From b8cf9e88f4c5c64d9406af533d8948deb050d695 Mon Sep 17 00:00:00 2001 From: shmel1k Date: Sun, 26 Nov 2023 18:16:14 +0300 Subject: add kikimr_configure --- contrib/python/Automat/py2/.dist-info/METADATA | 487 + .../python/Automat/py2/.dist-info/entry_points.txt | 3 + .../python/Automat/py2/.dist-info/top_level.txt | 1 + contrib/python/Automat/py2/LICENSE | 21 + contrib/python/Automat/py2/README.md | 430 + contrib/python/Automat/py2/automat/__init__.py | 8 + contrib/python/Automat/py2/automat/_core.py | 165 + contrib/python/Automat/py2/automat/_discover.py | 144 + .../python/Automat/py2/automat/_introspection.py | 45 + contrib/python/Automat/py2/automat/_methodical.py | 474 + contrib/python/Automat/py2/automat/_visualize.py | 182 + contrib/python/Automat/py2/ya.make | 38 + contrib/python/Automat/py3/.dist-info/METADATA | 27 + .../python/Automat/py3/.dist-info/entry_points.txt | 2 + .../python/Automat/py3/.dist-info/top_level.txt | 1 + contrib/python/Automat/py3/LICENSE | 21 + contrib/python/Automat/py3/README.md | 430 + contrib/python/Automat/py3/automat/__init__.py | 8 + contrib/python/Automat/py3/automat/_core.py | 165 + contrib/python/Automat/py3/automat/_discover.py | 144 + .../python/Automat/py3/automat/_introspection.py | 46 + contrib/python/Automat/py3/automat/_methodical.py | 472 + contrib/python/Automat/py3/automat/_visualize.py | 182 + contrib/python/Automat/py3/ya.make | 38 + contrib/python/Automat/ya.make | 18 + contrib/python/Twisted/py2/.dist-info/METADATA | 217 + .../python/Twisted/py2/.dist-info/entry_points.txt | 11 + .../python/Twisted/py2/.dist-info/top_level.txt | 1 + contrib/python/Twisted/py2/LICENSE | 73 + contrib/python/Twisted/py2/README.rst | 104 + contrib/python/Twisted/py2/twisted/__init__.py | 23 + .../Twisted/py2/twisted/_threads/__init__.py | 25 + .../Twisted/py2/twisted/_threads/_convenience.py | 46 + .../Twisted/py2/twisted/_threads/_ithreads.py | 61 + .../python/Twisted/py2/twisted/_threads/_memory.py | 71 + .../python/Twisted/py2/twisted/_threads/_pool.py | 69 + .../python/Twisted/py2/twisted/_threads/_team.py | 231 + .../Twisted/py2/twisted/_threads/_threadworker.py | 123 + contrib/python/Twisted/py2/twisted/_version.py | 11 + .../Twisted/py2/twisted/application/__init__.py | 6 + .../python/Twisted/py2/twisted/application/app.py | 708 ++ .../Twisted/py2/twisted/application/internet.py | 1157 ++ .../Twisted/py2/twisted/application/reactors.py | 85 + .../py2/twisted/application/runner/__init__.py | 7 + .../py2/twisted/application/runner/_exit.py | 138 + .../py2/twisted/application/runner/_pidfile.py | 303 + .../py2/twisted/application/runner/_runner.py | 185 + .../Twisted/py2/twisted/application/service.py | 424 + .../Twisted/py2/twisted/application/strports.py | 70 + .../py2/twisted/application/twist/__init__.py | 7 + .../py2/twisted/application/twist/_options.py | 205 + .../py2/twisted/application/twist/_twist.py | 128 + .../python/Twisted/py2/twisted/conch/__init__.py | 7 + contrib/python/Twisted/py2/twisted/conch/avatar.py | 45 + .../python/Twisted/py2/twisted/conch/checkers.py | 592 + .../Twisted/py2/twisted/conch/client/__init__.py | 9 + .../Twisted/py2/twisted/conch/client/agent.py | 73 + .../Twisted/py2/twisted/conch/client/connect.py | 21 + .../Twisted/py2/twisted/conch/client/default.py | 349 + .../Twisted/py2/twisted/conch/client/direct.py | 109 + .../Twisted/py2/twisted/conch/client/knownhosts.py | 630 + .../Twisted/py2/twisted/conch/client/options.py | 103 + .../python/Twisted/py2/twisted/conch/endpoints.py | 872 ++ contrib/python/Twisted/py2/twisted/conch/error.py | 103 + .../Twisted/py2/twisted/conch/insults/__init__.py | 4 + .../Twisted/py2/twisted/conch/insults/helper.py | 517 + .../Twisted/py2/twisted/conch/insults/insults.py | 1289 ++ .../Twisted/py2/twisted/conch/insults/text.py | 176 + .../Twisted/py2/twisted/conch/insults/window.py | 1027 ++ .../python/Twisted/py2/twisted/conch/interfaces.py | 444 + contrib/python/Twisted/py2/twisted/conch/ls.py | 83 + .../python/Twisted/py2/twisted/conch/manhole.py | 401 + .../Twisted/py2/twisted/conch/manhole_ssh.py | 141 + .../Twisted/py2/twisted/conch/manhole_tap.py | 165 + contrib/python/Twisted/py2/twisted/conch/mixin.py | 55 + .../py2/twisted/conch/openssh_compat/__init__.py | 11 + .../py2/twisted/conch/openssh_compat/factory.py | 72 + .../py2/twisted/conch/openssh_compat/primes.py | 30 + .../python/Twisted/py2/twisted/conch/recvline.py | 374 + .../Twisted/py2/twisted/conch/ssh/__init__.py | 10 + .../python/Twisted/py2/twisted/conch/ssh/_kex.py | 294 + .../Twisted/py2/twisted/conch/ssh/address.py | 47 + .../python/Twisted/py2/twisted/conch/ssh/agent.py | 296 + .../Twisted/py2/twisted/conch/ssh/channel.py | 320 + .../python/Twisted/py2/twisted/conch/ssh/common.py | 93 + .../Twisted/py2/twisted/conch/ssh/connection.py | 653 + .../Twisted/py2/twisted/conch/ssh/factory.py | 123 + .../Twisted/py2/twisted/conch/ssh/filetransfer.py | 1055 ++ .../Twisted/py2/twisted/conch/ssh/forwarding.py | 269 + .../python/Twisted/py2/twisted/conch/ssh/keys.py | 1678 +++ .../Twisted/py2/twisted/conch/ssh/service.py | 48 + .../Twisted/py2/twisted/conch/ssh/session.py | 362 + .../python/Twisted/py2/twisted/conch/ssh/sexpy.py | 45 + .../Twisted/py2/twisted/conch/ssh/transport.py | 2127 ++++ .../Twisted/py2/twisted/conch/ssh/userauth.py | 770 ++ contrib/python/Twisted/py2/twisted/conch/stdio.py | 120 + contrib/python/Twisted/py2/twisted/conch/tap.py | 86 + contrib/python/Twisted/py2/twisted/conch/telnet.py | 1194 ++ .../python/Twisted/py2/twisted/conch/ttymodes.py | 121 + .../Twisted/py2/twisted/conch/ui/__init__.py | 11 + .../python/Twisted/py2/twisted/conch/ui/ansi.py | 240 + .../python/Twisted/py2/twisted/conch/ui/tkvt100.py | 202 + contrib/python/Twisted/py2/twisted/conch/unix.py | 535 + contrib/python/Twisted/py2/twisted/copyright.py | 43 + .../python/Twisted/py2/twisted/cred/__init__.py | 7 + contrib/python/Twisted/py2/twisted/cred/_digest.py | 132 + .../python/Twisted/py2/twisted/cred/checkers.py | 329 + .../python/Twisted/py2/twisted/cred/credentials.py | 510 + contrib/python/Twisted/py2/twisted/cred/error.py | 45 + contrib/python/Twisted/py2/twisted/cred/portal.py | 124 + contrib/python/Twisted/py2/twisted/cred/strcred.py | 272 + .../Twisted/py2/twisted/enterprise/__init__.py | 8 + .../Twisted/py2/twisted/enterprise/adbapi.py | 502 + .../Twisted/py2/twisted/internet/__init__.py | 12 + .../Twisted/py2/twisted/internet/_baseprocess.py | 66 + .../Twisted/py2/twisted/internet/_dumbwin32proc.py | 433 + .../Twisted/py2/twisted/internet/_glibbase.py | 390 + .../python/Twisted/py2/twisted/internet/_idna.py | 54 + .../python/Twisted/py2/twisted/internet/_newtls.py | 271 + .../Twisted/py2/twisted/internet/_pollingfile.py | 300 + .../py2/twisted/internet/_posixserialport.py | 73 + .../Twisted/py2/twisted/internet/_posixstdio.py | 168 + .../py2/twisted/internet/_producer_helpers.py | 125 + .../Twisted/py2/twisted/internet/_resolver.py | 279 + .../Twisted/py2/twisted/internet/_signals.py | 68 + .../Twisted/py2/twisted/internet/_sslverify.py | 2058 ++++ .../py2/twisted/internet/_threadedselect.py | 354 + .../py2/twisted/internet/_win32serialport.py | 133 + .../Twisted/py2/twisted/internet/_win32stdio.py | 133 + .../Twisted/py2/twisted/internet/abstract.py | 546 + .../python/Twisted/py2/twisted/internet/address.py | 180 + .../Twisted/py2/twisted/internet/asyncioreactor.py | 322 + .../python/Twisted/py2/twisted/internet/base.py | 1304 ++ .../Twisted/py2/twisted/internet/cfreactor.py | 502 + .../python/Twisted/py2/twisted/internet/default.py | 56 + .../python/Twisted/py2/twisted/internet/defer.py | 2019 ++++ .../Twisted/py2/twisted/internet/endpoints.py | 2269 ++++ .../Twisted/py2/twisted/internet/epollreactor.py | 249 + .../python/Twisted/py2/twisted/internet/error.py | 517 + .../python/Twisted/py2/twisted/internet/fdesc.py | 118 + .../Twisted/py2/twisted/internet/gireactor.py | 188 + .../Twisted/py2/twisted/internet/glib2reactor.py | 44 + .../Twisted/py2/twisted/internet/gtk2reactor.py | 121 + .../Twisted/py2/twisted/internet/gtk3reactor.py | 80 + .../python/Twisted/py2/twisted/internet/inotify.py | 419 + .../Twisted/py2/twisted/internet/interfaces.py | 2915 +++++ .../py2/twisted/internet/iocpreactor/__init__.py | 10 + .../py2/twisted/internet/iocpreactor/abstract.py | 399 + .../py2/twisted/internet/iocpreactor/const.py | 26 + .../py2/twisted/internet/iocpreactor/interfaces.py | 47 + .../internet/iocpreactor/iocpsupport/iocpsupport.c | 11990 +++++++++++++++++++ .../iocpreactor/iocpsupport/winsock_pointers.c | 62 + .../iocpreactor/iocpsupport/winsock_pointers.h | 51 + .../py2/twisted/internet/iocpreactor/notes.txt | 24 + .../py2/twisted/internet/iocpreactor/reactor.py | 273 + .../py2/twisted/internet/iocpreactor/setup.py | 23 + .../py2/twisted/internet/iocpreactor/tcp.py | 617 + .../py2/twisted/internet/iocpreactor/udp.py | 428 + .../Twisted/py2/twisted/internet/kqreactor.py | 320 + .../python/Twisted/py2/twisted/internet/main.py | 37 + .../Twisted/py2/twisted/internet/pollreactor.py | 189 + .../Twisted/py2/twisted/internet/posixbase.py | 798 ++ .../python/Twisted/py2/twisted/internet/process.py | 1114 ++ .../Twisted/py2/twisted/internet/protocol.py | 933 ++ .../Twisted/py2/twisted/internet/pyuisupport.py | 37 + .../python/Twisted/py2/twisted/internet/reactor.py | 39 + .../Twisted/py2/twisted/internet/selectreactor.py | 200 + .../Twisted/py2/twisted/internet/serialport.py | 89 + contrib/python/Twisted/py2/twisted/internet/ssl.py | 255 + .../python/Twisted/py2/twisted/internet/stdio.py | 37 + .../python/Twisted/py2/twisted/internet/task.py | 948 ++ contrib/python/Twisted/py2/twisted/internet/tcp.py | 1555 +++ .../python/Twisted/py2/twisted/internet/testing.py | 1010 ++ .../python/Twisted/py2/twisted/internet/threads.py | 127 + .../Twisted/py2/twisted/internet/tksupport.py | 78 + contrib/python/Twisted/py2/twisted/internet/udp.py | 541 + .../python/Twisted/py2/twisted/internet/unix.py | 624 + .../python/Twisted/py2/twisted/internet/utils.py | 245 + .../py2/twisted/internet/win32eventreactor.py | 429 + .../Twisted/py2/twisted/internet/wxreactor.py | 188 + .../Twisted/py2/twisted/internet/wxsupport.py | 59 + .../python/Twisted/py2/twisted/logger/__init__.py | 130 + .../python/Twisted/py2/twisted/logger/_buffer.py | 59 + .../python/Twisted/py2/twisted/logger/_capture.py | 24 + contrib/python/Twisted/py2/twisted/logger/_file.py | 86 + .../python/Twisted/py2/twisted/logger/_filter.py | 231 + .../python/Twisted/py2/twisted/logger/_flatten.py | 178 + .../python/Twisted/py2/twisted/logger/_format.py | 421 + .../python/Twisted/py2/twisted/logger/_global.py | 240 + contrib/python/Twisted/py2/twisted/logger/_io.py | 202 + contrib/python/Twisted/py2/twisted/logger/_json.py | 355 + .../python/Twisted/py2/twisted/logger/_legacy.py | 154 + .../python/Twisted/py2/twisted/logger/_levels.py | 110 + .../python/Twisted/py2/twisted/logger/_logger.py | 275 + .../python/Twisted/py2/twisted/logger/_observer.py | 158 + .../python/Twisted/py2/twisted/logger/_stdlib.py | 147 + contrib/python/Twisted/py2/twisted/logger/_util.py | 48 + .../python/Twisted/py2/twisted/mail/__init__.py | 6 + contrib/python/Twisted/py2/twisted/mail/_cred.py | 122 + contrib/python/Twisted/py2/twisted/mail/_except.py | 392 + contrib/python/Twisted/py2/twisted/mail/alias.py | 799 ++ contrib/python/Twisted/py2/twisted/mail/bounce.py | 107 + contrib/python/Twisted/py2/twisted/mail/imap4.py | 6404 ++++++++++ .../python/Twisted/py2/twisted/mail/interfaces.py | 1110 ++ contrib/python/Twisted/py2/twisted/mail/mail.py | 749 ++ contrib/python/Twisted/py2/twisted/mail/maildir.py | 944 ++ contrib/python/Twisted/py2/twisted/mail/pb.py | 124 + contrib/python/Twisted/py2/twisted/mail/pop3.py | 1749 +++ .../python/Twisted/py2/twisted/mail/pop3client.py | 1264 ++ .../python/Twisted/py2/twisted/mail/protocols.py | 404 + contrib/python/Twisted/py2/twisted/mail/relay.py | 180 + .../Twisted/py2/twisted/mail/relaymanager.py | 1161 ++ contrib/python/Twisted/py2/twisted/mail/smtp.py | 2247 ++++ contrib/python/Twisted/py2/twisted/mail/tap.py | 394 + .../python/Twisted/py2/twisted/names/__init__.py | 6 + .../python/Twisted/py2/twisted/names/_rfc1982.py | 278 + .../python/Twisted/py2/twisted/names/authority.py | 543 + contrib/python/Twisted/py2/twisted/names/cache.py | 125 + contrib/python/Twisted/py2/twisted/names/client.py | 776 ++ contrib/python/Twisted/py2/twisted/names/common.py | 289 + contrib/python/Twisted/py2/twisted/names/dns.py | 3214 +++++ contrib/python/Twisted/py2/twisted/names/error.py | 97 + contrib/python/Twisted/py2/twisted/names/hosts.py | 153 + .../python/Twisted/py2/twisted/names/resolve.py | 99 + contrib/python/Twisted/py2/twisted/names/root.py | 333 + .../python/Twisted/py2/twisted/names/secondary.py | 221 + contrib/python/Twisted/py2/twisted/names/server.py | 590 + .../python/Twisted/py2/twisted/names/srvconnect.py | 273 + contrib/python/Twisted/py2/twisted/names/tap.py | 150 + .../python/Twisted/py2/twisted/news/__init__.py | 6 + .../python/Twisted/py2/twisted/news/database.py | 1046 ++ contrib/python/Twisted/py2/twisted/news/news.py | 92 + contrib/python/Twisted/py2/twisted/news/nntp.py | 1050 ++ contrib/python/Twisted/py2/twisted/news/tap.py | 143 + .../python/Twisted/py2/twisted/pair/__init__.py | 13 + .../python/Twisted/py2/twisted/pair/ethernet.py | 56 + contrib/python/Twisted/py2/twisted/pair/ip.py | 71 + contrib/python/Twisted/py2/twisted/pair/raw.py | 40 + contrib/python/Twisted/py2/twisted/pair/rawudp.py | 59 + contrib/python/Twisted/py2/twisted/pair/testing.py | 572 + contrib/python/Twisted/py2/twisted/pair/tuntap.py | 433 + .../Twisted/py2/twisted/persisted/__init__.py | 6 + .../python/Twisted/py2/twisted/persisted/aot.py | 625 + .../Twisted/py2/twisted/persisted/crefutil.py | 159 + .../python/Twisted/py2/twisted/persisted/dirdbm.py | 389 + .../python/Twisted/py2/twisted/persisted/sob.py | 194 + .../python/Twisted/py2/twisted/persisted/styles.py | 423 + contrib/python/Twisted/py2/twisted/plugin.py | 259 + .../python/Twisted/py2/twisted/plugins/__init__.py | 19 + .../Twisted/py2/twisted/plugins/cred_anonymous.py | 41 + .../Twisted/py2/twisted/plugins/cred_file.py | 61 + .../Twisted/py2/twisted/plugins/cred_memory.py | 70 + .../Twisted/py2/twisted/plugins/cred_sshkeys.py | 53 + .../Twisted/py2/twisted/plugins/cred_unix.py | 185 + .../Twisted/py2/twisted/plugins/twisted_conch.py | 18 + .../Twisted/py2/twisted/plugins/twisted_core.py | 18 + .../Twisted/py2/twisted/plugins/twisted_ftp.py | 10 + .../Twisted/py2/twisted/plugins/twisted_inet.py | 10 + .../Twisted/py2/twisted/plugins/twisted_mail.py | 10 + .../Twisted/py2/twisted/plugins/twisted_names.py | 10 + .../Twisted/py2/twisted/plugins/twisted_news.py | 10 + .../py2/twisted/plugins/twisted_portforward.py | 10 + .../py2/twisted/plugins/twisted_reactors.py | 71 + .../Twisted/py2/twisted/plugins/twisted_runner.py | 10 + .../Twisted/py2/twisted/plugins/twisted_socks.py | 10 + .../Twisted/py2/twisted/plugins/twisted_trial.py | 62 + .../Twisted/py2/twisted/plugins/twisted_web.py | 11 + .../Twisted/py2/twisted/plugins/twisted_words.py | 47 + .../Twisted/py2/twisted/positioning/__init__.py | 8 + .../Twisted/py2/twisted/positioning/_sentence.py | 122 + .../python/Twisted/py2/twisted/positioning/base.py | 947 ++ .../py2/twisted/positioning/ipositioning.py | 122 + .../python/Twisted/py2/twisted/positioning/nmea.py | 984 ++ .../Twisted/py2/twisted/protocols/__init__.py | 15 + .../python/Twisted/py2/twisted/protocols/amp.py | 2897 +++++ .../python/Twisted/py2/twisted/protocols/basic.py | 953 ++ .../python/Twisted/py2/twisted/protocols/dict.py | 415 + .../python/Twisted/py2/twisted/protocols/finger.py | 42 + .../python/Twisted/py2/twisted/protocols/ftp.py | 3374 ++++++ .../py2/twisted/protocols/haproxy/__init__.py | 13 + .../py2/twisted/protocols/haproxy/_exceptions.py | 52 + .../Twisted/py2/twisted/protocols/haproxy/_info.py | 36 + .../py2/twisted/protocols/haproxy/_interfaces.py | 64 + .../py2/twisted/protocols/haproxy/_parser.py | 71 + .../py2/twisted/protocols/haproxy/_v1parser.py | 143 + .../py2/twisted/protocols/haproxy/_v2parser.py | 215 + .../py2/twisted/protocols/haproxy/_wrapper.py | 106 + .../python/Twisted/py2/twisted/protocols/htb.py | 295 + .../python/Twisted/py2/twisted/protocols/ident.py | 255 + .../Twisted/py2/twisted/protocols/loopback.py | 385 + .../Twisted/py2/twisted/protocols/memcache.py | 766 ++ .../python/Twisted/py2/twisted/protocols/pcp.py | 203 + .../Twisted/py2/twisted/protocols/policies.py | 751 ++ .../Twisted/py2/twisted/protocols/portforward.py | 99 + .../Twisted/py2/twisted/protocols/postfix.py | 158 + .../Twisted/py2/twisted/protocols/shoutcast.py | 111 + .../python/Twisted/py2/twisted/protocols/sip.py | 1294 ++ .../python/Twisted/py2/twisted/protocols/socks.py | 255 + .../Twisted/py2/twisted/protocols/stateful.py | 49 + .../python/Twisted/py2/twisted/protocols/tls.py | 830 ++ .../python/Twisted/py2/twisted/protocols/wire.py | 124 + .../python/Twisted/py2/twisted/python/__init__.py | 29 + .../python/Twisted/py2/twisted/python/_appdirs.py | 32 + .../python/Twisted/py2/twisted/python/_inotify.py | 110 + .../python/Twisted/py2/twisted/python/_oldstyle.py | 99 + .../python/Twisted/py2/twisted/python/_pydoctor.py | 269 + .../python/Twisted/py2/twisted/python/_release.py | 576 + .../python/Twisted/py2/twisted/python/_sendmsg.c | 519 + .../python/Twisted/py2/twisted/python/_setup.py | 452 + .../Twisted/py2/twisted/python/_shellcomp.py | 677 ++ .../Twisted/py2/twisted/python/_textattributes.py | 320 + .../python/Twisted/py2/twisted/python/_tzhelper.py | 119 + contrib/python/Twisted/py2/twisted/python/_url.py | 13 + .../python/Twisted/py2/twisted/python/compat.py | 928 ++ .../Twisted/py2/twisted/python/components.py | 430 + .../python/Twisted/py2/twisted/python/constants.py | 18 + .../python/Twisted/py2/twisted/python/context.py | 137 + .../python/Twisted/py2/twisted/python/deprecate.py | 797 ++ .../python/Twisted/py2/twisted/python/failure.py | 798 ++ .../python/Twisted/py2/twisted/python/fakepwd.py | 220 + .../python/Twisted/py2/twisted/python/filepath.py | 1766 +++ .../python/Twisted/py2/twisted/python/finalize.py | 48 + .../Twisted/py2/twisted/python/formmethod.py | 377 + contrib/python/Twisted/py2/twisted/python/hook.py | 176 + .../python/Twisted/py2/twisted/python/htmlizer.py | 131 + .../python/Twisted/py2/twisted/python/lockfile.py | 248 + contrib/python/Twisted/py2/twisted/python/log.py | 767 ++ .../python/Twisted/py2/twisted/python/logfile.py | 340 + .../python/Twisted/py2/twisted/python/modules.py | 789 ++ .../python/Twisted/py2/twisted/python/monkey.py | 75 + .../python/Twisted/py2/twisted/python/procutils.py | 51 + .../python/Twisted/py2/twisted/python/randbytes.py | 150 + .../python/Twisted/py2/twisted/python/rebuild.py | 310 + .../python/Twisted/py2/twisted/python/reflect.py | 634 + .../python/Twisted/py2/twisted/python/release.py | 67 + contrib/python/Twisted/py2/twisted/python/roots.py | 257 + .../python/Twisted/py2/twisted/python/runtime.py | 231 + .../python/Twisted/py2/twisted/python/sendmsg.py | 106 + .../python/Twisted/py2/twisted/python/shortcut.py | 85 + .../python/Twisted/py2/twisted/python/syslog.py | 109 + .../python/Twisted/py2/twisted/python/systemd.py | 89 + contrib/python/Twisted/py2/twisted/python/text.py | 208 + .../Twisted/py2/twisted/python/threadable.py | 141 + .../Twisted/py2/twisted/python/threadpool.py | 320 + contrib/python/Twisted/py2/twisted/python/url.py | 15 + .../python/Twisted/py2/twisted/python/urlpath.py | 294 + contrib/python/Twisted/py2/twisted/python/usage.py | 1001 ++ contrib/python/Twisted/py2/twisted/python/util.py | 1027 ++ .../python/Twisted/py2/twisted/python/versions.py | 14 + contrib/python/Twisted/py2/twisted/python/win32.py | 136 + contrib/python/Twisted/py2/twisted/python/ya.make | 31 + .../python/Twisted/py2/twisted/python/zippath.py | 295 + .../python/Twisted/py2/twisted/python/zipstream.py | 336 + .../python/Twisted/py2/twisted/runner/__init__.py | 6 + contrib/python/Twisted/py2/twisted/runner/inetd.py | 70 + .../python/Twisted/py2/twisted/runner/inetdconf.py | 198 + .../python/Twisted/py2/twisted/runner/inetdtap.py | 109 + .../python/Twisted/py2/twisted/runner/procmon.py | 426 + .../Twisted/py2/twisted/runner/procmontap.py | 73 + .../python/Twisted/py2/twisted/scripts/__init__.py | 9 + .../Twisted/py2/twisted/scripts/_twistd_unix.py | 453 + .../python/Twisted/py2/twisted/scripts/_twistw.py | 54 + .../python/Twisted/py2/twisted/scripts/htmlizer.py | 74 + .../python/Twisted/py2/twisted/scripts/trial.py | 627 + .../python/Twisted/py2/twisted/scripts/twistd.py | 34 + .../python/Twisted/py2/twisted/spread/__init__.py | 8 + .../python/Twisted/py2/twisted/spread/banana.py | 398 + .../python/Twisted/py2/twisted/spread/flavors.py | 642 + .../Twisted/py2/twisted/spread/interfaces.py | 31 + contrib/python/Twisted/py2/twisted/spread/jelly.py | 1131 ++ contrib/python/Twisted/py2/twisted/spread/pb.py | 1677 +++ .../python/Twisted/py2/twisted/spread/publish.py | 142 + contrib/python/Twisted/py2/twisted/spread/util.py | 215 + contrib/python/Twisted/py2/twisted/tap/__init__.py | 6 + contrib/python/Twisted/py2/twisted/tap/ftp.py | 69 + .../python/Twisted/py2/twisted/tap/portforward.py | 27 + contrib/python/Twisted/py2/twisted/tap/socks.py | 39 + .../python/Twisted/py2/twisted/trial/__init__.py | 50 + .../python/Twisted/py2/twisted/trial/__main__.py | 10 + .../Twisted/py2/twisted/trial/_asyncrunner.py | 185 + .../python/Twisted/py2/twisted/trial/_asynctest.py | 405 + .../Twisted/py2/twisted/trial/_dist/__init__.py | 47 + .../py2/twisted/trial/_dist/distreporter.py | 93 + .../Twisted/py2/twisted/trial/_dist/disttrial.py | 258 + .../py2/twisted/trial/_dist/managercommands.py | 86 + .../Twisted/py2/twisted/trial/_dist/options.py | 30 + .../Twisted/py2/twisted/trial/_dist/worker.py | 333 + .../py2/twisted/trial/_dist/workercommands.py | 31 + .../py2/twisted/trial/_dist/workerreporter.py | 154 + .../Twisted/py2/twisted/trial/_dist/workertrial.py | 111 + .../python/Twisted/py2/twisted/trial/_synctest.py | 1416 +++ contrib/python/Twisted/py2/twisted/trial/itrial.py | 259 + .../python/Twisted/py2/twisted/trial/reporter.py | 1322 ++ contrib/python/Twisted/py2/twisted/trial/runner.py | 1064 ++ .../python/Twisted/py2/twisted/trial/unittest.py | 35 + contrib/python/Twisted/py2/twisted/trial/util.py | 411 + contrib/python/Twisted/py2/twisted/web/__init__.py | 12 + .../Twisted/py2/twisted/web/_auth/__init__.py | 7 + .../python/Twisted/py2/twisted/web/_auth/basic.py | 61 + .../python/Twisted/py2/twisted/web/_auth/digest.py | 56 + .../Twisted/py2/twisted/web/_auth/wrapper.py | 236 + contrib/python/Twisted/py2/twisted/web/_element.py | 185 + contrib/python/Twisted/py2/twisted/web/_flatten.py | 421 + contrib/python/Twisted/py2/twisted/web/_http2.py | 1356 +++ .../python/Twisted/py2/twisted/web/_newclient.py | 1778 +++ .../python/Twisted/py2/twisted/web/_responses.py | 114 + contrib/python/Twisted/py2/twisted/web/_stan.py | 330 + contrib/python/Twisted/py2/twisted/web/client.py | 2336 ++++ contrib/python/Twisted/py2/twisted/web/demo.py | 26 + contrib/python/Twisted/py2/twisted/web/distrib.py | 386 + .../python/Twisted/py2/twisted/web/domhelpers.py | 272 + contrib/python/Twisted/py2/twisted/web/error.py | 407 + contrib/python/Twisted/py2/twisted/web/guard.py | 20 + contrib/python/Twisted/py2/twisted/web/html.py | 57 + contrib/python/Twisted/py2/twisted/web/http.py | 3170 +++++ .../python/Twisted/py2/twisted/web/http_headers.py | 294 + contrib/python/Twisted/py2/twisted/web/iweb.py | 828 ++ contrib/python/Twisted/py2/twisted/web/microdom.py | 1145 ++ contrib/python/Twisted/py2/twisted/web/proxy.py | 303 + contrib/python/Twisted/py2/twisted/web/resource.py | 422 + contrib/python/Twisted/py2/twisted/web/rewrite.py | 52 + contrib/python/Twisted/py2/twisted/web/script.py | 182 + contrib/python/Twisted/py2/twisted/web/server.py | 911 ++ contrib/python/Twisted/py2/twisted/web/soap.py | 154 + contrib/python/Twisted/py2/twisted/web/static.py | 1103 ++ contrib/python/Twisted/py2/twisted/web/sux.py | 637 + contrib/python/Twisted/py2/twisted/web/tap.py | 316 + contrib/python/Twisted/py2/twisted/web/template.py | 575 + .../Twisted/py2/twisted/web/test/requesthelper.py | 486 + contrib/python/Twisted/py2/twisted/web/twcgi.py | 321 + contrib/python/Twisted/py2/twisted/web/util.py | 443 + contrib/python/Twisted/py2/twisted/web/vhost.py | 138 + contrib/python/Twisted/py2/twisted/web/wsgi.py | 596 + contrib/python/Twisted/py2/twisted/web/xmlrpc.py | 591 + .../python/Twisted/py2/twisted/words/__init__.py | 8 + contrib/python/Twisted/py2/twisted/words/ewords.py | 34 + .../Twisted/py2/twisted/words/im/__init__.py | 8 + .../Twisted/py2/twisted/words/im/baseaccount.py | 62 + .../Twisted/py2/twisted/words/im/basechat.py | 512 + .../Twisted/py2/twisted/words/im/basesupport.py | 269 + .../Twisted/py2/twisted/words/im/interfaces.py | 398 + .../Twisted/py2/twisted/words/im/ircsupport.py | 293 + .../python/Twisted/py2/twisted/words/im/locals.py | 26 + .../Twisted/py2/twisted/words/im/pbsupport.py | 262 + contrib/python/Twisted/py2/twisted/words/iwords.py | 267 + .../py2/twisted/words/protocols/__init__.py | 6 + .../Twisted/py2/twisted/words/protocols/irc.py | 4074 +++++++ .../py2/twisted/words/protocols/jabber/__init__.py | 8 + .../py2/twisted/words/protocols/jabber/client.py | 408 + .../twisted/words/protocols/jabber/component.py | 475 + .../py2/twisted/words/protocols/jabber/error.py | 331 + .../py2/twisted/words/protocols/jabber/ijabber.py | 201 + .../py2/twisted/words/protocols/jabber/jid.py | 253 + .../twisted/words/protocols/jabber/jstrports.py | 33 + .../py2/twisted/words/protocols/jabber/sasl.py | 233 + .../words/protocols/jabber/sasl_mechanisms.py | 293 + .../twisted/words/protocols/jabber/xmlstream.py | 1170 ++ .../words/protocols/jabber/xmpp_stringprep.py | 244 + .../python/Twisted/py2/twisted/words/service.py | 1269 ++ contrib/python/Twisted/py2/twisted/words/tap.py | 74 + .../Twisted/py2/twisted/words/xish/__init__.py | 10 + .../Twisted/py2/twisted/words/xish/domish.py | 899 ++ .../Twisted/py2/twisted/words/xish/utility.py | 375 + .../Twisted/py2/twisted/words/xish/xmlstream.py | 279 + .../python/Twisted/py2/twisted/words/xish/xpath.py | 337 + .../Twisted/py2/twisted/words/xish/xpathparser.py | 650 + .../Twisted/py2/twisted/words/xmpproutertap.py | 30 + contrib/python/Twisted/py2/ya.make | 472 + contrib/python/Twisted/py3/.dist-info/METADATA | 224 + .../python/Twisted/py3/.dist-info/entry_points.txt | 10 + .../python/Twisted/py3/.dist-info/top_level.txt | 1 + contrib/python/Twisted/py3/LICENSE | 74 + contrib/python/Twisted/py3/README.rst | 123 + contrib/python/Twisted/py3/twisted/11715.misc | 0 contrib/python/Twisted/py3/twisted/__init__.py | 12 + contrib/python/Twisted/py3/twisted/__main__.py | 14 + .../Twisted/py3/twisted/_threads/__init__.py | 24 + .../Twisted/py3/twisted/_threads/_convenience.py | 43 + .../Twisted/py3/twisted/_threads/_ithreads.py | 61 + .../python/Twisted/py3/twisted/_threads/_memory.py | 70 + .../python/Twisted/py3/twisted/_threads/_pool.py | 73 + .../python/Twisted/py3/twisted/_threads/_team.py | 232 + .../Twisted/py3/twisted/_threads/_threadworker.py | 121 + contrib/python/Twisted/py3/twisted/_version.py | 11 + .../Twisted/py3/twisted/application/__init__.py | 6 + .../python/Twisted/py3/twisted/application/app.py | 706 ++ .../Twisted/py3/twisted/application/internet.py | 1205 ++ .../twisted/application/newsfragments/10146.misc | 0 .../twisted/application/newsfragments/9746.misc | 1 + .../Twisted/py3/twisted/application/reactors.py | 87 + .../py3/twisted/application/runner/__init__.py | 7 + .../py3/twisted/application/runner/_exit.py | 99 + .../py3/twisted/application/runner/_pidfile.py | 282 + .../py3/twisted/application/runner/_runner.py | 166 + .../Twisted/py3/twisted/application/service.py | 420 + .../Twisted/py3/twisted/application/strports.py | 83 + .../py3/twisted/application/twist/__init__.py | 7 + .../py3/twisted/application/twist/_options.py | 207 + .../py3/twisted/application/twist/_twist.py | 114 + .../python/Twisted/py3/twisted/conch/__init__.py | 7 + contrib/python/Twisted/py3/twisted/conch/avatar.py | 56 + .../python/Twisted/py3/twisted/conch/checkers.py | 640 + .../Twisted/py3/twisted/conch/client/__init__.py | 9 + .../Twisted/py3/twisted/conch/client/agent.py | 65 + .../Twisted/py3/twisted/conch/client/connect.py | 24 + .../Twisted/py3/twisted/conch/client/default.py | 331 + .../Twisted/py3/twisted/conch/client/direct.py | 98 + .../Twisted/py3/twisted/conch/client/knownhosts.py | 620 + .../Twisted/py3/twisted/conch/client/options.py | 109 + .../python/Twisted/py3/twisted/conch/endpoints.py | 875 ++ contrib/python/Twisted/py3/twisted/conch/error.py | 96 + .../Twisted/py3/twisted/conch/insults/__init__.py | 4 + .../Twisted/py3/twisted/conch/insults/helper.py | 556 + .../Twisted/py3/twisted/conch/insults/insults.py | 1223 ++ .../Twisted/py3/twisted/conch/insults/text.py | 176 + .../Twisted/py3/twisted/conch/insults/window.py | 928 ++ .../python/Twisted/py3/twisted/conch/interfaces.py | 454 + contrib/python/Twisted/py3/twisted/conch/ls.py | 104 + .../python/Twisted/py3/twisted/conch/manhole.py | 392 + .../Twisted/py3/twisted/conch/manhole_ssh.py | 148 + .../Twisted/py3/twisted/conch/manhole_tap.py | 180 + contrib/python/Twisted/py3/twisted/conch/mixin.py | 54 + .../py3/twisted/conch/newsfragments/.gitignore | 1 + .../py3/twisted/conch/openssh_compat/__init__.py | 10 + .../py3/twisted/conch/openssh_compat/factory.py | 74 + .../py3/twisted/conch/openssh_compat/primes.py | 31 + .../python/Twisted/py3/twisted/conch/recvline.py | 569 + .../Twisted/py3/twisted/conch/scripts/__init__.py | 1 + .../Twisted/py3/twisted/conch/scripts/cftp.py | 1002 ++ .../Twisted/py3/twisted/conch/scripts/ckeygen.py | 400 + .../Twisted/py3/twisted/conch/scripts/conch.py | 578 + .../Twisted/py3/twisted/conch/scripts/tkconch.py | 673 ++ .../Twisted/py3/twisted/conch/ssh/__init__.py | 10 + .../python/Twisted/py3/twisted/conch/ssh/_kex.py | 293 + .../Twisted/py3/twisted/conch/ssh/address.py | 43 + .../python/Twisted/py3/twisted/conch/ssh/agent.py | 278 + .../Twisted/py3/twisted/conch/ssh/channel.py | 312 + .../python/Twisted/py3/twisted/conch/ssh/common.py | 85 + .../Twisted/py3/twisted/conch/ssh/connection.py | 679 ++ .../Twisted/py3/twisted/conch/ssh/factory.py | 129 + .../Twisted/py3/twisted/conch/ssh/filetransfer.py | 1069 ++ .../Twisted/py3/twisted/conch/ssh/forwarding.py | 272 + .../python/Twisted/py3/twisted/conch/ssh/keys.py | 1818 +++ .../Twisted/py3/twisted/conch/ssh/service.py | 56 + .../Twisted/py3/twisted/conch/ssh/session.py | 440 + .../python/Twisted/py3/twisted/conch/ssh/sexpy.py | 40 + .../Twisted/py3/twisted/conch/ssh/transport.py | 2266 ++++ .../Twisted/py3/twisted/conch/ssh/userauth.py | 764 ++ contrib/python/Twisted/py3/twisted/conch/stdio.py | 114 + contrib/python/Twisted/py3/twisted/conch/tap.py | 91 + contrib/python/Twisted/py3/twisted/conch/telnet.py | 1144 ++ .../python/Twisted/py3/twisted/conch/ttymodes.py | 122 + .../Twisted/py3/twisted/conch/ui/__init__.py | 11 + .../python/Twisted/py3/twisted/conch/ui/ansi.py | 253 + .../python/Twisted/py3/twisted/conch/ui/tkvt100.py | 249 + contrib/python/Twisted/py3/twisted/conch/unix.py | 524 + contrib/python/Twisted/py3/twisted/copyright.py | 44 + .../python/Twisted/py3/twisted/cred/__init__.py | 7 + contrib/python/Twisted/py3/twisted/cred/_digest.py | 132 + .../python/Twisted/py3/twisted/cred/checkers.py | 334 + .../python/Twisted/py3/twisted/cred/credentials.py | 508 + contrib/python/Twisted/py3/twisted/cred/error.py | 38 + contrib/python/Twisted/py3/twisted/cred/portal.py | 154 + contrib/python/Twisted/py3/twisted/cred/strcred.py | 250 + .../Twisted/py3/twisted/enterprise/__init__.py | 8 + .../Twisted/py3/twisted/enterprise/adbapi.py | 478 + .../Twisted/py3/twisted/internet/__init__.py | 12 + .../Twisted/py3/twisted/internet/_baseprocess.py | 68 + .../Twisted/py3/twisted/internet/_deprecate.py | 25 + .../Twisted/py3/twisted/internet/_dumbwin32proc.py | 397 + .../Twisted/py3/twisted/internet/_glibbase.py | 369 + .../python/Twisted/py3/twisted/internet/_idna.py | 51 + .../python/Twisted/py3/twisted/internet/_newtls.py | 256 + .../Twisted/py3/twisted/internet/_pollingfile.py | 291 + .../py3/twisted/internet/_posixserialport.py | 81 + .../Twisted/py3/twisted/internet/_posixstdio.py | 178 + .../py3/twisted/internet/_producer_helpers.py | 124 + .../Twisted/py3/twisted/internet/_resolver.py | 342 + .../Twisted/py3/twisted/internet/_signals.py | 445 + .../Twisted/py3/twisted/internet/_sslverify.py | 2017 ++++ .../py3/twisted/internet/_threadedselect.py | 337 + .../py3/twisted/internet/_win32serialport.py | 156 + .../Twisted/py3/twisted/internet/_win32stdio.py | 127 + .../Twisted/py3/twisted/internet/abstract.py | 542 + .../python/Twisted/py3/twisted/internet/address.py | 182 + .../Twisted/py3/twisted/internet/asyncioreactor.py | 307 + .../python/Twisted/py3/twisted/internet/base.py | 1345 +++ .../Twisted/py3/twisted/internet/cfreactor.py | 593 + .../python/Twisted/py3/twisted/internet/default.py | 55 + .../python/Twisted/py3/twisted/internet/defer.py | 2697 +++++ .../Twisted/py3/twisted/internet/endpoints.py | 2338 ++++ .../Twisted/py3/twisted/internet/epollreactor.py | 259 + .../python/Twisted/py3/twisted/internet/error.py | 510 + .../python/Twisted/py3/twisted/internet/fdesc.py | 121 + .../Twisted/py3/twisted/internet/gireactor.py | 122 + .../Twisted/py3/twisted/internet/glib2reactor.py | 50 + .../Twisted/py3/twisted/internet/gtk2reactor.py | 119 + .../Twisted/py3/twisted/internet/gtk3reactor.py | 22 + .../python/Twisted/py3/twisted/internet/inotify.py | 426 + .../Twisted/py3/twisted/internet/interfaces.py | 2756 +++++ .../py3/twisted/internet/iocpreactor/__init__.py | 10 + .../py3/twisted/internet/iocpreactor/abstract.py | 387 + .../py3/twisted/internet/iocpreactor/const.py | 25 + .../py3/twisted/internet/iocpreactor/interfaces.py | 42 + .../twisted/internet/iocpreactor/iocpsupport.py | 27 + .../py3/twisted/internet/iocpreactor/notes.txt | 24 + .../py3/twisted/internet/iocpreactor/reactor.py | 285 + .../py3/twisted/internet/iocpreactor/tcp.py | 608 + .../py3/twisted/internet/iocpreactor/udp.py | 428 + .../Twisted/py3/twisted/internet/kqreactor.py | 324 + .../python/Twisted/py3/twisted/internet/main.py | 37 + .../Twisted/py3/twisted/internet/pollreactor.py | 189 + .../Twisted/py3/twisted/internet/posixbase.py | 653 + .../python/Twisted/py3/twisted/internet/process.py | 1293 ++ .../Twisted/py3/twisted/internet/protocol.py | 900 ++ .../Twisted/py3/twisted/internet/pyuisupport.py | 39 + .../python/Twisted/py3/twisted/internet/reactor.py | 40 + .../Twisted/py3/twisted/internet/selectreactor.py | 197 + .../Twisted/py3/twisted/internet/serialport.py | 100 + contrib/python/Twisted/py3/twisted/internet/ssl.py | 278 + .../python/Twisted/py3/twisted/internet/stdio.py | 37 + .../python/Twisted/py3/twisted/internet/task.py | 976 ++ contrib/python/Twisted/py3/twisted/internet/tcp.py | 1523 +++ .../python/Twisted/py3/twisted/internet/testing.py | 969 ++ .../python/Twisted/py3/twisted/internet/threads.py | 144 + .../Twisted/py3/twisted/internet/tksupport.py | 78 + contrib/python/Twisted/py3/twisted/internet/udp.py | 533 + .../python/Twisted/py3/twisted/internet/unix.py | 645 + .../python/Twisted/py3/twisted/internet/utils.py | 256 + .../py3/twisted/internet/win32eventreactor.py | 425 + .../Twisted/py3/twisted/internet/wxreactor.py | 188 + .../Twisted/py3/twisted/internet/wxsupport.py | 57 + .../python/Twisted/py3/twisted/logger/__init__.py | 135 + .../python/Twisted/py3/twisted/logger/_buffer.py | 54 + .../python/Twisted/py3/twisted/logger/_capture.py | 25 + contrib/python/Twisted/py3/twisted/logger/_file.py | 77 + .../python/Twisted/py3/twisted/logger/_filter.py | 211 + .../python/Twisted/py3/twisted/logger/_flatten.py | 175 + .../python/Twisted/py3/twisted/logger/_format.py | 373 + .../python/Twisted/py3/twisted/logger/_global.py | 226 + .../Twisted/py3/twisted/logger/_interfaces.py | 63 + contrib/python/Twisted/py3/twisted/logger/_io.py | 187 + contrib/python/Twisted/py3/twisted/logger/_json.py | 285 + .../python/Twisted/py3/twisted/logger/_legacy.py | 147 + .../python/Twisted/py3/twisted/logger/_levels.py | 81 + .../python/Twisted/py3/twisted/logger/_logger.py | 269 + .../python/Twisted/py3/twisted/logger/_observer.py | 112 + .../python/Twisted/py3/twisted/logger/_stdlib.py | 131 + contrib/python/Twisted/py3/twisted/logger/_util.py | 51 + .../python/Twisted/py3/twisted/mail/__init__.py | 6 + contrib/python/Twisted/py3/twisted/mail/_cred.py | 105 + contrib/python/Twisted/py3/twisted/mail/_except.py | 350 + .../python/Twisted/py3/twisted/mail/_pop3client.py | 1235 ++ contrib/python/Twisted/py3/twisted/mail/alias.py | 765 ++ contrib/python/Twisted/py3/twisted/mail/bounce.py | 107 + contrib/python/Twisted/py3/twisted/mail/imap4.py | 6233 ++++++++++ .../python/Twisted/py3/twisted/mail/interfaces.py | 1050 ++ contrib/python/Twisted/py3/twisted/mail/mail.py | 706 ++ contrib/python/Twisted/py3/twisted/mail/maildir.py | 910 ++ .../py3/twisted/mail/newsfragments/.gitignore | 1 + contrib/python/Twisted/py3/twisted/mail/pb.py | 117 + contrib/python/Twisted/py3/twisted/mail/pop3.py | 1704 +++ .../python/Twisted/py3/twisted/mail/pop3client.py | 22 + .../python/Twisted/py3/twisted/mail/protocols.py | 385 + contrib/python/Twisted/py3/twisted/mail/relay.py | 164 + .../Twisted/py3/twisted/mail/relaymanager.py | 1135 ++ .../Twisted/py3/twisted/mail/scripts/__init__.py | 1 + .../Twisted/py3/twisted/mail/scripts/mailmail.py | 386 + contrib/python/Twisted/py3/twisted/mail/smtp.py | 2270 ++++ contrib/python/Twisted/py3/twisted/mail/tap.py | 384 + .../python/Twisted/py3/twisted/names/__init__.py | 6 + .../python/Twisted/py3/twisted/names/_rfc1982.py | 261 + .../python/Twisted/py3/twisted/names/authority.py | 503 + contrib/python/Twisted/py3/twisted/names/cache.py | 131 + contrib/python/Twisted/py3/twisted/names/client.py | 734 ++ contrib/python/Twisted/py3/twisted/names/common.py | 263 + contrib/python/Twisted/py3/twisted/names/dns.py | 3390 ++++++ contrib/python/Twisted/py3/twisted/names/error.py | 94 + contrib/python/Twisted/py3/twisted/names/hosts.py | 151 + .../py3/twisted/names/newsfragments/.gitignore | 1 + .../python/Twisted/py3/twisted/names/resolve.py | 91 + contrib/python/Twisted/py3/twisted/names/root.py | 331 + .../python/Twisted/py3/twisted/names/secondary.py | 216 + contrib/python/Twisted/py3/twisted/names/server.py | 569 + .../python/Twisted/py3/twisted/names/srvconnect.py | 271 + contrib/python/Twisted/py3/twisted/names/tap.py | 149 + .../Twisted/py3/twisted/newsfragments/.gitignore | 1 + .../python/Twisted/py3/twisted/pair/__init__.py | 13 + .../python/Twisted/py3/twisted/pair/ethernet.py | 59 + contrib/python/Twisted/py3/twisted/pair/ip.py | 78 + contrib/python/Twisted/py3/twisted/pair/raw.py | 54 + contrib/python/Twisted/py3/twisted/pair/rawudp.py | 59 + contrib/python/Twisted/py3/twisted/pair/testing.py | 555 + contrib/python/Twisted/py3/twisted/pair/tuntap.py | 423 + .../Twisted/py3/twisted/persisted/__init__.py | 6 + .../python/Twisted/py3/twisted/persisted/_token.py | 150 + .../Twisted/py3/twisted/persisted/_tokenize.py | 897 ++ .../python/Twisted/py3/twisted/persisted/aot.py | 631 + .../Twisted/py3/twisted/persisted/crefutil.py | 160 + .../python/Twisted/py3/twisted/persisted/dirdbm.py | 361 + .../py3/twisted/persisted/newsfragments/9831.misc | 0 .../python/Twisted/py3/twisted/persisted/sob.py | 200 + .../python/Twisted/py3/twisted/persisted/styles.py | 391 + contrib/python/Twisted/py3/twisted/plugin.py | 260 + .../python/Twisted/py3/twisted/plugins/__init__.py | 22 + .../Twisted/py3/twisted/plugins/cred_anonymous.py | 38 + .../Twisted/py3/twisted/plugins/cred_file.py | 59 + .../Twisted/py3/twisted/plugins/cred_memory.py | 65 + .../Twisted/py3/twisted/plugins/cred_sshkeys.py | 48 + .../Twisted/py3/twisted/plugins/cred_unix.py | 184 + .../Twisted/py3/twisted/plugins/twisted_conch.py | 19 + .../Twisted/py3/twisted/plugins/twisted_core.py | 19 + .../Twisted/py3/twisted/plugins/twisted_ftp.py | 6 + .../Twisted/py3/twisted/plugins/twisted_inet.py | 11 + .../Twisted/py3/twisted/plugins/twisted_mail.py | 8 + .../Twisted/py3/twisted/plugins/twisted_names.py | 8 + .../py3/twisted/plugins/twisted_portforward.py | 11 + .../py3/twisted/plugins/twisted_reactors.py | 59 + .../Twisted/py3/twisted/plugins/twisted_runner.py | 11 + .../Twisted/py3/twisted/plugins/twisted_socks.py | 8 + .../Twisted/py3/twisted/plugins/twisted_trial.py | 172 + .../Twisted/py3/twisted/plugins/twisted_web.py | 14 + .../Twisted/py3/twisted/plugins/twisted_words.py | 38 + .../Twisted/py3/twisted/positioning/__init__.py | 8 + .../Twisted/py3/twisted/positioning/_sentence.py | 118 + .../python/Twisted/py3/twisted/positioning/base.py | 926 ++ .../py3/twisted/positioning/ipositioning.py | 113 + .../python/Twisted/py3/twisted/positioning/nmea.py | 932 ++ .../Twisted/py3/twisted/protocols/__init__.py | 6 + .../python/Twisted/py3/twisted/protocols/amp.py | 2833 +++++ .../python/Twisted/py3/twisted/protocols/basic.py | 912 ++ .../python/Twisted/py3/twisted/protocols/finger.py | 42 + .../python/Twisted/py3/twisted/protocols/ftp.py | 3253 +++++ .../py3/twisted/protocols/haproxy/__init__.py | 10 + .../py3/twisted/protocols/haproxy/_exceptions.py | 49 + .../Twisted/py3/twisted/protocols/haproxy/_info.py | 34 + .../py3/twisted/protocols/haproxy/_interfaces.py | 63 + .../py3/twisted/protocols/haproxy/_parser.py | 75 + .../py3/twisted/protocols/haproxy/_v1parser.py | 142 + .../py3/twisted/protocols/haproxy/_v2parser.py | 217 + .../py3/twisted/protocols/haproxy/_wrapper.py | 109 + .../python/Twisted/py3/twisted/protocols/htb.py | 306 + .../python/Twisted/py3/twisted/protocols/ident.py | 253 + .../Twisted/py3/twisted/protocols/loopback.py | 387 + .../Twisted/py3/twisted/protocols/memcache.py | 733 ++ .../python/Twisted/py3/twisted/protocols/pcp.py | 211 + .../Twisted/py3/twisted/protocols/policies.py | 696 ++ .../Twisted/py3/twisted/protocols/portforward.py | 90 + .../Twisted/py3/twisted/protocols/postfix.py | 137 + .../Twisted/py3/twisted/protocols/shoutcast.py | 111 + .../python/Twisted/py3/twisted/protocols/sip.py | 1251 ++ .../python/Twisted/py3/twisted/protocols/socks.py | 249 + .../Twisted/py3/twisted/protocols/stateful.py | 52 + .../python/Twisted/py3/twisted/protocols/tls.py | 936 ++ .../python/Twisted/py3/twisted/protocols/wire.py | 112 + contrib/python/Twisted/py3/twisted/py.typed | 0 .../python/Twisted/py3/twisted/python/__init__.py | 31 + .../python/Twisted/py3/twisted/python/_appdirs.py | 32 + .../python/Twisted/py3/twisted/python/_inotify.py | 100 + .../python/_pydoctortemplates/subheader.html | 29 + .../python/Twisted/py3/twisted/python/_release.py | 281 + .../Twisted/py3/twisted/python/_shellcomp.py | 684 ++ .../Twisted/py3/twisted/python/_textattributes.py | 304 + .../python/Twisted/py3/twisted/python/_tzhelper.py | 105 + contrib/python/Twisted/py3/twisted/python/_url.py | 11 + .../python/Twisted/py3/twisted/python/compat.py | 650 + .../Twisted/py3/twisted/python/components.py | 431 + .../python/Twisted/py3/twisted/python/constants.py | 21 + .../python/Twisted/py3/twisted/python/context.py | 135 + .../python/Twisted/py3/twisted/python/deprecate.py | 820 ++ .../python/Twisted/py3/twisted/python/failure.py | 810 ++ .../python/Twisted/py3/twisted/python/fakepwd.py | 263 + .../python/Twisted/py3/twisted/python/filepath.py | 1784 +++ .../Twisted/py3/twisted/python/formmethod.py | 446 + .../python/Twisted/py3/twisted/python/htmlizer.py | 133 + .../python/Twisted/py3/twisted/python/lockfile.py | 241 + contrib/python/Twisted/py3/twisted/python/log.py | 738 ++ .../python/Twisted/py3/twisted/python/logfile.py | 341 + .../python/Twisted/py3/twisted/python/modules.py | 781 ++ .../python/Twisted/py3/twisted/python/monkey.py | 73 + .../python/Twisted/py3/twisted/python/procutils.py | 50 + .../python/Twisted/py3/twisted/python/randbytes.py | 128 + .../python/Twisted/py3/twisted/python/rebuild.py | 250 + .../python/Twisted/py3/twisted/python/reflect.py | 686 ++ .../python/Twisted/py3/twisted/python/release.py | 63 + contrib/python/Twisted/py3/twisted/python/roots.py | 242 + .../python/Twisted/py3/twisted/python/runtime.py | 204 + .../python/Twisted/py3/twisted/python/sendmsg.py | 76 + .../python/Twisted/py3/twisted/python/shortcut.py | 88 + .../python/Twisted/py3/twisted/python/syslog.py | 106 + .../python/Twisted/py3/twisted/python/systemd.py | 154 + contrib/python/Twisted/py3/twisted/python/text.py | 205 + .../Twisted/py3/twisted/python/threadable.py | 137 + .../Twisted/py3/twisted/python/threadpool.py | 340 + .../py3/twisted/python/twisted-completion.zsh | 33 + contrib/python/Twisted/py3/twisted/python/url.py | 15 + .../python/Twisted/py3/twisted/python/urlpath.py | 278 + contrib/python/Twisted/py3/twisted/python/usage.py | 1013 ++ contrib/python/Twisted/py3/twisted/python/util.py | 987 ++ .../python/Twisted/py3/twisted/python/versions.py | 13 + contrib/python/Twisted/py3/twisted/python/win32.py | 163 + .../python/Twisted/py3/twisted/python/zippath.py | 352 + .../python/Twisted/py3/twisted/python/zipstream.py | 319 + .../python/Twisted/py3/twisted/runner/__init__.py | 6 + contrib/python/Twisted/py3/twisted/runner/inetd.py | 80 + .../python/Twisted/py3/twisted/runner/inetdconf.py | 203 + .../python/Twisted/py3/twisted/runner/inetdtap.py | 109 + .../py3/twisted/runner/newsfragments/11681.misc | 0 .../py3/twisted/runner/newsfragments/9657.doc | 1 + .../python/Twisted/py3/twisted/runner/procmon.py | 407 + .../Twisted/py3/twisted/runner/procmontap.py | 96 + .../python/Twisted/py3/twisted/scripts/__init__.py | 9 + .../Twisted/py3/twisted/scripts/_twistd_unix.py | 455 + .../python/Twisted/py3/twisted/scripts/_twistw.py | 55 + .../python/Twisted/py3/twisted/scripts/htmlizer.py | 74 + .../py3/twisted/scripts/newsfragments/761.bugfix | 1 + .../python/Twisted/py3/twisted/scripts/trial.py | 654 + .../python/Twisted/py3/twisted/scripts/twistd.py | 38 + .../python/Twisted/py3/twisted/spread/__init__.py | 8 + .../python/Twisted/py3/twisted/spread/banana.py | 403 + .../python/Twisted/py3/twisted/spread/flavors.py | 651 + .../Twisted/py3/twisted/spread/interfaces.py | 30 + contrib/python/Twisted/py3/twisted/spread/jelly.py | 1092 ++ contrib/python/Twisted/py3/twisted/spread/pb.py | 1674 +++ .../python/Twisted/py3/twisted/spread/publish.py | 144 + contrib/python/Twisted/py3/twisted/spread/util.py | 217 + contrib/python/Twisted/py3/twisted/tap/__init__.py | 6 + contrib/python/Twisted/py3/twisted/tap/ftp.py | 66 + .../python/Twisted/py3/twisted/tap/portforward.py | 26 + contrib/python/Twisted/py3/twisted/tap/socks.py | 42 + .../python/Twisted/py3/twisted/trial/__init__.py | 50 + .../python/Twisted/py3/twisted/trial/__main__.py | 9 + .../Twisted/py3/twisted/trial/_asyncrunner.py | 176 + .../python/Twisted/py3/twisted/trial/_asynctest.py | 413 + .../Twisted/py3/twisted/trial/_dist/__init__.py | 47 + .../py3/twisted/trial/_dist/distreporter.py | 90 + .../Twisted/py3/twisted/trial/_dist/disttrial.py | 512 + .../Twisted/py3/twisted/trial/_dist/functional.py | 125 + .../py3/twisted/trial/_dist/managercommands.py | 89 + .../Twisted/py3/twisted/trial/_dist/options.py | 28 + .../Twisted/py3/twisted/trial/_dist/stream.py | 100 + .../Twisted/py3/twisted/trial/_dist/worker.py | 465 + .../py3/twisted/trial/_dist/workercommands.py | 30 + .../py3/twisted/trial/_dist/workerreporter.py | 354 + .../Twisted/py3/twisted/trial/_dist/workertrial.py | 93 + .../python/Twisted/py3/twisted/trial/_synctest.py | 1464 +++ contrib/python/Twisted/py3/twisted/trial/itrial.py | 157 + .../py3/twisted/trial/newsfragments/.gitignore | 1 + .../python/Twisted/py3/twisted/trial/reporter.py | 1278 ++ contrib/python/Twisted/py3/twisted/trial/runner.py | 987 ++ .../python/Twisted/py3/twisted/trial/unittest.py | 39 + contrib/python/Twisted/py3/twisted/trial/util.py | 407 + contrib/python/Twisted/py3/twisted/web/__init__.py | 12 + .../Twisted/py3/twisted/web/_auth/__init__.py | 7 + .../python/Twisted/py3/twisted/web/_auth/basic.py | 58 + .../python/Twisted/py3/twisted/web/_auth/digest.py | 56 + .../Twisted/py3/twisted/web/_auth/wrapper.py | 236 + contrib/python/Twisted/py3/twisted/web/_element.py | 200 + contrib/python/Twisted/py3/twisted/web/_flatten.py | 487 + contrib/python/Twisted/py3/twisted/web/_http2.py | 1283 ++ .../python/Twisted/py3/twisted/web/_newclient.py | 1727 +++ .../python/Twisted/py3/twisted/web/_responses.py | 110 + contrib/python/Twisted/py3/twisted/web/_stan.py | 360 + .../Twisted/py3/twisted/web/_template_util.py | 1112 ++ contrib/python/Twisted/py3/twisted/web/client.py | 1789 +++ contrib/python/Twisted/py3/twisted/web/demo.py | 27 + contrib/python/Twisted/py3/twisted/web/distrib.py | 390 + .../python/Twisted/py3/twisted/web/domhelpers.py | 313 + contrib/python/Twisted/py3/twisted/web/error.py | 442 + contrib/python/Twisted/py3/twisted/web/guard.py | 21 + contrib/python/Twisted/py3/twisted/web/html.py | 56 + contrib/python/Twisted/py3/twisted/web/http.py | 3305 +++++ .../python/Twisted/py3/twisted/web/http_headers.py | 295 + contrib/python/Twisted/py3/twisted/web/iweb.py | 830 ++ contrib/python/Twisted/py3/twisted/web/microdom.py | 1217 ++ .../py3/twisted/web/newsfragments/.gitignore | 1 + contrib/python/Twisted/py3/twisted/web/pages.py | 134 + contrib/python/Twisted/py3/twisted/web/proxy.py | 296 + contrib/python/Twisted/py3/twisted/web/resource.py | 458 + contrib/python/Twisted/py3/twisted/web/rewrite.py | 55 + contrib/python/Twisted/py3/twisted/web/script.py | 193 + contrib/python/Twisted/py3/twisted/web/server.py | 906 ++ contrib/python/Twisted/py3/twisted/web/soap.py | 166 + contrib/python/Twisted/py3/twisted/web/static.py | 1078 ++ contrib/python/Twisted/py3/twisted/web/sux.py | 644 + contrib/python/Twisted/py3/twisted/web/tap.py | 322 + contrib/python/Twisted/py3/twisted/web/template.py | 60 + .../Twisted/py3/twisted/web/test/requesthelper.py | 512 + contrib/python/Twisted/py3/twisted/web/twcgi.py | 343 + contrib/python/Twisted/py3/twisted/web/util.py | 38 + contrib/python/Twisted/py3/twisted/web/vhost.py | 137 + contrib/python/Twisted/py3/twisted/web/wsgi.py | 589 + contrib/python/Twisted/py3/twisted/web/xmlrpc.py | 633 + .../python/Twisted/py3/twisted/words/__init__.py | 8 + contrib/python/Twisted/py3/twisted/words/ewords.py | 41 + .../Twisted/py3/twisted/words/im/__init__.py | 7 + .../Twisted/py3/twisted/words/im/baseaccount.py | 69 + .../Twisted/py3/twisted/words/im/basechat.py | 486 + .../Twisted/py3/twisted/words/im/basesupport.py | 275 + .../py3/twisted/words/im/instancemessenger.glade | 3164 +++++ .../Twisted/py3/twisted/words/im/interfaces.py | 362 + .../Twisted/py3/twisted/words/im/ircsupport.py | 273 + .../python/Twisted/py3/twisted/words/im/locals.py | 30 + .../Twisted/py3/twisted/words/im/pbsupport.py | 278 + contrib/python/Twisted/py3/twisted/words/iwords.py | 281 + .../py3/twisted/words/newsfragments/.gitignore | 1 + .../py3/twisted/words/protocols/__init__.py | 6 + .../Twisted/py3/twisted/words/protocols/irc.py | 4118 +++++++ .../py3/twisted/words/protocols/jabber/__init__.py | 8 + .../py3/twisted/words/protocols/jabber/client.py | 394 + .../twisted/words/protocols/jabber/component.py | 456 + .../py3/twisted/words/protocols/jabber/error.py | 323 + .../py3/twisted/words/protocols/jabber/ijabber.py | 188 + .../py3/twisted/words/protocols/jabber/jid.py | 259 + .../twisted/words/protocols/jabber/jstrports.py | 34 + .../py3/twisted/words/protocols/jabber/sasl.py | 229 + .../words/protocols/jabber/sasl_mechanisms.py | 307 + .../twisted/words/protocols/jabber/xmlstream.py | 1145 ++ .../words/protocols/jabber/xmpp_stringprep.py | 257 + .../python/Twisted/py3/twisted/words/service.py | 1278 ++ contrib/python/Twisted/py3/twisted/words/tap.py | 89 + .../Twisted/py3/twisted/words/xish/__init__.py | 10 + .../Twisted/py3/twisted/words/xish/domish.py | 901 ++ .../Twisted/py3/twisted/words/xish/utility.py | 364 + .../Twisted/py3/twisted/words/xish/xmlstream.py | 274 + .../python/Twisted/py3/twisted/words/xish/xpath.py | 337 + .../Twisted/py3/twisted/words/xish/xpathparser.g | 524 + .../Twisted/py3/twisted/words/xish/xpathparser.py | 652 + .../Twisted/py3/twisted/words/xmpproutertap.py | 29 + contrib/python/Twisted/py3/ya.make | 492 + contrib/python/Twisted/ya.make | 18 + contrib/python/chardet/py2/test.py | 151 + contrib/python/chardet/py2/tests/ya.make | 21 + contrib/python/chardet/py3/.dist-info/METADATA | 97 + .../python/chardet/py3/.dist-info/entry_points.txt | 2 + .../python/chardet/py3/.dist-info/top_level.txt | 1 + contrib/python/chardet/py3/chardet/__init__.py | 115 + contrib/python/chardet/py3/chardet/__main__.py | 6 + contrib/python/chardet/py3/chardet/big5freq.py | 386 + contrib/python/chardet/py3/chardet/big5prober.py | 47 + .../python/chardet/py3/chardet/chardistribution.py | 261 + .../chardet/py3/chardet/charsetgroupprober.py | 106 + .../python/chardet/py3/chardet/charsetprober.py | 147 + contrib/python/chardet/py3/chardet/cli/__init__.py | 0 .../python/chardet/py3/chardet/cli/chardetect.py | 112 + .../chardet/py3/chardet/codingstatemachine.py | 90 + .../chardet/py3/chardet/codingstatemachinedict.py | 19 + contrib/python/chardet/py3/chardet/cp949prober.py | 49 + contrib/python/chardet/py3/chardet/enums.py | 85 + contrib/python/chardet/py3/chardet/escprober.py | 102 + contrib/python/chardet/py3/chardet/escsm.py | 261 + contrib/python/chardet/py3/chardet/eucjpprober.py | 102 + contrib/python/chardet/py3/chardet/euckrfreq.py | 196 + contrib/python/chardet/py3/chardet/euckrprober.py | 47 + contrib/python/chardet/py3/chardet/euctwfreq.py | 388 + contrib/python/chardet/py3/chardet/euctwprober.py | 47 + contrib/python/chardet/py3/chardet/gb2312freq.py | 284 + contrib/python/chardet/py3/chardet/gb2312prober.py | 47 + contrib/python/chardet/py3/chardet/hebrewprober.py | 316 + contrib/python/chardet/py3/chardet/jisfreq.py | 325 + contrib/python/chardet/py3/chardet/johabfreq.py | 2382 ++++ contrib/python/chardet/py3/chardet/johabprober.py | 47 + contrib/python/chardet/py3/chardet/jpcntx.py | 238 + .../chardet/py3/chardet/langbulgarianmodel.py | 4649 +++++++ .../python/chardet/py3/chardet/langgreekmodel.py | 4397 +++++++ .../python/chardet/py3/chardet/langhebrewmodel.py | 4380 +++++++ .../chardet/py3/chardet/langhungarianmodel.py | 4649 +++++++ .../python/chardet/py3/chardet/langrussianmodel.py | 5725 +++++++++ .../python/chardet/py3/chardet/langthaimodel.py | 4380 +++++++ .../python/chardet/py3/chardet/langturkishmodel.py | 4380 +++++++ contrib/python/chardet/py3/chardet/latin1prober.py | 147 + .../python/chardet/py3/chardet/macromanprober.py | 162 + .../python/chardet/py3/chardet/mbcharsetprober.py | 95 + .../python/chardet/py3/chardet/mbcsgroupprober.py | 57 + contrib/python/chardet/py3/chardet/mbcssm.py | 661 + .../chardet/py3/chardet/metadata/__init__.py | 0 .../chardet/py3/chardet/metadata/languages.py | 352 + contrib/python/chardet/py3/chardet/py.typed | 0 contrib/python/chardet/py3/chardet/resultdict.py | 16 + .../python/chardet/py3/chardet/sbcharsetprober.py | 162 + .../python/chardet/py3/chardet/sbcsgroupprober.py | 88 + contrib/python/chardet/py3/chardet/sjisprober.py | 105 + .../chardet/py3/chardet/universaldetector.py | 362 + .../python/chardet/py3/chardet/utf1632prober.py | 225 + contrib/python/chardet/py3/chardet/utf8prober.py | 82 + contrib/python/chardet/py3/chardet/version.py | 9 + contrib/python/chardet/py3/test.py | 240 + contrib/python/chardet/py3/tests/ya.make | 21 + contrib/python/chardet/py3/ya.make | 76 + contrib/python/constantly/py2/.dist-info/METADATA | 39 + .../python/constantly/py2/.dist-info/top_level.txt | 1 + contrib/python/constantly/py2/LICENSE | 21 + contrib/python/constantly/py2/README.rst | 16 + .../python/constantly/py2/constantly/__init__.py | 24 + .../python/constantly/py2/constantly/_constants.py | 500 + .../python/constantly/py2/constantly/_version.py | 21 + contrib/python/constantly/py2/ya.make | 24 + contrib/python/constantly/py3/.dist-info/METADATA | 59 + .../python/constantly/py3/.dist-info/top_level.txt | 1 + contrib/python/constantly/py3/LICENSE | 21 + contrib/python/constantly/py3/README.rst | 33 + .../python/constantly/py3/constantly/__init__.py | 23 + .../python/constantly/py3/constantly/_constants.py | 498 + .../python/constantly/py3/constantly/_version.py | 21 + contrib/python/constantly/py3/ya.make | 24 + contrib/python/constantly/ya.make | 18 + contrib/python/hyperlink/py2/.dist-info/METADATA | 38 + .../python/hyperlink/py2/.dist-info/top_level.txt | 1 + contrib/python/hyperlink/py2/LICENSE | 29 + contrib/python/hyperlink/py2/README.md | 67 + contrib/python/hyperlink/py2/hyperlink/__init__.py | 17 + contrib/python/hyperlink/py2/hyperlink/_socket.py | 53 + contrib/python/hyperlink/py2/hyperlink/_url.py | 2448 ++++ .../python/hyperlink/py2/hyperlink/hypothesis.py | 324 + .../py2/hyperlink/idna-tables-properties.csv.gz | Bin 0 -> 25555 bytes contrib/python/hyperlink/py2/hyperlink/py.typed | 1 + contrib/python/hyperlink/py2/ya.make | 36 + contrib/python/hyperlink/py3/.dist-info/METADATA | 38 + .../python/hyperlink/py3/.dist-info/top_level.txt | 1 + contrib/python/hyperlink/py3/LICENSE | 29 + contrib/python/hyperlink/py3/README.md | 67 + contrib/python/hyperlink/py3/hyperlink/__init__.py | 17 + contrib/python/hyperlink/py3/hyperlink/_socket.py | 53 + contrib/python/hyperlink/py3/hyperlink/_url.py | 2448 ++++ .../python/hyperlink/py3/hyperlink/hypothesis.py | 324 + .../py3/hyperlink/idna-tables-properties.csv.gz | Bin 0 -> 25555 bytes contrib/python/hyperlink/py3/hyperlink/py.typed | 1 + contrib/python/hyperlink/py3/ya.make | 35 + contrib/python/hyperlink/ya.make | 18 + contrib/python/incremental/py2/.dist-info/METADATA | 136 + .../incremental/py2/.dist-info/entry_points.txt | 2 + .../incremental/py2/.dist-info/top_level.txt | 1 + contrib/python/incremental/py2/LICENSE | 74 + contrib/python/incremental/py2/README.rst | 108 + .../python/incremental/py2/incremental/__init__.py | 395 + .../python/incremental/py2/incremental/_version.py | 11 + .../python/incremental/py2/incremental/py.typed | 0 .../python/incremental/py2/incremental/update.py | 331 + contrib/python/incremental/py2/ya.make | 30 + contrib/python/incremental/py3/.dist-info/METADATA | 136 + .../incremental/py3/.dist-info/entry_points.txt | 2 + .../incremental/py3/.dist-info/top_level.txt | 1 + contrib/python/incremental/py3/LICENSE | 74 + contrib/python/incremental/py3/README.rst | 108 + .../python/incremental/py3/incremental/__init__.py | 395 + .../python/incremental/py3/incremental/_version.py | 11 + .../python/incremental/py3/incremental/py.typed | 0 .../python/incremental/py3/incremental/update.py | 331 + contrib/python/incremental/py3/ya.make | 30 + contrib/python/incremental/ya.make | 18 + contrib/python/jsonschema/py2/.dist-info/METADATA | 224 + .../jsonschema/py2/.dist-info/entry_points.txt | 3 + .../python/jsonschema/py2/.dist-info/top_level.txt | 1 + contrib/python/jsonschema/py2/COPYING | 19 + contrib/python/jsonschema/py2/README.rst | 179 + .../python/jsonschema/py2/jsonschema/__init__.py | 34 + .../python/jsonschema/py2/jsonschema/__main__.py | 2 + .../python/jsonschema/py2/jsonschema/_format.py | 425 + .../py2/jsonschema/_legacy_validators.py | 141 + .../python/jsonschema/py2/jsonschema/_reflect.py | 155 + contrib/python/jsonschema/py2/jsonschema/_types.py | 188 + contrib/python/jsonschema/py2/jsonschema/_utils.py | 212 + .../jsonschema/py2/jsonschema/_validators.py | 373 + contrib/python/jsonschema/py2/jsonschema/cli.py | 90 + contrib/python/jsonschema/py2/jsonschema/compat.py | 55 + .../python/jsonschema/py2/jsonschema/exceptions.py | 374 + .../jsonschema/py2/jsonschema/schemas/draft3.json | 199 + .../jsonschema/py2/jsonschema/schemas/draft4.json | 222 + .../jsonschema/py2/jsonschema/schemas/draft6.json | 153 + .../jsonschema/py2/jsonschema/schemas/draft7.json | 166 + .../jsonschema/py2/jsonschema/tests/__init__.py | 0 .../jsonschema/py2/jsonschema/tests/_helpers.py | 5 + .../jsonschema/py2/jsonschema/tests/test_cli.py | 143 + .../py2/jsonschema/tests/test_exceptions.py | 462 + .../jsonschema/py2/jsonschema/tests/test_format.py | 89 + .../jsonschema/py2/jsonschema/tests/test_types.py | 190 + .../py2/jsonschema/tests/test_validators.py | 1762 +++ .../python/jsonschema/py2/jsonschema/validators.py | 970 ++ contrib/python/jsonschema/py2/tests/ya.make | 32 + contrib/python/jsonschema/py2/ya.make | 51 + contrib/python/jsonschema/py3/.dist-info/METADATA | 224 + .../jsonschema/py3/.dist-info/entry_points.txt | 3 + .../python/jsonschema/py3/.dist-info/top_level.txt | 1 + contrib/python/jsonschema/py3/COPYING | 19 + contrib/python/jsonschema/py3/README.rst | 179 + .../python/jsonschema/py3/jsonschema/__init__.py | 34 + .../python/jsonschema/py3/jsonschema/__main__.py | 2 + .../python/jsonschema/py3/jsonschema/_format.py | 425 + .../py3/jsonschema/_legacy_validators.py | 141 + .../python/jsonschema/py3/jsonschema/_reflect.py | 155 + contrib/python/jsonschema/py3/jsonschema/_types.py | 188 + contrib/python/jsonschema/py3/jsonschema/_utils.py | 212 + .../jsonschema/py3/jsonschema/_validators.py | 373 + contrib/python/jsonschema/py3/jsonschema/cli.py | 90 + contrib/python/jsonschema/py3/jsonschema/compat.py | 55 + .../python/jsonschema/py3/jsonschema/exceptions.py | 374 + .../jsonschema/py3/jsonschema/schemas/draft3.json | 199 + .../jsonschema/py3/jsonschema/schemas/draft4.json | 222 + .../jsonschema/py3/jsonschema/schemas/draft6.json | 153 + .../jsonschema/py3/jsonschema/schemas/draft7.json | 166 + .../jsonschema/py3/jsonschema/tests/__init__.py | 0 .../jsonschema/py3/jsonschema/tests/_helpers.py | 5 + .../jsonschema/py3/jsonschema/tests/test_cli.py | 143 + .../py3/jsonschema/tests/test_exceptions.py | 462 + .../jsonschema/py3/jsonschema/tests/test_format.py | 89 + .../jsonschema/py3/jsonschema/tests/test_types.py | 190 + .../py3/jsonschema/tests/test_validators.py | 1762 +++ .../python/jsonschema/py3/jsonschema/validators.py | 970 ++ contrib/python/jsonschema/py3/tests/ya.make | 32 + contrib/python/jsonschema/py3/ya.make | 49 + contrib/python/jsonschema/ya.make | 18 + contrib/python/pyOpenSSL/py2/.dist-info/METADATA | 198 + .../python/pyOpenSSL/py2/.dist-info/top_level.txt | 1 + contrib/python/pyOpenSSL/py2/LICENSE | 202 + contrib/python/pyOpenSSL/py2/OpenSSL/SSL.py | 2505 ++++ contrib/python/pyOpenSSL/py2/OpenSSL/__init__.py | 32 + contrib/python/pyOpenSSL/py2/OpenSSL/_util.py | 155 + contrib/python/pyOpenSSL/py2/OpenSSL/crypto.py | 3288 +++++ contrib/python/pyOpenSSL/py2/OpenSSL/debug.py | 42 + contrib/python/pyOpenSSL/py2/OpenSSL/rand.py | 40 + contrib/python/pyOpenSSL/py2/OpenSSL/version.py | 28 + contrib/python/pyOpenSSL/py2/README.rst | 46 + contrib/python/pyOpenSSL/py2/ya.make | 33 + contrib/python/pyOpenSSL/py3/LICENSE | 202 + contrib/python/pyOpenSSL/py3/README.rst | 46 + contrib/python/pyOpenSSL/ya.make | 18 + contrib/python/pyrsistent/py2/.dist-info/METADATA | 744 ++ .../python/pyrsistent/py2/.dist-info/top_level.txt | 3 + contrib/python/pyrsistent/py2/README.rst | 723 ++ .../python/pyrsistent/py2/_pyrsistent_version.py | 1 + .../python/pyrsistent/py2/pyrsistent/__init__.py | 47 + .../pyrsistent/py2/pyrsistent/_checked_types.py | 542 + .../python/pyrsistent/py2/pyrsistent/_compat.py | 31 + .../pyrsistent/py2/pyrsistent/_field_common.py | 330 + .../python/pyrsistent/py2/pyrsistent/_helpers.py | 82 + .../python/pyrsistent/py2/pyrsistent/_immutable.py | 105 + contrib/python/pyrsistent/py2/pyrsistent/_pbag.py | 267 + .../python/pyrsistent/py2/pyrsistent/_pclass.py | 264 + .../python/pyrsistent/py2/pyrsistent/_pdeque.py | 376 + contrib/python/pyrsistent/py2/pyrsistent/_plist.py | 313 + contrib/python/pyrsistent/py2/pyrsistent/_pmap.py | 460 + .../python/pyrsistent/py2/pyrsistent/_precord.py | 169 + contrib/python/pyrsistent/py2/pyrsistent/_pset.py | 229 + .../python/pyrsistent/py2/pyrsistent/_pvector.py | 713 ++ contrib/python/pyrsistent/py2/pyrsistent/_toolz.py | 83 + .../pyrsistent/py2/pyrsistent/_transformations.py | 143 + contrib/python/pyrsistent/py2/pyrsistent/py.typed | 0 contrib/python/pyrsistent/py2/pyrsistent/typing.py | 80 + contrib/python/pyrsistent/py2/tests/bag_test.py | 150 + .../pyrsistent/py2/tests/checked_map_test.py | 151 + .../pyrsistent/py2/tests/checked_set_test.py | 85 + .../pyrsistent/py2/tests/checked_vector_test.py | 213 + contrib/python/pyrsistent/py2/tests/class_test.py | 477 + contrib/python/pyrsistent/py2/tests/deque_test.py | 293 + contrib/python/pyrsistent/py2/tests/field_test.py | 27 + contrib/python/pyrsistent/py2/tests/freeze_test.py | 101 + .../pyrsistent/py2/tests/hypothesis_vector_test.py | 304 + .../pyrsistent/py2/tests/immutable_object_test.py | 67 + contrib/python/pyrsistent/py2/tests/list_test.py | 209 + contrib/python/pyrsistent/py2/tests/map_test.py | 497 + .../pyrsistent/py2/tests/memory_profiling.py | 44 + contrib/python/pyrsistent/py2/tests/record_test.py | 864 ++ .../python/pyrsistent/py2/tests/regression_test.py | 30 + contrib/python/pyrsistent/py2/tests/set_test.py | 178 + contrib/python/pyrsistent/py2/tests/toolz_test.py | 6 + .../python/pyrsistent/py2/tests/transform_test.py | 117 + contrib/python/pyrsistent/py2/tests/vector_test.py | 934 ++ contrib/python/pyrsistent/py2/tests/ya.make | 27 + contrib/python/pyrsistent/py2/ya.make | 52 + contrib/python/pyrsistent/py3/.dist-info/METADATA | 789 ++ .../python/pyrsistent/py3/.dist-info/top_level.txt | 3 + contrib/python/pyrsistent/py3/LICENSE.mit | 22 + contrib/python/pyrsistent/py3/README.rst | 767 ++ .../python/pyrsistent/py3/_pyrsistent_version.py | 1 + .../python/pyrsistent/py3/pyrsistent/__init__.py | 47 + .../pyrsistent/py3/pyrsistent/_checked_types.py | 547 + .../pyrsistent/py3/pyrsistent/_field_common.py | 332 + .../python/pyrsistent/py3/pyrsistent/_helpers.py | 101 + .../python/pyrsistent/py3/pyrsistent/_immutable.py | 97 + contrib/python/pyrsistent/py3/pyrsistent/_pbag.py | 270 + .../python/pyrsistent/py3/pyrsistent/_pclass.py | 262 + .../python/pyrsistent/py3/pyrsistent/_pdeque.py | 379 + contrib/python/pyrsistent/py3/pyrsistent/_plist.py | 316 + contrib/python/pyrsistent/py3/pyrsistent/_pmap.py | 583 + .../python/pyrsistent/py3/pyrsistent/_precord.py | 167 + contrib/python/pyrsistent/py3/pyrsistent/_pset.py | 230 + .../python/pyrsistent/py3/pyrsistent/_pvector.py | 715 ++ contrib/python/pyrsistent/py3/pyrsistent/_toolz.py | 83 + .../pyrsistent/py3/pyrsistent/_transformations.py | 143 + contrib/python/pyrsistent/py3/pyrsistent/py.typed | 0 contrib/python/pyrsistent/py3/pyrsistent/typing.py | 82 + contrib/python/pyrsistent/py3/tests/bag_test.py | 150 + .../pyrsistent/py3/tests/checked_map_test.py | 152 + .../pyrsistent/py3/tests/checked_set_test.py | 85 + .../pyrsistent/py3/tests/checked_vector_test.py | 213 + contrib/python/pyrsistent/py3/tests/class_test.py | 474 + contrib/python/pyrsistent/py3/tests/deque_test.py | 293 + contrib/python/pyrsistent/py3/tests/field_test.py | 23 + contrib/python/pyrsistent/py3/tests/freeze_test.py | 174 + .../pyrsistent/py3/tests/hypothesis_vector_test.py | 304 + .../pyrsistent/py3/tests/immutable_object_test.py | 67 + contrib/python/pyrsistent/py3/tests/list_test.py | 209 + contrib/python/pyrsistent/py3/tests/map_test.py | 551 + .../pyrsistent/py3/tests/memory_profiling.py | 48 + contrib/python/pyrsistent/py3/tests/record_test.py | 878 ++ .../python/pyrsistent/py3/tests/regression_test.py | 30 + contrib/python/pyrsistent/py3/tests/set_test.py | 181 + contrib/python/pyrsistent/py3/tests/toolz_test.py | 6 + .../python/pyrsistent/py3/tests/transform_test.py | 122 + contrib/python/pyrsistent/py3/tests/vector_test.py | 934 ++ contrib/python/pyrsistent/py3/tests/ya.make | 27 + contrib/python/pyrsistent/py3/ya.make | 47 + contrib/python/pyrsistent/ya.make | 18 + .../python/zope.interface/py2/.dist-info/METADATA | 1094 ++ .../zope.interface/py2/.dist-info/top_level.txt | 1 + contrib/python/zope.interface/py2/COPYRIGHT.txt | 1 + contrib/python/zope.interface/py2/LICENSE.txt | 44 + contrib/python/zope.interface/py2/README.rst | 31 + contrib/python/zope.interface/py2/ya.make | 61 + .../zope.interface/py2/zope/interface/__init__.py | 96 + .../zope.interface/py2/zope/interface/_compat.py | 170 + .../zope.interface/py2/zope/interface/_flatten.py | 35 + .../interface/_zope_interface_coptimizations.c | 2122 ++++ .../zope.interface/py2/zope/interface/adapter.py | 1018 ++ .../zope.interface/py2/zope/interface/advice.py | 213 + .../py2/zope/interface/common/__init__.py | 272 + .../py2/zope/interface/common/builtins.py | 125 + .../py2/zope/interface/common/collections.py | 284 + .../py2/zope/interface/common/idatetime.py | 606 + .../py2/zope/interface/common/interfaces.py | 212 + .../zope.interface/py2/zope/interface/common/io.py | 53 + .../py2/zope/interface/common/mapping.py | 184 + .../py2/zope/interface/common/numbers.py | 84 + .../py2/zope/interface/common/sequence.py | 215 + .../py2/zope/interface/declarations.py | 1313 ++ .../zope.interface/py2/zope/interface/document.py | 124 + .../py2/zope/interface/exceptions.py | 275 + .../zope.interface/py2/zope/interface/interface.py | 1153 ++ .../py2/zope/interface/interfaces.py | 1593 +++ .../zope.interface/py2/zope/interface/registry.py | 726 ++ .../python/zope.interface/py2/zope/interface/ro.py | 666 + .../zope.interface/py2/zope/interface/verify.py | 218 + .../python/zope.interface/py3/.dist-info/METADATA | 1120 ++ .../zope.interface/py3/.dist-info/top_level.txt | 1 + contrib/python/zope.interface/py3/COPYRIGHT.txt | 1 + contrib/python/zope.interface/py3/LICENSE.txt | 44 + contrib/python/zope.interface/py3/README.rst | 31 + contrib/python/zope.interface/py3/ya.make | 61 + .../zope.interface/py3/zope/interface/__init__.py | 93 + .../zope.interface/py3/zope/interface/_compat.py | 135 + .../zope.interface/py3/zope/interface/_flatten.py | 35 + .../interface/_zope_interface_coptimizations.c | 2101 ++++ .../zope.interface/py3/zope/interface/adapter.py | 1015 ++ .../zope.interface/py3/zope/interface/advice.py | 118 + .../py3/zope/interface/common/__init__.py | 272 + .../py3/zope/interface/common/builtins.py | 119 + .../py3/zope/interface/common/collections.py | 253 + .../py3/zope/interface/common/idatetime.py | 606 + .../py3/zope/interface/common/interfaces.py | 208 + .../zope.interface/py3/zope/interface/common/io.py | 43 + .../py3/zope/interface/common/mapping.py | 168 + .../py3/zope/interface/common/numbers.py | 65 + .../py3/zope/interface/common/sequence.py | 189 + .../py3/zope/interface/declarations.py | 1188 ++ .../zope.interface/py3/zope/interface/document.py | 124 + .../py3/zope/interface/exceptions.py | 275 + .../zope.interface/py3/zope/interface/interface.py | 1131 ++ .../py3/zope/interface/interfaces.py | 1480 +++ .../zope.interface/py3/zope/interface/registry.py | 723 ++ .../python/zope.interface/py3/zope/interface/ro.py | 665 + .../zope.interface/py3/zope/interface/verify.py | 185 + contrib/python/zope.interface/ya.make | 18 + 1272 files changed, 476841 insertions(+) create mode 100644 contrib/python/Automat/py2/.dist-info/METADATA create mode 100644 contrib/python/Automat/py2/.dist-info/entry_points.txt create mode 100644 contrib/python/Automat/py2/.dist-info/top_level.txt create mode 100644 contrib/python/Automat/py2/LICENSE create mode 100644 contrib/python/Automat/py2/README.md create mode 100644 contrib/python/Automat/py2/automat/__init__.py create mode 100644 contrib/python/Automat/py2/automat/_core.py create mode 100644 contrib/python/Automat/py2/automat/_discover.py create mode 100644 contrib/python/Automat/py2/automat/_introspection.py create mode 100644 contrib/python/Automat/py2/automat/_methodical.py create mode 100644 contrib/python/Automat/py2/automat/_visualize.py create mode 100644 contrib/python/Automat/py2/ya.make create mode 100644 contrib/python/Automat/py3/.dist-info/METADATA create mode 100644 contrib/python/Automat/py3/.dist-info/entry_points.txt create mode 100644 contrib/python/Automat/py3/.dist-info/top_level.txt create mode 100644 contrib/python/Automat/py3/LICENSE create mode 100644 contrib/python/Automat/py3/README.md create mode 100644 contrib/python/Automat/py3/automat/__init__.py create mode 100644 contrib/python/Automat/py3/automat/_core.py create mode 100644 contrib/python/Automat/py3/automat/_discover.py create mode 100644 contrib/python/Automat/py3/automat/_introspection.py create mode 100644 contrib/python/Automat/py3/automat/_methodical.py create mode 100644 contrib/python/Automat/py3/automat/_visualize.py create mode 100644 contrib/python/Automat/py3/ya.make create mode 100644 contrib/python/Automat/ya.make create mode 100644 contrib/python/Twisted/py2/.dist-info/METADATA create mode 100644 contrib/python/Twisted/py2/.dist-info/entry_points.txt create mode 100644 contrib/python/Twisted/py2/.dist-info/top_level.txt create mode 100644 contrib/python/Twisted/py2/LICENSE create mode 100644 contrib/python/Twisted/py2/README.rst create mode 100644 contrib/python/Twisted/py2/twisted/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/_threads/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/_threads/_convenience.py create mode 100644 contrib/python/Twisted/py2/twisted/_threads/_ithreads.py create mode 100644 contrib/python/Twisted/py2/twisted/_threads/_memory.py create mode 100644 contrib/python/Twisted/py2/twisted/_threads/_pool.py create mode 100644 contrib/python/Twisted/py2/twisted/_threads/_team.py create mode 100644 contrib/python/Twisted/py2/twisted/_threads/_threadworker.py create mode 100644 contrib/python/Twisted/py2/twisted/_version.py create mode 100644 contrib/python/Twisted/py2/twisted/application/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/application/app.py create mode 100644 contrib/python/Twisted/py2/twisted/application/internet.py create mode 100644 contrib/python/Twisted/py2/twisted/application/reactors.py create mode 100644 contrib/python/Twisted/py2/twisted/application/runner/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/application/runner/_exit.py create mode 100644 contrib/python/Twisted/py2/twisted/application/runner/_pidfile.py create mode 100644 contrib/python/Twisted/py2/twisted/application/runner/_runner.py create mode 100644 contrib/python/Twisted/py2/twisted/application/service.py create mode 100644 contrib/python/Twisted/py2/twisted/application/strports.py create mode 100644 contrib/python/Twisted/py2/twisted/application/twist/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/application/twist/_options.py create mode 100644 contrib/python/Twisted/py2/twisted/application/twist/_twist.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/avatar.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/checkers.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/client/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/client/agent.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/client/connect.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/client/default.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/client/direct.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/client/knownhosts.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/client/options.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/endpoints.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/error.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/insults/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/insults/helper.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/insults/insults.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/insults/text.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/insults/window.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/interfaces.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ls.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/manhole.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/manhole_ssh.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/manhole_tap.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/mixin.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/openssh_compat/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/openssh_compat/factory.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/openssh_compat/primes.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/recvline.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/_kex.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/address.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/agent.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/channel.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/common.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/connection.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/factory.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/filetransfer.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/forwarding.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/keys.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/service.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/session.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/sexpy.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/transport.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ssh/userauth.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/stdio.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/tap.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/telnet.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ttymodes.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ui/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ui/ansi.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/ui/tkvt100.py create mode 100644 contrib/python/Twisted/py2/twisted/conch/unix.py create mode 100644 contrib/python/Twisted/py2/twisted/copyright.py create mode 100644 contrib/python/Twisted/py2/twisted/cred/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/cred/_digest.py create mode 100644 contrib/python/Twisted/py2/twisted/cred/checkers.py create mode 100644 contrib/python/Twisted/py2/twisted/cred/credentials.py create mode 100644 contrib/python/Twisted/py2/twisted/cred/error.py create mode 100644 contrib/python/Twisted/py2/twisted/cred/portal.py create mode 100644 contrib/python/Twisted/py2/twisted/cred/strcred.py create mode 100644 contrib/python/Twisted/py2/twisted/enterprise/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/enterprise/adbapi.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_baseprocess.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_dumbwin32proc.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_glibbase.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_idna.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_newtls.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_pollingfile.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_posixserialport.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_posixstdio.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_producer_helpers.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_resolver.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_signals.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_sslverify.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_threadedselect.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_win32serialport.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/_win32stdio.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/abstract.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/address.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/asyncioreactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/base.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/cfreactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/default.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/defer.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/endpoints.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/epollreactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/error.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/fdesc.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/gireactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/glib2reactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/gtk2reactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/gtk3reactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/inotify.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/interfaces.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/abstract.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/const.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/interfaces.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/iocpsupport.c create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.h create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/notes.txt create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/reactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/setup.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/tcp.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/iocpreactor/udp.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/kqreactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/main.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/pollreactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/posixbase.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/process.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/protocol.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/pyuisupport.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/reactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/selectreactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/serialport.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/ssl.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/stdio.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/task.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/tcp.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/testing.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/threads.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/tksupport.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/udp.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/unix.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/utils.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/win32eventreactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/wxreactor.py create mode 100644 contrib/python/Twisted/py2/twisted/internet/wxsupport.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_buffer.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_capture.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_file.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_filter.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_flatten.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_format.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_global.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_io.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_json.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_legacy.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_levels.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_logger.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_observer.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_stdlib.py create mode 100644 contrib/python/Twisted/py2/twisted/logger/_util.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/_cred.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/_except.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/alias.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/bounce.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/imap4.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/interfaces.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/mail.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/maildir.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/pb.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/pop3.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/pop3client.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/protocols.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/relay.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/relaymanager.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/smtp.py create mode 100644 contrib/python/Twisted/py2/twisted/mail/tap.py create mode 100644 contrib/python/Twisted/py2/twisted/names/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/names/_rfc1982.py create mode 100644 contrib/python/Twisted/py2/twisted/names/authority.py create mode 100644 contrib/python/Twisted/py2/twisted/names/cache.py create mode 100644 contrib/python/Twisted/py2/twisted/names/client.py create mode 100644 contrib/python/Twisted/py2/twisted/names/common.py create mode 100644 contrib/python/Twisted/py2/twisted/names/dns.py create mode 100644 contrib/python/Twisted/py2/twisted/names/error.py create mode 100644 contrib/python/Twisted/py2/twisted/names/hosts.py create mode 100644 contrib/python/Twisted/py2/twisted/names/resolve.py create mode 100644 contrib/python/Twisted/py2/twisted/names/root.py create mode 100644 contrib/python/Twisted/py2/twisted/names/secondary.py create mode 100644 contrib/python/Twisted/py2/twisted/names/server.py create mode 100644 contrib/python/Twisted/py2/twisted/names/srvconnect.py create mode 100644 contrib/python/Twisted/py2/twisted/names/tap.py create mode 100644 contrib/python/Twisted/py2/twisted/news/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/news/database.py create mode 100644 contrib/python/Twisted/py2/twisted/news/news.py create mode 100644 contrib/python/Twisted/py2/twisted/news/nntp.py create mode 100644 contrib/python/Twisted/py2/twisted/news/tap.py create mode 100644 contrib/python/Twisted/py2/twisted/pair/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/pair/ethernet.py create mode 100644 contrib/python/Twisted/py2/twisted/pair/ip.py create mode 100644 contrib/python/Twisted/py2/twisted/pair/raw.py create mode 100644 contrib/python/Twisted/py2/twisted/pair/rawudp.py create mode 100644 contrib/python/Twisted/py2/twisted/pair/testing.py create mode 100644 contrib/python/Twisted/py2/twisted/pair/tuntap.py create mode 100644 contrib/python/Twisted/py2/twisted/persisted/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/persisted/aot.py create mode 100644 contrib/python/Twisted/py2/twisted/persisted/crefutil.py create mode 100644 contrib/python/Twisted/py2/twisted/persisted/dirdbm.py create mode 100644 contrib/python/Twisted/py2/twisted/persisted/sob.py create mode 100644 contrib/python/Twisted/py2/twisted/persisted/styles.py create mode 100644 contrib/python/Twisted/py2/twisted/plugin.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/cred_anonymous.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/cred_file.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/cred_memory.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/cred_sshkeys.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/cred_unix.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_conch.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_core.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_ftp.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_inet.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_mail.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_names.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_news.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_portforward.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_reactors.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_runner.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_socks.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_trial.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_web.py create mode 100644 contrib/python/Twisted/py2/twisted/plugins/twisted_words.py create mode 100644 contrib/python/Twisted/py2/twisted/positioning/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/positioning/_sentence.py create mode 100644 contrib/python/Twisted/py2/twisted/positioning/base.py create mode 100644 contrib/python/Twisted/py2/twisted/positioning/ipositioning.py create mode 100644 contrib/python/Twisted/py2/twisted/positioning/nmea.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/amp.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/basic.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/dict.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/finger.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/ftp.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/haproxy/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/haproxy/_exceptions.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/haproxy/_info.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/haproxy/_interfaces.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/haproxy/_parser.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/haproxy/_v1parser.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/haproxy/_v2parser.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/haproxy/_wrapper.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/htb.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/ident.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/loopback.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/memcache.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/pcp.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/policies.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/portforward.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/postfix.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/shoutcast.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/sip.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/socks.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/stateful.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/tls.py create mode 100644 contrib/python/Twisted/py2/twisted/protocols/wire.py create mode 100644 contrib/python/Twisted/py2/twisted/python/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_appdirs.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_inotify.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_oldstyle.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_pydoctor.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_release.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_sendmsg.c create mode 100644 contrib/python/Twisted/py2/twisted/python/_setup.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_shellcomp.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_textattributes.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_tzhelper.py create mode 100644 contrib/python/Twisted/py2/twisted/python/_url.py create mode 100644 contrib/python/Twisted/py2/twisted/python/compat.py create mode 100644 contrib/python/Twisted/py2/twisted/python/components.py create mode 100644 contrib/python/Twisted/py2/twisted/python/constants.py create mode 100644 contrib/python/Twisted/py2/twisted/python/context.py create mode 100644 contrib/python/Twisted/py2/twisted/python/deprecate.py create mode 100644 contrib/python/Twisted/py2/twisted/python/failure.py create mode 100644 contrib/python/Twisted/py2/twisted/python/fakepwd.py create mode 100644 contrib/python/Twisted/py2/twisted/python/filepath.py create mode 100644 contrib/python/Twisted/py2/twisted/python/finalize.py create mode 100644 contrib/python/Twisted/py2/twisted/python/formmethod.py create mode 100644 contrib/python/Twisted/py2/twisted/python/hook.py create mode 100644 contrib/python/Twisted/py2/twisted/python/htmlizer.py create mode 100644 contrib/python/Twisted/py2/twisted/python/lockfile.py create mode 100644 contrib/python/Twisted/py2/twisted/python/log.py create mode 100644 contrib/python/Twisted/py2/twisted/python/logfile.py create mode 100644 contrib/python/Twisted/py2/twisted/python/modules.py create mode 100644 contrib/python/Twisted/py2/twisted/python/monkey.py create mode 100644 contrib/python/Twisted/py2/twisted/python/procutils.py create mode 100644 contrib/python/Twisted/py2/twisted/python/randbytes.py create mode 100644 contrib/python/Twisted/py2/twisted/python/rebuild.py create mode 100644 contrib/python/Twisted/py2/twisted/python/reflect.py create mode 100644 contrib/python/Twisted/py2/twisted/python/release.py create mode 100644 contrib/python/Twisted/py2/twisted/python/roots.py create mode 100644 contrib/python/Twisted/py2/twisted/python/runtime.py create mode 100644 contrib/python/Twisted/py2/twisted/python/sendmsg.py create mode 100644 contrib/python/Twisted/py2/twisted/python/shortcut.py create mode 100644 contrib/python/Twisted/py2/twisted/python/syslog.py create mode 100644 contrib/python/Twisted/py2/twisted/python/systemd.py create mode 100644 contrib/python/Twisted/py2/twisted/python/text.py create mode 100644 contrib/python/Twisted/py2/twisted/python/threadable.py create mode 100644 contrib/python/Twisted/py2/twisted/python/threadpool.py create mode 100644 contrib/python/Twisted/py2/twisted/python/url.py create mode 100644 contrib/python/Twisted/py2/twisted/python/urlpath.py create mode 100644 contrib/python/Twisted/py2/twisted/python/usage.py create mode 100644 contrib/python/Twisted/py2/twisted/python/util.py create mode 100644 contrib/python/Twisted/py2/twisted/python/versions.py create mode 100644 contrib/python/Twisted/py2/twisted/python/win32.py create mode 100644 contrib/python/Twisted/py2/twisted/python/ya.make create mode 100644 contrib/python/Twisted/py2/twisted/python/zippath.py create mode 100644 contrib/python/Twisted/py2/twisted/python/zipstream.py create mode 100644 contrib/python/Twisted/py2/twisted/runner/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/runner/inetd.py create mode 100644 contrib/python/Twisted/py2/twisted/runner/inetdconf.py create mode 100644 contrib/python/Twisted/py2/twisted/runner/inetdtap.py create mode 100644 contrib/python/Twisted/py2/twisted/runner/procmon.py create mode 100644 contrib/python/Twisted/py2/twisted/runner/procmontap.py create mode 100644 contrib/python/Twisted/py2/twisted/scripts/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/scripts/_twistd_unix.py create mode 100644 contrib/python/Twisted/py2/twisted/scripts/_twistw.py create mode 100644 contrib/python/Twisted/py2/twisted/scripts/htmlizer.py create mode 100644 contrib/python/Twisted/py2/twisted/scripts/trial.py create mode 100644 contrib/python/Twisted/py2/twisted/scripts/twistd.py create mode 100644 contrib/python/Twisted/py2/twisted/spread/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/spread/banana.py create mode 100644 contrib/python/Twisted/py2/twisted/spread/flavors.py create mode 100644 contrib/python/Twisted/py2/twisted/spread/interfaces.py create mode 100644 contrib/python/Twisted/py2/twisted/spread/jelly.py create mode 100644 contrib/python/Twisted/py2/twisted/spread/pb.py create mode 100644 contrib/python/Twisted/py2/twisted/spread/publish.py create mode 100644 contrib/python/Twisted/py2/twisted/spread/util.py create mode 100644 contrib/python/Twisted/py2/twisted/tap/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/tap/ftp.py create mode 100644 contrib/python/Twisted/py2/twisted/tap/portforward.py create mode 100644 contrib/python/Twisted/py2/twisted/tap/socks.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/__main__.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_asyncrunner.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_asynctest.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/distreporter.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/disttrial.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/managercommands.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/options.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/worker.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/workercommands.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/workerreporter.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_dist/workertrial.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/_synctest.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/itrial.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/reporter.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/runner.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/unittest.py create mode 100644 contrib/python/Twisted/py2/twisted/trial/util.py create mode 100644 contrib/python/Twisted/py2/twisted/web/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_auth/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_auth/basic.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_auth/digest.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_auth/wrapper.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_element.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_flatten.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_http2.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_newclient.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_responses.py create mode 100644 contrib/python/Twisted/py2/twisted/web/_stan.py create mode 100644 contrib/python/Twisted/py2/twisted/web/client.py create mode 100644 contrib/python/Twisted/py2/twisted/web/demo.py create mode 100644 contrib/python/Twisted/py2/twisted/web/distrib.py create mode 100644 contrib/python/Twisted/py2/twisted/web/domhelpers.py create mode 100644 contrib/python/Twisted/py2/twisted/web/error.py create mode 100644 contrib/python/Twisted/py2/twisted/web/guard.py create mode 100644 contrib/python/Twisted/py2/twisted/web/html.py create mode 100644 contrib/python/Twisted/py2/twisted/web/http.py create mode 100644 contrib/python/Twisted/py2/twisted/web/http_headers.py create mode 100644 contrib/python/Twisted/py2/twisted/web/iweb.py create mode 100644 contrib/python/Twisted/py2/twisted/web/microdom.py create mode 100644 contrib/python/Twisted/py2/twisted/web/proxy.py create mode 100644 contrib/python/Twisted/py2/twisted/web/resource.py create mode 100644 contrib/python/Twisted/py2/twisted/web/rewrite.py create mode 100644 contrib/python/Twisted/py2/twisted/web/script.py create mode 100644 contrib/python/Twisted/py2/twisted/web/server.py create mode 100644 contrib/python/Twisted/py2/twisted/web/soap.py create mode 100644 contrib/python/Twisted/py2/twisted/web/static.py create mode 100644 contrib/python/Twisted/py2/twisted/web/sux.py create mode 100644 contrib/python/Twisted/py2/twisted/web/tap.py create mode 100644 contrib/python/Twisted/py2/twisted/web/template.py create mode 100644 contrib/python/Twisted/py2/twisted/web/test/requesthelper.py create mode 100644 contrib/python/Twisted/py2/twisted/web/twcgi.py create mode 100644 contrib/python/Twisted/py2/twisted/web/util.py create mode 100644 contrib/python/Twisted/py2/twisted/web/vhost.py create mode 100644 contrib/python/Twisted/py2/twisted/web/wsgi.py create mode 100644 contrib/python/Twisted/py2/twisted/web/xmlrpc.py create mode 100644 contrib/python/Twisted/py2/twisted/words/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/words/ewords.py create mode 100644 contrib/python/Twisted/py2/twisted/words/im/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/words/im/baseaccount.py create mode 100644 contrib/python/Twisted/py2/twisted/words/im/basechat.py create mode 100644 contrib/python/Twisted/py2/twisted/words/im/basesupport.py create mode 100644 contrib/python/Twisted/py2/twisted/words/im/interfaces.py create mode 100644 contrib/python/Twisted/py2/twisted/words/im/ircsupport.py create mode 100644 contrib/python/Twisted/py2/twisted/words/im/locals.py create mode 100644 contrib/python/Twisted/py2/twisted/words/im/pbsupport.py create mode 100644 contrib/python/Twisted/py2/twisted/words/iwords.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/irc.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/client.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/component.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/error.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/ijabber.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/jid.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/jstrports.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/sasl.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/sasl_mechanisms.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/xmlstream.py create mode 100644 contrib/python/Twisted/py2/twisted/words/protocols/jabber/xmpp_stringprep.py create mode 100644 contrib/python/Twisted/py2/twisted/words/service.py create mode 100644 contrib/python/Twisted/py2/twisted/words/tap.py create mode 100644 contrib/python/Twisted/py2/twisted/words/xish/__init__.py create mode 100644 contrib/python/Twisted/py2/twisted/words/xish/domish.py create mode 100644 contrib/python/Twisted/py2/twisted/words/xish/utility.py create mode 100644 contrib/python/Twisted/py2/twisted/words/xish/xmlstream.py create mode 100644 contrib/python/Twisted/py2/twisted/words/xish/xpath.py create mode 100644 contrib/python/Twisted/py2/twisted/words/xish/xpathparser.py create mode 100644 contrib/python/Twisted/py2/twisted/words/xmpproutertap.py create mode 100644 contrib/python/Twisted/py2/ya.make create mode 100644 contrib/python/Twisted/py3/.dist-info/METADATA create mode 100644 contrib/python/Twisted/py3/.dist-info/entry_points.txt create mode 100644 contrib/python/Twisted/py3/.dist-info/top_level.txt create mode 100644 contrib/python/Twisted/py3/LICENSE create mode 100644 contrib/python/Twisted/py3/README.rst create mode 100644 contrib/python/Twisted/py3/twisted/11715.misc create mode 100644 contrib/python/Twisted/py3/twisted/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/__main__.py create mode 100644 contrib/python/Twisted/py3/twisted/_threads/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/_threads/_convenience.py create mode 100644 contrib/python/Twisted/py3/twisted/_threads/_ithreads.py create mode 100644 contrib/python/Twisted/py3/twisted/_threads/_memory.py create mode 100644 contrib/python/Twisted/py3/twisted/_threads/_pool.py create mode 100644 contrib/python/Twisted/py3/twisted/_threads/_team.py create mode 100644 contrib/python/Twisted/py3/twisted/_threads/_threadworker.py create mode 100644 contrib/python/Twisted/py3/twisted/_version.py create mode 100644 contrib/python/Twisted/py3/twisted/application/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/application/app.py create mode 100644 contrib/python/Twisted/py3/twisted/application/internet.py create mode 100644 contrib/python/Twisted/py3/twisted/application/newsfragments/10146.misc create mode 100644 contrib/python/Twisted/py3/twisted/application/newsfragments/9746.misc create mode 100644 contrib/python/Twisted/py3/twisted/application/reactors.py create mode 100644 contrib/python/Twisted/py3/twisted/application/runner/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/application/runner/_exit.py create mode 100644 contrib/python/Twisted/py3/twisted/application/runner/_pidfile.py create mode 100644 contrib/python/Twisted/py3/twisted/application/runner/_runner.py create mode 100644 contrib/python/Twisted/py3/twisted/application/service.py create mode 100644 contrib/python/Twisted/py3/twisted/application/strports.py create mode 100644 contrib/python/Twisted/py3/twisted/application/twist/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/application/twist/_options.py create mode 100644 contrib/python/Twisted/py3/twisted/application/twist/_twist.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/avatar.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/checkers.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/client/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/client/agent.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/client/connect.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/client/default.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/client/direct.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/client/knownhosts.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/client/options.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/endpoints.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/error.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/insults/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/insults/helper.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/insults/insults.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/insults/text.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/insults/window.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/interfaces.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ls.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/manhole.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/manhole_ssh.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/manhole_tap.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/mixin.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/newsfragments/.gitignore create mode 100644 contrib/python/Twisted/py3/twisted/conch/openssh_compat/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/openssh_compat/factory.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/openssh_compat/primes.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/recvline.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/scripts/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/scripts/cftp.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/scripts/ckeygen.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/scripts/conch.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/scripts/tkconch.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/_kex.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/address.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/agent.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/channel.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/common.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/connection.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/factory.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/filetransfer.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/forwarding.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/keys.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/service.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/session.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/sexpy.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/transport.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ssh/userauth.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/stdio.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/tap.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/telnet.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ttymodes.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ui/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ui/ansi.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/ui/tkvt100.py create mode 100644 contrib/python/Twisted/py3/twisted/conch/unix.py create mode 100644 contrib/python/Twisted/py3/twisted/copyright.py create mode 100644 contrib/python/Twisted/py3/twisted/cred/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/cred/_digest.py create mode 100644 contrib/python/Twisted/py3/twisted/cred/checkers.py create mode 100644 contrib/python/Twisted/py3/twisted/cred/credentials.py create mode 100644 contrib/python/Twisted/py3/twisted/cred/error.py create mode 100644 contrib/python/Twisted/py3/twisted/cred/portal.py create mode 100644 contrib/python/Twisted/py3/twisted/cred/strcred.py create mode 100644 contrib/python/Twisted/py3/twisted/enterprise/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/enterprise/adbapi.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_baseprocess.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_deprecate.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_dumbwin32proc.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_glibbase.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_idna.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_newtls.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_pollingfile.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_posixserialport.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_posixstdio.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_producer_helpers.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_resolver.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_signals.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_sslverify.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_threadedselect.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_win32serialport.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/_win32stdio.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/abstract.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/address.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/asyncioreactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/base.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/cfreactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/default.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/defer.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/endpoints.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/epollreactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/error.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/fdesc.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/gireactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/glib2reactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/gtk2reactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/gtk3reactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/inotify.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/interfaces.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/abstract.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/const.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/interfaces.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/iocpsupport.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/notes.txt create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/reactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/tcp.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/iocpreactor/udp.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/kqreactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/main.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/pollreactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/posixbase.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/process.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/protocol.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/pyuisupport.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/reactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/selectreactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/serialport.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/ssl.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/stdio.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/task.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/tcp.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/testing.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/threads.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/tksupport.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/udp.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/unix.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/utils.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/win32eventreactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/wxreactor.py create mode 100644 contrib/python/Twisted/py3/twisted/internet/wxsupport.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_buffer.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_capture.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_file.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_filter.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_flatten.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_format.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_global.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_interfaces.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_io.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_json.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_legacy.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_levels.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_logger.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_observer.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_stdlib.py create mode 100644 contrib/python/Twisted/py3/twisted/logger/_util.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/_cred.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/_except.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/_pop3client.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/alias.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/bounce.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/imap4.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/interfaces.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/mail.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/maildir.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/newsfragments/.gitignore create mode 100644 contrib/python/Twisted/py3/twisted/mail/pb.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/pop3.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/pop3client.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/protocols.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/relay.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/relaymanager.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/scripts/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/scripts/mailmail.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/smtp.py create mode 100644 contrib/python/Twisted/py3/twisted/mail/tap.py create mode 100644 contrib/python/Twisted/py3/twisted/names/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/names/_rfc1982.py create mode 100644 contrib/python/Twisted/py3/twisted/names/authority.py create mode 100644 contrib/python/Twisted/py3/twisted/names/cache.py create mode 100644 contrib/python/Twisted/py3/twisted/names/client.py create mode 100644 contrib/python/Twisted/py3/twisted/names/common.py create mode 100644 contrib/python/Twisted/py3/twisted/names/dns.py create mode 100644 contrib/python/Twisted/py3/twisted/names/error.py create mode 100644 contrib/python/Twisted/py3/twisted/names/hosts.py create mode 100644 contrib/python/Twisted/py3/twisted/names/newsfragments/.gitignore create mode 100644 contrib/python/Twisted/py3/twisted/names/resolve.py create mode 100644 contrib/python/Twisted/py3/twisted/names/root.py create mode 100644 contrib/python/Twisted/py3/twisted/names/secondary.py create mode 100644 contrib/python/Twisted/py3/twisted/names/server.py create mode 100644 contrib/python/Twisted/py3/twisted/names/srvconnect.py create mode 100644 contrib/python/Twisted/py3/twisted/names/tap.py create mode 100644 contrib/python/Twisted/py3/twisted/newsfragments/.gitignore create mode 100644 contrib/python/Twisted/py3/twisted/pair/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/pair/ethernet.py create mode 100644 contrib/python/Twisted/py3/twisted/pair/ip.py create mode 100644 contrib/python/Twisted/py3/twisted/pair/raw.py create mode 100644 contrib/python/Twisted/py3/twisted/pair/rawudp.py create mode 100644 contrib/python/Twisted/py3/twisted/pair/testing.py create mode 100644 contrib/python/Twisted/py3/twisted/pair/tuntap.py create mode 100644 contrib/python/Twisted/py3/twisted/persisted/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/persisted/_token.py create mode 100644 contrib/python/Twisted/py3/twisted/persisted/_tokenize.py create mode 100644 contrib/python/Twisted/py3/twisted/persisted/aot.py create mode 100644 contrib/python/Twisted/py3/twisted/persisted/crefutil.py create mode 100644 contrib/python/Twisted/py3/twisted/persisted/dirdbm.py create mode 100644 contrib/python/Twisted/py3/twisted/persisted/newsfragments/9831.misc create mode 100644 contrib/python/Twisted/py3/twisted/persisted/sob.py create mode 100644 contrib/python/Twisted/py3/twisted/persisted/styles.py create mode 100644 contrib/python/Twisted/py3/twisted/plugin.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/cred_anonymous.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/cred_file.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/cred_memory.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/cred_sshkeys.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/cred_unix.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_conch.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_core.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_ftp.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_inet.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_mail.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_names.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_portforward.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_reactors.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_runner.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_socks.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_trial.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_web.py create mode 100644 contrib/python/Twisted/py3/twisted/plugins/twisted_words.py create mode 100644 contrib/python/Twisted/py3/twisted/positioning/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/positioning/_sentence.py create mode 100644 contrib/python/Twisted/py3/twisted/positioning/base.py create mode 100644 contrib/python/Twisted/py3/twisted/positioning/ipositioning.py create mode 100644 contrib/python/Twisted/py3/twisted/positioning/nmea.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/amp.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/basic.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/finger.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/ftp.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/haproxy/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/haproxy/_exceptions.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/haproxy/_info.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/haproxy/_interfaces.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/haproxy/_parser.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/haproxy/_v1parser.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/haproxy/_v2parser.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/haproxy/_wrapper.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/htb.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/ident.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/loopback.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/memcache.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/pcp.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/policies.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/portforward.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/postfix.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/shoutcast.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/sip.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/socks.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/stateful.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/tls.py create mode 100644 contrib/python/Twisted/py3/twisted/protocols/wire.py create mode 100644 contrib/python/Twisted/py3/twisted/py.typed create mode 100644 contrib/python/Twisted/py3/twisted/python/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/python/_appdirs.py create mode 100644 contrib/python/Twisted/py3/twisted/python/_inotify.py create mode 100644 contrib/python/Twisted/py3/twisted/python/_pydoctortemplates/subheader.html create mode 100644 contrib/python/Twisted/py3/twisted/python/_release.py create mode 100644 contrib/python/Twisted/py3/twisted/python/_shellcomp.py create mode 100644 contrib/python/Twisted/py3/twisted/python/_textattributes.py create mode 100644 contrib/python/Twisted/py3/twisted/python/_tzhelper.py create mode 100644 contrib/python/Twisted/py3/twisted/python/_url.py create mode 100644 contrib/python/Twisted/py3/twisted/python/compat.py create mode 100644 contrib/python/Twisted/py3/twisted/python/components.py create mode 100644 contrib/python/Twisted/py3/twisted/python/constants.py create mode 100644 contrib/python/Twisted/py3/twisted/python/context.py create mode 100644 contrib/python/Twisted/py3/twisted/python/deprecate.py create mode 100644 contrib/python/Twisted/py3/twisted/python/failure.py create mode 100644 contrib/python/Twisted/py3/twisted/python/fakepwd.py create mode 100644 contrib/python/Twisted/py3/twisted/python/filepath.py create mode 100644 contrib/python/Twisted/py3/twisted/python/formmethod.py create mode 100644 contrib/python/Twisted/py3/twisted/python/htmlizer.py create mode 100644 contrib/python/Twisted/py3/twisted/python/lockfile.py create mode 100644 contrib/python/Twisted/py3/twisted/python/log.py create mode 100644 contrib/python/Twisted/py3/twisted/python/logfile.py create mode 100644 contrib/python/Twisted/py3/twisted/python/modules.py create mode 100644 contrib/python/Twisted/py3/twisted/python/monkey.py create mode 100644 contrib/python/Twisted/py3/twisted/python/procutils.py create mode 100644 contrib/python/Twisted/py3/twisted/python/randbytes.py create mode 100644 contrib/python/Twisted/py3/twisted/python/rebuild.py create mode 100644 contrib/python/Twisted/py3/twisted/python/reflect.py create mode 100644 contrib/python/Twisted/py3/twisted/python/release.py create mode 100644 contrib/python/Twisted/py3/twisted/python/roots.py create mode 100644 contrib/python/Twisted/py3/twisted/python/runtime.py create mode 100644 contrib/python/Twisted/py3/twisted/python/sendmsg.py create mode 100644 contrib/python/Twisted/py3/twisted/python/shortcut.py create mode 100644 contrib/python/Twisted/py3/twisted/python/syslog.py create mode 100644 contrib/python/Twisted/py3/twisted/python/systemd.py create mode 100644 contrib/python/Twisted/py3/twisted/python/text.py create mode 100644 contrib/python/Twisted/py3/twisted/python/threadable.py create mode 100644 contrib/python/Twisted/py3/twisted/python/threadpool.py create mode 100644 contrib/python/Twisted/py3/twisted/python/twisted-completion.zsh create mode 100644 contrib/python/Twisted/py3/twisted/python/url.py create mode 100644 contrib/python/Twisted/py3/twisted/python/urlpath.py create mode 100644 contrib/python/Twisted/py3/twisted/python/usage.py create mode 100644 contrib/python/Twisted/py3/twisted/python/util.py create mode 100644 contrib/python/Twisted/py3/twisted/python/versions.py create mode 100644 contrib/python/Twisted/py3/twisted/python/win32.py create mode 100644 contrib/python/Twisted/py3/twisted/python/zippath.py create mode 100644 contrib/python/Twisted/py3/twisted/python/zipstream.py create mode 100644 contrib/python/Twisted/py3/twisted/runner/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/runner/inetd.py create mode 100644 contrib/python/Twisted/py3/twisted/runner/inetdconf.py create mode 100644 contrib/python/Twisted/py3/twisted/runner/inetdtap.py create mode 100644 contrib/python/Twisted/py3/twisted/runner/newsfragments/11681.misc create mode 100644 contrib/python/Twisted/py3/twisted/runner/newsfragments/9657.doc create mode 100644 contrib/python/Twisted/py3/twisted/runner/procmon.py create mode 100644 contrib/python/Twisted/py3/twisted/runner/procmontap.py create mode 100644 contrib/python/Twisted/py3/twisted/scripts/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/scripts/_twistd_unix.py create mode 100644 contrib/python/Twisted/py3/twisted/scripts/_twistw.py create mode 100644 contrib/python/Twisted/py3/twisted/scripts/htmlizer.py create mode 100644 contrib/python/Twisted/py3/twisted/scripts/newsfragments/761.bugfix create mode 100644 contrib/python/Twisted/py3/twisted/scripts/trial.py create mode 100644 contrib/python/Twisted/py3/twisted/scripts/twistd.py create mode 100644 contrib/python/Twisted/py3/twisted/spread/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/spread/banana.py create mode 100644 contrib/python/Twisted/py3/twisted/spread/flavors.py create mode 100644 contrib/python/Twisted/py3/twisted/spread/interfaces.py create mode 100644 contrib/python/Twisted/py3/twisted/spread/jelly.py create mode 100644 contrib/python/Twisted/py3/twisted/spread/pb.py create mode 100644 contrib/python/Twisted/py3/twisted/spread/publish.py create mode 100644 contrib/python/Twisted/py3/twisted/spread/util.py create mode 100644 contrib/python/Twisted/py3/twisted/tap/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/tap/ftp.py create mode 100644 contrib/python/Twisted/py3/twisted/tap/portforward.py create mode 100644 contrib/python/Twisted/py3/twisted/tap/socks.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/__main__.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_asyncrunner.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_asynctest.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/distreporter.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/disttrial.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/functional.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/managercommands.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/options.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/stream.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/worker.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/workercommands.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/workerreporter.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_dist/workertrial.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/_synctest.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/itrial.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/newsfragments/.gitignore create mode 100644 contrib/python/Twisted/py3/twisted/trial/reporter.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/runner.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/unittest.py create mode 100644 contrib/python/Twisted/py3/twisted/trial/util.py create mode 100644 contrib/python/Twisted/py3/twisted/web/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_auth/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_auth/basic.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_auth/digest.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_auth/wrapper.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_element.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_flatten.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_http2.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_newclient.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_responses.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_stan.py create mode 100644 contrib/python/Twisted/py3/twisted/web/_template_util.py create mode 100644 contrib/python/Twisted/py3/twisted/web/client.py create mode 100644 contrib/python/Twisted/py3/twisted/web/demo.py create mode 100644 contrib/python/Twisted/py3/twisted/web/distrib.py create mode 100644 contrib/python/Twisted/py3/twisted/web/domhelpers.py create mode 100644 contrib/python/Twisted/py3/twisted/web/error.py create mode 100644 contrib/python/Twisted/py3/twisted/web/guard.py create mode 100644 contrib/python/Twisted/py3/twisted/web/html.py create mode 100644 contrib/python/Twisted/py3/twisted/web/http.py create mode 100644 contrib/python/Twisted/py3/twisted/web/http_headers.py create mode 100644 contrib/python/Twisted/py3/twisted/web/iweb.py create mode 100644 contrib/python/Twisted/py3/twisted/web/microdom.py create mode 100644 contrib/python/Twisted/py3/twisted/web/newsfragments/.gitignore create mode 100644 contrib/python/Twisted/py3/twisted/web/pages.py create mode 100644 contrib/python/Twisted/py3/twisted/web/proxy.py create mode 100644 contrib/python/Twisted/py3/twisted/web/resource.py create mode 100644 contrib/python/Twisted/py3/twisted/web/rewrite.py create mode 100644 contrib/python/Twisted/py3/twisted/web/script.py create mode 100644 contrib/python/Twisted/py3/twisted/web/server.py create mode 100644 contrib/python/Twisted/py3/twisted/web/soap.py create mode 100644 contrib/python/Twisted/py3/twisted/web/static.py create mode 100644 contrib/python/Twisted/py3/twisted/web/sux.py create mode 100644 contrib/python/Twisted/py3/twisted/web/tap.py create mode 100644 contrib/python/Twisted/py3/twisted/web/template.py create mode 100644 contrib/python/Twisted/py3/twisted/web/test/requesthelper.py create mode 100644 contrib/python/Twisted/py3/twisted/web/twcgi.py create mode 100644 contrib/python/Twisted/py3/twisted/web/util.py create mode 100644 contrib/python/Twisted/py3/twisted/web/vhost.py create mode 100644 contrib/python/Twisted/py3/twisted/web/wsgi.py create mode 100644 contrib/python/Twisted/py3/twisted/web/xmlrpc.py create mode 100644 contrib/python/Twisted/py3/twisted/words/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/words/ewords.py create mode 100644 contrib/python/Twisted/py3/twisted/words/im/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/words/im/baseaccount.py create mode 100644 contrib/python/Twisted/py3/twisted/words/im/basechat.py create mode 100644 contrib/python/Twisted/py3/twisted/words/im/basesupport.py create mode 100644 contrib/python/Twisted/py3/twisted/words/im/instancemessenger.glade create mode 100644 contrib/python/Twisted/py3/twisted/words/im/interfaces.py create mode 100644 contrib/python/Twisted/py3/twisted/words/im/ircsupport.py create mode 100644 contrib/python/Twisted/py3/twisted/words/im/locals.py create mode 100644 contrib/python/Twisted/py3/twisted/words/im/pbsupport.py create mode 100644 contrib/python/Twisted/py3/twisted/words/iwords.py create mode 100644 contrib/python/Twisted/py3/twisted/words/newsfragments/.gitignore create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/irc.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/client.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/component.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/error.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/ijabber.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/jid.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/jstrports.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/sasl.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/sasl_mechanisms.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/xmlstream.py create mode 100644 contrib/python/Twisted/py3/twisted/words/protocols/jabber/xmpp_stringprep.py create mode 100644 contrib/python/Twisted/py3/twisted/words/service.py create mode 100644 contrib/python/Twisted/py3/twisted/words/tap.py create mode 100644 contrib/python/Twisted/py3/twisted/words/xish/__init__.py create mode 100644 contrib/python/Twisted/py3/twisted/words/xish/domish.py create mode 100644 contrib/python/Twisted/py3/twisted/words/xish/utility.py create mode 100644 contrib/python/Twisted/py3/twisted/words/xish/xmlstream.py create mode 100644 contrib/python/Twisted/py3/twisted/words/xish/xpath.py create mode 100644 contrib/python/Twisted/py3/twisted/words/xish/xpathparser.g create mode 100644 contrib/python/Twisted/py3/twisted/words/xish/xpathparser.py create mode 100644 contrib/python/Twisted/py3/twisted/words/xmpproutertap.py create mode 100644 contrib/python/Twisted/py3/ya.make create mode 100644 contrib/python/Twisted/ya.make create mode 100644 contrib/python/chardet/py2/test.py create mode 100644 contrib/python/chardet/py2/tests/ya.make create mode 100644 contrib/python/chardet/py3/.dist-info/METADATA create mode 100644 contrib/python/chardet/py3/.dist-info/entry_points.txt create mode 100644 contrib/python/chardet/py3/.dist-info/top_level.txt create mode 100644 contrib/python/chardet/py3/chardet/__init__.py create mode 100644 contrib/python/chardet/py3/chardet/__main__.py create mode 100644 contrib/python/chardet/py3/chardet/big5freq.py create mode 100644 contrib/python/chardet/py3/chardet/big5prober.py create mode 100644 contrib/python/chardet/py3/chardet/chardistribution.py create mode 100644 contrib/python/chardet/py3/chardet/charsetgroupprober.py create mode 100644 contrib/python/chardet/py3/chardet/charsetprober.py create mode 100644 contrib/python/chardet/py3/chardet/cli/__init__.py create mode 100644 contrib/python/chardet/py3/chardet/cli/chardetect.py create mode 100644 contrib/python/chardet/py3/chardet/codingstatemachine.py create mode 100644 contrib/python/chardet/py3/chardet/codingstatemachinedict.py create mode 100644 contrib/python/chardet/py3/chardet/cp949prober.py create mode 100644 contrib/python/chardet/py3/chardet/enums.py create mode 100644 contrib/python/chardet/py3/chardet/escprober.py create mode 100644 contrib/python/chardet/py3/chardet/escsm.py create mode 100644 contrib/python/chardet/py3/chardet/eucjpprober.py create mode 100644 contrib/python/chardet/py3/chardet/euckrfreq.py create mode 100644 contrib/python/chardet/py3/chardet/euckrprober.py create mode 100644 contrib/python/chardet/py3/chardet/euctwfreq.py create mode 100644 contrib/python/chardet/py3/chardet/euctwprober.py create mode 100644 contrib/python/chardet/py3/chardet/gb2312freq.py create mode 100644 contrib/python/chardet/py3/chardet/gb2312prober.py create mode 100644 contrib/python/chardet/py3/chardet/hebrewprober.py create mode 100644 contrib/python/chardet/py3/chardet/jisfreq.py create mode 100644 contrib/python/chardet/py3/chardet/johabfreq.py create mode 100644 contrib/python/chardet/py3/chardet/johabprober.py create mode 100644 contrib/python/chardet/py3/chardet/jpcntx.py create mode 100644 contrib/python/chardet/py3/chardet/langbulgarianmodel.py create mode 100644 contrib/python/chardet/py3/chardet/langgreekmodel.py create mode 100644 contrib/python/chardet/py3/chardet/langhebrewmodel.py create mode 100644 contrib/python/chardet/py3/chardet/langhungarianmodel.py create mode 100644 contrib/python/chardet/py3/chardet/langrussianmodel.py create mode 100644 contrib/python/chardet/py3/chardet/langthaimodel.py create mode 100644 contrib/python/chardet/py3/chardet/langturkishmodel.py create mode 100644 contrib/python/chardet/py3/chardet/latin1prober.py create mode 100644 contrib/python/chardet/py3/chardet/macromanprober.py create mode 100644 contrib/python/chardet/py3/chardet/mbcharsetprober.py create mode 100644 contrib/python/chardet/py3/chardet/mbcsgroupprober.py create mode 100644 contrib/python/chardet/py3/chardet/mbcssm.py create mode 100644 contrib/python/chardet/py3/chardet/metadata/__init__.py create mode 100644 contrib/python/chardet/py3/chardet/metadata/languages.py create mode 100644 contrib/python/chardet/py3/chardet/py.typed create mode 100644 contrib/python/chardet/py3/chardet/resultdict.py create mode 100644 contrib/python/chardet/py3/chardet/sbcharsetprober.py create mode 100644 contrib/python/chardet/py3/chardet/sbcsgroupprober.py create mode 100644 contrib/python/chardet/py3/chardet/sjisprober.py create mode 100644 contrib/python/chardet/py3/chardet/universaldetector.py create mode 100644 contrib/python/chardet/py3/chardet/utf1632prober.py create mode 100644 contrib/python/chardet/py3/chardet/utf8prober.py create mode 100644 contrib/python/chardet/py3/chardet/version.py create mode 100644 contrib/python/chardet/py3/test.py create mode 100644 contrib/python/chardet/py3/tests/ya.make create mode 100644 contrib/python/chardet/py3/ya.make create mode 100644 contrib/python/constantly/py2/.dist-info/METADATA create mode 100644 contrib/python/constantly/py2/.dist-info/top_level.txt create mode 100644 contrib/python/constantly/py2/LICENSE create mode 100644 contrib/python/constantly/py2/README.rst create mode 100644 contrib/python/constantly/py2/constantly/__init__.py create mode 100644 contrib/python/constantly/py2/constantly/_constants.py create mode 100644 contrib/python/constantly/py2/constantly/_version.py create mode 100644 contrib/python/constantly/py2/ya.make create mode 100644 contrib/python/constantly/py3/.dist-info/METADATA create mode 100644 contrib/python/constantly/py3/.dist-info/top_level.txt create mode 100644 contrib/python/constantly/py3/LICENSE create mode 100644 contrib/python/constantly/py3/README.rst create mode 100644 contrib/python/constantly/py3/constantly/__init__.py create mode 100644 contrib/python/constantly/py3/constantly/_constants.py create mode 100644 contrib/python/constantly/py3/constantly/_version.py create mode 100644 contrib/python/constantly/py3/ya.make create mode 100644 contrib/python/constantly/ya.make create mode 100644 contrib/python/hyperlink/py2/.dist-info/METADATA create mode 100644 contrib/python/hyperlink/py2/.dist-info/top_level.txt create mode 100644 contrib/python/hyperlink/py2/LICENSE create mode 100644 contrib/python/hyperlink/py2/README.md create mode 100644 contrib/python/hyperlink/py2/hyperlink/__init__.py create mode 100644 contrib/python/hyperlink/py2/hyperlink/_socket.py create mode 100644 contrib/python/hyperlink/py2/hyperlink/_url.py create mode 100644 contrib/python/hyperlink/py2/hyperlink/hypothesis.py create mode 100644 contrib/python/hyperlink/py2/hyperlink/idna-tables-properties.csv.gz create mode 100644 contrib/python/hyperlink/py2/hyperlink/py.typed create mode 100644 contrib/python/hyperlink/py2/ya.make create mode 100644 contrib/python/hyperlink/py3/.dist-info/METADATA create mode 100644 contrib/python/hyperlink/py3/.dist-info/top_level.txt create mode 100644 contrib/python/hyperlink/py3/LICENSE create mode 100644 contrib/python/hyperlink/py3/README.md create mode 100644 contrib/python/hyperlink/py3/hyperlink/__init__.py create mode 100644 contrib/python/hyperlink/py3/hyperlink/_socket.py create mode 100644 contrib/python/hyperlink/py3/hyperlink/_url.py create mode 100644 contrib/python/hyperlink/py3/hyperlink/hypothesis.py create mode 100644 contrib/python/hyperlink/py3/hyperlink/idna-tables-properties.csv.gz create mode 100644 contrib/python/hyperlink/py3/hyperlink/py.typed create mode 100644 contrib/python/hyperlink/py3/ya.make create mode 100644 contrib/python/hyperlink/ya.make create mode 100644 contrib/python/incremental/py2/.dist-info/METADATA create mode 100644 contrib/python/incremental/py2/.dist-info/entry_points.txt create mode 100644 contrib/python/incremental/py2/.dist-info/top_level.txt create mode 100644 contrib/python/incremental/py2/LICENSE create mode 100644 contrib/python/incremental/py2/README.rst create mode 100644 contrib/python/incremental/py2/incremental/__init__.py create mode 100644 contrib/python/incremental/py2/incremental/_version.py create mode 100644 contrib/python/incremental/py2/incremental/py.typed create mode 100644 contrib/python/incremental/py2/incremental/update.py create mode 100644 contrib/python/incremental/py2/ya.make create mode 100644 contrib/python/incremental/py3/.dist-info/METADATA create mode 100644 contrib/python/incremental/py3/.dist-info/entry_points.txt create mode 100644 contrib/python/incremental/py3/.dist-info/top_level.txt create mode 100644 contrib/python/incremental/py3/LICENSE create mode 100644 contrib/python/incremental/py3/README.rst create mode 100644 contrib/python/incremental/py3/incremental/__init__.py create mode 100644 contrib/python/incremental/py3/incremental/_version.py create mode 100644 contrib/python/incremental/py3/incremental/py.typed create mode 100644 contrib/python/incremental/py3/incremental/update.py create mode 100644 contrib/python/incremental/py3/ya.make create mode 100644 contrib/python/incremental/ya.make create mode 100644 contrib/python/jsonschema/py2/.dist-info/METADATA create mode 100644 contrib/python/jsonschema/py2/.dist-info/entry_points.txt create mode 100644 contrib/python/jsonschema/py2/.dist-info/top_level.txt create mode 100644 contrib/python/jsonschema/py2/COPYING create mode 100644 contrib/python/jsonschema/py2/README.rst create mode 100644 contrib/python/jsonschema/py2/jsonschema/__init__.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/__main__.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/_format.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/_legacy_validators.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/_reflect.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/_types.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/_utils.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/_validators.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/cli.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/compat.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/exceptions.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/schemas/draft3.json create mode 100644 contrib/python/jsonschema/py2/jsonschema/schemas/draft4.json create mode 100644 contrib/python/jsonschema/py2/jsonschema/schemas/draft6.json create mode 100644 contrib/python/jsonschema/py2/jsonschema/schemas/draft7.json create mode 100644 contrib/python/jsonschema/py2/jsonschema/tests/__init__.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/tests/_helpers.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/tests/test_cli.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/tests/test_exceptions.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/tests/test_format.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/tests/test_types.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/tests/test_validators.py create mode 100644 contrib/python/jsonschema/py2/jsonschema/validators.py create mode 100644 contrib/python/jsonschema/py2/tests/ya.make create mode 100644 contrib/python/jsonschema/py2/ya.make create mode 100644 contrib/python/jsonschema/py3/.dist-info/METADATA create mode 100644 contrib/python/jsonschema/py3/.dist-info/entry_points.txt create mode 100644 contrib/python/jsonschema/py3/.dist-info/top_level.txt create mode 100644 contrib/python/jsonschema/py3/COPYING create mode 100644 contrib/python/jsonschema/py3/README.rst create mode 100644 contrib/python/jsonschema/py3/jsonschema/__init__.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/__main__.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/_format.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/_legacy_validators.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/_reflect.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/_types.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/_utils.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/_validators.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/cli.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/compat.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/exceptions.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/schemas/draft3.json create mode 100644 contrib/python/jsonschema/py3/jsonschema/schemas/draft4.json create mode 100644 contrib/python/jsonschema/py3/jsonschema/schemas/draft6.json create mode 100644 contrib/python/jsonschema/py3/jsonschema/schemas/draft7.json create mode 100644 contrib/python/jsonschema/py3/jsonschema/tests/__init__.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/tests/_helpers.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/tests/test_cli.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/tests/test_exceptions.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/tests/test_format.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/tests/test_types.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/tests/test_validators.py create mode 100644 contrib/python/jsonschema/py3/jsonschema/validators.py create mode 100644 contrib/python/jsonschema/py3/tests/ya.make create mode 100644 contrib/python/jsonschema/py3/ya.make create mode 100644 contrib/python/jsonschema/ya.make create mode 100644 contrib/python/pyOpenSSL/py2/.dist-info/METADATA create mode 100644 contrib/python/pyOpenSSL/py2/.dist-info/top_level.txt create mode 100644 contrib/python/pyOpenSSL/py2/LICENSE create mode 100644 contrib/python/pyOpenSSL/py2/OpenSSL/SSL.py create mode 100644 contrib/python/pyOpenSSL/py2/OpenSSL/__init__.py create mode 100644 contrib/python/pyOpenSSL/py2/OpenSSL/_util.py create mode 100644 contrib/python/pyOpenSSL/py2/OpenSSL/crypto.py create mode 100644 contrib/python/pyOpenSSL/py2/OpenSSL/debug.py create mode 100644 contrib/python/pyOpenSSL/py2/OpenSSL/rand.py create mode 100644 contrib/python/pyOpenSSL/py2/OpenSSL/version.py create mode 100644 contrib/python/pyOpenSSL/py2/README.rst create mode 100644 contrib/python/pyOpenSSL/py2/ya.make create mode 100644 contrib/python/pyOpenSSL/py3/LICENSE create mode 100644 contrib/python/pyOpenSSL/py3/README.rst create mode 100644 contrib/python/pyOpenSSL/ya.make create mode 100644 contrib/python/pyrsistent/py2/.dist-info/METADATA create mode 100644 contrib/python/pyrsistent/py2/.dist-info/top_level.txt create mode 100644 contrib/python/pyrsistent/py2/README.rst create mode 100644 contrib/python/pyrsistent/py2/_pyrsistent_version.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/__init__.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_checked_types.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_compat.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_field_common.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_helpers.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_immutable.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_pbag.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_pclass.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_pdeque.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_plist.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_pmap.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_precord.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_pset.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_pvector.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_toolz.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/_transformations.py create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/py.typed create mode 100644 contrib/python/pyrsistent/py2/pyrsistent/typing.py create mode 100644 contrib/python/pyrsistent/py2/tests/bag_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/checked_map_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/checked_set_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/checked_vector_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/class_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/deque_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/field_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/freeze_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/hypothesis_vector_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/immutable_object_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/list_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/map_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/memory_profiling.py create mode 100644 contrib/python/pyrsistent/py2/tests/record_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/regression_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/set_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/toolz_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/transform_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/vector_test.py create mode 100644 contrib/python/pyrsistent/py2/tests/ya.make create mode 100644 contrib/python/pyrsistent/py2/ya.make create mode 100644 contrib/python/pyrsistent/py3/.dist-info/METADATA create mode 100644 contrib/python/pyrsistent/py3/.dist-info/top_level.txt create mode 100644 contrib/python/pyrsistent/py3/LICENSE.mit create mode 100644 contrib/python/pyrsistent/py3/README.rst create mode 100644 contrib/python/pyrsistent/py3/_pyrsistent_version.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/__init__.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_checked_types.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_field_common.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_helpers.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_immutable.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_pbag.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_pclass.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_pdeque.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_plist.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_pmap.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_precord.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_pset.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_pvector.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_toolz.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/_transformations.py create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/py.typed create mode 100644 contrib/python/pyrsistent/py3/pyrsistent/typing.py create mode 100644 contrib/python/pyrsistent/py3/tests/bag_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/checked_map_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/checked_set_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/checked_vector_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/class_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/deque_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/field_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/freeze_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/hypothesis_vector_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/immutable_object_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/list_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/map_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/memory_profiling.py create mode 100644 contrib/python/pyrsistent/py3/tests/record_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/regression_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/set_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/toolz_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/transform_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/vector_test.py create mode 100644 contrib/python/pyrsistent/py3/tests/ya.make create mode 100644 contrib/python/pyrsistent/py3/ya.make create mode 100644 contrib/python/pyrsistent/ya.make create mode 100644 contrib/python/zope.interface/py2/.dist-info/METADATA create mode 100644 contrib/python/zope.interface/py2/.dist-info/top_level.txt create mode 100644 contrib/python/zope.interface/py2/COPYRIGHT.txt create mode 100644 contrib/python/zope.interface/py2/LICENSE.txt create mode 100644 contrib/python/zope.interface/py2/README.rst create mode 100644 contrib/python/zope.interface/py2/ya.make create mode 100644 contrib/python/zope.interface/py2/zope/interface/__init__.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/_compat.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/_flatten.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/_zope_interface_coptimizations.c create mode 100644 contrib/python/zope.interface/py2/zope/interface/adapter.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/advice.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/__init__.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/builtins.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/collections.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/idatetime.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/interfaces.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/io.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/mapping.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/numbers.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/common/sequence.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/declarations.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/document.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/exceptions.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/interface.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/interfaces.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/registry.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/ro.py create mode 100644 contrib/python/zope.interface/py2/zope/interface/verify.py create mode 100644 contrib/python/zope.interface/py3/.dist-info/METADATA create mode 100644 contrib/python/zope.interface/py3/.dist-info/top_level.txt create mode 100644 contrib/python/zope.interface/py3/COPYRIGHT.txt create mode 100644 contrib/python/zope.interface/py3/LICENSE.txt create mode 100644 contrib/python/zope.interface/py3/README.rst create mode 100644 contrib/python/zope.interface/py3/ya.make create mode 100644 contrib/python/zope.interface/py3/zope/interface/__init__.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/_compat.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/_flatten.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/_zope_interface_coptimizations.c create mode 100644 contrib/python/zope.interface/py3/zope/interface/adapter.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/advice.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/__init__.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/builtins.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/collections.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/idatetime.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/interfaces.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/io.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/mapping.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/numbers.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/common/sequence.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/declarations.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/document.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/exceptions.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/interface.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/interfaces.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/registry.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/ro.py create mode 100644 contrib/python/zope.interface/py3/zope/interface/verify.py create mode 100644 contrib/python/zope.interface/ya.make (limited to 'contrib/python') diff --git a/contrib/python/Automat/py2/.dist-info/METADATA b/contrib/python/Automat/py2/.dist-info/METADATA new file mode 100644 index 00000000000..8b4c9cecf78 --- /dev/null +++ b/contrib/python/Automat/py2/.dist-info/METADATA @@ -0,0 +1,487 @@ +Metadata-Version: 2.1 +Name: Automat +Version: 20.2.0 +Summary: Self-service finite-state machines for the programmer on the go. +Home-page: https://github.com/glyph/Automat +Author: Glyph +Author-email: glyph@twistedmatrix.com +License: MIT +Keywords: fsm finite state machine automata +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Requires-Dist: attrs (>=19.2.0) +Requires-Dist: six +Provides-Extra: visualize +Requires-Dist: graphviz (>0.5.1); extra == 'visualize' +Requires-Dist: Twisted (>=16.1.1); extra == 'visualize' + + +Automat +======= + + +.. image:: https://readthedocs.org/projects/automat/badge/?version=latest + :target: http://automat.readthedocs.io/en/latest/ + :alt: Documentation Status + + +.. image:: https://travis-ci.org/glyph/automat.svg?branch=master + :target: https://travis-ci.org/glyph/automat + :alt: Build Status + + +.. image:: https://coveralls.io/repos/glyph/automat/badge.png + :target: https://coveralls.io/r/glyph/automat + :alt: Coverage Status + + +Self-service finite-state machines for the programmer on the go. +---------------------------------------------------------------- + +Automat is a library for concise, idiomatic Python expression of finite-state +automata (particularly deterministic finite-state transducers). + +Read more here, or on `Read the Docs `_\ , or watch the following videos for an overview and presentation + +Overview and presentation by **Glyph Lefkowitz** at the first talk of the first Pyninsula meetup, on February 21st, 2017: + +.. image:: https://img.youtube.com/vi/0wOZBpD1VVk/0.jpg + :target: https://www.youtube.com/watch?v=0wOZBpD1VVk + :alt: Glyph Lefkowitz - Automat - Pyninsula #0 + + +Presentation by **Clinton Roy** at PyCon Australia, on August 6th 2017: + +.. image:: https://img.youtube.com/vi/TedUKXhu9kE/0.jpg + :target: https://www.youtube.com/watch?v=TedUKXhu9kE + :alt: Clinton Roy - State Machines - Pycon Australia 2017 + + +Why use state machines? +^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes you have to create an object whose behavior varies with its state, +but still wishes to present a consistent interface to its callers. + +For example, let's say you're writing the software for a coffee machine. It +has a lid that can be opened or closed, a chamber for water, a chamber for +coffee beans, and a button for "brew". + +There are a number of possible states for the coffee machine. It might or +might not have water. It might or might not have beans. The lid might be open +or closed. The "brew" button should only actually attempt to brew coffee in +one of these configurations, and the "open lid" button should only work if the +coffee is not, in fact, brewing. + +With diligence and attention to detail, you can implement this correctly using +a collection of attributes on an object; ``has_water``\ , ``has_beans``\ , +``is_lid_open`` and so on. However, you have to keep all these attributes +consistent. As the coffee maker becomes more complex - perhaps you add an +additional chamber for flavorings so you can make hazelnut coffee, for +example - you have to keep adding more and more checks and more and more +reasoning about which combinations of states are allowed. + +Rather than adding tedious 'if' checks to every single method to make sure that +each of these flags are exactly what you expect, you can use a state machine to +ensure that if your code runs at all, it will be run with all the required +values initialized, because they have to be called in the order you declare +them. + +You can read about state machines and their advantages for Python programmers +in considerably more detail +`in this excellent series of articles from ClusterHQ `_. + +What makes Automat different? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are +`dozens of libraries on PyPI implementing state machines `_. +So it behooves me to say why yet another one would be a good idea. + +Automat is designed around this principle: while organizing your code around +state machines is a good idea, your callers don't, and shouldn't have to, care +that you've done so. In Python, the "input" to a stateful system is a method +call; the "output" may be a method call, if you need to invoke a side effect, +or a return value, if you are just performing a computation in memory. Most +other state-machine libraries require you to explicitly create an input object, +provide that object to a generic "input" method, and then receive results, +sometimes in terms of that library's interfaces and sometimes in terms of +classes you define yourself. + +For example, a snippet of the coffee-machine example above might be implemented +as follows in naive Python: + +.. code-block:: python + + class CoffeeMachine(object): + def brew_button(self): + if self.has_water and self.has_beans and not self.is_lid_open: + self.heat_the_heating_element() + # ... + +With Automat, you'd create a class with a ``MethodicalMachine`` attribute: + +.. code-block:: python + + from automat import MethodicalMachine + + class CoffeeBrewer(object): + _machine = MethodicalMachine() + +and then you would break the above logic into two pieces - the ``brew_button`` +*input*\ , declared like so: + +.. code-block:: python + + @_machine.input() + def brew_button(self): + "The user pressed the 'brew' button." + +It wouldn't do any good to declare a method *body* on this, however, because +input methods don't actually execute their bodies when called; doing actual +work is the *output*\ 's job: + +.. code-block:: python + + @_machine.output() + def _heat_the_heating_element(self): + "Heat up the heating element, which should cause coffee to happen." + self._heating_element.turn_on() + +As well as a couple of *states* - and for simplicity's sake let's say that the +only two states are ``have_beans`` and ``dont_have_beans``\ : + +.. code-block:: python + + @_machine.state() + def have_beans(self): + "In this state, you have some beans." + @_machine.state(initial=True) + def dont_have_beans(self): + "In this state, you don't have any beans." + +``dont_have_beans`` is the ``initial`` state because ``CoffeeBrewer`` starts without beans +in it. + +(And another input to put some beans in:) + +.. code-block:: python + + @_machine.input() + def put_in_beans(self): + "The user put in some beans." + +Finally, you hook everything together with the ``upon`` method of the functions +decorated with ``_machine.state``\ : + +.. code-block:: python + + + # When we don't have beans, upon putting in beans, we will then have beans + # (and produce no output) + dont_have_beans.upon(put_in_beans, enter=have_beans, outputs=[]) + + # When we have beans, upon pressing the brew button, we will then not have + # beans any more (as they have been entered into the brewing chamber) and + # our output will be heating the heating element. + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element]) + +To *users* of this coffee machine class though, it still looks like a POPO +(Plain Old Python Object): + +.. code-block:: python + + >>> coffee_machine = CoffeeMachine() + >>> coffee_machine.put_in_beans() + >>> coffee_machine.brew_button() + +All of the *inputs* are provided by calling them like methods, all of the +*outputs* are automatically invoked when they are produced according to the +outputs specified to ``upon`` and all of the states are simply opaque tokens - +although the fact that they're defined as methods like inputs and outputs +allows you to put docstrings on them easily to document them. + +How do I get the current state of a state machine? +-------------------------------------------------- + +Don't do that. + +One major reason for having a state machine is that you want the callers of the +state machine to just provide the appropriate input to the machine at the +appropriate time, and *not have to check themselves* what state the machine is +in. So if you are tempted to write some code like this: + +.. code-block:: python + + if connection_state_machine.state == "CONNECTED": + connection_state_machine.send_message() + else: + print("not connected") + +Instead, just make your calling code do this: + +.. code-block:: python + + connection_state_machine.send_message() + +and then change your state machine to look like this: + +.. code-block:: python + + @_machine.state() + def connected(self): + "connected" + @_machine.state() + def not_connected(self): + "not connected" + @_machine.input() + def send_message(self): + "send a message" + @_machine.output() + def _actually_send_message(self): + self._transport.send(b"message") + @_machine.output() + def _report_sending_failure(self): + print("not connected") + connected.upon(send_message, enter=connected, [_actually_send_message]) + not_connected.upon(send_message, enter=not_connected, [_report_sending_failure]) + +so that the responsibility for knowing which state the state machine is in +remains within the state machine itself. + +Input for Inputs and Output for Outputs +--------------------------------------- + +Quite often you want to be able to pass parameters to your methods, as well as +inspecting their results. For example, when you brew the coffee, you might +expect a cup of coffee to result, and you would like to see what kind of coffee +it is. And if you were to put delicious hand-roasted small-batch artisanal +beans into the machine, you would expect a *better* cup of coffee than if you +were to use mass-produced beans. You would do this in plain old Python by +adding a parameter, so that's how you do it in Automat as well. + +.. code-block:: python + + @_machine.input() + def put_in_beans(self, beans): + "The user put in some beans." + +However, one important difference here is that *we can't add any +implementation code to the input method*. Inputs are purely a declaration of +the interface; the behavior must all come from outputs. Therefore, the change +in the state of the coffee machine must be represented as an output. We can +add an output method like this: + +.. code-block:: python + + @_machine.output() + def _save_beans(self, beans): + "The beans are now in the machine; save them." + self._beans = beans + +and then connect it to the ``put_in_beans`` by changing the transition from +``dont_have_beans`` to ``have_beans`` like so: + +.. code-block:: python + + dont_have_beans.upon(put_in_beans, enter=have_beans, + outputs=[_save_beans]) + +Now, when you call: + +.. code-block:: python + + coffee_machine.put_in_beans("real good beans") + +the machine will remember the beans for later. + +So how do we get the beans back out again? One of our outputs needs to have a +return value. It would make sense if our ``brew_button`` method returned the cup +of coffee that it made, so we should add an output. So, in addition to heating +the heating element, let's add a return value that describes the coffee. First +a new output: + +.. code-block:: python + + @_machine.output() + def _describe_coffee(self): + return "A cup of coffee made with {}.".format(self._beans) + +Note that we don't need to check first whether ``self._beans`` exists or not, +because we can only reach this output method if the state machine says we've +gone through a set of states that sets this attribute. + +Now, we need to hook up ``_describe_coffee`` to the process of brewing, so change +the brewing transition to: + +.. code-block:: python + + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element, + _describe_coffee]) + +Now, we can call it: + +.. code-block:: python + + >>> coffee_machine.brew_button() + [None, 'A cup of coffee made with real good beans.'] + +Except... wait a second, what's that ``None`` doing there? + +Since every input can produce multiple outputs, in automat, the default return +value from every input invocation is a ``list``. In this case, we have both +``_heat_the_heating_element`` and ``_describe_coffee`` outputs, so we're seeing +both of their return values. However, this can be customized, with the +``collector`` argument to ``upon``\ ; the ``collector`` is a callable which takes an +iterable of all the outputs' return values and "collects" a single return value +to return to the caller of the state machine. + +In this case, we only care about the last output, so we can adjust the call to +``upon`` like this: + +.. code-block:: python + + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element, + _describe_coffee], + collector=lambda iterable: list(iterable)[-1] + ) + +And now, we'll get just the return value we want: + +.. code-block:: python + + >>> coffee_machine.brew_button() + 'A cup of coffee made with real good beans.' + +If I can't get the state of the state machine, how can I save it to (a database, an API response, a file on disk...) +-------------------------------------------------------------------------------------------------------------------- + +There are APIs for serializing the state machine. + +First, you have to decide on a persistent representation of each state, via the +``serialized=`` argument to the ``MethodicalMachine.state()`` decorator. + +Let's take this very simple "light switch" state machine, which can be on or +off, and flipped to reverse its state: + +.. code-block:: python + + class LightSwitch(object): + _machine = MethodicalMachine() + @_machine.state(serialized="on") + def on_state(self): + "the switch is on" + @_machine.state(serialized="off", initial=True) + def off_state(self): + "the switch is off" + @_machine.input() + def flip(self): + "flip the switch" + on_state.upon(flip, enter=off_state, outputs=[]) + off_state.upon(flip, enter=on_state, outputs=[]) + +In this case, we've chosen a serialized representation for each state via the +``serialized`` argument. The on state is represented by the string ``"on"``\ , and +the off state is represented by the string ``"off"``. + +Now, let's just add an input that lets us tell if the switch is on or not. + +.. code-block:: python + + @_machine.input() + def query_power(self): + "return True if powered, False otherwise" + @_machine.output() + def _is_powered(self): + return True + @_machine.output() + def _not_powered(self): + return False + on_state.upon(query_power, enter=on_state, outputs=[_is_powered], + collector=next) + off_state.upon(query_power, enter=off_state, outputs=[_not_powered], + collector=next) + +To save the state, we have the ``MethodicalMachine.serializer()`` method. A +method decorated with ``@serializer()`` gets an extra argument injected at the +beginning of its argument list: the serialized identifier for the state. In +this case, either ``"on"`` or ``"off"``. Since state machine output methods can +also affect other state on the object, a serializer method is expected to +return *all* relevant state for serialization. + +For our simple light switch, such a method might look like this: + +.. code-block:: python + + @_machine.serializer() + def save(self, state): + return {"is-it-on": state} + +Serializers can be public methods, and they can return whatever you like. If +necessary, you can have different serializers - just multiple methods decorated +with ``@_machine.serializer()`` - for different formats; return one data-structure +for JSON, one for XML, one for a database row, and so on. + +When it comes time to unserialize, though, you generally want a private method, +because an unserializer has to take a not-fully-initialized instance and +populate it with state. It is expected to *return* the serialized machine +state token that was passed to the serializer, but it can take whatever +arguments you like. Of course, in order to return that, it probably has to +take it somewhere in its arguments, so it will generally take whatever a paired +serializer has returned as an argument. + +So our unserializer would look like this: + +.. code-block:: python + + @_machine.unserializer() + def _restore(self, blob): + return blob["is-it-on"] + +Generally you will want a classmethod deserialization constructor which you +write yourself to call this, so that you know how to create an instance of your +own object, like so: + +.. code-block:: python + + @classmethod + def from_blob(cls, blob): + self = cls() + self._restore(blob) + return self + +Saving and loading our ``LightSwitch`` along with its state-machine state can now +be accomplished as follows: + +.. code-block:: python + + >>> switch1 = LightSwitch() + >>> switch1.query_power() + False + >>> switch1.flip() + [] + >>> switch1.query_power() + True + >>> blob = switch1.save() + >>> switch2 = LightSwitch.from_blob(blob) + >>> switch2.query_power() + True + +More comprehensive (tested, working) examples are present in ``docs/examples``. + +Go forth and machine all the state! + + diff --git a/contrib/python/Automat/py2/.dist-info/entry_points.txt b/contrib/python/Automat/py2/.dist-info/entry_points.txt new file mode 100644 index 00000000000..d79319995c2 --- /dev/null +++ b/contrib/python/Automat/py2/.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +automat-visualize = automat._visualize:tool + diff --git a/contrib/python/Automat/py2/.dist-info/top_level.txt b/contrib/python/Automat/py2/.dist-info/top_level.txt new file mode 100644 index 00000000000..b69387ba180 --- /dev/null +++ b/contrib/python/Automat/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +automat diff --git a/contrib/python/Automat/py2/LICENSE b/contrib/python/Automat/py2/LICENSE new file mode 100644 index 00000000000..9773501b17d --- /dev/null +++ b/contrib/python/Automat/py2/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) 2014 +Rackspace + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/Automat/py2/README.md b/contrib/python/Automat/py2/README.md new file mode 100644 index 00000000000..d830ee66217 --- /dev/null +++ b/contrib/python/Automat/py2/README.md @@ -0,0 +1,430 @@ +# Automat # + +[![Documentation Status](https://readthedocs.org/projects/automat/badge/?version=latest)](http://automat.readthedocs.io/en/latest/) +[![Build Status](https://travis-ci.org/glyph/automat.svg?branch=master)](https://travis-ci.org/glyph/automat) +[![Coverage Status](https://coveralls.io/repos/glyph/automat/badge.png)](https://coveralls.io/r/glyph/automat) + +## Self-service finite-state machines for the programmer on the go. ## + +Automat is a library for concise, idiomatic Python expression of finite-state +automata (particularly deterministic finite-state transducers). + +Read more here, or on [Read the Docs](https://automat.readthedocs.io/), or watch the following videos for an overview and presentation + +Overview and presentation by **Glyph Lefkowitz** at the first talk of the first Pyninsula meetup, on February 21st, 2017: +[![Glyph Lefkowitz - Automat - Pyninsula #0](https://img.youtube.com/vi/0wOZBpD1VVk/0.jpg)](https://www.youtube.com/watch?v=0wOZBpD1VVk) + +Presentation by **Clinton Roy** at PyCon Australia, on August 6th 2017: +[![Clinton Roy - State Machines - Pycon Australia 2017](https://img.youtube.com/vi/TedUKXhu9kE/0.jpg)](https://www.youtube.com/watch?v=TedUKXhu9kE) + +### Why use state machines? ### + +Sometimes you have to create an object whose behavior varies with its state, +but still wishes to present a consistent interface to its callers. + +For example, let's say you're writing the software for a coffee machine. It +has a lid that can be opened or closed, a chamber for water, a chamber for +coffee beans, and a button for "brew". + +There are a number of possible states for the coffee machine. It might or +might not have water. It might or might not have beans. The lid might be open +or closed. The "brew" button should only actually attempt to brew coffee in +one of these configurations, and the "open lid" button should only work if the +coffee is not, in fact, brewing. + +With diligence and attention to detail, you can implement this correctly using +a collection of attributes on an object; `has_water`, `has_beans`, +`is_lid_open` and so on. However, you have to keep all these attributes +consistent. As the coffee maker becomes more complex - perhaps you add an +additional chamber for flavorings so you can make hazelnut coffee, for +example - you have to keep adding more and more checks and more and more +reasoning about which combinations of states are allowed. + +Rather than adding tedious 'if' checks to every single method to make sure that +each of these flags are exactly what you expect, you can use a state machine to +ensure that if your code runs at all, it will be run with all the required +values initialized, because they have to be called in the order you declare +them. + +You can read about state machines and their advantages for Python programmers +in considerably more detail +[in this excellent series of articles from ClusterHQ](https://clusterhq.com/blog/what-is-a-state-machine/). + +### What makes Automat different? ### + +There are +[dozens of libraries on PyPI implementing state machines](https://pypi.org/search/?q=finite+state+machine). +So it behooves me to say why yet another one would be a good idea. + +Automat is designed around this principle: while organizing your code around +state machines is a good idea, your callers don't, and shouldn't have to, care +that you've done so. In Python, the "input" to a stateful system is a method +call; the "output" may be a method call, if you need to invoke a side effect, +or a return value, if you are just performing a computation in memory. Most +other state-machine libraries require you to explicitly create an input object, +provide that object to a generic "input" method, and then receive results, +sometimes in terms of that library's interfaces and sometimes in terms of +classes you define yourself. + +For example, a snippet of the coffee-machine example above might be implemented +as follows in naive Python: + +```python +class CoffeeMachine(object): + def brew_button(self): + if self.has_water and self.has_beans and not self.is_lid_open: + self.heat_the_heating_element() + # ... +``` + +With Automat, you'd create a class with a `MethodicalMachine` attribute: + +```python +from automat import MethodicalMachine + +class CoffeeBrewer(object): + _machine = MethodicalMachine() +``` + +and then you would break the above logic into two pieces - the `brew_button` +*input*, declared like so: + +```python + @_machine.input() + def brew_button(self): + "The user pressed the 'brew' button." +``` + +It wouldn't do any good to declare a method *body* on this, however, because +input methods don't actually execute their bodies when called; doing actual +work is the *output*'s job: + +```python + @_machine.output() + def _heat_the_heating_element(self): + "Heat up the heating element, which should cause coffee to happen." + self._heating_element.turn_on() +``` + +As well as a couple of *states* - and for simplicity's sake let's say that the +only two states are `have_beans` and `dont_have_beans`: + +```python + @_machine.state() + def have_beans(self): + "In this state, you have some beans." + @_machine.state(initial=True) + def dont_have_beans(self): + "In this state, you don't have any beans." +``` + +`dont_have_beans` is the `initial` state because `CoffeeBrewer` starts without beans +in it. + +(And another input to put some beans in:) + +```python + @_machine.input() + def put_in_beans(self): + "The user put in some beans." +``` + +Finally, you hook everything together with the `upon` method of the functions +decorated with `_machine.state`: + +```python + + # When we don't have beans, upon putting in beans, we will then have beans + # (and produce no output) + dont_have_beans.upon(put_in_beans, enter=have_beans, outputs=[]) + + # When we have beans, upon pressing the brew button, we will then not have + # beans any more (as they have been entered into the brewing chamber) and + # our output will be heating the heating element. + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element]) +``` + +To *users* of this coffee machine class though, it still looks like a POPO +(Plain Old Python Object): + +```python +>>> coffee_machine = CoffeeMachine() +>>> coffee_machine.put_in_beans() +>>> coffee_machine.brew_button() +``` + +All of the *inputs* are provided by calling them like methods, all of the +*outputs* are automatically invoked when they are produced according to the +outputs specified to `upon` and all of the states are simply opaque tokens - +although the fact that they're defined as methods like inputs and outputs +allows you to put docstrings on them easily to document them. + +## How do I get the current state of a state machine? + +Don't do that. + +One major reason for having a state machine is that you want the callers of the +state machine to just provide the appropriate input to the machine at the +appropriate time, and *not have to check themselves* what state the machine is +in. So if you are tempted to write some code like this: + +```python +if connection_state_machine.state == "CONNECTED": + connection_state_machine.send_message() +else: + print("not connected") +``` + +Instead, just make your calling code do this: + +```python +connection_state_machine.send_message() +``` + +and then change your state machine to look like this: + +```python + @_machine.state() + def connected(self): + "connected" + @_machine.state() + def not_connected(self): + "not connected" + @_machine.input() + def send_message(self): + "send a message" + @_machine.output() + def _actually_send_message(self): + self._transport.send(b"message") + @_machine.output() + def _report_sending_failure(self): + print("not connected") + connected.upon(send_message, enter=connected, [_actually_send_message]) + not_connected.upon(send_message, enter=not_connected, [_report_sending_failure]) +``` + +so that the responsibility for knowing which state the state machine is in +remains within the state machine itself. + +## Input for Inputs and Output for Outputs + +Quite often you want to be able to pass parameters to your methods, as well as +inspecting their results. For example, when you brew the coffee, you might +expect a cup of coffee to result, and you would like to see what kind of coffee +it is. And if you were to put delicious hand-roasted small-batch artisanal +beans into the machine, you would expect a *better* cup of coffee than if you +were to use mass-produced beans. You would do this in plain old Python by +adding a parameter, so that's how you do it in Automat as well. + +```python + @_machine.input() + def put_in_beans(self, beans): + "The user put in some beans." +``` + +However, one important difference here is that *we can't add any +implementation code to the input method*. Inputs are purely a declaration of +the interface; the behavior must all come from outputs. Therefore, the change +in the state of the coffee machine must be represented as an output. We can +add an output method like this: + +```python + @_machine.output() + def _save_beans(self, beans): + "The beans are now in the machine; save them." + self._beans = beans +``` + +and then connect it to the `put_in_beans` by changing the transition from +`dont_have_beans` to `have_beans` like so: + +```python + dont_have_beans.upon(put_in_beans, enter=have_beans, + outputs=[_save_beans]) +``` + +Now, when you call: + +```python +coffee_machine.put_in_beans("real good beans") +``` + +the machine will remember the beans for later. + +So how do we get the beans back out again? One of our outputs needs to have a +return value. It would make sense if our `brew_button` method returned the cup +of coffee that it made, so we should add an output. So, in addition to heating +the heating element, let's add a return value that describes the coffee. First +a new output: + +```python + @_machine.output() + def _describe_coffee(self): + return "A cup of coffee made with {}.".format(self._beans) +``` + +Note that we don't need to check first whether `self._beans` exists or not, +because we can only reach this output method if the state machine says we've +gone through a set of states that sets this attribute. + +Now, we need to hook up `_describe_coffee` to the process of brewing, so change +the brewing transition to: + +```python + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element, + _describe_coffee]) +``` + +Now, we can call it: + +```python +>>> coffee_machine.brew_button() +[None, 'A cup of coffee made with real good beans.'] +``` + +Except... wait a second, what's that `None` doing there? + +Since every input can produce multiple outputs, in automat, the default return +value from every input invocation is a `list`. In this case, we have both +`_heat_the_heating_element` and `_describe_coffee` outputs, so we're seeing +both of their return values. However, this can be customized, with the +`collector` argument to `upon`; the `collector` is a callable which takes an +iterable of all the outputs' return values and "collects" a single return value +to return to the caller of the state machine. + +In this case, we only care about the last output, so we can adjust the call to +`upon` like this: + +```python + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element, + _describe_coffee], + collector=lambda iterable: list(iterable)[-1] + ) +``` + +And now, we'll get just the return value we want: + +```python +>>> coffee_machine.brew_button() +'A cup of coffee made with real good beans.' +``` + +## If I can't get the state of the state machine, how can I save it to (a database, an API response, a file on disk...) + +There are APIs for serializing the state machine. + +First, you have to decide on a persistent representation of each state, via the +`serialized=` argument to the `MethodicalMachine.state()` decorator. + +Let's take this very simple "light switch" state machine, which can be on or +off, and flipped to reverse its state: + +```python +class LightSwitch(object): + _machine = MethodicalMachine() + @_machine.state(serialized="on") + def on_state(self): + "the switch is on" + @_machine.state(serialized="off", initial=True) + def off_state(self): + "the switch is off" + @_machine.input() + def flip(self): + "flip the switch" + on_state.upon(flip, enter=off_state, outputs=[]) + off_state.upon(flip, enter=on_state, outputs=[]) +``` + +In this case, we've chosen a serialized representation for each state via the +`serialized` argument. The on state is represented by the string `"on"`, and +the off state is represented by the string `"off"`. + +Now, let's just add an input that lets us tell if the switch is on or not. + +```python + @_machine.input() + def query_power(self): + "return True if powered, False otherwise" + @_machine.output() + def _is_powered(self): + return True + @_machine.output() + def _not_powered(self): + return False + on_state.upon(query_power, enter=on_state, outputs=[_is_powered], + collector=next) + off_state.upon(query_power, enter=off_state, outputs=[_not_powered], + collector=next) +``` + +To save the state, we have the `MethodicalMachine.serializer()` method. A +method decorated with `@serializer()` gets an extra argument injected at the +beginning of its argument list: the serialized identifier for the state. In +this case, either `"on"` or `"off"`. Since state machine output methods can +also affect other state on the object, a serializer method is expected to +return *all* relevant state for serialization. + +For our simple light switch, such a method might look like this: + +```python + @_machine.serializer() + def save(self, state): + return {"is-it-on": state} +``` + +Serializers can be public methods, and they can return whatever you like. If +necessary, you can have different serializers - just multiple methods decorated +with `@_machine.serializer()` - for different formats; return one data-structure +for JSON, one for XML, one for a database row, and so on. + +When it comes time to unserialize, though, you generally want a private method, +because an unserializer has to take a not-fully-initialized instance and +populate it with state. It is expected to *return* the serialized machine +state token that was passed to the serializer, but it can take whatever +arguments you like. Of course, in order to return that, it probably has to +take it somewhere in its arguments, so it will generally take whatever a paired +serializer has returned as an argument. + +So our unserializer would look like this: + +```python + @_machine.unserializer() + def _restore(self, blob): + return blob["is-it-on"] +``` + +Generally you will want a classmethod deserialization constructor which you +write yourself to call this, so that you know how to create an instance of your +own object, like so: + +```python + @classmethod + def from_blob(cls, blob): + self = cls() + self._restore(blob) + return self +``` + +Saving and loading our `LightSwitch` along with its state-machine state can now +be accomplished as follows: + +```python +>>> switch1 = LightSwitch() +>>> switch1.query_power() +False +>>> switch1.flip() +[] +>>> switch1.query_power() +True +>>> blob = switch1.save() +>>> switch2 = LightSwitch.from_blob(blob) +>>> switch2.query_power() +True +``` + +More comprehensive (tested, working) examples are present in `docs/examples`. + +Go forth and machine all the state! diff --git a/contrib/python/Automat/py2/automat/__init__.py b/contrib/python/Automat/py2/automat/__init__.py new file mode 100644 index 00000000000..570b84f9951 --- /dev/null +++ b/contrib/python/Automat/py2/automat/__init__.py @@ -0,0 +1,8 @@ +# -*- test-case-name: automat -*- +from ._methodical import MethodicalMachine +from ._core import NoTransition + +__all__ = [ + 'MethodicalMachine', + 'NoTransition', +] diff --git a/contrib/python/Automat/py2/automat/_core.py b/contrib/python/Automat/py2/automat/_core.py new file mode 100644 index 00000000000..4118a4b070a --- /dev/null +++ b/contrib/python/Automat/py2/automat/_core.py @@ -0,0 +1,165 @@ +# -*- test-case-name: automat._test.test_core -*- + +""" +A core state-machine abstraction. + +Perhaps something that could be replaced with or integrated into machinist. +""" + +from itertools import chain + +_NO_STATE = "" + + +class NoTransition(Exception): + """ + A finite state machine in C{state} has no transition for C{symbol}. + + @param state: the finite state machine's state at the time of the + illegal transition. + + @param symbol: the input symbol for which no transition exists. + """ + + def __init__(self, state, symbol): + self.state = state + self.symbol = symbol + super(Exception, self).__init__( + "no transition for {} in {}".format(symbol, state) + ) + + +class Automaton(object): + """ + A declaration of a finite state machine. + + Note that this is not the machine itself; it is immutable. + """ + + def __init__(self): + """ + Initialize the set of transitions and the initial state. + """ + self._initialState = _NO_STATE + self._transitions = set() + + + @property + def initialState(self): + """ + Return this automaton's initial state. + """ + return self._initialState + + + @initialState.setter + def initialState(self, state): + """ + Set this automaton's initial state. Raises a ValueError if + this automaton already has an initial state. + """ + + if self._initialState is not _NO_STATE: + raise ValueError( + "initial state already set to {}".format(self._initialState)) + + self._initialState = state + + + def addTransition(self, inState, inputSymbol, outState, outputSymbols): + """ + Add the given transition to the outputSymbol. Raise ValueError if + there is already a transition with the same inState and inputSymbol. + """ + # keeping self._transitions in a flat list makes addTransition + # O(n^2), but state machines don't tend to have hundreds of + # transitions. + for (anInState, anInputSymbol, anOutState, _) in self._transitions: + if (anInState == inState and anInputSymbol == inputSymbol): + raise ValueError( + "already have transition from {} via {}".format(inState, inputSymbol)) + self._transitions.add( + (inState, inputSymbol, outState, tuple(outputSymbols)) + ) + + + def allTransitions(self): + """ + All transitions. + """ + return frozenset(self._transitions) + + + def inputAlphabet(self): + """ + The full set of symbols acceptable to this automaton. + """ + return {inputSymbol for (inState, inputSymbol, outState, + outputSymbol) in self._transitions} + + + def outputAlphabet(self): + """ + The full set of symbols which can be produced by this automaton. + """ + return set( + chain.from_iterable( + outputSymbols for + (inState, inputSymbol, outState, outputSymbols) + in self._transitions + ) + ) + + + def states(self): + """ + All valid states; "Q" in the mathematical description of a state + machine. + """ + return frozenset( + chain.from_iterable( + (inState, outState) + for + (inState, inputSymbol, outState, outputSymbol) + in self._transitions + ) + ) + + + def outputForInput(self, inState, inputSymbol): + """ + A 2-tuple of (outState, outputSymbols) for inputSymbol. + """ + for (anInState, anInputSymbol, + outState, outputSymbols) in self._transitions: + if (inState, inputSymbol) == (anInState, anInputSymbol): + return (outState, list(outputSymbols)) + raise NoTransition(state=inState, symbol=inputSymbol) + + +class Transitioner(object): + """ + The combination of a current state and an L{Automaton}. + """ + + def __init__(self, automaton, initialState): + self._automaton = automaton + self._state = initialState + self._tracer = None + + def setTrace(self, tracer): + self._tracer = tracer + + def transition(self, inputSymbol): + """ + Transition between states, returning any outputs. + """ + outState, outputSymbols = self._automaton.outputForInput(self._state, + inputSymbol) + outTracer = None + if self._tracer: + outTracer = self._tracer(self._state._name(), + inputSymbol._name(), + outState._name()) + self._state = outState + return (outputSymbols, outTracer) diff --git a/contrib/python/Automat/py2/automat/_discover.py b/contrib/python/Automat/py2/automat/_discover.py new file mode 100644 index 00000000000..c0d88baea4e --- /dev/null +++ b/contrib/python/Automat/py2/automat/_discover.py @@ -0,0 +1,144 @@ +import collections +import inspect +from automat import MethodicalMachine +from twisted.python.modules import PythonModule, getModule + + +def isOriginalLocation(attr): + """ + Attempt to discover if this appearance of a PythonAttribute + representing a class refers to the module where that class was + defined. + """ + sourceModule = inspect.getmodule(attr.load()) + if sourceModule is None: + return False + + currentModule = attr + while not isinstance(currentModule, PythonModule): + currentModule = currentModule.onObject + + return currentModule.name == sourceModule.__name__ + + +def findMachinesViaWrapper(within): + """ + Recursively yield L{MethodicalMachine}s and their FQPNs within a + L{PythonModule} or a L{twisted.python.modules.PythonAttribute} + wrapper object. + + Note that L{PythonModule}s may refer to packages, as well. + + The discovery heuristic considers L{MethodicalMachine} instances + that are module-level attributes or class-level attributes + accessible from module scope. Machines inside nested classes will + be discovered, but those returned from functions or methods will not be. + + @type within: L{PythonModule} or L{twisted.python.modules.PythonAttribute} + @param within: Where to start the search. + + @return: a generator which yields FQPN, L{MethodicalMachine} pairs. + """ + queue = collections.deque([within]) + visited = set() + + while queue: + attr = queue.pop() + value = attr.load() + + if isinstance(value, MethodicalMachine) and value not in visited: + visited.add(value) + yield attr.name, value + elif (inspect.isclass(value) and isOriginalLocation(attr) and + value not in visited): + visited.add(value) + queue.extendleft(attr.iterAttributes()) + elif isinstance(attr, PythonModule) and value not in visited: + visited.add(value) + queue.extendleft(attr.iterAttributes()) + queue.extendleft(attr.iterModules()) + + +class InvalidFQPN(Exception): + """ + The given FQPN was not a dot-separated list of Python objects. + """ + + +class NoModule(InvalidFQPN): + """ + A prefix of the FQPN was not an importable module or package. + """ + + +class NoObject(InvalidFQPN): + """ + A suffix of the FQPN was not an accessible object + """ + + +def wrapFQPN(fqpn): + """ + Given an FQPN, retrieve the object via the global Python module + namespace and wrap it with a L{PythonModule} or a + L{twisted.python.modules.PythonAttribute}. + """ + # largely cribbed from t.p.reflect.namedAny + + if not fqpn: + raise InvalidFQPN("FQPN was empty") + + components = collections.deque(fqpn.split('.')) + + if '' in components: + raise InvalidFQPN( + "name must be a string giving a '.'-separated list of Python " + "identifiers, not %r" % (fqpn,)) + + component = components.popleft() + try: + module = getModule(component) + except KeyError: + raise NoModule(component) + + # find the bottom-most module + while components: + component = components.popleft() + try: + module = module[component] + except KeyError: + components.appendleft(component) + break + else: + module.load() + else: + return module + + # find the bottom-most attribute + attribute = module + for component in components: + try: + attribute = next(child for child in attribute.iterAttributes() + if child.name.rsplit('.', 1)[-1] == component) + except StopIteration: + raise NoObject('{}.{}'.format(attribute.name, component)) + + return attribute + + +def findMachines(fqpn): + """ + Recursively yield L{MethodicalMachine}s and their FQPNs in and + under the a Python object specified by an FQPN. + + The discovery heuristic considers L{MethodicalMachine} instances + that are module-level attributes or class-level attributes + accessible from module scope. Machines inside nested classes will + be discovered, but those returned from functions or methods will not be. + + @type within: an FQPN + @param within: Where to start the search. + + @return: a generator which yields FQPN, L{MethodicalMachine} pairs. + """ + return findMachinesViaWrapper(wrapFQPN(fqpn)) diff --git a/contrib/python/Automat/py2/automat/_introspection.py b/contrib/python/Automat/py2/automat/_introspection.py new file mode 100644 index 00000000000..3f7307d8df9 --- /dev/null +++ b/contrib/python/Automat/py2/automat/_introspection.py @@ -0,0 +1,45 @@ +""" +Python introspection helpers. +""" + +from types import CodeType as code, FunctionType as function + + +def copycode(template, changes): + names = [ + "argcount", "nlocals", "stacksize", "flags", "code", "consts", + "names", "varnames", "filename", "name", "firstlineno", "lnotab", + "freevars", "cellvars" + ] + if hasattr(code, "co_kwonlyargcount"): + names.insert(1, "kwonlyargcount") + if hasattr(code, "co_posonlyargcount"): + # PEP 570 added "positional only arguments" + names.insert(1, "posonlyargcount") + values = [ + changes.get(name, getattr(template, "co_" + name)) + for name in names + ] + return code(*values) + + + +def copyfunction(template, funcchanges, codechanges): + names = [ + "globals", "name", "defaults", "closure", + ] + values = [ + funcchanges.get(name, getattr(template, "__" + name + "__")) + for name in names + ] + return function(copycode(template.__code__, codechanges), *values) + + +def preserveName(f): + """ + Preserve the name of the given function on the decorated function. + """ + def decorator(decorated): + return copyfunction(decorated, + dict(name=f.__name__), dict(name=f.__name__)) + return decorator diff --git a/contrib/python/Automat/py2/automat/_methodical.py b/contrib/python/Automat/py2/automat/_methodical.py new file mode 100644 index 00000000000..84fcd362a6d --- /dev/null +++ b/contrib/python/Automat/py2/automat/_methodical.py @@ -0,0 +1,474 @@ +# -*- test-case-name: automat._test.test_methodical -*- + +import collections +from functools import wraps +from itertools import count + +try: + # Python 3 + from inspect import getfullargspec as getArgsSpec +except ImportError: + # Python 2 + from inspect import getargspec as getArgsSpec + +import attr +import six + +from ._core import Transitioner, Automaton +from ._introspection import preserveName + + +ArgSpec = collections.namedtuple('ArgSpec', ['args', 'varargs', 'varkw', + 'defaults', 'kwonlyargs', + 'kwonlydefaults', 'annotations']) + + +def _getArgSpec(func): + """ + Normalize inspect.ArgSpec across python versions + and convert mutable attributes to immutable types. + + :param Callable func: A function. + :return: The function's ArgSpec. + :rtype: ArgSpec + """ + spec = getArgsSpec(func) + return ArgSpec( + args=tuple(spec.args), + varargs=spec.varargs, + varkw=spec.varkw if six.PY3 else spec.keywords, + defaults=spec.defaults if spec.defaults else (), + kwonlyargs=tuple(spec.kwonlyargs) if six.PY3 else (), + kwonlydefaults=( + tuple(spec.kwonlydefaults.items()) + if spec.kwonlydefaults else () + ) if six.PY3 else (), + annotations=tuple(spec.annotations.items()) if six.PY3 else (), + ) + + +def _getArgNames(spec): + """ + Get the name of all arguments defined in a function signature. + + The name of * and ** arguments is normalized to "*args" and "**kwargs". + + :param ArgSpec spec: A function to interrogate for a signature. + :return: The set of all argument names in `func`s signature. + :rtype: Set[str] + """ + return set( + spec.args + + spec.kwonlyargs + + (('*args',) if spec.varargs else ()) + + (('**kwargs',) if spec.varkw else ()) + + spec.annotations + ) + + +def _keywords_only(f): + """ + Decorate a function so all its arguments must be passed by keyword. + + A useful utility for decorators that take arguments so that they don't + accidentally get passed the thing they're decorating as their first + argument. + + Only works for methods right now. + """ + @wraps(f) + def g(self, **kw): + return f(self, **kw) + return g + + +@attr.s(frozen=True) +class MethodicalState(object): + """ + A state for a L{MethodicalMachine}. + """ + machine = attr.ib(repr=False) + method = attr.ib() + serialized = attr.ib(repr=False) + + def upon(self, input, enter, outputs, collector=list): + """ + Declare a state transition within the :class:`automat.MethodicalMachine` + associated with this :class:`automat.MethodicalState`: + upon the receipt of the `input`, enter the `state`, + emitting each output in `outputs`. + + :param MethodicalInput input: The input triggering a state transition. + :param MethodicalState enter: The resulting state. + :param Iterable[MethodicalOutput] outputs: The outputs to be triggered + as a result of the declared state transition. + :param Callable collector: The function to be used when collecting + output return values. + + :raises TypeError: if any of the `outputs` signatures do not match + the `inputs` signature. + :raises ValueError: if the state transition from `self` via `input` + has already been defined. + """ + inputArgs = _getArgNames(input.argSpec) + for output in outputs: + outputArgs = _getArgNames(output.argSpec) + if not outputArgs.issubset(inputArgs): + raise TypeError( + "method {input} signature {inputSignature} " + "does not match output {output} " + "signature {outputSignature}".format( + input=input.method.__name__, + output=output.method.__name__, + inputSignature=getArgsSpec(input.method), + outputSignature=getArgsSpec(output.method), + )) + self.machine._oneTransition(self, input, enter, outputs, collector) + + def _name(self): + return self.method.__name__ + + +def _transitionerFromInstance(oself, symbol, automaton): + """ + Get a L{Transitioner} + """ + transitioner = getattr(oself, symbol, None) + if transitioner is None: + transitioner = Transitioner( + automaton, + automaton.initialState, + ) + setattr(oself, symbol, transitioner) + return transitioner + + +def _empty(): + pass + +def _docstring(): + """docstring""" + +def assertNoCode(inst, attribute, f): + # The function body must be empty, i.e. "pass" or "return None", which + # both yield the same bytecode: LOAD_CONST (None), RETURN_VALUE. We also + # accept functions with only a docstring, which yields slightly different + # bytecode, because the "None" is put in a different constant slot. + + # Unfortunately, this does not catch function bodies that return a + # constant value, e.g. "return 1", because their code is identical to a + # "return None". They differ in the contents of their constant table, but + # checking that would require us to parse the bytecode, find the index + # being returned, then making sure the table has a None at that index. + + if f.__code__.co_code not in (_empty.__code__.co_code, + _docstring.__code__.co_code): + raise ValueError("function body must be empty") + + +def _filterArgs(args, kwargs, inputSpec, outputSpec): + """ + Filter out arguments that were passed to input that output won't accept. + + :param tuple args: The *args that input received. + :param dict kwargs: The **kwargs that input received. + :param ArgSpec inputSpec: The input's arg spec. + :param ArgSpec outputSpec: The output's arg spec. + :return: The args and kwargs that output will accept. + :rtype: Tuple[tuple, dict] + """ + named_args = tuple(zip(inputSpec.args[1:], args)) + if outputSpec.varargs: + # Only return all args if the output accepts *args. + return_args = args + else: + # Filter out arguments that don't appear + # in the output's method signature. + return_args = [v for n, v in named_args if n in outputSpec.args] + + # Get any of input's default arguments that were not passed. + passed_arg_names = tuple(kwargs) + for name, value in named_args: + passed_arg_names += (name, value) + defaults = zip(inputSpec.args[::-1], inputSpec.defaults[::-1]) + full_kwargs = {n: v for n, v in defaults if n not in passed_arg_names} + full_kwargs.update(kwargs) + + if outputSpec.varkw: + # Only pass all kwargs if the output method accepts **kwargs. + return_kwargs = full_kwargs + else: + # Filter out names that the output method does not accept. + all_accepted_names = outputSpec.args[1:] + outputSpec.kwonlyargs + return_kwargs = {n: v for n, v in full_kwargs.items() + if n in all_accepted_names} + + return return_args, return_kwargs + + +@attr.s(eq=False, hash=False) +class MethodicalInput(object): + """ + An input for a L{MethodicalMachine}. + """ + automaton = attr.ib(repr=False) + method = attr.ib(validator=assertNoCode) + symbol = attr.ib(repr=False) + collectors = attr.ib(default=attr.Factory(dict), repr=False) + argSpec = attr.ib(init=False, repr=False) + + @argSpec.default + def _buildArgSpec(self): + return _getArgSpec(self.method) + + def __get__(self, oself, type=None): + """ + Return a function that takes no arguments and returns values returned + by output functions produced by the given L{MethodicalInput} in + C{oself}'s current state. + """ + transitioner = _transitionerFromInstance(oself, self.symbol, + self.automaton) + @preserveName(self.method) + @wraps(self.method) + def doInput(*args, **kwargs): + self.method(oself, *args, **kwargs) + previousState = transitioner._state + (outputs, outTracer) = transitioner.transition(self) + collector = self.collectors[previousState] + values = [] + for output in outputs: + if outTracer: + outTracer(output._name()) + a, k = _filterArgs(args, kwargs, self.argSpec, output.argSpec) + value = output(oself, *a, **k) + values.append(value) + return collector(values) + return doInput + + def _name(self): + return self.method.__name__ + + +@attr.s(frozen=True) +class MethodicalOutput(object): + """ + An output for a L{MethodicalMachine}. + """ + machine = attr.ib(repr=False) + method = attr.ib() + argSpec = attr.ib(init=False, repr=False) + + @argSpec.default + def _buildArgSpec(self): + return _getArgSpec(self.method) + + def __get__(self, oself, type=None): + """ + Outputs are private, so raise an exception when we attempt to get one. + """ + raise AttributeError( + "{cls}.{method} is a state-machine output method; " + "to produce this output, call an input method instead.".format( + cls=type.__name__, + method=self.method.__name__ + ) + ) + + + def __call__(self, oself, *args, **kwargs): + """ + Call the underlying method. + """ + return self.method(oself, *args, **kwargs) + + def _name(self): + return self.method.__name__ + +@attr.s(eq=False, hash=False) +class MethodicalTracer(object): + automaton = attr.ib(repr=False) + symbol = attr.ib(repr=False) + + + def __get__(self, oself, type=None): + transitioner = _transitionerFromInstance(oself, self.symbol, + self.automaton) + def setTrace(tracer): + transitioner.setTrace(tracer) + return setTrace + + + +counter = count() +def gensym(): + """ + Create a unique Python identifier. + """ + return "_symbol_" + str(next(counter)) + + + +class MethodicalMachine(object): + """ + A :class:`MethodicalMachine` is an interface to an `Automaton` + that uses methods on a class. + """ + + def __init__(self): + self._automaton = Automaton() + self._reducers = {} + self._symbol = gensym() + + + def __get__(self, oself, type=None): + """ + L{MethodicalMachine} is an implementation detail for setting up + class-level state; applications should never need to access it on an + instance. + """ + if oself is not None: + raise AttributeError( + "MethodicalMachine is an implementation detail.") + return self + + + @_keywords_only + def state(self, initial=False, terminal=False, + serialized=None): + """ + Declare a state, possibly an initial state or a terminal state. + + This is a decorator for methods, but it will modify the method so as + not to be callable any more. + + :param bool initial: is this state the initial state? + Only one state on this :class:`automat.MethodicalMachine` + may be an initial state; more than one is an error. + + :param bool terminal: Is this state a terminal state? + i.e. a state that the machine can end up in? + (This is purely informational at this point.) + + :param Hashable serialized: a serializable value + to be used to represent this state to external systems. + This value should be hashable; + :py:func:`unicode` is a good type to use. + """ + def decorator(stateMethod): + state = MethodicalState(machine=self, + method=stateMethod, + serialized=serialized) + if initial: + self._automaton.initialState = state + return state + return decorator + + + @_keywords_only + def input(self): + """ + Declare an input. + + This is a decorator for methods. + """ + def decorator(inputMethod): + return MethodicalInput(automaton=self._automaton, + method=inputMethod, + symbol=self._symbol) + return decorator + + + @_keywords_only + def output(self): + """ + Declare an output. + + This is a decorator for methods. + + This method will be called when the state machine transitions to this + state as specified in the decorated `output` method. + """ + def decorator(outputMethod): + return MethodicalOutput(machine=self, method=outputMethod) + return decorator + + + def _oneTransition(self, startState, inputToken, endState, outputTokens, + collector): + """ + See L{MethodicalState.upon}. + """ + # FIXME: tests for all of this (some of it is wrong) + # if not isinstance(startState, MethodicalState): + # raise NotImplementedError("start state {} isn't a state" + # .format(startState)) + # if not isinstance(inputToken, MethodicalInput): + # raise NotImplementedError("start state {} isn't an input" + # .format(inputToken)) + # if not isinstance(endState, MethodicalState): + # raise NotImplementedError("end state {} isn't a state" + # .format(startState)) + # for output in outputTokens: + # if not isinstance(endState, MethodicalState): + # raise NotImplementedError("output state {} isn't a state" + # .format(endState)) + self._automaton.addTransition(startState, inputToken, endState, + tuple(outputTokens)) + inputToken.collectors[startState] = collector + + + @_keywords_only + def serializer(self): + """ + + """ + def decorator(decoratee): + @wraps(decoratee) + def serialize(oself): + transitioner = _transitionerFromInstance(oself, self._symbol, + self._automaton) + return decoratee(oself, transitioner._state.serialized) + return serialize + return decorator + + @_keywords_only + def unserializer(self): + """ + + """ + def decorator(decoratee): + @wraps(decoratee) + def unserialize(oself, *args, **kwargs): + state = decoratee(oself, *args, **kwargs) + mapping = {} + for eachState in self._automaton.states(): + mapping[eachState.serialized] = eachState + transitioner = _transitionerFromInstance( + oself, self._symbol, self._automaton) + transitioner._state = mapping[state] + return None # it's on purpose + return unserialize + return decorator + + @property + def _setTrace(self): + return MethodicalTracer(self._automaton, self._symbol) + + def asDigraph(self): + """ + Generate a L{graphviz.Digraph} that represents this machine's + states and transitions. + + @return: L{graphviz.Digraph} object; for more information, please + see the documentation for + U{graphviz} + + """ + from ._visualize import makeDigraph + return makeDigraph( + self._automaton, + stateAsString=lambda state: state.method.__name__, + inputAsString=lambda input: input.method.__name__, + outputAsString=lambda output: output.method.__name__, + ) diff --git a/contrib/python/Automat/py2/automat/_visualize.py b/contrib/python/Automat/py2/automat/_visualize.py new file mode 100644 index 00000000000..7a9c8c6eb55 --- /dev/null +++ b/contrib/python/Automat/py2/automat/_visualize.py @@ -0,0 +1,182 @@ +from __future__ import print_function +import argparse +import sys + +import graphviz + +from ._discover import findMachines + + +def _gvquote(s): + return '"{}"'.format(s.replace('"', r'\"')) + + +def _gvhtml(s): + return '<{}>'.format(s) + + +def elementMaker(name, *children, **attrs): + """ + Construct a string from the HTML element description. + """ + formattedAttrs = ' '.join('{}={}'.format(key, _gvquote(str(value))) + for key, value in sorted(attrs.items())) + formattedChildren = ''.join(children) + return u'<{name} {attrs}>{children}'.format( + name=name, + attrs=formattedAttrs, + children=formattedChildren) + + +def tableMaker(inputLabel, outputLabels, port, _E=elementMaker): + """ + Construct an HTML table to label a state transition. + """ + colspan = {} + if outputLabels: + colspan['colspan'] = str(len(outputLabels)) + + inputLabelCell = _E("td", + _E("font", + inputLabel, + face="menlo-italic"), + color="purple", + port=port, + **colspan) + + pointSize = {"point-size": "9"} + outputLabelCells = [_E("td", + _E("font", + outputLabel, + **pointSize), + color="pink") + for outputLabel in outputLabels] + + rows = [_E("tr", inputLabelCell)] + + if outputLabels: + rows.append(_E("tr", *outputLabelCells)) + + return _E("table", *rows) + + +def makeDigraph(automaton, inputAsString=repr, + outputAsString=repr, + stateAsString=repr): + """ + Produce a L{graphviz.Digraph} object from an automaton. + """ + digraph = graphviz.Digraph(graph_attr={'pack': 'true', + 'dpi': '100'}, + node_attr={'fontname': 'Menlo'}, + edge_attr={'fontname': 'Menlo'}) + + for state in automaton.states(): + if state is automaton.initialState: + stateShape = "bold" + fontName = "Menlo-Bold" + else: + stateShape = "" + fontName = "Menlo" + digraph.node(stateAsString(state), + fontame=fontName, + shape="ellipse", + style=stateShape, + color="blue") + for n, eachTransition in enumerate(automaton.allTransitions()): + inState, inputSymbol, outState, outputSymbols = eachTransition + thisTransition = "t{}".format(n) + inputLabel = inputAsString(inputSymbol) + + port = "tableport" + table = tableMaker(inputLabel, [outputAsString(outputSymbol) + for outputSymbol in outputSymbols], + port=port) + + digraph.node(thisTransition, + label=_gvhtml(table), margin="0.2", shape="none") + + digraph.edge(stateAsString(inState), + '{}:{}:w'.format(thisTransition, port), + arrowhead="none") + digraph.edge('{}:{}:e'.format(thisTransition, port), + stateAsString(outState)) + + return digraph + + +def tool(_progname=sys.argv[0], + _argv=sys.argv[1:], + _syspath=sys.path, + _findMachines=findMachines, + _print=print): + """ + Entry point for command line utility. + """ + + DESCRIPTION = """ + Visualize automat.MethodicalMachines as graphviz graphs. + """ + EPILOG = """ + You must have the graphviz tool suite installed. Please visit + http://www.graphviz.org for more information. + """ + if _syspath[0]: + _syspath.insert(0, '') + argumentParser = argparse.ArgumentParser( + prog=_progname, + description=DESCRIPTION, + epilog=EPILOG) + argumentParser.add_argument('fqpn', + help="A Fully Qualified Path name" + " representing where to find machines.") + argumentParser.add_argument('--quiet', '-q', + help="suppress output", + default=False, + action="store_true") + argumentParser.add_argument('--dot-directory', '-d', + help="Where to write out .dot files.", + default=".automat_visualize") + argumentParser.add_argument('--image-directory', '-i', + help="Where to write out image files.", + default=".automat_visualize") + argumentParser.add_argument('--image-type', '-t', + help="The image format.", + choices=graphviz.FORMATS, + default='png') + argumentParser.add_argument('--view', '-v', + help="View rendered graphs with" + " default image viewer", + default=False, + action="store_true") + args = argumentParser.parse_args(_argv) + + explicitlySaveDot = (args.dot_directory + and (not args.image_directory + or args.image_directory != args.dot_directory)) + if args.quiet: + def _print(*args): + pass + + for fqpn, machine in _findMachines(args.fqpn): + _print(fqpn, '...discovered') + + digraph = machine.asDigraph() + + if explicitlySaveDot: + digraph.save(filename="{}.dot".format(fqpn), + directory=args.dot_directory) + _print(fqpn, "...wrote dot into", args.dot_directory) + + if args.image_directory: + deleteDot = not args.dot_directory or explicitlySaveDot + digraph.format = args.image_type + digraph.render(filename="{}.dot".format(fqpn), + directory=args.image_directory, + view=args.view, + cleanup=deleteDot) + if deleteDot: + msg = "...wrote image into" + else: + msg = "...wrote image and dot into" + _print(fqpn, msg, args.image_directory) diff --git a/contrib/python/Automat/py2/ya.make b/contrib/python/Automat/py2/ya.make new file mode 100644 index 00000000000..f7abf294a0a --- /dev/null +++ b/contrib/python/Automat/py2/ya.make @@ -0,0 +1,38 @@ +# Generated by devtools/yamaker (pypi). + +PY2_LIBRARY() + +VERSION(20.2.0) + +LICENSE(MIT) + +PEERDIR( + contrib/python/attrs + contrib/python/six +) + +NO_LINT() + +NO_CHECK_IMPORTS( + automat._discover + automat._visualize +) + +PY_SRCS( + TOP_LEVEL + automat/__init__.py + automat/_core.py + automat/_discover.py + automat/_introspection.py + automat/_methodical.py + automat/_visualize.py +) + +RESOURCE_FILES( + PREFIX contrib/python/Automat/py2/ + .dist-info/METADATA + .dist-info/entry_points.txt + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/Automat/py3/.dist-info/METADATA b/contrib/python/Automat/py3/.dist-info/METADATA new file mode 100644 index 00000000000..1df3dba6c8f --- /dev/null +++ b/contrib/python/Automat/py3/.dist-info/METADATA @@ -0,0 +1,27 @@ +Metadata-Version: 2.1 +Name: Automat +Version: 22.10.0 +Summary: Self-service finite-state machines for the programmer on the go. +Home-page: https://github.com/glyph/Automat +Author: Glyph +Author-email: glyph@twistedmatrix.com +License: MIT +Keywords: fsm finite state machine automata +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +License-File: LICENSE +Requires-Dist: attrs (>=19.2.0) +Requires-Dist: six +Provides-Extra: visualize +Requires-Dist: graphviz (>0.5.1) ; extra == 'visualize' +Requires-Dist: Twisted (>=16.1.1) ; extra == 'visualize' + diff --git a/contrib/python/Automat/py3/.dist-info/entry_points.txt b/contrib/python/Automat/py3/.dist-info/entry_points.txt new file mode 100644 index 00000000000..ec4b91a0f32 --- /dev/null +++ b/contrib/python/Automat/py3/.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +automat-visualize = automat._visualize:tool diff --git a/contrib/python/Automat/py3/.dist-info/top_level.txt b/contrib/python/Automat/py3/.dist-info/top_level.txt new file mode 100644 index 00000000000..b69387ba180 --- /dev/null +++ b/contrib/python/Automat/py3/.dist-info/top_level.txt @@ -0,0 +1 @@ +automat diff --git a/contrib/python/Automat/py3/LICENSE b/contrib/python/Automat/py3/LICENSE new file mode 100644 index 00000000000..9773501b17d --- /dev/null +++ b/contrib/python/Automat/py3/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) 2014 +Rackspace + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/Automat/py3/README.md b/contrib/python/Automat/py3/README.md new file mode 100644 index 00000000000..488a6c4043f --- /dev/null +++ b/contrib/python/Automat/py3/README.md @@ -0,0 +1,430 @@ +# Automat # + +[![Documentation Status](https://readthedocs.org/projects/automat/badge/?version=latest)](http://automat.readthedocs.io/en/latest/) +[![Build Status](https://travis-ci.org/glyph/automat.svg?branch=master)](https://travis-ci.org/glyph/automat) +[![Coverage Status](https://coveralls.io/repos/glyph/automat/badge.png)](https://coveralls.io/r/glyph/automat) + +## Self-service finite-state machines for the programmer on the go. ## + +Automat is a library for concise, idiomatic Python expression of finite-state +automata (particularly deterministic finite-state transducers). + +Read more here, or on [Read the Docs](https://automat.readthedocs.io/), or watch the following videos for an overview and presentation + +Overview and presentation by **Glyph Lefkowitz** at the first talk of the first Pyninsula meetup, on February 21st, 2017: +[![Glyph Lefkowitz - Automat - Pyninsula #0](https://img.youtube.com/vi/0wOZBpD1VVk/0.jpg)](https://www.youtube.com/watch?v=0wOZBpD1VVk) + +Presentation by **Clinton Roy** at PyCon Australia, on August 6th 2017: +[![Clinton Roy - State Machines - Pycon Australia 2017](https://img.youtube.com/vi/TedUKXhu9kE/0.jpg)](https://www.youtube.com/watch?v=TedUKXhu9kE) + +### Why use state machines? ### + +Sometimes you have to create an object whose behavior varies with its state, +but still wishes to present a consistent interface to its callers. + +For example, let's say you're writing the software for a coffee machine. It +has a lid that can be opened or closed, a chamber for water, a chamber for +coffee beans, and a button for "brew". + +There are a number of possible states for the coffee machine. It might or +might not have water. It might or might not have beans. The lid might be open +or closed. The "brew" button should only actually attempt to brew coffee in +one of these configurations, and the "open lid" button should only work if the +coffee is not, in fact, brewing. + +With diligence and attention to detail, you can implement this correctly using +a collection of attributes on an object; `has_water`, `has_beans`, +`is_lid_open` and so on. However, you have to keep all these attributes +consistent. As the coffee maker becomes more complex - perhaps you add an +additional chamber for flavorings so you can make hazelnut coffee, for +example - you have to keep adding more and more checks and more and more +reasoning about which combinations of states are allowed. + +Rather than adding tedious 'if' checks to every single method to make sure that +each of these flags are exactly what you expect, you can use a state machine to +ensure that if your code runs at all, it will be run with all the required +values initialized, because they have to be called in the order you declare +them. + +You can read about state machines and their advantages for Python programmers +in more detail [in this excellent article by Jean-Paul +Calderone](https://web.archive.org/web/20160507053658/https://clusterhq.com/2013/12/05/what-is-a-state-machine/). + +### What makes Automat different? ### + +There are +[dozens of libraries on PyPI implementing state machines](https://pypi.org/search/?q=finite+state+machine). +So it behooves me to say why yet another one would be a good idea. + +Automat is designed around this principle: while organizing your code around +state machines is a good idea, your callers don't, and shouldn't have to, care +that you've done so. In Python, the "input" to a stateful system is a method +call; the "output" may be a method call, if you need to invoke a side effect, +or a return value, if you are just performing a computation in memory. Most +other state-machine libraries require you to explicitly create an input object, +provide that object to a generic "input" method, and then receive results, +sometimes in terms of that library's interfaces and sometimes in terms of +classes you define yourself. + +For example, a snippet of the coffee-machine example above might be implemented +as follows in naive Python: + +```python +class CoffeeMachine(object): + def brew_button(self): + if self.has_water and self.has_beans and not self.is_lid_open: + self.heat_the_heating_element() + # ... +``` + +With Automat, you'd create a class with a `MethodicalMachine` attribute: + +```python +from automat import MethodicalMachine + +class CoffeeBrewer(object): + _machine = MethodicalMachine() +``` + +and then you would break the above logic into two pieces - the `brew_button` +*input*, declared like so: + +```python + @_machine.input() + def brew_button(self): + "The user pressed the 'brew' button." +``` + +It wouldn't do any good to declare a method *body* on this, however, because +input methods don't actually execute their bodies when called; doing actual +work is the *output*'s job: + +```python + @_machine.output() + def _heat_the_heating_element(self): + "Heat up the heating element, which should cause coffee to happen." + self._heating_element.turn_on() +``` + +As well as a couple of *states* - and for simplicity's sake let's say that the +only two states are `have_beans` and `dont_have_beans`: + +```python + @_machine.state() + def have_beans(self): + "In this state, you have some beans." + @_machine.state(initial=True) + def dont_have_beans(self): + "In this state, you don't have any beans." +``` + +`dont_have_beans` is the `initial` state because `CoffeeBrewer` starts without beans +in it. + +(And another input to put some beans in:) + +```python + @_machine.input() + def put_in_beans(self): + "The user put in some beans." +``` + +Finally, you hook everything together with the `upon` method of the functions +decorated with `_machine.state`: + +```python + + # When we don't have beans, upon putting in beans, we will then have beans + # (and produce no output) + dont_have_beans.upon(put_in_beans, enter=have_beans, outputs=[]) + + # When we have beans, upon pressing the brew button, we will then not have + # beans any more (as they have been entered into the brewing chamber) and + # our output will be heating the heating element. + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element]) +``` + +To *users* of this coffee machine class though, it still looks like a POPO +(Plain Old Python Object): + +```python +>>> coffee_machine = CoffeeMachine() +>>> coffee_machine.put_in_beans() +>>> coffee_machine.brew_button() +``` + +All of the *inputs* are provided by calling them like methods, all of the +*outputs* are automatically invoked when they are produced according to the +outputs specified to `upon` and all of the states are simply opaque tokens - +although the fact that they're defined as methods like inputs and outputs +allows you to put docstrings on them easily to document them. + +## How do I get the current state of a state machine? + +Don't do that. + +One major reason for having a state machine is that you want the callers of the +state machine to just provide the appropriate input to the machine at the +appropriate time, and *not have to check themselves* what state the machine is +in. So if you are tempted to write some code like this: + +```python +if connection_state_machine.state == "CONNECTED": + connection_state_machine.send_message() +else: + print("not connected") +``` + +Instead, just make your calling code do this: + +```python +connection_state_machine.send_message() +``` + +and then change your state machine to look like this: + +```python + @_machine.state() + def connected(self): + "connected" + @_machine.state() + def not_connected(self): + "not connected" + @_machine.input() + def send_message(self): + "send a message" + @_machine.output() + def _actually_send_message(self): + self._transport.send(b"message") + @_machine.output() + def _report_sending_failure(self): + print("not connected") + connected.upon(send_message, enter=connected, [_actually_send_message]) + not_connected.upon(send_message, enter=not_connected, [_report_sending_failure]) +``` + +so that the responsibility for knowing which state the state machine is in +remains within the state machine itself. + +## Input for Inputs and Output for Outputs + +Quite often you want to be able to pass parameters to your methods, as well as +inspecting their results. For example, when you brew the coffee, you might +expect a cup of coffee to result, and you would like to see what kind of coffee +it is. And if you were to put delicious hand-roasted small-batch artisanal +beans into the machine, you would expect a *better* cup of coffee than if you +were to use mass-produced beans. You would do this in plain old Python by +adding a parameter, so that's how you do it in Automat as well. + +```python + @_machine.input() + def put_in_beans(self, beans): + "The user put in some beans." +``` + +However, one important difference here is that *we can't add any +implementation code to the input method*. Inputs are purely a declaration of +the interface; the behavior must all come from outputs. Therefore, the change +in the state of the coffee machine must be represented as an output. We can +add an output method like this: + +```python + @_machine.output() + def _save_beans(self, beans): + "The beans are now in the machine; save them." + self._beans = beans +``` + +and then connect it to the `put_in_beans` by changing the transition from +`dont_have_beans` to `have_beans` like so: + +```python + dont_have_beans.upon(put_in_beans, enter=have_beans, + outputs=[_save_beans]) +``` + +Now, when you call: + +```python +coffee_machine.put_in_beans("real good beans") +``` + +the machine will remember the beans for later. + +So how do we get the beans back out again? One of our outputs needs to have a +return value. It would make sense if our `brew_button` method returned the cup +of coffee that it made, so we should add an output. So, in addition to heating +the heating element, let's add a return value that describes the coffee. First +a new output: + +```python + @_machine.output() + def _describe_coffee(self): + return "A cup of coffee made with {}.".format(self._beans) +``` + +Note that we don't need to check first whether `self._beans` exists or not, +because we can only reach this output method if the state machine says we've +gone through a set of states that sets this attribute. + +Now, we need to hook up `_describe_coffee` to the process of brewing, so change +the brewing transition to: + +```python + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element, + _describe_coffee]) +``` + +Now, we can call it: + +```python +>>> coffee_machine.brew_button() +[None, 'A cup of coffee made with real good beans.'] +``` + +Except... wait a second, what's that `None` doing there? + +Since every input can produce multiple outputs, in automat, the default return +value from every input invocation is a `list`. In this case, we have both +`_heat_the_heating_element` and `_describe_coffee` outputs, so we're seeing +both of their return values. However, this can be customized, with the +`collector` argument to `upon`; the `collector` is a callable which takes an +iterable of all the outputs' return values and "collects" a single return value +to return to the caller of the state machine. + +In this case, we only care about the last output, so we can adjust the call to +`upon` like this: + +```python + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element, + _describe_coffee], + collector=lambda iterable: list(iterable)[-1] + ) +``` + +And now, we'll get just the return value we want: + +```python +>>> coffee_machine.brew_button() +'A cup of coffee made with real good beans.' +``` + +## If I can't get the state of the state machine, how can I save it to (a database, an API response, a file on disk...) + +There are APIs for serializing the state machine. + +First, you have to decide on a persistent representation of each state, via the +`serialized=` argument to the `MethodicalMachine.state()` decorator. + +Let's take this very simple "light switch" state machine, which can be on or +off, and flipped to reverse its state: + +```python +class LightSwitch(object): + _machine = MethodicalMachine() + @_machine.state(serialized="on") + def on_state(self): + "the switch is on" + @_machine.state(serialized="off", initial=True) + def off_state(self): + "the switch is off" + @_machine.input() + def flip(self): + "flip the switch" + on_state.upon(flip, enter=off_state, outputs=[]) + off_state.upon(flip, enter=on_state, outputs=[]) +``` + +In this case, we've chosen a serialized representation for each state via the +`serialized` argument. The on state is represented by the string `"on"`, and +the off state is represented by the string `"off"`. + +Now, let's just add an input that lets us tell if the switch is on or not. + +```python + @_machine.input() + def query_power(self): + "return True if powered, False otherwise" + @_machine.output() + def _is_powered(self): + return True + @_machine.output() + def _not_powered(self): + return False + on_state.upon(query_power, enter=on_state, outputs=[_is_powered], + collector=next) + off_state.upon(query_power, enter=off_state, outputs=[_not_powered], + collector=next) +``` + +To save the state, we have the `MethodicalMachine.serializer()` method. A +method decorated with `@serializer()` gets an extra argument injected at the +beginning of its argument list: the serialized identifier for the state. In +this case, either `"on"` or `"off"`. Since state machine output methods can +also affect other state on the object, a serializer method is expected to +return *all* relevant state for serialization. + +For our simple light switch, such a method might look like this: + +```python + @_machine.serializer() + def save(self, state): + return {"is-it-on": state} +``` + +Serializers can be public methods, and they can return whatever you like. If +necessary, you can have different serializers - just multiple methods decorated +with `@_machine.serializer()` - for different formats; return one data-structure +for JSON, one for XML, one for a database row, and so on. + +When it comes time to unserialize, though, you generally want a private method, +because an unserializer has to take a not-fully-initialized instance and +populate it with state. It is expected to *return* the serialized machine +state token that was passed to the serializer, but it can take whatever +arguments you like. Of course, in order to return that, it probably has to +take it somewhere in its arguments, so it will generally take whatever a paired +serializer has returned as an argument. + +So our unserializer would look like this: + +```python + @_machine.unserializer() + def _restore(self, blob): + return blob["is-it-on"] +``` + +Generally you will want a classmethod deserialization constructor which you +write yourself to call this, so that you know how to create an instance of your +own object, like so: + +```python + @classmethod + def from_blob(cls, blob): + self = cls() + self._restore(blob) + return self +``` + +Saving and loading our `LightSwitch` along with its state-machine state can now +be accomplished as follows: + +```python +>>> switch1 = LightSwitch() +>>> switch1.query_power() +False +>>> switch1.flip() +[] +>>> switch1.query_power() +True +>>> blob = switch1.save() +>>> switch2 = LightSwitch.from_blob(blob) +>>> switch2.query_power() +True +``` + +More comprehensive (tested, working) examples are present in `docs/examples`. + +Go forth and machine all the state! diff --git a/contrib/python/Automat/py3/automat/__init__.py b/contrib/python/Automat/py3/automat/__init__.py new file mode 100644 index 00000000000..570b84f9951 --- /dev/null +++ b/contrib/python/Automat/py3/automat/__init__.py @@ -0,0 +1,8 @@ +# -*- test-case-name: automat -*- +from ._methodical import MethodicalMachine +from ._core import NoTransition + +__all__ = [ + 'MethodicalMachine', + 'NoTransition', +] diff --git a/contrib/python/Automat/py3/automat/_core.py b/contrib/python/Automat/py3/automat/_core.py new file mode 100644 index 00000000000..4118a4b070a --- /dev/null +++ b/contrib/python/Automat/py3/automat/_core.py @@ -0,0 +1,165 @@ +# -*- test-case-name: automat._test.test_core -*- + +""" +A core state-machine abstraction. + +Perhaps something that could be replaced with or integrated into machinist. +""" + +from itertools import chain + +_NO_STATE = "" + + +class NoTransition(Exception): + """ + A finite state machine in C{state} has no transition for C{symbol}. + + @param state: the finite state machine's state at the time of the + illegal transition. + + @param symbol: the input symbol for which no transition exists. + """ + + def __init__(self, state, symbol): + self.state = state + self.symbol = symbol + super(Exception, self).__init__( + "no transition for {} in {}".format(symbol, state) + ) + + +class Automaton(object): + """ + A declaration of a finite state machine. + + Note that this is not the machine itself; it is immutable. + """ + + def __init__(self): + """ + Initialize the set of transitions and the initial state. + """ + self._initialState = _NO_STATE + self._transitions = set() + + + @property + def initialState(self): + """ + Return this automaton's initial state. + """ + return self._initialState + + + @initialState.setter + def initialState(self, state): + """ + Set this automaton's initial state. Raises a ValueError if + this automaton already has an initial state. + """ + + if self._initialState is not _NO_STATE: + raise ValueError( + "initial state already set to {}".format(self._initialState)) + + self._initialState = state + + + def addTransition(self, inState, inputSymbol, outState, outputSymbols): + """ + Add the given transition to the outputSymbol. Raise ValueError if + there is already a transition with the same inState and inputSymbol. + """ + # keeping self._transitions in a flat list makes addTransition + # O(n^2), but state machines don't tend to have hundreds of + # transitions. + for (anInState, anInputSymbol, anOutState, _) in self._transitions: + if (anInState == inState and anInputSymbol == inputSymbol): + raise ValueError( + "already have transition from {} via {}".format(inState, inputSymbol)) + self._transitions.add( + (inState, inputSymbol, outState, tuple(outputSymbols)) + ) + + + def allTransitions(self): + """ + All transitions. + """ + return frozenset(self._transitions) + + + def inputAlphabet(self): + """ + The full set of symbols acceptable to this automaton. + """ + return {inputSymbol for (inState, inputSymbol, outState, + outputSymbol) in self._transitions} + + + def outputAlphabet(self): + """ + The full set of symbols which can be produced by this automaton. + """ + return set( + chain.from_iterable( + outputSymbols for + (inState, inputSymbol, outState, outputSymbols) + in self._transitions + ) + ) + + + def states(self): + """ + All valid states; "Q" in the mathematical description of a state + machine. + """ + return frozenset( + chain.from_iterable( + (inState, outState) + for + (inState, inputSymbol, outState, outputSymbol) + in self._transitions + ) + ) + + + def outputForInput(self, inState, inputSymbol): + """ + A 2-tuple of (outState, outputSymbols) for inputSymbol. + """ + for (anInState, anInputSymbol, + outState, outputSymbols) in self._transitions: + if (inState, inputSymbol) == (anInState, anInputSymbol): + return (outState, list(outputSymbols)) + raise NoTransition(state=inState, symbol=inputSymbol) + + +class Transitioner(object): + """ + The combination of a current state and an L{Automaton}. + """ + + def __init__(self, automaton, initialState): + self._automaton = automaton + self._state = initialState + self._tracer = None + + def setTrace(self, tracer): + self._tracer = tracer + + def transition(self, inputSymbol): + """ + Transition between states, returning any outputs. + """ + outState, outputSymbols = self._automaton.outputForInput(self._state, + inputSymbol) + outTracer = None + if self._tracer: + outTracer = self._tracer(self._state._name(), + inputSymbol._name(), + outState._name()) + self._state = outState + return (outputSymbols, outTracer) diff --git a/contrib/python/Automat/py3/automat/_discover.py b/contrib/python/Automat/py3/automat/_discover.py new file mode 100644 index 00000000000..c0d88baea4e --- /dev/null +++ b/contrib/python/Automat/py3/automat/_discover.py @@ -0,0 +1,144 @@ +import collections +import inspect +from automat import MethodicalMachine +from twisted.python.modules import PythonModule, getModule + + +def isOriginalLocation(attr): + """ + Attempt to discover if this appearance of a PythonAttribute + representing a class refers to the module where that class was + defined. + """ + sourceModule = inspect.getmodule(attr.load()) + if sourceModule is None: + return False + + currentModule = attr + while not isinstance(currentModule, PythonModule): + currentModule = currentModule.onObject + + return currentModule.name == sourceModule.__name__ + + +def findMachinesViaWrapper(within): + """ + Recursively yield L{MethodicalMachine}s and their FQPNs within a + L{PythonModule} or a L{twisted.python.modules.PythonAttribute} + wrapper object. + + Note that L{PythonModule}s may refer to packages, as well. + + The discovery heuristic considers L{MethodicalMachine} instances + that are module-level attributes or class-level attributes + accessible from module scope. Machines inside nested classes will + be discovered, but those returned from functions or methods will not be. + + @type within: L{PythonModule} or L{twisted.python.modules.PythonAttribute} + @param within: Where to start the search. + + @return: a generator which yields FQPN, L{MethodicalMachine} pairs. + """ + queue = collections.deque([within]) + visited = set() + + while queue: + attr = queue.pop() + value = attr.load() + + if isinstance(value, MethodicalMachine) and value not in visited: + visited.add(value) + yield attr.name, value + elif (inspect.isclass(value) and isOriginalLocation(attr) and + value not in visited): + visited.add(value) + queue.extendleft(attr.iterAttributes()) + elif isinstance(attr, PythonModule) and value not in visited: + visited.add(value) + queue.extendleft(attr.iterAttributes()) + queue.extendleft(attr.iterModules()) + + +class InvalidFQPN(Exception): + """ + The given FQPN was not a dot-separated list of Python objects. + """ + + +class NoModule(InvalidFQPN): + """ + A prefix of the FQPN was not an importable module or package. + """ + + +class NoObject(InvalidFQPN): + """ + A suffix of the FQPN was not an accessible object + """ + + +def wrapFQPN(fqpn): + """ + Given an FQPN, retrieve the object via the global Python module + namespace and wrap it with a L{PythonModule} or a + L{twisted.python.modules.PythonAttribute}. + """ + # largely cribbed from t.p.reflect.namedAny + + if not fqpn: + raise InvalidFQPN("FQPN was empty") + + components = collections.deque(fqpn.split('.')) + + if '' in components: + raise InvalidFQPN( + "name must be a string giving a '.'-separated list of Python " + "identifiers, not %r" % (fqpn,)) + + component = components.popleft() + try: + module = getModule(component) + except KeyError: + raise NoModule(component) + + # find the bottom-most module + while components: + component = components.popleft() + try: + module = module[component] + except KeyError: + components.appendleft(component) + break + else: + module.load() + else: + return module + + # find the bottom-most attribute + attribute = module + for component in components: + try: + attribute = next(child for child in attribute.iterAttributes() + if child.name.rsplit('.', 1)[-1] == component) + except StopIteration: + raise NoObject('{}.{}'.format(attribute.name, component)) + + return attribute + + +def findMachines(fqpn): + """ + Recursively yield L{MethodicalMachine}s and their FQPNs in and + under the a Python object specified by an FQPN. + + The discovery heuristic considers L{MethodicalMachine} instances + that are module-level attributes or class-level attributes + accessible from module scope. Machines inside nested classes will + be discovered, but those returned from functions or methods will not be. + + @type within: an FQPN + @param within: Where to start the search. + + @return: a generator which yields FQPN, L{MethodicalMachine} pairs. + """ + return findMachinesViaWrapper(wrapFQPN(fqpn)) diff --git a/contrib/python/Automat/py3/automat/_introspection.py b/contrib/python/Automat/py3/automat/_introspection.py new file mode 100644 index 00000000000..403cddb15e9 --- /dev/null +++ b/contrib/python/Automat/py3/automat/_introspection.py @@ -0,0 +1,46 @@ +""" +Python introspection helpers. +""" + +from types import CodeType as code, FunctionType as function + + +def copycode(template, changes): + if hasattr(code, "replace"): + return template.replace(**{"co_" + k : v for k, v in changes.items()}) + names = [ + "argcount", "nlocals", "stacksize", "flags", "code", "consts", + "names", "varnames", "filename", "name", "firstlineno", "lnotab", + "freevars", "cellvars" + ] + if hasattr(code, "co_kwonlyargcount"): + names.insert(1, "kwonlyargcount") + if hasattr(code, "co_posonlyargcount"): + # PEP 570 added "positional only arguments" + names.insert(1, "posonlyargcount") + values = [ + changes.get(name, getattr(template, "co_" + name)) + for name in names + ] + return code(*values) + + +def copyfunction(template, funcchanges, codechanges): + names = [ + "globals", "name", "defaults", "closure", + ] + values = [ + funcchanges.get(name, getattr(template, "__" + name + "__")) + for name in names + ] + return function(copycode(template.__code__, codechanges), *values) + + +def preserveName(f): + """ + Preserve the name of the given function on the decorated function. + """ + def decorator(decorated): + return copyfunction(decorated, + dict(name=f.__name__), dict(name=f.__name__)) + return decorator diff --git a/contrib/python/Automat/py3/automat/_methodical.py b/contrib/python/Automat/py3/automat/_methodical.py new file mode 100644 index 00000000000..6c9060cbb04 --- /dev/null +++ b/contrib/python/Automat/py3/automat/_methodical.py @@ -0,0 +1,472 @@ +# -*- test-case-name: automat._test.test_methodical -*- + +import collections +from functools import wraps +from itertools import count + +from inspect import getfullargspec as getArgsSpec + +import attr + +from ._core import Transitioner, Automaton +from ._introspection import preserveName + + +ArgSpec = collections.namedtuple('ArgSpec', ['args', 'varargs', 'varkw', + 'defaults', 'kwonlyargs', + 'kwonlydefaults', 'annotations']) + + +def _getArgSpec(func): + """ + Normalize inspect.ArgSpec across python versions + and convert mutable attributes to immutable types. + + :param Callable func: A function. + :return: The function's ArgSpec. + :rtype: ArgSpec + """ + spec = getArgsSpec(func) + return ArgSpec( + args=tuple(spec.args), + varargs=spec.varargs, + varkw=spec.varkw, + defaults=spec.defaults if spec.defaults else (), + kwonlyargs=tuple(spec.kwonlyargs), + kwonlydefaults=( + tuple(spec.kwonlydefaults.items()) + if spec.kwonlydefaults else () + ), + annotations=tuple(spec.annotations.items()), + ) + + +def _getArgNames(spec): + """ + Get the name of all arguments defined in a function signature. + + The name of * and ** arguments is normalized to "*args" and "**kwargs". + + :param ArgSpec spec: A function to interrogate for a signature. + :return: The set of all argument names in `func`s signature. + :rtype: Set[str] + """ + return set( + spec.args + + spec.kwonlyargs + + (('*args',) if spec.varargs else ()) + + (('**kwargs',) if spec.varkw else ()) + + spec.annotations + ) + + +def _keywords_only(f): + """ + Decorate a function so all its arguments must be passed by keyword. + + A useful utility for decorators that take arguments so that they don't + accidentally get passed the thing they're decorating as their first + argument. + + Only works for methods right now. + """ + @wraps(f) + def g(self, **kw): + return f(self, **kw) + return g + + +@attr.s(frozen=True) +class MethodicalState(object): + """ + A state for a L{MethodicalMachine}. + """ + machine = attr.ib(repr=False) + method = attr.ib() + serialized = attr.ib(repr=False) + + def upon(self, input, enter=None, outputs=None, collector=list): + """ + Declare a state transition within the :class:`automat.MethodicalMachine` + associated with this :class:`automat.MethodicalState`: + upon the receipt of the `input`, enter the `state`, + emitting each output in `outputs`. + + :param MethodicalInput input: The input triggering a state transition. + :param MethodicalState enter: The resulting state. + :param Iterable[MethodicalOutput] outputs: The outputs to be triggered + as a result of the declared state transition. + :param Callable collector: The function to be used when collecting + output return values. + + :raises TypeError: if any of the `outputs` signatures do not match + the `inputs` signature. + :raises ValueError: if the state transition from `self` via `input` + has already been defined. + """ + if enter is None: + enter = self + if outputs is None: + outputs = [] + inputArgs = _getArgNames(input.argSpec) + for output in outputs: + outputArgs = _getArgNames(output.argSpec) + if not outputArgs.issubset(inputArgs): + raise TypeError( + "method {input} signature {inputSignature} " + "does not match output {output} " + "signature {outputSignature}".format( + input=input.method.__name__, + output=output.method.__name__, + inputSignature=getArgsSpec(input.method), + outputSignature=getArgsSpec(output.method), + )) + self.machine._oneTransition(self, input, enter, outputs, collector) + + def _name(self): + return self.method.__name__ + + +def _transitionerFromInstance(oself, symbol, automaton): + """ + Get a L{Transitioner} + """ + transitioner = getattr(oself, symbol, None) + if transitioner is None: + transitioner = Transitioner( + automaton, + automaton.initialState, + ) + setattr(oself, symbol, transitioner) + return transitioner + + +def _empty(): + pass + +def _docstring(): + """docstring""" + +def assertNoCode(inst, attribute, f): + # The function body must be empty, i.e. "pass" or "return None", which + # both yield the same bytecode: LOAD_CONST (None), RETURN_VALUE. We also + # accept functions with only a docstring, which yields slightly different + # bytecode, because the "None" is put in a different constant slot. + + # Unfortunately, this does not catch function bodies that return a + # constant value, e.g. "return 1", because their code is identical to a + # "return None". They differ in the contents of their constant table, but + # checking that would require us to parse the bytecode, find the index + # being returned, then making sure the table has a None at that index. + + if f.__code__.co_code not in (_empty.__code__.co_code, + _docstring.__code__.co_code): + raise ValueError("function body must be empty") + + +def _filterArgs(args, kwargs, inputSpec, outputSpec): + """ + Filter out arguments that were passed to input that output won't accept. + + :param tuple args: The *args that input received. + :param dict kwargs: The **kwargs that input received. + :param ArgSpec inputSpec: The input's arg spec. + :param ArgSpec outputSpec: The output's arg spec. + :return: The args and kwargs that output will accept. + :rtype: Tuple[tuple, dict] + """ + named_args = tuple(zip(inputSpec.args[1:], args)) + if outputSpec.varargs: + # Only return all args if the output accepts *args. + return_args = args + else: + # Filter out arguments that don't appear + # in the output's method signature. + return_args = [v for n, v in named_args if n in outputSpec.args] + + # Get any of input's default arguments that were not passed. + passed_arg_names = tuple(kwargs) + for name, value in named_args: + passed_arg_names += (name, value) + defaults = zip(inputSpec.args[::-1], inputSpec.defaults[::-1]) + full_kwargs = {n: v for n, v in defaults if n not in passed_arg_names} + full_kwargs.update(kwargs) + + if outputSpec.varkw: + # Only pass all kwargs if the output method accepts **kwargs. + return_kwargs = full_kwargs + else: + # Filter out names that the output method does not accept. + all_accepted_names = outputSpec.args[1:] + outputSpec.kwonlyargs + return_kwargs = {n: v for n, v in full_kwargs.items() + if n in all_accepted_names} + + return return_args, return_kwargs + + +@attr.s(eq=False, hash=False) +class MethodicalInput(object): + """ + An input for a L{MethodicalMachine}. + """ + automaton = attr.ib(repr=False) + method = attr.ib(validator=assertNoCode) + symbol = attr.ib(repr=False) + collectors = attr.ib(default=attr.Factory(dict), repr=False) + argSpec = attr.ib(init=False, repr=False) + + @argSpec.default + def _buildArgSpec(self): + return _getArgSpec(self.method) + + def __get__(self, oself, type=None): + """ + Return a function that takes no arguments and returns values returned + by output functions produced by the given L{MethodicalInput} in + C{oself}'s current state. + """ + transitioner = _transitionerFromInstance(oself, self.symbol, + self.automaton) + @preserveName(self.method) + @wraps(self.method) + def doInput(*args, **kwargs): + self.method(oself, *args, **kwargs) + previousState = transitioner._state + (outputs, outTracer) = transitioner.transition(self) + collector = self.collectors[previousState] + values = [] + for output in outputs: + if outTracer: + outTracer(output._name()) + a, k = _filterArgs(args, kwargs, self.argSpec, output.argSpec) + value = output(oself, *a, **k) + values.append(value) + return collector(values) + return doInput + + def _name(self): + return self.method.__name__ + + +@attr.s(frozen=True) +class MethodicalOutput(object): + """ + An output for a L{MethodicalMachine}. + """ + machine = attr.ib(repr=False) + method = attr.ib() + argSpec = attr.ib(init=False, repr=False) + + @argSpec.default + def _buildArgSpec(self): + return _getArgSpec(self.method) + + def __get__(self, oself, type=None): + """ + Outputs are private, so raise an exception when we attempt to get one. + """ + raise AttributeError( + "{cls}.{method} is a state-machine output method; " + "to produce this output, call an input method instead.".format( + cls=type.__name__, + method=self.method.__name__ + ) + ) + + + def __call__(self, oself, *args, **kwargs): + """ + Call the underlying method. + """ + return self.method(oself, *args, **kwargs) + + def _name(self): + return self.method.__name__ + +@attr.s(eq=False, hash=False) +class MethodicalTracer(object): + automaton = attr.ib(repr=False) + symbol = attr.ib(repr=False) + + + def __get__(self, oself, type=None): + transitioner = _transitionerFromInstance(oself, self.symbol, + self.automaton) + def setTrace(tracer): + transitioner.setTrace(tracer) + return setTrace + + + +counter = count() +def gensym(): + """ + Create a unique Python identifier. + """ + return "_symbol_" + str(next(counter)) + + + +class MethodicalMachine(object): + """ + A :class:`MethodicalMachine` is an interface to an `Automaton` + that uses methods on a class. + """ + + def __init__(self): + self._automaton = Automaton() + self._reducers = {} + self._symbol = gensym() + + + def __get__(self, oself, type=None): + """ + L{MethodicalMachine} is an implementation detail for setting up + class-level state; applications should never need to access it on an + instance. + """ + if oself is not None: + raise AttributeError( + "MethodicalMachine is an implementation detail.") + return self + + + @_keywords_only + def state(self, initial=False, terminal=False, + serialized=None): + """ + Declare a state, possibly an initial state or a terminal state. + + This is a decorator for methods, but it will modify the method so as + not to be callable any more. + + :param bool initial: is this state the initial state? + Only one state on this :class:`automat.MethodicalMachine` + may be an initial state; more than one is an error. + + :param bool terminal: Is this state a terminal state? + i.e. a state that the machine can end up in? + (This is purely informational at this point.) + + :param Hashable serialized: a serializable value + to be used to represent this state to external systems. + This value should be hashable; + :py:func:`unicode` is a good type to use. + """ + def decorator(stateMethod): + state = MethodicalState(machine=self, + method=stateMethod, + serialized=serialized) + if initial: + self._automaton.initialState = state + return state + return decorator + + + @_keywords_only + def input(self): + """ + Declare an input. + + This is a decorator for methods. + """ + def decorator(inputMethod): + return MethodicalInput(automaton=self._automaton, + method=inputMethod, + symbol=self._symbol) + return decorator + + + @_keywords_only + def output(self): + """ + Declare an output. + + This is a decorator for methods. + + This method will be called when the state machine transitions to this + state as specified in the decorated `output` method. + """ + def decorator(outputMethod): + return MethodicalOutput(machine=self, method=outputMethod) + return decorator + + + def _oneTransition(self, startState, inputToken, endState, outputTokens, + collector): + """ + See L{MethodicalState.upon}. + """ + # FIXME: tests for all of this (some of it is wrong) + # if not isinstance(startState, MethodicalState): + # raise NotImplementedError("start state {} isn't a state" + # .format(startState)) + # if not isinstance(inputToken, MethodicalInput): + # raise NotImplementedError("start state {} isn't an input" + # .format(inputToken)) + # if not isinstance(endState, MethodicalState): + # raise NotImplementedError("end state {} isn't a state" + # .format(startState)) + # for output in outputTokens: + # if not isinstance(endState, MethodicalState): + # raise NotImplementedError("output state {} isn't a state" + # .format(endState)) + self._automaton.addTransition(startState, inputToken, endState, + tuple(outputTokens)) + inputToken.collectors[startState] = collector + + + @_keywords_only + def serializer(self): + """ + + """ + def decorator(decoratee): + @wraps(decoratee) + def serialize(oself): + transitioner = _transitionerFromInstance(oself, self._symbol, + self._automaton) + return decoratee(oself, transitioner._state.serialized) + return serialize + return decorator + + @_keywords_only + def unserializer(self): + """ + + """ + def decorator(decoratee): + @wraps(decoratee) + def unserialize(oself, *args, **kwargs): + state = decoratee(oself, *args, **kwargs) + mapping = {} + for eachState in self._automaton.states(): + mapping[eachState.serialized] = eachState + transitioner = _transitionerFromInstance( + oself, self._symbol, self._automaton) + transitioner._state = mapping[state] + return None # it's on purpose + return unserialize + return decorator + + @property + def _setTrace(self): + return MethodicalTracer(self._automaton, self._symbol) + + def asDigraph(self): + """ + Generate a L{graphviz.Digraph} that represents this machine's + states and transitions. + + @return: L{graphviz.Digraph} object; for more information, please + see the documentation for + U{graphviz} + + """ + from ._visualize import makeDigraph + return makeDigraph( + self._automaton, + stateAsString=lambda state: state.method.__name__, + inputAsString=lambda input: input.method.__name__, + outputAsString=lambda output: output.method.__name__, + ) diff --git a/contrib/python/Automat/py3/automat/_visualize.py b/contrib/python/Automat/py3/automat/_visualize.py new file mode 100644 index 00000000000..7a9c8c6eb55 --- /dev/null +++ b/contrib/python/Automat/py3/automat/_visualize.py @@ -0,0 +1,182 @@ +from __future__ import print_function +import argparse +import sys + +import graphviz + +from ._discover import findMachines + + +def _gvquote(s): + return '"{}"'.format(s.replace('"', r'\"')) + + +def _gvhtml(s): + return '<{}>'.format(s) + + +def elementMaker(name, *children, **attrs): + """ + Construct a string from the HTML element description. + """ + formattedAttrs = ' '.join('{}={}'.format(key, _gvquote(str(value))) + for key, value in sorted(attrs.items())) + formattedChildren = ''.join(children) + return u'<{name} {attrs}>{children}'.format( + name=name, + attrs=formattedAttrs, + children=formattedChildren) + + +def tableMaker(inputLabel, outputLabels, port, _E=elementMaker): + """ + Construct an HTML table to label a state transition. + """ + colspan = {} + if outputLabels: + colspan['colspan'] = str(len(outputLabels)) + + inputLabelCell = _E("td", + _E("font", + inputLabel, + face="menlo-italic"), + color="purple", + port=port, + **colspan) + + pointSize = {"point-size": "9"} + outputLabelCells = [_E("td", + _E("font", + outputLabel, + **pointSize), + color="pink") + for outputLabel in outputLabels] + + rows = [_E("tr", inputLabelCell)] + + if outputLabels: + rows.append(_E("tr", *outputLabelCells)) + + return _E("table", *rows) + + +def makeDigraph(automaton, inputAsString=repr, + outputAsString=repr, + stateAsString=repr): + """ + Produce a L{graphviz.Digraph} object from an automaton. + """ + digraph = graphviz.Digraph(graph_attr={'pack': 'true', + 'dpi': '100'}, + node_attr={'fontname': 'Menlo'}, + edge_attr={'fontname': 'Menlo'}) + + for state in automaton.states(): + if state is automaton.initialState: + stateShape = "bold" + fontName = "Menlo-Bold" + else: + stateShape = "" + fontName = "Menlo" + digraph.node(stateAsString(state), + fontame=fontName, + shape="ellipse", + style=stateShape, + color="blue") + for n, eachTransition in enumerate(automaton.allTransitions()): + inState, inputSymbol, outState, outputSymbols = eachTransition + thisTransition = "t{}".format(n) + inputLabel = inputAsString(inputSymbol) + + port = "tableport" + table = tableMaker(inputLabel, [outputAsString(outputSymbol) + for outputSymbol in outputSymbols], + port=port) + + digraph.node(thisTransition, + label=_gvhtml(table), margin="0.2", shape="none") + + digraph.edge(stateAsString(inState), + '{}:{}:w'.format(thisTransition, port), + arrowhead="none") + digraph.edge('{}:{}:e'.format(thisTransition, port), + stateAsString(outState)) + + return digraph + + +def tool(_progname=sys.argv[0], + _argv=sys.argv[1:], + _syspath=sys.path, + _findMachines=findMachines, + _print=print): + """ + Entry point for command line utility. + """ + + DESCRIPTION = """ + Visualize automat.MethodicalMachines as graphviz graphs. + """ + EPILOG = """ + You must have the graphviz tool suite installed. Please visit + http://www.graphviz.org for more information. + """ + if _syspath[0]: + _syspath.insert(0, '') + argumentParser = argparse.ArgumentParser( + prog=_progname, + description=DESCRIPTION, + epilog=EPILOG) + argumentParser.add_argument('fqpn', + help="A Fully Qualified Path name" + " representing where to find machines.") + argumentParser.add_argument('--quiet', '-q', + help="suppress output", + default=False, + action="store_true") + argumentParser.add_argument('--dot-directory', '-d', + help="Where to write out .dot files.", + default=".automat_visualize") + argumentParser.add_argument('--image-directory', '-i', + help="Where to write out image files.", + default=".automat_visualize") + argumentParser.add_argument('--image-type', '-t', + help="The image format.", + choices=graphviz.FORMATS, + default='png') + argumentParser.add_argument('--view', '-v', + help="View rendered graphs with" + " default image viewer", + default=False, + action="store_true") + args = argumentParser.parse_args(_argv) + + explicitlySaveDot = (args.dot_directory + and (not args.image_directory + or args.image_directory != args.dot_directory)) + if args.quiet: + def _print(*args): + pass + + for fqpn, machine in _findMachines(args.fqpn): + _print(fqpn, '...discovered') + + digraph = machine.asDigraph() + + if explicitlySaveDot: + digraph.save(filename="{}.dot".format(fqpn), + directory=args.dot_directory) + _print(fqpn, "...wrote dot into", args.dot_directory) + + if args.image_directory: + deleteDot = not args.dot_directory or explicitlySaveDot + digraph.format = args.image_type + digraph.render(filename="{}.dot".format(fqpn), + directory=args.image_directory, + view=args.view, + cleanup=deleteDot) + if deleteDot: + msg = "...wrote image into" + else: + msg = "...wrote image and dot into" + _print(fqpn, msg, args.image_directory) diff --git a/contrib/python/Automat/py3/ya.make b/contrib/python/Automat/py3/ya.make new file mode 100644 index 00000000000..03a66263d74 --- /dev/null +++ b/contrib/python/Automat/py3/ya.make @@ -0,0 +1,38 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(22.10.0) + +LICENSE(MIT) + +PEERDIR( + contrib/python/attrs + contrib/python/six +) + +NO_LINT() + +NO_CHECK_IMPORTS( + automat._discover + automat._visualize +) + +PY_SRCS( + TOP_LEVEL + automat/__init__.py + automat/_core.py + automat/_discover.py + automat/_introspection.py + automat/_methodical.py + automat/_visualize.py +) + +RESOURCE_FILES( + PREFIX contrib/python/Automat/py3/ + .dist-info/METADATA + .dist-info/entry_points.txt + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/Automat/ya.make b/contrib/python/Automat/ya.make new file mode 100644 index 00000000000..8df839fcc1f --- /dev/null +++ b/contrib/python/Automat/ya.make @@ -0,0 +1,18 @@ +PY23_LIBRARY() + +LICENSE(Service-Py23-Proxy) + +IF (PYTHON2) + PEERDIR(contrib/python/Automat/py2) +ELSE() + PEERDIR(contrib/python/Automat/py3) +ENDIF() + +NO_LINT() + +END() + +RECURSE( + py2 + py3 +) diff --git a/contrib/python/Twisted/py2/.dist-info/METADATA b/contrib/python/Twisted/py2/.dist-info/METADATA new file mode 100644 index 00000000000..e6e4e8267ea --- /dev/null +++ b/contrib/python/Twisted/py2/.dist-info/METADATA @@ -0,0 +1,217 @@ +Metadata-Version: 2.1 +Name: Twisted +Version: 20.3.0 +Summary: An asynchronous networking framework written in Python +Home-page: https://twistedmatrix.com/ +Author: Twisted Matrix Laboratories +Author-email: twisted-python@twistedmatrix.com +Maintainer: Glyph Lefkowitz +Maintainer-email: glyph@twistedmatrix.com +License: MIT +Project-URL: Documentation, https://twistedmatrix.com/documents/current/ +Project-URL: Source, https://github.com/twisted/twisted +Project-URL: Issues, https://twistedmatrix.com/trac/report +Platform: UNKNOWN +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.* +Description-Content-Type: text/x-rst +Requires-Dist: zope.interface (>=4.4.2) +Requires-Dist: constantly (>=15.1) +Requires-Dist: incremental (>=16.10.1) +Requires-Dist: Automat (>=0.3.0) +Requires-Dist: hyperlink (>=17.1.1) +Requires-Dist: PyHamcrest (!=1.10.0,>=1.9.0) +Requires-Dist: attrs (>=19.2.0) +Provides-Extra: all_non_platform +Requires-Dist: pyopenssl (>=16.0.0) ; extra == 'all_non_platform' +Requires-Dist: service-identity (>=18.1.0) ; extra == 'all_non_platform' +Requires-Dist: idna (!=2.3,>=0.6) ; extra == 'all_non_platform' +Requires-Dist: pyasn1 ; extra == 'all_non_platform' +Requires-Dist: cryptography (>=2.5) ; extra == 'all_non_platform' +Requires-Dist: appdirs (>=1.4.0) ; extra == 'all_non_platform' +Requires-Dist: bcrypt (>=3.0.0) ; extra == 'all_non_platform' +Requires-Dist: soappy ; extra == 'all_non_platform' +Requires-Dist: pyserial (>=3.0) ; extra == 'all_non_platform' +Requires-Dist: h2 (<4.0,>=3.0) ; extra == 'all_non_platform' +Requires-Dist: priority (<2.0,>=1.1.0) ; extra == 'all_non_platform' +Requires-Dist: pywin32 (!=226) ; (platform_system == "Windows") and extra == 'all_non_platform' +Provides-Extra: conch +Requires-Dist: pyasn1 ; extra == 'conch' +Requires-Dist: cryptography (>=2.5) ; extra == 'conch' +Requires-Dist: appdirs (>=1.4.0) ; extra == 'conch' +Requires-Dist: bcrypt (>=3.0.0) ; extra == 'conch' +Provides-Extra: dev +Requires-Dist: pyflakes (>=1.0.0) ; extra == 'dev' +Requires-Dist: twisted-dev-tools (>=0.0.2) ; extra == 'dev' +Requires-Dist: python-subunit ; extra == 'dev' +Requires-Dist: sphinx (>=1.3.1) ; extra == 'dev' +Requires-Dist: towncrier (>=17.4.0) ; extra == 'dev' +Provides-Extra: http2 +Requires-Dist: h2 (<4.0,>=3.0) ; extra == 'http2' +Requires-Dist: priority (<2.0,>=1.1.0) ; extra == 'http2' +Provides-Extra: macos_platform +Requires-Dist: pyobjc-core ; extra == 'macos_platform' +Requires-Dist: pyobjc-framework-CFNetwork ; extra == 'macos_platform' +Requires-Dist: pyobjc-framework-Cocoa ; extra == 'macos_platform' +Requires-Dist: pyopenssl (>=16.0.0) ; extra == 'macos_platform' +Requires-Dist: service-identity (>=18.1.0) ; extra == 'macos_platform' +Requires-Dist: idna (!=2.3,>=0.6) ; extra == 'macos_platform' +Requires-Dist: pyasn1 ; extra == 'macos_platform' +Requires-Dist: cryptography (>=2.5) ; extra == 'macos_platform' +Requires-Dist: appdirs (>=1.4.0) ; extra == 'macos_platform' +Requires-Dist: bcrypt (>=3.0.0) ; extra == 'macos_platform' +Requires-Dist: soappy ; extra == 'macos_platform' +Requires-Dist: pyserial (>=3.0) ; extra == 'macos_platform' +Requires-Dist: h2 (<4.0,>=3.0) ; extra == 'macos_platform' +Requires-Dist: priority (<2.0,>=1.1.0) ; extra == 'macos_platform' +Requires-Dist: pywin32 (!=226) ; (platform_system == "Windows") and extra == 'macos_platform' +Provides-Extra: osx_platform +Requires-Dist: pyobjc-core ; extra == 'osx_platform' +Requires-Dist: pyobjc-framework-CFNetwork ; extra == 'osx_platform' +Requires-Dist: pyobjc-framework-Cocoa ; extra == 'osx_platform' +Requires-Dist: pyopenssl (>=16.0.0) ; extra == 'osx_platform' +Requires-Dist: service-identity (>=18.1.0) ; extra == 'osx_platform' +Requires-Dist: idna (!=2.3,>=0.6) ; extra == 'osx_platform' +Requires-Dist: pyasn1 ; extra == 'osx_platform' +Requires-Dist: cryptography (>=2.5) ; extra == 'osx_platform' +Requires-Dist: appdirs (>=1.4.0) ; extra == 'osx_platform' +Requires-Dist: bcrypt (>=3.0.0) ; extra == 'osx_platform' +Requires-Dist: soappy ; extra == 'osx_platform' +Requires-Dist: pyserial (>=3.0) ; extra == 'osx_platform' +Requires-Dist: h2 (<4.0,>=3.0) ; extra == 'osx_platform' +Requires-Dist: priority (<2.0,>=1.1.0) ; extra == 'osx_platform' +Requires-Dist: pywin32 (!=226) ; (platform_system == "Windows") and extra == 'osx_platform' +Provides-Extra: serial +Requires-Dist: pyserial (>=3.0) ; extra == 'serial' +Requires-Dist: pywin32 (!=226) ; (platform_system == "Windows") and extra == 'serial' +Provides-Extra: soap +Requires-Dist: soappy ; extra == 'soap' +Provides-Extra: tls +Requires-Dist: pyopenssl (>=16.0.0) ; extra == 'tls' +Requires-Dist: service-identity (>=18.1.0) ; extra == 'tls' +Requires-Dist: idna (!=2.3,>=0.6) ; extra == 'tls' +Provides-Extra: windows_platform +Requires-Dist: pywin32 (!=226) ; extra == 'windows_platform' +Requires-Dist: pyopenssl (>=16.0.0) ; extra == 'windows_platform' +Requires-Dist: service-identity (>=18.1.0) ; extra == 'windows_platform' +Requires-Dist: idna (!=2.3,>=0.6) ; extra == 'windows_platform' +Requires-Dist: pyasn1 ; extra == 'windows_platform' +Requires-Dist: cryptography (>=2.5) ; extra == 'windows_platform' +Requires-Dist: appdirs (>=1.4.0) ; extra == 'windows_platform' +Requires-Dist: bcrypt (>=3.0.0) ; extra == 'windows_platform' +Requires-Dist: soappy ; extra == 'windows_platform' +Requires-Dist: pyserial (>=3.0) ; extra == 'windows_platform' +Requires-Dist: h2 (<4.0,>=3.0) ; extra == 'windows_platform' +Requires-Dist: priority (<2.0,>=1.1.0) ; extra == 'windows_platform' +Requires-Dist: pywin32 (!=226) ; (platform_system == "Windows") and extra == 'windows_platform' + +Twisted +======= + +|pypi|_ +|travis|_ +|circleci|_ + +For information on changes in this release, see the `NEWS `_ file. + + +What is this? +------------- + +Twisted is an event-based framework for internet applications, supporting Python 2.7 and Python 3.5+. +It includes modules for many different purposes, including the following: + +- ``twisted.web``: HTTP clients and servers, HTML templating, and a WSGI server +- ``twisted.conch``: SSHv2 and Telnet clients and servers and terminal emulators +- ``twisted.words``: Clients and servers for IRC, XMPP, and other IM protocols +- ``twisted.mail``: IMAPv4, POP3, SMTP clients and servers +- ``twisted.positioning``: Tools for communicating with NMEA-compatible GPS receivers +- ``twisted.names``: DNS client and tools for making your own DNS servers +- ``twisted.trial``: A unit testing framework that integrates well with Twisted-based code. + +Twisted supports all major system event loops -- ``select`` (all platforms), ``poll`` (most POSIX platforms), ``epoll`` (Linux), ``kqueue`` (FreeBSD, macOS), IOCP (Windows), and various GUI event loops (GTK+2/3, Qt, wxWidgets). +Third-party reactors can plug into Twisted, and provide support for additional event loops. + + +Installing +---------- + +To install the latest version of Twisted using pip:: + + $ pip install twisted + +Additional instructions for installing this software are in `the installation instructions `_. + + +Documentation and Support +------------------------- + +Twisted's documentation is available from the `Twisted Matrix website `_. +This documentation contains how-tos, code examples, and an API reference. + +Help is also available on the `Twisted mailing list `_. + +There is also a pair of very lively IRC channels, ``#twisted`` (for general Twisted questions) and ``#twisted.web`` (for Twisted Web), on ``chat.freenode.net``. + + +Unit Tests +---------- + +Twisted has a comprehensive test suite, which can be run by ``tox``:: + + $ tox -l # to view all test environments + $ tox -e py27-tests # to run the tests for Python 2.7 + $ tox -e py35-tests # to run the tests for Python 3.5 + + +You can test running the test suite under the different reactors with the ``TWISTED_REACTOR`` environment variable:: + + $ env TWISTED_REACTOR=epoll tox -e py27-tests + + +Some of these tests may fail if you: + +* don't have the dependencies required for a particular subsystem installed, +* have a firewall blocking some ports (or things like Multicast, which Linux NAT has shown itself to do), or +* run them as root. + + +Copyright +--------- + +All of the code in this distribution is Copyright (c) 2001-2020 Twisted Matrix Laboratories. + +Twisted is made available under the MIT license. +The included `LICENSE `_ file describes this in detail. + + +Warranty +-------- + + THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER + EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS + TO THE USE OF THIS SOFTWARE IS WITH YOU. + + IN NO EVENT WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY + AND/OR REDISTRIBUTE THE LIBRARY, BE LIABLE TO YOU FOR ANY DAMAGES, EVEN IF + SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH + DAMAGES. + +Again, see the included `LICENSE `_ file for specific legal details. + + +.. |pypi| image:: http://img.shields.io/pypi/v/twisted.svg +.. _pypi: https://pypi.python.org/pypi/twisted + +.. |travis| image:: https://travis-ci.org/twisted/twisted.svg?branch=trunk +.. _travis: https://travis-ci.org/twisted/twisted + +.. |circleci| image:: https://circleci.com/gh/twisted/twisted.svg?style=svg +.. _circleci: https://circleci.com/gh/twisted/twisted + + diff --git a/contrib/python/Twisted/py2/.dist-info/entry_points.txt b/contrib/python/Twisted/py2/.dist-info/entry_points.txt new file mode 100644 index 00000000000..8db754520c4 --- /dev/null +++ b/contrib/python/Twisted/py2/.dist-info/entry_points.txt @@ -0,0 +1,11 @@ +[console_scripts] +cftp = twisted.conch.scripts.cftp:run +ckeygen = twisted.conch.scripts.ckeygen:run +conch = twisted.conch.scripts.conch:run +mailmail = twisted.mail.scripts.mailmail:run +pyhtmlizer = twisted.scripts.htmlizer:run +tkconch = twisted.conch.scripts.tkconch:run +trial = twisted.scripts.trial:run +twist = twisted.application.twist._twist:Twist.main +twistd = twisted.scripts.twistd:run + diff --git a/contrib/python/Twisted/py2/.dist-info/top_level.txt b/contrib/python/Twisted/py2/.dist-info/top_level.txt new file mode 100644 index 00000000000..3eb29f049fb --- /dev/null +++ b/contrib/python/Twisted/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +twisted diff --git a/contrib/python/Twisted/py2/LICENSE b/contrib/python/Twisted/py2/LICENSE new file mode 100644 index 00000000000..79380d7dbf4 --- /dev/null +++ b/contrib/python/Twisted/py2/LICENSE @@ -0,0 +1,73 @@ +Copyright (c) 2001-2020 +Allen Short +Amber Hawkie Brown +Andrew Bennetts +Andy Gayton +Antoine Pitrou +Apple Computer, Inc. +Ashwini Oruganti +Benjamin Bruheim +Bob Ippolito +Canonical Limited +Christopher Armstrong +Ciena Corporation +David Reid +Divmod Inc. +Donovan Preston +Eric Mangold +Eyal Lotem +Google Inc. +Hybrid Logic Ltd. +Hynek Schlawack +Itamar Turner-Trauring +James Knight +Jason A. Mobarak +Jean-Paul Calderone +Jessica McKellar +Jonathan D. Simms +Jonathan Jacobs +Jonathan Lange +Julian Berman +Jürgen Hermann +Kevin Horn +Kevin Turner +Laurens Van Houtven +Mary Gardiner +Massachusetts Institute of Technology +Matthew Lefkowitz +Moshe Zadka +Paul Swartz +Pavel Pergamenshchik +Rackspace, US Inc. +Ralph Meijer +Richard Wall +Sean Riley +Software Freedom Conservancy +Tavendo GmbH +Thijs Triemstra +Thomas Herve +Timothy Allen +Tom Most +Tom Prince +Travis B. Hartwell + +and others that have contributed code to the public domain. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/Twisted/py2/README.rst b/contrib/python/Twisted/py2/README.rst new file mode 100644 index 00000000000..d28319f33d1 --- /dev/null +++ b/contrib/python/Twisted/py2/README.rst @@ -0,0 +1,104 @@ +Twisted +======= + +|pypi|_ +|travis|_ +|circleci|_ + +For information on changes in this release, see the `NEWS `_ file. + + +What is this? +------------- + +Twisted is an event-based framework for internet applications, supporting Python 2.7 and Python 3.5+. +It includes modules for many different purposes, including the following: + +- ``twisted.web``: HTTP clients and servers, HTML templating, and a WSGI server +- ``twisted.conch``: SSHv2 and Telnet clients and servers and terminal emulators +- ``twisted.words``: Clients and servers for IRC, XMPP, and other IM protocols +- ``twisted.mail``: IMAPv4, POP3, SMTP clients and servers +- ``twisted.positioning``: Tools for communicating with NMEA-compatible GPS receivers +- ``twisted.names``: DNS client and tools for making your own DNS servers +- ``twisted.trial``: A unit testing framework that integrates well with Twisted-based code. + +Twisted supports all major system event loops -- ``select`` (all platforms), ``poll`` (most POSIX platforms), ``epoll`` (Linux), ``kqueue`` (FreeBSD, macOS), IOCP (Windows), and various GUI event loops (GTK+2/3, Qt, wxWidgets). +Third-party reactors can plug into Twisted, and provide support for additional event loops. + + +Installing +---------- + +To install the latest version of Twisted using pip:: + + $ pip install twisted + +Additional instructions for installing this software are in `the installation instructions `_. + + +Documentation and Support +------------------------- + +Twisted's documentation is available from the `Twisted Matrix website `_. +This documentation contains how-tos, code examples, and an API reference. + +Help is also available on the `Twisted mailing list `_. + +There is also a pair of very lively IRC channels, ``#twisted`` (for general Twisted questions) and ``#twisted.web`` (for Twisted Web), on ``chat.freenode.net``. + + +Unit Tests +---------- + +Twisted has a comprehensive test suite, which can be run by ``tox``:: + + $ tox -l # to view all test environments + $ tox -e py27-tests # to run the tests for Python 2.7 + $ tox -e py35-tests # to run the tests for Python 3.5 + + +You can test running the test suite under the different reactors with the ``TWISTED_REACTOR`` environment variable:: + + $ env TWISTED_REACTOR=epoll tox -e py27-tests + + +Some of these tests may fail if you: + +* don't have the dependencies required for a particular subsystem installed, +* have a firewall blocking some ports (or things like Multicast, which Linux NAT has shown itself to do), or +* run them as root. + + +Copyright +--------- + +All of the code in this distribution is Copyright (c) 2001-2020 Twisted Matrix Laboratories. + +Twisted is made available under the MIT license. +The included `LICENSE `_ file describes this in detail. + + +Warranty +-------- + + THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER + EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS + TO THE USE OF THIS SOFTWARE IS WITH YOU. + + IN NO EVENT WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY + AND/OR REDISTRIBUTE THE LIBRARY, BE LIABLE TO YOU FOR ANY DAMAGES, EVEN IF + SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH + DAMAGES. + +Again, see the included `LICENSE `_ file for specific legal details. + + +.. |pypi| image:: http://img.shields.io/pypi/v/twisted.svg +.. _pypi: https://pypi.python.org/pypi/twisted + +.. |travis| image:: https://travis-ci.org/twisted/twisted.svg?branch=trunk +.. _travis: https://travis-ci.org/twisted/twisted + +.. |circleci| image:: https://circleci.com/gh/twisted/twisted.svg?style=svg +.. _circleci: https://circleci.com/gh/twisted/twisted diff --git a/contrib/python/Twisted/py2/twisted/__init__.py b/contrib/python/Twisted/py2/twisted/__init__.py new file mode 100644 index 00000000000..9d281a6db84 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/__init__.py @@ -0,0 +1,23 @@ +# -*- test-case-name: twisted -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted: The Framework Of Your Internet. +""" + +# setup version +from twisted._version import __version__ as version +__version__ = version.short() + + + +from incremental import Version +from twisted.python.deprecate import deprecatedModuleAttribute +deprecatedModuleAttribute( + Version('Twisted', 20, 3, 0), + "morituri nolumus mori", + "twisted", + "news" +) diff --git a/contrib/python/Twisted/py2/twisted/_threads/__init__.py b/contrib/python/Twisted/py2/twisted/_threads/__init__.py new file mode 100644 index 00000000000..7c98afd93c2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/_threads/__init__.py @@ -0,0 +1,25 @@ +# -*- test-case-name: twisted.test.test_paths -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted integration with operating system threads. +""" + +from __future__ import absolute_import, division, print_function + +from ._threadworker import ThreadWorker, LockWorker +from ._ithreads import IWorker, AlreadyQuit +from ._team import Team +from ._memory import createMemoryWorker +from ._pool import pool + +__all__ = [ + "ThreadWorker", + "LockWorker", + "IWorker", + "AlreadyQuit", + "Team", + "createMemoryWorker", + "pool", +] diff --git a/contrib/python/Twisted/py2/twisted/_threads/_convenience.py b/contrib/python/Twisted/py2/twisted/_threads/_convenience.py new file mode 100644 index 00000000000..deacad61ada --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/_threads/_convenience.py @@ -0,0 +1,46 @@ +# -*- test-case-name: twisted._threads.test.test_convenience -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Common functionality used within the implementation of various workers. +""" + +from __future__ import absolute_import, division, print_function + +from ._ithreads import AlreadyQuit + + +class Quit(object): + """ + A flag representing whether a worker has been quit. + + @ivar isSet: Whether this flag is set. + @type isSet: L{bool} + """ + + def __init__(self): + """ + Create a L{Quit} un-set. + """ + self.isSet = False + + + def set(self): + """ + Set the flag if it has not been set. + + @raise AlreadyQuit: If it has been set. + """ + self.check() + self.isSet = True + + + def check(self): + """ + Check if the flag has been set. + + @raise AlreadyQuit: If it has been set. + """ + if self.isSet: + raise AlreadyQuit() diff --git a/contrib/python/Twisted/py2/twisted/_threads/_ithreads.py b/contrib/python/Twisted/py2/twisted/_threads/_ithreads.py new file mode 100644 index 00000000000..4e8861ba363 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/_threads/_ithreads.py @@ -0,0 +1,61 @@ +# -*- test-case-name: twisted._threads.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Interfaces related to threads. +""" + +from __future__ import absolute_import, division, print_function + +from zope.interface import Interface + + +class AlreadyQuit(Exception): + """ + This worker worker is dead and cannot execute more instructions. + """ + + + +class IWorker(Interface): + """ + A worker that can perform some work concurrently. + + All methods on this interface must be thread-safe. + """ + + def do(task): + """ + Perform the given task. + + As an interface, this method makes no specific claims about concurrent + execution. An L{IWorker}'s C{do} implementation may defer execution + for later on the same thread, immediately on a different thread, or + some combination of the two. It is valid for a C{do} method to + schedule C{task} in such a way that it may never be executed. + + It is important for some implementations to provide specific properties + with respect to where C{task} is executed, of course, and client code + may rely on a more specific implementation of C{do} than L{IWorker}. + + @param task: a task to call in a thread or other concurrent context. + @type task: 0-argument callable + + @raise AlreadyQuit: if C{quit} has been called. + """ + + def quit(): + """ + Free any resources associated with this L{IWorker} and cause it to + reject all future work. + + @raise: L{AlreadyQuit} if this method has already been called. + """ + + +class IExclusiveWorker(IWorker): + """ + Like L{IWorker}, but with the additional guarantee that the callables + passed to C{do} will not be called exclusively with each other. + """ diff --git a/contrib/python/Twisted/py2/twisted/_threads/_memory.py b/contrib/python/Twisted/py2/twisted/_threads/_memory.py new file mode 100644 index 00000000000..2587a70226e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/_threads/_memory.py @@ -0,0 +1,71 @@ +# -*- test-case-name: twisted._threads.test.test_memory -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of an in-memory worker that defers execution. +""" + +from __future__ import absolute_import, division, print_function + +from zope.interface import implementer + +from . import IWorker +from ._convenience import Quit + +NoMoreWork = object() + +@implementer(IWorker) +class MemoryWorker(object): + """ + An L{IWorker} that queues work for later performance. + + @ivar _quit: a flag indicating + @type _quit: L{Quit} + """ + + def __init__(self, pending=list): + """ + Create a L{MemoryWorker}. + """ + self._quit = Quit() + self._pending = pending() + + + def do(self, work): + """ + Queue some work for to perform later; see L{createMemoryWorker}. + + @param work: The work to perform. + """ + self._quit.check() + self._pending.append(work) + + + def quit(self): + """ + Quit this worker. + """ + self._quit.set() + self._pending.append(NoMoreWork) + + + +def createMemoryWorker(): + """ + Create an L{IWorker} that does nothing but defer work, to be performed + later. + + @return: a worker that will enqueue work to perform later, and a callable + that will perform one element of that work. + @rtype: 2-L{tuple} of (L{IWorker}, L{callable}) + """ + def perform(): + if not worker._pending: + return False + if worker._pending[0] is NoMoreWork: + return False + worker._pending.pop(0)() + return True + worker = MemoryWorker() + return (worker, perform) diff --git a/contrib/python/Twisted/py2/twisted/_threads/_pool.py b/contrib/python/Twisted/py2/twisted/_threads/_pool.py new file mode 100644 index 00000000000..043c43ecf84 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/_threads/_pool.py @@ -0,0 +1,69 @@ +# -*- test-case-name: twisted._threads.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Top level thread pool interface, used to implement +L{twisted.python.threadpool}. +""" + +from __future__ import absolute_import, division, print_function + +from threading import Thread, Lock, local as LocalStorage +try: + from Queue import Queue +except ImportError: + from queue import Queue + +from twisted.python.log import err + +from ._threadworker import LockWorker +from ._team import Team +from ._threadworker import ThreadWorker + + +def pool(currentLimit, threadFactory=Thread): + """ + Construct a L{Team} that spawns threads as a thread pool, with the given + limiting function. + + @note: Future maintainers: while the public API for the eventual move to + twisted.threads should look I{something} like this, and while this + function is necessary to implement the API described by + L{twisted.python.threadpool}, I am starting to think the idea of a hard + upper limit on threadpool size is just bad (turning memory performance + issues into correctness issues well before we run into memory + pressure), and instead we should build something with reactor + integration for slowly releasing idle threads when they're not needed + and I{rate} limiting the creation of new threads rather than just + hard-capping it. + + @param currentLimit: a callable that returns the current limit on the + number of workers that the returned L{Team} should create; if it + already has more workers than that value, no new workers will be + created. + @type currentLimit: 0-argument callable returning L{int} + + @param reactor: If passed, the L{IReactorFromThreads} / L{IReactorCore} to + be used to coordinate actions on the L{Team} itself. Otherwise, a + L{LockWorker} will be used. + + @return: a new L{Team}. + """ + + def startThread(target): + return threadFactory(target=target).start() + + def limitedWorkerCreator(): + stats = team.statistics() + if stats.busyWorkerCount + stats.idleWorkerCount >= currentLimit(): + return None + return ThreadWorker(startThread, Queue()) + + team = Team(coordinator=LockWorker(Lock(), LocalStorage()), + createWorker=limitedWorkerCreator, + logException=err) + return team + + + diff --git a/contrib/python/Twisted/py2/twisted/_threads/_team.py b/contrib/python/Twisted/py2/twisted/_threads/_team.py new file mode 100644 index 00000000000..83b777abd34 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/_threads/_team.py @@ -0,0 +1,231 @@ +# -*- test-case-name: twisted._threads.test.test_team -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of a L{Team} of workers; a thread-pool that can allocate work to +workers. +""" + +from __future__ import absolute_import, division, print_function + +from collections import deque +from zope.interface import implementer + +from . import IWorker +from ._convenience import Quit + + + +class Statistics(object): + """ + Statistics about a L{Team}'s current activity. + + @ivar idleWorkerCount: The number of idle workers. + @type idleWorkerCount: L{int} + + @ivar busyWorkerCount: The number of busy workers. + @type busyWorkerCount: L{int} + + @ivar backloggedWorkCount: The number of work items passed to L{Team.do} + which have not yet been sent to a worker to be performed because not + enough workers are available. + @type backloggedWorkCount: L{int} + """ + + def __init__(self, idleWorkerCount, busyWorkerCount, + backloggedWorkCount): + self.idleWorkerCount = idleWorkerCount + self.busyWorkerCount = busyWorkerCount + self.backloggedWorkCount = backloggedWorkCount + + + +@implementer(IWorker) +class Team(object): + """ + A composite L{IWorker} implementation. + + @ivar _quit: A L{Quit} flag indicating whether this L{Team} has been quit + yet. This may be set by an arbitrary thread since L{Team.quit} may be + called from anywhere. + + @ivar _coordinator: the L{IExclusiveWorker} coordinating access to this + L{Team}'s internal resources. + + @ivar _createWorker: a callable that will create new workers. + + @ivar _logException: a 0-argument callable called in an exception context + when there is an unhandled error from a task passed to L{Team.do} + + @ivar _idle: a L{set} of idle workers. + + @ivar _busyCount: the number of workers currently busy. + + @ivar _pending: a C{deque} of tasks - that is, 0-argument callables passed + to L{Team.do} - that are outstanding. + + @ivar _shouldQuitCoordinator: A flag indicating that the coordinator should + be quit at the next available opportunity. Unlike L{Team._quit}, this + flag is only set by the coordinator. + + @ivar _toShrink: the number of workers to shrink this L{Team} by at the + next available opportunity; set in the coordinator. + """ + + def __init__(self, coordinator, createWorker, logException): + """ + @param coordinator: an L{IExclusiveWorker} which will coordinate access + to resources on this L{Team}; that is to say, an + L{IExclusiveWorker} whose C{do} method ensures that its given work + will be executed in a mutually exclusive context, not in parallel + with other work enqueued by C{do} (although possibly in parallel + with the caller). + + @param createWorker: A 0-argument callable that will create an + L{IWorker} to perform work. + + @param logException: A 0-argument callable called in an exception + context when the work passed to C{do} raises an exception. + """ + self._quit = Quit() + self._coordinator = coordinator + self._createWorker = createWorker + self._logException = logException + + # Don't touch these except from the coordinator. + self._idle = set() + self._busyCount = 0 + self._pending = deque() + self._shouldQuitCoordinator = False + self._toShrink = 0 + + + def statistics(self): + """ + Gather information on the current status of this L{Team}. + + @return: a L{Statistics} describing the current state of this L{Team}. + """ + return Statistics(len(self._idle), self._busyCount, len(self._pending)) + + + def grow(self, n): + """ + Increase the the number of idle workers by C{n}. + + @param n: The number of new idle workers to create. + @type n: L{int} + """ + self._quit.check() + @self._coordinator.do + def createOneWorker(): + for x in range(n): + worker = self._createWorker() + if worker is None: + return + self._recycleWorker(worker) + + + def shrink(self, n=None): + """ + Decrease the number of idle workers by C{n}. + + @param n: The number of idle workers to shut down, or L{None} (or + unspecified) to shut down all workers. + @type n: L{int} or L{None} + """ + self._quit.check() + self._coordinator.do(lambda: self._quitIdlers(n)) + + + def _quitIdlers(self, n=None): + """ + The implmentation of C{shrink}, performed by the coordinator worker. + + @param n: see L{Team.shrink} + """ + if n is None: + n = len(self._idle) + self._busyCount + for x in range(n): + if self._idle: + self._idle.pop().quit() + else: + self._toShrink += 1 + if self._shouldQuitCoordinator and self._busyCount == 0: + self._coordinator.quit() + + + def do(self, task): + """ + Perform some work in a worker created by C{createWorker}. + + @param task: the callable to run + """ + self._quit.check() + self._coordinator.do(lambda: self._coordinateThisTask(task)) + + + def _coordinateThisTask(self, task): + """ + Select a worker to dispatch to, either an idle one or a new one, and + perform it. + + This method should run on the coordinator worker. + + @param task: the task to dispatch + @type task: 0-argument callable + """ + worker = (self._idle.pop() if self._idle + else self._createWorker()) + if worker is None: + # The createWorker method may return None if we're out of resources + # to create workers. + self._pending.append(task) + return + self._busyCount += 1 + @worker.do + def doWork(): + try: + task() + except: + self._logException() + + @self._coordinator.do + def idleAndPending(): + self._busyCount -= 1 + self._recycleWorker(worker) + + + def _recycleWorker(self, worker): + """ + Called only from coordinator. + + Recycle the given worker into the idle pool. + + @param worker: a worker created by C{createWorker} and now idle. + @type worker: L{IWorker} + """ + self._idle.add(worker) + if self._pending: + # Re-try the first enqueued thing. + # (Explicitly do _not_ honor _quit.) + self._coordinateThisTask(self._pending.popleft()) + elif self._shouldQuitCoordinator: + self._quitIdlers() + elif self._toShrink > 0: + self._toShrink -= 1 + self._idle.remove(worker) + worker.quit() + + + def quit(self): + """ + Stop doing work and shut down all idle workers. + """ + self._quit.set() + # In case all the workers are idle when we do this. + @self._coordinator.do + def startFinishing(): + self._shouldQuitCoordinator = True + self._quitIdlers() diff --git a/contrib/python/Twisted/py2/twisted/_threads/_threadworker.py b/contrib/python/Twisted/py2/twisted/_threads/_threadworker.py new file mode 100644 index 00000000000..152625a1f87 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/_threads/_threadworker.py @@ -0,0 +1,123 @@ +# -*- test-case-name: twisted._threads.test.test_threadworker -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of an L{IWorker} based on native threads and queues. +""" + +from __future__ import absolute_import, division, print_function + +from zope.interface import implementer +from ._ithreads import IExclusiveWorker +from ._convenience import Quit + + +_stop = object() + +@implementer(IExclusiveWorker) +class ThreadWorker(object): + """ + An L{IExclusiveWorker} implemented based on a single thread and a queue. + + This worker ensures exclusivity (i.e. it is an L{IExclusiveWorker} and not + an L{IWorker}) by performing all of the work passed to C{do} on the I{same} + thread. + """ + + def __init__(self, startThread, queue): + """ + Create a L{ThreadWorker} with a function to start a thread and a queue + to use to communicate with that thread. + + @param startThread: a callable that takes a callable to run in another + thread. + @type startThread: callable taking a 0-argument callable and returning + nothing. + + @param queue: A L{Queue} to use to give tasks to the thread created by + C{startThread}. + @param queue: L{Queue} + """ + self._q = queue + self._hasQuit = Quit() + def work(): + for task in iter(queue.get, _stop): + task() + startThread(work) + + + def do(self, task): + """ + Perform the given task on the thread owned by this L{ThreadWorker}. + + @param task: the function to call on a thread. + """ + self._hasQuit.check() + self._q.put(task) + + + def quit(self): + """ + Reject all future work and stop the thread started by C{__init__}. + """ + # Reject all future work. Set this _before_ enqueueing _stop, so + # that no work is ever enqueued _after_ _stop. + self._hasQuit.set() + self._q.put(_stop) + + + +@implementer(IExclusiveWorker) +class LockWorker(object): + """ + An L{IWorker} implemented based on a mutual-exclusion lock. + """ + + def __init__(self, lock, local): + """ + @param lock: A mutual-exclusion lock, with C{acquire} and C{release} + methods. + @type lock: L{threading.Lock} + + @param local: Local storage. + @type local: L{threading.local} + """ + self._quit = Quit() + self._lock = lock + self._local = local + + + def do(self, work): + """ + Do the given work on this thread, with the mutex acquired. If this is + called re-entrantly, return and wait for the outer invocation to do the + work. + + @param work: the work to do with the lock held. + """ + lock = self._lock + local = self._local + self._quit.check() + working = getattr(local, "working", None) + if working is None: + working = local.working = [] + working.append(work) + lock.acquire() + try: + while working: + working.pop(0)() + finally: + lock.release() + local.working = None + else: + working.append(work) + + + def quit(self): + """ + Quit this L{LockWorker}. + """ + self._quit.set() + self._lock = None + diff --git a/contrib/python/Twisted/py2/twisted/_version.py b/contrib/python/Twisted/py2/twisted/_version.py new file mode 100644 index 00000000000..13073dd4204 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/_version.py @@ -0,0 +1,11 @@ +""" +Provides Twisted version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update Twisted` to change this file. + +from incremental import Version + +__version__ = Version('Twisted', 20, 3, 0) +__all__ = ["__version__"] diff --git a/contrib/python/Twisted/py2/twisted/application/__init__.py b/contrib/python/Twisted/py2/twisted/application/__init__.py new file mode 100644 index 00000000000..a462fa6b24a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Configuration objects for Twisted Applications. +""" diff --git a/contrib/python/Twisted/py2/twisted/application/app.py b/contrib/python/Twisted/py2/twisted/application/app.py new file mode 100644 index 00000000000..02dc3f5dd85 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/app.py @@ -0,0 +1,708 @@ +# -*- test-case-name: twisted.test.test_application,twisted.test.test_twistd -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division, print_function + +import sys +import os +import pdb +import getpass +import traceback +import signal +import warnings + +from operator import attrgetter + +from twisted import copyright, plugin, logger +from twisted.application import service, reactors +from twisted.internet import defer +from twisted.persisted import sob +from twisted.python import runtime, log, usage, failure, util, logfile +from twisted.python._oldstyle import _oldStyle +from twisted.python.reflect import (qual, namedAny, namedModule) +from twisted.internet.interfaces import _ISupportsExitSignalCapturing + +# Expose the new implementation of installReactor at the old location. +from twisted.application.reactors import installReactor +from twisted.application.reactors import NoSuchReactor + + +class _BasicProfiler(object): + """ + @ivar saveStats: if C{True}, save the stats information instead of the + human readable format + @type saveStats: C{bool} + + @ivar profileOutput: the name of the file use to print profile data. + @type profileOutput: C{str} + """ + + def __init__(self, profileOutput, saveStats): + self.profileOutput = profileOutput + self.saveStats = saveStats + + + def _reportImportError(self, module, e): + """ + Helper method to report an import error with a profile module. This + has to be explicit because some of these modules are removed by + distributions due to them being non-free. + """ + s = "Failed to import module %s: %s" % (module, e) + s += """ +This is most likely caused by your operating system not including +the module due to it being non-free. Either do not use the option +--profile, or install the module; your operating system vendor +may provide it in a separate package. +""" + raise SystemExit(s) + + + +class ProfileRunner(_BasicProfiler): + """ + Runner for the standard profile module. + """ + + def run(self, reactor): + """ + Run reactor under the standard profiler. + """ + try: + import profile + except ImportError as e: + self._reportImportError("profile", e) + + p = profile.Profile() + p.runcall(reactor.run) + if self.saveStats: + p.dump_stats(self.profileOutput) + else: + tmp, sys.stdout = sys.stdout, open(self.profileOutput, 'a') + try: + p.print_stats() + finally: + sys.stdout, tmp = tmp, sys.stdout + tmp.close() + + + +class CProfileRunner(_BasicProfiler): + """ + Runner for the cProfile module. + """ + + def run(self, reactor): + """ + Run reactor under the cProfile profiler. + """ + try: + import cProfile + import pstats + except ImportError as e: + self._reportImportError("cProfile", e) + + p = cProfile.Profile() + p.runcall(reactor.run) + if self.saveStats: + p.dump_stats(self.profileOutput) + else: + with open(self.profileOutput, 'w') as stream: + s = pstats.Stats(p, stream=stream) + s.strip_dirs() + s.sort_stats(-1) + s.print_stats() + + + +class AppProfiler(object): + """ + Class which selects a specific profile runner based on configuration + options. + + @ivar profiler: the name of the selected profiler. + @type profiler: C{str} + """ + profilers = {"profile": ProfileRunner, "cprofile": CProfileRunner} + + def __init__(self, options): + saveStats = options.get("savestats", False) + profileOutput = options.get("profile", None) + self.profiler = options.get("profiler", "cprofile").lower() + if self.profiler in self.profilers: + profiler = self.profilers[self.profiler](profileOutput, saveStats) + self.run = profiler.run + else: + raise SystemExit("Unsupported profiler name: %s" % + (self.profiler,)) + + + +class AppLogger(object): + """ + An L{AppLogger} attaches the configured log observer specified on the + commandline to a L{ServerOptions} object, a custom L{logger.ILogObserver}, + or a legacy custom {log.ILogObserver}. + + @ivar _logfilename: The name of the file to which to log, if other than the + default. + @type _logfilename: C{str} + + @ivar _observerFactory: Callable object that will create a log observer, or + None. + + @ivar _observer: log observer added at C{start} and removed at C{stop}. + @type _observer: a callable that implements L{logger.ILogObserver} or + L{log.ILogObserver}. + """ + _observer = None + + def __init__(self, options): + """ + Initialize an L{AppLogger} with a L{ServerOptions}. + """ + self._logfilename = options.get("logfile", "") + self._observerFactory = options.get("logger") or None + + + def start(self, application): + """ + Initialize the global logging system for the given application. + + If a custom logger was specified on the command line it will be used. + If not, and an L{logger.ILogObserver} or legacy L{log.ILogObserver} + component has been set on C{application}, then it will be used as the + log observer. Otherwise a log observer will be created based on the + command line options for built-in loggers (e.g. C{--logfile}). + + @param application: The application on which to check for an + L{logger.ILogObserver} or legacy L{log.ILogObserver}. + @type application: L{twisted.python.components.Componentized} + """ + if self._observerFactory is not None: + observer = self._observerFactory() + else: + observer = application.getComponent(logger.ILogObserver, None) + if observer is None: + # If there's no new ILogObserver, try the legacy one + observer = application.getComponent(log.ILogObserver, None) + + if observer is None: + observer = self._getLogObserver() + self._observer = observer + + if logger.ILogObserver.providedBy(self._observer): + observers = [self._observer] + elif log.ILogObserver.providedBy(self._observer): + observers = [logger.LegacyLogObserverWrapper(self._observer)] + else: + warnings.warn( + ("Passing a logger factory which makes log observers which do " + "not implement twisted.logger.ILogObserver or " + "twisted.python.log.ILogObserver to " + "twisted.application.app.AppLogger was deprecated in " + "Twisted 16.2. Please use a factory that produces " + "twisted.logger.ILogObserver (or the legacy " + "twisted.python.log.ILogObserver) implementing objects " + "instead."), + DeprecationWarning, + stacklevel=2) + observers = [logger.LegacyLogObserverWrapper(self._observer)] + + logger.globalLogBeginner.beginLoggingTo(observers) + self._initialLog() + + + def _initialLog(self): + """ + Print twistd start log message. + """ + from twisted.internet import reactor + logger._loggerFor(self).info( + "twistd {version} ({exe} {pyVersion}) starting up.", + version=copyright.version, exe=sys.executable, + pyVersion=runtime.shortPythonVersion()) + logger._loggerFor(self).info('reactor class: {reactor}.', + reactor=qual(reactor.__class__)) + + + def _getLogObserver(self): + """ + Create a log observer to be added to the logging system before running + this application. + """ + if self._logfilename == '-' or not self._logfilename: + logFile = sys.stdout + else: + logFile = logfile.LogFile.fromFullPath(self._logfilename) + return logger.textFileLogObserver(logFile) + + + def stop(self): + """ + Remove all log observers previously set up by L{AppLogger.start}. + """ + logger._loggerFor(self).info("Server Shut Down.") + if self._observer is not None: + logger.globalLogPublisher.removeObserver(self._observer) + self._observer = None + + + +def fixPdb(): + def do_stop(self, arg): + self.clear_all_breaks() + self.set_continue() + from twisted.internet import reactor + reactor.callLater(0, reactor.stop) + return 1 + + + def help_stop(self): + print("stop - Continue execution, then cleanly shutdown the twisted " + "reactor.") + + + def set_quit(self): + os._exit(0) + + pdb.Pdb.set_quit = set_quit + pdb.Pdb.do_stop = do_stop + pdb.Pdb.help_stop = help_stop + + + +def runReactorWithLogging(config, oldstdout, oldstderr, profiler=None, + reactor=None): + """ + Start the reactor, using profiling if specified by the configuration, and + log any error happening in the process. + + @param config: configuration of the twistd application. + @type config: L{ServerOptions} + + @param oldstdout: initial value of C{sys.stdout}. + @type oldstdout: C{file} + + @param oldstderr: initial value of C{sys.stderr}. + @type oldstderr: C{file} + + @param profiler: object used to run the reactor with profiling. + @type profiler: L{AppProfiler} + + @param reactor: The reactor to use. If L{None}, the global reactor will + be used. + """ + if reactor is None: + from twisted.internet import reactor + try: + if config['profile']: + if profiler is not None: + profiler.run(reactor) + elif config['debug']: + sys.stdout = oldstdout + sys.stderr = oldstderr + if runtime.platformType == 'posix': + signal.signal(signal.SIGUSR2, lambda *args: pdb.set_trace()) + signal.signal(signal.SIGINT, lambda *args: pdb.set_trace()) + fixPdb() + pdb.runcall(reactor.run) + else: + reactor.run() + except: + close = False + if config['nodaemon']: + file = oldstdout + else: + file = open("TWISTD-CRASH.log", "a") + close = True + try: + traceback.print_exc(file=file) + file.flush() + finally: + if close: + file.close() + + + +def getPassphrase(needed): + if needed: + return getpass.getpass('Passphrase: ') + else: + return None + + + +def getSavePassphrase(needed): + if needed: + return util.getPassword("Encryption passphrase: ") + else: + return None + + + +class ApplicationRunner(object): + """ + An object which helps running an application based on a config object. + + Subclass me and implement preApplication and postApplication + methods. postApplication generally will want to run the reactor + after starting the application. + + @ivar config: The config object, which provides a dict-like interface. + + @ivar application: Available in postApplication, but not + preApplication. This is the application object. + + @ivar profilerFactory: Factory for creating a profiler object, able to + profile the application if options are set accordingly. + + @ivar profiler: Instance provided by C{profilerFactory}. + + @ivar loggerFactory: Factory for creating object responsible for logging. + + @ivar logger: Instance provided by C{loggerFactory}. + """ + profilerFactory = AppProfiler + loggerFactory = AppLogger + + def __init__(self, config): + self.config = config + self.profiler = self.profilerFactory(config) + self.logger = self.loggerFactory(config) + + + def run(self): + """ + Run the application. + """ + self.preApplication() + self.application = self.createOrGetApplication() + + self.logger.start(self.application) + + self.postApplication() + self.logger.stop() + + + def startReactor(self, reactor, oldstdout, oldstderr): + """ + Run the reactor with the given configuration. Subclasses should + probably call this from C{postApplication}. + + @see: L{runReactorWithLogging} + """ + if reactor is None: + from twisted.internet import reactor + runReactorWithLogging( + self.config, oldstdout, oldstderr, self.profiler, reactor) + + if _ISupportsExitSignalCapturing.providedBy(reactor): + self._exitSignal = reactor._exitSignal + else: + self._exitSignal = None + + + def preApplication(self): + """ + Override in subclass. + + This should set up any state necessary before loading and + running the Application. + """ + raise NotImplementedError() + + + def postApplication(self): + """ + Override in subclass. + + This will be called after the application has been loaded (so + the C{application} attribute will be set). Generally this + should start the application and run the reactor. + """ + raise NotImplementedError() + + + def createOrGetApplication(self): + """ + Create or load an Application based on the parameters found in the + given L{ServerOptions} instance. + + If a subcommand was used, the L{service.IServiceMaker} that it + represents will be used to construct a service to be added to + a newly-created Application. + + Otherwise, an application will be loaded based on parameters in + the config. + """ + if self.config.subCommand: + # If a subcommand was given, it's our responsibility to create + # the application, instead of load it from a file. + + # loadedPlugins is set up by the ServerOptions.subCommands + # property, which is iterated somewhere in the bowels of + # usage.Options. + plg = self.config.loadedPlugins[self.config.subCommand] + ser = plg.makeService(self.config.subOptions) + application = service.Application(plg.tapname) + ser.setServiceParent(application) + else: + passphrase = getPassphrase(self.config['encrypted']) + application = getApplication(self.config, passphrase) + return application + + + +def getApplication(config, passphrase): + s = [(config[t], t) + for t in ['python', 'source', 'file'] if config[t]][0] + filename, style = s[0], {'file': 'pickle'}.get(s[1], s[1]) + try: + log.msg("Loading %s..." % filename) + application = service.loadApplication(filename, style, passphrase) + log.msg("Loaded.") + except Exception as e: + s = "Failed to load application: %s" % e + if isinstance(e, KeyError) and e.args[0] == "application": + s += """ +Could not find 'application' in the file. To use 'twistd -y', your .tac +file must create a suitable object (e.g., by calling service.Application()) +and store it in a variable named 'application'. twistd loads your .tac file +and scans the global variables for one of this name. + +Please read the 'Using Application' HOWTO for details. +""" + traceback.print_exc(file=log.logfile) + log.msg(s) + log.deferr() + sys.exit('\n' + s + '\n') + return application + + + +def _reactorAction(): + return usage.CompleteList([r.shortName for r in + reactors.getReactorTypes()]) + + + +@_oldStyle +class ReactorSelectionMixin: + """ + Provides options for selecting a reactor to install. + + If a reactor is installed, the short name which was used to locate it is + saved as the value for the C{"reactor"} key. + """ + compData = usage.Completions( + optActions={"reactor": _reactorAction}) + + messageOutput = sys.stdout + _getReactorTypes = staticmethod(reactors.getReactorTypes) + + + def opt_help_reactors(self): + """ + Display a list of possibly available reactor names. + """ + rcts = sorted(self._getReactorTypes(), key=attrgetter('shortName')) + notWorkingReactors = "" + for r in rcts: + try: + namedModule(r.moduleName) + self.messageOutput.write(' %-4s\t%s\n' % + (r.shortName, r.description)) + except ImportError as e: + notWorkingReactors += (' !%-4s\t%s (%s)\n' % + (r.shortName, r.description, e.args[0])) + + if notWorkingReactors: + self.messageOutput.write('\n') + self.messageOutput.write(' reactors not available ' + 'on this platform:\n\n') + self.messageOutput.write(notWorkingReactors) + raise SystemExit(0) + + + def opt_reactor(self, shortName): + """ + Which reactor to use (see --help-reactors for a list of possibilities) + """ + # Actually actually actually install the reactor right at this very + # moment, before any other code (for example, a sub-command plugin) + # runs and accidentally imports and installs the default reactor. + # + # This could probably be improved somehow. + try: + installReactor(shortName) + except NoSuchReactor: + msg = ("The specified reactor does not exist: '%s'.\n" + "See the list of available reactors with " + "--help-reactors" % (shortName,)) + raise usage.UsageError(msg) + except Exception as e: + msg = ("The specified reactor cannot be used, failed with error: " + "%s.\nSee the list of available reactors with " + "--help-reactors" % (e,)) + raise usage.UsageError(msg) + else: + self["reactor"] = shortName + opt_r = opt_reactor + + + +class ServerOptions(usage.Options, ReactorSelectionMixin): + + longdesc = ("twistd reads a twisted.application.service.Application out " + "of a file and runs it.") + + optFlags = [['savestats', None, + "save the Stats object rather than the text output of " + "the profiler."], + ['no_save', 'o', "do not save state on shutdown"], + ['encrypted', 'e', + "The specified tap/aos file is encrypted."]] + + optParameters = [['logfile', 'l', None, + "log to a specified file, - for stdout"], + ['logger', None, None, + "A fully-qualified name to a log observer factory to " + "use for the initial log observer. Takes precedence " + "over --logfile and --syslog (when available)."], + ['profile', 'p', None, + "Run in profile mode, dumping results to specified " + "file."], + ['profiler', None, "cprofile", + "Name of the profiler to use (%s)." % + ", ".join(AppProfiler.profilers)], + ['file', 'f', 'twistd.tap', + "read the given .tap file"], + ['python', 'y', None, + "read an application from within a Python file " + "(implies -o)"], + ['source', 's', None, + "Read an application from a .tas file (AOT format)."], + ['rundir', 'd', '.', + 'Change to a supplied directory before running']] + + compData = usage.Completions( + mutuallyExclusive=[("file", "python", "source")], + optActions={"file": usage.CompleteFiles("*.tap"), + "python": usage.CompleteFiles("*.(tac|py)"), + "source": usage.CompleteFiles("*.tas"), + "rundir": usage.CompleteDirs()} + ) + + _getPlugins = staticmethod(plugin.getPlugins) + + def __init__(self, *a, **kw): + self['debug'] = False + if 'stdout' in kw: + self.stdout = kw['stdout'] + else: + self.stdout = sys.stdout + usage.Options.__init__(self) + + + def opt_debug(self): + """ + Run the application in the Python Debugger (implies nodaemon), + sending SIGUSR2 will drop into debugger + """ + defer.setDebugging(True) + failure.startDebugMode() + self['debug'] = True + opt_b = opt_debug + + + def opt_spew(self): + """ + Print an insanely verbose log of everything that happens. + Useful when debugging freezes or locks in complex code. + """ + sys.settrace(util.spewer) + try: + import threading + except ImportError: + return + threading.settrace(util.spewer) + + + def parseOptions(self, options=None): + if options is None: + options = sys.argv[1:] or ["--help"] + usage.Options.parseOptions(self, options) + + + def postOptions(self): + if self.subCommand or self['python']: + self['no_save'] = True + if self['logger'] is not None: + try: + self['logger'] = namedAny(self['logger']) + except Exception as e: + raise usage.UsageError("Logger '%s' could not be imported: %s" + % (self['logger'], e)) + + + def subCommands(self): + plugins = self._getPlugins(service.IServiceMaker) + self.loadedPlugins = {} + for plug in sorted(plugins, key=attrgetter('tapname')): + self.loadedPlugins[plug.tapname] = plug + yield (plug.tapname, + None, + # Avoid resolving the options attribute right away, in case + # it's a property with a non-trivial getter (eg, one which + # imports modules). + lambda plug=plug: plug.options(), + plug.description) + subCommands = property(subCommands) + + + +def run(runApp, ServerOptions): + config = ServerOptions() + try: + config.parseOptions() + except usage.error as ue: + print(config) + print("%s: %s" % (sys.argv[0], ue)) + else: + runApp(config) + + + +def convertStyle(filein, typein, passphrase, fileout, typeout, encrypt): + application = service.loadApplication(filein, typein, passphrase) + sob.IPersistable(application).setStyle(typeout) + passphrase = getSavePassphrase(encrypt) + if passphrase: + fileout = None + sob.IPersistable(application).save(filename=fileout, passphrase=passphrase) + + + +def startApplication(application, save): + from twisted.internet import reactor + service.IService(application).startService() + if save: + p = sob.IPersistable(application) + reactor.addSystemEventTrigger('after', 'shutdown', p.save, 'shutdown') + reactor.addSystemEventTrigger('before', 'shutdown', + service.IService(application).stopService) + + + +def _exitWithSignal(sig): + """ + Force the application to terminate with the specified signal by replacing + the signal handler with the default and sending the signal to ourselves. + + @param sig: Signal to use to terminate the process with C{os.kill}. + @type sig: C{int} + """ + signal.signal(sig, signal.SIG_DFL) + os.kill(os.getpid(), sig) diff --git a/contrib/python/Twisted/py2/twisted/application/internet.py b/contrib/python/Twisted/py2/twisted/application/internet.py new file mode 100644 index 00000000000..a8582f4ebe7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/internet.py @@ -0,0 +1,1157 @@ +# -*- test-case-name: twisted.application.test.test_internet,twisted.test.test_application,twisted.test.test_cooperator -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Reactor-based Services + +Here are services to run clients, servers and periodic services using +the reactor. + +If you want to run a server service, L{StreamServerEndpointService} defines a +service that can wrap an arbitrary L{IStreamServerEndpoint +} +as an L{IService}. See also L{twisted.application.strports.service} for +constructing one of these directly from a descriptive string. + +Additionally, this module (dynamically) defines various Service subclasses that +let you represent clients and servers in a Service hierarchy. Endpoints APIs +should be preferred for stream server services, but since those APIs do not yet +exist for clients or datagram services, many of these are still useful. + +They are as follows:: + + TCPServer, TCPClient, + UNIXServer, UNIXClient, + SSLServer, SSLClient, + UDPServer, + UNIXDatagramServer, UNIXDatagramClient, + MulticastServer + +These classes take arbitrary arguments in their constructors and pass +them straight on to their respective reactor.listenXXX or +reactor.connectXXX calls. + +For example, the following service starts a web server on port 8080: +C{TCPServer(8080, server.Site(r))}. See the documentation for the +reactor.listen/connect* methods for more information. +""" + +from __future__ import absolute_import, division + +from random import random as _goodEnoughRandom + +from twisted.python import log +from twisted.logger import Logger + +from twisted.application import service +from twisted.internet import task +from twisted.python.failure import Failure +from twisted.internet.defer import ( + CancelledError, Deferred, succeed, fail, maybeDeferred +) + +from automat import MethodicalMachine + + +def _maybeGlobalReactor(maybeReactor): + """ + @return: the argument, or the global reactor if the argument is L{None}. + """ + if maybeReactor is None: + from twisted.internet import reactor + return reactor + else: + return maybeReactor + + + +class _VolatileDataService(service.Service): + + volatile = [] + + def __getstate__(self): + d = service.Service.__getstate__(self) + for attr in self.volatile: + if attr in d: + del d[attr] + return d + + + +class _AbstractServer(_VolatileDataService): + """ + @cvar volatile: list of attribute to remove from pickling. + @type volatile: C{list} + + @ivar method: the type of method to call on the reactor, one of B{TCP}, + B{UDP}, B{SSL} or B{UNIX}. + @type method: C{str} + + @ivar reactor: the current running reactor. + @type reactor: a provider of C{IReactorTCP}, C{IReactorUDP}, + C{IReactorSSL} or C{IReactorUnix}. + + @ivar _port: instance of port set when the service is started. + @type _port: a provider of L{twisted.internet.interfaces.IListeningPort}. + """ + + volatile = ['_port'] + method = None + reactor = None + + _port = None + + def __init__(self, *args, **kwargs): + self.args = args + if 'reactor' in kwargs: + self.reactor = kwargs.pop("reactor") + self.kwargs = kwargs + + + def privilegedStartService(self): + service.Service.privilegedStartService(self) + self._port = self._getPort() + + + def startService(self): + service.Service.startService(self) + if self._port is None: + self._port = self._getPort() + + + def stopService(self): + service.Service.stopService(self) + # TODO: if startup failed, should shutdown skip stopListening? + # _port won't exist + if self._port is not None: + d = self._port.stopListening() + del self._port + return d + + + def _getPort(self): + """ + Wrapper around the appropriate listen method of the reactor. + + @return: the port object returned by the listen method. + @rtype: an object providing + L{twisted.internet.interfaces.IListeningPort}. + """ + return getattr(_maybeGlobalReactor(self.reactor), + 'listen%s' % (self.method,))(*self.args, **self.kwargs) + + + +class _AbstractClient(_VolatileDataService): + """ + @cvar volatile: list of attribute to remove from pickling. + @type volatile: C{list} + + @ivar method: the type of method to call on the reactor, one of B{TCP}, + B{UDP}, B{SSL} or B{UNIX}. + @type method: C{str} + + @ivar reactor: the current running reactor. + @type reactor: a provider of C{IReactorTCP}, C{IReactorUDP}, + C{IReactorSSL} or C{IReactorUnix}. + + @ivar _connection: instance of connection set when the service is started. + @type _connection: a provider of L{twisted.internet.interfaces.IConnector}. + """ + + volatile = ['_connection'] + method = None + reactor = None + + _connection = None + + def __init__(self, *args, **kwargs): + self.args = args + if 'reactor' in kwargs: + self.reactor = kwargs.pop("reactor") + self.kwargs = kwargs + + + def startService(self): + service.Service.startService(self) + self._connection = self._getConnection() + + + def stopService(self): + service.Service.stopService(self) + if self._connection is not None: + self._connection.disconnect() + del self._connection + + + def _getConnection(self): + """ + Wrapper around the appropriate connect method of the reactor. + + @return: the port object returned by the connect method. + @rtype: an object providing L{twisted.internet.interfaces.IConnector}. + """ + return getattr(_maybeGlobalReactor(self.reactor), + 'connect%s' % (self.method,))(*self.args, **self.kwargs) + + + +_doc={ +'Client': +"""Connect to %(tran)s + +Call reactor.connect%(tran)s when the service starts, with the +arguments given to the constructor. +""", +'Server': +"""Serve %(tran)s clients + +Call reactor.listen%(tran)s when the service starts, with the +arguments given to the constructor. When the service stops, +stop listening. See twisted.internet.interfaces for documentation +on arguments to the reactor method. +""", +} + +for tran in 'TCP UNIX SSL UDP UNIXDatagram Multicast'.split(): + for side in 'Server Client'.split(): + if tran == "Multicast" and side == "Client": + continue + if tran == "UDP" and side == "Client": + continue + base = globals()['_Abstract'+side] + doc = _doc[side] % vars() + + klass = type(tran+side, (base,), {'method': tran, '__doc__': doc}) + globals()[tran+side] = klass + + + +class TimerService(_VolatileDataService): + """ + Service to periodically call a function + + Every C{step} seconds call the given function with the given arguments. + The service starts the calls when it starts, and cancels them + when it stops. + + @ivar clock: Source of time. This defaults to L{None} which is + causes L{twisted.internet.reactor} to be used. + Feel free to set this to something else, but it probably ought to be + set *before* calling L{startService}. + @type clock: L{IReactorTime} + + @ivar call: Function and arguments to call periodically. + @type call: L{tuple} of C{(callable, args, kwargs)} + """ + + volatile = ['_loop', '_loopFinished'] + + def __init__(self, step, callable, *args, **kwargs): + """ + @param step: The number of seconds between calls. + @type step: L{float} + + @param callable: Function to call + @type callable: L{callable} + + @param args: Positional arguments to pass to function + @param kwargs: Keyword arguments to pass to function + """ + self.step = step + self.call = (callable, args, kwargs) + self.clock = None + + def startService(self): + service.Service.startService(self) + callable, args, kwargs = self.call + # we have to make a new LoopingCall each time we're started, because + # an active LoopingCall remains active when serialized. If + # LoopingCall were a _VolatileDataService, we wouldn't need to do + # this. + self._loop = task.LoopingCall(callable, *args, **kwargs) + self._loop.clock = _maybeGlobalReactor(self.clock) + self._loopFinished = self._loop.start(self.step, now=True) + self._loopFinished.addErrback(self._failed) + + def _failed(self, why): + # make a note that the LoopingCall is no longer looping, so we don't + # try to shut it down a second time in stopService. I think this + # should be in LoopingCall. -warner + self._loop.running = False + log.err(why) + + def stopService(self): + """ + Stop the service. + + @rtype: L{Deferred} + @return: a L{Deferred} which is fired when the + currently running call (if any) is finished. + """ + if self._loop.running: + self._loop.stop() + self._loopFinished.addCallback(lambda _: + service.Service.stopService(self)) + return self._loopFinished + + + +class CooperatorService(service.Service): + """ + Simple L{service.IService} which starts and stops a L{twisted.internet.task.Cooperator}. + """ + def __init__(self): + self.coop = task.Cooperator(started=False) + + + def coiterate(self, iterator): + return self.coop.coiterate(iterator) + + + def startService(self): + self.coop.start() + + + def stopService(self): + self.coop.stop() + + + +class StreamServerEndpointService(service.Service, object): + """ + A L{StreamServerEndpointService} is an L{IService} which runs a server on a + listening port described by an L{IStreamServerEndpoint + }. + + @ivar factory: A server factory which will be used to listen on the + endpoint. + + @ivar endpoint: An L{IStreamServerEndpoint + } provider + which will be used to listen when the service starts. + + @ivar _waitingForPort: a Deferred, if C{listen} has yet been invoked on the + endpoint, otherwise None. + + @ivar _raiseSynchronously: Defines error-handling behavior for the case + where C{listen(...)} raises an exception before C{startService} or + C{privilegedStartService} have completed. + + @type _raiseSynchronously: C{bool} + + @since: 10.2 + """ + + _raiseSynchronously = False + + def __init__(self, endpoint, factory): + self.endpoint = endpoint + self.factory = factory + self._waitingForPort = None + + + def privilegedStartService(self): + """ + Start listening on the endpoint. + """ + service.Service.privilegedStartService(self) + self._waitingForPort = self.endpoint.listen(self.factory) + raisedNow = [] + def handleIt(err): + if self._raiseSynchronously: + raisedNow.append(err) + elif not err.check(CancelledError): + log.err(err) + self._waitingForPort.addErrback(handleIt) + if raisedNow: + raisedNow[0].raiseException() + self._raiseSynchronously = False + + + def startService(self): + """ + Start listening on the endpoint, unless L{privilegedStartService} got + around to it already. + """ + service.Service.startService(self) + if self._waitingForPort is None: + self.privilegedStartService() + + + def stopService(self): + """ + Stop listening on the port if it is already listening, otherwise, + cancel the attempt to listen. + + @return: a L{Deferred} which fires + with L{None} when the port has stopped listening. + """ + self._waitingForPort.cancel() + def stopIt(port): + if port is not None: + return port.stopListening() + d = self._waitingForPort.addCallback(stopIt) + def stop(passthrough): + self.running = False + return passthrough + d.addBoth(stop) + return d + + + +class _ReconnectingProtocolProxy(object): + """ + A proxy for a Protocol to provide connectionLost notification to a client + connection service, in support of reconnecting when connections are lost. + """ + + def __init__(self, protocol, lostNotification): + """ + Create a L{_ReconnectingProtocolProxy}. + + @param protocol: the application-provided L{interfaces.IProtocol} + provider. + @type protocol: provider of L{interfaces.IProtocol} which may + additionally provide L{interfaces.IHalfCloseableProtocol} and + L{interfaces.IFileDescriptorReceiver}. + + @param lostNotification: a 1-argument callable to invoke with the + C{reason} when the connection is lost. + """ + self._protocol = protocol + self._lostNotification = lostNotification + + + def connectionLost(self, reason): + """ + The connection was lost. Relay this information. + + @param reason: The reason the connection was lost. + + @return: the underlying protocol's result + """ + try: + return self._protocol.connectionLost(reason) + finally: + self._lostNotification(reason) + + + def __getattr__(self, item): + return getattr(self._protocol, item) + + + def __repr__(self): + return '<%s wrapping %r>' % ( + self.__class__.__name__, self._protocol) + + + +class _DisconnectFactory(object): + """ + A L{_DisconnectFactory} is a proxy for L{IProtocolFactory} that catches + C{connectionLost} notifications and relays them. + """ + + def __init__(self, protocolFactory, protocolDisconnected): + self._protocolFactory = protocolFactory + self._protocolDisconnected = protocolDisconnected + + + def buildProtocol(self, addr): + """ + Create a L{_ReconnectingProtocolProxy} with the disconnect-notification + callback we were called with. + + @param addr: The address the connection is coming from. + + @return: a L{_ReconnectingProtocolProxy} for a protocol produced by + C{self._protocolFactory} + """ + return _ReconnectingProtocolProxy( + self._protocolFactory.buildProtocol(addr), + self._protocolDisconnected + ) + + + def __getattr__(self, item): + return getattr(self._protocolFactory, item) + + + def __repr__(self): + return '<%s wrapping %r>' % ( + self.__class__.__name__, self._protocolFactory) + + + +def backoffPolicy(initialDelay=1.0, maxDelay=60.0, factor=1.5, + jitter=_goodEnoughRandom): + """ + A timeout policy for L{ClientService} which computes an exponential backoff + interval with configurable parameters. + + @since: 16.1.0 + + @param initialDelay: Delay for the first reconnection attempt (default + 1.0s). + @type initialDelay: L{float} + + @param maxDelay: Maximum number of seconds between connection attempts + (default 60 seconds, or one minute). Note that this value is before + jitter is applied, so the actual maximum possible delay is this value + plus the maximum possible result of C{jitter()}. + @type maxDelay: L{float} + + @param factor: A multiplicative factor by which the delay grows on each + failed reattempt. Default: 1.5. + @type factor: L{float} + + @param jitter: A 0-argument callable that introduces noise into the delay. + By default, C{random.random}, i.e. a pseudorandom floating-point value + between zero and one. + @type jitter: 0-argument callable returning L{float} + + @return: a 1-argument callable that, given an attempt count, returns a + floating point number; the number of seconds to delay. + @rtype: see L{ClientService.__init__}'s C{retryPolicy} argument. + """ + def policy(attempt): + try: + delay = min(initialDelay * (factor ** min(100, attempt)), maxDelay) + except OverflowError: + delay = maxDelay + return delay + jitter() + return policy + + + +_defaultPolicy = backoffPolicy() + + +def _firstResult(gen): + """ + Return the first element of a generator and exhaust it. + + C{MethodicalMachine.upon}'s C{collector} argument takes a generator of + output results. If the generator is exhausted, the later outputs aren't + actually run. + + @param gen: Generator to extract values from + + @return: The first element of the generator. + """ + return list(gen)[0] + + + +class _ClientMachine(object): + """ + State machine for maintaining a single outgoing connection to an endpoint. + + @see: L{ClientService} + """ + + _machine = MethodicalMachine() + + def __init__(self, endpoint, factory, retryPolicy, clock, + prepareConnection, log): + """ + @see: L{ClientService.__init__} + + @param log: The logger for the L{ClientService} instance this state + machine is associated to. + @type log: L{Logger} + + @ivar _awaitingConnected: notifications to make when connection + succeeds, fails, or is cancelled + @type _awaitingConnected: list of (Deferred, count) tuples + """ + self._endpoint = endpoint + self._failedAttempts = 0 + self._stopped = False + self._factory = factory + self._timeoutForAttempt = retryPolicy + self._clock = clock + self._prepareConnection = prepareConnection + self._connectionInProgress = succeed(None) + + self._awaitingConnected = [] + + self._stopWaiters = [] + self._log = log + + + @_machine.state(initial=True) + def _init(self): + """ + The service has not been started. + """ + + @_machine.state() + def _connecting(self): + """ + The service has started connecting. + """ + + @_machine.state() + def _waiting(self): + """ + The service is waiting for the reconnection period + before reconnecting. + """ + + @_machine.state() + def _connected(self): + """ + The service is connected. + """ + + @_machine.state() + def _disconnecting(self): + """ + The service is disconnecting after being asked to shutdown. + """ + + @_machine.state() + def _restarting(self): + """ + The service is disconnecting and has been asked to restart. + """ + + @_machine.state() + def _stopped(self): + """ + The service has been stopped and is disconnected. + """ + + @_machine.input() + def start(self): + """ + Start this L{ClientService}, initiating the connection retry loop. + """ + + @_machine.output() + def _connect(self): + """ + Start a connection attempt. + """ + factoryProxy = _DisconnectFactory(self._factory, + lambda _: self._clientDisconnected()) + + self._connectionInProgress = ( + self._endpoint.connect(factoryProxy) + .addCallback(self._runPrepareConnection) + .addCallback(self._connectionMade) + .addErrback(self._connectionFailed)) + + + def _runPrepareConnection(self, protocol): + """ + Run any C{prepareConnection} callback with the connected protocol, + ignoring its return value but propagating any failure. + + @param protocol: The protocol of the connection. + @type protocol: L{IProtocol} + + @return: Either: + + - A L{Deferred} that succeeds with the protocol when the + C{prepareConnection} callback has executed successfully. + + - A L{Deferred} that fails when the C{prepareConnection} callback + throws or returns a failed L{Deferred}. + + - The protocol, when no C{prepareConnection} callback is defined. + """ + if self._prepareConnection: + return (maybeDeferred(self._prepareConnection, protocol) + .addCallback(lambda _: protocol)) + return protocol + + + @_machine.output() + def _resetFailedAttempts(self): + """ + Reset the number of failed attempts. + """ + self._failedAttempts = 0 + + + @_machine.input() + def stop(self): + """ + Stop trying to connect and disconnect any current connection. + + @return: a L{Deferred} that fires when all outstanding connections are + closed and all in-progress connection attempts halted. + """ + + @_machine.output() + def _waitForStop(self): + """ + Return a deferred that will fire when the service has finished + disconnecting. + + @return: L{Deferred} that fires when the service has finished + disconnecting. + """ + self._stopWaiters.append(Deferred()) + return self._stopWaiters[-1] + + + @_machine.output() + def _stopConnecting(self): + """ + Stop pending connection attempt. + """ + self._connectionInProgress.cancel() + + + @_machine.output() + def _stopRetrying(self): + """ + Stop pending attempt to reconnect. + """ + self._retryCall.cancel() + del self._retryCall + + + @_machine.output() + def _disconnect(self): + """ + Disconnect the current connection. + """ + self._currentConnection.transport.loseConnection() + + + @_machine.input() + def _connectionMade(self, protocol): + """ + A connection has been made. + + @param protocol: The protocol of the connection. + @type protocol: L{IProtocol} + """ + + @_machine.output() + def _notifyWaiters(self, protocol): + """ + Notify all pending requests for a connection that a connection has been + made. + + @param protocol: The protocol of the connection. + @type protocol: L{IProtocol} + """ + # This should be in _resetFailedAttempts but the signature doesn't + # match. + self._failedAttempts = 0 + + self._currentConnection = protocol._protocol + self._unawait(self._currentConnection) + + + @_machine.input() + def _connectionFailed(self, f): + """ + The current connection attempt failed. + """ + + + @_machine.output() + def _wait(self): + """ + Schedule a retry attempt. + """ + self._doWait() + + @_machine.output() + def _ignoreAndWait(self, f): + """ + Schedule a retry attempt, and ignore the Failure passed in. + """ + return self._doWait() + + def _doWait(self): + self._failedAttempts += 1 + delay = self._timeoutForAttempt(self._failedAttempts) + self._log.info("Scheduling retry {attempt} to connect {endpoint} " + "in {delay} seconds.", attempt=self._failedAttempts, + endpoint=self._endpoint, delay=delay) + self._retryCall = self._clock.callLater(delay, self._reconnect) + + + @_machine.input() + def _reconnect(self): + """ + The wait between connection attempts is done. + """ + + @_machine.input() + def _clientDisconnected(self): + """ + The current connection has been disconnected. + """ + + @_machine.output() + def _forgetConnection(self): + """ + Forget the current connection. + """ + del self._currentConnection + + + @_machine.output() + def _cancelConnectWaiters(self): + """ + Notify all pending requests for a connection that no more connections + are expected. + """ + self._unawait(Failure(CancelledError())) + + @_machine.output() + def _ignoreAndCancelConnectWaiters(self, f): + """ + Notify all pending requests for a connection that no more connections + are expected, after ignoring the Failure passed in. + """ + self._unawait(Failure(CancelledError())) + + + @_machine.output() + def _finishStopping(self): + """ + Notify all deferreds waiting on the service stopping. + """ + self._doFinishStopping() + + @_machine.output() + def _ignoreAndFinishStopping(self, f): + """ + Notify all deferreds waiting on the service stopping, and ignore the + Failure passed in. + """ + self._doFinishStopping() + + def _doFinishStopping(self): + self._stopWaiters, waiting = [], self._stopWaiters + for w in waiting: + w.callback(None) + + + @_machine.input() + def whenConnected(self, failAfterFailures=None): + """ + Retrieve the currently-connected L{Protocol}, or the next one to + connect. + + @param failAfterFailures: number of connection failures after which + the Deferred will deliver a Failure (None means the Deferred will + only fail if/when the service is stopped). Set this to 1 to make + the very first connection failure signal an error. Use 2 to + allow one failure but signal an error if the subsequent retry + then fails. + @type failAfterFailures: L{int} or None + + @return: a Deferred that fires with a protocol produced by the + factory passed to C{__init__} + @rtype: L{Deferred} that may: + + - fire with L{IProtocol} + + - fail with L{CancelledError} when the service is stopped + + - fail with e.g. + L{DNSLookupError} or + L{ConnectionRefusedError} + when the number of consecutive failed connection attempts + equals the value of "failAfterFailures" + """ + + @_machine.output() + def _currentConnection(self, failAfterFailures=None): + """ + Return the currently connected protocol. + + @return: L{Deferred} that is fired with currently connected protocol. + """ + return succeed(self._currentConnection) + + + @_machine.output() + def _noConnection(self, failAfterFailures=None): + """ + Notify the caller that no connection is expected. + + @return: L{Deferred} that is fired with L{CancelledError}. + """ + return fail(CancelledError()) + + + @_machine.output() + def _awaitingConnection(self, failAfterFailures=None): + """ + Return a deferred that will fire with the next connected protocol. + + @return: L{Deferred} that will fire with the next connected protocol. + """ + result = Deferred() + self._awaitingConnected.append((result, failAfterFailures)) + return result + + + @_machine.output() + def _deferredSucceededWithNone(self): + """ + Return a deferred that has already fired with L{None}. + + @return: A L{Deferred} that has already fired with L{None}. + """ + return succeed(None) + + + def _unawait(self, value): + """ + Fire all outstanding L{ClientService.whenConnected} L{Deferred}s. + + @param value: the value to fire the L{Deferred}s with. + """ + self._awaitingConnected, waiting = [], self._awaitingConnected + for (w, remaining) in waiting: + w.callback(value) + + @_machine.output() + def _deliverConnectionFailure(self, f): + """ + Deliver connection failures to any L{ClientService.whenConnected} + L{Deferred}s that have met their failAfterFailures threshold. + + @param f: the Failure to fire the L{Deferred}s with. + """ + ready = [] + notReady = [] + for (w, remaining) in self._awaitingConnected: + if remaining is None: + notReady.append((w, remaining)) + elif remaining <= 1: + ready.append(w) + else: + notReady.append((w, remaining-1)) + self._awaitingConnected = notReady + for w in ready: + w.callback(f) + + # State Transitions + + _init.upon(start, enter=_connecting, + outputs=[_connect]) + _init.upon(stop, enter=_stopped, + outputs=[_deferredSucceededWithNone], + collector=_firstResult) + + _connecting.upon(start, enter=_connecting, outputs=[]) + # Note that this synchonously triggers _connectionFailed in the + # _disconnecting state. + _connecting.upon(stop, enter=_disconnecting, + outputs=[_waitForStop, _stopConnecting], + collector=_firstResult) + _connecting.upon(_connectionMade, enter=_connected, + outputs=[_notifyWaiters]) + _connecting.upon(_connectionFailed, enter=_waiting, + outputs=[_ignoreAndWait, _deliverConnectionFailure]) + + _waiting.upon(start, enter=_waiting, + outputs=[]) + _waiting.upon(stop, enter=_stopped, + outputs=[_waitForStop, + _cancelConnectWaiters, + _stopRetrying, + _finishStopping], + collector=_firstResult) + _waiting.upon(_reconnect, enter=_connecting, + outputs=[_connect]) + + _connected.upon(start, enter=_connected, + outputs=[]) + _connected.upon(stop, enter=_disconnecting, + outputs=[_waitForStop, _disconnect], + collector=_firstResult) + _connected.upon(_clientDisconnected, enter=_waiting, + outputs=[_forgetConnection, _wait]) + + _disconnecting.upon(start, enter=_restarting, + outputs=[_resetFailedAttempts]) + _disconnecting.upon(stop, enter=_disconnecting, + outputs=[_waitForStop], + collector=_firstResult) + _disconnecting.upon(_clientDisconnected, enter=_stopped, + outputs=[_cancelConnectWaiters, + _finishStopping, + _forgetConnection]) + # Note that this is triggered synchonously with the transition from + # _connecting + _disconnecting.upon(_connectionFailed, enter=_stopped, + outputs=[_ignoreAndCancelConnectWaiters, + _ignoreAndFinishStopping]) + + _restarting.upon(start, enter=_restarting, + outputs=[]) + _restarting.upon(stop, enter=_disconnecting, + outputs=[_waitForStop], + collector=_firstResult) + _restarting.upon(_clientDisconnected, enter=_connecting, + outputs=[_finishStopping, _connect]) + + _stopped.upon(start, enter=_connecting, + outputs=[_connect]) + _stopped.upon(stop, enter=_stopped, + outputs=[_deferredSucceededWithNone], + collector=_firstResult) + + _init.upon(whenConnected, enter=_init, + outputs=[_awaitingConnection], + collector=_firstResult) + _connecting.upon(whenConnected, enter=_connecting, + outputs=[_awaitingConnection], + collector=_firstResult) + _waiting.upon(whenConnected, enter=_waiting, + outputs=[_awaitingConnection], + collector=_firstResult) + _connected.upon(whenConnected, enter=_connected, + outputs=[_currentConnection], + collector=_firstResult) + _disconnecting.upon(whenConnected, enter=_disconnecting, + outputs=[_awaitingConnection], + collector=_firstResult) + _restarting.upon(whenConnected, enter=_restarting, + outputs=[_awaitingConnection], + collector=_firstResult) + _stopped.upon(whenConnected, enter=_stopped, + outputs=[_noConnection], + collector=_firstResult) + + + +class ClientService(service.Service, object): + """ + A L{ClientService} maintains a single outgoing connection to a client + endpoint, reconnecting after a configurable timeout when a connection + fails, either before or after connecting. + + @since: 16.1.0 + """ + + _log = Logger() + + def __init__(self, endpoint, factory, retryPolicy=None, clock=None, + prepareConnection=None): + """ + @param endpoint: A L{stream client endpoint + } provider which will be used to + connect when the service starts. + + @param factory: A L{protocol factory } + which will be used to create clients for the endpoint. + + @param retryPolicy: A policy configuring how long L{ClientService} will + wait between attempts to connect to C{endpoint}. + @type retryPolicy: callable taking (the number of failed connection + attempts made in a row (L{int})) and returning the number of + seconds to wait before making another attempt. + + @param clock: The clock used to schedule reconnection. It's mainly + useful to be parametrized in tests. If the factory is serialized, + this attribute will not be serialized, and the default value (the + reactor) will be restored when deserialized. + @type clock: L{IReactorTime} + + @param prepareConnection: A single argument L{callable} that may return + a L{Deferred}. It will be called once with the L{protocol + } each time a new connection is made. It may + call methods on the protocol to prepare it for use (e.g. + authenticate) or validate it (check its health). + + The C{prepareConnection} callable may raise an exception or return + a L{Deferred} which fails to reject the connection. A rejected + connection is not used to fire an L{Deferred} returned by + L{whenConnected}. Instead, L{ClientService} handles the failure + and continues as if the connection attempt were a failure + (incrementing the counter passed to C{retryPolicy}). + + L{Deferred}s returned by L{whenConnected} will not fire until + any L{Deferred} returned by the C{prepareConnection} callable + fire. Otherwise its successful return value is consumed, but + ignored. + + Present Since Twisted 18.7.0 + + @type prepareConnection: L{callable} + + """ + clock = _maybeGlobalReactor(clock) + retryPolicy = _defaultPolicy if retryPolicy is None else retryPolicy + + self._machine = _ClientMachine( + endpoint, factory, retryPolicy, clock, + prepareConnection=prepareConnection, log=self._log, + ) + + + def whenConnected(self, failAfterFailures=None): + """ + Retrieve the currently-connected L{Protocol}, or the next one to + connect. + + @param failAfterFailures: number of connection failures after which + the Deferred will deliver a Failure (None means the Deferred will + only fail if/when the service is stopped). Set this to 1 to make + the very first connection failure signal an error. Use 2 to + allow one failure but signal an error if the subsequent retry + then fails. + @type failAfterFailures: L{int} or None + + @return: a Deferred that fires with a protocol produced by the + factory passed to C{__init__} + @rtype: L{Deferred} that may: + + - fire with L{IProtocol} + + - fail with L{CancelledError} when the service is stopped + + - fail with e.g. + L{DNSLookupError} or + L{ConnectionRefusedError} + when the number of consecutive failed connection attempts + equals the value of "failAfterFailures" + """ + return self._machine.whenConnected(failAfterFailures) + + + def startService(self): + """ + Start this L{ClientService}, initiating the connection retry loop. + """ + if self.running: + self._log.warn("Duplicate ClientService.startService {log_source}") + return + super(ClientService, self).startService() + self._machine.start() + + + def stopService(self): + """ + Stop attempting to reconnect and close any existing connections. + + @return: a L{Deferred} that fires when all outstanding connections are + closed and all in-progress connection attempts halted. + """ + super(ClientService, self).stopService() + return self._machine.stop() + + +__all__ = (['TimerService', 'CooperatorService', 'MulticastServer', + 'StreamServerEndpointService', 'UDPServer', + 'ClientService'] + + [tran + side + for tran in 'TCP UNIX SSL UNIXDatagram'.split() + for side in 'Server Client'.split()]) diff --git a/contrib/python/Twisted/py2/twisted/application/reactors.py b/contrib/python/Twisted/py2/twisted/application/reactors.py new file mode 100644 index 00000000000..ab2726482f7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/reactors.py @@ -0,0 +1,85 @@ +# -*- test-case-name: twisted.test.test_application -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Plugin-based system for enumerating available reactors and installing one of +them. +""" + +from __future__ import absolute_import, division + +from zope.interface import Interface, Attribute, implementer + +from twisted.plugin import IPlugin, getPlugins +from twisted.python.reflect import namedAny + + +class IReactorInstaller(Interface): + """ + Definition of a reactor which can probably be installed. + """ + shortName = Attribute(""" + A brief string giving the user-facing name of this reactor. + """) + + description = Attribute(""" + A longer string giving a user-facing description of this reactor. + """) + + def install(): + """ + Install this reactor. + """ + + # TODO - A method which provides a best-guess as to whether this reactor + # can actually be used in the execution environment. + + + +class NoSuchReactor(KeyError): + """ + Raised when an attempt is made to install a reactor which cannot be found. + """ + + + +@implementer(IPlugin, IReactorInstaller) +class Reactor(object): + """ + @ivar moduleName: The fully-qualified Python name of the module of which + the install callable is an attribute. + """ + def __init__(self, shortName, moduleName, description): + self.shortName = shortName + self.moduleName = moduleName + self.description = description + + + def install(self): + namedAny(self.moduleName).install() + + + +def getReactorTypes(): + """ + Return an iterator of L{IReactorInstaller} plugins. + """ + return getPlugins(IReactorInstaller) + + + +def installReactor(shortName): + """ + Install the reactor with the given C{shortName} attribute. + + @raise NoSuchReactor: If no reactor is found with a matching C{shortName}. + + @raise: anything that the specified reactor can raise when installed. + """ + for installer in getReactorTypes(): + if installer.shortName == shortName: + installer.install() + from twisted.internet import reactor + return reactor + raise NoSuchReactor(shortName) diff --git a/contrib/python/Twisted/py2/twisted/application/runner/__init__.py b/contrib/python/Twisted/py2/twisted/application/runner/__init__.py new file mode 100644 index 00000000000..6da0ac04e7c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/runner/__init__.py @@ -0,0 +1,7 @@ +# -*- test-case-name: twisted.application.runner.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Facilities for running a Twisted application. +""" diff --git a/contrib/python/Twisted/py2/twisted/application/runner/_exit.py b/contrib/python/Twisted/py2/twisted/application/runner/_exit.py new file mode 100644 index 00000000000..ffccc417bd4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/runner/_exit.py @@ -0,0 +1,138 @@ +# -*- test-case-name: twisted.application.runner.test.test_exit -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +System exit support. +""" + +from sys import stdout, stderr, exit as sysexit + +from constantly import Values, ValueConstant + + + +def exit(status, message=None): + """ + Exit the python interpreter with the given status and an optional message. + + @param status: An exit status. + @type status: L{int} or L{ValueConstant} from L{ExitStatus}. + + @param message: An options message to print. + @type status: L{str} + """ + if isinstance(status, ValueConstant): + code = status.value + else: + code = int(status) + + if message: + if code == 0: + out = stdout + else: + out = stderr + out.write(message) + out.write("\n") + + sysexit(code) + + + +try: + import posix as Status +except ImportError: + class Status(object): + """ + Object to hang C{EX_*} values off of as a substitute for L{posix}. + """ + EX__BASE = 64 + + EX_OK = 0 + EX_USAGE = EX__BASE + EX_DATAERR = EX__BASE + 1 + EX_NOINPUT = EX__BASE + 2 + EX_NOUSER = EX__BASE + 3 + EX_NOHOST = EX__BASE + 4 + EX_UNAVAILABLE = EX__BASE + 5 + EX_SOFTWARE = EX__BASE + 6 + EX_OSERR = EX__BASE + 7 + EX_OSFILE = EX__BASE + 8 + EX_CANTCREAT = EX__BASE + 9 + EX_IOERR = EX__BASE + 10 + EX_TEMPFAIL = EX__BASE + 11 + EX_PROTOCOL = EX__BASE + 12 + EX_NOPERM = EX__BASE + 13 + EX_CONFIG = EX__BASE + 14 + + + +class ExitStatus(Values): + """ + Standard exit status codes for system programs. + + @cvar EX_OK: Successful termination. + @type EX_OK: L{ValueConstant} + + @cvar EX_USAGE: Command line usage error. + @type EX_USAGE: L{ValueConstant} + + @cvar EX_DATAERR: Data format error. + @type EX_DATAERR: L{ValueConstant} + + @cvar EX_NOINPUT: Cannot open input. + @type EX_NOINPUT: L{ValueConstant} + + @cvar EX_NOUSER: Addressee unknown. + @type EX_NOUSER: L{ValueConstant} + + @cvar EX_NOHOST: Host name unknown. + @type EX_NOHOST: L{ValueConstant} + + @cvar EX_UNAVAILABLE: Service unavailable. + @type EX_UNAVAILABLE: L{ValueConstant} + + @cvar EX_SOFTWARE: Internal software error. + @type EX_SOFTWARE: L{ValueConstant} + + @cvar EX_OSERR: System error (e.g., can't fork). + @type EX_OSERR: L{ValueConstant} + + @cvar EX_OSFILE: Critical OS file missing. + @type EX_OSFILE: L{ValueConstant} + + @cvar EX_CANTCREAT: Can't create (user) output file. + @type EX_CANTCREAT: L{ValueConstant} + + @cvar EX_IOERR: Input/output error. + @type EX_IOERR: L{ValueConstant} + + @cvar EX_TEMPFAIL: Temporary failure; the user is invited to retry. + @type EX_TEMPFAIL: L{ValueConstant} + + @cvar EX_PROTOCOL: Remote error in protocol. + @type EX_PROTOCOL: L{ValueConstant} + + @cvar EX_NOPERM: Permission denied. + @type EX_NOPERM: L{ValueConstant} + + @cvar EX_CONFIG: Configuration error. + @type EX_CONFIG: L{ValueConstant} + """ + + EX_OK = ValueConstant(Status.EX_OK) + EX_USAGE = ValueConstant(Status.EX_USAGE) + EX_DATAERR = ValueConstant(Status.EX_DATAERR) + EX_NOINPUT = ValueConstant(Status.EX_NOINPUT) + EX_NOUSER = ValueConstant(Status.EX_NOUSER) + EX_NOHOST = ValueConstant(Status.EX_NOHOST) + EX_UNAVAILABLE = ValueConstant(Status.EX_UNAVAILABLE) + EX_SOFTWARE = ValueConstant(Status.EX_SOFTWARE) + EX_OSERR = ValueConstant(Status.EX_OSERR) + EX_OSFILE = ValueConstant(Status.EX_OSFILE) + EX_CANTCREAT = ValueConstant(Status.EX_CANTCREAT) + EX_IOERR = ValueConstant(Status.EX_IOERR) + EX_TEMPFAIL = ValueConstant(Status.EX_TEMPFAIL) + EX_PROTOCOL = ValueConstant(Status.EX_PROTOCOL) + EX_NOPERM = ValueConstant(Status.EX_NOPERM) + EX_CONFIG = ValueConstant(Status.EX_CONFIG) diff --git a/contrib/python/Twisted/py2/twisted/application/runner/_pidfile.py b/contrib/python/Twisted/py2/twisted/application/runner/_pidfile.py new file mode 100644 index 00000000000..50b8aed68d6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/runner/_pidfile.py @@ -0,0 +1,303 @@ +# -*- test-case-name: twisted.application.runner.test.test_pidfile -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +PID file. +""" + +import errno +from os import getpid, kill, name as SYSTEM_NAME + +from zope.interface import Interface, implementer + +from twisted.logger import Logger + + + +class IPIDFile(Interface): + """ + Manages a file that remembers a process ID. + """ + + def read(): + """ + Read the process ID stored in this PID file. + + @return: The contained process ID. + @rtype: L{int} + + @raise NoPIDFound: If this PID file does not exist. + @raise EnvironmentError: If this PID file cannot be read. + @raise ValueError: If this PID file's content is invalid. + """ + + + def writeRunningPID(): + """ + Store the PID of the current process in this PID file. + + @raise EnvironmentError: If this PID file cannot be written. + """ + + + def remove(): + """ + Remove this PID file. + + @raise EnvironmentError: If this PID file cannot be removed. + """ + + + def isRunning(): + """ + Determine whether there is a running process corresponding to the PID + in this PID file. + + @return: True if this PID file contains a PID and a process with that + PID is currently running; false otherwise. + @rtype: L{bool} + + @raise EnvironmentError: If this PID file cannot be read. + @raise InvalidPIDFileError: If this PID file's content is invalid. + @raise StalePIDFileError: If this PID file's content refers to a PID + for which there is no corresponding running process. + """ + + + def __enter__(): + """ + Enter a context using this PIDFile. + + Writes the PID file with the PID of the running process. + + @raise AlreadyRunningError: A process corresponding to the PID in this + PID file is already running. + """ + + + def __exit__(excType, excValue, traceback): + """ + Exit a context using this PIDFile. + + Removes the PID file. + """ + + + +@implementer(IPIDFile) +class PIDFile(object): + """ + Concrete implementation of L{IPIDFile} based on C{IFilePath}. + + This implementation is presently not supported on non-POSIX platforms. + Specifically, calling L{PIDFile.isRunning} will raise + L{NotImplementedError}. + """ + + _log = Logger() + + + @staticmethod + def _format(pid): + """ + Format a PID file's content. + + @param pid: A process ID. + @type pid: int + + @return: Formatted PID file contents. + @rtype: L{bytes} + """ + return u"{}\n".format(int(pid)).encode("utf-8") + + + def __init__(self, filePath): + """ + @param filePath: The path to the PID file on disk. + @type filePath: L{IFilePath} + """ + self.filePath = filePath + + + def read(self): + pidString = b"" + try: + with self.filePath.open() as fh: + for pidString in fh: + break + except OSError as e: + if e.errno == errno.ENOENT: # No such file + raise NoPIDFound("PID file does not exist") + raise + + try: + return int(pidString) + except ValueError: + raise InvalidPIDFileError( + "non-integer PID value in PID file: {!r}".format(pidString) + ) + + + def _write(self, pid): + """ + Store a PID in this PID file. + + @param pid: A PID to store. + @type pid: L{int} + + @raise EnvironmentError: If this PID file cannot be written. + """ + self.filePath.setContent(self._format(pid=pid)) + + + def writeRunningPID(self): + self._write(getpid()) + + + def remove(self): + self.filePath.remove() + + + def isRunning(self): + try: + pid = self.read() + except NoPIDFound: + return False + + if SYSTEM_NAME == "posix": + return self._pidIsRunningPOSIX(pid) + else: + raise NotImplementedError( + "isRunning is not implemented on {}".format(SYSTEM_NAME) + ) + + + @staticmethod + def _pidIsRunningPOSIX(pid): + """ + POSIX implementation for running process check. + + Determine whether there is a running process corresponding to the given + PID. + + @return: True if the given PID is currently running; false otherwise. + @rtype: L{bool} + + @raise EnvironmentError: If this PID file cannot be read. + @raise InvalidPIDFileError: If this PID file's content is invalid. + @raise StalePIDFileError: If this PID file's content refers to a PID + for which there is no corresponding running process. + """ + try: + kill(pid, 0) + except OSError as e: + if e.errno == errno.ESRCH: # No such process + raise StalePIDFileError( + "PID file refers to non-existing process" + ) + elif e.errno == errno.EPERM: # Not permitted to kill + return True + else: + raise + else: + return True + + + def __enter__(self): + try: + if self.isRunning(): + raise AlreadyRunningError() + except StalePIDFileError: + self._log.info("Replacing stale PID file: {log_source}") + self.writeRunningPID() + return self + + + def __exit__(self, excType, excValue, traceback): + self.remove() + + + +@implementer(IPIDFile) +class NonePIDFile(object): + """ + PID file implementation that does nothing. + + This is meant to be used as a "active None" object in place of a PID file + when no PID file is desired. + """ + + def __init__(self): + pass + + + def read(self): + raise NoPIDFound("PID file does not exist") + + + def _write(self, pid): + """ + Store a PID in this PID file. + + @param pid: A PID to store. + @type pid: L{int} + + @raise EnvironmentError: If this PID file cannot be written. + + @note: This implementation always raises an L{EnvironmentError}. + """ + raise OSError(errno.EPERM, "Operation not permitted") + + + def writeRunningPID(self): + self._write(0) + + + def remove(self): + raise OSError(errno.ENOENT, "No such file or directory") + + + def isRunning(self): + return False + + + def __enter__(self): + return self + + + def __exit__(self, excType, excValue, traceback): + pass + + + +nonePIDFile = NonePIDFile() + + + +class AlreadyRunningError(Exception): + """ + Process is already running. + """ + + + +class InvalidPIDFileError(Exception): + """ + PID file contents are invalid. + """ + + + +class StalePIDFileError(Exception): + """ + PID file contents are valid, but there is no process with the referenced + PID. + """ + + + +class NoPIDFound(Exception): + """ + No PID found in PID file. + """ diff --git a/contrib/python/Twisted/py2/twisted/application/runner/_runner.py b/contrib/python/Twisted/py2/twisted/application/runner/_runner.py new file mode 100644 index 00000000000..7542a5b9a94 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/runner/_runner.py @@ -0,0 +1,185 @@ +# -*- test-case-name: twisted.application.runner.test.test_runner -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted application runner. +""" + +from sys import stderr +from signal import SIGTERM +from os import kill + +from attr import attrib, attrs, Factory + +from twisted.logger import ( + globalLogBeginner, textFileLogObserver, + FilteringLogObserver, LogLevelFilterPredicate, + LogLevel, Logger, +) + +from ._exit import exit, ExitStatus +from ._pidfile import nonePIDFile, AlreadyRunningError, InvalidPIDFileError + + + +@attrs(frozen=True) +class Runner(object): + """ + Twisted application runner. + + @cvar _log: The logger attached to this class. + @type _log: L{Logger} + + @ivar _reactor: The reactor to start and run the application in. + @type _reactor: L{IReactorCore} + + @ivar _pidFile: The file to store the running process ID in. + @type _pidFile: L{IPIDFile} + + @ivar _kill: Whether this runner should kill an existing running + instance of the application. + @type _kill: L{bool} + + @ivar _defaultLogLevel: The default log level to start the logging + system with. + @type _defaultLogLevel: L{constantly.NamedConstant} from L{LogLevel} + + @ivar _logFile: A file stream to write logging output to. + @type _logFile: writable file-like object + + @ivar _fileLogObserverFactory: A factory for the file log observer to + use when starting the logging system. + @type _pidFile: callable that takes a single writable file-like object + argument and returns a L{twisted.logger.FileLogObserver} + + @ivar _whenRunning: Hook to call after the reactor is running; + this is where the application code that relies on the reactor gets + called. + @type _whenRunning: callable that takes the keyword arguments specified + by C{whenRunningArguments} + + @ivar _whenRunningArguments: Keyword arguments to pass to + C{whenRunning} when it is called. + @type _whenRunningArguments: L{dict} + + @ivar _reactorExited: Hook to call after the reactor exits. + @type _reactorExited: callable that takes the keyword arguments + specified by C{reactorExitedArguments} + + @ivar _reactorExitedArguments: Keyword arguments to pass to + C{reactorExited} when it is called. + @type _reactorExitedArguments: L{dict} + """ + + _log = Logger() + + _reactor = attrib() + _pidFile = attrib(default=nonePIDFile) + _kill = attrib(default=False) + _defaultLogLevel = attrib(default=LogLevel.info) + _logFile = attrib(default=stderr) + _fileLogObserverFactory = attrib(default=textFileLogObserver) + _whenRunning = attrib(default=lambda **_: None) + _whenRunningArguments = attrib(default=Factory(dict)) + _reactorExited = attrib(default=lambda **_: None) + _reactorExitedArguments = attrib(default=Factory(dict)) + + + def run(self): + """ + Run this command. + """ + pidFile = self._pidFile + + self.killIfRequested() + + try: + with pidFile: + self.startLogging() + self.startReactor() + self.reactorExited() + + except AlreadyRunningError: + exit(ExitStatus.EX_CONFIG, "Already running.") + return # When testing, patched exit doesn't exit + + + def killIfRequested(self): + """ + If C{self._kill} is true, attempt to kill a running instance of the + application. + """ + pidFile = self._pidFile + + if self._kill: + if pidFile is nonePIDFile: + exit(ExitStatus.EX_USAGE, "No PID file specified.") + return # When testing, patched exit doesn't exit + + try: + pid = pidFile.read() + except EnvironmentError: + exit(ExitStatus.EX_IOERR, "Unable to read PID file.") + return # When testing, patched exit doesn't exit + except InvalidPIDFileError: + exit(ExitStatus.EX_DATAERR, "Invalid PID file.") + return # When testing, patched exit doesn't exit + + self.startLogging() + self._log.info("Terminating process: {pid}", pid=pid) + + kill(pid, SIGTERM) + + exit(ExitStatus.EX_OK) + return # When testing, patched exit doesn't exit + + + def startLogging(self): + """ + Start the L{twisted.logger} logging system. + """ + logFile = self._logFile + + fileLogObserverFactory = self._fileLogObserverFactory + + fileLogObserver = fileLogObserverFactory(logFile) + + logLevelPredicate = LogLevelFilterPredicate( + defaultLogLevel=self._defaultLogLevel + ) + + filteringObserver = FilteringLogObserver( + fileLogObserver, [logLevelPredicate] + ) + + globalLogBeginner.beginLoggingTo([filteringObserver]) + + + def startReactor(self): + """ + Register C{self._whenRunning} with the reactor so that it is called + once the reactor is running, then start the reactor. + """ + self._reactor.callWhenRunning(self.whenRunning) + + self._log.info("Starting reactor...") + self._reactor.run() + + + def whenRunning(self): + """ + Call C{self._whenRunning} with C{self._whenRunningArguments}. + + @note: This method is called after the reactor starts running. + """ + self._whenRunning(**self._whenRunningArguments) + + + def reactorExited(self): + """ + Call C{self._reactorExited} with C{self._reactorExitedArguments}. + + @note: This method is called after the reactor exits. + """ + self._reactorExited(**self._reactorExitedArguments) diff --git a/contrib/python/Twisted/py2/twisted/application/service.py b/contrib/python/Twisted/py2/twisted/application/service.py new file mode 100644 index 00000000000..c6da29bf61e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/service.py @@ -0,0 +1,424 @@ +# -*- test-case-name: twisted.application.test.test_service -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Service architecture for Twisted. + +Services are arranged in a hierarchy. At the leafs of the hierarchy, +the services which actually interact with the outside world are started. +Services can be named or anonymous -- usually, they will be named if +there is need to access them through the hierarchy (from a parent or +a sibling). + +Maintainer: Moshe Zadka +""" + +from __future__ import absolute_import, division + +from zope.interface import implementer, Interface, Attribute + +from twisted.persisted import sob +from twisted.python.reflect import namedAny +from twisted.python import components +from twisted.python._oldstyle import _oldStyle +from twisted.internet import defer +from twisted.plugin import IPlugin + + +class IServiceMaker(Interface): + """ + An object which can be used to construct services in a flexible + way. + + This interface should most often be implemented along with + L{twisted.plugin.IPlugin}, and will most often be used by the + 'twistd' command. + """ + tapname = Attribute( + "A short string naming this Twisted plugin, for example 'web' or " + "'pencil'. This name will be used as the subcommand of 'twistd'.") + + description = Attribute( + "A brief summary of the features provided by this " + "Twisted application plugin.") + + options = Attribute( + "A C{twisted.python.usage.Options} subclass defining the " + "configuration options for this application.") + + + def makeService(options): + """ + Create and return an object providing + L{twisted.application.service.IService}. + + @param options: A mapping (typically a C{dict} or + L{twisted.python.usage.Options} instance) of configuration + options to desired configuration values. + """ + + + +@implementer(IPlugin, IServiceMaker) +class ServiceMaker(object): + """ + Utility class to simplify the definition of L{IServiceMaker} plugins. + """ + def __init__(self, name, module, description, tapname): + self.name = name + self.module = module + self.description = description + self.tapname = tapname + + + def options(): + def get(self): + return namedAny(self.module).Options + return get, + options = property(*options()) + + + def makeService(): + def get(self): + return namedAny(self.module).makeService + return get, + makeService = property(*makeService()) + + + +class IService(Interface): + """ + A service. + + Run start-up and shut-down code at the appropriate times. + """ + + name = Attribute( + "A C{str} which is the name of the service or C{None}.") + + running = Attribute( + "A C{boolean} which indicates whether the service is running.") + + parent = Attribute( + "An C{IServiceCollection} which is the parent or C{None}.") + + def setName(name): + """ + Set the name of the service. + + @type name: C{str} + @raise RuntimeError: Raised if the service already has a parent. + """ + + def setServiceParent(parent): + """ + Set the parent of the service. This method is responsible for setting + the C{parent} attribute on this service (the child service). + + @type parent: L{IServiceCollection} + @raise RuntimeError: Raised if the service already has a parent + or if the service has a name and the parent already has a child + by that name. + """ + + def disownServiceParent(): + """ + Use this API to remove an L{IService} from an L{IServiceCollection}. + + This method is used symmetrically with L{setServiceParent} in that it + sets the C{parent} attribute on the child. + + @rtype: L{Deferred} + @return: a L{Deferred} which is triggered when the + service has finished shutting down. If shutting down is immediate, + a value can be returned (usually, L{None}). + """ + + def startService(): + """ + Start the service. + """ + + def stopService(): + """ + Stop the service. + + @rtype: L{Deferred} + @return: a L{Deferred} which is triggered when the + service has finished shutting down. If shutting down is immediate, + a value can be returned (usually, L{None}). + """ + + def privilegedStartService(): + """ + Do preparation work for starting the service. + + Here things which should be done before changing directory, + root or shedding privileges are done. + """ + + + +@implementer(IService) +class Service(object): + """ + Base class for services. + + Most services should inherit from this class. It handles the + book-keeping responsibilities of starting and stopping, as well + as not serializing this book-keeping information. + """ + + running = 0 + name = None + parent = None + + def __getstate__(self): + dict = self.__dict__.copy() + if "running" in dict: + del dict['running'] + return dict + + def setName(self, name): + if self.parent is not None: + raise RuntimeError("cannot change name when parent exists") + self.name = name + + def setServiceParent(self, parent): + if self.parent is not None: + self.disownServiceParent() + parent = IServiceCollection(parent, parent) + self.parent = parent + self.parent.addService(self) + + def disownServiceParent(self): + d = self.parent.removeService(self) + self.parent = None + return d + + def privilegedStartService(self): + pass + + def startService(self): + self.running = 1 + + def stopService(self): + self.running = 0 + + + +class IServiceCollection(Interface): + """ + Collection of services. + + Contain several services, and manage their start-up/shut-down. + Services can be accessed by name if they have a name, and it + is always possible to iterate over them. + """ + + def getServiceNamed(name): + """ + Get the child service with a given name. + + @type name: C{str} + @rtype: L{IService} + @raise KeyError: Raised if the service has no child with the + given name. + """ + + def __iter__(): + """ + Get an iterator over all child services. + """ + + def addService(service): + """ + Add a child service. + + Only implementations of L{IService.setServiceParent} should use this + method. + + @type service: L{IService} + @raise RuntimeError: Raised if the service has a child with + the given name. + """ + + def removeService(service): + """ + Remove a child service. + + Only implementations of L{IService.disownServiceParent} should + use this method. + + @type service: L{IService} + @raise ValueError: Raised if the given service is not a child. + @rtype: L{Deferred} + @return: a L{Deferred} which is triggered when the + service has finished shutting down. If shutting down is immediate, + a value can be returned (usually, L{None}). + """ + + + +@implementer(IServiceCollection) +class MultiService(Service): + """ + Straightforward Service Container. + + Hold a collection of services, and manage them in a simplistic + way. No service will wait for another, but this object itself + will not finish shutting down until all of its child services + will finish. + """ + + def __init__(self): + self.services = [] + self.namedServices = {} + self.parent = None + + def privilegedStartService(self): + Service.privilegedStartService(self) + for service in self: + service.privilegedStartService() + + def startService(self): + Service.startService(self) + for service in self: + service.startService() + + def stopService(self): + Service.stopService(self) + l = [] + services = list(self) + services.reverse() + for service in services: + l.append(defer.maybeDeferred(service.stopService)) + return defer.DeferredList(l) + + def getServiceNamed(self, name): + return self.namedServices[name] + + def __iter__(self): + return iter(self.services) + + def addService(self, service): + if service.name is not None: + if service.name in self.namedServices: + raise RuntimeError("cannot have two services with same name" + " '%s'" % service.name) + self.namedServices[service.name] = service + self.services.append(service) + if self.running: + # It may be too late for that, but we will do our best + service.privilegedStartService() + service.startService() + + def removeService(self, service): + if service.name: + del self.namedServices[service.name] + self.services.remove(service) + if self.running: + # Returning this so as not to lose information from the + # MultiService.stopService deferred. + return service.stopService() + else: + return None + + + +class IProcess(Interface): + """ + Process running parameters. + + Represents parameters for how processes should be run. + """ + processName = Attribute( + """ + A C{str} giving the name the process should have in ps (or L{None} + to leave the name alone). + """) + + uid = Attribute( + """ + An C{int} giving the user id as which the process should run (or + L{None} to leave the UID alone). + """) + + gid = Attribute( + """ + An C{int} giving the group id as which the process should run (or + L{None} to leave the GID alone). + """) + + + +@implementer(IProcess) +@_oldStyle +class Process: + """ + Process running parameters. + + Sets up uid/gid in the constructor, and has a default + of L{None} as C{processName}. + """ + processName = None + + def __init__(self, uid=None, gid=None): + """ + Set uid and gid. + + @param uid: The user ID as whom to execute the process. If + this is L{None}, no attempt will be made to change the UID. + + @param gid: The group ID as whom to execute the process. If + this is L{None}, no attempt will be made to change the GID. + """ + self.uid = uid + self.gid = gid + + + +def Application(name, uid=None, gid=None): + """ + Return a compound class. + + Return an object supporting the L{IService}, L{IServiceCollection}, + L{IProcess} and L{sob.IPersistable} interfaces, with the given + parameters. Always access the return value by explicit casting to + one of the interfaces. + """ + ret = components.Componentized() + availableComponents = [MultiService(), Process(uid, gid), + sob.Persistent(ret, name)] + + for comp in availableComponents: + ret.addComponent(comp, ignoreClass=1) + IService(ret).setName(name) + return ret + + + +def loadApplication(filename, kind, passphrase=None): + """ + Load Application from a given file. + + The serialization format it was saved in should be given as + C{kind}, and is one of C{pickle}, C{source}, C{xml} or C{python}. If + C{passphrase} is given, the application was encrypted with the + given passphrase. + + @type filename: C{str} + @type kind: C{str} + @type passphrase: C{str} + """ + if kind == 'python': + application = sob.loadValueFromFile(filename, 'application') + else: + application = sob.load(filename, kind) + return application + + +__all__ = ['IServiceMaker', 'IService', 'Service', + 'IServiceCollection', 'MultiService', + 'IProcess', 'Process', 'Application', 'loadApplication'] diff --git a/contrib/python/Twisted/py2/twisted/application/strports.py b/contrib/python/Twisted/py2/twisted/application/strports.py new file mode 100644 index 00000000000..b6b081eb99b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/strports.py @@ -0,0 +1,70 @@ +# -*- test-case-name: twisted.test.test_strports -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Construct listening port services from a simple string description. + +@see: L{twisted.internet.endpoints.serverFromString} +@see: L{twisted.internet.endpoints.clientFromString} +""" + +from __future__ import absolute_import, division + +from twisted.application.internet import StreamServerEndpointService +from twisted.internet import endpoints + + +def service(description, factory, reactor=None): + """ + Return the service corresponding to a description. + + @param description: The description of the listening port, in the syntax + described by L{twisted.internet.endpoints.serverFromString}. + @type description: C{str} + + @param factory: The protocol factory which will build protocols for + connections to this service. + @type factory: L{twisted.internet.interfaces.IProtocolFactory} + + @rtype: C{twisted.application.service.IService} + @return: the service corresponding to a description of a reliable stream + server. + + @see: L{twisted.internet.endpoints.serverFromString} + """ + if reactor is None: + from twisted.internet import reactor + + svc = StreamServerEndpointService( + endpoints.serverFromString(reactor, description), factory) + svc._raiseSynchronously = True + return svc + + + +def listen(description, factory): + """ + Listen on a port corresponding to a description. + + @param description: The description of the connecting port, in the syntax + described by L{twisted.internet.endpoints.serverFromString}. + @type description: L{str} + + @param factory: The protocol factory which will build protocols on + connection. + @type factory: L{twisted.internet.interfaces.IProtocolFactory} + + @rtype: L{twisted.internet.interfaces.IListeningPort} + @return: the port corresponding to a description of a reliable virtual + circuit server. + + @see: L{twisted.internet.endpoints.serverFromString} + """ + from twisted.internet import reactor + name, args, kw = endpoints._parseServer(description, factory) + return getattr(reactor, 'listen' + name)(*args, **kw) + + + +__all__ = ['service', 'listen'] diff --git a/contrib/python/Twisted/py2/twisted/application/twist/__init__.py b/contrib/python/Twisted/py2/twisted/application/twist/__init__.py new file mode 100644 index 00000000000..ea7c5d29ce7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/twist/__init__.py @@ -0,0 +1,7 @@ +# -*- test-case-name: twisted.application.twist.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +C{twist} command line tool. +""" diff --git a/contrib/python/Twisted/py2/twisted/application/twist/_options.py b/contrib/python/Twisted/py2/twisted/application/twist/_options.py new file mode 100644 index 00000000000..2b7d3075155 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/twist/_options.py @@ -0,0 +1,205 @@ +# -*- test-case-name: twisted.application.twist.test.test_options -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Command line options for C{twist}. +""" + +from sys import stdout, stderr +from textwrap import dedent + +from twisted.copyright import version +from twisted.python.usage import Options, UsageError +from twisted.logger import ( + LogLevel, InvalidLogLevelError, + textFileLogObserver, jsonFileLogObserver, +) +from twisted.plugin import getPlugins + +from ..reactors import installReactor, NoSuchReactor, getReactorTypes +from ..runner._exit import exit, ExitStatus +from ..service import IServiceMaker + +openFile = open + + + +class TwistOptions(Options): + """ + Command line options for C{twist}. + """ + + defaultReactorName = "default" + defaultLogLevel = LogLevel.info + + + def __init__(self): + Options.__init__(self) + + self["reactorName"] = self.defaultReactorName + self["logLevel"] = self.defaultLogLevel + self["logFile"] = stdout + + + def getSynopsis(self): + return "{} plugin [plugin_options]".format( + Options.getSynopsis(self) + ) + + + def opt_version(self): + """ + Print version and exit. + """ + exit(ExitStatus.EX_OK, "{}".format(version)) + + + def opt_reactor(self, name): + """ + The name of the reactor to use. + (options: {options}) + """ + # Actually actually actually install the reactor right at this very + # moment, before any other code (for example, a sub-command plugin) + # runs and accidentally imports and installs the default reactor. + try: + self["reactor"] = self.installReactor(name) + except NoSuchReactor: + raise UsageError("Unknown reactor: {}".format(name)) + else: + self["reactorName"] = name + + opt_reactor.__doc__ = dedent(opt_reactor.__doc__).format( + options=", ".join( + '"{}"'.format(rt.shortName) for rt in getReactorTypes() + ), + ) + + + def installReactor(self, name): + """ + Install the reactor. + """ + if name == self.defaultReactorName: + from twisted.internet import reactor + return reactor + else: + return installReactor(name) + + + def opt_log_level(self, levelName): + """ + Set default log level. + (options: {options}; default: "{default}") + """ + try: + self["logLevel"] = LogLevel.levelWithName(levelName) + except InvalidLogLevelError: + raise UsageError("Invalid log level: {}".format(levelName)) + + opt_log_level.__doc__ = dedent(opt_log_level.__doc__).format( + options=", ".join( + '"{}"'.format(l.name) for l in LogLevel.iterconstants() + ), + default=defaultLogLevel.name, + ) + + + def opt_log_file(self, fileName): + """ + Log to file. ("-" for stdout, "+" for stderr; default: "-") + """ + if fileName == "-": + self["logFile"] = stdout + return + + if fileName == "+": + self["logFile"] = stderr + return + + try: + self["logFile"] = openFile(fileName, "a") + except EnvironmentError as e: + exit( + ExitStatus.EX_IOERR, + "Unable to open log file {!r}: {}".format(fileName, e) + ) + + + def opt_log_format(self, format): + """ + Log file format. + (options: "text", "json"; default: "text" if the log file is a tty, + otherwise "json") + """ + format = format.lower() + + if format == "text": + self["fileLogObserverFactory"] = textFileLogObserver + elif format == "json": + self["fileLogObserverFactory"] = jsonFileLogObserver + else: + raise UsageError("Invalid log format: {}".format(format)) + self["logFormat"] = format + + opt_log_format.__doc__ = dedent(opt_log_format.__doc__) + + + def selectDefaultLogObserver(self): + """ + Set C{fileLogObserverFactory} to the default appropriate for the + chosen C{logFile}. + """ + if "fileLogObserverFactory" not in self: + logFile = self["logFile"] + + if hasattr(logFile, "isatty") and logFile.isatty(): + self["fileLogObserverFactory"] = textFileLogObserver + self["logFormat"] = "text" + else: + self["fileLogObserverFactory"] = jsonFileLogObserver + self["logFormat"] = "json" + + + def parseOptions(self, options=None): + self.selectDefaultLogObserver() + + Options.parseOptions(self, options=options) + + if "reactor" not in self: + self["reactor"] = self.installReactor(self["reactorName"]) + + + @property + def plugins(self): + if "plugins" not in self: + plugins = {} + for plugin in getPlugins(IServiceMaker): + plugins[plugin.tapname] = plugin + self["plugins"] = plugins + + return self["plugins"] + + + @property + def subCommands(self): + plugins = self.plugins + for name in sorted(plugins): + plugin = plugins[name] + yield ( + plugin.tapname, + None, + # Avoid resolving the options attribute right away, in case + # it's a property with a non-trivial getter (eg, one which + # imports modules). + lambda plugin=plugin: plugin.options(), + plugin.description, + ) + + + def postOptions(self): + Options.postOptions(self) + + if self.subCommand is None: + raise UsageError("No plugin specified.") diff --git a/contrib/python/Twisted/py2/twisted/application/twist/_twist.py b/contrib/python/Twisted/py2/twisted/application/twist/_twist.py new file mode 100644 index 00000000000..ec8bc34739d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/application/twist/_twist.py @@ -0,0 +1,128 @@ +# -*- test-case-name: twisted.application.twist.test.test_twist -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Run a Twisted application. +""" + +import sys + +from twisted.python.usage import UsageError +from ..service import Application, IService +from ..runner._exit import exit, ExitStatus +from ..runner._runner import Runner +from ._options import TwistOptions +from twisted.application.app import _exitWithSignal +from twisted.internet.interfaces import _ISupportsExitSignalCapturing + + + +class Twist(object): + """ + Run a Twisted application. + """ + + @staticmethod + def options(argv): + """ + Parse command line options. + + @param argv: Command line arguments. + @type argv: L{list} + + @return: The parsed options. + @rtype: L{TwistOptions} + """ + options = TwistOptions() + + try: + options.parseOptions(argv[1:]) + except UsageError as e: + exit(ExitStatus.EX_USAGE, "Error: {}\n\n{}".format(e, options)) + + return options + + + @staticmethod + def service(plugin, options): + """ + Create the application service. + + @param plugin: The name of the plugin that implements the service + application to run. + @type plugin: L{str} + + @param options: Options to pass to the application. + @type options: L{twisted.python.usage.Options} + + @return: The created application service. + @rtype: L{IService} + """ + service = plugin.makeService(options) + application = Application(plugin.tapname) + service.setServiceParent(application) + + return IService(application) + + + @staticmethod + def startService(reactor, service): + """ + Start the application service. + + @param reactor: The reactor to run the service with. + @type reactor: L{twisted.internet.interfaces.IReactorCore} + + @param service: The application service to run. + @type service: L{IService} + """ + service.startService() + + # Ask the reactor to stop the service before shutting down + reactor.addSystemEventTrigger( + "before", "shutdown", service.stopService + ) + + + @staticmethod + def run(twistOptions): + """ + Run the application service. + + @param twistOptions: Command line options to convert to runner + arguments. + @type twistOptions: L{TwistOptions} + """ + runner = Runner( + reactor=twistOptions["reactor"], + defaultLogLevel=twistOptions["logLevel"], + logFile=twistOptions["logFile"], + fileLogObserverFactory=twistOptions["fileLogObserverFactory"], + ) + runner.run() + reactor = twistOptions["reactor"] + if _ISupportsExitSignalCapturing.providedBy(reactor): + if reactor._exitSignal is not None: + _exitWithSignal(reactor._exitSignal) + + + @classmethod + def main(cls, argv=sys.argv): + """ + Executable entry point for L{Twist}. + Processes options and run a twisted reactor with a service. + + @param argv: Command line arguments. + @type argv: L{list} + """ + options = cls.options(argv) + + reactor = options["reactor"] + service = cls.service( + plugin=options.plugins[options.subCommand], + options=options.subOptions, + ) + + cls.startService(reactor, service) + cls.run(options) diff --git a/contrib/python/Twisted/py2/twisted/conch/__init__.py b/contrib/python/Twisted/py2/twisted/conch/__init__.py new file mode 100644 index 00000000000..adc49d01e64 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/__init__.py @@ -0,0 +1,7 @@ +# -*- test-case-name: twisted.conch.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Conch: The Twisted Shell. Terminal emulation, SSHv2 and telnet. +""" diff --git a/contrib/python/Twisted/py2/twisted/conch/avatar.py b/contrib/python/Twisted/py2/twisted/conch/avatar.py new file mode 100644 index 00000000000..2f6850dbb8e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/avatar.py @@ -0,0 +1,45 @@ +# -*- test-case-name: twisted.conch.test.test_conch -*- + +from __future__ import absolute_import, division + +from zope.interface import implementer + +from twisted.conch.error import ConchError +from twisted.conch.interfaces import IConchUser +from twisted.conch.ssh.connection import OPEN_UNKNOWN_CHANNEL_TYPE +from twisted.python import log +from twisted.python.compat import nativeString + + +@implementer(IConchUser) +class ConchUser: + def __init__(self): + self.channelLookup = {} + self.subsystemLookup = {} + + + def lookupChannel(self, channelType, windowSize, maxPacket, data): + klass = self.channelLookup.get(channelType, None) + if not klass: + raise ConchError(OPEN_UNKNOWN_CHANNEL_TYPE, "unknown channel") + else: + return klass(remoteWindow=windowSize, + remoteMaxPacket=maxPacket, + data=data, avatar=self) + + + def lookupSubsystem(self, subsystem, data): + log.msg(repr(self.subsystemLookup)) + klass = self.subsystemLookup.get(subsystem, None) + if not klass: + return False + return klass(data, avatar=self) + + + def gotGlobalRequest(self, requestType, data): + # XXX should this use method dispatch? + requestType = nativeString(requestType.replace(b'-', b'_')) + f = getattr(self, "global_%s" % requestType, None) + if not f: + return 0 + return f(data) diff --git a/contrib/python/Twisted/py2/twisted/conch/checkers.py b/contrib/python/Twisted/py2/twisted/conch/checkers.py new file mode 100644 index 00000000000..d4fcf5bc78d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/checkers.py @@ -0,0 +1,592 @@ +# -*- test-case-name: twisted.conch.test.test_checkers -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Provide L{ICredentialsChecker} implementations to be used in Conch protocols. +""" + +from __future__ import absolute_import, division + +import sys +import binascii +import errno + +try: + import pwd +except ImportError: + pwd = None +else: + import crypt + +try: + import spwd +except ImportError: + spwd = None + +from zope.interface import providedBy, implementer, Interface + +from incremental import Version + +from twisted.conch import error +from twisted.conch.ssh import keys +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword, ISSHPrivateKey +from twisted.cred.error import UnauthorizedLogin, UnhandledCredentials +from twisted.internet import defer +from twisted.python.compat import _keys, _PY3, _b64decodebytes +from twisted.python import failure, reflect, log +from twisted.python.deprecate import deprecatedModuleAttribute +from twisted.python.util import runAsEffectiveUser +from twisted.python.filepath import FilePath + + + + +def verifyCryptedPassword(crypted, pw): + """ + Check that the password, when crypted, matches the stored crypted password. + + @param crypted: The stored crypted password. + @type crypted: L{str} + @param pw: The password the user has given. + @type pw: L{str} + + @rtype: L{bool} + """ + return crypt.crypt(pw, crypted) == crypted + + + +def _pwdGetByName(username): + """ + Look up a user in the /etc/passwd database using the pwd module. If the + pwd module is not available, return None. + + @param username: the username of the user to return the passwd database + information for. + @type username: L{str} + """ + if pwd is None: + return None + return pwd.getpwnam(username) + + + +def _shadowGetByName(username): + """ + Look up a user in the /etc/shadow database using the spwd module. If it is + not available, return L{None}. + + @param username: the username of the user to return the shadow database + information for. + @type username: L{str} + """ + if spwd is not None: + f = spwd.getspnam + else: + return None + return runAsEffectiveUser(0, 0, f, username) + + + +@implementer(ICredentialsChecker) +class UNIXPasswordDatabase: + """ + A checker which validates users out of the UNIX password databases, or + databases of a compatible format. + + @ivar _getByNameFunctions: a C{list} of functions which are called in order + to valid a user. The default value is such that the C{/etc/passwd} + database will be tried first, followed by the C{/etc/shadow} database. + """ + credentialInterfaces = IUsernamePassword, + + def __init__(self, getByNameFunctions=None): + if getByNameFunctions is None: + getByNameFunctions = [_pwdGetByName, _shadowGetByName] + self._getByNameFunctions = getByNameFunctions + + + def requestAvatarId(self, credentials): + # We get bytes, but the Py3 pwd module uses str. So attempt to decode + # it using the same method that CPython does for the file on disk. + if _PY3: + username = credentials.username.decode(sys.getfilesystemencoding()) + password = credentials.password.decode(sys.getfilesystemencoding()) + else: + username = credentials.username + password = credentials.password + + for func in self._getByNameFunctions: + try: + pwnam = func(username) + except KeyError: + return defer.fail(UnauthorizedLogin("invalid username")) + else: + if pwnam is not None: + crypted = pwnam[1] + if crypted == '': + continue + + if verifyCryptedPassword(crypted, password): + return defer.succeed(credentials.username) + # fallback + return defer.fail(UnauthorizedLogin("unable to verify password")) + + + +@implementer(ICredentialsChecker) +class SSHPublicKeyDatabase: + """ + Checker that authenticates SSH public keys, based on public keys listed in + authorized_keys and authorized_keys2 files in user .ssh/ directories. + """ + credentialInterfaces = (ISSHPrivateKey,) + + _userdb = pwd + + def requestAvatarId(self, credentials): + d = defer.maybeDeferred(self.checkKey, credentials) + d.addCallback(self._cbRequestAvatarId, credentials) + d.addErrback(self._ebRequestAvatarId) + return d + + + def _cbRequestAvatarId(self, validKey, credentials): + """ + Check whether the credentials themselves are valid, now that we know + if the key matches the user. + + @param validKey: A boolean indicating whether or not the public key + matches a key in the user's authorized_keys file. + + @param credentials: The credentials offered by the user. + @type credentials: L{ISSHPrivateKey} provider + + @raise UnauthorizedLogin: (as a failure) if the key does not match the + user in C{credentials}. Also raised if the user provides an invalid + signature. + + @raise ValidPublicKey: (as a failure) if the key matches the user but + the credentials do not include a signature. See + L{error.ValidPublicKey} for more information. + + @return: The user's username, if authentication was successful. + """ + if not validKey: + return failure.Failure(UnauthorizedLogin("invalid key")) + if not credentials.signature: + return failure.Failure(error.ValidPublicKey()) + else: + try: + pubKey = keys.Key.fromString(credentials.blob) + if pubKey.verify(credentials.signature, credentials.sigData): + return credentials.username + except: # any error should be treated as a failed login + log.err() + return failure.Failure(UnauthorizedLogin('error while verifying key')) + return failure.Failure(UnauthorizedLogin("unable to verify key")) + + + def getAuthorizedKeysFiles(self, credentials): + """ + Return a list of L{FilePath} instances for I{authorized_keys} files + which might contain information about authorized keys for the given + credentials. + + On OpenSSH servers, the default location of the file containing the + list of authorized public keys is + U{$HOME/.ssh/authorized_keys}. + + I{$HOME/.ssh/authorized_keys2} is also returned, though it has been + U{deprecated by OpenSSH since + 2001}. + + @return: A list of L{FilePath} instances to files with the authorized keys. + """ + pwent = self._userdb.getpwnam(credentials.username) + root = FilePath(pwent.pw_dir).child('.ssh') + files = ['authorized_keys', 'authorized_keys2'] + return [root.child(f) for f in files] + + + def checkKey(self, credentials): + """ + Retrieve files containing authorized keys and check against user + credentials. + """ + ouid, ogid = self._userdb.getpwnam(credentials.username)[2:4] + for filepath in self.getAuthorizedKeysFiles(credentials): + if not filepath.exists(): + continue + try: + lines = filepath.open() + except IOError as e: + if e.errno == errno.EACCES: + lines = runAsEffectiveUser(ouid, ogid, filepath.open) + else: + raise + with lines: + for l in lines: + l2 = l.split() + if len(l2) < 2: + continue + try: + if _b64decodebytes(l2[1]) == credentials.blob: + return True + except binascii.Error: + continue + return False + + + def _ebRequestAvatarId(self, f): + if not f.check(UnauthorizedLogin): + log.msg(f) + return failure.Failure(UnauthorizedLogin("unable to get avatar id")) + return f + + + +@implementer(ICredentialsChecker) +class SSHProtocolChecker: + """ + SSHProtocolChecker is a checker that requires multiple authentications + to succeed. To add a checker, call my registerChecker method with + the checker and the interface. + + After each successful authenticate, I call my areDone method with the + avatar id. To get a list of the successful credentials for an avatar id, + use C{SSHProcotolChecker.successfulCredentials[avatarId]}. If L{areDone} + returns True, the authentication has succeeded. + """ + + def __init__(self): + self.checkers = {} + self.successfulCredentials = {} + + + def get_credentialInterfaces(self): + return _keys(self.checkers) + + credentialInterfaces = property(get_credentialInterfaces) + + def registerChecker(self, checker, *credentialInterfaces): + if not credentialInterfaces: + credentialInterfaces = checker.credentialInterfaces + for credentialInterface in credentialInterfaces: + self.checkers[credentialInterface] = checker + + + def requestAvatarId(self, credentials): + """ + Part of the L{ICredentialsChecker} interface. Called by a portal with + some credentials to check if they'll authenticate a user. We check the + interfaces that the credentials provide against our list of acceptable + checkers. If one of them matches, we ask that checker to verify the + credentials. If they're valid, we call our L{_cbGoodAuthentication} + method to continue. + + @param credentials: the credentials the L{Portal} wants us to verify + """ + ifac = providedBy(credentials) + for i in ifac: + c = self.checkers.get(i) + if c is not None: + d = defer.maybeDeferred(c.requestAvatarId, credentials) + return d.addCallback(self._cbGoodAuthentication, + credentials) + return defer.fail(UnhandledCredentials("No checker for %s" % \ + ', '.join(map(reflect.qual, ifac)))) + + + def _cbGoodAuthentication(self, avatarId, credentials): + """ + Called if a checker has verified the credentials. We call our + L{areDone} method to see if the whole of the successful authentications + are enough. If they are, we return the avatar ID returned by the first + checker. + """ + if avatarId not in self.successfulCredentials: + self.successfulCredentials[avatarId] = [] + self.successfulCredentials[avatarId].append(credentials) + if self.areDone(avatarId): + del self.successfulCredentials[avatarId] + return avatarId + else: + raise error.NotEnoughAuthentication() + + + def areDone(self, avatarId): + """ + Override to determine if the authentication is finished for a given + avatarId. + + @param avatarId: the avatar returned by the first checker. For + this checker to function correctly, all the checkers must + return the same avatar ID. + """ + return True + + + +deprecatedModuleAttribute( + Version("Twisted", 15, 0, 0), + ("Please use twisted.conch.checkers.SSHPublicKeyChecker, " + "initialized with an instance of " + "twisted.conch.checkers.UNIXAuthorizedKeysFiles instead."), + __name__, "SSHPublicKeyDatabase") + + + +class IAuthorizedKeysDB(Interface): + """ + An object that provides valid authorized ssh keys mapped to usernames. + + @since: 15.0 + """ + def getAuthorizedKeys(avatarId): + """ + Gets an iterable of authorized keys that are valid for the given + C{avatarId}. + + @param avatarId: the ID of the avatar + @type avatarId: valid return value of + L{twisted.cred.checkers.ICredentialsChecker.requestAvatarId} + + @return: an iterable of L{twisted.conch.ssh.keys.Key} + """ + + + +def readAuthorizedKeyFile(fileobj, parseKey=keys.Key.fromString): + """ + Reads keys from an authorized keys file. Any non-comment line that cannot + be parsed as a key will be ignored, although that particular line will + be logged. + + @param fileobj: something from which to read lines which can be parsed + as keys + @type fileobj: L{file}-like object + + @param parseKey: a callable that takes a string and returns a + L{twisted.conch.ssh.keys.Key}, mainly to be used for testing. The + default is L{twisted.conch.ssh.keys.Key.fromString}. + @type parseKey: L{callable} + + @return: an iterable of L{twisted.conch.ssh.keys.Key} + @rtype: iterable + + @since: 15.0 + """ + for line in fileobj: + line = line.strip() + if line and not line.startswith(b'#'): # for comments + try: + yield parseKey(line) + except keys.BadKeyError as e: + log.msg('Unable to parse line "{0}" as a key: {1!s}' + .format(line, e)) + + + +def _keysFromFilepaths(filepaths, parseKey): + """ + Helper function that turns an iterable of filepaths into a generator of + keys. If any file cannot be read, a message is logged but it is + otherwise ignored. + + @param filepaths: iterable of L{twisted.python.filepath.FilePath}. + @type filepaths: iterable + + @param parseKey: a callable that takes a string and returns a + L{twisted.conch.ssh.keys.Key} + @type parseKey: L{callable} + + @return: generator of L{twisted.conch.ssh.keys.Key} + @rtype: generator + + @since: 15.0 + """ + for fp in filepaths: + if fp.exists(): + try: + with fp.open() as f: + for key in readAuthorizedKeyFile(f, parseKey): + yield key + except (IOError, OSError) as e: + log.msg("Unable to read {0}: {1!s}".format(fp.path, e)) + + + +@implementer(IAuthorizedKeysDB) +class InMemorySSHKeyDB(object): + """ + Object that provides SSH public keys based on a dictionary of usernames + mapped to L{twisted.conch.ssh.keys.Key}s. + + @since: 15.0 + """ + def __init__(self, mapping): + """ + Initializes a new L{InMemorySSHKeyDB}. + + @param mapping: mapping of usernames to iterables of + L{twisted.conch.ssh.keys.Key}s + @type mapping: L{dict} + + """ + self._mapping = mapping + + + def getAuthorizedKeys(self, username): + return self._mapping.get(username, []) + + + +@implementer(IAuthorizedKeysDB) +class UNIXAuthorizedKeysFiles(object): + """ + Object that provides SSH public keys based on public keys listed in + authorized_keys and authorized_keys2 files in UNIX user .ssh/ directories. + If any of the files cannot be read, a message is logged but that file is + otherwise ignored. + + @since: 15.0 + """ + def __init__(self, userdb=None, parseKey=keys.Key.fromString): + """ + Initializes a new L{UNIXAuthorizedKeysFiles}. + + @param userdb: access to the Unix user account and password database + (default is the Python module L{pwd}) + @type userdb: L{pwd}-like object + + @param parseKey: a callable that takes a string and returns a + L{twisted.conch.ssh.keys.Key}, mainly to be used for testing. The + default is L{twisted.conch.ssh.keys.Key.fromString}. + @type parseKey: L{callable} + """ + self._userdb = userdb + self._parseKey = parseKey + if userdb is None: + self._userdb = pwd + + + def getAuthorizedKeys(self, username): + try: + passwd = self._userdb.getpwnam(username) + except KeyError: + return () + + root = FilePath(passwd.pw_dir).child('.ssh') + files = ['authorized_keys', 'authorized_keys2'] + return _keysFromFilepaths((root.child(f) for f in files), + self._parseKey) + + + +@implementer(ICredentialsChecker) +class SSHPublicKeyChecker(object): + """ + Checker that authenticates SSH public keys, based on public keys listed in + authorized_keys and authorized_keys2 files in user .ssh/ directories. + + Initializing this checker with a L{UNIXAuthorizedKeysFiles} should be + used instead of L{twisted.conch.checkers.SSHPublicKeyDatabase}. + + @since: 15.0 + """ + credentialInterfaces = (ISSHPrivateKey,) + + def __init__(self, keydb): + """ + Initializes a L{SSHPublicKeyChecker}. + + @param keydb: a provider of L{IAuthorizedKeysDB} + @type keydb: L{IAuthorizedKeysDB} provider + """ + self._keydb = keydb + + + def requestAvatarId(self, credentials): + d = defer.maybeDeferred(self._sanityCheckKey, credentials) + d.addCallback(self._checkKey, credentials) + d.addCallback(self._verifyKey, credentials) + return d + + + def _sanityCheckKey(self, credentials): + """ + Checks whether the provided credentials are a valid SSH key with a + signature (does not actually verify the signature). + + @param credentials: the credentials offered by the user + @type credentials: L{ISSHPrivateKey} provider + + @raise ValidPublicKey: the credentials do not include a signature. See + L{error.ValidPublicKey} for more information. + + @raise BadKeyError: The key included with the credentials is not + recognized as a key. + + @return: the key in the credentials + @rtype: L{twisted.conch.ssh.keys.Key} + """ + if not credentials.signature: + raise error.ValidPublicKey() + + return keys.Key.fromString(credentials.blob) + + + def _checkKey(self, pubKey, credentials): + """ + Checks the public key against all authorized keys (if any) for the + user. + + @param pubKey: the key in the credentials (just to prevent it from + having to be calculated again) + @type pubKey: + + @param credentials: the credentials offered by the user + @type credentials: L{ISSHPrivateKey} provider + + @raise UnauthorizedLogin: If the key is not authorized, or if there + was any error obtaining a list of authorized keys for the user. + + @return: C{pubKey} if the key is authorized + @rtype: L{twisted.conch.ssh.keys.Key} + """ + if any(key == pubKey for key in + self._keydb.getAuthorizedKeys(credentials.username)): + return pubKey + + raise UnauthorizedLogin("Key not authorized") + + + def _verifyKey(self, pubKey, credentials): + """ + Checks whether the credentials themselves are valid, now that we know + if the key matches the user. + + @param pubKey: the key in the credentials (just to prevent it from + having to be calculated again) + @type pubKey: L{twisted.conch.ssh.keys.Key} + + @param credentials: the credentials offered by the user + @type credentials: L{ISSHPrivateKey} provider + + @raise UnauthorizedLogin: If the key signature is invalid or there + was any error verifying the signature. + + @return: The user's username, if authentication was successful + @rtype: L{bytes} + """ + try: + if pubKey.verify(credentials.signature, credentials.sigData): + return credentials.username + except: # Any error should be treated as a failed login + log.err() + raise UnauthorizedLogin('Error while verifying key') + + raise UnauthorizedLogin("Key signature invalid.") diff --git a/contrib/python/Twisted/py2/twisted/conch/client/__init__.py b/contrib/python/Twisted/py2/twisted/conch/client/__init__.py new file mode 100644 index 00000000000..f55d474db4c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/client/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# +""" +Client support code for Conch. + +Maintainer: Paul Swartz +""" diff --git a/contrib/python/Twisted/py2/twisted/conch/client/agent.py b/contrib/python/Twisted/py2/twisted/conch/client/agent.py new file mode 100644 index 00000000000..fdf08356f14 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/client/agent.py @@ -0,0 +1,73 @@ +# -*- test-case-name: twisted.conch.test.test_default -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Accesses the key agent for user authentication. + +Maintainer: Paul Swartz +""" + +import os + +from twisted.conch.ssh import agent, channel, keys +from twisted.internet import protocol, reactor +from twisted.python import log + + + +class SSHAgentClient(agent.SSHAgentClient): + + def __init__(self): + agent.SSHAgentClient.__init__(self) + self.blobs = [] + + + def getPublicKeys(self): + return self.requestIdentities().addCallback(self._cbPublicKeys) + + + def _cbPublicKeys(self, blobcomm): + log.msg('got %i public keys' % len(blobcomm)) + self.blobs = [x[0] for x in blobcomm] + + + def getPublicKey(self): + """ + Return a L{Key} from the first blob in C{self.blobs}, if any, or + return L{None}. + """ + if self.blobs: + return keys.Key.fromString(self.blobs.pop(0)) + return None + + + +class SSHAgentForwardingChannel(channel.SSHChannel): + + def channelOpen(self, specificData): + cc = protocol.ClientCreator(reactor, SSHAgentForwardingLocal) + d = cc.connectUNIX(os.environ['SSH_AUTH_SOCK']) + d.addCallback(self._cbGotLocal) + d.addErrback(lambda x:self.loseConnection()) + self.buf = '' + + + def _cbGotLocal(self, local): + self.local = local + self.dataReceived = self.local.transport.write + self.local.dataReceived = self.write + + + def dataReceived(self, data): + self.buf += data + + + def closed(self): + if self.local: + self.local.loseConnection() + self.local = None + + +class SSHAgentForwardingLocal(protocol.Protocol): + pass diff --git a/contrib/python/Twisted/py2/twisted/conch/client/connect.py b/contrib/python/Twisted/py2/twisted/conch/client/connect.py new file mode 100644 index 00000000000..ac47187e49b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/client/connect.py @@ -0,0 +1,21 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# +from twisted.conch.client import direct + +connectTypes = {"direct" : direct.connect} + +def connect(host, port, options, verifyHostKey, userAuthObject): + useConnects = ['direct'] + return _ebConnect(None, useConnects, host, port, options, verifyHostKey, + userAuthObject) + +def _ebConnect(f, useConnects, host, port, options, vhk, uao): + if not useConnects: + return f + connectType = useConnects.pop(0) + f = connectTypes[connectType] + d = f(host, port, options, vhk, uao) + d.addErrback(_ebConnect, useConnects, host, port, options, vhk, uao) + return d diff --git a/contrib/python/Twisted/py2/twisted/conch/client/default.py b/contrib/python/Twisted/py2/twisted/conch/client/default.py new file mode 100644 index 00000000000..ff2d6353140 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/client/default.py @@ -0,0 +1,349 @@ +# -*- test-case-name: twisted.conch.test.test_knownhosts,twisted.conch.test.test_default -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Various classes and functions for implementing user-interaction in the +command-line conch client. + +You probably shouldn't use anything in this module directly, since it assumes +you are sitting at an interactive terminal. For example, to programmatically +interact with a known_hosts database, use L{twisted.conch.client.knownhosts}. +""" + +from __future__ import print_function + +from twisted.python import log +from twisted.python.compat import ( + nativeString, raw_input, _PY3, _b64decodebytes as decodebytes) +from twisted.python.filepath import FilePath + +from twisted.conch.error import ConchError +from twisted.conch.ssh import common, keys, userauth +from twisted.internet import defer, protocol, reactor + +from twisted.conch.client.knownhosts import KnownHostsFile, ConsoleUI + +from twisted.conch.client import agent + +import os, sys, getpass, contextlib + +if _PY3: + import io + +# The default location of the known hosts file (probably should be parsed out +# of an ssh config file someday). +_KNOWN_HOSTS = "~/.ssh/known_hosts" + + +# This name is bound so that the unit tests can use 'patch' to override it. +_open = open + +def verifyHostKey(transport, host, pubKey, fingerprint): + """ + Verify a host's key. + + This function is a gross vestige of some bad factoring in the client + internals. The actual implementation, and a better signature of this logic + is in L{KnownHostsFile.verifyHostKey}. This function is not deprecated yet + because the callers have not yet been rehabilitated, but they should + eventually be changed to call that method instead. + + However, this function does perform two functions not implemented by + L{KnownHostsFile.verifyHostKey}. It determines the path to the user's + known_hosts file based on the options (which should really be the options + object's job), and it provides an opener to L{ConsoleUI} which opens + '/dev/tty' so that the user will be prompted on the tty of the process even + if the input and output of the process has been redirected. This latter + part is, somewhat obviously, not portable, but I don't know of a portable + equivalent that could be used. + + @param host: Due to a bug in L{SSHClientTransport.verifyHostKey}, this is + always the dotted-quad IP address of the host being connected to. + @type host: L{str} + + @param transport: the client transport which is attempting to connect to + the given host. + @type transport: L{SSHClientTransport} + + @param fingerprint: the fingerprint of the given public key, in + xx:xx:xx:... format. This is ignored in favor of getting the fingerprint + from the key itself. + @type fingerprint: L{str} + + @param pubKey: The public key of the server being connected to. + @type pubKey: L{str} + + @return: a L{Deferred} which fires with C{1} if the key was successfully + verified, or fails if the key could not be successfully verified. Failure + types may include L{HostKeyChanged}, L{UserRejectedKey}, L{IOError} or + L{KeyboardInterrupt}. + """ + actualHost = transport.factory.options['host'] + actualKey = keys.Key.fromString(pubKey) + kh = KnownHostsFile.fromPath(FilePath( + transport.factory.options['known-hosts'] + or os.path.expanduser(_KNOWN_HOSTS) + )) + ui = ConsoleUI(lambda : _open("/dev/tty", "r+b", buffering=0)) + return kh.verifyHostKey(ui, actualHost, host, actualKey) + + + +def isInKnownHosts(host, pubKey, options): + """ + Checks to see if host is in the known_hosts file for the user. + + @return: 0 if it isn't, 1 if it is and is the same, 2 if it's changed. + @rtype: L{int} + """ + keyType = common.getNS(pubKey)[0] + retVal = 0 + + if not options['known-hosts'] and not os.path.exists(os.path.expanduser('~/.ssh/')): + print('Creating ~/.ssh directory...') + os.mkdir(os.path.expanduser('~/.ssh')) + kh_file = options['known-hosts'] or _KNOWN_HOSTS + try: + known_hosts = open(os.path.expanduser(kh_file), 'rb') + except IOError: + return 0 + with known_hosts: + for line in known_hosts.readlines(): + split = line.split() + if len(split) < 3: + continue + hosts, hostKeyType, encodedKey = split[:3] + if host not in hosts.split(b','): # incorrect host + continue + if hostKeyType != keyType: # incorrect type of key + continue + try: + decodedKey = decodebytes(encodedKey) + except: + continue + if decodedKey == pubKey: + return 1 + else: + retVal = 2 + return retVal + + + +def getHostKeyAlgorithms(host, options): + """ + Look in known_hosts for a key corresponding to C{host}. + This can be used to change the order of supported key types + in the KEXINIT packet. + + @type host: L{str} + @param host: the host to check in known_hosts + @type options: L{twisted.conch.client.options.ConchOptions} + @param options: options passed to client + @return: L{list} of L{str} representing key types or L{None}. + """ + knownHosts = KnownHostsFile.fromPath(FilePath( + options['known-hosts'] + or os.path.expanduser(_KNOWN_HOSTS) + )) + keyTypes = [] + for entry in knownHosts.iterentries(): + if entry.matchesHost(host): + if entry.keyType not in keyTypes: + keyTypes.append(entry.keyType) + return keyTypes or None + + + +class SSHUserAuthClient(userauth.SSHUserAuthClient): + + def __init__(self, user, options, *args): + userauth.SSHUserAuthClient.__init__(self, user, *args) + self.keyAgent = None + self.options = options + self.usedFiles = [] + if not options.identitys: + options.identitys = ['~/.ssh/id_rsa', '~/.ssh/id_dsa'] + + + def serviceStarted(self): + if 'SSH_AUTH_SOCK' in os.environ and not self.options['noagent']: + log.msg('using agent') + cc = protocol.ClientCreator(reactor, agent.SSHAgentClient) + d = cc.connectUNIX(os.environ['SSH_AUTH_SOCK']) + d.addCallback(self._setAgent) + d.addErrback(self._ebSetAgent) + else: + userauth.SSHUserAuthClient.serviceStarted(self) + + + def serviceStopped(self): + if self.keyAgent: + self.keyAgent.transport.loseConnection() + self.keyAgent = None + + + def _setAgent(self, a): + self.keyAgent = a + d = self.keyAgent.getPublicKeys() + d.addBoth(self._ebSetAgent) + return d + + + def _ebSetAgent(self, f): + userauth.SSHUserAuthClient.serviceStarted(self) + + + def _getPassword(self, prompt): + """ + Prompt for a password using L{getpass.getpass}. + + @param prompt: Written on tty to ask for the input. + @type prompt: L{str} + @return: The input. + @rtype: L{str} + """ + with self._replaceStdoutStdin(): + try: + p = getpass.getpass(prompt) + return p + except (KeyboardInterrupt, IOError): + print() + raise ConchError('PEBKAC') + + + def getPassword(self, prompt = None): + if prompt: + prompt = nativeString(prompt) + else: + prompt = ("%s@%s's password: " % + (nativeString(self.user), self.transport.transport.getPeer().host)) + try: + # We don't know the encoding the other side is using, + # signaling that is not part of the SSH protocol. But + # using our defaultencoding is better than just going for + # ASCII. + p = self._getPassword(prompt).encode(sys.getdefaultencoding()) + return defer.succeed(p) + except ConchError: + return defer.fail() + + + def getPublicKey(self): + """ + Get a public key from the key agent if possible, otherwise look in + the next configured identity file for one. + """ + if self.keyAgent: + key = self.keyAgent.getPublicKey() + if key is not None: + return key + files = [x for x in self.options.identitys if x not in self.usedFiles] + log.msg(str(self.options.identitys)) + log.msg(str(files)) + if not files: + return None + file = files[0] + log.msg(file) + self.usedFiles.append(file) + file = os.path.expanduser(file) + file += '.pub' + if not os.path.exists(file): + return self.getPublicKey() # try again + try: + return keys.Key.fromFile(file) + except keys.BadKeyError: + return self.getPublicKey() # try again + + + def signData(self, publicKey, signData): + """ + Extend the base signing behavior by using an SSH agent to sign the + data, if one is available. + + @type publicKey: L{Key} + @type signData: L{bytes} + """ + if not self.usedFiles: # agent key + return self.keyAgent.signData(publicKey.blob(), signData) + else: + return userauth.SSHUserAuthClient.signData(self, publicKey, signData) + + + def getPrivateKey(self): + """ + Try to load the private key from the last used file identified by + C{getPublicKey}, potentially asking for the passphrase if the key is + encrypted. + """ + file = os.path.expanduser(self.usedFiles[-1]) + if not os.path.exists(file): + return None + try: + return defer.succeed(keys.Key.fromFile(file)) + except keys.EncryptedKeyError: + for i in range(3): + prompt = "Enter passphrase for key '%s': " % self.usedFiles[-1] + try: + p = self._getPassword(prompt).encode( + sys.getfilesystemencoding()) + return defer.succeed(keys.Key.fromFile(file, passphrase=p)) + except (keys.BadKeyError, ConchError): + pass + return defer.fail(ConchError('bad password')) + raise + except KeyboardInterrupt: + print() + reactor.stop() + + + def getGenericAnswers(self, name, instruction, prompts): + responses = [] + with self._replaceStdoutStdin(): + if name: + print(name.decode("utf-8")) + if instruction: + print(instruction.decode("utf-8")) + for prompt, echo in prompts: + prompt = prompt.decode("utf-8") + if echo: + responses.append(raw_input(prompt)) + else: + responses.append(getpass.getpass(prompt)) + return defer.succeed(responses) + + + @classmethod + def _openTty(cls): + """ + Open /dev/tty as two streams one in read, one in write mode, + and return them. + + @return: File objects for reading and writing to /dev/tty, + corresponding to standard input and standard output. + @rtype: A L{tuple} of L{io.TextIOWrapper} on Python 3. + A L{tuple} of binary files on Python 2. + """ + stdin = open("/dev/tty", "rb") + stdout = open("/dev/tty", "wb") + if _PY3: + stdin = io.TextIOWrapper(stdin) + stdout = io.TextIOWrapper(stdout) + return stdin, stdout + + + @classmethod + @contextlib.contextmanager + def _replaceStdoutStdin(cls): + """ + Contextmanager that replaces stdout and stdin with /dev/tty + and resets them when it is done. + """ + oldout, oldin = sys.stdout, sys.stdin + sys.stdin, sys.stdout = cls._openTty() + try: + yield + finally: + sys.stdout.close() + sys.stdin.close() + sys.stdout, sys.stdin = oldout, oldin diff --git a/contrib/python/Twisted/py2/twisted/conch/client/direct.py b/contrib/python/Twisted/py2/twisted/conch/client/direct.py new file mode 100644 index 00000000000..601a9d2dc31 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/client/direct.py @@ -0,0 +1,109 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +from __future__ import print_function + +from twisted.internet import defer, protocol, reactor +from twisted.conch import error +from twisted.conch.ssh import transport +from twisted.python import log + + + +class SSHClientFactory(protocol.ClientFactory): + + def __init__(self, d, options, verifyHostKey, userAuthObject): + self.d = d + self.options = options + self.verifyHostKey = verifyHostKey + self.userAuthObject = userAuthObject + + + def clientConnectionLost(self, connector, reason): + if self.options['reconnect']: + connector.connect() + + + def clientConnectionFailed(self, connector, reason): + if self.d is None: + return + d, self.d = self.d, None + d.errback(reason) + + + def buildProtocol(self, addr): + trans = SSHClientTransport(self) + if self.options['ciphers']: + trans.supportedCiphers = self.options['ciphers'] + if self.options['macs']: + trans.supportedMACs = self.options['macs'] + if self.options['compress']: + trans.supportedCompressions[0:1] = ['zlib'] + if self.options['host-key-algorithms']: + trans.supportedPublicKeys = self.options['host-key-algorithms'] + return trans + + + +class SSHClientTransport(transport.SSHClientTransport): + + def __init__(self, factory): + self.factory = factory + self.unixServer = None + + + def connectionLost(self, reason): + if self.unixServer: + d = self.unixServer.stopListening() + self.unixServer = None + else: + d = defer.succeed(None) + d.addCallback(lambda x: + transport.SSHClientTransport.connectionLost(self, reason)) + + + def receiveError(self, code, desc): + if self.factory.d is None: + return + d, self.factory.d = self.factory.d, None + d.errback(error.ConchError(desc, code)) + + + def sendDisconnect(self, code, reason): + if self.factory.d is None: + return + d, self.factory.d = self.factory.d, None + transport.SSHClientTransport.sendDisconnect(self, code, reason) + d.errback(error.ConchError(reason, code)) + + + def receiveDebug(self, alwaysDisplay, message, lang): + log.msg('Received Debug Message: %s' % message) + if alwaysDisplay: # XXX what should happen here? + print(message) + + + def verifyHostKey(self, pubKey, fingerprint): + return self.factory.verifyHostKey(self, self.transport.getPeer().host, pubKey, + fingerprint) + + + def setService(self, service): + log.msg('setting client server to %s' % service) + transport.SSHClientTransport.setService(self, service) + if service.name != 'ssh-userauth' and self.factory.d is not None: + d, self.factory.d = self.factory.d, None + d.callback(None) + + + def connectionSecure(self): + self.requestService(self.factory.userAuthObject) + + + +def connect(host, port, options, verifyHostKey, userAuthObject): + d = defer.Deferred() + factory = SSHClientFactory(d, options, verifyHostKey, userAuthObject) + reactor.connectTCP(host, port, factory) + return d diff --git a/contrib/python/Twisted/py2/twisted/conch/client/knownhosts.py b/contrib/python/Twisted/py2/twisted/conch/client/knownhosts.py new file mode 100644 index 00000000000..aa0a622d411 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/client/knownhosts.py @@ -0,0 +1,630 @@ +# -*- test-case-name: twisted.conch.test.test_knownhosts -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An implementation of the OpenSSH known_hosts database. + +@since: 8.2 +""" + +from __future__ import absolute_import, division + +import hmac +from binascii import Error as DecodeError, b2a_base64, a2b_base64 +from contextlib import closing +from hashlib import sha1 +import sys + +from zope.interface import implementer + +from twisted.conch.interfaces import IKnownHostEntry +from twisted.conch.error import HostKeyChanged, UserRejectedKey, InvalidEntry +from twisted.conch.ssh.keys import Key, BadKeyError, FingerprintFormats +from twisted.internet import defer +from twisted.python import log +from twisted.python.compat import nativeString, unicode +from twisted.python.randbytes import secureRandom +from twisted.python.util import FancyEqMixin + + +def _b64encode(s): + """ + Encode a binary string as base64 with no trailing newline. + + @param s: The string to encode. + @type s: L{bytes} + + @return: The base64-encoded string. + @rtype: L{bytes} + """ + return b2a_base64(s).strip() + + + +def _extractCommon(string): + """ + Extract common elements of base64 keys from an entry in a hosts file. + + @param string: A known hosts file entry (a single line). + @type string: L{bytes} + + @return: a 4-tuple of hostname data (L{bytes}), ssh key type (L{bytes}), key + (L{Key}), and comment (L{bytes} or L{None}). The hostname data is + simply the beginning of the line up to the first occurrence of + whitespace. + @rtype: L{tuple} + """ + elements = string.split(None, 2) + if len(elements) != 3: + raise InvalidEntry() + hostnames, keyType, keyAndComment = elements + splitkey = keyAndComment.split(None, 1) + if len(splitkey) == 2: + keyString, comment = splitkey + comment = comment.rstrip(b"\n") + else: + keyString = splitkey[0] + comment = None + key = Key.fromString(a2b_base64(keyString)) + return hostnames, keyType, key, comment + + + +class _BaseEntry(object): + """ + Abstract base of both hashed and non-hashed entry objects, since they + represent keys and key types the same way. + + @ivar keyType: The type of the key; either ssh-dss or ssh-rsa. + @type keyType: L{bytes} + + @ivar publicKey: The server public key indicated by this line. + @type publicKey: L{twisted.conch.ssh.keys.Key} + + @ivar comment: Trailing garbage after the key line. + @type comment: L{bytes} + """ + + def __init__(self, keyType, publicKey, comment): + self.keyType = keyType + self.publicKey = publicKey + self.comment = comment + + + def matchesKey(self, keyObject): + """ + Check to see if this entry matches a given key object. + + @param keyObject: A public key object to check. + @type keyObject: L{Key} + + @return: C{True} if this entry's key matches C{keyObject}, C{False} + otherwise. + @rtype: L{bool} + """ + return self.publicKey == keyObject + + + +@implementer(IKnownHostEntry) +class PlainEntry(_BaseEntry): + """ + A L{PlainEntry} is a representation of a plain-text entry in a known_hosts + file. + + @ivar _hostnames: the list of all host-names associated with this entry. + @type _hostnames: L{list} of L{bytes} + """ + + def __init__(self, hostnames, keyType, publicKey, comment): + self._hostnames = hostnames + super(PlainEntry, self).__init__(keyType, publicKey, comment) + + + @classmethod + def fromString(cls, string): + """ + Parse a plain-text entry in a known_hosts file, and return a + corresponding L{PlainEntry}. + + @param string: a space-separated string formatted like "hostname + key-type base64-key-data comment". + + @type string: L{bytes} + + @raise DecodeError: if the key is not valid encoded as valid base64. + + @raise InvalidEntry: if the entry does not have the right number of + elements and is therefore invalid. + + @raise BadKeyError: if the key, once decoded from base64, is not + actually an SSH key. + + @return: an IKnownHostEntry representing the hostname and key in the + input line. + + @rtype: L{PlainEntry} + """ + hostnames, keyType, key, comment = _extractCommon(string) + self = cls(hostnames.split(b","), keyType, key, comment) + return self + + + def matchesHost(self, hostname): + """ + Check to see if this entry matches a given hostname. + + @param hostname: A hostname or IP address literal to check against this + entry. + @type hostname: L{bytes} + + @return: C{True} if this entry is for the given hostname or IP address, + C{False} otherwise. + @rtype: L{bool} + """ + if isinstance(hostname, unicode): + hostname = hostname.encode("utf-8") + return hostname in self._hostnames + + + def toString(self): + """ + Implement L{IKnownHostEntry.toString} by recording the comma-separated + hostnames, key type, and base-64 encoded key. + + @return: The string representation of this entry, with unhashed hostname + information. + @rtype: L{bytes} + """ + fields = [b','.join(self._hostnames), + self.keyType, + _b64encode(self.publicKey.blob())] + if self.comment is not None: + fields.append(self.comment) + return b' '.join(fields) + + + +@implementer(IKnownHostEntry) +class UnparsedEntry(object): + """ + L{UnparsedEntry} is an entry in a L{KnownHostsFile} which can't actually be + parsed; therefore it matches no keys and no hosts. + """ + + def __init__(self, string): + """ + Create an unparsed entry from a line in a known_hosts file which cannot + otherwise be parsed. + """ + self._string = string + + + def matchesHost(self, hostname): + """ + Always returns False. + """ + return False + + + def matchesKey(self, key): + """ + Always returns False. + """ + return False + + + def toString(self): + """ + Returns the input line, without its newline if one was given. + + @return: The string representation of this entry, almost exactly as was + used to initialize this entry but without a trailing newline. + @rtype: L{bytes} + """ + return self._string.rstrip(b"\n") + + + +def _hmacedString(key, string): + """ + Return the SHA-1 HMAC hash of the given key and string. + + @param key: The HMAC key. + @type key: L{bytes} + + @param string: The string to be hashed. + @type string: L{bytes} + + @return: The keyed hash value. + @rtype: L{bytes} + """ + hash = hmac.HMAC(key, digestmod=sha1) + if isinstance(string, unicode): + string = string.encode("utf-8") + hash.update(string) + return hash.digest() + + + +@implementer(IKnownHostEntry) +class HashedEntry(_BaseEntry, FancyEqMixin): + """ + A L{HashedEntry} is a representation of an entry in a known_hosts file + where the hostname has been hashed and salted. + + @ivar _hostSalt: the salt to combine with a hostname for hashing. + + @ivar _hostHash: the hashed representation of the hostname. + + @cvar MAGIC: the 'hash magic' string used to identify a hashed line in a + known_hosts file as opposed to a plaintext one. + """ + + MAGIC = b'|1|' + + compareAttributes = ( + "_hostSalt", "_hostHash", "keyType", "publicKey", "comment") + + def __init__(self, hostSalt, hostHash, keyType, publicKey, comment): + self._hostSalt = hostSalt + self._hostHash = hostHash + super(HashedEntry, self).__init__(keyType, publicKey, comment) + + + @classmethod + def fromString(cls, string): + """ + Load a hashed entry from a string representing a line in a known_hosts + file. + + @param string: A complete single line from a I{known_hosts} file, + formatted as defined by OpenSSH. + @type string: L{bytes} + + @raise DecodeError: if the key, the hostname, or the is not valid + encoded as valid base64 + + @raise InvalidEntry: if the entry does not have the right number of + elements and is therefore invalid, or the host/hash portion contains + more items than just the host and hash. + + @raise BadKeyError: if the key, once decoded from base64, is not + actually an SSH key. + + @return: The newly created L{HashedEntry} instance, initialized with the + information from C{string}. + """ + stuff, keyType, key, comment = _extractCommon(string) + saltAndHash = stuff[len(cls.MAGIC):].split(b"|") + if len(saltAndHash) != 2: + raise InvalidEntry() + hostSalt, hostHash = saltAndHash + self = cls(a2b_base64(hostSalt), a2b_base64(hostHash), + keyType, key, comment) + return self + + + def matchesHost(self, hostname): + """ + Implement L{IKnownHostEntry.matchesHost} to compare the hash of the + input to the stored hash. + + @param hostname: A hostname or IP address literal to check against this + entry. + @type hostname: L{bytes} + + @return: C{True} if this entry is for the given hostname or IP address, + C{False} otherwise. + @rtype: L{bool} + """ + return (_hmacedString(self._hostSalt, hostname) == self._hostHash) + + + def toString(self): + """ + Implement L{IKnownHostEntry.toString} by base64-encoding the salt, host + hash, and key. + + @return: The string representation of this entry, with the hostname part + hashed. + @rtype: L{bytes} + """ + fields = [self.MAGIC + b'|'.join([_b64encode(self._hostSalt), + _b64encode(self._hostHash)]), + self.keyType, + _b64encode(self.publicKey.blob())] + if self.comment is not None: + fields.append(self.comment) + return b' '.join(fields) + + + +class KnownHostsFile(object): + """ + A structured representation of an OpenSSH-format ~/.ssh/known_hosts file. + + @ivar _added: A list of L{IKnownHostEntry} providers which have been added + to this instance in memory but not yet saved. + + @ivar _clobber: A flag indicating whether the current contents of the save + path will be disregarded and potentially overwritten or not. If + C{True}, this will be done. If C{False}, entries in the save path will + be read and new entries will be saved by appending rather than + overwriting. + @type _clobber: L{bool} + + @ivar _savePath: See C{savePath} parameter of L{__init__}. + """ + + def __init__(self, savePath): + """ + Create a new, empty KnownHostsFile. + + Unless you want to erase the current contents of C{savePath}, you want + to use L{KnownHostsFile.fromPath} instead. + + @param savePath: The L{FilePath} to which to save new entries. + @type savePath: L{FilePath} + """ + self._added = [] + self._savePath = savePath + self._clobber = True + + + @property + def savePath(self): + """ + @see: C{savePath} parameter of L{__init__} + """ + return self._savePath + + + def iterentries(self): + """ + Iterate over the host entries in this file. + + @return: An iterable the elements of which provide L{IKnownHostEntry}. + There is an element for each entry in the file as well as an element + for each added but not yet saved entry. + @rtype: iterable of L{IKnownHostEntry} providers + """ + for entry in self._added: + yield entry + + if self._clobber: + return + + try: + fp = self._savePath.open() + except IOError: + return + + with fp: + for line in fp: + try: + if line.startswith(HashedEntry.MAGIC): + entry = HashedEntry.fromString(line) + else: + entry = PlainEntry.fromString(line) + except (DecodeError, InvalidEntry, BadKeyError): + entry = UnparsedEntry(line) + yield entry + + + def hasHostKey(self, hostname, key): + """ + Check for an entry with matching hostname and key. + + @param hostname: A hostname or IP address literal to check for. + @type hostname: L{bytes} + + @param key: The public key to check for. + @type key: L{Key} + + @return: C{True} if the given hostname and key are present in this file, + C{False} if they are not. + @rtype: L{bool} + + @raise HostKeyChanged: if the host key found for the given hostname + does not match the given key. + """ + for lineidx, entry in enumerate(self.iterentries(), -len(self._added)): + if entry.matchesHost(hostname) and entry.keyType == key.sshType(): + if entry.matchesKey(key): + return True + else: + # Notice that lineidx is 0-based but HostKeyChanged.lineno + # is 1-based. + if lineidx < 0: + line = None + path = None + else: + line = lineidx + 1 + path = self._savePath + raise HostKeyChanged(entry, path, line) + return False + + + def verifyHostKey(self, ui, hostname, ip, key): + """ + Verify the given host key for the given IP and host, asking for + confirmation from, and notifying, the given UI about changes to this + file. + + @param ui: The user interface to request an IP address from. + + @param hostname: The hostname that the user requested to connect to. + + @param ip: The string representation of the IP address that is actually + being connected to. + + @param key: The public key of the server. + + @return: a L{Deferred} that fires with True when the key has been + verified, or fires with an errback when the key either cannot be + verified or has changed. + @rtype: L{Deferred} + """ + hhk = defer.maybeDeferred(self.hasHostKey, hostname, key) + def gotHasKey(result): + if result: + if not self.hasHostKey(ip, key): + ui.warn("Warning: Permanently added the %s host key for " + "IP address '%s' to the list of known hosts." % + (key.type(), nativeString(ip))) + self.addHostKey(ip, key) + self.save() + return result + else: + def promptResponse(response): + if response: + self.addHostKey(hostname, key) + self.addHostKey(ip, key) + self.save() + return response + else: + raise UserRejectedKey() + + keytype = key.type() + + if keytype == "EC": + keytype = "ECDSA" + + prompt = ( + "The authenticity of host '%s (%s)' " + "can't be established.\n" + "%s key fingerprint is SHA256:%s.\n" + "Are you sure you want to continue connecting (yes/no)? " % + (nativeString(hostname), nativeString(ip), keytype, + key.fingerprint(format=FingerprintFormats.SHA256_BASE64))) + proceed = ui.prompt(prompt.encode(sys.getdefaultencoding())) + return proceed.addCallback(promptResponse) + return hhk.addCallback(gotHasKey) + + + def addHostKey(self, hostname, key): + """ + Add a new L{HashedEntry} to the key database. + + Note that you still need to call L{KnownHostsFile.save} if you wish + these changes to be persisted. + + @param hostname: A hostname or IP address literal to associate with the + new entry. + @type hostname: L{bytes} + + @param key: The public key to associate with the new entry. + @type key: L{Key} + + @return: The L{HashedEntry} that was added. + @rtype: L{HashedEntry} + """ + salt = secureRandom(20) + keyType = key.sshType() + entry = HashedEntry(salt, _hmacedString(salt, hostname), + keyType, key, None) + self._added.append(entry) + return entry + + + def save(self): + """ + Save this L{KnownHostsFile} to the path it was loaded from. + """ + p = self._savePath.parent() + if not p.isdir(): + p.makedirs() + + if self._clobber: + mode = "wb" + else: + mode = "ab" + + with self._savePath.open(mode) as hostsFileObj: + if self._added: + hostsFileObj.write( + b"\n".join([entry.toString() for entry in self._added]) + + b"\n") + self._added = [] + self._clobber = False + + + @classmethod + def fromPath(cls, path): + """ + Create a new L{KnownHostsFile}, potentially reading existing known + hosts information from the given file. + + @param path: A path object to use for both reading contents from and + later saving to. If no file exists at this path, it is not an + error; a L{KnownHostsFile} with no entries is returned. + @type path: L{FilePath} + + @return: A L{KnownHostsFile} initialized with entries from C{path}. + @rtype: L{KnownHostsFile} + """ + knownHosts = cls(path) + knownHosts._clobber = False + return knownHosts + + + +class ConsoleUI(object): + """ + A UI object that can ask true/false questions and post notifications on the + console, to be used during key verification. + """ + def __init__(self, opener): + """ + @param opener: A no-argument callable which should open a console + binary-mode file-like object to be used for reading and writing. + This initializes the C{opener} attribute. + @type opener: callable taking no arguments and returning a read/write + file-like object + """ + self.opener = opener + + + def prompt(self, text): + """ + Write the given text as a prompt to the console output, then read a + result from the console input. + + @param text: Something to present to a user to solicit a yes or no + response. + @type text: L{bytes} + + @return: a L{Deferred} which fires with L{True} when the user answers + 'yes' and L{False} when the user answers 'no'. It may errback if + there were any I/O errors. + """ + d = defer.succeed(None) + def body(ignored): + with closing(self.opener()) as f: + f.write(text) + while True: + answer = f.readline().strip().lower() + if answer == b'yes': + return True + elif answer == b'no': + return False + else: + f.write(b"Please type 'yes' or 'no': ") + return d.addCallback(body) + + + def warn(self, text): + """ + Notify the user (non-interactively) of the provided text, by writing it + to the console. + + @param text: Some information the user is to be made aware of. + @type text: L{bytes} + """ + try: + with closing(self.opener()) as f: + f.write(text) + except: + log.err() diff --git a/contrib/python/Twisted/py2/twisted/conch/client/options.py b/contrib/python/Twisted/py2/twisted/conch/client/options.py new file mode 100644 index 00000000000..5630fce2509 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/client/options.py @@ -0,0 +1,103 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# +from twisted.conch.ssh.transport import SSHClientTransport, SSHCiphers +from twisted.python import usage +from twisted.python.compat import unicode + +import sys + +class ConchOptions(usage.Options): + + optParameters = [['user', 'l', None, 'Log in using this user name.'], + ['identity', 'i', None], + ['ciphers', 'c', None], + ['macs', 'm', None], + ['port', 'p', None, 'Connect to this port. Server must be on the same port.'], + ['option', 'o', None, 'Ignored OpenSSH options'], + ['host-key-algorithms', '', None], + ['known-hosts', '', None, 'File to check for host keys'], + ['user-authentications', '', None, 'Types of user authentications to use.'], + ['logfile', '', None, 'File to log to, or - for stdout'], + ] + + optFlags = [['version', 'V', 'Display version number only.'], + ['compress', 'C', 'Enable compression.'], + ['log', 'v', 'Enable logging (defaults to stderr)'], + ['nox11', 'x', 'Disable X11 connection forwarding (default)'], + ['agent', 'A', 'Enable authentication agent forwarding'], + ['noagent', 'a', 'Disable authentication agent forwarding (default)'], + ['reconnect', 'r', 'Reconnect to the server if the connection is lost.'], + ] + + compData = usage.Completions( + mutuallyExclusive=[("agent", "noagent")], + optActions={ + "user": usage.CompleteUsernames(), + "ciphers": usage.CompleteMultiList( + SSHCiphers.cipherMap.keys(), + descr='ciphers to choose from'), + "macs": usage.CompleteMultiList( + SSHCiphers.macMap.keys(), + descr='macs to choose from'), + "host-key-algorithms": usage.CompleteMultiList( + SSHClientTransport.supportedPublicKeys, + descr='host key algorithms to choose from'), + #"user-authentications": usage.CompleteMultiList(? + # descr='user authentication types' ), + }, + extraActions=[usage.CompleteUserAtHost(), + usage.Completer(descr="command"), + usage.Completer(descr='argument', + repeat=True)] + ) + + def __init__(self, *args, **kw): + usage.Options.__init__(self, *args, **kw) + self.identitys = [] + self.conns = None + + def opt_identity(self, i): + """Identity for public-key authentication""" + self.identitys.append(i) + + def opt_ciphers(self, ciphers): + "Select encryption algorithms" + ciphers = ciphers.split(',') + for cipher in ciphers: + if cipher not in SSHCiphers.cipherMap: + sys.exit("Unknown cipher type '%s'" % cipher) + self['ciphers'] = ciphers + + + def opt_macs(self, macs): + "Specify MAC algorithms" + if isinstance(macs, unicode): + macs = macs.encode("utf-8") + macs = macs.split(b',') + for mac in macs: + if mac not in SSHCiphers.macMap: + sys.exit("Unknown mac type '%r'" % mac) + self['macs'] = macs + + def opt_host_key_algorithms(self, hkas): + "Select host key algorithms" + if isinstance(hkas, unicode): + hkas = hkas.encode("utf-8") + hkas = hkas.split(b',') + for hka in hkas: + if hka not in SSHClientTransport.supportedPublicKeys: + sys.exit("Unknown host key type '%r'" % hka) + self['host-key-algorithms'] = hkas + + def opt_user_authentications(self, uas): + "Choose how to authenticate to the remote server" + if isinstance(uas, unicode): + uas = uas.encode("utf-8") + self['user-authentications'] = uas.split(b',') + +# def opt_compress(self): +# "Enable compression" +# self.enableCompression = 1 +# SSHClientTransport.supportedCompressions[0:1] = ['zlib'] diff --git a/contrib/python/Twisted/py2/twisted/conch/endpoints.py b/contrib/python/Twisted/py2/twisted/conch/endpoints.py new file mode 100644 index 00000000000..2e19d0870f3 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/endpoints.py @@ -0,0 +1,872 @@ +# -*- test-case-name: twisted.conch.test.test_endpoints -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Endpoint implementations of various SSH interactions. +""" + +__all__ = [ + 'AuthenticationFailed', 'SSHCommandAddress', 'SSHCommandClientEndpoint'] + +from struct import unpack +from os.path import expanduser + +import signal + +from zope.interface import Interface, implementer + +from twisted.logger import Logger +from twisted.python.compat import nativeString, networkString +from twisted.python.filepath import FilePath +from twisted.python.failure import Failure +from twisted.internet.error import ConnectionDone, ProcessTerminated +from twisted.internet.interfaces import IStreamClientEndpoint +from twisted.internet.protocol import Factory +from twisted.internet.defer import Deferred, succeed, CancelledError +from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol + +from twisted.conch.ssh.keys import Key +from twisted.conch.ssh.common import getNS, NS +from twisted.conch.ssh.transport import SSHClientTransport +from twisted.conch.ssh.connection import SSHConnection +from twisted.conch.ssh.userauth import SSHUserAuthClient +from twisted.conch.ssh.channel import SSHChannel +from twisted.conch.client.knownhosts import ConsoleUI, KnownHostsFile +from twisted.conch.client.agent import SSHAgentClient +from twisted.conch.client.default import _KNOWN_HOSTS + + +class AuthenticationFailed(Exception): + """ + An SSH session could not be established because authentication was not + successful. + """ + + + +# This should be public. See #6541. +class _ISSHConnectionCreator(Interface): + """ + An L{_ISSHConnectionCreator} knows how to create SSH connections somehow. + """ + def secureConnection(): + """ + Return a new, connected, secured, but not yet authenticated instance of + L{twisted.conch.ssh.transport.SSHServerTransport} or + L{twisted.conch.ssh.transport.SSHClientTransport}. + """ + + + def cleanupConnection(connection, immediate): + """ + Perform cleanup necessary for a connection object previously returned + from this creator's C{secureConnection} method. + + @param connection: An L{twisted.conch.ssh.transport.SSHServerTransport} + or L{twisted.conch.ssh.transport.SSHClientTransport} returned by a + previous call to C{secureConnection}. It is no longer needed by + the caller of that method and may be closed or otherwise cleaned up + as necessary. + + @param immediate: If C{True} don't wait for any network communication, + just close the connection immediately and as aggressively as + necessary. + """ + + + +class SSHCommandAddress(object): + """ + An L{SSHCommandAddress} instance represents the address of an SSH server, a + username which was used to authenticate with that server, and a command + which was run there. + + @ivar server: See L{__init__} + @ivar username: See L{__init__} + @ivar command: See L{__init__} + """ + def __init__(self, server, username, command): + """ + @param server: The address of the SSH server on which the command is + running. + @type server: L{IAddress} provider + + @param username: An authentication username which was used to + authenticate against the server at the given address. + @type username: L{bytes} + + @param command: A command which was run in a session channel on the + server at the given address. + @type command: L{bytes} + """ + self.server = server + self.username = username + self.command = command + + + +class _CommandChannel(SSHChannel): + """ + A L{_CommandChannel} executes a command in a session channel and connects + its input and output to an L{IProtocol} provider. + + @ivar _creator: See L{__init__} + @ivar _command: See L{__init__} + @ivar _protocolFactory: See L{__init__} + @ivar _commandConnected: See L{__init__} + @ivar _protocol: An L{IProtocol} provider created using C{_protocolFactory} + which is hooked up to the running command's input and output streams. + """ + name = b'session' + _log = Logger() + + def __init__(self, creator, command, protocolFactory, commandConnected): + """ + @param creator: The L{_ISSHConnectionCreator} provider which was used + to get the connection which this channel exists on. + @type creator: L{_ISSHConnectionCreator} provider + + @param command: The command to be executed. + @type command: L{bytes} + + @param protocolFactory: A client factory to use to build a L{IProtocol} + provider to use to associate with the running command. + + @param commandConnected: A L{Deferred} to use to signal that execution + of the command has failed or that it has succeeded and the command + is now running. + @type commandConnected: L{Deferred} + """ + SSHChannel.__init__(self) + self._creator = creator + self._command = command + self._protocolFactory = protocolFactory + self._commandConnected = commandConnected + self._reason = None + + + def openFailed(self, reason): + """ + When the request to open a new channel to run this command in fails, + fire the C{commandConnected} deferred with a failure indicating that. + """ + self._commandConnected.errback(reason) + + + def channelOpen(self, ignored): + """ + When the request to open a new channel to run this command in succeeds, + issue an C{"exec"} request to run the command. + """ + command = self.conn.sendRequest( + self, b'exec', NS(self._command), wantReply=True) + command.addCallbacks(self._execSuccess, self._execFailure) + + + def _execFailure(self, reason): + """ + When the request to execute the command in this channel fails, fire the + C{commandConnected} deferred with a failure indicating this. + + @param reason: The cause of the command execution failure. + @type reason: L{Failure} + """ + self._commandConnected.errback(reason) + + + def _execSuccess(self, ignored): + """ + When the request to execute the command in this channel succeeds, use + C{protocolFactory} to build a protocol to handle the command's input + and output and connect the protocol to a transport representing those + streams. + + Also fire C{commandConnected} with the created protocol after it is + connected to its transport. + + @param ignored: The (ignored) result of the execute request + """ + self._protocol = self._protocolFactory.buildProtocol( + SSHCommandAddress( + self.conn.transport.transport.getPeer(), + self.conn.transport.creator.username, + self.conn.transport.creator.command)) + self._protocol.makeConnection(self) + self._commandConnected.callback(self._protocol) + + + def dataReceived(self, data): + """ + When the command's stdout data arrives over the channel, deliver it to + the protocol instance. + + @param data: The bytes from the command's stdout. + @type data: L{bytes} + """ + self._protocol.dataReceived(data) + + + def request_exit_status(self, data): + """ + When the server sends the command's exit status, record it for later + delivery to the protocol. + + @param data: The network-order four byte representation of the exit + status of the command. + @type data: L{bytes} + """ + (status,) = unpack('>L', data) + if status != 0: + self._reason = ProcessTerminated(status, None, None) + + + def request_exit_signal(self, data): + """ + When the server sends the command's exit status, record it for later + delivery to the protocol. + + @param data: The network-order four byte representation of the exit + signal of the command. + @type data: L{bytes} + """ + shortSignalName, data = getNS(data) + coreDumped, data = bool(ord(data[0:1])), data[1:] + errorMessage, data = getNS(data) + languageTag, data = getNS(data) + signalName = "SIG%s" % (nativeString(shortSignalName),) + signalID = getattr(signal, signalName, -1) + self._log.info( + "Process exited with signal {shortSignalName!r};" + " core dumped: {coreDumped};" + " error message: {errorMessage};" + " language: {languageTag!r}", + shortSignalName=shortSignalName, + coreDumped=coreDumped, + errorMessage=errorMessage.decode('utf-8'), + languageTag=languageTag, + ) + self._reason = ProcessTerminated(None, signalID, None) + + + def closed(self): + """ + When the channel closes, deliver disconnection notification to the + protocol. + """ + self._creator.cleanupConnection(self.conn, False) + if self._reason is None: + reason = ConnectionDone("ssh channel closed") + else: + reason = self._reason + self._protocol.connectionLost(Failure(reason)) + + + +class _ConnectionReady(SSHConnection): + """ + L{_ConnectionReady} is an L{SSHConnection} (an SSH service) which only + propagates the I{serviceStarted} event to a L{Deferred} to be handled + elsewhere. + """ + def __init__(self, ready): + """ + @param ready: A L{Deferred} which should be fired when + I{serviceStarted} happens. + """ + SSHConnection.__init__(self) + self._ready = ready + + + def serviceStarted(self): + """ + When the SSH I{connection} I{service} this object represents is ready + to be used, fire the C{connectionReady} L{Deferred} to publish that + event to some other interested party. + + """ + self._ready.callback(self) + del self._ready + + + +class _UserAuth(SSHUserAuthClient): + """ + L{_UserAuth} implements the client part of SSH user authentication in the + convenient way a user might expect if they are familiar with the + interactive I{ssh} command line client. + + L{_UserAuth} supports key-based authentication, password-based + authentication, and delegating authentication to an agent. + """ + password = None + keys = None + agent = None + + def getPublicKey(self): + """ + Retrieve the next public key object to offer to the server, possibly + delegating to an authentication agent if there is one. + + @return: The public part of a key pair that could be used to + authenticate with the server, or L{None} if there are no more + public keys to try. + @rtype: L{twisted.conch.ssh.keys.Key} or L{None} + """ + if self.agent is not None: + return self.agent.getPublicKey() + + if self.keys: + self.key = self.keys.pop(0) + else: + self.key = None + return self.key.public() + + + def signData(self, publicKey, signData): + """ + Extend the base signing behavior by using an SSH agent to sign the + data, if one is available. + + @type publicKey: L{Key} + @type signData: L{str} + """ + if self.agent is not None: + return self.agent.signData(publicKey.blob(), signData) + else: + return SSHUserAuthClient.signData(self, publicKey, signData) + + + def getPrivateKey(self): + """ + Get the private part of a key pair to use for authentication. The key + corresponds to the public part most recently returned from + C{getPublicKey}. + + @return: A L{Deferred} which fires with the private key. + @rtype: L{Deferred} + """ + return succeed(self.key) + + + def getPassword(self): + """ + Get the password to use for authentication. + + @return: A L{Deferred} which fires with the password, or L{None} if the + password was not specified. + """ + if self.password is None: + return + return succeed(self.password) + + + def ssh_USERAUTH_SUCCESS(self, packet): + """ + Handle user authentication success in the normal way, but also make a + note of the state change on the L{_CommandTransport}. + """ + self.transport._state = b'CHANNELLING' + return SSHUserAuthClient.ssh_USERAUTH_SUCCESS(self, packet) + + + def connectToAgent(self, endpoint): + """ + Set up a connection to the authentication agent and trigger its + initialization. + + @param endpoint: An endpoint which can be used to connect to the + authentication agent. + @type endpoint: L{IStreamClientEndpoint} provider + + @return: A L{Deferred} which fires when the agent connection is ready + for use. + """ + factory = Factory() + factory.protocol = SSHAgentClient + d = endpoint.connect(factory) + def connected(agent): + self.agent = agent + return agent.getPublicKeys() + d.addCallback(connected) + return d + + + def loseAgentConnection(self): + """ + Disconnect the agent. + """ + if self.agent is None: + return + self.agent.transport.loseConnection() + + + +class _CommandTransport(SSHClientTransport): + """ + L{_CommandTransport} is an SSH client I{transport} which includes a host + key verification step before it will proceed to secure the connection. + + L{_CommandTransport} also knows how to set up a connection to an + authentication agent if it is told where it can connect to one. + + @ivar _userauth: The L{_UserAuth} instance which is in charge of the + overall authentication process or L{None} if the SSH connection has not + reach yet the C{user-auth} service. + @type _userauth: L{_UserAuth} + """ + # STARTING -> SECURING -> AUTHENTICATING -> CHANNELLING -> RUNNING + _state = b'STARTING' + + _hostKeyFailure = None + + _userauth = None + + + def __init__(self, creator): + """ + @param creator: The L{_NewConnectionHelper} that created this + connection. + + @type creator: L{_NewConnectionHelper}. + """ + self.connectionReady = Deferred( + lambda d: self.transport.abortConnection()) + # Clear the reference to that deferred to help the garbage collector + # and to signal to other parts of this implementation (in particular + # connectionLost) that it has already been fired and does not need to + # be fired again. + def readyFired(result): + self.connectionReady = None + return result + self.connectionReady.addBoth(readyFired) + self.creator = creator + + + def verifyHostKey(self, hostKey, fingerprint): + """ + Ask the L{KnownHostsFile} provider available on the factory which + created this protocol this protocol to verify the given host key. + + @return: A L{Deferred} which fires with the result of + L{KnownHostsFile.verifyHostKey}. + """ + hostname = self.creator.hostname + ip = networkString(self.transport.getPeer().host) + + self._state = b'SECURING' + d = self.creator.knownHosts.verifyHostKey( + self.creator.ui, hostname, ip, Key.fromString(hostKey)) + d.addErrback(self._saveHostKeyFailure) + return d + + + def _saveHostKeyFailure(self, reason): + """ + When host key verification fails, record the reason for the failure in + order to fire a L{Deferred} with it later. + + @param reason: The cause of the host key verification failure. + @type reason: L{Failure} + + @return: C{reason} + @rtype: L{Failure} + """ + self._hostKeyFailure = reason + return reason + + + def connectionSecure(self): + """ + When the connection is secure, start the authentication process. + """ + self._state = b'AUTHENTICATING' + + command = _ConnectionReady(self.connectionReady) + + self._userauth = _UserAuth(self.creator.username, command) + self._userauth.password = self.creator.password + if self.creator.keys: + self._userauth.keys = list(self.creator.keys) + + if self.creator.agentEndpoint is not None: + d = self._userauth.connectToAgent(self.creator.agentEndpoint) + else: + d = succeed(None) + + def maybeGotAgent(ignored): + self.requestService(self._userauth) + d.addBoth(maybeGotAgent) + + + def connectionLost(self, reason): + """ + When the underlying connection to the SSH server is lost, if there were + any connection setup errors, propagate them. Also, clean up the + connection to the ssh agent if one was created. + """ + if self._userauth: + self._userauth.loseAgentConnection() + + if self._state == b'RUNNING' or self.connectionReady is None: + return + if self._state == b'SECURING' and self._hostKeyFailure is not None: + reason = self._hostKeyFailure + elif self._state == b'AUTHENTICATING': + reason = Failure( + AuthenticationFailed("Connection lost while authenticating")) + self.connectionReady.errback(reason) + + + +@implementer(IStreamClientEndpoint) +class SSHCommandClientEndpoint(object): + """ + L{SSHCommandClientEndpoint} exposes the command-executing functionality of + SSH servers. + + L{SSHCommandClientEndpoint} can set up a new SSH connection, authenticate + it in any one of a number of different ways (keys, passwords, agents), + launch a command over that connection and then associate its input and + output with a protocol. + + It can also re-use an existing, already-authenticated SSH connection + (perhaps one which already has some SSH channels being used for other + purposes). In this case it creates a new SSH channel to use to execute the + command. Notably this means it supports multiplexing several different + command invocations over a single SSH connection. + """ + + def __init__(self, creator, command): + """ + @param creator: An L{_ISSHConnectionCreator} provider which will be + used to set up the SSH connection which will be used to run a + command. + @type creator: L{_ISSHConnectionCreator} provider + + @param command: The command line to execute on the SSH server. This + byte string is interpreted by a shell on the SSH server, so it may + have a value like C{"ls /"}. Take care when trying to run a + command like C{"/Volumes/My Stuff/a-program"} - spaces (and other + special bytes) may require escaping. + @type command: L{bytes} + + """ + self._creator = creator + self._command = command + + + @classmethod + def newConnection(cls, reactor, command, username, hostname, port=None, + keys=None, password=None, agentEndpoint=None, + knownHosts=None, ui=None): + """ + Create and return a new endpoint which will try to create a new + connection to an SSH server and run a command over it. It will also + close the connection if there are problems leading up to the command + being executed, after the command finishes, or if the connection + L{Deferred} is cancelled. + + @param reactor: The reactor to use to establish the connection. + @type reactor: L{IReactorTCP} provider + + @param command: See L{__init__}'s C{command} argument. + + @param username: The username with which to authenticate to the SSH + server. + @type username: L{bytes} + + @param hostname: The hostname of the SSH server. + @type hostname: L{bytes} + + @param port: The port number of the SSH server. By default, the + standard SSH port number is used. + @type port: L{int} + + @param keys: Private keys with which to authenticate to the SSH server, + if key authentication is to be attempted (otherwise L{None}). + @type keys: L{list} of L{Key} + + @param password: The password with which to authenticate to the SSH + server, if password authentication is to be attempted (otherwise + L{None}). + @type password: L{bytes} or L{None} + + @param agentEndpoint: An L{IStreamClientEndpoint} provider which may be + used to connect to an SSH agent, if one is to be used to help with + authentication. + @type agentEndpoint: L{IStreamClientEndpoint} provider + + @param knownHosts: The currently known host keys, used to check the + host key presented by the server we actually connect to. + @type knownHosts: L{KnownHostsFile} + + @param ui: An object for interacting with users to make decisions about + whether to accept the server host keys. If L{None}, a L{ConsoleUI} + connected to /dev/tty will be used; if /dev/tty is unavailable, an + object which answers C{b"no"} to all prompts will be used. + @type ui: L{None} or L{ConsoleUI} + + @return: A new instance of C{cls} (probably + L{SSHCommandClientEndpoint}). + """ + helper = _NewConnectionHelper( + reactor, hostname, port, command, username, keys, password, + agentEndpoint, knownHosts, ui) + return cls(helper, command) + + + @classmethod + def existingConnection(cls, connection, command): + """ + Create and return a new endpoint which will try to open a new channel + on an existing SSH connection and run a command over it. It will + B{not} close the connection if there is a problem executing the command + or after the command finishes. + + @param connection: An existing connection to an SSH server. + @type connection: L{SSHConnection} + + @param command: See L{SSHCommandClientEndpoint.newConnection}'s + C{command} parameter. + @type command: L{bytes} + + @return: A new instance of C{cls} (probably + L{SSHCommandClientEndpoint}). + """ + helper = _ExistingConnectionHelper(connection) + return cls(helper, command) + + + def connect(self, protocolFactory): + """ + Set up an SSH connection, use a channel from that connection to launch + a command, and hook the stdin and stdout of that command up as a + transport for a protocol created by the given factory. + + @param protocolFactory: A L{Factory} to use to create the protocol + which will be connected to the stdin and stdout of the command on + the SSH server. + + @return: A L{Deferred} which will fire with an error if the connection + cannot be set up for any reason or with the protocol instance + created by C{protocolFactory} once it has been connected to the + command. + """ + d = self._creator.secureConnection() + d.addCallback(self._executeCommand, protocolFactory) + return d + + + def _executeCommand(self, connection, protocolFactory): + """ + Given a secured SSH connection, try to execute a command in a new + channel created on it and associate the result with a protocol from the + given factory. + + @param connection: See L{SSHCommandClientEndpoint.existingConnection}'s + C{connection} parameter. + + @param protocolFactory: See L{SSHCommandClientEndpoint.connect}'s + C{protocolFactory} parameter. + + @return: See L{SSHCommandClientEndpoint.connect}'s return value. + """ + commandConnected = Deferred() + + def disconnectOnFailure(passthrough): + # Close the connection immediately in case of cancellation, since + # that implies user wants it gone immediately (e.g. a timeout): + immediate = passthrough.check(CancelledError) + self._creator.cleanupConnection(connection, immediate) + return passthrough + commandConnected.addErrback(disconnectOnFailure) + + channel = _CommandChannel( + self._creator, self._command, protocolFactory, commandConnected) + connection.openChannel(channel) + return commandConnected + + + +class _ReadFile(object): + """ + A weakly file-like object which can be used with L{KnownHostsFile} to + respond in the negative to all prompts for decisions. + """ + def __init__(self, contents): + """ + @param contents: L{bytes} which will be returned from every C{readline} + call. + """ + self._contents = contents + + + def write(self, data): + """ + No-op. + + @param data: ignored + """ + + + def readline(self, count=-1): + """ + Always give back the byte string that this L{_ReadFile} was initialized + with. + + @param count: ignored + + @return: A fixed byte-string. + @rtype: L{bytes} + """ + return self._contents + + + def close(self): + """ + No-op. + """ + + + +@implementer(_ISSHConnectionCreator) +class _NewConnectionHelper(object): + """ + L{_NewConnectionHelper} implements L{_ISSHConnectionCreator} by + establishing a brand new SSH connection, securing it, and authenticating. + """ + _KNOWN_HOSTS = _KNOWN_HOSTS + port = 22 + + def __init__(self, reactor, hostname, port, command, username, keys, + password, agentEndpoint, knownHosts, ui, + tty=FilePath(b"/dev/tty")): + """ + @param tty: The path of the tty device to use in case C{ui} is L{None}. + @type tty: L{FilePath} + + @see: L{SSHCommandClientEndpoint.newConnection} + """ + self.reactor = reactor + self.hostname = hostname + if port is not None: + self.port = port + self.command = command + self.username = username + self.keys = keys + self.password = password + self.agentEndpoint = agentEndpoint + if knownHosts is None: + knownHosts = self._knownHosts() + self.knownHosts = knownHosts + + if ui is None: + ui = ConsoleUI(self._opener) + self.ui = ui + self.tty = tty + + + def _opener(self): + """ + Open the tty if possible, otherwise give back a file-like object from + which C{b"no"} can be read. + + For use as the opener argument to L{ConsoleUI}. + """ + try: + return self.tty.open("rb+") + except: + # Give back a file-like object from which can be read a byte string + # that KnownHostsFile recognizes as rejecting some option (b"no"). + return _ReadFile(b"no") + + + @classmethod + def _knownHosts(cls): + """ + + @return: A L{KnownHostsFile} instance pointed at the user's personal + I{known hosts} file. + @type: L{KnownHostsFile} + """ + return KnownHostsFile.fromPath(FilePath(expanduser(cls._KNOWN_HOSTS))) + + + def secureConnection(self): + """ + Create and return a new SSH connection which has been secured and on + which authentication has already happened. + + @return: A L{Deferred} which fires with the ready-to-use connection or + with a failure if something prevents the connection from being + setup, secured, or authenticated. + """ + protocol = _CommandTransport(self) + ready = protocol.connectionReady + + sshClient = TCP4ClientEndpoint( + self.reactor, nativeString(self.hostname), self.port) + + d = connectProtocol(sshClient, protocol) + d.addCallback(lambda ignored: ready) + return d + + + def cleanupConnection(self, connection, immediate): + """ + Clean up the connection by closing it. The command running on the + endpoint has ended so the connection is no longer needed. + + @param connection: The L{SSHConnection} to close. + @type connection: L{SSHConnection} + + @param immediate: Whether to close connection immediately. + @type immediate: L{bool}. + """ + if immediate: + # We're assuming the underlying connection is an ITCPTransport, + # which is what the current implementation is restricted to: + connection.transport.transport.abortConnection() + else: + connection.transport.loseConnection() + + + +@implementer(_ISSHConnectionCreator) +class _ExistingConnectionHelper(object): + """ + L{_ExistingConnectionHelper} implements L{_ISSHConnectionCreator} by + handing out an existing SSH connection which is supplied to its + initializer. + """ + + def __init__(self, connection): + """ + @param connection: See L{SSHCommandClientEndpoint.existingConnection}'s + C{connection} parameter. + """ + self.connection = connection + + + def secureConnection(self): + """ + + @return: A L{Deferred} that fires synchronously with the + already-established connection object. + """ + return succeed(self.connection) + + + def cleanupConnection(self, connection, immediate): + """ + Do not do any cleanup on the connection. Leave that responsibility to + whatever code created it in the first place. + + @param connection: The L{SSHConnection} which will not be modified in + any way. + @type connection: L{SSHConnection} + + @param immediate: An argument which will be ignored. + @type immediate: L{bool}. + """ diff --git a/contrib/python/Twisted/py2/twisted/conch/error.py b/contrib/python/Twisted/py2/twisted/conch/error.py new file mode 100644 index 00000000000..c8297c3964b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/error.py @@ -0,0 +1,103 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An error to represent bad things happening in Conch. + +Maintainer: Paul Swartz +""" + +from __future__ import absolute_import, division + +from twisted.cred.error import UnauthorizedLogin + + +class ConchError(Exception): + def __init__(self, value, data = None): + Exception.__init__(self, value, data) + self.value = value + self.data = data + + + +class NotEnoughAuthentication(Exception): + """ + This is thrown if the authentication is valid, but is not enough to + successfully verify the user. i.e. don't retry this type of + authentication, try another one. + """ + + + +class ValidPublicKey(UnauthorizedLogin): + """ + Raised by public key checkers when they receive public key credentials + that don't contain a signature at all, but are valid in every other way. + (e.g. the public key matches one in the user's authorized_keys file). + + Protocol code (eg + L{SSHUserAuthServer}) which + attempts to log in using + L{ISSHPrivateKey} credentials + should be prepared to handle a failure of this type by telling the user to + re-authenticate using the same key and to include a signature with the new + attempt. + + See U{http://www.ietf.org/rfc/rfc4252.txt} section 7 for more details. + """ + + + +class IgnoreAuthentication(Exception): + """ + This is thrown to let the UserAuthServer know it doesn't need to handle the + authentication anymore. + """ + + + +class MissingKeyStoreError(Exception): + """ + Raised if an SSHAgentServer starts receiving data without its factory + providing a keys dict on which to read/write key data. + """ + + + +class UserRejectedKey(Exception): + """ + The user interactively rejected a key. + """ + + + +class InvalidEntry(Exception): + """ + An entry in a known_hosts file could not be interpreted as a valid entry. + """ + + + +class HostKeyChanged(Exception): + """ + The host key of a remote host has changed. + + @ivar offendingEntry: The entry which contains the persistent host key that + disagrees with the given host key. + + @type offendingEntry: L{twisted.conch.interfaces.IKnownHostEntry} + + @ivar path: a reference to the known_hosts file that the offending entry + was loaded from + + @type path: L{twisted.python.filepath.FilePath} + + @ivar lineno: The line number of the offending entry in the given path. + + @type lineno: L{int} + """ + def __init__(self, offendingEntry, path, lineno): + Exception.__init__(self) + self.offendingEntry = offendingEntry + self.path = path + self.lineno = lineno diff --git a/contrib/python/Twisted/py2/twisted/conch/insults/__init__.py b/contrib/python/Twisted/py2/twisted/conch/insults/__init__.py new file mode 100644 index 00000000000..3d838766989 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/insults/__init__.py @@ -0,0 +1,4 @@ +""" +Insults: a replacement for Curses/S-Lang. + +Very basic at the moment.""" diff --git a/contrib/python/Twisted/py2/twisted/conch/insults/helper.py b/contrib/python/Twisted/py2/twisted/conch/insults/helper.py new file mode 100644 index 00000000000..0485bfdbe63 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/insults/helper.py @@ -0,0 +1,517 @@ +# -*- test-case-name: twisted.conch.test.test_helper -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Partial in-memory terminal emulator + +@author: Jp Calderone +""" + +from __future__ import print_function + +import re, string + +from zope.interface import implementer + +from incremental import Version + +from twisted.internet import defer, protocol, reactor +from twisted.python import log, _textattributes +from twisted.python.compat import iterbytes +from twisted.python.deprecate import deprecated, deprecatedModuleAttribute +from twisted.conch.insults import insults + +FOREGROUND = 30 +BACKGROUND = 40 +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, N_COLORS = range(9) + + + +class _FormattingState(_textattributes._FormattingStateMixin): + """ + Represents the formatting state/attributes of a single character. + + Character set, intensity, underlinedness, blinkitude, video + reversal, as well as foreground and background colors made up a + character's attributes. + """ + compareAttributes = ( + 'charset', 'bold', 'underline', 'blink', 'reverseVideo', 'foreground', + 'background', '_subtracting') + + + def __init__(self, charset=insults.G0, bold=False, underline=False, + blink=False, reverseVideo=False, foreground=WHITE, + background=BLACK, _subtracting=False): + self.charset = charset + self.bold = bold + self.underline = underline + self.blink = blink + self.reverseVideo = reverseVideo + self.foreground = foreground + self.background = background + self._subtracting = _subtracting + + + @deprecated(Version('Twisted', 13, 1, 0)) + def wantOne(self, **kw): + """ + Add a character attribute to a copy of this formatting state. + + @param **kw: An optional attribute name and value can be provided with + a keyword argument. + + @return: A formatting state instance with the new attribute. + + @see: L{DefaultFormattingState._withAttribute}. + """ + k, v = kw.popitem() + return self._withAttribute(k, v) + + + def toVT102(self): + # Spit out a vt102 control sequence that will set up + # all the attributes set here. Except charset. + attrs = [] + if self._subtracting: + attrs.append(0) + if self.bold: + attrs.append(insults.BOLD) + if self.underline: + attrs.append(insults.UNDERLINE) + if self.blink: + attrs.append(insults.BLINK) + if self.reverseVideo: + attrs.append(insults.REVERSE_VIDEO) + if self.foreground != WHITE: + attrs.append(FOREGROUND + self.foreground) + if self.background != BLACK: + attrs.append(BACKGROUND + self.background) + if attrs: + return '\x1b[' + ';'.join(map(str, attrs)) + 'm' + return '' + +CharacterAttribute = _FormattingState + +deprecatedModuleAttribute( + Version('Twisted', 13, 1, 0), + 'Use twisted.conch.insults.text.assembleFormattedText instead.', + 'twisted.conch.insults.helper', + 'CharacterAttribute') + + + +# XXX - need to support scroll regions and scroll history +@implementer(insults.ITerminalTransport) +class TerminalBuffer(protocol.Protocol): + """ + An in-memory terminal emulator. + """ + for keyID in (b'UP_ARROW', b'DOWN_ARROW', b'RIGHT_ARROW', b'LEFT_ARROW', + b'HOME', b'INSERT', b'DELETE', b'END', b'PGUP', b'PGDN', + b'F1', b'F2', b'F3', b'F4', b'F5', b'F6', b'F7', b'F8', b'F9', + b'F10', b'F11', b'F12'): + execBytes = keyID + b" = object()" + execStr = execBytes.decode("ascii") + exec(execStr) + + TAB = b'\t' + BACKSPACE = b'\x7f' + + width = 80 + height = 24 + + fill = b' ' + void = object() + + def getCharacter(self, x, y): + return self.lines[y][x] + + + def connectionMade(self): + self.reset() + + + def write(self, data): + """ + Add the given printable bytes to the terminal. + + Line feeds in L{bytes} will be replaced with carriage return / line + feed pairs. + """ + for b in iterbytes(data.replace(b'\n', b'\r\n')): + self.insertAtCursor(b) + + + def _currentFormattingState(self): + return _FormattingState(self.activeCharset, **self.graphicRendition) + + + def insertAtCursor(self, b): + """ + Add one byte to the terminal at the cursor and make consequent state + updates. + + If b is a carriage return, move the cursor to the beginning of the + current row. + + If b is a line feed, move the cursor to the next row or scroll down if + the cursor is already in the last row. + + Otherwise, if b is printable, put it at the cursor position (inserting + or overwriting as dictated by the current mode) and move the cursor. + """ + if b == b'\r': + self.x = 0 + elif b == b'\n': + self._scrollDown() + elif b in string.printable.encode("ascii"): + if self.x >= self.width: + self.nextLine() + ch = (b, self._currentFormattingState()) + if self.modes.get(insults.modes.IRM): + self.lines[self.y][self.x:self.x] = [ch] + self.lines[self.y].pop() + else: + self.lines[self.y][self.x] = ch + self.x += 1 + + + def _emptyLine(self, width): + return [(self.void, self._currentFormattingState()) + for i in range(width)] + + + def _scrollDown(self): + self.y += 1 + if self.y >= self.height: + self.y -= 1 + del self.lines[0] + self.lines.append(self._emptyLine(self.width)) + + + def _scrollUp(self): + self.y -= 1 + if self.y < 0: + self.y = 0 + del self.lines[-1] + self.lines.insert(0, self._emptyLine(self.width)) + + + def cursorUp(self, n=1): + self.y = max(0, self.y - n) + + + def cursorDown(self, n=1): + self.y = min(self.height - 1, self.y + n) + + + def cursorBackward(self, n=1): + self.x = max(0, self.x - n) + + + def cursorForward(self, n=1): + self.x = min(self.width, self.x + n) + + + def cursorPosition(self, column, line): + self.x = column + self.y = line + + + def cursorHome(self): + self.x = self.home.x + self.y = self.home.y + + + def index(self): + self._scrollDown() + + + def reverseIndex(self): + self._scrollUp() + + + def nextLine(self): + """ + Update the cursor position attributes and scroll down if appropriate. + """ + self.x = 0 + self._scrollDown() + + + def saveCursor(self): + self._savedCursor = (self.x, self.y) + + + def restoreCursor(self): + self.x, self.y = self._savedCursor + del self._savedCursor + + + def setModes(self, modes): + for m in modes: + self.modes[m] = True + + + def resetModes(self, modes): + for m in modes: + try: + del self.modes[m] + except KeyError: + pass + + + def setPrivateModes(self, modes): + """ + Enable the given modes. + + Track which modes have been enabled so that the implementations of + other L{insults.ITerminalTransport} methods can be properly implemented + to respect these settings. + + @see: L{resetPrivateModes} + @see: L{insults.ITerminalTransport.setPrivateModes} + """ + for m in modes: + self.privateModes[m] = True + + + def resetPrivateModes(self, modes): + """ + Disable the given modes. + + @see: L{setPrivateModes} + @see: L{insults.ITerminalTransport.resetPrivateModes} + """ + for m in modes: + try: + del self.privateModes[m] + except KeyError: + pass + + + def applicationKeypadMode(self): + self.keypadMode = 'app' + + + def numericKeypadMode(self): + self.keypadMode = 'num' + + + def selectCharacterSet(self, charSet, which): + self.charsets[which] = charSet + + + def shiftIn(self): + self.activeCharset = insults.G0 + + + def shiftOut(self): + self.activeCharset = insults.G1 + + + def singleShift2(self): + oldActiveCharset = self.activeCharset + self.activeCharset = insults.G2 + f = self.insertAtCursor + def insertAtCursor(b): + f(b) + del self.insertAtCursor + self.activeCharset = oldActiveCharset + self.insertAtCursor = insertAtCursor + + + def singleShift3(self): + oldActiveCharset = self.activeCharset + self.activeCharset = insults.G3 + f = self.insertAtCursor + def insertAtCursor(b): + f(b) + del self.insertAtCursor + self.activeCharset = oldActiveCharset + self.insertAtCursor = insertAtCursor + + + def selectGraphicRendition(self, *attributes): + for a in attributes: + if a == insults.NORMAL: + self.graphicRendition = { + 'bold': False, + 'underline': False, + 'blink': False, + 'reverseVideo': False, + 'foreground': WHITE, + 'background': BLACK} + elif a == insults.BOLD: + self.graphicRendition['bold'] = True + elif a == insults.UNDERLINE: + self.graphicRendition['underline'] = True + elif a == insults.BLINK: + self.graphicRendition['blink'] = True + elif a == insults.REVERSE_VIDEO: + self.graphicRendition['reverseVideo'] = True + else: + try: + v = int(a) + except ValueError: + log.msg("Unknown graphic rendition attribute: " + repr(a)) + else: + if FOREGROUND <= v <= FOREGROUND + N_COLORS: + self.graphicRendition['foreground'] = v - FOREGROUND + elif BACKGROUND <= v <= BACKGROUND + N_COLORS: + self.graphicRendition['background'] = v - BACKGROUND + else: + log.msg("Unknown graphic rendition attribute: " + repr(a)) + + + def eraseLine(self): + self.lines[self.y] = self._emptyLine(self.width) + + + def eraseToLineEnd(self): + width = self.width - self.x + self.lines[self.y][self.x:] = self._emptyLine(width) + + + def eraseToLineBeginning(self): + self.lines[self.y][:self.x + 1] = self._emptyLine(self.x + 1) + + + def eraseDisplay(self): + self.lines = [self._emptyLine(self.width) for i in range(self.height)] + + + def eraseToDisplayEnd(self): + self.eraseToLineEnd() + height = self.height - self.y - 1 + self.lines[self.y + 1:] = [self._emptyLine(self.width) for i in range(height)] + + + def eraseToDisplayBeginning(self): + self.eraseToLineBeginning() + self.lines[:self.y] = [self._emptyLine(self.width) for i in range(self.y)] + + + def deleteCharacter(self, n=1): + del self.lines[self.y][self.x:self.x+n] + self.lines[self.y].extend(self._emptyLine(min(self.width - self.x, n))) + + + def insertLine(self, n=1): + self.lines[self.y:self.y] = [self._emptyLine(self.width) for i in range(n)] + del self.lines[self.height:] + + + def deleteLine(self, n=1): + del self.lines[self.y:self.y+n] + self.lines.extend([self._emptyLine(self.width) for i in range(n)]) + + + def reportCursorPosition(self): + return (self.x, self.y) + + + def reset(self): + self.home = insults.Vector(0, 0) + self.x = self.y = 0 + self.modes = {} + self.privateModes = {} + self.setPrivateModes([insults.privateModes.AUTO_WRAP, + insults.privateModes.CURSOR_MODE]) + self.numericKeypad = 'app' + self.activeCharset = insults.G0 + self.graphicRendition = { + 'bold': False, + 'underline': False, + 'blink': False, + 'reverseVideo': False, + 'foreground': WHITE, + 'background': BLACK} + self.charsets = { + insults.G0: insults.CS_US, + insults.G1: insults.CS_US, + insults.G2: insults.CS_ALTERNATE, + insults.G3: insults.CS_ALTERNATE_SPECIAL} + self.eraseDisplay() + + + def unhandledControlSequence(self, buf): + print('Could not handle', repr(buf)) + + + def __bytes__(self): + lines = [] + for L in self.lines: + buf = [] + length = 0 + for (ch, attr) in L: + if ch is not self.void: + buf.append(ch) + length = len(buf) + else: + buf.append(self.fill) + lines.append(b''.join(buf[:length])) + return b'\n'.join(lines) + + + +class ExpectationTimeout(Exception): + pass + + + +class ExpectableBuffer(TerminalBuffer): + _mark = 0 + + def connectionMade(self): + TerminalBuffer.connectionMade(self) + self._expecting = [] + + + def write(self, data): + TerminalBuffer.write(self, data) + self._checkExpected() + + + def cursorHome(self): + TerminalBuffer.cursorHome(self) + self._mark = 0 + + + def _timeoutExpected(self, d): + d.errback(ExpectationTimeout()) + self._checkExpected() + + + def _checkExpected(self): + s = self.__bytes__()[self._mark:] + while self._expecting: + expr, timer, deferred = self._expecting[0] + if timer and not timer.active(): + del self._expecting[0] + continue + for match in expr.finditer(s): + if timer: + timer.cancel() + del self._expecting[0] + self._mark += match.end() + s = s[match.end():] + deferred.callback(match) + break + else: + return + + + def expect(self, expression, timeout=None, scheduler=reactor): + d = defer.Deferred() + timer = None + if timeout: + timer = scheduler.callLater(timeout, self._timeoutExpected, d) + self._expecting.append((re.compile(expression), timer, d)) + self._checkExpected() + return d + +__all__ = [ + 'CharacterAttribute', 'TerminalBuffer', 'ExpectableBuffer'] diff --git a/contrib/python/Twisted/py2/twisted/conch/insults/insults.py b/contrib/python/Twisted/py2/twisted/conch/insults/insults.py new file mode 100644 index 00000000000..a583174415a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/insults/insults.py @@ -0,0 +1,1289 @@ +# -*- test-case-name: twisted.conch.test.test_insults -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +VT102 and VT220 terminal manipulation. + +@author: Jp Calderone +""" + +from zope.interface import implementer, Interface + +from twisted.internet import protocol, defer, interfaces as iinternet +from twisted.python.compat import intToBytes, iterbytes, networkString + + + +class ITerminalProtocol(Interface): + def makeConnection(transport): + """ + Called with an L{ITerminalTransport} when a connection is established. + """ + + def keystrokeReceived(keyID, modifier): + """ + A keystroke was received. + + Each keystroke corresponds to one invocation of this method. + keyID is a string identifier for that key. Printable characters + are represented by themselves. Control keys, such as arrows and + function keys, are represented with symbolic constants on + L{ServerProtocol}. + """ + + def terminalSize(width, height): + """ + Called to indicate the size of the terminal. + + A terminal of 80x24 should be assumed if this method is not + called. This method might not be called for real terminals. + """ + + def unhandledControlSequence(seq): + """ + Called when an unsupported control sequence is received. + + @type seq: L{str} + @param seq: The whole control sequence which could not be interpreted. + """ + + def connectionLost(reason): + """ + Called when the connection has been lost. + + reason is a Failure describing why. + """ + + + +@implementer(ITerminalProtocol) +class TerminalProtocol(object): + def makeConnection(self, terminal): + # assert ITerminalTransport.providedBy(transport), "TerminalProtocol.makeConnection must be passed an ITerminalTransport implementor" + self.terminal = terminal + self.connectionMade() + + + def connectionMade(self): + """ + Called after a connection has been established. + """ + + + def keystrokeReceived(self, keyID, modifier): + pass + + + def terminalSize(self, width, height): + pass + + + def unhandledControlSequence(self, seq): + pass + + + def connectionLost(self, reason): + pass + + + +class ITerminalTransport(iinternet.ITransport): + def cursorUp(n=1): + """ + Move the cursor up n lines. + """ + + + def cursorDown(n=1): + """ + Move the cursor down n lines. + """ + + + def cursorForward(n=1): + """ + Move the cursor right n columns. + """ + + + def cursorBackward(n=1): + """ + Move the cursor left n columns. + """ + + + def cursorPosition(column, line): + """ + Move the cursor to the given line and column. + """ + + + def cursorHome(): + """ + Move the cursor home. + """ + + + def index(): + """ + Move the cursor down one line, performing scrolling if necessary. + """ + + + def reverseIndex(): + """ + Move the cursor up one line, performing scrolling if necessary. + """ + + + def nextLine(): + """ + Move the cursor to the first position on the next line, performing scrolling if necessary. + """ + + + def saveCursor(): + """ + Save the cursor position, character attribute, character set, and origin mode selection. + """ + + + def restoreCursor(): + """ + Restore the previously saved cursor position, character attribute, character set, and origin mode selection. + + If no cursor state was previously saved, move the cursor to the home position. + """ + + + def setModes(modes): + """ + Set the given modes on the terminal. + """ + + def resetModes(mode): + """ + Reset the given modes on the terminal. + """ + + + def setPrivateModes(modes): + """ + Set the given DEC private modes on the terminal. + """ + + + def resetPrivateModes(modes): + """ + Reset the given DEC private modes on the terminal. + """ + + + def applicationKeypadMode(): + """ + Cause keypad to generate control functions. + + Cursor key mode selects the type of characters generated by cursor keys. + """ + + + def numericKeypadMode(): + """ + Cause keypad to generate normal characters. + """ + + + def selectCharacterSet(charSet, which): + """ + Select a character set. + + charSet should be one of CS_US, CS_UK, CS_DRAWING, CS_ALTERNATE, or + CS_ALTERNATE_SPECIAL. + + which should be one of G0 or G1. + """ + + + def shiftIn(): + """ + Activate the G0 character set. + """ + + + def shiftOut(): + """ + Activate the G1 character set. + """ + + + def singleShift2(): + """ + Shift to the G2 character set for a single character. + """ + + + def singleShift3(): + """ + Shift to the G3 character set for a single character. + """ + + + def selectGraphicRendition(*attributes): + """ + Enabled one or more character attributes. + + Arguments should be one or more of UNDERLINE, REVERSE_VIDEO, BLINK, or BOLD. + NORMAL may also be specified to disable all character attributes. + """ + + + def horizontalTabulationSet(): + """ + Set a tab stop at the current cursor position. + """ + + + def tabulationClear(): + """ + Clear the tab stop at the current cursor position. + """ + + + def tabulationClearAll(): + """ + Clear all tab stops. + """ + + + def doubleHeightLine(top=True): + """ + Make the current line the top or bottom half of a double-height, double-width line. + + If top is True, the current line is the top half. Otherwise, it is the bottom half. + """ + + + def singleWidthLine(): + """ + Make the current line a single-width, single-height line. + """ + + + def doubleWidthLine(): + """ + Make the current line a double-width line. + """ + + + def eraseToLineEnd(): + """ + Erase from the cursor to the end of line, including cursor position. + """ + + + def eraseToLineBeginning(): + """ + Erase from the cursor to the beginning of the line, including the cursor position. + """ + + + def eraseLine(): + """ + Erase the entire cursor line. + """ + + + def eraseToDisplayEnd(): + """ + Erase from the cursor to the end of the display, including the cursor position. + """ + + + def eraseToDisplayBeginning(): + """ + Erase from the cursor to the beginning of the display, including the cursor position. + """ + + + def eraseDisplay(): + """ + Erase the entire display. + """ + + + def deleteCharacter(n=1): + """ + Delete n characters starting at the cursor position. + + Characters to the right of deleted characters are shifted to the left. + """ + + + def insertLine(n=1): + """ + Insert n lines at the cursor position. + + Lines below the cursor are shifted down. Lines moved past the bottom margin are lost. + This command is ignored when the cursor is outside the scroll region. + """ + + + def deleteLine(n=1): + """ + Delete n lines starting at the cursor position. + + Lines below the cursor are shifted up. This command is ignored when the cursor is outside + the scroll region. + """ + + + def reportCursorPosition(): + """ + Return a Deferred that fires with a two-tuple of (x, y) indicating the cursor position. + """ + + + def reset(): + """ + Reset the terminal to its initial state. + """ + + + def unhandledControlSequence(seq): + """ + Called when an unsupported control sequence is received. + + @type seq: L{str} + @param seq: The whole control sequence which could not be interpreted. + """ + + +CSI = b'\x1b' +CST = {b'~': b'tilde'} + +class modes: + """ + ECMA 48 standardized modes + """ + + # BREAKS YOPUR KEYBOARD MOFO + KEYBOARD_ACTION = KAM = 2 + + # When set, enables character insertion. New display characters + # move old display characters to the right. Characters moved past + # the right margin are lost. + + # When reset, enables replacement mode (disables character + # insertion). New display characters replace old display + # characters at cursor position. The old character is erased. + INSERTION_REPLACEMENT = IRM = 4 + + # Set causes a received linefeed, form feed, or vertical tab to + # move cursor to first column of next line. RETURN transmits both + # a carriage return and linefeed. This selection is also called + # new line option. + + # Reset causes a received linefeed, form feed, or vertical tab to + # move cursor to next line in current column. RETURN transmits a + # carriage return. + LINEFEED_NEWLINE = LNM = 20 + + + +class privateModes: + """ + ANSI-Compatible Private Modes + """ + ERROR = 0 + CURSOR_KEY = 1 + ANSI_VT52 = 2 + COLUMN = 3 + SCROLL = 4 + SCREEN = 5 + ORIGIN = 6 + AUTO_WRAP = 7 + AUTO_REPEAT = 8 + PRINTER_FORM_FEED = 18 + PRINTER_EXTENT = 19 + + # Toggle cursor visibility (reset hides it) + CURSOR_MODE = 25 + + +# Character sets +CS_US = b'CS_US' +CS_UK = b'CS_UK' +CS_DRAWING = b'CS_DRAWING' +CS_ALTERNATE = b'CS_ALTERNATE' +CS_ALTERNATE_SPECIAL = b'CS_ALTERNATE_SPECIAL' + +# Groupings (or something?? These are like variables that can be bound to character sets) +G0 = b'G0' +G1 = b'G1' + +# G2 and G3 cannot be changed, but they can be shifted to. +G2 = b'G2' +G3 = b'G3' + +# Character attributes + +NORMAL = 0 +BOLD = 1 +UNDERLINE = 4 +BLINK = 5 +REVERSE_VIDEO = 7 + +class Vector: + def __init__(self, x, y): + self.x = x + self.y = y + + + +def log(s): + with open('log', 'a') as f: + f.write(str(s) + '\n') + +# XXX TODO - These attributes are really part of the +# ITerminalTransport interface, I think. +_KEY_NAMES = ('UP_ARROW', 'DOWN_ARROW', 'RIGHT_ARROW', 'LEFT_ARROW', + 'HOME', 'INSERT', 'DELETE', 'END', 'PGUP', 'PGDN', 'NUMPAD_MIDDLE', + 'F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F7', 'F8', 'F9', + 'F10', 'F11', 'F12', + + 'ALT', 'SHIFT', 'CONTROL') + +class _const(object): + """ + @ivar name: A string naming this constant + """ + def __init__(self, name): + self.name = name + + + def __repr__(self): + return '[' + self.name + ']' + + + def __bytes__(self): + return ('[' + self.name + ']').encode("ascii") + + +FUNCTION_KEYS = [ + _const(_name).__bytes__() for _name in _KEY_NAMES] + + + +@implementer(ITerminalTransport) +class ServerProtocol(protocol.Protocol): + protocolFactory = None + terminalProtocol = None + + TAB = b'\t' + BACKSPACE = b'\x7f' + ## + + lastWrite = b'' + + state = b'data' + + termSize = Vector(80, 24) + cursorPos = Vector(0, 0) + scrollRegion = None + + # Factory who instantiated me + factory = None + + def __init__(self, protocolFactory=None, *a, **kw): + """ + @param protocolFactory: A callable which will be invoked with + *a, **kw and should return an ITerminalProtocol implementor. + This will be invoked when a connection to this ServerProtocol + is established. + + @param a: Any positional arguments to pass to protocolFactory. + @param kw: Any keyword arguments to pass to protocolFactory. + """ + # assert protocolFactory is None or ITerminalProtocol.implementedBy(protocolFactory), "ServerProtocol.__init__ must be passed an ITerminalProtocol implementor" + if protocolFactory is not None: + self.protocolFactory = protocolFactory + self.protocolArgs = a + self.protocolKwArgs = kw + + self._cursorReports = [] + + + def connectionMade(self): + if self.protocolFactory is not None: + self.terminalProtocol = self.protocolFactory(*self.protocolArgs, **self.protocolKwArgs) + + try: + factory = self.factory + except AttributeError: + pass + else: + self.terminalProtocol.factory = factory + + self.terminalProtocol.makeConnection(self) + + + def dataReceived(self, data): + for ch in iterbytes(data): + if self.state == b'data': + if ch == b'\x1b': + self.state = b'escaped' + else: + self.terminalProtocol.keystrokeReceived(ch, None) + elif self.state == b'escaped': + if ch == b'[': + self.state = b'bracket-escaped' + self.escBuf = [] + elif ch == b'O': + self.state = b'low-function-escaped' + else: + self.state = b'data' + self._handleShortControlSequence(ch) + elif self.state == b'bracket-escaped': + if ch == b'O': + self.state = b'low-function-escaped' + elif ch.isalpha() or ch == b'~': + self._handleControlSequence(b''.join(self.escBuf) + ch) + del self.escBuf + self.state = b'data' + else: + self.escBuf.append(ch) + elif self.state == b'low-function-escaped': + self._handleLowFunctionControlSequence(ch) + self.state = b'data' + else: + raise ValueError("Illegal state") + + + def _handleShortControlSequence(self, ch): + self.terminalProtocol.keystrokeReceived(ch, self.ALT) + + + def _handleControlSequence(self, buf): + buf = b'\x1b[' + buf + f = getattr(self.controlSequenceParser, + CST.get(buf[-1:], buf[-1:]).decode("ascii"), + None) + if f is None: + self.unhandledControlSequence(buf) + else: + f(self, self.terminalProtocol, buf[:-1]) + + + def unhandledControlSequence(self, buf): + self.terminalProtocol.unhandledControlSequence(buf) + + + def _handleLowFunctionControlSequence(self, ch): + functionKeys = {b'P': self.F1, b'Q': self.F2, + b'R': self.F3, b'S': self.F4} + keyID = functionKeys.get(ch) + if keyID is not None: + self.terminalProtocol.keystrokeReceived(keyID, None) + else: + self.terminalProtocol.unhandledControlSequence(b'\x1b[O' + ch) + + + class ControlSequenceParser: + def A(self, proto, handler, buf): + if buf == b'\x1b[': + handler.keystrokeReceived(proto.UP_ARROW, None) + else: + handler.unhandledControlSequence(buf + b'A') + + + def B(self, proto, handler, buf): + if buf == b'\x1b[': + handler.keystrokeReceived(proto.DOWN_ARROW, None) + else: + handler.unhandledControlSequence(buf + b'B') + + + def C(self, proto, handler, buf): + if buf == b'\x1b[': + handler.keystrokeReceived(proto.RIGHT_ARROW, None) + else: + handler.unhandledControlSequence(buf + b'C') + + + def D(self, proto, handler, buf): + if buf == b'\x1b[': + handler.keystrokeReceived(proto.LEFT_ARROW, None) + else: + handler.unhandledControlSequence(buf + b'D') + + + def E(self, proto, handler, buf): + if buf == b'\x1b[': + handler.keystrokeReceived(proto.NUMPAD_MIDDLE, None) + else: + handler.unhandledControlSequence(buf + b'E') + + + def F(self, proto, handler, buf): + if buf == b'\x1b[': + handler.keystrokeReceived(proto.END, None) + else: + handler.unhandledControlSequence(buf + b'F') + + + def H(self, proto, handler, buf): + if buf == b'\x1b[': + handler.keystrokeReceived(proto.HOME, None) + else: + handler.unhandledControlSequence(buf + b'H') + + + def R(self, proto, handler, buf): + if not proto._cursorReports: + handler.unhandledControlSequence(buf + b'R') + elif buf.startswith(b'\x1b['): + report = buf[2:] + parts = report.split(b';') + if len(parts) != 2: + handler.unhandledControlSequence(buf + b'R') + else: + Pl, Pc = parts + try: + Pl, Pc = int(Pl), int(Pc) + except ValueError: + handler.unhandledControlSequence(buf + b'R') + else: + d = proto._cursorReports.pop(0) + d.callback((Pc - 1, Pl - 1)) + else: + handler.unhandledControlSequence(buf + b'R') + + + def Z(self, proto, handler, buf): + if buf == b'\x1b[': + handler.keystrokeReceived(proto.TAB, proto.SHIFT) + else: + handler.unhandledControlSequence(buf + b'Z') + + + def tilde(self, proto, handler, buf): + map = {1: proto.HOME, 2: proto.INSERT, 3: proto.DELETE, + 4: proto.END, 5: proto.PGUP, 6: proto.PGDN, + + 15: proto.F5, 17: proto.F6, 18: proto.F7, + 19: proto.F8, 20: proto.F9, 21: proto.F10, + 23: proto.F11, 24: proto.F12} + + if buf.startswith(b'\x1b['): + ch = buf[2:] + try: + v = int(ch) + except ValueError: + handler.unhandledControlSequence(buf + b'~') + else: + symbolic = map.get(v) + if symbolic is not None: + handler.keystrokeReceived(map[v], None) + else: + handler.unhandledControlSequence(buf + b'~') + else: + handler.unhandledControlSequence(buf + b'~') + + controlSequenceParser = ControlSequenceParser() + + + # ITerminalTransport + def cursorUp(self, n=1): + assert n >= 1 + self.cursorPos.y = max(self.cursorPos.y - n, 0) + self.write(b'\x1b[' + intToBytes(n) + b'A') + + + def cursorDown(self, n=1): + assert n >= 1 + self.cursorPos.y = min(self.cursorPos.y + n, self.termSize.y - 1) + self.write(b'\x1b[' + intToBytes(n) + b'B') + + + def cursorForward(self, n=1): + assert n >= 1 + self.cursorPos.x = min(self.cursorPos.x + n, self.termSize.x - 1) + self.write(b'\x1b[' + intToBytes(n) + b'C') + + + def cursorBackward(self, n=1): + assert n >= 1 + self.cursorPos.x = max(self.cursorPos.x - n, 0) + self.write(b'\x1b[' + intToBytes(n) + b'D') + + + def cursorPosition(self, column, line): + self.write(b'\x1b[' + + intToBytes(line + 1) + + b';' + + intToBytes(column + 1) + + b'H') + + + def cursorHome(self): + self.cursorPos.x = self.cursorPos.y = 0 + self.write(b'\x1b[H') + + + def index(self): + # ECMA48 5th Edition removes this + self.cursorPos.y = min(self.cursorPos.y + 1, self.termSize.y - 1) + self.write(b'\x1bD') + + + def reverseIndex(self): + self.cursorPos.y = max(self.cursorPos.y - 1, 0) + self.write(b'\x1bM') + + + def nextLine(self): + self.cursorPos.x = 0 + self.cursorPos.y = min(self.cursorPos.y + 1, self.termSize.y - 1) + self.write(b'\n') + + + def saveCursor(self): + self._savedCursorPos = Vector(self.cursorPos.x, self.cursorPos.y) + self.write(b'\x1b7') + + + def restoreCursor(self): + self.cursorPos = self._savedCursorPos + del self._savedCursorPos + self.write(b'\x1b8') + + + def setModes(self, modes): + # XXX Support ANSI-Compatible private modes + modesBytes = b';'.join([intToBytes(mode) for mode in modes]) + self.write(b'\x1b[' + modesBytes + b'h') + + + def setPrivateModes(self, modes): + modesBytes = b';'.join([intToBytes(mode) for mode in modes]) + self.write(b'\x1b[?' + modesBytes + b'h') + + + def resetModes(self, modes): + # XXX Support ANSI-Compatible private modes + modesBytes = b';'.join([intToBytes(mode) for mode in modes]) + self.write(b'\x1b[' + modesBytes + b'l') + + + def resetPrivateModes(self, modes): + modesBytes = b';'.join([intToBytes(mode) for mode in modes]) + self.write(b'\x1b[?' + modesBytes + b'l') + + + def applicationKeypadMode(self): + self.write(b'\x1b=') + + + def numericKeypadMode(self): + self.write(b'\x1b>') + + + def selectCharacterSet(self, charSet, which): + # XXX Rewrite these as dict lookups + if which == G0: + which = b'(' + elif which == G1: + which = b')' + else: + raise ValueError("`which' argument to selectCharacterSet must be G0 or G1") + if charSet == CS_UK: + charSet = b'A' + elif charSet == CS_US: + charSet = b'B' + elif charSet == CS_DRAWING: + charSet = b'0' + elif charSet == CS_ALTERNATE: + charSet = b'1' + elif charSet == CS_ALTERNATE_SPECIAL: + charSet = b'2' + else: + raise ValueError("Invalid `charSet' argument to selectCharacterSet") + self.write(b'\x1b' + which + charSet) + + + def shiftIn(self): + self.write(b'\x15') + + + def shiftOut(self): + self.write(b'\x14') + + + def singleShift2(self): + self.write(b'\x1bN') + + + def singleShift3(self): + self.write(b'\x1bO') + + + def selectGraphicRendition(self, *attributes): + # each member of attributes must be a native string + attrs = [] + for a in attributes: + attrs.append(networkString(a)) + self.write(b'\x1b[' + + b';'.join(attrs) + + b'm') + + + def horizontalTabulationSet(self): + self.write(b'\x1bH') + + + def tabulationClear(self): + self.write(b'\x1b[q') + + + def tabulationClearAll(self): + self.write(b'\x1b[3q') + + + def doubleHeightLine(self, top=True): + if top: + self.write(b'\x1b#3') + else: + self.write(b'\x1b#4') + + + def singleWidthLine(self): + self.write(b'\x1b#5') + + + def doubleWidthLine(self): + self.write(b'\x1b#6') + + + def eraseToLineEnd(self): + self.write(b'\x1b[K') + + + def eraseToLineBeginning(self): + self.write(b'\x1b[1K') + + + def eraseLine(self): + self.write(b'\x1b[2K') + + + def eraseToDisplayEnd(self): + self.write(b'\x1b[J') + + + def eraseToDisplayBeginning(self): + self.write(b'\x1b[1J') + + + def eraseDisplay(self): + self.write(b'\x1b[2J') + + + def deleteCharacter(self, n=1): + self.write(b'\x1b[' + intToBytes(n) + b'P') + + + def insertLine(self, n=1): + self.write(b'\x1b[' + intToBytes(n) + b'L') + + + def deleteLine(self, n=1): + self.write(b'\x1b[' + intToBytes(n) + b'M') + + + def setScrollRegion(self, first=None, last=None): + if first is not None: + first = intToBytes(first) + else: + first = b'' + if last is not None: + last = intToBytes(last) + else: + last = b'' + self.write(b'\x1b[' + first + b';' + last + b'r') + + + def resetScrollRegion(self): + self.setScrollRegion() + + + def reportCursorPosition(self): + d = defer.Deferred() + self._cursorReports.append(d) + self.write(b'\x1b[6n') + return d + + + def reset(self): + self.cursorPos.x = self.cursorPos.y = 0 + try: + del self._savedCursorPos + except AttributeError: + pass + self.write(b'\x1bc') + + + # ITransport + def write(self, data): + if data: + if not isinstance(data, bytes): + data = data.encode("utf-8") + self.lastWrite = data + self.transport.write(b'\r\n'.join(data.split(b'\n'))) + + + def writeSequence(self, data): + self.write(b''.join(data)) + + + def loseConnection(self): + self.reset() + self.transport.loseConnection() + + + def connectionLost(self, reason): + if self.terminalProtocol is not None: + try: + self.terminalProtocol.connectionLost(reason) + finally: + self.terminalProtocol = None +# Add symbolic names for function keys +for name, const in zip(_KEY_NAMES, FUNCTION_KEYS): + setattr(ServerProtocol, name, const) + + + +class ClientProtocol(protocol.Protocol): + + terminalFactory = None + terminal = None + + state = b'data' + + _escBuf = None + + _shorts = { + b'D': b'index', + b'M': b'reverseIndex', + b'E': b'nextLine', + b'7': b'saveCursor', + b'8': b'restoreCursor', + b'=': b'applicationKeypadMode', + b'>': b'numericKeypadMode', + b'N': b'singleShift2', + b'O': b'singleShift3', + b'H': b'horizontalTabulationSet', + b'c': b'reset'} + + _longs = { + b'[': b'bracket-escape', + b'(': b'select-g0', + b')': b'select-g1', + b'#': b'select-height-width'} + + _charsets = { + b'A': CS_UK, + b'B': CS_US, + b'0': CS_DRAWING, + b'1': CS_ALTERNATE, + b'2': CS_ALTERNATE_SPECIAL} + + # Factory who instantiated me + factory = None + + def __init__(self, terminalFactory=None, *a, **kw): + """ + @param terminalFactory: A callable which will be invoked with + *a, **kw and should return an ITerminalTransport provider. + This will be invoked when this ClientProtocol establishes a + connection. + + @param a: Any positional arguments to pass to terminalFactory. + @param kw: Any keyword arguments to pass to terminalFactory. + """ + # assert terminalFactory is None or ITerminalTransport.implementedBy(terminalFactory), "ClientProtocol.__init__ must be passed an ITerminalTransport implementor" + if terminalFactory is not None: + self.terminalFactory = terminalFactory + self.terminalArgs = a + self.terminalKwArgs = kw + + + def connectionMade(self): + if self.terminalFactory is not None: + self.terminal = self.terminalFactory(*self.terminalArgs, **self.terminalKwArgs) + self.terminal.factory = self.factory + self.terminal.makeConnection(self) + + + def connectionLost(self, reason): + if self.terminal is not None: + try: + self.terminal.connectionLost(reason) + finally: + del self.terminal + + + def dataReceived(self, data): + """ + Parse the given data from a terminal server, dispatching to event + handlers defined by C{self.terminal}. + """ + toWrite = [] + for b in iterbytes(data): + if self.state == b'data': + if b == b'\x1b': + if toWrite: + self.terminal.write(b''.join(toWrite)) + del toWrite[:] + self.state = b'escaped' + elif b == b'\x14': + if toWrite: + self.terminal.write(b''.join(toWrite)) + del toWrite[:] + self.terminal.shiftOut() + elif b == b'\x15': + if toWrite: + self.terminal.write(b''.join(toWrite)) + del toWrite[:] + self.terminal.shiftIn() + elif b == b'\x08': + if toWrite: + self.terminal.write(b''.join(toWrite)) + del toWrite[:] + self.terminal.cursorBackward() + else: + toWrite.append(b) + elif self.state == b'escaped': + fName = self._shorts.get(b) + if fName is not None: + self.state = b'data' + getattr(self.terminal, fName.decode("ascii"))() + else: + state = self._longs.get(b) + if state is not None: + self.state = state + else: + self.terminal.unhandledControlSequence(b'\x1b' + b) + self.state = b'data' + elif self.state == b'bracket-escape': + if self._escBuf is None: + self._escBuf = [] + if b.isalpha() or b == b'~': + self._handleControlSequence(b''.join(self._escBuf), b) + del self._escBuf + self.state = b'data' + else: + self._escBuf.append(b) + elif self.state == b'select-g0': + self.terminal.selectCharacterSet(self._charsets.get(b, b), G0) + self.state = b'data' + elif self.state == b'select-g1': + self.terminal.selectCharacterSet(self._charsets.get(b, b), G1) + self.state = b'data' + elif self.state == b'select-height-width': + self._handleHeightWidth(b) + self.state = b'data' + else: + raise ValueError("Illegal state") + if toWrite: + self.terminal.write(b''.join(toWrite)) + + + def _handleControlSequence(self, buf, terminal): + f = getattr(self.controlSequenceParser, CST.get(terminal, terminal).decode("ascii"), None) + if f is None: + self.terminal.unhandledControlSequence(b'\x1b[' + buf + terminal) + else: + f(self, self.terminal, buf) + + + class ControlSequenceParser: + def _makeSimple(ch, fName): + n = 'cursor' + fName + def simple(self, proto, handler, buf): + if not buf: + getattr(handler, n)(1) + else: + try: + m = int(buf) + except ValueError: + handler.unhandledControlSequence(b'\x1b[' + buf + ch) + else: + getattr(handler, n)(m) + return simple + + for (ch, fName) in (('A', 'Up'), + ('B', 'Down'), + ('C', 'Forward'), + ('D', 'Backward')): + exec(ch + " = _makeSimple(ch, fName)") + del _makeSimple + + + def h(self, proto, handler, buf): + # XXX - Handle '?' to introduce ANSI-Compatible private modes. + try: + modes = [int(mode) for mode in buf.split(b';')] + except ValueError: + handler.unhandledControlSequence(b'\x1b[' + buf + b'h') + else: + handler.setModes(modes) + + + def l(self, proto, handler, buf): + # XXX - Handle '?' to introduce ANSI-Compatible private modes. + try: + modes = [int(mode) for mode in buf.split(b';')] + except ValueError: + handler.unhandledControlSequence(b'\x1b[' + buf + 'l') + else: + handler.resetModes(modes) + + + def r(self, proto, handler, buf): + parts = buf.split(b';') + if len(parts) == 1: + handler.setScrollRegion(None, None) + elif len(parts) == 2: + try: + if parts[0]: + pt = int(parts[0]) + else: + pt = None + if parts[1]: + pb = int(parts[1]) + else: + pb = None + except ValueError: + handler.unhandledControlSequence(b'\x1b[' + buf + b'r') + else: + handler.setScrollRegion(pt, pb) + else: + handler.unhandledControlSequence(b'\x1b[' + buf + b'r') + + def K(self, proto, handler, buf): + if not buf: + handler.eraseToLineEnd() + elif buf == b'1': + handler.eraseToLineBeginning() + elif buf == b'2': + handler.eraseLine() + else: + handler.unhandledControlSequence(b'\x1b[' + buf + b'K') + + + def H(self, proto, handler, buf): + handler.cursorHome() + + + def J(self, proto, handler, buf): + if not buf: + handler.eraseToDisplayEnd() + elif buf == b'1': + handler.eraseToDisplayBeginning() + elif buf == b'2': + handler.eraseDisplay() + else: + handler.unhandledControlSequence(b'\x1b[' + buf + b'J') + + + def P(self, proto, handler, buf): + if not buf: + handler.deleteCharacter(1) + else: + try: + n = int(buf) + except ValueError: + handler.unhandledControlSequence(b'\x1b[' + buf + b'P') + else: + handler.deleteCharacter(n) + + def L(self, proto, handler, buf): + if not buf: + handler.insertLine(1) + else: + try: + n = int(buf) + except ValueError: + handler.unhandledControlSequence(b'\x1b[' + buf + b'L') + else: + handler.insertLine(n) + + + def M(self, proto, handler, buf): + if not buf: + handler.deleteLine(1) + else: + try: + n = int(buf) + except ValueError: + handler.unhandledControlSequence(b'\x1b[' + buf + b'M') + else: + handler.deleteLine(n) + + + def n(self, proto, handler, buf): + if buf == b'6': + x, y = handler.reportCursorPosition() + proto.transport.write(b'\x1b[' + + intToBytes(x+1) + + b';' + + intToBytes(y+1) + + b'R') + else: + handler.unhandledControlSequence(b'\x1b[' + buf + b'n') + + + def m(self, proto, handler, buf): + if not buf: + handler.selectGraphicRendition(NORMAL) + else: + attrs = [] + for a in buf.split(b';'): + try: + a = int(a) + except ValueError: + pass + attrs.append(a) + handler.selectGraphicRendition(*attrs) + + controlSequenceParser = ControlSequenceParser() + + + def _handleHeightWidth(self, b): + if b == b'3': + self.terminal.doubleHeightLine(True) + elif b == b'4': + self.terminal.doubleHeightLine(False) + elif b == b'5': + self.terminal.singleWidthLine() + elif b == b'6': + self.terminal.doubleWidthLine() + else: + self.terminal.unhandledControlSequence(b'\x1b#' + b) + + +__all__ = [ + # Interfaces + 'ITerminalProtocol', 'ITerminalTransport', + + # Symbolic constants + 'modes', 'privateModes', 'FUNCTION_KEYS', + + 'CS_US', 'CS_UK', 'CS_DRAWING', 'CS_ALTERNATE', 'CS_ALTERNATE_SPECIAL', + 'G0', 'G1', 'G2', 'G3', + + 'UNDERLINE', 'REVERSE_VIDEO', 'BLINK', 'BOLD', 'NORMAL', + + # Protocol classes + 'ServerProtocol', 'ClientProtocol'] diff --git a/contrib/python/Twisted/py2/twisted/conch/insults/text.py b/contrib/python/Twisted/py2/twisted/conch/insults/text.py new file mode 100644 index 00000000000..54476f71a12 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/insults/text.py @@ -0,0 +1,176 @@ +# -*- test-case-name: twisted.conch.test.test_text -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Character attribute manipulation API. + +This module provides a domain-specific language (using Python syntax) +for the creation of text with additional display attributes associated +with it. It is intended as an alternative to manually building up +strings containing ECMA 48 character attribute control codes. It +currently supports foreground and background colors (black, red, +green, yellow, blue, magenta, cyan, and white), intensity selection, +underlining, blinking and reverse video. Character set selection +support is planned. + +Character attributes are specified by using two Python operations: +attribute lookup and indexing. For example, the string \"Hello +world\" with red foreground and all other attributes set to their +defaults, assuming the name twisted.conch.insults.text.attributes has +been imported and bound to the name \"A\" (with the statement C{from +twisted.conch.insults.text import attributes as A}, for example) one +uses this expression:: + + A.fg.red[\"Hello world\"] + +Other foreground colors are set by substituting their name for +\"red\". To set both a foreground and a background color, this +expression is used:: + + A.fg.red[A.bg.green[\"Hello world\"]] + +Note that either A.bg.green can be nested within A.fg.red or vice +versa. Also note that multiple items can be nested within a single +index operation by separating them with commas:: + + A.bg.green[A.fg.red[\"Hello\"], " ", A.fg.blue[\"world\"]] + +Other character attributes are set in a similar fashion. To specify a +blinking version of the previous expression:: + + A.blink[A.bg.green[A.fg.red[\"Hello\"], " ", A.fg.blue[\"world\"]]] + +C{A.reverseVideo}, C{A.underline}, and C{A.bold} are also valid. + +A third operation is actually supported: unary negation. This turns +off an attribute when an enclosing expression would otherwise have +caused it to be on. For example:: + + A.underline[A.fg.red[\"Hello\", -A.underline[\" world\"]]] + +A formatting structure can then be serialized into a string containing the +necessary VT102 control codes with L{assembleFormattedText}. + +@see: L{twisted.conch.insults.text._CharacterAttributes} +@author: Jp Calderone +""" + +from incremental import Version + +from twisted.conch.insults import helper, insults +from twisted.python import _textattributes +from twisted.python.deprecate import deprecatedModuleAttribute + + + +flatten = _textattributes.flatten + +deprecatedModuleAttribute( + Version('Twisted', 13, 1, 0), + 'Use twisted.conch.insults.text.assembleFormattedText instead.', + 'twisted.conch.insults.text', + 'flatten') + +_TEXT_COLORS = { + 'black': helper.BLACK, + 'red': helper.RED, + 'green': helper.GREEN, + 'yellow': helper.YELLOW, + 'blue': helper.BLUE, + 'magenta': helper.MAGENTA, + 'cyan': helper.CYAN, + 'white': helper.WHITE} + + + +class _CharacterAttributes(_textattributes.CharacterAttributesMixin): + """ + Factory for character attributes, including foreground and background color + and non-color attributes such as bold, reverse video and underline. + + Character attributes are applied to actual text by using object + indexing-syntax (C{obj['abc']}) after accessing a factory attribute, for + example:: + + attributes.bold['Some text'] + + These can be nested to mix attributes:: + + attributes.bold[attributes.underline['Some text']] + + And multiple values can be passed:: + + attributes.normal[attributes.bold['Some'], ' text'] + + Non-color attributes can be accessed by attribute name, available + attributes are: + + - bold + - blink + - reverseVideo + - underline + + Available colors are: + + 0. black + 1. red + 2. green + 3. yellow + 4. blue + 5. magenta + 6. cyan + 7. white + + @ivar fg: Foreground colors accessed by attribute name, see above + for possible names. + + @ivar bg: Background colors accessed by attribute name, see above + for possible names. + """ + fg = _textattributes._ColorAttribute( + _textattributes._ForegroundColorAttr, _TEXT_COLORS) + bg = _textattributes._ColorAttribute( + _textattributes._BackgroundColorAttr, _TEXT_COLORS) + + attrs = { + 'bold': insults.BOLD, + 'blink': insults.BLINK, + 'underline': insults.UNDERLINE, + 'reverseVideo': insults.REVERSE_VIDEO} + + + +def assembleFormattedText(formatted): + """ + Assemble formatted text from structured information. + + Currently handled formatting includes: bold, blink, reverse, underline and + color codes. + + For example:: + + from twisted.conch.insults.text import attributes as A + assembleFormattedText( + A.normal[A.bold['Time: '], A.fg.lightRed['Now!']]) + + Would produce "Time: " in bold formatting, followed by "Now!" with a + foreground color of light red and without any additional formatting. + + @param formatted: Structured text and attributes. + + @rtype: L{str} + @return: String containing VT102 control sequences that mimic those + specified by C{formatted}. + + @see: L{twisted.conch.insults.text._CharacterAttributes} + @since: 13.1 + """ + return _textattributes.flatten( + formatted, helper._FormattingState(), 'toVT102') + + + +attributes = _CharacterAttributes() + +__all__ = ['attributes', 'flatten'] diff --git a/contrib/python/Twisted/py2/twisted/conch/insults/window.py b/contrib/python/Twisted/py2/twisted/conch/insults/window.py new file mode 100644 index 00000000000..d3caf7d3f4e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/insults/window.py @@ -0,0 +1,1027 @@ +# -*- test-case-name: twisted.conch.test.test_window -*- + +""" +Simple insults-based widget library + +@author: Jp Calderone +""" + +import array + +from twisted.conch.insults import insults, helper +from twisted.python import text as tptext +from twisted.python.compat import (_PY3, _bytesChr as chr) + +class YieldFocus(Exception): + """ + Input focus manipulation exception + """ + + + +class BoundedTerminalWrapper(object): + def __init__(self, terminal, width, height, xoff, yoff): + self.width = width + self.height = height + self.xoff = xoff + self.yoff = yoff + self.terminal = terminal + self.cursorForward = terminal.cursorForward + self.selectCharacterSet = terminal.selectCharacterSet + self.selectGraphicRendition = terminal.selectGraphicRendition + self.saveCursor = terminal.saveCursor + self.restoreCursor = terminal.restoreCursor + + + def cursorPosition(self, x, y): + return self.terminal.cursorPosition( + self.xoff + min(self.width, x), + self.yoff + min(self.height, y) + ) + + + def cursorHome(self): + return self.terminal.cursorPosition( + self.xoff, self.yoff) + + + def write(self, data): + return self.terminal.write(data) + + + +class Widget(object): + focused = False + parent = None + dirty = False + width = height = None + + def repaint(self): + if not self.dirty: + self.dirty = True + if self.parent is not None and not self.parent.dirty: + self.parent.repaint() + + + def filthy(self): + self.dirty = True + + + def redraw(self, width, height, terminal): + self.filthy() + self.draw(width, height, terminal) + + + def draw(self, width, height, terminal): + if width != self.width or height != self.height or self.dirty: + self.width = width + self.height = height + self.dirty = False + self.render(width, height, terminal) + + + def render(self, width, height, terminal): + pass + + + def sizeHint(self): + return None + + + def keystrokeReceived(self, keyID, modifier): + if keyID == b'\t': + self.tabReceived(modifier) + elif keyID == b'\x7f': + self.backspaceReceived() + elif keyID in insults.FUNCTION_KEYS: + self.functionKeyReceived(keyID, modifier) + else: + self.characterReceived(keyID, modifier) + + + def tabReceived(self, modifier): + # XXX TODO - Handle shift+tab + raise YieldFocus() + + + def focusReceived(self): + """ + Called when focus is being given to this widget. + + May raise YieldFocus is this widget does not want focus. + """ + self.focused = True + self.repaint() + + + def focusLost(self): + self.focused = False + self.repaint() + + + def backspaceReceived(self): + pass + + + def functionKeyReceived(self, keyID, modifier): + name = keyID + if not isinstance(keyID, str): + name = name.decode("utf-8") + func = getattr(self, 'func_' + name, None) + if func is not None: + func(modifier) + + + def characterReceived(self, keyID, modifier): + pass + + + +class ContainerWidget(Widget): + """ + @ivar focusedChild: The contained widget which currently has + focus, or None. + """ + focusedChild = None + focused = False + + def __init__(self): + Widget.__init__(self) + self.children = [] + + + def addChild(self, child): + assert child.parent is None + child.parent = self + self.children.append(child) + if self.focusedChild is None and self.focused: + try: + child.focusReceived() + except YieldFocus: + pass + else: + self.focusedChild = child + self.repaint() + + + def remChild(self, child): + assert child.parent is self + child.parent = None + self.children.remove(child) + self.repaint() + + + def filthy(self): + for ch in self.children: + ch.filthy() + Widget.filthy(self) + + + def render(self, width, height, terminal): + for ch in self.children: + ch.draw(width, height, terminal) + + + def changeFocus(self): + self.repaint() + + if self.focusedChild is not None: + self.focusedChild.focusLost() + focusedChild = self.focusedChild + self.focusedChild = None + try: + curFocus = self.children.index(focusedChild) + 1 + except ValueError: + raise YieldFocus() + else: + curFocus = 0 + while curFocus < len(self.children): + try: + self.children[curFocus].focusReceived() + except YieldFocus: + curFocus += 1 + else: + self.focusedChild = self.children[curFocus] + return + # None of our children wanted focus + raise YieldFocus() + + + def focusReceived(self): + self.changeFocus() + self.focused = True + + + def keystrokeReceived(self, keyID, modifier): + if self.focusedChild is not None: + try: + self.focusedChild.keystrokeReceived(keyID, modifier) + except YieldFocus: + self.changeFocus() + self.repaint() + else: + Widget.keystrokeReceived(self, keyID, modifier) + + + +class TopWindow(ContainerWidget): + """ + A top-level container object which provides focus wrap-around and paint + scheduling. + + @ivar painter: A no-argument callable which will be invoked when this + widget needs to be redrawn. + + @ivar scheduler: A one-argument callable which will be invoked with a + no-argument callable and should arrange for it to invoked at some point in + the near future. The no-argument callable will cause this widget and all + its children to be redrawn. It is typically beneficial for the no-argument + callable to be invoked at the end of handling for whatever event is + currently active; for example, it might make sense to call it at the end of + L{twisted.conch.insults.insults.ITerminalProtocol.keystrokeReceived}. + Note, however, that since calls to this may also be made in response to no + apparent event, arrangements should be made for the function to be called + even if an event handler such as C{keystrokeReceived} is not on the call + stack (eg, using + L{reactor.callLater} + with a short timeout). + """ + focused = True + + def __init__(self, painter, scheduler): + ContainerWidget.__init__(self) + self.painter = painter + self.scheduler = scheduler + + _paintCall = None + + + def repaint(self): + if self._paintCall is None: + self._paintCall = object() + self.scheduler(self._paint) + ContainerWidget.repaint(self) + + + def _paint(self): + self._paintCall = None + self.painter() + + + def changeFocus(self): + try: + ContainerWidget.changeFocus(self) + except YieldFocus: + try: + ContainerWidget.changeFocus(self) + except YieldFocus: + pass + + + def keystrokeReceived(self, keyID, modifier): + try: + ContainerWidget.keystrokeReceived(self, keyID, modifier) + except YieldFocus: + self.changeFocus() + + + +class AbsoluteBox(ContainerWidget): + def moveChild(self, child, x, y): + for n in range(len(self.children)): + if self.children[n][0] is child: + self.children[n] = (child, x, y) + break + else: + raise ValueError("No such child", child) + + + def render(self, width, height, terminal): + for (ch, x, y) in self.children: + wrap = BoundedTerminalWrapper(terminal, width - x, height - y, x, y) + ch.draw(width, height, wrap) + + + +class _Box(ContainerWidget): + TOP, CENTER, BOTTOM = range(3) + + def __init__(self, gravity=CENTER): + ContainerWidget.__init__(self) + self.gravity = gravity + + + def sizeHint(self): + height = 0 + width = 0 + for ch in self.children: + hint = ch.sizeHint() + if hint is None: + hint = (None, None) + + if self.variableDimension == 0: + if hint[0] is None: + width = None + elif width is not None: + width += hint[0] + if hint[1] is None: + height = None + elif height is not None: + height = max(height, hint[1]) + else: + if hint[0] is None: + width = None + elif width is not None: + width = max(width, hint[0]) + if hint[1] is None: + height = None + elif height is not None: + height += hint[1] + + return width, height + + + def render(self, width, height, terminal): + if not self.children: + return + + greedy = 0 + wants = [] + for ch in self.children: + hint = ch.sizeHint() + if hint is None: + hint = (None, None) + if hint[self.variableDimension] is None: + greedy += 1 + wants.append(hint[self.variableDimension]) + + length = (width, height)[self.variableDimension] + totalWant = sum([w for w in wants if w is not None]) + if greedy: + leftForGreedy = int((length - totalWant) / greedy) + + widthOffset = heightOffset = 0 + + for want, ch in zip(wants, self.children): + if want is None: + want = leftForGreedy + + subWidth, subHeight = width, height + if self.variableDimension == 0: + subWidth = want + else: + subHeight = want + + wrap = BoundedTerminalWrapper( + terminal, + subWidth, + subHeight, + widthOffset, + heightOffset, + ) + ch.draw(subWidth, subHeight, wrap) + if self.variableDimension == 0: + widthOffset += want + else: + heightOffset += want + + + +class HBox(_Box): + variableDimension = 0 + + + +class VBox(_Box): + variableDimension = 1 + + + +class Packer(ContainerWidget): + def render(self, width, height, terminal): + if not self.children: + return + + root = int(len(self.children) ** 0.5 + 0.5) + boxes = [VBox() for n in range(root)] + for n, ch in enumerate(self.children): + boxes[n % len(boxes)].addChild(ch) + h = HBox() + map(h.addChild, boxes) + h.render(width, height, terminal) + + + +class Canvas(Widget): + focused = False + + contents = None + + def __init__(self): + Widget.__init__(self) + self.resize(1, 1) + + + def resize(self, width, height): + contents = array.array('B', b' ' * width * height) + if self.contents is not None: + for x in range(min(width, self._width)): + for y in range(min(height, self._height)): + contents[width * y + x] = self[x, y] + self.contents = contents + self._width = width + self._height = height + if self.x >= width: + self.x = width - 1 + if self.y >= height: + self.y = height - 1 + + + def __getitem__(self, index): + (x, y) = index + return self.contents[(self._width * y) + x] + + + def __setitem__(self, index, value): + (x, y) = index + self.contents[(self._width * y) + x] = value + + + def clear(self): + self.contents = array.array('B', b' ' * len(self.contents)) + + + def render(self, width, height, terminal): + if not width or not height: + return + + if width != self._width or height != self._height: + self.resize(width, height) + for i in range(height): + terminal.cursorPosition(0, i) + if _PY3: + text = self.contents[self._width * i: + self._width * i + self._width + ].tobytes() + else: + text = self.contents[self._width * i: + self._width * i + self._width + ].tostring() + text = text[:width] + terminal.write(text) + + + +def horizontalLine(terminal, y, left, right): + terminal.selectCharacterSet(insults.CS_DRAWING, insults.G0) + terminal.cursorPosition(left, y) + terminal.write(chr(0o161) * (right - left)) + terminal.selectCharacterSet(insults.CS_US, insults.G0) + + + +def verticalLine(terminal, x, top, bottom): + terminal.selectCharacterSet(insults.CS_DRAWING, insults.G0) + for n in range(top, bottom): + terminal.cursorPosition(x, n) + terminal.write(chr(0o170)) + terminal.selectCharacterSet(insults.CS_US, insults.G0) + + +def rectangle(terminal, position, dimension): + """ + Draw a rectangle + + @type position: L{tuple} + @param position: A tuple of the (top, left) coordinates of the rectangle. + @type dimension: L{tuple} + @param dimension: A tuple of the (width, height) size of the rectangle. + """ + (top, left) = position + (width, height) = dimension + terminal.selectCharacterSet(insults.CS_DRAWING, insults.G0) + + terminal.cursorPosition(top, left) + terminal.write(chr(0o154)) + terminal.write(chr(0o161) * (width - 2)) + terminal.write(chr(0o153)) + for n in range(height - 2): + terminal.cursorPosition(left, top + n + 1) + terminal.write(chr(0o170)) + terminal.cursorForward(width - 2) + terminal.write(chr(0o170)) + terminal.cursorPosition(0, top + height - 1) + terminal.write(chr(0o155)) + terminal.write(chr(0o161) * (width - 2)) + terminal.write(chr(0o152)) + + terminal.selectCharacterSet(insults.CS_US, insults.G0) + + + +class Border(Widget): + def __init__(self, containee): + Widget.__init__(self) + self.containee = containee + self.containee.parent = self + + + def focusReceived(self): + return self.containee.focusReceived() + + + def focusLost(self): + return self.containee.focusLost() + + + def keystrokeReceived(self, keyID, modifier): + return self.containee.keystrokeReceived(keyID, modifier) + + + def sizeHint(self): + hint = self.containee.sizeHint() + if hint is None: + hint = (None, None) + if hint[0] is None: + x = None + else: + x = hint[0] + 2 + if hint[1] is None: + y = None + else: + y = hint[1] + 2 + return x, y + + + def filthy(self): + self.containee.filthy() + Widget.filthy(self) + + + def render(self, width, height, terminal): + if self.containee.focused: + terminal.write(b'\x1b[31m') + rectangle(terminal, (0, 0), (width, height)) + terminal.write(b'\x1b[0m') + wrap = BoundedTerminalWrapper(terminal, width - 2, height - 2, 1, 1) + self.containee.draw(width - 2, height - 2, wrap) + + + +class Button(Widget): + def __init__(self, label, onPress): + Widget.__init__(self) + self.label = label + self.onPress = onPress + + + def sizeHint(self): + return len(self.label), 1 + + + def characterReceived(self, keyID, modifier): + if keyID == b'\r': + self.onPress() + + + def render(self, width, height, terminal): + terminal.cursorPosition(0, 0) + if self.focused: + terminal.write(b'\x1b[1m' + self.label + b'\x1b[0m') + else: + terminal.write(self.label) + + + +class TextInput(Widget): + def __init__(self, maxwidth, onSubmit): + Widget.__init__(self) + self.onSubmit = onSubmit + self.maxwidth = maxwidth + self.buffer = b'' + self.cursor = 0 + + + def setText(self, text): + self.buffer = text[:self.maxwidth] + self.cursor = len(self.buffer) + self.repaint() + + + def func_LEFT_ARROW(self, modifier): + if self.cursor > 0: + self.cursor -= 1 + self.repaint() + + + def func_RIGHT_ARROW(self, modifier): + if self.cursor < len(self.buffer): + self.cursor += 1 + self.repaint() + + + def backspaceReceived(self): + if self.cursor > 0: + self.buffer = self.buffer[:self.cursor - 1] + self.buffer[self.cursor:] + self.cursor -= 1 + self.repaint() + + + def characterReceived(self, keyID, modifier): + if keyID == b'\r': + self.onSubmit(self.buffer) + else: + if len(self.buffer) < self.maxwidth: + self.buffer = self.buffer[:self.cursor] + keyID + self.buffer[self.cursor:] + self.cursor += 1 + self.repaint() + + + def sizeHint(self): + return self.maxwidth + 1, 1 + + + def render(self, width, height, terminal): + currentText = self._renderText() + terminal.cursorPosition(0, 0) + if self.focused: + terminal.write(currentText[:self.cursor]) + cursor(terminal, currentText[self.cursor:self.cursor+1] or b' ') + terminal.write(currentText[self.cursor+1:]) + terminal.write(b' ' * (self.maxwidth - len(currentText) + 1)) + else: + more = self.maxwidth - len(currentText) + terminal.write(currentText + b'_' * more) + + + def _renderText(self): + return self.buffer + + + +class PasswordInput(TextInput): + def _renderText(self): + return '*' * len(self.buffer) + + + +class TextOutput(Widget): + text = b'' + + def __init__(self, size=None): + Widget.__init__(self) + self.size = size + + + + def sizeHint(self): + return self.size + + + + def render(self, width, height, terminal): + terminal.cursorPosition(0, 0) + text = self.text[:width] + terminal.write(text + b' ' * (width - len(text))) + + + + def setText(self, text): + self.text = text + self.repaint() + + + def focusReceived(self): + raise YieldFocus() + + + +class TextOutputArea(TextOutput): + WRAP, TRUNCATE = range(2) + + def __init__(self, size=None, longLines=WRAP): + TextOutput.__init__(self, size) + self.longLines = longLines + + + def render(self, width, height, terminal): + n = 0 + inputLines = self.text.splitlines() + outputLines = [] + while inputLines: + if self.longLines == self.WRAP: + line = inputLines.pop(0) + if not isinstance(line, str): + line = line.decode("utf-8") + wrappedLines = [] + for wrappedLine in tptext.greedyWrap(line, width): + if not isinstance(wrappedLine, bytes): + wrappedLine = wrappedLine.encode("utf-8") + wrappedLines.append(wrappedLine) + outputLines.extend(wrappedLines or [b'']) + else: + outputLines.append(inputLines.pop(0)[:width]) + if len(outputLines) >= height: + break + for n, L in enumerate(outputLines[:height]): + terminal.cursorPosition(0, n) + terminal.write(L) + + + +class Viewport(Widget): + _xOffset = 0 + _yOffset = 0 + + def xOffset(): + def get(self): + return self._xOffset + def set(self, value): + if self._xOffset != value: + self._xOffset = value + self.repaint() + return get, set + xOffset = property(*xOffset()) + + + def yOffset(): + def get(self): + return self._yOffset + def set(self, value): + if self._yOffset != value: + self._yOffset = value + self.repaint() + return get, set + yOffset = property(*yOffset()) + + _width = 160 + _height = 24 + + + def __init__(self, containee): + Widget.__init__(self) + self.containee = containee + self.containee.parent = self + + self._buf = helper.TerminalBuffer() + self._buf.width = self._width + self._buf.height = self._height + self._buf.connectionMade() + + + def filthy(self): + self.containee.filthy() + Widget.filthy(self) + + + def render(self, width, height, terminal): + self.containee.draw(self._width, self._height, self._buf) + + # XXX /Lame/ + for y, line in enumerate(self._buf.lines[self._yOffset:self._yOffset + height]): + terminal.cursorPosition(0, y) + n = 0 + for n, (ch, attr) in enumerate(line[self._xOffset:self._xOffset + width]): + if ch is self._buf.void: + ch = b' ' + terminal.write(ch) + if n < width: + terminal.write(b' ' * (width - n - 1)) + + + +class _Scrollbar(Widget): + def __init__(self, onScroll): + Widget.__init__(self) + self.onScroll = onScroll + self.percent = 0.0 + + + def smaller(self): + self.percent = min(1.0, max(0.0, self.onScroll(-1))) + self.repaint() + + + def bigger(self): + self.percent = min(1.0, max(0.0, self.onScroll(+1))) + self.repaint() + + + +class HorizontalScrollbar(_Scrollbar): + def sizeHint(self): + return (None, 1) + + + def func_LEFT_ARROW(self, modifier): + self.smaller() + + + def func_RIGHT_ARROW(self, modifier): + self.bigger() + + _left = u'\N{BLACK LEFT-POINTING TRIANGLE}' + _right = u'\N{BLACK RIGHT-POINTING TRIANGLE}' + _bar = u'\N{LIGHT SHADE}' + _slider = u'\N{DARK SHADE}' + + + def render(self, width, height, terminal): + terminal.cursorPosition(0, 0) + n = width - 3 + before = int(n * self.percent) + after = n - before + me = self._left + (self._bar * before) + self._slider + (self._bar * after) + self._right + terminal.write(me.encode('utf-8')) + + + +class VerticalScrollbar(_Scrollbar): + def sizeHint(self): + return (1, None) + + + def func_UP_ARROW(self, modifier): + self.smaller() + + + def func_DOWN_ARROW(self, modifier): + self.bigger() + + _up = u'\N{BLACK UP-POINTING TRIANGLE}' + _down = u'\N{BLACK DOWN-POINTING TRIANGLE}' + _bar = u'\N{LIGHT SHADE}' + _slider = u'\N{DARK SHADE}' + + + def render(self, width, height, terminal): + terminal.cursorPosition(0, 0) + knob = int(self.percent * (height - 2)) + terminal.write(self._up.encode('utf-8')) + for i in range(1, height - 1): + terminal.cursorPosition(0, i) + if i != (knob + 1): + terminal.write(self._bar.encode('utf-8')) + else: + terminal.write(self._slider.encode('utf-8')) + terminal.cursorPosition(0, height - 1) + terminal.write(self._down.encode('utf-8')) + + + +class ScrolledArea(Widget): + """ + A L{ScrolledArea} contains another widget wrapped in a viewport and + vertical and horizontal scrollbars for moving the viewport around. + """ + def __init__(self, containee): + Widget.__init__(self) + self._viewport = Viewport(containee) + self._horiz = HorizontalScrollbar(self._horizScroll) + self._vert = VerticalScrollbar(self._vertScroll) + + for w in self._viewport, self._horiz, self._vert: + w.parent = self + + + def _horizScroll(self, n): + self._viewport.xOffset += n + self._viewport.xOffset = max(0, self._viewport.xOffset) + return self._viewport.xOffset / 25.0 + + + def _vertScroll(self, n): + self._viewport.yOffset += n + self._viewport.yOffset = max(0, self._viewport.yOffset) + return self._viewport.yOffset / 25.0 + + + def func_UP_ARROW(self, modifier): + self._vert.smaller() + + + def func_DOWN_ARROW(self, modifier): + self._vert.bigger() + + + def func_LEFT_ARROW(self, modifier): + self._horiz.smaller() + + + def func_RIGHT_ARROW(self, modifier): + self._horiz.bigger() + + + def filthy(self): + self._viewport.filthy() + self._horiz.filthy() + self._vert.filthy() + Widget.filthy(self) + + + def render(self, width, height, terminal): + wrapper = BoundedTerminalWrapper(terminal, width - 2, height - 2, 1, 1) + self._viewport.draw(width - 2, height - 2, wrapper) + if self.focused: + terminal.write(b'\x1b[31m') + horizontalLine(terminal, 0, 1, width - 1) + verticalLine(terminal, 0, 1, height - 1) + self._vert.draw(1, height - 1, BoundedTerminalWrapper(terminal, 1, height - 1, width - 1, 0)) + self._horiz.draw(width, 1, BoundedTerminalWrapper(terminal, width, 1, 0, height - 1)) + terminal.write(b'\x1b[0m') + + + +def cursor(terminal, ch): + terminal.saveCursor() + terminal.selectGraphicRendition(str(insults.REVERSE_VIDEO)) + terminal.write(ch) + terminal.restoreCursor() + terminal.cursorForward() + + + +class Selection(Widget): + # Index into the sequence + focusedIndex = 0 + + # Offset into the displayed subset of the sequence + renderOffset = 0 + + def __init__(self, sequence, onSelect, minVisible=None): + Widget.__init__(self) + self.sequence = sequence + self.onSelect = onSelect + self.minVisible = minVisible + if minVisible is not None: + self._width = max(map(len, self.sequence)) + + + def sizeHint(self): + if self.minVisible is not None: + return self._width, self.minVisible + + + def func_UP_ARROW(self, modifier): + if self.focusedIndex > 0: + self.focusedIndex -= 1 + if self.renderOffset > 0: + self.renderOffset -= 1 + self.repaint() + + + def func_PGUP(self, modifier): + if self.renderOffset != 0: + self.focusedIndex -= self.renderOffset + self.renderOffset = 0 + else: + self.focusedIndex = max(0, self.focusedIndex - self.height) + self.repaint() + + + def func_DOWN_ARROW(self, modifier): + if self.focusedIndex < len(self.sequence) - 1: + self.focusedIndex += 1 + if self.renderOffset < self.height - 1: + self.renderOffset += 1 + self.repaint() + + + def func_PGDN(self, modifier): + if self.renderOffset != self.height - 1: + change = self.height - self.renderOffset - 1 + if change + self.focusedIndex >= len(self.sequence): + change = len(self.sequence) - self.focusedIndex - 1 + self.focusedIndex += change + self.renderOffset = self.height - 1 + else: + self.focusedIndex = min(len(self.sequence) - 1, self.focusedIndex + self.height) + self.repaint() + + + def characterReceived(self, keyID, modifier): + if keyID == b'\r': + self.onSelect(self.sequence[self.focusedIndex]) + + + def render(self, width, height, terminal): + self.height = height + start = self.focusedIndex - self.renderOffset + if start > len(self.sequence) - height: + start = max(0, len(self.sequence) - height) + + elements = self.sequence[start:start+height] + + for n, ele in enumerate(elements): + terminal.cursorPosition(0, n) + if n == self.renderOffset: + terminal.saveCursor() + if self.focused: + modes = str(insults.REVERSE_VIDEO), str(insults.BOLD) + else: + modes = str(insults.REVERSE_VIDEO), + terminal.selectGraphicRendition(*modes) + text = ele[:width] + terminal.write(text + (b' ' * (width - len(text)))) + if n == self.renderOffset: + terminal.restoreCursor() diff --git a/contrib/python/Twisted/py2/twisted/conch/interfaces.py b/contrib/python/Twisted/py2/twisted/conch/interfaces.py new file mode 100644 index 00000000000..cdf5489898a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/interfaces.py @@ -0,0 +1,444 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module contains interfaces defined for the L{twisted.conch} package. +""" + +from zope.interface import Interface, Attribute + +class IConchUser(Interface): + """ + A user who has been authenticated to Cred through Conch. This is + the interface between the SSH connection and the user. + """ + + conn = Attribute('The SSHConnection object for this user.') + + def lookupChannel(channelType, windowSize, maxPacket, data): + """ + The other side requested a channel of some sort. + + C{channelType} is the type of channel being requested, + as an ssh connection protocol channel type. + C{data} is any other packet data (often nothing). + + We return a subclass of L{SSHChannel}. If + the channel type is unknown, we return C{None}. + + For other failures, we raise an exception. If a + L{ConchError} is raised, the C{.value} will + be the message, and the C{.data} will be the error code. + + @param channelType: The requested channel type + @type channelType: L{bytes} + @param windowSize: The initial size of the remote window + @type windowSize: L{int} + @param maxPacket: The largest packet we should send + @type maxPacket: L{int} + @param data: Additional request data + @type data: L{bytes} + @rtype: a subclass of L{SSHChannel} or L{None} + """ + + def lookupSubsystem(subsystem, data): + """ + The other side requested a subsystem. + + We return a L{Protocol} implementing the requested subsystem. + If the subsystem is not available, we return C{None}. + + @param subsystem: The name of the subsystem being requested + @type subsystem: L{bytes} + @param data: Additional request data (often nothing) + @type data: L{bytes} + @rtype: L{Protocol} or L{None} + """ + + def gotGlobalRequest(requestType, data): + """ + A global request was sent from the other side. + + We return a true value on success or a false value on failure. + If we indicate success by returning a tuple, its second item + will be sent to the other side as additional response data. + + @param requestType: The type of the request + @type requestType: L{bytes} + @param data: Additional request data + @type data: L{bytes} + @rtype: boolean or L{tuple} + """ + + + +class ISession(Interface): + + def getPty(term, windowSize, modes): + """ + Get a pseudo-terminal for use by a shell or command. + + If a pseudo-terminal is not available, or the request otherwise + fails, raise an exception. + """ + + def openShell(proto): + """ + Open a shell and connect it to proto. + + @param proto: a L{ProcessProtocol} instance. + """ + + def execCommand(proto, command): + """ + Execute a command. + + @param proto: a L{ProcessProtocol} instance. + """ + + def windowChanged(newWindowSize): + """ + Called when the size of the remote screen has changed. + """ + + def eofReceived(): + """ + Called when the other side has indicated no more data will be sent. + """ + + def closed(): + """ + Called when the session is closed. + """ + + + +class ISFTPServer(Interface): + """ + SFTP subsystem for server-side communication. + + Each method should check to verify that the user has permission for + their actions. + """ + + avatar = Attribute( + """ + The avatar returned by the Realm that we are authenticated with, + and represents the logged-in user. + """) + + + def gotVersion(otherVersion, extData): + """ + Called when the client sends their version info. + + otherVersion is an integer representing the version of the SFTP + protocol they are claiming. + extData is a dictionary of extended_name : extended_data items. + These items are sent by the client to indicate additional features. + + This method should return a dictionary of extended_name : extended_data + items. These items are the additional features (if any) supported + by the server. + """ + return {} + + + def openFile(filename, flags, attrs): + """ + Called when the clients asks to open a file. + + @param filename: a string representing the file to open. + + @param flags: an integer of the flags to open the file with, ORed + together. The flags and their values are listed at the bottom of + L{twisted.conch.ssh.filetransfer} as FXF_*. + + @param attrs: a list of attributes to open the file with. It is a + dictionary, consisting of 0 or more keys. The possible keys are:: + + size: the size of the file in bytes + uid: the user ID of the file as an integer + gid: the group ID of the file as an integer + permissions: the permissions of the file with as an integer. + the bit representation of this field is defined by POSIX. + atime: the access time of the file as seconds since the epoch. + mtime: the modification time of the file as seconds since the epoch. + ext_*: extended attributes. The server is not required to + understand this, but it may. + + NOTE: there is no way to indicate text or binary files. it is up + to the SFTP client to deal with this. + + This method returns an object that meets the ISFTPFile interface. + Alternatively, it can return a L{Deferred} that will be called back + with the object. + """ + + + def removeFile(filename): + """ + Remove the given file. + + This method returns when the remove succeeds, or a Deferred that is + called back when it succeeds. + + @param filename: the name of the file as a string. + """ + + + def renameFile(oldpath, newpath): + """ + Rename the given file. + + This method returns when the rename succeeds, or a L{Deferred} that is + called back when it succeeds. If the rename fails, C{renameFile} will + raise an implementation-dependent exception. + + @param oldpath: the current location of the file. + @param newpath: the new file name. + """ + + + def makeDirectory(path, attrs): + """ + Make a directory. + + This method returns when the directory is created, or a Deferred that + is called back when it is created. + + @param path: the name of the directory to create as a string. + @param attrs: a dictionary of attributes to create the directory with. + Its meaning is the same as the attrs in the L{openFile} method. + """ + + + def removeDirectory(path): + """ + Remove a directory (non-recursively) + + It is an error to remove a directory that has files or directories in + it. + + This method returns when the directory is removed, or a Deferred that + is called back when it is removed. + + @param path: the directory to remove. + """ + + + def openDirectory(path): + """ + Open a directory for scanning. + + This method returns an iterable object that has a close() method, + or a Deferred that is called back with same. + + The close() method is called when the client is finished reading + from the directory. At this point, the iterable will no longer + be used. + + The iterable should return triples of the form (filename, + longname, attrs) or Deferreds that return the same. The + sequence must support __getitem__, but otherwise may be any + 'sequence-like' object. + + filename is the name of the file relative to the directory. + logname is an expanded format of the filename. The recommended format + is: + -rwxr-xr-x 1 mjos staff 348911 Mar 25 14:29 t-filexfer + 1234567890 123 12345678 12345678 12345678 123456789012 + + The first line is sample output, the second is the length of the field. + The fields are: permissions, link count, user owner, group owner, + size in bytes, modification time. + + attrs is a dictionary in the format of the attrs argument to openFile. + + @param path: the directory to open. + """ + + + def getAttrs(path, followLinks): + """ + Return the attributes for the given path. + + This method returns a dictionary in the same format as the attrs + argument to openFile or a Deferred that is called back with same. + + @param path: the path to return attributes for as a string. + @param followLinks: a boolean. If it is True, follow symbolic links + and return attributes for the real path at the base. If it is False, + return attributes for the specified path. + """ + + + def setAttrs(path, attrs): + """ + Set the attributes for the path. + + This method returns when the attributes are set or a Deferred that is + called back when they are. + + @param path: the path to set attributes for as a string. + @param attrs: a dictionary in the same format as the attrs argument to + L{openFile}. + """ + + + def readLink(path): + """ + Find the root of a set of symbolic links. + + This method returns the target of the link, or a Deferred that + returns the same. + + @param path: the path of the symlink to read. + """ + + + def makeLink(linkPath, targetPath): + """ + Create a symbolic link. + + This method returns when the link is made, or a Deferred that + returns the same. + + @param linkPath: the pathname of the symlink as a string. + @param targetPath: the path of the target of the link as a string. + """ + + + def realPath(path): + """ + Convert any path to an absolute path. + + This method returns the absolute path as a string, or a Deferred + that returns the same. + + @param path: the path to convert as a string. + """ + + + def extendedRequest(extendedName, extendedData): + """ + This is the extension mechanism for SFTP. The other side can send us + arbitrary requests. + + If we don't implement the request given by extendedName, raise + NotImplementedError. + + The return value is a string, or a Deferred that will be called + back with a string. + + @param extendedName: the name of the request as a string. + @param extendedData: the data the other side sent with the request, + as a string. + """ + + + +class IKnownHostEntry(Interface): + """ + A L{IKnownHostEntry} is an entry in an OpenSSH-formatted C{known_hosts} + file. + + @since: 8.2 + """ + + def matchesKey(key): + """ + Return True if this entry matches the given Key object, False + otherwise. + + @param key: The key object to match against. + @type key: L{twisted.conch.ssh.keys.Key} + """ + + + def matchesHost(hostname): + """ + Return True if this entry matches the given hostname, False otherwise. + + Note that this does no name resolution; if you want to match an IP + address, you have to resolve it yourself, and pass it in as a dotted + quad string. + + @param hostname: The hostname to match against. + @type hostname: L{str} + """ + + + def toString(): + """ + + @return: a serialized string representation of this entry, suitable for + inclusion in a known_hosts file. (Newline not included.) + + @rtype: L{str} + """ + + + +class ISFTPFile(Interface): + """ + This represents an open file on the server. An object adhering to this + interface should be returned from L{openFile}(). + """ + + def close(): + """ + Close the file. + + This method returns nothing if the close succeeds immediately, or a + Deferred that is called back when the close succeeds. + """ + + + def readChunk(offset, length): + """ + Read from the file. + + If EOF is reached before any data is read, raise EOFError. + + This method returns the data as a string, or a Deferred that is + called back with same. + + @param offset: an integer that is the index to start from in the file. + @param length: the maximum length of data to return. The actual amount + returned may less than this. For normal disk files, however, + this should read the requested number (up to the end of the file). + """ + + + def writeChunk(offset, data): + """ + Write to the file. + + This method returns when the write completes, or a Deferred that is + called when it completes. + + @param offset: an integer that is the index to start from in the file. + @param data: a string that is the data to write. + """ + + + def getAttrs(): + """ + Return the attributes for the file. + + This method returns a dictionary in the same format as the attrs + argument to L{openFile} or a L{Deferred} that is called back with same. + """ + + + def setAttrs(attrs): + """ + Set the attributes for the file. + + This method returns when the attributes are set or a Deferred that is + called back when they are. + + @param attrs: a dictionary in the same format as the attrs argument to + L{openFile}. + """ diff --git a/contrib/python/Twisted/py2/twisted/conch/ls.py b/contrib/python/Twisted/py2/twisted/conch/ls.py new file mode 100644 index 00000000000..85da665dc93 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ls.py @@ -0,0 +1,83 @@ +# -*- test-case-name: twisted.conch.test.test_cftp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +import array +import stat + +from time import time, strftime, localtime +from twisted.python.compat import _PY3 + +# Locale-independent month names to use instead of strftime's +_MONTH_NAMES = dict(list(zip( + list(range(1, 13)), + "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split()))) + + +def lsLine(name, s): + """ + Build an 'ls' line for a file ('file' in its generic sense, it + can be of any type). + """ + mode = s.st_mode + perms = array.array('B', b'-'*10) + ft = stat.S_IFMT(mode) + if stat.S_ISDIR(ft): perms[0] = ord('d') + elif stat.S_ISCHR(ft): perms[0] = ord('c') + elif stat.S_ISBLK(ft): perms[0] = ord('b') + elif stat.S_ISREG(ft): perms[0] = ord('-') + elif stat.S_ISFIFO(ft): perms[0] = ord('f') + elif stat.S_ISLNK(ft): perms[0] = ord('l') + elif stat.S_ISSOCK(ft): perms[0] = ord('s') + else: perms[0] = ord('!') + # User + if mode&stat.S_IRUSR:perms[1] = ord('r') + if mode&stat.S_IWUSR:perms[2] = ord('w') + if mode&stat.S_IXUSR:perms[3] = ord('x') + # Group + if mode&stat.S_IRGRP:perms[4] = ord('r') + if mode&stat.S_IWGRP:perms[5] = ord('w') + if mode&stat.S_IXGRP:perms[6] = ord('x') + # Other + if mode&stat.S_IROTH:perms[7] = ord('r') + if mode&stat.S_IWOTH:perms[8] = ord('w') + if mode&stat.S_IXOTH:perms[9] = ord('x') + # Suid/sgid + if mode&stat.S_ISUID: + if perms[3] == ord('x'): perms[3] = ord('s') + else: perms[3] = ord('S') + if mode&stat.S_ISGID: + if perms[6] == ord('x'): perms[6] = ord('s') + else: perms[6] = ord('S') + + if _PY3: + if isinstance(name, bytes): + name = name.decode("utf-8") + lsPerms = perms.tobytes() + lsPerms = lsPerms.decode("utf-8") + else: + lsPerms = perms.tostring() + + lsresult = [ + lsPerms, + str(s.st_nlink).rjust(5), + ' ', + str(s.st_uid).ljust(9), + str(s.st_gid).ljust(9), + str(s.st_size).rjust(8), + ' ', + ] + # Need to specify the month manually, as strftime depends on locale + ttup = localtime(s.st_mtime) + sixmonths = 60 * 60 * 24 * 7 * 26 + if s.st_mtime + sixmonths < time(): # Last edited more than 6mo ago + strtime = strftime("%%s %d %Y ", ttup) + else: + strtime = strftime("%%s %d %H:%M ", ttup) + lsresult.append(strtime % (_MONTH_NAMES[ttup[1]],)) + + lsresult.append(name) + return ''.join(lsresult) + + +__all__ = ['lsLine'] diff --git a/contrib/python/Twisted/py2/twisted/conch/manhole.py b/contrib/python/Twisted/py2/twisted/conch/manhole.py new file mode 100644 index 00000000000..70e16b70cd7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/manhole.py @@ -0,0 +1,401 @@ +# -*- test-case-name: twisted.conch.test.test_manhole -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Line-input oriented interactive interpreter loop. + +Provides classes for handling Python source input and arbitrary output +interactively from a Twisted application. Also included is syntax coloring +code with support for VT102 terminals, control code handling (^C, ^D, ^Q), +and reasonable handling of Deferreds. + +@author: Jp Calderone +""" + +import code, sys, tokenize +from io import BytesIO + +from twisted.conch import recvline + +from twisted.internet import defer +from twisted.python.compat import _tokenize, _get_async_param +from twisted.python.htmlizer import TokenPrinter + + + +class FileWrapper: + """ + Minimal write-file-like object. + + Writes are translated into addOutput calls on an object passed to + __init__. Newlines are also converted from network to local style. + """ + + softspace = 0 + state = 'normal' + + def __init__(self, o): + self.o = o + + + def flush(self): + pass + + + def write(self, data): + self.o.addOutput(data.replace('\r\n', '\n')) + + + def writelines(self, lines): + self.write(''.join(lines)) + + + +class ManholeInterpreter(code.InteractiveInterpreter): + """ + Interactive Interpreter with special output and Deferred support. + + Aside from the features provided by L{code.InteractiveInterpreter}, this + class captures sys.stdout output and redirects it to the appropriate + location (the Manhole protocol instance). It also treats Deferreds + which reach the top-level specially: each is formatted to the user with + a unique identifier and a new callback and errback added to it, each of + which will format the unique identifier and the result with which the + Deferred fires and then pass it on to the next participant in the + callback chain. + """ + + numDeferreds = 0 + def __init__(self, handler, locals=None, filename=""): + code.InteractiveInterpreter.__init__(self, locals) + self._pendingDeferreds = {} + self.handler = handler + self.filename = filename + self.resetBuffer() + + + def resetBuffer(self): + """ + Reset the input buffer. + """ + self.buffer = [] + + + def push(self, line): + """ + Push a line to the interpreter. + + The line should not have a trailing newline; it may have + internal newlines. The line is appended to a buffer and the + interpreter's runsource() method is called with the + concatenated contents of the buffer as source. If this + indicates that the command was executed or invalid, the buffer + is reset; otherwise, the command is incomplete, and the buffer + is left as it was after the line was appended. The return + value is 1 if more input is required, 0 if the line was dealt + with in some way (this is the same as runsource()). + + @param line: line of text + @type line: L{bytes} + @return: L{bool} from L{code.InteractiveInterpreter.runsource} + """ + self.buffer.append(line) + source = b"\n".join(self.buffer) + source = source.decode("utf-8") + more = self.runsource(source, self.filename) + if not more: + self.resetBuffer() + return more + + + def runcode(self, *a, **kw): + orighook, sys.displayhook = sys.displayhook, self.displayhook + try: + origout, sys.stdout = sys.stdout, FileWrapper(self.handler) + try: + code.InteractiveInterpreter.runcode(self, *a, **kw) + finally: + sys.stdout = origout + finally: + sys.displayhook = orighook + + + def displayhook(self, obj): + self.locals['_'] = obj + if isinstance(obj, defer.Deferred): + # XXX Ick, where is my "hasFired()" interface? + if hasattr(obj, "result"): + self.write(repr(obj)) + elif id(obj) in self._pendingDeferreds: + self.write("" % (self._pendingDeferreds[id(obj)][0],)) + else: + d = self._pendingDeferreds + k = self.numDeferreds + d[id(obj)] = (k, obj) + self.numDeferreds += 1 + obj.addCallbacks(self._cbDisplayDeferred, self._ebDisplayDeferred, + callbackArgs=(k, obj), errbackArgs=(k, obj)) + self.write("" % (k,)) + elif obj is not None: + self.write(repr(obj)) + + + def _cbDisplayDeferred(self, result, k, obj): + self.write("Deferred #%d called back: %r" % (k, result), True) + del self._pendingDeferreds[id(obj)] + return result + + + def _ebDisplayDeferred(self, failure, k, obj): + self.write("Deferred #%d failed: %r" % (k, failure.getErrorMessage()), True) + del self._pendingDeferreds[id(obj)] + return failure + + + def write(self, data, isAsync=None, **kwargs): + isAsync = _get_async_param(isAsync, **kwargs) + self.handler.addOutput(data, isAsync) + + + +CTRL_C = b'\x03' +CTRL_D = b'\x04' +CTRL_BACKSLASH = b'\x1c' +CTRL_L = b'\x0c' +CTRL_A = b'\x01' +CTRL_E = b'\x05' + + + +class Manhole(recvline.HistoricRecvLine): + """ + Mediator between a fancy line source and an interactive interpreter. + + This accepts lines from its transport and passes them on to a + L{ManholeInterpreter}. Control commands (^C, ^D, ^\) are also handled + with something approximating their normal terminal-mode behavior. It + can optionally be constructed with a dict which will be used as the + local namespace for any code executed. + """ + + namespace = None + + def __init__(self, namespace=None): + recvline.HistoricRecvLine.__init__(self) + if namespace is not None: + self.namespace = namespace.copy() + + + def connectionMade(self): + recvline.HistoricRecvLine.connectionMade(self) + self.interpreter = ManholeInterpreter(self, self.namespace) + self.keyHandlers[CTRL_C] = self.handle_INT + self.keyHandlers[CTRL_D] = self.handle_EOF + self.keyHandlers[CTRL_L] = self.handle_FF + self.keyHandlers[CTRL_A] = self.handle_HOME + self.keyHandlers[CTRL_E] = self.handle_END + self.keyHandlers[CTRL_BACKSLASH] = self.handle_QUIT + + + def handle_INT(self): + """ + Handle ^C as an interrupt keystroke by resetting the current input + variables to their initial state. + """ + self.pn = 0 + self.lineBuffer = [] + self.lineBufferIndex = 0 + self.interpreter.resetBuffer() + + self.terminal.nextLine() + self.terminal.write(b"KeyboardInterrupt") + self.terminal.nextLine() + self.terminal.write(self.ps[self.pn]) + + + def handle_EOF(self): + if self.lineBuffer: + self.terminal.write(b'\a') + else: + self.handle_QUIT() + + + def handle_FF(self): + """ + Handle a 'form feed' byte - generally used to request a screen + refresh/redraw. + """ + self.terminal.eraseDisplay() + self.terminal.cursorHome() + self.drawInputLine() + + + def handle_QUIT(self): + self.terminal.loseConnection() + + + def _needsNewline(self): + w = self.terminal.lastWrite + return not w.endswith(b'\n') and not w.endswith(b'\x1bE') + + + def addOutput(self, data, isAsync=None, **kwargs): + isAsync = _get_async_param(isAsync, **kwargs) + if isAsync: + self.terminal.eraseLine() + self.terminal.cursorBackward(len(self.lineBuffer) + + len(self.ps[self.pn])) + + self.terminal.write(data) + + if isAsync: + if self._needsNewline(): + self.terminal.nextLine() + + self.terminal.write(self.ps[self.pn]) + + if self.lineBuffer: + oldBuffer = self.lineBuffer + self.lineBuffer = [] + self.lineBufferIndex = 0 + + self._deliverBuffer(oldBuffer) + + + def lineReceived(self, line): + more = self.interpreter.push(line) + self.pn = bool(more) + if self._needsNewline(): + self.terminal.nextLine() + self.terminal.write(self.ps[self.pn]) + + + +class VT102Writer: + """ + Colorizer for Python tokens. + + A series of tokens are written to instances of this object. Each is + colored in a particular way. The final line of the result of this is + generally added to the output. + """ + + typeToColor = { + 'identifier': b'\x1b[31m', + 'keyword': b'\x1b[32m', + 'parameter': b'\x1b[33m', + 'variable': b'\x1b[1;33m', + 'string': b'\x1b[35m', + 'number': b'\x1b[36m', + 'op': b'\x1b[37m'} + + normalColor = b'\x1b[0m' + + def __init__(self): + self.written = [] + + + def color(self, type): + r = self.typeToColor.get(type, b'') + return r + + + def write(self, token, type=None): + if token and token != b'\r': + c = self.color(type) + if c: + self.written.append(c) + self.written.append(token) + if c: + self.written.append(self.normalColor) + + + def __bytes__(self): + s = b''.join(self.written) + return s.strip(b'\n').splitlines()[-1] + + if bytes == str: + # Compat with Python 2.7 + __str__ = __bytes__ + + + +def lastColorizedLine(source): + """ + Tokenize and colorize the given Python source. + + Returns a VT102-format colorized version of the last line of C{source}. + + @param source: Python source code + @type source: L{str} or L{bytes} + @return: L{bytes} of colorized source + """ + if not isinstance(source, bytes): + source = source.encode("utf-8") + w = VT102Writer() + p = TokenPrinter(w.write).printtoken + s = BytesIO(source) + + for token in _tokenize(s.readline): + (tokenType, string, start, end, line) = token + p(tokenType, string, start, end, line) + + return bytes(w) + + + +class ColoredManhole(Manhole): + """ + A REPL which syntax colors input as users type it. + """ + + def getSource(self): + """ + Return a string containing the currently entered source. + + This is only the code which will be considered for execution + next. + """ + return (b'\n'.join(self.interpreter.buffer) + + b'\n' + + b''.join(self.lineBuffer)) + + + def characterReceived(self, ch, moreCharactersComing): + if self.mode == 'insert': + self.lineBuffer.insert(self.lineBufferIndex, ch) + else: + self.lineBuffer[self.lineBufferIndex:self.lineBufferIndex+1] = [ch] + self.lineBufferIndex += 1 + + if moreCharactersComing: + # Skip it all, we'll get called with another character in + # like 2 femtoseconds. + return + + if ch == b' ': + # Don't bother to try to color whitespace + self.terminal.write(ch) + return + + source = self.getSource() + + # Try to write some junk + try: + coloredLine = lastColorizedLine(source) + except tokenize.TokenError: + # We couldn't do it. Strange. Oh well, just add the character. + self.terminal.write(ch) + else: + # Success! Clear the source on this line. + self.terminal.eraseLine() + self.terminal.cursorBackward(len(self.lineBuffer) + len(self.ps[self.pn]) - 1) + + # And write a new, colorized one. + self.terminal.write(self.ps[self.pn] + coloredLine) + + # And move the cursor to where it belongs + n = len(self.lineBuffer) - self.lineBufferIndex + if n: + self.terminal.cursorBackward(n) diff --git a/contrib/python/Twisted/py2/twisted/conch/manhole_ssh.py b/contrib/python/Twisted/py2/twisted/conch/manhole_ssh.py new file mode 100644 index 00000000000..84b242f24e9 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/manhole_ssh.py @@ -0,0 +1,141 @@ +# -*- test-case-name: twisted.conch.test.test_manhole -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +insults/SSH integration support. + +@author: Jp Calderone +""" + +from zope.interface import implementer + +from twisted.conch import avatar, interfaces as iconch, error as econch +from twisted.conch.ssh import factory, session +from twisted.python import components + +from twisted.conch.insults import insults + + +class _Glue: + """ + A feeble class for making one attribute look like another. + + This should be replaced with a real class at some point, probably. + Try not to write new code that uses it. + """ + def __init__(self, **kw): + self.__dict__.update(kw) + + + def __getattr__(self, name): + raise AttributeError(self.name, "has no attribute", name) + + + +class TerminalSessionTransport: + def __init__(self, proto, chainedProtocol, avatar, width, height): + self.proto = proto + self.avatar = avatar + self.chainedProtocol = chainedProtocol + + protoSession = self.proto.session + + self.proto.makeConnection( + _Glue(write=self.chainedProtocol.dataReceived, + loseConnection=lambda: avatar.conn.sendClose(protoSession), + name="SSH Proto Transport")) + + def loseConnection(): + self.proto.loseConnection() + + self.chainedProtocol.makeConnection( + _Glue(write=self.proto.write, + loseConnection=loseConnection, + name="Chained Proto Transport")) + + # XXX TODO + # chainedProtocol is supposed to be an ITerminalTransport, + # maybe. That means perhaps its terminalProtocol attribute is + # an ITerminalProtocol, it could be. So calling terminalSize + # on that should do the right thing But it'd be nice to clean + # this bit up. + self.chainedProtocol.terminalProtocol.terminalSize(width, height) + + + +@implementer(iconch.ISession) +class TerminalSession(components.Adapter): + transportFactory = TerminalSessionTransport + chainedProtocolFactory = insults.ServerProtocol + + def getPty(self, term, windowSize, attrs): + self.height, self.width = windowSize[:2] + + + def openShell(self, proto): + self.transportFactory( + proto, self.chainedProtocolFactory(), + iconch.IConchUser(self.original), + self.width, self.height) + + + def execCommand(self, proto, cmd): + raise econch.ConchError("Cannot execute commands") + + + def closed(self): + pass + + + +class TerminalUser(avatar.ConchUser, components.Adapter): + def __init__(self, original, avatarId): + components.Adapter.__init__(self, original) + avatar.ConchUser.__init__(self) + self.channelLookup[b'session'] = session.SSHSession + + + +class TerminalRealm: + userFactory = TerminalUser + sessionFactory = TerminalSession + + transportFactory = TerminalSessionTransport + chainedProtocolFactory = insults.ServerProtocol + + def _getAvatar(self, avatarId): + comp = components.Componentized() + user = self.userFactory(comp, avatarId) + sess = self.sessionFactory(comp) + + sess.transportFactory = self.transportFactory + sess.chainedProtocolFactory = self.chainedProtocolFactory + + comp.setComponent(iconch.IConchUser, user) + comp.setComponent(iconch.ISession, sess) + + return user + + + def __init__(self, transportFactory=None): + if transportFactory is not None: + self.transportFactory = transportFactory + + + def requestAvatar(self, avatarId, mind, *interfaces): + for i in interfaces: + if i is iconch.IConchUser: + return (iconch.IConchUser, + self._getAvatar(avatarId), + lambda: None) + raise NotImplementedError() + + + +class ConchFactory(factory.SSHFactory): + publicKeys = {} + privateKeys = {} + + def __init__(self, portal): + self.portal = portal diff --git a/contrib/python/Twisted/py2/twisted/conch/manhole_tap.py b/contrib/python/Twisted/py2/twisted/conch/manhole_tap.py new file mode 100644 index 00000000000..b5fb78b60ca --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/manhole_tap.py @@ -0,0 +1,165 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +TAP plugin for creating telnet- and ssh-accessible manhole servers. + +@author: Jp Calderone +""" + +from zope.interface import implementer + +from twisted.internet import protocol +from twisted.application import service, strports +from twisted.cred import portal, checkers +from twisted.python import usage, filepath + +from twisted.conch import manhole, manhole_ssh, telnet +from twisted.conch.insults import insults +from twisted.conch.ssh import keys + + + +class makeTelnetProtocol: + def __init__(self, portal): + self.portal = portal + + def __call__(self): + auth = telnet.AuthenticatingTelnetProtocol + args = (self.portal,) + return telnet.TelnetTransport(auth, *args) + + + +class chainedProtocolFactory: + def __init__(self, namespace): + self.namespace = namespace + + def __call__(self): + return insults.ServerProtocol(manhole.ColoredManhole, self.namespace) + + + +@implementer(portal.IRealm) +class _StupidRealm: + def __init__(self, proto, *a, **kw): + self.protocolFactory = proto + self.protocolArgs = a + self.protocolKwArgs = kw + + def requestAvatar(self, avatarId, *interfaces): + if telnet.ITelnetProtocol in interfaces: + return (telnet.ITelnetProtocol, + self.protocolFactory(*self.protocolArgs, + **self.protocolKwArgs), + lambda: None) + raise NotImplementedError() + + + +class Options(usage.Options): + optParameters = [ + ["telnetPort", "t", None, + ("strports description of the address on which to listen for telnet " + "connections")], + ["sshPort", "s", None, + ("strports description of the address on which to listen for ssh " + "connections")], + ["passwd", "p", "/etc/passwd", + "name of a passwd(5)-format username/password file"], + ["sshKeyDir", None, "", + "Directory where the autogenerated SSH key is kept."], + ["sshKeyName", None, "server.key", + "Filename of the autogenerated SSH key."], + ["sshKeySize", None, 4096, + "Size of the automatically generated SSH key."], + ] + + def __init__(self): + usage.Options.__init__(self) + self['namespace'] = None + + def postOptions(self): + if self['telnetPort'] is None and self['sshPort'] is None: + raise usage.UsageError( + "At least one of --telnetPort and --sshPort must be specified") + + + +def makeService(options): + """ + Create a manhole server service. + + @type options: L{dict} + @param options: A mapping describing the configuration of + the desired service. Recognized key/value pairs are:: + + "telnetPort": strports description of the address on which + to listen for telnet connections. If None, + no telnet service will be started. + + "sshPort": strports description of the address on which to + listen for ssh connections. If None, no ssh + service will be started. + + "namespace": dictionary containing desired initial locals + for manhole connections. If None, an empty + dictionary will be used. + + "passwd": Name of a passwd(5)-format username/password file. + + "sshKeyDir": The folder that the SSH server key will be kept in. + + "sshKeyName": The filename of the key. + + "sshKeySize": The size of the key, in bits. Default is 4096. + + @rtype: L{twisted.application.service.IService} + @return: A manhole service. + """ + svc = service.MultiService() + + namespace = options['namespace'] + if namespace is None: + namespace = {} + + checker = checkers.FilePasswordDB(options['passwd']) + + if options['telnetPort']: + telnetRealm = _StupidRealm(telnet.TelnetBootstrapProtocol, + insults.ServerProtocol, + manhole.ColoredManhole, + namespace) + + telnetPortal = portal.Portal(telnetRealm, [checker]) + + telnetFactory = protocol.ServerFactory() + telnetFactory.protocol = makeTelnetProtocol(telnetPortal) + telnetService = strports.service(options['telnetPort'], + telnetFactory) + telnetService.setServiceParent(svc) + + if options['sshPort']: + sshRealm = manhole_ssh.TerminalRealm() + sshRealm.chainedProtocolFactory = chainedProtocolFactory(namespace) + + sshPortal = portal.Portal(sshRealm, [checker]) + sshFactory = manhole_ssh.ConchFactory(sshPortal) + + if options['sshKeyDir'] != "": + keyDir = options['sshKeyDir'] + else: + from twisted.python._appdirs import getDataDirectory + keyDir = getDataDirectory() + + keyLocation = filepath.FilePath(keyDir).child(options['sshKeyName']) + + sshKey = keys._getPersistentRSAKey(keyLocation, + int(options['sshKeySize'])) + sshFactory.publicKeys[b"ssh-rsa"] = sshKey + sshFactory.privateKeys[b"ssh-rsa"] = sshKey + + sshService = strports.service(options['sshPort'], sshFactory) + sshService.setServiceParent(svc) + + return svc diff --git a/contrib/python/Twisted/py2/twisted/conch/mixin.py b/contrib/python/Twisted/py2/twisted/conch/mixin.py new file mode 100644 index 00000000000..976e9ad18f6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/mixin.py @@ -0,0 +1,55 @@ +# -*- test-case-name: twisted.conch.test.test_mixin -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Experimental optimization + +This module provides a single mixin class which allows protocols to +collapse numerous small writes into a single larger one. + +@author: Jp Calderone +""" + +from twisted.internet import reactor + +class BufferingMixin: + """ + Mixin which adds write buffering. + """ + _delayedWriteCall = None + data = None + + DELAY = 0.0 + + def schedule(self): + return reactor.callLater(self.DELAY, self.flush) + + + def reschedule(self, token): + token.reset(self.DELAY) + + + def write(self, data): + """ + Buffer some bytes to be written soon. + + Every call to this function delays the real write by C{self.DELAY} + seconds. When the delay expires, all collected bytes are written + to the underlying transport using L{ITransport.writeSequence}. + """ + if self._delayedWriteCall is None: + self.data = [] + self._delayedWriteCall = self.schedule() + else: + self.reschedule(self._delayedWriteCall) + self.data.append(data) + + + def flush(self): + """ + Flush the buffer immediately. + """ + self._delayedWriteCall = None + self.transport.writeSequence(self.data) + self.data = None diff --git a/contrib/python/Twisted/py2/twisted/conch/openssh_compat/__init__.py b/contrib/python/Twisted/py2/twisted/conch/openssh_compat/__init__.py new file mode 100644 index 00000000000..69d5927d1fe --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/openssh_compat/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + +""" +Support for OpenSSH configuration files. + +Maintainer: Paul Swartz +""" + diff --git a/contrib/python/Twisted/py2/twisted/conch/openssh_compat/factory.py b/contrib/python/Twisted/py2/twisted/conch/openssh_compat/factory.py new file mode 100644 index 00000000000..eeea8232262 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/openssh_compat/factory.py @@ -0,0 +1,72 @@ +# -*- test-case-name: twisted.conch.test.test_openssh_compat -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Factory for reading openssh configuration files: public keys, private keys, and +moduli file. +""" + +import os, errno + +from twisted.python import log +from twisted.python.util import runAsEffectiveUser + +from twisted.conch.ssh import keys, factory, common +from twisted.conch.openssh_compat import primes + + + +class OpenSSHFactory(factory.SSHFactory): + dataRoot = '/usr/local/etc' + # For openbsd which puts moduli in a different directory from keys. + moduliRoot = '/usr/local/etc' + + + def getPublicKeys(self): + """ + Return the server public keys. + """ + ks = {} + for filename in os.listdir(self.dataRoot): + if filename[:9] == 'ssh_host_' and filename[-8:]=='_key.pub': + try: + k = keys.Key.fromFile( + os.path.join(self.dataRoot, filename)) + t = common.getNS(k.blob())[0] + ks[t] = k + except Exception as e: + log.msg('bad public key file %s: %s' % (filename, e)) + return ks + + + def getPrivateKeys(self): + """ + Return the server private keys. + """ + privateKeys = {} + for filename in os.listdir(self.dataRoot): + if filename[:9] == 'ssh_host_' and filename[-4:]=='_key': + fullPath = os.path.join(self.dataRoot, filename) + try: + key = keys.Key.fromFile(fullPath) + except IOError as e: + if e.errno == errno.EACCES: + # Not allowed, let's switch to root + key = runAsEffectiveUser( + 0, 0, keys.Key.fromFile, fullPath) + privateKeys[key.sshType()] = key + else: + raise + except Exception as e: + log.msg('bad private key file %s: %s' % (filename, e)) + else: + privateKeys[key.sshType()] = key + return privateKeys + + + def getPrimes(self): + try: + return primes.parseModuliFile(self.moduliRoot+'/moduli') + except IOError: + return None diff --git a/contrib/python/Twisted/py2/twisted/conch/openssh_compat/primes.py b/contrib/python/Twisted/py2/twisted/conch/openssh_compat/primes.py new file mode 100644 index 00000000000..79cc7ff1251 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/openssh_compat/primes.py @@ -0,0 +1,30 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + +""" +Parsing for the moduli file, which contains Diffie-Hellman prime groups. + +Maintainer: Paul Swartz +""" + +from twisted.python.compat import long + + +def parseModuliFile(filename): + with open(filename) as f: + lines = f.readlines() + primes = {} + for l in lines: + l = l.strip() + if not l or l[0]=='#': + continue + tim, typ, tst, tri, size, gen, mod = l.split() + size = int(size) + 1 + gen = long(gen) + mod = long(mod, 16) + if size not in primes: + primes[size] = [] + primes[size].append((gen, mod)) + return primes diff --git a/contrib/python/Twisted/py2/twisted/conch/recvline.py b/contrib/python/Twisted/py2/twisted/conch/recvline.py new file mode 100644 index 00000000000..f7801b720ee --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/recvline.py @@ -0,0 +1,374 @@ +# -*- test-case-name: twisted.conch.test.test_recvline -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Basic line editing support. + +@author: Jp Calderone +""" + +import string + +from zope.interface import implementer + +from twisted.conch.insults import insults, helper + +from twisted.python import log, reflect +from twisted.python.compat import iterbytes + +_counters = {} +class Logging(object): + """ + Wrapper which logs attribute lookups. + + This was useful in debugging something, I guess. I forget what. + It can probably be deleted or moved somewhere more appropriate. + Nothing special going on here, really. + """ + def __init__(self, original): + self.original = original + key = reflect.qual(original.__class__) + count = _counters.get(key, 0) + _counters[key] = count + 1 + self._logFile = open(key + '-' + str(count), 'w') + + + def __str__(self): + return str(super(Logging, self).__getattribute__('original')) + + + def __repr__(self): + return repr(super(Logging, self).__getattribute__('original')) + + + def __getattribute__(self, name): + original = super(Logging, self).__getattribute__('original') + logFile = super(Logging, self).__getattribute__('_logFile') + logFile.write(name + '\n') + return getattr(original, name) + + + +@implementer(insults.ITerminalTransport) +class TransportSequence(object): + """ + An L{ITerminalTransport} implementation which forwards calls to + one or more other L{ITerminalTransport}s. + + This is a cheap way for servers to keep track of the state they + expect the client to see, since all terminal manipulations can be + send to the real client and to a terminal emulator that lives in + the server process. + """ + + for keyID in (b'UP_ARROW', b'DOWN_ARROW', b'RIGHT_ARROW', b'LEFT_ARROW', + b'HOME', b'INSERT', b'DELETE', b'END', b'PGUP', b'PGDN', + b'F1', b'F2', b'F3', b'F4', b'F5', b'F6', b'F7', b'F8', + b'F9', b'F10', b'F11', b'F12'): + execBytes = keyID + b" = object()" + execStr = execBytes.decode("ascii") + exec(execStr) + + TAB = b'\t' + BACKSPACE = b'\x7f' + + def __init__(self, *transports): + assert transports, ( + "Cannot construct a TransportSequence with no transports") + self.transports = transports + + for method in insults.ITerminalTransport: + exec("""\ +def %s(self, *a, **kw): + for tpt in self.transports: + result = tpt.%s(*a, **kw) + return result +""" % (method, method)) + + + +class LocalTerminalBufferMixin(object): + """ + A mixin for RecvLine subclasses which records the state of the terminal. + + This is accomplished by performing all L{ITerminalTransport} operations on both + the transport passed to makeConnection and an instance of helper.TerminalBuffer. + + @ivar terminalCopy: A L{helper.TerminalBuffer} instance which efforts + will be made to keep up to date with the actual terminal + associated with this protocol instance. + """ + + def makeConnection(self, transport): + self.terminalCopy = helper.TerminalBuffer() + self.terminalCopy.connectionMade() + return super(LocalTerminalBufferMixin, self).makeConnection( + TransportSequence(transport, self.terminalCopy)) + + + def __str__(self): + return str(self.terminalCopy) + + + +class RecvLine(insults.TerminalProtocol): + """ + L{TerminalProtocol} which adds line editing features. + + Clients will be prompted for lines of input with all the usual + features: character echoing, left and right arrow support for + moving the cursor to different areas of the line buffer, backspace + and delete for removing characters, and insert for toggling + between typeover and insert mode. Tabs will be expanded to enough + spaces to move the cursor to the next tabstop (every four + characters by default). Enter causes the line buffer to be + cleared and the line to be passed to the lineReceived() method + which, by default, does nothing. Subclasses are responsible for + redrawing the input prompt (this will probably change). + """ + width = 80 + height = 24 + + TABSTOP = 4 + + ps = (b'>>> ', b'... ') + pn = 0 + _printableChars = string.printable.encode("ascii") + + def connectionMade(self): + # A list containing the characters making up the current line + self.lineBuffer = [] + + # A zero-based (wtf else?) index into self.lineBuffer. + # Indicates the current cursor position. + self.lineBufferIndex = 0 + + t = self.terminal + # A map of keyIDs to bound instance methods. + self.keyHandlers = { + t.LEFT_ARROW: self.handle_LEFT, + t.RIGHT_ARROW: self.handle_RIGHT, + t.TAB: self.handle_TAB, + + # Both of these should not be necessary, but figuring out + # which is necessary is a huge hassle. + b'\r': self.handle_RETURN, + b'\n': self.handle_RETURN, + + t.BACKSPACE: self.handle_BACKSPACE, + t.DELETE: self.handle_DELETE, + t.INSERT: self.handle_INSERT, + t.HOME: self.handle_HOME, + t.END: self.handle_END} + + self.initializeScreen() + + + def initializeScreen(self): + # Hmm, state sucks. Oh well. + # For now we will just take over the whole terminal. + self.terminal.reset() + self.terminal.write(self.ps[self.pn]) + # XXX Note: I would prefer to default to starting in insert + # mode, however this does not seem to actually work! I do not + # know why. This is probably of interest to implementors + # subclassing RecvLine. + + # XXX XXX Note: But the unit tests all expect the initial mode + # to be insert right now. Fuck, there needs to be a way to + # query the current mode or something. + # self.setTypeoverMode() + self.setInsertMode() + + + def currentLineBuffer(self): + s = b''.join(self.lineBuffer) + return s[:self.lineBufferIndex], s[self.lineBufferIndex:] + + + def setInsertMode(self): + self.mode = 'insert' + self.terminal.setModes([insults.modes.IRM]) + + + def setTypeoverMode(self): + self.mode = 'typeover' + self.terminal.resetModes([insults.modes.IRM]) + + + def drawInputLine(self): + """ + Write a line containing the current input prompt and the current line + buffer at the current cursor position. + """ + self.terminal.write(self.ps[self.pn] + b''.join(self.lineBuffer)) + + + def terminalSize(self, width, height): + # XXX - Clear the previous input line, redraw it at the new + # cursor position + self.terminal.eraseDisplay() + self.terminal.cursorHome() + self.width = width + self.height = height + self.drawInputLine() + + + def unhandledControlSequence(self, seq): + pass + + + def keystrokeReceived(self, keyID, modifier): + m = self.keyHandlers.get(keyID) + if m is not None: + m() + elif keyID in self._printableChars: + self.characterReceived(keyID, False) + else: + log.msg("Received unhandled keyID: %r" % (keyID,)) + + + def characterReceived(self, ch, moreCharactersComing): + if self.mode == 'insert': + self.lineBuffer.insert(self.lineBufferIndex, ch) + else: + self.lineBuffer[self.lineBufferIndex:self.lineBufferIndex+1] = [ch] + self.lineBufferIndex += 1 + self.terminal.write(ch) + + + def handle_TAB(self): + n = self.TABSTOP - (len(self.lineBuffer) % self.TABSTOP) + self.terminal.cursorForward(n) + self.lineBufferIndex += n + self.lineBuffer.extend(iterbytes(b' ' * n)) + + + def handle_LEFT(self): + if self.lineBufferIndex > 0: + self.lineBufferIndex -= 1 + self.terminal.cursorBackward() + + + def handle_RIGHT(self): + if self.lineBufferIndex < len(self.lineBuffer): + self.lineBufferIndex += 1 + self.terminal.cursorForward() + + + def handle_HOME(self): + if self.lineBufferIndex: + self.terminal.cursorBackward(self.lineBufferIndex) + self.lineBufferIndex = 0 + + + def handle_END(self): + offset = len(self.lineBuffer) - self.lineBufferIndex + if offset: + self.terminal.cursorForward(offset) + self.lineBufferIndex = len(self.lineBuffer) + + + def handle_BACKSPACE(self): + if self.lineBufferIndex > 0: + self.lineBufferIndex -= 1 + del self.lineBuffer[self.lineBufferIndex] + self.terminal.cursorBackward() + self.terminal.deleteCharacter() + + + def handle_DELETE(self): + if self.lineBufferIndex < len(self.lineBuffer): + del self.lineBuffer[self.lineBufferIndex] + self.terminal.deleteCharacter() + + + def handle_RETURN(self): + line = b''.join(self.lineBuffer) + self.lineBuffer = [] + self.lineBufferIndex = 0 + self.terminal.nextLine() + self.lineReceived(line) + + + def handle_INSERT(self): + assert self.mode in ('typeover', 'insert') + if self.mode == 'typeover': + self.setInsertMode() + else: + self.setTypeoverMode() + + + def lineReceived(self, line): + pass + + + +class HistoricRecvLine(RecvLine): + """ + L{TerminalProtocol} which adds both basic line-editing features and input history. + + Everything supported by L{RecvLine} is also supported by this class. In addition, the + up and down arrows traverse the input history. Each received line is automatically + added to the end of the input history. + """ + def connectionMade(self): + RecvLine.connectionMade(self) + + self.historyLines = [] + self.historyPosition = 0 + + t = self.terminal + self.keyHandlers.update({t.UP_ARROW: self.handle_UP, + t.DOWN_ARROW: self.handle_DOWN}) + + + def currentHistoryBuffer(self): + b = tuple(self.historyLines) + return b[:self.historyPosition], b[self.historyPosition:] + + + def _deliverBuffer(self, buf): + if buf: + for ch in iterbytes(buf[:-1]): + self.characterReceived(ch, True) + self.characterReceived(buf[-1:], False) + + + def handle_UP(self): + if self.lineBuffer and self.historyPosition == len(self.historyLines): + self.historyLines.append(b''.join(self.lineBuffer)) + if self.historyPosition > 0: + self.handle_HOME() + self.terminal.eraseToLineEnd() + + self.historyPosition -= 1 + self.lineBuffer = [] + + self._deliverBuffer(self.historyLines[self.historyPosition]) + + + def handle_DOWN(self): + if self.historyPosition < len(self.historyLines) - 1: + self.handle_HOME() + self.terminal.eraseToLineEnd() + + self.historyPosition += 1 + self.lineBuffer = [] + + self._deliverBuffer(self.historyLines[self.historyPosition]) + else: + self.handle_HOME() + self.terminal.eraseToLineEnd() + + self.historyPosition = len(self.historyLines) + self.lineBuffer = [] + self.lineBufferIndex = 0 + + + def handle_RETURN(self): + if self.lineBuffer: + self.historyLines.append(b''.join(self.lineBuffer)) + self.historyPosition = len(self.historyLines) + return RecvLine.handle_RETURN(self) diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/__init__.py b/contrib/python/Twisted/py2/twisted/conch/ssh/__init__.py new file mode 100644 index 00000000000..4b7f024b996 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + +""" +An SSHv2 implementation for Twisted. Part of the Twisted.Conch package. + +Maintainer: Paul Swartz +""" diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/_kex.py b/contrib/python/Twisted/py2/twisted/conch/ssh/_kex.py new file mode 100644 index 00000000000..922cf8b6a1e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/_kex.py @@ -0,0 +1,294 @@ +# -*- test-case-name: twisted.conch.test.test_transport -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +SSH key exchange handling. +""" + +from __future__ import absolute_import, division + +from hashlib import sha1, sha256, sha384, sha512 + +from zope.interface import Attribute, implementer, Interface + +from twisted.conch import error +from twisted.python.compat import long + + +class _IKexAlgorithm(Interface): + """ + An L{_IKexAlgorithm} describes a key exchange algorithm. + """ + + preference = Attribute( + "An L{int} giving the preference of the algorithm when negotiating " + "key exchange. Algorithms with lower precedence values are more " + "preferred.") + + hashProcessor = Attribute( + "A callable hash algorithm constructor (e.g. C{hashlib.sha256}) " + "suitable for use with this key exchange algorithm.") + + + +class _IFixedGroupKexAlgorithm(_IKexAlgorithm): + """ + An L{_IFixedGroupKexAlgorithm} describes a key exchange algorithm with a + fixed prime / generator group. + """ + + prime = Attribute( + "A L{long} giving the prime number used in Diffie-Hellman key " + "exchange, or L{None} if not applicable.") + + generator = Attribute( + "A L{long} giving the generator number used in Diffie-Hellman key " + "exchange, or L{None} if not applicable. (This is not related to " + "Python generator functions.)") + + + +class _IEllipticCurveExchangeKexAlgorithm(_IKexAlgorithm): + """ + An L{_IEllipticCurveExchangeKexAlgorithm} describes a key exchange algorithm + that uses an elliptic curve exchange between the client and server. + """ + + + +class _IGroupExchangeKexAlgorithm(_IKexAlgorithm): + """ + An L{_IGroupExchangeKexAlgorithm} describes a key exchange algorithm + that uses group exchange between the client and server. + + A prime / generator group should be chosen at run time based on the + requested size. See RFC 4419. + """ + + + +@implementer(_IEllipticCurveExchangeKexAlgorithm) +class _Curve25519SHA256(object): + """ + Elliptic Curve Key Exchange using Curve25519 and SHA256. Defined in + U{https://datatracker.ietf.org/doc/draft-ietf-curdle-ssh-curves/}. + """ + preference = 1 + hashProcessor = sha256 + + + +@implementer(_IEllipticCurveExchangeKexAlgorithm) +class _Curve25519SHA256LibSSH(object): + """ + As L{_Curve25519SHA256}, but with a pre-standardized algorithm name. + """ + preference = 2 + hashProcessor = sha256 + + + +@implementer(_IEllipticCurveExchangeKexAlgorithm) +class _ECDH256(object): + """ + Elliptic Curve Key Exchange with SHA-256 as HASH. Defined in + RFC 5656. + """ + preference = 3 + hashProcessor = sha256 + + + +@implementer(_IEllipticCurveExchangeKexAlgorithm) +class _ECDH384(object): + """ + Elliptic Curve Key Exchange with SHA-384 as HASH. Defined in + RFC 5656. + """ + preference = 4 + hashProcessor = sha384 + + + +@implementer(_IEllipticCurveExchangeKexAlgorithm) +class _ECDH512(object): + """ + Elliptic Curve Key Exchange with SHA-512 as HASH. Defined in + RFC 5656. + """ + preference = 5 + hashProcessor = sha512 + + + +@implementer(_IGroupExchangeKexAlgorithm) +class _DHGroupExchangeSHA256(object): + """ + Diffie-Hellman Group and Key Exchange with SHA-256 as HASH. Defined in + RFC 4419, 4.2. + """ + + preference = 6 + hashProcessor = sha256 + + + +@implementer(_IGroupExchangeKexAlgorithm) +class _DHGroupExchangeSHA1(object): + """ + Diffie-Hellman Group and Key Exchange with SHA-1 as HASH. Defined in + RFC 4419, 4.1. + """ + + preference = 7 + hashProcessor = sha1 + + + +@implementer(_IFixedGroupKexAlgorithm) +class _DHGroup14SHA1(object): + """ + Diffie-Hellman key exchange with SHA-1 as HASH and Oakley Group 14 + (2048-bit MODP Group). Defined in RFC 4253, 8.2. + """ + + preference = 8 + hashProcessor = sha1 + # Diffie-Hellman primes from Oakley Group 14 (RFC 3526, 3). + prime = long('32317006071311007300338913926423828248817941241140239112842' + '00975140074170663435422261968941736356934711790173790970419175460587' + '32091950288537589861856221532121754125149017745202702357960782362488' + '84246189477587641105928646099411723245426622522193230540919037680524' + '23551912567971587011700105805587765103886184728025797605490356973256' + '15261670813393617995413364765591603683178967290731783845896806396719' + '00977202194168647225871031411336429319536193471636533209717077448227' + '98858856536920864529663607725026895550592836275112117409697299806841' + '05543595848665832916421362182310789909994486524682624169720359118525' + '07045361090559') + generator = 2 + + + +# Which ECDH hash function to use is dependent on the size. +_kexAlgorithms = { + b"curve25519-sha256": _Curve25519SHA256(), + b"curve25519-sha256@libssh.org": _Curve25519SHA256LibSSH(), + b"diffie-hellman-group-exchange-sha256": _DHGroupExchangeSHA256(), + b"diffie-hellman-group-exchange-sha1": _DHGroupExchangeSHA1(), + b"diffie-hellman-group14-sha1": _DHGroup14SHA1(), + b"ecdh-sha2-nistp256": _ECDH256(), + b"ecdh-sha2-nistp384": _ECDH384(), + b"ecdh-sha2-nistp521": _ECDH512(), + } + + + +def getKex(kexAlgorithm): + """ + Get a description of a named key exchange algorithm. + + @param kexAlgorithm: The key exchange algorithm name. + @type kexAlgorithm: L{bytes} + + @return: A description of the key exchange algorithm named by + C{kexAlgorithm}. + @rtype: L{_IKexAlgorithm} + + @raises ConchError: if the key exchange algorithm is not found. + """ + if kexAlgorithm not in _kexAlgorithms: + raise error.ConchError( + "Unsupported key exchange algorithm: %s" % (kexAlgorithm,)) + return _kexAlgorithms[kexAlgorithm] + + + +def isEllipticCurve(kexAlgorithm): + """ + Returns C{True} if C{kexAlgorithm} is an elliptic curve. + + @param kexAlgorithm: The key exchange algorithm name. + @type kexAlgorithm: C{str} + + @return: C{True} if C{kexAlgorithm} is an elliptic curve, + otherwise C{False}. + @rtype: C{bool} + """ + return _IEllipticCurveExchangeKexAlgorithm.providedBy(getKex(kexAlgorithm)) + + + +def isFixedGroup(kexAlgorithm): + """ + Returns C{True} if C{kexAlgorithm} has a fixed prime / generator group. + + @param kexAlgorithm: The key exchange algorithm name. + @type kexAlgorithm: L{bytes} + + @return: C{True} if C{kexAlgorithm} has a fixed prime / generator group, + otherwise C{False}. + @rtype: L{bool} + """ + return _IFixedGroupKexAlgorithm.providedBy(getKex(kexAlgorithm)) + + + +def getHashProcessor(kexAlgorithm): + """ + Get the hash algorithm callable to use in key exchange. + + @param kexAlgorithm: The key exchange algorithm name. + @type kexAlgorithm: L{bytes} + + @return: A callable hash algorithm constructor (e.g. C{hashlib.sha256}). + @rtype: C{callable} + """ + kex = getKex(kexAlgorithm) + return kex.hashProcessor + + + +def getDHGeneratorAndPrime(kexAlgorithm): + """ + Get the generator and the prime to use in key exchange. + + @param kexAlgorithm: The key exchange algorithm name. + @type kexAlgorithm: L{bytes} + + @return: A L{tuple} containing L{long} generator and L{long} prime. + @rtype: L{tuple} + """ + kex = getKex(kexAlgorithm) + return kex.generator, kex.prime + + + +def getSupportedKeyExchanges(): + """ + Get a list of supported key exchange algorithm names in order of + preference. + + @return: A C{list} of supported key exchange algorithm names. + @rtype: C{list} of L{bytes} + """ + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.asymmetric import ec + from twisted.conch.ssh.keys import _curveTable + + backend = default_backend() + kexAlgorithms = _kexAlgorithms.copy() + for keyAlgorithm in list(kexAlgorithms): + if keyAlgorithm.startswith(b"ecdh"): + keyAlgorithmDsa = keyAlgorithm.replace(b"ecdh", b"ecdsa") + supported = backend.elliptic_curve_exchange_algorithm_supported( + ec.ECDH(), _curveTable[keyAlgorithmDsa]) + elif keyAlgorithm.startswith(b"curve25519-sha256"): + supported = backend.x25519_supported() + else: + supported = True + if not supported: + kexAlgorithms.pop(keyAlgorithm) + return sorted( + kexAlgorithms, + key=lambda kexAlgorithm: kexAlgorithms[kexAlgorithm].preference) diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/address.py b/contrib/python/Twisted/py2/twisted/conch/ssh/address.py new file mode 100644 index 00000000000..969740268c7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/address.py @@ -0,0 +1,47 @@ +# -*- test-case-name: twisted.conch.test.test_address -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Address object for SSH network connections. + +Maintainer: Paul Swartz + +@since: 12.1 +""" + +from __future__ import division, absolute_import + +from zope.interface import implementer + +from twisted.internet.interfaces import IAddress +from twisted.python import util + + + +@implementer(IAddress) +class SSHTransportAddress(util.FancyEqMixin, object): + """ + Object representing an SSH Transport endpoint. + + This is used to ensure that any code inspecting this address and + attempting to construct a similar connection based upon it is not + mislead into creating a transport which is not similar to the one it is + indicating. + + @ivar address: An instance of an object which implements I{IAddress} to + which this transport address is connected. + """ + + compareAttributes = ('address',) + + def __init__(self, address): + self.address = address + + + def __repr__(self): + return 'SSHTransportAddress(%r)' % (self.address,) + + + def __hash__(self): + return hash(('SSH', self.address)) diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/agent.py b/contrib/python/Twisted/py2/twisted/conch/ssh/agent.py new file mode 100644 index 00000000000..03c0de80681 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/agent.py @@ -0,0 +1,296 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implements the SSH v2 key agent protocol. This protocol is documented in the +SSH source code, in the file +U{PROTOCOL.agent}. + +Maintainer: Paul Swartz +""" + +from __future__ import absolute_import, division + +import struct + +from twisted.conch.ssh.common import NS, getNS, getMP +from twisted.conch.error import ConchError, MissingKeyStoreError +from twisted.conch.ssh import keys +from twisted.internet import defer, protocol +from twisted.python.compat import itervalues + + + +class SSHAgentClient(protocol.Protocol): + """ + The client side of the SSH agent protocol. This is equivalent to + ssh-add(1) and can be used with either ssh-agent(1) or the SSHAgentServer + protocol, also in this package. + """ + + def __init__(self): + self.buf = b'' + self.deferreds = [] + + + def dataReceived(self, data): + self.buf += data + while 1: + if len(self.buf) <= 4: + return + packLen = struct.unpack('!L', self.buf[:4])[0] + if len(self.buf) < 4 + packLen: + return + packet, self.buf = self.buf[4:4 + packLen], self.buf[4 + packLen:] + reqType = ord(packet[0:1]) + d = self.deferreds.pop(0) + if reqType == AGENT_FAILURE: + d.errback(ConchError('agent failure')) + elif reqType == AGENT_SUCCESS: + d.callback(b'') + else: + d.callback(packet) + + + def sendRequest(self, reqType, data): + pack = struct.pack('!LB',len(data) + 1, reqType) + data + self.transport.write(pack) + d = defer.Deferred() + self.deferreds.append(d) + return d + + + def requestIdentities(self): + """ + @return: A L{Deferred} which will fire with a list of all keys found in + the SSH agent. The list of keys is comprised of (public key blob, + comment) tuples. + """ + d = self.sendRequest(AGENTC_REQUEST_IDENTITIES, b'') + d.addCallback(self._cbRequestIdentities) + return d + + + def _cbRequestIdentities(self, data): + """ + Unpack a collection of identities into a list of tuples comprised of + public key blobs and comments. + """ + if ord(data[0:1]) != AGENT_IDENTITIES_ANSWER: + raise ConchError('unexpected response: %i' % ord(data[0:1])) + numKeys = struct.unpack('!L', data[1:5])[0] + result = [] + data = data[5:] + for i in range(numKeys): + blob, data = getNS(data) + comment, data = getNS(data) + result.append((blob, comment)) + return result + + + def addIdentity(self, blob, comment = b''): + """ + Add a private key blob to the agent's collection of keys. + """ + req = blob + req += NS(comment) + return self.sendRequest(AGENTC_ADD_IDENTITY, req) + + + def signData(self, blob, data): + """ + Request that the agent sign the given C{data} with the private key + which corresponds to the public key given by C{blob}. The private + key should have been added to the agent already. + + @type blob: L{bytes} + @type data: L{bytes} + @return: A L{Deferred} which fires with a signature for given data + created with the given key. + """ + req = NS(blob) + req += NS(data) + req += b'\000\000\000\000' # flags + return self.sendRequest(AGENTC_SIGN_REQUEST, req).addCallback(self._cbSignData) + + + def _cbSignData(self, data): + if ord(data[0:1]) != AGENT_SIGN_RESPONSE: + raise ConchError('unexpected data: %i' % ord(data[0:1])) + signature = getNS(data[1:])[0] + return signature + + + def removeIdentity(self, blob): + """ + Remove the private key corresponding to the public key in blob from the + running agent. + """ + req = NS(blob) + return self.sendRequest(AGENTC_REMOVE_IDENTITY, req) + + + def removeAllIdentities(self): + """ + Remove all keys from the running agent. + """ + return self.sendRequest(AGENTC_REMOVE_ALL_IDENTITIES, b'') + + + +class SSHAgentServer(protocol.Protocol): + """ + The server side of the SSH agent protocol. This is equivalent to + ssh-agent(1) and can be used with either ssh-add(1) or the SSHAgentClient + protocol, also in this package. + """ + + def __init__(self): + self.buf = b'' + + + def dataReceived(self, data): + self.buf += data + while 1: + if len(self.buf) <= 4: + return + packLen = struct.unpack('!L', self.buf[:4])[0] + if len(self.buf) < 4 + packLen: + return + packet, self.buf = self.buf[4:4 + packLen], self.buf[4 + packLen:] + reqType = ord(packet[0:1]) + reqName = messages.get(reqType, None) + if not reqName: + self.sendResponse(AGENT_FAILURE, b'') + else: + f = getattr(self, 'agentc_%s' % reqName) + if getattr(self.factory, 'keys', None) is None: + self.sendResponse(AGENT_FAILURE, b'') + raise MissingKeyStoreError() + f(packet[1:]) + + + def sendResponse(self, reqType, data): + pack = struct.pack('!LB', len(data) + 1, reqType) + data + self.transport.write(pack) + + + def agentc_REQUEST_IDENTITIES(self, data): + """ + Return all of the identities that have been added to the server + """ + assert data == b'' + numKeys = len(self.factory.keys) + resp = [] + + resp.append(struct.pack('!L', numKeys)) + for key, comment in itervalues(self.factory.keys): + resp.append(NS(key.blob())) # yes, wrapped in an NS + resp.append(NS(comment)) + self.sendResponse(AGENT_IDENTITIES_ANSWER, b''.join(resp)) + + + def agentc_SIGN_REQUEST(self, data): + """ + Data is a structure with a reference to an already added key object and + some data that the clients wants signed with that key. If the key + object wasn't loaded, return AGENT_FAILURE, else return the signature. + """ + blob, data = getNS(data) + if blob not in self.factory.keys: + return self.sendResponse(AGENT_FAILURE, b'') + signData, data = getNS(data) + assert data == b'\000\000\000\000' + self.sendResponse(AGENT_SIGN_RESPONSE, NS(self.factory.keys[blob][0].sign(signData))) + + + def agentc_ADD_IDENTITY(self, data): + """ + Adds a private key to the agent's collection of identities. On + subsequent interactions, the private key can be accessed using only the + corresponding public key. + """ + + # need to pre-read the key data so we can get past it to the comment string + keyType, rest = getNS(data) + if keyType == b'ssh-rsa': + nmp = 6 + elif keyType == b'ssh-dss': + nmp = 5 + else: + raise keys.BadKeyError('unknown blob type: %s' % keyType) + + rest = getMP(rest, nmp)[-1] # ignore the key data for now, we just want the comment + comment, rest = getNS(rest) # the comment, tacked onto the end of the key blob + + k = keys.Key.fromString(data, type='private_blob') # not wrapped in NS here + self.factory.keys[k.blob()] = (k, comment) + self.sendResponse(AGENT_SUCCESS, b'') + + + def agentc_REMOVE_IDENTITY(self, data): + """ + Remove a specific key from the agent's collection of identities. + """ + blob, _ = getNS(data) + k = keys.Key.fromString(blob, type='blob') + del self.factory.keys[k.blob()] + self.sendResponse(AGENT_SUCCESS, b'') + + + def agentc_REMOVE_ALL_IDENTITIES(self, data): + """ + Remove all keys from the agent's collection of identities. + """ + assert data == b'' + self.factory.keys = {} + self.sendResponse(AGENT_SUCCESS, b'') + + # v1 messages that we ignore because we don't keep v1 keys + # open-ssh sends both v1 and v2 commands, so we have to + # do no-ops for v1 commands or we'll get "bad request" errors + + def agentc_REQUEST_RSA_IDENTITIES(self, data): + """ + v1 message for listing RSA1 keys; superseded by + agentc_REQUEST_IDENTITIES, which handles different key types. + """ + self.sendResponse(AGENT_RSA_IDENTITIES_ANSWER, struct.pack('!L', 0)) + + + def agentc_REMOVE_RSA_IDENTITY(self, data): + """ + v1 message for removing RSA1 keys; superseded by + agentc_REMOVE_IDENTITY, which handles different key types. + """ + self.sendResponse(AGENT_SUCCESS, b'') + + + def agentc_REMOVE_ALL_RSA_IDENTITIES(self, data): + """ + v1 message for removing all RSA1 keys; superseded by + agentc_REMOVE_ALL_IDENTITIES, which handles different key types. + """ + self.sendResponse(AGENT_SUCCESS, b'') + + +AGENTC_REQUEST_RSA_IDENTITIES = 1 +AGENT_RSA_IDENTITIES_ANSWER = 2 +AGENT_FAILURE = 5 +AGENT_SUCCESS = 6 + +AGENTC_REMOVE_RSA_IDENTITY = 8 +AGENTC_REMOVE_ALL_RSA_IDENTITIES = 9 + +AGENTC_REQUEST_IDENTITIES = 11 +AGENT_IDENTITIES_ANSWER = 12 +AGENTC_SIGN_REQUEST = 13 +AGENT_SIGN_RESPONSE = 14 +AGENTC_ADD_IDENTITY = 17 +AGENTC_REMOVE_IDENTITY = 18 +AGENTC_REMOVE_ALL_IDENTITIES = 19 + +messages = {} +for name, value in locals().copy().items(): + if name[:7] == 'AGENTC_': + messages[value] = name[7:] # doesn't handle doubles diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/channel.py b/contrib/python/Twisted/py2/twisted/conch/ssh/channel.py new file mode 100644 index 00000000000..51e2a2f9141 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/channel.py @@ -0,0 +1,320 @@ +# -*- test-case-name: twisted.conch.test.test_channel -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +The parent class for all the SSH Channels. Currently implemented channels +are session, direct-tcp, and forwarded-tcp. + +Maintainer: Paul Swartz +""" + +from __future__ import division, absolute_import + +from zope.interface import implementer + +from twisted.python import log +from twisted.python.compat import nativeString, intToBytes +from twisted.internet import interfaces + + + +@implementer(interfaces.ITransport) +class SSHChannel(log.Logger): + """ + A class that represents a multiplexed channel over an SSH connection. + The channel has a local window which is the maximum amount of data it will + receive, and a remote which is the maximum amount of data the remote side + will accept. There is also a maximum packet size for any individual data + packet going each way. + + @ivar name: the name of the channel. + @type name: L{bytes} + @ivar localWindowSize: the maximum size of the local window in bytes. + @type localWindowSize: L{int} + @ivar localWindowLeft: how many bytes are left in the local window. + @type localWindowLeft: L{int} + @ivar localMaxPacket: the maximum size of packet we will accept in bytes. + @type localMaxPacket: L{int} + @ivar remoteWindowLeft: how many bytes are left in the remote window. + @type remoteWindowLeft: L{int} + @ivar remoteMaxPacket: the maximum size of a packet the remote side will + accept in bytes. + @type remoteMaxPacket: L{int} + @ivar conn: the connection this channel is multiplexed through. + @type conn: L{SSHConnection} + @ivar data: any data to send to the other side when the channel is + requested. + @type data: L{bytes} + @ivar avatar: an avatar for the logged-in user (if a server channel) + @ivar localClosed: True if we aren't accepting more data. + @type localClosed: L{bool} + @ivar remoteClosed: True if the other side isn't accepting more data. + @type remoteClosed: L{bool} + """ + + name = None # only needed for client channels + + def __init__(self, localWindow = 0, localMaxPacket = 0, + remoteWindow = 0, remoteMaxPacket = 0, + conn = None, data=None, avatar = None): + self.localWindowSize = localWindow or 131072 + self.localWindowLeft = self.localWindowSize + self.localMaxPacket = localMaxPacket or 32768 + self.remoteWindowLeft = remoteWindow + self.remoteMaxPacket = remoteMaxPacket + self.areWriting = 1 + self.conn = conn + self.data = data + self.avatar = avatar + self.specificData = b'' + self.buf = b'' + self.extBuf = [] + self.closing = 0 + self.localClosed = 0 + self.remoteClosed = 0 + self.id = None # gets set later by SSHConnection + + + def __str__(self): + return nativeString(self.__bytes__()) + + + def __bytes__(self): + """ + Return a byte string representation of the channel + """ + name = self.name + if not name: + name = b'None' + + return (b'') + + + def logPrefix(self): + id = (self.id is not None and str(self.id)) or "unknown" + name = self.name + if name: + name = nativeString(name) + return "SSHChannel %s (%s) on %s" % (name, id, + self.conn.logPrefix()) + + + def channelOpen(self, specificData): + """ + Called when the channel is opened. specificData is any data that the + other side sent us when opening the channel. + + @type specificData: L{bytes} + """ + log.msg('channel open') + + + def openFailed(self, reason): + """ + Called when the open failed for some reason. + reason.desc is a string descrption, reason.code the SSH error code. + + @type reason: L{error.ConchError} + """ + log.msg('other side refused open\nreason: %s'% reason) + + + def addWindowBytes(self, data): + """ + Called when bytes are added to the remote window. By default it clears + the data buffers. + + @type data: L{bytes} + """ + self.remoteWindowLeft = self.remoteWindowLeft+data + if not self.areWriting and not self.closing: + self.areWriting = True + self.startWriting() + if self.buf: + b = self.buf + self.buf = b'' + self.write(b) + if self.extBuf: + b = self.extBuf + self.extBuf = [] + for (type, data) in b: + self.writeExtended(type, data) + + + def requestReceived(self, requestType, data): + """ + Called when a request is sent to this channel. By default it delegates + to self.request_. + If this function returns true, the request succeeded, otherwise it + failed. + + @type requestType: L{bytes} + @type data: L{bytes} + @rtype: L{bool} + """ + foo = nativeString(requestType.replace(b'-', b'_')) + f = getattr(self, 'request_%s'%foo, None) + if f: + return f(data) + log.msg('unhandled request for %s'%requestType) + return 0 + + + def dataReceived(self, data): + """ + Called when we receive data. + + @type data: L{bytes} + """ + log.msg('got data %s'%repr(data)) + + + def extReceived(self, dataType, data): + """ + Called when we receive extended data (usually standard error). + + @type dataType: L{int} + @type data: L{str} + """ + log.msg('got extended data %s %s'%(dataType, repr(data))) + + + def eofReceived(self): + """ + Called when the other side will send no more data. + """ + log.msg('remote eof') + + + def closeReceived(self): + """ + Called when the other side has closed the channel. + """ + log.msg('remote close') + self.loseConnection() + + + def closed(self): + """ + Called when the channel is closed. This means that both our side and + the remote side have closed the channel. + """ + log.msg('closed') + + + def write(self, data): + """ + Write some data to the channel. If there is not enough remote window + available, buffer until it is. Otherwise, split the data into + packets of length remoteMaxPacket and send them. + + @type data: L{bytes} + """ + if self.buf: + self.buf += data + return + top = len(data) + if top > self.remoteWindowLeft: + data, self.buf = (data[:self.remoteWindowLeft], + data[self.remoteWindowLeft:]) + self.areWriting = 0 + self.stopWriting() + top = self.remoteWindowLeft + rmp = self.remoteMaxPacket + write = self.conn.sendData + r = range(0, top, rmp) + for offset in r: + write(self, data[offset: offset+rmp]) + self.remoteWindowLeft -= top + if self.closing and not self.buf: + self.loseConnection() # try again + + + def writeExtended(self, dataType, data): + """ + Send extended data to this channel. If there is not enough remote + window available, buffer until there is. Otherwise, split the data + into packets of length remoteMaxPacket and send them. + + @type dataType: L{int} + @type data: L{bytes} + """ + if self.extBuf: + if self.extBuf[-1][0] == dataType: + self.extBuf[-1][1] += data + else: + self.extBuf.append([dataType, data]) + return + if len(data) > self.remoteWindowLeft: + data, self.extBuf = (data[:self.remoteWindowLeft], + [[dataType, data[self.remoteWindowLeft:]]]) + self.areWriting = 0 + self.stopWriting() + while len(data) > self.remoteMaxPacket: + self.conn.sendExtendedData(self, dataType, + data[:self.remoteMaxPacket]) + data = data[self.remoteMaxPacket:] + self.remoteWindowLeft -= self.remoteMaxPacket + if data: + self.conn.sendExtendedData(self, dataType, data) + self.remoteWindowLeft -= len(data) + if self.closing: + self.loseConnection() # try again + + + def writeSequence(self, data): + """ + Part of the Transport interface. Write a list of strings to the + channel. + + @type data: C{list} of L{str} + """ + self.write(b''.join(data)) + + + def loseConnection(self): + """ + Close the channel if there is no buferred data. Otherwise, note the + request and return. + """ + self.closing = 1 + if not self.buf and not self.extBuf: + self.conn.sendClose(self) + + + def getPeer(self): + """ + See: L{ITransport.getPeer} + + @return: The remote address of this connection. + @rtype: L{SSHTransportAddress}. + """ + return self.conn.transport.getPeer() + + + def getHost(self): + """ + See: L{ITransport.getHost} + + @return: An address describing this side of the connection. + @rtype: L{SSHTransportAddress}. + """ + return self.conn.transport.getHost() + + + def stopWriting(self): + """ + Called when the remote buffer is full, as a hint to stop writing. + This can be ignored, but it can be helpful. + """ + + + def startWriting(self): + """ + Called when the remote buffer has more room, as a hint to continue + writing. + """ diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/common.py b/contrib/python/Twisted/py2/twisted/conch/ssh/common.py new file mode 100644 index 00000000000..8a0f136c360 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/common.py @@ -0,0 +1,93 @@ +# -*- test-case-name: twisted.conch.test.test_ssh -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Common functions for the SSH classes. + +Maintainer: Paul Swartz +""" + +from __future__ import absolute_import, division + +import struct + +from cryptography.utils import int_from_bytes, int_to_bytes + +from twisted.python.compat import unicode +from twisted.python.deprecate import deprecated +from twisted.python.versions import Version + +__all__ = ["NS", "getNS", "MP", "getMP", "ffs"] + + + +def NS(t): + """ + net string + """ + if isinstance(t, unicode): + t = t.encode("utf-8") + return struct.pack('!L', len(t)) + t + + + +def getNS(s, count=1): + """ + get net string + """ + ns = [] + c = 0 + for i in range(count): + l, = struct.unpack('!L', s[c:c + 4]) + ns.append(s[c + 4:4 + l + c]) + c += 4 + l + return tuple(ns) + (s[c:],) + + + +def MP(number): + if number == 0: + return b'\000' * 4 + assert number > 0 + bn = int_to_bytes(number) + if ord(bn[0:1]) & 128: + bn = b'\000' + bn + return struct.pack('>L', len(bn)) + bn + + + +def getMP(data, count=1): + """ + Get multiple precision integer out of the string. A multiple precision + integer is stored as a 4-byte length followed by length bytes of the + integer. If count is specified, get count integers out of the string. + The return value is a tuple of count integers followed by the rest of + the data. + """ + mp = [] + c = 0 + for i in range(count): + length, = struct.unpack('>L', data[c:c + 4]) + mp.append(int_from_bytes(data[c + 4:c + 4 + length], 'big')) + c += 4 + length + return tuple(mp) + (data[c:],) + + + +def ffs(c, s): + """ + first from second + goes through the first list, looking for items in the second, returns the first one + """ + for i in c: + if i in s: + return i + + + +@deprecated(Version("Twisted", 16, 5, 0)) +def install(): + # This used to install gmpy, but is technically public API, so just do + # nothing. + pass diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/connection.py b/contrib/python/Twisted/py2/twisted/conch/ssh/connection.py new file mode 100644 index 00000000000..16ef6444a05 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/connection.py @@ -0,0 +1,653 @@ +# -*- test-case-name: twisted.conch.test.test_connection -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module contains the implementation of the ssh-connection service, which +allows access to the shell and port-forwarding. + +Maintainer: Paul Swartz +""" +from __future__ import division, absolute_import + +import string +import struct + +import twisted.internet.error +from twisted.conch.ssh import service, common +from twisted.conch import error +from twisted.internet import defer +from twisted.python import log +from twisted.python.compat import ( + nativeString, networkString, long, _bytesChr as chr) + + + +class SSHConnection(service.SSHService): + """ + An implementation of the 'ssh-connection' service. It is used to + multiplex multiple channels over the single SSH connection. + + @ivar localChannelID: the next number to use as a local channel ID. + @type localChannelID: L{int} + @ivar channels: a L{dict} mapping a local channel ID to C{SSHChannel} + subclasses. + @type channels: L{dict} + @ivar localToRemoteChannel: a L{dict} mapping a local channel ID to a + remote channel ID. + @type localToRemoteChannel: L{dict} + @ivar channelsToRemoteChannel: a L{dict} mapping a C{SSHChannel} subclass + to remote channel ID. + @type channelsToRemoteChannel: L{dict} + @ivar deferreds: a L{dict} mapping a local channel ID to a C{list} of + C{Deferreds} for outstanding channel requests. Also, the 'global' + key stores the C{list} of pending global request C{Deferred}s. + """ + name = b'ssh-connection' + + def __init__(self): + self.localChannelID = 0 # this is the current # to use for channel ID + self.localToRemoteChannel = {} # local channel ID -> remote channel ID + self.channels = {} # local channel ID -> subclass of SSHChannel + self.channelsToRemoteChannel = {} # subclass of SSHChannel -> + # remote channel ID + self.deferreds = {"global": []} # local channel -> list of deferreds + # for pending requests or 'global' -> list of + # deferreds for global requests + self.transport = None # gets set later + + + def serviceStarted(self): + if hasattr(self.transport, 'avatar'): + self.transport.avatar.conn = self + + + def serviceStopped(self): + """ + Called when the connection is stopped. + """ + # Close any fully open channels + for channel in list(self.channelsToRemoteChannel.keys()): + self.channelClosed(channel) + # Indicate failure to any channels that were in the process of + # opening but not yet open. + while self.channels: + (_, channel) = self.channels.popitem() + log.callWithLogger(channel, channel.openFailed, + twisted.internet.error.ConnectionLost()) + # Errback any unfinished global requests. + self._cleanupGlobalDeferreds() + + + def _cleanupGlobalDeferreds(self): + """ + All pending requests that have returned a deferred must be errbacked + when this service is stopped, otherwise they might be left uncalled and + uncallable. + """ + for d in self.deferreds["global"]: + d.errback(error.ConchError("Connection stopped.")) + del self.deferreds["global"][:] + + + # packet methods + def ssh_GLOBAL_REQUEST(self, packet): + """ + The other side has made a global request. Payload:: + string request type + bool want reply + + + This dispatches to self.gotGlobalRequest. + """ + requestType, rest = common.getNS(packet) + wantReply, rest = ord(rest[0:1]), rest[1:] + ret = self.gotGlobalRequest(requestType, rest) + if wantReply: + reply = MSG_REQUEST_FAILURE + data = b'' + if ret: + reply = MSG_REQUEST_SUCCESS + if isinstance(ret, (tuple, list)): + data = ret[1] + self.transport.sendPacket(reply, data) + + def ssh_REQUEST_SUCCESS(self, packet): + """ + Our global request succeeded. Get the appropriate Deferred and call + it back with the packet we received. + """ + log.msg('RS') + self.deferreds['global'].pop(0).callback(packet) + + def ssh_REQUEST_FAILURE(self, packet): + """ + Our global request failed. Get the appropriate Deferred and errback + it with the packet we received. + """ + log.msg('RF') + self.deferreds['global'].pop(0).errback( + error.ConchError('global request failed', packet)) + + def ssh_CHANNEL_OPEN(self, packet): + """ + The other side wants to get a channel. Payload:: + string channel name + uint32 remote channel number + uint32 remote window size + uint32 remote maximum packet size + + + We get a channel from self.getChannel(), give it a local channel number + and notify the other side. Then notify the channel by calling its + channelOpen method. + """ + channelType, rest = common.getNS(packet) + senderChannel, windowSize, maxPacket = struct.unpack('>3L', rest[:12]) + packet = rest[12:] + try: + channel = self.getChannel(channelType, windowSize, maxPacket, + packet) + localChannel = self.localChannelID + self.localChannelID += 1 + channel.id = localChannel + self.channels[localChannel] = channel + self.channelsToRemoteChannel[channel] = senderChannel + self.localToRemoteChannel[localChannel] = senderChannel + self.transport.sendPacket(MSG_CHANNEL_OPEN_CONFIRMATION, + struct.pack('>4L', senderChannel, localChannel, + channel.localWindowSize, + channel.localMaxPacket)+channel.specificData) + log.callWithLogger(channel, channel.channelOpen, packet) + except Exception as e: + log.err(e, 'channel open failed') + if isinstance(e, error.ConchError): + textualInfo, reason = e.args + if isinstance(textualInfo, (int, long)): + # See #3657 and #3071 + textualInfo, reason = reason, textualInfo + else: + reason = OPEN_CONNECT_FAILED + textualInfo = "unknown failure" + self.transport.sendPacket( + MSG_CHANNEL_OPEN_FAILURE, + struct.pack('>2L', senderChannel, reason) + + common.NS(networkString(textualInfo)) + common.NS(b'')) + + def ssh_CHANNEL_OPEN_CONFIRMATION(self, packet): + """ + The other side accepted our MSG_CHANNEL_OPEN request. Payload:: + uint32 local channel number + uint32 remote channel number + uint32 remote window size + uint32 remote maximum packet size + + + Find the channel using the local channel number and notify its + channelOpen method. + """ + (localChannel, remoteChannel, windowSize, + maxPacket) = struct.unpack('>4L', packet[: 16]) + specificData = packet[16:] + channel = self.channels[localChannel] + channel.conn = self + self.localToRemoteChannel[localChannel] = remoteChannel + self.channelsToRemoteChannel[channel] = remoteChannel + channel.remoteWindowLeft = windowSize + channel.remoteMaxPacket = maxPacket + log.callWithLogger(channel, channel.channelOpen, specificData) + + def ssh_CHANNEL_OPEN_FAILURE(self, packet): + """ + The other side did not accept our MSG_CHANNEL_OPEN request. Payload:: + uint32 local channel number + uint32 reason code + string reason description + + Find the channel using the local channel number and notify it by + calling its openFailed() method. + """ + localChannel, reasonCode = struct.unpack('>2L', packet[:8]) + reasonDesc = common.getNS(packet[8:])[0] + channel = self.channels[localChannel] + del self.channels[localChannel] + channel.conn = self + reason = error.ConchError(reasonDesc, reasonCode) + log.callWithLogger(channel, channel.openFailed, reason) + + def ssh_CHANNEL_WINDOW_ADJUST(self, packet): + """ + The other side is adding bytes to its window. Payload:: + uint32 local channel number + uint32 bytes to add + + Call the channel's addWindowBytes() method to add new bytes to the + remote window. + """ + localChannel, bytesToAdd = struct.unpack('>2L', packet[:8]) + channel = self.channels[localChannel] + log.callWithLogger(channel, channel.addWindowBytes, bytesToAdd) + + def ssh_CHANNEL_DATA(self, packet): + """ + The other side is sending us data. Payload:: + uint32 local channel number + string data + + Check to make sure the other side hasn't sent too much data (more + than what's in the window, or more than the maximum packet size). If + they have, close the channel. Otherwise, decrease the available + window and pass the data to the channel's dataReceived(). + """ + localChannel, dataLength = struct.unpack('>2L', packet[:8]) + channel = self.channels[localChannel] + # XXX should this move to dataReceived to put client in charge? + if (dataLength > channel.localWindowLeft or + dataLength > channel.localMaxPacket): # more data than we want + log.callWithLogger(channel, log.msg, 'too much data') + self.sendClose(channel) + return + #packet = packet[:channel.localWindowLeft+4] + data = common.getNS(packet[4:])[0] + channel.localWindowLeft -= dataLength + if channel.localWindowLeft < channel.localWindowSize // 2: + self.adjustWindow(channel, channel.localWindowSize - \ + channel.localWindowLeft) + #log.msg('local window left: %s/%s' % (channel.localWindowLeft, + # channel.localWindowSize)) + log.callWithLogger(channel, channel.dataReceived, data) + + def ssh_CHANNEL_EXTENDED_DATA(self, packet): + """ + The other side is sending us exteneded data. Payload:: + uint32 local channel number + uint32 type code + string data + + Check to make sure the other side hasn't sent too much data (more + than what's in the window, or than the maximum packet size). If + they have, close the channel. Otherwise, decrease the available + window and pass the data and type code to the channel's + extReceived(). + """ + localChannel, typeCode, dataLength = struct.unpack('>3L', packet[:12]) + channel = self.channels[localChannel] + if (dataLength > channel.localWindowLeft or + dataLength > channel.localMaxPacket): + log.callWithLogger(channel, log.msg, 'too much extdata') + self.sendClose(channel) + return + data = common.getNS(packet[8:])[0] + channel.localWindowLeft -= dataLength + if channel.localWindowLeft < channel.localWindowSize // 2: + self.adjustWindow(channel, channel.localWindowSize - + channel.localWindowLeft) + log.callWithLogger(channel, channel.extReceived, typeCode, data) + + def ssh_CHANNEL_EOF(self, packet): + """ + The other side is not sending any more data. Payload:: + uint32 local channel number + + Notify the channel by calling its eofReceived() method. + """ + localChannel = struct.unpack('>L', packet[:4])[0] + channel = self.channels[localChannel] + log.callWithLogger(channel, channel.eofReceived) + + def ssh_CHANNEL_CLOSE(self, packet): + """ + The other side is closing its end; it does not want to receive any + more data. Payload:: + uint32 local channel number + + Notify the channnel by calling its closeReceived() method. If + the channel has also sent a close message, call self.channelClosed(). + """ + localChannel = struct.unpack('>L', packet[:4])[0] + channel = self.channels[localChannel] + log.callWithLogger(channel, channel.closeReceived) + channel.remoteClosed = True + if channel.localClosed and channel.remoteClosed: + self.channelClosed(channel) + + def ssh_CHANNEL_REQUEST(self, packet): + """ + The other side is sending a request to a channel. Payload:: + uint32 local channel number + string request name + bool want reply + + + Pass the message to the channel's requestReceived method. If the + other side wants a reply, add callbacks which will send the + reply. + """ + localChannel = struct.unpack('>L', packet[:4])[0] + requestType, rest = common.getNS(packet[4:]) + wantReply = ord(rest[0:1]) + channel = self.channels[localChannel] + d = defer.maybeDeferred(log.callWithLogger, channel, + channel.requestReceived, requestType, rest[1:]) + if wantReply: + d.addCallback(self._cbChannelRequest, localChannel) + d.addErrback(self._ebChannelRequest, localChannel) + return d + + def _cbChannelRequest(self, result, localChannel): + """ + Called back if the other side wanted a reply to a channel request. If + the result is true, send a MSG_CHANNEL_SUCCESS. Otherwise, raise + a C{error.ConchError} + + @param result: the value returned from the channel's requestReceived() + method. If it's False, the request failed. + @type result: L{bool} + @param localChannel: the local channel ID of the channel to which the + request was made. + @type localChannel: L{int} + @raises ConchError: if the result is False. + """ + if not result: + raise error.ConchError('failed request') + self.transport.sendPacket(MSG_CHANNEL_SUCCESS, struct.pack('>L', + self.localToRemoteChannel[localChannel])) + + def _ebChannelRequest(self, result, localChannel): + """ + Called if the other wisde wanted a reply to the channel requeset and + the channel request failed. + + @param result: a Failure, but it's not used. + @param localChannel: the local channel ID of the channel to which the + request was made. + @type localChannel: L{int} + """ + self.transport.sendPacket(MSG_CHANNEL_FAILURE, struct.pack('>L', + self.localToRemoteChannel[localChannel])) + + def ssh_CHANNEL_SUCCESS(self, packet): + """ + Our channel request to the other side succeeded. Payload:: + uint32 local channel number + + Get the C{Deferred} out of self.deferreds and call it back. + """ + localChannel = struct.unpack('>L', packet[:4])[0] + if self.deferreds.get(localChannel): + d = self.deferreds[localChannel].pop(0) + log.callWithLogger(self.channels[localChannel], + d.callback, '') + + def ssh_CHANNEL_FAILURE(self, packet): + """ + Our channel request to the other side failed. Payload:: + uint32 local channel number + + Get the C{Deferred} out of self.deferreds and errback it with a + C{error.ConchError}. + """ + localChannel = struct.unpack('>L', packet[:4])[0] + if self.deferreds.get(localChannel): + d = self.deferreds[localChannel].pop(0) + log.callWithLogger(self.channels[localChannel], + d.errback, + error.ConchError('channel request failed')) + + # methods for users of the connection to call + + def sendGlobalRequest(self, request, data, wantReply=0): + """ + Send a global request for this connection. Current this is only used + for remote->local TCP forwarding. + + @type request: L{bytes} + @type data: L{bytes} + @type wantReply: L{bool} + @rtype C{Deferred}/L{None} + """ + self.transport.sendPacket(MSG_GLOBAL_REQUEST, + common.NS(request) + + (wantReply and b'\xff' or b'\x00') + + data) + if wantReply: + d = defer.Deferred() + self.deferreds['global'].append(d) + return d + + def openChannel(self, channel, extra=b''): + """ + Open a new channel on this connection. + + @type channel: subclass of C{SSHChannel} + @type extra: L{bytes} + """ + log.msg('opening channel %s with %s %s'%(self.localChannelID, + channel.localWindowSize, channel.localMaxPacket)) + self.transport.sendPacket(MSG_CHANNEL_OPEN, common.NS(channel.name) + + struct.pack('>3L', self.localChannelID, + channel.localWindowSize, channel.localMaxPacket) + + extra) + channel.id = self.localChannelID + self.channels[self.localChannelID] = channel + self.localChannelID += 1 + + def sendRequest(self, channel, requestType, data, wantReply=0): + """ + Send a request to a channel. + + @type channel: subclass of C{SSHChannel} + @type requestType: L{bytes} + @type data: L{bytes} + @type wantReply: L{bool} + @rtype C{Deferred}/L{None} + """ + if channel.localClosed: + return + log.msg('sending request %r' % (requestType)) + self.transport.sendPacket(MSG_CHANNEL_REQUEST, struct.pack('>L', + self.channelsToRemoteChannel[channel]) + + common.NS(requestType)+chr(wantReply) + + data) + if wantReply: + d = defer.Deferred() + self.deferreds.setdefault(channel.id, []).append(d) + return d + + def adjustWindow(self, channel, bytesToAdd): + """ + Tell the other side that we will receive more data. This should not + normally need to be called as it is managed automatically. + + @type channel: subclass of L{SSHChannel} + @type bytesToAdd: L{int} + """ + if channel.localClosed: + return # we're already closed + self.transport.sendPacket(MSG_CHANNEL_WINDOW_ADJUST, struct.pack('>2L', + self.channelsToRemoteChannel[channel], + bytesToAdd)) + log.msg('adding %i to %i in channel %i' % (bytesToAdd, + channel.localWindowLeft, channel.id)) + channel.localWindowLeft += bytesToAdd + + def sendData(self, channel, data): + """ + Send data to a channel. This should not normally be used: instead use + channel.write(data) as it manages the window automatically. + + @type channel: subclass of L{SSHChannel} + @type data: L{bytes} + """ + if channel.localClosed: + return # we're already closed + self.transport.sendPacket(MSG_CHANNEL_DATA, struct.pack('>L', + self.channelsToRemoteChannel[channel]) + + common.NS(data)) + + def sendExtendedData(self, channel, dataType, data): + """ + Send extended data to a channel. This should not normally be used: + instead use channel.writeExtendedData(data, dataType) as it manages + the window automatically. + + @type channel: subclass of L{SSHChannel} + @type dataType: L{int} + @type data: L{bytes} + """ + if channel.localClosed: + return # we're already closed + self.transport.sendPacket(MSG_CHANNEL_EXTENDED_DATA, struct.pack('>2L', + self.channelsToRemoteChannel[channel],dataType) \ + + common.NS(data)) + + def sendEOF(self, channel): + """ + Send an EOF (End of File) for a channel. + + @type channel: subclass of L{SSHChannel} + """ + if channel.localClosed: + return # we're already closed + log.msg('sending eof') + self.transport.sendPacket(MSG_CHANNEL_EOF, struct.pack('>L', + self.channelsToRemoteChannel[channel])) + + def sendClose(self, channel): + """ + Close a channel. + + @type channel: subclass of L{SSHChannel} + """ + if channel.localClosed: + return # we're already closed + log.msg('sending close %i' % channel.id) + self.transport.sendPacket(MSG_CHANNEL_CLOSE, struct.pack('>L', + self.channelsToRemoteChannel[channel])) + channel.localClosed = True + if channel.localClosed and channel.remoteClosed: + self.channelClosed(channel) + + # methods to override + def getChannel(self, channelType, windowSize, maxPacket, data): + """ + The other side requested a channel of some sort. + channelType is the type of channel being requested, + windowSize is the initial size of the remote window, + maxPacket is the largest packet we should send, + data is any other packet data (often nothing). + + We return a subclass of L{SSHChannel}. + + By default, this dispatches to a method 'channel_channelType' with any + non-alphanumerics in the channelType replace with _'s. If it cannot + find a suitable method, it returns an OPEN_UNKNOWN_CHANNEL_TYPE error. + The method is called with arguments of windowSize, maxPacket, data. + + @type channelType: L{bytes} + @type windowSize: L{int} + @type maxPacket: L{int} + @type data: L{bytes} + @rtype: subclass of L{SSHChannel}/L{tuple} + """ + log.msg('got channel %r request' % (channelType)) + if hasattr(self.transport, "avatar"): # this is a server! + chan = self.transport.avatar.lookupChannel(channelType, + windowSize, + maxPacket, + data) + else: + channelType = channelType.translate(TRANSLATE_TABLE) + attr = 'channel_%s' % nativeString(channelType) + f = getattr(self, attr, None) + if f is not None: + chan = f(windowSize, maxPacket, data) + else: + chan = None + if chan is None: + raise error.ConchError('unknown channel', + OPEN_UNKNOWN_CHANNEL_TYPE) + else: + chan.conn = self + return chan + + def gotGlobalRequest(self, requestType, data): + """ + We got a global request. pretty much, this is just used by the client + to request that we forward a port from the server to the client. + Returns either: + - 1: request accepted + - 1, : request accepted with request specific data + - 0: request denied + + By default, this dispatches to a method 'global_requestType' with + -'s in requestType replaced with _'s. The found method is passed data. + If this method cannot be found, this method returns 0. Otherwise, it + returns the return value of that method. + + @type requestType: L{bytes} + @type data: L{bytes} + @rtype: L{int}/L{tuple} + """ + log.msg('got global %s request' % requestType) + if hasattr(self.transport, 'avatar'): # this is a server! + return self.transport.avatar.gotGlobalRequest(requestType, data) + + requestType = nativeString(requestType.replace(b'-',b'_')) + f = getattr(self, 'global_%s' % requestType, None) + if not f: + return 0 + return f(data) + + def channelClosed(self, channel): + """ + Called when a channel is closed. + It clears the local state related to the channel, and calls + channel.closed(). + MAKE SURE YOU CALL THIS METHOD, even if you subclass L{SSHConnection}. + If you don't, things will break mysteriously. + + @type channel: L{SSHChannel} + """ + if channel in self.channelsToRemoteChannel: # actually open + channel.localClosed = channel.remoteClosed = True + del self.localToRemoteChannel[channel.id] + del self.channels[channel.id] + del self.channelsToRemoteChannel[channel] + for d in self.deferreds.pop(channel.id, []): + d.errback(error.ConchError("Channel closed.")) + log.callWithLogger(channel, channel.closed) + + + +MSG_GLOBAL_REQUEST = 80 +MSG_REQUEST_SUCCESS = 81 +MSG_REQUEST_FAILURE = 82 +MSG_CHANNEL_OPEN = 90 +MSG_CHANNEL_OPEN_CONFIRMATION = 91 +MSG_CHANNEL_OPEN_FAILURE = 92 +MSG_CHANNEL_WINDOW_ADJUST = 93 +MSG_CHANNEL_DATA = 94 +MSG_CHANNEL_EXTENDED_DATA = 95 +MSG_CHANNEL_EOF = 96 +MSG_CHANNEL_CLOSE = 97 +MSG_CHANNEL_REQUEST = 98 +MSG_CHANNEL_SUCCESS = 99 +MSG_CHANNEL_FAILURE = 100 + +OPEN_ADMINISTRATIVELY_PROHIBITED = 1 +OPEN_CONNECT_FAILED = 2 +OPEN_UNKNOWN_CHANNEL_TYPE = 3 +OPEN_RESOURCE_SHORTAGE = 4 + +EXTENDED_DATA_STDERR = 1 + +messages = {} +for name, value in locals().copy().items(): + if name[:4] == 'MSG_': + messages[value] = name # Doesn't handle doubles + +alphanums = networkString(string.ascii_letters + string.digits) +TRANSLATE_TABLE = b''.join([chr(i) in alphanums and chr(i) or b'_' + for i in range(256)]) +SSHConnection.protocolMessages = messages diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/factory.py b/contrib/python/Twisted/py2/twisted/conch/ssh/factory.py new file mode 100644 index 00000000000..16658e7c592 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/factory.py @@ -0,0 +1,123 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A Factory for SSH servers. + +See also L{twisted.conch.openssh_compat.factory} for OpenSSH compatibility. + +Maintainer: Paul Swartz +""" + +from __future__ import division, absolute_import + +from twisted.internet import protocol +from twisted.python import log + +from twisted.conch import error +from twisted.conch.ssh import (_kex, transport, userauth, connection) + +import random + + +class SSHFactory(protocol.Factory): + """ + A Factory for SSH servers. + """ + protocol = transport.SSHServerTransport + + services = { + b'ssh-userauth':userauth.SSHUserAuthServer, + b'ssh-connection':connection.SSHConnection + } + def startFactory(self): + """ + Check for public and private keys. + """ + if not hasattr(self,'publicKeys'): + self.publicKeys = self.getPublicKeys() + if not hasattr(self,'privateKeys'): + self.privateKeys = self.getPrivateKeys() + if not self.publicKeys or not self.privateKeys: + raise error.ConchError('no host keys, failing') + if not hasattr(self,'primes'): + self.primes = self.getPrimes() + + + def buildProtocol(self, addr): + """ + Create an instance of the server side of the SSH protocol. + + @type addr: L{twisted.internet.interfaces.IAddress} provider + @param addr: The address at which the server will listen. + + @rtype: L{twisted.conch.ssh.transport.SSHServerTransport} + @return: The built transport. + """ + t = protocol.Factory.buildProtocol(self, addr) + t.supportedPublicKeys = self.privateKeys.keys() + if not self.primes: + log.msg('disabling non-fixed-group key exchange algorithms ' + 'because we cannot find moduli file') + t.supportedKeyExchanges = [ + kexAlgorithm for kexAlgorithm in t.supportedKeyExchanges + if _kex.isFixedGroup(kexAlgorithm) or + _kex.isEllipticCurve(kexAlgorithm)] + return t + + + def getPublicKeys(self): + """ + Called when the factory is started to get the public portions of the + servers host keys. Returns a dictionary mapping SSH key types to + public key strings. + + @rtype: L{dict} + """ + raise NotImplementedError('getPublicKeys unimplemented') + + + def getPrivateKeys(self): + """ + Called when the factory is started to get the private portions of the + servers host keys. Returns a dictionary mapping SSH key types to + L{twisted.conch.ssh.keys.Key} objects. + + @rtype: L{dict} + """ + raise NotImplementedError('getPrivateKeys unimplemented') + + + def getPrimes(self): + """ + Called when the factory is started to get Diffie-Hellman generators and + primes to use. Returns a dictionary mapping number of bits to lists + of tuple of (generator, prime). + + @rtype: L{dict} + """ + + + def getDHPrime(self, bits): + """ + Return a tuple of (g, p) for a Diffe-Hellman process, with p being as + close to bits bits as possible. + + @type bits: L{int} + @rtype: L{tuple} + """ + primesKeys = sorted(self.primes.keys(), key=lambda i: abs(i - bits)) + realBits = primesKeys[0] + return random.choice(self.primes[realBits]) + + + def getService(self, transport, service): + """ + Return a class to use as a service for the given transport. + + @type transport: L{transport.SSHServerTransport} + @type service: L{bytes} + @rtype: subclass of L{service.SSHService} + """ + if service == b'ssh-userauth' or hasattr(transport, 'avatar'): + return self.services[service] diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/filetransfer.py b/contrib/python/Twisted/py2/twisted/conch/ssh/filetransfer.py new file mode 100644 index 00000000000..cd739e5361a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/filetransfer.py @@ -0,0 +1,1055 @@ +# -*- test-case-name: twisted.conch.test.test_filetransfer -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import division, absolute_import + +import errno +import struct +import warnings + +from zope.interface import implementer + +from twisted.conch.interfaces import ISFTPServer, ISFTPFile +from twisted.conch.ssh.common import NS, getNS +from twisted.internet import defer, protocol +from twisted.python import failure, log +from twisted.python.compat import ( + _PY3, range, itervalues, nativeString, networkString) + + + +class FileTransferBase(protocol.Protocol): + + versions = (3, ) + + packetTypes = {} + + def __init__(self): + self.buf = b'' + self.otherVersion = None # This gets set + + + def sendPacket(self, kind, data): + self.transport.write(struct.pack('!LB', len(data)+1, kind) + data) + + + def dataReceived(self, data): + self.buf += data + while len(self.buf) > 5: + length, kind = struct.unpack('!LB', self.buf[:5]) + if len(self.buf) < 4 + length: + return + data, self.buf = self.buf[5:4+length], self.buf[4+length:] + packetType = self.packetTypes.get(kind, None) + if not packetType: + log.msg('no packet type for', kind) + continue + f = getattr(self, 'packet_{}'.format(packetType), None) + if not f: + log.msg('not implemented: {}'.format(packetType)) + log.msg(repr(data[4:])) + reqId, = struct.unpack('!L', data[:4]) + self._sendStatus(reqId, FX_OP_UNSUPPORTED, + "don't understand {}".format(packetType)) + # XXX not implemented + continue + try: + f(data) + except Exception: + log.err() + continue + + + def _parseAttributes(self, data): + (flags,) = struct.unpack('!L', data[:4]) + attrs = {} + data = data[4:] + if flags & FILEXFER_ATTR_SIZE == FILEXFER_ATTR_SIZE: + (size,) = struct.unpack('!Q', data[:8]) + attrs['size'] = size + data = data[8:] + if flags & FILEXFER_ATTR_OWNERGROUP == FILEXFER_ATTR_OWNERGROUP: + uid, gid = struct.unpack('!2L', data[:8]) + attrs['uid'] = uid + attrs['gid'] = gid + data = data[8:] + if flags & FILEXFER_ATTR_PERMISSIONS == FILEXFER_ATTR_PERMISSIONS: + (perms,) = struct.unpack('!L', data[:4]) + attrs['permissions'] = perms + data = data[4:] + if flags & FILEXFER_ATTR_ACMODTIME == FILEXFER_ATTR_ACMODTIME: + atime, mtime = struct.unpack('!2L', data[:8]) + attrs['atime'] = atime + attrs['mtime'] = mtime + data = data[8:] + if flags & FILEXFER_ATTR_EXTENDED == FILEXFER_ATTR_EXTENDED: + (extendedCount,) = struct.unpack('!L', data[:4]) + data = data[4:] + for i in range(extendedCount): + (extendedType, data) = getNS(data) + (extendedData, data) = getNS(data) + attrs['ext_{}'.format(nativeString(extendedType))] = \ + extendedData + return attrs, data + + + def _packAttributes(self, attrs): + flags = 0 + data = b'' + if 'size' in attrs: + data += struct.pack('!Q', attrs['size']) + flags |= FILEXFER_ATTR_SIZE + if 'uid' in attrs and 'gid' in attrs: + data += struct.pack('!2L', attrs['uid'], attrs['gid']) + flags |= FILEXFER_ATTR_OWNERGROUP + if 'permissions' in attrs: + data += struct.pack('!L', attrs['permissions']) + flags |= FILEXFER_ATTR_PERMISSIONS + if 'atime' in attrs and 'mtime' in attrs: + data += struct.pack('!2L', attrs['atime'], attrs['mtime']) + flags |= FILEXFER_ATTR_ACMODTIME + extended = [] + for k in attrs: + if k.startswith('ext_'): + extType = NS(networkString(k[4:])) + extData = NS(attrs[k]) + extended.append(extType + extData) + if extended: + data += struct.pack('!L', len(extended)) + data += b''.join(extended) + flags |= FILEXFER_ATTR_EXTENDED + return struct.pack('!L', flags) + data + + + +class FileTransferServer(FileTransferBase): + + def __init__(self, data=None, avatar=None): + FileTransferBase.__init__(self) + self.client = ISFTPServer(avatar) # yay interfaces + self.openFiles = {} + self.openDirs = {} + + + def packet_INIT(self, data): + (version,) = struct.unpack('!L', data[:4]) + self.version = min(list(self.versions) + [version]) + data = data[4:] + ext = {} + while data: + extName, data = getNS(data) + extData, data = getNS(data) + ext[extName] = extData + ourExt = self.client.gotVersion(version, ext) + ourExtData = b"" + for (k, v) in ourExt.items(): + ourExtData += NS(k) + NS(v) + self.sendPacket(FXP_VERSION, struct.pack('!L', self.version) + + ourExtData) + + + def packet_OPEN(self, data): + requestId = data[:4] + data = data[4:] + filename, data = getNS(data) + (flags,) = struct.unpack('!L', data[:4]) + data = data[4:] + attrs, data = self._parseAttributes(data) + assert data == b'', 'still have data in OPEN: {!r}'.format(data) + d = defer.maybeDeferred(self.client.openFile, filename, flags, attrs) + d.addCallback(self._cbOpenFile, requestId) + d.addErrback(self._ebStatus, requestId, b"open failed") + + + def _cbOpenFile(self, fileObj, requestId): + fileId = networkString(str(hash(fileObj))) + if fileId in self.openFiles: + raise KeyError('id already open') + self.openFiles[fileId] = fileObj + self.sendPacket(FXP_HANDLE, requestId + NS(fileId)) + + + def packet_CLOSE(self, data): + requestId = data[:4] + data = data[4:] + handle, data = getNS(data) + assert data == b'', 'still have data in CLOSE: {!r}'.format(data) + if handle in self.openFiles: + fileObj = self.openFiles[handle] + d = defer.maybeDeferred(fileObj.close) + d.addCallback(self._cbClose, handle, requestId) + d.addErrback(self._ebStatus, requestId, b"close failed") + elif handle in self.openDirs: + dirObj = self.openDirs[handle][0] + d = defer.maybeDeferred(dirObj.close) + d.addCallback(self._cbClose, handle, requestId, 1) + d.addErrback(self._ebStatus, requestId, b"close failed") + else: + self._ebClose(failure.Failure(KeyError()), requestId) + + + def _cbClose(self, result, handle, requestId, isDir=0): + if isDir: + del self.openDirs[handle] + else: + del self.openFiles[handle] + self._sendStatus(requestId, FX_OK, b'file closed') + + + def packet_READ(self, data): + requestId = data[:4] + data = data[4:] + handle, data = getNS(data) + (offset, length), data = struct.unpack('!QL', data[:12]), data[12:] + assert data == b'', 'still have data in READ: {!r}'.format(data) + if handle not in self.openFiles: + self._ebRead(failure.Failure(KeyError()), requestId) + else: + fileObj = self.openFiles[handle] + d = defer.maybeDeferred(fileObj.readChunk, offset, length) + d.addCallback(self._cbRead, requestId) + d.addErrback(self._ebStatus, requestId, b"read failed") + + + def _cbRead(self, result, requestId): + if result == b'': # Python's read will return this for EOF + raise EOFError() + self.sendPacket(FXP_DATA, requestId + NS(result)) + + + def packet_WRITE(self, data): + requestId = data[:4] + data = data[4:] + handle, data = getNS(data) + offset, = struct.unpack('!Q', data[:8]) + data = data[8:] + writeData, data = getNS(data) + assert data == b'', 'still have data in WRITE: {!r}'.format(data) + if handle not in self.openFiles: + self._ebWrite(failure.Failure(KeyError()), requestId) + else: + fileObj = self.openFiles[handle] + d = defer.maybeDeferred(fileObj.writeChunk, offset, writeData) + d.addCallback(self._cbStatus, requestId, b"write succeeded") + d.addErrback(self._ebStatus, requestId, b"write failed") + + + def packet_REMOVE(self, data): + requestId = data[:4] + data = data[4:] + filename, data = getNS(data) + assert data == b'', 'still have data in REMOVE: {!r}'.format(data) + d = defer.maybeDeferred(self.client.removeFile, filename) + d.addCallback(self._cbStatus, requestId, b"remove succeeded") + d.addErrback(self._ebStatus, requestId, b"remove failed") + + + def packet_RENAME(self, data): + requestId = data[:4] + data = data[4:] + oldPath, data = getNS(data) + newPath, data = getNS(data) + assert data == b'', 'still have data in RENAME: {!r}'.format(data) + d = defer.maybeDeferred(self.client.renameFile, oldPath, newPath) + d.addCallback(self._cbStatus, requestId, b"rename succeeded") + d.addErrback(self._ebStatus, requestId, b"rename failed") + + + def packet_MKDIR(self, data): + requestId = data[:4] + data = data[4:] + path, data = getNS(data) + attrs, data = self._parseAttributes(data) + assert data == b'', 'still have data in MKDIR: {!r}'.format(data) + d = defer.maybeDeferred(self.client.makeDirectory, path, attrs) + d.addCallback(self._cbStatus, requestId, b"mkdir succeeded") + d.addErrback(self._ebStatus, requestId, b"mkdir failed") + + + def packet_RMDIR(self, data): + requestId = data[:4] + data = data[4:] + path, data = getNS(data) + assert data == b'', 'still have data in RMDIR: {!r}'.format(data) + d = defer.maybeDeferred(self.client.removeDirectory, path) + d.addCallback(self._cbStatus, requestId, b"rmdir succeeded") + d.addErrback(self._ebStatus, requestId, b"rmdir failed") + + + def packet_OPENDIR(self, data): + requestId = data[:4] + data = data[4:] + path, data = getNS(data) + assert data == b'', 'still have data in OPENDIR: {!r}'.format(data) + d = defer.maybeDeferred(self.client.openDirectory, path) + d.addCallback(self._cbOpenDirectory, requestId) + d.addErrback(self._ebStatus, requestId, b"opendir failed") + + + def _cbOpenDirectory(self, dirObj, requestId): + handle = networkString((str(hash(dirObj)))) + if handle in self.openDirs: + raise KeyError("already opened this directory") + self.openDirs[handle] = [dirObj, iter(dirObj)] + self.sendPacket(FXP_HANDLE, requestId + NS(handle)) + + + def packet_READDIR(self, data): + requestId = data[:4] + data = data[4:] + handle, data = getNS(data) + assert data == b'', 'still have data in READDIR: {!r}'.format(data) + if handle not in self.openDirs: + self._ebStatus(failure.Failure(KeyError()), requestId) + else: + dirObj, dirIter = self.openDirs[handle] + d = defer.maybeDeferred(self._scanDirectory, dirIter, []) + d.addCallback(self._cbSendDirectory, requestId) + d.addErrback(self._ebStatus, requestId, b"scan directory failed") + + + def _scanDirectory(self, dirIter, f): + while len(f) < 250: + try: + info = next(dirIter) + except StopIteration: + if not f: + raise EOFError + return f + if isinstance(info, defer.Deferred): + info.addCallback(self._cbScanDirectory, dirIter, f) + return + else: + f.append(info) + return f + + + def _cbScanDirectory(self, result, dirIter, f): + f.append(result) + return self._scanDirectory(dirIter, f) + + + def _cbSendDirectory(self, result, requestId): + data = b'' + for (filename, longname, attrs) in result: + data += NS(filename) + data += NS(longname) + data += self._packAttributes(attrs) + self.sendPacket(FXP_NAME, requestId + + struct.pack('!L', len(result))+data) + + + def packet_STAT(self, data, followLinks=1): + requestId = data[:4] + data = data[4:] + path, data = getNS(data) + assert data == b'', 'still have data in STAT/LSTAT: {!r}'.format(data) + d = defer.maybeDeferred(self.client.getAttrs, path, followLinks) + d.addCallback(self._cbStat, requestId) + d.addErrback(self._ebStatus, requestId, b'stat/lstat failed') + + + def packet_LSTAT(self, data): + self.packet_STAT(data, 0) + + + def packet_FSTAT(self, data): + requestId = data[:4] + data = data[4:] + handle, data = getNS(data) + assert data == b'', 'still have data in FSTAT: {!r}'.format(data) + if handle not in self.openFiles: + self._ebStatus(failure.Failure(KeyError( + '{} not in self.openFiles'.format(handle))), requestId) + else: + fileObj = self.openFiles[handle] + d = defer.maybeDeferred(fileObj.getAttrs) + d.addCallback(self._cbStat, requestId) + d.addErrback(self._ebStatus, requestId, b'fstat failed') + + + def _cbStat(self, result, requestId): + data = requestId + self._packAttributes(result) + self.sendPacket(FXP_ATTRS, data) + + + def packet_SETSTAT(self, data): + requestId = data[:4] + data = data[4:] + path, data = getNS(data) + attrs, data = self._parseAttributes(data) + if data != b'': + log.msg('WARN: still have data in SETSTAT: {!r}'.format(data)) + d = defer.maybeDeferred(self.client.setAttrs, path, attrs) + d.addCallback(self._cbStatus, requestId, b'setstat succeeded') + d.addErrback(self._ebStatus, requestId, b'setstat failed') + + + def packet_FSETSTAT(self, data): + requestId = data[:4] + data = data[4:] + handle, data = getNS(data) + attrs, data = self._parseAttributes(data) + assert data == b'', 'still have data in FSETSTAT: {!r}'.format(data) + if handle not in self.openFiles: + self._ebStatus(failure.Failure(KeyError()), requestId) + else: + fileObj = self.openFiles[handle] + d = defer.maybeDeferred(fileObj.setAttrs, attrs) + d.addCallback(self._cbStatus, requestId, b'fsetstat succeeded') + d.addErrback(self._ebStatus, requestId, b'fsetstat failed') + + + def packet_READLINK(self, data): + requestId = data[:4] + data = data[4:] + path, data = getNS(data) + assert data == b'', 'still have data in READLINK: {!r}'.format(data) + d = defer.maybeDeferred(self.client.readLink, path) + d.addCallback(self._cbReadLink, requestId) + d.addErrback(self._ebStatus, requestId, b'readlink failed') + + + def _cbReadLink(self, result, requestId): + self._cbSendDirectory([(result, b'', {})], requestId) + + + def packet_SYMLINK(self, data): + requestId = data[:4] + data = data[4:] + linkPath, data = getNS(data) + targetPath, data = getNS(data) + d = defer.maybeDeferred(self.client.makeLink, linkPath, targetPath) + d.addCallback(self._cbStatus, requestId, b'symlink succeeded') + d.addErrback(self._ebStatus, requestId, b'symlink failed') + + + def packet_REALPATH(self, data): + requestId = data[:4] + data = data[4:] + path, data = getNS(data) + assert data == b'', 'still have data in REALPATH: {!r}'.format(data) + d = defer.maybeDeferred(self.client.realPath, path) + d.addCallback(self._cbReadLink, requestId) # Same return format + d.addErrback(self._ebStatus, requestId, b'realpath failed') + + + def packet_EXTENDED(self, data): + requestId = data[:4] + data = data[4:] + extName, extData = getNS(data) + d = defer.maybeDeferred(self.client.extendedRequest, extName, extData) + d.addCallback(self._cbExtended, requestId) + d.addErrback(self._ebStatus, requestId, + b'extended ' + extName + b' failed') + + + def _cbExtended(self, data, requestId): + self.sendPacket(FXP_EXTENDED_REPLY, requestId + data) + + + def _cbStatus(self, result, requestId, msg=b"request succeeded"): + self._sendStatus(requestId, FX_OK, msg) + + + def _ebStatus(self, reason, requestId, msg=b"request failed"): + code = FX_FAILURE + message = msg + if isinstance(reason.value, (IOError, OSError)): + if reason.value.errno == errno.ENOENT: # No such file + code = FX_NO_SUCH_FILE + message = networkString(reason.value.strerror) + elif reason.value.errno == errno.EACCES: # Permission denied + code = FX_PERMISSION_DENIED + message = networkString(reason.value.strerror) + elif reason.value.errno == errno.EEXIST: + code = FX_FILE_ALREADY_EXISTS + else: + log.err(reason) + elif isinstance(reason.value, EOFError): # EOF + code = FX_EOF + if reason.value.args: + message = networkString(reason.value.args[0]) + elif isinstance(reason.value, NotImplementedError): + code = FX_OP_UNSUPPORTED + if reason.value.args: + message = networkString(reason.value.args[0]) + elif isinstance(reason.value, SFTPError): + code = reason.value.code + message = networkString(reason.value.message) + else: + log.err(reason) + self._sendStatus(requestId, code, message) + + + def _sendStatus(self, requestId, code, message, lang=b''): + """ + Helper method to send a FXP_STATUS message. + """ + data = requestId + struct.pack('!L', code) + data += NS(message) + data += NS(lang) + self.sendPacket(FXP_STATUS, data) + + + def connectionLost(self, reason): + """ + Clean all opened files and directories. + """ + for fileObj in self.openFiles.values(): + fileObj.close() + self.openFiles = {} + for (dirObj, dirIter) in self.openDirs.values(): + dirObj.close() + self.openDirs = {} + + + +class FileTransferClient(FileTransferBase): + + def __init__(self, extData={}): + """ + @param extData: a dict of extended_name : extended_data items + to be sent to the server. + """ + FileTransferBase.__init__(self) + self.extData = {} + self.counter = 0 + self.openRequests = {} # id -> Deferred + + + def connectionMade(self): + data = struct.pack('!L', max(self.versions)) + for (k, v) in itervalues(self.extData): + data += NS(k) + NS(v) + self.sendPacket(FXP_INIT, data) + + + def _sendRequest(self, msg, data): + data = struct.pack('!L', self.counter) + data + d = defer.Deferred() + self.openRequests[self.counter] = d + self.counter += 1 + self.sendPacket(msg, data) + return d + + + def _parseRequest(self, data): + (id,) = struct.unpack('!L', data[:4]) + d = self.openRequests[id] + del self.openRequests[id] + return d, data[4:] + + + def openFile(self, filename, flags, attrs): + """ + Open a file. + + This method returns a L{Deferred} that is called back with an object + that provides the L{ISFTPFile} interface. + + @type filename: L{bytes} + @param filename: a string representing the file to open. + + @param flags: an integer of the flags to open the file with, ORed together. + The flags and their values are listed at the bottom of this file. + + @param attrs: a list of attributes to open the file with. It is a + dictionary, consisting of 0 or more keys. The possible keys are:: + + size: the size of the file in bytes + uid: the user ID of the file as an integer + gid: the group ID of the file as an integer + permissions: the permissions of the file with as an integer. + the bit representation of this field is defined by POSIX. + atime: the access time of the file as seconds since the epoch. + mtime: the modification time of the file as seconds since the epoch. + ext_*: extended attributes. The server is not required to + understand this, but it may. + + NOTE: there is no way to indicate text or binary files. it is up + to the SFTP client to deal with this. + """ + data = NS(filename) + struct.pack('!L', flags) + self._packAttributes(attrs) + d = self._sendRequest(FXP_OPEN, data) + d.addCallback(self._cbOpenHandle, ClientFile, filename) + return d + + + def _cbOpenHandle(self, handle, handleClass, name): + """ + Callback invoked when an OPEN or OPENDIR request succeeds. + + @param handle: The handle returned by the server + @type handle: L{bytes} + @param handleClass: The class that will represent the + newly-opened file or directory to the user (either L{ClientFile} or + L{ClientDirectory}). + @param name: The name of the file or directory represented + by C{handle}. + @type name: L{bytes} + """ + cb = handleClass(self, handle) + cb.name = name + return cb + + + def removeFile(self, filename): + """ + Remove the given file. + + This method returns a Deferred that is called back when it succeeds. + + @type filename: L{bytes} + @param filename: the name of the file as a string. + """ + return self._sendRequest(FXP_REMOVE, NS(filename)) + + + def renameFile(self, oldpath, newpath): + """ + Rename the given file. + + This method returns a Deferred that is called back when it succeeds. + + @type oldpath: L{bytes} + @param oldpath: the current location of the file. + @type newpath: L{bytes} + @param newpath: the new file name. + """ + return self._sendRequest(FXP_RENAME, NS(oldpath)+NS(newpath)) + + + def makeDirectory(self, path, attrs): + """ + Make a directory. + + This method returns a Deferred that is called back when it is + created. + + @type path: L{bytes} + @param path: the name of the directory to create as a string. + + @param attrs: a dictionary of attributes to create the directory + with. Its meaning is the same as the attrs in the openFile method. + """ + return self._sendRequest(FXP_MKDIR, NS(path)+self._packAttributes(attrs)) + + + def removeDirectory(self, path): + """ + Remove a directory (non-recursively) + + It is an error to remove a directory that has files or directories in + it. + + This method returns a Deferred that is called back when it is removed. + + @type path: L{bytes} + @param path: the directory to remove. + """ + return self._sendRequest(FXP_RMDIR, NS(path)) + + + def openDirectory(self, path): + """ + Open a directory for scanning. + + This method returns a Deferred that is called back with an iterable + object that has a close() method. + + The close() method is called when the client is finished reading + from the directory. At this point, the iterable will no longer + be used. + + The iterable returns triples of the form (filename, longname, attrs) + or a Deferred that returns the same. The sequence must support + __getitem__, but otherwise may be any 'sequence-like' object. + + filename is the name of the file relative to the directory. + logname is an expanded format of the filename. The recommended format + is: + -rwxr-xr-x 1 mjos staff 348911 Mar 25 14:29 t-filexfer + 1234567890 123 12345678 12345678 12345678 123456789012 + + The first line is sample output, the second is the length of the field. + The fields are: permissions, link count, user owner, group owner, + size in bytes, modification time. + + attrs is a dictionary in the format of the attrs argument to openFile. + + @type path: L{bytes} + @param path: the directory to open. + """ + d = self._sendRequest(FXP_OPENDIR, NS(path)) + d.addCallback(self._cbOpenHandle, ClientDirectory, path) + return d + + + def getAttrs(self, path, followLinks=0): + """ + Return the attributes for the given path. + + This method returns a dictionary in the same format as the attrs + argument to openFile or a Deferred that is called back with same. + + @type path: L{bytes} + @param path: the path to return attributes for as a string. + @param followLinks: a boolean. if it is True, follow symbolic links + and return attributes for the real path at the base. if it is False, + return attributes for the specified path. + """ + if followLinks: m = FXP_STAT + else: m = FXP_LSTAT + return self._sendRequest(m, NS(path)) + + + def setAttrs(self, path, attrs): + """ + Set the attributes for the path. + + This method returns when the attributes are set or a Deferred that is + called back when they are. + + @type path: L{bytes} + @param path: the path to set attributes for as a string. + @param attrs: a dictionary in the same format as the attrs argument to + openFile. + """ + data = NS(path) + self._packAttributes(attrs) + return self._sendRequest(FXP_SETSTAT, data) + + + def readLink(self, path): + """ + Find the root of a set of symbolic links. + + This method returns the target of the link, or a Deferred that + returns the same. + + @type path: L{bytes} + @param path: the path of the symlink to read. + """ + d = self._sendRequest(FXP_READLINK, NS(path)) + return d.addCallback(self._cbRealPath) + + + def makeLink(self, linkPath, targetPath): + """ + Create a symbolic link. + + This method returns when the link is made, or a Deferred that + returns the same. + + @type linkPath: L{bytes} + @param linkPath: the pathname of the symlink as a string + @type targetPath: L{bytes} + @param targetPath: the path of the target of the link as a string. + """ + return self._sendRequest(FXP_SYMLINK, NS(linkPath)+NS(targetPath)) + + + def realPath(self, path): + """ + Convert any path to an absolute path. + + This method returns the absolute path as a string, or a Deferred + that returns the same. + + @type path: L{bytes} + @param path: the path to convert as a string. + """ + d = self._sendRequest(FXP_REALPATH, NS(path)) + return d.addCallback(self._cbRealPath) + + + def _cbRealPath(self, result): + name, longname, attrs = result[0] + if _PY3: + name = name.decode("utf-8") + return name + + + def extendedRequest(self, request, data): + """ + Make an extended request of the server. + + The method returns a Deferred that is called back with + the result of the extended request. + + @type request: L{bytes} + @param request: the name of the extended request to make. + @type data: L{bytes} + @param data: any other data that goes along with the request. + """ + return self._sendRequest(FXP_EXTENDED, NS(request) + data) + + + def packet_VERSION(self, data): + version, = struct.unpack('!L', data[:4]) + data = data[4:] + d = {} + while data: + k, data = getNS(data) + v, data = getNS(data) + d[k]=v + self.version = version + self.gotServerVersion(version, d) + + + def packet_STATUS(self, data): + d, data = self._parseRequest(data) + code, = struct.unpack('!L', data[:4]) + data = data[4:] + if len(data) >= 4: + msg, data = getNS(data) + if len(data) >= 4: + lang, data = getNS(data) + else: + lang = b'' + else: + msg = b'' + lang = b'' + if code == FX_OK: + d.callback((msg, lang)) + elif code == FX_EOF: + d.errback(EOFError(msg)) + elif code == FX_OP_UNSUPPORTED: + d.errback(NotImplementedError(msg)) + else: + d.errback(SFTPError(code, nativeString(msg), lang)) + + + def packet_HANDLE(self, data): + d, data = self._parseRequest(data) + handle, _ = getNS(data) + d.callback(handle) + + + def packet_DATA(self, data): + d, data = self._parseRequest(data) + d.callback(getNS(data)[0]) + + + def packet_NAME(self, data): + d, data = self._parseRequest(data) + count, = struct.unpack('!L', data[:4]) + data = data[4:] + files = [] + for i in range(count): + filename, data = getNS(data) + longname, data = getNS(data) + attrs, data = self._parseAttributes(data) + files.append((filename, longname, attrs)) + d.callback(files) + + + def packet_ATTRS(self, data): + d, data = self._parseRequest(data) + d.callback(self._parseAttributes(data)[0]) + + + def packet_EXTENDED_REPLY(self, data): + d, data = self._parseRequest(data) + d.callback(data) + + + def gotServerVersion(self, serverVersion, extData): + """ + Called when the client sends their version info. + + @param otherVersion: an integer representing the version of the SFTP + protocol they are claiming. + @param extData: a dictionary of extended_name : extended_data items. + These items are sent by the client to indicate additional features. + """ + + + +@implementer(ISFTPFile) +class ClientFile: + def __init__(self, parent, handle): + self.parent = parent + self.handle = NS(handle) + + + def close(self): + return self.parent._sendRequest(FXP_CLOSE, self.handle) + + + def readChunk(self, offset, length): + data = self.handle + struct.pack("!QL", offset, length) + return self.parent._sendRequest(FXP_READ, data) + + + def writeChunk(self, offset, chunk): + data = self.handle + struct.pack("!Q", offset) + NS(chunk) + return self.parent._sendRequest(FXP_WRITE, data) + + + def getAttrs(self): + return self.parent._sendRequest(FXP_FSTAT, self.handle) + + + def setAttrs(self, attrs): + data = self.handle + self.parent._packAttributes(attrs) + return self.parent._sendRequest(FXP_FSTAT, data) + + + +class ClientDirectory: + + def __init__(self, parent, handle): + self.parent = parent + self.handle = NS(handle) + self.filesCache = [] + + + def read(self): + return self.parent._sendRequest(FXP_READDIR, self.handle) + + + def close(self): + if self.handle is None: + return defer.succeed(None) + d = self.parent._sendRequest(FXP_CLOSE, self.handle) + self.handle = None + return d + + + def __iter__(self): + return self + + + def __next__(self): + warnings.warn( + ('Using twisted.conch.ssh.filetransfer.ClientDirectory ' + 'as an iterator was deprecated in Twisted 18.9.0.'), + category=DeprecationWarning, + stacklevel=2) + if self.filesCache: + return self.filesCache.pop(0) + if self.filesCache is None: + raise StopIteration() + d = self.read() + d.addCallbacks(self._cbReadDir, self._ebReadDir) + return d + + next = __next__ + + + def _cbReadDir(self, names): + self.filesCache = names[1:] + return names[0] + + + def _ebReadDir(self, reason): + reason.trap(EOFError) + self.filesCache = None + return failure.Failure(StopIteration()) + + + +class SFTPError(Exception): + + def __init__(self, errorCode, errorMessage, lang=''): + Exception.__init__(self) + self.code = errorCode + self._message = errorMessage + self.lang = lang + + + def message(self): + """ + A string received over the network that explains the error to a human. + """ + # Python 2.6 deprecates assigning to the 'message' attribute of an + # exception. We define this read-only property here in order to + # prevent the warning about deprecation while maintaining backwards + # compatibility with object clients that rely on the 'message' + # attribute being set correctly. See bug #3897. + return self._message + message = property(message) + + + def __str__(self): + return 'SFTPError {}: {}'.format(self.code, self.message) + + + +FXP_INIT = 1 +FXP_VERSION = 2 +FXP_OPEN = 3 +FXP_CLOSE = 4 +FXP_READ = 5 +FXP_WRITE = 6 +FXP_LSTAT = 7 +FXP_FSTAT = 8 +FXP_SETSTAT = 9 +FXP_FSETSTAT = 10 +FXP_OPENDIR = 11 +FXP_READDIR = 12 +FXP_REMOVE = 13 +FXP_MKDIR = 14 +FXP_RMDIR = 15 +FXP_REALPATH = 16 +FXP_STAT = 17 +FXP_RENAME = 18 +FXP_READLINK = 19 +FXP_SYMLINK = 20 +FXP_STATUS = 101 +FXP_HANDLE = 102 +FXP_DATA = 103 +FXP_NAME = 104 +FXP_ATTRS = 105 +FXP_EXTENDED = 200 +FXP_EXTENDED_REPLY = 201 + +FILEXFER_ATTR_SIZE = 0x00000001 +FILEXFER_ATTR_UIDGID = 0x00000002 +FILEXFER_ATTR_OWNERGROUP = FILEXFER_ATTR_UIDGID +FILEXFER_ATTR_PERMISSIONS = 0x00000004 +FILEXFER_ATTR_ACMODTIME = 0x00000008 +FILEXFER_ATTR_EXTENDED = 0x80000000 + +FILEXFER_TYPE_REGULAR = 1 +FILEXFER_TYPE_DIRECTORY = 2 +FILEXFER_TYPE_SYMLINK = 3 +FILEXFER_TYPE_SPECIAL = 4 +FILEXFER_TYPE_UNKNOWN = 5 + +FXF_READ = 0x00000001 +FXF_WRITE = 0x00000002 +FXF_APPEND = 0x00000004 +FXF_CREAT = 0x00000008 +FXF_TRUNC = 0x00000010 +FXF_EXCL = 0x00000020 +FXF_TEXT = 0x00000040 + +FX_OK = 0 +FX_EOF = 1 +FX_NO_SUCH_FILE = 2 +FX_PERMISSION_DENIED = 3 +FX_FAILURE = 4 +FX_BAD_MESSAGE = 5 +FX_NO_CONNECTION = 6 +FX_CONNECTION_LOST = 7 +FX_OP_UNSUPPORTED = 8 +FX_FILE_ALREADY_EXISTS = 11 +# http://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/ defines more +# useful error codes, but so far OpenSSH doesn't implement them. We use them +# internally for clarity, but for now define them all as FX_FAILURE to be +# compatible with existing software. +FX_NOT_A_DIRECTORY = FX_FAILURE +FX_FILE_IS_A_DIRECTORY = FX_FAILURE + + +# initialize FileTransferBase.packetTypes: +g = globals() +for name in list(g.keys()): + if name.startswith('FXP_'): + value = g[name] + FileTransferBase.packetTypes[value] = name[4:] +del g, name, value diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/forwarding.py b/contrib/python/Twisted/py2/twisted/conch/ssh/forwarding.py new file mode 100644 index 00000000000..dd61e75f649 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/forwarding.py @@ -0,0 +1,269 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module contains the implementation of the TCP forwarding, which allows +clients and servers to forward arbitrary TCP data across the connection. + +Maintainer: Paul Swartz +""" + +from __future__ import division, absolute_import + +import struct + +from twisted.internet import protocol, reactor +from twisted.internet.endpoints import HostnameEndpoint, connectProtocol +from twisted.python import log +from twisted.python.compat import _PY3, unicode + +from twisted.conch.ssh import common, channel + +class SSHListenForwardingFactory(protocol.Factory): + def __init__(self, connection, hostport, klass): + self.conn = connection + self.hostport = hostport # tuple + self.klass = klass + + def buildProtocol(self, addr): + channel = self.klass(conn = self.conn) + client = SSHForwardingClient(channel) + channel.client = client + addrTuple = (addr.host, addr.port) + channelOpenData = packOpen_direct_tcpip(self.hostport, addrTuple) + self.conn.openChannel(channel, channelOpenData) + return client + +class SSHListenForwardingChannel(channel.SSHChannel): + + def channelOpen(self, specificData): + log.msg('opened forwarding channel %s' % self.id) + if len(self.client.buf)>1: + b = self.client.buf[1:] + self.write(b) + self.client.buf = b'' + + def openFailed(self, reason): + self.closed() + + def dataReceived(self, data): + self.client.transport.write(data) + + def eofReceived(self): + self.client.transport.loseConnection() + + def closed(self): + if hasattr(self, 'client'): + log.msg('closing local forwarding channel %s' % self.id) + self.client.transport.loseConnection() + del self.client + +class SSHListenClientForwardingChannel(SSHListenForwardingChannel): + + name = b'direct-tcpip' + +class SSHListenServerForwardingChannel(SSHListenForwardingChannel): + + name = b'forwarded-tcpip' + + + +class SSHConnectForwardingChannel(channel.SSHChannel): + """ + Channel used for handling server side forwarding request. + It acts as a client for the remote forwarding destination. + + @ivar hostport: C{(host, port)} requested by client as forwarding + destination. + @type hostport: L{tuple} or a C{sequence} + + @ivar client: Protocol connected to the forwarding destination. + @type client: L{protocol.Protocol} + + @ivar clientBuf: Data received while forwarding channel is not yet + connected. + @type clientBuf: L{bytes} + + @var _reactor: Reactor used for TCP connections. + @type _reactor: A reactor. + + @ivar _channelOpenDeferred: Deferred used in testing to check the + result of C{channelOpen}. + @type _channelOpenDeferred: L{twisted.internet.defer.Deferred} + """ + _reactor = reactor + + def __init__(self, hostport, *args, **kw): + channel.SSHChannel.__init__(self, *args, **kw) + self.hostport = hostport + self.client = None + self.clientBuf = b'' + + + def channelOpen(self, specificData): + """ + See: L{channel.SSHChannel} + """ + log.msg("connecting to %s:%i" % self.hostport) + ep = HostnameEndpoint( + self._reactor, self.hostport[0], self.hostport[1]) + d = connectProtocol(ep, SSHForwardingClient(self)) + d.addCallbacks(self._setClient, self._close) + self._channelOpenDeferred = d + + def _setClient(self, client): + """ + Called when the connection was established to the forwarding + destination. + + @param client: Client protocol connected to the forwarding destination. + @type client: L{protocol.Protocol} + """ + self.client = client + log.msg("connected to %s:%i" % self.hostport) + if self.clientBuf: + self.client.transport.write(self.clientBuf) + self.clientBuf = None + if self.client.buf[1:]: + self.write(self.client.buf[1:]) + self.client.buf = b'' + + + def _close(self, reason): + """ + Called when failed to connect to the forwarding destination. + + @param reason: Reason why connection failed. + @type reason: L{twisted.python.failure.Failure} + """ + log.msg("failed to connect: %s" % reason) + self.loseConnection() + + + def dataReceived(self, data): + """ + See: L{channel.SSHChannel} + """ + if self.client: + self.client.transport.write(data) + else: + self.clientBuf += data + + + def closed(self): + """ + See: L{channel.SSHChannel} + """ + if self.client: + log.msg('closed remote forwarding channel %s' % self.id) + if self.client.channel: + self.loseConnection() + self.client.transport.loseConnection() + del self.client + + + +def openConnectForwardingClient(remoteWindow, remoteMaxPacket, data, avatar): + remoteHP, origHP = unpackOpen_direct_tcpip(data) + return SSHConnectForwardingChannel(remoteHP, + remoteWindow=remoteWindow, + remoteMaxPacket=remoteMaxPacket, + avatar=avatar) + +class SSHForwardingClient(protocol.Protocol): + + def __init__(self, channel): + self.channel = channel + self.buf = b'\000' + + def dataReceived(self, data): + if self.buf: + self.buf += data + else: + self.channel.write(data) + + def connectionLost(self, reason): + if self.channel: + self.channel.loseConnection() + self.channel = None + + +def packOpen_direct_tcpip(destination, source): + """ + Pack the data suitable for sending in a CHANNEL_OPEN packet. + + @type destination: L{tuple} + @param destination: A tuple of the (host, port) of the destination host. + + @type source: L{tuple} + @param source: A tuple of the (host, port) of the source host. + """ + (connHost, connPort) = destination + (origHost, origPort) = source + if isinstance(connHost, unicode): + connHost = connHost.encode("utf-8") + if isinstance(origHost, unicode): + origHost = origHost.encode("utf-8") + conn = common.NS(connHost) + struct.pack('>L', connPort) + orig = common.NS(origHost) + struct.pack('>L', origPort) + return conn + orig + +packOpen_forwarded_tcpip = packOpen_direct_tcpip + +def unpackOpen_direct_tcpip(data): + """Unpack the data to a usable format. + """ + connHost, rest = common.getNS(data) + if _PY3 and isinstance(connHost, bytes): + connHost = connHost.decode("utf-8") + connPort = int(struct.unpack('>L', rest[:4])[0]) + origHost, rest = common.getNS(rest[4:]) + if _PY3 and isinstance(origHost, bytes): + origHost = origHost.decode("utf-8") + origPort = int(struct.unpack('>L', rest[:4])[0]) + return (connHost, connPort), (origHost, origPort) + +unpackOpen_forwarded_tcpip = unpackOpen_direct_tcpip + + + +def packGlobal_tcpip_forward(peer): + """ + Pack the data for tcpip forwarding. + + @param peer: A tuple of the (host, port) . + @type peer: L{tuple} + """ + (host, port) = peer + return common.NS(host) + struct.pack('>L', port) + + + +def unpackGlobal_tcpip_forward(data): + host, rest = common.getNS(data) + if _PY3 and isinstance(host, bytes): + host = host.decode("utf-8") + port = int(struct.unpack('>L', rest[:4])[0]) + return host, port + +"""This is how the data -> eof -> close stuff /should/ work. + +debug3: channel 1: waiting for connection +debug1: channel 1: connected +debug1: channel 1: read<=0 rfd 7 len 0 +debug1: channel 1: read failed +debug1: channel 1: close_read +debug1: channel 1: input open -> drain +debug1: channel 1: ibuf empty +debug1: channel 1: send eof +debug1: channel 1: input drain -> closed +debug1: channel 1: rcvd eof +debug1: channel 1: output open -> drain +debug1: channel 1: obuf empty +debug1: channel 1: close_write +debug1: channel 1: output drain -> closed +debug1: channel 1: rcvd close +debug3: channel 1: will not send data after close +debug1: channel 1: send close +debug1: channel 1: is dead +""" diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/keys.py b/contrib/python/Twisted/py2/twisted/conch/ssh/keys.py new file mode 100644 index 00000000000..fcbf9d28665 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/keys.py @@ -0,0 +1,1678 @@ +# -*- test-case-name: twisted.conch.test.test_keys -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Handling of RSA, DSA, and EC keys. +""" + +from __future__ import absolute_import, division + +import binascii +import itertools + +from hashlib import md5, sha256 +import base64 +import struct +import warnings + +import bcrypt +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import dsa, rsa, padding, ec +from cryptography.hazmat.primitives.serialization import ( + load_pem_private_key, load_ssh_public_key) +from cryptography import utils + +try: + + from cryptography.hazmat.primitives.asymmetric.utils import ( + encode_dss_signature, decode_dss_signature) +except ImportError: + from cryptography.hazmat.primitives.asymmetric.utils import ( + encode_rfc6979_signature as encode_dss_signature, + decode_rfc6979_signature as decode_dss_signature) +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +from pyasn1.error import PyAsn1Error +from pyasn1.type import univ +from pyasn1.codec.ber import decoder as berDecoder +from pyasn1.codec.ber import encoder as berEncoder + +from twisted.conch.ssh import common, sexpy +from twisted.conch.ssh.common import int_from_bytes, int_to_bytes +from twisted.python import randbytes +from twisted.python.compat import ( + iterbytes, long, izip, nativeString, unicode, _PY3, + _b64decodebytes as decodebytes, _b64encodebytes as encodebytes, + _bytesChr as chr) +from twisted.python.constants import NamedConstant, Names +from twisted.python.deprecate import _mutuallyExclusiveArguments + +# Curve lookup table +_curveTable = { + b'ecdsa-sha2-nistp256': ec.SECP256R1(), + b'ecdsa-sha2-nistp384': ec.SECP384R1(), + b'ecdsa-sha2-nistp521': ec.SECP521R1(), +} + +_secToNist = { + b'secp256r1' : b'nistp256', + b'secp384r1' : b'nistp384', + b'secp521r1' : b'nistp521', +} + + + + + +class BadKeyError(Exception): + """ + Raised when a key isn't what we expected from it. + + XXX: we really need to check for bad keys + """ + + + +class EncryptedKeyError(Exception): + """ + Raised when an encrypted key is presented to fromString/fromFile without + a password. + """ + + + +class BadFingerPrintFormat(Exception): + """ + Raises when unsupported fingerprint formats are presented to fingerprint. + """ + + + +class FingerprintFormats(Names): + """ + Constants representing the supported formats of key fingerprints. + + @cvar MD5_HEX: Named constant representing fingerprint format generated + using md5[RFC1321] algorithm in hexadecimal encoding. + @type MD5_HEX: L{twisted.python.constants.NamedConstant} + + @cvar SHA256_BASE64: Named constant representing fingerprint format + generated using sha256[RFC4634] algorithm in base64 encoding + @type SHA256_BASE64: L{twisted.python.constants.NamedConstant} + """ + MD5_HEX = NamedConstant() + SHA256_BASE64 = NamedConstant() + + + +class Key(object): + """ + An object representing a key. A key can be either a public or + private key. A public key can verify a signature; a private key can + create or verify a signature. To generate a string that can be stored + on disk, use the toString method. If you have a private key, but want + the string representation of the public key, use Key.public().toString(). + """ + + @classmethod + def fromFile(cls, filename, type=None, passphrase=None): + """ + Load a key from a file. + + @param filename: The path to load key data from. + + @type type: L{str} or L{None} + @param type: A string describing the format the key data is in, or + L{None} to attempt detection of the type. + + @type passphrase: L{bytes} or L{None} + @param passphrase: The passphrase the key is encrypted with, or L{None} + if there is no encryption. + + @rtype: L{Key} + @return: The loaded key. + """ + with open(filename, 'rb') as f: + return cls.fromString(f.read(), type, passphrase) + + + @classmethod + def fromString(cls, data, type=None, passphrase=None): + """ + Return a Key object corresponding to the string data. + type is optionally the type of string, matching a _fromString_* + method. Otherwise, the _guessStringType() classmethod will be used + to guess a type. If the key is encrypted, passphrase is used as + the decryption key. + + @type data: L{bytes} + @param data: The key data. + + @type type: L{str} or L{None} + @param type: A string describing the format the key data is in, or + L{None} to attempt detection of the type. + + @type passphrase: L{bytes} or L{None} + @param passphrase: The passphrase the key is encrypted with, or L{None} + if there is no encryption. + + @rtype: L{Key} + @return: The loaded key. + """ + if isinstance(data, unicode): + data = data.encode("utf-8") + if isinstance(passphrase, unicode): + passphrase = passphrase.encode("utf-8") + if type is None: + type = cls._guessStringType(data) + if type is None: + raise BadKeyError('cannot guess the type of %r' % (data,)) + method = getattr(cls, '_fromString_%s' % (type.upper(),), None) + if method is None: + raise BadKeyError('no _fromString method for %s' % (type,)) + if method.__code__.co_argcount == 2: # No passphrase + if passphrase: + raise BadKeyError('key not encrypted') + return method(data) + else: + return method(data, passphrase) + + + @classmethod + def _fromString_BLOB(cls, blob): + """ + Return a public key object corresponding to this public key blob. + The format of a RSA public key blob is:: + string 'ssh-rsa' + integer e + integer n + + The format of a DSA public key blob is:: + string 'ssh-dss' + integer p + integer q + integer g + integer y + + The format of ECDSA-SHA2-* public key blob is:: + string 'ecdsa-sha2-[identifier]' + integer x + integer y + + identifier is the standard NIST curve name. + + @type blob: L{bytes} + @param blob: The key data. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if the key type (the first string) is unknown. + """ + keyType, rest = common.getNS(blob) + if keyType == b'ssh-rsa': + e, n, rest = common.getMP(rest, 2) + return cls( + rsa.RSAPublicNumbers(e, n).public_key(default_backend())) + elif keyType == b'ssh-dss': + p, q, g, y, rest = common.getMP(rest, 4) + return cls( + dsa.DSAPublicNumbers( + y=y, + parameter_numbers=dsa.DSAParameterNumbers( + p=p, + q=q, + g=g + ) + ).public_key(default_backend()) + ) + elif keyType in _curveTable: + return cls( + ec.EllipticCurvePublicKey.from_encoded_point( + _curveTable[keyType], common.getNS(rest, 2)[1] + ) + ) + else: + raise BadKeyError('unknown blob type: %s' % (keyType,)) + + + @classmethod + def _fromString_PRIVATE_BLOB(cls, blob): + """ + Return a private key object corresponding to this private key blob. + The blob formats are as follows: + + RSA keys:: + string 'ssh-rsa' + integer n + integer e + integer d + integer u + integer p + integer q + + DSA keys:: + string 'ssh-dss' + integer p + integer q + integer g + integer y + integer x + + EC keys:: + string 'ecdsa-sha2-[identifier]' + string identifier + string q + integer privateValue + + identifier is the standard NIST curve name. + + + @type blob: L{bytes} + @param blob: The key data. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if + * the key type (the first string) is unknown + * the curve name of an ECDSA key does not match the key type + """ + keyType, rest = common.getNS(blob) + + if keyType == b'ssh-rsa': + n, e, d, u, p, q, rest = common.getMP(rest, 6) + return cls._fromRSAComponents(n=n, e=e, d=d, p=p, q=q) + elif keyType == b'ssh-dss': + p, q, g, y, x, rest = common.getMP(rest, 5) + return cls._fromDSAComponents(y=y, g=g, p=p, q=q, x=x) + elif keyType in _curveTable: + curve = _curveTable[keyType] + curveName, q, rest = common.getNS(rest, 2) + if curveName != _secToNist[curve.name.encode('ascii')]: + raise BadKeyError('ECDSA curve name %r does not match key ' + 'type %r' % (curveName, keyType)) + privateValue, rest = common.getMP(rest) + return cls._fromECEncodedPoint( + encodedPoint=q, curve=keyType, privateValue=privateValue) + else: + raise BadKeyError('unknown blob type: %s' % (keyType,)) + + + @classmethod + def _fromString_PUBLIC_OPENSSH(cls, data): + """ + Return a public key object corresponding to this OpenSSH public key + string. The format of an OpenSSH public key string is:: + + + @type data: L{bytes} + @param data: The key data. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if the blob type is unknown. + """ + # ECDSA keys don't need base64 decoding which is required + # for RSA or DSA key. + if data.startswith(b'ecdsa-sha2'): + return cls(load_ssh_public_key(data, default_backend())) + blob = decodebytes(data.split()[1]) + return cls._fromString_BLOB(blob) + + + @classmethod + def _fromPrivateOpenSSH_v1(cls, data, passphrase): + """ + Return a private key object corresponding to this OpenSSH private key + string, in the "openssh-key-v1" format introduced in OpenSSH 6.5. + + The format of an openssh-key-v1 private key string is:: + -----BEGIN OPENSSH PRIVATE KEY----- + + -----END OPENSSH PRIVATE KEY----- + + The SSH protocol string is as described in + U{PROTOCOL.key}. + + @type data: L{bytes} + @param data: The key data. + + @type passphrase: L{bytes} or L{None} + @param passphrase: The passphrase the key is encrypted with, or L{None} + if it is not encrypted. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if + * a passphrase is provided for an unencrypted key + * the SSH protocol encoding is incorrect + @raises EncryptedKeyError: if + * a passphrase is not provided for an encrypted key + """ + lines = data.strip().splitlines() + keyList = decodebytes(b''.join(lines[1:-1])) + if not keyList.startswith(b'openssh-key-v1\0'): + raise BadKeyError('unknown OpenSSH private key format') + keyList = keyList[len(b'openssh-key-v1\0'):] + cipher, kdf, kdfOptions, rest = common.getNS(keyList, 3) + n = struct.unpack('!L', rest[:4])[0] + if n != 1: + raise BadKeyError('only OpenSSH private key files containing ' + 'a single key are supported') + # Ignore public key + _, encPrivKeyList, _ = common.getNS(rest[4:], 2) + if cipher != b'none': + if not passphrase: + raise EncryptedKeyError('Passphrase must be provided ' + 'for an encrypted key') + # Determine cipher + if cipher in (b'aes128-ctr', b'aes192-ctr', b'aes256-ctr'): + algorithmClass = algorithms.AES + blockSize = 16 + keySize = int(cipher[3:6]) // 8 + ivSize = blockSize + else: + raise BadKeyError('unknown encryption type %r' % (cipher,)) + if kdf == b'bcrypt': + salt, rest = common.getNS(kdfOptions) + rounds = struct.unpack('!L', rest[:4])[0] + decKey = bcrypt.kdf( + passphrase, salt, keySize + ivSize, rounds, + # We can only use the number of rounds that OpenSSH used. + ignore_few_rounds=True) + else: + raise BadKeyError('unknown KDF type %r' % (kdf,)) + if (len(encPrivKeyList) % blockSize) != 0: + raise BadKeyError('bad padding') + decryptor = Cipher( + algorithmClass(decKey[:keySize]), + modes.CTR(decKey[keySize:keySize + ivSize]), + backend=default_backend() + ).decryptor() + privKeyList = ( + decryptor.update(encPrivKeyList) + decryptor.finalize()) + else: + if kdf != b'none': + raise BadKeyError('private key specifies KDF %r but no ' + 'cipher' % (kdf,)) + privKeyList = encPrivKeyList + check1 = struct.unpack('!L', privKeyList[:4])[0] + check2 = struct.unpack('!L', privKeyList[4:8])[0] + if check1 != check2: + raise BadKeyError('check values do not match: %d != %d' % + (check1, check2)) + return cls._fromString_PRIVATE_BLOB(privKeyList[8:]) + + + @classmethod + def _fromPrivateOpenSSH_PEM(cls, data, passphrase): + """ + Return a private key object corresponding to this OpenSSH private key + string, in the old PEM-based format. + + The format of a PEM-based OpenSSH private key string is:: + -----BEGIN PRIVATE KEY----- + [Proc-Type: 4,ENCRYPTED + DEK-Info: DES-EDE3-CBC,] + + ------END PRIVATE KEY------ + + The ASN.1 structure of a RSA key is:: + (0, n, e, d, p, q) + + The ASN.1 structure of a DSA key is:: + (0, p, q, g, y, x) + + The ASN.1 structure of a ECDSA key is:: + (ECParameters, OID, NULL) + + @type data: L{bytes} + @param data: The key data. + + @type passphrase: L{bytes} or L{None} + @param passphrase: The passphrase the key is encrypted with, or L{None} + if it is not encrypted. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if + * a passphrase is provided for an unencrypted key + * the ASN.1 encoding is incorrect + @raises EncryptedKeyError: if + * a passphrase is not provided for an encrypted key + """ + lines = data.strip().splitlines() + kind = lines[0][11:-17] + if lines[1].startswith(b'Proc-Type: 4,ENCRYPTED'): + if not passphrase: + raise EncryptedKeyError('Passphrase must be provided ' + 'for an encrypted key') + + # Determine cipher and initialization vector + try: + _, cipherIVInfo = lines[2].split(b' ', 1) + cipher, ivdata = cipherIVInfo.rstrip().split(b',', 1) + except ValueError: + raise BadKeyError('invalid DEK-info %r' % (lines[2],)) + + if cipher in (b'AES-128-CBC', b'AES-256-CBC'): + algorithmClass = algorithms.AES + keySize = int(cipher.split(b'-')[1]) // 8 + if len(ivdata) != 32: + raise BadKeyError('AES encrypted key with a bad IV') + elif cipher == b'DES-EDE3-CBC': + algorithmClass = algorithms.TripleDES + keySize = 24 + if len(ivdata) != 16: + raise BadKeyError('DES encrypted key with a bad IV') + else: + raise BadKeyError('unknown encryption type %r' % (cipher,)) + + # Extract keyData for decoding + iv = bytes(bytearray([int(ivdata[i:i + 2], 16) + for i in range(0, len(ivdata), 2)])) + ba = md5(passphrase + iv[:8]).digest() + bb = md5(ba + passphrase + iv[:8]).digest() + decKey = (ba + bb)[:keySize] + b64Data = decodebytes(b''.join(lines[3:-1])) + + decryptor = Cipher( + algorithmClass(decKey), + modes.CBC(iv), + backend=default_backend() + ).decryptor() + keyData = decryptor.update(b64Data) + decryptor.finalize() + + removeLen = ord(keyData[-1:]) + keyData = keyData[:-removeLen] + else: + b64Data = b''.join(lines[1:-1]) + keyData = decodebytes(b64Data) + + try: + decodedKey = berDecoder.decode(keyData)[0] + except PyAsn1Error as e: + raise BadKeyError( + 'Failed to decode key (Bad Passphrase?): %s' % (e,)) + + if kind == b'EC': + return cls( + load_pem_private_key(data, passphrase, default_backend())) + + if kind == b'RSA': + if len(decodedKey) == 2: # Alternate RSA key + decodedKey = decodedKey[0] + if len(decodedKey) < 6: + raise BadKeyError('RSA key failed to decode properly') + + n, e, d, p, q, dmp1, dmq1, iqmp = [ + long(value) for value in decodedKey[1:9] + ] + return cls( + rsa.RSAPrivateNumbers( + p=p, + q=q, + d=d, + dmp1=dmp1, + dmq1=dmq1, + iqmp=iqmp, + public_numbers=rsa.RSAPublicNumbers(e=e, n=n), + ).private_key(default_backend()) + ) + elif kind == b'DSA': + p, q, g, y, x = [long(value) for value in decodedKey[1: 6]] + if len(decodedKey) < 6: + raise BadKeyError('DSA key failed to decode properly') + return cls( + dsa.DSAPrivateNumbers( + x=x, + public_numbers=dsa.DSAPublicNumbers( + y=y, + parameter_numbers=dsa.DSAParameterNumbers( + p=p, + q=q, + g=g + ) + ) + ).private_key(backend=default_backend()) + ) + else: + raise BadKeyError("unknown key type %s" % (kind,)) + + + @classmethod + def _fromString_PRIVATE_OPENSSH(cls, data, passphrase): + """ + Return a private key object corresponding to this OpenSSH private key + string. If the key is encrypted, passphrase MUST be provided. + Providing a passphrase for an unencrypted key is an error. + + @type data: L{bytes} + @param data: The key data. + + @type passphrase: L{bytes} or L{None} + @param passphrase: The passphrase the key is encrypted with, or L{None} + if it is not encrypted. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if + * a passphrase is provided for an unencrypted key + * the encoding is incorrect + @raises EncryptedKeyError: if + * a passphrase is not provided for an encrypted key + """ + if data.strip().splitlines()[0][11:-17] == b'OPENSSH': + # New-format (openssh-key-v1) key + return cls._fromPrivateOpenSSH_v1(data, passphrase) + else: + # Old-format (PEM) key + return cls._fromPrivateOpenSSH_PEM(data, passphrase) + + @classmethod + def _fromString_PUBLIC_LSH(cls, data): + """ + Return a public key corresponding to this LSH public key string. + The LSH public key string format is:: + , ()+))> + + The names for a RSA (key type 'rsa-pkcs1-sha1') key are: n, e. + The names for a DSA (key type 'dsa') key are: y, g, p, q. + + @type data: L{bytes} + @param data: The key data. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if the key type is unknown + """ + sexp = sexpy.parse(decodebytes(data[1:-1])) + assert sexp[0] == b'public-key' + kd = {} + for name, data in sexp[1][1:]: + kd[name] = common.getMP(common.NS(data))[0] + if sexp[1][0] == b'dsa': + return cls._fromDSAComponents( + y=kd[b'y'], g=kd[b'g'], p=kd[b'p'], q=kd[b'q']) + + elif sexp[1][0] == b'rsa-pkcs1-sha1': + return cls._fromRSAComponents(n=kd[b'n'], e=kd[b'e']) + else: + raise BadKeyError('unknown lsh key type %s' % (sexp[1][0],)) + + @classmethod + def _fromString_PRIVATE_LSH(cls, data): + """ + Return a private key corresponding to this LSH private key string. + The LSH private key string format is:: + , (, )+))> + + The names for a RSA (key type 'rsa-pkcs1-sha1') key are: n, e, d, p, q. + The names for a DSA (key type 'dsa') key are: y, g, p, q, x. + + @type data: L{bytes} + @param data: The key data. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if the key type is unknown + """ + sexp = sexpy.parse(data) + assert sexp[0] == b'private-key' + kd = {} + for name, data in sexp[1][1:]: + kd[name] = common.getMP(common.NS(data))[0] + if sexp[1][0] == b'dsa': + assert len(kd) == 5, len(kd) + return cls._fromDSAComponents( + y=kd[b'y'], g=kd[b'g'], p=kd[b'p'], q=kd[b'q'], x=kd[b'x']) + elif sexp[1][0] == b'rsa-pkcs1': + assert len(kd) == 8, len(kd) + if kd[b'p'] > kd[b'q']: # Make p smaller than q + kd[b'p'], kd[b'q'] = kd[b'q'], kd[b'p'] + return cls._fromRSAComponents( + n=kd[b'n'], e=kd[b'e'], d=kd[b'd'], p=kd[b'p'], q=kd[b'q']) + + else: + raise BadKeyError('unknown lsh key type %s' % (sexp[1][0],)) + + @classmethod + def _fromString_AGENTV3(cls, data): + """ + Return a private key object corresponsing to the Secure Shell Key + Agent v3 format. + + The SSH Key Agent v3 format for a RSA key is:: + string 'ssh-rsa' + integer e + integer d + integer n + integer u + integer p + integer q + + The SSH Key Agent v3 format for a DSA key is:: + string 'ssh-dss' + integer p + integer q + integer g + integer y + integer x + + @type data: L{bytes} + @param data: The key data. + + @return: A new key. + @rtype: L{twisted.conch.ssh.keys.Key} + @raises BadKeyError: if the key type (the first string) is unknown + """ + keyType, data = common.getNS(data) + if keyType == b'ssh-dss': + p, data = common.getMP(data) + q, data = common.getMP(data) + g, data = common.getMP(data) + y, data = common.getMP(data) + x, data = common.getMP(data) + return cls._fromDSAComponents(y=y, g=g, p=p, q=q, x=x) + elif keyType == b'ssh-rsa': + e, data = common.getMP(data) + d, data = common.getMP(data) + n, data = common.getMP(data) + u, data = common.getMP(data) + p, data = common.getMP(data) + q, data = common.getMP(data) + return cls._fromRSAComponents(n=n, e=e, d=d, p=p, q=q, u=u) + else: + raise BadKeyError("unknown key type %s" % (keyType,)) + + @classmethod + def _guessStringType(cls, data): + """ + Guess the type of key in data. The types map to _fromString_* + methods. + + @type data: L{bytes} + @param data: The key data. + """ + if data.startswith(b'ssh-') or data.startswith(b'ecdsa-sha2-'): + return 'public_openssh' + elif data.startswith(b'-----BEGIN'): + return 'private_openssh' + elif data.startswith(b'{'): + return 'public_lsh' + elif data.startswith(b'('): + return 'private_lsh' + elif data.startswith(b'\x00\x00\x00\x07ssh-') or data.startswith(b'\x00\x00\x00\x13ecdsa-'): + ignored, rest = common.getNS(data) + count = 0 + while rest: + count += 1 + ignored, rest = common.getMP(rest) + if count > 4: + return 'agentv3' + else: + return 'blob' + + @classmethod + def _fromRSAComponents(cls, n, e, d=None, p=None, q=None, u=None): + """ + Build a key from RSA numerical components. + + @type n: L{int} + @param n: The 'n' RSA variable. + + @type e: L{int} + @param e: The 'e' RSA variable. + + @type d: L{int} or L{None} + @param d: The 'd' RSA variable (optional for a public key). + + @type p: L{int} or L{None} + @param p: The 'p' RSA variable (optional for a public key). + + @type q: L{int} or L{None} + @param q: The 'q' RSA variable (optional for a public key). + + @type u: L{int} or L{None} + @param u: The 'u' RSA variable. Ignored, as its value is determined by + p and q. + + @rtype: L{Key} + @return: An RSA key constructed from the values as given. + """ + publicNumbers = rsa.RSAPublicNumbers(e=e, n=n) + if d is None: + # We have public components. + keyObject = publicNumbers.public_key(default_backend()) + else: + privateNumbers = rsa.RSAPrivateNumbers( + p=p, + q=q, + d=d, + dmp1=rsa.rsa_crt_dmp1(d, p), + dmq1=rsa.rsa_crt_dmq1(d, q), + iqmp=rsa.rsa_crt_iqmp(p, q), + public_numbers=publicNumbers, + ) + keyObject = privateNumbers.private_key(default_backend()) + + return cls(keyObject) + + @classmethod + def _fromDSAComponents(cls, y, p, q, g, x=None): + """ + Build a key from DSA numerical components. + + @type y: L{int} + @param y: The 'y' DSA variable. + + @type p: L{int} + @param p: The 'p' DSA variable. + + @type q: L{int} + @param q: The 'q' DSA variable. + + @type g: L{int} + @param g: The 'g' DSA variable. + + @type x: L{int} or L{None} + @param x: The 'x' DSA variable (optional for a public key) + + @rtype: L{Key} + @return: A DSA key constructed from the values as given. + """ + publicNumbers = dsa.DSAPublicNumbers( + y=y, parameter_numbers=dsa.DSAParameterNumbers(p=p, q=q, g=g)) + if x is None: + # We have public components. + keyObject = publicNumbers.public_key(default_backend()) + else: + privateNumbers = dsa.DSAPrivateNumbers( + x=x, public_numbers=publicNumbers) + keyObject = privateNumbers.private_key(default_backend()) + + return cls(keyObject) + + @classmethod + def _fromECComponents(cls, x, y, curve, privateValue=None): + """ + Build a key from EC components. + + @param x: The affine x component of the public point used for verifying. + @type x: L{int} + + @param y: The affine y component of the public point used for verifying. + @type y: L{int} + + @param curve: NIST name of elliptic curve. + @type curve: L{bytes} + + @param privateValue: The private value. + @type privateValue: L{int} + """ + + publicNumbers = ec.EllipticCurvePublicNumbers( + x=x, y=y, curve=_curveTable[curve]) + if privateValue is None: + # We have public components. + keyObject = publicNumbers.public_key(default_backend()) + else: + privateNumbers = ec.EllipticCurvePrivateNumbers( + private_value=privateValue, public_numbers=publicNumbers) + keyObject = privateNumbers.private_key(default_backend()) + + return cls(keyObject) + + @classmethod + def _fromECEncodedPoint(cls, encodedPoint, curve, privateValue=None): + """ + Build a key from an EC encoded point. + + @param encodedPoint: The public point encoded as in SEC 1 v2.0 + section 2.3.3. + @type encodedPoint: L{bytes} + + @param curve: NIST name of elliptic curve. + @type curve: L{bytes} + + @param privateValue: The private value. + @type privateValue: L{int} + """ + + if privateValue is None: + # We have public components. + keyObject = ec.EllipticCurvePublicKey.from_encoded_point( + _curveTable[curve], encodedPoint + ) + else: + keyObject = ec.derive_private_key( + privateValue, _curveTable[curve], default_backend() + ) + + return cls(keyObject) + + def __init__(self, keyObject): + """ + Initialize with a private or public + C{cryptography.hazmat.primitives.asymmetric} key. + + @param keyObject: Low level key. + @type keyObject: C{cryptography.hazmat.primitives.asymmetric} key. + """ + self._keyObject = keyObject + + def __eq__(self, other): + """ + Return True if other represents an object with the same key. + """ + if type(self) == type(other): + return self.type() == other.type() and self.data() == other.data() + else: + return NotImplemented + + def __ne__(self, other): + """ + Return True if other represents anything other than this key. + """ + result = self.__eq__(other) + if result == NotImplemented: + return result + return not result + + def __repr__(self): + """ + Return a pretty representation of this object. + """ + if self.type() == 'EC': + data = self.data() + name = data['curve'].decode('utf-8') + + if self.isPublic(): + out = '\n" + else: + lines = [ + '<%s %s (%s bits)' % ( + nativeString(self.type()), + self.isPublic() and 'Public Key' or 'Private Key', + self._keyObject.key_size)] + for k, v in sorted(self.data().items()): + lines.append('attr %s:' % (k,)) + by = common.MP(v)[4:] + while by: + m = by[:15] + by = by[15:] + o = '' + for c in iterbytes(m): + o = o + '%02x:' % (ord(c),) + if len(m) < 15: + o = o[:-1] + lines.append('\t' + o) + lines[-1] = lines[-1] + '>' + return '\n'.join(lines) + + def isPublic(self): + """ + Check if this instance is a public key. + + @return: C{True} if this is a public key. + """ + return isinstance( + self._keyObject, + (rsa.RSAPublicKey, dsa.DSAPublicKey, ec.EllipticCurvePublicKey)) + + def public(self): + """ + Returns a version of this key containing only the public key data. + If this is a public key, this may or may not be the same object + as self. + + @rtype: L{Key} + @return: A public key. + """ + if self.isPublic(): + return self + else: + return Key(self._keyObject.public_key()) + + def fingerprint(self, format=FingerprintFormats.MD5_HEX): + """ + The fingerprint of a public key consists of the output of the + message-digest algorithm in the specified format. + Supported formats include L{FingerprintFormats.MD5_HEX} and + L{FingerprintFormats.SHA256_BASE64} + + The input to the algorithm is the public key data as specified by [RFC4253]. + + The output of sha256[RFC4634] algorithm is presented to the + user in the form of base64 encoded sha256 hashes. + Example: C{US5jTUa0kgX5ZxdqaGF0yGRu8EgKXHNmoT8jHKo1StM=} + + The output of the MD5[RFC1321](default) algorithm is presented to the user as + a sequence of 16 octets printed as hexadecimal with lowercase letters + and separated by colons. + Example: C{c1:b1:30:29:d7:b8:de:6c:97:77:10:d7:46:41:63:87} + + @param format: Format for fingerprint generation. Consists + hash function and representation format. + Default is L{FingerprintFormats.MD5_HEX} + + @since: 8.2 + + @return: the user presentation of this L{Key}'s fingerprint, as a + string. + + @rtype: L{str} + """ + if format is FingerprintFormats.SHA256_BASE64: + return nativeString(base64.b64encode( + sha256(self.blob()).digest())) + elif format is FingerprintFormats.MD5_HEX: + return nativeString( + b':'.join([binascii.hexlify(x) + for x in iterbytes(md5(self.blob()).digest())])) + else: + raise BadFingerPrintFormat( + 'Unsupported fingerprint format: %s' % (format,)) + + def type(self): + """ + Return the type of the object we wrap. Currently this can only be + 'RSA', 'DSA', or 'EC'. + + @rtype: L{str} + @raises RuntimeError: If the object type is unknown. + """ + if isinstance( + self._keyObject, (rsa.RSAPublicKey, rsa.RSAPrivateKey)): + return 'RSA' + elif isinstance( + self._keyObject, (dsa.DSAPublicKey, dsa.DSAPrivateKey)): + return 'DSA' + elif isinstance( + self._keyObject, (ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey)): + return 'EC' + else: + raise RuntimeError( + 'unknown type of object: %r' % (self._keyObject,)) + + def sshType(self): + """ + Get the type of the object we wrap as defined in the SSH protocol, + defined in RFC 4253, Section 6.6. Currently this can only be b'ssh-rsa', + b'ssh-dss' or b'ecdsa-sha2-[identifier]'. + + identifier is the standard NIST curve name + + @return: The key type format. + @rtype: L{bytes} + """ + if self.type() == 'EC': + return b'ecdsa-sha2-' + _secToNist[self._keyObject.curve.name.encode('ascii')] + else: + return {'RSA': b'ssh-rsa', 'DSA': b'ssh-dss'}[self.type()] + + def size(self): + """ + Return the size of the object we wrap. + + @return: The size of the key. + @rtype: L{int} + """ + if self._keyObject is None: + return 0 + elif self.type() == 'EC': + return self._keyObject.curve.key_size + return self._keyObject.key_size + + def data(self): + """ + Return the values of the public key as a dictionary. + + @rtype: L{dict} + """ + if isinstance(self._keyObject, rsa.RSAPublicKey): + numbers = self._keyObject.public_numbers() + return { + "n": numbers.n, + "e": numbers.e, + } + elif isinstance(self._keyObject, rsa.RSAPrivateKey): + numbers = self._keyObject.private_numbers() + return { + "n": numbers.public_numbers.n, + "e": numbers.public_numbers.e, + "d": numbers.d, + "p": numbers.p, + "q": numbers.q, + # Use a trick: iqmp is q^-1 % p, u is p^-1 % q + "u": rsa.rsa_crt_iqmp(numbers.q, numbers.p), + } + elif isinstance(self._keyObject, dsa.DSAPublicKey): + numbers = self._keyObject.public_numbers() + return { + "y": numbers.y, + "g": numbers.parameter_numbers.g, + "p": numbers.parameter_numbers.p, + "q": numbers.parameter_numbers.q, + } + elif isinstance(self._keyObject, dsa.DSAPrivateKey): + numbers = self._keyObject.private_numbers() + return { + "x": numbers.x, + "y": numbers.public_numbers.y, + "g": numbers.public_numbers.parameter_numbers.g, + "p": numbers.public_numbers.parameter_numbers.p, + "q": numbers.public_numbers.parameter_numbers.q, + } + elif isinstance(self._keyObject, ec.EllipticCurvePublicKey): + numbers = self._keyObject.public_numbers() + return { + "x": numbers.x, + "y": numbers.y, + "curve": self.sshType(), + } + elif isinstance(self._keyObject, ec.EllipticCurvePrivateKey): + numbers = self._keyObject.private_numbers() + return { + "x": numbers.public_numbers.x, + "y": numbers.public_numbers.y, + "privateValue": numbers.private_value, + "curve": self.sshType(), + } + + else: + raise RuntimeError("Unexpected key type: %s" % (self._keyObject,)) + + def blob(self): + """ + Return the public key blob for this key. The blob is the + over-the-wire format for public keys. + + SECSH-TRANS RFC 4253 Section 6.6. + + RSA keys:: + string 'ssh-rsa' + integer e + integer n + + DSA keys:: + string 'ssh-dss' + integer p + integer q + integer g + integer y + + EC keys:: + string 'ecdsa-sha2-[identifier]' + integer x + integer y + + identifier is the standard NIST curve name + + @rtype: L{bytes} + """ + type = self.type() + data = self.data() + if type == 'RSA': + return (common.NS(b'ssh-rsa') + common.MP(data['e']) + + common.MP(data['n'])) + elif type == 'DSA': + return (common.NS(b'ssh-dss') + common.MP(data['p']) + + common.MP(data['q']) + common.MP(data['g']) + + common.MP(data['y'])) + else: # EC + byteLength = (self._keyObject.curve.key_size + 7) // 8 + return (common.NS(data['curve']) + common.NS(data["curve"][-8:]) + + common.NS(b'\x04' + utils.int_to_bytes(data['x'], byteLength) + + utils.int_to_bytes(data['y'], byteLength))) + + + def privateBlob(self): + """ + Return the private key blob for this key. The blob is the + over-the-wire format for private keys: + + Specification in OpenSSH PROTOCOL.agent + + RSA keys:: + string 'ssh-rsa' + integer n + integer e + integer d + integer u + integer p + integer q + + DSA keys:: + string 'ssh-dss' + integer p + integer q + integer g + integer y + integer x + + EC keys:: + string 'ecdsa-sha2-[identifier]' + integer x + integer y + integer privateValue + + identifier is the NIST standard curve name. + """ + type = self.type() + data = self.data() + if type == 'RSA': + iqmp = rsa.rsa_crt_iqmp(data['p'], data['q']) + return (common.NS(b'ssh-rsa') + common.MP(data['n']) + + common.MP(data['e']) + common.MP(data['d']) + + common.MP(iqmp) + common.MP(data['p']) + + common.MP(data['q'])) + elif type == 'DSA': + return (common.NS(b'ssh-dss') + common.MP(data['p']) + + common.MP(data['q']) + common.MP(data['g']) + + common.MP(data['y']) + common.MP(data['x'])) + else: # EC + encPub = self._keyObject.public_key().public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint + ) + return (common.NS(data['curve']) + common.NS(data['curve'][-8:]) + + common.NS(encPub) + common.MP(data['privateValue'])) + + @_mutuallyExclusiveArguments([ + ['extra', 'comment'], + ['extra', 'passphrase'], + ]) + def toString(self, type, extra=None, subtype=None, comment=None, + passphrase=None): + """ + Create a string representation of this key. If the key is a private + key and you want the representation of its public key, use + C{key.public().toString()}. type maps to a _toString_* method. + + @param type: The type of string to emit. Currently supported values + are C{'OPENSSH'}, C{'LSH'}, and C{'AGENTV3'}. + @type type: L{str} + + @param extra: Any extra data supported by the selected format which + is not part of the key itself. For public OpenSSH keys, this is + a comment. For private OpenSSH keys, this is a passphrase to + encrypt with. (Deprecated since Twisted 20.3.0; use C{comment} + or C{passphrase} as appropriate instead.) + @type extra: L{bytes} or L{unicode} or L{None} + + @param subtype: A subtype of the requested C{type} to emit. Only + supported for private OpenSSH keys, for which the currently + supported subtypes are C{'PEM'} and C{'v1'}. If not given, an + appropriate default is used. + @type subtype: L{str} or L{None} + + @param comment: A comment to include with the key. Only supported + for OpenSSH keys. + + Present since Twisted 20.3.0. + + @type comment: L{bytes} or L{unicode} or L{None} + + @param passphrase: A passphrase to encrypt the key with. Only + supported for private OpenSSH keys. + + Present since Twisted 20.3.0. + + @type passphrase: L{bytes} or L{unicode} or L{None} + + @rtype: L{bytes} + """ + if extra is not None: + # Compatibility with old parameter format. + warnings.warn( + "The 'extra' argument to " + "twisted.conch.ssh.keys.Key.toString was deprecated in " + "Twisted 20.3.0; use 'comment' or 'passphrase' instead.", + DeprecationWarning, stacklevel=3) + if self.isPublic(): + comment = extra + else: + passphrase = extra + if isinstance(comment, unicode): + comment = comment.encode("utf-8") + if isinstance(passphrase, unicode): + passphrase = passphrase.encode("utf-8") + method = getattr(self, '_toString_%s' % (type.upper(),), None) + if method is None: + raise BadKeyError('unknown key type: %s' % (type,)) + return method(subtype=subtype, comment=comment, passphrase=passphrase) + + def _toPublicOpenSSH(self, comment=None): + """ + Return a public OpenSSH key string. + + See _fromString_PUBLIC_OPENSSH for the string format. + + @type comment: L{bytes} or L{None} + @param comment: A comment to include with the key, or L{None} to + omit the comment. + """ + if self.type() == 'EC': + if not comment: + comment = b'' + return (self._keyObject.public_bytes( + serialization.Encoding.OpenSSH, + serialization.PublicFormat.OpenSSH + ) + b' ' + comment).strip() + + b64Data = encodebytes(self.blob()).replace(b'\n', b'') + if not comment: + comment = b'' + return (self.sshType() + b' ' + b64Data + b' ' + comment).strip() + + def _toPrivateOpenSSH_v1(self, comment=None, passphrase=None): + """ + Return a private OpenSSH key string, in the "openssh-key-v1" format + introduced in OpenSSH 6.5. + + See _fromPrivateOpenSSH_v1 for the string format. + + @type passphrase: L{bytes} or L{None} + @param passphrase: The passphrase to encrypt the key with, or L{None} + if it is not encrypted. + """ + if passphrase: + # For now we just hardcode the cipher to the one used by + # OpenSSH. We could make this configurable later if it's + # needed. + cipher = algorithms.AES + cipherName = b'aes256-ctr' + kdfName = b'bcrypt' + blockSize = cipher.block_size // 8 + keySize = 32 + ivSize = blockSize + salt = randbytes.secureRandom(ivSize) + rounds = 100 + kdfOptions = common.NS(salt) + struct.pack('!L', rounds) + else: + cipherName = b'none' + kdfName = b'none' + blockSize = 8 + kdfOptions = b'' + check = randbytes.secureRandom(4) + privKeyList = ( + check + check + self.privateBlob() + common.NS(comment or b'')) + padByte = 0 + while len(privKeyList) % blockSize: + padByte += 1 + privKeyList += chr(padByte & 0xFF) + if passphrase: + encKey = bcrypt.kdf(passphrase, salt, keySize + ivSize, 100) + encryptor = Cipher( + cipher(encKey[:keySize]), + modes.CTR(encKey[keySize:keySize + ivSize]), + backend=default_backend() + ).encryptor() + encPrivKeyList = ( + encryptor.update(privKeyList) + encryptor.finalize()) + else: + encPrivKeyList = privKeyList + blob = ( + b'openssh-key-v1\0' + + common.NS(cipherName) + + common.NS(kdfName) + common.NS(kdfOptions) + + struct.pack('!L', 1) + + common.NS(self.blob()) + + common.NS(encPrivKeyList)) + b64Data = encodebytes(blob).replace(b'\n', b'') + lines = ( + [b'-----BEGIN OPENSSH PRIVATE KEY-----'] + + [b64Data[i:i + 64] for i in range(0, len(b64Data), 64)] + + [b'-----END OPENSSH PRIVATE KEY-----']) + return b'\n'.join(lines) + b'\n' + + def _toPrivateOpenSSH_PEM(self, passphrase=None): + """ + Return a private OpenSSH key string, in the old PEM-based format. + + See _fromPrivateOpenSSH_PEM for the string format. + + @type passphrase: L{bytes} or L{None} + @param passphrase: The passphrase to encrypt the key with, or L{None} + if it is not encrypted. + """ + if self.type() == 'EC': + # EC keys has complex ASN.1 structure hence we do this this way. + if not passphrase: + # unencrypted private key + encryptor = serialization.NoEncryption() + else: + encryptor = serialization.BestAvailableEncryption(passphrase) + + return self._keyObject.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + encryptor) + + data = self.data() + lines = [b''.join((b'-----BEGIN ', self.type().encode('ascii'), + b' PRIVATE KEY-----'))] + if self.type() == 'RSA': + p, q = data['p'], data['q'] + iqmp = rsa.rsa_crt_iqmp(p, q) + objData = (0, data['n'], data['e'], data['d'], p, q, + data['d'] % (p - 1), data['d'] % (q - 1), + iqmp) + else: + objData = (0, data['p'], data['q'], data['g'], data['y'], + data['x']) + asn1Sequence = univ.Sequence() + for index, value in izip(itertools.count(), objData): + asn1Sequence.setComponentByPosition(index, univ.Integer(value)) + asn1Data = berEncoder.encode(asn1Sequence) + if passphrase: + iv = randbytes.secureRandom(8) + hexiv = ''.join(['%02X' % (ord(x),) for x in iterbytes(iv)]) + hexiv = hexiv.encode('ascii') + lines.append(b'Proc-Type: 4,ENCRYPTED') + lines.append(b'DEK-Info: DES-EDE3-CBC,' + hexiv + b'\n') + ba = md5(passphrase + iv).digest() + bb = md5(ba + passphrase + iv).digest() + encKey = (ba + bb)[:24] + padLen = 8 - (len(asn1Data) % 8) + asn1Data += chr(padLen) * padLen + + encryptor = Cipher( + algorithms.TripleDES(encKey), + modes.CBC(iv), + backend=default_backend() + ).encryptor() + + asn1Data = encryptor.update(asn1Data) + encryptor.finalize() + + b64Data = encodebytes(asn1Data).replace(b'\n', b'') + lines += [b64Data[i:i + 64] for i in range(0, len(b64Data), 64)] + lines.append(b''.join((b'-----END ', self.type().encode('ascii'), + b' PRIVATE KEY-----'))) + return b'\n'.join(lines) + + def _toString_OPENSSH(self, subtype=None, comment=None, passphrase=None): + """ + Return a public or private OpenSSH string. See + _fromString_PUBLIC_OPENSSH and _fromPrivateOpenSSH_PEM for the + string formats. If extra is present, it represents a comment for a + public key, or a passphrase for a private key. + + @param extra: Comment for a public key or passphrase for a + private key + @type extra: L{bytes} + + @rtype: L{bytes} + """ + if self.isPublic(): + return self._toPublicOpenSSH(comment=comment) + elif subtype is None or subtype == 'PEM': + return self._toPrivateOpenSSH_PEM(passphrase=passphrase) + elif subtype == 'v1': + return self._toPrivateOpenSSH_v1( + comment=comment, passphrase=passphrase) + else: + raise ValueError('unknown subtype %s' % (subtype,)) + + def _toString_LSH(self, **kwargs): + """ + Return a public or private LSH key. See _fromString_PUBLIC_LSH and + _fromString_PRIVATE_LSH for the key formats. + + @rtype: L{bytes} + """ + data = self.data() + type = self.type() + if self.isPublic(): + if type == 'RSA': + keyData = sexpy.pack([[b'public-key', + [b'rsa-pkcs1-sha1', + [b'n', common.MP(data['n'])[4:]], + [b'e', common.MP(data['e'])[4:]]]]]) + elif type == 'DSA': + keyData = sexpy.pack([[b'public-key', + [b'dsa', + [b'p', common.MP(data['p'])[4:]], + [b'q', common.MP(data['q'])[4:]], + [b'g', common.MP(data['g'])[4:]], + [b'y', common.MP(data['y'])[4:]]]]]) + else: + raise BadKeyError("unknown key type %s" % (type,)) + return (b'{' + encodebytes(keyData).replace(b'\n', b'') + + b'}') + else: + if type == 'RSA': + p, q = data['p'], data['q'] + iqmp = rsa.rsa_crt_iqmp(p, q) + return sexpy.pack([[b'private-key', + [b'rsa-pkcs1', + [b'n', common.MP(data['n'])[4:]], + [b'e', common.MP(data['e'])[4:]], + [b'd', common.MP(data['d'])[4:]], + [b'p', common.MP(q)[4:]], + [b'q', common.MP(p)[4:]], + [b'a', common.MP( + data['d'] % (q - 1))[4:]], + [b'b', common.MP( + data['d'] % (p - 1))[4:]], + [b'c', common.MP(iqmp)[4:]]]]]) + elif type == 'DSA': + return sexpy.pack([[b'private-key', + [b'dsa', + [b'p', common.MP(data['p'])[4:]], + [b'q', common.MP(data['q'])[4:]], + [b'g', common.MP(data['g'])[4:]], + [b'y', common.MP(data['y'])[4:]], + [b'x', common.MP(data['x'])[4:]]]]]) + else: + raise BadKeyError("unknown key type %s'" % (type,)) + + def _toString_AGENTV3(self, **kwargs): + """ + Return a private Secure Shell Agent v3 key. See + _fromString_AGENTV3 for the key format. + + @rtype: L{bytes} + """ + data = self.data() + if not self.isPublic(): + if self.type() == 'RSA': + values = (data['e'], data['d'], data['n'], data['u'], + data['p'], data['q']) + elif self.type() == 'DSA': + values = (data['p'], data['q'], data['g'], data['y'], + data['x']) + return common.NS(self.sshType()) + b''.join(map(common.MP, values)) + + def sign(self, data): + """ + Sign some data with this key. + + SECSH-TRANS RFC 4253 Section 6.6. + + @type data: L{bytes} + @param data: The data to sign. + + @rtype: L{bytes} + @return: A signature for the given data. + """ + keyType = self.type() + if keyType == 'RSA': + sig = self._keyObject.sign(data, padding.PKCS1v15(), hashes.SHA1()) + ret = common.NS(sig) + + elif keyType == 'DSA': + sig = self._keyObject.sign(data, hashes.SHA1()) + (r, s) = decode_dss_signature(sig) + # SSH insists that the DSS signature blob be two 160-bit integers + # concatenated together. The sig[0], [1] numbers from obj.sign + # are just numbers, and could be any length from 0 to 160 bits. + # Make sure they are padded out to 160 bits (20 bytes each) + ret = common.NS(int_to_bytes(r, 20) + int_to_bytes(s, 20)) + + elif keyType == 'EC': # Pragma: no branch + # Hash size depends on key size + keySize = self.size() + if keySize <= 256: + hashSize = hashes.SHA256() + elif keySize <= 384: + hashSize = hashes.SHA384() + else: + hashSize = hashes.SHA512() + signature = self._keyObject.sign(data, ec.ECDSA(hashSize)) + (r, s) = decode_dss_signature(signature) + + rb = int_to_bytes(r) + sb = int_to_bytes(s) + + # Int_to_bytes returns rb[0] as a str in python2 + # and an as int in python3 + if type(rb[0]) is str: + rcomp = ord(rb[0]) + else: + rcomp = rb[0] + + # If the MSB is set, prepend a null byte for correct formatting. + if rcomp & 0x80: + rb = b"\x00" + rb + + if type(sb[0]) is str: + scomp = ord(sb[0]) + else: + scomp = sb[0] + + if scomp & 0x80: + sb = b"\x00" + sb + + ret = common.NS(common.NS(rb) + common.NS(sb)) + return common.NS(self.sshType()) + ret + + def verify(self, signature, data): + """ + Verify a signature using this key. + + @type signature: L{bytes} + @param signature: The signature to verify. + + @type data: L{bytes} + @param data: The signed data. + + @rtype: L{bool} + @return: C{True} if the signature is valid. + """ + if len(signature) == 40: + # DSA key with no padding + signatureType, signature = b'ssh-dss', common.NS(signature) + else: + signatureType, signature = common.getNS(signature) + + if signatureType != self.sshType(): + return False + + keyType = self.type() + if keyType == 'RSA': + k = self._keyObject + if not self.isPublic(): + k = k.public_key() + args = ( + common.getNS(signature)[0], + data, + padding.PKCS1v15(), + hashes.SHA1(), + ) + elif keyType == 'DSA': + concatenatedSignature = common.getNS(signature)[0] + r = int_from_bytes(concatenatedSignature[:20], 'big') + s = int_from_bytes(concatenatedSignature[20:], 'big') + signature = encode_dss_signature(r, s) + k = self._keyObject + if not self.isPublic(): + k = k.public_key() + args = (signature, data, hashes.SHA1()) + + elif keyType == 'EC': # Pragma: no branch + concatenatedSignature = common.getNS(signature)[0] + rstr, sstr, rest = common.getNS(concatenatedSignature, 2) + r = int_from_bytes(rstr, 'big') + s = int_from_bytes(sstr, 'big') + signature = encode_dss_signature(r, s) + + k = self._keyObject + if not self.isPublic(): + k = k.public_key() + + keySize = self.size() + if keySize <= 256: # Hash size depends on key size + hashSize = hashes.SHA256() + elif keySize <= 384: + hashSize = hashes.SHA384() + else: + hashSize = hashes.SHA512() + args = (signature, data, ec.ECDSA(hashSize)) + + try: + k.verify(*args) + except InvalidSignature: + return False + else: + return True + + +def _getPersistentRSAKey(location, keySize=4096): + """ + This function returns a persistent L{Key}. + + The key is loaded from a PEM file in C{location}. If it does not exist, a + key with the key size of C{keySize} is generated and saved. + + @param location: Where the key is stored. + @type location: L{twisted.python.filepath.FilePath} + + @param keySize: The size of the key, if it needs to be generated. + @type keySize: L{int} + + @returns: A persistent key. + @rtype: L{Key} + """ + location.parent().makedirs(ignoreExistingDirectory=True) + + # If it doesn't exist, we want to generate a new key and save it + if not location.exists(): + privateKey = rsa.generate_private_key( + public_exponent=65537, + key_size=keySize, + backend=default_backend() + ) + + pem = privateKey.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + + location.setContent(pem) + + # By this point (save any hilarious race conditions) we should have a + # working PEM file. Load it! + # (Future archaeological readers: I chose not to short circuit above, + # because then there's two exit paths to this code!) + with location.open("rb") as keyFile: + privateKey = serialization.load_pem_private_key( + keyFile.read(), + password=None, + backend=default_backend() + ) + return Key(privateKey) diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/service.py b/contrib/python/Twisted/py2/twisted/conch/ssh/service.py new file mode 100644 index 00000000000..94a34cce9be --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/service.py @@ -0,0 +1,48 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +The parent class for all the SSH services. Currently implemented services +are ssh-userauth and ssh-connection. + +Maintainer: Paul Swartz +""" + +from __future__ import division, absolute_import + +from twisted.python import log + +class SSHService(log.Logger): + name = None # this is the ssh name for the service + protocolMessages = {} # these map #'s -> protocol names + transport = None # gets set later + + def serviceStarted(self): + """ + called when the service is active on the transport. + """ + + def serviceStopped(self): + """ + called when the service is stopped, either by the connection ending + or by another service being started + """ + + def logPrefix(self): + return "SSHService %r on %s" % (self.name, + self.transport.transport.logPrefix()) + + def packetReceived(self, messageNum, packet): + """ + called when we receive a packet on the transport + """ + #print self.protocolMessages + if messageNum in self.protocolMessages: + messageType = self.protocolMessages[messageNum] + f = getattr(self,'ssh_%s' % messageType[4:], + None) + if f is not None: + return f(packet) + log.msg("couldn't handle %r" % messageNum) + log.msg(repr(packet)) + self.transport.sendUnimplemented() diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/session.py b/contrib/python/Twisted/py2/twisted/conch/ssh/session.py new file mode 100644 index 00000000000..3a2f5d54c7a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/session.py @@ -0,0 +1,362 @@ +# -*- test-case-name: twisted.conch.test.test_session -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module contains the implementation of SSHSession, which (by default) +allows access to a shell and a python interpreter over SSH. + +Maintainer: Paul Swartz +""" + +from __future__ import division, absolute_import + +import struct +import signal +import sys +import os + +from zope.interface import implementer + +from twisted.internet import interfaces, protocol +from twisted.python import log +from twisted.python.compat import _bytesChr as chr, networkString +from twisted.conch.interfaces import ISession +from twisted.conch.ssh import common, channel, connection + + +class SSHSession(channel.SSHChannel): + + name = b'session' + def __init__(self, *args, **kw): + channel.SSHChannel.__init__(self, *args, **kw) + self.buf = b'' + self.client = None + self.session = None + + def request_subsystem(self, data): + subsystem, ignored= common.getNS(data) + log.msg('asking for subsystem "%s"' % subsystem) + client = self.avatar.lookupSubsystem(subsystem, data) + if client: + pp = SSHSessionProcessProtocol(self) + proto = wrapProcessProtocol(pp) + client.makeConnection(proto) + pp.makeConnection(wrapProtocol(client)) + self.client = pp + return 1 + else: + log.msg('failed to get subsystem') + return 0 + + def request_shell(self, data): + log.msg('getting shell') + if not self.session: + self.session = ISession(self.avatar) + try: + pp = SSHSessionProcessProtocol(self) + self.session.openShell(pp) + except: + log.deferr() + return 0 + else: + self.client = pp + return 1 + + def request_exec(self, data): + if not self.session: + self.session = ISession(self.avatar) + f,data = common.getNS(data) + log.msg('executing command "%s"' % f) + try: + pp = SSHSessionProcessProtocol(self) + self.session.execCommand(pp, f) + except: + log.deferr() + return 0 + else: + self.client = pp + return 1 + + def request_pty_req(self, data): + if not self.session: + self.session = ISession(self.avatar) + term, windowSize, modes = parseRequest_pty_req(data) + log.msg('pty request: %r %r' % (term, windowSize)) + try: + self.session.getPty(term, windowSize, modes) + except: + log.err() + return 0 + else: + return 1 + + def request_window_change(self, data): + if not self.session: + self.session = ISession(self.avatar) + winSize = parseRequest_window_change(data) + try: + self.session.windowChanged(winSize) + except: + log.msg('error changing window size') + log.err() + return 0 + else: + return 1 + + def dataReceived(self, data): + if not self.client: + #self.conn.sendClose(self) + self.buf += data + return + self.client.transport.write(data) + + def extReceived(self, dataType, data): + if dataType == connection.EXTENDED_DATA_STDERR: + if self.client and hasattr(self.client.transport, 'writeErr'): + self.client.transport.writeErr(data) + else: + log.msg('weird extended data: %s'%dataType) + + def eofReceived(self): + if self.session: + self.session.eofReceived() + elif self.client: + self.conn.sendClose(self) + + def closed(self): + if self.session: + self.session.closed() + elif self.client: + self.client.transport.loseConnection() + + #def closeReceived(self): + # self.loseConnection() # don't know what to do with this + + def loseConnection(self): + if self.client: + self.client.transport.loseConnection() + channel.SSHChannel.loseConnection(self) + +class _ProtocolWrapper(protocol.ProcessProtocol): + """ + This class wraps a L{Protocol} instance in a L{ProcessProtocol} instance. + """ + def __init__(self, proto): + self.proto = proto + + def connectionMade(self): self.proto.connectionMade() + + def outReceived(self, data): self.proto.dataReceived(data) + + def processEnded(self, reason): self.proto.connectionLost(reason) + +class _DummyTransport: + + def __init__(self, proto): + self.proto = proto + + def dataReceived(self, data): + self.proto.transport.write(data) + + def write(self, data): + self.proto.dataReceived(data) + + def writeSequence(self, seq): + self.write(b''.join(seq)) + + def loseConnection(self): + self.proto.connectionLost(protocol.connectionDone) + +def wrapProcessProtocol(inst): + if isinstance(inst, protocol.Protocol): + return _ProtocolWrapper(inst) + else: + return inst + +def wrapProtocol(proto): + return _DummyTransport(proto) + + + +# SUPPORTED_SIGNALS is a list of signals that every session channel is supposed +# to accept. See RFC 4254 +SUPPORTED_SIGNALS = ["ABRT", "ALRM", "FPE", "HUP", "ILL", "INT", "KILL", + "PIPE", "QUIT", "SEGV", "TERM", "USR1", "USR2"] + + + +@implementer(interfaces.ITransport) +class SSHSessionProcessProtocol(protocol.ProcessProtocol): + """I am both an L{IProcessProtocol} and an L{ITransport}. + + I am a transport to the remote endpoint and a process protocol to the + local subsystem. + """ + + # once initialized, a dictionary mapping signal values to strings + # that follow RFC 4254. + _signalValuesToNames = None + + def __init__(self, session): + self.session = session + self.lostOutOrErrFlag = False + + def connectionMade(self): + if self.session.buf: + self.transport.write(self.session.buf) + self.session.buf = None + + def outReceived(self, data): + self.session.write(data) + + def errReceived(self, err): + self.session.writeExtended(connection.EXTENDED_DATA_STDERR, err) + + def outConnectionLost(self): + """ + EOF should only be sent when both STDOUT and STDERR have been closed. + """ + if self.lostOutOrErrFlag: + self.session.conn.sendEOF(self.session) + else: + self.lostOutOrErrFlag = True + + def errConnectionLost(self): + """ + See outConnectionLost(). + """ + self.outConnectionLost() + + def connectionLost(self, reason = None): + self.session.loseConnection() + + + def _getSignalName(self, signum): + """ + Get a signal name given a signal number. + """ + if self._signalValuesToNames is None: + self._signalValuesToNames = {} + # make sure that the POSIX ones are the defaults + for signame in SUPPORTED_SIGNALS: + signame = 'SIG' + signame + sigvalue = getattr(signal, signame, None) + if sigvalue is not None: + self._signalValuesToNames[sigvalue] = signame + for k, v in signal.__dict__.items(): + # Check for platform specific signals, ignoring Python specific + # SIG_DFL and SIG_IGN + if k.startswith('SIG') and not k.startswith('SIG_'): + if v not in self._signalValuesToNames: + self._signalValuesToNames[v] = k + '@' + sys.platform + return self._signalValuesToNames[signum] + + + def processEnded(self, reason=None): + """ + When we are told the process ended, try to notify the other side about + how the process ended using the exit-signal or exit-status requests. + Also, close the channel. + """ + if reason is not None: + err = reason.value + if err.signal is not None: + signame = self._getSignalName(err.signal) + if (getattr(os, 'WCOREDUMP', None) is not None and + os.WCOREDUMP(err.status)): + log.msg('exitSignal: %s (core dumped)' % (signame,)) + coreDumped = 1 + else: + log.msg('exitSignal: %s' % (signame,)) + coreDumped = 0 + self.session.conn.sendRequest( + self.session, b'exit-signal', + common.NS(networkString(signame[3:])) + chr(coreDumped) + + common.NS(b'') + common.NS(b'')) + elif err.exitCode is not None: + log.msg('exitCode: %r' % (err.exitCode,)) + self.session.conn.sendRequest(self.session, b'exit-status', + struct.pack('>L', err.exitCode)) + self.session.loseConnection() + + + def getHost(self): + """ + Return the host from my session's transport. + """ + return self.session.conn.transport.getHost() + + + def getPeer(self): + """ + Return the peer from my session's transport. + """ + return self.session.conn.transport.getPeer() + + + def write(self, data): + self.session.write(data) + + + def writeSequence(self, seq): + self.session.write(b''.join(seq)) + + + def loseConnection(self): + self.session.loseConnection() + + + +class SSHSessionClient(protocol.Protocol): + + def dataReceived(self, data): + if self.transport: + self.transport.write(data) + +# methods factored out to make live easier on server writers +def parseRequest_pty_req(data): + """Parse the data from a pty-req request into usable data. + + @returns: a tuple of (terminal type, (rows, cols, xpixel, ypixel), modes) + """ + term, rest = common.getNS(data) + cols, rows, xpixel, ypixel = struct.unpack('>4L', rest[: 16]) + modes, ignored= common.getNS(rest[16:]) + winSize = (rows, cols, xpixel, ypixel) + modes = [(ord(modes[i:i+1]), struct.unpack('>L', modes[i+1: i+5])[0]) + for i in range(0, len(modes)-1, 5)] + return term, winSize, modes + +def packRequest_pty_req(term, geometry, modes): + """ + Pack a pty-req request so that it is suitable for sending. + + NOTE: modes must be packed before being sent here. + + @type geometry: L{tuple} + @param geometry: A tuple of (rows, columns, xpixel, ypixel) + """ + (rows, cols, xpixel, ypixel) = geometry + termPacked = common.NS(term) + winSizePacked = struct.pack('>4L', cols, rows, xpixel, ypixel) + modesPacked = common.NS(modes) # depend on the client packing modes + return termPacked + winSizePacked + modesPacked + +def parseRequest_window_change(data): + """Parse the data from a window-change request into usuable data. + + @returns: a tuple of (rows, cols, xpixel, ypixel) + """ + cols, rows, xpixel, ypixel = struct.unpack('>4L', data) + return rows, cols, xpixel, ypixel + +def packRequest_window_change(geometry): + """ + Pack a window-change request so that it is suitable for sending. + + @type geometry: L{tuple} + @param geometry: A tuple of (rows, columns, xpixel, ypixel) + """ + (rows, cols, xpixel, ypixel) = geometry + return struct.pack('>4L', cols, rows, xpixel, ypixel) diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/sexpy.py b/contrib/python/Twisted/py2/twisted/conch/ssh/sexpy.py new file mode 100644 index 00000000000..c5f102e4f1c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/sexpy.py @@ -0,0 +1,45 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division + +from twisted.python.compat import intToBytes + + +def parse(s): + s = s.strip() + expr = [] + while s: + if s[0:1] == b'(': + newSexp = [] + if expr: + expr[-1].append(newSexp) + expr.append(newSexp) + s = s[1:] + continue + if s[0:1] == b')': + aList = expr.pop() + s=s[1:] + if not expr: + assert not s + return aList + continue + i = 0 + while s[i:i+1].isdigit(): i+=1 + assert i + length = int(s[:i]) + data = s[i+1:i+1+length] + expr[-1].append(data) + s=s[i+1+length:] + assert 0, "this should not happen" + +def pack(sexp): + s = b"" + for o in sexp: + if type(o) in (type(()), type([])): + s+=b'(' + s+=pack(o) + s+=b')' + else: + s+=intToBytes(len(o)) + b":" + o + return s diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/transport.py b/contrib/python/Twisted/py2/twisted/conch/ssh/transport.py new file mode 100644 index 00000000000..e5d75eab18b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/transport.py @@ -0,0 +1,2127 @@ +# -*- test-case-name: twisted.conch.test.test_transport -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +The lowest level SSH protocol. This handles the key negotiation, the +encryption and the compression. The transport layer is described in +RFC 4253. + +Maintainer: Paul Swartz +""" + +from __future__ import absolute_import, division + +import binascii +import hmac +import struct +import zlib + +from hashlib import md5, sha1, sha256, sha384, sha512 + +from cryptography.exceptions import UnsupportedAlgorithm +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher +from cryptography.hazmat.primitives.asymmetric import dh, ec, x25519 + +from twisted import __version__ as twisted_version +from twisted.internet import protocol, defer +from twisted.python import log, randbytes +from twisted.python.compat import iterbytes, _bytesChr as chr, networkString + +# This import is needed if SHA256 hashing is used. +# from twisted.python.compat import nativeString + +from twisted.conch.ssh import address, keys, _kex +from twisted.conch.ssh.common import ( + NS, getNS, MP, getMP, ffs, int_from_bytes +) + + + +def _mpFromBytes(data): + """Make an SSH multiple-precision integer from big-endian L{bytes}. + + Used in ECDH key exchange. + + @type data: L{bytes} + @param data: The input data, interpreted as a big-endian octet string. + + @rtype: L{bytes} + @return: The given data encoded as an SSH multiple-precision integer. + """ + return MP(int_from_bytes(data, 'big')) + + + +class _MACParams(tuple): + """ + L{_MACParams} represents the parameters necessary to compute SSH MAC + (Message Authenticate Codes). + + L{_MACParams} is a L{tuple} subclass to maintain compatibility with older + versions of the code. The elements of a L{_MACParams} are:: + + 0. The digest object used for the MAC + 1. The inner pad ("ipad") string + 2. The outer pad ("opad") string + 3. The size of the digest produced by the digest object + + L{_MACParams} is also an object lesson in why tuples are a bad type for + public APIs. + + @ivar key: The HMAC key which will be used. + """ + + + +class SSHCiphers: + """ + SSHCiphers represents all the encryption operations that need to occur + to encrypt and authenticate the SSH connection. + + @cvar cipherMap: A dictionary mapping SSH encryption names to 3-tuples of + (, + , ) + @cvar macMap: A dictionary mapping SSH MAC names to hash modules. + + @ivar outCipType: the string type of the outgoing cipher. + @ivar inCipType: the string type of the incoming cipher. + @ivar outMACType: the string type of the incoming MAC. + @ivar inMACType: the string type of the incoming MAC. + @ivar encBlockSize: the block size of the outgoing cipher. + @ivar decBlockSize: the block size of the incoming cipher. + @ivar verifyDigestSize: the size of the incoming MAC. + @ivar outMAC: a tuple of (, , , + ) representing the outgoing MAC. + @ivar inMAc: see outMAC, but for the incoming MAC. + """ + + cipherMap = { + b'3des-cbc': (algorithms.TripleDES, 24, modes.CBC), + b'blowfish-cbc': (algorithms.Blowfish, 16, modes.CBC), + b'aes256-cbc': (algorithms.AES, 32, modes.CBC), + b'aes192-cbc': (algorithms.AES, 24, modes.CBC), + b'aes128-cbc': (algorithms.AES, 16, modes.CBC), + b'cast128-cbc': (algorithms.CAST5, 16, modes.CBC), + b'aes128-ctr': (algorithms.AES, 16, modes.CTR), + b'aes192-ctr': (algorithms.AES, 24, modes.CTR), + b'aes256-ctr': (algorithms.AES, 32, modes.CTR), + b'3des-ctr': (algorithms.TripleDES, 24, modes.CTR), + b'blowfish-ctr': (algorithms.Blowfish, 16, modes.CTR), + b'cast128-ctr': (algorithms.CAST5, 16, modes.CTR), + b'none': (None, 0, modes.CBC), + } + macMap = { + b'hmac-sha2-512': sha512, + b'hmac-sha2-384': sha384, + b'hmac-sha2-256': sha256, + b'hmac-sha1': sha1, + b'hmac-md5': md5, + b'none': None + } + + + def __init__(self, outCip, inCip, outMac, inMac): + self.outCipType = outCip + self.inCipType = inCip + self.outMACType = outMac + self.inMACType = inMac + self.encBlockSize = 0 + self.decBlockSize = 0 + self.verifyDigestSize = 0 + self.outMAC = (None, b'', b'', 0) + self.inMAC = (None, b'', b'', 0) + + + def setKeys(self, outIV, outKey, inIV, inKey, outInteg, inInteg): + """ + Set up the ciphers and hashes using the given keys, + + @param outIV: the outgoing initialization vector + @param outKey: the outgoing encryption key + @param inIV: the incoming initialization vector + @param inKey: the incoming encryption key + @param outInteg: the outgoing integrity key + @param inInteg: the incoming integrity key. + """ + o = self._getCipher(self.outCipType, outIV, outKey) + self.encryptor = o.encryptor() + self.encBlockSize = o.algorithm.block_size // 8 + o = self._getCipher(self.inCipType, inIV, inKey) + self.decryptor = o.decryptor() + self.decBlockSize = o.algorithm.block_size // 8 + self.outMAC = self._getMAC(self.outMACType, outInteg) + self.inMAC = self._getMAC(self.inMACType, inInteg) + if self.inMAC: + self.verifyDigestSize = self.inMAC[3] + + + def _getCipher(self, cip, iv, key): + """ + Creates an initialized cipher object. + + @param cip: the name of the cipher, maps into cipherMap + @param iv: the initialzation vector + @param key: the encryption key + + @return: the cipher object. + """ + algorithmClass, keySize, modeClass = self.cipherMap[cip] + if algorithmClass is None: + return _DummyCipher() + + return Cipher( + algorithmClass(key[:keySize]), + modeClass(iv[:algorithmClass.block_size // 8]), + backend=default_backend(), + ) + + + def _getMAC(self, mac, key): + """ + Gets a 4-tuple representing the message authentication code. + (, , , + ) + + @type mac: L{bytes} + @param mac: a key mapping into macMap + + @type key: L{bytes} + @param key: the MAC key. + + @rtype: L{bytes} + @return: The MAC components. + """ + mod = self.macMap[mac] + if not mod: + return (None, b'', b'', 0) + + # With stdlib we can only get attributes fron an instantiated object. + hashObject = mod() + digestSize = hashObject.digest_size + blockSize = hashObject.block_size + + # Truncation here appears to contravene RFC 2104, section 2. However, + # implementing the hashing behavior prescribed by the RFC breaks + # interoperability with OpenSSH (at least version 5.5p1). + key = key[:digestSize] + (b'\x00' * (blockSize - digestSize)) + i = key.translate(hmac.trans_36) + o = key.translate(hmac.trans_5C) + result = _MACParams((mod, i, o, digestSize)) + result.key = key + return result + + + def encrypt(self, blocks): + """ + Encrypt some data. + + @type blocks: L{bytes} + @param blocks: The data to encrypt. + + @rtype: L{bytes} + @return: The encrypted data. + """ + return self.encryptor.update(blocks) + + + def decrypt(self, blocks): + """ + Decrypt some data. + + @type blocks: L{bytes} + @param blocks: The data to decrypt. + + @rtype: L{bytes} + @return: The decrypted data. + """ + return self.decryptor.update(blocks) + + + def makeMAC(self, seqid, data): + """ + Create a message authentication code (MAC) for the given packet using + the outgoing MAC values. + + @type seqid: L{int} + @param seqid: The sequence ID of the outgoing packet. + + @type data: L{bytes} + @param data: The data to create a MAC for. + + @rtype: L{str} + @return: The serialized MAC. + """ + if not self.outMAC[0]: + return b'' + data = struct.pack('>L', seqid) + data + return hmac.HMAC(self.outMAC.key, data, self.outMAC[0]).digest() + + + def verify(self, seqid, data, mac): + """ + Verify an incoming MAC using the incoming MAC values. + + @type seqid: L{int} + @param seqid: The sequence ID of the incoming packet. + + @type data: L{bytes} + @param data: The packet data to verify. + + @type mac: L{bytes} + @param mac: The MAC sent with the packet. + + @rtype: L{bool} + @return: C{True} if the MAC is valid. + """ + if not self.inMAC[0]: + return mac == b'' + data = struct.pack('>L', seqid) + data + outer = hmac.HMAC(self.inMAC.key, data, self.inMAC[0]).digest() + return mac == outer + + + +def _getSupportedCiphers(): + """ + Build a list of ciphers that are supported by the backend in use. + + @return: a list of supported ciphers. + @rtype: L{list} of L{str} + """ + supportedCiphers = [] + cs = [b'aes256-ctr', b'aes256-cbc', b'aes192-ctr', b'aes192-cbc', + b'aes128-ctr', b'aes128-cbc', b'cast128-ctr', b'cast128-cbc', + b'blowfish-ctr', b'blowfish-cbc', b'3des-ctr', b'3des-cbc'] + for cipher in cs: + algorithmClass, keySize, modeClass = SSHCiphers.cipherMap[cipher] + try: + Cipher( + algorithmClass(b' ' * keySize), + modeClass(b' ' * (algorithmClass.block_size // 8)), + backend=default_backend(), + ).encryptor() + except UnsupportedAlgorithm: + pass + else: + supportedCiphers.append(cipher) + return supportedCiphers + + + +class SSHTransportBase(protocol.Protocol): + """ + Protocol supporting basic SSH functionality: sending/receiving packets + and message dispatch. To connect to or run a server, you must use + SSHClientTransport or SSHServerTransport. + + @ivar protocolVersion: A string representing the version of the SSH + protocol we support. Currently defaults to '2.0'. + + @ivar version: A string representing the version of the server or client. + Currently defaults to 'Twisted'. + + @ivar comment: An optional string giving more information about the + server or client. + + @ivar supportedCiphers: A list of strings representing the encryption + algorithms supported, in order from most-preferred to least. + + @ivar supportedMACs: A list of strings representing the message + authentication codes (hashes) supported, in order from most-preferred + to least. Both this and supportedCiphers can include 'none' to use + no encryption or authentication, but that must be done manually, + + @ivar supportedKeyExchanges: A list of strings representing the + key exchanges supported, in order from most-preferred to least. + + @ivar supportedPublicKeys: A list of strings representing the + public key types supported, in order from most-preferred to least. + + @ivar supportedCompressions: A list of strings representing compression + types supported, from most-preferred to least. + + @ivar supportedLanguages: A list of strings representing languages + supported, from most-preferred to least. + + @ivar supportedVersions: A container of strings representing supported ssh + protocol version numbers. + + @ivar isClient: A boolean indicating whether this is a client or server. + + @ivar gotVersion: A boolean indicating whether we have received the + version string from the other side. + + @ivar buf: Data we've received but hasn't been parsed into a packet. + + @ivar outgoingPacketSequence: the sequence number of the next packet we + will send. + + @ivar incomingPacketSequence: the sequence number of the next packet we + are expecting from the other side. + + @ivar outgoingCompression: an object supporting the .compress(str) and + .flush() methods, or None if there is no outgoing compression. Used to + compress outgoing data. + + @ivar outgoingCompressionType: A string representing the outgoing + compression type. + + @ivar incomingCompression: an object supporting the .decompress(str) + method, or None if there is no incoming compression. Used to + decompress incoming data. + + @ivar incomingCompressionType: A string representing the incoming + compression type. + + @ivar ourVersionString: the version string that we sent to the other side. + Used in the key exchange. + + @ivar otherVersionString: the version string sent by the other side. Used + in the key exchange. + + @ivar ourKexInitPayload: the MSG_KEXINIT payload we sent. Used in the key + exchange. + + @ivar otherKexInitPayload: the MSG_KEXINIT payload we received. Used in + the key exchange + + @ivar sessionID: a string that is unique to this SSH session. Created as + part of the key exchange, sessionID is used to generate the various + encryption and authentication keys. + + @ivar service: an SSHService instance, or None. If it's set to an object, + it's the currently running service. + + @ivar kexAlg: the agreed-upon key exchange algorithm. + + @ivar keyAlg: the agreed-upon public key type for the key exchange. + + @ivar currentEncryptions: an SSHCiphers instance. It represents the + current encryption and authentication options for the transport. + + @ivar nextEncryptions: an SSHCiphers instance. Held here until the + MSG_NEWKEYS messages are exchanged, when nextEncryptions is + transitioned to currentEncryptions. + + @ivar first: the first bytes of the next packet. In order to avoid + decrypting data twice, the first bytes are decrypted and stored until + the whole packet is available. + + @ivar _keyExchangeState: The current protocol state with respect to key + exchange. This is either C{_KEY_EXCHANGE_NONE} if no key exchange is + in progress (and returns to this value after any key exchange + completqes), C{_KEY_EXCHANGE_REQUESTED} if this side of the connection + initiated a key exchange, and C{_KEY_EXCHANGE_PROGRESSING} if the other + side of the connection initiated a key exchange. C{_KEY_EXCHANGE_NONE} + is the initial value (however SSH connections begin with key exchange, + so it will quickly change to another state). + + @ivar _blockedByKeyExchange: Whenever C{_keyExchangeState} is not + C{_KEY_EXCHANGE_NONE}, this is a C{list} of pending messages which were + passed to L{sendPacket} but could not be sent because it is not legal + to send them while a key exchange is in progress. When the key + exchange completes, another attempt is made to send these messages. + """ + protocolVersion = b'2.0' + version = b'Twisted_' + twisted_version.encode('ascii') + comment = b'' + ourVersionString = (b'SSH-' + protocolVersion + b'-' + version + b' ' + + comment).strip() + + # L{None} is supported as cipher and hmac. For security they are disabled + # by default. To enable them, subclass this class and add it, or do: + # SSHTransportBase.supportedCiphers.append('none') + # List ordered by preference. + supportedCiphers = _getSupportedCiphers() + supportedMACs = [ + b'hmac-sha2-512', + b'hmac-sha2-384', + b'hmac-sha2-256', + b'hmac-sha1', + b'hmac-md5', + # `none`, + ] + + supportedKeyExchanges = _kex.getSupportedKeyExchanges() + supportedPublicKeys = [] + + # Add the supported EC keys, and change the name from ecdh* to ecdsa* + for eckey in supportedKeyExchanges: + if eckey.find(b'ecdh') != -1: + supportedPublicKeys += [eckey.replace(b'ecdh', b'ecdsa')] + + supportedPublicKeys += [b'ssh-rsa', b'ssh-dss'] + + supportedCompressions = [b'none', b'zlib'] + supportedLanguages = () + supportedVersions = (b'1.99', b'2.0') + isClient = False + gotVersion = False + buf = b'' + outgoingPacketSequence = 0 + incomingPacketSequence = 0 + outgoingCompression = None + incomingCompression = None + sessionID = None + service = None + + # There is no key exchange activity in progress. + _KEY_EXCHANGE_NONE = '_KEY_EXCHANGE_NONE' + + # Key exchange is in progress and we started it. + _KEY_EXCHANGE_REQUESTED = '_KEY_EXCHANGE_REQUESTED' + + # Key exchange is in progress and both sides have sent KEXINIT messages. + _KEY_EXCHANGE_PROGRESSING = '_KEY_EXCHANGE_PROGRESSING' + + # There is a fourth conceptual state not represented here: KEXINIT received + # but not sent. Since we always send a KEXINIT as soon as we get it, we + # can't ever be in that state. + + # The current key exchange state. + _keyExchangeState = _KEY_EXCHANGE_NONE + _blockedByKeyExchange = None + + def connectionLost(self, reason): + """ + When the underlying connection is closed, stop the running service (if + any), and log out the avatar (if any). + + @type reason: L{twisted.python.failure.Failure} + @param reason: The cause of the connection being closed. + """ + if self.service: + self.service.serviceStopped() + if hasattr(self, 'avatar'): + self.logoutFunction() + log.msg('connection lost') + + + def connectionMade(self): + """ + Called when the connection is made to the other side. We sent our + version and the MSG_KEXINIT packet. + """ + self.transport.write(self.ourVersionString + b'\r\n') + self.currentEncryptions = SSHCiphers(b'none', b'none', b'none', + b'none') + self.currentEncryptions.setKeys(b'', b'', b'', b'', b'', b'') + self.sendKexInit() + + + def sendKexInit(self): + """ + Send a I{KEXINIT} message to initiate key exchange or to respond to a + key exchange initiated by the peer. + + @raise RuntimeError: If a key exchange has already been started and it + is not appropriate to send a I{KEXINIT} message at this time. + + @return: L{None} + """ + if self._keyExchangeState != self._KEY_EXCHANGE_NONE: + raise RuntimeError( + "Cannot send KEXINIT while key exchange state is %r" % ( + self._keyExchangeState,)) + + self.ourKexInitPayload = b''.join([ + chr(MSG_KEXINIT), + randbytes.secureRandom(16), + NS(b','.join(self.supportedKeyExchanges)), + NS(b','.join(self.supportedPublicKeys)), + NS(b','.join(self.supportedCiphers)), + NS(b','.join(self.supportedCiphers)), + NS(b','.join(self.supportedMACs)), + NS(b','.join(self.supportedMACs)), + NS(b','.join(self.supportedCompressions)), + NS(b','.join(self.supportedCompressions)), + NS(b','.join(self.supportedLanguages)), + NS(b','.join(self.supportedLanguages)), + b'\000\000\000\000\000']) + self.sendPacket(MSG_KEXINIT, self.ourKexInitPayload[1:]) + self._keyExchangeState = self._KEY_EXCHANGE_REQUESTED + self._blockedByKeyExchange = [] + + + def _allowedKeyExchangeMessageType(self, messageType): + """ + Determine if the given message type may be sent while key exchange is + in progress. + + @param messageType: The type of message + @type messageType: L{int} + + @return: C{True} if the given type of message may be sent while key + exchange is in progress, C{False} if it may not. + @rtype: L{bool} + + @see: U{http://tools.ietf.org/html/rfc4253#section-7.1} + """ + # Written somewhat peculularly to reflect the way the specification + # defines the allowed message types. + if 1 <= messageType <= 19: + return messageType not in (MSG_SERVICE_REQUEST, MSG_SERVICE_ACCEPT) + if 20 <= messageType <= 29: + return messageType not in (MSG_KEXINIT,) + return 30 <= messageType <= 49 + + + def sendPacket(self, messageType, payload): + """ + Sends a packet. If it's been set up, compress the data, encrypt it, + and authenticate it before sending. If key exchange is in progress and + the message is not part of key exchange, queue it to be sent later. + + @param messageType: The type of the packet; generally one of the + MSG_* values. + @type messageType: L{int} + @param payload: The payload for the message. + @type payload: L{str} + """ + if self._keyExchangeState != self._KEY_EXCHANGE_NONE: + if not self._allowedKeyExchangeMessageType(messageType): + self._blockedByKeyExchange.append((messageType, payload)) + return + + payload = chr(messageType) + payload + if self.outgoingCompression: + payload = (self.outgoingCompression.compress(payload) + + self.outgoingCompression.flush(2)) + bs = self.currentEncryptions.encBlockSize + # 4 for the packet length and 1 for the padding length + totalSize = 5 + len(payload) + lenPad = bs - (totalSize % bs) + if lenPad < 4: + lenPad = lenPad + bs + packet = (struct.pack('!LB', + totalSize + lenPad - 4, lenPad) + + payload + randbytes.secureRandom(lenPad)) + encPacket = ( + self.currentEncryptions.encrypt(packet) + + self.currentEncryptions.makeMAC( + self.outgoingPacketSequence, packet)) + self.transport.write(encPacket) + self.outgoingPacketSequence += 1 + + + def getPacket(self): + """ + Try to return a decrypted, authenticated, and decompressed packet + out of the buffer. If there is not enough data, return None. + + @rtype: L{str} or L{None} + @return: The decoded packet, if any. + """ + bs = self.currentEncryptions.decBlockSize + ms = self.currentEncryptions.verifyDigestSize + if len(self.buf) < bs: + # Not enough data for a block + return + if not hasattr(self, 'first'): + first = self.currentEncryptions.decrypt(self.buf[:bs]) + else: + first = self.first + del self.first + packetLen, paddingLen = struct.unpack('!LB', first[:5]) + if packetLen > 1048576: # 1024 ** 2 + self.sendDisconnect( + DISCONNECT_PROTOCOL_ERROR, + networkString('bad packet length {}'.format(packetLen))) + return + if len(self.buf) < packetLen + 4 + ms: + # Not enough data for a packet + self.first = first + return + if (packetLen + 4) % bs != 0: + self.sendDisconnect( + DISCONNECT_PROTOCOL_ERROR, + networkString( + 'bad packet mod (%i%%%i == %i)' % ( + packetLen + 4, bs, (packetLen + 4) % bs))) + return + encData, self.buf = self.buf[:4 + packetLen], self.buf[4 + packetLen:] + packet = first + self.currentEncryptions.decrypt(encData[bs:]) + if len(packet) != 4 + packetLen: + self.sendDisconnect(DISCONNECT_PROTOCOL_ERROR, + b'bad decryption') + return + if ms: + macData, self.buf = self.buf[:ms], self.buf[ms:] + if not self.currentEncryptions.verify(self.incomingPacketSequence, + packet, macData): + self.sendDisconnect(DISCONNECT_MAC_ERROR, b'bad MAC') + return + payload = packet[5:-paddingLen] + if self.incomingCompression: + try: + payload = self.incomingCompression.decompress(payload) + except: + # Tolerate any errors in decompression + log.err() + self.sendDisconnect(DISCONNECT_COMPRESSION_ERROR, + b'compression error') + return + self.incomingPacketSequence += 1 + return payload + + + def _unsupportedVersionReceived(self, remoteVersion): + """ + Called when an unsupported version of the ssh protocol is received from + the remote endpoint. + + @param remoteVersion: remote ssh protocol version which is unsupported + by us. + @type remoteVersion: L{str} + """ + self.sendDisconnect(DISCONNECT_PROTOCOL_VERSION_NOT_SUPPORTED, + b'bad version ' + remoteVersion) + + + def dataReceived(self, data): + """ + First, check for the version string (SSH-2.0-*). After that has been + received, this method adds data to the buffer, and pulls out any + packets. + + @type data: L{bytes} + @param data: The data that was received. + """ + self.buf = self.buf + data + if not self.gotVersion: + if self.buf.find(b'\n', self.buf.find(b'SSH-')) == -1: + return + + # RFC 4253 section 4.2 ask for strict `\r\n` line ending. + # Here we are a bit more relaxed and accept implementations ending + # only in '\n'. + # https://tools.ietf.org/html/rfc4253#section-4.2 + lines = self.buf.split(b'\n') + for p in lines: + if p.startswith(b'SSH-'): + self.gotVersion = True + # Since the line was split on '\n' and most of the time + # it uses '\r\n' we may get an extra '\r'. + self.otherVersionString = p.rstrip(b'\r') + remoteVersion = p.split(b'-')[1] + if remoteVersion not in self.supportedVersions: + self._unsupportedVersionReceived(remoteVersion) + return + i = lines.index(p) + self.buf = b'\n'.join(lines[i + 1:]) + packet = self.getPacket() + while packet: + messageNum = ord(packet[0:1]) + self.dispatchMessage(messageNum, packet[1:]) + packet = self.getPacket() + + + def dispatchMessage(self, messageNum, payload): + """ + Send a received message to the appropriate method. + + @type messageNum: L{int} + @param messageNum: The message number. + + @type payload: L{bytes} + @param payload: The message payload. + """ + if messageNum < 50 and messageNum in messages: + messageType = messages[messageNum][4:] + f = getattr(self, 'ssh_%s' % (messageType,), None) + if f is not None: + f(payload) + else: + log.msg("couldn't handle %s" % messageType) + log.msg(repr(payload)) + self.sendUnimplemented() + elif self.service: + log.callWithLogger(self.service, self.service.packetReceived, + messageNum, payload) + else: + log.msg("couldn't handle %s" % messageNum) + log.msg(repr(payload)) + self.sendUnimplemented() + + + def getPeer(self): + """ + Returns an L{SSHTransportAddress} corresponding to the other (peer) + side of this transport. + + @return: L{SSHTransportAddress} for the peer + @rtype: L{SSHTransportAddress} + @since: 12.1 + """ + return address.SSHTransportAddress(self.transport.getPeer()) + + + def getHost(self): + """ + Returns an L{SSHTransportAddress} corresponding to the this side of + transport. + + @return: L{SSHTransportAddress} for the peer + @rtype: L{SSHTransportAddress} + @since: 12.1 + """ + return address.SSHTransportAddress(self.transport.getHost()) + + + @property + def kexAlg(self): + """ + The key exchange algorithm name agreed between client and server. + """ + return self._kexAlg + + + @kexAlg.setter + def kexAlg(self, value): + """ + Set the key exchange algorithm name. + """ + self._kexAlg = value + + # Client-initiated rekeying looks like this: + # + # C> MSG_KEXINIT + # S> MSG_KEXINIT + # C> MSG_KEX_DH_GEX_REQUEST or MSG_KEXDH_INIT + # S> MSG_KEX_DH_GEX_GROUP or MSG_KEXDH_REPLY + # C> MSG_KEX_DH_GEX_INIT or -- + # S> MSG_KEX_DH_GEX_REPLY or -- + # C> MSG_NEWKEYS + # S> MSG_NEWKEYS + # + # Server-initiated rekeying is the same, only the first two messages are + # switched. + + + def ssh_KEXINIT(self, packet): + """ + Called when we receive a MSG_KEXINIT message. Payload:: + bytes[16] cookie + string keyExchangeAlgorithms + string keyAlgorithms + string incomingEncryptions + string outgoingEncryptions + string incomingAuthentications + string outgoingAuthentications + string incomingCompressions + string outgoingCompressions + string incomingLanguages + string outgoingLanguages + bool firstPacketFollows + unit32 0 (reserved) + + Starts setting up the key exchange, keys, encryptions, and + authentications. Extended by ssh_KEXINIT in SSHServerTransport and + SSHClientTransport. + + @type packet: L{bytes} + @param packet: The message data. + + @return: A L{tuple} of negotiated key exchange algorithms, key + algorithms, and unhandled data, or L{None} if something went wrong. + """ + self.otherKexInitPayload = chr(MSG_KEXINIT) + packet + # This is useless to us: + # cookie = packet[: 16] + k = getNS(packet[16:], 10) + strings, rest = k[:-1], k[-1] + (kexAlgs, keyAlgs, encCS, encSC, macCS, macSC, compCS, compSC, langCS, + langSC) = [s.split(b',') for s in strings] + # These are the server directions + outs = [encSC, macSC, compSC] + ins = [encCS, macSC, compCS] + if self.isClient: + outs, ins = ins, outs # Switch directions + server = (self.supportedKeyExchanges, self.supportedPublicKeys, + self.supportedCiphers, self.supportedCiphers, + self.supportedMACs, self.supportedMACs, + self.supportedCompressions, self.supportedCompressions) + client = (kexAlgs, keyAlgs, outs[0], ins[0], outs[1], ins[1], + outs[2], ins[2]) + if self.isClient: + server, client = client, server + self.kexAlg = ffs(client[0], server[0]) + self.keyAlg = ffs(client[1], server[1]) + self.nextEncryptions = SSHCiphers( + ffs(client[2], server[2]), + ffs(client[3], server[3]), + ffs(client[4], server[4]), + ffs(client[5], server[5])) + self.outgoingCompressionType = ffs(client[6], server[6]) + self.incomingCompressionType = ffs(client[7], server[7]) + if None in (self.kexAlg, self.keyAlg, self.outgoingCompressionType, + self.incomingCompressionType): + self.sendDisconnect(DISCONNECT_KEY_EXCHANGE_FAILED, + b"couldn't match all kex parts") + return + if None in self.nextEncryptions.__dict__.values(): + self.sendDisconnect(DISCONNECT_KEY_EXCHANGE_FAILED, + b"couldn't match all kex parts") + return + log.msg('kex alg, key alg: %r %r' % (self.kexAlg, self.keyAlg)) + log.msg('outgoing: %r %r %r' % (self.nextEncryptions.outCipType, + self.nextEncryptions.outMACType, + self.outgoingCompressionType)) + log.msg('incoming: %r %r %r' % (self.nextEncryptions.inCipType, + self.nextEncryptions.inMACType, + self.incomingCompressionType)) + + if self._keyExchangeState == self._KEY_EXCHANGE_REQUESTED: + self._keyExchangeState = self._KEY_EXCHANGE_PROGRESSING + else: + self.sendKexInit() + + return kexAlgs, keyAlgs, rest # For SSHServerTransport to use + + + def ssh_DISCONNECT(self, packet): + """ + Called when we receive a MSG_DISCONNECT message. Payload:: + long code + string description + + This means that the other side has disconnected. Pass the message up + and disconnect ourselves. + + @type packet: L{bytes} + @param packet: The message data. + """ + reasonCode = struct.unpack('>L', packet[: 4])[0] + description, foo = getNS(packet[4:]) + self.receiveError(reasonCode, description) + self.transport.loseConnection() + + + def ssh_IGNORE(self, packet): + """ + Called when we receive a MSG_IGNORE message. No payload. + This means nothing; we simply return. + + @type packet: L{bytes} + @param packet: The message data. + """ + + + def ssh_UNIMPLEMENTED(self, packet): + """ + Called when we receive a MSG_UNIMPLEMENTED message. Payload:: + long packet + + This means that the other side did not implement one of our packets. + + @type packet: L{bytes} + @param packet: The message data. + """ + seqnum, = struct.unpack('>L', packet) + self.receiveUnimplemented(seqnum) + + + def ssh_DEBUG(self, packet): + """ + Called when we receive a MSG_DEBUG message. Payload:: + bool alwaysDisplay + string message + string language + + This means the other side has passed along some debugging info. + + @type packet: L{bytes} + @param packet: The message data. + """ + alwaysDisplay = bool(ord(packet[0:1])) + message, lang, foo = getNS(packet[1:], 2) + self.receiveDebug(alwaysDisplay, message, lang) + + + def setService(self, service): + """ + Set our service to service and start it running. If we were + running a service previously, stop it first. + + @type service: C{SSHService} + @param service: The service to attach. + """ + log.msg('starting service %r' % (service.name,)) + if self.service: + self.service.serviceStopped() + self.service = service + service.transport = self + self.service.serviceStarted() + + + def sendDebug(self, message, alwaysDisplay=False, language=b''): + """ + Send a debug message to the other side. + + @param message: the message to send. + @type message: L{str} + @param alwaysDisplay: if True, tell the other side to always + display this message. + @type alwaysDisplay: L{bool} + @param language: optionally, the language the message is in. + @type language: L{str} + """ + self.sendPacket(MSG_DEBUG, chr(alwaysDisplay) + NS(message) + + NS(language)) + + + def sendIgnore(self, message): + """ + Send a message that will be ignored by the other side. This is + useful to fool attacks based on guessing packet sizes in the + encrypted stream. + + @param message: data to send with the message + @type message: L{str} + """ + self.sendPacket(MSG_IGNORE, NS(message)) + + + def sendUnimplemented(self): + """ + Send a message to the other side that the last packet was not + understood. + """ + seqnum = self.incomingPacketSequence + self.sendPacket(MSG_UNIMPLEMENTED, struct.pack('!L', seqnum)) + + + def sendDisconnect(self, reason, desc): + """ + Send a disconnect message to the other side and then disconnect. + + @param reason: the reason for the disconnect. Should be one of the + DISCONNECT_* values. + @type reason: L{int} + @param desc: a descrption of the reason for the disconnection. + @type desc: L{str} + """ + self.sendPacket( + MSG_DISCONNECT, struct.pack('>L', reason) + NS(desc) + NS(b'')) + log.msg('Disconnecting with error, code %s\nreason: %s' % (reason, + desc)) + self.transport.loseConnection() + + + def _startEphemeralDH(self): + """ + Prepares for a Diffie-Hellman key agreement exchange. + + Creates an ephemeral keypair in the group defined by (self.g, + self.p) and stores it. + """ + + numbers = dh.DHParameterNumbers(self.p, self.g) + parameters = numbers.parameters(default_backend()) + self.dhSecretKey = parameters.generate_private_key() + y = self.dhSecretKey.public_key().public_numbers().y + self.dhSecretKeyPublicMP = MP(y) + + + def _finishEphemeralDH(self, remoteDHpublicKey): + """ + Completes the Diffie-Hellman key agreement started by + _startEphemeralDH, and forgets the ephemeral secret key. + + @type remoteDHpublicKey: L{int} + @rtype: L{bytes} + @return: The new shared secret, in SSH C{mpint} format. + + """ + + remoteKey = dh.DHPublicNumbers( + remoteDHpublicKey, + dh.DHParameterNumbers(self.p, self.g) + ).public_key(default_backend()) + secret = self.dhSecretKey.exchange(remoteKey) + del self.dhSecretKey + + # The result of a Diffie-Hellman exchange is an integer, but + # the Cryptography module returns it as bytes in a form that + # is only vaguely documented. We fix it up to match the SSH + # MP-integer format as described in RFC4251. + secret = secret.lstrip(b'\x00') + ch = ord(secret[0:1]) + if ch & 0x80: # High bit set? + # Make room for the sign bit + prefix = struct.pack('>L', len(secret) + 1) + b'\x00' + else: + prefix = struct.pack('>L', len(secret)) + return prefix + secret + + + def _getKey(self, c, sharedSecret, exchangeHash): + """ + Get one of the keys for authentication/encryption. + + @type c: L{bytes} + @param c: The letter identifying which key this is. + + @type sharedSecret: L{bytes} + @param sharedSecret: The shared secret K. + + @type exchangeHash: L{bytes} + @param exchangeHash: The hash H from key exchange. + + @rtype: L{bytes} + @return: The derived key. + """ + hashProcessor = _kex.getHashProcessor(self.kexAlg) + k1 = hashProcessor(sharedSecret + exchangeHash + c + self.sessionID) + k1 = k1.digest() + k2 = hashProcessor(sharedSecret + exchangeHash + k1).digest() + k3 = hashProcessor(sharedSecret + exchangeHash + k1 + k2).digest() + k4 = hashProcessor(sharedSecret + exchangeHash + k1 + k2 + k3).digest() + return k1 + k2 + k3 + k4 + + + def _keySetup(self, sharedSecret, exchangeHash): + """ + Set up the keys for the connection and sends MSG_NEWKEYS when + finished, + + @param sharedSecret: a secret string agreed upon using a Diffie- + Hellman exchange, so it is only shared between + the server and the client. + @type sharedSecret: L{str} + @param exchangeHash: A hash of various data known by both sides. + @type exchangeHash: L{str} + """ + if not self.sessionID: + self.sessionID = exchangeHash + initIVCS = self._getKey(b'A', sharedSecret, exchangeHash) + initIVSC = self._getKey(b'B', sharedSecret, exchangeHash) + encKeyCS = self._getKey(b'C', sharedSecret, exchangeHash) + encKeySC = self._getKey(b'D', sharedSecret, exchangeHash) + integKeyCS = self._getKey(b'E', sharedSecret, exchangeHash) + integKeySC = self._getKey(b'F', sharedSecret, exchangeHash) + outs = [initIVSC, encKeySC, integKeySC] + ins = [initIVCS, encKeyCS, integKeyCS] + if self.isClient: # Reverse for the client + log.msg('REVERSE') + outs, ins = ins, outs + self.nextEncryptions.setKeys(outs[0], outs[1], ins[0], ins[1], + outs[2], ins[2]) + self.sendPacket(MSG_NEWKEYS, b'') + + + def _newKeys(self): + """ + Called back by a subclass once a I{MSG_NEWKEYS} message has been + received. This indicates key exchange has completed and new encryption + and compression parameters should be adopted. Any messages which were + queued during key exchange will also be flushed. + """ + log.msg('NEW KEYS') + self.currentEncryptions = self.nextEncryptions + if self.outgoingCompressionType == b'zlib': + self.outgoingCompression = zlib.compressobj(6) + if self.incomingCompressionType == b'zlib': + self.incomingCompression = zlib.decompressobj() + + self._keyExchangeState = self._KEY_EXCHANGE_NONE + messages = self._blockedByKeyExchange + self._blockedByKeyExchange = None + for (messageType, payload) in messages: + self.sendPacket(messageType, payload) + + + def isEncrypted(self, direction="out"): + """ + Check if the connection is encrypted in the given direction. + + @type direction: L{str} + @param direction: The direction: one of 'out', 'in', or 'both'. + + @rtype: L{bool} + @return: C{True} if it is encrypted. + """ + if direction == "out": + return self.currentEncryptions.outCipType != b'none' + elif direction == "in": + return self.currentEncryptions.inCipType != b'none' + elif direction == "both": + return self.isEncrypted("in") and self.isEncrypted("out") + else: + raise TypeError('direction must be "out", "in", or "both"') + + + def isVerified(self, direction="out"): + """ + Check if the connection is verified/authentication in the given direction. + + @type direction: L{str} + @param direction: The direction: one of 'out', 'in', or 'both'. + + @rtype: L{bool} + @return: C{True} if it is verified. + """ + if direction == "out": + return self.currentEncryptions.outMACType != b'none' + elif direction == "in": + return self.currentEncryptions.inMACType != b'none' + elif direction == "both": + return self.isVerified("in") and self.isVerified("out") + else: + raise TypeError('direction must be "out", "in", or "both"') + + + def loseConnection(self): + """ + Lose the connection to the other side, sending a + DISCONNECT_CONNECTION_LOST message. + """ + self.sendDisconnect(DISCONNECT_CONNECTION_LOST, + b"user closed connection") + + # Client methods + + + def receiveError(self, reasonCode, description): + """ + Called when we receive a disconnect error message from the other + side. + + @param reasonCode: the reason for the disconnect, one of the + DISCONNECT_ values. + @type reasonCode: L{int} + @param description: a human-readable description of the + disconnection. + @type description: L{str} + """ + log.msg('Got remote error, code %s\nreason: %s' % (reasonCode, + description)) + + + def receiveUnimplemented(self, seqnum): + """ + Called when we receive an unimplemented packet message from the other + side. + + @param seqnum: the sequence number that was not understood. + @type seqnum: L{int} + """ + log.msg('other side unimplemented packet #%s' % (seqnum,)) + + + def receiveDebug(self, alwaysDisplay, message, lang): + """ + Called when we receive a debug message from the other side. + + @param alwaysDisplay: if True, this message should always be + displayed. + @type alwaysDisplay: L{bool} + @param message: the debug message + @type message: L{str} + @param lang: optionally the language the message is in. + @type lang: L{str} + """ + if alwaysDisplay: + log.msg('Remote Debug Message: %s' % (message,)) + + + def _generateECPrivateKey(self): + """ + Generate an private key for ECDH key exchange. + + @rtype: The appropriate private key type matching C{self.kexAlg}: + L{EllipticCurvePrivateKey} for C{ecdh-sha2-nistp*}, or + L{X25519PrivateKey} for C{curve25519-sha256}. + @return: The generated private key. + """ + if self.kexAlg.startswith(b'ecdh-sha2-nistp'): + try: + curve = keys._curveTable[b'ecdsa' + self.kexAlg[4:]] + except KeyError: + raise UnsupportedAlgorithm('unused-key') + + return ec.generate_private_key(curve, default_backend()) + elif self.kexAlg in ( + b'curve25519-sha256', b'curve25519-sha256@libssh.org'): + return x25519.X25519PrivateKey.generate() + else: + raise UnsupportedAlgorithm( + 'Cannot generate elliptic curve private key for %r' % + (self.kexAlg,)) + + + def _encodeECPublicKey(self, ecPub): + """ + Encode an elliptic curve public key to bytes. + + @type ecPub: The appropriate public key type matching + C{self.kexAlg}: L{EllipticCurvePublicKey} for + C{ecdh-sha2-nistp*}, or L{X25519PublicKey} for + C{curve25519-sha256}. + @param ecPub: The public key to encode. + + @rtype: L{bytes} + @return: The encoded public key. + """ + if self.kexAlg.startswith(b'ecdh-sha2-nistp'): + return ecPub.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint + ) + elif self.kexAlg in ( + b'curve25519-sha256', b'curve25519-sha256@libssh.org'): + return ecPub.public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw + ) + else: + raise UnsupportedAlgorithm( + 'Cannot encode elliptic curve public key for %r' % + (self.kexAlg,)) + + + def _generateECSharedSecret(self, ecPriv, theirECPubBytes): + """ + Generate a shared secret for ECDH key exchange. + + @type ecPriv: The appropriate private key type matching + C{self.kexAlg}: L{EllipticCurvePrivateKey} for + C{ecdh-sha2-nistp*}, or L{X25519PrivateKey} for + C{curve25519-sha256}. + @param ecPriv: Our private key. + + @rtype: L{bytes} + @return: The generated shared secret, as an SSH multiple-precision + integer. + """ + if self.kexAlg.startswith(b'ecdh-sha2-nistp'): + try: + curve = keys._curveTable[b'ecdsa' + self.kexAlg[4:]] + except KeyError: + raise UnsupportedAlgorithm('unused-key') + + theirECPub = ec.EllipticCurvePublicKey.from_encoded_point( + curve, theirECPubBytes) + sharedSecret = ecPriv.exchange(ec.ECDH(), theirECPub) + elif self.kexAlg in ( + b'curve25519-sha256', b'curve25519-sha256@libssh.org'): + theirECPub = x25519.X25519PublicKey.from_public_bytes( + theirECPubBytes) + sharedSecret = ecPriv.exchange(theirECPub) + else: + raise UnsupportedAlgorithm( + 'Cannot generate elliptic curve shared secret for %r' % + (self.kexAlg,)) + + return _mpFromBytes(sharedSecret) + + + +class SSHServerTransport(SSHTransportBase): + """ + SSHServerTransport implements the server side of the SSH protocol. + + @ivar isClient: since we are never the client, this is always False. + + @ivar ignoreNextPacket: if True, ignore the next key exchange packet. This + is set when the client sends a guessed key exchange packet but with + an incorrect guess. + + @ivar dhGexRequest: the KEX_DH_GEX_REQUEST(_OLD) that the client sent. + The key generation needs this to be stored. + + @ivar g: the Diffie-Hellman group generator. + + @ivar p: the Diffie-Hellman group prime. + """ + isClient = False + ignoreNextPacket = 0 + + + def ssh_KEXINIT(self, packet): + """ + Called when we receive a MSG_KEXINIT message. For a description + of the packet, see SSHTransportBase.ssh_KEXINIT(). Additionally, + this method checks if a guessed key exchange packet was sent. If + it was sent, and it guessed incorrectly, the next key exchange + packet MUST be ignored. + """ + retval = SSHTransportBase.ssh_KEXINIT(self, packet) + if not retval: # Disconnected + return + else: + kexAlgs, keyAlgs, rest = retval + if ord(rest[0:1]): # Flag first_kex_packet_follows? + if (kexAlgs[0] != self.supportedKeyExchanges[0] or + keyAlgs[0] != self.supportedPublicKeys[0]): + self.ignoreNextPacket = True # Guess was wrong + + + def _ssh_KEX_ECDH_INIT(self, packet): + """ + Called from L{ssh_KEX_DH_GEX_REQUEST_OLD} to handle + elliptic curve key exchanges. + + Payload:: + + string client Elliptic Curve Diffie-Hellman public key + + Just like L{_ssh_KEXDH_INIT} this message type is also not dispatched + directly. Extra check to determine if this is really KEX_ECDH_INIT + is required. + + First we load the host's public/private keys. + Then we generate the ECDH public/private keypair for the given curve. + With that we generate the shared secret key. + Then we compute the hash to sign and send back to the client + Along with the server's public key and the ECDH public key. + + @type packet: L{bytes} + @param packet: The message data. + + @return: None. + """ + # Get the raw client public key. + pktPub, packet = getNS(packet) + + # Get the host's public and private keys + pubHostKey = self.factory.publicKeys[self.keyAlg] + privHostKey = self.factory.privateKeys[self.keyAlg] + + # Generate the private key + ecPriv = self._generateECPrivateKey() + + # Get the public key + self.ecPub = ecPriv.public_key() + encPub = self._encodeECPublicKey(self.ecPub) + + # Generate the shared secret + sharedSecret = self._generateECSharedSecret(ecPriv, pktPub) + + # Finish update and digest + h = _kex.getHashProcessor(self.kexAlg)() + h.update(NS(self.otherVersionString)) + h.update(NS(self.ourVersionString)) + h.update(NS(self.otherKexInitPayload)) + h.update(NS(self.ourKexInitPayload)) + h.update(NS(pubHostKey.blob())) + h.update(NS(pktPub)) + h.update(NS(encPub)) + h.update(sharedSecret) + exchangeHash = h.digest() + + self.sendPacket( + MSG_KEXDH_REPLY, + NS(pubHostKey.blob()) + NS(encPub) + + NS(privHostKey.sign(exchangeHash))) + self._keySetup(sharedSecret, exchangeHash) + + + def _ssh_KEXDH_INIT(self, packet): + """ + Called to handle the beginning of a non-group key exchange. + + Unlike other message types, this is not dispatched automatically. It + is called from C{ssh_KEX_DH_GEX_REQUEST_OLD} because an extra check is + required to determine if this is really a KEXDH_INIT message or if it + is a KEX_DH_GEX_REQUEST_OLD message. + + The KEXDH_INIT payload:: + + integer e (the client's Diffie-Hellman public key) + + We send the KEXDH_REPLY with our host key and signature. + + @type packet: L{bytes} + @param packet: The message data. + """ + clientDHpublicKey, foo = getMP(packet) + self.g, self.p = _kex.getDHGeneratorAndPrime(self.kexAlg) + self._startEphemeralDH() + sharedSecret = self._finishEphemeralDH(clientDHpublicKey) + h = sha1() + h.update(NS(self.otherVersionString)) + h.update(NS(self.ourVersionString)) + h.update(NS(self.otherKexInitPayload)) + h.update(NS(self.ourKexInitPayload)) + h.update(NS(self.factory.publicKeys[self.keyAlg].blob())) + h.update(MP(clientDHpublicKey)) + h.update(self.dhSecretKeyPublicMP) + h.update(sharedSecret) + exchangeHash = h.digest() + self.sendPacket( + MSG_KEXDH_REPLY, + NS(self.factory.publicKeys[self.keyAlg].blob()) + + self.dhSecretKeyPublicMP + + NS(self.factory.privateKeys[self.keyAlg].sign(exchangeHash))) + self._keySetup(sharedSecret, exchangeHash) + + + def ssh_KEX_DH_GEX_REQUEST_OLD(self, packet): + """ + This represents different key exchange methods that share the same + integer value. If the message is determined to be a KEXDH_INIT, + L{_ssh_KEXDH_INIT} is called to handle it. If it is a KEX_ECDH_INIT, + L{_ssh_KEX_ECDH_INIT} is called. + Otherwise, for KEX_DH_GEX_REQUEST_OLD payload:: + + integer ideal (ideal size for the Diffie-Hellman prime) + + We send the KEX_DH_GEX_GROUP message with the group that is + closest in size to ideal. + + If we were told to ignore the next key exchange packet by ssh_KEXINIT, + drop it on the floor and return. + + @type packet: L{bytes} + @param packet: The message data. + """ + if self.ignoreNextPacket: + self.ignoreNextPacket = 0 + return + + # KEXDH_INIT, KEX_ECDH_INIT, and KEX_DH_GEX_REQUEST_OLD + # have the same value, so use another cue + # to decide what kind of message the peer sent us. + if _kex.isFixedGroup(self.kexAlg): + return self._ssh_KEXDH_INIT(packet) + elif _kex.isEllipticCurve(self.kexAlg): + return self._ssh_KEX_ECDH_INIT(packet) + else: + self.dhGexRequest = packet + ideal = struct.unpack('>L', packet)[0] + self.g, self.p = self.factory.getDHPrime(ideal) + self._startEphemeralDH() + self.sendPacket(MSG_KEX_DH_GEX_GROUP, MP(self.p) + MP(self.g)) + + + def ssh_KEX_DH_GEX_REQUEST(self, packet): + """ + Called when we receive a MSG_KEX_DH_GEX_REQUEST message. Payload:: + integer minimum + integer ideal + integer maximum + + The client is asking for a Diffie-Hellman group between minimum and + maximum size, and close to ideal if possible. We reply with a + MSG_KEX_DH_GEX_GROUP message. + + If we were told to ignore the next key exchange packet by ssh_KEXINIT, + drop it on the floor and return. + + @type packet: L{bytes} + @param packet: The message data. + """ + if self.ignoreNextPacket: + self.ignoreNextPacket = 0 + return + self.dhGexRequest = packet + min, ideal, max = struct.unpack('>3L', packet) + self.g, self.p = self.factory.getDHPrime(ideal) + self._startEphemeralDH() + self.sendPacket(MSG_KEX_DH_GEX_GROUP, MP(self.p) + MP(self.g)) + + + def ssh_KEX_DH_GEX_INIT(self, packet): + """ + Called when we get a MSG_KEX_DH_GEX_INIT message. Payload:: + integer e (client DH public key) + + We send the MSG_KEX_DH_GEX_REPLY message with our host key and + signature. + + @type packet: L{bytes} + @param packet: The message data. + """ + clientDHpublicKey, foo = getMP(packet) + # TODO: we should also look at the value they send to us and reject + # insecure values of f (if g==2 and f has a single '1' bit while the + # rest are '0's, then they must have used a small y also). + + # TODO: This could be computed when self.p is set up + # or do as openssh does and scan f for a single '1' bit instead + + sharedSecret = self._finishEphemeralDH(clientDHpublicKey) + h = _kex.getHashProcessor(self.kexAlg)() + h.update(NS(self.otherVersionString)) + h.update(NS(self.ourVersionString)) + h.update(NS(self.otherKexInitPayload)) + h.update(NS(self.ourKexInitPayload)) + h.update(NS(self.factory.publicKeys[self.keyAlg].blob())) + h.update(self.dhGexRequest) + h.update(MP(self.p)) + h.update(MP(self.g)) + h.update(MP(clientDHpublicKey)) + h.update(self.dhSecretKeyPublicMP) + h.update(sharedSecret) + exchangeHash = h.digest() + self.sendPacket( + MSG_KEX_DH_GEX_REPLY, + NS(self.factory.publicKeys[self.keyAlg].blob()) + + self.dhSecretKeyPublicMP + + NS(self.factory.privateKeys[self.keyAlg].sign(exchangeHash))) + self._keySetup(sharedSecret, exchangeHash) + + + def ssh_NEWKEYS(self, packet): + """ + Called when we get a MSG_NEWKEYS message. No payload. + When we get this, the keys have been set on both sides, and we + start using them to encrypt and authenticate the connection. + + @type packet: L{bytes} + @param packet: The message data. + """ + if packet != b'': + self.sendDisconnect(DISCONNECT_PROTOCOL_ERROR, + b"NEWKEYS takes no data") + return + self._newKeys() + + + def ssh_SERVICE_REQUEST(self, packet): + """ + Called when we get a MSG_SERVICE_REQUEST message. Payload:: + string serviceName + + The client has requested a service. If we can start the service, + start it; otherwise, disconnect with + DISCONNECT_SERVICE_NOT_AVAILABLE. + + @type packet: L{bytes} + @param packet: The message data. + """ + service, rest = getNS(packet) + cls = self.factory.getService(self, service) + if not cls: + self.sendDisconnect(DISCONNECT_SERVICE_NOT_AVAILABLE, + b"don't have service " + service) + return + else: + self.sendPacket(MSG_SERVICE_ACCEPT, NS(service)) + self.setService(cls()) + + + +class SSHClientTransport(SSHTransportBase): + """ + SSHClientTransport implements the client side of the SSH protocol. + + @ivar isClient: since we are always the client, this is always True. + + @ivar _gotNewKeys: if we receive a MSG_NEWKEYS message before we are + ready to transition to the new keys, this is set to True so we + can transition when the keys are ready locally. + + @ivar x: our Diffie-Hellman private key. + + @ivar e: our Diffie-Hellman public key. + + @ivar g: the Diffie-Hellman group generator. + + @ivar p: the Diffie-Hellman group prime + + @ivar instance: the SSHService object we are requesting. + + @ivar _dhMinimalGroupSize: Minimal acceptable group size advertised by the + client in MSG_KEX_DH_GEX_REQUEST. + @type _dhMinimalGroupSize: int + + @ivar _dhMaximalGroupSize: Maximal acceptable group size advertised by the + client in MSG_KEX_DH_GEX_REQUEST. + @type _dhMaximalGroupSize: int + + @ivar _dhPreferredGroupSize: Preferred group size advertised by the client + in MSG_KEX_DH_GEX_REQUEST. + @type _dhPreferredGroupSize: int + """ + isClient = True + + # Recommended minimal and maximal values from RFC 4419, 3. + _dhMinimalGroupSize = 1024 + _dhMaximalGroupSize = 8192 + # FIXME: https://twistedmatrix.com/trac/ticket/8103 + # This may need to be more dynamic; compare kexgex_client in + # OpenSSH. + _dhPreferredGroupSize = 2048 + + def connectionMade(self): + """ + Called when the connection is started with the server. Just sets + up a private instance variable. + """ + SSHTransportBase.connectionMade(self) + self._gotNewKeys = 0 + + + def ssh_KEXINIT(self, packet): + """ + Called when we receive a MSG_KEXINIT message. For a description + of the packet, see SSHTransportBase.ssh_KEXINIT(). Additionally, + this method sends the first key exchange packet. + + If the agreed-upon exchange is ECDH, generate a key pair for the + corresponding curve and send the public key. + + If the agreed-upon exchange has a fixed prime/generator group, + generate a public key and send it in a MSG_KEXDH_INIT message. + Otherwise, ask for a 2048 bit group with a MSG_KEX_DH_GEX_REQUEST + message. + """ + if SSHTransportBase.ssh_KEXINIT(self, packet) is None: + # Connection was disconnected while doing base processing. + # Maybe no common protocols were agreed. + return + # Are we using ECDH? + if _kex.isEllipticCurve(self.kexAlg): + # Generate the keys + self.ecPriv = self._generateECPrivateKey() + self.ecPub = self.ecPriv.public_key() + + # DH_GEX_REQUEST_OLD is the same number we need. + self.sendPacket( + MSG_KEX_DH_GEX_REQUEST_OLD, + NS(self._encodeECPublicKey(self.ecPub)) + ) + elif _kex.isFixedGroup(self.kexAlg): + # We agreed on a fixed group key exchange algorithm. + self.g, self.p = _kex.getDHGeneratorAndPrime(self.kexAlg) + self._startEphemeralDH() + self.sendPacket(MSG_KEXDH_INIT, self.dhSecretKeyPublicMP) + else: + # We agreed on a dynamic group. Tell the server what range of + # group sizes we accept, and what size we prefer; the server + # will then select a group. + self.sendPacket( + MSG_KEX_DH_GEX_REQUEST, + struct.pack( + '!LLL', + self._dhMinimalGroupSize, + self._dhPreferredGroupSize, + self._dhMaximalGroupSize, + )) + + + def _ssh_KEX_ECDH_REPLY(self, packet): + """ + Called to handle a reply to a ECDH exchange message(KEX_ECDH_INIT). + + Like the handler for I{KEXDH_INIT}, this message type has an + overlapping value. This method is called from C{ssh_KEX_DH_GEX_GROUP} + if that method detects a non-group key exchange is in progress. + + Payload:: + + string serverHostKey + string server Elliptic Curve Diffie-Hellman public key + string signature + + We verify the host key and continue if it passes verificiation. + Otherwise raise an exception and return. + + @type packet: L{bytes} + @param packet: The message data. + + @return: A deferred firing when key exchange is complete. + """ + def _continue_KEX_ECDH_REPLY(ignored, hostKey, pubKey, signature): + # Save off the host public key. + theirECHost = hostKey + + sharedSecret = self._generateECSharedSecret(self.ecPriv, pubKey) + + h = _kex.getHashProcessor(self.kexAlg)() + h.update(NS(self.ourVersionString)) + h.update(NS(self.otherVersionString)) + h.update(NS(self.ourKexInitPayload)) + h.update(NS(self.otherKexInitPayload)) + h.update(NS(theirECHost)) + h.update(NS(self._encodeECPublicKey(self.ecPub))) + h.update(NS(pubKey)) + h.update(sharedSecret) + + exchangeHash = h.digest() + + if not keys.Key.fromString(theirECHost).verify( + signature, exchangeHash): + self.sendDisconnect(DISCONNECT_KEY_EXCHANGE_FAILED, + b'bad signature') + else: + self._keySetup(sharedSecret, exchangeHash) + + # Get the host public key, + # the raw ECDH public key bytes and the signature + hostKey, pubKey, signature, packet = getNS(packet, 3) + + # Easier to comment this out for now than to update all of the tests. + #fingerprint = nativeString(base64.b64encode( + # sha256(hostKey).digest())) + + fingerprint = b':'.join( + [binascii.hexlify(ch) for ch in iterbytes(md5(hostKey).digest())]) + d = self.verifyHostKey(hostKey, fingerprint) + d.addCallback(_continue_KEX_ECDH_REPLY, hostKey, pubKey, signature) + d.addErrback( + lambda unused: self.sendDisconnect( + DISCONNECT_HOST_KEY_NOT_VERIFIABLE, b'bad host key')) + return d + + + def _ssh_KEXDH_REPLY(self, packet): + """ + Called to handle a reply to a non-group key exchange message + (KEXDH_INIT). + + Like the handler for I{KEXDH_INIT}, this message type has an + overlapping value. This method is called from C{ssh_KEX_DH_GEX_GROUP} + if that method detects a non-group key exchange is in progress. + + Payload:: + + string serverHostKey + integer f (server Diffie-Hellman public key) + string signature + + We verify the host key by calling verifyHostKey, then continue in + _continueKEXDH_REPLY. + + @type packet: L{bytes} + @param packet: The message data. + + @return: A deferred firing when key exchange is complete. + """ + pubKey, packet = getNS(packet) + f, packet = getMP(packet) + signature, packet = getNS(packet) + fingerprint = b':'.join([binascii.hexlify(ch) for ch in + iterbytes(md5(pubKey).digest())]) + d = self.verifyHostKey(pubKey, fingerprint) + d.addCallback(self._continueKEXDH_REPLY, pubKey, f, signature) + d.addErrback( + lambda unused: self.sendDisconnect( + DISCONNECT_HOST_KEY_NOT_VERIFIABLE, b'bad host key')) + return d + + + def ssh_KEX_DH_GEX_GROUP(self, packet): + """ + This handles different messages which share an integer value. + + If the key exchange does not have a fixed prime/generator group, + we generate a Diffie-Hellman public key and send it in a + MSG_KEX_DH_GEX_INIT message. + + Payload:: + string g (group generator) + string p (group prime) + + @type packet: L{bytes} + @param packet: The message data. + """ + if _kex.isFixedGroup(self.kexAlg): + return self._ssh_KEXDH_REPLY(packet) + elif _kex.isEllipticCurve(self.kexAlg): + return self._ssh_KEX_ECDH_REPLY(packet) + else: + self.p, rest = getMP(packet) + self.g, rest = getMP(rest) + self._startEphemeralDH() + self.sendPacket(MSG_KEX_DH_GEX_INIT, self.dhSecretKeyPublicMP) + + + def _continueKEXDH_REPLY(self, ignored, pubKey, f, signature): + """ + The host key has been verified, so we generate the keys. + + @param ignored: Ignored. + + @param pubKey: the public key blob for the server's public key. + @type pubKey: L{str} + @param f: the server's Diffie-Hellman public key. + @type f: L{long} + @param signature: the server's signature, verifying that it has the + correct private key. + @type signature: L{str} + """ + serverKey = keys.Key.fromString(pubKey) + sharedSecret = self._finishEphemeralDH(f) + h = sha1() + h.update(NS(self.ourVersionString)) + h.update(NS(self.otherVersionString)) + h.update(NS(self.ourKexInitPayload)) + h.update(NS(self.otherKexInitPayload)) + h.update(NS(pubKey)) + h.update(self.dhSecretKeyPublicMP) + h.update(MP(f)) + h.update(sharedSecret) + exchangeHash = h.digest() + if not serverKey.verify(signature, exchangeHash): + self.sendDisconnect(DISCONNECT_KEY_EXCHANGE_FAILED, + b'bad signature') + return + self._keySetup(sharedSecret, exchangeHash) + + + def ssh_KEX_DH_GEX_REPLY(self, packet): + """ + Called when we receive a MSG_KEX_DH_GEX_REPLY message. Payload:: + string server host key + integer f (server DH public key) + + We verify the host key by calling verifyHostKey, then continue in + _continueGEX_REPLY. + + @type packet: L{bytes} + @param packet: The message data. + + @return: A deferred firing once key exchange is complete. + """ + pubKey, packet = getNS(packet) + f, packet = getMP(packet) + signature, packet = getNS(packet) + fingerprint = b':'.join( + [binascii.hexlify(c) for c in iterbytes(md5(pubKey).digest())]) + d = self.verifyHostKey(pubKey, fingerprint) + d.addCallback(self._continueGEX_REPLY, pubKey, f, signature) + d.addErrback( + lambda unused: self.sendDisconnect( + DISCONNECT_HOST_KEY_NOT_VERIFIABLE, b'bad host key')) + return d + + + def _continueGEX_REPLY(self, ignored, pubKey, f, signature): + """ + The host key has been verified, so we generate the keys. + + @param ignored: Ignored. + + @param pubKey: the public key blob for the server's public key. + @type pubKey: L{str} + @param f: the server's Diffie-Hellman public key. + @type f: L{long} + @param signature: the server's signature, verifying that it has the + correct private key. + @type signature: L{str} + """ + serverKey = keys.Key.fromString(pubKey) + sharedSecret = self._finishEphemeralDH(f) + h = _kex.getHashProcessor(self.kexAlg)() + h.update(NS(self.ourVersionString)) + h.update(NS(self.otherVersionString)) + h.update(NS(self.ourKexInitPayload)) + h.update(NS(self.otherKexInitPayload)) + h.update(NS(pubKey)) + h.update(struct.pack( + '!LLL', + self._dhMinimalGroupSize, + self._dhPreferredGroupSize, + self._dhMaximalGroupSize, + )) + h.update(MP(self.p)) + h.update(MP(self.g)) + h.update(self.dhSecretKeyPublicMP) + h.update(MP(f)) + h.update(sharedSecret) + exchangeHash = h.digest() + if not serverKey.verify(signature, exchangeHash): + self.sendDisconnect(DISCONNECT_KEY_EXCHANGE_FAILED, + b'bad signature') + return + self._keySetup(sharedSecret, exchangeHash) + + + def _keySetup(self, sharedSecret, exchangeHash): + """ + See SSHTransportBase._keySetup(). + """ + SSHTransportBase._keySetup(self, sharedSecret, exchangeHash) + if self._gotNewKeys: + self.ssh_NEWKEYS(b'') + + + def ssh_NEWKEYS(self, packet): + """ + Called when we receive a MSG_NEWKEYS message. No payload. + If we've finished setting up our own keys, start using them. + Otherwise, remember that we've received this message. + + @type packet: L{bytes} + @param packet: The message data. + """ + if packet != b'': + self.sendDisconnect(DISCONNECT_PROTOCOL_ERROR, + b"NEWKEYS takes no data") + return + if not self.nextEncryptions.encBlockSize: + self._gotNewKeys = 1 + return + self._newKeys() + self.connectionSecure() + + + def ssh_SERVICE_ACCEPT(self, packet): + """ + Called when we receive a MSG_SERVICE_ACCEPT message. Payload:: + string service name + + Start the service we requested. + + @type packet: L{bytes} + @param packet: The message data. + """ + if packet == b'': + log.msg('got SERVICE_ACCEPT without payload') + else: + name = getNS(packet)[0] + if name != self.instance.name: + self.sendDisconnect( + DISCONNECT_PROTOCOL_ERROR, + b"received accept for service we did not request") + self.setService(self.instance) + + + def requestService(self, instance): + """ + Request that a service be run over this transport. + + @type instance: subclass of L{twisted.conch.ssh.service.SSHService} + @param instance: The service to run. + """ + self.sendPacket(MSG_SERVICE_REQUEST, NS(instance.name)) + self.instance = instance + + # Client methods + + + def verifyHostKey(self, hostKey, fingerprint): + """ + Returns a Deferred that gets a callback if it is a valid key, or + an errback if not. + + @type hostKey: L{bytes} + @param hostKey: The host key to verify. + + @type fingerprint: L{bytes} + @param fingerprint: The fingerprint of the key. + + @return: A deferred firing with C{True} if the key is valid. + """ + return defer.fail(NotImplementedError()) + + + def connectionSecure(self): + """ + Called when the encryption has been set up. Generally, + requestService() is called to run another service over the transport. + """ + raise NotImplementedError() + + + +class _NullEncryptionContext(object): + """ + An encryption context that does not actually encrypt anything. + """ + def update(self, data): + """ + 'Encrypt' new data by doing nothing. + + @type data: L{bytes} + @param data: The data to 'encrypt'. + + @rtype: L{bytes} + @return: The 'encrypted' data. + """ + return data + + + +class _DummyAlgorithm(object): + """ + An encryption algorithm that does not actually encrypt anything. + """ + block_size = 64 + + + +class _DummyCipher(object): + """ + A cipher for the none encryption method. + + @ivar block_size: the block size of the encryption. In the case of the + none cipher, this is 8 bytes. + """ + algorithm = _DummyAlgorithm() + + + def encryptor(self): + """ + Construct a noop encryptor. + + @return: The encryptor. + """ + return _NullEncryptionContext() + + + def decryptor(self): + """ + Construct a noop decryptor. + + @return: The decryptor. + """ + return _NullEncryptionContext() + + + +DH_GENERATOR, DH_PRIME = _kex.getDHGeneratorAndPrime( + b'diffie-hellman-group14-sha1') + + +MSG_DISCONNECT = 1 +MSG_IGNORE = 2 +MSG_UNIMPLEMENTED = 3 +MSG_DEBUG = 4 +MSG_SERVICE_REQUEST = 5 +MSG_SERVICE_ACCEPT = 6 +MSG_KEXINIT = 20 +MSG_NEWKEYS = 21 +MSG_KEXDH_INIT = 30 +MSG_KEXDH_REPLY = 31 +MSG_KEX_DH_GEX_REQUEST_OLD = 30 +MSG_KEX_DH_GEX_REQUEST = 34 +MSG_KEX_DH_GEX_GROUP = 31 +MSG_KEX_DH_GEX_INIT = 32 +MSG_KEX_DH_GEX_REPLY = 33 + + + +DISCONNECT_HOST_NOT_ALLOWED_TO_CONNECT = 1 +DISCONNECT_PROTOCOL_ERROR = 2 +DISCONNECT_KEY_EXCHANGE_FAILED = 3 +DISCONNECT_RESERVED = 4 +DISCONNECT_MAC_ERROR = 5 +DISCONNECT_COMPRESSION_ERROR = 6 +DISCONNECT_SERVICE_NOT_AVAILABLE = 7 +DISCONNECT_PROTOCOL_VERSION_NOT_SUPPORTED = 8 +DISCONNECT_HOST_KEY_NOT_VERIFIABLE = 9 +DISCONNECT_CONNECTION_LOST = 10 +DISCONNECT_BY_APPLICATION = 11 +DISCONNECT_TOO_MANY_CONNECTIONS = 12 +DISCONNECT_AUTH_CANCELLED_BY_USER = 13 +DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE = 14 +DISCONNECT_ILLEGAL_USER_NAME = 15 + + + +messages = {} +for name, value in list(globals().items()): + # Avoid legacy messages which overlap with never ones + if name.startswith('MSG_') and not name.startswith('MSG_KEXDH_'): + messages[value] = name +# Check for regressions (#5352) +if 'MSG_KEXDH_INIT' in messages or 'MSG_KEXDH_REPLY' in messages: + raise RuntimeError( + "legacy SSH mnemonics should not end up in messages dict") diff --git a/contrib/python/Twisted/py2/twisted/conch/ssh/userauth.py b/contrib/python/Twisted/py2/twisted/conch/ssh/userauth.py new file mode 100644 index 00000000000..8fab81603af --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ssh/userauth.py @@ -0,0 +1,770 @@ +# -*- test-case-name: twisted.conch.test.test_userauth -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of the ssh-userauth service. +Currently implemented authentication types are public-key and password. + +Maintainer: Paul Swartz +""" + +from __future__ import absolute_import, division + +import struct + +from twisted.conch import error, interfaces +from twisted.conch.ssh import keys, transport, service +from twisted.conch.ssh.common import NS, getNS +from twisted.cred import credentials +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer, reactor +from twisted.python import failure, log +from twisted.python.compat import nativeString, _bytesChr as chr + + + +class SSHUserAuthServer(service.SSHService): + """ + A service implementing the server side of the 'ssh-userauth' service. It + is used to authenticate the user on the other side as being able to access + this server. + + @ivar name: the name of this service: 'ssh-userauth' + @type name: L{bytes} + @ivar authenticatedWith: a list of authentication methods that have + already been used. + @type authenticatedWith: L{list} + @ivar loginTimeout: the number of seconds we wait before disconnecting + the user for taking too long to authenticate + @type loginTimeout: L{int} + @ivar attemptsBeforeDisconnect: the number of failed login attempts we + allow before disconnecting. + @type attemptsBeforeDisconnect: L{int} + @ivar loginAttempts: the number of login attempts that have been made + @type loginAttempts: L{int} + @ivar passwordDelay: the number of seconds to delay when the user gives + an incorrect password + @type passwordDelay: L{int} + @ivar interfaceToMethod: a L{dict} mapping credential interfaces to + authentication methods. The server checks to see which of the + cred interfaces have checkers and tells the client that those methods + are valid for authentication. + @type interfaceToMethod: L{dict} + @ivar supportedAuthentications: A list of the supported authentication + methods. + @type supportedAuthentications: L{list} of L{bytes} + @ivar user: the last username the client tried to authenticate with + @type user: L{bytes} + @ivar method: the current authentication method + @type method: L{bytes} + @ivar nextService: the service the user wants started after authentication + has been completed. + @type nextService: L{bytes} + @ivar portal: the L{twisted.cred.portal.Portal} we are using for + authentication + @type portal: L{twisted.cred.portal.Portal} + @ivar clock: an object with a callLater method. Stubbed out for testing. + """ + + name = b'ssh-userauth' + loginTimeout = 10 * 60 * 60 + # 10 minutes before we disconnect them + attemptsBeforeDisconnect = 20 + # 20 login attempts before a disconnect + passwordDelay = 1 # number of seconds to delay on a failed password + clock = reactor + interfaceToMethod = { + credentials.ISSHPrivateKey : b'publickey', + credentials.IUsernamePassword : b'password', + } + + + def serviceStarted(self): + """ + Called when the userauth service is started. Set up instance + variables, check if we should allow password authentication (only + allow if the outgoing connection is encrypted) and set up a login + timeout. + """ + self.authenticatedWith = [] + self.loginAttempts = 0 + self.user = None + self.nextService = None + self.portal = self.transport.factory.portal + + self.supportedAuthentications = [] + for i in self.portal.listCredentialsInterfaces(): + if i in self.interfaceToMethod: + self.supportedAuthentications.append(self.interfaceToMethod[i]) + + if not self.transport.isEncrypted('in'): + # don't let us transport password in plaintext + if b'password' in self.supportedAuthentications: + self.supportedAuthentications.remove(b'password') + self._cancelLoginTimeout = self.clock.callLater( + self.loginTimeout, + self.timeoutAuthentication) + + + def serviceStopped(self): + """ + Called when the userauth service is stopped. Cancel the login timeout + if it's still going. + """ + if self._cancelLoginTimeout: + self._cancelLoginTimeout.cancel() + self._cancelLoginTimeout = None + + + def timeoutAuthentication(self): + """ + Called when the user has timed out on authentication. Disconnect + with a DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE message. + """ + self._cancelLoginTimeout = None + self.transport.sendDisconnect( + transport.DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, + b'you took too long') + + + def tryAuth(self, kind, user, data): + """ + Try to authenticate the user with the given method. Dispatches to a + auth_* method. + + @param kind: the authentication method to try. + @type kind: L{bytes} + @param user: the username the client is authenticating with. + @type user: L{bytes} + @param data: authentication specific data sent by the client. + @type data: L{bytes} + @return: A Deferred called back if the method succeeded, or erred back + if it failed. + @rtype: C{defer.Deferred} + """ + log.msg('%r trying auth %r' % (user, kind)) + if kind not in self.supportedAuthentications: + return defer.fail( + error.ConchError('unsupported authentication, failing')) + kind = nativeString(kind.replace(b'-', b'_')) + f = getattr(self, 'auth_%s' % (kind,), None) + if f: + ret = f(data) + if not ret: + return defer.fail( + error.ConchError( + '%s return None instead of a Deferred' + % (kind, ))) + else: + return ret + return defer.fail(error.ConchError('bad auth type: %s' % (kind,))) + + + def ssh_USERAUTH_REQUEST(self, packet): + """ + The client has requested authentication. Payload:: + string user + string next service + string method + + + @type packet: L{bytes} + """ + user, nextService, method, rest = getNS(packet, 3) + if user != self.user or nextService != self.nextService: + self.authenticatedWith = [] # clear auth state + self.user = user + self.nextService = nextService + self.method = method + d = self.tryAuth(method, user, rest) + if not d: + self._ebBadAuth( + failure.Failure(error.ConchError('auth returned none'))) + return + d.addCallback(self._cbFinishedAuth) + d.addErrback(self._ebMaybeBadAuth) + d.addErrback(self._ebBadAuth) + return d + + + def _cbFinishedAuth(self, result): + """ + The callback when user has successfully been authenticated. For a + description of the arguments, see L{twisted.cred.portal.Portal.login}. + We start the service requested by the user. + """ + (interface, avatar, logout) = result + self.transport.avatar = avatar + self.transport.logoutFunction = logout + service = self.transport.factory.getService(self.transport, + self.nextService) + if not service: + raise error.ConchError('could not get next service: %s' + % self.nextService) + log.msg('%r authenticated with %r' % (self.user, self.method)) + self.transport.sendPacket(MSG_USERAUTH_SUCCESS, b'') + self.transport.setService(service()) + + + def _ebMaybeBadAuth(self, reason): + """ + An intermediate errback. If the reason is + error.NotEnoughAuthentication, we send a MSG_USERAUTH_FAILURE, but + with the partial success indicator set. + + @type reason: L{twisted.python.failure.Failure} + """ + reason.trap(error.NotEnoughAuthentication) + self.transport.sendPacket(MSG_USERAUTH_FAILURE, + NS(b','.join(self.supportedAuthentications)) + b'\xff') + + + def _ebBadAuth(self, reason): + """ + The final errback in the authentication chain. If the reason is + error.IgnoreAuthentication, we simply return; the authentication + method has sent its own response. Otherwise, send a failure message + and (if the method is not 'none') increment the number of login + attempts. + + @type reason: L{twisted.python.failure.Failure} + """ + if reason.check(error.IgnoreAuthentication): + return + if self.method != b'none': + log.msg('%r failed auth %r' % (self.user, self.method)) + if reason.check(UnauthorizedLogin): + log.msg('unauthorized login: %s' % reason.getErrorMessage()) + elif reason.check(error.ConchError): + log.msg('reason: %s' % reason.getErrorMessage()) + else: + log.msg(reason.getTraceback()) + self.loginAttempts += 1 + if self.loginAttempts > self.attemptsBeforeDisconnect: + self.transport.sendDisconnect( + transport.DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, + b'too many bad auths') + return + self.transport.sendPacket( + MSG_USERAUTH_FAILURE, + NS(b','.join(self.supportedAuthentications)) + b'\x00') + + + def auth_publickey(self, packet): + """ + Public key authentication. Payload:: + byte has signature + string algorithm name + string key blob + [string signature] (if has signature is True) + + Create a SSHPublicKey credential and verify it using our portal. + """ + hasSig = ord(packet[0:1]) + algName, blob, rest = getNS(packet[1:], 2) + + try: + pubKey = keys.Key.fromString(blob) + except keys.BadKeyError: + error = "Unsupported key type %s or bad key" % ( + algName.decode('ascii'),) + log.msg(error) + return defer.fail(UnauthorizedLogin(error)) + + signature = hasSig and getNS(rest)[0] or None + if hasSig: + b = (NS(self.transport.sessionID) + chr(MSG_USERAUTH_REQUEST) + + NS(self.user) + NS(self.nextService) + NS(b'publickey') + + chr(hasSig) + NS(pubKey.sshType()) + NS(blob)) + c = credentials.SSHPrivateKey(self.user, algName, blob, b, + signature) + return self.portal.login(c, None, interfaces.IConchUser) + else: + c = credentials.SSHPrivateKey(self.user, algName, blob, None, None) + return self.portal.login(c, None, + interfaces.IConchUser).addErrback(self._ebCheckKey, + packet[1:]) + + + def _ebCheckKey(self, reason, packet): + """ + Called back if the user did not sent a signature. If reason is + error.ValidPublicKey then this key is valid for the user to + authenticate with. Send MSG_USERAUTH_PK_OK. + """ + reason.trap(error.ValidPublicKey) + # if we make it here, it means that the publickey is valid + self.transport.sendPacket(MSG_USERAUTH_PK_OK, packet) + return failure.Failure(error.IgnoreAuthentication()) + + + def auth_password(self, packet): + """ + Password authentication. Payload:: + string password + + Make a UsernamePassword credential and verify it with our portal. + """ + password = getNS(packet[1:])[0] + c = credentials.UsernamePassword(self.user, password) + return self.portal.login(c, None, interfaces.IConchUser).addErrback( + self._ebPassword) + + + def _ebPassword(self, f): + """ + If the password is invalid, wait before sending the failure in order + to delay brute-force password guessing. + """ + d = defer.Deferred() + self.clock.callLater(self.passwordDelay, d.callback, f) + return d + + + +class SSHUserAuthClient(service.SSHService): + """ + A service implementing the client side of 'ssh-userauth'. + + This service will try all authentication methods provided by the server, + making callbacks for more information when necessary. + + @ivar name: the name of this service: 'ssh-userauth' + @type name: L{str} + @ivar preferredOrder: a list of authentication methods that should be used + first, in order of preference, if supported by the server + @type preferredOrder: L{list} + @ivar user: the name of the user to authenticate as + @type user: L{bytes} + @ivar instance: the service to start after authentication has finished + @type instance: L{service.SSHService} + @ivar authenticatedWith: a list of strings of authentication methods we've tried + @type authenticatedWith: L{list} of L{bytes} + @ivar triedPublicKeys: a list of public key objects that we've tried to + authenticate with + @type triedPublicKeys: L{list} of L{Key} + @ivar lastPublicKey: the last public key object we've tried to authenticate + with + @type lastPublicKey: L{Key} + """ + + name = b'ssh-userauth' + preferredOrder = [b'publickey', b'password', b'keyboard-interactive'] + + + def __init__(self, user, instance): + self.user = user + self.instance = instance + + + def serviceStarted(self): + self.authenticatedWith = [] + self.triedPublicKeys = [] + self.lastPublicKey = None + self.askForAuth(b'none', b'') + + + def askForAuth(self, kind, extraData): + """ + Send a MSG_USERAUTH_REQUEST. + + @param kind: the authentication method to try. + @type kind: L{bytes} + @param extraData: method-specific data to go in the packet + @type extraData: L{bytes} + """ + self.lastAuth = kind + self.transport.sendPacket(MSG_USERAUTH_REQUEST, NS(self.user) + + NS(self.instance.name) + NS(kind) + extraData) + + + def tryAuth(self, kind): + """ + Dispatch to an authentication method. + + @param kind: the authentication method + @type kind: L{bytes} + """ + kind = nativeString(kind.replace(b'-', b'_')) + log.msg('trying to auth with %s' % (kind,)) + f = getattr(self,'auth_%s' % (kind,), None) + if f: + return f() + + + def _ebAuth(self, ignored, *args): + """ + Generic callback for a failed authentication attempt. Respond by + asking for the list of accepted methods (the 'none' method) + """ + self.askForAuth(b'none', b'') + + + def ssh_USERAUTH_SUCCESS(self, packet): + """ + We received a MSG_USERAUTH_SUCCESS. The server has accepted our + authentication, so start the next service. + """ + self.transport.setService(self.instance) + + + def ssh_USERAUTH_FAILURE(self, packet): + """ + We received a MSG_USERAUTH_FAILURE. Payload:: + string methods + byte partial success + + If partial success is C{True}, then the previous method succeeded but is + not sufficient for authentication. C{methods} is a comma-separated list + of accepted authentication methods. + + We sort the list of methods by their position in C{self.preferredOrder}, + removing methods that have already succeeded. We then call + C{self.tryAuth} with the most preferred method. + + @param packet: the C{MSG_USERAUTH_FAILURE} payload. + @type packet: L{bytes} + + @return: a L{defer.Deferred} that will be callbacked with L{None} as + soon as all authentication methods have been tried, or L{None} if no + more authentication methods are available. + @rtype: C{defer.Deferred} or L{None} + """ + canContinue, partial = getNS(packet) + partial = ord(partial) + if partial: + self.authenticatedWith.append(self.lastAuth) + + def orderByPreference(meth): + """ + Invoked once per authentication method in order to extract a + comparison key which is then used for sorting. + + @param meth: the authentication method. + @type meth: L{bytes} + + @return: the comparison key for C{meth}. + @rtype: L{int} + """ + if meth in self.preferredOrder: + return self.preferredOrder.index(meth) + else: + # put the element at the end of the list. + return len(self.preferredOrder) + + canContinue = sorted([meth for meth in canContinue.split(b',') + if meth not in self.authenticatedWith], + key=orderByPreference) + + log.msg('can continue with: %s' % canContinue) + return self._cbUserauthFailure(None, iter(canContinue)) + + + def _cbUserauthFailure(self, result, iterator): + if result: + return + try: + method = next(iterator) + except StopIteration: + self.transport.sendDisconnect( + transport.DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, + b'no more authentication methods available') + else: + d = defer.maybeDeferred(self.tryAuth, method) + d.addCallback(self._cbUserauthFailure, iterator) + return d + + + def ssh_USERAUTH_PK_OK(self, packet): + """ + This message (number 60) can mean several different messages depending + on the current authentication type. We dispatch to individual methods + in order to handle this request. + """ + func = getattr(self, 'ssh_USERAUTH_PK_OK_%s' % + nativeString(self.lastAuth.replace(b'-', b'_')), None) + if func is not None: + return func(packet) + else: + self.askForAuth(b'none', b'') + + + def ssh_USERAUTH_PK_OK_publickey(self, packet): + """ + This is MSG_USERAUTH_PK. Our public key is valid, so we create a + signature and try to authenticate with it. + """ + publicKey = self.lastPublicKey + b = (NS(self.transport.sessionID) + chr(MSG_USERAUTH_REQUEST) + + NS(self.user) + NS(self.instance.name) + NS(b'publickey') + + b'\x01' + NS(publicKey.sshType()) + NS(publicKey.blob())) + d = self.signData(publicKey, b) + if not d: + self.askForAuth(b'none', b'') + # this will fail, we'll move on + return + d.addCallback(self._cbSignedData) + d.addErrback(self._ebAuth) + + + def ssh_USERAUTH_PK_OK_password(self, packet): + """ + This is MSG_USERAUTH_PASSWD_CHANGEREQ. The password given has expired. + We ask for an old password and a new password, then send both back to + the server. + """ + prompt, language, rest = getNS(packet, 2) + self._oldPass = self._newPass = None + d = self.getPassword(b'Old Password: ') + d = d.addCallbacks(self._setOldPass, self._ebAuth) + d.addCallback(lambda ignored: self.getPassword(prompt)) + d.addCallbacks(self._setNewPass, self._ebAuth) + + + def ssh_USERAUTH_PK_OK_keyboard_interactive(self, packet): + """ + This is MSG_USERAUTH_INFO_RESPONSE. The server has sent us the + questions it wants us to answer, so we ask the user and sent the + responses. + """ + name, instruction, lang, data = getNS(packet, 3) + numPrompts = struct.unpack('!L', data[:4])[0] + data = data[4:] + prompts = [] + for i in range(numPrompts): + prompt, data = getNS(data) + echo = bool(ord(data[0:1])) + data = data[1:] + prompts.append((prompt, echo)) + d = self.getGenericAnswers(name, instruction, prompts) + d.addCallback(self._cbGenericAnswers) + d.addErrback(self._ebAuth) + + + def _cbSignedData(self, signedData): + """ + Called back out of self.signData with the signed data. Send the + authentication request with the signature. + + @param signedData: the data signed by the user's private key. + @type signedData: L{bytes} + """ + publicKey = self.lastPublicKey + self.askForAuth(b'publickey', b'\x01' + NS(publicKey.sshType()) + + NS(publicKey.blob()) + NS(signedData)) + + + def _setOldPass(self, op): + """ + Called back when we are choosing a new password. Simply store the old + password for now. + + @param op: the old password as entered by the user + @type op: L{bytes} + """ + self._oldPass = op + + + def _setNewPass(self, np): + """ + Called back when we are choosing a new password. Get the old password + and send the authentication message with both. + + @param np: the new password as entered by the user + @type np: L{bytes} + """ + op = self._oldPass + self._oldPass = None + self.askForAuth(b'password', b'\xff' + NS(op) + NS(np)) + + + def _cbGenericAnswers(self, responses): + """ + Called back when we are finished answering keyboard-interactive + questions. Send the info back to the server in a + MSG_USERAUTH_INFO_RESPONSE. + + @param responses: a list of L{bytes} responses + @type responses: L{list} + """ + data = struct.pack('!L', len(responses)) + for r in responses: + data += NS(r.encode('UTF8')) + self.transport.sendPacket(MSG_USERAUTH_INFO_RESPONSE, data) + + + def auth_publickey(self): + """ + Try to authenticate with a public key. Ask the user for a public key; + if the user has one, send the request to the server and return True. + Otherwise, return False. + + @rtype: L{bool} + """ + d = defer.maybeDeferred(self.getPublicKey) + d.addBoth(self._cbGetPublicKey) + return d + + + def _cbGetPublicKey(self, publicKey): + if not isinstance(publicKey, keys.Key): # failure or None + publicKey = None + if publicKey is not None: + self.lastPublicKey = publicKey + self.triedPublicKeys.append(publicKey) + log.msg('using key of type %s' % publicKey.type()) + self.askForAuth(b'publickey', b'\x00' + NS(publicKey.sshType()) + + NS(publicKey.blob())) + return True + else: + return False + + + def auth_password(self): + """ + Try to authenticate with a password. Ask the user for a password. + If the user will return a password, return True. Otherwise, return + False. + + @rtype: L{bool} + """ + d = self.getPassword() + if d: + d.addCallbacks(self._cbPassword, self._ebAuth) + return True + else: # returned None, don't do password auth + return False + + + def auth_keyboard_interactive(self): + """ + Try to authenticate with keyboard-interactive authentication. Send + the request to the server and return True. + + @rtype: L{bool} + """ + log.msg('authing with keyboard-interactive') + self.askForAuth(b'keyboard-interactive', NS(b'') + NS(b'')) + return True + + + def _cbPassword(self, password): + """ + Called back when the user gives a password. Send the request to the + server. + + @param password: the password the user entered + @type password: L{bytes} + """ + self.askForAuth(b'password', b'\x00' + NS(password)) + + + def signData(self, publicKey, signData): + """ + Sign the given data with the given public key. + + By default, this will call getPrivateKey to get the private key, + then sign the data using Key.sign(). + + This method is factored out so that it can be overridden to use + alternate methods, such as a key agent. + + @param publicKey: The public key object returned from L{getPublicKey} + @type publicKey: L{keys.Key} + + @param signData: the data to be signed by the private key. + @type signData: L{bytes} + @return: a Deferred that's called back with the signature + @rtype: L{defer.Deferred} + """ + key = self.getPrivateKey() + if not key: + return + return key.addCallback(self._cbSignData, signData) + + + def _cbSignData(self, privateKey, signData): + """ + Called back when the private key is returned. Sign the data and + return the signature. + + @param privateKey: the private key object + @type publicKey: L{keys.Key} + @param signData: the data to be signed by the private key. + @type signData: L{bytes} + @return: the signature + @rtype: L{bytes} + """ + return privateKey.sign(signData) + + + def getPublicKey(self): + """ + Return a public key for the user. If no more public keys are + available, return L{None}. + + This implementation always returns L{None}. Override it in a + subclass to actually find and return a public key object. + + @rtype: L{Key} or L{None} + """ + return None + + + def getPrivateKey(self): + """ + Return a L{Deferred} that will be called back with the private key + object corresponding to the last public key from getPublicKey(). + If the private key is not available, errback on the Deferred. + + @rtype: L{Deferred} called back with L{Key} + """ + return defer.fail(NotImplementedError()) + + + def getPassword(self, prompt = None): + """ + Return a L{Deferred} that will be called back with a password. + prompt is a string to display for the password, or None for a generic + 'user@hostname's password: '. + + @type prompt: L{bytes}/L{None} + @rtype: L{defer.Deferred} + """ + return defer.fail(NotImplementedError()) + + + def getGenericAnswers(self, name, instruction, prompts): + """ + Returns a L{Deferred} with the responses to the promopts. + + @param name: The name of the authentication currently in progress. + @param instruction: Describes what the authentication wants. + @param prompts: A list of (prompt, echo) pairs, where prompt is a + string to display and echo is a boolean indicating whether the + user's response should be echoed as they type it. + """ + return defer.fail(NotImplementedError()) + + +MSG_USERAUTH_REQUEST = 50 +MSG_USERAUTH_FAILURE = 51 +MSG_USERAUTH_SUCCESS = 52 +MSG_USERAUTH_BANNER = 53 +MSG_USERAUTH_INFO_RESPONSE = 61 +MSG_USERAUTH_PK_OK = 60 + +messages = {} +for k, v in list(locals().items()): + if k[:4] == 'MSG_': + messages[v] = k + +SSHUserAuthServer.protocolMessages = messages +SSHUserAuthClient.protocolMessages = messages +del messages +del v + +# Doubles, not included in the protocols' mappings +MSG_USERAUTH_PASSWD_CHANGEREQ = 60 +MSG_USERAUTH_INFO_REQUEST = 60 diff --git a/contrib/python/Twisted/py2/twisted/conch/stdio.py b/contrib/python/Twisted/py2/twisted/conch/stdio.py new file mode 100644 index 00000000000..78a88d88862 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/stdio.py @@ -0,0 +1,120 @@ +# -*- test-case-name: twisted.conch.test.test_manhole -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Asynchronous local terminal input handling + +@author: Jp Calderone +""" + +import os, tty, sys, termios + +from twisted.internet import reactor, stdio, protocol, defer +from twisted.python import failure, reflect, log + +from twisted.conch.insults.insults import ServerProtocol +from twisted.conch.manhole import ColoredManhole + +class UnexpectedOutputError(Exception): + pass + + + +class TerminalProcessProtocol(protocol.ProcessProtocol): + def __init__(self, proto): + self.proto = proto + self.onConnection = defer.Deferred() + + + def connectionMade(self): + self.proto.makeConnection(self) + self.onConnection.callback(None) + self.onConnection = None + + + def write(self, data): + """ + Write to the terminal. + + @param data: Data to write. + @type data: L{bytes} + """ + self.transport.write(data) + + + def outReceived(self, data): + """ + Receive data from the terminal. + + @param data: Data received. + @type data: L{bytes} + """ + self.proto.dataReceived(data) + + + def errReceived(self, data): + """ + Report an error. + + @param data: Data to include in L{Failure}. + @type data: L{bytes} + """ + self.transport.loseConnection() + if self.proto is not None: + self.proto.connectionLost(failure.Failure(UnexpectedOutputError(data))) + self.proto = None + + + def childConnectionLost(self, childFD): + if self.proto is not None: + self.proto.childConnectionLost(childFD) + + + def processEnded(self, reason): + if self.proto is not None: + self.proto.connectionLost(reason) + self.proto = None + + + +class ConsoleManhole(ColoredManhole): + """ + A manhole protocol specifically for use with L{stdio.StandardIO}. + """ + def connectionLost(self, reason): + """ + When the connection is lost, there is nothing more to do. Stop the + reactor so that the process can exit. + """ + reactor.stop() + + + +def runWithProtocol(klass): + fd = sys.__stdin__.fileno() + oldSettings = termios.tcgetattr(fd) + tty.setraw(fd) + try: + stdio.StandardIO(ServerProtocol(klass)) + reactor.run() + finally: + termios.tcsetattr(fd, termios.TCSANOW, oldSettings) + os.write(fd, b"\r\x1bc\r") + + + +def main(argv=None): + log.startLogging(open('child.log', 'w')) + + if argv is None: + argv = sys.argv[1:] + if argv: + klass = reflect.namedClass(argv[0]) + else: + klass = ConsoleManhole + runWithProtocol(klass) + + +if __name__ == '__main__': + main() diff --git a/contrib/python/Twisted/py2/twisted/conch/tap.py b/contrib/python/Twisted/py2/twisted/conch/tap.py new file mode 100644 index 00000000000..f622854a068 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/tap.py @@ -0,0 +1,86 @@ +# -*- test-case-name: twisted.conch.test.test_tap -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support module for making SSH servers with twistd. +""" + +from twisted.conch import unix +from twisted.conch import checkers as conch_checkers +from twisted.conch.openssh_compat import factory +from twisted.cred import portal, strcred +from twisted.python import usage +from twisted.application import strports + + +class Options(usage.Options, strcred.AuthOptionMixin): + synopsis = "[-i ] [-p ] [-d ] " + longdesc = ("Makes a Conch SSH server. If no authentication methods are " + "specified, the default authentication methods are UNIX passwords " + "and SSH public keys. If --auth options are " + "passed, only the measures specified will be used.") + optParameters = [ + ["interface", "i", "", "local interface to which we listen"], + ["port", "p", "tcp:22", "Port on which to listen"], + ["data", "d", "/etc", "directory to look for host keys in"], + ["moduli", "", None, "directory to look for moduli in " + "(if different from --data)"] + ] + compData = usage.Completions( + optActions={"data": usage.CompleteDirs(descr="data directory"), + "moduli": usage.CompleteDirs(descr="moduli directory"), + "interface": usage.CompleteNetInterfaces()} + ) + + + def __init__(self, *a, **kw): + usage.Options.__init__(self, *a, **kw) + + # Call the default addCheckers (for backwards compatibility) that will + # be used if no --auth option is provided - note that conch's + # UNIXPasswordDatabase is used, instead of twisted.plugins.cred_unix's + # checker + super(Options, self).addChecker(conch_checkers.UNIXPasswordDatabase()) + super(Options, self).addChecker(conch_checkers.SSHPublicKeyChecker( + conch_checkers.UNIXAuthorizedKeysFiles())) + self._usingDefaultAuth = True + + + def addChecker(self, checker): + """ + Add the checker specified. If any checkers are added, the default + checkers are automatically cleared and the only checkers will be the + specified one(s). + """ + if self._usingDefaultAuth: + self['credCheckers'] = [] + self['credInterfaces'] = {} + self._usingDefaultAuth = False + super(Options, self).addChecker(checker) + + + +def makeService(config): + """ + Construct a service for operating a SSH server. + + @param config: An L{Options} instance specifying server options, including + where server keys are stored and what authentication methods to use. + + @return: A L{twisted.application.service.IService} provider which contains + the requested SSH server. + """ + + t = factory.OpenSSHFactory() + + r = unix.UnixSSHRealm() + t.portal = portal.Portal(r, config.get('credCheckers', [])) + t.dataRoot = config['data'] + t.moduliRoot = config['moduli'] or config['data'] + + port = config['port'] + if config['interface']: + # Add warning here + port += ':interface=' + config['interface'] + return strports.service(port, t) diff --git a/contrib/python/Twisted/py2/twisted/conch/telnet.py b/contrib/python/Twisted/py2/twisted/conch/telnet.py new file mode 100644 index 00000000000..daa27b8aba1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/telnet.py @@ -0,0 +1,1194 @@ +# -*- test-case-name: twisted.conch.test.test_telnet -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Telnet protocol implementation. + +@author: Jean-Paul Calderone +""" + +from __future__ import absolute_import, division + +import struct + +from zope.interface import implementer + +from twisted.internet import protocol, interfaces as iinternet, defer +from twisted.python import log +from twisted.python.compat import _bytesChr as chr, iterbytes + +MODE = chr(1) +EDIT = 1 +TRAPSIG = 2 +MODE_ACK = 4 +SOFT_TAB = 8 +LIT_ECHO = 16 + +# Characters gleaned from the various (and conflicting) RFCs. Not all of these are correct. + +NULL = chr(0) # No operation. +BEL = chr(7) # Produces an audible or + # visible signal (which does + # NOT move the print head). +BS = chr(8) # Moves the print head one + # character position towards + # the left margin. +HT = chr(9) # Moves the printer to the + # next horizontal tab stop. + # It remains unspecified how + # either party determines or + # establishes where such tab + # stops are located. +LF = chr(10) # Moves the printer to the + # next print line, keeping the + # same horizontal position. +VT = chr(11) # Moves the printer to the + # next vertical tab stop. It + # remains unspecified how + # either party determines or + # establishes where such tab + # stops are located. +FF = chr(12) # Moves the printer to the top + # of the next page, keeping + # the same horizontal position. +CR = chr(13) # Moves the printer to the left + # margin of the current line. + +ECHO = chr(1) # User-to-Server: Asks the server to send + # Echos of the transmitted data. +SGA = chr(3) # Suppress Go Ahead. Go Ahead is silly + # and most modern servers should suppress + # it. +NAWS = chr(31) # Negotiate About Window Size. Indicate that + # information about the size of the terminal + # can be communicated. +LINEMODE = chr(34) # Allow line buffering to be + # negotiated about. + +SE = chr(240) # End of subnegotiation parameters. +NOP = chr(241) # No operation. +DM = chr(242) # "Data Mark": The data stream portion + # of a Synch. This should always be + # accompanied by a TCP Urgent + # notification. +BRK = chr(243) # NVT character Break. +IP = chr(244) # The function Interrupt Process. +AO = chr(245) # The function Abort Output +AYT = chr(246) # The function Are You There. +EC = chr(247) # The function Erase Character. +EL = chr(248) # The function Erase Line +GA = chr(249) # The Go Ahead signal. +SB = chr(250) # Indicates that what follows is + # subnegotiation of the indicated + # option. +WILL = chr(251) # Indicates the desire to begin + # performing, or confirmation that + # you are now performing, the + # indicated option. +WONT = chr(252) # Indicates the refusal to perform, + # or continue performing, the + # indicated option. +DO = chr(253) # Indicates the request that the + # other party perform, or + # confirmation that you are expecting + # the other party to perform, the + # indicated option. +DONT = chr(254) # Indicates the demand that the + # other party stop performing, + # or confirmation that you are no + # longer expecting the other party + # to perform, the indicated option. +IAC = chr(255) # Data Byte 255. Introduces a + # telnet command. + +LINEMODE_MODE = chr(1) +LINEMODE_EDIT = chr(1) +LINEMODE_TRAPSIG = chr(2) +LINEMODE_MODE_ACK = chr(4) +LINEMODE_SOFT_TAB = chr(8) +LINEMODE_LIT_ECHO = chr(16) +LINEMODE_FORWARDMASK = chr(2) +LINEMODE_SLC = chr(3) +LINEMODE_SLC_SYNCH = chr(1) +LINEMODE_SLC_BRK = chr(2) +LINEMODE_SLC_IP = chr(3) +LINEMODE_SLC_AO = chr(4) +LINEMODE_SLC_AYT = chr(5) +LINEMODE_SLC_EOR = chr(6) +LINEMODE_SLC_ABORT = chr(7) +LINEMODE_SLC_EOF = chr(8) +LINEMODE_SLC_SUSP = chr(9) +LINEMODE_SLC_EC = chr(10) +LINEMODE_SLC_EL = chr(11) + +LINEMODE_SLC_EW = chr(12) +LINEMODE_SLC_RP = chr(13) +LINEMODE_SLC_LNEXT = chr(14) +LINEMODE_SLC_XON = chr(15) +LINEMODE_SLC_XOFF = chr(16) +LINEMODE_SLC_FORW1 = chr(17) +LINEMODE_SLC_FORW2 = chr(18) +LINEMODE_SLC_MCL = chr(19) +LINEMODE_SLC_MCR = chr(20) +LINEMODE_SLC_MCWL = chr(21) +LINEMODE_SLC_MCWR = chr(22) +LINEMODE_SLC_MCBOL = chr(23) +LINEMODE_SLC_MCEOL = chr(24) +LINEMODE_SLC_INSRT = chr(25) +LINEMODE_SLC_OVER = chr(26) +LINEMODE_SLC_ECR = chr(27) +LINEMODE_SLC_EWR = chr(28) +LINEMODE_SLC_EBOL = chr(29) +LINEMODE_SLC_EEOL = chr(30) + +LINEMODE_SLC_DEFAULT = chr(3) +LINEMODE_SLC_VALUE = chr(2) +LINEMODE_SLC_CANTCHANGE = chr(1) +LINEMODE_SLC_NOSUPPORT = chr(0) +LINEMODE_SLC_LEVELBITS = chr(3) + +LINEMODE_SLC_ACK = chr(128) +LINEMODE_SLC_FLUSHIN = chr(64) +LINEMODE_SLC_FLUSHOUT = chr(32) +LINEMODE_EOF = chr(236) +LINEMODE_SUSP = chr(237) +LINEMODE_ABORT = chr(238) + +class ITelnetProtocol(iinternet.IProtocol): + def unhandledCommand(command, argument): + """ + A command was received but not understood. + + @param command: the command received. + @type command: L{str}, a single character. + @param argument: the argument to the received command. + @type argument: L{str}, a single character, or None if the command that + was unhandled does not provide an argument. + """ + + + def unhandledSubnegotiation(command, data): + """ + A subnegotiation command was received but not understood. + + @param command: the command being subnegotiated. That is, the first + byte after the SB command. + @type command: L{str}, a single character. + @param data: all other bytes of the subneogation. That is, all but the + first bytes between SB and SE, with IAC un-escaping applied. + @type data: L{bytes}, each a single character + """ + + + def enableLocal(option): + """ + Enable the given option locally. + + This should enable the given option on this side of the + telnet connection and return True. If False is returned, + the option will be treated as still disabled and the peer + will be notified. + + @param option: the option to be enabled. + @type option: L{bytes}, a single character. + """ + + + def enableRemote(option): + """ + Indicate whether the peer should be allowed to enable this option. + + Returns True if the peer should be allowed to enable this option, + False otherwise. + + @param option: the option to be enabled. + @type option: L{bytes}, a single character. + """ + + + def disableLocal(option): + """ + Disable the given option locally. + + Unlike enableLocal, this method cannot fail. The option must be + disabled. + + @param option: the option to be disabled. + @type option: L{bytes}, a single character. + """ + + + def disableRemote(option): + """ + Indicate that the peer has disabled this option. + + @param option: the option to be disabled. + @type option: L{bytes}, a single character. + """ + + + +class ITelnetTransport(iinternet.ITransport): + def do(option): + """ + Indicate a desire for the peer to begin performing the given option. + + Returns a Deferred that fires with True when the peer begins performing + the option, or fails with L{OptionRefused} when the peer refuses to + perform it. If the peer is already performing the given option, the + Deferred will fail with L{AlreadyEnabled}. If a negotiation regarding + this option is already in progress, the Deferred will fail with + L{AlreadyNegotiating}. + + Note: It is currently possible that this Deferred will never fire, + if the peer never responds, or if the peer believes the option to + already be enabled. + """ + + + def dont(option): + """ + Indicate a desire for the peer to cease performing the given option. + + Returns a Deferred that fires with True when the peer ceases performing + the option. If the peer is not performing the given option, the + Deferred will fail with L{AlreadyDisabled}. If negotiation regarding + this option is already in progress, the Deferred will fail with + L{AlreadyNegotiating}. + + Note: It is currently possible that this Deferred will never fire, + if the peer never responds, or if the peer believes the option to + already be disabled. + """ + + + def will(option): + """ + Indicate our willingness to begin performing this option locally. + + Returns a Deferred that fires with True when the peer agrees to allow us + to begin performing this option, or fails with L{OptionRefused} if the + peer refuses to allow us to begin performing it. If the option is + already enabled locally, the Deferred will fail with L{AlreadyEnabled}. + If negotiation regarding this option is already in progress, the + Deferred will fail with L{AlreadyNegotiating}. + + Note: It is currently possible that this Deferred will never fire, + if the peer never responds, or if the peer believes the option to + already be enabled. + """ + + + def wont(option): + """ + Indicate that we will stop performing the given option. + + Returns a Deferred that fires with True when the peer acknowledges + we have stopped performing this option. If the option is already + disabled locally, the Deferred will fail with L{AlreadyDisabled}. + If negotiation regarding this option is already in progress, + the Deferred will fail with L{AlreadyNegotiating}. + + Note: It is currently possible that this Deferred will never fire, + if the peer never responds, or if the peer believes the option to + already be disabled. + """ + + + def requestNegotiation(about, data): + """ + Send a subnegotiation request. + + @param about: A byte indicating the feature being negotiated. + @param data: Any number of L{bytes} containing specific information + about the negotiation being requested. No values in this string + need to be escaped, as this function will escape any value which + requires it. + """ + + + +class TelnetError(Exception): + pass + + + +class NegotiationError(TelnetError): + def __str__(self): + return self.__class__.__module__ + '.' + self.__class__.__name__ + ':' + repr(self.args[0]) + + + +class OptionRefused(NegotiationError): + pass + + + +class AlreadyEnabled(NegotiationError): + pass + + + +class AlreadyDisabled(NegotiationError): + pass + + + +class AlreadyNegotiating(NegotiationError): + pass + + + +@implementer(ITelnetProtocol) +class TelnetProtocol(protocol.Protocol): + def unhandledCommand(self, command, argument): + pass + + + def unhandledSubnegotiation(self, command, data): + pass + + + def enableLocal(self, option): + pass + + + def enableRemote(self, option): + pass + + + def disableLocal(self, option): + pass + + + def disableRemote(self, option): + pass + + + +class Telnet(protocol.Protocol): + """ + @ivar commandMap: A mapping of bytes to callables. When a + telnet command is received, the command byte (the first byte + after IAC) is looked up in this dictionary. If a callable is + found, it is invoked with the argument of the command, or None + if the command takes no argument. Values should be added to + this dictionary if commands wish to be handled. By default, + only WILL, WONT, DO, and DONT are handled. These should not + be overridden, as this class handles them correctly and + provides an API for interacting with them. + + @ivar negotiationMap: A mapping of bytes to callables. When + a subnegotiation command is received, the command byte (the + first byte after SB) is looked up in this dictionary. If + a callable is found, it is invoked with the argument of the + subnegotiation. Values should be added to this dictionary if + subnegotiations are to be handled. By default, no values are + handled. + + @ivar options: A mapping of option bytes to their current + state. This state is likely of little use to user code. + Changes should not be made to it. + + @ivar state: A string indicating the current parse state. It + can take on the values "data", "escaped", "command", "newline", + "subnegotiation", and "subnegotiation-escaped". Changes + should not be made to it. + + @ivar transport: This protocol's transport object. + """ + + # One of a lot of things + state = 'data' + + def __init__(self): + self.options = {} + self.negotiationMap = {} + self.commandMap = { + WILL: self.telnet_WILL, + WONT: self.telnet_WONT, + DO: self.telnet_DO, + DONT: self.telnet_DONT} + + + def _write(self, data): + self.transport.write(data) + + + class _OptionState: + """ + Represents the state of an option on both sides of a telnet + connection. + + @ivar us: The state of the option on this side of the connection. + + @ivar him: The state of the option on the other side of the + connection. + """ + class _Perspective: + """ + Represents the state of an option on side of the telnet + connection. Some options can be enabled on a particular side of + the connection (RFC 1073 for example: only the client can have + NAWS enabled). Other options can be enabled on either or both + sides (such as RFC 1372: each side can have its own flow control + state). + + @ivar state: C{'yes'} or C{'no'} indicating whether or not this + option is enabled on one side of the connection. + + @ivar negotiating: A boolean tracking whether negotiation about + this option is in progress. + + @ivar onResult: When negotiation about this option has been + initiated by this side of the connection, a L{Deferred} + which will fire with the result of the negotiation. L{None} + at other times. + """ + state = 'no' + negotiating = False + onResult = None + + def __str__(self): + return self.state + ('*' * self.negotiating) + + + def __init__(self): + self.us = self._Perspective() + self.him = self._Perspective() + + + def __repr__(self): + return '<_OptionState us=%s him=%s>' % (self.us, self.him) + + + def getOptionState(self, opt): + return self.options.setdefault(opt, self._OptionState()) + + + def _do(self, option): + self._write(IAC + DO + option) + + + def _dont(self, option): + self._write(IAC + DONT + option) + + + def _will(self, option): + self._write(IAC + WILL + option) + + + def _wont(self, option): + self._write(IAC + WONT + option) + + + def will(self, option): + """ + Indicate our willingness to enable an option. + """ + s = self.getOptionState(option) + if s.us.negotiating or s.him.negotiating: + return defer.fail(AlreadyNegotiating(option)) + elif s.us.state == 'yes': + return defer.fail(AlreadyEnabled(option)) + else: + s.us.negotiating = True + s.us.onResult = d = defer.Deferred() + self._will(option) + return d + + + def wont(self, option): + """ + Indicate we are not willing to enable an option. + """ + s = self.getOptionState(option) + if s.us.negotiating or s.him.negotiating: + return defer.fail(AlreadyNegotiating(option)) + elif s.us.state == 'no': + return defer.fail(AlreadyDisabled(option)) + else: + s.us.negotiating = True + s.us.onResult = d = defer.Deferred() + self._wont(option) + return d + + + def do(self, option): + s = self.getOptionState(option) + if s.us.negotiating or s.him.negotiating: + return defer.fail(AlreadyNegotiating(option)) + elif s.him.state == 'yes': + return defer.fail(AlreadyEnabled(option)) + else: + s.him.negotiating = True + s.him.onResult = d = defer.Deferred() + self._do(option) + return d + + + def dont(self, option): + s = self.getOptionState(option) + if s.us.negotiating or s.him.negotiating: + return defer.fail(AlreadyNegotiating(option)) + elif s.him.state == 'no': + return defer.fail(AlreadyDisabled(option)) + else: + s.him.negotiating = True + s.him.onResult = d = defer.Deferred() + self._dont(option) + return d + + + def requestNegotiation(self, about, data): + """ + Send a negotiation message for the option C{about} with C{data} as the + payload. + + @param data: the payload + @type data: L{bytes} + @see: L{ITelnetTransport.requestNegotiation} + """ + data = data.replace(IAC, IAC * 2) + self._write(IAC + SB + about + data + IAC + SE) + + + def dataReceived(self, data): + appDataBuffer = [] + + for b in iterbytes(data): + if self.state == 'data': + if b == IAC: + self.state = 'escaped' + elif b == b'\r': + self.state = 'newline' + else: + appDataBuffer.append(b) + elif self.state == 'escaped': + if b == IAC: + appDataBuffer.append(b) + self.state = 'data' + elif b == SB: + self.state = 'subnegotiation' + self.commands = [] + elif b in (NOP, DM, BRK, IP, AO, AYT, EC, EL, GA): + self.state = 'data' + if appDataBuffer: + self.applicationDataReceived(b''.join(appDataBuffer)) + del appDataBuffer[:] + self.commandReceived(b, None) + elif b in (WILL, WONT, DO, DONT): + self.state = 'command' + self.command = b + else: + raise ValueError("Stumped", b) + elif self.state == 'command': + self.state = 'data' + command = self.command + del self.command + if appDataBuffer: + self.applicationDataReceived(b''.join(appDataBuffer)) + del appDataBuffer[:] + self.commandReceived(command, b) + elif self.state == 'newline': + self.state = 'data' + if b == b'\n': + appDataBuffer.append(b'\n') + elif b == b'\0': + appDataBuffer.append(b'\r') + elif b == IAC: + # IAC isn't really allowed after \r, according to the + # RFC, but handling it this way is less surprising than + # delivering the IAC to the app as application data. + # The purpose of the restriction is to allow terminals + # to unambiguously interpret the behavior of the CR + # after reading only one more byte. CR LF is supposed + # to mean one thing (cursor to next line, first column), + # CR NUL another (cursor to first column). Absent the + # NUL, it still makes sense to interpret this as CR and + # then apply all the usual interpretation to the IAC. + appDataBuffer.append(b'\r') + self.state = 'escaped' + else: + appDataBuffer.append(b'\r' + b) + elif self.state == 'subnegotiation': + if b == IAC: + self.state = 'subnegotiation-escaped' + else: + self.commands.append(b) + elif self.state == 'subnegotiation-escaped': + if b == SE: + self.state = 'data' + commands = self.commands + del self.commands + if appDataBuffer: + self.applicationDataReceived(b''.join(appDataBuffer)) + del appDataBuffer[:] + self.negotiate(commands) + else: + self.state = 'subnegotiation' + self.commands.append(b) + else: + raise ValueError("How'd you do this?") + + if appDataBuffer: + self.applicationDataReceived(b''.join(appDataBuffer)) + + + def connectionLost(self, reason): + for state in self.options.values(): + if state.us.onResult is not None: + d = state.us.onResult + state.us.onResult = None + d.errback(reason) + if state.him.onResult is not None: + d = state.him.onResult + state.him.onResult = None + d.errback(reason) + + + def applicationDataReceived(self, data): + """ + Called with application-level data. + """ + + def unhandledCommand(self, command, argument): + """ + Called for commands for which no handler is installed. + """ + + + def commandReceived(self, command, argument): + cmdFunc = self.commandMap.get(command) + if cmdFunc is None: + self.unhandledCommand(command, argument) + else: + cmdFunc(argument) + + + def unhandledSubnegotiation(self, command, data): + """ + Called for subnegotiations for which no handler is installed. + """ + + + def negotiate(self, data): + command, data = data[0], data[1:] + cmdFunc = self.negotiationMap.get(command) + if cmdFunc is None: + self.unhandledSubnegotiation(command, data) + else: + cmdFunc(data) + + + def telnet_WILL(self, option): + s = self.getOptionState(option) + self.willMap[s.him.state, s.him.negotiating](self, s, option) + + + def will_no_false(self, state, option): + # He is unilaterally offering to enable an option. + if self.enableRemote(option): + state.him.state = 'yes' + self._do(option) + else: + self._dont(option) + + + def will_no_true(self, state, option): + # Peer agreed to enable an option in response to our request. + state.him.state = 'yes' + state.him.negotiating = False + d = state.him.onResult + state.him.onResult = None + d.callback(True) + assert self.enableRemote(option), "enableRemote must return True in this context (for option %r)" % (option,) + + + def will_yes_false(self, state, option): + # He is unilaterally offering to enable an already-enabled option. + # Ignore this. + pass + + + def will_yes_true(self, state, option): + # This is a bogus state. It is here for completeness. It will + # never be entered. + assert False, "will_yes_true can never be entered, but was called with %r, %r" % (state, option) + + willMap = {('no', False): will_no_false, ('no', True): will_no_true, + ('yes', False): will_yes_false, ('yes', True): will_yes_true} + + + def telnet_WONT(self, option): + s = self.getOptionState(option) + self.wontMap[s.him.state, s.him.negotiating](self, s, option) + + + def wont_no_false(self, state, option): + # He is unilaterally demanding that an already-disabled option be/remain disabled. + # Ignore this (although we could record it and refuse subsequent enable attempts + # from our side - he can always refuse them again though, so we won't) + pass + + + def wont_no_true(self, state, option): + # Peer refused to enable an option in response to our request. + state.him.negotiating = False + d = state.him.onResult + state.him.onResult = None + d.errback(OptionRefused(option)) + + + def wont_yes_false(self, state, option): + # Peer is unilaterally demanding that an option be disabled. + state.him.state = 'no' + self.disableRemote(option) + self._dont(option) + + + def wont_yes_true(self, state, option): + # Peer agreed to disable an option at our request. + state.him.state = 'no' + state.him.negotiating = False + d = state.him.onResult + state.him.onResult = None + d.callback(True) + self.disableRemote(option) + + wontMap = {('no', False): wont_no_false, ('no', True): wont_no_true, + ('yes', False): wont_yes_false, ('yes', True): wont_yes_true} + + + def telnet_DO(self, option): + s = self.getOptionState(option) + self.doMap[s.us.state, s.us.negotiating](self, s, option) + + + def do_no_false(self, state, option): + # Peer is unilaterally requesting that we enable an option. + if self.enableLocal(option): + state.us.state = 'yes' + self._will(option) + else: + self._wont(option) + + + def do_no_true(self, state, option): + # Peer agreed to allow us to enable an option at our request. + state.us.state = 'yes' + state.us.negotiating = False + d = state.us.onResult + state.us.onResult = None + d.callback(True) + self.enableLocal(option) + + + def do_yes_false(self, state, option): + # Peer is unilaterally requesting us to enable an already-enabled option. + # Ignore this. + pass + + + def do_yes_true(self, state, option): + # This is a bogus state. It is here for completeness. It will never be + # entered. + assert False, "do_yes_true can never be entered, but was called with %r, %r" % (state, option) + + doMap = {('no', False): do_no_false, ('no', True): do_no_true, + ('yes', False): do_yes_false, ('yes', True): do_yes_true} + + + def telnet_DONT(self, option): + s = self.getOptionState(option) + self.dontMap[s.us.state, s.us.negotiating](self, s, option) + + + def dont_no_false(self, state, option): + # Peer is unilaterally demanding us to disable an already-disabled option. + # Ignore this. + pass + + + def dont_no_true(self, state, option): + # Offered option was refused. Fail the Deferred returned by the + # previous will() call. + state.us.negotiating = False + d = state.us.onResult + state.us.onResult = None + d.errback(OptionRefused(option)) + + + def dont_yes_false(self, state, option): + # Peer is unilaterally demanding we disable an option. + state.us.state = 'no' + self.disableLocal(option) + self._wont(option) + + + def dont_yes_true(self, state, option): + # Peer acknowledged our notice that we will disable an option. + state.us.state = 'no' + state.us.negotiating = False + d = state.us.onResult + state.us.onResult = None + d.callback(True) + self.disableLocal(option) + + dontMap = {('no', False): dont_no_false, ('no', True): dont_no_true, + ('yes', False): dont_yes_false, ('yes', True): dont_yes_true} + + + def enableLocal(self, option): + """ + Reject all attempts to enable options. + """ + return False + + + def enableRemote(self, option): + """ + Reject all attempts to enable options. + """ + return False + + + def disableLocal(self, option): + """ + Signal a programming error by raising an exception. + + L{enableLocal} must return true for the given value of C{option} in + order for this method to be called. If a subclass of L{Telnet} + overrides enableLocal to allow certain options to be enabled, it must + also override disableLocal to disable those options. + + @raise NotImplementedError: Always raised. + """ + raise NotImplementedError( + "Don't know how to disable local telnet option %r" % (option,)) + + + def disableRemote(self, option): + """ + Signal a programming error by raising an exception. + + L{enableRemote} must return true for the given value of C{option} in + order for this method to be called. If a subclass of L{Telnet} + overrides enableRemote to allow certain options to be enabled, it must + also override disableRemote tto disable those options. + + @raise NotImplementedError: Always raised. + """ + raise NotImplementedError( + "Don't know how to disable remote telnet option %r" % (option,)) + + + +class ProtocolTransportMixin: + def write(self, data): + self.transport.write(data.replace(b'\n', b'\r\n')) + + + def writeSequence(self, seq): + self.transport.writeSequence(seq) + + + def loseConnection(self): + self.transport.loseConnection() + + + def getHost(self): + return self.transport.getHost() + + + def getPeer(self): + return self.transport.getPeer() + + + +class TelnetTransport(Telnet, ProtocolTransportMixin): + """ + @ivar protocol: An instance of the protocol to which this + transport is connected, or None before the connection is + established and after it is lost. + + @ivar protocolFactory: A callable which returns protocol instances + which provide L{ITelnetProtocol}. This will be invoked when a + connection is established. It is passed *protocolArgs and + **protocolKwArgs. + + @ivar protocolArgs: A tuple of additional arguments to + pass to protocolFactory. + + @ivar protocolKwArgs: A dictionary of additional arguments + to pass to protocolFactory. + """ + + disconnecting = False + + protocolFactory = None + protocol = None + + def __init__(self, protocolFactory=None, *a, **kw): + Telnet.__init__(self) + if protocolFactory is not None: + self.protocolFactory = protocolFactory + self.protocolArgs = a + self.protocolKwArgs = kw + + + def connectionMade(self): + if self.protocolFactory is not None: + self.protocol = self.protocolFactory(*self.protocolArgs, **self.protocolKwArgs) + assert ITelnetProtocol.providedBy(self.protocol) + try: + factory = self.factory + except AttributeError: + pass + else: + self.protocol.factory = factory + self.protocol.makeConnection(self) + + + def connectionLost(self, reason): + Telnet.connectionLost(self, reason) + if self.protocol is not None: + try: + self.protocol.connectionLost(reason) + finally: + del self.protocol + + + def enableLocal(self, option): + return self.protocol.enableLocal(option) + + + def enableRemote(self, option): + return self.protocol.enableRemote(option) + + + def disableLocal(self, option): + return self.protocol.disableLocal(option) + + + def disableRemote(self, option): + return self.protocol.disableRemote(option) + + + def unhandledSubnegotiation(self, command, data): + self.protocol.unhandledSubnegotiation(command, data) + + + def unhandledCommand(self, command, argument): + self.protocol.unhandledCommand(command, argument) + + + def applicationDataReceived(self, data): + self.protocol.dataReceived(data) + + + def write(self, data): + ProtocolTransportMixin.write(self, data.replace(b'\xff', b'\xff\xff')) + + + +class TelnetBootstrapProtocol(TelnetProtocol, ProtocolTransportMixin): + protocol = None + + def __init__(self, protocolFactory, *args, **kw): + self.protocolFactory = protocolFactory + self.protocolArgs = args + self.protocolKwArgs = kw + + + def connectionMade(self): + self.transport.negotiationMap[NAWS] = self.telnet_NAWS + self.transport.negotiationMap[LINEMODE] = self.telnet_LINEMODE + + for opt in (LINEMODE, NAWS, SGA): + self.transport.do(opt).addErrback(log.err) + for opt in (ECHO,): + self.transport.will(opt).addErrback(log.err) + + self.protocol = self.protocolFactory(*self.protocolArgs, **self.protocolKwArgs) + + try: + factory = self.factory + except AttributeError: + pass + else: + self.protocol.factory = factory + + self.protocol.makeConnection(self) + + + def connectionLost(self, reason): + if self.protocol is not None: + try: + self.protocol.connectionLost(reason) + finally: + del self.protocol + + + def dataReceived(self, data): + self.protocol.dataReceived(data) + + + def enableLocal(self, opt): + if opt == ECHO: + return True + elif opt == SGA: + return True + else: + return False + + + def enableRemote(self, opt): + if opt == LINEMODE: + self.transport.requestNegotiation(LINEMODE, MODE + chr(TRAPSIG)) + return True + elif opt == NAWS: + return True + elif opt == SGA: + return True + else: + return False + + + def telnet_NAWS(self, data): + # NAWS is client -> server *only*. self.protocol will + # therefore be an ITerminalTransport, the `.protocol' + # attribute of which will be an ITerminalProtocol. Maybe. + # You know what, XXX TODO clean this up. + if len(data) == 4: + width, height = struct.unpack('!HH', b''.join(data)) + self.protocol.terminalProtocol.terminalSize(width, height) + else: + log.msg("Wrong number of NAWS bytes") + + linemodeSubcommands = { + LINEMODE_SLC: 'SLC'} + def telnet_LINEMODE(self, data): + linemodeSubcommand = data[0] + if 0: + # XXX TODO: This should be enabled to parse linemode subnegotiation. + getattr(self, 'linemode_' + self.linemodeSubcommands[linemodeSubcommand])(data[1:]) + + + def linemode_SLC(self, data): + chunks = zip(*[iter(data)]*3) + for slcFunction, slcValue, slcWhat in chunks: + # Later, we should parse stuff. + 'SLC', ord(slcFunction), ord(slcValue), ord(slcWhat) + + +from twisted.protocols import basic + +class StatefulTelnetProtocol(basic.LineReceiver, TelnetProtocol): + delimiter = b'\n' + + state = 'Discard' + + def connectionLost(self, reason): + basic.LineReceiver.connectionLost(self, reason) + TelnetProtocol.connectionLost(self, reason) + + + def lineReceived(self, line): + oldState = self.state + newState = getattr(self, "telnet_" + oldState)(line) + if newState is not None: + if self.state == oldState: + self.state = newState + else: + log.msg("Warning: state changed and new state returned") + + + def telnet_Discard(self, line): + pass + + +from twisted.cred import credentials + +class AuthenticatingTelnetProtocol(StatefulTelnetProtocol): + """ + A protocol which prompts for credentials and attempts to authenticate them. + + Username and password prompts are given (the password is obscured). When the + information is collected, it is passed to a portal and an avatar implementing + L{ITelnetProtocol} is requested. If an avatar is returned, it connected to this + protocol's transport, and this protocol's transport is connected to it. + Otherwise, the user is re-prompted for credentials. + """ + + state = "User" + protocol = None + + def __init__(self, portal): + self.portal = portal + + + def connectionMade(self): + self.transport.write(b"Username: ") + + + def connectionLost(self, reason): + StatefulTelnetProtocol.connectionLost(self, reason) + if self.protocol is not None: + try: + self.protocol.connectionLost(reason) + self.logout() + finally: + del self.protocol, self.logout + + + def telnet_User(self, line): + self.username = line + self.transport.will(ECHO) + self.transport.write(b"Password: ") + return 'Password' + + + def telnet_Password(self, line): + username, password = self.username, line + del self.username + def login(ignored): + creds = credentials.UsernamePassword(username, password) + d = self.portal.login(creds, None, ITelnetProtocol) + d.addCallback(self._cbLogin) + d.addErrback(self._ebLogin) + self.transport.wont(ECHO).addCallback(login) + return 'Discard' + + + def _cbLogin(self, ial): + interface, protocol, logout = ial + assert interface is ITelnetProtocol + self.protocol = protocol + self.logout = logout + self.state = 'Command' + + protocol.makeConnection(self.transport) + self.transport.protocol = protocol + + + def _ebLogin(self, failure): + self.transport.write(b"\nAuthentication failed\n") + self.transport.write(b"Username: ") + self.state = "User" + + +__all__ = [ + # Exceptions + 'TelnetError', 'NegotiationError', 'OptionRefused', + 'AlreadyNegotiating', 'AlreadyEnabled', 'AlreadyDisabled', + + # Interfaces + 'ITelnetProtocol', 'ITelnetTransport', + + # Other stuff, protocols, etc. + 'Telnet', 'TelnetProtocol', 'TelnetTransport', + 'TelnetBootstrapProtocol', + + ] diff --git a/contrib/python/Twisted/py2/twisted/conch/ttymodes.py b/contrib/python/Twisted/py2/twisted/conch/ttymodes.py new file mode 100644 index 00000000000..00b4495f3a8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ttymodes.py @@ -0,0 +1,121 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + +import tty +# this module was autogenerated. + +VINTR = 1 +VQUIT = 2 +VERASE = 3 +VKILL = 4 +VEOF = 5 +VEOL = 6 +VEOL2 = 7 +VSTART = 8 +VSTOP = 9 +VSUSP = 10 +VDSUSP = 11 +VREPRINT = 12 +VWERASE = 13 +VLNEXT = 14 +VFLUSH = 15 +VSWTCH = 16 +VSTATUS = 17 +VDISCARD = 18 +IGNPAR = 30 +PARMRK = 31 +INPCK = 32 +ISTRIP = 33 +INLCR = 34 +IGNCR = 35 +ICRNL = 36 +IUCLC = 37 +IXON = 38 +IXANY = 39 +IXOFF = 40 +IMAXBEL = 41 +ISIG = 50 +ICANON = 51 +XCASE = 52 +ECHO = 53 +ECHOE = 54 +ECHOK = 55 +ECHONL = 56 +NOFLSH = 57 +TOSTOP = 58 +IEXTEN = 59 +ECHOCTL = 60 +ECHOKE = 61 +PENDIN = 62 +OPOST = 70 +OLCUC = 71 +ONLCR = 72 +OCRNL = 73 +ONOCR = 74 +ONLRET = 75 +CS7 = 90 +CS8 = 91 +PARENB = 92 +PARODD = 93 +TTY_OP_ISPEED = 128 +TTY_OP_OSPEED = 129 + +TTYMODES = { + 1 : 'VINTR', + 2 : 'VQUIT', + 3 : 'VERASE', + 4 : 'VKILL', + 5 : 'VEOF', + 6 : 'VEOL', + 7 : 'VEOL2', + 8 : 'VSTART', + 9 : 'VSTOP', + 10 : 'VSUSP', + 11 : 'VDSUSP', + 12 : 'VREPRINT', + 13 : 'VWERASE', + 14 : 'VLNEXT', + 15 : 'VFLUSH', + 16 : 'VSWTCH', + 17 : 'VSTATUS', + 18 : 'VDISCARD', + 30 : (tty.IFLAG, 'IGNPAR'), + 31 : (tty.IFLAG, 'PARMRK'), + 32 : (tty.IFLAG, 'INPCK'), + 33 : (tty.IFLAG, 'ISTRIP'), + 34 : (tty.IFLAG, 'INLCR'), + 35 : (tty.IFLAG, 'IGNCR'), + 36 : (tty.IFLAG, 'ICRNL'), + 37 : (tty.IFLAG, 'IUCLC'), + 38 : (tty.IFLAG, 'IXON'), + 39 : (tty.IFLAG, 'IXANY'), + 40 : (tty.IFLAG, 'IXOFF'), + 41 : (tty.IFLAG, 'IMAXBEL'), + 50 : (tty.LFLAG, 'ISIG'), + 51 : (tty.LFLAG, 'ICANON'), + 52 : (tty.LFLAG, 'XCASE'), + 53 : (tty.LFLAG, 'ECHO'), + 54 : (tty.LFLAG, 'ECHOE'), + 55 : (tty.LFLAG, 'ECHOK'), + 56 : (tty.LFLAG, 'ECHONL'), + 57 : (tty.LFLAG, 'NOFLSH'), + 58 : (tty.LFLAG, 'TOSTOP'), + 59 : (tty.LFLAG, 'IEXTEN'), + 60 : (tty.LFLAG, 'ECHOCTL'), + 61 : (tty.LFLAG, 'ECHOKE'), + 62 : (tty.LFLAG, 'PENDIN'), + 70 : (tty.OFLAG, 'OPOST'), + 71 : (tty.OFLAG, 'OLCUC'), + 72 : (tty.OFLAG, 'ONLCR'), + 73 : (tty.OFLAG, 'OCRNL'), + 74 : (tty.OFLAG, 'ONOCR'), + 75 : (tty.OFLAG, 'ONLRET'), +# 90 : (tty.CFLAG, 'CS7'), +# 91 : (tty.CFLAG, 'CS8'), + 92 : (tty.CFLAG, 'PARENB'), + 93 : (tty.CFLAG, 'PARODD'), + 128 : 'ISPEED', + 129 : 'OSPEED' +} diff --git a/contrib/python/Twisted/py2/twisted/conch/ui/__init__.py b/contrib/python/Twisted/py2/twisted/conch/ui/__init__.py new file mode 100644 index 00000000000..ea0eea83183 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ui/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + + +""" +twisted.conch.ui is home to the UI elements for tkconch. + +Maintainer: Paul Swartz +""" diff --git a/contrib/python/Twisted/py2/twisted/conch/ui/ansi.py b/contrib/python/Twisted/py2/twisted/conch/ui/ansi.py new file mode 100644 index 00000000000..b0735326425 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ui/ansi.py @@ -0,0 +1,240 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# +"""Module to parse ANSI escape sequences + +Maintainer: Jean-Paul Calderone +""" + +import string + +# Twisted imports +from twisted.python import log + +class ColorText: + """ + Represents an element of text along with the texts colors and + additional attributes. + """ + + # The colors to use + COLORS = ('b', 'r', 'g', 'y', 'l', 'm', 'c', 'w') + BOLD_COLORS = tuple([x.upper() for x in COLORS]) + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(len(COLORS)) + + # Color names + COLOR_NAMES = ( + 'Black', 'Red', 'Green', 'Yellow', 'Blue', 'Magenta', 'Cyan', 'White' + ) + + def __init__(self, text, fg, bg, display, bold, underline, flash, reverse): + self.text, self.fg, self.bg = text, fg, bg + self.display = display + self.bold = bold + self.underline = underline + self.flash = flash + self.reverse = reverse + if self.reverse: + self.fg, self.bg = self.bg, self.fg + + +class AnsiParser: + """ + Parser class for ANSI codes. + """ + + # Terminators for cursor movement ansi controls - unsupported + CURSOR_SET = ('H', 'f', 'A', 'B', 'C', 'D', 'R', 's', 'u', 'd','G') + + # Terminators for erasure ansi controls - unsupported + ERASE_SET = ('J', 'K', 'P') + + # Terminators for mode change ansi controls - unsupported + MODE_SET = ('h', 'l') + + # Terminators for keyboard assignment ansi controls - unsupported + ASSIGN_SET = ('p',) + + # Terminators for color change ansi controls - supported + COLOR_SET = ('m',) + + SETS = (CURSOR_SET, ERASE_SET, MODE_SET, ASSIGN_SET, COLOR_SET) + + def __init__(self, defaultFG, defaultBG): + self.defaultFG, self.defaultBG = defaultFG, defaultBG + self.currentFG, self.currentBG = self.defaultFG, self.defaultBG + self.bold, self.flash, self.underline, self.reverse = 0, 0, 0, 0 + self.display = 1 + self.prepend = '' + + + def stripEscapes(self, string): + """ + Remove all ANSI color escapes from the given string. + """ + result = '' + show = 1 + i = 0 + L = len(string) + while i < L: + if show == 0 and string[i] in _sets: + show = 1 + elif show: + n = string.find('\x1B', i) + if n == -1: + return result + string[i:] + else: + result = result + string[i:n] + i = n + show = 0 + i = i + 1 + return result + + def writeString(self, colorstr): + pass + + def parseString(self, str): + """ + Turn a string input into a list of L{ColorText} elements. + """ + + if self.prepend: + str = self.prepend + str + self.prepend = '' + parts = str.split('\x1B') + + if len(parts) == 1: + self.writeString(self.formatText(parts[0])) + else: + self.writeString(self.formatText(parts[0])) + for s in parts[1:]: + L = len(s) + i = 0 + type = None + while i < L: + if s[i] not in string.digits+'[;?': + break + i+=1 + if not s: + self.prepend = '\x1b' + return + if s[0]!='[': + self.writeString(self.formatText(s[i+1:])) + continue + else: + s=s[1:] + i-=1 + if i==L-1: + self.prepend = '\x1b[' + return + type = _setmap.get(s[i], None) + if type is None: + continue + + if type == AnsiParser.COLOR_SET: + self.parseColor(s[:i + 1]) + s = s[i + 1:] + self.writeString(self.formatText(s)) + elif type == AnsiParser.CURSOR_SET: + cursor, s = s[:i+1], s[i+1:] + self.parseCursor(cursor) + self.writeString(self.formatText(s)) + elif type == AnsiParser.ERASE_SET: + erase, s = s[:i+1], s[i+1:] + self.parseErase(erase) + self.writeString(self.formatText(s)) + elif type == AnsiParser.MODE_SET: + s = s[i+1:] + #self.parseErase('2J') + self.writeString(self.formatText(s)) + elif i == L: + self.prepend = '\x1B[' + s + else: + log.msg('Unhandled ANSI control type: %c' % (s[i],)) + s = s[i + 1:] + self.writeString(self.formatText(s)) + + def parseColor(self, str): + """ + Handle a single ANSI color sequence + """ + # Drop the trailing 'm' + str = str[:-1] + + if not str: + str = '0' + + try: + parts = map(int, str.split(';')) + except ValueError: + log.msg('Invalid ANSI color sequence (%d): %s' % (len(str), str)) + self.currentFG, self.currentBG = self.defaultFG, self.defaultBG + return + + for x in parts: + if x == 0: + self.currentFG, self.currentBG = self.defaultFG, self.defaultBG + self.bold, self.flash, self.underline, self.reverse = 0, 0, 0, 0 + self.display = 1 + elif x == 1: + self.bold = 1 + elif 30 <= x <= 37: + self.currentFG = x - 30 + elif 40 <= x <= 47: + self.currentBG = x - 40 + elif x == 39: + self.currentFG = self.defaultFG + elif x == 49: + self.currentBG = self.defaultBG + elif x == 4: + self.underline = 1 + elif x == 5: + self.flash = 1 + elif x == 7: + self.reverse = 1 + elif x == 8: + self.display = 0 + elif x == 22: + self.bold = 0 + elif x == 24: + self.underline = 0 + elif x == 25: + self.blink = 0 + elif x == 27: + self.reverse = 0 + elif x == 28: + self.display = 1 + else: + log.msg('Unrecognised ANSI color command: %d' % (x,)) + + def parseCursor(self, cursor): + pass + + def parseErase(self, erase): + pass + + + def pickColor(self, value, mode, BOLD = ColorText.BOLD_COLORS): + if mode: + return ColorText.COLORS[value] + else: + return self.bold and BOLD[value] or ColorText.COLORS[value] + + + def formatText(self, text): + return ColorText( + text, + self.pickColor(self.currentFG, 0), + self.pickColor(self.currentBG, 1), + self.display, self.bold, self.underline, self.flash, self.reverse + ) + + +_sets = ''.join(map(''.join, AnsiParser.SETS)) + +_setmap = {} +for s in AnsiParser.SETS: + for r in s: + _setmap[r] = s +del s diff --git a/contrib/python/Twisted/py2/twisted/conch/ui/tkvt100.py b/contrib/python/Twisted/py2/twisted/conch/ui/tkvt100.py new file mode 100644 index 00000000000..0ce4db29152 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/ui/tkvt100.py @@ -0,0 +1,202 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + +"""Module to emulate a VT100 terminal in Tkinter. + +Maintainer: Paul Swartz +""" + +try: + import tkinter as Tkinter + import tkinter.font as tkFont +except ImportError: + import Tkinter, tkFont +import string +from . import ansi + +ttyFont = None#tkFont.Font(family = 'Courier', size = 10) +fontWidth, fontHeight = None,None#max(map(ttyFont.measure, string.letters+string.digits)), int(ttyFont.metrics()['linespace']) + +colorKeys = ( + 'b', 'r', 'g', 'y', 'l', 'm', 'c', 'w', + 'B', 'R', 'G', 'Y', 'L', 'M', 'C', 'W' +) + +colorMap = { + 'b': '#000000', 'r': '#c40000', 'g': '#00c400', 'y': '#c4c400', + 'l': '#000080', 'm': '#c400c4', 'c': '#00c4c4', 'w': '#c4c4c4', + 'B': '#626262', 'R': '#ff0000', 'G': '#00ff00', 'Y': '#ffff00', + 'L': '#0000ff', 'M': '#ff00ff', 'C': '#00ffff', 'W': '#ffffff', +} + +class VT100Frame(Tkinter.Frame): + def __init__(self, *args, **kw): + global ttyFont, fontHeight, fontWidth + ttyFont = tkFont.Font(family = 'Courier', size = 10) + fontWidth = max(map(ttyFont.measure, string.ascii_letters+string.digits)) + fontHeight = int(ttyFont.metrics()['linespace']) + self.width = kw.get('width', 80) + self.height = kw.get('height', 25) + self.callback = kw['callback'] + del kw['callback'] + kw['width'] = w = fontWidth * self.width + kw['height'] = h = fontHeight * self.height + Tkinter.Frame.__init__(self, *args, **kw) + self.canvas = Tkinter.Canvas(bg='#000000', width=w, height=h) + self.canvas.pack(side=Tkinter.TOP, fill=Tkinter.BOTH, expand=1) + self.canvas.bind('', self.keyPressed) + self.canvas.bind('<1>', lambda x: 'break') + self.canvas.bind('', self.upPressed) + self.canvas.bind('', self.downPressed) + self.canvas.bind('', self.leftPressed) + self.canvas.bind('', self.rightPressed) + self.canvas.focus() + + self.ansiParser = ansi.AnsiParser(ansi.ColorText.WHITE, ansi.ColorText.BLACK) + self.ansiParser.writeString = self.writeString + self.ansiParser.parseCursor = self.parseCursor + self.ansiParser.parseErase = self.parseErase + #for (a, b) in colorMap.items(): + # self.canvas.tag_config(a, foreground=b) + # self.canvas.tag_config('b'+a, background=b) + #self.canvas.tag_config('underline', underline=1) + + self.x = 0 + self.y = 0 + self.cursor = self.canvas.create_rectangle(0,0,fontWidth-1,fontHeight-1,fill='green',outline='green') + + def _delete(self, sx, sy, ex, ey): + csx = sx*fontWidth + 1 + csy = sy*fontHeight + 1 + cex = ex*fontWidth + 3 + cey = ey*fontHeight + 3 + items = self.canvas.find_overlapping(csx,csy, cex,cey) + for item in items: + self.canvas.delete(item) + + def _write(self, ch, fg, bg): + if self.x == self.width: + self.x = 0 + self.y+=1 + if self.y == self.height: + [self.canvas.move(x,0,-fontHeight) for x in self.canvas.find_all()] + self.y-=1 + canvasX = self.x*fontWidth + 1 + canvasY = self.y*fontHeight + 1 + items = self.canvas.find_overlapping(canvasX, canvasY, canvasX+2, canvasY+2) + if items: + [self.canvas.delete(item) for item in items] + if bg: + self.canvas.create_rectangle(canvasX, canvasY, canvasX+fontWidth-1, canvasY+fontHeight-1, fill=bg, outline=bg) + self.canvas.create_text(canvasX, canvasY, anchor=Tkinter.NW, font=ttyFont, text=ch, fill=fg) + self.x+=1 + + def write(self, data): + #print self.x,self.y,repr(data) + #if len(data)>5: raw_input() + self.ansiParser.parseString(data) + self.canvas.delete(self.cursor) + canvasX = self.x*fontWidth + 1 + canvasY = self.y*fontHeight + 1 + self.cursor = self.canvas.create_rectangle(canvasX,canvasY,canvasX+fontWidth-1,canvasY+fontHeight-1, fill='green', outline='green') + self.canvas.lower(self.cursor) + + def writeString(self, i): + if not i.display: + return + fg = colorMap[i.fg] + bg = i.bg != 'b' and colorMap[i.bg] + for ch in i.text: + b = ord(ch) + if b == 7: # bell + self.bell() + elif b == 8: # BS + if self.x: + self.x-=1 + elif b == 9: # TAB + [self._write(' ',fg,bg) for index in range(8)] + elif b == 10: + if self.y == self.height-1: + self._delete(0,0,self.width,0) + [self.canvas.move(x,0,-fontHeight) for x in self.canvas.find_all()] + else: + self.y+=1 + elif b == 13: + self.x = 0 + elif 32 <= b < 127: + self._write(ch, fg, bg) + + def parseErase(self, erase): + if ';' in erase: + end = erase[-1] + parts = erase[:-1].split(';') + [self.parseErase(x+end) for x in parts] + return + start = 0 + x,y = self.x, self.y + if len(erase) > 1: + start = int(erase[:-1]) + if erase[-1] == 'J': + if start == 0: + self._delete(x,y,self.width,self.height) + else: + self._delete(0,0,self.width,self.height) + self.x = 0 + self.y = 0 + elif erase[-1] == 'K': + if start == 0: + self._delete(x,y,self.width,y) + elif start == 1: + self._delete(0,y,x,y) + self.x = 0 + else: + self._delete(0,y,self.width,y) + self.x = 0 + elif erase[-1] == 'P': + self._delete(x,y,x+start,y) + + def parseCursor(self, cursor): + #if ';' in cursor and cursor[-1]!='H': + # end = cursor[-1] + # parts = cursor[:-1].split(';') + # [self.parseCursor(x+end) for x in parts] + # return + start = 1 + if len(cursor) > 1 and cursor[-1]!='H': + start = int(cursor[:-1]) + if cursor[-1] == 'C': + self.x+=start + elif cursor[-1] == 'D': + self.x-=start + elif cursor[-1]=='d': + self.y=start-1 + elif cursor[-1]=='G': + self.x=start-1 + elif cursor[-1]=='H': + if len(cursor)>1: + y,x = map(int, cursor[:-1].split(';')) + y-=1 + x-=1 + else: + x,y=0,0 + self.x = x + self.y = y + + def keyPressed(self, event): + if self.callback and event.char: + self.callback(event.char) + return 'break' + + def upPressed(self, event): + self.callback('\x1bOA') + + def downPressed(self, event): + self.callback('\x1bOB') + + def rightPressed(self, event): + self.callback('\x1bOC') + + def leftPressed(self, event): + self.callback('\x1bOD') diff --git a/contrib/python/Twisted/py2/twisted/conch/unix.py b/contrib/python/Twisted/py2/twisted/conch/unix.py new file mode 100644 index 00000000000..d9c3f05c89a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/conch/unix.py @@ -0,0 +1,535 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A UNIX SSH server. +""" + +import fcntl +import grp +import os +import pty +import pwd +import socket +import struct +import time +import tty + +from zope.interface import implementer + +from twisted.conch import ttymodes +from twisted.conch.avatar import ConchUser +from twisted.conch.error import ConchError +from twisted.conch.ls import lsLine +from twisted.conch.ssh import session, forwarding, filetransfer +from twisted.conch.ssh.filetransfer import ( + FXF_READ, FXF_WRITE, FXF_APPEND, FXF_CREAT, FXF_TRUNC, FXF_EXCL +) +from twisted.conch.interfaces import ISession, ISFTPServer, ISFTPFile +from twisted.cred import portal +from twisted.internet.error import ProcessExitedAlready +from twisted.python import components, log +from twisted.python.compat import _bytesChr as chr, nativeString + +try: + import utmp +except ImportError: + utmp = None + + + +@implementer(portal.IRealm) +class UnixSSHRealm: + def requestAvatar(self, username, mind, *interfaces): + user = UnixConchUser(username) + return interfaces[0], user, user.logout + + + +class UnixConchUser(ConchUser): + + def __init__(self, username): + ConchUser.__init__(self) + self.username = username + self.pwdData = pwd.getpwnam(self.username) + l = [self.pwdData[3]] + for groupname, password, gid, userlist in grp.getgrall(): + if username in userlist: + l.append(gid) + self.otherGroups = l + self.listeners = {} # Dict mapping (interface, port) -> listener + self.channelLookup.update( + {b"session": session.SSHSession, + b"direct-tcpip": forwarding.openConnectForwardingClient}) + + self.subsystemLookup.update( + {b"sftp": filetransfer.FileTransferServer}) + + + def getUserGroupId(self): + return self.pwdData[2:4] + + + def getOtherGroups(self): + return self.otherGroups + + + def getHomeDir(self): + return self.pwdData[5] + + + def getShell(self): + return self.pwdData[6] + + + def global_tcpip_forward(self, data): + hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data) + from twisted.internet import reactor + try: + listener = self._runAsUser( + reactor.listenTCP, portToBind, + forwarding.SSHListenForwardingFactory( + self.conn, + (hostToBind, portToBind), + forwarding.SSHListenServerForwardingChannel), + interface=hostToBind) + except: + return 0 + else: + self.listeners[(hostToBind, portToBind)] = listener + if portToBind == 0: + portToBind = listener.getHost()[2] # The port + return 1, struct.pack('>L', portToBind) + else: + return 1 + + + def global_cancel_tcpip_forward(self, data): + hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data) + listener = self.listeners.get((hostToBind, portToBind), None) + if not listener: + return 0 + del self.listeners[(hostToBind, portToBind)] + self._runAsUser(listener.stopListening) + return 1 + + + def logout(self): + # Remove all listeners. + for listener in self.listeners.values(): + self._runAsUser(listener.stopListening) + log.msg( + 'avatar %s logging out (%i)' + % (self.username, len(self.listeners))) + + + def _runAsUser(self, f, *args, **kw): + euid = os.geteuid() + egid = os.getegid() + groups = os.getgroups() + uid, gid = self.getUserGroupId() + os.setegid(0) + os.seteuid(0) + os.setgroups(self.getOtherGroups()) + os.setegid(gid) + os.seteuid(uid) + try: + f = iter(f) + except TypeError: + f = [(f, args, kw)] + try: + for i in f: + func = i[0] + args = len(i) > 1 and i[1] or () + kw = len(i) > 2 and i[2] or {} + r = func(*args, **kw) + finally: + os.setegid(0) + os.seteuid(0) + os.setgroups(groups) + os.setegid(egid) + os.seteuid(euid) + return r + + + +@implementer(ISession) +class SSHSessionForUnixConchUser: + def __init__(self, avatar, reactor=None): + """ + Construct an C{SSHSessionForUnixConchUser}. + + @param avatar: The L{UnixConchUser} for whom this is an SSH session. + @param reactor: An L{IReactorProcess} used to handle shell and exec + requests. Uses the default reactor if None. + """ + if reactor is None: + from twisted.internet import reactor + self._reactor = reactor + self.avatar = avatar + self.environ = {'PATH': '/bin:/usr/bin:/usr/local/bin'} + self.pty = None + self.ptyTuple = 0 + + + def addUTMPEntry(self, loggedIn=1): + if not utmp: + return + ipAddress = self.avatar.conn.transport.transport.getPeer().host + packedIp, = struct.unpack('L', socket.inet_aton(ipAddress)) + ttyName = self.ptyTuple[2][5:] + t = time.time() + t1 = int(t) + t2 = int((t-t1) * 1e6) + entry = utmp.UtmpEntry() + entry.ut_type = loggedIn and utmp.USER_PROCESS or utmp.DEAD_PROCESS + entry.ut_pid = self.pty.pid + entry.ut_line = ttyName + entry.ut_id = ttyName[-4:] + entry.ut_tv = (t1, t2) + if loggedIn: + entry.ut_user = self.avatar.username + entry.ut_host = socket.gethostbyaddr(ipAddress)[0] + entry.ut_addr_v6 = (packedIp, 0, 0, 0) + a = utmp.UtmpRecord(utmp.UTMP_FILE) + a.pututline(entry) + a.endutent() + b = utmp.UtmpRecord(utmp.WTMP_FILE) + b.pututline(entry) + b.endutent() + + + def getPty(self, term, windowSize, modes): + self.environ['TERM'] = term + self.winSize = windowSize + self.modes = modes + master, slave = pty.openpty() + ttyname = os.ttyname(slave) + self.environ['SSH_TTY'] = ttyname + self.ptyTuple = (master, slave, ttyname) + + + def openShell(self, proto): + if not self.ptyTuple: # We didn't get a pty-req. + log.msg('tried to get shell without pty, failing') + raise ConchError("no pty") + uid, gid = self.avatar.getUserGroupId() + homeDir = self.avatar.getHomeDir() + shell = self.avatar.getShell() + self.environ['USER'] = self.avatar.username + self.environ['HOME'] = homeDir + self.environ['SHELL'] = shell + shellExec = os.path.basename(shell) + peer = self.avatar.conn.transport.transport.getPeer() + host = self.avatar.conn.transport.transport.getHost() + self.environ['SSH_CLIENT'] = '%s %s %s' % ( + peer.host, peer.port, host.port) + self.getPtyOwnership() + self.pty = self._reactor.spawnProcess( + proto, shell, ['-%s' % (shellExec,)], self.environ, homeDir, uid, + gid, usePTY=self.ptyTuple) + self.addUTMPEntry() + fcntl.ioctl(self.pty.fileno(), tty.TIOCSWINSZ, + struct.pack('4H', *self.winSize)) + if self.modes: + self.setModes() + self.oldWrite = proto.transport.write + proto.transport.write = self._writeHack + self.avatar.conn.transport.transport.setTcpNoDelay(1) + + + def execCommand(self, proto, cmd): + uid, gid = self.avatar.getUserGroupId() + homeDir = self.avatar.getHomeDir() + shell = self.avatar.getShell() or '/bin/sh' + self.environ['HOME'] = homeDir + command = (shell, '-c', cmd) + peer = self.avatar.conn.transport.transport.getPeer() + host = self.avatar.conn.transport.transport.getHost() + self.environ['SSH_CLIENT'] = '%s %s %s' % ( + peer.host, peer.port, host.port) + if self.ptyTuple: + self.getPtyOwnership() + self.pty = self._reactor.spawnProcess( + proto, shell, command, self.environ, homeDir, uid, gid, + usePTY=self.ptyTuple or 0) + if self.ptyTuple: + self.addUTMPEntry() + if self.modes: + self.setModes() + self.avatar.conn.transport.transport.setTcpNoDelay(1) + + + def getPtyOwnership(self): + ttyGid = os.stat(self.ptyTuple[2])[5] + uid, gid = self.avatar.getUserGroupId() + euid, egid = os.geteuid(), os.getegid() + os.setegid(0) + os.seteuid(0) + try: + os.chown(self.ptyTuple[2], uid, ttyGid) + finally: + os.setegid(egid) + os.seteuid(euid) + + + def setModes(self): + pty = self.pty + attr = tty.tcgetattr(pty.fileno()) + for mode, modeValue in self.modes: + if mode not in ttymodes.TTYMODES: + continue + ttyMode = ttymodes.TTYMODES[mode] + if len(ttyMode) == 2: # Flag. + flag, ttyAttr = ttyMode + if not hasattr(tty, ttyAttr): + continue + ttyval = getattr(tty, ttyAttr) + if modeValue: + attr[flag] = attr[flag] | ttyval + else: + attr[flag] = attr[flag] & ~ttyval + elif ttyMode == 'OSPEED': + attr[tty.OSPEED] = getattr(tty, 'B%s' % (modeValue,)) + elif ttyMode == 'ISPEED': + attr[tty.ISPEED] = getattr(tty, 'B%s' % (modeValue,)) + else: + if not hasattr(tty, ttyMode): + continue + ttyval = getattr(tty, ttyMode) + attr[tty.CC][ttyval] = chr(modeValue) + tty.tcsetattr(pty.fileno(), tty.TCSANOW, attr) + + + def eofReceived(self): + if self.pty: + self.pty.closeStdin() + + + def closed(self): + if self.ptyTuple and os.path.exists(self.ptyTuple[2]): + ttyGID = os.stat(self.ptyTuple[2])[5] + os.chown(self.ptyTuple[2], 0, ttyGID) + if self.pty: + try: + self.pty.signalProcess('HUP') + except (OSError, ProcessExitedAlready): + pass + self.pty.loseConnection() + self.addUTMPEntry(0) + log.msg('shell closed') + + + def windowChanged(self, winSize): + self.winSize = winSize + fcntl.ioctl( + self.pty.fileno(), tty.TIOCSWINSZ, + struct.pack('4H', *self.winSize)) + + + def _writeHack(self, data): + """ + Hack to send ignore messages when we aren't echoing. + """ + if self.pty is not None: + attr = tty.tcgetattr(self.pty.fileno())[3] + if not attr & tty.ECHO and attr & tty.ICANON: # No echo. + self.avatar.conn.transport.sendIgnore('\x00'*(8+len(data))) + self.oldWrite(data) + + + +@implementer(ISFTPServer) +class SFTPServerForUnixConchUser: + def __init__(self, avatar): + self.avatar = avatar + + + def _setAttrs(self, path, attrs): + """ + NOTE: this function assumes it runs as the logged-in user: + i.e. under _runAsUser() + """ + if "uid" in attrs and "gid" in attrs: + os.chown(path, attrs["uid"], attrs["gid"]) + if "permissions" in attrs: + os.chmod(path, attrs["permissions"]) + if "atime" in attrs and "mtime" in attrs: + os.utime(path, (attrs["atime"], attrs["mtime"])) + + + def _getAttrs(self, s): + return { + "size": s.st_size, + "uid": s.st_uid, + "gid": s.st_gid, + "permissions": s.st_mode, + "atime": int(s.st_atime), + "mtime": int(s.st_mtime) + } + + + def _absPath(self, path): + home = self.avatar.getHomeDir() + return os.path.join(nativeString(home.path), nativeString(path)) + + + def gotVersion(self, otherVersion, extData): + return {} + + + def openFile(self, filename, flags, attrs): + return UnixSFTPFile(self, self._absPath(filename), flags, attrs) + + + def removeFile(self, filename): + filename = self._absPath(filename) + return self.avatar._runAsUser(os.remove, filename) + + + def renameFile(self, oldpath, newpath): + oldpath = self._absPath(oldpath) + newpath = self._absPath(newpath) + return self.avatar._runAsUser(os.rename, oldpath, newpath) + + + def makeDirectory(self, path, attrs): + path = self._absPath(path) + return self.avatar._runAsUser( + [(os.mkdir, (path,)), (self._setAttrs, (path, attrs))]) + + + def removeDirectory(self, path): + path = self._absPath(path) + self.avatar._runAsUser(os.rmdir, path) + + + def openDirectory(self, path): + return UnixSFTPDirectory(self, self._absPath(path)) + + + def getAttrs(self, path, followLinks): + path = self._absPath(path) + if followLinks: + s = self.avatar._runAsUser(os.stat, path) + else: + s = self.avatar._runAsUser(os.lstat, path) + return self._getAttrs(s) + + + def setAttrs(self, path, attrs): + path = self._absPath(path) + self.avatar._runAsUser(self._setAttrs, path, attrs) + + + def readLink(self, path): + path = self._absPath(path) + return self.avatar._runAsUser(os.readlink, path) + + + def makeLink(self, linkPath, targetPath): + linkPath = self._absPath(linkPath) + targetPath = self._absPath(targetPath) + return self.avatar._runAsUser(os.symlink, targetPath, linkPath) + + + def realPath(self, path): + return os.path.realpath(self._absPath(path)) + + + def extendedRequest(self, extName, extData): + raise NotImplementedError + + + +@implementer(ISFTPFile) +class UnixSFTPFile: + def __init__(self, server, filename, flags, attrs): + self.server = server + openFlags = 0 + if flags & FXF_READ == FXF_READ and flags & FXF_WRITE == 0: + openFlags = os.O_RDONLY + if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == 0: + openFlags = os.O_WRONLY + if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == FXF_READ: + openFlags = os.O_RDWR + if flags & FXF_APPEND == FXF_APPEND: + openFlags |= os.O_APPEND + if flags & FXF_CREAT == FXF_CREAT: + openFlags |= os.O_CREAT + if flags & FXF_TRUNC == FXF_TRUNC: + openFlags |= os.O_TRUNC + if flags & FXF_EXCL == FXF_EXCL: + openFlags |= os.O_EXCL + if "permissions" in attrs: + mode = attrs["permissions"] + del attrs["permissions"] + else: + mode = 0o777 + fd = server.avatar._runAsUser(os.open, filename, openFlags, mode) + if attrs: + server.avatar._runAsUser(server._setAttrs, filename, attrs) + self.fd = fd + + + def close(self): + return self.server.avatar._runAsUser(os.close, self.fd) + + + def readChunk(self, offset, length): + return self.server.avatar._runAsUser( + [(os.lseek, (self.fd, offset, 0)), + (os.read, (self.fd, length))]) + + + def writeChunk(self, offset, data): + return self.server.avatar._runAsUser( + [(os.lseek, (self.fd, offset, 0)), + (os.write, (self.fd, data))]) + + + def getAttrs(self): + s = self.server.avatar._runAsUser(os.fstat, self.fd) + return self.server._getAttrs(s) + + + def setAttrs(self, attrs): + raise NotImplementedError + + + +class UnixSFTPDirectory: + + def __init__(self, server, directory): + self.server = server + self.files = server.avatar._runAsUser(os.listdir, directory) + self.dir = directory + + + def __iter__(self): + return self + + + def __next__(self): + try: + f = self.files.pop(0) + except IndexError: + raise StopIteration + else: + s = self.server.avatar._runAsUser( + os.lstat, os.path.join(self.dir, f)) + longname = lsLine(f, s) + attrs = self.server._getAttrs(s) + return (f, longname, attrs) + + next = __next__ + + def close(self): + self.files = [] + + + +components.registerAdapter( + SFTPServerForUnixConchUser, UnixConchUser, filetransfer.ISFTPServer) +components.registerAdapter( + SSHSessionForUnixConchUser, UnixConchUser, session.ISession) diff --git a/contrib/python/Twisted/py2/twisted/copyright.py b/contrib/python/Twisted/py2/twisted/copyright.py new file mode 100644 index 00000000000..b6e32f740a8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/copyright.py @@ -0,0 +1,43 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Copyright information for Twisted. +""" + +from __future__ import division, absolute_import + +__all__ = ['copyright', 'disclaimer', 'longversion' ,'version'] + +from twisted import __version__ as version, version as longversion + +longversion = str(longversion) + +copyright = """\ +Copyright (c) 2001-2020 Twisted Matrix Laboratories. +See LICENSE for details.""" + +disclaimer = ''' +Twisted, the Framework of Your Internet +%s + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +''' % (copyright,) diff --git a/contrib/python/Twisted/py2/twisted/cred/__init__.py b/contrib/python/Twisted/py2/twisted/cred/__init__.py new file mode 100644 index 00000000000..2ee268c5e90 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/cred/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Cred: Support for verifying credentials, and providing services to user +based on those credentials. +""" diff --git a/contrib/python/Twisted/py2/twisted/cred/_digest.py b/contrib/python/Twisted/py2/twisted/cred/_digest.py new file mode 100644 index 00000000000..4ed1ce317a6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/cred/_digest.py @@ -0,0 +1,132 @@ +# -*- test-case-name: twisted.cred.test.test_digestauth -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Calculations for HTTP Digest authentication. + +@see: U{http://www.faqs.org/rfcs/rfc2617.html} +""" + +from __future__ import division, absolute_import + +from binascii import hexlify +from hashlib import md5, sha1 + + + +# The digest math + +algorithms = { + b'md5': md5, + + # md5-sess is more complicated than just another algorithm. It requires + # H(A1) state to be remembered from the first WWW-Authenticate challenge + # issued and re-used to process any Authorization header in response to + # that WWW-Authenticate challenge. It is *not* correct to simply + # recalculate H(A1) each time an Authorization header is received. Read + # RFC 2617, section 3.2.2.2 and do not try to make DigestCredentialFactory + # support this unless you completely understand it. -exarkun + b'md5-sess': md5, + + b'sha': sha1, +} + +# DigestCalcHA1 +def calcHA1(pszAlg, pszUserName, pszRealm, pszPassword, pszNonce, pszCNonce, + preHA1=None): + """ + Compute H(A1) from RFC 2617. + + @param pszAlg: The name of the algorithm to use to calculate the digest. + Currently supported are md5, md5-sess, and sha. + @param pszUserName: The username + @param pszRealm: The realm + @param pszPassword: The password + @param pszNonce: The nonce + @param pszCNonce: The cnonce + + @param preHA1: If available this is a str containing a previously + calculated H(A1) as a hex string. If this is given then the values for + pszUserName, pszRealm, and pszPassword must be L{None} and are ignored. + """ + + if (preHA1 and (pszUserName or pszRealm or pszPassword)): + raise TypeError(("preHA1 is incompatible with the pszUserName, " + "pszRealm, and pszPassword arguments")) + + if preHA1 is None: + # We need to calculate the HA1 from the username:realm:password + m = algorithms[pszAlg]() + m.update(pszUserName) + m.update(b":") + m.update(pszRealm) + m.update(b":") + m.update(pszPassword) + HA1 = hexlify(m.digest()) + else: + # We were given a username:realm:password + HA1 = preHA1 + + if pszAlg == b"md5-sess": + m = algorithms[pszAlg]() + m.update(HA1) + m.update(b":") + m.update(pszNonce) + m.update(b":") + m.update(pszCNonce) + HA1 = hexlify(m.digest()) + + return HA1 + + +def calcHA2(algo, pszMethod, pszDigestUri, pszQop, pszHEntity): + """ + Compute H(A2) from RFC 2617. + + @param pszAlg: The name of the algorithm to use to calculate the digest. + Currently supported are md5, md5-sess, and sha. + @param pszMethod: The request method. + @param pszDigestUri: The request URI. + @param pszQop: The Quality-of-Protection value. + @param pszHEntity: The hash of the entity body or L{None} if C{pszQop} is + not C{'auth-int'}. + @return: The hash of the A2 value for the calculation of the response + digest. + """ + m = algorithms[algo]() + m.update(pszMethod) + m.update(b":") + m.update(pszDigestUri) + if pszQop == b"auth-int": + m.update(b":") + m.update(pszHEntity) + return hexlify(m.digest()) + + +def calcResponse(HA1, HA2, algo, pszNonce, pszNonceCount, pszCNonce, pszQop): + """ + Compute the digest for the given parameters. + + @param HA1: The H(A1) value, as computed by L{calcHA1}. + @param HA2: The H(A2) value, as computed by L{calcHA2}. + @param pszNonce: The challenge nonce. + @param pszNonceCount: The (client) nonce count value for this response. + @param pszCNonce: The client nonce. + @param pszQop: The Quality-of-Protection value. + """ + m = algorithms[algo]() + m.update(HA1) + m.update(b":") + m.update(pszNonce) + m.update(b":") + if pszNonceCount and pszCNonce: + m.update(pszNonceCount) + m.update(b":") + m.update(pszCNonce) + m.update(b":") + m.update(pszQop) + m.update(b":") + m.update(HA2) + respHash = hexlify(m.digest()) + return respHash diff --git a/contrib/python/Twisted/py2/twisted/cred/checkers.py b/contrib/python/Twisted/py2/twisted/cred/checkers.py new file mode 100644 index 00000000000..7a29809cb60 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/cred/checkers.py @@ -0,0 +1,329 @@ +# -*- test-case-name: twisted.cred.test.test_cred -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Basic credential checkers + +@var ANONYMOUS: An empty tuple used to represent the anonymous avatar ID. +""" + +from __future__ import division, absolute_import + +import os + +from zope.interface import implementer, Interface, Attribute + +from twisted.logger import Logger +from twisted.internet import defer +from twisted.python import failure +from twisted.cred import error, credentials + + + +class ICredentialsChecker(Interface): + """ + An object that can check sub-interfaces of L{ICredentials}. + """ + + credentialInterfaces = Attribute(( + 'A list of sub-interfaces of L{ICredentials} which specifies which I ' + 'may check.' + )) + + + def requestAvatarId(credentials): + """ + Validate credentials and produce an avatar ID. + + @param credentials: something which implements one of the interfaces in + C{credentialInterfaces}. + + @return: a L{Deferred} which will fire with a L{bytes} that identifies + an avatar, an empty tuple to specify an authenticated anonymous user + (provided as L{twisted.cred.checkers.ANONYMOUS}) or fail with + L{UnauthorizedLogin}. Alternatively, return the result itself. + + @see: L{twisted.cred.credentials} + """ + + + +# A note on anonymity - We do not want None as the value for anonymous +# because it is too easy to accidentally return it. We do not want the +# empty string, because it is too easy to mistype a password file. For +# example, an .htpasswd file may contain the lines: ['hello:asdf', +# 'world:asdf', 'goodbye', ':world']. This misconfiguration will have an +# ill effect in any case, but accidentally granting anonymous access is a +# worse failure mode than simply granting access to an untypeable +# username. We do not want an instance of 'object', because that would +# create potential problems with persistence. + +ANONYMOUS = () + + + +@implementer(ICredentialsChecker) +class AllowAnonymousAccess: + """ + A credentials checker that unconditionally grants anonymous access. + + @cvar credentialInterfaces: Tuple containing L{IAnonymous}. + """ + credentialInterfaces = credentials.IAnonymous, + + def requestAvatarId(self, credentials): + """ + Succeed with the L{ANONYMOUS} avatar ID. + + @return: L{Deferred} that fires with L{twisted.cred.checkers.ANONYMOUS} + """ + return defer.succeed(ANONYMOUS) + + + +@implementer(ICredentialsChecker) +class InMemoryUsernamePasswordDatabaseDontUse(object): + """ + An extremely simple credentials checker. + + This is only of use in one-off test programs or examples which don't + want to focus too much on how credentials are verified. + + You really don't want to use this for anything else. It is, at best, a + toy. If you need a simple credentials checker for a real application, + see L{FilePasswordDB}. + + @cvar credentialInterfaces: Tuple of L{IUsernamePassword} and + L{IUsernameHashedPassword}. + + @ivar users: Mapping of usernames to passwords. + @type users: L{dict} mapping L{bytes} to L{bytes} + """ + credentialInterfaces = (credentials.IUsernamePassword, + credentials.IUsernameHashedPassword) + + def __init__(self, **users): + """ + Initialize the in-memory database. + + For example:: + + db = InMemoryUsernamePasswordDatabaseDontUse( + user1=b'sesame', + user2=b'hunter2', + ) + + @param users: Usernames and passwords to seed the database with. + Each username given as a keyword is encoded to L{bytes} as ASCII. + Passwords must be given as L{bytes}. + @type users: L{dict} of L{str} to L{bytes} + """ + self.users = {x.encode('ascii'): y for x, y in users.items()} + + + def addUser(self, username, password): + """ + Set a user's password. + + @param username: Name of the user. + @type username: L{bytes} + + @param password: Password to associate with the username. + @type password: L{bytes} + """ + self.users[username] = password + + + def _cbPasswordMatch(self, matched, username): + if matched: + return username + else: + return failure.Failure(error.UnauthorizedLogin()) + + + def requestAvatarId(self, credentials): + if credentials.username in self.users: + return defer.maybeDeferred( + credentials.checkPassword, + self.users[credentials.username]).addCallback( + self._cbPasswordMatch, credentials.username) + else: + return defer.fail(error.UnauthorizedLogin()) + + + +@implementer(ICredentialsChecker) +class FilePasswordDB: + """ + A file-based, text-based username/password database. + + Records in the datafile for this class are delimited by a particular + string. The username appears in a fixed field of the columns delimited + by this string, as does the password. Both fields are specifiable. If + the passwords are not stored plaintext, a hash function must be supplied + to convert plaintext passwords to the form stored on disk and this + CredentialsChecker will only be able to check L{IUsernamePassword} + credentials. If the passwords are stored plaintext, + L{IUsernameHashedPassword} credentials will be checkable as well. + """ + + cache = False + _credCache = None + _cacheTimestamp = 0 + _log = Logger() + + def __init__(self, filename, delim=b':', usernameField=0, passwordField=1, + caseSensitive=True, hash=None, cache=False): + """ + @type filename: L{str} + @param filename: The name of the file from which to read username and + password information. + + @type delim: L{bytes} + @param delim: The field delimiter used in the file. + + @type usernameField: L{int} + @param usernameField: The index of the username after splitting a + line on the delimiter. + + @type passwordField: L{int} + @param passwordField: The index of the password after splitting a + line on the delimiter. + + @type caseSensitive: L{bool} + @param caseSensitive: If true, consider the case of the username when + performing a lookup. Ignore it otherwise. + + @type hash: Three-argument callable or L{None} + @param hash: A function used to transform the plaintext password + received over the network to a format suitable for comparison + against the version stored on disk. The arguments to the callable + are the username, the network-supplied password, and the in-file + version of the password. If the return value compares equal to the + version stored on disk, the credentials are accepted. + + @type cache: L{bool} + @param cache: If true, maintain an in-memory cache of the + contents of the password file. On lookups, the mtime of the + file will be checked, and the file will only be re-parsed if + the mtime is newer than when the cache was generated. + """ + self.filename = filename + self.delim = delim + self.ufield = usernameField + self.pfield = passwordField + self.caseSensitive = caseSensitive + self.hash = hash + self.cache = cache + + if self.hash is None: + # The passwords are stored plaintext. We can support both + # plaintext and hashed passwords received over the network. + self.credentialInterfaces = ( + credentials.IUsernamePassword, + credentials.IUsernameHashedPassword + ) + else: + # The passwords are hashed on disk. We can support only + # plaintext passwords received over the network. + self.credentialInterfaces = ( + credentials.IUsernamePassword, + ) + + + def __getstate__(self): + d = dict(vars(self)) + for k in '_credCache', '_cacheTimestamp': + try: + del d[k] + except KeyError: + pass + return d + + + def _cbPasswordMatch(self, matched, username): + if matched: + return username + else: + return failure.Failure(error.UnauthorizedLogin()) + + + def _loadCredentials(self): + """ + Loads the credentials from the configured file. + + @return: An iterable of C{username, password} couples. + @rtype: C{iterable} + + @raise UnauthorizedLogin: when failing to read the credentials from the + file. + """ + try: + with open(self.filename, "rb") as f: + for line in f: + line = line.rstrip() + parts = line.split(self.delim) + + if self.ufield >= len(parts) or self.pfield >= len(parts): + continue + if self.caseSensitive: + yield parts[self.ufield], parts[self.pfield] + else: + yield parts[self.ufield].lower(), parts[self.pfield] + except IOError as e: + self._log.error("Unable to load credentials db: {e!r}", e=e) + raise error.UnauthorizedLogin() + + + def getUser(self, username): + """ + Look up the credentials for a username. + + @param username: The username to look up. + @type username: L{bytes} + + @returns: Two-tuple of the canonicalicalized username (i.e. lowercase + if the database is not case sensitive) and the associated password + value, both L{bytes}. + @rtype: L{tuple} + + @raises KeyError: When lookup of the username fails. + """ + if not self.caseSensitive: + username = username.lower() + + if self.cache: + if self._credCache is None or os.path.getmtime(self.filename) > self._cacheTimestamp: + self._cacheTimestamp = os.path.getmtime(self.filename) + self._credCache = dict(self._loadCredentials()) + return username, self._credCache[username] + else: + for u, p in self._loadCredentials(): + if u == username: + return u, p + raise KeyError(username) + + + def requestAvatarId(self, c): + try: + u, p = self.getUser(c.username) + except KeyError: + return defer.fail(error.UnauthorizedLogin()) + else: + up = credentials.IUsernamePassword(c, None) + if self.hash: + if up is not None: + h = self.hash(up.username, up.password, p) + if h == p: + return defer.succeed(u) + return defer.fail(error.UnauthorizedLogin()) + else: + return defer.maybeDeferred(c.checkPassword, p + ).addCallback(self._cbPasswordMatch, u) + + + +# For backwards compatibility +# Allow access as the old name. +OnDiskUsernamePasswordDatabase = FilePasswordDB diff --git a/contrib/python/Twisted/py2/twisted/cred/credentials.py b/contrib/python/Twisted/py2/twisted/cred/credentials.py new file mode 100644 index 00000000000..5469e51587d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/cred/credentials.py @@ -0,0 +1,510 @@ +# -*- test-case-name: twisted.cred.test.test_cred-*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module defines L{ICredentials}, an interface for objects that represent +authentication credentials to provide, and also includes a number of useful +implementations of that interface. +""" + +from __future__ import division, absolute_import + +from zope.interface import implementer, Interface + +import base64 +import hmac +import random +import re +import time + +from binascii import hexlify +from hashlib import md5 + +from twisted.python.randbytes import secureRandom +from twisted.python.compat import networkString, nativeString +from twisted.python.compat import intToBytes, unicode +from twisted.cred._digest import calcResponse, calcHA1, calcHA2 +from twisted.cred import error + + + +class ICredentials(Interface): + """ + I check credentials. + + Implementors I{must} specify the sub-interfaces of ICredentials + to which it conforms, using L{zope.interface.declarations.implementer}. + """ + + + +class IUsernameDigestHash(ICredentials): + """ + This credential is used when a CredentialChecker has access to the hash + of the username:realm:password as in an Apache .htdigest file. + """ + def checkHash(digestHash): + """ + @param digestHash: The hashed username:realm:password to check against. + + @return: C{True} if the credentials represented by this object match + the given hash, C{False} if they do not, or a L{Deferred} which + will be called back with one of these values. + """ + + + +class IUsernameHashedPassword(ICredentials): + """ + I encapsulate a username and a hashed password. + + This credential is used when a hashed password is received from the + party requesting authentication. CredentialCheckers which check this + kind of credential must store the passwords in plaintext (or as + password-equivalent hashes) form so that they can be hashed in a manner + appropriate for the particular credentials class. + + @type username: L{bytes} + @ivar username: The username associated with these credentials. + """ + + def checkPassword(password): + """ + Validate these credentials against the correct password. + + @type password: L{bytes} + @param password: The correct, plaintext password against which to + check. + + @rtype: C{bool} or L{Deferred} + @return: C{True} if the credentials represented by this object match the + given password, C{False} if they do not, or a L{Deferred} which will + be called back with one of these values. + """ + + + +class IUsernamePassword(ICredentials): + """ + I encapsulate a username and a plaintext password. + + This encapsulates the case where the password received over the network + has been hashed with the identity function (That is, not at all). The + CredentialsChecker may store the password in whatever format it desires, + it need only transform the stored password in a similar way before + performing the comparison. + + @type username: L{bytes} + @ivar username: The username associated with these credentials. + + @type password: L{bytes} + @ivar password: The password associated with these credentials. + """ + + def checkPassword(password): + """ + Validate these credentials against the correct password. + + @type password: L{bytes} + @param password: The correct, plaintext password against which to + check. + + @rtype: C{bool} or L{Deferred} + @return: C{True} if the credentials represented by this object match the + given password, C{False} if they do not, or a L{Deferred} which will + be called back with one of these values. + """ + + + +class IAnonymous(ICredentials): + """ + I am an explicitly anonymous request for access. + + @see: L{twisted.cred.checkers.AllowAnonymousAccess} + """ + + + +@implementer(IUsernameHashedPassword, IUsernameDigestHash) +class DigestedCredentials(object): + """ + Yet Another Simple HTTP Digest authentication scheme. + """ + + def __init__(self, username, method, realm, fields): + self.username = username + self.method = method + self.realm = realm + self.fields = fields + + + def checkPassword(self, password): + """ + Verify that the credentials represented by this object agree with the + given plaintext C{password} by hashing C{password} in the same way the + response hash represented by this object was generated and comparing + the results. + """ + response = self.fields.get('response') + uri = self.fields.get('uri') + nonce = self.fields.get('nonce') + cnonce = self.fields.get('cnonce') + nc = self.fields.get('nc') + algo = self.fields.get('algorithm', b'md5').lower() + qop = self.fields.get('qop', b'auth') + + expected = calcResponse( + calcHA1(algo, self.username, self.realm, password, nonce, cnonce), + calcHA2(algo, self.method, uri, qop, None), + algo, nonce, nc, cnonce, qop) + + return expected == response + + + def checkHash(self, digestHash): + """ + Verify that the credentials represented by this object agree with the + credentials represented by the I{H(A1)} given in C{digestHash}. + + @param digestHash: A precomputed H(A1) value based on the username, + realm, and password associate with this credentials object. + """ + response = self.fields.get('response') + uri = self.fields.get('uri') + nonce = self.fields.get('nonce') + cnonce = self.fields.get('cnonce') + nc = self.fields.get('nc') + algo = self.fields.get('algorithm', b'md5').lower() + qop = self.fields.get('qop', b'auth') + + expected = calcResponse( + calcHA1(algo, None, None, None, nonce, cnonce, preHA1=digestHash), + calcHA2(algo, self.method, uri, qop, None), + algo, nonce, nc, cnonce, qop) + + return expected == response + + + +class DigestCredentialFactory(object): + """ + Support for RFC2617 HTTP Digest Authentication + + @cvar CHALLENGE_LIFETIME_SECS: The number of seconds for which an + opaque should be valid. + + @type privateKey: L{bytes} + @ivar privateKey: A random string used for generating the secure opaque. + + @type algorithm: L{bytes} + @param algorithm: Case insensitive string specifying the hash algorithm to + use. Must be either C{'md5'} or C{'sha'}. C{'md5-sess'} is B{not} + supported. + + @type authenticationRealm: L{bytes} + @param authenticationRealm: case sensitive string that specifies the realm + portion of the challenge + """ + + _parseparts = re.compile( + b'([^= ]+)' # The key + b'=' # Conventional key/value separator (literal) + b'(?:' # Group together a couple options + b'"([^"]*)"' # A quoted string of length 0 or more + b'|' # The other option in the group is coming + b'([^,]+)' # An unquoted string of length 1 or more, up to a comma + b')' # That non-matching group ends + b',?') # There might be a comma at the end (none on last pair) + + CHALLENGE_LIFETIME_SECS = 15 * 60 # 15 minutes + + scheme = b"digest" + + def __init__(self, algorithm, authenticationRealm): + self.algorithm = algorithm + self.authenticationRealm = authenticationRealm + self.privateKey = secureRandom(12) + + + def getChallenge(self, address): + """ + Generate the challenge for use in the WWW-Authenticate header. + + @param address: The client address to which this challenge is being + sent. + + @return: The L{dict} that can be used to generate a WWW-Authenticate + header. + """ + c = self._generateNonce() + o = self._generateOpaque(c, address) + + return {'nonce': c, + 'opaque': o, + 'qop': b'auth', + 'algorithm': self.algorithm, + 'realm': self.authenticationRealm} + + + def _generateNonce(self): + """ + Create a random value suitable for use as the nonce parameter of a + WWW-Authenticate challenge. + + @rtype: L{bytes} + """ + return hexlify(secureRandom(12)) + + + def _getTime(self): + """ + Parameterize the time based seed used in C{_generateOpaque} + so we can deterministically unittest it's behavior. + """ + return time.time() + + + def _generateOpaque(self, nonce, clientip): + """ + Generate an opaque to be returned to the client. This is a unique + string that can be returned to us and verified. + """ + # Now, what we do is encode the nonce, client ip and a timestamp in the + # opaque value with a suitable digest. + now = intToBytes(int(self._getTime())) + + if not clientip: + clientip = b'' + elif isinstance(clientip, unicode): + clientip = clientip.encode('ascii') + + key = b",".join((nonce, clientip, now)) + digest = hexlify(md5(key + self.privateKey).digest()) + ekey = base64.b64encode(key) + return b"-".join((digest, ekey.replace(b'\n', b''))) + + + def _verifyOpaque(self, opaque, nonce, clientip): + """ + Given the opaque and nonce from the request, as well as the client IP + that made the request, verify that the opaque was generated by us. + And that it's not too old. + + @param opaque: The opaque value from the Digest response + @param nonce: The nonce value from the Digest response + @param clientip: The remote IP address of the client making the request + or L{None} if the request was submitted over a channel where this + does not make sense. + + @return: C{True} if the opaque was successfully verified. + + @raise error.LoginFailed: if C{opaque} could not be parsed or + contained the wrong values. + """ + # First split the digest from the key + opaqueParts = opaque.split(b'-') + if len(opaqueParts) != 2: + raise error.LoginFailed('Invalid response, invalid opaque value') + + if not clientip: + clientip = b'' + elif isinstance(clientip, unicode): + clientip = clientip.encode('ascii') + + # Verify the key + key = base64.b64decode(opaqueParts[1]) + keyParts = key.split(b',') + + if len(keyParts) != 3: + raise error.LoginFailed('Invalid response, invalid opaque value') + + if keyParts[0] != nonce: + raise error.LoginFailed( + 'Invalid response, incompatible opaque/nonce values') + + if keyParts[1] != clientip: + raise error.LoginFailed( + 'Invalid response, incompatible opaque/client values') + + try: + when = int(keyParts[2]) + except ValueError: + raise error.LoginFailed( + 'Invalid response, invalid opaque/time values') + + if (int(self._getTime()) - when > + DigestCredentialFactory.CHALLENGE_LIFETIME_SECS): + + raise error.LoginFailed( + 'Invalid response, incompatible opaque/nonce too old') + + # Verify the digest + digest = hexlify(md5(key + self.privateKey).digest()) + if digest != opaqueParts[0]: + raise error.LoginFailed('Invalid response, invalid opaque value') + + return True + + + def decode(self, response, method, host): + """ + Decode the given response and attempt to generate a + L{DigestedCredentials} from it. + + @type response: L{bytes} + @param response: A string of comma separated key=value pairs + + @type method: L{bytes} + @param method: The action requested to which this response is addressed + (GET, POST, INVITE, OPTIONS, etc). + + @type host: L{bytes} + @param host: The address the request was sent from. + + @raise error.LoginFailed: If the response does not contain a username, + a nonce, an opaque, or if the opaque is invalid. + + @return: L{DigestedCredentials} + """ + response = b' '.join(response.splitlines()) + parts = self._parseparts.findall(response) + auth = {} + for (key, bare, quoted) in parts: + value = (quoted or bare).strip() + auth[nativeString(key.strip())] = value + + username = auth.get('username') + if not username: + raise error.LoginFailed('Invalid response, no username given.') + + if 'opaque' not in auth: + raise error.LoginFailed('Invalid response, no opaque given.') + + if 'nonce' not in auth: + raise error.LoginFailed('Invalid response, no nonce given.') + + # Now verify the nonce/opaque values for this client + if self._verifyOpaque(auth.get('opaque'), auth.get('nonce'), host): + return DigestedCredentials(username, + method, + self.authenticationRealm, + auth) + + + +@implementer(IUsernameHashedPassword) +class CramMD5Credentials(object): + """ + An encapsulation of some CramMD5 hashed credentials. + + @ivar challenge: The challenge to be sent to the client. + @type challenge: L{bytes} + + @ivar response: The hashed response from the client. + @type response: L{bytes} + + @ivar username: The username from the response from the client. + @type username: L{bytes} or L{None} if not yet provided. + """ + username = None + challenge = b'' + response = b'' + + def __init__(self, host=None): + self.host = host + + + def getChallenge(self): + if self.challenge: + return self.challenge + # The data encoded in the first ready response contains an + # presumptively arbitrary string of random digits, a timestamp, and + # the fully-qualified primary host name of the server. The syntax of + # the unencoded form must correspond to that of an RFC 822 'msg-id' + # [RFC822] as described in [POP3]. + # -- RFC 2195 + r = random.randrange(0x7fffffff) + t = time.time() + self.challenge = networkString('<%d.%d@%s>' % ( + r, t, nativeString(self.host) if self.host else None)) + return self.challenge + + + def setResponse(self, response): + self.username, self.response = response.split(None, 1) + + + def moreChallenges(self): + return False + + + def checkPassword(self, password): + verify = hexlify(hmac.HMAC(password, self.challenge).digest()) + return verify == self.response + + + +@implementer(IUsernameHashedPassword) +class UsernameHashedPassword: + + def __init__(self, username, hashed): + self.username = username + self.hashed = hashed + + def checkPassword(self, password): + return self.hashed == password + + + +@implementer(IUsernamePassword) +class UsernamePassword: + + def __init__(self, username, password): + self.username = username + self.password = password + + def checkPassword(self, password): + return self.password == password + + + +@implementer(IAnonymous) +class Anonymous: + pass + + + +class ISSHPrivateKey(ICredentials): + """ + L{ISSHPrivateKey} credentials encapsulate an SSH public key to be checked + against a user's private key. + + @ivar username: The username associated with these credentials. + @type username: L{bytes} + + @ivar algName: The algorithm name for the blob. + @type algName: L{bytes} + + @ivar blob: The public key blob as sent by the client. + @type blob: L{bytes} + + @ivar sigData: The data the signature was made from. + @type sigData: L{bytes} + + @ivar signature: The signed data. This is checked to verify that the user + owns the private key. + @type signature: L{bytes} or L{None} + """ + + + +@implementer(ISSHPrivateKey) +class SSHPrivateKey: + def __init__(self, username, algName, blob, sigData, signature): + self.username = username + self.algName = algName + self.blob = blob + self.sigData = sigData + self.signature = signature diff --git a/contrib/python/Twisted/py2/twisted/cred/error.py b/contrib/python/Twisted/py2/twisted/cred/error.py new file mode 100644 index 00000000000..efd3ec34266 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/cred/error.py @@ -0,0 +1,45 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Cred errors. +""" + +from __future__ import division, absolute_import + + +class Unauthorized(Exception): + """Standard unauthorized error.""" + + + +class LoginFailed(Exception): + """ + The user's request to log in failed for some reason. + """ + + + +class UnauthorizedLogin(LoginFailed, Unauthorized): + """The user was not authorized to log in. + """ + + + +class UnhandledCredentials(LoginFailed): + """A type of credentials were passed in with no knowledge of how to check + them. This is a server configuration error - it means that a protocol was + connected to a Portal without a CredentialChecker that can check all of its + potential authentication strategies. + """ + + + +class LoginDenied(LoginFailed): + """ + The realm rejected this login for some reason. + + Examples of reasons this might be raised include an avatar logging in + too frequently, a quota having been fully used, or the overall server + load being too high. + """ diff --git a/contrib/python/Twisted/py2/twisted/cred/portal.py b/contrib/python/Twisted/py2/twisted/cred/portal.py new file mode 100644 index 00000000000..4c81d5f12cd --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/cred/portal.py @@ -0,0 +1,124 @@ +# -*- test-case-name: twisted.cred.test.test_cred -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +The point of integration of application and authentication. +""" + +from __future__ import division, absolute_import + +from twisted.internet import defer +from twisted.internet.defer import maybeDeferred +from twisted.python import failure, reflect +from twisted.cred import error +from zope.interface import providedBy, Interface + + +class IRealm(Interface): + """ + The realm connects application-specific objects to the + authentication system. + """ + def requestAvatar(avatarId, mind, *interfaces): + """ + Return avatar which provides one of the given interfaces. + + @param avatarId: a string that identifies an avatar, as returned by + L{ICredentialsChecker.requestAvatarId} + (via a Deferred). Alternatively, it may be + C{twisted.cred.checkers.ANONYMOUS}. + @param mind: usually None. See the description of mind in + L{Portal.login}. + @param interfaces: the interface(s) the returned avatar should + implement, e.g. C{IMailAccount}. See the description of + L{Portal.login}. + + @returns: a deferred which will fire a tuple of (interface, + avatarAspect, logout), or the tuple itself. The interface will be + one of the interfaces passed in the 'interfaces' argument. The + 'avatarAspect' will implement that interface. The 'logout' object + is a callable which will detach the mind from the avatar. + """ + + +class Portal(object): + """ + A mediator between clients and a realm. + + A portal is associated with one Realm and zero or more credentials checkers. + When a login is attempted, the portal finds the appropriate credentials + checker for the credentials given, invokes it, and if the credentials are + valid, retrieves the appropriate avatar from the Realm. + + This class is not intended to be subclassed. Customization should be done + in the realm object and in the credentials checker objects. + """ + def __init__(self, realm, checkers=()): + """ + Create a Portal to a L{IRealm}. + """ + self.realm = realm + self.checkers = {} + for checker in checkers: + self.registerChecker(checker) + + + def listCredentialsInterfaces(self): + """ + Return list of credentials interfaces that can be used to login. + """ + return list(self.checkers.keys()) + + + def registerChecker(self, checker, *credentialInterfaces): + if not credentialInterfaces: + credentialInterfaces = checker.credentialInterfaces + for credentialInterface in credentialInterfaces: + self.checkers[credentialInterface] = checker + + + def login(self, credentials, mind, *interfaces): + """ + @param credentials: an implementor of + L{twisted.cred.credentials.ICredentials} + + @param mind: an object which implements a client-side interface for + your particular realm. In many cases, this may be None, so if the + word 'mind' confuses you, just ignore it. + + @param interfaces: list of interfaces for the perspective that the mind + wishes to attach to. Usually, this will be only one interface, for + example IMailAccount. For highly dynamic protocols, however, this + may be a list like (IMailAccount, IUserChooser, IServiceInfo). To + expand: if we are speaking to the system over IMAP, any information + that will be relayed to the user MUST be returned as an + IMailAccount implementor; IMAP clients would not be able to + understand anything else. Any information about unusual status + would have to be relayed as a single mail message in an + otherwise-empty mailbox. However, in a web-based mail system, or a + PB-based client, the ``mind'' object inside the web server + (implemented with a dynamic page-viewing mechanism such as a + Twisted Web Resource) or on the user's client program may be + intelligent enough to respond to several ``server''-side + interfaces. + + @return: A deferred which will fire a tuple of (interface, + avatarAspect, logout). The interface will be one of the interfaces + passed in the 'interfaces' argument. The 'avatarAspect' will + implement that interface. The 'logout' object is a callable which + will detach the mind from the avatar. It must be called when the + user has conceptually disconnected from the service. Although in + some cases this will not be in connectionLost (such as in a + web-based session), it will always be at the end of a user's + interactive session. + """ + for i in self.checkers: + if i.providedBy(credentials): + return maybeDeferred(self.checkers[i].requestAvatarId, credentials + ).addCallback(self.realm.requestAvatar, mind, *interfaces + ) + ifac = providedBy(credentials) + return defer.fail(failure.Failure(error.UnhandledCredentials( + "No checker for %s" % ', '.join(map(reflect.qual, ifac))))) diff --git a/contrib/python/Twisted/py2/twisted/cred/strcred.py b/contrib/python/Twisted/py2/twisted/cred/strcred.py new file mode 100644 index 00000000000..4c3e5bc95e4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/cred/strcred.py @@ -0,0 +1,272 @@ +# -*- test-case-name: twisted.cred.test.test_strcred -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +# + +""" +Support for resolving command-line strings that represent different +checkers available to cred. + +Examples: + - passwd:/etc/passwd + - memory:admin:asdf:user:lkj + - unix +""" + +from __future__ import absolute_import, division + +import sys + +from zope.interface import Interface, Attribute + +from twisted.plugin import getPlugins +from twisted.python import usage + + + +class ICheckerFactory(Interface): + """ + A factory for objects which provide + L{twisted.cred.checkers.ICredentialsChecker}. + + It's implemented by twistd plugins creating checkers. + """ + + authType = Attribute( + 'A tag that identifies the authentication method.') + + + authHelp = Attribute( + 'A detailed (potentially multi-line) description of precisely ' + 'what functionality this CheckerFactory provides.') + + + argStringFormat = Attribute( + 'A short (one-line) description of the argument string format.') + + + credentialInterfaces = Attribute( + 'A list of credentials interfaces that this factory will support.') + + + def generateChecker(argstring): + """ + Return an L{twisted.cred.checkers.ICredentialsChecker} provider using the supplied + argument string. + """ + + + +class StrcredException(Exception): + """ + Base exception class for strcred. + """ + + + +class InvalidAuthType(StrcredException): + """ + Raised when a user provides an invalid identifier for the + authentication plugin (known as the authType). + """ + + + +class InvalidAuthArgumentString(StrcredException): + """ + Raised by an authentication plugin when the argument string + provided is formatted incorrectly. + """ + + + +class UnsupportedInterfaces(StrcredException): + """ + Raised when an application is given a checker to use that does not + provide any of the application's supported credentials interfaces. + """ + + + +# This will be used to warn the users whenever they view help for an +# authType that is not supported by the application. +notSupportedWarning = ("WARNING: This authType is not supported by " + "this application.") + + + +def findCheckerFactories(): + """ + Find all objects that implement L{ICheckerFactory}. + """ + return getPlugins(ICheckerFactory) + + + +def findCheckerFactory(authType): + """ + Find the first checker factory that supports the given authType. + """ + for factory in findCheckerFactories(): + if factory.authType == authType: + return factory + raise InvalidAuthType(authType) + + + +def makeChecker(description): + """ + Returns an L{twisted.cred.checkers.ICredentialsChecker} based on the + contents of a descriptive string. Similar to + L{twisted.application.strports}. + """ + if ':' in description: + authType, argstring = description.split(':', 1) + else: + authType = description + argstring = '' + return findCheckerFactory(authType).generateChecker(argstring) + + + +class AuthOptionMixin: + """ + Defines helper methods that can be added on to any + L{usage.Options} subclass that needs authentication. + + This mixin implements three new options methods: + + The opt_auth method (--auth) will write two new values to the + 'self' dictionary: C{credInterfaces} (a dict of lists) and + C{credCheckers} (a list). + + The opt_help_auth method (--help-auth) will search for all + available checker plugins and list them for the user; it will exit + when finished. + + The opt_help_auth_type method (--help-auth-type) will display + detailed help for a particular checker plugin. + + @cvar supportedInterfaces: An iterable object that returns + credential interfaces which this application is able to support. + + @cvar authOutput: A writeable object to which this options class + will send all help-related output. Default: L{sys.stdout} + """ + + supportedInterfaces = None + authOutput = sys.stdout + + + def supportsInterface(self, interface): + """ + Returns whether a particular credentials interface is supported. + """ + return (self.supportedInterfaces is None + or interface in self.supportedInterfaces) + + + def supportsCheckerFactory(self, factory): + """ + Returns whether a checker factory will provide at least one of + the credentials interfaces that we care about. + """ + for interface in factory.credentialInterfaces: + if self.supportsInterface(interface): + return True + return False + + + def addChecker(self, checker): + """ + Supply a supplied credentials checker to the Options class. + """ + # First figure out which interfaces we're willing to support. + supported = [] + if self.supportedInterfaces is None: + supported = checker.credentialInterfaces + else: + for interface in checker.credentialInterfaces: + if self.supportsInterface(interface): + supported.append(interface) + if not supported: + raise UnsupportedInterfaces(checker.credentialInterfaces) + # If we get this far, then we know we can use this checker. + if 'credInterfaces' not in self: + self['credInterfaces'] = {} + if 'credCheckers' not in self: + self['credCheckers'] = [] + self['credCheckers'].append(checker) + for interface in supported: + self['credInterfaces'].setdefault(interface, []).append(checker) + + + def opt_auth(self, description): + """ + Specify an authentication method for the server. + """ + try: + self.addChecker(makeChecker(description)) + except UnsupportedInterfaces as e: + raise usage.UsageError( + 'Auth plugin not supported: %s' % e.args[0]) + except InvalidAuthType as e: + raise usage.UsageError( + 'Auth plugin not recognized: %s' % e.args[0]) + except Exception as e: + raise usage.UsageError('Unexpected error: %s' % e) + + + def _checkerFactoriesForOptHelpAuth(self): + """ + Return a list of which authTypes will be displayed by --help-auth. + This makes it a lot easier to test this module. + """ + for factory in findCheckerFactories(): + for interface in factory.credentialInterfaces: + if self.supportsInterface(interface): + yield factory + break + + + def opt_help_auth(self): + """ + Show all authentication methods available. + """ + self.authOutput.write("Usage: --auth AuthType[:ArgString]\n") + self.authOutput.write("For detailed help: --help-auth-type AuthType\n") + self.authOutput.write('\n') + # Figure out the right width for our columns + firstLength = 0 + for factory in self._checkerFactoriesForOptHelpAuth(): + if len(factory.authType) > firstLength: + firstLength = len(factory.authType) + formatString = ' %%-%is\t%%s\n' % firstLength + self.authOutput.write(formatString % ('AuthType', 'ArgString format')) + self.authOutput.write(formatString % ('========', '================')) + for factory in self._checkerFactoriesForOptHelpAuth(): + self.authOutput.write( + formatString % (factory.authType, factory.argStringFormat)) + self.authOutput.write('\n') + raise SystemExit(0) + + + def opt_help_auth_type(self, authType): + """ + Show help for a particular authentication type. + """ + try: + cf = findCheckerFactory(authType) + except InvalidAuthType: + raise usage.UsageError("Invalid auth type: %s" % authType) + self.authOutput.write("Usage: --auth %s[:ArgString]\n" % authType) + self.authOutput.write("ArgString format: %s\n" % cf.argStringFormat) + self.authOutput.write('\n') + for line in cf.authHelp.strip().splitlines(): + self.authOutput.write(' %s\n' % line.rstrip()) + self.authOutput.write('\n') + if not self.supportsCheckerFactory(cf): + self.authOutput.write(' %s\n' % notSupportedWarning) + self.authOutput.write('\n') + raise SystemExit(0) diff --git a/contrib/python/Twisted/py2/twisted/enterprise/__init__.py b/contrib/python/Twisted/py2/twisted/enterprise/__init__.py new file mode 100644 index 00000000000..03f17ceaeae --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/enterprise/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Enterprise: Database support for Twisted services. +""" + +__all__ = ['adbapi'] diff --git a/contrib/python/Twisted/py2/twisted/enterprise/adbapi.py b/contrib/python/Twisted/py2/twisted/enterprise/adbapi.py new file mode 100644 index 00000000000..e240a69fb59 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/enterprise/adbapi.py @@ -0,0 +1,502 @@ +# -*- test-case-name: twisted.test.test_adbapi -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An asynchronous mapping to U{DB-API +2.0}. +""" + +import sys + +from twisted.internet import threads +from twisted.python import reflect, log, compat + + +class ConnectionLost(Exception): + """ + This exception means that a db connection has been lost. Client code may + try again. + """ + + + +class Connection(object): + """ + A wrapper for a DB-API connection instance. + + The wrapper passes almost everything to the wrapped connection and so has + the same API. However, the L{Connection} knows about its pool and also + handle reconnecting should when the real connection dies. + """ + + def __init__(self, pool): + self._pool = pool + self._connection = None + self.reconnect() + + + def close(self): + # The way adbapi works right now means that closing a connection is + # a really bad thing as it leaves a dead connection associated with + # a thread in the thread pool. + # Really, I think closing a pooled connection should return it to the + # pool but that's handled by the runWithConnection method already so, + # rather than upsetting anyone by raising an exception, let's ignore + # the request + pass + + + def rollback(self): + if not self._pool.reconnect: + self._connection.rollback() + return + + try: + self._connection.rollback() + curs = self._connection.cursor() + curs.execute(self._pool.good_sql) + curs.close() + self._connection.commit() + return + except: + log.err(None, "Rollback failed") + + self._pool.disconnect(self._connection) + + if self._pool.noisy: + log.msg("Connection lost.") + + raise ConnectionLost() + + + def reconnect(self): + if self._connection is not None: + self._pool.disconnect(self._connection) + self._connection = self._pool.connect() + + + def __getattr__(self, name): + return getattr(self._connection, name) + + + +class Transaction: + """ + A lightweight wrapper for a DB-API 'cursor' object. + + Relays attribute access to the DB cursor. That is, you can call + C{execute()}, C{fetchall()}, etc., and they will be called on the + underlying DB-API cursor object. Attributes will also be retrieved from + there. + """ + _cursor = None + + def __init__(self, pool, connection): + self._pool = pool + self._connection = connection + self.reopen() + + + def close(self): + _cursor = self._cursor + self._cursor = None + _cursor.close() + + + def reopen(self): + if self._cursor is not None: + self.close() + + try: + self._cursor = self._connection.cursor() + return + except: + if not self._pool.reconnect: + raise + else: + log.err(None, "Cursor creation failed") + + if self._pool.noisy: + log.msg('Connection lost, reconnecting') + + self.reconnect() + self._cursor = self._connection.cursor() + + + def reconnect(self): + self._connection.reconnect() + self._cursor = None + + + def __getattr__(self, name): + return getattr(self._cursor, name) + + + +class ConnectionPool: + """ + Represent a pool of connections to a DB-API 2.0 compliant database. + + @ivar connectionFactory: factory for connections, default to L{Connection}. + @type connectionFactory: any callable. + + @ivar transactionFactory: factory for transactions, default to + L{Transaction}. + @type transactionFactory: any callable + + @ivar shutdownID: L{None} or a handle on the shutdown event trigger which + will be used to stop the connection pool workers when the reactor + stops. + + @ivar _reactor: The reactor which will be used to schedule startup and + shutdown events. + @type _reactor: L{IReactorCore} provider + """ + + CP_ARGS = "min max name noisy openfun reconnect good_sql".split() + + noisy = False # If true, generate informational log messages + min = 3 # Minimum number of connections in pool + max = 5 # Maximum number of connections in pool + name = None # Name to assign to thread pool for debugging + openfun = None # A function to call on new connections + reconnect = False # Reconnect when connections fail + good_sql = 'select 1' # A query which should always succeed + + running = False # True when the pool is operating + connectionFactory = Connection + transactionFactory = Transaction + + # Initialize this to None so it's available in close() even if start() + # never runs. + shutdownID = None + + def __init__(self, dbapiName, *connargs, **connkw): + """ + Create a new L{ConnectionPool}. + + Any positional or keyword arguments other than those documented here + are passed to the DB-API object when connecting. Use these arguments to + pass database names, usernames, passwords, etc. + + @param dbapiName: an import string to use to obtain a DB-API compatible + module (e.g. C{'pyPgSQL.PgSQL'}) + + @param cp_min: the minimum number of connections in pool (default 3) + + @param cp_max: the maximum number of connections in pool (default 5) + + @param cp_noisy: generate informational log messages during operation + (default C{False}) + + @param cp_openfun: a callback invoked after every C{connect()} on the + underlying DB-API object. The callback is passed a new DB-API + connection object. This callback can setup per-connection state + such as charset, timezone, etc. + + @param cp_reconnect: detect connections which have failed and reconnect + (default C{False}). Failed connections may result in + L{ConnectionLost} exceptions, which indicate the query may need to + be re-sent. + + @param cp_good_sql: an sql query which should always succeed and change + no state (default C{'select 1'}) + + @param cp_reactor: use this reactor instead of the global reactor + (added in Twisted 10.2). + @type cp_reactor: L{IReactorCore} provider + """ + self.dbapiName = dbapiName + self.dbapi = reflect.namedModule(dbapiName) + + if getattr(self.dbapi, 'apilevel', None) != '2.0': + log.msg('DB API module not DB API 2.0 compliant.') + + if getattr(self.dbapi, 'threadsafety', 0) < 1: + log.msg('DB API module not sufficiently thread-safe.') + + reactor = connkw.pop('cp_reactor', None) + if reactor is None: + from twisted.internet import reactor + self._reactor = reactor + + self.connargs = connargs + self.connkw = connkw + + for arg in self.CP_ARGS: + cpArg = 'cp_%s' % (arg,) + if cpArg in connkw: + setattr(self, arg, connkw[cpArg]) + del connkw[cpArg] + + self.min = min(self.min, self.max) + self.max = max(self.min, self.max) + + # All connections, hashed on thread id + self.connections = {} + + # These are optional so import them here + from twisted.python import threadpool + from twisted.python import threadable + + self.threadID = threadable.getThreadID + self.threadpool = threadpool.ThreadPool(self.min, self.max) + self.startID = self._reactor.callWhenRunning(self._start) + + + def _start(self): + self.startID = None + return self.start() + + + def start(self): + """ + Start the connection pool. + + If you are using the reactor normally, this function does *not* + need to be called. + """ + if not self.running: + self.threadpool.start() + self.shutdownID = self._reactor.addSystemEventTrigger( + 'during', 'shutdown', self.finalClose) + self.running = True + + + def runWithConnection(self, func, *args, **kw): + """ + Execute a function with a database connection and return the result. + + @param func: A callable object of one argument which will be executed + in a thread with a connection from the pool. It will be passed as + its first argument a L{Connection} instance (whose interface is + mostly identical to that of a connection object for your DB-API + module of choice), and its results will be returned as a + L{Deferred}. If the method raises an exception the transaction will + be rolled back. Otherwise, the transaction will be committed. + B{Note} that this function is B{not} run in the main thread: it + must be threadsafe. + + @param *args: positional arguments to be passed to func + + @param **kw: keyword arguments to be passed to func + + @return: a L{Deferred} which will fire the return value of + C{func(Transaction(...), *args, **kw)}, or a + L{twisted.python.failure.Failure}. + """ + return threads.deferToThreadPool(self._reactor, self.threadpool, + self._runWithConnection, + func, *args, **kw) + + + def _runWithConnection(self, func, *args, **kw): + conn = self.connectionFactory(self) + try: + result = func(conn, *args, **kw) + conn.commit() + return result + except: + excType, excValue, excTraceback = sys.exc_info() + try: + conn.rollback() + except: + log.err(None, "Rollback failed") + compat.reraise(excValue, excTraceback) + + + def runInteraction(self, interaction, *args, **kw): + """ + Interact with the database and return the result. + + The 'interaction' is a callable object which will be executed in a + thread using a pooled connection. It will be passed an L{Transaction} + object as an argument (whose interface is identical to that of the + database cursor for your DB-API module of choice), and its results will + be returned as a L{Deferred}. If running the method raises an + exception, the transaction will be rolled back. If the method returns a + value, the transaction will be committed. + + NOTE that the function you pass is *not* run in the main thread: you + may have to worry about thread-safety in the function you pass to this + if it tries to use non-local objects. + + @param interaction: a callable object whose first argument is an + L{adbapi.Transaction}. + + @param *args: additional positional arguments to be passed to + interaction + + @param **kw: keyword arguments to be passed to interaction + + @return: a Deferred which will fire the return value of + C{interaction(Transaction(...), *args, **kw)}, or a + L{twisted.python.failure.Failure}. + """ + return threads.deferToThreadPool(self._reactor, self.threadpool, + self._runInteraction, + interaction, *args, **kw) + + + def runQuery(self, *args, **kw): + """ + Execute an SQL query and return the result. + + A DB-API cursor which will be invoked with C{cursor.execute(*args, + **kw)}. The exact nature of the arguments will depend on the specific + flavor of DB-API being used, but the first argument in C{*args} be an + SQL statement. The result of a subsequent C{cursor.fetchall()} will be + fired to the L{Deferred} which is returned. If either the 'execute' or + 'fetchall' methods raise an exception, the transaction will be rolled + back and a L{twisted.python.failure.Failure} returned. + + The C{*args} and C{**kw} arguments will be passed to the DB-API + cursor's 'execute' method. + + @return: a L{Deferred} which will fire the return value of a DB-API + cursor's 'fetchall' method, or a L{twisted.python.failure.Failure}. + """ + return self.runInteraction(self._runQuery, *args, **kw) + + + def runOperation(self, *args, **kw): + """ + Execute an SQL query and return L{None}. + + A DB-API cursor which will be invoked with C{cursor.execute(*args, + **kw)}. The exact nature of the arguments will depend on the specific + flavor of DB-API being used, but the first argument in C{*args} will be + an SQL statement. This method will not attempt to fetch any results + from the query and is thus suitable for C{INSERT}, C{DELETE}, and other + SQL statements which do not return values. If the 'execute' method + raises an exception, the transaction will be rolled back and a + L{Failure} returned. + + The C{*args} and C{*kw} arguments will be passed to the DB-API cursor's + 'execute' method. + + @return: a L{Deferred} which will fire with L{None} or a + L{twisted.python.failure.Failure}. + """ + return self.runInteraction(self._runOperation, *args, **kw) + + + def close(self): + """ + Close all pool connections and shutdown the pool. + """ + if self.shutdownID: + self._reactor.removeSystemEventTrigger(self.shutdownID) + self.shutdownID = None + if self.startID: + self._reactor.removeSystemEventTrigger(self.startID) + self.startID = None + self.finalClose() + + + def finalClose(self): + """ + This should only be called by the shutdown trigger. + """ + self.shutdownID = None + self.threadpool.stop() + self.running = False + for conn in self.connections.values(): + self._close(conn) + self.connections.clear() + + + def connect(self): + """ + Return a database connection when one becomes available. + + This method blocks and should be run in a thread from the internal + threadpool. Don't call this method directly from non-threaded code. + Using this method outside the external threadpool may exceed the + maximum number of connections in the pool. + + @return: a database connection from the pool. + """ + + tid = self.threadID() + conn = self.connections.get(tid) + if conn is None: + if self.noisy: + log.msg('adbapi connecting: %s' % (self.dbapiName,)) + conn = self.dbapi.connect(*self.connargs, **self.connkw) + if self.openfun is not None: + self.openfun(conn) + self.connections[tid] = conn + return conn + + + def disconnect(self, conn): + """ + Disconnect a database connection associated with this pool. + + Note: This function should only be used by the same thread which called + L{ConnectionPool.connect}. As with C{connect}, this function is not + used in normal non-threaded Twisted code. + """ + tid = self.threadID() + if conn is not self.connections.get(tid): + raise Exception("wrong connection for thread") + if conn is not None: + self._close(conn) + del self.connections[tid] + + + def _close(self, conn): + if self.noisy: + log.msg('adbapi closing: %s' % (self.dbapiName,)) + try: + conn.close() + except: + log.err(None, "Connection close failed") + + + def _runInteraction(self, interaction, *args, **kw): + conn = self.connectionFactory(self) + trans = self.transactionFactory(self, conn) + try: + result = interaction(trans, *args, **kw) + trans.close() + conn.commit() + return result + except: + excType, excValue, excTraceback = sys.exc_info() + try: + conn.rollback() + except: + log.err(None, "Rollback failed") + compat.reraise(excValue, excTraceback) + + + def _runQuery(self, trans, *args, **kw): + trans.execute(*args, **kw) + return trans.fetchall() + + + def _runOperation(self, trans, *args, **kw): + trans.execute(*args, **kw) + + + def __getstate__(self): + return {'dbapiName': self.dbapiName, + 'min': self.min, + 'max': self.max, + 'noisy': self.noisy, + 'reconnect': self.reconnect, + 'good_sql': self.good_sql, + 'connargs': self.connargs, + 'connkw': self.connkw} + + + def __setstate__(self, state): + self.__dict__ = state + self.__init__(self.dbapiName, *self.connargs, **self.connkw) + + + +__all__ = ['Transaction', 'ConnectionPool'] diff --git a/contrib/python/Twisted/py2/twisted/internet/__init__.py b/contrib/python/Twisted/py2/twisted/internet/__init__.py new file mode 100644 index 00000000000..a3d851d1983 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Internet: Asynchronous I/O and Events. + +Twisted Internet is a collection of compatible event-loops for Python. It contains +the code to dispatch events to interested observers and a portable API so that +observers need not care about which event loop is running. Thus, it is possible +to use the same code for different loops, from Twisted's basic, yet portable, +select-based loop to the loops of various GUI toolkits like GTK+ or Tk. +""" diff --git a/contrib/python/Twisted/py2/twisted/internet/_baseprocess.py b/contrib/python/Twisted/py2/twisted/internet/_baseprocess.py new file mode 100644 index 00000000000..f02eff7aba7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_baseprocess.py @@ -0,0 +1,66 @@ +# -*- test-case-name: twisted.test.test_process -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Cross-platform process-related functionality used by different +L{IReactorProcess} implementations. +""" + +from twisted.python.reflect import qual +from twisted.python.deprecate import getWarningMethod +from twisted.python.failure import Failure +from twisted.python.log import err + +_missingProcessExited = ("Since Twisted 8.2, IProcessProtocol.processExited " + "is required. %s must implement it.") + + + +class BaseProcess(object): + pid = None + status = None + lostProcess = 0 + proto = None + + def __init__(self, protocol): + self.proto = protocol + + + def _callProcessExited(self, reason): + default = object() + processExited = getattr(self.proto, 'processExited', default) + if processExited is default: + getWarningMethod()( + _missingProcessExited % (qual(self.proto.__class__),), + DeprecationWarning, stacklevel=0) + else: + try: + processExited(Failure(reason)) + except: + err(None, "unexpected error in processExited") + + + def processEnded(self, status): + """ + This is called when the child terminates. + """ + self.status = status + self.lostProcess += 1 + self.pid = None + self._callProcessExited(self._getReason(status)) + self.maybeCallProcessEnded() + + + def maybeCallProcessEnded(self): + """ + Call processEnded on protocol after final cleanup. + """ + if self.proto is not None: + reason = self._getReason(self.status) + proto = self.proto + self.proto = None + try: + proto.processEnded(Failure(reason)) + except: + err(None, "unexpected error in processEnded") diff --git a/contrib/python/Twisted/py2/twisted/internet/_dumbwin32proc.py b/contrib/python/Twisted/py2/twisted/internet/_dumbwin32proc.py new file mode 100644 index 00000000000..4fa6574bf37 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_dumbwin32proc.py @@ -0,0 +1,433 @@ +# -*- test-case-name: twisted.test.test_process -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +http://isometri.cc/strips/gates_in_the_head +""" + +from __future__ import absolute_import, division, print_function + +import os +import sys + +# Win32 imports +import win32api +import win32con +import win32event +import win32file +import win32pipe +import win32process +import win32security + +import pywintypes + +# Security attributes for pipes +PIPE_ATTRS_INHERITABLE = win32security.SECURITY_ATTRIBUTES() +PIPE_ATTRS_INHERITABLE.bInheritHandle = 1 + +from zope.interface import implementer +from twisted.internet.interfaces import IProcessTransport, IConsumer, IProducer + +from twisted.python.compat import items, _PY3 +from twisted.python.win32 import quoteArguments +from twisted.python.util import _replaceIf + +from twisted.internet import error + +from twisted.internet import _pollingfile +from twisted.internet._baseprocess import BaseProcess + + +@_replaceIf(_PY3, getattr(os, 'fsdecode', None)) +def _fsdecode(x): + """ + Decode a string to a L{unicode} representation, passing + through existing L{unicode} unchanged. + + @param x: The string to be conditionally decoded. + @type x: L{bytes} or L{unicode} + + @return: L{unicode} + """ + if isinstance(x, bytes): + return x.decode(sys.getfilesystemencoding()) + else: + return x + + + +def debug(msg): + print(msg) + sys.stdout.flush() + + + +class _Reaper(_pollingfile._PollableResource): + + def __init__(self, proc): + self.proc = proc + + + def checkWork(self): + if win32event.WaitForSingleObject(self.proc.hProcess, 0) != win32event.WAIT_OBJECT_0: + return 0 + exitCode = win32process.GetExitCodeProcess(self.proc.hProcess) + self.deactivate() + self.proc.processEnded(exitCode) + return 0 + + + +def _findShebang(filename): + """ + Look for a #! line, and return the value following the #! if one exists, or + None if this file is not a script. + + I don't know if there are any conventions for quoting in Windows shebang + lines, so this doesn't support any; therefore, you may not pass any + arguments to scripts invoked as filters. That's probably wrong, so if + somebody knows more about the cultural expectations on Windows, please feel + free to fix. + + This shebang line support was added in support of the CGI tests; + appropriately enough, I determined that shebang lines are culturally + accepted in the Windows world through this page:: + + http://www.cgi101.com/learn/connect/winxp.html + + @param filename: str representing a filename + + @return: a str representing another filename. + """ + with open(filename, 'rU') as f: + if f.read(2) == '#!': + exe = f.readline(1024).strip('\n') + return exe + + + +def _invalidWin32App(pywinerr): + """ + Determine if a pywintypes.error is telling us that the given process is + 'not a valid win32 application', i.e. not a PE format executable. + + @param pywinerr: a pywintypes.error instance raised by CreateProcess + + @return: a boolean + """ + + # Let's do this better in the future, but I have no idea what this error + # is; MSDN doesn't mention it, and there is no symbolic constant in + # win32process module that represents 193. + + return pywinerr.args[0] == 193 + + + +@implementer(IProcessTransport, IConsumer, IProducer) +class Process(_pollingfile._PollingTimer, BaseProcess): + """ + A process that integrates with the Twisted event loop. + + If your subprocess is a python program, you need to: + + - Run python.exe with the '-u' command line option - this turns on + unbuffered I/O. Buffering stdout/err/in can cause problems, see e.g. + http://support.microsoft.com/default.aspx?scid=kb;EN-US;q1903 + + - If you don't want Windows messing with data passed over + stdin/out/err, set the pipes to be in binary mode:: + + import os, sys, mscvrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) + + """ + closedNotifies = 0 + + def __init__(self, reactor, protocol, command, args, environment, path): + """ + Create a new child process. + """ + _pollingfile._PollingTimer.__init__(self, reactor) + BaseProcess.__init__(self, protocol) + + # security attributes for pipes + sAttrs = win32security.SECURITY_ATTRIBUTES() + sAttrs.bInheritHandle = 1 + + # create the pipes which will connect to the secondary process + self.hStdoutR, hStdoutW = win32pipe.CreatePipe(sAttrs, 0) + self.hStderrR, hStderrW = win32pipe.CreatePipe(sAttrs, 0) + hStdinR, self.hStdinW = win32pipe.CreatePipe(sAttrs, 0) + + win32pipe.SetNamedPipeHandleState(self.hStdinW, + win32pipe.PIPE_NOWAIT, + None, + None) + + # set the info structure for the new process. + StartupInfo = win32process.STARTUPINFO() + StartupInfo.hStdOutput = hStdoutW + StartupInfo.hStdError = hStderrW + StartupInfo.hStdInput = hStdinR + StartupInfo.dwFlags = win32process.STARTF_USESTDHANDLES + + # Create new handles whose inheritance property is false + currentPid = win32api.GetCurrentProcess() + + tmp = win32api.DuplicateHandle(currentPid, self.hStdoutR, currentPid, 0, 0, + win32con.DUPLICATE_SAME_ACCESS) + win32file.CloseHandle(self.hStdoutR) + self.hStdoutR = tmp + + tmp = win32api.DuplicateHandle(currentPid, self.hStderrR, currentPid, 0, 0, + win32con.DUPLICATE_SAME_ACCESS) + win32file.CloseHandle(self.hStderrR) + self.hStderrR = tmp + + tmp = win32api.DuplicateHandle(currentPid, self.hStdinW, currentPid, 0, 0, + win32con.DUPLICATE_SAME_ACCESS) + win32file.CloseHandle(self.hStdinW) + self.hStdinW = tmp + + # Add the specified environment to the current environment - this is + # necessary because certain operations are only supported on Windows + # if certain environment variables are present. + + env = os.environ.copy() + env.update(environment or {}) + newenv = {} + for key, value in items(env): + + key = _fsdecode(key) + value = _fsdecode(value) + + newenv[key] = value + + env = newenv + + # Make sure all the arguments are Unicode. + args = [_fsdecode(x) for x in args] + + cmdline = quoteArguments(args) + + # The command, too, needs to be Unicode, if it is a value. + command = _fsdecode(command) if command else command + path = _fsdecode(path) if path else path + + # TODO: error detection here. See #2787 and #4184. + def doCreate(): + flags = win32con.CREATE_NO_WINDOW + self.hProcess, self.hThread, self.pid, dwTid = win32process.CreateProcess( + command, cmdline, None, None, 1, flags, env, path, StartupInfo) + try: + doCreate() + except pywintypes.error as pwte: + if not _invalidWin32App(pwte): + # This behavior isn't _really_ documented, but let's make it + # consistent with the behavior that is documented. + raise OSError(pwte) + else: + # look for a shebang line. Insert the original 'command' + # (actually a script) into the new arguments list. + sheb = _findShebang(command) + if sheb is None: + raise OSError( + "%r is neither a Windows executable, " + "nor a script with a shebang line" % command) + else: + args = list(args) + args.insert(0, command) + cmdline = quoteArguments(args) + origcmd = command + command = sheb + try: + # Let's try again. + doCreate() + except pywintypes.error as pwte2: + # d'oh, failed again! + if _invalidWin32App(pwte2): + raise OSError( + "%r has an invalid shebang line: " + "%r is not a valid executable" % ( + origcmd, sheb)) + raise OSError(pwte2) + + # close handles which only the child will use + win32file.CloseHandle(hStderrW) + win32file.CloseHandle(hStdoutW) + win32file.CloseHandle(hStdinR) + + # set up everything + self.stdout = _pollingfile._PollableReadPipe( + self.hStdoutR, + lambda data: self.proto.childDataReceived(1, data), + self.outConnectionLost) + + self.stderr = _pollingfile._PollableReadPipe( + self.hStderrR, + lambda data: self.proto.childDataReceived(2, data), + self.errConnectionLost) + + self.stdin = _pollingfile._PollableWritePipe( + self.hStdinW, self.inConnectionLost) + + for pipewatcher in self.stdout, self.stderr, self.stdin: + self._addPollableResource(pipewatcher) + + # notify protocol + self.proto.makeConnection(self) + + self._addPollableResource(_Reaper(self)) + + + def signalProcess(self, signalID): + if self.pid is None: + raise error.ProcessExitedAlready() + if signalID in ("INT", "TERM", "KILL"): + win32process.TerminateProcess(self.hProcess, 1) + + + def _getReason(self, status): + if status == 0: + return error.ProcessDone(status) + return error.ProcessTerminated(status) + + + def write(self, data): + """ + Write data to the process' stdin. + + @type data: C{bytes} + """ + self.stdin.write(data) + + + def writeSequence(self, seq): + """ + Write data to the process' stdin. + + @type data: C{list} of C{bytes} + """ + self.stdin.writeSequence(seq) + + + def writeToChild(self, fd, data): + """ + Similar to L{ITransport.write} but also allows the file descriptor in + the child process which will receive the bytes to be specified. + + This implementation is limited to writing to the child's standard input. + + @param fd: The file descriptor to which to write. Only stdin (C{0}) is + supported. + @type fd: C{int} + + @param data: The bytes to write. + @type data: C{bytes} + + @return: L{None} + + @raise KeyError: If C{fd} is anything other than the stdin file + descriptor (C{0}). + """ + if fd == 0: + self.stdin.write(data) + else: + raise KeyError(fd) + + + def closeChildFD(self, fd): + if fd == 0: + self.closeStdin() + elif fd == 1: + self.closeStdout() + elif fd == 2: + self.closeStderr() + else: + raise NotImplementedError("Only standard-IO file descriptors available on win32") + + + def closeStdin(self): + """Close the process' stdin. + """ + self.stdin.close() + + + def closeStderr(self): + self.stderr.close() + + + def closeStdout(self): + self.stdout.close() + + + def loseConnection(self): + """ + Close the process' stdout, in and err. + """ + self.closeStdin() + self.closeStdout() + self.closeStderr() + + + def outConnectionLost(self): + self.proto.childConnectionLost(1) + self.connectionLostNotify() + + + def errConnectionLost(self): + self.proto.childConnectionLost(2) + self.connectionLostNotify() + + + def inConnectionLost(self): + self.proto.childConnectionLost(0) + self.connectionLostNotify() + + + def connectionLostNotify(self): + """ + Will be called 3 times, by stdout/err threads and process handle. + """ + self.closedNotifies += 1 + self.maybeCallProcessEnded() + + + def maybeCallProcessEnded(self): + if self.closedNotifies == 3 and self.lostProcess: + win32file.CloseHandle(self.hProcess) + win32file.CloseHandle(self.hThread) + self.hProcess = None + self.hThread = None + BaseProcess.maybeCallProcessEnded(self) + + # IConsumer + def registerProducer(self, producer, streaming): + self.stdin.registerProducer(producer, streaming) + + + def unregisterProducer(self): + self.stdin.unregisterProducer() + + # IProducer + def pauseProducing(self): + self._pause() + + + def resumeProducing(self): + self._unpause() + + + def stopProducing(self): + self.loseConnection() + + + def __repr__(self): + """ + Return a string representation of the process. + """ + return "<%s pid=%s>" % (self.__class__.__name__, self.pid) diff --git a/contrib/python/Twisted/py2/twisted/internet/_glibbase.py b/contrib/python/Twisted/py2/twisted/internet/_glibbase.py new file mode 100644 index 00000000000..47e162d96a6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_glibbase.py @@ -0,0 +1,390 @@ +# -*- test-case-name: twisted.internet.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module provides base support for Twisted to interact with the glib/gtk +mainloops. + +The classes in this module should not be used directly, but rather you should +import gireactor or gtk3reactor for GObject Introspection based applications, +or glib2reactor or gtk2reactor for applications using legacy static bindings. +""" + +from __future__ import division, absolute_import + +import sys + +from zope.interface import implementer + +from twisted.internet import base, posixbase, selectreactor +from twisted.internet.interfaces import IReactorFDSet +from twisted.python import log + + + +def ensureNotImported(moduleNames, errorMessage, preventImports=[]): + """ + Check whether the given modules were imported, and if requested, ensure + they will not be importable in the future. + + @param moduleNames: A list of module names we make sure aren't imported. + @type moduleNames: C{list} of C{str} + + @param preventImports: A list of module name whose future imports should + be prevented. + @type preventImports: C{list} of C{str} + + @param errorMessage: Message to use when raising an C{ImportError}. + @type errorMessage: C{str} + + @raises: C{ImportError} with given error message if a given module name + has already been imported. + """ + for name in moduleNames: + if sys.modules.get(name) is not None: + raise ImportError(errorMessage) + + # Disable module imports to avoid potential problems. + for name in preventImports: + sys.modules[name] = None + + + +class GlibWaker(posixbase._UnixWaker): + """ + Run scheduled events after waking up. + """ + + def doRead(self): + posixbase._UnixWaker.doRead(self) + self.reactor._simulate() + + + +@implementer(IReactorFDSet) +class GlibReactorBase(posixbase.PosixReactorBase, posixbase._PollLikeMixin): + """ + Base class for GObject event loop reactors. + + Notification for I/O events (reads and writes on file descriptors) is done + by the gobject-based event loop. File descriptors are registered with + gobject with the appropriate flags for read/write/disconnect notification. + + Time-based events, the results of C{callLater} and C{callFromThread}, are + handled differently. Rather than registering each event with gobject, a + single gobject timeout is registered for the earliest scheduled event, the + output of C{reactor.timeout()}. For example, if there are timeouts in 1, 2 + and 3.4 seconds, a single timeout is registered for 1 second in the + future. When this timeout is hit, C{_simulate} is called, which calls the + appropriate Twisted-level handlers, and a new timeout is added to gobject + by the C{_reschedule} method. + + To handle C{callFromThread} events, we use a custom waker that calls + C{_simulate} whenever it wakes up. + + @ivar _sources: A dictionary mapping L{FileDescriptor} instances to + GSource handles. + + @ivar _reads: A set of L{FileDescriptor} instances currently monitored for + reading. + + @ivar _writes: A set of L{FileDescriptor} instances currently monitored for + writing. + + @ivar _simtag: A GSource handle for the next L{simulate} call. + """ + + # Install a waker that knows it needs to call C{_simulate} in order to run + # callbacks queued from a thread: + _wakerFactory = GlibWaker + + def __init__(self, glib_module, gtk_module, useGtk=False): + self._simtag = None + self._reads = set() + self._writes = set() + self._sources = {} + self._glib = glib_module + self._gtk = gtk_module + posixbase.PosixReactorBase.__init__(self) + + self._source_remove = self._glib.source_remove + self._timeout_add = self._glib.timeout_add + + def _mainquit(): + if self._gtk.main_level(): + self._gtk.main_quit() + + if useGtk: + self._pending = self._gtk.events_pending + self._iteration = self._gtk.main_iteration_do + self._crash = _mainquit + self._run = self._gtk.main + else: + self.context = self._glib.main_context_default() + self._pending = self.context.pending + self._iteration = self.context.iteration + self.loop = self._glib.MainLoop() + self._crash = lambda: self._glib.idle_add(self.loop.quit) + self._run = self.loop.run + + + def _handleSignals(self): + # First, install SIGINT and friends: + base._SignalReactorMixin._handleSignals(self) + # Next, since certain versions of gtk will clobber our signal handler, + # set all signal handlers again after the event loop has started to + # ensure they're *really* set. We don't call this twice so we don't + # leak file descriptors created in the SIGCHLD initialization: + self.callLater(0, posixbase.PosixReactorBase._handleSignals, self) + + + # The input_add function in pygtk1 checks for objects with a + # 'fileno' method and, if present, uses the result of that method + # as the input source. The pygtk2 input_add does not do this. The + # function below replicates the pygtk1 functionality. + + # In addition, pygtk maps gtk.input_add to _gobject.io_add_watch, and + # g_io_add_watch() takes different condition bitfields than + # gtk_input_add(). We use g_io_add_watch() here in case pygtk fixes this + # bug. + def input_add(self, source, condition, callback): + if hasattr(source, 'fileno'): + # handle python objects + def wrapper(ignored, condition): + return callback(source, condition) + fileno = source.fileno() + else: + fileno = source + wrapper = callback + return self._glib.io_add_watch( + fileno, condition, wrapper, + priority=self._glib.PRIORITY_DEFAULT_IDLE) + + + def _ioEventCallback(self, source, condition): + """ + Called by event loop when an I/O event occurs. + """ + log.callWithLogger( + source, self._doReadOrWrite, source, source, condition) + return True # True = don't auto-remove the source + + + def _add(self, source, primary, other, primaryFlag, otherFlag): + """ + Add the given L{FileDescriptor} for monitoring either for reading or + writing. If the file is already monitored for the other operation, we + delete the previous registration and re-register it for both reading + and writing. + """ + if source in primary: + return + flags = primaryFlag + if source in other: + self._source_remove(self._sources[source]) + flags |= otherFlag + self._sources[source] = self.input_add( + source, flags, self._ioEventCallback) + primary.add(source) + + + def addReader(self, reader): + """ + Add a L{FileDescriptor} for monitoring of data available to read. + """ + self._add(reader, self._reads, self._writes, + self.INFLAGS, self.OUTFLAGS) + + + def addWriter(self, writer): + """ + Add a L{FileDescriptor} for monitoring ability to write data. + """ + self._add(writer, self._writes, self._reads, + self.OUTFLAGS, self.INFLAGS) + + + def getReaders(self): + """ + Retrieve the list of current L{FileDescriptor} monitored for reading. + """ + return list(self._reads) + + + def getWriters(self): + """ + Retrieve the list of current L{FileDescriptor} monitored for writing. + """ + return list(self._writes) + + + def removeAll(self): + """ + Remove monitoring for all registered L{FileDescriptor}s. + """ + return self._removeAll(self._reads, self._writes) + + + def _remove(self, source, primary, other, flags): + """ + Remove monitoring the given L{FileDescriptor} for either reading or + writing. If it's still monitored for the other operation, we + re-register the L{FileDescriptor} for only that operation. + """ + if source not in primary: + return + self._source_remove(self._sources[source]) + primary.remove(source) + if source in other: + self._sources[source] = self.input_add( + source, flags, self._ioEventCallback) + else: + self._sources.pop(source) + + + def removeReader(self, reader): + """ + Stop monitoring the given L{FileDescriptor} for reading. + """ + self._remove(reader, self._reads, self._writes, self.OUTFLAGS) + + + def removeWriter(self, writer): + """ + Stop monitoring the given L{FileDescriptor} for writing. + """ + self._remove(writer, self._writes, self._reads, self.INFLAGS) + + + def iterate(self, delay=0): + """ + One iteration of the event loop, for trial's use. + + This is not used for actual reactor runs. + """ + self.runUntilCurrent() + while self._pending(): + self._iteration(0) + + + def crash(self): + """ + Crash the reactor. + """ + posixbase.PosixReactorBase.crash(self) + self._crash() + + + def stop(self): + """ + Stop the reactor. + """ + posixbase.PosixReactorBase.stop(self) + # The base implementation only sets a flag, to ensure shutting down is + # not reentrant. Unfortunately, this flag is not meaningful to the + # gobject event loop. We therefore call wakeUp() to ensure the event + # loop will call back into Twisted once this iteration is done. This + # will result in self.runUntilCurrent() being called, where the stop + # flag will trigger the actual shutdown process, eventually calling + # crash() which will do the actual gobject event loop shutdown. + self.wakeUp() + + + def run(self, installSignalHandlers=True): + """ + Run the reactor. + """ + self.callWhenRunning(self._reschedule) + self.startRunning(installSignalHandlers=installSignalHandlers) + if self._started: + self._run() + + + def callLater(self, *args, **kwargs): + """ + Schedule a C{DelayedCall}. + """ + result = posixbase.PosixReactorBase.callLater(self, *args, **kwargs) + # Make sure we'll get woken up at correct time to handle this new + # scheduled call: + self._reschedule() + return result + + + def _reschedule(self): + """ + Schedule a glib timeout for C{_simulate}. + """ + if self._simtag is not None: + self._source_remove(self._simtag) + self._simtag = None + timeout = self.timeout() + if timeout is not None: + self._simtag = self._timeout_add( + int(timeout * 1000), self._simulate, + priority=self._glib.PRIORITY_DEFAULT_IDLE) + + + def _simulate(self): + """ + Run timers, and then reschedule glib timeout for next scheduled event. + """ + self.runUntilCurrent() + self._reschedule() + + + +class PortableGlibReactorBase(selectreactor.SelectReactor): + """ + Base class for GObject event loop reactors that works on Windows. + + Sockets aren't supported by GObject's input_add on Win32. + """ + def __init__(self, glib_module, gtk_module, useGtk=False): + self._simtag = None + self._glib = glib_module + self._gtk = gtk_module + selectreactor.SelectReactor.__init__(self) + + self._source_remove = self._glib.source_remove + self._timeout_add = self._glib.timeout_add + + def _mainquit(): + if self._gtk.main_level(): + self._gtk.main_quit() + + if useGtk: + self._crash = _mainquit + self._run = self._gtk.main + else: + self.loop = self._glib.MainLoop() + self._crash = lambda: self._glib.idle_add(self.loop.quit) + self._run = self.loop.run + + + def crash(self): + selectreactor.SelectReactor.crash(self) + self._crash() + + + def run(self, installSignalHandlers=True): + self.startRunning(installSignalHandlers=installSignalHandlers) + self._timeout_add(0, self.simulate) + if self._started: + self._run() + + + def simulate(self): + """ + Run simulation loops and reschedule callbacks. + """ + if self._simtag is not None: + self._source_remove(self._simtag) + self.iterate() + timeout = min(self.timeout(), 0.01) + if timeout is None: + timeout = 0.01 + self._simtag = self._timeout_add( + int(timeout * 1000), self.simulate, + priority=self._glib.PRIORITY_DEFAULT_IDLE) diff --git a/contrib/python/Twisted/py2/twisted/internet/_idna.py b/contrib/python/Twisted/py2/twisted/internet/_idna.py new file mode 100644 index 00000000000..997dabaadfb --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_idna.py @@ -0,0 +1,54 @@ +# -*- test-case-name: twisted.test.test_sslverify -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Shared interface to IDNA encoding and decoding, using the C{idna} PyPI package +if available, otherwise the stdlib implementation. +""" + +def _idnaBytes(text): + """ + Convert some text typed by a human into some ASCII bytes. + + This is provided to allow us to use the U{partially-broken IDNA + implementation in the standard library } + if the more-correct U{idna } package is + not available; C{service_identity} is somewhat stricter about this. + + @param text: A domain name, hopefully. + @type text: L{unicode} + + @return: The domain name's IDNA representation, encoded as bytes. + @rtype: L{bytes} + """ + try: + import idna + except ImportError: + return text.encode("idna") + else: + return idna.encode(text) + + + +def _idnaText(octets): + """ + Convert some IDNA-encoded octets into some human-readable text. + + Currently only used by the tests. + + @param octets: Some bytes representing a hostname. + @type octets: L{bytes} + + @return: A human-readable domain name. + @rtype: L{unicode} + """ + try: + import idna + except ImportError: + return octets.decode("idna") + else: + return idna.decode(octets) + + + diff --git a/contrib/python/Twisted/py2/twisted/internet/_newtls.py b/contrib/python/Twisted/py2/twisted/internet/_newtls.py new file mode 100644 index 00000000000..0eb33121546 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_newtls.py @@ -0,0 +1,271 @@ +# -*- test-case-name: twisted.test.test_ssl -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module implements memory BIO based TLS support. It is the preferred +implementation and will be used whenever pyOpenSSL 0.10 or newer is installed +(whenever L{twisted.protocols.tls} is importable). + +@since: 11.1 +""" + +from __future__ import division, absolute_import + +from zope.interface import implementer +from zope.interface import directlyProvides + +from twisted.internet.interfaces import ITLSTransport, ISSLTransport +from twisted.internet.abstract import FileDescriptor + +from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol + + +class _BypassTLS(object): + """ + L{_BypassTLS} is used as the transport object for the TLS protocol object + used to implement C{startTLS}. Its methods skip any TLS logic which + C{startTLS} enables. + + @ivar _base: A transport class L{_BypassTLS} has been mixed in with to which + methods will be forwarded. This class is only responsible for sending + bytes over the connection, not doing TLS. + + @ivar _connection: A L{Connection} which TLS has been started on which will + be proxied to by this object. Any method which has its behavior + altered after C{startTLS} will be skipped in favor of the base class's + implementation. This allows the TLS protocol object to have direct + access to the transport, necessary to actually implement TLS. + """ + def __init__(self, base, connection): + self._base = base + self._connection = connection + + + def __getattr__(self, name): + """ + Forward any extra attribute access to the original transport object. + For example, this exposes C{getHost}, the behavior of which does not + change after TLS is enabled. + """ + return getattr(self._connection, name) + + + def write(self, data): + """ + Write some bytes directly to the connection. + """ + return self._base.write(self._connection, data) + + + def writeSequence(self, iovec): + """ + Write a some bytes directly to the connection. + """ + return self._base.writeSequence(self._connection, iovec) + + + def loseConnection(self, *args, **kwargs): + """ + Close the underlying connection. + """ + return self._base.loseConnection(self._connection, *args, **kwargs) + + + def registerProducer(self, producer, streaming): + """ + Register a producer with the underlying connection. + """ + return self._base.registerProducer(self._connection, producer, streaming) + + + def unregisterProducer(self): + """ + Unregister a producer with the underlying connection. + """ + return self._base.unregisterProducer(self._connection) + + + +def startTLS(transport, contextFactory, normal, bypass): + """ + Add a layer of SSL to a transport. + + @param transport: The transport which will be modified. This can either by + a L{FileDescriptor} or a + L{FileHandle}. The + actual requirements of this instance are that it have: + + - a C{_tlsClientDefault} attribute indicating whether the transport is + a client (C{True}) or a server (C{False}) + - a settable C{TLS} attribute which can be used to mark the fact + that SSL has been started + - settable C{getHandle} and C{getPeerCertificate} attributes so + these L{ISSLTransport} methods can be added to it + - a C{protocol} attribute referring to the L{IProtocol} currently + connected to the transport, which can also be set to a new + L{IProtocol} for the transport to deliver data to + + @param contextFactory: An SSL context factory defining SSL parameters for + the new SSL layer. + @type contextFactory: L{twisted.internet.interfaces.IOpenSSLContextFactory} + + @param normal: A flag indicating whether SSL will go in the same direction + as the underlying transport goes. That is, if the SSL client will be + the underlying client and the SSL server will be the underlying server. + C{True} means it is the same, C{False} means they are switched. + @type param: L{bool} + + @param bypass: A transport base class to call methods on to bypass the new + SSL layer (so that the SSL layer itself can send its bytes). + @type bypass: L{type} + """ + # Figure out which direction the SSL goes in. If normal is True, + # we'll go in the direction indicated by the subclass. Otherwise, + # we'll go the other way (client = not normal ^ _tlsClientDefault, + # in other words). + if normal: + client = transport._tlsClientDefault + else: + client = not transport._tlsClientDefault + + # If we have a producer, unregister it, and then re-register it below once + # we've switched to TLS mode, so it gets hooked up correctly: + producer, streaming = None, None + if transport.producer is not None: + producer, streaming = transport.producer, transport.streamingProducer + transport.unregisterProducer() + + tlsFactory = TLSMemoryBIOFactory(contextFactory, client, None) + tlsProtocol = TLSMemoryBIOProtocol(tlsFactory, transport.protocol, False) + transport.protocol = tlsProtocol + + transport.getHandle = tlsProtocol.getHandle + transport.getPeerCertificate = tlsProtocol.getPeerCertificate + + # Mark the transport as secure. + directlyProvides(transport, ISSLTransport) + + # Remember we did this so that write and writeSequence can send the + # data to the right place. + transport.TLS = True + + # Hook it up + transport.protocol.makeConnection(_BypassTLS(bypass, transport)) + + # Restore producer if necessary: + if producer: + transport.registerProducer(producer, streaming) + + + +@implementer(ITLSTransport) +class ConnectionMixin(object): + """ + A mixin for L{twisted.internet.abstract.FileDescriptor} which adds an + L{ITLSTransport} implementation. + + @ivar TLS: A flag indicating whether TLS is currently in use on this + transport. This is not a good way for applications to check for TLS, + instead use L{twisted.internet.interfaces.ISSLTransport}. + """ + + TLS = False + + def startTLS(self, ctx, normal=True): + """ + @see: L{ITLSTransport.startTLS} + """ + startTLS(self, ctx, normal, FileDescriptor) + + + def write(self, bytes): + """ + Write some bytes to this connection, passing them through a TLS layer if + necessary, or discarding them if the connection has already been lost. + """ + if self.TLS: + if self.connected: + self.protocol.write(bytes) + else: + FileDescriptor.write(self, bytes) + + + def writeSequence(self, iovec): + """ + Write some bytes to this connection, scatter/gather-style, passing them + through a TLS layer if necessary, or discarding them if the connection + has already been lost. + """ + if self.TLS: + if self.connected: + self.protocol.writeSequence(iovec) + else: + FileDescriptor.writeSequence(self, iovec) + + + def loseConnection(self): + """ + Close this connection after writing all pending data. + + If TLS has been negotiated, perform a TLS shutdown. + """ + if self.TLS: + if self.connected and not self.disconnecting: + self.protocol.loseConnection() + else: + FileDescriptor.loseConnection(self) + + + def registerProducer(self, producer, streaming): + """ + Register a producer. + + If TLS is enabled, the TLS connection handles this. + """ + if self.TLS: + # Registering a producer before we're connected shouldn't be a + # problem. If we end up with a write(), that's already handled in + # the write() code above, and there are no other potential + # side-effects. + self.protocol.registerProducer(producer, streaming) + else: + FileDescriptor.registerProducer(self, producer, streaming) + + + def unregisterProducer(self): + """ + Unregister a producer. + + If TLS is enabled, the TLS connection handles this. + """ + if self.TLS: + self.protocol.unregisterProducer() + else: + FileDescriptor.unregisterProducer(self) + + + +class ClientMixin(object): + """ + A mixin for L{twisted.internet.tcp.Client} which just marks it as a client + for the purposes of the default TLS handshake. + + @ivar _tlsClientDefault: Always C{True}, indicating that this is a client + connection, and by default when TLS is negotiated this class will act as + a TLS client. + """ + _tlsClientDefault = True + + + +class ServerMixin(object): + """ + A mixin for L{twisted.internet.tcp.Server} which just marks it as a server + for the purposes of the default TLS handshake. + + @ivar _tlsClientDefault: Always C{False}, indicating that this is a server + connection, and by default when TLS is negotiated this class will act as + a TLS server. + """ + _tlsClientDefault = False diff --git a/contrib/python/Twisted/py2/twisted/internet/_pollingfile.py b/contrib/python/Twisted/py2/twisted/internet/_pollingfile.py new file mode 100644 index 00000000000..6df0a6da28a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_pollingfile.py @@ -0,0 +1,300 @@ +# -*- test-case-name: twisted.internet.test.test_pollingfile -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implements a simple polling interface for file descriptors that don't work with +select() - this is pretty much only useful on Windows. +""" + +from __future__ import absolute_import, division + +from zope.interface import implementer +from twisted.internet.interfaces import IConsumer, IPushProducer +from twisted.python.compat import unicode + + +MIN_TIMEOUT = 0.000000001 +MAX_TIMEOUT = 0.1 + + + +class _PollableResource: + active = True + + def activate(self): + self.active = True + + + def deactivate(self): + self.active = False + + + +class _PollingTimer: + # Everything is private here because it is really an implementation detail. + + def __init__(self, reactor): + self.reactor = reactor + self._resources = [] + self._pollTimer = None + self._currentTimeout = MAX_TIMEOUT + self._paused = False + + def _addPollableResource(self, res): + self._resources.append(res) + self._checkPollingState() + + def _checkPollingState(self): + for resource in self._resources: + if resource.active: + self._startPolling() + break + else: + self._stopPolling() + + def _startPolling(self): + if self._pollTimer is None: + self._pollTimer = self._reschedule() + + def _stopPolling(self): + if self._pollTimer is not None: + self._pollTimer.cancel() + self._pollTimer = None + + def _pause(self): + self._paused = True + + def _unpause(self): + self._paused = False + self._checkPollingState() + + def _reschedule(self): + if not self._paused: + return self.reactor.callLater(self._currentTimeout, self._pollEvent) + + def _pollEvent(self): + workUnits = 0. + anyActive = [] + for resource in self._resources: + if resource.active: + workUnits += resource.checkWork() + # Check AFTER work has been done + if resource.active: + anyActive.append(resource) + + newTimeout = self._currentTimeout + if workUnits: + newTimeout = self._currentTimeout / (workUnits + 1.) + if newTimeout < MIN_TIMEOUT: + newTimeout = MIN_TIMEOUT + else: + newTimeout = self._currentTimeout * 2. + if newTimeout > MAX_TIMEOUT: + newTimeout = MAX_TIMEOUT + self._currentTimeout = newTimeout + if anyActive: + self._pollTimer = self._reschedule() + + +# If we ever (let's hope not) need the above functionality on UNIX, this could +# be factored into a different module. + +import win32pipe +import win32file +import win32api +import pywintypes + +@implementer(IPushProducer) +class _PollableReadPipe(_PollableResource): + + def __init__(self, pipe, receivedCallback, lostCallback): + # security attributes for pipes + self.pipe = pipe + self.receivedCallback = receivedCallback + self.lostCallback = lostCallback + + def checkWork(self): + finished = 0 + fullDataRead = [] + + while 1: + try: + buffer, bytesToRead, result = win32pipe.PeekNamedPipe(self.pipe, 1) + # finished = (result == -1) + if not bytesToRead: + break + hr, data = win32file.ReadFile(self.pipe, bytesToRead, None) + fullDataRead.append(data) + except win32api.error: + finished = 1 + break + + dataBuf = b''.join(fullDataRead) + if dataBuf: + self.receivedCallback(dataBuf) + if finished: + self.cleanup() + return len(dataBuf) + + def cleanup(self): + self.deactivate() + self.lostCallback() + + def close(self): + try: + win32api.CloseHandle(self.pipe) + except pywintypes.error: + # You can't close std handles...? + pass + + def stopProducing(self): + self.close() + + def pauseProducing(self): + self.deactivate() + + def resumeProducing(self): + self.activate() + + +FULL_BUFFER_SIZE = 64 * 1024 + +@implementer(IConsumer) +class _PollableWritePipe(_PollableResource): + + def __init__(self, writePipe, lostCallback): + self.disconnecting = False + self.producer = None + self.producerPaused = False + self.streamingProducer = 0 + self.outQueue = [] + self.writePipe = writePipe + self.lostCallback = lostCallback + try: + win32pipe.SetNamedPipeHandleState(writePipe, + win32pipe.PIPE_NOWAIT, + None, + None) + except pywintypes.error: + # Maybe it's an invalid handle. Who knows. + pass + + def close(self): + self.disconnecting = True + + def bufferFull(self): + if self.producer is not None: + self.producerPaused = True + self.producer.pauseProducing() + + def bufferEmpty(self): + if self.producer is not None and ((not self.streamingProducer) or + self.producerPaused): + self.producer.producerPaused = False + self.producer.resumeProducing() + return True + return False + + # almost-but-not-quite-exact copy-paste from abstract.FileDescriptor... ugh + + def registerProducer(self, producer, streaming): + """Register to receive data from a producer. + + This sets this selectable to be a consumer for a producer. When this + selectable runs out of data on a write() call, it will ask the producer + to resumeProducing(). A producer should implement the IProducer + interface. + + FileDescriptor provides some infrastructure for producer methods. + """ + if self.producer is not None: + raise RuntimeError( + "Cannot register producer %s, because producer %s was never " + "unregistered." % (producer, self.producer)) + if not self.active: + producer.stopProducing() + else: + self.producer = producer + self.streamingProducer = streaming + if not streaming: + producer.resumeProducing() + + def unregisterProducer(self): + """Stop consuming data from a producer, without disconnecting. + """ + self.producer = None + + def writeConnectionLost(self): + self.deactivate() + try: + win32api.CloseHandle(self.writePipe) + except pywintypes.error: + # OMG what + pass + self.lostCallback() + + + def writeSequence(self, seq): + """ + Append a C{list} or C{tuple} of bytes to the output buffer. + + @param seq: C{list} or C{tuple} of C{str} instances to be appended to + the output buffer. + + @raise TypeError: If C{seq} contains C{unicode}. + """ + if unicode in map(type, seq): + raise TypeError("Unicode not allowed in output buffer.") + self.outQueue.extend(seq) + + + def write(self, data): + """ + Append some bytes to the output buffer. + + @param data: C{str} to be appended to the output buffer. + @type data: C{str}. + + @raise TypeError: If C{data} is C{unicode} instead of C{str}. + """ + if isinstance(data, unicode): + raise TypeError("Unicode not allowed in output buffer.") + if self.disconnecting: + return + self.outQueue.append(data) + if sum(map(len, self.outQueue)) > FULL_BUFFER_SIZE: + self.bufferFull() + + + def checkWork(self): + numBytesWritten = 0 + if not self.outQueue: + if self.disconnecting: + self.writeConnectionLost() + return 0 + try: + win32file.WriteFile(self.writePipe, b'', None) + except pywintypes.error: + self.writeConnectionLost() + return numBytesWritten + while self.outQueue: + data = self.outQueue.pop(0) + errCode = 0 + try: + errCode, nBytesWritten = win32file.WriteFile(self.writePipe, + data, None) + except win32api.error: + self.writeConnectionLost() + break + else: + # assert not errCode, "wtf an error code???" + numBytesWritten += nBytesWritten + if len(data) > nBytesWritten: + self.outQueue.insert(0, data[nBytesWritten:]) + break + else: + resumed = self.bufferEmpty() + if not resumed and self.disconnecting: + self.writeConnectionLost() + return numBytesWritten diff --git a/contrib/python/Twisted/py2/twisted/internet/_posixserialport.py b/contrib/python/Twisted/py2/twisted/internet/_posixserialport.py new file mode 100644 index 00000000000..7d4730c3b55 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_posixserialport.py @@ -0,0 +1,73 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Serial Port Protocol +""" + +from __future__ import division, absolute_import + +# dependent on pyserial ( http://pyserial.sf.net/ ) +# only tested w/ 1.18 (5 Dec 2002) +from serial import PARITY_NONE +from serial import STOPBITS_ONE +from serial import EIGHTBITS + +from twisted.internet.serialport import BaseSerialPort + +from twisted.internet import abstract, fdesc + + + +class SerialPort(BaseSerialPort, abstract.FileDescriptor): + """ + A select()able serial device, acting as a transport. + """ + + connected = 1 + + def __init__(self, protocol, deviceNameOrPortNumber, reactor, + baudrate = 9600, bytesize = EIGHTBITS, parity = PARITY_NONE, + stopbits = STOPBITS_ONE, timeout = 0, xonxoff = 0, rtscts = 0): + abstract.FileDescriptor.__init__(self, reactor) + self._serial = self._serialFactory( + deviceNameOrPortNumber, baudrate=baudrate, bytesize=bytesize, + parity=parity, stopbits=stopbits, timeout=timeout, + xonxoff=xonxoff, rtscts=rtscts) + self.reactor = reactor + self.flushInput() + self.flushOutput() + self.protocol = protocol + self.protocol.makeConnection(self) + self.startReading() + + + def fileno(self): + return self._serial.fd + + + def writeSomeData(self, data): + """ + Write some data to the serial device. + """ + return fdesc.writeToFD(self.fileno(), data) + + + def doRead(self): + """ + Some data's readable from serial device. + """ + return fdesc.readFromFD(self.fileno(), self.protocol.dataReceived) + + + def connectionLost(self, reason): + """ + Called when the serial port disconnects. + + Will call C{connectionLost} on the protocol that is handling the + serial data. + """ + abstract.FileDescriptor.connectionLost(self, reason) + self._serial.close() + self.protocol.connectionLost(reason) diff --git a/contrib/python/Twisted/py2/twisted/internet/_posixstdio.py b/contrib/python/Twisted/py2/twisted/internet/_posixstdio.py new file mode 100644 index 00000000000..9d521bc7a7e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_posixstdio.py @@ -0,0 +1,168 @@ +# -*- test-case-name: twisted.test.test_stdio -*- + +"""Standard input/out/err support. + +Future Plans:: + + support for stderr, perhaps + Rewrite to use the reactor instead of an ad-hoc mechanism for connecting + protocols to transport. + +Maintainer: James Y Knight +""" + +from zope.interface import implementer + +from twisted.internet import process, error, interfaces +from twisted.python import log, failure + + +@implementer(interfaces.IAddress) +class PipeAddress(object): + pass + + +@implementer(interfaces.ITransport, interfaces.IProducer, + interfaces.IConsumer, interfaces.IHalfCloseableDescriptor) +class StandardIO(object): + + _reader = None + _writer = None + disconnected = False + disconnecting = False + + def __init__(self, proto, stdin=0, stdout=1, reactor=None): + if reactor is None: + from twisted.internet import reactor + self.protocol = proto + + self._writer = process.ProcessWriter(reactor, self, 'write', stdout) + self._reader = process.ProcessReader(reactor, self, 'read', stdin) + self._reader.startReading() + self.protocol.makeConnection(self) + + # ITransport + + # XXX Actually, see #3597. + def loseWriteConnection(self): + if self._writer is not None: + self._writer.loseConnection() + + def write(self, data): + if self._writer is not None: + self._writer.write(data) + + def writeSequence(self, data): + if self._writer is not None: + self._writer.writeSequence(data) + + def loseConnection(self): + self.disconnecting = True + + if self._writer is not None: + self._writer.loseConnection() + if self._reader is not None: + # Don't loseConnection, because we don't want to SIGPIPE it. + self._reader.stopReading() + + def getPeer(self): + return PipeAddress() + + def getHost(self): + return PipeAddress() + + + # Callbacks from process.ProcessReader/ProcessWriter + def childDataReceived(self, fd, data): + self.protocol.dataReceived(data) + + def childConnectionLost(self, fd, reason): + if self.disconnected: + return + + if reason.value.__class__ == error.ConnectionDone: + # Normal close + if fd == 'read': + self._readConnectionLost(reason) + else: + self._writeConnectionLost(reason) + else: + self.connectionLost(reason) + + def connectionLost(self, reason): + self.disconnected = True + + # Make sure to cleanup the other half + _reader = self._reader + _writer = self._writer + protocol = self.protocol + self._reader = self._writer = None + self.protocol = None + + if _writer is not None and not _writer.disconnected: + _writer.connectionLost(reason) + + if _reader is not None and not _reader.disconnected: + _reader.connectionLost(reason) + + try: + protocol.connectionLost(reason) + except: + log.err() + + def _writeConnectionLost(self, reason): + self._writer=None + if self.disconnecting: + self.connectionLost(reason) + return + + p = interfaces.IHalfCloseableProtocol(self.protocol, None) + if p: + try: + p.writeConnectionLost() + except: + log.err() + self.connectionLost(failure.Failure()) + + def _readConnectionLost(self, reason): + self._reader=None + p = interfaces.IHalfCloseableProtocol(self.protocol, None) + if p: + try: + p.readConnectionLost() + except: + log.err() + self.connectionLost(failure.Failure()) + else: + self.connectionLost(reason) + + # IConsumer + def registerProducer(self, producer, streaming): + if self._writer is None: + producer.stopProducing() + else: + self._writer.registerProducer(producer, streaming) + + def unregisterProducer(self): + if self._writer is not None: + self._writer.unregisterProducer() + + # IProducer + def stopProducing(self): + self.loseConnection() + + def pauseProducing(self): + if self._reader is not None: + self._reader.pauseProducing() + + def resumeProducing(self): + if self._reader is not None: + self._reader.resumeProducing() + + def stopReading(self): + """Compatibility only, don't use. Call pauseProducing.""" + self.pauseProducing() + + def startReading(self): + """Compatibility only, don't use. Call resumeProducing.""" + self.resumeProducing() diff --git a/contrib/python/Twisted/py2/twisted/internet/_producer_helpers.py b/contrib/python/Twisted/py2/twisted/internet/_producer_helpers.py new file mode 100644 index 00000000000..9b51ffaf302 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_producer_helpers.py @@ -0,0 +1,125 @@ +# -*- test-case-name: twisted.test.test_producer_helpers -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Helpers for working with producers. +""" +from __future__ import division, absolute_import + +from zope.interface import implementer + +from twisted.internet.interfaces import IPushProducer +from twisted.internet.task import cooperate +from twisted.python import log +from twisted.python.reflect import safe_str + + +# This module exports nothing public, it's for internal Twisted use only. +__all__ = [] + + +@implementer(IPushProducer) +class _PullToPush(object): + """ + An adapter that converts a non-streaming to a streaming producer. + + Because of limitations of the producer API, this adapter requires the + cooperation of the consumer. When the consumer's C{registerProducer} is + called with a non-streaming producer, it must wrap it with L{_PullToPush} + and then call C{startStreaming} on the resulting object. When the + consumer's C{unregisterProducer} is called, it must call + C{stopStreaming} on the L{_PullToPush} instance. + + If the underlying producer throws an exception from C{resumeProducing}, + the producer will be unregistered from the consumer. + + @ivar _producer: the underling non-streaming producer. + + @ivar _consumer: the consumer with which the underlying producer was + registered. + + @ivar _finished: C{bool} indicating whether the producer has finished. + + @ivar _coopTask: the result of calling L{cooperate}, the task driving the + streaming producer. + """ + + _finished = False + + + def __init__(self, pullProducer, consumer): + self._producer = pullProducer + self._consumer = consumer + + + def _pull(self): + """ + A generator that calls C{resumeProducing} on the underlying producer + forever. + + If C{resumeProducing} throws an exception, the producer is + unregistered, which should result in streaming stopping. + """ + while True: + try: + self._producer.resumeProducing() + except: + log.err(None, "%s failed, producing will be stopped:" % + (safe_str(self._producer),)) + try: + self._consumer.unregisterProducer() + # The consumer should now call stopStreaming() on us, + # thus stopping the streaming. + except: + # Since the consumer blew up, we may not have had + # stopStreaming() called, so we just stop on our own: + log.err(None, "%s failed to unregister producer:" % + (safe_str(self._consumer),)) + self._finished = True + return + yield None + + + def startStreaming(self): + """ + This should be called by the consumer when the producer is registered. + + Start streaming data to the consumer. + """ + self._coopTask = cooperate(self._pull()) + + + def stopStreaming(self): + """ + This should be called by the consumer when the producer is + unregistered. + + Stop streaming data to the consumer. + """ + if self._finished: + return + self._finished = True + self._coopTask.stop() + + + def pauseProducing(self): + """ + @see: C{IPushProducer.pauseProducing} + """ + self._coopTask.pause() + + + def resumeProducing(self): + """ + @see: C{IPushProducer.resumeProducing} + """ + self._coopTask.resume() + + + def stopProducing(self): + """ + @see: C{IPushProducer.stopProducing} + """ + self.stopStreaming() + self._producer.stopProducing() diff --git a/contrib/python/Twisted/py2/twisted/internet/_resolver.py b/contrib/python/Twisted/py2/twisted/internet/_resolver.py new file mode 100644 index 00000000000..1c16174a281 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_resolver.py @@ -0,0 +1,279 @@ +# -*- test-case-name: twisted.internet.test.test_resolver -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +IPv6-aware hostname resolution. + +@see: L{IHostnameResolver} +""" + +from __future__ import division, absolute_import + +__metaclass__ = type + +from socket import (getaddrinfo, AF_INET, AF_INET6, AF_UNSPEC, SOCK_STREAM, + SOCK_DGRAM, gaierror) + +from zope.interface import implementer + +from twisted.internet.interfaces import (IHostnameResolver, IHostResolution, + IResolverSimple, IResolutionReceiver) +from twisted.internet.error import DNSLookupError +from twisted.internet.defer import Deferred +from twisted.internet.threads import deferToThreadPool +from twisted.internet.address import IPv4Address, IPv6Address +from twisted.python.compat import nativeString +from twisted.internet._idna import _idnaBytes +from twisted.logger import Logger + + +@implementer(IHostResolution) +class HostResolution(object): + """ + The in-progress resolution of a given hostname. + """ + + def __init__(self, name): + """ + Create a L{HostResolution} with the given name. + """ + self.name = name + + + +_any = frozenset([IPv4Address, IPv6Address]) + +_typesToAF = { + frozenset([IPv4Address]): AF_INET, + frozenset([IPv6Address]): AF_INET6, + _any: AF_UNSPEC, +} + +_afToType = { + AF_INET: IPv4Address, + AF_INET6: IPv6Address, +} + +_transportToSocket = { + 'TCP': SOCK_STREAM, + 'UDP': SOCK_DGRAM, +} + +_socktypeToType = { + SOCK_STREAM: 'TCP', + SOCK_DGRAM: 'UDP', +} + + + +@implementer(IHostnameResolver) +class GAIResolver(object): + """ + L{IHostnameResolver} implementation that resolves hostnames by calling + L{getaddrinfo} in a thread. + """ + + def __init__(self, reactor, getThreadPool=None, getaddrinfo=getaddrinfo): + """ + Create a L{GAIResolver}. + + @param reactor: the reactor to schedule result-delivery on + @type reactor: L{IReactorThreads} + + @param getThreadPool: a function to retrieve the thread pool to use for + scheduling name resolutions. If not supplied, the use the given + C{reactor}'s thread pool. + @type getThreadPool: 0-argument callable returning a + L{twisted.python.threadpool.ThreadPool} + + @param getaddrinfo: a reference to the L{getaddrinfo} to use - mainly + parameterized for testing. + @type getaddrinfo: callable with the same signature as L{getaddrinfo} + """ + self._reactor = reactor + self._getThreadPool = (reactor.getThreadPool if getThreadPool is None + else getThreadPool) + self._getaddrinfo = getaddrinfo + + + def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, + addressTypes=None, transportSemantics='TCP'): + """ + See L{IHostnameResolver.resolveHostName} + + @param resolutionReceiver: see interface + + @param hostName: see interface + + @param portNumber: see interface + + @param addressTypes: see interface + + @param transportSemantics: see interface + + @return: see interface + """ + pool = self._getThreadPool() + addressFamily = _typesToAF[_any if addressTypes is None + else frozenset(addressTypes)] + socketType = _transportToSocket[transportSemantics] + def get(): + try: + return self._getaddrinfo(hostName, portNumber, addressFamily, + socketType) + except gaierror: + return [] + d = deferToThreadPool(self._reactor, pool, get) + resolution = HostResolution(hostName) + resolutionReceiver.resolutionBegan(resolution) + @d.addCallback + def deliverResults(result): + for family, socktype, proto, cannoname, sockaddr in result: + addrType = _afToType[family] + resolutionReceiver.addressResolved( + addrType(_socktypeToType.get(socktype, 'TCP'), *sockaddr) + ) + resolutionReceiver.resolutionComplete() + return resolution + + + +@implementer(IHostnameResolver) +class SimpleResolverComplexifier(object): + """ + A converter from L{IResolverSimple} to L{IHostnameResolver}. + """ + + _log = Logger() + + def __init__(self, simpleResolver): + """ + Construct a L{SimpleResolverComplexifier} with an L{IResolverSimple}. + """ + self._simpleResolver = simpleResolver + + + def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, + addressTypes=None, transportSemantics='TCP'): + """ + See L{IHostnameResolver.resolveHostName} + + @param resolutionReceiver: see interface + + @param hostName: see interface + + @param portNumber: see interface + + @param addressTypes: see interface + + @param transportSemantics: see interface + + @return: see interface + """ + # If it's str, we need to make sure that it's just ASCII. + try: + hostName = hostName.encode('ascii') + except UnicodeEncodeError: + # If it's not just ASCII, IDNA it. We don't want to give a Unicode + # string with non-ASCII in it to Python 3, as if anyone passes that + # to a Python 3 stdlib function, it will probably use the wrong + # IDNA version and break absolutely everything + hostName = _idnaBytes(hostName) + + # Make sure it's passed down as a native str, to maintain the interface + hostName = nativeString(hostName) + + resolution = HostResolution(hostName) + resolutionReceiver.resolutionBegan(resolution) + onAddress = self._simpleResolver.getHostByName(hostName) + def addressReceived(address): + resolutionReceiver.addressResolved(IPv4Address('TCP', address, + portNumber)) + def errorReceived(error): + if not error.check(DNSLookupError): + self._log.failure("while looking up {name} with {resolver}", + error, name=hostName, + resolver=self._simpleResolver) + onAddress.addCallbacks(addressReceived, errorReceived) + def finish(result): + resolutionReceiver.resolutionComplete() + onAddress.addCallback(finish) + return resolution + + + +@implementer(IResolutionReceiver) +class FirstOneWins(object): + """ + An L{IResolutionReceiver} which fires a L{Deferred} with its first result. + """ + + def __init__(self, deferred): + """ + @param deferred: The L{Deferred} to fire when the first resolution + result arrives. + """ + self._deferred = deferred + self._resolved = False + + + def resolutionBegan(self, resolution): + """ + See L{IResolutionReceiver.resolutionBegan} + + @param resolution: See L{IResolutionReceiver.resolutionBegan} + """ + self._resolution = resolution + + + def addressResolved(self, address): + """ + See L{IResolutionReceiver.addressResolved} + + @param address: See L{IResolutionReceiver.addressResolved} + """ + if self._resolved: + return + self._resolved = True + self._deferred.callback(address.host) + + + def resolutionComplete(self): + """ + See L{IResolutionReceiver.resolutionComplete} + """ + if self._resolved: + return + self._deferred.errback(DNSLookupError(self._resolution.name)) + + + +@implementer(IResolverSimple) +class ComplexResolverSimplifier(object): + """ + A converter from L{IHostnameResolver} to L{IResolverSimple} + """ + def __init__(self, nameResolver): + """ + Create a L{ComplexResolverSimplifier} with an L{IHostnameResolver}. + + @param nameResolver: The L{IHostnameResolver} to use. + """ + self._nameResolver = nameResolver + + + def getHostByName(self, name, timeouts=()): + """ + See L{IResolverSimple.getHostByName} + + @param name: see L{IResolverSimple.getHostByName} + + @param timeouts: see L{IResolverSimple.getHostByName} + + @return: see L{IResolverSimple.getHostByName} + """ + result = Deferred() + self._nameResolver.resolveHostName(FirstOneWins(result), name, 0, + [IPv4Address]) + return result diff --git a/contrib/python/Twisted/py2/twisted/internet/_signals.py b/contrib/python/Twisted/py2/twisted/internet/_signals.py new file mode 100644 index 00000000000..4335727fe89 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_signals.py @@ -0,0 +1,68 @@ +# -*- test-case-name: twisted.internet.test.test_sigchld -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module is used to integrate child process termination into a +reactor event loop. This is a challenging feature to provide because +most platforms indicate process termination via SIGCHLD and do not +provide a way to wait for that signal and arbitrary I/O events at the +same time. The naive implementation involves installing a Python +SIGCHLD handler; unfortunately this leads to other syscalls being +interrupted (whenever SIGCHLD is received) and failing with EINTR +(which almost no one is prepared to handle). This interruption can be +disabled via siginterrupt(2) (or one of the equivalent mechanisms); +however, if the SIGCHLD is delivered by the platform to a non-main +thread (not a common occurrence, but difficult to prove impossible), +the main thread (waiting on select() or another event notification +API) may not wake up leading to an arbitrary delay before the child +termination is noticed. + +The basic solution to all these issues involves enabling SA_RESTART +(ie, disabling system call interruption) and registering a C signal +handler which writes a byte to a pipe. The other end of the pipe is +registered with the event loop, allowing it to wake up shortly after +SIGCHLD is received. See L{twisted.internet.posixbase._SIGCHLDWaker} +for the implementation of the event loop side of this solution. The +use of a pipe this way is known as the U{self-pipe +trick}. + +From Python version 2.6, C{signal.siginterrupt} and C{signal.set_wakeup_fd} +provide the necessary C signal handler which writes to the pipe to be +registered with C{SA_RESTART}. +""" + +from __future__ import division, absolute_import + +import signal + + +def installHandler(fd): + """ + Install a signal handler which will write a byte to C{fd} when + I{SIGCHLD} is received. + + This is implemented by installing a SIGCHLD handler that does nothing, + setting the I{SIGCHLD} handler as not allowed to interrupt system calls, + and using L{signal.set_wakeup_fd} to do the actual writing. + + @param fd: The file descriptor to which to write when I{SIGCHLD} is + received. + @type fd: C{int} + """ + if fd == -1: + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + else: + def noopSignalHandler(*args): + pass + signal.signal(signal.SIGCHLD, noopSignalHandler) + signal.siginterrupt(signal.SIGCHLD, False) + return signal.set_wakeup_fd(fd) + + + +def isDefaultHandler(): + """ + Determine whether the I{SIGCHLD} handler is the default or not. + """ + return signal.getsignal(signal.SIGCHLD) == signal.SIG_DFL diff --git a/contrib/python/Twisted/py2/twisted/internet/_sslverify.py b/contrib/python/Twisted/py2/twisted/internet/_sslverify.py new file mode 100644 index 00000000000..c44ec143246 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_sslverify.py @@ -0,0 +1,2058 @@ +# -*- test-case-name: twisted.test.test_sslverify -*- +# Copyright (c) 2005 Divmod, Inc. +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import division, absolute_import + +import warnings + +from constantly import Names, NamedConstant +from hashlib import md5 + +from OpenSSL import SSL, crypto +from OpenSSL._util import lib as pyOpenSSLlib + +from twisted.internet.abstract import isIPAddress, isIPv6Address +from twisted.python import log +from twisted.python.randbytes import secureRandom +from twisted.python._oldstyle import _oldStyle +from ._idna import _idnaBytes + +from zope.interface import Interface, implementer +from constantly import Flags, FlagConstant +from incremental import Version + +from twisted.internet.defer import Deferred +from twisted.internet.error import VerifyError, CertificateError +from twisted.internet.interfaces import ( + IAcceptableCiphers, ICipher, IOpenSSLClientConnectionCreator, + IOpenSSLContextFactory +) + +from twisted.python import util +from twisted.python.deprecate import _mutuallyExclusiveArguments +from twisted.python.compat import nativeString, unicode +from twisted.python.failure import Failure +from twisted.python.util import FancyEqMixin + +from twisted.python.deprecate import deprecated + + + +class TLSVersion(Names): + """ + TLS versions that we can negotiate with the client/server. + """ + SSLv3 = NamedConstant() + TLSv1_0 = NamedConstant() + TLSv1_1 = NamedConstant() + TLSv1_2 = NamedConstant() + TLSv1_3 = NamedConstant() + + + +_tlsDisableFlags = { + TLSVersion.SSLv3: SSL.OP_NO_SSLv3, + TLSVersion.TLSv1_0: SSL.OP_NO_TLSv1, + TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1, + TLSVersion.TLSv1_2: SSL.OP_NO_TLSv1_2, + + # If we don't have TLS v1.3 yet, we can't disable it -- this is just so + # when it makes it into OpenSSL, connections knowingly bracketed to v1.2 + # don't end up going to v1.3 + TLSVersion.TLSv1_3: getattr(SSL, "OP_NO_TLSv1_3", 0x00), +} + + + +def _getExcludedTLSProtocols(oldest, newest): + """ + Given a pair of L{TLSVersion} constants, figure out what versions we want + to disable (as OpenSSL is an exclusion based API). + + @param oldest: The oldest L{TLSVersion} we want to allow. + @type oldest: L{TLSVersion} constant + + @param newest: The newest L{TLSVersion} we want to allow, or L{None} for no + upper limit. + @type newest: L{TLSVersion} constant or L{None} + + @return: The versions we want to disable. + @rtype: L{list} of L{TLSVersion} constants. + """ + versions = list(TLSVersion.iterconstants()) + excludedVersions = [x for x in versions[:versions.index(oldest)]] + + if newest: + excludedVersions.extend([x for x in versions[versions.index(newest):]]) + + return excludedVersions + + + + +class SimpleVerificationError(Exception): + """ + Not a very useful verification error. + """ + + + +def simpleVerifyHostname(connection, hostname): + """ + Check only the common name in the certificate presented by the peer and + only for an exact match. + + This is to provide I{something} in the way of hostname verification to + users who haven't installed C{service_identity}. This check is overly + strict, relies on a deprecated TLS feature (you're supposed to ignore the + commonName if the subjectAlternativeName extensions are present, I + believe), and lots of valid certificates will fail. + + @param connection: the OpenSSL connection to verify. + @type connection: L{OpenSSL.SSL.Connection} + + @param hostname: The hostname expected by the user. + @type hostname: L{unicode} + + @raise twisted.internet.ssl.VerificationError: if the common name and + hostname don't match. + """ + commonName = connection.get_peer_certificate().get_subject().commonName + if commonName != hostname: + raise SimpleVerificationError(repr(commonName) + "!=" + + repr(hostname)) + + + +def simpleVerifyIPAddress(connection, hostname): + """ + Always fails validation of IP addresses + + @param connection: the OpenSSL connection to verify. + @type connection: L{OpenSSL.SSL.Connection} + + @param hostname: The hostname expected by the user. + @type hostname: L{unicode} + + @raise twisted.internet.ssl.VerificationError: Always raised + """ + raise SimpleVerificationError("Cannot verify certificate IP addresses") + + + +def _usablePyOpenSSL(version): + """ + Check pyOpenSSL version string whether we can use it for host verification. + + @param version: A pyOpenSSL version string. + @type version: L{str} + + @rtype: L{bool} + """ + major, minor = (int(part) for part in version.split(".")[:2]) + return (major, minor) >= (0, 12) + + + +def _selectVerifyImplementation(): + """ + Determine if C{service_identity} is installed. If so, use it. If not, use + simplistic and incorrect checking as implemented in + L{simpleVerifyHostname}. + + @return: 2-tuple of (C{verify_hostname}, C{VerificationError}) + @rtype: L{tuple} + """ + + whatsWrong = ( + "Without the service_identity module, Twisted can perform only " + "rudimentary TLS client hostname verification. Many valid " + "certificate/hostname mappings may be rejected." + ) + + try: + from service_identity import VerificationError + from service_identity.pyopenssl import ( + verify_hostname, verify_ip_address + ) + return verify_hostname, verify_ip_address, VerificationError + except ImportError as e: + warnings.warn_explicit( + "You do not have a working installation of the " + "service_identity module: '" + str(e) + "'. " + "Please install it from " + " and make " + "sure all of its dependencies are satisfied. " + + whatsWrong, + # Unfortunately the lineno is required. + category=UserWarning, filename="", lineno=0) + + return simpleVerifyHostname, simpleVerifyIPAddress, SimpleVerificationError + + + +verifyHostname, verifyIPAddress, VerificationError = \ + _selectVerifyImplementation() + + + +class ProtocolNegotiationSupport(Flags): + """ + L{ProtocolNegotiationSupport} defines flags which are used to indicate the + level of NPN/ALPN support provided by the TLS backend. + + @cvar NOSUPPORT: There is no support for NPN or ALPN. This is exclusive + with both L{NPN} and L{ALPN}. + @cvar NPN: The implementation supports Next Protocol Negotiation. + @cvar ALPN: The implementation supports Application Layer Protocol + Negotiation. + """ + NPN = FlagConstant(0x0001) + ALPN = FlagConstant(0x0002) + +# FIXME: https://twistedmatrix.com/trac/ticket/8074 +# Currently flags with literal zero values behave incorrectly. However, +# creating a flag by NOTing a flag with itself appears to work totally fine, so +# do that instead. +ProtocolNegotiationSupport.NOSUPPORT = ( + ProtocolNegotiationSupport.NPN ^ ProtocolNegotiationSupport.NPN +) + + +def protocolNegotiationMechanisms(): + """ + Checks whether your versions of PyOpenSSL and OpenSSL are recent enough to + support protocol negotiation, and if they are, what kind of protocol + negotiation is supported. + + @return: A combination of flags from L{ProtocolNegotiationSupport} that + indicate which mechanisms for protocol negotiation are supported. + @rtype: L{constantly.FlagConstant} + """ + support = ProtocolNegotiationSupport.NOSUPPORT + ctx = SSL.Context(SSL.SSLv23_METHOD) + + try: + ctx.set_npn_advertise_callback(lambda c: None) + except (AttributeError, NotImplementedError): + pass + else: + support |= ProtocolNegotiationSupport.NPN + + try: + ctx.set_alpn_select_callback(lambda c: None) + except (AttributeError, NotImplementedError): + pass + else: + support |= ProtocolNegotiationSupport.ALPN + + return support + + + +_x509names = { + 'CN': 'commonName', + 'commonName': 'commonName', + + 'O': 'organizationName', + 'organizationName': 'organizationName', + + 'OU': 'organizationalUnitName', + 'organizationalUnitName': 'organizationalUnitName', + + 'L': 'localityName', + 'localityName': 'localityName', + + 'ST': 'stateOrProvinceName', + 'stateOrProvinceName': 'stateOrProvinceName', + + 'C': 'countryName', + 'countryName': 'countryName', + + 'emailAddress': 'emailAddress'} + + + +class DistinguishedName(dict): + """ + Identify and describe an entity. + + Distinguished names are used to provide a minimal amount of identifying + information about a certificate issuer or subject. They are commonly + created with one or more of the following fields:: + + commonName (CN) + organizationName (O) + organizationalUnitName (OU) + localityName (L) + stateOrProvinceName (ST) + countryName (C) + emailAddress + + A L{DistinguishedName} should be constructed using keyword arguments whose + keys can be any of the field names above (as a native string), and the + values are either Unicode text which is encodable to ASCII, or L{bytes} + limited to the ASCII subset. Any fields passed to the constructor will be + set as attributes, accessible using both their extended name and their + shortened acronym. The attribute values will be the ASCII-encoded + bytes. For example:: + + >>> dn = DistinguishedName(commonName=b'www.example.com', + ... C='US') + >>> dn.C + b'US' + >>> dn.countryName + b'US' + >>> hasattr(dn, "organizationName") + False + + L{DistinguishedName} instances can also be used as dictionaries; the keys + are extended name of the fields:: + + >>> dn.keys() + ['countryName', 'commonName'] + >>> dn['countryName'] + b'US' + + """ + __slots__ = () + + def __init__(self, **kw): + for k, v in kw.items(): + setattr(self, k, v) + + + def _copyFrom(self, x509name): + for name in _x509names: + value = getattr(x509name, name, None) + if value is not None: + setattr(self, name, value) + + + def _copyInto(self, x509name): + for k, v in self.items(): + setattr(x509name, k, nativeString(v)) + + + def __repr__(self): + return '' % (dict.__repr__(self)[1:-1]) + + + def __getattr__(self, attr): + try: + return self[_x509names[attr]] + except KeyError: + raise AttributeError(attr) + + + def __setattr__(self, attr, value): + if attr not in _x509names: + raise AttributeError("%s is not a valid OpenSSL X509 name field" % (attr,)) + realAttr = _x509names[attr] + if not isinstance(value, bytes): + value = value.encode("ascii") + self[realAttr] = value + + + def inspect(self): + """ + Return a multi-line, human-readable representation of this DN. + + @rtype: L{str} + """ + l = [] + lablen = 0 + def uniqueValues(mapping): + return set(mapping.values()) + for k in sorted(uniqueValues(_x509names)): + label = util.nameToLabel(k) + lablen = max(len(label), lablen) + v = getattr(self, k, None) + if v is not None: + l.append((label, nativeString(v))) + lablen += 2 + for n, (label, attr) in enumerate(l): + l[n] = (label.rjust(lablen)+': '+ attr) + return '\n'.join(l) + +DN = DistinguishedName + + + +@_oldStyle +class CertBase: + """ + Base class for public (certificate only) and private (certificate + key + pair) certificates. + + @ivar original: The underlying OpenSSL certificate object. + @type original: L{OpenSSL.crypto.X509} + """ + + def __init__(self, original): + self.original = original + + + def _copyName(self, suffix): + dn = DistinguishedName() + dn._copyFrom(getattr(self.original, 'get_'+suffix)()) + return dn + + + def getSubject(self): + """ + Retrieve the subject of this certificate. + + @return: A copy of the subject of this certificate. + @rtype: L{DistinguishedName} + """ + return self._copyName('subject') + + + def __conform__(self, interface): + """ + Convert this L{CertBase} into a provider of the given interface. + + @param interface: The interface to conform to. + @type interface: L{zope.interface.interfaces.IInterface} + + @return: an L{IOpenSSLTrustRoot} provider or L{NotImplemented} + @rtype: L{IOpenSSLTrustRoot} or L{NotImplemented} + """ + if interface is IOpenSSLTrustRoot: + return OpenSSLCertificateAuthorities([self.original]) + return NotImplemented + + + +def _handleattrhelper(Class, transport, methodName): + """ + (private) Helper for L{Certificate.peerFromTransport} and + L{Certificate.hostFromTransport} which checks for incompatible handle types + and null certificates and raises the appropriate exception or returns the + appropriate certificate object. + """ + method = getattr(transport.getHandle(), + "get_%s_certificate" % (methodName,), None) + if method is None: + raise CertificateError( + "non-TLS transport %r did not have %s certificate" % (transport, methodName)) + cert = method() + if cert is None: + raise CertificateError( + "TLS transport %r did not have %s certificate" % (transport, methodName)) + return Class(cert) + + + +class Certificate(CertBase): + """ + An x509 certificate. + """ + def __repr__(self): + return '<%s Subject=%s Issuer=%s>' % (self.__class__.__name__, + self.getSubject().commonName, + self.getIssuer().commonName) + + + def __eq__(self, other): + if isinstance(other, Certificate): + return self.dump() == other.dump() + return False + + + def __ne__(self, other): + return not self.__eq__(other) + + + @classmethod + def load(Class, requestData, format=crypto.FILETYPE_ASN1, args=()): + """ + Load a certificate from an ASN.1- or PEM-format string. + + @rtype: C{Class} + """ + return Class(crypto.load_certificate(format, requestData), *args) + + # We can't use super() because it is old style still, so we have to hack + # around things wanting to call the parent function + _load = load + + + def dumpPEM(self): + """ + Dump this certificate to a PEM-format data string. + + @rtype: L{str} + """ + return self.dump(crypto.FILETYPE_PEM) + + + @classmethod + def loadPEM(Class, data): + """ + Load a certificate from a PEM-format data string. + + @rtype: C{Class} + """ + return Class.load(data, crypto.FILETYPE_PEM) + + + @classmethod + def peerFromTransport(Class, transport): + """ + Get the certificate for the remote end of the given transport. + + @param transport: an L{ISystemHandle} provider + + @rtype: C{Class} + + @raise: L{CertificateError}, if the given transport does not have a peer + certificate. + """ + return _handleattrhelper(Class, transport, 'peer') + + + @classmethod + def hostFromTransport(Class, transport): + """ + Get the certificate for the local end of the given transport. + + @param transport: an L{ISystemHandle} provider; the transport we will + + @rtype: C{Class} + + @raise: L{CertificateError}, if the given transport does not have a host + certificate. + """ + return _handleattrhelper(Class, transport, 'host') + + + def getPublicKey(self): + """ + Get the public key for this certificate. + + @rtype: L{PublicKey} + """ + return PublicKey(self.original.get_pubkey()) + + + def dump(self, format=crypto.FILETYPE_ASN1): + return crypto.dump_certificate(format, self.original) + + + def serialNumber(self): + """ + Retrieve the serial number of this certificate. + + @rtype: L{int} + """ + return self.original.get_serial_number() + + + def digest(self, method='md5'): + """ + Return a digest hash of this certificate using the specified hash + algorithm. + + @param method: One of C{'md5'} or C{'sha'}. + + @return: The digest of the object, formatted as b":"-delimited hex + pairs + @rtype: L{bytes} + """ + return self.original.digest(method) + + + def _inspect(self): + return '\n'.join(['Certificate For Subject:', + self.getSubject().inspect(), + '\nIssuer:', + self.getIssuer().inspect(), + '\nSerial Number: %d' % self.serialNumber(), + 'Digest: %s' % nativeString(self.digest())]) + + + def inspect(self): + """ + Return a multi-line, human-readable representation of this + Certificate, including information about the subject, issuer, and + public key. + """ + return '\n'.join((self._inspect(), self.getPublicKey().inspect())) + + + def getIssuer(self): + """ + Retrieve the issuer of this certificate. + + @rtype: L{DistinguishedName} + @return: A copy of the issuer of this certificate. + """ + return self._copyName('issuer') + + + def options(self, *authorities): + raise NotImplementedError('Possible, but doubtful we need this yet') + + + +class CertificateRequest(CertBase): + """ + An x509 certificate request. + + Certificate requests are given to certificate authorities to be signed and + returned resulting in an actual certificate. + """ + @classmethod + def load(Class, requestData, requestFormat=crypto.FILETYPE_ASN1): + req = crypto.load_certificate_request(requestFormat, requestData) + dn = DistinguishedName() + dn._copyFrom(req.get_subject()) + if not req.verify(req.get_pubkey()): + raise VerifyError("Can't verify that request for %r is self-signed." % (dn,)) + return Class(req) + + + def dump(self, format=crypto.FILETYPE_ASN1): + return crypto.dump_certificate_request(format, self.original) + + + +class PrivateCertificate(Certificate): + """ + An x509 certificate and private key. + """ + def __repr__(self): + return Certificate.__repr__(self) + ' with ' + repr(self.privateKey) + + + def _setPrivateKey(self, privateKey): + if not privateKey.matches(self.getPublicKey()): + raise VerifyError( + "Certificate public and private keys do not match.") + self.privateKey = privateKey + return self + + + def newCertificate(self, newCertData, format=crypto.FILETYPE_ASN1): + """ + Create a new L{PrivateCertificate} from the given certificate data and + this instance's private key. + """ + return self.load(newCertData, self.privateKey, format) + + + @classmethod + def load(Class, data, privateKey, format=crypto.FILETYPE_ASN1): + return Class._load(data, format)._setPrivateKey(privateKey) + + + def inspect(self): + return '\n'.join([Certificate._inspect(self), + self.privateKey.inspect()]) + + + def dumpPEM(self): + """ + Dump both public and private parts of a private certificate to + PEM-format data. + """ + return self.dump(crypto.FILETYPE_PEM) + self.privateKey.dump(crypto.FILETYPE_PEM) + + + @classmethod + def loadPEM(Class, data): + """ + Load both private and public parts of a private certificate from a + chunk of PEM-format data. + """ + return Class.load(data, KeyPair.load(data, crypto.FILETYPE_PEM), + crypto.FILETYPE_PEM) + + + @classmethod + def fromCertificateAndKeyPair(Class, certificateInstance, privateKey): + privcert = Class(certificateInstance.original) + return privcert._setPrivateKey(privateKey) + + + def options(self, *authorities): + """ + Create a context factory using this L{PrivateCertificate}'s certificate + and private key. + + @param authorities: A list of L{Certificate} object + + @return: A context factory. + @rtype: L{CertificateOptions } + """ + options = dict(privateKey=self.privateKey.original, + certificate=self.original) + if authorities: + options.update(dict(trustRoot=OpenSSLCertificateAuthorities( + [auth.original for auth in authorities] + ))) + return OpenSSLCertificateOptions(**options) + + + def certificateRequest(self, format=crypto.FILETYPE_ASN1, + digestAlgorithm='sha256'): + return self.privateKey.certificateRequest( + self.getSubject(), + format, + digestAlgorithm) + + + def signCertificateRequest(self, + requestData, + verifyDNCallback, + serialNumber, + requestFormat=crypto.FILETYPE_ASN1, + certificateFormat=crypto.FILETYPE_ASN1): + issuer = self.getSubject() + return self.privateKey.signCertificateRequest( + issuer, + requestData, + verifyDNCallback, + serialNumber, + requestFormat, + certificateFormat) + + + def signRequestObject(self, certificateRequest, serialNumber, + secondsToExpiry=60 * 60 * 24 * 365, # One year + digestAlgorithm='sha256'): + return self.privateKey.signRequestObject(self.getSubject(), + certificateRequest, + serialNumber, + secondsToExpiry, + digestAlgorithm) + + + +@_oldStyle +class PublicKey: + """ + A L{PublicKey} is a representation of the public part of a key pair. + + You can't do a whole lot with it aside from comparing it to other + L{PublicKey} objects. + + @note: If constructing a L{PublicKey} manually, be sure to pass only a + L{OpenSSL.crypto.PKey} that does not contain a private key! + + @ivar original: The original private key. + """ + + def __init__(self, osslpkey): + """ + @param osslpkey: The underlying pyOpenSSL key object. + @type osslpkey: L{OpenSSL.crypto.PKey} + """ + self.original = osslpkey + + + def matches(self, otherKey): + """ + Does this L{PublicKey} contain the same value as another L{PublicKey}? + + @param otherKey: The key to compare C{self} to. + @type otherKey: L{PublicKey} + + @return: L{True} if these keys match, L{False} if not. + @rtype: L{bool} + """ + return self.keyHash() == otherKey.keyHash() + + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.keyHash()) + + + def keyHash(self): + """ + Compute a hash of the underlying PKey object. + + The purpose of this method is to allow you to determine if two + certificates share the same public key; it is not really useful for + anything else. + + In versions of Twisted prior to 15.0, C{keyHash} used a technique + involving certificate requests for computing the hash that was not + stable in the face of changes to the underlying OpenSSL library. + + @return: Return a 32-character hexadecimal string uniquely identifying + this public key, I{for this version of Twisted}. + @rtype: native L{str} + """ + raw = crypto.dump_publickey(crypto.FILETYPE_ASN1, self.original) + h = md5() + h.update(raw) + return h.hexdigest() + + + def inspect(self): + return 'Public Key with Hash: %s' % (self.keyHash(),) + + + +class KeyPair(PublicKey): + + @classmethod + def load(Class, data, format=crypto.FILETYPE_ASN1): + return Class(crypto.load_privatekey(format, data)) + + + def dump(self, format=crypto.FILETYPE_ASN1): + return crypto.dump_privatekey(format, self.original) + + + def __getstate__(self): + return self.dump() + + + def __setstate__(self, state): + self.__init__(crypto.load_privatekey(crypto.FILETYPE_ASN1, state)) + + + def inspect(self): + t = self.original.type() + if t == crypto.TYPE_RSA: + ts = 'RSA' + elif t == crypto.TYPE_DSA: + ts = 'DSA' + else: + ts = '(Unknown Type!)' + L = (self.original.bits(), ts, self.keyHash()) + return '%s-bit %s Key Pair with Hash: %s' % L + + + @classmethod + def generate(Class, kind=crypto.TYPE_RSA, size=2048): + pkey = crypto.PKey() + pkey.generate_key(kind, size) + return Class(pkey) + + + def newCertificate(self, newCertData, format=crypto.FILETYPE_ASN1): + return PrivateCertificate.load(newCertData, self, format) + + + def requestObject(self, distinguishedName, digestAlgorithm='sha256'): + req = crypto.X509Req() + req.set_pubkey(self.original) + distinguishedName._copyInto(req.get_subject()) + req.sign(self.original, digestAlgorithm) + return CertificateRequest(req) + + + def certificateRequest(self, distinguishedName, + format=crypto.FILETYPE_ASN1, + digestAlgorithm='sha256'): + """ + Create a certificate request signed with this key. + + @return: a string, formatted according to the 'format' argument. + """ + return self.requestObject(distinguishedName, digestAlgorithm).dump(format) + + + def signCertificateRequest(self, + issuerDistinguishedName, + requestData, + verifyDNCallback, + serialNumber, + requestFormat=crypto.FILETYPE_ASN1, + certificateFormat=crypto.FILETYPE_ASN1, + secondsToExpiry=60 * 60 * 24 * 365, # One year + digestAlgorithm='sha256'): + """ + Given a blob of certificate request data and a certificate authority's + DistinguishedName, return a blob of signed certificate data. + + If verifyDNCallback returns a Deferred, I will return a Deferred which + fires the data when that Deferred has completed. + """ + hlreq = CertificateRequest.load(requestData, requestFormat) + + dn = hlreq.getSubject() + vval = verifyDNCallback(dn) + + def verified(value): + if not value: + raise VerifyError("DN callback %r rejected request DN %r" % (verifyDNCallback, dn)) + return self.signRequestObject(issuerDistinguishedName, hlreq, + serialNumber, secondsToExpiry, digestAlgorithm).dump(certificateFormat) + + if isinstance(vval, Deferred): + return vval.addCallback(verified) + else: + return verified(vval) + + + def signRequestObject(self, + issuerDistinguishedName, + requestObject, + serialNumber, + secondsToExpiry=60 * 60 * 24 * 365, # One year + digestAlgorithm='sha256'): + """ + Sign a CertificateRequest instance, returning a Certificate instance. + """ + req = requestObject.original + cert = crypto.X509() + issuerDistinguishedName._copyInto(cert.get_issuer()) + cert.set_subject(req.get_subject()) + cert.set_pubkey(req.get_pubkey()) + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(secondsToExpiry) + cert.set_serial_number(serialNumber) + cert.sign(self.original, digestAlgorithm) + return Certificate(cert) + + + def selfSignedCert(self, serialNumber, **kw): + dn = DN(**kw) + return PrivateCertificate.fromCertificateAndKeyPair( + self.signRequestObject(dn, self.requestObject(dn), serialNumber), + self) + +KeyPair.__getstate__ = deprecated(Version("Twisted", 15, 0, 0), + "a real persistence system")(KeyPair.__getstate__) +KeyPair.__setstate__ = deprecated(Version("Twisted", 15, 0, 0), + "a real persistence system")(KeyPair.__setstate__) + + + +class IOpenSSLTrustRoot(Interface): + """ + Trust settings for an OpenSSL context. + + Note that this interface's methods are private, so things outside of + Twisted shouldn't implement it. + """ + + def _addCACertsToContext(context): + """ + Add certificate-authority certificates to an SSL context whose + connections should trust those authorities. + + @param context: An SSL context for a connection which should be + verified by some certificate authority. + @type context: L{OpenSSL.SSL.Context} + + @return: L{None} + """ + + + +@implementer(IOpenSSLTrustRoot) +class OpenSSLCertificateAuthorities(object): + """ + Trust an explicitly specified set of certificates, represented by a list of + L{OpenSSL.crypto.X509} objects. + """ + + def __init__(self, caCerts): + """ + @param caCerts: The certificate authorities to trust when using this + object as a C{trustRoot} for L{OpenSSLCertificateOptions}. + @type caCerts: L{list} of L{OpenSSL.crypto.X509} + """ + self._caCerts = caCerts + + + def _addCACertsToContext(self, context): + store = context.get_cert_store() + for cert in self._caCerts: + store.add_cert(cert) + + + +def trustRootFromCertificates(certificates): + """ + Builds an object that trusts multiple root L{Certificate}s. + + When passed to L{optionsForClientTLS}, connections using those options will + reject any server certificate not signed by at least one of the + certificates in the `certificates` list. + + @since: 16.0 + + @param certificates: All certificates which will be trusted. + @type certificates: C{iterable} of L{CertBase} + + @rtype: L{IOpenSSLTrustRoot} + @return: an object suitable for use as the trustRoot= keyword argument to + L{optionsForClientTLS} + """ + + certs = [] + for cert in certificates: + # PrivateCertificate or Certificate are both okay + if isinstance(cert, CertBase): + cert = cert.original + else: + raise TypeError( + "certificates items must be twisted.internet.ssl.CertBase" + " instances" + ) + certs.append(cert) + return OpenSSLCertificateAuthorities(certs) + + + +@implementer(IOpenSSLTrustRoot) +class OpenSSLDefaultPaths(object): + """ + Trust the set of default verify paths that OpenSSL was built with, as + specified by U{SSL_CTX_set_default_verify_paths + }. + """ + + def _addCACertsToContext(self, context): + context.set_default_verify_paths() + + + +def platformTrust(): + """ + Attempt to discover a set of trusted certificate authority certificates + (or, in other words: trust roots, or root certificates) whose trust is + managed and updated by tools outside of Twisted. + + If you are writing any client-side TLS code with Twisted, you should use + this as the C{trustRoot} argument to L{CertificateOptions + }. + + The result of this function should be like the up-to-date list of + certificates in a web browser. When developing code that uses + C{platformTrust}, you can think of it that way. However, the choice of + which certificate authorities to trust is never Twisted's responsibility. + Unless you're writing a very unusual application or library, it's not your + code's responsibility either. The user may use platform-specific tools for + defining which server certificates should be trusted by programs using TLS. + The purpose of using this API is to respect that decision as much as + possible. + + This should be a set of trust settings most appropriate for I{client} TLS + connections; i.e. those which need to verify a server's authenticity. You + should probably use this by default for any client TLS connection that you + create. For servers, however, client certificates are typically not + verified; or, if they are, their verification will depend on a custom, + application-specific certificate authority. + + @since: 14.0 + + @note: Currently, L{platformTrust} depends entirely upon your OpenSSL build + supporting a set of "L{default verify paths }" + which correspond to certificate authority trust roots. Unfortunately, + whether this is true of your system is both outside of Twisted's + control and difficult (if not impossible) for Twisted to detect + automatically. + + Nevertheless, this ought to work as desired by default on: + + - Ubuntu Linux machines with the U{ca-certificates + } package + installed, + + - macOS when using the system-installed version of OpenSSL (i.e. + I{not} one installed via MacPorts or Homebrew), + + - any build of OpenSSL which has had certificate authority + certificates installed into its default verify paths (by default, + C{/usr/local/ssl/certs} if you've built your own OpenSSL), or + + - any process where the C{SSL_CERT_FILE} environment variable is + set to the path of a file containing your desired CA certificates + bundle. + + Hopefully soon, this API will be updated to use more sophisticated + trust-root discovery mechanisms. Until then, you can follow tickets in + the Twisted tracker for progress on this implementation on U{Microsoft + Windows }, U{macOS + }, and U{a fallback for + other platforms which do not have native trust management tools + }. + + @return: an appropriate trust settings object for your platform. + @rtype: L{IOpenSSLTrustRoot} + + @raise NotImplementedError: if this platform is not yet supported by + Twisted. At present, only OpenSSL is supported. + """ + return OpenSSLDefaultPaths() + + + +def _tolerateErrors(wrapped): + """ + Wrap up an C{info_callback} for pyOpenSSL so that if something goes wrong + the error is immediately logged and the connection is dropped if possible. + + This wrapper exists because some versions of pyOpenSSL don't handle errors + from callbacks at I{all}, and those which do write tracebacks directly to + stderr rather than to a supplied logging system. This reports unexpected + errors to the Twisted logging system. + + Also, this terminates the connection immediately if possible because if + you've got bugs in your verification logic it's much safer to just give up. + + @param wrapped: A valid C{info_callback} for pyOpenSSL. + @type wrapped: L{callable} + + @return: A valid C{info_callback} for pyOpenSSL that handles any errors in + C{wrapped}. + @rtype: L{callable} + """ + def infoCallback(connection, where, ret): + try: + return wrapped(connection, where, ret) + except: + f = Failure() + log.err(f, "Error during info_callback") + connection.get_app_data().failVerification(f) + return infoCallback + + + +@implementer(IOpenSSLClientConnectionCreator) +class ClientTLSOptions(object): + """ + Client creator for TLS. + + Private implementation type (not exposed to applications) for public + L{optionsForClientTLS} API. + + @ivar _ctx: The context to use for new connections. + @type _ctx: L{OpenSSL.SSL.Context} + + @ivar _hostname: The hostname to verify, as specified by the application, + as some human-readable text. + @type _hostname: L{unicode} + + @ivar _hostnameBytes: The hostname to verify, decoded into IDNA-encoded + bytes. This is passed to APIs which think that hostnames are bytes, + such as OpenSSL's SNI implementation. + @type _hostnameBytes: L{bytes} + + @ivar _hostnameASCII: The hostname, as transcoded into IDNA ASCII-range + unicode code points. This is pre-transcoded because the + C{service_identity} package is rather strict about requiring the + C{idna} package from PyPI for internationalized domain names, rather + than working with Python's built-in (but sometimes broken) IDNA + encoding. ASCII values, however, will always work. + @type _hostnameASCII: L{unicode} + + @ivar _hostnameIsDnsName: Whether or not the C{_hostname} is a DNSName. + Will be L{False} if C{_hostname} is an IP address or L{True} if + C{_hostname} is a DNSName + @type _hostnameIsDnsName: L{bool} + """ + + def __init__(self, hostname, ctx): + """ + Initialize L{ClientTLSOptions}. + + @param hostname: The hostname to verify as input by a human. + @type hostname: L{unicode} + + @param ctx: an L{OpenSSL.SSL.Context} to use for new connections. + @type ctx: L{OpenSSL.SSL.Context}. + """ + self._ctx = ctx + self._hostname = hostname + + if isIPAddress(hostname) or isIPv6Address(hostname): + self._hostnameBytes = hostname.encode('ascii') + self._hostnameIsDnsName = False + else: + self._hostnameBytes = _idnaBytes(hostname) + self._hostnameIsDnsName = True + + self._hostnameASCII = self._hostnameBytes.decode("ascii") + ctx.set_info_callback( + _tolerateErrors(self._identityVerifyingInfoCallback) + ) + + + def clientConnectionForTLS(self, tlsProtocol): + """ + Create a TLS connection for a client. + + @note: This will call C{set_app_data} on its connection. If you're + delegating to this implementation of this method, don't ever call + C{set_app_data} or C{set_info_callback} on the returned connection, + or you'll break the implementation of various features of this + class. + + @param tlsProtocol: the TLS protocol initiating the connection. + @type tlsProtocol: L{twisted.protocols.tls.TLSMemoryBIOProtocol} + + @return: the configured client connection. + @rtype: L{OpenSSL.SSL.Connection} + """ + context = self._ctx + connection = SSL.Connection(context, None) + connection.set_app_data(tlsProtocol) + return connection + + + def _identityVerifyingInfoCallback(self, connection, where, ret): + """ + U{info_callback + + } for pyOpenSSL that verifies the hostname in the presented certificate + matches the one passed to this L{ClientTLSOptions}. + + @param connection: the connection which is handshaking. + @type connection: L{OpenSSL.SSL.Connection} + + @param where: flags indicating progress through a TLS handshake. + @type where: L{int} + + @param ret: ignored + @type ret: ignored + """ + # Literal IPv4 and IPv6 addresses are not permitted + # as host names according to the RFCs + if where & SSL.SSL_CB_HANDSHAKE_START and self._hostnameIsDnsName: + connection.set_tlsext_host_name(self._hostnameBytes) + elif where & SSL.SSL_CB_HANDSHAKE_DONE: + try: + if self._hostnameIsDnsName: + verifyHostname(connection, self._hostnameASCII) + else: + verifyIPAddress(connection, self._hostnameASCII) + except VerificationError: + f = Failure() + transport = connection.get_app_data() + transport.failVerification(f) + + + +def optionsForClientTLS(hostname, trustRoot=None, clientCertificate=None, + acceptableProtocols=None, **kw): + """ + Create a L{client connection creator } for + use with APIs such as L{SSL4ClientEndpoint + }, L{connectSSL + }, and L{startTLS + }. + + @since: 14.0 + + @param hostname: The expected name of the remote host. This serves two + purposes: first, and most importantly, it verifies that the certificate + received from the server correctly identifies the specified hostname. + The second purpose is to use the U{Server Name Indication extension + } to indicate to + the server which certificate should be used. + @type hostname: L{unicode} + + @param trustRoot: Specification of trust requirements of peers. This may be + a L{Certificate} or the result of L{platformTrust}. By default it is + L{platformTrust} and you probably shouldn't adjust it unless you really + know what you're doing. Be aware that clients using this interface + I{must} verify the server; you cannot explicitly pass L{None} since + that just means to use L{platformTrust}. + @type trustRoot: L{IOpenSSLTrustRoot} + + @param clientCertificate: The certificate and private key that the client + will use to authenticate to the server. If unspecified, the client will + not authenticate. + @type clientCertificate: L{PrivateCertificate} + + @param acceptableProtocols: The protocols this peer is willing to speak + after the TLS negotiation has completed, advertised over both ALPN and + NPN. If this argument is specified, and no overlap can be found with + the other peer, the connection will fail to be established. If the + remote peer does not offer NPN or ALPN, the connection will be + established, but no protocol wil be negotiated. Protocols earlier in + the list are preferred over those later in the list. + @type acceptableProtocols: L{list} of L{bytes} + + @param extraCertificateOptions: keyword-only argument; this is a dictionary + of additional keyword arguments to be presented to + L{CertificateOptions}. Please avoid using this unless you absolutely + need to; any time you need to pass an option here that is a bug in this + interface. + @type extraCertificateOptions: L{dict} + + @param kw: (Backwards compatibility hack to allow keyword-only arguments on + Python 2. Please ignore; arbitrary keyword arguments will be errors.) + @type kw: L{dict} + + @return: A client connection creator. + @rtype: L{IOpenSSLClientConnectionCreator} + """ + extraCertificateOptions = kw.pop('extraCertificateOptions', None) or {} + if trustRoot is None: + trustRoot = platformTrust() + if kw: + raise TypeError( + "optionsForClientTLS() got an unexpected keyword argument" + " '{arg}'".format( + arg=kw.popitem()[0] + ) + ) + if not isinstance(hostname, unicode): + raise TypeError( + "optionsForClientTLS requires text for host names, not " + + hostname.__class__.__name__ + ) + if clientCertificate: + extraCertificateOptions.update( + privateKey=clientCertificate.privateKey.original, + certificate=clientCertificate.original + ) + certificateOptions = OpenSSLCertificateOptions( + trustRoot=trustRoot, + acceptableProtocols=acceptableProtocols, + **extraCertificateOptions + ) + return ClientTLSOptions(hostname, certificateOptions.getContext()) + + + +@implementer(IOpenSSLContextFactory) +class OpenSSLCertificateOptions(object): + """ + A L{CertificateOptions } specifies + the security properties for a client or server TLS connection used with + OpenSSL. + + @ivar _options: Any option flags to set on the L{OpenSSL.SSL.Context} + object that will be created. + @type _options: L{int} + + @ivar _cipherString: An OpenSSL-specific cipher string. + @type _cipherString: L{unicode} + + @ivar _defaultMinimumTLSVersion: The default TLS version that will be + negotiated. This should be a "safe default", with wide client and + server support, vs an optimally secure one that excludes a large number + of users. As of late 2016, TLSv1.0 is that safe default. + @type _defaultMinimumTLSVersion: L{TLSVersion} constant + """ + + # Factory for creating contexts. Configurable for testability. + _contextFactory = SSL.Context + _context = None + + _OP_NO_TLSv1_3 = _tlsDisableFlags[TLSVersion.TLSv1_3] + + _defaultMinimumTLSVersion = TLSVersion.TLSv1_0 + + @_mutuallyExclusiveArguments([ + ['trustRoot', 'requireCertificate'], + ['trustRoot', 'verify'], + ['trustRoot', 'caCerts'], + ['method', 'insecurelyLowerMinimumTo'], + ['method', 'raiseMinimumTo'], + ['raiseMinimumTo', 'insecurelyLowerMinimumTo'], + ['method', 'lowerMaximumSecurityTo'], + ]) + def __init__(self, + privateKey=None, + certificate=None, + method=None, + verify=False, + caCerts=None, + verifyDepth=9, + requireCertificate=True, + verifyOnce=True, + enableSingleUseKeys=True, + enableSessions=True, + fixBrokenPeers=False, + enableSessionTickets=False, + extraCertChain=None, + acceptableCiphers=None, + dhParameters=None, + trustRoot=None, + acceptableProtocols=None, + raiseMinimumTo=None, + insecurelyLowerMinimumTo=None, + lowerMaximumSecurityTo=None, + ): + """ + Create an OpenSSL context SSL connection context factory. + + @param privateKey: A PKey object holding the private key. + + @param certificate: An X509 object holding the certificate. + + @param method: Deprecated, use a combination of + C{insecurelyLowerMinimumTo}, C{raiseMinimumTo}, or + C{lowerMaximumSecurityTo} instead. The SSL protocol to use, one of + C{SSLv23_METHOD}, C{SSLv2_METHOD}, C{SSLv3_METHOD}, C{TLSv1_METHOD} + (or any other method constants provided by pyOpenSSL). By default, + a setting will be used which allows TLSv1.0, TLSv1.1, and TLSv1.2. + Can not be used with C{insecurelyLowerMinimumTo}, + C{raiseMinimumTo}, or C{lowerMaximumSecurityTo} + + @param verify: Please use a C{trustRoot} keyword argument instead, + since it provides the same functionality in a less error-prone way. + By default this is L{False}. + + If L{True}, verify certificates received from the peer and fail the + handshake if verification fails. Otherwise, allow anonymous + sessions and sessions with certificates which fail validation. + + @param caCerts: Please use a C{trustRoot} keyword argument instead, + since it provides the same functionality in a less error-prone way. + + List of certificate authority certificate objects to use to verify + the peer's certificate. Only used if verify is L{True} and will be + ignored otherwise. Since verify is L{False} by default, this is + L{None} by default. + + @type caCerts: L{list} of L{OpenSSL.crypto.X509} + + @param verifyDepth: Depth in certificate chain down to which to verify. + If unspecified, use the underlying default (9). + + @param requireCertificate: Please use a C{trustRoot} keyword argument + instead, since it provides the same functionality in a less + error-prone way. + + If L{True}, do not allow anonymous sessions; defaults to L{True}. + + @param verifyOnce: If True, do not re-verify the certificate on session + resumption. + + @param enableSingleUseKeys: If L{True}, generate a new key whenever + ephemeral DH and ECDH parameters are used to prevent small subgroup + attacks and to ensure perfect forward secrecy. + + @param enableSessions: If True, set a session ID on each context. This + allows a shortened handshake to be used when a known client + reconnects. + + @param fixBrokenPeers: If True, enable various non-spec protocol fixes + for broken SSL implementations. This should be entirely safe, + according to the OpenSSL documentation, but YMMV. This option is + now off by default, because it causes problems with connections + between peers using OpenSSL 0.9.8a. + + @param enableSessionTickets: If L{True}, enable session ticket + extension for session resumption per RFC 5077. Note there is no + support for controlling session tickets. This option is off by + default, as some server implementations don't correctly process + incoming empty session ticket extensions in the hello. + + @param extraCertChain: List of certificates that I{complete} your + verification chain if the certificate authority that signed your + C{certificate} isn't widely supported. Do I{not} add + C{certificate} to it. + @type extraCertChain: C{list} of L{OpenSSL.crypto.X509} + + @param acceptableCiphers: Ciphers that are acceptable for connections. + Uses a secure default if left L{None}. + @type acceptableCiphers: L{IAcceptableCiphers} + + @param dhParameters: Key generation parameters that are required for + Diffie-Hellman key exchange. If this argument is left L{None}, + C{EDH} ciphers are I{disabled} regardless of C{acceptableCiphers}. + @type dhParameters: L{DiffieHellmanParameters + } + + @param trustRoot: Specification of trust requirements of peers. If + this argument is specified, the peer is verified. It requires a + certificate, and that certificate must be signed by one of the + certificate authorities specified by this object. + + Note that since this option specifies the same information as + C{caCerts}, C{verify}, and C{requireCertificate}, specifying any of + those options in combination with this one will raise a + L{TypeError}. + + @type trustRoot: L{IOpenSSLTrustRoot} + + @param acceptableProtocols: The protocols this peer is willing to speak + after the TLS negotiation has completed, advertised over both ALPN + and NPN. If this argument is specified, and no overlap can be + found with the other peer, the connection will fail to be + established. If the remote peer does not offer NPN or ALPN, the + connection will be established, but no protocol wil be negotiated. + Protocols earlier in the list are preferred over those later in the + list. + @type acceptableProtocols: L{list} of L{bytes} + + @param raiseMinimumTo: The minimum TLS version that you want to use, or + Twisted's default if it is higher. Use this if you want to make + your client/server more secure than Twisted's default, but will + accept Twisted's default instead if it moves higher than this + value. You probably want to use this over + C{insecurelyLowerMinimumTo}. + @type raiseMinimumTo: L{TLSVersion} constant + + @param insecurelyLowerMinimumTo: The minimum TLS version to use, + possibly lower than Twisted's default. If not specified, it is a + generally considered safe default (TLSv1.0). If you want to raise + your minimum TLS version to above that of this default, use + C{raiseMinimumTo}. DO NOT use this argument unless you are + absolutely sure this is what you want. + @type insecurelyLowerMinimumTo: L{TLSVersion} constant + + @param lowerMaximumSecurityTo: The maximum TLS version to use. If not + specified, it is the most recent your OpenSSL supports. You only + want to set this if the peer that you are communicating with has + problems with more recent TLS versions, it lowers your security + when communicating with newer peers. DO NOT use this argument + unless you are absolutely sure this is what you want. + @type lowerMaximumSecurityTo: L{TLSVersion} constant + + @raise ValueError: when C{privateKey} or C{certificate} are set without + setting the respective other. + @raise ValueError: when C{verify} is L{True} but C{caCerts} doesn't + specify any CA certificates. + @raise ValueError: when C{extraCertChain} is passed without specifying + C{privateKey} or C{certificate}. + @raise ValueError: when C{acceptableCiphers} doesn't yield any usable + ciphers for the current platform. + + @raise TypeError: if C{trustRoot} is passed in combination with + C{caCert}, C{verify}, or C{requireCertificate}. Please prefer + C{trustRoot} in new code, as its semantics are less tricky. + @raise TypeError: if C{method} is passed in combination with + C{tlsProtocols}. Please prefer the more explicit C{tlsProtocols} + in new code. + + @raises NotImplementedError: If acceptableProtocols were provided but + no negotiation mechanism is available. + """ + + if (privateKey is None) != (certificate is None): + raise ValueError( + "Specify neither or both of privateKey and certificate") + self.privateKey = privateKey + self.certificate = certificate + + # Set basic security options: disallow insecure SSLv2, disallow TLS + # compression to avoid CRIME attack, make the server choose the + # ciphers. + self._options = ( + SSL.OP_NO_SSLv2 | SSL.OP_NO_COMPRESSION | + SSL.OP_CIPHER_SERVER_PREFERENCE + ) + + # Set the mode to Release Buffers, which demallocs send/recv buffers on + # idle TLS connections to save memory + self._mode = SSL.MODE_RELEASE_BUFFERS + + if method is None: + self.method = SSL.SSLv23_METHOD + + if raiseMinimumTo: + if (lowerMaximumSecurityTo and + raiseMinimumTo > lowerMaximumSecurityTo): + raise ValueError( + ("raiseMinimumTo needs to be lower than " + "lowerMaximumSecurityTo")) + + if raiseMinimumTo > self._defaultMinimumTLSVersion: + insecurelyLowerMinimumTo = raiseMinimumTo + + if insecurelyLowerMinimumTo is None: + insecurelyLowerMinimumTo = self._defaultMinimumTLSVersion + + # If you set the max lower than the default, but don't set the + # minimum, pull it down to that + if (lowerMaximumSecurityTo and + insecurelyLowerMinimumTo > lowerMaximumSecurityTo): + insecurelyLowerMinimumTo = lowerMaximumSecurityTo + + if (lowerMaximumSecurityTo and + insecurelyLowerMinimumTo > lowerMaximumSecurityTo): + raise ValueError( + ("insecurelyLowerMinimumTo needs to be lower than " + "lowerMaximumSecurityTo")) + + excludedVersions = _getExcludedTLSProtocols( + insecurelyLowerMinimumTo, lowerMaximumSecurityTo) + + for version in excludedVersions: + self._options |= _tlsDisableFlags[version] + else: + warnings.warn( + ("Passing method to twisted.internet.ssl.CertificateOptions " + "was deprecated in Twisted 17.1.0. Please use a combination " + "of insecurelyLowerMinimumTo, raiseMinimumTo, and " + "lowerMaximumSecurityTo instead, as Twisted will correctly " + "configure the method."), + DeprecationWarning, stacklevel=3) + + # Otherwise respect the application decision. + self.method = method + + if verify and not caCerts: + raise ValueError("Specify client CA certificate information if and" + " only if enabling certificate verification") + self.verify = verify + if extraCertChain is not None and None in (privateKey, certificate): + raise ValueError("A private key and a certificate are required " + "when adding a supplemental certificate chain.") + if extraCertChain is not None: + self.extraCertChain = extraCertChain + else: + self.extraCertChain = [] + + self.caCerts = caCerts + self.verifyDepth = verifyDepth + self.requireCertificate = requireCertificate + self.verifyOnce = verifyOnce + self.enableSingleUseKeys = enableSingleUseKeys + if enableSingleUseKeys: + self._options |= SSL.OP_SINGLE_DH_USE | SSL.OP_SINGLE_ECDH_USE + self.enableSessions = enableSessions + self.fixBrokenPeers = fixBrokenPeers + if fixBrokenPeers: + self._options |= SSL.OP_ALL + self.enableSessionTickets = enableSessionTickets + + if not enableSessionTickets: + self._options |= SSL.OP_NO_TICKET + self.dhParameters = dhParameters + + self._ecChooser = _ChooseDiffieHellmanEllipticCurve( + SSL.OPENSSL_VERSION_NUMBER, + openSSLlib=pyOpenSSLlib, + openSSLcrypto=crypto, + ) + + if acceptableCiphers is None: + acceptableCiphers = defaultCiphers + # This needs to run when method and _options are finalized. + self._cipherString = u':'.join( + c.fullName + for c in acceptableCiphers.selectCiphers( + _expandCipherString(u'ALL', self.method, self._options) + ) + ) + if self._cipherString == u'': + raise ValueError( + 'Supplied IAcceptableCiphers yielded no usable ciphers ' + 'on this platform.' + ) + + if trustRoot is None: + if self.verify: + trustRoot = OpenSSLCertificateAuthorities(caCerts) + else: + self.verify = True + self.requireCertificate = True + trustRoot = IOpenSSLTrustRoot(trustRoot) + self.trustRoot = trustRoot + + if acceptableProtocols is not None and not protocolNegotiationMechanisms(): + raise NotImplementedError( + "No support for protocol negotiation on this platform." + ) + + self._acceptableProtocols = acceptableProtocols + + + def __getstate__(self): + d = self.__dict__.copy() + try: + del d['_context'] + except KeyError: + pass + return d + + + def __setstate__(self, state): + self.__dict__ = state + + + def getContext(self): + """ + Return an L{OpenSSL.SSL.Context} object. + """ + if self._context is None: + self._context = self._makeContext() + return self._context + + + def _makeContext(self): + ctx = self._contextFactory(self.method) + ctx.set_options(self._options) + ctx.set_mode(self._mode) + + if self.certificate is not None and self.privateKey is not None: + ctx.use_certificate(self.certificate) + ctx.use_privatekey(self.privateKey) + for extraCert in self.extraCertChain: + ctx.add_extra_chain_cert(extraCert) + # Sanity check + ctx.check_privatekey() + + verifyFlags = SSL.VERIFY_NONE + if self.verify: + verifyFlags = SSL.VERIFY_PEER + if self.requireCertificate: + verifyFlags |= SSL.VERIFY_FAIL_IF_NO_PEER_CERT + if self.verifyOnce: + verifyFlags |= SSL.VERIFY_CLIENT_ONCE + self.trustRoot._addCACertsToContext(ctx) + + # It'd be nice if pyOpenSSL let us pass None here for this behavior (as + # the underlying OpenSSL API call allows NULL to be passed). It + # doesn't, so we'll supply a function which does the same thing. + def _verifyCallback(conn, cert, errno, depth, preverify_ok): + return preverify_ok + ctx.set_verify(verifyFlags, _verifyCallback) + if self.verifyDepth is not None: + ctx.set_verify_depth(self.verifyDepth) + + if self.enableSessions: + # 32 bytes is the maximum length supported + # Unfortunately pyOpenSSL doesn't provide SSL_MAX_SESSION_ID_LENGTH + sessionName = secureRandom(32) + ctx.set_session_id(sessionName) + + if self.dhParameters: + ctx.load_tmp_dh(self.dhParameters._dhFile.path) + ctx.set_cipher_list(self._cipherString.encode('ascii')) + + self._ecChooser.configureECDHCurve(ctx) + + if self._acceptableProtocols: + # Try to set NPN and ALPN. _acceptableProtocols cannot be set by + # the constructor unless at least one mechanism is supported. + _setAcceptableProtocols(ctx, self._acceptableProtocols) + + return ctx + + +OpenSSLCertificateOptions.__getstate__ = deprecated( + Version("Twisted", 15, 0, 0), + "a real persistence system")(OpenSSLCertificateOptions.__getstate__) +OpenSSLCertificateOptions.__setstate__ = deprecated( + Version("Twisted", 15, 0, 0), + "a real persistence system")(OpenSSLCertificateOptions.__setstate__) + + + +@implementer(ICipher) +class OpenSSLCipher(FancyEqMixin, object): + """ + A representation of an OpenSSL cipher. + """ + compareAttributes = ('fullName',) + + def __init__(self, fullName): + """ + @param fullName: The full name of the cipher. For example + C{u"ECDHE-RSA-AES256-GCM-SHA384"}. + @type fullName: L{unicode} + """ + self.fullName = fullName + + + def __repr__(self): + """ + A runnable representation of the cipher. + """ + return 'OpenSSLCipher({0!r})'.format(self.fullName) + + + +def _expandCipherString(cipherString, method, options): + """ + Expand C{cipherString} according to C{method} and C{options} to a list + of explicit ciphers that are supported by the current platform. + + @param cipherString: An OpenSSL cipher string to expand. + @type cipherString: L{unicode} + + @param method: An OpenSSL method like C{SSL.TLSv1_METHOD} used for + determining the effective ciphers. + + @param options: OpenSSL options like C{SSL.OP_NO_SSLv3} ORed together. + @type options: L{int} + + @return: The effective list of explicit ciphers that results from the + arguments on the current platform. + @rtype: L{list} of L{ICipher} + """ + ctx = SSL.Context(method) + ctx.set_options(options) + try: + ctx.set_cipher_list(cipherString.encode('ascii')) + except SSL.Error as e: + # OpenSSL 1.1.1 turns an invalid cipher list into TLS 1.3 + # ciphers, so pyOpenSSL >= 19.0.0 raises an artificial Error + # that lacks a corresponding OpenSSL error if the cipher list + # consists only of these after a call to set_cipher_list. + if not e.args[0]: + return [] + if e.args[0][0][2] == 'no cipher match': + return [] + else: + raise + conn = SSL.Connection(ctx, None) + ciphers = conn.get_cipher_list() + if isinstance(ciphers[0], unicode): + return [OpenSSLCipher(cipher) for cipher in ciphers] + else: + return [OpenSSLCipher(cipher.decode('ascii')) for cipher in ciphers] + + + +@implementer(IAcceptableCiphers) +class OpenSSLAcceptableCiphers(object): + """ + A representation of ciphers that are acceptable for TLS connections. + """ + def __init__(self, ciphers): + self._ciphers = ciphers + + + def selectCiphers(self, availableCiphers): + return [cipher + for cipher in self._ciphers + if cipher in availableCiphers] + + + @classmethod + def fromOpenSSLCipherString(cls, cipherString): + """ + Create a new instance using an OpenSSL cipher string. + + @param cipherString: An OpenSSL cipher string that describes what + cipher suites are acceptable. + See the documentation of U{OpenSSL + } or + U{Apache + } + for details. + @type cipherString: L{unicode} + + @return: Instance representing C{cipherString}. + @rtype: L{twisted.internet.ssl.AcceptableCiphers} + """ + return cls(_expandCipherString( + nativeString(cipherString), + SSL.SSLv23_METHOD, SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) + ) + + +# A secure default. +# Sources for more information on TLS ciphers: +# +# - https://wiki.mozilla.org/Security/Server_Side_TLS +# - https://www.ssllabs.com/projects/best-practices/index.html +# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ +# +# The general intent is: +# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), +# - prefer ECDHE over DHE for better performance, +# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and +# security, +# - prefer AES-GCM to ChaCha20 because AES hardware support is common, +# - disable NULL authentication, MD5 MACs and DSS for security reasons. +# +defaultCiphers = OpenSSLAcceptableCiphers.fromOpenSSLCipherString( + "TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:" + "TLS13-AES-128-GCM-SHA256:" + "ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:DH+CHACHA20:ECDH+AES256:DH+AES256:" + "ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:" + "!aNULL:!MD5:!DSS" +) +_defaultCurveName = u"prime256v1" + + + +class _ChooseDiffieHellmanEllipticCurve(object): + """ + Chooses the best elliptic curve for Elliptic Curve Diffie-Hellman + key exchange, and provides a C{configureECDHCurve} method to set + the curve, when appropriate, on a new L{OpenSSL.SSL.Context}. + + The C{configureECDHCurve} method will be set to one of the + following based on the provided OpenSSL version and configuration: + + - L{_configureOpenSSL110} + + - L{_configureOpenSSL102} + + - L{_configureOpenSSL101} + + - L{_configureOpenSSL101NoCurves}. + + @param openSSLVersion: The OpenSSL version number. + @type openSSLVersion: L{int} + + @see: L{OpenSSL.SSL.OPENSSL_VERSION_NUMBER} + + @param openSSLlib: The OpenSSL C{cffi} library module. + @param openSSLlib: The OpenSSL L{crypto} module. + + @see: L{crypto} + """ + + def __init__(self, openSSLVersion, openSSLlib, openSSLcrypto): + self._openSSLlib = openSSLlib + self._openSSLcrypto = openSSLcrypto + if openSSLVersion >= 0x10100000: + self.configureECDHCurve = self._configureOpenSSL110 + elif openSSLVersion >= 0x10002000: + self.configureECDHCurve = self._configureOpenSSL102 + else: + try: + self._ecCurve = openSSLcrypto.get_elliptic_curve( + _defaultCurveName) + except ValueError: + # The get_elliptic_curve method raises a ValueError + # when the curve does not exist. + self.configureECDHCurve = self._configureOpenSSL101NoCurves + else: + self.configureECDHCurve = self._configureOpenSSL101 + + + def _configureOpenSSL110(self, ctx): + """ + OpenSSL 1.1.0 Contexts are preconfigured with an optimal set + of ECDH curves. This method does nothing. + + @param ctx: L{OpenSSL.SSL.Context} + """ + + + def _configureOpenSSL102(self, ctx): + """ + Have the context automatically choose elliptic curves for + ECDH. Run on OpenSSL 1.0.2 and OpenSSL 1.1.0+, but only has + an effect on OpenSSL 1.0.2. + + @param ctx: The context which . + @type ctx: L{OpenSSL.SSL.Context} + """ + ctxPtr = ctx._context + try: + self._openSSLlib.SSL_CTX_set_ecdh_auto(ctxPtr, True) + except: + pass + + + def _configureOpenSSL101(self, ctx): + """ + Set the default elliptic curve for ECDH on the context. Only + run on OpenSSL 1.0.1. + + @param ctx: The context on which to set the ECDH curve. + @type ctx: L{OpenSSL.SSL.Context} + """ + try: + ctx.set_tmp_ecdh(self._ecCurve) + except: + pass + + + def _configureOpenSSL101NoCurves(self, ctx): + """ + No elliptic curves are available on OpenSSL 1.0.1. We can't + set anything, so do nothing. + + @param ctx: The context on which to set the ECDH curve. + @type ctx: L{OpenSSL.SSL.Context} + """ + + + +class OpenSSLDiffieHellmanParameters(object): + """ + A representation of key generation parameters that are required for + Diffie-Hellman key exchange. + """ + def __init__(self, parameters): + self._dhFile = parameters + + + @classmethod + def fromFile(cls, filePath): + """ + Load parameters from a file. + + Such a file can be generated using the C{openssl} command line tool as + following: + + C{openssl dhparam -out dh_param_2048.pem -2 2048} + + Please refer to U{OpenSSL's C{dhparam} documentation + } for further details. + + @param filePath: A file containing parameters for Diffie-Hellman key + exchange. + @type filePath: L{FilePath } + + @return: An instance that loads its parameters from C{filePath}. + @rtype: L{DiffieHellmanParameters + } + """ + return cls(filePath) + + + +def _setAcceptableProtocols(context, acceptableProtocols): + """ + Called to set up the L{OpenSSL.SSL.Context} for doing NPN and/or ALPN + negotiation. + + @param context: The context which is set up. + @type context: L{OpenSSL.SSL.Context} + + @param acceptableProtocols: The protocols this peer is willing to speak + after the TLS negotiation has completed, advertised over both ALPN and + NPN. If this argument is specified, and no overlap can be found with + the other peer, the connection will fail to be established. If the + remote peer does not offer NPN or ALPN, the connection will be + established, but no protocol wil be negotiated. Protocols earlier in + the list are preferred over those later in the list. + @type acceptableProtocols: L{list} of L{bytes} + """ + def protoSelectCallback(conn, protocols): + """ + NPN client-side and ALPN server-side callback used to select + the next protocol. Prefers protocols found earlier in + C{_acceptableProtocols}. + + @param conn: The context which is set up. + @type conn: L{OpenSSL.SSL.Connection} + + @param conn: Protocols advertised by the other side. + @type conn: L{list} of L{bytes} + """ + overlap = set(protocols) & set(acceptableProtocols) + + for p in acceptableProtocols: + if p in overlap: + return p + else: + return b'' + + # If we don't actually have protocols to negotiate, don't set anything up. + # Depending on OpenSSL version, failing some of the selection callbacks can + # cause the handshake to fail, which is presumably not what was intended + # here. + if not acceptableProtocols: + return + + supported = protocolNegotiationMechanisms() + + if supported & ProtocolNegotiationSupport.NPN: + def npnAdvertiseCallback(conn): + return acceptableProtocols + + context.set_npn_advertise_callback(npnAdvertiseCallback) + context.set_npn_select_callback(protoSelectCallback) + + if supported & ProtocolNegotiationSupport.ALPN: + context.set_alpn_select_callback(protoSelectCallback) + context.set_alpn_protos(acceptableProtocols) diff --git a/contrib/python/Twisted/py2/twisted/internet/_threadedselect.py b/contrib/python/Twisted/py2/twisted/internet/_threadedselect.py new file mode 100644 index 00000000000..113646a7081 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_threadedselect.py @@ -0,0 +1,354 @@ +# -*- test-case-name: twisted.test.test_internet -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Threaded select reactor + +The threadedselectreactor is a specialized reactor for integrating with +arbitrary foreign event loop, such as those you find in GUI toolkits. + +There are three things you'll need to do to use this reactor. + +Install the reactor at the beginning of your program, before importing +the rest of Twisted:: + + | from twisted.internet import _threadedselect + | _threadedselect.install() + +Interleave this reactor with your foreign event loop, at some point after +your event loop is initialized:: + + | from twisted.internet import reactor + | reactor.interleave(foreignEventLoopWakerFunction) + | self.addSystemEventTrigger('after', 'shutdown', foreignEventLoopStop) + +Instead of shutting down the foreign event loop directly, shut down the +reactor:: + + | from twisted.internet import reactor + | reactor.stop() + +In order for Twisted to do its work in the main thread (the thread that +interleave is called from), a waker function is necessary. The waker function +will be called from a "background" thread with one argument: func. +The waker function's purpose is to call func() from the main thread. +Many GUI toolkits ship with appropriate waker functions. +Some examples of this are wxPython's wx.callAfter (may be wxCallAfter in +older versions of wxPython) or PyObjC's PyObjCTools.AppHelper.callAfter. +These would be used in place of "foreignEventLoopWakerFunction" in the above +example. + +The other integration point at which the foreign event loop and this reactor +must integrate is shutdown. In order to ensure clean shutdown of Twisted, +you must allow for Twisted to come to a complete stop before quitting the +application. Typically, you will do this by setting up an after shutdown +trigger to stop your foreign event loop, and call reactor.stop() where you +would normally have initiated the shutdown procedure for the foreign event +loop. Shutdown functions that could be used in place of +"foreignEventloopStop" would be the ExitMainLoop method of the wxApp instance +with wxPython, or the PyObjCTools.AppHelper.stopEventLoop function. +""" + +from functools import partial +from threading import Thread + +try: + from queue import Queue, Empty +except ImportError: + from Queue import Queue, Empty +import sys + +from zope.interface import implementer + +from twisted.internet.interfaces import IReactorFDSet +from twisted.internet import posixbase +from twisted.internet.posixbase import _NO_FILENO, _NO_FILEDESC +from twisted.python import log, failure, threadable + +import select +from errno import EINTR, EBADF + +from twisted.internet.selectreactor import _select + + +def dictRemove(dct, value): + try: + del dct[value] + except KeyError: + pass + + +def raiseException(e): + raise e + + +@implementer(IReactorFDSet) +class ThreadedSelectReactor(posixbase.PosixReactorBase): + """A threaded select() based reactor - runs on all POSIX platforms and on + Win32. + """ + + def __init__(self): + threadable.init(1) + self.reads = {} + self.writes = {} + self.toThreadQueue = Queue() + self.toMainThread = Queue() + self.workerThread = None + self.mainWaker = None + posixbase.PosixReactorBase.__init__(self) + self.addSystemEventTrigger('after', 'shutdown', self._mainLoopShutdown) + + def wakeUp(self): + # we want to wake up from any thread + self.waker.wakeUp() + + def callLater(self, *args, **kw): + tple = posixbase.PosixReactorBase.callLater(self, *args, **kw) + self.wakeUp() + return tple + + def _sendToMain(self, msg, *args): + self.toMainThread.put((msg, args)) + if self.mainWaker is not None: + self.mainWaker() + + def _sendToThread(self, fn, *args): + self.toThreadQueue.put((fn, args)) + + def _preenDescriptorsInThread(self): + log.msg("Malformed file descriptor found. Preening lists.") + readers = self.reads.keys() + writers = self.writes.keys() + self.reads.clear() + self.writes.clear() + for selDict, selList in ((self.reads, readers), (self.writes, writers)): + for selectable in selList: + try: + select.select([selectable], [selectable], [selectable], 0) + except: + log.msg("bad descriptor %s" % selectable) + else: + selDict[selectable] = 1 + + def _workerInThread(self): + try: + while 1: + fn, args = self.toThreadQueue.get() + fn(*args) + except SystemExit: + pass # Exception indicates this thread should exit + except: + f = failure.Failure() + self._sendToMain('Failure', f) + + def _doSelectInThread(self, timeout): + """Run one iteration of the I/O monitor loop. + + This will run all selectables who had input or output readiness + waiting for them. + """ + reads = self.reads + writes = self.writes + while 1: + try: + r, w, ignored = _select(reads.keys(), + writes.keys(), + [], timeout) + break + except ValueError: + # Possibly a file descriptor has gone negative? + log.err() + self._preenDescriptorsInThread() + except TypeError: + # Something *totally* invalid (object w/o fileno, non-integral + # result) was passed + log.err() + self._preenDescriptorsInThread() + except (select.error, IOError) as se: + # select(2) encountered an error + if se.args[0] in (0, 2): + # windows does this if it got an empty list + if (not reads) and (not writes): + return + else: + raise + elif se.args[0] == EINTR: + return + elif se.args[0] == EBADF: + self._preenDescriptorsInThread() + else: + # OK, I really don't know what's going on. Blow up. + raise + self._sendToMain('Notify', r, w) + + def _process_Notify(self, r, w): + reads = self.reads + writes = self.writes + + _drdw = self._doReadOrWrite + _logrun = log.callWithLogger + for selectables, method, dct in ( + (r, "doRead", reads), (w, "doWrite", writes)): + for selectable in selectables: + # if this was disconnected in another thread, kill it. + if selectable not in dct: + continue + # This for pausing input when we're not ready for more. + _logrun(selectable, _drdw, selectable, method, dct) + + def _process_Failure(self, f): + f.raiseException() + + _doIterationInThread = _doSelectInThread + + def ensureWorkerThread(self): + if self.workerThread is None or not self.workerThread.isAlive(): + self.workerThread = Thread(target=self._workerInThread) + self.workerThread.start() + + def doThreadIteration(self, timeout): + self._sendToThread(self._doIterationInThread, timeout) + self.ensureWorkerThread() + msg, args = self.toMainThread.get() + getattr(self, '_process_' + msg)(*args) + + doIteration = doThreadIteration + + def _interleave(self): + while self.running: + self.runUntilCurrent() + t2 = self.timeout() + t = self.running and t2 + self._sendToThread(self._doIterationInThread, t) + yield None + msg, args = self.toMainThread.get_nowait() + getattr(self, '_process_' + msg)(*args) + + def interleave(self, waker, *args, **kw): + """ + interleave(waker) interleaves this reactor with the + current application by moving the blocking parts of + the reactor (select() in this case) to a separate + thread. This is typically useful for integration with + GUI applications which have their own event loop + already running. + + See the module docstring for more information. + """ + self.startRunning(*args, **kw) + loop = self._interleave() + + def mainWaker(waker=waker, loop=loop): + waker(partial(next, loop)) + + self.mainWaker = mainWaker + next(loop) + self.ensureWorkerThread() + + def _mainLoopShutdown(self): + self.mainWaker = None + if self.workerThread is not None: + self._sendToThread(raiseException, SystemExit) + self.wakeUp() + try: + while 1: + msg, args = self.toMainThread.get_nowait() + except Empty: + pass + self.workerThread.join() + self.workerThread = None + try: + while 1: + fn, args = self.toThreadQueue.get_nowait() + if fn is self._doIterationInThread: + log.msg('Iteration is still in the thread queue!') + elif fn is raiseException and args[0] is SystemExit: + pass + else: + fn(*args) + except Empty: + pass + + def _doReadOrWrite(self, selectable, method, dict): + try: + why = getattr(selectable, method)() + handfn = getattr(selectable, 'fileno', None) + if not handfn: + why = _NO_FILENO + elif handfn() == -1: + why = _NO_FILEDESC + except: + why = sys.exc_info()[1] + log.err() + if why: + self._disconnectSelectable(selectable, why, method == "doRead") + + def addReader(self, reader): + """Add a FileDescriptor for notification of data available to read. + """ + self._sendToThread(self.reads.__setitem__, reader, 1) + self.wakeUp() + + def addWriter(self, writer): + """Add a FileDescriptor for notification of data available to write. + """ + self._sendToThread(self.writes.__setitem__, writer, 1) + self.wakeUp() + + def removeReader(self, reader): + """Remove a Selectable for notification of data available to read. + """ + self._sendToThread(dictRemove, self.reads, reader) + + def removeWriter(self, writer): + """Remove a Selectable for notification of data available to write. + """ + self._sendToThread(dictRemove, self.writes, writer) + + def removeAll(self): + return self._removeAll(self.reads, self.writes) + + + def getReaders(self): + return list(self.reads.keys()) + + + def getWriters(self): + return list(self.writes.keys()) + + + def stop(self): + """ + Extend the base stop implementation to also wake up the select thread so + that C{runUntilCurrent} notices the reactor should stop. + """ + posixbase.PosixReactorBase.stop(self) + self.wakeUp() + + + def run(self, installSignalHandlers=True): + self.startRunning(installSignalHandlers=installSignalHandlers) + self.mainLoop() + + def mainLoop(self): + q = Queue() + self.interleave(q.put) + while self.running: + try: + q.get()() + except StopIteration: + break + + +def install(): + """Configure the twisted mainloop to be run using the select() reactor. + """ + reactor = ThreadedSelectReactor() + from twisted.internet.main import installReactor + installReactor(reactor) + return reactor + + +__all__ = ['install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/_win32serialport.py b/contrib/python/Twisted/py2/twisted/internet/_win32serialport.py new file mode 100644 index 00000000000..68c7e6e2738 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_win32serialport.py @@ -0,0 +1,133 @@ +# -*- test-case-name: twisted.internet.test.test_win32serialport -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Serial port support for Windows. + +Requires PySerial and pywin32. +""" + +from __future__ import division, absolute_import + +# system imports +from serial import PARITY_NONE +from serial import STOPBITS_ONE +from serial import EIGHTBITS +from serial.serialutil import to_bytes +import win32file, win32event + +# twisted imports +from twisted.internet import abstract + +# sibling imports +from twisted.internet.serialport import BaseSerialPort + + +class SerialPort(BaseSerialPort, abstract.FileDescriptor): + """A serial device, acting as a transport, that uses a win32 event.""" + + connected = 1 + + def __init__(self, protocol, deviceNameOrPortNumber, reactor, + baudrate = 9600, bytesize = EIGHTBITS, parity = PARITY_NONE, + stopbits = STOPBITS_ONE, xonxoff = 0, rtscts = 0): + self._serial = self._serialFactory( + deviceNameOrPortNumber, baudrate=baudrate, bytesize=bytesize, + parity=parity, stopbits=stopbits, timeout=None, + xonxoff=xonxoff, rtscts=rtscts) + self.flushInput() + self.flushOutput() + self.reactor = reactor + self.protocol = protocol + self.outQueue = [] + self.closed = 0 + self.closedNotifies = 0 + self.writeInProgress = 0 + + self.protocol = protocol + self._overlappedRead = win32file.OVERLAPPED() + self._overlappedRead.hEvent = win32event.CreateEvent(None, 1, 0, None) + self._overlappedWrite = win32file.OVERLAPPED() + self._overlappedWrite.hEvent = win32event.CreateEvent(None, 0, 0, None) + + self.reactor.addEvent(self._overlappedRead.hEvent, self, 'serialReadEvent') + self.reactor.addEvent(self._overlappedWrite.hEvent, self, 'serialWriteEvent') + + self.protocol.makeConnection(self) + self._finishPortSetup() + + + def _finishPortSetup(self): + """ + Finish setting up the serial port. + + This is a separate method to facilitate testing. + """ + flags, comstat = self._clearCommError() + rc, self.read_buf = win32file.ReadFile(self._serial._port_handle, + win32file.AllocateReadBuffer(1), + self._overlappedRead) + + + def _clearCommError(self): + return win32file.ClearCommError(self._serial._port_handle) + + + def serialReadEvent(self): + #get that character we set up + n = win32file.GetOverlappedResult(self._serial._port_handle, self._overlappedRead, 0) + first = to_bytes(self.read_buf[:n]) + #now we should get everything that is already in the buffer + flags, comstat = self._clearCommError() + if comstat.cbInQue: + win32event.ResetEvent(self._overlappedRead.hEvent) + rc, buf = win32file.ReadFile(self._serial._port_handle, + win32file.AllocateReadBuffer(comstat.cbInQue), + self._overlappedRead) + n = win32file.GetOverlappedResult(self._serial._port_handle, self._overlappedRead, 1) + #handle all the received data: + self.protocol.dataReceived(first + to_bytes(buf[:n])) + else: + #handle all the received data: + self.protocol.dataReceived(first) + + #set up next one + win32event.ResetEvent(self._overlappedRead.hEvent) + rc, self.read_buf = win32file.ReadFile(self._serial._port_handle, + win32file.AllocateReadBuffer(1), + self._overlappedRead) + + + def write(self, data): + if data: + if self.writeInProgress: + self.outQueue.append(data) + else: + self.writeInProgress = 1 + win32file.WriteFile(self._serial._port_handle, data, self._overlappedWrite) + + + def serialWriteEvent(self): + try: + dataToWrite = self.outQueue.pop(0) + except IndexError: + self.writeInProgress = 0 + return + else: + win32file.WriteFile(self._serial._port_handle, dataToWrite, self._overlappedWrite) + + + def connectionLost(self, reason): + """ + Called when the serial port disconnects. + + Will call C{connectionLost} on the protocol that is handling the + serial data. + """ + self.reactor.removeEvent(self._overlappedRead.hEvent) + self.reactor.removeEvent(self._overlappedWrite.hEvent) + abstract.FileDescriptor.connectionLost(self, reason) + self._serial.close() + self.protocol.connectionLost(reason) diff --git a/contrib/python/Twisted/py2/twisted/internet/_win32stdio.py b/contrib/python/Twisted/py2/twisted/internet/_win32stdio.py new file mode 100644 index 00000000000..6ce3aadda25 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/_win32stdio.py @@ -0,0 +1,133 @@ +# -*- test-case-name: twisted.test.test_stdio -*- + +""" +Windows-specific implementation of the L{twisted.internet.stdio} interface. +""" + +from __future__ import absolute_import, division + +import win32api +import os +import msvcrt + +from zope.interface import implementer + +from twisted.internet.interfaces import (IHalfCloseableProtocol, ITransport, + IConsumer, IPushProducer, IAddress) + +from twisted.internet import _pollingfile, main +from twisted.python.failure import Failure + +@implementer(IAddress) +class Win32PipeAddress(object): + pass + + +@implementer(ITransport, IConsumer, IPushProducer) +class StandardIO(_pollingfile._PollingTimer): + + disconnecting = False + disconnected = False + + def __init__(self, proto, reactor=None): + """ + Start talking to standard IO with the given protocol. + + Also, put it stdin/stdout/stderr into binary mode. + """ + if reactor is None: + from twisted.internet import reactor + + for stdfd in range(0, 1, 2): + msvcrt.setmode(stdfd, os.O_BINARY) + + _pollingfile._PollingTimer.__init__(self, reactor) + self.proto = proto + + hstdin = win32api.GetStdHandle(win32api.STD_INPUT_HANDLE) + hstdout = win32api.GetStdHandle(win32api.STD_OUTPUT_HANDLE) + + self.stdin = _pollingfile._PollableReadPipe( + hstdin, self.dataReceived, self.readConnectionLost) + + self.stdout = _pollingfile._PollableWritePipe( + hstdout, self.writeConnectionLost) + + self._addPollableResource(self.stdin) + self._addPollableResource(self.stdout) + + self.proto.makeConnection(self) + + + def dataReceived(self, data): + self.proto.dataReceived(data) + + + def readConnectionLost(self): + if IHalfCloseableProtocol.providedBy(self.proto): + self.proto.readConnectionLost() + self.checkConnLost() + + + def writeConnectionLost(self): + if IHalfCloseableProtocol.providedBy(self.proto): + self.proto.writeConnectionLost() + self.checkConnLost() + + connsLost = 0 + + + def checkConnLost(self): + self.connsLost += 1 + if self.connsLost >= 2: + self.disconnecting = True + self.disconnected = True + self.proto.connectionLost(Failure(main.CONNECTION_DONE)) + + # ITransport + + def write(self, data): + self.stdout.write(data) + + + def writeSequence(self, seq): + self.stdout.write(b''.join(seq)) + + + def loseConnection(self): + self.disconnecting = True + self.stdin.close() + self.stdout.close() + + + def getPeer(self): + return Win32PipeAddress() + + + def getHost(self): + return Win32PipeAddress() + + # IConsumer + + def registerProducer(self, producer, streaming): + return self.stdout.registerProducer(producer, streaming) + + + def unregisterProducer(self): + return self.stdout.unregisterProducer() + + # def write() above + + # IProducer + + def stopProducing(self): + self.stdin.stopProducing() + + # IPushProducer + + def pauseProducing(self): + self.stdin.pauseProducing() + + + def resumeProducing(self): + self.stdin.resumeProducing() diff --git a/contrib/python/Twisted/py2/twisted/internet/abstract.py b/contrib/python/Twisted/py2/twisted/internet/abstract.py new file mode 100644 index 00000000000..4b560dc4e6b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/abstract.py @@ -0,0 +1,546 @@ +# -*- test-case-name: twisted.test.test_abstract -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for generic select()able objects. +""" + +from __future__ import division, absolute_import + +from socket import AF_INET, AF_INET6, inet_pton, error + +from zope.interface import implementer + +# Twisted Imports +from twisted.python.compat import unicode, lazyByteSlice, _PY3 +from twisted.python import reflect, failure +from twisted.internet import interfaces, main + +if _PY3: + # Python 3.4+ can join bytes and memoryviews; using a + # memoryview prevents the slice from copying + def _concatenate(bObj, offset, bArray): + return b''.join([memoryview(bObj)[offset:]] + bArray) +else: + from __builtin__ import buffer + + def _concatenate(bObj, offset, bArray): + # Avoid one extra string copy by using a buffer to limit what + # we include in the result. + return buffer(bObj, offset) + b"".join(bArray) + + + +class _ConsumerMixin(object): + """ + L{IConsumer} implementations can mix this in to get C{registerProducer} and + C{unregisterProducer} methods which take care of keeping track of a + producer's state. + + Subclasses must provide three attributes which L{_ConsumerMixin} will read + but not write: + + - connected: A C{bool} which is C{True} as long as the consumer has + someplace to send bytes (for example, a TCP connection), and then + C{False} when it no longer does. + + - disconnecting: A C{bool} which is C{False} until something like + L{ITransport.loseConnection} is called, indicating that the send buffer + should be flushed and the connection lost afterwards. Afterwards, + C{True}. + + - disconnected: A C{bool} which is C{False} until the consumer no longer + has a place to send bytes, then C{True}. + + Subclasses must also override the C{startWriting} method. + + @ivar producer: L{None} if no producer is registered, otherwise the + registered producer. + + @ivar producerPaused: A flag indicating whether the producer is currently + paused. + @type producerPaused: L{bool} + + @ivar streamingProducer: A flag indicating whether the producer was + registered as a streaming (ie push) producer or not (ie a pull + producer). This will determine whether the consumer may ever need to + pause and resume it, or if it can merely call C{resumeProducing} on it + when buffer space is available. + @ivar streamingProducer: C{bool} or C{int} + + """ + producer = None + producerPaused = False + streamingProducer = False + + def startWriting(self): + """ + Override in a subclass to cause the reactor to monitor this selectable + for write events. This will be called once in C{unregisterProducer} if + C{loseConnection} has previously been called, so that the connection can + actually close. + """ + raise NotImplementedError("%r did not implement startWriting") + + + def registerProducer(self, producer, streaming): + """ + Register to receive data from a producer. + + This sets this selectable to be a consumer for a producer. When this + selectable runs out of data on a write() call, it will ask the producer + to resumeProducing(). When the FileDescriptor's internal data buffer is + filled, it will ask the producer to pauseProducing(). If the connection + is lost, FileDescriptor calls producer's stopProducing() method. + + If streaming is true, the producer should provide the IPushProducer + interface. Otherwise, it is assumed that producer provides the + IPullProducer interface. In this case, the producer won't be asked to + pauseProducing(), but it has to be careful to write() data only when its + resumeProducing() method is called. + """ + if self.producer is not None: + raise RuntimeError( + "Cannot register producer %s, because producer %s was never " + "unregistered." % (producer, self.producer)) + if self.disconnected: + producer.stopProducing() + else: + self.producer = producer + self.streamingProducer = streaming + if not streaming: + producer.resumeProducing() + + + def unregisterProducer(self): + """ + Stop consuming data from a producer, without disconnecting. + """ + self.producer = None + if self.connected and self.disconnecting: + self.startWriting() + + + +@implementer(interfaces.ILoggingContext) +class _LogOwner(object): + """ + Mixin to help implement L{interfaces.ILoggingContext} for transports which + have a protocol, the log prefix of which should also appear in the + transport's log prefix. + """ + + def _getLogPrefix(self, applicationObject): + """ + Determine the log prefix to use for messages related to + C{applicationObject}, which may or may not be an + L{interfaces.ILoggingContext} provider. + + @return: A C{str} giving the log prefix to use. + """ + if interfaces.ILoggingContext.providedBy(applicationObject): + return applicationObject.logPrefix() + return applicationObject.__class__.__name__ + + + def logPrefix(self): + """ + Override this method to insert custom logging behavior. Its + return value will be inserted in front of every line. It may + be called more times than the number of output lines. + """ + return "-" + + + +@implementer( + interfaces.IPushProducer, interfaces.IReadWriteDescriptor, + interfaces.IConsumer, interfaces.ITransport, + interfaces.IHalfCloseableDescriptor) +class FileDescriptor(_ConsumerMixin, _LogOwner): + """ + An object which can be operated on by select(). + + This is an abstract superclass of all objects which may be notified when + they are readable or writable; e.g. they have a file-descriptor that is + valid to be passed to select(2). + """ + connected = 0 + disconnected = 0 + disconnecting = 0 + _writeDisconnecting = False + _writeDisconnected = False + dataBuffer = b"" + offset = 0 + + SEND_LIMIT = 128*1024 + + def __init__(self, reactor=None): + """ + @param reactor: An L{IReactorFDSet} provider which this descriptor will + use to get readable and writeable event notifications. If no value + is given, the global reactor will be used. + """ + if not reactor: + from twisted.internet import reactor + self.reactor = reactor + self._tempDataBuffer = [] # will be added to dataBuffer in doWrite + self._tempDataLen = 0 + + + def connectionLost(self, reason): + """The connection was lost. + + This is called when the connection on a selectable object has been + lost. It will be called whether the connection was closed explicitly, + an exception occurred in an event handler, or the other end of the + connection closed it first. + + Clean up state here, but make sure to call back up to FileDescriptor. + """ + self.disconnected = 1 + self.connected = 0 + if self.producer is not None: + self.producer.stopProducing() + self.producer = None + self.stopReading() + self.stopWriting() + + + def writeSomeData(self, data): + """ + Write as much as possible of the given data, immediately. + + This is called to invoke the lower-level writing functionality, such + as a socket's send() method, or a file's write(); this method + returns an integer or an exception. If an integer, it is the number + of bytes written (possibly zero); if an exception, it indicates the + connection was lost. + """ + raise NotImplementedError("%s does not implement writeSomeData" % + reflect.qual(self.__class__)) + + + def doRead(self): + """ + Called when data is available for reading. + + Subclasses must override this method. The result will be interpreted + in the same way as a result of doWrite(). + """ + raise NotImplementedError("%s does not implement doRead" % + reflect.qual(self.__class__)) + + def doWrite(self): + """ + Called when data can be written. + + @return: L{None} on success, an exception or a negative integer on + failure. + + @see: L{twisted.internet.interfaces.IWriteDescriptor.doWrite}. + """ + if len(self.dataBuffer) - self.offset < self.SEND_LIMIT: + # If there is currently less than SEND_LIMIT bytes left to send + # in the string, extend it with the array data. + self.dataBuffer = _concatenate( + self.dataBuffer, self.offset, self._tempDataBuffer) + self.offset = 0 + self._tempDataBuffer = [] + self._tempDataLen = 0 + + # Send as much data as you can. + if self.offset: + l = self.writeSomeData(lazyByteSlice(self.dataBuffer, self.offset)) + else: + l = self.writeSomeData(self.dataBuffer) + + # There is no writeSomeData implementation in Twisted which returns + # < 0, but the documentation for writeSomeData used to claim negative + # integers meant connection lost. Keep supporting this here, + # although it may be worth deprecating and removing at some point. + if isinstance(l, Exception) or l < 0: + return l + self.offset += l + # If there is nothing left to send, + if self.offset == len(self.dataBuffer) and not self._tempDataLen: + self.dataBuffer = b"" + self.offset = 0 + # stop writing. + self.stopWriting() + # If I've got a producer who is supposed to supply me with data, + if self.producer is not None and ((not self.streamingProducer) + or self.producerPaused): + # tell them to supply some more. + self.producerPaused = False + self.producer.resumeProducing() + elif self.disconnecting: + # But if I was previously asked to let the connection die, do + # so. + return self._postLoseConnection() + elif self._writeDisconnecting: + # I was previously asked to half-close the connection. We + # set _writeDisconnected before calling handler, in case the + # handler calls loseConnection(), which will want to check for + # this attribute. + self._writeDisconnected = True + result = self._closeWriteConnection() + return result + return None + + def _postLoseConnection(self): + """Called after a loseConnection(), when all data has been written. + + Whatever this returns is then returned by doWrite. + """ + # default implementation, telling reactor we're finished + return main.CONNECTION_DONE + + def _closeWriteConnection(self): + # override in subclasses + pass + + def writeConnectionLost(self, reason): + # in current code should never be called + self.connectionLost(reason) + + def readConnectionLost(self, reason): + # override in subclasses + self.connectionLost(reason) + + + def _isSendBufferFull(self): + """ + Determine whether the user-space send buffer for this transport is full + or not. + + When the buffer contains more than C{self.bufferSize} bytes, it is + considered full. This might be improved by considering the size of the + kernel send buffer and how much of it is free. + + @return: C{True} if it is full, C{False} otherwise. + """ + return len(self.dataBuffer) + self._tempDataLen > self.bufferSize + + + def _maybePauseProducer(self): + """ + Possibly pause a producer, if there is one and the send buffer is full. + """ + # If we are responsible for pausing our producer, + if self.producer is not None and self.streamingProducer: + # and our buffer is full, + if self._isSendBufferFull(): + # pause it. + self.producerPaused = True + self.producer.pauseProducing() + + + def write(self, data): + """Reliably write some data. + + The data is buffered until the underlying file descriptor is ready + for writing. If there is more than C{self.bufferSize} data in the + buffer and this descriptor has a registered streaming producer, its + C{pauseProducing()} method will be called. + """ + if isinstance(data, unicode): # no, really, I mean it + raise TypeError("Data must not be unicode") + if not self.connected or self._writeDisconnected: + return + if data: + self._tempDataBuffer.append(data) + self._tempDataLen += len(data) + self._maybePauseProducer() + self.startWriting() + + + def writeSequence(self, iovec): + """ + Reliably write a sequence of data. + + Currently, this is a convenience method roughly equivalent to:: + + for chunk in iovec: + fd.write(chunk) + + It may have a more efficient implementation at a later time or in a + different reactor. + + As with the C{write()} method, if a buffer size limit is reached and a + streaming producer is registered, it will be paused until the buffered + data is written to the underlying file descriptor. + """ + for i in iovec: + if isinstance(i, unicode): # no, really, I mean it + raise TypeError("Data must not be unicode") + if not self.connected or not iovec or self._writeDisconnected: + return + self._tempDataBuffer.extend(iovec) + for i in iovec: + self._tempDataLen += len(i) + self._maybePauseProducer() + self.startWriting() + + + def loseConnection(self, _connDone=failure.Failure(main.CONNECTION_DONE)): + """Close the connection at the next available opportunity. + + Call this to cause this FileDescriptor to lose its connection. It will + first write any data that it has buffered. + + If there is data buffered yet to be written, this method will cause the + transport to lose its connection as soon as it's done flushing its + write buffer. If you have a producer registered, the connection won't + be closed until the producer is finished. Therefore, make sure you + unregister your producer when it's finished, or the connection will + never close. + """ + + if self.connected and not self.disconnecting: + if self._writeDisconnected: + # doWrite won't trigger the connection close anymore + self.stopReading() + self.stopWriting() + self.connectionLost(_connDone) + else: + self.stopReading() + self.startWriting() + self.disconnecting = 1 + + def loseWriteConnection(self): + self._writeDisconnecting = True + self.startWriting() + + def stopReading(self): + """Stop waiting for read availability. + + Call this to remove this selectable from being notified when it is + ready for reading. + """ + self.reactor.removeReader(self) + + def stopWriting(self): + """Stop waiting for write availability. + + Call this to remove this selectable from being notified when it is ready + for writing. + """ + self.reactor.removeWriter(self) + + def startReading(self): + """Start waiting for read availability. + """ + self.reactor.addReader(self) + + def startWriting(self): + """Start waiting for write availability. + + Call this to have this FileDescriptor be notified whenever it is ready for + writing. + """ + self.reactor.addWriter(self) + + # Producer/consumer implementation + + # first, the consumer stuff. This requires no additional work, as + # any object you can write to can be a consumer, really. + + producer = None + bufferSize = 2**2**2**2 + + def stopConsuming(self): + """Stop consuming data. + + This is called when a producer has lost its connection, to tell the + consumer to go lose its connection (and break potential circular + references). + """ + self.unregisterProducer() + self.loseConnection() + + # producer interface implementation + + def resumeProducing(self): + if self.connected and not self.disconnecting: + self.startReading() + + def pauseProducing(self): + self.stopReading() + + def stopProducing(self): + self.loseConnection() + + + def fileno(self): + """File Descriptor number for select(). + + This method must be overridden or assigned in subclasses to + indicate a valid file descriptor for the operating system. + """ + return -1 + + + +def isIPAddress(addr, family=AF_INET): + """ + Determine whether the given string represents an IP address of the given + family; by default, an IPv4 address. + + @type addr: C{str} + @param addr: A string which may or may not be the decimal dotted + representation of an IPv4 address. + + @param family: The address family to test for; one of the C{AF_*} constants + from the L{socket} module. (This parameter has only been available + since Twisted 17.1.0; previously L{isIPAddress} could only test for IPv4 + addresses.) + @type family: C{int} + + @rtype: C{bool} + @return: C{True} if C{addr} represents an IPv4 address, C{False} otherwise. + """ + if isinstance(addr, bytes): + try: + addr = addr.decode("ascii") + except UnicodeDecodeError: + return False + if family == AF_INET6: + # On some platforms, inet_ntop fails unless the scope ID is valid; this + # is a test for whether the given string *is* an IP address, so strip + # any potential scope ID before checking. + addr = addr.split(u"%", 1)[0] + elif family == AF_INET: + # On Windows, where 3.5+ implement inet_pton, "0" is considered a valid + # IPv4 address, but we want to ensure we have all 4 segments. + if addr.count(u".") != 3: + return False + else: + raise ValueError("unknown address family {!r}".format(family)) + try: + # This might be a native implementation or the one from + # twisted.python.compat. + inet_pton(family, addr) + except (ValueError, error): + return False + return True + + + +def isIPv6Address(addr): + """ + Determine whether the given string represents an IPv6 address. + + @param addr: A string which may or may not be the hex + representation of an IPv6 address. + @type addr: C{str} + + @return: C{True} if C{addr} represents an IPv6 address, C{False} + otherwise. + @rtype: C{bool} + """ + return isIPAddress(addr, AF_INET6) + + +__all__ = ["FileDescriptor", "isIPAddress", "isIPv6Address"] diff --git a/contrib/python/Twisted/py2/twisted/internet/address.py b/contrib/python/Twisted/py2/twisted/internet/address.py new file mode 100644 index 00000000000..609e6e8e34f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/address.py @@ -0,0 +1,180 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Address objects for network connections. +""" + +from __future__ import division, absolute_import + +import attr +import warnings, os + +from zope.interface import implementer +from twisted.internet.interfaces import IAddress +from twisted.python.filepath import _asFilesystemBytes +from twisted.python.filepath import _coerceToFilesystemEncoding +from twisted.python.runtime import platform +from twisted.python.compat import _PY3 + + +@implementer(IAddress) +@attr.s(hash=True) +class IPv4Address(object): + """ + An L{IPv4Address} represents the address of an IPv4 socket endpoint. + + @ivar type: A string describing the type of transport, either 'TCP' or + 'UDP'. + + @ivar host: A string containing a dotted-quad IPv4 address; for example, + "127.0.0.1". + @type host: C{str} + + @ivar port: An integer representing the port number. + @type port: C{int} + """ + type = attr.ib(validator=attr.validators.in_(["TCP", "UDP"])) + host = attr.ib() + port = attr.ib() + + + +@implementer(IAddress) +@attr.s(hash=True) +class IPv6Address(object): + """ + An L{IPv6Address} represents the address of an IPv6 socket endpoint. + + @ivar type: A string describing the type of transport, either 'TCP' or + 'UDP'. + + @ivar host: A string containing a colon-separated, hexadecimal formatted + IPv6 address; for example, "::1". + @type host: C{str} + + @ivar port: An integer representing the port number. + @type port: C{int} + + @ivar flowInfo: the IPv6 flow label. This can be used by QoS routers to + identify flows of traffic; you may generally safely ignore it. + @type flowInfo: L{int} + + @ivar scopeID: the IPv6 scope identifier - roughly analagous to what + interface traffic destined for this address must be transmitted over. + @type scopeID: L{int} or L{str} + """ + type = attr.ib(validator=attr.validators.in_(["TCP", "UDP"])) + host = attr.ib() + port = attr.ib() + flowInfo = attr.ib(default=0) + scopeID = attr.ib(default=0) + + + +@implementer(IAddress) +class _ProcessAddress(object): + """ + An L{interfaces.IAddress} provider for process transports. + """ + + + +@attr.s(hash=True) +@implementer(IAddress) +class HostnameAddress(object): + """ + A L{HostnameAddress} represents the address of a L{HostnameEndpoint}. + + @ivar hostname: A hostname byte string; for example, b"example.com". + @type hostname: L{bytes} + + @ivar port: An integer representing the port number. + @type port: L{int} + """ + + hostname = attr.ib() + port = attr.ib() + + + +@attr.s(hash=False, repr=False, eq=False) +@implementer(IAddress) +class UNIXAddress(object): + """ + Object representing a UNIX socket endpoint. + + @ivar name: The filename associated with this socket. + @type name: C{bytes} + """ + + name = attr.ib(converter=attr.converters.optional(_asFilesystemBytes)) + + if getattr(os.path, 'samefile', None) is not None: + def __eq__(self, other): + """ + Overriding C{attrs} to ensure the os level samefile + check is done if the name attributes do not match. + """ + if isinstance(other, self.__class__): + res = self.name == other.name + else: + return False + if not res and self.name and other.name: + try: + return os.path.samefile(self.name, other.name) + except OSError: + pass + except (TypeError, ValueError) as e: + # On Linux, abstract namespace UNIX sockets start with a + # \0, which os.path doesn't like. + if not _PY3 and not platform.isLinux(): + raise e + return res + else: + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.name == other.name + return False + + + def __ne__(self, other): + if isinstance(other, self.__class__): + return not self.__eq__(other) + return True + + + def __repr__(self): + name = self.name + if name: + name = _coerceToFilesystemEncoding('', self.name) + return 'UNIXAddress(%r)' % (name,) + + + def __hash__(self): + if self.name is None: + return hash((self.__class__, None)) + try: + s1 = os.stat(self.name) + return hash((s1.st_ino, s1.st_dev)) + except OSError: + return hash(self.name) + + + +# These are for buildFactory backwards compatibility due to +# stupidity-induced inconsistency. + +class _ServerFactoryIPv4Address(IPv4Address): + """Backwards compatibility hack. Just like IPv4Address in practice.""" + + def __eq__(self, other): + if isinstance(other, tuple): + warnings.warn("IPv4Address.__getitem__ is deprecated. Use attributes instead.", + category=DeprecationWarning, stacklevel=2) + return (self.host, self.port) == other + elif isinstance(other, IPv4Address): + a = (self.type, self.host, self.port) + b = (other.type, other.host, other.port) + return a == b + return False diff --git a/contrib/python/Twisted/py2/twisted/internet/asyncioreactor.py b/contrib/python/Twisted/py2/twisted/internet/asyncioreactor.py new file mode 100644 index 00000000000..a2896fb6851 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/asyncioreactor.py @@ -0,0 +1,322 @@ +# -*- test-case-name: twisted.test.test_internet -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +asyncio-based reactor implementation. +""" + +from __future__ import absolute_import, division + +import errno + +from zope.interface import implementer + +from twisted.logger import Logger +from twisted.internet.base import DelayedCall +from twisted.internet.posixbase import (PosixReactorBase, _NO_FILEDESC, + _ContinuousPolling) +from twisted.python.log import callWithLogger +from twisted.internet.interfaces import IReactorFDSet + +try: + from asyncio import get_event_loop +except ImportError: + raise ImportError("Requires asyncio.") + +# As per ImportError above, this module is never imported on python 2, but +# pyflakes still runs on python 2, so let's tell it where the errors come from. +from builtins import PermissionError, BrokenPipeError + + +class _DCHandle(object): + """ + Wraps ephemeral L{asyncio.Handle} instances. Callbacks can close + over this and use it as a mutable reference to asyncio C{Handles}. + + @ivar handle: The current L{asyncio.Handle} + """ + def __init__(self, handle): + self.handle = handle + + + def cancel(self): + """ + Cancel the inner L{asyncio.Handle}. + """ + self.handle.cancel() + + + +@implementer(IReactorFDSet) +class AsyncioSelectorReactor(PosixReactorBase): + """ + Reactor running on top of L{asyncio.SelectorEventLoop}. + """ + _asyncClosed = False + _log = Logger() + + def __init__(self, eventloop=None): + + if eventloop is None: + eventloop = get_event_loop() + + self._asyncioEventloop = eventloop + self._writers = {} + self._readers = {} + self._delayedCalls = set() + self._continuousPolling = _ContinuousPolling(self) + super().__init__() + + + def _unregisterFDInAsyncio(self, fd): + """ + Compensate for a bug in asyncio where it will not unregister a FD that + it cannot handle in the epoll loop. It touches internal asyncio code. + + A description of the bug by markrwilliams: + + The C{add_writer} method of asyncio event loops isn't atomic because + all the Selector classes in the selector module internally record a + file object before passing it to the platform's selector + implementation. If the platform's selector decides the file object + isn't acceptable, the resulting exception doesn't cause the Selector to + un-track the file object. + + The failing/hanging stdio test goes through the following sequence of + events (roughly): + + * The first C{connection.write(intToByte(value))} call hits the asyncio + reactor's C{addWriter} method. + + * C{addWriter} calls the asyncio loop's C{add_writer} method, which + happens to live on C{_BaseSelectorEventLoop}. + + * The asyncio loop's C{add_writer} method checks if the file object has + been registered before via the selector's C{get_key} method. + + * It hasn't, so the KeyError block runs and calls the selector's + register method + + * Code examples that follow use EpollSelector, but the code flow holds + true for any other selector implementation. The selector's register + method first calls through to the next register method in the MRO + + * That next method is always C{_BaseSelectorImpl.register} which + creates a C{SelectorKey} instance for the file object, stores it under + the file object's file descriptor, and then returns it. + + * Control returns to the concrete selector implementation, which asks + the operating system to track the file descriptor using the right API. + + * The operating system refuses! An exception is raised that, in this + case, the asyncio reactor handles by creating a C{_ContinuousPolling} + object to watch the file descriptor. + + * The second C{connection.write(intToByte(value))} call hits the + asyncio reactor's C{addWriter} method, which hits the C{add_writer} + method. But the loop's selector's get_key method now returns a + C{SelectorKey}! Now the asyncio reactor's C{addWriter} method thinks + the asyncio loop will watch the file descriptor, even though it won't. + """ + try: + self._asyncioEventloop._selector.unregister(fd) + except: + pass + + + def _readOrWrite(self, selectable, read): + method = selectable.doRead if read else selectable.doWrite + + if selectable.fileno() == -1: + self._disconnectSelectable(selectable, _NO_FILEDESC, read) + return + + try: + why = method() + except Exception as e: + why = e + self._log.failure(None) + if why: + self._disconnectSelectable(selectable, why, read) + + + def addReader(self, reader): + if reader in self._readers.keys() or \ + reader in self._continuousPolling._readers: + return + + fd = reader.fileno() + try: + self._asyncioEventloop.add_reader(fd, callWithLogger, reader, + self._readOrWrite, reader, + True) + self._readers[reader] = fd + except IOError as e: + self._unregisterFDInAsyncio(fd) + if e.errno == errno.EPERM: + # epoll(7) doesn't support certain file descriptors, + # e.g. filesystem files, so for those we just poll + # continuously: + self._continuousPolling.addReader(reader) + else: + raise + + + def addWriter(self, writer): + if writer in self._writers.keys() or \ + writer in self._continuousPolling._writers: + return + + fd = writer.fileno() + try: + self._asyncioEventloop.add_writer(fd, callWithLogger, writer, + self._readOrWrite, writer, + False) + self._writers[writer] = fd + except PermissionError: + self._unregisterFDInAsyncio(fd) + # epoll(7) doesn't support certain file descriptors, + # e.g. filesystem files, so for those we just poll + # continuously: + self._continuousPolling.addWriter(writer) + except BrokenPipeError: + # The kqueuereactor will raise this if there is a broken pipe + self._unregisterFDInAsyncio(fd) + except: + self._unregisterFDInAsyncio(fd) + raise + + + def removeReader(self, reader): + + # First, see if they're trying to remove a reader that we don't have. + if not (reader in self._readers.keys() \ + or self._continuousPolling.isReading(reader)): + # We don't have it, so just return OK. + return + + # If it was a cont. polling reader, check there first. + if self._continuousPolling.isReading(reader): + self._continuousPolling.removeReader(reader) + return + + fd = reader.fileno() + if fd == -1: + # If the FD is -1, we want to know what its original FD was, to + # remove it. + fd = self._readers.pop(reader) + else: + self._readers.pop(reader) + + self._asyncioEventloop.remove_reader(fd) + + + def removeWriter(self, writer): + + # First, see if they're trying to remove a writer that we don't have. + if not (writer in self._writers.keys() \ + or self._continuousPolling.isWriting(writer)): + # We don't have it, so just return OK. + return + + # If it was a cont. polling writer, check there first. + if self._continuousPolling.isWriting(writer): + self._continuousPolling.removeWriter(writer) + return + + fd = writer.fileno() + + if fd == -1: + # If the FD is -1, we want to know what its original FD was, to + # remove it. + fd = self._writers.pop(writer) + else: + self._writers.pop(writer) + + self._asyncioEventloop.remove_writer(fd) + + + def removeAll(self): + return (self._removeAll(self._readers.keys(), self._writers.keys()) + + self._continuousPolling.removeAll()) + + + def getReaders(self): + return (list(self._readers.keys()) + + self._continuousPolling.getReaders()) + + + def getWriters(self): + return (list(self._writers.keys()) + + self._continuousPolling.getWriters()) + + + def getDelayedCalls(self): + return list(self._delayedCalls) + + + def iterate(self, timeout): + self._asyncioEventloop.call_later(timeout + 0.01, + self._asyncioEventloop.stop) + self._asyncioEventloop.run_forever() + + + def run(self, installSignalHandlers=True): + self.startRunning(installSignalHandlers=installSignalHandlers) + self._asyncioEventloop.run_forever() + if self._justStopped: + self._justStopped = False + + + def stop(self): + super().stop() + self.callLater(0, self.fireSystemEvent, "shutdown") + + + def crash(self): + super().crash() + self._asyncioEventloop.stop() + + + def seconds(self): + return self._asyncioEventloop.time() + + + def callLater(self, seconds, f, *args, **kwargs): + def run(): + dc.called = True + self._delayedCalls.remove(dc) + f(*args, **kwargs) + handle = self._asyncioEventloop.call_later(seconds, run) + dchandle = _DCHandle(handle) + + def cancel(dc): + self._delayedCalls.remove(dc) + dchandle.cancel() + + def reset(dc): + dchandle.handle = self._asyncioEventloop.call_at(dc.time, run) + + dc = DelayedCall(self.seconds() + seconds, run, (), {}, + cancel, reset, seconds=self.seconds) + self._delayedCalls.add(dc) + return dc + + + def callFromThread(self, f, *args, **kwargs): + g = lambda: self.callLater(0, f, *args, **kwargs) + self._asyncioEventloop.call_soon_threadsafe(g) + + + +def install(eventloop=None): + """ + Install an asyncio-based reactor. + + @param eventloop: The asyncio eventloop to wrap. If default, the global one + is selected. + """ + reactor = AsyncioSelectorReactor(eventloop) + from twisted.internet.main import installReactor + installReactor(reactor) diff --git a/contrib/python/Twisted/py2/twisted/internet/base.py b/contrib/python/Twisted/py2/twisted/internet/base.py new file mode 100644 index 00000000000..cfde7c2a11e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/base.py @@ -0,0 +1,1304 @@ +# -*- test-case-name: twisted.test.test_internet,twisted.internet.test.test_core -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Very basic functionality for a Reactor implementation. +""" + +from __future__ import division, absolute_import + +import socket # needed only for sync-dns +from zope.interface import implementer, classImplements + +import sys +import warnings +from heapq import heappush, heappop, heapify + +import traceback + +from twisted.internet.interfaces import ( + IReactorCore, IReactorTime, IReactorThreads, IResolverSimple, + IReactorPluggableResolver, IReactorPluggableNameResolver, IConnector, + IDelayedCall, _ISupportsExitSignalCapturing +) + +from twisted.internet import fdesc, main, error, abstract, defer, threads +from twisted.internet._resolver import ( + GAIResolver as _GAIResolver, + ComplexResolverSimplifier as _ComplexResolverSimplifier, + SimpleResolverComplexifier as _SimpleResolverComplexifier, +) +from twisted.python import log, failure, reflect +from twisted.python.compat import unicode, iteritems +from twisted.python.runtime import seconds as runtimeSeconds, platform +from twisted.internet.defer import Deferred, DeferredList +from twisted.python._oldstyle import _oldStyle + +# This import is for side-effects! Even if you don't see any code using it +# in this module, don't delete it. +from twisted.python import threadable + + +@implementer(IDelayedCall) +@_oldStyle +class DelayedCall: + + # enable .debug to record creator call stack, and it will be logged if + # an exception occurs while the function is being run + debug = False + _repr = None + + def __init__(self, time, func, args, kw, cancel, reset, + seconds=runtimeSeconds): + """ + @param time: Seconds from the epoch at which to call C{func}. + @param func: The callable to call. + @param args: The positional arguments to pass to the callable. + @param kw: The keyword arguments to pass to the callable. + @param cancel: A callable which will be called with this + DelayedCall before cancellation. + @param reset: A callable which will be called with this + DelayedCall after changing this DelayedCall's scheduled + execution time. The callable should adjust any necessary + scheduling details to ensure this DelayedCall is invoked + at the new appropriate time. + @param seconds: If provided, a no-argument callable which will be + used to determine the current time any time that information is + needed. + """ + self.time, self.func, self.args, self.kw = time, func, args, kw + self.resetter = reset + self.canceller = cancel + self.seconds = seconds + self.cancelled = self.called = 0 + self.delayed_time = 0 + if self.debug: + self.creator = traceback.format_stack()[:-2] + + def getTime(self): + """Return the time at which this call will fire + + @rtype: C{float} + @return: The number of seconds after the epoch at which this call is + scheduled to be made. + """ + return self.time + self.delayed_time + + def cancel(self): + """Unschedule this call + + @raise AlreadyCancelled: Raised if this call has already been + unscheduled. + + @raise AlreadyCalled: Raised if this call has already been made. + """ + if self.cancelled: + raise error.AlreadyCancelled + elif self.called: + raise error.AlreadyCalled + else: + self.canceller(self) + self.cancelled = 1 + if self.debug: + self._repr = repr(self) + del self.func, self.args, self.kw + + def reset(self, secondsFromNow): + """Reschedule this call for a different time + + @type secondsFromNow: C{float} + @param secondsFromNow: The number of seconds from the time of the + C{reset} call at which this call will be scheduled. + + @raise AlreadyCancelled: Raised if this call has been cancelled. + @raise AlreadyCalled: Raised if this call has already been made. + """ + if self.cancelled: + raise error.AlreadyCancelled + elif self.called: + raise error.AlreadyCalled + else: + newTime = self.seconds() + secondsFromNow + if newTime < self.time: + self.delayed_time = 0 + self.time = newTime + self.resetter(self) + else: + self.delayed_time = newTime - self.time + + def delay(self, secondsLater): + """Reschedule this call for a later time + + @type secondsLater: C{float} + @param secondsLater: The number of seconds after the originally + scheduled time for which to reschedule this call. + + @raise AlreadyCancelled: Raised if this call has been cancelled. + @raise AlreadyCalled: Raised if this call has already been made. + """ + if self.cancelled: + raise error.AlreadyCancelled + elif self.called: + raise error.AlreadyCalled + else: + self.delayed_time += secondsLater + if self.delayed_time < 0: + self.activate_delay() + self.resetter(self) + + def activate_delay(self): + self.time += self.delayed_time + self.delayed_time = 0 + + def active(self): + """Determine whether this call is still pending + + @rtype: C{bool} + @return: True if this call has not yet been made or cancelled, + False otherwise. + """ + return not (self.cancelled or self.called) + + + def __le__(self, other): + """ + Implement C{<=} operator between two L{DelayedCall} instances. + + Comparison is based on the C{time} attribute (unadjusted by the + delayed time). + """ + return self.time <= other.time + + + def __lt__(self, other): + """ + Implement C{<} operator between two L{DelayedCall} instances. + + Comparison is based on the C{time} attribute (unadjusted by the + delayed time). + """ + return self.time < other.time + + + def __repr__(self): + """ + Implement C{repr()} for L{DelayedCall} instances. + + @rtype: C{str} + @returns: String containing details of the L{DelayedCall}. + """ + if self._repr is not None: + return self._repr + if hasattr(self, 'func'): + # This code should be replaced by a utility function in reflect; + # see ticket #6066: + if hasattr(self.func, '__qualname__'): + func = self.func.__qualname__ + elif hasattr(self.func, '__name__'): + func = self.func.func_name + if hasattr(self.func, 'im_class'): + func = self.func.im_class.__name__ + '.' + func + else: + func = reflect.safe_repr(self.func) + else: + func = None + + now = self.seconds() + L = ["') + + return "".join(L) + + + +@implementer(IResolverSimple) +class ThreadedResolver(object): + """ + L{ThreadedResolver} uses a reactor, a threadpool, and + L{socket.gethostbyname} to perform name lookups without blocking the + reactor thread. It also supports timeouts indepedently from whatever + timeout logic L{socket.gethostbyname} might have. + + @ivar reactor: The reactor the threadpool of which will be used to call + L{socket.gethostbyname} and the I/O thread of which the result will be + delivered. + """ + + def __init__(self, reactor): + self.reactor = reactor + self._runningQueries = {} + + + def _fail(self, name, err): + err = error.DNSLookupError("address %r not found: %s" % (name, err)) + return failure.Failure(err) + + + def _cleanup(self, name, lookupDeferred): + userDeferred, cancelCall = self._runningQueries[lookupDeferred] + del self._runningQueries[lookupDeferred] + userDeferred.errback(self._fail(name, "timeout error")) + + + def _checkTimeout(self, result, name, lookupDeferred): + try: + userDeferred, cancelCall = self._runningQueries[lookupDeferred] + except KeyError: + pass + else: + del self._runningQueries[lookupDeferred] + cancelCall.cancel() + + if isinstance(result, failure.Failure): + userDeferred.errback(self._fail(name, result.getErrorMessage())) + else: + userDeferred.callback(result) + + + def getHostByName(self, name, timeout = (1, 3, 11, 45)): + """ + See L{twisted.internet.interfaces.IResolverSimple.getHostByName}. + + Note that the elements of C{timeout} are summed and the result is used + as a timeout for the lookup. Any intermediate timeout or retry logic + is left up to the platform via L{socket.gethostbyname}. + """ + if timeout: + timeoutDelay = sum(timeout) + else: + timeoutDelay = 60 + userDeferred = defer.Deferred() + lookupDeferred = threads.deferToThreadPool( + self.reactor, self.reactor.getThreadPool(), + socket.gethostbyname, name) + cancelCall = self.reactor.callLater( + timeoutDelay, self._cleanup, name, lookupDeferred) + self._runningQueries[lookupDeferred] = (userDeferred, cancelCall) + lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred) + return userDeferred + + + +@implementer(IResolverSimple) +@_oldStyle +class BlockingResolver: + + def getHostByName(self, name, timeout = (1, 3, 11, 45)): + try: + address = socket.gethostbyname(name) + except socket.error: + msg = "address %r not found" % (name,) + err = error.DNSLookupError(msg) + return defer.fail(err) + else: + return defer.succeed(address) + + +class _ThreePhaseEvent(object): + """ + Collection of callables (with arguments) which can be invoked as a group in + a particular order. + + This provides the underlying implementation for the reactor's system event + triggers. An instance of this class tracks triggers for all phases of a + single type of event. + + @ivar before: A list of the before-phase triggers containing three-tuples + of a callable, a tuple of positional arguments, and a dict of keyword + arguments + + @ivar finishedBefore: A list of the before-phase triggers which have + already been executed. This is only populated in the C{'BEFORE'} state. + + @ivar during: A list of the during-phase triggers containing three-tuples + of a callable, a tuple of positional arguments, and a dict of keyword + arguments + + @ivar after: A list of the after-phase triggers containing three-tuples + of a callable, a tuple of positional arguments, and a dict of keyword + arguments + + @ivar state: A string indicating what is currently going on with this + object. One of C{'BASE'} (for when nothing in particular is happening; + this is the initial value), C{'BEFORE'} (when the before-phase triggers + are in the process of being executed). + """ + def __init__(self): + self.before = [] + self.during = [] + self.after = [] + self.state = 'BASE' + + + def addTrigger(self, phase, callable, *args, **kwargs): + """ + Add a trigger to the indicate phase. + + @param phase: One of C{'before'}, C{'during'}, or C{'after'}. + + @param callable: An object to be called when this event is triggered. + @param *args: Positional arguments to pass to C{callable}. + @param **kwargs: Keyword arguments to pass to C{callable}. + + @return: An opaque handle which may be passed to L{removeTrigger} to + reverse the effects of calling this method. + """ + if phase not in ('before', 'during', 'after'): + raise KeyError("invalid phase") + getattr(self, phase).append((callable, args, kwargs)) + return phase, callable, args, kwargs + + + def removeTrigger(self, handle): + """ + Remove a previously added trigger callable. + + @param handle: An object previously returned by L{addTrigger}. The + trigger added by that call will be removed. + + @raise ValueError: If the trigger associated with C{handle} has already + been removed or if C{handle} is not a valid handle. + """ + return getattr(self, 'removeTrigger_' + self.state)(handle) + + + def removeTrigger_BASE(self, handle): + """ + Just try to remove the trigger. + + @see: removeTrigger + """ + try: + phase, callable, args, kwargs = handle + except (TypeError, ValueError): + raise ValueError("invalid trigger handle") + else: + if phase not in ('before', 'during', 'after'): + raise KeyError("invalid phase") + getattr(self, phase).remove((callable, args, kwargs)) + + + def removeTrigger_BEFORE(self, handle): + """ + Remove the trigger if it has yet to be executed, otherwise emit a + warning that in the future an exception will be raised when removing an + already-executed trigger. + + @see: removeTrigger + """ + phase, callable, args, kwargs = handle + if phase != 'before': + return self.removeTrigger_BASE(handle) + if (callable, args, kwargs) in self.finishedBefore: + warnings.warn( + "Removing already-fired system event triggers will raise an " + "exception in a future version of Twisted.", + category=DeprecationWarning, + stacklevel=3) + else: + self.removeTrigger_BASE(handle) + + + def fireEvent(self): + """ + Call the triggers added to this event. + """ + self.state = 'BEFORE' + self.finishedBefore = [] + beforeResults = [] + while self.before: + callable, args, kwargs = self.before.pop(0) + self.finishedBefore.append((callable, args, kwargs)) + try: + result = callable(*args, **kwargs) + except: + log.err() + else: + if isinstance(result, Deferred): + beforeResults.append(result) + DeferredList(beforeResults).addCallback(self._continueFiring) + + + def _continueFiring(self, ignored): + """ + Call the during and after phase triggers for this event. + """ + self.state = 'BASE' + self.finishedBefore = [] + for phase in self.during, self.after: + while phase: + callable, args, kwargs = phase.pop(0) + try: + callable(*args, **kwargs) + except: + log.err() + + + +@implementer(IReactorPluggableNameResolver, IReactorPluggableResolver) +class PluggableResolverMixin(object): + """ + A mixin which implements the pluggable resolver reactor interfaces. + + @ivar resolver: The installed L{IResolverSimple}. + @ivar _nameResolver: The installed L{IHostnameResolver}. + """ + resolver = BlockingResolver() + _nameResolver = _SimpleResolverComplexifier(resolver) + + # IReactorPluggableResolver + def installResolver(self, resolver): + """ + See L{IReactorPluggableResolver}. + + @param resolver: see L{IReactorPluggableResolver}. + + @return: see L{IReactorPluggableResolver}. + """ + assert IResolverSimple.providedBy(resolver) + oldResolver = self.resolver + self.resolver = resolver + self._nameResolver = _SimpleResolverComplexifier(resolver) + return oldResolver + + + # IReactorPluggableNameResolver + def installNameResolver(self, resolver): + """ + See L{IReactorPluggableNameResolver}. + + @param resolver: See L{IReactorPluggableNameResolver}. + + @return: see L{IReactorPluggableNameResolver}. + """ + previousNameResolver = self._nameResolver + self._nameResolver = resolver + self.resolver = _ComplexResolverSimplifier(resolver) + return previousNameResolver + + + @property + def nameResolver(self): + """ + Implementation of read-only + L{IReactorPluggableNameResolver.nameResolver}. + """ + return self._nameResolver + + + +@implementer(IReactorCore, IReactorTime, _ISupportsExitSignalCapturing) +class ReactorBase(PluggableResolverMixin): + """ + Default base class for Reactors. + + @type _stopped: C{bool} + @ivar _stopped: A flag which is true between paired calls to C{reactor.run} + and C{reactor.stop}. This should be replaced with an explicit state + machine. + + @type _justStopped: C{bool} + @ivar _justStopped: A flag which is true between the time C{reactor.stop} + is called and the time the shutdown system event is fired. This is + used to determine whether that event should be fired after each + iteration through the mainloop. This should be replaced with an + explicit state machine. + + @type _started: C{bool} + @ivar _started: A flag which is true from the time C{reactor.run} is called + until the time C{reactor.run} returns. This is used to prevent calls + to C{reactor.run} on a running reactor. This should be replaced with + an explicit state machine. + + @ivar running: See L{IReactorCore.running} + + @ivar _registerAsIOThread: A flag controlling whether the reactor will + register the thread it is running in as the I/O thread when it starts. + If C{True}, registration will be done, otherwise it will not be. + + @ivar _exitSignal: See L{_ISupportsExitSignalCapturing._exitSignal} + """ + + _registerAsIOThread = True + + _stopped = True + installed = False + usingThreads = False + _exitSignal = None + + __name__ = "twisted.internet.reactor" + + def __init__(self): + super(ReactorBase, self).__init__() + self.threadCallQueue = [] + self._eventTriggers = {} + self._pendingTimedCalls = [] + self._newTimedCalls = [] + self._cancellations = 0 + self.running = False + self._started = False + self._justStopped = False + self._startedBefore = False + # reactor internal readers, e.g. the waker. + self._internalReaders = set() + self.waker = None + + # Arrange for the running attribute to change to True at the right time + # and let a subclass possibly do other things at that time (eg install + # signal handlers). + self.addSystemEventTrigger( + 'during', 'startup', self._reallyStartRunning) + self.addSystemEventTrigger('during', 'shutdown', self.crash) + self.addSystemEventTrigger('during', 'shutdown', self.disconnectAll) + + if platform.supportsThreads(): + self._initThreads() + self.installWaker() + + # override in subclasses + + _lock = None + + def installWaker(self): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement installWaker") + + + def wakeUp(self): + """ + Wake up the event loop. + """ + if self.waker: + self.waker.wakeUp() + # if the waker isn't installed, the reactor isn't running, and + # therefore doesn't need to be woken up + + def doIteration(self, delay): + """ + Do one iteration over the readers and writers which have been added. + """ + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement doIteration") + + def addReader(self, reader): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement addReader") + + def addWriter(self, writer): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement addWriter") + + def removeReader(self, reader): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement removeReader") + + def removeWriter(self, writer): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement removeWriter") + + def removeAll(self): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement removeAll") + + + def getReaders(self): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement getReaders") + + + def getWriters(self): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement getWriters") + + + # IReactorCore + def resolve(self, name, timeout=(1, 3, 11, 45)): + """Return a Deferred that will resolve a hostname. + """ + if not name: + # XXX - This is *less than* '::', and will screw up IPv6 servers + return defer.succeed('0.0.0.0') + if abstract.isIPAddress(name): + return defer.succeed(name) + return self.resolver.getHostByName(name, timeout) + + + def stop(self): + """ + See twisted.internet.interfaces.IReactorCore.stop. + """ + if self._stopped: + raise error.ReactorNotRunning( + "Can't stop reactor that isn't running.") + self._stopped = True + self._justStopped = True + self._startedBefore = True + + + def crash(self): + """ + See twisted.internet.interfaces.IReactorCore.crash. + + Reset reactor state tracking attributes and re-initialize certain + state-transition helpers which were set up in C{__init__} but later + destroyed (through use). + """ + self._started = False + self.running = False + self.addSystemEventTrigger( + 'during', 'startup', self._reallyStartRunning) + + def sigInt(self, *args): + """ + Handle a SIGINT interrupt. + + @param args: See handler specification in L{signal.signal} + """ + log.msg("Received SIGINT, shutting down.") + self.callFromThread(self.stop) + self._exitSignal = args[0] + + + def sigBreak(self, *args): + """ + Handle a SIGBREAK interrupt. + + @param args: See handler specification in L{signal.signal} + """ + log.msg("Received SIGBREAK, shutting down.") + self.callFromThread(self.stop) + self._exitSignal = args[0] + + + def sigTerm(self, *args): + """ + Handle a SIGTERM interrupt. + + @param args: See handler specification in L{signal.signal} + """ + log.msg("Received SIGTERM, shutting down.") + self.callFromThread(self.stop) + self._exitSignal = args[0] + + + def disconnectAll(self): + """Disconnect every reader, and writer in the system. + """ + selectables = self.removeAll() + for reader in selectables: + log.callWithLogger(reader, + reader.connectionLost, + failure.Failure(main.CONNECTION_LOST)) + + + def iterate(self, delay=0): + """See twisted.internet.interfaces.IReactorCore.iterate. + """ + self.runUntilCurrent() + self.doIteration(delay) + + + def fireSystemEvent(self, eventType): + """See twisted.internet.interfaces.IReactorCore.fireSystemEvent. + """ + event = self._eventTriggers.get(eventType) + if event is not None: + event.fireEvent() + + + def addSystemEventTrigger(self, _phase, _eventType, _f, *args, **kw): + """See twisted.internet.interfaces.IReactorCore.addSystemEventTrigger. + """ + assert callable(_f), "%s is not callable" % _f + if _eventType not in self._eventTriggers: + self._eventTriggers[_eventType] = _ThreePhaseEvent() + return (_eventType, self._eventTriggers[_eventType].addTrigger( + _phase, _f, *args, **kw)) + + + def removeSystemEventTrigger(self, triggerID): + """See twisted.internet.interfaces.IReactorCore.removeSystemEventTrigger. + """ + eventType, handle = triggerID + self._eventTriggers[eventType].removeTrigger(handle) + + + def callWhenRunning(self, _callable, *args, **kw): + """See twisted.internet.interfaces.IReactorCore.callWhenRunning. + """ + if self.running: + _callable(*args, **kw) + else: + return self.addSystemEventTrigger('after', 'startup', + _callable, *args, **kw) + + def startRunning(self): + """ + Method called when reactor starts: do some initialization and fire + startup events. + + Don't call this directly, call reactor.run() instead: it should take + care of calling this. + + This method is somewhat misnamed. The reactor will not necessarily be + in the running state by the time this method returns. The only + guarantee is that it will be on its way to the running state. + """ + if self._started: + raise error.ReactorAlreadyRunning() + if self._startedBefore: + raise error.ReactorNotRestartable() + self._started = True + self._stopped = False + if self._registerAsIOThread: + threadable.registerAsIOThread() + self.fireSystemEvent('startup') + + + def _reallyStartRunning(self): + """ + Method called to transition to the running state. This should happen + in the I{during startup} event trigger phase. + """ + self.running = True + + # IReactorTime + + seconds = staticmethod(runtimeSeconds) + + def callLater(self, _seconds, _f, *args, **kw): + """See twisted.internet.interfaces.IReactorTime.callLater. + """ + assert callable(_f), "%s is not callable" % _f + assert _seconds >= 0, \ + "%s is not greater than or equal to 0 seconds" % (_seconds,) + tple = DelayedCall(self.seconds() + _seconds, _f, args, kw, + self._cancelCallLater, + self._moveCallLaterSooner, + seconds=self.seconds) + self._newTimedCalls.append(tple) + return tple + + def _moveCallLaterSooner(self, tple): + # Linear time find: slow. + heap = self._pendingTimedCalls + try: + pos = heap.index(tple) + + # Move elt up the heap until it rests at the right place. + elt = heap[pos] + while pos != 0: + parent = (pos-1) // 2 + if heap[parent] <= elt: + break + # move parent down + heap[pos] = heap[parent] + pos = parent + heap[pos] = elt + except ValueError: + # element was not found in heap - oh well... + pass + + def _cancelCallLater(self, tple): + self._cancellations+=1 + + + def getDelayedCalls(self): + """ + Return all the outstanding delayed calls in the system. + They are returned in no particular order. + This method is not efficient -- it is really only meant for + test cases. + + @return: A list of outstanding delayed calls. + @type: L{list} of L{DelayedCall} + """ + return [x for x in (self._pendingTimedCalls + self._newTimedCalls) if not x.cancelled] + + + def _insertNewDelayedCalls(self): + for call in self._newTimedCalls: + if call.cancelled: + self._cancellations-=1 + else: + call.activate_delay() + heappush(self._pendingTimedCalls, call) + self._newTimedCalls = [] + + + def timeout(self): + """ + Determine the longest time the reactor may sleep (waiting on I/O + notification, perhaps) before it must wake up to service a time-related + event. + + @return: The maximum number of seconds the reactor may sleep. + @rtype: L{float} + """ + # insert new delayed calls to make sure to include them in timeout value + self._insertNewDelayedCalls() + + if not self._pendingTimedCalls: + return None + + delay = self._pendingTimedCalls[0].time - self.seconds() + + # Pick a somewhat arbitrary maximum possible value for the timeout. + # This value is 2 ** 31 / 1000, which is the number of seconds which can + # be represented as an integer number of milliseconds in a signed 32 bit + # integer. This particular limit is imposed by the epoll_wait(3) + # interface which accepts a timeout as a C "int" type and treats it as + # representing a number of milliseconds. + longest = 2147483 + + # Don't let the delay be in the past (negative) or exceed a plausible + # maximum (platform-imposed) interval. + return max(0, min(longest, delay)) + + + def runUntilCurrent(self): + """ + Run all pending timed calls. + """ + if self.threadCallQueue: + # Keep track of how many calls we actually make, as we're + # making them, in case another call is added to the queue + # while we're in this loop. + count = 0 + total = len(self.threadCallQueue) + for (f, a, kw) in self.threadCallQueue: + try: + f(*a, **kw) + except: + log.err() + count += 1 + if count == total: + break + del self.threadCallQueue[:count] + if self.threadCallQueue: + self.wakeUp() + + # insert new delayed calls now + self._insertNewDelayedCalls() + + now = self.seconds() + while self._pendingTimedCalls and (self._pendingTimedCalls[0].time <= now): + call = heappop(self._pendingTimedCalls) + if call.cancelled: + self._cancellations-=1 + continue + + if call.delayed_time > 0: + call.activate_delay() + heappush(self._pendingTimedCalls, call) + continue + + try: + call.called = 1 + call.func(*call.args, **call.kw) + except: + log.deferr() + if hasattr(call, "creator"): + e = "\n" + e += " C: previous exception occurred in " + \ + "a DelayedCall created here:\n" + e += " C:" + e += "".join(call.creator).rstrip().replace("\n","\n C:") + e += "\n" + log.msg(e) + + + if (self._cancellations > 50 and + self._cancellations > len(self._pendingTimedCalls) >> 1): + self._cancellations = 0 + self._pendingTimedCalls = [x for x in self._pendingTimedCalls + if not x.cancelled] + heapify(self._pendingTimedCalls) + + if self._justStopped: + self._justStopped = False + self.fireSystemEvent("shutdown") + + # IReactorProcess + + def _checkProcessArgs(self, args, env): + """ + Check for valid arguments and environment to spawnProcess. + + @return: A two element tuple giving values to use when creating the + process. The first element of the tuple is a C{list} of C{bytes} + giving the values for argv of the child process. The second element + of the tuple is either L{None} if C{env} was L{None} or a C{dict} + mapping C{bytes} environment keys to C{bytes} environment values. + """ + # Any unicode string which Python would successfully implicitly + # encode to a byte string would have worked before these explicit + # checks were added. Anything which would have failed with a + # UnicodeEncodeError during that implicit encoding step would have + # raised an exception in the child process and that would have been + # a pain in the butt to debug. + # + # So, we will explicitly attempt the same encoding which Python + # would implicitly do later. If it fails, we will report an error + # without ever spawning a child process. If it succeeds, we'll save + # the result so that Python doesn't need to do it implicitly later. + # + # -exarkun + + defaultEncoding = sys.getfilesystemencoding() + + # Common check function + def argChecker(arg): + """ + Return either L{bytes} or L{None}. If the given value is not + allowable for some reason, L{None} is returned. Otherwise, a + possibly different object which should be used in place of arg is + returned. This forces unicode encoding to happen now, rather than + implicitly later. + """ + if isinstance(arg, unicode): + try: + arg = arg.encode(defaultEncoding) + except UnicodeEncodeError: + return None + if isinstance(arg, bytes) and b'\0' not in arg: + return arg + + return None + + # Make a few tests to check input validity + if not isinstance(args, (tuple, list)): + raise TypeError("Arguments must be a tuple or list") + + outputArgs = [] + for arg in args: + arg = argChecker(arg) + if arg is None: + raise TypeError("Arguments contain a non-string value") + else: + outputArgs.append(arg) + + outputEnv = None + if env is not None: + outputEnv = {} + for key, val in iteritems(env): + key = argChecker(key) + if key is None: + raise TypeError("Environment contains a non-string key") + val = argChecker(val) + if val is None: + raise TypeError("Environment contains a non-string value") + outputEnv[key] = val + return outputArgs, outputEnv + + # IReactorThreads + if platform.supportsThreads(): + threadpool = None + # ID of the trigger starting the threadpool + _threadpoolStartupID = None + # ID of the trigger stopping the threadpool + threadpoolShutdownID = None + + def _initThreads(self): + self.installNameResolver(_GAIResolver(self, self.getThreadPool)) + self.usingThreads = True + + + def callFromThread(self, f, *args, **kw): + """ + See + L{twisted.internet.interfaces.IReactorFromThreads.callFromThread}. + """ + assert callable(f), "%s is not callable" % (f,) + # lists are thread-safe in CPython, but not in Jython + # this is probably a bug in Jython, but until fixed this code + # won't work in Jython. + self.threadCallQueue.append((f, args, kw)) + self.wakeUp() + + def _initThreadPool(self): + """ + Create the threadpool accessible with callFromThread. + """ + from twisted.python import threadpool + self.threadpool = threadpool.ThreadPool( + 0, 10, 'twisted.internet.reactor') + self._threadpoolStartupID = self.callWhenRunning( + self.threadpool.start) + self.threadpoolShutdownID = self.addSystemEventTrigger( + 'during', 'shutdown', self._stopThreadPool) + + def _uninstallHandler(self): + pass + + def _stopThreadPool(self): + """ + Stop the reactor threadpool. This method is only valid if there + is currently a threadpool (created by L{_initThreadPool}). It + is not intended to be called directly; instead, it will be + called by a shutdown trigger created in L{_initThreadPool}. + """ + triggers = [self._threadpoolStartupID, self.threadpoolShutdownID] + for trigger in filter(None, triggers): + try: + self.removeSystemEventTrigger(trigger) + except ValueError: + pass + self._threadpoolStartupID = None + self.threadpoolShutdownID = None + self.threadpool.stop() + self.threadpool = None + + + def getThreadPool(self): + """ + See L{twisted.internet.interfaces.IReactorThreads.getThreadPool}. + """ + if self.threadpool is None: + self._initThreadPool() + return self.threadpool + + + def callInThread(self, _callable, *args, **kwargs): + """ + See L{twisted.internet.interfaces.IReactorInThreads.callInThread}. + """ + self.getThreadPool().callInThread(_callable, *args, **kwargs) + + + def suggestThreadPoolSize(self, size): + """ + See L{twisted.internet.interfaces.IReactorThreads.suggestThreadPoolSize}. + """ + self.getThreadPool().adjustPoolsize(maxthreads=size) + else: + # This is for signal handlers. + def callFromThread(self, f, *args, **kw): + assert callable(f), "%s is not callable" % (f,) + # See comment in the other callFromThread implementation. + self.threadCallQueue.append((f, args, kw)) + +if platform.supportsThreads(): + classImplements(ReactorBase, IReactorThreads) + + +@implementer(IConnector) +@_oldStyle +class BaseConnector: + """Basic implementation of connector. + + State can be: "connecting", "connected", "disconnected" + """ + timeoutID = None + factoryStarted = 0 + + def __init__(self, factory, timeout, reactor): + self.state = "disconnected" + self.reactor = reactor + self.factory = factory + self.timeout = timeout + + def disconnect(self): + """Disconnect whatever our state is.""" + if self.state == 'connecting': + self.stopConnecting() + elif self.state == 'connected': + self.transport.loseConnection() + + def connect(self): + """Start connection to remote server.""" + if self.state != "disconnected": + raise RuntimeError("can't connect in this state") + + self.state = "connecting" + if not self.factoryStarted: + self.factory.doStart() + self.factoryStarted = 1 + self.transport = transport = self._makeTransport() + if self.timeout is not None: + self.timeoutID = self.reactor.callLater(self.timeout, transport.failIfNotConnected, error.TimeoutError()) + self.factory.startedConnecting(self) + + def stopConnecting(self): + """Stop attempting to connect.""" + if self.state != "connecting": + raise error.NotConnectingError("we're not trying to connect") + + self.state = "disconnected" + self.transport.failIfNotConnected(error.UserError()) + del self.transport + + def cancelTimeout(self): + if self.timeoutID is not None: + try: + self.timeoutID.cancel() + except ValueError: + pass + del self.timeoutID + + def buildProtocol(self, addr): + self.state = "connected" + self.cancelTimeout() + return self.factory.buildProtocol(addr) + + def connectionFailed(self, reason): + self.cancelTimeout() + self.transport = None + self.state = "disconnected" + self.factory.clientConnectionFailed(self, reason) + if self.state == "disconnected": + # factory hasn't called our connect() method + self.factory.doStop() + self.factoryStarted = 0 + + def connectionLost(self, reason): + self.state = "disconnected" + self.factory.clientConnectionLost(self, reason) + if self.state == "disconnected": + # factory hasn't called our connect() method + self.factory.doStop() + self.factoryStarted = 0 + + def getDestination(self): + raise NotImplementedError( + reflect.qual(self.__class__) + " did not implement " + "getDestination") + + def __repr__(self): + return "<%s instance at 0x%x %s %s>" % ( + reflect.qual(self.__class__), id(self), self.state, + self.getDestination()) + + + +class BasePort(abstract.FileDescriptor): + """Basic implementation of a ListeningPort. + + Note: This does not actually implement IListeningPort. + """ + + addressFamily = None + socketType = None + + def createInternetSocket(self): + s = socket.socket(self.addressFamily, self.socketType) + s.setblocking(0) + fdesc._setCloseOnExec(s.fileno()) + return s + + + def doWrite(self): + """Raises a RuntimeError""" + raise RuntimeError( + "doWrite called on a %s" % reflect.qual(self.__class__)) + + + +class _SignalReactorMixin(object): + """ + Private mixin to manage signals: it installs signal handlers at start time, + and define run method. + + It can only be used mixed in with L{ReactorBase}, and has to be defined + first in the inheritance (so that method resolution order finds + startRunning first). + + @type _installSignalHandlers: C{bool} + @ivar _installSignalHandlers: A flag which indicates whether any signal + handlers will be installed during startup. This includes handlers for + SIGCHLD to monitor child processes, and SIGINT, SIGTERM, and SIGBREAK + to stop the reactor. + """ + + _installSignalHandlers = False + + def _handleSignals(self): + """ + Install the signal handlers for the Twisted event loop. + """ + try: + import signal + except ImportError: + log.msg("Warning: signal module unavailable -- " + "not installing signal handlers.") + return + + if signal.getsignal(signal.SIGINT) == signal.default_int_handler: + # only handle if there isn't already a handler, e.g. for Pdb. + signal.signal(signal.SIGINT, self.sigInt) + signal.signal(signal.SIGTERM, self.sigTerm) + + # Catch Ctrl-Break in windows + if hasattr(signal, "SIGBREAK"): + signal.signal(signal.SIGBREAK, self.sigBreak) + + + def startRunning(self, installSignalHandlers=True): + """ + Extend the base implementation in order to remember whether signal + handlers should be installed later. + + @type installSignalHandlers: C{bool} + @param installSignalHandlers: A flag which, if set, indicates that + handlers for a number of (implementation-defined) signals should be + installed during startup. + """ + self._installSignalHandlers = installSignalHandlers + ReactorBase.startRunning(self) + + + def _reallyStartRunning(self): + """ + Extend the base implementation by also installing signal handlers, if + C{self._installSignalHandlers} is true. + """ + ReactorBase._reallyStartRunning(self) + if self._installSignalHandlers: + # Make sure this happens before after-startup events, since the + # expectation of after-startup is that the reactor is fully + # initialized. Don't do it right away for historical reasons + # (perhaps some before-startup triggers don't want there to be a + # custom SIGCHLD handler so that they can run child processes with + # some blocking api). + self._handleSignals() + + + def run(self, installSignalHandlers=True): + self.startRunning(installSignalHandlers=installSignalHandlers) + self.mainLoop() + + + def mainLoop(self): + while self._started: + try: + while self._started: + # Advance simulation time in delayed event + # processors. + self.runUntilCurrent() + t2 = self.timeout() + t = self.running and t2 + self.doIteration(t) + except: + log.msg("Unexpected error in main loop.") + log.err() + else: + log.msg('Main loop terminated.') + + + +__all__ = [] diff --git a/contrib/python/Twisted/py2/twisted/internet/cfreactor.py b/contrib/python/Twisted/py2/twisted/internet/cfreactor.py new file mode 100644 index 00000000000..d161807439e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/cfreactor.py @@ -0,0 +1,502 @@ +# -*- test-case-name: twisted.internet.test.test_core -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A reactor for integrating with U{CFRunLoop}, the +CoreFoundation main loop used by macOS. + +This is useful for integrating Twisted with U{PyObjC} +applications. +""" + +__all__ = [ + 'install', + 'CFReactor' +] + +import sys + +from zope.interface import implementer + +from twisted.internet.interfaces import IReactorFDSet +from twisted.internet.posixbase import PosixReactorBase, _Waker +from twisted.internet.posixbase import _NO_FILEDESC + +from twisted.python import log + +from CoreFoundation import ( + CFRunLoopAddSource, CFRunLoopRemoveSource, CFRunLoopGetMain, CFRunLoopRun, + CFRunLoopStop, CFRunLoopTimerCreate, CFRunLoopAddTimer, + CFRunLoopTimerInvalidate, kCFAllocatorDefault, kCFRunLoopCommonModes, + CFAbsoluteTimeGetCurrent) + +from CFNetwork import ( + CFSocketCreateWithNative, CFSocketSetSocketFlags, CFSocketEnableCallBacks, + CFSocketCreateRunLoopSource, CFSocketDisableCallBacks, CFSocketInvalidate, + kCFSocketWriteCallBack, kCFSocketReadCallBack, kCFSocketConnectCallBack, + kCFSocketAutomaticallyReenableReadCallBack, + kCFSocketAutomaticallyReenableWriteCallBack) + + +_READ = 0 +_WRITE = 1 +_preserveSOError = 1 << 6 + + +class _WakerPlus(_Waker): + """ + The normal Twisted waker will simply wake up the main loop, which causes an + iteration to run, which in turn causes L{ReactorBase.runUntilCurrent} + to get invoked. + + L{CFReactor} has a slightly different model of iteration, though: rather + than have each iteration process the thread queue, then timed calls, then + file descriptors, each callback is run as it is dispatched by the CFRunLoop + observer which triggered it. + + So this waker needs to not only unblock the loop, but also make sure the + work gets done; so, it reschedules the invocation of C{runUntilCurrent} to + be immediate (0 seconds from now) even if there is no timed call work to + do. + """ + + def doRead(self): + """ + Wake up the loop and force C{runUntilCurrent} to run immediately in the + next timed iteration. + """ + result = _Waker.doRead(self) + self.reactor._scheduleSimulate(True) + return result + + + +@implementer(IReactorFDSet) +class CFReactor(PosixReactorBase): + """ + The CoreFoundation reactor. + + You probably want to use this via the L{install} API. + + @ivar _fdmap: a dictionary, mapping an integer (a file descriptor) to a + 4-tuple of: + + - source: a C{CFRunLoopSource}; the source associated with this + socket. + - socket: a C{CFSocket} wrapping the file descriptor. + - descriptor: an L{IReadDescriptor} and/or L{IWriteDescriptor} + provider. + - read-write: a 2-C{list} of booleans: respectively, whether this + descriptor is currently registered for reading or registered for + writing. + + @ivar _idmap: a dictionary, mapping the id() of an L{IReadDescriptor} or + L{IWriteDescriptor} to a C{fd} in L{_fdmap}. Implemented in this + manner so that we don't have to rely (even more) on the hashability of + L{IReadDescriptor} providers, and we know that they won't be collected + since these are kept in sync with C{_fdmap}. Necessary because the + .fileno() of a file descriptor may change at will, so we need to be + able to look up what its file descriptor I{used} to be, so that we can + look it up in C{_fdmap} + + @ivar _cfrunloop: the C{CFRunLoop} pyobjc object wrapped + by this reactor. + + @ivar _inCFLoop: Is C{CFRunLoopRun} currently running? + + @type _inCFLoop: L{bool} + + @ivar _currentSimulator: if a CFTimer is currently scheduled with the CF + run loop to run Twisted callLater calls, this is a reference to it. + Otherwise, it is L{None} + """ + def __init__(self, runLoop=None, runner=None): + self._fdmap = {} + self._idmap = {} + if runner is None: + runner = CFRunLoopRun + self._runner = runner + + if runLoop is None: + runLoop = CFRunLoopGetMain() + self._cfrunloop = runLoop + PosixReactorBase.__init__(self) + + + def installWaker(self): + """ + Override C{installWaker} in order to use L{_WakerPlus}; otherwise this + should be exactly the same as the parent implementation. + """ + if not self.waker: + self.waker = _WakerPlus(self) + self._internalReaders.add(self.waker) + self.addReader(self.waker) + + + def _socketCallback(self, cfSocket, callbackType, + ignoredAddress, ignoredData, context): + """ + The socket callback issued by CFRunLoop. This will issue C{doRead} or + C{doWrite} calls to the L{IReadDescriptor} and L{IWriteDescriptor} + registered with the file descriptor that we are being notified of. + + @param cfSocket: The C{CFSocket} which has got some activity. + + @param callbackType: The type of activity that we are being notified + of. Either C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack}. + + @param ignoredAddress: Unused, because this is not used for either of + the callback types we register for. + + @param ignoredData: Unused, because this is not used for either of the + callback types we register for. + + @param context: The data associated with this callback by + C{CFSocketCreateWithNative} (in C{CFReactor._watchFD}). A 2-tuple + of C{(int, CFRunLoopSource)}. + """ + (fd, smugglesrc) = context + if fd not in self._fdmap: + # Spurious notifications seem to be generated sometimes if you + # CFSocketDisableCallBacks in the middle of an event. I don't know + # about this FD, any more, so let's get rid of it. + CFRunLoopRemoveSource( + self._cfrunloop, smugglesrc, kCFRunLoopCommonModes + ) + return + + src, skt, readWriteDescriptor, rw = self._fdmap[fd] + + def _drdw(): + why = None + isRead = False + + try: + if readWriteDescriptor.fileno() == -1: + why = _NO_FILEDESC + else: + isRead = callbackType == kCFSocketReadCallBack + # CFSocket seems to deliver duplicate read/write + # notifications sometimes, especially a duplicate + # writability notification when first registering the + # socket. This bears further investigation, since I may + # have been mis-interpreting the behavior I was seeing. + # (Running the full Twisted test suite, while thorough, is + # not always entirely clear.) Until this has been more + # thoroughly investigated , we consult our own + # reading/writing state flags to determine whether we + # should actually attempt a doRead/doWrite first. -glyph + if isRead: + if rw[_READ]: + why = readWriteDescriptor.doRead() + else: + if rw[_WRITE]: + why = readWriteDescriptor.doWrite() + except: + why = sys.exc_info()[1] + log.err() + if why: + self._disconnectSelectable(readWriteDescriptor, why, isRead) + + log.callWithLogger(readWriteDescriptor, _drdw) + + + def _watchFD(self, fd, descr, flag): + """ + Register a file descriptor with the C{CFRunLoop}, or modify its state + so that it's listening for both notifications (read and write) rather + than just one; used to implement C{addReader} and C{addWriter}. + + @param fd: The file descriptor. + + @type fd: L{int} + + @param descr: the L{IReadDescriptor} or L{IWriteDescriptor} + + @param flag: the flag to register for callbacks on, either + C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack} + """ + if fd == -1: + raise RuntimeError("Invalid file descriptor.") + if fd in self._fdmap: + src, cfs, gotdescr, rw = self._fdmap[fd] + # do I need to verify that it's the same descr? + else: + ctx = [] + ctx.append(fd) + cfs = CFSocketCreateWithNative( + kCFAllocatorDefault, fd, + kCFSocketReadCallBack | kCFSocketWriteCallBack | + kCFSocketConnectCallBack, + self._socketCallback, ctx + ) + CFSocketSetSocketFlags( + cfs, + kCFSocketAutomaticallyReenableReadCallBack | + kCFSocketAutomaticallyReenableWriteCallBack | + + # This extra flag is to ensure that CF doesn't (destructively, + # because destructively is the only way to do it) retrieve + # SO_ERROR and thereby break twisted.internet.tcp.BaseClient, + # which needs SO_ERROR to tell it whether or not it needs to + # call connect_ex a second time. + _preserveSOError + ) + src = CFSocketCreateRunLoopSource(kCFAllocatorDefault, cfs, 0) + ctx.append(src) + CFRunLoopAddSource(self._cfrunloop, src, kCFRunLoopCommonModes) + CFSocketDisableCallBacks( + cfs, + kCFSocketReadCallBack | kCFSocketWriteCallBack | + kCFSocketConnectCallBack + ) + rw = [False, False] + self._idmap[id(descr)] = fd + self._fdmap[fd] = src, cfs, descr, rw + rw[self._flag2idx(flag)] = True + CFSocketEnableCallBacks(cfs, flag) + + + def _flag2idx(self, flag): + """ + Convert a C{kCFSocket...} constant to an index into the read/write + state list (C{_READ} or C{_WRITE}) (the 4th element of the value of + C{self._fdmap}). + + @param flag: C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack} + + @return: C{_READ} or C{_WRITE} + """ + return {kCFSocketReadCallBack: _READ, + kCFSocketWriteCallBack: _WRITE}[flag] + + + def _unwatchFD(self, fd, descr, flag): + """ + Unregister a file descriptor with the C{CFRunLoop}, or modify its state + so that it's listening for only one notification (read or write) as + opposed to both; used to implement C{removeReader} and C{removeWriter}. + + @param fd: a file descriptor + + @type fd: C{int} + + @param descr: an L{IReadDescriptor} or L{IWriteDescriptor} + + @param flag: C{kCFSocketWriteCallBack} C{kCFSocketReadCallBack} + """ + if id(descr) not in self._idmap: + return + if fd == -1: + # need to deal with it in this case, I think. + realfd = self._idmap[id(descr)] + else: + realfd = fd + src, cfs, descr, rw = self._fdmap[realfd] + CFSocketDisableCallBacks(cfs, flag) + rw[self._flag2idx(flag)] = False + if not rw[_READ] and not rw[_WRITE]: + del self._idmap[id(descr)] + del self._fdmap[realfd] + CFRunLoopRemoveSource(self._cfrunloop, src, kCFRunLoopCommonModes) + CFSocketInvalidate(cfs) + + + def addReader(self, reader): + """ + Implement L{IReactorFDSet.addReader}. + """ + self._watchFD(reader.fileno(), reader, kCFSocketReadCallBack) + + + def addWriter(self, writer): + """ + Implement L{IReactorFDSet.addWriter}. + """ + self._watchFD(writer.fileno(), writer, kCFSocketWriteCallBack) + + + def removeReader(self, reader): + """ + Implement L{IReactorFDSet.removeReader}. + """ + self._unwatchFD(reader.fileno(), reader, kCFSocketReadCallBack) + + + def removeWriter(self, writer): + """ + Implement L{IReactorFDSet.removeWriter}. + """ + self._unwatchFD(writer.fileno(), writer, kCFSocketWriteCallBack) + + + def removeAll(self): + """ + Implement L{IReactorFDSet.removeAll}. + """ + allDesc = set([descr for src, cfs, descr, rw in self._fdmap.values()]) + allDesc -= set(self._internalReaders) + for desc in allDesc: + self.removeReader(desc) + self.removeWriter(desc) + return list(allDesc) + + + def getReaders(self): + """ + Implement L{IReactorFDSet.getReaders}. + """ + return [descr for src, cfs, descr, rw in self._fdmap.values() + if rw[_READ]] + + + def getWriters(self): + """ + Implement L{IReactorFDSet.getWriters}. + """ + return [descr for src, cfs, descr, rw in self._fdmap.values() + if rw[_WRITE]] + + + def _moveCallLaterSooner(self, tple): + """ + Override L{PosixReactorBase}'s implementation of L{IDelayedCall.reset} + so that it will immediately reschedule. Normally + C{_moveCallLaterSooner} depends on the fact that C{runUntilCurrent} is + always run before the mainloop goes back to sleep, so this forces it to + immediately recompute how long the loop needs to stay asleep. + """ + result = PosixReactorBase._moveCallLaterSooner(self, tple) + self._scheduleSimulate() + return result + + + _inCFLoop = False + + def mainLoop(self): + """ + Run the runner (C{CFRunLoopRun} or something that calls it), which runs + the run loop until C{crash()} is called. + """ + self._inCFLoop = True + try: + self._runner() + finally: + self._inCFLoop = False + + + _currentSimulator = None + + def _scheduleSimulate(self, force=False): + """ + Schedule a call to C{self.runUntilCurrent}. This will cancel the + currently scheduled call if it is already scheduled. + + @param force: Even if there are no timed calls, make sure that + C{runUntilCurrent} runs immediately (in a 0-seconds-from-now + C{CFRunLoopTimer}). This is necessary for calls which need to + trigger behavior of C{runUntilCurrent} other than running timed + calls, such as draining the thread call queue or calling C{crash()} + when the appropriate flags are set. + + @type force: C{bool} + """ + if self._currentSimulator is not None: + CFRunLoopTimerInvalidate(self._currentSimulator) + self._currentSimulator = None + timeout = self.timeout() + if force: + timeout = 0.0 + if timeout is not None: + fireDate = (CFAbsoluteTimeGetCurrent() + timeout) + def simulate(cftimer, extra): + self._currentSimulator = None + self.runUntilCurrent() + self._scheduleSimulate() + c = self._currentSimulator = CFRunLoopTimerCreate( + kCFAllocatorDefault, fireDate, + 0, 0, 0, simulate, None + ) + CFRunLoopAddTimer(self._cfrunloop, c, kCFRunLoopCommonModes) + + + def callLater(self, _seconds, _f, *args, **kw): + """ + Implement L{IReactorTime.callLater}. + """ + delayedCall = PosixReactorBase.callLater( + self, _seconds, _f, *args, **kw + ) + self._scheduleSimulate() + return delayedCall + + + def stop(self): + """ + Implement L{IReactorCore.stop}. + """ + PosixReactorBase.stop(self) + self._scheduleSimulate(True) + + + def crash(self): + """ + Implement L{IReactorCore.crash} + """ + wasStarted = self._started + PosixReactorBase.crash(self) + if self._inCFLoop: + self._stopNow() + else: + if wasStarted: + self.callLater(0, self._stopNow) + + + def _stopNow(self): + """ + Immediately stop the CFRunLoop (which must be running!). + """ + CFRunLoopStop(self._cfrunloop) + + + def iterate(self, delay=0): + """ + Emulate the behavior of C{iterate()} for things that want to call it, + by letting the loop run for a little while and then scheduling a timed + call to exit it. + """ + self.callLater(delay, self._stopNow) + self.mainLoop() + + + +def install(runLoop=None, runner=None): + """ + Configure the twisted mainloop to be run inside CFRunLoop. + + @param runLoop: the run loop to use. + + @param runner: the function to call in order to actually invoke the main + loop. This will default to C{CFRunLoopRun} if not specified. However, + this is not an appropriate choice for GUI applications, as you need to + run NSApplicationMain (or something like it). For example, to run the + Twisted mainloop in a PyObjC application, your C{main.py} should look + something like this:: + + from PyObjCTools import AppHelper + from twisted.internet.cfreactor import install + install(runner=AppHelper.runEventLoop) + # initialize your application + reactor.run() + + @return: The installed reactor. + + @rtype: C{CFReactor} + """ + + reactor = CFReactor(runLoop=runLoop, runner=runner) + from twisted.internet.main import installReactor + installReactor(reactor) + return reactor diff --git a/contrib/python/Twisted/py2/twisted/internet/default.py b/contrib/python/Twisted/py2/twisted/internet/default.py new file mode 100644 index 00000000000..c78fb18520e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/default.py @@ -0,0 +1,56 @@ +# -*- test-case-name: twisted.internet.test.test_default -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +The most suitable default reactor for the current platform. + +Depending on a specific application's needs, some other reactor may in +fact be better. +""" + +from __future__ import division, absolute_import + +__all__ = ["install"] + +from twisted.python.runtime import platform + + +def _getInstallFunction(platform): + """ + Return a function to install the reactor most suited for the given platform. + + @param platform: The platform for which to select a reactor. + @type platform: L{twisted.python.runtime.Platform} + + @return: A zero-argument callable which will install the selected + reactor. + """ + # Linux: epoll(7) is the default, since it scales well. + # + # macOS: poll(2) is not exposed by Python because it doesn't support all + # file descriptors (in particular, lack of PTY support is a problem) -- + # see . kqueue has the same restrictions + # as poll(2) as far PTY support goes. + # + # Windows: IOCP should eventually be default, but still has some serious + # bugs, e.g. . + # + # We therefore choose epoll(7) on Linux, poll(2) on other non-macOS POSIX + # platforms, and select(2) everywhere else. + try: + if platform.isLinux(): + try: + from twisted.internet.epollreactor import install + except ImportError: + from twisted.internet.pollreactor import install + elif platform.getType() == 'posix' and not platform.isMacOSX(): + from twisted.internet.pollreactor import install + else: + from twisted.internet.selectreactor import install + except ImportError: + from twisted.internet.selectreactor import install + return install + + +install = _getInstallFunction(platform) diff --git a/contrib/python/Twisted/py2/twisted/internet/defer.py b/contrib/python/Twisted/py2/twisted/internet/defer.py new file mode 100644 index 00000000000..96fa3245125 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/defer.py @@ -0,0 +1,2019 @@ +# -*- test-case-name: twisted.test.test_defer -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for results that aren't immediately available. + +Maintainer: Glyph Lefkowitz + +@var _NO_RESULT: The result used to represent the fact that there is no + result. B{Never ever ever use this as an actual result for a Deferred}. You + have been warned. + +@var _CONTINUE: A marker left in L{Deferred.callback}s to indicate a Deferred + chain. Always accompanied by a Deferred instance in the args tuple pointing + at the Deferred which is chained to the Deferred which has this marker. +""" + +from __future__ import division, absolute_import, print_function + +import attr +import traceback +import types +import warnings +from sys import exc_info, version_info +from functools import wraps +from incremental import Version + +# Twisted imports +from twisted.python.compat import cmp, comparable +from twisted.python import lockfile, failure +from twisted.logger import Logger +from twisted.python.deprecate import warnAboutFunction, deprecated +from twisted.python._oldstyle import _oldStyle + +log = Logger() + + +class AlreadyCalledError(Exception): + pass + + + +class CancelledError(Exception): + """ + This error is raised by default when a L{Deferred} is cancelled. + """ + + +class TimeoutError(Exception): + """ + This error is raised by default when a L{Deferred} times out. + """ + + + +def logError(err): + """ + Log and return failure. + + This method can be used as an errback that passes the failure on to the + next errback unmodified. Note that if this is the last errback, and the + deferred gets garbage collected after being this errback has been called, + the clean up code logs it again. + """ + log.failure(None, err) + return err + + + +def succeed(result): + """ + Return a L{Deferred} that has already had C{.callback(result)} called. + + This is useful when you're writing synchronous code to an + asynchronous interface: i.e., some code is calling you expecting a + L{Deferred} result, but you don't actually need to do anything + asynchronous. Just return C{defer.succeed(theResult)}. + + See L{fail} for a version of this function that uses a failing + L{Deferred} rather than a successful one. + + @param result: The result to give to the Deferred's 'callback' + method. + + @rtype: L{Deferred} + """ + d = Deferred() + d.callback(result) + return d + + + +def fail(result=None): + """ + Return a L{Deferred} that has already had C{.errback(result)} called. + + See L{succeed}'s docstring for rationale. + + @param result: The same argument that L{Deferred.errback} takes. + + @raise NoCurrentExceptionError: If C{result} is L{None} but there is no + current exception state. + + @rtype: L{Deferred} + """ + d = Deferred() + d.errback(result) + return d + + + +def execute(callable, *args, **kw): + """ + Create a L{Deferred} from a callable and arguments. + + Call the given function with the given arguments. Return a L{Deferred} + which has been fired with its callback as the result of that invocation + or its C{errback} with a L{Failure} for the exception thrown. + """ + try: + result = callable(*args, **kw) + except: + return fail() + else: + return succeed(result) + + + +def maybeDeferred(f, *args, **kw): + """ + Invoke a function that may or may not return a L{Deferred}. + + Call the given function with the given arguments. If the returned + object is a L{Deferred}, return it. If the returned object is a L{Failure}, + wrap it with L{fail} and return it. Otherwise, wrap it in L{succeed} and + return it. If an exception is raised, convert it to a L{Failure}, wrap it + in L{fail}, and then return it. + + @type f: Any callable + @param f: The callable to invoke + + @param args: The arguments to pass to C{f} + @param kw: The keyword arguments to pass to C{f} + + @rtype: L{Deferred} + @return: The result of the function call, wrapped in a L{Deferred} if + necessary. + """ + try: + result = f(*args, **kw) + except: + return fail(failure.Failure(captureVars=Deferred.debug)) + + if isinstance(result, Deferred): + return result + elif isinstance(result, failure.Failure): + return fail(result) + else: + return succeed(result) + + + +@deprecated(Version('Twisted', 17, 1, 0), + replacement='twisted.internet.defer.Deferred.addTimeout') +def timeout(deferred): + deferred.errback(failure.Failure(TimeoutError("Callback timed out"))) + + + +def passthru(arg): + return arg + + + +def setDebugging(on): + """ + Enable or disable L{Deferred} debugging. + + When debugging is on, the call stacks from creation and invocation are + recorded, and added to any L{AlreadyCalledError}s we raise. + """ + Deferred.debug=bool(on) + + + +def getDebugging(): + """ + Determine whether L{Deferred} debugging is enabled. + """ + return Deferred.debug + + +# See module docstring. +_NO_RESULT = object() +_CONTINUE = object() + + + +@_oldStyle +class Deferred: + """ + This is a callback which will be put off until later. + + Why do we want this? Well, in cases where a function in a threaded + program would block until it gets a result, for Twisted it should + not block. Instead, it should return a L{Deferred}. + + This can be implemented for protocols that run over the network by + writing an asynchronous protocol for L{twisted.internet}. For methods + that come from outside packages that are not under our control, we use + threads (see for example L{twisted.enterprise.adbapi}). + + For more information about Deferreds, see doc/core/howto/defer.html or + U{http://twistedmatrix.com/documents/current/core/howto/defer.html} + + When creating a Deferred, you may provide a canceller function, which + will be called by d.cancel() to let you do any clean-up necessary if the + user decides not to wait for the deferred to complete. + + @ivar called: A flag which is C{False} until either C{callback} or + C{errback} is called and afterwards always C{True}. + @type called: L{bool} + + @ivar paused: A counter of how many unmatched C{pause} calls have been made + on this instance. + @type paused: L{int} + + @ivar _suppressAlreadyCalled: A flag used by the cancellation mechanism + which is C{True} if the Deferred has no canceller and has been + cancelled, C{False} otherwise. If C{True}, it can be expected that + C{callback} or C{errback} will eventually be called and the result + should be silently discarded. + @type _suppressAlreadyCalled: L{bool} + + @ivar _runningCallbacks: A flag which is C{True} while this instance is + executing its callback chain, used to stop recursive execution of + L{_runCallbacks} + @type _runningCallbacks: L{bool} + + @ivar _chainedTo: If this L{Deferred} is waiting for the result of another + L{Deferred}, this is a reference to the other Deferred. Otherwise, + L{None}. + """ + + called = False + paused = False + _debugInfo = None + _suppressAlreadyCalled = False + + # Are we currently running a user-installed callback? Meant to prevent + # recursive running of callbacks when a reentrant call to add a callback is + # used. + _runningCallbacks = False + + # Keep this class attribute for now, for compatibility with code that + # sets it directly. + debug = False + + _chainedTo = None + + def __init__(self, canceller=None): + """ + Initialize a L{Deferred}. + + @param canceller: a callable used to stop the pending operation + scheduled by this L{Deferred} when L{Deferred.cancel} is + invoked. The canceller will be passed the deferred whose + cancelation is requested (i.e., self). + + If a canceller is not given, or does not invoke its argument's + C{callback} or C{errback} method, L{Deferred.cancel} will + invoke L{Deferred.errback} with a L{CancelledError}. + + Note that if a canceller is not given, C{callback} or + C{errback} may still be invoked exactly once, even though + defer.py will have already invoked C{errback}, as described + above. This allows clients of code which returns a L{Deferred} + to cancel it without requiring the L{Deferred} instantiator to + provide any specific implementation support for cancellation. + New in 10.1. + + @type canceller: a 1-argument callable which takes a L{Deferred}. The + return result is ignored. + """ + self.callbacks = [] + self._canceller = canceller + if self.debug: + self._debugInfo = DebugInfo() + self._debugInfo.creator = traceback.format_stack()[:-1] + + + def addCallbacks(self, callback, errback=None, + callbackArgs=None, callbackKeywords=None, + errbackArgs=None, errbackKeywords=None): + """ + Add a pair of callbacks (success and error) to this L{Deferred}. + + These will be executed when the 'master' callback is run. + + @return: C{self}. + @rtype: a L{Deferred} + """ + assert callable(callback) + assert errback is None or callable(errback) + cbs = ((callback, callbackArgs, callbackKeywords), + (errback or (passthru), errbackArgs, errbackKeywords)) + self.callbacks.append(cbs) + + if self.called: + self._runCallbacks() + return self + + + def addCallback(self, callback, *args, **kw): + """ + Convenience method for adding just a callback. + + See L{addCallbacks}. + """ + return self.addCallbacks(callback, callbackArgs=args, + callbackKeywords=kw) + + + def addErrback(self, errback, *args, **kw): + """ + Convenience method for adding just an errback. + + See L{addCallbacks}. + """ + return self.addCallbacks(passthru, errback, + errbackArgs=args, + errbackKeywords=kw) + + + def addBoth(self, callback, *args, **kw): + """ + Convenience method for adding a single callable as both a callback + and an errback. + + See L{addCallbacks}. + """ + return self.addCallbacks(callback, callback, + callbackArgs=args, errbackArgs=args, + callbackKeywords=kw, errbackKeywords=kw) + + + def addTimeout(self, timeout, clock, onTimeoutCancel=None): + """ + Time out this L{Deferred} by scheduling it to be cancelled after + C{timeout} seconds. + + The timeout encompasses all the callbacks and errbacks added to this + L{defer.Deferred} before the call to L{addTimeout}, and none added + after the call. + + If this L{Deferred} gets timed out, it errbacks with a L{TimeoutError}, + unless a cancelable function was passed to its initialization or unless + a different C{onTimeoutCancel} callable is provided. + + @param timeout: number of seconds to wait before timing out this + L{Deferred} + @type timeout: L{int} + + @param clock: The object which will be used to schedule the timeout. + @type clock: L{twisted.internet.interfaces.IReactorTime} + + @param onTimeoutCancel: A callable which is called immediately after + this L{Deferred} times out, and not if this L{Deferred} is + otherwise cancelled before the timeout. It takes an arbitrary + value, which is the value of this L{Deferred} at that exact point + in time (probably a L{CancelledError} L{Failure}), and the + C{timeout}. The default callable (if none is provided) will + translate a L{CancelledError} L{Failure} into a L{TimeoutError}. + @type onTimeoutCancel: L{callable} + + @return: C{self}. + @rtype: a L{Deferred} + + @since: 16.5 + """ + timedOut = [False] + + def timeItOut(): + timedOut[0] = True + self.cancel() + + delayedCall = clock.callLater(timeout, timeItOut) + + def convertCancelled(value): + # if C{deferred} was timed out, call the translation function, + # if provdied, otherwise just use L{cancelledToTimedOutError} + if timedOut[0]: + toCall = onTimeoutCancel or _cancelledToTimedOutError + return toCall(value, timeout) + return value + + self.addBoth(convertCancelled) + + def cancelTimeout(result): + # stop the pending call to cancel the deferred if it's been fired + if delayedCall.active(): + delayedCall.cancel() + return result + + self.addBoth(cancelTimeout) + return self + + + def chainDeferred(self, d): + """ + Chain another L{Deferred} to this L{Deferred}. + + This method adds callbacks to this L{Deferred} to call C{d}'s callback + or errback, as appropriate. It is merely a shorthand way of performing + the following:: + + self.addCallbacks(d.callback, d.errback) + + When you chain a deferred d2 to another deferred d1 with + d1.chainDeferred(d2), you are making d2 participate in the callback + chain of d1. Thus any event that fires d1 will also fire d2. + However, the converse is B{not} true; if d2 is fired d1 will not be + affected. + + Note that unlike the case where chaining is caused by a L{Deferred} + being returned from a callback, it is possible to cause the call + stack size limit to be exceeded by chaining many L{Deferred}s + together with C{chainDeferred}. + + @return: C{self}. + @rtype: a L{Deferred} + """ + d._chainedTo = self + return self.addCallbacks(d.callback, d.errback) + + + def callback(self, result): + """ + Run all success callbacks that have been added to this L{Deferred}. + + Each callback will have its result passed as the first argument to + the next; this way, the callbacks act as a 'processing chain'. If + the success-callback returns a L{Failure} or raises an L{Exception}, + processing will continue on the *error* callback chain. If a + callback (or errback) returns another L{Deferred}, this L{Deferred} + will be chained to it (and further callbacks will not run until that + L{Deferred} has a result). + + An instance of L{Deferred} may only have either L{callback} or + L{errback} called on it, and only once. + + @param result: The object which will be passed to the first callback + added to this L{Deferred} (via L{addCallback}). + + @raise AlreadyCalledError: If L{callback} or L{errback} has already been + called on this L{Deferred}. + """ + assert not isinstance(result, Deferred) + self._startRunCallbacks(result) + + + def errback(self, fail=None): + """ + Run all error callbacks that have been added to this L{Deferred}. + + Each callback will have its result passed as the first + argument to the next; this way, the callbacks act as a + 'processing chain'. Also, if the error-callback returns a non-Failure + or doesn't raise an L{Exception}, processing will continue on the + *success*-callback chain. + + If the argument that's passed to me is not a L{failure.Failure} instance, + it will be embedded in one. If no argument is passed, a + L{failure.Failure} instance will be created based on the current + traceback stack. + + Passing a string as `fail' is deprecated, and will be punished with + a warning message. + + An instance of L{Deferred} may only have either L{callback} or + L{errback} called on it, and only once. + + @param fail: The L{Failure} object which will be passed to the first + errback added to this L{Deferred} (via L{addErrback}). + Alternatively, a L{Exception} instance from which a L{Failure} will + be constructed (with no traceback) or L{None} to create a L{Failure} + instance from the current exception state (with a traceback). + + @raise AlreadyCalledError: If L{callback} or L{errback} has already been + called on this L{Deferred}. + + @raise NoCurrentExceptionError: If C{fail} is L{None} but there is + no current exception state. + """ + if fail is None: + fail = failure.Failure(captureVars=self.debug) + elif not isinstance(fail, failure.Failure): + fail = failure.Failure(fail) + + self._startRunCallbacks(fail) + + + def pause(self): + """ + Stop processing on a L{Deferred} until L{unpause}() is called. + """ + self.paused = self.paused + 1 + + + def unpause(self): + """ + Process all callbacks made since L{pause}() was called. + """ + self.paused = self.paused - 1 + if self.paused: + return + if self.called: + self._runCallbacks() + + + def cancel(self): + """ + Cancel this L{Deferred}. + + If the L{Deferred} has not yet had its C{errback} or C{callback} method + invoked, call the canceller function provided to the constructor. If + that function does not invoke C{callback} or C{errback}, or if no + canceller function was provided, errback with L{CancelledError}. + + If this L{Deferred} is waiting on another L{Deferred}, forward the + cancellation to the other L{Deferred}. + """ + if not self.called: + canceller = self._canceller + if canceller: + canceller(self) + else: + # Arrange to eat the callback that will eventually be fired + # since there was no real canceller. + self._suppressAlreadyCalled = True + if not self.called: + # There was no canceller, or the canceller didn't call + # callback or errback. + self.errback(failure.Failure(CancelledError())) + elif isinstance(self.result, Deferred): + # Waiting for another deferred -- cancel it instead. + self.result.cancel() + + + def _startRunCallbacks(self, result): + if self.called: + if self._suppressAlreadyCalled: + self._suppressAlreadyCalled = False + return + if self.debug: + if self._debugInfo is None: + self._debugInfo = DebugInfo() + extra = "\n" + self._debugInfo._getDebugTracebacks() + raise AlreadyCalledError(extra) + raise AlreadyCalledError + if self.debug: + if self._debugInfo is None: + self._debugInfo = DebugInfo() + self._debugInfo.invoker = traceback.format_stack()[:-2] + self.called = True + self.result = result + self._runCallbacks() + + + def _continuation(self): + """ + Build a tuple of callback and errback with L{_CONTINUE}. + """ + return ((_CONTINUE, (self,), None), + (_CONTINUE, (self,), None)) + + + def _runCallbacks(self): + """ + Run the chain of callbacks once a result is available. + + This consists of a simple loop over all of the callbacks, calling each + with the current result and making the current result equal to the + return value (or raised exception) of that call. + + If L{_runningCallbacks} is true, this loop won't run at all, since + it is already running above us on the call stack. If C{self.paused} is + true, the loop also won't run, because that's what it means to be + paused. + + The loop will terminate before processing all of the callbacks if a + L{Deferred} without a result is encountered. + + If a L{Deferred} I{with} a result is encountered, that result is taken + and the loop proceeds. + + @note: The implementation is complicated slightly by the fact that + chaining (associating two L{Deferred}s with each other such that one + will wait for the result of the other, as happens when a Deferred is + returned from a callback on another L{Deferred}) is supported + iteratively rather than recursively, to avoid running out of stack + frames when processing long chains. + """ + if self._runningCallbacks: + # Don't recursively run callbacks + return + + # Keep track of all the Deferreds encountered while propagating results + # up a chain. The way a Deferred gets onto this stack is by having + # added its _continuation() to the callbacks list of a second Deferred + # and then that second Deferred being fired. ie, if ever had _chainedTo + # set to something other than None, you might end up on this stack. + chain = [self] + + while chain: + current = chain[-1] + + if current.paused: + # This Deferred isn't going to produce a result at all. All the + # Deferreds up the chain waiting on it will just have to... + # wait. + return + + finished = True + current._chainedTo = None + while current.callbacks: + item = current.callbacks.pop(0) + callback, args, kw = item[ + isinstance(current.result, failure.Failure)] + args = args or () + kw = kw or {} + + # Avoid recursion if we can. + if callback is _CONTINUE: + # Give the waiting Deferred our current result and then + # forget about that result ourselves. + chainee = args[0] + chainee.result = current.result + current.result = None + # Making sure to update _debugInfo + if current._debugInfo is not None: + current._debugInfo.failResult = None + chainee.paused -= 1 + chain.append(chainee) + # Delay cleaning this Deferred and popping it from the chain + # until after we've dealt with chainee. + finished = False + break + + try: + current._runningCallbacks = True + try: + current.result = callback(current.result, *args, **kw) + if current.result is current: + warnAboutFunction( + callback, + "Callback returned the Deferred " + "it was attached to; this breaks the " + "callback chain and will raise an " + "exception in the future.") + finally: + current._runningCallbacks = False + except: + # Including full frame information in the Failure is quite + # expensive, so we avoid it unless self.debug is set. + current.result = failure.Failure(captureVars=self.debug) + else: + if isinstance(current.result, Deferred): + # The result is another Deferred. If it has a result, + # we can take it and keep going. + resultResult = getattr(current.result, 'result', _NO_RESULT) + if resultResult is _NO_RESULT or isinstance(resultResult, Deferred) or current.result.paused: + # Nope, it didn't. Pause and chain. + current.pause() + current._chainedTo = current.result + # Note: current.result has no result, so it's not + # running its callbacks right now. Therefore we can + # append to the callbacks list directly instead of + # using addCallbacks. + current.result.callbacks.append(current._continuation()) + break + else: + # Yep, it did. Steal it. + current.result.result = None + # Make sure _debugInfo's failure state is updated. + if current.result._debugInfo is not None: + current.result._debugInfo.failResult = None + current.result = resultResult + + if finished: + # As much of the callback chain - perhaps all of it - as can be + # processed right now has been. The current Deferred is waiting on + # another Deferred or for more callbacks. Before finishing with it, + # make sure its _debugInfo is in the proper state. + if isinstance(current.result, failure.Failure): + # Stash the Failure in the _debugInfo for unhandled error + # reporting. + current.result.cleanFailure() + if current._debugInfo is None: + current._debugInfo = DebugInfo() + current._debugInfo.failResult = current.result + else: + # Clear out any Failure in the _debugInfo, since the result + # is no longer a Failure. + if current._debugInfo is not None: + current._debugInfo.failResult = None + + # This Deferred is done, pop it from the chain and move back up + # to the Deferred which supplied us with our result. + chain.pop() + + + def __str__(self): + """ + Return a string representation of this C{Deferred}. + """ + cname = self.__class__.__name__ + result = getattr(self, 'result', _NO_RESULT) + myID = id(self) + if self._chainedTo is not None: + result = ' waiting on Deferred at 0x%x' % (id(self._chainedTo),) + elif result is _NO_RESULT: + result = '' + else: + result = ' current result: %r' % (result,) + return "<%s at 0x%x%s>" % (cname, myID, result) + __repr__ = __str__ + + + def __iter__(self): + return self + + + @failure._extraneous + def send(self, value=None): + if self.paused: + # If we're paused, we have no result to give + return self + + result = getattr(self, 'result', _NO_RESULT) + if result is _NO_RESULT: + return self + if isinstance(result, failure.Failure): + # Clear the failure on debugInfo so it doesn't raise "unhandled + # exception" + self._debugInfo.failResult = None + result.value.__failure__ = result + raise result.value + else: + raise StopIteration(result) + + + # For PEP-492 support (async/await) + __await__ = __iter__ + __next__ = send + + + def asFuture(self, loop): + """ + Adapt a L{Deferred} into a L{asyncio.Future} which is bound to C{loop}. + + @note: converting a L{Deferred} to an L{asyncio.Future} consumes both + its result and its errors, so this method implicitly converts + C{self} into a L{Deferred} firing with L{None}, regardless of what + its result previously would have been. + + @since: Twisted 17.5.0 + + @param loop: The asyncio event loop to bind the L{asyncio.Future} to. + @type loop: L{asyncio.AbstractEventLoop} or similar + + @param deferred: The Deferred to adapt. + @type deferred: L{Deferred} + + @return: A Future which will fire when the Deferred fires. + @rtype: L{asyncio.Future} + """ + try: + createFuture = loop.create_future + except AttributeError: + from asyncio import Future + def createFuture(): + return Future(loop=loop) + future = createFuture() + def checkCancel(futureAgain): + if futureAgain.cancelled(): + self.cancel() + def maybeFail(failure): + if not future.cancelled(): + future.set_exception(failure.value) + def maybeSucceed(result): + if not future.cancelled(): + future.set_result(result) + self.addCallbacks(maybeSucceed, maybeFail) + future.add_done_callback(checkCancel) + return future + + + @classmethod + def fromFuture(cls, future): + """ + Adapt an L{asyncio.Future} to a L{Deferred}. + + @note: This creates a L{Deferred} from a L{asyncio.Future}, I{not} from + a C{coroutine}; in other words, you will need to call + L{asyncio.ensure_future}, + L{asyncio.loop.create_task} or create an + L{asyncio.Task} yourself to get from a C{coroutine} to a + L{asyncio.Future} if what you have is an awaitable coroutine and + not a L{asyncio.Future}. (The length of this list of techniques is + exactly why we have left it to the caller!) + + @since: Twisted 17.5.0 + + @param future: The Future to adapt. + @type future: L{asyncio.Future} + + @return: A Deferred which will fire when the Future fires. + @rtype: L{Deferred} + """ + def adapt(result): + try: + extracted = result.result() + except: + extracted = failure.Failure() + adapt.actual.callback(extracted) + futureCancel = object() + def cancel(reself): + future.cancel() + reself.callback(futureCancel) + self = cls(cancel) + adapt.actual = self + def uncancel(result): + if result is futureCancel: + adapt.actual = Deferred() + return adapt.actual + return result + self.addCallback(uncancel) + future.add_done_callback(adapt) + return self + + + +def _cancelledToTimedOutError(value, timeout): + """ + A default translation function that translates L{Failure}s that are + L{CancelledError}s to L{TimeoutError}s. + + @param value: Anything + @type value: Anything + + @param timeout: The timeout + @type timeout: L{int} + + @rtype: C{value} + @raise: L{TimeoutError} + + @since: 16.5 + """ + if isinstance(value, failure.Failure): + value.trap(CancelledError) + raise TimeoutError(timeout, "Deferred") + return value + + + +def ensureDeferred(coro): + """ + Schedule the execution of a coroutine that awaits/yields from L{Deferred}s, + wrapping it in a L{Deferred} that will fire on success/failure of the + coroutine. If a Deferred is passed to this function, it will be returned + directly (mimicing C{asyncio}'s C{ensure_future} function). + + Coroutine functions return a coroutine object, similar to how generators + work. This function turns that coroutine into a Deferred, meaning that it + can be used in regular Twisted code. For example:: + + import treq + from twisted.internet.defer import ensureDeferred + from twisted.internet.task import react + + async def crawl(pages): + results = {} + for page in pages: + results[page] = await treq.content(await treq.get(page)) + return results + + def main(reactor): + pages = [ + "http://localhost:8080" + ] + d = ensureDeferred(crawl(pages)) + d.addCallback(print) + return d + + react(main) + + @param coro: The coroutine object to schedule, or a L{Deferred}. + @type coro: A Python 3.5+ C{async def} C{coroutine}, a Python 3.4+ + C{yield from} using L{types.GeneratorType}, or a L{Deferred}. + + @rtype: L{Deferred} + """ + from types import GeneratorType + + if version_info >= (3, 4, 0): + from asyncio import iscoroutine + + if iscoroutine(coro) or isinstance(coro, GeneratorType): + return _cancellableInlineCallbacks(coro) + + if not isinstance(coro, Deferred): + raise ValueError("%r is not a coroutine or a Deferred" % (coro,)) + + # Must be a Deferred + return coro + + + + +@_oldStyle +class DebugInfo: + """ + Deferred debug helper. + """ + + failResult = None + + def _getDebugTracebacks(self): + info = '' + if hasattr(self, "creator"): + info += " C: Deferred was created:\n C:" + info += "".join(self.creator).rstrip().replace("\n", "\n C:") + info += "\n" + if hasattr(self, "invoker"): + info += " I: First Invoker was:\n I:" + info += "".join(self.invoker).rstrip().replace("\n", "\n I:") + info += "\n" + return info + + + def __del__(self): + """ + Print tracebacks and die. + + If the *last* (and I do mean *last*) callback leaves me in an error + state, print a traceback (if said errback is a L{Failure}). + """ + if self.failResult is not None: + # Note: this is two separate messages for compatibility with + # earlier tests; arguably it should be a single error message. + log.critical("Unhandled error in Deferred:", + isError=True) + + debugInfo = self._getDebugTracebacks() + if debugInfo: + format = "(debug: {debugInfo})" + else: + format = None + + log.failure(format, + self.failResult, + debugInfo=debugInfo) + + + +@comparable +class FirstError(Exception): + """ + First error to occur in a L{DeferredList} if C{fireOnOneErrback} is set. + + @ivar subFailure: The L{Failure} that occurred. + @type subFailure: L{Failure} + + @ivar index: The index of the L{Deferred} in the L{DeferredList} where + it happened. + @type index: L{int} + """ + def __init__(self, failure, index): + Exception.__init__(self, failure, index) + self.subFailure = failure + self.index = index + + + def __repr__(self): + """ + The I{repr} of L{FirstError} instances includes the repr of the + wrapped failure's exception and the index of the L{FirstError}. + """ + return 'FirstError[#%d, %r]' % (self.index, self.subFailure.value) + + + def __str__(self): + """ + The I{str} of L{FirstError} instances includes the I{str} of the + entire wrapped failure (including its traceback and exception) and + the index of the L{FirstError}. + """ + return 'FirstError[#%d, %s]' % (self.index, self.subFailure) + + + def __cmp__(self, other): + """ + Comparison between L{FirstError} and other L{FirstError} instances + is defined as the comparison of the index and sub-failure of each + instance. L{FirstError} instances don't compare equal to anything + that isn't a L{FirstError} instance. + + @since: 8.2 + """ + if isinstance(other, FirstError): + return cmp( + (self.index, self.subFailure), + (other.index, other.subFailure)) + return -1 + + + +class DeferredList(Deferred): + """ + L{DeferredList} is a tool for collecting the results of several Deferreds. + + This tracks a list of L{Deferred}s for their results, and makes a single + callback when they have all completed. By default, the ultimate result is a + list of (success, result) tuples, 'success' being a boolean. + L{DeferredList} exposes the same API that L{Deferred} does, so callbacks and + errbacks can be added to it in the same way. + + L{DeferredList} is implemented by adding callbacks and errbacks to each + L{Deferred} in the list passed to it. This means callbacks and errbacks + added to the Deferreds before they are passed to L{DeferredList} will change + the result that L{DeferredList} sees (i.e., L{DeferredList} is not special). + Callbacks and errbacks can also be added to the Deferreds after they are + passed to L{DeferredList} and L{DeferredList} may change the result that + they see. + + See the documentation for the C{__init__} arguments for more information. + + @ivar _deferredList: The L{list} of L{Deferred}s to track. + """ + + fireOnOneCallback = False + fireOnOneErrback = False + + def __init__(self, deferredList, fireOnOneCallback=False, + fireOnOneErrback=False, consumeErrors=False): + """ + Initialize a DeferredList. + + @param deferredList: The list of deferreds to track. + @type deferredList: L{list} of L{Deferred}s + + @param fireOnOneCallback: (keyword param) a flag indicating that this + L{DeferredList} will fire when the first L{Deferred} in + C{deferredList} fires with a non-failure result without waiting for + any of the other Deferreds. When this flag is set, the DeferredList + will fire with a two-tuple: the first element is the result of the + Deferred which fired; the second element is the index in + C{deferredList} of that Deferred. + @type fireOnOneCallback: L{bool} + + @param fireOnOneErrback: (keyword param) a flag indicating that this + L{DeferredList} will fire when the first L{Deferred} in + C{deferredList} fires with a failure result without waiting for any + of the other Deferreds. When this flag is set, if a Deferred in the + list errbacks, the DeferredList will errback with a L{FirstError} + failure wrapping the failure of that Deferred. + @type fireOnOneErrback: L{bool} + + @param consumeErrors: (keyword param) a flag indicating that failures in + any of the included L{Deferred}s should not be propagated to + errbacks added to the individual L{Deferred}s after this + L{DeferredList} is constructed. After constructing the + L{DeferredList}, any errors in the individual L{Deferred}s will be + converted to a callback result of L{None}. This is useful to + prevent spurious 'Unhandled error in Deferred' messages from being + logged. This does not prevent C{fireOnOneErrback} from working. + @type consumeErrors: L{bool} + """ + self._deferredList = list(deferredList) + self.resultList = [None] * len(self._deferredList) + Deferred.__init__(self) + if len(self._deferredList) == 0 and not fireOnOneCallback: + self.callback(self.resultList) + + # These flags need to be set *before* attaching callbacks to the + # deferreds, because the callbacks use these flags, and will run + # synchronously if any of the deferreds are already fired. + self.fireOnOneCallback = fireOnOneCallback + self.fireOnOneErrback = fireOnOneErrback + self.consumeErrors = consumeErrors + self.finishedCount = 0 + + index = 0 + for deferred in self._deferredList: + deferred.addCallbacks(self._cbDeferred, self._cbDeferred, + callbackArgs=(index,SUCCESS), + errbackArgs=(index,FAILURE)) + index = index + 1 + + + def _cbDeferred(self, result, index, succeeded): + """ + (internal) Callback for when one of my deferreds fires. + """ + self.resultList[index] = (succeeded, result) + + self.finishedCount += 1 + if not self.called: + if succeeded == SUCCESS and self.fireOnOneCallback: + self.callback((result, index)) + elif succeeded == FAILURE and self.fireOnOneErrback: + self.errback(failure.Failure(FirstError(result, index))) + elif self.finishedCount == len(self.resultList): + self.callback(self.resultList) + + if succeeded == FAILURE and self.consumeErrors: + result = None + + return result + + + def cancel(self): + """ + Cancel this L{DeferredList}. + + If the L{DeferredList} hasn't fired yet, cancel every L{Deferred} in + the list. + + If the L{DeferredList} has fired, including the case where the + C{fireOnOneCallback}/C{fireOnOneErrback} flag is set and the + L{DeferredList} fires because one L{Deferred} in the list fires with a + non-failure/failure result, do nothing in the C{cancel} method. + """ + if not self.called: + for deferred in self._deferredList: + try: + deferred.cancel() + except: + log.failure( + "Exception raised from user supplied canceller" + ) + + + +def _parseDListResult(l, fireOnOneErrback=False): + if __debug__: + for success, value in l: + assert success + return [x[1] for x in l] + + + +def gatherResults(deferredList, consumeErrors=False): + """ + Returns, via a L{Deferred}, a list with the results of the given + L{Deferred}s - in effect, a "join" of multiple deferred operations. + + The returned L{Deferred} will fire when I{all} of the provided L{Deferred}s + have fired, or when any one of them has failed. + + This method can be cancelled by calling the C{cancel} method of the + L{Deferred}, all the L{Deferred}s in the list will be cancelled. + + This differs from L{DeferredList} in that you don't need to parse + the result for success/failure. + + @type deferredList: L{list} of L{Deferred}s + + @param consumeErrors: (keyword param) a flag, defaulting to False, + indicating that failures in any of the given L{Deferred}s should not be + propagated to errbacks added to the individual L{Deferred}s after this + L{gatherResults} invocation. Any such errors in the individual + L{Deferred}s will be converted to a callback result of L{None}. This + is useful to prevent spurious 'Unhandled error in Deferred' messages + from being logged. This parameter is available since 11.1.0. + @type consumeErrors: L{bool} + """ + d = DeferredList(deferredList, fireOnOneErrback=True, + consumeErrors=consumeErrors) + d.addCallback(_parseDListResult) + return d + + + +# Constants for use with DeferredList + +SUCCESS = True +FAILURE = False + + + +## deferredGenerator +@_oldStyle +class waitForDeferred: + """ + See L{deferredGenerator}. + """ + + def __init__(self, d): + warnings.warn( + "twisted.internet.defer.waitForDeferred was deprecated in " + "Twisted 15.0.0; please use twisted.internet.defer.inlineCallbacks " + "instead", DeprecationWarning, stacklevel=2) + + if not isinstance(d, Deferred): + raise TypeError("You must give waitForDeferred a Deferred. You gave it %r." % (d,)) + self.d = d + + + def getResult(self): + if isinstance(self.result, failure.Failure): + self.result.raiseException() + return self.result + + + +def _deferGenerator(g, deferred): + """ + See L{deferredGenerator}. + """ + result = None + + # This function is complicated by the need to prevent unbounded recursion + # arising from repeatedly yielding immediately ready deferreds. This while + # loop and the waiting variable solve that by manually unfolding the + # recursion. + + waiting = [True, # defgen is waiting for result? + None] # result + + while 1: + try: + result = next(g) + except StopIteration: + deferred.callback(result) + return deferred + except: + deferred.errback() + return deferred + + # Deferred.callback(Deferred) raises an error; we catch this case + # early here and give a nicer error message to the user in case + # they yield a Deferred. + if isinstance(result, Deferred): + return fail(TypeError("Yield waitForDeferred(d), not d!")) + + if isinstance(result, waitForDeferred): + # a waitForDeferred was yielded, get the result. + # Pass result in so it don't get changed going around the loop + # This isn't a problem for waiting, as it's only reused if + # gotResult has already been executed. + def gotResult(r, result=result): + result.result = r + if waiting[0]: + waiting[0] = False + waiting[1] = r + else: + _deferGenerator(g, deferred) + result.d.addBoth(gotResult) + if waiting[0]: + # Haven't called back yet, set flag so that we get reinvoked + # and return from the loop + waiting[0] = False + return deferred + # Reset waiting to initial values for next loop + waiting[0] = True + waiting[1] = None + + result = None + + + +@deprecated(Version('Twisted', 15, 0, 0), + "twisted.internet.defer.inlineCallbacks") +def deferredGenerator(f): + """ + L{deferredGenerator} and L{waitForDeferred} help you write + L{Deferred}-using code that looks like a regular sequential function. + Consider the use of L{inlineCallbacks} instead, which can accomplish + the same thing in a more concise manner. + + There are two important functions involved: L{waitForDeferred}, and + L{deferredGenerator}. They are used together, like this:: + + @deferredGenerator + def thingummy(): + thing = waitForDeferred(makeSomeRequestResultingInDeferred()) + yield thing + thing = thing.getResult() + print(thing) #the result! hoorj! + + L{waitForDeferred} returns something that you should immediately yield; when + your generator is resumed, calling C{thing.getResult()} will either give you + the result of the L{Deferred} if it was a success, or raise an exception if it + was a failure. Calling C{getResult} is B{absolutely mandatory}. If you do + not call it, I{your program will not work}. + + L{deferredGenerator} takes one of these waitForDeferred-using generator + functions and converts it into a function that returns a L{Deferred}. The + result of the L{Deferred} will be the last value that your generator yielded + unless the last value is a L{waitForDeferred} instance, in which case the + result will be L{None}. If the function raises an unhandled exception, the + L{Deferred} will errback instead. Remember that C{return result} won't work; + use C{yield result; return} in place of that. + + Note that not yielding anything from your generator will make the L{Deferred} + result in L{None}. Yielding a L{Deferred} from your generator is also an error + condition; always yield C{waitForDeferred(d)} instead. + + The L{Deferred} returned from your deferred generator may also errback if your + generator raised an exception. For example:: + + @deferredGenerator + def thingummy(): + thing = waitForDeferred(makeSomeRequestResultingInDeferred()) + yield thing + thing = thing.getResult() + if thing == 'I love Twisted': + # will become the result of the Deferred + yield 'TWISTED IS GREAT!' + return + else: + # will trigger an errback + raise Exception('DESTROY ALL LIFE') + + Put succinctly, these functions connect deferred-using code with this 'fake + blocking' style in both directions: L{waitForDeferred} converts from a + L{Deferred} to the 'blocking' style, and L{deferredGenerator} converts from the + 'blocking' style to a L{Deferred}. + """ + @wraps(f) + def unwindGenerator(*args, **kwargs): + return _deferGenerator(f(*args, **kwargs), Deferred()) + return unwindGenerator + + +## inlineCallbacks + + + +class _DefGen_Return(BaseException): + def __init__(self, value): + self.value = value + + + +def returnValue(val): + """ + Return val from a L{inlineCallbacks} generator. + + Note: this is currently implemented by raising an exception + derived from L{BaseException}. You might want to change any + 'except:' clauses to an 'except Exception:' clause so as not to + catch this exception. + + Also: while this function currently will work when called from + within arbitrary functions called from within the generator, do + not rely upon this behavior. + """ + raise _DefGen_Return(val) + + + +@attr.s +class _CancellationStatus(object): + """ + Cancellation status of an L{inlineCallbacks} invocation. + + @ivar waitingOn: the L{Deferred} being waited upon (which + L{_inlineCallbacks} must fill out before returning) + + @ivar deferred: the L{Deferred} to callback or errback when the generator + invocation has finished. + """ + + deferred = attr.ib() + waitingOn = attr.ib(default=None) + + + +@failure._extraneous +def _inlineCallbacks(result, g, status): + """ + Carry out the work of L{inlineCallbacks}. + + Iterate the generator produced by an C{@}L{inlineCallbacks}-decorated + function, C{g}, C{send()}ing it the results of each value C{yield}ed by + that generator, until a L{Deferred} is yielded, at which point a callback + is added to that L{Deferred} to call this function again. + + @param result: The last result seen by this generator. Note that this is + never a L{Deferred} - by the time this function is invoked, the + L{Deferred} has been called back and this will be a particular result + at a point in its callback chain. + + @param g: a generator object returned by calling a function or method + decorated with C{@}L{inlineCallbacks} + + @param status: a L{_CancellationStatus} tracking the current status of C{g} + """ + # This function is complicated by the need to prevent unbounded recursion + # arising from repeatedly yielding immediately ready deferreds. This while + # loop and the waiting variable solve that by manually unfolding the + # recursion. + + waiting = [True, # waiting for result? + None] # result + + while 1: + try: + # Send the last result back as the result of the yield expression. + isFailure = isinstance(result, failure.Failure) + if isFailure: + result = result.throwExceptionIntoGenerator(g) + else: + result = g.send(result) + except StopIteration as e: + # fell off the end, or "return" statement + status.deferred.callback(getattr(e, "value", None)) + return + except _DefGen_Return as e: + # returnValue() was called; time to give a result to the original + # Deferred. First though, let's try to identify the potentially + # confusing situation which results when returnValue() is + # accidentally invoked from a different function, one that wasn't + # decorated with @inlineCallbacks. + + # The traceback starts in this frame (the one for + # _inlineCallbacks); the next one down should be the application + # code. + appCodeTrace = exc_info()[2].tb_next + if isFailure: + # If we invoked this generator frame by throwing an exception + # into it, then throwExceptionIntoGenerator will consume an + # additional stack frame itself, so we need to skip that too. + appCodeTrace = appCodeTrace.tb_next + # Now that we've identified the frame being exited by the + # exception, let's figure out if returnValue was called from it + # directly. returnValue itself consumes a stack frame, so the + # application code will have a tb_next, but it will *not* have a + # second tb_next. + if appCodeTrace.tb_next.tb_next: + # If returnValue was invoked non-local to the frame which it is + # exiting, identify the frame that ultimately invoked + # returnValue so that we can warn the user, as this behavior is + # confusing. + ultimateTrace = appCodeTrace + while ultimateTrace.tb_next.tb_next: + ultimateTrace = ultimateTrace.tb_next + filename = ultimateTrace.tb_frame.f_code.co_filename + lineno = ultimateTrace.tb_lineno + warnings.warn_explicit( + "returnValue() in %r causing %r to exit: " + "returnValue should only be invoked by functions decorated " + "with inlineCallbacks" % ( + ultimateTrace.tb_frame.f_code.co_name, + appCodeTrace.tb_frame.f_code.co_name), + DeprecationWarning, filename, lineno) + status.deferred.callback(e.value) + return + except: + status.deferred.errback() + return + + if isinstance(result, Deferred): + # a deferred was yielded, get the result. + def gotResult(r): + if waiting[0]: + waiting[0] = False + waiting[1] = r + else: + # We are not waiting for deferred result any more + _inlineCallbacks(r, g, status) + + result.addBoth(gotResult) + if waiting[0]: + # Haven't called back yet, set flag so that we get reinvoked + # and return from the loop + waiting[0] = False + status.waitingOn = result + return + + result = waiting[1] + # Reset waiting to initial values for next loop. gotResult uses + # waiting, but this isn't a problem because gotResult is only + # executed once, and if it hasn't been executed yet, the return + # branch above would have been taken. + + waiting[0] = True + waiting[1] = None + + + +def _cancellableInlineCallbacks(g): + """ + Make an C{@}L{inlineCallbacks} cancellable. + + @param g: a generator object returned by calling a function or method + decorated with C{@}L{inlineCallbacks} + + @return: L{Deferred} for the C{@}L{inlineCallbacks} that is cancellable. + """ + def cancel(it): + it.callbacks, tmp = [], it.callbacks + it.addErrback(handleCancel) + it.callbacks.extend(tmp) + it.errback(_InternalInlineCallbacksCancelledError()) + deferred = Deferred(cancel) + status = _CancellationStatus(deferred) + def handleCancel(result): + """ + Propagate the cancellation of an C{@}L{inlineCallbacks} to the + L{Deferred} it is waiting on. + + @param result: An L{_InternalInlineCallbacksCancelledError} from + C{cancel()}. + @return: A new L{Deferred} that the C{@}L{inlineCallback} generator + can callback or errback through. + """ + result.trap(_InternalInlineCallbacksCancelledError) + status.deferred = Deferred(cancel) + # We would only end up here if the inlineCallback is waiting on + # another Deferred. It needs to be cancelled. + awaited = status.waitingOn + awaited.cancel() + return status.deferred + _inlineCallbacks(None, g, status) + return deferred + + + +class _InternalInlineCallbacksCancelledError(Exception): + """ + A unique exception used only in L{_cancellableInlineCallbacks} to verify + that an L{inlineCallbacks} is being cancelled as expected. + """ + + + +def inlineCallbacks(f): + """ + L{inlineCallbacks} helps you write L{Deferred}-using code that looks like a + regular sequential function. For example:: + + @inlineCallbacks + def thingummy(): + thing = yield makeSomeRequestResultingInDeferred() + print(thing) # the result! hoorj! + + When you call anything that results in a L{Deferred}, you can simply yield it; + your generator will automatically be resumed when the Deferred's result is + available. The generator will be sent the result of the L{Deferred} with the + 'send' method on generators, or if the result was a failure, 'throw'. + + Things that are not L{Deferred}s may also be yielded, and your generator + will be resumed with the same object sent back. This means C{yield} + performs an operation roughly equivalent to L{maybeDeferred}. + + Your inlineCallbacks-enabled generator will return a L{Deferred} object, which + will result in the return value of the generator (or will fail with a + failure object if your generator raises an unhandled exception). Note that + you can't use C{return result} to return a value; use C{returnValue(result)} + instead. Falling off the end of the generator, or simply using C{return} + will cause the L{Deferred} to have a result of L{None}. + + Be aware that L{returnValue} will not accept a L{Deferred} as a parameter. + If you believe the thing you'd like to return could be a L{Deferred}, do + this:: + + result = yield result + returnValue(result) + + The L{Deferred} returned from your deferred generator may errback if your + generator raised an exception:: + + @inlineCallbacks + def thingummy(): + thing = yield makeSomeRequestResultingInDeferred() + if thing == 'I love Twisted': + # will become the result of the Deferred + returnValue('TWISTED IS GREAT!') + else: + # will trigger an errback + raise Exception('DESTROY ALL LIFE') + + It is possible to use the C{return} statement instead of L{returnValue}:: + + @inlineCallbacks + def loadData(url): + response = yield makeRequest(url) + return json.loads(response) + + You can cancel the L{Deferred} returned from your L{inlineCallbacks} + generator before it is fired by your generator completing (either by + reaching its end, a C{return} statement, or by calling L{returnValue}). + A C{CancelledError} will be raised from the C{yield}ed L{Deferred} that + has been cancelled if that C{Deferred} does not otherwise suppress it. + """ + @wraps(f) + def unwindGenerator(*args, **kwargs): + try: + gen = f(*args, **kwargs) + except _DefGen_Return: + raise TypeError( + "inlineCallbacks requires %r to produce a generator; instead" + "caught returnValue being used in a non-generator" % (f,)) + if not isinstance(gen, types.GeneratorType): + raise TypeError( + "inlineCallbacks requires %r to produce a generator; " + "instead got %r" % (f, gen)) + return _cancellableInlineCallbacks(gen) + return unwindGenerator + + +## DeferredLock/DeferredQueue + +class _ConcurrencyPrimitive(object): + def __init__(self): + self.waiting = [] + + + def _releaseAndReturn(self, r): + self.release() + return r + + + def run(*args, **kwargs): + """ + Acquire, run, release. + + This function takes a callable as its first argument and any + number of other positional and keyword arguments. When the + lock or semaphore is acquired, the callable will be invoked + with those arguments. + + The callable may return a L{Deferred}; if it does, the lock or + semaphore won't be released until that L{Deferred} fires. + + @return: L{Deferred} of function result. + """ + if len(args) < 2: + if not args: + raise TypeError("run() takes at least 2 arguments, none given.") + raise TypeError("%s.run() takes at least 2 arguments, 1 given" % ( + args[0].__class__.__name__,)) + self, f = args[:2] + args = args[2:] + + def execute(ignoredResult): + d = maybeDeferred(f, *args, **kwargs) + d.addBoth(self._releaseAndReturn) + return d + + d = self.acquire() + d.addCallback(execute) + return d + + + def __aenter__(self): + """ + We can be used as an asynchronous context manager. + """ + return self.acquire() + + + def __aexit__(self, exc_type, exc_val, exc_tb): + self.release() + # We return False to indicate that we have not consumed the + # exception, if any. + return succeed(False) + + + +class DeferredLock(_ConcurrencyPrimitive): + """ + A lock for event driven systems. + + @ivar locked: C{True} when this Lock has been acquired, false at all other + times. Do not change this value, but it is useful to examine for the + equivalent of a "non-blocking" acquisition. + """ + + locked = False + + + def _cancelAcquire(self, d): + """ + Remove a deferred d from our waiting list, as the deferred has been + canceled. + + Note: We do not need to wrap this in a try/except to catch d not + being in self.waiting because this canceller will not be called if + d has fired. release() pops a deferred out of self.waiting and + calls it, so the canceller will no longer be called. + + @param d: The deferred that has been canceled. + """ + self.waiting.remove(d) + + + def acquire(self): + """ + Attempt to acquire the lock. Returns a L{Deferred} that fires on + lock acquisition with the L{DeferredLock} as the value. If the lock + is locked, then the Deferred is placed at the end of a waiting list. + + @return: a L{Deferred} which fires on lock acquisition. + @rtype: a L{Deferred} + """ + d = Deferred(canceller=self._cancelAcquire) + if self.locked: + self.waiting.append(d) + else: + self.locked = True + d.callback(self) + return d + + + def release(self): + """ + Release the lock. If there is a waiting list, then the first + L{Deferred} in that waiting list will be called back. + + Should be called by whomever did the L{acquire}() when the shared + resource is free. + """ + assert self.locked, "Tried to release an unlocked lock" + self.locked = False + if self.waiting: + # someone is waiting to acquire lock + self.locked = True + d = self.waiting.pop(0) + d.callback(self) + + + +class DeferredSemaphore(_ConcurrencyPrimitive): + """ + A semaphore for event driven systems. + + If you are looking into this as a means of limiting parallelism, you might + find L{twisted.internet.task.Cooperator} more useful. + + @ivar limit: At most this many users may acquire this semaphore at + once. + @type limit: L{int} + + @ivar tokens: The difference between C{limit} and the number of users + which have currently acquired this semaphore. + @type tokens: L{int} + """ + + def __init__(self, tokens): + """ + @param tokens: initial value of L{tokens} and L{limit} + @type tokens: L{int} + """ + _ConcurrencyPrimitive.__init__(self) + if tokens < 1: + raise ValueError("DeferredSemaphore requires tokens >= 1") + self.tokens = tokens + self.limit = tokens + + + def _cancelAcquire(self, d): + """ + Remove a deferred d from our waiting list, as the deferred has been + canceled. + + Note: We do not need to wrap this in a try/except to catch d not + being in self.waiting because this canceller will not be called if + d has fired. release() pops a deferred out of self.waiting and + calls it, so the canceller will no longer be called. + + @param d: The deferred that has been canceled. + """ + self.waiting.remove(d) + + + def acquire(self): + """ + Attempt to acquire the token. + + @return: a L{Deferred} which fires on token acquisition. + """ + assert self.tokens >= 0, "Internal inconsistency?? tokens should never be negative" + d = Deferred(canceller=self._cancelAcquire) + if not self.tokens: + self.waiting.append(d) + else: + self.tokens = self.tokens - 1 + d.callback(self) + return d + + + def release(self): + """ + Release the token. + + Should be called by whoever did the L{acquire}() when the shared + resource is free. + """ + assert self.tokens < self.limit, "Someone released me too many times: too many tokens!" + self.tokens = self.tokens + 1 + if self.waiting: + # someone is waiting to acquire token + self.tokens = self.tokens - 1 + d = self.waiting.pop(0) + d.callback(self) + + + +class QueueOverflow(Exception): + pass + + + +class QueueUnderflow(Exception): + pass + + + +class DeferredQueue(object): + """ + An event driven queue. + + Objects may be added as usual to this queue. When an attempt is + made to retrieve an object when the queue is empty, a L{Deferred} is + returned which will fire when an object becomes available. + + @ivar size: The maximum number of objects to allow into the queue + at a time. When an attempt to add a new object would exceed this + limit, L{QueueOverflow} is raised synchronously. L{None} for no limit. + + @ivar backlog: The maximum number of L{Deferred} gets to allow at + one time. When an attempt is made to get an object which would + exceed this limit, L{QueueUnderflow} is raised synchronously. L{None} + for no limit. + """ + + def __init__(self, size=None, backlog=None): + self.waiting = [] + self.pending = [] + self.size = size + self.backlog = backlog + + + def _cancelGet(self, d): + """ + Remove a deferred d from our waiting list, as the deferred has been + canceled. + + Note: We do not need to wrap this in a try/except to catch d not + being in self.waiting because this canceller will not be called if + d has fired. put() pops a deferred out of self.waiting and calls + it, so the canceller will no longer be called. + + @param d: The deferred that has been canceled. + """ + self.waiting.remove(d) + + + def put(self, obj): + """ + Add an object to this queue. + + @raise QueueOverflow: Too many objects are in this queue. + """ + if self.waiting: + self.waiting.pop(0).callback(obj) + elif self.size is None or len(self.pending) < self.size: + self.pending.append(obj) + else: + raise QueueOverflow() + + + def get(self): + """ + Attempt to retrieve and remove an object from the queue. + + @return: a L{Deferred} which fires with the next object available in + the queue. + + @raise QueueUnderflow: Too many (more than C{backlog}) + L{Deferred}s are already waiting for an object from this queue. + """ + if self.pending: + return succeed(self.pending.pop(0)) + elif self.backlog is None or len(self.waiting) < self.backlog: + d = Deferred(canceller=self._cancelGet) + self.waiting.append(d) + return d + else: + raise QueueUnderflow() + + + +class AlreadyTryingToLockError(Exception): + """ + Raised when L{DeferredFilesystemLock.deferUntilLocked} is called twice on a + single L{DeferredFilesystemLock}. + """ + + + +class DeferredFilesystemLock(lockfile.FilesystemLock): + """ + A L{FilesystemLock} that allows for a L{Deferred} to be fired when the lock is + acquired. + + @ivar _scheduler: The object in charge of scheduling retries. In this + implementation this is parameterized for testing. + + @ivar _interval: The retry interval for an L{IReactorTime} based scheduler. + + @ivar _tryLockCall: A L{DelayedCall} based on C{_interval} that will manage + the next retry for acquiring the lock. + + @ivar _timeoutCall: A L{DelayedCall} based on C{deferUntilLocked}'s timeout + argument. This is in charge of timing out our attempt to acquire the + lock. + """ + _interval = 1 + _tryLockCall = None + _timeoutCall = None + + + def __init__(self, name, scheduler=None): + """ + @param name: The name of the lock to acquire + @param scheduler: An object which provides L{IReactorTime} + """ + lockfile.FilesystemLock.__init__(self, name) + + if scheduler is None: + from twisted.internet import reactor + scheduler = reactor + + self._scheduler = scheduler + + + def deferUntilLocked(self, timeout=None): + """ + Wait until we acquire this lock. This method is not safe for + concurrent use. + + @type timeout: L{float} or L{int} + @param timeout: the number of seconds after which to time out if the + lock has not been acquired. + + @return: a L{Deferred} which will callback when the lock is acquired, or + errback with a L{TimeoutError} after timing out or an + L{AlreadyTryingToLockError} if the L{deferUntilLocked} has already + been called and not successfully locked the file. + """ + if self._tryLockCall is not None: + return fail( + AlreadyTryingToLockError( + "deferUntilLocked isn't safe for concurrent use.")) + + def _cancelLock(reason): + """ + Cancel a L{DeferredFilesystemLock.deferUntilLocked} call. + + @type reason: L{failure.Failure} + @param reason: The reason why the call is cancelled. + """ + self._tryLockCall.cancel() + self._tryLockCall = None + if self._timeoutCall is not None and self._timeoutCall.active(): + self._timeoutCall.cancel() + self._timeoutCall = None + + if self.lock(): + d.callback(None) + else: + d.errback(reason) + + d = Deferred(lambda deferred: _cancelLock(CancelledError())) + + def _tryLock(): + if self.lock(): + if self._timeoutCall is not None: + self._timeoutCall.cancel() + self._timeoutCall = None + + self._tryLockCall = None + + d.callback(None) + else: + if timeout is not None and self._timeoutCall is None: + reason = failure.Failure(TimeoutError( + "Timed out acquiring lock: %s after %fs" % ( + self.name, + timeout))) + self._timeoutCall = self._scheduler.callLater( + timeout, _cancelLock, reason) + + self._tryLockCall = self._scheduler.callLater( + self._interval, _tryLock) + + _tryLock() + + return d + + + +__all__ = ["Deferred", "DeferredList", "succeed", "fail", "FAILURE", "SUCCESS", + "AlreadyCalledError", "TimeoutError", "gatherResults", + "maybeDeferred", "ensureDeferred", + "waitForDeferred", "deferredGenerator", "inlineCallbacks", + "returnValue", + "DeferredLock", "DeferredSemaphore", "DeferredQueue", + "DeferredFilesystemLock", "AlreadyTryingToLockError", + "CancelledError", + ] + diff --git a/contrib/python/Twisted/py2/twisted/internet/endpoints.py b/contrib/python/Twisted/py2/twisted/internet/endpoints.py new file mode 100644 index 00000000000..11925bbd4a4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/endpoints.py @@ -0,0 +1,2269 @@ +# -*- test-case-name: twisted.internet.test.test_endpoints -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementations of L{IStreamServerEndpoint} and L{IStreamClientEndpoint} that +wrap the L{IReactorTCP}, L{IReactorSSL}, and L{IReactorUNIX} interfaces. + +This also implements an extensible mini-language for describing endpoints, +parsed by the L{clientFromString} and L{serverFromString} functions. + +@since: 10.1 +""" + +from __future__ import division, absolute_import + +import os +import re +import socket +from unicodedata import normalize +import warnings + +from constantly import NamedConstant, Names +from incremental import Version + +from zope.interface import implementer, directlyProvides, provider + +from twisted.internet import interfaces, defer, error, fdesc, threads +from twisted.internet.abstract import isIPv6Address, isIPAddress +from twisted.internet.address import ( + _ProcessAddress, HostnameAddress, IPv4Address, IPv6Address +) +from twisted.internet.interfaces import ( + IStreamServerEndpointStringParser, + IStreamClientEndpointStringParserWithReactor, IResolutionReceiver, + IReactorPluggableNameResolver, + IHostnameResolver, +) +from twisted.internet.protocol import ClientFactory, Factory +from twisted.internet.protocol import ProcessProtocol, Protocol + +try: + from twisted.internet.stdio import StandardIO, PipeAddress +except ImportError: + # fallback if pywin32 is not installed + StandardIO = None + PipeAddress = None + +from twisted.internet.task import LoopingCall +from twisted.internet._resolver import HostResolution +from twisted.logger import Logger +from twisted.plugin import IPlugin, getPlugins +from twisted.python import deprecate, log +from twisted.python.compat import nativeString, unicode, _matchingString +from twisted.python.components import proxyForInterface +from twisted.python.failure import Failure +from twisted.python.filepath import FilePath +from twisted.python.compat import iterbytes +from twisted.internet.defer import Deferred +from twisted.python.systemd import ListenFDs + +from ._idna import _idnaBytes, _idnaText + +try: + from twisted.protocols.tls import TLSMemoryBIOFactory + from twisted.internet.ssl import ( + optionsForClientTLS, PrivateCertificate, Certificate, KeyPair, + CertificateOptions, trustRootFromCertificates + ) + from OpenSSL.SSL import Error as SSLError +except ImportError: + TLSMemoryBIOFactory = None + +__all__ = ["clientFromString", "serverFromString", + "TCP4ServerEndpoint", "TCP6ServerEndpoint", + "TCP4ClientEndpoint", "TCP6ClientEndpoint", + "UNIXServerEndpoint", "UNIXClientEndpoint", + "SSL4ServerEndpoint", "SSL4ClientEndpoint", + "AdoptedStreamServerEndpoint", "StandardIOEndpoint", + "ProcessEndpoint", "HostnameEndpoint", + "StandardErrorBehavior", "connectProtocol", + "wrapClientTLS"] + + + +class _WrappingProtocol(Protocol): + """ + Wrap another protocol in order to notify my user when a connection has + been made. + """ + + def __init__(self, connectedDeferred, wrappedProtocol): + """ + @param connectedDeferred: The L{Deferred} that will callback + with the C{wrappedProtocol} when it is connected. + + @param wrappedProtocol: An L{IProtocol} provider that will be + connected. + """ + self._connectedDeferred = connectedDeferred + self._wrappedProtocol = wrappedProtocol + + for iface in [interfaces.IHalfCloseableProtocol, + interfaces.IFileDescriptorReceiver, + interfaces.IHandshakeListener]: + if iface.providedBy(self._wrappedProtocol): + directlyProvides(self, iface) + + + def logPrefix(self): + """ + Transparently pass through the wrapped protocol's log prefix. + """ + if interfaces.ILoggingContext.providedBy(self._wrappedProtocol): + return self._wrappedProtocol.logPrefix() + return self._wrappedProtocol.__class__.__name__ + + + def connectionMade(self): + """ + Connect the C{self._wrappedProtocol} to our C{self.transport} and + callback C{self._connectedDeferred} with the C{self._wrappedProtocol} + """ + self._wrappedProtocol.makeConnection(self.transport) + self._connectedDeferred.callback(self._wrappedProtocol) + + + def dataReceived(self, data): + """ + Proxy C{dataReceived} calls to our C{self._wrappedProtocol} + """ + return self._wrappedProtocol.dataReceived(data) + + + def fileDescriptorReceived(self, descriptor): + """ + Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol} + """ + return self._wrappedProtocol.fileDescriptorReceived(descriptor) + + + def connectionLost(self, reason): + """ + Proxy C{connectionLost} calls to our C{self._wrappedProtocol} + """ + return self._wrappedProtocol.connectionLost(reason) + + + def readConnectionLost(self): + """ + Proxy L{IHalfCloseableProtocol.readConnectionLost} to our + C{self._wrappedProtocol} + """ + self._wrappedProtocol.readConnectionLost() + + + def writeConnectionLost(self): + """ + Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our + C{self._wrappedProtocol} + """ + self._wrappedProtocol.writeConnectionLost() + + + def handshakeCompleted(self): + """ + Proxy L{interfaces.IHandshakeListener} to our + C{self._wrappedProtocol}. + """ + self._wrappedProtocol.handshakeCompleted() + + + +class _WrappingFactory(ClientFactory): + """ + Wrap a factory in order to wrap the protocols it builds. + + @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol + method will be called and whose resulting protocol will be wrapped. + + @ivar _onConnection: A L{Deferred} that fires when the protocol is + connected + + @ivar _connector: A L{connector } + that is managing the current or previous connection attempt. + """ + protocol = _WrappingProtocol + + def __init__(self, wrappedFactory): + """ + @param wrappedFactory: A provider of I{IProtocolFactory} whose + buildProtocol method will be called and whose resulting protocol + will be wrapped. + """ + self._wrappedFactory = wrappedFactory + self._onConnection = defer.Deferred(canceller=self._canceller) + + + def startedConnecting(self, connector): + """ + A connection attempt was started. Remember the connector which started + said attempt, for use later. + """ + self._connector = connector + + + def _canceller(self, deferred): + """ + The outgoing connection attempt was cancelled. Fail that L{Deferred} + with an L{error.ConnectingCancelledError}. + + @param deferred: The L{Deferred } that was cancelled; + should be the same as C{self._onConnection}. + @type deferred: L{Deferred } + + @note: This relies on startedConnecting having been called, so it may + seem as though there's a race condition where C{_connector} may not + have been set. However, using public APIs, this condition is + impossible to catch, because a connection API + (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a + L{_WrappingFactory}'s L{Deferred } is returned to + C{connect()}'s caller. + + @return: L{None} + """ + deferred.errback( + error.ConnectingCancelledError( + self._connector.getDestination())) + self._connector.stopConnecting() + + + def doStart(self): + """ + Start notifications are passed straight through to the wrapped factory. + """ + self._wrappedFactory.doStart() + + + def doStop(self): + """ + Stop notifications are passed straight through to the wrapped factory. + """ + self._wrappedFactory.doStop() + + + def buildProtocol(self, addr): + """ + Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback the + C{self._onConnection} L{Deferred} if the wrapped factory raises an + exception or returns L{None}. + + @return: An instance of L{_WrappingProtocol} or L{None} + """ + try: + proto = self._wrappedFactory.buildProtocol(addr) + if proto is None: + raise error.NoProtocol() + except: + self._onConnection.errback() + else: + return self.protocol(self._onConnection, proto) + + + def clientConnectionFailed(self, connector, reason): + """ + Errback the C{self._onConnection} L{Deferred} when the + client connection fails. + """ + if not self._onConnection.called: + self._onConnection.errback(reason) + + + +@implementer(interfaces.IStreamServerEndpoint) +class StandardIOEndpoint(object): + """ + A Standard Input/Output endpoint + + @ivar _stdio: a callable, like L{stdio.StandardIO}, which takes an + L{IProtocol} provider and a C{reactor} keyword argument (interface + dependent upon your platform). + """ + + _stdio = StandardIO + + def __init__(self, reactor): + """ + @param reactor: The reactor for the endpoint. + """ + self._reactor = reactor + + + def listen(self, stdioProtocolFactory): + """ + Implement L{IStreamServerEndpoint.listen} to listen on stdin/stdout + """ + return defer.execute(self._stdio, + stdioProtocolFactory.buildProtocol(PipeAddress()), + reactor=self._reactor) + + + +class _IProcessTransportWithConsumerAndProducer(interfaces.IProcessTransport, + interfaces.IConsumer, + interfaces.IPushProducer): + """ + An L{_IProcessTransportWithConsumerAndProducer} combines various interfaces + to work around the issue that L{interfaces.IProcessTransport} is + incompletely defined and doesn't specify flow-control interfaces, and that + L{proxyForInterface} doesn't allow for multiple interfaces. + """ + + + +class _ProcessEndpointTransport( + proxyForInterface(_IProcessTransportWithConsumerAndProducer, + '_process')): + """ + An L{ITransport}, L{IProcessTransport}, L{IConsumer}, and L{IPushProducer} + provider for the L{IProtocol} instance passed to the process endpoint. + + @ivar _process: An active process transport which will be used by write + methods on this object to write data to a child process. + @type _process: L{interfaces.IProcessTransport} provider + """ + + + +class _WrapIProtocol(ProcessProtocol): + """ + An L{IProcessProtocol} provider that wraps an L{IProtocol}. + + @ivar transport: A L{_ProcessEndpointTransport} provider that is hooked to + the wrapped L{IProtocol} provider. + + @see: L{protocol.ProcessProtocol} + """ + + def __init__(self, proto, executable, errFlag): + """ + @param proto: An L{IProtocol} provider. + @param errFlag: A constant belonging to L{StandardErrorBehavior} + that determines if stderr is logged or dropped. + @param executable: The file name (full path) to spawn. + """ + self.protocol = proto + self.errFlag = errFlag + self.executable = executable + + + def makeConnection(self, process): + """ + Call L{IProtocol} provider's makeConnection method with an + L{ITransport} provider. + + @param process: An L{IProcessTransport} provider. + """ + self.transport = _ProcessEndpointTransport(process) + return self.protocol.makeConnection(self.transport) + + + def childDataReceived(self, childFD, data): + """ + This is called with data from the process's stdout or stderr pipes. It + checks the status of the errFlag to setermine if stderr should be + logged (default) or dropped. + """ + if childFD == 1: + return self.protocol.dataReceived(data) + elif childFD == 2 and self.errFlag == StandardErrorBehavior.LOG: + log.msg( + format="Process %(executable)r wrote stderr unhandled by " + "%(protocol)s: %(data)s", + executable=self.executable, protocol=self.protocol, + data=data) + + + def processEnded(self, reason): + """ + If the process ends with L{error.ProcessDone}, this method calls the + L{IProtocol} provider's L{connectionLost} with a + L{error.ConnectionDone} + + @see: L{ProcessProtocol.processEnded} + """ + if (reason.check(error.ProcessDone) == error.ProcessDone) and ( + reason.value.status == 0): + return self.protocol.connectionLost( + Failure(error.ConnectionDone())) + else: + return self.protocol.connectionLost(reason) + + + +class StandardErrorBehavior(Names): + """ + Constants used in ProcessEndpoint to decide what to do with stderr. + + @cvar LOG: Indicates that stderr is to be logged. + @cvar DROP: Indicates that stderr is to be dropped (and not logged). + + @since: 13.1 + """ + LOG = NamedConstant() + DROP = NamedConstant() + + + +@implementer(interfaces.IStreamClientEndpoint) +class ProcessEndpoint(object): + """ + An endpoint for child processes + + @ivar _spawnProcess: A hook used for testing the spawning of child process. + + @since: 13.1 + """ + def __init__(self, reactor, executable, args=(), env={}, path=None, + uid=None, gid=None, usePTY=0, childFDs=None, + errFlag=StandardErrorBehavior.LOG): + """ + See L{IReactorProcess.spawnProcess}. + + @param errFlag: Determines if stderr should be logged. + @type errFlag: L{endpoints.StandardErrorBehavior} + """ + self._reactor = reactor + self._executable = executable + self._args = args + self._env = env + self._path = path + self._uid = uid + self._gid = gid + self._usePTY = usePTY + self._childFDs = childFDs + self._errFlag = errFlag + self._spawnProcess = self._reactor.spawnProcess + + + def connect(self, protocolFactory): + """ + Implement L{IStreamClientEndpoint.connect} to launch a child process + and connect it to a protocol created by C{protocolFactory}. + + @param protocolFactory: A factory for an L{IProtocol} provider which + will be notified of all events related to the created process. + """ + proto = protocolFactory.buildProtocol(_ProcessAddress()) + try: + self._spawnProcess( + _WrapIProtocol(proto, self._executable, self._errFlag), + self._executable, self._args, self._env, self._path, self._uid, + self._gid, self._usePTY, self._childFDs) + except: + return defer.fail() + else: + return defer.succeed(proto) + + + +@implementer(interfaces.IStreamServerEndpoint) +class _TCPServerEndpoint(object): + """ + A TCP server endpoint interface + """ + + def __init__(self, reactor, port, backlog, interface): + """ + @param reactor: An L{IReactorTCP} provider. + + @param port: The port number used for listening + @type port: int + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to + @type interface: str + """ + self._reactor = reactor + self._port = port + self._backlog = backlog + self._interface = interface + + + def listen(self, protocolFactory): + """ + Implement L{IStreamServerEndpoint.listen} to listen on a TCP + socket + """ + return defer.execute(self._reactor.listenTCP, + self._port, + protocolFactory, + backlog=self._backlog, + interface=self._interface) + + + +class TCP4ServerEndpoint(_TCPServerEndpoint): + """ + Implements TCP server endpoint with an IPv4 configuration + """ + def __init__(self, reactor, port, backlog=50, interface=''): + """ + @param reactor: An L{IReactorTCP} provider. + + @param port: The port number used for listening + @type port: int + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to, defaults to '' (all) + @type interface: str + """ + _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) + + + +class TCP6ServerEndpoint(_TCPServerEndpoint): + """ + Implements TCP server endpoint with an IPv6 configuration + """ + def __init__(self, reactor, port, backlog=50, interface='::'): + """ + @param reactor: An L{IReactorTCP} provider. + + @param port: The port number used for listening + @type port: int + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to, defaults to C{::} (all) + @type interface: str + """ + _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) + + + +@implementer(interfaces.IStreamClientEndpoint) +class TCP4ClientEndpoint(object): + """ + TCP client endpoint with an IPv4 configuration. + """ + + def __init__(self, reactor, host, port, timeout=30, bindAddress=None): + """ + @param reactor: An L{IReactorTCP} provider + + @param host: A hostname, used when connecting + @type host: str + + @param port: The port number, used when connecting + @type port: int + + @param timeout: The number of seconds to wait before assuming the + connection has failed. + @type timeout: L{float} or L{int} + + @param bindAddress: A (host, port) tuple of local address to bind to, + or None. + @type bindAddress: tuple + """ + self._reactor = reactor + self._host = host + self._port = port + self._timeout = timeout + self._bindAddress = bindAddress + + + def connect(self, protocolFactory): + """ + Implement L{IStreamClientEndpoint.connect} to connect via TCP. + """ + try: + wf = _WrappingFactory(protocolFactory) + self._reactor.connectTCP( + self._host, self._port, wf, + timeout=self._timeout, bindAddress=self._bindAddress) + return wf._onConnection + except: + return defer.fail() + + + +@implementer(interfaces.IStreamClientEndpoint) +class TCP6ClientEndpoint(object): + """ + TCP client endpoint with an IPv6 configuration. + + @ivar _getaddrinfo: A hook used for testing name resolution. + + @ivar _deferToThread: A hook used for testing deferToThread. + + @ivar _GAI_ADDRESS: Index of the address portion in result of + getaddrinfo to be used. + + @ivar _GAI_ADDRESS_HOST: Index of the actual host-address in the + 5-tuple L{_GAI_ADDRESS}. + """ + + _getaddrinfo = staticmethod(socket.getaddrinfo) + _deferToThread = staticmethod(threads.deferToThread) + _GAI_ADDRESS = 4 + _GAI_ADDRESS_HOST = 0 + + def __init__(self, reactor, host, port, timeout=30, bindAddress=None): + """ + @param host: An IPv6 address literal or a hostname with an + IPv6 address + + @see: L{twisted.internet.interfaces.IReactorTCP.connectTCP} + """ + self._reactor = reactor + self._host = host + self._port = port + self._timeout = timeout + self._bindAddress = bindAddress + + + def connect(self, protocolFactory): + """ + Implement L{IStreamClientEndpoint.connect} to connect via TCP, + once the hostname resolution is done. + """ + if isIPv6Address(self._host): + d = self._resolvedHostConnect(self._host, protocolFactory) + else: + d = self._nameResolution(self._host) + d.addCallback(lambda result: result[0][self._GAI_ADDRESS] + [self._GAI_ADDRESS_HOST]) + d.addCallback(self._resolvedHostConnect, protocolFactory) + return d + + + def _nameResolution(self, host): + """ + Resolve the hostname string into a tuple containing the host + IPv6 address. + """ + return self._deferToThread( + self._getaddrinfo, host, 0, socket.AF_INET6) + + + def _resolvedHostConnect(self, resolvedHost, protocolFactory): + """ + Connect to the server using the resolved hostname. + """ + try: + wf = _WrappingFactory(protocolFactory) + self._reactor.connectTCP(resolvedHost, self._port, wf, + timeout=self._timeout, bindAddress=self._bindAddress) + return wf._onConnection + except: + return defer.fail() + + + +@implementer(IHostnameResolver) +class _SimpleHostnameResolver(object): + """ + An L{IHostnameResolver} provider that invokes a provided callable + to resolve hostnames. + + @ivar _nameResolution: the callable L{resolveHostName} invokes to + resolve hostnames. + @type _nameResolution: A L{callable} that accepts two arguments: + the host to resolve and the port number to include in the + result. + """ + _log = Logger() + + def __init__(self, nameResolution): + """ + Create a L{_SimpleHostnameResolver} instance. + """ + self._nameResolution = nameResolution + + + def resolveHostName(self, resolutionReceiver, + hostName, + portNumber=0, + addressTypes=None, + transportSemantics='TCP'): + """ + Initiate a hostname resolution. + + @param resolutionReceiver: an object that will receive each resolved + address as it arrives. + @type resolutionReceiver: L{IResolutionReceiver} + + @param hostName: see interface + + @param portNumber: see interface + + @param addressTypes: Ignored in this implementation. + + @param transportSemantics: Ignored in this implementation. + + @return: The resolution in progress. + @rtype: L{IResolutionReceiver} + """ + resolutionReceiver.resolutionBegan(HostResolution(hostName)) + d = self._nameResolution(hostName, portNumber) + + def cbDeliver(gairesult): + for family, socktype, proto, canonname, sockaddr in gairesult: + if family == socket.AF_INET6: + resolutionReceiver.addressResolved( + IPv6Address('TCP', *sockaddr)) + elif family == socket.AF_INET: + resolutionReceiver.addressResolved( + IPv4Address('TCP', *sockaddr)) + + + def ebLog(error): + self._log.failure("while looking up {name} with {callable}", + error, name=hostName, + callable=self._nameResolution) + + d.addCallback(cbDeliver) + d.addErrback(ebLog) + d.addBoth(lambda ignored: resolutionReceiver.resolutionComplete()) + return resolutionReceiver + + + + +@implementer(interfaces.IStreamClientEndpoint) +class HostnameEndpoint(object): + """ + A name-based endpoint that connects to the fastest amongst the resolved + host addresses. + + @cvar _DEFAULT_ATTEMPT_DELAY: The default time to use between attempts, in + seconds, when no C{attemptDelay} is given to + L{HostnameEndpoint.__init__}. + + @ivar _hostText: the textual representation of the hostname passed to the + constructor. Used to pass to the reactor's hostname resolver. + @type _hostText: L{unicode} + + @ivar _hostBytes: the encoded bytes-representation of the hostname passed + to the constructor. Used to construct the L{HostnameAddress} + associated with this endpoint. + @type _hostBytes: L{bytes} + + @ivar _hostStr: the native-string representation of the hostname passed to + the constructor, used for exception construction + @type _hostStr: native L{str} + + @ivar _badHostname: a flag - hopefully false! - indicating that an invalid + hostname was passed to the constructor. This might be a textual + hostname that isn't valid IDNA, or non-ASCII bytes. + @type _badHostname: L{bool} + """ + _getaddrinfo = staticmethod(socket.getaddrinfo) + _deferToThread = staticmethod(threads.deferToThread) + _DEFAULT_ATTEMPT_DELAY = 0.3 + + def __init__(self, reactor, host, port, timeout=30, bindAddress=None, + attemptDelay=None): + """ + Create a L{HostnameEndpoint}. + + @param reactor: The reactor to use for connections and delayed calls. + @type reactor: provider of L{IReactorTCP}, L{IReactorTime} and either + L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}. + + @param host: A hostname to connect to. + @type host: L{bytes} or L{unicode} + + @param port: The port number to connect to. + @type port: L{int} + + @param timeout: For each individual connection attempt, the number of + seconds to wait before assuming the connection has failed. + @type timeout: L{float} or L{int} + + @param bindAddress: the local address of the network interface to make + the connections from. + @type bindAddress: L{bytes} + + @param attemptDelay: The number of seconds to delay between connection + attempts. + @type attemptDelay: L{float} + + @see: L{twisted.internet.interfaces.IReactorTCP.connectTCP} + """ + + self._reactor = reactor + self._nameResolver = self._getNameResolverAndMaybeWarn(reactor) + [self._badHostname, self._hostBytes, self._hostText] = ( + self._hostAsBytesAndText(host) + ) + self._hostStr = self._hostBytes if bytes is str else self._hostText + self._port = port + self._timeout = timeout + self._bindAddress = bindAddress + if attemptDelay is None: + attemptDelay = self._DEFAULT_ATTEMPT_DELAY + self._attemptDelay = attemptDelay + + + def __repr__(self): + """ + Produce a string representation of the L{HostnameEndpoint}. + + @return: A L{str} + """ + if self._badHostname: + # Use the backslash-encoded version of the string passed to the + # constructor, which is already a native string. + host = self._hostStr + elif isIPv6Address(self._hostStr): + host = '[{}]'.format(self._hostStr) + else: + # Convert the bytes representation to a native string to ensure + # that we display the punycoded version of the hostname, which is + # more useful than any IDN version as it can be easily copy-pasted + # into debugging tools. + host = nativeString(self._hostBytes) + return "".join([""]) + + + def _getNameResolverAndMaybeWarn(self, reactor): + """ + Retrieve a C{nameResolver} callable and warn the caller's + caller that using a reactor which doesn't provide + L{IReactorPluggableNameResolver} is deprecated. + + @param reactor: The reactor to check. + + @return: A L{IHostnameResolver} provider. + """ + if not IReactorPluggableNameResolver.providedBy(reactor): + warningString = deprecate.getDeprecationWarningString( + reactor.__class__, + Version('Twisted', 17, 5, 0), + format=("Passing HostnameEndpoint a reactor that does not" + " provide IReactorPluggableNameResolver (%(fqpn)s)" + " was deprecated in %(version)s"), + replacement=("a reactor that provides" + " IReactorPluggableNameResolver"), + ) + warnings.warn(warningString, DeprecationWarning, stacklevel=3) + return _SimpleHostnameResolver(self._fallbackNameResolution) + return reactor.nameResolver + + + @staticmethod + def _hostAsBytesAndText(host): + """ + For various reasons (documented in the C{@ivar}'s in the class + docstring) we need both a textual and a binary representation of the + hostname given to the constructor. For compatibility and convenience, + we accept both textual and binary representations of the hostname, save + the form that was passed, and convert into the other form. This is + mostly just because L{HostnameAddress} chose somewhat poorly to define + its attribute as bytes; hopefully we can find a compatible way to clean + this up in the future and just operate in terms of text internally. + + @param host: A hostname to convert. + @type host: L{bytes} or C{str} + + @return: a 3-tuple of C{(invalid, bytes, text)} where C{invalid} is a + boolean indicating the validity of the hostname, C{bytes} is a + binary representation of C{host}, and C{text} is a textual + representation of C{host}. + """ + if isinstance(host, bytes): + if isIPAddress(host) or isIPv6Address(host): + return False, host, host.decode("ascii") + else: + try: + return False, host, _idnaText(host) + except UnicodeError: + # Convert the host to _some_ kind of text, to handle below. + host = host.decode("charmap") + else: + host = normalize('NFC', host) + if isIPAddress(host) or isIPv6Address(host): + return False, host.encode("ascii"), host + else: + try: + return False, _idnaBytes(host), host + except UnicodeError: + pass + # `host` has been converted to text by this point either way; it's + # invalid as a hostname, and so may contain unprintable characters and + # such. escape it with backslashes so the user can get _some_ guess as + # to what went wrong. + asciibytes = host.encode('ascii', 'backslashreplace') + return True, asciibytes, asciibytes.decode('ascii') + + + def connect(self, protocolFactory): + """ + Attempts a connection to each resolved address, and returns a + connection which is established first. + + @param protocolFactory: The protocol factory whose protocol + will be connected. + @type protocolFactory: + L{IProtocolFactory} + + @return: A L{Deferred} that fires with the connected protocol + or fails a connection-related error. + """ + if self._badHostname: + return defer.fail( + ValueError("invalid hostname: {}".format(self._hostStr)) + ) + + d = Deferred() + addresses = [] + @provider(IResolutionReceiver) + class EndpointReceiver(object): + @staticmethod + def resolutionBegan(resolutionInProgress): + pass + @staticmethod + def addressResolved(address): + addresses.append(address) + @staticmethod + def resolutionComplete(): + d.callback(addresses) + + self._nameResolver.resolveHostName( + EndpointReceiver, self._hostText, portNumber=self._port + ) + + d.addErrback(lambda ignored: defer.fail(error.DNSLookupError( + "Couldn't find the hostname '{}'".format(self._hostStr)))) + @d.addCallback + def resolvedAddressesToEndpoints(addresses): + # Yield an endpoint for every address resolved from the name. + for eachAddress in addresses: + if isinstance(eachAddress, IPv6Address): + yield TCP6ClientEndpoint( + self._reactor, eachAddress.host, eachAddress.port, + self._timeout, self._bindAddress + ) + if isinstance(eachAddress, IPv4Address): + yield TCP4ClientEndpoint( + self._reactor, eachAddress.host, eachAddress.port, + self._timeout, self._bindAddress + ) + d.addCallback(list) + + def _canceller(d): + # This canceller must remain defined outside of + # `startConnectionAttempts`, because Deferred should not + # participate in cycles with their cancellers; that would create a + # potentially problematic circular reference and possibly + # gc.garbage. + d.errback(error.ConnectingCancelledError( + HostnameAddress(self._hostBytes, self._port))) + + @d.addCallback + def startConnectionAttempts(endpoints): + """ + Given a sequence of endpoints obtained via name resolution, start + connecting to a new one every C{self._attemptDelay} seconds until + one of the connections succeeds, all of them fail, or the attempt + is cancelled. + + @param endpoints: a list of all the endpoints we might try to + connect to, as determined by name resolution. + @type endpoints: L{list} of L{IStreamServerEndpoint} + + @return: a Deferred that fires with the result of the + C{endpoint.connect} method that completes the fastest, or fails + with the first connection error it encountered if none of them + succeed. + @rtype: L{Deferred} failing with L{error.ConnectingCancelledError} + or firing with L{IProtocol} + """ + if not endpoints: + raise error.DNSLookupError( + "no results for hostname lookup: {}".format(self._hostStr) + ) + iterEndpoints = iter(endpoints) + pending = [] + failures = [] + winner = defer.Deferred(canceller=_canceller) + + def checkDone(): + if pending or checkDone.completed or checkDone.endpointsLeft: + return + winner.errback(failures.pop()) + checkDone.completed = False + checkDone.endpointsLeft = True + + @LoopingCall + def iterateEndpoint(): + endpoint = next(iterEndpoints, None) + if endpoint is None: + # The list of endpoints ends. + checkDone.endpointsLeft = False + checkDone() + return + + eachAttempt = endpoint.connect(protocolFactory) + pending.append(eachAttempt) + @eachAttempt.addBoth + def noLongerPending(result): + pending.remove(eachAttempt) + return result + @eachAttempt.addCallback + def succeeded(result): + winner.callback(result) + @eachAttempt.addErrback + def failed(reason): + failures.append(reason) + checkDone() + + iterateEndpoint.clock = self._reactor + iterateEndpoint.start(self._attemptDelay) + + @winner.addBoth + def cancelRemainingPending(result): + checkDone.completed = True + for remaining in pending[:]: + remaining.cancel() + if iterateEndpoint.running: + iterateEndpoint.stop() + return result + return winner + + return d + + + def _fallbackNameResolution(self, host, port): + """ + Resolve the hostname string into a tuple containing the host + address. This is method is only used when the reactor does + not provide L{IReactorPluggableNameResolver}. + + @param host: A unicode hostname to resolve. + + @param port: The port to include in the resolution. + + @return: A L{Deferred} that fires with L{_getaddrinfo}'s + return value. + """ + return self._deferToThread(self._getaddrinfo, host, port, 0, + socket.SOCK_STREAM) + + + +@implementer(interfaces.IStreamServerEndpoint) +class SSL4ServerEndpoint(object): + """ + SSL secured TCP server endpoint with an IPv4 configuration. + """ + + def __init__(self, reactor, port, sslContextFactory, + backlog=50, interface=''): + """ + @param reactor: An L{IReactorSSL} provider. + + @param port: The port number used for listening + @type port: int + + @param sslContextFactory: An instance of + L{interfaces.IOpenSSLContextFactory}. + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to, defaults to '' (all) + @type interface: str + """ + self._reactor = reactor + self._port = port + self._sslContextFactory = sslContextFactory + self._backlog = backlog + self._interface = interface + + + def listen(self, protocolFactory): + """ + Implement L{IStreamServerEndpoint.listen} to listen for SSL on a + TCP socket. + """ + return defer.execute(self._reactor.listenSSL, self._port, + protocolFactory, + contextFactory=self._sslContextFactory, + backlog=self._backlog, + interface=self._interface) + + + +@implementer(interfaces.IStreamClientEndpoint) +class SSL4ClientEndpoint(object): + """ + SSL secured TCP client endpoint with an IPv4 configuration + """ + + def __init__(self, reactor, host, port, sslContextFactory, + timeout=30, bindAddress=None): + """ + @param reactor: An L{IReactorSSL} provider. + + @param host: A hostname, used when connecting + @type host: str + + @param port: The port number, used when connecting + @type port: int + + @param sslContextFactory: SSL Configuration information as an instance + of L{interfaces.IOpenSSLContextFactory}. + + @param timeout: Number of seconds to wait before assuming the + connection has failed. + @type timeout: int + + @param bindAddress: A (host, port) tuple of local address to bind to, + or None. + @type bindAddress: tuple + """ + self._reactor = reactor + self._host = host + self._port = port + self._sslContextFactory = sslContextFactory + self._timeout = timeout + self._bindAddress = bindAddress + + + def connect(self, protocolFactory): + """ + Implement L{IStreamClientEndpoint.connect} to connect with SSL over + TCP. + """ + try: + wf = _WrappingFactory(protocolFactory) + self._reactor.connectSSL( + self._host, self._port, wf, self._sslContextFactory, + timeout=self._timeout, bindAddress=self._bindAddress) + return wf._onConnection + except: + return defer.fail() + + + +@implementer(interfaces.IStreamServerEndpoint) +class UNIXServerEndpoint(object): + """ + UnixSocket server endpoint. + """ + def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0): + """ + @param reactor: An L{IReactorUNIX} provider. + @param address: The path to the Unix socket file, used when listening + @param backlog: number of connections to allow in backlog. + @param mode: mode to set on the unix socket. This parameter is + deprecated. Permissions should be set on the directory which + contains the UNIX socket. + @param wantPID: If True, create a pidfile for the socket. + """ + self._reactor = reactor + self._address = address + self._backlog = backlog + self._mode = mode + self._wantPID = wantPID + + + def listen(self, protocolFactory): + """ + Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket. + """ + return defer.execute(self._reactor.listenUNIX, self._address, + protocolFactory, + backlog=self._backlog, + mode=self._mode, + wantPID=self._wantPID) + + + +@implementer(interfaces.IStreamClientEndpoint) +class UNIXClientEndpoint(object): + """ + UnixSocket client endpoint. + """ + def __init__(self, reactor, path, timeout=30, checkPID=0): + """ + @param reactor: An L{IReactorUNIX} provider. + + @param path: The path to the Unix socket file, used when connecting + @type path: str + + @param timeout: Number of seconds to wait before assuming the + connection has failed. + @type timeout: int + + @param checkPID: If True, check for a pid file to verify that a server + is listening. + @type checkPID: bool + """ + self._reactor = reactor + self._path = path + self._timeout = timeout + self._checkPID = checkPID + + + def connect(self, protocolFactory): + """ + Implement L{IStreamClientEndpoint.connect} to connect via a + UNIX Socket + """ + try: + wf = _WrappingFactory(protocolFactory) + self._reactor.connectUNIX( + self._path, wf, + timeout=self._timeout, + checkPID=self._checkPID) + return wf._onConnection + except: + return defer.fail() + + + +@implementer(interfaces.IStreamServerEndpoint) +class AdoptedStreamServerEndpoint(object): + """ + An endpoint for listening on a file descriptor initialized outside of + Twisted. + + @ivar _used: A C{bool} indicating whether this endpoint has been used to + listen with a factory yet. C{True} if so. + """ + _close = os.close + _setNonBlocking = staticmethod(fdesc.setNonBlocking) + + def __init__(self, reactor, fileno, addressFamily): + """ + @param reactor: An L{IReactorSocket} provider. + + @param fileno: An integer file descriptor corresponding to a listening + I{SOCK_STREAM} socket. + + @param addressFamily: The address family of the socket given by + C{fileno}. + """ + self.reactor = reactor + self.fileno = fileno + self.addressFamily = addressFamily + self._used = False + + + def listen(self, factory): + """ + Implement L{IStreamServerEndpoint.listen} to start listening on, and + then close, C{self._fileno}. + """ + if self._used: + return defer.fail(error.AlreadyListened()) + self._used = True + + try: + self._setNonBlocking(self.fileno) + port = self.reactor.adoptStreamPort( + self.fileno, self.addressFamily, factory) + self._close(self.fileno) + except: + return defer.fail() + return defer.succeed(port) + + + +def _parseTCP(factory, port, interface="", backlog=50): + """ + Internal parser function for L{_parseServer} to convert the string + arguments for a TCP(IPv4) stream endpoint into the structured arguments. + + @param factory: the protocol factory being parsed, or L{None}. (This was a + leftover argument from when this code was in C{strports}, and is now + mostly None and unused.) + + @type factory: L{IProtocolFactory} or L{None} + + @param port: the integer port number to bind + @type port: C{str} + + @param interface: the interface IP to listen on + @param backlog: the length of the listen queue + @type backlog: C{str} + + @return: a 2-tuple of (args, kwargs), describing the parameters to + L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments + to L{TCP4ServerEndpoint}. + """ + return (int(port), factory), {'interface': interface, + 'backlog': int(backlog)} + + + +def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True): + """ + Internal parser function for L{_parseServer} to convert the string + arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the + structured arguments. + + @param factory: the protocol factory being parsed, or L{None}. (This was a + leftover argument from when this code was in C{strports}, and is now + mostly None and unused.) + + @type factory: L{IProtocolFactory} or L{None} + + @param address: the pathname of the unix socket + @type address: C{str} + + @param backlog: the length of the listen queue + @type backlog: C{str} + + @param lockfile: A string '0' or '1', mapping to True and False + respectively. See the C{wantPID} argument to C{listenUNIX} + + @return: a 2-tuple of (args, kwargs), describing the parameters to + L{twisted.internet.interfaces.IReactorUNIX.listenUNIX} (or, + modulo argument 2, the factory, arguments to L{UNIXServerEndpoint}. + """ + return ( + (address, factory), + {'mode': int(mode, 8), 'backlog': int(backlog), + 'wantPID': bool(int(lockfile))}) + + + +def _parseSSL(factory, port, privateKey="server.pem", certKey=None, + sslmethod=None, interface='', backlog=50, extraCertChain=None, + dhParameters=None): + """ + Internal parser function for L{_parseServer} to convert the string + arguments for an SSL (over TCP/IPv4) stream endpoint into the structured + arguments. + + @param factory: the protocol factory being parsed, or L{None}. (This was a + leftover argument from when this code was in C{strports}, and is now + mostly None and unused.) + @type factory: L{IProtocolFactory} or L{None} + + @param port: the integer port number to bind + @type port: C{str} + + @param interface: the interface IP to listen on + @param backlog: the length of the listen queue + @type backlog: C{str} + + @param privateKey: The file name of a PEM format private key file. + @type privateKey: C{str} + + @param certKey: The file name of a PEM format certificate file. + @type certKey: C{str} + + @param sslmethod: The string name of an SSL method, based on the name of a + constant in C{OpenSSL.SSL}. Must be one of: "SSLv23_METHOD", + "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD". + @type sslmethod: C{str} + + @param extraCertChain: The path of a file containing one or more + certificates in PEM format that establish the chain from a root CA to + the CA that signed your C{certKey}. + @type extraCertChain: L{str} + + @param dhParameters: The file name of a file containing parameters that are + required for Diffie-Hellman key exchange. If this is not specified, + the forward secret C{DHE} ciphers aren't available for servers. + @type dhParameters: L{str} + + @return: a 2-tuple of (args, kwargs), describing the parameters to + L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments + to L{SSL4ServerEndpoint}. + """ + from twisted.internet import ssl + if certKey is None: + certKey = privateKey + kw = {} + if sslmethod is not None: + kw['method'] = getattr(ssl.SSL, sslmethod) + certPEM = FilePath(certKey).getContent() + keyPEM = FilePath(privateKey).getContent() + privateCertificate = ssl.PrivateCertificate.loadPEM( + certPEM + b'\n' + keyPEM) + if extraCertChain is not None: + matches = re.findall( + r'(-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----)', + nativeString(FilePath(extraCertChain).getContent()), + flags=re.DOTALL + ) + chainCertificates = [ssl.Certificate.loadPEM(chainCertPEM).original + for chainCertPEM in matches] + if not chainCertificates: + raise ValueError( + "Specified chain file '%s' doesn't contain any valid " + "certificates in PEM format." % (extraCertChain,) + ) + else: + chainCertificates = None + if dhParameters is not None: + dhParameters = ssl.DiffieHellmanParameters.fromFile( + FilePath(dhParameters), + ) + + cf = ssl.CertificateOptions( + privateKey=privateCertificate.privateKey.original, + certificate=privateCertificate.original, + extraCertChain=chainCertificates, + dhParameters=dhParameters, + **kw + ) + return ((int(port), factory, cf), + {'interface': interface, 'backlog': int(backlog)}) + + + +@implementer(IPlugin, IStreamServerEndpointStringParser) +class _StandardIOParser(object): + """ + Stream server endpoint string parser for the Standard I/O type. + + @ivar prefix: See L{IStreamServerEndpointStringParser.prefix}. + """ + prefix = "stdio" + + def _parseServer(self, reactor): + """ + Internal parser function for L{_parseServer} to convert the string + arguments into structured arguments for the L{StandardIOEndpoint} + + @param reactor: Reactor for the endpoint + """ + return StandardIOEndpoint(reactor) + + + def parseStreamServer(self, reactor, *args, **kwargs): + # Redirects to another function (self._parseServer), tricks zope.interface + # into believing the interface is correctly implemented. + return self._parseServer(reactor) + + + +@implementer(IPlugin, IStreamServerEndpointStringParser) +class _SystemdParser(object): + """ + Stream server endpoint string parser for the I{systemd} endpoint type. + + @ivar prefix: See L{IStreamServerEndpointStringParser.prefix}. + + @ivar _sddaemon: A L{ListenFDs} instance used to translate an index into an + actual file descriptor. + """ + _sddaemon = ListenFDs.fromEnvironment() + + prefix = "systemd" + + def _parseServer(self, reactor, domain, index): + """ + Internal parser function for L{_parseServer} to convert the string + arguments for a systemd server endpoint into structured arguments for + L{AdoptedStreamServerEndpoint}. + + @param reactor: An L{IReactorSocket} provider. + + @param domain: The domain (or address family) of the socket inherited + from systemd. This is a string like C{"INET"} or C{"UNIX"}, ie the + name of an address family from the L{socket} module, without the + C{"AF_"} prefix. + @type domain: C{str} + + @param index: An offset into the list of file descriptors inherited from + systemd. + @type index: C{str} + + @return: A two-tuple of parsed positional arguments and parsed keyword + arguments (a tuple and a dictionary). These can be used to + construct an L{AdoptedStreamServerEndpoint}. + """ + index = int(index) + fileno = self._sddaemon.inheritedDescriptors()[index] + addressFamily = getattr(socket, 'AF_' + domain) + return AdoptedStreamServerEndpoint(reactor, fileno, addressFamily) + + + def parseStreamServer(self, reactor, *args, **kwargs): + # Delegate to another function with a sane signature. This function has + # an insane signature to trick zope.interface into believing the + # interface is correctly implemented. + return self._parseServer(reactor, *args, **kwargs) + + + +@implementer(IPlugin, IStreamServerEndpointStringParser) +class _TCP6ServerParser(object): + """ + Stream server endpoint string parser for the TCP6ServerEndpoint type. + + @ivar prefix: See L{IStreamServerEndpointStringParser.prefix}. + """ + prefix = "tcp6" # Used in _parseServer to identify the plugin with the endpoint type + + def _parseServer(self, reactor, port, backlog=50, interface='::'): + """ + Internal parser function for L{_parseServer} to convert the string + arguments into structured arguments for the L{TCP6ServerEndpoint} + + @param reactor: An L{IReactorTCP} provider. + + @param port: The port number used for listening + @type port: int + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to + @type interface: str + """ + port = int(port) + backlog = int(backlog) + return TCP6ServerEndpoint(reactor, port, backlog, interface) + + + def parseStreamServer(self, reactor, *args, **kwargs): + # Redirects to another function (self._parseServer), tricks zope.interface + # into believing the interface is correctly implemented. + return self._parseServer(reactor, *args, **kwargs) + + + +_serverParsers = {"tcp": _parseTCP, + "unix": _parseUNIX, + "ssl": _parseSSL, + } + +_OP, _STRING = range(2) + +def _tokenize(description): + """ + Tokenize a strports string and yield each token. + + @param description: a string as described by L{serverFromString} or + L{clientFromString}. + @type description: L{str} or L{bytes} + + @return: an iterable of 2-tuples of (C{_OP} or C{_STRING}, string). Tuples + starting with C{_OP} will contain a second element of either ':' (i.e. + 'next parameter') or '=' (i.e. 'assign parameter value'). For example, + the string 'hello:greeting=world' would result in a generator yielding + these values:: + + _STRING, 'hello' + _OP, ':' + _STRING, 'greet=ing' + _OP, '=' + _STRING, 'world' + """ + empty = _matchingString(u'', description) + colon = _matchingString(u':', description) + equals = _matchingString(u'=', description) + backslash = _matchingString(u'\x5c', description) + current = empty + + ops = colon + equals + nextOps = {colon: colon + equals, equals: colon} + iterdesc = iter(iterbytes(description)) + for n in iterdesc: + if n in iterbytes(ops): + yield _STRING, current + yield _OP, n + current = empty + ops = nextOps[n] + elif n == backslash: + current += next(iterdesc) + else: + current += n + yield _STRING, current + + + +def _parse(description): + """ + Convert a description string into a list of positional and keyword + parameters, using logic vaguely like what Python does. + + @param description: a string as described by L{serverFromString} or + L{clientFromString}. + + @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all + ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of + all C{str}s which do contain an '='. For example, the result of + C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}. + """ + args, kw = [], {} + colon = _matchingString(u':', description) + def add(sofar): + if len(sofar) == 1: + args.append(sofar[0]) + else: + kw[nativeString(sofar[0])] = sofar[1] + sofar = () + for (type, value) in _tokenize(description): + if type is _STRING: + sofar += (value,) + elif value == colon: + add(sofar) + sofar = () + add(sofar) + return args, kw + + +# Mappings from description "names" to endpoint constructors. +_endpointServerFactories = { + 'TCP': TCP4ServerEndpoint, + 'SSL': SSL4ServerEndpoint, + 'UNIX': UNIXServerEndpoint, + } + +_endpointClientFactories = { + 'TCP': TCP4ClientEndpoint, + 'SSL': SSL4ClientEndpoint, + 'UNIX': UNIXClientEndpoint, + } + + +def _parseServer(description, factory): + """ + Parse a strports description into a 2-tuple of arguments and keyword + values. + + @param description: A description in the format explained by + L{serverFromString}. + @type description: C{str} + + @param factory: A 'factory' argument; this is left-over from + twisted.application.strports, it's not really used. + @type factory: L{IProtocolFactory} or L{None} + + @return: a 3-tuple of (plugin or name, arguments, keyword arguments) + """ + args, kw = _parse(description) + endpointType = args[0] + parser = _serverParsers.get(endpointType) + if parser is None: + # If the required parser is not found in _server, check if + # a plugin exists for the endpointType + plugin = _matchPluginToPrefix( + getPlugins(IStreamServerEndpointStringParser), endpointType + ) + return (plugin, args[1:], kw) + return (endpointType.upper(),) + parser(factory, *args[1:], **kw) + + + +def _matchPluginToPrefix(plugins, endpointType): + """ + Match plugin to prefix. + """ + endpointType = endpointType.lower() + for plugin in plugins: + if (_matchingString(plugin.prefix.lower(), + endpointType) == endpointType): + return plugin + raise ValueError("Unknown endpoint type: '%s'" % (endpointType,)) + + + +def serverFromString(reactor, description): + """ + Construct a stream server endpoint from an endpoint description string. + + The format for server endpoint descriptions is a simple byte string. It is + a prefix naming the type of endpoint, then a colon, then the arguments for + that endpoint. + + For example, you can call it like this to create an endpoint that will + listen on TCP port 80:: + + serverFromString(reactor, "tcp:80") + + Additional arguments may be specified as keywords, separated with colons. + For example, you can specify the interface for a TCP server endpoint to + bind to like this:: + + serverFromString(reactor, "tcp:80:interface=127.0.0.1") + + SSL server endpoints may be specified with the 'ssl' prefix, and the + private key and certificate files may be specified by the C{privateKey} and + C{certKey} arguments:: + + serverFromString( + reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem") + + If a private key file name (C{privateKey}) isn't provided, a "server.pem" + file is assumed to exist which contains the private key. If the certificate + file name (C{certKey}) isn't provided, the private key file is assumed to + contain the certificate as well. + + You may escape colons in arguments with a backslash, which you will need to + use if you want to specify a full pathname argument on Windows:: + + serverFromString(reactor, + "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem") + + finally, the 'unix' prefix may be used to specify a filesystem UNIX socket, + optionally with a 'mode' argument to specify the mode of the socket file + created by C{listen}:: + + serverFromString(reactor, "unix:/var/run/finger") + serverFromString(reactor, "unix:/var/run/finger:mode=660") + + This function is also extensible; new endpoint types may be registered as + L{IStreamServerEndpointStringParser} plugins. See that interface for more + information. + + @param reactor: The server endpoint will be constructed with this reactor. + + @param description: The strports description to parse. + @type description: L{str} + + @return: A new endpoint which can be used to listen with the parameters + given by C{description}. + + @rtype: L{IStreamServerEndpoint} + + @raise ValueError: when the 'description' string cannot be parsed. + + @since: 10.2 + """ + nameOrPlugin, args, kw = _parseServer(description, None) + if type(nameOrPlugin) is not str: + plugin = nameOrPlugin + return plugin.parseStreamServer(reactor, *args, **kw) + else: + name = nameOrPlugin + # Chop out the factory. + args = args[:1] + args[2:] + return _endpointServerFactories[name](reactor, *args, **kw) + + + +def quoteStringArgument(argument): + """ + Quote an argument to L{serverFromString} and L{clientFromString}. Since + arguments are separated with colons and colons are escaped with + backslashes, some care is necessary if, for example, you have a pathname, + you may be tempted to interpolate into a string like this:: + + serverFromString(reactor, "ssl:443:privateKey=%s" % (myPathName,)) + + This may appear to work, but will have portability issues (Windows + pathnames, for example). Usually you should just construct the appropriate + endpoint type rather than interpolating strings, which in this case would + be L{SSL4ServerEndpoint}. There are some use-cases where you may need to + generate such a string, though; for example, a tool to manipulate a + configuration file which has strports descriptions in it. To be correct in + those cases, do this instead:: + + serverFromString(reactor, "ssl:443:privateKey=%s" % + (quoteStringArgument(myPathName),)) + + @param argument: The part of the endpoint description string you want to + pass through. + + @type argument: C{str} + + @return: The quoted argument. + + @rtype: C{str} + """ + backslash, colon = '\\:' + for c in backslash, colon: + argument = argument.replace(c, backslash + c) + return argument + + + +def _parseClientTCP(*args, **kwargs): + """ + Perform any argument value coercion necessary for TCP client parameters. + + Valid positional arguments to this function are host and port. + + Valid keyword arguments to this function are all L{IReactorTCP.connectTCP} + arguments. + + @return: The coerced values as a C{dict}. + """ + + if len(args) == 2: + kwargs['port'] = int(args[1]) + kwargs['host'] = args[0] + elif len(args) == 1: + if 'host' in kwargs: + kwargs['port'] = int(args[0]) + else: + kwargs['host'] = args[0] + + try: + kwargs['port'] = int(kwargs['port']) + except KeyError: + pass + + try: + kwargs['timeout'] = int(kwargs['timeout']) + except KeyError: + pass + + try: + kwargs['bindAddress'] = (kwargs['bindAddress'], 0) + except KeyError: + pass + + return kwargs + + + +def _loadCAsFromDir(directoryPath): + """ + Load certificate-authority certificate objects in a given directory. + + @param directoryPath: a L{unicode} or L{bytes} pointing at a directory to + load .pem files from, or L{None}. + + @return: an L{IOpenSSLTrustRoot} provider. + """ + caCerts = {} + for child in directoryPath.children(): + if not child.asTextMode().basename().split(u'.')[-1].lower() == u'pem': + continue + try: + data = child.getContent() + except IOError: + # Permission denied, corrupt disk, we don't care. + continue + try: + theCert = Certificate.loadPEM(data) + except SSLError: + # Duplicate certificate, invalid certificate, etc. We don't care. + pass + else: + caCerts[theCert.digest()] = theCert + return trustRootFromCertificates(caCerts.values()) + + + +def _parseTrustRootPath(pathName): + """ + Parse a string referring to a directory full of certificate authorities + into a trust root. + + @param pathName: path name + @type pathName: L{unicode} or L{bytes} or L{None} + + @return: L{None} or L{IOpenSSLTrustRoot} + """ + if pathName is None: + return None + return _loadCAsFromDir(FilePath(pathName)) + + + +def _privateCertFromPaths(certificatePath, keyPath): + """ + Parse a certificate path and key path, either or both of which might be + L{None}, into a certificate object. + + @param certificatePath: the certificate path + @type certificatePath: L{bytes} or L{unicode} or L{None} + + @param keyPath: the private key path + @type keyPath: L{bytes} or L{unicode} or L{None} + + @return: a L{PrivateCertificate} or L{None} + """ + if certificatePath is None: + return None + certBytes = FilePath(certificatePath).getContent() + if keyPath is None: + return PrivateCertificate.loadPEM(certBytes) + else: + return PrivateCertificate.fromCertificateAndKeyPair( + Certificate.loadPEM(certBytes), + KeyPair.load(FilePath(keyPath).getContent(), 1) + ) + + + +def _parseClientSSLOptions(kwargs): + """ + Parse common arguments for SSL endpoints, creating an L{CertificateOptions} + instance. + + @param kwargs: A dict of keyword arguments to be parsed, potentially + containing keys C{certKey}, C{privateKey}, C{caCertsDir}, and + C{hostname}. See L{_parseClientSSL}. + @type kwargs: L{dict} + + @return: The remaining arguments, including a new key C{sslContextFactory}. + """ + hostname = kwargs.pop('hostname', None) + clientCertificate = _privateCertFromPaths(kwargs.pop('certKey', None), + kwargs.pop('privateKey', None)) + trustRoot = _parseTrustRootPath(kwargs.pop('caCertsDir', None)) + if hostname is not None: + configuration = optionsForClientTLS( + _idnaText(hostname), trustRoot=trustRoot, + clientCertificate=clientCertificate + ) + else: + # _really_ though, you should specify a hostname. + if clientCertificate is not None: + privateKeyOpenSSL = clientCertificate.privateKey.original + certificateOpenSSL = clientCertificate.original + else: + privateKeyOpenSSL = None + certificateOpenSSL = None + configuration = CertificateOptions( + trustRoot=trustRoot, + privateKey=privateKeyOpenSSL, + certificate=certificateOpenSSL, + ) + kwargs['sslContextFactory'] = configuration + return kwargs + + + +def _parseClientSSL(*args, **kwargs): + """ + Perform any argument value coercion necessary for SSL client parameters. + + Valid keyword arguments to this function are all L{IReactorSSL.connectSSL} + arguments except for C{contextFactory}. Instead, C{certKey} (the path name + of the certificate file) C{privateKey} (the path name of the private key + associated with the certificate) are accepted and used to construct a + context factory. + + Valid positional arguments to this function are host and port. + + @param caCertsDir: The one parameter which is not part of + L{IReactorSSL.connectSSL}'s signature, this is a path name used to + construct a list of certificate authority certificates. The directory + will be scanned for files ending in C{.pem}, all of which will be + considered valid certificate authorities for this connection. + + @type caCertsDir: L{str} + + @param hostname: The hostname to use for validating the server's + certificate. + @type hostname: L{unicode} + + @return: The coerced values as a L{dict}. + """ + kwargs = _parseClientTCP(*args, **kwargs) + return _parseClientSSLOptions(kwargs) + + + +def _parseClientUNIX(*args, **kwargs): + """ + Perform any argument value coercion necessary for UNIX client parameters. + + Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX} + keyword arguments except for C{checkPID}. Instead, C{lockfile} is accepted + and has the same meaning. Also C{path} is used instead of C{address}. + + Valid positional arguments to this function are C{path}. + + @return: The coerced values as a C{dict}. + """ + if len(args) == 1: + kwargs['path'] = args[0] + + try: + kwargs['checkPID'] = bool(int(kwargs.pop('lockfile'))) + except KeyError: + pass + try: + kwargs['timeout'] = int(kwargs['timeout']) + except KeyError: + pass + return kwargs + +_clientParsers = { + 'TCP': _parseClientTCP, + 'SSL': _parseClientSSL, + 'UNIX': _parseClientUNIX, + } + + + +def clientFromString(reactor, description): + """ + Construct a client endpoint from a description string. + + Client description strings are much like server description strings, + although they take all of their arguments as keywords, aside from host and + port. + + You can create a TCP client endpoint with the 'host' and 'port' arguments, + like so:: + + clientFromString(reactor, "tcp:host=www.example.com:port=80") + + or, without specifying host and port keywords:: + + clientFromString(reactor, "tcp:www.example.com:80") + + Or you can specify only one or the other, as in the following 2 examples:: + + clientFromString(reactor, "tcp:host=www.example.com:80") + clientFromString(reactor, "tcp:www.example.com:port=80") + + or an SSL client endpoint with those arguments, plus the arguments used by + the server SSL, for a client certificate:: + + clientFromString(reactor, "ssl:web.example.com:443:" + "privateKey=foo.pem:certKey=foo.pem") + + to specify your certificate trust roots, you can identify a directory with + PEM files in it with the C{caCertsDir} argument:: + + clientFromString(reactor, "ssl:host=web.example.com:port=443:" + "caCertsDir=/etc/ssl/certs") + + Both TCP and SSL client endpoint description strings can include a + 'bindAddress' keyword argument, whose value should be a local IPv4 + address. This fixes the client socket to that IP address:: + + clientFromString(reactor, "tcp:www.example.com:80:" + "bindAddress=192.0.2.100") + + NB: Fixed client ports are not currently supported in TCP or SSL + client endpoints. The client socket will always use an ephemeral + port assigned by the operating system + + You can create a UNIX client endpoint with the 'path' argument and optional + 'lockfile' and 'timeout' arguments:: + + clientFromString( + reactor, b"unix:path=/var/foo/bar:lockfile=1:timeout=9") + + or, with the path as a positional argument with or without optional + arguments as in the following 2 examples:: + + clientFromString(reactor, "unix:/var/foo/bar") + clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9") + + This function is also extensible; new endpoint types may be registered as + L{IStreamClientEndpointStringParserWithReactor} plugins. See that + interface for more information. + + @param reactor: The client endpoint will be constructed with this reactor. + + @param description: The strports description to parse. + @type description: L{str} + + @return: A new endpoint which can be used to connect with the parameters + given by C{description}. + @rtype: L{IStreamClientEndpoint} + + @since: 10.2 + """ + args, kwargs = _parse(description) + aname = args.pop(0) + name = aname.upper() + if name not in _clientParsers: + plugin = _matchPluginToPrefix( + getPlugins(IStreamClientEndpointStringParserWithReactor), name + ) + return plugin.parseStreamClient(reactor, *args, **kwargs) + kwargs = _clientParsers[name](*args, **kwargs) + return _endpointClientFactories[name](reactor, **kwargs) + + + +def connectProtocol(endpoint, protocol): + """ + Connect a protocol instance to an endpoint. + + This allows using a client endpoint without having to create a factory. + + @param endpoint: A client endpoint to connect to. + + @param protocol: A protocol instance. + + @return: The result of calling C{connect} on the endpoint, i.e. a + L{Deferred} that will fire with the protocol when connected, or an + appropriate error. + + @since: 13.1 + """ + class OneShotFactory(Factory): + def buildProtocol(self, addr): + return protocol + return endpoint.connect(OneShotFactory()) + + + +@implementer(interfaces.IStreamClientEndpoint) +class _WrapperEndpoint(object): + """ + An endpoint that wraps another endpoint. + """ + + def __init__(self, wrappedEndpoint, wrapperFactory): + """ + Construct a L{_WrapperEndpoint}. + """ + self._wrappedEndpoint = wrappedEndpoint + self._wrapperFactory = wrapperFactory + + + def connect(self, protocolFactory): + """ + Connect the given protocol factory and unwrap its result. + """ + return self._wrappedEndpoint.connect( + self._wrapperFactory(protocolFactory) + ).addCallback(lambda protocol: protocol.wrappedProtocol) + + + +@implementer(interfaces.IStreamServerEndpoint) +class _WrapperServerEndpoint(object): + """ + A server endpoint that wraps another server endpoint. + """ + + def __init__(self, wrappedEndpoint, wrapperFactory): + """ + Construct a L{_WrapperServerEndpoint}. + """ + self._wrappedEndpoint = wrappedEndpoint + self._wrapperFactory = wrapperFactory + + + def listen(self, protocolFactory): + """ + Connect the given protocol factory and unwrap its result. + """ + return self._wrappedEndpoint.listen( + self._wrapperFactory(protocolFactory) + ) + + + +def wrapClientTLS(connectionCreator, wrappedEndpoint): + """ + Wrap an endpoint which upgrades to TLS as soon as the connection is + established. + + @since: 16.0 + + @param connectionCreator: The TLS options to use when connecting; see + L{twisted.internet.ssl.optionsForClientTLS} for how to construct this. + @type connectionCreator: + L{twisted.internet.interfaces.IOpenSSLClientConnectionCreator} + + @param wrappedEndpoint: The endpoint to wrap. + @type wrappedEndpoint: An L{IStreamClientEndpoint} provider. + + @return: an endpoint that provides transport level encryption layered on + top of C{wrappedEndpoint} + @rtype: L{twisted.internet.interfaces.IStreamClientEndpoint} + """ + if TLSMemoryBIOFactory is None: + raise NotImplementedError( + "OpenSSL not available. Try `pip install twisted[tls]`." + ) + return _WrapperEndpoint( + wrappedEndpoint, + lambda protocolFactory: + TLSMemoryBIOFactory(connectionCreator, True, protocolFactory) + ) + + + +def _parseClientTLS(reactor, host, port, timeout=b'30', bindAddress=None, + certificate=None, privateKey=None, trustRoots=None, + endpoint=None, **kwargs): + """ + Internal method to construct an endpoint from string parameters. + + @param reactor: The reactor passed to L{clientFromString}. + + @param host: The hostname to connect to. + @type host: L{bytes} or L{unicode} + + @param port: The port to connect to. + @type port: L{bytes} or L{unicode} + + @param timeout: For each individual connection attempt, the number of + seconds to wait before assuming the connection has failed. + @type timeout: L{bytes} or L{unicode} + + @param bindAddress: The address to which to bind outgoing connections. + @type bindAddress: L{bytes} or L{unicode} + + @param certificate: a string representing a filesystem path to a + PEM-encoded certificate. + @type certificate: L{bytes} or L{unicode} + + @param privateKey: a string representing a filesystem path to a PEM-encoded + certificate. + @type privateKey: L{bytes} or L{unicode} + + @param endpoint: an optional string endpoint description of an endpoint to + wrap; if this is passed then C{host} is used only for certificate + verification. + @type endpoint: L{bytes} or L{unicode} + + @return: a client TLS endpoint + @rtype: L{IStreamClientEndpoint} + """ + if kwargs: + raise TypeError('unrecognized keyword arguments present', + list(kwargs.keys())) + host = host if isinstance(host, unicode) else host.decode("utf-8") + bindAddress = (bindAddress + if isinstance(bindAddress, unicode) or bindAddress is None + else bindAddress.decode("utf-8")) + port = int(port) + timeout = int(timeout) + return wrapClientTLS( + optionsForClientTLS( + host, trustRoot=_parseTrustRootPath(trustRoots), + clientCertificate=_privateCertFromPaths(certificate, + privateKey)), + clientFromString(reactor, endpoint) if endpoint is not None + else HostnameEndpoint(reactor, _idnaBytes(host), port, timeout, + bindAddress) + ) + + + +@implementer(IPlugin, IStreamClientEndpointStringParserWithReactor) +class _TLSClientEndpointParser(object): + """ + Stream client endpoint string parser for L{wrapClientTLS} with + L{HostnameEndpoint}. + + @ivar prefix: See + L{IStreamClientEndpointStringParserWithReactor.prefix}. + """ + prefix = 'tls' + + @staticmethod + def parseStreamClient(reactor, *args, **kwargs): + """ + Redirects to another function L{_parseClientTLS}; tricks zope.interface + into believing the interface is correctly implemented, since the + signature is (C{reactor}, C{*args}, C{**kwargs}). See + L{_parseClientTLS} for the specific signature description for this + endpoint parser. + + @param reactor: The reactor passed to L{clientFromString}. + + @param args: The positional arguments in the endpoint description. + @type args: L{tuple} + + @param kwargs: The named arguments in the endpoint description. + @type kwargs: L{dict} + + @return: a client TLS endpoint + @rtype: L{IStreamClientEndpoint} + """ + return _parseClientTLS(reactor, *args, **kwargs) diff --git a/contrib/python/Twisted/py2/twisted/internet/epollreactor.py b/contrib/python/Twisted/py2/twisted/internet/epollreactor.py new file mode 100644 index 00000000000..a9fe298e938 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/epollreactor.py @@ -0,0 +1,249 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An epoll() based implementation of the twisted main loop. + +To install the event loop (and you should do this before any connections, +listeners or connectors are added):: + + from twisted.internet import epollreactor + epollreactor.install() +""" + +from __future__ import division, absolute_import + +from select import epoll, EPOLLHUP, EPOLLERR, EPOLLIN, EPOLLOUT +import errno + +from zope.interface import implementer + +from twisted.internet.interfaces import IReactorFDSet + +from twisted.python import log +from twisted.internet import posixbase + + +@implementer(IReactorFDSet) +class EPollReactor(posixbase.PosixReactorBase, posixbase._PollLikeMixin): + """ + A reactor that uses epoll(7). + + @ivar _poller: A C{epoll} which will be used to check for I/O + readiness. + + @ivar _selectables: A dictionary mapping integer file descriptors to + instances of C{FileDescriptor} which have been registered with the + reactor. All C{FileDescriptors} which are currently receiving read or + write readiness notifications will be present as values in this + dictionary. + + @ivar _reads: A set containing integer file descriptors. Values in this + set will be registered with C{_poller} for read readiness notifications + which will be dispatched to the corresponding C{FileDescriptor} + instances in C{_selectables}. + + @ivar _writes: A set containing integer file descriptors. Values in this + set will be registered with C{_poller} for write readiness + notifications which will be dispatched to the corresponding + C{FileDescriptor} instances in C{_selectables}. + + @ivar _continuousPolling: A L{_ContinuousPolling} instance, used to handle + file descriptors (e.g. filesystem files) that are not supported by + C{epoll(7)}. + """ + + # Attributes for _PollLikeMixin + _POLL_DISCONNECTED = (EPOLLHUP | EPOLLERR) + _POLL_IN = EPOLLIN + _POLL_OUT = EPOLLOUT + + def __init__(self): + """ + Initialize epoll object, file descriptor tracking dictionaries, and the + base class. + """ + # Create the poller we're going to use. The 1024 here is just a hint + # to the kernel, it is not a hard maximum. After Linux 2.6.8, the size + # argument is completely ignored. + self._poller = epoll(1024) + self._reads = set() + self._writes = set() + self._selectables = {} + self._continuousPolling = posixbase._ContinuousPolling(self) + posixbase.PosixReactorBase.__init__(self) + + + def _add(self, xer, primary, other, selectables, event, antievent): + """ + Private method for adding a descriptor from the event loop. + + It takes care of adding it if new or modifying it if already added + for another state (read -> read/write for example). + """ + fd = xer.fileno() + if fd not in primary: + flags = event + # epoll_ctl can raise all kinds of IOErrors, and every one + # indicates a bug either in the reactor or application-code. + # Let them all through so someone sees a traceback and fixes + # something. We'll do the same thing for every other call to + # this method in this file. + if fd in other: + flags |= antievent + self._poller.modify(fd, flags) + else: + self._poller.register(fd, flags) + + # Update our own tracking state *only* after the epoll call has + # succeeded. Otherwise we may get out of sync. + primary.add(fd) + selectables[fd] = xer + + + def addReader(self, reader): + """ + Add a FileDescriptor for notification of data available to read. + """ + try: + self._add(reader, self._reads, self._writes, self._selectables, + EPOLLIN, EPOLLOUT) + except IOError as e: + if e.errno == errno.EPERM: + # epoll(7) doesn't support certain file descriptors, + # e.g. filesystem files, so for those we just poll + # continuously: + self._continuousPolling.addReader(reader) + else: + raise + + + def addWriter(self, writer): + """ + Add a FileDescriptor for notification of data available to write. + """ + try: + self._add(writer, self._writes, self._reads, self._selectables, + EPOLLOUT, EPOLLIN) + except IOError as e: + if e.errno == errno.EPERM: + # epoll(7) doesn't support certain file descriptors, + # e.g. filesystem files, so for those we just poll + # continuously: + self._continuousPolling.addWriter(writer) + else: + raise + + + def _remove(self, xer, primary, other, selectables, event, antievent): + """ + Private method for removing a descriptor from the event loop. + + It does the inverse job of _add, and also add a check in case of the fd + has gone away. + """ + fd = xer.fileno() + if fd == -1: + for fd, fdes in selectables.items(): + if xer is fdes: + break + else: + return + if fd in primary: + if fd in other: + flags = antievent + # See comment above modify call in _add. + self._poller.modify(fd, flags) + else: + del selectables[fd] + # See comment above _control call in _add. + self._poller.unregister(fd) + primary.remove(fd) + + + def removeReader(self, reader): + """ + Remove a Selectable for notification of data available to read. + """ + if self._continuousPolling.isReading(reader): + self._continuousPolling.removeReader(reader) + return + self._remove(reader, self._reads, self._writes, self._selectables, + EPOLLIN, EPOLLOUT) + + + def removeWriter(self, writer): + """ + Remove a Selectable for notification of data available to write. + """ + if self._continuousPolling.isWriting(writer): + self._continuousPolling.removeWriter(writer) + return + self._remove(writer, self._writes, self._reads, self._selectables, + EPOLLOUT, EPOLLIN) + + + def removeAll(self): + """ + Remove all selectables, and return a list of them. + """ + return (self._removeAll( + [self._selectables[fd] for fd in self._reads], + [self._selectables[fd] for fd in self._writes]) + + self._continuousPolling.removeAll()) + + + def getReaders(self): + return ([self._selectables[fd] for fd in self._reads] + + self._continuousPolling.getReaders()) + + + def getWriters(self): + return ([self._selectables[fd] for fd in self._writes] + + self._continuousPolling.getWriters()) + + + def doPoll(self, timeout): + """ + Poll the poller for new events. + """ + if timeout is None: + timeout = -1 # Wait indefinitely. + + try: + # Limit the number of events to the number of io objects we're + # currently tracking (because that's maybe a good heuristic) and + # the amount of time we block to the value specified by our + # caller. + l = self._poller.poll(timeout, len(self._selectables)) + except IOError as err: + if err.errno == errno.EINTR: + return + # See epoll_wait(2) for documentation on the other conditions + # under which this can fail. They can only be due to a serious + # programming error on our part, so let's just announce them + # loudly. + raise + + _drdw = self._doReadOrWrite + for fd, event in l: + try: + selectable = self._selectables[fd] + except KeyError: + pass + else: + log.callWithLogger(selectable, _drdw, selectable, fd, event) + + doIteration = doPoll + + +def install(): + """ + Install the epoll() reactor. + """ + p = EPollReactor() + from twisted.internet.main import installReactor + installReactor(p) + + +__all__ = ["EPollReactor", "install"] diff --git a/contrib/python/Twisted/py2/twisted/internet/error.py b/contrib/python/Twisted/py2/twisted/internet/error.py new file mode 100644 index 00000000000..20a29089c20 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/error.py @@ -0,0 +1,517 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Exceptions and errors for use in twisted.internet modules. +""" + +from __future__ import division, absolute_import + +import socket + +from twisted.python import deprecate +from incremental import Version + + + +class BindError(Exception): + """An error occurred binding to an interface""" + + def __str__(self): + s = self.__doc__ + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class CannotListenError(BindError): + """ + This gets raised by a call to startListening, when the object cannotstart + listening. + + @ivar interface: the interface I tried to listen on + @ivar port: the port I tried to listen on + @ivar socketError: the exception I got when I tried to listen + @type socketError: L{socket.error} + """ + def __init__(self, interface, port, socketError): + BindError.__init__(self, interface, port, socketError) + self.interface = interface + self.port = port + self.socketError = socketError + + def __str__(self): + iface = self.interface or 'any' + return "Couldn't listen on %s:%s: %s." % (iface, self.port, + self.socketError) + + + +class MulticastJoinError(Exception): + """ + An attempt to join a multicast group failed. + """ + + + +class MessageLengthError(Exception): + """Message is too long to send""" + + def __str__(self): + s = self.__doc__ + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class DNSLookupError(IOError): + """DNS lookup failed""" + + def __str__(self): + s = self.__doc__ + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class ConnectInProgressError(Exception): + """A connect operation was started and isn't done yet.""" + + +# connection errors + +class ConnectError(Exception): + """An error occurred while connecting""" + + def __init__(self, osError=None, string=""): + self.osError = osError + Exception.__init__(self, string) + + def __str__(self): + s = self.__doc__ or self.__class__.__name__ + if self.osError: + s = '%s: %s' % (s, self.osError) + if self.args[0]: + s = '%s: %s' % (s, self.args[0]) + s = '%s.' % s + return s + + + +class ConnectBindError(ConnectError): + """Couldn't bind""" + + + +class UnknownHostError(ConnectError): + """Hostname couldn't be looked up""" + + + +class NoRouteError(ConnectError): + """No route to host""" + + + +class ConnectionRefusedError(ConnectError): + """Connection was refused by other side""" + + + +class TCPTimedOutError(ConnectError): + """TCP connection timed out""" + + + +class BadFileError(ConnectError): + """File used for UNIX socket is no good""" + + + +class ServiceNameUnknownError(ConnectError): + """Service name given as port is unknown""" + + + +class UserError(ConnectError): + """User aborted connection""" + + + +class TimeoutError(UserError): + """User timeout caused connection failure""" + + + +class SSLError(ConnectError): + """An SSL error occurred""" + + + +class VerifyError(Exception): + """Could not verify something that was supposed to be signed. + """ + + + +class PeerVerifyError(VerifyError): + """The peer rejected our verify error. + """ + + + +class CertificateError(Exception): + """ + We did not find a certificate where we expected to find one. + """ + + + +try: + import errno + errnoMapping = { + errno.ENETUNREACH: NoRouteError, + errno.ECONNREFUSED: ConnectionRefusedError, + errno.ETIMEDOUT: TCPTimedOutError, + } + if hasattr(errno, "WSAECONNREFUSED"): + errnoMapping[errno.WSAECONNREFUSED] = ConnectionRefusedError + errnoMapping[errno.WSAENETUNREACH] = NoRouteError +except ImportError: + errnoMapping = {} + + + +def getConnectError(e): + """Given a socket exception, return connection error.""" + if isinstance(e, Exception): + args = e.args + else: + args = e + try: + number, string = args + except ValueError: + return ConnectError(string=e) + + if hasattr(socket, 'gaierror') and isinstance(e, socket.gaierror): + # Only works in 2.2 in newer. Really that means always; #5978 covers + # this and other weirdnesses in this function. + klass = UnknownHostError + else: + klass = errnoMapping.get(number, ConnectError) + return klass(number, string) + + + +class ConnectionClosed(Exception): + """ + Connection was closed, whether cleanly or non-cleanly. + """ + + + +class ConnectionLost(ConnectionClosed): + """Connection to the other side was lost in a non-clean fashion""" + + def __str__(self): + s = self.__doc__.strip().splitlines()[0] + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class ConnectionAborted(ConnectionLost): + """ + Connection was aborted locally, using + L{twisted.internet.interfaces.ITCPTransport.abortConnection}. + + @since: 11.1 + """ + + def __str__(self): + s = [( + "Connection was aborted locally using" + " ITCPTransport.abortConnection" + )] + if self.args: + s.append(': ') + s.append(' '.join(self.args)) + s.append('.') + return ''.join(s) + + + +class ConnectionDone(ConnectionClosed): + """Connection was closed cleanly""" + + def __str__(self): + s = self.__doc__ + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class FileDescriptorOverrun(ConnectionLost): + """ + A mis-use of L{IUNIXTransport.sendFileDescriptor} caused the connection to + be closed. + + Each file descriptor sent using C{sendFileDescriptor} must be associated + with at least one byte sent using L{ITransport.write}. If at any point + fewer bytes have been written than file descriptors have been sent, the + connection is closed with this exception. + """ + + + +class ConnectionFdescWentAway(ConnectionLost): + """Uh""" #TODO + + + +class AlreadyCalled(ValueError): + """Tried to cancel an already-called event""" + + def __str__(self): + s = self.__doc__ + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class AlreadyCancelled(ValueError): + """Tried to cancel an already-cancelled event""" + + def __str__(self): + s = self.__doc__ + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class PotentialZombieWarning(Warning): + """ + Emitted when L{IReactorProcess.spawnProcess} is called in a way which may + result in termination of the created child process not being reported. + + Deprecated in Twisted 10.0. + """ + MESSAGE = ( + "spawnProcess called, but the SIGCHLD handler is not " + "installed. This probably means you have not yet " + "called reactor.run, or called " + "reactor.run(installSignalHandler=0). You will probably " + "never see this process finish, and it may become a " + "zombie process.") + +deprecate.deprecatedModuleAttribute( + Version("Twisted", 10, 0, 0), + "There is no longer any potential for zombie process.", + __name__, + "PotentialZombieWarning") + + + +class ProcessDone(ConnectionDone): + """A process has ended without apparent errors""" + + def __init__(self, status): + Exception.__init__(self, "process finished with exit code 0") + self.exitCode = 0 + self.signal = None + self.status = status + + + +class ProcessTerminated(ConnectionLost): + """ + A process has ended with a probable error condition + + @ivar exitCode: See L{__init__} + @ivar signal: See L{__init__} + @ivar status: See L{__init__} + """ + def __init__(self, exitCode=None, signal=None, status=None): + """ + @param exitCode: The exit status of the process. This is roughly like + the value you might pass to L{os.exit}. This is L{None} if the + process exited due to a signal. + @type exitCode: L{int} or L{None} + + @param signal: The exit signal of the process. This is L{None} if the + process did not exit due to a signal. + @type signal: L{int} or L{None} + + @param status: The exit code of the process. This is a platform + specific combination of the exit code and the exit signal. See + L{os.WIFEXITED} and related functions. + @type status: L{int} + """ + self.exitCode = exitCode + self.signal = signal + self.status = status + s = "process ended" + if exitCode is not None: s = s + " with exit code %s" % exitCode + if signal is not None: s = s + " by signal %s" % signal + Exception.__init__(self, s) + + + +class ProcessExitedAlready(Exception): + """ + The process has already exited and the operation requested can no longer + be performed. + """ + + + +class NotConnectingError(RuntimeError): + """The Connector was not connecting when it was asked to stop connecting""" + + def __str__(self): + s = self.__doc__ + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class NotListeningError(RuntimeError): + """The Port was not listening when it was asked to stop listening""" + + def __str__(self): + s = self.__doc__ + if self.args: + s = '%s: %s' % (s, ' '.join(self.args)) + s = '%s.' % s + return s + + + +class ReactorNotRunning(RuntimeError): + """ + Error raised when trying to stop a reactor which is not running. + """ + + +class ReactorNotRestartable(RuntimeError): + """ + Error raised when trying to run a reactor which was stopped. + """ + + + +class ReactorAlreadyRunning(RuntimeError): + """ + Error raised when trying to start the reactor multiple times. + """ + + +class ReactorAlreadyInstalledError(AssertionError): + """ + Could not install reactor because one is already installed. + """ + + + +class ConnectingCancelledError(Exception): + """ + An C{Exception} that will be raised when an L{IStreamClientEndpoint} is + cancelled before it connects. + + @ivar address: The L{IAddress} that is the destination of the + cancelled L{IStreamClientEndpoint}. + """ + + def __init__(self, address): + """ + @param address: The L{IAddress} that is the destination of the + L{IStreamClientEndpoint} that was cancelled. + """ + Exception.__init__(self, address) + self.address = address + + + +class NoProtocol(Exception): + """ + An C{Exception} that will be raised when the factory given to a + L{IStreamClientEndpoint} returns L{None} from C{buildProtocol}. + """ + + + +class UnsupportedAddressFamily(Exception): + """ + An attempt was made to use a socket with an address family (eg I{AF_INET}, + I{AF_INET6}, etc) which is not supported by the reactor. + """ + + + +class UnsupportedSocketType(Exception): + """ + An attempt was made to use a socket of a type (eg I{SOCK_STREAM}, + I{SOCK_DGRAM}, etc) which is not supported by the reactor. + """ + + +class AlreadyListened(Exception): + """ + An attempt was made to listen on a file descriptor which can only be + listened on once. + """ + + + +class InvalidAddressError(ValueError): + """ + An invalid address was specified (i.e. neither IPv4 or IPv6, or expected + one and got the other). + + @ivar address: See L{__init__} + @ivar message: See L{__init__} + """ + + def __init__(self, address, message): + """ + @param address: The address that was provided. + @type address: L{bytes} + @param message: A native string of additional information provided by + the calling context. + @type address: L{str} + """ + self.address = address + self.message = message + + + +__all__ = [ + 'BindError', 'CannotListenError', 'MulticastJoinError', + 'MessageLengthError', 'DNSLookupError', 'ConnectInProgressError', + 'ConnectError', 'ConnectBindError', 'UnknownHostError', 'NoRouteError', + 'ConnectionRefusedError', 'TCPTimedOutError', 'BadFileError', + 'ServiceNameUnknownError', 'UserError', 'TimeoutError', 'SSLError', + 'VerifyError', 'PeerVerifyError', 'CertificateError', + 'getConnectError', 'ConnectionClosed', 'ConnectionLost', + 'ConnectionDone', 'ConnectionFdescWentAway', 'AlreadyCalled', + 'AlreadyCancelled', 'PotentialZombieWarning', 'ProcessDone', + 'ProcessTerminated', 'ProcessExitedAlready', 'NotConnectingError', + 'NotListeningError', 'ReactorNotRunning', 'ReactorAlreadyRunning', + 'ReactorAlreadyInstalledError', 'ConnectingCancelledError', + 'UnsupportedAddressFamily', 'UnsupportedSocketType', 'InvalidAddressError'] diff --git a/contrib/python/Twisted/py2/twisted/internet/fdesc.py b/contrib/python/Twisted/py2/twisted/internet/fdesc.py new file mode 100644 index 00000000000..e5a760d7b99 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/fdesc.py @@ -0,0 +1,118 @@ +# -*- test-case-name: twisted.test.test_fdesc -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Utility functions for dealing with POSIX file descriptors. +""" + +import os +import errno +try: + import fcntl +except ImportError: + fcntl = None + +# twisted imports +from twisted.internet.main import CONNECTION_LOST, CONNECTION_DONE + + +def setNonBlocking(fd): + """ + Set the file description of the given file descriptor to non-blocking. + """ + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + flags = flags | os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, flags) + + +def setBlocking(fd): + """ + Set the file description of the given file descriptor to blocking. + """ + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + flags = flags & ~os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, flags) + + +if fcntl is None: + # fcntl isn't available on Windows. By default, handles aren't + # inherited on Windows, so we can do nothing here. + _setCloseOnExec = _unsetCloseOnExec = lambda fd: None +else: + def _setCloseOnExec(fd): + """ + Make a file descriptor close-on-exec. + """ + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags = flags | fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + + def _unsetCloseOnExec(fd): + """ + Make a file descriptor close-on-exec. + """ + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags = flags & ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + +def readFromFD(fd, callback): + """ + Read from file descriptor, calling callback with resulting data. + + If successful, call 'callback' with a single argument: the + resulting data. + + Returns same thing FileDescriptor.doRead would: CONNECTION_LOST, + CONNECTION_DONE, or None. + + @type fd: C{int} + @param fd: non-blocking file descriptor to be read from. + @param callback: a callable which accepts a single argument. If + data is read from the file descriptor it will be called with this + data. Handling exceptions from calling the callback is up to the + caller. + + Note that if the descriptor is still connected but no data is read, + None will be returned but callback will not be called. + + @return: CONNECTION_LOST on error, CONNECTION_DONE when fd is + closed, otherwise None. + """ + try: + output = os.read(fd, 8192) + except (OSError, IOError) as ioe: + if ioe.args[0] in (errno.EAGAIN, errno.EINTR): + return + else: + return CONNECTION_LOST + if not output: + return CONNECTION_DONE + callback(output) + + +def writeToFD(fd, data): + """ + Write data to file descriptor. + + Returns same thing FileDescriptor.writeSomeData would. + + @type fd: C{int} + @param fd: non-blocking file descriptor to be written to. + @type data: C{str} or C{buffer} + @param data: bytes to write to fd. + + @return: number of bytes written, or CONNECTION_LOST. + """ + try: + return os.write(fd, data) + except (OSError, IOError) as io: + if io.errno in (errno.EAGAIN, errno.EINTR): + return 0 + return CONNECTION_LOST + + +__all__ = ["setNonBlocking", "setBlocking", "readFromFD", "writeToFD"] diff --git a/contrib/python/Twisted/py2/twisted/internet/gireactor.py b/contrib/python/Twisted/py2/twisted/internet/gireactor.py new file mode 100644 index 00000000000..f1b0d273337 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/gireactor.py @@ -0,0 +1,188 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module provides support for Twisted to interact with the glib +mainloop via GObject Introspection. + +In order to use this support, simply do the following:: + + from twisted.internet import gireactor + gireactor.install() + +If you wish to use a GApplication, register it with the reactor:: + + from twisted.internet import reactor + reactor.registerGApplication(app) + +Then use twisted.internet APIs as usual. + +On Python 3, pygobject v3.4 or later is required. +""" + +from __future__ import division, absolute_import + +from twisted.python.compat import _PY3 +from twisted.internet.error import ReactorAlreadyRunning +from twisted.internet import _glibbase +from twisted.python import runtime + +if _PY3: + # We require a sufficiently new version of pygobject, so always exists: + _pygtkcompatPresent = True +else: + # We can't just try to import gi.pygtkcompat, because that would import + # gi, and the goal here is to not import gi in cases where that would + # cause segfault. + from twisted.python.modules import theSystemPath + _pygtkcompatPresent = True + try: + theSystemPath["gi.pygtkcompat"] + except KeyError: + _pygtkcompatPresent = False + + +# Modules that we want to ensure aren't imported if we're on older versions of +# GI: +_PYGTK_MODULES = ['gobject', 'glib', 'gio', 'gtk'] + +def _oldGiInit(): + """ + Make sure pygtk and gi aren't loaded at the same time, and import Glib if + possible. + """ + # We can't immediately prevent imports, because that confuses some buggy + # code in gi: + _glibbase.ensureNotImported( + _PYGTK_MODULES, + "Introspected and static glib/gtk bindings must not be mixed; can't " + "import gireactor since pygtk2 module is already imported.") + + global GLib + from gi.repository import GLib + if getattr(GLib, "threads_init", None) is not None: + GLib.threads_init() + + _glibbase.ensureNotImported([], "", + preventImports=_PYGTK_MODULES) + + +if not _pygtkcompatPresent: + # Older versions of gi don't have compatibility layer, so just enforce no + # imports of pygtk and gi at same time: + _oldGiInit() +else: + # Newer version of gi, so we can try to initialize compatibility layer; if + # real pygtk was already imported we'll get ImportError at this point + # rather than segfault, so unconditional import is fine. + import gi.pygtkcompat + gi.pygtkcompat.enable() + # At this point importing gobject will get you gi version, and importing + # e.g. gtk will either fail in non-segfaulty way or use gi version if user + # does gi.pygtkcompat.enable_gtk(). So, no need to prevent imports of + # old school pygtk modules. + from gi.repository import GLib + if getattr(GLib, "threads_init", None) is not None: + GLib.threads_init() + + + +class GIReactor(_glibbase.GlibReactorBase): + """ + GObject-introspection event loop reactor. + + @ivar _gapplication: A C{Gio.Application} instance that was registered + with C{registerGApplication}. + """ + _POLL_DISCONNECTED = (GLib.IOCondition.HUP | GLib.IOCondition.ERR | + GLib.IOCondition.NVAL) + _POLL_IN = GLib.IOCondition.IN + _POLL_OUT = GLib.IOCondition.OUT + + # glib's iochannel sources won't tell us about any events that we haven't + # asked for, even if those events aren't sensible inputs to the poll() + # call. + INFLAGS = _POLL_IN | _POLL_DISCONNECTED + OUTFLAGS = _POLL_OUT | _POLL_DISCONNECTED + + # By default no Application is registered: + _gapplication = None + + + def __init__(self, useGtk=False): + _gtk = None + if useGtk is True: + from gi.repository import Gtk as _gtk + + _glibbase.GlibReactorBase.__init__(self, GLib, _gtk, useGtk=useGtk) + + + def registerGApplication(self, app): + """ + Register a C{Gio.Application} or C{Gtk.Application}, whose main loop + will be used instead of the default one. + + We will C{hold} the application so it doesn't exit on its own. In + versions of C{python-gi} 3.2 and later, we exit the event loop using + the C{app.quit} method which overrides any holds. Older versions are + not supported. + """ + if self._gapplication is not None: + raise RuntimeError( + "Can't register more than one application instance.") + if self._started: + raise ReactorAlreadyRunning( + "Can't register application after reactor was started.") + if not hasattr(app, "quit"): + raise RuntimeError("Application registration is not supported in" + " versions of PyGObject prior to 3.2.") + self._gapplication = app + def run(): + app.hold() + app.run(None) + self._run = run + + self._crash = app.quit + + + +class PortableGIReactor(_glibbase.PortableGlibReactorBase): + """ + Portable GObject Introspection event loop reactor. + """ + def __init__(self, useGtk=False): + _gtk = None + if useGtk is True: + from gi.repository import Gtk as _gtk + + _glibbase.PortableGlibReactorBase.__init__(self, GLib, _gtk, + useGtk=useGtk) + + + def registerGApplication(self, app): + """ + Register a C{Gio.Application} or C{Gtk.Application}, whose main loop + will be used instead of the default one. + """ + raise NotImplementedError("GApplication is not currently supported on Windows.") + + + +def install(useGtk=False): + """ + Configure the twisted mainloop to be run inside the glib mainloop. + + @param useGtk: should GTK+ rather than glib event loop be + used (this will be slightly slower but does support GUI). + """ + if runtime.platform.getType() == 'posix': + reactor = GIReactor(useGtk=useGtk) + else: + reactor = PortableGIReactor(useGtk=useGtk) + + from twisted.internet.main import installReactor + installReactor(reactor) + return reactor + + +__all__ = ['install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/glib2reactor.py b/contrib/python/Twisted/py2/twisted/internet/glib2reactor.py new file mode 100644 index 00000000000..5275efd82b3 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/glib2reactor.py @@ -0,0 +1,44 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module provides support for Twisted to interact with the glib mainloop. +This is like gtk2, but slightly faster and does not require a working +$DISPLAY. However, you cannot run GUIs under this reactor: for that you must +use the gtk2reactor instead. + +In order to use this support, simply do the following:: + + from twisted.internet import glib2reactor + glib2reactor.install() + +Then use twisted.internet APIs as usual. The other methods here are not +intended to be called directly. +""" + +from twisted.internet import gtk2reactor + + +class Glib2Reactor(gtk2reactor.Gtk2Reactor): + """ + The reactor using the glib mainloop. + """ + + def __init__(self): + """ + Override init to set the C{useGtk} flag. + """ + gtk2reactor.Gtk2Reactor.__init__(self, useGtk=False) + + + +def install(): + """ + Configure the twisted mainloop to be run inside the glib mainloop. + """ + reactor = Glib2Reactor() + from twisted.internet.main import installReactor + installReactor(reactor) + + +__all__ = ['install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/gtk2reactor.py b/contrib/python/Twisted/py2/twisted/internet/gtk2reactor.py new file mode 100644 index 00000000000..faf1234645d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/gtk2reactor.py @@ -0,0 +1,121 @@ +# -*- test-case-name: twisted.internet.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +This module provides support for Twisted to interact with the glib/gtk2 +mainloop. + +In order to use this support, simply do the following:: + + from twisted.internet import gtk2reactor + gtk2reactor.install() + +Then use twisted.internet APIs as usual. The other methods here are not +intended to be called directly. +""" + +# System Imports +import sys + +# Twisted Imports +from twisted.internet import _glibbase +from twisted.python import runtime + +# Certain old versions of pygtk and gi crash if imported at the same +# time. This is a problem when running Twisted's unit tests, since they will +# attempt to run both gtk2 and gtk3/gi tests. However, gireactor makes sure +# that if we are in such an old version, and gireactor was imported, +# gtk2reactor will not be importable. So we don't *need* to enforce that here +# as well; whichever is imported first will still win. Moreover, additional +# enforcement in this module is unnecessary in modern versions, and downright +# problematic in certain versions where for some reason importing gtk also +# imports some subset of gi. So we do nothing here, relying on gireactor to +# prevent the crash. + +try: + if not hasattr(sys, 'frozen'): + # Don't want to check this for py2exe + import pygtk + pygtk.require('2.0') +except (ImportError, AttributeError): + pass # maybe we're using pygtk before this hack existed. + +import gobject +if hasattr(gobject, "threads_init"): + # recent versions of python-gtk expose this. python-gtk=2.4.1 + # (wrapping glib-2.4.7) does. python-gtk=2.0.0 (wrapping + # glib-2.2.3) does not. + gobject.threads_init() + + + +class Gtk2Reactor(_glibbase.GlibReactorBase): + """ + PyGTK+ 2 event loop reactor. + """ + _POLL_DISCONNECTED = gobject.IO_HUP | gobject.IO_ERR | gobject.IO_NVAL + _POLL_IN = gobject.IO_IN + _POLL_OUT = gobject.IO_OUT + + # glib's iochannel sources won't tell us about any events that we haven't + # asked for, even if those events aren't sensible inputs to the poll() + # call. + INFLAGS = _POLL_IN | _POLL_DISCONNECTED + OUTFLAGS = _POLL_OUT | _POLL_DISCONNECTED + + def __init__(self, useGtk=True): + _gtk = None + if useGtk is True: + import gtk as _gtk + + _glibbase.GlibReactorBase.__init__(self, gobject, _gtk, useGtk=useGtk) + + + +class PortableGtkReactor(_glibbase.PortableGlibReactorBase): + """ + Reactor that works on Windows. + + Sockets aren't supported by GTK+'s input_add on Win32. + """ + def __init__(self, useGtk=True): + _gtk = None + if useGtk is True: + import gtk as _gtk + + _glibbase.PortableGlibReactorBase.__init__(self, gobject, _gtk, + useGtk=useGtk) + + +def install(useGtk=True): + """ + Configure the twisted mainloop to be run inside the gtk mainloop. + + @param useGtk: should glib rather than GTK+ event loop be + used (this will be slightly faster but does not support GUI). + """ + reactor = Gtk2Reactor(useGtk) + from twisted.internet.main import installReactor + installReactor(reactor) + return reactor + + +def portableInstall(useGtk=True): + """ + Configure the twisted mainloop to be run inside the gtk mainloop. + """ + reactor = PortableGtkReactor() + from twisted.internet.main import installReactor + installReactor(reactor) + return reactor + + +if runtime.platform.getType() == 'posix': + install = install +else: + install = portableInstall + + +__all__ = ['install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/gtk3reactor.py b/contrib/python/Twisted/py2/twisted/internet/gtk3reactor.py new file mode 100644 index 00000000000..256b6985f40 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/gtk3reactor.py @@ -0,0 +1,80 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module provides support for Twisted to interact with the gtk3 mainloop +via Gobject introspection. This is like gi, but slightly slower and requires a +working $DISPLAY. + +In order to use this support, simply do the following:: + + from twisted.internet import gtk3reactor + gtk3reactor.install() + +If you wish to use a GApplication, register it with the reactor:: + + from twisted.internet import reactor + reactor.registerGApplication(app) + +Then use twisted.internet APIs as usual. +""" + +from __future__ import division, absolute_import + +import os + +from twisted.internet import gireactor +from twisted.python import runtime + +# Newer versions of gtk3/pygoject raise a RuntimeError, or just break in a +# confusing manner, if the program is not running under X11. We therefore try +# to fail in a more reasonable manner, and check for $DISPLAY as a reasonable +# approximation of availability of X11. This is somewhat over-aggressive, +# since some older versions of gtk3/pygobject do work with missing $DISPLAY, +# but it's too hard to figure out which, so we always require it. +if (runtime.platform.getType() == 'posix' and + not runtime.platform.isMacOSX() and not os.environ.get("DISPLAY")): + raise ImportError( + "Gtk3 requires X11, and no DISPLAY environment variable is set") + + +class Gtk3Reactor(gireactor.GIReactor): + """ + A reactor using the gtk3+ event loop. + """ + + def __init__(self): + """ + Override init to set the C{useGtk} flag. + """ + gireactor.GIReactor.__init__(self, useGtk=True) + + + +class PortableGtk3Reactor(gireactor.PortableGIReactor): + """ + Portable GTK+ 3.x reactor. + """ + def __init__(self): + """ + Override init to set the C{useGtk} flag. + """ + gireactor.PortableGIReactor.__init__(self, useGtk=True) + + + +def install(): + """ + Configure the Twisted mainloop to be run inside the gtk3+ mainloop. + """ + if runtime.platform.getType() == 'posix': + reactor = Gtk3Reactor() + else: + reactor = PortableGtk3Reactor() + + from twisted.internet.main import installReactor + installReactor(reactor) + return reactor + + +__all__ = ['install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/inotify.py b/contrib/python/Twisted/py2/twisted/internet/inotify.py new file mode 100644 index 00000000000..23d8e8285c0 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/inotify.py @@ -0,0 +1,419 @@ +# -*- test-case-name: twisted.internet.test.test_inotify -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module provides support for Twisted to linux inotify API. + +In order to use this support, simply do the following (and start a reactor +at some point):: + + from twisted.internet import inotify + from twisted.python import filepath + + def notify(ignored, filepath, mask): + \""" + For historical reasons, an opaque handle is passed as first + parameter. This object should never be used. + + @param filepath: FilePath on which the event happened. + @param mask: inotify event as hexadecimal masks + \""" + print("event %s on %s" % ( + ', '.join(inotify.humanReadableMask(mask)), filepath)) + + notifier = inotify.INotify() + notifier.startReading() + notifier.watch(filepath.FilePath("/some/directory"), callbacks=[notify]) + notifier.watch(filepath.FilePath(b"/some/directory2"), callbacks=[notify]) + +Note that in the above example, a L{FilePath} which is a L{bytes} path name +or L{str} path name may be used. However, no matter what type of +L{FilePath} is passed to this module, internally the L{FilePath} is +converted to L{bytes} according to L{sys.getfilesystemencoding}. +For any L{FilePath} returned by this module, the caller is responsible for +converting from a L{bytes} path name to a L{str} path name. + +@since: 10.1 +""" + +from __future__ import print_function + +import os +import struct + +from twisted.internet import fdesc +from twisted.internet.abstract import FileDescriptor +from twisted.python import log, _inotify + + +# from /usr/src/linux/include/linux/inotify.h + +IN_ACCESS = 0x00000001 # File was accessed +IN_MODIFY = 0x00000002 # File was modified +IN_ATTRIB = 0x00000004 # Metadata changed +IN_CLOSE_WRITE = 0x00000008 # Writeable file was closed +IN_CLOSE_NOWRITE = 0x00000010 # Unwriteable file closed +IN_OPEN = 0x00000020 # File was opened +IN_MOVED_FROM = 0x00000040 # File was moved from X +IN_MOVED_TO = 0x00000080 # File was moved to Y +IN_CREATE = 0x00000100 # Subfile was created +IN_DELETE = 0x00000200 # Subfile was delete +IN_DELETE_SELF = 0x00000400 # Self was deleted +IN_MOVE_SELF = 0x00000800 # Self was moved +IN_UNMOUNT = 0x00002000 # Backing fs was unmounted +IN_Q_OVERFLOW = 0x00004000 # Event queued overflowed +IN_IGNORED = 0x00008000 # File was ignored + +IN_ONLYDIR = 0x01000000 # only watch the path if it is a directory +IN_DONT_FOLLOW = 0x02000000 # don't follow a sym link +IN_MASK_ADD = 0x20000000 # add to the mask of an already existing watch +IN_ISDIR = 0x40000000 # event occurred against dir +IN_ONESHOT = 0x80000000 # only send event once + +IN_CLOSE = IN_CLOSE_WRITE | IN_CLOSE_NOWRITE # closes +IN_MOVED = IN_MOVED_FROM | IN_MOVED_TO # moves +IN_CHANGED = IN_MODIFY | IN_ATTRIB # changes + +IN_WATCH_MASK = (IN_MODIFY | IN_ATTRIB | + IN_CREATE | IN_DELETE | + IN_DELETE_SELF | IN_MOVE_SELF | + IN_UNMOUNT | IN_MOVED_FROM | IN_MOVED_TO) + + +_FLAG_TO_HUMAN = [ + (IN_ACCESS, 'access'), + (IN_MODIFY, 'modify'), + (IN_ATTRIB, 'attrib'), + (IN_CLOSE_WRITE, 'close_write'), + (IN_CLOSE_NOWRITE, 'close_nowrite'), + (IN_OPEN, 'open'), + (IN_MOVED_FROM, 'moved_from'), + (IN_MOVED_TO, 'moved_to'), + (IN_CREATE, 'create'), + (IN_DELETE, 'delete'), + (IN_DELETE_SELF, 'delete_self'), + (IN_MOVE_SELF, 'move_self'), + (IN_UNMOUNT, 'unmount'), + (IN_Q_OVERFLOW, 'queue_overflow'), + (IN_IGNORED, 'ignored'), + (IN_ONLYDIR, 'only_dir'), + (IN_DONT_FOLLOW, 'dont_follow'), + (IN_MASK_ADD, 'mask_add'), + (IN_ISDIR, 'is_dir'), + (IN_ONESHOT, 'one_shot') +] + + + +def humanReadableMask(mask): + """ + Auxiliary function that converts a hexadecimal mask into a series + of human readable flags. + """ + s = [] + for k, v in _FLAG_TO_HUMAN: + if k & mask: + s.append(v) + return s + + + +class _Watch(object): + """ + Watch object that represents a Watch point in the filesystem. The + user should let INotify to create these objects + + @ivar path: The path over which this watch point is monitoring + @ivar mask: The events monitored by this watchpoint + @ivar autoAdd: Flag that determines whether this watch point + should automatically add created subdirectories + @ivar callbacks: L{list} of callback functions that will be called + when an event occurs on this watch. + """ + def __init__(self, path, mask=IN_WATCH_MASK, autoAdd=False, + callbacks=None): + self.path = path.asBytesMode() + self.mask = mask + self.autoAdd = autoAdd + if callbacks is None: + callbacks = [] + self.callbacks = callbacks + + + def _notify(self, filepath, events): + """ + Callback function used by L{INotify} to dispatch an event. + """ + filepath = filepath.asBytesMode() + for callback in self.callbacks: + callback(self, filepath, events) + + + +class INotify(FileDescriptor, object): + """ + The INotify file descriptor, it basically does everything related + to INotify, from reading to notifying watch points. + + @ivar _buffer: a L{bytes} containing the data read from the inotify fd. + + @ivar _watchpoints: a L{dict} that maps from inotify watch ids to + watchpoints objects + + @ivar _watchpaths: a L{dict} that maps from watched paths to the + inotify watch ids + """ + _inotify = _inotify + + def __init__(self, reactor=None): + FileDescriptor.__init__(self, reactor=reactor) + + # Smart way to allow parametrization of libc so I can override + # it and test for the system errors. + self._fd = self._inotify.init() + + fdesc.setNonBlocking(self._fd) + fdesc._setCloseOnExec(self._fd) + + # The next 2 lines are needed to have self.loseConnection() + # to call connectionLost() on us. Since we already created the + # fd that talks to inotify we want to be notified even if we + # haven't yet started reading. + self.connected = 1 + self._writeDisconnected = True + + self._buffer = b'' + self._watchpoints = {} + self._watchpaths = {} + + + def _addWatch(self, path, mask, autoAdd, callbacks): + """ + Private helper that abstracts the use of ctypes. + + Calls the internal inotify API and checks for any errors after the + call. If there's an error L{INotify._addWatch} can raise an + INotifyError. If there's no error it proceeds creating a watchpoint and + adding a watchpath for inverse lookup of the file descriptor from the + path. + """ + path = path.asBytesMode() + wd = self._inotify.add(self._fd, path, mask) + + iwp = _Watch(path, mask, autoAdd, callbacks) + + self._watchpoints[wd] = iwp + self._watchpaths[path] = wd + + return wd + + + def _rmWatch(self, wd): + """ + Private helper that abstracts the use of ctypes. + + Calls the internal inotify API to remove an fd from inotify then + removes the corresponding watchpoint from the internal mapping together + with the file descriptor from the watchpath. + """ + self._inotify.remove(self._fd, wd) + iwp = self._watchpoints.pop(wd) + self._watchpaths.pop(iwp.path) + + + def connectionLost(self, reason): + """ + Release the inotify file descriptor and do the necessary cleanup + """ + FileDescriptor.connectionLost(self, reason) + if self._fd >= 0: + try: + os.close(self._fd) + except OSError as e: + log.err(e, "Couldn't close INotify file descriptor.") + + + def fileno(self): + """ + Get the underlying file descriptor from this inotify observer. + Required by L{abstract.FileDescriptor} subclasses. + """ + return self._fd + + + def doRead(self): + """ + Read some data from the observed file descriptors + """ + fdesc.readFromFD(self._fd, self._doRead) + + + def _doRead(self, in_): + """ + Work on the data just read from the file descriptor. + """ + self._buffer += in_ + while len(self._buffer) >= 16: + + wd, mask, cookie, size = struct.unpack("=LLLL", self._buffer[0:16]) + + if size: + name = self._buffer[16:16 + size].rstrip(b'\0') + else: + name = None + + self._buffer = self._buffer[16 + size:] + + try: + iwp = self._watchpoints[wd] + except KeyError: + continue + + path = iwp.path.asBytesMode() + if name: + path = path.child(name) + iwp._notify(path, mask) + + if (iwp.autoAdd and mask & IN_ISDIR and mask & IN_CREATE): + # mask & IN_ISDIR already guarantees that the path is a + # directory. There's no way you can get here without a + # directory anyway, so no point in checking for that again. + new_wd = self.watch( + path, mask=iwp.mask, autoAdd=True, + callbacks=iwp.callbacks + ) + # This is very very very hacky and I'd rather not do this but + # we have no other alternative that is less hacky other than + # surrender. We use callLater because we don't want to have + # too many events waiting while we process these subdirs, we + # must always answer events as fast as possible or the overflow + # might come. + self.reactor.callLater(0, + self._addChildren, self._watchpoints[new_wd]) + if mask & IN_DELETE_SELF: + self._rmWatch(wd) + + + def _addChildren(self, iwp): + """ + This is a very private method, please don't even think about using it. + + Note that this is a fricking hack... it's because we cannot be fast + enough in adding a watch to a directory and so we basically end up + getting here too late if some operations have already been going on in + the subdir, we basically need to catchup. This eventually ends up + meaning that we generate double events, your app must be resistant. + """ + try: + listdir = iwp.path.children() + except OSError: + # Somebody or something (like a test) removed this directory while + # we were in the callLater(0...) waiting. It doesn't make sense to + # process it anymore + return + + # note that it's true that listdir will only see the subdirs inside + # path at the moment of the call but path is monitored already so if + # something is created we will receive an event. + for f in listdir: + # It's a directory, watch it and then add its children + if f.isdir(): + wd = self.watch( + f, mask=iwp.mask, autoAdd=True, + callbacks=iwp.callbacks + ) + iwp._notify(f, IN_ISDIR|IN_CREATE) + # now f is watched, we can add its children the callLater is to + # avoid recursion + self.reactor.callLater(0, + self._addChildren, self._watchpoints[wd]) + + # It's a file and we notify it. + if f.isfile(): + iwp._notify(f, IN_CREATE|IN_CLOSE_WRITE) + + + def watch(self, path, mask=IN_WATCH_MASK, autoAdd=False, + callbacks=None, recursive=False): + """ + Watch the 'mask' events in given path. Can raise C{INotifyError} when + there's a problem while adding a directory. + + @param path: The path needing monitoring + @type path: L{FilePath} + + @param mask: The events that should be watched + @type mask: L{int} + + @param autoAdd: if True automatically add newly created + subdirectories + @type autoAdd: L{bool} + + @param callbacks: A list of callbacks that should be called + when an event happens in the given path. + The callback should accept 3 arguments: + (ignored, filepath, mask) + @type callbacks: L{list} of callables + + @param recursive: Also add all the subdirectories in this path + @type recursive: L{bool} + """ + if recursive: + # This behavior is needed to be compatible with the windows + # interface for filesystem changes: + # http://msdn.microsoft.com/en-us/library/aa365465(VS.85).aspx + # ReadDirectoryChangesW can do bWatchSubtree so it doesn't + # make sense to implement this at a higher abstraction + # level when other platforms support it already + for child in path.walk(): + if child.isdir(): + self.watch(child, mask, autoAdd, callbacks, + recursive=False) + else: + wd = self._isWatched(path) + if wd: + return wd + + mask = mask | IN_DELETE_SELF # need this to remove the watch + + return self._addWatch(path, mask, autoAdd, callbacks) + + + def ignore(self, path): + """ + Remove the watch point monitoring the given path + + @param path: The path that should be ignored + @type path: L{FilePath} + """ + path = path.asBytesMode() + wd = self._isWatched(path) + if wd is None: + raise KeyError("%r is not watched" % (path,)) + else: + self._rmWatch(wd) + + + def _isWatched(self, path): + """ + Helper function that checks if the path is already monitored + and returns its watchdescriptor if so or None otherwise. + + @param path: The path that should be checked + @type path: L{FilePath} + """ + path = path.asBytesMode() + return self._watchpaths.get(path, None) + + +INotifyError = _inotify.INotifyError + + +__all__ = ["INotify", "humanReadableMask", "IN_WATCH_MASK", "IN_ACCESS", + "IN_MODIFY", "IN_ATTRIB", "IN_CLOSE_NOWRITE", "IN_CLOSE_WRITE", + "IN_OPEN", "IN_MOVED_FROM", "IN_MOVED_TO", "IN_CREATE", + "IN_DELETE", "IN_DELETE_SELF", "IN_MOVE_SELF", "IN_UNMOUNT", + "IN_Q_OVERFLOW", "IN_IGNORED", "IN_ONLYDIR", "IN_DONT_FOLLOW", + "IN_MASK_ADD", "IN_ISDIR", "IN_ONESHOT", "IN_CLOSE", + "IN_MOVED", "IN_CHANGED"] diff --git a/contrib/python/Twisted/py2/twisted/internet/interfaces.py b/contrib/python/Twisted/py2/twisted/internet/interfaces.py new file mode 100644 index 00000000000..4b4984325d7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/interfaces.py @@ -0,0 +1,2915 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Interface documentation. + +Maintainer: Itamar Shtull-Trauring +""" + +from __future__ import division, absolute_import + +from zope.interface import Interface, Attribute + + +class IAddress(Interface): + """ + An address, e.g. a TCP C{(host, port)}. + + Default implementations are in L{twisted.internet.address}. + """ + +### Reactor Interfaces + +class IConnector(Interface): + """ + Object used to interface between connections and protocols. + + Each L{IConnector} manages one connection. + """ + + def stopConnecting(): + """ + Stop attempting to connect. + """ + + def disconnect(): + """ + Disconnect regardless of the connection state. + + If we are connected, disconnect, if we are trying to connect, + stop trying. + """ + + def connect(): + """ + Try to connect to remote address. + """ + + def getDestination(): + """ + Return destination this will try to connect to. + + @return: An object which provides L{IAddress}. + """ + + + +class IResolverSimple(Interface): + def getHostByName(name, timeout = (1, 3, 11, 45)): + """ + Resolve the domain name C{name} into an IP address. + + @type name: C{bytes} or C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{twisted.internet.defer.Deferred} + @return: The callback of the Deferred that is returned will be + passed a string that represents the IP address of the + specified name, or the errback will be called if the + lookup times out. If multiple types of address records + are associated with the name, A6 records will be returned + in preference to AAAA records, which will be returned in + preference to A records. If there are multiple records of + the type to be returned, one will be selected at random. + + @raise twisted.internet.defer.TimeoutError: Raised + (asynchronously) if the name cannot be resolved within the + specified timeout period. + """ + + + +class IHostResolution(Interface): + """ + An L{IHostResolution} represents represents an in-progress recursive query + for a DNS name. + + @since: Twisted 17.1.0 + """ + + name = Attribute( + """ + L{unicode}; the name of the host being resolved. + """ + ) + + def cancel(): + """ + Stop the hostname resolution in progress. + """ + + + +class IResolutionReceiver(Interface): + """ + An L{IResolutionReceiver} receives the results of a hostname resolution in + progress, initiated by an L{IHostnameResolver}. + + @since: Twisted 17.1.0 + """ + + def resolutionBegan(resolutionInProgress): + """ + A hostname resolution began. + + @param resolutionInProgress: an L{IHostResolution}. + """ + + + def addressResolved(address): + """ + An internet address. This is called when an address for the given name + is discovered. In the current implementation this practically means + L{IPv4Address} or L{IPv6Address}, but implementations of this interface + should be lenient to other types being passed to this interface as + well, for future-proofing. + + @param address: An address object. + @type address: L{IAddress} + """ + + + def resolutionComplete(): + """ + Resolution has completed; no further addresses will be relayed to + L{IResolutionReceiver.addressResolved}. + """ + + + +class IHostnameResolver(Interface): + """ + An L{IHostnameResolver} can resolve a host name and port number into a + series of L{IAddress} objects. + + @since: Twisted 17.1.0 + """ + + def resolveHostName(resolutionReceiver, hostName, portNumber=0, + addressTypes=None, transportSemantics='TCP'): + """ + Initiate a hostname resolution. + + @param resolutionReceiver: an object that will receive each resolved + address as it arrives. + @type resolutionReceiver: L{IResolutionReceiver} + + @param hostName: The name of the host to resolve. If this contains + non-ASCII code points, they will be converted to IDNA first. + @type hostName: L{unicode} + + @param portNumber: The port number that the returned addresses should + include. + @type portNumber: L{int} greater than or equal to 0 and less than 65536 + + @param addressTypes: An iterable of implementors of L{IAddress} that + are acceptable values for C{resolutionReceiver} to receive to its + L{addressResolved }. In + practice, this means an iterable containing + L{twisted.internet.address.IPv4Address}, + L{twisted.internet.address.IPv6Address}, both, or neither. + @type addressTypes: L{collections.abc.Iterable} of L{type} + + @param transportSemantics: A string describing the semantics of the + transport; either C{'TCP'} for stream-oriented transports or + C{'UDP'} for datagram-oriented; see + L{twisted.internet.address.IPv6Address.type} and + L{twisted.internet.address.IPv4Address.type}. + @type transportSemantics: native L{str} + + @return: The resolution in progress. + @rtype: L{IResolutionReceiver} + """ + + + +class IResolver(IResolverSimple): + def query(query, timeout=None): + """ + Dispatch C{query} to the method which can handle its type. + + @type query: L{twisted.names.dns.Query} + @param query: The DNS query being issued, to which a response is to be + generated. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAddress(name, timeout=None): + """ + Perform an A record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAddress6(name, timeout=None): + """ + Perform an A6 record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupIPV6Address(name, timeout=None): + """ + Perform an AAAA record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailExchange(name, timeout=None): + """ + Perform an MX record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupNameservers(name, timeout=None): + """ + Perform an NS record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupCanonicalName(name, timeout=None): + """ + Perform a CNAME record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailBox(name, timeout=None): + """ + Perform an MB record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailGroup(name, timeout=None): + """ + Perform an MG record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailRename(name, timeout=None): + """ + Perform an MR record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupPointer(name, timeout=None): + """ + Perform a PTR record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAuthority(name, timeout=None): + """ + Perform an SOA record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupNull(name, timeout=None): + """ + Perform a NULL record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupWellKnownServices(name, timeout=None): + """ + Perform a WKS record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupHostInfo(name, timeout=None): + """ + Perform a HINFO record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailboxInfo(name, timeout=None): + """ + Perform an MINFO record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupText(name, timeout=None): + """ + Perform a TXT record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupResponsibility(name, timeout=None): + """ + Perform an RP record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAFSDatabase(name, timeout=None): + """ + Perform an AFSDB record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupService(name, timeout=None): + """ + Perform an SRV record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAllRecords(name, timeout=None): + """ + Perform an ALL_RECORD lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupSenderPolicy(name, timeout= 10): + """ + Perform a SPF record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupNamingAuthorityPointer(name, timeout=None): + """ + Perform a NAPTR record lookup. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupZone(name, timeout=None): + """ + Perform an AXFR record lookup. + + NB This is quite different from other DNS requests. See + U{http://cr.yp.to/djbdns/axfr-notes.html} for more + information. + + NB Unlike other C{lookup*} methods, the timeout here is not a + list of ints, it is a single int. + + @type name: L{bytes} or L{str} + @param name: DNS name to resolve. + + @type timeout: C{int} + @param timeout: When this timeout expires, the query is + considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. + The first element of the tuple gives answers. + The second and third elements are always empty. + The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + +class IReactorTCP(Interface): + + def listenTCP(port, factory, backlog=50, interface=''): + """ + Connects a given protocol factory to the given numeric TCP/IP port. + + @param port: a port number on which to listen + + @param factory: a L{twisted.internet.protocol.ServerFactory} instance + + @param backlog: size of the listen queue + + @param interface: The local IPv4 or IPv6 address to which to bind; + defaults to '', ie all IPv4 addresses. To bind to all IPv4 and IPv6 + addresses, you must call this method twice. + + @return: an object that provides L{IListeningPort}. + + @raise CannotListenError: as defined here + L{twisted.internet.error.CannotListenError}, + if it cannot listen on this port (e.g., it + cannot bind to the required port number) + """ + + def connectTCP(host, port, factory, timeout=30, bindAddress=None): + """ + Connect a TCP client. + + @param host: A hostname or an IPv4 or IPv6 address literal. + + @type host: L{bytes} + + @param port: a port number + + @param factory: a L{twisted.internet.protocol.ClientFactory} instance + + @param timeout: number of seconds to wait before assuming the + connection has failed. + + @param bindAddress: a (host, port) tuple of local address to bind + to, or None. + + @return: An object which provides L{IConnector}. This connector will + call various callbacks on the factory when a connection is + made, failed, or lost - see + L{ClientFactory} + docs for details. + """ + +class IReactorSSL(Interface): + + def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None): + """ + Connect a client Protocol to a remote SSL socket. + + @param host: a host name + + @param port: a port number + + @param factory: a L{twisted.internet.protocol.ClientFactory} instance + + @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object. + + @param timeout: number of seconds to wait before assuming the + connection has failed. + + @param bindAddress: a (host, port) tuple of local address to bind to, + or L{None}. + + @return: An object which provides L{IConnector}. + """ + + def listenSSL(port, factory, contextFactory, backlog=50, interface=''): + """ + Connects a given protocol factory to the given numeric TCP/IP port. + The connection is a SSL one, using contexts created by the context + factory. + + @param port: a port number on which to listen + + @param factory: a L{twisted.internet.protocol.ServerFactory} instance + + @param contextFactory: an implementor of L{IOpenSSLContextFactory} + + @param backlog: size of the listen queue + + @param interface: the hostname to bind to, defaults to '' (all) + """ + + + +class IReactorUNIX(Interface): + """ + UNIX socket methods. + """ + + def connectUNIX(address, factory, timeout=30, checkPID=0): + """ + Connect a client protocol to a UNIX socket. + + @param address: a path to a unix socket on the filesystem. + + @param factory: a L{twisted.internet.protocol.ClientFactory} instance + + @param timeout: number of seconds to wait before assuming the connection + has failed. + + @param checkPID: if True, check for a pid file to verify that a server + is listening. If C{address} is a Linux abstract namespace path, + this must be C{False}. + + @return: An object which provides L{IConnector}. + """ + + + def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0): + """ + Listen on a UNIX socket. + + @param address: a path to a unix socket on the filesystem. + + @param factory: a L{twisted.internet.protocol.Factory} instance. + + @param backlog: number of connections to allow in backlog. + + @param mode: The mode (B{not} umask) to set on the unix socket. See + platform specific documentation for information about how this + might affect connection attempts. + @type mode: C{int} + + @param wantPID: if True, create a pidfile for the socket. If C{address} + is a Linux abstract namespace path, this must be C{False}. + + @return: An object which provides L{IListeningPort}. + """ + + + +class IReactorUNIXDatagram(Interface): + """ + Datagram UNIX socket methods. + """ + + def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None): + """ + Connect a client protocol to a datagram UNIX socket. + + @param address: a path to a unix socket on the filesystem. + + @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance + + @param maxPacketSize: maximum packet size to accept + + @param mode: The mode (B{not} umask) to set on the unix socket. See + platform specific documentation for information about how this + might affect connection attempts. + @type mode: C{int} + + @param bindAddress: address to bind to + + @return: An object which provides L{IConnector}. + """ + + + def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666): + """ + Listen on a datagram UNIX socket. + + @param address: a path to a unix socket on the filesystem. + + @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance. + + @param maxPacketSize: maximum packet size to accept + + @param mode: The mode (B{not} umask) to set on the unix socket. See + platform specific documentation for information about how this + might affect connection attempts. + @type mode: C{int} + + @return: An object which provides L{IListeningPort}. + """ + + + +class IReactorWin32Events(Interface): + """ + Win32 Event API methods + + @since: 10.2 + """ + + def addEvent(event, fd, action): + """ + Add a new win32 event to the event loop. + + @param event: a Win32 event object created using win32event.CreateEvent() + + @param fd: an instance of L{twisted.internet.abstract.FileDescriptor} + + @param action: a string that is a method name of the fd instance. + This method is called in response to the event. + + @return: None + """ + + + def removeEvent(event): + """ + Remove an event. + + @param event: a Win32 event object added using L{IReactorWin32Events.addEvent} + + @return: None + """ + + + +class IReactorUDP(Interface): + """ + UDP socket methods. + """ + + def listenUDP(port, protocol, interface='', maxPacketSize=8192): + """ + Connects a given L{DatagramProtocol} to the given numeric UDP port. + + @param port: A port number on which to listen. + @type port: C{int} + + @param protocol: A L{DatagramProtocol} instance which will be + connected to the given C{port}. + @type protocol: L{DatagramProtocol} + + @param interface: The local IPv4 or IPv6 address to which to bind; + defaults to '', ie all IPv4 addresses. + @type interface: C{str} + + @param maxPacketSize: The maximum packet size to accept. + @type maxPacketSize: C{int} + + @return: object which provides L{IListeningPort}. + """ + + + +class IReactorMulticast(Interface): + """ + UDP socket methods that support multicast. + + IMPORTANT: This is an experimental new interface. It may change + without backwards compatibility. Suggestions are welcome. + """ + + def listenMulticast(port, protocol, interface='', maxPacketSize=8192, + listenMultiple=False): + """ + Connects a given + L{DatagramProtocol} to the + given numeric UDP port. + + @param listenMultiple: If set to True, allows multiple sockets to + bind to the same address and port number at the same time. + @type listenMultiple: C{bool} + + @returns: An object which provides L{IListeningPort}. + + @see: L{twisted.internet.interfaces.IMulticastTransport} + @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} + """ + + + +class IReactorSocket(Interface): + """ + Methods which allow a reactor to use externally created sockets. + + For example, to use C{adoptStreamPort} to implement behavior equivalent + to that of L{IReactorTCP.listenTCP}, you might write code like this:: + + from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket + portSocket = socket(AF_INET, SOCK_STREAM) + # Set FD_CLOEXEC on port, left as an exercise. Then make it into a + # non-blocking listening port: + portSocket.setblocking(False) + portSocket.bind(('192.168.1.2', 12345)) + portSocket.listen(SOMAXCONN) + + # Now have the reactor use it as a TCP port + port = reactor.adoptStreamPort( + portSocket.fileno(), AF_INET, YourFactory()) + + # portSocket itself is no longer necessary, and needs to be cleaned + # up by us. + portSocket.close() + + # Whenever the server is no longer needed, stop it as usual. + stoppedDeferred = port.stopListening() + + Another potential use is to inherit a listening descriptor from a parent + process (for example, systemd or launchd), or to receive one over a UNIX + domain socket. + + Some plans for extending this interface exist. See: + + - U{http://twistedmatrix.com/trac/ticket/6594}: AF_UNIX SOCK_DGRAM ports + """ + + def adoptStreamPort(fileDescriptor, addressFamily, factory): + """ + Add an existing listening I{SOCK_STREAM} socket to the reactor to + monitor for new connections to accept and handle. + + @param fileDescriptor: A file descriptor associated with a socket which + is already bound to an address and marked as listening. The socket + must be set non-blocking. Any additional flags (for example, + close-on-exec) must also be set by application code. Application + code is responsible for closing the file descriptor, which may be + done as soon as C{adoptStreamPort} returns. + @type fileDescriptor: C{int} + + @param addressFamily: The address family (or I{domain}) of the socket. + For example, L{socket.AF_INET6}. + + @param factory: A L{ServerFactory} instance to use to create new + protocols to handle connections accepted via this socket. + + @return: An object providing L{IListeningPort}. + + @raise twisted.internet.error.UnsupportedAddressFamily: If the + given address family is not supported by this reactor, or + not supported with the given socket type. + + @raise twisted.internet.error.UnsupportedSocketType: If the + given socket type is not supported by this reactor, or not + supported with the given socket type. + """ + + + def adoptStreamConnection(fileDescriptor, addressFamily, factory): + """ + Add an existing connected I{SOCK_STREAM} socket to the reactor to + monitor for data. + + Note that the given factory won't have its C{startFactory} and + C{stopFactory} methods called, as there is no sensible time to call + them in this situation. + + @param fileDescriptor: A file descriptor associated with a socket which + is already connected. The socket must be set non-blocking. Any + additional flags (for example, close-on-exec) must also be set by + application code. Application code is responsible for closing the + file descriptor, which may be done as soon as + C{adoptStreamConnection} returns. + @type fileDescriptor: C{int} + + @param addressFamily: The address family (or I{domain}) of the socket. + For example, L{socket.AF_INET6}. + + @param factory: A L{ServerFactory} instance to use to create a new + protocol to handle the connection via this socket. + + @raise UnsupportedAddressFamily: If the given address family is not + supported by this reactor, or not supported with the given socket + type. + + @raise UnsupportedSocketType: If the given socket type is not supported + by this reactor, or not supported with the given socket type. + """ + + + def adoptDatagramPort(fileDescriptor, addressFamily, protocol, + maxPacketSize=8192): + """ + Add an existing listening I{SOCK_DGRAM} socket to the reactor to + monitor for read and write readiness. + + @param fileDescriptor: A file descriptor associated with a socket which + is already bound to an address and marked as listening. The socket + must be set non-blocking. Any additional flags (for example, + close-on-exec) must also be set by application code. Application + code is responsible for closing the file descriptor, which may be + done as soon as C{adoptDatagramPort} returns. + @type fileDescriptor: C{int} + + @param addressFamily: The address family or I{domain} of the socket. + For example, L{socket.AF_INET6}. + @type addressFamily: C{int} + + @param protocol: A L{DatagramProtocol} instance to connect to + a UDP transport. + @type protocol: L{DatagramProtocol} + + @param maxPacketSize: The maximum packet size to accept. + @type maxPacketSize: C{int} + + @return: An object providing L{IListeningPort}. + + @raise UnsupportedAddressFamily: If the given address family is not + supported by this reactor, or not supported with the given socket + type. + + @raise UnsupportedSocketType: If the given socket type is not supported + by this reactor, or not supported with the given socket type. + """ + + + +class IReactorProcess(Interface): + + def spawnProcess(processProtocol, executable, args=(), env={}, path=None, + uid=None, gid=None, usePTY=0, childFDs=None): + """ + Spawn a process, with a process protocol. + + Arguments given to this function that are listed as L{bytes} or + L{unicode} may be encoded or decoded depending on the platform and the + argument type given. On UNIX systems (Linux, FreeBSD, macOS) and + Python 2 on Windows, L{unicode} arguments will be encoded down to + L{bytes} using the encoding given by L{os.getfilesystemencoding}, to be + used with the "narrow" OS APIs. On Python 3 on Windows, L{bytes} + arguments will be decoded up to L{unicode} using the encoding given by + L{os.getfilesystemencoding} (C{mbcs} before Python 3.6, C{utf8} + thereafter) and given to Windows's native "wide" APIs. + + @type processProtocol: L{IProcessProtocol} provider + @param processProtocol: An object which will be notified of all events + related to the created process. + + @param executable: the file name to spawn - the full path should be + used. + @type executable: L{bytes} or L{unicode} + + @param args: the command line arguments to pass to the process; a + sequence of strings. The first string should be the executable's + name. + @type args: L{list} with L{bytes} or L{unicode} items. + + @type env: a L{dict} mapping L{bytes}/L{unicode} keys to + L{bytes}/L{unicode} items, or L{None}. + @param env: the environment variables to pass to the child process. + The resulting behavior varies between platforms. If: + + - C{env} is not set: + - On POSIX: pass an empty environment. + - On Windows: pass L{os.environ}. + - C{env} is L{None}: + - On POSIX: pass L{os.environ}. + - On Windows: pass L{os.environ}. + - C{env} is a L{dict}: + - On POSIX: pass the key/value pairs in C{env} as the + complete environment. + - On Windows: update L{os.environ} with the key/value + pairs in the L{dict} before passing it. As a + consequence of U{bug #1640 + }, passing + keys with empty values in an effort to unset + environment variables I{won't} unset them. + + @param path: the path to run the subprocess in - defaults to the + current directory. + @type path: L{bytes} or L{unicode} or L{None} + + @param uid: user ID to run the subprocess as. (Only available on POSIX + systems.) + + @param gid: group ID to run the subprocess as. (Only available on + POSIX systems.) + + @param usePTY: if true, run this process in a pseudo-terminal. + optionally a tuple of C{(masterfd, slavefd, ttyname)}, in which + case use those file descriptors. (Not available on all systems.) + + @param childFDs: A dictionary mapping file descriptors in the new child + process to an integer or to the string 'r' or 'w'. + + If the value is an integer, it specifies a file descriptor in the + parent process which will be mapped to a file descriptor (specified + by the key) in the child process. This is useful for things like + inetd and shell-like file redirection. + + If it is the string 'r', a pipe will be created and attached to the + child at that file descriptor: the child will be able to write to + that file descriptor and the parent will receive read notification + via the L{IProcessProtocol.childDataReceived} callback. This is + useful for the child's stdout and stderr. + + If it is the string 'w', similar setup to the previous case will + occur, with the pipe being readable by the child instead of + writeable. The parent process can write to that file descriptor + using L{IProcessTransport.writeToChild}. This is useful for the + child's stdin. + + If childFDs is not passed, the default behaviour is to use a + mapping that opens the usual stdin/stdout/stderr pipes. + @type childFDs: L{dict} of L{int} to L{int} or L{str} + + @see: L{twisted.internet.protocol.ProcessProtocol} + + @return: An object which provides L{IProcessTransport}. + + @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are + insufficient system resources to create a new process. + """ + +class IReactorTime(Interface): + """ + Time methods that a Reactor should implement. + """ + + def seconds(): + """ + Get the current time in seconds. + + @return: A number-like object of some sort. + """ + + + def callLater(delay, callable, *args, **kw): + """ + Call a function later. + + @type delay: C{float} + @param delay: the number of seconds to wait. + + @param callable: the callable object to call later. + + @param args: the arguments to call it with. + + @param kw: the keyword arguments to call it with. + + @return: An object which provides L{IDelayedCall} and can be used to + cancel the scheduled call, by calling its C{cancel()} method. + It also may be rescheduled by calling its C{delay()} or + C{reset()} methods. + """ + + + def getDelayedCalls(): + """ + Retrieve all currently scheduled delayed calls. + + @return: A list of L{IDelayedCall} providers representing all + currently scheduled calls. This is everything that has been + returned by C{callLater} but not yet called or cancelled. + """ + + +class IDelayedCall(Interface): + """ + A scheduled call. + + There are probably other useful methods we can add to this interface; + suggestions are welcome. + """ + + def getTime(): + """ + Get time when delayed call will happen. + + @return: time in seconds since epoch (a float). + """ + + def cancel(): + """ + Cancel the scheduled call. + + @raises twisted.internet.error.AlreadyCalled: if the call has already + happened. + @raises twisted.internet.error.AlreadyCancelled: if the call has already + been cancelled. + """ + + def delay(secondsLater): + """ + Delay the scheduled call. + + @param secondsLater: how many seconds from its current firing time to delay + + @raises twisted.internet.error.AlreadyCalled: if the call has already + happened. + @raises twisted.internet.error.AlreadyCancelled: if the call has already + been cancelled. + """ + + def reset(secondsFromNow): + """ + Reset the scheduled call's timer. + + @param secondsFromNow: how many seconds from now it should fire, + equivalent to C{.cancel()} and then doing another + C{reactor.callLater(secondsLater, ...)} + + @raises twisted.internet.error.AlreadyCalled: if the call has already + happened. + @raises twisted.internet.error.AlreadyCancelled: if the call has already + been cancelled. + """ + + def active(): + """ + @return: True if this call is still active, False if it has been + called or cancelled. + """ + + + +class IReactorFromThreads(Interface): + """ + This interface is the set of thread-safe methods which may be invoked on + the reactor from other threads. + + @since: 15.4 + """ + + def callFromThread(callable, *args, **kw): + """ + Cause a function to be executed by the reactor thread. + + Use this method when you want to run a function in the reactor's thread + from another thread. Calling L{callFromThread} should wake up the main + thread (where L{reactor.run() } is executing) and run + the given callable in that thread. + + If you're writing a multi-threaded application the C{callable} may need + to be thread safe, but this method doesn't require it as such. If you + want to call a function in the next mainloop iteration, but you're in + the same thread, use L{callLater} with a delay of 0. + """ + + +class IReactorInThreads(Interface): + """ + This interface contains the methods exposed by a reactor which will let you + run functions in another thread. + + @since: 15.4 + """ + + def callInThread(callable, *args, **kwargs): + """ + Run the given callable object in a separate thread, with the given + arguments and keyword arguments. + """ + + + +class IReactorThreads(IReactorFromThreads, IReactorInThreads): + """ + Dispatch methods to be run in threads. + + Internally, this should use a thread pool and dispatch methods to them. + """ + + def getThreadPool(): + """ + Return the threadpool used by L{IReactorInThreads.callInThread}. + Create it first if necessary. + + @rtype: L{twisted.python.threadpool.ThreadPool} + """ + + + def suggestThreadPoolSize(size): + """ + Suggest the size of the internal threadpool used to dispatch functions + passed to L{IReactorInThreads.callInThread}. + """ + + + +class IReactorCore(Interface): + """ + Core methods that a Reactor must implement. + """ + + running = Attribute( + "A C{bool} which is C{True} from I{during startup} to " + "I{during shutdown} and C{False} the rest of the time.") + + + def resolve(name, timeout=10): + """ + Return a L{twisted.internet.defer.Deferred} that will resolve a hostname. + """ + + def run(): + """ + Fire 'startup' System Events, move the reactor to the 'running' + state, then run the main loop until it is stopped with C{stop()} or + C{crash()}. + """ + + def stop(): + """ + Fire 'shutdown' System Events, which will move the reactor to the + 'stopped' state and cause C{reactor.run()} to exit. + """ + + def crash(): + """ + Stop the main loop *immediately*, without firing any system events. + + This is named as it is because this is an extremely "rude" thing to do; + it is possible to lose data and put your system in an inconsistent + state by calling this. However, it is necessary, as sometimes a system + can become wedged in a pre-shutdown call. + """ + + def iterate(delay=0): + """ + Run the main loop's I/O polling function for a period of time. + + This is most useful in applications where the UI is being drawn "as + fast as possible", such as games. All pending L{IDelayedCall}s will + be called. + + The reactor must have been started (via the C{run()} method) prior to + any invocations of this method. It must also be stopped manually + after the last call to this method (via the C{stop()} method). This + method is not re-entrant: you must not call it recursively; in + particular, you must not call it while the reactor is running. + """ + + def fireSystemEvent(eventType): + """ + Fire a system-wide event. + + System-wide events are things like 'startup', 'shutdown', and + 'persist'. + """ + + def addSystemEventTrigger(phase, eventType, callable, *args, **kw): + """ + Add a function to be called when a system event occurs. + + Each "system event" in Twisted, such as 'startup', 'shutdown', and + 'persist', has 3 phases: 'before', 'during', and 'after' (in that + order, of course). These events will be fired internally by the + Reactor. + + An implementor of this interface must only implement those events + described here. + + Callbacks registered for the "before" phase may return either None or a + Deferred. The "during" phase will not execute until all of the + Deferreds from the "before" phase have fired. + + Once the "during" phase is running, all of the remaining triggers must + execute; their return values must be ignored. + + @param phase: a time to call the event -- either the string 'before', + 'after', or 'during', describing when to call it + relative to the event's execution. + + @param eventType: this is a string describing the type of event. + + @param callable: the object to call before shutdown. + + @param args: the arguments to call it with. + + @param kw: the keyword arguments to call it with. + + @return: an ID that can be used to remove this call with + removeSystemEventTrigger. + """ + + def removeSystemEventTrigger(triggerID): + """ + Removes a trigger added with addSystemEventTrigger. + + @param triggerID: a value returned from addSystemEventTrigger. + + @raise KeyError: If there is no system event trigger for the given + C{triggerID}. + + @raise ValueError: If there is no system event trigger for the given + C{triggerID}. + + @raise TypeError: If there is no system event trigger for the given + C{triggerID}. + """ + + def callWhenRunning(callable, *args, **kw): + """ + Call a function when the reactor is running. + + If the reactor has not started, the callable will be scheduled + to run when it does start. Otherwise, the callable will be invoked + immediately. + + @param callable: the callable object to call later. + + @param args: the arguments to call it with. + + @param kw: the keyword arguments to call it with. + + @return: None if the callable was invoked, otherwise a system + event id for the scheduled call. + """ + + + +class IReactorPluggableResolver(Interface): + """ + An L{IReactorPluggableResolver} is a reactor which can be customized with + an L{IResolverSimple}. This is a fairly limited interface, that supports + only IPv4; you should use L{IReactorPluggableNameResolver} instead. + + @see: L{IReactorPluggableNameResolver} + """ + + def installResolver(resolver): + """ + Set the internal resolver to use to for name lookups. + + @type resolver: An object implementing the L{IResolverSimple} interface + @param resolver: The new resolver to use. + + @return: The previously installed resolver. + @rtype: L{IResolverSimple} + """ + + + +class IReactorPluggableNameResolver(Interface): + """ + An L{IReactorPluggableNameResolver} is a reactor whose name resolver can be + set to a user-supplied object. + """ + + nameResolver = Attribute( + """ + Read-only attribute; the resolver installed with L{installResolver}. + An L{IHostnameResolver}. + """ + ) + + def installNameResolver(resolver): + """ + Set the internal resolver to use for name lookups. + + @type resolver: An object providing the L{IHostnameResolver} interface. + @param resolver: The new resolver to use. + + @return: The previously installed resolver. + @rtype: L{IHostnameResolver} + """ + + + +class IReactorDaemonize(Interface): + """ + A reactor which provides hooks that need to be called before and after + daemonization. + + Notes: + - This interface SHOULD NOT be called by applications. + - This interface should only be implemented by reactors as a workaround + (in particular, it's implemented currently only by kqueue()). + For details please see the comments on ticket #1918. + """ + + def beforeDaemonize(): + """ + Hook to be called immediately before daemonization. No reactor methods + may be called until L{afterDaemonize} is called. + + @return: L{None}. + """ + + + def afterDaemonize(): + """ + Hook to be called immediately after daemonization. This may only be + called after L{beforeDaemonize} had been called previously. + + @return: L{None}. + """ + + + +class IReactorFDSet(Interface): + """ + Implement me to be able to use L{IFileDescriptor} type resources. + + This assumes that your main-loop uses UNIX-style numeric file descriptors + (or at least similarly opaque IDs returned from a .fileno() method) + """ + + def addReader(reader): + """ + I add reader to the set of file descriptors to get read events for. + + @param reader: An L{IReadDescriptor} provider that will be checked for + read events until it is removed from the reactor with + L{removeReader}. + + @return: L{None}. + """ + + def addWriter(writer): + """ + I add writer to the set of file descriptors to get write events for. + + @param writer: An L{IWriteDescriptor} provider that will be checked for + write events until it is removed from the reactor with + L{removeWriter}. + + @return: L{None}. + """ + + def removeReader(reader): + """ + Removes an object previously added with L{addReader}. + + @return: L{None}. + """ + + def removeWriter(writer): + """ + Removes an object previously added with L{addWriter}. + + @return: L{None}. + """ + + def removeAll(): + """ + Remove all readers and writers. + + Should not remove reactor internal reactor connections (like a waker). + + @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers + which were removed. + """ + + def getReaders(): + """ + Return the list of file descriptors currently monitored for input + events by the reactor. + + @return: the list of file descriptors monitored for input events. + @rtype: C{list} of C{IReadDescriptor} + """ + + def getWriters(): + """ + Return the list file descriptors currently monitored for output events + by the reactor. + + @return: the list of file descriptors monitored for output events. + @rtype: C{list} of C{IWriteDescriptor} + """ + + +class IListeningPort(Interface): + """ + A listening port. + """ + + def startListening(): + """ + Start listening on this port. + + @raise CannotListenError: If it cannot listen on this port (e.g., it is + a TCP port and it cannot bind to the required + port number). + """ + + def stopListening(): + """ + Stop listening on this port. + + If it does not complete immediately, will return Deferred that fires + upon completion. + """ + + def getHost(): + """ + Get the host that this port is listening for. + + @return: An L{IAddress} provider. + """ + + +class ILoggingContext(Interface): + """ + Give context information that will be used to log events generated by + this item. + """ + + def logPrefix(): + """ + @return: Prefix used during log formatting to indicate context. + @rtype: C{str} + """ + + + +class IFileDescriptor(ILoggingContext): + """ + An interface representing a UNIX-style numeric file descriptor. + """ + + def fileno(): + """ + @raise: If the descriptor no longer has a valid file descriptor + number associated with it. + + @return: The platform-specified representation of a file descriptor + number. Or C{-1} if the descriptor no longer has a valid file + descriptor number associated with it. As long as the descriptor + is valid, calls to this method on a particular instance must + return the same value. + """ + + + def connectionLost(reason): + """ + Called when the connection was lost. + + This is called when the connection on a selectable object has been + lost. It will be called whether the connection was closed explicitly, + an exception occurred in an event handler, or the other end of the + connection closed it first. + + See also L{IHalfCloseableDescriptor} if your descriptor wants to be + notified separately of the two halves of the connection being closed. + + @param reason: A failure instance indicating the reason why the + connection was lost. L{error.ConnectionLost} and + L{error.ConnectionDone} are of special note, but the + failure may be of other classes as well. + """ + + + +class IReadDescriptor(IFileDescriptor): + """ + An L{IFileDescriptor} that can read. + + This interface is generally used in conjunction with L{IReactorFDSet}. + """ + + def doRead(): + """ + Some data is available for reading on your descriptor. + + @return: If an error is encountered which causes the descriptor to + no longer be valid, a L{Failure} should be returned. Otherwise, + L{None}. + """ + + +class IWriteDescriptor(IFileDescriptor): + """ + An L{IFileDescriptor} that can write. + + This interface is generally used in conjunction with L{IReactorFDSet}. + """ + + def doWrite(): + """ + Some data can be written to your descriptor. + + @return: If an error is encountered which causes the descriptor to + no longer be valid, a L{Failure} should be returned. Otherwise, + L{None}. + """ + + +class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor): + """ + An L{IFileDescriptor} that can both read and write. + """ + + +class IHalfCloseableDescriptor(Interface): + """ + A descriptor that can be half-closed. + """ + + def writeConnectionLost(reason): + """ + Indicates write connection was lost. + """ + + def readConnectionLost(reason): + """ + Indicates read connection was lost. + """ + + +class ISystemHandle(Interface): + """ + An object that wraps a networking OS-specific handle. + """ + + def getHandle(): + """ + Return a system- and reactor-specific handle. + + This might be a socket.socket() object, or some other type of + object, depending on which reactor is being used. Use and + manipulate at your own risk. + + This might be used in cases where you want to set specific + options not exposed by the Twisted APIs. + """ + + +class IConsumer(Interface): + """ + A consumer consumes data from a producer. + """ + + def registerProducer(producer, streaming): + """ + Register to receive data from a producer. + + This sets self to be a consumer for a producer. When this object runs + out of data (as when a send(2) call on a socket succeeds in moving the + last data from a userspace buffer into a kernelspace buffer), it will + ask the producer to resumeProducing(). + + For L{IPullProducer} providers, C{resumeProducing} will be called once + each time data is required. + + For L{IPushProducer} providers, C{pauseProducing} will be called + whenever the write buffer fills up and C{resumeProducing} will only be + called when it empties. The consumer will only call C{resumeProducing} + to balance a previous C{pauseProducing} call; the producer is assumed + to start in an un-paused state. + + @type producer: L{IProducer} provider + + @type streaming: C{bool} + @param streaming: C{True} if C{producer} provides L{IPushProducer}, + C{False} if C{producer} provides L{IPullProducer}. + + @raise RuntimeError: If a producer is already registered. + + @return: L{None} + """ + + + def unregisterProducer(): + """ + Stop consuming data from a producer, without disconnecting. + """ + + + def write(data): + """ + The producer will write data by calling this method. + + The implementation must be non-blocking and perform whatever + buffering is necessary. If the producer has provided enough data + for now and it is a L{IPushProducer}, the consumer may call its + C{pauseProducing} method. + """ + + + +class IProducer(Interface): + """ + A producer produces data for a consumer. + + Typically producing is done by calling the C{write} method of a class + implementing L{IConsumer}. + """ + + def stopProducing(): + """ + Stop producing data. + + This tells a producer that its consumer has died, so it must stop + producing data for good. + """ + + +class IPushProducer(IProducer): + """ + A push producer, also known as a streaming producer is expected to + produce (write to this consumer) data on a continuous basis, unless + it has been paused. A paused push producer will resume producing + after its C{resumeProducing()} method is called. For a push producer + which is not pauseable, these functions may be noops. + """ + + def pauseProducing(): + """ + Pause producing data. + + Tells a producer that it has produced too much data to process for + the time being, and to stop until C{resumeProducing()} is called. + """ + def resumeProducing(): + """ + Resume producing data. + + This tells a producer to re-add itself to the main loop and produce + more data for its consumer. + """ + + + +class IPullProducer(IProducer): + """ + A pull producer, also known as a non-streaming producer, is + expected to produce data each time L{resumeProducing()} is called. + """ + + def resumeProducing(): + """ + Produce data for the consumer a single time. + + This tells a producer to produce data for the consumer once + (not repeatedly, once only). Typically this will be done + by calling the consumer's C{write} method a single time with + produced data. The producer should produce data before returning + from C{resumeProducing()}, that is, it should not schedule a deferred + write. + """ + + + +class IProtocol(Interface): + + def dataReceived(data): + """ + Called whenever data is received. + + Use this method to translate to a higher-level message. Usually, some + callback will be made upon the receipt of each complete protocol + message. + + Please keep in mind that you will probably need to buffer some data + as partial (or multiple) protocol messages may be received! We + recommend that unit tests for protocols call through to this method + with differing chunk sizes, down to one byte at a time. + + @param data: bytes of indeterminate length + @type data: L{bytes} + """ + + def connectionLost(reason): + """ + Called when the connection is shut down. + + Clear any circular references here, and any external references + to this Protocol. The connection has been closed. The C{reason} + Failure wraps a L{twisted.internet.error.ConnectionDone} or + L{twisted.internet.error.ConnectionLost} instance (or a subclass + of one of those). + + @type reason: L{twisted.python.failure.Failure} + """ + + def makeConnection(transport): + """ + Make a connection to a transport and a server. + """ + + def connectionMade(): + """ + Called when a connection is made. + + This may be considered the initializer of the protocol, because + it is called when the connection is completed. For clients, + this is called once the connection to the server has been + established; for servers, this is called after an accept() call + stops blocking and a socket has been received. If you need to + send any greeting or initial message, do it here. + """ + + +class IProcessProtocol(Interface): + """ + Interface for process-related event handlers. + """ + + def makeConnection(process): + """ + Called when the process has been created. + + @type process: L{IProcessTransport} provider + @param process: An object representing the process which has been + created and associated with this protocol. + """ + + + def childDataReceived(childFD, data): + """ + Called when data arrives from the child process. + + @type childFD: L{int} + @param childFD: The file descriptor from which the data was + received. + + @type data: L{bytes} + @param data: The data read from the child's file descriptor. + """ + + + def childConnectionLost(childFD): + """ + Called when a file descriptor associated with the child process is + closed. + + @type childFD: C{int} + @param childFD: The file descriptor which was closed. + """ + + + def processExited(reason): + """ + Called when the child process exits. + + @type reason: L{twisted.python.failure.Failure} + @param reason: A failure giving the reason the child process + terminated. The type of exception for this failure is either + L{twisted.internet.error.ProcessDone} or + L{twisted.internet.error.ProcessTerminated}. + + @since: 8.2 + """ + + + def processEnded(reason): + """ + Called when the child process exits and all file descriptors associated + with it have been closed. + + @type reason: L{twisted.python.failure.Failure} + @param reason: A failure giving the reason the child process + terminated. The type of exception for this failure is either + L{twisted.internet.error.ProcessDone} or + L{twisted.internet.error.ProcessTerminated}. + """ + + + +class IHalfCloseableProtocol(Interface): + """ + Implemented to indicate they want notification of half-closes. + + TCP supports the notion of half-closing the connection, e.g. + closing the write side but still not stopping reading. A protocol + that implements this interface will be notified of such events, + instead of having connectionLost called. + """ + + def readConnectionLost(): + """ + Notification of the read connection being closed. + + This indicates peer did half-close of write side. It is now + the responsibility of the this protocol to call + loseConnection(). In addition, the protocol MUST make sure a + reference to it still exists (i.e. by doing a callLater with + one of its methods, etc.) as the reactor will only have a + reference to it if it is writing. + + If the protocol does not do so, it might get garbage collected + without the connectionLost method ever being called. + """ + + def writeConnectionLost(): + """ + Notification of the write connection being closed. + + This will never be called for TCP connections as TCP does not + support notification of this type of half-close. + """ + + + +class IHandshakeListener(Interface): + """ + An interface implemented by a L{IProtocol} to indicate that it would like + to be notified when TLS handshakes complete when run over a TLS-based + transport. + + This interface is only guaranteed to be called when run over a TLS-based + transport: non TLS-based transports will not respect this interface. + """ + + def handshakeCompleted(): + """ + Notification of the TLS handshake being completed. + + This notification fires when OpenSSL has completed the TLS handshake. + At this point the TLS connection is established, and the protocol can + interrogate its transport (usually an L{ISSLTransport}) for details of + the TLS connection. + + This notification *also* fires whenever the TLS session is + renegotiated. As a result, protocols that have certain minimum security + requirements should implement this interface to ensure that they are + able to re-evaluate the security of the TLS session if it changes. + """ + + + +class IFileDescriptorReceiver(Interface): + """ + Protocols may implement L{IFileDescriptorReceiver} to receive file + descriptors sent to them. This is useful in conjunction with + L{IUNIXTransport}, which allows file descriptors to be sent between + processes on a single host. + """ + def fileDescriptorReceived(descriptor): + """ + Called when a file descriptor is received over the connection. + + @param descriptor: The descriptor which was received. + @type descriptor: C{int} + + @return: L{None} + """ + + + +class IProtocolFactory(Interface): + """ + Interface for protocol factories. + """ + + def buildProtocol(addr): + """ + Called when a connection has been established to addr. + + If None is returned, the connection is assumed to have been refused, + and the Port will close the connection. + + @type addr: (host, port) + @param addr: The address of the newly-established connection + + @return: None if the connection was refused, otherwise an object + providing L{IProtocol}. + """ + + def doStart(): + """ + Called every time this is connected to a Port or Connector. + """ + + def doStop(): + """ + Called every time this is unconnected from a Port or Connector. + """ + + +class ITransport(Interface): + """ + I am a transport for bytes. + + I represent (and wrap) the physical connection and synchronicity + of the framework which is talking to the network. I make no + representations about whether calls to me will happen immediately + or require returning to a control loop, or whether they will happen + in the same or another thread. Consider methods of this class + (aside from getPeer) to be 'thrown over the wall', to happen at some + indeterminate time. + """ + + def write(data): + """ + Write some data to the physical connection, in sequence, in a + non-blocking fashion. + + If possible, make sure that it is all written. No data will + ever be lost, although (obviously) the connection may be closed + before it all gets through. + + @type data: L{bytes} + @param data: The data to write. + """ + + def writeSequence(data): + """ + Write an iterable of byte strings to the physical connection. + + If possible, make sure that all of the data is written to + the socket at once, without first copying it all into a + single byte string. + + @type data: an iterable of L{bytes} + @param data: The data to write. + """ + + def loseConnection(): + """ + Close my connection, after writing all pending data. + + Note that if there is a registered producer on a transport it + will not be closed until the producer has been unregistered. + """ + + def getPeer(): + """ + Get the remote address of this connection. + + Treat this method with caution. It is the unfortunate result of the + CGI and Jabber standards, but should not be considered reliable for + the usual host of reasons; port forwarding, proxying, firewalls, IP + masquerading, etc. + + @return: An L{IAddress} provider. + """ + + def getHost(): + """ + Similar to getPeer, but returns an address describing this side of the + connection. + + @return: An L{IAddress} provider. + """ + + +class ITCPTransport(ITransport): + """ + A TCP based transport. + """ + + def loseWriteConnection(): + """ + Half-close the write side of a TCP connection. + + If the protocol instance this is attached to provides + IHalfCloseableProtocol, it will get notified when the operation is + done. When closing write connection, as with loseConnection this will + only happen when buffer has emptied and there is no registered + producer. + """ + + + def abortConnection(): + """ + Close the connection abruptly. + + Discards any buffered data, stops any registered producer, + and, if possible, notifies the other end of the unclean + closure. + + @since: 11.1 + """ + + + def getTcpNoDelay(): + """ + Return if C{TCP_NODELAY} is enabled. + """ + + def setTcpNoDelay(enabled): + """ + Enable/disable C{TCP_NODELAY}. + + Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are + sent sooner, possibly at the expense of overall throughput. + """ + + def getTcpKeepAlive(): + """ + Return if C{SO_KEEPALIVE} is enabled. + """ + + def setTcpKeepAlive(enabled): + """ + Enable/disable C{SO_KEEPALIVE}. + + Enabling C{SO_KEEPALIVE} sends packets periodically when the connection + is otherwise idle, usually once every two hours. They are intended + to allow detection of lost peers in a non-infinite amount of time. + """ + + def getHost(): + """ + Returns L{IPv4Address} or L{IPv6Address}. + """ + + def getPeer(): + """ + Returns L{IPv4Address} or L{IPv6Address}. + """ + + + +class IUNIXTransport(ITransport): + """ + Transport for stream-oriented unix domain connections. + """ + def sendFileDescriptor(descriptor): + """ + Send a duplicate of this (file, socket, pipe, etc) descriptor to the + other end of this connection. + + The send is non-blocking and will be queued if it cannot be performed + immediately. The send will be processed in order with respect to other + C{sendFileDescriptor} calls on this transport, but not necessarily with + respect to C{write} calls on this transport. The send can only be + processed if there are also bytes in the normal connection-oriented send + buffer (ie, you must call C{write} at least as many times as you call + C{sendFileDescriptor}). + + @param descriptor: An C{int} giving a valid file descriptor in this + process. Note that a I{file descriptor} may actually refer to a + socket, a pipe, or anything else POSIX tries to treat in the same + way as a file. + + @return: L{None} + """ + + + +class IOpenSSLServerConnectionCreator(Interface): + """ + A provider of L{IOpenSSLServerConnectionCreator} can create + L{OpenSSL.SSL.Connection} objects for TLS servers. + + @see: L{twisted.internet.ssl} + + @note: Creating OpenSSL connection objects is subtle, error-prone, and + security-critical. Before implementing this interface yourself, + consider using L{twisted.internet.ssl.CertificateOptions} as your + C{contextFactory}. (For historical reasons, that class does not + actually I{implement} this interface; nevertheless it is usable in all + Twisted APIs which require a provider of this interface.) + """ + + def serverConnectionForTLS(tlsProtocol): + """ + Create a connection for the given server protocol. + + @param tlsProtocol: the protocol server making the request. + @type tlsProtocol: L{twisted.protocols.tls.TLSMemoryBIOProtocol}. + + @return: an OpenSSL connection object configured appropriately for the + given Twisted protocol. + @rtype: L{OpenSSL.SSL.Connection} + """ + + + +class IOpenSSLClientConnectionCreator(Interface): + """ + A provider of L{IOpenSSLClientConnectionCreator} can create + L{OpenSSL.SSL.Connection} objects for TLS clients. + + @see: L{twisted.internet.ssl} + + @note: Creating OpenSSL connection objects is subtle, error-prone, and + security-critical. Before implementing this interface yourself, + consider using L{twisted.internet.ssl.optionsForClientTLS} as your + C{contextFactory}. + """ + + def clientConnectionForTLS(tlsProtocol): + """ + Create a connection for the given client protocol. + + @param tlsProtocol: the client protocol making the request. + @type tlsProtocol: L{twisted.protocols.tls.TLSMemoryBIOProtocol}. + + @return: an OpenSSL connection object configured appropriately for the + given Twisted protocol. + @rtype: L{OpenSSL.SSL.Connection} + """ + + + +class IProtocolNegotiationFactory(Interface): + """ + A provider of L{IProtocolNegotiationFactory} can provide information about + the various protocols that the factory can create implementations of. This + can be used, for example, to provide protocol names for Next Protocol + Negotiation and Application Layer Protocol Negotiation. + + @see: L{twisted.internet.ssl} + """ + + def acceptableProtocols(): + """ + Returns a list of protocols that can be spoken by the connection + factory in the form of ALPN tokens, as laid out in the IANA registry + for ALPN tokens. + + @return: a list of ALPN tokens in order of preference. + @rtype: L{list} of L{bytes} + """ + + + +class IOpenSSLContextFactory(Interface): + """ + A provider of L{IOpenSSLContextFactory} is capable of generating + L{OpenSSL.SSL.Context} classes suitable for configuring TLS on a + connection. A provider will store enough state to be able to generate these + contexts as needed for individual connections. + + @see: L{twisted.internet.ssl} + """ + + def getContext(): + """ + Returns a TLS context object, suitable for securing a TLS connection. + This context object will be appropriately customized for the connection + based on the state in this object. + + @return: A TLS context object. + @rtype: L{OpenSSL.SSL.Context} + """ + + + +class ITLSTransport(ITCPTransport): + """ + A TCP transport that supports switching to TLS midstream. + + Once TLS mode is started the transport will implement L{ISSLTransport}. + """ + + def startTLS(contextFactory): + """ + Initiate TLS negotiation. + + @param contextFactory: An object which creates appropriately configured + TLS connections. + + For clients, use L{twisted.internet.ssl.optionsForClientTLS}; for + servers, use L{twisted.internet.ssl.CertificateOptions}. + + @type contextFactory: L{IOpenSSLClientConnectionCreator} or + L{IOpenSSLServerConnectionCreator}, depending on whether this + L{ITLSTransport} is a server or not. If the appropriate interface + is not provided by the value given for C{contextFactory}, it must + be an implementor of L{IOpenSSLContextFactory}. + """ + + + +class ISSLTransport(ITCPTransport): + """ + A SSL/TLS based transport. + """ + + def getPeerCertificate(): + """ + Return an object with the peer's certificate info. + """ + + + +class INegotiated(ISSLTransport): + """ + A TLS based transport that supports using ALPN/NPN to negotiate the + protocol to be used inside the encrypted tunnel. + """ + negotiatedProtocol = Attribute( + """ + The protocol selected to be spoken using ALPN/NPN. The result from ALPN + is preferred to the result from NPN if both were used. If the remote + peer does not support ALPN or NPN, or neither NPN or ALPN are available + on this machine, will be L{None}. Otherwise, will be the name of the + selected protocol as C{bytes}. Note that until the handshake has + completed this property may incorrectly return L{None}: wait until data + has been received before trusting it (see + https://twistedmatrix.com/trac/ticket/6024). + """ + ) + + + +class ICipher(Interface): + """ + A TLS cipher. + """ + fullName = Attribute( + "The fully qualified name of the cipher in L{unicode}." + ) + + + +class IAcceptableCiphers(Interface): + """ + A list of acceptable ciphers for a TLS context. + """ + def selectCiphers(availableCiphers): + """ + Choose which ciphers to allow to be negotiated on a TLS connection. + + @param availableCiphers: A L{list} of L{ICipher} which gives the names + of all ciphers supported by the TLS implementation in use. + + @return: A L{list} of L{ICipher} which represents the ciphers + which may be negotiated on the TLS connection. The result is + ordered by preference with more preferred ciphers appearing + earlier. + """ + + + +class IProcessTransport(ITransport): + """ + A process transport. + """ + + pid = Attribute( + "From before L{IProcessProtocol.makeConnection} is called to before " + "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} " + "giving the platform process ID of this process. C{pid} is L{None} " + "at all other times.") + + def closeStdin(): + """ + Close stdin after all data has been written out. + """ + + def closeStdout(): + """ + Close stdout. + """ + + def closeStderr(): + """ + Close stderr. + """ + + def closeChildFD(descriptor): + """ + Close a file descriptor which is connected to the child process, identified + by its FD in the child process. + """ + + def writeToChild(childFD, data): + """ + Similar to L{ITransport.write} but also allows the file descriptor in + the child process which will receive the bytes to be specified. + + @type childFD: L{int} + @param childFD: The file descriptor to which to write. + + @type data: L{bytes} + @param data: The bytes to write. + + @return: L{None} + + @raise KeyError: If C{childFD} is not a file descriptor that was mapped + in the child when L{IReactorProcess.spawnProcess} was used to create + it. + """ + + def loseConnection(): + """ + Close stdin, stderr and stdout. + """ + + def signalProcess(signalID): + """ + Send a signal to the process. + + @param signalID: can be + - one of C{"KILL"}, C{"TERM"}, or C{"INT"}. + These will be implemented in a + cross-platform manner, and so should be used + if possible. + - an integer, where it represents a POSIX + signal ID. + + @raise twisted.internet.error.ProcessExitedAlready: If the process has + already exited. + @raise OSError: If the C{os.kill} call fails with an errno different + from C{ESRCH}. + """ + + +class IServiceCollection(Interface): + """ + An object which provides access to a collection of services. + """ + + def getServiceNamed(serviceName): + """ + Retrieve the named service from this application. + + Raise a C{KeyError} if there is no such service name. + """ + + def addService(service): + """ + Add a service to this collection. + """ + + def removeService(service): + """ + Remove a service from this collection. + """ + + +class IUDPTransport(Interface): + """ + Transport for UDP DatagramProtocols. + """ + + def write(packet, addr=None): + """ + Write packet to given address. + + @param addr: a tuple of (ip, port). For connected transports must + be the address the transport is connected to, or None. + In non-connected mode this is mandatory. + + @raise twisted.internet.error.MessageLengthError: C{packet} was too + long. + """ + + def connect(host, port): + """ + Connect the transport to an address. + + This changes it to connected mode. Datagrams can only be sent to + this address, and will only be received from this address. In addition + the protocol's connectionRefused method might get called if destination + is not receiving datagrams. + + @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost') + @param port: port to connect to. + """ + + def getHost(): + """ + Get this port's host address. + + @return: an address describing the listening port. + @rtype: L{IPv4Address} or L{IPv6Address}. + """ + + def stopListening(): + """ + Stop listening on this port. + + If it does not complete immediately, will return L{Deferred} that fires + upon completion. + """ + + def setBroadcastAllowed(enabled): + """ + Set whether this port may broadcast. + + @param enabled: Whether the port may broadcast. + @type enabled: L{bool} + """ + + def getBroadcastAllowed(): + """ + Checks if broadcast is currently allowed on this port. + + @return: Whether this port may broadcast. + @rtype: L{bool} + """ + + +class IUNIXDatagramTransport(Interface): + """ + Transport for UDP PacketProtocols. + """ + + def write(packet, address): + """ + Write packet to given address. + """ + + def getHost(): + """ + Returns L{UNIXAddress}. + """ + + +class IUNIXDatagramConnectedTransport(Interface): + """ + Transport for UDP ConnectedPacketProtocols. + """ + + def write(packet): + """ + Write packet to address we are connected to. + """ + + def getHost(): + """ + Returns L{UNIXAddress}. + """ + + def getPeer(): + """ + Returns L{UNIXAddress}. + """ + + +class IMulticastTransport(Interface): + """ + Additional functionality for multicast UDP. + """ + + def getOutgoingInterface(): + """ + Return interface of outgoing multicast packets. + """ + + def setOutgoingInterface(addr): + """ + Set interface for outgoing multicast packets. + + Returns Deferred of success. + """ + + def getLoopbackMode(): + """ + Return if loopback mode is enabled. + """ + + def setLoopbackMode(mode): + """ + Set if loopback mode is enabled. + """ + + def getTTL(): + """ + Get time to live for multicast packets. + """ + + def setTTL(ttl): + """ + Set time to live on multicast packets. + """ + + def joinGroup(addr, interface=""): + """ + Join a multicast group. Returns L{Deferred} of success or failure. + + If an error occurs, the returned L{Deferred} will fail with + L{error.MulticastJoinError}. + """ + + def leaveGroup(addr, interface=""): + """ + Leave multicast group, return L{Deferred} of success. + """ + + +class IStreamClientEndpoint(Interface): + """ + A stream client endpoint is a place that L{ClientFactory} can connect to. + For example, a remote TCP host/port pair would be a TCP client endpoint. + + @since: 10.1 + """ + + def connect(protocolFactory): + """ + Connect the C{protocolFactory} to the location specified by this + L{IStreamClientEndpoint} provider. + + @param protocolFactory: A provider of L{IProtocolFactory} + @return: A L{Deferred} that results in an L{IProtocol} upon successful + connection otherwise a L{Failure} wrapping L{ConnectError} or + L{NoProtocol }. + """ + + + +class IStreamServerEndpoint(Interface): + """ + A stream server endpoint is a place that a L{Factory} can listen for + incoming connections. + + @since: 10.1 + """ + + def listen(protocolFactory): + """ + Listen with C{protocolFactory} at the location specified by this + L{IStreamServerEndpoint} provider. + + @param protocolFactory: A provider of L{IProtocolFactory} + @return: A L{Deferred} that results in an L{IListeningPort} or an + L{CannotListenError} + """ + + + +class IStreamServerEndpointStringParser(Interface): + """ + An L{IStreamServerEndpointStringParser} is like an + L{IStreamClientEndpointStringParserWithReactor}, except for + L{IStreamServerEndpoint}s instead of clients. It integrates with + L{endpoints.serverFromString} in much the same way. + """ + + prefix = Attribute( + """ + A C{str}, the description prefix to respond to. For example, an + L{IStreamServerEndpointStringParser} plugin which had C{"foo"} for its + C{prefix} attribute would be called for endpoint descriptions like + C{"foo:bar:baz"} or C{"foo:"}. + """ + ) + + + def parseStreamServer(reactor, *args, **kwargs): + """ + Parse a stream server endpoint from a reactor and string-only arguments + and keyword arguments. + + @see: L{IStreamClientEndpointStringParserWithReactor.parseStreamClient} + + @return: a stream server endpoint + @rtype: L{IStreamServerEndpoint} + """ + + +class IStreamClientEndpointStringParserWithReactor(Interface): + """ + An L{IStreamClientEndpointStringParserWithReactor} is a parser which can + convert a set of string C{*args} and C{**kwargs} into an + L{IStreamClientEndpoint} provider. + + This interface is really only useful in the context of the plugin system + for L{endpoints.clientFromString}. See the document entitled "I{The + Twisted Plugin System}" for more details on how to write a plugin. + + If you place an L{IStreamClientEndpointStringParserWithReactor} plugin in + the C{twisted.plugins} package, that plugin's C{parseStreamClient} method + will be used to produce endpoints for any description string that begins + with the result of that L{IStreamClientEndpointStringParserWithReactor}'s + prefix attribute. + """ + + prefix = Attribute( + """ + L{bytes}, the description prefix to respond to. For example, an + L{IStreamClientEndpointStringParserWithReactor} plugin which had + C{b"foo"} for its C{prefix} attribute would be called for endpoint + descriptions like C{b"foo:bar:baz"} or C{b"foo:"}. + """ + ) + + + def parseStreamClient(reactor, *args, **kwargs): + """ + This method is invoked by L{endpoints.clientFromString}, if the type of + endpoint matches the return value from this + L{IStreamClientEndpointStringParserWithReactor}'s C{prefix} method. + + @param reactor: The reactor passed to L{endpoints.clientFromString}. + + @param args: The byte string arguments, minus the endpoint type, in the + endpoint description string, parsed according to the rules + described in L{endpoints.quoteStringArgument}. For example, if the + description were C{b"my-type:foo:bar:baz=qux"}, C{args} would be + C{(b'foo', b'bar')} + + @param kwargs: The byte string arguments from the endpoint description + passed as keyword arguments. For example, if the description were + C{b"my-type:foo:bar:baz=qux"}, C{kwargs} would be + C{dict(baz=b'qux')}. + + @return: a client endpoint + @rtype: a provider of L{IStreamClientEndpoint} + """ + + + +class _ISupportsExitSignalCapturing(Interface): + """ + An implementor of L{_ISupportsExitSignalCapturing} will capture the + value of any delivered exit signal (SIGINT, SIGTERM, SIGBREAK) for which + it has installed a handler. The caught signal number is made available in + the _exitSignal attribute. + """ + + _exitSignal = Attribute( + """ + C{int} or C{None}, the integer exit signal delivered to the + application, or None if no signal was delivered. + """ + ) diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/__init__.py b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/__init__.py new file mode 100644 index 00000000000..c403e517643 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +I/O Completion Ports reactor +""" + +from twisted.internet.iocpreactor.reactor import install + +__all__ = ['install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/abstract.py b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/abstract.py new file mode 100644 index 00000000000..f3c8e6d37e9 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/abstract.py @@ -0,0 +1,399 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Abstract file handle class +""" + +from twisted.internet import main, error, interfaces +from twisted.internet.abstract import _ConsumerMixin, _LogOwner +from twisted.python import failure +from twisted.python.compat import unicode + +from zope.interface import implementer +import errno + +from twisted.internet.iocpreactor.const import ERROR_HANDLE_EOF +from twisted.internet.iocpreactor.const import ERROR_IO_PENDING +from twisted.internet.iocpreactor import iocpsupport as _iocp + + +@implementer(interfaces.IPushProducer, interfaces.IConsumer, + interfaces.ITransport, interfaces.IHalfCloseableDescriptor) +class FileHandle(_ConsumerMixin, _LogOwner): + """ + File handle that can read and write asynchronously + """ + # read stuff + maxReadBuffers = 16 + readBufferSize = 4096 + reading = False + dynamicReadBuffers = True # set this to false if subclass doesn't do iovecs + _readNextBuffer = 0 + _readSize = 0 # how much data we have in the read buffer + _readScheduled = None + _readScheduledInOS = False + + + def startReading(self): + self.reactor.addActiveHandle(self) + if not self._readScheduled and not self.reading: + self.reading = True + self._readScheduled = self.reactor.callLater(0, + self._resumeReading) + + + def stopReading(self): + if self._readScheduled: + self._readScheduled.cancel() + self._readScheduled = None + self.reading = False + + + def _resumeReading(self): + self._readScheduled = None + if self._dispatchData() and not self._readScheduledInOS: + self.doRead() + + + def _dispatchData(self): + """ + Dispatch previously read data. Return True if self.reading and we don't + have any more data + """ + if not self._readSize: + return self.reading + size = self._readSize + full_buffers = size // self.readBufferSize + while self._readNextBuffer < full_buffers: + self.dataReceived(self._readBuffers[self._readNextBuffer]) + self._readNextBuffer += 1 + if not self.reading: + return False + remainder = size % self.readBufferSize + if remainder: + self.dataReceived(self._readBuffers[full_buffers][0:remainder]) + if self.dynamicReadBuffers: + total_buffer_size = self.readBufferSize * len(self._readBuffers) + # we have one buffer too many + if size < total_buffer_size - self.readBufferSize: + del self._readBuffers[-1] + # we filled all buffers, so allocate one more + elif (size == total_buffer_size and + len(self._readBuffers) < self.maxReadBuffers): + self._readBuffers.append(bytearray(self.readBufferSize)) + self._readNextBuffer = 0 + self._readSize = 0 + return self.reading + + + def _cbRead(self, rc, data, evt): + self._readScheduledInOS = False + if self._handleRead(rc, data, evt): + self.doRead() + + + def _handleRead(self, rc, data, evt): + """ + Returns False if we should stop reading for now + """ + if self.disconnected: + return False + # graceful disconnection + if (not (rc or data)) or rc in (errno.WSAEDISCON, ERROR_HANDLE_EOF): + self.reactor.removeActiveHandle(self) + self.readConnectionLost(failure.Failure(main.CONNECTION_DONE)) + return False + # XXX: not handling WSAEWOULDBLOCK + # ("too many outstanding overlapped I/O requests") + elif rc: + self.connectionLost(failure.Failure( + error.ConnectionLost("read error -- %s (%s)" % + (errno.errorcode.get(rc, 'unknown'), rc)))) + return False + else: + assert self._readSize == 0 + assert self._readNextBuffer == 0 + self._readSize = data + return self._dispatchData() + + + def doRead(self): + evt = _iocp.Event(self._cbRead, self) + + evt.buff = buff = self._readBuffers + rc, numBytesRead = self.readFromHandle(buff, evt) + + if not rc or rc == ERROR_IO_PENDING: + self._readScheduledInOS = True + else: + self._handleRead(rc, numBytesRead, evt) + + + def readFromHandle(self, bufflist, evt): + raise NotImplementedError() # TODO: this should default to ReadFile + + + def dataReceived(self, data): + raise NotImplementedError + + + def readConnectionLost(self, reason): + self.connectionLost(reason) + + + # write stuff + dataBuffer = b'' + offset = 0 + writing = False + _writeScheduled = None + _writeDisconnecting = False + _writeDisconnected = False + writeBufferSize = 2**2**2**2 + + + def loseWriteConnection(self): + self._writeDisconnecting = True + self.startWriting() + + + def _closeWriteConnection(self): + # override in subclasses + pass + + + def writeConnectionLost(self, reason): + # in current code should never be called + self.connectionLost(reason) + + + def startWriting(self): + self.reactor.addActiveHandle(self) + + if not self._writeScheduled and not self.writing: + self.writing = True + self._writeScheduled = self.reactor.callLater(0, + self._resumeWriting) + + + def stopWriting(self): + if self._writeScheduled: + self._writeScheduled.cancel() + self._writeScheduled = None + self.writing = False + + + def _resumeWriting(self): + self._writeScheduled = None + self.doWrite() + + + def _cbWrite(self, rc, numBytesWritten, evt): + if self._handleWrite(rc, numBytesWritten, evt): + self.doWrite() + + + def _handleWrite(self, rc, numBytesWritten, evt): + """ + Returns false if we should stop writing for now + """ + if self.disconnected or self._writeDisconnected: + return False + # XXX: not handling WSAEWOULDBLOCK + # ("too many outstanding overlapped I/O requests") + if rc: + self.connectionLost(failure.Failure( + error.ConnectionLost("write error -- %s (%s)" % + (errno.errorcode.get(rc, 'unknown'), rc)))) + return False + else: + self.offset += numBytesWritten + # If there is nothing left to send, + if self.offset == len(self.dataBuffer) and not self._tempDataLen: + self.dataBuffer = b"" + self.offset = 0 + # stop writing + self.stopWriting() + # If I've got a producer who is supposed to supply me with data + if self.producer is not None and ((not self.streamingProducer) + or self.producerPaused): + # tell them to supply some more. + self.producerPaused = True + self.producer.resumeProducing() + elif self.disconnecting: + # But if I was previously asked to let the connection die, + # do so. + self.connectionLost(failure.Failure(main.CONNECTION_DONE)) + elif self._writeDisconnecting: + # I was previously asked to half-close the connection. + self._writeDisconnected = True + self._closeWriteConnection() + return False + else: + return True + + + def doWrite(self): + if len(self.dataBuffer) - self.offset < self.SEND_LIMIT: + # If there is currently less than SEND_LIMIT bytes left to send + # in the string, extend it with the array data. + self.dataBuffer = (self.dataBuffer[self.offset:] + + b"".join(self._tempDataBuffer)) + self.offset = 0 + self._tempDataBuffer = [] + self._tempDataLen = 0 + + evt = _iocp.Event(self._cbWrite, self) + + # Send as much data as you can. + if self.offset: + sendView = memoryview(self.dataBuffer) + evt.buff = buff = sendView[self.offset:] + else: + evt.buff = buff = self.dataBuffer + rc, data = self.writeToHandle(buff, evt) + if rc and rc != ERROR_IO_PENDING: + self._handleWrite(rc, data, evt) + + + def writeToHandle(self, buff, evt): + raise NotImplementedError() # TODO: this should default to WriteFile + + + def write(self, data): + """Reliably write some data. + + The data is buffered until his file descriptor is ready for writing. + """ + if isinstance(data, unicode): # no, really, I mean it + raise TypeError("Data must not be unicode") + if not self.connected or self._writeDisconnected: + return + if data: + self._tempDataBuffer.append(data) + self._tempDataLen += len(data) + if self.producer is not None and self.streamingProducer: + if (len(self.dataBuffer) + self._tempDataLen + > self.writeBufferSize): + self.producerPaused = True + self.producer.pauseProducing() + self.startWriting() + + + def writeSequence(self, iovec): + for i in iovec: + if isinstance(i, unicode): # no, really, I mean it + raise TypeError("Data must not be unicode") + if not self.connected or not iovec or self._writeDisconnected: + return + self._tempDataBuffer.extend(iovec) + for i in iovec: + self._tempDataLen += len(i) + if self.producer is not None and self.streamingProducer: + if len(self.dataBuffer) + self._tempDataLen > self.writeBufferSize: + self.producerPaused = True + self.producer.pauseProducing() + self.startWriting() + + + # general stuff + connected = False + disconnected = False + disconnecting = False + logstr = "Uninitialized" + + SEND_LIMIT = 128*1024 + + + def __init__(self, reactor = None): + if not reactor: + from twisted.internet import reactor + self.reactor = reactor + self._tempDataBuffer = [] # will be added to dataBuffer in doWrite + self._tempDataLen = 0 + self._readBuffers = [bytearray(self.readBufferSize)] + + + def connectionLost(self, reason): + """ + The connection was lost. + + This is called when the connection on a selectable object has been + lost. It will be called whether the connection was closed explicitly, + an exception occurred in an event handler, or the other end of the + connection closed it first. + + Clean up state here, but make sure to call back up to FileDescriptor. + """ + + self.disconnected = True + self.connected = False + if self.producer is not None: + self.producer.stopProducing() + self.producer = None + self.stopReading() + self.stopWriting() + self.reactor.removeActiveHandle(self) + + + def getFileHandle(self): + return -1 + + + def loseConnection(self, _connDone=failure.Failure(main.CONNECTION_DONE)): + """ + Close the connection at the next available opportunity. + + Call this to cause this FileDescriptor to lose its connection. It will + first write any data that it has buffered. + + If there is data buffered yet to be written, this method will cause the + transport to lose its connection as soon as it's done flushing its + write buffer. If you have a producer registered, the connection won't + be closed until the producer is finished. Therefore, make sure you + unregister your producer when it's finished, or the connection will + never close. + """ + + if self.connected and not self.disconnecting: + if self._writeDisconnected: + # doWrite won't trigger the connection close anymore + self.stopReading() + self.stopWriting + self.connectionLost(_connDone) + else: + self.stopReading() + self.startWriting() + self.disconnecting = 1 + + + # Producer/consumer implementation + + def stopConsuming(self): + """ + Stop consuming data. + + This is called when a producer has lost its connection, to tell the + consumer to go lose its connection (and break potential circular + references). + """ + self.unregisterProducer() + self.loseConnection() + + + # producer interface implementation + + def resumeProducing(self): + if self.connected and not self.disconnecting: + self.startReading() + + + def pauseProducing(self): + self.stopReading() + + + def stopProducing(self): + self.loseConnection() + + +__all__ = ['FileHandle'] diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/const.py b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/const.py new file mode 100644 index 00000000000..dbeb094b293 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/const.py @@ -0,0 +1,26 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Windows constants for IOCP +""" + + +# this stuff should really be gotten from Windows headers via pyrex, but it +# probably is not going to change + +ERROR_PORT_UNREACHABLE = 1234 +ERROR_NETWORK_UNREACHABLE = 1231 +ERROR_CONNECTION_REFUSED = 1225 +ERROR_IO_PENDING = 997 +ERROR_OPERATION_ABORTED = 995 +WAIT_TIMEOUT = 258 +ERROR_NETNAME_DELETED = 64 +ERROR_HANDLE_EOF = 38 + +INFINITE = -1 + +SO_UPDATE_CONNECT_CONTEXT = 0x7010 +SO_UPDATE_ACCEPT_CONTEXT = 0x700B + diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/interfaces.py b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/interfaces.py new file mode 100644 index 00000000000..9e4d3ca4166 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/interfaces.py @@ -0,0 +1,47 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Interfaces for iocpreactor +""" + + +from zope.interface import Interface + + + +class IReadHandle(Interface): + def readFromHandle(bufflist, evt): + """ + Read into the given buffers from this handle. + + @param buff: the buffers to read into + @type buff: list of objects implementing the read/write buffer protocol + + @param evt: an IOCP Event object + + @return: tuple (return code, number of bytes read) + """ + + + +class IWriteHandle(Interface): + def writeToHandle(buff, evt): + """ + Write the given buffer to this handle. + + @param buff: the buffer to write + @type buff: any object implementing the buffer protocol + + @param evt: an IOCP Event object + + @return: tuple (return code, number of bytes written) + """ + + + +class IReadWriteHandle(IReadHandle, IWriteHandle): + pass + + diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/iocpsupport.c b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/iocpsupport.c new file mode 100644 index 00000000000..2c0eaa9536f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/iocpsupport.c @@ -0,0 +1,11990 @@ +/* Generated by Cython 0.28.5 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000) + #error Cython requires Python 2.6+ or Python 3.3+. +#else +#define CYTHON_ABI "0_28_5" +#define CYTHON_FUTURE_DIVISION 0 +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #if PY_VERSION_HEX >= 0x02070000 + #define HAVE_LONG_LONG + #endif +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#ifdef PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 +#elif defined(PYSTON_VERSION) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #elif !defined(CYTHON_USE_PYTYPE_LOOKUP) + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #if PY_MAJOR_VERSION < 3 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #ifndef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if PY_VERSION_HEX < 0x030300F0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT (0 && PY_VERSION_HEX >= 0x03050000) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PY_VERSION_HEX >= 0x030400a1) + #endif +#endif +#if !defined(CYTHON_FAST_PYCCALL) +#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #include "longintrepr.h" + #undef SHIFT + #undef BASE + #undef MASK +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + #endif + #endif +#else + #include +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) && __cplusplus >= 201103L + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #elif __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__ ) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif + +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #elif defined(__GNUC__) + #define CYTHON_INLINE __inline__ + #elif defined(_MSC_VER) + #define CYTHON_INLINE __inline + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_INLINE inline + #else + #define CYTHON_INLINE + #endif +#endif + +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) + #define Py_OptimizeFlag 0 +#endif +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyClass_Type +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#if PY_VERSION_HEX <= 0x030700A3 || !defined(METH_FASTCALL) + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #define __Pyx_PyCFunctionFast _PyCFunctionFast + #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords +#endif +#if CYTHON_FAST_PYCCALL +#define __Pyx_PyFastCFunction_Check(func)\ + ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS))))) +#else +#define __Pyx_PyFastCFunction_Check(func) 0 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_PYSTON + #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if !CYTHON_FAST_THREAD_STATE || PY_VERSION_HEX < 0x02070000 + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x03060000 + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#elif PY_VERSION_HEX >= 0x03000000 + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_Current +#endif +#if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT) +#include "pythread.h" +#define Py_tss_NEEDS_INIT 0 +typedef int Py_tss_t; +static CYTHON_INLINE int PyThread_tss_create(Py_tss_t *key) { + *key = PyThread_create_key(); + return 0; // PyThread_create_key reports success always +} +static CYTHON_INLINE Py_tss_t * PyThread_tss_alloc(void) { + Py_tss_t *key = (Py_tss_t *)PyObject_Malloc(sizeof(Py_tss_t)); + *key = Py_tss_NEEDS_INIT; + return key; +} +static CYTHON_INLINE void PyThread_tss_free(Py_tss_t *key) { + PyObject_Free(key); +} +static CYTHON_INLINE int PyThread_tss_is_created(Py_tss_t *key) { + return *key != Py_tss_NEEDS_INIT; +} +static CYTHON_INLINE void PyThread_tss_delete(Py_tss_t *key) { + PyThread_delete_key(*key); + *key = Py_tss_NEEDS_INIT; +} +static CYTHON_INLINE int PyThread_tss_set(Py_tss_t *key, void *value) { + return PyThread_set_key_value(*key, value); +} +static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { + return PyThread_get_key_value(*key); +} +#endif // TSS (Thread Specific Storage) API +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#if PY_MAJOR_VERSION >= 3 || CYTHON_FUTURE_DIVISION + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStr(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +#else +#define __Pyx_PyDict_GetItemStr(dict, name) PyDict_GetItem(dict, name) +#endif +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) +#else + #define CYTHON_PEP393_ENABLED 0 + #define PyUnicode_1BYTE_KIND 1 + #define PyUnicode_2BYTE_KIND 2 + #define PyUnicode_4BYTE_KIND 4 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) + #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#endif +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) +#else + #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) +#endif +#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) + #define PyObject_ASCII(o) PyObject_Repr(o) +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyStringObject PyUnicodeObject + #define PyString_Type PyUnicode_Type + #define PyString_Check PyUnicode_Check + #define PyString_CheckExact PyUnicode_CheckExact + #define PyObject_Unicode PyObject_Str +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) + #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) +#else + #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) + #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) +#else + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyIntObject PyLongObject + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define PyNumber_Int PyNumber_Long +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBoolObject PyLongObject +#endif +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY + #ifndef PyUnicode_InternFromString + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) + #endif +#endif +#if PY_VERSION_HEX < 0x030200A4 + typedef long Py_hash_t; + #define __Pyx_PyInt_FromHash_t PyInt_FromLong + #define __Pyx_PyInt_AsHash_t PyInt_AsLong +#else + #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t + #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : (Py_INCREF(func), func)) +#else + #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) +#endif +#if CYTHON_USE_ASYNC_SLOTS + #if PY_VERSION_HEX >= 0x030500B1 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods + #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) + #else + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef __Pyx_PyAsyncMethodsStruct + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + } __Pyx_PyAsyncMethodsStruct; +#endif + +#if defined(WIN32) || defined(MS_WINDOWS) + #define _USE_MATH_DEFINES +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + + +#define __PYX_ERR(f_index, lineno, Ln_error) \ +{ \ + __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ +} + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#define __PYX_HAVE__iocpsupport +#define __PYX_HAVE_API__iocpsupport +/* Early includes */ +#include "io.h" +#include "errno.h" +#include "winsock2.h" +#include "ws2tcpip.h" +#include "windows.h" +#include "Python.h" +#include "string.h" +#include "wchar.h" +#include "winsock_pointers.h" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; + const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#else + #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize +#endif +#define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyObject_AsWritableString(s) ((char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { + const Py_UNICODE *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) +#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) +#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +#if CYTHON_ASSUME_SAFE_MACROS +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#else +#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#endif +#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#else +#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) +#endif +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII +static int __Pyx_sys_getdefaultencoding_not_ascii; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + PyObject* ascii_chars_u = NULL; + PyObject* ascii_chars_b = NULL; + const char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + if (strcmp(default_encoding_c, "ascii") == 0) { + __Pyx_sys_getdefaultencoding_not_ascii = 0; + } else { + char ascii_chars[128]; + int c; + for (c = 0; c < 128; c++) { + ascii_chars[c] = c; + } + __Pyx_sys_getdefaultencoding_not_ascii = 1; + ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); + if (!ascii_chars_u) goto bad; + ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); + if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { + PyErr_Format( + PyExc_ValueError, + "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", + default_encoding_c); + goto bad; + } + Py_DECREF(ascii_chars_u); + Py_DECREF(ascii_chars_b); + } + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return -1; +} +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#else +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +static char* __PYX_DEFAULT_STRING_ENCODING; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; + strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + return -1; +} +#endif +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } + +static PyObject *__pyx_m = NULL; +static PyObject *__pyx_d; +static PyObject *__pyx_b; +static PyObject *__pyx_cython_runtime = NULL; +static PyObject *__pyx_empty_tuple; +static PyObject *__pyx_empty_bytes; +static PyObject *__pyx_empty_unicode; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; + + +static const char *__pyx_f[] = { + "src/twisted/internet/iocpreactor/iocpsupport/iocpsupport.pyx", + "stringsource", + "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi", + "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi", + "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi", + "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi", +}; + +/* "iocpsupport.pyx":7 + * + * # HANDLE and SOCKET are pointer-sized (they are 64 bit wide in 64-bit builds) + * ctypedef size_t HANDLE # <<<<<<<<<<<<<< + * ctypedef size_t SOCKET + * ctypedef unsigned long DWORD + */ +typedef size_t __pyx_t_11iocpsupport_HANDLE; + +/* "iocpsupport.pyx":8 + * # HANDLE and SOCKET are pointer-sized (they are 64 bit wide in 64-bit builds) + * ctypedef size_t HANDLE + * ctypedef size_t SOCKET # <<<<<<<<<<<<<< + * ctypedef unsigned long DWORD + * # it's really a pointer, but we use it as an integer + */ +typedef size_t __pyx_t_11iocpsupport_SOCKET; + +/* "iocpsupport.pyx":9 + * ctypedef size_t HANDLE + * ctypedef size_t SOCKET + * ctypedef unsigned long DWORD # <<<<<<<<<<<<<< + * # it's really a pointer, but we use it as an integer + * ctypedef size_t ULONG_PTR + */ +typedef unsigned long __pyx_t_11iocpsupport_DWORD; + +/* "iocpsupport.pyx":11 + * ctypedef unsigned long DWORD + * # it's really a pointer, but we use it as an integer + * ctypedef size_t ULONG_PTR # <<<<<<<<<<<<<< + * ctypedef int BOOL + * + */ +typedef size_t __pyx_t_11iocpsupport_ULONG_PTR; + +/* "iocpsupport.pyx":12 + * # it's really a pointer, but we use it as an integer + * ctypedef size_t ULONG_PTR + * ctypedef int BOOL # <<<<<<<<<<<<<< + * + * cdef extern from 'io.h': + */ +typedef int __pyx_t_11iocpsupport_BOOL; + +/*--- Type declarations ---*/ +struct __pyx_obj_11iocpsupport_CompletionPort; +struct __pyx_t_11iocpsupport_myOVERLAPPED; +struct __pyx_opt_args_11iocpsupport_makeOV; + +/* "iocpsupport.pyx":128 + * + * + * cdef struct myOVERLAPPED: # <<<<<<<<<<<<<< + * # myOVERLAPPED is the C-level structure that is fed into and read out of a + * # L{CompletionPort}. + */ +struct __pyx_t_11iocpsupport_myOVERLAPPED { + OVERLAPPED ov; + PyObject *attached; +}; + +/* "iocpsupport.pyx":152 + * + * + * cdef myOVERLAPPED *makeOV(object evt, object other=None) except NULL: # <<<<<<<<<<<<<< + * """ + * Make a myOVERLAPPED structure for passing along to a low-level C object. + */ +struct __pyx_opt_args_11iocpsupport_makeOV { + int __pyx_n; + PyObject *other; +}; + +/* "iocpsupport.pyx":190 + * setattr(self, k, v) + * + * cdef class CompletionPort: # <<<<<<<<<<<<<< + * # A wrapper around an I/O completion port created with + * # C{CreateIoCompletionPort}. + */ +struct __pyx_obj_11iocpsupport_CompletionPort { + PyObject_HEAD + __pyx_t_11iocpsupport_HANDLE port; +}; + + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*SetupContext)(const char*, int, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; +#ifdef WITH_THREAD + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + } +#else + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) +#endif + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* GetModuleGlobalName.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#define __Pyx_PyErr_Occurred() __pyx_tstate->curexc_type +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ + const char* function_name); + +/* PyObjectSetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o, n, NULL) +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value); +#else +#define __Pyx_PyObject_DelAttrStr(o,n) PyObject_DelAttr(o,n) +#define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v) +#endif + +/* py_dict_items.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d); + +/* UnpackUnboundCMethod.proto */ +typedef struct { + PyObject *type; + PyObject **method_name; + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; + +/* CallUnboundCMethod0.proto */ +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CallUnboundCMethod0(cfunc, self)\ + (likely((cfunc)->func) ?\ + (likely((cfunc)->flag == METH_NOARGS) ? (*((cfunc)->func))(self, NULL) :\ + (PY_VERSION_HEX >= 0x030600B1 && likely((cfunc)->flag == METH_FASTCALL) ?\ + (PY_VERSION_HEX >= 0x030700A0 ?\ + (*(__Pyx_PyCFunctionFast)(cfunc)->func)(self, &__pyx_empty_tuple, 0) :\ + (*(__Pyx_PyCFunctionFastWithKeywords)(cfunc)->func)(self, &__pyx_empty_tuple, 0, NULL)) :\ + (PY_VERSION_HEX >= 0x030700A0 && (cfunc)->flag == (METH_FASTCALL | METH_KEYWORDS) ?\ + (*(__Pyx_PyCFunctionFastWithKeywords)(cfunc)->func)(self, &__pyx_empty_tuple, 0, NULL) :\ + (likely((cfunc)->flag == (METH_VARARGS | METH_KEYWORDS)) ? ((*(PyCFunctionWithKeywords)(cfunc)->func)(self, __pyx_empty_tuple, NULL)) :\ + ((cfunc)->flag == METH_VARARGS ? (*((cfunc)->func))(self, __pyx_empty_tuple) :\ + __Pyx__CallUnboundCMethod0(cfunc, self)))))) :\ + __Pyx__CallUnboundCMethod0(cfunc, self)) +#else +#define __Pyx_CallUnboundCMethod0(cfunc, self) __Pyx__CallUnboundCMethod0(cfunc, self) +#endif + +/* RaiseTooManyValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* RaiseNeedMoreValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* IterFinish.proto */ +static CYTHON_INLINE int __Pyx_IterFinish(void); + +/* UnpackItemEndCheck.proto */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); + +/* KeywordStringCheck.proto */ +static int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* GetAttr.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *, PyObject *); + +/* GetAttr3.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); + +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck); + +/* IncludeStringH.proto */ +#include + +/* BytesEquals.proto */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* StrEquals.proto */ +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyString_Equals __Pyx_PyUnicode_Equals +#else +#define __Pyx_PyString_Equals __Pyx_PyBytes_Equals +#endif + +/* SliceObject.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice( + PyObject* obj, Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** py_start, PyObject** py_stop, PyObject** py_slice, + int has_cstart, int has_cstop, int wraparound); + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* HasAttr.proto */ +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); + +/* PyObject_GenericGetAttrNoDict.proto */ +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GenericGetAttrNoDict PyObject_GenericGetAttr +#endif + +/* PyObject_GenericGetAttr.proto */ +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject* __Pyx_PyObject_GenericGetAttr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GenericGetAttr PyObject_GenericGetAttr +#endif + +/* SetupReduce.proto */ +static int __Pyx_setup_reduce(PyObject* type_obj); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); + +/* CythonFunction.proto */ +#define __Pyx_CyFunction_USED 1 +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { + PyCFunctionObject func; +#if PY_VERSION_HEX < 0x030500A0 + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; + PyObject *func_classobj; + void *defaults; + int defaults_pyobjects; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; +} __pyx_CyFunctionObject; +static PyTypeObject *__pyx_CyFunctionType = 0; +#define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code)\ + __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) +static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *self, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, + size_t size, + int pyobjects); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(void); + +/* SetNameInClass.proto */ +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 +#define __Pyx_SetNameInClass(ns, name, value)\ + (likely(PyDict_CheckExact(ns)) ? _PyDict_SetItem_KnownHash(ns, name, value, ((PyASCIIObject *) name)->hash) : PyObject_SetItem(ns, name, value)) +#elif CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_SetNameInClass(ns, name, value)\ + (likely(PyDict_CheckExact(ns)) ? PyDict_SetItem(ns, name, value) : PyObject_SetItem(ns, name, value)) +#else +#define __Pyx_SetNameInClass(ns, name, value) PyObject_SetItem(ns, name, value) +#endif + +/* CalculateMetaclass.proto */ +static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases); + +/* Py3ClassCreate.proto */ +static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, + PyObject *mkw, PyObject *modname, PyObject *doc); +static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, + PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); + +/* CLineInTraceback.proto */ +#ifdef CYTHON_CLINE_IN_TRACEBACK +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#else +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#endif + +/* CodeObjectCache.proto */ +typedef struct { + PyCodeObject* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; +}; +static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static PyCodeObject *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_long(unsigned long value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_short(unsigned short value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE unsigned long __Pyx_PyInt_As_unsigned_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE unsigned short __Pyx_PyInt_As_unsigned_short(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +#define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) +#endif +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(void); + +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); + + +/* Module declarations from 'cpython.version' */ + +/* Module declarations from 'iocpsupport' */ +static PyTypeObject *__pyx_ptype_11iocpsupport_CompletionPort = 0; +static struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_f_11iocpsupport_makeOV(PyObject *, struct __pyx_opt_args_11iocpsupport_makeOV *__pyx_optional_args); /*proto*/ +static void __pyx_f_11iocpsupport_unmakeOV(struct __pyx_t_11iocpsupport_myOVERLAPPED *); /*proto*/ +static void __pyx_f_11iocpsupport_raise_error(int, PyObject *); /*proto*/ +static PyObject *__pyx_f_11iocpsupport__makesockaddr(struct sockaddr *, Py_ssize_t); /*proto*/ +static PyObject *__pyx_f_11iocpsupport_fillinetaddr(struct sockaddr_in *, PyObject *); /*proto*/ +static PyObject *__pyx_f_11iocpsupport_fillinet6addr(struct sockaddr_in6 *, PyObject *); /*proto*/ +static int __pyx_f_11iocpsupport_getAddrFamily(__pyx_t_11iocpsupport_SOCKET); /*proto*/ +static PyObject *__pyx_f_11iocpsupport___pyx_unpickle_CompletionPort__set_state(struct __pyx_obj_11iocpsupport_CompletionPort *, PyObject *); /*proto*/ +#define __Pyx_MODULE_NAME "iocpsupport" +extern int __pyx_module_is_main_iocpsupport; +int __pyx_module_is_main_iocpsupport = 0; + +/* Implementation of 'iocpsupport' */ +static PyObject *__pyx_builtin_ValueError; +static PyObject *__pyx_builtin_MemoryError; +static PyObject *__pyx_builtin_RuntimeError; +static const char __pyx_k_[] = ":"; +static const char __pyx_k_i[] = "i"; +static const char __pyx_k_k[] = "k"; +static const char __pyx_k_s[] = "s"; +static const char __pyx_k_v[] = "v"; +static const char __pyx_k__4[] = "["; +static const char __pyx_k__5[] = "]"; +static const char __pyx_k__7[] = "%"; +static const char __pyx_k_kw[] = "kw"; +static const char __pyx_k_ov[] = "ov"; +static const char __pyx_k_rc[] = "rc"; +static const char __pyx_k_doc[] = "__doc__"; +static const char __pyx_k_key[] = "key"; +static const char __pyx_k_new[] = "__new__"; +static const char __pyx_k_obj[] = "obj"; +static const char __pyx_k_res[] = "res"; +static const char __pyx_k_addr[] = "addr"; +static const char __pyx_k_buff[] = "buff"; +static const char __pyx_k_dict[] = "__dict__"; +static const char __pyx_k_init[] = "__init__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "__name__"; +static const char __pyx_k_recv[] = "recv"; +static const char __pyx_k_self[] = "self"; +static const char __pyx_k_send[] = "send"; +static const char __pyx_k_size[] = "size"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_Event[] = "Event"; +static const char __pyx_k_bytes[] = "bytes"; +static const char __pyx_k_flags[] = "flags"; +static const char __pyx_k_items[] = "items"; +static const char __pyx_k_owner[] = "owner"; +static const char __pyx_k_split[] = "split"; +static const char __pyx_k_utf_8[] = "utf-8"; +static const char __pyx_k_accept[] = "accept"; +static const char __pyx_k_family[] = "family"; +static const char __pyx_k_handle[] = "handle"; +static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_name_2[] = "name"; +static const char __pyx_k_pickle[] = "pickle"; +static const char __pyx_k_reduce[] = "__reduce__"; +static const char __pyx_k_rsplit[] = "rsplit"; +static const char __pyx_k_socket[] = "socket"; +static const char __pyx_k_update[] = "update"; +static const char __pyx_k_ws_buf[] = "ws_buf"; +static const char __pyx_k_wsa_pi[] = "wsa_pi"; +static const char __pyx_k_buffers[] = "buffers"; +static const char __pyx_k_connect[] = "connect"; +static const char __pyx_k_namelen[] = "namelen"; +static const char __pyx_k_prepare[] = "__prepare__"; +static const char __pyx_k_bufflist[] = "bufflist"; +static const char __pyx_k_callback[] = "callback"; +static const char __pyx_k_getstate[] = "__getstate__"; +static const char __pyx_k_locallen[] = "locallen"; +static const char __pyx_k_pyx_type[] = "__pyx_type"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_recvfrom[] = "recvfrom"; +static const char __pyx_k_setstate[] = "__setstate__"; +static const char __pyx_k_accepting[] = "accepting"; +static const char __pyx_k_addr_buff[] = "addr_buff"; +static const char __pyx_k_buffcount[] = "buffcount"; +static const char __pyx_k_ipv4_name[] = "ipv4_name"; +static const char __pyx_k_ipv6_name[] = "ipv6_name"; +static const char __pyx_k_listening[] = "listening"; +static const char __pyx_k_localaddr[] = "localaddr"; +static const char __pyx_k_metaclass[] = "__metaclass__"; +static const char __pyx_k_pyx_state[] = "__pyx_state"; +static const char __pyx_k_reduce_ex[] = "__reduce_ex__"; +static const char __pyx_k_remotelen[] = "remotelen"; +static const char __pyx_k_ValueError[] = "ValueError"; +static const char __pyx_k_getsockopt[] = "getsockopt"; +static const char __pyx_k_maxAddrLen[] = "maxAddrLen"; +static const char __pyx_k_mem_buffer[] = "mem_buffer"; +static const char __pyx_k_pyx_result[] = "__pyx_result"; +static const char __pyx_k_remoteaddr[] = "remoteaddr"; +static const char __pyx_k_MemoryError[] = "MemoryError"; +static const char __pyx_k_PickleError[] = "PickleError"; +static const char __pyx_k_c_addr_buff[] = "c_addr_buff"; +static const char __pyx_k_iocpsupport[] = "iocpsupport"; +static const char __pyx_k_Event___init[] = "Event.__init__"; +static const char __pyx_k_RuntimeError[] = "RuntimeError"; +static const char __pyx_k_WindowsError[] = "WindowsError"; +static const char __pyx_k_makesockaddr[] = "makesockaddr"; +static const char __pyx_k_pyx_checksum[] = "__pyx_checksum"; +static const char __pyx_k_stringsource[] = "stringsource"; +static const char __pyx_k_addr_len_buff[] = "addr_len_buff"; +static const char __pyx_k_reduce_cython[] = "__reduce_cython__"; +static const char __pyx_k_have_connectex[] = "have_connectex"; +static const char __pyx_k_c_addr_buff_len[] = "c_addr_buff_len"; +static const char __pyx_k_c_addr_len_buff[] = "c_addr_len_buff"; +static const char __pyx_k_pyx_PickleError[] = "__pyx_PickleError"; +static const char __pyx_k_setstate_cython[] = "__setstate_cython__"; +static const char __pyx_k_get_accept_addrs[] = "get_accept_addrs"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_WSAAddressToStringW[] = "WSAAddressToStringW"; +static const char __pyx_k_c_addr_len_buff_len[] = "c_addr_len_buff_len"; +static const char __pyx_k_invalid_IP_address_r[] = "invalid IP address %r"; +static const char __pyx_k_CreateIoCompletionPort[] = "CreateIoCompletionPort"; +static const char __pyx_k_invalid_IPv6_address_r[] = "invalid IPv6 address %r"; +static const char __pyx_k_PostQueuedCompletionStatus[] = "PostQueuedCompletionStatus"; +static const char __pyx_k_unsupported_address_family[] = "unsupported address family"; +static const char __pyx_k_pyx_unpickle_CompletionPort[] = "__pyx_unpickle_CompletionPort"; +static const char __pyx_k_unsupported_address_family_d[] = "unsupported address family %d"; +static const char __pyx_k_length_of_address_length_buffer[] = "length of address length buffer needs to be sizeof(int)"; +static const char __pyx_k_undefined_error_occurred_during[] = "undefined error occurred during address parsing"; +static const char __pyx_k_ConnectEx_is_not_available_on_th[] = "ConnectEx is not available on this system"; +static const char __pyx_k_Failed_to_initialize_Winsock_fun[] = "Failed to initialize Winsock function vectors"; +static const char __pyx_k_Incompatible_checksums_s_vs_0x90[] = "Incompatible checksums (%s vs 0x901555f = (port))"; +static const char __pyx_k_src_twisted_internet_iocpreactor[] = "src/twisted/internet/iocpreactor/iocpsupport/iocpsupport.pyx"; +static const char __pyx_k_src_twisted_internet_iocpreactor_2[] = "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi"; +static const char __pyx_k_src_twisted_internet_iocpreactor_3[] = "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi"; +static const char __pyx_k_src_twisted_internet_iocpreactor_4[] = "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi"; +static const char __pyx_k_src_twisted_internet_iocpreactor_5[] = "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi"; +static PyObject *__pyx_kp_u_; +static PyObject *__pyx_kp_s_ConnectEx_is_not_available_on_th; +static PyObject *__pyx_n_s_CreateIoCompletionPort; +static PyObject *__pyx_n_s_Event; +static PyObject *__pyx_n_s_Event___init; +static PyObject *__pyx_kp_s_Failed_to_initialize_Winsock_fun; +static PyObject *__pyx_kp_s_Incompatible_checksums_s_vs_0x90; +static PyObject *__pyx_n_s_MemoryError; +static PyObject *__pyx_n_s_PickleError; +static PyObject *__pyx_n_s_PostQueuedCompletionStatus; +static PyObject *__pyx_n_s_RuntimeError; +static PyObject *__pyx_n_s_ValueError; +static PyObject *__pyx_n_s_WSAAddressToStringW; +static PyObject *__pyx_n_s_WindowsError; +static PyObject *__pyx_kp_s__4; +static PyObject *__pyx_kp_s__5; +static PyObject *__pyx_kp_s__7; +static PyObject *__pyx_n_s_accept; +static PyObject *__pyx_n_s_accepting; +static PyObject *__pyx_n_s_addr; +static PyObject *__pyx_n_s_addr_buff; +static PyObject *__pyx_n_s_addr_len_buff; +static PyObject *__pyx_n_s_buff; +static PyObject *__pyx_n_s_buffcount; +static PyObject *__pyx_n_s_buffers; +static PyObject *__pyx_n_s_bufflist; +static PyObject *__pyx_n_s_bytes; +static PyObject *__pyx_n_s_c_addr_buff; +static PyObject *__pyx_n_s_c_addr_buff_len; +static PyObject *__pyx_n_s_c_addr_len_buff; +static PyObject *__pyx_n_s_c_addr_len_buff_len; +static PyObject *__pyx_n_s_callback; +static PyObject *__pyx_n_s_cline_in_traceback; +static PyObject *__pyx_n_s_connect; +static PyObject *__pyx_n_s_dict; +static PyObject *__pyx_n_s_doc; +static PyObject *__pyx_n_s_family; +static PyObject *__pyx_n_s_flags; +static PyObject *__pyx_n_s_get_accept_addrs; +static PyObject *__pyx_n_s_getsockopt; +static PyObject *__pyx_n_s_getstate; +static PyObject *__pyx_n_s_handle; +static PyObject *__pyx_n_s_have_connectex; +static PyObject *__pyx_n_s_i; +static PyObject *__pyx_n_s_import; +static PyObject *__pyx_n_s_init; +static PyObject *__pyx_kp_s_invalid_IP_address_r; +static PyObject *__pyx_kp_s_invalid_IPv6_address_r; +static PyObject *__pyx_n_s_iocpsupport; +static PyObject *__pyx_n_s_ipv4_name; +static PyObject *__pyx_n_s_ipv6_name; +static PyObject *__pyx_n_s_items; +static PyObject *__pyx_n_s_k; +static PyObject *__pyx_n_s_key; +static PyObject *__pyx_n_s_kw; +static PyObject *__pyx_kp_s_length_of_address_length_buffer; +static PyObject *__pyx_n_s_listening; +static PyObject *__pyx_n_s_localaddr; +static PyObject *__pyx_n_s_locallen; +static PyObject *__pyx_n_s_main; +static PyObject *__pyx_n_s_makesockaddr; +static PyObject *__pyx_n_s_maxAddrLen; +static PyObject *__pyx_n_s_mem_buffer; +static PyObject *__pyx_n_s_metaclass; +static PyObject *__pyx_n_s_module; +static PyObject *__pyx_n_s_name; +static PyObject *__pyx_n_s_name_2; +static PyObject *__pyx_n_s_namelen; +static PyObject *__pyx_n_s_new; +static PyObject *__pyx_n_s_obj; +static PyObject *__pyx_n_s_ov; +static PyObject *__pyx_n_s_owner; +static PyObject *__pyx_n_s_pickle; +static PyObject *__pyx_n_s_prepare; +static PyObject *__pyx_n_s_pyx_PickleError; +static PyObject *__pyx_n_s_pyx_checksum; +static PyObject *__pyx_n_s_pyx_result; +static PyObject *__pyx_n_s_pyx_state; +static PyObject *__pyx_n_s_pyx_type; +static PyObject *__pyx_n_s_pyx_unpickle_CompletionPort; +static PyObject *__pyx_n_s_qualname; +static PyObject *__pyx_n_s_rc; +static PyObject *__pyx_n_s_recv; +static PyObject *__pyx_n_s_recvfrom; +static PyObject *__pyx_n_s_reduce; +static PyObject *__pyx_n_s_reduce_cython; +static PyObject *__pyx_n_s_reduce_ex; +static PyObject *__pyx_n_s_remoteaddr; +static PyObject *__pyx_n_s_remotelen; +static PyObject *__pyx_n_s_res; +static PyObject *__pyx_n_s_rsplit; +static PyObject *__pyx_n_s_s; +static PyObject *__pyx_n_s_self; +static PyObject *__pyx_n_s_send; +static PyObject *__pyx_n_s_setstate; +static PyObject *__pyx_n_s_setstate_cython; +static PyObject *__pyx_n_s_size; +static PyObject *__pyx_n_s_socket; +static PyObject *__pyx_n_s_split; +static PyObject *__pyx_kp_s_src_twisted_internet_iocpreactor; +static PyObject *__pyx_kp_s_src_twisted_internet_iocpreactor_2; +static PyObject *__pyx_kp_s_src_twisted_internet_iocpreactor_3; +static PyObject *__pyx_kp_s_src_twisted_internet_iocpreactor_4; +static PyObject *__pyx_kp_s_src_twisted_internet_iocpreactor_5; +static PyObject *__pyx_kp_s_stringsource; +static PyObject *__pyx_n_s_test; +static PyObject *__pyx_kp_s_undefined_error_occurred_during; +static PyObject *__pyx_kp_s_unsupported_address_family; +static PyObject *__pyx_kp_s_unsupported_address_family_d; +static PyObject *__pyx_n_s_update; +static PyObject *__pyx_kp_s_utf_8; +static PyObject *__pyx_n_s_v; +static PyObject *__pyx_n_s_ws_buf; +static PyObject *__pyx_n_s_wsa_pi; +static PyObject *__pyx_pf_11iocpsupport_5Event___init__(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_callback, PyObject *__pyx_v_owner, PyObject *__pyx_v_kw); /* proto */ +static int __pyx_pf_11iocpsupport_14CompletionPort___init__(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_2addHandle(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self, __pyx_t_11iocpsupport_HANDLE __pyx_v_handle, size_t __pyx_v_key); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_4getEvent(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self, long __pyx_v_timeout); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_6postEvent(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self, unsigned long __pyx_v_bytes, size_t __pyx_v_key, PyObject *__pyx_v_obj); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_8__del__(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_10__reduce_cython__(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_12__setstate_cython__(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_makesockaddr(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_buff); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_2maxAddrLen(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_4accept(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_listening, long __pyx_v_accepting, PyObject *__pyx_v_buff, PyObject *__pyx_v_obj); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_6get_accept_addrs(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED long __pyx_v_s, PyObject *__pyx_v_buff); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_8connect(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s, PyObject *__pyx_v_addr, PyObject *__pyx_v_obj); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_10recv(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s, PyObject *__pyx_v_bufflist, PyObject *__pyx_v_obj, unsigned long __pyx_v_flags); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_12recvfrom(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s, PyObject *__pyx_v_buff, PyObject *__pyx_v_addr_buff, PyObject *__pyx_v_addr_len_buff, PyObject *__pyx_v_obj, unsigned long __pyx_v_flags); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_14send(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s, PyObject *__pyx_v_buff, PyObject *__pyx_v_obj, unsigned long __pyx_v_flags); /* proto */ +static PyObject *__pyx_pf_11iocpsupport_16__pyx_unpickle_CompletionPort(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_tp_new_11iocpsupport_CompletionPort(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_items = {0, &__pyx_n_s_items, 0, 0, 0}; +static PyObject *__pyx_int_0; +static PyObject *__pyx_int_1; +static PyObject *__pyx_int_151082335; +static PyObject *__pyx_int_neg_1; +static PyObject *__pyx_slice__6; +static PyObject *__pyx_tuple__2; +static PyObject *__pyx_tuple__3; +static PyObject *__pyx_tuple__8; +static PyObject *__pyx_tuple__9; +static PyObject *__pyx_tuple__11; +static PyObject *__pyx_tuple__13; +static PyObject *__pyx_tuple__15; +static PyObject *__pyx_tuple__17; +static PyObject *__pyx_tuple__19; +static PyObject *__pyx_tuple__21; +static PyObject *__pyx_tuple__23; +static PyObject *__pyx_tuple__25; +static PyObject *__pyx_tuple__27; +static PyObject *__pyx_codeobj__10; +static PyObject *__pyx_codeobj__12; +static PyObject *__pyx_codeobj__14; +static PyObject *__pyx_codeobj__16; +static PyObject *__pyx_codeobj__18; +static PyObject *__pyx_codeobj__20; +static PyObject *__pyx_codeobj__22; +static PyObject *__pyx_codeobj__24; +static PyObject *__pyx_codeobj__26; +static PyObject *__pyx_codeobj__28; +/* Late includes */ + +/* "iocpsupport.pyx":152 + * + * + * cdef myOVERLAPPED *makeOV(object evt, object other=None) except NULL: # <<<<<<<<<<<<<< + * """ + * Make a myOVERLAPPED structure for passing along to a low-level C object. + */ + +static struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_f_11iocpsupport_makeOV(PyObject *__pyx_v_evt, struct __pyx_opt_args_11iocpsupport_makeOV *__pyx_optional_args) { + PyObject *__pyx_v_other = ((PyObject *)Py_None); + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_res; + PyObject *__pyx_v_tupl = NULL; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_r; + __Pyx_RefNannyDeclarations + void *__pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + __Pyx_RefNannySetupContext("makeOV", 0); + if (__pyx_optional_args) { + if (__pyx_optional_args->__pyx_n > 0) { + __pyx_v_other = __pyx_optional_args->other; + } + } + + /* "iocpsupport.pyx":156 + * Make a myOVERLAPPED structure for passing along to a low-level C object. + * """ + * cdef myOVERLAPPED *res = PyMem_Malloc(sizeof(myOVERLAPPED)) # <<<<<<<<<<<<<< + * if not res: + * raise MemoryError + */ + __pyx_t_1 = PyMem_Malloc((sizeof(struct __pyx_t_11iocpsupport_myOVERLAPPED))); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(0, 156, __pyx_L1_error) + __pyx_v_res = ((struct __pyx_t_11iocpsupport_myOVERLAPPED *)__pyx_t_1); + + /* "iocpsupport.pyx":157 + * """ + * cdef myOVERLAPPED *res = PyMem_Malloc(sizeof(myOVERLAPPED)) + * if not res: # <<<<<<<<<<<<<< + * raise MemoryError + * memset(res, 0, sizeof(myOVERLAPPED)) + */ + __pyx_t_2 = ((!(__pyx_v_res != 0)) != 0); + if (unlikely(__pyx_t_2)) { + + /* "iocpsupport.pyx":158 + * cdef myOVERLAPPED *res = PyMem_Malloc(sizeof(myOVERLAPPED)) + * if not res: + * raise MemoryError # <<<<<<<<<<<<<< + * memset(res, 0, sizeof(myOVERLAPPED)) + * tupl = (evt, other) + */ + PyErr_NoMemory(); __PYX_ERR(0, 158, __pyx_L1_error) + + /* "iocpsupport.pyx":157 + * """ + * cdef myOVERLAPPED *res = PyMem_Malloc(sizeof(myOVERLAPPED)) + * if not res: # <<<<<<<<<<<<<< + * raise MemoryError + * memset(res, 0, sizeof(myOVERLAPPED)) + */ + } + + /* "iocpsupport.pyx":159 + * if not res: + * raise MemoryError + * memset(res, 0, sizeof(myOVERLAPPED)) # <<<<<<<<<<<<<< + * tupl = (evt, other) + * res.attached = tupl + */ + (void)(memset(__pyx_v_res, 0, (sizeof(struct __pyx_t_11iocpsupport_myOVERLAPPED)))); + + /* "iocpsupport.pyx":160 + * raise MemoryError + * memset(res, 0, sizeof(myOVERLAPPED)) + * tupl = (evt, other) # <<<<<<<<<<<<<< + * res.attached = tupl + * Py_XINCREF(tupl) + */ + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 160, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_evt); + __Pyx_GIVEREF(__pyx_v_evt); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_evt); + __Pyx_INCREF(__pyx_v_other); + __Pyx_GIVEREF(__pyx_v_other); + PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_v_other); + __pyx_v_tupl = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "iocpsupport.pyx":161 + * memset(res, 0, sizeof(myOVERLAPPED)) + * tupl = (evt, other) + * res.attached = tupl # <<<<<<<<<<<<<< + * Py_XINCREF(tupl) + * return res + */ + __pyx_v_res->attached = ((PyObject *)__pyx_v_tupl); + + /* "iocpsupport.pyx":162 + * tupl = (evt, other) + * res.attached = tupl + * Py_XINCREF(tupl) # <<<<<<<<<<<<<< + * return res + * + */ + Py_XINCREF(__pyx_v_tupl); + + /* "iocpsupport.pyx":163 + * res.attached = tupl + * Py_XINCREF(tupl) + * return res # <<<<<<<<<<<<<< + * + * + */ + __pyx_r = __pyx_v_res; + goto __pyx_L0; + + /* "iocpsupport.pyx":152 + * + * + * cdef myOVERLAPPED *makeOV(object evt, object other=None) except NULL: # <<<<<<<<<<<<<< + * """ + * Make a myOVERLAPPED structure for passing along to a low-level C object. + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("iocpsupport.makeOV", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_tupl); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":167 + * + * + * cdef void unmakeOV(myOVERLAPPED* ov): # <<<<<<<<<<<<<< + * """ + * Clean up a myOVERLAPPED structure, decrefing the Python object and freeing + */ + +static void __pyx_f_11iocpsupport_unmakeOV(struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_ov) { + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("unmakeOV", 0); + + /* "iocpsupport.pyx":172 + * its memory. + * """ + * Py_XDECREF(ov.attached) # <<<<<<<<<<<<<< + * memset(ov, 0, sizeof(myOVERLAPPED)) + * PyMem_Free(ov) + */ + __pyx_t_1 = ((PyObject *)__pyx_v_ov->attached); + __Pyx_INCREF(__pyx_t_1); + Py_XDECREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "iocpsupport.pyx":173 + * """ + * Py_XDECREF(ov.attached) + * memset(ov, 0, sizeof(myOVERLAPPED)) # <<<<<<<<<<<<<< + * PyMem_Free(ov) + * + */ + (void)(memset(__pyx_v_ov, 0, (sizeof(struct __pyx_t_11iocpsupport_myOVERLAPPED)))); + + /* "iocpsupport.pyx":174 + * Py_XDECREF(ov.attached) + * memset(ov, 0, sizeof(myOVERLAPPED)) + * PyMem_Free(ov) # <<<<<<<<<<<<<< + * + * + */ + PyMem_Free(__pyx_v_ov); + + /* "iocpsupport.pyx":167 + * + * + * cdef void unmakeOV(myOVERLAPPED* ov): # <<<<<<<<<<<<<< + * """ + * Clean up a myOVERLAPPED structure, decrefing the Python object and freeing + */ + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +/* "iocpsupport.pyx":178 + * + * + * cdef void raise_error(int err, object message) except *: # <<<<<<<<<<<<<< + * if not err: + * err = GetLastError() + */ + +static void __pyx_f_11iocpsupport_raise_error(int __pyx_v_err, PyObject *__pyx_v_message) { + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannySetupContext("raise_error", 0); + + /* "iocpsupport.pyx":179 + * + * cdef void raise_error(int err, object message) except *: + * if not err: # <<<<<<<<<<<<<< + * err = GetLastError() + * raise WindowsError(message, err) + */ + __pyx_t_1 = ((!(__pyx_v_err != 0)) != 0); + if (__pyx_t_1) { + + /* "iocpsupport.pyx":180 + * cdef void raise_error(int err, object message) except *: + * if not err: + * err = GetLastError() # <<<<<<<<<<<<<< + * raise WindowsError(message, err) + * + */ + __pyx_v_err = GetLastError(); + + /* "iocpsupport.pyx":179 + * + * cdef void raise_error(int err, object message) except *: + * if not err: # <<<<<<<<<<<<<< + * err = GetLastError() + * raise WindowsError(message, err) + */ + } + + /* "iocpsupport.pyx":181 + * if not err: + * err = GetLastError() + * raise WindowsError(message, err) # <<<<<<<<<<<<<< + * + * class Event: + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_WindowsError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 181, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_err); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 181, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 181, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_message); + __Pyx_GIVEREF(__pyx_v_message); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_message); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 181, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 181, __pyx_L1_error) + + /* "iocpsupport.pyx":178 + * + * + * cdef void raise_error(int err, object message) except *: # <<<<<<<<<<<<<< + * if not err: + * err = GetLastError() + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("iocpsupport.raise_error", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); +} + +/* "iocpsupport.pyx":184 + * + * class Event: + * def __init__(self, callback, owner, **kw): # <<<<<<<<<<<<<< + * self.callback = callback + * self.owner = owner + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_5Event_1__init__(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_11iocpsupport_5Event_1__init__ = {"__init__", (PyCFunction)__pyx_pw_11iocpsupport_5Event_1__init__, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_11iocpsupport_5Event_1__init__(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + PyObject *__pyx_v_callback = 0; + PyObject *__pyx_v_owner = 0; + PyObject *__pyx_v_kw = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + __pyx_v_kw = PyDict_New(); if (unlikely(!__pyx_v_kw)) return NULL; + __Pyx_GOTREF(__pyx_v_kw); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_self,&__pyx_n_s_callback,&__pyx_n_s_owner,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_self)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_callback)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__init__", 1, 3, 3, 1); __PYX_ERR(0, 184, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_owner)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__init__", 1, 3, 3, 2); __PYX_ERR(0, 184, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kw, values, pos_args, "__init__") < 0)) __PYX_ERR(0, 184, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_self = values[0]; + __pyx_v_callback = values[1]; + __pyx_v_owner = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 184, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_DECREF(__pyx_v_kw); __pyx_v_kw = 0; + __Pyx_AddTraceback("iocpsupport.Event.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_5Event___init__(__pyx_self, __pyx_v_self, __pyx_v_callback, __pyx_v_owner, __pyx_v_kw); + + /* function exit code */ + __Pyx_XDECREF(__pyx_v_kw); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_5Event___init__(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_callback, PyObject *__pyx_v_owner, PyObject *__pyx_v_kw) { + PyObject *__pyx_v_k = NULL; + PyObject *__pyx_v_v = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *(*__pyx_t_8)(PyObject *); + int __pyx_t_9; + __Pyx_RefNannySetupContext("__init__", 0); + + /* "iocpsupport.pyx":185 + * class Event: + * def __init__(self, callback, owner, **kw): + * self.callback = callback # <<<<<<<<<<<<<< + * self.owner = owner + * for k, v in kw.items(): + */ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_n_s_callback, __pyx_v_callback) < 0) __PYX_ERR(0, 185, __pyx_L1_error) + + /* "iocpsupport.pyx":186 + * def __init__(self, callback, owner, **kw): + * self.callback = callback + * self.owner = owner # <<<<<<<<<<<<<< + * for k, v in kw.items(): + * setattr(self, k, v) + */ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_n_s_owner, __pyx_v_owner) < 0) __PYX_ERR(0, 186, __pyx_L1_error) + + /* "iocpsupport.pyx":187 + * self.callback = callback + * self.owner = owner + * for k, v in kw.items(): # <<<<<<<<<<<<<< + * setattr(self, k, v) + * + */ + __pyx_t_1 = __Pyx_PyDict_Items(__pyx_v_kw); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { + __pyx_t_2 = __pyx_t_1; __Pyx_INCREF(__pyx_t_2); __pyx_t_3 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 187, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_2)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(0, 187, __pyx_L1_error) + #else + __pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + #endif + } else { + if (__pyx_t_3 >= PyTuple_GET_SIZE(__pyx_t_2)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(0, 187, __pyx_L1_error) + #else + __pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + #endif + } + } else { + __pyx_t_1 = __pyx_t_4(__pyx_t_2); + if (unlikely(!__pyx_t_1)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else __PYX_ERR(0, 187, __pyx_L1_error) + } + break; + } + __Pyx_GOTREF(__pyx_t_1); + } + if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 187, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_5 = PyTuple_GET_ITEM(sequence, 0); + __pyx_t_6 = PyTuple_GET_ITEM(sequence, 1); + } else { + __pyx_t_5 = PyList_GET_ITEM(sequence, 0); + __pyx_t_6 = PyList_GET_ITEM(sequence, 1); + } + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(__pyx_t_6); + #else + __pyx_t_5 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; + index = 0; __pyx_t_5 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_5)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_5); + index = 1; __pyx_t_6 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_6)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_7), 2) < 0) __PYX_ERR(0, 187, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L6_unpacking_done; + __pyx_L5_unpacking_failed:; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 187, __pyx_L1_error) + __pyx_L6_unpacking_done:; + } + __Pyx_XDECREF_SET(__pyx_v_k, __pyx_t_5); + __pyx_t_5 = 0; + __Pyx_XDECREF_SET(__pyx_v_v, __pyx_t_6); + __pyx_t_6 = 0; + + /* "iocpsupport.pyx":188 + * self.owner = owner + * for k, v in kw.items(): + * setattr(self, k, v) # <<<<<<<<<<<<<< + * + * cdef class CompletionPort: + */ + __pyx_t_9 = PyObject_SetAttr(__pyx_v_self, __pyx_v_k, __pyx_v_v); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 188, __pyx_L1_error) + + /* "iocpsupport.pyx":187 + * self.callback = callback + * self.owner = owner + * for k, v in kw.items(): # <<<<<<<<<<<<<< + * setattr(self, k, v) + * + */ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "iocpsupport.pyx":184 + * + * class Event: + * def __init__(self, callback, owner, **kw): # <<<<<<<<<<<<<< + * self.callback = callback + * self.owner = owner + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("iocpsupport.Event.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_k); + __Pyx_XDECREF(__pyx_v_v); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":198 + * + * cdef HANDLE port + * def __init__(self): # <<<<<<<<<<<<<< + * cdef HANDLE res + * res = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0) + */ + +/* Python wrapper */ +static int __pyx_pw_11iocpsupport_14CompletionPort_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_11iocpsupport_14CompletionPort_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) { + __Pyx_RaiseArgtupleInvalid("__init__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;} + if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__init__", 0))) return -1; + __pyx_r = __pyx_pf_11iocpsupport_14CompletionPort___init__(((struct __pyx_obj_11iocpsupport_CompletionPort *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_11iocpsupport_14CompletionPort___init__(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self) { + __pyx_t_11iocpsupport_HANDLE __pyx_v_res; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + __Pyx_RefNannySetupContext("__init__", 0); + + /* "iocpsupport.pyx":200 + * def __init__(self): + * cdef HANDLE res + * res = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0) # <<<<<<<<<<<<<< + * if not res: + * raise_error(0, 'CreateIoCompletionPort') + */ + __pyx_v_res = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0); + + /* "iocpsupport.pyx":201 + * cdef HANDLE res + * res = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0) + * if not res: # <<<<<<<<<<<<<< + * raise_error(0, 'CreateIoCompletionPort') + * self.port = res + */ + __pyx_t_1 = ((!(__pyx_v_res != 0)) != 0); + if (__pyx_t_1) { + + /* "iocpsupport.pyx":202 + * res = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0) + * if not res: + * raise_error(0, 'CreateIoCompletionPort') # <<<<<<<<<<<<<< + * self.port = res + * + */ + __pyx_f_11iocpsupport_raise_error(0, __pyx_n_s_CreateIoCompletionPort); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 202, __pyx_L1_error) + + /* "iocpsupport.pyx":201 + * cdef HANDLE res + * res = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0) + * if not res: # <<<<<<<<<<<<<< + * raise_error(0, 'CreateIoCompletionPort') + * self.port = res + */ + } + + /* "iocpsupport.pyx":203 + * if not res: + * raise_error(0, 'CreateIoCompletionPort') + * self.port = res # <<<<<<<<<<<<<< + * + * def addHandle(self, HANDLE handle, size_t key=0): + */ + __pyx_v_self->port = __pyx_v_res; + + /* "iocpsupport.pyx":198 + * + * cdef HANDLE port + * def __init__(self): # <<<<<<<<<<<<<< + * cdef HANDLE res + * res = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0) + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("iocpsupport.CompletionPort.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":205 + * self.port = res + * + * def addHandle(self, HANDLE handle, size_t key=0): # <<<<<<<<<<<<<< + * cdef HANDLE res + * res = CreateIoCompletionPort(handle, self.port, key, 0) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_3addHandle(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_3addHandle(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + __pyx_t_11iocpsupport_HANDLE __pyx_v_handle; + size_t __pyx_v_key; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("addHandle (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_handle,&__pyx_n_s_key,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_handle)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (kw_args > 0) { + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_key); + if (value) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "addHandle") < 0)) __PYX_ERR(0, 205, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_handle = __Pyx_PyInt_As_size_t(values[0]); if (unlikely((__pyx_v_handle == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 205, __pyx_L3_error) + if (values[1]) { + __pyx_v_key = __Pyx_PyInt_As_size_t(values[1]); if (unlikely((__pyx_v_key == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 205, __pyx_L3_error) + } else { + __pyx_v_key = ((size_t)0); + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("addHandle", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 205, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.CompletionPort.addHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_14CompletionPort_2addHandle(((struct __pyx_obj_11iocpsupport_CompletionPort *)__pyx_v_self), __pyx_v_handle, __pyx_v_key); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_2addHandle(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self, __pyx_t_11iocpsupport_HANDLE __pyx_v_handle, size_t __pyx_v_key) { + __pyx_t_11iocpsupport_HANDLE __pyx_v_res; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + __Pyx_RefNannySetupContext("addHandle", 0); + + /* "iocpsupport.pyx":207 + * def addHandle(self, HANDLE handle, size_t key=0): + * cdef HANDLE res + * res = CreateIoCompletionPort(handle, self.port, key, 0) # <<<<<<<<<<<<<< + * if not res: + * raise_error(0, 'CreateIoCompletionPort') + */ + __pyx_v_res = CreateIoCompletionPort(__pyx_v_handle, __pyx_v_self->port, __pyx_v_key, 0); + + /* "iocpsupport.pyx":208 + * cdef HANDLE res + * res = CreateIoCompletionPort(handle, self.port, key, 0) + * if not res: # <<<<<<<<<<<<<< + * raise_error(0, 'CreateIoCompletionPort') + * + */ + __pyx_t_1 = ((!(__pyx_v_res != 0)) != 0); + if (__pyx_t_1) { + + /* "iocpsupport.pyx":209 + * res = CreateIoCompletionPort(handle, self.port, key, 0) + * if not res: + * raise_error(0, 'CreateIoCompletionPort') # <<<<<<<<<<<<<< + * + * def getEvent(self, long timeout): + */ + __pyx_f_11iocpsupport_raise_error(0, __pyx_n_s_CreateIoCompletionPort); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 209, __pyx_L1_error) + + /* "iocpsupport.pyx":208 + * cdef HANDLE res + * res = CreateIoCompletionPort(handle, self.port, key, 0) + * if not res: # <<<<<<<<<<<<<< + * raise_error(0, 'CreateIoCompletionPort') + * + */ + } + + /* "iocpsupport.pyx":205 + * self.port = res + * + * def addHandle(self, HANDLE handle, size_t key=0): # <<<<<<<<<<<<<< + * cdef HANDLE res + * res = CreateIoCompletionPort(handle, self.port, key, 0) + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("iocpsupport.CompletionPort.addHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":211 + * raise_error(0, 'CreateIoCompletionPort') + * + * def getEvent(self, long timeout): # <<<<<<<<<<<<<< + * cdef PyThreadState *_save + * cdef unsigned long bytes, rc + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_5getEvent(PyObject *__pyx_v_self, PyObject *__pyx_arg_timeout); /*proto*/ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_5getEvent(PyObject *__pyx_v_self, PyObject *__pyx_arg_timeout) { + long __pyx_v_timeout; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("getEvent (wrapper)", 0); + assert(__pyx_arg_timeout); { + __pyx_v_timeout = __Pyx_PyInt_As_long(__pyx_arg_timeout); if (unlikely((__pyx_v_timeout == (long)-1) && PyErr_Occurred())) __PYX_ERR(0, 211, __pyx_L3_error) + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.CompletionPort.getEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_14CompletionPort_4getEvent(((struct __pyx_obj_11iocpsupport_CompletionPort *)__pyx_v_self), ((long)__pyx_v_timeout)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_4getEvent(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self, long __pyx_v_timeout) { + PyThreadState *__pyx_v__save; + unsigned long __pyx_v_bytes; + unsigned long __pyx_v_rc; + size_t __pyx_v_key; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_ov; + PyObject *__pyx_v_obj = NULL; + CYTHON_UNUSED PyObject *__pyx_v_ignored = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *(*__pyx_t_6)(PyObject *); + __Pyx_RefNannySetupContext("getEvent", 0); + + /* "iocpsupport.pyx":217 + * cdef myOVERLAPPED *ov + * + * _save = PyEval_SaveThread() # <<<<<<<<<<<<<< + * rc = GetQueuedCompletionStatus(self.port, &bytes, &key, &ov, timeout) + * PyEval_RestoreThread(_save) + */ + __pyx_v__save = PyEval_SaveThread(); + + /* "iocpsupport.pyx":218 + * + * _save = PyEval_SaveThread() + * rc = GetQueuedCompletionStatus(self.port, &bytes, &key, &ov, timeout) # <<<<<<<<<<<<<< + * PyEval_RestoreThread(_save) + * + */ + __pyx_v_rc = GetQueuedCompletionStatus(__pyx_v_self->port, (&__pyx_v_bytes), (&__pyx_v_key), ((OVERLAPPED **)(&__pyx_v_ov)), __pyx_v_timeout); + + /* "iocpsupport.pyx":219 + * _save = PyEval_SaveThread() + * rc = GetQueuedCompletionStatus(self.port, &bytes, &key, &ov, timeout) + * PyEval_RestoreThread(_save) # <<<<<<<<<<<<<< + * + * if not rc: + */ + PyEval_RestoreThread(__pyx_v__save); + + /* "iocpsupport.pyx":221 + * PyEval_RestoreThread(_save) + * + * if not rc: # <<<<<<<<<<<<<< + * rc = GetLastError() + * else: + */ + __pyx_t_1 = ((!(__pyx_v_rc != 0)) != 0); + if (__pyx_t_1) { + + /* "iocpsupport.pyx":222 + * + * if not rc: + * rc = GetLastError() # <<<<<<<<<<<<<< + * else: + * rc = 0 + */ + __pyx_v_rc = GetLastError(); + + /* "iocpsupport.pyx":221 + * PyEval_RestoreThread(_save) + * + * if not rc: # <<<<<<<<<<<<<< + * rc = GetLastError() + * else: + */ + goto __pyx_L3; + } + + /* "iocpsupport.pyx":224 + * rc = GetLastError() + * else: + * rc = 0 # <<<<<<<<<<<<<< + * + * obj = None + */ + /*else*/ { + __pyx_v_rc = 0; + } + __pyx_L3:; + + /* "iocpsupport.pyx":226 + * rc = 0 + * + * obj = None # <<<<<<<<<<<<<< + * if ov: + * obj, ignored = ov.attached + */ + __Pyx_INCREF(Py_None); + __pyx_v_obj = Py_None; + + /* "iocpsupport.pyx":227 + * + * obj = None + * if ov: # <<<<<<<<<<<<<< + * obj, ignored = ov.attached + * unmakeOV(ov) + */ + __pyx_t_1 = (__pyx_v_ov != 0); + if (__pyx_t_1) { + + /* "iocpsupport.pyx":228 + * obj = None + * if ov: + * obj, ignored = ov.attached # <<<<<<<<<<<<<< + * unmakeOV(ov) + * + */ + __pyx_t_2 = ((PyObject *)__pyx_v_ov->attached); + __Pyx_INCREF(__pyx_t_2); + if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 228, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + } else { + __pyx_t_3 = PyList_GET_ITEM(sequence, 0); + __pyx_t_4 = PyList_GET_ITEM(sequence, 1); + } + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + #else + __pyx_t_3 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_5 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_6 = Py_TYPE(__pyx_t_5)->tp_iternext; + index = 0; __pyx_t_3 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_4 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_4)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 2) < 0) __PYX_ERR(0, 228, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L6_unpacking_done; + __pyx_L5_unpacking_failed:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 228, __pyx_L1_error) + __pyx_L6_unpacking_done:; + } + __Pyx_DECREF_SET(__pyx_v_obj, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_v_ignored = __pyx_t_4; + __pyx_t_4 = 0; + + /* "iocpsupport.pyx":229 + * if ov: + * obj, ignored = ov.attached + * unmakeOV(ov) # <<<<<<<<<<<<<< + * + * return (rc, bytes, key, obj) + */ + __pyx_f_11iocpsupport_unmakeOV(__pyx_v_ov); + + /* "iocpsupport.pyx":227 + * + * obj = None + * if ov: # <<<<<<<<<<<<<< + * obj, ignored = ov.attached + * unmakeOV(ov) + */ + } + + /* "iocpsupport.pyx":231 + * unmakeOV(ov) + * + * return (rc, bytes, key, obj) # <<<<<<<<<<<<<< + * + * def postEvent(self, unsigned long bytes, size_t key, obj): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyInt_From_unsigned_long(__pyx_v_rc); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyInt_From_unsigned_long(__pyx_v_bytes); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_3 = __Pyx_PyInt_FromSize_t(__pyx_v_key); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = PyTuple_New(4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_t_3); + __Pyx_INCREF(__pyx_v_obj); + __Pyx_GIVEREF(__pyx_v_obj); + PyTuple_SET_ITEM(__pyx_t_5, 3, __pyx_v_obj); + __pyx_t_2 = 0; + __pyx_t_4 = 0; + __pyx_t_3 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "iocpsupport.pyx":211 + * raise_error(0, 'CreateIoCompletionPort') + * + * def getEvent(self, long timeout): # <<<<<<<<<<<<<< + * cdef PyThreadState *_save + * cdef unsigned long bytes, rc + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("iocpsupport.CompletionPort.getEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XDECREF(__pyx_v_ignored); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":233 + * return (rc, bytes, key, obj) + * + * def postEvent(self, unsigned long bytes, size_t key, obj): # <<<<<<<<<<<<<< + * cdef myOVERLAPPED *ov + * cdef unsigned long rc + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_7postEvent(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_7postEvent(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + unsigned long __pyx_v_bytes; + size_t __pyx_v_key; + PyObject *__pyx_v_obj = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("postEvent (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_bytes,&__pyx_n_s_key,&__pyx_n_s_obj,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_bytes)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_key)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("postEvent", 1, 3, 3, 1); __PYX_ERR(0, 233, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_obj)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("postEvent", 1, 3, 3, 2); __PYX_ERR(0, 233, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "postEvent") < 0)) __PYX_ERR(0, 233, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_bytes = __Pyx_PyInt_As_unsigned_long(values[0]); if (unlikely((__pyx_v_bytes == (unsigned long)-1) && PyErr_Occurred())) __PYX_ERR(0, 233, __pyx_L3_error) + __pyx_v_key = __Pyx_PyInt_As_size_t(values[1]); if (unlikely((__pyx_v_key == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 233, __pyx_L3_error) + __pyx_v_obj = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("postEvent", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 233, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.CompletionPort.postEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_14CompletionPort_6postEvent(((struct __pyx_obj_11iocpsupport_CompletionPort *)__pyx_v_self), __pyx_v_bytes, __pyx_v_key, __pyx_v_obj); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_6postEvent(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self, unsigned long __pyx_v_bytes, size_t __pyx_v_key, PyObject *__pyx_v_obj) { + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_ov; + unsigned long __pyx_v_rc; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_t_3; + __Pyx_RefNannySetupContext("postEvent", 0); + + /* "iocpsupport.pyx":237 + * cdef unsigned long rc + * + * if obj is not None: # <<<<<<<<<<<<<< + * ov = makeOV(obj) + * else: + */ + __pyx_t_1 = (__pyx_v_obj != Py_None); + __pyx_t_2 = (__pyx_t_1 != 0); + if (__pyx_t_2) { + + /* "iocpsupport.pyx":238 + * + * if obj is not None: + * ov = makeOV(obj) # <<<<<<<<<<<<<< + * else: + * ov = NULL + */ + __pyx_t_3 = __pyx_f_11iocpsupport_makeOV(__pyx_v_obj, NULL); if (unlikely(__pyx_t_3 == ((struct __pyx_t_11iocpsupport_myOVERLAPPED *)NULL))) __PYX_ERR(0, 238, __pyx_L1_error) + __pyx_v_ov = __pyx_t_3; + + /* "iocpsupport.pyx":237 + * cdef unsigned long rc + * + * if obj is not None: # <<<<<<<<<<<<<< + * ov = makeOV(obj) + * else: + */ + goto __pyx_L3; + } + + /* "iocpsupport.pyx":240 + * ov = makeOV(obj) + * else: + * ov = NULL # <<<<<<<<<<<<<< + * + * rc = PostQueuedCompletionStatus(self.port, bytes, key, ov) + */ + /*else*/ { + __pyx_v_ov = NULL; + } + __pyx_L3:; + + /* "iocpsupport.pyx":242 + * ov = NULL + * + * rc = PostQueuedCompletionStatus(self.port, bytes, key, ov) # <<<<<<<<<<<<<< + * if not rc: + * if ov: + */ + __pyx_v_rc = PostQueuedCompletionStatus(__pyx_v_self->port, __pyx_v_bytes, __pyx_v_key, ((OVERLAPPED *)__pyx_v_ov)); + + /* "iocpsupport.pyx":243 + * + * rc = PostQueuedCompletionStatus(self.port, bytes, key, ov) + * if not rc: # <<<<<<<<<<<<<< + * if ov: + * unmakeOV(ov) + */ + __pyx_t_2 = ((!(__pyx_v_rc != 0)) != 0); + if (__pyx_t_2) { + + /* "iocpsupport.pyx":244 + * rc = PostQueuedCompletionStatus(self.port, bytes, key, ov) + * if not rc: + * if ov: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * raise_error(0, 'PostQueuedCompletionStatus') + */ + __pyx_t_2 = (__pyx_v_ov != 0); + if (__pyx_t_2) { + + /* "iocpsupport.pyx":245 + * if not rc: + * if ov: + * unmakeOV(ov) # <<<<<<<<<<<<<< + * raise_error(0, 'PostQueuedCompletionStatus') + * + */ + __pyx_f_11iocpsupport_unmakeOV(__pyx_v_ov); + + /* "iocpsupport.pyx":244 + * rc = PostQueuedCompletionStatus(self.port, bytes, key, ov) + * if not rc: + * if ov: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * raise_error(0, 'PostQueuedCompletionStatus') + */ + } + + /* "iocpsupport.pyx":246 + * if ov: + * unmakeOV(ov) + * raise_error(0, 'PostQueuedCompletionStatus') # <<<<<<<<<<<<<< + * + * def __del__(self): + */ + __pyx_f_11iocpsupport_raise_error(0, __pyx_n_s_PostQueuedCompletionStatus); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 246, __pyx_L1_error) + + /* "iocpsupport.pyx":243 + * + * rc = PostQueuedCompletionStatus(self.port, bytes, key, ov) + * if not rc: # <<<<<<<<<<<<<< + * if ov: + * unmakeOV(ov) + */ + } + + /* "iocpsupport.pyx":233 + * return (rc, bytes, key, obj) + * + * def postEvent(self, unsigned long bytes, size_t key, obj): # <<<<<<<<<<<<<< + * cdef myOVERLAPPED *ov + * cdef unsigned long rc + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("iocpsupport.CompletionPort.postEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":248 + * raise_error(0, 'PostQueuedCompletionStatus') + * + * def __del__(self): # <<<<<<<<<<<<<< + * CloseHandle(self.port) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_9__del__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_9__del__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__del__ (wrapper)", 0); + __pyx_r = __pyx_pf_11iocpsupport_14CompletionPort_8__del__(((struct __pyx_obj_11iocpsupport_CompletionPort *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_8__del__(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__del__", 0); + + /* "iocpsupport.pyx":249 + * + * def __del__(self): + * CloseHandle(self.port) # <<<<<<<<<<<<<< + * + * def makesockaddr(object buff): + */ + (void)(CloseHandle(__pyx_v_self->port)); + + /* "iocpsupport.pyx":248 + * raise_error(0, 'PostQueuedCompletionStatus') + * + * def __del__(self): # <<<<<<<<<<<<<< + * CloseHandle(self.port) + * + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef bint use_setstate + * state = (self.port,) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_11__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_11__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_11iocpsupport_14CompletionPort_10__reduce_cython__(((struct __pyx_obj_11iocpsupport_CompletionPort *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_10__reduce_cython__(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self) { + int __pyx_v_use_setstate; + PyObject *__pyx_v_state = NULL; + PyObject *__pyx_v__dict = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * cdef bint use_setstate + * state = (self.port,) # <<<<<<<<<<<<<< + * _dict = getattr(self, '__dict__', None) + * if _dict is not None: + */ + __pyx_t_1 = __Pyx_PyInt_FromSize_t(__pyx_v_self->port); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 3, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 3, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_v_state = __pyx_t_2; + __pyx_t_2 = 0; + + /* "(tree fragment)":4 + * cdef bint use_setstate + * state = (self.port,) + * _dict = getattr(self, '__dict__', None) # <<<<<<<<<<<<<< + * if _dict is not None: + * state += (_dict,) + */ + __pyx_t_2 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_n_s_dict, Py_None); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v__dict = __pyx_t_2; + __pyx_t_2 = 0; + + /* "(tree fragment)":5 + * state = (self.port,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True + */ + __pyx_t_3 = (__pyx_v__dict != Py_None); + __pyx_t_4 = (__pyx_t_3 != 0); + if (__pyx_t_4) { + + /* "(tree fragment)":6 + * _dict = getattr(self, '__dict__', None) + * if _dict is not None: + * state += (_dict,) # <<<<<<<<<<<<<< + * use_setstate = True + * else: + */ + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_v__dict); + __Pyx_GIVEREF(__pyx_v__dict); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v__dict); + __pyx_t_1 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF_SET(__pyx_v_state, __pyx_t_1); + __pyx_t_1 = 0; + + /* "(tree fragment)":7 + * if _dict is not None: + * state += (_dict,) + * use_setstate = True # <<<<<<<<<<<<<< + * else: + * use_setstate = False + */ + __pyx_v_use_setstate = 1; + + /* "(tree fragment)":5 + * state = (self.port,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True + */ + goto __pyx_L3; + } + + /* "(tree fragment)":9 + * use_setstate = True + * else: + * use_setstate = False # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, None), state + */ + /*else*/ { + __pyx_v_use_setstate = 0; + } + __pyx_L3:; + + /* "(tree fragment)":10 + * else: + * use_setstate = False + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, None), state + * else: + */ + __pyx_t_4 = (__pyx_v_use_setstate != 0); + if (__pyx_t_4) { + + /* "(tree fragment)":11 + * use_setstate = False + * if use_setstate: + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, None), state # <<<<<<<<<<<<<< + * else: + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, state) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_pyx_unpickle_CompletionPort); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_INCREF(__pyx_int_151082335); + __Pyx_GIVEREF(__pyx_int_151082335); + PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_int_151082335); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + PyTuple_SET_ITEM(__pyx_t_2, 2, Py_None); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_2); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state); + __pyx_t_1 = 0; + __pyx_t_2 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "(tree fragment)":10 + * else: + * use_setstate = False + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, None), state + * else: + */ + } + + /* "(tree fragment)":13 + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, None), state + * else: + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, state) # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_CompletionPort__set_state(self, __pyx_state) + */ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_pyx_unpickle_CompletionPort); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_2 = PyTuple_New(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_INCREF(__pyx_int_151082335); + __Pyx_GIVEREF(__pyx_int_151082335); + PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_int_151082335); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + PyTuple_SET_ITEM(__pyx_t_2, 2, __pyx_v_state); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_2); + __pyx_t_5 = 0; + __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + } + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef bint use_setstate + * state = (self.port,) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("iocpsupport.CompletionPort.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_state); + __Pyx_XDECREF(__pyx_v__dict); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":14 + * else: + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_CompletionPort__set_state(self, __pyx_state) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_13__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_11iocpsupport_14CompletionPort_13__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_11iocpsupport_14CompletionPort_12__setstate_cython__(((struct __pyx_obj_11iocpsupport_CompletionPort *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_14CompletionPort_12__setstate_cython__(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":15 + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, state) + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_CompletionPort__set_state(self, __pyx_state) # <<<<<<<<<<<<<< + */ + if (!(likely(PyTuple_CheckExact(__pyx_v___pyx_state))||((__pyx_v___pyx_state) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "tuple", Py_TYPE(__pyx_v___pyx_state)->tp_name), 0))) __PYX_ERR(1, 15, __pyx_L1_error) + __pyx_t_1 = __pyx_f_11iocpsupport___pyx_unpickle_CompletionPort__set_state(__pyx_v_self, ((PyObject*)__pyx_v___pyx_state)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "(tree fragment)":14 + * else: + * return __pyx_unpickle_CompletionPort, (type(self), 0x901555f, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_CompletionPort__set_state(self, __pyx_state) + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("iocpsupport.CompletionPort.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":251 + * CloseHandle(self.port) + * + * def makesockaddr(object buff): # <<<<<<<<<<<<<< + * cdef void *mem_buffer + * cdef Py_ssize_t size + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_1makesockaddr(PyObject *__pyx_self, PyObject *__pyx_v_buff); /*proto*/ +static PyMethodDef __pyx_mdef_11iocpsupport_1makesockaddr = {"makesockaddr", (PyCFunction)__pyx_pw_11iocpsupport_1makesockaddr, METH_O, 0}; +static PyObject *__pyx_pw_11iocpsupport_1makesockaddr(PyObject *__pyx_self, PyObject *__pyx_v_buff) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("makesockaddr (wrapper)", 0); + __pyx_r = __pyx_pf_11iocpsupport_makesockaddr(__pyx_self, ((PyObject *)__pyx_v_buff)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_makesockaddr(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_buff) { + void *__pyx_v_mem_buffer; + Py_ssize_t __pyx_v_size; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("makesockaddr", 0); + + /* "iocpsupport.pyx":255 + * cdef Py_ssize_t size + * + * PyObject_AsReadBuffer(buff, &mem_buffer, &size) # <<<<<<<<<<<<<< + * # XXX: this should really return the address family as well + * return _makesockaddr(mem_buffer, size) + */ + __pyx_t_1 = PyObject_AsReadBuffer(__pyx_v_buff, (&__pyx_v_mem_buffer), (&__pyx_v_size)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 255, __pyx_L1_error) + + /* "iocpsupport.pyx":257 + * PyObject_AsReadBuffer(buff, &mem_buffer, &size) + * # XXX: this should really return the address family as well + * return _makesockaddr(mem_buffer, size) # <<<<<<<<<<<<<< + * + * cdef object _makesockaddr(sockaddr *addr, Py_ssize_t len): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_f_11iocpsupport__makesockaddr(((struct sockaddr *)__pyx_v_mem_buffer), __pyx_v_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 257, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "iocpsupport.pyx":251 + * CloseHandle(self.port) + * + * def makesockaddr(object buff): # <<<<<<<<<<<<<< + * cdef void *mem_buffer + * cdef Py_ssize_t size + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("iocpsupport.makesockaddr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":259 + * return _makesockaddr(mem_buffer, size) + * + * cdef object _makesockaddr(sockaddr *addr, Py_ssize_t len): # <<<<<<<<<<<<<< + * cdef sockaddr_in *sin + * cdef sockaddr_in6 *sin6 + */ + +static PyObject *__pyx_f_11iocpsupport__makesockaddr(struct sockaddr *__pyx_v_addr, Py_ssize_t __pyx_v_len) { + struct sockaddr_in *__pyx_v_sin; + struct sockaddr_in6 *__pyx_v_sin6; + WCHAR __pyx_v_buff[0x100]; + __pyx_t_11iocpsupport_DWORD __pyx_v_buffWcharLen; + int __pyx_v_rc; + unsigned short __pyx_v_sa_port; + PyObject *__pyx_v_host = NULL; + PyObject *__pyx_v_port = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + size_t __pyx_t_1; + size_t __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *(*__pyx_t_8)(PyObject *); + __Pyx_RefNannySetupContext("_makesockaddr", 0); + + /* "iocpsupport.pyx":263 + * cdef sockaddr_in6 *sin6 + * cdef WCHAR buff[256] + * cdef DWORD buffWcharLen = (sizeof(buff) / sizeof(WCHAR)) - 1 # <<<<<<<<<<<<<< + * cdef int rc + * if not len: + */ + __pyx_t_1 = (sizeof(__pyx_v_buff)); + __pyx_t_2 = (sizeof(WCHAR)); + if (unlikely(__pyx_t_2 == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(0, 263, __pyx_L1_error) + } + __pyx_v_buffWcharLen = (((__pyx_t_11iocpsupport_DWORD)(__pyx_t_1 / __pyx_t_2)) - 1); + + /* "iocpsupport.pyx":265 + * cdef DWORD buffWcharLen = (sizeof(buff) / sizeof(WCHAR)) - 1 + * cdef int rc + * if not len: # <<<<<<<<<<<<<< + * return None + * + */ + __pyx_t_3 = ((!(__pyx_v_len != 0)) != 0); + if (__pyx_t_3) { + + /* "iocpsupport.pyx":266 + * cdef int rc + * if not len: + * return None # <<<<<<<<<<<<<< + * + * memset(buff, 0, sizeof(buff)) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "iocpsupport.pyx":265 + * cdef DWORD buffWcharLen = (sizeof(buff) / sizeof(WCHAR)) - 1 + * cdef int rc + * if not len: # <<<<<<<<<<<<<< + * return None + * + */ + } + + /* "iocpsupport.pyx":268 + * return None + * + * memset(buff, 0, sizeof(buff)) # <<<<<<<<<<<<<< + * + * if addr.sa_family == AF_INET: + */ + (void)(memset(__pyx_v_buff, 0, (sizeof(__pyx_v_buff)))); + + /* "iocpsupport.pyx":270 + * memset(buff, 0, sizeof(buff)) + * + * if addr.sa_family == AF_INET: # <<<<<<<<<<<<<< + * sin = addr + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in), NULL, buff, + */ + switch (__pyx_v_addr->sa_family) { + case AF_INET: + + /* "iocpsupport.pyx":271 + * + * if addr.sa_family == AF_INET: + * sin = addr # <<<<<<<<<<<<<< + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in), NULL, buff, + * &buffWcharLen) + */ + __pyx_v_sin = ((struct sockaddr_in *)__pyx_v_addr); + + /* "iocpsupport.pyx":272 + * if addr.sa_family == AF_INET: + * sin = addr + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in), NULL, buff, # <<<<<<<<<<<<<< + * &buffWcharLen) + * if rc == SOCKET_ERROR: + */ + __pyx_v_rc = WSAAddressToStringW(__pyx_v_addr, (sizeof(struct sockaddr_in)), NULL, __pyx_v_buff, (&__pyx_v_buffWcharLen)); + + /* "iocpsupport.pyx":274 + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in), NULL, buff, + * &buffWcharLen) + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise_error(0, 'WSAAddressToStringW') + * sa_port = ntohs(sin.sin_port) + */ + __pyx_t_3 = ((__pyx_v_rc == SOCKET_ERROR) != 0); + if (__pyx_t_3) { + + /* "iocpsupport.pyx":275 + * &buffWcharLen) + * if rc == SOCKET_ERROR: + * raise_error(0, 'WSAAddressToStringW') # <<<<<<<<<<<<<< + * sa_port = ntohs(sin.sin_port) + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + */ + __pyx_f_11iocpsupport_raise_error(0, __pyx_n_s_WSAAddressToStringW); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 275, __pyx_L1_error) + + /* "iocpsupport.pyx":274 + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in), NULL, buff, + * &buffWcharLen) + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise_error(0, 'WSAAddressToStringW') + * sa_port = ntohs(sin.sin_port) + */ + } + + /* "iocpsupport.pyx":276 + * if rc == SOCKET_ERROR: + * raise_error(0, 'WSAAddressToStringW') + * sa_port = ntohs(sin.sin_port) # <<<<<<<<<<<<<< + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + * host, port = host.rsplit(u':', 1) + */ + __pyx_v_sa_port = ntohs(__pyx_v_sin->sin_port); + + /* "iocpsupport.pyx":277 + * raise_error(0, 'WSAAddressToStringW') + * sa_port = ntohs(sin.sin_port) + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) # <<<<<<<<<<<<<< + * host, port = host.rsplit(u':', 1) + * port = int(port) + */ + __pyx_t_4 = PyUnicode_FromWideChar(__pyx_v_buff, wcslen(__pyx_v_buff)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 277, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_host = __pyx_t_4; + __pyx_t_4 = 0; + + /* "iocpsupport.pyx":278 + * sa_port = ntohs(sin.sin_port) + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + * host, port = host.rsplit(u':', 1) # <<<<<<<<<<<<<< + * port = int(port) + * assert port == sa_port + */ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_host, __pyx_n_s_rsplit); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 278, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 278, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { + PyObject* sequence = __pyx_t_5; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 278, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); + __pyx_t_6 = PyTuple_GET_ITEM(sequence, 1); + } else { + __pyx_t_4 = PyList_GET_ITEM(sequence, 0); + __pyx_t_6 = PyList_GET_ITEM(sequence, 1); + } + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx_t_6); + #else + __pyx_t_4 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 278, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 278, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + #endif + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_7 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 278, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; + index = 0; __pyx_t_4 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_4)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + index = 1; __pyx_t_6 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_6)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_7), 2) < 0) __PYX_ERR(0, 278, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L6_unpacking_done; + __pyx_L5_unpacking_failed:; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 278, __pyx_L1_error) + __pyx_L6_unpacking_done:; + } + __Pyx_DECREF_SET(__pyx_v_host, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_v_port = __pyx_t_6; + __pyx_t_6 = 0; + + /* "iocpsupport.pyx":279 + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + * host, port = host.rsplit(u':', 1) + * port = int(port) # <<<<<<<<<<<<<< + * assert port == sa_port + * + */ + __pyx_t_5 = __Pyx_PyNumber_Int(__pyx_v_port); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 279, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF_SET(__pyx_v_port, __pyx_t_5); + __pyx_t_5 = 0; + + /* "iocpsupport.pyx":280 + * host, port = host.rsplit(u':', 1) + * port = int(port) + * assert port == sa_port # <<<<<<<<<<<<<< + * + * return host, port + */ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(!Py_OptimizeFlag)) { + __pyx_t_5 = __Pyx_PyInt_From_unsigned_short(__pyx_v_sa_port); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyObject_RichCompare(__pyx_v_port, __pyx_t_5, Py_EQ); __Pyx_XGOTREF(__pyx_t_6); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_6); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_3)) { + PyErr_SetNone(PyExc_AssertionError); + __PYX_ERR(0, 280, __pyx_L1_error) + } + } + #endif + + /* "iocpsupport.pyx":282 + * assert port == sa_port + * + * return host, port # <<<<<<<<<<<<<< + * elif addr.sa_family == AF_INET6: + * sin6 = addr + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 282, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_INCREF(__pyx_v_host); + __Pyx_GIVEREF(__pyx_v_host); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_host); + __Pyx_INCREF(__pyx_v_port); + __Pyx_GIVEREF(__pyx_v_port); + PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_v_port); + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "iocpsupport.pyx":270 + * memset(buff, 0, sizeof(buff)) + * + * if addr.sa_family == AF_INET: # <<<<<<<<<<<<<< + * sin = addr + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in), NULL, buff, + */ + break; + + /* "iocpsupport.pyx":283 + * + * return host, port + * elif addr.sa_family == AF_INET6: # <<<<<<<<<<<<<< + * sin6 = addr + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in6), NULL, buff, + */ + case AF_INET6: + + /* "iocpsupport.pyx":284 + * return host, port + * elif addr.sa_family == AF_INET6: + * sin6 = addr # <<<<<<<<<<<<<< + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in6), NULL, buff, + * &buffWcharLen) + */ + __pyx_v_sin6 = ((struct sockaddr_in6 *)__pyx_v_addr); + + /* "iocpsupport.pyx":285 + * elif addr.sa_family == AF_INET6: + * sin6 = addr + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in6), NULL, buff, # <<<<<<<<<<<<<< + * &buffWcharLen) + * if rc == SOCKET_ERROR: + */ + __pyx_v_rc = WSAAddressToStringW(__pyx_v_addr, (sizeof(struct sockaddr_in6)), NULL, __pyx_v_buff, (&__pyx_v_buffWcharLen)); + + /* "iocpsupport.pyx":287 + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in6), NULL, buff, + * &buffWcharLen) + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise_error(0, 'WSAAddressToStringW') + * sa_port = ntohs(sin6.sin6_port) + */ + __pyx_t_3 = ((__pyx_v_rc == SOCKET_ERROR) != 0); + if (__pyx_t_3) { + + /* "iocpsupport.pyx":288 + * &buffWcharLen) + * if rc == SOCKET_ERROR: + * raise_error(0, 'WSAAddressToStringW') # <<<<<<<<<<<<<< + * sa_port = ntohs(sin6.sin6_port) + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + */ + __pyx_f_11iocpsupport_raise_error(0, __pyx_n_s_WSAAddressToStringW); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 288, __pyx_L1_error) + + /* "iocpsupport.pyx":287 + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in6), NULL, buff, + * &buffWcharLen) + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise_error(0, 'WSAAddressToStringW') + * sa_port = ntohs(sin6.sin6_port) + */ + } + + /* "iocpsupport.pyx":289 + * if rc == SOCKET_ERROR: + * raise_error(0, 'WSAAddressToStringW') + * sa_port = ntohs(sin6.sin6_port) # <<<<<<<<<<<<<< + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + * host, port = host.rsplit(u':', 1) + */ + __pyx_v_sa_port = ntohs(__pyx_v_sin6->sin6_port); + + /* "iocpsupport.pyx":290 + * raise_error(0, 'WSAAddressToStringW') + * sa_port = ntohs(sin6.sin6_port) + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) # <<<<<<<<<<<<<< + * host, port = host.rsplit(u':', 1) + * port = int(port) + */ + __pyx_t_6 = PyUnicode_FromWideChar(__pyx_v_buff, wcslen(__pyx_v_buff)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 290, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_v_host = __pyx_t_6; + __pyx_t_6 = 0; + + /* "iocpsupport.pyx":291 + * sa_port = ntohs(sin6.sin6_port) + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + * host, port = host.rsplit(u':', 1) # <<<<<<<<<<<<<< + * port = int(port) + * assert host[0] == '[' + */ + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_host, __pyx_n_s_rsplit); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { + PyObject* sequence = __pyx_t_5; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 291, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + } else { + __pyx_t_6 = PyList_GET_ITEM(sequence, 0); + __pyx_t_4 = PyList_GET_ITEM(sequence, 1); + } + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(__pyx_t_4); + #else + __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_4 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_7 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; + index = 0; __pyx_t_6 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_6)) goto __pyx_L8_unpacking_failed; + __Pyx_GOTREF(__pyx_t_6); + index = 1; __pyx_t_4 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_4)) goto __pyx_L8_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_7), 2) < 0) __PYX_ERR(0, 291, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L9_unpacking_done; + __pyx_L8_unpacking_failed:; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 291, __pyx_L1_error) + __pyx_L9_unpacking_done:; + } + __Pyx_DECREF_SET(__pyx_v_host, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_v_port = __pyx_t_4; + __pyx_t_4 = 0; + + /* "iocpsupport.pyx":292 + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + * host, port = host.rsplit(u':', 1) + * port = int(port) # <<<<<<<<<<<<<< + * assert host[0] == '[' + * assert host[-1] == ']' + */ + __pyx_t_5 = __Pyx_PyNumber_Int(__pyx_v_port); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 292, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF_SET(__pyx_v_port, __pyx_t_5); + __pyx_t_5 = 0; + + /* "iocpsupport.pyx":293 + * host, port = host.rsplit(u':', 1) + * port = int(port) + * assert host[0] == '[' # <<<<<<<<<<<<<< + * assert host[-1] == ']' + * assert port == sa_port + */ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(!Py_OptimizeFlag)) { + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_host, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 293, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = (__Pyx_PyString_Equals(__pyx_t_5, __pyx_kp_s__4, Py_EQ)); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 293, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) { + PyErr_SetNone(PyExc_AssertionError); + __PYX_ERR(0, 293, __pyx_L1_error) + } + } + #endif + + /* "iocpsupport.pyx":294 + * port = int(port) + * assert host[0] == '[' + * assert host[-1] == ']' # <<<<<<<<<<<<<< + * assert port == sa_port + * + */ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(!Py_OptimizeFlag)) { + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_host, -1L, long, 1, __Pyx_PyInt_From_long, 0, 1, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = (__Pyx_PyString_Equals(__pyx_t_5, __pyx_kp_s__5, Py_EQ)); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) { + PyErr_SetNone(PyExc_AssertionError); + __PYX_ERR(0, 294, __pyx_L1_error) + } + } + #endif + + /* "iocpsupport.pyx":295 + * assert host[0] == '[' + * assert host[-1] == ']' + * assert port == sa_port # <<<<<<<<<<<<<< + * + * return host[1:-1], port + */ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(!Py_OptimizeFlag)) { + __pyx_t_5 = __Pyx_PyInt_From_unsigned_short(__pyx_v_sa_port); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 295, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = PyObject_RichCompare(__pyx_v_port, __pyx_t_5, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 295, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 295, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_3)) { + PyErr_SetNone(PyExc_AssertionError); + __PYX_ERR(0, 295, __pyx_L1_error) + } + } + #endif + + /* "iocpsupport.pyx":297 + * assert port == sa_port + * + * return host[1:-1], port # <<<<<<<<<<<<<< + * else: + * raise_error(0, "unsupported address family %d" % (addr.sa_family)) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __Pyx_PyObject_GetSlice(__pyx_v_host, 1, -1L, NULL, NULL, &__pyx_slice__6, 1, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 297, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 297, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); + __Pyx_INCREF(__pyx_v_port); + __Pyx_GIVEREF(__pyx_v_port); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_port); + __pyx_t_4 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "iocpsupport.pyx":283 + * + * return host, port + * elif addr.sa_family == AF_INET6: # <<<<<<<<<<<<<< + * sin6 = addr + * rc = WSAAddressToStringW(addr, sizeof(sockaddr_in6), NULL, buff, + */ + break; + default: + + /* "iocpsupport.pyx":299 + * return host[1:-1], port + * else: + * raise_error(0, "unsupported address family %d" % (addr.sa_family)) # <<<<<<<<<<<<<< + * + * + */ + __pyx_t_5 = __Pyx_PyInt_From_unsigned_short(__pyx_v_addr->sa_family); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 299, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_unsupported_address_family_d, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 299, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_f_11iocpsupport_raise_error(0, __pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 299, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + break; + } + + /* "iocpsupport.pyx":259 + * return _makesockaddr(mem_buffer, size) + * + * cdef object _makesockaddr(sockaddr *addr, Py_ssize_t len): # <<<<<<<<<<<<<< + * cdef sockaddr_in *sin + * cdef sockaddr_in6 *sin6 + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("iocpsupport._makesockaddr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_host); + __Pyx_XDECREF(__pyx_v_port); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":302 + * + * + * cdef object fillinetaddr(sockaddr_in *dest, object addr): # <<<<<<<<<<<<<< + * cdef unsigned short port + * cdef WCHAR hostStr[256] # slightly larger than longest valid DNS hostname + */ + +static PyObject *__pyx_f_11iocpsupport_fillinetaddr(struct sockaddr_in *__pyx_v_dest, PyObject *__pyx_v_addr) { + unsigned short __pyx_v_port; + WCHAR __pyx_v_hostStr[0x100]; + Py_ssize_t __pyx_v_hostStrWcharLen; + int __pyx_v_addrlen; + Py_ssize_t __pyx_v_rc; + PyObject *__pyx_v_host = NULL; + int __pyx_v_parseresult; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + size_t __pyx_t_1; + size_t __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *(*__pyx_t_6)(PyObject *); + unsigned short __pyx_t_7; + int __pyx_t_8; + int __pyx_t_9; + __Pyx_RefNannySetupContext("fillinetaddr", 0); + + /* "iocpsupport.pyx":305 + * cdef unsigned short port + * cdef WCHAR hostStr[256] # slightly larger than longest valid DNS hostname + * cdef Py_ssize_t hostStrWcharLen = (sizeof(hostStr) / sizeof(WCHAR)) - 1 # <<<<<<<<<<<<<< + * cdef int addrlen = sizeof(sockaddr_in) + * cdef Py_ssize_t rc + */ + __pyx_t_1 = (sizeof(__pyx_v_hostStr)); + __pyx_t_2 = (sizeof(WCHAR)); + if (unlikely(__pyx_t_2 == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(0, 305, __pyx_L1_error) + } + __pyx_v_hostStrWcharLen = ((__pyx_t_1 / __pyx_t_2) - 1); + + /* "iocpsupport.pyx":306 + * cdef WCHAR hostStr[256] # slightly larger than longest valid DNS hostname + * cdef Py_ssize_t hostStrWcharLen = (sizeof(hostStr) / sizeof(WCHAR)) - 1 + * cdef int addrlen = sizeof(sockaddr_in) # <<<<<<<<<<<<<< + * cdef Py_ssize_t rc + * host, port = addr + */ + __pyx_v_addrlen = (sizeof(struct sockaddr_in)); + + /* "iocpsupport.pyx":308 + * cdef int addrlen = sizeof(sockaddr_in) + * cdef Py_ssize_t rc + * host, port = addr # <<<<<<<<<<<<<< + * + * if PY_MAJOR_VERSION < 3: + */ + if ((likely(PyTuple_CheckExact(__pyx_v_addr))) || (PyList_CheckExact(__pyx_v_addr))) { + PyObject* sequence = __pyx_v_addr; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 308, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + } else { + __pyx_t_3 = PyList_GET_ITEM(sequence, 0); + __pyx_t_4 = PyList_GET_ITEM(sequence, 1); + } + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + #else + __pyx_t_3 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 308, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 308, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + } else { + Py_ssize_t index = -1; + __pyx_t_5 = PyObject_GetIter(__pyx_v_addr); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 308, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = Py_TYPE(__pyx_t_5)->tp_iternext; + index = 0; __pyx_t_3 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_4 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_4)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 2) < 0) __PYX_ERR(0, 308, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 308, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + __pyx_t_7 = __Pyx_PyInt_As_unsigned_short(__pyx_t_4); if (unlikely((__pyx_t_7 == (unsigned short)-1) && PyErr_Occurred())) __PYX_ERR(0, 308, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_host = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v_port = __pyx_t_7; + + /* "iocpsupport.pyx":310 + * host, port = addr + * + * if PY_MAJOR_VERSION < 3: # <<<<<<<<<<<<<< + * if (isinstance(host, str)): + * host = unicode(host, "utf-8") + */ + __pyx_t_8 = ((PY_MAJOR_VERSION < 3) != 0); + if (__pyx_t_8) { + + /* "iocpsupport.pyx":311 + * + * if PY_MAJOR_VERSION < 3: + * if (isinstance(host, str)): # <<<<<<<<<<<<<< + * host = unicode(host, "utf-8") + * + */ + __pyx_t_8 = PyString_Check(__pyx_v_host); + __pyx_t_9 = (__pyx_t_8 != 0); + if (__pyx_t_9) { + + /* "iocpsupport.pyx":312 + * if PY_MAJOR_VERSION < 3: + * if (isinstance(host, str)): + * host = unicode(host, "utf-8") # <<<<<<<<<<<<<< + * + * memset(hostStr, 0, sizeof(hostStr)) + */ + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 312, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_host); + __Pyx_GIVEREF(__pyx_v_host); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_host); + __Pyx_INCREF(__pyx_kp_s_utf_8); + __Pyx_GIVEREF(__pyx_kp_s_utf_8); + PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_kp_s_utf_8); + __pyx_t_3 = __Pyx_PyObject_Call(((PyObject *)(&PyUnicode_Type)), __pyx_t_4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 312, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF_SET(__pyx_v_host, __pyx_t_3); + __pyx_t_3 = 0; + + /* "iocpsupport.pyx":311 + * + * if PY_MAJOR_VERSION < 3: + * if (isinstance(host, str)): # <<<<<<<<<<<<<< + * host = unicode(host, "utf-8") + * + */ + } + + /* "iocpsupport.pyx":310 + * host, port = addr + * + * if PY_MAJOR_VERSION < 3: # <<<<<<<<<<<<<< + * if (isinstance(host, str)): + * host = unicode(host, "utf-8") + */ + } + + /* "iocpsupport.pyx":314 + * host = unicode(host, "utf-8") + * + * memset(hostStr, 0, sizeof(hostStr)) # <<<<<<<<<<<<<< + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) + * if rc == -1: + */ + (void)(memset(__pyx_v_hostStr, 0, (sizeof(__pyx_v_hostStr)))); + + /* "iocpsupport.pyx":315 + * + * memset(hostStr, 0, sizeof(hostStr)) + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) # <<<<<<<<<<<<<< + * if rc == -1: + * raise ValueError, 'invalid IP address %r' % (host,) + */ + __pyx_v_rc = PyUnicode_AsWideChar(__pyx_v_host, __pyx_v_hostStr, __pyx_v_hostStrWcharLen); + + /* "iocpsupport.pyx":316 + * memset(hostStr, 0, sizeof(hostStr)) + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) + * if rc == -1: # <<<<<<<<<<<<<< + * raise ValueError, 'invalid IP address %r' % (host,) + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET, NULL, + */ + __pyx_t_9 = ((__pyx_v_rc == -1L) != 0); + if (unlikely(__pyx_t_9)) { + + /* "iocpsupport.pyx":317 + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) + * if rc == -1: + * raise ValueError, 'invalid IP address %r' % (host,) # <<<<<<<<<<<<<< + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET, NULL, + * dest, &addrlen) + */ + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 317, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_host); + __Pyx_GIVEREF(__pyx_v_host); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_host); + __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_invalid_IP_address_r, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 317, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(__pyx_builtin_ValueError, __pyx_t_4, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 317, __pyx_L1_error) + + /* "iocpsupport.pyx":316 + * memset(hostStr, 0, sizeof(hostStr)) + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) + * if rc == -1: # <<<<<<<<<<<<<< + * raise ValueError, 'invalid IP address %r' % (host,) + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET, NULL, + */ + } + + /* "iocpsupport.pyx":318 + * if rc == -1: + * raise ValueError, 'invalid IP address %r' % (host,) + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET, NULL, # <<<<<<<<<<<<<< + * dest, &addrlen) + * if parseresult == SOCKET_ERROR: + */ + __pyx_v_parseresult = WSAStringToAddressW(__pyx_v_hostStr, AF_INET, NULL, ((struct sockaddr *)__pyx_v_dest), (&__pyx_v_addrlen)); + + /* "iocpsupport.pyx":320 + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET, NULL, + * dest, &addrlen) + * if parseresult == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise ValueError, 'invalid IP address %r' % (host,) + * + */ + __pyx_t_9 = ((__pyx_v_parseresult == SOCKET_ERROR) != 0); + if (unlikely(__pyx_t_9)) { + + /* "iocpsupport.pyx":321 + * dest, &addrlen) + * if parseresult == SOCKET_ERROR: + * raise ValueError, 'invalid IP address %r' % (host,) # <<<<<<<<<<<<<< + * + * dest.sin_port = htons(port) + */ + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_host); + __Pyx_GIVEREF(__pyx_v_host); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_host); + __pyx_t_3 = __Pyx_PyString_Format(__pyx_kp_s_invalid_IP_address_r, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(__pyx_builtin_ValueError, __pyx_t_3, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 321, __pyx_L1_error) + + /* "iocpsupport.pyx":320 + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET, NULL, + * dest, &addrlen) + * if parseresult == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise ValueError, 'invalid IP address %r' % (host,) + * + */ + } + + /* "iocpsupport.pyx":323 + * raise ValueError, 'invalid IP address %r' % (host,) + * + * dest.sin_port = htons(port) # <<<<<<<<<<<<<< + * + * + */ + __pyx_v_dest->sin_port = htons(__pyx_v_port); + + /* "iocpsupport.pyx":302 + * + * + * cdef object fillinetaddr(sockaddr_in *dest, object addr): # <<<<<<<<<<<<<< + * cdef unsigned short port + * cdef WCHAR hostStr[256] # slightly larger than longest valid DNS hostname + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("iocpsupport.fillinetaddr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_host); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":326 + * + * + * cdef object fillinet6addr(sockaddr_in6 *dest, object addr): # <<<<<<<<<<<<<< + * cdef unsigned short port + * cdef unsigned long res + */ + +static PyObject *__pyx_f_11iocpsupport_fillinet6addr(struct sockaddr_in6 *__pyx_v_dest, PyObject *__pyx_v_addr) { + unsigned short __pyx_v_port; + WCHAR __pyx_v_hostStr[0x100]; + Py_ssize_t __pyx_v_hostStrWcharLen; + Py_ssize_t __pyx_v_rc; + int __pyx_v_addrlen; + PyObject *__pyx_v_host = NULL; + PyObject *__pyx_v_flow = NULL; + PyObject *__pyx_v_scope = NULL; + int __pyx_v_parseresult; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + size_t __pyx_t_1; + size_t __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *(*__pyx_t_8)(PyObject *); + unsigned short __pyx_t_9; + int __pyx_t_10; + int __pyx_t_11; + unsigned long __pyx_t_12; + __Pyx_RefNannySetupContext("fillinet6addr", 0); + + /* "iocpsupport.pyx":330 + * cdef unsigned long res + * cdef WCHAR hostStr[256] + * cdef Py_ssize_t hostStrWcharLen = (sizeof(hostStr) / sizeof(WCHAR)) - 1 # <<<<<<<<<<<<<< + * cdef Py_ssize_t rc + * cdef int addrlen = sizeof(sockaddr_in6) + */ + __pyx_t_1 = (sizeof(__pyx_v_hostStr)); + __pyx_t_2 = (sizeof(WCHAR)); + if (unlikely(__pyx_t_2 == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(0, 330, __pyx_L1_error) + } + __pyx_v_hostStrWcharLen = ((__pyx_t_1 / __pyx_t_2) - 1); + + /* "iocpsupport.pyx":332 + * cdef Py_ssize_t hostStrWcharLen = (sizeof(hostStr) / sizeof(WCHAR)) - 1 + * cdef Py_ssize_t rc + * cdef int addrlen = sizeof(sockaddr_in6) # <<<<<<<<<<<<<< + * host, port, flow, scope = addr + * host = host.split("%")[0] # remove scope ID, if any + */ + __pyx_v_addrlen = (sizeof(struct sockaddr_in6)); + + /* "iocpsupport.pyx":333 + * cdef Py_ssize_t rc + * cdef int addrlen = sizeof(sockaddr_in6) + * host, port, flow, scope = addr # <<<<<<<<<<<<<< + * host = host.split("%")[0] # remove scope ID, if any + * + */ + if ((likely(PyTuple_CheckExact(__pyx_v_addr))) || (PyList_CheckExact(__pyx_v_addr))) { + PyObject* sequence = __pyx_v_addr; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 4)) { + if (size > 4) __Pyx_RaiseTooManyValuesError(4); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 333, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + __pyx_t_5 = PyTuple_GET_ITEM(sequence, 2); + __pyx_t_6 = PyTuple_GET_ITEM(sequence, 3); + } else { + __pyx_t_3 = PyList_GET_ITEM(sequence, 0); + __pyx_t_4 = PyList_GET_ITEM(sequence, 1); + __pyx_t_5 = PyList_GET_ITEM(sequence, 2); + __pyx_t_6 = PyList_GET_ITEM(sequence, 3); + } + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(__pyx_t_6); + #else + { + Py_ssize_t i; + PyObject** temps[4] = {&__pyx_t_3,&__pyx_t_4,&__pyx_t_5,&__pyx_t_6}; + for (i=0; i < 4; i++) { + PyObject* item = PySequence_ITEM(sequence, i); if (unlikely(!item)) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_GOTREF(item); + *(temps[i]) = item; + } + } + #endif + } else { + Py_ssize_t index = -1; + PyObject** temps[4] = {&__pyx_t_3,&__pyx_t_4,&__pyx_t_5,&__pyx_t_6}; + __pyx_t_7 = PyObject_GetIter(__pyx_v_addr); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; + for (index=0; index < 4; index++) { + PyObject* item = __pyx_t_8(__pyx_t_7); if (unlikely(!item)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(item); + *(temps[index]) = item; + } + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_7), 4) < 0) __PYX_ERR(0, 333, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 333, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + __pyx_t_9 = __Pyx_PyInt_As_unsigned_short(__pyx_t_4); if (unlikely((__pyx_t_9 == (unsigned short)-1) && PyErr_Occurred())) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_host = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v_port = __pyx_t_9; + __pyx_v_flow = __pyx_t_5; + __pyx_t_5 = 0; + __pyx_v_scope = __pyx_t_6; + __pyx_t_6 = 0; + + /* "iocpsupport.pyx":334 + * cdef int addrlen = sizeof(sockaddr_in6) + * host, port, flow, scope = addr + * host = host.split("%")[0] # remove scope ID, if any # <<<<<<<<<<<<<< + * + * if PY_MAJOR_VERSION < 3: + */ + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_host, __pyx_n_s_split); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_GetItemInt(__pyx_t_5, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF_SET(__pyx_v_host, __pyx_t_6); + __pyx_t_6 = 0; + + /* "iocpsupport.pyx":336 + * host = host.split("%")[0] # remove scope ID, if any + * + * if PY_MAJOR_VERSION < 3: # <<<<<<<<<<<<<< + * if (isinstance(host, str)): + * host = unicode(host, "utf-8") + */ + __pyx_t_10 = ((PY_MAJOR_VERSION < 3) != 0); + if (__pyx_t_10) { + + /* "iocpsupport.pyx":337 + * + * if PY_MAJOR_VERSION < 3: + * if (isinstance(host, str)): # <<<<<<<<<<<<<< + * host = unicode(host, "utf-8") + * + */ + __pyx_t_10 = PyString_Check(__pyx_v_host); + __pyx_t_11 = (__pyx_t_10 != 0); + if (__pyx_t_11) { + + /* "iocpsupport.pyx":338 + * if PY_MAJOR_VERSION < 3: + * if (isinstance(host, str)): + * host = unicode(host, "utf-8") # <<<<<<<<<<<<<< + * + * memset(hostStr, 0, sizeof(hostStr)) + */ + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 338, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_INCREF(__pyx_v_host); + __Pyx_GIVEREF(__pyx_v_host); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_host); + __Pyx_INCREF(__pyx_kp_s_utf_8); + __Pyx_GIVEREF(__pyx_kp_s_utf_8); + PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_kp_s_utf_8); + __pyx_t_5 = __Pyx_PyObject_Call(((PyObject *)(&PyUnicode_Type)), __pyx_t_6, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 338, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF_SET(__pyx_v_host, __pyx_t_5); + __pyx_t_5 = 0; + + /* "iocpsupport.pyx":337 + * + * if PY_MAJOR_VERSION < 3: + * if (isinstance(host, str)): # <<<<<<<<<<<<<< + * host = unicode(host, "utf-8") + * + */ + } + + /* "iocpsupport.pyx":336 + * host = host.split("%")[0] # remove scope ID, if any + * + * if PY_MAJOR_VERSION < 3: # <<<<<<<<<<<<<< + * if (isinstance(host, str)): + * host = unicode(host, "utf-8") + */ + } + + /* "iocpsupport.pyx":340 + * host = unicode(host, "utf-8") + * + * memset(hostStr, 0, sizeof(hostStr)) # <<<<<<<<<<<<<< + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) + * if rc == -1: + */ + (void)(memset(__pyx_v_hostStr, 0, (sizeof(__pyx_v_hostStr)))); + + /* "iocpsupport.pyx":341 + * + * memset(hostStr, 0, sizeof(hostStr)) + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) # <<<<<<<<<<<<<< + * if rc == -1: + * raise ValueError, 'invalid IPv6 address %r' % (host,) + */ + __pyx_v_rc = PyUnicode_AsWideChar(__pyx_v_host, __pyx_v_hostStr, __pyx_v_hostStrWcharLen); + + /* "iocpsupport.pyx":342 + * memset(hostStr, 0, sizeof(hostStr)) + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) + * if rc == -1: # <<<<<<<<<<<<<< + * raise ValueError, 'invalid IPv6 address %r' % (host,) + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET6, NULL, + */ + __pyx_t_11 = ((__pyx_v_rc == -1L) != 0); + if (unlikely(__pyx_t_11)) { + + /* "iocpsupport.pyx":343 + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) + * if rc == -1: + * raise ValueError, 'invalid IPv6 address %r' % (host,) # <<<<<<<<<<<<<< + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET6, NULL, + * dest, &addrlen) + */ + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_v_host); + __Pyx_GIVEREF(__pyx_v_host); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_v_host); + __pyx_t_6 = __Pyx_PyString_Format(__pyx_kp_s_invalid_IPv6_address_r, __pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_Raise(__pyx_builtin_ValueError, __pyx_t_6, 0, 0); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __PYX_ERR(0, 343, __pyx_L1_error) + + /* "iocpsupport.pyx":342 + * memset(hostStr, 0, sizeof(hostStr)) + * rc = PyUnicode_AsWideChar(host, hostStr, hostStrWcharLen) + * if rc == -1: # <<<<<<<<<<<<<< + * raise ValueError, 'invalid IPv6 address %r' % (host,) + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET6, NULL, + */ + } + + /* "iocpsupport.pyx":344 + * if rc == -1: + * raise ValueError, 'invalid IPv6 address %r' % (host,) + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET6, NULL, # <<<<<<<<<<<<<< + * dest, &addrlen) + * if parseresult == SOCKET_ERROR: + */ + __pyx_v_parseresult = WSAStringToAddressW(__pyx_v_hostStr, AF_INET6, NULL, ((struct sockaddr *)__pyx_v_dest), (&__pyx_v_addrlen)); + + /* "iocpsupport.pyx":346 + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET6, NULL, + * dest, &addrlen) + * if parseresult == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise ValueError, 'invalid IPv6 address %r' % (host,) + * if parseresult != 0: + */ + __pyx_t_11 = ((__pyx_v_parseresult == SOCKET_ERROR) != 0); + if (unlikely(__pyx_t_11)) { + + /* "iocpsupport.pyx":347 + * dest, &addrlen) + * if parseresult == SOCKET_ERROR: + * raise ValueError, 'invalid IPv6 address %r' % (host,) # <<<<<<<<<<<<<< + * if parseresult != 0: + * raise RuntimeError, 'undefined error occurred during address parsing' + */ + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_INCREF(__pyx_v_host); + __Pyx_GIVEREF(__pyx_v_host); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_host); + __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_invalid_IPv6_address_r, __pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_Raise(__pyx_builtin_ValueError, __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(0, 347, __pyx_L1_error) + + /* "iocpsupport.pyx":346 + * cdef int parseresult = WSAStringToAddressW(hostStr, AF_INET6, NULL, + * dest, &addrlen) + * if parseresult == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise ValueError, 'invalid IPv6 address %r' % (host,) + * if parseresult != 0: + */ + } + + /* "iocpsupport.pyx":348 + * if parseresult == SOCKET_ERROR: + * raise ValueError, 'invalid IPv6 address %r' % (host,) + * if parseresult != 0: # <<<<<<<<<<<<<< + * raise RuntimeError, 'undefined error occurred during address parsing' + * # sin6_host field was handled by WSAStringToAddress + */ + __pyx_t_11 = ((__pyx_v_parseresult != 0) != 0); + if (unlikely(__pyx_t_11)) { + + /* "iocpsupport.pyx":349 + * raise ValueError, 'invalid IPv6 address %r' % (host,) + * if parseresult != 0: + * raise RuntimeError, 'undefined error occurred during address parsing' # <<<<<<<<<<<<<< + * # sin6_host field was handled by WSAStringToAddress + * dest.sin6_port = htons(port) + */ + __Pyx_Raise(__pyx_builtin_RuntimeError, __pyx_kp_s_undefined_error_occurred_during, 0, 0); + __PYX_ERR(0, 349, __pyx_L1_error) + + /* "iocpsupport.pyx":348 + * if parseresult == SOCKET_ERROR: + * raise ValueError, 'invalid IPv6 address %r' % (host,) + * if parseresult != 0: # <<<<<<<<<<<<<< + * raise RuntimeError, 'undefined error occurred during address parsing' + * # sin6_host field was handled by WSAStringToAddress + */ + } + + /* "iocpsupport.pyx":351 + * raise RuntimeError, 'undefined error occurred during address parsing' + * # sin6_host field was handled by WSAStringToAddress + * dest.sin6_port = htons(port) # <<<<<<<<<<<<<< + * dest.sin6_flowinfo = flow + * dest.sin6_scope_id = scope + */ + __pyx_v_dest->sin6_port = htons(__pyx_v_port); + + /* "iocpsupport.pyx":352 + * # sin6_host field was handled by WSAStringToAddress + * dest.sin6_port = htons(port) + * dest.sin6_flowinfo = flow # <<<<<<<<<<<<<< + * dest.sin6_scope_id = scope + * + */ + __pyx_t_12 = __Pyx_PyInt_As_unsigned_long(__pyx_v_flow); if (unlikely((__pyx_t_12 == (unsigned long)-1) && PyErr_Occurred())) __PYX_ERR(0, 352, __pyx_L1_error) + __pyx_v_dest->sin6_flowinfo = __pyx_t_12; + + /* "iocpsupport.pyx":353 + * dest.sin6_port = htons(port) + * dest.sin6_flowinfo = flow + * dest.sin6_scope_id = scope # <<<<<<<<<<<<<< + * + * + */ + __pyx_t_12 = __Pyx_PyInt_As_unsigned_long(__pyx_v_scope); if (unlikely((__pyx_t_12 == (unsigned long)-1) && PyErr_Occurred())) __PYX_ERR(0, 353, __pyx_L1_error) + __pyx_v_dest->sin6_scope_id = __pyx_t_12; + + /* "iocpsupport.pyx":326 + * + * + * cdef object fillinet6addr(sockaddr_in6 *dest, object addr): # <<<<<<<<<<<<<< + * cdef unsigned short port + * cdef unsigned long res + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("iocpsupport.fillinet6addr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_host); + __Pyx_XDECREF(__pyx_v_flow); + __Pyx_XDECREF(__pyx_v_scope); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":356 + * + * + * def maxAddrLen(long s): # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int size, rc + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_3maxAddrLen(PyObject *__pyx_self, PyObject *__pyx_arg_s); /*proto*/ +static PyMethodDef __pyx_mdef_11iocpsupport_3maxAddrLen = {"maxAddrLen", (PyCFunction)__pyx_pw_11iocpsupport_3maxAddrLen, METH_O, 0}; +static PyObject *__pyx_pw_11iocpsupport_3maxAddrLen(PyObject *__pyx_self, PyObject *__pyx_arg_s) { + long __pyx_v_s; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("maxAddrLen (wrapper)", 0); + assert(__pyx_arg_s); { + __pyx_v_s = __Pyx_PyInt_As_long(__pyx_arg_s); if (unlikely((__pyx_v_s == (long)-1) && PyErr_Occurred())) __PYX_ERR(0, 356, __pyx_L3_error) + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.maxAddrLen", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_2maxAddrLen(__pyx_self, ((long)__pyx_v_s)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_2maxAddrLen(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s) { + WSAPROTOCOL_INFO __pyx_v_wsa_pi; + int __pyx_v_size; + int __pyx_v_rc; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("maxAddrLen", 0); + + /* "iocpsupport.pyx":360 + * cdef int size, rc + * + * size = sizeof(wsa_pi) # <<<<<<<<<<<<<< + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) + * if rc == SOCKET_ERROR: + */ + __pyx_v_size = (sizeof(__pyx_v_wsa_pi)); + + /* "iocpsupport.pyx":361 + * + * size = sizeof(wsa_pi) + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) # <<<<<<<<<<<<<< + * if rc == SOCKET_ERROR: + * raise_error(WSAGetLastError(), 'getsockopt') + */ + __pyx_v_rc = getsockopt(__pyx_v_s, SOL_SOCKET, SO_PROTOCOL_INFO, ((char *)(&__pyx_v_wsa_pi)), (&__pyx_v_size)); + + /* "iocpsupport.pyx":362 + * size = sizeof(wsa_pi) + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise_error(WSAGetLastError(), 'getsockopt') + * return wsa_pi.iMaxSockAddr + */ + __pyx_t_1 = ((__pyx_v_rc == SOCKET_ERROR) != 0); + if (__pyx_t_1) { + + /* "iocpsupport.pyx":363 + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) + * if rc == SOCKET_ERROR: + * raise_error(WSAGetLastError(), 'getsockopt') # <<<<<<<<<<<<<< + * return wsa_pi.iMaxSockAddr + * + */ + __pyx_f_11iocpsupport_raise_error(WSAGetLastError(), __pyx_n_s_getsockopt); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 363, __pyx_L1_error) + + /* "iocpsupport.pyx":362 + * size = sizeof(wsa_pi) + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise_error(WSAGetLastError(), 'getsockopt') + * return wsa_pi.iMaxSockAddr + */ + } + + /* "iocpsupport.pyx":364 + * if rc == SOCKET_ERROR: + * raise_error(WSAGetLastError(), 'getsockopt') + * return wsa_pi.iMaxSockAddr # <<<<<<<<<<<<<< + * + * cdef int getAddrFamily(SOCKET s) except *: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_v_wsa_pi.iMaxSockAddr); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 364, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "iocpsupport.pyx":356 + * + * + * def maxAddrLen(long s): # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int size, rc + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("iocpsupport.maxAddrLen", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "iocpsupport.pyx":366 + * return wsa_pi.iMaxSockAddr + * + * cdef int getAddrFamily(SOCKET s) except *: # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int size, rc + */ + +static int __pyx_f_11iocpsupport_getAddrFamily(__pyx_t_11iocpsupport_SOCKET __pyx_v_s) { + WSAPROTOCOL_INFO __pyx_v_wsa_pi; + int __pyx_v_size; + int __pyx_v_rc; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + __Pyx_RefNannySetupContext("getAddrFamily", 0); + + /* "iocpsupport.pyx":370 + * cdef int size, rc + * + * size = sizeof(wsa_pi) # <<<<<<<<<<<<<< + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) + * if rc == SOCKET_ERROR: + */ + __pyx_v_size = (sizeof(__pyx_v_wsa_pi)); + + /* "iocpsupport.pyx":371 + * + * size = sizeof(wsa_pi) + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) # <<<<<<<<<<<<<< + * if rc == SOCKET_ERROR: + * raise_error(WSAGetLastError(), 'getsockopt') + */ + __pyx_v_rc = getsockopt(__pyx_v_s, SOL_SOCKET, SO_PROTOCOL_INFO, ((char *)(&__pyx_v_wsa_pi)), (&__pyx_v_size)); + + /* "iocpsupport.pyx":372 + * size = sizeof(wsa_pi) + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise_error(WSAGetLastError(), 'getsockopt') + * return wsa_pi.iAddressFamily + */ + __pyx_t_1 = ((__pyx_v_rc == SOCKET_ERROR) != 0); + if (__pyx_t_1) { + + /* "iocpsupport.pyx":373 + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) + * if rc == SOCKET_ERROR: + * raise_error(WSAGetLastError(), 'getsockopt') # <<<<<<<<<<<<<< + * return wsa_pi.iAddressFamily + * + */ + __pyx_f_11iocpsupport_raise_error(WSAGetLastError(), __pyx_n_s_getsockopt); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 373, __pyx_L1_error) + + /* "iocpsupport.pyx":372 + * size = sizeof(wsa_pi) + * rc = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO, &wsa_pi, &size) + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * raise_error(WSAGetLastError(), 'getsockopt') + * return wsa_pi.iAddressFamily + */ + } + + /* "iocpsupport.pyx":374 + * if rc == SOCKET_ERROR: + * raise_error(WSAGetLastError(), 'getsockopt') + * return wsa_pi.iAddressFamily # <<<<<<<<<<<<<< + * + * import socket # for WSAStartup + */ + __pyx_r = __pyx_v_wsa_pi.iAddressFamily; + goto __pyx_L0; + + /* "iocpsupport.pyx":366 + * return wsa_pi.iMaxSockAddr + * + * cdef int getAddrFamily(SOCKET s) except *: # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int size, rc + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("iocpsupport.getAddrFamily", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":5 + * + * + * def accept(long listening, long accepting, object buff, object obj): # <<<<<<<<<<<<<< + * """ + * CAUTION: unlike system AcceptEx(), this function returns 0 on success + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_5accept(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_11iocpsupport_4accept[] = "\n CAUTION: unlike system AcceptEx(), this function returns 0 on success\n "; +static PyMethodDef __pyx_mdef_11iocpsupport_5accept = {"accept", (PyCFunction)__pyx_pw_11iocpsupport_5accept, METH_VARARGS|METH_KEYWORDS, __pyx_doc_11iocpsupport_4accept}; +static PyObject *__pyx_pw_11iocpsupport_5accept(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + long __pyx_v_listening; + long __pyx_v_accepting; + PyObject *__pyx_v_buff = 0; + PyObject *__pyx_v_obj = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("accept (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_listening,&__pyx_n_s_accepting,&__pyx_n_s_buff,&__pyx_n_s_obj,0}; + PyObject* values[4] = {0,0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_listening)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_accepting)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("accept", 1, 4, 4, 1); __PYX_ERR(2, 5, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_buff)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("accept", 1, 4, 4, 2); __PYX_ERR(2, 5, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 3: + if (likely((values[3] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_obj)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("accept", 1, 4, 4, 3); __PYX_ERR(2, 5, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "accept") < 0)) __PYX_ERR(2, 5, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 4) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + } + __pyx_v_listening = __Pyx_PyInt_As_long(values[0]); if (unlikely((__pyx_v_listening == (long)-1) && PyErr_Occurred())) __PYX_ERR(2, 5, __pyx_L3_error) + __pyx_v_accepting = __Pyx_PyInt_As_long(values[1]); if (unlikely((__pyx_v_accepting == (long)-1) && PyErr_Occurred())) __PYX_ERR(2, 5, __pyx_L3_error) + __pyx_v_buff = values[2]; + __pyx_v_obj = values[3]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("accept", 1, 4, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 5, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.accept", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_4accept(__pyx_self, __pyx_v_listening, __pyx_v_accepting, __pyx_v_buff, __pyx_v_obj); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_4accept(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_listening, long __pyx_v_accepting, PyObject *__pyx_v_buff, PyObject *__pyx_v_obj) { + unsigned long __pyx_v_bytes; + int __pyx_v_rc; + Py_ssize_t __pyx_v_size; + void *__pyx_v_mem_buffer; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_ov; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_t_2; + struct __pyx_opt_args_11iocpsupport_makeOV __pyx_t_3; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannySetupContext("accept", 0); + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":15 + * cdef myOVERLAPPED *ov + * + * PyObject_AsWriteBuffer(buff, &mem_buffer, &size) # <<<<<<<<<<<<<< + * + * ov = makeOV(obj, buff) + */ + __pyx_t_1 = PyObject_AsWriteBuffer(__pyx_v_buff, (&__pyx_v_mem_buffer), (&__pyx_v_size)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 15, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":17 + * PyObject_AsWriteBuffer(buff, &mem_buffer, &size) + * + * ov = makeOV(obj, buff) # <<<<<<<<<<<<<< + * + * rc = lpAcceptEx(listening, accepting, mem_buffer, 0, + */ + __pyx_t_3.__pyx_n = 1; + __pyx_t_3.other = __pyx_v_buff; + __pyx_t_2 = __pyx_f_11iocpsupport_makeOV(__pyx_v_obj, &__pyx_t_3); if (unlikely(__pyx_t_2 == ((struct __pyx_t_11iocpsupport_myOVERLAPPED *)NULL))) __PYX_ERR(2, 17, __pyx_L1_error) + __pyx_v_ov = __pyx_t_2; + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":19 + * ov = makeOV(obj, buff) + * + * rc = lpAcceptEx(listening, accepting, mem_buffer, 0, # <<<<<<<<<<<<<< + * size / 2, size / 2, + * &bytes, ov) + */ + __pyx_v_rc = lpAcceptEx(__pyx_v_listening, __pyx_v_accepting, __pyx_v_mem_buffer, 0, (((__pyx_t_11iocpsupport_DWORD)__pyx_v_size) / 2), (((__pyx_t_11iocpsupport_DWORD)__pyx_v_size) / 2), (&__pyx_v_bytes), ((OVERLAPPED *)__pyx_v_ov)); + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":22 + * size / 2, size / 2, + * &bytes, ov) + * if not rc: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + __pyx_t_4 = ((!(__pyx_v_rc != 0)) != 0); + if (__pyx_t_4) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":23 + * &bytes, ov) + * if not rc: + * rc = WSAGetLastError() # <<<<<<<<<<<<<< + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + */ + __pyx_v_rc = WSAGetLastError(); + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":24 + * if not rc: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc + */ + __pyx_t_4 = ((__pyx_v_rc != ERROR_IO_PENDING) != 0); + if (__pyx_t_4) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":25 + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) # <<<<<<<<<<<<<< + * return rc + * + */ + __pyx_f_11iocpsupport_unmakeOV(__pyx_v_ov); + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":26 + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + * return rc # <<<<<<<<<<<<<< + * + * return 0 + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_5 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 26, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":24 + * if not rc: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":22 + * size / 2, size / 2, + * &bytes, ov) + * if not rc: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":28 + * return rc + * + * return 0 # <<<<<<<<<<<<<< + * + * def get_accept_addrs(long s, object buff): + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_int_0); + __pyx_r = __pyx_int_0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":5 + * + * + * def accept(long listening, long accepting, object buff, object obj): # <<<<<<<<<<<<<< + * """ + * CAUTION: unlike system AcceptEx(), this function returns 0 on success + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("iocpsupport.accept", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":30 + * return 0 + * + * def get_accept_addrs(long s, object buff): # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int locallen, remotelen + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_7get_accept_addrs(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_11iocpsupport_7get_accept_addrs = {"get_accept_addrs", (PyCFunction)__pyx_pw_11iocpsupport_7get_accept_addrs, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_11iocpsupport_7get_accept_addrs(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED long __pyx_v_s; + PyObject *__pyx_v_buff = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("get_accept_addrs (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_s,&__pyx_n_s_buff,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_s)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_buff)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("get_accept_addrs", 1, 2, 2, 1); __PYX_ERR(2, 30, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "get_accept_addrs") < 0)) __PYX_ERR(2, 30, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + } + __pyx_v_s = __Pyx_PyInt_As_long(values[0]); if (unlikely((__pyx_v_s == (long)-1) && PyErr_Occurred())) __PYX_ERR(2, 30, __pyx_L3_error) + __pyx_v_buff = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("get_accept_addrs", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 30, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.get_accept_addrs", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_6get_accept_addrs(__pyx_self, __pyx_v_s, __pyx_v_buff); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_6get_accept_addrs(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED long __pyx_v_s, PyObject *__pyx_v_buff) { + int __pyx_v_locallen; + int __pyx_v_remotelen; + Py_ssize_t __pyx_v_size; + void *__pyx_v_mem_buffer; + struct sockaddr *__pyx_v_localaddr; + struct sockaddr *__pyx_v_remoteaddr; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannySetupContext("get_accept_addrs", 0); + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":38 + * cdef sockaddr *remoteaddr + * + * PyObject_AsReadBuffer(buff, &mem_buffer, &size) # <<<<<<<<<<<<<< + * + * lpGetAcceptExSockaddrs(mem_buffer, 0, size / 2, size / 2, + */ + __pyx_t_1 = PyObject_AsReadBuffer(__pyx_v_buff, (&__pyx_v_mem_buffer), (&__pyx_v_size)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 38, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":40 + * PyObject_AsReadBuffer(buff, &mem_buffer, &size) + * + * lpGetAcceptExSockaddrs(mem_buffer, 0, size / 2, size / 2, # <<<<<<<<<<<<<< + * &localaddr, &locallen, &remoteaddr, &remotelen) + * return remoteaddr.sa_family, _makesockaddr(localaddr, locallen), _makesockaddr(remoteaddr, remotelen) + */ + lpGetAcceptExSockaddrs(__pyx_v_mem_buffer, 0, (((__pyx_t_11iocpsupport_DWORD)__pyx_v_size) / 2), (((__pyx_t_11iocpsupport_DWORD)__pyx_v_size) / 2), (&__pyx_v_localaddr), (&__pyx_v_locallen), (&__pyx_v_remoteaddr), (&__pyx_v_remotelen)); + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":42 + * lpGetAcceptExSockaddrs(mem_buffer, 0, size / 2, size / 2, + * &localaddr, &locallen, &remoteaddr, &remotelen) + * return remoteaddr.sa_family, _makesockaddr(localaddr, locallen), _makesockaddr(remoteaddr, remotelen) # <<<<<<<<<<<<<< + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyInt_From_unsigned_short(__pyx_v_remoteaddr->sa_family); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __pyx_f_11iocpsupport__makesockaddr(__pyx_v_localaddr, __pyx_v_locallen); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __pyx_f_11iocpsupport__makesockaddr(__pyx_v_remoteaddr, __pyx_v_remotelen); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_t_4); + __pyx_t_2 = 0; + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":30 + * return 0 + * + * def get_accept_addrs(long s, object buff): # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int locallen, remotelen + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("iocpsupport.get_accept_addrs", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":5 + * + * + * def connect(long s, object addr, object obj): # <<<<<<<<<<<<<< + * """ + * CAUTION: unlike system ConnectEx(), this function returns 0 on success + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_9connect(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_11iocpsupport_8connect[] = "\n CAUTION: unlike system ConnectEx(), this function returns 0 on success\n "; +static PyMethodDef __pyx_mdef_11iocpsupport_9connect = {"connect", (PyCFunction)__pyx_pw_11iocpsupport_9connect, METH_VARARGS|METH_KEYWORDS, __pyx_doc_11iocpsupport_8connect}; +static PyObject *__pyx_pw_11iocpsupport_9connect(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + long __pyx_v_s; + PyObject *__pyx_v_addr = 0; + PyObject *__pyx_v_obj = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("connect (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_s,&__pyx_n_s_addr,&__pyx_n_s_obj,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_s)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_addr)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("connect", 1, 3, 3, 1); __PYX_ERR(3, 5, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_obj)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("connect", 1, 3, 3, 2); __PYX_ERR(3, 5, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "connect") < 0)) __PYX_ERR(3, 5, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_s = __Pyx_PyInt_As_long(values[0]); if (unlikely((__pyx_v_s == (long)-1) && PyErr_Occurred())) __PYX_ERR(3, 5, __pyx_L3_error) + __pyx_v_addr = values[1]; + __pyx_v_obj = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("connect", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(3, 5, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.connect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_8connect(__pyx_self, __pyx_v_s, __pyx_v_addr, __pyx_v_obj); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_8connect(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s, PyObject *__pyx_v_addr, PyObject *__pyx_v_obj) { + int __pyx_v_family; + int __pyx_v_rc; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_ov; + struct sockaddr_in __pyx_v_ipv4_name; + struct sockaddr_in6 __pyx_v_ipv6_name; + struct sockaddr *__pyx_v_name; + int __pyx_v_namelen; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_t_5; + __Pyx_RefNannySetupContext("connect", 0); + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":16 + * cdef int namelen + * + * if not have_connectex: # <<<<<<<<<<<<<< + * raise ValueError, 'ConnectEx is not available on this system' + * + */ + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_have_connectex); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(3, 16, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = ((!__pyx_t_2) != 0); + if (unlikely(__pyx_t_3)) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":17 + * + * if not have_connectex: + * raise ValueError, 'ConnectEx is not available on this system' # <<<<<<<<<<<<<< + * + * family = getAddrFamily(s) + */ + __Pyx_Raise(__pyx_builtin_ValueError, __pyx_kp_s_ConnectEx_is_not_available_on_th, 0, 0); + __PYX_ERR(3, 17, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":16 + * cdef int namelen + * + * if not have_connectex: # <<<<<<<<<<<<<< + * raise ValueError, 'ConnectEx is not available on this system' + * + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":19 + * raise ValueError, 'ConnectEx is not available on this system' + * + * family = getAddrFamily(s) # <<<<<<<<<<<<<< + * if family == AF_INET: + * name = &ipv4_name + */ + __pyx_t_4 = __pyx_f_11iocpsupport_getAddrFamily(__pyx_v_s); if (unlikely(__pyx_t_4 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(3, 19, __pyx_L1_error) + __pyx_v_family = __pyx_t_4; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":20 + * + * family = getAddrFamily(s) + * if family == AF_INET: # <<<<<<<<<<<<<< + * name = &ipv4_name + * namelen = sizeof(ipv4_name) + */ + switch (__pyx_v_family) { + case AF_INET: + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":21 + * family = getAddrFamily(s) + * if family == AF_INET: + * name = &ipv4_name # <<<<<<<<<<<<<< + * namelen = sizeof(ipv4_name) + * fillinetaddr(&ipv4_name, addr) + */ + __pyx_v_name = ((struct sockaddr *)(&__pyx_v_ipv4_name)); + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":22 + * if family == AF_INET: + * name = &ipv4_name + * namelen = sizeof(ipv4_name) # <<<<<<<<<<<<<< + * fillinetaddr(&ipv4_name, addr) + * elif family == AF_INET6: + */ + __pyx_v_namelen = (sizeof(__pyx_v_ipv4_name)); + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":23 + * name = &ipv4_name + * namelen = sizeof(ipv4_name) + * fillinetaddr(&ipv4_name, addr) # <<<<<<<<<<<<<< + * elif family == AF_INET6: + * name = &ipv6_name + */ + __pyx_t_1 = __pyx_f_11iocpsupport_fillinetaddr((&__pyx_v_ipv4_name), __pyx_v_addr); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 23, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":20 + * + * family = getAddrFamily(s) + * if family == AF_INET: # <<<<<<<<<<<<<< + * name = &ipv4_name + * namelen = sizeof(ipv4_name) + */ + break; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":24 + * namelen = sizeof(ipv4_name) + * fillinetaddr(&ipv4_name, addr) + * elif family == AF_INET6: # <<<<<<<<<<<<<< + * name = &ipv6_name + * namelen = sizeof(ipv6_name) + */ + case AF_INET6: + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":25 + * fillinetaddr(&ipv4_name, addr) + * elif family == AF_INET6: + * name = &ipv6_name # <<<<<<<<<<<<<< + * namelen = sizeof(ipv6_name) + * fillinet6addr(&ipv6_name, addr) + */ + __pyx_v_name = ((struct sockaddr *)(&__pyx_v_ipv6_name)); + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":26 + * elif family == AF_INET6: + * name = &ipv6_name + * namelen = sizeof(ipv6_name) # <<<<<<<<<<<<<< + * fillinet6addr(&ipv6_name, addr) + * else: + */ + __pyx_v_namelen = (sizeof(__pyx_v_ipv6_name)); + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":27 + * name = &ipv6_name + * namelen = sizeof(ipv6_name) + * fillinet6addr(&ipv6_name, addr) # <<<<<<<<<<<<<< + * else: + * raise ValueError, 'unsupported address family' + */ + __pyx_t_1 = __pyx_f_11iocpsupport_fillinet6addr((&__pyx_v_ipv6_name), __pyx_v_addr); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 27, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":24 + * namelen = sizeof(ipv4_name) + * fillinetaddr(&ipv4_name, addr) + * elif family == AF_INET6: # <<<<<<<<<<<<<< + * name = &ipv6_name + * namelen = sizeof(ipv6_name) + */ + break; + default: + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":29 + * fillinet6addr(&ipv6_name, addr) + * else: + * raise ValueError, 'unsupported address family' # <<<<<<<<<<<<<< + * name.sa_family = family + * + */ + __Pyx_Raise(__pyx_builtin_ValueError, __pyx_kp_s_unsupported_address_family, 0, 0); + __PYX_ERR(3, 29, __pyx_L1_error) + break; + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":30 + * else: + * raise ValueError, 'unsupported address family' + * name.sa_family = family # <<<<<<<<<<<<<< + * + * ov = makeOV(obj) + */ + __pyx_v_name->sa_family = __pyx_v_family; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":32 + * name.sa_family = family + * + * ov = makeOV(obj) # <<<<<<<<<<<<<< + * + * rc = lpConnectEx(s, name, namelen, NULL, 0, NULL, ov) + */ + __pyx_t_5 = __pyx_f_11iocpsupport_makeOV(__pyx_v_obj, NULL); if (unlikely(__pyx_t_5 == ((struct __pyx_t_11iocpsupport_myOVERLAPPED *)NULL))) __PYX_ERR(3, 32, __pyx_L1_error) + __pyx_v_ov = __pyx_t_5; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":34 + * ov = makeOV(obj) + * + * rc = lpConnectEx(s, name, namelen, NULL, 0, NULL, ov) # <<<<<<<<<<<<<< + * + * if not rc: + */ + __pyx_v_rc = lpConnectEx(__pyx_v_s, __pyx_v_name, __pyx_v_namelen, NULL, 0, NULL, ((OVERLAPPED *)__pyx_v_ov)); + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":36 + * rc = lpConnectEx(s, name, namelen, NULL, 0, NULL, ov) + * + * if not rc: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + __pyx_t_3 = ((!(__pyx_v_rc != 0)) != 0); + if (__pyx_t_3) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":37 + * + * if not rc: + * rc = WSAGetLastError() # <<<<<<<<<<<<<< + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + */ + __pyx_v_rc = WSAGetLastError(); + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":38 + * if not rc: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc + */ + __pyx_t_3 = ((__pyx_v_rc != ERROR_IO_PENDING) != 0); + if (__pyx_t_3) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":39 + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) # <<<<<<<<<<<<<< + * return rc + * + */ + __pyx_f_11iocpsupport_unmakeOV(__pyx_v_ov); + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":40 + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + * return rc # <<<<<<<<<<<<<< + * + * return 0 + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":38 + * if not rc: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":36 + * rc = lpConnectEx(s, name, namelen, NULL, 0, NULL, ov) + * + * if not rc: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":42 + * return rc + * + * return 0 # <<<<<<<<<<<<<< + * + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_int_0); + __pyx_r = __pyx_int_0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":5 + * + * + * def connect(long s, object addr, object obj): # <<<<<<<<<<<<<< + * """ + * CAUTION: unlike system ConnectEx(), this function returns 0 on success + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("iocpsupport.connect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":5 + * + * + * def recv(long s, object bufflist, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc, res + * cdef myOVERLAPPED *ov + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_11recv(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_11iocpsupport_11recv = {"recv", (PyCFunction)__pyx_pw_11iocpsupport_11recv, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_11iocpsupport_11recv(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + long __pyx_v_s; + PyObject *__pyx_v_bufflist = 0; + PyObject *__pyx_v_obj = 0; + unsigned long __pyx_v_flags; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("recv (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_s,&__pyx_n_s_bufflist,&__pyx_n_s_obj,&__pyx_n_s_flags,0}; + PyObject* values[4] = {0,0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_s)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_bufflist)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("recv", 0, 3, 4, 1); __PYX_ERR(4, 5, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_obj)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("recv", 0, 3, 4, 2); __PYX_ERR(4, 5, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 3: + if (kw_args > 0) { + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_flags); + if (value) { values[3] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "recv") < 0)) __PYX_ERR(4, 5, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_s = __Pyx_PyInt_As_long(values[0]); if (unlikely((__pyx_v_s == (long)-1) && PyErr_Occurred())) __PYX_ERR(4, 5, __pyx_L3_error) + __pyx_v_bufflist = values[1]; + __pyx_v_obj = values[2]; + if (values[3]) { + __pyx_v_flags = __Pyx_PyInt_As_unsigned_long(values[3]); if (unlikely((__pyx_v_flags == (unsigned long)-1) && PyErr_Occurred())) __PYX_ERR(4, 5, __pyx_L3_error) + } else { + __pyx_v_flags = ((unsigned long)0); + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("recv", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(4, 5, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.recv", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_10recv(__pyx_self, __pyx_v_s, __pyx_v_bufflist, __pyx_v_obj, __pyx_v_flags); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_10recv(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s, PyObject *__pyx_v_bufflist, PyObject *__pyx_v_obj, unsigned long __pyx_v_flags) { + int __pyx_v_rc; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_ov; + WSABUF *__pyx_v_ws_buf; + unsigned long __pyx_v_bytes; + PyObject **__pyx_v_buffers; + Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_size; + Py_ssize_t __pyx_v_buffcount; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + void *__pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_t_5; + struct __pyx_opt_args_11iocpsupport_makeOV __pyx_t_6; + int __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + int __pyx_t_10; + char const *__pyx_t_11; + PyObject *__pyx_t_12 = NULL; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + PyObject *__pyx_t_15 = NULL; + PyObject *__pyx_t_16 = NULL; + PyObject *__pyx_t_17 = NULL; + __Pyx_RefNannySetupContext("recv", 0); + __Pyx_INCREF(__pyx_v_bufflist); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":13 + * cdef Py_ssize_t i, size, buffcount + * + * bufflist = PySequence_Fast(bufflist, 'second argument needs to be a list') # <<<<<<<<<<<<<< + * buffcount = PySequence_Fast_GET_SIZE(bufflist) + * buffers = PySequence_Fast_ITEMS(bufflist) + */ + __pyx_t_1 = PySequence_Fast(__pyx_v_bufflist, ((char *)"second argument needs to be a list")); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF_SET(__pyx_v_bufflist, __pyx_t_1); + __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":14 + * + * bufflist = PySequence_Fast(bufflist, 'second argument needs to be a list') + * buffcount = PySequence_Fast_GET_SIZE(bufflist) # <<<<<<<<<<<<<< + * buffers = PySequence_Fast_ITEMS(bufflist) + * + */ + __pyx_v_buffcount = PySequence_Fast_GET_SIZE(__pyx_v_bufflist); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":15 + * bufflist = PySequence_Fast(bufflist, 'second argument needs to be a list') + * buffcount = PySequence_Fast_GET_SIZE(bufflist) + * buffers = PySequence_Fast_ITEMS(bufflist) # <<<<<<<<<<<<<< + * + * ws_buf = PyMem_Malloc(buffcount*sizeof(WSABUF)) + */ + __pyx_v_buffers = PySequence_Fast_ITEMS(__pyx_v_bufflist); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":17 + * buffers = PySequence_Fast_ITEMS(bufflist) + * + * ws_buf = PyMem_Malloc(buffcount*sizeof(WSABUF)) # <<<<<<<<<<<<<< + * + * try: + */ + __pyx_t_2 = PyMem_Malloc((__pyx_v_buffcount * (sizeof(WSABUF)))); if (unlikely(__pyx_t_2 == ((void *)NULL))) __PYX_ERR(4, 17, __pyx_L1_error) + __pyx_v_ws_buf = ((WSABUF *)__pyx_t_2); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":19 + * ws_buf = PyMem_Malloc(buffcount*sizeof(WSABUF)) + * + * try: # <<<<<<<<<<<<<< + * for i from 0 <= i < buffcount: + * PyObject_AsWriteBuffer(buffers[i], &ws_buf[i].buf, &size) + */ + /*try:*/ { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":20 + * + * try: + * for i from 0 <= i < buffcount: # <<<<<<<<<<<<<< + * PyObject_AsWriteBuffer(buffers[i], &ws_buf[i].buf, &size) + * ws_buf[i].len = size + */ + __pyx_t_3 = __pyx_v_buffcount; + for (__pyx_v_i = 0; __pyx_v_i < __pyx_t_3; __pyx_v_i++) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":21 + * try: + * for i from 0 <= i < buffcount: + * PyObject_AsWriteBuffer(buffers[i], &ws_buf[i].buf, &size) # <<<<<<<<<<<<<< + * ws_buf[i].len = size + * + */ + __pyx_t_1 = ((PyObject *)(__pyx_v_buffers[__pyx_v_i])); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = PyObject_AsWriteBuffer(__pyx_t_1, ((void **)(&(__pyx_v_ws_buf[__pyx_v_i]).buf)), (&__pyx_v_size)); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(4, 21, __pyx_L4_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":22 + * for i from 0 <= i < buffcount: + * PyObject_AsWriteBuffer(buffers[i], &ws_buf[i].buf, &size) + * ws_buf[i].len = size # <<<<<<<<<<<<<< + * + * ov = makeOV(obj, bufflist) + */ + (__pyx_v_ws_buf[__pyx_v_i]).len = ((__pyx_t_11iocpsupport_DWORD)__pyx_v_size); + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":24 + * ws_buf[i].len = size + * + * ov = makeOV(obj, bufflist) # <<<<<<<<<<<<<< + * + * rc = WSARecv(s, ws_buf, buffcount, &bytes, &flags, ov, NULL) + */ + __pyx_t_6.__pyx_n = 1; + __pyx_t_6.other = __pyx_v_bufflist; + __pyx_t_5 = __pyx_f_11iocpsupport_makeOV(__pyx_v_obj, &__pyx_t_6); if (unlikely(__pyx_t_5 == ((struct __pyx_t_11iocpsupport_myOVERLAPPED *)NULL))) __PYX_ERR(4, 24, __pyx_L4_error) + __pyx_v_ov = __pyx_t_5; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":26 + * ov = makeOV(obj, bufflist) + * + * rc = WSARecv(s, ws_buf, buffcount, &bytes, &flags, ov, NULL) # <<<<<<<<<<<<<< + * + * if rc == SOCKET_ERROR: + */ + __pyx_v_rc = WSARecv(__pyx_v_s, __pyx_v_ws_buf, ((__pyx_t_11iocpsupport_DWORD)__pyx_v_buffcount), (&__pyx_v_bytes), (&__pyx_v_flags), ((OVERLAPPED *)__pyx_v_ov), NULL); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":28 + * rc = WSARecv(s, ws_buf, buffcount, &bytes, &flags, ov, NULL) + * + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + __pyx_t_7 = ((__pyx_v_rc == SOCKET_ERROR) != 0); + if (__pyx_t_7) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":29 + * + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() # <<<<<<<<<<<<<< + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + */ + __pyx_v_rc = WSAGetLastError(); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":30 + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc, 0 + */ + __pyx_t_7 = ((__pyx_v_rc != ERROR_IO_PENDING) != 0); + if (__pyx_t_7) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":31 + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) # <<<<<<<<<<<<<< + * return rc, 0 + * + */ + __pyx_f_11iocpsupport_unmakeOV(__pyx_v_ov); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":32 + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + * return rc, 0 # <<<<<<<<<<<<<< + * + * return rc, bytes + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 32, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(4, 32, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_1); + __Pyx_INCREF(__pyx_int_0); + __Pyx_GIVEREF(__pyx_int_0); + PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_int_0); + __pyx_t_1 = 0; + __pyx_r = __pyx_t_8; + __pyx_t_8 = 0; + goto __pyx_L3_return; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":30 + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc, 0 + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":28 + * rc = WSARecv(s, ws_buf, buffcount, &bytes, &flags, ov, NULL) + * + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":34 + * return rc, 0 + * + * return rc, bytes # <<<<<<<<<<<<<< + * finally: + * PyMem_Free(ws_buf) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_8)) __PYX_ERR(4, 34, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyInt_From_unsigned_long(__pyx_v_bytes); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 34, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(4, 34, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_8); + PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_8); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_1); + __pyx_t_8 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_9; + __pyx_t_9 = 0; + goto __pyx_L3_return; + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":36 + * return rc, bytes + * finally: + * PyMem_Free(ws_buf) # <<<<<<<<<<<<<< + * + * def recvfrom(long s, object buff, object addr_buff, object addr_len_buff, object obj, unsigned long flags = 0): + */ + /*finally:*/ { + __pyx_L4_error:; + /*exception exit:*/{ + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __pyx_t_12 = 0; __pyx_t_13 = 0; __pyx_t_14 = 0; __pyx_t_15 = 0; __pyx_t_16 = 0; __pyx_t_17 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + if (PY_MAJOR_VERSION >= 3) __Pyx_ExceptionSwap(&__pyx_t_15, &__pyx_t_16, &__pyx_t_17); + if ((PY_MAJOR_VERSION < 3) || unlikely(__Pyx_GetException(&__pyx_t_12, &__pyx_t_13, &__pyx_t_14) < 0)) __Pyx_ErrFetch(&__pyx_t_12, &__pyx_t_13, &__pyx_t_14); + __Pyx_XGOTREF(__pyx_t_12); + __Pyx_XGOTREF(__pyx_t_13); + __Pyx_XGOTREF(__pyx_t_14); + __Pyx_XGOTREF(__pyx_t_15); + __Pyx_XGOTREF(__pyx_t_16); + __Pyx_XGOTREF(__pyx_t_17); + __pyx_t_4 = __pyx_lineno; __pyx_t_10 = __pyx_clineno; __pyx_t_11 = __pyx_filename; + { + PyMem_Free(__pyx_v_ws_buf); + } + if (PY_MAJOR_VERSION >= 3) { + __Pyx_XGIVEREF(__pyx_t_15); + __Pyx_XGIVEREF(__pyx_t_16); + __Pyx_XGIVEREF(__pyx_t_17); + __Pyx_ExceptionReset(__pyx_t_15, __pyx_t_16, __pyx_t_17); + } + __Pyx_XGIVEREF(__pyx_t_12); + __Pyx_XGIVEREF(__pyx_t_13); + __Pyx_XGIVEREF(__pyx_t_14); + __Pyx_ErrRestore(__pyx_t_12, __pyx_t_13, __pyx_t_14); + __pyx_t_12 = 0; __pyx_t_13 = 0; __pyx_t_14 = 0; __pyx_t_15 = 0; __pyx_t_16 = 0; __pyx_t_17 = 0; + __pyx_lineno = __pyx_t_4; __pyx_clineno = __pyx_t_10; __pyx_filename = __pyx_t_11; + goto __pyx_L1_error; + } + __pyx_L3_return: { + __pyx_t_17 = __pyx_r; + __pyx_r = 0; + PyMem_Free(__pyx_v_ws_buf); + __pyx_r = __pyx_t_17; + __pyx_t_17 = 0; + goto __pyx_L0; + } + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":5 + * + * + * def recv(long s, object bufflist, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc, res + * cdef myOVERLAPPED *ov + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("iocpsupport.recv", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_bufflist); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":38 + * PyMem_Free(ws_buf) + * + * def recvfrom(long s, object buff, object addr_buff, object addr_len_buff, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc, c_addr_buff_len, c_addr_len_buff_len + * cdef myOVERLAPPED *ov + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_13recvfrom(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_11iocpsupport_13recvfrom = {"recvfrom", (PyCFunction)__pyx_pw_11iocpsupport_13recvfrom, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_11iocpsupport_13recvfrom(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + long __pyx_v_s; + PyObject *__pyx_v_buff = 0; + PyObject *__pyx_v_addr_buff = 0; + PyObject *__pyx_v_addr_len_buff = 0; + PyObject *__pyx_v_obj = 0; + unsigned long __pyx_v_flags; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("recvfrom (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_s,&__pyx_n_s_buff,&__pyx_n_s_addr_buff,&__pyx_n_s_addr_len_buff,&__pyx_n_s_obj,&__pyx_n_s_flags,0}; + PyObject* values[6] = {0,0,0,0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); + CYTHON_FALLTHROUGH; + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + CYTHON_FALLTHROUGH; + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_s)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_buff)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("recvfrom", 0, 5, 6, 1); __PYX_ERR(4, 38, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_addr_buff)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("recvfrom", 0, 5, 6, 2); __PYX_ERR(4, 38, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 3: + if (likely((values[3] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_addr_len_buff)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("recvfrom", 0, 5, 6, 3); __PYX_ERR(4, 38, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 4: + if (likely((values[4] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_obj)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("recvfrom", 0, 5, 6, 4); __PYX_ERR(4, 38, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 5: + if (kw_args > 0) { + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_flags); + if (value) { values[5] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "recvfrom") < 0)) __PYX_ERR(4, 38, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); + CYTHON_FALLTHROUGH; + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_s = __Pyx_PyInt_As_long(values[0]); if (unlikely((__pyx_v_s == (long)-1) && PyErr_Occurred())) __PYX_ERR(4, 38, __pyx_L3_error) + __pyx_v_buff = values[1]; + __pyx_v_addr_buff = values[2]; + __pyx_v_addr_len_buff = values[3]; + __pyx_v_obj = values[4]; + if (values[5]) { + __pyx_v_flags = __Pyx_PyInt_As_unsigned_long(values[5]); if (unlikely((__pyx_v_flags == (unsigned long)-1) && PyErr_Occurred())) __PYX_ERR(4, 38, __pyx_L3_error) + } else { + __pyx_v_flags = ((unsigned long)0); + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("recvfrom", 0, 5, 6, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(4, 38, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.recvfrom", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_12recvfrom(__pyx_self, __pyx_v_s, __pyx_v_buff, __pyx_v_addr_buff, __pyx_v_addr_len_buff, __pyx_v_obj, __pyx_v_flags); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_12recvfrom(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s, PyObject *__pyx_v_buff, PyObject *__pyx_v_addr_buff, PyObject *__pyx_v_addr_len_buff, PyObject *__pyx_v_obj, unsigned long __pyx_v_flags) { + int __pyx_v_rc; + int __pyx_v_c_addr_buff_len; + int __pyx_v_c_addr_len_buff_len; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_ov; + WSABUF __pyx_v_ws_buf; + unsigned long __pyx_v_bytes; + struct sockaddr *__pyx_v_c_addr_buff; + int *__pyx_v_c_addr_len_buff; + Py_ssize_t __pyx_v_size; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_t_4; + struct __pyx_opt_args_11iocpsupport_makeOV __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + __Pyx_RefNannySetupContext("recvfrom", 0); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":47 + * cdef Py_ssize_t size + * + * PyObject_AsWriteBuffer(buff, &ws_buf.buf, &size) # <<<<<<<<<<<<<< + * ws_buf.len = size + * PyObject_AsWriteBuffer(addr_buff, &c_addr_buff, &size) + */ + __pyx_t_1 = PyObject_AsWriteBuffer(__pyx_v_buff, ((void **)(&__pyx_v_ws_buf.buf)), (&__pyx_v_size)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(4, 47, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":48 + * + * PyObject_AsWriteBuffer(buff, &ws_buf.buf, &size) + * ws_buf.len = size # <<<<<<<<<<<<<< + * PyObject_AsWriteBuffer(addr_buff, &c_addr_buff, &size) + * c_addr_buff_len = size + */ + __pyx_v_ws_buf.len = ((__pyx_t_11iocpsupport_DWORD)__pyx_v_size); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":49 + * PyObject_AsWriteBuffer(buff, &ws_buf.buf, &size) + * ws_buf.len = size + * PyObject_AsWriteBuffer(addr_buff, &c_addr_buff, &size) # <<<<<<<<<<<<<< + * c_addr_buff_len = size + * PyObject_AsWriteBuffer(addr_len_buff, &c_addr_len_buff, &size) + */ + __pyx_t_1 = PyObject_AsWriteBuffer(__pyx_v_addr_buff, ((void **)(&__pyx_v_c_addr_buff)), (&__pyx_v_size)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(4, 49, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":50 + * ws_buf.len = size + * PyObject_AsWriteBuffer(addr_buff, &c_addr_buff, &size) + * c_addr_buff_len = size # <<<<<<<<<<<<<< + * PyObject_AsWriteBuffer(addr_len_buff, &c_addr_len_buff, &size) + * c_addr_len_buff_len = size + */ + __pyx_v_c_addr_buff_len = ((int)__pyx_v_size); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":51 + * PyObject_AsWriteBuffer(addr_buff, &c_addr_buff, &size) + * c_addr_buff_len = size + * PyObject_AsWriteBuffer(addr_len_buff, &c_addr_len_buff, &size) # <<<<<<<<<<<<<< + * c_addr_len_buff_len = size + * + */ + __pyx_t_1 = PyObject_AsWriteBuffer(__pyx_v_addr_len_buff, ((void **)(&__pyx_v_c_addr_len_buff)), (&__pyx_v_size)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(4, 51, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":52 + * c_addr_buff_len = size + * PyObject_AsWriteBuffer(addr_len_buff, &c_addr_len_buff, &size) + * c_addr_len_buff_len = size # <<<<<<<<<<<<<< + * + * if c_addr_len_buff_len != sizeof(int): + */ + __pyx_v_c_addr_len_buff_len = ((int)__pyx_v_size); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":54 + * c_addr_len_buff_len = size + * + * if c_addr_len_buff_len != sizeof(int): # <<<<<<<<<<<<<< + * raise ValueError, 'length of address length buffer needs to be sizeof(int)' + * + */ + __pyx_t_2 = ((__pyx_v_c_addr_len_buff_len != (sizeof(int))) != 0); + if (unlikely(__pyx_t_2)) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":55 + * + * if c_addr_len_buff_len != sizeof(int): + * raise ValueError, 'length of address length buffer needs to be sizeof(int)' # <<<<<<<<<<<<<< + * + * c_addr_len_buff[0] = c_addr_buff_len + */ + __Pyx_Raise(__pyx_builtin_ValueError, __pyx_kp_s_length_of_address_length_buffer, 0, 0); + __PYX_ERR(4, 55, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":54 + * c_addr_len_buff_len = size + * + * if c_addr_len_buff_len != sizeof(int): # <<<<<<<<<<<<<< + * raise ValueError, 'length of address length buffer needs to be sizeof(int)' + * + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":57 + * raise ValueError, 'length of address length buffer needs to be sizeof(int)' + * + * c_addr_len_buff[0] = c_addr_buff_len # <<<<<<<<<<<<<< + * + * ov = makeOV(obj, (buff, addr_buff, addr_len_buff)) + */ + (__pyx_v_c_addr_len_buff[0]) = __pyx_v_c_addr_buff_len; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":59 + * c_addr_len_buff[0] = c_addr_buff_len + * + * ov = makeOV(obj, (buff, addr_buff, addr_len_buff)) # <<<<<<<<<<<<<< + * + * rc = WSARecvFrom(s, &ws_buf, 1, &bytes, &flags, c_addr_buff, c_addr_len_buff, ov, NULL) + */ + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(4, 59, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_buff); + __Pyx_GIVEREF(__pyx_v_buff); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_buff); + __Pyx_INCREF(__pyx_v_addr_buff); + __Pyx_GIVEREF(__pyx_v_addr_buff); + PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_v_addr_buff); + __Pyx_INCREF(__pyx_v_addr_len_buff); + __Pyx_GIVEREF(__pyx_v_addr_len_buff); + PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_v_addr_len_buff); + __pyx_t_5.__pyx_n = 1; + __pyx_t_5.other = __pyx_t_3; + __pyx_t_4 = __pyx_f_11iocpsupport_makeOV(__pyx_v_obj, &__pyx_t_5); if (unlikely(__pyx_t_4 == ((struct __pyx_t_11iocpsupport_myOVERLAPPED *)NULL))) __PYX_ERR(4, 59, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_ov = __pyx_t_4; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":61 + * ov = makeOV(obj, (buff, addr_buff, addr_len_buff)) + * + * rc = WSARecvFrom(s, &ws_buf, 1, &bytes, &flags, c_addr_buff, c_addr_len_buff, ov, NULL) # <<<<<<<<<<<<<< + * + * if rc == SOCKET_ERROR: + */ + __pyx_v_rc = WSARecvFrom(__pyx_v_s, (&__pyx_v_ws_buf), 1, (&__pyx_v_bytes), (&__pyx_v_flags), __pyx_v_c_addr_buff, __pyx_v_c_addr_len_buff, ((OVERLAPPED *)__pyx_v_ov), NULL); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":63 + * rc = WSARecvFrom(s, &ws_buf, 1, &bytes, &flags, c_addr_buff, c_addr_len_buff, ov, NULL) + * + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + __pyx_t_2 = ((__pyx_v_rc == SOCKET_ERROR) != 0); + if (__pyx_t_2) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":64 + * + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() # <<<<<<<<<<<<<< + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + */ + __pyx_v_rc = WSAGetLastError(); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":65 + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc, 0 + */ + __pyx_t_2 = ((__pyx_v_rc != ERROR_IO_PENDING) != 0); + if (__pyx_t_2) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":66 + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) # <<<<<<<<<<<<<< + * return rc, 0 + * + */ + __pyx_f_11iocpsupport_unmakeOV(__pyx_v_ov); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":67 + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + * return rc, 0 # <<<<<<<<<<<<<< + * + * return rc, bytes + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_3)) __PYX_ERR(4, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(4, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); + __Pyx_INCREF(__pyx_int_0); + __Pyx_GIVEREF(__pyx_int_0); + PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_int_0); + __pyx_t_3 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":65 + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc, 0 + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":63 + * rc = WSARecvFrom(s, &ws_buf, 1, &bytes, &flags, c_addr_buff, c_addr_len_buff, ov, NULL) + * + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":69 + * return rc, 0 + * + * return rc, bytes # <<<<<<<<<<<<<< + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_6 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_6)) __PYX_ERR(4, 69, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_3 = __Pyx_PyInt_From_unsigned_long(__pyx_v_bytes); if (unlikely(!__pyx_t_3)) __PYX_ERR(4, 69, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(4, 69, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_3); + __pyx_t_6 = 0; + __pyx_t_3 = 0; + __pyx_r = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":38 + * PyMem_Free(ws_buf) + * + * def recvfrom(long s, object buff, object addr_buff, object addr_len_buff, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc, c_addr_buff_len, c_addr_len_buff_len + * cdef myOVERLAPPED *ov + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("iocpsupport.recvfrom", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":5 + * + * + * def send(long s, object buff, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc + * cdef myOVERLAPPED *ov + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_15send(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_11iocpsupport_15send = {"send", (PyCFunction)__pyx_pw_11iocpsupport_15send, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_11iocpsupport_15send(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + long __pyx_v_s; + PyObject *__pyx_v_buff = 0; + PyObject *__pyx_v_obj = 0; + unsigned long __pyx_v_flags; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("send (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_s,&__pyx_n_s_buff,&__pyx_n_s_obj,&__pyx_n_s_flags,0}; + PyObject* values[4] = {0,0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_s)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_buff)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("send", 0, 3, 4, 1); __PYX_ERR(5, 5, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_obj)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("send", 0, 3, 4, 2); __PYX_ERR(5, 5, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 3: + if (kw_args > 0) { + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_flags); + if (value) { values[3] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "send") < 0)) __PYX_ERR(5, 5, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_s = __Pyx_PyInt_As_long(values[0]); if (unlikely((__pyx_v_s == (long)-1) && PyErr_Occurred())) __PYX_ERR(5, 5, __pyx_L3_error) + __pyx_v_buff = values[1]; + __pyx_v_obj = values[2]; + if (values[3]) { + __pyx_v_flags = __Pyx_PyInt_As_unsigned_long(values[3]); if (unlikely((__pyx_v_flags == (unsigned long)-1) && PyErr_Occurred())) __PYX_ERR(5, 5, __pyx_L3_error) + } else { + __pyx_v_flags = ((unsigned long)0); + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("send", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(5, 5, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.send", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_14send(__pyx_self, __pyx_v_s, __pyx_v_buff, __pyx_v_obj, __pyx_v_flags); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_14send(CYTHON_UNUSED PyObject *__pyx_self, long __pyx_v_s, PyObject *__pyx_v_buff, PyObject *__pyx_v_obj, unsigned long __pyx_v_flags) { + int __pyx_v_rc; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_v_ov; + WSABUF __pyx_v_ws_buf; + unsigned long __pyx_v_bytes; + Py_ssize_t __pyx_v_size; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + struct __pyx_t_11iocpsupport_myOVERLAPPED *__pyx_t_2; + struct __pyx_opt_args_11iocpsupport_makeOV __pyx_t_3; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + __Pyx_RefNannySetupContext("send", 0); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":12 + * cdef Py_ssize_t size + * + * PyObject_AsReadBuffer(buff, &ws_buf.buf, &size) # <<<<<<<<<<<<<< + * ws_buf.len = size + * + */ + __pyx_t_1 = PyObject_AsReadBuffer(__pyx_v_buff, ((void **)(&__pyx_v_ws_buf.buf)), (&__pyx_v_size)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(5, 12, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":13 + * + * PyObject_AsReadBuffer(buff, &ws_buf.buf, &size) + * ws_buf.len = size # <<<<<<<<<<<<<< + * + * ov = makeOV(obj, buff) + */ + __pyx_v_ws_buf.len = ((__pyx_t_11iocpsupport_DWORD)__pyx_v_size); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":15 + * ws_buf.len = size + * + * ov = makeOV(obj, buff) # <<<<<<<<<<<<<< + * + * rc = WSASend(s, &ws_buf, 1, &bytes, flags, ov, NULL) + */ + __pyx_t_3.__pyx_n = 1; + __pyx_t_3.other = __pyx_v_buff; + __pyx_t_2 = __pyx_f_11iocpsupport_makeOV(__pyx_v_obj, &__pyx_t_3); if (unlikely(__pyx_t_2 == ((struct __pyx_t_11iocpsupport_myOVERLAPPED *)NULL))) __PYX_ERR(5, 15, __pyx_L1_error) + __pyx_v_ov = __pyx_t_2; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":17 + * ov = makeOV(obj, buff) + * + * rc = WSASend(s, &ws_buf, 1, &bytes, flags, ov, NULL) # <<<<<<<<<<<<<< + * + * if rc == SOCKET_ERROR: + */ + __pyx_v_rc = WSASend(__pyx_v_s, (&__pyx_v_ws_buf), 1, (&__pyx_v_bytes), __pyx_v_flags, ((OVERLAPPED *)__pyx_v_ov), NULL); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":19 + * rc = WSASend(s, &ws_buf, 1, &bytes, flags, ov, NULL) + * + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + __pyx_t_4 = ((__pyx_v_rc == SOCKET_ERROR) != 0); + if (__pyx_t_4) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":20 + * + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() # <<<<<<<<<<<<<< + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + */ + __pyx_v_rc = WSAGetLastError(); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":21 + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc, bytes + */ + __pyx_t_4 = ((__pyx_v_rc != ERROR_IO_PENDING) != 0); + if (__pyx_t_4) { + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":22 + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) # <<<<<<<<<<<<<< + * return rc, bytes + * + */ + __pyx_f_11iocpsupport_unmakeOV(__pyx_v_ov); + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":23 + * if rc != ERROR_IO_PENDING: + * unmakeOV(ov) + * return rc, bytes # <<<<<<<<<<<<<< + * + * return rc, bytes + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_5 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_5)) __PYX_ERR(5, 23, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyInt_From_unsigned_long(__pyx_v_bytes); if (unlikely(!__pyx_t_6)) __PYX_ERR(5, 23, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(5, 23, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6); + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_r = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":21 + * if rc == SOCKET_ERROR: + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: # <<<<<<<<<<<<<< + * unmakeOV(ov) + * return rc, bytes + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":19 + * rc = WSASend(s, &ws_buf, 1, &bytes, flags, ov, NULL) + * + * if rc == SOCKET_ERROR: # <<<<<<<<<<<<<< + * rc = WSAGetLastError() + * if rc != ERROR_IO_PENDING: + */ + } + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":25 + * return rc, bytes + * + * return rc, bytes # <<<<<<<<<<<<<< + * + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_7 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_7)) __PYX_ERR(5, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = __Pyx_PyInt_From_unsigned_long(__pyx_v_bytes); if (unlikely(!__pyx_t_6)) __PYX_ERR(5, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(5, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_7); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_6); + __pyx_t_7 = 0; + __pyx_t_6 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":5 + * + * + * def send(long s, object buff, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc + * cdef myOVERLAPPED *ov + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("iocpsupport.send", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __pyx_unpickle_CompletionPort(__pyx_type, long __pyx_checksum, __pyx_state): # <<<<<<<<<<<<<< + * if __pyx_checksum != 0x901555f: + * from pickle import PickleError as __pyx_PickleError + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11iocpsupport_17__pyx_unpickle_CompletionPort(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_11iocpsupport_17__pyx_unpickle_CompletionPort = {"__pyx_unpickle_CompletionPort", (PyCFunction)__pyx_pw_11iocpsupport_17__pyx_unpickle_CompletionPort, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_11iocpsupport_17__pyx_unpickle_CompletionPort(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v___pyx_type = 0; + long __pyx_v___pyx_checksum; + PyObject *__pyx_v___pyx_state = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_unpickle_CompletionPort (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_pyx_type,&__pyx_n_s_pyx_checksum,&__pyx_n_s_pyx_state,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_pyx_type)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_pyx_checksum)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_CompletionPort", 1, 3, 3, 1); __PYX_ERR(1, 1, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_pyx_state)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_CompletionPort", 1, 3, 3, 2); __PYX_ERR(1, 1, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__pyx_unpickle_CompletionPort") < 0)) __PYX_ERR(1, 1, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v___pyx_type = values[0]; + __pyx_v___pyx_checksum = __Pyx_PyInt_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(1, 1, __pyx_L3_error) + __pyx_v___pyx_state = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_CompletionPort", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 1, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("iocpsupport.__pyx_unpickle_CompletionPort", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_11iocpsupport_16__pyx_unpickle_CompletionPort(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11iocpsupport_16__pyx_unpickle_CompletionPort(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_v___pyx_PickleError = NULL; + PyObject *__pyx_v___pyx_result = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + int __pyx_t_7; + __Pyx_RefNannySetupContext("__pyx_unpickle_CompletionPort", 0); + + /* "(tree fragment)":2 + * def __pyx_unpickle_CompletionPort(__pyx_type, long __pyx_checksum, __pyx_state): + * if __pyx_checksum != 0x901555f: # <<<<<<<<<<<<<< + * from pickle import PickleError as __pyx_PickleError + * raise __pyx_PickleError("Incompatible checksums (%s vs 0x901555f = (port))" % __pyx_checksum) + */ + __pyx_t_1 = ((__pyx_v___pyx_checksum != 0x901555f) != 0); + if (__pyx_t_1) { + + /* "(tree fragment)":3 + * def __pyx_unpickle_CompletionPort(__pyx_type, long __pyx_checksum, __pyx_state): + * if __pyx_checksum != 0x901555f: + * from pickle import PickleError as __pyx_PickleError # <<<<<<<<<<<<<< + * raise __pyx_PickleError("Incompatible checksums (%s vs 0x901555f = (port))" % __pyx_checksum) + * __pyx_result = CompletionPort.__new__(__pyx_type) + */ + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 3, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_n_s_PickleError); + __Pyx_GIVEREF(__pyx_n_s_PickleError); + PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_PickleError); + __pyx_t_3 = __Pyx_Import(__pyx_n_s_pickle, __pyx_t_2, -1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 3, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_n_s_PickleError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 3, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_t_2); + __pyx_v___pyx_PickleError = __pyx_t_2; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "(tree fragment)":4 + * if __pyx_checksum != 0x901555f: + * from pickle import PickleError as __pyx_PickleError + * raise __pyx_PickleError("Incompatible checksums (%s vs 0x901555f = (port))" % __pyx_checksum) # <<<<<<<<<<<<<< + * __pyx_result = CompletionPort.__new__(__pyx_type) + * if __pyx_state is not None: + */ + __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_v___pyx_checksum); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Incompatible_checksums_s_vs_0x90, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_INCREF(__pyx_v___pyx_PickleError); + __pyx_t_2 = __pyx_v___pyx_PickleError; __pyx_t_5 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_5)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + } + } + if (!__pyx_t_5) { + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_3); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_t_4}; + __pyx_t_3 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_t_4}; + __pyx_t_3 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __pyx_t_5 = NULL; + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":2 + * def __pyx_unpickle_CompletionPort(__pyx_type, long __pyx_checksum, __pyx_state): + * if __pyx_checksum != 0x901555f: # <<<<<<<<<<<<<< + * from pickle import PickleError as __pyx_PickleError + * raise __pyx_PickleError("Incompatible checksums (%s vs 0x901555f = (port))" % __pyx_checksum) + */ + } + + /* "(tree fragment)":5 + * from pickle import PickleError as __pyx_PickleError + * raise __pyx_PickleError("Incompatible checksums (%s vs 0x901555f = (port))" % __pyx_checksum) + * __pyx_result = CompletionPort.__new__(__pyx_type) # <<<<<<<<<<<<<< + * if __pyx_state is not None: + * __pyx_unpickle_CompletionPort__set_state( __pyx_result, __pyx_state) + */ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_11iocpsupport_CompletionPort), __pyx_n_s_new); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + } + } + if (!__pyx_t_6) { + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v___pyx_type); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v___pyx_type}; + __pyx_t_3 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_3); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v___pyx_type}; + __pyx_t_3 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_3); + } else + #endif + { + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_INCREF(__pyx_v___pyx_type); + __Pyx_GIVEREF(__pyx_v___pyx_type); + PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v___pyx_type); + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v___pyx_result = __pyx_t_3; + __pyx_t_3 = 0; + + /* "(tree fragment)":6 + * raise __pyx_PickleError("Incompatible checksums (%s vs 0x901555f = (port))" % __pyx_checksum) + * __pyx_result = CompletionPort.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_CompletionPort__set_state( __pyx_result, __pyx_state) + * return __pyx_result + */ + __pyx_t_1 = (__pyx_v___pyx_state != Py_None); + __pyx_t_7 = (__pyx_t_1 != 0); + if (__pyx_t_7) { + + /* "(tree fragment)":7 + * __pyx_result = CompletionPort.__new__(__pyx_type) + * if __pyx_state is not None: + * __pyx_unpickle_CompletionPort__set_state( __pyx_result, __pyx_state) # <<<<<<<<<<<<<< + * return __pyx_result + * cdef __pyx_unpickle_CompletionPort__set_state(CompletionPort __pyx_result, tuple __pyx_state): + */ + if (!(likely(PyTuple_CheckExact(__pyx_v___pyx_state))||((__pyx_v___pyx_state) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "tuple", Py_TYPE(__pyx_v___pyx_state)->tp_name), 0))) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_3 = __pyx_f_11iocpsupport___pyx_unpickle_CompletionPort__set_state(((struct __pyx_obj_11iocpsupport_CompletionPort *)__pyx_v___pyx_result), ((PyObject*)__pyx_v___pyx_state)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "(tree fragment)":6 + * raise __pyx_PickleError("Incompatible checksums (%s vs 0x901555f = (port))" % __pyx_checksum) + * __pyx_result = CompletionPort.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_CompletionPort__set_state( __pyx_result, __pyx_state) + * return __pyx_result + */ + } + + /* "(tree fragment)":8 + * if __pyx_state is not None: + * __pyx_unpickle_CompletionPort__set_state( __pyx_result, __pyx_state) + * return __pyx_result # <<<<<<<<<<<<<< + * cdef __pyx_unpickle_CompletionPort__set_state(CompletionPort __pyx_result, tuple __pyx_state): + * __pyx_result.port = __pyx_state[0] + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v___pyx_result); + __pyx_r = __pyx_v___pyx_result; + goto __pyx_L0; + + /* "(tree fragment)":1 + * def __pyx_unpickle_CompletionPort(__pyx_type, long __pyx_checksum, __pyx_state): # <<<<<<<<<<<<<< + * if __pyx_checksum != 0x901555f: + * from pickle import PickleError as __pyx_PickleError + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("iocpsupport.__pyx_unpickle_CompletionPort", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v___pyx_PickleError); + __Pyx_XDECREF(__pyx_v___pyx_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":9 + * __pyx_unpickle_CompletionPort__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_CompletionPort__set_state(CompletionPort __pyx_result, tuple __pyx_state): # <<<<<<<<<<<<<< + * __pyx_result.port = __pyx_state[0] + * if len(__pyx_state) > 1 and hasattr(__pyx_result, '__dict__'): + */ + +static PyObject *__pyx_f_11iocpsupport___pyx_unpickle_CompletionPort__set_state(struct __pyx_obj_11iocpsupport_CompletionPort *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __pyx_t_11iocpsupport_HANDLE __pyx_t_2; + int __pyx_t_3; + Py_ssize_t __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + __Pyx_RefNannySetupContext("__pyx_unpickle_CompletionPort__set_state", 0); + + /* "(tree fragment)":10 + * return __pyx_result + * cdef __pyx_unpickle_CompletionPort__set_state(CompletionPort __pyx_result, tuple __pyx_state): + * __pyx_result.port = __pyx_state[0] # <<<<<<<<<<<<<< + * if len(__pyx_state) > 1 and hasattr(__pyx_result, '__dict__'): + * __pyx_result.__dict__.update(__pyx_state[1]) + */ + if (unlikely(__pyx_v___pyx_state == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(1, 10, __pyx_L1_error) + } + __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_size_t(__pyx_t_1); if (unlikely((__pyx_t_2 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v___pyx_result->port = __pyx_t_2; + + /* "(tree fragment)":11 + * cdef __pyx_unpickle_CompletionPort__set_state(CompletionPort __pyx_result, tuple __pyx_state): + * __pyx_result.port = __pyx_state[0] + * if len(__pyx_state) > 1 and hasattr(__pyx_result, '__dict__'): # <<<<<<<<<<<<<< + * __pyx_result.__dict__.update(__pyx_state[1]) + */ + if (unlikely(__pyx_v___pyx_state == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 11, __pyx_L1_error) + } + __pyx_t_4 = PyTuple_GET_SIZE(__pyx_v___pyx_state); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(1, 11, __pyx_L1_error) + __pyx_t_5 = ((__pyx_t_4 > 1) != 0); + if (__pyx_t_5) { + } else { + __pyx_t_3 = __pyx_t_5; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_5 = __Pyx_HasAttr(((PyObject *)__pyx_v___pyx_result), __pyx_n_s_dict); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 11, __pyx_L1_error) + __pyx_t_6 = (__pyx_t_5 != 0); + __pyx_t_3 = __pyx_t_6; + __pyx_L4_bool_binop_done:; + if (__pyx_t_3) { + + /* "(tree fragment)":12 + * __pyx_result.port = __pyx_state[0] + * if len(__pyx_state) > 1 and hasattr(__pyx_result, '__dict__'): + * __pyx_result.__dict__.update(__pyx_state[1]) # <<<<<<<<<<<<<< + */ + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v___pyx_result), __pyx_n_s_dict); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_update); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(__pyx_v___pyx_state == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(1, 12, __pyx_L1_error) + } + __pyx_t_7 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 1, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_9 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_9 = PyMethod_GET_SELF(__pyx_t_8); + if (likely(__pyx_t_9)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_9); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_8, function); + } + } + if (!__pyx_t_9) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_8)) { + PyObject *__pyx_temp[2] = {__pyx_t_9, __pyx_t_7}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { + PyObject *__pyx_temp[2] = {__pyx_t_9, __pyx_t_7}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + } else + #endif + { + __pyx_t_10 = PyTuple_New(1+1); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_GIVEREF(__pyx_t_9); PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_t_9); __pyx_t_9 = NULL; + __Pyx_GIVEREF(__pyx_t_7); + PyTuple_SET_ITEM(__pyx_t_10, 0+1, __pyx_t_7); + __pyx_t_7 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_10, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + } + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "(tree fragment)":11 + * cdef __pyx_unpickle_CompletionPort__set_state(CompletionPort __pyx_result, tuple __pyx_state): + * __pyx_result.port = __pyx_state[0] + * if len(__pyx_state) > 1 and hasattr(__pyx_result, '__dict__'): # <<<<<<<<<<<<<< + * __pyx_result.__dict__.update(__pyx_state[1]) + */ + } + + /* "(tree fragment)":9 + * __pyx_unpickle_CompletionPort__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_CompletionPort__set_state(CompletionPort __pyx_result, tuple __pyx_state): # <<<<<<<<<<<<<< + * __pyx_result.port = __pyx_state[0] + * if len(__pyx_state) > 1 and hasattr(__pyx_result, '__dict__'): + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("iocpsupport.__pyx_unpickle_CompletionPort__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_11iocpsupport_CompletionPort(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + return o; +} + +static void __pyx_tp_dealloc_11iocpsupport_CompletionPort(PyObject *o) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + (*Py_TYPE(o)->tp_free)(o); +} + +static PyMethodDef __pyx_methods_11iocpsupport_CompletionPort[] = { + {"addHandle", (PyCFunction)__pyx_pw_11iocpsupport_14CompletionPort_3addHandle, METH_VARARGS|METH_KEYWORDS, 0}, + {"getEvent", (PyCFunction)__pyx_pw_11iocpsupport_14CompletionPort_5getEvent, METH_O, 0}, + {"postEvent", (PyCFunction)__pyx_pw_11iocpsupport_14CompletionPort_7postEvent, METH_VARARGS|METH_KEYWORDS, 0}, + {"__del__", (PyCFunction)__pyx_pw_11iocpsupport_14CompletionPort_9__del__, METH_NOARGS, 0}, + {"__reduce_cython__", (PyCFunction)__pyx_pw_11iocpsupport_14CompletionPort_11__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_11iocpsupport_14CompletionPort_13__setstate_cython__, METH_O, 0}, + {0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_11iocpsupport_CompletionPort = { + PyVarObject_HEAD_INIT(0, 0) + "iocpsupport.CompletionPort", /*tp_name*/ + sizeof(struct __pyx_obj_11iocpsupport_CompletionPort), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_11iocpsupport_CompletionPort, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_11iocpsupport_CompletionPort, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + __pyx_pw_11iocpsupport_14CompletionPort_1__init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_11iocpsupport_CompletionPort, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; + +#if PY_MAJOR_VERSION >= 3 +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_iocpsupport(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_iocpsupport}, + {0, NULL} +}; +#endif + +static struct PyModuleDef __pyx_moduledef = { + PyModuleDef_HEAD_INIT, + "iocpsupport", + 0, /* m_doc */ + #if CYTHON_PEP489_MULTI_PHASE_INIT + 0, /* m_size */ + #else + -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp_u_, __pyx_k_, sizeof(__pyx_k_), 0, 1, 0, 0}, + {&__pyx_kp_s_ConnectEx_is_not_available_on_th, __pyx_k_ConnectEx_is_not_available_on_th, sizeof(__pyx_k_ConnectEx_is_not_available_on_th), 0, 0, 1, 0}, + {&__pyx_n_s_CreateIoCompletionPort, __pyx_k_CreateIoCompletionPort, sizeof(__pyx_k_CreateIoCompletionPort), 0, 0, 1, 1}, + {&__pyx_n_s_Event, __pyx_k_Event, sizeof(__pyx_k_Event), 0, 0, 1, 1}, + {&__pyx_n_s_Event___init, __pyx_k_Event___init, sizeof(__pyx_k_Event___init), 0, 0, 1, 1}, + {&__pyx_kp_s_Failed_to_initialize_Winsock_fun, __pyx_k_Failed_to_initialize_Winsock_fun, sizeof(__pyx_k_Failed_to_initialize_Winsock_fun), 0, 0, 1, 0}, + {&__pyx_kp_s_Incompatible_checksums_s_vs_0x90, __pyx_k_Incompatible_checksums_s_vs_0x90, sizeof(__pyx_k_Incompatible_checksums_s_vs_0x90), 0, 0, 1, 0}, + {&__pyx_n_s_MemoryError, __pyx_k_MemoryError, sizeof(__pyx_k_MemoryError), 0, 0, 1, 1}, + {&__pyx_n_s_PickleError, __pyx_k_PickleError, sizeof(__pyx_k_PickleError), 0, 0, 1, 1}, + {&__pyx_n_s_PostQueuedCompletionStatus, __pyx_k_PostQueuedCompletionStatus, sizeof(__pyx_k_PostQueuedCompletionStatus), 0, 0, 1, 1}, + {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, + {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, + {&__pyx_n_s_WSAAddressToStringW, __pyx_k_WSAAddressToStringW, sizeof(__pyx_k_WSAAddressToStringW), 0, 0, 1, 1}, + {&__pyx_n_s_WindowsError, __pyx_k_WindowsError, sizeof(__pyx_k_WindowsError), 0, 0, 1, 1}, + {&__pyx_kp_s__4, __pyx_k__4, sizeof(__pyx_k__4), 0, 0, 1, 0}, + {&__pyx_kp_s__5, __pyx_k__5, sizeof(__pyx_k__5), 0, 0, 1, 0}, + {&__pyx_kp_s__7, __pyx_k__7, sizeof(__pyx_k__7), 0, 0, 1, 0}, + {&__pyx_n_s_accept, __pyx_k_accept, sizeof(__pyx_k_accept), 0, 0, 1, 1}, + {&__pyx_n_s_accepting, __pyx_k_accepting, sizeof(__pyx_k_accepting), 0, 0, 1, 1}, + {&__pyx_n_s_addr, __pyx_k_addr, sizeof(__pyx_k_addr), 0, 0, 1, 1}, + {&__pyx_n_s_addr_buff, __pyx_k_addr_buff, sizeof(__pyx_k_addr_buff), 0, 0, 1, 1}, + {&__pyx_n_s_addr_len_buff, __pyx_k_addr_len_buff, sizeof(__pyx_k_addr_len_buff), 0, 0, 1, 1}, + {&__pyx_n_s_buff, __pyx_k_buff, sizeof(__pyx_k_buff), 0, 0, 1, 1}, + {&__pyx_n_s_buffcount, __pyx_k_buffcount, sizeof(__pyx_k_buffcount), 0, 0, 1, 1}, + {&__pyx_n_s_buffers, __pyx_k_buffers, sizeof(__pyx_k_buffers), 0, 0, 1, 1}, + {&__pyx_n_s_bufflist, __pyx_k_bufflist, sizeof(__pyx_k_bufflist), 0, 0, 1, 1}, + {&__pyx_n_s_bytes, __pyx_k_bytes, sizeof(__pyx_k_bytes), 0, 0, 1, 1}, + {&__pyx_n_s_c_addr_buff, __pyx_k_c_addr_buff, sizeof(__pyx_k_c_addr_buff), 0, 0, 1, 1}, + {&__pyx_n_s_c_addr_buff_len, __pyx_k_c_addr_buff_len, sizeof(__pyx_k_c_addr_buff_len), 0, 0, 1, 1}, + {&__pyx_n_s_c_addr_len_buff, __pyx_k_c_addr_len_buff, sizeof(__pyx_k_c_addr_len_buff), 0, 0, 1, 1}, + {&__pyx_n_s_c_addr_len_buff_len, __pyx_k_c_addr_len_buff_len, sizeof(__pyx_k_c_addr_len_buff_len), 0, 0, 1, 1}, + {&__pyx_n_s_callback, __pyx_k_callback, sizeof(__pyx_k_callback), 0, 0, 1, 1}, + {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, + {&__pyx_n_s_connect, __pyx_k_connect, sizeof(__pyx_k_connect), 0, 0, 1, 1}, + {&__pyx_n_s_dict, __pyx_k_dict, sizeof(__pyx_k_dict), 0, 0, 1, 1}, + {&__pyx_n_s_doc, __pyx_k_doc, sizeof(__pyx_k_doc), 0, 0, 1, 1}, + {&__pyx_n_s_family, __pyx_k_family, sizeof(__pyx_k_family), 0, 0, 1, 1}, + {&__pyx_n_s_flags, __pyx_k_flags, sizeof(__pyx_k_flags), 0, 0, 1, 1}, + {&__pyx_n_s_get_accept_addrs, __pyx_k_get_accept_addrs, sizeof(__pyx_k_get_accept_addrs), 0, 0, 1, 1}, + {&__pyx_n_s_getsockopt, __pyx_k_getsockopt, sizeof(__pyx_k_getsockopt), 0, 0, 1, 1}, + {&__pyx_n_s_getstate, __pyx_k_getstate, sizeof(__pyx_k_getstate), 0, 0, 1, 1}, + {&__pyx_n_s_handle, __pyx_k_handle, sizeof(__pyx_k_handle), 0, 0, 1, 1}, + {&__pyx_n_s_have_connectex, __pyx_k_have_connectex, sizeof(__pyx_k_have_connectex), 0, 0, 1, 1}, + {&__pyx_n_s_i, __pyx_k_i, sizeof(__pyx_k_i), 0, 0, 1, 1}, + {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, + {&__pyx_n_s_init, __pyx_k_init, sizeof(__pyx_k_init), 0, 0, 1, 1}, + {&__pyx_kp_s_invalid_IP_address_r, __pyx_k_invalid_IP_address_r, sizeof(__pyx_k_invalid_IP_address_r), 0, 0, 1, 0}, + {&__pyx_kp_s_invalid_IPv6_address_r, __pyx_k_invalid_IPv6_address_r, sizeof(__pyx_k_invalid_IPv6_address_r), 0, 0, 1, 0}, + {&__pyx_n_s_iocpsupport, __pyx_k_iocpsupport, sizeof(__pyx_k_iocpsupport), 0, 0, 1, 1}, + {&__pyx_n_s_ipv4_name, __pyx_k_ipv4_name, sizeof(__pyx_k_ipv4_name), 0, 0, 1, 1}, + {&__pyx_n_s_ipv6_name, __pyx_k_ipv6_name, sizeof(__pyx_k_ipv6_name), 0, 0, 1, 1}, + {&__pyx_n_s_items, __pyx_k_items, sizeof(__pyx_k_items), 0, 0, 1, 1}, + {&__pyx_n_s_k, __pyx_k_k, sizeof(__pyx_k_k), 0, 0, 1, 1}, + {&__pyx_n_s_key, __pyx_k_key, sizeof(__pyx_k_key), 0, 0, 1, 1}, + {&__pyx_n_s_kw, __pyx_k_kw, sizeof(__pyx_k_kw), 0, 0, 1, 1}, + {&__pyx_kp_s_length_of_address_length_buffer, __pyx_k_length_of_address_length_buffer, sizeof(__pyx_k_length_of_address_length_buffer), 0, 0, 1, 0}, + {&__pyx_n_s_listening, __pyx_k_listening, sizeof(__pyx_k_listening), 0, 0, 1, 1}, + {&__pyx_n_s_localaddr, __pyx_k_localaddr, sizeof(__pyx_k_localaddr), 0, 0, 1, 1}, + {&__pyx_n_s_locallen, __pyx_k_locallen, sizeof(__pyx_k_locallen), 0, 0, 1, 1}, + {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, + {&__pyx_n_s_makesockaddr, __pyx_k_makesockaddr, sizeof(__pyx_k_makesockaddr), 0, 0, 1, 1}, + {&__pyx_n_s_maxAddrLen, __pyx_k_maxAddrLen, sizeof(__pyx_k_maxAddrLen), 0, 0, 1, 1}, + {&__pyx_n_s_mem_buffer, __pyx_k_mem_buffer, sizeof(__pyx_k_mem_buffer), 0, 0, 1, 1}, + {&__pyx_n_s_metaclass, __pyx_k_metaclass, sizeof(__pyx_k_metaclass), 0, 0, 1, 1}, + {&__pyx_n_s_module, __pyx_k_module, sizeof(__pyx_k_module), 0, 0, 1, 1}, + {&__pyx_n_s_name, __pyx_k_name, sizeof(__pyx_k_name), 0, 0, 1, 1}, + {&__pyx_n_s_name_2, __pyx_k_name_2, sizeof(__pyx_k_name_2), 0, 0, 1, 1}, + {&__pyx_n_s_namelen, __pyx_k_namelen, sizeof(__pyx_k_namelen), 0, 0, 1, 1}, + {&__pyx_n_s_new, __pyx_k_new, sizeof(__pyx_k_new), 0, 0, 1, 1}, + {&__pyx_n_s_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 0, 0, 1, 1}, + {&__pyx_n_s_ov, __pyx_k_ov, sizeof(__pyx_k_ov), 0, 0, 1, 1}, + {&__pyx_n_s_owner, __pyx_k_owner, sizeof(__pyx_k_owner), 0, 0, 1, 1}, + {&__pyx_n_s_pickle, __pyx_k_pickle, sizeof(__pyx_k_pickle), 0, 0, 1, 1}, + {&__pyx_n_s_prepare, __pyx_k_prepare, sizeof(__pyx_k_prepare), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_PickleError, __pyx_k_pyx_PickleError, sizeof(__pyx_k_pyx_PickleError), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_checksum, __pyx_k_pyx_checksum, sizeof(__pyx_k_pyx_checksum), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_result, __pyx_k_pyx_result, sizeof(__pyx_k_pyx_result), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_state, __pyx_k_pyx_state, sizeof(__pyx_k_pyx_state), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_type, __pyx_k_pyx_type, sizeof(__pyx_k_pyx_type), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_unpickle_CompletionPort, __pyx_k_pyx_unpickle_CompletionPort, sizeof(__pyx_k_pyx_unpickle_CompletionPort), 0, 0, 1, 1}, + {&__pyx_n_s_qualname, __pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 0, 1, 1}, + {&__pyx_n_s_rc, __pyx_k_rc, sizeof(__pyx_k_rc), 0, 0, 1, 1}, + {&__pyx_n_s_recv, __pyx_k_recv, sizeof(__pyx_k_recv), 0, 0, 1, 1}, + {&__pyx_n_s_recvfrom, __pyx_k_recvfrom, sizeof(__pyx_k_recvfrom), 0, 0, 1, 1}, + {&__pyx_n_s_reduce, __pyx_k_reduce, sizeof(__pyx_k_reduce), 0, 0, 1, 1}, + {&__pyx_n_s_reduce_cython, __pyx_k_reduce_cython, sizeof(__pyx_k_reduce_cython), 0, 0, 1, 1}, + {&__pyx_n_s_reduce_ex, __pyx_k_reduce_ex, sizeof(__pyx_k_reduce_ex), 0, 0, 1, 1}, + {&__pyx_n_s_remoteaddr, __pyx_k_remoteaddr, sizeof(__pyx_k_remoteaddr), 0, 0, 1, 1}, + {&__pyx_n_s_remotelen, __pyx_k_remotelen, sizeof(__pyx_k_remotelen), 0, 0, 1, 1}, + {&__pyx_n_s_res, __pyx_k_res, sizeof(__pyx_k_res), 0, 0, 1, 1}, + {&__pyx_n_s_rsplit, __pyx_k_rsplit, sizeof(__pyx_k_rsplit), 0, 0, 1, 1}, + {&__pyx_n_s_s, __pyx_k_s, sizeof(__pyx_k_s), 0, 0, 1, 1}, + {&__pyx_n_s_self, __pyx_k_self, sizeof(__pyx_k_self), 0, 0, 1, 1}, + {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, + {&__pyx_n_s_setstate, __pyx_k_setstate, sizeof(__pyx_k_setstate), 0, 0, 1, 1}, + {&__pyx_n_s_setstate_cython, __pyx_k_setstate_cython, sizeof(__pyx_k_setstate_cython), 0, 0, 1, 1}, + {&__pyx_n_s_size, __pyx_k_size, sizeof(__pyx_k_size), 0, 0, 1, 1}, + {&__pyx_n_s_socket, __pyx_k_socket, sizeof(__pyx_k_socket), 0, 0, 1, 1}, + {&__pyx_n_s_split, __pyx_k_split, sizeof(__pyx_k_split), 0, 0, 1, 1}, + {&__pyx_kp_s_src_twisted_internet_iocpreactor, __pyx_k_src_twisted_internet_iocpreactor, sizeof(__pyx_k_src_twisted_internet_iocpreactor), 0, 0, 1, 0}, + {&__pyx_kp_s_src_twisted_internet_iocpreactor_2, __pyx_k_src_twisted_internet_iocpreactor_2, sizeof(__pyx_k_src_twisted_internet_iocpreactor_2), 0, 0, 1, 0}, + {&__pyx_kp_s_src_twisted_internet_iocpreactor_3, __pyx_k_src_twisted_internet_iocpreactor_3, sizeof(__pyx_k_src_twisted_internet_iocpreactor_3), 0, 0, 1, 0}, + {&__pyx_kp_s_src_twisted_internet_iocpreactor_4, __pyx_k_src_twisted_internet_iocpreactor_4, sizeof(__pyx_k_src_twisted_internet_iocpreactor_4), 0, 0, 1, 0}, + {&__pyx_kp_s_src_twisted_internet_iocpreactor_5, __pyx_k_src_twisted_internet_iocpreactor_5, sizeof(__pyx_k_src_twisted_internet_iocpreactor_5), 0, 0, 1, 0}, + {&__pyx_kp_s_stringsource, __pyx_k_stringsource, sizeof(__pyx_k_stringsource), 0, 0, 1, 0}, + {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, + {&__pyx_kp_s_undefined_error_occurred_during, __pyx_k_undefined_error_occurred_during, sizeof(__pyx_k_undefined_error_occurred_during), 0, 0, 1, 0}, + {&__pyx_kp_s_unsupported_address_family, __pyx_k_unsupported_address_family, sizeof(__pyx_k_unsupported_address_family), 0, 0, 1, 0}, + {&__pyx_kp_s_unsupported_address_family_d, __pyx_k_unsupported_address_family_d, sizeof(__pyx_k_unsupported_address_family_d), 0, 0, 1, 0}, + {&__pyx_n_s_update, __pyx_k_update, sizeof(__pyx_k_update), 0, 0, 1, 1}, + {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, + {&__pyx_n_s_v, __pyx_k_v, sizeof(__pyx_k_v), 0, 0, 1, 1}, + {&__pyx_n_s_ws_buf, __pyx_k_ws_buf, sizeof(__pyx_k_ws_buf), 0, 0, 1, 1}, + {&__pyx_n_s_wsa_pi, __pyx_k_wsa_pi, sizeof(__pyx_k_wsa_pi), 0, 0, 1, 1}, + {0, 0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 378, __pyx_L1_error) + __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 158, __pyx_L1_error) + __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 349, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitCachedConstants(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "iocpsupport.pyx":278 + * sa_port = ntohs(sin.sin_port) + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + * host, port = host.rsplit(u':', 1) # <<<<<<<<<<<<<< + * port = int(port) + * assert port == sa_port + */ + __pyx_tuple__2 = PyTuple_Pack(2, __pyx_kp_u_, __pyx_int_1); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(0, 278, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__2); + __Pyx_GIVEREF(__pyx_tuple__2); + + /* "iocpsupport.pyx":291 + * sa_port = ntohs(sin6.sin6_port) + * host = PyUnicode_FromWideChar(buff, wcslen(buff)) + * host, port = host.rsplit(u':', 1) # <<<<<<<<<<<<<< + * port = int(port) + * assert host[0] == '[' + */ + __pyx_tuple__3 = PyTuple_Pack(2, __pyx_kp_u_, __pyx_int_1); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__3); + __Pyx_GIVEREF(__pyx_tuple__3); + + /* "iocpsupport.pyx":297 + * assert port == sa_port + * + * return host[1:-1], port # <<<<<<<<<<<<<< + * else: + * raise_error(0, "unsupported address family %d" % (addr.sa_family)) + */ + __pyx_slice__6 = PySlice_New(__pyx_int_1, __pyx_int_neg_1, Py_None); if (unlikely(!__pyx_slice__6)) __PYX_ERR(0, 297, __pyx_L1_error) + __Pyx_GOTREF(__pyx_slice__6); + __Pyx_GIVEREF(__pyx_slice__6); + + /* "iocpsupport.pyx":334 + * cdef int addrlen = sizeof(sockaddr_in6) + * host, port, flow, scope = addr + * host = host.split("%")[0] # remove scope ID, if any # <<<<<<<<<<<<<< + * + * if PY_MAJOR_VERSION < 3: + */ + __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_s__7); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); + + /* "iocpsupport.pyx":184 + * + * class Event: + * def __init__(self, callback, owner, **kw): # <<<<<<<<<<<<<< + * self.callback = callback + * self.owner = owner + */ + __pyx_tuple__9 = PyTuple_Pack(6, __pyx_n_s_self, __pyx_n_s_callback, __pyx_n_s_owner, __pyx_n_s_kw, __pyx_n_s_k, __pyx_n_s_v); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__9); + __Pyx_GIVEREF(__pyx_tuple__9); + __pyx_codeobj__10 = (PyObject*)__Pyx_PyCode_New(3, 0, 6, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__9, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor, __pyx_n_s_init, 184, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__10)) __PYX_ERR(0, 184, __pyx_L1_error) + + /* "iocpsupport.pyx":251 + * CloseHandle(self.port) + * + * def makesockaddr(object buff): # <<<<<<<<<<<<<< + * cdef void *mem_buffer + * cdef Py_ssize_t size + */ + __pyx_tuple__11 = PyTuple_Pack(3, __pyx_n_s_buff, __pyx_n_s_mem_buffer, __pyx_n_s_size); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__11); + __Pyx_GIVEREF(__pyx_tuple__11); + __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(1, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor, __pyx_n_s_makesockaddr, 251, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(0, 251, __pyx_L1_error) + + /* "iocpsupport.pyx":356 + * + * + * def maxAddrLen(long s): # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int size, rc + */ + __pyx_tuple__13 = PyTuple_Pack(5, __pyx_n_s_s, __pyx_n_s_s, __pyx_n_s_wsa_pi, __pyx_n_s_size, __pyx_n_s_rc); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__13); + __Pyx_GIVEREF(__pyx_tuple__13); + __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(1, 0, 5, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor, __pyx_n_s_maxAddrLen, 356, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(0, 356, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":5 + * + * + * def accept(long listening, long accepting, object buff, object obj): # <<<<<<<<<<<<<< + * """ + * CAUTION: unlike system AcceptEx(), this function returns 0 on success + */ + __pyx_tuple__15 = PyTuple_Pack(9, __pyx_n_s_listening, __pyx_n_s_accepting, __pyx_n_s_buff, __pyx_n_s_obj, __pyx_n_s_bytes, __pyx_n_s_rc, __pyx_n_s_size, __pyx_n_s_mem_buffer, __pyx_n_s_ov); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__15); + __Pyx_GIVEREF(__pyx_tuple__15); + __pyx_codeobj__16 = (PyObject*)__Pyx_PyCode_New(4, 0, 9, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor_2, __pyx_n_s_accept, 5, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__16)) __PYX_ERR(2, 5, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":30 + * return 0 + * + * def get_accept_addrs(long s, object buff): # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int locallen, remotelen + */ + __pyx_tuple__17 = PyTuple_Pack(9, __pyx_n_s_s, __pyx_n_s_buff, __pyx_n_s_wsa_pi, __pyx_n_s_locallen, __pyx_n_s_remotelen, __pyx_n_s_size, __pyx_n_s_mem_buffer, __pyx_n_s_localaddr, __pyx_n_s_remoteaddr); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(2, 30, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__17); + __Pyx_GIVEREF(__pyx_tuple__17); + __pyx_codeobj__18 = (PyObject*)__Pyx_PyCode_New(2, 0, 9, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__17, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor_2, __pyx_n_s_get_accept_addrs, 30, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__18)) __PYX_ERR(2, 30, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":5 + * + * + * def connect(long s, object addr, object obj): # <<<<<<<<<<<<<< + * """ + * CAUTION: unlike system ConnectEx(), this function returns 0 on success + */ + __pyx_tuple__19 = PyTuple_Pack(10, __pyx_n_s_s, __pyx_n_s_addr, __pyx_n_s_obj, __pyx_n_s_family, __pyx_n_s_rc, __pyx_n_s_ov, __pyx_n_s_ipv4_name, __pyx_n_s_ipv6_name, __pyx_n_s_name_2, __pyx_n_s_namelen); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(3, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__19); + __Pyx_GIVEREF(__pyx_tuple__19); + __pyx_codeobj__20 = (PyObject*)__Pyx_PyCode_New(3, 0, 10, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__19, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor_3, __pyx_n_s_connect, 5, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__20)) __PYX_ERR(3, 5, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":5 + * + * + * def recv(long s, object bufflist, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc, res + * cdef myOVERLAPPED *ov + */ + __pyx_tuple__21 = PyTuple_Pack(13, __pyx_n_s_s, __pyx_n_s_bufflist, __pyx_n_s_obj, __pyx_n_s_flags, __pyx_n_s_rc, __pyx_n_s_res, __pyx_n_s_ov, __pyx_n_s_ws_buf, __pyx_n_s_bytes, __pyx_n_s_buffers, __pyx_n_s_i, __pyx_n_s_size, __pyx_n_s_buffcount); if (unlikely(!__pyx_tuple__21)) __PYX_ERR(4, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__21); + __Pyx_GIVEREF(__pyx_tuple__21); + __pyx_codeobj__22 = (PyObject*)__Pyx_PyCode_New(4, 0, 13, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__21, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor_4, __pyx_n_s_recv, 5, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__22)) __PYX_ERR(4, 5, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":38 + * PyMem_Free(ws_buf) + * + * def recvfrom(long s, object buff, object addr_buff, object addr_len_buff, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc, c_addr_buff_len, c_addr_len_buff_len + * cdef myOVERLAPPED *ov + */ + __pyx_tuple__23 = PyTuple_Pack(15, __pyx_n_s_s, __pyx_n_s_buff, __pyx_n_s_addr_buff, __pyx_n_s_addr_len_buff, __pyx_n_s_obj, __pyx_n_s_flags, __pyx_n_s_rc, __pyx_n_s_c_addr_buff_len, __pyx_n_s_c_addr_len_buff_len, __pyx_n_s_ov, __pyx_n_s_ws_buf, __pyx_n_s_bytes, __pyx_n_s_c_addr_buff, __pyx_n_s_c_addr_len_buff, __pyx_n_s_size); if (unlikely(!__pyx_tuple__23)) __PYX_ERR(4, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__23); + __Pyx_GIVEREF(__pyx_tuple__23); + __pyx_codeobj__24 = (PyObject*)__Pyx_PyCode_New(6, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__23, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor_4, __pyx_n_s_recvfrom, 38, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__24)) __PYX_ERR(4, 38, __pyx_L1_error) + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":5 + * + * + * def send(long s, object buff, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc + * cdef myOVERLAPPED *ov + */ + __pyx_tuple__25 = PyTuple_Pack(9, __pyx_n_s_s, __pyx_n_s_buff, __pyx_n_s_obj, __pyx_n_s_flags, __pyx_n_s_rc, __pyx_n_s_ov, __pyx_n_s_ws_buf, __pyx_n_s_bytes, __pyx_n_s_size); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(5, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__25); + __Pyx_GIVEREF(__pyx_tuple__25); + __pyx_codeobj__26 = (PyObject*)__Pyx_PyCode_New(4, 0, 9, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_internet_iocpreactor_5, __pyx_n_s_send, 5, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__26)) __PYX_ERR(5, 5, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __pyx_unpickle_CompletionPort(__pyx_type, long __pyx_checksum, __pyx_state): # <<<<<<<<<<<<<< + * if __pyx_checksum != 0x901555f: + * from pickle import PickleError as __pyx_PickleError + */ + __pyx_tuple__27 = PyTuple_Pack(5, __pyx_n_s_pyx_type, __pyx_n_s_pyx_checksum, __pyx_n_s_pyx_state, __pyx_n_s_pyx_PickleError, __pyx_n_s_pyx_result); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__27); + __Pyx_GIVEREF(__pyx_tuple__27); + __pyx_codeobj__28 = (PyObject*)__Pyx_PyCode_New(3, 0, 5, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_pyx_unpickle_CompletionPort, 1, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__28)) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_InitGlobals(void) { + __pyx_umethod_PyDict_Type_items.type = (PyObject*)&PyDict_Type; + if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_151082335 = PyInt_FromLong(151082335L); if (unlikely(!__pyx_int_151082335)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_neg_1 = PyInt_FromLong(-1); if (unlikely(!__pyx_int_neg_1)) __PYX_ERR(0, 1, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_modinit_global_init_code(void); /*proto*/ +static int __Pyx_modinit_variable_export_code(void); /*proto*/ +static int __Pyx_modinit_function_export_code(void); /*proto*/ +static int __Pyx_modinit_type_init_code(void); /*proto*/ +static int __Pyx_modinit_type_import_code(void); /*proto*/ +static int __Pyx_modinit_variable_import_code(void); /*proto*/ +static int __Pyx_modinit_function_import_code(void); /*proto*/ + +static int __Pyx_modinit_global_init_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_init_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_11iocpsupport_CompletionPort) < 0) __PYX_ERR(0, 190, __pyx_L1_error) + __pyx_type_11iocpsupport_CompletionPort.tp_print = 0; + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type_11iocpsupport_CompletionPort.tp_dictoffset && __pyx_type_11iocpsupport_CompletionPort.tp_getattro == PyObject_GenericGetAttr)) { + __pyx_type_11iocpsupport_CompletionPort.tp_getattro = __Pyx_PyObject_GenericGetAttr; + } + if (PyObject_SetAttrString(__pyx_m, "CompletionPort", (PyObject *)&__pyx_type_11iocpsupport_CompletionPort) < 0) __PYX_ERR(0, 190, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_11iocpsupport_CompletionPort) < 0) __PYX_ERR(0, 190, __pyx_L1_error) + __pyx_ptype_11iocpsupport_CompletionPort = &__pyx_type_11iocpsupport_CompletionPort; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + + +#if PY_MAJOR_VERSION < 3 +#ifdef CYTHON_NO_PYINIT_EXPORT +#define __Pyx_PyMODINIT_FUNC void +#else +#define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#endif +#else +#ifdef CYTHON_NO_PYINIT_EXPORT +#define __Pyx_PyMODINIT_FUNC PyObject * +#else +#define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#endif +#endif +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (!(defined(__cplusplus)) || (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4))) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + + +#if PY_MAJOR_VERSION < 3 +__Pyx_PyMODINIT_FUNC initiocpsupport(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC initiocpsupport(void) +#else +__Pyx_PyMODINIT_FUNC PyInit_iocpsupport(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_iocpsupport(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +static int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name) { + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + result = PyDict_SetItemString(moddict, to_name, value); + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static PyObject* __pyx_pymod_create(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__") < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static int __pyx_pymod_exec_iocpsupport(PyObject *__pyx_pyinit_module) +#endif +#endif +{ + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m && __pyx_m == __pyx_pyinit_module) return 0; + #elif PY_MAJOR_VERSION >= 3 + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + __Pyx_RefNannySetupContext("__Pyx_PyMODINIT_FUNC PyInit_iocpsupport(void)", 0); + if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_StopAsyncIteration_USED + if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_m = __pyx_pyinit_module; + Py_INCREF(__pyx_m); + #else + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4("iocpsupport", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_d); + __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_cython_runtime = PyImport_AddModule((char *) "cython_runtime"); if (unlikely(!__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + #if CYTHON_COMPILING_IN_PYPY + Py_INCREF(__pyx_b); + #endif + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + if (__pyx_module_is_main_iocpsupport) { + if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + } + #if PY_MAJOR_VERSION >= 3 + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "iocpsupport")) { + if (unlikely(PyDict_SetItemString(modules, "iocpsupport", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #endif + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(); + (void)__Pyx_modinit_variable_export_code(); + (void)__Pyx_modinit_function_export_code(); + if (unlikely(__Pyx_modinit_type_init_code() != 0)) goto __pyx_L1_error; + (void)__Pyx_modinit_type_import_code(); + (void)__Pyx_modinit_variable_import_code(); + (void)__Pyx_modinit_function_import_code(); + /*--- Execution code ---*/ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + + /* "iocpsupport.pyx":183 + * raise WindowsError(message, err) + * + * class Event: # <<<<<<<<<<<<<< + * def __init__(self, callback, owner, **kw): + * self.callback = callback + */ + __pyx_t_1 = __Pyx_Py3MetaclassPrepare((PyObject *) NULL, __pyx_empty_tuple, __pyx_n_s_Event, __pyx_n_s_Event, (PyObject *) NULL, __pyx_n_s_iocpsupport, (PyObject *) NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 183, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + + /* "iocpsupport.pyx":184 + * + * class Event: + * def __init__(self, callback, owner, **kw): # <<<<<<<<<<<<<< + * self.callback = callback + * self.owner = owner + */ + __pyx_t_2 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11iocpsupport_5Event_1__init__, 0, __pyx_n_s_Event___init, NULL, __pyx_n_s_iocpsupport, __pyx_d, ((PyObject *)__pyx_codeobj__10)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_SetNameInClass(__pyx_t_1, __pyx_n_s_init, __pyx_t_2) < 0) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "iocpsupport.pyx":183 + * raise WindowsError(message, err) + * + * class Event: # <<<<<<<<<<<<<< + * def __init__(self, callback, owner, **kw): + * self.callback = callback + */ + __pyx_t_2 = __Pyx_Py3ClassCreate(((PyObject*)&__Pyx_DefaultClassType), __pyx_n_s_Event, __pyx_empty_tuple, __pyx_t_1, NULL, 0, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 183, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Event, __pyx_t_2) < 0) __PYX_ERR(0, 183, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "iocpsupport.pyx":251 + * CloseHandle(self.port) + * + * def makesockaddr(object buff): # <<<<<<<<<<<<<< + * cdef void *mem_buffer + * cdef Py_ssize_t size + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_1makesockaddr, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_makesockaddr, __pyx_t_1) < 0) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "iocpsupport.pyx":356 + * + * + * def maxAddrLen(long s): # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int size, rc + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_3maxAddrLen, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_maxAddrLen, __pyx_t_1) < 0) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "iocpsupport.pyx":376 + * return wsa_pi.iAddressFamily + * + * import socket # for WSAStartup # <<<<<<<<<<<<<< + * if not initWinsockPointers(): + * raise ValueError, 'Failed to initialize Winsock function vectors' + */ + __pyx_t_1 = __Pyx_Import(__pyx_n_s_socket, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 376, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_socket, __pyx_t_1) < 0) __PYX_ERR(0, 376, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "iocpsupport.pyx":377 + * + * import socket # for WSAStartup + * if not initWinsockPointers(): # <<<<<<<<<<<<<< + * raise ValueError, 'Failed to initialize Winsock function vectors' + * + */ + __pyx_t_3 = ((!(initWinsockPointers() != 0)) != 0); + if (unlikely(__pyx_t_3)) { + + /* "iocpsupport.pyx":378 + * import socket # for WSAStartup + * if not initWinsockPointers(): + * raise ValueError, 'Failed to initialize Winsock function vectors' # <<<<<<<<<<<<<< + * + * have_connectex = (lpConnectEx != NULL) + */ + __Pyx_Raise(__pyx_builtin_ValueError, __pyx_kp_s_Failed_to_initialize_Winsock_fun, 0, 0); + __PYX_ERR(0, 378, __pyx_L1_error) + + /* "iocpsupport.pyx":377 + * + * import socket # for WSAStartup + * if not initWinsockPointers(): # <<<<<<<<<<<<<< + * raise ValueError, 'Failed to initialize Winsock function vectors' + * + */ + } + + /* "iocpsupport.pyx":380 + * raise ValueError, 'Failed to initialize Winsock function vectors' + * + * have_connectex = (lpConnectEx != NULL) # <<<<<<<<<<<<<< + * + * include 'acceptex.pxi' + */ + __pyx_t_1 = __Pyx_PyBool_FromLong((lpConnectEx != NULL)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 380, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_have_connectex, __pyx_t_1) < 0) __PYX_ERR(0, 380, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":5 + * + * + * def accept(long listening, long accepting, object buff, object obj): # <<<<<<<<<<<<<< + * """ + * CAUTION: unlike system AcceptEx(), this function returns 0 on success + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_5accept, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_accept, __pyx_t_1) < 0) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/acceptex.pxi":30 + * return 0 + * + * def get_accept_addrs(long s, object buff): # <<<<<<<<<<<<<< + * cdef WSAPROTOCOL_INFO wsa_pi + * cdef int locallen, remotelen + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_7get_accept_addrs, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 30, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_get_accept_addrs, __pyx_t_1) < 0) __PYX_ERR(2, 30, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/connectex.pxi":5 + * + * + * def connect(long s, object addr, object obj): # <<<<<<<<<<<<<< + * """ + * CAUTION: unlike system ConnectEx(), this function returns 0 on success + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_9connect, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_connect, __pyx_t_1) < 0) __PYX_ERR(3, 5, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":5 + * + * + * def recv(long s, object bufflist, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc, res + * cdef myOVERLAPPED *ov + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_11recv, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_recv, __pyx_t_1) < 0) __PYX_ERR(4, 5, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsarecv.pxi":38 + * PyMem_Free(ws_buf) + * + * def recvfrom(long s, object buff, object addr_buff, object addr_len_buff, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc, c_addr_buff_len, c_addr_len_buff_len + * cdef myOVERLAPPED *ov + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_13recvfrom, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_recvfrom, __pyx_t_1) < 0) __PYX_ERR(4, 38, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "src/twisted/internet/iocpreactor/iocpsupport/wsasend.pxi":5 + * + * + * def send(long s, object buff, object obj, unsigned long flags = 0): # <<<<<<<<<<<<<< + * cdef int rc + * cdef myOVERLAPPED *ov + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_15send, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(5, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_send, __pyx_t_1) < 0) __PYX_ERR(5, 5, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "(tree fragment)":1 + * def __pyx_unpickle_CompletionPort(__pyx_type, long __pyx_checksum, __pyx_state): # <<<<<<<<<<<<<< + * if __pyx_checksum != 0x901555f: + * from pickle import PickleError as __pyx_PickleError + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_11iocpsupport_17__pyx_unpickle_CompletionPort, NULL, __pyx_n_s_iocpsupport); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_pyx_unpickle_CompletionPort, __pyx_t_1) < 0) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "iocpsupport.pyx":1 + * # Copyright (c) Twisted Matrix Laboratories. # <<<<<<<<<<<<<< + * # See LICENSE for details. + * + */ + __pyx_t_1 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + if (__pyx_m) { + if (__pyx_d) { + __Pyx_AddTraceback("init iocpsupport", 0, __pyx_lineno, __pyx_filename); + } + Py_DECREF(__pyx_m); __pyx_m = 0; + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init iocpsupport"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #elif PY_MAJOR_VERSION >= 3 + return __pyx_m; + #else + return; + #endif +} + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); + if (unlikely(!result)) { + PyErr_Format(PyExc_NameError, +#if PY_MAJOR_VERSION >= 3 + "name '%U' is not defined", name); +#else + "name '%.200s' is not defined", PyString_AS_STRING(name)); +#endif + } + return result; +} + +/* GetModuleGlobalName */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 + result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); + if (likely(result)) { + Py_INCREF(result); + } else if (unlikely(PyErr_Occurred())) { + result = NULL; + } else { +#else + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#endif +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + +/* PyObjectCall */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyErrFetchRestore */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} +#endif + +/* RaiseException */ + #if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + __Pyx_PyThreadState_declare + Py_XINCREF(type); + if (!value || value == Py_None) + value = NULL; + else + Py_INCREF(value); + if (!tb || tb == Py_None) + tb = NULL; + else { + Py_INCREF(tb); + if (!PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + } + if (PyType_Check(type)) { +#if CYTHON_COMPILING_IN_PYPY + if (!value) { + Py_INCREF(Py_None); + value = Py_None; + } +#endif + PyErr_NormalizeException(&type, &value, &tb); + } else { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + value = type; + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + } + __Pyx_PyThreadState_assign + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} +#endif + +/* RaiseArgTupleInvalid */ + static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* RaiseDoubleKeywords */ + static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AsString(kw_name)); + #endif +} + +/* ParseKeywords */ + static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + continue; + } + name = first_kw_arg; + #if PY_MAJOR_VERSION < 3 + if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + while (*name) { + if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) + && _PyString_Eq(**name, key)) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + if ((**argname == key) || ( + (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) + && _PyString_Eq(**argname, key))) { + goto arg_passed_twice; + } + argname++; + } + } + } else + #endif + if (likely(PyUnicode_Check(key))) { + while (*name) { + int cmp = (**name == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**name, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + int cmp = (**argname == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**argname, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + argname++; + } + } + } else + goto invalid_keyword_type; + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +/* PyObjectSetAttrStr */ + #if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_setattro)) + return tp->tp_setattro(obj, attr_name, value); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_setattr)) + return tp->tp_setattr(obj, PyString_AS_STRING(attr_name), value); +#endif + return PyObject_SetAttr(obj, attr_name, value); +} +#endif + +/* UnpackUnboundCMethod */ + static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { + PyObject *method; + method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); + if (unlikely(!method)) + return -1; + target->method = method; +#if CYTHON_COMPILING_IN_CPYTHON + #if PY_MAJOR_VERSION >= 3 + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + #endif + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + } +#endif + return 0; +} + +/* CallUnboundCMethod0 */ + static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + PyObject *args, *result = NULL; + if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_ASSUME_SAFE_MACROS + args = PyTuple_New(1); + if (unlikely(!args)) goto bad; + Py_INCREF(self); + PyTuple_SET_ITEM(args, 0, self); +#else + args = PyTuple_Pack(1, self); + if (unlikely(!args)) goto bad; +#endif + result = __Pyx_PyObject_Call(cfunc->method, args, NULL); + Py_DECREF(args); +bad: + return result; +} + +/* py_dict_items */ + static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) { + if (PY_MAJOR_VERSION >= 3) + return __Pyx_CallUnboundCMethod0(&__pyx_umethod_PyDict_Type_items, d); + else + return PyDict_Items(d); +} + +/* RaiseTooManyValuesToUnpack */ + static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* RaiseNeedMoreValuesToUnpack */ + static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* IterFinish */ + static CYTHON_INLINE int __Pyx_IterFinish(void) { +#if CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* exc_type = tstate->curexc_type; + if (unlikely(exc_type)) { + if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) { + PyObject *exc_value, *exc_tb; + exc_value = tstate->curexc_value; + exc_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + Py_DECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + return 0; + } else { + return -1; + } + } + return 0; +#else + if (unlikely(PyErr_Occurred())) { + if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) { + PyErr_Clear(); + return 0; + } else { + return -1; + } + } + return 0; +#endif +} + +/* UnpackItemEndCheck */ + static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { + if (unlikely(retval)) { + Py_DECREF(retval); + __Pyx_RaiseTooManyValuesError(expected); + return -1; + } else { + return __Pyx_IterFinish(); + } + return 0; +} + +/* KeywordStringCheck */ + static int __Pyx_CheckKeywordStrings( + PyObject *kwdict, + const char* function_name, + int kw_allowed) +{ + PyObject* key = 0; + Py_ssize_t pos = 0; +#if CYTHON_COMPILING_IN_PYPY + if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) + goto invalid_keyword; + return 1; +#else + while (PyDict_Next(kwdict, &pos, &key, 0)) { + #if PY_MAJOR_VERSION < 3 + if (unlikely(!PyString_Check(key))) + #endif + if (unlikely(!PyUnicode_Check(key))) + goto invalid_keyword_type; + } + if ((!kw_allowed) && unlikely(key)) + goto invalid_keyword; + return 1; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + return 0; +#endif +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif + return 0; +} + +/* PyErrExceptionMatches */ + #if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); +#if PY_MAJOR_VERSION >= 3 + for (i=0; icurexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; + if (unlikely(PyTuple_Check(err))) + return __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + return __Pyx_PyErr_GivenExceptionMatches(exc_type, err); +} +#endif + +/* GetAttr */ + static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { +#if CYTHON_USE_TYPE_SLOTS +#if PY_MAJOR_VERSION >= 3 + if (likely(PyUnicode_Check(n))) +#else + if (likely(PyString_Check(n))) +#endif + return __Pyx_PyObject_GetAttrStr(o, n); +#endif + return PyObject_GetAttr(o, n); +} + +/* GetAttr3 */ + static PyObject *__Pyx_GetAttr3Default(PyObject *d) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(d); + return d; +} +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { + PyObject *r = __Pyx_GetAttr(o, n); + return (likely(r)) ? r : __Pyx_GetAttr3Default(d); +} + +/* GetItemInt */ + static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (!j) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((!boundscheck) || likely((0 <= wrapped_i) & (wrapped_i < PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely((0 <= wrapped_i) & (wrapped_i < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((!boundscheck) || (likely((n >= 0) & (n < PyList_GET_SIZE(o))))) { + PyObject *r = PyList_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } + else if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely((n >= 0) & (n < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } else { + PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; + if (likely(m && m->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { + Py_ssize_t l = m->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return m->sq_item(o, i); + } + } +#else + if (is_list || PySequence_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +} + +/* BytesEquals */ + static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals */ + static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY + return PyObject_RichCompareBool(s1, s2, equals); +#else +#if PY_MAJOR_VERSION < 3 + PyObject* owned_ref = NULL; +#endif + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); +#if PY_MAJOR_VERSION < 3 + if ((s1_is_unicode & (!s2_is_unicode)) && PyString_CheckExact(s2)) { + owned_ref = PyUnicode_FromObject(s2); + if (unlikely(!owned_ref)) + return -1; + s2 = owned_ref; + s2_is_unicode = 1; + } else if ((s2_is_unicode & (!s1_is_unicode)) && PyString_CheckExact(s1)) { + owned_ref = PyUnicode_FromObject(s1); + if (unlikely(!owned_ref)) + return -1; + s1 = owned_ref; + s1_is_unicode = 1; + } else if (((!s2_is_unicode) & (!s1_is_unicode))) { + return __Pyx_PyBytes_Equals(s1, s2, equals); + } +#endif + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length; + int kind; + void *data1, *data2; + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + length = __Pyx_PyUnicode_GET_LENGTH(s1); + if (length != __Pyx_PyUnicode_GET_LENGTH(s2)) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + #if CYTHON_PEP393_ENABLED + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + #else + hash1 = ((PyUnicodeObject*)s1)->hash; + hash2 = ((PyUnicodeObject*)s2)->hash; + #endif + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(owned_ref); + #endif + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(owned_ref); + #endif + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(owned_ref); + #endif + return (equals == Py_EQ); +return_ne: + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(owned_ref); + #endif + return (equals == Py_NE); +#endif +} + +/* SliceObject */ + static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice(PyObject* obj, + Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, + int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { +#if CYTHON_USE_TYPE_SLOTS + PyMappingMethods* mp; +#if PY_MAJOR_VERSION < 3 + PySequenceMethods* ms = Py_TYPE(obj)->tp_as_sequence; + if (likely(ms && ms->sq_slice)) { + if (!has_cstart) { + if (_py_start && (*_py_start != Py_None)) { + cstart = __Pyx_PyIndex_AsSsize_t(*_py_start); + if ((cstart == (Py_ssize_t)-1) && PyErr_Occurred()) goto bad; + } else + cstart = 0; + } + if (!has_cstop) { + if (_py_stop && (*_py_stop != Py_None)) { + cstop = __Pyx_PyIndex_AsSsize_t(*_py_stop); + if ((cstop == (Py_ssize_t)-1) && PyErr_Occurred()) goto bad; + } else + cstop = PY_SSIZE_T_MAX; + } + if (wraparound && unlikely((cstart < 0) | (cstop < 0)) && likely(ms->sq_length)) { + Py_ssize_t l = ms->sq_length(obj); + if (likely(l >= 0)) { + if (cstop < 0) { + cstop += l; + if (cstop < 0) cstop = 0; + } + if (cstart < 0) { + cstart += l; + if (cstart < 0) cstart = 0; + } + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + goto bad; + PyErr_Clear(); + } + } + return ms->sq_slice(obj, cstart, cstop); + } +#endif + mp = Py_TYPE(obj)->tp_as_mapping; + if (likely(mp && mp->mp_subscript)) +#endif + { + PyObject* result; + PyObject *py_slice, *py_start, *py_stop; + if (_py_slice) { + py_slice = *_py_slice; + } else { + PyObject* owned_start = NULL; + PyObject* owned_stop = NULL; + if (_py_start) { + py_start = *_py_start; + } else { + if (has_cstart) { + owned_start = py_start = PyInt_FromSsize_t(cstart); + if (unlikely(!py_start)) goto bad; + } else + py_start = Py_None; + } + if (_py_stop) { + py_stop = *_py_stop; + } else { + if (has_cstop) { + owned_stop = py_stop = PyInt_FromSsize_t(cstop); + if (unlikely(!py_stop)) { + Py_XDECREF(owned_start); + goto bad; + } + } else + py_stop = Py_None; + } + py_slice = PySlice_New(py_start, py_stop, Py_None); + Py_XDECREF(owned_start); + Py_XDECREF(owned_stop); + if (unlikely(!py_slice)) goto bad; + } +#if CYTHON_USE_TYPE_SLOTS + result = mp->mp_subscript(obj, py_slice); +#else + result = PyObject_GetItem(obj, py_slice); +#endif + if (!_py_slice) { + Py_DECREF(py_slice); + } + return result; + } + PyErr_Format(PyExc_TypeError, + "'%.200s' object is unsliceable", Py_TYPE(obj)->tp_name); +bad: + return NULL; +} + +/* GetException */ + #if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { +#endif + PyObject *local_type, *local_value, *local_tb; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + #if PY_MAJOR_VERSION >= 3 + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } + #endif + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if PY_VERSION_HEX >= 0x030700A3 + tmp_type = tstate->exc_state.exc_type; + tmp_value = tstate->exc_state.exc_value; + tmp_tb = tstate->exc_state.exc_traceback; + tstate->exc_state.exc_type = local_type; + tstate->exc_state.exc_value = local_value; + tstate->exc_state.exc_traceback = local_tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +} + +/* SwapException */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030700A3 + tmp_type = tstate->exc_state.exc_type; + tmp_value = tstate->exc_state.exc_value; + tmp_tb = tstate->exc_state.exc_traceback; + tstate->exc_state.exc_type = *type; + tstate->exc_state.exc_value = *value; + tstate->exc_state.exc_traceback = *tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + #endif + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* SaveResetException */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if PY_VERSION_HEX >= 0x030700A3 + *type = tstate->exc_state.exc_type; + *value = tstate->exc_state.exc_value; + *tb = tstate->exc_state.exc_traceback; + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + #endif + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030700A3 + tmp_type = tstate->exc_state.exc_type; + tmp_value = tstate->exc_state.exc_value; + tmp_tb = tstate->exc_state.exc_traceback; + tstate->exc_state.exc_type = type; + tstate->exc_state.exc_value = value; + tstate->exc_state.exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +#endif + +/* Import */ + static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + #if PY_MAJOR_VERSION < 3 + PyObject *py_import; + py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); + if (!py_import) + goto bad; + #endif + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + { + #if PY_MAJOR_VERSION >= 3 + if (level == -1) { + if (strchr(__Pyx_MODULE_NAME, '.')) { + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, 1); + if (!module) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + #endif + if (!module) { + #if PY_MAJOR_VERSION < 3 + PyObject *py_level = PyInt_FromLong(level); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, level); + #endif + } + } +bad: + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(py_import); + #endif + Py_XDECREF(empty_list); + Py_XDECREF(empty_dict); + return module; +} + +/* ImportFrom */ + static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Format(PyExc_ImportError, + #if PY_MAJOR_VERSION < 3 + "cannot import name %.230s", PyString_AS_STRING(name)); + #else + "cannot import name %S", name); + #endif + } + return value; +} + +/* PyCFunctionFastCall */ + #if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + int flags = PyCFunction_GET_FLAGS(func); + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS))); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { + return (*((__Pyx_PyCFunctionFastWithKeywords)meth)) (self, args, nargs, NULL); + } else { + return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs); + } +} +#endif + +/* PyFunctionFastCall */ + #if CYTHON_FAST_PYCALL +#include "frameobject.h" +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif +#endif + +/* PyObjectCallMethO */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif + if (likely(PyCFunction_Check(func))) { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif + +/* HasAttr */ + static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!__Pyx_PyBaseString_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_GetAttr(o, n); + if (unlikely(!r)) { + PyErr_Clear(); + return 0; + } else { + Py_DECREF(r); + return 1; + } +} + +/* PyObject_GenericGetAttrNoDict */ + #if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { + PyErr_Format(PyExc_AttributeError, +#if PY_MAJOR_VERSION >= 3 + "'%.50s' object has no attribute '%U'", + tp->tp_name, attr_name); +#else + "'%.50s' object has no attribute '%.400s'", + tp->tp_name, PyString_AS_STRING(attr_name)); +#endif + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { + PyObject *descr; + PyTypeObject *tp = Py_TYPE(obj); + if (unlikely(!PyString_Check(attr_name))) { + return PyObject_GenericGetAttr(obj, attr_name); + } + assert(!tp->tp_dictoffset); + descr = _PyType_Lookup(tp, attr_name); + if (unlikely(!descr)) { + return __Pyx_RaiseGenericGetAttributeError(tp, attr_name); + } + Py_INCREF(descr); + #if PY_MAJOR_VERSION < 3 + if (likely(PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_HAVE_CLASS))) + #endif + { + descrgetfunc f = Py_TYPE(descr)->tp_descr_get; + if (unlikely(f)) { + PyObject *res = f(descr, obj, (PyObject *)tp); + Py_DECREF(descr); + return res; + } + } + return descr; +} +#endif + +/* PyObject_GenericGetAttr */ + #if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject* __Pyx_PyObject_GenericGetAttr(PyObject* obj, PyObject* attr_name) { + if (unlikely(Py_TYPE(obj)->tp_dictoffset)) { + return PyObject_GenericGetAttr(obj, attr_name); + } + return __Pyx_PyObject_GenericGetAttrNoDict(obj, attr_name); +} +#endif + +/* SetupReduce */ + static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + name_attr = __Pyx_PyObject_GetAttrStr(meth, __pyx_n_s_name); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + Py_XDECREF(name_attr); + return ret; +} +static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; +#if CYTHON_USE_PYTYPE_LOOKUP + if (_PyType_Lookup((PyTypeObject*)type_obj, __pyx_n_s_getstate)) goto GOOD; +#else + if (PyObject_HasAttr(type_obj, __pyx_n_s_getstate)) goto GOOD; +#endif +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_n_s_reduce_ex); if (!object_reduce_ex) goto BAD; +#else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_n_s_reduce_ex); if (!object_reduce_ex) goto BAD; +#endif + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce_ex); if (unlikely(!reduce_ex)) goto BAD; + if (reduce_ex == object_reduce_ex) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_n_s_reduce); if (!object_reduce) goto BAD; +#else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_n_s_reduce); if (!object_reduce) goto BAD; +#endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce); if (unlikely(!reduce)) goto BAD; + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_n_s_reduce_cython)) { + reduce_cython = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce_cython); if (unlikely(!reduce_cython)) goto BAD; + ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_reduce, reduce_cython); if (unlikely(ret < 0)) goto BAD; + ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_reduce_cython); if (unlikely(ret < 0)) goto BAD; + setstate = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_setstate); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_n_s_setstate_cython)) { + setstate_cython = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_setstate_cython); if (unlikely(!setstate_cython)) goto BAD; + ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_setstate, setstate_cython); if (unlikely(ret < 0)) goto BAD; + ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_setstate_cython); if (unlikely(ret < 0)) goto BAD; + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto GOOD; +BAD: + if (!PyErr_Occurred()) + PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name); + ret = -1; +GOOD: +#if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); +#endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; +} + +/* FetchCommonType */ + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* fake_module; + PyTypeObject* cached_type = NULL; + fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); + if (!fake_module) return NULL; + Py_INCREF(fake_module); + cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + if (cached_type) { + if (!PyType_Check((PyObject*)cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", + type->tp_name); + goto bad; + } + if (cached_type->tp_basicsize != type->tp_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + type->tp_name); + goto bad; + } + } else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + } +done: + Py_DECREF(fake_module); + return cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CythonFunction */ + #include +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) +{ + if (unlikely(op->func_doc == NULL)) { + if (op->func.m_ml->ml_doc) { +#if PY_MAJOR_VERSION >= 3 + op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); +#else + op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); +#endif + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp = op->func_doc; + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + op->func_doc = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if PY_MAJOR_VERSION >= 3 + op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); +#else + op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + tmp = op->func_name; + Py_INCREF(value); + op->func_name = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op) +{ + Py_INCREF(op->func_qualname); + return op->func_qualname; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + tmp = op->func_qualname; + Py_INCREF(value); + op->func_qualname = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) +{ + PyObject *self; + self = m->func_closure; + if (self == NULL) + self = Py_None; + Py_INCREF(self); + return self; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + tmp = op->func_dict; + Py_INCREF(value); + op->func_dict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op) +{ + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op) +{ + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value) { + value = Py_None; + } else if (value != Py_None && !PyTuple_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + Py_INCREF(value); + tmp = op->defaults_tuple; + op->defaults_tuple = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value) { + value = Py_None; + } else if (value != Py_None && !PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + Py_INCREF(value); + tmp = op->defaults_kwdict; + op->defaults_kwdict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value || value == Py_None) { + value = NULL; + } else if (!PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + tmp = op->func_annotations; + op->func_annotations = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {(char *) "func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, + {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { + {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromString(m->func.m_ml->ml_name); +#else + return PyString_FromString(m->func.m_ml->ml_name); +#endif +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if PY_VERSION_HEX < 0x030500A0 +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); + if (op == NULL) + return NULL; + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; + op->func.m_ml = ml; + op->func.m_self = (PyObject *) op; + Py_XINCREF(closure); + op->func_closure = closure; + Py_XINCREF(module); + op->func.m_module = module; + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; + op->func_classobj = NULL; + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults_pyobjects = 0; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + PyObject_GC_Track(op); + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); + Py_CLEAR(m->func.m_module); + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); + Py_CLEAR(m->func_classobj); + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_XDECREF(pydefaults[i]); + PyObject_Free(m->defaults); + m->defaults = NULL; + } + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + PyObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + Py_VISIT(m->func_closure); + Py_VISIT(m->func.m_module); + Py_VISIT(m->func_dict); + Py_VISIT(m->func_name); + Py_VISIT(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + Py_VISIT(m->func_code); + Py_VISIT(m->func_classobj); + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_VISIT(pydefaults[i]); + } + return 0; +} +static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) +{ + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { + Py_INCREF(func); + return func; + } + if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { + if (type == NULL) + type = (PyObject *)(Py_TYPE(obj)); + return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); + } + if (obj == Py_None) + obj = NULL; + return __Pyx_PyMethod_New(func, obj, type); +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromFormat("", + op->func_qualname, (void *)op); +#else + return PyString_FromFormat("", + PyString_AsString(op->func_qualname), (void *)op); +#endif +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + Py_ssize_t size; + switch (f->m_ml->ml_flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + size = PyTuple_GET_SIZE(arg); + if (likely(size == 0)) + return (*meth)(self, NULL); + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", + f->m_ml->ml_name, size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + size = PyTuple_GET_SIZE(arg); + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", + f->m_ml->ml_name, size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags in " + "__Pyx_CyFunction_Call. METH_OLDARGS is no " + "longer supported!"); + return NULL; + } + PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", + f->m_ml->ml_name); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + return __Pyx_CyFunction_CallMethod(func, ((PyCFunctionObject*)func)->m_self, arg, kw); +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; + argc = PyTuple_GET_SIZE(args); + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +static PyTypeObject __pyx_CyFunctionType_type = { + PyVarObject_HEAD_INIT(0, 0) + "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, + (destructor) __Pyx_CyFunction_dealloc, + 0, + 0, + 0, +#if PY_MAJOR_VERSION < 3 + 0, +#else + 0, +#endif + (reprfunc) __Pyx_CyFunction_repr, + 0, + 0, + 0, + 0, + __Pyx_CyFunction_CallAsMethod, + 0, + 0, + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + 0, + (traverseproc) __Pyx_CyFunction_traverse, + (inquiry) __Pyx_CyFunction_clear, + 0, +#if PY_VERSION_HEX < 0x030500A0 + offsetof(__pyx_CyFunctionObject, func_weakreflist), +#else + offsetof(PyCFunctionObject, m_weakreflist), +#endif + 0, + 0, + __pyx_CyFunction_methods, + __pyx_CyFunction_members, + __pyx_CyFunction_getsets, + 0, + 0, + __Pyx_CyFunction_descr_get, + 0, + offsetof(__pyx_CyFunctionObject, func_dict), + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, +#if PY_VERSION_HEX >= 0x030400a1 + 0, +#endif +}; +static int __pyx_CyFunction_init(void) { + __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); + if (unlikely(__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_Malloc(size); + if (unlikely(!m->defaults)) + return PyErr_NoMemory(); + memset(m->defaults, 0, size); + m->defaults_pyobjects = pyobjects; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CalculateMetaclass */ + static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases) { + Py_ssize_t i, nbases = PyTuple_GET_SIZE(bases); + for (i=0; i < nbases; i++) { + PyTypeObject *tmptype; + PyObject *tmp = PyTuple_GET_ITEM(bases, i); + tmptype = Py_TYPE(tmp); +#if PY_MAJOR_VERSION < 3 + if (tmptype == &PyClass_Type) + continue; +#endif + if (!metaclass) { + metaclass = tmptype; + continue; + } + if (PyType_IsSubtype(metaclass, tmptype)) + continue; + if (PyType_IsSubtype(tmptype, metaclass)) { + metaclass = tmptype; + continue; + } + PyErr_SetString(PyExc_TypeError, + "metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases"); + return NULL; + } + if (!metaclass) { +#if PY_MAJOR_VERSION < 3 + metaclass = &PyClass_Type; +#else + metaclass = &PyType_Type; +#endif + } + Py_INCREF((PyObject*) metaclass); + return (PyObject*) metaclass; +} + +/* Py3ClassCreate */ + static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, + PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { + PyObject *ns; + if (metaclass) { + PyObject *prep = __Pyx_PyObject_GetAttrStr(metaclass, __pyx_n_s_prepare); + if (prep) { + PyObject *pargs = PyTuple_Pack(2, name, bases); + if (unlikely(!pargs)) { + Py_DECREF(prep); + return NULL; + } + ns = PyObject_Call(prep, pargs, mkw); + Py_DECREF(prep); + Py_DECREF(pargs); + } else { + if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + PyErr_Clear(); + ns = PyDict_New(); + } + } else { + ns = PyDict_New(); + } + if (unlikely(!ns)) + return NULL; + if (unlikely(PyObject_SetItem(ns, __pyx_n_s_module, modname) < 0)) goto bad; + if (unlikely(PyObject_SetItem(ns, __pyx_n_s_qualname, qualname) < 0)) goto bad; + if (unlikely(doc && PyObject_SetItem(ns, __pyx_n_s_doc, doc) < 0)) goto bad; + return ns; +bad: + Py_DECREF(ns); + return NULL; +} +static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, + PyObject *dict, PyObject *mkw, + int calculate_metaclass, int allow_py2_metaclass) { + PyObject *result, *margs; + PyObject *owned_metaclass = NULL; + if (allow_py2_metaclass) { + owned_metaclass = PyObject_GetItem(dict, __pyx_n_s_metaclass); + if (owned_metaclass) { + metaclass = owned_metaclass; + } else if (likely(PyErr_ExceptionMatches(PyExc_KeyError))) { + PyErr_Clear(); + } else { + return NULL; + } + } + if (calculate_metaclass && (!metaclass || PyType_Check(metaclass))) { + metaclass = __Pyx_CalculateMetaclass((PyTypeObject*) metaclass, bases); + Py_XDECREF(owned_metaclass); + if (unlikely(!metaclass)) + return NULL; + owned_metaclass = metaclass; + } + margs = PyTuple_Pack(3, name, bases, dict); + if (unlikely(!margs)) { + result = NULL; + } else { + result = PyObject_Call(metaclass, margs, mkw); + Py_DECREF(margs); + } + Py_XDECREF(owned_metaclass); + return result; +} + +/* CLineInTraceback */ + #ifndef CYTHON_CLINE_IN_TRACEBACK +static int __Pyx_CLineForTraceback(CYTHON_UNUSED PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + if (unlikely(!__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + use_cline = __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_n_s_cline_in_traceback); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + PyObject_SetAttr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback, Py_False); + } + else if (PyObject_Not(use_cline) != 0) { + c_line = 0; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); +} + +/* AddTraceback */ + #include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, + 0, + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + } +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_long(unsigned long value) { + const unsigned long neg_one = (unsigned long) -1, const_zero = (unsigned long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(unsigned long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(unsigned long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(unsigned long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(unsigned long), + little, !is_unsigned); + } +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_short(unsigned short value) { + const unsigned short neg_one = (unsigned short) -1, const_zero = (unsigned short) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(unsigned short) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(unsigned short) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned short) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(unsigned short) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned short) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(unsigned short), + little, !is_unsigned); + } +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + +/* CIntFromPy */ + static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (size_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) + case -2: + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } +#endif + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + size_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (size_t) -1; + } + } else { + size_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case -2: + if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (long) -1; + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (long) -1; + val = __Pyx_PyInt_As_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE unsigned long __Pyx_PyInt_As_unsigned_long(PyObject *x) { + const unsigned long neg_one = (unsigned long) -1, const_zero = (unsigned long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(unsigned long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(unsigned long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (unsigned long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (unsigned long) 0; + case 1: __PYX_VERIFY_RETURN_INT(unsigned long, digit, digits[0]) + case 2: + if (8 * sizeof(unsigned long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) >= 2 * PyLong_SHIFT) { + return (unsigned long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) >= 3 * PyLong_SHIFT) { + return (unsigned long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) >= 4 * PyLong_SHIFT) { + return (unsigned long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (unsigned long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(unsigned long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (unsigned long) 0; + case -1: __PYX_VERIFY_RETURN_INT(unsigned long, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(unsigned long, digit, +digits[0]) + case -2: + if (8 * sizeof(unsigned long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) - 1 > 2 * PyLong_SHIFT) { + return (unsigned long) (((unsigned long)-1)*(((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(unsigned long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) - 1 > 2 * PyLong_SHIFT) { + return (unsigned long) ((((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(unsigned long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) - 1 > 3 * PyLong_SHIFT) { + return (unsigned long) (((unsigned long)-1)*(((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) - 1 > 3 * PyLong_SHIFT) { + return (unsigned long) ((((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(unsigned long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) - 1 > 4 * PyLong_SHIFT) { + return (unsigned long) (((unsigned long)-1)*(((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned long) - 1 > 4 * PyLong_SHIFT) { + return (unsigned long) ((((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(unsigned long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + unsigned long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (unsigned long) -1; + } + } else { + unsigned long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (unsigned long) -1; + val = __Pyx_PyInt_As_unsigned_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned long"); + return (unsigned long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE unsigned short __Pyx_PyInt_As_unsigned_short(PyObject *x) { + const unsigned short neg_one = (unsigned short) -1, const_zero = (unsigned short) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(unsigned short) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(unsigned short, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (unsigned short) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (unsigned short) 0; + case 1: __PYX_VERIFY_RETURN_INT(unsigned short, digit, digits[0]) + case 2: + if (8 * sizeof(unsigned short) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) >= 2 * PyLong_SHIFT) { + return (unsigned short) (((((unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(unsigned short) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) >= 3 * PyLong_SHIFT) { + return (unsigned short) (((((((unsigned short)digits[2]) << PyLong_SHIFT) | (unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(unsigned short) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) >= 4 * PyLong_SHIFT) { + return (unsigned short) (((((((((unsigned short)digits[3]) << PyLong_SHIFT) | (unsigned short)digits[2]) << PyLong_SHIFT) | (unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (unsigned short) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(unsigned short) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned short, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned short) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned short, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (unsigned short) 0; + case -1: __PYX_VERIFY_RETURN_INT(unsigned short, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(unsigned short, digit, +digits[0]) + case -2: + if (8 * sizeof(unsigned short) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) - 1 > 2 * PyLong_SHIFT) { + return (unsigned short) (((unsigned short)-1)*(((((unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(unsigned short) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) - 1 > 2 * PyLong_SHIFT) { + return (unsigned short) ((((((unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(unsigned short) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) - 1 > 3 * PyLong_SHIFT) { + return (unsigned short) (((unsigned short)-1)*(((((((unsigned short)digits[2]) << PyLong_SHIFT) | (unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(unsigned short) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) - 1 > 3 * PyLong_SHIFT) { + return (unsigned short) ((((((((unsigned short)digits[2]) << PyLong_SHIFT) | (unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(unsigned short) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) - 1 > 4 * PyLong_SHIFT) { + return (unsigned short) (((unsigned short)-1)*(((((((((unsigned short)digits[3]) << PyLong_SHIFT) | (unsigned short)digits[2]) << PyLong_SHIFT) | (unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(unsigned short) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned short, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned short) - 1 > 4 * PyLong_SHIFT) { + return (unsigned short) ((((((((((unsigned short)digits[3]) << PyLong_SHIFT) | (unsigned short)digits[2]) << PyLong_SHIFT) | (unsigned short)digits[1]) << PyLong_SHIFT) | (unsigned short)digits[0]))); + } + } + break; + } +#endif + if (sizeof(unsigned short) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned short, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned short) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned short, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + unsigned short val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (unsigned short) -1; + } + } else { + unsigned short val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (unsigned short) -1; + val = __Pyx_PyInt_As_unsigned_short(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned short"); + return (unsigned short) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned short"); + return (unsigned short) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case -2: + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int) -1; + } + } else { + int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FastTypeChecks */ + #if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = a->tp_base; + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +#if PY_MAJOR_VERSION == 2 +static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { + PyObject *exception, *value, *tb; + int res; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&exception, &value, &tb); + res = exc_type1 ? PyObject_IsSubclass(err, exc_type1) : 0; + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + if (!res) { + res = PyObject_IsSubclass(err, exc_type2); + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + } + __Pyx_ErrRestore(exception, value, tb); + return res; +} +#else +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; + if (!res) { + res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } + return res; +} +#endif +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); +#if PY_MAJOR_VERSION >= 3 + for (i=0; ip) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else + if (t->is_unicode | t->is_str) { + if (t->intern) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); + } else { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + if (PyObject_Hash(*t->p) == -1) + return -1; + ++t; + } + return 0; +} + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +#if !CYTHON_PEP393_ENABLED +static const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; + } + } + } +#endif + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +} +#else +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +} +#endif +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif +#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) + if (PyByteArray_Check(o)) { + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); + } else +#endif + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { +#if PY_MAJOR_VERSION >= 3 + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type %.200s). " + "The ability to return an instance of a strict subclass of int " + "is deprecated, and may be removed in a future version of Python.", + Py_TYPE(result)->tp_name)) { + Py_DECREF(result); + return NULL; + } + return result; + } +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + type_name, type_name, Py_TYPE(result)->tp_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + const char *name = NULL; + PyObject *res = NULL; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x) || PyLong_Check(x))) +#else + if (likely(PyLong_Check(x))) +#endif + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + #if PY_MAJOR_VERSION < 3 + if (m && m->nb_int) { + name = "int"; + res = m->nb_int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = m->nb_long(x); + } + #else + if (likely(m && m->nb_int)) { + name = "int"; + res = m->nb_int(x); + } + #endif +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Int(x); + } +#endif + if (likely(res)) { +#if PY_MAJOR_VERSION < 3 + if (unlikely(!PyInt_Check(res) && !PyLong_Check(res))) { +#else + if (unlikely(!PyLong_CheckExact(res))) { +#endif + return __Pyx_PyNumber_IntOrLongWrongResultType(res, name); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(b))) { + if (sizeof(Py_ssize_t) >= sizeof(long)) + return PyInt_AS_LONG(b); + else + return PyInt_AsSsize_t(x); + } +#endif + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)b)->ob_digit; + const Py_ssize_t size = Py_SIZE(b); + if (likely(__Pyx_sst_abs(size) <= 1)) { + ival = likely(size) ? digits[0] : 0; + if (size == -1) ival = -ival; + return ival; + } else { + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { + return PyInt_FromSize_t(ival); +} + + +#endif /* Py_PYTHON_H */ diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c new file mode 100644 index 00000000000..07bbfab0d0b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c @@ -0,0 +1,62 @@ +/* Copyright (c) 2008 Twisted Matrix Laboratories. + * See LICENSE for details. + */ + + +#include +#include +#include +#include + +#include "winsock_pointers.h" + +#ifndef WSAID_CONNECTEX +#define WSAID_CONNECTEX {0x25a207b9,0xddf3,0x4660,{0x8e,0xe9,0x76,0xe5,0x8c,0x74,0x06,0x3e}} +#endif +#ifndef WSAID_GETACCEPTEXSOCKADDRS +#define WSAID_GETACCEPTEXSOCKADDRS {0xb5367df2,0xcbac,0x11cf,{0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92}} +#endif +#ifndef WSAID_ACCEPTEX +#define WSAID_ACCEPTEX {0xb5367df1,0xcbac,0x11cf,{0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92}} +#endif +/*#ifndef WSAID_TRANSMITFILE +#define WSAID_TRANSMITFILE {0xb5367df0,0xcbac,0x11cf,{0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92}} +#endif*/ + + +int initPointer(SOCKET s, void **fun, GUID guid) { + int res; + DWORD bytes; + + *fun = NULL; + res = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, + &guid, sizeof(guid), + fun, sizeof(fun), + &bytes, NULL, NULL); + return !res; +} + +int initWinsockPointers() { + SOCKET s = socket(AF_INET, SOCK_STREAM, 0); + /* I hate C */ + GUID guid1 = WSAID_ACCEPTEX; + GUID guid2 = WSAID_GETACCEPTEXSOCKADDRS; + GUID guid3 = WSAID_CONNECTEX; + /*GUID guid4 = WSAID_TRANSMITFILE;*/ + if (!s) { + return 0; + } + if (!initPointer(s, (void **)&lpAcceptEx, guid1)) + { + return 0; + } + if (!initPointer(s, (void **)&lpGetAcceptExSockaddrs, guid2)) { + return 0; + } + if (!initPointer(s, (void **)&lpConnectEx, guid3)) { + return 0; + }; + /*initPointer(s, &lpTransmitFile, guid4);*/ + return 1; +} + diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.h b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.h new file mode 100644 index 00000000000..7301756ce7b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2008 Twisted Matrix Laboratories. + * See LICENSE for details. + */ + + +#include + +int initWinsockPointers(); +BOOL +(PASCAL FAR * lpAcceptEx)( + IN SOCKET sListenSocket, + IN SOCKET sAcceptSocket, + IN PVOID lpOutputBuffer, + IN DWORD dwReceiveDataLength, + IN DWORD dwLocalAddressLength, + IN DWORD dwRemoteAddressLength, + OUT LPDWORD lpdwBytesReceived, + IN LPOVERLAPPED lpOverlapped + ); +VOID +(PASCAL FAR * lpGetAcceptExSockaddrs)( + IN PVOID lpOutputBuffer, + IN DWORD dwReceiveDataLength, + IN DWORD dwLocalAddressLength, + IN DWORD dwRemoteAddressLength, + OUT struct sockaddr **LocalSockaddr, + OUT LPINT LocalSockaddrLength, + OUT struct sockaddr **RemoteSockaddr, + OUT LPINT RemoteSockaddrLength + ); +BOOL +(PASCAL FAR * lpConnectEx) ( + IN SOCKET s, + IN const struct sockaddr FAR *name, + IN int namelen, + IN PVOID lpSendBuffer OPTIONAL, + IN DWORD dwSendDataLength, + OUT LPDWORD lpdwBytesSent, + IN LPOVERLAPPED lpOverlapped + ); +/*BOOL +(PASCAL FAR * lpTransmitFile)( + IN SOCKET hSocket, + IN HANDLE hFile, + IN DWORD nNumberOfBytesToWrite, + IN DWORD nNumberOfBytesPerSend, + IN LPOVERLAPPED lpOverlapped, + IN LPTRANSMIT_FILE_BUFFERS lpTransmitBuffers, + IN DWORD dwReserved + );*/ + diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/notes.txt b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/notes.txt new file mode 100644 index 00000000000..4caffb882f1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/notes.txt @@ -0,0 +1,24 @@ +test specifically: +failed accept error message -- similar to test_tcp_internals +immediate success on accept/connect/recv, including Event.ignore +parametrize iocpsupport somehow -- via reactor? + +do: +break handling -- WaitForSingleObject on the IOCP handle? +iovecs for write buffer +do not wait for a mainloop iteration if resumeProducing (in _handleWrite) does startWriting +don't addActiveHandle in every call to startWriting/startReading +iocpified process support + win32er-in-a-thread (or run GQCS in a thread -- it can't receive SIGBREAK) +blocking in sendto() -- I think Windows can do that, especially with local UDP + +buildbot: +run in vmware +start from a persistent snapshot + +use a stub inside the vm to svnup/run tests/collect stdio +lift logs through SMB? or ship them via tcp beams to the VM host + +have a timeout on the test run +if we time out, take a screenshot, save it, kill the VM + diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/reactor.py b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/reactor.py new file mode 100644 index 00000000000..394477382ed --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/reactor.py @@ -0,0 +1,273 @@ +# -*- test-case-name: twisted.internet.test.test_iocp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Reactor that uses IO completion ports +""" + +import warnings, socket, sys + +from zope.interface import implementer + +from twisted.internet import base, interfaces, main, error +from twisted.python import log, failure +from twisted.internet._dumbwin32proc import Process +from twisted.internet.win32eventreactor import _ThreadedWin32EventsMixin + +from twisted.internet.iocpreactor import iocpsupport as _iocp +from twisted.internet.iocpreactor.const import WAIT_TIMEOUT +from twisted.internet.iocpreactor import tcp, udp + +try: + from twisted.protocols.tls import TLSMemoryBIOFactory +except ImportError: + # Either pyOpenSSL isn't installed, or it is too old for this code to work. + # The reactor won't provide IReactorSSL. + TLSMemoryBIOFactory = None + _extraInterfaces = () + warnings.warn( + "pyOpenSSL 0.10 or newer is required for SSL support in iocpreactor. " + "It is missing, so the reactor will not support SSL APIs.") +else: + _extraInterfaces = (interfaces.IReactorSSL,) + +MAX_TIMEOUT = 2000 # 2 seconds, see doIteration for explanation + +EVENTS_PER_LOOP = 1000 # XXX: what's a good value here? + +# keys to associate with normal and waker events +KEY_NORMAL, KEY_WAKEUP = range(2) + +_NO_GETHANDLE = error.ConnectionFdescWentAway( + 'Handler has no getFileHandle method') +_NO_FILEDESC = error.ConnectionFdescWentAway('Filedescriptor went away') + + + +@implementer(interfaces.IReactorTCP, interfaces.IReactorUDP, + interfaces.IReactorMulticast, interfaces.IReactorProcess, + *_extraInterfaces) +class IOCPReactor(base._SignalReactorMixin, base.ReactorBase, + _ThreadedWin32EventsMixin): + + + port = None + + def __init__(self): + base.ReactorBase.__init__(self) + self.port = _iocp.CompletionPort() + self.handles = set() + + + def addActiveHandle(self, handle): + self.handles.add(handle) + + + def removeActiveHandle(self, handle): + self.handles.discard(handle) + + + def doIteration(self, timeout): + """ + Poll the IO completion port for new events. + """ + # This function sits and waits for an IO completion event. + # + # There are two requirements: process IO events as soon as they arrive + # and process ctrl-break from the user in a reasonable amount of time. + # + # There are three kinds of waiting. + # 1) GetQueuedCompletionStatus (self.port.getEvent) to wait for IO + # events only. + # 2) Msg* family of wait functions that can stop waiting when + # ctrl-break is detected (then, I think, Python converts it into a + # KeyboardInterrupt) + # 3) *Ex family of wait functions that put the thread into an + # "alertable" wait state which is supposedly triggered by IO completion + # + # 2) and 3) can be combined. Trouble is, my IO completion is not + # causing 3) to trigger, possibly because I do not use an IO completion + # callback. Windows is weird. + # There are two ways to handle this. I could use MsgWaitForSingleObject + # here and GetQueuedCompletionStatus in a thread. Or I could poll with + # a reasonable interval. Guess what! Threads are hard. + + processed_events = 0 + if timeout is None: + timeout = MAX_TIMEOUT + else: + timeout = min(MAX_TIMEOUT, int(1000*timeout)) + rc, numBytes, key, evt = self.port.getEvent(timeout) + while 1: + if rc == WAIT_TIMEOUT: + break + if key != KEY_WAKEUP: + assert key == KEY_NORMAL + log.callWithLogger(evt.owner, self._callEventCallback, + rc, numBytes, evt) + processed_events += 1 + if processed_events >= EVENTS_PER_LOOP: + break + rc, numBytes, key, evt = self.port.getEvent(0) + + + def _callEventCallback(self, rc, numBytes, evt): + owner = evt.owner + why = None + try: + evt.callback(rc, numBytes, evt) + handfn = getattr(owner, 'getFileHandle', None) + if not handfn: + why = _NO_GETHANDLE + elif handfn() == -1: + why = _NO_FILEDESC + if why: + return # ignore handles that were closed + except: + why = sys.exc_info()[1] + log.err() + if why: + owner.loseConnection(failure.Failure(why)) + + + def installWaker(self): + pass + + + def wakeUp(self): + self.port.postEvent(0, KEY_WAKEUP, None) + + + def registerHandle(self, handle): + self.port.addHandle(handle, KEY_NORMAL) + + + def createSocket(self, af, stype): + skt = socket.socket(af, stype) + self.registerHandle(skt.fileno()) + return skt + + + def listenTCP(self, port, factory, backlog=50, interface=''): + """ + @see: twisted.internet.interfaces.IReactorTCP.listenTCP + """ + p = tcp.Port(port, factory, backlog, interface, self) + p.startListening() + return p + + + def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): + """ + @see: twisted.internet.interfaces.IReactorTCP.connectTCP + """ + c = tcp.Connector(host, port, factory, timeout, bindAddress, self) + c.connect() + return c + + + if TLSMemoryBIOFactory is not None: + def listenSSL(self, port, factory, contextFactory, backlog=50, interface=''): + """ + @see: twisted.internet.interfaces.IReactorSSL.listenSSL + """ + port = self.listenTCP( + port, + TLSMemoryBIOFactory(contextFactory, False, factory), + backlog, interface) + port._type = 'TLS' + return port + + + def connectSSL(self, host, port, factory, contextFactory, timeout=30, bindAddress=None): + """ + @see: twisted.internet.interfaces.IReactorSSL.connectSSL + """ + return self.connectTCP( + host, port, + TLSMemoryBIOFactory(contextFactory, True, factory), + timeout, bindAddress) + else: + def listenSSL(self, port, factory, contextFactory, backlog=50, interface=''): + """ + Non-implementation of L{IReactorSSL.listenSSL}. Some dependency + is not satisfied. This implementation always raises + L{NotImplementedError}. + """ + raise NotImplementedError( + "pyOpenSSL 0.10 or newer is required for SSL support in " + "iocpreactor. It is missing, so the reactor does not support " + "SSL APIs.") + + + def connectSSL(self, host, port, factory, contextFactory, timeout=30, bindAddress=None): + """ + Non-implementation of L{IReactorSSL.connectSSL}. Some dependency + is not satisfied. This implementation always raises + L{NotImplementedError}. + """ + raise NotImplementedError( + "pyOpenSSL 0.10 or newer is required for SSL support in " + "iocpreactor. It is missing, so the reactor does not support " + "SSL APIs.") + + + def listenUDP(self, port, protocol, interface='', maxPacketSize=8192): + """ + Connects a given L{DatagramProtocol} to the given numeric UDP port. + + @returns: object conforming to L{IListeningPort}. + """ + p = udp.Port(port, protocol, interface, maxPacketSize, self) + p.startListening() + return p + + + def listenMulticast(self, port, protocol, interface='', maxPacketSize=8192, + listenMultiple=False): + """ + Connects a given DatagramProtocol to the given numeric UDP port. + + EXPERIMENTAL. + + @returns: object conforming to IListeningPort. + """ + p = udp.MulticastPort(port, protocol, interface, maxPacketSize, self, + listenMultiple) + p.startListening() + return p + + + def spawnProcess(self, processProtocol, executable, args=(), env={}, + path=None, uid=None, gid=None, usePTY=0, childFDs=None): + """ + Spawn a process. + """ + if uid is not None: + raise ValueError("Setting UID is unsupported on this platform.") + if gid is not None: + raise ValueError("Setting GID is unsupported on this platform.") + if usePTY: + raise ValueError("PTYs are unsupported on this platform.") + if childFDs is not None: + raise ValueError( + "Custom child file descriptor mappings are unsupported on " + "this platform.") + args, env = self._checkProcessArgs(args, env) + return Process(self, processProtocol, executable, args, env, path) + + + def removeAll(self): + res = list(self.handles) + self.handles.clear() + return res + + + +def install(): + r = IOCPReactor() + main.installReactor(r) + + +__all__ = ['IOCPReactor', 'install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/setup.py b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/setup.py new file mode 100644 index 00000000000..b110fc53946 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/setup.py @@ -0,0 +1,23 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Distutils file for building low-level IOCP bindings from their Pyrex source +""" + + +from distutils.core import setup +from distutils.extension import Extension +from Cython.Distutils import build_ext + +setup(name='iocpsupport', + ext_modules=[Extension('iocpsupport', + ['iocpsupport/iocpsupport.pyx', + 'iocpsupport/winsock_pointers.c'], + libraries = ['ws2_32'], + ) + ], + cmdclass = {'build_ext': build_ext}, + ) + diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/tcp.py b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/tcp.py new file mode 100644 index 00000000000..0af92ba9fe2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/tcp.py @@ -0,0 +1,617 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +TCP support for IOCP reactor +""" + +import socket, operator, errno, struct + +from zope.interface import implementer, classImplements + +from twisted.internet import interfaces, error, address, main, defer +from twisted.internet.protocol import Protocol +from twisted.internet.abstract import _LogOwner, isIPv6Address +from twisted.internet.tcp import ( + _SocketCloser, Connector as TCPConnector, _AbortingMixin, _BaseBaseClient, + _BaseTCPClient, _resolveIPv6, _getsockname) +from twisted.python import log, failure, reflect +from twisted.python.compat import _PY3, nativeString + +from twisted.internet.iocpreactor import iocpsupport as _iocp, abstract +from twisted.internet.iocpreactor.interfaces import IReadWriteHandle +from twisted.internet.iocpreactor.const import ERROR_IO_PENDING +from twisted.internet.iocpreactor.const import SO_UPDATE_CONNECT_CONTEXT +from twisted.internet.iocpreactor.const import SO_UPDATE_ACCEPT_CONTEXT +from twisted.internet.iocpreactor.const import ERROR_CONNECTION_REFUSED +from twisted.internet.iocpreactor.const import ERROR_NETWORK_UNREACHABLE + +try: + from twisted.internet._newtls import startTLS as _startTLS +except ImportError: + _startTLS = None + +# ConnectEx returns these. XXX: find out what it does for timeout +connectExErrors = { + ERROR_CONNECTION_REFUSED: errno.WSAECONNREFUSED, + ERROR_NETWORK_UNREACHABLE: errno.WSAENETUNREACH, + } + +@implementer(IReadWriteHandle, interfaces.ITCPTransport, + interfaces.ISystemHandle) +class Connection(abstract.FileHandle, _SocketCloser, _AbortingMixin): + """ + @ivar TLS: C{False} to indicate the connection is in normal TCP mode, + C{True} to indicate that TLS has been started and that operations must + be routed through the L{TLSMemoryBIOProtocol} instance. + """ + TLS = False + + + def __init__(self, sock, proto, reactor=None): + abstract.FileHandle.__init__(self, reactor) + self.socket = sock + self.getFileHandle = sock.fileno + self.protocol = proto + + + def getHandle(self): + return self.socket + + + def dataReceived(self, rbuffer): + """ + @param rbuffer: Data received. + @type rbuffer: L{bytes} or L{bytearray} + """ + if isinstance(rbuffer, bytes): + pass + elif isinstance(rbuffer, bytearray): + # XXX: some day, we'll have protocols that can handle raw buffers + rbuffer = bytes(rbuffer) + else: + raise TypeError("data must be bytes or bytearray, not " + + type(rbuffer)) + + self.protocol.dataReceived(rbuffer) + + + def readFromHandle(self, bufflist, evt): + return _iocp.recv(self.getFileHandle(), bufflist, evt) + + + def writeToHandle(self, buff, evt): + """ + Send C{buff} to current file handle using C{_iocp.send}. The buffer + sent is limited to a size of C{self.SEND_LIMIT}. + """ + writeView = memoryview(buff) + return _iocp.send(self.getFileHandle(), + writeView[0:self.SEND_LIMIT].tobytes(), evt) + + + def _closeWriteConnection(self): + try: + self.socket.shutdown(1) + except socket.error: + pass + p = interfaces.IHalfCloseableProtocol(self.protocol, None) + if p: + try: + p.writeConnectionLost() + except: + f = failure.Failure() + log.err() + self.connectionLost(f) + + + def readConnectionLost(self, reason): + p = interfaces.IHalfCloseableProtocol(self.protocol, None) + if p: + try: + p.readConnectionLost() + except: + log.err() + self.connectionLost(failure.Failure()) + else: + self.connectionLost(reason) + + + def connectionLost(self, reason): + if self.disconnected: + return + abstract.FileHandle.connectionLost(self, reason) + isClean = (reason is None or + not reason.check(error.ConnectionAborted)) + self._closeSocket(isClean) + protocol = self.protocol + del self.protocol + del self.socket + del self.getFileHandle + protocol.connectionLost(reason) + + + def logPrefix(self): + """ + Return the prefix to log with when I own the logging thread. + """ + return self.logstr + + + def getTcpNoDelay(self): + return operator.truth(self.socket.getsockopt(socket.IPPROTO_TCP, + socket.TCP_NODELAY)) + + + def setTcpNoDelay(self, enabled): + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, enabled) + + + def getTcpKeepAlive(self): + return operator.truth(self.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_KEEPALIVE)) + + + def setTcpKeepAlive(self, enabled): + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, enabled) + + + if _startTLS is not None: + def startTLS(self, contextFactory, normal=True): + """ + @see: L{ITLSTransport.startTLS} + """ + _startTLS(self, contextFactory, normal, abstract.FileHandle) + + + def write(self, data): + """ + Write some data, either directly to the underlying handle or, if TLS + has been started, to the L{TLSMemoryBIOProtocol} for it to encrypt and + send. + + @see: L{twisted.internet.interfaces.ITransport.write} + """ + if self.disconnected: + return + if self.TLS: + self.protocol.write(data) + else: + abstract.FileHandle.write(self, data) + + + def writeSequence(self, iovec): + """ + Write some data, either directly to the underlying handle or, if TLS + has been started, to the L{TLSMemoryBIOProtocol} for it to encrypt and + send. + + @see: L{twisted.internet.interfaces.ITransport.writeSequence} + """ + if self.disconnected: + return + if self.TLS: + self.protocol.writeSequence(iovec) + else: + abstract.FileHandle.writeSequence(self, iovec) + + + def loseConnection(self, reason=None): + """ + Close the underlying handle or, if TLS has been started, first shut it + down. + + @see: L{twisted.internet.interfaces.ITransport.loseConnection} + """ + if self.TLS: + if self.connected and not self.disconnecting: + self.protocol.loseConnection() + else: + abstract.FileHandle.loseConnection(self, reason) + + + def registerProducer(self, producer, streaming): + """ + Register a producer. + + If TLS is enabled, the TLS connection handles this. + """ + if self.TLS: + # Registering a producer before we're connected shouldn't be a + # problem. If we end up with a write(), that's already handled in + # the write() code above, and there are no other potential + # side-effects. + self.protocol.registerProducer(producer, streaming) + else: + abstract.FileHandle.registerProducer(self, producer, streaming) + + + def unregisterProducer(self): + """ + Unregister a producer. + + If TLS is enabled, the TLS connection handles this. + """ + if self.TLS: + self.protocol.unregisterProducer() + else: + abstract.FileHandle.unregisterProducer(self) + +if _startTLS is not None: + classImplements(Connection, interfaces.ITLSTransport) + + + +class Client(_BaseBaseClient, _BaseTCPClient, Connection): + """ + @ivar _tlsClientDefault: Always C{True}, indicating that this is a client + connection, and by default when TLS is negotiated this class will act as + a TLS client. + """ + addressFamily = socket.AF_INET + socketType = socket.SOCK_STREAM + + _tlsClientDefault = True + _commonConnection = Connection + + def __init__(self, host, port, bindAddress, connector, reactor): + # ConnectEx documentation says socket _has_ to be bound + if bindAddress is None: + bindAddress = ('', 0) + self.reactor = reactor # createInternetSocket needs this + _BaseTCPClient.__init__(self, host, port, bindAddress, connector, + reactor) + + + def createInternetSocket(self): + """ + Create a socket registered with the IOCP reactor. + + @see: L{_BaseTCPClient} + """ + return self.reactor.createSocket(self.addressFamily, self.socketType) + + + def _collectSocketDetails(self): + """ + Clean up potentially circular references to the socket and to its + C{getFileHandle} method. + + @see: L{_BaseBaseClient} + """ + del self.socket, self.getFileHandle + + + def _stopReadingAndWriting(self): + """ + Remove the active handle from the reactor. + + @see: L{_BaseBaseClient} + """ + self.reactor.removeActiveHandle(self) + + + def cbConnect(self, rc, data, evt): + if rc: + rc = connectExErrors.get(rc, rc) + self.failIfNotConnected(error.getConnectError((rc, + errno.errorcode.get(rc, 'Unknown error')))) + else: + self.socket.setsockopt( + socket.SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, + struct.pack('P', self.socket.fileno())) + self.protocol = self.connector.buildProtocol(self.getPeer()) + self.connected = True + logPrefix = self._getLogPrefix(self.protocol) + self.logstr = logPrefix + ",client" + if self.protocol is None: + # Factory.buildProtocol is allowed to return None. In that + # case, make up a protocol to satisfy the rest of the + # implementation; connectionLost is going to be called on + # something, for example. This is easier than adding special + # case support for a None protocol throughout the rest of the + # transport implementation. + self.protocol = Protocol() + # But dispose of the connection quickly. + self.loseConnection() + else: + self.protocol.makeConnection(self) + self.startReading() + + + def doConnect(self): + if not hasattr(self, "connector"): + # this happens if we connector.stopConnecting in + # factory.startedConnecting + return + assert _iocp.have_connectex + self.reactor.addActiveHandle(self) + evt = _iocp.Event(self.cbConnect, self) + + rc = _iocp.connect(self.socket.fileno(), self.realAddress, evt) + if rc and rc != ERROR_IO_PENDING: + self.cbConnect(rc, 0, evt) + + + +class Server(Connection): + """ + Serverside socket-stream connection class. + + I am a serverside network connection transport; a socket which came from an + accept() on a server. + + @ivar _tlsClientDefault: Always C{False}, indicating that this is a server + connection, and by default when TLS is negotiated this class will act as + a TLS server. + """ + + _tlsClientDefault = False + + + def __init__(self, sock, protocol, clientAddr, serverAddr, sessionno, reactor): + """ + Server(sock, protocol, client, server, sessionno) + + Initialize me with a socket, a protocol, a descriptor for my peer (a + tuple of host, port describing the other end of the connection), an + instance of Port, and a session number. + """ + Connection.__init__(self, sock, protocol, reactor) + self.serverAddr = serverAddr + self.clientAddr = clientAddr + self.sessionno = sessionno + logPrefix = self._getLogPrefix(self.protocol) + self.logstr = "%s,%s,%s" % (logPrefix, sessionno, self.clientAddr.host) + self.repstr = "<%s #%s on %s>" % (self.protocol.__class__.__name__, + self.sessionno, self.serverAddr.port) + self.connected = True + self.startReading() + + + def __repr__(self): + """ + A string representation of this connection. + """ + return self.repstr + + + def getHost(self): + """ + Returns an IPv4Address. + + This indicates the server's address. + """ + return self.serverAddr + + + def getPeer(self): + """ + Returns an IPv4Address. + + This indicates the client's address. + """ + return self.clientAddr + + + +class Connector(TCPConnector): + def _makeTransport(self): + return Client(self.host, self.port, self.bindAddress, self, + self.reactor) + + + +@implementer(interfaces.IListeningPort) +class Port(_SocketCloser, _LogOwner): + + connected = False + disconnected = False + disconnecting = False + addressFamily = socket.AF_INET + socketType = socket.SOCK_STREAM + _addressType = address.IPv4Address + sessionno = 0 + + # Actual port number being listened on, only set to a non-None + # value when we are actually listening. + _realPortNumber = None + + # A string describing the connections which will be created by this port. + # Normally this is C{"TCP"}, since this is a TCP port, but when the TLS + # implementation re-uses this class it overrides the value with C{"TLS"}. + # Only used for logging. + _type = 'TCP' + + def __init__(self, port, factory, backlog=50, interface='', reactor=None): + self.port = port + self.factory = factory + self.backlog = backlog + self.interface = interface + self.reactor = reactor + if isIPv6Address(interface): + self.addressFamily = socket.AF_INET6 + self._addressType = address.IPv6Address + + + def __repr__(self): + if self._realPortNumber is not None: + return "<%s of %s on %s>" % (self.__class__, + self.factory.__class__, + self._realPortNumber) + else: + return "<%s of %s (not listening)>" % (self.__class__, + self.factory.__class__) + + + def startListening(self): + try: + skt = self.reactor.createSocket(self.addressFamily, + self.socketType) + # TODO: resolve self.interface if necessary + if self.addressFamily == socket.AF_INET6: + addr = _resolveIPv6(self.interface, self.port) + else: + addr = (self.interface, self.port) + skt.bind(addr) + except socket.error as le: + raise error.CannotListenError(self.interface, self.port, le) + + self.addrLen = _iocp.maxAddrLen(skt.fileno()) + + # Make sure that if we listened on port 0, we update that to + # reflect what the OS actually assigned us. + self._realPortNumber = skt.getsockname()[1] + + log.msg("%s starting on %s" % (self._getLogPrefix(self.factory), + self._realPortNumber)) + + self.factory.doStart() + skt.listen(self.backlog) + self.connected = True + self.disconnected = False + self.reactor.addActiveHandle(self) + self.socket = skt + self.getFileHandle = self.socket.fileno + self.doAccept() + + + def loseConnection(self, connDone=failure.Failure(main.CONNECTION_DONE)): + """ + Stop accepting connections on this port. + + This will shut down my socket and call self.connectionLost(). + It returns a deferred which will fire successfully when the + port is actually closed. + """ + self.disconnecting = True + if self.connected: + self.deferred = defer.Deferred() + self.reactor.callLater(0, self.connectionLost, connDone) + return self.deferred + + stopListening = loseConnection + + + def _logConnectionLostMsg(self): + """ + Log message for closing port + """ + log.msg('(%s Port %s Closed)' % (self._type, self._realPortNumber)) + + + def connectionLost(self, reason): + """ + Cleans up the socket. + """ + self._logConnectionLostMsg() + self._realPortNumber = None + d = None + if hasattr(self, "deferred"): + d = self.deferred + del self.deferred + + self.disconnected = True + self.reactor.removeActiveHandle(self) + self.connected = False + self._closeSocket(True) + del self.socket + del self.getFileHandle + + try: + self.factory.doStop() + except: + self.disconnecting = False + if d is not None: + d.errback(failure.Failure()) + else: + raise + else: + self.disconnecting = False + if d is not None: + d.callback(None) + + + def logPrefix(self): + """ + Returns the name of my class, to prefix log entries with. + """ + return reflect.qual(self.factory.__class__) + + + def getHost(self): + """ + Returns an IPv4Address or IPv6Address. + + This indicates the server's address. + """ + return self._addressType('TCP', *_getsockname(self.socket)) + + + def cbAccept(self, rc, data, evt): + self.handleAccept(rc, evt) + if not (self.disconnecting or self.disconnected): + self.doAccept() + + + def handleAccept(self, rc, evt): + if self.disconnecting or self.disconnected: + return False + + # possible errors: + # (WSAEMFILE, WSAENOBUFS, WSAENFILE, WSAENOMEM, WSAECONNABORTED) + if rc: + log.msg("Could not accept new connection -- %s (%s)" % + (errno.errorcode.get(rc, 'unknown error'), rc)) + return False + else: + evt.newskt.setsockopt( + socket.SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, + struct.pack('P', self.socket.fileno())) + family, lAddr, rAddr = _iocp.get_accept_addrs(evt.newskt.fileno(), + evt.buff) + if not _PY3: + # In _makesockaddr(), we use the Win32 API which + # gives us an address of the form: (unicode host, port). + # Only on Python 2 do we need to convert it to a + # non-unicode str. + # On Python 3, we leave it alone as unicode. + lAddr = (nativeString(lAddr[0]), lAddr[1]) + rAddr = (nativeString(rAddr[0]), rAddr[1]) + assert family == self.addressFamily + + # Build an IPv6 address that includes the scopeID, if necessary + if "%" in lAddr[0]: + scope = int(lAddr[0].split("%")[1]) + lAddr = (lAddr[0], lAddr[1], 0, scope) + if "%" in rAddr[0]: + scope = int(rAddr[0].split("%")[1]) + rAddr = (rAddr[0], rAddr[1], 0, scope) + + protocol = self.factory.buildProtocol( + self._addressType('TCP', *rAddr)) + if protocol is None: + evt.newskt.close() + else: + s = self.sessionno + self.sessionno = s+1 + transport = Server(evt.newskt, protocol, + self._addressType('TCP', *rAddr), + self._addressType('TCP', *lAddr), + s, self.reactor) + protocol.makeConnection(transport) + return True + + + def doAccept(self): + evt = _iocp.Event(self.cbAccept, self) + + # see AcceptEx documentation + evt.buff = buff = bytearray(2 * (self.addrLen + 16)) + + evt.newskt = newskt = self.reactor.createSocket(self.addressFamily, + self.socketType) + rc = _iocp.accept(self.socket.fileno(), newskt.fileno(), buff, evt) + + if rc and rc != ERROR_IO_PENDING: + self.handleAccept(rc, evt) diff --git a/contrib/python/Twisted/py2/twisted/internet/iocpreactor/udp.py b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/udp.py new file mode 100644 index 00000000000..8419d899fa0 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/iocpreactor/udp.py @@ -0,0 +1,428 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +UDP support for IOCP reactor +""" + +import socket, operator, struct, warnings, errno + +from zope.interface import implementer + +from twisted.internet import defer, address, error, interfaces +from twisted.internet.abstract import isIPAddress, isIPv6Address +from twisted.python import log, failure + +from twisted.internet.iocpreactor.const import ERROR_IO_PENDING +from twisted.internet.iocpreactor.const import ERROR_CONNECTION_REFUSED +from twisted.internet.iocpreactor.const import ERROR_PORT_UNREACHABLE +from twisted.internet.iocpreactor.interfaces import IReadWriteHandle +from twisted.internet.iocpreactor import iocpsupport as _iocp, abstract + + + +@implementer(IReadWriteHandle, interfaces.IListeningPort, + interfaces.IUDPTransport, interfaces.ISystemHandle) +class Port(abstract.FileHandle): + """ + UDP port, listening for packets. + + @ivar addressFamily: L{socket.AF_INET} or L{socket.AF_INET6}, depending on + whether this port is listening on an IPv4 address or an IPv6 address. + """ + addressFamily = socket.AF_INET + socketType = socket.SOCK_DGRAM + dynamicReadBuffers = False + + # Actual port number being listened on, only set to a non-None + # value when we are actually listening. + _realPortNumber = None + + + def __init__(self, port, proto, interface='', maxPacketSize=8192, + reactor=None): + """ + Initialize with a numeric port to listen on. + """ + self.port = port + self.protocol = proto + self.readBufferSize = maxPacketSize + self.interface = interface + self.setLogStr() + self._connectedAddr = None + self._setAddressFamily() + + abstract.FileHandle.__init__(self, reactor) + + skt = socket.socket(self.addressFamily, self.socketType) + addrLen = _iocp.maxAddrLen(skt.fileno()) + self.addressBuffer = bytearray(addrLen) + # WSARecvFrom takes an int + self.addressLengthBuffer = bytearray(struct.calcsize('i')) + + + def _setAddressFamily(self): + """ + Resolve address family for the socket. + """ + if isIPv6Address(self.interface): + self.addressFamily = socket.AF_INET6 + elif isIPAddress(self.interface): + self.addressFamily = socket.AF_INET + elif self.interface: + raise error.InvalidAddressError( + self.interface, 'not an IPv4 or IPv6 address') + + + def __repr__(self): + if self._realPortNumber is not None: + return ("<%s on %s>" % + (self.protocol.__class__, self._realPortNumber)) + else: + return "<%s not connected>" % (self.protocol.__class__,) + + + def getHandle(self): + """ + Return a socket object. + """ + return self.socket + + + def startListening(self): + """ + Create and bind my socket, and begin listening on it. + + This is called on unserialization, and must be called after creating a + server to begin listening on the specified port. + """ + self._bindSocket() + self._connectToProtocol() + + + def createSocket(self): + return self.reactor.createSocket(self.addressFamily, self.socketType) + + + def _bindSocket(self): + try: + skt = self.createSocket() + skt.bind((self.interface, self.port)) + except socket.error as le: + raise error.CannotListenError(self.interface, self.port, le) + + # Make sure that if we listened on port 0, we update that to + # reflect what the OS actually assigned us. + self._realPortNumber = skt.getsockname()[1] + + log.msg("%s starting on %s" % ( + self._getLogPrefix(self.protocol), self._realPortNumber)) + + self.connected = True + self.socket = skt + self.getFileHandle = self.socket.fileno + + + def _connectToProtocol(self): + self.protocol.makeConnection(self) + self.startReading() + self.reactor.addActiveHandle(self) + + + def cbRead(self, rc, data, evt): + if self.reading: + self.handleRead(rc, data, evt) + self.doRead() + + + def handleRead(self, rc, data, evt): + if rc in (errno.WSAECONNREFUSED, errno.WSAECONNRESET, + ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE): + if self._connectedAddr: + self.protocol.connectionRefused() + elif rc: + log.msg("error in recvfrom -- %s (%s)" % + (errno.errorcode.get(rc, 'unknown error'), rc)) + else: + try: + self.protocol.datagramReceived(bytes(evt.buff[:data]), + _iocp.makesockaddr(evt.addr_buff)) + except: + log.err() + + + def doRead(self): + evt = _iocp.Event(self.cbRead, self) + + evt.buff = buff = self._readBuffers[0] + evt.addr_buff = addr_buff = self.addressBuffer + evt.addr_len_buff = addr_len_buff = self.addressLengthBuffer + rc, data = _iocp.recvfrom(self.getFileHandle(), buff, + addr_buff, addr_len_buff, evt) + + if rc and rc != ERROR_IO_PENDING: + self.handleRead(rc, data, evt) + + + def write(self, datagram, addr=None): + """ + Write a datagram. + + @param addr: should be a tuple (ip, port), can be None in connected + mode. + """ + if self._connectedAddr: + assert addr in (None, self._connectedAddr) + try: + return self.socket.send(datagram) + except socket.error as se: + no = se.args[0] + if no == errno.WSAEINTR: + return self.write(datagram) + elif no == errno.WSAEMSGSIZE: + raise error.MessageLengthError("message too long") + elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET, + ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE): + self.protocol.connectionRefused() + else: + raise + else: + assert addr != None + if (not isIPAddress(addr[0]) and not isIPv6Address(addr[0]) + and addr[0] != ""): + raise error.InvalidAddressError( + addr[0], + "write() only accepts IP addresses, not hostnames") + if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6: + raise error.InvalidAddressError( + addr[0], "IPv6 port write() called with IPv4 address") + if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET: + raise error.InvalidAddressError( + addr[0], "IPv4 port write() called with IPv6 address") + try: + return self.socket.sendto(datagram, addr) + except socket.error as se: + no = se.args[0] + if no == errno.WSAEINTR: + return self.write(datagram, addr) + elif no == errno.WSAEMSGSIZE: + raise error.MessageLengthError("message too long") + elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET, + ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE): + # in non-connected UDP ECONNREFUSED is platform dependent, + # I think and the info is not necessarily useful. + # Nevertheless maybe we should call connectionRefused? XXX + return + else: + raise + + + def writeSequence(self, seq, addr): + self.write(b"".join(seq), addr) + + + def connect(self, host, port): + """ + 'Connect' to remote server. + """ + if self._connectedAddr: + raise RuntimeError( + "already connected, reconnecting is not currently supported " + "(talk to itamar if you want this)") + if not isIPAddress(host) and not isIPv6Address(host): + raise error.InvalidAddressError( + host, 'not an IPv4 or IPv6 address.') + self._connectedAddr = (host, port) + self.socket.connect((host, port)) + + + def _loseConnection(self): + self.stopReading() + self.reactor.removeActiveHandle(self) + if self.connected: # actually means if we are *listening* + self.reactor.callLater(0, self.connectionLost) + + + def stopListening(self): + if self.connected: + result = self.d = defer.Deferred() + else: + result = None + self._loseConnection() + return result + + + def loseConnection(self): + warnings.warn("Please use stopListening() to disconnect port", + DeprecationWarning, stacklevel=2) + self.stopListening() + + + def connectionLost(self, reason=None): + """ + Cleans up my socket. + """ + log.msg('(UDP Port %s Closed)' % self._realPortNumber) + self._realPortNumber = None + abstract.FileHandle.connectionLost(self, reason) + self.protocol.doStop() + self.socket.close() + del self.socket + del self.getFileHandle + if hasattr(self, "d"): + self.d.callback(None) + del self.d + + + def setLogStr(self): + """ + Initialize the C{logstr} attribute to be used by C{logPrefix}. + """ + logPrefix = self._getLogPrefix(self.protocol) + self.logstr = "%s (UDP)" % logPrefix + + + def logPrefix(self): + """ + Returns the name of my class, to prefix log entries with. + """ + return self.logstr + + + def getHost(self): + """ + Return the local address of the UDP connection + + @returns: the local address of the UDP connection + @rtype: L{IPv4Address} or L{IPv6Address} + """ + addr = self.socket.getsockname() + if self.addressFamily == socket.AF_INET: + return address.IPv4Address('UDP', *addr) + elif self.addressFamily == socket.AF_INET6: + return address.IPv6Address('UDP', *(addr[:2])) + + + def setBroadcastAllowed(self, enabled): + """ + Set whether this port may broadcast. This is disabled by default. + + @param enabled: Whether the port may broadcast. + @type enabled: L{bool} + """ + self.socket.setsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST, enabled) + + + def getBroadcastAllowed(self): + """ + Checks if broadcast is currently allowed on this port. + + @return: Whether this port may broadcast. + @rtype: L{bool} + """ + return operator.truth( + self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST)) + + + +class MulticastMixin: + """ + Implement multicast functionality. + """ + + + def getOutgoingInterface(self): + i = self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF) + return socket.inet_ntoa(struct.pack("@i", i)) + + + def setOutgoingInterface(self, addr): + """ + Returns Deferred of success. + """ + return self.reactor.resolve(addr).addCallback(self._setInterface) + + + def _setInterface(self, addr): + i = socket.inet_aton(addr) + self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, i) + return 1 + + + def getLoopbackMode(self): + return self.socket.getsockopt(socket.IPPROTO_IP, + socket.IP_MULTICAST_LOOP) + + + def setLoopbackMode(self, mode): + mode = struct.pack("b", operator.truth(mode)) + self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, + mode) + + + def getTTL(self): + return self.socket.getsockopt(socket.IPPROTO_IP, + socket.IP_MULTICAST_TTL) + + + def setTTL(self, ttl): + ttl = struct.pack("B", ttl) + self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl) + + + def joinGroup(self, addr, interface=""): + """ + Join a multicast group. Returns Deferred of success. + """ + return self.reactor.resolve(addr).addCallback(self._joinAddr1, + interface, 1) + + + def _joinAddr1(self, addr, interface, join): + return self.reactor.resolve(interface).addCallback(self._joinAddr2, + addr, join) + + + def _joinAddr2(self, interface, addr, join): + addr = socket.inet_aton(addr) + interface = socket.inet_aton(interface) + if join: + cmd = socket.IP_ADD_MEMBERSHIP + else: + cmd = socket.IP_DROP_MEMBERSHIP + try: + self.socket.setsockopt(socket.IPPROTO_IP, cmd, addr + interface) + except socket.error as e: + return failure.Failure(error.MulticastJoinError(addr, interface, + *e.args)) + + + def leaveGroup(self, addr, interface=""): + """ + Leave multicast group, return Deferred of success. + """ + return self.reactor.resolve(addr).addCallback(self._joinAddr1, + interface, 0) + + + +@implementer(interfaces.IMulticastTransport) +class MulticastPort(MulticastMixin, Port): + """ + UDP Port that supports multicasting. + """ + + def __init__(self, port, proto, interface='', maxPacketSize=8192, + reactor=None, listenMultiple=False): + Port.__init__(self, port, proto, interface, maxPacketSize, reactor) + self.listenMultiple = listenMultiple + + + def createSocket(self): + skt = Port.createSocket(self) + if self.listenMultiple: + skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, "SO_REUSEPORT"): + skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + return skt diff --git a/contrib/python/Twisted/py2/twisted/internet/kqreactor.py b/contrib/python/Twisted/py2/twisted/internet/kqreactor.py new file mode 100644 index 00000000000..ffc40c385a8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/kqreactor.py @@ -0,0 +1,320 @@ +# -*- test-case-name: twisted.test.test_kqueuereactor -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A kqueue()/kevent() based implementation of the Twisted main loop. + +To use this reactor, start your application specifying the kqueue reactor:: + + twistd --reactor kqueue ... + +To install the event loop from code (and you should do this before any +connections, listeners or connectors are added):: + + from twisted.internet import kqreactor + kqreactor.install() +""" + +from __future__ import division, absolute_import + +import errno +import select + +from select import KQ_FILTER_READ, KQ_FILTER_WRITE +from select import KQ_EV_DELETE, KQ_EV_ADD, KQ_EV_EOF + +from zope.interface import implementer, declarations, Interface, Attribute + +from twisted.internet import main, posixbase +from twisted.internet.interfaces import IReactorFDSet, IReactorDaemonize +from twisted.python import log, failure + + + +class _IKQueue(Interface): + """ + An interface for KQueue implementations. + """ + kqueue = Attribute("An implementation of kqueue(2).") + kevent = Attribute("An implementation of kevent(2).") + +declarations.directlyProvides(select, _IKQueue) + + + +@implementer(IReactorFDSet, IReactorDaemonize) +class KQueueReactor(posixbase.PosixReactorBase): + """ + A reactor that uses kqueue(2)/kevent(2) and relies on Python 2.6 or higher + which has built in support for kqueue in the select module. + + @ivar _kq: A C{kqueue} which will be used to check for I/O readiness. + + @ivar _impl: The implementation of L{_IKQueue} to use. + + @ivar _selectables: A dictionary mapping integer file descriptors to + instances of L{FileDescriptor} which have been registered with the + reactor. All L{FileDescriptor}s which are currently receiving read or + write readiness notifications will be present as values in this + dictionary. + + @ivar _reads: A set containing integer file descriptors. Values in this + set will be registered with C{_kq} for read readiness notifications + which will be dispatched to the corresponding L{FileDescriptor} + instances in C{_selectables}. + + @ivar _writes: A set containing integer file descriptors. Values in this + set will be registered with C{_kq} for write readiness notifications + which will be dispatched to the corresponding L{FileDescriptor} + instances in C{_selectables}. + """ + + def __init__(self, _kqueueImpl=select): + """ + Initialize kqueue object, file descriptor tracking dictionaries, and + the base class. + + See: + - http://docs.python.org/library/select.html + - www.freebsd.org/cgi/man.cgi?query=kqueue + - people.freebsd.org/~jlemon/papers/kqueue.pdf + + @param _kqueueImpl: The implementation of L{_IKQueue} to use. A + hook for testing. + """ + self._impl = _kqueueImpl + self._kq = self._impl.kqueue() + self._reads = set() + self._writes = set() + self._selectables = {} + posixbase.PosixReactorBase.__init__(self) + + + def _updateRegistration(self, fd, filter, op): + """ + Private method for changing kqueue registration on a given FD + filtering for events given filter/op. This will never block and + returns nothing. + """ + self._kq.control([self._impl.kevent(fd, filter, op)], 0, 0) + + + def beforeDaemonize(self): + """ + Implement L{IReactorDaemonize.beforeDaemonize}. + """ + # Twisted-internal method called during daemonization (when application + # is started via twistd). This is called right before the magic double + # forking done for daemonization. We cleanly close the kqueue() and later + # recreate it. This is needed since a) kqueue() are not inherited across + # forks and b) twistd will create the reactor already before daemonization + # (and will also add at least 1 reader to the reactor, an instance of + # twisted.internet.posixbase._UnixWaker). + # + # See: twisted.scripts._twistd_unix.daemonize() + self._kq.close() + self._kq = None + + + def afterDaemonize(self): + """ + Implement L{IReactorDaemonize.afterDaemonize}. + """ + # Twisted-internal method called during daemonization. This is called right + # after daemonization and recreates the kqueue() and any readers/writers + # that were added before. Note that you MUST NOT call any reactor methods + # in between beforeDaemonize() and afterDaemonize()! + self._kq = self._impl.kqueue() + for fd in self._reads: + self._updateRegistration(fd, KQ_FILTER_READ, KQ_EV_ADD) + for fd in self._writes: + self._updateRegistration(fd, KQ_FILTER_WRITE, KQ_EV_ADD) + + + def addReader(self, reader): + """ + Implement L{IReactorFDSet.addReader}. + """ + fd = reader.fileno() + if fd not in self._reads: + try: + self._updateRegistration(fd, KQ_FILTER_READ, KQ_EV_ADD) + except OSError: + pass + finally: + self._selectables[fd] = reader + self._reads.add(fd) + + + def addWriter(self, writer): + """ + Implement L{IReactorFDSet.addWriter}. + """ + fd = writer.fileno() + if fd not in self._writes: + try: + self._updateRegistration(fd, KQ_FILTER_WRITE, KQ_EV_ADD) + except OSError: + pass + finally: + self._selectables[fd] = writer + self._writes.add(fd) + + + def removeReader(self, reader): + """ + Implement L{IReactorFDSet.removeReader}. + """ + wasLost = False + try: + fd = reader.fileno() + except: + fd = -1 + if fd == -1: + for fd, fdes in self._selectables.items(): + if reader is fdes: + wasLost = True + break + else: + return + if fd in self._reads: + self._reads.remove(fd) + if fd not in self._writes: + del self._selectables[fd] + if not wasLost: + try: + self._updateRegistration(fd, KQ_FILTER_READ, KQ_EV_DELETE) + except OSError: + pass + + + def removeWriter(self, writer): + """ + Implement L{IReactorFDSet.removeWriter}. + """ + wasLost = False + try: + fd = writer.fileno() + except: + fd = -1 + if fd == -1: + for fd, fdes in self._selectables.items(): + if writer is fdes: + wasLost = True + break + else: + return + if fd in self._writes: + self._writes.remove(fd) + if fd not in self._reads: + del self._selectables[fd] + if not wasLost: + try: + self._updateRegistration(fd, KQ_FILTER_WRITE, KQ_EV_DELETE) + except OSError: + pass + + + def removeAll(self): + """ + Implement L{IReactorFDSet.removeAll}. + """ + return self._removeAll( + [self._selectables[fd] for fd in self._reads], + [self._selectables[fd] for fd in self._writes]) + + + def getReaders(self): + """ + Implement L{IReactorFDSet.getReaders}. + """ + return [self._selectables[fd] for fd in self._reads] + + + def getWriters(self): + """ + Implement L{IReactorFDSet.getWriters}. + """ + return [self._selectables[fd] for fd in self._writes] + + + def doKEvent(self, timeout): + """ + Poll the kqueue for new events. + """ + if timeout is None: + timeout = 1 + + try: + events = self._kq.control([], len(self._selectables), timeout) + except OSError as e: + # Since this command blocks for potentially a while, it's possible + # EINTR can be raised for various reasons (for example, if the user + # hits ^C). + if e.errno == errno.EINTR: + return + else: + raise + + _drdw = self._doWriteOrRead + for event in events: + fd = event.ident + try: + selectable = self._selectables[fd] + except KeyError: + # Handles the infrequent case where one selectable's + # handler disconnects another. + continue + else: + log.callWithLogger(selectable, _drdw, selectable, fd, event) + + + def _doWriteOrRead(self, selectable, fd, event): + """ + Private method called when a FD is ready for reading, writing or was + lost. Do the work and raise errors where necessary. + """ + why = None + inRead = False + (filter, flags, data, fflags) = ( + event.filter, event.flags, event.data, event.fflags) + + if flags & KQ_EV_EOF and data and fflags: + why = main.CONNECTION_LOST + else: + try: + if selectable.fileno() == -1: + inRead = False + why = posixbase._NO_FILEDESC + else: + if filter == KQ_FILTER_READ: + inRead = True + why = selectable.doRead() + if filter == KQ_FILTER_WRITE: + inRead = False + why = selectable.doWrite() + except: + # Any exception from application code gets logged and will + # cause us to disconnect the selectable. + why = failure.Failure() + log.err(why, "An exception was raised from application code" \ + " while processing a reactor selectable") + + if why: + self._disconnectSelectable(selectable, why, inRead) + + doIteration = doKEvent + + + +def install(): + """ + Install the kqueue() reactor. + """ + p = KQueueReactor() + from twisted.internet.main import installReactor + installReactor(p) + + +__all__ = ["KQueueReactor", "install"] diff --git a/contrib/python/Twisted/py2/twisted/internet/main.py b/contrib/python/Twisted/py2/twisted/internet/main.py new file mode 100644 index 00000000000..f7efeabfd99 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/main.py @@ -0,0 +1,37 @@ +# -*- test-case-name: twisted.internet.test.test_main -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Backwards compatibility, and utility functions. + +In general, this module should not be used, other than by reactor authors +who need to use the 'installReactor' method. +""" + +from __future__ import division, absolute_import + +from twisted.internet import error + +CONNECTION_DONE = error.ConnectionDone('Connection done') +CONNECTION_LOST = error.ConnectionLost('Connection lost') + + + +def installReactor(reactor): + """ + Install reactor C{reactor}. + + @param reactor: An object that provides one or more IReactor* interfaces. + """ + # this stuff should be common to all reactors. + import twisted.internet + import sys + if 'twisted.internet.reactor' in sys.modules: + raise error.ReactorAlreadyInstalledError("reactor already installed") + twisted.internet.reactor = reactor + sys.modules['twisted.internet.reactor'] = reactor + + +__all__ = ["CONNECTION_LOST", "CONNECTION_DONE", "installReactor"] diff --git a/contrib/python/Twisted/py2/twisted/internet/pollreactor.py b/contrib/python/Twisted/py2/twisted/internet/pollreactor.py new file mode 100644 index 00000000000..fe2fea56489 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/pollreactor.py @@ -0,0 +1,189 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A poll() based implementation of the twisted main loop. + +To install the event loop (and you should do this before any connections, +listeners or connectors are added):: + + from twisted.internet import pollreactor + pollreactor.install() +""" + +from __future__ import division, absolute_import + +# System imports +import errno +from select import error as SelectError, poll +from select import POLLIN, POLLOUT, POLLHUP, POLLERR, POLLNVAL + +from zope.interface import implementer + +# Twisted imports +from twisted.python import log +from twisted.internet import posixbase +from twisted.internet.interfaces import IReactorFDSet + + + +@implementer(IReactorFDSet) +class PollReactor(posixbase.PosixReactorBase, posixbase._PollLikeMixin): + """ + A reactor that uses poll(2). + + @ivar _poller: A L{select.poll} which will be used to check for I/O + readiness. + + @ivar _selectables: A dictionary mapping integer file descriptors to + instances of L{FileDescriptor} which have been registered with the + reactor. All L{FileDescriptor}s which are currently receiving read or + write readiness notifications will be present as values in this + dictionary. + + @ivar _reads: A dictionary mapping integer file descriptors to arbitrary + values (this is essentially a set). Keys in this dictionary will be + registered with C{_poller} for read readiness notifications which will + be dispatched to the corresponding L{FileDescriptor} instances in + C{_selectables}. + + @ivar _writes: A dictionary mapping integer file descriptors to arbitrary + values (this is essentially a set). Keys in this dictionary will be + registered with C{_poller} for write readiness notifications which will + be dispatched to the corresponding L{FileDescriptor} instances in + C{_selectables}. + """ + + _POLL_DISCONNECTED = (POLLHUP | POLLERR | POLLNVAL) + _POLL_IN = POLLIN + _POLL_OUT = POLLOUT + + def __init__(self): + """ + Initialize polling object, file descriptor tracking dictionaries, and + the base class. + """ + self._poller = poll() + self._selectables = {} + self._reads = {} + self._writes = {} + posixbase.PosixReactorBase.__init__(self) + + + def _updateRegistration(self, fd): + """Register/unregister an fd with the poller.""" + try: + self._poller.unregister(fd) + except KeyError: + pass + + mask = 0 + if fd in self._reads: + mask = mask | POLLIN + if fd in self._writes: + mask = mask | POLLOUT + if mask != 0: + self._poller.register(fd, mask) + else: + if fd in self._selectables: + del self._selectables[fd] + + def _dictRemove(self, selectable, mdict): + try: + # the easy way + fd = selectable.fileno() + # make sure the fd is actually real. In some situations we can get + # -1 here. + mdict[fd] + except: + # the hard way: necessary because fileno() may disappear at any + # moment, thanks to python's underlying sockets impl + for fd, fdes in self._selectables.items(): + if selectable is fdes: + break + else: + # Hmm, maybe not the right course of action? This method can't + # fail, because it happens inside error detection... + return + if fd in mdict: + del mdict[fd] + self._updateRegistration(fd) + + def addReader(self, reader): + """Add a FileDescriptor for notification of data available to read. + """ + fd = reader.fileno() + if fd not in self._reads: + self._selectables[fd] = reader + self._reads[fd] = 1 + self._updateRegistration(fd) + + def addWriter(self, writer): + """Add a FileDescriptor for notification of data available to write. + """ + fd = writer.fileno() + if fd not in self._writes: + self._selectables[fd] = writer + self._writes[fd] = 1 + self._updateRegistration(fd) + + def removeReader(self, reader): + """Remove a Selectable for notification of data available to read. + """ + return self._dictRemove(reader, self._reads) + + def removeWriter(self, writer): + """Remove a Selectable for notification of data available to write. + """ + return self._dictRemove(writer, self._writes) + + def removeAll(self): + """ + Remove all selectables, and return a list of them. + """ + return self._removeAll( + [self._selectables[fd] for fd in self._reads], + [self._selectables[fd] for fd in self._writes]) + + + def doPoll(self, timeout): + """Poll the poller for new events.""" + if timeout is not None: + timeout = int(timeout * 1000) # convert seconds to milliseconds + + try: + l = self._poller.poll(timeout) + except SelectError as e: + if e.args[0] == errno.EINTR: + return + else: + raise + _drdw = self._doReadOrWrite + for fd, event in l: + try: + selectable = self._selectables[fd] + except KeyError: + # Handles the infrequent case where one selectable's + # handler disconnects another. + continue + log.callWithLogger(selectable, _drdw, selectable, fd, event) + + doIteration = doPoll + + def getReaders(self): + return [self._selectables[fd] for fd in self._reads] + + + def getWriters(self): + return [self._selectables[fd] for fd in self._writes] + + + +def install(): + """Install the poll() reactor.""" + p = PollReactor() + from twisted.internet.main import installReactor + installReactor(p) + + +__all__ = ["PollReactor", "install"] diff --git a/contrib/python/Twisted/py2/twisted/internet/posixbase.py b/contrib/python/Twisted/py2/twisted/internet/posixbase.py new file mode 100644 index 00000000000..1b0e5096419 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/posixbase.py @@ -0,0 +1,798 @@ +# -*- test-case-name: twisted.test.test_internet,twisted.internet.test.test_posixbase -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Posix reactor base class +""" + +from __future__ import division, absolute_import + +import socket +import errno +import os +import sys + +from zope.interface import implementer, classImplements + +from twisted.internet import error, udp, tcp +from twisted.internet.base import ReactorBase, _SignalReactorMixin +from twisted.internet.main import CONNECTION_DONE, CONNECTION_LOST +from twisted.internet.interfaces import ( + IReactorUNIX, IReactorUNIXDatagram, IReactorTCP, IReactorUDP, IReactorSSL, + IReactorSocket, IHalfCloseableDescriptor, IReactorProcess, + IReactorMulticast, IReactorFDSet) + +from twisted.python import log, failure, util +from twisted.python.runtime import platformType, platform + +# Exceptions that doSelect might return frequently +_NO_FILENO = error.ConnectionFdescWentAway('Handler has no fileno method') +_NO_FILEDESC = error.ConnectionFdescWentAway('File descriptor lost') + + +try: + from twisted.protocols import tls +except ImportError: + tls = None + try: + from twisted.internet import ssl + except ImportError: + ssl = None + +unixEnabled = (platformType == 'posix') + +processEnabled = False +if unixEnabled: + from twisted.internet import fdesc, unix + from twisted.internet import process, _signals + processEnabled = True + + +if platform.isWindows(): + try: + import win32process + processEnabled = True + except ImportError: + win32process = None + + +class _SocketWaker(log.Logger): + """ + The I{self-pipe trick}, implemented + using a pair of sockets rather than pipes (due to the lack of support in + select() on Windows for pipes), used to wake up the main loop from + another thread. + """ + disconnected = 0 + + def __init__(self, reactor): + """Initialize. + """ + self.reactor = reactor + # Following select_trigger (from asyncore)'s example; + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + server.bind(('127.0.0.1', 0)) + server.listen(1) + client.connect(server.getsockname()) + reader, clientaddr = server.accept() + client.setblocking(0) + reader.setblocking(0) + self.r = reader + self.w = client + self.fileno = self.r.fileno + + def wakeUp(self): + """Send a byte to my connection. + """ + try: + util.untilConcludes(self.w.send, b'x') + except socket.error as e: + if e.args[0] != errno.WSAEWOULDBLOCK: + raise + + def doRead(self): + """Read some data from my connection. + """ + try: + self.r.recv(8192) + except socket.error: + pass + + def connectionLost(self, reason): + self.r.close() + self.w.close() + + + +class _FDWaker(log.Logger, object): + """ + The I{self-pipe trick}, used to wake + up the main loop from another thread or a signal handler. + + L{_FDWaker} is a base class for waker implementations based on + writing to a pipe being monitored by the reactor. + + @ivar o: The file descriptor for the end of the pipe which can be + written to wake up a reactor monitoring this waker. + + @ivar i: The file descriptor which should be monitored in order to + be awoken by this waker. + """ + disconnected = 0 + + i = None + o = None + + def __init__(self, reactor): + """Initialize. + """ + self.reactor = reactor + self.i, self.o = os.pipe() + fdesc.setNonBlocking(self.i) + fdesc._setCloseOnExec(self.i) + fdesc.setNonBlocking(self.o) + fdesc._setCloseOnExec(self.o) + self.fileno = lambda: self.i + + + def doRead(self): + """ + Read some bytes from the pipe and discard them. + """ + fdesc.readFromFD(self.fileno(), lambda data: None) + + + def connectionLost(self, reason): + """Close both ends of my pipe. + """ + if not hasattr(self, "o"): + return + for fd in self.i, self.o: + try: + os.close(fd) + except IOError: + pass + del self.i, self.o + + + +class _UnixWaker(_FDWaker): + """ + This class provides a simple interface to wake up the event loop. + + This is used by threads or signals to wake up the event loop. + """ + + def wakeUp(self): + """Write one byte to the pipe, and flush it. + """ + # We don't use fdesc.writeToFD since we need to distinguish + # between EINTR (try again) and EAGAIN (do nothing). + if self.o is not None: + try: + util.untilConcludes(os.write, self.o, b'x') + except OSError as e: + # XXX There is no unit test for raising the exception + # for other errnos. See #4285. + if e.errno != errno.EAGAIN: + raise + + + +if platformType == 'posix': + _Waker = _UnixWaker +else: + # Primarily Windows and Jython. + _Waker = _SocketWaker + + +class _SIGCHLDWaker(_FDWaker): + """ + L{_SIGCHLDWaker} can wake up a reactor whenever C{SIGCHLD} is + received. + + @see: L{twisted.internet._signals} + """ + def __init__(self, reactor): + _FDWaker.__init__(self, reactor) + + + def install(self): + """ + Install the handler necessary to make this waker active. + """ + _signals.installHandler(self.o) + + + def uninstall(self): + """ + Remove the handler which makes this waker active. + """ + _signals.installHandler(-1) + + + def doRead(self): + """ + Having woken up the reactor in response to receipt of + C{SIGCHLD}, reap the process which exited. + + This is called whenever the reactor notices the waker pipe is + writeable, which happens soon after any call to the C{wakeUp} + method. + """ + _FDWaker.doRead(self) + process.reapAllProcesses() + + + + +class _DisconnectSelectableMixin(object): + """ + Mixin providing the C{_disconnectSelectable} method. + """ + + def _disconnectSelectable(self, selectable, why, isRead, faildict={ + error.ConnectionDone: failure.Failure(error.ConnectionDone()), + error.ConnectionLost: failure.Failure(error.ConnectionLost()) + }): + """ + Utility function for disconnecting a selectable. + + Supports half-close notification, isRead should be boolean indicating + whether error resulted from doRead(). + """ + self.removeReader(selectable) + f = faildict.get(why.__class__) + if f: + if (isRead and why.__class__ == error.ConnectionDone + and IHalfCloseableDescriptor.providedBy(selectable)): + selectable.readConnectionLost(f) + else: + self.removeWriter(selectable) + selectable.connectionLost(f) + else: + self.removeWriter(selectable) + selectable.connectionLost(failure.Failure(why)) + + + +@implementer(IReactorTCP, IReactorUDP, IReactorMulticast) +class PosixReactorBase(_SignalReactorMixin, _DisconnectSelectableMixin, + ReactorBase): + """ + A basis for reactors that use file descriptors. + + @ivar _childWaker: L{None} or a reference to the L{_SIGCHLDWaker} + which is used to properly notice child process termination. + """ + + # Callable that creates a waker, overrideable so that subclasses can + # substitute their own implementation: + _wakerFactory = _Waker + + def installWaker(self): + """ + Install a `waker' to allow threads and signals to wake up the IO thread. + + We use the self-pipe trick (http://cr.yp.to/docs/selfpipe.html) to wake + the reactor. On Windows we use a pair of sockets. + """ + if not self.waker: + self.waker = self._wakerFactory(self) + self._internalReaders.add(self.waker) + self.addReader(self.waker) + + + _childWaker = None + def _handleSignals(self): + """ + Extend the basic signal handling logic to also support + handling SIGCHLD to know when to try to reap child processes. + """ + _SignalReactorMixin._handleSignals(self) + if platformType == 'posix' and processEnabled: + if not self._childWaker: + self._childWaker = _SIGCHLDWaker(self) + self._internalReaders.add(self._childWaker) + self.addReader(self._childWaker) + self._childWaker.install() + # Also reap all processes right now, in case we missed any + # signals before we installed the SIGCHLD waker/handler. + # This should only happen if someone used spawnProcess + # before calling reactor.run (and the process also exited + # already). + process.reapAllProcesses() + + def _uninstallHandler(self): + """ + If a child waker was created and installed, uninstall it now. + + Since this disables reactor functionality and is only called + when the reactor is stopping, it doesn't provide any directly + useful functionality, but the cleanup of reactor-related + process-global state that it does helps in unit tests + involving multiple reactors and is generally just a nice + thing. + """ + # XXX This would probably be an alright place to put all of + # the cleanup code for all internal readers (here and in the + # base class, anyway). See #3063 for that cleanup task. + if self._childWaker: + self._childWaker.uninstall() + + # IReactorProcess + + def spawnProcess(self, processProtocol, executable, args=(), + env={}, path=None, + uid=None, gid=None, usePTY=0, childFDs=None): + args, env = self._checkProcessArgs(args, env) + if platformType == 'posix': + if usePTY: + if childFDs is not None: + raise ValueError("Using childFDs is not supported with usePTY=True.") + return process.PTYProcess(self, executable, args, env, path, + processProtocol, uid, gid, usePTY) + else: + return process.Process(self, executable, args, env, path, + processProtocol, uid, gid, childFDs) + elif platformType == "win32": + if uid is not None: + raise ValueError("Setting UID is unsupported on this platform.") + if gid is not None: + raise ValueError("Setting GID is unsupported on this platform.") + if usePTY: + raise ValueError("The usePTY parameter is not supported on Windows.") + if childFDs: + raise ValueError("Customizing childFDs is not supported on Windows.") + + if win32process: + from twisted.internet._dumbwin32proc import Process + return Process(self, processProtocol, executable, args, env, path) + else: + raise NotImplementedError( + "spawnProcess not available since pywin32 is not installed.") + else: + raise NotImplementedError( + "spawnProcess only available on Windows or POSIX.") + + # IReactorUDP + + def listenUDP(self, port, protocol, interface='', maxPacketSize=8192): + """Connects a given L{DatagramProtocol} to the given numeric UDP port. + + @returns: object conforming to L{IListeningPort}. + """ + p = udp.Port(port, protocol, interface, maxPacketSize, self) + p.startListening() + return p + + # IReactorMulticast + + def listenMulticast(self, port, protocol, interface='', maxPacketSize=8192, listenMultiple=False): + """Connects a given DatagramProtocol to the given numeric UDP port. + + EXPERIMENTAL. + + @returns: object conforming to IListeningPort. + """ + p = udp.MulticastPort(port, protocol, interface, maxPacketSize, self, listenMultiple) + p.startListening() + return p + + + # IReactorUNIX + + def connectUNIX(self, address, factory, timeout=30, checkPID=0): + assert unixEnabled, "UNIX support is not present" + c = unix.Connector(address, factory, timeout, self, checkPID) + c.connect() + return c + + def listenUNIX(self, address, factory, backlog=50, mode=0o666, wantPID=0): + assert unixEnabled, "UNIX support is not present" + p = unix.Port(address, factory, backlog, mode, self, wantPID) + p.startListening() + return p + + + # IReactorUNIXDatagram + + def listenUNIXDatagram(self, address, protocol, maxPacketSize=8192, + mode=0o666): + """ + Connects a given L{DatagramProtocol} to the given path. + + EXPERIMENTAL. + + @returns: object conforming to L{IListeningPort}. + """ + assert unixEnabled, "UNIX support is not present" + p = unix.DatagramPort(address, protocol, maxPacketSize, mode, self) + p.startListening() + return p + + def connectUNIXDatagram(self, address, protocol, maxPacketSize=8192, + mode=0o666, bindAddress=None): + """ + Connects a L{ConnectedDatagramProtocol} instance to a path. + + EXPERIMENTAL. + """ + assert unixEnabled, "UNIX support is not present" + p = unix.ConnectedDatagramPort(address, protocol, maxPacketSize, mode, bindAddress, self) + p.startListening() + return p + + + # IReactorSocket (no AF_UNIX on Windows) + + if unixEnabled: + _supportedAddressFamilies = ( + socket.AF_INET, socket.AF_INET6, socket.AF_UNIX, + ) + else: + _supportedAddressFamilies = ( + socket.AF_INET, socket.AF_INET6, + ) + + def adoptStreamPort(self, fileDescriptor, addressFamily, factory): + """ + Create a new L{IListeningPort} from an already-initialized socket. + + This just dispatches to a suitable port implementation (eg from + L{IReactorTCP}, etc) based on the specified C{addressFamily}. + + @see: L{twisted.internet.interfaces.IReactorSocket.adoptStreamPort} + """ + if addressFamily not in self._supportedAddressFamilies: + raise error.UnsupportedAddressFamily(addressFamily) + + if unixEnabled and addressFamily == socket.AF_UNIX: + p = unix.Port._fromListeningDescriptor( + self, fileDescriptor, factory) + else: + p = tcp.Port._fromListeningDescriptor( + self, fileDescriptor, addressFamily, factory) + p.startListening() + return p + + def adoptStreamConnection(self, fileDescriptor, addressFamily, factory): + """ + @see: + L{twisted.internet.interfaces.IReactorSocket.adoptStreamConnection} + """ + if addressFamily not in self._supportedAddressFamilies: + raise error.UnsupportedAddressFamily(addressFamily) + + if unixEnabled and addressFamily == socket.AF_UNIX: + return unix.Server._fromConnectedSocket( + fileDescriptor, factory, self) + else: + return tcp.Server._fromConnectedSocket( + fileDescriptor, addressFamily, factory, self) + + + def adoptDatagramPort(self, fileDescriptor, addressFamily, protocol, + maxPacketSize=8192): + if addressFamily not in (socket.AF_INET, socket.AF_INET6): + raise error.UnsupportedAddressFamily(addressFamily) + + p = udp.Port._fromListeningDescriptor( + self, fileDescriptor, addressFamily, protocol, + maxPacketSize=maxPacketSize) + p.startListening() + return p + + + + # IReactorTCP + + def listenTCP(self, port, factory, backlog=50, interface=''): + p = tcp.Port(port, factory, backlog, interface, self) + p.startListening() + return p + + def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): + c = tcp.Connector(host, port, factory, timeout, bindAddress, self) + c.connect() + return c + + # IReactorSSL (sometimes, not implemented) + + def connectSSL(self, host, port, factory, contextFactory, timeout=30, bindAddress=None): + if tls is not None: + tlsFactory = tls.TLSMemoryBIOFactory(contextFactory, True, factory) + return self.connectTCP(host, port, tlsFactory, timeout, bindAddress) + elif ssl is not None: + c = ssl.Connector( + host, port, factory, contextFactory, timeout, bindAddress, self) + c.connect() + return c + else: + assert False, "SSL support is not present" + + + + def listenSSL(self, port, factory, contextFactory, backlog=50, interface=''): + if tls is not None: + tlsFactory = tls.TLSMemoryBIOFactory(contextFactory, False, factory) + port = self.listenTCP(port, tlsFactory, backlog, interface) + port._type = 'TLS' + return port + elif ssl is not None: + p = ssl.Port( + port, factory, contextFactory, backlog, interface, self) + p.startListening() + return p + else: + assert False, "SSL support is not present" + + + def _removeAll(self, readers, writers): + """ + Remove all readers and writers, and list of removed L{IReadDescriptor}s + and L{IWriteDescriptor}s. + + Meant for calling from subclasses, to implement removeAll, like:: + + def removeAll(self): + return self._removeAll(self._reads, self._writes) + + where C{self._reads} and C{self._writes} are iterables. + """ + removedReaders = set(readers) - self._internalReaders + for reader in removedReaders: + self.removeReader(reader) + + removedWriters = set(writers) + for writer in removedWriters: + self.removeWriter(writer) + + return list(removedReaders | removedWriters) + + +class _PollLikeMixin(object): + """ + Mixin for poll-like reactors. + + Subclasses must define the following attributes:: + + - _POLL_DISCONNECTED - Bitmask for events indicating a connection was + lost. + - _POLL_IN - Bitmask for events indicating there is input to read. + - _POLL_OUT - Bitmask for events indicating output can be written. + + Must be mixed in to a subclass of PosixReactorBase (for + _disconnectSelectable). + """ + + def _doReadOrWrite(self, selectable, fd, event): + """ + fd is available for read or write, do the work and raise errors if + necessary. + """ + why = None + inRead = False + if event & self._POLL_DISCONNECTED and not (event & self._POLL_IN): + # Handle disconnection. But only if we finished processing all + # the pending input. + if fd in self._reads: + # If we were reading from the descriptor then this is a + # clean shutdown. We know there are no read events pending + # because we just checked above. It also might be a + # half-close (which is why we have to keep track of inRead). + inRead = True + why = CONNECTION_DONE + else: + # If we weren't reading, this is an error shutdown of some + # sort. + why = CONNECTION_LOST + else: + # Any non-disconnect event turns into a doRead or a doWrite. + try: + # First check to see if the descriptor is still valid. This + # gives fileno() a chance to raise an exception, too. + # Ideally, disconnection would always be indicated by the + # return value of doRead or doWrite (or an exception from + # one of those methods), but calling fileno here helps make + # buggy applications more transparent. + if selectable.fileno() == -1: + # -1 is sort of a historical Python artifact. Python + # files and sockets used to change their file descriptor + # to -1 when they closed. For the time being, we'll + # continue to support this anyway in case applications + # replicated it, plus abstract.FileDescriptor.fileno + # returns -1. Eventually it'd be good to deprecate this + # case. + why = _NO_FILEDESC + else: + if event & self._POLL_IN: + # Handle a read event. + why = selectable.doRead() + inRead = True + if not why and event & self._POLL_OUT: + # Handle a write event, as long as doRead didn't + # disconnect us. + why = selectable.doWrite() + inRead = False + except: + # Any exception from application code gets logged and will + # cause us to disconnect the selectable. + why = sys.exc_info()[1] + log.err() + if why: + self._disconnectSelectable(selectable, why, inRead) + + + +@implementer(IReactorFDSet) +class _ContinuousPolling(_PollLikeMixin, _DisconnectSelectableMixin): + """ + Schedule reads and writes based on the passage of time, rather than + notification. + + This is useful for supporting polling filesystem files, which C{epoll(7)} + does not support. + + The implementation uses L{_PollLikeMixin}, which is a bit hacky, but + re-implementing and testing the relevant code yet again is unappealing. + + @ivar _reactor: The L{EPollReactor} that is using this instance. + + @ivar _loop: A C{LoopingCall} that drives the polling, or L{None}. + + @ivar _readers: A C{set} of C{FileDescriptor} objects that should be read + from. + + @ivar _writers: A C{set} of C{FileDescriptor} objects that should be + written to. + """ + + # Attributes for _PollLikeMixin + _POLL_DISCONNECTED = 1 + _POLL_IN = 2 + _POLL_OUT = 4 + + + def __init__(self, reactor): + self._reactor = reactor + self._loop = None + self._readers = set() + self._writers = set() + + + def _checkLoop(self): + """ + Start or stop a C{LoopingCall} based on whether there are readers and + writers. + """ + if self._readers or self._writers: + if self._loop is None: + from twisted.internet.task import LoopingCall, _EPSILON + self._loop = LoopingCall(self.iterate) + self._loop.clock = self._reactor + # LoopingCall seems unhappy with timeout of 0, so use very + # small number: + self._loop.start(_EPSILON, now=False) + elif self._loop: + self._loop.stop() + self._loop = None + + + def iterate(self): + """ + Call C{doRead} and C{doWrite} on all readers and writers respectively. + """ + for reader in list(self._readers): + self._doReadOrWrite(reader, reader, self._POLL_IN) + for writer in list(self._writers): + self._doReadOrWrite(writer, writer, self._POLL_OUT) + + + def addReader(self, reader): + """ + Add a C{FileDescriptor} for notification of data available to read. + """ + self._readers.add(reader) + self._checkLoop() + + + def addWriter(self, writer): + """ + Add a C{FileDescriptor} for notification of data available to write. + """ + self._writers.add(writer) + self._checkLoop() + + + def removeReader(self, reader): + """ + Remove a C{FileDescriptor} from notification of data available to read. + """ + try: + self._readers.remove(reader) + except KeyError: + return + self._checkLoop() + + + def removeWriter(self, writer): + """ + Remove a C{FileDescriptor} from notification of data available to + write. + """ + try: + self._writers.remove(writer) + except KeyError: + return + self._checkLoop() + + + def removeAll(self): + """ + Remove all readers and writers. + """ + result = list(self._readers | self._writers) + # Don't reset to new value, since self.isWriting and .isReading refer + # to the existing instance: + self._readers.clear() + self._writers.clear() + return result + + + def getReaders(self): + """ + Return a list of the readers. + """ + return list(self._readers) + + + def getWriters(self): + """ + Return a list of the writers. + """ + return list(self._writers) + + + def isReading(self, fd): + """ + Checks if the file descriptor is currently being observed for read + readiness. + + @param fd: The file descriptor being checked. + @type fd: L{twisted.internet.abstract.FileDescriptor} + @return: C{True} if the file descriptor is being observed for read + readiness, C{False} otherwise. + @rtype: C{bool} + """ + return fd in self._readers + + + def isWriting(self, fd): + """ + Checks if the file descriptor is currently being observed for write + readiness. + + @param fd: The file descriptor being checked. + @type fd: L{twisted.internet.abstract.FileDescriptor} + @return: C{True} if the file descriptor is being observed for write + readiness, C{False} otherwise. + @rtype: C{bool} + """ + return fd in self._writers + + + +if tls is not None or ssl is not None: + classImplements(PosixReactorBase, IReactorSSL) +if unixEnabled: + classImplements(PosixReactorBase, IReactorUNIX, IReactorUNIXDatagram) +if processEnabled: + classImplements(PosixReactorBase, IReactorProcess) +if getattr(socket, 'fromfd', None) is not None: + classImplements(PosixReactorBase, IReactorSocket) + +__all__ = ["PosixReactorBase"] diff --git a/contrib/python/Twisted/py2/twisted/internet/process.py b/contrib/python/Twisted/py2/twisted/internet/process.py new file mode 100644 index 00000000000..de94cb838a1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/process.py @@ -0,0 +1,1114 @@ +# -*- test-case-name: twisted.test.test_process -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +UNIX Process management. + +Do NOT use this module directly - use reactor.spawnProcess() instead. + +Maintainer: Itamar Shtull-Trauring +""" + +from __future__ import division, absolute_import, print_function + +from twisted.python.runtime import platform + +if platform.isWindows(): + raise ImportError(("twisted.internet.process does not work on Windows. " + "Use the reactor.spawnProcess() API instead.")) + +import errno +import gc +import os +import io +import signal +import stat +import sys +import traceback + +try: + import pty +except ImportError: + pty = None + +try: + import fcntl, termios +except ImportError: + fcntl = None + +from zope.interface import implementer + +from twisted.python import log, failure +from twisted.python.util import switchUID +from twisted.python.compat import items, range, _PY3 +from twisted.internet import fdesc, abstract, error +from twisted.internet.main import CONNECTION_LOST, CONNECTION_DONE +from twisted.internet._baseprocess import BaseProcess +from twisted.internet.interfaces import IProcessTransport + +# Some people were importing this, which is incorrect, just keeping it +# here for backwards compatibility: +ProcessExitedAlready = error.ProcessExitedAlready + +reapProcessHandlers = {} + +def reapAllProcesses(): + """ + Reap all registered processes. + """ + # Coerce this to a list, as reaping the process changes the dictionary and + # causes a "size changed during iteration" exception + for process in list(reapProcessHandlers.values()): + process.reapProcess() + + + +def registerReapProcessHandler(pid, process): + """ + Register a process handler for the given pid, in case L{reapAllProcesses} + is called. + + @param pid: the pid of the process. + @param process: a process handler. + """ + if pid in reapProcessHandlers: + raise RuntimeError("Try to register an already registered process.") + try: + auxPID, status = os.waitpid(pid, os.WNOHANG) + except: + log.msg('Failed to reap %d:' % pid) + log.err() + auxPID = None + if auxPID: + process.processEnded(status) + else: + # if auxPID is 0, there are children but none have exited + reapProcessHandlers[pid] = process + + + +def unregisterReapProcessHandler(pid, process): + """ + Unregister a process handler previously registered with + L{registerReapProcessHandler}. + """ + if not (pid in reapProcessHandlers + and reapProcessHandlers[pid] == process): + raise RuntimeError("Try to unregister a process not registered.") + del reapProcessHandlers[pid] + + + +class ProcessWriter(abstract.FileDescriptor): + """ + (Internal) Helper class to write into a Process's input pipe. + + I am a helper which describes a selectable asynchronous writer to a + process's input pipe, including stdin. + + @ivar enableReadHack: A flag which determines how readability on this + write descriptor will be handled. If C{True}, then readability may + indicate the reader for this write descriptor has been closed (ie, + the connection has been lost). If C{False}, then readability events + are ignored. + """ + connected = 1 + ic = 0 + enableReadHack = False + + def __init__(self, reactor, proc, name, fileno, forceReadHack=False): + """ + Initialize, specifying a Process instance to connect to. + """ + abstract.FileDescriptor.__init__(self, reactor) + fdesc.setNonBlocking(fileno) + self.proc = proc + self.name = name + self.fd = fileno + + if not stat.S_ISFIFO(os.fstat(self.fileno()).st_mode): + # If the fd is not a pipe, then the read hack is never + # applicable. This case arises when ProcessWriter is used by + # StandardIO and stdout is redirected to a normal file. + self.enableReadHack = False + elif forceReadHack: + self.enableReadHack = True + else: + # Detect if this fd is actually a write-only fd. If it's + # valid to read, don't try to detect closing via read. + # This really only means that we cannot detect a TTY's write + # pipe being closed. + try: + os.read(self.fileno(), 0) + except OSError: + # It's a write-only pipe end, enable hack + self.enableReadHack = True + + if self.enableReadHack: + self.startReading() + + + def fileno(self): + """ + Return the fileno() of my process's stdin. + """ + return self.fd + + + def writeSomeData(self, data): + """ + Write some data to the open process. + """ + rv = fdesc.writeToFD(self.fd, data) + if rv == len(data) and self.enableReadHack: + # If the send buffer is now empty and it is necessary to monitor + # this descriptor for readability to detect close, try detecting + # readability now. + self.startReading() + return rv + + + def write(self, data): + self.stopReading() + abstract.FileDescriptor.write(self, data) + + + def doRead(self): + """ + The only way a write pipe can become "readable" is at EOF, because the + child has closed it, and we're using a reactor which doesn't + distinguish between readable and closed (such as the select reactor). + + Except that's not true on linux < 2.6.11. It has the following + characteristics: write pipe is completely empty => POLLOUT (writable in + select), write pipe is not completely empty => POLLIN (readable in + select), write pipe's reader closed => POLLIN|POLLERR (readable and + writable in select) + + That's what this funky code is for. If linux was not broken, this + function could be simply "return CONNECTION_LOST". + """ + if self.enableReadHack: + return CONNECTION_LOST + else: + self.stopReading() + + + def connectionLost(self, reason): + """ + See abstract.FileDescriptor.connectionLost. + """ + # At least on macOS 10.4, exiting while stdout is non-blocking can + # result in data loss. For some reason putting the file descriptor + # back into blocking mode seems to resolve this issue. + fdesc.setBlocking(self.fd) + + abstract.FileDescriptor.connectionLost(self, reason) + self.proc.childConnectionLost(self.name, reason) + + + +class ProcessReader(abstract.FileDescriptor): + """ + ProcessReader + + I am a selectable representation of a process's output pipe, such as + stdout and stderr. + """ + connected = True + + def __init__(self, reactor, proc, name, fileno): + """ + Initialize, specifying a process to connect to. + """ + abstract.FileDescriptor.__init__(self, reactor) + fdesc.setNonBlocking(fileno) + self.proc = proc + self.name = name + self.fd = fileno + self.startReading() + + + def fileno(self): + """ + Return the fileno() of my process's stderr. + """ + return self.fd + + + def writeSomeData(self, data): + # the only time this is actually called is after .loseConnection Any + # actual write attempt would fail, so we must avoid that. This hack + # allows us to use .loseConnection on both readers and writers. + assert data == b"" + return CONNECTION_LOST + + + def doRead(self): + """ + This is called when the pipe becomes readable. + """ + return fdesc.readFromFD(self.fd, self.dataReceived) + + + def dataReceived(self, data): + self.proc.childDataReceived(self.name, data) + + + def loseConnection(self): + if self.connected and not self.disconnecting: + self.disconnecting = 1 + self.stopReading() + self.reactor.callLater(0, self.connectionLost, + failure.Failure(CONNECTION_DONE)) + + + def connectionLost(self, reason): + """ + Close my end of the pipe, signal the Process (which signals the + ProcessProtocol). + """ + abstract.FileDescriptor.connectionLost(self, reason) + self.proc.childConnectionLost(self.name, reason) + + + +class _BaseProcess(BaseProcess, object): + """ + Base class for Process and PTYProcess. + """ + status = None + pid = None + + def reapProcess(self): + """ + Try to reap a process (without blocking) via waitpid. + + This is called when sigchild is caught or a Process object loses its + "connection" (stdout is closed) This ought to result in reaping all + zombie processes, since it will be called twice as often as it needs + to be. + + (Unfortunately, this is a slightly experimental approach, since + UNIX has no way to be really sure that your process is going to + go away w/o blocking. I don't want to block.) + """ + try: + try: + pid, status = os.waitpid(self.pid, os.WNOHANG) + except OSError as e: + if e.errno == errno.ECHILD: + # no child process + pid = None + else: + raise + except: + log.msg('Failed to reap %d:' % self.pid) + log.err() + pid = None + if pid: + self.processEnded(status) + unregisterReapProcessHandler(pid, self) + + + def _getReason(self, status): + exitCode = sig = None + if os.WIFEXITED(status): + exitCode = os.WEXITSTATUS(status) + else: + sig = os.WTERMSIG(status) + if exitCode or sig: + return error.ProcessTerminated(exitCode, sig, status) + return error.ProcessDone(status) + + + def signalProcess(self, signalID): + """ + Send the given signal C{signalID} to the process. It'll translate a + few signals ('HUP', 'STOP', 'INT', 'KILL', 'TERM') from a string + representation to its int value, otherwise it'll pass directly the + value provided + + @type signalID: C{str} or C{int} + """ + if signalID in ('HUP', 'STOP', 'INT', 'KILL', 'TERM'): + signalID = getattr(signal, 'SIG%s' % (signalID,)) + if self.pid is None: + raise ProcessExitedAlready() + try: + os.kill(self.pid, signalID) + except OSError as e: + if e.errno == errno.ESRCH: + raise ProcessExitedAlready() + else: + raise + + + def _resetSignalDisposition(self): + # The Python interpreter ignores some signals, and our child + # process will inherit that behaviour. To have a child process + # that responds to signals normally, we need to reset our + # child process's signal handling (just) after we fork and + # before we execvpe. + for signalnum in range(1, signal.NSIG): + if signal.getsignal(signalnum) == signal.SIG_IGN: + # Reset signal handling to the default + signal.signal(signalnum, signal.SIG_DFL) + + + def _fork(self, path, uid, gid, executable, args, environment, **kwargs): + """ + Fork and then exec sub-process. + + @param path: the path where to run the new process. + @type path: L{bytes} or L{unicode} + @param uid: if defined, the uid used to run the new process. + @type uid: L{int} + @param gid: if defined, the gid used to run the new process. + @type gid: L{int} + @param executable: the executable to run in a new process. + @type executable: L{str} + @param args: arguments used to create the new process. + @type args: L{list}. + @param environment: environment used for the new process. + @type environment: L{dict}. + @param kwargs: keyword arguments to L{_setupChild} method. + """ + collectorEnabled = gc.isenabled() + gc.disable() + try: + self.pid = os.fork() + except: + # Still in the parent process + if collectorEnabled: + gc.enable() + raise + else: + if self.pid == 0: + # A return value of 0 from fork() indicates that we are now + # executing in the child process. + + # Do not put *ANY* code outside the try block. The child + # process must either exec or _exit. If it gets outside this + # block (due to an exception that is not handled here, but + # which might be handled higher up), there will be two copies + # of the parent running in parallel, doing all kinds of damage. + + # After each change to this code, review it to make sure there + # are no exit paths. + + try: + # Stop debugging. If I am, I don't care anymore. + sys.settrace(None) + self._setupChild(**kwargs) + self._execChild(path, uid, gid, executable, args, + environment) + except: + # If there are errors, try to write something descriptive + # to stderr before exiting. + + # The parent's stderr isn't *necessarily* fd 2 anymore, or + # even still available; however, even libc assumes that + # write(2, err) is a useful thing to attempt. + + try: + stderr = os.fdopen(2, 'wb') + msg = ("Upon execvpe {0} {1} in environment id {2}" + "\n:").format(executable, str(args), + id(environment)) + + if _PY3: + + # On Python 3, print_exc takes a text stream, but + # on Python 2 it still takes a byte stream. So on + # Python 3 we will wrap up the byte stream returned + # by os.fdopen using TextIOWrapper. + + # We hard-code UTF-8 as the encoding here, rather + # than looking at something like + # getfilesystemencoding() or sys.stderr.encoding, + # because we want an encoding that will be able to + # encode the full range of code points. We are + # (most likely) talking to the parent process on + # the other end of this pipe and not the filesystem + # or the original sys.stderr, so there's no point + # in trying to match the encoding of one of those + # objects. + + stderr = io.TextIOWrapper(stderr, encoding="utf-8") + + stderr.write(msg) + traceback.print_exc(file=stderr) + stderr.flush() + + for fd in range(3): + os.close(fd) + except: + # Handle all errors during the error-reporting process + # silently to ensure that the child terminates. + pass + + # See comment above about making sure that we reach this line + # of code. + os._exit(1) + + # we are now in parent process + if collectorEnabled: + gc.enable() + self.status = -1 # this records the exit status of the child + + + def _setupChild(self, *args, **kwargs): + """ + Setup the child process. Override in subclasses. + """ + raise NotImplementedError() + + + def _execChild(self, path, uid, gid, executable, args, environment): + """ + The exec() which is done in the forked child. + """ + if path: + os.chdir(path) + if uid is not None or gid is not None: + if uid is None: + uid = os.geteuid() + if gid is None: + gid = os.getegid() + # set the UID before I actually exec the process + os.setuid(0) + os.setgid(0) + switchUID(uid, gid) + os.execvpe(executable, args, environment) + + + def __repr__(self): + """ + String representation of a process. + """ + return "<%s pid=%s status=%s>" % (self.__class__.__name__, + self.pid, self.status) + + + +class _FDDetector(object): + """ + This class contains the logic necessary to decide which of the available + system techniques should be used to detect the open file descriptors for + the current process. The chosen technique gets monkey-patched into the + _listOpenFDs method of this class so that the detection only needs to occur + once. + + @ivar listdir: The implementation of listdir to use. This gets overwritten + by the test cases. + @ivar getpid: The implementation of getpid to use, returns the PID of the + running process. + @ivar openfile: The implementation of open() to use, by default the Python + builtin. + """ + # So that we can unit test this + listdir = os.listdir + getpid = os.getpid + openfile = open + + def __init__(self): + self._implementations = [ + self._procFDImplementation, self._devFDImplementation, + self._fallbackFDImplementation] + + + def _listOpenFDs(self): + """ + Return an iterable of file descriptors which I{may} be open in this + process. + + This will try to return the fewest possible descriptors without missing + any. + """ + self._listOpenFDs = self._getImplementation() + return self._listOpenFDs() + + + def _getImplementation(self): + """ + Pick a method which gives correct results for C{_listOpenFDs} in this + runtime environment. + + This involves a lot of very platform-specific checks, some of which may + be relatively expensive. Therefore the returned method should be saved + and re-used, rather than always calling this method to determine what it + is. + + See the implementation for the details of how a method is selected. + """ + for impl in self._implementations: + try: + before = impl() + except: + continue + with self.openfile("/dev/null", "r"): + after = impl() + if before != after: + return impl + # If no implementation can detect the newly opened file above, then just + # return the last one. The last one should therefore always be one + # which makes a simple static guess which includes all possible open + # file descriptors, but perhaps also many other values which do not + # correspond to file descriptors. For example, the scheme implemented + # by _fallbackFDImplementation is suitable to be the last entry. + return impl + + + def _devFDImplementation(self): + """ + Simple implementation for systems where /dev/fd actually works. + See: http://www.freebsd.org/cgi/man.cgi?fdescfs + """ + dname = "/dev/fd" + result = [int(fd) for fd in self.listdir(dname)] + return result + + + def _procFDImplementation(self): + """ + Simple implementation for systems where /proc/pid/fd exists (we assume + it works). + """ + dname = "/proc/%d/fd" % (self.getpid(),) + return [int(fd) for fd in self.listdir(dname)] + + + def _fallbackFDImplementation(self): + """ + Fallback implementation where either the resource module can inform us + about the upper bound of how many FDs to expect, or where we just guess + a constant maximum if there is no resource module. + + All possible file descriptors from 0 to that upper bound are returned + with no attempt to exclude invalid file descriptor values. + """ + try: + import resource + except ImportError: + maxfds = 1024 + else: + # OS-X reports 9223372036854775808. That's a lot of fds to close. + # OS-X should get the /dev/fd implementation instead, so mostly + # this check probably isn't necessary. + maxfds = min(1024, resource.getrlimit(resource.RLIMIT_NOFILE)[1]) + return range(maxfds) + + + +detector = _FDDetector() + +def _listOpenFDs(): + """ + Use the global detector object to figure out which FD implementation to + use. + """ + return detector._listOpenFDs() + + + +@implementer(IProcessTransport) +class Process(_BaseProcess): + """ + An operating-system Process. + + This represents an operating-system process with arbitrary input/output + pipes connected to it. Those pipes may represent standard input, + standard output, and standard error, or any other file descriptor. + + On UNIX, this is implemented using fork(), exec(), pipe() + and fcntl(). These calls may not exist elsewhere so this + code is not cross-platform. (also, windows can only select + on sockets...) + """ + debug = False + debug_child = False + + status = -1 + pid = None + + processWriterFactory = ProcessWriter + processReaderFactory = ProcessReader + + def __init__(self, + reactor, executable, args, environment, path, proto, + uid=None, gid=None, childFDs=None): + """ + Spawn an operating-system process. + + This is where the hard work of disconnecting all currently open + files / forking / executing the new process happens. (This is + executed automatically when a Process is instantiated.) + + This will also run the subprocess as a given user ID and group ID, if + specified. (Implementation Note: this doesn't support all the arcane + nuances of setXXuid on UNIX: it will assume that either your effective + or real UID is 0.) + """ + if not proto: + assert 'r' not in childFDs.values() + assert 'w' not in childFDs.values() + _BaseProcess.__init__(self, proto) + + self.pipes = {} + # keys are childFDs, we can sense them closing + # values are ProcessReader/ProcessWriters + + helpers = {} + # keys are childFDs + # values are parentFDs + + if childFDs is None: + childFDs = {0: "w", # we write to the child's stdin + 1: "r", # we read from their stdout + 2: "r", # and we read from their stderr + } + + debug = self.debug + if debug: print("childFDs", childFDs) + + _openedPipes = [] + def pipe(): + r, w = os.pipe() + _openedPipes.extend([r, w]) + return r, w + + # fdmap.keys() are filenos of pipes that are used by the child. + fdmap = {} # maps childFD to parentFD + try: + for childFD, target in items(childFDs): + if debug: print("[%d]" % childFD, target) + if target == "r": + # we need a pipe that the parent can read from + readFD, writeFD = pipe() + if debug: print("readFD=%d, writeFD=%d" % (readFD, writeFD)) + fdmap[childFD] = writeFD # child writes to this + helpers[childFD] = readFD # parent reads from this + elif target == "w": + # we need a pipe that the parent can write to + readFD, writeFD = pipe() + if debug: print("readFD=%d, writeFD=%d" % (readFD, writeFD)) + fdmap[childFD] = readFD # child reads from this + helpers[childFD] = writeFD # parent writes to this + else: + assert type(target) == int, '%r should be an int' % (target,) + fdmap[childFD] = target # parent ignores this + if debug: print("fdmap", fdmap) + if debug: print("helpers", helpers) + # the child only cares about fdmap.values() + + self._fork(path, uid, gid, executable, args, environment, fdmap=fdmap) + except: + for pipe in _openedPipes: + os.close(pipe) + raise + + # we are the parent process: + self.proto = proto + + # arrange for the parent-side pipes to be read and written + for childFD, parentFD in items(helpers): + os.close(fdmap[childFD]) + if childFDs[childFD] == "r": + reader = self.processReaderFactory(reactor, self, childFD, + parentFD) + self.pipes[childFD] = reader + + if childFDs[childFD] == "w": + writer = self.processWriterFactory(reactor, self, childFD, + parentFD, forceReadHack=True) + self.pipes[childFD] = writer + + try: + # the 'transport' is used for some compatibility methods + if self.proto is not None: + self.proto.makeConnection(self) + except: + log.err() + + # The reactor might not be running yet. This might call back into + # processEnded synchronously, triggering an application-visible + # callback. That's probably not ideal. The replacement API for + # spawnProcess should improve upon this situation. + registerReapProcessHandler(self.pid, self) + + + def _setupChild(self, fdmap): + """ + fdmap[childFD] = parentFD + + The child wants to end up with 'childFD' attached to what used to be + the parent's parentFD. As an example, a bash command run like + 'command 2>&1' would correspond to an fdmap of {0:0, 1:1, 2:1}. + 'command >foo.txt' would be {0:0, 1:os.open('foo.txt'), 2:2}. + + This is accomplished in two steps:: + + 1. close all file descriptors that aren't values of fdmap. This + means 0 .. maxfds (or just the open fds within that range, if + the platform supports '/proc//fd'). + + 2. for each childFD:: + + - if fdmap[childFD] == childFD, the descriptor is already in + place. Make sure the CLOEXEC flag is not set, then delete + the entry from fdmap. + + - if childFD is in fdmap.values(), then the target descriptor + is busy. Use os.dup() to move it elsewhere, update all + fdmap[childFD] items that point to it, then close the + original. Then fall through to the next case. + + - now fdmap[childFD] is not in fdmap.values(), and is free. + Use os.dup2() to move it to the right place, then close the + original. + """ + debug = self.debug_child + if debug: + errfd = sys.stderr + errfd.write("starting _setupChild\n") + + destList = fdmap.values() + for fd in _listOpenFDs(): + if fd in destList: + continue + if debug and fd == errfd.fileno(): + continue + try: + os.close(fd) + except: + pass + + # at this point, the only fds still open are the ones that need to + # be moved to their appropriate positions in the child (the targets + # of fdmap, i.e. fdmap.values() ) + + if debug: print("fdmap", fdmap, file=errfd) + for child in sorted(fdmap.keys()): + target = fdmap[child] + if target == child: + # fd is already in place + if debug: print("%d already in place" % target, file=errfd) + fdesc._unsetCloseOnExec(child) + else: + if child in fdmap.values(): + # we can't replace child-fd yet, as some other mapping + # still needs the fd it wants to target. We must preserve + # that old fd by duping it to a new home. + newtarget = os.dup(child) # give it a safe home + if debug: print("os.dup(%d) -> %d" % (child, newtarget), + file=errfd) + os.close(child) # close the original + for c, p in items(fdmap): + if p == child: + fdmap[c] = newtarget # update all pointers + # now it should be available + if debug: print("os.dup2(%d,%d)" % (target, child), file=errfd) + os.dup2(target, child) + + # At this point, the child has everything it needs. We want to close + # everything that isn't going to be used by the child, i.e. + # everything not in fdmap.keys(). The only remaining fds open are + # those in fdmap.values(). + + # Any given fd may appear in fdmap.values() multiple times, so we + # need to remove duplicates first. + + old = [] + for fd in fdmap.values(): + if not fd in old: + if not fd in fdmap.keys(): + old.append(fd) + if debug: print("old", old, file=errfd) + for fd in old: + os.close(fd) + + self._resetSignalDisposition() + + + def writeToChild(self, childFD, data): + self.pipes[childFD].write(data) + + + def closeChildFD(self, childFD): + # for writer pipes, loseConnection tries to write the remaining data + # out to the pipe before closing it + # if childFD is not in the list of pipes, assume that it is already + # closed + if childFD in self.pipes: + self.pipes[childFD].loseConnection() + + + def pauseProducing(self): + for p in self.pipes.itervalues(): + if isinstance(p, ProcessReader): + p.stopReading() + + + def resumeProducing(self): + for p in self.pipes.itervalues(): + if isinstance(p, ProcessReader): + p.startReading() + + # compatibility + def closeStdin(self): + """ + Call this to close standard input on this process. + """ + self.closeChildFD(0) + + + def closeStdout(self): + self.closeChildFD(1) + + + def closeStderr(self): + self.closeChildFD(2) + + + def loseConnection(self): + self.closeStdin() + self.closeStderr() + self.closeStdout() + + + def write(self, data): + """ + Call this to write to standard input on this process. + + NOTE: This will silently lose data if there is no standard input. + """ + if 0 in self.pipes: + self.pipes[0].write(data) + + + def registerProducer(self, producer, streaming): + """ + Call this to register producer for standard input. + + If there is no standard input producer.stopProducing() will + be called immediately. + """ + if 0 in self.pipes: + self.pipes[0].registerProducer(producer, streaming) + else: + producer.stopProducing() + + + def unregisterProducer(self): + """ + Call this to unregister producer for standard input.""" + if 0 in self.pipes: + self.pipes[0].unregisterProducer() + + + def writeSequence(self, seq): + """ + Call this to write to standard input on this process. + + NOTE: This will silently lose data if there is no standard input. + """ + if 0 in self.pipes: + self.pipes[0].writeSequence(seq) + + + def childDataReceived(self, name, data): + self.proto.childDataReceived(name, data) + + + def childConnectionLost(self, childFD, reason): + # this is called when one of the helpers (ProcessReader or + # ProcessWriter) notices their pipe has been closed + os.close(self.pipes[childFD].fileno()) + del self.pipes[childFD] + try: + self.proto.childConnectionLost(childFD) + except: + log.err() + self.maybeCallProcessEnded() + + + def maybeCallProcessEnded(self): + # we don't call ProcessProtocol.processEnded until: + # the child has terminated, AND + # all writers have indicated an error status, AND + # all readers have indicated EOF + # This insures that we've gathered all output from the process. + if self.pipes: + return + if not self.lostProcess: + self.reapProcess() + return + _BaseProcess.maybeCallProcessEnded(self) + + + +@implementer(IProcessTransport) +class PTYProcess(abstract.FileDescriptor, _BaseProcess): + """ + An operating-system Process that uses PTY support. + """ + + status = -1 + pid = None + + def __init__(self, reactor, executable, args, environment, path, proto, + uid=None, gid=None, usePTY=None): + """ + Spawn an operating-system process. + + This is where the hard work of disconnecting all currently open + files / forking / executing the new process happens. (This is + executed automatically when a Process is instantiated.) + + This will also run the subprocess as a given user ID and group ID, if + specified. (Implementation Note: this doesn't support all the arcane + nuances of setXXuid on UNIX: it will assume that either your effective + or real UID is 0.) + """ + if pty is None and not isinstance(usePTY, (tuple, list)): + # no pty module and we didn't get a pty to use + raise NotImplementedError( + "cannot use PTYProcess on platforms without the pty module.") + abstract.FileDescriptor.__init__(self, reactor) + _BaseProcess.__init__(self, proto) + + if isinstance(usePTY, (tuple, list)): + masterfd, slavefd, _ = usePTY + else: + masterfd, slavefd = pty.openpty() + + try: + self._fork(path, uid, gid, executable, args, environment, + masterfd=masterfd, slavefd=slavefd) + except: + if not isinstance(usePTY, (tuple, list)): + os.close(masterfd) + os.close(slavefd) + raise + + # we are now in parent process: + os.close(slavefd) + fdesc.setNonBlocking(masterfd) + self.fd = masterfd + self.startReading() + self.connected = 1 + self.status = -1 + try: + self.proto.makeConnection(self) + except: + log.err() + registerReapProcessHandler(self.pid, self) + + + def _setupChild(self, masterfd, slavefd): + """ + Set up child process after C{fork()} but before C{exec()}. + + This involves: + + - closing C{masterfd}, since it is not used in the subprocess + + - creating a new session with C{os.setsid} + + - changing the controlling terminal of the process (and the new + session) to point at C{slavefd} + + - duplicating C{slavefd} to standard input, output, and error + + - closing all other open file descriptors (according to + L{_listOpenFDs}) + + - re-setting all signal handlers to C{SIG_DFL} + + @param masterfd: The master end of a PTY file descriptors opened with + C{openpty}. + @type masterfd: L{int} + + @param slavefd: The slave end of a PTY opened with C{openpty}. + @type slavefd: L{int} + """ + os.close(masterfd) + os.setsid() + fcntl.ioctl(slavefd, termios.TIOCSCTTY, '') + + for fd in range(3): + if fd != slavefd: + os.close(fd) + + os.dup2(slavefd, 0) # stdin + os.dup2(slavefd, 1) # stdout + os.dup2(slavefd, 2) # stderr + + for fd in _listOpenFDs(): + if fd > 2: + try: + os.close(fd) + except: + pass + + self._resetSignalDisposition() + + + def closeStdin(self): + # PTYs do not have stdin/stdout/stderr. They only have in and out, just + # like sockets. You cannot close one without closing off the entire PTY + pass + + + def closeStdout(self): + pass + + + def closeStderr(self): + pass + + + def doRead(self): + """ + Called when my standard output stream is ready for reading. + """ + return fdesc.readFromFD( + self.fd, + lambda data: self.proto.childDataReceived(1, data)) + + + def fileno(self): + """ + This returns the file number of standard output on this process. + """ + return self.fd + + + def maybeCallProcessEnded(self): + # two things must happen before we call the ProcessProtocol's + # processEnded method. 1: the child process must die and be reaped + # (which calls our own processEnded method). 2: the child must close + # their stdin/stdout/stderr fds, causing the pty to close, causing + # our connectionLost method to be called. #2 can also be triggered + # by calling .loseConnection(). + if self.lostProcess == 2: + _BaseProcess.maybeCallProcessEnded(self) + + + def connectionLost(self, reason): + """ + I call this to clean up when one or all of my connections has died. + """ + abstract.FileDescriptor.connectionLost(self, reason) + os.close(self.fd) + self.lostProcess += 1 + self.maybeCallProcessEnded() + + + def writeSomeData(self, data): + """ + Write some data to the open process. + """ + return fdesc.writeToFD(self.fd, data) diff --git a/contrib/python/Twisted/py2/twisted/internet/protocol.py b/contrib/python/Twisted/py2/twisted/internet/protocol.py new file mode 100644 index 00000000000..37364287e22 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/protocol.py @@ -0,0 +1,933 @@ +# -*- test-case-name: twisted.test.test_factories,twisted.internet.test.test_protocol -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Standard implementations of Twisted protocol-related interfaces. + +Start here if you are looking to write a new protocol implementation for +Twisted. The Protocol class contains some introductory material. +""" + +from __future__ import division, absolute_import + +import random +from zope.interface import implementer + +from twisted.python import log, failure, components +from twisted.internet import interfaces, error, defer +from twisted.logger import _loggerFor +from twisted.python._oldstyle import _oldStyle + + +@implementer(interfaces.IProtocolFactory, interfaces.ILoggingContext) +@_oldStyle +class Factory: + """ + This is a factory which produces protocols. + + By default, buildProtocol will create a protocol of the class given in + self.protocol. + """ + + # Put a subclass of Protocol here: + protocol = None + + numPorts = 0 + noisy = True + + @classmethod + def forProtocol(cls, protocol, *args, **kwargs): + """ + Create a factory for the given protocol. + + It sets the C{protocol} attribute and returns the constructed factory + instance. + + @param protocol: A L{Protocol} subclass + + @param args: Positional arguments for the factory. + + @param kwargs: Keyword arguments for the factory. + + @return: A L{Factory} instance wired up to C{protocol}. + """ + factory = cls(*args, **kwargs) + factory.protocol = protocol + return factory + + + def logPrefix(self): + """ + Describe this factory for log messages. + """ + return self.__class__.__name__ + + + def doStart(self): + """ + Make sure startFactory is called. + + Users should not call this function themselves! + """ + if not self.numPorts: + if self.noisy: + _loggerFor(self).info("Starting factory {factory!r}", + factory=self) + self.startFactory() + self.numPorts = self.numPorts + 1 + + + def doStop(self): + """ + Make sure stopFactory is called. + + Users should not call this function themselves! + """ + if self.numPorts == 0: + # This shouldn't happen, but does sometimes and this is better + # than blowing up in assert as we did previously. + return + self.numPorts = self.numPorts - 1 + if not self.numPorts: + if self.noisy: + _loggerFor(self).info("Stopping factory {factory!r}", + factory=self) + self.stopFactory() + + + def startFactory(self): + """ + This will be called before I begin listening on a Port or Connector. + + It will only be called once, even if the factory is connected + to multiple ports. + + This can be used to perform 'unserialization' tasks that + are best put off until things are actually running, such + as connecting to a database, opening files, etcetera. + """ + + + def stopFactory(self): + """ + This will be called before I stop listening on all Ports/Connectors. + + This can be overridden to perform 'shutdown' tasks such as disconnecting + database connections, closing files, etc. + + It will be called, for example, before an application shuts down, + if it was connected to a port. User code should not call this function + directly. + """ + + + def buildProtocol(self, addr): + """ + Create an instance of a subclass of Protocol. + + The returned instance will handle input on an incoming server + connection, and an attribute "factory" pointing to the creating + factory. + + Alternatively, L{None} may be returned to immediately close the + new connection. + + Override this method to alter how Protocol instances get created. + + @param addr: an object implementing L{twisted.internet.interfaces.IAddress} + """ + p = self.protocol() + p.factory = self + return p + + + +class ClientFactory(Factory): + """ + A Protocol factory for clients. + + This can be used together with the various connectXXX methods in + reactors. + """ + + def startedConnecting(self, connector): + """ + Called when a connection has been started. + + You can call connector.stopConnecting() to stop the connection attempt. + + @param connector: a Connector object. + """ + + + def clientConnectionFailed(self, connector, reason): + """ + Called when a connection has failed to connect. + + It may be useful to call connector.connect() - this will reconnect. + + @type reason: L{twisted.python.failure.Failure} + """ + + + def clientConnectionLost(self, connector, reason): + """ + Called when an established connection is lost. + + It may be useful to call connector.connect() - this will reconnect. + + @type reason: L{twisted.python.failure.Failure} + """ + + + +class _InstanceFactory(ClientFactory): + """ + Factory used by ClientCreator. + + @ivar deferred: The L{Deferred} which represents this connection attempt and + which will be fired when it succeeds or fails. + + @ivar pending: After a connection attempt succeeds or fails, a delayed call + which will fire the L{Deferred} representing this connection attempt. + """ + + noisy = False + pending = None + + def __init__(self, reactor, instance, deferred): + self.reactor = reactor + self.instance = instance + self.deferred = deferred + + + def __repr__(self): + return "" % (self.instance, ) + + + def buildProtocol(self, addr): + """ + Return the pre-constructed protocol instance and arrange to fire the + waiting L{Deferred} to indicate success establishing the connection. + """ + self.pending = self.reactor.callLater( + 0, self.fire, self.deferred.callback, self.instance) + self.deferred = None + return self.instance + + + def clientConnectionFailed(self, connector, reason): + """ + Arrange to fire the waiting L{Deferred} with the given failure to + indicate the connection could not be established. + """ + self.pending = self.reactor.callLater( + 0, self.fire, self.deferred.errback, reason) + self.deferred = None + + + def fire(self, func, value): + """ + Clear C{self.pending} to avoid a reference cycle and then invoke func + with the value. + """ + self.pending = None + func(value) + + + +@_oldStyle +class ClientCreator: + """ + Client connections that do not require a factory. + + The various connect* methods create a protocol instance using the given + protocol class and arguments, and connect it, returning a Deferred of the + resulting protocol instance. + + Useful for cases when we don't really need a factory. Mainly this + is when there is no shared state between protocol instances, and no need + to reconnect. + + The C{connectTCP}, C{connectUNIX}, and C{connectSSL} methods each return a + L{Deferred} which will fire with an instance of the protocol class passed to + L{ClientCreator.__init__}. These Deferred can be cancelled to abort the + connection attempt (in a very unlikely case, cancelling the Deferred may not + prevent the protocol from being instantiated and connected to a transport; + if this happens, it will be disconnected immediately afterwards and the + Deferred will still errback with L{CancelledError}). + """ + + def __init__(self, reactor, protocolClass, *args, **kwargs): + self.reactor = reactor + self.protocolClass = protocolClass + self.args = args + self.kwargs = kwargs + + + def _connect(self, method, *args, **kwargs): + """ + Initiate a connection attempt. + + @param method: A callable which will actually start the connection + attempt. For example, C{reactor.connectTCP}. + + @param *args: Positional arguments to pass to C{method}, excluding the + factory. + + @param **kwargs: Keyword arguments to pass to C{method}. + + @return: A L{Deferred} which fires with an instance of the protocol + class passed to this L{ClientCreator}'s initializer or fails if the + connection cannot be set up for some reason. + """ + def cancelConnect(deferred): + connector.disconnect() + if f.pending is not None: + f.pending.cancel() + d = defer.Deferred(cancelConnect) + f = _InstanceFactory( + self.reactor, self.protocolClass(*self.args, **self.kwargs), d) + connector = method(factory=f, *args, **kwargs) + return d + + + def connectTCP(self, host, port, timeout=30, bindAddress=None): + """ + Connect to a TCP server. + + The parameters are all the same as to L{IReactorTCP.connectTCP} except + that the factory parameter is omitted. + + @return: A L{Deferred} which fires with an instance of the protocol + class passed to this L{ClientCreator}'s initializer or fails if the + connection cannot be set up for some reason. + """ + return self._connect( + self.reactor.connectTCP, host, port, timeout=timeout, + bindAddress=bindAddress) + + + def connectUNIX(self, address, timeout=30, checkPID=False): + """ + Connect to a Unix socket. + + The parameters are all the same as to L{IReactorUNIX.connectUNIX} except + that the factory parameter is omitted. + + @return: A L{Deferred} which fires with an instance of the protocol + class passed to this L{ClientCreator}'s initializer or fails if the + connection cannot be set up for some reason. + """ + return self._connect( + self.reactor.connectUNIX, address, timeout=timeout, + checkPID=checkPID) + + + def connectSSL(self, host, port, contextFactory, timeout=30, bindAddress=None): + """ + Connect to an SSL server. + + The parameters are all the same as to L{IReactorSSL.connectSSL} except + that the factory parameter is omitted. + + @return: A L{Deferred} which fires with an instance of the protocol + class passed to this L{ClientCreator}'s initializer or fails if the + connection cannot be set up for some reason. + """ + return self._connect( + self.reactor.connectSSL, host, port, + contextFactory=contextFactory, timeout=timeout, + bindAddress=bindAddress) + + + +class ReconnectingClientFactory(ClientFactory): + """ + Factory which auto-reconnects clients with an exponential back-off. + + Note that clients should call my resetDelay method after they have + connected successfully. + + @ivar maxDelay: Maximum number of seconds between connection attempts. + @ivar initialDelay: Delay for the first reconnection attempt. + @ivar factor: A multiplicitive factor by which the delay grows + @ivar jitter: Percentage of randomness to introduce into the delay length + to prevent stampeding. + @ivar clock: The clock used to schedule reconnection. It's mainly useful to + be parametrized in tests. If the factory is serialized, this attribute + will not be serialized, and the default value (the reactor) will be + restored when deserialized. + @type clock: L{IReactorTime} + @ivar maxRetries: Maximum number of consecutive unsuccessful connection + attempts, after which no further connection attempts will be made. If + this is not explicitly set, no maximum is applied. + """ + maxDelay = 3600 + initialDelay = 1.0 + # Note: These highly sensitive factors have been precisely measured by + # the National Institute of Science and Technology. Take extreme care + # in altering them, or you may damage your Internet! + # (Seriously: ) + factor = 2.7182818284590451 # (math.e) + # Phi = 1.6180339887498948 # (Phi is acceptable for use as a + # factor if e is too large for your application.) + + # This is the value of the molar Planck constant times c, joule + # meter/mole. The value is attributable to + # https://physics.nist.gov/cgi-bin/cuu/Value?nahc|search_for=molar+planck+constant+times+c + jitter = 0.119626565582 + + delay = initialDelay + retries = 0 + maxRetries = None + _callID = None + connector = None + clock = None + + continueTrying = 1 + + + def clientConnectionFailed(self, connector, reason): + if self.continueTrying: + self.connector = connector + self.retry() + + + def clientConnectionLost(self, connector, unused_reason): + if self.continueTrying: + self.connector = connector + self.retry() + + + def retry(self, connector=None): + """ + Have this connector connect again, after a suitable delay. + """ + if not self.continueTrying: + if self.noisy: + log.msg("Abandoning %s on explicit request" % (connector,)) + return + + if connector is None: + if self.connector is None: + raise ValueError("no connector to retry") + else: + connector = self.connector + + self.retries += 1 + if self.maxRetries is not None and (self.retries > self.maxRetries): + if self.noisy: + log.msg("Abandoning %s after %d retries." % + (connector, self.retries)) + return + + self.delay = min(self.delay * self.factor, self.maxDelay) + if self.jitter: + self.delay = random.normalvariate(self.delay, + self.delay * self.jitter) + + if self.noisy: + log.msg("%s will retry in %d seconds" % (connector, self.delay,)) + + def reconnector(): + self._callID = None + connector.connect() + if self.clock is None: + from twisted.internet import reactor + self.clock = reactor + self._callID = self.clock.callLater(self.delay, reconnector) + + + def stopTrying(self): + """ + Put a stop to any attempt to reconnect in progress. + """ + # ??? Is this function really stopFactory? + if self._callID: + self._callID.cancel() + self._callID = None + self.continueTrying = 0 + if self.connector: + try: + self.connector.stopConnecting() + except error.NotConnectingError: + pass + + + def resetDelay(self): + """ + Call this method after a successful connection: it resets the delay and + the retry counter. + """ + self.delay = self.initialDelay + self.retries = 0 + self._callID = None + self.continueTrying = 1 + + + def __getstate__(self): + """ + Remove all of the state which is mutated by connection attempts and + failures, returning just the state which describes how reconnections + should be attempted. This will make the unserialized instance + behave just as this one did when it was first instantiated. + """ + state = self.__dict__.copy() + for key in ['connector', 'retries', 'delay', + 'continueTrying', '_callID', 'clock']: + if key in state: + del state[key] + return state + + + +class ServerFactory(Factory): + """ + Subclass this to indicate that your protocol.Factory is only usable for servers. + """ + + + +@_oldStyle +class BaseProtocol: + """ + This is the abstract superclass of all protocols. + + Some methods have helpful default implementations here so that they can + easily be shared, but otherwise the direct subclasses of this class are more + interesting, L{Protocol} and L{ProcessProtocol}. + """ + connected = 0 + transport = None + + def makeConnection(self, transport): + """ + Make a connection to a transport and a server. + + This sets the 'transport' attribute of this Protocol, and calls the + connectionMade() callback. + """ + self.connected = 1 + self.transport = transport + self.connectionMade() + + + def connectionMade(self): + """ + Called when a connection is made. + + This may be considered the initializer of the protocol, because + it is called when the connection is completed. For clients, + this is called once the connection to the server has been + established; for servers, this is called after an accept() call + stops blocking and a socket has been received. If you need to + send any greeting or initial message, do it here. + """ + +connectionDone = failure.Failure(error.ConnectionDone()) +connectionDone.cleanFailure() + + +@implementer(interfaces.IProtocol, interfaces.ILoggingContext) +class Protocol(BaseProtocol): + """ + This is the base class for streaming connection-oriented protocols. + + If you are going to write a new connection-oriented protocol for Twisted, + start here. Any protocol implementation, either client or server, should + be a subclass of this class. + + The API is quite simple. Implement L{dataReceived} to handle both + event-based and synchronous input; output can be sent through the + 'transport' attribute, which is to be an instance that implements + L{twisted.internet.interfaces.ITransport}. Override C{connectionLost} to be + notified when the connection ends. + + Some subclasses exist already to help you write common types of protocols: + see the L{twisted.protocols.basic} module for a few of them. + """ + + def logPrefix(self): + """ + Return a prefix matching the class name, to identify log messages + related to this protocol instance. + """ + return self.__class__.__name__ + + + def dataReceived(self, data): + """ + Called whenever data is received. + + Use this method to translate to a higher-level message. Usually, some + callback will be made upon the receipt of each complete protocol + message. + + @param data: a string of indeterminate length. Please keep in mind + that you will probably need to buffer some data, as partial + (or multiple) protocol messages may be received! I recommend + that unit tests for protocols call through to this method with + differing chunk sizes, down to one byte at a time. + """ + + def connectionLost(self, reason=connectionDone): + """ + Called when the connection is shut down. + + Clear any circular references here, and any external references + to this Protocol. The connection has been closed. + + @type reason: L{twisted.python.failure.Failure} + """ + + + +@implementer(interfaces.IConsumer) +class ProtocolToConsumerAdapter(components.Adapter): + + def write(self, data): + self.original.dataReceived(data) + + + def registerProducer(self, producer, streaming): + pass + + + def unregisterProducer(self): + pass + + +components.registerAdapter(ProtocolToConsumerAdapter, interfaces.IProtocol, + interfaces.IConsumer) + + + +@implementer(interfaces.IProtocol) +class ConsumerToProtocolAdapter(components.Adapter): + + def dataReceived(self, data): + self.original.write(data) + + + def connectionLost(self, reason): + pass + + + def makeConnection(self, transport): + pass + + + def connectionMade(self): + pass + + +components.registerAdapter(ConsumerToProtocolAdapter, interfaces.IConsumer, + interfaces.IProtocol) + + + +@implementer(interfaces.IProcessProtocol) +class ProcessProtocol(BaseProtocol): + """ + Base process protocol implementation which does simple dispatching for + stdin, stdout, and stderr file descriptors. + """ + + def childDataReceived(self, childFD, data): + if childFD == 1: + self.outReceived(data) + elif childFD == 2: + self.errReceived(data) + + + def outReceived(self, data): + """ + Some data was received from stdout. + """ + + + def errReceived(self, data): + """ + Some data was received from stderr. + """ + + + def childConnectionLost(self, childFD): + if childFD == 0: + self.inConnectionLost() + elif childFD == 1: + self.outConnectionLost() + elif childFD == 2: + self.errConnectionLost() + + + def inConnectionLost(self): + """ + This will be called when stdin is closed. + """ + + + def outConnectionLost(self): + """ + This will be called when stdout is closed. + """ + + + def errConnectionLost(self): + """ + This will be called when stderr is closed. + """ + + + def processExited(self, reason): + """ + This will be called when the subprocess exits. + + @type reason: L{twisted.python.failure.Failure} + """ + + + def processEnded(self, reason): + """ + Called when the child process exits and all file descriptors + associated with it have been closed. + + @type reason: L{twisted.python.failure.Failure} + """ + + + +@_oldStyle +class AbstractDatagramProtocol: + """ + Abstract protocol for datagram-oriented transports, e.g. IP, ICMP, ARP, UDP. + """ + + transport = None + numPorts = 0 + noisy = True + + def __getstate__(self): + d = self.__dict__.copy() + d['transport'] = None + return d + + + def doStart(self): + """ + Make sure startProtocol is called. + + This will be called by makeConnection(), users should not call it. + """ + if not self.numPorts: + if self.noisy: + log.msg("Starting protocol %s" % self) + self.startProtocol() + self.numPorts = self.numPorts + 1 + + + def doStop(self): + """ + Make sure stopProtocol is called. + + This will be called by the port, users should not call it. + """ + assert self.numPorts > 0 + self.numPorts = self.numPorts - 1 + self.transport = None + if not self.numPorts: + if self.noisy: + log.msg("Stopping protocol %s" % self) + self.stopProtocol() + + + def startProtocol(self): + """ + Called when a transport is connected to this protocol. + + Will only be called once, even if multiple ports are connected. + """ + + + def stopProtocol(self): + """ + Called when the transport is disconnected. + + Will only be called once, after all ports are disconnected. + """ + + + def makeConnection(self, transport): + """ + Make a connection to a transport and a server. + + This sets the 'transport' attribute of this DatagramProtocol, and calls the + doStart() callback. + """ + assert self.transport == None + self.transport = transport + self.doStart() + + + def datagramReceived(self, datagram, addr): + """ + Called when a datagram is received. + + @param datagram: the string received from the transport. + @param addr: tuple of source of datagram. + """ + + + +@implementer(interfaces.ILoggingContext) +class DatagramProtocol(AbstractDatagramProtocol): + """ + Protocol for datagram-oriented transport, e.g. UDP. + + @type transport: L{None} or + L{IUDPTransport} provider + @ivar transport: The transport with which this protocol is associated, + if it is associated with one. + """ + + def logPrefix(self): + """ + Return a prefix matching the class name, to identify log messages + related to this protocol instance. + """ + return self.__class__.__name__ + + + def connectionRefused(self): + """ + Called due to error from write in connected mode. + + Note this is a result of ICMP message generated by *previous* + write. + """ + + + +class ConnectedDatagramProtocol(DatagramProtocol): + """ + Protocol for connected datagram-oriented transport. + + No longer necessary for UDP. + """ + + def datagramReceived(self, datagram): + """ + Called when a datagram is received. + + @param datagram: the string received from the transport. + """ + + def connectionFailed(self, failure): + """ + Called if connecting failed. + + Usually this will be due to a DNS lookup failure. + """ + + + +@implementer(interfaces.ITransport) +@_oldStyle +class FileWrapper: + """ + A wrapper around a file-like object to make it behave as a Transport. + + This doesn't actually stream the file to the attached protocol, + and is thus useful mainly as a utility for debugging protocols. + """ + + closed = 0 + disconnecting = 0 + producer = None + streamingProducer = 0 + + def __init__(self, file): + self.file = file + + + def write(self, data): + try: + self.file.write(data) + except: + self.handleException() + + + def _checkProducer(self): + # Cheating; this is called at "idle" times to allow producers to be + # found and dealt with + if self.producer: + self.producer.resumeProducing() + + + def registerProducer(self, producer, streaming): + """ + From abstract.FileDescriptor + """ + self.producer = producer + self.streamingProducer = streaming + if not streaming: + producer.resumeProducing() + + + def unregisterProducer(self): + self.producer = None + + + def stopConsuming(self): + self.unregisterProducer() + self.loseConnection() + + + def writeSequence(self, iovec): + self.write(b"".join(iovec)) + + + def loseConnection(self): + self.closed = 1 + try: + self.file.close() + except (IOError, OSError): + self.handleException() + + + def getPeer(self): + # FIXME: https://twistedmatrix.com/trac/ticket/7820 + # According to ITransport, this should return an IAddress! + return 'file', 'file' + + + def getHost(self): + # FIXME: https://twistedmatrix.com/trac/ticket/7820 + # According to ITransport, this should return an IAddress! + return 'file' + + + def handleException(self): + pass + + + def resumeProducing(self): + # Never sends data anyways + pass + + + def pauseProducing(self): + # Never sends data anyways + pass + + + def stopProducing(self): + self.loseConnection() + + +__all__ = ["Factory", "ClientFactory", "ReconnectingClientFactory", "connectionDone", + "Protocol", "ProcessProtocol", "FileWrapper", "ServerFactory", + "AbstractDatagramProtocol", "DatagramProtocol", "ConnectedDatagramProtocol", + "ClientCreator"] diff --git a/contrib/python/Twisted/py2/twisted/internet/pyuisupport.py b/contrib/python/Twisted/py2/twisted/internet/pyuisupport.py new file mode 100644 index 00000000000..1e7def59118 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/pyuisupport.py @@ -0,0 +1,37 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +This module integrates PyUI with twisted.internet's mainloop. + +Maintainer: Jp Calderone + +See doc/examples/pyuidemo.py for example usage. +""" + +# System imports +import pyui + +def _guiUpdate(reactor, delay): + pyui.draw() + if pyui.update() == 0: + pyui.quit() + reactor.stop() + else: + reactor.callLater(delay, _guiUpdate, reactor, delay) + + +def install(ms=10, reactor=None, args=(), kw={}): + """ + Schedule PyUI's display to be updated approximately every C{ms} + milliseconds, and initialize PyUI with the specified arguments. + """ + d = pyui.init(*args, **kw) + + if reactor is None: + from twisted.internet import reactor + _guiUpdate(reactor, ms / 1000.0) + return d + +__all__ = ["install"] diff --git a/contrib/python/Twisted/py2/twisted/internet/reactor.py b/contrib/python/Twisted/py2/twisted/internet/reactor.py new file mode 100644 index 00000000000..6dd72af0c81 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/reactor.py @@ -0,0 +1,39 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +The reactor is the Twisted event loop within Twisted, the loop which drives +applications using Twisted. The reactor provides APIs for networking, +threading, dispatching events, and more. + +The default reactor depends on the platform and will be installed if this +module is imported without another reactor being explicitly installed +beforehand. Regardless of which reactor is installed, importing this module is +the correct way to get a reference to it. + +New application code should prefer to pass and accept the reactor as a +parameter where it is needed, rather than relying on being able to import this +module to get a reference. This simplifies unit testing and may make it easier +to one day support multiple reactors (as a performance enhancement), though +this is not currently possible. + +@see: L{IReactorCore} +@see: L{IReactorTime} +@see: L{IReactorProcess} +@see: L{IReactorTCP} +@see: L{IReactorSSL} +@see: L{IReactorUDP} +@see: L{IReactorMulticast} +@see: L{IReactorUNIX} +@see: L{IReactorUNIXDatagram} +@see: L{IReactorFDSet} +@see: L{IReactorThreads} +@see: L{IReactorPluggableResolver} +""" + +from __future__ import division, absolute_import + +import sys +del sys.modules['twisted.internet.reactor'] +from twisted.internet import default +default.install() diff --git a/contrib/python/Twisted/py2/twisted/internet/selectreactor.py b/contrib/python/Twisted/py2/twisted/internet/selectreactor.py new file mode 100644 index 00000000000..90eba54cef7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/selectreactor.py @@ -0,0 +1,200 @@ +# -*- test-case-name: twisted.test.test_internet -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Select reactor +""" + +from __future__ import division, absolute_import + +from time import sleep +import sys, select, socket +from errno import EINTR, EBADF + +from zope.interface import implementer + +from twisted.internet.interfaces import IReactorFDSet +from twisted.internet import posixbase +from twisted.python import log +from twisted.python.runtime import platformType + + +def win32select(r, w, e, timeout=None): + """Win32 select wrapper.""" + if not (r or w): + # windows select() exits immediately when no sockets + if timeout is None: + timeout = 0.01 + else: + timeout = min(timeout, 0.001) + sleep(timeout) + return [], [], [] + # windows doesn't process 'signals' inside select(), so we set a max + # time or ctrl-c will never be recognized + if timeout is None or timeout > 0.5: + timeout = 0.5 + r, w, e = select.select(r, w, w, timeout) + return r, w + e, [] + +if platformType == "win32": + _select = win32select +else: + _select = select.select + + +try: + from twisted.internet.win32eventreactor import _ThreadedWin32EventsMixin +except ImportError: + _extraBase = object +else: + _extraBase = _ThreadedWin32EventsMixin + + +@implementer(IReactorFDSet) +class SelectReactor(posixbase.PosixReactorBase, _extraBase): + """ + A select() based reactor - runs on all POSIX platforms and on Win32. + + @ivar _reads: A set containing L{FileDescriptor} instances which will be + checked for read events. + + @ivar _writes: A set containing L{FileDescriptor} instances which will be + checked for writability. + """ + + def __init__(self): + """ + Initialize file descriptor tracking dictionaries and the base class. + """ + self._reads = set() + self._writes = set() + posixbase.PosixReactorBase.__init__(self) + + + def _preenDescriptors(self): + log.msg("Malformed file descriptor found. Preening lists.") + readers = list(self._reads) + writers = list(self._writes) + self._reads.clear() + self._writes.clear() + for selSet, selList in ((self._reads, readers), + (self._writes, writers)): + for selectable in selList: + try: + select.select([selectable], [selectable], [selectable], 0) + except Exception as e: + log.msg("bad descriptor %s" % selectable) + self._disconnectSelectable(selectable, e, False) + else: + selSet.add(selectable) + + + def doSelect(self, timeout): + """ + Run one iteration of the I/O monitor loop. + + This will run all selectables who had input or output readiness + waiting for them. + """ + try: + r, w, ignored = _select(self._reads, + self._writes, + [], timeout) + except ValueError: + # Possibly a file descriptor has gone negative? + self._preenDescriptors() + return + except TypeError: + # Something *totally* invalid (object w/o fileno, non-integral + # result) was passed + log.err() + self._preenDescriptors() + return + except (select.error, socket.error, IOError) as se: + # select(2) encountered an error, perhaps while calling the fileno() + # method of a socket. (Python 2.6 socket.error is an IOError + # subclass, but on Python 2.5 and earlier it is not.) + if se.args[0] in (0, 2): + # windows does this if it got an empty list + if (not self._reads) and (not self._writes): + return + else: + raise + elif se.args[0] == EINTR: + return + elif se.args[0] == EBADF: + self._preenDescriptors() + return + else: + # OK, I really don't know what's going on. Blow up. + raise + + _drdw = self._doReadOrWrite + _logrun = log.callWithLogger + for selectables, method, fdset in ((r, "doRead", self._reads), + (w,"doWrite", self._writes)): + for selectable in selectables: + # if this was disconnected in another thread, kill it. + # ^^^^ --- what the !@#*? serious! -exarkun + if selectable not in fdset: + continue + # This for pausing input when we're not ready for more. + _logrun(selectable, _drdw, selectable, method) + + doIteration = doSelect + + def _doReadOrWrite(self, selectable, method): + try: + why = getattr(selectable, method)() + except: + why = sys.exc_info()[1] + log.err() + if why: + self._disconnectSelectable(selectable, why, method=="doRead") + + def addReader(self, reader): + """ + Add a FileDescriptor for notification of data available to read. + """ + self._reads.add(reader) + + def addWriter(self, writer): + """ + Add a FileDescriptor for notification of data available to write. + """ + self._writes.add(writer) + + def removeReader(self, reader): + """ + Remove a Selectable for notification of data available to read. + """ + self._reads.discard(reader) + + def removeWriter(self, writer): + """ + Remove a Selectable for notification of data available to write. + """ + self._writes.discard(writer) + + def removeAll(self): + return self._removeAll(self._reads, self._writes) + + + def getReaders(self): + return list(self._reads) + + + def getWriters(self): + return list(self._writes) + + + +def install(): + """Configure the twisted mainloop to be run using the select() reactor. + """ + reactor = SelectReactor() + from twisted.internet.main import installReactor + installReactor(reactor) + +__all__ = ['install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/serialport.py b/contrib/python/Twisted/py2/twisted/internet/serialport.py new file mode 100644 index 00000000000..d0cdd7f76a2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/serialport.py @@ -0,0 +1,89 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Serial Port Protocol +""" + +from __future__ import division, absolute_import + +# http://twistedmatrix.com/trac/ticket/3725#comment:24 +# Apparently applications use these names even though they should +# be imported from pyserial +__all__ = ["serial", "PARITY_ODD", "PARITY_EVEN", "PARITY_NONE", + "STOPBITS_TWO", "STOPBITS_ONE", "FIVEBITS", + "EIGHTBITS", "SEVENBITS", "SIXBITS", +# Name this module is actually trying to export + "SerialPort"] + +# all of them require pyserial at the moment, so check that first +import serial +from serial import PARITY_NONE, PARITY_EVEN, PARITY_ODD +from serial import STOPBITS_ONE, STOPBITS_TWO +from serial import FIVEBITS, SIXBITS, SEVENBITS, EIGHTBITS + +from twisted.python._oldstyle import _oldStyle +from twisted.python.runtime import platform + + + +@_oldStyle +class BaseSerialPort: + """ + Base class for Windows and POSIX serial ports. + + @ivar _serialFactory: a pyserial C{serial.Serial} factory, used to create + the instance stored in C{self._serial}. Overrideable to enable easier + testing. + + @ivar _serial: a pyserial C{serial.Serial} instance used to manage the + options on the serial port. + """ + + _serialFactory = serial.Serial + + + def setBaudRate(self, baudrate): + if hasattr(self._serial, "setBaudrate"): + self._serial.setBaudrate(baudrate) + else: + self._serial.setBaudRate(baudrate) + + def inWaiting(self): + return self._serial.inWaiting() + + def flushInput(self): + self._serial.flushInput() + + def flushOutput(self): + self._serial.flushOutput() + + def sendBreak(self): + self._serial.sendBreak() + + def getDSR(self): + return self._serial.getDSR() + + def getCD(self): + return self._serial.getCD() + + def getRI(self): + return self._serial.getRI() + + def getCTS(self): + return self._serial.getCTS() + + def setDTR(self, on = 1): + self._serial.setDTR(on) + + def setRTS(self, on = 1): + self._serial.setRTS(on) + + + +# Expert appropriate implementation of SerialPort. +if platform.isWindows(): + from twisted.internet._win32serialport import SerialPort +else: + from twisted.internet._posixserialport import SerialPort diff --git a/contrib/python/Twisted/py2/twisted/internet/ssl.py b/contrib/python/Twisted/py2/twisted/internet/ssl.py new file mode 100644 index 00000000000..591df8cb82d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/ssl.py @@ -0,0 +1,255 @@ +# -*- test-case-name: twisted.test.test_ssl -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module implements Transport Layer Security (TLS) support for Twisted. It +requires U{PyOpenSSL }. + +If you wish to establish a TLS connection, please use one of the following +APIs: + + - SSL endpoints for L{servers + } and L{clients + } + + - L{startTLS } + + - L{connectSSL } + + - L{listenSSL } + +These APIs all require a C{contextFactory} argument that specifies their +security properties, such as certificate, private key, certificate authorities +to verify the peer, allowed TLS protocol versions, cipher suites, and so on. +The recommended value for this argument is a L{CertificateOptions} instance; +see its documentation for an explanation of the available options. + +The C{contextFactory} name is a bit of an anachronism now, as context factories +have been replaced with "connection creators", but these objects serve the same +role. + +Be warned that implementing your own connection creator (i.e.: value for the +C{contextFactory}) is both difficult and dangerous; the Twisted team has worked +hard to make L{CertificateOptions}' API comprehensible and unsurprising, and +the Twisted team is actively maintaining it to ensure that it becomes more +secure over time. + +If you are really absolutely sure that you want to take on the risk of +implementing your own connection creator based on the pyOpenSSL API, see the +L{server connection creator +} and L{client +connection creator +} interfaces. + +Developers using Twisted, please ignore the L{Port}, L{Connector}, and +L{Client} classes defined here, as these are details of certain reactors' TLS +implementations, exposed by accident (and remaining here only for compatibility +reasons). If you wish to establish a TLS connection, please use one of the +APIs listed above. + +@note: "SSL" (Secure Sockets Layer) is an antiquated synonym for "TLS" + (Transport Layer Security). You may see these terms used interchangeably + throughout the documentation. +""" + +from __future__ import division, absolute_import + +# System imports +from OpenSSL import SSL +supported = True + +from zope.interface import implementer, implementer_only, implementedBy + +# Twisted imports +from twisted.internet import tcp, interfaces +from twisted.python._oldstyle import _oldStyle + + +@implementer(interfaces.IOpenSSLContextFactory) +@_oldStyle +class ContextFactory: + """A factory for SSL context objects, for server SSL connections.""" + + isClient = 0 + + def getContext(self): + """Return a SSL.Context object. override in subclasses.""" + raise NotImplementedError + + + +class DefaultOpenSSLContextFactory(ContextFactory): + """ + L{DefaultOpenSSLContextFactory} is a factory for server-side SSL context + objects. These objects define certain parameters related to SSL + handshakes and the subsequent connection. + + @ivar _contextFactory: A callable which will be used to create new + context objects. This is typically L{OpenSSL.SSL.Context}. + """ + _context = None + + def __init__(self, privateKeyFileName, certificateFileName, + sslmethod=SSL.SSLv23_METHOD, _contextFactory=SSL.Context): + """ + @param privateKeyFileName: Name of a file containing a private key + @param certificateFileName: Name of a file containing a certificate + @param sslmethod: The SSL method to use + """ + self.privateKeyFileName = privateKeyFileName + self.certificateFileName = certificateFileName + self.sslmethod = sslmethod + self._contextFactory = _contextFactory + + # Create a context object right now. This is to force validation of + # the given parameters so that errors are detected earlier rather + # than later. + self.cacheContext() + + + def cacheContext(self): + if self._context is None: + ctx = self._contextFactory(self.sslmethod) + # Disallow SSLv2! It's insecure! SSLv3 has been around since + # 1996. It's time to move on. + ctx.set_options(SSL.OP_NO_SSLv2) + ctx.use_certificate_file(self.certificateFileName) + ctx.use_privatekey_file(self.privateKeyFileName) + self._context = ctx + + + def __getstate__(self): + d = self.__dict__.copy() + del d['_context'] + return d + + + def __setstate__(self, state): + self.__dict__ = state + + + def getContext(self): + """ + Return an SSL context. + """ + return self._context + + + +@implementer(interfaces.IOpenSSLContextFactory) +@_oldStyle +class ClientContextFactory: + """A context factory for SSL clients.""" + + isClient = 1 + + # SSLv23_METHOD allows SSLv2, SSLv3, and TLSv1. We disable SSLv2 below, + # though. + method = SSL.SSLv23_METHOD + + _contextFactory = SSL.Context + + def getContext(self): + ctx = self._contextFactory(self.method) + # See comment in DefaultOpenSSLContextFactory about SSLv2. + ctx.set_options(SSL.OP_NO_SSLv2) + return ctx + + + +@implementer_only(interfaces.ISSLTransport, + *[i for i in implementedBy(tcp.Client) + if i != interfaces.ITLSTransport]) +class Client(tcp.Client): + """ + I am an SSL client. + """ + + def __init__(self, host, port, bindAddress, ctxFactory, connector, reactor=None): + # tcp.Client.__init__ depends on self.ctxFactory being set + self.ctxFactory = ctxFactory + tcp.Client.__init__(self, host, port, bindAddress, connector, reactor) + + def _connectDone(self): + self.startTLS(self.ctxFactory) + self.startWriting() + tcp.Client._connectDone(self) + + + +@implementer(interfaces.ISSLTransport) +class Server(tcp.Server): + """ + I am an SSL server. + """ + + def __init__(self, *args, **kwargs): + tcp.Server.__init__(self, *args, **kwargs) + self.startTLS(self.server.ctxFactory) + + + +class Port(tcp.Port): + """ + I am an SSL port. + """ + transport = Server + + _type = 'TLS' + + def __init__(self, port, factory, ctxFactory, backlog=50, interface='', reactor=None): + tcp.Port.__init__(self, port, factory, backlog, interface, reactor) + self.ctxFactory = ctxFactory + + + def _getLogPrefix(self, factory): + """ + Override the normal prefix to include an annotation indicating this is a + port for TLS connections. + """ + return tcp.Port._getLogPrefix(self, factory) + ' (TLS)' + + + +class Connector(tcp.Connector): + def __init__(self, host, port, factory, contextFactory, timeout, bindAddress, reactor=None): + self.contextFactory = contextFactory + tcp.Connector.__init__(self, host, port, factory, timeout, bindAddress, reactor) + + # Force some parameter checking in pyOpenSSL. It's better to fail now + # than after we've set up the transport. + contextFactory.getContext() + + + def _makeTransport(self): + return Client(self.host, self.port, self.bindAddress, self.contextFactory, self, self.reactor) + + + +from twisted.internet._sslverify import ( + KeyPair, DistinguishedName, DN, Certificate, + CertificateRequest, PrivateCertificate, + OpenSSLAcceptableCiphers as AcceptableCiphers, + OpenSSLCertificateOptions as CertificateOptions, + OpenSSLDiffieHellmanParameters as DiffieHellmanParameters, + platformTrust, OpenSSLDefaultPaths, VerificationError, + optionsForClientTLS, ProtocolNegotiationSupport, + protocolNegotiationMechanisms, + trustRootFromCertificates, + TLSVersion, +) + +__all__ = [ + "ContextFactory", "DefaultOpenSSLContextFactory", "ClientContextFactory", + + 'DistinguishedName', 'DN', + 'Certificate', 'CertificateRequest', 'PrivateCertificate', + 'KeyPair', + 'AcceptableCiphers', 'CertificateOptions', 'DiffieHellmanParameters', + 'platformTrust', 'OpenSSLDefaultPaths', 'TLSVersion', + + 'VerificationError', 'optionsForClientTLS', + 'ProtocolNegotiationSupport', 'protocolNegotiationMechanisms', + 'trustRootFromCertificates', +] diff --git a/contrib/python/Twisted/py2/twisted/internet/stdio.py b/contrib/python/Twisted/py2/twisted/internet/stdio.py new file mode 100644 index 00000000000..843a47277e5 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/stdio.py @@ -0,0 +1,37 @@ +# -*- test-case-name: twisted.test.test_stdio -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Standard input/out/err support. + +This module exposes one name, StandardIO, which is a factory that takes an +IProtocol provider as an argument. It connects that protocol to standard input +and output on the current process. + +It should work on any UNIX and also on Win32 (with some caveats: due to +platform limitations, it will perform very poorly on Win32). + +Future Plans:: + + support for stderr, perhaps + Rewrite to use the reactor instead of an ad-hoc mechanism for connecting + protocols to transport. + + +Maintainer: James Y Knight +""" + +from __future__ import absolute_import, division + +from twisted.python.runtime import platform + +if platform.isWindows(): + from twisted.internet import _win32stdio + StandardIO = _win32stdio.StandardIO + PipeAddress = _win32stdio.Win32PipeAddress + +else: + from twisted.internet._posixstdio import StandardIO, PipeAddress + +__all__ = ['StandardIO', 'PipeAddress'] diff --git a/contrib/python/Twisted/py2/twisted/internet/task.py b/contrib/python/Twisted/py2/twisted/internet/task.py new file mode 100644 index 00000000000..25042c949fa --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/task.py @@ -0,0 +1,948 @@ +# -*- test-case-name: twisted.test.test_task,twisted.test.test_cooperator -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Scheduling utility methods and classes. +""" + +from __future__ import division, absolute_import + +__metaclass__ = type + +import sys +import time +import warnings + +from zope.interface import implementer + +from twisted.python import log +from twisted.python import reflect +from twisted.python.deprecate import _getDeprecationWarningString +from twisted.python.failure import Failure +from incremental import Version + +from twisted.internet import base, defer +from twisted.internet.interfaces import IReactorTime +from twisted.internet.error import ReactorNotRunning + + +class LoopingCall: + """Call a function repeatedly. + + If C{f} returns a deferred, rescheduling will not take place until the + deferred has fired. The result value is ignored. + + @ivar f: The function to call. + @ivar a: A tuple of arguments to pass the function. + @ivar kw: A dictionary of keyword arguments to pass to the function. + @ivar clock: A provider of + L{twisted.internet.interfaces.IReactorTime}. The default is + L{twisted.internet.reactor}. Feel free to set this to + something else, but it probably ought to be set *before* + calling L{start}. + + @type running: C{bool} + @ivar running: A flag which is C{True} while C{f} is scheduled to be called + (or is currently being called). It is set to C{True} when L{start} is + called and set to C{False} when L{stop} is called or if C{f} raises an + exception. In either case, it will be C{False} by the time the + C{Deferred} returned by L{start} fires its callback or errback. + + @type _realLastTime: C{float} + @ivar _realLastTime: When counting skips, the time at which the skip + counter was last invoked. + + @type _runAtStart: C{bool} + @ivar _runAtStart: A flag indicating whether the 'now' argument was passed + to L{LoopingCall.start}. + """ + + call = None + running = False + _deferred = None + interval = None + _runAtStart = False + starttime = None + + def __init__(self, f, *a, **kw): + self.f = f + self.a = a + self.kw = kw + from twisted.internet import reactor + self.clock = reactor + + @property + def deferred(self): + """ + DEPRECATED. L{Deferred} fired when loop stops or fails. + + Use the L{Deferred} returned by L{LoopingCall.start}. + """ + warningString = _getDeprecationWarningString( + "twisted.internet.task.LoopingCall.deferred", + Version("Twisted", 16, 0, 0), + replacement='the deferred returned by start()') + warnings.warn(warningString, DeprecationWarning, stacklevel=2) + + return self._deferred + + def withCount(cls, countCallable): + """ + An alternate constructor for L{LoopingCall} that makes available the + number of calls which should have occurred since it was last invoked. + + Note that this number is an C{int} value; It represents the discrete + number of calls that should have been made. For example, if you are + using a looping call to display an animation with discrete frames, this + number would be the number of frames to advance. + + The count is normally 1, but can be higher. For example, if the reactor + is blocked and takes too long to invoke the L{LoopingCall}, a Deferred + returned from a previous call is not fired before an interval has + elapsed, or if the callable itself blocks for longer than an interval, + preventing I{itself} from being called. + + When running with an interval if 0, count will be always 1. + + @param countCallable: A callable that will be invoked each time the + resulting LoopingCall is run, with an integer specifying the number + of calls that should have been invoked. + + @type countCallable: 1-argument callable which takes an C{int} + + @return: An instance of L{LoopingCall} with call counting enabled, + which provides the count as the first positional argument. + + @rtype: L{LoopingCall} + + @since: 9.0 + """ + + def counter(): + now = self.clock.seconds() + + if self.interval == 0: + self._realLastTime = now + return countCallable(1) + + lastTime = self._realLastTime + if lastTime is None: + lastTime = self.starttime + if self._runAtStart: + lastTime -= self.interval + lastInterval = self._intervalOf(lastTime) + thisInterval = self._intervalOf(now) + count = thisInterval - lastInterval + if count > 0: + self._realLastTime = now + return countCallable(count) + + self = cls(counter) + + self._realLastTime = None + + return self + + withCount = classmethod(withCount) + + + def _intervalOf(self, t): + """ + Determine the number of intervals passed as of the given point in + time. + + @param t: The specified time (from the start of the L{LoopingCall}) to + be measured in intervals + + @return: The C{int} number of intervals which have passed as of the + given point in time. + """ + elapsedTime = t - self.starttime + intervalNum = int(elapsedTime / self.interval) + return intervalNum + + + def start(self, interval, now=True): + """ + Start running function every interval seconds. + + @param interval: The number of seconds between calls. May be + less than one. Precision will depend on the underlying + platform, the available hardware, and the load on the system. + + @param now: If True, run this call right now. Otherwise, wait + until the interval has elapsed before beginning. + + @return: A Deferred whose callback will be invoked with + C{self} when C{self.stop} is called, or whose errback will be + invoked when the function raises an exception or returned a + deferred that has its errback invoked. + """ + assert not self.running, ("Tried to start an already running " + "LoopingCall.") + if interval < 0: + raise ValueError("interval must be >= 0") + self.running = True + # Loop might fail to start and then self._deferred will be cleared. + # This why the local C{deferred} variable is used. + deferred = self._deferred = defer.Deferred() + self.starttime = self.clock.seconds() + self.interval = interval + self._runAtStart = now + if now: + self() + else: + self._scheduleFrom(self.starttime) + return deferred + + def stop(self): + """Stop running function. + """ + assert self.running, ("Tried to stop a LoopingCall that was " + "not running.") + self.running = False + if self.call is not None: + self.call.cancel() + self.call = None + d, self._deferred = self._deferred, None + d.callback(self) + + def reset(self): + """ + Skip the next iteration and reset the timer. + + @since: 11.1 + """ + assert self.running, ("Tried to reset a LoopingCall that was " + "not running.") + if self.call is not None: + self.call.cancel() + self.call = None + self.starttime = self.clock.seconds() + self._scheduleFrom(self.starttime) + + def __call__(self): + def cb(result): + if self.running: + self._scheduleFrom(self.clock.seconds()) + else: + d, self._deferred = self._deferred, None + d.callback(self) + + def eb(failure): + self.running = False + d, self._deferred = self._deferred, None + d.errback(failure) + + self.call = None + d = defer.maybeDeferred(self.f, *self.a, **self.kw) + d.addCallback(cb) + d.addErrback(eb) + + + def _scheduleFrom(self, when): + """ + Schedule the next iteration of this looping call. + + @param when: The present time from whence the call is scheduled. + """ + def howLong(): + # How long should it take until the next invocation of our + # callable? Split out into a function because there are multiple + # places we want to 'return' out of this. + if self.interval == 0: + # If the interval is 0, just go as fast as possible, always + # return zero, call ourselves ASAP. + return 0 + # Compute the time until the next interval; how long has this call + # been running for? + runningFor = when - self.starttime + # And based on that start time, when does the current interval end? + untilNextInterval = self.interval - (runningFor % self.interval) + # Now that we know how long it would be, we have to tell if the + # number is effectively zero. However, we can't just test against + # zero. If a number with a small exponent is added to a number + # with a large exponent, it may be so small that the digits just + # fall off the end, which means that adding the increment makes no + # difference; it's time to tick over into the next interval. + if when == when + untilNextInterval: + # If it's effectively zero, then we need to add another + # interval. + return self.interval + # Finally, if everything else is normal, we just return the + # computed delay. + return untilNextInterval + self.call = self.clock.callLater(howLong(), self) + + + def __repr__(self): + if hasattr(self.f, '__qualname__'): + func = self.f.__qualname__ + elif hasattr(self.f, '__name__'): + func = self.f.__name__ + if hasattr(self.f, 'im_class'): + func = self.f.im_class.__name__ + '.' + func + else: + func = reflect.safe_repr(self.f) + + return 'LoopingCall<%r>(%s, *%s, **%s)' % ( + self.interval, func, reflect.safe_repr(self.a), + reflect.safe_repr(self.kw)) + + + +class SchedulerError(Exception): + """ + The operation could not be completed because the scheduler or one of its + tasks was in an invalid state. This exception should not be raised + directly, but is a superclass of various scheduler-state-related + exceptions. + """ + + + +class SchedulerStopped(SchedulerError): + """ + The operation could not complete because the scheduler was stopped in + progress or was already stopped. + """ + + + +class TaskFinished(SchedulerError): + """ + The operation could not complete because the task was already completed, + stopped, encountered an error or otherwise permanently stopped running. + """ + + + +class TaskDone(TaskFinished): + """ + The operation could not complete because the task was already completed. + """ + + + +class TaskStopped(TaskFinished): + """ + The operation could not complete because the task was stopped. + """ + + + +class TaskFailed(TaskFinished): + """ + The operation could not complete because the task died with an unhandled + error. + """ + + + +class NotPaused(SchedulerError): + """ + This exception is raised when a task is resumed which was not previously + paused. + """ + + + +class _Timer(object): + MAX_SLICE = 0.01 + def __init__(self): + self.end = time.time() + self.MAX_SLICE + + + def __call__(self): + return time.time() >= self.end + + + +_EPSILON = 0.00000001 +def _defaultScheduler(x): + from twisted.internet import reactor + return reactor.callLater(_EPSILON, x) + + +class CooperativeTask(object): + """ + A L{CooperativeTask} is a task object inside a L{Cooperator}, which can be + paused, resumed, and stopped. It can also have its completion (or + termination) monitored. + + @see: L{Cooperator.cooperate} + + @ivar _iterator: the iterator to iterate when this L{CooperativeTask} is + asked to do work. + + @ivar _cooperator: the L{Cooperator} that this L{CooperativeTask} + participates in, which is used to re-insert it upon resume. + + @ivar _deferreds: the list of L{defer.Deferred}s to fire when this task + completes, fails, or finishes. + + @type _deferreds: C{list} + + @type _cooperator: L{Cooperator} + + @ivar _pauseCount: the number of times that this L{CooperativeTask} has + been paused; if 0, it is running. + + @type _pauseCount: C{int} + + @ivar _completionState: The completion-state of this L{CooperativeTask}. + L{None} if the task is not yet completed, an instance of L{TaskStopped} + if C{stop} was called to stop this task early, of L{TaskFailed} if the + application code in the iterator raised an exception which caused it to + terminate, and of L{TaskDone} if it terminated normally via raising + C{StopIteration}. + + @type _completionState: L{TaskFinished} + """ + + def __init__(self, iterator, cooperator): + """ + A private constructor: to create a new L{CooperativeTask}, see + L{Cooperator.cooperate}. + """ + self._iterator = iterator + self._cooperator = cooperator + self._deferreds = [] + self._pauseCount = 0 + self._completionState = None + self._completionResult = None + cooperator._addTask(self) + + + def whenDone(self): + """ + Get a L{defer.Deferred} notification of when this task is complete. + + @return: a L{defer.Deferred} that fires with the C{iterator} that this + L{CooperativeTask} was created with when the iterator has been + exhausted (i.e. its C{next} method has raised C{StopIteration}), or + fails with the exception raised by C{next} if it raises some other + exception. + + @rtype: L{defer.Deferred} + """ + d = defer.Deferred() + if self._completionState is None: + self._deferreds.append(d) + else: + d.callback(self._completionResult) + return d + + + def pause(self): + """ + Pause this L{CooperativeTask}. Stop doing work until + L{CooperativeTask.resume} is called. If C{pause} is called more than + once, C{resume} must be called an equal number of times to resume this + task. + + @raise TaskFinished: if this task has already finished or completed. + """ + self._checkFinish() + self._pauseCount += 1 + if self._pauseCount == 1: + self._cooperator._removeTask(self) + + + def resume(self): + """ + Resume processing of a paused L{CooperativeTask}. + + @raise NotPaused: if this L{CooperativeTask} is not paused. + """ + if self._pauseCount == 0: + raise NotPaused() + self._pauseCount -= 1 + if self._pauseCount == 0 and self._completionState is None: + self._cooperator._addTask(self) + + + def _completeWith(self, completionState, deferredResult): + """ + @param completionState: a L{TaskFinished} exception or a subclass + thereof, indicating what exception should be raised when subsequent + operations are performed. + + @param deferredResult: the result to fire all the deferreds with. + """ + self._completionState = completionState + self._completionResult = deferredResult + if not self._pauseCount: + self._cooperator._removeTask(self) + + # The Deferreds need to be invoked after all this is completed, because + # a Deferred may want to manipulate other tasks in a Cooperator. For + # example, if you call "stop()" on a cooperator in a callback on a + # Deferred returned from whenDone(), this CooperativeTask must be gone + # from the Cooperator by that point so that _completeWith is not + # invoked reentrantly; that would cause these Deferreds to blow up with + # an AlreadyCalledError, or the _removeTask to fail with a ValueError. + for d in self._deferreds: + d.callback(deferredResult) + + + def stop(self): + """ + Stop further processing of this task. + + @raise TaskFinished: if this L{CooperativeTask} has previously + completed, via C{stop}, completion, or failure. + """ + self._checkFinish() + self._completeWith(TaskStopped(), Failure(TaskStopped())) + + + def _checkFinish(self): + """ + If this task has been stopped, raise the appropriate subclass of + L{TaskFinished}. + """ + if self._completionState is not None: + raise self._completionState + + + def _oneWorkUnit(self): + """ + Perform one unit of work for this task, retrieving one item from its + iterator, stopping if there are no further items in the iterator, and + pausing if the result was a L{defer.Deferred}. + """ + try: + result = next(self._iterator) + except StopIteration: + self._completeWith(TaskDone(), self._iterator) + except: + self._completeWith(TaskFailed(), Failure()) + else: + if isinstance(result, defer.Deferred): + self.pause() + def failLater(f): + self._completeWith(TaskFailed(), f) + result.addCallbacks(lambda result: self.resume(), + failLater) + + + +class Cooperator(object): + """ + Cooperative task scheduler. + + A cooperative task is an iterator where each iteration represents an + atomic unit of work. When the iterator yields, it allows the + L{Cooperator} to decide which of its tasks to execute next. If the + iterator yields a L{defer.Deferred} then work will pause until the + L{defer.Deferred} fires and completes its callback chain. + + When a L{Cooperator} has more than one task, it distributes work between + all tasks. + + There are two ways to add tasks to a L{Cooperator}, L{cooperate} and + L{coiterate}. L{cooperate} is the more useful of the two, as it returns a + L{CooperativeTask}, which can be L{paused}, + L{resumed} and L{waited + on}. L{coiterate} has the same effect, but + returns only a L{defer.Deferred} that fires when the task is done. + + L{Cooperator} can be used for many things, including but not limited to: + + - running one or more computationally intensive tasks without blocking + - limiting parallelism by running a subset of the total tasks + simultaneously + - doing one thing, waiting for a L{Deferred} to fire, + doing the next thing, repeat (i.e. serializing a sequence of + asynchronous tasks) + + Multiple L{Cooperator}s do not cooperate with each other, so for most + cases you should use the L{global cooperator}. + """ + + def __init__(self, + terminationPredicateFactory=_Timer, + scheduler=_defaultScheduler, + started=True): + """ + Create a scheduler-like object to which iterators may be added. + + @param terminationPredicateFactory: A no-argument callable which will + be invoked at the beginning of each step and should return a + no-argument callable which will return True when the step should be + terminated. The default factory is time-based and allows iterators to + run for 1/100th of a second at a time. + + @param scheduler: A one-argument callable which takes a no-argument + callable and should invoke it at some future point. This will be used + to schedule each step of this Cooperator. + + @param started: A boolean which indicates whether iterators should be + stepped as soon as they are added, or if they will be queued up until + L{Cooperator.start} is called. + """ + self._tasks = [] + self._metarator = iter(()) + self._terminationPredicateFactory = terminationPredicateFactory + self._scheduler = scheduler + self._delayedCall = None + self._stopped = False + self._started = started + + + def coiterate(self, iterator, doneDeferred=None): + """ + Add an iterator to the list of iterators this L{Cooperator} is + currently running. + + Equivalent to L{cooperate}, but returns a L{defer.Deferred} that will + be fired when the task is done. + + @param doneDeferred: If specified, this will be the Deferred used as + the completion deferred. It is suggested that you use the default, + which creates a new Deferred for you. + + @return: a Deferred that will fire when the iterator finishes. + """ + if doneDeferred is None: + doneDeferred = defer.Deferred() + CooperativeTask(iterator, self).whenDone().chainDeferred(doneDeferred) + return doneDeferred + + + def cooperate(self, iterator): + """ + Start running the given iterator as a long-running cooperative task, by + calling next() on it as a periodic timed event. + + @param iterator: the iterator to invoke. + + @return: a L{CooperativeTask} object representing this task. + """ + return CooperativeTask(iterator, self) + + + def _addTask(self, task): + """ + Add a L{CooperativeTask} object to this L{Cooperator}. + """ + if self._stopped: + self._tasks.append(task) # XXX silly, I know, but _completeWith + # does the inverse + task._completeWith(SchedulerStopped(), Failure(SchedulerStopped())) + else: + self._tasks.append(task) + self._reschedule() + + + def _removeTask(self, task): + """ + Remove a L{CooperativeTask} from this L{Cooperator}. + """ + self._tasks.remove(task) + # If no work left to do, cancel the delayed call: + if not self._tasks and self._delayedCall: + self._delayedCall.cancel() + self._delayedCall = None + + + def _tasksWhileNotStopped(self): + """ + Yield all L{CooperativeTask} objects in a loop as long as this + L{Cooperator}'s termination condition has not been met. + """ + terminator = self._terminationPredicateFactory() + while self._tasks: + for t in self._metarator: + yield t + if terminator(): + return + self._metarator = iter(self._tasks) + + + def _tick(self): + """ + Run one scheduler tick. + """ + self._delayedCall = None + for taskObj in self._tasksWhileNotStopped(): + taskObj._oneWorkUnit() + self._reschedule() + + + _mustScheduleOnStart = False + def _reschedule(self): + if not self._started: + self._mustScheduleOnStart = True + return + if self._delayedCall is None and self._tasks: + self._delayedCall = self._scheduler(self._tick) + + + def start(self): + """ + Begin scheduling steps. + """ + self._stopped = False + self._started = True + if self._mustScheduleOnStart: + del self._mustScheduleOnStart + self._reschedule() + + + def stop(self): + """ + Stop scheduling steps. Errback the completion Deferreds of all + iterators which have been added and forget about them. + """ + self._stopped = True + for taskObj in self._tasks: + taskObj._completeWith(SchedulerStopped(), + Failure(SchedulerStopped())) + self._tasks = [] + if self._delayedCall is not None: + self._delayedCall.cancel() + self._delayedCall = None + + + @property + def running(self): + """ + Is this L{Cooperator} is currently running? + + @return: C{True} if the L{Cooperator} is running, C{False} otherwise. + @rtype: C{bool} + """ + return (self._started and not self._stopped) + + + +_theCooperator = Cooperator() + +def coiterate(iterator): + """ + Cooperatively iterate over the given iterator, dividing runtime between it + and all other iterators which have been passed to this function and not yet + exhausted. + + @param iterator: the iterator to invoke. + + @return: a Deferred that will fire when the iterator finishes. + """ + return _theCooperator.coiterate(iterator) + + + +def cooperate(iterator): + """ + Start running the given iterator as a long-running cooperative task, by + calling next() on it as a periodic timed event. + + This is very useful if you have computationally expensive tasks that you + want to run without blocking the reactor. Just break each task up so that + it yields frequently, pass it in here and the global L{Cooperator} will + make sure work is distributed between them without blocking longer than a + single iteration of a single task. + + @param iterator: the iterator to invoke. + + @return: a L{CooperativeTask} object representing this task. + """ + return _theCooperator.cooperate(iterator) + + + +@implementer(IReactorTime) +class Clock: + """ + Provide a deterministic, easily-controlled implementation of + L{IReactorTime.callLater}. This is commonly useful for writing + deterministic unit tests for code which schedules events using this API. + """ + + rightNow = 0.0 + + def __init__(self): + self.calls = [] + + + def seconds(self): + """ + Pretend to be time.time(). This is used internally when an operation + such as L{IDelayedCall.reset} needs to determine a time value + relative to the current time. + + @rtype: C{float} + @return: The time which should be considered the current time. + """ + return self.rightNow + + + def _sortCalls(self): + """ + Sort the pending calls according to the time they are scheduled. + """ + self.calls.sort(key=lambda a: a.getTime()) + + + def callLater(self, when, what, *a, **kw): + """ + See L{twisted.internet.interfaces.IReactorTime.callLater}. + """ + dc = base.DelayedCall(self.seconds() + when, + what, a, kw, + self.calls.remove, + lambda c: None, + self.seconds) + self.calls.append(dc) + self._sortCalls() + return dc + + + def getDelayedCalls(self): + """ + See L{twisted.internet.interfaces.IReactorTime.getDelayedCalls} + """ + return self.calls + + + def advance(self, amount): + """ + Move time on this clock forward by the given amount and run whatever + pending calls should be run. + + @type amount: C{float} + @param amount: The number of seconds which to advance this clock's + time. + """ + self.rightNow += amount + self._sortCalls() + while self.calls and self.calls[0].getTime() <= self.seconds(): + call = self.calls.pop(0) + call.called = 1 + call.func(*call.args, **call.kw) + self._sortCalls() + + + def pump(self, timings): + """ + Advance incrementally by the given set of times. + + @type timings: iterable of C{float} + """ + for amount in timings: + self.advance(amount) + + + +def deferLater(clock, delay, callable=None, *args, **kw): + """ + Call the given function after a certain period of time has passed. + + @type clock: L{IReactorTime} provider + @param clock: The object which will be used to schedule the delayed + call. + + @type delay: C{float} or C{int} + @param delay: The number of seconds to wait before calling the function. + + @param callable: The object to call after the delay. + + @param *args: The positional arguments to pass to C{callable}. + + @param **kw: The keyword arguments to pass to C{callable}. + + @rtype: L{defer.Deferred} + + @return: A deferred that fires with the result of the callable when the + specified time has elapsed. + """ + def deferLaterCancel(deferred): + delayedCall.cancel() + d = defer.Deferred(deferLaterCancel) + if callable is not None: + d.addCallback(lambda ignored: callable(*args, **kw)) + delayedCall = clock.callLater(delay, d.callback, None) + return d + + + +def react(main, argv=(), _reactor=None): + """ + Call C{main} and run the reactor until the L{Deferred} it returns fires. + + This is intended as the way to start up an application with a well-defined + completion condition. Use it to write clients or one-off asynchronous + operations. Prefer this to calling C{reactor.run} directly, as this + function will also: + + - Take care to call C{reactor.stop} once and only once, and at the right + time. + - Log any failures from the C{Deferred} returned by C{main}. + - Exit the application when done, with exit code 0 in case of success and + 1 in case of failure. If C{main} fails with a C{SystemExit} error, the + code returned is used. + + The following demonstrates the signature of a C{main} function which can be + used with L{react}:: + def main(reactor, username, password): + return defer.succeed('ok') + + task.react(main, ('alice', 'secret')) + + @param main: A callable which returns a L{Deferred}. It should + take the reactor as its first parameter, followed by the elements of + C{argv}. + + @param argv: A list of arguments to pass to C{main}. If omitted the + callable will be invoked with no additional arguments. + + @param _reactor: An implementation detail to allow easier unit testing. Do + not supply this parameter. + + @since: 12.3 + """ + if _reactor is None: + from twisted.internet import reactor as _reactor + finished = main(_reactor, *argv) + codes = [0] + + stopping = [] + _reactor.addSystemEventTrigger('before', 'shutdown', stopping.append, True) + + def stop(result, stopReactor): + if stopReactor: + try: + _reactor.stop() + except ReactorNotRunning: + pass + + if isinstance(result, Failure): + if result.check(SystemExit) is not None: + code = result.value.code + else: + log.err(result, "main function encountered error") + code = 1 + codes[0] = code + + def cbFinish(result): + if stopping: + stop(result, False) + else: + _reactor.callWhenRunning(stop, result, True) + + finished.addBoth(cbFinish) + _reactor.run() + sys.exit(codes[0]) + + +__all__ = [ + 'LoopingCall', + + 'Clock', + + 'SchedulerStopped', 'Cooperator', 'coiterate', + + 'deferLater', 'react'] diff --git a/contrib/python/Twisted/py2/twisted/internet/tcp.py b/contrib/python/Twisted/py2/twisted/internet/tcp.py new file mode 100644 index 00000000000..7b00019268b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/tcp.py @@ -0,0 +1,1555 @@ +# -*- test-case-name: twisted.test.test_tcp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Various asynchronous TCP/IP classes. + +End users shouldn't use this module directly - use the reactor APIs instead. +""" + +from __future__ import division, absolute_import +# System Imports +import socket +import sys +import operator +import os +import struct + +import attr + +from zope.interface import Interface, implementer + +from twisted.logger import Logger +from twisted.python.compat import lazyByteSlice, unicode +from twisted.python.runtime import platformType +from twisted.python import versions, deprecate + +try: + # Try to get the memory BIO based startTLS implementation, available since + # pyOpenSSL 0.10 + from twisted.internet._newtls import ( + ConnectionMixin as _TLSConnectionMixin, + ClientMixin as _TLSClientMixin, + ServerMixin as _TLSServerMixin) +except ImportError: + # There is no version of startTLS available + class _TLSConnectionMixin(object): + TLS = False + + + class _TLSClientMixin(object): + pass + + + class _TLSServerMixin(object): + pass + + +if platformType == 'win32': + # no such thing as WSAEPERM or error code 10001 according to winsock.h or MSDN + EPERM = object() + from errno import WSAEINVAL as EINVAL + from errno import WSAEWOULDBLOCK as EWOULDBLOCK + from errno import WSAEINPROGRESS as EINPROGRESS + from errno import WSAEALREADY as EALREADY + from errno import WSAEISCONN as EISCONN + from errno import WSAENOBUFS as ENOBUFS + from errno import WSAEMFILE as EMFILE + # No such thing as WSAENFILE, either. + ENFILE = object() + # Nor ENOMEM + ENOMEM = object() + EAGAIN = EWOULDBLOCK + from errno import WSAECONNRESET as ECONNABORTED + + from twisted.python.win32 import formatError as strerror +else: + from errno import EPERM + from errno import EINVAL + from errno import EWOULDBLOCK + from errno import EINPROGRESS + from errno import EALREADY + from errno import EISCONN + from errno import ENOBUFS + from errno import EMFILE + from errno import ENFILE + from errno import ENOMEM + from errno import EAGAIN + from errno import ECONNABORTED + + from os import strerror + + +from errno import errorcode + +# Twisted Imports +from twisted.internet import base, address, fdesc +from twisted.internet.task import deferLater +from twisted.python import log, failure, reflect +from twisted.python.util import untilConcludes +from twisted.internet.error import CannotListenError +from twisted.internet import abstract, main, interfaces, error +from twisted.internet.protocol import Protocol + +# Not all platforms have, or support, this flag. +_AI_NUMERICSERV = getattr(socket, "AI_NUMERICSERV", 0) + + +# The type for service names passed to socket.getservbyname: +_portNameType = (str, unicode) + + +def _getrealname(addr): + """ + Return a 2-tuple of socket IP and port for IPv4 and a 4-tuple of + socket IP, port, flowInfo, and scopeID for IPv6. For IPv6, it + returns the interface portion (the part after the %) as a part of + the IPv6 address, which Python 3.7+ does not include. + + @param addr: A 2-tuple for IPv4 information or a 4-tuple for IPv6 + information. + """ + if len(addr) == 4: + # IPv6 + host = socket.getnameinfo( + addr, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV)[0] + return tuple([host] + list(addr[1:])) + else: + return addr[:2] + + + +def _getpeername(skt): + """ + See L{_getrealname}. + """ + return _getrealname(skt.getpeername()) + + + +def _getsockname(skt): + """ + See L{_getrealname}. + """ + return _getrealname(skt.getsockname()) + + + +class _SocketCloser(object): + """ + @ivar _shouldShutdown: Set to C{True} if C{shutdown} should be called + before calling C{close} on the underlying socket. + @type _shouldShutdown: C{bool} + """ + _shouldShutdown = True + + def _closeSocket(self, orderly): + # The call to shutdown() before close() isn't really necessary, because + # we set FD_CLOEXEC now, which will ensure this is the only process + # holding the FD, thus ensuring close() really will shutdown the TCP + # socket. However, do it anyways, just to be safe. + skt = self.socket + try: + if orderly: + if self._shouldShutdown: + skt.shutdown(2) + else: + # Set SO_LINGER to 1,0 which, by convention, causes a + # connection reset to be sent when close is called, + # instead of the standard FIN shutdown sequence. + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack("ii", 1, 0)) + + except socket.error: + pass + try: + skt.close() + except socket.error: + pass + + + +class _AbortingMixin(object): + """ + Common implementation of C{abortConnection}. + + @ivar _aborting: Set to C{True} when C{abortConnection} is called. + @type _aborting: C{bool} + """ + _aborting = False + + def abortConnection(self): + """ + Aborts the connection immediately, dropping any buffered data. + + @since: 11.1 + """ + if self.disconnected or self._aborting: + return + self._aborting = True + self.stopReading() + self.stopWriting() + self.doRead = lambda *args, **kwargs: None + self.doWrite = lambda *args, **kwargs: None + self.reactor.callLater(0, self.connectionLost, + failure.Failure(error.ConnectionAborted())) + + + +@implementer(interfaces.ITCPTransport, interfaces.ISystemHandle) +class Connection(_TLSConnectionMixin, abstract.FileDescriptor, _SocketCloser, + _AbortingMixin): + """ + Superclass of all socket-based FileDescriptors. + + This is an abstract superclass of all objects which represent a TCP/IP + connection based socket. + + @ivar logstr: prefix used when logging events related to this connection. + @type logstr: C{str} + """ + + + def __init__(self, skt, protocol, reactor=None): + abstract.FileDescriptor.__init__(self, reactor=reactor) + self.socket = skt + self.socket.setblocking(0) + self.fileno = skt.fileno + self.protocol = protocol + + + def getHandle(self): + """Return the socket for this connection.""" + return self.socket + + + def doRead(self): + """Calls self.protocol.dataReceived with all available data. + + This reads up to self.bufferSize bytes of data from its socket, then + calls self.dataReceived(data) to process it. If the connection is not + lost through an error in the physical recv(), this function will return + the result of the dataReceived call. + """ + try: + data = self.socket.recv(self.bufferSize) + except socket.error as se: + if se.args[0] == EWOULDBLOCK: + return + else: + return main.CONNECTION_LOST + + return self._dataReceived(data) + + + def _dataReceived(self, data): + if not data: + return main.CONNECTION_DONE + rval = self.protocol.dataReceived(data) + if rval is not None: + offender = self.protocol.dataReceived + warningFormat = ( + 'Returning a value other than None from %(fqpn)s is ' + 'deprecated since %(version)s.') + warningString = deprecate.getDeprecationWarningString( + offender, versions.Version('Twisted', 11, 0, 0), + format=warningFormat) + deprecate.warnAboutFunction(offender, warningString) + return rval + + + def writeSomeData(self, data): + """ + Write as much as possible of the given data to this TCP connection. + + This sends up to C{self.SEND_LIMIT} bytes from C{data}. If the + connection is lost, an exception is returned. Otherwise, the number + of bytes successfully written is returned. + """ + # Limit length of buffer to try to send, because some OSes are too + # stupid to do so themselves (ahem windows) + limitedData = lazyByteSlice(data, 0, self.SEND_LIMIT) + + try: + return untilConcludes(self.socket.send, limitedData) + except socket.error as se: + if se.args[0] in (EWOULDBLOCK, ENOBUFS): + return 0 + else: + return main.CONNECTION_LOST + + + def _closeWriteConnection(self): + try: + self.socket.shutdown(1) + except socket.error: + pass + p = interfaces.IHalfCloseableProtocol(self.protocol, None) + if p: + try: + p.writeConnectionLost() + except: + f = failure.Failure() + log.err() + self.connectionLost(f) + + + def readConnectionLost(self, reason): + p = interfaces.IHalfCloseableProtocol(self.protocol, None) + if p: + try: + p.readConnectionLost() + except: + log.err() + self.connectionLost(failure.Failure()) + else: + self.connectionLost(reason) + + + + def connectionLost(self, reason): + """See abstract.FileDescriptor.connectionLost(). + """ + # Make sure we're not called twice, which can happen e.g. if + # abortConnection() is called from protocol's dataReceived and then + # code immediately after throws an exception that reaches the + # reactor. We can't rely on "disconnected" attribute for this check + # since twisted.internet._oldtls does evil things to it: + if not hasattr(self, "socket"): + return + abstract.FileDescriptor.connectionLost(self, reason) + self._closeSocket(not reason.check(error.ConnectionAborted)) + protocol = self.protocol + del self.protocol + del self.socket + del self.fileno + protocol.connectionLost(reason) + + + logstr = "Uninitialized" + + def logPrefix(self): + """Return the prefix to log with when I own the logging thread. + """ + return self.logstr + + def getTcpNoDelay(self): + return operator.truth(self.socket.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)) + + def setTcpNoDelay(self, enabled): + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, enabled) + + def getTcpKeepAlive(self): + return operator.truth(self.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_KEEPALIVE)) + + def setTcpKeepAlive(self, enabled): + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, enabled) + + + + +class _BaseBaseClient(object): + """ + Code shared with other (non-POSIX) reactors for management of general + outgoing connections. + + Requirements upon subclasses are documented as instance variables rather + than abstract methods, in order to avoid MRO confusion, since this base is + mixed in to unfortunately weird and distinctive multiple-inheritance + hierarchies and many of these attributes are provided by peer classes + rather than descendant classes in those hierarchies. + + @ivar addressFamily: The address family constant (C{socket.AF_INET}, + C{socket.AF_INET6}, C{socket.AF_UNIX}) of the underlying socket of this + client connection. + @type addressFamily: C{int} + + @ivar socketType: The socket type constant (C{socket.SOCK_STREAM} or + C{socket.SOCK_DGRAM}) of the underlying socket. + @type socketType: C{int} + + @ivar _requiresResolution: A flag indicating whether the address of this + client will require name resolution. C{True} if the hostname of said + address indicates a name that must be resolved by hostname lookup, + C{False} if it indicates an IP address literal. + @type _requiresResolution: C{bool} + + @cvar _commonConnection: Subclasses must provide this attribute, which + indicates the L{Connection}-alike class to invoke C{__init__} and + C{connectionLost} on. + @type _commonConnection: C{type} + + @ivar _stopReadingAndWriting: Subclasses must implement in order to remove + this transport from its reactor's notifications in response to a + terminated connection attempt. + @type _stopReadingAndWriting: 0-argument callable returning L{None} + + @ivar _closeSocket: Subclasses must implement in order to close the socket + in response to a terminated connection attempt. + @type _closeSocket: 1-argument callable; see L{_SocketCloser._closeSocket} + + @ivar _collectSocketDetails: Clean up references to the attached socket in + its underlying OS resource (such as a file descriptor or file handle), + as part of post connection-failure cleanup. + @type _collectSocketDetails: 0-argument callable returning L{None}. + + @ivar reactor: The class pointed to by C{_commonConnection} should set this + attribute in its constructor. + @type reactor: L{twisted.internet.interfaces.IReactorTime}, + L{twisted.internet.interfaces.IReactorCore}, + L{twisted.internet.interfaces.IReactorFDSet} + """ + + addressFamily = socket.AF_INET + socketType = socket.SOCK_STREAM + + def _finishInit(self, whenDone, skt, error, reactor): + """ + Called by subclasses to continue to the stage of initialization where + the socket connect attempt is made. + + @param whenDone: A 0-argument callable to invoke once the connection is + set up. This is L{None} if the connection could not be prepared + due to a previous error. + + @param skt: The socket object to use to perform the connection. + @type skt: C{socket._socketobject} + + @param error: The error to fail the connection with. + + @param reactor: The reactor to use for this client. + @type reactor: L{twisted.internet.interfaces.IReactorTime} + """ + if whenDone: + self._commonConnection.__init__(self, skt, None, reactor) + reactor.callLater(0, whenDone) + else: + reactor.callLater(0, self.failIfNotConnected, error) + + + def resolveAddress(self): + """ + Resolve the name that was passed to this L{_BaseBaseClient}, if + necessary, and then move on to attempting the connection once an + address has been determined. (The connection will be attempted + immediately within this function if either name resolution can be + synchronous or the address was an IP address literal.) + + @note: You don't want to call this method from outside, as it won't do + anything useful; it's just part of the connection bootstrapping + process. Also, although this method is on L{_BaseBaseClient} for + historical reasons, it's not used anywhere except for L{Client} + itself. + + @return: L{None} + """ + if self._requiresResolution: + d = self.reactor.resolve(self.addr[0]) + d.addCallback(lambda n: (n,) + self.addr[1:]) + d.addCallbacks(self._setRealAddress, self.failIfNotConnected) + else: + self._setRealAddress(self.addr) + + + def _setRealAddress(self, address): + """ + Set the resolved address of this L{_BaseBaseClient} and initiate the + connection attempt. + + @param address: Depending on whether this is an IPv4 or IPv6 connection + attempt, a 2-tuple of C{(host, port)} or a 4-tuple of C{(host, + port, flow, scope)}. At this point it is a fully resolved address, + and the 'host' portion will always be an IP address, not a DNS + name. + """ + if len(address) == 4: + # IPv6, make sure we have the scopeID associated + hostname = socket.getnameinfo( + address, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV)[0] + self.realAddress = tuple([hostname] + list(address[1:])) + else: + self.realAddress = address + self.doConnect() + + + def failIfNotConnected(self, err): + """ + Generic method called when the attempts to connect failed. It basically + cleans everything it can: call connectionFailed, stop read and write, + delete socket related members. + """ + if (self.connected or self.disconnected or + not hasattr(self, "connector")): + return + + self._stopReadingAndWriting() + try: + self._closeSocket(True) + except AttributeError: + pass + else: + self._collectSocketDetails() + self.connector.connectionFailed(failure.Failure(err)) + del self.connector + + + def stopConnecting(self): + """ + If a connection attempt is still outstanding (i.e. no connection is + yet established), immediately stop attempting to connect. + """ + self.failIfNotConnected(error.UserError()) + + + def connectionLost(self, reason): + """ + Invoked by lower-level logic when it's time to clean the socket up. + Depending on the state of the connection, either inform the attached + L{Connector} that the connection attempt has failed, or inform the + connected L{IProtocol} that the established connection has been lost. + + @param reason: the reason that the connection was terminated + @type reason: L{Failure} + """ + if not self.connected: + self.failIfNotConnected(error.ConnectError(string=reason)) + else: + self._commonConnection.connectionLost(self, reason) + self.connector.connectionLost(reason) + + + +class BaseClient(_BaseBaseClient, _TLSClientMixin, Connection): + """ + A base class for client TCP (and similar) sockets. + + @ivar realAddress: The address object that will be used for socket.connect; + this address is an address tuple (the number of elements dependent upon + the address family) which does not contain any names which need to be + resolved. + @type realAddress: C{tuple} + + @ivar _base: L{Connection}, which is the base class of this class which has + all of the useful file descriptor methods. This is used by + L{_TLSServerMixin} to call the right methods to directly manipulate the + transport, as is necessary for writing TLS-encrypted bytes (whereas + those methods on L{Server} will go through another layer of TLS if it + has been enabled). + """ + + _base = Connection + _commonConnection = Connection + + def _stopReadingAndWriting(self): + """ + Implement the POSIX-ish (i.e. + L{twisted.internet.interfaces.IReactorFDSet}) method of detaching this + socket from the reactor for L{_BaseBaseClient}. + """ + if hasattr(self, "reactor"): + # this doesn't happen if we failed in __init__ + self.stopReading() + self.stopWriting() + + + def _collectSocketDetails(self): + """ + Clean up references to the socket and its file descriptor. + + @see: L{_BaseBaseClient} + """ + del self.socket, self.fileno + + + def createInternetSocket(self): + """(internal) Create a non-blocking socket using + self.addressFamily, self.socketType. + """ + s = socket.socket(self.addressFamily, self.socketType) + s.setblocking(0) + fdesc._setCloseOnExec(s.fileno()) + return s + + + def doConnect(self): + """ + Initiate the outgoing connection attempt. + + @note: Applications do not need to call this method; it will be invoked + internally as part of L{IReactorTCP.connectTCP}. + """ + self.doWrite = self.doConnect + self.doRead = self.doConnect + if not hasattr(self, "connector"): + # this happens when connection failed but doConnect + # was scheduled via a callLater in self._finishInit + return + + err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err: + self.failIfNotConnected(error.getConnectError((err, strerror(err)))) + return + + # doConnect gets called twice. The first time we actually need to + # start the connection attempt. The second time we don't really + # want to (SO_ERROR above will have taken care of any errors, and if + # it reported none, the mere fact that doConnect was called again is + # sufficient to indicate that the connection has succeeded), but it + # is not /particularly/ detrimental to do so. This should get + # cleaned up some day, though. + try: + connectResult = self.socket.connect_ex(self.realAddress) + except socket.error as se: + connectResult = se.args[0] + if connectResult: + if connectResult == EISCONN: + pass + # on Windows EINVAL means sometimes that we should keep trying: + # http://msdn.microsoft.com/library/default.asp?url=/library/en-us/winsock/winsock/connect_2.asp + elif ((connectResult in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or + (connectResult == EINVAL and platformType == "win32")): + self.startReading() + self.startWriting() + return + else: + self.failIfNotConnected(error.getConnectError((connectResult, strerror(connectResult)))) + return + + # If I have reached this point without raising or returning, that means + # that the socket is connected. + del self.doWrite + del self.doRead + # we first stop and then start, to reset any references to the old doRead + self.stopReading() + self.stopWriting() + self._connectDone() + + + def _connectDone(self): + """ + This is a hook for when a connection attempt has succeeded. + + Here, we build the protocol from the + L{twisted.internet.protocol.ClientFactory} that was passed in, compute + a log string, begin reading so as to send traffic to the newly built + protocol, and finally hook up the protocol itself. + + This hook is overridden by L{ssl.Client} to initiate the TLS protocol. + """ + self.protocol = self.connector.buildProtocol(self.getPeer()) + self.connected = 1 + logPrefix = self._getLogPrefix(self.protocol) + self.logstr = "%s,client" % logPrefix + if self.protocol is None: + # Factory.buildProtocol is allowed to return None. In that case, + # make up a protocol to satisfy the rest of the implementation; + # connectionLost is going to be called on something, for example. + # This is easier than adding special case support for a None + # protocol throughout the rest of the transport implementation. + self.protocol = Protocol() + # But dispose of the connection quickly. + self.loseConnection() + else: + self.startReading() + self.protocol.makeConnection(self) + + + +_NUMERIC_ONLY = socket.AI_NUMERICHOST | _AI_NUMERICSERV + +def _resolveIPv6(ip, port): + """ + Resolve an IPv6 literal into an IPv6 address. + + This is necessary to resolve any embedded scope identifiers to the relevant + C{sin6_scope_id} for use with C{socket.connect()}, C{socket.listen()}, or + C{socket.bind()}; see U{RFC 3493 } for + more information. + + @param ip: An IPv6 address literal. + @type ip: C{str} + + @param port: A port number. + @type port: C{int} + + @return: a 4-tuple of C{(host, port, flow, scope)}, suitable for use as an + IPv6 address. + + @raise socket.gaierror: if either the IP or port is not numeric as it + should be. + """ + return socket.getaddrinfo(ip, port, 0, 0, 0, _NUMERIC_ONLY)[0][4] + + + +class _BaseTCPClient(object): + """ + Code shared with other (non-POSIX) reactors for management of outgoing TCP + connections (both TCPv4 and TCPv6). + + @note: In order to be functional, this class must be mixed into the same + hierarchy as L{_BaseBaseClient}. It would subclass L{_BaseBaseClient} + directly, but the class hierarchy here is divided in strange ways out + of the need to share code along multiple axes; specifically, with the + IOCP reactor and also with UNIX clients in other reactors. + + @ivar _addressType: The Twisted _IPAddress implementation for this client + @type _addressType: L{IPv4Address} or L{IPv6Address} + + @ivar connector: The L{Connector} which is driving this L{_BaseTCPClient}'s + connection attempt. + + @ivar addr: The address that this socket will be connecting to. + @type addr: If IPv4, a 2-C{tuple} of C{(str host, int port)}. If IPv6, a + 4-C{tuple} of (C{str host, int port, int ignored, int scope}). + + @ivar createInternetSocket: Subclasses must implement this as a method to + create a python socket object of the appropriate address family and + socket type. + @type createInternetSocket: 0-argument callable returning + C{socket._socketobject}. + """ + + _addressType = address.IPv4Address + + def __init__(self, host, port, bindAddress, connector, reactor=None): + # BaseClient.__init__ is invoked later + self.connector = connector + self.addr = (host, port) + + whenDone = self.resolveAddress + err = None + skt = None + + if abstract.isIPAddress(host): + self._requiresResolution = False + elif abstract.isIPv6Address(host): + self._requiresResolution = False + self.addr = _resolveIPv6(host, port) + self.addressFamily = socket.AF_INET6 + self._addressType = address.IPv6Address + else: + self._requiresResolution = True + try: + skt = self.createInternetSocket() + except socket.error as se: + err = error.ConnectBindError(se.args[0], se.args[1]) + whenDone = None + if whenDone and bindAddress is not None: + try: + if abstract.isIPv6Address(bindAddress[0]): + bindinfo = _resolveIPv6(*bindAddress) + else: + bindinfo = bindAddress + skt.bind(bindinfo) + except socket.error as se: + err = error.ConnectBindError(se.args[0], se.args[1]) + whenDone = None + self._finishInit(whenDone, skt, err, reactor) + + + def getHost(self): + """ + Returns an L{IPv4Address} or L{IPv6Address}. + + This indicates the address from which I am connecting. + """ + return self._addressType('TCP', *_getsockname(self.socket)) + + + def getPeer(self): + """ + Returns an L{IPv4Address} or L{IPv6Address}. + + This indicates the address that I am connected to. + """ + return self._addressType('TCP', *self.realAddress) + + + def __repr__(self): + s = '<%s to %s at %x>' % (self.__class__, self.addr, id(self)) + return s + + + +class Client(_BaseTCPClient, BaseClient): + """ + A transport for a TCP protocol; either TCPv4 or TCPv6. + + Do not create these directly; use L{IReactorTCP.connectTCP}. + """ + + + +class Server(_TLSServerMixin, Connection): + """ + Serverside socket-stream connection class. + + This is a serverside network connection transport; a socket which came from + an accept() on a server. + + @ivar _base: L{Connection}, which is the base class of this class which has + all of the useful file descriptor methods. This is used by + L{_TLSServerMixin} to call the right methods to directly manipulate the + transport, as is necessary for writing TLS-encrypted bytes (whereas + those methods on L{Server} will go through another layer of TLS if it + has been enabled). + """ + _base = Connection + + _addressType = address.IPv4Address + + def __init__(self, sock, protocol, client, server, sessionno, reactor): + """ + Server(sock, protocol, client, server, sessionno) + + Initialize it with a socket, a protocol, a descriptor for my peer (a + tuple of host, port describing the other end of the connection), an + instance of Port, and a session number. + """ + Connection.__init__(self, sock, protocol, reactor) + if len(client) != 2: + self._addressType = address.IPv6Address + self.server = server + self.client = client + self.sessionno = sessionno + self.hostname = client[0] + + logPrefix = self._getLogPrefix(self.protocol) + self.logstr = "%s,%s,%s" % (logPrefix, + sessionno, + self.hostname) + if self.server is not None: + self.repstr = "<%s #%s on %s>" % (self.protocol.__class__.__name__, + self.sessionno, + self.server._realPortNumber) + self.startReading() + self.connected = 1 + + def __repr__(self): + """ + A string representation of this connection. + """ + return self.repstr + + + @classmethod + def _fromConnectedSocket(cls, fileDescriptor, addressFamily, factory, + reactor): + """ + Create a new L{Server} based on an existing connected I{SOCK_STREAM} + socket. + + Arguments are the same as to L{Server.__init__}, except where noted. + + @param fileDescriptor: An integer file descriptor associated with a + connected socket. The socket must be in non-blocking mode. Any + additional attributes desired, such as I{FD_CLOEXEC}, must also be + set already. + + @param addressFamily: The address family (sometimes called I{domain}) + of the existing socket. For example, L{socket.AF_INET}. + + @return: A new instance of C{cls} wrapping the socket given by + C{fileDescriptor}. + """ + addressType = address.IPv4Address + if addressFamily == socket.AF_INET6: + addressType = address.IPv6Address + skt = socket.fromfd(fileDescriptor, addressFamily, socket.SOCK_STREAM) + addr = _getpeername(skt) + protocolAddr = addressType('TCP', *addr) + localPort = skt.getsockname()[1] + + protocol = factory.buildProtocol(protocolAddr) + if protocol is None: + skt.close() + return + + self = cls(skt, protocol, addr, None, addr[1], reactor) + self.repstr = "<%s #%s on %s>" % ( + self.protocol.__class__.__name__, self.sessionno, localPort) + protocol.makeConnection(self) + return self + + + def getHost(self): + """ + Returns an L{IPv4Address} or L{IPv6Address}. + + This indicates the server's address. + """ + addr = _getsockname(self.socket) + return self._addressType('TCP', *addr) + + + def getPeer(self): + """ + Returns an L{IPv4Address} or L{IPv6Address}. + + This indicates the client's address. + """ + return self._addressType('TCP', *self.client) + + + +class _IFileDescriptorReservation(Interface): + """ + An open file that represents an emergency reservation in the + process' file descriptor table. If L{Port} encounters C{EMFILE} + on C{accept(2)}, it can close this file descriptor, retry the + C{accept} so that the incoming connection occupies this file + descriptor's space, and then close that connection and reopen this + one. + + Calling L{_IFileDescriptorReservation.reserve} attempts to open + the reserve file descriptor if it is not already open. + L{_IFileDescriptorReservation.available} returns L{True} if the + underlying file is open and its descriptor claimed. + + L{_IFileDescriptorReservation} instances are context managers; + entering them releases the underlying file descriptor, while + exiting them attempts to reacquire it. The block can take + advantage of the free slot in the process' file descriptor table + accept and close a client connection. + + Because another thread might open a file descriptor between the + time the context manager is entered and the time C{accept} is + called, opening the reserve descriptor is best-effort only. + """ + + def available(): + """ + Is the reservation available? + + @return: L{True} if the reserved file descriptor is open and + can thus be closed to allow a new file to be opened in its + place; L{False} if it is not open. + """ + + + def reserve(): + """ + Attempt to open the reserved file descriptor; if this fails + because of C{EMFILE}, internal state is reset so that another + reservation attempt can be made. + + @raises: Any exception except an L{OSError} or L{IOError} + whose errno is L{EMFILE}. + """ + + + def __enter__(): + """ + Release the underlying file descriptor so that code within the + context manager can open a new file. + """ + + + def __exit__(excType, excValue, traceback): + """ + Attempt to re-open the reserved file descriptor. See + L{reserve} for caveats. + + @param excType: See L{object.__exit__} + @param excValue: See L{object.__exit__} + @param traceback: See L{object.__exit__} + """ + + + +@implementer(_IFileDescriptorReservation) +@attr.s +class _FileDescriptorReservation(object): + """ + L{_IFileDescriptorReservation} implementation. + + @ivar fileFactory: A factory that will be called to reserve a + file descriptor. + @type fileFactory: A L{callable} that accepts no arguments and + returns an object with a C{close} method. + """ + _log = Logger() + + _fileFactory = attr.ib() + _fileDescriptor = attr.ib(init=False, default=None) + + + def available(self): + """ + See L{_IFileDescriptorReservation.available}. + + @return: L{True} if the reserved file descriptor is open and + can thus be closed to allow a new file to be opened in its + place; L{False} if it is not open. + """ + return self._fileDescriptor is not None + + + def reserve(self): + """ + See L{_IFileDescriptorReservation.reserve}. + """ + if self._fileDescriptor is None: + try: + fileDescriptor = self._fileFactory() + except (IOError, OSError) as e: + if e.errno == EMFILE: + self._log.failure( + "Could not reserve EMFILE recovery file descriptor.") + else: + raise + else: + self._fileDescriptor = fileDescriptor + + + def __enter__(self): + """ + See L{_IFileDescriptorReservation.__enter__}. + """ + if self._fileDescriptor is None: + raise RuntimeError( + "No file reserved. Have you called my reserve method?") + self._fileDescriptor.close() + self._fileDescriptor = None + + + def __exit__(self, excValue, excType, traceback): + """ + See L{_IFileDescriptorReservation.__exit__}. + """ + try: + self.reserve() + except Exception: + self._log.failure( + "Could not re-reserve EMFILE recovery file descriptor.") + + + +@implementer(_IFileDescriptorReservation) +class _NullFileDescriptorReservation(object): + """ + A null implementation of L{_IFileDescriptorReservation}. + """ + + def available(self): + """ + The reserved file is never available. See + L{_IFileDescriptorReservation.available}. + + @return: L{False} + """ + return False + + + def reserve(self): + """ + Do nothing. See L{_IFileDescriptorReservation.reserve}. + """ + + + def __enter__(self): + """ + Do nothing. See L{_IFileDescriptorReservation.__enter__} + + @return: L{False} + """ + + + def __exit__(self, excValue, excType, traceback): + """ + Do nothing. See L{_IFileDescriptorReservation.__exit__}. + + @param excType: See L{object.__exit__} + @param excValue: See L{object.__exit__} + @param traceback: See L{object.__exit__} + """ + + + +# Don't keep a reserve file descriptor for coping with file descriptor +# exhaustion on Windows. + +# WSAEMFILE occurs when a process has run out of memory, not when a +# specific limit has been reached. Windows sockets are handles, which +# differ from UNIX's file descriptors in that they can refer to any +# "named kernel object", including user interface resources like menu +# and icons. The generality of handles results in a much higher limit +# than UNIX imposes on file descriptors: a single Windows process can +# allocate up to 16,777,216 handles. Because they're indexes into a +# three level table whose upper two layers are allocated from +# swappable pages, handles compete for heap space with other kernel +# objects, not with each other. Closing a given socket handle may not +# release enough memory to allow the process to make progress. +# +# This fundamental difference between file descriptors and handles +# makes a reserve file descriptor useless on Windows. Note that other +# event loops, such as libuv and libevent, also do not special case +# WSAEMFILE. +# +# For an explanation of handles, see the "Object Manager" +# (pp. 140-175) section of +# +# Windows Internals, Part 1: Covering Windows Server 2008 R2 and +# Windows 7 (6th ed.) +# Mark E. Russinovich, David A. Solomon, and Alex +# Ionescu. 2012. Microsoft Press. +if platformType == 'win32': + _reservedFD = _NullFileDescriptorReservation() +else: + _reservedFD = _FileDescriptorReservation(lambda: open(os.devnull)) + + +# Linux and other UNIX-like operating systems return EMFILE when a +# process has reached its soft limit of file descriptors. *BSD and +# Win32 raise (WSA)ENOBUFS when socket limits are reached. Linux can +# give ENFILE if the system is out of inodes, or ENOMEM if there is +# insufficient memory to allocate a new dentry. ECONNABORTED is +# documented as possible on all relevant platforms (Linux, Windows, +# macOS, and the BSDs) but occurs only on the BSDs. It occurs when a +# client sends a FIN or RST after the server sends a SYN|ACK but +# before application code calls accept(2). On Linux, calling +# accept(2) on such a listener returns a connection that fails as +# though the it were terminated after being fully established. This +# appears to be an implementation choice (see inet_accept in +# inet/ipv4/af_inet.c). On macOS, such a listener is not considered +# readable, so accept(2) will never be called. Calling accept(2) on +# such a listener, however, does not return at all. +_ACCEPT_ERRORS = (EMFILE, ENOBUFS, ENFILE, ENOMEM, ECONNABORTED) + + + +@attr.s +class _BuffersLogs(object): + """ + A context manager that buffers any log events until after its + block exits. + + @ivar _namespace: The namespace of the buffered events. + @type _namespace: L{str}. + + @ivar _observer: The observer to which buffered log events will be + written + @type _observer: L{twisted.logger.ILogObserver}. + """ + _namespace = attr.ib() + _observer = attr.ib() + _logs = attr.ib(default=attr.Factory(list)) + + def __enter__(self): + """ + Enter a log buffering context. + + @return: A logger that buffers log events. + @rtype: L{Logger}. + """ + return Logger(namespace=self._namespace, observer=self._logs.append) + + + def __exit__(self, excValue, excType, traceback): + """ + Exit a log buffering context and log all buffered events to + the provided observer. + + @param excType: See L{object.__exit__} + @param excValue: See L{object.__exit__} + @param traceback: See L{object.__exit__} + """ + for event in self._logs: + self._observer(event) + + + +def _accept(logger, accepts, listener, reservedFD): + """ + Return a generator that yields client sockets from the provided + listening socket until there are none left or an unrecoverable + error occurs. + + @param logger: A logger to which C{accept}-related events will be + logged. This should not log to arbitrary observers that might + open a file descriptor to avoid claiming the C{EMFILE} file + descriptor on UNIX-like systems. + @type logger: L{Logger} + + @param accepts: An iterable iterated over to limit the number + consecutive C{accept}s. + @type accepts: An iterable. + + @param listener: The listening socket. + @type listener: L{socket.socket} + + @param reservedFD: A reserved file descriptor that can be used to + recover from C{EMFILE} on UNIX-like systems. + @type reservedFD: L{_IFileDescriptorReservation} + + @return: A generator that yields C{(socket, addr)} tuples from + L{socket.socket.accept} + """ + for _ in accepts: + try: + client, address = listener.accept() + except socket.error as e: + if e.args[0] in (EWOULDBLOCK, EAGAIN): + # No more clients. + return + elif e.args[0] == EPERM: + # Netfilter on Linux may have rejected the + # connection, but we get told to try to accept() + # anyway. + continue + elif e.args[0] == EMFILE and reservedFD.available(): + # Linux and other UNIX-like operating systems return + # EMFILE when a process has reached its soft limit of + # file descriptors. The reserved file descriptor is + # available, so it can be released to free up a + # descriptor for use by listener.accept()'s clients. + # Each client socket will be closed until the listener + # returns EAGAIN. + logger.info("EMFILE encountered;" + " releasing reserved file descriptor.") + # The following block should not run arbitrary code + # that might acquire its own file descriptor. + with reservedFD: + clientsToClose = _accept( + logger, accepts, listener, reservedFD) + for clientToClose, closedAddress in clientsToClose: + clientToClose.close() + logger.info("EMFILE recovery:" + " Closed socket from {address}", + address=closedAddress) + logger.info( + "Re-reserving EMFILE recovery file descriptor.") + return + elif e.args[0] in _ACCEPT_ERRORS: + logger.info("Could not accept new connection ({acceptError})", + acceptError=errorcode[e.args[0]]) + return + else: + raise + else: + yield client, address + + + +@implementer(interfaces.IListeningPort) +class Port(base.BasePort, _SocketCloser): + """ + A TCP server port, listening for connections. + + When a connection is accepted, this will call a factory's buildProtocol + with the incoming address as an argument, according to the specification + described in L{twisted.internet.interfaces.IProtocolFactory}. + + If you wish to change the sort of transport that will be used, the + C{transport} attribute will be called with the signature expected for + C{Server.__init__}, so it can be replaced. + + @ivar deferred: a deferred created when L{stopListening} is called, and + that will fire when connection is lost. This is not to be used it + directly: prefer the deferred returned by L{stopListening} instead. + @type deferred: L{defer.Deferred} + + @ivar disconnecting: flag indicating that the L{stopListening} method has + been called and that no connections should be accepted anymore. + @type disconnecting: C{bool} + + @ivar connected: flag set once the listen has successfully been called on + the socket. + @type connected: C{bool} + + @ivar _type: A string describing the connections which will be created by + this port. Normally this is C{"TCP"}, since this is a TCP port, but + when the TLS implementation re-uses this class it overrides the value + with C{"TLS"}. Only used for logging. + + @ivar _preexistingSocket: If not L{None}, a L{socket.socket} instance which + was created and initialized outside of the reactor and will be used to + listen for connections (instead of a new socket being created by this + L{Port}). + """ + + socketType = socket.SOCK_STREAM + + transport = Server + sessionno = 0 + interface = '' + backlog = 50 + + _type = 'TCP' + + # Actual port number being listened on, only set to a non-None + # value when we are actually listening. + _realPortNumber = None + + # An externally initialized socket that we will use, rather than creating + # our own. + _preexistingSocket = None + + addressFamily = socket.AF_INET + _addressType = address.IPv4Address + _logger = Logger() + + def __init__(self, port, factory, backlog=50, interface='', reactor=None): + """Initialize with a numeric port to listen on. + """ + base.BasePort.__init__(self, reactor=reactor) + self.port = port + self.factory = factory + self.backlog = backlog + if abstract.isIPv6Address(interface): + self.addressFamily = socket.AF_INET6 + self._addressType = address.IPv6Address + self.interface = interface + + + @classmethod + def _fromListeningDescriptor(cls, reactor, fd, addressFamily, factory): + """ + Create a new L{Port} based on an existing listening I{SOCK_STREAM} + socket. + + Arguments are the same as to L{Port.__init__}, except where noted. + + @param fd: An integer file descriptor associated with a listening + socket. The socket must be in non-blocking mode. Any additional + attributes desired, such as I{FD_CLOEXEC}, must also be set already. + + @param addressFamily: The address family (sometimes called I{domain}) of + the existing socket. For example, L{socket.AF_INET}. + + @return: A new instance of C{cls} wrapping the socket given by C{fd}. + """ + port = socket.fromfd(fd, addressFamily, cls.socketType) + interface = _getsockname(port)[0] + self = cls(None, factory, None, interface, reactor) + self._preexistingSocket = port + return self + + + def __repr__(self): + if self._realPortNumber is not None: + return "<%s of %s on %s>" % (self.__class__, + self.factory.__class__, self._realPortNumber) + else: + return "<%s of %s (not listening)>" % (self.__class__, self.factory.__class__) + + def createInternetSocket(self): + s = base.BasePort.createInternetSocket(self) + if platformType == "posix" and sys.platform != "cygwin": + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s + + + def startListening(self): + """Create and bind my socket, and begin listening on it. + + This is called on unserialization, and must be called after creating a + server to begin listening on the specified port. + """ + _reservedFD.reserve() + if self._preexistingSocket is None: + # Create a new socket and make it listen + try: + skt = self.createInternetSocket() + if self.addressFamily == socket.AF_INET6: + addr = _resolveIPv6(self.interface, self.port) + else: + addr = (self.interface, self.port) + skt.bind(addr) + except socket.error as le: + raise CannotListenError(self.interface, self.port, le) + skt.listen(self.backlog) + else: + # Re-use the externally specified socket + skt = self._preexistingSocket + self._preexistingSocket = None + # Avoid shutting it down at the end. + self._shouldShutdown = False + + # Make sure that if we listened on port 0, we update that to + # reflect what the OS actually assigned us. + self._realPortNumber = skt.getsockname()[1] + + log.msg("%s starting on %s" % ( + self._getLogPrefix(self.factory), self._realPortNumber)) + + # The order of the next 5 lines is kind of bizarre. If no one + # can explain it, perhaps we should re-arrange them. + self.factory.doStart() + self.connected = True + self.socket = skt + self.fileno = self.socket.fileno + self.numberAccepts = 100 + + self.startReading() + + def _buildAddr(self, address): + return self._addressType('TCP', *address) + + def doRead(self): + """ + Called when my socket is ready for reading. + + This accepts a connection and calls self.protocol() to handle the + wire-level protocol. + """ + try: + if platformType == "posix": + numAccepts = self.numberAccepts + else: + # win32 event loop breaks if we do more than one accept() + # in an iteration of the event loop. + numAccepts = 1 + + with _BuffersLogs(self._logger.namespace, + self._logger.observer) as bufferingLogger: + accepted = 0 + clients = _accept(bufferingLogger, + range(numAccepts), + self.socket, + _reservedFD) + + for accepted, (skt, addr) in enumerate(clients, 1): + fdesc._setCloseOnExec(skt.fileno()) + + if len(addr) == 4: + # IPv6, make sure we get the scopeID if it + # exists + host = socket.getnameinfo( + addr, + socket.NI_NUMERICHOST | socket.NI_NUMERICSERV) + addr = tuple([host[0]] + list(addr[1:])) + + protocol = self.factory.buildProtocol( + self._buildAddr(addr)) + if protocol is None: + skt.close() + continue + s = self.sessionno + self.sessionno = s + 1 + transport = self.transport( + skt, protocol, addr, self, s, self.reactor) + protocol.makeConnection(transport) + + # Scale our synchronous accept loop according to traffic + # Reaching our limit on consecutive accept calls indicates + # there might be still more clients to serve the next time + # the reactor calls us. Prepare to accept some more. + if accepted == self.numberAccepts: + self.numberAccepts += 20 + # Otherwise, don't attempt to accept any more clients than + # we just accepted or any less than 1. + else: + self.numberAccepts = max(1, accepted) + except BaseException: + # Note that in TLS mode, this will possibly catch SSL.Errors + # raised by self.socket.accept() + # + # There is no "except SSL.Error:" above because SSL may be + # None if there is no SSL support. In any case, all the + # "except SSL.Error:" suite would probably do is log.deferr() + # and return, so handling it here works just as well. + log.deferr() + + def loseConnection(self, connDone=failure.Failure(main.CONNECTION_DONE)): + """ + Stop accepting connections on this port. + + This will shut down the socket and call self.connectionLost(). It + returns a deferred which will fire successfully when the port is + actually closed, or with a failure if an error occurs shutting down. + """ + self.disconnecting = True + self.stopReading() + if self.connected: + self.deferred = deferLater( + self.reactor, 0, self.connectionLost, connDone) + return self.deferred + + stopListening = loseConnection + + def _logConnectionLostMsg(self): + """ + Log message for closing port + """ + log.msg('(%s Port %s Closed)' % (self._type, self._realPortNumber)) + + + def connectionLost(self, reason): + """ + Cleans up the socket. + """ + self._logConnectionLostMsg() + self._realPortNumber = None + + base.BasePort.connectionLost(self, reason) + self.connected = False + self._closeSocket(True) + del self.socket + del self.fileno + + try: + self.factory.doStop() + finally: + self.disconnecting = False + + + def logPrefix(self): + """Returns the name of my class, to prefix log entries with. + """ + return reflect.qual(self.factory.__class__) + + + def getHost(self): + """ + Return an L{IPv4Address} or L{IPv6Address} indicating the listening + address of this port. + """ + addr = _getsockname(self.socket) + return self._addressType('TCP', *addr) + + + +class Connector(base.BaseConnector): + """ + A L{Connector} provides of L{twisted.internet.interfaces.IConnector} for + all POSIX-style reactors. + + @ivar _addressType: the type returned by L{Connector.getDestination}. + Either L{IPv4Address} or L{IPv6Address}, depending on the type of + address. + @type _addressType: C{type} + """ + _addressType = address.IPv4Address + + def __init__(self, host, port, factory, timeout, bindAddress, reactor=None): + if isinstance(port, _portNameType): + try: + port = socket.getservbyname(port, 'tcp') + except socket.error as e: + raise error.ServiceNameUnknownError(string="%s (%r)" % (e, port)) + self.host, self.port = host, port + if abstract.isIPv6Address(host): + self._addressType = address.IPv6Address + self.bindAddress = bindAddress + base.BaseConnector.__init__(self, factory, timeout, reactor) + + + def _makeTransport(self): + """ + Create a L{Client} bound to this L{Connector}. + + @return: a new L{Client} + @rtype: L{Client} + """ + return Client(self.host, self.port, self.bindAddress, self, self.reactor) + + + def getDestination(self): + """ + @see: L{twisted.internet.interfaces.IConnector.getDestination}. + """ + return self._addressType('TCP', self.host, self.port) diff --git a/contrib/python/Twisted/py2/twisted/internet/testing.py b/contrib/python/Twisted/py2/twisted/internet/testing.py new file mode 100644 index 00000000000..8602831ad55 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/testing.py @@ -0,0 +1,1010 @@ +# -*- test-case-name: twisted.internet.test.test_testing -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Assorted functionality which is commonly useful when writing unit tests. +""" + +from __future__ import division, absolute_import + +from socket import AF_INET, AF_INET6 +from io import BytesIO + +from zope.interface import implementer, implementedBy +from zope.interface.verify import verifyClass + +from twisted.python import failure +from twisted.python.compat import unicode, intToBytes, Sequence +from twisted.internet.defer import Deferred +from twisted.internet.interfaces import ( + ITransport, IConsumer, IPushProducer, IConnector, + IReactorCore, IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket, + IListeningPort, IReactorFDSet, +) +from twisted.internet.abstract import isIPv6Address +from twisted.internet.error import UnsupportedAddressFamily +from twisted.protocols import basic +from twisted.internet import protocol, error, address, task + +from twisted.internet.task import Clock +from twisted.internet.address import IPv4Address, UNIXAddress, IPv6Address +from twisted.logger import ILogObserver + + + +__all__ = [ + 'AccumulatingProtocol', + 'LineSendingProtocol', + 'FakeDatagramTransport', + 'StringTransport', + 'StringTransportWithDisconnection', + 'StringIOWithoutClosing', + '_FakeConnector', + '_FakePort', + 'MemoryReactor', + 'MemoryReactorClock', + 'RaisingMemoryReactor', + 'NonStreamingProducer', + 'waitUntilAllDisconnected', + 'EventLoggingObserver' +] + + + +class AccumulatingProtocol(protocol.Protocol): + """ + L{AccumulatingProtocol} is an L{IProtocol} implementation which collects + the data delivered to it and can fire a Deferred when it is connected or + disconnected. + + @ivar made: A flag indicating whether C{connectionMade} has been called. + @ivar data: Bytes giving all the data passed to C{dataReceived}. + @ivar closed: A flag indicated whether C{connectionLost} has been called. + @ivar closedReason: The value of the I{reason} parameter passed to + C{connectionLost}. + @ivar closedDeferred: If set to a L{Deferred}, this will be fired when + C{connectionLost} is called. + """ + made = closed = 0 + closedReason = None + + closedDeferred = None + + data = b"" + + factory = None + + def connectionMade(self): + self.made = 1 + if (self.factory is not None and self.factory.protocolConnectionMade + is not None): + d = self.factory.protocolConnectionMade + self.factory.protocolConnectionMade = None + d.callback(self) + + def dataReceived(self, data): + self.data += data + + def connectionLost(self, reason): + self.closed = 1 + self.closedReason = reason + if self.closedDeferred is not None: + d, self.closedDeferred = self.closedDeferred, None + d.callback(None) + + + +class LineSendingProtocol(basic.LineReceiver): + lostConn = False + + def __init__(self, lines, start=True): + self.lines = lines[:] + self.response = [] + self.start = start + + def connectionMade(self): + if self.start: + for line in self.lines: + self.sendLine(line) + + def lineReceived(self, line): + if not self.start: + for line in self.lines: + self.sendLine(line) + self.lines = [] + self.response.append(line) + + def connectionLost(self, reason): + self.lostConn = True + + + +class FakeDatagramTransport: + noAddr = object() + + def __init__(self): + self.written = [] + + def write(self, packet, addr=noAddr): + self.written.append((packet, addr)) + + + +@implementer(ITransport, IConsumer, IPushProducer) +class StringTransport: + """ + A transport implementation which buffers data in memory and keeps track of + its other state without providing any behavior. + + L{StringTransport} has a number of attributes which are not part of any of + the interfaces it claims to implement. These attributes are provided for + testing purposes. Implementation code should not use any of these + attributes; they are not provided by other transports. + + @ivar disconnecting: A C{bool} which is C{False} until L{loseConnection} is + called, then C{True}. + + @ivar disconnected: A C{bool} which is C{False} until L{abortConnection} is + called, then C{True}. + + @ivar producer: If a producer is currently registered, C{producer} is a + reference to it. Otherwise, L{None}. + + @ivar streaming: If a producer is currently registered, C{streaming} refers + to the value of the second parameter passed to C{registerProducer}. + + @ivar hostAddr: L{None} or an object which will be returned as the host + address of this transport. If L{None}, a nasty tuple will be returned + instead. + + @ivar peerAddr: L{None} or an object which will be returned as the peer + address of this transport. If L{None}, a nasty tuple will be returned + instead. + + @ivar producerState: The state of this L{StringTransport} in its capacity + as an L{IPushProducer}. One of C{'producing'}, C{'paused'}, or + C{'stopped'}. + + @ivar io: A L{io.BytesIO} which holds the data which has been written to + this transport since the last call to L{clear}. Use L{value} instead + of accessing this directly. + + @ivar _lenient: By default L{StringTransport} enforces that + L{resumeProducing} is not called after the connection is lost. This is + to ensure that any code that does call L{resumeProducing} after the + connection is lost is not blindly expecting L{resumeProducing} to have + any impact. + + However, if your test case is calling L{resumeProducing} after + connection close on purpose, and you know it won't block expecting + further data to show up, this flag may safely be set to L{True}. + + Defaults to L{False}. + @type lenient: L{bool} + """ + + disconnecting = False + disconnected = False + + producer = None + streaming = None + + hostAddr = None + peerAddr = None + + producerState = 'producing' + + def __init__(self, hostAddress=None, peerAddress=None, lenient=False): + self.clear() + if hostAddress is not None: + self.hostAddr = hostAddress + if peerAddress is not None: + self.peerAddr = peerAddress + self.connected = True + self._lenient = lenient + + def clear(self): + """ + Discard all data written to this transport so far. + + This is not a transport method. It is intended for tests. Do not use + it in implementation code. + """ + self.io = BytesIO() + + + def value(self): + """ + Retrieve all data which has been buffered by this transport. + + This is not a transport method. It is intended for tests. Do not use + it in implementation code. + + @return: A C{bytes} giving all data written to this transport since the + last call to L{clear}. + @rtype: C{bytes} + """ + return self.io.getvalue() + + + # ITransport + def write(self, data): + if isinstance(data, unicode): # no, really, I mean it + raise TypeError("Data must not be unicode") + self.io.write(data) + + + def writeSequence(self, data): + self.io.write(b''.join(data)) + + + def loseConnection(self): + """ + Close the connection. Does nothing besides toggle the C{disconnecting} + instance variable to C{True}. + """ + self.disconnecting = True + + + def abortConnection(self): + """ + Abort the connection. Same as C{loseConnection}, but also toggles the + C{aborted} instance variable to C{True}. + """ + self.disconnected = True + self.loseConnection() + + + def getPeer(self): + if self.peerAddr is None: + return address.IPv4Address('TCP', '192.168.1.1', 54321) + return self.peerAddr + + + def getHost(self): + if self.hostAddr is None: + return address.IPv4Address('TCP', '10.0.0.1', 12345) + return self.hostAddr + + + # IConsumer + def registerProducer(self, producer, streaming): + if self.producer is not None: + raise RuntimeError("Cannot register two producers") + self.producer = producer + self.streaming = streaming + + + def unregisterProducer(self): + if self.producer is None: + raise RuntimeError( + "Cannot unregister a producer unless one is registered") + self.producer = None + self.streaming = None + + + # IPushProducer + def _checkState(self): + if self.disconnecting and not self._lenient: + raise RuntimeError( + "Cannot resume producing after loseConnection") + if self.producerState == 'stopped': + raise RuntimeError("Cannot resume a stopped producer") + + + def pauseProducing(self): + self._checkState() + self.producerState = 'paused' + + + def stopProducing(self): + self.producerState = 'stopped' + + + def resumeProducing(self): + self._checkState() + self.producerState = 'producing' + + + +class StringTransportWithDisconnection(StringTransport): + """ + A L{StringTransport} which on disconnection will trigger the connection + lost on the attached protocol. + """ + + def loseConnection(self): + if self.connected: + self.connected = False + self.protocol.connectionLost( + failure.Failure(error.ConnectionDone("Bye."))) + + + +class StringIOWithoutClosing(BytesIO): + """ + A BytesIO that can't be closed. + """ + def close(self): + """ + Do nothing. + """ + + + +@implementer(IListeningPort) +class _FakePort(object): + """ + A fake L{IListeningPort} to be used in tests. + + @ivar _hostAddress: The L{IAddress} this L{IListeningPort} is pretending + to be listening on. + """ + + def __init__(self, hostAddress): + """ + @param hostAddress: An L{IAddress} this L{IListeningPort} should + pretend to be listening on. + """ + self._hostAddress = hostAddress + + + def startListening(self): + """ + Fake L{IListeningPort.startListening} that doesn't do anything. + """ + + + def stopListening(self): + """ + Fake L{IListeningPort.stopListening} that doesn't do anything. + """ + + + def getHost(self): + """ + Fake L{IListeningPort.getHost} that returns our L{IAddress}. + """ + return self._hostAddress + + + +@implementer(IConnector) +class _FakeConnector(object): + """ + A fake L{IConnector} that allows us to inspect if it has been told to stop + connecting. + + @ivar stoppedConnecting: has this connector's + L{_FakeConnector.stopConnecting} method been invoked yet? + + @ivar _address: An L{IAddress} provider that represents our destination. + """ + _disconnected = False + stoppedConnecting = False + + def __init__(self, address): + """ + @param address: An L{IAddress} provider that represents this + connector's destination. + """ + self._address = address + + + def stopConnecting(self): + """ + Implement L{IConnector.stopConnecting} and set + L{_FakeConnector.stoppedConnecting} to C{True} + """ + self.stoppedConnecting = True + + + def disconnect(self): + """ + Implement L{IConnector.disconnect} as a no-op. + """ + self._disconnected = True + + + def connect(self): + """ + Implement L{IConnector.connect} as a no-op. + """ + + + def getDestination(self): + """ + Implement L{IConnector.getDestination} to return the C{address} passed + to C{__init__}. + """ + return self._address + + + +@implementer( + IReactorCore, + IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket, IReactorFDSet +) +class MemoryReactor(object): + """ + A fake reactor to be used in tests. This reactor doesn't actually do + much that's useful yet. It accepts TCP connection setup attempts, but + they will never succeed. + + @ivar hasInstalled: Keeps track of whether this reactor has been installed. + @type hasInstalled: L{bool} + + @ivar running: Keeps track of whether this reactor is running. + @type running: L{bool} + + @ivar hasStopped: Keeps track of whether this reactor has been stopped. + @type hasStopped: L{bool} + + @ivar hasCrashed: Keeps track of whether this reactor has crashed. + @type hasCrashed: L{bool} + + @ivar whenRunningHooks: Keeps track of hooks registered with + C{callWhenRunning}. + @type whenRunningHooks: L{list} + + @ivar triggers: Keeps track of hooks registered with + C{addSystemEventTrigger}. + @type triggers: L{dict} + + @ivar tcpClients: Keeps track of connection attempts (ie, calls to + C{connectTCP}). + @type tcpClients: L{list} + + @ivar tcpServers: Keeps track of server listen attempts (ie, calls to + C{listenTCP}). + @type tcpServers: L{list} + + @ivar sslClients: Keeps track of connection attempts (ie, calls to + C{connectSSL}). + @type sslClients: L{list} + + @ivar sslServers: Keeps track of server listen attempts (ie, calls to + C{listenSSL}). + @type sslServers: L{list} + + @ivar unixClients: Keeps track of connection attempts (ie, calls to + C{connectUNIX}). + @type unixClients: L{list} + + @ivar unixServers: Keeps track of server listen attempts (ie, calls to + C{listenUNIX}). + @type unixServers: L{list} + + @ivar adoptedPorts: Keeps track of server listen attempts (ie, calls to + C{adoptStreamPort}). + + @ivar adoptedStreamConnections: Keeps track of stream-oriented + connections added using C{adoptStreamConnection}. + """ + + def __init__(self): + """ + Initialize the tracking lists. + """ + self.hasInstalled = False + + self.running = False + self.hasRun = True + self.hasStopped = True + self.hasCrashed = True + + self.whenRunningHooks = [] + self.triggers = {} + + self.tcpClients = [] + self.tcpServers = [] + self.sslClients = [] + self.sslServers = [] + self.unixClients = [] + self.unixServers = [] + self.adoptedPorts = [] + self.adoptedStreamConnections = [] + self.connectors = [] + + self.readers = set() + self.writers = set() + + + def install(self): + """ + Fake install callable to emulate reactor module installation. + """ + self.hasInstalled = True + + + def resolve(self, name, timeout=10): + """ + Not implemented; raises L{NotImplementedError}. + """ + raise NotImplementedError() + + + def run(self): + """ + Fake L{IReactorCore.run}. + Sets C{self.running} to L{True}, runs all of the hooks passed to + C{self.callWhenRunning}, then calls C{self.stop} to simulate a request + to stop the reactor. + Sets C{self.hasRun} to L{True}. + """ + assert self.running is False + self.running = True + self.hasRun = True + + for f, args, kwargs in self.whenRunningHooks: + f(*args, **kwargs) + + self.stop() + # That we stopped means we can return, phew. + + + def stop(self): + """ + Fake L{IReactorCore.run}. + Sets C{self.running} to L{False}. + Sets C{self.hasStopped} to L{True}. + """ + self.running = False + self.hasStopped = True + + + def crash(self): + """ + Fake L{IReactorCore.crash}. + Sets C{self.running} to L{None}, because that feels crashy. + Sets C{self.hasCrashed} to L{True}. + """ + self.running = None + self.hasCrashed = True + + + def iterate(self, delay=0): + """ + Not implemented; raises L{NotImplementedError}. + """ + raise NotImplementedError() + + + def fireSystemEvent(self, eventType): + """ + Not implemented; raises L{NotImplementedError}. + """ + raise NotImplementedError() + + + def addSystemEventTrigger(self, phase, eventType, callable, *args, **kw): + """ + Fake L{IReactorCore.run}. + Keep track of trigger by appending it to + self.triggers[phase][eventType]. + """ + phaseTriggers = self.triggers.setdefault(phase, {}) + eventTypeTriggers = phaseTriggers.setdefault(eventType, []) + eventTypeTriggers.append((callable, args, kw)) + + + def removeSystemEventTrigger(self, triggerID): + """ + Not implemented; raises L{NotImplementedError}. + """ + raise NotImplementedError() + + + def callWhenRunning(self, callable, *args, **kw): + """ + Fake L{IReactorCore.callWhenRunning}. + Keeps a list of invocations to make in C{self.whenRunningHooks}. + """ + self.whenRunningHooks.append((callable, args, kw)) + + + def adoptStreamPort(self, fileno, addressFamily, factory): + """ + Fake L{IReactorSocket.adoptStreamPort}, that logs the call and returns + an L{IListeningPort}. + """ + if addressFamily == AF_INET: + addr = IPv4Address('TCP', '0.0.0.0', 1234) + elif addressFamily == AF_INET6: + addr = IPv6Address('TCP', '::', 1234) + else: + raise UnsupportedAddressFamily() + + self.adoptedPorts.append((fileno, addressFamily, factory)) + return _FakePort(addr) + + + def adoptStreamConnection(self, fileDescriptor, addressFamily, factory): + """ + Record the given stream connection in C{adoptedStreamConnections}. + + @see: + L{twisted.internet.interfaces.IReactorSocket.adoptStreamConnection} + """ + self.adoptedStreamConnections.append(( + fileDescriptor, addressFamily, factory)) + + + def adoptDatagramPort(self, fileno, addressFamily, protocol, + maxPacketSize=8192): + """ + Fake L{IReactorSocket.adoptDatagramPort}, that logs the call and + returns a fake L{IListeningPort}. + + @see: L{twisted.internet.interfaces.IReactorSocket.adoptDatagramPort} + """ + if addressFamily == AF_INET: + addr = IPv4Address('UDP', '0.0.0.0', 1234) + elif addressFamily == AF_INET6: + addr = IPv6Address('UDP', '::', 1234) + else: + raise UnsupportedAddressFamily() + + self.adoptedPorts.append( + (fileno, addressFamily, protocol, maxPacketSize)) + return _FakePort(addr) + + + def listenTCP(self, port, factory, backlog=50, interface=''): + """ + Fake L{IReactorTCP.listenTCP}, that logs the call and + returns an L{IListeningPort}. + """ + self.tcpServers.append((port, factory, backlog, interface)) + if isIPv6Address(interface): + address = IPv6Address('TCP', interface, port) + else: + address = IPv4Address('TCP', '0.0.0.0', port) + return _FakePort(address) + + + def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): + """ + Fake L{IReactorTCP.connectTCP}, that logs the call and + returns an L{IConnector}. + """ + self.tcpClients.append((host, port, factory, timeout, bindAddress)) + if isIPv6Address(host): + conn = _FakeConnector(IPv6Address('TCP', host, port)) + else: + conn = _FakeConnector(IPv4Address('TCP', host, port)) + factory.startedConnecting(conn) + self.connectors.append(conn) + return conn + + + def listenSSL(self, port, factory, contextFactory, + backlog=50, interface=''): + """ + Fake L{IReactorSSL.listenSSL}, that logs the call and + returns an L{IListeningPort}. + """ + self.sslServers.append((port, factory, contextFactory, + backlog, interface)) + return _FakePort(IPv4Address('TCP', '0.0.0.0', port)) + + + def connectSSL(self, host, port, factory, contextFactory, + timeout=30, bindAddress=None): + """ + Fake L{IReactorSSL.connectSSL}, that logs the call and returns an + L{IConnector}. + """ + self.sslClients.append((host, port, factory, contextFactory, + timeout, bindAddress)) + conn = _FakeConnector(IPv4Address('TCP', host, port)) + factory.startedConnecting(conn) + self.connectors.append(conn) + return conn + + + def listenUNIX(self, address, factory, + backlog=50, mode=0o666, wantPID=0): + """ + Fake L{IReactorUNIX.listenUNIX}, that logs the call and returns an + L{IListeningPort}. + """ + self.unixServers.append((address, factory, backlog, mode, wantPID)) + return _FakePort(UNIXAddress(address)) + + + def connectUNIX(self, address, factory, timeout=30, checkPID=0): + """ + Fake L{IReactorUNIX.connectUNIX}, that logs the call and returns an + L{IConnector}. + """ + self.unixClients.append((address, factory, timeout, checkPID)) + conn = _FakeConnector(UNIXAddress(address)) + factory.startedConnecting(conn) + self.connectors.append(conn) + return conn + + + def addReader(self, reader): + """ + Fake L{IReactorFDSet.addReader} which adds the reader to a local set. + """ + self.readers.add(reader) + + + def removeReader(self, reader): + """ + Fake L{IReactorFDSet.removeReader} which removes the reader from a + local set. + """ + self.readers.discard(reader) + + + def addWriter(self, writer): + """ + Fake L{IReactorFDSet.addWriter} which adds the writer to a local set. + """ + self.writers.add(writer) + + + def removeWriter(self, writer): + """ + Fake L{IReactorFDSet.removeWriter} which removes the writer from a + local set. + """ + self.writers.discard(writer) + + + def getReaders(self): + """ + Fake L{IReactorFDSet.getReaders} which returns a list of readers from + the local set. + """ + return list(self.readers) + + + def getWriters(self): + """ + Fake L{IReactorFDSet.getWriters} which returns a list of writers from + the local set. + """ + return list(self.writers) + + + def removeAll(self): + """ + Fake L{IReactorFDSet.removeAll} which removed all readers and writers + from the local sets. + """ + self.readers.clear() + self.writers.clear() + + + +for iface in implementedBy(MemoryReactor): + verifyClass(iface, MemoryReactor) + + + +class MemoryReactorClock(MemoryReactor, Clock): + def __init__(self): + MemoryReactor.__init__(self) + Clock.__init__(self) + + + +@implementer(IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket) +class RaisingMemoryReactor(object): + """ + A fake reactor to be used in tests. It accepts TCP connection setup + attempts, but they will fail. + + @ivar _listenException: An instance of an L{Exception} + @ivar _connectException: An instance of an L{Exception} + """ + + def __init__(self, listenException=None, connectException=None): + """ + @param listenException: An instance of an L{Exception} to raise + when any C{listen} method is called. + + @param connectException: An instance of an L{Exception} to raise + when any C{connect} method is called. + """ + self._listenException = listenException + self._connectException = connectException + + + def adoptStreamPort(self, fileno, addressFamily, factory): + """ + Fake L{IReactorSocket.adoptStreamPort}, that raises + L{_listenException}. + """ + raise self._listenException + + + def listenTCP(self, port, factory, backlog=50, interface=''): + """ + Fake L{IReactorTCP.listenTCP}, that raises L{_listenException}. + """ + raise self._listenException + + + def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): + """ + Fake L{IReactorTCP.connectTCP}, that raises L{_connectException}. + """ + raise self._connectException + + + def listenSSL(self, port, factory, contextFactory, + backlog=50, interface=''): + """ + Fake L{IReactorSSL.listenSSL}, that raises L{_listenException}. + """ + raise self._listenException + + + def connectSSL(self, host, port, factory, contextFactory, + timeout=30, bindAddress=None): + """ + Fake L{IReactorSSL.connectSSL}, that raises L{_connectException}. + """ + raise self._connectException + + + def listenUNIX(self, address, factory, + backlog=50, mode=0o666, wantPID=0): + """ + Fake L{IReactorUNIX.listenUNIX}, that raises L{_listenException}. + """ + raise self._listenException + + + def connectUNIX(self, address, factory, timeout=30, checkPID=0): + """ + Fake L{IReactorUNIX.connectUNIX}, that raises L{_connectException}. + """ + raise self._connectException + + + +class NonStreamingProducer(object): + """ + A pull producer which writes 10 times only. + """ + + counter = 0 + stopped = False + + def __init__(self, consumer): + self.consumer = consumer + self.result = Deferred() + + + def resumeProducing(self): + """ + Write the counter value once. + """ + if self.consumer is None or self.counter >= 10: + raise RuntimeError("BUG: resume after unregister/stop.") + else: + self.consumer.write(intToBytes(self.counter)) + self.counter += 1 + if self.counter == 10: + self.consumer.unregisterProducer() + self._done() + + + def pauseProducing(self): + """ + An implementation of C{IPushProducer.pauseProducing}. This should never + be called on a pull producer, so this just raises an error. + """ + raise RuntimeError("BUG: pause should never be called.") + + + def _done(self): + """ + Fire a L{Deferred} so that users can wait for this to complete. + """ + self.consumer = None + d = self.result + del self.result + d.callback(None) + + + def stopProducing(self): + """ + Stop all production. + """ + self.stopped = True + self._done() + + + +def waitUntilAllDisconnected(reactor, protocols): + """ + Take a list of disconnecting protocols, callback a L{Deferred} when they're + all done. + + This is a hack to make some older tests less flaky, as + L{ITransport.loseConnection} is not atomic on all reactors (for example, + the CoreFoundation, which sometimes takes a reactor turn for CFSocket to + realise). New tests should either not use real sockets in testing, or take + the advice in + I{https://jml.io/pages/how-to-disconnect-in-twisted-really.html} to heart. + + @param reactor: The reactor to schedule the checks on. + @type reactor: L{IReactorTime} + + @param protocols: The protocols to wait for disconnecting. + @type protocols: A L{list} of L{IProtocol}s. + """ + lc = None + + def _check(): + if True not in [x.transport.connected for x in protocols]: + lc.stop() + + lc = task.LoopingCall(_check) + lc.clock = reactor + return lc.start(0.01, now=True) + + + +@implementer(ILogObserver) +class EventLoggingObserver(Sequence): + """ + L{ILogObserver} That stores its events in a list for later inspection. + This class is similar to L{LimitedHistoryLogObserver} save that the + internal buffer is public and intended for external inspection. The + observer implements the sequence protocol to ease iteration of the events. + + @ivar _events: The events captured by this observer + @type _events: L{list} + """ + def __init__(self): + self._events = [] + + + def __len__(self): + return len(self._events) + + + def __getitem__(self, index): + return self._events[index] + + + def __iter__(self): + return iter(self._events) + + + def __call__(self, event): + """ + @see: L{ILogObserver} + """ + self._events.append(event) + + + @classmethod + def createWithCleanup(cls, testInstance, publisher): + """ + Create an L{EventLoggingObserver} instance that observes the provided + publisher and will be cleaned up with addCleanup(). + + @param testInstance: Test instance in which this logger is used. + @type testInstance: L{twisted.trial.unittest.TestCase} + + @param publisher: Log publisher to observe. + @type publisher: twisted.logger.LogPublisher + + @return: An EventLoggingObserver configured to observe the provided + publisher. + @rtype: L{twisted.test.proto_helpers.EventLoggingObserver} + """ + obs = cls() + publisher.addObserver(obs) + testInstance.addCleanup(lambda: publisher.removeObserver(obs)) + return obs diff --git a/contrib/python/Twisted/py2/twisted/internet/threads.py b/contrib/python/Twisted/py2/twisted/internet/threads.py new file mode 100644 index 00000000000..8852d002650 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/threads.py @@ -0,0 +1,127 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Extended thread dispatching support. + +For basic support see reactor threading API docs. +""" + +from __future__ import division, absolute_import + +from twisted.python.compat import _PY3 +if not _PY3: + import Queue +else: + import queue as Queue + +from twisted.python import failure +from twisted.internet import defer + + +def deferToThreadPool(reactor, threadpool, f, *args, **kwargs): + """ + Call the function C{f} using a thread from the given threadpool and return + the result as a Deferred. + + This function is only used by client code which is maintaining its own + threadpool. To run a function in the reactor's threadpool, use + C{deferToThread}. + + @param reactor: The reactor in whose main thread the Deferred will be + invoked. + + @param threadpool: An object which supports the C{callInThreadWithCallback} + method of C{twisted.python.threadpool.ThreadPool}. + + @param f: The function to call. + @param *args: positional arguments to pass to f. + @param **kwargs: keyword arguments to pass to f. + + @return: A Deferred which fires a callback with the result of f, or an + errback with a L{twisted.python.failure.Failure} if f throws an + exception. + """ + d = defer.Deferred() + + def onResult(success, result): + if success: + reactor.callFromThread(d.callback, result) + else: + reactor.callFromThread(d.errback, result) + + threadpool.callInThreadWithCallback(onResult, f, *args, **kwargs) + + return d + + +def deferToThread(f, *args, **kwargs): + """ + Run a function in a thread and return the result as a Deferred. + + @param f: The function to call. + @param *args: positional arguments to pass to f. + @param **kwargs: keyword arguments to pass to f. + + @return: A Deferred which fires a callback with the result of f, + or an errback with a L{twisted.python.failure.Failure} if f throws + an exception. + """ + from twisted.internet import reactor + return deferToThreadPool(reactor, reactor.getThreadPool(), + f, *args, **kwargs) + + +def _runMultiple(tupleList): + """ + Run a list of functions. + """ + for f, args, kwargs in tupleList: + f(*args, **kwargs) + + +def callMultipleInThread(tupleList): + """ + Run a list of functions in the same thread. + + tupleList should be a list of (function, argsList, kwargsDict) tuples. + """ + from twisted.internet import reactor + reactor.callInThread(_runMultiple, tupleList) + + +def blockingCallFromThread(reactor, f, *a, **kw): + """ + Run a function in the reactor from a thread, and wait for the result + synchronously. If the function returns a L{Deferred}, wait for its + result and return that. + + @param reactor: The L{IReactorThreads} provider which will be used to + schedule the function call. + @param f: the callable to run in the reactor thread + @type f: any callable. + @param a: the arguments to pass to C{f}. + @param kw: the keyword arguments to pass to C{f}. + + @return: the result of the L{Deferred} returned by C{f}, or the result + of C{f} if it returns anything other than a L{Deferred}. + + @raise: If C{f} raises a synchronous exception, + C{blockingCallFromThread} will raise that exception. If C{f} + returns a L{Deferred} which fires with a L{Failure}, + C{blockingCallFromThread} will raise that failure's exception (see + L{Failure.raiseException}). + """ + queue = Queue.Queue() + def _callFromThread(): + result = defer.maybeDeferred(f, *a, **kw) + result.addBoth(queue.put) + reactor.callFromThread(_callFromThread) + result = queue.get() + if isinstance(result, failure.Failure): + result.raiseException() + return result + + +__all__ = ["deferToThread", "deferToThreadPool", "callMultipleInThread", + "blockingCallFromThread"] diff --git a/contrib/python/Twisted/py2/twisted/internet/tksupport.py b/contrib/python/Twisted/py2/twisted/internet/tksupport.py new file mode 100644 index 00000000000..846cff0f74a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/tksupport.py @@ -0,0 +1,78 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +This module integrates Tkinter with twisted.internet's mainloop. + +Maintainer: Itamar Shtull-Trauring + +To use, do:: + + | tksupport.install(rootWidget) + +and then run your reactor as usual - do *not* call Tk's mainloop(), +use Twisted's regular mechanism for running the event loop. + +Likewise, to stop your program you will need to stop Twisted's +event loop. For example, if you want closing your root widget to +stop Twisted:: + + | root.protocol('WM_DELETE_WINDOW', reactor.stop) + +When using Aqua Tcl/Tk on macOS the standard Quit menu item in +your application might become unresponsive without the additional +fix:: + + | root.createcommand("::tk::mac::Quit", reactor.stop) + +@see: U{Tcl/TkAqua FAQ for more info} +""" + +from twisted.internet import task +from twisted.python.compat import _PY3 + +if _PY3: + import tkinter.simpledialog as tkSimpleDialog + import tkinter.messagebox as tkMessageBox +else: + import tkSimpleDialog, tkMessageBox + + + +_task = None + +def install(widget, ms=10, reactor=None): + """Install a Tkinter.Tk() object into the reactor.""" + installTkFunctions() + global _task + _task = task.LoopingCall(widget.update) + _task.start(ms / 1000.0, False) + +def uninstall(): + """Remove the root Tk widget from the reactor. + + Call this before destroy()ing the root widget. + """ + global _task + _task.stop() + _task = None + + +def installTkFunctions(): + import twisted.python.util + twisted.python.util.getPassword = getPassword + + +def getPassword(prompt = '', confirm = 0): + while 1: + try1 = tkSimpleDialog.askstring('Password Dialog', prompt, show='*') + if not confirm: + return try1 + try2 = tkSimpleDialog.askstring('Password Dialog', 'Confirm Password', show='*') + if try1 == try2: + return try1 + else: + tkMessageBox.showerror('Password Mismatch', 'Passwords did not match, starting over') + +__all__ = ["install", "uninstall"] diff --git a/contrib/python/Twisted/py2/twisted/internet/udp.py b/contrib/python/Twisted/py2/twisted/internet/udp.py new file mode 100644 index 00000000000..c9e79c1edf2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/udp.py @@ -0,0 +1,541 @@ +# -*- test-case-name: twisted.test.test_udp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Various asynchronous UDP classes. + +Please do not use this module directly. + +@var _sockErrReadIgnore: list of symbolic error constants (from the C{errno} + module) representing socket errors where the error is temporary and can be + ignored. + +@var _sockErrReadRefuse: list of symbolic error constants (from the C{errno} + module) representing socket errors that indicate connection refused. +""" + +from __future__ import division, absolute_import + +# System Imports +import socket +import operator +import struct +import warnings + +from zope.interface import implementer + +from twisted.python.runtime import platformType +if platformType == 'win32': + from errno import WSAEWOULDBLOCK + from errno import WSAEINTR, WSAEMSGSIZE, WSAETIMEDOUT + from errno import WSAECONNREFUSED, WSAECONNRESET, WSAENETRESET + from errno import WSAEINPROGRESS + from errno import WSAENOPROTOOPT as ENOPROTOOPT + + # Classify read and write errors + _sockErrReadIgnore = [WSAEINTR, WSAEWOULDBLOCK, WSAEMSGSIZE, WSAEINPROGRESS] + _sockErrReadRefuse = [WSAECONNREFUSED, WSAECONNRESET, WSAENETRESET, + WSAETIMEDOUT] + + # POSIX-compatible write errors + EMSGSIZE = WSAEMSGSIZE + ECONNREFUSED = WSAECONNREFUSED + EAGAIN = WSAEWOULDBLOCK + EINTR = WSAEINTR +else: + from errno import EWOULDBLOCK, EINTR, EMSGSIZE, ECONNREFUSED, EAGAIN + from errno import ENOPROTOOPT + _sockErrReadIgnore = [EAGAIN, EINTR, EWOULDBLOCK] + _sockErrReadRefuse = [ECONNREFUSED] + +# Twisted Imports +from twisted.internet import base, defer, address +from twisted.python import log, failure +from twisted.python._oldstyle import _oldStyle +from twisted.internet import abstract, error, interfaces + + + +@implementer( + interfaces.IListeningPort, interfaces.IUDPTransport, + interfaces.ISystemHandle) +class Port(base.BasePort): + """ + UDP port, listening for packets. + + @ivar maxThroughput: Maximum number of bytes read in one event + loop iteration. + + @ivar addressFamily: L{socket.AF_INET} or L{socket.AF_INET6}, depending on + whether this port is listening on an IPv4 address or an IPv6 address. + + @ivar _realPortNumber: Actual port number being listened on. The + value will be L{None} until this L{Port} is listening. + + @ivar _preexistingSocket: If not L{None}, a L{socket.socket} instance which + was created and initialized outside of the reactor and will be used to + listen for connections (instead of a new socket being created by this + L{Port}). + """ + + addressFamily = socket.AF_INET + socketType = socket.SOCK_DGRAM + maxThroughput = 256 * 1024 + + _realPortNumber = None + _preexistingSocket = None + + def __init__(self, port, proto, interface='', maxPacketSize=8192, reactor=None): + """ + @param port: A port number on which to listen. + @type port: L{int} + + @param proto: A C{DatagramProtocol} instance which will be + connected to the given C{port}. + @type proto: L{twisted.internet.protocol.DatagramProtocol} + + @param interface: The local IPv4 or IPv6 address to which to bind; + defaults to '', ie all IPv4 addresses. + @type interface: L{str} + + @param maxPacketSize: The maximum packet size to accept. + @type maxPacketSize: L{int} + + @param reactor: A reactor which will notify this C{Port} when + its socket is ready for reading or writing. Defaults to + L{None}, ie the default global reactor. + @type reactor: L{interfaces.IReactorFDSet} + """ + base.BasePort.__init__(self, reactor) + self.port = port + self.protocol = proto + self.maxPacketSize = maxPacketSize + self.interface = interface + self.setLogStr() + self._connectedAddr = None + self._setAddressFamily() + + + @classmethod + def _fromListeningDescriptor(cls, reactor, fd, addressFamily, protocol, + maxPacketSize): + """ + Create a new L{Port} based on an existing listening + I{SOCK_DGRAM} socket. + + @param reactor: A reactor which will notify this L{Port} when + its socket is ready for reading or writing. Defaults to + L{None}, ie the default global reactor. + @type reactor: L{interfaces.IReactorFDSet} + + @param fd: An integer file descriptor associated with a listening + socket. The socket must be in non-blocking mode. Any additional + attributes desired, such as I{FD_CLOEXEC}, must also be set already. + @type fd: L{int} + + @param addressFamily: The address family (sometimes called I{domain}) of + the existing socket. For example, L{socket.AF_INET}. + @param addressFamily: L{int} + + @param protocol: A C{DatagramProtocol} instance which will be + connected to the C{port}. + @type proto: L{twisted.internet.protocol.DatagramProtocol} + + @param maxPacketSize: The maximum packet size to accept. + @type maxPacketSize: L{int} + + @return: A new instance of C{cls} wrapping the socket given by C{fd}. + @rtype: L{Port} + """ + port = socket.fromfd(fd, addressFamily, cls.socketType) + interface = port.getsockname()[0] + self = cls(None, protocol, interface=interface, reactor=reactor, + maxPacketSize=maxPacketSize) + self._preexistingSocket = port + return self + + + def __repr__(self): + if self._realPortNumber is not None: + return "<%s on %s>" % (self.protocol.__class__, self._realPortNumber) + else: + return "<%s not connected>" % (self.protocol.__class__,) + + def getHandle(self): + """ + Return a socket object. + """ + return self.socket + + def startListening(self): + """ + Create and bind my socket, and begin listening on it. + + This is called on unserialization, and must be called after creating a + server to begin listening on the specified port. + """ + self._bindSocket() + self._connectToProtocol() + + + def _bindSocket(self): + """ + Prepare and assign a L{socket.socket} instance to + C{self.socket}. + + Either creates a new SOCK_DGRAM L{socket.socket} bound to + C{self.interface} and C{self.port} or takes an existing + L{socket.socket} provided via the + L{interfaces.IReactorSocket.adoptDatagramPort} interface. + """ + if self._preexistingSocket is None: + # Create a new socket and make it listen + try: + skt = self.createInternetSocket() + skt.bind((self.interface, self.port)) + except socket.error as le: + raise error.CannotListenError(self.interface, self.port, le) + else: + # Re-use the externally specified socket + skt = self._preexistingSocket + self._preexistingSocket = None + + # Make sure that if we listened on port 0, we update that to + # reflect what the OS actually assigned us. + self._realPortNumber = skt.getsockname()[1] + + log.msg("%s starting on %s" % ( + self._getLogPrefix(self.protocol), self._realPortNumber)) + + self.connected = 1 + self.socket = skt + self.fileno = self.socket.fileno + + + def _connectToProtocol(self): + self.protocol.makeConnection(self) + self.startReading() + + + def doRead(self): + """ + Called when my socket is ready for reading. + """ + read = 0 + while read < self.maxThroughput: + try: + data, addr = self.socket.recvfrom(self.maxPacketSize) + except socket.error as se: + no = se.args[0] + if no in _sockErrReadIgnore: + return + if no in _sockErrReadRefuse: + if self._connectedAddr: + self.protocol.connectionRefused() + return + raise + else: + read += len(data) + if self.addressFamily == socket.AF_INET6: + # Remove the flow and scope ID from the address tuple, + # reducing it to a tuple of just (host, port). + # + # TODO: This should be amended to return an object that can + # unpack to (host, port) but also includes the flow info + # and scope ID. See http://tm.tl/6826 + addr = addr[:2] + try: + self.protocol.datagramReceived(data, addr) + except: + log.err() + + + def write(self, datagram, addr=None): + """ + Write a datagram. + + @type datagram: L{bytes} + @param datagram: The datagram to be sent. + + @type addr: L{tuple} containing L{str} as first element and L{int} as + second element, or L{None} + @param addr: A tuple of (I{stringified IPv4 or IPv6 address}, + I{integer port number}); can be L{None} in connected mode. + """ + if self._connectedAddr: + assert addr in (None, self._connectedAddr) + try: + return self.socket.send(datagram) + except socket.error as se: + no = se.args[0] + if no == EINTR: + return self.write(datagram) + elif no == EMSGSIZE: + raise error.MessageLengthError("message too long") + elif no == ECONNREFUSED: + self.protocol.connectionRefused() + else: + raise + else: + assert addr != None + if (not abstract.isIPAddress(addr[0]) + and not abstract.isIPv6Address(addr[0]) + and addr[0] != ""): + raise error.InvalidAddressError( + addr[0], + "write() only accepts IP addresses, not hostnames") + if ((abstract.isIPAddress(addr[0]) or addr[0] == "") + and self.addressFamily == socket.AF_INET6): + raise error.InvalidAddressError( + addr[0], + "IPv6 port write() called with IPv4 or broadcast address") + if (abstract.isIPv6Address(addr[0]) + and self.addressFamily == socket.AF_INET): + raise error.InvalidAddressError( + addr[0], "IPv4 port write() called with IPv6 address") + try: + return self.socket.sendto(datagram, addr) + except socket.error as se: + no = se.args[0] + if no == EINTR: + return self.write(datagram, addr) + elif no == EMSGSIZE: + raise error.MessageLengthError("message too long") + elif no == ECONNREFUSED: + # in non-connected UDP ECONNREFUSED is platform dependent, I + # think and the info is not necessarily useful. Nevertheless + # maybe we should call connectionRefused? XXX + return + else: + raise + + + def writeSequence(self, seq, addr): + """ + Write a datagram constructed from an iterable of L{bytes}. + + @param seq: The data that will make up the complete datagram to be + written. + @type seq: an iterable of L{bytes} + + @type addr: L{tuple} containing L{str} as first element and L{int} as + second element, or L{None} + @param addr: A tuple of (I{stringified IPv4 or IPv6 address}, + I{integer port number}); can be L{None} in connected mode. + """ + self.write(b"".join(seq), addr) + + + def connect(self, host, port): + """ + 'Connect' to remote server. + """ + if self._connectedAddr: + raise RuntimeError("already connected, reconnecting is not currently supported") + if not abstract.isIPAddress(host) and not abstract.isIPv6Address(host): + raise error.InvalidAddressError( + host, 'not an IPv4 or IPv6 address.') + self._connectedAddr = (host, port) + self.socket.connect((host, port)) + + + def _loseConnection(self): + self.stopReading() + if self.connected: # actually means if we are *listening* + self.reactor.callLater(0, self.connectionLost) + + + def stopListening(self): + if self.connected: + result = self.d = defer.Deferred() + else: + result = None + self._loseConnection() + return result + + + def loseConnection(self): + warnings.warn("Please use stopListening() to disconnect port", DeprecationWarning, stacklevel=2) + self.stopListening() + + + def connectionLost(self, reason=None): + """ + Cleans up my socket. + """ + log.msg('(UDP Port %s Closed)' % self._realPortNumber) + self._realPortNumber = None + self.maxThroughput = -1 + base.BasePort.connectionLost(self, reason) + self.protocol.doStop() + self.socket.close() + del self.socket + del self.fileno + if hasattr(self, "d"): + self.d.callback(None) + del self.d + + + def setLogStr(self): + """ + Initialize the C{logstr} attribute to be used by C{logPrefix}. + """ + logPrefix = self._getLogPrefix(self.protocol) + self.logstr = "%s (UDP)" % logPrefix + + + def _setAddressFamily(self): + """ + Resolve address family for the socket. + """ + if abstract.isIPv6Address(self.interface): + self.addressFamily = socket.AF_INET6 + elif abstract.isIPAddress(self.interface): + self.addressFamily = socket.AF_INET + elif self.interface: + raise error.InvalidAddressError( + self.interface, 'not an IPv4 or IPv6 address.') + + + def logPrefix(self): + """ + Return the prefix to log with. + """ + return self.logstr + + + def getHost(self): + """ + Return the local address of the UDP connection + + @returns: the local address of the UDP connection + @rtype: L{IPv4Address} or L{IPv6Address} + """ + addr = self.socket.getsockname() + if self.addressFamily == socket.AF_INET: + return address.IPv4Address('UDP', *addr) + elif self.addressFamily == socket.AF_INET6: + return address.IPv6Address('UDP', *(addr[:2])) + + + def setBroadcastAllowed(self, enabled): + """ + Set whether this port may broadcast. This is disabled by default. + + @param enabled: Whether the port may broadcast. + @type enabled: L{bool} + """ + self.socket.setsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST, enabled) + + + def getBroadcastAllowed(self): + """ + Checks if broadcast is currently allowed on this port. + + @return: Whether this port may broadcast. + @rtype: L{bool} + """ + return operator.truth( + self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST)) + + + +@_oldStyle +class MulticastMixin: + """ + Implement multicast functionality. + """ + + def getOutgoingInterface(self): + i = self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF) + return socket.inet_ntoa(struct.pack("@i", i)) + + + def setOutgoingInterface(self, addr): + """Returns Deferred of success.""" + return self.reactor.resolve(addr).addCallback(self._setInterface) + + + def _setInterface(self, addr): + i = socket.inet_aton(addr) + self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, i) + return 1 + + + def getLoopbackMode(self): + return self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP) + + + def setLoopbackMode(self, mode): + mode = struct.pack("b", operator.truth(mode)) + self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, mode) + + + def getTTL(self): + return self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL) + + + def setTTL(self, ttl): + ttl = struct.pack("B", ttl) + self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl) + + + def joinGroup(self, addr, interface=""): + """Join a multicast group. Returns Deferred of success.""" + return self.reactor.resolve(addr).addCallback(self._joinAddr1, interface, 1) + + + def _joinAddr1(self, addr, interface, join): + return self.reactor.resolve(interface).addCallback(self._joinAddr2, addr, join) + + + def _joinAddr2(self, interface, addr, join): + addr = socket.inet_aton(addr) + interface = socket.inet_aton(interface) + if join: + cmd = socket.IP_ADD_MEMBERSHIP + else: + cmd = socket.IP_DROP_MEMBERSHIP + try: + self.socket.setsockopt(socket.IPPROTO_IP, cmd, addr + interface) + except socket.error as e: + return failure.Failure(error.MulticastJoinError(addr, interface, *e.args)) + + + def leaveGroup(self, addr, interface=""): + """Leave multicast group, return Deferred of success.""" + return self.reactor.resolve(addr).addCallback(self._joinAddr1, interface, 0) + + + +@implementer(interfaces.IMulticastTransport) +class MulticastPort(MulticastMixin, Port): + """ + UDP Port that supports multicasting. + """ + + def __init__(self, port, proto, interface='', maxPacketSize=8192, + reactor=None, listenMultiple=False): + """ + @see: L{twisted.internet.interfaces.IReactorMulticast.listenMulticast} + """ + Port.__init__(self, port, proto, interface, maxPacketSize, reactor) + self.listenMultiple = listenMultiple + + + def createInternetSocket(self): + skt = Port.createInternetSocket(self) + if self.listenMultiple: + skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, "SO_REUSEPORT"): + try: + skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + except socket.error as le: + # RHEL6 defines SO_REUSEPORT but it doesn't work + if le.errno == ENOPROTOOPT: + pass + else: + raise + return skt diff --git a/contrib/python/Twisted/py2/twisted/internet/unix.py b/contrib/python/Twisted/py2/twisted/internet/unix.py new file mode 100644 index 00000000000..45553d6cccc --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/unix.py @@ -0,0 +1,624 @@ +# -*- test-case-name: twisted.test.test_unix,twisted.internet.test.test_unix,twisted.internet.test.test_posixbase -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +UNIX socket support for Twisted. + +End users shouldn't use this module directly - use the reactor APIs instead. + +Maintainer: Itamar Shtull-Trauring +""" + +from __future__ import division, absolute_import + +import os +import stat +import socket +import struct +from errno import EINTR, EMSGSIZE, EAGAIN, EWOULDBLOCK, ECONNREFUSED, ENOBUFS + +from zope.interface import implementer, implementer_only, implementedBy + +if not hasattr(socket, 'AF_UNIX'): + raise ImportError("UNIX sockets not supported on this platform") + +from twisted.internet import main, base, tcp, udp, error, interfaces +from twisted.internet import protocol, address +from twisted.python import lockfile, log, reflect, failure +from twisted.python.filepath import _coerceToFilesystemEncoding +from twisted.python.util import untilConcludes +from twisted.python.compat import lazyByteSlice + + +try: + from twisted.python import sendmsg +except ImportError: + sendmsg = None + + + +def _ancillaryDescriptor(fd): + """ + Pack an integer into an ancillary data structure suitable for use with + L{sendmsg.sendmsg}. + """ + packed = struct.pack("i", fd) + return [(socket.SOL_SOCKET, sendmsg.SCM_RIGHTS, packed)] + + + +@implementer(interfaces.IUNIXTransport) +class _SendmsgMixin(object): + """ + Mixin for stream-oriented UNIX transports which uses sendmsg and recvmsg to + offer additional functionality, such as copying file descriptors into other + processes. + + @ivar _writeSomeDataBase: The class which provides the basic implementation + of C{writeSomeData}. Ultimately this should be a subclass of + L{twisted.internet.abstract.FileDescriptor}. Subclasses which mix in + L{_SendmsgMixin} must define this. + + @ivar _sendmsgQueue: A C{list} of C{int} holding file descriptors which are + currently buffered before being sent. + + @ivar _fileDescriptorBufferSize: An C{int} giving the maximum number of file + descriptors to accept and queue for sending before pausing the + registered producer, if there is one. + """ + + _writeSomeDataBase = None + _fileDescriptorBufferSize = 64 + + def __init__(self): + self._sendmsgQueue = [] + + + def _isSendBufferFull(self): + """ + Determine whether the user-space send buffer for this transport is full + or not. + + This extends the base determination by adding consideration of how many + file descriptors need to be sent using L{sendmsg.sendmsg}. When there + are more than C{self._fileDescriptorBufferSize}, the buffer is + considered full. + + @return: C{True} if it is full, C{False} otherwise. + """ + # There must be some bytes in the normal send buffer, checked by + # _writeSomeDataBase._isSendBufferFull, in order to send file + # descriptors from _sendmsgQueue. That means that the buffer will + # eventually be considered full even without this additional logic. + # However, since we send only one byte per file descriptor, having lots + # of elements in _sendmsgQueue incurs more overhead and perhaps slows + # things down. Anyway, try this for now, maybe rethink it later. + return ( + len(self._sendmsgQueue) > self._fileDescriptorBufferSize + or self._writeSomeDataBase._isSendBufferFull(self)) + + + def sendFileDescriptor(self, fileno): + """ + Queue the given file descriptor to be sent and start trying to send it. + """ + self._sendmsgQueue.append(fileno) + self._maybePauseProducer() + self.startWriting() + + + def writeSomeData(self, data): + """ + Send as much of C{data} as possible. Also send any pending file + descriptors. + """ + # Make it a programming error to send more file descriptors than you + # send regular bytes. Otherwise, due to the limitation mentioned + # below, we could end up with file descriptors left, but no bytes to + # send with them, therefore no way to send those file descriptors. + if len(self._sendmsgQueue) > len(data): + return error.FileDescriptorOverrun() + + # If there are file descriptors to send, try sending them first, using + # a little bit of data from the stream-oriented write buffer too. It + # is not possible to send a file descriptor without sending some + # regular data. + index = 0 + try: + while index < len(self._sendmsgQueue): + fd = self._sendmsgQueue[index] + try: + untilConcludes( + sendmsg.sendmsg, self.socket, data[index:index+1], + _ancillaryDescriptor(fd)) + except socket.error as se: + if se.args[0] in (EWOULDBLOCK, ENOBUFS): + return index + else: + return main.CONNECTION_LOST + else: + index += 1 + finally: + del self._sendmsgQueue[:index] + + # Hand the remaining data to the base implementation. Avoid slicing in + # favor of a buffer, in case that happens to be any faster. + limitedData = lazyByteSlice(data, index) + result = self._writeSomeDataBase.writeSomeData(self, limitedData) + try: + return index + result + except TypeError: + return result + + + def doRead(self): + """ + Calls {IProtocol.dataReceived} with all available data and + L{IFileDescriptorReceiver.fileDescriptorReceived} once for each + received file descriptor in ancillary data. + + This reads up to C{self.bufferSize} bytes of data from its socket, then + dispatches the data to protocol callbacks to be handled. If the + connection is not lost through an error in the underlying recvmsg(), + this function will return the result of the dataReceived call. + """ + try: + data, ancillary, flags = untilConcludes( + sendmsg.recvmsg, self.socket, self.bufferSize) + except socket.error as se: + if se.args[0] == EWOULDBLOCK: + return + else: + return main.CONNECTION_LOST + + for cmsgLevel, cmsgType, cmsgData in ancillary: + if (cmsgLevel == socket.SOL_SOCKET and + cmsgType == sendmsg.SCM_RIGHTS): + self._ancillaryLevelSOLSOCKETTypeSCMRIGHTS(cmsgData) + else: + log.msg( + format=( + "%(protocolName)s (on %(hostAddress)r) " + "received unsupported ancillary data " + "(level=%(cmsgLevel)r, type=%(cmsgType)r) " + "from %(peerAddress)r."), + hostAddress=self.getHost(), peerAddress=self.getPeer(), + protocolName=self._getLogPrefix(self.protocol), + cmsgLevel=cmsgLevel, cmsgType=cmsgType, + ) + + return self._dataReceived(data) + + + def _ancillaryLevelSOLSOCKETTypeSCMRIGHTS(self, cmsgData): + """ + Processes ancillary data with level SOL_SOCKET and type SCM_RIGHTS, + indicating that the ancillary data payload holds file descriptors. + + Calls L{IFileDescriptorReceiver.fileDescriptorReceived} once for each + received file descriptor or logs a message if the protocol does not + implement L{IFileDescriptorReceiver}. + + @param cmsgData: Ancillary data payload. + @type cmsgData: L{bytes} + """ + + fdCount = len(cmsgData) // 4 + fds = struct.unpack('i'*fdCount, cmsgData) + if interfaces.IFileDescriptorReceiver.providedBy(self.protocol): + for fd in fds: + self.protocol.fileDescriptorReceived(fd) + else: + log.msg( + format=( + "%(protocolName)s (on %(hostAddress)r) does not " + "provide IFileDescriptorReceiver; closing file " + "descriptor received (from %(peerAddress)r)."), + hostAddress=self.getHost(), peerAddress=self.getPeer(), + protocolName=self._getLogPrefix(self.protocol), + ) + for fd in fds: + os.close(fd) + + +class _UnsupportedSendmsgMixin(object): + """ + Behaviorless placeholder used when C{twisted.python.sendmsg} is not + available, preventing L{IUNIXTransport} from being supported. + """ + + + +if sendmsg: + _SendmsgMixin = _SendmsgMixin +else: + _SendmsgMixin = _UnsupportedSendmsgMixin + + + +class Server(_SendmsgMixin, tcp.Server): + + _writeSomeDataBase = tcp.Server + + def __init__(self, sock, protocol, client, server, sessionno, reactor): + _SendmsgMixin.__init__(self) + tcp.Server.__init__(self, sock, protocol, (client, None), server, sessionno, reactor) + + @classmethod + def _fromConnectedSocket(cls, fileDescriptor, factory, reactor): + """ + Create a new L{Server} based on an existing connected I{SOCK_STREAM} + socket. + + Arguments are the same as to L{Server.__init__}, except where noted. + + @param fileDescriptor: An integer file descriptor associated with a + connected socket. The socket must be in non-blocking mode. Any + additional attributes desired, such as I{FD_CLOEXEC}, must also be + set already. + + @return: A new instance of C{cls} wrapping the socket given by + C{fileDescriptor}. + """ + skt = socket.fromfd(fileDescriptor, socket.AF_UNIX, socket.SOCK_STREAM) + protocolAddr = address.UNIXAddress(skt.getsockname()) + + proto = factory.buildProtocol(protocolAddr) + if proto is None: + skt.close() + return + + # FIXME: is this a suitable sessionno? + sessionno = 0 + self = cls(skt, proto, skt.getpeername(), None, sessionno, reactor) + self.repstr = "<%s #%s on %s>" % ( + self.protocol.__class__.__name__, self.sessionno, skt.getsockname()) + self.logstr = "%s,%s,%s" % ( + self.protocol.__class__.__name__, self.sessionno, skt.getsockname()) + proto.makeConnection(self) + return self + + def getHost(self): + return address.UNIXAddress(self.socket.getsockname()) + + def getPeer(self): + return address.UNIXAddress(self.hostname or None) + + + +def _inFilesystemNamespace(path): + """ + Determine whether the given unix socket path is in a filesystem namespace. + + While most PF_UNIX sockets are entries in the filesystem, Linux 2.2 and + above support PF_UNIX sockets in an "abstract namespace" that does not + correspond to any path. This function returns C{True} if the given socket + path is stored in the filesystem and C{False} if the path is in this + abstract namespace. + """ + return path[:1] not in (b"\0", u"\0") + + +class _UNIXPort(object): + def getHost(self): + """ + Returns a UNIXAddress. + + This indicates the server's address. + """ + return address.UNIXAddress(self.socket.getsockname()) + + + +class Port(_UNIXPort, tcp.Port): + addressFamily = socket.AF_UNIX + socketType = socket.SOCK_STREAM + + transport = Server + lockFile = None + + def __init__(self, fileName, factory, backlog=50, mode=0o666, reactor=None, + wantPID = 0): + tcp.Port.__init__(self, self._buildAddr(fileName).name, factory, + backlog, reactor=reactor) + self.mode = mode + self.wantPID = wantPID + self._preexistingSocket = None + + @classmethod + def _fromListeningDescriptor(cls, reactor, fd, factory): + """ + Create a new L{Port} based on an existing listening I{SOCK_STREAM} + socket. + + Arguments are the same as to L{Port.__init__}, except where noted. + + @param fd: An integer file descriptor associated with a listening + socket. The socket must be in non-blocking mode. Any additional + attributes desired, such as I{FD_CLOEXEC}, must also be set already. + + @return: A new instance of C{cls} wrapping the socket given by C{fd}. + """ + port = socket.fromfd(fd, cls.addressFamily, cls.socketType) + self = cls(port.getsockname(), factory, reactor=reactor) + self._preexistingSocket = port + return self + + def __repr__(self): + factoryName = reflect.qual(self.factory.__class__) + if hasattr(self, 'socket'): + return '<%s on %r>' % ( + factoryName, _coerceToFilesystemEncoding('', self.port)) + else: + return '<%s (not listening)>' % (factoryName,) + + def _buildAddr(self, name): + return address.UNIXAddress(name) + + def startListening(self): + """ + Create and bind my socket, and begin listening on it. + + This is called on unserialization, and must be called after creating a + server to begin listening on the specified port. + """ + tcp._reservedFD.reserve() + log.msg("%s starting on %r" % ( + self._getLogPrefix(self.factory), + _coerceToFilesystemEncoding('', self.port))) + if self.wantPID: + self.lockFile = lockfile.FilesystemLock(self.port + b".lock") + if not self.lockFile.lock(): + raise error.CannotListenError(None, self.port, + "Cannot acquire lock") + else: + if not self.lockFile.clean: + try: + # This is a best-attempt at cleaning up + # left-over unix sockets on the filesystem. + # If it fails, there's not much else we can + # do. The bind() below will fail with an + # exception that actually propagates. + if stat.S_ISSOCK(os.stat(self.port).st_mode): + os.remove(self.port) + except: + pass + + self.factory.doStart() + + try: + if self._preexistingSocket is not None: + skt = self._preexistingSocket + self._preexistingSocket = None + else: + skt = self.createInternetSocket() + skt.bind(self.port) + except socket.error as le: + raise error.CannotListenError(None, self.port, le) + else: + if _inFilesystemNamespace(self.port): + # Make the socket readable and writable to the world. + os.chmod(self.port, self.mode) + skt.listen(self.backlog) + self.connected = True + self.socket = skt + self.fileno = self.socket.fileno + self.numberAccepts = 100 + self.startReading() + + + def _logConnectionLostMsg(self): + """ + Log message for closing socket + """ + log.msg('(UNIX Port %s Closed)' % ( + _coerceToFilesystemEncoding('', self.port,))) + + + def connectionLost(self, reason): + if _inFilesystemNamespace(self.port): + os.unlink(self.port) + if self.lockFile is not None: + self.lockFile.unlock() + tcp.Port.connectionLost(self, reason) + + + +class Client(_SendmsgMixin, tcp.BaseClient): + """A client for Unix sockets.""" + addressFamily = socket.AF_UNIX + socketType = socket.SOCK_STREAM + _writeSomeDataBase = tcp.BaseClient + + def __init__(self, filename, connector, reactor=None, checkPID = 0): + _SendmsgMixin.__init__(self) + # Normalise the filename using UNIXAddress + filename = address.UNIXAddress(filename).name + self.connector = connector + self.realAddress = self.addr = filename + if checkPID and not lockfile.isLocked(filename + b".lock"): + self._finishInit(None, None, error.BadFileError(filename), reactor) + self._finishInit(self.doConnect, self.createInternetSocket(), + None, reactor) + + def getPeer(self): + return address.UNIXAddress(self.addr) + + def getHost(self): + return address.UNIXAddress(None) + + +class Connector(base.BaseConnector): + def __init__(self, address, factory, timeout, reactor, checkPID): + base.BaseConnector.__init__(self, factory, timeout, reactor) + self.address = address + self.checkPID = checkPID + + def _makeTransport(self): + return Client(self.address, self, self.reactor, self.checkPID) + + def getDestination(self): + return address.UNIXAddress(self.address) + + +@implementer(interfaces.IUNIXDatagramTransport) +class DatagramPort(_UNIXPort, udp.Port): + """ + Datagram UNIX port, listening for packets. + """ + + addressFamily = socket.AF_UNIX + + def __init__(self, addr, proto, maxPacketSize=8192, mode=0o666, reactor=None): + """Initialize with address to listen on. + """ + udp.Port.__init__(self, addr, proto, maxPacketSize=maxPacketSize, reactor=reactor) + self.mode = mode + + + def __repr__(self): + protocolName = reflect.qual(self.protocol.__class__,) + if hasattr(self, 'socket'): + return '<%s on %r>' % (protocolName, self.port) + else: + return '<%s (not listening)>' % (protocolName,) + + + def _bindSocket(self): + log.msg("%s starting on %s"%(self.protocol.__class__, repr(self.port))) + try: + skt = self.createInternetSocket() # XXX: haha misnamed method + if self.port: + skt.bind(self.port) + except socket.error as le: + raise error.CannotListenError(None, self.port, le) + if self.port and _inFilesystemNamespace(self.port): + # Make the socket readable and writable to the world. + os.chmod(self.port, self.mode) + self.connected = 1 + self.socket = skt + self.fileno = self.socket.fileno + + def write(self, datagram, address): + """Write a datagram.""" + try: + return self.socket.sendto(datagram, address) + except socket.error as se: + no = se.args[0] + if no == EINTR: + return self.write(datagram, address) + elif no == EMSGSIZE: + raise error.MessageLengthError("message too long") + elif no == EAGAIN: + # oh, well, drop the data. The only difference from UDP + # is that UDP won't ever notice. + # TODO: add TCP-like buffering + pass + else: + raise + + def connectionLost(self, reason=None): + """Cleans up my socket. + """ + log.msg('(Port %s Closed)' % repr(self.port)) + base.BasePort.connectionLost(self, reason) + if hasattr(self, "protocol"): + # we won't have attribute in ConnectedPort, in cases + # where there was an error in connection process + self.protocol.doStop() + self.connected = 0 + self.socket.close() + del self.socket + del self.fileno + if hasattr(self, "d"): + self.d.callback(None) + del self.d + + def setLogStr(self): + self.logstr = reflect.qual(self.protocol.__class__) + " (UDP)" + + + +@implementer_only(interfaces.IUNIXDatagramConnectedTransport, + *(implementedBy(base.BasePort))) +class ConnectedDatagramPort(DatagramPort): + """ + A connected datagram UNIX socket. + """ + def __init__(self, addr, proto, maxPacketSize=8192, mode=0o666, + bindAddress=None, reactor=None): + assert isinstance(proto, protocol.ConnectedDatagramProtocol) + DatagramPort.__init__(self, bindAddress, proto, maxPacketSize, mode, + reactor) + self.remoteaddr = addr + + + def startListening(self): + try: + self._bindSocket() + self.socket.connect(self.remoteaddr) + self._connectToProtocol() + except: + self.connectionFailed(failure.Failure()) + + + def connectionFailed(self, reason): + """ + Called when a connection fails. Stop listening on the socket. + + @type reason: L{Failure} + @param reason: Why the connection failed. + """ + self.stopListening() + self.protocol.connectionFailed(reason) + del self.protocol + + + def doRead(self): + """ + Called when my socket is ready for reading. + """ + read = 0 + while read < self.maxThroughput: + try: + data, addr = self.socket.recvfrom(self.maxPacketSize) + read += len(data) + self.protocol.datagramReceived(data) + except socket.error as se: + no = se.args[0] + if no in (EAGAIN, EINTR, EWOULDBLOCK): + return + if no == ECONNREFUSED: + self.protocol.connectionRefused() + else: + raise + except: + log.deferr() + + + def write(self, data): + """ + Write a datagram. + """ + try: + return self.socket.send(data) + except socket.error as se: + no = se.args[0] + if no == EINTR: + return self.write(data) + elif no == EMSGSIZE: + raise error.MessageLengthError("message too long") + elif no == ECONNREFUSED: + self.protocol.connectionRefused() + elif no == EAGAIN: + # oh, well, drop the data. The only difference from UDP + # is that UDP won't ever notice. + # TODO: add TCP-like buffering + pass + else: + raise + + + def getPeer(self): + return address.UNIXAddress(self.remoteaddr) diff --git a/contrib/python/Twisted/py2/twisted/internet/utils.py b/contrib/python/Twisted/py2/twisted/internet/utils.py new file mode 100644 index 00000000000..100fca60236 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/utils.py @@ -0,0 +1,245 @@ +# -*- test-case-name: twisted.test.test_iutils -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Utility methods. +""" + +from __future__ import division, absolute_import + +import sys, warnings +from functools import wraps + +from twisted.internet import protocol, defer +from twisted.python import failure +from twisted.python.compat import reraise + +from io import BytesIO + + + +def _callProtocolWithDeferred(protocol, executable, args, env, path, + reactor=None, protoArgs=()): + if reactor is None: + from twisted.internet import reactor + + d = defer.Deferred() + p = protocol(d, *protoArgs) + reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path) + return d + + + +class _UnexpectedErrorOutput(IOError): + """ + Standard error data was received where it was not expected. This is a + subclass of L{IOError} to preserve backward compatibility with the previous + error behavior of L{getProcessOutput}. + + @ivar processEnded: A L{Deferred} which will fire when the process which + produced the data on stderr has ended (exited and all file descriptors + closed). + """ + + def __init__(self, text, processEnded): + IOError.__init__(self, "got stderr: %r" % (text,)) + self.processEnded = processEnded + + + +class _BackRelay(protocol.ProcessProtocol): + """ + Trivial protocol for communicating with a process and turning its output + into the result of a L{Deferred}. + + @ivar deferred: A L{Deferred} which will be called back with all of stdout + and, if C{errortoo} is true, all of stderr as well (mixed together in + one string). If C{errortoo} is false and any bytes are received over + stderr, this will fire with an L{_UnexpectedErrorOutput} instance and + the attribute will be set to L{None}. + + @ivar onProcessEnded: If C{errortoo} is false and bytes are received over + stderr, this attribute will refer to a L{Deferred} which will be called + back when the process ends. This C{Deferred} is also associated with + the L{_UnexpectedErrorOutput} which C{deferred} fires with earlier in + this case so that users can determine when the process has actually + ended, in addition to knowing when bytes have been received via stderr. + """ + + def __init__(self, deferred, errortoo=0): + self.deferred = deferred + self.s = BytesIO() + if errortoo: + self.errReceived = self.errReceivedIsGood + else: + self.errReceived = self.errReceivedIsBad + + def errReceivedIsBad(self, text): + if self.deferred is not None: + self.onProcessEnded = defer.Deferred() + err = _UnexpectedErrorOutput(text, self.onProcessEnded) + self.deferred.errback(failure.Failure(err)) + self.deferred = None + self.transport.loseConnection() + + def errReceivedIsGood(self, text): + self.s.write(text) + + def outReceived(self, text): + self.s.write(text) + + def processEnded(self, reason): + if self.deferred is not None: + self.deferred.callback(self.s.getvalue()) + elif self.onProcessEnded is not None: + self.onProcessEnded.errback(reason) + + + +def getProcessOutput(executable, args=(), env={}, path=None, reactor=None, + errortoo=0): + """ + Spawn a process and return its output as a deferred returning a L{bytes}. + + @param executable: The file name to run and get the output of - the + full path should be used. + + @param args: the command line arguments to pass to the process; a + sequence of strings. The first string should B{NOT} be the + executable's name. + + @param env: the environment variables to pass to the process; a + dictionary of strings. + + @param path: the path to run the subprocess in - defaults to the + current directory. + + @param reactor: the reactor to use - defaults to the default reactor + + @param errortoo: If true, include stderr in the result. If false, if + stderr is received the returned L{Deferred} will errback with an + L{IOError} instance with a C{processEnded} attribute. The + C{processEnded} attribute refers to a L{Deferred} which fires when the + executed process ends. + """ + return _callProtocolWithDeferred(lambda d: + _BackRelay(d, errortoo=errortoo), + executable, args, env, path, + reactor) + + +class _ValueGetter(protocol.ProcessProtocol): + + def __init__(self, deferred): + self.deferred = deferred + + def processEnded(self, reason): + self.deferred.callback(reason.value.exitCode) + + + +def getProcessValue(executable, args=(), env={}, path=None, reactor=None): + """Spawn a process and return its exit code as a Deferred.""" + return _callProtocolWithDeferred(_ValueGetter, executable, args, env, path, + reactor) + + + +class _EverythingGetter(protocol.ProcessProtocol): + + def __init__(self, deferred, stdinBytes=None): + self.deferred = deferred + self.outBuf = BytesIO() + self.errBuf = BytesIO() + self.outReceived = self.outBuf.write + self.errReceived = self.errBuf.write + self.stdinBytes = stdinBytes + + def connectionMade(self): + if self.stdinBytes is not None: + self.transport.writeToChild(0, self.stdinBytes) + # The only compelling reason not to _always_ close stdin here is + # backwards compatibility. + self.transport.closeStdin() + + def processEnded(self, reason): + out = self.outBuf.getvalue() + err = self.errBuf.getvalue() + e = reason.value + code = e.exitCode + if e.signal: + self.deferred.errback((out, err, e.signal)) + else: + self.deferred.callback((out, err, code)) + + + +def getProcessOutputAndValue(executable, args=(), env={}, path=None, + reactor=None, stdinBytes=None): + """Spawn a process and returns a Deferred that will be called back with + its output (from stdout and stderr) and it's exit code as (out, err, code) + If a signal is raised, the Deferred will errback with the stdout and + stderr up to that point, along with the signal, as (out, err, signalNum) + """ + return _callProtocolWithDeferred( + _EverythingGetter, + executable, + args, + env, + path, + reactor, + protoArgs=(stdinBytes,), + ) + + + +def _resetWarningFilters(passthrough, addedFilters): + for f in addedFilters: + try: + warnings.filters.remove(f) + except ValueError: + pass + return passthrough + + +def runWithWarningsSuppressed(suppressedWarnings, f, *a, **kw): + """Run the function C{f}, but with some warnings suppressed. + + @param suppressedWarnings: A list of arguments to pass to filterwarnings. + Must be a sequence of 2-tuples (args, kwargs). + @param f: A callable, followed by its arguments and keyword arguments + """ + for args, kwargs in suppressedWarnings: + warnings.filterwarnings(*args, **kwargs) + addedFilters = warnings.filters[:len(suppressedWarnings)] + try: + result = f(*a, **kw) + except: + exc_info = sys.exc_info() + _resetWarningFilters(None, addedFilters) + reraise(exc_info[1], exc_info[2]) + else: + if isinstance(result, defer.Deferred): + result.addBoth(_resetWarningFilters, addedFilters) + else: + _resetWarningFilters(None, addedFilters) + return result + + +def suppressWarnings(f, *suppressedWarnings): + """ + Wrap C{f} in a callable which suppresses the indicated warnings before + invoking C{f} and unsuppresses them afterwards. If f returns a Deferred, + warnings will remain suppressed until the Deferred fires. + """ + @wraps(f) + def warningSuppressingWrapper(*a, **kw): + return runWithWarningsSuppressed(suppressedWarnings, f, *a, **kw) + return warningSuppressingWrapper + + +__all__ = [ + "runWithWarningsSuppressed", "suppressWarnings", + "getProcessOutput", "getProcessValue", "getProcessOutputAndValue", + ] diff --git a/contrib/python/Twisted/py2/twisted/internet/win32eventreactor.py b/contrib/python/Twisted/py2/twisted/internet/win32eventreactor.py new file mode 100644 index 00000000000..e26c230797d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/win32eventreactor.py @@ -0,0 +1,429 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +A win32event based implementation of the Twisted main loop. + +This requires pywin32 (formerly win32all) or ActivePython to be installed. + +To install the event loop (and you should do this before any connections, +listeners or connectors are added):: + + from twisted.internet import win32eventreactor + win32eventreactor.install() + +LIMITATIONS: + 1. WaitForMultipleObjects and thus the event loop can only handle 64 objects. + 2. Process running has some problems (see L{twisted.internet.process} docstring). + + +TODO: + 1. Event loop handling of writes is *very* problematic (this is causing failed tests). + Switch to doing it the correct way, whatever that means (see below). + 2. Replace icky socket loopback waker with event based waker (use dummyEvent object) + 3. Switch everyone to using Free Software so we don't have to deal with proprietary APIs. + + +ALTERNATIVE SOLUTIONS: + - IIRC, sockets can only be registered once. So we switch to a structure + like the poll() reactor, thus allowing us to deal with write events in + a decent fashion. This should allow us to pass tests, but we're still + limited to 64 events. + +Or: + + - Instead of doing a reactor, we make this an addon to the select reactor. + The WFMO event loop runs in a separate thread. This means no need to maintain + separate code for networking, 64 event limit doesn't apply to sockets, + we can run processes and other win32 stuff in default event loop. The + only problem is that we're stuck with the icky socket based waker. + Another benefit is that this could be extended to support >64 events + in a simpler manner than the previous solution. + +The 2nd solution is probably what will get implemented. +""" + +# System imports +import time +import sys +from threading import Thread +from weakref import WeakKeyDictionary + +from zope.interface import implementer + +# Win32 imports +from win32file import FD_READ, FD_CLOSE, FD_ACCEPT, FD_CONNECT, WSAEventSelect +try: + # WSAEnumNetworkEvents was added in pywin32 215 + from win32file import WSAEnumNetworkEvents +except ImportError: + import warnings + warnings.warn( + 'Reliable disconnection notification requires pywin32 215 or later', + category=UserWarning) + def WSAEnumNetworkEvents(fd, event): + return set([FD_READ]) + +from win32event import CreateEvent, MsgWaitForMultipleObjects +from win32event import WAIT_OBJECT_0, WAIT_TIMEOUT, QS_ALLINPUT + +import win32gui + +# Twisted imports +from twisted.internet import posixbase +from twisted.python import log, threadable, failure +from twisted.internet.interfaces import IReactorFDSet +from twisted.internet.interfaces import IReactorWin32Events +from twisted.internet.threads import blockingCallFromThread + + +@implementer(IReactorFDSet, IReactorWin32Events) +class Win32Reactor(posixbase.PosixReactorBase): + """ + Reactor that uses Win32 event APIs. + + @ivar _reads: A dictionary mapping L{FileDescriptor} instances to a + win32 event object used to check for read events for that descriptor. + + @ivar _writes: A dictionary mapping L{FileDescriptor} instances to a + arbitrary value. Keys in this dictionary will be given a chance to + write out their data. + + @ivar _events: A dictionary mapping win32 event object to tuples of + L{FileDescriptor} instances and event masks. + + @ivar _closedAndReading: Along with C{_closedAndNotReading}, keeps track of + descriptors which have had close notification delivered from the OS but + which we have not finished reading data from. MsgWaitForMultipleObjects + will only deliver close notification to us once, so we remember it in + these two dictionaries until we're ready to act on it. The OS has + delivered close notification for each descriptor in this dictionary, and + the descriptors are marked as allowed to handle read events in the + reactor, so they can be processed. When a descriptor is marked as not + allowed to handle read events in the reactor (ie, it is passed to + L{IReactorFDSet.removeReader}), it is moved out of this dictionary and + into C{_closedAndNotReading}. The descriptors are keys in this + dictionary. The values are arbitrary. + @type _closedAndReading: C{dict} + + @ivar _closedAndNotReading: These descriptors have had close notification + delivered from the OS, but are not marked as allowed to handle read + events in the reactor. They are saved here to record their closed + state, but not processed at all. When one of these descriptors is + passed to L{IReactorFDSet.addReader}, it is moved out of this dictionary + and into C{_closedAndReading}. The descriptors are keys in this + dictionary. The values are arbitrary. This is a weak key dictionary so + that if an application tells the reactor to stop reading from a + descriptor and then forgets about that descriptor itself, the reactor + will also forget about it. + @type _closedAndNotReading: C{WeakKeyDictionary} + """ + dummyEvent = CreateEvent(None, 0, 0, None) + + def __init__(self): + self._reads = {} + self._writes = {} + self._events = {} + self._closedAndReading = {} + self._closedAndNotReading = WeakKeyDictionary() + posixbase.PosixReactorBase.__init__(self) + + + def _makeSocketEvent(self, fd, action, why): + """ + Make a win32 event object for a socket. + """ + event = CreateEvent(None, 0, 0, None) + WSAEventSelect(fd, event, why) + self._events[event] = (fd, action) + return event + + + def addEvent(self, event, fd, action): + """ + Add a new win32 event to the event loop. + """ + self._events[event] = (fd, action) + + + def removeEvent(self, event): + """ + Remove an event. + """ + del self._events[event] + + + def addReader(self, reader): + """ + Add a socket FileDescriptor for notification of data available to read. + """ + if reader not in self._reads: + self._reads[reader] = self._makeSocketEvent( + reader, 'doRead', FD_READ | FD_ACCEPT | FD_CONNECT | FD_CLOSE) + # If the reader is closed, move it over to the dictionary of reading + # descriptors. + if reader in self._closedAndNotReading: + self._closedAndReading[reader] = True + del self._closedAndNotReading[reader] + + + def addWriter(self, writer): + """ + Add a socket FileDescriptor for notification of data available to write. + """ + if writer not in self._writes: + self._writes[writer] = 1 + + + def removeReader(self, reader): + """Remove a Selectable for notification of data available to read. + """ + if reader in self._reads: + del self._events[self._reads[reader]] + del self._reads[reader] + + # If the descriptor is closed, move it out of the dictionary of + # reading descriptors into the dictionary of waiting descriptors. + if reader in self._closedAndReading: + self._closedAndNotReading[reader] = True + del self._closedAndReading[reader] + + + def removeWriter(self, writer): + """Remove a Selectable for notification of data available to write. + """ + if writer in self._writes: + del self._writes[writer] + + + def removeAll(self): + """ + Remove all selectables, and return a list of them. + """ + return self._removeAll(self._reads, self._writes) + + + def getReaders(self): + return list(self._reads.keys()) + + + def getWriters(self): + return list(self._writes.keys()) + + + def doWaitForMultipleEvents(self, timeout): + log.msg(channel='system', event='iteration', reactor=self) + if timeout is None: + timeout = 100 + + # Keep track of whether we run any application code before we get to the + # MsgWaitForMultipleObjects. If so, there's a chance it will schedule a + # new timed call or stop the reactor or do something else that means we + # shouldn't block in MsgWaitForMultipleObjects for the full timeout. + ranUserCode = False + + # If any descriptors are trying to close, try to get them out of the way + # first. + for reader in list(self._closedAndReading.keys()): + ranUserCode = True + self._runAction('doRead', reader) + + for fd in list(self._writes.keys()): + ranUserCode = True + log.callWithLogger(fd, self._runWrite, fd) + + if ranUserCode: + # If application code *might* have scheduled an event, assume it + # did. If we're wrong, we'll get back here shortly anyway. If + # we're right, we'll be sure to handle the event (including reactor + # shutdown) in a timely manner. + timeout = 0 + + if not (self._events or self._writes): + # sleep so we don't suck up CPU time + time.sleep(timeout) + return + + handles = list(self._events.keys()) or [self.dummyEvent] + timeout = int(timeout * 1000) + val = MsgWaitForMultipleObjects(handles, 0, timeout, QS_ALLINPUT) + if val == WAIT_TIMEOUT: + return + elif val == WAIT_OBJECT_0 + len(handles): + exit = win32gui.PumpWaitingMessages() + if exit: + self.callLater(0, self.stop) + return + elif val >= WAIT_OBJECT_0 and val < WAIT_OBJECT_0 + len(handles): + event = handles[val - WAIT_OBJECT_0] + fd, action = self._events[event] + + if fd in self._reads: + # Before anything, make sure it's still a valid file descriptor. + fileno = fd.fileno() + if fileno == -1: + self._disconnectSelectable(fd, posixbase._NO_FILEDESC, False) + return + + # Since it's a socket (not another arbitrary event added via + # addEvent) and we asked for FD_READ | FD_CLOSE, check to see if + # we actually got FD_CLOSE. This needs a special check because + # it only gets delivered once. If we miss it, it's gone forever + # and we'll never know that the connection is closed. + events = WSAEnumNetworkEvents(fileno, event) + if FD_CLOSE in events: + self._closedAndReading[fd] = True + log.callWithLogger(fd, self._runAction, action, fd) + + + def _runWrite(self, fd): + closed = 0 + try: + closed = fd.doWrite() + except: + closed = sys.exc_info()[1] + log.deferr() + + if closed: + self.removeReader(fd) + self.removeWriter(fd) + try: + fd.connectionLost(failure.Failure(closed)) + except: + log.deferr() + elif closed is None: + return 1 + + def _runAction(self, action, fd): + try: + closed = getattr(fd, action)() + except: + closed = sys.exc_info()[1] + log.deferr() + if closed: + self._disconnectSelectable(fd, closed, action == 'doRead') + + doIteration = doWaitForMultipleEvents + + + +class _ThreadFDWrapper(object): + """ + This wraps an event handler and translates notification in the helper + L{Win32Reactor} thread into a notification in the primary reactor thread. + + @ivar _reactor: The primary reactor, the one to which event notification + will be sent. + + @ivar _fd: The L{FileDescriptor} to which the event will be dispatched. + + @ivar _action: A C{str} giving the method of C{_fd} which handles the event. + + @ivar _logPrefix: The pre-fetched log prefix string for C{_fd}, so that + C{_fd.logPrefix} does not need to be called in a non-main thread. + """ + def __init__(self, reactor, fd, action, logPrefix): + self._reactor = reactor + self._fd = fd + self._action = action + self._logPrefix = logPrefix + + + def logPrefix(self): + """ + Return the original handler's log prefix, as it was given to + C{__init__}. + """ + return self._logPrefix + + + def _execute(self): + """ + Callback fired when the associated event is set. Run the C{action} + callback on the wrapped descriptor in the main reactor thread and raise + or return whatever it raises or returns to cause this event handler to + be removed from C{self._reactor} if appropriate. + """ + return blockingCallFromThread( + self._reactor, lambda: getattr(self._fd, self._action)()) + + + def connectionLost(self, reason): + """ + Pass through to the wrapped descriptor, but in the main reactor thread + instead of the helper C{Win32Reactor} thread. + """ + self._reactor.callFromThread(self._fd.connectionLost, reason) + + + +@implementer(IReactorWin32Events) +class _ThreadedWin32EventsMixin(object): + """ + This mixin implements L{IReactorWin32Events} for another reactor by running + a L{Win32Reactor} in a separate thread and dispatching work to it. + + @ivar _reactor: The L{Win32Reactor} running in the other thread. This is + L{None} until it is actually needed. + + @ivar _reactorThread: The L{threading.Thread} which is running the + L{Win32Reactor}. This is L{None} until it is actually needed. + """ + + _reactor = None + _reactorThread = None + + + def _unmakeHelperReactor(self): + """ + Stop and discard the reactor started by C{_makeHelperReactor}. + """ + self._reactor.callFromThread(self._reactor.stop) + self._reactor = None + + + def _makeHelperReactor(self): + """ + Create and (in a new thread) start a L{Win32Reactor} instance to use for + the implementation of L{IReactorWin32Events}. + """ + self._reactor = Win32Reactor() + # This is a helper reactor, it is not the global reactor and its thread + # is not "the" I/O thread. Prevent it from registering it as such. + self._reactor._registerAsIOThread = False + self._reactorThread = Thread( + target=self._reactor.run, args=(False,)) + self.addSystemEventTrigger( + 'after', 'shutdown', self._unmakeHelperReactor) + self._reactorThread.start() + + + def addEvent(self, event, fd, action): + """ + @see: L{IReactorWin32Events} + """ + if self._reactor is None: + self._makeHelperReactor() + self._reactor.callFromThread( + self._reactor.addEvent, + event, _ThreadFDWrapper(self, fd, action, fd.logPrefix()), + "_execute") + + + def removeEvent(self, event): + """ + @see: L{IReactorWin32Events} + """ + self._reactor.callFromThread(self._reactor.removeEvent, event) + + + +def install(): + threadable.init(1) + r = Win32Reactor() + from . import main + main.installReactor(r) + + +__all__ = ["Win32Reactor", "install"] diff --git a/contrib/python/Twisted/py2/twisted/internet/wxreactor.py b/contrib/python/Twisted/py2/twisted/internet/wxreactor.py new file mode 100644 index 00000000000..ae24c599e1b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/wxreactor.py @@ -0,0 +1,188 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module provides wxPython event loop support for Twisted. + +In order to use this support, simply do the following:: + + | from twisted.internet import wxreactor + | wxreactor.install() + +Then, when your root wxApp has been created:: + + | from twisted.internet import reactor + | reactor.registerWxApp(yourApp) + | reactor.run() + +Then use twisted.internet APIs as usual. Stop the event loop using +reactor.stop(), not yourApp.ExitMainLoop(). + +IMPORTANT: tests will fail when run under this reactor. This is +expected and probably does not reflect on the reactor's ability to run +real applications. +""" + +try: + from queue import Empty, Queue +except ImportError: + from Queue import Empty, Queue + +try: + from wx import PySimpleApp as wxPySimpleApp, CallAfter as wxCallAfter, \ + Timer as wxTimer +except ImportError: + # older version of wxPython: + from wxPython.wx import wxPySimpleApp, wxCallAfter, wxTimer + +from twisted.python import log, runtime +from twisted.internet import _threadedselect + + +class ProcessEventsTimer(wxTimer): + """ + Timer that tells wx to process pending events. + + This is necessary on macOS, probably due to a bug in wx, if we want + wxCallAfters to be handled when modal dialogs, menus, etc. are open. + """ + def __init__(self, wxapp): + wxTimer.__init__(self) + self.wxapp = wxapp + + + def Notify(self): + """ + Called repeatedly by wx event loop. + """ + self.wxapp.ProcessPendingEvents() + + + +class WxReactor(_threadedselect.ThreadedSelectReactor): + """ + wxPython reactor. + + wxPython drives the event loop, select() runs in a thread. + """ + + _stopping = False + + def registerWxApp(self, wxapp): + """ + Register wxApp instance with the reactor. + """ + self.wxapp = wxapp + + + def _installSignalHandlersAgain(self): + """ + wx sometimes removes our own signal handlers, so re-add them. + """ + try: + # make _handleSignals happy: + import signal + signal.signal(signal.SIGINT, signal.default_int_handler) + except ImportError: + return + self._handleSignals() + + + def stop(self): + """ + Stop the reactor. + """ + if self._stopping: + return + self._stopping = True + _threadedselect.ThreadedSelectReactor.stop(self) + + + def _runInMainThread(self, f): + """ + Schedule function to run in main wx/Twisted thread. + + Called by the select() thread. + """ + if hasattr(self, "wxapp"): + wxCallAfter(f) + else: + # wx shutdown but twisted hasn't + self._postQueue.put(f) + + + def _stopWx(self): + """ + Stop the wx event loop if it hasn't already been stopped. + + Called during Twisted event loop shutdown. + """ + if hasattr(self, "wxapp"): + self.wxapp.ExitMainLoop() + + + def run(self, installSignalHandlers=True): + """ + Start the reactor. + """ + self._postQueue = Queue() + if not hasattr(self, "wxapp"): + log.msg("registerWxApp() was not called on reactor, " + "registering my own wxApp instance.") + self.registerWxApp(wxPySimpleApp()) + + # start select() thread: + self.interleave(self._runInMainThread, + installSignalHandlers=installSignalHandlers) + if installSignalHandlers: + self.callLater(0, self._installSignalHandlersAgain) + + # add cleanup events: + self.addSystemEventTrigger("after", "shutdown", self._stopWx) + self.addSystemEventTrigger("after", "shutdown", + lambda: self._postQueue.put(None)) + + # On macOS, work around wx bug by starting timer to ensure + # wxCallAfter calls are always processed. We don't wake up as + # often as we could since that uses too much CPU. + if runtime.platform.isMacOSX(): + t = ProcessEventsTimer(self.wxapp) + t.Start(2) # wake up every 2ms + + self.wxapp.MainLoop() + wxapp = self.wxapp + del self.wxapp + + if not self._stopping: + # wx event loop exited without reactor.stop() being + # called. At this point events from select() thread will + # be added to _postQueue, but some may still be waiting + # unprocessed in wx, thus the ProcessPendingEvents() + # below. + self.stop() + wxapp.ProcessPendingEvents() # deal with any queued wxCallAfters + while 1: + try: + f = self._postQueue.get(timeout=0.01) + except Empty: + continue + else: + if f is None: + break + try: + f() + except: + log.err() + + +def install(): + """ + Configure the twisted mainloop to be run inside the wxPython mainloop. + """ + reactor = WxReactor() + from twisted.internet.main import installReactor + installReactor(reactor) + return reactor + + +__all__ = ['install'] diff --git a/contrib/python/Twisted/py2/twisted/internet/wxsupport.py b/contrib/python/Twisted/py2/twisted/internet/wxsupport.py new file mode 100644 index 00000000000..b16ff473a78 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/internet/wxsupport.py @@ -0,0 +1,59 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# +"""Old method of wxPython support for Twisted. + +twisted.internet.wxreactor is probably a better choice. + +To use:: + + | # given a wxApp instance called myWxAppInstance: + | from twisted.internet import wxsupport + | wxsupport.install(myWxAppInstance) + +Use Twisted's APIs for running and stopping the event loop, don't use +wxPython's methods. + +On Windows the Twisted event loop might block when dialogs are open +or menus are selected. + +Maintainer: Itamar Shtull-Trauring +""" + +import warnings +warnings.warn("wxsupport is not fully functional on Windows, wxreactor is better.") + +from twisted.python._oldstyle import _oldStyle +from twisted.internet import reactor + + + +@_oldStyle +class wxRunner: + """Make sure GUI events are handled.""" + + def __init__(self, app): + self.app = app + + def run(self): + """ + Execute pending WX events followed by WX idle events and + reschedule. + """ + # run wx events + while self.app.Pending(): + self.app.Dispatch() + + # run wx idle events + self.app.ProcessIdle() + reactor.callLater(0.02, self.run) + + +def install(app): + """Install the wxPython support, given a wxApp instance""" + runner = wxRunner(app) + reactor.callLater(0.02, runner.run) + + +__all__ = ["install"] diff --git a/contrib/python/Twisted/py2/twisted/logger/__init__.py b/contrib/python/Twisted/py2/twisted/logger/__init__.py new file mode 100644 index 00000000000..d80a704911b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/__init__.py @@ -0,0 +1,130 @@ +# -*- test-case-name: twisted.logger.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Logger: Classes and functions to do granular logging. + +Example usage in a module C{some.module}:: + + from twisted.logger import Logger + log = Logger() + + def handleData(data): + log.debug("Got data: {data!r}.", data=data) + +Or in a class:: + + from twisted.logger import Logger + + class Foo(object): + log = Logger() + + def oops(self, data): + self.log.error("Oops! Invalid data from server: {data!r}", + data=data) + +C{Logger}s have namespaces, for which logging can be configured independently. +Namespaces may be specified by passing in a C{namespace} argument to L{Logger} +when instantiating it, but if none is given, the logger will derive its own +namespace by using the module name of the callable that instantiated it, or, in +the case of a class, by using the fully qualified name of the class. + +In the first example above, the namespace would be C{some.module}, and in the +second example, it would be C{some.module.Foo}. + +@var globalLogPublisher: The L{LogPublisher} that all L{Logger} instances that + are not otherwise parameterized will point to by default. +@type globalLogPublisher: L{LogPublisher} + +@var globalLogBeginner: The L{LogBeginner} used to activate the main log + observer, whether it's a log file, or an observer pointing at stderr. +@type globalLogBeginner: L{LogBeginner} +""" + +__all__ = [ + # From ._levels + "InvalidLogLevelError", "LogLevel", + + # From ._format + "formatEvent", "formatEventAsClassicLogText", + "formatTime", "timeFormatRFC3339", + "eventAsText", + + # From ._flatten + "extractField", + + # From ._logger + "Logger", "_loggerFor", + + # From ._observer + "ILogObserver", "LogPublisher", + + # From ._buffer + "LimitedHistoryLogObserver", + + # From ._file + "FileLogObserver", "textFileLogObserver", + + # From ._filter + "PredicateResult", "ILogFilterPredicate", + "FilteringLogObserver", "LogLevelFilterPredicate", + + # From ._stdlib + "STDLibLogObserver", + + # From ._io + "LoggingFile", + + # From ._legacy + "LegacyLogObserverWrapper", + + # From ._global + "globalLogPublisher", "globalLogBeginner", "LogBeginner", + + # From ._json + "eventAsJSON", "eventFromJSON", + "jsonFileLogObserver", "eventsFromJSONLogFile", + + # From ._capture + "capturedLogs", +] + +from ._levels import InvalidLogLevelError, LogLevel + +from ._flatten import extractField + +from ._format import ( + formatEvent, formatEventAsClassicLogText, formatTime, timeFormatRFC3339, + eventAsText +) + +from ._logger import Logger, _loggerFor + +from ._observer import ILogObserver, LogPublisher + +from ._buffer import LimitedHistoryLogObserver + +from ._file import FileLogObserver, textFileLogObserver + +from ._filter import ( + PredicateResult, ILogFilterPredicate, FilteringLogObserver, + LogLevelFilterPredicate +) + +from ._stdlib import STDLibLogObserver + +from ._io import LoggingFile + +from ._legacy import LegacyLogObserverWrapper + +from ._global import ( + globalLogPublisher, globalLogBeginner, LogBeginner +) + +from ._json import ( + eventAsJSON, eventFromJSON, + jsonFileLogObserver, eventsFromJSONLogFile +) + +from ._capture import capturedLogs diff --git a/contrib/python/Twisted/py2/twisted/logger/_buffer.py b/contrib/python/Twisted/py2/twisted/logger/_buffer.py new file mode 100644 index 00000000000..6d32166d1c7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_buffer.py @@ -0,0 +1,59 @@ +# -*- test-case-name: twisted.logger.test.test_buffer -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Log observer that maintains a buffer. +""" + +from collections import deque + +from zope.interface import implementer + +from ._observer import ILogObserver + + +_DEFAULT_BUFFER_MAXIMUM = 64 * 1024 + + + +@implementer(ILogObserver) +class LimitedHistoryLogObserver(object): + """ + L{ILogObserver} that stores events in a buffer of a fixed size:: + + >>> from twisted.logger import LimitedHistoryLogObserver + >>> history = LimitedHistoryLogObserver(5) + >>> for n in range(10): history({'n': n}) + ... + >>> repeats = [] + >>> history.replayTo(repeats.append) + >>> len(repeats) + 5 + >>> repeats + [{'n': 5}, {'n': 6}, {'n': 7}, {'n': 8}, {'n': 9}] + >>> + """ + + def __init__(self, size=_DEFAULT_BUFFER_MAXIMUM): + """ + @param size: The maximum number of events to buffer. If L{None}, the + buffer is unbounded. + @type size: L{int} + """ + self._buffer = deque(maxlen=size) + + + def __call__(self, event): + self._buffer.append(event) + + + def replayTo(self, otherObserver): + """ + Re-play the buffered events to another log observer. + + @param otherObserver: An observer to replay events to. + @type otherObserver: L{ILogObserver} + """ + for event in self._buffer: + otherObserver(event) diff --git a/contrib/python/Twisted/py2/twisted/logger/_capture.py b/contrib/python/Twisted/py2/twisted/logger/_capture.py new file mode 100644 index 00000000000..ee9096e9db1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_capture.py @@ -0,0 +1,24 @@ +# -*- test-case-name: twisted.logger.test.test_capture -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Context manager for capturing logs. +""" + +from contextlib import contextmanager + +from twisted.logger import globalLogPublisher + + + +@contextmanager +def capturedLogs(): + events = [] + observer = events.append + + globalLogPublisher.addObserver(observer) + + yield events + + globalLogPublisher.removeObserver(observer) diff --git a/contrib/python/Twisted/py2/twisted/logger/_file.py b/contrib/python/Twisted/py2/twisted/logger/_file.py new file mode 100644 index 00000000000..4913070ef27 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_file.py @@ -0,0 +1,86 @@ +# -*- test-case-name: twisted.logger.test.test_file -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +File log observer. +""" + +from zope.interface import implementer + +from twisted.python.compat import ioType, unicode +from ._observer import ILogObserver +from ._format import formatTime +from ._format import timeFormatRFC3339 +from ._format import formatEventAsClassicLogText + + + +@implementer(ILogObserver) +class FileLogObserver(object): + """ + Log observer that writes to a file-like object. + """ + def __init__(self, outFile, formatEvent): + """ + @param outFile: A file-like object. Ideally one should be passed which + accepts L{unicode} data. Otherwise, UTF-8 L{bytes} will be used. + @type outFile: L{io.IOBase} + + @param formatEvent: A callable that formats an event. + @type formatEvent: L{callable} that takes an C{event} argument and + returns a formatted event as L{unicode}. + """ + if ioType(outFile) is not unicode: + self._encoding = "utf-8" + else: + self._encoding = None + + self._outFile = outFile + self.formatEvent = formatEvent + + + def __call__(self, event): + """ + Write event to file. + + @param event: An event. + @type event: L{dict} + """ + text = self.formatEvent(event) + + if text is None: + text = u"" + + if self._encoding is not None: + text = text.encode(self._encoding) + + if text: + self._outFile.write(text) + self._outFile.flush() + + + +def textFileLogObserver(outFile, timeFormat=timeFormatRFC3339): + """ + Create a L{FileLogObserver} that emits text to a specified (writable) + file-like object. + + @param outFile: A file-like object. Ideally one should be passed which + accepts L{unicode} data. Otherwise, UTF-8 L{bytes} will be used. + @type outFile: L{io.IOBase} + + @param timeFormat: The format to use when adding timestamp prefixes to + logged events. If L{None}, or for events with no C{"log_timestamp"} + key, the default timestamp prefix of C{u"-"} is used. + @type timeFormat: L{unicode} or L{None} + + @return: A file log observer. + @rtype: L{FileLogObserver} + """ + def formatEvent(event): + return formatEventAsClassicLogText( + event, formatTime=lambda e: formatTime(e, timeFormat) + ) + + return FileLogObserver(outFile, formatEvent) diff --git a/contrib/python/Twisted/py2/twisted/logger/_filter.py b/contrib/python/Twisted/py2/twisted/logger/_filter.py new file mode 100644 index 00000000000..fdbd13c261e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_filter.py @@ -0,0 +1,231 @@ +# -*- test-case-name: twisted.logger.test.test_filter -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Filtering log observer. +""" + +from functools import partial + +from zope.interface import Interface, implementer + +from constantly import NamedConstant, Names + +from ._levels import InvalidLogLevelError, LogLevel +from ._observer import ILogObserver + + + +class PredicateResult(Names): + """ + Predicate results. + + @see: L{LogLevelFilterPredicate} + + @cvar yes: Log the specified event. When this value is used, + L{FilteringLogObserver} will always log the message, without + evaluating other predicates. + + @cvar no: Do not log the specified event. When this value is used, + L{FilteringLogObserver} will I{not} log the message, without + evaluating other predicates. + + @cvar maybe: Do not have an opinion on the event. When this value is used, + L{FilteringLogObserver} will consider subsequent predicate results; + if returned by the last predicate being considered, then the event will + be logged. + """ + yes = NamedConstant() + no = NamedConstant() + maybe = NamedConstant() + + + +class ILogFilterPredicate(Interface): + """ + A predicate that determined whether an event should be logged. + """ + + def __call__(event): + """ + Determine whether an event should be logged. + + @returns: a L{PredicateResult}. + """ + + + +def shouldLogEvent(predicates, event): + """ + Determine whether an event should be logged, based on the result of + C{predicates}. + + By default, the result is C{True}; so if there are no predicates, + everything will be logged. + + If any predicate returns C{yes}, then we will immediately return C{True}. + + If any predicate returns C{no}, then we will immediately return C{False}. + + As predicates return C{maybe}, we keep calling the next predicate until we + run out, at which point we return C{True}. + + @param predicates: The predicates to use. + @type predicates: iterable of L{ILogFilterPredicate} + + @param event: An event + @type event: L{dict} + + @return: True if the message should be forwarded on, C{False} if not. + @rtype: L{bool} + """ + for predicate in predicates: + result = predicate(event) + if result == PredicateResult.yes: + return True + if result == PredicateResult.no: + return False + if result == PredicateResult.maybe: + continue + raise TypeError("Invalid predicate result: {0!r}".format(result)) + return True + + + +@implementer(ILogObserver) +class FilteringLogObserver(object): + """ + L{ILogObserver} that wraps another L{ILogObserver}, but filters out events + based on applying a series of L{ILogFilterPredicate}s. + """ + + def __init__( + self, observer, predicates, + negativeObserver=lambda event: None + ): + """ + @param observer: An observer to which this observer will forward + events when C{predictates} yield a positive result. + @type observer: L{ILogObserver} + + @param predicates: Predicates to apply to events before forwarding to + the wrapped observer. + @type predicates: ordered iterable of predicates + + @param negativeObserver: An observer to which this observer will + forward events when C{predictates} yield a negative result. + @type negativeObserver: L{ILogObserver} + """ + self._observer = observer + self._shouldLogEvent = partial(shouldLogEvent, list(predicates)) + self._negativeObserver = negativeObserver + + + def __call__(self, event): + """ + Forward to next observer if predicate allows it. + """ + if self._shouldLogEvent(event): + if "log_trace" in event: + event["log_trace"].append((self, self.observer)) + self._observer(event) + else: + self._negativeObserver(event) + + + +@implementer(ILogFilterPredicate) +class LogLevelFilterPredicate(object): + """ + L{ILogFilterPredicate} that filters out events with a log level lower than + the log level for the event's namespace. + + Events that not not have a log level or namespace are also dropped. + """ + + def __init__(self, defaultLogLevel=LogLevel.info): + """ + @param defaultLogLevel: The default minimum log level. + @type defaultLogLevel: L{LogLevel} + """ + self._logLevelsByNamespace = {} + self.defaultLogLevel = defaultLogLevel + self.clearLogLevels() + + + def logLevelForNamespace(self, namespace): + """ + Determine an appropriate log level for the given namespace. + + This respects dots in namespaces; for example, if you have previously + invoked C{setLogLevelForNamespace("mypackage", LogLevel.debug)}, then + C{logLevelForNamespace("mypackage.subpackage")} will return + C{LogLevel.debug}. + + @param namespace: A logging namespace, or L{None} for the default + namespace. + @type namespace: L{str} (native string) + + @return: The log level for the specified namespace. + @rtype: L{LogLevel} + """ + if not namespace: + return self._logLevelsByNamespace[None] + + if namespace in self._logLevelsByNamespace: + return self._logLevelsByNamespace[namespace] + + segments = namespace.split(".") + index = len(segments) - 1 + + while index > 0: + namespace = ".".join(segments[:index]) + if namespace in self._logLevelsByNamespace: + return self._logLevelsByNamespace[namespace] + index -= 1 + + return self._logLevelsByNamespace[None] + + + def setLogLevelForNamespace(self, namespace, level): + """ + Sets the log level for a logging namespace. + + @param namespace: A logging namespace. + @type namespace: L{str} (native string) + + @param level: The log level for the given namespace. + @type level: L{LogLevel} + """ + if level not in LogLevel.iterconstants(): + raise InvalidLogLevelError(level) + + if namespace: + self._logLevelsByNamespace[namespace] = level + else: + self._logLevelsByNamespace[None] = level + + + def clearLogLevels(self): + """ + Clears all log levels to the default. + """ + self._logLevelsByNamespace.clear() + self._logLevelsByNamespace[None] = self.defaultLogLevel + + + def __call__(self, event): + eventLevel = event.get("log_level", None) + namespace = event.get("log_namespace", None) + namespaceLevel = self.logLevelForNamespace(namespace) + + if ( + eventLevel is None or + namespace is None or + LogLevel._priorityForLevel(eventLevel) < + LogLevel._priorityForLevel(namespaceLevel) + ): + return PredicateResult.no + + return PredicateResult.maybe diff --git a/contrib/python/Twisted/py2/twisted/logger/_flatten.py b/contrib/python/Twisted/py2/twisted/logger/_flatten.py new file mode 100644 index 00000000000..a6f06ed60d7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_flatten.py @@ -0,0 +1,178 @@ +# -*- test-case-name: twisted.logger.test.test_flatten -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Code related to "flattening" events; that is, extracting a description of all +relevant fields from the format string and persisting them for later +examination. +""" + +from string import Formatter +from collections import defaultdict + +from twisted.python.compat import unicode + +aFormatter = Formatter() + + + +class KeyFlattener(object): + """ + A L{KeyFlattener} computes keys for the things within curly braces in + PEP-3101-style format strings as parsed by L{string.Formatter.parse}. + """ + + def __init__(self): + """ + Initialize a L{KeyFlattener}. + """ + self.keys = defaultdict(lambda: 0) + + + def flatKey(self, fieldName, formatSpec, conversion): + """ + Compute a string key for a given field/format/conversion. + + @param fieldName: A format field name. + @type fieldName: L{str} + + @param formatSpec: A format spec. + @type formatSpec: L{str} + + @param conversion: A format field conversion type. + @type conversion: L{str} + + @return: A key specific to the given field, format and conversion, as + well as the occurrence of that combination within this + L{KeyFlattener}'s lifetime. + @rtype: L{str} + """ + result = ( + "{fieldName}!{conversion}:{formatSpec}" + .format( + fieldName=fieldName, + formatSpec=(formatSpec or ""), + conversion=(conversion or ""), + ) + ) + self.keys[result] += 1 + n = self.keys[result] + if n != 1: + result += "/" + str(self.keys[result]) + return result + + + +def flattenEvent(event): + """ + Flatten the given event by pre-associating format fields with specific + objects and callable results in a L{dict} put into the C{"log_flattened"} + key in the event. + + @param event: A logging event. + @type event: L{dict} + """ + if event.get("log_format", None) is None: + return + + if "log_flattened" in event: + fields = event["log_flattened"] + else: + fields = {} + + keyFlattener = KeyFlattener() + + for (literalText, fieldName, formatSpec, conversion) in ( + aFormatter.parse(event["log_format"]) + ): + if fieldName is None: + continue + + if conversion != "r": + conversion = "s" + + flattenedKey = keyFlattener.flatKey(fieldName, formatSpec, conversion) + structuredKey = keyFlattener.flatKey(fieldName, formatSpec, "") + + if flattenedKey in fields: + # We've already seen and handled this key + continue + + if fieldName.endswith(u"()"): + fieldName = fieldName[:-2] + callit = True + else: + callit = False + + field = aFormatter.get_field(fieldName, (), event) + fieldValue = field[0] + + if conversion == "r": + conversionFunction = repr + else: # Above: if conversion is not "r", it's "s" + conversionFunction = unicode + + if callit: + fieldValue = fieldValue() + + flattenedValue = conversionFunction(fieldValue) + fields[flattenedKey] = flattenedValue + fields[structuredKey] = fieldValue + + if fields: + event["log_flattened"] = fields + + + +def extractField(field, event): + """ + Extract a given format field from the given event. + + @param field: A string describing a format field or log key. This is the + text that would normally fall between a pair of curly braces in a + format string: for example, C{"key[2].attribute"}. If a conversion is + specified (the thing after the C{"!"} character in a format field) then + the result will always be L{unicode}. + @type field: L{str} (native string) + + @param event: A log event. + @type event: L{dict} + + @return: A value extracted from the field. + @rtype: L{object} + + @raise KeyError: if the field is not found in the given event. + """ + keyFlattener = KeyFlattener() + [[literalText, fieldName, formatSpec, conversion]] = aFormatter.parse( + "{" + field + "}" + ) + key = keyFlattener.flatKey(fieldName, formatSpec, conversion) + if "log_flattened" not in event: + flattenEvent(event) + return event["log_flattened"][key] + + + +def flatFormat(event): + """ + Format an event which has been flattened with L{flattenEvent}. + + @param event: A logging event. + @type event: L{dict} + + @return: A formatted string. + @rtype: L{unicode} + """ + fieldValues = event["log_flattened"] + s = [] + keyFlattener = KeyFlattener() + formatFields = aFormatter.parse(event["log_format"]) + for literalText, fieldName, formatSpec, conversion in formatFields: + s.append(literalText) + if fieldName is not None: + key = keyFlattener.flatKey( + fieldName, formatSpec, conversion or "s") + s.append(unicode(fieldValues[key])) + return u"".join(s) diff --git a/contrib/python/Twisted/py2/twisted/logger/_format.py b/contrib/python/Twisted/py2/twisted/logger/_format.py new file mode 100644 index 00000000000..b7cc3cbf0c7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_format.py @@ -0,0 +1,421 @@ +# -*- test-case-name: twisted.logger.test.test_format -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Tools for formatting logging events. +""" + +from datetime import datetime as DateTime + +from twisted.python.compat import unicode +from twisted.python.failure import Failure +from twisted.python.reflect import safe_repr +from twisted.python._tzhelper import FixedOffsetTimeZone + +from ._flatten import flatFormat, aFormatter + +timeFormatRFC3339 = "%Y-%m-%dT%H:%M:%S%z" + + + +def formatEvent(event): + """ + Formats an event as a L{unicode}, using the format in + C{event["log_format"]}. + + This implementation should never raise an exception; if the formatting + cannot be done, the returned string will describe the event generically so + that a useful message is emitted regardless. + + @param event: A logging event. + @type event: L{dict} + + @return: A formatted string. + @rtype: L{unicode} + """ + return eventAsText( + event, + includeTraceback=False, + includeTimestamp=False, + includeSystem=False, + ) + + + +def formatUnformattableEvent(event, error): + """ + Formats an event as a L{unicode} that describes the event generically and a + formatting error. + + @param event: A logging event. + @type event: L{dict} + + @param error: The formatting error. + @type error: L{Exception} + + @return: A formatted string. + @rtype: L{unicode} + """ + try: + return ( + u"Unable to format event {event!r}: {error}" + .format(event=event, error=error) + ) + except BaseException: + # Yikes, something really nasty happened. + # + # Try to recover as much formattable data as possible; hopefully at + # least the namespace is sane, which will help you find the offending + # logger. + failure = Failure() + + text = u", ".join( + u" = ".join((safe_repr(key), safe_repr(value))) + for key, value in event.items() + ) + + return ( + u"MESSAGE LOST: unformattable object logged: {error}\n" + u"Recoverable data: {text}\n" + u"Exception during formatting:\n{failure}" + .format(error=safe_repr(error), failure=failure, text=text) + ) + + + +def formatTime(when, timeFormat=timeFormatRFC3339, default=u"-"): + """ + Format a timestamp as text. + + Example:: + + >>> from time import time + >>> from twisted.logger import formatTime + >>> + >>> t = time() + >>> formatTime(t) + u'2013-10-22T14:19:11-0700' + >>> formatTime(t, timeFormat="%Y/%W") # Year and week number + u'2013/42' + >>> + + @param when: A timestamp. + @type then: L{float} + + @param timeFormat: A time format. + @type timeFormat: L{unicode} or L{None} + + @param default: Text to return if C{when} or C{timeFormat} is L{None}. + @type default: L{unicode} + + @return: A formatted time. + @rtype: L{unicode} + """ + if (timeFormat is None or when is None): + return default + else: + tz = FixedOffsetTimeZone.fromLocalTimeStamp(when) + datetime = DateTime.fromtimestamp(when, tz) + return unicode(datetime.strftime(timeFormat)) + + + +def formatEventAsClassicLogText(event, formatTime=formatTime): + """ + Format an event as a line of human-readable text for, e.g. traditional log + file output. + + The output format is C{u"{timeStamp} [{system}] {event}\\n"}, where: + + - C{timeStamp} is computed by calling the given C{formatTime} callable + on the event's C{"log_time"} value + + - C{system} is the event's C{"log_system"} value, if set, otherwise, + the C{"log_namespace"} and C{"log_level"}, joined by a C{u"#"}. Each + defaults to C{u"-"} is not set. + + - C{event} is the event, as formatted by L{formatEvent}. + + Example:: + + >>> from __future__ import print_function + >>> from time import time + >>> from twisted.logger import formatEventAsClassicLogText + >>> from twisted.logger import LogLevel + >>> + >>> formatEventAsClassicLogText(dict()) # No format, returns None + >>> formatEventAsClassicLogText(dict(log_format=u"Hello!")) + u'- [-#-] Hello!\\n' + >>> formatEventAsClassicLogText(dict( + ... log_format=u"Hello!", + ... log_time=time(), + ... log_namespace="my_namespace", + ... log_level=LogLevel.info, + ... )) + u'2013-10-22T17:30:02-0700 [my_namespace#info] Hello!\\n' + >>> formatEventAsClassicLogText(dict( + ... log_format=u"Hello!", + ... log_time=time(), + ... log_system="my_system", + ... )) + u'2013-11-11T17:22:06-0800 [my_system] Hello!\\n' + >>> + + @param event: an event. + @type event: L{dict} + + @param formatTime: A time formatter + @type formatTime: L{callable} that takes an C{event} argument and returns + a L{unicode} + + @return: A formatted event, or L{None} if no output is appropriate. + @rtype: L{unicode} or L{None} + """ + eventText = eventAsText(event, formatTime=formatTime) + if not eventText: + return None + eventText = eventText.replace(u"\n", u"\n\t") + return eventText + u"\n" + + + +class CallMapping(object): + """ + Read-only mapping that turns a C{()}-suffix in key names into an invocation + of the key rather than a lookup of the key. + + Implementation support for L{formatWithCall}. + """ + def __init__(self, submapping): + """ + @param submapping: Another read-only mapping which will be used to look + up items. + """ + self._submapping = submapping + + + def __getitem__(self, key): + """ + Look up an item in the submapping for this L{CallMapping}, calling it + if C{key} ends with C{"()"}. + """ + callit = key.endswith(u"()") + realKey = key[:-2] if callit else key + value = self._submapping[realKey] + if callit: + value = value() + return value + + + +def formatWithCall(formatString, mapping): + """ + Format a string like L{unicode.format}, but: + + - taking only a name mapping; no positional arguments + + - with the additional syntax that an empty set of parentheses + correspond to a formatting item that should be called, and its result + C{str}'d, rather than calling C{str} on the element directly as + normal. + + For example:: + + >>> formatWithCall("{string}, {function()}.", + ... dict(string="just a string", + ... function=lambda: "a function")) + 'just a string, a function.' + + @param formatString: A PEP-3101 format string. + @type formatString: L{unicode} + + @param mapping: A L{dict}-like object to format. + + @return: The string with formatted values interpolated. + @rtype: L{unicode} + """ + return unicode( + aFormatter.vformat(formatString, (), CallMapping(mapping)) + ) + + + +def _formatEvent(event): + """ + Formats an event as a L{unicode}, using the format in + C{event["log_format"]}. + + This implementation should never raise an exception; if the formatting + cannot be done, the returned string will describe the event generically so + that a useful message is emitted regardless. + + @param event: A logging event. + @type event: L{dict} + + @return: A formatted string. + @rtype: L{unicode} + """ + try: + if "log_flattened" in event: + return flatFormat(event) + + format = event.get("log_format", None) + if format is None: + return u"" + + # Make sure format is unicode. + if isinstance(format, bytes): + # If we get bytes, assume it's UTF-8 bytes + format = format.decode("utf-8") + elif not isinstance(format, unicode): + raise TypeError( + "Log format must be unicode or bytes, not {0!r}".format(format) + ) + + return formatWithCall(format, event) + + except BaseException as e: + return formatUnformattableEvent(event, e) + + + +def _formatTraceback(failure): + """ + Format a failure traceback, assuming UTF-8 and using a replacement + strategy for errors. Every effort is made to provide a usable + traceback, but should not that not be possible, a message and the + captured exception are logged. + + @param failure: The failure to retrieve a traceback from. + @type failure: L{twisted.python.failure.Failure} + + @return: The formatted traceback. + @rtype: L{unicode} + """ + try: + traceback = failure.getTraceback() + if isinstance(traceback, bytes): + traceback = traceback.decode('utf-8', errors='replace') + except BaseException as e: + traceback = ( + u"(UNABLE TO OBTAIN TRACEBACK FROM EVENT):" + unicode(e) + ) + return traceback + + + +def _formatSystem(event): + """ + Format the system specified in the event in the "log_system" key if set, + otherwise the C{"log_namespace"} and C{"log_level"}, joined by a C{u"#"}. + Each defaults to C{u"-"} is not set. If formatting fails completely, + "UNFORMATTABLE" is returned. + + @param event: The event containing the system specification. + @type event: L{dict} + + @return: A formatted string representing the "log_system" key. + @rtype: L{unicode} + """ + system = event.get("log_system", None) + if system is None: + level = event.get("log_level", None) + if level is None: + levelName = u"-" + else: + levelName = level.name + + system = u"{namespace}#{level}".format( + namespace=event.get("log_namespace", u"-"), + level=levelName, + ) + else: + try: + system = unicode(system) + except Exception: + system = u"UNFORMATTABLE" + return system + + + +def eventAsText( + event, + includeTraceback=True, + includeTimestamp=True, + includeSystem=True, + formatTime=formatTime, +): + r""" + Format an event as a unicode string. Optionally, attach timestamp, + traceback, and system information. + + The full output format is: + C{u"{timeStamp} [{system}] {event}\n{traceback}\n"} where: + + - C{timeStamp} is the event's C{"log_time"} value formatted with + the provided C{formatTime} callable. + + - C{system} is the event's C{"log_system"} value, if set, otherwise, + the C{"log_namespace"} and C{"log_level"}, joined by a C{u"#"}. Each + defaults to C{u"-"} is not set. + + - C{event} is the event, as formatted by L{formatEvent}. + + - C{traceback} is the traceback if the event contains a + C{"log_failure"} key. In the event the original traceback cannot + be formatted, a message indicating the failure will be substituted. + + If the event cannot be formatted, and no traceback exists, an empty string + is returned, even if includeSystem or includeTimestamp are true. + + @param event: A logging event. + @type event: L{dict} + + @param includeTraceback: If true and a C{"log_failure"} key exists, append + a traceback. + @type includeTraceback: L{bool} + + @param includeTimestamp: If true include a formatted timestamp before the + event. + @type includeTimestamp: L{bool} + + @param includeSystem: If true, include the event's C{"log_system"} value. + @type includeSystem: L{bool} + + @param formatTime: A time formatter + @type formatTime: L{callable} that takes an C{event} argument and returns + a L{unicode} + + @return: A formatted string with specified options. + @rtype: L{unicode} + + @since: Twisted 18.9.0 + """ + eventText = _formatEvent(event) + if includeTraceback and 'log_failure' in event: + f = event['log_failure'] + traceback = _formatTraceback(f) + eventText = u"\n".join((eventText, traceback)) + + if not eventText: + return eventText + + timeStamp = u"" + if includeTimestamp: + timeStamp = u"".join([formatTime(event.get("log_time", None)), " "]) + + system = u"" + if includeSystem: + system = u"".join([ + u"[", + _formatSystem(event), + u"]", + u" " + ]) + + return u"{timeStamp}{system}{eventText}".format( + timeStamp=timeStamp, + system=system, + eventText=eventText, + ) diff --git a/contrib/python/Twisted/py2/twisted/logger/_global.py b/contrib/python/Twisted/py2/twisted/logger/_global.py new file mode 100644 index 00000000000..45c0bc3f368 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_global.py @@ -0,0 +1,240 @@ +# -*- test-case-name: twisted.logger.test.test_global -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module includes process-global state associated with the logging system, +and implementation of logic for managing that global state. +""" + +import sys +import warnings + +from twisted.python.compat import currentframe +from twisted.python.reflect import qual + +from ._buffer import LimitedHistoryLogObserver +from ._observer import LogPublisher +from ._filter import FilteringLogObserver, LogLevelFilterPredicate +from ._logger import Logger +from ._format import eventAsText +from ._levels import LogLevel +from ._io import LoggingFile +from ._file import FileLogObserver + + + +MORE_THAN_ONCE_WARNING = ( + "Warning: primary log target selected twice at <{fileNow}:{lineNow}> - " + "previously selected at <{fileThen}:{lineThen}>. Remove one of the calls " + "to beginLoggingTo." +) + + + +class LogBeginner(object): + """ + A L{LogBeginner} holds state related to logging before logging has begun, + and begins logging when told to do so. Logging "begins" when someone has + selected a set of observers, like, for example, a L{FileLogObserver} that + writes to a file on disk, or to standard output. + + Applications will not typically need to instantiate this class, except + those which intend to initialize the global logging system themselves, + which may wish to instantiate this for testing. The global instance for + the current process is exposed as + L{twisted.logger.globalLogBeginner}. + + Before logging has begun, a L{LogBeginner} will: + + 1. Log any critical messages (e.g.: unhandled exceptions) to the given + file-like object. + + 2. Save (a limited number of) log events in a + L{LimitedHistoryLogObserver}. + + @ivar _initialBuffer: A buffer of messages logged before logging began. + @type _initialBuffer: L{LimitedHistoryLogObserver} + + @ivar _publisher: The log publisher passed in to L{LogBeginner}'s + constructor. + @type _publisher: L{LogPublisher} + + @ivar _log: The logger used to log messages about the operation of the + L{LogBeginner} itself. + @type _log: L{Logger} + + @ivar _temporaryObserver: If not L{None}, an L{ILogObserver} that observes + events on C{_publisher} for this L{LogBeginner}. + @type _temporaryObserver: L{ILogObserver} or L{None} + + @ivar _stdio: An object with C{stderr} and C{stdout} attributes (like the + L{sys} module) which will be replaced when redirecting standard I/O. + @type _stdio: L{object} + + @cvar _DEFAULT_BUFFER_SIZE: The default size for the initial log events + buffer. + @type _DEFAULT_BUFFER_SIZE: L{int} + """ + + _DEFAULT_BUFFER_SIZE = 200 + + def __init__( + self, publisher, errorStream, stdio, warningsModule, + initialBufferSize=None, + ): + """ + Initialize this L{LogBeginner}. + + @param initialBufferSize: The size of the event buffer into which + events are collected until C{beginLoggingTo} is called. Or + C{None} to use the default size. + @type initialBufferSize: L{int} or L{types.NoneType} + """ + if initialBufferSize is None: + initialBufferSize = self._DEFAULT_BUFFER_SIZE + self._initialBuffer = LimitedHistoryLogObserver(size=initialBufferSize) + self._publisher = publisher + self._log = Logger(observer=publisher) + self._stdio = stdio + self._warningsModule = warningsModule + self._temporaryObserver = LogPublisher( + self._initialBuffer, + FilteringLogObserver( + FileLogObserver( + errorStream, + lambda event: eventAsText( + event, + includeTimestamp=False, + includeSystem=False, + ) + '\n' + ), + [LogLevelFilterPredicate(defaultLogLevel=LogLevel.critical)] + ) + ) + publisher.addObserver(self._temporaryObserver) + self._oldshowwarning = warningsModule.showwarning + + + def beginLoggingTo( + self, observers, discardBuffer=False, redirectStandardIO=True + ): + """ + Begin logging to the given set of observers. This will: + + 1. Add all the observers given in C{observers} to the + L{LogPublisher} associated with this L{LogBeginner}. + + 2. Optionally re-direct standard output and standard error streams + to the logging system. + + 3. Re-play any messages that were previously logged to that + publisher to the new observers, if C{discardBuffer} is not set. + + 4. Stop logging critical errors from the L{LogPublisher} as strings + to the C{errorStream} associated with this L{LogBeginner}, and + allow them to be logged normally. + + 5. Re-direct warnings from the L{warnings} module associated with + this L{LogBeginner} to log messages. + + @note: Since a L{LogBeginner} is designed to encapsulate the transition + between process-startup and log-system-configuration, this method + is intended to be invoked I{once}. + + @param observers: The observers to register. + @type observers: iterable of L{ILogObserver}s + + @param discardBuffer: Whether to discard the buffer and not re-play it + to the added observers. (This argument is provided mainly for + compatibility with legacy concerns.) + @type discardBuffer: L{bool} + + @param redirectStandardIO: If true, redirect standard output and + standard error to the observers. + @type redirectStandardIO: L{bool} + """ + caller = currentframe(1) + filename, lineno = caller.f_code.co_filename, caller.f_lineno + + for observer in observers: + self._publisher.addObserver(observer) + + if self._temporaryObserver is not None: + self._publisher.removeObserver(self._temporaryObserver) + if not discardBuffer: + self._initialBuffer.replayTo(self._publisher) + self._temporaryObserver = None + self._warningsModule.showwarning = self.showwarning + else: + previousFile, previousLine = self._previousBegin + self._log.warn( + MORE_THAN_ONCE_WARNING, + fileNow=filename, lineNow=lineno, + fileThen=previousFile, lineThen=previousLine, + ) + + self._previousBegin = filename, lineno + if redirectStandardIO: + streams = [("stdout", LogLevel.info), ("stderr", LogLevel.error)] + else: + streams = [] + + for (stream, level) in streams: + oldStream = getattr(self._stdio, stream) + loggingFile = LoggingFile( + logger=Logger(namespace=stream, observer=self._publisher), + level=level, + encoding=getattr(oldStream, "encoding", None), + ) + setattr(self._stdio, stream, loggingFile) + + + def showwarning( + self, message, category, filename, lineno, file=None, line=None + ): + """ + Twisted-enabled wrapper around L{warnings.showwarning}. + + If C{file} is L{None}, the default behaviour is to emit the warning to + the log system, otherwise the original L{warnings.showwarning} Python + function is called. + + @param message: A warning message to emit. + @type message: L{str} + + @param category: A warning category to associate with C{message}. + @type category: L{warnings.Warning} + + @param filename: A file name for the source code file issuing the + warning. + @type warning: L{str} + + @param lineno: A line number in the source file where the warning was + issued. + @type lineno: L{int} + + @param file: A file to write the warning message to. If L{None}, + write to L{sys.stderr}. + @type file: file-like object + + @param line: A line of source code to include with the warning message. + If L{None}, attempt to read the line from C{filename} and + C{lineno}. + @type line: L{str} + """ + if file is None: + self._log.warn( + "{filename}:{lineno}: {category}: {warning}", + warning=message, category=qual(category), + filename=filename, lineno=lineno, + ) + else: + self._oldshowwarning( + message, category, filename, lineno, file, line + ) + + + +globalLogPublisher = LogPublisher() +globalLogBeginner = LogBeginner(globalLogPublisher, sys.stderr, sys, warnings) diff --git a/contrib/python/Twisted/py2/twisted/logger/_io.py b/contrib/python/Twisted/py2/twisted/logger/_io.py new file mode 100644 index 00000000000..bfb6cc6960a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_io.py @@ -0,0 +1,202 @@ +# -*- test-case-name: twisted.logger.test.test_io -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +File-like object that logs. +""" + +import sys + +from ._levels import LogLevel + + + +class LoggingFile(object): + """ + File-like object that turns C{write()} calls into logging events. + + Note that because event formats are C{unicode}, C{bytes} received via + C{write()} are converted to C{unicode}, which is the opposite of what + C{file} does. + + @ivar softspace: File-like L{'softspace' attribute }; zero + or one. + @type softspace: L{int} + """ + + softspace = 0 + + + def __init__(self, logger, level=LogLevel.info, encoding=None): + """ + @param logger: the logger to log through. + + @param level: the log level to emit events with. + + @param encoding: The encoding to expect when receiving bytes via + C{write()}. If L{None}, use C{sys.getdefaultencoding()}. + @type encoding: L{str} + + @param log: The logger to send events to. + @type log: L{Logger} + """ + self.level = level + self.log = logger + + if encoding is None: + self._encoding = sys.getdefaultencoding() + else: + self._encoding = encoding + + self._buffer = "" + self._closed = False + + + @property + def closed(self): + """ + Read-only property. Is the file closed? + + @return: true if closed, otherwise false. + @rtype: L{bool} + """ + return self._closed + + + @property + def encoding(self): + """ + Read-only property. File encoding. + + @return: an encoding. + @rtype: L{str} + """ + return self._encoding + + + @property + def mode(self): + """ + Read-only property. File mode. + + @return: "w" + @rtype: L{str} + """ + return "w" + + + @property + def newlines(self): + """ + Read-only property. Types of newlines encountered. + + @return: L{None} + @rtype: L{None} + """ + return None + + + @property + def name(self): + """ + The name of this file; a repr-style string giving information about its + namespace. + + @return: A file name. + @rtype: L{str} + """ + return ( + "<{0} {1}#{2}>".format( + self.__class__.__name__, + self.log.namespace, + self.level.name, + ) + ) + + + def close(self): + """ + Close this file so it can no longer be written to. + """ + self._closed = True + + + def flush(self): + """ + No-op; this file does not buffer. + """ + pass + + + def fileno(self): + """ + Returns an invalid file descriptor, since this is not backed by an FD. + + @return: C{-1} + @rtype: L{int} + """ + return -1 + + + def isatty(self): + """ + A L{LoggingFile} is not a TTY. + + @return: C{False} + @rtype: L{bool} + """ + return False + + + def write(self, string): + """ + Log the given message. + + @param string: Data to write. + @type string: L{bytes} in this file's preferred encoding or L{unicode} + """ + if self._closed: + raise ValueError("I/O operation on closed file") + + if isinstance(string, bytes): + string = string.decode(self._encoding) + + lines = (self._buffer + string).split("\n") + self._buffer = lines[-1] + lines = lines[0:-1] + + for line in lines: + self.log.emit(self.level, format=u"{log_io}", log_io=line) + + + def writelines(self, lines): + """ + Log each of the given lines as a separate message. + + @param lines: Data to write. + @type lines: iterable of L{unicode} or L{bytes} in this file's + declared encoding + """ + for line in lines: + self.write(line) + + + def _unsupported(self, *args): + """ + Template for unsupported operations. + + @param args: Arguments. + @type args: tuple of L{object} + """ + raise IOError("unsupported operation") + + + read = _unsupported + next = _unsupported + readline = _unsupported + readlines = _unsupported + xreadlines = _unsupported + seek = _unsupported + tell = _unsupported + truncate = _unsupported diff --git a/contrib/python/Twisted/py2/twisted/logger/_json.py b/contrib/python/Twisted/py2/twisted/logger/_json.py new file mode 100644 index 00000000000..d220e291410 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_json.py @@ -0,0 +1,355 @@ +# -*- test-case-name: twisted.logger.test.test_json -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Tools for saving and loading log events in a structured format. +""" + +import types + +from constantly import NamedConstant +from json import dumps, loads +from uuid import UUID + +from ._flatten import flattenEvent +from ._file import FileLogObserver +from ._levels import LogLevel +from ._logger import Logger + +from twisted.python.compat import unicode, _PY3 +from twisted.python.failure import Failure + +log = Logger() + + + +def failureAsJSON(failure): + """ + Convert a failure to a JSON-serializable data structure. + + @param failure: A failure to serialize. + @type failure: L{Failure} + + @return: a mapping of strings to ... stuff, mostly reminiscent of + L{Failure.__getstate__} + @rtype: L{dict} + """ + return dict( + failure.__getstate__(), + type=dict( + __module__=failure.type.__module__, + __name__=failure.type.__name__, + ) + ) + + + +def asBytes(obj): + """ + On Python 2, we really need native strings in a variety of places; + attribute names will sort of work in a __dict__, but they're subtly wrong; + however, printing tracebacks relies on I/O to containers that only support + bytes. This function converts _all_ native strings within a + JSON-deserialized object to bytes. + + @param obj: An object to convert to bytes. + @type obj: L{object} + + @return: A string of UTF-8 bytes. + @rtype: L{bytes} + """ + if isinstance(obj, list): + return map(asBytes, obj) + elif isinstance(obj, dict): + return dict((asBytes(k), asBytes(v)) for k, v in obj.items()) + elif isinstance(obj, unicode): + return obj.encode("utf-8") + else: + return obj + + + +def failureFromJSON(failureDict): + """ + Load a L{Failure} from a dictionary deserialized from JSON. + + @param failureDict: a JSON-deserialized object like one previously returned + by L{failureAsJSON}. + @type failureDict: L{dict} mapping L{unicode} to attributes + + @return: L{Failure} + @rtype: L{Failure} + """ + # InstanceType() is only available in Python 2 and lower. + # __new__ is only available on new-style classes. + newFailure = getattr(Failure, "__new__", None) + if newFailure is None: + f = types.InstanceType(Failure) + else: + f = newFailure(Failure) + + if not _PY3: + # Python 2 needs the failure dictionary as purely bytes, not text + failureDict = asBytes(failureDict) + + typeInfo = failureDict["type"] + failureDict["type"] = type(typeInfo["__name__"], (), typeInfo) + f.__dict__ = failureDict + return f + + + +classInfo = [ + ( + lambda level: ( + isinstance(level, NamedConstant) and + getattr(LogLevel, level.name, None) is level + ), + UUID("02E59486-F24D-46AD-8224-3ACDF2A5732A"), + lambda level: dict(name=level.name), + lambda level: getattr(LogLevel, level["name"], None) + ), + + ( + lambda o: isinstance(o, Failure), + UUID("E76887E2-20ED-49BF-A8F8-BA25CC586F2D"), + failureAsJSON, failureFromJSON + ), +] + + + +uuidToLoader = dict([ + (uuid, loader) for (predicate, uuid, saver, loader) in classInfo +]) + + + +def objectLoadHook(aDict): + """ + Dictionary-to-object-translation hook for certain value types used within + the logging system. + + @see: the C{object_hook} parameter to L{json.load} + + @param aDict: A dictionary loaded from a JSON object. + @type aDict: L{dict} + + @return: C{aDict} itself, or the object represented by C{aDict} + @rtype: L{object} + """ + if "__class_uuid__" in aDict: + return uuidToLoader[UUID(aDict["__class_uuid__"])](aDict) + return aDict + + + +def objectSaveHook(pythonObject): + """ + Object-to-serializable hook for certain value types used within the logging + system. + + @see: the C{default} parameter to L{json.dump} + + @param pythonObject: Any object. + @type pythonObject: L{object} + + @return: If the object is one of the special types the logging system + supports, a specially-formatted dictionary; otherwise, a marker + dictionary indicating that it could not be serialized. + """ + for (predicate, uuid, saver, loader) in classInfo: + if predicate(pythonObject): + result = saver(pythonObject) + result["__class_uuid__"] = str(uuid) + return result + return {"unpersistable": True} + + + +def eventAsJSON(event): + """ + Encode an event as JSON, flattening it if necessary to preserve as much + structure as possible. + + Not all structure from the log event will be preserved when it is + serialized. + + @param event: A log event dictionary. + @type event: L{dict} with arbitrary keys and values + + @return: A string of the serialized JSON; note that this will contain no + newline characters, and may thus safely be stored in a line-delimited + file. + @rtype: L{unicode} + """ + if bytes is str: + kw = dict(default=objectSaveHook, encoding="charmap", skipkeys=True) + else: + def default(unencodable): + """ + Serialize an object not otherwise serializable by L{dumps}. + + @param unencodable: An unencodable object. + @return: C{unencodable}, serialized + """ + if isinstance(unencodable, bytes): + return unencodable.decode("charmap") + return objectSaveHook(unencodable) + + kw = dict(default=default, skipkeys=True) + + flattenEvent(event) + result = dumps(event, **kw) + if not isinstance(result, unicode): + return unicode(result, "utf-8", "replace") + return result + + + +def eventFromJSON(eventText): + """ + Decode a log event from JSON. + + @param eventText: The output of a previous call to L{eventAsJSON} + @type eventText: L{unicode} + + @return: A reconstructed version of the log event. + @rtype: L{dict} + """ + loaded = loads(eventText, object_hook=objectLoadHook) + return loaded + + + +def jsonFileLogObserver(outFile, recordSeparator=u"\x1e"): + """ + Create a L{FileLogObserver} that emits JSON-serialized events to a + specified (writable) file-like object. + + Events are written in the following form:: + + RS + JSON + NL + + C{JSON} is the serialized event, which is JSON text. C{NL} is a newline + (C{u"\\n"}). C{RS} is a record separator. By default, this is a single + RS character (C{u"\\x1e"}), which makes the default output conform to the + IETF draft document "draft-ietf-json-text-sequence-13". + + @param outFile: A file-like object. Ideally one should be passed which + accepts L{unicode} data. Otherwise, UTF-8 L{bytes} will be used. + @type outFile: L{io.IOBase} + + @param recordSeparator: The record separator to use. + @type recordSeparator: L{unicode} + + @return: A file log observer. + @rtype: L{FileLogObserver} + """ + return FileLogObserver( + outFile, + lambda event: u"{0}{1}\n".format(recordSeparator, eventAsJSON(event)) + ) + + + +def eventsFromJSONLogFile(inFile, recordSeparator=None, bufferSize=4096): + """ + Load events from a file previously saved with L{jsonFileLogObserver}. + Event records that are truncated or otherwise unreadable are ignored. + + @param inFile: A (readable) file-like object. Data read from C{inFile} + should be L{unicode} or UTF-8 L{bytes}. + @type inFile: iterable of lines + + @param recordSeparator: The expected record separator. + If L{None}, attempt to automatically detect the record separator from + one of C{u"\\x1e"} or C{u""}. + @type recordSeparator: L{unicode} + + @param bufferSize: The size of the read buffer used while reading from + C{inFile}. + @type bufferSize: integer + + @return: Log events as read from C{inFile}. + @rtype: iterable of L{dict} + """ + def asBytes(s): + if type(s) is bytes: + return s + else: + return s.encode("utf-8") + + def eventFromBytearray(record): + try: + text = bytes(record).decode("utf-8") + except UnicodeDecodeError: + log.error( + u"Unable to decode UTF-8 for JSON record: {record!r}", + record=bytes(record) + ) + return None + + try: + return eventFromJSON(text) + except ValueError: + log.error( + u"Unable to read JSON record: {record!r}", + record=bytes(record) + ) + return None + + if recordSeparator is None: + first = asBytes(inFile.read(1)) + + if first == b"\x1e": + # This looks json-text-sequence compliant. + recordSeparator = first + else: + # Default to simpler newline-separated stream, which does not use + # a record separator. + recordSeparator = b"" + + else: + recordSeparator = asBytes(recordSeparator) + first = b"" + + if recordSeparator == b"": + recordSeparator = b"\n" # Split on newlines below + + eventFromRecord = eventFromBytearray + + else: + def eventFromRecord(record): + if record[-1] == ord("\n"): + return eventFromBytearray(record) + else: + log.error( + u"Unable to read truncated JSON record: {record!r}", + record=bytes(record) + ) + return None + + buffer = bytearray(first) + + while True: + newData = inFile.read(bufferSize) + + if not newData: + if len(buffer) > 0: + event = eventFromRecord(buffer) + if event is not None: + yield event + break + + buffer += asBytes(newData) + records = buffer.split(recordSeparator) + + for record in records[:-1]: + if len(record) > 0: + event = eventFromRecord(record) + if event is not None: + yield event + + buffer = records[-1] diff --git a/contrib/python/Twisted/py2/twisted/logger/_legacy.py b/contrib/python/Twisted/py2/twisted/logger/_legacy.py new file mode 100644 index 00000000000..b3e23d5281c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_legacy.py @@ -0,0 +1,154 @@ +# -*- test-case-name: twisted.logger.test.test_legacy -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Integration with L{twisted.python.log}. +""" + +from zope.interface import implementer + +from ._levels import LogLevel +from ._format import formatEvent +from ._observer import ILogObserver +from ._stdlib import fromStdlibLogLevelMapping, StringifiableFromEvent + + + +@implementer(ILogObserver) +class LegacyLogObserverWrapper(object): + """ + L{ILogObserver} that wraps an L{twisted.python.log.ILogObserver}. + + Received (new-style) events are modified prior to forwarding to + the legacy observer to ensure compatibility with observers that + expect legacy events. + """ + + def __init__(self, legacyObserver): + """ + @param legacyObserver: a legacy observer to which this observer will + forward events. + @type legacyObserver: L{twisted.python.log.ILogObserver} + """ + self.legacyObserver = legacyObserver + + + def __repr__(self): + return ( + "{self.__class__.__name__}({self.legacyObserver})" + .format(self=self) + ) + + + def __call__(self, event): + """ + Forward events to the legacy observer after editing them to + ensure compatibility. + + @param event: an event + @type event: L{dict} + """ + + # The "message" key is required by textFromEventDict() + if "message" not in event: + event["message"] = () + + if "time" not in event: + event["time"] = event["log_time"] + + if "system" not in event: + event["system"] = event.get("log_system", "-") + + # Format new style -> old style + if "format" not in event and event.get("log_format", None) is not None: + # Create an object that implements __str__() in order to defer the + # work of formatting until it's needed by a legacy log observer. + event["format"] = "%(log_legacy)s" + event["log_legacy"] = StringifiableFromEvent(event.copy()) + + # In the old-style system, the 'message' key always holds a tuple + # of messages. If we find the 'message' key here to not be a + # tuple, it has been passed as new-style parameter. We drop it + # here because we render it using the old-style 'format' key, + # which otherwise doesn't get precedence, and the original event + # has been copied above. + if not isinstance(event["message"], tuple): + event["message"] = () + + # From log.failure() -> isError blah blah + if "log_failure" in event: + if "failure" not in event: + event["failure"] = event["log_failure"] + if "isError" not in event: + event["isError"] = 1 + if "why" not in event: + event["why"] = formatEvent(event) + elif "isError" not in event: + if event["log_level"] in (LogLevel.error, LogLevel.critical): + event["isError"] = 1 + else: + event["isError"] = 0 + + self.legacyObserver(event) + + + +def publishToNewObserver(observer, eventDict, textFromEventDict): + """ + Publish an old-style (L{twisted.python.log}) event to a new-style + (L{twisted.logger}) observer. + + @note: It's possible that a new-style event was sent to a + L{LegacyLogObserverWrapper}, and may now be getting sent back to a + new-style observer. In this case, it's already a new-style event, + adapted to also look like an old-style event, and we don't need to + tweak it again to be a new-style event, hence the checks for + already-defined new-style keys. + + @param observer: A new-style observer to handle this event. + @type observer: L{ILogObserver} + + @param eventDict: An L{old-style }, log event. + @type eventDict: L{dict} + + @param textFromEventDict: callable that can format an old-style event as a + string. Passed here rather than imported to avoid circular dependency. + @type textFromEventDict: 1-arg L{callable} taking L{dict} returning L{str} + + @return: L{None} + """ + + if "log_time" not in eventDict: + eventDict["log_time"] = eventDict["time"] + + if "log_format" not in eventDict: + text = textFromEventDict(eventDict) + if text is not None: + eventDict["log_text"] = text + eventDict["log_format"] = u"{log_text}" + + if "log_level" not in eventDict: + if "logLevel" in eventDict: + try: + level = fromStdlibLogLevelMapping[eventDict["logLevel"]] + except KeyError: + level = None + elif "isError" in eventDict: + if eventDict["isError"]: + level = LogLevel.critical + else: + level = LogLevel.info + else: + level = LogLevel.info + + if level is not None: + eventDict["log_level"] = level + + if "log_namespace" not in eventDict: + eventDict["log_namespace"] = u"log_legacy" + + if "log_system" not in eventDict and "system" in eventDict: + eventDict["log_system"] = eventDict["system"] + + observer(eventDict) diff --git a/contrib/python/Twisted/py2/twisted/logger/_levels.py b/contrib/python/Twisted/py2/twisted/logger/_levels.py new file mode 100644 index 00000000000..7cf81835faa --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_levels.py @@ -0,0 +1,110 @@ +# -*- test-case-name: twisted.logger.test.test_levels -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Log levels. +""" + +from constantly import NamedConstant, Names + + + +class InvalidLogLevelError(Exception): + """ + Someone tried to use a L{LogLevel} that is unknown to the logging system. + """ + def __init__(self, level): + """ + @param level: A log level. + @type level: L{LogLevel} + """ + super(InvalidLogLevelError, self).__init__(str(level)) + self.level = level + + + +class LogLevel(Names): + """ + Constants describing log levels. + + @cvar debug: Debugging events: Information of use to a developer of the + software, not generally of interest to someone running the software + unless they are attempting to diagnose a software issue. + + @cvar info: Informational events: Routine information about the status of + an application, such as incoming connections, startup of a subsystem, + etc. + + @cvar warn: Warning events: Events that may require greater attention than + informational events but are not a systemic failure condition, such as + authorization failures, bad data from a network client, etc. Such + events are of potential interest to system administrators, and should + ideally be phrased in such a way, or documented, so as to indicate an + action that an administrator might take to mitigate the warning. + + @cvar error: Error conditions: Events indicating a systemic failure, such + as programming errors in the form of unhandled exceptions, loss of + connectivity to an external system without which no useful work can + proceed, such as a database or API endpoint, or resource exhaustion. + Similarly to warnings, errors that are related to operational + parameters may be actionable to system administrators and should + provide references to resources which an administrator might use to + resolve them. + + @cvar critical: Critical failures: Errors indicating systemic failure (ie. + service outage), data corruption, imminent data loss, etc. which must + be handled immediately. This includes errors unanticipated by the + software, such as unhandled exceptions, wherein the cause and + consequences are unknown. + """ + + debug = NamedConstant() + info = NamedConstant() + warn = NamedConstant() + error = NamedConstant() + critical = NamedConstant() + + + @classmethod + def levelWithName(cls, name): + """ + Get the log level with the given name. + + @param name: The name of a log level. + @type name: L{str} (native string) + + @return: The L{LogLevel} with the specified C{name}. + @rtype: L{LogLevel} + + @raise InvalidLogLevelError: if the C{name} does not name a valid log + level. + """ + try: + return cls.lookupByName(name) + except ValueError: + raise InvalidLogLevelError(name) + + + @classmethod + def _priorityForLevel(cls, level): + """ + We want log levels to have defined ordering - the order of definition - + but they aren't value constants (the only value is the name). This is + arguably a bug in Twisted, so this is just a workaround for U{until + this is fixed in some way + }. + + @param level: A log level. + @type level: L{LogLevel} + + @return: A numeric index indicating priority (lower is higher level). + @rtype: L{int} + """ + return cls._levelPriorities[level] + + +LogLevel._levelPriorities = dict( + (level, index) for (index, level) in + (enumerate(LogLevel.iterconstants())) +) diff --git a/contrib/python/Twisted/py2/twisted/logger/_logger.py b/contrib/python/Twisted/py2/twisted/logger/_logger.py new file mode 100644 index 00000000000..8252250bc2e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_logger.py @@ -0,0 +1,275 @@ +# -*- test-case-name: twisted.logger.test.test_logger -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Logger class. +""" + +from time import time + +from twisted.python.compat import currentframe +from twisted.python.failure import Failure +from ._levels import InvalidLogLevelError, LogLevel + + + +class Logger(object): + """ + A L{Logger} emits log messages to an observer. You should instantiate it + as a class or module attribute, as documented in L{this module's + documentation }. + + @type namespace: L{str} + @ivar namespace: the namespace for this logger + + @type source: L{object} + @ivar source: The object which is emitting events via this logger + + @type: L{ILogObserver} + @ivar observer: The observer that this logger will send events to. + """ + + @staticmethod + def _namespaceFromCallingContext(): + """ + Derive a namespace from the module containing the caller's caller. + + @return: the fully qualified python name of a module. + @rtype: L{str} (native string) + """ + try: + return currentframe(2).f_globals["__name__"] + except KeyError: + return "" + + + def __init__(self, namespace=None, source=None, observer=None): + """ + @param namespace: The namespace for this logger. Uses a dotted + notation, as used by python modules. If not L{None}, then the name + of the module of the caller is used. + @type namespace: L{str} (native string) + + @param source: The object which is emitting events via this + logger; this is automatically set on instances of a class + if this L{Logger} is an attribute of that class. + @type source: L{object} + + @param observer: The observer that this logger will send events to. + If L{None}, use the L{global log publisher }. + @type observer: L{ILogObserver} + """ + if namespace is None: + namespace = self._namespaceFromCallingContext() + + self.namespace = namespace + self.source = source + + if observer is None: + from ._global import globalLogPublisher + self.observer = globalLogPublisher + else: + self.observer = observer + + + def __get__(self, oself, type=None): + """ + When used as a descriptor, i.e.:: + + # File: athing.py + class Something(object): + log = Logger() + def hello(self): + self.log.info("Hello") + + a L{Logger}'s namespace will be set to the name of the class it is + declared on. In the above example, the namespace would be + C{athing.Something}. + + Additionally, its source will be set to the actual object referring to + the L{Logger}. In the above example, C{Something.log.source} would be + C{Something}, and C{Something().log.source} would be an instance of + C{Something}. + """ + if oself is None: + source = type + else: + source = oself + + return self.__class__( + ".".join([type.__module__, type.__name__]), + source, + observer=self.observer, + ) + + + def __repr__(self): + return "<%s %r>" % (self.__class__.__name__, self.namespace) + + + def emit(self, level, format=None, **kwargs): + """ + Emit a log event to all log observers at the given level. + + @param level: a L{LogLevel} + + @param format: a message format using new-style (PEP 3101) + formatting. The logging event (which is a L{dict}) is + used to render this format string. + + @param kwargs: additional key/value pairs to include in the event. + Note that values which are later mutated may result in + non-deterministic behavior from observers that schedule work for + later execution. + """ + if level not in LogLevel.iterconstants(): + self.failure( + "Got invalid log level {invalidLevel!r} in {logger}.emit().", + Failure(InvalidLogLevelError(level)), + invalidLevel=level, + logger=self, + ) + return + + event = kwargs + event.update( + log_logger=self, log_level=level, log_namespace=self.namespace, + log_source=self.source, log_format=format, log_time=time(), + ) + + if "log_trace" in event: + event["log_trace"].append((self, self.observer)) + + self.observer(event) + + + def failure(self, format, failure=None, level=LogLevel.critical, **kwargs): + """ + Log a failure and emit a traceback. + + For example:: + + try: + frob(knob) + except Exception: + log.failure("While frobbing {knob}", knob=knob) + + or:: + + d = deferredFrob(knob) + d.addErrback(lambda f: log.failure("While frobbing {knob}", + f, knob=knob)) + + This method is generally meant to capture unexpected exceptions in + code; an exception that is caught and handled somehow should be logged, + if appropriate, via L{Logger.error} instead. If some unknown exception + occurs and your code doesn't know how to handle it, as in the above + example, then this method provides a means to describe the failure in + nerd-speak. This is done at L{LogLevel.critical} by default, since no + corrective guidance can be offered to an user/administrator, and the + impact of the condition is unknown. + + @param format: a message format using new-style (PEP 3101) formatting. + The logging event (which is a L{dict}) is used to render this + format string. + + @param failure: a L{Failure} to log. If L{None}, a L{Failure} is + created from the exception in flight. + + @param level: a L{LogLevel} to use. + + @param kwargs: additional key/value pairs to include in the event. + Note that values which are later mutated may result in + non-deterministic behavior from observers that schedule work for + later execution. + """ + if failure is None: + failure = Failure() + + self.emit(level, format, log_failure=failure, **kwargs) + + + def debug(self, format=None, **kwargs): + """ + Emit a log event at log level L{LogLevel.debug}. + + @param format: a message format using new-style (PEP 3101) formatting. + The logging event (which is a L{dict}) is used to render this + format string. + + @param kwargs: additional key/value pairs to include in the event. + Note that values which are later mutated may result in + non-deterministic behavior from observers that schedule work for + later execution. + """ + self.emit(LogLevel.debug, format, **kwargs) + + + def info(self, format=None, **kwargs): + """ + Emit a log event at log level L{LogLevel.info}. + + @param format: a message format using new-style (PEP 3101) formatting. + The logging event (which is a L{dict}) is used to render this + format string. + + @param kwargs: additional key/value pairs to include in the event. + Note that values which are later mutated may result in + non-deterministic behavior from observers that schedule work for + later execution. + """ + self.emit(LogLevel.info, format, **kwargs) + + + def warn(self, format=None, **kwargs): + """ + Emit a log event at log level L{LogLevel.warn}. + + @param format: a message format using new-style (PEP 3101) formatting. + The logging event (which is a L{dict}) is used to render this + format string. + + @param kwargs: additional key/value pairs to include in the event. + Note that values which are later mutated may result in + non-deterministic behavior from observers that schedule work for + later execution. + """ + self.emit(LogLevel.warn, format, **kwargs) + + + def error(self, format=None, **kwargs): + """ + Emit a log event at log level L{LogLevel.error}. + + @param format: a message format using new-style (PEP 3101) formatting. + The logging event (which is a L{dict}) is used to render this + format string. + + @param kwargs: additional key/value pairs to include in the event. + Note that values which are later mutated may result in + non-deterministic behavior from observers that schedule work for + later execution. + """ + self.emit(LogLevel.error, format, **kwargs) + + + def critical(self, format=None, **kwargs): + """ + Emit a log event at log level L{LogLevel.critical}. + + @param format: a message format using new-style (PEP 3101) formatting. + The logging event (which is a L{dict}) is used to render this + format string. + + @param kwargs: additional key/value pairs to include in the event. + Note that values which are later mutated may result in + non-deterministic behavior from observers that schedule work for + later execution. + """ + self.emit(LogLevel.critical, format, **kwargs) + + + +_log = Logger() +_loggerFor = lambda obj:_log.__get__(obj, obj.__class__) diff --git a/contrib/python/Twisted/py2/twisted/logger/_observer.py b/contrib/python/Twisted/py2/twisted/logger/_observer.py new file mode 100644 index 00000000000..d1c03a51dc0 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_observer.py @@ -0,0 +1,158 @@ +# -*- test-case-name: twisted.logger.test.test_observer -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Basic log observers. +""" + +from zope.interface import Interface, implementer + +from twisted.python.failure import Failure +from ._logger import Logger + + + +OBSERVER_DISABLED = ( + "Temporarily disabling observer {observer} due to exception: {log_failure}" +) + + + +class ILogObserver(Interface): + """ + An observer which can handle log events. + + Unlike most interfaces within Twisted, an L{ILogObserver} I{must be + thread-safe}. Log observers may be called indiscriminately from many + different threads, as any thread may wish to log a message at any time. + """ + + def __call__(event): + """ + Log an event. + + @type event: C{dict} with (native) C{str} keys. + @param event: A dictionary with arbitrary keys as defined by the + application emitting logging events, as well as keys added by the + logging system. The logging system reserves the right to set any + key beginning with the prefix C{"log_"}; applications should not + use any key so named. Currently, the following keys are used by + the logging system in some way, if they are present (they are all + optional): + + - C{"log_format"}: a PEP-3101-style format string which draws + upon the keys in the event as its values, used to format the + event for human consumption. + + - C{"log_flattened"}: a dictionary mapping keys derived from + the names and format values used in the C{"log_format"} + string to their values. This is used to preserve some + structured information for use with + L{twisted.logger.extractField}. + + - C{"log_trace"}: A L{list} designed to capture information + about which L{LogPublisher}s have observed the event. + + - C{"log_level"}: a L{log level + } constant, indicating the + importance of and audience for this event. + + - C{"log_namespace"}: a namespace for the emitter of the event, + given as a unicode string. + + - C{"log_system"}: a string indicating the network event or + method call which resulted in the message being logged. + """ + + + +@implementer(ILogObserver) +class LogPublisher(object): + """ + I{ILogObserver} that fans out events to other observers. + + Keeps track of a set of L{ILogObserver} objects and forwards + events to each. + """ + + def __init__(self, *observers): + self._observers = list(observers) + self.log = Logger(observer=self) + + + def addObserver(self, observer): + """ + Registers an observer with this publisher. + + @param observer: An L{ILogObserver} to add. + """ + if not callable(observer): + raise TypeError("Observer is not callable: {0!r}".format(observer)) + if observer not in self._observers: + self._observers.append(observer) + + + def removeObserver(self, observer): + """ + Unregisters an observer with this publisher. + + @param observer: An L{ILogObserver} to remove. + """ + try: + self._observers.remove(observer) + except ValueError: + pass + + + def __call__(self, event): + """ + Forward events to contained observers. + """ + if "log_trace" in event: + def trace(observer): + """ + Add tracing information for an observer. + + @param observer: an observer being forwarded to + @type observer: L{ILogObserver} + """ + event["log_trace"].append((self, observer)) + else: + trace = None + + brokenObservers = [] + + for observer in self._observers: + if trace is not None: + trace(observer) + + try: + observer(event) + except Exception: + brokenObservers.append((observer, Failure())) + + for brokenObserver, failure in brokenObservers: + errorLogger = self._errorLoggerForObserver(brokenObserver) + errorLogger.failure( + OBSERVER_DISABLED, + failure=failure, + observer=brokenObserver, + ) + + + def _errorLoggerForObserver(self, observer): + """ + Create an error-logger based on this logger, which does not contain the + given bad observer. + + @param observer: The observer which previously had an error. + @type observer: L{ILogObserver} + + @return: L{None} + """ + errorPublisher = LogPublisher(*[ + obs for obs in self._observers + if obs is not observer + ]) + return Logger(observer=errorPublisher) diff --git a/contrib/python/Twisted/py2/twisted/logger/_stdlib.py b/contrib/python/Twisted/py2/twisted/logger/_stdlib.py new file mode 100644 index 00000000000..07b65923c3d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_stdlib.py @@ -0,0 +1,147 @@ +# -*- test-case-name: twisted.logger.test.test_stdlib -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Integration with Python standard library logging. +""" + +import logging as stdlibLogging + +from zope.interface import implementer + +from twisted.python.compat import _PY3, currentframe, unicode +from ._levels import LogLevel +from ._format import formatEvent +from ._observer import ILogObserver + + + +# Mappings to Python's logging module +toStdlibLogLevelMapping = { + LogLevel.debug: stdlibLogging.DEBUG, + LogLevel.info: stdlibLogging.INFO, + LogLevel.warn: stdlibLogging.WARNING, + LogLevel.error: stdlibLogging.ERROR, + LogLevel.critical: stdlibLogging.CRITICAL, +} + +def _reverseLogLevelMapping(): + """ + Reverse the above mapping, adding both the numerical keys used above and + the corresponding string keys also used by python logging. + @return: the reversed mapping + """ + mapping = {} + for logLevel, pyLogLevel in toStdlibLogLevelMapping.items(): + mapping[pyLogLevel] = logLevel + mapping[stdlibLogging.getLevelName(pyLogLevel)] = logLevel + return mapping + +fromStdlibLogLevelMapping = _reverseLogLevelMapping() + + + +@implementer(ILogObserver) +class STDLibLogObserver(object): + """ + Log observer that writes to the python standard library's C{logging} + module. + + @note: Warning: specific logging configurations (example: network) can lead + to this observer blocking. Nothing is done here to prevent that, so be + sure to not to configure the standard library logging module to block + when used in conjunction with this module: code within Twisted, such as + twisted.web, assumes that logging does not block. + + @cvar defaultStackDepth: This is the default number of frames that it takes + to get from L{STDLibLogObserver} through the logging module, plus one; + in other words, the number of frames if you were to call a + L{STDLibLogObserver} directly. This is useful to use as an offset for + the C{stackDepth} parameter to C{__init__}, to add frames for other + publishers. + """ + + defaultStackDepth = 4 + + def __init__(self, name="twisted", stackDepth=defaultStackDepth): + """ + @param loggerName: logger identifier. + @type loggerName: C{str} + + @param stackDepth: The depth of the stack to investigate for caller + metadata. + @type stackDepth: L{int} + """ + self.logger = stdlibLogging.getLogger(name) + self.logger.findCaller = self._findCaller + self.stackDepth = stackDepth + + + def _findCaller(self, stackInfo=False, stackLevel=1): + """ + Based on the stack depth passed to this L{STDLibLogObserver}, identify + the calling function. + + @param stackInfo: Whether or not to construct stack information. + (Currently ignored.) + @type stackInfo: L{bool} + + @param stackLevel: The number of stack frames to skip when determining + the caller (currently ignored; use stackDepth on the instance). + @type stackLevel: L{int} + + @return: Depending on Python version, either a 3-tuple of (filename, + lineno, name) or a 4-tuple of that plus stack information. + @rtype: L{tuple} + """ + f = currentframe(self.stackDepth) + co = f.f_code + if _PY3: + extra = (None,) + else: + extra = () + return (co.co_filename, f.f_lineno, co.co_name) + extra + + + def __call__(self, event): + """ + Format an event and bridge it to Python logging. + """ + level = event.get("log_level", LogLevel.info) + failure = event.get('log_failure') + if failure is None: + excInfo = None + else: + excInfo = ( + failure.type, failure.value, failure.getTracebackObject()) + stdlibLevel = toStdlibLogLevelMapping.get(level, stdlibLogging.INFO) + self.logger.log( + stdlibLevel, StringifiableFromEvent(event), exc_info=excInfo) + + + +class StringifiableFromEvent(object): + """ + An object that implements C{__str__()} in order to defer the work of + formatting until it's converted into a C{str}. + """ + def __init__(self, event): + """ + @param event: An event. + @type event: L{dict} + """ + self.event = event + + + def __unicode__(self): + return formatEvent(self.event) + + + def __bytes__(self): + return unicode(self).encode("utf-8") + + if _PY3: + __str__ = __unicode__ + else: + __str__ = __bytes__ diff --git a/contrib/python/Twisted/py2/twisted/logger/_util.py b/contrib/python/Twisted/py2/twisted/logger/_util.py new file mode 100644 index 00000000000..84c45a857b8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/logger/_util.py @@ -0,0 +1,48 @@ +# -*- test-case-name: twisted.logger.test.test_util -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Logging utilities. +""" + + + +def formatTrace(trace): + """ + Format a trace (that is, the contents of the C{log_trace} key of a log + event) as a visual indication of the message's propagation through various + observers. + + @param trace: the contents of the C{log_trace} key from an event. + @type trace: object + + @return: A multi-line string with indentation and arrows indicating the + flow of the message through various observers. + @rtype: L{unicode} + """ + def formatWithName(obj): + if hasattr(obj, "name"): + return u"{0} ({1})".format(obj, obj.name) + else: + return u"{0}".format(obj) + + result = [] + lineage = [] + + for parent, child in trace: + if not lineage or lineage[-1] is not parent: + if parent in lineage: + while lineage[-1] is not parent: + lineage.pop() + + else: + if not lineage: + result.append(u"{0}\n".format(formatWithName(parent))) + + lineage.append(parent) + + result.append(u" " * len(lineage)) + result.append(u"-> {0}\n".format(formatWithName(child))) + + return u"".join(result) diff --git a/contrib/python/Twisted/py2/twisted/mail/__init__.py b/contrib/python/Twisted/py2/twisted/mail/__init__.py new file mode 100644 index 00000000000..0f1604e8a5f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Mail: Servers and clients for POP3, ESMTP, and IMAP. +""" diff --git a/contrib/python/Twisted/py2/twisted/mail/_cred.py b/contrib/python/Twisted/py2/twisted/mail/_cred.py new file mode 100644 index 00000000000..1fd2a33cf8f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/_cred.py @@ -0,0 +1,122 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Credential managers for L{twisted.mail}. +""" + +from __future__ import absolute_import, division + +import hmac +import hashlib + +from zope.interface import implementer + +from twisted.cred import credentials +from twisted.python.compat import nativeString +from twisted.mail._except import IllegalClientResponse +from twisted.mail.interfaces import IClientAuthentication, IChallengeResponse + + +@implementer(IClientAuthentication) +class CramMD5ClientAuthenticator: + def __init__(self, user): + self.user = user + + + def getName(self): + return b"CRAM-MD5" + + + def challengeResponse(self, secret, chal): + response = hmac.HMAC(secret, chal, digestmod=hashlib.md5).hexdigest() + return self.user + b' ' + response.encode('ascii') + + + +@implementer(IClientAuthentication) +class LOGINAuthenticator: + def __init__(self, user): + self.user = user + self.challengeResponse = self.challengeUsername + + + def getName(self): + return b"LOGIN" + + + def challengeUsername(self, secret, chal): + # Respond to something like "Username:" + self.challengeResponse = self.challengeSecret + return self.user + + + def challengeSecret(self, secret, chal): + # Respond to something like "Password:" + return secret + + + +@implementer(IClientAuthentication) +class PLAINAuthenticator: + def __init__(self, user): + self.user = user + + + def getName(self): + return b"PLAIN" + + + def challengeResponse(self, secret, chal): + return b'\0' + self.user + b'\0' + secret + + + +@implementer(IChallengeResponse) +class LOGINCredentials(credentials.UsernamePassword): + def __init__(self): + self.challenges = [b'Password\0', b'User Name\0'] + self.responses = [b'password', b'username'] + credentials.UsernamePassword.__init__(self, None, None) + + + def getChallenge(self): + return self.challenges.pop() + + + def setResponse(self, response): + setattr(self, nativeString(self.responses.pop()), response) + + + def moreChallenges(self): + return bool(self.challenges) + + + +@implementer(IChallengeResponse) +class PLAINCredentials(credentials.UsernamePassword): + def __init__(self): + credentials.UsernamePassword.__init__(self, None, None) + + + def getChallenge(self): + return b'' + + + def setResponse(self, response): + parts = response.split(b'\0') + if len(parts) != 3: + raise IllegalClientResponse( + "Malformed Response - wrong number of parts") + useless, self.username, self.password = parts + + + def moreChallenges(self): + return False + + +__all__ = [ + "CramMD5ClientAuthenticator", + "LOGINCredentials", "LOGINAuthenticator", + "PLAINCredentials", "PLAINAuthenticator", +] diff --git a/contrib/python/Twisted/py2/twisted/mail/_except.py b/contrib/python/Twisted/py2/twisted/mail/_except.py new file mode 100644 index 00000000000..60db6121c48 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/_except.py @@ -0,0 +1,392 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Exceptions in L{twisted.mail}. +""" + +from __future__ import absolute_import, division + +from twisted.python.compat import _PY3, unicode + + +class IMAP4Exception(Exception): + pass + + + +class IllegalClientResponse(IMAP4Exception): + pass + + + +class IllegalOperation(IMAP4Exception): + pass + + + +class IllegalMailboxEncoding(IMAP4Exception): + pass + + + +class MailboxException(IMAP4Exception): + pass + + + +class MailboxCollision(MailboxException): + def __str__(self): + return 'Mailbox named %s already exists' % self.args + + + +class NoSuchMailbox(MailboxException): + def __str__(self): + return 'No mailbox named %s exists' % self.args + + + +class ReadOnlyMailbox(MailboxException): + def __str__(self): + return 'Mailbox open in read-only state' + + +class UnhandledResponse(IMAP4Exception): + pass + + + +class NegativeResponse(IMAP4Exception): + pass + + + +class NoSupportedAuthentication(IMAP4Exception): + def __init__(self, serverSupports, clientSupports): + IMAP4Exception.__init__( + self, 'No supported authentication schemes available') + self.serverSupports = serverSupports + self.clientSupports = clientSupports + + def __str__(self): + return (IMAP4Exception.__str__(self) + + ': Server supports %r, client supports %r' + % (self.serverSupports, self.clientSupports)) + + + +class IllegalServerResponse(IMAP4Exception): + pass + + + +class IllegalIdentifierError(IMAP4Exception): + pass + + + +class IllegalQueryError(IMAP4Exception): + pass + + + +class MismatchedNesting(IMAP4Exception): + pass + + + +class MismatchedQuoting(IMAP4Exception): + pass + + + +class SMTPError(Exception): + pass + + + +class SMTPClientError(SMTPError): + """ + Base class for SMTP client errors. + """ + def __init__(self, code, resp, log=None, addresses=None, isFatal=False, + retry=False): + """ + @param code: The SMTP response code associated with this error. + + @param resp: The string response associated with this error. + + @param log: A string log of the exchange leading up to and including + the error. + @type log: L{bytes} + + @param isFatal: A boolean indicating whether this connection can + proceed or not. If True, the connection will be dropped. + + @param retry: A boolean indicating whether the delivery should be + retried. If True and the factory indicates further retries are + desirable, they will be attempted, otherwise the delivery will be + failed. + """ + self.code = code + self.resp = resp + self.log = log + self.addresses = addresses + self.isFatal = isFatal + self.retry = retry + + + def __str__(self): + if _PY3: + return self.__bytes__().decode("utf-8") + else: + return self.__bytes__() + + + def __bytes__(self): + if self.code > 0: + res = [u"{:03d} {}".format(self.code, self.resp)] + else: + res = [self.resp] + if self.log: + res.append(self.log) + res.append(b'') + for (i, r) in enumerate(res): + if isinstance(r, unicode): + res[i] = r.encode('utf-8') + return b'\n'.join(res) + + + +class ESMTPClientError(SMTPClientError): + """ + Base class for ESMTP client errors. + """ + + + +class EHLORequiredError(ESMTPClientError): + """ + The server does not support EHLO. + + This is considered a non-fatal error (the connection will not be dropped). + """ + + + +class AUTHRequiredError(ESMTPClientError): + """ + Authentication was required but the server does not support it. + + This is considered a non-fatal error (the connection will not be dropped). + """ + + + +class TLSRequiredError(ESMTPClientError): + """ + Transport security was required but the server does not support it. + + This is considered a non-fatal error (the connection will not be dropped). + """ + + + +class AUTHDeclinedError(ESMTPClientError): + """ + The server rejected our credentials. + + Either the username, password, or challenge response + given to the server was rejected. + + This is considered a non-fatal error (the connection will not be + dropped). + """ + + + +class AuthenticationError(ESMTPClientError): + """ + An error occurred while authenticating. + + Either the server rejected our request for authentication or the + challenge received was malformed. + + This is considered a non-fatal error (the connection will not be + dropped). + """ + + + +class SMTPTLSError(ESMTPClientError): + """ + An error occurred while negiotiating for transport security. + + This is considered a non-fatal error (the connection will not be dropped). + """ + + + +class SMTPConnectError(SMTPClientError): + """ + Failed to connect to the mail exchange host. + + This is considered a fatal error. A retry will be made. + """ + def __init__(self, code, resp, log=None, addresses=None, isFatal=True, + retry=True): + SMTPClientError.__init__(self, code, resp, log, addresses, isFatal, + retry) + + + +class SMTPTimeoutError(SMTPClientError): + """ + Failed to receive a response from the server in the expected time period. + + This is considered a fatal error. A retry will be made. + """ + def __init__(self, code, resp, log=None, addresses=None, isFatal=True, + retry=True): + SMTPClientError.__init__(self, code, resp, log, addresses, isFatal, + retry) + + + +class SMTPProtocolError(SMTPClientError): + """ + The server sent a mangled response. + + This is considered a fatal error. A retry will not be made. + """ + def __init__(self, code, resp, log=None, addresses=None, isFatal=True, + retry=False): + SMTPClientError.__init__(self, code, resp, log, addresses, isFatal, + retry) + + + +class SMTPDeliveryError(SMTPClientError): + """ + Indicates that a delivery attempt has had an error. + """ + + + +class SMTPServerError(SMTPError): + def __init__(self, code, resp): + self.code = code + self.resp = resp + + + def __str__(self): + return "%.3d %s" % (self.code, self.resp) + + + +class SMTPAddressError(SMTPServerError): + def __init__(self, addr, code, resp): + from twisted.mail.smtp import Address + + SMTPServerError.__init__(self, code, resp) + self.addr = Address(addr) + + + def __str__(self): + return "%.3d <%s>... %s" % (self.code, self.addr, self.resp) + + + +class SMTPBadRcpt(SMTPAddressError): + def __init__(self, addr, code=550, + resp='Cannot receive for specified address'): + SMTPAddressError.__init__(self, addr, code, resp) + + + +class SMTPBadSender(SMTPAddressError): + def __init__(self, addr, code=550, resp='Sender not acceptable'): + SMTPAddressError.__init__(self, addr, code, resp) + + + +class AddressError(SMTPError): + """ + Parse error in address + """ + + +class POP3Error(Exception): + """ + The base class for POP3 errors. + """ + pass + + + +class _POP3MessageDeleted(Exception): + """ + An internal control-flow error which indicates that a deleted message was + requested. + """ + + + +class POP3ClientError(Exception): + """ + The base class for all exceptions raised by POP3Client. + """ + + + +class InsecureAuthenticationDisallowed(POP3ClientError): + """ + An error indicating secure authentication was required but no mechanism + could be found. + """ + + + +class TLSError(POP3ClientError): + """ + An error indicating secure authentication was required but either the + transport does not support TLS or no TLS context factory was supplied. + """ + + + +class TLSNotSupportedError(POP3ClientError): + """ + An error indicating secure authentication was required but the server does + not support TLS. + """ + + + +class ServerErrorResponse(POP3ClientError): + """ + An error indicating that the server returned an error response to a + request. + + @ivar consumer: See L{__init__} + """ + def __init__(self, reason, consumer=None): + """ + @type reason: L{bytes} + @param reason: The server response minus the status indicator. + + @type consumer: callable that takes L{object} + @param consumer: The function meant to handle the values for a + multi-line response. + """ + POP3ClientError.__init__(self, reason) + self.consumer = consumer + + + +class LineTooLong(POP3ClientError): + """ + An error indicating that the server sent a line which exceeded the + maximum line length (L{LineOnlyReceiver.MAX_LENGTH}). + """ diff --git a/contrib/python/Twisted/py2/twisted/mail/alias.py b/contrib/python/Twisted/py2/twisted/mail/alias.py new file mode 100644 index 00000000000..32229bda7d2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/alias.py @@ -0,0 +1,799 @@ +# -*- test-case-name: twisted.mail.test.test_mail -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Support for aliases(5) configuration files. + +@author: Jp Calderone +""" + +import os +import tempfile + +from twisted.mail import smtp +from twisted.mail.interfaces import IAlias +from twisted.internet import reactor +from twisted.internet import protocol +from twisted.internet import defer +from twisted.python import failure +from twisted.python import log +from zope.interface import implementer + + +def handle(result, line, filename, lineNo): + """ + Parse a line from an aliases file. + + @type result: L{dict} mapping L{bytes} to L{list} of L{bytes} + @param result: A dictionary mapping username to aliases to which + the results of parsing the line are added. + + @type line: L{bytes} + @param line: A line from an aliases file. + + @type filename: L{bytes} + @param filename: The full or relative path to the aliases file. + + @type lineNo: L{int} + @param lineNo: The position of the line within the aliases file. + """ + parts = [p.strip() for p in line.split(':', 1)] + if len(parts) != 2: + fmt = "Invalid format on line %d of alias file %s." + arg = (lineNo, filename) + log.err(fmt % arg) + else: + user, alias = parts + result.setdefault(user.strip(), []).extend(map(str.strip, alias.split(','))) + + + +def loadAliasFile(domains, filename=None, fp=None): + """ + Load a file containing email aliases. + + Lines in the file should be formatted like so:: + + username: alias1, alias2, ..., aliasN + + Aliases beginning with a C{|} will be treated as programs, will be run, and + the message will be written to their stdin. + + Aliases beginning with a C{:} will be treated as a file containing + additional aliases for the username. + + Aliases beginning with a C{/} will be treated as the full pathname to a file + to which the message will be appended. + + Aliases without a host part will be assumed to be addresses on localhost. + + If a username is specified multiple times, the aliases for each are joined + together as if they had all been on one line. + + Lines beginning with a space or a tab are continuations of the previous + line. + + Lines beginning with a C{#} are comments. + + @type domains: L{dict} mapping L{bytes} to L{IDomain} provider + @param domains: A mapping of domain name to domain object. + + @type filename: L{bytes} or L{None} + @param filename: The full or relative path to a file from which to load + aliases. If omitted, the C{fp} parameter must be specified. + + @type fp: file-like object or L{None} + @param fp: The file from which to load aliases. If specified, + the C{filename} parameter is ignored. + + @rtype: L{dict} mapping L{bytes} to L{AliasGroup} + @return: A mapping from username to group of aliases. + """ + result = {} + close = False + if fp is None: + fp = open(filename) + close = True + else: + filename = getattr(fp, 'name', '') + i = 0 + prev = '' + try: + for line in fp: + i += 1 + line = line.rstrip() + if line.lstrip().startswith('#'): + continue + elif line.startswith(' ') or line.startswith('\t'): + prev = prev + line + else: + if prev: + handle(result, prev, filename, i) + prev = line + finally: + if close: + fp.close() + if prev: + handle(result, prev, filename, i) + for (u, a) in result.items(): + result[u] = AliasGroup(a, domains, u) + return result + + + +class AliasBase: + """ + The default base class for aliases. + + @ivar domains: See L{__init__}. + + @type original: L{Address} + @ivar original: The original address being aliased. + """ + def __init__(self, domains, original): + """ + @type domains: L{dict} mapping L{bytes} to L{IDomain} provider + @param domains: A mapping of domain name to domain object. + + @type original: L{bytes} + @param original: The original address being aliased. + """ + self.domains = domains + self.original = smtp.Address(original) + + + def domain(self): + """ + Return the domain associated with original address. + + @rtype: L{IDomain} provider + @return: The domain for the original address. + """ + return self.domains[self.original.domain] + + + def resolve(self, aliasmap, memo=None): + """ + Map this alias to its ultimate destination. + + @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase} + @param aliasmap: A mapping of username to alias or group of aliases. + + @type memo: L{None} or L{dict} of L{AliasBase} + @param memo: A record of the aliases already considered in the + resolution process. If provided, C{memo} is modified to include + this alias. + + @rtype: L{IMessage } or L{None} + @return: A message receiver for the ultimate destination or None for + an invalid destination. + """ + if memo is None: + memo = {} + if str(self) in memo: + return None + memo[str(self)] = None + return self.createMessageReceiver() + + + +@implementer(IAlias) +class AddressAlias(AliasBase): + """ + An alias which translates one email address into another. + + @type alias : L{Address} + @ivar alias: The destination address. + """ + def __init__(self, alias, *args): + """ + @type alias: L{Address}, L{User}, L{bytes} or object which can be + converted into L{bytes} + @param alias: The destination address. + + @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain} + provider, (1) L{bytes} + @param args: Arguments for L{AliasBase.__init__}. + """ + AliasBase.__init__(self, *args) + self.alias = smtp.Address(alias) + + + def __str__(self): + """ + Build a string representation of this L{AddressAlias} instance. + + @rtype: L{bytes} + @return: A string containing the destination address. + """ + return '
' % (self.alias,) + + + def createMessageReceiver(self): + """ + Create a message receiver which delivers a message to + the destination address. + + @rtype: L{IMessage } provider + @return: A message receiver. + """ + return self.domain().exists(str(self.alias)) + + + def resolve(self, aliasmap, memo=None): + """ + Map this alias to its ultimate destination. + + @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase} + @param aliasmap: A mapping of username to alias or group of aliases. + + @type memo: L{None} or L{dict} of L{AliasBase} + @param memo: A record of the aliases already considered in the + resolution process. If provided, C{memo} is modified to include + this alias. + + @rtype: L{IMessage } or L{None} + @return: A message receiver for the ultimate destination or None for + an invalid destination. + """ + if memo is None: + memo = {} + if str(self) in memo: + return None + memo[str(self)] = None + try: + return self.domain().exists(smtp.User(self.alias, None, None, None), memo)() + except smtp.SMTPBadRcpt: + pass + if self.alias.local in aliasmap: + return aliasmap[self.alias.local].resolve(aliasmap, memo) + return None + + + +@implementer(smtp.IMessage) +class FileWrapper: + """ + A message receiver which delivers a message to a file. + + @type fp: file-like object + @ivar fp: A file used for temporary storage of the message. + + @type finalname: L{bytes} + @ivar finalname: The name of the file in which the message should be + stored. + """ + def __init__(self, filename): + """ + @type filename: L{bytes} + @param filename: The name of the file in which the message should be + stored. + """ + self.fp = tempfile.TemporaryFile() + self.finalname = filename + + + def lineReceived(self, line): + """ + Write a received line to the temporary file. + + @type line: L{bytes} + @param line: A received line of the message. + """ + self.fp.write(line + '\n') + + + def eomReceived(self): + """ + Handle end of message by writing the message to the file. + + @rtype: L{Deferred } which successfully results in + L{bytes} + @return: A deferred which succeeds with the name of the file to which + the message has been stored or fails if the message cannot be + saved to the file. + """ + self.fp.seek(0, 0) + try: + f = open(self.finalname, 'a') + except: + return defer.fail(failure.Failure()) + + with f: + f.write(self.fp.read()) + self.fp.close() + + return defer.succeed(self.finalname) + + + def connectionLost(self): + """ + Close the temporary file when the connection is lost. + """ + self.fp.close() + self.fp = None + + + def __str__(self): + """ + Build a string representation of this L{FileWrapper} instance. + + @rtype: L{bytes} + @return: A string containing the file name of the message. + """ + return '' % (self.finalname,) + + + +@implementer(IAlias) +class FileAlias(AliasBase): + """ + An alias which translates an address to a file. + + @ivar filename: See L{__init__}. + """ + def __init__(self, filename, *args): + """ + @type filename: L{bytes} + @param filename: The name of the file in which to store the message. + + @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain} + provider, (1) L{bytes} + @param args: Arguments for L{AliasBase.__init__}. + """ + AliasBase.__init__(self, *args) + self.filename = filename + + + def __str__(self): + """ + Build a string representation of this L{FileAlias} instance. + + @rtype: L{bytes} + @return: A string containing the name of the file. + """ + return '' % (self.filename,) + + + def createMessageReceiver(self): + """ + Create a message receiver which delivers a message to the file. + + @rtype: L{FileWrapper} + @return: A message receiver which writes a message to the file. + """ + return FileWrapper(self.filename) + + + +class ProcessAliasTimeout(Exception): + """ + An error indicating that a timeout occurred while waiting for a process + to complete. + """ + + + +@implementer(smtp.IMessage) +class MessageWrapper: + """ + A message receiver which delivers a message to a child process. + + @type completionTimeout: L{int} or L{float} + @ivar completionTimeout: The number of seconds to wait for the child + process to exit before reporting the delivery as a failure. + + @type _timeoutCallID: L{None} or + L{IDelayedCall } provider + @ivar _timeoutCallID: The call used to time out delivery, started when the + connection to the child process is closed. + + @type done: L{bool} + @ivar done: A flag indicating whether the child process has exited + (C{True}) or not (C{False}). + + @type reactor: L{IReactorTime } + provider + @ivar reactor: A reactor which will be used to schedule timeouts. + + @ivar protocol: See L{__init__}. + + @type processName: L{bytes} or L{None} + @ivar processName: The process name. + + @type completion: L{Deferred } + @ivar completion: The deferred which will be triggered by the protocol + when the child process exits. + """ + done = False + + completionTimeout = 60 + _timeoutCallID = None + + reactor = reactor + + def __init__(self, protocol, process=None, reactor=None): + """ + @type protocol: L{ProcessAliasProtocol} + @param protocol: The protocol associated with the child process. + + @type process: L{bytes} or L{None} + @param process: The process name. + + @type reactor: L{None} or L{IReactorTime + } provider + @param reactor: A reactor which will be used to schedule timeouts. + """ + self.processName = process + self.protocol = protocol + self.completion = defer.Deferred() + self.protocol.onEnd = self.completion + self.completion.addBoth(self._processEnded) + + if reactor is not None: + self.reactor = reactor + + + def _processEnded(self, result): + """ + Record process termination and cancel the timeout call if it is active. + + @type result: L{Failure } + @param result: The reason the child process terminated. + + @rtype: L{None} or L{Failure } + @return: None, if the process end is expected, or the reason the child + process terminated, if the process end is unexpected. + """ + self.done = True + if self._timeoutCallID is not None: + # eomReceived was called, we're actually waiting for the process to + # exit. + self._timeoutCallID.cancel() + self._timeoutCallID = None + else: + # eomReceived was not called, this is unexpected, propagate the + # error. + return result + + + def lineReceived(self, line): + """ + Write a received line to the child process. + + @type line: L{bytes} + @param line: A received line of the message. + """ + if self.done: + return + self.protocol.transport.write(line + '\n') + + + def eomReceived(self): + """ + Disconnect from the child process and set up a timeout to wait for it + to exit. + + @rtype: L{Deferred } + @return: A deferred which will be called back when the child process + exits. + """ + if not self.done: + self.protocol.transport.loseConnection() + self._timeoutCallID = self.reactor.callLater( + self.completionTimeout, self._completionCancel) + return self.completion + + + def _completionCancel(self): + """ + Handle the expiration of the timeout for the child process to exit by + terminating the child process forcefully and issuing a failure to the + L{completion} deferred. + """ + self._timeoutCallID = None + self.protocol.transport.signalProcess('KILL') + exc = ProcessAliasTimeout( + "No answer after %s seconds" % (self.completionTimeout,)) + self.protocol.onEnd = None + self.completion.errback(failure.Failure(exc)) + + + def connectionLost(self): + """ + Ignore notification of lost connection. + """ + + + def __str__(self): + """ + Build a string representation of this L{MessageWrapper} instance. + + @rtype: L{bytes} + @return: A string containing the name of the process. + """ + return '' % (self.processName,) + + + +class ProcessAliasProtocol(protocol.ProcessProtocol): + """ + A process protocol which errbacks a deferred when the associated + process ends. + + @type onEnd: L{None} or L{Deferred } + @ivar onEnd: If set, a deferred on which to errback when the process ends. + """ + onEnd = None + + def processEnded(self, reason): + """ + Call an errback. + + @type reason: L{Failure } + @param reason: The reason the child process terminated. + """ + if self.onEnd is not None: + self.onEnd.errback(reason) + + + +@implementer(IAlias) +class ProcessAlias(AliasBase): + """ + An alias which is handled by the execution of a program. + + @type path: L{list} of L{bytes} + @ivar path: The arguments to pass to the process. The first string is + the executable's name. + + @type program: L{bytes} + @ivar program: The path of the program to be executed. + + @type reactor: L{IReactorTime } + and L{IReactorProcess } + provider + @ivar reactor: A reactor which will be used to create and timeout the + child process. + """ + reactor = reactor + + def __init__(self, path, *args): + """ + @type path: L{bytes} + @param path: The command to invoke the program consisting of the path + to the executable followed by any arguments. + + @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain} + provider, (1) L{bytes} + @param args: Arguments for L{AliasBase.__init__}. + """ + + AliasBase.__init__(self, *args) + self.path = path.split() + self.program = self.path[0] + + + def __str__(self): + """ + Build a string representation of this L{ProcessAlias} instance. + + @rtype: L{bytes} + @return: A string containing the command used to invoke the process. + """ + return '' % (self.path,) + + + def spawnProcess(self, proto, program, path): + """ + Spawn a process. + + This wraps the L{spawnProcess + } method on + L{reactor} so that it can be customized for test purposes. + + @type proto: L{IProcessProtocol + } provider + @param proto: An object which will be notified of all events related to + the created process. + + @type program: L{bytes} + @param program: The full path name of the file to execute. + + @type path: L{list} of L{bytes} + @param path: The arguments to pass to the process. The first string + should be the executable's name. + + @rtype: L{IProcessTransport + } provider + @return: A process transport. + """ + return self.reactor.spawnProcess(proto, program, path) + + + def createMessageReceiver(self): + """ + Launch a process and create a message receiver to pass a message + to the process. + + @rtype: L{MessageWrapper} + @return: A message receiver which delivers a message to the process. + """ + p = ProcessAliasProtocol() + m = MessageWrapper(p, self.program, self.reactor) + self.spawnProcess(p, self.program, self.path) + return m + + + +@implementer(smtp.IMessage) +class MultiWrapper: + """ + A message receiver which delivers a single message to multiple other + message receivers. + + @ivar objs: See L{__init__}. + """ + def __init__(self, objs): + """ + @type objs: L{list} of L{IMessage } provider + @param objs: Message receivers to which the incoming message should be + directed. + """ + self.objs = objs + + + def lineReceived(self, line): + """ + Pass a received line to the message receivers. + + @type line: L{bytes} + @param line: A line of the message. + """ + for o in self.objs: + o.lineReceived(line) + + + def eomReceived(self): + """ + Pass the end of message along to the message receivers. + + @rtype: L{DeferredList } whose successful results + are L{bytes} or L{None} + @return: A deferred list which triggers when all of the message + receivers have finished handling their end of message. + """ + return defer.DeferredList([ + o.eomReceived() for o in self.objs + ]) + + + def connectionLost(self): + """ + Inform the message receivers that the connection has been lost. + """ + for o in self.objs: + o.connectionLost() + + + def __str__(self): + """ + Build a string representation of this L{MultiWrapper} instance. + + @rtype: L{bytes} + @return: A string containing a list of the message receivers. + """ + return '' % (map(str, self.objs),) + + + +@implementer(IAlias) +class AliasGroup(AliasBase): + """ + An alias which points to multiple destination aliases. + + @type processAliasFactory: no-argument callable which returns + L{ProcessAlias} + @ivar processAliasFactory: A factory for process aliases. + + @type aliases: L{list} of L{AliasBase} which implements L{IAlias} + @ivar aliases: The destination aliases. + """ + processAliasFactory = ProcessAlias + + def __init__(self, items, *args): + """ + Create a group of aliases. + + Parse a list of alias strings and, for each, create an appropriate + alias object. + + @type items: L{list} of L{bytes} + @param items: Aliases. + + @type args: n-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain} + provider, (1) L{bytes} + @param args: Arguments for L{AliasBase.__init__}. + """ + + AliasBase.__init__(self, *args) + self.aliases = [] + while items: + addr = items.pop().strip() + if addr.startswith(':'): + try: + f = open(addr[1:]) + except: + log.err("Invalid filename in alias file %r" % (addr[1:],)) + else: + with f: + addr = ' '.join([l.strip() for l in f]) + items.extend(addr.split(',')) + elif addr.startswith('|'): + self.aliases.append(self.processAliasFactory(addr[1:], *args)) + elif addr.startswith('/'): + if os.path.isdir(addr): + log.err("Directory delivery not supported") + else: + self.aliases.append(FileAlias(addr, *args)) + else: + self.aliases.append(AddressAlias(addr, *args)) + + + def __len__(self): + """ + Return the number of aliases in the group. + + @rtype: L{int} + @return: The number of aliases in the group. + """ + return len(self.aliases) + + + def __str__(self): + """ + Build a string representation of this L{AliasGroup} instance. + + @rtype: L{bytes} + @return: A string containing the aliases in the group. + """ + return '' % (', '.join(map(str, self.aliases))) + + + def createMessageReceiver(self): + """ + Create a message receiver for each alias and return a message receiver + which will pass on a message to each of those. + + @rtype: L{MultiWrapper} + @return: A message receiver which passes a message on to message + receivers for each alias in the group. + """ + return MultiWrapper([a.createMessageReceiver() for a in self.aliases]) + + + def resolve(self, aliasmap, memo=None): + """ + Map each of the aliases in the group to its ultimate destination. + + @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase} + @param aliasmap: A mapping of username to alias or group of aliases. + + @type memo: L{None} or L{dict} of L{AliasBase} + @param memo: A record of the aliases already considered in the + resolution process. If provided, C{memo} is modified to include + this alias. + + @rtype: L{MultiWrapper} + @return: A message receiver which passes the message on to message + receivers for the ultimate destination of each alias in the group. + """ + if memo is None: + memo = {} + r = [] + for a in self.aliases: + r.append(a.resolve(aliasmap, memo)) + return MultiWrapper(filter(None, r)) diff --git a/contrib/python/Twisted/py2/twisted/mail/bounce.py b/contrib/python/Twisted/py2/twisted/mail/bounce.py new file mode 100644 index 00000000000..9e6b73d2bf0 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/bounce.py @@ -0,0 +1,107 @@ +# -*- test-case-name: twisted.mail.test.test_bounce -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Support for bounce message generation. +""" +import email.utils +import time +import os +from io import StringIO, SEEK_SET, SEEK_END + +from twisted.mail import smtp + +BOUNCE_FORMAT = u"""\ +From: postmaster@{failedDomain} +To: {failedFrom} +Subject: Returned Mail: see transcript for details +Message-ID: {messageID} +Content-Type: multipart/report; report-type=delivery-status; + boundary="{boundary}" + +--{boundary} + +{transcript} + +--{boundary} +Content-Type: message/delivery-status +Arrival-Date: {ctime} +Final-Recipient: RFC822; {failedTo} +""" + + + +def generateBounce(message, failedFrom, failedTo, transcript='', + encoding='utf-8'): + """ + Generate a bounce message for an undeliverable email message. + + @type message: a file-like object + @param message: The undeliverable message. + + @type failedFrom: L{bytes} or L{unicode} + @param failedFrom: The originator of the undeliverable message. + + @type failedTo: L{bytes} or L{unicode} + @param failedTo: The destination of the undeliverable message. + + @type transcript: L{bytes} or L{unicode} + @param transcript: An error message to include in the bounce message. + + @type encoding: L{str} or L{unicode} + @param encoding: Encoding to use, default: utf-8 + + @rtype: 3-L{tuple} of (E{1}) L{bytes}, (E{2}) L{bytes}, (E{3}) L{bytes} + @return: The originator, the destination and the contents of the bounce + message. The destination of the bounce message is the originator of + the undeliverable message. + """ + + if isinstance(failedFrom, bytes): + failedFrom = failedFrom.decode(encoding) + + if isinstance(failedTo, bytes): + failedTo = failedTo.decode(encoding) + + if not transcript: + transcript = u'''\ +I'm sorry, the following address has permanent errors: {failedTo}. +I've given up, and I will not retry the message again. +'''.format(failedTo=failedTo) + + failedAddress = email.utils.parseaddr(failedTo)[1] + data = { + 'boundary': "{}_{}_{}".format(time.time(), os.getpid(), 'XXXXX'), + 'ctime': time.ctime(time.time()), + 'failedAddress': failedAddress, + 'failedDomain': failedAddress.split('@', 1)[1], + 'failedFrom': failedFrom, + 'failedTo': failedTo, + 'messageID': smtp.messageid(uniq='bounce'), + 'message': message, + 'transcript': transcript, + } + + fp = StringIO() + fp.write(BOUNCE_FORMAT.format(**data)) + orig = message.tell() + message.seek(0, SEEK_END) + sz = message.tell() + message.seek(orig, SEEK_SET) + if sz > 10000: + while 1: + line = message.readline() + if isinstance(line, bytes): + line = line.decode(encoding) + if len(line) <= 0: + break + fp.write(line) + else: + messageContent = message.read() + if isinstance(messageContent, bytes): + messageContent = messageContent.decode(encoding) + fp.write(messageContent) + return b'', failedFrom.encode(encoding), fp.getvalue().encode(encoding) diff --git a/contrib/python/Twisted/py2/twisted/mail/imap4.py b/contrib/python/Twisted/py2/twisted/mail/imap4.py new file mode 100644 index 00000000000..7949ef51ac8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/imap4.py @@ -0,0 +1,6404 @@ +# -*- test-case-name: twisted.mail.test.test_imap.IMAP4HelperTests -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An IMAP4 protocol implementation + +@author: Jp Calderone + +To do:: + Suspend idle timeout while server is processing + Use an async message parser instead of buffering in memory + Figure out a way to not queue multi-message client requests (Flow? A simple callback?) + Clarify some API docs (Query, etc) + Make APPEND recognize (again) non-existent mailboxes before accepting the literal +""" + +import binascii +import codecs +import copy +import functools +import re +import string +import tempfile +import time +import uuid + +import email.utils + +from itertools import chain +from io import BytesIO + +from zope.interface import implementer + +from twisted.protocols import basic +from twisted.protocols import policies +from twisted.internet import defer +from twisted.internet import error +from twisted.internet.defer import maybeDeferred +from twisted.python import log, text +from twisted.python.compat import ( + _bytesChr, unichr as chr, _b64decodebytes as decodebytes, + _b64encodebytes as encodebytes, + intToBytes, iterbytes, long, nativeString, networkString, unicode, + _matchingString, _PY3, _get_async_param, +) +from twisted.internet import interfaces + +from twisted.cred import credentials +from twisted.cred.error import UnauthorizedLogin, UnhandledCredentials + +# Re-exported for compatibility reasons +from twisted.mail.interfaces import ( + IClientAuthentication, INamespacePresenter, + IAccountIMAP as IAccount, + IMessageIMAPPart as IMessagePart, + IMessageIMAP as IMessage, + IMessageIMAPFile as IMessageFile, + ISearchableIMAPMailbox as ISearchableMailbox, + IMessageIMAPCopier as IMessageCopier, + IMailboxIMAPInfo as IMailboxInfo, + IMailboxIMAP as IMailbox, + ICloseableMailboxIMAP as ICloseableMailbox, + IMailboxIMAPListener as IMailboxListener +) +from twisted.mail._cred import ( + CramMD5ClientAuthenticator, + LOGINAuthenticator, LOGINCredentials, + PLAINAuthenticator, PLAINCredentials) +from twisted.mail._except import ( + IMAP4Exception, IllegalClientResponse, IllegalOperation, MailboxException, + IllegalMailboxEncoding, MailboxCollision, NoSuchMailbox, ReadOnlyMailbox, + UnhandledResponse, NegativeResponse, NoSupportedAuthentication, + IllegalIdentifierError, IllegalQueryError, MismatchedNesting, + MismatchedQuoting, IllegalServerResponse, +) + +# locale-independent month names to use instead of strftime's +_MONTH_NAMES = dict(zip( + range(1, 13), + "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split())) + + +def _swap(this, that, ifIs): + """ + Swap C{this} with C{that} if C{this} is C{ifIs}. + + @param this: The object that may be replaced. + + @param that: The object that may replace C{this}. + + @param ifIs: An object whose identity will be compared to + C{this}. + """ + return that if this is ifIs else this + + +def _swapAllPairs(of, that, ifIs): + """ + Swap each element in each pair in C{of} with C{that} it is + C{ifIs}. + + @param of: A list of 2-L{tuple}s, whose members may be the object + C{that} + @type of: L{list} of 2-L{tuple}s + + @param ifIs: An object whose identity will be compared to members + of each pair in C{of} + + @return: A L{list} of 2-L{tuple}s with all occurences of C{ifIs} + replaced with C{that} + """ + return [(_swap(first, that, ifIs), _swap(second, that, ifIs)) + for first, second in of] + + +class MessageSet(object): + """ + A set of message identifiers usable by both L{IMAP4Client} and + L{IMAP4Server} via L{IMailboxIMAP.store} and + L{IMailboxIMAP.fetch}. + + These identifiers can be either message sequence numbers or unique + identifiers. See Section 2.3.1, "Message Numbers", RFC 3501. + + This represents the C{sequence-set} described in Section 9, + "Formal Syntax" of RFC 3501: + + - A L{MessageSet} can describe a single identifier, e.g. + C{MessageSet(1)} + + - A L{MessageSet} can describe C{*} via L{None}, e.g. + C{MessageSet(None)} + + - A L{MessageSet} can describe a range of identifiers, e.g. + C{MessageSet(1, 2)}. The range is inclusive and unordered + (see C{seq-range} in RFC 3501, Section 9), so that + C{Message(2, 1)} is equivalent to C{MessageSet(1, 2)}, and + both describe messages 1 and 2. Ranges can include C{*} by + specifying L{None}, e.g. C{MessageSet(None, 1)}. In all + cases ranges are normalized so that the smallest identifier + comes first, and L{None} always comes last; C{Message(2, 1)} + becomes C{MessageSet(1, 2)} and C{MessageSet(None, 1)} + becomes C{MessageSet(1, None)} + + - A L{MessageSet} can describe a sequence of single + identifiers and ranges, constructed by addition. + C{MessageSet(1) + MessageSet(5, 10)} refers the message + identified by C{1} and the messages identified by C{5} + through C{10}. + + B{NB: The meaning of * varies, but it always represents the + largest number in use}. + + B{For servers}: Your L{IMailboxIMAP} provider must set + L{MessageSet.last} to the highest-valued identifier (unique or + message sequence) before iterating over it. + + B{For clients}: C{*} consumes ranges smaller than it, e.g. + C{MessageSet(1, 100) + MessageSet(50, None)} is equivalent to + C{1:*}. + + @type getnext: Function taking L{int} returning L{int} + @ivar getnext: A function that returns the next message number, + used when iterating through the L{MessageSet}. By default, a + function returning the next integer is supplied, but as this + can be rather inefficient for sparse UID iterations, it is + recommended to supply one when messages are requested by UID. + The argument is provided as a hint to the implementation and + may be ignored if it makes sense to do so (eg, if an iterator + is being used that maintains its own state, it is guaranteed + that it will not be called out-of-order). + """ + _empty = [] + _infinity = float('inf') + + def __init__(self, start=_empty, end=_empty): + """ + Create a new MessageSet() + + @type start: Optional L{int} + @param start: Start of range, or only message number + + @type end: Optional L{int} + @param end: End of range. + """ + self._last = self._empty # Last message/UID in use + self.ranges = [] # List of ranges included + self.getnext = lambda x: x+1 # A function which will return the next + # message id. Handy for UID requests. + + if start is self._empty: + return + + if isinstance(start, list): + self.ranges = start[:] + self.clean() + else: + self.add(start,end) + + + # Ooo. A property. + def last(): + def _setLast(self, value): + if self._last is not self._empty: + raise ValueError("last already set") + + self._last = value + for i, (l, h) in enumerate(self.ranges): + if l is None: + l = value + if h is None: + h = value + if l > h: + l, h = h, l + self.ranges[i] = (l, h) + self.clean() + + def _getLast(self): + return self._last + + doc = ''' + Replaces all occurrences of "*". This should be the + largest number in use. Must be set before attempting to + use the MessageSet as a container. + + @raises: L{ValueError} if a largest value has already + been set. + ''' + return _getLast, _setLast, None, doc + last = property(*last()) + + + def add(self, start, end=_empty): + """ + Add another range + + @type start: L{int} + @param start: Start of range, or only message number + + @type end: Optional L{int} + @param end: End of range. + """ + if end is self._empty: + end = start + + if self._last is not self._empty: + if start is None: + start = self.last + if end is None: + end = self.last + + start, end = sorted( + [start, end], + key=functools.partial(_swap, that=self._infinity, ifIs=None)) + self.ranges.append((start, end)) + self.clean() + + + def __add__(self, other): + if isinstance(other, MessageSet): + ranges = self.ranges + other.ranges + return MessageSet(ranges) + else: + res = MessageSet(self.ranges) + if self.last is not self._empty: + res.last = self.last + try: + res.add(*other) + except TypeError: + res.add(other) + return res + + + def extend(self, other): + """ + Extend our messages with another message or set of messages. + + @param other: The messages to include. + @type other: L{MessageSet}, L{tuple} of two L{int}s, or a + single L{int} + """ + if isinstance(other, MessageSet): + self.ranges.extend(other.ranges) + self.clean() + else: + try: + self.add(*other) + except TypeError: + self.add(other) + + return self + + + def clean(self): + """ + Clean ranges list, combining adjacent ranges + """ + + ranges = sorted(_swapAllPairs(self.ranges, + that=self._infinity, + ifIs=None)) + + mergedRanges = [(float('-inf'), float('-inf'))] + + + for low, high in ranges: + previousLow, previousHigh = mergedRanges[-1] + + if previousHigh < low - 1: + mergedRanges.append((low, high)) + continue + + mergedRanges[-1] = (min(previousLow, low), + max(previousHigh, high)) + + self.ranges = _swapAllPairs(mergedRanges[1:], + that=None, + ifIs=self._infinity) + + + def _noneInRanges(self): + """ + Is there a L{None} in our ranges? + + L{MessageSet.clean} merges overlapping or consecutive ranges. + None is represents a value larger than any number. There are + thus two cases: + + 1. C{(x, *) + (y, z)} such that C{x} is smaller than C{y} + + 2. C{(z, *) + (x, y)} such that C{z} is larger than C{y} + + (Other cases, such as C{y < x < z}, can be split into these + two cases; for example C{(y - 1, y)} + C{(x, x) + (z, z + 1)}) + + In case 1, C{* > y} and C{* > z}, so C{(x, *) + (y, z) = (x, + *)} + + In case 2, C{z > x and z > y}, so the intervals do not merge, + and the ranges are sorted as C{[(x, y), (z, *)]}. C{*} is + represented as C{(*, *)}, so this is the same as 2. but with + a C{z} that is greater than everything. + + The result is that there is a maximum of two L{None}s, and one + of them has to be the high element in the last tuple in + C{self.ranges}. That means checking if C{self.ranges[-1][-1]} + is L{None} suffices to check if I{any} element is L{None}. + + @return: L{True} if L{None} is in some range in ranges and + L{False} if otherwise. + """ + return self.ranges[-1][-1] is None + + + def __contains__(self, value): + """ + May raise TypeError if we encounter an open-ended range + + @param value: Is this in our ranges? + @type value: L{int} + """ + + if self._noneInRanges(): + raise TypeError( + "Can't determine membership; last value not set") + + for low, high in self.ranges: + if low <= value <= high: + return True + + return False + + + def _iterator(self): + for l, h in self.ranges: + l = self.getnext(l-1) + while l <= h: + yield l + l = self.getnext(l) + + + def __iter__(self): + if self._noneInRanges(): + raise TypeError("Can't iterate; last value not set") + + return self._iterator() + + + def __len__(self): + res = 0 + for l, h in self.ranges: + if l is None: + res += 1 + elif h is None: + raise TypeError("Can't size object; last value not set") + else: + res += (h - l) + 1 + + return res + + + def __str__(self): + p = [] + for low, high in self.ranges: + if low == high: + if low is None: + p.append('*') + else: + p.append(str(low)) + elif high is None: + p.append('%d:*' % (low,)) + else: + p.append('%d:%d' % (low, high)) + return ','.join(p) + + + def __repr__(self): + return '' % (str(self),) + + + def __eq__(self, other): + if isinstance(other, MessageSet): + return self.ranges == other.ranges + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class LiteralString: + def __init__(self, size, defered): + self.size = size + self.data = [] + self.defer = defered + + + def write(self, data): + self.size -= len(data) + passon = None + if self.size > 0: + self.data.append(data) + else: + if self.size: + data, passon = data[:self.size], data[self.size:] + else: + passon = b'' + if data: + self.data.append(data) + + return passon + + + def callback(self, line): + """ + Call deferred with data and rest of line + """ + self.defer.callback((b''.join(self.data), line)) + + + +class LiteralFile: + _memoryFileLimit = 1024 * 1024 * 10 + + def __init__(self, size, defered): + self.size = size + self.defer = defered + if size > self._memoryFileLimit: + self.data = tempfile.TemporaryFile() + else: + self.data = BytesIO() + + + def write(self, data): + self.size -= len(data) + passon = None + if self.size > 0: + self.data.write(data) + else: + if self.size: + data, passon = data[:self.size], data[self.size:] + else: + passon = b'' + if data: + self.data.write(data) + return passon + + + def callback(self, line): + """ + Call deferred with data and rest of line + """ + self.data.seek(0,0) + self.defer.callback((self.data, line)) + + + +class WriteBuffer: + """ + Buffer up a bunch of writes before sending them all to a transport at once. + """ + def __init__(self, transport, size=8192): + self.bufferSize = size + self.transport = transport + self._length = 0 + self._writes = [] + + + def write(self, s): + self._length += len(s) + self._writes.append(s) + if self._length > self.bufferSize: + self.flush() + + + def flush(self): + if self._writes: + self.transport.writeSequence(self._writes) + self._writes = [] + self._length = 0 + + + +class Command: + _1_RESPONSES = (b'CAPABILITY', b'FLAGS', b'LIST', b'LSUB', b'STATUS', b'SEARCH', b'NAMESPACE') + _2_RESPONSES = (b'EXISTS', b'EXPUNGE', b'FETCH', b'RECENT') + _OK_RESPONSES = (b'UIDVALIDITY', b'UNSEEN', b'READ-WRITE', b'READ-ONLY', b'UIDNEXT', b'PERMANENTFLAGS') + defer = None + + def __init__(self, command, args=None, wantResponse=(), + continuation=None, *contArgs, **contKw): + self.command = command + self.args = args + self.wantResponse = wantResponse + self.continuation = lambda x: continuation(x, *contArgs, **contKw) + self.lines = [] + + + def __repr__(self): + return "".format( + self.command, self.args, self.wantResponse, self.continuation, + self.lines + ) + + + def format(self, tag): + if self.args is None: + return b' '.join((tag, self.command)) + return b' '.join((tag, self.command, self.args)) + + + def finish(self, lastLine, unusedCallback): + send = [] + unuse = [] + for L in self.lines: + names = parseNestedParens(L) + N = len(names) + if (N >= 1 and names[0] in self._1_RESPONSES or + N >= 2 and names[1] in self._2_RESPONSES or + N >= 2 and names[0] == b'OK' and isinstance(names[1], list) + and names[1][0] in self._OK_RESPONSES): + send.append(names) + else: + unuse.append(names) + d, self.defer = self.defer, None + d.callback((send, lastLine)) + if unuse: + unusedCallback(unuse) + + + +# Some constants to help define what an atom is and is not - see the grammar +# section of the IMAP4 RFC - . +# Some definitions (SP, CTL, DQUOTE) are also from the ABNF RFC - +# . +_SP = b' ' +_CTL = b''.join(_bytesChr(ch) for ch in chain(range(0x21), range(0x80, 0x100))) + +# It is easier to define ATOM-CHAR in terms of what it does not match than in +# terms of what it does match. +_nonAtomChars = b']\\\\(){%*"' + _SP + _CTL + +# _nonAtomRE is only used in Query, so it uses native strings. +if _PY3: + # + _nativeNonAtomChars = _nonAtomChars.decode('charmap') +else: + _nativeNonAtomChars = _nonAtomChars +_nonAtomRE = re.compile('[' + _nativeNonAtomChars + ']') + +# This is all the bytes that match the ATOM-CHAR from the grammar in the RFC. +_atomChars = b''.join(_bytesChr(ch) for ch in list(range(0x100)) if _bytesChr(ch) not in _nonAtomChars) + +@implementer(IMailboxListener) +class IMAP4Server(basic.LineReceiver, policies.TimeoutMixin): + """ + Protocol implementation for an IMAP4rev1 server. + + The server can be in any of four states: + - Non-authenticated + - Authenticated + - Selected + - Logout + """ + # Identifier for this server software + IDENT = b'Twisted IMAP4rev1 Ready' + + # Number of seconds before idle timeout + # Initially 1 minute. Raised to 30 minutes after login. + timeOut = 60 + + POSTAUTH_TIMEOUT = 60 * 30 + + # Whether STARTTLS has been issued successfully yet or not. + startedTLS = False + + # Whether our transport supports TLS + canStartTLS = False + + # Mapping of tags to commands we have received + tags = None + + # The object which will handle logins for us + portal = None + + # The account object for this connection + account = None + + # Logout callback + _onLogout = None + + # The currently selected mailbox + mbox = None + + # Command data to be processed when literal data is received + _pendingLiteral = None + + # Maximum length to accept for a "short" string literal + _literalStringLimit = 4096 + + # IChallengeResponse factories for AUTHENTICATE command + challengers = None + + # Search terms the implementation of which needs to be passed both the last + # message identifier (UID) and the last sequence id. + _requiresLastMessageInfo = set([b"OR", b"NOT", b"UID"]) + + state = 'unauth' + + parseState = 'command' + + def __init__(self, chal = None, contextFactory = None, scheduler = None): + if chal is None: + chal = {} + self.challengers = chal + self.ctx = contextFactory + if scheduler is None: + scheduler = iterateInReactor + self._scheduler = scheduler + self._queuedAsync = [] + + + def capabilities(self): + cap = {b'AUTH': list(self.challengers.keys())} + if self.ctx and self.canStartTLS: + if not self.startedTLS and interfaces.ISSLTransport(self.transport, None) is None: + cap[b'LOGINDISABLED'] = None + cap[b'STARTTLS'] = None + cap[b'NAMESPACE'] = None + cap[b'IDLE'] = None + return cap + + + def connectionMade(self): + self.tags = {} + self.canStartTLS = interfaces.ITLSTransport(self.transport, None) is not None + self.setTimeout(self.timeOut) + self.sendServerGreeting() + + + def connectionLost(self, reason): + self.setTimeout(None) + if self._onLogout: + self._onLogout() + self._onLogout = None + + + def timeoutConnection(self): + self.sendLine(b'* BYE Autologout; connection idle too long') + self.transport.loseConnection() + if self.mbox: + self.mbox.removeListener(self) + cmbx = ICloseableMailbox(self.mbox, None) + if cmbx is not None: + maybeDeferred(cmbx.close).addErrback(log.err) + self.mbox = None + self.state = 'timeout' + + + def rawDataReceived(self, data): + self.resetTimeout() + passon = self._pendingLiteral.write(data) + if passon is not None: + self.setLineMode(passon) + + # Avoid processing commands while buffers are being dumped to + # our transport + blocked = None + + def _unblock(self): + commands = self.blocked + self.blocked = None + while commands and self.blocked is None: + self.lineReceived(commands.pop(0)) + if self.blocked is not None: + self.blocked.extend(commands) + + + def lineReceived(self, line): + if self.blocked is not None: + self.blocked.append(line) + return + + self.resetTimeout() + f = getattr(self, 'parse_' + self.parseState) + try: + f(line) + except Exception as e: + self.sendUntaggedResponse(b'BAD Server error: ' + networkString(str(e))) + log.err() + + + def parse_command(self, line): + args = line.split(None, 2) + rest = None + if len(args) == 3: + tag, cmd, rest = args + elif len(args) == 2: + tag, cmd = args + elif len(args) == 1: + tag = args[0] + self.sendBadResponse(tag, b'Missing command') + return None + else: + self.sendBadResponse(None, b'Null command') + return None + + cmd = cmd.upper() + try: + return self.dispatchCommand(tag, cmd, rest) + except IllegalClientResponse as e: + self.sendBadResponse(tag, b'Illegal syntax: ' + networkString(str(e))) + except IllegalOperation as e: + self.sendNegativeResponse(tag, b'Illegal operation: ' + networkString(str(e))) + except IllegalMailboxEncoding as e: + self.sendNegativeResponse(tag, b'Illegal mailbox name: ' + networkString(str(e))) + + + def parse_pending(self, line): + d = self._pendingLiteral + self._pendingLiteral = None + self.parseState = 'command' + d.callback(line) + + + def dispatchCommand(self, tag, cmd, rest, uid=None): + f = self.lookupCommand(cmd) + if f: + fn = f[0] + parseargs = f[1:] + self.__doCommand(tag, fn, [self, tag], parseargs, rest, uid) + else: + self.sendBadResponse(tag, b'Unsupported command') + + + def lookupCommand(self, cmd): + return getattr(self, '_'.join((self.state, nativeString(cmd.upper()))), None) + + + def __doCommand(self, tag, handler, args, parseargs, line, uid): + for (i, arg) in enumerate(parseargs): + if callable(arg): + parseargs = parseargs[i+1:] + maybeDeferred(arg, self, line).addCallback( + self.__cbDispatch, tag, handler, args, + parseargs, uid).addErrback(self.__ebDispatch, tag) + return + else: + args.append(arg) + + if line: + # Too many arguments + raise IllegalClientResponse("Too many arguments for command: " + repr(line)) + + if uid is not None: + handler(uid=uid, *args) + else: + handler(*args) + + + def __cbDispatch(self, result, tag, fn, args, parseargs, uid): + (arg, rest) = result + args.append(arg) + self.__doCommand(tag, fn, args, parseargs, rest, uid) + + + def __ebDispatch(self, failure, tag): + if failure.check(IllegalClientResponse): + self.sendBadResponse(tag, b'Illegal syntax: ' + networkString(str(failure.value))) + elif failure.check(IllegalOperation): + self.sendNegativeResponse(tag, b'Illegal operation: ' + + networkString(str(failure.value))) + elif failure.check(IllegalMailboxEncoding): + self.sendNegativeResponse(tag, b'Illegal mailbox name: ' + + networkString(str(failure.value))) + else: + self.sendBadResponse(tag, b'Server error: ' + networkString(str(failure.value))) + log.err(failure) + + + def _stringLiteral(self, size): + if size > self._literalStringLimit: + raise IllegalClientResponse( + "Literal too long! I accept at most %d octets" % + (self._literalStringLimit,)) + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralString(size, d) + self.sendContinuationRequest( + networkString('Ready for %d octets of text' % size)) + self.setRawMode() + return d + + + def _fileLiteral(self, size): + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralFile(size, d) + self.sendContinuationRequest( + networkString('Ready for %d octets of data' % size)) + self.setRawMode() + return d + + + def arg_finalastring(self, line): + """ + Parse an astring from line that represents a command's final + argument. This special case exists to enable parsing empty + string literals. + + @param line: A line that contains a string literal. + @type line: L{bytes} + + @return: A 2-tuple containing the parsed argument and any + trailing data, or a L{Deferred} that fires with that + 2-tuple + @rtype: L{tuple} of (L{bytes}, L{bytes}) or a L{Deferred} + + @see: https://twistedmatrix.com/trac/ticket/9207 + """ + return self.arg_astring(line, final=True) + + + def arg_astring(self, line, final=False): + """ + Parse an astring from the line, return (arg, rest), possibly + via a deferred (to handle literals) + + @param line: A line that contains a string literal. + @type line: L{bytes} + + @param final: Is this the final argument? + @type final L{bool} + + @return: A 2-tuple containing the parsed argument and any + trailing data, or a L{Deferred} that fires with that + 2-tuple + @rtype: L{tuple} of (L{bytes}, L{bytes}) or a L{Deferred} + + """ + line = line.strip() + if not line: + raise IllegalClientResponse("Missing argument") + d = None + arg, rest = None, None + if line[0:1] == b'"': + try: + spam, arg, rest = line.split(b'"',2) + rest = rest[1:] # Strip space + except ValueError: + raise IllegalClientResponse("Unmatched quotes") + elif line[0:1] == b'{': + # literal + if line[-1:] != b'}': + raise IllegalClientResponse("Malformed literal") + try: + size = int(line[1:-1]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + repr(line[1:-1])) + if final and not size: + return (b'', b'') + d = self._stringLiteral(size) + else: + arg = line.split(b' ',1) + if len(arg) == 1: + arg.append(b'') + arg, rest = arg + return d or (arg, rest) + + # ATOM: Any CHAR except ( ) { % * " \ ] CTL SP (CHAR is 7bit) + atomre = re.compile(b'(?P[' + re.escape(_atomChars) + b']+)( (?P.*$)|$)') + + + def arg_atom(self, line): + """ + Parse an atom from the line + """ + if not line: + raise IllegalClientResponse("Missing argument") + m = self.atomre.match(line) + if m: + return m.group('atom'), m.group('rest') + else: + raise IllegalClientResponse("Malformed ATOM") + + + def arg_plist(self, line): + """ + Parse a (non-nested) parenthesised list from the line + """ + if not line: + raise IllegalClientResponse("Missing argument") + + if line[:1] != b"(": + raise IllegalClientResponse("Missing parenthesis") + + i = line.find(b")") + + if i == -1: + raise IllegalClientResponse("Mismatched parenthesis") + + return (parseNestedParens(line[1:i],0), line[i+2:]) + + + def arg_literal(self, line): + """ + Parse a literal from the line + """ + if not line: + raise IllegalClientResponse("Missing argument") + + if line[:1] != b'{': + raise IllegalClientResponse("Missing literal") + + if line[-1:] != b'}': + raise IllegalClientResponse("Malformed literal") + + try: + size = int(line[1:-1]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: {!r}".format(line[1:-1])) + + return self._fileLiteral(size) + + + def arg_searchkeys(self, line): + """ + searchkeys + """ + query = parseNestedParens(line) + # XXX Should really use list of search terms and parse into + # a proper tree + return (query, b'') + + + def arg_seqset(self, line): + """ + sequence-set + """ + rest = b'' + arg = line.split(b' ',1) + if len(arg) == 2: + rest = arg[1] + arg = arg[0] + + try: + return (parseIdList(arg), rest) + except IllegalIdentifierError as e: + raise IllegalClientResponse("Bad message number " + str(e)) + + + def arg_fetchatt(self, line): + """ + fetch-att + """ + p = _FetchParser() + p.parseString(line) + return (p.result, b'') + + + def arg_flaglist(self, line): + """ + Flag part of store-att-flag + """ + flags = [] + if line[0:1] == b'(': + if line[-1:] != b')': + raise IllegalClientResponse("Mismatched parenthesis") + line = line[1:-1] + + while line: + m = self.atomre.search(line) + if not m: + raise IllegalClientResponse("Malformed flag") + if line[0:1] == b'\\' and m.start() == 1: + flags.append(b'\\' + m.group('atom')) + elif m.start() == 0: + flags.append(m.group('atom')) + else: + raise IllegalClientResponse("Malformed flag") + line = m.group('rest') + + return (flags, b'') + + + def arg_line(self, line): + """ + Command line of UID command + """ + return (line, b'') + + + def opt_plist(self, line): + """ + Optional parenthesised list + """ + if line.startswith(b'('): + return self.arg_plist(line) + else: + return (None, line) + + + def opt_datetime(self, line): + """ + Optional date-time string + """ + if line.startswith(b'"'): + try: + spam, date, rest = line.split(b'"',2) + except ValueError: + raise IllegalClientResponse("Malformed date-time") + return (date, rest[1:]) + else: + return (None, line) + + + def opt_charset(self, line): + """ + Optional charset of SEARCH command + """ + if line[:7].upper() == b'CHARSET': + arg = line.split(b' ',2) + if len(arg) == 1: + raise IllegalClientResponse("Missing charset identifier") + if len(arg) == 2: + arg.append(b'') + spam, arg, rest = arg + return (arg, rest) + else: + return (None, line) + + + def sendServerGreeting(self): + msg = (b'[CAPABILITY ' + b' '.join(self.listCapabilities()) + b'] ' + + self.IDENT) + self.sendPositiveResponse(message=msg) + + + def sendBadResponse(self, tag = None, message = b''): + self._respond(b'BAD', tag, message) + + + def sendPositiveResponse(self, tag = None, message = b''): + self._respond(b'OK', tag, message) + + + def sendNegativeResponse(self, tag = None, message = b''): + self._respond(b'NO', tag, message) + + + def sendUntaggedResponse(self, message, isAsync=None, **kwargs): + isAsync = _get_async_param(isAsync, **kwargs) + if not isAsync or (self.blocked is None): + self._respond(message, None, None) + else: + self._queuedAsync.append(message) + + + def sendContinuationRequest(self, msg = b'Ready for additional command text'): + if msg: + self.sendLine(b'+ ' + msg) + else: + self.sendLine(b'+') + + + def _respond(self, state, tag, message): + if state in (b'OK', b'NO', b'BAD') and self._queuedAsync: + lines = self._queuedAsync + self._queuedAsync = [] + for msg in lines: + self._respond(msg, None, None) + if not tag: + tag = b'*' + if message: + self.sendLine(b' '.join((tag, state, message))) + else: + self.sendLine(b' '.join((tag, state))) + + + def listCapabilities(self): + caps = [b'IMAP4rev1'] + for c, v in self.capabilities().items(): + if v is None: + caps.append(c) + elif len(v): + caps.extend([(c + b'=' + cap) for cap in v]) + return caps + + + def do_CAPABILITY(self, tag): + self.sendUntaggedResponse(b'CAPABILITY ' + b' '.join(self.listCapabilities())) + self.sendPositiveResponse(tag, b'CAPABILITY completed') + + unauth_CAPABILITY = (do_CAPABILITY,) + auth_CAPABILITY = unauth_CAPABILITY + select_CAPABILITY = unauth_CAPABILITY + logout_CAPABILITY = unauth_CAPABILITY + + + def do_LOGOUT(self, tag): + self.sendUntaggedResponse(b'BYE Nice talking to you') + self.sendPositiveResponse(tag, b'LOGOUT successful') + self.transport.loseConnection() + + unauth_LOGOUT = (do_LOGOUT,) + auth_LOGOUT = unauth_LOGOUT + select_LOGOUT = unauth_LOGOUT + logout_LOGOUT = unauth_LOGOUT + + + def do_NOOP(self, tag): + self.sendPositiveResponse(tag, b'NOOP No operation performed') + + unauth_NOOP = (do_NOOP,) + auth_NOOP = unauth_NOOP + select_NOOP = unauth_NOOP + logout_NOOP = unauth_NOOP + + + def do_AUTHENTICATE(self, tag, args): + args = args.upper().strip() + if args not in self.challengers: + self.sendNegativeResponse(tag, b'AUTHENTICATE method unsupported') + else: + self.authenticate(self.challengers[args](), tag) + + unauth_AUTHENTICATE = (do_AUTHENTICATE, arg_atom) + + + def authenticate(self, chal, tag): + if self.portal is None: + self.sendNegativeResponse(tag, b'Temporary authentication failure') + return + + self._setupChallenge(chal, tag) + + + def _setupChallenge(self, chal, tag): + try: + challenge = chal.getChallenge() + except Exception as e: + self.sendBadResponse(tag, b'Server error: ' + networkString(str(e))) + else: + coded = encodebytes(challenge)[:-1] + self.parseState = 'pending' + self._pendingLiteral = defer.Deferred() + self.sendContinuationRequest(coded) + self._pendingLiteral.addCallback(self.__cbAuthChunk, chal, tag) + self._pendingLiteral.addErrback(self.__ebAuthChunk, tag) + + + def __cbAuthChunk(self, result, chal, tag): + try: + uncoded = decodebytes(result) + except binascii.Error: + raise IllegalClientResponse("Malformed Response - not base64") + + chal.setResponse(uncoded) + if chal.moreChallenges(): + self._setupChallenge(chal, tag) + else: + self.portal.login(chal, None, IAccount).addCallbacks( + self.__cbAuthResp, + self.__ebAuthResp, + (tag,), None, (tag,), None + ) + + + def __cbAuthResp(self, result, tag): + (iface, avatar, logout) = result + assert iface is IAccount, "IAccount is the only supported interface" + self.account = avatar + self.state = 'auth' + self._onLogout = logout + self.sendPositiveResponse(tag, b'Authentication successful') + self.setTimeout(self.POSTAUTH_TIMEOUT) + + + def __ebAuthResp(self, failure, tag): + if failure.check(UnauthorizedLogin): + self.sendNegativeResponse(tag, b'Authentication failed: unauthorized') + elif failure.check(UnhandledCredentials): + self.sendNegativeResponse(tag, b'Authentication failed: server misconfigured') + else: + self.sendBadResponse(tag, b'Server error: login failed unexpectedly') + log.err(failure) + + + def __ebAuthChunk(self, failure, tag): + self.sendNegativeResponse(tag, b'Authentication failed: ' + networkString(str(failure.value))) + + + def do_STARTTLS(self, tag): + if self.startedTLS: + self.sendNegativeResponse(tag, b'TLS already negotiated') + elif self.ctx and self.canStartTLS: + self.sendPositiveResponse(tag, b'Begin TLS negotiation now') + self.transport.startTLS(self.ctx) + self.startedTLS = True + self.challengers = self.challengers.copy() + if b'LOGIN' not in self.challengers: + self.challengers[b'LOGIN'] = LOGINCredentials + if b'PLAIN' not in self.challengers: + self.challengers[b'PLAIN'] = PLAINCredentials + else: + self.sendNegativeResponse(tag, b'TLS not available') + + unauth_STARTTLS = (do_STARTTLS,) + + + def do_LOGIN(self, tag, user, passwd): + if b'LOGINDISABLED' in self.capabilities(): + self.sendBadResponse(tag, b'LOGIN is disabled before STARTTLS') + return + + maybeDeferred(self.authenticateLogin, user, passwd + ).addCallback(self.__cbLogin, tag + ).addErrback(self.__ebLogin, tag + ) + + unauth_LOGIN = (do_LOGIN, arg_astring, arg_finalastring) + + + def authenticateLogin(self, user, passwd): + """ + Lookup the account associated with the given parameters + + Override this method to define the desired authentication behavior. + + The default behavior is to defer authentication to C{self.portal} + if it is not None, or to deny the login otherwise. + + @type user: L{str} + @param user: The username to lookup + + @type passwd: L{str} + @param passwd: The password to login with + """ + if self.portal: + return self.portal.login( + credentials.UsernamePassword(user, passwd), + None, IAccount + ) + raise UnauthorizedLogin() + + + def __cbLogin(self, result, tag): + (iface, avatar, logout) = result + if iface is not IAccount: + self.sendBadResponse(tag, b'Server error: login returned unexpected value') + log.err("__cbLogin called with %r, IAccount expected" % (iface,)) + else: + self.account = avatar + self._onLogout = logout + self.sendPositiveResponse(tag, b'LOGIN succeeded') + self.state = 'auth' + self.setTimeout(self.POSTAUTH_TIMEOUT) + + + def __ebLogin(self, failure, tag): + if failure.check(UnauthorizedLogin): + self.sendNegativeResponse(tag, b'LOGIN failed') + else: + self.sendBadResponse(tag, b'Server error: ' + networkString(str(failure.value))) + log.err(failure) + + + def do_NAMESPACE(self, tag): + personal = public = shared = None + np = INamespacePresenter(self.account, None) + if np is not None: + personal = np.getPersonalNamespaces() + public = np.getSharedNamespaces() + shared = np.getSharedNamespaces() + self.sendUntaggedResponse(b'NAMESPACE ' + collapseNestedLists([personal, public, shared])) + self.sendPositiveResponse(tag, b"NAMESPACE command completed") + + auth_NAMESPACE = (do_NAMESPACE,) + select_NAMESPACE = auth_NAMESPACE + + + def _selectWork(self, tag, name, rw, cmdName): + if self.mbox: + self.mbox.removeListener(self) + cmbx = ICloseableMailbox(self.mbox, None) + if cmbx is not None: + maybeDeferred(cmbx.close).addErrback(log.err) + self.mbox = None + self.state = 'auth' + + name = _parseMbox(name) + maybeDeferred(self.account.select, _parseMbox(name), rw + ).addCallback(self._cbSelectWork, cmdName, tag + ).addErrback(self._ebSelectWork, cmdName, tag + ) + + + def _ebSelectWork(self, failure, cmdName, tag): + self.sendBadResponse(tag, cmdName + b" failed: Server error") + log.err(failure) + + + def _cbSelectWork(self, mbox, cmdName, tag): + if mbox is None: + self.sendNegativeResponse(tag, b'No such mailbox') + return + if '\\noselect' in [s.lower() for s in mbox.getFlags()]: + self.sendNegativeResponse(tag, 'Mailbox cannot be selected') + return + + flags = [networkString(flag) for flag in mbox.getFlags()] + self.sendUntaggedResponse(intToBytes(mbox.getMessageCount()) + b' EXISTS') + self.sendUntaggedResponse(intToBytes(mbox.getRecentCount()) + b' RECENT') + self.sendUntaggedResponse(b'FLAGS (' + b' '.join(flags) + b')') + self.sendPositiveResponse(None, b'[UIDVALIDITY ' + intToBytes(mbox.getUIDValidity()) + b']') + + s = mbox.isWriteable() and b'READ-WRITE' or b'READ-ONLY' + mbox.addListener(self) + self.sendPositiveResponse(tag, b'[' + s + b'] ' + cmdName + b' successful') + self.state = 'select' + self.mbox = mbox + + auth_SELECT = ( _selectWork, arg_astring, 1, b'SELECT' ) + select_SELECT = auth_SELECT + + auth_EXAMINE = ( _selectWork, arg_astring, 0, b'EXAMINE' ) + select_EXAMINE = auth_EXAMINE + + + def do_IDLE(self, tag): + self.sendContinuationRequest(None) + self.parseTag = tag + self.lastState = self.parseState + self.parseState = 'idle' + + + def parse_idle(self, *args): + self.parseState = self.lastState + del self.lastState + self.sendPositiveResponse(self.parseTag, b"IDLE terminated") + del self.parseTag + + select_IDLE = ( do_IDLE, ) + auth_IDLE = select_IDLE + + + def do_CREATE(self, tag, name): + name = _parseMbox(name) + try: + result = self.account.create(name) + except MailboxException as c: + self.sendNegativeResponse(tag, networkString(str(c))) + except: + self.sendBadResponse(tag, b"Server error encountered while creating mailbox") + log.err() + else: + if result: + self.sendPositiveResponse(tag, b'Mailbox created') + else: + self.sendNegativeResponse(tag, b'Mailbox not created') + + auth_CREATE = (do_CREATE, arg_finalastring) + select_CREATE = auth_CREATE + + + def do_DELETE(self, tag, name): + name = _parseMbox(name) + if name.lower() == 'inbox': + self.sendNegativeResponse(tag, b'You cannot delete the inbox') + return + try: + self.account.delete(name) + except MailboxException as m: + self.sendNegativeResponse(tag, str(m).encode("imap4-utf-7")) + except: + self.sendBadResponse(tag, b"Server error encountered while deleting mailbox") + log.err() + else: + self.sendPositiveResponse(tag, b'Mailbox deleted') + + auth_DELETE = (do_DELETE, arg_finalastring) + select_DELETE = auth_DELETE + + + def do_RENAME(self, tag, oldname, newname): + oldname, newname = [_parseMbox(n) for n in (oldname, newname)] + if oldname.lower() == 'inbox' or newname.lower() == 'inbox': + self.sendNegativeResponse(tag, b'You cannot rename the inbox, or rename another mailbox to inbox.') + return + try: + self.account.rename(oldname, newname) + except TypeError: + self.sendBadResponse(tag, b'Invalid command syntax') + except MailboxException as m: + self.sendNegativeResponse(tag, networkString(str(m))) + except: + self.sendBadResponse(tag, b"Server error encountered while renaming mailbox") + log.err() + else: + self.sendPositiveResponse(tag, b'Mailbox renamed') + + auth_RENAME = (do_RENAME, arg_astring, arg_finalastring) + select_RENAME = auth_RENAME + + + def do_SUBSCRIBE(self, tag, name): + name = _parseMbox(name) + try: + self.account.subscribe(name) + except MailboxException as m: + self.sendNegativeResponse(tag, networkString(str(m))) + except: + self.sendBadResponse(tag, b"Server error encountered while subscribing to mailbox") + log.err() + else: + self.sendPositiveResponse(tag, b'Subscribed') + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_finalastring) + select_SUBSCRIBE = auth_SUBSCRIBE + + + def do_UNSUBSCRIBE(self, tag, name): + name = _parseMbox(name) + try: + self.account.unsubscribe(name) + except MailboxException as m: + self.sendNegativeResponse(tag, networkString(str(m))) + except: + self.sendBadResponse(tag, b"Server error encountered while unsubscribing from mailbox") + log.err() + else: + self.sendPositiveResponse(tag, b'Unsubscribed') + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_finalastring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + + def _listWork(self, tag, ref, mbox, sub, cmdName): + mbox = _parseMbox(mbox) + ref = _parseMbox(ref) + maybeDeferred(self.account.listMailboxes, ref, mbox + ).addCallback(self._cbListWork, tag, sub, cmdName + ).addErrback(self._ebListWork, tag + ) + + + def _cbListWork(self, mailboxes, tag, sub, cmdName): + for (name, box) in mailboxes: + if not sub or self.account.isSubscribed(name): + flags = [networkString(flag) for flag in box.getFlags()] + delim = box.getHierarchicalDelimiter().encode('imap4-utf-7') + resp = (DontQuoteMe(cmdName), map(DontQuoteMe, flags), delim, name.encode('imap4-utf-7')) + self.sendUntaggedResponse(collapseNestedLists(resp)) + self.sendPositiveResponse(tag, cmdName + b' completed') + + + def _ebListWork(self, failure, tag): + self.sendBadResponse(tag, b"Server error encountered while listing mailboxes.") + log.err(failure) + + auth_LIST = (_listWork, arg_astring, arg_astring, 0, b'LIST') + select_LIST = auth_LIST + + auth_LSUB = (_listWork, arg_astring, arg_astring, 1, b'LSUB') + select_LSUB = auth_LSUB + + + def do_STATUS(self, tag, mailbox, names): + nativeNames = [] + for name in names: + nativeNames.append(nativeString(name)) + + mailbox = _parseMbox(mailbox) + + maybeDeferred(self.account.select, mailbox, 0 + ).addCallback(self._cbStatusGotMailbox, tag, mailbox, nativeNames + ).addErrback(self._ebStatusGotMailbox, tag + ) + + + def _cbStatusGotMailbox(self, mbox, tag, mailbox, names): + if mbox: + maybeDeferred(mbox.requestStatus, names).addCallbacks( + self.__cbStatus, self.__ebStatus, + (tag, mailbox), None, (tag, mailbox), None + ) + else: + self.sendNegativeResponse(tag, b"Could not open mailbox") + + + def _ebStatusGotMailbox(self, failure, tag): + self.sendBadResponse(tag, b"Server error encountered while opening mailbox.") + log.err(failure) + + auth_STATUS = (do_STATUS, arg_astring, arg_plist) + select_STATUS = auth_STATUS + + + def __cbStatus(self, status, tag, box): + # STATUS names should only be ASCII + line = networkString(' '.join(['%s %s' % x for x in status.items()])) + self.sendUntaggedResponse(b'STATUS ' + box.encode('imap4-utf-7') + b' ('+ line + b')') + self.sendPositiveResponse(tag, b'STATUS complete') + + + def __ebStatus(self, failure, tag, box): + self.sendBadResponse(tag, b'STATUS '+ box + b' failed: ' + + networkString(str(failure.value))) + + + def do_APPEND(self, tag, mailbox, flags, date, message): + mailbox = _parseMbox(mailbox) + maybeDeferred(self.account.select, mailbox + ).addCallback(self._cbAppendGotMailbox, tag, flags, date, message + ).addErrback(self._ebAppendGotMailbox, tag + ) + + + def _cbAppendGotMailbox(self, mbox, tag, flags, date, message): + if not mbox: + self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox') + return + + decodedFlags = [nativeString(flag) for flag in flags] + d = mbox.addMessage(message, decodedFlags, date) + d.addCallback(self.__cbAppend, tag, mbox) + d.addErrback(self.__ebAppend, tag) + + + def _ebAppendGotMailbox(self, failure, tag): + self.sendBadResponse(tag, b"Server error encountered while opening mailbox.") + log.err(failure) + + auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, + arg_literal) + select_APPEND = auth_APPEND + + + def __cbAppend(self, result, tag, mbox): + self.sendUntaggedResponse(intToBytes(mbox.getMessageCount()) + b' EXISTS') + self.sendPositiveResponse(tag, b'APPEND complete') + + + def __ebAppend(self, failure, tag): + self.sendBadResponse(tag, b'APPEND failed: ' + + networkString(str(failure.value))) + + + def do_CHECK(self, tag): + d = self.checkpoint() + if d is None: + self.__cbCheck(None, tag) + else: + d.addCallbacks( + self.__cbCheck, + self.__ebCheck, + callbackArgs=(tag,), + errbackArgs=(tag,) + ) + select_CHECK = (do_CHECK,) + + + def __cbCheck(self, result, tag): + self.sendPositiveResponse(tag, b'CHECK completed') + + + def __ebCheck(self, failure, tag): + self.sendBadResponse(tag, b'CHECK failed: ' + + networkString(str(failure.value))) + + + def checkpoint(self): + """ + Called when the client issues a CHECK command. + + This should perform any checkpoint operations required by the server. + It may be a long running operation, but may not block. If it returns + a deferred, the client will only be informed of success (or failure) + when the deferred's callback (or errback) is invoked. + """ + return None + + + def do_CLOSE(self, tag): + d = None + if self.mbox.isWriteable(): + d = maybeDeferred(self.mbox.expunge) + cmbx = ICloseableMailbox(self.mbox, None) + if cmbx is not None: + if d is not None: + d.addCallback(lambda result: cmbx.close()) + else: + d = maybeDeferred(cmbx.close) + if d is not None: + d.addCallbacks(self.__cbClose, self.__ebClose, (tag,), None, (tag,), None) + else: + self.__cbClose(None, tag) + + select_CLOSE = (do_CLOSE,) + + + def __cbClose(self, result, tag): + self.sendPositiveResponse(tag, b'CLOSE completed') + self.mbox.removeListener(self) + self.mbox = None + self.state = 'auth' + + + def __ebClose(self, failure, tag): + self.sendBadResponse(tag, b'CLOSE failed: ' + + networkString(str(failure.value))) + + + def do_EXPUNGE(self, tag): + if self.mbox.isWriteable(): + maybeDeferred(self.mbox.expunge).addCallbacks( + self.__cbExpunge, self.__ebExpunge, (tag,), None, (tag,), None + ) + else: + self.sendNegativeResponse(tag, b'EXPUNGE ignored on read-only mailbox') + + select_EXPUNGE = (do_EXPUNGE,) + + + def __cbExpunge(self, result, tag): + for e in result: + self.sendUntaggedResponse(intToBytes(e) + b' EXPUNGE') + self.sendPositiveResponse(tag, b'EXPUNGE completed') + + + def __ebExpunge(self, failure, tag): + self.sendBadResponse(tag, b'EXPUNGE failed: ' + + networkString(str(failure.value))) + log.err(failure) + + + def do_SEARCH(self, tag, charset, query, uid=0): + sm = ISearchableMailbox(self.mbox, None) + if sm is not None: + maybeDeferred(sm.search, query, uid=uid + ).addCallback(self.__cbSearch, tag, self.mbox, uid + ).addErrback(self.__ebSearch, tag) + else: + # that's not the ideal way to get all messages, there should be a + # method on mailboxes that gives you all of them + s = parseIdList(b'1:*') + maybeDeferred(self.mbox.fetch, s, uid=uid + ).addCallback(self.__cbManualSearch, + tag, self.mbox, query, uid + ).addErrback(self.__ebSearch, tag) + + select_SEARCH = (do_SEARCH, opt_charset, arg_searchkeys) + + + def __cbSearch(self, result, tag, mbox, uid): + if uid: + result = map(mbox.getUID, result) + ids = networkString(' '.join([str(i) for i in result])) + self.sendUntaggedResponse(b'SEARCH ' + ids) + self.sendPositiveResponse(tag, b'SEARCH completed') + + + def __cbManualSearch(self, result, tag, mbox, query, uid, + searchResults=None): + """ + Apply the search filter to a set of messages. Send the response to the + client. + + @type result: L{list} of L{tuple} of (L{int}, provider of + L{imap4.IMessage}) + @param result: A list two tuples of messages with their sequence ids, + sorted by the ids in descending order. + + @type tag: L{str} + @param tag: A command tag. + + @type mbox: Provider of L{imap4.IMailbox} + @param mbox: The searched mailbox. + + @type query: L{list} + @param query: A list representing the parsed form of the search query. + + @param uid: A flag indicating whether the search is over message + sequence numbers or UIDs. + + @type searchResults: L{list} + @param searchResults: The search results so far or L{None} if no + results yet. + """ + if searchResults is None: + searchResults = [] + i = 0 + + # result is a list of tuples (sequenceId, Message) + lastSequenceId = result and result[-1][0] + lastMessageId = result and result[-1][1].getUID() + for (i, (msgId, msg)) in list(zip(range(5), result)): + # searchFilter and singleSearchStep will mutate the query. Dang. + # Copy it here or else things will go poorly for subsequent + # messages. + if self._searchFilter(copy.deepcopy(query), msgId, msg, + lastSequenceId, lastMessageId): + if uid: + searchResults.append(intToBytes(msg.getUID())) + else: + searchResults.append(intToBytes(msgId)) + + if i == 4: + from twisted.internet import reactor + reactor.callLater( + 0, self.__cbManualSearch, list(result[5:]), tag, mbox, query, uid, + searchResults) + else: + if searchResults: + self.sendUntaggedResponse(b'SEARCH ' + b' '.join(searchResults)) + self.sendPositiveResponse(tag, b'SEARCH completed') + + + def _searchFilter(self, query, id, msg, lastSequenceId, lastMessageId): + """ + Pop search terms from the beginning of C{query} until there are none + left and apply them to the given message. + + @param query: A list representing the parsed form of the search query. + + @param id: The sequence number of the message being checked. + + @param msg: The message being checked. + + @type lastSequenceId: L{int} + @param lastSequenceId: The highest sequence number of any message in + the mailbox being searched. + + @type lastMessageId: L{int} + @param lastMessageId: The highest UID of any message in the mailbox + being searched. + + @return: Boolean indicating whether all of the query terms match the + message. + """ + while query: + if not self._singleSearchStep(query, id, msg, + lastSequenceId, lastMessageId): + return False + return True + + + def _singleSearchStep(self, query, msgId, msg, lastSequenceId, lastMessageId): + """ + Pop one search term from the beginning of C{query} (possibly more than + one element) and return whether it matches the given message. + + @param query: A list representing the parsed form of the search query. + + @param msgId: The sequence number of the message being checked. + + @param msg: The message being checked. + + @param lastSequenceId: The highest sequence number of any message in + the mailbox being searched. + + @param lastMessageId: The highest UID of any message in the mailbox + being searched. + + @return: Boolean indicating whether the query term matched the message. + """ + + q = query.pop(0) + if isinstance(q, list): + if not self._searchFilter(q, msgId, msg, + lastSequenceId, lastMessageId): + return False + else: + c = q.upper() + if not c[:1].isalpha(): + # A search term may be a word like ALL, ANSWERED, BCC, etc (see + # below) or it may be a message sequence set. Here we + # recognize a message sequence set "N:M". + messageSet = parseIdList(c, lastSequenceId) + return msgId in messageSet + else: + f = getattr(self, 'search_' + nativeString(c), None) + if f is None: + raise IllegalQueryError("Invalid search command %s" % nativeString(c)) + + if c in self._requiresLastMessageInfo: + result = f(query, msgId, msg, (lastSequenceId, + lastMessageId)) + else: + result = f(query, msgId, msg) + + if not result: + return False + return True + + + def search_ALL(self, query, id, msg): + """ + Returns C{True} if the message matches the ALL search key (always). + + @type query: A L{list} of L{str} + @param query: A list representing the parsed query string. + + @type id: L{int} + @param id: The sequence number of the message being checked. + + @type msg: Provider of L{imap4.IMessage} + """ + return True + + + def search_ANSWERED(self, query, id, msg): + """ + Returns C{True} if the message has been answered. + + @type query: A L{list} of L{str} + @param query: A list representing the parsed query string. + + @type id: L{int} + @param id: The sequence number of the message being checked. + + @type msg: Provider of L{imap4.IMessage} + """ + return '\\Answered' in msg.getFlags() + + + def search_BCC(self, query, id, msg): + """ + Returns C{True} if the message has a BCC address matching the query. + + @type query: A L{list} of L{str} + @param query: A list whose first element is a BCC L{str} + + @type id: L{int} + @param id: The sequence number of the message being checked. + + @type msg: Provider of L{imap4.IMessage} + """ + bcc = msg.getHeaders(False, 'bcc').get('bcc', '') + return bcc.lower().find(query.pop(0).lower()) != -1 + + + def search_BEFORE(self, query, id, msg): + date = parseTime(query.pop(0)) + return email.utils.parsedate(nativeString(msg.getInternalDate())) < date + + + def search_BODY(self, query, id, msg): + body = query.pop(0).lower() + return text.strFile(body, msg.getBodyFile(), False) + + + def search_CC(self, query, id, msg): + cc = msg.getHeaders(False, 'cc').get('cc', '') + return cc.lower().find(query.pop(0).lower()) != -1 + + + def search_DELETED(self, query, id, msg): + return '\\Deleted' in msg.getFlags() + + + def search_DRAFT(self, query, id, msg): + return '\\Draft' in msg.getFlags() + + + def search_FLAGGED(self, query, id, msg): + return '\\Flagged' in msg.getFlags() + + + def search_FROM(self, query, id, msg): + fm = msg.getHeaders(False, 'from').get('from', '') + return fm.lower().find(query.pop(0).lower()) != -1 + + + def search_HEADER(self, query, id, msg): + hdr = query.pop(0).lower() + hdr = msg.getHeaders(False, hdr).get(hdr, '') + return hdr.lower().find(query.pop(0).lower()) != -1 + + + def search_KEYWORD(self, query, id, msg): + query.pop(0) + return False + + + def search_LARGER(self, query, id, msg): + return int(query.pop(0)) < msg.getSize() + + + def search_NEW(self, query, id, msg): + return '\\Recent' in msg.getFlags() and '\\Seen' not in msg.getFlags() + + + def search_NOT(self, query, id, msg, lastIDs): + """ + Returns C{True} if the message does not match the query. + + @type query: A L{list} of L{str} + @param query: A list representing the parsed form of the search query. + + @type id: L{int} + @param id: The sequence number of the message being checked. + + @type msg: Provider of L{imap4.IMessage} + @param msg: The message being checked. + + @type lastIDs: L{tuple} + @param lastIDs: A tuple of (last sequence id, last message id). + The I{last sequence id} is an L{int} containing the highest sequence + number of a message in the mailbox. The I{last message id} is an + L{int} containing the highest UID of a message in the mailbox. + """ + (lastSequenceId, lastMessageId) = lastIDs + return not self._singleSearchStep(query, id, msg, + lastSequenceId, lastMessageId) + + + def search_OLD(self, query, id, msg): + return '\\Recent' not in msg.getFlags() + + + def search_ON(self, query, id, msg): + date = parseTime(query.pop(0)) + return email.utils.parsedate(msg.getInternalDate()) == date + + + def search_OR(self, query, id, msg, lastIDs): + """ + Returns C{True} if the message matches any of the first two query + items. + + @type query: A L{list} of L{str} + @param query: A list representing the parsed form of the search query. + + @type id: L{int} + @param id: The sequence number of the message being checked. + + @type msg: Provider of L{imap4.IMessage} + @param msg: The message being checked. + + @type lastIDs: L{tuple} + @param lastIDs: A tuple of (last sequence id, last message id). + The I{last sequence id} is an L{int} containing the highest sequence + number of a message in the mailbox. The I{last message id} is an + L{int} containing the highest UID of a message in the mailbox. + """ + (lastSequenceId, lastMessageId) = lastIDs + a = self._singleSearchStep(query, id, msg, + lastSequenceId, lastMessageId) + b = self._singleSearchStep(query, id, msg, + lastSequenceId, lastMessageId) + return a or b + + + def search_RECENT(self, query, id, msg): + return '\\Recent' in msg.getFlags() + + + def search_SEEN(self, query, id, msg): + return '\\Seen' in msg.getFlags() + + + def search_SENTBEFORE(self, query, id, msg): + """ + Returns C{True} if the message date is earlier than the query date. + + @type query: A L{list} of L{str} + @param query: A list whose first element starts with a stringified date + that is a fragment of an L{imap4.Query()}. The date must be in the + format 'DD-Mon-YYYY', for example '03-March-2003' or '03-Mar-2003'. + + @type id: L{int} + @param id: The sequence number of the message being checked. + + @type msg: Provider of L{imap4.IMessage} + """ + date = msg.getHeaders(False, 'date').get('date', '') + date = email.utils.parsedate(date) + return date < parseTime(query.pop(0)) + + + def search_SENTON(self, query, id, msg): + """ + Returns C{True} if the message date is the same as the query date. + + @type query: A L{list} of L{str} + @param query: A list whose first element starts with a stringified date + that is a fragment of an L{imap4.Query()}. The date must be in the + format 'DD-Mon-YYYY', for example '03-March-2003' or '03-Mar-2003'. + + @type msg: Provider of L{imap4.IMessage} + """ + date = msg.getHeaders(False, 'date').get('date', '') + date = email.utils.parsedate(date) + return date[:3] == parseTime(query.pop(0))[:3] + + + def search_SENTSINCE(self, query, id, msg): + """ + Returns C{True} if the message date is later than the query date. + + @type query: A L{list} of L{str} + @param query: A list whose first element starts with a stringified date + that is a fragment of an L{imap4.Query()}. The date must be in the + format 'DD-Mon-YYYY', for example '03-March-2003' or '03-Mar-2003'. + + @type msg: Provider of L{imap4.IMessage} + """ + date = msg.getHeaders(False, 'date').get('date', '') + date = email.utils.parsedate(date) + return date > parseTime(query.pop(0)) + + + def search_SINCE(self, query, id, msg): + date = parseTime(query.pop(0)) + return email.utils.parsedate(msg.getInternalDate()) > date + + + def search_SMALLER(self, query, id, msg): + return int(query.pop(0)) > msg.getSize() + + + def search_SUBJECT(self, query, id, msg): + subj = msg.getHeaders(False, 'subject').get('subject', '') + return subj.lower().find(query.pop(0).lower()) != -1 + + + def search_TEXT(self, query, id, msg): + # XXX - This must search headers too + body = query.pop(0).lower() + return text.strFile(body, msg.getBodyFile(), False) + + + def search_TO(self, query, id, msg): + to = msg.getHeaders(False, 'to').get('to', '') + return to.lower().find(query.pop(0).lower()) != -1 + + + def search_UID(self, query, id, msg, lastIDs): + """ + Returns C{True} if the message UID is in the range defined by the + search query. + + @type query: A L{list} of L{bytes} + @param query: A list representing the parsed form of the search + query. Its first element should be a L{str} that can be interpreted + as a sequence range, for example '2:4,5:*'. + + @type id: L{int} + @param id: The sequence number of the message being checked. + + @type msg: Provider of L{imap4.IMessage} + @param msg: The message being checked. + + @type lastIDs: L{tuple} + @param lastIDs: A tuple of (last sequence id, last message id). + The I{last sequence id} is an L{int} containing the highest sequence + number of a message in the mailbox. The I{last message id} is an + L{int} containing the highest UID of a message in the mailbox. + """ + (lastSequenceId, lastMessageId) = lastIDs + c = query.pop(0) + m = parseIdList(c, lastMessageId) + return msg.getUID() in m + + + def search_UNANSWERED(self, query, id, msg): + return '\\Answered' not in msg.getFlags() + + + def search_UNDELETED(self, query, id, msg): + return '\\Deleted' not in msg.getFlags() + + + def search_UNDRAFT(self, query, id, msg): + return '\\Draft' not in msg.getFlags() + + + def search_UNFLAGGED(self, query, id, msg): + return '\\Flagged' not in msg.getFlags() + + + def search_UNKEYWORD(self, query, id, msg): + query.pop(0) + return False + + + def search_UNSEEN(self, query, id, msg): + return '\\Seen' not in msg.getFlags() + + + def __ebSearch(self, failure, tag): + self.sendBadResponse(tag, b'SEARCH failed: ' + + networkString(str(failure.value))) + log.err(failure) + + + def do_FETCH(self, tag, messages, query, uid=0): + if query: + self._oldTimeout = self.setTimeout(None) + maybeDeferred(self.mbox.fetch, messages, uid=uid + ).addCallback(iter + ).addCallback(self.__cbFetch, tag, query, uid + ).addErrback(self.__ebFetch, tag + ) + else: + self.sendPositiveResponse(tag, b'FETCH complete') + + select_FETCH = (do_FETCH, arg_seqset, arg_fetchatt) + + + def __cbFetch(self, results, tag, query, uid): + if self.blocked is None: + self.blocked = [] + try: + id, msg = next(results) + except StopIteration: + # The idle timeout was suspended while we delivered results, + # restore it now. + self.setTimeout(self._oldTimeout) + del self._oldTimeout + + # All results have been processed, deliver completion notification. + + # It's important to run this *after* resetting the timeout to "rig + # a race" in some test code. writing to the transport will + # synchronously call test code, which synchronously loses the + # connection, calling our connectionLost method, which cancels the + # timeout. We want to make sure that timeout is cancelled *after* + # we reset it above, so that the final state is no timed + # calls. This avoids reactor uncleanliness errors in the test + # suite. + # XXX: Perhaps loopback should be fixed to not call the user code + # synchronously in transport.write? + self.sendPositiveResponse(tag, b'FETCH completed') + + # Instance state is now consistent again (ie, it is as though + # the fetch command never ran), so allow any pending blocked + # commands to execute. + self._unblock() + else: + self.spewMessage(id, msg, query, uid + ).addCallback(lambda _: self.__cbFetch(results, tag, query, uid) + ).addErrback(self.__ebSpewMessage + ) + + + def __ebSpewMessage(self, failure): + # This indicates a programming error. + # There's no reliable way to indicate anything to the client, since we + # may have already written an arbitrary amount of data in response to + # the command. + log.err(failure) + self.transport.loseConnection() + + + def spew_envelope(self, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + _w(b'ENVELOPE ' + collapseNestedLists([getEnvelope(msg)])) + + + def spew_flags(self, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.writen + encodedFlags = [networkString(flag) for flag in msg.getFlags()] + _w(b'FLAGS ' + b'(' + b' '.join(encodedFlags) + b')') + + + def spew_internaldate(self, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + idate = msg.getInternalDate() + ttup = email.utils.parsedate_tz(nativeString(idate)) + if ttup is None: + log.msg("%d:%r: unpareseable internaldate: %r" % (id, msg, idate)) + raise IMAP4Exception("Internal failure generating INTERNALDATE") + + # need to specify the month manually, as strftime depends on locale + strdate = time.strftime("%d-%%s-%Y %H:%M:%S ", ttup[:9]) + odate = networkString(strdate % (_MONTH_NAMES[ttup[1]],)) + if ttup[9] is None: + odate = odate + b"+0000" + else: + if ttup[9] >= 0: + sign = b"+" + else: + sign = b"-" + odate = odate + sign + intToBytes( + ((abs(ttup[9]) // 3600) * 100 + + (abs(ttup[9]) % 3600) // 60) + ).zfill(4) + _w(b'INTERNALDATE ' + _quote(odate)) + + + def spew_rfc822header(self, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + hdrs = _formatHeaders(msg.getHeaders(True)) + _w(b'RFC822.HEADER ' + _literal(hdrs)) + + + def spew_rfc822text(self, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + _w(b'RFC822.TEXT ') + _f() + return FileProducer(msg.getBodyFile() + ).beginProducing(self.transport + ) + + + def spew_rfc822size(self, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + _w(b'RFC822.SIZE ' + intToBytes(msg.getSize())) + + + def spew_rfc822(self, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + _w(b'RFC822 ') + _f() + mf = IMessageFile(msg, None) + if mf is not None: + return FileProducer(mf.open() + ).beginProducing(self.transport + ) + return MessageProducer(msg, None, self._scheduler + ).beginProducing(self.transport + ) + + + def spew_uid(self, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + _w(b'UID ' + intToBytes(msg.getUID())) + + + def spew_bodystructure(self, id, msg, _w=None, _f=None): + _w(b'BODYSTRUCTURE ' + collapseNestedLists([getBodyStructure(msg, True)])) + + + def spew_body(self, part, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + for p in part.part: + if msg.isMultipart(): + msg = msg.getSubPart(p) + elif p > 0: + # Non-multipart messages have an implicit first part but no + # other parts - reject any request for any other part. + raise TypeError("Requested subpart of non-multipart message") + + if part.header: + hdrs = msg.getHeaders(part.header.negate, *part.header.fields) + hdrs = _formatHeaders(hdrs) + _w(part.__bytes__() + b' ' + _literal(hdrs)) + elif part.text: + _w(part.__bytes__() + b' ') + _f() + return FileProducer(msg.getBodyFile() + ).beginProducing(self.transport + ) + elif part.mime: + hdrs = _formatHeaders(msg.getHeaders(True)) + _w(part.__bytes__() + b' ' + _literal(hdrs)) + elif part.empty: + _w(part.__bytes__() + b' ') + _f() + if part.part: + return FileProducer(msg.getBodyFile() + ).beginProducing(self.transport + ) + else: + mf = IMessageFile(msg, None) + if mf is not None: + return FileProducer(mf.open()).beginProducing(self.transport) + return MessageProducer(msg, None, self._scheduler).beginProducing(self.transport) + + else: + _w(b'BODY ' + collapseNestedLists([getBodyStructure(msg)])) + + + def spewMessage(self, id, msg, query, uid): + wbuf = WriteBuffer(self.transport) + write = wbuf.write + flush = wbuf.flush + def start(): + write(b'* ' + intToBytes(id) + b' FETCH (') + def finish(): + write(b')\r\n') + def space(): + write(b' ') + + def spew(): + seenUID = False + start() + for part in query: + if part.type == 'uid': + seenUID = True + if part.type == 'body': + yield self.spew_body(part, id, msg, write, flush) + else: + f = getattr(self, 'spew_' + part.type) + yield f(id, msg, write, flush) + if part is not query[-1]: + space() + if uid and not seenUID: + space() + yield self.spew_uid(id, msg, write, flush) + finish() + flush() + return self._scheduler(spew()) + + + def __ebFetch(self, failure, tag): + self.setTimeout(self._oldTimeout) + del self._oldTimeout + log.err(failure) + self.sendBadResponse(tag, b'FETCH failed: ' + + networkString(str(failure.value))) + + + def do_STORE(self, tag, messages, mode, flags, uid=0): + mode = mode.upper() + silent = mode.endswith(b'SILENT') + if mode.startswith(b'+'): + mode = 1 + elif mode.startswith(b'-'): + mode = -1 + else: + mode = 0 + + flags = [nativeString(flag) for flag in flags] + maybeDeferred(self.mbox.store, messages, flags, mode, uid=uid).addCallbacks( + self.__cbStore, self.__ebStore, (tag, self.mbox, uid, silent), None, (tag,), None + ) + + select_STORE = (do_STORE, arg_seqset, arg_atom, arg_flaglist) + + + def __cbStore(self, result, tag, mbox, uid, silent): + if result and not silent: + for (k, v) in result.items(): + if uid: + uidstr = b' UID ' + intToBytes(mbox.getUID(k)) + else: + uidstr = b'' + + flags = [networkString(flag) for flag in v] + self.sendUntaggedResponse( + intToBytes(k) + + b' FETCH (FLAGS ('+ b' '.join(flags) + b')' + + uidstr + b')') + self.sendPositiveResponse(tag, b'STORE completed') + + + def __ebStore(self, failure, tag): + self.sendBadResponse(tag, b'Server error: ' + + networkString(str(failure.value))) + + + def do_COPY(self, tag, messages, mailbox, uid=0): + mailbox = self._parseMbox(mailbox) + maybeDeferred(self.account.select, mailbox + ).addCallback(self._cbCopySelectedMailbox, tag, messages, mailbox, uid + ).addErrback(self._ebCopySelectedMailbox, tag + ) + select_COPY = (do_COPY, arg_seqset, arg_finalastring) + + + def _cbCopySelectedMailbox(self, mbox, tag, messages, mailbox, uid): + if not mbox: + self.sendNegativeResponse(tag, 'No such mailbox: ' + mailbox) + else: + maybeDeferred(self.mbox.fetch, messages, uid + ).addCallback(self.__cbCopy, tag, mbox + ).addCallback(self.__cbCopied, tag, mbox + ).addErrback(self.__ebCopy, tag + ) + + + def _ebCopySelectedMailbox(self, failure, tag): + self.sendBadResponse(tag, b'Server error: ' + + networkString(str(failure.value))) + + + def __cbCopy(self, messages, tag, mbox): + # XXX - This should handle failures with a rollback or something + addedDeferreds = [] + + fastCopyMbox = IMessageCopier(mbox, None) + for (id, msg) in messages: + if fastCopyMbox is not None: + d = maybeDeferred(fastCopyMbox.copy, msg) + addedDeferreds.append(d) + continue + + # XXX - The following should be an implementation of IMessageCopier.copy + # on an IMailbox->IMessageCopier adapter. + + flags = msg.getFlags() + date = msg.getInternalDate() + + body = IMessageFile(msg, None) + if body is not None: + bodyFile = body.open() + d = maybeDeferred(mbox.addMessage, bodyFile, flags, date) + else: + def rewind(f): + f.seek(0) + return f + buffer = tempfile.TemporaryFile() + d = MessageProducer(msg, buffer, self._scheduler + ).beginProducing(None + ).addCallback(lambda _, b=buffer, f=flags, d=date: mbox.addMessage(rewind(b), f, d) + ) + addedDeferreds.append(d) + return defer.DeferredList(addedDeferreds) + + + def __cbCopied(self, deferredIds, tag, mbox): + ids = [] + failures = [] + for (status, result) in deferredIds: + if status: + ids.append(result) + else: + failures.append(result.value) + if failures: + self.sendNegativeResponse(tag, '[ALERT] Some messages were not copied') + else: + self.sendPositiveResponse(tag, b'COPY completed') + + + def __ebCopy(self, failure, tag): + self.sendBadResponse(tag, b'COPY failed:' + + networkString(str(failure.value))) + log.err(failure) + + + def do_UID(self, tag, command, line): + command = command.upper() + + if command not in (b'COPY', b'FETCH', b'STORE', b'SEARCH'): + raise IllegalClientResponse(command) + + self.dispatchCommand(tag, command, line, uid=1) + + select_UID = (do_UID, arg_atom, arg_line) + + + # + # IMailboxListener implementation + # + def modeChanged(self, writeable): + if writeable: + self.sendUntaggedResponse(message=b'[READ-WRITE]', isAsync=True) + else: + self.sendUntaggedResponse(message=b'[READ-ONLY]', isAsync=True) + + + def flagsChanged(self, newFlags): + for (mId, flags) in newFlags.items(): + encodedFlags = [networkString(flag) for flag in flags] + msg = intToBytes(mId) + ( + b' FETCH (FLAGS (' + b' '.join(encodedFlags) + b'))' + ) + self.sendUntaggedResponse(msg, isAsync=True) + + + def newMessages(self, exists, recent): + if exists is not None: + self.sendUntaggedResponse( + intToBytes(exists) + b' EXISTS', isAsync=True) + if recent is not None: + self.sendUntaggedResponse( + intToBytes(recent) + b' RECENT', isAsync=True) + + + +TIMEOUT_ERROR = error.TimeoutError() + +@implementer(IMailboxListener) +class IMAP4Client(basic.LineReceiver, policies.TimeoutMixin): + """IMAP4 client protocol implementation + + @ivar state: A string representing the state the connection is currently + in. + """ + tags = None + waiting = None + queued = None + tagID = 1 + state = None + + startedTLS = False + + # Number of seconds to wait before timing out a connection. + # If the number is <= 0 no timeout checking will be performed. + timeout = 0 + + # Capabilities are not allowed to change during the session + # So cache the first response and use that for all later + # lookups + _capCache = None + + _memoryFileLimit = 1024 * 1024 * 10 + + # Authentication is pluggable. This maps names to IClientAuthentication + # objects. + authenticators = None + + STATUS_CODES = ('OK', 'NO', 'BAD', 'PREAUTH', 'BYE') + + STATUS_TRANSFORMATIONS = { + 'MESSAGES': int, 'RECENT': int, 'UNSEEN': int + } + + context = None + + def __init__(self, contextFactory = None): + self.tags = {} + self.queued = [] + self.authenticators = {} + self.context = contextFactory + + self._tag = None + self._parts = None + self._lastCmd = None + + + def registerAuthenticator(self, auth): + """ + Register a new form of authentication + + When invoking the authenticate() method of IMAP4Client, the first + matching authentication scheme found will be used. The ordering is + that in which the server lists support authentication schemes. + + @type auth: Implementor of C{IClientAuthentication} + @param auth: The object to use to perform the client + side of this authentication scheme. + """ + self.authenticators[auth.getName().upper()] = auth + + + def rawDataReceived(self, data): + if self.timeout > 0: + self.resetTimeout() + + self._pendingSize -= len(data) + if self._pendingSize > 0: + self._pendingBuffer.write(data) + else: + passon = b'' + if self._pendingSize < 0: + data, passon = data[:self._pendingSize], data[self._pendingSize:] + self._pendingBuffer.write(data) + rest = self._pendingBuffer + self._pendingBuffer = None + self._pendingSize = None + rest.seek(0, 0) + self._parts.append(rest.read()) + self.setLineMode(passon.lstrip(b'\r\n')) + +# def sendLine(self, line): +# print 'S:', repr(line) +# return basic.LineReceiver.sendLine(self, line) + + + def _setupForLiteral(self, rest, octets): + self._pendingBuffer = self.messageFile(octets) + self._pendingSize = octets + if self._parts is None: + self._parts = [rest, b'\r\n'] + else: + self._parts.extend([rest, b'\r\n']) + self.setRawMode() + + + def connectionMade(self): + if self.timeout > 0: + self.setTimeout(self.timeout) + + + def connectionLost(self, reason): + """ + We are no longer connected + """ + if self.timeout > 0: + self.setTimeout(None) + if self.queued is not None: + queued = self.queued + self.queued = None + for cmd in queued: + cmd.defer.errback(reason) + if self.tags is not None: + tags = self.tags + self.tags = None + for cmd in tags.values(): + if cmd is not None and cmd.defer is not None: + cmd.defer.errback(reason) + + + def lineReceived(self, line): + """ + Attempt to parse a single line from the server. + + @type line: L{bytes} + @param line: The line from the server, without the line delimiter. + + @raise IllegalServerResponse: If the line or some part of the line + does not represent an allowed message from the server at this time. + """ +# print('C: ' + repr(line)) + if self.timeout > 0: + self.resetTimeout() + + lastPart = line.rfind(b'{') + if lastPart != -1: + lastPart = line[lastPart + 1:] + if lastPart.endswith(b'}'): + # It's a literal a-comin' in + try: + octets = int(lastPart[:-1]) + except ValueError: + raise IllegalServerResponse(line) + if self._parts is None: + self._tag, parts = line.split(None, 1) + else: + parts = line + self._setupForLiteral(parts, octets) + return + + if self._parts is None: + # It isn't a literal at all + self._regularDispatch(line) + else: + # If an expression is in progress, no tag is required here + # Since we didn't find a literal indicator, this expression + # is done. + self._parts.append(line) + tag, rest = self._tag, b''.join(self._parts) + self._tag = self._parts = None + self.dispatchCommand(tag, rest) + + + def timeoutConnection(self): + if self._lastCmd and self._lastCmd.defer is not None: + d, self._lastCmd.defer = self._lastCmd.defer, None + d.errback(TIMEOUT_ERROR) + + if self.queued: + for cmd in self.queued: + if cmd.defer is not None: + d, cmd.defer = cmd.defer, d + d.errback(TIMEOUT_ERROR) + + self.transport.loseConnection() + + + def _regularDispatch(self, line): + parts = line.split(None, 1) + if len(parts) != 2: + parts.append(b'') + tag, rest = parts + self.dispatchCommand(tag, rest) + + + def messageFile(self, octets): + """ + Create a file to which an incoming message may be written. + + @type octets: L{int} + @param octets: The number of octets which will be written to the file + + @rtype: Any object which implements C{write(string)} and + C{seek(int, int)} + @return: A file-like object + """ + if octets > self._memoryFileLimit: + return tempfile.TemporaryFile() + else: + return BytesIO() + + + def makeTag(self): + tag = (u'%0.4X' % self.tagID).encode("ascii") + self.tagID += 1 + return tag + + + def dispatchCommand(self, tag, rest): + if self.state is None: + f = self.response_UNAUTH + else: + f = getattr(self, 'response_' + self.state.upper(), None) + if f: + try: + f(tag, rest) + except: + log.err() + self.transport.loseConnection() + else: + log.err("Cannot dispatch: %s, %r, %r" % (self.state, tag, rest)) + self.transport.loseConnection() + + + def response_UNAUTH(self, tag, rest): + if self.state is None: + # Server greeting, this is + status, rest = rest.split(None, 1) + if status.upper() == b'OK': + self.state = 'unauth' + elif status.upper() == b'PREAUTH': + self.state = 'auth' + else: + # XXX - This is rude. + self.transport.loseConnection() + raise IllegalServerResponse(tag + b' ' + rest) + + b, e = rest.find(b'['), rest.find(b']') + if b != -1 and e != -1: + self.serverGreeting( + self.__cbCapabilities( + ([parseNestedParens(rest[b + 1:e])], None))) + else: + self.serverGreeting(None) + else: + self._defaultHandler(tag, rest) + + + def response_AUTH(self, tag, rest): + self._defaultHandler(tag, rest) + + + def _defaultHandler(self, tag, rest): + if tag == b'*' or tag == b'+': + if not self.waiting: + self._extraInfo([parseNestedParens(rest)]) + else: + cmd = self.tags[self.waiting] + if tag == b'+': + cmd.continuation(rest) + else: + cmd.lines.append(rest) + else: + try: + cmd = self.tags[tag] + except KeyError: + # XXX - This is rude. + self.transport.loseConnection() + raise IllegalServerResponse(tag + b' ' + rest) + else: + status, line = rest.split(None, 1) + if status == b'OK': + # Give them this last line, too + cmd.finish(rest, self._extraInfo) + else: + cmd.defer.errback(IMAP4Exception(line)) + del self.tags[tag] + self.waiting = None + self._flushQueue() + + + def _flushQueue(self): + if self.queued: + cmd = self.queued.pop(0) + t = self.makeTag() + self.tags[t] = cmd + self.sendLine(cmd.format(t)) + self.waiting = t + + + def _extraInfo(self, lines): + # XXX - This is terrible. + # XXX - Also, this should collapse temporally proximate calls into single + # invocations of IMailboxListener methods, where possible. + flags = {} + recent = exists = None + for response in lines: + elements = len(response) + if elements == 1 and response[0] == [b'READ-ONLY']: + self.modeChanged(False) + elif elements == 1 and response[0] == [b'READ-WRITE']: + self.modeChanged(True) + elif elements == 2 and response[1] == b'EXISTS': + exists = int(response[0]) + elif elements == 2 and response[1] == b'RECENT': + recent = int(response[0]) + elif elements == 3 and response[1] == b'FETCH': + mId = int(response[0]) + values, _ = self._parseFetchPairs(response[2]) + flags.setdefault(mId, []).extend(values.get('FLAGS', ())) + else: + log.msg('Unhandled unsolicited response: %s' % (response,)) + + if flags: + self.flagsChanged(flags) + if recent is not None or exists is not None: + self.newMessages(exists, recent) + + + def sendCommand(self, cmd): + cmd.defer = defer.Deferred() + if self.waiting: + self.queued.append(cmd) + return cmd.defer + t = self.makeTag() + self.tags[t] = cmd + self.sendLine(cmd.format(t)) + self.waiting = t + self._lastCmd = cmd + return cmd.defer + + + def getCapabilities(self, useCache=1): + """ + Request the capabilities available on this server. + + This command is allowed in any state of connection. + + @type useCache: C{bool} + @param useCache: Specify whether to use the capability-cache or to + re-retrieve the capabilities from the server. Server capabilities + should never change, so for normal use, this flag should never be + false. + + @rtype: L{Deferred} + @return: A deferred whose callback will be invoked with a + dictionary mapping capability types to lists of supported + mechanisms, or to None if a support list is not applicable. + """ + if useCache and self._capCache is not None: + return defer.succeed(self._capCache) + cmd = b'CAPABILITY' + resp = (b'CAPABILITY',) + d = self.sendCommand(Command(cmd, wantResponse=resp)) + d.addCallback(self.__cbCapabilities) + return d + + + def __cbCapabilities(self, result): + (lines, tagline) = result + caps = {} + for rest in lines: + for cap in rest[1:]: + parts = cap.split(b'=', 1) + if len(parts) == 1: + category, value = parts[0], None + else: + category, value = parts + caps.setdefault(category, []).append(value) + + # Preserve a non-ideal API for backwards compatibility. It would + # probably be entirely sensible to have an object with a wider API than + # dict here so this could be presented less insanely. + for category in caps: + if caps[category] == [None]: + caps[category] = None + self._capCache = caps + return caps + + + def logout(self): + """ + Inform the server that we are done with the connection. + + This command is allowed in any state of connection. + + @rtype: L{Deferred} + @return: A deferred whose callback will be invoked with None + when the proper server acknowledgement has been received. + """ + d = self.sendCommand(Command(b'LOGOUT', wantResponse=(b'BYE',))) + d.addCallback(self.__cbLogout) + return d + + + def __cbLogout(self, result): + (lines, tagline) = result + self.transport.loseConnection() + # We don't particularly care what the server said + return None + + + def noop(self): + """ + Perform no operation. + + This command is allowed in any state of connection. + + @rtype: L{Deferred} + @return: A deferred whose callback will be invoked with a list + of untagged status updates the server responds with. + """ + d = self.sendCommand(Command(b'NOOP')) + d.addCallback(self.__cbNoop) + return d + + + def __cbNoop(self, result): + # Conceivable, this is elidable. + # It is, afterall, a no-op. + (lines, tagline) = result + return lines + + + def startTLS(self, contextFactory=None): + """ + Initiates a 'STARTTLS' request and negotiates the TLS / SSL + Handshake. + + @param contextFactory: The TLS / SSL Context Factory to + leverage. If the contextFactory is None the IMAP4Client will + either use the current TLS / SSL Context Factory or attempt to + create a new one. + + @type contextFactory: C{ssl.ClientContextFactory} + + @return: A Deferred which fires when the transport has been + secured according to the given contextFactory, or which fails + if the transport cannot be secured. + """ + assert not self.startedTLS, "Client and Server are currently communicating via TLS" + if contextFactory is None: + contextFactory = self._getContextFactory() + + if contextFactory is None: + return defer.fail(IMAP4Exception( + "IMAP4Client requires a TLS context to " + "initiate the STARTTLS handshake")) + + if b'STARTTLS' not in self._capCache: + return defer.fail(IMAP4Exception( + "Server does not support secure communication " + "via TLS / SSL")) + + tls = interfaces.ITLSTransport(self.transport, None) + if tls is None: + return defer.fail(IMAP4Exception( + "IMAP4Client transport does not implement " + "interfaces.ITLSTransport")) + + d = self.sendCommand(Command(b'STARTTLS')) + d.addCallback(self._startedTLS, contextFactory) + d.addCallback(lambda _: self.getCapabilities()) + return d + + + def authenticate(self, secret): + """ + Attempt to enter the authenticated state with the server + + This command is allowed in the Non-Authenticated state. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked if the authentication + succeeds and whose errback will be invoked otherwise. + """ + if self._capCache is None: + d = self.getCapabilities() + else: + d = defer.succeed(self._capCache) + d.addCallback(self.__cbAuthenticate, secret) + return d + + + def __cbAuthenticate(self, caps, secret): + auths = caps.get(b'AUTH', ()) + for scheme in auths: + if scheme.upper() in self.authenticators: + cmd = Command(b'AUTHENTICATE', scheme, (), + self.__cbContinueAuth, scheme, + secret) + return self.sendCommand(cmd) + + if self.startedTLS: + return defer.fail(NoSupportedAuthentication( + auths, self.authenticators.keys())) + else: + def ebStartTLS(err): + err.trap(IMAP4Exception) + # We couldn't negotiate TLS for some reason + return defer.fail(NoSupportedAuthentication( + auths, self.authenticators.keys())) + + d = self.startTLS() + d.addErrback(ebStartTLS) + d.addCallback(lambda _: self.getCapabilities()) + d.addCallback(self.__cbAuthTLS, secret) + return d + + + def __cbContinueAuth(self, rest, scheme, secret): + try: + chal = decodebytes(rest + b'\n') + except binascii.Error: + self.sendLine(b'*') + raise IllegalServerResponse(rest) + else: + auth = self.authenticators[scheme] + chal = auth.challengeResponse(secret, chal) + self.sendLine(encodebytes(chal).strip()) + + + def __cbAuthTLS(self, caps, secret): + auths = caps.get(b'AUTH', ()) + for scheme in auths: + if scheme.upper() in self.authenticators: + cmd = Command(b'AUTHENTICATE', scheme, (), + self.__cbContinueAuth, scheme, + secret) + return self.sendCommand(cmd) + raise NoSupportedAuthentication(auths, self.authenticators.keys()) + + + def login(self, username, password): + """ + Authenticate with the server using a username and password + + This command is allowed in the Non-Authenticated state. If the + server supports the STARTTLS capability and our transport supports + TLS, TLS is negotiated before the login command is issued. + + A more secure way to log in is to use C{startTLS} or + C{authenticate} or both. + + @type username: L{str} + @param username: The username to log in with + + @type password: L{str} + @param password: The password to log in with + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked if login is successful + and whose errback is invoked otherwise. + """ + d = maybeDeferred(self.getCapabilities) + d.addCallback(self.__cbLoginCaps, username, password) + return d + + + def serverGreeting(self, caps): + """ + Called when the server has sent us a greeting. + + @type caps: C{dict} + @param caps: Capabilities the server advertised in its greeting. + """ + + + def _getContextFactory(self): + if self.context is not None: + return self.context + try: + from twisted.internet import ssl + except ImportError: + return None + else: + context = ssl.ClientContextFactory() + context.method = ssl.SSL.TLSv1_METHOD + return context + + + def __cbLoginCaps(self, capabilities, username, password): + # If the server advertises STARTTLS, we might want to try to switch to TLS + tryTLS = b'STARTTLS' in capabilities + + # If our transport supports switching to TLS, we might want to try to switch to TLS. + tlsableTransport = interfaces.ITLSTransport(self.transport, None) is not None + + # If our transport is not already using TLS, we might want to try to switch to TLS. + nontlsTransport = interfaces.ISSLTransport(self.transport, None) is None + + if not self.startedTLS and tryTLS and tlsableTransport and nontlsTransport: + d = self.startTLS() + + d.addCallbacks( + self.__cbLoginTLS, + self.__ebLoginTLS, + callbackArgs=(username, password), + ) + return d + else: + if nontlsTransport: + log.msg("Server has no TLS support. logging in over cleartext!") + args = b' '.join((_quote(username), _quote(password))) + return self.sendCommand(Command(b'LOGIN', args)) + + + def _startedTLS(self, result, context): + self.transport.startTLS(context) + self._capCache = None + self.startedTLS = True + return result + + + def __cbLoginTLS(self, result, username, password): + args = b' '.join((_quote(username), _quote(password))) + return self.sendCommand(Command(b'LOGIN', args)) + + + def __ebLoginTLS(self, failure): + log.err(failure) + return failure + + + def namespace(self): + """ + Retrieve information about the namespaces available to this account + + This command is allowed in the Authenticated and Selected states. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with namespace + information. An example of this information is:: + + [[['', '/']], [], []] + + which indicates a single personal namespace called '' with '/' + as its hierarchical delimiter, and no shared or user namespaces. + """ + cmd = b'NAMESPACE' + resp = (b'NAMESPACE',) + d = self.sendCommand(Command(cmd, wantResponse=resp)) + d.addCallback(self.__cbNamespace) + return d + + + def __cbNamespace(self, result): + (lines, last) = result + + # Namespaces and their delimiters qualify and delimit + # mailboxes, so they should be native strings + # + # On Python 2, no decoding is necessary to maintain + # the API contract. + # + # On Python 3, users specify mailboxes with native strings, so + # they should receive namespaces and delimiters as native + # strings. Both cases are possible because of the imap4-utf-7 + # encoding. + if _PY3: + def _prepareNamespaceOrDelimiter(namespaceList): + return [ + element.decode('imap4-utf-7') for element in namespaceList + ] + else: + def _prepareNamespaceOrDelimiter(element): + return element + + for parts in lines: + if len(parts) == 4 and parts[0] == b'NAMESPACE': + return [ + [] + if pairOrNone is None else + [ + _prepareNamespaceOrDelimiter(value) + for value in pairOrNone + ] + for pairOrNone in parts[1:] + ] + log.err("No NAMESPACE response to NAMESPACE command") + return [[], [], []] + + + def select(self, mailbox): + """ + Select a mailbox + + This command is allowed in the Authenticated and Selected states. + + @type mailbox: L{str} + @param mailbox: The name of the mailbox to select + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with mailbox + information if the select is successful and whose errback is + invoked otherwise. Mailbox information consists of a dictionary + with the following L{str} keys and values:: + + FLAGS: A list of strings containing the flags settable on + messages in this mailbox. + + EXISTS: An integer indicating the number of messages in this + mailbox. + + RECENT: An integer indicating the number of "recent" + messages in this mailbox. + + UNSEEN: The message sequence number (an integer) of the + first unseen message in the mailbox. + + PERMANENTFLAGS: A list of strings containing the flags that + can be permanently set on messages in this mailbox. + + UIDVALIDITY: An integer uniquely identifying this mailbox. + """ + cmd = b'SELECT' + args = _prepareMailboxName(mailbox) + # This appears not to be used, so we can use native strings to + # indicate that the return type is native strings. + resp = ('FLAGS', 'EXISTS', 'RECENT', + 'UNSEEN', 'PERMANENTFLAGS', 'UIDVALIDITY') + d = self.sendCommand(Command(cmd, args, wantResponse=resp)) + d.addCallback(self.__cbSelect, 1) + return d + + + def examine(self, mailbox): + """ + Select a mailbox in read-only mode + + This command is allowed in the Authenticated and Selected states. + + @type mailbox: L{str} + @param mailbox: The name of the mailbox to examine + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with mailbox + information if the examine is successful and whose errback + is invoked otherwise. Mailbox information consists of a dictionary + with the following keys and values:: + + 'FLAGS': A list of strings containing the flags settable on + messages in this mailbox. + + 'EXISTS': An integer indicating the number of messages in this + mailbox. + + 'RECENT': An integer indicating the number of \"recent\" + messages in this mailbox. + + 'UNSEEN': An integer indicating the number of messages not + flagged \\Seen in this mailbox. + + 'PERMANENTFLAGS': A list of strings containing the flags that + can be permanently set on messages in this mailbox. + + 'UIDVALIDITY': An integer uniquely identifying this mailbox. + """ + cmd = b'EXAMINE' + args = _prepareMailboxName(mailbox) + resp = (b'FLAGS', b'EXISTS', b'RECENT', b'UNSEEN', b'PERMANENTFLAGS', b'UIDVALIDITY') + d = self.sendCommand(Command(cmd, args, wantResponse=resp)) + d.addCallback(self.__cbSelect, 0) + return d + + + def _intOrRaise(self, value, phrase): + """ + Parse C{value} as an integer and return the result or raise + L{IllegalServerResponse} with C{phrase} as an argument if C{value} + cannot be parsed as an integer. + """ + try: + return int(value) + except ValueError: + raise IllegalServerResponse(phrase) + + + def __cbSelect(self, result, rw): + """ + Handle lines received in response to a SELECT or EXAMINE command. + + See RFC 3501, section 6.3.1. + """ + (lines, tagline) = result + # In the absence of specification, we are free to assume: + # READ-WRITE access + datum = {'READ-WRITE': rw} + lines.append(parseNestedParens(tagline)) + for split in lines: + if len(split) > 0 and split[0].upper() == b'OK': + # Handle all the kinds of OK response. + content = split[1] + if isinstance(content, list): + key = content[0] + else: + # not multi-valued, like OK LOGIN + key = content + key = key.upper() + if key == b'READ-ONLY': + datum['READ-WRITE'] = False + elif key == b'READ-WRITE': + datum['READ-WRITE'] = True + elif key == b'UIDVALIDITY': + datum['UIDVALIDITY'] = self._intOrRaise(content[1], split) + elif key == b'UNSEEN': + datum['UNSEEN'] = self._intOrRaise(content[1], split) + elif key == b'UIDNEXT': + datum['UIDNEXT'] = self._intOrRaise(content[1], split) + elif key == b'PERMANENTFLAGS': + datum['PERMANENTFLAGS'] = tuple( + nativeString(flag) for flag in content[1]) + else: + log.err('Unhandled SELECT response (2): %s' % (split,)) + elif len(split) == 2: + # Handle FLAGS, EXISTS, and RECENT + if split[0].upper() == b'FLAGS': + datum['FLAGS'] = tuple( + nativeString(flag) for flag in split[1]) + elif isinstance(split[1], bytes): + # Must make sure things are strings before treating them as + # strings since some other forms of response have nesting in + # places which results in lists instead. + if split[1].upper() == b'EXISTS': + datum['EXISTS'] = self._intOrRaise(split[0], split) + elif split[1].upper() == b'RECENT': + datum['RECENT'] = self._intOrRaise(split[0], split) + else: + log.err('Unhandled SELECT response (0): %s' % (split,)) + else: + log.err('Unhandled SELECT response (1): %s' % (split,)) + else: + log.err('Unhandled SELECT response (4): %s' % (split,)) + return datum + + + def create(self, name): + """ + Create a new mailbox on the server + + This command is allowed in the Authenticated and Selected states. + + @type name: L{str} + @param name: The name of the mailbox to create. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked if the mailbox creation + is successful and whose errback is invoked otherwise. + """ + return self.sendCommand(Command(b'CREATE', _prepareMailboxName(name))) + + + def delete(self, name): + """ + Delete a mailbox + + This command is allowed in the Authenticated and Selected states. + + @type name: L{str} + @param name: The name of the mailbox to delete. + + @rtype: L{Deferred} + @return: A deferred whose calblack is invoked if the mailbox is + deleted successfully and whose errback is invoked otherwise. + """ + return self.sendCommand(Command(b'DELETE', _prepareMailboxName(name))) + + + def rename(self, oldname, newname): + """ + Rename a mailbox + + This command is allowed in the Authenticated and Selected states. + + @type oldname: L{str} + @param oldname: The current name of the mailbox to rename. + + @type newname: L{str} + @param newname: The new name to give the mailbox. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked if the rename is + successful and whose errback is invoked otherwise. + """ + oldname = _prepareMailboxName(oldname) + newname = _prepareMailboxName(newname) + return self.sendCommand(Command(b'RENAME', b' '.join((oldname, newname)))) + + + def subscribe(self, name): + """ + Add a mailbox to the subscription list + + This command is allowed in the Authenticated and Selected states. + + @type name: L{str} + @param name: The mailbox to mark as 'active' or 'subscribed' + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked if the subscription + is successful and whose errback is invoked otherwise. + """ + return self.sendCommand(Command(b'SUBSCRIBE', _prepareMailboxName(name))) + + + def unsubscribe(self, name): + """ + Remove a mailbox from the subscription list + + This command is allowed in the Authenticated and Selected states. + + @type name: L{str} + @param name: The mailbox to unsubscribe + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked if the unsubscription + is successful and whose errback is invoked otherwise. + """ + return self.sendCommand(Command(b'UNSUBSCRIBE', _prepareMailboxName(name))) + + + def list(self, reference, wildcard): + """ + List a subset of the available mailboxes + + This command is allowed in the Authenticated and Selected + states. + + @type reference: L{str} + @param reference: The context in which to interpret + C{wildcard} + + @type wildcard: L{str} + @param wildcard: The pattern of mailbox names to match, + optionally including either or both of the '*' and '%' + wildcards. '*' will match zero or more characters and + cross hierarchical boundaries. '%' will also match zero + or more characters, but is limited to a single + hierarchical level. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a list of + L{tuple}s, the first element of which is a L{tuple} of + mailbox flags, the second element of which is the + hierarchy delimiter for this mailbox, and the third of + which is the mailbox name; if the command is unsuccessful, + the deferred's errback is invoked instead. B{NB}: the + delimiter and the mailbox name are L{str}s. + """ + cmd = b'LIST' + args = ('"%s" "%s"' % (reference, wildcard)).encode("imap4-utf-7") + resp = (b'LIST',) + d = self.sendCommand(Command(cmd, args, wantResponse=resp)) + d.addCallback(self.__cbList, b'LIST') + return d + + + def lsub(self, reference, wildcard): + """ + List a subset of the subscribed available mailboxes + + This command is allowed in the Authenticated and Selected states. + + The parameters and returned object are the same as for the L{list} + method, with one slight difference: Only mailboxes which have been + subscribed can be included in the resulting list. + """ + cmd = b'LSUB' + + encodedReference = reference.encode('ascii') + encodedWildcard = wildcard.encode('imap4-utf-7') + args = b"".join([ + b'"', encodedReference, b'"' + b' "', encodedWildcard, b'"', + ]) + resp = (b'LSUB',) + d = self.sendCommand(Command(cmd, args, wantResponse=resp)) + d.addCallback(self.__cbList, b'LSUB') + return d + + + def __cbList(self, result, command): + (lines, last) = result + results = [] + + for parts in lines: + if len(parts) == 4 and parts[0] == command: + # flags + parts[1] = tuple(nativeString(flag) for flag in parts[1]) + + # The mailbox should be a native string. + # On Python 2, this maintains the API's contract. + # + # On Python 3, users specify mailboxes with native + # strings, so they should receive mailboxes as native + # strings. Both cases are possible because of the + # imap4-utf-7 encoding. + # + # Mailbox names contain the hierarchical delimiter, so + # it too should be a native string. + if _PY3: + # delimiter + parts[2] = parts[2].decode('imap4-utf-7') + # mailbox + parts[3] = parts[3].decode('imap4-utf-7') + + results.append(tuple(parts[1:])) + return results + + + _statusNames = { + name: name.encode('ascii') for name in ( + 'MESSAGES', + 'RECENT', + 'UIDNEXT', + 'UIDVALIDITY', + 'UNSEEN', + ) + } + + def status(self, mailbox, *names): + """ + Retrieve the status of the given mailbox + + This command is allowed in the Authenticated and Selected states. + + @type mailbox: L{str} + @param mailbox: The name of the mailbox to query + + @type *names: L{bytes} + @param *names: The status names to query. These may be any number of: + C{'MESSAGES'}, C{'RECENT'}, C{'UIDNEXT'}, C{'UIDVALIDITY'}, and + C{'UNSEEN'}. + + @rtype: L{Deferred} + @return: A deferred which fires with the status information if the + command is successful and whose errback is invoked otherwise. The + status information is in the form of a C{dict}. Each element of + C{names} is a key in the dictionary. The value for each key is the + corresponding response from the server. + """ + cmd = b'STATUS' + + preparedMailbox = _prepareMailboxName(mailbox) + try: + names = b' '.join(self._statusNames[name] for name in names) + except KeyError: + raise ValueError("Unknown names: {!r}".format( + set(names) - set(self._statusNames) + )) + + args = b''.join([preparedMailbox, + b" (", names, b")"]) + resp = (b'STATUS',) + d = self.sendCommand(Command(cmd, args, wantResponse=resp)) + d.addCallback(self.__cbStatus) + return d + + + def __cbStatus(self, result): + (lines, last) = result + status = {} + for parts in lines: + if parts[0] == b'STATUS': + items = parts[2] + items = [items[i:i+2] for i in range(0, len(items), 2)] + for k, v in items: + try: + status[nativeString(k)] = v + except UnicodeDecodeError: + raise IllegalServerResponse(repr(items)) + for k in status.keys(): + t = self.STATUS_TRANSFORMATIONS.get(k) + if t: + try: + status[k] = t(status[k]) + except Exception as e: + raise IllegalServerResponse('(' + k + ' '+ status[k] + '): ' + str(e)) + return status + + + def append(self, mailbox, message, flags = (), date = None): + """ + Add the given message to the given mailbox. + + This command is allowed in the Authenticated and Selected states. + + @type mailbox: L{str} + @param mailbox: The mailbox to which to add this message. + + @type message: Any file-like object opened in B{binary mode}. + @param message: The message to add, in RFC822 format. Newlines + in this file should be \\r\\n-style. + + @type flags: Any iterable of L{str} + @param flags: The flags to associated with this message. + + @type date: L{str} + @param date: The date to associate with this message. This should + be of the format DD-MM-YYYY HH:MM:SS +/-HHMM. For example, in + Eastern Standard Time, on July 1st 2004 at half past 1 PM, + \"01-07-2004 13:30:00 -0500\". + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked when this command + succeeds or whose errback is invoked if it fails. + """ + message.seek(0, 2) + L = message.tell() + message.seek(0, 0) + if date: + date = networkString(' "%s"' % nativeString(date)) + else: + date = b'' + + encodedFlags = [networkString(flag) for flag in flags] + + cmd = b''.join([ + _prepareMailboxName(mailbox), + b" (", b" ".join(encodedFlags), b")", + date, + b" {", intToBytes(L), b"}", + ]) + + d = self.sendCommand(Command(b'APPEND', cmd, (), self.__cbContinueAppend, message)) + return d + + + def __cbContinueAppend(self, lines, message): + s = basic.FileSender() + return s.beginFileTransfer(message, self.transport, None + ).addCallback(self.__cbFinishAppend) + + + def __cbFinishAppend(self, foo): + self.sendLine(b'') + + + def check(self): + """ + Tell the server to perform a checkpoint + + This command is allowed in the Selected state. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked when this command + succeeds or whose errback is invoked if it fails. + """ + return self.sendCommand(Command(b'CHECK')) + + + def close(self): + """ + Return the connection to the Authenticated state. + + This command is allowed in the Selected state. + + Issuing this command will also remove all messages flagged \\Deleted + from the selected mailbox if it is opened in read-write mode, + otherwise it indicates success by no messages are removed. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked when the command + completes successfully or whose errback is invoked if it fails. + """ + return self.sendCommand(Command(b'CLOSE')) + + + def expunge(self): + """ + Return the connection to the Authenticate state. + + This command is allowed in the Selected state. + + Issuing this command will perform the same actions as issuing the + close command, but will also generate an 'expunge' response for + every message deleted. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a list of the + 'expunge' responses when this command is successful or whose errback + is invoked otherwise. + """ + cmd = b'EXPUNGE' + resp = (b'EXPUNGE',) + d = self.sendCommand(Command(cmd, wantResponse=resp)) + d.addCallback(self.__cbExpunge) + return d + + + def __cbExpunge(self, result): + (lines, last) = result + ids = [] + for parts in lines: + if len(parts) == 2 and parts[1] == b'EXPUNGE': + ids.append(self._intOrRaise(parts[0], parts)) + return ids + + + def search(self, *queries, **kwarg): + """ + Search messages in the currently selected mailbox + + This command is allowed in the Selected state. + + Any non-zero number of queries are accepted by this method, as returned + by the C{Query}, C{Or}, and C{Not} functions. + + @param uid: if true, the server is asked to return message UIDs instead + of message sequence numbers. (This is a keyword-only argument.) + @type uid: L{bool} + + @rtype: L{Deferred} + @return: A deferred whose callback will be invoked with a list of all + the message sequence numbers return by the search, or whose errback + will be invoked if there is an error. + """ + # Queries should be encoded as ASCII unless a charset + # identifier is provided. See #9201. + if _PY3: + queries = [query.encode('charmap') for query in queries] + + if kwarg.get('uid'): + cmd = b'UID SEARCH' + else: + cmd = b'SEARCH' + args = b' '.join(queries) + d = self.sendCommand(Command(cmd, args, wantResponse=(cmd,))) + d.addCallback(self.__cbSearch) + return d + + + def __cbSearch(self, result): + (lines, end) = result + ids = [] + for parts in lines: + if len(parts) > 0 and parts[0] == b'SEARCH': + ids.extend([self._intOrRaise(p, parts) for p in parts[1:]]) + return ids + + + def fetchUID(self, messages, uid=0): + """ + Retrieve the unique identifier for one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message sequence numbers to unique message identifiers, or whose + errback is invoked if there is an error. + """ + return self._fetch(messages, useUID=uid, uid=1) + + + def fetchFlags(self, messages, uid=0): + """ + Retrieve the flags for one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: The messages for which to retrieve flags. + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to lists of flags, or whose errback is invoked if + there is an error. + """ + return self._fetch(messages, useUID=uid, flags=1) + + + def fetchInternalDate(self, messages, uid=0): + """ + Retrieve the internal date associated with one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: The messages for which to retrieve the internal date. + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to date strings, or whose errback is invoked + if there is an error. Date strings take the format of + \"day-month-year time timezone\". + """ + return self._fetch(messages, useUID=uid, internaldate=1) + + + def fetchEnvelope(self, messages, uid=0): + """ + Retrieve the envelope data for one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: The messages for which to retrieve envelope + data. + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of + message numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict + mapping message numbers to envelope data, or whose errback + is invoked if there is an error. Envelope data consists + of a sequence of the date, subject, from, sender, + reply-to, to, cc, bcc, in-reply-to, and message-id header + fields. The date, subject, in-reply-to, and message-id + fields are L{str}, while the from, sender, reply-to, to, + cc, and bcc fields contain address data as L{str}s. + Address data consists of a sequence of name, source route, + mailbox name, and hostname. Fields which are not present + for a particular address may be L{None}. + """ + return self._fetch(messages, useUID=uid, envelope=1) + + + def fetchBodyStructure(self, messages, uid=0): + """ + Retrieve the structure of the body of one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: The messages for which to retrieve body structure + data. + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to body structure data, or whose errback is invoked + if there is an error. Body structure data describes the MIME-IMB + format of a message and consists of a sequence of mime type, mime + subtype, parameters, content id, description, encoding, and size. + The fields following the size field are variable: if the mime + type/subtype is message/rfc822, the contained message's envelope + information, body structure data, and number of lines of text; if + the mime type is text, the number of lines of text. Extension fields + may also be included; if present, they are: the MD5 hash of the body, + body disposition, body language. + """ + return self._fetch(messages, useUID=uid, bodystructure=1) + + + def fetchSimplifiedBody(self, messages, uid=0): + """ + Retrieve the simplified body structure of one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: C{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to body data, or whose errback is invoked + if there is an error. The simplified body structure is the same + as the body structure, except that extension fields will never be + present. + """ + return self._fetch(messages, useUID=uid, body=1) + + + def fetchMessage(self, messages, uid=0): + """ + Retrieve one or more entire messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: C{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + + @return: A L{Deferred} which will fire with a C{dict} mapping message + sequence numbers to C{dict}s giving message data for the + corresponding message. If C{uid} is true, the inner dictionaries + have a C{'UID'} key mapped to a L{str} giving the UID for the + message. The text of the message is a L{str} associated with the + C{'RFC822'} key in each dictionary. + """ + return self._fetch(messages, useUID=uid, rfc822=1) + + + def fetchHeaders(self, messages, uid=0): + """ + Retrieve headers of one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to dicts of message headers, or whose errback is + invoked if there is an error. + """ + return self._fetch(messages, useUID=uid, rfc822header=1) + + + def fetchBody(self, messages, uid=0): + """ + Retrieve body text of one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to file-like objects containing body text, or whose + errback is invoked if there is an error. + """ + return self._fetch(messages, useUID=uid, rfc822text=1) + + + def fetchSize(self, messages, uid=0): + """ + Retrieve the size, in octets, of one or more messages + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to sizes, or whose errback is invoked if there is + an error. + """ + return self._fetch(messages, useUID=uid, rfc822size=1) + + + def fetchFull(self, messages, uid=0): + """ + Retrieve several different fields of one or more messages + + This command is allowed in the Selected state. This is equivalent + to issuing all of the C{fetchFlags}, C{fetchInternalDate}, + C{fetchSize}, C{fetchEnvelope}, and C{fetchSimplifiedBody} + functions. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to dict of the retrieved data values, or whose + errback is invoked if there is an error. They dictionary keys + are "flags", "date", "size", "envelope", and "body". + """ + return self._fetch( + messages, useUID=uid, flags=1, internaldate=1, + rfc822size=1, envelope=1, body=1) + + + def fetchAll(self, messages, uid=0): + """ + Retrieve several different fields of one or more messages + + This command is allowed in the Selected state. This is equivalent + to issuing all of the C{fetchFlags}, C{fetchInternalDate}, + C{fetchSize}, and C{fetchEnvelope} functions. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to dict of the retrieved data values, or whose + errback is invoked if there is an error. They dictionary keys + are "flags", "date", "size", and "envelope". + """ + return self._fetch( + messages, useUID=uid, flags=1, internaldate=1, + rfc822size=1, envelope=1) + + + def fetchFast(self, messages, uid=0): + """ + Retrieve several different fields of one or more messages + + This command is allowed in the Selected state. This is equivalent + to issuing all of the C{fetchFlags}, C{fetchInternalDate}, and + C{fetchSize} functions. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a dict mapping + message numbers to dict of the retrieved data values, or whose + errback is invoked if there is an error. They dictionary keys are + "flags", "date", and "size". + """ + return self._fetch( + messages, useUID=uid, flags=1, internaldate=1, rfc822size=1) + + + def _parseFetchPairs(self, fetchResponseList): + """ + Given the result of parsing a single I{FETCH} response, construct a + L{dict} mapping response keys to response values. + + @param fetchResponseList: The result of parsing a I{FETCH} response + with L{parseNestedParens} and extracting just the response data + (that is, just the part that comes after C{"FETCH"}). The form + of this input (and therefore the output of this method) is very + disagreeable. A valuable improvement would be to enumerate the + possible keys (representing them as structured objects of some + sort) rather than using strings and tuples of tuples of strings + and so forth. This would allow the keys to be documented more + easily and would allow for a much simpler application-facing API + (one not based on looking up somewhat hard to predict keys in a + dict). Since C{fetchResponseList} notionally represents a + flattened sequence of pairs (identifying keys followed by their + associated values), collapsing such complex elements of this + list as C{["BODY", ["HEADER.FIELDS", ["SUBJECT"]]]} into a + single object would also greatly simplify the implementation of + this method. + + @return: A C{dict} of the response data represented by C{pairs}. Keys + in this dictionary are things like C{"RFC822.TEXT"}, C{"FLAGS"}, or + C{("BODY", ("HEADER.FIELDS", ("SUBJECT",)))}. Values are entirely + dependent on the key with which they are associated, but retain the + same structured as produced by L{parseNestedParens}. + """ + + # TODO: RFC 3501 Section 7.4.2, "FETCH Response", says for + # BODY responses that "8-bit textual data is permitted if a + # charset identifier is part of the body parameter + # parenthesized list". Every other component is 7-bit. This + # should parse out the charset identifier and use it to decode + # 8-bit bodies. Until then, on Python 2 it should continue to + # return native (byte) strings, while on Python 3 it should + # decode bytes to native strings via charmap, ensuring data + # fidelity at the cost of mojibake. + if _PY3: + def nativeStringResponse(thing): + if isinstance(thing, bytes): + return thing.decode('charmap') + elif isinstance(thing, list): + return [nativeStringResponse(subthing) + for subthing in thing] + else: + def nativeStringResponse(thing): + return thing + + values = {} + unstructured = [] + + responseParts = iter(fetchResponseList) + while True: + try: + key = next(responseParts) + except StopIteration: + break + + try: + value = next(responseParts) + except StopIteration: + raise IllegalServerResponse( + b"Not enough arguments", fetchResponseList) + + # The parsed forms of responses like: + # + # BODY[] VALUE + # BODY[TEXT] VALUE + # BODY[HEADER.FIELDS (SUBJECT)] VALUE + # BODY[HEADER.FIELDS (SUBJECT)] VALUE + # + # are: + # + # ["BODY", [], VALUE] + # ["BODY", ["TEXT"], VALUE] + # ["BODY", ["HEADER.FIELDS", ["SUBJECT"]], VALUE] + # ["BODY", ["HEADER.FIELDS", ["SUBJECT"]], "", VALUE] + # + # Additionally, BODY responses for multipart messages are + # represented as: + # + # ["BODY", VALUE] + # + # with list as the type of VALUE and the type of VALUE[0]. + # + # See #6281 for ideas on how this might be improved. + + if key not in (b"BODY", b"BODY.PEEK"): + # Only BODY (and by extension, BODY.PEEK) responses can have + # body sections. + hasSection = False + elif not isinstance(value, list): + # A BODY section is always represented as a list. Any non-list + # is not a BODY section. + hasSection = False + elif len(value) > 2: + # The list representing a BODY section has at most two elements. + hasSection = False + elif value and isinstance(value[0], list): + # A list containing a list represents the body structure of a + # multipart message, instead. + hasSection = False + else: + # Otherwise it must have a BODY section to examine. + hasSection = True + + # If it has a BODY section, grab some extra elements and shuffle + # around the shape of the key a little bit. + + key = nativeString(key) + unstructured.append(key) + + if hasSection: + if len(value) < 2: + value = [nativeString(v) for v in value] + unstructured.append(value) + + key = (key, tuple(value)) + else: + valueHead = nativeString(value[0]) + valueTail = [nativeString(v) for v in value[1]] + unstructured.append([valueHead, valueTail]) + + key = (key, (valueHead, tuple(valueTail))) + try: + value = next(responseParts) + except StopIteration: + raise IllegalServerResponse( + b"Not enough arguments", fetchResponseList) + + # Handle partial ranges + if value.startswith(b'<') and value.endswith(b'>'): + try: + int(value[1:-1]) + except ValueError: + # This isn't really a range, it's some content. + pass + else: + value = nativeString(value) + unstructured.append(value) + key = key + (value,) + try: + value = next(responseParts) + except StopIteration: + raise IllegalServerResponse( + b"Not enough arguments", fetchResponseList) + + value = nativeStringResponse(value) + unstructured.append(value) + values[key] = value + + return values, unstructured + + + def _cbFetch(self, result, requestedParts, structured): + (lines, last) = result + info = {} + for parts in lines: + if len(parts) == 3 and parts[1] == b'FETCH': + id = self._intOrRaise(parts[0], parts) + if id not in info: + info[id] = [parts[2]] + else: + info[id][0].extend(parts[2]) + + results = {} + decodedInfo = {} + for (messageId, values) in info.items(): + structuredMap, unstructuredList = self._parseFetchPairs(values[0]) + decodedInfo.setdefault(messageId, [[]])[0].extend(unstructuredList) + results.setdefault(messageId, {}).update(structuredMap) + info = decodedInfo + + flagChanges = {} + for messageId in list(results.keys()): + values = results[messageId] + for part in list(values.keys()): + if part not in requestedParts and part == 'FLAGS': + flagChanges[messageId] = values['FLAGS'] + # Find flags in the result and get rid of them. + for i in range(len(info[messageId][0])): + if info[messageId][0][i] == 'FLAGS': + del info[messageId][0][i:i+2] + break + del values['FLAGS'] + if not values: + del results[messageId] + + if flagChanges: + self.flagsChanged(flagChanges) + + if structured: + return results + else: + return info + + + def fetchSpecific(self, messages, uid=0, headerType=None, + headerNumber=None, headerArgs=None, peek=None, + offset=None, length=None): + """ + Retrieve a specific section of one or more messages + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @type headerType: L{str} + @param headerType: If specified, must be one of HEADER, HEADER.FIELDS, + HEADER.FIELDS.NOT, MIME, or TEXT, and will determine which part of + the message is retrieved. For HEADER.FIELDS and HEADER.FIELDS.NOT, + C{headerArgs} must be a sequence of header names. For MIME, + C{headerNumber} must be specified. + + @type headerNumber: L{int} or L{int} sequence + @param headerNumber: The nested rfc822 index specifying the entity to + retrieve. For example, C{1} retrieves the first entity of the + message, and C{(2, 1, 3}) retrieves the 3rd entity inside the first + entity inside the second entity of the message. + + @type headerArgs: A sequence of L{str} + @param headerArgs: If C{headerType} is HEADER.FIELDS, these are the + headers to retrieve. If it is HEADER.FIELDS.NOT, these are the + headers to exclude from retrieval. + + @type peek: C{bool} + @param peek: If true, cause the server to not set the \\Seen flag on + this message as a result of this command. + + @type offset: L{int} + @param offset: The number of octets at the beginning of the result to + skip. + + @type length: L{int} + @param length: The number of octets to retrieve. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a mapping of message + numbers to retrieved data, or whose errback is invoked if there is + an error. + """ + fmt = '%s BODY%s[%s%s%s]%s' + if headerNumber is None: + number = '' + elif isinstance(headerNumber, int): + number = str(headerNumber) + else: + number = '.'.join(map(str, headerNumber)) + if headerType is None: + header = '' + elif number: + header = '.' + headerType + else: + header = headerType + if header and headerType in ('HEADER.FIELDS', 'HEADER.FIELDS.NOT'): + if headerArgs is not None: + payload = ' (%s)' % ' '.join(headerArgs) + else: + payload = ' ()' + else: + payload = '' + if offset is None: + extra = '' + else: + extra = '<%d.%d>' % (offset, length) + fetch = uid and b'UID FETCH' or b'FETCH' + cmd = fmt % (messages, peek and '.PEEK' or '', number, header, payload, extra) + + # APPEND components should be encoded as ASCII unless a + # charset identifier is provided. See #9201. + if _PY3: + cmd = cmd.encode('charmap') + + d = self.sendCommand(Command(fetch, cmd, wantResponse=(b'FETCH',))) + d.addCallback(self._cbFetch, (), False) + return d + + + def _fetch(self, messages, useUID=0, **terms): + messages = str(messages).encode('ascii') + fetch = useUID and b'UID FETCH' or b'FETCH' + + if 'rfc822text' in terms: + del terms['rfc822text'] + terms['rfc822.text'] = True + if 'rfc822size' in terms: + del terms['rfc822size'] + terms['rfc822.size'] = True + if 'rfc822header' in terms: + del terms['rfc822header'] + terms['rfc822.header'] = True + + # The terms in 6.4.5 are all ASCII congruent, so wing it. + # Note that this isn't a public API, so terms in responses + # should not be decoded to native strings. + encodedTerms = [networkString(s) for s in terms] + cmd = messages + b' (' + b' '.join( + [s.upper() for s in encodedTerms] + ) + b')' + + d = self.sendCommand(Command(fetch, cmd, wantResponse=(b'FETCH',))) + d.addCallback(self._cbFetch, [t.upper() for t in terms.keys()], True) + return d + + + def setFlags(self, messages, flags, silent=1, uid=0): + """ + Set the flags for one or more messages. + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type flags: Any iterable of L{str} + @param flags: The flags to set + + @type silent: L{bool} + @param silent: If true, cause the server to suppress its verbose + response. + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a list of the + server's responses (C{[]} if C{silent} is true) or whose + errback is invoked if there is an error. + """ + return self._store(messages, b'FLAGS', silent, flags, uid) + + + def addFlags(self, messages, flags, silent=1, uid=0): + """ + Add to the set flags for one or more messages. + + This command is allowed in the Selected state. + + @type messages: C{MessageSet} or L{str} + @param messages: A message sequence set + + @type flags: Any iterable of L{str} + @param flags: The flags to set + + @type silent: C{bool} + @param silent: If true, cause the server to suppress its verbose + response. + + @type uid: C{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a list of the + server's responses (C{[]} if C{silent} is true) or whose + errback is invoked if there is an error. + """ + return self._store(messages, b'+FLAGS', silent, flags, uid) + + + def removeFlags(self, messages, flags, silent=1, uid=0): + """ + Remove from the set flags for one or more messages. + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type flags: Any iterable of L{str} + @param flags: The flags to set + + @type silent: L{bool} + @param silent: If true, cause the server to suppress its verbose + response. + + @type uid: L{bool} + @param uid: Indicates whether the message sequence set is of message + numbers or of unique message IDs. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a list of the + server's responses (C{[]} if C{silent} is true) or whose + errback is invoked if there is an error. + """ + return self._store(messages, b'-FLAGS', silent, flags, uid) + + + def _store(self, messages, cmd, silent, flags, uid): + messages = str(messages).encode('ascii') + encodedFlags = [networkString(flag) for flag in flags] + if silent: + cmd = cmd + b'.SILENT' + store = uid and b'UID STORE' or b'STORE' + args = b' '.join((messages, cmd, b'('+ b' '.join(encodedFlags) + b')')) + d = self.sendCommand(Command(store, args, wantResponse=(b'FETCH',))) + expected = () + if not silent: + expected = ('FLAGS',) + d.addCallback(self._cbFetch, expected, True) + return d + + + def copy(self, messages, mailbox, uid): + """ + Copy the specified messages to the specified mailbox. + + This command is allowed in the Selected state. + + @type messages: L{MessageSet} or L{str} + @param messages: A message sequence set + + @type mailbox: L{str} + @param mailbox: The mailbox to which to copy the messages + + @type uid: C{bool} + @param uid: If true, the C{messages} refers to message UIDs, rather + than message sequence numbers. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with a true value + when the copy is successful, or whose errback is invoked if there + is an error. + """ + messages = str(messages).encode('ascii') + if uid: + cmd = b'UID COPY' + else: + cmd = b'COPY' + args = b' '.join([messages, _prepareMailboxName(mailbox)]) + return self.sendCommand(Command(cmd, args)) + + # + # IMailboxListener methods + # + def modeChanged(self, writeable): + """Override me""" + + def flagsChanged(self, newFlags): + """Override me""" + + def newMessages(self, exists, recent): + """Override me""" + + + +def parseIdList(s, lastMessageId=None): + """ + Parse a message set search key into a C{MessageSet}. + + @type s: L{bytes} + @param s: A string description of an id list, for example "1:3, 4:*" + + @type lastMessageId: L{int} + @param lastMessageId: The last message sequence id or UID, depending on + whether we are parsing the list in UID or sequence id context. The + caller should pass in the correct value. + + @rtype: C{MessageSet} + @return: A C{MessageSet} that contains the ids defined in the list + """ + res = MessageSet() + parts = s.split(b',') + for p in parts: + if b':' in p: + low, high = p.split(b':', 1) + try: + if low == b'*': + low = None + else: + low = int(low) + if high == b'*': + high = None + else: + high = int(high) + if low is high is None: + # *:* does not make sense + raise IllegalIdentifierError(p) + # non-positive values are illegal according to RFC 3501 + if ((low is not None and low <= 0) or + (high is not None and high <= 0)): + raise IllegalIdentifierError(p) + # star means "highest value of an id in the mailbox" + high = high or lastMessageId + low = low or lastMessageId + + res.add(low, high) + except ValueError: + raise IllegalIdentifierError(p) + else: + try: + if p == b'*': + p = None + else: + p = int(p) + if p is not None and p <= 0: + raise IllegalIdentifierError(p) + except ValueError: + raise IllegalIdentifierError(p) + else: + res.extend(p or lastMessageId) + return res + + + +_SIMPLE_BOOL = ( + 'ALL', 'ANSWERED', 'DELETED', 'DRAFT', 'FLAGGED', 'NEW', 'OLD', + 'RECENT', 'SEEN', 'UNANSWERED', 'UNDELETED', 'UNDRAFT', 'UNFLAGGED', + 'UNSEEN' +) + +_NO_QUOTES = ( + 'LARGER', 'SMALLER', 'UID' +) + +_sorted = sorted + +def Query(sorted=0, **kwarg): + """ + Create a query string + + Among the accepted keywords are:: + + all : If set to a true value, search all messages in the + current mailbox + + answered : If set to a true value, search messages flagged with + \\Answered + + bcc : A substring to search the BCC header field for + + before : Search messages with an internal date before this + value. The given date should be a string in the format + of 'DD-Mon-YYYY'. For example, '03-Mar-2003'. + + body : A substring to search the body of the messages for + + cc : A substring to search the CC header field for + + deleted : If set to a true value, search messages flagged with + \\Deleted + + draft : If set to a true value, search messages flagged with + \\Draft + + flagged : If set to a true value, search messages flagged with + \\Flagged + + from : A substring to search the From header field for + + header : A two-tuple of a header name and substring to search + for in that header + + keyword : Search for messages with the given keyword set + + larger : Search for messages larger than this number of octets + + messages : Search only the given message sequence set. + + new : If set to a true value, search messages flagged with + \\Recent but not \\Seen + + old : If set to a true value, search messages not flagged with + \\Recent + + on : Search messages with an internal date which is on this + date. The given date should be a string in the format + of 'DD-Mon-YYYY'. For example, '03-Mar-2003'. + + recent : If set to a true value, search for messages flagged with + \\Recent + + seen : If set to a true value, search for messages flagged with + \\Seen + + sentbefore : Search for messages with an RFC822 'Date' header before + this date. The given date should be a string in the format + of 'DD-Mon-YYYY'. For example, '03-Mar-2003'. + + senton : Search for messages with an RFC822 'Date' header which is + on this date The given date should be a string in the format + of 'DD-Mon-YYYY'. For example, '03-Mar-2003'. + + sentsince : Search for messages with an RFC822 'Date' header which is + after this date. The given date should be a string in the format + of 'DD-Mon-YYYY'. For example, '03-Mar-2003'. + + since : Search for messages with an internal date that is after + this date.. The given date should be a string in the format + of 'DD-Mon-YYYY'. For example, '03-Mar-2003'. + + smaller : Search for messages smaller than this number of octets + + subject : A substring to search the 'subject' header for + + text : A substring to search the entire message for + + to : A substring to search the 'to' header for + + uid : Search only the messages in the given message set + + unanswered : If set to a true value, search for messages not + flagged with \\Answered + + undeleted : If set to a true value, search for messages not + flagged with \\Deleted + + undraft : If set to a true value, search for messages not + flagged with \\Draft + + unflagged : If set to a true value, search for messages not + flagged with \\Flagged + + unkeyword : Search for messages without the given keyword set + + unseen : If set to a true value, search for messages not + flagged with \\Seen + + @type sorted: C{bool} + @param sorted: If true, the output will be sorted, alphabetically. + The standard does not require it, but it makes testing this function + easier. The default is zero, and this should be acceptable for any + application. + + @rtype: L{str} + @return: The formatted query string + """ + cmd = [] + keys = kwarg.keys() + if sorted: + keys = _sorted(keys) + for k in keys: + v = kwarg[k] + k = k.upper() + if k in _SIMPLE_BOOL and v: + cmd.append(k) + elif k == 'HEADER': + cmd.extend([k, str(v[0]), str(v[1])]) + elif k == 'KEYWORD' or k == 'UNKEYWORD': + # Discard anything that does not fit into an "atom". Perhaps turn + # the case where this actually removes bytes from the value into a + # warning and then an error, eventually. See #6277. + v = _nonAtomRE.sub("", v) + cmd.extend([k, v]) + elif k not in _NO_QUOTES: + if isinstance(v, MessageSet): + fmt = '"%s"' + elif isinstance(v, str): + fmt = '"%s"' + else: + fmt = '"%d"' + cmd.extend([k, fmt % (v,)]) + elif isinstance(v, int): + cmd.extend([k, '%d' % (v,)]) + else: + cmd.extend([k, '%s' % (v,)]) + if len(cmd) > 1: + return '(' + ' '.join(cmd) + ')' + else: + return ' '.join(cmd) + + + +def Or(*args): + """ + The disjunction of two or more queries + """ + if len(args) < 2: + raise IllegalQueryError(args) + elif len(args) == 2: + return '(OR %s %s)' % args + else: + return '(OR %s %s)' % (args[0], Or(*args[1:])) + +def Not(query): + """The negation of a query""" + return '(NOT %s)' % (query,) + + +def wildcardToRegexp(wildcard, delim=None): + wildcard = wildcard.replace('*', '(?:.*?)') + if delim is None: + wildcard = wildcard.replace('%', '(?:.*?)') + else: + wildcard = wildcard.replace('%', '(?:(?:[^%s])*?)' % re.escape(delim)) + return re.compile(wildcard, re.I) + + + +def splitQuoted(s): + """ + Split a string into whitespace delimited tokens + + Tokens that would otherwise be separated but are surrounded by \" + remain as a single token. Any token that is not quoted and is + equal to \"NIL\" is tokenized as L{None}. + + @type s: L{bytes} + @param s: The string to be split + + @rtype: L{list} of L{bytes} + @return: A list of the resulting tokens + + @raise MismatchedQuoting: Raised if an odd number of quotes are present + """ + s = s.strip() + result = [] + word = [] + inQuote = inWord = False + qu = _matchingString('"', s) + esc = _matchingString('\x5c', s) + empty = _matchingString('', s) + nil = _matchingString('NIL', s) + for i, c in enumerate(iterbytes(s)): + if c == qu: + if i and s[i-1:i] == esc: + word.pop() + word.append(qu) + elif not inQuote: + inQuote = True + else: + inQuote = False + result.append(empty.join(word)) + word = [] + elif ( + not inWord and not inQuote and + c not in (qu + (string.whitespace.encode("ascii"))) + ): + inWord = True + word.append(c) + elif inWord and not inQuote and c in string.whitespace.encode("ascii"): + w = empty.join(word) + if w == nil: + result.append(None) + else: + result.append(w) + word = [] + inWord = False + elif inWord or inQuote: + word.append(c) + + if inQuote: + raise MismatchedQuoting(s) + if inWord: + w = empty.join(word) + if w == nil: + result.append(None) + else: + result.append(w) + + return result + + + +def splitOn(sequence, predicate, transformers): + result = [] + mode = predicate(sequence[0]) + tmp = [sequence[0]] + for e in sequence[1:]: + p = predicate(e) + if p != mode: + result.extend(transformers[mode](tmp)) + tmp = [e] + mode = p + else: + tmp.append(e) + result.extend(transformers[mode](tmp)) + return result + + + +def collapseStrings(results): + """ + Turns a list of length-one strings and lists into a list of longer + strings and lists. For example, + + ['a', 'b', ['c', 'd']] is returned as ['ab', ['cd']] + + @type results: L{list} of L{bytes} and L{list} + @param results: The list to be collapsed + + @rtype: L{list} of L{bytes} and L{list} + @return: A new list which is the collapsed form of C{results} + """ + copy = [] + begun = None + + pred = lambda e: isinstance(e, tuple) + tran = { + 0: lambda e: splitQuoted(b''.join(e)), + 1: lambda e: [b''.join([i[0] for i in e])] + } + for i, c in enumerate(results): + if isinstance(c, list): + if begun is not None: + copy.extend(splitOn(results[begun:i], pred, tran)) + begun = None + copy.append(collapseStrings(c)) + elif begun is None: + begun = i + if begun is not None: + copy.extend(splitOn(results[begun:], pred, tran)) + return copy + + + +def parseNestedParens(s, handleLiteral = 1): + """ + Parse an s-exp-like string into a more useful data structure. + + @type s: L{bytes} + @param s: The s-exp-like string to parse + + @rtype: L{list} of L{bytes} and L{list} + @return: A list containing the tokens present in the input. + + @raise MismatchedNesting: Raised if the number or placement + of opening or closing parenthesis is invalid. + """ + s = s.strip() + inQuote = 0 + contentStack = [[]] + try: + i = 0 + L = len(s) + while i < L: + c = s[i:i+1] + if inQuote: + if c == b'\\': + contentStack[-1].append(s[i:i+2]) + i += 2 + continue + elif c == b'"': + inQuote = not inQuote + contentStack[-1].append(c) + i += 1 + else: + if c == b'"': + contentStack[-1].append(c) + inQuote = not inQuote + i += 1 + elif handleLiteral and c == b'{': + end = s.find(b'}', i) + if end == -1: + raise ValueError("Malformed literal") + literalSize = int(s[i+1:end]) + contentStack[-1].append((s[end+3:end+3+literalSize],)) + i = end + 3 + literalSize + elif c == b'(' or c == b'[': + contentStack.append([]) + i += 1 + elif c == b')' or c == b']': + contentStack[-2].append(contentStack.pop()) + i += 1 + else: + contentStack[-1].append(c) + i += 1 + except IndexError: + raise MismatchedNesting(s) + if len(contentStack) != 1: + raise MismatchedNesting(s) + return collapseStrings(contentStack[0]) + + + +def _quote(s): + qu = _matchingString('"', s) + esc = _matchingString('\x5c', s) + return qu + s.replace(esc, esc + esc).replace(qu, esc + qu) + qu + + + +def _literal(s): + return b'{' + intToBytes(len(s)) + b'}\r\n' + s + + + +class DontQuoteMe: + def __init__(self, value): + self.value = value + + + def __str__(self): + return str(self.value) + + + +_ATOM_SPECIALS = b'(){ %*"' +def _needsQuote(s): + if s == b'': + return 1 + for c in iterbytes(s): + if c < b'\x20' or c > b'\x7f': + return 1 + if c in _ATOM_SPECIALS: + return 1 + return 0 + + + +def _parseMbox(name): + if isinstance(name, unicode): + return name + try: + return name.decode('imap4-utf-7') + except: + log.err() + raise IllegalMailboxEncoding(name) + + + +def _prepareMailboxName(name): + if not isinstance(name, unicode): + name = name.decode("charmap") + name = name.encode('imap4-utf-7') + if _needsQuote(name): + return _quote(name) + return name + + + + +def _needsLiteral(s): + # change this to "return 1" to wig out stupid clients + cr = _matchingString("\n", s) + lf = _matchingString("\r", s) + return cr in s or lf in s or len(s) > 1000 + + + +def collapseNestedLists(items): + """ + Turn a nested list structure into an s-exp-like string. + + Strings in C{items} will be sent as literals if they contain CR or LF, + otherwise they will be quoted. References to None in C{items} will be + translated to the atom NIL. Objects with a 'read' attribute will have + it called on them with no arguments and the returned string will be + inserted into the output as a literal. Integers will be converted to + strings and inserted into the output unquoted. Instances of + C{DontQuoteMe} will be converted to strings and inserted into the output + unquoted. + + This function used to be much nicer, and only quote things that really + needed to be quoted (and C{DontQuoteMe} did not exist), however, many + broken IMAP4 clients were unable to deal with this level of sophistication, + forcing the current behavior to be adopted for practical reasons. + + @type items: Any iterable + + @rtype: L{str} + """ + pieces = [] + for i in items: + if isinstance(i, unicode): + # anything besides ASCII will have to wait for an RFC 5738 + # implementation. See + # https://twistedmatrix.com/trac/ticket/9258 + i = i.encode("ascii") + if i is None: + pieces.extend([b' ', b'NIL']) + elif isinstance(i, (int, long)): + pieces.extend([b' ', networkString(str(i))]) + elif isinstance(i, DontQuoteMe): + pieces.extend([b' ', i.value]) + elif isinstance(i, bytes): + # XXX warning + if _needsLiteral(i): + pieces.extend([b' ', b'{', intToBytes(len(i)), b'}', + IMAP4Server.delimiter, i]) + else: + pieces.extend([b' ', _quote(i)]) + elif hasattr(i, 'read'): + d = i.read() + pieces.extend([b' ', b'{', intToBytes(len(d)), b'}', + IMAP4Server.delimiter, d]) + else: + pieces.extend([b' ', b'(' + collapseNestedLists(i) + b')']) + return b''.join(pieces[1:]) + + + +@implementer(IAccount) +class MemoryAccountWithoutNamespaces(object): + mailboxes = None + subscriptions = None + top_id = 0 + + def __init__(self, name): + self.name = name + self.mailboxes = {} + self.subscriptions = [] + + + def allocateID(self): + id = self.top_id + self.top_id += 1 + return id + + + ## + ## IAccount + ## + def addMailbox(self, name, mbox = None): + name = _parseMbox(name.upper()) + if name in self.mailboxes: + raise MailboxCollision(name) + if mbox is None: + mbox = self._emptyMailbox(name, self.allocateID()) + self.mailboxes[name] = mbox + return 1 + + + def create(self, pathspec): + paths = [path for path in pathspec.split('/') if path] + for accum in range(1, len(paths)): + try: + self.addMailbox('/'.join(paths[:accum])) + except MailboxCollision: + pass + try: + self.addMailbox('/'.join(paths)) + except MailboxCollision: + if not pathspec.endswith('/'): + return False + return True + + + def _emptyMailbox(self, name, id): + raise NotImplementedError + + + def select(self, name, readwrite=1): + return self.mailboxes.get(_parseMbox(name.upper())) + + + def delete(self, name): + name = _parseMbox(name.upper()) + # See if this mailbox exists at all + mbox = self.mailboxes.get(name) + if not mbox: + raise MailboxException("No such mailbox") + # See if this box is flagged \Noselect + if r'\Noselect' in mbox.getFlags(): + # Check for hierarchically inferior mailboxes with this one + # as part of their root. + for others in self.mailboxes.keys(): + if others != name and others.startswith(name): + raise MailboxException("Hierarchically inferior mailboxes exist and \\Noselect is set") + mbox.destroy() + + # iff there are no hierarchically inferior names, we will + # delete it from our ken. + if len(self._inferiorNames(name)) > 1: + raise MailboxException( + 'Name "%s" has inferior hierarchical names' % (name,)) + del self.mailboxes[name] + + + def rename(self, oldname, newname): + oldname = _parseMbox(oldname.upper()) + newname = _parseMbox(newname.upper()) + if oldname not in self.mailboxes: + raise NoSuchMailbox(oldname) + + inferiors = self._inferiorNames(oldname) + inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + + for (old, new) in inferiors: + if new in self.mailboxes: + raise MailboxCollision(new) + + for (old, new) in inferiors: + self.mailboxes[new] = self.mailboxes[old] + del self.mailboxes[old] + + + def _inferiorNames(self, name): + inferiors = [] + for infname in self.mailboxes.keys(): + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + + def isSubscribed(self, name): + return _parseMbox(name.upper()) in self.subscriptions + + + def subscribe(self, name): + name = _parseMbox(name.upper()) + if name not in self.subscriptions: + self.subscriptions.append(name) + + + def unsubscribe(self, name): + name = _parseMbox(name.upper()) + if name not in self.subscriptions: + raise MailboxException("Not currently subscribed to %s" % (name,)) + self.subscriptions.remove(name) + + + def listMailboxes(self, ref, wildcard): + ref = self._inferiorNames(_parseMbox(ref.upper())) + wildcard = wildcardToRegexp(wildcard, '/') + return [(i, self.mailboxes[i]) for i in ref if wildcard.match(i)] + + +@implementer(INamespacePresenter) +class MemoryAccount(MemoryAccountWithoutNamespaces): + ## + ## INamespacePresenter + ## + def getPersonalNamespaces(self): + return [[b"", b"/"]] + + + def getSharedNamespaces(self): + return None + + + def getOtherNamespaces(self): + return None + + + +_statusRequestDict = { + 'MESSAGES': 'getMessageCount', + 'RECENT': 'getRecentCount', + 'UIDNEXT': 'getUIDNext', + 'UIDVALIDITY': 'getUIDValidity', + 'UNSEEN': 'getUnseenCount' +} + +def statusRequestHelper(mbox, names): + r = {} + for n in names: + r[n] = getattr(mbox, _statusRequestDict[n.upper()])() + return r + + + +def parseAddr(addr): + if addr is None: + return [(None, None, None),] + addr = email.utils.getaddresses([addr]) + return [[fn or None, None] + address.split('@') for fn, address in addr] + + + +def getEnvelope(msg): + headers = msg.getHeaders(True) + date = headers.get('date') + subject = headers.get('subject') + from_ = headers.get('from') + sender = headers.get('sender', from_) + reply_to = headers.get('reply-to', from_) + to = headers.get('to') + cc = headers.get('cc') + bcc = headers.get('bcc') + in_reply_to = headers.get('in-reply-to') + mid = headers.get('message-id') + return (date, subject, parseAddr(from_), parseAddr(sender), + reply_to and parseAddr(reply_to), to and parseAddr(to), + cc and parseAddr(cc), bcc and parseAddr(bcc), in_reply_to, mid) + + + +def getLineCount(msg): + # XXX - Super expensive, CACHE THIS VALUE FOR LATER RE-USE + # XXX - This must be the number of lines in the ENCODED version + lines = 0 + for _ in msg.getBodyFile(): + lines += 1 + return lines + + + +def unquote(s): + if s[0] == s[-1] == '"': + return s[1:-1] + return s + + + +def _getContentType(msg): + """ + Return a two-tuple of the main and subtype of the given message. + """ + attrs = None + mm = msg.getHeaders(False, 'content-type').get('content-type', '') + mm = ''.join(mm.splitlines()) + if mm: + mimetype = mm.split(';') + type = mimetype[0].split('/', 1) + if len(type) == 1: + major = type[0] + minor = None + else: + # length must be 2, because of split('/', 1) + major, minor = type + attrs = dict(x.strip().lower().split('=', 1) for x in mimetype[1:]) + else: + major = minor = None + return major, minor, attrs + + + +def _getMessageStructure(message): + """ + Construct an appropriate type of message structure object for the given + message object. + + @param message: A L{IMessagePart} provider + + @return: A L{_MessageStructure} instance of the most specific type available + for the given message, determined by inspecting the MIME type of the + message. + """ + main, subtype, attrs = _getContentType(message) + if main is not None: + main = main.lower() + if subtype is not None: + subtype = subtype.lower() + if main == 'multipart': + return _MultipartMessageStructure(message, subtype, attrs) + elif (main, subtype) == ('message', 'rfc822'): + return _RFC822MessageStructure(message, main, subtype, attrs) + elif main == 'text': + return _TextMessageStructure(message, main, subtype, attrs) + else: + return _SinglepartMessageStructure(message, main, subtype, attrs) + + + +class _MessageStructure(object): + """ + L{_MessageStructure} is a helper base class for message structure classes + representing the structure of particular kinds of messages, as defined by + their MIME type. + """ + def __init__(self, message, attrs): + """ + @param message: An L{IMessagePart} provider which this structure object + reports on. + + @param attrs: A C{dict} giving the parameters of the I{Content-Type} + header of the message. + """ + self.message = message + self.attrs = attrs + + + def _disposition(self, disp): + """ + Parse a I{Content-Disposition} header into a two-sequence of the + disposition and a flattened list of its parameters. + + @return: L{None} if there is no disposition header value, a L{list} with + two elements otherwise. + """ + if disp: + disp = disp.split('; ') + if len(disp) == 1: + disp = (disp[0].lower(), None) + elif len(disp) > 1: + # XXX Poorly tested parser + params = [x for param in disp[1:] for x in param.split('=', 1)] + disp = [disp[0].lower(), params] + return disp + else: + return None + + + def _unquotedAttrs(self): + """ + @return: The I{Content-Type} parameters, unquoted, as a flat list with + each Nth element giving a parameter name and N+1th element giving + the corresponding parameter value. + """ + if self.attrs: + unquoted = [(k, unquote(v)) for (k, v) in self.attrs.items()] + return [y for x in sorted(unquoted) for y in x] + return None + + + +class _SinglepartMessageStructure(_MessageStructure): + """ + L{_SinglepartMessageStructure} represents the message structure of a + non-I{multipart/*} message. + """ + _HEADERS = [ + 'content-id', 'content-description', + 'content-transfer-encoding'] + + def __init__(self, message, main, subtype, attrs): + """ + @param message: An L{IMessagePart} provider which this structure object + reports on. + + @param main: A L{str} giving the main MIME type of the message (for + example, C{"text"}). + + @param subtype: A L{str} giving the MIME subtype of the message (for + example, C{"plain"}). + + @param attrs: A C{dict} giving the parameters of the I{Content-Type} + header of the message. + """ + _MessageStructure.__init__(self, message, attrs) + self.main = main + self.subtype = subtype + self.attrs = attrs + + + def _basicFields(self): + """ + Return a list of the basic fields for a single-part message. + """ + headers = self.message.getHeaders(False, *self._HEADERS) + + # Number of octets total + size = self.message.getSize() + + major, minor = self.main, self.subtype + + # content-type parameter list + unquotedAttrs = self._unquotedAttrs() + + return [ + major, minor, unquotedAttrs, + headers.get('content-id'), + headers.get('content-description'), + headers.get('content-transfer-encoding'), + size, + ] + + + def encode(self, extended): + """ + Construct and return a list of the basic and extended fields for a + single-part message. The list suitable to be encoded into a BODY or + BODYSTRUCTURE response. + """ + result = self._basicFields() + if extended: + result.extend(self._extended()) + return result + + + def _extended(self): + """ + The extension data of a non-multipart body part are in the + following order: + + 1. body MD5 + + A string giving the body MD5 value as defined in [MD5]. + + 2. body disposition + + A parenthesized list with the same content and function as + the body disposition for a multipart body part. + + 3. body language + + A string or parenthesized list giving the body language + value as defined in [LANGUAGE-TAGS]. + + 4. body location + + A string list giving the body content URI as defined in + [LOCATION]. + + """ + result = [] + headers = self.message.getHeaders( + False, 'content-md5', 'content-disposition', + 'content-language', 'content-language') + + result.append(headers.get('content-md5')) + result.append(self._disposition(headers.get('content-disposition'))) + result.append(headers.get('content-language')) + result.append(headers.get('content-location')) + + return result + + + +class _TextMessageStructure(_SinglepartMessageStructure): + """ + L{_TextMessageStructure} represents the message structure of a I{text/*} + message. + """ + def encode(self, extended): + """ + A body type of type TEXT contains, immediately after the basic + fields, the size of the body in text lines. Note that this + size is the size in its content transfer encoding and not the + resulting size after any decoding. + """ + result = _SinglepartMessageStructure._basicFields(self) + result.append(getLineCount(self.message)) + if extended: + result.extend(self._extended()) + return result + + + +class _RFC822MessageStructure(_SinglepartMessageStructure): + """ + L{_RFC822MessageStructure} represents the message structure of a + I{message/rfc822} message. + """ + def encode(self, extended): + """ + A body type of type MESSAGE and subtype RFC822 contains, + immediately after the basic fields, the envelope structure, + body structure, and size in text lines of the encapsulated + message. + """ + result = _SinglepartMessageStructure.encode(self, extended) + contained = self.message.getSubPart(0) + result.append(getEnvelope(contained)) + result.append(getBodyStructure(contained, False)) + result.append(getLineCount(contained)) + return result + + + +class _MultipartMessageStructure(_MessageStructure): + """ + L{_MultipartMessageStructure} represents the message structure of a + I{multipart/*} message. + """ + def __init__(self, message, subtype, attrs): + """ + @param message: An L{IMessagePart} provider which this structure object + reports on. + + @param subtype: A L{str} giving the MIME subtype of the message (for + example, C{"plain"}). + + @param attrs: A C{dict} giving the parameters of the I{Content-Type} + header of the message. + """ + _MessageStructure.__init__(self, message, attrs) + self.subtype = subtype + + + def _getParts(self): + """ + Return an iterator over all of the sub-messages of this message. + """ + i = 0 + while True: + try: + part = self.message.getSubPart(i) + except IndexError: + break + else: + yield part + i += 1 + + + def encode(self, extended): + """ + Encode each sub-message and added the additional I{multipart} fields. + """ + result = [_getMessageStructure(p).encode(extended) for p in self._getParts()] + result.append(self.subtype) + if extended: + result.extend(self._extended()) + return result + + + def _extended(self): + """ + The extension data of a multipart body part are in the following order: + + 1. body parameter parenthesized list + A parenthesized list of attribute/value pairs [e.g., ("foo" + "bar" "baz" "rag") where "bar" is the value of "foo", and + "rag" is the value of "baz"] as defined in [MIME-IMB]. + + 2. body disposition + A parenthesized list, consisting of a disposition type + string, followed by a parenthesized list of disposition + attribute/value pairs as defined in [DISPOSITION]. + + 3. body language + A string or parenthesized list giving the body language + value as defined in [LANGUAGE-TAGS]. + + 4. body location + A string list giving the body content URI as defined in + [LOCATION]. + """ + result = [] + headers = self.message.getHeaders( + False, 'content-language', 'content-location', + 'content-disposition') + + result.append(self._unquotedAttrs()) + result.append(self._disposition(headers.get('content-disposition'))) + result.append(headers.get('content-language', None)) + result.append(headers.get('content-location', None)) + + return result + + + +def getBodyStructure(msg, extended=False): + """ + RFC 3501, 7.4.2, BODYSTRUCTURE:: + + A parenthesized list that describes the [MIME-IMB] body structure of a + message. This is computed by the server by parsing the [MIME-IMB] header + fields, defaulting various fields as necessary. + + For example, a simple text message of 48 lines and 2279 octets can have + a body structure of: ("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL + "7BIT" 2279 48) + + This is represented as:: + + ["TEXT", "PLAIN", ["CHARSET", "US-ASCII"], None, None, "7BIT", 2279, 48] + + These basic fields are documented in the RFC as: + + 1. body type + + A string giving the content media type name as defined in + [MIME-IMB]. + + 2. body subtype + + A string giving the content subtype name as defined in + [MIME-IMB]. + + 3. body parameter parenthesized list + + A parenthesized list of attribute/value pairs [e.g., ("foo" + "bar" "baz" "rag") where "bar" is the value of "foo" and + "rag" is the value of "baz"] as defined in [MIME-IMB]. + + 4. body id + + A string giving the content id as defined in [MIME-IMB]. + + 5. body description + + A string giving the content description as defined in + [MIME-IMB]. + + 6. body encoding + + A string giving the content transfer encoding as defined in + [MIME-IMB]. + + 7. body size + + A number giving the size of the body in octets. Note that this size is + the size in its transfer encoding and not the resulting size after any + decoding. + + Put another way, the body structure is a list of seven elements. The + semantics of the elements of this list are: + + 1. Byte string giving the major MIME type + 2. Byte string giving the minor MIME type + 3. A list giving the Content-Type parameters of the message + 4. A byte string giving the content identifier for the message part, or + None if it has no content identifier. + 5. A byte string giving the content description for the message part, or + None if it has no content description. + 6. A byte string giving the Content-Encoding of the message body + 7. An integer giving the number of octets in the message body + + The RFC goes on:: + + Multiple parts are indicated by parenthesis nesting. Instead of a body + type as the first element of the parenthesized list, there is a sequence + of one or more nested body structures. The second element of the + parenthesized list is the multipart subtype (mixed, digest, parallel, + alternative, etc.). + + For example, a two part message consisting of a text and a + BASE64-encoded text attachment can have a body structure of: (("TEXT" + "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 1152 23)("TEXT" "PLAIN" + ("CHARSET" "US-ASCII" "NAME" "cc.diff") + "<960723163407.20117h@cac.washington.edu>" "Compiler diff" "BASE64" 4554 + 73) "MIXED") + + This is represented as:: + + [["TEXT", "PLAIN", ["CHARSET", "US-ASCII"], None, None, "7BIT", 1152, + 23], + ["TEXT", "PLAIN", ["CHARSET", "US-ASCII", "NAME", "cc.diff"], + "<960723163407.20117h@cac.washington.edu>", "Compiler diff", + "BASE64", 4554, 73], + "MIXED"] + + In other words, a list of N + 1 elements, where N is the number of parts in + the message. The first N elements are structures as defined by the previous + section. The last element is the minor MIME subtype of the multipart + message. + + Additionally, the RFC describes extension data:: + + Extension data follows the multipart subtype. Extension data is never + returned with the BODY fetch, but can be returned with a BODYSTRUCTURE + fetch. Extension data, if present, MUST be in the defined order. + + The C{extended} flag controls whether extension data might be returned with + the normal data. + """ + return _getMessageStructure(msg).encode(extended) + + +def _formatHeaders(headers): + # TODO: This should use email.header.Header, which handles encoding + hdrs = [': '.join((k.title(), '\r\n'.join(v.splitlines()))) for (k, v) + in headers.items()] + hdrs = '\r\n'.join(hdrs) + '\r\n' + return networkString(hdrs) + +def subparts(m): + i = 0 + try: + while True: + yield m.getSubPart(i) + i += 1 + except IndexError: + pass + + + +def iterateInReactor(i): + """ + Consume an interator at most a single iteration per reactor iteration. + + If the iterator produces a Deferred, the next iteration will not occur + until the Deferred fires, otherwise the next iteration will be taken + in the next reactor iteration. + + @rtype: C{Deferred} + @return: A deferred which fires (with None) when the iterator is + exhausted or whose errback is called if there is an exception. + """ + from twisted.internet import reactor + d = defer.Deferred() + def go(last): + try: + r = next(i) + except StopIteration: + d.callback(last) + except: + d.errback() + else: + if isinstance(r, defer.Deferred): + r.addCallback(go) + else: + reactor.callLater(0, go, r) + go(None) + return d + + + +class MessageProducer: + CHUNK_SIZE = 2 ** 2 ** 2 ** 2 + _uuid4 = staticmethod(uuid.uuid4) + + def __init__(self, msg, buffer = None, scheduler = None): + """ + Produce this message. + + @param msg: The message I am to produce. + @type msg: L{IMessage} + + @param buffer: A buffer to hold the message in. If None, I will + use a L{tempfile.TemporaryFile}. + @type buffer: file-like + """ + self.msg = msg + if buffer is None: + buffer = tempfile.TemporaryFile() + self.buffer = buffer + if scheduler is None: + scheduler = iterateInReactor + self.scheduler = scheduler + self.write = self.buffer.write + + + def beginProducing(self, consumer): + self.consumer = consumer + return self.scheduler(self._produce()) + + + def _produce(self): + headers = self.msg.getHeaders(True) + boundary = None + if self.msg.isMultipart(): + content = headers.get('content-type') + parts = [x.split('=', 1) for x in content.split(';')[1:]] + parts = dict([(k.lower().strip(), v) for (k, v) in parts]) + boundary = parts.get('boundary') + if boundary is None: + # Bastards + boundary = '----=%s' % (self._uuid4().hex,) + headers['content-type'] += '; boundary="%s"' % (boundary,) + else: + if boundary.startswith('"') and boundary.endswith('"'): + boundary = boundary[1:-1] + boundary = networkString(boundary) + + self.write(_formatHeaders(headers)) + self.write(b'\r\n') + if self.msg.isMultipart(): + for p in subparts(self.msg): + self.write(b'\r\n--' + boundary + b'\r\n') + yield MessageProducer(p, self.buffer, self.scheduler + ).beginProducing(None + ) + self.write(b'\r\n--' + boundary + b'--\r\n' ) + else: + f = self.msg.getBodyFile() + while True: + b = f.read(self.CHUNK_SIZE) + if b: + self.buffer.write(b) + yield None + else: + break + if self.consumer: + self.buffer.seek(0, 0) + yield FileProducer(self.buffer + ).beginProducing(self.consumer + ).addCallback(lambda _: self + ) + + + +class _FetchParser: + class Envelope: + # Response should be a list of fields from the message: + # date, subject, from, sender, reply-to, to, cc, bcc, in-reply-to, + # and message-id. + # + # from, sender, reply-to, to, cc, and bcc are themselves lists of + # address information: + # personal name, source route, mailbox name, host name + # + # reply-to and sender must not be None. If not present in a message + # they should be defaulted to the value of the from field. + type = 'envelope' + __str__ = lambda self: 'envelope' + + + class Flags: + type = 'flags' + __str__ = lambda self: 'flags' + + + class InternalDate: + type = 'internaldate' + __str__ = lambda self: 'internaldate' + + + class RFC822Header: + type = 'rfc822header' + __str__ = lambda self: 'rfc822.header' + + + class RFC822Text: + type = 'rfc822text' + __str__ = lambda self: 'rfc822.text' + + + class RFC822Size: + type = 'rfc822size' + __str__ = lambda self: 'rfc822.size' + + + class RFC822: + type = 'rfc822' + __str__ = lambda self: 'rfc822' + + + class UID: + type = 'uid' + __str__ = lambda self: 'uid' + + + class Body: + type = 'body' + peek = False + header = None + mime = None + text = None + part = () + empty = False + partialBegin = None + partialLength = None + + def __str__(self): + return nativeString(self.__bytes__()) + + def __bytes__(self): + base = b'BODY' + part = b'' + separator = b'' + if self.part: + part = b'.'.join([unicode(x + 1).encode("ascii") + for x in self.part]) + separator = b'.' +# if self.peek: +# base += '.PEEK' + if self.header: + base += (b'[' + part + separator + + str(self.header).encode("ascii") + b']') + elif self.text: + base += b'[' + part + separator + b'TEXT]' + elif self.mime: + base += b'[' + part + separator + b'MIME]' + elif self.empty: + base += b'[' + part + b']' + if self.partialBegin is not None: + base += b'<' + intToBytes(self.partialBegin) + b'.' + intToBytes(self.partialLength) + b'>' + return base + + + class BodyStructure: + type = 'bodystructure' + __str__ = lambda self: 'bodystructure' + + + # These three aren't top-level, they don't need type indicators + class Header: + negate = False + fields = None + part = None + + def __str__(self): + return nativeString(self.__bytes__()) + + + def __bytes__(self): + base = b'HEADER' + if self.fields: + base += b'.FIELDS' + if self.negate: + base += b'.NOT' + fields = [] + for f in self.fields: + f = f.title() + if _needsQuote(f): + f = _quote(f) + fields.append(f) + base += b' (' + b' '.join(fields) + b')' + if self.part: + # TODO: _FetchParser never assigns Header.part - dead + # code? + base = b'.'.join([(x + 1).__bytes__() for x in self.part]) + b'.' + base + return base + + + class Text: + pass + + + class MIME: + pass + + parts = None + + _simple_fetch_att = [ + (b'envelope', Envelope), + (b'flags', Flags), + (b'internaldate', InternalDate), + (b'rfc822.header', RFC822Header), + (b'rfc822.text', RFC822Text), + (b'rfc822.size', RFC822Size), + (b'rfc822', RFC822), + (b'uid', UID), + (b'bodystructure', BodyStructure), + ] + + def __init__(self): + self.state = ['initial'] + self.result = [] + self.remaining = b'' + + + def parseString(self, s): + s = self.remaining + s + try: + while s or self.state: + if not self.state: + raise IllegalClientResponse("Invalid Argument") + # print 'Entering state_' + self.state[-1] + ' with', repr(s) + state = self.state.pop() + try: + used = getattr(self, 'state_' + state)(s) + except: + self.state.append(state) + raise + else: + # print state, 'consumed', repr(s[:used]) + s = s[used:] + finally: + self.remaining = s + + + def state_initial(self, s): + # In the initial state, the literals "ALL", "FULL", and "FAST" + # are accepted, as is a ( indicating the beginning of a fetch_att + # token, as is the beginning of a fetch_att token. + if s == b'': + return 0 + + l = s.lower() + if l.startswith(b'all'): + self.result.extend(( + self.Flags(), self.InternalDate(), + self.RFC822Size(), self.Envelope() + )) + return 3 + if l.startswith(b'full'): + self.result.extend(( + self.Flags(), self.InternalDate(), + self.RFC822Size(), self.Envelope(), + self.Body() + )) + return 4 + if l.startswith(b'fast'): + self.result.extend(( + self.Flags(), self.InternalDate(), self.RFC822Size(), + )) + return 4 + + if l.startswith(b'('): + self.state.extend(('close_paren', 'maybe_fetch_att', 'fetch_att')) + return 1 + + self.state.append('fetch_att') + return 0 + + + def state_close_paren(self, s): + if s.startswith(b')'): + return 1 + # TODO: does maybe_fetch_att's startswith(b')') make this dead + # code? + raise Exception("Missing )") + + + def state_whitespace(self, s): + # Eat up all the leading whitespace + if not s or not s[0:1].isspace(): + raise Exception("Whitespace expected, none found") + i = 0 + for i in range(len(s)): + if not s[i:i + 1].isspace(): + break + return i + + + def state_maybe_fetch_att(self, s): + if not s.startswith(b')'): + self.state.extend(('maybe_fetch_att', 'fetch_att', 'whitespace')) + return 0 + + + def state_fetch_att(self, s): + # Allowed fetch_att tokens are "ENVELOPE", "FLAGS", "INTERNALDATE", + # "RFC822", "RFC822.HEADER", "RFC822.SIZE", "RFC822.TEXT", "BODY", + # "BODYSTRUCTURE", "UID", + # "BODY [".PEEK"] [
] ["<" "." ">"] + + l = s.lower() + for (name, cls) in self._simple_fetch_att: + if l.startswith(name): + self.result.append(cls()) + return len(name) + + b = self.Body() + if l.startswith(b'body.peek'): + b.peek = True + used = 9 + elif l.startswith(b'body'): + used = 4 + else: + raise Exception("Nothing recognized in fetch_att: %s" % (l,)) + + self.pending_body = b + self.state.extend(('got_body', 'maybe_partial', 'maybe_section')) + return used + + + def state_got_body(self, s): + self.result.append(self.pending_body) + del self.pending_body + return 0 + + + def state_maybe_section(self, s): + if not s.startswith(b"["): + return 0 + + self.state.extend(('section', 'part_number')) + return 1 + + _partExpr = re.compile(b'(\d+(?:\.\d+)*)\.?') + + + def state_part_number(self, s): + m = self._partExpr.match(s) + if m is not None: + self.parts = [int(p) - 1 for p in m.groups()[0].split(b'.')] + return m.end() + else: + self.parts = [] + return 0 + + + def state_section(self, s): + # Grab "HEADER]" or "HEADER.FIELDS (Header list)]" or + # "HEADER.FIELDS.NOT (Header list)]" or "TEXT]" or "MIME]" or + # just "]". + + l = s.lower() + used = 0 + if l.startswith(b']'): + self.pending_body.empty = True + used += 1 + elif l.startswith(b'header]'): + h = self.pending_body.header = self.Header() + h.negate = True + h.fields = () + used += 7 + elif l.startswith(b'text]'): + self.pending_body.text = self.Text() + used += 5 + elif l.startswith(b'mime]'): + self.pending_body.mime = self.MIME() + used += 5 + else: + h = self.Header() + if l.startswith(b'header.fields.not'): + h.negate = True + used += 17 + elif l.startswith(b'header.fields'): + used += 13 + else: + raise Exception("Unhandled section contents: %r" % (l,)) + + self.pending_body.header = h + self.state.extend(('finish_section', 'header_list', 'whitespace')) + self.pending_body.part = tuple(self.parts) + self.parts = None + return used + + + def state_finish_section(self, s): + if not s.startswith(b']'): + raise Exception("section must end with ]") + return 1 + + + def state_header_list(self, s): + if not s.startswith(b'('): + raise Exception("Header list must begin with (") + end = s.find(b')') + if end == -1: + raise Exception("Header list must end with )") + + headers = s[1:end].split() + self.pending_body.header.fields = [h.upper() for h in headers] + return end + 1 + + + def state_maybe_partial(self, s): + # Grab or nothing at all + if not s.startswith(b'<'): + return 0 + end = s.find(b'>') + if end == -1: + raise Exception("Found < but not >") + + partial = s[1:end] + parts = partial.split(b'.', 1) + if len(parts) != 2: + raise Exception("Partial specification did not include two .-delimited integers") + begin, length = map(int, parts) + self.pending_body.partialBegin = begin + self.pending_body.partialLength = length + + return end + 1 + + + +class FileProducer: + CHUNK_SIZE = 2 ** 2 ** 2 ** 2 + + firstWrite = True + + def __init__(self, f): + self.f = f + + + def beginProducing(self, consumer): + self.consumer = consumer + self.produce = consumer.write + d = self._onDone = defer.Deferred() + self.consumer.registerProducer(self, False) + return d + + + def resumeProducing(self): + b = b'' + if self.firstWrite: + b = b'{' + intToBytes(self._size()) + b'}\r\n' + self.firstWrite = False + if not self.f: + return + b = b + self.f.read(self.CHUNK_SIZE) + if not b: + self.consumer.unregisterProducer() + self._onDone.callback(self) + self._onDone = self.f = self.consumer = None + else: + self.produce(b) + + + def pauseProducing(self): + """ + Pause the producer. This does nothing. + """ + + + def stopProducing(self): + """ + Stop the producer. This does nothing. + """ + + + def _size(self): + b = self.f.tell() + self.f.seek(0, 2) + e = self.f.tell() + self.f.seek(b, 0) + return e - b + + + +def parseTime(s): + # XXX - This may require localization :( + months = [ + 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', + 'nov', 'dec', 'january', 'february', 'march', 'april', 'may', 'june', + 'july', 'august', 'september', 'october', 'november', 'december' + ] + expr = { + 'day': r"(?P3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", + 'mon': r"(?P\w+)", + 'year': r"(?P\d\d\d\d)" + } + m = re.match('%(day)s-%(mon)s-%(year)s' % expr, s) + if not m: + raise ValueError("Cannot parse time string %r" % (s,)) + d = m.groupdict() + try: + d['mon'] = 1 + (months.index(d['mon'].lower()) % 12) + d['year'] = int(d['year']) + d['day'] = int(d['day']) + except ValueError: + raise ValueError("Cannot parse time string %r" % (s,)) + else: + return time.struct_time( + (d['year'], d['mon'], d['day'], 0, 0, 0, -1, -1, -1) + ) + +# we need to cast Python >=3.3 memoryview to chars (from unsigned bytes), but +# cast is absent in previous versions: thus, the lambda returns the +# memoryview instance while ignoring the format +memory_cast = getattr(memoryview, "cast", lambda *x: x[0]) + +def modified_base64(s): + s_utf7 = s.encode('utf-7') + return s_utf7[1:-1].replace(b'/', b',') + +def modified_unbase64(s): + s_utf7 = b'+' + s.replace(b',', b'/') + b'-' + return s_utf7.decode('utf-7') + +def encoder(s, errors=None): + """ + Encode the given C{unicode} string using the IMAP4 specific variation of + UTF-7. + + @type s: C{unicode} + @param s: The text to encode. + + @param errors: Policy for handling encoding errors. Currently ignored. + + @return: L{tuple} of a L{str} giving the encoded bytes and an L{int} + giving the number of code units consumed from the input. + """ + r = bytearray() + _in = [] + valid_chars = set(map(chr, range(0x20,0x7f))) - {u"&"} + for c in s: + if c in valid_chars: + if _in: + r += b'&' + modified_base64(''.join(_in)) + b'-' + del _in[:] + r.append(ord(c)) + elif c == u'&': + if _in: + r += b'&' + modified_base64(''.join(_in)) + b'-' + del _in[:] + r += b'&-' + else: + _in.append(c) + if _in: + r.extend(b'&' + modified_base64(''.join(_in)) + b'-') + return (bytes(r), len(s)) + + + + +def decoder(s, errors=None): + """ + Decode the given L{str} using the IMAP4 specific variation of UTF-7. + + @type s: L{str} + @param s: The bytes to decode. + + @param errors: Policy for handling decoding errors. Currently ignored. + + @return: a L{tuple} of a C{unicode} string giving the text which was + decoded and an L{int} giving the number of bytes consumed from the + input. + """ + r = [] + decode = [] + s = memory_cast(memoryview(s), 'c') + for c in s: + if c == b'&' and not decode: + decode.append(b'&') + elif c == b'-' and decode: + if len(decode) == 1: + r.append(u'&') + else: + r.append(modified_unbase64(b''.join(decode[1:]))) + decode = [] + elif decode: + decode.append(c) + else: + r.append(c.decode()) + if decode: + r.append(modified_unbase64(b''.join(decode[1:]))) + return (u''.join(r), len(s)) + + + +class StreamReader(codecs.StreamReader): + def decode(self, s, errors='strict'): + return decoder(s) + + + +class StreamWriter(codecs.StreamWriter): + def encode(self, s, errors='strict'): + return encoder(s) + + +_codecInfo = codecs.CodecInfo(encoder, decoder, StreamReader, StreamWriter) + + +def imap4_utf_7(name): + if name == 'imap4-utf-7': + return _codecInfo + +codecs.register(imap4_utf_7) + +__all__ = [ + # Protocol classes + 'IMAP4Server', 'IMAP4Client', + + # Interfaces + 'IMailboxListener', 'IClientAuthentication', 'IAccount', 'IMailbox', + 'INamespacePresenter', 'ICloseableMailbox', 'IMailboxInfo', + 'IMessage', 'IMessageCopier', 'IMessageFile', 'ISearchableMailbox', + 'IMessagePart', + + # Exceptions + 'IMAP4Exception', 'IllegalClientResponse', 'IllegalOperation', + 'IllegalMailboxEncoding', 'UnhandledResponse', 'NegativeResponse', + 'NoSupportedAuthentication', 'IllegalServerResponse', + 'IllegalIdentifierError', 'IllegalQueryError', 'MismatchedNesting', + 'MismatchedQuoting', 'MailboxException', 'MailboxCollision', + 'NoSuchMailbox', 'ReadOnlyMailbox', + + # Auth objects + 'CramMD5ClientAuthenticator', 'PLAINAuthenticator', 'LOGINAuthenticator', + 'PLAINCredentials', 'LOGINCredentials', + + # Simple query interface + 'Query', 'Not', 'Or', + + # Miscellaneous + 'MemoryAccount', + 'statusRequestHelper', +] diff --git a/contrib/python/Twisted/py2/twisted/mail/interfaces.py b/contrib/python/Twisted/py2/twisted/mail/interfaces.py new file mode 100644 index 00000000000..9dc133df455 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/interfaces.py @@ -0,0 +1,1110 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Interfaces for L{twisted.mail}. + +@since: 16.5 +""" + +from __future__ import absolute_import, division + +from zope.interface import Interface + + +class IChallengeResponse(Interface): + """ + An C{IMAPrev4} authorization challenge mechanism. + """ + + def getChallenge(): + """ + Return a client challenge. + + @return: A challenge. + @rtype: L{bytes} + """ + + + def setResponse(response): + """ + Extract a username and possibly a password from a response and + assign them to C{username} and C{password} instance variables. + + @param response: A decoded response. + @type response: L{bytes} + + @see: L{credentials.IUsernamePassword} or + L{credentials.IUsernameHashedPassword} + """ + + + def moreChallenges(): + """ + Are there more challenges than just the first? If so, callers + should challenge clients with the result of L{getChallenge}, + and check their response with L{setResponse} in a loop until + this returns L{False} + + @return: Are there more challenges? + @rtype: L{bool} + """ + + + +class IClientAuthentication(Interface): + + def getName(): + """ + Return an identifier associated with this authentication scheme. + + @rtype: L{bytes} + """ + + def challengeResponse(secret, challenge): + """ + Generate a challenge response string. + """ + + + +class IServerFactoryPOP3(Interface): + """ + An interface for querying capabilities of a POP3 server. + + Any cap_* method may raise L{NotImplementedError} if the particular + capability is not supported. If L{cap_EXPIRE()} does not raise + L{NotImplementedError}, L{perUserExpiration()} must be implemented, + otherwise they are optional. If L{cap_LOGIN_DELAY()} is implemented, + L{perUserLoginDelay()} must be implemented, otherwise they are optional. + + @type challengers: L{dict} of L{bytes} -> L{IUsernameHashedPassword + } + @ivar challengers: A mapping of challenger names to + L{IUsernameHashedPassword } + provider. + """ + def cap_IMPLEMENTATION(): + """ + Return a string describing the POP3 server implementation. + + @rtype: L{bytes} + @return: Server implementation information. + """ + + + def cap_EXPIRE(): + """ + Return the minimum number of days messages are retained. + + @rtype: L{int} or L{None} + @return: The minimum number of days messages are retained or none, if + the server never deletes messages. + """ + + + def perUserExpiration(): + """ + Indicate whether the message expiration policy differs per user. + + @rtype: L{bool} + @return: C{True} when the message expiration policy differs per user, + C{False} otherwise. + """ + + + def cap_LOGIN_DELAY(): + """ + Return the minimum number of seconds between client logins. + + @rtype: L{int} + @return: The minimum number of seconds between client logins. + """ + + + def perUserLoginDelay(): + """ + Indicate whether the login delay period differs per user. + + @rtype: L{bool} + @return: C{True} when the login delay differs per user, C{False} + otherwise. + """ + + + +class IMailboxPOP3(Interface): + """ + An interface for mailbox access. + + Message indices are 0-based. + + @type loginDelay: L{int} + @ivar loginDelay: The number of seconds between allowed logins for the + user associated with this mailbox. + + @type messageExpiration: L{int} + @ivar messageExpiration: The number of days messages in this mailbox will + remain on the server before being deleted. + """ + def listMessages(index=None): + """ + Retrieve the size of a message, or, if none is specified, the size of + each message in the mailbox. + + @type index: L{int} or L{None} + @param index: The 0-based index of the message. + + @rtype: L{int}, sequence of L{int}, or L{Deferred } + @return: The number of octets in the specified message, or, if an + index is not specified, a sequence of the number of octets for + all messages in the mailbox or a deferred which fires with + one of those. Any value which corresponds to a deleted message + is set to 0. + + @raise ValueError or IndexError: When the index does not correspond to + a message in the mailbox. The use of ValueError is preferred. + """ + + + def getMessage(index): + """ + Retrieve a file containing the contents of a message. + + @type index: L{int} + @param index: The 0-based index of a message. + + @rtype: file-like object + @return: A file containing the message. + + @raise ValueError or IndexError: When the index does not correspond to + a message in the mailbox. The use of ValueError is preferred. + """ + + + def getUidl(index): + """ + Get a unique identifier for a message. + + @type index: L{int} + @param index: The 0-based index of a message. + + @rtype: L{bytes} + @return: A string of printable characters uniquely identifying the + message for all time. + + @raise ValueError or IndexError: When the index does not correspond to + a message in the mailbox. The use of ValueError is preferred. + """ + + + def deleteMessage(index): + """ + Mark a message for deletion. + + This must not change the number of messages in this mailbox. Further + requests for the size of the deleted message should return 0. Further + requests for the message itself may raise an exception. + + @type index: L{int} + @param index: The 0-based index of a message. + + @raise ValueError or IndexError: When the index does not correspond to + a message in the mailbox. The use of ValueError is preferred. + """ + + + def undeleteMessages(): + """ + Undelete all messages marked for deletion. + + Any message which can be undeleted should be returned to its original + position in the message sequence and retain its original UID. + """ + + + def sync(): + """ + Discard the contents of any message marked for deletion. + """ + + + +class IDomain(Interface): + """ + An interface for email domains. + """ + def exists(user): + """ + Check whether a user exists in this domain. + + @type user: L{User} + @param user: A user. + + @rtype: no-argument callable which returns L{IMessageSMTP} provider + @return: A function which takes no arguments and returns a message + receiver for the user. + + @raise SMTPBadRcpt: When the given user does not exist in this domain. + """ + + + def addUser(user, password): + """ + Add a user to this domain. + + @type user: L{bytes} + @param user: A username. + + @type password: L{bytes} + @param password: A password. + """ + + + def getCredentialsCheckers(): + """ + Return credentials checkers for this domain. + + @rtype: L{list} of L{ICredentialsChecker + } provider + @return: Credentials checkers for this domain. + """ + + + +class IAlias(Interface): + """ + An interface for aliases. + """ + def createMessageReceiver(): + """ + Create a message receiver. + + @rtype: L{IMessageSMTP} provider + @return: A message receiver. + """ + + + +class IAliasableDomain(IDomain): + """ + An interface for email domains which can be aliased to other domains. + """ + def setAliasGroup(aliases): + """ + Set the group of defined aliases for this domain. + + @type aliases: L{dict} of L{bytes} -> L{IAlias} provider + @param aliases: A mapping of domain name to alias. + """ + + + def exists(user, memo=None): + """ + Check whether a user exists in this domain or an alias of it. + + @type user: L{User} + @param user: A user. + + @type memo: L{None} or L{dict} of + L{AliasBase } + @param memo: A record of the addresses already considered while + resolving aliases. The default value should be used by all external + code. + + @rtype: no-argument callable which returns L{IMessageSMTP} provider + @return: A function which takes no arguments and returns a message + receiver for the user. + + @raise SMTPBadRcpt: When the given user does not exist in this domain + or an alias of it. + """ + + + +class IMessageDelivery(Interface): + + def receivedHeader(helo, origin, recipients): + """ + Generate the Received header for a message. + + @type helo: 2-L{tuple} of L{bytes} and L{bytes}. + @param helo: The argument to the HELO command and the client's IP + address. + + @type origin: L{Address} + @param origin: The address the message is from + + @type recipients: L{list} of L{User} + @param recipients: A list of the addresses for which this message + is bound. + + @rtype: L{bytes} + @return: The full C{"Received"} header string. + """ + + def validateTo(user): + """ + Validate the address for which the message is destined. + + @type user: L{User} + @param user: The address to validate. + + @rtype: no-argument callable + @return: A L{Deferred} which becomes, or a callable which takes no + arguments and returns an object implementing L{IMessageSMTP}. This + will be called and the returned object used to deliver the message + when it arrives. + + @raise SMTPBadRcpt: Raised if messages to the address are not to be + accepted. + """ + + def validateFrom(helo, origin): + """ + Validate the address from which the message originates. + + @type helo: 2-L{tuple} of L{bytes} and L{bytes}. + @param helo: The argument to the HELO command and the client's IP + address. + + @type origin: L{Address} + @param origin: The address the message is from + + @rtype: L{Deferred} or L{Address} + @return: C{origin} or a L{Deferred} whose callback will be + passed C{origin}. + + @raise SMTPBadSender: Raised of messages from this address are + not to be accepted. + """ + + + +class IMessageDeliveryFactory(Interface): + """ + An alternate interface to implement for handling message delivery. + + It is useful to implement this interface instead of L{IMessageDelivery} + directly because it allows the implementor to distinguish between different + messages delivery over the same connection. This can be used to optimize + delivery of a single message to multiple recipients, something which cannot + be done by L{IMessageDelivery} implementors due to their lack of + information. + """ + def getMessageDelivery(): + """ + Return an L{IMessageDelivery} object. + + This will be called once per message. + """ + + + +class IMessageSMTP(Interface): + """ + Interface definition for messages that can be sent via SMTP. + """ + + def lineReceived(line): + """ + Handle another line. + """ + + def eomReceived(): + """ + Handle end of message. + + return a deferred. The deferred should be called with either: + callback(string) or errback(error) + + @rtype: L{Deferred} + """ + + def connectionLost(): + """ + Handle message truncated. + + semantics should be to discard the message + """ + + + +class IMessageIMAPPart(Interface): + def getHeaders(negate, *names): + """ + Retrieve a group of message headers. + + @type names: L{tuple} of L{str} + @param names: The names of the headers to retrieve or omit. + + @type negate: L{bool} + @param negate: If True, indicates that the headers listed in C{names} + should be omitted from the return value, rather than included. + + @rtype: L{dict} + @return: A mapping of header field names to header field values + """ + + + def getBodyFile(): + """ + Retrieve a file object containing only the body of this message. + """ + + + def getSize(): + """ + Retrieve the total size, in octets, of this message. + + @rtype: L{int} + """ + + + def isMultipart(): + """ + Indicate whether this message has subparts. + + @rtype: L{bool} + """ + + + def getSubPart(part): + """ + Retrieve a MIME sub-message + + @type part: L{int} + @param part: The number of the part to retrieve, indexed from 0. + + @raise IndexError: Raised if the specified part does not exist. + @raise TypeError: Raised if this message is not multipart. + + @rtype: Any object implementing L{IMessageIMAPPart}. + @return: The specified sub-part. + """ + + + +class IMessageIMAP(IMessageIMAPPart): + + def getUID(): + """ + Retrieve the unique identifier associated with this message. + """ + + + def getFlags(): + """ + Retrieve the flags associated with this message. + + @rtype: C{iterable} + @return: The flags, represented as strings. + """ + + + def getInternalDate(): + """ + Retrieve the date internally associated with this message. + + @rtype: L{bytes} + @return: An RFC822-formatted date string. + """ + + + +class IMessageIMAPFile(Interface): + """ + Optional message interface for representing messages as files. + + If provided by message objects, this interface will be used instead the + more complex MIME-based interface. + """ + + def open(): + """ + Return a file-like object opened for reading. + + Reading from the returned file will return all the bytes of which this + message consists. + """ + + + +class ISearchableIMAPMailbox(Interface): + + def search(query, uid): + """ + Search for messages that meet the given query criteria. + + If this interface is not implemented by the mailbox, + L{IMailboxIMAP.fetch} and various methods of L{IMessageIMAP} will be + used instead. + + Implementations which wish to offer better performance than the default + implementation should implement this interface. + + @type query: L{list} + @param query: The search criteria + + @type uid: L{bool} + @param uid: If true, the IDs specified in the query are UIDs; otherwise + they are message sequence IDs. + + @rtype: L{list} or L{Deferred} + @return: A list of message sequence numbers or message UIDs which match + the search criteria or a L{Deferred} whose callback will be invoked + with such a list. + + @raise IllegalQueryError: Raised when query is not valid. + """ + + + +class IMailboxIMAPListener(Interface): + """ + Interface for objects interested in mailbox events + """ + + def modeChanged(writeable): + """ + Indicates that the write status of a mailbox has changed. + + @type writeable: L{bool} + @param writeable: A true value if write is now allowed, false + otherwise. + """ + + + def flagsChanged(newFlags): + """ + Indicates that the flags of one or more messages have changed. + + @type newFlags: L{dict} + @param newFlags: A mapping of message identifiers to tuples of flags + now set on that message. + """ + + + def newMessages(exists, recent): + """ + Indicates that the number of messages in a mailbox has changed. + + @type exists: L{int} or L{None} + @param exists: The total number of messages now in this mailbox. If the + total number of messages has not changed, this should be L{None}. + + @type recent: L{int} + @param recent: The number of messages now flagged C{\\Recent}. If the + number of recent messages has not changed, this should be L{None}. + """ + + + +class IMessageIMAPCopier(Interface): + def copy(messageObject): + """ + Copy the given message object into this mailbox. + + The message object will be one which was previously returned by + L{IMailboxIMAP.fetch}. + + Implementations which wish to offer better performance than the default + implementation should implement this interface. + + If this interface is not implemented by the mailbox, + L{IMailboxIMAP.addMessage} will be used instead. + + @rtype: L{Deferred} or L{int} + @return: Either the UID of the message or a Deferred which fires with + the UID when the copy finishes. + """ + + + +class IMailboxIMAPInfo(Interface): + """ + Interface specifying only the methods required for C{listMailboxes}. + + Implementations can return objects implementing only these methods for + return to C{listMailboxes} if it can allow them to operate more + efficiently. + """ + + def getFlags(): + """ + Return the flags defined in this mailbox + + Flags with the \\ prefix are reserved for use as system flags. + + @rtype: L{list} of L{str} + @return: A list of the flags that can be set on messages in this + mailbox. + """ + + + def getHierarchicalDelimiter(): + """ + Get the character which delimits namespaces for in this mailbox. + + @rtype: L{bytes} + """ + + + +class IMailboxIMAP(IMailboxIMAPInfo): + def getUIDValidity(): + """ + Return the unique validity identifier for this mailbox. + + @rtype: L{int} + """ + + + def getUIDNext(): + """ + Return the likely UID for the next message added to this mailbox. + + @rtype: L{int} + """ + + + def getUID(message): + """ + Return the UID of a message in the mailbox + + @type message: L{int} + @param message: The message sequence number + + @rtype: L{int} + @return: The UID of the message. + """ + + + def getMessageCount(): + """ + Return the number of messages in this mailbox. + + @rtype: L{int} + """ + + + def getRecentCount(): + """ + Return the number of messages with the 'Recent' flag. + + @rtype: L{int} + """ + + + def getUnseenCount(): + """ + Return the number of messages with the 'Unseen' flag. + + @rtype: L{int} + """ + + + def isWriteable(): + """ + Get the read/write status of the mailbox. + + @rtype: L{int} + @return: A true value if write permission is allowed, a false value + otherwise. + """ + + + def destroy(): + """ + Called before this mailbox is deleted, permanently. + + If necessary, all resources held by this mailbox should be cleaned up + here. This function _must_ set the \\Noselect flag on this mailbox. + """ + + + def requestStatus(names): + """ + Return status information about this mailbox. + + Mailboxes which do not intend to do any special processing to generate + the return value, C{statusRequestHelper} can be used to build the + dictionary by calling the other interface methods which return the data + for each name. + + @type names: Any iterable + @param names: The status names to return information regarding. The + possible values for each name are: MESSAGES, RECENT, UIDNEXT, + UIDVALIDITY, UNSEEN. + + @rtype: L{dict} or L{Deferred} + @return: A dictionary containing status information about the requested + names is returned. If the process of looking this information up + would be costly, a deferred whose callback will eventually be + passed this dictionary is returned instead. + """ + + + def addListener(listener): + """ + Add a mailbox change listener + + @type listener: Any object which implements C{IMailboxIMAPListener} + @param listener: An object to add to the set of those which will be + notified when the contents of this mailbox change. + """ + + + def removeListener(listener): + """ + Remove a mailbox change listener + + @type listener: Any object previously added to and not removed from + this mailbox as a listener. + @param listener: The object to remove from the set of listeners. + + @raise ValueError: Raised when the given object is not a listener for + this mailbox. + """ + + + def addMessage(message, flags=(), date=None): + """ + Add the given message to this mailbox. + + @type message: A file-like object + @param message: The RFC822 formatted message + + @type flags: Any iterable of L{bytes} + @param flags: The flags to associate with this message + + @type date: L{bytes} + @param date: If specified, the date to associate with this message. + + @rtype: L{Deferred} + @return: A deferred whose callback is invoked with the message id if + the message is added successfully and whose errback is invoked + otherwise. + + @raise ReadOnlyMailbox: Raised if this Mailbox is not open for + read-write. + """ + + + def expunge(): + """ + Remove all messages flagged \\Deleted. + + @rtype: L{list} or L{Deferred} + @return: The list of message sequence numbers which were deleted, or a + L{Deferred} whose callback will be invoked with such a list. + + @raise ReadOnlyMailbox: Raised if this Mailbox is not open for + read-write. + """ + + + def fetch(messages, uid): + """ + Retrieve one or more messages. + + @type messages: C{MessageSet} + @param messages: The identifiers of messages to retrieve information + about + + @type uid: L{bool} + @param uid: If true, the IDs specified in the query are UIDs; otherwise + they are message sequence IDs. + + @rtype: Any iterable of two-tuples of message sequence numbers and + implementors of C{IMessageIMAP}. + """ + + + def store(messages, flags, mode, uid): + """ + Set the flags of one or more messages. + + @type messages: A MessageSet object with the list of messages requested + @param messages: The identifiers of the messages to set the flags of. + + @type flags: sequence of L{str} + @param flags: The flags to set, unset, or add. + + @type mode: -1, 0, or 1 + @param mode: If mode is -1, these flags should be removed from the + specified messages. If mode is 1, these flags should be added to + the specified messages. If mode is 0, all existing flags should be + cleared and these flags should be added. + + @type uid: L{bool} + @param uid: If true, the IDs specified in the query are UIDs; otherwise + they are message sequence IDs. + + @rtype: L{dict} or L{Deferred} + @return: A L{dict} mapping message sequence numbers to sequences of + L{str} representing the flags set on the message after this + operation has been performed, or a L{Deferred} whose callback will + be invoked with such a L{dict}. + + @raise ReadOnlyMailbox: Raised if this mailbox is not open for + read-write. + """ + + + +class ICloseableMailboxIMAP(Interface): + """ + A supplementary interface for mailboxes which require cleanup on close. + + Implementing this interface is optional. If it is implemented, the protocol + code will call the close method defined whenever a mailbox is closed. + """ + + def close(): + """ + Close this mailbox. + + @return: A L{Deferred} which fires when this mailbox has been closed, + or None if the mailbox can be closed immediately. + """ + + + +class IAccountIMAP(Interface): + """ + Interface for Account classes + + Implementors of this interface should consider implementing + C{INamespacePresenter}. + """ + + def addMailbox(name, mbox=None): + """ + Add a new mailbox to this account + + @type name: L{bytes} + @param name: The name associated with this mailbox. It may not contain + multiple hierarchical parts. + + @type mbox: An object implementing C{IMailboxIMAP} + @param mbox: The mailbox to associate with this name. If L{None}, a + suitable default is created and used. + + @rtype: L{Deferred} or L{bool} + @return: A true value if the creation succeeds, or a deferred whose + callback will be invoked when the creation succeeds. + + @raise MailboxException: Raised if this mailbox cannot be added for + some reason. This may also be raised asynchronously, if a + L{Deferred} is returned. + """ + + + def create(pathspec): + """ + Create a new mailbox from the given hierarchical name. + + @type pathspec: L{bytes} + @param pathspec: The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one do not exist, + they are created as well. + + @rtype: L{Deferred} or L{bool} + @return: A true value if the creation succeeds, or a deferred whose + callback will be invoked when the creation succeeds. + + @raise MailboxException: Raised if this mailbox cannot be added. This + may also be raised asynchronously, if a L{Deferred} is returned. + """ + + + def select(name, rw=True): + """ + Acquire a mailbox, given its name. + + @type name: L{bytes} + @param name: The mailbox to acquire + + @type rw: L{bool} + @param rw: If a true value, request a read-write version of this + mailbox. If a false value, request a read-only version. + + @rtype: Any object implementing C{IMailboxIMAP} or L{Deferred} + @return: The mailbox object, or a L{Deferred} whose callback will be + invoked with the mailbox object. None may be returned if the + specified mailbox may not be selected for any reason. + """ + + + def delete(name): + """ + Delete the mailbox with the specified name. + + @type name: L{bytes} + @param name: The mailbox to delete. + + @rtype: L{Deferred} or L{bool} + @return: A true value if the mailbox is successfully deleted, or a + L{Deferred} whose callback will be invoked when the deletion + completes. + + @raise MailboxException: Raised if this mailbox cannot be deleted. This + may also be raised asynchronously, if a L{Deferred} is returned. + """ + + + def rename(oldname, newname): + """ + Rename a mailbox + + @type oldname: L{bytes} + @param oldname: The current name of the mailbox to rename. + + @type newname: L{bytes} + @param newname: The new name to associate with the mailbox. + + @rtype: L{Deferred} or L{bool} + @return: A true value if the mailbox is successfully renamed, or a + L{Deferred} whose callback will be invoked when the rename + operation is completed. + + @raise MailboxException: Raised if this mailbox cannot be renamed. This + may also be raised asynchronously, if a L{Deferred} is returned. + """ + + + def isSubscribed(name): + """ + Check the subscription status of a mailbox + + @type name: L{bytes} + @param name: The name of the mailbox to check + + @rtype: L{Deferred} or L{bool} + @return: A true value if the given mailbox is currently subscribed to, + a false value otherwise. A L{Deferred} may also be returned whose + callback will be invoked with one of these values. + """ + + + def subscribe(name): + """ + Subscribe to a mailbox + + @type name: L{bytes} + @param name: The name of the mailbox to subscribe to + + @rtype: L{Deferred} or L{bool} + @return: A true value if the mailbox is subscribed to successfully, or + a Deferred whose callback will be invoked with this value when the + subscription is successful. + + @raise MailboxException: Raised if this mailbox cannot be subscribed + to. This may also be raised asynchronously, if a L{Deferred} is + returned. + """ + + + def unsubscribe(name): + """ + Unsubscribe from a mailbox + + @type name: L{bytes} + @param name: The name of the mailbox to unsubscribe from + + @rtype: L{Deferred} or L{bool} + @return: A true value if the mailbox is unsubscribed from successfully, + or a Deferred whose callback will be invoked with this value when + the unsubscription is successful. + + @raise MailboxException: Raised if this mailbox cannot be unsubscribed + from. This may also be raised asynchronously, if a L{Deferred} is + returned. + """ + + + def listMailboxes(ref, wildcard): + """ + List all the mailboxes that meet a certain criteria + + @type ref: L{bytes} + @param ref: The context in which to apply the wildcard + + @type wildcard: L{bytes} + @param wildcard: An expression against which to match mailbox names. + '*' matches any number of characters in a mailbox name, and '%' + matches similarly, but will not match across hierarchical + boundaries. + + @rtype: L{list} of L{tuple} + @return: A list of C{(mailboxName, mailboxObject)} which meet the given + criteria. C{mailboxObject} should implement either + C{IMailboxIMAPInfo} or C{IMailboxIMAP}. A Deferred may also be + returned. + """ + + + +class INamespacePresenter(Interface): + + def getPersonalNamespaces(): + """ + Report the available personal namespaces. + + Typically there should be only one personal namespace. A common name + for it is C{\"\"}, and its hierarchical delimiter is usually C{\"/\"}. + + @rtype: iterable of two-tuples of strings + @return: The personal namespaces and their hierarchical delimiters. If + no namespaces of this type exist, None should be returned. + """ + + + def getSharedNamespaces(): + """ + Report the available shared namespaces. + + Shared namespaces do not belong to any individual user but are usually + to one or more of them. Examples of shared namespaces might be + C{\"#news\"} for a usenet gateway. + + @rtype: iterable of two-tuples of strings + @return: The shared namespaces and their hierarchical delimiters. If no + namespaces of this type exist, None should be returned. + """ + + + def getUserNamespaces(): + """ + Report the available user namespaces. + + These are namespaces that contain folders belonging to other users + access to which this account has been granted. + + @rtype: iterable of two-tuples of strings + @return: The user namespaces and their hierarchical delimiters. If no + namespaces of this type exist, None should be returned. + """ + + + +__all__ = [ + # IMAP + 'IAccountIMAP', 'ICloseableMailboxIMAP', 'IMailboxIMAP', + 'IMailboxIMAPInfo', 'IMailboxIMAPListener', 'IMessageIMAP', + 'IMessageIMAPCopier', 'IMessageIMAPFile', 'IMessageIMAPPart', + 'ISearchableIMAPMailbox', 'INamespacePresenter', + + # SMTP + 'IMessageDelivery', 'IMessageDeliveryFactory', 'IMessageSMTP', + + # Domains and aliases + 'IDomain', 'IAlias', 'IAliasableDomain', + + # POP3 + 'IMailboxPOP3', 'IServerFactoryPOP3', + + # Authentication + 'IClientAuthentication', +] diff --git a/contrib/python/Twisted/py2/twisted/mail/mail.py b/contrib/python/Twisted/py2/twisted/mail/mail.py new file mode 100644 index 00000000000..a1f041f544c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/mail.py @@ -0,0 +1,749 @@ +# -*- test-case-name: twisted.mail.test.test_mail -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Mail service support. +""" + +import warnings + +# Twisted imports +from twisted.internet import defer +from twisted.application import service, internet +from twisted.python import util +from twisted.python import log +from twisted.mail.interfaces import IAliasableDomain, IDomain +from twisted.cred.portal import Portal + +# Sibling imports +from twisted.mail import protocols, smtp + +# System imports +import os +from zope.interface import implementer + + +class DomainWithDefaultDict: + """ + A simulated dictionary for mapping domain names to domain objects with + a default value for non-existing keys. + + @ivar domains: See L{__init__} + @ivar default: See L{__init__} + """ + def __init__(self, domains, default): + """ + @type domains: L{dict} of L{bytes} -> L{IDomain} provider + @param domains: A mapping of domain name to domain object. + + @type default: L{IDomain} provider + @param default: The default domain. + """ + self.domains = domains + self.default = default + + + def setDefaultDomain(self, domain): + """ + Set the default domain. + + @type domain: L{IDomain} provider + @param domain: The default domain. + """ + self.default = domain + + + def has_key(self, name): + """ + Test for the presence of a domain name in this dictionary. + + This always returns C{True} because a default value will be returned + if the name doesn't exist in this dictionary. + + @type name: L{bytes} + @param name: A domain name. + + @rtype: L{bool} + @return: C{True} to indicate that the domain name is in this + dictionary. + """ + warnings.warn( + 'twisted.mail.mail.DomainWithDefaultDict.has_key was deprecated ' + 'in Twisted 16.3.0. ' + 'Use the `in` keyword instead.', + category=DeprecationWarning, + stacklevel=2) + return 1 + + + def fromkeys(klass, keys, value=None): + """ + Create a new L{DomainWithDefaultDict} with the specified keys. + + @type keys: iterable of L{bytes} + @param keys: Domain names to serve as keys in the new dictionary. + + @type value: L{None} or L{IDomain} provider + @param value: A domain object to serve as the value for all new keys + in the dictionary. + + @rtype: L{DomainWithDefaultDict} + @return: A new dictionary. + """ + d = klass() + for k in keys: + d[k] = value + return d + fromkeys = classmethod(fromkeys) + + + def __contains__(self, name): + """ + Test for the presence of a domain name in this dictionary. + + This always returns C{True} because a default value will be returned + if the name doesn't exist in this dictionary. + + @type name: L{bytes} + @param name: A domain name. + + @rtype: L{bool} + @return: C{True} to indicate that the domain name is in this + dictionary. + """ + return 1 + + + def __getitem__(self, name): + """ + Look up a domain name and, if it is present, return the domain object + associated with it. Otherwise return the default domain. + + @type name: L{bytes} + @param name: A domain name. + + @rtype: L{IDomain} provider or L{None} + @return: A domain object. + """ + return self.domains.get(name, self.default) + + + def __setitem__(self, name, value): + """ + Associate a domain object with a domain name in this dictionary. + + @type name: L{bytes} + @param name: A domain name. + + @type value: L{IDomain} provider + @param value: A domain object. + """ + self.domains[name] = value + + + def __delitem__(self, name): + """ + Delete the entry for a domain name in this dictionary. + + @type name: L{bytes} + @param name: A domain name. + """ + del self.domains[name] + + + def __iter__(self): + """ + Return an iterator over the domain names in this dictionary. + + @rtype: iterator over L{bytes} + @return: An iterator over the domain names. + """ + return iter(self.domains) + + + def __len__(self): + """ + Return the number of domains in this dictionary. + + @rtype: L{int} + @return: The number of domains in this dictionary. + """ + return len(self.domains) + + + def __str__(self): + """ + Build an informal string representation of this dictionary. + + @rtype: L{bytes} + @return: A string containing the mapping of domain names to domain + objects. + """ + return '' % (self.domains,) + + + def __repr__(self): + """ + Build an "official" string representation of this dictionary. + + @rtype: L{bytes} + @return: A pseudo-executable string describing the underlying domain + mapping of this object. + """ + return 'DomainWithDefaultDict(%s)' % (self.domains,) + + + def get(self, key, default=None): + """ + Look up a domain name in this dictionary. + + @type key: L{bytes} + @param key: A domain name. + + @type default: L{IDomain} provider or L{None} + @param default: A domain object to be returned if the domain name is + not in this dictionary. + + @rtype: L{IDomain} provider or L{None} + @return: The domain object associated with the domain name if it is in + this dictionary. Otherwise, the default value. + """ + return self.domains.get(key, default) + + + def copy(self): + """ + Make a copy of this dictionary. + + @rtype: L{DomainWithDefaultDict} + @return: A copy of this dictionary. + """ + return DomainWithDefaultDict(self.domains.copy(), self.default) + + + def iteritems(self): + """ + Return an iterator over the domain name/domain object pairs in the + dictionary. + + Using the returned iterator while adding or deleting entries from the + dictionary may result in a L{RuntimeError } or + failing to iterate over all the domain name/domain object pairs. + + @rtype: iterator over 2-L{tuple} of (E{1}) L{bytes}, + (E{2}) L{IDomain} provider or L{None} + @return: An iterator over the domain name/domain object pairs. + """ + return self.domains.iteritems() + + + def iterkeys(self): + """ + Return an iterator over the domain names in this dictionary. + + Using the returned iterator while adding or deleting entries from the + dictionary may result in a L{RuntimeError } or + failing to iterate over all the domain names. + + @rtype: iterator over L{bytes} + @return: An iterator over the domain names. + """ + return self.domains.iterkeys() + + + def itervalues(self): + """ + Return an iterator over the domain objects in this dictionary. + + Using the returned iterator while adding or deleting entries from the + dictionary may result in a L{RuntimeError } + or failing to iterate over all the domain objects. + + @rtype: iterator over L{IDomain} provider or + L{None} + @return: An iterator over the domain objects. + """ + return self.domains.itervalues() + + + def keys(self): + """ + Return a list of all domain names in this dictionary. + + @rtype: L{list} of L{bytes} + @return: The domain names in this dictionary. + + """ + return self.domains.keys() + + + def values(self): + """ + Return a list of all domain objects in this dictionary. + + @rtype: L{list} of L{IDomain} provider or L{None} + @return: The domain objects in this dictionary. + """ + return self.domains.values() + + + def items(self): + """ + Return a list of all domain name/domain object pairs in this + dictionary. + + @rtype: L{list} of 2-L{tuple} of (E{1}) L{bytes}, (E{2}) L{IDomain} + provider or L{None} + @return: Domain name/domain object pairs in this dictionary. + """ + return self.domains.items() + + + def popitem(self): + """ + Remove a random domain name/domain object pair from this dictionary and + return it as a tuple. + + @rtype: 2-L{tuple} of (E{1}) L{bytes}, (E{2}) L{IDomain} provider or + L{None} + @return: A domain name/domain object pair. + + @raise KeyError: When this dictionary is empty. + """ + return self.domains.popitem() + + + def update(self, other): + """ + Update this dictionary with domain name/domain object pairs from + another dictionary. + + When this dictionary contains a domain name which is in the other + dictionary, its value will be overwritten. + + @type other: L{dict} of L{bytes} -> L{IDomain} provider and/or + L{bytes} -> L{None} + @param other: Another dictionary of domain name/domain object pairs. + + @rtype: L{None} + @return: None. + """ + return self.domains.update(other) + + + def clear(self): + """ + Remove all items from this dictionary. + + @rtype: L{None} + @return: None. + """ + return self.domains.clear() + + + def setdefault(self, key, default): + """ + Return the domain object associated with the domain name if it is + present in this dictionary. Otherwise, set the value for the + domain name to the default and return that value. + + @type key: L{bytes} + @param key: A domain name. + + @type default: L{IDomain} provider + @param default: A domain object. + + @rtype: L{IDomain} provider or L{None} + @return: The domain object associated with the domain name. + """ + return self.domains.setdefault(key, default) + + + + + + +@implementer(IDomain) +class BounceDomain: + """ + A domain with no users. + + This can be used to block off a domain. + """ + def exists(self, user): + """ + Raise an exception to indicate that the user does not exist in this + domain. + + @type user: L{User} + @param user: A user. + + @raise SMTPBadRcpt: When the given user does not exist in this domain. + """ + raise smtp.SMTPBadRcpt(user) + + + def willRelay(self, user, protocol): + """ + Indicate that this domain will not relay. + + @type user: L{Address} + @param user: The destination address. + + @type protocol: L{Protocol } + @param protocol: The protocol over which the message to be relayed is + being received. + + @rtype: L{bool} + @return: C{False}. + """ + return False + + + def addUser(self, user, password): + """ + Ignore attempts to add a user to this domain. + + @type user: L{bytes} + @param user: A username. + + @type password: L{bytes} + @param password: A password. + """ + pass + + + def getCredentialsCheckers(self): + """ + Return no credentials checkers for this domain. + + @rtype: L{list} + @return: The empty list. + """ + return [] + + + +@implementer(smtp.IMessage) +class FileMessage: + """ + A message receiver which delivers a message to a file. + + @ivar fp: See L{__init__}. + @ivar name: See L{__init__}. + @ivar finalName: See L{__init__}. + """ + def __init__(self, fp, name, finalName): + """ + @type fp: file-like object + @param fp: The file in which to store the message while it is being + received. + + @type name: L{bytes} + @param name: The full path name of the temporary file. + + @type finalName: L{bytes} + @param finalName: The full path name that should be given to the file + holding the message after it has been fully received. + """ + self.fp = fp + self.name = name + self.finalName = finalName + + + def lineReceived(self, line): + """ + Write a received line to the file. + + @type line: L{bytes} + @param line: A received line. + """ + self.fp.write(line+'\n') + + + def eomReceived(self): + """ + At the end of message, rename the file holding the message to its + final name. + + @rtype: L{Deferred} which successfully results in L{bytes} + @return: A deferred which returns the final name of the file. + """ + self.fp.close() + os.rename(self.name, self.finalName) + return defer.succeed(self.finalName) + + + def connectionLost(self): + """ + Delete the file holding the partially received message. + """ + self.fp.close() + os.remove(self.name) + + + +class MailService(service.MultiService): + """ + An email service. + + @type queue: L{Queue} or L{None} + @ivar queue: A queue for outgoing messages. + + @type domains: L{dict} of L{bytes} -> L{IDomain} provider + @ivar domains: A mapping of supported domain name to domain object. + + @type portals: L{dict} of L{bytes} -> L{Portal} + @ivar portals: A mapping of domain name to authentication portal. + + @type aliases: L{None} or L{dict} of + L{bytes} -> L{IAlias} provider + @ivar aliases: A mapping of domain name to alias. + + @type smtpPortal: L{Portal} + @ivar smtpPortal: A portal for authentication for the SMTP server. + + @type monitor: L{FileMonitoringService} + @ivar monitor: A service to monitor changes to files. + """ + queue = None + domains = None + portals = None + aliases = None + smtpPortal = None + + def __init__(self): + """ + Initialize the mail service. + """ + service.MultiService.__init__(self) + # Domains and portals for "client" protocols - POP3, IMAP4, etc + self.domains = DomainWithDefaultDict({}, BounceDomain()) + self.portals = {} + + self.monitor = FileMonitoringService() + self.monitor.setServiceParent(self) + self.smtpPortal = Portal(self) + + + def getPOP3Factory(self): + """ + Create a POP3 protocol factory. + + @rtype: L{POP3Factory} + @return: A POP3 protocol factory. + """ + return protocols.POP3Factory(self) + + + def getSMTPFactory(self): + """ + Create an SMTP protocol factory. + + @rtype: L{SMTPFactory } + @return: An SMTP protocol factory. + """ + return protocols.SMTPFactory(self, self.smtpPortal) + + + def getESMTPFactory(self): + """ + Create an ESMTP protocol factory. + + @rtype: L{ESMTPFactory } + @return: An ESMTP protocol factory. + """ + return protocols.ESMTPFactory(self, self.smtpPortal) + + + def addDomain(self, name, domain): + """ + Add a domain for which the service will accept email. + + @type name: L{bytes} + @param name: A domain name. + + @type domain: L{IDomain} provider + @param domain: A domain object. + """ + portal = Portal(domain) + map(portal.registerChecker, domain.getCredentialsCheckers()) + self.domains[name] = domain + self.portals[name] = portal + if self.aliases and IAliasableDomain.providedBy(domain): + domain.setAliasGroup(self.aliases) + + + def setQueue(self, queue): + """ + Set the queue for outgoing emails. + + @type queue: L{Queue} + @param queue: A queue for outgoing messages. + """ + self.queue = queue + + + def requestAvatar(self, avatarId, mind, *interfaces): + """ + Return a message delivery for an authenticated SMTP user. + + @type avatarId: L{bytes} + @param avatarId: A string which identifies an authenticated user. + + @type mind: L{None} + @param mind: Unused. + + @type interfaces: n-L{tuple} of C{zope.interface.Interface} + @param interfaces: A group of interfaces one of which the avatar must + support. + + @rtype: 3-L{tuple} of (E{1}) L{IMessageDelivery}, + (E{2}) L{ESMTPDomainDelivery}, (E{3}) no-argument callable + @return: A tuple of the supported interface, a message delivery, and + a logout function. + + @raise NotImplementedError: When the given interfaces do not include + L{IMessageDelivery}. + """ + if smtp.IMessageDelivery in interfaces: + a = protocols.ESMTPDomainDelivery(self, avatarId) + return smtp.IMessageDelivery, a, lambda: None + raise NotImplementedError() + + + def lookupPortal(self, name): + """ + Find the portal for a domain. + + @type name: L{bytes} + @param name: A domain name. + + @rtype: L{Portal} + @return: A portal. + """ + return self.portals[name] + + + def defaultPortal(self): + """ + Return the portal for the default domain. + + The default domain is named ''. + + @rtype: L{Portal} + @return: The portal for the default domain. + """ + return self.portals[''] + + + +class FileMonitoringService(internet.TimerService): + """ + A service for monitoring changes to files. + + @type files: L{list} of L{list} of (E{1}) L{float}, (E{2}) L{bytes}, + (E{3}) callable which takes a L{bytes} argument, (E{4}) L{float} + @ivar files: Information about files to be monitored. Each list entry + provides the following information for a file: interval in seconds + between checks, filename, callback function, time of last modification + to the file. + + @type intervals: L{_IntervalDifferentialIterator + } + @ivar intervals: Intervals between successive file checks. + + @type _call: L{IDelayedCall } + provider + @ivar _call: The next scheduled call to check a file. + + @type index: L{int} + @ivar index: The index of the next file to be checked. + """ + def __init__(self): + """ + Initialize the file monitoring service. + """ + self.files = [] + self.intervals = iter(util.IntervalDifferential([], 60)) + + + def startService(self): + """ + Start the file monitoring service. + """ + service.Service.startService(self) + self._setupMonitor() + + + def _setupMonitor(self): + """ + Schedule the next monitoring call. + """ + from twisted.internet import reactor + t, self.index = self.intervals.next() + self._call = reactor.callLater(t, self._monitor) + + + def stopService(self): + """ + Stop the file monitoring service. + """ + service.Service.stopService(self) + if self._call: + self._call.cancel() + self._call = None + + + def monitorFile(self, name, callback, interval=10): + """ + Start monitoring a file for changes. + + @type name: L{bytes} + @param name: The name of a file to monitor. + + @type callback: callable which takes a L{bytes} argument + @param callback: The function to call when the file has changed. + + @type interval: L{float} + @param interval: The interval in seconds between checks. + """ + try: + mtime = os.path.getmtime(name) + except: + mtime = 0 + self.files.append([interval, name, callback, mtime]) + self.intervals.addInterval(interval) + + + def unmonitorFile(self, name): + """ + Stop monitoring a file. + + @type name: L{bytes} + @param name: A file name. + """ + for i in range(len(self.files)): + if name == self.files[i][1]: + self.intervals.removeInterval(self.files[i][0]) + del self.files[i] + break + + + def _monitor(self): + """ + Monitor a file and make a callback if it has changed. + """ + self._call = None + if self.index is not None: + name, callback, mtime = self.files[self.index][1:] + try: + now = os.path.getmtime(name) + except: + now = 0 + if now > mtime: + log.msg("%s changed, notifying listener" % (name,)) + self.files[self.index][3] = now + callback(name) + self._setupMonitor() diff --git a/contrib/python/Twisted/py2/twisted/mail/maildir.py b/contrib/python/Twisted/py2/twisted/mail/maildir.py new file mode 100644 index 00000000000..034074d0a97 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/maildir.py @@ -0,0 +1,944 @@ +# -*- test-case-name: twisted.mail.test.test_mail -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Maildir-style mailbox support. +""" + +import os +import stat +import socket +from hashlib import md5 + +from zope.interface import implementer + +try: + import cStringIO as StringIO +except ImportError: + import StringIO + +from twisted.mail import pop3 +from twisted.mail import smtp +from twisted.protocols import basic +from twisted.persisted import dirdbm +from twisted.python import log, failure +from twisted.mail import mail +from twisted.internet import interfaces, defer, reactor +from twisted.cred import portal, credentials, checkers +from twisted.cred.error import UnauthorizedLogin + +INTERNAL_ERROR = '''\ +From: Twisted.mail Internals +Subject: An Error Occurred + + An internal server error has occurred. Please contact the + server administrator. +''' + + + +class _MaildirNameGenerator: + """ + A utility class to generate a unique maildir name. + + @type n: L{int} + @ivar n: A counter used to generate unique integers. + + @type p: L{int} + @ivar p: The ID of the current process. + + @type s: L{bytes} + @ivar s: A representation of the hostname. + + @ivar _clock: See C{clock} parameter of L{__init__}. + """ + n = 0 + p = os.getpid() + s = socket.gethostname().replace('/', r'\057').replace(':', r'\072') + + def __init__(self, clock): + """ + @type clock: L{IReactorTime } provider + @param clock: A reactor which will be used to learn the current time. + """ + self._clock = clock + + + def generate(self): + """ + Generate a string which is intended to be unique across all calls to + this function (across all processes, reboots, etc). + + Strings returned by earlier calls to this method will compare less + than strings returned by later calls as long as the clock provided + doesn't go backwards. + + @rtype: L{bytes} + @return: A unique string. + """ + self.n = self.n + 1 + t = self._clock.seconds() + seconds = str(int(t)) + microseconds = '%07d' % (int((t - int(t)) * 10e6),) + return '%s.M%sP%sQ%s.%s' % (seconds, microseconds, + self.p, self.n, self.s) + +_generateMaildirName = _MaildirNameGenerator(reactor).generate + + + +def initializeMaildir(dir): + """ + Create a maildir user directory if it doesn't already exist. + + @type dir: L{bytes} + @param dir: The path name for a user directory. + """ + if not os.path.isdir(dir): + os.mkdir(dir, 0o700) + for subdir in ['new', 'cur', 'tmp', '.Trash']: + os.mkdir(os.path.join(dir, subdir), 0o700) + for subdir in ['new', 'cur', 'tmp']: + os.mkdir(os.path.join(dir, '.Trash', subdir), 0o700) + # touch + open(os.path.join(dir, '.Trash', 'maildirfolder'), 'w').close() + + + +class MaildirMessage(mail.FileMessage): + """ + A message receiver which adds a header and delivers a message to a file + whose name includes the size of the message. + + @type size: L{int} + @ivar size: The number of octets in the message. + """ + size = None + + def __init__(self, address, fp, *a, **kw): + """ + @type address: L{bytes} + @param address: The address of the message recipient. + + @type fp: file-like object + @param fp: The file in which to store the message while it is being + received. + + @type a: 2-L{tuple} of (0) L{bytes}, (1) L{bytes} + @param a: Positional arguments for L{FileMessage.__init__}. + + @type kw: L{dict} + @param kw: Keyword arguments for L{FileMessage.__init__}. + """ + header = "Delivered-To: %s\n" % address + fp.write(header) + self.size = len(header) + mail.FileMessage.__init__(self, fp, *a, **kw) + + + def lineReceived(self, line): + """ + Write a line to the file. + + @type line: L{bytes} + @param line: A received line. + """ + mail.FileMessage.lineReceived(self, line) + self.size += len(line)+1 + + + def eomReceived(self): + """ + At the end of message, rename the file holding the message to its final + name concatenated with the size of the file. + + @rtype: L{Deferred } which successfully results in + L{bytes} + @return: A deferred which returns the name of the file holding the + message. + """ + self.finalName = self.finalName+',S=%d' % self.size + return mail.FileMessage.eomReceived(self) + + + +@implementer(mail.IAliasableDomain) +class AbstractMaildirDomain: + """ + An abstract maildir-backed domain. + + @type alias: L{None} or L{dict} mapping + L{bytes} to L{AliasBase} + @ivar alias: A mapping of username to alias. + + @ivar root: See L{__init__}. + """ + alias = None + root = None + + def __init__(self, service, root): + """ + @type service: L{MailService} + @param service: An email service. + + @type root: L{bytes} + @param root: The maildir root directory. + """ + self.root = root + + + def userDirectory(self, user): + """ + Return the maildir directory for a user. + + @type user: L{bytes} + @param user: A username. + + @rtype: L{bytes} or L{None} + @return: The user's mail directory for a valid user. Otherwise, + L{None}. + """ + return None + + + def setAliasGroup(self, alias): + """ + Set the group of defined aliases for this domain. + + @type alias: L{dict} mapping L{bytes} to L{IAlias} provider. + @param alias: A mapping of domain name to alias. + """ + self.alias = alias + + + def exists(self, user, memo=None): + """ + Check whether a user exists in this domain or an alias of it. + + @type user: L{User} + @param user: A user. + + @type memo: L{None} or L{dict} of L{AliasBase} + @param memo: A record of the addresses already considered while + resolving aliases. The default value should be used by all + external code. + + @rtype: no-argument callable which returns L{IMessage } + provider. + @return: A function which takes no arguments and returns a message + receiver for the user. + + @raises SMTPBadRcpt: When the given user does not exist in this domain + or an alias of it. + """ + if self.userDirectory(user.dest.local) is not None: + return lambda: self.startMessage(user) + try: + a = self.alias[user.dest.local] + except: + raise smtp.SMTPBadRcpt(user) + else: + aliases = a.resolve(self.alias, memo) + if aliases: + return lambda: aliases + log.err("Bad alias configuration: " + str(user)) + raise smtp.SMTPBadRcpt(user) + + + def startMessage(self, user): + """ + Create a maildir message for a user. + + @type user: L{bytes} + @param user: A username. + + @rtype: L{MaildirMessage} + @return: A message receiver for this user. + """ + if isinstance(user, str): + name, domain = user.split('@', 1) + else: + name, domain = user.dest.local, user.dest.domain + dir = self.userDirectory(name) + fname = _generateMaildirName() + filename = os.path.join(dir, 'tmp', fname) + fp = open(filename, 'w') + return MaildirMessage('%s@%s' % (name, domain), fp, filename, + os.path.join(dir, 'new', fname)) + + + def willRelay(self, user, protocol): + """ + Check whether this domain will relay. + + @type user: L{Address} + @param user: The destination address. + + @type protocol: L{SMTP} + @param protocol: The protocol over which the message to be relayed is + being received. + + @rtype: L{bool} + @return: An indication of whether this domain will relay the message to + the destination. + """ + return False + + + def addUser(self, user, password): + """ + Add a user to this domain. + + Subclasses should override this method. + + @type user: L{bytes} + @param user: A username. + + @type password: L{bytes} + @param password: A password. + """ + raise NotImplementedError + + + def getCredentialsCheckers(self): + """ + Return credentials checkers for this domain. + + Subclasses should override this method. + + @rtype: L{list} of L{ICredentialsChecker + } provider + @return: Credentials checkers for this domain. + """ + raise NotImplementedError + + + +@implementer(interfaces.IConsumer) +class _MaildirMailboxAppendMessageTask: + """ + A task which adds a message to a maildir mailbox. + + @ivar mbox: See L{__init__}. + + @type defer: L{Deferred } which successfully returns + L{None} + @ivar defer: A deferred which fires when the task has completed. + + @type opencall: L{IDelayedCall } provider or + L{None} + @ivar opencall: A scheduled call to L{prodProducer}. + + @type msg: file-like object + @ivar msg: The message to add. + + @type tmpname: L{bytes} + @ivar tmpname: The pathname of the temporary file holding the message while + it is being transferred. + + @type fh: file + @ivar fh: The new maildir file. + + @type filesender: L{FileSender } + @ivar filesender: A file sender which sends the message. + + @type myproducer: L{IProducer } + @ivar myproducer: The registered producer. + + @type streaming: L{bool} + @ivar streaming: Indicates whether the registered producer provides a + streaming interface. + """ + osopen = staticmethod(os.open) + oswrite = staticmethod(os.write) + osclose = staticmethod(os.close) + osrename = staticmethod(os.rename) + + def __init__(self, mbox, msg): + """ + @type mbox: L{MaildirMailbox} + @param mbox: A maildir mailbox. + + @type msg: L{bytes} or file-like object + @param msg: The message to add. + """ + self.mbox = mbox + self.defer = defer.Deferred() + self.openCall = None + if not hasattr(msg, "read"): + msg = StringIO.StringIO(msg) + self.msg = msg + + + def startUp(self): + """ + Start transferring the message to the mailbox. + """ + self.createTempFile() + if self.fh != -1: + self.filesender = basic.FileSender() + self.filesender.beginFileTransfer(self.msg, self) + + + def registerProducer(self, producer, streaming): + """ + Register a producer and start asking it for data if it is + non-streaming. + + @type producer: L{IProducer } + @param producer: A producer. + + @type streaming: L{bool} + @param streaming: A flag indicating whether the producer provides a + streaming interface. + """ + self.myproducer = producer + self.streaming = streaming + if not streaming: + self.prodProducer() + + + def prodProducer(self): + """ + Repeatedly prod a non-streaming producer to produce data. + """ + self.openCall = None + if self.myproducer is not None: + self.openCall = reactor.callLater(0, self.prodProducer) + self.myproducer.resumeProducing() + + + def unregisterProducer(self): + """ + Finish transferring the message to the mailbox. + """ + self.myproducer = None + self.streaming = None + self.osclose(self.fh) + self.moveFileToNew() + + + def write(self, data): + """ + Write data to the maildir file. + + @type data: L{bytes} + @param data: Data to be written to the file. + """ + try: + self.oswrite(self.fh, data) + except: + self.fail() + + + def fail(self, err=None): + """ + Fire the deferred to indicate the task completed with a failure. + + @type err: L{Failure } + @param err: The error that occurred. + """ + if err is None: + err = failure.Failure() + if self.openCall is not None: + self.openCall.cancel() + self.defer.errback(err) + self.defer = None + + + def moveFileToNew(self): + """ + Place the message in the I{new/} directory, add it to the mailbox and + fire the deferred to indicate that the task has completed + successfully. + """ + while True: + newname = os.path.join(self.mbox.path, "new", _generateMaildirName()) + try: + self.osrename(self.tmpname, newname) + break + except OSError as e: + (err, estr) = e.args + import errno + # if the newname exists, retry with a new newname. + if err != errno.EEXIST: + self.fail() + newname = None + break + if newname is not None: + self.mbox.list.append(newname) + self.defer.callback(None) + self.defer = None + + + def createTempFile(self): + """ + Create a temporary file to hold the message as it is being transferred. + """ + attr = (os.O_RDWR | os.O_CREAT | os.O_EXCL + | getattr(os, "O_NOINHERIT", 0) + | getattr(os, "O_NOFOLLOW", 0)) + tries = 0 + self.fh = -1 + while True: + self.tmpname = os.path.join(self.mbox.path, "tmp", _generateMaildirName()) + try: + self.fh = self.osopen(self.tmpname, attr, 0o600) + return None + except OSError: + tries += 1 + if tries > 500: + self.defer.errback(RuntimeError("Could not create tmp file for %s" % self.mbox.path)) + self.defer = None + return None + + + +class MaildirMailbox(pop3.Mailbox): + """ + A maildir-backed mailbox. + + @ivar path: See L{__init__}. + + @type list: L{list} of L{int} or 2-L{tuple} of (0) file-like object, + (1) L{bytes} + @ivar list: Information about the messages in the mailbox. For undeleted + messages, the file containing the message and the + full path name of the file are stored. Deleted messages are indicated + by 0. + + @type deleted: L{dict} mapping 2-L{tuple} of (0) file-like object, + (1) L{bytes} to L{bytes} + @type deleted: A mapping of the information about a file before it was + deleted to the full path name of the deleted file in the I{.Trash/} + subfolder. + """ + AppendFactory = _MaildirMailboxAppendMessageTask + + def __init__(self, path): + """ + @type path: L{bytes} + @param path: The directory name for a maildir mailbox. + """ + self.path = path + self.list = [] + self.deleted = {} + initializeMaildir(path) + for name in ('cur', 'new'): + for file in os.listdir(os.path.join(path, name)): + self.list.append((file, os.path.join(path, name, file))) + self.list.sort() + self.list = [e[1] for e in self.list] + + + def listMessages(self, i=None): + """ + Retrieve the size of a message, or, if none is specified, the size of + each message in the mailbox. + + @type i: L{int} or L{None} + @param i: The 0-based index of a message. + + @rtype: L{int} or L{list} of L{int} + @return: The number of octets in the specified message, or, if an index + is not specified, a list of the number of octets for all messages + in the mailbox. Any value which corresponds to a deleted message + is set to 0. + + @raise IndexError: When the index does not correspond to a message in + the mailbox. + """ + if i is None: + ret = [] + for mess in self.list: + if mess: + ret.append(os.stat(mess)[stat.ST_SIZE]) + else: + ret.append(0) + return ret + return self.list[i] and os.stat(self.list[i])[stat.ST_SIZE] or 0 + + + def getMessage(self, i): + """ + Retrieve a file-like object with the contents of a message. + + @type i: L{int} + @param i: The 0-based index of a message. + + @rtype: file-like object + @return: A file containing the message. + + @raise IndexError: When the index does not correspond to a message in + the mailbox. + """ + return open(self.list[i]) + + + def getUidl(self, i): + """ + Get a unique identifier for a message. + + @type i: L{int} + @param i: The 0-based index of a message. + + @rtype: L{bytes} + @return: A string of printable characters uniquely identifying the + message for all time. + + @raise IndexError: When the index does not correspond to a message in + the mailbox. + """ + # Returning the actual filename is a mistake. Hash it. + base = os.path.basename(self.list[i]) + return md5(base).hexdigest() + + + def deleteMessage(self, i): + """ + Mark a message for deletion. + + Move the message to the I{.Trash/} subfolder so it can be undeleted + by an administrator. + + @type i: L{int} + @param i: The 0-based index of a message. + + @raise IndexError: When the index does not correspond to a message in + the mailbox. + """ + trashFile = os.path.join( + self.path, '.Trash', 'cur', os.path.basename(self.list[i]) + ) + os.rename(self.list[i], trashFile) + self.deleted[self.list[i]] = trashFile + self.list[i] = 0 + + + def undeleteMessages(self): + """ + Undelete all messages marked for deletion. + + Move each message marked for deletion from the I{.Trash/} subfolder back + to its original position. + """ + for (real, trash) in self.deleted.items(): + try: + os.rename(trash, real) + except OSError as e: + (err, estr) = e.args + import errno + # If the file has been deleted from disk, oh well! + if err != errno.ENOENT: + raise + # This is a pass + else: + try: + self.list[self.list.index(0)] = real + except ValueError: + self.list.append(real) + self.deleted.clear() + + + def appendMessage(self, txt): + """ + Add a message to the mailbox. + + @type txt: L{bytes} or file-like object + @param txt: A message to add. + + @rtype: L{Deferred } + @return: A deferred which fires when the message has been added to + the mailbox. + """ + task = self.AppendFactory(self, txt) + result = task.defer + task.startUp() + return result + + + +@implementer(pop3.IMailbox) +class StringListMailbox: + """ + An in-memory mailbox. + + @ivar msgs: See L{__init__}. + + @type _delete: L{set} of L{int} + @ivar _delete: The indices of messages which have been marked for deletion. + """ + def __init__(self, msgs): + """ + @type msgs: L{list} of L{bytes} + @param msgs: The contents of each message in the mailbox. + """ + self.msgs = msgs + self._delete = set() + + + def listMessages(self, i=None): + """ + Retrieve the size of a message, or, if none is specified, the size of + each message in the mailbox. + + @type i: L{int} or L{None} + @param i: The 0-based index of a message. + + @rtype: L{int} or L{list} of L{int} + @return: The number of octets in the specified message, or, if an index + is not specified, a list of the number of octets in each message in + the mailbox. Any value which corresponds to a deleted message is + set to 0. + + @raise IndexError: When the index does not correspond to a message in + the mailbox. + """ + if i is None: + return [self.listMessages(msg) for msg in range(len(self.msgs))] + if i in self._delete: + return 0 + return len(self.msgs[i]) + + + def getMessage(self, i): + """ + Return an in-memory file-like object with the contents of a message. + + @type i: L{int} + @param i: The 0-based index of a message. + + @rtype: L{StringIO } + @return: An in-memory file-like object containing the message. + + @raise IndexError: When the index does not correspond to a message in + the mailbox. + """ + return StringIO.StringIO(self.msgs[i]) + + + def getUidl(self, i): + """ + Get a unique identifier for a message. + + @type i: L{int} + @param i: The 0-based index of a message. + + @rtype: L{bytes} + @return: A hash of the contents of the message at the given index. + + @raise IndexError: When the index does not correspond to a message in + the mailbox. + """ + return md5(self.msgs[i]).hexdigest() + + + def deleteMessage(self, i): + """ + Mark a message for deletion. + + @type i: L{int} + @param i: The 0-based index of a message to delete. + + @raise IndexError: When the index does not correspond to a message in + the mailbox. + """ + self._delete.add(i) + + + def undeleteMessages(self): + """ + Undelete any messages which have been marked for deletion. + """ + self._delete = set() + + + def sync(self): + """ + Discard the contents of any messages marked for deletion. + """ + for index in self._delete: + self.msgs[index] = "" + self._delete = set() + + + +@implementer(portal.IRealm) +class MaildirDirdbmDomain(AbstractMaildirDomain): + """ + A maildir-backed domain where membership is checked with a + L{DirDBM } database. + + The directory structure of a MaildirDirdbmDomain is: + + /passwd <-- a DirDBM directory + + /USER/{cur, new, del} <-- each user has these three directories + + @ivar postmaster: See L{__init__}. + + @type dbm: L{DirDBM } + @ivar dbm: The authentication database for the domain. + """ + portal = None + _credcheckers = None + + def __init__(self, service, root, postmaster=0): + """ + @type service: L{MailService} + @param service: An email service. + + @type root: L{bytes} + @param root: The maildir root directory. + + @type postmaster: L{bool} + @param postmaster: A flag indicating whether non-existent addresses + should be forwarded to the postmaster (C{True}) or + bounced (C{False}). + """ + AbstractMaildirDomain.__init__(self, service, root) + dbm = os.path.join(root, 'passwd') + if not os.path.exists(dbm): + os.makedirs(dbm) + self.dbm = dirdbm.open(dbm) + self.postmaster = postmaster + + + def userDirectory(self, name): + """ + Return the path to a user's mail directory. + + @type name: L{bytes} + @param name: A username. + + @rtype: L{bytes} or L{None} + @return: The path to the user's mail directory for a valid user. For + an invalid user, the path to the postmaster's mailbox if bounces + are redirected there. Otherwise, L{None}. + """ + if name not in self.dbm: + if not self.postmaster: + return None + name = 'postmaster' + dir = os.path.join(self.root, name) + if not os.path.exists(dir): + initializeMaildir(dir) + return dir + + + def addUser(self, user, password): + """ + Add a user to this domain by adding an entry in the authentication + database and initializing the user's mail directory. + + @type user: L{bytes} + @param user: A username. + + @type password: L{bytes} + @param password: A password. + """ + self.dbm[user] = password + # Ensure it is initialized + self.userDirectory(user) + + + def getCredentialsCheckers(self): + """ + Return credentials checkers for this domain. + + @rtype: L{list} of L{ICredentialsChecker + } provider + @return: Credentials checkers for this domain. + """ + if self._credcheckers is None: + self._credcheckers = [DirdbmDatabase(self.dbm)] + return self._credcheckers + + + def requestAvatar(self, avatarId, mind, *interfaces): + """ + Get the mailbox for an authenticated user. + + The mailbox for the authenticated user will be returned only if the + given interfaces include L{IMailbox }. Requests for + anonymous access will be met with a mailbox containing a message + indicating that an internal error has occurred. + + @type avatarId: L{bytes} or C{twisted.cred.checkers.ANONYMOUS} + @param avatarId: A string which identifies a user or an object which + signals a request for anonymous access. + + @type mind: L{None} + @param mind: Unused. + + @type interfaces: n-L{tuple} of C{zope.interface.Interface} + @param interfaces: A group of interfaces, one of which the avatar + must support. + + @rtype: 3-L{tuple} of (0) L{IMailbox }, + (1) L{IMailbox } provider, (2) no-argument + callable + @return: A tuple of the supported interface, a mailbox, and a + logout function. + + @raise NotImplementedError: When the given interfaces do not include + L{IMailbox }. + """ + if pop3.IMailbox not in interfaces: + raise NotImplementedError("No interface") + if avatarId == checkers.ANONYMOUS: + mbox = StringListMailbox([INTERNAL_ERROR]) + else: + mbox = MaildirMailbox(os.path.join(self.root, avatarId)) + + return ( + pop3.IMailbox, + mbox, + lambda: None + ) + + + +@implementer(checkers.ICredentialsChecker) +class DirdbmDatabase: + """ + A credentials checker which authenticates users out of a + L{DirDBM } database. + + @type dirdbm: L{DirDBM } + @ivar dirdbm: An authentication database. + """ + # credentialInterfaces is not used by the class + credentialInterfaces = ( + credentials.IUsernamePassword, + credentials.IUsernameHashedPassword + ) + + def __init__(self, dbm): + """ + @type dbm: L{DirDBM } + @param dbm: An authentication database. + """ + self.dirdbm = dbm + + + def requestAvatarId(self, c): + """ + Authenticate a user and, if successful, return their username. + + @type c: L{IUsernamePassword } or + L{IUsernameHashedPassword } + provider. + @param c: Credentials. + + @rtype: L{bytes} + @return: A string which identifies an user. + + @raise UnauthorizedLogin: When the credentials check fails. + """ + if c.username in self.dirdbm: + if c.checkPassword(self.dirdbm[c.username]): + return c.username + raise UnauthorizedLogin() diff --git a/contrib/python/Twisted/py2/twisted/mail/pb.py b/contrib/python/Twisted/py2/twisted/mail/pb.py new file mode 100644 index 00000000000..104d363ea53 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/pb.py @@ -0,0 +1,124 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +from twisted.spread import pb + +import os + + +class Maildir(pb.Referenceable): + + def __init__(self, directory, rootDirectory): + self.virtualDirectory = directory + self.rootDirectory = rootDirectory + self.directory = os.path.join(rootDirectory, directory) + + + def getFolderMessage(self, folder, name): + if '/' in name: + raise IOError("can only open files in '%s' directory'" % folder) + with open(os.path.join(self.directory, 'new', name)) as fp: + return fp.read() + + + def deleteFolderMessage(self, folder, name): + if '/' in name: + raise IOError("can only delete files in '%s' directory'" % folder) + os.rename(os.path.join(self.directory, folder, name), + os.path.join(self.rootDirectory, '.Trash', folder, name)) + + + def deleteNewMessage(self, name): + return self.deleteFolderMessage('new', name) + remote_deleteNewMessage = deleteNewMessage + + + def deleteCurMessage(self, name): + return self.deleteFolderMessage('cur', name) + remote_deleteCurMessage = deleteCurMessage + + + def getNewMessages(self): + return os.listdir(os.path.join(self.directory, 'new')) + remote_getNewMessages = getNewMessages + + + def getCurMessages(self): + return os.listdir(os.path.join(self.directory, 'cur')) + remote_getCurMessages = getCurMessages + + + def getNewMessage(self, name): + return self.getFolderMessage('new', name) + remote_getNewMessage = getNewMessage + + + def getCurMessage(self, name): + return self.getFolderMessage('cur', name) + remote_getCurMessage = getCurMessage + + + def getSubFolder(self, name): + if name[0] == '.': + raise IOError("subfolder name cannot begin with a '.'") + name = name.replace('/', ':') + if self.virtualDirectoy == '.': + name = '.'+name + else: + name = self.virtualDirectory+':'+name + if not self._isSubFolder(name): + raise IOError("not a subfolder") + return Maildir(name, self.rootDirectory) + remote_getSubFolder = getSubFolder + + + def _isSubFolder(self, name): + return (not os.path.isdir(os.path.join(self.rootDirectory, name)) or + not os.path.isfile(os.path.join(self.rootDirectory, name, + 'maildirfolder'))) + + + +class MaildirCollection(pb.Referenceable): + def __init__(self, root): + self.root = root + + + def getSubFolders(self): + return os.listdir(self.getRoot()) + remote_getSubFolders = getSubFolders + + + def getSubFolder(self, name): + if '/' in name or name[0] == '.': + raise IOError("invalid name") + return Maildir('.', os.path.join(self.getRoot(), name)) + remote_getSubFolder = getSubFolder + + + +class MaildirBroker(pb.Broker): + def proto_getCollection(self, requestID, name, domain, password): + collection = self._getCollection() + if collection is None: + self.sendError(requestID, "permission denied") + else: + self.sendAnswer(requestID, collection) + + + def getCollection(self, name, domain, password): + if domain not in self.domains: + return + domain = self.domains[domain] + if (name in domain.dbm and + domain.dbm[name] == password): + return MaildirCollection(domain.userDirectory(name)) + + + +class MaildirClient(pb.Broker): + def getCollection(self, name, domain, password, callback, errback): + requestID = self.newRequestID() + self.waitingForAnswers[requestID] = callback, errback + self.sendCall("getCollection", requestID, name, domain, password) diff --git a/contrib/python/Twisted/py2/twisted/mail/pop3.py b/contrib/python/Twisted/py2/twisted/mail/pop3.py new file mode 100644 index 00000000000..ffe9714c940 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/pop3.py @@ -0,0 +1,1749 @@ +# -*- test-case-name: twisted.mail.test.test_pop3 -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Post-office Protocol version 3. + +@author: Glyph Lefkowitz +@author: Jp Calderone +""" + +import base64 +import binascii +import warnings +from hashlib import md5 + +from zope.interface import implementer + +from twisted import cred +from twisted.internet import task +from twisted.internet import defer +from twisted.internet import interfaces +from twisted.mail import smtp +from twisted.mail.interfaces import ( + IServerFactoryPOP3 as IServerFactory, + IMailboxPOP3 as IMailbox, +) +from twisted.mail._except import ( + POP3Error, _POP3MessageDeleted, POP3ClientError +) +from twisted.protocols import basic +from twisted.protocols import policies +from twisted.python import log +from twisted.python.compat import _PY3, intToBytes + +# Authentication +@implementer(cred.credentials.IUsernamePassword) +class APOPCredentials: + """ + Credentials for use in APOP authentication. + + @ivar magic: See L{__init__} + @ivar username: See L{__init__} + @ivar digest: See L{__init__} + """ + def __init__(self, magic, username, digest): + """ + @type magic: L{bytes} + @param magic: The challenge string used to encrypt the password. + + @type username: L{bytes} + @param username: The username associated with these credentials. + + @type digest: L{bytes} + @param digest: An encrypted version of the user's password. Should be + generated as an MD5 hash of the challenge string concatenated with + the plaintext password. + """ + self.magic = magic + self.username = username + self.digest = digest + + + def checkPassword(self, password): + """ + Validate a plaintext password against the credentials. + + @type password: L{bytes} + @param password: A plaintext password. + + @rtype: L{bool} + @return: C{True} if the credentials represented by this object match + the given password, C{False} if they do not. + """ + seed = self.magic + password + myDigest = md5(seed).hexdigest() + return myDigest == self.digest + + + +class _HeadersPlusNLines: + """ + A utility class to retrieve the header and some lines of the body of a mail + message. + + @ivar _file: See L{__init__} + @ivar _extraLines: See L{__init__} + + @type linecount: L{int} + @ivar linecount: The number of full lines of the message body scanned. + + @type headers: L{bool} + @ivar headers: An indication of which part of the message is being scanned. + C{True} for the header and C{False} for the body. + + @type done: L{bool} + @ivar done: A flag indicating when the desired part of the message has been + scanned. + + @type buf: L{bytes} + @ivar buf: The portion of the message body that has been scanned, up to + C{n} lines. + """ + def __init__(self, file, extraLines): + """ + @type file: file-like object + @param file: A file containing a mail message. + + @type extraLines: L{int} + @param extraLines: The number of lines of the message body to retrieve. + """ + self._file = file + self._extraLines = extraLines + self.linecount = 0 + self.headers = 1 + self.done = 0 + self.buf = b'' + + + def read(self, bytes): + """ + Scan bytes from the file. + + @type bytes: L{int} + @param bytes: The number of bytes to read from the file. + + @rtype: L{bytes} + @return: Each portion of the header as it is scanned. Then, full lines + of the message body as they are scanned. When more than one line + of the header and/or body has been scanned, the result is the + concatenation of the lines. When the scan results in no full + lines, the empty string is returned. + """ + if self.done: + return b'' + data = self._file.read(bytes) + if not data: + return data + if self.headers: + df, sz = data.find(b'\r\n\r\n'), 4 + if df == -1: + df, sz = data.find(b'\n\n'), 2 + if df != -1: + df += sz + val = data[:df] + data = data[df:] + self.linecount = 1 + self.headers = 0 + else: + val = b'' + if self.linecount > 0: + dsplit = (self.buf + data).split(b'\n') + self.buf = dsplit[-1] + for ln in dsplit[:-1]: + if self.linecount > self._extraLines: + self.done = 1 + return val + val += (ln + b'\n') + self.linecount += 1 + return val + else: + return data + + + +class _IteratorBuffer(object): + """ + An iterator which buffers the elements of a container and periodically + passes them as input to a writer. + + @ivar write: See L{__init__}. + @ivar memoryBufferSize: See L{__init__}. + + @type bufSize: L{int} + @ivar bufSize: The number of bytes currently in the buffer. + + @type lines: L{list} of L{bytes} + @ivar lines: The buffer, which is a list of strings. + + @type iterator: iterator which yields L{bytes} + @ivar iterator: An iterator over a container of strings. + """ + bufSize = 0 + + def __init__(self, write, iterable, memoryBufferSize=None): + """ + @type write: callable that takes L{list} of L{bytes} + @param write: A writer which is a callable that takes a list of + strings. + + @type iterable: iterable which yields L{bytes} + @param iterable: An iterable container of strings. + + @type memoryBufferSize: L{int} or L{None} + @param memoryBufferSize: The number of bytes to buffer before flushing + the buffer to the writer. + """ + self.lines = [] + self.write = write + self.iterator = iter(iterable) + if memoryBufferSize is None: + memoryBufferSize = 2 ** 16 + self.memoryBufferSize = memoryBufferSize + + + def __iter__(self): + """ + Return an iterator. + + @rtype: iterator which yields L{bytes} + @return: An iterator over strings. + """ + return self + + + def __next__(self): + """ + Get the next string from the container, buffer it, and possibly send + the buffer to the writer. + + The contents of the buffer are written when it is full or when no + further values are available from the container. + + @raise StopIteration: When no further values are available from the + container. + """ + try: + v = next(self.iterator) + except StopIteration: + if self.lines: + self.write(self.lines) + # Drop some references, in case they're edges in a cycle. + del self.iterator, self.lines, self.write + raise + else: + if v is not None: + self.lines.append(v) + self.bufSize += len(v) + if self.bufSize > self.memoryBufferSize: + self.write(self.lines) + self.lines = [] + self.bufSize = 0 + + if not _PY3: + next = __next__ + + + +def iterateLineGenerator(proto, gen): + """ + Direct the output of an iterator to the transport of a protocol and arrange + for iteration to take place. + + @type proto: L{POP3} + @param proto: A POP3 server protocol. + + @type gen: iterator which yields L{bytes} + @param gen: An iterator over strings. + + @rtype: L{Deferred } + @return: A deferred which fires when the iterator finishes. + """ + coll = _IteratorBuffer(proto.transport.writeSequence, gen) + return proto.schedule(coll) + + + +def successResponse(response): + """ + Format an object as a positive response. + + @type response: stringifyable L{object} + @param response: An object with a string representation. + + @rtype: L{bytes} + @return: A positive POP3 response string. + """ + if not isinstance(response, bytes): + response = str(response).encode("utf-8") + return b'+OK ' + response + b'\r\n' + + + +def formatStatResponse(msgs): + """ + Format a list of message sizes into a STAT response. + + This generator function is intended to be used with + L{Cooperator }. + + @type msgs: L{list} of L{int} + @param msgs: A list of message sizes. + + @rtype: L{None} or L{bytes} + @return: Yields none until a result is available, then a string that is + suitable for use in a STAT response. The string consists of the number + of messages and the total size of the messages in octets. + """ + i = 0 + bytes = 0 + for size in msgs: + i += 1 + bytes += size + yield None + yield successResponse(intToBytes(i) + b' ' + intToBytes(bytes)) + + + +def formatListLines(msgs): + """ + Format a list of message sizes for use in a LIST response. + + @type msgs: L{list} of L{int} + @param msgs: A list of message sizes. + + @rtype: L{bytes} + @return: Yields a series of strings that are suitable for use as scan + listings in a LIST response. Each string consists of a message number + and its size in octets. + """ + i = 0 + for size in msgs: + i += 1 + yield intToBytes(i) + b' ' + intToBytes(size) + b'\r\n' + + + +def formatListResponse(msgs): + """ + Format a list of message sizes into a complete LIST response. + + This generator function is intended to be used with + L{Cooperator }. + + @type msgs: L{list} of L{int} + @param msgs: A list of message sizes. + + @rtype: L{bytes} + @return: Yields a series of strings which make up a complete LIST response. + """ + yield successResponse(intToBytes(len(msgs))) + for ele in formatListLines(msgs): + yield ele + yield b'.\r\n' + + + +def formatUIDListLines(msgs, getUidl): + """ + Format a list of message sizes for use in a UIDL response. + + @param msgs: See L{formatUIDListResponse} + @param getUidl: See L{formatUIDListResponse} + + @rtype: L{bytes} + @return: Yields a series of strings that are suitable for use as unique-id + listings in a UIDL response. Each string consists of a message number + and its unique id. + """ + for i, m in enumerate(msgs): + if m is not None: + uid = getUidl(i) + if not isinstance(uid, bytes): + uid = str(uid).encode("utf-8") + yield intToBytes(i + 1) + b' ' + uid + b'\r\n' + + + +def formatUIDListResponse(msgs, getUidl): + """ + Format a list of message sizes into a complete UIDL response. + + This generator function is intended to be used with + L{Cooperator }. + + @type msgs: L{list} of L{int} + @param msgs: A list of message sizes. + + @type getUidl: one-argument callable returning bytes + @param getUidl: A callable which takes a message index number and returns + the UID of the corresponding message in the mailbox. + + @rtype: L{bytes} + @return: Yields a series of strings which make up a complete UIDL response. + """ + yield successResponse('') + for ele in formatUIDListLines(msgs, getUidl): + yield ele + yield b'.\r\n' + + + +@implementer(interfaces.IProducer) +class POP3(basic.LineOnlyReceiver, policies.TimeoutMixin): + """ + A POP3 server protocol. + + @type portal: L{Portal} + @ivar portal: A portal for authentication. + + @type factory: L{IServerFactory} provider + @ivar factory: A server factory which provides an interface for querying + capabilities of the server. + + @type timeOut: L{int} + @ivar timeOut: The number of seconds to wait for a command from the client + before disconnecting. + + @type schedule: callable that takes interator and returns + L{Deferred } + @ivar schedule: A callable that arranges for an iterator to be + cooperatively iterated over along with all other iterators which have + been passed to it such that runtime is divided between all of them. It + returns a deferred which fires when the iterator finishes. + + @type magic: L{bytes} or L{None} + @ivar magic: An APOP challenge. If not set, an APOP challenge string + will be generated when a connection is made. + + @type _userIs: L{bytes} or L{None} + @ivar _userIs: The username sent with the USER command. + + @type _onLogout: no-argument callable or L{None} + @ivar _onLogout: The function to be executed when the connection is + lost. + + @type mbox: L{IMailbox} provider + @ivar mbox: The mailbox for the authenticated user. + + @type state: L{bytes} + @ivar state: The state which indicates what type of messages are expected + from the client. Valid states are 'COMMAND' and 'AUTH' + + @type blocked: L{None} or L{list} of 2-L{tuple} of + (E{1}) L{bytes} (E{2}) L{tuple} of L{bytes} + @ivar blocked: A list of blocked commands. While a response to a command + is being generated by the server, other commands are blocked. When + no command is outstanding, C{blocked} is set to none. Otherwise, it + contains a list of information about blocked commands. Each list + entry consists of the command and the arguments to the command. + + @type _highest: L{int} + @ivar _highest: The 1-based index of the highest message retrieved. + + @type _auth: L{IUsernameHashedPassword + } provider + @ivar _auth: Authorization credentials. + """ + magic = None + _userIs = None + _onLogout = None + + AUTH_CMDS = [b'CAPA', b'USER', b'PASS', b'APOP', b'AUTH', b'RPOP', b'QUIT'] + + portal = None + factory = None + + # The mailbox we're serving + mbox = None + + # Set this pretty low -- POP3 clients are expected to log in, download + # everything, and log out. + timeOut = 300 + + state = "COMMAND" + + # PIPELINE + blocked = None + + # Cooperate and suchlike. + schedule = staticmethod(task.coiterate) + + _highest = 0 + + def connectionMade(self): + """ + Send a greeting to the client after the connection has been made. + """ + if self.magic is None: + self.magic = self.generateMagic() + self.successResponse(self.magic) + self.setTimeout(self.timeOut) + if getattr(self.factory, 'noisy', True): + log.msg("New connection from " + str(self.transport.getPeer())) + + + def connectionLost(self, reason): + """ + Clean up when the connection has been lost. + + @type reason: L{Failure} + @param reason: The reason the connection was terminated. + """ + if self._onLogout is not None: + self._onLogout() + self._onLogout = None + self.setTimeout(None) + + + def generateMagic(self): + """ + Generate an APOP challenge. + + @rtype: L{bytes} + @return: An RFC 822 message id format string. + """ + return smtp.messageid() + + + def successResponse(self, message=''): + """ + Send a response indicating success. + + @type message: stringifyable L{object} + @param message: An object whose string representation should be + included in the response. + """ + self.transport.write(successResponse(message)) + + + def failResponse(self, message=b''): + """ + Send a response indicating failure. + + @type message: stringifyable L{object} + @param message: An object whose string representation should be + included in the response. + """ + if not isinstance(message, bytes): + message = str(message).encode("utf-8") + self.sendLine(b'-ERR ' + message) + + + def lineReceived(self, line): + """ + Pass a received line to a state machine function. + + @type line: L{bytes} + @param line: A received line. + """ + self.resetTimeout() + getattr(self, 'state_' + self.state)(line) + + + def _unblock(self, _): + """ + Process as many blocked commands as possible. + + If there are no more blocked commands, set up for the next command to + be sent immediately. + + @type _: L{object} + @param _: Ignored. + """ + commands = self.blocked + self.blocked = None + while commands and self.blocked is None: + cmd, args = commands.pop(0) + self.processCommand(cmd, *args) + if self.blocked is not None: + self.blocked.extend(commands) + + + def state_COMMAND(self, line): + """ + Handle received lines for the COMMAND state in which commands from the + client are expected. + + @type line: L{bytes} + @param line: A received command. + """ + try: + return self.processCommand(*line.split(b' ')) + except (ValueError, AttributeError, POP3Error, TypeError) as e: + log.err() + self.failResponse(b': '.join([b'bad protocol or server', + e.__class__.__name__.encode('utf-8'), + b''.join(e.args)])) + + + def processCommand(self, command, *args): + """ + Dispatch a command from the client for handling. + + @type command: L{bytes} + @param command: A POP3 command. + + @type args: L{tuple} of L{bytes} + @param args: Arguments to the command. + + @raise POP3Error: When the command is invalid or the command requires + prior authentication which hasn't been performed. + """ + if self.blocked is not None: + self.blocked.append((command, args)) + return + + command = command.upper() + authCmd = command in self.AUTH_CMDS + if not self.mbox and not authCmd: + raise POP3Error(b"not authenticated yet: cannot do " + command) + f = getattr(self, 'do_' + command.decode("utf-8"), None) + if f: + return f(*args) + raise POP3Error(b"Unknown protocol command: " + command) + + + def listCapabilities(self): + """ + Return a list of server capabilities suitable for use in a CAPA + response. + + @rtype: L{list} of L{bytes} + @return: A list of server capabilities. + """ + baseCaps = [ + b"TOP", + b"USER", + b"UIDL", + b"PIPELINE", + b"CELERITY", + b"AUSPEX", + b"POTENCE", + ] + + if IServerFactory.providedBy(self.factory): + # Oh my god. We can't just loop over a list of these because + # each has spectacularly different return value semantics! + try: + v = self.factory.cap_IMPLEMENTATION() + if v and not isinstance(v, bytes): + v = str(v).encode("utf-8") + except NotImplementedError: + pass + except: + log.err() + else: + baseCaps.append(b"IMPLEMENTATION " + v) + + try: + v = self.factory.cap_EXPIRE() + if v and not isinstance(v, bytes): + v = str(v).encode("utf-8") + except NotImplementedError: + pass + except: + log.err() + else: + if v is None: + v = b"NEVER" + if self.factory.perUserExpiration(): + if self.mbox: + v = str(self.mbox.messageExpiration).encode("utf-8") + else: + v = v + b" USER" + baseCaps.append(b"EXPIRE " + v) + + try: + v = self.factory.cap_LOGIN_DELAY() + if v and not isinstance(v, bytes): + v = str(v).encode("utf-8") + except NotImplementedError: + pass + except: + log.err() + else: + if self.factory.perUserLoginDelay(): + if self.mbox: + v = str(self.mbox.loginDelay).encode("utf-8") + else: + v = v + b" USER" + baseCaps.append(b"LOGIN-DELAY " + v) + + try: + v = self.factory.challengers + except AttributeError: + pass + except: + log.err() + else: + baseCaps.append(b"SASL " + b' '.join(v.keys())) + return baseCaps + + + def do_CAPA(self): + """ + Handle a CAPA command. + + Respond with the server capabilities. + """ + self.successResponse(b"I can do the following:") + for cap in self.listCapabilities(): + self.sendLine(cap) + self.sendLine(b".") + + + def do_AUTH(self, args=None): + """ + Handle an AUTH command. + + If the AUTH extension is not supported, send an error response. If an + authentication mechanism was not specified in the command, send a list + of all supported authentication methods. Otherwise, send an + authentication challenge to the client and transition to the + AUTH state. + + @type args: L{bytes} or L{None} + @param args: The name of an authentication mechanism. + """ + if not getattr(self.factory, 'challengers', None): + self.failResponse(b"AUTH extension unsupported") + return + + if args is None: + self.successResponse("Supported authentication methods:") + for a in self.factory.challengers: + self.sendLine(a.upper()) + self.sendLine(b".") + return + + auth = self.factory.challengers.get(args.strip().upper()) + if not self.portal or not auth: + self.failResponse(b"Unsupported SASL selected") + return + + self._auth = auth() + chal = self._auth.getChallenge() + + self.sendLine(b'+ ' + base64.encodestring(chal).rstrip(b'\n')) + self.state = 'AUTH' + + + def state_AUTH(self, line): + """ + Handle received lines for the AUTH state in which an authentication + challenge response from the client is expected. + + Transition back to the COMMAND state. Check the credentials and + complete the authorization process with the L{_cbMailbox} + callback function on success or the L{_ebMailbox} and L{_ebUnexpected} + errback functions on failure. + + @type line: L{bytes} + @param line: The challenge response. + """ + self.state = "COMMAND" + try: + parts = base64.decodestring(line).split(None, 1) + except binascii.Error: + self.failResponse(b"Invalid BASE64 encoding") + else: + if len(parts) != 2: + self.failResponse(b"Invalid AUTH response") + return + self._auth.username = parts[0] + self._auth.response = parts[1] + d = self.portal.login(self._auth, None, IMailbox) + d.addCallback(self._cbMailbox, parts[0]) + d.addErrback(self._ebMailbox) + d.addErrback(self._ebUnexpected) + + + def do_APOP(self, user, digest): + """ + Handle an APOP command. + + Perform APOP authentication and complete the authorization process with + the L{_cbMailbox} callback function on success or the L{_ebMailbox} + and L{_ebUnexpected} errback functions on failure. + + @type user: L{bytes} + @param user: A username. + + @type digest: L{bytes} + @param digest: An MD5 digest string. + """ + d = defer.maybeDeferred(self.authenticateUserAPOP, user, digest) + d.addCallbacks(self._cbMailbox, self._ebMailbox, callbackArgs=(user,) + ).addErrback(self._ebUnexpected) + + + def _cbMailbox(self, result, user): + """ + Complete successful authentication. + + Save the mailbox and logout function for the authenticated user and + send a successful response to the client. + + @type result: C{tuple} + @param interface_avatar_logout: The first item of the tuple is a + C{zope.interface.Interface} which is the interface + supported by the avatar. The second item of the tuple is a + L{IMailbox} provider which is the mailbox for the + authenticated user. The third item of the tuple is a no-argument + callable which is a function to be invoked when the session is + terminated. + + @type user: L{bytes} + @param user: The user being authenticated. + """ + (interface, avatar, logout) = result + if interface is not IMailbox: + self.failResponse(b'Authentication failed') + log.err( + "_cbMailbox() called with an interface other than IMailbox" + ) + return + + self.mbox = avatar + self._onLogout = logout + self.successResponse('Authentication succeeded') + if getattr(self.factory, 'noisy', True): + log.msg(b"Authenticated login for " + user) + + + def _ebMailbox(self, failure): + """ + Handle an expected authentication failure. + + Send an appropriate error response for a L{LoginDenied} or + L{LoginFailed} authentication failure. + + @type failure: L{Failure} + @param failure: The authentication error. + """ + failure = failure.trap(cred.error.LoginDenied, cred.error.LoginFailed) + if issubclass(failure, cred.error.LoginDenied): + self.failResponse("Access denied: " + str(failure)) + elif issubclass(failure, cred.error.LoginFailed): + self.failResponse(b'Authentication failed') + if getattr(self.factory, 'noisy', True): + log.msg( + "Denied login attempt from " + str(self.transport.getPeer()) + ) + + + def _ebUnexpected(self, failure): + """ + Handle an unexpected authentication failure. + + Send an error response for an unexpected authentication failure. + + @type failure: L{Failure} + @param failure: The authentication error. + """ + self.failResponse('Server error: ' + failure.getErrorMessage()) + log.err(failure) + + + def do_USER(self, user): + """ + Handle a USER command. + + Save the username and send a successful response prompting the client + for the password. + + @type user: L{bytes} + @param user: A username. + """ + self._userIs = user + self.successResponse(b'USER accepted, send PASS') + + + def do_PASS(self, password, *words): + """ + Handle a PASS command. + + If a USER command was previously received, authenticate the user and + complete the authorization process with the L{_cbMailbox} callback + function on success or the L{_ebMailbox} and L{_ebUnexpected} errback + functions on failure. If a USER command was not previously received, + send an error response. + + @type password: L{bytes} + @param password: A password. + + @type words: L{tuple} of L{bytes} + @param words: Other parts of the password split by spaces. + """ + if self._userIs is None: + self.failResponse(b"USER required before PASS") + return + user = self._userIs + self._userIs = None + password = b' '.join((password,) + words) + d = defer.maybeDeferred(self.authenticateUserPASS, user, password) + d.addCallbacks(self._cbMailbox, self._ebMailbox, + callbackArgs=(user,)).addErrback(self._ebUnexpected) + + + def _longOperation(self, d): + """ + Stop timeouts and block further command processing while a long + operation completes. + + @type d: L{Deferred } + @param d: A deferred which triggers at the completion of a long + operation. + + @rtype: L{Deferred } + @return: A deferred which triggers after command processing resumes and + timeouts restart after the completion of a long operation. + """ + timeOut = self.timeOut + self.setTimeout(None) + self.blocked = [] + d.addCallback(self._unblock) + d.addCallback(lambda ign: self.setTimeout(timeOut)) + return d + + + def _coiterate(self, gen): + """ + Direct the output of an iterator to the transport and arrange for + iteration to take place. + + @type gen: iterable which yields L{bytes} + @param gen: An iterator over strings. + + @rtype: L{Deferred } + @return: A deferred which fires when the iterator finishes. + """ + return self.schedule( + _IteratorBuffer(self.transport.writeSequence, gen) + ) + + + def do_STAT(self): + """ + Handle a STAT command. + + @rtype: L{Deferred } + @return: A deferred which triggers after the response to the STAT + command has been issued. + """ + d = defer.maybeDeferred(self.mbox.listMessages) + def cbMessages(msgs): + return self._coiterate(formatStatResponse(msgs)) + def ebMessages(err): + self.failResponse(err.getErrorMessage()) + log.msg("Unexpected do_STAT failure:") + log.err(err) + return self._longOperation(d.addCallbacks(cbMessages, ebMessages)) + + + def do_LIST(self, i=None): + """ + Handle a LIST command. + + @type i: L{bytes} or L{None} + @param i: A 1-based message index. + + @rtype: L{Deferred } + @return: A deferred which triggers after the response to the LIST + command has been issued. + """ + if i is None: + d = defer.maybeDeferred(self.mbox.listMessages) + def cbMessages(msgs): + return self._coiterate(formatListResponse(msgs)) + def ebMessages(err): + self.failResponse(err.getErrorMessage()) + log.msg("Unexpected do_LIST failure:") + log.err(err) + return self._longOperation(d.addCallbacks(cbMessages, ebMessages)) + else: + try: + i = int(i) + if i < 1: + raise ValueError() + except ValueError: + if not isinstance(i, bytes): + i = str(i).encode("utf-8") + self.failResponse(b"Invalid message-number: " + i) + else: + d = defer.maybeDeferred(self.mbox.listMessages, i - 1) + def cbMessage(msg): + self.successResponse(intToBytes(i) + b' ' + + intToBytes(msg)) + def ebMessage(err): + errcls = err.check(ValueError, IndexError) + if errcls is not None: + if errcls is IndexError: + # IndexError was supported for a while, but really + # shouldn't be. One error condition, one exception + # type. See ticket #6669. + warnings.warn( + "twisted.mail.pop3.IMailbox.listMessages may " + "not raise IndexError for out-of-bounds " + "message numbers: raise ValueError instead.", + PendingDeprecationWarning) + invalidNum = i + if invalidNum and not isinstance(invalidNum, bytes): + invalidNum = str(invalidNum).encode("utf-8") + self.failResponse(b"Invalid message-number: " + + invalidNum) + else: + self.failResponse(err.getErrorMessage()) + log.msg("Unexpected do_LIST failure:") + log.err(err) + d.addCallbacks(cbMessage, ebMessage) + return self._longOperation(d) + + + def do_UIDL(self, i=None): + """ + Handle a UIDL command. + + @type i: L{bytes} or L{None} + @param i: A 1-based message index. + + @rtype: L{Deferred } + @return: A deferred which triggers after the response to the UIDL + command has been issued. + """ + if i is None: + d = defer.maybeDeferred(self.mbox.listMessages) + def cbMessages(msgs): + return self._coiterate( + formatUIDListResponse(msgs, self.mbox.getUidl), + ) + def ebMessages(err): + self.failResponse(err.getErrorMessage()) + log.msg("Unexpected do_UIDL failure:") + log.err(err) + return self._longOperation(d.addCallbacks(cbMessages, ebMessages)) + else: + try: + i = int(i) + if i < 1: + raise ValueError() + except ValueError: + self.failResponse("Bad message number argument") + else: + try: + msg = self.mbox.getUidl(i - 1) + except IndexError: + # XXX TODO See above comment regarding IndexError. + warnings.warn( + "twisted.mail.pop3.IMailbox.getUidl may not " + "raise IndexError for out-of-bounds message numbers: " + "raise ValueError instead.", + PendingDeprecationWarning) + self.failResponse("Bad message number argument") + except ValueError: + self.failResponse("Bad message number argument") + else: + if not isinstance(msg, bytes): + msg = str(msg).encode("utf-8") + self.successResponse(msg) + + + def _getMessageFile(self, i): + """ + Retrieve the size and contents of a message. + + @type i: L{bytes} + @param i: A 1-based message index. + + @rtype: L{Deferred } which successfully fires with + 2-L{tuple} of (E{1}) L{int}, (E{2}) file-like object + @return: A deferred which successfully fires with the size of the + message and a file containing the contents of the message. + """ + try: + msg = int(i) - 1 + if msg < 0: + raise ValueError() + except ValueError: + self.failResponse("Bad message number argument") + return defer.succeed(None) + + sizeDeferred = defer.maybeDeferred(self.mbox.listMessages, msg) + def cbMessageSize(size): + if not size: + return defer.fail(_POP3MessageDeleted()) + fileDeferred = defer.maybeDeferred(self.mbox.getMessage, msg) + fileDeferred.addCallback(lambda fObj: (size, fObj)) + return fileDeferred + + def ebMessageSomething(err): + errcls = err.check(_POP3MessageDeleted, ValueError, IndexError) + if errcls is _POP3MessageDeleted: + self.failResponse("message deleted") + elif errcls in (ValueError, IndexError): + if errcls is IndexError: + # XXX TODO See above comment regarding IndexError. + warnings.warn( + "twisted.mail.pop3.IMailbox.listMessages may not " + "raise IndexError for out-of-bounds message numbers: " + "raise ValueError instead.", + PendingDeprecationWarning) + self.failResponse("Bad message number argument") + else: + log.msg("Unexpected _getMessageFile failure:") + log.err(err) + return None + + sizeDeferred.addCallback(cbMessageSize) + sizeDeferred.addErrback(ebMessageSomething) + return sizeDeferred + + + def _sendMessageContent(self, i, fpWrapper, successResponse): + """ + Send the contents of a message. + + @type i: L{bytes} + @param i: A 1-based message index. + + @type fpWrapper: callable that takes a file-like object and returns + a file-like object + @param fpWrapper: + + @type successResponse: callable that takes L{int} and returns + L{bytes} + @param successResponse: + + @rtype: L{Deferred} + @return: A deferred which triggers after the message has been sent. + """ + d = self._getMessageFile(i) + def cbMessageFile(info): + if info is None: + # Some error occurred - a failure response has been sent + # already, just give up. + return + + self._highest = max(self._highest, int(i)) + resp, fp = info + fp = fpWrapper(fp) + self.successResponse(successResponse(resp)) + s = basic.FileSender() + d = s.beginFileTransfer(fp, self.transport, self.transformChunk) + + def cbFileTransfer(lastsent): + if lastsent != b'\n': + line = b'\r\n.' + else: + line = b'.' + self.sendLine(line) + + def ebFileTransfer(err): + self.transport.loseConnection() + log.msg("Unexpected error in _sendMessageContent:") + log.err(err) + + d.addCallback(cbFileTransfer) + d.addErrback(ebFileTransfer) + return d + return self._longOperation(d.addCallback(cbMessageFile)) + + + def do_TOP(self, i, size): + """ + Handle a TOP command. + + @type i: L{bytes} + @param i: A 1-based message index. + + @type size: L{bytes} + @param size: The number of lines of the message to retrieve. + + @rtype: L{Deferred} + @return: A deferred which triggers after the response to the TOP + command has been issued. + """ + try: + size = int(size) + if size < 0: + raise ValueError + except ValueError: + self.failResponse("Bad line count argument") + else: + return self._sendMessageContent( + i, + lambda fp: _HeadersPlusNLines(fp, size), + lambda size: "Top of message follows") + + + def do_RETR(self, i): + """ + Handle a RETR command. + + @type i: L{bytes} + @param i: A 1-based message index. + + @rtype: L{Deferred} + @return: A deferred which triggers after the response to the RETR + command has been issued. + """ + return self._sendMessageContent( + i, + lambda fp: fp, + lambda size: "%d" % (size,)) + + + def transformChunk(self, chunk): + """ + Transform a chunk of a message to POP3 message format. + + Make sure each line ends with C{'\\r\\n'} and byte-stuff the + termination character (C{'.'}) by adding an extra one when one appears + at the beginning of a line. + + @type chunk: L{bytes} + @param chunk: A string to transform. + + @rtype: L{bytes} + @return: The transformed string. + """ + return chunk.replace(b'\n', b'\r\n').replace(b'\r\n.', b'\r\n..') + + + def finishedFileTransfer(self, lastsent): + """ + Send the termination sequence. + + @type lastsent: L{bytes} + @param lastsent: The last character of the file. + """ + if lastsent != b'\n': + line = b'\r\n.' + else: + line = b'.' + self.sendLine(line) + + + def do_DELE(self, i): + """ + Handle a DELE command. + + Mark a message for deletion and issue a successful response. + + @type i: L{int} + @param i: A 1-based message index. + """ + i = int(i)-1 + self.mbox.deleteMessage(i) + self.successResponse() + + + def do_NOOP(self): + """ + Handle a NOOP command. + + Do nothing but issue a successful response. + """ + self.successResponse() + + + def do_RSET(self): + """ + Handle a RSET command. + + Unmark any messages that have been flagged for deletion. + """ + try: + self.mbox.undeleteMessages() + except: + log.err() + self.failResponse() + else: + self._highest = 0 + self.successResponse() + + + def do_LAST(self): + """ + Handle a LAST command. + + Respond with the 1-based index of the highest retrieved message. + """ + self.successResponse(self._highest) + + + def do_RPOP(self, user): + """ + Handle an RPOP command. + + RPOP is not supported. Send an error response. + + @type user: L{bytes} + @param user: A username. + + """ + self.failResponse('permission denied, sucker') + + + def do_QUIT(self): + """ + Handle a QUIT command. + + Remove any messages marked for deletion, issue a successful response, + and drop the connection. + """ + if self.mbox: + self.mbox.sync() + self.successResponse() + self.transport.loseConnection() + + + def authenticateUserAPOP(self, user, digest): + """ + Perform APOP authentication. + + @type user: L{bytes} + @param user: The name of the user attempting to log in. + + @type digest: L{bytes} + @param digest: The challenge response. + + @rtype: L{Deferred } which successfully results in + 3-L{tuple} of (E{1}) L{IMailbox }, (E{2}) + L{IMailbox } provider, (E{3}) no-argument callable + @return: A deferred which fires when authentication is complete. If + successful, it returns an L{IMailbox } interface, a + mailbox, and a function to be invoked with the session is + terminated. If authentication fails, the deferred fails with an + L{UnathorizedLogin } error. + + @raise cred.error.UnauthorizedLogin: When authentication fails. + """ + if self.portal is not None: + return self.portal.login( + APOPCredentials(self.magic, user, digest), + None, + IMailbox + ) + raise cred.error.UnauthorizedLogin() + + + def authenticateUserPASS(self, user, password): + """ + Perform authentication for a username/password login. + + @type user: L{bytes} + @param user: The name of the user attempting to log in. + + @type password: L{bytes} + @param password: The password to authenticate with. + + @rtype: L{Deferred } which successfully results in + 3-L{tuple} of (E{1}) L{IMailbox }, (E{2}) L{IMailbox + } provider, (E{3}) no-argument callable + @return: A deferred which fires when authentication is complete. If + successful, it returns a L{pop3.IMailbox} interface, a mailbox, + and a function to be invoked with the session is terminated. + If authentication fails, the deferred fails with an + L{UnathorizedLogin } error. + + @raise cred.error.UnauthorizedLogin: When authentication fails. + """ + if self.portal is not None: + return self.portal.login( + cred.credentials.UsernamePassword(user, password), + None, + IMailbox + ) + raise cred.error.UnauthorizedLogin() + + + +@implementer(IMailbox) +class Mailbox: + """ + A base class for mailboxes. + """ + def listMessages(self, i=None): + """ + Retrieve the size of a message, or, if none is specified, the size of + each message in the mailbox. + + @type i: L{int} or L{None} + @param i: The 0-based index of the message. + + @rtype: L{int}, sequence of L{int}, or L{Deferred } + @return: The number of octets in the specified message, or, if an + index is not specified, a sequence of the number of octets for + all messages in the mailbox or a deferred which fires with + one of those. Any value which corresponds to a deleted message + is set to 0. + + @raise ValueError or IndexError: When the index does not correspond to + a message in the mailbox. The use of ValueError is preferred. + """ + return [] + + + def getMessage(self, i): + """ + Retrieve a file containing the contents of a message. + + @type i: L{int} + @param i: The 0-based index of a message. + + @rtype: file-like object + @return: A file containing the message. + + @raise ValueError or IndexError: When the index does not correspond to + a message in the mailbox. The use of ValueError is preferred. + """ + raise ValueError + + + def getUidl(self, i): + """ + Get a unique identifier for a message. + + @type i: L{int} + @param i: The 0-based index of a message. + + @rtype: L{bytes} + @return: A string of printable characters uniquely identifying the + message for all time. + + @raise ValueError or IndexError: When the index does not correspond to + a message in the mailbox. The use of ValueError is preferred. + """ + raise ValueError + + + def deleteMessage(self, i): + """ + Mark a message for deletion. + + This must not change the number of messages in this mailbox. Further + requests for the size of the deleted message should return 0. Further + requests for the message itself may raise an exception. + + @type i: L{int} + @param i: The 0-based index of a message. + + @raise ValueError or IndexError: When the index does not correspond to + a message in the mailbox. The use of ValueError is preferred. + """ + raise ValueError + + + def undeleteMessages(self): + """ + Undelete all messages marked for deletion. + + Any message which can be undeleted should be returned to its original + position in the message sequence and retain its original UID. + """ + pass + + + def sync(self): + """ + Discard the contents of any message marked for deletion. + """ + pass + + + +NONE, SHORT, FIRST_LONG, LONG = range(4) + +NEXT = {} +NEXT[NONE] = NONE +NEXT[SHORT] = NONE +NEXT[FIRST_LONG] = LONG +NEXT[LONG] = NONE + + + +class POP3Client(basic.LineOnlyReceiver): + """ + A POP3 client protocol. + + @type mode: L{int} + @ivar mode: The type of response expected from the server. Choices include + none (0), a one line response (1), the first line of a multi-line + response (2), and subsequent lines of a multi-line response (3). + + @type command: L{bytes} + @ivar command: The command most recently sent to the server. + + @type welcomeRe: L{RegexObject } + @ivar welcomeRe: A regular expression which matches the APOP challenge in + the server greeting. + + @type welcomeCode: L{bytes} + @ivar welcomeCode: The APOP challenge passed in the server greeting. + """ + mode = SHORT + command = b'WELCOME' + import re + welcomeRe = re.compile(b'<(.*)>') + + def __init__(self): + """ + Issue deprecation warning. + """ + import warnings + warnings.warn("twisted.mail.pop3.POP3Client is deprecated, " + "please use twisted.mail.pop3.AdvancedPOP3Client " + "instead.", DeprecationWarning, + stacklevel=3) + + + def sendShort(self, command, params=None): + """ + Send a POP3 command to which a short response is expected. + + @type command: L{bytes} + @param command: A POP3 command. + + @type params: stringifyable L{object} or L{None} + @param params: Command arguments. + """ + if params is not None: + if not isinstance(params, bytes): + params = str(params).encode("utf-8") + self.sendLine(command + b' ' + params) + else: + self.sendLine(command) + self.command = command + self.mode = SHORT + + + def sendLong(self, command, params): + """ + Send a POP3 command to which a long response is expected. + + @type command: L{bytes} + @param command: A POP3 command. + + @type params: stringifyable L{object} + @param params: Command arguments. + """ + if params: + if not isinstance(params, bytes): + params = str(params).encode("utf-8") + self.sendLine(command + b' ' + params) + else: + self.sendLine(command) + self.command = command + self.mode = FIRST_LONG + + + def handle_default(self, line): + """ + Handle responses from the server for which no other handler exists. + + @type line: L{bytes} + @param line: A received line. + """ + if line[:-4] == b'-ERR': + self.mode = NONE + + + def handle_WELCOME(self, line): + """ + Handle a server response which is expected to be a server greeting. + + @type line: L{bytes} + @param line: A received line. + """ + code, data = line.split(b' ', 1) + if code != b'+OK': + self.transport.loseConnection() + else: + m = self.welcomeRe.match(line) + if m: + self.welcomeCode = m.group(1) + + + def _dispatch(self, command, default, *args): + """ + Dispatch a response from the server for handling. + + Command X is dispatched to handle_X() if it exists. If not, it is + dispatched to the default handler. + + @type command: L{bytes} + @param command: The command. + + @type default: callable that takes L{bytes} or + L{None} + @param default: The default handler. + + @type args: L{tuple} or L{None} + @param args: Arguments to the handler function. + """ + try: + method = getattr(self, 'handle_' + command.decode("utf-8"), + default) + if method is not None: + method(*args) + except: + log.err() + + + def lineReceived(self, line): + """ + Dispatch a received line for processing. + + The choice of function to handle the received line is based on the + type of response expected to the command sent to the server and how + much of that response has been received. + + An expected one line response to command X is handled by handle_X(). + The first line of a multi-line response to command X is also handled by + handle_X(). Subsequent lines of the multi-line response are handled by + handle_X_continue() except for the last line which is handled by + handle_X_end(). + + @type line: L{bytes} + @param line: A received line. + """ + if self.mode == SHORT or self.mode == FIRST_LONG: + self.mode = NEXT[self.mode] + self._dispatch(self.command, self.handle_default, line) + elif self.mode == LONG: + if line == b'.': + self.mode = NEXT[self.mode] + self._dispatch(self.command + b'_end', None) + return + if line[:1] == b'.': + line = line[1:] + self._dispatch(self.command + b"_continue", None, line) + + + def apopAuthenticate(self, user, password, magic): + """ + Perform an authenticated login. + + @type user: L{bytes} + @param user: The username with which to log in. + + @type password: L{bytes} + @param password: The password with which to log in. + + @type magic: L{bytes} + @param magic: The challenge provided by the server. + """ + digest = md5(magic + password).hexdigest().encode("ascii") + self.apop(user, digest) + + + def apop(self, user, digest): + """ + Send an APOP command to perform authenticated login. + + @type user: L{bytes} + @param user: The username with which to log in. + + @type digest: L{bytes} + @param digest: The challenge response with which to authenticate. + """ + self.sendLong(b'APOP', b' '.join((user, digest))) + + + def retr(self, i): + """ + Send a RETR command to retrieve a message from the server. + + @type i: L{int} or L{bytes} + @param i: A 0-based message index. + """ + self.sendLong(b'RETR', i) + + + def dele(self, i): + """ + Send a DELE command to delete a message from the server. + + @type i: L{int} or L{bytes} + @param i: A 0-based message index. + """ + self.sendShort(b'DELE', i) + + + def list(self, i=''): + """ + Send a LIST command to retrieve the size of a message or, if no message + is specified, the sizes of all messages. + + @type i: L{int} or L{bytes} + @param i: A 0-based message index or the empty string to specify all + messages. + """ + self.sendLong(b'LIST', i) + + + def uidl(self, i=''): + """ + Send a UIDL command to retrieve the unique identifier of a message or, + if no message is specified, the unique identifiers of all messages. + + @type i: L{int} or L{bytes} + @param i: A 0-based message index or the empty string to specify all + messages. + """ + self.sendLong(b'UIDL', i) + + + def user(self, name): + """ + Send a USER command to perform the first half of a plaintext login. + + @type name: L{bytes} + @param name: The username with which to log in. + """ + self.sendShort(b'USER', name) + + + def password(self, password): + """ + Perform the second half of a plaintext login. + + @type password: L{bytes} + @param password: The plaintext password with which to authenticate. + """ + self.sendShort(b'PASS', password) + + pass_ = password + + def quit(self): + """ + Send a QUIT command to disconnect from the server. + """ + self.sendShort(b'QUIT') + + +from twisted.mail.pop3client import POP3Client as AdvancedPOP3Client +from twisted.mail.pop3client import InsecureAuthenticationDisallowed +from twisted.mail.pop3client import ServerErrorResponse +from twisted.mail.pop3client import LineTooLong +from twisted.mail.pop3client import TLSError +from twisted.mail.pop3client import TLSNotSupportedError + +__all__ = [ + # Interfaces + 'IMailbox', 'IServerFactory', + + # Exceptions + 'POP3Error', 'POP3ClientError', 'InsecureAuthenticationDisallowed', + 'ServerErrorResponse', 'LineTooLong', 'TLSError', 'TLSNotSupportedError', + + # Protocol classes + 'POP3', 'POP3Client', 'AdvancedPOP3Client', + + # Misc + 'APOPCredentials', 'Mailbox'] diff --git a/contrib/python/Twisted/py2/twisted/mail/pop3client.py b/contrib/python/Twisted/py2/twisted/mail/pop3client.py new file mode 100644 index 00000000000..fc27bb4cc56 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/pop3client.py @@ -0,0 +1,1264 @@ +# -*- test-case-name: twisted.mail.test.test_pop3client -*- +# Copyright (c) 2001-2004 Divmod Inc. +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A POP3 client protocol implementation. + +Don't use this module directly. Use twisted.mail.pop3 instead. + +@author: Jp Calderone +""" + +import re +from hashlib import md5 + +from twisted.python import log +from twisted.python.compat import intToBytes +from twisted.internet import defer +from twisted.protocols import basic +from twisted.protocols import policies +from twisted.internet import error +from twisted.internet import interfaces +from twisted.mail._except import ( + InsecureAuthenticationDisallowed, TLSError, + TLSNotSupportedError, ServerErrorResponse, LineTooLong) + +OK = b'+OK' +ERR = b'-ERR' + + + +class _ListSetter: + """ + A utility class to construct a list from a multi-line response accounting + for deleted messages. + + POP3 responses sometimes occur in the form of a list of lines containing + two pieces of data, a message index and a value of some sort. When a + message is deleted, it is omitted from these responses. The L{setitem} + method of this class is meant to be called with these two values. In the + cases where indices are skipped, it takes care of padding out the missing + values with L{None}. + + @ivar L: See L{__init__} + """ + def __init__(self, L): + """ + @type L: L{list} of L{object} + @param L: The list being constructed. An empty list should be + passed in. + """ + self.L = L + + + def setitem(self, itemAndValue): + """ + Add the value at the specified position, padding out missing entries. + + @type itemAndValue: C{tuple} + @param item: A tuple of (item, value). The I{item} is the 0-based + index in the list at which the value should be placed. The value is + is an L{object} to put in the list. + """ + (item, value) = itemAndValue + diff = item - len(self.L) + 1 + if diff > 0: + self.L.extend([None] * diff) + self.L[item] = value + + + +def _statXform(line): + """ + Parse the response to a STAT command. + + @type line: L{bytes} + @param line: The response from the server to a STAT command minus the + status indicator. + + @rtype: 2-L{tuple} of (0) L{int}, (1) L{int} + @return: The number of messages in the mailbox and the size of the mailbox. + """ + numMsgs, totalSize = line.split(None, 1) + return int(numMsgs), int(totalSize) + + + +def _listXform(line): + """ + Parse a line of the response to a LIST command. + + The line from the LIST response consists of a 1-based message number + followed by a size. + + @type line: L{bytes} + @param line: A non-initial line from the multi-line response to a LIST + command. + + @rtype: 2-L{tuple} of (0) L{int}, (1) L{int} + @return: The 0-based index of the message and the size of the message. + """ + index, size = line.split(None, 1) + return int(index) - 1, int(size) + + + +def _uidXform(line): + """ + Parse a line of the response to a UIDL command. + + The line from the UIDL response consists of a 1-based message number + followed by a unique id. + + @type line: L{bytes} + @param line: A non-initial line from the multi-line response to a UIDL + command. + + @rtype: 2-L{tuple} of (0) L{int}, (1) L{bytes} + @return: The 0-based index of the message and the unique identifier + for the message. + """ + index, uid = line.split(None, 1) + return int(index) - 1, uid + + + +def _codeStatusSplit(line): + """ + Parse the first line of a multi-line server response. + + @type line: L{bytes} + @param line: The first line of a multi-line server response. + + @rtype: 2-tuple of (0) L{bytes}, (1) L{bytes} + @return: The status indicator and the rest of the server response. + """ + parts = line.split(b' ', 1) + if len(parts) == 1: + return parts[0], b'' + return parts + + + +def _dotUnquoter(line): + """ + Remove a byte-stuffed termination character at the beginning of a line if + present. + + When the termination character (C{'.'}) appears at the beginning of a line, + the server byte-stuffs it by adding another termination character to + avoid confusion with the terminating sequence (C{'.\\r\\n'}). + + @type line: L{bytes} + @param line: A received line. + + @rtype: L{bytes} + @return: The line without the byte-stuffed termination character at the + beginning if it was present. Otherwise, the line unchanged. + """ + if line.startswith(b'..'): + return line[1:] + return line + + + +class POP3Client(basic.LineOnlyReceiver, policies.TimeoutMixin): + """ + A POP3 client protocol. + + Instances of this class provide a convenient, efficient API for + retrieving and deleting messages from a POP3 server. + + This API provides a pipelining interface but POP3 pipelining + on the network is not yet supported. + + @type startedTLS: L{bool} + @ivar startedTLS: An indication of whether TLS has been negotiated + successfully. + + @type allowInsecureLogin: L{bool} + @ivar allowInsecureLogin: An indication of whether plaintext login should + be allowed when the server offers no authentication challenge and the + transport does not offer any protection via encryption. + + @type serverChallenge: L{bytes} or L{None} + @ivar serverChallenge: The challenge received in the server greeting. + + @type timeout: L{int} + @ivar timeout: The number of seconds to wait on a response from the server + before timing out a connection. If the number is <= 0, no timeout + checking will be performed. + + @type _capCache: L{None} or L{dict} mapping L{bytes} + to L{list} of L{bytes} and/or L{bytes} to L{None} + @ivar _capCache: The cached server capabilities. Capabilities are not + allowed to change during the session (except when TLS is negotiated), + so the first response to a capabilities command can be used for + later lookups. + + @type _challengeMagicRe: L{RegexObject } + @ivar _challengeMagicRe: A regular expression which matches the + challenge in the server greeting. + + @type _blockedQueue: L{None} or L{list} of 3-L{tuple} + of (0) L{Deferred }, (1) callable which results + in a L{Deferred }, (2) L{tuple} + @ivar _blockedQueue: A list of blocked commands. While a command is + awaiting a response from the server, other commands are blocked. When + no command is outstanding, C{_blockedQueue} is set to L{None}. + Otherwise, it contains a list of information about blocked commands. + Each list entry provides the following information about a blocked + command: the deferred that should be called when the response to the + command is received, the function that sends the command, and the + arguments to the function. + + @type _waiting: L{Deferred } or + L{None} + @ivar _waiting: A deferred which fires when the response to the + outstanding command is received from the server. + + @type _timedOut: L{bool} + @ivar _timedOut: An indication of whether the connection was dropped + because of a timeout. + + @type _greetingError: L{bytes} or L{None} + @ivar _greetingError: The server greeting minus the status indicator, when + the connection was dropped because of an error in the server greeting. + Otherwise, L{None}. + + @type state: L{bytes} + @ivar state: The state which indicates what type of response is expected + from the server. Valid states are: 'WELCOME', 'WAITING', 'SHORT', + 'LONG_INITIAL', 'LONG'. + + @type _xform: L{None} or callable that takes L{bytes} + and returns L{object} + @ivar _xform: The transform function which is used to convert each + line of a multi-line response into usable values for use by the + consumer function. If L{None}, each line of the multi-line response + is sent directly to the consumer function. + + @type _consumer: callable that takes L{object} + @ivar _consumer: The consumer function which is used to store the + values derived by the transform function from each line of a + multi-line response into a list. + """ + startedTLS = False + allowInsecureLogin = False + timeout = 0 + serverChallenge = None + + _capCache = None + _challengeMagicRe = re.compile(b'(<[^>]+>)') + _blockedQueue = None + _waiting = None + _timedOut = False + _greetingError = None + + def _blocked(self, f, *a): + """ + Block a command, if necessary. + + If commands are being blocked, append information about the function + which sends the command to a list and return a deferred that will be + chained with the return value of the function when it eventually runs. + Otherwise, set up for subsequent commands to be blocked and return + L{None}. + + @type f: callable + @param f: A function which sends a command. + + @type a: L{tuple} + @param a: Arguments to the function. + + @rtype: L{None} or L{Deferred } + @return: L{None} if the command can run immediately. Otherwise, + a deferred that will eventually trigger with the return value of + the function. + """ + if self._blockedQueue is not None: + d = defer.Deferred() + self._blockedQueue.append((d, f, a)) + return d + self._blockedQueue = [] + return None + + + def _unblock(self): + """ + Send the next blocked command. + + If there are no more commands in the blocked queue, set up for the next + command to be sent immediately. + """ + if self._blockedQueue == []: + self._blockedQueue = None + elif self._blockedQueue is not None: + _blockedQueue = self._blockedQueue + self._blockedQueue = None + + d, f, a = _blockedQueue.pop(0) + d2 = f(*a) + d2.chainDeferred(d) + # f is a function which uses _blocked (otherwise it wouldn't + # have gotten into the blocked queue), which means it will have + # re-set _blockedQueue to an empty list, so we can put the rest + # of the blocked queue back into it now. + self._blockedQueue.extend(_blockedQueue) + + + def sendShort(self, cmd, args): + """ + Send a POP3 command to which a short response is expected. + + Block all further commands from being sent until the response is + received. Transition the state to SHORT. + + @type cmd: L{bytes} + @param cmd: A POP3 command. + + @type args: L{bytes} + @param args: The command arguments. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the entire response is received. + On an OK response, it returns the response from the server minus + the status indicator. On an ERR response, it issues a server + error response failure with the response from the server minus the + status indicator. + """ + d = self._blocked(self.sendShort, cmd, args) + if d is not None: + return d + + if args: + self.sendLine(cmd + b' ' + args) + else: + self.sendLine(cmd) + self.state = 'SHORT' + self._waiting = defer.Deferred() + return self._waiting + + + def sendLong(self, cmd, args, consumer, xform): + """ + Send a POP3 command to which a multi-line response is expected. + + Block all further commands from being sent until the entire response is + received. Transition the state to LONG_INITIAL. + + @type cmd: L{bytes} + @param cmd: A POP3 command. + + @type args: L{bytes} + @param args: The command arguments. + + @type consumer: callable that takes L{object} + @param consumer: A consumer function which should be used to put + the values derived by a transform function from each line of the + multi-line response into a list. + + @type xform: L{None} or callable that takes + L{bytes} and returns L{object} + @param xform: A transform function which should be used to transform + each line of the multi-line response into usable values for use by + a consumer function. If L{None}, each line of the multi-line + response should be sent directly to the consumer function. + + @rtype: L{Deferred } which successfully fires with + callable that takes L{object} and fails with L{ServerErrorResponse} + @return: A deferred which fires when the entire response is received. + On an OK response, it returns the consumer function. On an ERR + response, it issues a server error response failure with the + response from the server minus the status indicator and the + consumer function. + """ + d = self._blocked(self.sendLong, cmd, args, consumer, xform) + if d is not None: + return d + + if args: + self.sendLine(cmd + b' ' + args) + else: + self.sendLine(cmd) + self.state = 'LONG_INITIAL' + self._xform = xform + self._consumer = consumer + self._waiting = defer.Deferred() + return self._waiting + + + # Twisted protocol callback + def connectionMade(self): + """ + Wait for a greeting from the server after the connection has been made. + + Start the connection in the WELCOME state. + """ + if self.timeout > 0: + self.setTimeout(self.timeout) + + self.state = 'WELCOME' + self._blockedQueue = [] + + + def timeoutConnection(self): + """ + Drop the connection when the server does not respond in time. + """ + self._timedOut = True + self.transport.loseConnection() + + + def connectionLost(self, reason): + """ + Clean up when the connection has been lost. + + When the loss of connection was initiated by the client due to a + timeout, the L{_timedOut} flag will be set. When it was initiated by + the client due to an error in the server greeting, L{_greetingError} + will be set to the server response minus the status indicator. + + @type reason: L{Failure } + @param reason: The reason the connection was terminated. + """ + if self.timeout > 0: + self.setTimeout(None) + + if self._timedOut: + reason = error.TimeoutError() + elif self._greetingError: + reason = ServerErrorResponse(self._greetingError) + + d = [] + if self._waiting is not None: + d.append(self._waiting) + self._waiting = None + if self._blockedQueue is not None: + d.extend([deferred for (deferred, f, a) in self._blockedQueue]) + self._blockedQueue = None + for w in d: + w.errback(reason) + + + def lineReceived(self, line): + """ + Pass a received line to a state machine function and + transition to the next state. + + @type line: L{bytes} + @param line: A received line. + """ + if self.timeout > 0: + self.resetTimeout() + + state = self.state + self.state = None + state = getattr(self, 'state_' + state)(line) or state + if self.state is None: + self.state = state + + + def lineLengthExceeded(self, buffer): + """ + Drop the connection when a server response exceeds the maximum line + length (L{LineOnlyReceiver.MAX_LENGTH}). + + @type buffer: L{bytes} + @param buffer: A received line which exceeds the maximum line length. + """ + # XXX - We need to be smarter about this + if self._waiting is not None: + waiting, self._waiting = self._waiting, None + waiting.errback(LineTooLong()) + self.transport.loseConnection() + + + # POP3 Client state logic - don't touch this. + def state_WELCOME(self, line): + """ + Handle server responses for the WELCOME state in which the server + greeting is expected. + + WELCOME is the first state. The server should send one line of text + with a greeting and possibly an APOP challenge. Transition the state + to WAITING. + + @type line: L{bytes} + @param line: A line received from the server. + + @rtype: L{bytes} + @return: The next state. + """ + code, status = _codeStatusSplit(line) + if code != OK: + self._greetingError = status + self.transport.loseConnection() + else: + m = self._challengeMagicRe.search(status) + + if m is not None: + self.serverChallenge = m.group(1) + + self.serverGreeting(status) + + self._unblock() + return 'WAITING' + + + def state_WAITING(self, line): + """ + Log an error for server responses received in the WAITING state during + which the server is not expected to send anything. + + @type line: L{bytes} + @param line: A line received from the server. + """ + log.msg("Illegal line from server: " + repr(line)) + + + def state_SHORT(self, line): + """ + Handle server responses for the SHORT state in which the server is + expected to send a single line response. + + Parse the response and fire the deferred which is waiting on receipt of + a complete response. Transition the state back to WAITING. + + @type line: L{bytes} + @param line: A line received from the server. + + @rtype: L{bytes} + @return: The next state. + """ + deferred, self._waiting = self._waiting, None + self._unblock() + code, status = _codeStatusSplit(line) + if code == OK: + deferred.callback(status) + else: + deferred.errback(ServerErrorResponse(status)) + return 'WAITING' + + + def state_LONG_INITIAL(self, line): + """ + Handle server responses for the LONG_INITIAL state in which the server + is expected to send the first line of a multi-line response. + + Parse the response. On an OK response, transition the state to + LONG. On an ERR response, cleanup and transition the state to + WAITING. + + @type line: L{bytes} + @param line: A line received from the server. + + @rtype: L{bytes} + @return: The next state. + """ + code, status = _codeStatusSplit(line) + if code == OK: + return 'LONG' + consumer = self._consumer + deferred = self._waiting + self._consumer = self._waiting = self._xform = None + self._unblock() + deferred.errback(ServerErrorResponse(status, consumer)) + return 'WAITING' + + + def state_LONG(self, line): + """ + Handle server responses for the LONG state in which the server is + expected to send a non-initial line of a multi-line response. + + On receipt of the last line of the response, clean up, fire the + deferred which is waiting on receipt of a complete response, and + transition the state to WAITING. Otherwise, pass the line to the + transform function, if provided, and then the consumer function. + + @type line: L{bytes} + @param line: A line received from the server. + + @rtype: L{bytes} + @return: The next state. + """ + # This is the state for each line of a long response. + if line == b'.': + consumer = self._consumer + deferred = self._waiting + self._consumer = self._waiting = self._xform = None + self._unblock() + deferred.callback(consumer) + return 'WAITING' + else: + if self._xform is not None: + self._consumer(self._xform(line)) + else: + self._consumer(line) + return 'LONG' + + + # Callbacks - override these + def serverGreeting(self, greeting): + """ + Handle the server greeting. + + @type greeting: L{bytes} + @param greeting: The server greeting minus the status indicator. + For servers implementing APOP authentication, this will contain a + challenge string. + """ + + + # External API - call these (most of 'em anyway) + def startTLS(self, contextFactory=None): + """ + Switch to encrypted communication using TLS. + + The first step of switching to encrypted communication is obtaining + the server's capabilities. When that is complete, the L{_startTLS} + callback function continues the switching process. + + @type contextFactory: L{None} or + L{ClientContextFactory } + @param contextFactory: The context factory with which to negotiate TLS. + If not provided, try to create a new one. + + @rtype: L{Deferred } which successfully results in + L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes} to + L{None} or fails with L{TLSError} + @return: A deferred which fires when the transport has been + secured according to the given context factory with the server + capabilities, or which fails with a TLS error if the transport + cannot be secured. + """ + tls = interfaces.ITLSTransport(self.transport, None) + if tls is None: + return defer.fail(TLSError( + "POP3Client transport does not implement " + "interfaces.ITLSTransport")) + + if contextFactory is None: + contextFactory = self._getContextFactory() + + if contextFactory is None: + return defer.fail(TLSError( + "POP3Client requires a TLS context to " + "initiate the STLS handshake")) + + d = self.capabilities() + d.addCallback(self._startTLS, contextFactory, tls) + return d + + + def _startTLS(self, caps, contextFactory, tls): + """ + Continue the process of switching to encrypted communication. + + This callback function runs after the server capabilities are received. + + The next step is sending the server an STLS command to request a + switch to encrypted communication. When an OK response is received, + the L{_startedTLS} callback function completes the switch to encrypted + communication. Then, the new server capabilities are requested. + + @type caps: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or + L{bytes} to L{None} + @param caps: The server capabilities. + + @type contextFactory: L{ClientContextFactory + } + @param contextFactory: A context factory with which to negotiate TLS. + + @type tls: L{ITLSTransport } + @param tls: A TCP transport that supports switching to TLS midstream. + + @rtype: L{Deferred } which successfully triggers with + L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes} to + L{None} or fails with L{TLSNotSupportedError} + @return: A deferred which successfully fires when the response from + the server to the request to start TLS has been received and the + new server capabilities have been received or fails when the server + does not support TLS. + """ + assert not self.startedTLS, "Client and Server are currently communicating via TLS" + + if b'STLS' not in caps: + return defer.fail(TLSNotSupportedError( + "Server does not support secure communication " + "via TLS / SSL")) + + d = self.sendShort(b'STLS', None) + d.addCallback(self._startedTLS, contextFactory, tls) + d.addCallback(lambda _: self.capabilities()) + return d + + + def _startedTLS(self, result, context, tls): + """ + Complete the process of switching to encrypted communication. + + This callback function runs after the response to the STLS command has + been received. + + The final steps are discarding the cached capabilities and initiating + TLS negotiation on the transport. + + @type result: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or + L{bytes} to L{None} + @param result: The server capabilities. + + @type context: L{ClientContextFactory + } + @param context: A context factory with which to negotiate TLS. + + @type tls: L{ITLSTransport } + @param tls: A TCP transport that supports switching to TLS midstream. + + @rtype: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes} + to L{None} + @return: The server capabilities. + """ + self.transport = tls + self.transport.startTLS(context) + self._capCache = None + self.startedTLS = True + return result + + + def _getContextFactory(self): + """ + Get a context factory with which to negotiate TLS. + + @rtype: L{None} or + L{ClientContextFactory } + @return: A context factory or L{None} if TLS is not supported on the + client. + """ + try: + from twisted.internet import ssl + except ImportError: + return None + else: + context = ssl.ClientContextFactory() + context.method = ssl.SSL.TLSv1_METHOD + return context + + + def login(self, username, password): + """ + Log in to the server. + + If APOP is available it will be used. Otherwise, if TLS is + available, an encrypted session will be started and plaintext + login will proceed. Otherwise, if L{allowInsecureLogin} is set, + insecure plaintext login will proceed. Otherwise, + L{InsecureAuthenticationDisallowed} will be raised. + + The first step of logging into the server is obtaining the server's + capabilities. When that is complete, the L{_login} callback function + continues the login process. + + @type username: L{bytes} + @param username: The username with which to log in. + + @type password: L{bytes} + @param password: The password with which to log in. + + @rtype: L{Deferred } which successfully fires with + L{bytes} + @return: A deferred which fires when the login process is complete. + On a successful login, it returns the server's response minus the + status indicator. + """ + d = self.capabilities() + d.addCallback(self._login, username, password) + return d + + + def _login(self, caps, username, password): + """ + Continue the process of logging in to the server. + + This callback function runs after the server capabilities are received. + + If the server provided a challenge in the greeting, proceed with an + APOP login. Otherwise, if the server and the transport support + encrypted communication, try to switch to TLS and then complete + the login process with the L{_loginTLS} callback function. Otherwise, + if insecure authentication is allowed, do a plaintext login. + Otherwise, fail with an L{InsecureAuthenticationDisallowed} error. + + @type caps: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or + L{bytes} to L{None} + @param caps: The server capabilities. + + @type username: L{bytes} + @param username: The username with which to log in. + + @type password: L{bytes} + @param password: The password with which to log in. + + @rtype: L{Deferred } which successfully fires with + L{bytes} + @return: A deferred which fires when the login process is complete. + On a successful login, it returns the server's response minus the + status indicator. + """ + if self.serverChallenge is not None: + return self._apop(username, password, self.serverChallenge) + + tryTLS = b'STLS' in caps + + # If our transport supports switching to TLS, we might want to + # try to switch to TLS. + tlsableTransport = interfaces.ITLSTransport(self.transport, None) is not None + + # If our transport is not already using TLS, we might want to + # try to switch to TLS. + nontlsTransport = interfaces.ISSLTransport(self.transport, None) is None + + if not self.startedTLS and tryTLS and tlsableTransport and nontlsTransport: + d = self.startTLS() + + d.addCallback(self._loginTLS, username, password) + return d + + elif self.startedTLS or not nontlsTransport or self.allowInsecureLogin: + return self._plaintext(username, password) + else: + return defer.fail(InsecureAuthenticationDisallowed()) + + + def _loginTLS(self, res, username, password): + """ + Do a plaintext login over an encrypted transport. + + This callback function runs after the transport switches to encrypted + communication. + + @type res: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or + L{bytes} to L{None} + @param res: The server capabilities. + + @type username: L{bytes} + @param username: The username with which to log in. + + @type password: L{bytes} + @param password: The password with which to log in. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server accepts the username + and password or fails when the server rejects either. On a + successful login, it returns the server's response minus the + status indicator. + """ + return self._plaintext(username, password) + + + def _plaintext(self, username, password): + """ + Perform a plaintext login. + + @type username: L{bytes} + @param username: The username with which to log in. + + @type password: L{bytes} + @param password: The password with which to log in. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server accepts the username + and password or fails when the server rejects either. On a + successful login, it returns the server's response minus the + status indicator. + """ + return self.user(username).addCallback(lambda r: self.password(password)) + + + def _apop(self, username, password, challenge): + """ + Perform an APOP login. + + @type username: L{bytes} + @param username: The username with which to log in. + + @type password: L{bytes} + @param password: The password with which to log in. + + @type challenge: L{bytes} + @param challenge: A challenge string. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On a successful login, it returns the server response minus + the status indicator. + """ + digest = md5(challenge + password).hexdigest().encode("ascii") + return self.apop(username, digest) + + + def apop(self, username, digest): + """ + Send an APOP command to perform authenticated login. + + This should be used in special circumstances only, when it is + known that the server supports APOP authentication, and APOP + authentication is absolutely required. For the common case, + use L{login} instead. + + @type username: L{bytes} + @param username: The username with which to log in. + + @type digest: L{bytes} + @param digest: The challenge response to authenticate with. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On an OK response, the deferred succeeds with the server + response minus the status indicator. On an ERR response, the + deferred fails with a server error response failure. + """ + return self.sendShort(b'APOP', username + b' ' + digest) + + + def user(self, username): + """ + Send a USER command to perform the first half of plaintext login. + + Unless this is absolutely required, use the L{login} method instead. + + @type username: L{bytes} + @param username: The username with which to log in. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On an OK response, the deferred succeeds with the server + response minus the status indicator. On an ERR response, the + deferred fails with a server error response failure. + """ + return self.sendShort(b'USER', username) + + + def password(self, password): + """ + Send a PASS command to perform the second half of plaintext login. + + Unless this is absolutely required, use the L{login} method instead. + + @type password: L{bytes} + @param password: The plaintext password with which to authenticate. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On an OK response, the deferred succeeds with the server + response minus the status indicator. On an ERR response, the + deferred fails with a server error response failure. + """ + return self.sendShort(b'PASS', password) + + + def delete(self, index): + """ + Send a DELE command to delete a message from the server. + + @type index: L{int} + @param index: The 0-based index of the message to delete. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On an OK response, the deferred succeeds with the server + response minus the status indicator. On an ERR response, the + deferred fails with a server error response failure. + """ + return self.sendShort(b'DELE', intToBytes(index + 1)) + + + def _consumeOrSetItem(self, cmd, args, consumer, xform): + """ + Send a command to which a long response is expected and process the + multi-line response into a list accounting for deleted messages. + + @type cmd: L{bytes} + @param cmd: A POP3 command to which a long response is expected. + + @type args: L{bytes} + @param args: The command arguments. + + @type consumer: L{None} or callable that takes + L{object} + @param consumer: L{None} or a function that consumes the output from + the transform function. + + @type xform: L{None}, callable that takes + L{bytes} and returns 2-L{tuple} of (0) L{int}, (1) L{object}, + or callable that takes L{bytes} and returns L{object} + @param xform: A function that parses a line from a multi-line response + and transforms the values into usable form for input to the + consumer function. If no consumer function is specified, the + output must be a message index and corresponding value. If no + transform function is specified, the line is used as is. + + @rtype: L{Deferred } which fires with L{list} of + L{object} or callable that takes L{list} of L{object} + @return: A deferred which fires when the entire response has been + received. When a consumer is not provided, the return value is a + list of the value for each message or L{None} for deleted messages. + Otherwise, it returns the consumer itself. + """ + if consumer is None: + L = [] + consumer = _ListSetter(L).setitem + return self.sendLong(cmd, args, consumer, xform).addCallback(lambda r: L) + return self.sendLong(cmd, args, consumer, xform) + + + def _consumeOrAppend(self, cmd, args, consumer, xform): + """ + Send a command to which a long response is expected and process the + multi-line response into a list. + + @type cmd: L{bytes} + @param cmd: A POP3 command which expects a long response. + + @type args: L{bytes} + @param args: The command arguments. + + @type consumer: L{None} or callable that takes + L{object} + @param consumer: L{None} or a function that consumes the output from the + transform function. + + @type xform: L{None} or callable that takes + L{bytes} and returns L{object} + @param xform: A function that transforms a line from a multi-line + response into usable form for input to the consumer function. If + no transform function is specified, the line is used as is. + + @rtype: L{Deferred } which fires with L{list} of + 2-L{tuple} of (0) L{int}, (1) L{object} or callable that + takes 2-L{tuple} of (0) L{int}, (1) L{object} + @return: A deferred which fires when the entire response has been + received. When a consumer is not provided, the return value is a + list of the transformed lines. Otherwise, it returns the consumer + itself. + """ + if consumer is None: + L = [] + consumer = L.append + return self.sendLong(cmd, args, consumer, xform).addCallback(lambda r: L) + return self.sendLong(cmd, args, consumer, xform) + + + def capabilities(self, useCache=True): + """ + Send a CAPA command to retrieve the capabilities supported by + the server. + + Not all servers support this command. If the server does not + support this, it is treated as though it returned a successful + response listing no capabilities. At some future time, this may be + changed to instead seek out information about a server's + capabilities in some other fashion (only if it proves useful to do + so, and only if there are servers still in use which do not support + CAPA but which do support POP3 extensions that are useful). + + @type useCache: L{bool} + @param useCache: A flag that determines whether previously retrieved + results should be used if available. + + @rtype: L{Deferred } which successfully results in + L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes} to + L{None} + @return: A deferred which fires with a mapping of capability name to + parameters. For example:: + + C: CAPA + S: +OK Capability list follows + S: TOP + S: USER + S: SASL CRAM-MD5 KERBEROS_V4 + S: RESP-CODES + S: LOGIN-DELAY 900 + S: PIPELINING + S: EXPIRE 60 + S: UIDL + S: IMPLEMENTATION Shlemazle-Plotz-v302 + S: . + + will be lead to a result of:: + + | {'TOP': None, + | 'USER': None, + | 'SASL': ['CRAM-MD5', 'KERBEROS_V4'], + | 'RESP-CODES': None, + | 'LOGIN-DELAY': ['900'], + | 'PIPELINING': None, + | 'EXPIRE': ['60'], + | 'UIDL': None, + | 'IMPLEMENTATION': ['Shlemazle-Plotz-v302']} + """ + if useCache and self._capCache is not None: + return defer.succeed(self._capCache) + + cache = {} + def consume(line): + tmp = line.split() + if len(tmp) == 1: + cache[tmp[0]] = None + elif len(tmp) > 1: + cache[tmp[0]] = tmp[1:] + + def capaNotSupported(err): + err.trap(ServerErrorResponse) + return None + + def gotCapabilities(result): + self._capCache = cache + return cache + + d = self._consumeOrAppend(b'CAPA', None, consume, None) + d.addErrback(capaNotSupported).addCallback(gotCapabilities) + return d + + + def noop(self): + """ + Send a NOOP command asking the server to do nothing but respond. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On an OK response, the deferred succeeds with the server + response minus the status indicator. On an ERR response, the + deferred fails with a server error response failure. + """ + return self.sendShort(b"NOOP", None) + + + def reset(self): + """ + Send a RSET command to unmark any messages that have been flagged + for deletion on the server. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On an OK response, the deferred succeeds with the server + response minus the status indicator. On an ERR response, the + deferred fails with a server error response failure. + """ + return self.sendShort(b"RSET", None) + + + def retrieve(self, index, consumer=None, lines=None): + """ + Send a RETR or TOP command to retrieve all or part of a message from + the server. + + @type index: L{int} + @param index: A 0-based message index. + + @type consumer: L{None} or callable that takes + L{bytes} + @param consumer: A function which consumes each transformed line from a + multi-line response as it is received. + + @type lines: L{None} or L{int} + @param lines: If specified, the number of lines of the message to be + retrieved. Otherwise, the entire message is retrieved. + + @rtype: L{Deferred } which fires with L{list} of + L{bytes}, or callable that takes 2-L{tuple} of (0) L{int}, + (1) L{object} + @return: A deferred which fires when the entire response has been + received. When a consumer is not provided, the return value is a + list of the transformed lines. Otherwise, it returns the consumer + itself. + """ + idx = intToBytes(index + 1) + if lines is None: + return self._consumeOrAppend(b'RETR', idx, consumer, _dotUnquoter) + + return self._consumeOrAppend(b'TOP', idx + b' ' + intToBytes(lines), + consumer, _dotUnquoter) + + + def stat(self): + """ + Send a STAT command to get information about the size of the mailbox. + + @rtype: L{Deferred } which successfully fires with + a 2-tuple of (0) L{int}, (1) L{int} or fails with + L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On an OK response, the deferred succeeds with the number of + messages in the mailbox and the size of the mailbox in octets. + On an ERR response, the deferred fails with a server error + response failure. + """ + return self.sendShort(b'STAT', None).addCallback(_statXform) + + + def listSize(self, consumer=None): + """ + Send a LIST command to retrieve the sizes of all messages on the + server. + + @type consumer: L{None} or callable that takes + 2-L{tuple} of (0) L{int}, (1) L{int} + @param consumer: A function which consumes the 0-based message index + and message size derived from the server response. + + @rtype: L{Deferred } which fires L{list} of L{int} or + callable that takes 2-L{tuple} of (0) L{int}, (1) L{int} + @return: A deferred which fires when the entire response has been + received. When a consumer is not provided, the return value is a + list of message sizes. Otherwise, it returns the consumer itself. + """ + return self._consumeOrSetItem(b'LIST', None, consumer, _listXform) + + + def listUID(self, consumer=None): + """ + Send a UIDL command to retrieve the UIDs of all messages on the server. + + @type consumer: L{None} or callable that takes + 2-L{tuple} of (0) L{int}, (1) L{bytes} + @param consumer: A function which consumes the 0-based message index + and UID derived from the server response. + + @rtype: L{Deferred } which fires with L{list} of + L{object} or callable that takes 2-L{tuple} of (0) L{int}, + (1) L{bytes} + @return: A deferred which fires when the entire response has been + received. When a consumer is not provided, the return value is a + list of message sizes. Otherwise, it returns the consumer itself. + """ + return self._consumeOrSetItem(b'UIDL', None, consumer, _uidXform) + + + def quit(self): + """ + Send a QUIT command to disconnect from the server. + + @rtype: L{Deferred } which successfully fires with + L{bytes} or fails with L{ServerErrorResponse} + @return: A deferred which fires when the server response is received. + On an OK response, the deferred succeeds with the server + response minus the status indicator. On an ERR response, the + deferred fails with a server error response failure. + """ + return self.sendShort(b'QUIT', None) + +__all__ = [] diff --git a/contrib/python/Twisted/py2/twisted/mail/protocols.py b/contrib/python/Twisted/py2/twisted/mail/protocols.py new file mode 100644 index 00000000000..25ab7661bb6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/protocols.py @@ -0,0 +1,404 @@ +# -*- test-case-name: twisted.mail.test.test_mail -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Mail protocol support. +""" + +from __future__ import absolute_import, division + +from twisted.mail import pop3 +from twisted.mail import smtp +from twisted.internet import protocol +from twisted.internet import defer +from twisted.copyright import longversion +from twisted.python import log + +from twisted.cred.credentials import CramMD5Credentials, UsernamePassword +from twisted.cred.error import UnauthorizedLogin + +from twisted.mail import relay + +from zope.interface import implementer + + + +@implementer(smtp.IMessageDelivery) +class DomainDeliveryBase: + """ + A base class for message delivery using the domains of a mail service. + + @ivar service: See L{__init__} + @ivar user: See L{__init__} + @ivar host: See L{__init__} + + @type protocolName: L{bytes} + @ivar protocolName: The protocol being used to deliver the mail. + Sub-classes should set this appropriately. + """ + service = None + protocolName = None + + def __init__(self, service, user, host=smtp.DNSNAME): + """ + @type service: L{MailService} + @param service: A mail service. + + @type user: L{bytes} or L{None} + @param user: The authenticated SMTP user. + + @type host: L{bytes} + @param host: The hostname. + """ + self.service = service + self.user = user + self.host = host + + + def receivedHeader(self, helo, origin, recipients): + """ + Generate a received header string for a message. + + @type helo: 2-L{tuple} of (L{bytes}, L{bytes}) + @param helo: The client's identity as sent in the HELO command and its + IP address. + + @type origin: L{Address} + @param origin: The origination address of the message. + + @type recipients: L{list} of L{User} + @param recipients: The destination addresses for the message. + + @rtype: L{bytes} + @return: A received header string. + """ + authStr = heloStr = b"" + if self.user: + authStr = b" auth=" + self.user.encode('xtext') + if helo[0]: + heloStr = b" helo=" + helo[0] + fromUser = (b"from " + helo[0] + b" ([" + helo[1] + b"]" + + heloStr + authStr) + by = (b"by " + self.host + b" with " + self.protocolName + + b" (" + longversion.encode("ascii") + b")") + forUser = (b"for <" + b' '.join(map(bytes, recipients)) + b"> " + + smtp.rfc822date()) + return (b"Received: " + fromUser + b"\n\t" + by + + b"\n\t" + forUser) + + + def validateTo(self, user): + """ + Validate the address for which a message is destined. + + @type user: L{User} + @param user: The destination address. + + @rtype: L{Deferred } which successfully fires with + no-argument callable which returns L{IMessage } + provider. + @return: A deferred which successfully fires with a no-argument + callable which returns a message receiver for the destination. + + @raise SMTPBadRcpt: When messages cannot be accepted for the + destination address. + """ + # XXX - Yick. This needs cleaning up. + if self.user and self.service.queue: + d = self.service.domains.get(user.dest.domain, None) + if d is None: + d = relay.DomainQueuer(self.service, True) + else: + d = self.service.domains[user.dest.domain] + return defer.maybeDeferred(d.exists, user) + + + def validateFrom(self, helo, origin): + """ + Validate the address from which a message originates. + + @type helo: 2-L{tuple} of (L{bytes}, L{bytes}) + @param helo: The client's identity as sent in the HELO command and its + IP address. + + @type origin: L{Address} + @param origin: The origination address of the message. + + @rtype: L{Address} + @return: The origination address. + + @raise SMTPBadSender: When messages cannot be accepted from the + origination address. + """ + if not helo: + raise smtp.SMTPBadSender(origin, 503, + "Who are you? Say HELO first.") + if origin.local != b'' and origin.domain == b'': + raise smtp.SMTPBadSender(origin, 501, + "Sender address must contain domain.") + return origin + + + +class SMTPDomainDelivery(DomainDeliveryBase): + """ + A domain delivery base class for use in an SMTP server. + """ + protocolName = b'smtp' + + + +class ESMTPDomainDelivery(DomainDeliveryBase): + """ + A domain delivery base class for use in an ESMTP server. + """ + protocolName = b'esmtp' + + + +class SMTPFactory(smtp.SMTPFactory): + """ + An SMTP server protocol factory. + + @ivar service: See L{__init__} + @ivar portal: See L{__init__} + + @type protocol: no-argument callable which returns a L{Protocol + } subclass + @ivar protocol: A callable which creates a protocol. The default value is + L{SMTP}. + """ + protocol = smtp.SMTP + portal = None + + def __init__(self, service, portal = None): + """ + @type service: L{MailService} + @param service: An email service. + + @type portal: L{Portal } or + L{None} + @param portal: A portal to use for authentication. + """ + smtp.SMTPFactory.__init__(self) + self.service = service + self.portal = portal + + + def buildProtocol(self, addr): + """ + Create an instance of an SMTP server protocol. + + @type addr: L{IAddress } provider + @param addr: The address of the SMTP client. + + @rtype: L{SMTP} + @return: An SMTP protocol. + """ + log.msg('Connection from %s' % (addr,)) + p = smtp.SMTPFactory.buildProtocol(self, addr) + p.service = self.service + p.portal = self.portal + return p + + + +class ESMTPFactory(SMTPFactory): + """ + An ESMTP server protocol factory. + + @type protocol: no-argument callable which returns a L{Protocol + } subclass + @ivar protocol: A callable which creates a protocol. The default value is + L{ESMTP}. + + @type context: L{IOpenSSLContextFactory + } or L{None} + @ivar context: A factory to generate contexts to be used in negotiating + encrypted communication. + + @type challengers: L{dict} mapping L{bytes} to no-argument callable which + returns L{ICredentials } + subclass provider. + @ivar challengers: A mapping of acceptable authorization mechanism to + callable which creates credentials to use for authentication. + """ + protocol = smtp.ESMTP + context = None + + def __init__(self, *args): + """ + @param args: Arguments for L{SMTPFactory.__init__} + + @see: L{SMTPFactory.__init__} + """ + SMTPFactory.__init__(self, *args) + self.challengers = { + b'CRAM-MD5': CramMD5Credentials + } + + + def buildProtocol(self, addr): + """ + Create an instance of an ESMTP server protocol. + + @type addr: L{IAddress } provider + @param addr: The address of the ESMTP client. + + @rtype: L{ESMTP} + @return: An ESMTP protocol. + """ + p = SMTPFactory.buildProtocol(self, addr) + p.challengers = self.challengers + p.ctx = self.context + return p + + + +class VirtualPOP3(pop3.POP3): + """ + A virtual hosting POP3 server. + + @type service: L{MailService} + @ivar service: The email service that created this server. This must be + set by the service. + + @type domainSpecifier: L{bytes} + @ivar domainSpecifier: The character to use to split an email address into + local-part and domain. The default is '@'. + """ + service = None + + domainSpecifier = b'@' # Gaagh! I hate POP3. No standardized way + # to indicate user@host. '@' doesn't work + # with NS, e.g. + + def authenticateUserAPOP(self, user, digest): + """ + Perform APOP authentication. + + Override the default lookup scheme to allow virtual domains. + + @type user: L{bytes} + @param user: The name of the user attempting to log in. + + @type digest: L{bytes} + @param digest: The challenge response. + + @rtype: L{Deferred} which successfully results in 3-L{tuple} of + (L{IMailbox }, L{IMailbox } + provider, no-argument callable) + @return: A deferred which fires when authentication is complete. + If successful, it returns an L{IMailbox } interface, + a mailbox and a logout function. If authentication fails, the + deferred fails with an L{UnauthorizedLogin + } error. + """ + user, domain = self.lookupDomain(user) + try: + portal = self.service.lookupPortal(domain) + except KeyError: + return defer.fail(UnauthorizedLogin()) + else: + return portal.login( + pop3.APOPCredentials(self.magic, user, digest), + None, + pop3.IMailbox + ) + + + def authenticateUserPASS(self, user, password): + """ + Perform authentication for a username/password login. + + Override the default lookup scheme to allow virtual domains. + + @type user: L{bytes} + @param user: The name of the user attempting to log in. + + @type password: L{bytes} + @param password: The password to authenticate with. + + @rtype: L{Deferred} which successfully results in 3-L{tuple} of + (L{IMailbox }, L{IMailbox } + provider, no-argument callable) + @return: A deferred which fires when authentication is complete. + If successful, it returns an L{IMailbox } interface, + a mailbox and a logout function. If authentication fails, the + deferred fails with an L{UnauthorizedLogin + } error. + """ + user, domain = self.lookupDomain(user) + try: + portal = self.service.lookupPortal(domain) + except KeyError: + return defer.fail(UnauthorizedLogin()) + else: + return portal.login( + UsernamePassword(user, password), + None, + pop3.IMailbox + ) + + + def lookupDomain(self, user): + """ + Check whether a domain is among the virtual domains supported by the + mail service. + + @type user: L{bytes} + @param user: An email address. + + @rtype: 2-L{tuple} of (L{bytes}, L{bytes}) + @return: The local part and the domain part of the email address if the + domain is supported. + + @raise POP3Error: When the domain is not supported by the mail service. + """ + try: + user, domain = user.split(self.domainSpecifier, 1) + except ValueError: + domain = b'' + if domain not in self.service.domains: + raise pop3.POP3Error( + "no such domain {}".format(domain.decode("utf-8"))) + return user, domain + + + +class POP3Factory(protocol.ServerFactory): + """ + A POP3 server protocol factory. + + @ivar service: See L{__init__} + + @type protocol: no-argument callable which returns a L{Protocol + } subclass + @ivar protocol: A callable which creates a protocol. The default value is + L{VirtualPOP3}. + """ + protocol = VirtualPOP3 + service = None + + def __init__(self, service): + """ + @type service: L{MailService} + @param service: An email service. + """ + self.service = service + + + def buildProtocol(self, addr): + """ + Create an instance of a POP3 server protocol. + + @type addr: L{IAddress } provider + @param addr: The address of the POP3 client. + + @rtype: L{POP3} + @return: A POP3 protocol. + """ + p = protocol.ServerFactory.buildProtocol(self, addr) + p.service = self.service + return p diff --git a/contrib/python/Twisted/py2/twisted/mail/relay.py b/contrib/python/Twisted/py2/twisted/mail/relay.py new file mode 100644 index 00000000000..ee4770d077e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/relay.py @@ -0,0 +1,180 @@ +# -*- test-case-name: twisted.mail.test.test_mail -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for relaying mail. +""" + +from twisted.mail import smtp +from twisted.python import log +from twisted.internet.address import UNIXAddress + +import os + +try: + import cPickle as pickle +except ImportError: + import pickle + + + +class DomainQueuer: + """ + An SMTP domain which add messages to a queue intended for relaying. + """ + + def __init__(self, service, authenticated=False): + self.service = service + self.authed = authenticated + + + def exists(self, user): + """ + Check whether mail can be relayed to a user. + + @type user: L{User} + @param user: A user. + + @rtype: no-argument callable which returns L{IMessage } + provider + @return: A function which takes no arguments and returns a message + receiver for the user. + + @raise SMTPBadRcpt: When mail cannot be relayed to the user. + """ + if self.willRelay(user.dest, user.protocol): + # The most cursor form of verification of the addresses + orig = filter(None, str(user.orig).split('@', 1)) + dest = filter(None, str(user.dest).split('@', 1)) + if len(orig) == 2 and len(dest) == 2: + return lambda: self.startMessage(user) + raise smtp.SMTPBadRcpt(user) + + + def willRelay(self, address, protocol): + """ + Check whether we agree to relay. + + The default is to relay for all connections over UNIX + sockets and all connections from localhost. + """ + peer = protocol.transport.getPeer() + return (self.authed or isinstance(peer, UNIXAddress) or + peer.host == '127.0.0.1') + + + def startMessage(self, user): + """ + Create an envelope and a message receiver for the relay queue. + + @type user: L{User} + @param user: A user. + + @rtype: L{IMessage } + @return: A message receiver. + """ + queue = self.service.queue + envelopeFile, smtpMessage = queue.createNewMessage() + with envelopeFile: + log.msg('Queueing mail %r -> %r' % (str(user.orig), + str(user.dest))) + pickle.dump([str(user.orig), str(user.dest)], envelopeFile) + return smtpMessage + + + +class RelayerMixin: + + # XXX - This is -totally- bogus + # It opens about a -hundred- -billion- files + # and -leaves- them open! + + def loadMessages(self, messagePaths): + self.messages = [] + self.names = [] + for message in messagePaths: + with open(message + '-H') as fp: + messageContents = pickle.load(fp) + fp = open(message + '-D') + messageContents.append(fp) + self.messages.append(messageContents) + self.names.append(message) + + + def getMailFrom(self): + if not self.messages: + return None + return self.messages[0][0] + + + def getMailTo(self): + if not self.messages: + return None + return [self.messages[0][1]] + + + def getMailData(self): + if not self.messages: + return None + return self.messages[0][2] + + + def sentMail(self, code, resp, numOk, addresses, log): + """Since we only use one recipient per envelope, this + will be called with 0 or 1 addresses. We probably want + to do something with the error message if we failed. + """ + if code in smtp.SUCCESS: + # At least one, i.e. all, recipients successfully delivered + os.remove(self.names[0] + '-D') + os.remove(self.names[0] + '-H') + del self.messages[0] + del self.names[0] + + + +class SMTPRelayer(RelayerMixin, smtp.SMTPClient): + """ + A base class for SMTP relayers. + """ + def __init__(self, messagePaths, *args, **kw): + """ + @type messagePaths: L{list} of L{bytes} + @param messagePaths: The base filename for each message to be relayed. + + @type args: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of + (0) L{bytes}, (1) L{int} + @param args: Positional arguments for L{SMTPClient.__init__} + + @type kw: L{dict} + @param kw: Keyword arguments for L{SMTPClient.__init__} + """ + smtp.SMTPClient.__init__(self, *args, **kw) + self.loadMessages(messagePaths) + + + +class ESMTPRelayer(RelayerMixin, smtp.ESMTPClient): + """ + A base class for ESMTP relayers. + """ + def __init__(self, messagePaths, *args, **kw): + """ + @type messagePaths: L{list} of L{bytes} + @param messagePaths: The base filename for each message to be relayed. + + @type args: 3-L{tuple} of (0) L{bytes}, (1) L{None} or + L{ClientContextFactory + }, + (2) L{bytes} or 4-L{tuple} of (0) L{bytes}, (1) L{None} + or L{ClientContextFactory + }, (2) L{bytes}, + (3) L{int} + @param args: Positional arguments for L{ESMTPClient.__init__} + + @type kw: L{dict} + @param kw: Keyword arguments for L{ESMTPClient.__init__} + """ + smtp.ESMTPClient.__init__(self, *args, **kw) + self.loadMessages(messagePaths) diff --git a/contrib/python/Twisted/py2/twisted/mail/relaymanager.py b/contrib/python/Twisted/py2/twisted/mail/relaymanager.py new file mode 100644 index 00000000000..312146abc37 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/relaymanager.py @@ -0,0 +1,1161 @@ +# -*- test-case-name: twisted.mail.test.test_mail -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Infrastructure for relaying mail through a smart host. + +Traditional peer-to-peer email has been increasingly replaced by smart host +configurations. Instead of sending mail directly to the recipient, a sender +sends mail to a smart host. The smart host finds the mail exchange server for +the recipient and sends on the message. +""" + +import email.utils +import os +import time + +try: + import cPickle as pickle +except ImportError: + import pickle + +from twisted.python import log +from twisted.python.failure import Failure +from twisted.mail import relay +from twisted.mail import bounce +from twisted.internet import protocol +from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.error import DNSLookupError +from twisted.mail import smtp +from twisted.application import internet + + + +class ManagedRelayerMixin: + """ + SMTP Relayer which notifies a manager + + Notify the manager about successful mail, failed mail + and broken connections + """ + def __init__(self, manager): + self.manager = manager + + + def sentMail(self, code, resp, numOk, addresses, log): + """ + called when e-mail has been sent + + we will always get 0 or 1 addresses. + """ + message = self.names[0] + if code in smtp.SUCCESS: + self.manager.notifySuccess(self.factory, message) + else: + self.manager.notifyFailure(self.factory, message) + del self.messages[0] + del self.names[0] + + + def connectionLost(self, reason): + """ + called when connection is broken + + notify manager we will try to send no more e-mail + """ + self.manager.notifyDone(self.factory) + + + +class SMTPManagedRelayer(ManagedRelayerMixin, relay.SMTPRelayer): + """ + An SMTP managed relayer. + + This managed relayer is an SMTP client which is responsible for sending a + set of messages and keeping an attempt manager informed about its progress. + + @type factory: L{SMTPManagedRelayerFactory} + @ivar factory: The factory that created this relayer. This must be set by + the factory. + """ + def __init__(self, messages, manager, *args, **kw): + """ + @type messages: L{list} of L{bytes} + @param messages: The base filenames of messages to be relayed. + + @type manager: L{_AttemptManager} + @param manager: An attempt manager. + + @type args: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of + (0) L{bytes}, (1) L{int} + @param args: Positional arguments for L{SMTPClient.__init__} + + @type kw: L{dict} + @param kw: Keyword arguments for L{SMTPClient.__init__} + """ + ManagedRelayerMixin.__init__(self, manager) + relay.SMTPRelayer.__init__(self, messages, *args, **kw) + + + +class ESMTPManagedRelayer(ManagedRelayerMixin, relay.ESMTPRelayer): + """ + An ESMTP managed relayer. + + This managed relayer is an ESMTP client which is responsible for sending a + set of messages and keeping an attempt manager informed about its progress. + """ + def __init__(self, messages, manager, *args, **kw): + """ + @type messages: L{list} of L{bytes} + @param messages: The base filenames of messages to be relayed. + + @type manager: L{_AttemptManager} + @param manager: An attempt manager. + + @type args: 3-L{tuple} of (0) L{bytes}, (1) L{None} or + L{ClientContextFactory + }, (2) L{bytes} or + 4-L{tuple} of (0) L{bytes}, (1) L{None} or + L{ClientContextFactory + }, (2) L{bytes}, + (3) L{int} + @param args: Positional arguments for L{ESMTPClient.__init__} + + @type kw: L{dict} + @param kw: Keyword arguments for L{ESMTPClient.__init__} + """ + ManagedRelayerMixin.__init__(self, manager) + relay.ESMTPRelayer.__init__(self, messages, *args, **kw) + + + +class SMTPManagedRelayerFactory(protocol.ClientFactory): + """ + A factory to create an L{SMTPManagedRelayer}. + + This factory creates a managed relayer which relays a set of messages over + SMTP and informs an attempt manager of its progress. + + @ivar messages: See L{__init__} + @ivar manager: See L{__init__} + + @type protocol: callable which returns L{SMTPManagedRelayer} + @ivar protocol: A callable which returns a managed relayer for SMTP. See + L{SMTPManagedRelayer.__init__} for parameters to the callable. + + @type pArgs: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of + (0) L{bytes}, (1), L{int} + @ivar pArgs: Positional arguments for L{SMTPClient.__init__} + + @type pKwArgs: L{dict} + @ivar pKwArgs: Keyword arguments for L{SMTPClient.__init__} + """ + protocol = SMTPManagedRelayer + + def __init__(self, messages, manager, *args, **kw): + """ + @type messages: L{list} of L{bytes} + @param messages: The base filenames of messages to be relayed. + + @type manager: L{_AttemptManager} + @param manager: An attempt manager. + + @type args: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of + (0) L{bytes}, (1), L{int} + @param args: Positional arguments for L{SMTPClient.__init__} + + @type kw: L{dict} + @param kw: Keyword arguments for L{SMTPClient.__init__} + """ + self.messages = messages + self.manager = manager + self.pArgs = args + self.pKwArgs = kw + + + def buildProtocol(self, addr): + """ + Create an L{SMTPManagedRelayer}. + + @type addr: L{IAddress } provider + @param addr: The address of the SMTP server. + + @rtype: L{SMTPManagedRelayer} + @return: A managed relayer for SMTP. + """ + protocol = self.protocol(self.messages, self.manager, *self.pArgs, + **self.pKwArgs) + protocol.factory = self + return protocol + + + def clientConnectionFailed(self, connector, reason): + """ + Notify the attempt manager that a connection could not be established. + + @type connector: L{IConnector } + provider + @param connector: A connector. + + @type reason: L{Failure} + @param reason: The reason the connection attempt failed. + """ + self.manager.notifyNoConnection(self) + self.manager.notifyDone(self) + + + +class ESMTPManagedRelayerFactory(SMTPManagedRelayerFactory): + """ + A factory to create an L{ESMTPManagedRelayer}. + + This factory creates a managed relayer which relays a set of messages over + ESMTP and informs an attempt manager of its progress. + + @type protocol: callable which returns L{ESMTPManagedRelayer} + @ivar protocol: A callable which returns a managed relayer for ESMTP. See + L{ESMTPManagedRelayer.__init__} for parameters to the callable. + + @ivar secret: See L{__init__} + @ivar contextFactory: See L{__init__} + """ + protocol = ESMTPManagedRelayer + + def __init__(self, messages, manager, secret, contextFactory, *args, **kw): + """ + @type messages: L{list} of L{bytes} + @param messages: The base filenames of messages to be relayed. + + @type manager: L{_AttemptManager} + @param manager: An attempt manager. + + @type secret: L{bytes} + @param secret: A string for the authentication challenge response. + + @type contextFactory: L{None} or + L{ClientContextFactory } + @param contextFactory: An SSL context factory. + + @type args: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of + (0) L{bytes}, (1), L{int} + @param args: Positional arguments for L{SMTPClient.__init__} + + @type pKwArgs: L{dict} + @param pKwArgs: Keyword arguments for L{SMTPClient.__init__} + """ + self.secret = secret + self.contextFactory = contextFactory + SMTPManagedRelayerFactory.__init__(self, messages, manager, *args, + **kw) + + + def buildProtocol(self, addr): + """ + Create an L{ESMTPManagedRelayer}. + + @type addr: L{IAddress } provider + @param addr: The address of the ESMTP server. + + @rtype: L{ESMTPManagedRelayer} + @return: A managed relayer for ESMTP. + """ + s = self.secret and self.secret(addr) + protocol = self.protocol(self.messages, self.manager, s, + self.contextFactory, *self.pArgs, **self.pKwArgs) + protocol.factory = self + return protocol + + + +class Queue: + """ + A queue for messages to be relayed. + + @ivar directory: See L{__init__} + + @type n: L{int} + @ivar n: A number used to form unique filenames. + + @type waiting: L{dict} of L{bytes} + @ivar waiting: The base filenames of messages waiting to be relayed. + + @type relayed: L{dict} of L{bytes} + @ivar relayed: The base filenames of messages in the process of being + relayed. + + @type noisy: L{bool} + @ivar noisy: A flag which determines whether informational log messages + will be generated (C{True}) or not (C{False}). + """ + noisy = True + + def __init__(self, directory): + """ + Initialize non-volatile state. + + @type directory: L{bytes} + @param directory: The pathname of the directory holding messages in the + queue. + """ + self.directory = directory + self._init() + + + def _init(self): + """ + Initialize volatile state. + """ + self.n = 0 + self.waiting = {} + self.relayed = {} + self.readDirectory() + + + def __getstate__(self): + """ + Create a representation of the non-volatile state of the queue. + + @rtype: L{dict} mapping L{bytes} to L{object} + @return: The non-volatile state of the queue. + """ + return {'directory': self.directory} + + + def __setstate__(self, state): + """ + Restore the non-volatile state of the queue and recreate the volatile + state. + + @type state: L{dict} mapping L{bytes} to L{object} + @param state: The non-volatile state of the queue. + """ + self.__dict__.update(state) + self._init() + + + def readDirectory(self): + """ + Scan the message directory for new messages. + """ + for message in os.listdir(self.directory): + # Skip non data files + if message[-2:] != '-D': + continue + self.addMessage(message[:-2]) + + + def getWaiting(self): + """ + Return the base filenames of messages waiting to be relayed. + + @rtype: L{list} of L{bytes} + @return: The base filenames of messages waiting to be relayed. + """ + return self.waiting.keys() + + + def hasWaiting(self): + """ + Return an indication of whether the queue has messages waiting to be + relayed. + + @rtype: L{bool} + @return: C{True} if messages are waiting to be relayed. C{False} + otherwise. + """ + return len(self.waiting) > 0 + + + def getRelayed(self): + """ + Return the base filenames of messages in the process of being relayed. + + @rtype: L{list} of L{bytes} + @return: The base filenames of messages in the process of being + relayed. + """ + return self.relayed.keys() + + + def setRelaying(self, message): + """ + Mark a message as being relayed. + + @type message: L{bytes} + @param message: The base filename of a message. + """ + del self.waiting[message] + self.relayed[message] = 1 + + + def setWaiting(self, message): + """ + Mark a message as waiting to be relayed. + + @type message: L{bytes} + @param message: The base filename of a message. + """ + del self.relayed[message] + self.waiting[message] = 1 + + + def addMessage(self, message): + """ + Mark a message as waiting to be relayed unless it is in the process of + being relayed. + + @type message: L{bytes} + @param message: The base filename of a message. + """ + if message not in self.relayed: + self.waiting[message] = 1 + if self.noisy: + log.msg('Set ' + message + ' waiting') + + + def done(self, message): + """ + Remove a message from the queue. + + @type message: L{bytes} + @param message: The base filename of a message. + """ + message = os.path.basename(message) + os.remove(self.getPath(message) + '-D') + os.remove(self.getPath(message) + '-H') + del self.relayed[message] + + + def getPath(self, message): + """ + Return the full base pathname of a message in the queue. + + @type message: L{bytes} + @param message: The base filename of a message. + + @rtype: L{bytes} + @return: The full base pathname of the message. + """ + return os.path.join(self.directory, message) + + + def getEnvelope(self, message): + """ + Get the envelope for a message. + + @type message: L{bytes} + @param message: The base filename of a message. + + @rtype: L{list} of two L{bytes} + @return: A list containing the origination and destination addresses + for the message. + """ + with self.getEnvelopeFile(message) as f: + return pickle.load(f) + + + def getEnvelopeFile(self, message): + """ + Return the envelope file for a message in the queue. + + @type message: L{bytes} + @param message: The base filename of a message. + + @rtype: L{file} + @return: The envelope file for the message. + """ + return open(os.path.join(self.directory, message + '-H'), 'rb') + + + def createNewMessage(self): + """ + Create a new message in the queue. + + @rtype: 2-L{tuple} of (0) L{file}, (1) L{FileMessage} + @return: The envelope file and a message receiver for a new message in + the queue. + """ + fname = "%s_%s_%s_%s" % (os.getpid(), time.time(), self.n, id(self)) + self.n = self.n + 1 + headerFile = open(os.path.join(self.directory, fname + '-H'), 'wb') + tempFilename = os.path.join(self.directory, fname + '-C') + finalFilename = os.path.join(self.directory, fname + '-D') + messageFile = open(tempFilename, 'wb') + + from twisted.mail.mail import FileMessage + return headerFile, FileMessage(messageFile, tempFilename, + finalFilename) + + + +class _AttemptManager(object): + """ + A manager for an attempt to relay a set of messages to a mail exchange + server. + + @ivar manager: See L{__init__} + + @type _completionDeferreds: L{list} of L{Deferred} + @ivar _completionDeferreds: Deferreds which are to be notified when the + attempt to relay is finished. + """ + def __init__(self, manager, noisy=True, reactor=None): + """ + @type manager: L{SmartHostSMTPRelayingManager} + @param manager: A smart host. + + @type noisy: L{bool} + @param noisy: A flag which determines whether informational log + messages will be generated (L{True}) or not (L{False}). + + @type reactor: L{IReactorTime + } provider + @param reactor: A reactor which will be used to schedule delayed calls. + """ + self.manager = manager + self._completionDeferreds = [] + self.noisy = noisy + + if not reactor: + from twisted.internet import reactor + self.reactor = reactor + + + def getCompletionDeferred(self): + """ + Return a deferred which will fire when the attempt to relay is + finished. + + @rtype: L{Deferred} + @return: A deferred which will fire when the attempt to relay is + finished. + """ + self._completionDeferreds.append(Deferred()) + return self._completionDeferreds[-1] + + + def _finish(self, relay, message): + """ + Remove a message from the relay queue and from the smart host's list of + messages being relayed. + + @type relay: L{SMTPManagedRelayerFactory} + @param relay: The factory for the relayer which sent the message. + + @type message: L{bytes} + @param message: The path of the file holding the message. + """ + self.manager.managed[relay].remove(os.path.basename(message)) + self.manager.queue.done(message) + + + def notifySuccess(self, relay, message): + """ + Remove a message from the relay queue after it has been successfully + sent. + + @type relay: L{SMTPManagedRelayerFactory} + @param relay: The factory for the relayer which sent the message. + + @type message: L{bytes} + @param message: The path of the file holding the message. + """ + if self.noisy: + log.msg("success sending %s, removing from queue" % message) + self._finish(relay, message) + + + def notifyFailure(self, relay, message): + """ + Generate a bounce message for a message which cannot be relayed. + + @type relay: L{SMTPManagedRelayerFactory} + @param relay: The factory for the relayer responsible for the message. + + @type message: L{bytes} + @param message: The path of the file holding the message. + """ + if self.noisy: + log.msg("could not relay " + message) + # Moshe - Bounce E-mail here + # Be careful: if it's a bounced bounce, silently + # discard it + message = os.path.basename(message) + with self.manager.queue.getEnvelopeFile(message) as fp: + from_, to = pickle.load(fp) + from_, to, bounceMessage = bounce.generateBounce( + open(self.manager.queue.getPath(message) + '-D'), from_, to) + fp, outgoingMessage = self.manager.queue.createNewMessage() + with fp: + pickle.dump([from_, to], fp) + for line in bounceMessage.splitlines(): + outgoingMessage.lineReceived(line) + outgoingMessage.eomReceived() + self._finish(relay, self.manager.queue.getPath(message)) + + + def notifyDone(self, relay): + """ + When the connection is lost or cannot be established, prepare to + resend unsent messages and fire all deferred which are waiting for + the completion of the attempt to relay. + + @type relay: L{SMTPManagedRelayerFactory} + @param relay: The factory for the relayer for the connection. + """ + for message in self.manager.managed.get(relay, ()): + if self.noisy: + log.msg("Setting " + message + " waiting") + self.manager.queue.setWaiting(message) + try: + del self.manager.managed[relay] + except KeyError: + pass + notifications = self._completionDeferreds + self._completionDeferreds = None + for d in notifications: + d.callback(None) + + + def notifyNoConnection(self, relay): + """ + When a connection to the mail exchange server cannot be established, + prepare to resend messages later. + + @type relay: L{SMTPManagedRelayerFactory} + @param relay: The factory for the relayer meant to use the connection. + """ + # Back off a bit + try: + msgs = self.manager.managed[relay] + except KeyError: + log.msg("notifyNoConnection passed unknown relay!") + return + + if self.noisy: + log.msg("Backing off on delivery of " + str(msgs)) + + def setWaiting(queue, messages): + map(queue.setWaiting, messages) + self.reactor.callLater(30, setWaiting, self.manager.queue, msgs) + del self.manager.managed[relay] + + + +class SmartHostSMTPRelayingManager: + """ + A smart host which uses SMTP managed relayers to send messages from the + relay queue. + + L{checkState} must be called periodically at which time the state of the + relay queue is checked and new relayers are created as needed. + + In order to relay a set of messages to a mail exchange server, a smart host + creates an attempt manager and a managed relayer factory for that set of + messages. When a connection is made with the mail exchange server, the + managed relayer factory creates a managed relayer to send the messages. + The managed relayer reports on its progress to the attempt manager which, + in turn, updates the smart host's relay queue and information about its + managed relayers. + + @ivar queue: See L{__init__}. + @ivar maxConnections: See L{__init__}. + @ivar maxMessagesPerConnection: See L{__init__}. + + @type fArgs: 3-L{tuple} of (0) L{list} of L{bytes}, + (1) L{_AttemptManager}, (2) L{bytes} or 4-L{tuple} of (0) L{list} + of L{bytes}, (1) L{_AttemptManager}, (2) L{bytes}, (3) L{int} + @ivar fArgs: Positional arguments for + L{SMTPManagedRelayerFactory.__init__}. + + @type fKwArgs: L{dict} + @ivar fKwArgs: Keyword arguments for L{SMTPManagedRelayerFactory.__init__}. + + @type factory: callable which returns L{SMTPManagedRelayerFactory} + @ivar factory: A callable which creates a factory for creating a managed + relayer. See L{SMTPManagedRelayerFactory.__init__} for parameters to + the callable. + + @type PORT: L{int} + @ivar PORT: The port over which to connect to the SMTP server. + + @type mxcalc: L{None} or L{MXCalculator} + @ivar mxcalc: A resource for mail exchange host lookups. + + @type managed: L{dict} mapping L{SMTPManagedRelayerFactory} to L{list} of + L{bytes} + @ivar managed: A mapping of factory for a managed relayer to + filenames of messages the managed relayer is responsible for. + """ + factory = SMTPManagedRelayerFactory + + PORT = 25 + + mxcalc = None + + def __init__(self, queue, maxConnections=2, maxMessagesPerConnection=10): + """ + Initialize a smart host. + + The default values specify connection limits appropriate for a + low-volume smart host. + + @type queue: L{Queue} + @param queue: A relay queue. + + @type maxConnections: L{int} + @param maxConnections: The maximum number of concurrent connections to + SMTP servers. + + @type maxMessagesPerConnection: L{int} + @param maxMessagesPerConnection: The maximum number of messages for + which a relayer will be given responsibility. + """ + self.maxConnections = maxConnections + self.maxMessagesPerConnection = maxMessagesPerConnection + self.managed = {} # SMTP clients we're managing + self.queue = queue + self.fArgs = () + self.fKwArgs = {} + + + def __getstate__(self): + """ + Create a representation of the non-volatile state of this object. + + @rtype: L{dict} mapping L{bytes} to L{object} + @return: The non-volatile state of the queue. + """ + dct = self.__dict__.copy() + del dct['managed'] + return dct + + + def __setstate__(self, state): + """ + Restore the non-volatile state of this object and recreate the volatile + state. + + @type state: L{dict} mapping L{bytes} to L{object} + @param state: The non-volatile state of the queue. + """ + self.__dict__.update(state) + self.managed = {} + + + def checkState(self): + """ + Check the state of the relay queue and, if possible, launch relayers to + handle waiting messages. + + @rtype: L{None} or L{Deferred} + @return: No return value if no further messages can be relayed or a + deferred which fires when all of the SMTP connections initiated by + this call have disconnected. + """ + self.queue.readDirectory() + if (len(self.managed) >= self.maxConnections): + return + if not self.queue.hasWaiting(): + return + + return self._checkStateMX() + + + def _checkStateMX(self): + nextMessages = self.queue.getWaiting() + nextMessages.reverse() + + exchanges = {} + for msg in nextMessages: + from_, to = self.queue.getEnvelope(msg) + name, addr = email.utils.parseaddr(to) + parts = addr.split('@', 1) + if len(parts) != 2: + log.err("Illegal message destination: " + to) + continue + domain = parts[1] + + self.queue.setRelaying(msg) + exchanges.setdefault(domain, []).append(self.queue.getPath(msg)) + if len(exchanges) >= (self.maxConnections - len(self.managed)): + break + + if self.mxcalc is None: + self.mxcalc = MXCalculator() + + relays = [] + for (domain, msgs) in exchanges.iteritems(): + manager = _AttemptManager(self, self.queue.noisy) + factory = self.factory(msgs, manager, *self.fArgs, **self.fKwArgs) + self.managed[factory] = map(os.path.basename, msgs) + relayAttemptDeferred = manager.getCompletionDeferred() + connectSetupDeferred = self.mxcalc.getMX(domain) + connectSetupDeferred.addCallback(lambda mx: str(mx.name)) + connectSetupDeferred.addCallback(self._cbExchange, self.PORT, + factory) + connectSetupDeferred.addErrback(lambda err: ( + relayAttemptDeferred.errback(err), err)[1]) + connectSetupDeferred.addErrback(self._ebExchange, factory, domain) + relays.append(relayAttemptDeferred) + return DeferredList(relays) + + + def _cbExchange(self, address, port, factory): + """ + Initiate a connection with a mail exchange server. + + This callback function runs after mail exchange server for the domain + has been looked up. + + @type address: L{bytes} + @param address: The hostname of a mail exchange server. + + @type port: L{int} + @param port: A port number. + + @type factory: L{SMTPManagedRelayerFactory} + @param factory: A factory which can create a relayer for the mail + exchange server. + """ + from twisted.internet import reactor + reactor.connectTCP(address, port, factory) + + + def _ebExchange(self, failure, factory, domain): + """ + Prepare to resend messages later. + + This errback function runs when no mail exchange server for the domain + can be found. + + @type failure: L{Failure} + @param failure: The reason the mail exchange lookup failed. + + @type factory: L{SMTPManagedRelayerFactory} + @param factory: A factory which can create a relayer for the mail + exchange server. + + @type domain: L{bytes} + @param domain: A domain. + """ + log.err('Error setting up managed relay factory for ' + domain) + log.err(failure) + + def setWaiting(queue, messages): + map(queue.setWaiting, messages) + + from twisted.internet import reactor + reactor.callLater(30, setWaiting, self.queue, self.managed[factory]) + del self.managed[factory] + + + +class SmartHostESMTPRelayingManager(SmartHostSMTPRelayingManager): + """ + A smart host which uses ESMTP managed relayers to send messages from the + relay queue. + + @type factory: callable which returns L{ESMTPManagedRelayerFactory} + @ivar factory: A callable which creates a factory for creating a managed + relayer. See L{ESMTPManagedRelayerFactory.__init__} for parameters to + the callable. + """ + factory = ESMTPManagedRelayerFactory + + + +def _checkState(manager): + """ + Prompt a relaying manager to check state. + + @type manager: L{SmartHostSMTPRelayingManager} + @param manager: A relaying manager. + """ + manager.checkState() + + + +def RelayStateHelper(manager, delay): + """ + Set up a periodic call to prompt a relaying manager to check state. + + @type manager: L{SmartHostSMTPRelayingManager} + @param manager: A relaying manager. + + @type delay: L{float} + @param delay: The number of seconds between calls. + + @rtype: L{TimerService } + @return: A service which periodically reminds a relaying manager to check + state. + """ + return internet.TimerService(delay, _checkState, manager) + + + +class CanonicalNameLoop(Exception): + """ + An error indicating that when trying to look up a mail exchange host, a set + of canonical name records was found which form a cycle and resolution was + abandoned. + """ + + + +class CanonicalNameChainTooLong(Exception): + """ + An error indicating that when trying to look up a mail exchange host, too + many canonical name records which point to other canonical name records + were encountered and resolution was abandoned. + """ + + + +class MXCalculator: + """ + A utility for looking up mail exchange hosts and tracking whether they are + working or not. + + @type clock: L{IReactorTime } + provider + @ivar clock: A reactor which will be used to schedule timeouts. + + @type resolver: L{IResolver } + @ivar resolver: A resolver. + + @type badMXs: L{dict} mapping L{bytes} to L{float} + @ivar badMXs: A mapping of non-functioning mail exchange hostname to time + at which another attempt at contacting it may be made. + + @type timeOutBadMX: L{int} + @ivar timeOutBadMX: Period in seconds between attempts to contact a + non-functioning mail exchange host. + + @type fallbackToDomain: L{bool} + @ivar fallbackToDomain: A flag indicating whether to attempt to use the + hostname directly when no mail exchange can be found (C{True}) or + not (C{False}). + """ + timeOutBadMX = 60 * 60 # One hour + fallbackToDomain = True + + def __init__(self, resolver=None, clock=None): + """ + @type resolver: L{IResolver } + provider or L{None} + @param: A resolver. + + @type clock: L{IReactorTime } + provider or L{None} + @param clock: A reactor which will be used to schedule timeouts. + """ + self.badMXs = {} + if resolver is None: + from twisted.names.client import createResolver + resolver = createResolver() + self.resolver = resolver + if clock is None: + from twisted.internet import reactor as clock + self.clock = clock + + + def markBad(self, mx): + """ + Record that a mail exchange host is not currently functioning. + + @type mx: L{bytes} + @param mx: The hostname of a mail exchange host. + """ + self.badMXs[str(mx)] = self.clock.seconds() + self.timeOutBadMX + + + def markGood(self, mx): + """ + Record that a mail exchange host is functioning. + + @type mx: L{bytes} + @param mx: The hostname of a mail exchange host. + """ + try: + del self.badMXs[mx] + except KeyError: + pass + + + def getMX(self, domain, maximumCanonicalChainLength=3): + """ + Find the name of a host that acts as a mail exchange server + for a domain. + + @type domain: L{bytes} + @param domain: A domain name. + + @type maximumCanonicalChainLength: L{int} + @param maximumCanonicalChainLength: The maximum number of unique + canonical name records to follow while looking up the mail exchange + host. + + @rtype: L{Deferred} which successfully fires with L{Record_MX} + @return: A deferred which succeeds with the MX record for the mail + exchange server for the domain or fails if none can be found. + """ + mailExchangeDeferred = self.resolver.lookupMailExchange(domain) + mailExchangeDeferred.addCallback(self._filterRecords) + mailExchangeDeferred.addCallback( + self._cbMX, domain, maximumCanonicalChainLength) + mailExchangeDeferred.addErrback(self._ebMX, domain) + return mailExchangeDeferred + + + def _filterRecords(self, records): + """ + Organize the records of a DNS response by record name. + + @type records: 3-L{tuple} of (0) L{list} of L{RRHeader + }, (1) L{list} of L{RRHeader + }, (2) L{list} of L{RRHeader + } + @param records: Answer resource records, authority resource records and + additional resource records. + + @rtype: L{dict} mapping L{bytes} to L{list} of L{IRecord + } provider + @return: A mapping of record name to record payload. + """ + recordBag = {} + for answer in records[0]: + recordBag.setdefault(str(answer.name), []).append(answer.payload) + return recordBag + + + def _cbMX(self, answers, domain, cnamesLeft): + """ + Try to find the mail exchange host for a domain from the given DNS + records. + + This will attempt to resolve canonical name record results. It can + recognize loops and will give up on non-cyclic chains after a specified + number of lookups. + + @type answers: L{dict} mapping L{bytes} to L{list} of L{IRecord + } provider + @param answers: A mapping of record name to record payload. + + @type domain: L{bytes} + @param domain: A domain name. + + @type cnamesLeft: L{int} + @param cnamesLeft: The number of unique canonical name records + left to follow while looking up the mail exchange host. + + @rtype: L{Record_MX } or L{Failure} + @return: An MX record for the mail exchange host or a failure if one + cannot be found. + """ + # Do this import here so that relaymanager.py doesn't depend on + # twisted.names, only MXCalculator will. + from twisted.names import dns, error + + seenAliases = set() + exchanges = [] + # Examine the answers for the domain we asked about + pertinentRecords = answers.get(domain, []) + while pertinentRecords: + record = pertinentRecords.pop() + + # If it's a CNAME, we'll need to do some more processing + if record.TYPE == dns.CNAME: + + # Remember that this name was an alias. + seenAliases.add(domain) + + canonicalName = str(record.name) + # See if we have some local records which might be relevant. + if canonicalName in answers: + + # Make sure it isn't a loop contained entirely within the + # results we have here. + if canonicalName in seenAliases: + return Failure(CanonicalNameLoop(record)) + + pertinentRecords = answers[canonicalName] + exchanges = [] + else: + if cnamesLeft: + # Request more information from the server. + return self.getMX(canonicalName, cnamesLeft - 1) + else: + # Give up. + return Failure(CanonicalNameChainTooLong(record)) + + # If it's an MX, collect it. + if record.TYPE == dns.MX: + exchanges.append((record.preference, record)) + + if exchanges: + exchanges.sort() + for (preference, record) in exchanges: + host = str(record.name) + if host not in self.badMXs: + return record + t = self.clock.seconds() - self.badMXs[host] + if t >= 0: + del self.badMXs[host] + return record + return exchanges[0][1] + else: + # Treat no answers the same as an error - jump to the errback to + # try to look up an A record. This provides behavior described as + # a special case in RFC 974 in the section headed I{Interpreting + # the List of MX RRs}. + return Failure( + error.DNSNameError("No MX records for %r" % (domain,))) + + + def _ebMX(self, failure, domain): + """ + Attempt to use the name of the domain directly when mail exchange + lookup fails. + + @type failure: L{Failure} + @param failure: The reason for the lookup failure. + + @type domain: L{bytes} + @param domain: The domain name. + + @rtype: L{Record_MX } or L{Failure} + @return: An MX record for the domain or a failure if the fallback to + domain option is not in effect and an error, other than not + finding an MX record, occurred during lookup. + + @raise IOError: When no MX record could be found and the fallback to + domain option is not in effect. + + @raise DNSLookupError: When no MX record could be found and the + fallback to domain option is in effect but no address for the + domain could be found. + """ + from twisted.names import error, dns + + if self.fallbackToDomain: + failure.trap(error.DNSNameError) + log.msg("MX lookup failed; attempting to use hostname (%s) directly" % (domain,)) + + # Alright, I admit, this is a bit icky. + d = self.resolver.getHostByName(domain) + + def cbResolved(addr): + return dns.Record_MX(name=addr) + + def ebResolved(err): + err.trap(error.DNSNameError) + raise DNSLookupError() + + d.addCallbacks(cbResolved, ebResolved) + return d + elif failure.check(error.DNSNameError): + raise IOError("No MX found for %r" % (domain,)) + return failure diff --git a/contrib/python/Twisted/py2/twisted/mail/smtp.py b/contrib/python/Twisted/py2/twisted/mail/smtp.py new file mode 100644 index 00000000000..7725ce5886e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/smtp.py @@ -0,0 +1,2247 @@ +# -*- test-case-name: twisted.mail.test.test_smtp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +# +# pylint: disable=I0011,C0103,C9302 + +""" +Simple Mail Transfer Protocol implementation. +""" + +from __future__ import absolute_import, division + +import time +import re +import base64 +import socket +import os +import random +import binascii +import warnings + +from email.utils import parseaddr + +from zope.interface import implementer + +from twisted import cred +from twisted.copyright import longversion +from twisted.protocols import basic +from twisted.protocols import policies +from twisted.internet import protocol +from twisted.internet import defer +from twisted.internet import error +from twisted.internet import reactor +from twisted.internet.interfaces import ITLSTransport, ISSLTransport +from twisted.python import log +from twisted.python import util +from twisted.python.compat import (_PY3, range, long, unicode, networkString, + nativeString, iteritems, _keys, _bytesChr, + iterbytes) +from twisted.python.runtime import platform + +from twisted.mail.interfaces import (IClientAuthentication, + IMessageSMTP as IMessage, + IMessageDeliveryFactory, IMessageDelivery) +from twisted.mail._cred import (CramMD5ClientAuthenticator, LOGINAuthenticator, + LOGINCredentials as _lcredentials) +from twisted.mail._except import ( + AUTHDeclinedError, AUTHRequiredError, AddressError, + AuthenticationError, EHLORequiredError, ESMTPClientError, + SMTPAddressError, SMTPBadRcpt, SMTPBadSender, SMTPClientError, + SMTPConnectError, SMTPDeliveryError, SMTPError, SMTPServerError, + SMTPTimeoutError, SMTPTLSError as TLSError, TLSRequiredError, + SMTPProtocolError) + + +from io import BytesIO + + +__all__ = [ + 'AUTHDeclinedError', 'AUTHRequiredError', 'AddressError', + 'AuthenticationError', 'EHLORequiredError', 'ESMTPClientError', + 'SMTPAddressError', 'SMTPBadRcpt', 'SMTPBadSender', 'SMTPClientError', + 'SMTPConnectError', 'SMTPDeliveryError', 'SMTPError', 'SMTPServerError', + 'SMTPTimeoutError', 'TLSError', 'TLSRequiredError', 'SMTPProtocolError', + + 'IClientAuthentication', 'IMessage', 'IMessageDelivery', + 'IMessageDeliveryFactory', + + 'CramMD5ClientAuthenticator', 'LOGINAuthenticator', 'LOGINCredentials', + 'PLAINAuthenticator', + + 'Address', 'User', 'sendmail', 'SenderMixin', + 'ESMTP', 'ESMTPClient', 'ESMTPSender', 'ESMTPSenderFactory', + 'SMTP', 'SMTPClient', 'SMTPFactory', 'SMTPSender', 'SMTPSenderFactory', + + 'idGenerator', 'messageid', 'quoteaddr', 'rfc822date', 'xtextStreamReader', + 'xtextStreamWriter', 'xtext_codec', 'xtext_decode', 'xtext_encode' +] + + +# Cache the hostname (XXX Yes - this is broken) +if platform.isMacOSX(): + # On macOS, getfqdn() is ridiculously slow - use the + # probably-identical-but-sometimes-not gethostname() there. + DNSNAME = socket.gethostname() +else: + DNSNAME = socket.getfqdn() + +# Encode the DNS name into something we can send over the wire +DNSNAME = DNSNAME.encode('ascii') + +# Used for fast success code lookup +SUCCESS = dict.fromkeys(range(200, 300)) + + + +def rfc822date(timeinfo=None, local=1): + """ + Format an RFC-2822 compliant date string. + + @param timeinfo: (optional) A sequence as returned by C{time.localtime()} + or C{time.gmtime()}. Default is now. + @param local: (optional) Indicates if the supplied time is local or + universal time, or if no time is given, whether now should be local or + universal time. Default is local, as suggested (SHOULD) by rfc-2822. + + @returns: A L{bytes} representing the time and date in RFC-2822 format. + """ + if not timeinfo: + if local: + timeinfo = time.localtime() + else: + timeinfo = time.gmtime() + if local: + if timeinfo[8]: + # DST + tz = -time.altzone + else: + tz = -time.timezone + + (tzhr, tzmin) = divmod(abs(tz), 3600) + if tz: + tzhr *= int(abs(tz)//tz) + (tzmin, tzsec) = divmod(tzmin, 60) + else: + (tzhr, tzmin) = (0, 0) + + return networkString("%s, %02d %s %04d %02d:%02d:%02d %+03d%02d" % ( + ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][timeinfo[6]], + timeinfo[2], + ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][timeinfo[1] - 1], + timeinfo[0], timeinfo[3], timeinfo[4], timeinfo[5], + tzhr, tzmin)) + + + +def idGenerator(): + i = 0 + while True: + yield i + i += 1 + +_gen = idGenerator() + + + +def messageid(uniq=None, N=lambda: next(_gen)): + """ + Return a globally unique random string in RFC 2822 Message-ID format + + + + Optional uniq string will be added to strengthen uniqueness if given. + """ + datetime = time.strftime('%Y%m%d%H%M%S', time.gmtime()) + pid = os.getpid() + rand = random.randrange(2**31-1) + if uniq is None: + uniq = '' + else: + uniq = '.' + uniq + + return '<%s.%s.%s%s.%s@%s>' % (datetime, pid, rand, uniq, N(), DNSNAME) + + + +def quoteaddr(addr): + """ + Turn an email address, possibly with realname part etc, into + a form suitable for and SMTP envelope. + """ + + if isinstance(addr, Address): + return b'<' + bytes(addr) + b'>' + + if isinstance(addr, bytes): + addr = addr.decode('ascii') + + res = parseaddr(addr) + + if res == (None, None): + # It didn't parse, use it as-is + return b'<' + bytes(addr) + b'>' + else: + return b'<' + res[1].encode('ascii') + b'>' + +COMMAND, DATA, AUTH = 'COMMAND', 'DATA', 'AUTH' + + +# Character classes for parsing addresses +atom = br"[-A-Za-z0-9!\#$%&'*+/=?^_`{|}~]" + +class Address: + """Parse and hold an RFC 2821 address. + + Source routes are stipped and ignored, UUCP-style bang-paths + and %-style routing are not parsed. + + @type domain: C{bytes} + @ivar domain: The domain within which this address resides. + + @type local: C{bytes} + @ivar local: The local (\"user\") portion of this address. + """ + + tstring = re.compile(br'''( # A string of + (?:"[^"]*" # quoted string + |\\. # backslash-escaped characted + |''' + atom + br''' # atom character + )+|.) # or any single character''', re.X) + atomre = re.compile(atom) # match any one atom character + + + def __init__(self, addr, defaultDomain=None): + if isinstance(addr, User): + addr = addr.dest + if isinstance(addr, Address): + self.__dict__ = addr.__dict__.copy() + return + elif not isinstance(addr, bytes): + addr = str(addr).encode('ascii') + + self.addrstr = addr + + # Tokenize + atl = list(filter(None, self.tstring.split(addr))) + local = [] + domain = [] + + while atl: + if atl[0] == b'<': + if atl[-1] != b'>': + raise AddressError("Unbalanced <>") + atl = atl[1:-1] + elif atl[0] == b'@': + atl = atl[1:] + if not local: + # Source route + while atl and atl[0] != b':': + # remove it + atl = atl[1:] + if not atl: + raise AddressError("Malformed source route") + atl = atl[1:] # remove : + elif domain: + raise AddressError("Too many @") + else: + # Now in domain + domain = [b''] + elif (len(atl[0]) == 1 and + not self.atomre.match(atl[0]) and + atl[0] != b'.'): + raise AddressError("Parse error at %r of %r" % (atl[0], (addr, atl))) + else: + if not domain: + local.append(atl[0]) + else: + domain.append(atl[0]) + atl = atl[1:] + + self.local = b''.join(local) + self.domain = b''.join(domain) + if self.local != b'' and self.domain == b'': + if defaultDomain is None: + defaultDomain = DNSNAME + self.domain = defaultDomain + + dequotebs = re.compile(br'\\(.)') + + + def dequote(self, addr): + """ + Remove RFC-2821 quotes from address. + """ + res = [] + + if not isinstance(addr, bytes): + addr = str(addr).encode('ascii') + + atl = filter(None, self.tstring.split(addr)) + + for t in atl: + if t[0] == b'"' and t[-1] == b'"': + res.append(t[1:-1]) + elif '\\' in t: + res.append(self.dequotebs.sub(br'\1', t)) + else: + res.append(t) + + return b''.join(res) + + if _PY3: + def __str__(self): + return nativeString(bytes(self)) + else: + def __str__(self): + return self.__bytes__() + + + def __bytes__(self): + if self.local or self.domain: + return b'@'.join((self.local, self.domain)) + else: + return b'' + + + def __repr__(self): + return "%s.%s(%s)" % (self.__module__, self.__class__.__name__, + repr(str(self))) + + + +class User: + """ + Hold information about and SMTP message recipient, + including information on where the message came from + """ + def __init__(self, destination, helo, protocol, orig): + try: + host = protocol.host + except AttributeError: + host = None + self.dest = Address(destination, host) + self.helo = helo + self.protocol = protocol + if isinstance(orig, Address): + self.orig = orig + else: + self.orig = Address(orig, host) + + + def __getstate__(self): + """ + Helper for pickle. + + protocol isn't picklabe, but we want User to be, so skip it in + the pickle. + """ + return { 'dest' : self.dest, + 'helo' : self.helo, + 'protocol' : None, + 'orig' : self.orig } + + + def __str__(self): + return nativeString(bytes(self.dest)) + + + def __bytes__(self): + return bytes(self.dest) + + + +class SMTP(basic.LineOnlyReceiver, policies.TimeoutMixin): + """ + SMTP server-side protocol. + + @ivar host: The hostname of this mail server. + @type host: L{bytes} + """ + + timeout = 600 + portal = None + + # Control whether we log SMTP events + noisy = True + + # A factory for IMessageDelivery objects. If an + # avatar implementing IMessageDeliveryFactory can + # be acquired from the portal, it will be used to + # create a new IMessageDelivery object for each + # message which is received. + deliveryFactory = None + + # An IMessageDelivery object. A new instance is + # used for each message received if we can get an + # IMessageDeliveryFactory from the portal. Otherwise, + # a single instance is used throughout the lifetime + # of the connection. + delivery = None + + # Cred cleanup function. + _onLogout = None + + def __init__(self, delivery=None, deliveryFactory=None): + self.mode = COMMAND + self._from = None + self._helo = None + self._to = [] + self.delivery = delivery + self.deliveryFactory = deliveryFactory + self.host = DNSNAME + + + @property + def host(self): + return self._host + + + @host.setter + def host(self, toSet): + if not isinstance(toSet, bytes): + toSet = str(toSet).encode('ascii') + self._host = toSet + + + + def timeoutConnection(self): + msg = self.host + b' Timeout. Try talking faster next time!' + self.sendCode(421, msg) + self.transport.loseConnection() + + + def greeting(self): + return self.host + b' NO UCE NO UBE NO RELAY PROBES' + + + def connectionMade(self): + # Ensure user-code always gets something sane for _helo + peer = self.transport.getPeer() + try: + host = peer.host + except AttributeError: # not an IPv4Address + host = str(peer) + self._helo = (None, host) + self.sendCode(220, self.greeting()) + self.setTimeout(self.timeout) + + + def sendCode(self, code, message=b''): + """ + Send an SMTP code with a message. + """ + lines = message.splitlines() + lastline = lines[-1:] + for line in lines[:-1]: + self.sendLine(networkString('%3.3d-' % (code,)) + line) + self.sendLine(networkString('%3.3d ' % (code,)) + + (lastline and lastline[0] or b'')) + + + def lineReceived(self, line): + self.resetTimeout() + return getattr(self, 'state_' + self.mode)(line) + + + def state_COMMAND(self, line): + # Ignore leading and trailing whitespace, as well as an arbitrary + # amount of whitespace between the command and its argument, though + # it is not required by the protocol, for it is a nice thing to do. + line = line.strip() + + parts = line.split(None, 1) + if parts: + method = self.lookupMethod(parts[0]) or self.do_UNKNOWN + if len(parts) == 2: + method(parts[1]) + else: + method(b'') + else: + self.sendSyntaxError() + + + def sendSyntaxError(self): + self.sendCode(500, b'Error: bad syntax') + + + def lookupMethod(self, command): + """ + + @param command: The command to get from this class. + @type command: L{str} + @return: The function which executes this command. + """ + if not isinstance(command, str): + command = nativeString(command) + + return getattr(self, 'do_' + command.upper(), None) + + + def lineLengthExceeded(self, line): + if self.mode is DATA: + for message in self.__messages: + message.connectionLost() + self.mode = COMMAND + del self.__messages + self.sendCode(500, b'Line too long') + + + def do_UNKNOWN(self, rest): + self.sendCode(500, b'Command not implemented') + + + def do_HELO(self, rest): + peer = self.transport.getPeer() + try: + host = peer.host + except AttributeError: + host = str(peer) + + if not isinstance(host, bytes): + host = host.encode('idna') + + self._helo = (rest, host) + self._from = None + self._to = [] + self.sendCode(250, + self.host + b' Hello ' + host + b', nice to meet you') + + + def do_QUIT(self, rest): + self.sendCode(221, b'See you later') + self.transport.loseConnection() + + # A string of quoted strings, backslash-escaped character or + # atom characters + '@.,:' + qstring = br'("[^"]*"|\\.|' + atom + br'|[@.,:])+' + + mail_re = re.compile(br'''\s*FROM:\s*(?P<> # Empty <> + |<''' + qstring + br'''> # + |''' + qstring + br''' # addr + )\s*(\s(?P.*))? # Optional WS + ESMTP options + $''', re.I|re.X) + rcpt_re = re.compile(br'\s*TO:\s*(?P<' + qstring + br'''> # + |''' + qstring + br''' # addr + )\s*(\s(?P.*))? # Optional WS + ESMTP options + $''', re.I|re.X) + + def do_MAIL(self, rest): + if self._from: + self.sendCode(503, b"Only one sender per message, please") + return + # Clear old recipient list + self._to = [] + m = self.mail_re.match(rest) + if not m: + self.sendCode(501, b"Syntax error") + return + + try: + addr = Address(m.group('path'), self.host) + except AddressError as e: + self.sendCode(553, networkString(str(e))) + return + + validated = defer.maybeDeferred(self.validateFrom, self._helo, addr) + validated.addCallbacks(self._cbFromValidate, self._ebFromValidate) + + + def _cbFromValidate(self, fromEmail, code=250, + msg=b'Sender address accepted'): + self._from = fromEmail + self.sendCode(code, msg) + + + def _ebFromValidate(self, failure): + if failure.check(SMTPBadSender): + self.sendCode(failure.value.code, + (b'Cannot receive from specified address ' + + quoteaddr(failure.value.addr) + b': ' + + networkString(failure.value.resp))) + elif failure.check(SMTPServerError): + self.sendCode(failure.value.code, + networkString(failure.value.resp)) + else: + log.err(failure, "SMTP sender validation failure") + self.sendCode( + 451, + b'Requested action aborted: local error in processing') + + + def do_RCPT(self, rest): + if not self._from: + self.sendCode(503, b"Must have sender before recipient") + return + m = self.rcpt_re.match(rest) + if not m: + self.sendCode(501, b"Syntax error") + return + + try: + user = User(m.group('path'), self._helo, self, self._from) + except AddressError as e: + self.sendCode(553, networkString(str(e))) + return + + d = defer.maybeDeferred(self.validateTo, user) + d.addCallbacks( + self._cbToValidate, + self._ebToValidate, + callbackArgs=(user,) + ) + + + def _cbToValidate(self, to, user=None, code=250, + msg=b'Recipient address accepted'): + if user is None: + user = to + self._to.append((user, to)) + self.sendCode(code, msg) + + + def _ebToValidate(self, failure): + if failure.check(SMTPBadRcpt, SMTPServerError): + self.sendCode(failure.value.code, + networkString(failure.value.resp)) + else: + log.err(failure) + self.sendCode( + 451, + b'Requested action aborted: local error in processing' + ) + + + def _disconnect(self, msgs): + for msg in msgs: + try: + msg.connectionLost() + except: + log.msg("msg raised exception from connectionLost") + log.err() + + + def do_DATA(self, rest): + if self._from is None or (not self._to): + self.sendCode(503, b'Must have valid receiver and originator') + return + self.mode = DATA + helo, origin = self._helo, self._from + recipients = self._to + + self._from = None + self._to = [] + self.datafailed = None + + msgs = [] + for (user, msgFunc) in recipients: + try: + msg = msgFunc() + rcvdhdr = self.receivedHeader(helo, origin, [user]) + if rcvdhdr: + msg.lineReceived(rcvdhdr) + msgs.append(msg) + except SMTPServerError as e: + self.sendCode(e.code, e.resp) + self.mode = COMMAND + self._disconnect(msgs) + return + except: + log.err() + self.sendCode(550, b"Internal server error") + self.mode = COMMAND + self._disconnect(msgs) + return + self.__messages = msgs + + self.__inheader = self.__inbody = 0 + self.sendCode(354, b'Continue') + + if self.noisy: + fmt = 'Receiving message for delivery: from=%s to=%s' + log.msg(fmt % (origin, [str(u) for (u, f) in recipients])) + + + def connectionLost(self, reason): + # self.sendCode(421, 'Dropping connection.') # This does nothing... + # Ideally, if we (rather than the other side) lose the connection, + # we should be able to tell the other side that we are going away. + # RFC-2821 requires that we try. + if self.mode is DATA: + try: + for message in self.__messages: + try: + message.connectionLost() + except: + log.err() + del self.__messages + except AttributeError: + pass + if self._onLogout: + self._onLogout() + self._onLogout = None + self.setTimeout(None) + + + def do_RSET(self, rest): + self._from = None + self._to = [] + self.sendCode(250, b'I remember nothing.') + + + def dataLineReceived(self, line): + if line[:1] == b'.': + if line == b'.': + self.mode = COMMAND + if self.datafailed: + self.sendCode(self.datafailed.code, + self.datafailed.resp) + return + if not self.__messages: + self._messageHandled("thrown away") + return + defer.DeferredList([ + m.eomReceived() for m in self.__messages + ], consumeErrors=True).addCallback(self._messageHandled + ) + del self.__messages + return + line = line[1:] + + if self.datafailed: + return + + try: + # Add a blank line between the generated Received:-header + # and the message body if the message comes in without any + # headers + if not self.__inheader and not self.__inbody: + if b':' in line: + self.__inheader = 1 + elif line: + for message in self.__messages: + message.lineReceived(b'') + self.__inbody = 1 + + if not line: + self.__inbody = 1 + + for message in self.__messages: + message.lineReceived(line) + except SMTPServerError as e: + self.datafailed = e + for message in self.__messages: + message.connectionLost() + state_DATA = dataLineReceived + + + def _messageHandled(self, resultList): + failures = 0 + for (success, result) in resultList: + if not success: + failures += 1 + log.err(result) + if failures: + msg = 'Could not send e-mail' + resultLen = len(resultList) + if resultLen > 1: + msg += ' (%d failures out of %d recipients)'.format( + failures, resultLen) + self.sendCode(550, networkString(msg)) + else: + self.sendCode(250, b'Delivery in progress') + + + def _cbAnonymousAuthentication(self, result): + """ + Save the state resulting from a successful anonymous cred login. + """ + (iface, avatar, logout) = result + if issubclass(iface, IMessageDeliveryFactory): + self.deliveryFactory = avatar + self.delivery = None + elif issubclass(iface, IMessageDelivery): + self.deliveryFactory = None + self.delivery = avatar + else: + raise RuntimeError("%s is not a supported interface" % (iface.__name__,)) + self._onLogout = logout + self.challenger = None + + + # overridable methods: + def validateFrom(self, helo, origin): + """ + Validate the address from which the message originates. + + @type helo: C{(bytes, bytes)} + @param helo: The argument to the HELO command and the client's IP + address. + + @type origin: C{Address} + @param origin: The address the message is from + + @rtype: C{Deferred} or C{Address} + @return: C{origin} or a C{Deferred} whose callback will be + passed C{origin}. + + @raise SMTPBadSender: Raised of messages from this address are + not to be accepted. + """ + if self.deliveryFactory is not None: + self.delivery = self.deliveryFactory.getMessageDelivery() + + if self.delivery is not None: + return defer.maybeDeferred(self.delivery.validateFrom, + helo, origin) + + # No login has been performed, no default delivery object has been + # provided: try to perform an anonymous login and then invoke this + # method again. + if self.portal: + + result = self.portal.login( + cred.credentials.Anonymous(), + None, + IMessageDeliveryFactory, IMessageDelivery) + + def ebAuthentication(err): + """ + Translate cred exceptions into SMTP exceptions so that the + protocol code which invokes C{validateFrom} can properly report + the failure. + """ + if err.check(cred.error.UnauthorizedLogin): + exc = SMTPBadSender(origin) + elif err.check(cred.error.UnhandledCredentials): + exc = SMTPBadSender( + origin, resp="Unauthenticated senders not allowed") + else: + return err + return defer.fail(exc) + + result.addCallbacks( + self._cbAnonymousAuthentication, ebAuthentication) + + def continueValidation(ignored): + """ + Re-attempt from address validation. + """ + return self.validateFrom(helo, origin) + + result.addCallback(continueValidation) + return result + + raise SMTPBadSender(origin) + + + def validateTo(self, user): + """ + Validate the address for which the message is destined. + + @type user: L{User} + @param user: The address to validate. + + @rtype: no-argument callable + @return: A C{Deferred} which becomes, or a callable which + takes no arguments and returns an object implementing C{IMessage}. + This will be called and the returned object used to deliver the + message when it arrives. + + @raise SMTPBadRcpt: Raised if messages to the address are + not to be accepted. + """ + if self.delivery is not None: + return self.delivery.validateTo(user) + raise SMTPBadRcpt(user) + + + def receivedHeader(self, helo, origin, recipients): + if self.delivery is not None: + return self.delivery.receivedHeader(helo, origin, recipients) + + heloStr = b"" + if helo[0]: + heloStr = b" helo=" + helo[0] + domain = networkString(self.transport.getHost().host) + + from_ = b"from " + helo[0] + b" ([" + helo[1] + b"]" + heloStr + b")" + by = b"by %s with %s (%s)" % (domain, + self.__class__.__name__, + longversion) + for_ = b"for %s; %s" % (' '.join(map(str, recipients)), + rfc822date()) + return b"Received: " + from_ + b"\n\t" + by + b"\n\t" + for_ + + + +class SMTPFactory(protocol.ServerFactory): + """ + Factory for SMTP. + """ + + # override in instances or subclasses + domain = DNSNAME + timeout = 600 + protocol = SMTP + + portal = None + + def __init__(self, portal = None): + self.portal = portal + + + def buildProtocol(self, addr): + p = protocol.ServerFactory.buildProtocol(self, addr) + p.portal = self.portal + p.host = self.domain + return p + + + +class SMTPClient(basic.LineReceiver, policies.TimeoutMixin): + """ + SMTP client for sending emails. + + After the client has connected to the SMTP server, it repeatedly calls + L{SMTPClient.getMailFrom}, L{SMTPClient.getMailTo} and + L{SMTPClient.getMailData} and uses this information to send an email. + It then calls L{SMTPClient.getMailFrom} again; if it returns L{None}, the + client will disconnect, otherwise it will continue as normal i.e. call + L{SMTPClient.getMailTo} and L{SMTPClient.getMailData} and send a new email. + """ + + # If enabled then log SMTP client server communication + debug = True + + # Number of seconds to wait before timing out a connection. If + # None, perform no timeout checking. + timeout = None + + def __init__(self, identity, logsize=10): + if isinstance(identity, unicode): + identity = identity.encode('ascii') + + self.identity = identity or b'' + self.toAddressesResult = [] + self.successAddresses = [] + self._from = None + self.resp = [] + self.code = -1 + self.log = util.LineLog(logsize) + + + def sendLine(self, line): + # Log sendLine only if you are in debug mode for performance + if self.debug: + self.log.append(b'>>> ' + line) + + basic.LineReceiver.sendLine(self,line) + + + def connectionMade(self): + self.setTimeout(self.timeout) + + self._expected = [ 220 ] + self._okresponse = self.smtpState_helo + self._failresponse = self.smtpConnectionFailed + + + def connectionLost(self, reason=protocol.connectionDone): + """ + We are no longer connected + """ + self.setTimeout(None) + self.mailFile = None + + + def timeoutConnection(self): + self.sendError( + SMTPTimeoutError( + -1, b"Timeout waiting for SMTP server response", + self.log.str())) + + + def lineReceived(self, line): + self.resetTimeout() + + # Log lineReceived only if you are in debug mode for performance + if self.debug: + self.log.append(b'<<< ' + line) + + why = None + + try: + self.code = int(line[:3]) + except ValueError: + # This is a fatal error and will disconnect the transport + # lineReceived will not be called again. + self.sendError(SMTPProtocolError(-1, + "Invalid response from SMTP server: {}".format(line), + self.log.str())) + return + + if line[0:1] == b'0': + # Verbose informational message, ignore it + return + + self.resp.append(line[4:]) + + if line[3:4] == b'-': + # Continuation + return + + if self.code in self._expected: + why = self._okresponse(self.code, b'\n'.join(self.resp)) + else: + why = self._failresponse(self.code, b'\n'.join(self.resp)) + + self.code = -1 + self.resp = [] + return why + + + def smtpConnectionFailed(self, code, resp): + self.sendError(SMTPConnectError(code, resp, self.log.str())) + + + def smtpTransferFailed(self, code, resp): + if code < 0: + self.sendError(SMTPProtocolError(code, resp, self.log.str())) + else: + self.smtpState_msgSent(code, resp) + + + def smtpState_helo(self, code, resp): + self.sendLine(b'HELO ' + self.identity) + self._expected = SUCCESS + self._okresponse = self.smtpState_from + + + def smtpState_from(self, code, resp): + self._from = self.getMailFrom() + self._failresponse = self.smtpTransferFailed + if self._from is not None: + self.sendLine(b'MAIL FROM:' + quoteaddr(self._from)) + self._expected = [250] + self._okresponse = self.smtpState_to + else: + # All messages have been sent, disconnect + self._disconnectFromServer() + + + def smtpState_disconnect(self, code, resp): + self.transport.loseConnection() + + + def smtpState_to(self, code, resp): + self.toAddresses = iter(self.getMailTo()) + self.toAddressesResult = [] + self.successAddresses = [] + self._okresponse = self.smtpState_toOrData + self._expected = range(0, 1000) + self.lastAddress = None + return self.smtpState_toOrData(0, b'') + + + def smtpState_toOrData(self, code, resp): + if self.lastAddress is not None: + self.toAddressesResult.append((self.lastAddress, code, resp)) + if code in SUCCESS: + self.successAddresses.append(self.lastAddress) + try: + self.lastAddress = next(self.toAddresses) + except StopIteration: + if self.successAddresses: + self.sendLine(b'DATA') + self._expected = [ 354 ] + self._okresponse = self.smtpState_data + else: + return self.smtpState_msgSent(code,'No recipients accepted') + else: + self.sendLine(b'RCPT TO:' + quoteaddr(self.lastAddress)) + + + def smtpState_data(self, code, resp): + s = basic.FileSender() + d = s.beginFileTransfer( + self.getMailData(), self.transport, self.transformChunk) + def ebTransfer(err): + self.sendError(err.value) + d.addCallbacks(self.finishedFileTransfer, ebTransfer) + self._expected = SUCCESS + self._okresponse = self.smtpState_msgSent + + + def smtpState_msgSent(self, code, resp): + if self._from is not None: + self.sentMail(code, resp, len(self.successAddresses), + self.toAddressesResult, self.log) + + self.toAddressesResult = [] + self._from = None + self.sendLine(b'RSET') + self._expected = SUCCESS + self._okresponse = self.smtpState_from + + + ## + ## Helpers for FileSender + ## + def transformChunk(self, chunk): + """ + Perform the necessary local to network newline conversion and escape + leading periods. + + This method also resets the idle timeout so that as long as process is + being made sending the message body, the client will not time out. + """ + self.resetTimeout() + return chunk.replace(b'\n', b'\r\n').replace(b'\r\n.', b'\r\n..') + + + def finishedFileTransfer(self, lastsent): + if lastsent != b'\n': + line = b'\r\n.' + else: + line = b'.' + self.sendLine(line) + + + ## + # these methods should be overridden in subclasses + def getMailFrom(self): + """ + Return the email address the mail is from. + """ + raise NotImplementedError + + + def getMailTo(self): + """ + Return a list of emails to send to. + """ + raise NotImplementedError + + + def getMailData(self): + """ + Return file-like object containing data of message to be sent. + + Lines in the file should be delimited by '\\n'. + """ + raise NotImplementedError + + + def sendError(self, exc): + """ + If an error occurs before a mail message is sent sendError will be + called. This base class method sends a QUIT if the error is + non-fatal and disconnects the connection. + + @param exc: The SMTPClientError (or child class) raised + @type exc: C{SMTPClientError} + """ + if isinstance(exc, SMTPClientError) and not exc.isFatal: + self._disconnectFromServer() + else: + # If the error was fatal then the communication channel with the + # SMTP Server is broken so just close the transport connection + self.smtpState_disconnect(-1, None) + + + def sentMail(self, code, resp, numOk, addresses, log): + """ + Called when an attempt to send an email is completed. + + If some addresses were accepted, code and resp are the response + to the DATA command. If no addresses were accepted, code is -1 + and resp is an informative message. + + @param code: the code returned by the SMTP Server + @param resp: The string response returned from the SMTP Server + @param numOK: the number of addresses accepted by the remote host. + @param addresses: is a list of tuples (address, code, resp) listing + the response to each RCPT command. + @param log: is the SMTP session log + """ + raise NotImplementedError + + + def _disconnectFromServer(self): + self._expected = range(0, 1000) + self._okresponse = self.smtpState_disconnect + self.sendLine(b'QUIT') + + + +class ESMTPClient(SMTPClient): + """ + A client for sending emails over ESMTP. + + @ivar heloFallback: Whether or not to fall back to plain SMTP if the C{EHLO} + command is not recognised by the server. If L{requireAuthentication} is + C{True}, or L{requireTransportSecurity} is C{True} and the connection is + not over TLS, this fallback flag will not be honored. + @type heloFallback: L{bool} + + @ivar requireAuthentication: If C{True}, refuse to proceed if authentication + cannot be performed. Overrides L{heloFallback}. + @type requireAuthentication: L{bool} + + @ivar requireTransportSecurity: If C{True}, refuse to proceed if the + transport cannot be secured. If the transport layer is not already + secured via TLS, this will override L{heloFallback}. + @type requireAuthentication: L{bool} + + @ivar context: The context factory to use for STARTTLS, if desired. + @type context: L{ssl.ClientContextFactory} + + @ivar _tlsMode: Whether or not the connection is over TLS. + @type _tlsMode: L{bool} + """ + heloFallback = True + requireAuthentication = False + requireTransportSecurity = False + context = None + _tlsMode = False + + def __init__(self, secret, contextFactory=None, *args, **kw): + SMTPClient.__init__(self, *args, **kw) + self.authenticators = [] + self.secret = secret + self.context = contextFactory + + + def __getattr__(self, name): + if name == "tlsMode": + warnings.warn( + "tlsMode attribute of twisted.mail.smtp.ESMTPClient " + "is deprecated since Twisted 13.0", + category=DeprecationWarning, stacklevel=2) + return self._tlsMode + else: + raise AttributeError( + '%s instance has no attribute %r' % ( + self.__class__.__name__, name,)) + + + def __setattr__(self, name, value): + if name == "tlsMode": + warnings.warn( + "tlsMode attribute of twisted.mail.smtp.ESMTPClient " + "is deprecated since Twisted 13.0", + category=DeprecationWarning, stacklevel=2) + self._tlsMode = value + else: + self.__dict__[name] = value + + + def esmtpEHLORequired(self, code=-1, resp=None): + """ + Fail because authentication is required, but the server does not support + ESMTP, which is required for authentication. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + """ + self.sendError(EHLORequiredError(502, b"Server does not support ESMTP " + b"Authentication", self.log.str())) + + + def esmtpAUTHRequired(self, code=-1, resp=None): + """ + Fail because authentication is required, but the server does not support + any schemes we support. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + """ + tmp = [] + + for a in self.authenticators: + tmp.append(a.getName().upper()) + + auth = b"[%s]" % b", ".join(tmp) + + self.sendError(AUTHRequiredError(502, b"Server does not support Client " + b"Authentication schemes %s" % auth, self.log.str())) + + + def esmtpTLSRequired(self, code=-1, resp=None): + """ + Fail because TLS is required and the server does not support it. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + """ + self.sendError(TLSRequiredError(502, b"Server does not support secure " + b"communication via TLS / SSL", self.log.str())) + + + def esmtpTLSFailed(self, code=-1, resp=None): + """ + Fail because the TLS handshake wasn't able to be completed. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + """ + self.sendError(TLSError(code, b"Could not complete the SSL/TLS " + b"handshake", self.log.str())) + + + def esmtpAUTHDeclined(self, code=-1, resp=None): + """ + Fail because the authentication was rejected. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + """ + self.sendError(AUTHDeclinedError(code, resp, self.log.str())) + + + def esmtpAUTHMalformedChallenge(self, code=-1, resp=None): + """ + Fail because the server sent a malformed authentication challenge. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + """ + self.sendError(AuthenticationError(501, b"Login failed because the " + b"SMTP Server returned a malformed Authentication Challenge", + self.log.str())) + + + def esmtpAUTHServerError(self, code=-1, resp=None): + """ + Fail because of some other authentication error. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + """ + self.sendError(AuthenticationError(code, resp, self.log.str())) + + + def registerAuthenticator(self, auth): + """ + Registers an Authenticator with the ESMTPClient. The ESMTPClient will + attempt to login to the SMTP Server in the order the Authenticators are + registered. The most secure Authentication mechanism should be + registered first. + + @param auth: The Authentication mechanism to register + @type auth: L{IClientAuthentication} implementor + + @return: L{None} + """ + self.authenticators.append(auth) + + + def connectionMade(self): + """ + Called when a connection has been made, and triggers sending an C{EHLO} + to the server. + """ + self._tlsMode = ISSLTransport.providedBy(self.transport) + SMTPClient.connectionMade(self) + self._okresponse = self.esmtpState_ehlo + + + def esmtpState_ehlo(self, code, resp): + """ + Send an C{EHLO} to the server. + + If L{heloFallback} is C{True}, and there is no requirement for TLS or + authentication, the client will fall back to basic SMTP. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + + @return: L{None} + """ + self._expected = SUCCESS + + self._okresponse = self.esmtpState_serverConfig + self._failresponse = self.esmtpEHLORequired + + if self._tlsMode: + needTLS = False + else: + needTLS = self.requireTransportSecurity + + if self.heloFallback and not self.requireAuthentication and not needTLS: + self._failresponse = self.smtpState_helo + + self.sendLine(b"EHLO " + self.identity) + + + def esmtpState_serverConfig(self, code, resp): + """ + Handle a positive response to the I{EHLO} command by parsing the + capabilities in the server's response and then taking the most + appropriate next step towards entering a mail transaction. + """ + items = {} + for line in resp.splitlines(): + e = line.split(None, 1) + if len(e) > 1: + items[e[0]] = e[1] + else: + items[e[0]] = None + + self.tryTLS(code, resp, items) + + + def tryTLS(self, code, resp, items): + """ + Take a necessary step towards being able to begin a mail transaction. + + The step may be to ask the server to being a TLS session. If TLS is + already in use or not necessary and not available then the step may be + to authenticate with the server. If TLS is necessary and not available, + fail the mail transmission attempt. + + This is an internal helper method. + + @param code: The server status code from the most recently received + server message. + @type code: L{int} + + @param resp: The server status response from the most recently received + server message. + @type resp: L{bytes} + + @param items: A mapping of ESMTP extensions offered by the server. Keys + are extension identifiers and values are the associated values. + @type items: L{dict} mapping L{bytes} to L{bytes} + + @return: L{None} + """ + + # has tls can tls must tls result + # t t t authenticate + # t t f authenticate + # t f t authenticate + # t f f authenticate + + # f t t STARTTLS + # f t f STARTTLS + # f f t esmtpTLSRequired + # f f f authenticate + + hasTLS = self._tlsMode + canTLS = self.context and b"STARTTLS" in items + mustTLS = self.requireTransportSecurity + + if hasTLS or not (canTLS or mustTLS): + self.authenticate(code, resp, items) + elif canTLS: + self._expected = [220] + self._okresponse = self.esmtpState_starttls + self._failresponse = self.esmtpTLSFailed + self.sendLine(b"STARTTLS") + else: + self.esmtpTLSRequired() + + + def esmtpState_starttls(self, code, resp): + """ + Handle a positive response to the I{STARTTLS} command by starting a new + TLS session on C{self.transport}. + + Upon success, re-handshake with the server to discover what capabilities + it has when TLS is in use. + """ + try: + self.transport.startTLS(self.context) + self._tlsMode = True + except: + log.err() + self.esmtpTLSFailed(451) + + # Send another EHLO once TLS has been started to + # get the TLS / AUTH schemes. Some servers only allow AUTH in TLS mode. + self.esmtpState_ehlo(code, resp) + + + def authenticate(self, code, resp, items): + if self.secret and items.get(b'AUTH'): + schemes = items[b'AUTH'].split() + tmpSchemes = {} + + #XXX: May want to come up with a more efficient way to do this + for s in schemes: + tmpSchemes[s.upper()] = 1 + + for a in self.authenticators: + auth = a.getName().upper() + + if auth in tmpSchemes: + self._authinfo = a + + # Special condition handled + if auth == b"PLAIN": + self._okresponse = self.smtpState_from + self._failresponse = self._esmtpState_plainAuth + self._expected = [235] + challenge = base64.b64encode( + self._authinfo.challengeResponse(self.secret, 1)) + self.sendLine(b"AUTH %s %s" % (auth, challenge)) + else: + self._expected = [334] + self._okresponse = self.esmtpState_challenge + # If some error occurs here, the server declined the + # AUTH before the user / password phase. This would be + # a very rare case + self._failresponse = self.esmtpAUTHServerError + self.sendLine(b'AUTH ' + auth) + return + + if self.requireAuthentication: + self.esmtpAUTHRequired() + else: + self.smtpState_from(code, resp) + + + def _esmtpState_plainAuth(self, code, resp): + self._okresponse = self.smtpState_from + self._failresponse = self.esmtpAUTHDeclined + self._expected = [235] + challenge = base64.b64encode( + self._authinfo.challengeResponse(self.secret, 2)) + self.sendLine(b'AUTH PLAIN ' + challenge) + + + def esmtpState_challenge(self, code, resp): + self._authResponse(self._authinfo, resp) + + + def _authResponse(self, auth, challenge): + self._failresponse = self.esmtpAUTHDeclined + try: + challenge = base64.b64decode(challenge) + except binascii.Error: + # Illegal challenge, give up, then quit + self.sendLine(b'*') + self._okresponse = self.esmtpAUTHMalformedChallenge + self._failresponse = self.esmtpAUTHMalformedChallenge + else: + resp = auth.challengeResponse(self.secret, challenge) + self._expected = [235, 334] + self._okresponse = self.smtpState_maybeAuthenticated + self.sendLine(base64.b64encode(resp)) + + + def smtpState_maybeAuthenticated(self, code, resp): + """ + Called to handle the next message from the server after sending a + response to a SASL challenge. The server response might be another + challenge or it might indicate authentication has succeeded. + """ + if code == 235: + # Yes, authenticated! + del self._authinfo + self.smtpState_from(code, resp) + else: + # No, not authenticated yet. Keep trying. + self._authResponse(self._authinfo, resp) + + + +class ESMTP(SMTP): + ctx = None + canStartTLS = False + startedTLS = False + + authenticated = False + + def __init__(self, chal=None, contextFactory=None): + SMTP.__init__(self) + if chal is None: + chal = {} + self.challengers = chal + self.authenticated = False + self.ctx = contextFactory + + + def connectionMade(self): + SMTP.connectionMade(self) + self.canStartTLS = ITLSTransport.providedBy(self.transport) + self.canStartTLS = self.canStartTLS and (self.ctx is not None) + + + def greeting(self): + return SMTP.greeting(self) + b' ESMTP' + + + def extensions(self): + """ + SMTP service extensions + + @return: the SMTP service extensions that are supported. + @rtype: L{dict} with L{bytes} keys and a value of either L{None} or a + L{list} of L{bytes}. + """ + ext = {b'AUTH': _keys(self.challengers)} + if self.canStartTLS and not self.startedTLS: + ext[b'STARTTLS'] = None + return ext + + + def lookupMethod(self, command): + command = nativeString(command) + + m = SMTP.lookupMethod(self, command) + if m is None: + m = getattr(self, 'ext_' + command.upper(), None) + return m + + + def listExtensions(self): + r = [] + for (c, v) in iteritems(self.extensions()): + if v is not None: + if v: + # Intentionally omit extensions with empty argument lists + r.append(c + b' ' + b' '.join(v)) + else: + r.append(c) + + return b'\n'.join(r) + + + def do_EHLO(self, rest): + peer = self.transport.getPeer().host + + if not isinstance(peer, bytes): + peer = peer.encode('idna') + + self._helo = (rest, peer) + self._from = None + self._to = [] + self.sendCode( + 250, + (self.host + b' Hello ' + peer + b', nice to meet you\n' + + self.listExtensions()) + ) + + + def ext_STARTTLS(self, rest): + if self.startedTLS: + self.sendCode(503, b'TLS already negotiated') + elif self.ctx and self.canStartTLS: + self.sendCode(220, b'Begin TLS negotiation now') + self.transport.startTLS(self.ctx) + self.startedTLS = True + else: + self.sendCode(454, b'TLS not available') + + + def ext_AUTH(self, rest): + if self.authenticated: + self.sendCode(503, b'Already authenticated') + return + parts = rest.split(None, 1) + chal = self.challengers.get(parts[0].upper(), lambda: None)() + if not chal: + self.sendCode(504, b'Unrecognized authentication type') + return + + self.mode = AUTH + self.challenger = chal + + if len(parts) > 1: + chal.getChallenge() # Discard it, apparently the client does not + # care about it. + rest = parts[1] + else: + rest = None + self.state_AUTH(rest) + + + def _cbAuthenticated(self, loginInfo): + """ + Save the state resulting from a successful cred login and mark this + connection as authenticated. + """ + result = SMTP._cbAnonymousAuthentication(self, loginInfo) + self.authenticated = True + return result + + + def _ebAuthenticated(self, reason): + """ + Handle cred login errors by translating them to the SMTP authenticate + failed. Translate all other errors into a generic SMTP error code and + log the failure for inspection. Stop all errors from propagating. + + @param reason: Reason for failure. + """ + self.challenge = None + if reason.check(cred.error.UnauthorizedLogin): + self.sendCode(535, b'Authentication failed') + else: + log.err(reason, "SMTP authentication failure") + self.sendCode( + 451, + b'Requested action aborted: local error in processing') + + + def state_AUTH(self, response): + """ + Handle one step of challenge/response authentication. + + @param response: The text of a response. If None, this + function has been called as a result of an AUTH command with + no initial response. A response of '*' aborts authentication, + as per RFC 2554. + """ + if self.portal is None: + self.sendCode(454, b'Temporary authentication failure') + self.mode = COMMAND + return + + if response is None: + challenge = self.challenger.getChallenge() + encoded = base64.b64encode(challenge) + self.sendCode(334, encoded) + return + + if response == b'*': + self.sendCode(501, b'Authentication aborted') + self.challenger = None + self.mode = COMMAND + return + + try: + uncoded = base64.b64decode(response) + except (TypeError, binascii.Error): + self.sendCode(501, b'Syntax error in parameters or arguments') + self.challenger = None + self.mode = COMMAND + return + + self.challenger.setResponse(uncoded) + if self.challenger.moreChallenges(): + challenge = self.challenger.getChallenge() + coded = base64.b64encode(challenge) + self.sendCode(334, coded) + return + + self.mode = COMMAND + result = self.portal.login( + self.challenger, None, + IMessageDeliveryFactory, IMessageDelivery) + result.addCallback(self._cbAuthenticated) + result.addCallback(lambda ign: self.sendCode(235, + b'Authentication successful.')) + result.addErrback(self._ebAuthenticated) + + + +class SenderMixin: + """ + Utility class for sending emails easily. + + Use with SMTPSenderFactory or ESMTPSenderFactory. + """ + done = 0 + + def getMailFrom(self): + if not self.done: + self.done = 1 + return str(self.factory.fromEmail) + else: + return None + + + def getMailTo(self): + return self.factory.toEmail + + + def getMailData(self): + return self.factory.file + + + def sendError(self, exc): + # Call the base class to close the connection with the SMTP server + SMTPClient.sendError(self, exc) + + # Do not retry to connect to SMTP Server if: + # 1. No more retries left (This allows the correct error to be returned to the errorback) + # 2. retry is false + # 3. The error code is not in the 4xx range (Communication Errors) + + if (self.factory.retries >= 0 or + (not exc.retry and not (exc.code >= 400 and exc.code < 500))): + self.factory.sendFinished = True + self.factory.result.errback(exc) + + + def sentMail(self, code, resp, numOk, addresses, log): + # Do not retry, the SMTP server acknowledged the request + self.factory.sendFinished = True + if code not in SUCCESS: + errlog = [] + for addr, acode, aresp in addresses: + if acode not in SUCCESS: + errlog.append((addr + b": " + + networkString("%03d" % (acode,)) + + b" " + aresp)) + + errlog.append(log.str()) + + exc = SMTPDeliveryError(code, resp, b'\n'.join(errlog), addresses) + self.factory.result.errback(exc) + else: + self.factory.result.callback((numOk, addresses)) + + + +class SMTPSender(SenderMixin, SMTPClient): + """ + SMTP protocol that sends a single email based on information it + gets from its factory, a L{SMTPSenderFactory}. + """ + + + +class SMTPSenderFactory(protocol.ClientFactory): + """ + Utility factory for sending emails easily. + + @type currentProtocol: L{SMTPSender} + @ivar currentProtocol: The current running protocol returned by + L{buildProtocol}. + + @type sendFinished: C{bool} + @ivar sendFinished: When the value is set to True, it means the message has + been sent or there has been an unrecoverable error or the sending has + been cancelled. The default value is False. + """ + + domain = DNSNAME + protocol = SMTPSender + + def __init__(self, fromEmail, toEmail, file, deferred, retries=5, + timeout=None): + """ + @param fromEmail: The RFC 2821 address from which to send this + message. + + @param toEmail: A sequence of RFC 2821 addresses to which to + send this message. + + @param file: A file-like object containing the message to send. + + @param deferred: A Deferred to callback or errback when sending + of this message completes. + @type deferred: L{defer.Deferred} + + @param retries: The number of times to retry delivery of this + message. + + @param timeout: Period, in seconds, for which to wait for + server responses, or None to wait forever. + """ + assert isinstance(retries, (int, long)) + + if isinstance(toEmail, unicode): + toEmail = [toEmail.encode('ascii')] + elif isinstance(toEmail, bytes): + toEmail = [toEmail] + else: + toEmailFinal = [] + for _email in toEmail: + if not isinstance(_email, bytes): + _email = _email.encode('ascii') + + toEmailFinal.append(_email) + toEmail = toEmailFinal + + self.fromEmail = Address(fromEmail) + self.nEmails = len(toEmail) + self.toEmail = toEmail + self.file = file + self.result = deferred + self.result.addBoth(self._removeDeferred) + self.sendFinished = False + self.currentProtocol = None + + self.retries = -retries + self.timeout = timeout + + + def _removeDeferred(self, result): + del self.result + return result + + + def clientConnectionFailed(self, connector, err): + self._processConnectionError(connector, err) + + + def clientConnectionLost(self, connector, err): + self._processConnectionError(connector, err) + + + def _processConnectionError(self, connector, err): + self.currentProtocol = None + if (self.retries < 0) and (not self.sendFinished): + log.msg("SMTP Client retrying server. Retry: %s" % -self.retries) + + # Rewind the file in case part of it was read while attempting to + # send the message. + self.file.seek(0, 0) + connector.connect() + self.retries += 1 + elif not self.sendFinished: + # If we were unable to communicate with the SMTP server a ConnectionDone will be + # returned. We want a more clear error message for debugging + if err.check(error.ConnectionDone): + err.value = SMTPConnectError(-1, "Unable to connect to server.") + self.result.errback(err.value) + + + def buildProtocol(self, addr): + p = self.protocol(self.domain, self.nEmails*2+2) + p.factory = self + p.timeout = self.timeout + self.currentProtocol = p + self.result.addBoth(self._removeProtocol) + return p + + + def _removeProtocol(self, result): + """ + Remove the protocol created in C{buildProtocol}. + + @param result: The result/error passed to the callback/errback of + L{defer.Deferred}. + + @return: The C{result} untouched. + """ + if self.currentProtocol: + self.currentProtocol = None + return result + + + +class LOGINCredentials(_lcredentials): + """ + L{LOGINCredentials} generates challenges for I{LOGIN} authentication. + + For interoperability with Outlook, the challenge generated does not exactly + match the one defined in the + U{draft specification}. + """ + + def __init__(self): + _lcredentials.__init__(self) + self.challenges = [b'Password:', b'Username:'] + + + +@implementer(IClientAuthentication) +class PLAINAuthenticator: + def __init__(self, user): + self.user = user + + + def getName(self): + return b"PLAIN" + + + def challengeResponse(self, secret, chal=1): + if chal == 1: + return self.user + b'\0' + self.user + b'\0' + secret + else: + return b'\0' + self.user + b'\0' + secret + + + +class ESMTPSender(SenderMixin, ESMTPClient): + + requireAuthentication = True + requireTransportSecurity = True + + def __init__(self, username, secret, contextFactory=None, *args, **kw): + self.heloFallback = 0 + self.username = username + + if contextFactory is None: + contextFactory = self._getContextFactory() + + ESMTPClient.__init__(self, secret, contextFactory, *args, **kw) + + self._registerAuthenticators() + + + def _registerAuthenticators(self): + # Register Authenticator in order from most secure to least secure + self.registerAuthenticator(CramMD5ClientAuthenticator(self.username)) + self.registerAuthenticator(LOGINAuthenticator(self.username)) + self.registerAuthenticator(PLAINAuthenticator(self.username)) + + + def _getContextFactory(self): + if self.context is not None: + return self.context + try: + from twisted.internet import ssl + except ImportError: + return None + else: + try: + context = ssl.ClientContextFactory() + context.method = ssl.SSL.TLSv1_METHOD + return context + except AttributeError: + return None + + + +class ESMTPSenderFactory(SMTPSenderFactory): + """ + Utility factory for sending emails easily. + + @type currentProtocol: L{ESMTPSender} + @ivar currentProtocol: The current running protocol as made by + L{buildProtocol}. + """ + protocol = ESMTPSender + + def __init__(self, username, password, fromEmail, toEmail, file, + deferred, retries=5, timeout=None, + contextFactory=None, heloFallback=False, + requireAuthentication=True, + requireTransportSecurity=True): + + SMTPSenderFactory.__init__(self, fromEmail, toEmail, file, deferred, retries, timeout) + self.username = username + self.password = password + self._contextFactory = contextFactory + self._heloFallback = heloFallback + self._requireAuthentication = requireAuthentication + self._requireTransportSecurity = requireTransportSecurity + + + def buildProtocol(self, addr): + """ + Build an L{ESMTPSender} protocol configured with C{heloFallback}, + C{requireAuthentication}, and C{requireTransportSecurity} as specified + in L{__init__}. + + This sets L{currentProtocol} on the factory, as well as returning it. + + @rtype: L{ESMTPSender} + """ + p = self.protocol(self.username, self.password, self._contextFactory, + self.domain, self.nEmails*2+2) + p.heloFallback = self._heloFallback + p.requireAuthentication = self._requireAuthentication + p.requireTransportSecurity = self._requireTransportSecurity + p.factory = self + p.timeout = self.timeout + self.currentProtocol = p + self.result.addBoth(self._removeProtocol) + return p + + + +def sendmail(smtphost, from_addr, to_addrs, msg, senderDomainName=None, port=25, + reactor=reactor, username=None, password=None, + requireAuthentication=False, requireTransportSecurity=False): + """ + Send an email. + + This interface is intended to be a replacement for L{smtplib.SMTP.sendmail} + and related methods. To maintain backwards compatibility, it will fall back + to plain SMTP, if ESMTP support is not available. If ESMTP support is + available, it will attempt to provide encryption via STARTTLS and + authentication if a secret is provided. + + @param smtphost: The host the message should be sent to. + @type smtphost: L{bytes} + + @param from_addr: The (envelope) address sending this mail. + @type from_addr: L{bytes} + + @param to_addrs: A list of addresses to send this mail to. A string will + be treated as a list of one address. + @type to_addr: L{list} of L{bytes} or L{bytes} + + @param msg: The message, including headers, either as a file or a string. + File-like objects need to support read() and close(). Lines must be + delimited by '\\n'. If you pass something that doesn't look like a file, + we try to convert it to a string (so you should be able to pass an + L{email.message} directly, but doing the conversion with + L{email.generator} manually will give you more control over the process). + + @param senderDomainName: Name by which to identify. If None, try to pick + something sane (but this depends on external configuration and may not + succeed). + @type senderDomainName: L{bytes} + + @param port: Remote port to which to connect. + @type port: L{int} + + @param username: The username to use, if wanting to authenticate. + @type username: L{bytes} or L{unicode} + + @param password: The secret to use, if wanting to authenticate. If you do + not specify this, SMTP authentication will not occur. + @type password: L{bytes} or L{unicode} + + @param requireTransportSecurity: Whether or not STARTTLS is required. + @type requireTransportSecurity: L{bool} + + @param requireAuthentication: Whether or not authentication is required. + @type requireAuthentication: L{bool} + + @param reactor: The L{reactor} used to make the TCP connection. + + @rtype: L{Deferred} + @returns: A cancellable L{Deferred}, its callback will be called if a + message is sent to ANY address, the errback if no message is sent. When + the C{cancel} method is called, it will stop retrying and disconnect + the connection immediately. + + The callback will be called with a tuple (numOk, addresses) where numOk + is the number of successful recipient addresses and addresses is a list + of tuples (address, code, resp) giving the response to the RCPT command + for each address. + """ + if not hasattr(msg, 'read'): + # It's not a file + msg = BytesIO(bytes(msg)) + + + def cancel(d): + """ + Cancel the L{twisted.mail.smtp.sendmail} call, tell the factory not to + retry and disconnect the connection. + + @param d: The L{defer.Deferred} to be cancelled. + """ + factory.sendFinished = True + if factory.currentProtocol: + factory.currentProtocol.transport.abortConnection() + else: + # Connection hasn't been made yet + connector.disconnect() + + d = defer.Deferred(cancel) + + if isinstance(username, unicode): + username = username.encode("utf-8") + if isinstance(password, unicode): + password = password.encode("utf-8") + + factory = ESMTPSenderFactory(username, password, from_addr, to_addrs, msg, + d, heloFallback=True, requireAuthentication=requireAuthentication, + requireTransportSecurity=requireTransportSecurity) + + if senderDomainName is not None: + factory.domain = networkString(senderDomainName) + + connector = reactor.connectTCP(smtphost, port, factory) + + return d + + + +import codecs +def xtext_encode(s, errors=None): + r = [] + for ch in iterbytes(s): + o = ord(ch) + if ch == '+' or ch == '=' or o < 33 or o > 126: + r.append(networkString('+%02X' % (o,))) + else: + r.append(_bytesChr(o)) + return (b''.join(r), len(s)) + + + +def xtext_decode(s, errors=None): + """ + Decode the xtext-encoded string C{s}. + + @param s: String to decode. + @param errors: codec error handling scheme. + @return: The decoded string. + """ + r = [] + i = 0 + while i < len(s): + if s[i:i+1] == b'+': + try: + r.append(chr(int(bytes(s[i + 1:i + 3]), 16))) + except ValueError: + r.append(ord(s[i:i + 3])) + i += 3 + else: + r.append(bytes(s[i:i+1]).decode('ascii')) + i += 1 + return (''.join(r), len(s)) + + + +class xtextStreamReader(codecs.StreamReader): + def decode(self, s, errors='strict'): + return xtext_decode(s) + + + +class xtextStreamWriter(codecs.StreamWriter): + def decode(self, s, errors='strict'): + return xtext_encode(s) + + + +def xtext_codec(name): + if name == 'xtext': + return (xtext_encode, xtext_decode, xtextStreamReader, xtextStreamWriter) +codecs.register(xtext_codec) diff --git a/contrib/python/Twisted/py2/twisted/mail/tap.py b/contrib/python/Twisted/py2/twisted/mail/tap.py new file mode 100644 index 00000000000..0a3a4317a55 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/mail/tap.py @@ -0,0 +1,394 @@ +# -*- test-case-name: twisted.mail.test.test_options -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Support for creating mail servers with twistd. +""" + +import os + +from twisted.mail import mail +from twisted.mail import maildir +from twisted.mail import relay +from twisted.mail import relaymanager +from twisted.mail import alias + +from twisted.internet import endpoints + +from twisted.python import usage + +from twisted.cred import checkers +from twisted.cred import strcred + +from twisted.application import internet + + +class Options(usage.Options, strcred.AuthOptionMixin): + """ + An options list parser for twistd mail. + + @type synopsis: L{bytes} + @ivar synopsis: A description of options for use in the usage message. + + @type optParameters: L{list} of L{list} of (0) L{bytes}, (1) L{bytes}, + (2) L{object}, (3) L{bytes}, (4) L{None} or + callable which takes L{bytes} and returns L{object} + @ivar optParameters: Information about supported parameters. See + L{Options } for details. + + @type optFlags: L{list} of L{list} of (0) L{bytes}, (1) L{bytes} or + L{None}, (2) L{bytes} + @ivar optFlags: Information about supported flags. See + L{Options } for details. + + @type _protoDefaults: L{dict} mapping L{bytes} to L{int} + @ivar _protoDefaults: A mapping of default service to port. + + @type compData: L{Completions } + @ivar compData: Metadata for the shell tab completion system. + + @type longdesc: L{bytes} + @ivar longdesc: A long description of the plugin for use in the usage + message. + + @type service: L{MailService} + @ivar service: The email service. + + @type last_domain: L{IDomain} provider or L{None} + @ivar last_domain: The most recently specified domain. + """ + synopsis = "[options]" + + optParameters = [ + ["relay", "R", None, + "Relay messages according to their envelope 'To', using " + "the given path as a queue directory."], + + ["hostname", "H", None, + "The hostname by which to identify this server."], + ] + + optFlags = [ + ["esmtp", "E", "Use RFC 1425/1869 SMTP extensions"], + ["disable-anonymous", None, + "Disallow non-authenticated SMTP connections"], + ["no-pop3", None, "Disable the default POP3 server."], + ["no-smtp", None, "Disable the default SMTP server."], + ] + + _protoDefaults = { + "pop3": 8110, + "smtp": 8025, + } + + compData = usage.Completions( + optActions={"hostname": usage.CompleteHostnames()} + ) + + longdesc = """ + An SMTP / POP3 email server plugin for twistd. + + Examples: + + 1. SMTP and POP server + + twistd mail --maildirdbmdomain=example.com=/tmp/example.com + --user=joe=password + + Starts an SMTP server that only accepts emails to joe@example.com and saves + them to /tmp/example.com. + + Also starts a POP mail server which will allow a client to log in using + username: joe@example.com and password: password and collect any email that + has been saved in /tmp/example.com. + + 2. SMTP relay + + twistd mail --relay=/tmp/mail_queue + + Starts an SMTP server that accepts emails to any email address and relays + them to an appropriate remote SMTP server. Queued emails will be + temporarily stored in /tmp/mail_queue. + """ + + def __init__(self): + """ + Parse options and create a mail service. + """ + usage.Options.__init__(self) + self.service = mail.MailService() + self.last_domain = None + for service in self._protoDefaults: + self[service] = [] + + + def addEndpoint(self, service, description): + """ + Add an endpoint to a service. + + @type service: L{bytes} + @param service: A service, either C{b'smtp'} or C{b'pop3'}. + + @type description: L{bytes} + @param description: An endpoint description string or a TCP port + number. + """ + from twisted.internet import reactor + self[service].append(endpoints.serverFromString(reactor, description)) + + + def opt_pop3(self, description): + """ + Add a POP3 port listener on the specified endpoint. + + You can listen on multiple ports by specifying multiple --pop3 options. + """ + self.addEndpoint('pop3', description) + opt_p = opt_pop3 + + + def opt_smtp(self, description): + """ + Add an SMTP port listener on the specified endpoint. + + You can listen on multiple ports by specifying multiple --smtp options. + """ + self.addEndpoint('smtp', description) + opt_s = opt_smtp + + + def opt_default(self): + """ + Make the most recently specified domain the default domain. + """ + if self.last_domain: + self.service.addDomain('', self.last_domain) + else: + raise usage.UsageError("Specify a domain before specifying using --default") + opt_D = opt_default + + + def opt_maildirdbmdomain(self, domain): + """ + Generate an SMTP/POP3 virtual domain. + + This option requires an argument of the form 'NAME=PATH' where NAME is + the DNS domain name for which email will be accepted and where PATH is + a the filesystem path to a Maildir folder. + [Example: 'example.com=/tmp/example.com'] + """ + try: + name, path = domain.split('=') + except ValueError: + raise usage.UsageError("Argument to --maildirdbmdomain must be of the form 'name=path'") + + self.last_domain = maildir.MaildirDirdbmDomain(self.service, os.path.abspath(path)) + self.service.addDomain(name, self.last_domain) + opt_d = opt_maildirdbmdomain + + def opt_user(self, user_pass): + """ + Add a user and password to the last specified domain. + """ + try: + user, password = user_pass.split('=', 1) + except ValueError: + raise usage.UsageError("Argument to --user must be of the form 'user=password'") + if self.last_domain: + self.last_domain.addUser(user, password) + else: + raise usage.UsageError("Specify a domain before specifying users") + opt_u = opt_user + + + def opt_bounce_to_postmaster(self): + """ + Send undeliverable messages to the postmaster. + """ + self.last_domain.postmaster = 1 + opt_b = opt_bounce_to_postmaster + + + def opt_aliases(self, filename): + """ + Specify an aliases(5) file to use for the last specified domain. + """ + if self.last_domain is not None: + if mail.IAliasableDomain.providedBy(self.last_domain): + aliases = alias.loadAliasFile(self.service.domains, filename) + self.last_domain.setAliasGroup(aliases) + self.service.monitor.monitorFile( + filename, + AliasUpdater(self.service.domains, self.last_domain) + ) + else: + raise usage.UsageError( + "%s does not support alias files" % ( + self.last_domain.__class__.__name__, + ) + ) + else: + raise usage.UsageError("Specify a domain before specifying aliases") + opt_A = opt_aliases + + + def _getEndpoints(self, reactor, service): + """ + Return a list of endpoints for the specified service, constructing + defaults if necessary. + + If no endpoints were configured for the service and the protocol + was not explicitly disabled with a I{--no-*} option, a default + endpoint for the service is created. + + @type reactor: L{IReactorTCP } + provider + @param reactor: If any endpoints are created, the reactor with + which they are created. + + @type service: L{bytes} + @param service: The type of service for which to retrieve endpoints, + either C{b'pop3'} or C{b'smtp'}. + + @rtype: L{list} of L{IStreamServerEndpoint + } provider + @return: The endpoints for the specified service as configured by the + command line parameters. + """ + if self[service]: + # If there are any services set up, just return those. + return self[service] + elif self['no-' + service]: + # If there are no services, but the service was explicitly disabled, + # return nothing. + return [] + else: + # Otherwise, return the old default service. + return [ + endpoints.TCP4ServerEndpoint( + reactor, self._protoDefaults[service])] + + + def postOptions(self): + """ + Check the validity of the specified set of options and + configure authentication. + + @raise UsageError: When the set of options is invalid. + """ + from twisted.internet import reactor + + if self['esmtp'] and self['hostname'] is None: + raise usage.UsageError("--esmtp requires --hostname") + + # If the --auth option was passed, this will be present -- otherwise, + # it won't be, which is also a perfectly valid state. + if 'credCheckers' in self: + for ch in self['credCheckers']: + self.service.smtpPortal.registerChecker(ch) + + if not self['disable-anonymous']: + self.service.smtpPortal.registerChecker(checkers.AllowAnonymousAccess()) + + anything = False + for service in self._protoDefaults: + self[service] = self._getEndpoints(reactor, service) + if self[service]: + anything = True + + if not anything: + raise usage.UsageError("You cannot disable all protocols") + + + +class AliasUpdater: + """ + A callable object which updates the aliases for a domain from an aliases(5) + file. + + @ivar domains: See L{__init__}. + @ivar domain: See L{__init__}. + """ + def __init__(self, domains, domain): + """ + @type domains: L{dict} mapping L{bytes} to L{IDomain} provider + @param domains: A mapping of domain name to domain object + + @type domain: L{IAliasableDomain} provider + @param domain: The domain to update. + """ + self.domains = domains + self.domain = domain + + + def __call__(self, new): + """ + Update the aliases for a domain from an aliases(5) file. + + @type new: L{bytes} + @param new: The name of an aliases(5) file. + """ + self.domain.setAliasGroup(alias.loadAliasFile(self.domains, new)) + + + +def makeService(config): + """ + Configure a service for operating a mail server. + + The returned service may include POP3 servers, SMTP servers, or both, + depending on the configuration passed in. If there are multiple servers, + they will share all of their non-network state (i.e. the same user accounts + are available on all of them). + + @type config: L{Options } + @param config: Configuration options specifying which servers to include in + the returned service and where they should keep mail data. + + @rtype: L{IService } provider + @return: A service which contains the requested mail servers. + """ + if config['esmtp']: + rmType = relaymanager.SmartHostESMTPRelayingManager + smtpFactory = config.service.getESMTPFactory + else: + rmType = relaymanager.SmartHostSMTPRelayingManager + smtpFactory = config.service.getSMTPFactory + + if config['relay']: + dir = config['relay'] + if not os.path.isdir(dir): + os.mkdir(dir) + + config.service.setQueue(relaymanager.Queue(dir)) + default = relay.DomainQueuer(config.service) + + manager = rmType(config.service.queue) + if config['esmtp']: + manager.fArgs += (None, None) + manager.fArgs += (config['hostname'],) + + helper = relaymanager.RelayStateHelper(manager, 1) + helper.setServiceParent(config.service) + config.service.domains.setDefaultDomain(default) + + if config['pop3']: + f = config.service.getPOP3Factory() + for endpoint in config['pop3']: + svc = internet.StreamServerEndpointService(endpoint, f) + svc.setServiceParent(config.service) + + if config['smtp']: + f = smtpFactory() + if config['hostname']: + f.domain = config['hostname'] + f.fArgs = (f.domain,) + if config['esmtp']: + f.fArgs = (None, None) + f.fArgs + for endpoint in config['smtp']: + svc = internet.StreamServerEndpointService(endpoint, f) + svc.setServiceParent(config.service) + + return config.service diff --git a/contrib/python/Twisted/py2/twisted/names/__init__.py b/contrib/python/Twisted/py2/twisted/names/__init__.py new file mode 100644 index 00000000000..ccdf8ba3319 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Names: DNS server and client implementations. +""" diff --git a/contrib/python/Twisted/py2/twisted/names/_rfc1982.py b/contrib/python/Twisted/py2/twisted/names/_rfc1982.py new file mode 100644 index 00000000000..a895d82c4e2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/_rfc1982.py @@ -0,0 +1,278 @@ +# -*- test-case-name: twisted.names.test.test_rfc1982 -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Utilities for handling RFC1982 Serial Number Arithmetic. + +@see: U{http://tools.ietf.org/html/rfc1982} + +@var RFC4034_TIME_FORMAT: RRSIG Time field presentation format. The Signature + Expiration Time and Inception Time field values MUST be represented either + as an unsigned decimal integer indicating seconds since 1 January 1970 + 00:00:00 UTC, or in the form YYYYMMDDHHmmSS in UTC. See U{RRSIG Presentation + Format} +""" + +from __future__ import division, absolute_import + +import calendar +from datetime import datetime, timedelta + +from twisted.python.compat import nativeString +from twisted.python.util import FancyStrMixin + + +RFC4034_TIME_FORMAT = '%Y%m%d%H%M%S' + + + +class SerialNumber(FancyStrMixin, object): + """ + An RFC1982 Serial Number. + + This class implements RFC1982 DNS Serial Number Arithmetic. + + SNA is used in DNS and specifically in DNSSEC as defined in RFC4034 in the + DNSSEC Signature Expiration and Inception Fields. + + @see: U{https://tools.ietf.org/html/rfc1982} + @see: U{https://tools.ietf.org/html/rfc4034} + + @ivar _serialBits: See C{serialBits} of L{__init__}. + @ivar _number: See C{number} of L{__init__}. + @ivar _modulo: The value at which wrapping will occur. + @ivar _halfRing: Half C{_modulo}. If another L{SerialNumber} value is larger + than this, it would lead to a wrapped value which is larger than the + first and comparisons are therefore ambiguous. + @ivar _maxAdd: Half C{_modulo} plus 1. If another L{SerialNumber} value is + larger than this, it would lead to a wrapped value which is larger than + the first. Comparisons with the original value would therefore be + ambiguous. + """ + + showAttributes = ( + ('_number', 'number', '%d'), + ('_serialBits', 'serialBits', '%d'), + ) + + def __init__(self, number, serialBits=32): + """ + Construct an L{SerialNumber} instance. + + @param number: An L{int} which will be stored as the modulo + C{number % 2 ^ serialBits} + @type number: L{int} + + @param serialBits: The size of the serial number space. The power of two + which results in one larger than the largest integer corresponding + to a serial number value. + @type serialBits: L{int} + """ + self._serialBits = serialBits + self._modulo = 2 ** serialBits + self._halfRing = 2 ** (serialBits - 1) + self._maxAdd = 2 ** (serialBits - 1) - 1 + self._number = int(number) % self._modulo + + + def _convertOther(self, other): + """ + Check that a foreign object is suitable for use in the comparison or + arithmetic magic methods of this L{SerialNumber} instance. Raise + L{TypeError} if not. + + @param other: The foreign L{object} to be checked. + @return: C{other} after compatibility checks and possible coercion. + @raises: L{TypeError} if C{other} is not compatible. + """ + if not isinstance(other, SerialNumber): + raise TypeError( + 'cannot compare or combine %r and %r' % (self, other)) + + if self._serialBits != other._serialBits: + raise TypeError( + 'cannot compare or combine SerialNumber instances with ' + 'different serialBits. %r and %r' % (self, other)) + + return other + + + def __str__(self): + """ + Return a string representation of this L{SerialNumber} instance. + + @rtype: L{nativeString} + """ + return nativeString('%d' % (self._number,)) + + + def __int__(self): + """ + @return: The integer value of this L{SerialNumber} instance. + @rtype: L{int} + """ + return self._number + + + def __eq__(self, other): + """ + Allow rich equality comparison with another L{SerialNumber} instance. + + @type other: L{SerialNumber} + """ + other = self._convertOther(other) + return other._number == self._number + + + def __ne__(self, other): + """ + Allow rich equality comparison with another L{SerialNumber} instance. + + @type other: L{SerialNumber} + """ + return not self.__eq__(other) + + + def __lt__(self, other): + """ + Allow I{less than} comparison with another L{SerialNumber} instance. + + @type other: L{SerialNumber} + """ + other = self._convertOther(other) + return ( + (self._number < other._number + and (other._number - self._number) < self._halfRing) + or + (self._number > other._number + and (self._number - other._number) > self._halfRing) + ) + + + def __gt__(self, other): + """ + Allow I{greater than} comparison with another L{SerialNumber} instance. + + @type other: L{SerialNumber} + @rtype: L{bool} + """ + other = self._convertOther(other) + return ( + (self._number < other._number + and (other._number - self._number) > self._halfRing) + or + (self._number > other._number + and (self._number - other._number) < self._halfRing) + ) + + + def __le__(self, other): + """ + Allow I{less than or equal} comparison with another L{SerialNumber} + instance. + + @type other: L{SerialNumber} + @rtype: L{bool} + """ + other = self._convertOther(other) + return self == other or self < other + + + def __ge__(self, other): + """ + Allow I{greater than or equal} comparison with another L{SerialNumber} + instance. + + @type other: L{SerialNumber} + @rtype: L{bool} + """ + other = self._convertOther(other) + return self == other or self > other + + + def __add__(self, other): + """ + Allow I{addition} with another L{SerialNumber} instance. + + Serial numbers may be incremented by the addition of a positive + integer n, where n is taken from the range of integers + [0 .. (2^(SERIAL_BITS - 1) - 1)]. For a sequence number s, the + result of such an addition, s', is defined as + + s' = (s + n) modulo (2 ^ SERIAL_BITS) + + where the addition and modulus operations here act upon values that are + non-negative values of unbounded size in the usual ways of integer + arithmetic. + + Addition of a value outside the range + [0 .. (2^(SERIAL_BITS - 1) - 1)] is undefined. + + @see: U{http://tools.ietf.org/html/rfc1982#section-3.1} + + @type other: L{SerialNumber} + @rtype: L{SerialNumber} + @raises: L{ArithmeticError} if C{other} is more than C{_maxAdd} + ie more than half the maximum value of this serial number. + """ + other = self._convertOther(other) + if other._number <= self._maxAdd: + return SerialNumber( + (self._number + other._number) % self._modulo, + serialBits=self._serialBits) + else: + raise ArithmeticError( + 'value %r outside the range 0 .. %r' % ( + other._number, self._maxAdd,)) + + + def __hash__(self): + """ + Allow L{SerialNumber} instances to be hashed for use as L{dict} keys. + + @rtype: L{int} + """ + return hash(self._number) + + + @classmethod + def fromRFC4034DateString(cls, utcDateString): + """ + Create an L{SerialNumber} instance from a date string in format + 'YYYYMMDDHHMMSS' described in U{RFC4034 + 3.2}. + + The L{SerialNumber} instance stores the date as a 32bit UNIX timestamp. + + @see: U{https://tools.ietf.org/html/rfc4034#section-3.1.5} + + @param utcDateString: A UTC date/time string of format I{YYMMDDhhmmss} + which will be converted to seconds since the UNIX epoch. + @type utcDateString: L{unicode} + + @return: An L{SerialNumber} instance containing the supplied date as a + 32bit UNIX timestamp. + """ + parsedDate = datetime.strptime(utcDateString, RFC4034_TIME_FORMAT) + secondsSinceEpoch = calendar.timegm(parsedDate.utctimetuple()) + return cls(secondsSinceEpoch, serialBits=32) + + + def toRFC4034DateString(self): + """ + Calculate a date by treating the current L{SerialNumber} value as a UNIX + timestamp and return a date string in the format described in + U{RFC4034 3.2}. + + @return: The date string. + """ + # Can't use datetime.utcfromtimestamp, because it seems to overflow the + # signed 32bit int used in the underlying C library. SNA is unsigned + # and capable of handling all timestamps up to 2**32. + d = datetime(1970, 1, 1) + timedelta(seconds=self._number) + return nativeString(d.strftime(RFC4034_TIME_FORMAT)) + + + +__all__ = ['SerialNumber'] diff --git a/contrib/python/Twisted/py2/twisted/names/authority.py b/contrib/python/Twisted/py2/twisted/names/authority.py new file mode 100644 index 00000000000..1abc4f828dc --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/authority.py @@ -0,0 +1,543 @@ +# -*- test-case-name: twisted.names.test.test_names -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Authoritative resolvers. +""" + +from __future__ import absolute_import, division + +import os +import time + +from twisted.names import dns, error, common +from twisted.internet import defer +from twisted.python import failure +from twisted.python.compat import execfile, nativeString, _PY3 +from twisted.python.filepath import FilePath + + + +def getSerial(filename='/tmp/twisted-names.serial'): + """ + Return a monotonically increasing (across program runs) integer. + + State is stored in the given file. If it does not exist, it is + created with rw-/---/--- permissions. + + This manipulates process-global state by calling C{os.umask()}, so it isn't + thread-safe. + + @param filename: Path to a file that is used to store the state across + program runs. + @type filename: L{str} + + @return: a monotonically increasing number + @rtype: L{str} + """ + serial = time.strftime('%Y%m%d') + + o = os.umask(0o177) + try: + if not os.path.exists(filename): + with open(filename, 'w') as f: + f.write(serial + ' 0') + finally: + os.umask(o) + + with open(filename, 'r') as serialFile: + lastSerial, zoneID = serialFile.readline().split() + + zoneID = (lastSerial == serial) and (int(zoneID) + 1) or 0 + + with open(filename, 'w') as serialFile: + serialFile.write('%s %d' % (serial, zoneID)) + + serial = serial + ('%02d' % (zoneID,)) + return serial + + + +class FileAuthority(common.ResolverBase): + """ + An Authority that is loaded from a file. + + This is an abstract class that implements record search logic. To create + a functional resolver, subclass it and override the L{loadFile} method. + + @ivar _ADDITIONAL_PROCESSING_TYPES: Record types for which additional + processing will be done. + + @ivar _ADDRESS_TYPES: Record types which are useful for inclusion in the + additional section generated during additional processing. + + @ivar soa: A 2-tuple containing the SOA domain name as a L{bytes} and a + L{dns.Record_SOA}. + + @ivar records: A mapping of domains (as lowercased L{bytes}) to records. + @type records: L{dict} with L{byte} keys + """ + # See https://twistedmatrix.com/trac/ticket/6650 + _ADDITIONAL_PROCESSING_TYPES = (dns.CNAME, dns.MX, dns.NS) + _ADDRESS_TYPES = (dns.A, dns.AAAA) + + soa = None + records = None + + def __init__(self, filename): + common.ResolverBase.__init__(self) + self.loadFile(filename) + self._cache = {} + + + def __setstate__(self, state): + self.__dict__ = state + + + def loadFile(self, filename): + """ + Load DNS records from a file. + + This method populates the I{soa} and I{records} attributes. It must be + overridden in a subclass. It is called once from the initializer. + + @param filename: The I{filename} parameter that was passed to the + initilizer. + + @returns: L{None} -- the return value is ignored + """ + + + def _additionalRecords(self, answer, authority, ttl): + """ + Find locally known information that could be useful to the consumer of + the response and construct appropriate records to include in the + I{additional} section of that response. + + Essentially, implement RFC 1034 section 4.3.2 step 6. + + @param answer: A L{list} of the records which will be included in the + I{answer} section of the response. + + @param authority: A L{list} of the records which will be included in + the I{authority} section of the response. + + @param ttl: The default TTL for records for which this is not otherwise + specified. + + @return: A generator of L{dns.RRHeader} instances for inclusion in the + I{additional} section. These instances represent extra information + about the records in C{answer} and C{authority}. + """ + for record in answer + authority: + if record.type in self._ADDITIONAL_PROCESSING_TYPES: + name = record.payload.name.name + for rec in self.records.get(name.lower(), ()): + if rec.TYPE in self._ADDRESS_TYPES: + yield dns.RRHeader( + name, rec.TYPE, dns.IN, + rec.ttl or ttl, rec, auth=True) + + + def _lookup(self, name, cls, type, timeout=None): + """ + Determine a response to a particular DNS query. + + @param name: The name which is being queried and for which to lookup a + response. + @type name: L{bytes} + + @param cls: The class which is being queried. Only I{IN} is + implemented here and this value is presently disregarded. + @type cls: L{int} + + @param type: The type of records being queried. See the types defined + in L{twisted.names.dns}. + @type type: L{int} + + @param timeout: All processing is done locally and a result is + available immediately, so the timeout value is ignored. + + @return: A L{Deferred} that fires with a L{tuple} of three sets of + response records (to comprise the I{answer}, I{authority}, and + I{additional} sections of a DNS response) or with a L{Failure} if + there is a problem processing the query. + """ + cnames = [] + results = [] + authority = [] + additional = [] + default_ttl = max(self.soa[1].minimum, self.soa[1].expire) + + domain_records = self.records.get(name.lower()) + + if domain_records: + for record in domain_records: + if record.ttl is not None: + ttl = record.ttl + else: + ttl = default_ttl + + if (record.TYPE == dns.NS and + name.lower() != self.soa[0].lower()): + # NS record belong to a child zone: this is a referral. As + # NS records are authoritative in the child zone, ours here + # are not. RFC 2181, section 6.1. + authority.append( + dns.RRHeader( + name, record.TYPE, dns.IN, ttl, record, auth=False + ) + ) + elif record.TYPE == type or type == dns.ALL_RECORDS: + results.append( + dns.RRHeader( + name, record.TYPE, dns.IN, ttl, record, auth=True + ) + ) + if record.TYPE == dns.CNAME: + cnames.append( + dns.RRHeader( + name, record.TYPE, dns.IN, ttl, record, auth=True + ) + ) + if not results: + results = cnames + + # Sort of https://tools.ietf.org/html/rfc1034#section-4.3.2 . + # See https://twistedmatrix.com/trac/ticket/6732 + additionalInformation = self._additionalRecords( + results, authority, default_ttl) + if cnames: + results.extend(additionalInformation) + else: + additional.extend(additionalInformation) + + if not results and not authority: + # Empty response. Include SOA record to allow clients to cache + # this response. RFC 1034, sections 3.7 and 4.3.4, and RFC 2181 + # section 7.1. + authority.append( + dns.RRHeader( + self.soa[0], dns.SOA, dns.IN, ttl, self.soa[1], + auth=True + ) + ) + return defer.succeed((results, authority, additional)) + else: + if dns._isSubdomainOf(name, self.soa[0]): + # We may be the authority and we didn't find it. + # XXX: The QNAME may also be in a delegated child zone. See + # #6581 and #6580 + return defer.fail( + failure.Failure(dns.AuthoritativeDomainError(name)) + ) + else: + # The QNAME is not a descendant of this zone. Fail with + # DomainError so that the next chained authority or + # resolver will be queried. + return defer.fail(failure.Failure(error.DomainError(name))) + + + def lookupZone(self, name, timeout=10): + name = dns.domainString(name) + if self.soa[0].lower() == name.lower(): + # Wee hee hee hooo yea + default_ttl = max(self.soa[1].minimum, self.soa[1].expire) + if self.soa[1].ttl is not None: + soa_ttl = self.soa[1].ttl + else: + soa_ttl = default_ttl + results = [ + dns.RRHeader( + self.soa[0], dns.SOA, dns.IN, soa_ttl, self.soa[1], + auth=True + ) + ] + for (k, r) in self.records.items(): + for rec in r: + if rec.ttl is not None: + ttl = rec.ttl + else: + ttl = default_ttl + if rec.TYPE != dns.SOA: + results.append( + dns.RRHeader( + k, rec.TYPE, dns.IN, ttl, rec, auth=True + ) + ) + results.append(results[0]) + return defer.succeed((results, (), ())) + return defer.fail(failure.Failure(dns.DomainError(name))) + + + def _cbAllRecords(self, results): + ans, auth, add = [], [], [] + for res in results: + if res[0]: + ans.extend(res[1][0]) + auth.extend(res[1][1]) + add.extend(res[1][2]) + return ans, auth, add + + + +class PySourceAuthority(FileAuthority): + """ + A FileAuthority that is built up from Python source code. + """ + def loadFile(self, filename): + g, l = self.setupConfigNamespace(), {} + execfile(filename, g, l) + if 'zone' not in l: + raise ValueError("No zone defined in " + filename) + + self.records = {} + for rr in l['zone']: + if isinstance(rr[1], dns.Record_SOA): + self.soa = rr + self.records.setdefault(rr[0].lower(), []).append(rr[1]) + + + def wrapRecord(self, type): + return lambda name, *arg, **kw: (name, type(*arg, **kw)) + + + def setupConfigNamespace(self): + r = {} + items = dns.__dict__.iterkeys() + for record in [x for x in items if x.startswith('Record_')]: + type = getattr(dns, record) + f = self.wrapRecord(type) + r[record[len('Record_'):]] = f + return r + + + +class BindAuthority(FileAuthority): + """ + An Authority that loads U{BIND zone files + }. + + Supports only C{$ORIGIN} and C{$TTL} directives. + """ + def loadFile(self, filename): + """ + Load records from C{filename}. + + @param filename: file to read from + @type filename: L{bytes} + """ + fp = FilePath(filename) + # Not the best way to set an origin. It can be set using $ORIGIN + # though. + self.origin = nativeString(fp.basename() + b'.') + + lines = fp.getContent().splitlines(True) + lines = self.stripComments(lines) + lines = self.collapseContinuations(lines) + self.parseLines(lines) + + + def stripComments(self, lines): + """ + Strip comments from C{lines}. + + @param lines: lines to work on + @type lines: iterable of L{bytes} + + @return: C{lines} sans comments. + """ + return ( + a.find(b';') == -1 and a or a[:a.find(b';')] for a in [ + b.strip() for b in lines + ] + ) + + + def collapseContinuations(self, lines): + """ + Transform multiline statements into single lines. + + @param lines: lines to work on + @type lines: iterable of L{bytes} + + @return: iterable of continuous lines + """ + l = [] + state = 0 + for line in lines: + if state == 0: + if line.find(b'(') == -1: + l.append(line) + else: + l.append(line[:line.find(b'(')]) + state = 1 + else: + if line.find(b')') != -1: + l[-1] += b' ' + line[:line.find(b')')] + state = 0 + else: + l[-1] += b' ' + line + return filter(None, (line.split() for line in l)) + + + def parseLines(self, lines): + """ + Parse C{lines}. + + @param lines: lines to work on + @type lines: iterable of L{bytes} + """ + ttl = 60 * 60 * 3 + origin = self.origin + + self.records = {} + + for line in lines: + if line[0] == b'$TTL': + ttl = dns.str2time(line[1]) + elif line[0] == b'$ORIGIN': + origin = line[1] + elif line[0] == b'$INCLUDE': + raise NotImplementedError('$INCLUDE directive not implemented') + elif line[0] == b'$GENERATE': + raise NotImplementedError( + '$GENERATE directive not implemented' + ) + else: + self.parseRecordLine(origin, ttl, line) + + # If the origin changed, reflect that within the instance. + self.origin = origin + + + def addRecord(self, owner, ttl, type, domain, cls, rdata): + """ + Add a record to our authority. Expand domain with origin if necessary. + + @param owner: origin? + @type owner: L{bytes} + + @param ttl: time to live for the record + @type ttl: L{int} + + @param domain: the domain for which the record is to be added + @type domain: L{bytes} + + @param type: record type + @type type: L{str} + + @param cls: record class + @type cls: L{str} + + @param rdata: record data + @type rdata: L{list} of L{bytes} + """ + if not domain.endswith(b'.'): + domain = domain + b'.' + owner[:-1] + else: + domain = domain[:-1] + f = getattr(self, 'class_%s' % (cls,), None) + if f: + f(ttl, type, domain, rdata) + else: + raise NotImplementedError( + "Record class %r not supported" % (cls,) + ) + + + def class_IN(self, ttl, type, domain, rdata): + """ + Simulate a class IN and recurse into the actual class. + + @param ttl: time to live for the record + @type ttl: L{int} + + @param type: record type + @type type: str + + @param domain: the domain + @type domain: bytes + + @param rdata: + @type rdate: bytes + """ + record = getattr(dns, 'Record_%s' % (nativeString(type),), None) + if record: + r = record(*rdata) + r.ttl = ttl + self.records.setdefault(domain.lower(), []).append(r) + + if type == 'SOA': + self.soa = (domain, r) + else: + raise NotImplementedError( + "Record type %r not supported" % (nativeString(type),) + ) + + + def parseRecordLine(self, origin, ttl, line): + """ + Parse a C{line} from a zone file respecting C{origin} and C{ttl}. + + Add resulting records to authority. + + @param origin: starting point for the zone + @type origin: L{bytes} + + @param ttl: time to live for the record + @type ttl: L{int} + + @param line: zone file line to parse; split by word + @type line: L{list} of L{bytes} + """ + if _PY3: + queryClasses = set( + qc.encode("ascii") for qc in dns.QUERY_CLASSES.values() + ) + queryTypes = set( + qt.encode("ascii") for qt in dns.QUERY_TYPES.values() + ) + else: + queryClasses = set(dns.QUERY_CLASSES.values()) + queryTypes = set(dns.QUERY_TYPES.values()) + + markers = queryClasses | queryTypes + + cls = b'IN' + owner = origin + + if line[0] == b'@': + line = line[1:] + owner = origin + elif not line[0].isdigit() and line[0] not in markers: + owner = line[0] + line = line[1:] + + if line[0].isdigit() or line[0] in markers: + domain = owner + owner = origin + else: + domain = line[0] + line = line[1:] + + if line[0] in queryClasses: + cls = line[0] + line = line[1:] + if line[0].isdigit(): + ttl = int(line[0]) + line = line[1:] + elif line[0].isdigit(): + ttl = int(line[0]) + line = line[1:] + if line[0] in queryClasses: + cls = line[0] + line = line[1:] + + type = line[0] + rdata = line[1:] + + self.addRecord( + owner, ttl, nativeString(type), domain, nativeString(cls), rdata + ) diff --git a/contrib/python/Twisted/py2/twisted/names/cache.py b/contrib/python/Twisted/py2/twisted/names/cache.py new file mode 100644 index 00000000000..c9062161260 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/cache.py @@ -0,0 +1,125 @@ +# -*- test-case-name: twisted.names.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An in-memory caching resolver. +""" + +from __future__ import division, absolute_import + +from twisted.names import dns, common +from twisted.python import failure, log +from twisted.internet import defer + + + +class CacheResolver(common.ResolverBase): + """ + A resolver that serves records from a local, memory cache. + + @ivar _reactor: A provider of L{interfaces.IReactorTime}. + """ + cache = None + + def __init__(self, cache=None, verbose=0, reactor=None): + common.ResolverBase.__init__(self) + + self.cache = {} + self.verbose = verbose + self.cancel = {} + if reactor is None: + from twisted.internet import reactor + self._reactor = reactor + + if cache: + for query, (seconds, payload) in cache.items(): + self.cacheResult(query, payload, seconds) + + + def __setstate__(self, state): + self.__dict__ = state + + now = self._reactor.seconds() + for (k, (when, (ans, add, ns))) in self.cache.items(): + diff = now - when + for rec in ans + add + ns: + if rec.ttl < diff: + del self.cache[k] + break + + + def __getstate__(self): + for c in self.cancel.values(): + c.cancel() + self.cancel.clear() + return self.__dict__ + + + def _lookup(self, name, cls, type, timeout): + now = self._reactor.seconds() + q = dns.Query(name, type, cls) + try: + when, (ans, auth, add) = self.cache[q] + except KeyError: + if self.verbose > 1: + log.msg('Cache miss for ' + repr(name)) + return defer.fail(failure.Failure(dns.DomainError(name))) + else: + if self.verbose: + log.msg('Cache hit for ' + repr(name)) + diff = now - when + + try: + result = ( + [dns.RRHeader(r.name.name, r.type, r.cls, r.ttl - diff, + r.payload) for r in ans], + [dns.RRHeader(r.name.name, r.type, r.cls, r.ttl - diff, + r.payload) for r in auth], + [dns.RRHeader(r.name.name, r.type, r.cls, r.ttl - diff, + r.payload) for r in add]) + except ValueError: + return defer.fail(failure.Failure(dns.DomainError(name))) + else: + return defer.succeed(result) + + + def lookupAllRecords(self, name, timeout = None): + return defer.fail(failure.Failure(dns.DomainError(name))) + + + def cacheResult(self, query, payload, cacheTime=None): + """ + Cache a DNS entry. + + @param query: a L{dns.Query} instance. + + @param payload: a 3-tuple of lists of L{dns.RRHeader} records, the + matching result of the query (answers, authority and additional). + + @param cacheTime: The time (seconds since epoch) at which the entry is + considered to have been added to the cache. If L{None} is given, + the current time is used. + """ + if self.verbose > 1: + log.msg('Adding %r to cache' % query) + + self.cache[query] = (cacheTime or self._reactor.seconds(), payload) + + if query in self.cancel: + self.cancel[query].cancel() + + s = list(payload[0]) + list(payload[1]) + list(payload[2]) + if s: + m = s[0].ttl + for r in s: + m = min(m, r.ttl) + else: + m = 0 + + self.cancel[query] = self._reactor.callLater(m, self.clearEntry, query) + + + def clearEntry(self, query): + del self.cache[query] + del self.cancel[query] diff --git a/contrib/python/Twisted/py2/twisted/names/client.py b/contrib/python/Twisted/py2/twisted/names/client.py new file mode 100644 index 00000000000..c89e0fc2a3d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/client.py @@ -0,0 +1,776 @@ +# -*- test-case-name: twisted.names.test.test_names -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Asynchronous client DNS + +The functions exposed in this module can be used for asynchronous name +resolution and dns queries. + +If you need to create a resolver with specific requirements, such as needing to +do queries against a particular host, the L{createResolver} function will +return an C{IResolver}. + +Future plans: Proper nameserver acquisition on Windows/MacOS, +better caching, respect timeouts +""" + +import os +import errno +import warnings + +from zope.interface import moduleProvides + +# Twisted imports +from twisted.python.compat import nativeString +from twisted.python.runtime import platform +from twisted.python.filepath import FilePath +from twisted.internet import error, defer, interfaces, protocol +from twisted.python import log, failure +from twisted.names import ( + dns, common, resolve, cache, root, hosts as hostsModule) +from twisted.internet.abstract import isIPv6Address + + + +moduleProvides(interfaces.IResolver) + + + +class Resolver(common.ResolverBase): + """ + @ivar _waiting: A C{dict} mapping tuple keys of query name/type/class to + Deferreds which will be called back with the result of those queries. + This is used to avoid issuing the same query more than once in + parallel. This is more efficient on the network and helps avoid a + "birthday paradox" attack by keeping the number of outstanding requests + for a particular query fixed at one instead of allowing the attacker to + raise it to an arbitrary number. + + @ivar _reactor: A provider of L{IReactorTCP}, L{IReactorUDP}, and + L{IReactorTime} which will be used to set up network resources and + track timeouts. + """ + index = 0 + timeout = None + + factory = None + servers = None + dynServers = () + pending = None + connections = None + + resolv = None + _lastResolvTime = None + _resolvReadInterval = 60 + + def __init__(self, resolv=None, servers=None, timeout=(1, 3, 11, 45), reactor=None): + """ + Construct a resolver which will query domain name servers listed in + the C{resolv.conf(5)}-format file given by C{resolv} as well as + those in the given C{servers} list. Servers are queried in a + round-robin fashion. If given, C{resolv} is periodically checked + for modification and re-parsed if it is noticed to have changed. + + @type servers: C{list} of C{(str, int)} or L{None} + @param servers: If not None, interpreted as a list of (host, port) + pairs specifying addresses of domain name servers to attempt to use + for this lookup. Host addresses should be in IPv4 dotted-quad + form. If specified, overrides C{resolv}. + + @type resolv: C{str} + @param resolv: Filename to read and parse as a resolver(5) + configuration file. + + @type timeout: Sequence of C{int} + @param timeout: Default number of seconds after which to reissue the + query. When the last timeout expires, the query is considered + failed. + + @param reactor: A provider of L{IReactorTime}, L{IReactorUDP}, and + L{IReactorTCP} which will be used to establish connections, listen + for DNS datagrams, and enforce timeouts. If not provided, the + global reactor will be used. + + @raise ValueError: Raised if no nameserver addresses can be found. + """ + common.ResolverBase.__init__(self) + + if reactor is None: + from twisted.internet import reactor + self._reactor = reactor + + self.timeout = timeout + + if servers is None: + self.servers = [] + else: + self.servers = servers + + self.resolv = resolv + + if not len(self.servers) and not resolv: + raise ValueError("No nameservers specified") + + self.factory = DNSClientFactory(self, timeout) + self.factory.noisy = 0 # Be quiet by default + + self.connections = [] + self.pending = [] + + self._waiting = {} + + self.maybeParseConfig() + + + def __getstate__(self): + d = self.__dict__.copy() + d['connections'] = [] + d['_parseCall'] = None + return d + + + def __setstate__(self, state): + self.__dict__.update(state) + self.maybeParseConfig() + + + def _openFile(self, path): + """ + Wrapper used for opening files in the class, exists primarily for unit + testing purposes. + """ + return FilePath(path).open() + + + def maybeParseConfig(self): + if self.resolv is None: + # Don't try to parse it, don't set up a call loop + return + + try: + resolvConf = self._openFile(self.resolv) + except IOError as e: + if e.errno == errno.ENOENT: + # Missing resolv.conf is treated the same as an empty resolv.conf + self.parseConfig(()) + else: + raise + else: + with resolvConf: + mtime = os.fstat(resolvConf.fileno()).st_mtime + if mtime != self._lastResolvTime: + log.msg('%s changed, reparsing' % (self.resolv,)) + self._lastResolvTime = mtime + self.parseConfig(resolvConf) + + # Check again in a little while + self._parseCall = self._reactor.callLater( + self._resolvReadInterval, self.maybeParseConfig) + + + def parseConfig(self, resolvConf): + servers = [] + for L in resolvConf: + L = L.strip() + if L.startswith(b'nameserver'): + resolver = (nativeString(L.split()[1]), dns.PORT) + servers.append(resolver) + log.msg("Resolver added %r to server list" % (resolver,)) + elif L.startswith(b'domain'): + try: + self.domain = L.split()[1] + except IndexError: + self.domain = b'' + self.search = None + elif L.startswith(b'search'): + self.search = L.split()[1:] + self.domain = None + if not servers: + servers.append(('127.0.0.1', dns.PORT)) + self.dynServers = servers + + + def pickServer(self): + """ + Return the address of a nameserver. + + TODO: Weight servers for response time so faster ones can be + preferred. + """ + if not self.servers and not self.dynServers: + return None + serverL = len(self.servers) + dynL = len(self.dynServers) + + self.index += 1 + self.index %= (serverL + dynL) + if self.index < serverL: + return self.servers[self.index] + else: + return self.dynServers[self.index - serverL] + + + def _connectedProtocol(self, interface=''): + """ + Return a new L{DNSDatagramProtocol} bound to a randomly selected port + number. + """ + failures = 0 + proto = dns.DNSDatagramProtocol(self, reactor=self._reactor) + + while True: + try: + self._reactor.listenUDP(dns.randomSource(), proto, + interface=interface) + except error.CannotListenError as e: + failures += 1 + + if (hasattr(e.socketError, "errno") and + e.socketError.errno == errno.EMFILE): + # We've run out of file descriptors. Stop trying. + raise + + if failures >= 1000: + # We've tried a thousand times and haven't found a port. + # This is almost impossible, and likely means something + # else weird is going on. Raise, as to not infinite loop. + raise + else: + return proto + + + def connectionMade(self, protocol): + """ + Called by associated L{dns.DNSProtocol} instances when they connect. + """ + self.connections.append(protocol) + for (d, q, t) in self.pending: + self.queryTCP(q, t).chainDeferred(d) + del self.pending[:] + + + def connectionLost(self, protocol): + """ + Called by associated L{dns.DNSProtocol} instances when they disconnect. + """ + if protocol in self.connections: + self.connections.remove(protocol) + + + def messageReceived(self, message, protocol, address = None): + log.msg("Unexpected message (%d) received from %r" % (message.id, address)) + + + def _query(self, *args): + """ + Get a new L{DNSDatagramProtocol} instance from L{_connectedProtocol}, + issue a query to it using C{*args}, and arrange for it to be + disconnected from its transport after the query completes. + + @param *args: Positional arguments to be passed to + L{DNSDatagramProtocol.query}. + + @return: A L{Deferred} which will be called back with the result of the + query. + """ + if isIPv6Address(args[0][0]): + protocol = self._connectedProtocol(interface='::') + else: + protocol = self._connectedProtocol() + d = protocol.query(*args) + def cbQueried(result): + protocol.transport.stopListening() + return result + d.addBoth(cbQueried) + return d + + + def queryUDP(self, queries, timeout = None): + """ + Make a number of DNS queries via UDP. + + @type queries: A C{list} of C{dns.Query} instances + @param queries: The queries to make. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: C{Deferred} + @raise C{twisted.internet.defer.TimeoutError}: When the query times + out. + """ + if timeout is None: + timeout = self.timeout + + addresses = self.servers + list(self.dynServers) + if not addresses: + return defer.fail(IOError("No domain name servers available")) + + # Make sure we go through servers in the list in the order they were + # specified. + addresses.reverse() + + used = addresses.pop() + d = self._query(used, queries, timeout[0]) + d.addErrback(self._reissue, addresses, [used], queries, timeout) + return d + + + def _reissue(self, reason, addressesLeft, addressesUsed, query, timeout): + reason.trap(dns.DNSQueryTimeoutError) + + # If there are no servers left to be tried, adjust the timeout + # to the next longest timeout period and move all the + # "used" addresses back to the list of addresses to try. + if not addressesLeft: + addressesLeft = addressesUsed + addressesLeft.reverse() + addressesUsed = [] + timeout = timeout[1:] + + # If all timeout values have been used this query has failed. Tell the + # protocol we're giving up on it and return a terminal timeout failure + # to our caller. + if not timeout: + return failure.Failure(defer.TimeoutError(query)) + + # Get an address to try. Take it out of the list of addresses + # to try and put it ino the list of already tried addresses. + address = addressesLeft.pop() + addressesUsed.append(address) + + # Issue a query to a server. Use the current timeout. Add this + # function as a timeout errback in case another retry is required. + d = self._query(address, query, timeout[0], reason.value.id) + d.addErrback(self._reissue, addressesLeft, addressesUsed, query, timeout) + return d + + + def queryTCP(self, queries, timeout = 10): + """ + Make a number of DNS queries via TCP. + + @type queries: Any non-zero number of C{dns.Query} instances + @param queries: The queries to make. + + @type timeout: C{int} + @param timeout: The number of seconds after which to fail. + + @rtype: C{Deferred} + """ + if not len(self.connections): + address = self.pickServer() + if address is None: + return defer.fail(IOError("No domain name servers available")) + host, port = address + self._reactor.connectTCP(host, port, self.factory) + self.pending.append((defer.Deferred(), queries, timeout)) + return self.pending[-1][0] + else: + return self.connections[0].query(queries, timeout) + + + def filterAnswers(self, message): + """ + Extract results from the given message. + + If the message was truncated, re-attempt the query over TCP and return + a Deferred which will fire with the results of that query. + + If the message's result code is not C{twisted.names.dns.OK}, return a + Failure indicating the type of error which occurred. + + Otherwise, return a three-tuple of lists containing the results from + the answers section, the authority section, and the additional section. + """ + if message.trunc: + return self.queryTCP(message.queries).addCallback(self.filterAnswers) + if message.rCode != dns.OK: + return failure.Failure(self.exceptionForCode(message.rCode)(message)) + return (message.answers, message.authority, message.additional) + + + def _lookup(self, name, cls, type, timeout): + """ + Build a L{dns.Query} for the given parameters and dispatch it via UDP. + + If this query is already outstanding, it will not be re-issued. + Instead, when the outstanding query receives a response, that response + will be re-used for this query as well. + + @type name: C{str} + @type type: C{int} + @type cls: C{int} + + @return: A L{Deferred} which fires with a three-tuple giving the + answer, authority, and additional sections of the response or with + a L{Failure} if the response code is anything other than C{dns.OK}. + """ + key = (name, type, cls) + waiting = self._waiting.get(key) + if waiting is None: + self._waiting[key] = [] + d = self.queryUDP([dns.Query(name, type, cls)], timeout) + def cbResult(result): + for d in self._waiting.pop(key): + d.callback(result) + return result + d.addCallback(self.filterAnswers) + d.addBoth(cbResult) + else: + d = defer.Deferred() + waiting.append(d) + return d + + + # This one doesn't ever belong on UDP + def lookupZone(self, name, timeout=10): + address = self.pickServer() + if address is None: + return defer.fail(IOError('No domain name servers available')) + host, port = address + d = defer.Deferred() + controller = AXFRController(name, d) + factory = DNSClientFactory(controller, timeout) + factory.noisy = False #stfu + + connector = self._reactor.connectTCP(host, port, factory) + controller.timeoutCall = self._reactor.callLater( + timeout or 10, self._timeoutZone, d, controller, + connector, timeout or 10) + + def eliminateTimeout(failure): + controller.timeoutCall.cancel() + controller.timeoutCall = None + return failure + + return d.addCallbacks(self._cbLookupZone, eliminateTimeout, + callbackArgs=(connector,)) + + + def _timeoutZone(self, d, controller, connector, seconds): + connector.disconnect() + controller.timeoutCall = None + controller.deferred = None + d.errback(error.TimeoutError("Zone lookup timed out after %d seconds" % (seconds,))) + + + def _cbLookupZone(self, result, connector): + connector.disconnect() + return (result, [], []) + + + +class AXFRController: + timeoutCall = None + + def __init__(self, name, deferred): + self.name = name + self.deferred = deferred + self.soa = None + self.records = [] + self.pending = [(deferred,)] + + + def connectionMade(self, protocol): + # dig saids recursion-desired to 0, so I will too + message = dns.Message(protocol.pickID(), recDes=0) + message.queries = [dns.Query(self.name, dns.AXFR, dns.IN)] + protocol.writeMessage(message) + + + def connectionLost(self, protocol): + # XXX Do something here - see #3428 + pass + + + def messageReceived(self, message, protocol): + # Caveat: We have to handle two cases: All records are in 1 + # message, or all records are in N messages. + + # According to http://cr.yp.to/djbdns/axfr-notes.html, + # 'authority' and 'additional' are always empty, and only + # 'answers' is present. + self.records.extend(message.answers) + if not self.records: + return + if not self.soa: + if self.records[0].type == dns.SOA: + #print "first SOA!" + self.soa = self.records[0] + if len(self.records) > 1 and self.records[-1].type == dns.SOA: + #print "It's the second SOA! We're done." + if self.timeoutCall is not None: + self.timeoutCall.cancel() + self.timeoutCall = None + if self.deferred is not None: + self.deferred.callback(self.records) + self.deferred = None + + + +from twisted.internet.base import ThreadedResolver as _ThreadedResolverImpl + +class ThreadedResolver(_ThreadedResolverImpl): + def __init__(self, reactor=None): + if reactor is None: + from twisted.internet import reactor + _ThreadedResolverImpl.__init__(self, reactor) + warnings.warn( + "twisted.names.client.ThreadedResolver is deprecated since " + "Twisted 9.0, use twisted.internet.base.ThreadedResolver " + "instead.", + category=DeprecationWarning, stacklevel=2) + + + +class DNSClientFactory(protocol.ClientFactory): + def __init__(self, controller, timeout = 10): + self.controller = controller + self.timeout = timeout + + + def clientConnectionLost(self, connector, reason): + pass + + + def clientConnectionFailed(self, connector, reason): + """ + Fail all pending TCP DNS queries if the TCP connection attempt + fails. + + @see: L{twisted.internet.protocol.ClientFactory} + + @param connector: Not used. + @type connector: L{twisted.internet.interfaces.IConnector} + + @param reason: A C{Failure} containing information about the + cause of the connection failure. This will be passed as the + argument to C{errback} on every pending TCP query + C{deferred}. + @type reason: L{twisted.python.failure.Failure} + """ + # Copy the current pending deferreds then reset the master + # pending list. This prevents triggering new deferreds which + # may be added by callback or errback functions on the current + # deferreds. + pending = self.controller.pending[:] + del self.controller.pending[:] + for pendingState in pending: + d = pendingState[0] + d.errback(reason) + + + def buildProtocol(self, addr): + p = dns.DNSProtocol(self.controller) + p.factory = self + return p + + + +def createResolver(servers=None, resolvconf=None, hosts=None): + """ + Create and return a Resolver. + + @type servers: C{list} of C{(str, int)} or L{None} + + @param servers: If not L{None}, interpreted as a list of domain name servers + to attempt to use. Each server is a tuple of address in C{str} dotted-quad + form and C{int} port number. + + @type resolvconf: C{str} or L{None} + @param resolvconf: If not L{None}, on posix systems will be interpreted as + an alternate resolv.conf to use. Will do nothing on windows systems. If + L{None}, /etc/resolv.conf will be used. + + @type hosts: C{str} or L{None} + @param hosts: If not L{None}, an alternate hosts file to use. If L{None} + on posix systems, /etc/hosts will be used. On windows, C:\windows\hosts + will be used. + + @rtype: C{IResolver} + """ + if platform.getType() == 'posix': + if resolvconf is None: + resolvconf = b'/etc/resolv.conf' + if hosts is None: + hosts = b'/etc/hosts' + theResolver = Resolver(resolvconf, servers) + hostResolver = hostsModule.Resolver(hosts) + else: + if hosts is None: + hosts = r'c:\windows\hosts' + from twisted.internet import reactor + bootstrap = _ThreadedResolverImpl(reactor) + hostResolver = hostsModule.Resolver(hosts) + theResolver = root.bootstrap(bootstrap, resolverFactory=Resolver) + + L = [hostResolver, cache.CacheResolver(), theResolver] + return resolve.ResolverChain(L) + + + +theResolver = None +def getResolver(): + """ + Get a Resolver instance. + + Create twisted.names.client.theResolver if it is L{None}, and then return + that value. + + @rtype: C{IResolver} + """ + global theResolver + if theResolver is None: + try: + theResolver = createResolver() + except ValueError: + theResolver = createResolver(servers=[('127.0.0.1', 53)]) + return theResolver + + + +def getHostByName(name, timeout=None, effort=10): + """ + Resolve a name to a valid ipv4 or ipv6 address. + + Will errback with C{DNSQueryTimeoutError} on a timeout, C{DomainError} or + C{AuthoritativeDomainError} (or subclasses) on other errors. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @type effort: C{int} + @param effort: How many times CNAME and NS records to follow while + resolving this name. + + @rtype: C{Deferred} + """ + return getResolver().getHostByName(name, timeout, effort) + + + +def query(query, timeout=None): + return getResolver().query(query, timeout) + + + +def lookupAddress(name, timeout=None): + return getResolver().lookupAddress(name, timeout) + + + +def lookupIPV6Address(name, timeout=None): + return getResolver().lookupIPV6Address(name, timeout) + + + +def lookupAddress6(name, timeout=None): + return getResolver().lookupAddress6(name, timeout) + + + +def lookupMailExchange(name, timeout=None): + return getResolver().lookupMailExchange(name, timeout) + + + +def lookupNameservers(name, timeout=None): + return getResolver().lookupNameservers(name, timeout) + + + +def lookupCanonicalName(name, timeout=None): + return getResolver().lookupCanonicalName(name, timeout) + + + +def lookupMailBox(name, timeout=None): + return getResolver().lookupMailBox(name, timeout) + + + +def lookupMailGroup(name, timeout=None): + return getResolver().lookupMailGroup(name, timeout) + + + +def lookupMailRename(name, timeout=None): + return getResolver().lookupMailRename(name, timeout) + + + +def lookupPointer(name, timeout=None): + return getResolver().lookupPointer(name, timeout) + + + +def lookupAuthority(name, timeout=None): + return getResolver().lookupAuthority(name, timeout) + + + +def lookupNull(name, timeout=None): + return getResolver().lookupNull(name, timeout) + + + +def lookupWellKnownServices(name, timeout=None): + return getResolver().lookupWellKnownServices(name, timeout) + + + +def lookupService(name, timeout=None): + return getResolver().lookupService(name, timeout) + + + +def lookupHostInfo(name, timeout=None): + return getResolver().lookupHostInfo(name, timeout) + + + +def lookupMailboxInfo(name, timeout=None): + return getResolver().lookupMailboxInfo(name, timeout) + + + +def lookupText(name, timeout=None): + return getResolver().lookupText(name, timeout) + + + +def lookupSenderPolicy(name, timeout=None): + return getResolver().lookupSenderPolicy(name, timeout) + + + +def lookupResponsibility(name, timeout=None): + return getResolver().lookupResponsibility(name, timeout) + + + +def lookupAFSDatabase(name, timeout=None): + return getResolver().lookupAFSDatabase(name, timeout) + + + +def lookupZone(name, timeout=None): + return getResolver().lookupZone(name, timeout) + + + +def lookupAllRecords(name, timeout=None): + return getResolver().lookupAllRecords(name, timeout) + + + +def lookupNamingAuthorityPointer(name, timeout=None): + return getResolver().lookupNamingAuthorityPointer(name, timeout) diff --git a/contrib/python/Twisted/py2/twisted/names/common.py b/contrib/python/Twisted/py2/twisted/names/common.py new file mode 100644 index 00000000000..c7dfdb8ad11 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/common.py @@ -0,0 +1,289 @@ +# -*- test-case-name: twisted.names.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Base functionality useful to various parts of Twisted Names. +""" + +from __future__ import division, absolute_import + +import socket + +from zope.interface import implementer + +from twisted.names import dns +from twisted.names.error import DNSFormatError, DNSServerError, DNSNameError +from twisted.names.error import DNSNotImplementedError, DNSQueryRefusedError +from twisted.names.error import DNSUnknownError + +from twisted.internet import defer, error, interfaces + +from twisted.logger import Logger + +# Helpers for indexing the three-tuples that get thrown around by this code a +# lot. +_ANS, _AUTH, _ADD = range(3) + +EMPTY_RESULT = (), (), () + + + +@implementer(interfaces.IResolver) +class ResolverBase: + """ + L{ResolverBase} is a base class for implementations of + L{interfaces.IResolver} which deals with a lot + of the boilerplate of implementing all of the lookup methods. + + @cvar _errormap: A C{dict} mapping DNS protocol failure response codes + to exception classes which will be used to represent those failures. + """ + _log = Logger() + _errormap = { + dns.EFORMAT: DNSFormatError, + dns.ESERVER: DNSServerError, + dns.ENAME: DNSNameError, + dns.ENOTIMP: DNSNotImplementedError, + dns.EREFUSED: DNSQueryRefusedError} + + typeToMethod = None + + def __init__(self): + self.typeToMethod = {} + for (k, v) in typeToMethod.items(): + self.typeToMethod[k] = getattr(self, v) + + + def exceptionForCode(self, responseCode): + """ + Convert a response code (one of the possible values of + L{dns.Message.rCode} to an exception instance representing it. + + @since: 10.0 + """ + return self._errormap.get(responseCode, DNSUnknownError) + + + def query(self, query, timeout=None): + try: + method = self.typeToMethod[query.type] + except KeyError: + self._log.debug( + 'Query of unknown type {query.type} for {query.name.name!r}', + query=query) + return defer.maybeDeferred( + self._lookup, query.name.name, dns.IN, query.type, timeout) + else: + return defer.maybeDeferred(method, query.name.name, timeout) + + + def _lookup(self, name, cls, type, timeout): + return defer.fail(NotImplementedError("ResolverBase._lookup")) + + + def lookupAddress(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.A, timeout) + + + def lookupIPV6Address(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.AAAA, timeout) + + + def lookupAddress6(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.A6, timeout) + + + def lookupMailExchange(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.MX, timeout) + + + def lookupNameservers(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.NS, timeout) + + + def lookupCanonicalName(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.CNAME, timeout) + + + def lookupMailBox(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.MB, timeout) + + + def lookupMailGroup(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.MG, timeout) + + + def lookupMailRename(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.MR, timeout) + + + def lookupPointer(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.PTR, timeout) + + + def lookupAuthority(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.SOA, timeout) + + + def lookupNull(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.NULL, timeout) + + + def lookupWellKnownServices(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.WKS, timeout) + + + def lookupService(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.SRV, timeout) + + + def lookupHostInfo(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.HINFO, timeout) + + + def lookupMailboxInfo(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.MINFO, timeout) + + + def lookupText(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.TXT, timeout) + + + def lookupSenderPolicy(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.SPF, timeout) + + + def lookupResponsibility(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.RP, timeout) + + + def lookupAFSDatabase(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.AFSDB, timeout) + + + def lookupZone(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.AXFR, timeout) + + + def lookupNamingAuthorityPointer(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.NAPTR, timeout) + + + def lookupAllRecords(self, name, timeout=None): + return self._lookup(dns.domainString(name), dns.IN, dns.ALL_RECORDS, + timeout) + + + # IResolverSimple + def getHostByName(self, name, timeout=None, effort=10): + name = dns.domainString(name) + # XXX - respect timeout + # XXX - this should do A and AAAA lookups, not ANY (see RFC 8482). + # https://twistedmatrix.com/trac/ticket/9691 + d = self.lookupAllRecords(name, timeout) + d.addCallback(self._cbRecords, name, effort) + return d + + + def _cbRecords(self, records, name, effort): + (ans, auth, add) = records + result = extractRecord(self, dns.Name(name), ans + auth + add, effort) + if not result: + raise error.DNSLookupError(name) + return result + + + +def extractRecord(resolver, name, answers, level=10): + """ + Resolve a name to an IP address, following I{CNAME} records and I{NS} + referrals recursively. + + This is an implementation detail of L{ResolverBase.getHostByName}. + + @param resolver: The resolver to use for the next query (unless handling + an I{NS} referral). + @type resolver: L{IResolver} + + @param name: The name being looked up. + @type name: L{dns.Name} + + @param answers: All of the records returned by the previous query (answers, + authority, and additional concatenated). + @type answers: L{list} of L{dns.RRHeader} + + @param level: Remaining recursion budget. This is decremented at each + recursion. The query returns L{None} when it reaches 0. + @type level: L{int} + + @returns: The first IPv4 or IPv6 address (as a dotted quad or colon + quibbles), or L{None} when no result is found. + @rtype: native L{str} or L{None} + """ + if not level: + return None + # FIXME: twisted.python.compat monkeypatches this if missing, so this + # condition is always true. https://twistedmatrix.com/trac/ticket/9753 + if hasattr(socket, 'inet_ntop'): + for r in answers: + if r.name == name and r.type == dns.A6: + return socket.inet_ntop(socket.AF_INET6, r.payload.address) + for r in answers: + if r.name == name and r.type == dns.AAAA: + return socket.inet_ntop(socket.AF_INET6, r.payload.address) + for r in answers: + if r.name == name and r.type == dns.A: + return socket.inet_ntop(socket.AF_INET, r.payload.address) + for r in answers: + if r.name == name and r.type == dns.CNAME: + result = extractRecord( + resolver, r.payload.name, answers, level - 1) + if not result: + return resolver.getHostByName( + r.payload.name.name, effort=level - 1) + return result + # No answers, but maybe there's a hint at who we should be asking about + # this + for r in answers: + if r.type != dns.NS: + continue + from twisted.names import client + nsResolver = client.Resolver(servers=[ + (r.payload.name.name.decode('ascii'), dns.PORT), + ]) + + def queryAgain(records): + (ans, auth, add) = records + return extractRecord(nsResolver, name, ans + auth + add, level - 1) + + return nsResolver.lookupAddress(name.name).addCallback(queryAgain) + + + +typeToMethod = { + dns.A: 'lookupAddress', + dns.AAAA: 'lookupIPV6Address', + dns.A6: 'lookupAddress6', + dns.NS: 'lookupNameservers', + dns.CNAME: 'lookupCanonicalName', + dns.SOA: 'lookupAuthority', + dns.MB: 'lookupMailBox', + dns.MG: 'lookupMailGroup', + dns.MR: 'lookupMailRename', + dns.NULL: 'lookupNull', + dns.WKS: 'lookupWellKnownServices', + dns.PTR: 'lookupPointer', + dns.HINFO: 'lookupHostInfo', + dns.MINFO: 'lookupMailboxInfo', + dns.MX: 'lookupMailExchange', + dns.TXT: 'lookupText', + dns.SPF: 'lookupSenderPolicy', + + dns.RP: 'lookupResponsibility', + dns.AFSDB: 'lookupAFSDatabase', + dns.SRV: 'lookupService', + dns.NAPTR: 'lookupNamingAuthorityPointer', + dns.AXFR: 'lookupZone', + dns.ALL_RECORDS: 'lookupAllRecords', +} diff --git a/contrib/python/Twisted/py2/twisted/names/dns.py b/contrib/python/Twisted/py2/twisted/names/dns.py new file mode 100644 index 00000000000..89509a644e4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/dns.py @@ -0,0 +1,3214 @@ +# -*- test-case-name: twisted.names.test.test_dns -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +DNS protocol implementation. + +Future Plans: + - Get rid of some toplevels, maybe. +""" + +from __future__ import division, absolute_import + +__all__ = [ + 'IEncodable', 'IRecord', + + 'A', 'A6', 'AAAA', 'AFSDB', 'CNAME', 'DNAME', 'HINFO', + 'MAILA', 'MAILB', 'MB', 'MD', 'MF', 'MG', 'MINFO', 'MR', 'MX', + 'NAPTR', 'NS', 'NULL', 'OPT', 'PTR', 'RP', 'SOA', 'SPF', 'SRV', 'TXT', + 'SSHFP', 'TSIG', 'WKS', + + 'ANY', 'CH', 'CS', 'HS', 'IN', + + 'ALL_RECORDS', 'AXFR', 'IXFR', + + 'EFORMAT', 'ENAME', 'ENOTIMP', 'EREFUSED', 'ESERVER', 'EBADVERSION', + 'EBADSIG', 'EBADKEY', 'EBADTIME', + + 'Record_A', 'Record_A6', 'Record_AAAA', 'Record_AFSDB', 'Record_CNAME', + 'Record_DNAME', 'Record_HINFO', 'Record_MB', 'Record_MD', 'Record_MF', + 'Record_MG', 'Record_MINFO', 'Record_MR', 'Record_MX', 'Record_NAPTR', + 'Record_NS', 'Record_NULL', 'Record_PTR', 'Record_RP', 'Record_SOA', + 'Record_SPF', 'Record_SRV', 'Record_SSHFP', 'Record_TSIG', 'Record_TXT', + 'Record_WKS', + 'UnknownRecord', + + 'QUERY_CLASSES', 'QUERY_TYPES', 'REV_CLASSES', 'REV_TYPES', 'EXT_QUERIES', + + 'Charstr', 'Message', 'Name', 'Query', 'RRHeader', 'SimpleRecord', + 'DNSDatagramProtocol', 'DNSMixin', 'DNSProtocol', + + 'OK', 'OP_INVERSE', 'OP_NOTIFY', 'OP_QUERY', 'OP_STATUS', 'OP_UPDATE', + 'PORT', + + 'AuthoritativeDomainError', 'DNSQueryTimeoutError', 'DomainError', + ] + + +# System imports +import inspect, struct, random, socket +from itertools import chain + +from io import BytesIO + +AF_INET6 = socket.AF_INET6 + +from zope.interface import implementer, Interface, Attribute + + +# Twisted imports +from twisted.internet import protocol, defer +from twisted.internet.error import CannotListenError +from twisted.python import log, failure +from twisted.python import util as tputil +from twisted.python import randbytes +from twisted.python.compat import _PY3, unicode, comparable, cmp, nativeString + + +if _PY3: + def _ord2bytes(ordinal): + """ + Construct a bytes object representing a single byte with the given + ordinal value. + + @type ordinal: L{int} + @rtype: L{bytes} + """ + return bytes([ordinal]) + + + def _nicebytes(bytes): + """ + Represent a mostly textful bytes object in a way suitable for + presentation to an end user. + + @param bytes: The bytes to represent. + @rtype: L{str} + """ + return repr(bytes)[1:] + + + def _nicebyteslist(list): + """ + Represent a list of mostly textful bytes objects in a way suitable for + presentation to an end user. + + @param list: The list of bytes to represent. + @rtype: L{str} + """ + return '[%s]' % ( + ', '.join([_nicebytes(b) for b in list]),) +else: + _ord2bytes = chr + _nicebytes = _nicebyteslist = repr + + + +def randomSource(): + """ + Wrapper around L{twisted.python.randbytes.RandomFactory.secureRandom} to + return 2 random bytes. + + @rtype: L{bytes} + """ + return struct.unpack('H', randbytes.secureRandom(2, fallback=True))[0] + + +PORT = 53 + +(A, NS, MD, MF, CNAME, SOA, MB, MG, MR, NULL, WKS, PTR, HINFO, MINFO, MX, TXT, + RP, AFSDB) = range(1, 19) +AAAA = 28 +SRV = 33 +NAPTR = 35 +A6 = 38 +DNAME = 39 +OPT = 41 +SSHFP = 44 +SPF = 99 + +# These record types do not exist in zones, but are transferred in +# messages the same way normal RRs are. +TKEY = 249 +TSIG = 250 + +QUERY_TYPES = { + A: 'A', + NS: 'NS', + MD: 'MD', + MF: 'MF', + CNAME: 'CNAME', + SOA: 'SOA', + MB: 'MB', + MG: 'MG', + MR: 'MR', + NULL: 'NULL', + WKS: 'WKS', + PTR: 'PTR', + HINFO: 'HINFO', + MINFO: 'MINFO', + MX: 'MX', + TXT: 'TXT', + RP: 'RP', + AFSDB: 'AFSDB', + + # 19 through 27? Eh, I'll get to 'em. + + AAAA: 'AAAA', + SRV: 'SRV', + NAPTR: 'NAPTR', + A6: 'A6', + DNAME: 'DNAME', + OPT: 'OPT', + SSHFP: 'SSHFP', + SPF: 'SPF', + + TKEY: 'TKEY', + TSIG: 'TSIG', +} + +IXFR, AXFR, MAILB, MAILA, ALL_RECORDS = range(251, 256) + +# "Extended" queries (Hey, half of these are deprecated, good job) +EXT_QUERIES = { + IXFR: 'IXFR', + AXFR: 'AXFR', + MAILB: 'MAILB', + MAILA: 'MAILA', + ALL_RECORDS: 'ALL_RECORDS' +} + +REV_TYPES = dict([ + (v, k) for (k, v) in chain(QUERY_TYPES.items(), EXT_QUERIES.items()) +]) + +IN, CS, CH, HS = range(1, 5) +ANY = 255 + +QUERY_CLASSES = { + IN: 'IN', + CS: 'CS', + CH: 'CH', + HS: 'HS', + ANY: 'ANY' +} +REV_CLASSES = dict([ + (v, k) for (k, v) in QUERY_CLASSES.items() +]) + + +# Opcodes +OP_QUERY, OP_INVERSE, OP_STATUS = range(3) +OP_NOTIFY = 4 # RFC 1996 +OP_UPDATE = 5 # RFC 2136 + + +# Response Codes +OK, EFORMAT, ESERVER, ENAME, ENOTIMP, EREFUSED = range(6) +# https://tools.ietf.org/html/rfc6891#section-9 +EBADVERSION = 16 +# RFC 2845 +EBADSIG, EBADKEY, EBADTIME = range(16, 19) + + + +class IRecord(Interface): + """ + A single entry in a zone of authority. + """ + + TYPE = Attribute("An indicator of what kind of record this is.") + + +# Backwards compatibility aliases - these should be deprecated or something I +# suppose. -exarkun +from twisted.names.error import DomainError, AuthoritativeDomainError +from twisted.names.error import DNSQueryTimeoutError + + + +def _nameToLabels(name): + """ + Split a domain name into its constituent labels. + + @type name: L{bytes} + @param name: A fully qualified domain name (with or without a + trailing dot). + + @return: A L{list} of labels ending with an empty label + representing the DNS root zone. + @rtype: L{list} of L{bytes} + """ + if name in (b'', b'.'): + return [b''] + labels = name.split(b'.') + if labels[-1] != b'': + labels.append(b'') + return labels + + + +def domainString(domain): + """ + Coerce a domain name string to bytes. + + L{twisted.names} represents domain names as L{bytes}, but many interfaces + accept L{bytes} or a text string (L{unicode} on Python 2, L{str} on Python + 3). This function coerces text strings using IDNA encoding --- see + L{encodings.idna}. + + Note that DNS is I{case insensitive} but I{case preserving}. This function + doesn't normalize case, so you'll still need to do that whenever comparing + the strings it returns. + + @param domain: A domain name. If passed as a text string it will be + C{idna} encoded. + @type domain: L{bytes} or L{str} + + @returns: L{bytes} suitable for network transmission. + @rtype: L{bytes} + + @since: Twisted 20.3.0 + """ + if isinstance(domain, unicode): + domain = domain.encode('idna') + if not isinstance(domain, bytes): + raise TypeError('Expected {} or {} but found {!r} of type {}'.format( + type(b'').__name__, type(u'').__name__, + domain, type(domain))) + return domain + + + +def _isSubdomainOf(descendantName, ancestorName): + """ + Test whether C{descendantName} is equal to or is a I{subdomain} of + C{ancestorName}. + + The names are compared case-insensitively. + + The names are treated as byte strings containing one or more + DNS labels separated by B{.}. + + C{descendantName} is considered equal if its sequence of labels + exactly matches the labels of C{ancestorName}. + + C{descendantName} is considered a I{subdomain} if its sequence of + labels ends with the labels of C{ancestorName}. + + @type descendantName: L{bytes} + @param descendantName: The DNS subdomain name. + + @type ancestorName: L{bytes} + @param ancestorName: The DNS parent or ancestor domain name. + + @return: C{True} if C{descendantName} is equal to or if it is a + subdomain of C{ancestorName}. Otherwise returns C{False}. + """ + descendantLabels = _nameToLabels(descendantName.lower()) + ancestorLabels = _nameToLabels(ancestorName.lower()) + return descendantLabels[-len(ancestorLabels):] == ancestorLabels + + + +def str2time(s): + """ + Parse a string description of an interval into an integer number of seconds. + + @param s: An interval definition constructed as an interval duration + followed by an interval unit. An interval duration is a base ten + representation of an integer. An interval unit is one of the following + letters: S (seconds), M (minutes), H (hours), D (days), W (weeks), or Y + (years). For example: C{"3S"} indicates an interval of three seconds; + C{"5D"} indicates an interval of five days. Alternatively, C{s} may be + any non-string and it will be returned unmodified. + @type s: text string (L{bytes} or L{unicode}) for parsing; anything else + for passthrough. + + @return: an L{int} giving the interval represented by the string C{s}, or + whatever C{s} is if it is not a string. + """ + suffixes = ( + ('S', 1), ('M', 60), ('H', 60 * 60), ('D', 60 * 60 * 24), + ('W', 60 * 60 * 24 * 7), ('Y', 60 * 60 * 24 * 365) + ) + if _PY3 and isinstance(s, bytes): + s = s.decode('ascii') + + if isinstance(s, str): + s = s.upper().strip() + for (suff, mult) in suffixes: + if s.endswith(suff): + return int(float(s[:-1]) * mult) + try: + s = int(s) + except ValueError: + raise ValueError("Invalid time interval specifier: " + s) + return s + + + +def readPrecisely(file, l): + buff = file.read(l) + if len(buff) < l: + raise EOFError + return buff + + + +class IEncodable(Interface): + """ + Interface for something which can be encoded to and decoded + to the DNS wire format. + + A binary-mode file object (such as L{io.BytesIO}) is used as a buffer when + encoding or decoding. + """ + + def encode(strio, compDict=None): + """ + Write a representation of this object to the given + file object. + + @type strio: File-like object + @param strio: The buffer to write to. It must have a C{tell()} method. + + @type compDict: L{dict} of L{bytes} to L{int} r L{None} + @param compDict: A mapping of names to byte offsets that have already + been written to the buffer, which may be used for compression (see RFC + 1035 section 4.1.4). When L{None}, encode without compression. + """ + + + def decode(strio, length=None): + """ + Reconstruct an object from data read from the given + file object. + + @type strio: File-like object + @param strio: A seekable buffer from which bytes may be read. + + @type length: L{int} or L{None} + @param length: The number of bytes in this RDATA field. Most + implementations can ignore this value. Only in the case of + records similar to TXT where the total length is in no way + encoded in the data is it necessary. + """ + + + +@implementer(IEncodable) +class Charstr(object): + + def __init__(self, string=b''): + if not isinstance(string, bytes): + raise ValueError("%r is not a byte string" % (string,)) + self.string = string + + + def encode(self, strio, compDict=None): + """ + Encode this Character string into the appropriate byte format. + + @type strio: file + @param strio: The byte representation of this Charstr will be written + to this file. + """ + string = self.string + ind = len(string) + strio.write(_ord2bytes(ind)) + strio.write(string) + + + def decode(self, strio, length=None): + """ + Decode a byte string into this Charstr. + + @type strio: file + @param strio: Bytes will be read from this file until the full string + is decoded. + + @raise EOFError: Raised when there are not enough bytes available from + C{strio}. + """ + self.string = b'' + l = ord(readPrecisely(strio, 1)) + self.string = readPrecisely(strio, l) + + + def __eq__(self, other): + if isinstance(other, Charstr): + return self.string == other.string + return NotImplemented + + + def __ne__(self, other): + if isinstance(other, Charstr): + return self.string != other.string + return NotImplemented + + + def __hash__(self): + return hash(self.string) + + + def __str__(self): + """ + Represent this L{Charstr} instance by its string value. + """ + return nativeString(self.string) + + + +@implementer(IEncodable) +class Name: + """ + A name in the domain name system, made up of multiple labels. For example, + I{twistedmatrix.com}. + + @ivar name: A byte string giving the name. + @type name: L{bytes} + """ + def __init__(self, name=b''): + """ + @param name: A name. + @type name: L{bytes} or L{str} + """ + self.name = domainString(name) + + + def encode(self, strio, compDict=None): + """ + Encode this Name into the appropriate byte format. + + @type strio: file + @param strio: The byte representation of this Name will be written to + this file. + + @type compDict: dict + @param compDict: dictionary of Names that have already been encoded + and whose addresses may be backreferenced by this Name (for the purpose + of reducing the message size). + """ + name = self.name + while name: + if compDict is not None: + if name in compDict: + strio.write( + struct.pack("!H", 0xc000 | compDict[name])) + return + else: + compDict[name] = strio.tell() + Message.headerSize + ind = name.find(b'.') + if ind > 0: + label, name = name[:ind], name[ind + 1:] + else: + # This is the last label, end the loop after handling it. + label = name + name = None + ind = len(label) + strio.write(_ord2bytes(ind)) + strio.write(label) + strio.write(b'\x00') + + + def decode(self, strio, length=None): + """ + Decode a byte string into this Name. + + @type strio: file + @param strio: Bytes will be read from this file until the full Name + is decoded. + + @raise EOFError: Raised when there are not enough bytes available + from C{strio}. + + @raise ValueError: Raised when the name cannot be decoded (for example, + because it contains a loop). + """ + visited = set() + self.name = b'' + off = 0 + while 1: + l = ord(readPrecisely(strio, 1)) + if l == 0: + if off > 0: + strio.seek(off) + return + if (l >> 6) == 3: + new_off = ((l&63) << 8 + | ord(readPrecisely(strio, 1))) + if new_off in visited: + raise ValueError("Compression loop in encoded name") + visited.add(new_off) + if off == 0: + off = strio.tell() + strio.seek(new_off) + continue + label = readPrecisely(strio, l) + if self.name == b'': + self.name = label + else: + self.name = self.name + b'.' + label + + def __eq__(self, other): + if isinstance(other, Name): + return self.name.lower() == other.name.lower() + return NotImplemented + + + def __ne__(self, other): + if isinstance(other, Name): + return self.name.lower() != other.name.lower() + return NotImplemented + + + def __hash__(self): + return hash(self.name) + + + def __str__(self): + """ + Represent this L{Name} instance by its string name. + """ + return nativeString(self.name) + + + +@comparable +@implementer(IEncodable) +class Query: + """ + Represent a single DNS query. + + @ivar name: The name about which this query is requesting information. + @type name: L{Name} + + @ivar type: The query type. + @type type: L{int} + + @ivar cls: The query class. + @type cls: L{int} + """ + name = None + type = None + cls = None + + def __init__(self, name=b'', type=A, cls=IN): + """ + @type name: L{bytes} or L{unicode} + @param name: See L{Query.name} + + @type type: L{int} + @param type: The query type. + + @type cls: L{int} + @param cls: The query class. + """ + self.name = Name(name) + self.type = type + self.cls = cls + + + def encode(self, strio, compDict=None): + self.name.encode(strio, compDict) + strio.write(struct.pack("!HH", self.type, self.cls)) + + + def decode(self, strio, length = None): + self.name.decode(strio) + buff = readPrecisely(strio, 4) + self.type, self.cls = struct.unpack("!HH", buff) + + + def __hash__(self): + return hash((self.name.name.lower(), self.type, self.cls)) + + + def __cmp__(self, other): + if isinstance(other, Query): + return cmp( + (self.name.name.lower(), self.type, self.cls), + (other.name.name.lower(), other.type, other.cls)) + return NotImplemented + + + def __str__(self): + t = QUERY_TYPES.get(self.type, EXT_QUERIES.get(self.type, 'UNKNOWN (%d)' % self.type)) + c = QUERY_CLASSES.get(self.cls, 'UNKNOWN (%d)' % self.cls) + return '' % (self.name, t, c) + + + def __repr__(self): + return 'Query(%r, %r, %r)' % (self.name.name, self.type, self.cls) + + + +@implementer(IEncodable) +class _OPTHeader(tputil.FancyStrMixin, tputil.FancyEqMixin, object): + """ + An OPT record header. + + @ivar name: The DNS name associated with this record. Since this + is a pseudo record, the name is always an L{Name} instance + with value b'', which represents the DNS root zone. This + attribute is a readonly property. + + @ivar type: The DNS record type. This is a fixed value of 41 + C{dns.OPT} for OPT Record. This attribute is a readonly + property. + + @see: L{_OPTHeader.__init__} for documentation of other public + instance attributes. + + @see: U{https://tools.ietf.org/html/rfc6891#section-6.1.2} + + @since: 13.2 + """ + showAttributes = ( + ('name', lambda n: nativeString(n.name)), 'type', 'udpPayloadSize', + 'extendedRCODE', 'version', 'dnssecOK', 'options') + + compareAttributes = ( + 'name', 'type', 'udpPayloadSize', 'extendedRCODE', 'version', + 'dnssecOK', 'options') + + def __init__(self, udpPayloadSize=4096, extendedRCODE=0, version=0, + dnssecOK=False, options=None): + """ + @type udpPayloadSize: L{int} + @param payload: The number of octets of the largest UDP + payload that can be reassembled and delivered in the + requestor's network stack. + + @type extendedRCODE: L{int} + @param extendedRCODE: Forms the upper 8 bits of extended + 12-bit RCODE (together with the 4 bits defined in + [RFC1035]. Note that EXTENDED-RCODE value 0 indicates + that an unextended RCODE is in use (values 0 through 15). + + @type version: L{int} + @param version: Indicates the implementation level of the + setter. Full conformance with this specification is + indicated by version C{0}. + + @type dnssecOK: L{bool} + @param dnssecOK: DNSSEC OK bit as defined by [RFC3225]. + + @type options: L{list} + @param options: A L{list} of 0 or more L{_OPTVariableOption} + instances. + """ + self.udpPayloadSize = udpPayloadSize + self.extendedRCODE = extendedRCODE + self.version = version + self.dnssecOK = dnssecOK + + if options is None: + options = [] + self.options = options + + + @property + def name(self): + """ + A readonly property for accessing the C{name} attribute of + this record. + + @return: The DNS name associated with this record. Since this + is a pseudo record, the name is always an L{Name} instance + with value b'', which represents the DNS root zone. + """ + return Name(b'') + + + @property + def type(self): + """ + A readonly property for accessing the C{type} attribute of + this record. + + @return: The DNS record type. This is a fixed value of 41 + (C{dns.OPT} for OPT Record. + """ + return OPT + + + def encode(self, strio, compDict=None): + """ + Encode this L{_OPTHeader} instance to bytes. + + @type strio: L{file} + @param strio: the byte representation of this L{_OPTHeader} + will be written to this file. + + @type compDict: L{dict} or L{None} + @param compDict: A dictionary of backreference addresses that + have already been written to this stream and that may + be used for DNS name compression. + """ + b = BytesIO() + for o in self.options: + o.encode(b) + optionBytes = b.getvalue() + + RRHeader( + name=self.name.name, + type=self.type, + cls=self.udpPayloadSize, + ttl=( + self.extendedRCODE << 24 + | self.version << 16 + | self.dnssecOK << 15), + payload=UnknownRecord(optionBytes) + ).encode(strio, compDict) + + + def decode(self, strio, length=None): + """ + Decode bytes into an L{_OPTHeader} instance. + + @type strio: L{file} + @param strio: Bytes will be read from this file until the full + L{_OPTHeader} is decoded. + + @type length: L{int} or L{None} + @param length: Not used. + """ + + h = RRHeader() + h.decode(strio, length) + h.payload = UnknownRecord(readPrecisely(strio, h.rdlength)) + + newOptHeader = self.fromRRHeader(h) + + for attrName in self.compareAttributes: + if attrName not in ('name', 'type'): + setattr(self, attrName, getattr(newOptHeader, attrName)) + + + @classmethod + def fromRRHeader(cls, rrHeader): + """ + A classmethod for constructing a new L{_OPTHeader} from the + attributes and payload of an existing L{RRHeader} instance. + + @type rrHeader: L{RRHeader} + @param rrHeader: An L{RRHeader} instance containing an + L{UnknownRecord} payload. + + @return: An instance of L{_OPTHeader}. + @rtype: L{_OPTHeader} + """ + options = None + if rrHeader.payload is not None: + options = [] + optionsBytes = BytesIO(rrHeader.payload.data) + optionsBytesLength = len(rrHeader.payload.data) + while optionsBytes.tell() < optionsBytesLength: + o = _OPTVariableOption() + o.decode(optionsBytes) + options.append(o) + + # Decode variable options if present + return cls( + udpPayloadSize=rrHeader.cls, + extendedRCODE=rrHeader.ttl >> 24, + version=rrHeader.ttl >> 16 & 0xff, + dnssecOK=(rrHeader.ttl & 0xffff) >> 15, + options=options + ) + + + +@implementer(IEncodable) +class _OPTVariableOption(tputil.FancyStrMixin, tputil.FancyEqMixin, object): + """ + A class to represent OPT record variable options. + + @see: L{_OPTVariableOption.__init__} for documentation of public + instance attributes. + + @see: U{https://tools.ietf.org/html/rfc6891#section-6.1.2} + + @since: 13.2 + """ + showAttributes = ('code', ('data', nativeString)) + compareAttributes = ('code', 'data') + + _fmt = '!HH' + + def __init__(self, code=0, data=b''): + """ + @type code: L{int} + @param code: The option code + + @type data: L{bytes} + @param data: The option data + """ + self.code = code + self.data = data + + + def encode(self, strio, compDict=None): + """ + Encode this L{_OPTVariableOption} to bytes. + + @type strio: L{file} + @param strio: the byte representation of this + L{_OPTVariableOption} will be written to this file. + + @type compDict: L{dict} or L{None} + @param compDict: A dictionary of backreference addresses that + have already been written to this stream and that may + be used for DNS name compression. + """ + strio.write( + struct.pack(self._fmt, self.code, len(self.data)) + self.data) + + + def decode(self, strio, length=None): + """ + Decode bytes into an L{_OPTVariableOption} instance. + + @type strio: L{file} + @param strio: Bytes will be read from this file until the full + L{_OPTVariableOption} is decoded. + + @type length: L{int} or L{None} + @param length: Not used. + """ + l = struct.calcsize(self._fmt) + buff = readPrecisely(strio, l) + self.code, length = struct.unpack(self._fmt, buff) + self.data = readPrecisely(strio, length) + + + +@implementer(IEncodable) +class RRHeader(tputil.FancyEqMixin): + """ + A resource record header. + + @cvar fmt: L{str} specifying the byte format of an RR. + + @ivar name: The name about which this reply contains information. + @type name: L{Name} + + @ivar type: The query type of the original request. + @type type: L{int} + + @ivar cls: The query class of the original request. + + @ivar ttl: The time-to-live for this record. + @type ttl: L{int} + + @ivar payload: An object that implements the L{IEncodable} interface + + @ivar auth: A L{bool} indicating whether this C{RRHeader} was parsed from + an authoritative message. + """ + compareAttributes = ('name', 'type', 'cls', 'ttl', 'payload', 'auth') + + fmt = "!HHIH" + + name = None + type = None + cls = None + ttl = None + payload = None + rdlength = None + + cachedResponse = None + + def __init__(self, name=b'', type=A, cls=IN, ttl=0, payload=None, + auth=False): + """ + @type name: L{bytes} or L{str} + @param name: See L{RRHeader.name} + + @type type: L{int} + @param type: The query type. + + @type cls: L{int} + @param cls: The query class. + + @type ttl: L{int} + @param ttl: Time to live for this record. This will be + converted to an L{int}. + + @type payload: An object implementing C{IEncodable} + @param payload: A Query Type specific data object. + + @raises TypeError: if the ttl cannot be converted to an L{int}. + @raises ValueError: if the ttl is negative. + """ + assert (payload is None) or isinstance(payload, UnknownRecord) or (payload.TYPE == type) + + integralTTL = int(ttl) + + if integralTTL < 0: + raise ValueError("TTL cannot be negative") + + self.name = Name(name) + self.type = type + self.cls = cls + self.ttl = integralTTL + self.payload = payload + self.auth = auth + + + def encode(self, strio, compDict=None): + self.name.encode(strio, compDict) + strio.write(struct.pack(self.fmt, self.type, self.cls, self.ttl, 0)) + if self.payload: + prefix = strio.tell() + self.payload.encode(strio, compDict) + aft = strio.tell() + strio.seek(prefix - 2, 0) + strio.write(struct.pack('!H', aft - prefix)) + strio.seek(aft, 0) + + + def decode(self, strio, length = None): + self.name.decode(strio) + l = struct.calcsize(self.fmt) + buff = readPrecisely(strio, l) + r = struct.unpack(self.fmt, buff) + self.type, self.cls, self.ttl, self.rdlength = r + + + def isAuthoritative(self): + return self.auth + + + def __str__(self): + t = QUERY_TYPES.get(self.type, EXT_QUERIES.get(self.type, 'UNKNOWN (%d)' % self.type)) + c = QUERY_CLASSES.get(self.cls, 'UNKNOWN (%d)' % self.cls) + return '' % (self.name, t, c, self.ttl, self.auth and 'True' or 'False') + + + __repr__ = __str__ + + + +@implementer(IEncodable, IRecord) +class SimpleRecord(tputil.FancyStrMixin, tputil.FancyEqMixin): + """ + A Resource Record which consists of a single RFC 1035 domain-name. + + @type name: L{Name} + @ivar name: The name associated with this record. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + """ + showAttributes = (('name', 'name', '%s'), 'ttl') + compareAttributes = ('name', 'ttl') + + TYPE = None + name = None + + def __init__(self, name=b'', ttl=None): + """ + @param name: See L{SimpleRecord.name} + @type name: L{bytes} or L{str} + """ + self.name = Name(name) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + self.name.encode(strio, compDict) + + + def decode(self, strio, length = None): + self.name = Name() + self.name.decode(strio) + + + def __hash__(self): + return hash(self.name) + + +# Kinds of RRs - oh my! +class Record_NS(SimpleRecord): + """ + An authoritative nameserver. + """ + TYPE = NS + fancybasename = 'NS' + + + +class Record_MD(SimpleRecord): + """ + A mail destination. + + This record type is obsolete. + + @see: L{Record_MX} + """ + TYPE = MD + fancybasename = 'MD' + + + +class Record_MF(SimpleRecord): + """ + A mail forwarder. + + This record type is obsolete. + + @see: L{Record_MX} + """ + TYPE = MF + fancybasename = 'MF' + + + +class Record_CNAME(SimpleRecord): + """ + The canonical name for an alias. + """ + TYPE = CNAME + fancybasename = 'CNAME' + + + +class Record_MB(SimpleRecord): + """ + A mailbox domain name. + + This is an experimental record type. + """ + TYPE = MB + fancybasename = 'MB' + + + +class Record_MG(SimpleRecord): + """ + A mail group member. + + This is an experimental record type. + """ + TYPE = MG + fancybasename = 'MG' + + + +class Record_MR(SimpleRecord): + """ + A mail rename domain name. + + This is an experimental record type. + """ + TYPE = MR + fancybasename = 'MR' + + + +class Record_PTR(SimpleRecord): + """ + A domain name pointer. + """ + TYPE = PTR + fancybasename = 'PTR' + + + +class Record_DNAME(SimpleRecord): + """ + A non-terminal DNS name redirection. + + This record type provides the capability to map an entire subtree of the + DNS name space to another domain. It differs from the CNAME record which + maps a single node of the name space. + + @see: U{http://www.faqs.org/rfcs/rfc2672.html} + @see: U{http://www.faqs.org/rfcs/rfc3363.html} + """ + TYPE = DNAME + fancybasename = 'DNAME' + + + +@implementer(IEncodable, IRecord) +class Record_A(tputil.FancyEqMixin): + """ + An IPv4 host address. + + @type address: L{bytes} + @ivar address: The packed network-order representation of the IPv4 address + associated with this record. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + """ + compareAttributes = ('address', 'ttl') + + TYPE = A + address = None + + def __init__(self, address='0.0.0.0', ttl=None): + """ + @type address: L{bytes} or L{unicode} + @param address: The IPv4 address associated with this record, in + quad-dotted notation. + """ + if _PY3 and isinstance(address, bytes): + address = address.decode('ascii') + + address = socket.inet_aton(address) + self.address = address + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + strio.write(self.address) + + + def decode(self, strio, length = None): + self.address = readPrecisely(strio, 4) + + + def __hash__(self): + return hash(self.address) + + + def __str__(self): + return '' % (self.dottedQuad(), self.ttl) + __repr__ = __str__ + + + def dottedQuad(self): + return socket.inet_ntoa(self.address) + + + +@implementer(IEncodable, IRecord) +class Record_SOA(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + Marks the start of a zone of authority. + + This record describes parameters which are shared by all records within a + particular zone. + + @type mname: L{Name} + @ivar mname: The domain-name of the name server that was the original or + primary source of data for this zone. + + @type rname: L{Name} + @ivar rname: A domain-name which specifies the mailbox of the person + responsible for this zone. + + @type serial: L{int} + @ivar serial: The unsigned 32 bit version number of the original copy of + the zone. Zone transfers preserve this value. This value wraps and + should be compared using sequence space arithmetic. + + @type refresh: L{int} + @ivar refresh: A 32 bit time interval before the zone should be refreshed. + + @type minimum: L{int} + @ivar minimum: The unsigned 32 bit minimum TTL field that should be + exported with any RR from this zone. + + @type expire: L{int} + @ivar expire: A 32 bit time value that specifies the upper limit on the + time interval that can elapse before the zone is no longer + authoritative. + + @type retry: L{int} + @ivar retry: A 32 bit time interval that should elapse before a failed + refresh should be retried. + + @type ttl: L{int} + @ivar ttl: The default TTL to use for records served from this zone. + """ + fancybasename = 'SOA' + compareAttributes = ('serial', 'mname', 'rname', 'refresh', 'expire', 'retry', 'minimum', 'ttl') + showAttributes = (('mname', 'mname', '%s'), ('rname', 'rname', '%s'), 'serial', 'refresh', 'retry', 'expire', 'minimum', 'ttl') + + TYPE = SOA + + def __init__(self, mname=b'', rname=b'', serial=0, refresh=0, retry=0, + expire=0, minimum=0, ttl=None): + """ + @param mname: See L{Record_SOA.mname} + @type mname: L{bytes} or L{unicode} + + @param rname: See L{Record_SOA.rname} + @type rname: L{bytes} or L{unicode} + """ + self.mname, self.rname = Name(mname), Name(rname) + self.serial, self.refresh = str2time(serial), str2time(refresh) + self.minimum, self.expire = str2time(minimum), str2time(expire) + self.retry = str2time(retry) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + self.mname.encode(strio, compDict) + self.rname.encode(strio, compDict) + strio.write( + struct.pack( + '!LlllL', + self.serial, self.refresh, self.retry, self.expire, + self.minimum + ) + ) + + + def decode(self, strio, length = None): + self.mname, self.rname = Name(), Name() + self.mname.decode(strio) + self.rname.decode(strio) + r = struct.unpack('!LlllL', readPrecisely(strio, 20)) + self.serial, self.refresh, self.retry, self.expire, self.minimum = r + + + def __hash__(self): + return hash(( + self.serial, self.mname, self.rname, + self.refresh, self.expire, self.retry + )) + + + +@implementer(IEncodable, IRecord) +class Record_NULL(tputil.FancyStrMixin, tputil.FancyEqMixin): + """ + A null record. + + This is an experimental record type. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + """ + fancybasename = 'NULL' + showAttributes = (('payload', _nicebytes), 'ttl') + compareAttributes = ('payload', 'ttl') + + TYPE = NULL + + def __init__(self, payload=None, ttl=None): + self.payload = payload + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + strio.write(self.payload) + + + def decode(self, strio, length = None): + self.payload = readPrecisely(strio, length) + + + def __hash__(self): + return hash(self.payload) + + + +@implementer(IEncodable, IRecord) +class Record_WKS(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + A well known service description. + + This record type is obsolete. See L{Record_SRV}. + + @type address: L{bytes} + @ivar address: The packed network-order representation of the IPv4 address + associated with this record. + + @type protocol: L{int} + @ivar protocol: The 8 bit IP protocol number for which this service map is + relevant. + + @type map: L{bytes} + @ivar map: A bitvector indicating the services available at the specified + address. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + """ + fancybasename = "WKS" + compareAttributes = ('address', 'protocol', 'map', 'ttl') + showAttributes = [('_address', 'address', '%s'), 'protocol', 'ttl'] + + TYPE = WKS + + _address = property(lambda self: socket.inet_ntoa(self.address)) + + def __init__(self, address='0.0.0.0', protocol=0, map=b'', ttl=None): + """ + @type address: L{bytes} or L{unicode} + @param address: The IPv4 address associated with this record, in + quad-dotted notation. + """ + if _PY3 and isinstance(address, bytes): + address = address.decode('idna') + + self.address = socket.inet_aton(address) + self.protocol, self.map = protocol, map + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + strio.write(self.address) + strio.write(struct.pack('!B', self.protocol)) + strio.write(self.map) + + + def decode(self, strio, length = None): + self.address = readPrecisely(strio, 4) + self.protocol = struct.unpack('!B', readPrecisely(strio, 1))[0] + self.map = readPrecisely(strio, length - 5) + + + def __hash__(self): + return hash((self.address, self.protocol, self.map)) + + + +@implementer(IEncodable, IRecord) +class Record_AAAA(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + An IPv6 host address. + + @type address: L{bytes} + @ivar address: The packed network-order representation of the IPv6 address + associated with this record. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + + @see: U{http://www.faqs.org/rfcs/rfc1886.html} + """ + TYPE = AAAA + + fancybasename = 'AAAA' + showAttributes = (('_address', 'address', '%s'), 'ttl') + compareAttributes = ('address', 'ttl') + + _address = property(lambda self: socket.inet_ntop(AF_INET6, self.address)) + + def __init__(self, address='::', ttl=None): + """ + @type address: L{bytes} or L{unicode} + @param address: The IPv6 address for this host, in RFC 2373 format. + """ + if _PY3 and isinstance(address, bytes): + address = address.decode('idna') + + self.address = socket.inet_pton(AF_INET6, address) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + strio.write(self.address) + + + def decode(self, strio, length = None): + self.address = readPrecisely(strio, 16) + + + def __hash__(self): + return hash(self.address) + + + +@implementer(IEncodable, IRecord) +class Record_A6(tputil.FancyStrMixin, tputil.FancyEqMixin): + """ + An IPv6 address. + + This is an experimental record type. + + @type prefixLen: L{int} + @ivar prefixLen: The length of the suffix. + + @type suffix: L{bytes} + @ivar suffix: An IPv6 address suffix in network order. + + @type prefix: L{Name} + @ivar prefix: If specified, a name which will be used as a prefix for other + A6 records. + + @type bytes: L{int} + @ivar bytes: The length of the prefix. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + + @see: U{http://www.faqs.org/rfcs/rfc2874.html} + @see: U{http://www.faqs.org/rfcs/rfc3363.html} + @see: U{http://www.faqs.org/rfcs/rfc3364.html} + """ + TYPE = A6 + + fancybasename = 'A6' + showAttributes = (('_suffix', 'suffix', '%s'), ('prefix', 'prefix', '%s'), 'ttl') + compareAttributes = ('prefixLen', 'prefix', 'suffix', 'ttl') + + _suffix = property(lambda self: socket.inet_ntop(AF_INET6, self.suffix)) + + def __init__(self, prefixLen=0, suffix='::', prefix=b'', ttl=None): + """ + @param suffix: An IPv6 address suffix in in RFC 2373 format. + @type suffix: L{bytes} or L{unicode} + + @param prefix: An IPv6 address prefix for other A6 records. + @type prefix: L{bytes} or L{unicode} + """ + if _PY3 and isinstance(suffix, bytes): + suffix = suffix.decode('idna') + + self.prefixLen = prefixLen + self.suffix = socket.inet_pton(AF_INET6, suffix) + self.prefix = Name(prefix) + self.bytes = int((128 - self.prefixLen) / 8.0) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + strio.write(struct.pack('!B', self.prefixLen)) + if self.bytes: + strio.write(self.suffix[-self.bytes:]) + if self.prefixLen: + # This may not be compressed + self.prefix.encode(strio, None) + + + def decode(self, strio, length = None): + self.prefixLen = struct.unpack('!B', readPrecisely(strio, 1))[0] + self.bytes = int((128 - self.prefixLen) / 8.0) + if self.bytes: + self.suffix = b'\x00' * (16 - self.bytes) + readPrecisely(strio, self.bytes) + if self.prefixLen: + self.prefix.decode(strio) + + + def __eq__(self, other): + if isinstance(other, Record_A6): + return (self.prefixLen == other.prefixLen and + self.suffix[-self.bytes:] == other.suffix[-self.bytes:] and + self.prefix == other.prefix and + self.ttl == other.ttl) + return NotImplemented + + + def __hash__(self): + return hash((self.prefixLen, self.suffix[-self.bytes:], self.prefix)) + + + def __str__(self): + return '' % ( + self.prefix, + socket.inet_ntop(AF_INET6, self.suffix), + self.prefixLen, self.ttl + ) + + + +@implementer(IEncodable, IRecord) +class Record_SRV(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + The location of the server(s) for a specific protocol and domain. + + This is an experimental record type. + + @type priority: L{int} + @ivar priority: The priority of this target host. A client MUST attempt to + contact the target host with the lowest-numbered priority it can reach; + target hosts with the same priority SHOULD be tried in an order defined + by the weight field. + + @type weight: L{int} + @ivar weight: Specifies a relative weight for entries with the same + priority. Larger weights SHOULD be given a proportionately higher + probability of being selected. + + @type port: L{int} + @ivar port: The port on this target host of this service. + + @type target: L{Name} + @ivar target: The domain name of the target host. There MUST be one or + more address records for this name, the name MUST NOT be an alias (in + the sense of RFC 1034 or RFC 2181). Implementors are urged, but not + required, to return the address record(s) in the Additional Data + section. Unless and until permitted by future standards action, name + compression is not to be used for this field. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + + @see: U{http://www.faqs.org/rfcs/rfc2782.html} + """ + TYPE = SRV + + fancybasename = 'SRV' + compareAttributes = ('priority', 'weight', 'target', 'port', 'ttl') + showAttributes = ('priority', 'weight', ('target', 'target', '%s'), 'port', 'ttl') + + def __init__(self, priority=0, weight=0, port=0, target=b'', ttl=None): + """ + @param target: See L{Record_SRV.target} + @type target: L{bytes} or L{unicode} + """ + self.priority = int(priority) + self.weight = int(weight) + self.port = int(port) + self.target = Name(target) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + strio.write(struct.pack('!HHH', self.priority, self.weight, self.port)) + # This can't be compressed + self.target.encode(strio, None) + + + def decode(self, strio, length = None): + r = struct.unpack('!HHH', readPrecisely(strio, struct.calcsize('!HHH'))) + self.priority, self.weight, self.port = r + self.target = Name() + self.target.decode(strio) + + + def __hash__(self): + return hash((self.priority, self.weight, self.port, self.target)) + + + +@implementer(IEncodable, IRecord) +class Record_NAPTR(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + The location of the server(s) for a specific protocol and domain. + + @type order: L{int} + @ivar order: An integer specifying the order in which the NAPTR records + MUST be processed to ensure the correct ordering of rules. Low numbers + are processed before high numbers. + + @type preference: L{int} + @ivar preference: An integer that specifies the order in which NAPTR + records with equal "order" values SHOULD be processed, low numbers + being processed before high numbers. + + @type flag: L{Charstr} + @ivar flag: A containing flags to control aspects of the + rewriting and interpretation of the fields in the record. Flags + are single characters from the set [A-Z0-9]. The case of the alphabetic + characters is not significant. + + At this time only four flags, "S", "A", "U", and "P", are defined. + + @type service: L{Charstr} + @ivar service: Specifies the service(s) available down this rewrite path. + It may also specify the particular protocol that is used to talk with a + service. A protocol MUST be specified if the flags field states that + the NAPTR is terminal. + + @type regexp: L{Charstr} + @ivar regexp: A STRING containing a substitution expression that is applied + to the original string held by the client in order to construct the + next domain name to lookup. + + @type replacement: L{Name} + @ivar replacement: The next NAME to query for NAPTR, SRV, or address + records depending on the value of the flags field. This MUST be a + fully qualified domain-name. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + + @see: U{http://www.faqs.org/rfcs/rfc2915.html} + """ + TYPE = NAPTR + + compareAttributes = ('order', 'preference', 'flags', 'service', 'regexp', + 'replacement') + fancybasename = 'NAPTR' + + showAttributes = ('order', 'preference', ('flags', 'flags', '%s'), + ('service', 'service', '%s'), ('regexp', 'regexp', '%s'), + ('replacement', 'replacement', '%s'), 'ttl') + + def __init__(self, order=0, preference=0, flags=b'', service=b'', + regexp=b'', replacement=b'', ttl=None): + """ + @param replacement: See L{Record_NAPTR.replacement} + @type replacement: L{bytes} or L{unicode} + """ + self.order = int(order) + self.preference = int(preference) + self.flags = Charstr(flags) + self.service = Charstr(service) + self.regexp = Charstr(regexp) + self.replacement = Name(replacement) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict=None): + strio.write(struct.pack('!HH', self.order, self.preference)) + # This can't be compressed + self.flags.encode(strio, None) + self.service.encode(strio, None) + self.regexp.encode(strio, None) + self.replacement.encode(strio, None) + + + def decode(self, strio, length=None): + r = struct.unpack('!HH', readPrecisely(strio, struct.calcsize('!HH'))) + self.order, self.preference = r + self.flags = Charstr() + self.service = Charstr() + self.regexp = Charstr() + self.replacement = Name() + self.flags.decode(strio) + self.service.decode(strio) + self.regexp.decode(strio) + self.replacement.decode(strio) + + + def __hash__(self): + return hash(( + self.order, self.preference, self.flags, + self.service, self.regexp, self.replacement)) + + + +@implementer(IEncodable, IRecord) +class Record_AFSDB(tputil.FancyStrMixin, tputil.FancyEqMixin): + """ + Map from a domain name to the name of an AFS cell database server. + + @type subtype: L{int} + @ivar subtype: In the case of subtype 1, the host has an AFS version 3.0 + Volume Location Server for the named AFS cell. In the case of subtype + 2, the host has an authenticated name server holding the cell-root + directory node for the named DCE/NCA cell. + + @type hostname: L{Name} + @ivar hostname: The domain name of a host that has a server for the cell + named by this record. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + + @see: U{http://www.faqs.org/rfcs/rfc1183.html} + """ + TYPE = AFSDB + + fancybasename = 'AFSDB' + compareAttributes = ('subtype', 'hostname', 'ttl') + showAttributes = ('subtype', ('hostname', 'hostname', '%s'), 'ttl') + + def __init__(self, subtype=0, hostname=b'', ttl=None): + """ + @param hostname: See L{Record_AFSDB.hostname} + @type hostname: L{bytes} or L{unicode} + """ + self.subtype = int(subtype) + self.hostname = Name(hostname) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + strio.write(struct.pack('!H', self.subtype)) + self.hostname.encode(strio, compDict) + + + def decode(self, strio, length = None): + r = struct.unpack('!H', readPrecisely(strio, struct.calcsize('!H'))) + self.subtype, = r + self.hostname.decode(strio) + + + def __hash__(self): + return hash((self.subtype, self.hostname)) + + + +@implementer(IEncodable, IRecord) +class Record_RP(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + The responsible person for a domain. + + @type mbox: L{Name} + @ivar mbox: A domain name that specifies the mailbox for the responsible + person. + + @type txt: L{Name} + @ivar txt: A domain name for which TXT RR's exist (indirection through + which allows information sharing about the contents of this RP record). + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + + @see: U{http://www.faqs.org/rfcs/rfc1183.html} + """ + TYPE = RP + + fancybasename = 'RP' + compareAttributes = ('mbox', 'txt', 'ttl') + showAttributes = (('mbox', 'mbox', '%s'), ('txt', 'txt', '%s'), 'ttl') + + def __init__(self, mbox=b'', txt=b'', ttl=None): + """ + @param mbox: See L{Record_RP.mbox}. + @type mbox: L{bytes} or L{unicode} + + @param txt: See L{Record_RP.txt} + @type txt: L{bytes} or L{unicode} + """ + self.mbox = Name(mbox) + self.txt = Name(txt) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + self.mbox.encode(strio, compDict) + self.txt.encode(strio, compDict) + + + def decode(self, strio, length = None): + self.mbox = Name() + self.txt = Name() + self.mbox.decode(strio) + self.txt.decode(strio) + + + def __hash__(self): + return hash((self.mbox, self.txt)) + + + +@implementer(IEncodable, IRecord) +class Record_HINFO(tputil.FancyStrMixin, tputil.FancyEqMixin): + """ + Host information. + + @type cpu: L{bytes} + @ivar cpu: Specifies the CPU type. + + @type os: L{bytes} + @ivar os: Specifies the OS. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + """ + TYPE = HINFO + + fancybasename = 'HINFO' + showAttributes = (('cpu', _nicebytes), ('os', _nicebytes), 'ttl') + compareAttributes = ('cpu', 'os', 'ttl') + + def __init__(self, cpu=b'', os=b'', ttl=None): + self.cpu, self.os = cpu, os + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + strio.write(struct.pack('!B', len(self.cpu)) + self.cpu) + strio.write(struct.pack('!B', len(self.os)) + self.os) + + + def decode(self, strio, length = None): + cpu = struct.unpack('!B', readPrecisely(strio, 1))[0] + self.cpu = readPrecisely(strio, cpu) + os = struct.unpack('!B', readPrecisely(strio, 1))[0] + self.os = readPrecisely(strio, os) + + + def __eq__(self, other): + if isinstance(other, Record_HINFO): + return (self.os.lower() == other.os.lower() and + self.cpu.lower() == other.cpu.lower() and + self.ttl == other.ttl) + return NotImplemented + + + def __hash__(self): + return hash((self.os.lower(), self.cpu.lower())) + + + +@implementer(IEncodable, IRecord) +class Record_MINFO(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + Mailbox or mail list information. + + This is an experimental record type. + + @type rmailbx: L{Name} + @ivar rmailbx: A domain-name which specifies a mailbox which is responsible + for the mailing list or mailbox. If this domain name names the root, + the owner of the MINFO RR is responsible for itself. + + @type emailbx: L{Name} + @ivar emailbx: A domain-name which specifies a mailbox which is to receive + error messages related to the mailing list or mailbox specified by the + owner of the MINFO record. If this domain name names the root, errors + should be returned to the sender of the message. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + """ + TYPE = MINFO + + rmailbx = None + emailbx = None + + fancybasename = 'MINFO' + compareAttributes = ('rmailbx', 'emailbx', 'ttl') + showAttributes = (('rmailbx', 'responsibility', '%s'), + ('emailbx', 'errors', '%s'), + 'ttl') + + def __init__(self, rmailbx=b'', emailbx=b'', ttl=None): + """ + @param rmailbx: See L{Record_MINFO.rmailbx}. + @type rmailbx: L{bytes} or L{unicode} + + @param emailbx: See L{Record_MINFO.rmailbx}. + @type emailbx: L{bytes} or L{unicode} + """ + self.rmailbx, self.emailbx = Name(rmailbx), Name(emailbx) + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict = None): + self.rmailbx.encode(strio, compDict) + self.emailbx.encode(strio, compDict) + + + def decode(self, strio, length = None): + self.rmailbx, self.emailbx = Name(), Name() + self.rmailbx.decode(strio) + self.emailbx.decode(strio) + + + def __hash__(self): + return hash((self.rmailbx, self.emailbx)) + + + +@implementer(IEncodable, IRecord) +class Record_MX(tputil.FancyStrMixin, tputil.FancyEqMixin): + """ + Mail exchange. + + @type preference: L{int} + @ivar preference: Specifies the preference given to this RR among others at + the same owner. Lower values are preferred. + + @type name: L{Name} + @ivar name: A domain-name which specifies a host willing to act as a mail + exchange. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be + cached. + """ + TYPE = MX + + fancybasename = 'MX' + compareAttributes = ('preference', 'name', 'ttl') + showAttributes = ('preference', ('name', 'name', '%s'), 'ttl') + + def __init__(self, preference=0, name=b'', ttl=None, **kwargs): + """ + @param name: See L{Record_MX.name}. + @type name: L{bytes} or L{unicode} + """ + self.preference = int(preference) + self.name = Name(kwargs.get('exchange', name)) + self.ttl = str2time(ttl) + + def encode(self, strio, compDict = None): + strio.write(struct.pack('!H', self.preference)) + self.name.encode(strio, compDict) + + + def decode(self, strio, length = None): + self.preference = struct.unpack('!H', readPrecisely(strio, 2))[0] + self.name = Name() + self.name.decode(strio) + + def __hash__(self): + return hash((self.preference, self.name)) + + + +@implementer(IEncodable, IRecord) +class Record_SSHFP(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + A record containing the fingerprint of an SSH key. + + @type algorithm: L{int} + @ivar algorithm: The SSH key's algorithm, such as L{ALGORITHM_RSA}. + Note that the numbering used for SSH key algorithms is specific + to the SSHFP record, and is not the same as the numbering + used for KEY or SIG records. + + @type fingerprintType: L{int} + @ivar fingerprintType: The fingerprint type, + such as L{FINGERPRINT_TYPE_SHA256}. + + @type fingerprint: L{bytes} + @ivar fingerprint: The key's fingerprint, e.g. a 32-byte SHA-256 digest. + + @cvar ALGORITHM_RSA: The algorithm value for C{ssh-rsa} keys. + @cvar ALGORITHM_DSS: The algorithm value for C{ssh-dss} keys. + @cvar ALGORITHM_ECDSA: The algorithm value for C{ecdsa-sha2-*} keys. + @cvar ALGORITHM_Ed25519: The algorithm value for C{ed25519} keys. + + @cvar FINGERPRINT_TYPE_SHA1: The type for SHA-1 fingerprints. + @cvar FINGERPRINT_TYPE_SHA256: The type for SHA-256 fingerprints. + + @see: U{RFC 4255 } + and + U{RFC 6594 } + """ + fancybasename = "SSHFP" + compareAttributes = ('algorithm', 'fingerprintType', 'fingerprint', 'ttl') + showAttributes = ('algorithm', 'fingerprintType', 'fingerprint') + + TYPE = SSHFP + + ALGORITHM_RSA = 1 + ALGORITHM_DSS = 2 + ALGORITHM_ECDSA = 3 + ALGORITHM_Ed25519 = 4 + + FINGERPRINT_TYPE_SHA1 = 1 + FINGERPRINT_TYPE_SHA256 = 2 + + def __init__(self, algorithm=0, fingerprintType=0, fingerprint=b'', ttl=0): + self.algorithm = algorithm + self.fingerprintType = fingerprintType + self.fingerprint = fingerprint + self.ttl = ttl + + + def encode(self, strio, compDict=None): + strio.write(struct.pack('!BB', + self.algorithm, self.fingerprintType)) + strio.write(self.fingerprint) + + + def decode(self, strio, length=None): + r = struct.unpack('!BB', readPrecisely(strio, 2)) + (self.algorithm, self.fingerprintType) = r + self.fingerprint = readPrecisely(strio, length - 2) + + + def __hash__(self): + return hash((self.algorithm, self.fingerprintType, self.fingerprint)) + + + +@implementer(IEncodable, IRecord) +class Record_TXT(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + Freeform text. + + @type data: L{list} of L{bytes} + @ivar data: Freeform text which makes up this record. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be cached. + """ + TYPE = TXT + + fancybasename = 'TXT' + showAttributes = (('data', _nicebyteslist), 'ttl') + compareAttributes = ('data', 'ttl') + + def __init__(self, *data, **kw): + self.data = list(data) + # arg man python sucks so bad + self.ttl = str2time(kw.get('ttl', None)) + + + def encode(self, strio, compDict=None): + for d in self.data: + strio.write(struct.pack('!B', len(d)) + d) + + + def decode(self, strio, length=None): + soFar = 0 + self.data = [] + while soFar < length: + L = struct.unpack('!B', readPrecisely(strio, 1))[0] + self.data.append(readPrecisely(strio, L)) + soFar += L + 1 + if soFar != length: + log.msg( + "Decoded %d bytes in %s record, but rdlength is %d" % ( + soFar, self.fancybasename, length + ) + ) + + + def __hash__(self): + return hash(tuple(self.data)) + + + +@implementer(IEncodable, IRecord) +class UnknownRecord(tputil.FancyEqMixin, tputil.FancyStrMixin, object): + """ + Encapsulate the wire data for unknown record types so that they can + pass through the system unchanged. + + @type data: L{bytes} + @ivar data: Wire data which makes up this record. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds which this record should be cached. + + @since: 11.1 + """ + fancybasename = 'UNKNOWN' + compareAttributes = ('data', 'ttl') + showAttributes = (('data', _nicebytes), 'ttl') + + def __init__(self, data=b'', ttl=None): + self.data = data + self.ttl = str2time(ttl) + + + def encode(self, strio, compDict=None): + """ + Write the raw bytes corresponding to this record's payload to the + stream. + """ + strio.write(self.data) + + + def decode(self, strio, length=None): + """ + Load the bytes which are part of this record from the stream and store + them unparsed and unmodified. + """ + if length is None: + raise Exception('must know length for unknown record types') + self.data = readPrecisely(strio, length) + + + def __hash__(self): + return hash((self.data, self.ttl)) + + + +class Record_SPF(Record_TXT): + """ + Structurally, freeform text. Semantically, a policy definition, formatted + as defined in U{rfc 4408}. + + @type data: L{list} of L{bytes} + @ivar data: Freeform text which makes up this record. + + @type ttl: L{int} + @ivar ttl: The maximum number of seconds + which this record should be cached. + """ + TYPE = SPF + fancybasename = 'SPF' + + + +@implementer(IEncodable, IRecord) +class Record_TSIG(tputil.FancyEqMixin, tputil.FancyStrMixin): + """ + A transaction signature, encapsulated in a RR, as described + in U{RFC 2845 }. + + @type algorithm: L{Name} + @ivar algorithm: The name of the signature or MAC algorithm. + + @type timeSigned: L{int} + @ivar timeSigned: Signing time, as seconds from the POSIX epoch. + + @type fudge: L{int} + @ivar fudge: Allowable time skew, in seconds. + + @type MAC: L{bytes} + @ivar MAC: The message digest or signature. + + @type originalID: L{int} + @ivar originalID: A message ID. + + @type error: L{int} + @ivar error: An error code (extended C{RCODE}) carried + in exceptional cases. + + @type otherData: L{bytes} + @ivar otherData: Other data carried in exceptional cases. + + """ + fancybasename = "TSIG" + compareAttributes = ('algorithm', 'timeSigned', 'fudge', + 'MAC', 'originalID', 'error', 'otherData', + 'ttl') + showAttributes = ['algorithm', 'timeSigned', 'MAC', 'error', 'otherData'] + + TYPE = TSIG + + def __init__(self, algorithm=None, timeSigned=None, + fudge=5, MAC=None, originalID=0, + error=OK, otherData=b'', ttl=0): + # All of our init arguments have to have defaults, because of + # the way IEncodable and Message.parseRecords() work, but for + # some of our arguments there is no reasonable default; we use + # invalid values here to prevent a user of this class from + # relying on what's really an internal implementation detail. + self.algorithm = None if algorithm is None else Name(algorithm) + self.timeSigned = timeSigned + self.fudge = str2time(fudge) + self.MAC = MAC + self.originalID = originalID + self.error = error + self.otherData = otherData + self.ttl = ttl + + + def encode(self, strio, compDict=None): + self.algorithm.encode(strio, compDict) + strio.write(struct.pack('!Q', self.timeSigned)[2:]) # 48-bit number + strio.write(struct.pack('!HH', self.fudge, len(self.MAC))) + strio.write(self.MAC) + strio.write(struct.pack('!HHH', + self.originalID, self.error, + len(self.otherData))) + strio.write(self.otherData) + + + def decode(self, strio, length=None): + algorithm = Name() + algorithm.decode(strio) + self.algorithm = algorithm + fields = struct.unpack('!QHH', b'\x00\x00' + readPrecisely(strio, 10)) + self.timeSigned, self.fudge, macLength = fields + self.MAC = readPrecisely(strio, macLength) + fields = struct.unpack('!HHH', readPrecisely(strio, 6)) + self.originalID, self.error, otherLength = fields + self.otherData = readPrecisely(strio, otherLength) + + + def __hash__(self): + return hash((self.algorithm, self.timeSigned, + self.MAC, self.originalID)) + + + +def _responseFromMessage(responseConstructor, message, **kwargs): + """ + Generate a L{Message} like instance suitable for use as the response to + C{message}. + + The C{queries}, C{id} attributes will be copied from C{message} and the + C{answer} flag will be set to L{True}. + + @param responseConstructor: A response message constructor with an + initializer signature matching L{dns.Message.__init__}. + @type responseConstructor: C{callable} + + @param message: A request message. + @type message: L{Message} + + @param kwargs: Keyword arguments which will be passed to the initialiser + of the response message. + @type kwargs: L{dict} + + @return: A L{Message} like response instance. + @rtype: C{responseConstructor} + """ + response = responseConstructor(id=message.id, answer=True, **kwargs) + response.queries = message.queries[:] + return response + + + +def _getDisplayableArguments(obj, alwaysShow, fieldNames): + """ + Inspect the function signature of C{obj}'s constructor, + and get a list of which arguments should be displayed. + This is a helper function for C{_compactRepr}. + + @param obj: The instance whose repr is being generated. + @param alwaysShow: A L{list} of field names which should always be shown. + @param fieldNames: A L{list} of field attribute names which should be shown + if they have non-default values. + @return: A L{list} of displayable arguments. + """ + displayableArgs = [] + if _PY3: + # Get the argument names and values from the constructor. + signature = inspect.signature(obj.__class__.__init__) + for name in fieldNames: + defaultValue = signature.parameters[name].default + fieldValue = getattr(obj, name, defaultValue) + if (name in alwaysShow) or (fieldValue != defaultValue): + displayableArgs.append(' %s=%r' % (name, fieldValue)) + else: + # Get the argument names and values from the constructor. + argspec = inspect.getargspec(obj.__class__.__init__) + # Reverse the args and defaults to avoid mapping positional arguments + # which don't have a default. + defaults = dict(zip(reversed(argspec.args), reversed(argspec.defaults))) + for name in fieldNames: + defaultValue = defaults.get(name) + fieldValue = getattr(obj, name, defaultValue) + if (name in alwaysShow) or (fieldValue != defaultValue): + displayableArgs.append(' %s=%r' % (name, fieldValue)) + + return displayableArgs + + + +def _compactRepr(obj, alwaysShow=None, flagNames=None, fieldNames=None, + sectionNames=None): + """ + Return a L{str} representation of C{obj} which only shows fields with + non-default values, flags which are True and sections which have been + explicitly set. + + @param obj: The instance whose repr is being generated. + @param alwaysShow: A L{list} of field names which should always be shown. + @param flagNames: A L{list} of flag attribute names which should be shown if + they are L{True}. + @param fieldNames: A L{list} of field attribute names which should be shown + if they have non-default values. + @param sectionNames: A L{list} of section attribute names which should be + shown if they have been assigned a value. + + @return: A L{str} representation of C{obj}. + """ + if alwaysShow is None: + alwaysShow = [] + + if flagNames is None: + flagNames = [] + + if fieldNames is None: + fieldNames = [] + + if sectionNames is None: + sectionNames = [] + + setFlags = [] + for name in flagNames: + if name in alwaysShow or getattr(obj, name, False) == True: + setFlags.append(name) + + displayableArgs = _getDisplayableArguments(obj, alwaysShow, fieldNames) + out = ['<', obj.__class__.__name__] + displayableArgs + + if setFlags: + out.append(' flags=%s' % (','.join(setFlags),)) + + for name in sectionNames: + section = getattr(obj, name, []) + if section: + out.append(' %s=%r' % (name, section)) + + out.append('>') + + return ''.join(out) + + + +class Message(tputil.FancyEqMixin): + """ + L{Message} contains all the information represented by a single + DNS request or response. + + @ivar id: See L{__init__} + @ivar answer: See L{__init__} + @ivar opCode: See L{__init__} + @ivar recDes: See L{__init__} + @ivar recAv: See L{__init__} + @ivar auth: See L{__init__} + @ivar rCode: See L{__init__} + @ivar trunc: See L{__init__} + @ivar maxSize: See L{__init__} + @ivar authenticData: See L{__init__} + @ivar checkingDisabled: See L{__init__} + + @ivar queries: The queries which are being asked of or answered by + DNS server. + @type queries: L{list} of L{Query} + + @ivar answers: Records containing the answers to C{queries} if + this is a response message. + @type answers: L{list} of L{RRHeader} + + @ivar authority: Records containing information about the + authoritative DNS servers for the names in C{queries}. + @type authority: L{list} of L{RRHeader} + + @ivar additional: Records containing IP addresses of host names + in C{answers} and C{authority}. + @type additional: L{list} of L{RRHeader} + + @ivar _flagNames: The names of attributes representing the flag header + fields. + @ivar _fieldNames: The names of attributes representing non-flag fixed + header fields. + @ivar _sectionNames: The names of attributes representing the record + sections of this message. + """ + compareAttributes = ( + 'id', 'answer', 'opCode', 'recDes', 'recAv', + 'auth', 'rCode', 'trunc', 'maxSize', + 'authenticData', 'checkingDisabled', + 'queries', 'answers', 'authority', 'additional' + ) + + headerFmt = "!H2B4H" + headerSize = struct.calcsize(headerFmt) + + # Question, answer, additional, and nameserver lists + queries = answers = add = ns = None + + def __init__(self, id=0, answer=0, opCode=0, recDes=0, recAv=0, + auth=0, rCode=OK, trunc=0, maxSize=512, + authenticData=0, checkingDisabled=0): + """ + @param id: A 16 bit identifier assigned by the program that + generates any kind of query. This identifier is copied to + the corresponding reply and can be used by the requester + to match up replies to outstanding queries. + @type id: L{int} + + @param answer: A one bit field that specifies whether this + message is a query (0), or a response (1). + @type answer: L{int} + + @param opCode: A four bit field that specifies kind of query in + this message. This value is set by the originator of a query + and copied into the response. + @type opCode: L{int} + + @param recDes: Recursion Desired - this bit may be set in a + query and is copied into the response. If RD is set, it + directs the name server to pursue the query recursively. + Recursive query support is optional. + @type recDes: L{int} + + @param recAv: Recursion Available - this bit is set or cleared + in a response and denotes whether recursive query support + is available in the name server. + @type recAv: L{int} + + @param auth: Authoritative Answer - this bit is valid in + responses and specifies that the responding name server + is an authority for the domain name in question section. + @type auth: L{int} + + @ivar rCode: A response code, used to indicate success or failure in a + message which is a response from a server to a client request. + @type rCode: C{0 <= int < 16} + + @param trunc: A flag indicating that this message was + truncated due to length greater than that permitted on the + transmission channel. + @type trunc: L{int} + + @param maxSize: The requestor's UDP payload size is the number + of octets of the largest UDP payload that can be + reassembled and delivered in the requestor's network + stack. + @type maxSize: L{int} + + @param authenticData: A flag indicating in a response that all + the data included in the answer and authority portion of + the response has been authenticated by the server + according to the policies of that server. + See U{RFC2535 section-6.1}. + @type authenticData: L{int} + + @param checkingDisabled: A flag indicating in a query that + pending (non-authenticated) data is acceptable to the + resolver sending the query. + See U{RFC2535 section-6.1}. + @type authenticData: L{int} + """ + self.maxSize = maxSize + self.id = id + self.answer = answer + self.opCode = opCode + self.auth = auth + self.trunc = trunc + self.recDes = recDes + self.recAv = recAv + self.rCode = rCode + self.authenticData = authenticData + self.checkingDisabled = checkingDisabled + + self.queries = [] + self.answers = [] + self.authority = [] + self.additional = [] + + + def __repr__(self): + """ + Generate a repr of this L{Message}. + + Only includes the non-default fields and sections and only includes + flags which are set. The C{id} is always shown. + + @return: The native string repr. + """ + return _compactRepr( + self, + flagNames=('answer', 'auth', 'trunc', 'recDes', 'recAv', + 'authenticData', 'checkingDisabled'), + fieldNames=('id', 'opCode', 'rCode', 'maxSize'), + sectionNames=('queries', 'answers', 'authority', 'additional'), + alwaysShow=('id',) + ) + + + def addQuery(self, name, type=ALL_RECORDS, cls=IN): + """ + Add another query to this Message. + + @type name: L{bytes} + @param name: The name to query. + + @type type: L{int} + @param type: Query type + + @type cls: L{int} + @param cls: Query class + """ + self.queries.append(Query(name, type, cls)) + + + def encode(self, strio): + compDict = {} + body_tmp = BytesIO() + for q in self.queries: + q.encode(body_tmp, compDict) + for q in self.answers: + q.encode(body_tmp, compDict) + for q in self.authority: + q.encode(body_tmp, compDict) + for q in self.additional: + q.encode(body_tmp, compDict) + body = body_tmp.getvalue() + size = len(body) + self.headerSize + if self.maxSize and size > self.maxSize: + self.trunc = 1 + body = body[:self.maxSize - self.headerSize] + byte3 = (( ( self.answer & 1 ) << 7 ) + | ((self.opCode & 0xf ) << 3 ) + | ((self.auth & 1 ) << 2 ) + | ((self.trunc & 1 ) << 1 ) + | ( self.recDes & 1 ) ) + byte4 = ( ( (self.recAv & 1 ) << 7 ) + | ((self.authenticData & 1) << 5) + | ((self.checkingDisabled & 1) << 4) + | (self.rCode & 0xf ) ) + + strio.write(struct.pack(self.headerFmt, self.id, byte3, byte4, + len(self.queries), len(self.answers), + len(self.authority), len(self.additional))) + strio.write(body) + + + def decode(self, strio, length=None): + self.maxSize = 0 + header = readPrecisely(strio, self.headerSize) + r = struct.unpack(self.headerFmt, header) + self.id, byte3, byte4, nqueries, nans, nns, nadd = r + self.answer = ( byte3 >> 7 ) & 1 + self.opCode = ( byte3 >> 3 ) & 0xf + self.auth = ( byte3 >> 2 ) & 1 + self.trunc = ( byte3 >> 1 ) & 1 + self.recDes = byte3 & 1 + self.recAv = ( byte4 >> 7 ) & 1 + self.authenticData = ( byte4 >> 5 ) & 1 + self.checkingDisabled = ( byte4 >> 4 ) & 1 + self.rCode = byte4 & 0xf + + self.queries = [] + for i in range(nqueries): + q = Query() + try: + q.decode(strio) + except EOFError: + return + self.queries.append(q) + + items = ( + (self.answers, nans), + (self.authority, nns), + (self.additional, nadd)) + + for (l, n) in items: + self.parseRecords(l, n, strio) + + + def parseRecords(self, list, num, strio): + for i in range(num): + header = RRHeader(auth=self.auth) + try: + header.decode(strio) + except EOFError: + return + t = self.lookupRecordType(header.type) + if not t: + continue + header.payload = t(ttl=header.ttl) + try: + header.payload.decode(strio, header.rdlength) + except EOFError: + return + list.append(header) + + + # Create a mapping from record types to their corresponding Record_* + # classes. This relies on the global state which has been created so + # far in initializing this module (so don't define Record classes after + # this). + _recordTypes = {} + for name in globals(): + if name.startswith('Record_'): + _recordTypes[globals()[name].TYPE] = globals()[name] + + # Clear the iteration variable out of the class namespace so it + # doesn't become an attribute. + del name + + + def lookupRecordType(self, type): + """ + Retrieve the L{IRecord} implementation for the given record type. + + @param type: A record type, such as C{A} or L{NS}. + @type type: L{int} + + @return: An object which implements L{IRecord} or L{None} if none + can be found for the given type. + @rtype: L{types.ClassType} + """ + return self._recordTypes.get(type, UnknownRecord) + + + def toStr(self): + """ + Encode this L{Message} into a byte string in the format described by RFC + 1035. + + @rtype: L{bytes} + """ + strio = BytesIO() + self.encode(strio) + return strio.getvalue() + + + def fromStr(self, str): + """ + Decode a byte string in the format described by RFC 1035 into this + L{Message}. + + @param str: L{bytes} + """ + strio = BytesIO(str) + self.decode(strio) + + + +class _EDNSMessage(tputil.FancyEqMixin, object): + """ + An I{EDNS} message. + + Designed for compatibility with L{Message} but with a narrower public + interface. + + Most importantly, L{_EDNSMessage.fromStr} will interpret and remove I{OPT} + records that are present in the additional records section. + + The I{OPT} records are used to populate certain I{EDNS} specific attributes. + + L{_EDNSMessage.toStr} will add suitable I{OPT} records to the additional + section to represent the extended EDNS information. + + @see: U{https://tools.ietf.org/html/rfc6891} + + @ivar id: See L{__init__} + @ivar answer: See L{__init__} + @ivar opCode: See L{__init__} + @ivar auth: See L{__init__} + @ivar trunc: See L{__init__} + @ivar recDes: See L{__init__} + @ivar recAv: See L{__init__} + @ivar rCode: See L{__init__} + @ivar ednsVersion: See L{__init__} + @ivar dnssecOK: See L{__init__} + @ivar authenticData: See L{__init__} + @ivar checkingDisabled: See L{__init__} + @ivar maxSize: See L{__init__} + + @ivar queries: See L{__init__} + @ivar answers: See L{__init__} + @ivar authority: See L{__init__} + @ivar additional: See L{__init__} + + @ivar _messageFactory: A constructor of L{Message} instances. Called by + C{_toMessage} and C{_fromMessage}. + """ + + compareAttributes = ( + 'id', 'answer', 'opCode', 'auth', 'trunc', + 'recDes', 'recAv', 'rCode', 'ednsVersion', 'dnssecOK', + 'authenticData', 'checkingDisabled', 'maxSize', + 'queries', 'answers', 'authority', 'additional') + + _messageFactory = Message + + def __init__(self, id=0, answer=False, opCode=OP_QUERY, auth=False, + trunc=False, recDes=False, recAv=False, rCode=0, + ednsVersion=0, dnssecOK=False, authenticData=False, + checkingDisabled=False, maxSize=512, + queries=None, answers=None, authority=None, additional=None): + """ + Construct a new L{_EDNSMessage} + + @see: U{RFC1035 section-4.1.1} + @see: U{RFC2535 section-6.1} + @see: U{RFC3225 section-3} + @see: U{RFC6891 section-6.1.3} + + @param id: A 16 bit identifier assigned by the program that generates + any kind of query. This identifier is copied the corresponding + reply and can be used by the requester to match up replies to + outstanding queries. + @type id: L{int} + + @param answer: A one bit field that specifies whether this message is a + query (0), or a response (1). + @type answer: L{bool} + + @param opCode: A four bit field that specifies kind of query in this + message. This value is set by the originator of a query and copied + into the response. + @type opCode: L{int} + + @param auth: Authoritative Answer - this bit is valid in responses, and + specifies that the responding name server is an authority for the + domain name in question section. + @type auth: L{bool} + + @param trunc: Truncation - specifies that this message was truncated due + to length greater than that permitted on the transmission channel. + @type trunc: L{bool} + + @param recDes: Recursion Desired - this bit may be set in a query and is + copied into the response. If set, it directs the name server to + pursue the query recursively. Recursive query support is optional. + @type recDes: L{bool} + + @param recAv: Recursion Available - this bit is set or cleared in a + response, and denotes whether recursive query support is available + in the name server. + @type recAv: L{bool} + + @param rCode: Extended 12-bit RCODE. Derived from the 4 bits defined in + U{RFC1035 4.1.1} + and the upper 8bits defined in U{RFC6891 + 6.1.3}. + @type rCode: L{int} + + @param ednsVersion: Indicates the EDNS implementation level. Set to + L{None} to prevent any EDNS attributes and options being added to + the encoded byte string. + @type ednsVersion: L{int} or L{None} + + @param dnssecOK: DNSSEC OK bit as defined by + U{RFC3225 3}. + @type dnssecOK: L{bool} + + @param authenticData: A flag indicating in a response that all the data + included in the answer and authority portion of the response has + been authenticated by the server according to the policies of that + server. + See U{RFC2535 section-6.1}. + @type authenticData: L{bool} + + @param checkingDisabled: A flag indicating in a query that pending + (non-authenticated) data is acceptable to the resolver sending the + query. + See U{RFC2535 section-6.1}. + @type authenticData: L{bool} + + @param maxSize: The requestor's UDP payload size is the number of octets + of the largest UDP payload that can be reassembled and delivered in + the requestor's network stack. + @type maxSize: L{int} + + @param queries: The L{list} of L{Query} associated with this message. + @type queries: L{list} of L{Query} + + @param answers: The L{list} of answers associated with this message. + @type answers: L{list} of L{RRHeader} + + @param authority: The L{list} of authority records associated with this + message. + @type authority: L{list} of L{RRHeader} + + @param additional: The L{list} of additional records associated with + this message. + @type additional: L{list} of L{RRHeader} + """ + self.id = id + self.answer = answer + self.opCode = opCode + self.auth = auth + self.trunc = trunc + self.recDes = recDes + self.recAv = recAv + self.rCode = rCode + self.ednsVersion = ednsVersion + self.dnssecOK = dnssecOK + self.authenticData = authenticData + self.checkingDisabled = checkingDisabled + self.maxSize = maxSize + + if queries is None: + queries = [] + self.queries = queries + + if answers is None: + answers = [] + self.answers = answers + + if authority is None: + authority = [] + self.authority = authority + + if additional is None: + additional = [] + self.additional = additional + + + def __repr__(self): + return _compactRepr( + self, + flagNames=('answer', 'auth', 'trunc', 'recDes', 'recAv', + 'authenticData', 'checkingDisabled', 'dnssecOK'), + fieldNames=('id', 'opCode', 'rCode', 'maxSize', 'ednsVersion'), + sectionNames=('queries', 'answers', 'authority', 'additional'), + alwaysShow=('id',) + ) + + + def _toMessage(self): + """ + Convert to a standard L{dns.Message}. + + If C{ednsVersion} is not None, an L{_OPTHeader} instance containing all + the I{EDNS} specific attributes and options will be appended to the list + of C{additional} records. + + @return: A L{dns.Message} + @rtype: L{dns.Message} + """ + m = self._messageFactory( + id=self.id, + answer=self.answer, + opCode=self.opCode, + auth=self.auth, + trunc=self.trunc, + recDes=self.recDes, + recAv=self.recAv, + # Assign the lower 4 bits to the message + rCode=self.rCode & 0xf, + authenticData=self.authenticData, + checkingDisabled=self.checkingDisabled) + + m.queries = self.queries[:] + m.answers = self.answers[:] + m.authority = self.authority[:] + m.additional = self.additional[:] + + if self.ednsVersion is not None: + o = _OPTHeader(version=self.ednsVersion, + dnssecOK=self.dnssecOK, + udpPayloadSize=self.maxSize, + # Assign the upper 8 bits to the OPT record + extendedRCODE=self.rCode >> 4) + m.additional.append(o) + + return m + + + def toStr(self): + """ + Encode to wire format by first converting to a standard L{dns.Message}. + + @return: A L{bytes} string. + """ + return self._toMessage().toStr() + + + @classmethod + def _fromMessage(cls, message): + """ + Construct and return a new L{_EDNSMessage} whose attributes and records + are derived from the attributes and records of C{message} (a L{Message} + instance). + + If present, an C{OPT} record will be extracted from the C{additional} + section and its attributes and options will be used to set the EDNS + specific attributes C{extendedRCODE}, C{ednsVersion}, C{dnssecOK}, + C{ednsOptions}. + + The C{extendedRCODE} will be combined with C{message.rCode} and assigned + to C{self.rCode}. + + @param message: The source L{Message}. + @type message: L{Message} + + @return: A new L{_EDNSMessage} + @rtype: L{_EDNSMessage} + """ + additional = [] + optRecords = [] + for r in message.additional: + if r.type == OPT: + optRecords.append(_OPTHeader.fromRRHeader(r)) + else: + additional.append(r) + + newMessage = cls( + id=message.id, + answer=message.answer, + opCode=message.opCode, + auth=message.auth, + trunc=message.trunc, + recDes=message.recDes, + recAv=message.recAv, + rCode=message.rCode, + authenticData=message.authenticData, + checkingDisabled=message.checkingDisabled, + # Default to None, it will be updated later when the OPT records are + # parsed. + ednsVersion=None, + dnssecOK=False, + queries=message.queries[:], + answers=message.answers[:], + authority=message.authority[:], + additional=additional, + ) + + if len(optRecords) == 1: + # XXX: If multiple OPT records are received, an EDNS server should + # respond with FORMERR. See ticket:5669#comment:1. + opt = optRecords[0] + newMessage.ednsVersion = opt.version + newMessage.dnssecOK = opt.dnssecOK + newMessage.maxSize = opt.udpPayloadSize + newMessage.rCode = opt.extendedRCODE << 4 | message.rCode + + return newMessage + + + def fromStr(self, bytes): + """ + Decode from wire format, saving flags, values and records to this + L{_EDNSMessage} instance in place. + + @param bytes: The full byte string to be decoded. + @type bytes: L{bytes} + """ + m = self._messageFactory() + m.fromStr(bytes) + + ednsMessage = self._fromMessage(m) + for attrName in self.compareAttributes: + setattr(self, attrName, getattr(ednsMessage, attrName)) + + + +class DNSMixin(object): + """ + DNS protocol mixin shared by UDP and TCP implementations. + + @ivar _reactor: A L{IReactorTime} and L{IReactorUDP} provider which will + be used to issue DNS queries and manage request timeouts. + """ + id = None + liveMessages = None + + def __init__(self, controller, reactor=None): + self.controller = controller + self.id = random.randrange(2 ** 10, 2 ** 15) + if reactor is None: + from twisted.internet import reactor + self._reactor = reactor + + + def pickID(self): + """ + Return a unique ID for queries. + """ + while True: + id = randomSource() + if id not in self.liveMessages: + return id + + + def callLater(self, period, func, *args): + """ + Wrapper around reactor.callLater, mainly for test purpose. + """ + return self._reactor.callLater(period, func, *args) + + + def _query(self, queries, timeout, id, writeMessage): + """ + Send out a message with the given queries. + + @type queries: L{list} of C{Query} instances + @param queries: The queries to transmit + + @type timeout: L{int} or C{float} + @param timeout: How long to wait before giving up + + @type id: L{int} + @param id: Unique key for this request + + @type writeMessage: C{callable} + @param writeMessage: One-parameter callback which writes the message + + @rtype: C{Deferred} + @return: a C{Deferred} which will be fired with the result of the + query, or errbacked with any errors that could happen (exceptions + during writing of the query, timeout errors, ...). + """ + m = Message(id, recDes=1) + m.queries = queries + + try: + writeMessage(m) + except: + return defer.fail() + + resultDeferred = defer.Deferred() + cancelCall = self.callLater(timeout, self._clearFailed, resultDeferred, id) + self.liveMessages[id] = (resultDeferred, cancelCall) + + return resultDeferred + + def _clearFailed(self, deferred, id): + """ + Clean the Deferred after a timeout. + """ + try: + del self.liveMessages[id] + except KeyError: + pass + deferred.errback(failure.Failure(DNSQueryTimeoutError(id))) + + +class DNSDatagramProtocol(DNSMixin, protocol.DatagramProtocol): + """ + DNS protocol over UDP. + """ + resends = None + + def stopProtocol(self): + """ + Stop protocol: reset state variables. + """ + self.liveMessages = {} + self.resends = {} + self.transport = None + + def startProtocol(self): + """ + Upon start, reset internal state. + """ + self.liveMessages = {} + self.resends = {} + + def writeMessage(self, message, address): + """ + Send a message holding DNS queries. + + @type message: L{Message} + """ + self.transport.write(message.toStr(), address) + + def startListening(self): + self._reactor.listenUDP(0, self, maxPacketSize=512) + + def datagramReceived(self, data, addr): + """ + Read a datagram, extract the message in it and trigger the associated + Deferred. + """ + m = Message() + try: + m.fromStr(data) + except EOFError: + log.msg("Truncated packet (%d bytes) from %s" % (len(data), addr)) + return + except: + # Nothing should trigger this, but since we're potentially + # invoking a lot of different decoding methods, we might as well + # be extra cautious. Anything that triggers this is itself + # buggy. + log.err(failure.Failure(), "Unexpected decoding error") + return + + if m.id in self.liveMessages: + d, canceller = self.liveMessages[m.id] + del self.liveMessages[m.id] + canceller.cancel() + # XXX we shouldn't need this hack of catching exception on callback() + try: + d.callback(m) + except: + log.err() + else: + if m.id not in self.resends: + self.controller.messageReceived(m, self, addr) + + + def removeResend(self, id): + """ + Mark message ID as no longer having duplication suppression. + """ + try: + del self.resends[id] + except KeyError: + pass + + def query(self, address, queries, timeout=10, id=None): + """ + Send out a message with the given queries. + + @type address: L{tuple} of L{str} and L{int} + @param address: The address to which to send the query + + @type queries: L{list} of C{Query} instances + @param queries: The queries to transmit + + @rtype: C{Deferred} + """ + if not self.transport: + # XXX transport might not get created automatically, use callLater? + try: + self.startListening() + except CannotListenError: + return defer.fail() + + if id is None: + id = self.pickID() + else: + self.resends[id] = 1 + + def writeMessage(m): + self.writeMessage(m, address) + + return self._query(queries, timeout, id, writeMessage) + + +class DNSProtocol(DNSMixin, protocol.Protocol): + """ + DNS protocol over TCP. + """ + length = None + buffer = b'' + + def writeMessage(self, message): + """ + Send a message holding DNS queries. + + @type message: L{Message} + """ + s = message.toStr() + self.transport.write(struct.pack('!H', len(s)) + s) + + def connectionMade(self): + """ + Connection is made: reset internal state, and notify the controller. + """ + self.liveMessages = {} + self.controller.connectionMade(self) + + + def connectionLost(self, reason): + """ + Notify the controller that this protocol is no longer + connected. + """ + self.controller.connectionLost(self) + + + def dataReceived(self, data): + self.buffer += data + + while self.buffer: + if self.length is None and len(self.buffer) >= 2: + self.length = struct.unpack('!H', self.buffer[:2])[0] + self.buffer = self.buffer[2:] + + if len(self.buffer) >= self.length: + myChunk = self.buffer[:self.length] + m = Message() + m.fromStr(myChunk) + + try: + d, canceller = self.liveMessages[m.id] + except KeyError: + self.controller.messageReceived(m, self) + else: + del self.liveMessages[m.id] + canceller.cancel() + # XXX we shouldn't need this hack + try: + d.callback(m) + except: + log.err() + + self.buffer = self.buffer[self.length:] + self.length = None + else: + break + + + def query(self, queries, timeout=60): + """ + Send out a message with the given queries. + + @type queries: L{list} of C{Query} instances + @param queries: The queries to transmit + + @rtype: C{Deferred} + """ + id = self.pickID() + return self._query(queries, timeout, id, self.writeMessage) diff --git a/contrib/python/Twisted/py2/twisted/names/error.py b/contrib/python/Twisted/py2/twisted/names/error.py new file mode 100644 index 00000000000..92a076b26e1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/error.py @@ -0,0 +1,97 @@ +# -*- test-case-name: twisted.names.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Exception class definitions for Twisted Names. +""" + +from __future__ import division, absolute_import + +from twisted.internet.defer import TimeoutError + + +class DomainError(ValueError): + """ + Indicates a lookup failed because there were no records matching the given + C{name, class, type} triple. + """ + + + +class AuthoritativeDomainError(ValueError): + """ + Indicates a lookup failed for a name for which this server is authoritative + because there were no records matching the given C{name, class, type} + triple. + """ + + + +class DNSQueryTimeoutError(TimeoutError): + """ + Indicates a lookup failed due to a timeout. + + @ivar id: The id of the message which timed out. + """ + def __init__(self, id): + TimeoutError.__init__(self) + self.id = id + + + +class DNSFormatError(DomainError): + """ + Indicates a query failed with a result of C{twisted.names.dns.EFORMAT}. + """ + + + +class DNSServerError(DomainError): + """ + Indicates a query failed with a result of C{twisted.names.dns.ESERVER}. + """ + + + +class DNSNameError(DomainError): + """ + Indicates a query failed with a result of C{twisted.names.dns.ENAME}. + """ + + + +class DNSNotImplementedError(DomainError): + """ + Indicates a query failed with a result of C{twisted.names.dns.ENOTIMP}. + """ + + + +class DNSQueryRefusedError(DomainError): + """ + Indicates a query failed with a result of C{twisted.names.dns.EREFUSED}. + """ + + + +class DNSUnknownError(DomainError): + """ + Indicates a query failed with an unknown result. + """ + + + +class ResolverError(Exception): + """ + Indicates a query failed because of a decision made by the local + resolver object. + """ + + +__all__ = [ + 'DomainError', 'AuthoritativeDomainError', 'DNSQueryTimeoutError', + + 'DNSFormatError', 'DNSServerError', 'DNSNameError', + 'DNSNotImplementedError', 'DNSQueryRefusedError', + 'DNSUnknownError', 'ResolverError'] diff --git a/contrib/python/Twisted/py2/twisted/names/hosts.py b/contrib/python/Twisted/py2/twisted/names/hosts.py new file mode 100644 index 00000000000..6673eac63ed --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/hosts.py @@ -0,0 +1,153 @@ +# -*- test-case-name: twisted.names.test.test_hosts -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +hosts(5) support. +""" + +from __future__ import division, absolute_import + +from twisted.python.compat import nativeString +from twisted.names import dns +from twisted.python import failure +from twisted.python.filepath import FilePath +from twisted.internet import defer +from twisted.internet.abstract import isIPAddress + +from twisted.names import common + +def searchFileForAll(hostsFile, name): + """ + Search the given file, which is in hosts(5) standard format, for an address + entry with a given name. + + @param hostsFile: The name of the hosts(5)-format file to search. + @type hostsFile: L{FilePath} + + @param name: The name to search for. + @type name: C{bytes} + + @return: L{None} if the name is not found in the file, otherwise a + C{str} giving the address in the file associated with the name. + """ + results = [] + try: + lines = hostsFile.getContent().splitlines() + except: + return results + + name = name.lower() + for line in lines: + idx = line.find(b'#') + if idx != -1: + line = line[:idx] + if not line: + continue + parts = line.split() + + if name.lower() in [s.lower() for s in parts[1:]]: + results.append(nativeString(parts[0])) + return results + + + +def searchFileFor(file, name): + """ + Grep given file, which is in hosts(5) standard format, for an address + entry with a given name. + + @param file: The name of the hosts(5)-format file to search. + @type file: C{str} or C{bytes} + + @param name: The name to search for. + @type name: C{bytes} + + @return: L{None} if the name is not found in the file, otherwise a + C{str} giving the first address in the file associated with + the name. + """ + addresses = searchFileForAll(FilePath(file), name) + if addresses: + return addresses[0] + return None + + + +class Resolver(common.ResolverBase): + """ + A resolver that services hosts(5) format files. + """ + def __init__(self, file=b'/etc/hosts', ttl = 60 * 60): + common.ResolverBase.__init__(self) + self.file = file + self.ttl = ttl + + + def _aRecords(self, name): + """ + Return a tuple of L{dns.RRHeader} instances for all of the IPv4 + addresses in the hosts file. + """ + return tuple([ + dns.RRHeader(name, dns.A, dns.IN, self.ttl, + dns.Record_A(addr, self.ttl)) + for addr + in searchFileForAll(FilePath(self.file), name) + if isIPAddress(addr)]) + + + def _aaaaRecords(self, name): + """ + Return a tuple of L{dns.RRHeader} instances for all of the IPv6 + addresses in the hosts file. + """ + return tuple([ + dns.RRHeader(name, dns.AAAA, dns.IN, self.ttl, + dns.Record_AAAA(addr, self.ttl)) + for addr + in searchFileForAll(FilePath(self.file), name) + if not isIPAddress(addr)]) + + + def _respond(self, name, records): + """ + Generate a response for the given name containing the given result + records, or a failure if there are no result records. + + @param name: The DNS name the response is for. + @type name: C{str} + + @param records: A tuple of L{dns.RRHeader} instances giving the results + that will go into the response. + + @return: A L{Deferred} which will fire with a three-tuple of result + records, authority records, and additional records, or which will + fail with L{dns.DomainError} if there are no result records. + """ + if records: + return defer.succeed((records, (), ())) + return defer.fail(failure.Failure(dns.DomainError(name))) + + + def lookupAddress(self, name, timeout=None): + """ + Read any IPv4 addresses from C{self.file} and return them as + L{Record_A} instances. + """ + name = dns.domainString(name) + return self._respond(name, self._aRecords(name)) + + + def lookupIPV6Address(self, name, timeout=None): + """ + Read any IPv6 addresses from C{self.file} and return them as + L{Record_AAAA} instances. + """ + name = dns.domainString(name) + return self._respond(name, self._aaaaRecords(name)) + + # Someday this should include IPv6 addresses too, but that will cause + # problems if users of the API (mainly via getHostByName) aren't updated to + # know about IPv6 first. + lookupAllRecords = lookupAddress diff --git a/contrib/python/Twisted/py2/twisted/names/resolve.py b/contrib/python/Twisted/py2/twisted/names/resolve.py new file mode 100644 index 00000000000..0cd39eb1c3e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/resolve.py @@ -0,0 +1,99 @@ +# -*- test-case-name: twisted.names.test.test_resolve -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Lookup a name using multiple resolvers. + +Future Plans: This needs someway to specify which resolver answered +the query, or someway to specify (authority|ttl|cache behavior|more?) +""" + +from __future__ import division, absolute_import + +from zope.interface import implementer + +from twisted.internet import defer, interfaces +from twisted.names import dns, common, error + + +class FailureHandler: + def __init__(self, resolver, query, timeout): + self.resolver = resolver + self.query = query + self.timeout = timeout + + + def __call__(self, failure): + # AuthoritativeDomainErrors should halt resolution attempts + failure.trap(dns.DomainError, defer.TimeoutError, NotImplementedError) + return self.resolver(self.query, self.timeout) + + + +@implementer(interfaces.IResolver) +class ResolverChain(common.ResolverBase): + """ + Lookup an address using multiple L{IResolver}s + """ + def __init__(self, resolvers): + """ + @type resolvers: L{list} + @param resolvers: A L{list} of L{IResolver} providers. + """ + common.ResolverBase.__init__(self) + self.resolvers = resolvers + + + def _lookup(self, name, cls, type, timeout): + """ + Build a L{dns.Query} for the given parameters and dispatch it + to each L{IResolver} in C{self.resolvers} until an answer or + L{error.AuthoritativeDomainError} is returned. + + @type name: C{str} + @param name: DNS name to resolve. + + @type type: C{int} + @param type: DNS record type. + + @type cls: C{int} + @param cls: DNS record class. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + if not self.resolvers: + return defer.fail(error.DomainError()) + q = dns.Query(name, type, cls) + d = self.resolvers[0].query(q, timeout) + for r in self.resolvers[1:]: + d = d.addErrback( + FailureHandler(r.query, q, timeout) + ) + return d + + + def lookupAllRecords(self, name, timeout=None): + # XXX: Why is this necessary? dns.ALL_RECORDS queries should + # be handled just the same as any other type by _lookup + # above. If I remove this method all names tests still + # pass. See #6604 -rwall + if not self.resolvers: + return defer.fail(error.DomainError()) + d = self.resolvers[0].lookupAllRecords(name, timeout) + for r in self.resolvers[1:]: + d = d.addErrback( + FailureHandler(r.lookupAllRecords, name, timeout) + ) + return d diff --git a/contrib/python/Twisted/py2/twisted/names/root.py b/contrib/python/Twisted/py2/twisted/names/root.py new file mode 100644 index 00000000000..5b942541096 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/root.py @@ -0,0 +1,333 @@ +# -*- test-case-name: twisted.names.test.test_rootresolve -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Resolver implementation for querying successive authoritative servers to +lookup a record, starting from the root nameservers. + +@author: Jp Calderone + +todo:: + robustify it + documentation +""" + +from twisted.python.failure import Failure +from twisted.internet import defer +from twisted.names import dns, common, error + + + +class _DummyController: + """ + A do-nothing DNS controller. This is useful when all messages received + will be responses to previously issued queries. Anything else received + will be ignored. + """ + def messageReceived(self, *args): + pass + + + +class Resolver(common.ResolverBase): + """ + L{Resolver} implements recursive lookup starting from a specified list of + root servers. + + @ivar hints: See C{hints} parameter of L{__init__} + @ivar _maximumQueries: See C{maximumQueries} parameter of L{__init__} + @ivar _reactor: See C{reactor} parameter of L{__init__} + @ivar _resolverFactory: See C{resolverFactory} parameter of L{__init__} + """ + def __init__(self, hints, maximumQueries=10, + reactor=None, resolverFactory=None): + """ + @param hints: A L{list} of L{str} giving the dotted quad + representation of IP addresses of root servers at which to + begin resolving names. + @type hints: L{list} of L{str} + + @param maximumQueries: An optional L{int} giving the maximum + number of queries which will be attempted to resolve a + single name. + @type maximumQueries: L{int} + + @param reactor: An optional L{IReactorTime} and L{IReactorUDP} + provider to use to bind UDP ports and manage timeouts. + @type reactor: L{IReactorTime} and L{IReactorUDP} provider + + @param resolverFactory: An optional callable which accepts C{reactor} + and C{servers} arguments and returns an instance that provides a + C{queryUDP} method. Defaults to L{twisted.names.client.Resolver}. + @type resolverFactory: callable + """ + common.ResolverBase.__init__(self) + self.hints = hints + self._maximumQueries = maximumQueries + self._reactor = reactor + if resolverFactory is None: + from twisted.names.client import Resolver as resolverFactory + self._resolverFactory = resolverFactory + + + def _roots(self): + """ + Return a list of two-tuples representing the addresses of the root + servers, as defined by C{self.hints}. + """ + return [(ip, dns.PORT) for ip in self.hints] + + + def _query(self, query, servers, timeout, filter): + """ + Issue one query and return a L{Deferred} which fires with its response. + + @param query: The query to issue. + @type query: L{dns.Query} + + @param servers: The servers which might have an answer for this + query. + @type servers: L{list} of L{tuple} of L{str} and L{int} + + @param timeout: A timeout on how long to wait for the response. + @type timeout: L{tuple} of L{int} + + @param filter: A flag indicating whether to filter the results. If + C{True}, the returned L{Deferred} will fire with a three-tuple of + lists of L{twisted.names.dns.RRHeader} (like the return value of + the I{lookup*} methods of L{IResolver}. IF C{False}, the result + will be a L{Message} instance. + @type filter: L{bool} + + @return: A L{Deferred} which fires with the response or a timeout + error. + @rtype: L{Deferred} + """ + r = self._resolverFactory(servers=servers, reactor=self._reactor) + d = r.queryUDP([query], timeout) + if filter: + d.addCallback(r.filterAnswers) + return d + + + def _lookup(self, name, cls, type, timeout): + """ + Implement name lookup by recursively discovering the authoritative + server for the name and then asking it, starting at one of the servers + in C{self.hints}. + """ + if timeout is None: + # A series of timeouts for semi-exponential backoff, summing to an + # arbitrary total of 60 seconds. + timeout = (1, 3, 11, 45) + return self._discoverAuthority( + dns.Query(name, type, cls), self._roots(), timeout, + self._maximumQueries) + + + def _discoverAuthority(self, query, servers, timeout, queriesLeft): + """ + Issue a query to a server and follow a delegation if necessary. + + @param query: The query to issue. + @type query: L{dns.Query} + + @param servers: The servers which might have an answer for this + query. + @type servers: L{list} of L{tuple} of L{str} and L{int} + + @param timeout: A C{tuple} of C{int} giving the timeout to use for this + query. + + @param queriesLeft: A C{int} giving the number of queries which may + yet be attempted to answer this query before the attempt will be + abandoned. + + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} giving the response, or with a + L{Failure} if there is a timeout or response error. + """ + # Stop now if we've hit the query limit. + if queriesLeft <= 0: + return Failure( + error.ResolverError("Query limit reached without result")) + + d = self._query(query, servers, timeout, False) + d.addCallback( + self._discoveredAuthority, query, timeout, queriesLeft - 1) + return d + + + def _discoveredAuthority(self, response, query, timeout, queriesLeft): + """ + Interpret the response to a query, checking for error codes and + following delegations if necessary. + + @param response: The L{Message} received in response to issuing C{query}. + @type response: L{Message} + + @param query: The L{dns.Query} which was issued. + @type query: L{dns.Query}. + + @param timeout: The timeout to use if another query is indicated by + this response. + @type timeout: L{tuple} of L{int} + + @param queriesLeft: A C{int} giving the number of queries which may + yet be attempted to answer this query before the attempt will be + abandoned. + + @return: A L{Failure} indicating a response error, a three-tuple of + lists of L{twisted.names.dns.RRHeader} giving the response to + C{query} or a L{Deferred} which will fire with one of those. + """ + if response.rCode != dns.OK: + return Failure(self.exceptionForCode(response.rCode)(response)) + + # Turn the answers into a structure that's a little easier to work with. + records = {} + for answer in response.answers: + records.setdefault(answer.name, []).append(answer) + + def findAnswerOrCName(name, type, cls): + cname = None + for record in records.get(name, []): + if record.cls == cls: + if record.type == type: + return record + elif record.type == dns.CNAME: + cname = record + # If there were any CNAME records, return the last one. There's + # only supposed to be zero or one, though. + return cname + + seen = set() + name = query.name + record = None + while True: + seen.add(name) + previous = record + record = findAnswerOrCName(name, query.type, query.cls) + if record is None: + if name == query.name: + # If there's no answer for the original name, then this may + # be a delegation. Code below handles it. + break + else: + # Try to resolve the CNAME with another query. + d = self._discoverAuthority( + dns.Query(str(name), query.type, query.cls), + self._roots(), timeout, queriesLeft) + # We also want to include the CNAME in the ultimate result, + # otherwise this will be pretty confusing. + def cbResolved(results): + answers, authority, additional = results + answers.insert(0, previous) + return (answers, authority, additional) + d.addCallback(cbResolved) + return d + elif record.type == query.type: + return ( + response.answers, + response.authority, + response.additional) + else: + # It's a CNAME record. Try to resolve it from the records + # in this response with another iteration around the loop. + if record.payload.name in seen: + raise error.ResolverError("Cycle in CNAME processing") + name = record.payload.name + + + # Build a map to use to convert NS names into IP addresses. + addresses = {} + for rr in response.additional: + if rr.type == dns.A: + addresses[rr.name.name] = rr.payload.dottedQuad() + + hints = [] + traps = [] + for rr in response.authority: + if rr.type == dns.NS: + ns = rr.payload.name.name + if ns in addresses: + hints.append((addresses[ns], dns.PORT)) + else: + traps.append(ns) + if hints: + return self._discoverAuthority( + query, hints, timeout, queriesLeft) + elif traps: + d = self.lookupAddress(traps[0], timeout) + def getOneAddress(results): + answers, authority, additional = results + return answers[0].payload.dottedQuad() + d.addCallback(getOneAddress) + d.addCallback( + lambda hint: self._discoverAuthority( + query, [(hint, dns.PORT)], timeout, queriesLeft - 1)) + return d + else: + return Failure(error.ResolverError( + "Stuck at response without answers or delegation")) + + + +def makePlaceholder(deferred, name): + def placeholder(*args, **kw): + deferred.addCallback(lambda r: getattr(r, name)(*args, **kw)) + return deferred + return placeholder + +class DeferredResolver: + def __init__(self, resolverDeferred): + self.waiting = [] + resolverDeferred.addCallback(self.gotRealResolver) + + def gotRealResolver(self, resolver): + w = self.waiting + self.__dict__ = resolver.__dict__ + self.__class__ = resolver.__class__ + for d in w: + d.callback(resolver) + + def __getattr__(self, name): + if name.startswith('lookup') or name in ('getHostByName', 'query'): + self.waiting.append(defer.Deferred()) + return makePlaceholder(self.waiting[-1], name) + raise AttributeError(name) + + + +def bootstrap(resolver, resolverFactory=None): + """ + Lookup the root nameserver addresses using the given resolver + + Return a Resolver which will eventually become a C{root.Resolver} + instance that has references to all the root servers that we were able + to look up. + + @param resolver: The resolver instance which will be used to + lookup the root nameserver addresses. + @type resolver: L{twisted.internet.interfaces.IResolverSimple} + + @param resolverFactory: An optional callable which returns a + resolver instance. It will passed as the C{resolverFactory} + argument to L{Resolver.__init__}. + @type resolverFactory: callable + + @return: A L{DeferredResolver} which will be dynamically replaced + with L{Resolver} when the root nameservers have been looked up. + """ + domains = [chr(ord('a') + i) for i in range(13)] + L = [resolver.getHostByName('%s.root-servers.net' % d) for d in domains] + d = defer.DeferredList(L, consumeErrors=True) + + def buildResolver(res): + return Resolver( + hints=[e[1] for e in res if e[0]], + resolverFactory=resolverFactory) + d.addCallback(buildResolver) + + return DeferredResolver(d) diff --git a/contrib/python/Twisted/py2/twisted/names/secondary.py b/contrib/python/Twisted/py2/twisted/names/secondary.py new file mode 100644 index 00000000000..ba73e1b7575 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/secondary.py @@ -0,0 +1,221 @@ +# -*- test-case-name: twisted.names.test.test_names -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division + +__all__ = ['SecondaryAuthority', 'SecondaryAuthorityService'] + +from twisted.internet import task, defer +from twisted.names import dns +from twisted.names import common +from twisted.names import client +from twisted.names import resolve +from twisted.names.authority import FileAuthority + +from twisted.python import log, failure +from twisted.python.compat import nativeString +from twisted.application import service + + + +class SecondaryAuthorityService(service.Service): + """ + A service that keeps one or more authorities up to date by doing hourly + zone transfers from a master. + + @ivar primary: IP address of the master. + @type primary: L{str} + + @ivar domains: An authority for each domain mirrored from the master. + @type domains: L{list} of L{SecondaryAuthority} + """ + calls = None + + _port = 53 + + def __init__(self, primary, domains): + """ + @param primary: The IP address of the server from which to perform + zone transfers. + @type primary: L{str} + + @param domains: A sequence of domain names for which to perform + zone transfers. + @type domains: L{list} of L{bytes} + """ + self.primary = nativeString(primary) + self.domains = [SecondaryAuthority(primary, d) for d in domains] + + + @classmethod + def fromServerAddressAndDomains(cls, serverAddress, domains): + """ + Construct a new L{SecondaryAuthorityService} from a tuple giving a + server address and a C{str} giving the name of a domain for which this + is an authority. + + @param serverAddress: A two-tuple, the first element of which is a + C{str} giving an IP address and the second element of which is a + C{int} giving a port number. Together, these define where zone + transfers will be attempted from. + + @param domain: A C{bytes} giving the domain to transfer. + + @return: A new instance of L{SecondaryAuthorityService}. + """ + primary, port = serverAddress + service = cls(primary, []) + service._port = port + service.domains = [ + SecondaryAuthority.fromServerAddressAndDomain(serverAddress, d) + for d in domains] + return service + + + def getAuthority(self): + """ + Get a resolver for the transferred domains. + + @rtype: L{ResolverChain} + """ + return resolve.ResolverChain(self.domains) + + def startService(self): + service.Service.startService(self) + self.calls = [task.LoopingCall(d.transfer) for d in self.domains] + i = 0 + from twisted.internet import reactor + for c in self.calls: + # XXX Add errbacks, respect proper timeouts + reactor.callLater(i, c.start, 60 * 60) + i += 1 + + def stopService(self): + service.Service.stopService(self) + for c in self.calls: + c.stop() + + + +class SecondaryAuthority(FileAuthority): + """ + An Authority that keeps itself updated by performing zone transfers. + + @ivar primary: The IP address of the server from which zone transfers will + be attempted. + @type primary: C{str} + + @ivar _port: The port number of the server from which zone transfers will + be attempted. + @type: C{int} + + @ivar domain: The domain for which this is the secondary authority. + @type: C{bytes} + + @ivar _reactor: The reactor to use to perform the zone transfers, or + L{None} to use the global reactor. + """ + + transferring = False + soa = records = None + _port = 53 + _reactor = None + + def __init__(self, primaryIP, domain): + """ + @param domain: The domain for which this will be the secondary + authority. + @type domain: L{bytes} or L{str} + """ + # Yep. Skip over FileAuthority.__init__. This is a hack until we have + # a good composition-based API for the complicated DNS record lookup + # logic we want to share. + common.ResolverBase.__init__(self) + self.primary = nativeString(primaryIP) + self.domain = dns.domainString(domain) + + + @classmethod + def fromServerAddressAndDomain(cls, serverAddress, domain): + """ + Construct a new L{SecondaryAuthority} from a tuple giving a server + address and a C{bytes} giving the name of a domain for which this is an + authority. + + @param serverAddress: A two-tuple, the first element of which is a + C{str} giving an IP address and the second element of which is a + C{int} giving a port number. Together, these define where zone + transfers will be attempted from. + + @param domain: A C{bytes} giving the domain to transfer. + @type domain: L{bytes} + + @return: A new instance of L{SecondaryAuthority}. + """ + primary, port = serverAddress + secondary = cls(primary, domain) + secondary._port = port + return secondary + + + def transfer(self): + """ + Attempt a zone transfer. + + @returns: A L{Deferred} that fires with L{None} when attempted zone + transfer has completed. + """ + # FIXME: This logic doesn't avoid duplicate transfers + # https://twistedmatrix.com/trac/ticket/9754 + if self.transferring: # <-- never true + return + self.transfering = True # <-- speling + + reactor = self._reactor + if reactor is None: + from twisted.internet import reactor + + resolver = client.Resolver( + servers=[(self.primary, self._port)], reactor=reactor) + return resolver.lookupZone(self.domain + ).addCallback(self._cbZone + ).addErrback(self._ebZone + ) + + + def _lookup(self, name, cls, type, timeout=None): + if not self.soa or not self.records: + # No transfer has occurred yet. Fail non-authoritatively so that + # the caller can try elsewhere. + return defer.fail(failure.Failure(dns.DomainError(name))) + return FileAuthority._lookup(self, name, cls, type, timeout) + + + def _cbZone(self, zone): + ans, _, _ = zone + self.records = r = {} + for rec in ans: + if not self.soa and rec.type == dns.SOA: + self.soa = (rec.name.name.lower(), rec.payload) + else: + r.setdefault(rec.name.name.lower(), []).append(rec.payload) + + + def _ebZone(self, failure): + log.msg("Updating %s from %s failed during zone transfer" % (self.domain, self.primary)) + log.err(failure) + + + def update(self): + self.transfer().addCallbacks(self._cbTransferred, self._ebTransferred) + + + def _cbTransferred(self, result): + self.transferring = False + + + def _ebTransferred(self, failure): + self.transferred = False + log.msg("Transferring %s from %s failed after zone transfer" % (self.domain, self.primary)) + log.err(failure) diff --git a/contrib/python/Twisted/py2/twisted/names/server.py b/contrib/python/Twisted/py2/twisted/names/server.py new file mode 100644 index 00000000000..893821b95f8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/server.py @@ -0,0 +1,590 @@ +# -*- test-case-name: twisted.names.test.test_names,twisted.names.test.test_server -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Async DNS server + +Future plans: + - Better config file format maybe + - Make sure to differentiate between different classes + - notice truncation bit + +Important: No additional processing is done on some of the record types. +This violates the most basic RFC and is just plain annoying +for resolvers to deal with. Fix it. + +@author: Jp Calderone +""" +from __future__ import division, absolute_import + +import time + +from twisted.internet import protocol +from twisted.names import dns, resolve +from twisted.python import log + + +class DNSServerFactory(protocol.ServerFactory): + """ + Server factory and tracker for L{DNSProtocol} connections. This class also + provides records for responses to DNS queries. + + @ivar cache: A L{Cache} instance whose + C{cacheResult} method is called when a response is received from one of + C{clients}. Defaults to L{None} if no caches are specified. See + C{caches} of L{__init__} for more details. + @type cache: L{Cache} or L{None} + + @ivar canRecurse: A flag indicating whether this server is capable of + performing recursive DNS resolution. + @type canRecurse: L{bool} + + @ivar resolver: A L{resolve.ResolverChain} containing an ordered list of + C{authorities}, C{caches} and C{clients} to which queries will be + dispatched. + @type resolver: L{resolve.ResolverChain} + + @ivar verbose: See L{__init__} + + @ivar connections: A list of all the connected L{DNSProtocol} instances + using this object as their controller. + @type connections: C{list} of L{DNSProtocol} instances + + @ivar protocol: A callable used for building a DNS stream protocol. Called + by L{DNSServerFactory.buildProtocol} and passed the L{DNSServerFactory} + instance as the one and only positional argument. Defaults to + L{dns.DNSProtocol}. + @type protocol: L{IProtocolFactory} constructor + + @ivar _messageFactory: A response message constructor with an initializer + signature matching L{dns.Message.__init__}. + @type _messageFactory: C{callable} + """ + + protocol = dns.DNSProtocol + cache = None + _messageFactory = dns.Message + + + def __init__(self, authorities=None, caches=None, clients=None, verbose=0): + """ + @param authorities: Resolvers which provide authoritative answers. + @type authorities: L{list} of L{IResolver} providers + + @param caches: Resolvers which provide cached non-authoritative + answers. The first cache instance is assigned to + C{DNSServerFactory.cache} and its C{cacheResult} method will be + called when a response is received from one of C{clients}. + @type caches: L{list} of L{Cache} instances + + @param clients: Resolvers which are capable of performing recursive DNS + lookups. + @type clients: L{list} of L{IResolver} providers + + @param verbose: An integer controlling the verbosity of logging of + queries and responses. Default is C{0} which means no logging. Set + to C{2} to enable logging of full query and response messages. + @type verbose: L{int} + """ + resolvers = [] + if authorities is not None: + resolvers.extend(authorities) + if caches is not None: + resolvers.extend(caches) + if clients is not None: + resolvers.extend(clients) + + self.canRecurse = not not clients + self.resolver = resolve.ResolverChain(resolvers) + self.verbose = verbose + if caches: + self.cache = caches[-1] + self.connections = [] + + + def _verboseLog(self, *args, **kwargs): + """ + Log a message only if verbose logging is enabled. + + @param args: Positional arguments which will be passed to C{log.msg} + @param kwargs: Keyword arguments which will be passed to C{log.msg} + """ + if self.verbose > 0: + log.msg(*args, **kwargs) + + + def buildProtocol(self, addr): + p = self.protocol(self) + p.factory = self + return p + + + def connectionMade(self, protocol): + """ + Track a newly connected L{DNSProtocol}. + + @param protocol: The protocol instance to be tracked. + @type protocol: L{dns.DNSProtocol} + """ + self.connections.append(protocol) + + + def connectionLost(self, protocol): + """ + Stop tracking a no-longer connected L{DNSProtocol}. + + @param protocol: The tracked protocol instance to be which has been + lost. + @type protocol: L{dns.DNSProtocol} + """ + self.connections.remove(protocol) + + + def sendReply(self, protocol, message, address): + """ + Send a response C{message} to a given C{address} via the supplied + C{protocol}. + + Message payload will be logged if C{DNSServerFactory.verbose} is C{>1}. + + @param protocol: The DNS protocol instance to which to send the message. + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param message: The DNS message to be sent. + @type message: L{dns.Message} + + @param address: The address to which the message will be sent or L{None} + if C{protocol} is a stream protocol. + @type address: L{tuple} or L{None} + """ + if self.verbose > 1: + s = ' '.join([str(a.payload) for a in message.answers]) + auth = ' '.join([str(a.payload) for a in message.authority]) + add = ' '.join([str(a.payload) for a in message.additional]) + if not s: + log.msg("Replying with no answers") + else: + log.msg("Answers are " + s) + log.msg("Authority is " + auth) + log.msg("Additional is " + add) + + if address is None: + protocol.writeMessage(message) + else: + protocol.writeMessage(message, address) + + self._verboseLog( + "Processed query in %0.3f seconds" % ( + time.time() - message.timeReceived)) + + + def _responseFromMessage(self, message, rCode=dns.OK, + answers=None, authority=None, additional=None): + """ + Generate a L{Message} instance suitable for use as the response to + C{message}. + + C{queries} will be copied from the request to the response. + + C{rCode}, C{answers}, C{authority} and C{additional} will be assigned to + the response, if supplied. + + The C{recAv} flag will be set on the response if the C{canRecurse} flag + on this L{DNSServerFactory} is set to L{True}. + + The C{auth} flag will be set on the response if *any* of the supplied + C{answers} have their C{auth} flag set to L{True}. + + The response will have the same C{maxSize} as the request. + + Additionally, the response will have a C{timeReceived} attribute whose + value is that of the original request and the + + @see: L{dns._responseFromMessage} + + @param message: The request message + @type message: L{Message} + + @param rCode: The response code which will be assigned to the response. + @type message: L{int} + + @param answers: An optional list of answer records which will be + assigned to the response. + @type answers: L{list} of L{dns.RRHeader} + + @param authority: An optional list of authority records which will be + assigned to the response. + @type authority: L{list} of L{dns.RRHeader} + + @param additional: An optional list of additional records which will be + assigned to the response. + @type additional: L{list} of L{dns.RRHeader} + + @return: A response L{Message} instance. + @rtype: L{Message} + """ + if answers is None: + answers = [] + if authority is None: + authority = [] + if additional is None: + additional = [] + authoritativeAnswer = False + for x in answers: + if x.isAuthoritative(): + authoritativeAnswer = True + break + + response = dns._responseFromMessage( + responseConstructor=self._messageFactory, + message=message, + recAv=self.canRecurse, + rCode=rCode, + auth=authoritativeAnswer + ) + + # XXX: Timereceived is a hack which probably shouldn't be tacked onto + # the message. Use getattr here so that we don't have to set the + # timereceived on every message in the tests. See #6957. + response.timeReceived = getattr(message, 'timeReceived', None) + + # XXX: This is another hack. dns.Message.decode sets maxSize=0 which + # means that responses are never truncated. I'll maintain that behaviour + # here until #6949 is resolved. + response.maxSize = message.maxSize + + response.answers = answers + response.authority = authority + response.additional = additional + + return response + + + def gotResolverResponse(self, response, protocol, message, address): + """ + A callback used by L{DNSServerFactory.handleQuery} for handling the + deferred response from C{self.resolver.query}. + + Constructs a response message by combining the original query message + with the resolved answer, authority and additional records. + + Marks the response message as authoritative if any of the resolved + answers are found to be authoritative. + + The resolved answers count will be logged if C{DNSServerFactory.verbose} + is C{>1}. + + @param response: Answer records, authority records and additional records + @type response: L{tuple} of L{list} of L{dns.RRHeader} instances + + @param protocol: The DNS protocol instance to which to send a response + message. + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param message: The original DNS query message for which a response + message will be constructed. + @type message: L{dns.Message} + + @param address: The address to which the response message will be sent + or L{None} if C{protocol} is a stream protocol. + @type address: L{tuple} or L{None} + """ + ans, auth, add = response + response = self._responseFromMessage( + message=message, rCode=dns.OK, + answers=ans, authority=auth, additional=add) + self.sendReply(protocol, response, address) + + l = len(ans) + len(auth) + len(add) + self._verboseLog("Lookup found %d record%s" % (l, l != 1 and "s" or "")) + + if self.cache and l: + self.cache.cacheResult( + message.queries[0], (ans, auth, add) + ) + + + def gotResolverError(self, failure, protocol, message, address): + """ + A callback used by L{DNSServerFactory.handleQuery} for handling deferred + errors from C{self.resolver.query}. + + Constructs a response message from the original query message by + assigning a suitable error code to C{rCode}. + + An error message will be logged if C{DNSServerFactory.verbose} is C{>1}. + + @param failure: The reason for the failed resolution (as reported by + C{self.resolver.query}). + @type failure: L{Failure} + + @param protocol: The DNS protocol instance to which to send a response + message. + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param message: The original DNS query message for which a response + message will be constructed. + @type message: L{dns.Message} + + @param address: The address to which the response message will be sent + or L{None} if C{protocol} is a stream protocol. + @type address: L{tuple} or L{None} + """ + if failure.check(dns.DomainError, dns.AuthoritativeDomainError): + rCode = dns.ENAME + else: + rCode = dns.ESERVER + log.err(failure) + + response = self._responseFromMessage(message=message, rCode=rCode) + + self.sendReply(protocol, response, address) + self._verboseLog("Lookup failed") + + + def handleQuery(self, message, protocol, address): + """ + Called by L{DNSServerFactory.messageReceived} when a query message is + received. + + Takes the first query from the received message and dispatches it to + C{self.resolver.query}. + + Adds callbacks L{DNSServerFactory.gotResolverResponse} and + L{DNSServerFactory.gotResolverError} to the resulting deferred. + + Note: Multiple queries in a single message are not supported because + there is no standard way to respond with multiple rCodes, auth, + etc. This is consistent with other DNS server implementations. See + U{http://tools.ietf.org/html/draft-ietf-dnsext-edns1-03} for a proposed + solution. + + @param protocol: The DNS protocol instance to which to send a response + message. + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param message: The original DNS query message for which a response + message will be constructed. + @type message: L{dns.Message} + + @param address: The address to which the response message will be sent + or L{None} if C{protocol} is a stream protocol. + @type address: L{tuple} or L{None} + + @return: A C{deferred} which fires with the resolved result or error of + the first query in C{message}. + @rtype: L{Deferred} + """ + query = message.queries[0] + + return self.resolver.query(query).addCallback( + self.gotResolverResponse, protocol, message, address + ).addErrback( + self.gotResolverError, protocol, message, address + ) + + + def handleInverseQuery(self, message, protocol, address): + """ + Called by L{DNSServerFactory.messageReceived} when an inverse query + message is received. + + Replies with a I{Not Implemented} error by default. + + An error message will be logged if C{DNSServerFactory.verbose} is C{>1}. + + Override in a subclass. + + @param protocol: The DNS protocol instance to which to send a response + message. + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param message: The original DNS query message for which a response + message will be constructed. + @type message: L{dns.Message} + + @param address: The address to which the response message will be sent + or L{None} if C{protocol} is a stream protocol. + @type address: L{tuple} or L{None} + """ + message.rCode = dns.ENOTIMP + self.sendReply(protocol, message, address) + self._verboseLog("Inverse query from %r" % (address,)) + + + def handleStatus(self, message, protocol, address): + """ + Called by L{DNSServerFactory.messageReceived} when a status message is + received. + + Replies with a I{Not Implemented} error by default. + + An error message will be logged if C{DNSServerFactory.verbose} is C{>1}. + + Override in a subclass. + + @param protocol: The DNS protocol instance to which to send a response + message. + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param message: The original DNS query message for which a response + message will be constructed. + @type message: L{dns.Message} + + @param address: The address to which the response message will be sent + or L{None} if C{protocol} is a stream protocol. + @type address: L{tuple} or L{None} + """ + message.rCode = dns.ENOTIMP + self.sendReply(protocol, message, address) + self._verboseLog("Status request from %r" % (address,)) + + + def handleNotify(self, message, protocol, address): + """ + Called by L{DNSServerFactory.messageReceived} when a notify message is + received. + + Replies with a I{Not Implemented} error by default. + + An error message will be logged if C{DNSServerFactory.verbose} is C{>1}. + + Override in a subclass. + + @param protocol: The DNS protocol instance to which to send a response + message. + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param message: The original DNS query message for which a response + message will be constructed. + @type message: L{dns.Message} + + @param address: The address to which the response message will be sent + or L{None} if C{protocol} is a stream protocol. + @type address: L{tuple} or L{None} + """ + message.rCode = dns.ENOTIMP + self.sendReply(protocol, message, address) + self._verboseLog("Notify message from %r" % (address,)) + + + def handleOther(self, message, protocol, address): + """ + Called by L{DNSServerFactory.messageReceived} when a message with + unrecognised I{OPCODE} is received. + + Replies with a I{Not Implemented} error by default. + + An error message will be logged if C{DNSServerFactory.verbose} is C{>1}. + + Override in a subclass. + + @param protocol: The DNS protocol instance to which to send a response + message. + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param message: The original DNS query message for which a response + message will be constructed. + @type message: L{dns.Message} + + @param address: The address to which the response message will be sent + or L{None} if C{protocol} is a stream protocol. + @type address: L{tuple} or L{None} + """ + message.rCode = dns.ENOTIMP + self.sendReply(protocol, message, address) + self._verboseLog( + "Unknown op code (%d) from %r" % (message.opCode, address)) + + + def messageReceived(self, message, proto, address=None): + """ + L{DNSServerFactory.messageReceived} is called by protocols which are + under the control of this L{DNSServerFactory} whenever they receive a + DNS query message or an unexpected / duplicate / late DNS response + message. + + L{DNSServerFactory.allowQuery} is called with the received message, + protocol and origin address. If it returns L{False}, a C{dns.EREFUSED} + response is sent back to the client. + + Otherwise the received message is dispatched to one of + L{DNSServerFactory.handleQuery}, L{DNSServerFactory.handleInverseQuery}, + L{DNSServerFactory.handleStatus}, L{DNSServerFactory.handleNotify}, or + L{DNSServerFactory.handleOther} depending on the I{OPCODE} of the + received message. + + If C{DNSServerFactory.verbose} is C{>0} all received messages will be + logged in more or less detail depending on the value of C{verbose}. + + @param message: The DNS message that was received. + @type message: L{dns.Message} + + @param proto: The DNS protocol instance which received the message + @type proto: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param address: The address from which the message was received. Only + provided for messages received by datagram protocols. The origin of + Messages received from stream protocols can be gleaned from the + protocol C{transport} attribute. + @type address: L{tuple} or L{None} + """ + message.timeReceived = time.time() + + if self.verbose: + if self.verbose > 1: + s = ' '.join([str(q) for q in message.queries]) + else: + s = ' '.join([dns.QUERY_TYPES.get(q.type, 'UNKNOWN') + for q in message.queries]) + if not len(s): + log.msg( + "Empty query from %r" % ( + (address or proto.transport.getPeer()),)) + else: + log.msg( + "%s query from %r" % ( + s, address or proto.transport.getPeer())) + + if not self.allowQuery(message, proto, address): + message.rCode = dns.EREFUSED + self.sendReply(proto, message, address) + elif message.opCode == dns.OP_QUERY: + self.handleQuery(message, proto, address) + elif message.opCode == dns.OP_INVERSE: + self.handleInverseQuery(message, proto, address) + elif message.opCode == dns.OP_STATUS: + self.handleStatus(message, proto, address) + elif message.opCode == dns.OP_NOTIFY: + self.handleNotify(message, proto, address) + else: + self.handleOther(message, proto, address) + + + def allowQuery(self, message, protocol, address): + """ + Called by L{DNSServerFactory.messageReceived} to decide whether to + process a received message or to reply with C{dns.EREFUSED}. + + This default implementation permits anything but empty queries. + + Override in a subclass to implement alternative policies. + + @param message: The DNS message that was received. + @type message: L{dns.Message} + + @param protocol: The DNS protocol instance which received the message + @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol} + + @param address: The address from which the message was received. Only + provided for messages received by datagram protocols. The origin of + Messages received from stream protocols can be gleaned from the + protocol C{transport} attribute. + @type address: L{tuple} or L{None} + + @return: L{True} if the received message contained one or more queries, + else L{False}. + @rtype: L{bool} + """ + return len(message.queries) diff --git a/contrib/python/Twisted/py2/twisted/names/srvconnect.py b/contrib/python/Twisted/py2/twisted/names/srvconnect.py new file mode 100644 index 00000000000..5346808d01b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/srvconnect.py @@ -0,0 +1,273 @@ +# -*- test-case-name: twisted.names.test.test_srvconnect -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division + +import random + +from zope.interface import implementer + +from twisted.internet import error, interfaces +from twisted.names import client, dns +from twisted.names.error import DNSNameError +from twisted.python.compat import nativeString + + + +class _SRVConnector_ClientFactoryWrapper: + def __init__(self, connector, wrappedFactory): + self.__connector = connector + self.__wrappedFactory = wrappedFactory + + + def startedConnecting(self, connector): + self.__wrappedFactory.startedConnecting(self.__connector) + + + def clientConnectionFailed(self, connector, reason): + self.__connector.connectionFailed(reason) + + + def clientConnectionLost(self, connector, reason): + self.__connector.connectionLost(reason) + + + def __getattr__(self, key): + return getattr(self.__wrappedFactory, key) + + + +@implementer(interfaces.IConnector) +class SRVConnector: + """ + A connector that looks up DNS SRV records. + + RFC 2782 details how SRV records should be interpreted and selected + for subsequent connection attempts. The algorithm for using the records' + priority and weight is implemented in L{pickServer}. + + @ivar servers: List of candidate server records for future connection + attempts. + @type servers: L{list} of L{dns.Record_SRV} + + @ivar orderedServers: List of server records that have already been tried + in this round of connection attempts. + @type orderedServers: L{list} of L{dns.Record_SRV} + """ + + stopAfterDNS = 0 + + def __init__(self, reactor, service, domain, factory, + protocol='tcp', connectFuncName='connectTCP', + connectFuncArgs=(), + connectFuncKwArgs={}, + defaultPort=None, + ): + """ + @param domain: The domain to connect to. If passed as a text + string, it will be encoded using C{idna} encoding. + @type domain: L{bytes} or L{str} + + @param defaultPort: Optional default port number to be used when SRV + lookup fails and the service name is unknown. This should be the + port number associated with the service name as defined by the IANA + registry. + @type defaultPort: L{int} + """ + self.reactor = reactor + self.service = service + self.domain = None if domain is None else dns.domainString(domain) + self.factory = factory + + self.protocol = protocol + self.connectFuncName = connectFuncName + self.connectFuncArgs = connectFuncArgs + self.connectFuncKwArgs = connectFuncKwArgs + self._defaultPort = defaultPort + + self.connector = None + self.servers = None + # list of servers already used in this round: + self.orderedServers = None + + + def connect(self): + """Start connection to remote server.""" + self.factory.doStart() + self.factory.startedConnecting(self) + + if not self.servers: + if self.domain is None: + self.connectionFailed( + error.DNSLookupError("Domain is not defined."), + ) + return + d = client.lookupService('_%s._%s.%s' % ( + nativeString(self.service), + nativeString(self.protocol), + nativeString(self.domain)), + ) + d.addCallbacks(self._cbGotServers, self._ebGotServers) + d.addCallback(lambda x, self=self: self._reallyConnect()) + if self._defaultPort: + d.addErrback(self._ebServiceUnknown) + d.addErrback(self.connectionFailed) + elif self.connector is None: + self._reallyConnect() + else: + self.connector.connect() + + + def _ebGotServers(self, failure): + failure.trap(DNSNameError) + + # Some DNS servers reply with NXDOMAIN when in fact there are + # just no SRV records for that domain. Act as if we just got an + # empty response and use fallback. + + self.servers = [] + self.orderedServers = [] + + + def _cbGotServers(self, result): + answers, auth, add = result + if len(answers) == 1 and answers[0].type == dns.SRV \ + and answers[0].payload \ + and answers[0].payload.target == dns.Name(b'.'): + # decidedly not available + raise error.DNSLookupError("Service %s not available for domain %s." + % (repr(self.service), repr(self.domain))) + + self.servers = [] + self.orderedServers = [] + for a in answers: + if a.type != dns.SRV or not a.payload: + continue + + self.orderedServers.append(a.payload) + + + def _ebServiceUnknown(self, failure): + """ + Connect to the default port when the service name is unknown. + + If no SRV records were found, the service name will be passed as the + port. If resolving the name fails with + L{error.ServiceNameUnknownError}, a final attempt is done using the + default port. + """ + failure.trap(error.ServiceNameUnknownError) + self.servers = [dns.Record_SRV(0, 0, self._defaultPort, self.domain)] + self.orderedServers = [] + self.connect() + + + def pickServer(self): + """ + Pick the next server. + + This selects the next server from the list of SRV records according + to their priority and weight values, as set out by the default + algorithm specified in RFC 2782. + + At the beginning of a round, L{servers} is populated with + L{orderedServers}, and the latter is made empty. L{servers} + is the list of candidates, and L{orderedServers} is the list of servers + that have already been tried. + + First, all records are ordered by priority and weight in ascending + order. Then for each priority level, a running sum is calculated + over the sorted list of records for that priority. Then a random value + between 0 and the final sum is compared to each record in order. The + first record that is greater than or equal to that random value is + chosen and removed from the list of candidates for this round. + + @return: A tuple of target hostname and port from the chosen DNS SRV + record. + @rtype: L{tuple} of native L{str} and L{int} + """ + assert self.servers is not None + assert self.orderedServers is not None + + if not self.servers and not self.orderedServers: + # no SRV record, fall back.. + return nativeString(self.domain), self.service + + if not self.servers and self.orderedServers: + # start new round + self.servers = self.orderedServers + self.orderedServers = [] + + assert self.servers + + self.servers.sort(key=lambda record: (record.priority, record.weight)) + minPriority = self.servers[0].priority + + index = 0 + weightSum = 0 + weightIndex = [] + for x in self.servers: + if x.priority == minPriority: + weightSum += x.weight + weightIndex.append((index, weightSum)) + index += 1 + + rand = random.randint(0, weightSum) + for index, weight in weightIndex: + if weight >= rand: + chosen = self.servers[index] + del self.servers[index] + self.orderedServers.append(chosen) + + return str(chosen.target), chosen.port + + raise RuntimeError( + 'Impossible %s pickServer result.' % (self.__class__.__name__,)) + + + def _reallyConnect(self): + if self.stopAfterDNS: + self.stopAfterDNS = 0 + return + + self.host, self.port = self.pickServer() + assert self.host is not None, 'Must have a host to connect to.' + assert self.port is not None, 'Must have a port to connect to.' + + connectFunc = getattr(self.reactor, self.connectFuncName) + self.connector = connectFunc( + self.host, self.port, + _SRVConnector_ClientFactoryWrapper(self, self.factory), + *self.connectFuncArgs, **self.connectFuncKwArgs) + + + def stopConnecting(self): + """Stop attempting to connect.""" + if self.connector: + self.connector.stopConnecting() + else: + self.stopAfterDNS = 1 + + + def disconnect(self): + """Disconnect whatever our are state is.""" + if self.connector is not None: + self.connector.disconnect() + else: + self.stopConnecting() + + + def getDestination(self): + assert self.connector + return self.connector.getDestination() + + + def connectionFailed(self, reason): + self.factory.clientConnectionFailed(self, reason) + self.factory.doStop() + + + def connectionLost(self, reason): + self.factory.clientConnectionLost(self, reason) + self.factory.doStop() diff --git a/contrib/python/Twisted/py2/twisted/names/tap.py b/contrib/python/Twisted/py2/twisted/names/tap.py new file mode 100644 index 00000000000..d0e3b1d0627 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/names/tap.py @@ -0,0 +1,150 @@ +# -*- test-case-name: twisted.names.test.test_tap -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Domain Name Server +""" + +import os, traceback + +from twisted.python import usage +from twisted.names import dns +from twisted.application import internet, service + +from twisted.names import server +from twisted.names import authority +from twisted.names import secondary + +class Options(usage.Options): + optParameters = [ + ["interface", "i", "", "The interface to which to bind"], + ["port", "p", "53", "The port on which to listen"], + ["resolv-conf", None, None, + "Override location of resolv.conf (implies --recursive)"], + ["hosts-file", None, None, "Perform lookups with a hosts file"], + ] + + optFlags = [ + ["cache", "c", "Enable record caching"], + ["recursive", "r", "Perform recursive lookups"], + ["verbose", "v", "Log verbosely"], + ] + + compData = usage.Completions( + optActions={"interface" : usage.CompleteNetInterfaces()} + ) + + zones = None + zonefiles = None + + def __init__(self): + usage.Options.__init__(self) + self['verbose'] = 0 + self.bindfiles = [] + self.zonefiles = [] + self.secondaries = [] + + + def opt_pyzone(self, filename): + """Specify the filename of a Python syntax zone definition""" + if not os.path.exists(filename): + raise usage.UsageError(filename + ": No such file") + self.zonefiles.append(filename) + + def opt_bindzone(self, filename): + """Specify the filename of a BIND9 syntax zone definition""" + if not os.path.exists(filename): + raise usage.UsageError(filename + ": No such file") + self.bindfiles.append(filename) + + + def opt_secondary(self, ip_domain): + """Act as secondary for the specified domain, performing + zone transfers from the specified IP (IP/domain) + """ + args = ip_domain.split('/', 1) + if len(args) != 2: + raise usage.UsageError("Argument must be of the form IP[:port]/domain") + address = args[0].split(':') + if len(address) == 1: + address = (address[0], dns.PORT) + else: + try: + port = int(address[1]) + except ValueError: + raise usage.UsageError( + "Specify an integer port number, not %r" % (address[1],)) + address = (address[0], port) + self.secondaries.append((address, [args[1]])) + + + def opt_verbose(self): + """Increment verbosity level""" + self['verbose'] += 1 + + + def postOptions(self): + if self['resolv-conf']: + self['recursive'] = True + + self.svcs = [] + self.zones = [] + for f in self.zonefiles: + try: + self.zones.append(authority.PySourceAuthority(f)) + except Exception: + traceback.print_exc() + raise usage.UsageError("Invalid syntax in " + f) + for f in self.bindfiles: + try: + self.zones.append(authority.BindAuthority(f)) + except Exception: + traceback.print_exc() + raise usage.UsageError("Invalid syntax in " + f) + for f in self.secondaries: + svc = secondary.SecondaryAuthorityService.fromServerAddressAndDomains(*f) + self.svcs.append(svc) + self.zones.append(self.svcs[-1].getAuthority()) + try: + self['port'] = int(self['port']) + except ValueError: + raise usage.UsageError("Invalid port: %r" % (self['port'],)) + + +def _buildResolvers(config): + """ + Build DNS resolver instances in an order which leaves recursive + resolving as a last resort. + + @type config: L{Options} instance + @param config: Parsed command-line configuration + + @return: Two-item tuple of a list of cache resovers and a list of client + resolvers + """ + from twisted.names import client, cache, hosts + + ca, cl = [], [] + if config['cache']: + ca.append(cache.CacheResolver(verbose=config['verbose'])) + if config['hosts-file']: + cl.append(hosts.Resolver(file=config['hosts-file'])) + if config['recursive']: + cl.append(client.createResolver(resolvconf=config['resolv-conf'])) + return ca, cl + + +def makeService(config): + ca, cl = _buildResolvers(config) + + f = server.DNSServerFactory(config.zones, ca, cl, config['verbose']) + p = dns.DNSDatagramProtocol(f) + f.noisy = 0 + ret = service.MultiService() + for (klass, arg) in [(internet.TCPServer, f), (internet.UDPServer, p)]: + s = klass(config['port'], arg, interface=config['interface']) + s.setServiceParent(ret) + for svc in config.svcs: + svc.setServiceParent(ret) + return ret diff --git a/contrib/python/Twisted/py2/twisted/news/__init__.py b/contrib/python/Twisted/py2/twisted/news/__init__.py new file mode 100644 index 00000000000..b31924820a0 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/news/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted News: A NNTP-based news service. +""" diff --git a/contrib/python/Twisted/py2/twisted/news/database.py b/contrib/python/Twisted/py2/twisted/news/database.py new file mode 100644 index 00000000000..5a25dd37b8b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/news/database.py @@ -0,0 +1,1046 @@ +# -*- test-case-name: twisted.news.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +News server backend implementations. +""" + +import getpass, pickle, time, socket +import os +import StringIO +from hashlib import md5 +from email.Message import Message +from email.Generator import Generator +from zope.interface import implementer, Interface + +from twisted.news.nntp import NNTPError +from twisted.mail import smtp +from twisted.internet import defer +from twisted.enterprise import adbapi +from twisted.persisted import dirdbm + + + +ERR_NOGROUP, ERR_NOARTICLE = range(2, 4) # XXX - put NNTP values here (I guess?) + +OVERVIEW_FMT = [ + 'Subject', 'From', 'Date', 'Message-ID', 'References', + 'Bytes', 'Lines', 'Xref' +] + +def hexdigest(md5): #XXX: argh. 1.5.2 doesn't have this. + return ''.join(map(lambda x: hex(ord(x))[2:], md5.digest())) + +class Article: + def __init__(self, head, body): + self.body = body + self.headers = {} + header = None + for line in head.split('\r\n'): + if line[0] in ' \t': + i = list(self.headers[header]) + i[1] += '\r\n' + line + else: + i = line.split(': ', 1) + header = i[0].lower() + self.headers[header] = tuple(i) + + if not self.getHeader('Message-ID'): + s = str(time.time()) + self.body + id = hexdigest(md5(s)) + '@' + socket.gethostname() + self.putHeader('Message-ID', '<%s>' % id) + + if not self.getHeader('Bytes'): + self.putHeader('Bytes', str(len(self.body))) + + if not self.getHeader('Lines'): + self.putHeader('Lines', str(self.body.count('\n'))) + + if not self.getHeader('Date'): + self.putHeader('Date', time.ctime(time.time())) + + + def getHeader(self, header): + h = header.lower() + if h in self.headers: + return self.headers[h][1] + else: + return '' + + + def putHeader(self, header, value): + self.headers[header.lower()] = (header, value) + + + def textHeaders(self): + headers = [] + for i in self.headers.values(): + headers.append('%s: %s' % i) + return '\r\n'.join(headers) + '\r\n' + + def overview(self): + xover = [] + for i in OVERVIEW_FMT: + xover.append(self.getHeader(i)) + return xover + + +class NewsServerError(Exception): + pass + + +class INewsStorage(Interface): + """ + An interface for storing and requesting news articles + """ + + def listRequest(): + """ + Returns a deferred whose callback will be passed a list of 4-tuples + containing (name, max index, min index, flags) for each news group + """ + + + def subscriptionRequest(): + """ + Returns a deferred whose callback will be passed the list of + recommended subscription groups for new server users + """ + + + def postRequest(message): + """ + Returns a deferred whose callback will be invoked if 'message' + is successfully posted to one or more specified groups and + whose errback will be invoked otherwise. + """ + + + def overviewRequest(): + """ + Returns a deferred whose callback will be passed the a list of + headers describing this server's overview format. + """ + + + def xoverRequest(group, low, high): + """ + Returns a deferred whose callback will be passed a list of xover + headers for the given group over the given range. If low is None, + the range starts at the first article. If high is None, the range + ends at the last article. + """ + + + def xhdrRequest(group, low, high, header): + """ + Returns a deferred whose callback will be passed a list of XHDR data + for the given group over the given range. If low is None, + the range starts at the first article. If high is None, the range + ends at the last article. + """ + + + def listGroupRequest(group): + """ + Returns a deferred whose callback will be passed a two-tuple of + (group name, [article indices]) + """ + + + def groupRequest(group): + """ + Returns a deferred whose callback will be passed a five-tuple of + (group name, article count, highest index, lowest index, group flags) + """ + + + def articleExistsRequest(id): + """ + Returns a deferred whose callback will be passed with a true value + if a message with the specified Message-ID exists in the database + and with a false value otherwise. + """ + + + def articleRequest(group, index, id = None): + """ + Returns a deferred whose callback will be passed a file-like object + containing the full article text (headers and body) for the article + of the specified index in the specified group, and whose errback + will be invoked if the article or group does not exist. If id is + not None, index is ignored and the article with the given Message-ID + will be returned instead, along with its index in the specified + group. + """ + + + def headRequest(group, index): + """ + Returns a deferred whose callback will be passed the header for + the article of the specified index in the specified group, and + whose errback will be invoked if the article or group does not + exist. + """ + + + def bodyRequest(group, index): + """ + Returns a deferred whose callback will be passed the body for + the article of the specified index in the specified group, and + whose errback will be invoked if the article or group does not + exist. + """ + +class NewsStorage: + """ + Backwards compatibility class -- There is no reason to inherit from this, + just implement INewsStorage instead. + """ + def listRequest(self): + raise NotImplementedError() + def subscriptionRequest(self): + raise NotImplementedError() + def postRequest(self, message): + raise NotImplementedError() + def overviewRequest(self): + return defer.succeed(OVERVIEW_FMT) + def xoverRequest(self, group, low, high): + raise NotImplementedError() + def xhdrRequest(self, group, low, high, header): + raise NotImplementedError() + def listGroupRequest(self, group): + raise NotImplementedError() + def groupRequest(self, group): + raise NotImplementedError() + def articleExistsRequest(self, id): + raise NotImplementedError() + def articleRequest(self, group, index, id = None): + raise NotImplementedError() + def headRequest(self, group, index): + raise NotImplementedError() + def bodyRequest(self, group, index): + raise NotImplementedError() + + + +class _ModerationMixin: + """ + Storage implementations can inherit from this class to get the easy-to-use + C{notifyModerators} method which will take care of sending messages which + require moderation to a list of moderators. + """ + sendmail = staticmethod(smtp.sendmail) + + def notifyModerators(self, moderators, article): + """ + Send an article to a list of group moderators to be moderated. + + @param moderators: A C{list} of C{str} giving RFC 2821 addresses of + group moderators to notify. + + @param article: The article requiring moderation. + @type article: L{Article} + + @return: A L{Deferred} which fires with the result of sending the email. + """ + # Moderated postings go through as long as they have an Approved + # header, regardless of what the value is + group = article.getHeader('Newsgroups') + subject = article.getHeader('Subject') + + if self._sender is None: + # This case should really go away. This isn't a good default. + sender = 'twisted-news@' + socket.gethostname() + else: + sender = self._sender + + msg = Message() + msg['Message-ID'] = smtp.messageid() + msg['From'] = sender + msg['To'] = ', '.join(moderators) + msg['Subject'] = 'Moderate new %s message: %s' % (group, subject) + msg['Content-Type'] = 'message/rfc822' + + payload = Message() + for header, value in article.headers.values(): + payload.add_header(header, value) + payload.set_payload(article.body) + + msg.attach(payload) + + out = StringIO.StringIO() + gen = Generator(out, False) + gen.flatten(msg) + msg = out.getvalue() + + return self.sendmail(self._mailhost, sender, moderators, msg) + + + +@implementer(INewsStorage) +class PickleStorage(_ModerationMixin): + """ + A trivial NewsStorage implementation using pickles + + Contains numerous flaws and is generally unsuitable for any + real applications. Consider yourself warned! + """ + sharedDBs = {} + + def __init__(self, filename, groups=None, moderators=(), + mailhost=None, sender=None): + """ + @param mailhost: A C{str} giving the mail exchange host which will + accept moderation emails from this server. Must accept emails + destined for any address specified as a moderator. + + @param sender: A C{str} giving the address which will be used as the + sender of any moderation email generated by this server. + """ + self.datafile = filename + self.load(filename, groups, moderators) + self._mailhost = mailhost + self._sender = sender + + + def getModerators(self, groups): + # first see if any groups are moderated. if so, nothing gets posted, + # but the whole messages gets forwarded to the moderator address + moderators = [] + for group in groups: + moderators.extend(self.db['moderators'].get(group, None)) + return filter(None, moderators) + + + def listRequest(self): + "Returns a list of 4-tuples: (name, max index, min index, flags)" + l = self.db['groups'] + r = [] + for i in l: + if len(self.db[i].keys()): + low = min(self.db[i].keys()) + high = max(self.db[i].keys()) + 1 + else: + low = high = 0 + if i in self.db['moderators']: + flags = 'm' + else: + flags = 'y' + r.append((i, high, low, flags)) + return defer.succeed(r) + + def subscriptionRequest(self): + return defer.succeed(['alt.test']) + + def postRequest(self, message): + cleave = message.find('\r\n\r\n') + headers, article = message[:cleave], message[cleave + 4:] + + a = Article(headers, article) + groups = a.getHeader('Newsgroups').split() + xref = [] + + # Check moderated status + moderators = self.getModerators(groups) + if moderators and not a.getHeader('Approved'): + return self.notifyModerators(moderators, a) + + for group in groups: + if group in self.db: + if len(self.db[group].keys()): + index = max(self.db[group].keys()) + 1 + else: + index = 1 + xref.append((group, str(index))) + self.db[group][index] = a + + if len(xref) == 0: + return defer.fail(None) + + a.putHeader('Xref', '%s %s' % ( + socket.gethostname().split()[0], + ''.join(map(lambda x: ':'.join(x), xref)) + )) + + self.flush() + return defer.succeed(None) + + + def overviewRequest(self): + return defer.succeed(OVERVIEW_FMT) + + + def xoverRequest(self, group, low, high): + if group not in self.db: + return defer.succeed([]) + r = [] + for i in self.db[group].keys(): + if (low is None or i >= low) and (high is None or i <= high): + r.append([str(i)] + self.db[group][i].overview()) + return defer.succeed(r) + + + def xhdrRequest(self, group, low, high, header): + if group not in self.db: + return defer.succeed([]) + r = [] + for i in self.db[group].keys(): + if low is None or i >= low and high is None or i <= high: + r.append((i, self.db[group][i].getHeader(header))) + return defer.succeed(r) + + + def listGroupRequest(self, group): + if group in self.db: + return defer.succeed((group, self.db[group].keys())) + else: + return defer.fail(None) + + def groupRequest(self, group): + if group in self.db: + if len(self.db[group].keys()): + num = len(self.db[group].keys()) + low = min(self.db[group].keys()) + high = max(self.db[group].keys()) + else: + num = low = high = 0 + flags = 'y' + return defer.succeed((group, num, high, low, flags)) + else: + return defer.fail(ERR_NOGROUP) + + + def articleExistsRequest(self, id): + for group in self.db['groups']: + for a in self.db[group].values(): + if a.getHeader('Message-ID') == id: + return defer.succeed(1) + return defer.succeed(0) + + + def articleRequest(self, group, index, id = None): + if id is not None: + raise NotImplementedError + + if group in self.db: + if index in self.db[group]: + a = self.db[group][index] + return defer.succeed(( + index, + a.getHeader('Message-ID'), + StringIO.StringIO(a.textHeaders() + '\r\n' + a.body) + )) + else: + return defer.fail(ERR_NOARTICLE) + else: + return defer.fail(ERR_NOGROUP) + + + def headRequest(self, group, index): + if group in self.db: + if index in self.db[group]: + a = self.db[group][index] + return defer.succeed((index, a.getHeader('Message-ID'), a.textHeaders())) + else: + return defer.fail(ERR_NOARTICLE) + else: + return defer.fail(ERR_NOGROUP) + + + def bodyRequest(self, group, index): + if group in self.db: + if index in self.db[group]: + a = self.db[group][index] + return defer.succeed((index, a.getHeader('Message-ID'), StringIO.StringIO(a.body))) + else: + return defer.fail(ERR_NOARTICLE) + else: + return defer.fail(ERR_NOGROUP) + + + def flush(self): + with open(self.datafile, 'w') as f: + pickle.dump(self.db, f) + + + def load(self, filename, groups = None, moderators = ()): + if filename in PickleStorage.sharedDBs: + self.db = PickleStorage.sharedDBs[filename] + else: + try: + with open(filename) as f: + self.db = pickle.load(f) + PickleStorage.sharedDBs[filename] = self.db + except IOError: + self.db = PickleStorage.sharedDBs[filename] = {} + self.db['groups'] = groups + if groups is not None: + for i in groups: + self.db[i] = {} + self.db['moderators'] = dict(moderators) + self.flush() + + +class Group: + name = None + flags = '' + minArticle = 1 + maxArticle = 0 + articles = None + + def __init__(self, name, flags = 'y'): + self.name = name + self.flags = flags + self.articles = {} + + +@implementer(INewsStorage) +class NewsShelf(_ModerationMixin): + """ + A NewStorage implementation using Twisted's dirdbm persistence module. + """ + def __init__(self, mailhost, path, sender=None): + """ + @param mailhost: A C{str} giving the mail exchange host which will + accept moderation emails from this server. Must accept emails + destined for any address specified as a moderator. + + @param sender: A C{str} giving the address which will be used as the + sender of any moderation email generated by this server. + """ + self.path = path + self._mailhost = self.mailhost = mailhost + self._sender = sender + + if not os.path.exists(path): + os.mkdir(path) + + self.dbm = dirdbm.Shelf(os.path.join(path, "newsshelf")) + if not len(self.dbm.keys()): + self.initialize() + + + def initialize(self): + # A dictionary of group name/Group instance items + self.dbm['groups'] = dirdbm.Shelf(os.path.join(self.path, 'groups')) + + # A dictionary of group name/email address + self.dbm['moderators'] = dirdbm.Shelf(os.path.join(self.path, 'moderators')) + + # A list of group names + self.dbm['subscriptions'] = [] + + # A dictionary of MessageID strings/xref lists + self.dbm['Message-IDs'] = dirdbm.Shelf(os.path.join(self.path, 'Message-IDs')) + + + def addGroup(self, name, flags): + self.dbm['groups'][name] = Group(name, flags) + + + def addSubscription(self, name): + self.dbm['subscriptions'] = self.dbm['subscriptions'] + [name] + + + def addModerator(self, group, email): + self.dbm['moderators'][group] = email + + + def listRequest(self): + result = [] + for g in self.dbm['groups'].values(): + result.append((g.name, g.maxArticle, g.minArticle, g.flags)) + return defer.succeed(result) + + + def subscriptionRequest(self): + return defer.succeed(self.dbm['subscriptions']) + + + def getModerator(self, groups): + # first see if any groups are moderated. if so, nothing gets posted, + # but the whole messages gets forwarded to the moderator address + for group in groups: + try: + return self.dbm['moderators'][group] + except KeyError: + pass + return None + + + def notifyModerator(self, moderator, article): + """ + Notify a single moderator about an article requiring moderation. + + C{notifyModerators} should be preferred. + """ + return self.notifyModerators([moderator], article) + + + def postRequest(self, message): + cleave = message.find('\r\n\r\n') + headers, article = message[:cleave], message[cleave + 4:] + + article = Article(headers, article) + groups = article.getHeader('Newsgroups').split() + xref = [] + + # Check for moderated status + moderator = self.getModerator(groups) + if moderator and not article.getHeader('Approved'): + return self.notifyModerators([moderator], article) + + + for group in groups: + try: + g = self.dbm['groups'][group] + except KeyError: + pass + else: + index = g.maxArticle + 1 + g.maxArticle += 1 + g.articles[index] = article + xref.append((group, str(index))) + self.dbm['groups'][group] = g + + if not xref: + return defer.fail(NewsServerError("No groups carried: " + ' '.join(groups))) + + article.putHeader('Xref', '%s %s' % (socket.gethostname().split()[0], ' '.join(map(lambda x: ':'.join(x), xref)))) + self.dbm['Message-IDs'][article.getHeader('Message-ID')] = xref + return defer.succeed(None) + + + def overviewRequest(self): + return defer.succeed(OVERVIEW_FMT) + + + def xoverRequest(self, group, low, high): + if group not in self.dbm['groups']: + return defer.succeed([]) + + if low is None: + low = 0 + if high is None: + high = self.dbm['groups'][group].maxArticle + r = [] + for i in range(low, high + 1): + if i in self.dbm['groups'][group].articles: + r.append([str(i)] + self.dbm['groups'][group].articles[i].overview()) + return defer.succeed(r) + + + def xhdrRequest(self, group, low, high, header): + if group not in self.dbm['groups']: + return defer.succeed([]) + + if low is None: + low = 0 + if high is None: + high = self.dbm['groups'][group].maxArticle + r = [] + for i in range(low, high + 1): + if i in self.dbm['groups'][group].articles: + r.append((i, self.dbm['groups'][group].articles[i].getHeader(header))) + return defer.succeed(r) + + + def listGroupRequest(self, group): + if group in self.dbm['groups']: + return defer.succeed((group, self.dbm['groups'][group].articles.keys())) + return defer.fail(NewsServerError("No such group: " + group)) + + + def groupRequest(self, group): + try: + g = self.dbm['groups'][group] + except KeyError: + return defer.fail(NewsServerError("No such group: " + group)) + else: + flags = g.flags + low = g.minArticle + high = g.maxArticle + num = high - low + 1 + return defer.succeed((group, num, high, low, flags)) + + + def articleExistsRequest(self, id): + return defer.succeed(id in self.dbm['Message-IDs']) + + + def articleRequest(self, group, index, id = None): + if id is not None: + try: + xref = self.dbm['Message-IDs'][id] + except KeyError: + return defer.fail(NewsServerError("No such article: " + id)) + else: + group, index = xref[0] + index = int(index) + + try: + a = self.dbm['groups'][group].articles[index] + except KeyError: + return defer.fail(NewsServerError("No such group: " + group)) + else: + return defer.succeed(( + index, + a.getHeader('Message-ID'), + StringIO.StringIO(a.textHeaders() + '\r\n' + a.body) + )) + + + def headRequest(self, group, index, id = None): + if id is not None: + try: + xref = self.dbm['Message-IDs'][id] + except KeyError: + return defer.fail(NewsServerError("No such article: " + id)) + else: + group, index = xref[0] + index = int(index) + + try: + a = self.dbm['groups'][group].articles[index] + except KeyError: + return defer.fail(NewsServerError("No such group: " + group)) + else: + return defer.succeed((index, a.getHeader('Message-ID'), a.textHeaders())) + + + def bodyRequest(self, group, index, id = None): + if id is not None: + try: + xref = self.dbm['Message-IDs'][id] + except KeyError: + return defer.fail(NewsServerError("No such article: " + id)) + else: + group, index = xref[0] + index = int(index) + + try: + a = self.dbm['groups'][group].articles[index] + except KeyError: + return defer.fail(NewsServerError("No such group: " + group)) + else: + return defer.succeed((index, a.getHeader('Message-ID'), StringIO.StringIO(a.body))) + + +@implementer(INewsStorage) +class NewsStorageAugmentation: + """ + A NewsStorage implementation using Twisted's asynchronous DB-API + """ + schema = """ + + CREATE TABLE groups ( + group_id SERIAL, + name VARCHAR(80) NOT NULL, + + flags INTEGER DEFAULT 0 NOT NULL + ); + + CREATE UNIQUE INDEX group_id_index ON groups (group_id); + CREATE UNIQUE INDEX name_id_index ON groups (name); + + CREATE TABLE articles ( + article_id SERIAL, + message_id TEXT, + + header TEXT, + body TEXT + ); + + CREATE UNIQUE INDEX article_id_index ON articles (article_id); + CREATE UNIQUE INDEX article_message_index ON articles (message_id); + + CREATE TABLE postings ( + group_id INTEGER, + article_id INTEGER, + article_index INTEGER NOT NULL + ); + + CREATE UNIQUE INDEX posting_article_index ON postings (article_id); + + CREATE TABLE subscriptions ( + group_id INTEGER + ); + + CREATE TABLE overview ( + header TEXT + ); + """ + + def __init__(self, info): + self.info = info + self.dbpool = adbapi.ConnectionPool(**self.info) + + + def __setstate__(self, state): + self.__dict__ = state + self.info['password'] = getpass.getpass('Database password for %s: ' % (self.info['user'],)) + self.dbpool = adbapi.ConnectionPool(**self.info) + del self.info['password'] + + + def listRequest(self): + # COALESCE may not be totally portable + # it is shorthand for + # CASE WHEN (first parameter) IS NOT NULL then (first parameter) ELSE (second parameter) END + sql = """ + SELECT groups.name, + COALESCE(MAX(postings.article_index), 0), + COALESCE(MIN(postings.article_index), 0), + groups.flags + FROM groups LEFT OUTER JOIN postings + ON postings.group_id = groups.group_id + GROUP BY groups.name, groups.flags + ORDER BY groups.name + """ + return self.dbpool.runQuery(sql) + + + def subscriptionRequest(self): + sql = """ + SELECT groups.name FROM groups,subscriptions WHERE groups.group_id = subscriptions.group_id + """ + return self.dbpool.runQuery(sql) + + + def postRequest(self, message): + cleave = message.find('\r\n\r\n') + headers, article = message[:cleave], message[cleave + 4:] + article = Article(headers, article) + return self.dbpool.runInteraction(self._doPost, article) + + + def _doPost(self, transaction, article): + # Get the group ids + groups = article.getHeader('Newsgroups').split() + if not len(groups): + raise NNTPError('Missing Newsgroups header') + + sql = """ + SELECT name, group_id FROM groups + WHERE name IN (%s) + """ % (', '.join([("'%s'" % (adbapi.safe(group),)) for group in groups]),) + + transaction.execute(sql) + result = transaction.fetchall() + + # No relevant groups, bye bye! + if not len(result): + raise NNTPError('None of groups in Newsgroup header carried') + + # Got some groups, now find the indices this article will have in each + sql = """ + SELECT groups.group_id, COALESCE(MAX(postings.article_index), 0) + 1 + FROM groups LEFT OUTER JOIN postings + ON postings.group_id = groups.group_id + WHERE groups.group_id IN (%s) + GROUP BY groups.group_id + """ % (', '.join([("%d" % (id,)) for (group, id) in result]),) + + transaction.execute(sql) + indices = transaction.fetchall() + + if not len(indices): + raise NNTPError('Internal server error - no indices found') + + # Associate indices with group names + gidToName = dict([(b, a) for (a, b) in result]) + gidToIndex = dict(indices) + + nameIndex = [] + for i in gidToName: + nameIndex.append((gidToName[i], gidToIndex[i])) + + # Build xrefs + xrefs = socket.gethostname().split()[0] + xrefs = xrefs + ' ' + ' '.join([('%s:%d' % (group, id)) for (group, id) in nameIndex]) + article.putHeader('Xref', xrefs) + + # Hey! The article is ready to be posted! God damn f'in finally. + sql = """ + INSERT INTO articles (message_id, header, body) + VALUES ('%s', '%s', '%s') + """ % ( + adbapi.safe(article.getHeader('Message-ID')), + adbapi.safe(article.textHeaders()), + adbapi.safe(article.body) + ) + + transaction.execute(sql) + + # Now update the posting to reflect the groups to which this belongs + for gid in gidToName: + sql = """ + INSERT INTO postings (group_id, article_id, article_index) + VALUES (%d, (SELECT last_value FROM articles_article_id_seq), %d) + """ % (gid, gidToIndex[gid]) + transaction.execute(sql) + + return len(nameIndex) + + + def overviewRequest(self): + sql = """ + SELECT header FROM overview + """ + return self.dbpool.runQuery(sql).addCallback(lambda result: [header[0] for header in result]) + + + def xoverRequest(self, group, low, high): + sql = """ + SELECT postings.article_index, articles.header + FROM articles,postings,groups + WHERE postings.group_id = groups.group_id + AND groups.name = '%s' + AND postings.article_id = articles.article_id + %s + %s + """ % ( + adbapi.safe(group), + low is not None and "AND postings.article_index >= %d" % (low,) or "", + high is not None and "AND postings.article_index <= %d" % (high,) or "" + ) + + return self.dbpool.runQuery(sql).addCallback( + lambda results: [ + [id] + Article(header, None).overview() for (id, header) in results + ] + ) + + + def xhdrRequest(self, group, low, high, header): + sql = """ + SELECT articles.header + FROM groups,postings,articles + WHERE groups.name = '%s' AND postings.group_id = groups.group_id + AND postings.article_index >= %d + AND postings.article_index <= %d + """ % (adbapi.safe(group), low, high) + + return self.dbpool.runQuery(sql).addCallback( + lambda results: [ + (i, Article(h, None).getHeader(h)) for (i, h) in results + ] + ) + + + def listGroupRequest(self, group): + sql = """ + SELECT postings.article_index FROM postings,groups + WHERE postings.group_id = groups.group_id + AND groups.name = '%s' + """ % (adbapi.safe(group),) + + return self.dbpool.runQuery(sql).addCallback( + lambda results, group = group: (group, [res[0] for res in results]) + ) + + + def groupRequest(self, group): + sql = """ + SELECT groups.name, + COUNT(postings.article_index), + COALESCE(MAX(postings.article_index), 0), + COALESCE(MIN(postings.article_index), 0), + groups.flags + FROM groups LEFT OUTER JOIN postings + ON postings.group_id = groups.group_id + WHERE groups.name = '%s' + GROUP BY groups.name, groups.flags + """ % (adbapi.safe(group),) + + return self.dbpool.runQuery(sql).addCallback( + lambda results: tuple(results[0]) + ) + + + def articleExistsRequest(self, id): + sql = """ + SELECT COUNT(message_id) FROM articles + WHERE message_id = '%s' + """ % (adbapi.safe(id),) + + return self.dbpool.runQuery(sql).addCallback( + lambda result: bool(result[0][0]) + ) + + + def articleRequest(self, group, index, id = None): + if id is not None: + sql = """ + SELECT postings.article_index, articles.message_id, articles.header, articles.body + FROM groups,postings LEFT OUTER JOIN articles + ON articles.message_id = '%s' + WHERE groups.name = '%s' + AND groups.group_id = postings.group_id + """ % (adbapi.safe(id), adbapi.safe(group)) + else: + sql = """ + SELECT postings.article_index, articles.message_id, articles.header, articles.body + FROM groups,articles LEFT OUTER JOIN postings + ON postings.article_id = articles.article_id + WHERE postings.article_index = %d + AND postings.group_id = groups.group_id + AND groups.name = '%s' + """ % (index, adbapi.safe(group)) + + return self.dbpool.runQuery(sql).addCallback( + lambda result: ( + result[0][0], + result[0][1], + StringIO.StringIO(result[0][2] + '\r\n' + result[0][3]) + ) + ) + + + def headRequest(self, group, index): + sql = """ + SELECT postings.article_index, articles.message_id, articles.header + FROM groups,articles LEFT OUTER JOIN postings + ON postings.article_id = articles.article_id + WHERE postings.article_index = %d + AND postings.group_id = groups.group_id + AND groups.name = '%s' + """ % (index, adbapi.safe(group)) + + return self.dbpool.runQuery(sql).addCallback(lambda result: result[0]) + + + def bodyRequest(self, group, index): + sql = """ + SELECT postings.article_index, articles.message_id, articles.body + FROM groups,articles LEFT OUTER JOIN postings + ON postings.article_id = articles.article_id + WHERE postings.article_index = %d + AND postings.group_id = groups.group_id + AND groups.name = '%s' + """ % (index, adbapi.safe(group)) + + return self.dbpool.runQuery(sql).addCallback( + lambda result: result[0] + ).addCallback( + # result is a tuple of (index, id, body) + lambda result: (result[0], result[1], StringIO.StringIO(result[2])) + ) + +#### +#### XXX - make these static methods some day +#### +def makeGroupSQL(groups): + res = '' + for g in groups: + res = res + """\n INSERT INTO groups (name) VALUES ('%s');\n""" % (adbapi.safe(g),) + return res + + +def makeOverviewSQL(): + res = '' + for o in OVERVIEW_FMT: + res = res + """\n INSERT INTO overview (header) VALUES ('%s');\n""" % (adbapi.safe(o),) + return res diff --git a/contrib/python/Twisted/py2/twisted/news/news.py b/contrib/python/Twisted/py2/twisted/news/news.py new file mode 100644 index 00000000000..6f1da245a79 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/news/news.py @@ -0,0 +1,92 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Maintainer: Jp Calderone +""" + +from __future__ import print_function + +from twisted.news import nntp +from twisted.internet import protocol, reactor + +import time + +class NNTPFactory(protocol.ServerFactory): + """A factory for NNTP server protocols.""" + + protocol = nntp.NNTPServer + + def __init__(self, backend): + self.backend = backend + + def buildProtocol(self, connection): + p = self.protocol() + p.factory = self + return p + + +class UsenetClientFactory(protocol.ClientFactory): + def __init__(self, groups, storage): + self.lastChecks = {} + self.groups = groups + self.storage = storage + + + def clientConnectionLost(self, connector, reason): + pass + + + def clientConnectionFailed(self, connector, reason): + print('Connection failed: ', reason) + + + def updateChecks(self, addr): + self.lastChecks[addr] = time.mktime(time.gmtime()) + + + def buildProtocol(self, addr): + last = self.lastChecks.setdefault(addr, time.mktime(time.gmtime()) - (60 * 60 * 24 * 7)) + p = nntp.UsenetClientProtocol(self.groups, last, self.storage) + p.factory = self + return p + + +# XXX - Maybe this inheritance doesn't make so much sense? +class UsenetServerFactory(NNTPFactory): + """A factory for NNTP Usenet server protocols.""" + + protocol = nntp.NNTPServer + + def __init__(self, backend, remoteHosts = None, updatePeriod = 60): + NNTPFactory.__init__(self, backend) + self.updatePeriod = updatePeriod + self.remoteHosts = remoteHosts or [] + self.clientFactory = UsenetClientFactory(self.remoteHosts, self.backend) + + + def startFactory(self): + self._updateCall = reactor.callLater(0, self.syncWithRemotes) + + + def stopFactory(self): + if self._updateCall: + self._updateCall.cancel() + self._updateCall = None + + + def buildProtocol(self, connection): + p = self.protocol() + p.factory = self + return p + + + def syncWithRemotes(self): + for remote in self.remoteHosts: + reactor.connectTCP(remote, 119, self.clientFactory) + self._updateCall = reactor.callLater(self.updatePeriod, self.syncWithRemotes) + + +# backwards compatibility +Factory = UsenetServerFactory diff --git a/contrib/python/Twisted/py2/twisted/news/nntp.py b/contrib/python/Twisted/py2/twisted/news/nntp.py new file mode 100644 index 00000000000..6c5d12b2778 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/news/nntp.py @@ -0,0 +1,1050 @@ +# -*- test-case-name: twisted.news.test.test_nntp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +NNTP protocol support. + +The following protocol commands are currently understood:: + + LIST LISTGROUP XOVER XHDR + POST GROUP ARTICLE STAT HEAD + BODY NEXT MODE STREAM MODE READER SLAVE + LAST QUIT HELP IHAVE XPATH + XINDEX XROVER TAKETHIS CHECK + +The following protocol commands require implementation:: + + NEWNEWS + XGTITLE XPAT + XTHREAD AUTHINFO NEWGROUPS + + +Other desired features: + + - A real backend + - More robust client input handling + - A control protocol +""" + +from __future__ import print_function + +import time + +from twisted.protocols import basic +from twisted.python import log + +def parseRange(text): + articles = text.split('-') + if len(articles) == 1: + try: + a = int(articles[0]) + return a, a + except ValueError: + return None, None + elif len(articles) == 2: + try: + if len(articles[0]): + l = int(articles[0]) + else: + l = None + if len(articles[1]): + h = int(articles[1]) + else: + h = None + except ValueError: + return None, None + return l, h + + +def extractCode(line): + line = line.split(' ', 1) + if len(line) != 2: + return None + try: + return int(line[0]), line[1] + except ValueError: + return None + + +class NNTPError(Exception): + def __init__(self, string): + self.string = string + + def __str__(self): + return 'NNTPError: %s' % self.string + + +class NNTPClient(basic.LineReceiver): + MAX_COMMAND_LENGTH = 510 + + def __init__(self): + self.currentGroup = None + + self._state = [] + self._error = [] + self._inputBuffers = [] + self._responseCodes = [] + self._responseHandlers = [] + + self._postText = [] + + self._newState(self._statePassive, None, self._headerInitial) + + + def gotAllGroups(self, groups): + "Override for notification when fetchGroups() action is completed" + + + def getAllGroupsFailed(self, error): + "Override for notification when fetchGroups() action fails" + + + def gotOverview(self, overview): + "Override for notification when fetchOverview() action is completed" + + + def getOverviewFailed(self, error): + "Override for notification when fetchOverview() action fails" + + + def gotSubscriptions(self, subscriptions): + "Override for notification when fetchSubscriptions() action is completed" + + + def getSubscriptionsFailed(self, error): + "Override for notification when fetchSubscriptions() action fails" + + + def gotGroup(self, group): + "Override for notification when fetchGroup() action is completed" + + + def getGroupFailed(self, error): + "Override for notification when fetchGroup() action fails" + + + def gotArticle(self, article): + "Override for notification when fetchArticle() action is completed" + + + def getArticleFailed(self, error): + "Override for notification when fetchArticle() action fails" + + + def gotHead(self, head): + "Override for notification when fetchHead() action is completed" + + + def getHeadFailed(self, error): + "Override for notification when fetchHead() action fails" + + + def gotBody(self, info): + "Override for notification when fetchBody() action is completed" + + + def getBodyFailed(self, body): + "Override for notification when fetchBody() action fails" + + + def postedOk(self): + "Override for notification when postArticle() action is successful" + + + def postFailed(self, error): + "Override for notification when postArticle() action fails" + + + def gotXHeader(self, headers): + "Override for notification when getXHeader() action is successful" + + + def getXHeaderFailed(self, error): + "Override for notification when getXHeader() action fails" + + + def gotNewNews(self, news): + "Override for notification when getNewNews() action is successful" + + + def getNewNewsFailed(self, error): + "Override for notification when getNewNews() action fails" + + + def gotNewGroups(self, groups): + "Override for notification when getNewGroups() action is successful" + + + def getNewGroupsFailed(self, error): + "Override for notification when getNewGroups() action fails" + + + def setStreamSuccess(self): + "Override for notification when setStream() action is successful" + + + def setStreamFailed(self, error): + "Override for notification when setStream() action fails" + + + def fetchGroups(self): + """ + Request a list of all news groups from the server. gotAllGroups() + is called on success, getGroupsFailed() on failure + """ + self.sendLine('LIST') + self._newState(self._stateList, self.getAllGroupsFailed) + + + def fetchOverview(self): + """ + Request the overview format from the server. gotOverview() is called + on success, getOverviewFailed() on failure + """ + self.sendLine('LIST OVERVIEW.FMT') + self._newState(self._stateOverview, self.getOverviewFailed) + + + def fetchSubscriptions(self): + """ + Request a list of the groups it is recommended a new user subscribe to. + gotSubscriptions() is called on success, getSubscriptionsFailed() on + failure + """ + self.sendLine('LIST SUBSCRIPTIONS') + self._newState(self._stateSubscriptions, self.getSubscriptionsFailed) + + + def fetchGroup(self, group): + """ + Get group information for the specified group from the server. gotGroup() + is called on success, getGroupFailed() on failure. + """ + self.sendLine('GROUP %s' % (group,)) + self._newState(None, self.getGroupFailed, self._headerGroup) + + + def fetchHead(self, index = ''): + """ + Get the header for the specified article (or the currently selected + article if index is '') from the server. gotHead() is called on + success, getHeadFailed() on failure + """ + self.sendLine('HEAD %s' % (index,)) + self._newState(self._stateHead, self.getHeadFailed) + + + def fetchBody(self, index = ''): + """ + Get the body for the specified article (or the currently selected + article if index is '') from the server. gotBody() is called on + success, getBodyFailed() on failure + """ + self.sendLine('BODY %s' % (index,)) + self._newState(self._stateBody, self.getBodyFailed) + + + def fetchArticle(self, index = ''): + """ + Get the complete article with the specified index (or the currently + selected article if index is '') or Message-ID from the server. + gotArticle() is called on success, getArticleFailed() on failure. + """ + self.sendLine('ARTICLE %s' % (index,)) + self._newState(self._stateArticle, self.getArticleFailed) + + + def postArticle(self, text): + """ + Attempt to post an article with the specified text to the server. 'text' + must consist of both head and body data, as specified by RFC 850. If the + article is posted successfully, postedOk() is called, otherwise postFailed() + is called. + """ + self.sendLine('POST') + self._newState(None, self.postFailed, self._headerPost) + self._postText.append(text) + + + def fetchNewNews(self, groups, date, distributions = ''): + """ + Get the Message-IDs for all new news posted to any of the given + groups since the specified date - in seconds since the epoch, GMT - + optionally restricted to the given distributions. gotNewNews() is + called on success, getNewNewsFailed() on failure. + + One invocation of this function may result in multiple invocations + of gotNewNews()/getNewNewsFailed(). + """ + date, timeStr = time.strftime('%y%m%d %H%M%S', time.gmtime(date)).split() + line = 'NEWNEWS %%s %s %s %s' % (date, timeStr, distributions) + groupPart = '' + while len(groups) and len(line) + len(groupPart) + len(groups[-1]) + 1 < NNTPClient.MAX_COMMAND_LENGTH: + group = groups.pop() + groupPart = groupPart + ',' + group + + self.sendLine(line % (groupPart,)) + self._newState(self._stateNewNews, self.getNewNewsFailed) + + if len(groups): + self.fetchNewNews(groups, date, distributions) + + + def fetchNewGroups(self, date, distributions): + """ + Get the names of all new groups created/added to the server since + the specified date - in seconds since the ecpoh, GMT - optionally + restricted to the given distributions. gotNewGroups() is called + on success, getNewGroupsFailed() on failure. + """ + date, timeStr = time.strftime('%y%m%d %H%M%S', time.gmtime(date)).split() + self.sendLine('NEWGROUPS %s %s %s' % (date, timeStr, distributions)) + self._newState(self._stateNewGroups, self.getNewGroupsFailed) + + + def fetchXHeader(self, header, low = None, high = None, id = None): + """ + Request a specific header from the server for an article or range + of articles. If 'id' is not None, a header for only the article + with that Message-ID will be requested. If both low and high are + None, a header for the currently selected article will be selected; + If both low and high are zero-length strings, headers for all articles + in the currently selected group will be requested; Otherwise, high + and low will be used as bounds - if one is None the first or last + article index will be substituted, as appropriate. + """ + if id is not None: + r = header + ' <%s>' % (id,) + elif low is high is None: + r = header + elif high is None: + r = header + ' %d-' % (low,) + elif low is None: + r = header + ' -%d' % (high,) + else: + r = header + ' %d-%d' % (low, high) + self.sendLine('XHDR ' + r) + self._newState(self._stateXHDR, self.getXHeaderFailed) + + + def setStream(self): + """ + Set the mode to STREAM, suspending the normal "lock-step" mode of + communications. setStreamSuccess() is called on success, + setStreamFailed() on failure. + """ + self.sendLine('MODE STREAM') + self._newState(None, self.setStreamFailed, self._headerMode) + + + def quit(self): + self.sendLine('QUIT') + self.transport.loseConnection() + + + def _newState(self, method, error, responseHandler = None): + self._inputBuffers.append([]) + self._responseCodes.append(None) + self._state.append(method) + self._error.append(error) + self._responseHandlers.append(responseHandler) + + + def _endState(self): + buf = self._inputBuffers[0] + del self._responseCodes[0] + del self._inputBuffers[0] + del self._state[0] + del self._error[0] + del self._responseHandlers[0] + return buf + + + def _newLine(self, line, check = 1): + if check and line and line[0] == '.': + line = line[1:] + self._inputBuffers[0].append(line) + + + def _setResponseCode(self, code): + self._responseCodes[0] = code + + + def _getResponseCode(self): + return self._responseCodes[0] + + + def lineReceived(self, line): + if not len(self._state): + self._statePassive(line) + elif self._getResponseCode() is None: + code = extractCode(line) + if code is None or not (200 <= code[0] < 400): # An error! + self._error[0](line) + self._endState() + else: + self._setResponseCode(code) + if self._responseHandlers[0]: + self._responseHandlers[0](code) + else: + self._state[0](line) + + + def _statePassive(self, line): + log.msg('Server said: %s' % line) + + + def _passiveError(self, error): + log.err('Passive Error: %s' % (error,)) + + + def _headerInitial(self, response): + (code, message) = response + if code == 200: + self.canPost = 1 + else: + self.canPost = 0 + self._endState() + + + def _stateList(self, line): + if line != '.': + data = filter(None, line.strip().split()) + self._newLine((data[0], int(data[1]), int(data[2]), data[3]), 0) + else: + self.gotAllGroups(self._endState()) + + + def _stateOverview(self, line): + if line != '.': + self._newLine(filter(None, line.strip().split()), 0) + else: + self.gotOverview(self._endState()) + + + def _stateSubscriptions(self, line): + if line != '.': + self._newLine(line.strip(), 0) + else: + self.gotSubscriptions(self._endState()) + + + def _headerGroup(self, response): + (code, line) = response + self.gotGroup(tuple(line.split())) + self._endState() + + + def _stateArticle(self, line): + if line != '.': + if line.startswith('.'): + line = line[1:] + self._newLine(line, 0) + else: + self.gotArticle('\n'.join(self._endState())+'\n') + + + def _stateHead(self, line): + if line != '.': + self._newLine(line, 0) + else: + self.gotHead('\n'.join(self._endState())) + + + def _stateBody(self, line): + if line != '.': + if line.startswith('.'): + line = line[1:] + self._newLine(line, 0) + else: + self.gotBody('\n'.join(self._endState())+'\n') + + + def _headerPost(self, response): + (code, message) = response + if code == 340: + self.transport.write(self._postText[0].replace('\n', '\r\n').replace('\r\n.', '\r\n..')) + if self._postText[0][-1:] != '\n': + self.sendLine('') + self.sendLine('.') + del self._postText[0] + self._newState(None, self.postFailed, self._headerPosted) + else: + self.postFailed('%d %s' % (code, message)) + self._endState() + + + def _headerPosted(self, response): + (code, message) = response + if code == 240: + self.postedOk() + else: + self.postFailed('%d %s' % (code, message)) + self._endState() + + + def _stateXHDR(self, line): + if line != '.': + self._newLine(line.split(), 0) + else: + self._gotXHeader(self._endState()) + + + def _stateNewNews(self, line): + if line != '.': + self._newLine(line, 0) + else: + self.gotNewNews(self._endState()) + + + def _stateNewGroups(self, line): + if line != '.': + self._newLine(line, 0) + else: + self.gotNewGroups(self._endState()) + + + def _headerMode(self, response): + (code, message) = response + if code == 203: + self.setStreamSuccess() + else: + self.setStreamFailed((code, message)) + self._endState() + + +class NNTPServer(basic.LineReceiver): + COMMANDS = [ + 'LIST', 'GROUP', 'ARTICLE', 'STAT', 'MODE', 'LISTGROUP', 'XOVER', + 'XHDR', 'HEAD', 'BODY', 'NEXT', 'LAST', 'POST', 'QUIT', 'IHAVE', + 'HELP', 'SLAVE', 'XPATH', 'XINDEX', 'XROVER', 'TAKETHIS', 'CHECK' + ] + + def __init__(self): + self.servingSlave = 0 + + + def connectionMade(self): + self.inputHandler = None + self.currentGroup = None + self.currentIndex = None + self.sendLine('200 server ready - posting allowed') + + def lineReceived(self, line): + if self.inputHandler is not None: + self.inputHandler(line) + else: + parts = line.strip().split() + if len(parts): + cmd, parts = parts[0].upper(), parts[1:] + if cmd in NNTPServer.COMMANDS: + func = getattr(self, 'do_%s' % cmd) + try: + func(*parts) + except TypeError: + self.sendLine('501 command syntax error') + log.msg("501 command syntax error") + log.msg("command was", line) + log.deferr() + except: + self.sendLine('503 program fault - command not performed') + log.msg("503 program fault") + log.msg("command was", line) + log.deferr() + else: + self.sendLine('500 command not recognized') + + + def do_LIST(self, subcmd = '', *dummy): + subcmd = subcmd.strip().lower() + if subcmd == 'newsgroups': + # XXX - this could use a real implementation, eh? + self.sendLine('215 Descriptions in form "group description"') + self.sendLine('.') + elif subcmd == 'overview.fmt': + defer = self.factory.backend.overviewRequest() + defer.addCallbacks(self._gotOverview, self._errOverview) + log.msg('overview') + elif subcmd == 'subscriptions': + defer = self.factory.backend.subscriptionRequest() + defer.addCallbacks(self._gotSubscription, self._errSubscription) + log.msg('subscriptions') + elif subcmd == '': + defer = self.factory.backend.listRequest() + defer.addCallbacks(self._gotList, self._errList) + else: + self.sendLine('500 command not recognized') + + + def _gotList(self, list): + self.sendLine('215 newsgroups in form "group high low flags"') + for i in list: + self.sendLine('%s %d %d %s' % tuple(i)) + self.sendLine('.') + + + def _errList(self, failure): + print('LIST failed: ', failure) + self.sendLine('503 program fault - command not performed') + + + def _gotSubscription(self, parts): + self.sendLine('215 information follows') + for i in parts: + self.sendLine(i) + self.sendLine('.') + + + def _errSubscription(self, failure): + print('SUBSCRIPTIONS failed: ', failure) + self.sendLine('503 program fault - comand not performed') + + + def _gotOverview(self, parts): + self.sendLine('215 Order of fields in overview database.') + for i in parts: + self.sendLine(i + ':') + self.sendLine('.') + + + def _errOverview(self, failure): + print('LIST OVERVIEW.FMT failed: ', failure) + self.sendLine('503 program fault - command not performed') + + + def do_LISTGROUP(self, group = None): + group = group or self.currentGroup + if group is None: + self.sendLine('412 Not currently in newsgroup') + else: + defer = self.factory.backend.listGroupRequest(group) + defer.addCallbacks(self._gotListGroup, self._errListGroup) + + + def _gotListGroup(self, result): + (group, articles) = result + self.currentGroup = group + if len(articles): + self.currentIndex = int(articles[0]) + else: + self.currentIndex = None + + self.sendLine('211 list of article numbers follow') + for i in articles: + self.sendLine(str(i)) + self.sendLine('.') + + + def _errListGroup(self, failure): + print('LISTGROUP failed: ', failure) + self.sendLine('502 no permission') + + + def do_XOVER(self, range): + if self.currentGroup is None: + self.sendLine('412 No news group currently selected') + else: + l, h = parseRange(range) + defer = self.factory.backend.xoverRequest(self.currentGroup, l, h) + defer.addCallbacks(self._gotXOver, self._errXOver) + + + def _gotXOver(self, parts): + self.sendLine('224 Overview information follows') + for i in parts: + self.sendLine('\t'.join(map(str, i))) + self.sendLine('.') + + + def _errXOver(self, failure): + print('XOVER failed: ', failure) + self.sendLine('420 No article(s) selected') + + + def xhdrWork(self, header, range): + if self.currentGroup is None: + self.sendLine('412 No news group currently selected') + else: + if range is None: + if self.currentIndex is None: + self.sendLine('420 No current article selected') + return + else: + l = h = self.currentIndex + else: + # FIXME: articles may be a message-id + l, h = parseRange(range) + + if l is h is None: + self.sendLine('430 no such article') + else: + return self.factory.backend.xhdrRequest(self.currentGroup, l, h, header) + + + def do_XHDR(self, header, range = None): + d = self.xhdrWork(header, range) + if d: + d.addCallbacks(self._gotXHDR, self._errXHDR) + + + def _gotXHDR(self, parts): + self.sendLine('221 Header follows') + for i in parts: + self.sendLine('%d %s' % i) + self.sendLine('.') + + def _errXHDR(self, failure): + print('XHDR failed: ', failure) + self.sendLine('502 no permission') + + + def do_POST(self): + self.inputHandler = self._doingPost + self.message = '' + self.sendLine('340 send article to be posted. End with .') + + + def _doingPost(self, line): + if line == '.': + self.inputHandler = None + article = self.message + self.message = '' + + defer = self.factory.backend.postRequest(article) + defer.addCallbacks(self._gotPost, self._errPost) + else: + self.message = self.message + line + '\r\n' + + + def _gotPost(self, parts): + self.sendLine('240 article posted ok') + + + def _errPost(self, failure): + print('POST failed: ', failure) + self.sendLine('441 posting failed') + + + def do_CHECK(self, id): + d = self.factory.backend.articleExistsRequest(id) + d.addCallbacks(self._gotCheck, self._errCheck) + + + def _gotCheck(self, result): + if result: + self.sendLine("438 already have it, please don't send it to me") + else: + self.sendLine('238 no such article found, please send it to me') + + + def _errCheck(self, failure): + print('CHECK failed: ', failure) + self.sendLine('431 try sending it again later') + + + def do_TAKETHIS(self, id): + self.inputHandler = self._doingTakeThis + self.message = '' + + + def _doingTakeThis(self, line): + if line == '.': + self.inputHandler = None + article = self.message + self.message = '' + d = self.factory.backend.postRequest(article) + d.addCallbacks(self._didTakeThis, self._errTakeThis) + else: + self.message = self.message + line + '\r\n' + + + def _didTakeThis(self, result): + self.sendLine('239 article transferred ok') + + + def _errTakeThis(self, failure): + print('TAKETHIS failed: ', failure) + self.sendLine('439 article transfer failed') + + + def do_GROUP(self, group): + defer = self.factory.backend.groupRequest(group) + defer.addCallbacks(self._gotGroup, self._errGroup) + + + def _gotGroup(self, result): + (name, num, high, low, flags) = result + self.currentGroup = name + self.currentIndex = low + self.sendLine('211 %d %d %d %s group selected' % (num, low, high, name)) + + + def _errGroup(self, failure): + print('GROUP failed: ', failure) + self.sendLine('411 no such group') + + + def articleWork(self, article, cmd, func): + if self.currentGroup is None: + self.sendLine('412 no newsgroup has been selected') + else: + if not article: + if self.currentIndex is None: + self.sendLine('420 no current article has been selected') + else: + article = self.currentIndex + else: + if article[0] == '<': + return func(self.currentGroup, index = None, id = article) + else: + try: + article = int(article) + return func(self.currentGroup, article) + except ValueError: + self.sendLine('501 command syntax error') + + + def do_ARTICLE(self, article = None): + defer = self.articleWork(article, 'ARTICLE', self.factory.backend.articleRequest) + if defer: + defer.addCallbacks(self._gotArticle, self._errArticle) + + + def _gotArticle(self, result): + (index, id, article) = result + self.currentIndex = index + self.sendLine('220 %d %s article' % (index, id)) + s = basic.FileSender() + d = s.beginFileTransfer(article, self.transport) + d.addCallback(self.finishedFileTransfer) + + ## + ## Helper for FileSender + ## + def finishedFileTransfer(self, lastsent): + if lastsent != '\n': + line = '\r\n.' + else: + line = '.' + self.sendLine(line) + ## + + def _errArticle(self, failure): + print('ARTICLE failed: ', failure) + self.sendLine('423 bad article number') + + + def do_STAT(self, article = None): + defer = self.articleWork(article, 'STAT', self.factory.backend.articleRequest) + if defer: + defer.addCallbacks(self._gotStat, self._errStat) + + + def _gotStat(self, result): + (index, id, article) = result + self.currentIndex = index + self.sendLine('223 %d %s article retreived - request text separately' % (index, id)) + + + def _errStat(self, failure): + print('STAT failed: ', failure) + self.sendLine('423 bad article number') + + + def do_HEAD(self, article = None): + defer = self.articleWork(article, 'HEAD', self.factory.backend.headRequest) + if defer: + defer.addCallbacks(self._gotHead, self._errHead) + + + def _gotHead(self, result): + (index, id, head) = result + self.currentIndex = index + self.sendLine('221 %d %s article retrieved' % (index, id)) + self.transport.write(head + '\r\n') + self.sendLine('.') + + + def _errHead(self, failure): + print('HEAD failed: ', failure) + self.sendLine('423 no such article number in this group') + + + def do_BODY(self, article): + defer = self.articleWork(article, 'BODY', self.factory.backend.bodyRequest) + if defer: + defer.addCallbacks(self._gotBody, self._errBody) + + + def _gotBody(self, result): + (index, id, body) = result + self.currentIndex = index + self.sendLine('221 %d %s article retrieved' % (index, id)) + self.lastsent = '' + s = basic.FileSender() + d = s.beginFileTransfer(body, self.transport) + d.addCallback(self.finishedFileTransfer) + + def _errBody(self, failure): + print('BODY failed: ', failure) + self.sendLine('423 no such article number in this group') + + + # NEXT and LAST are just STATs that increment currentIndex first. + # Accordingly, use the STAT callbacks. + def do_NEXT(self): + i = self.currentIndex + 1 + defer = self.factory.backend.articleRequest(self.currentGroup, i) + defer.addCallbacks(self._gotStat, self._errStat) + + + def do_LAST(self): + i = self.currentIndex - 1 + defer = self.factory.backend.articleRequest(self.currentGroup, i) + defer.addCallbacks(self._gotStat, self._errStat) + + + def do_MODE(self, cmd): + cmd = cmd.strip().upper() + if cmd == 'READER': + self.servingSlave = 0 + self.sendLine('200 Hello, you can post') + elif cmd == 'STREAM': + self.sendLine('500 Command not understood') + else: + # This is not a mistake + self.sendLine('500 Command not understood') + + + def do_QUIT(self): + self.sendLine('205 goodbye') + self.transport.loseConnection() + + + def do_HELP(self): + self.sendLine('100 help text follows') + self.sendLine('Read the RFC.') + self.sendLine('.') + + + def do_SLAVE(self): + self.sendLine('202 slave status noted') + self.servingeSlave = 1 + + + def do_XPATH(self, article): + # XPATH is a silly thing to have. No client has the right to ask + # for this piece of information from me, and so that is what I'll + # tell them. + self.sendLine('502 access restriction or permission denied') + + + def do_XINDEX(self, article): + # XINDEX is another silly command. The RFC suggests it be relegated + # to the history books, and who am I to disagree? + self.sendLine('502 access restriction or permission denied') + + + def do_XROVER(self, range=None): + """ + Handle a request for references of all messages in the currently + selected group. + + This generates the same response a I{XHDR References} request would + generate. + """ + self.do_XHDR('References', range) + + + def do_IHAVE(self, id): + self.factory.backend.articleExistsRequest(id).addCallback(self._foundArticle) + + + def _foundArticle(self, result): + if result: + self.sendLine('437 article rejected - do not try again') + else: + self.sendLine('335 send article to be transferred. End with .') + self.inputHandler = self._handleIHAVE + self.message = '' + + + def _handleIHAVE(self, line): + if line == '.': + self.inputHandler = None + self.factory.backend.postRequest( + self.message + ).addCallbacks(self._gotIHAVE, self._errIHAVE) + + self.message = '' + else: + self.message = self.message + line + '\r\n' + + + def _gotIHAVE(self, result): + self.sendLine('235 article transferred ok') + + + def _errIHAVE(self, failure): + print('IHAVE failed: ', failure) + self.sendLine('436 transfer failed - try again later') + + +class UsenetClientProtocol(NNTPClient): + """ + A client that connects to an NNTP server and asks for articles new + since a certain time. + """ + + def __init__(self, groups, date, storage): + """ + Fetch all new articles from the given groups since the + given date and dump them into the given storage. groups + is a list of group names. date is an integer or floating + point representing seconds since the epoch (GMT). storage is + any object that implements the NewsStorage interface. + """ + NNTPClient.__init__(self) + self.groups, self.date, self.storage = groups, date, storage + + + def connectionMade(self): + NNTPClient.connectionMade(self) + log.msg("Initiating update with remote host: " + str(self.transport.getPeer())) + self.setStream() + self.fetchNewNews(self.groups, self.date, '') + + + def articleExists(self, exists, article): + if exists: + self.fetchArticle(article) + else: + self.count = self.count - 1 + self.disregard = self.disregard + 1 + + + def gotNewNews(self, news): + self.disregard = 0 + self.count = len(news) + log.msg("Transferring " + str(self.count) + + " articles from remote host: " + str(self.transport.getPeer())) + for i in news: + self.storage.articleExistsRequest(i).addCallback(self.articleExists, i) + + + def getNewNewsFailed(self, reason): + log.msg("Updated failed (" + reason + ") with remote host: " + str(self.transport.getPeer())) + self.quit() + + + def gotArticle(self, article): + self.storage.postRequest(article) + self.count = self.count - 1 + if not self.count: + log.msg("Completed update with remote host: " + str(self.transport.getPeer())) + if self.disregard: + log.msg("Disregarded %d articles." % (self.disregard,)) + self.factory.updateChecks(self.transport.getPeer()) + self.quit() diff --git a/contrib/python/Twisted/py2/twisted/news/tap.py b/contrib/python/Twisted/py2/twisted/news/tap.py new file mode 100644 index 00000000000..d6b82934142 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/news/tap.py @@ -0,0 +1,143 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +from __future__ import print_function + +from twisted.news import news, database +from twisted.application import strports +from twisted.python import usage, log + +class DBOptions(usage.Options): + optParameters = [ + ['module', None, 'pyPgSQL.PgSQL', "DB-API 2.0 module to use"], + ['dbhost', None, 'localhost', "Host where database manager is listening"], + ['dbuser', None, 'news', "Username with which to connect to database"], + ['database', None, 'news', "Database name to use"], + ['schema', None, 'schema.sql', "File to which to write SQL schema initialisation"], + + # XXX - Hrm. + ["groups", "g", "groups.list", "File containing group list"], + ["servers", "s", "servers.list", "File containing server list"] + ] + + def postOptions(self): + # XXX - Hmmm. + with open(self['groups']) as f: + self['groups'] = [g.strip() for g in f.readlines() if not g.startswith('#')] + with open(self['servers']) as f: + self['servers'] = [s.strip() for s in f.readlines() if not s.startswith('#')] + + try: + __import__(self['module']) + except ImportError: + log.msg("Warning: Cannot import %s" % (self['module'],)) + + with open(self['schema'], 'w') as f: + f.write( + database.NewsStorageAugmentation.schema + '\n' + + database.makeGroupSQL(self['groups']) + '\n' + + database.makeOverviewSQL() + ) + + info = { + 'host': self['dbhost'], 'user': self['dbuser'], + 'database': self['database'], 'dbapiName': self['module'] + } + self.db = database.NewsStorageAugmentation(info) + + +class PickleOptions(usage.Options): + optParameters = [ + ['file', None, 'news.pickle', "File to which to save pickle"], + + # XXX - Hrm. + ["groups", "g", "groups.list", "File containing group list"], + ["servers", "s", "servers.list", "File containing server list"], + ["moderators", "m", "moderators.list", + "File containing moderators list"], + ] + + subCommands = None + + def postOptions(self): + # XXX - Hmmm. + filename = self['file'] + with open(self['groups']) as f: + self['groups'] = [g.strip() for g in f.readlines() + if not g.startswith('#')] + with open(self['servers']) as f: + self['servers'] = [s.strip() for s in f.readlines() + if not s.startswith('#')] + with open(self['moderators']) as f: + self['moderators'] = [s.split() for s in f.readlines() + if not s.startswith('#')] + self.db = database.PickleStorage(filename, self['groups'], + self['moderators']) + + +class Options(usage.Options): + synopsis = "[options]" + + groups = None + servers = None + subscriptions = None + + optParameters = [ + ["port", "p", "119", "Listen port"], + ["interface", "i", "", "Interface to which to bind"], + ["datadir", "d", "news.db", "Root data storage path"], + ["mailhost", "m", "localhost", "Host of SMTP server to use"] + ] + compData = usage.Completions( + optActions={"datadir" : usage.CompleteDirs(), + "mailhost" : usage.CompleteHostnames(), + "interface" : usage.CompleteNetInterfaces()} + ) + + def __init__(self): + usage.Options.__init__(self) + self.groups = [] + self.servers = [] + self.subscriptions = [] + + + def opt_group(self, group): + """The name of a newsgroup to carry.""" + self.groups.append([group, None]) + + + def opt_moderator(self, moderator): + """The email of the moderator for the most recently passed group.""" + self.groups[-1][1] = moderator + + + def opt_subscription(self, group): + """A newsgroup to list as a recommended subscription.""" + self.subscriptions.append(group) + + + def opt_server(self, server): + """The address of a Usenet server to pass messages to and receive messages from.""" + self.servers.append(server) + + +def makeService(config): + if not len(config.groups): + raise usage.UsageError("No newsgroups specified") + + db = database.NewsShelf(config['mailhost'], config['datadir']) + for (g, m) in config.groups: + if m: + db.addGroup(g, 'm') + db.addModerator(g, m) + else: + db.addGroup(g, 'y') + for s in config.subscriptions: + print(s) + db.addSubscription(s) + s = config['port'] + if config['interface']: + # Add a warning here + s += ':interface='+config['interface'] + return strports.service(s, news.UsenetServerFactory(db, config.servers)) diff --git a/contrib/python/Twisted/py2/twisted/pair/__init__.py b/contrib/python/Twisted/py2/twisted/pair/__init__.py new file mode 100644 index 00000000000..09fb3dc85db --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/pair/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Pair: The framework of your ethernet. + +Low-level networking transports and utilities. + +See also twisted.protocols.ethernet, twisted.protocols.ip, +twisted.protocols.raw and twisted.protocols.rawudp. + +Maintainer: Tommi Virtanen +""" diff --git a/contrib/python/Twisted/py2/twisted/pair/ethernet.py b/contrib/python/Twisted/py2/twisted/pair/ethernet.py new file mode 100644 index 00000000000..7c7f9a89965 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/pair/ethernet.py @@ -0,0 +1,56 @@ +# -*- test-case-name: twisted.pair.test.test_ethernet -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + + +"""Support for working directly with ethernet frames""" + +import struct + + +from twisted.internet import protocol +from twisted.pair import raw +from zope.interface import implementer, Interface + + +class IEthernetProtocol(Interface): + """An interface for protocols that handle Ethernet frames""" + def addProto(): + """Add an IRawPacketProtocol protocol""" + + def datagramReceived(): + """An Ethernet frame has been received""" + +class EthernetHeader: + def __init__(self, data): + + (self.dest, self.source, self.proto) \ + = struct.unpack("!6s6sH", data[:6+6+2]) + + + +@implementer(IEthernetProtocol) +class EthernetProtocol(protocol.AbstractDatagramProtocol): + def __init__(self): + self.etherProtos = {} + + def addProto(self, num, proto): + proto = raw.IRawPacketProtocol(proto) + if num < 0: + raise TypeError('Added protocol must be positive or zero') + if num >= 2**16: + raise TypeError('Added protocol must fit in 16 bits') + if num not in self.etherProtos: + self.etherProtos[num] = [] + self.etherProtos[num].append(proto) + + def datagramReceived(self, data, partial=0): + header = EthernetHeader(data[:14]) + for proto in self.etherProtos.get(header.proto, ()): + proto.datagramReceived(data=data[14:], + partial=partial, + dest=header.dest, + source=header.source, + protocol=header.proto) diff --git a/contrib/python/Twisted/py2/twisted/pair/ip.py b/contrib/python/Twisted/py2/twisted/pair/ip.py new file mode 100644 index 00000000000..47db7075d7f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/pair/ip.py @@ -0,0 +1,71 @@ +# -*- test-case-name: twisted.pair.test.test_ip -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + + +"""Support for working directly with IP packets""" + +import struct +import socket + +from twisted.internet import protocol +from twisted.pair import raw +from zope.interface import implementer + + +class IPHeader: + def __init__(self, data): + + (ihlversion, self.tos, self.tot_len, self.fragment_id, frag_off, + self.ttl, self.protocol, self.check, saddr, daddr) \ + = struct.unpack("!BBHHHBBH4s4s", data[:20]) + self.saddr = socket.inet_ntoa(saddr) + self.daddr = socket.inet_ntoa(daddr) + self.version = ihlversion & 0x0F + self.ihl = ((ihlversion & 0xF0) >> 4) << 2 + self.fragment_offset = frag_off & 0x1FFF + self.dont_fragment = (frag_off & 0x4000 != 0) + self.more_fragments = (frag_off & 0x2000 != 0) + +MAX_SIZE = 2**32 + +@implementer(raw.IRawPacketProtocol) +class IPProtocol(protocol.AbstractDatagramProtocol): + def __init__(self): + self.ipProtos = {} + + def addProto(self, num, proto): + proto = raw.IRawDatagramProtocol(proto) + if num < 0: + raise TypeError('Added protocol must be positive or zero') + if num >= MAX_SIZE: + raise TypeError('Added protocol must fit in 32 bits') + if num not in self.ipProtos: + self.ipProtos[num] = [] + self.ipProtos[num].append(proto) + + def datagramReceived(self, + data, + partial, + dest, + source, + protocol): + header = IPHeader(data) + for proto in self.ipProtos.get(header.protocol, ()): + proto.datagramReceived(data=data[20:], + partial=partial, + source=header.saddr, + dest=header.daddr, + protocol=header.protocol, + version=header.version, + ihl=header.ihl, + tos=header.tos, + tot_len=header.tot_len, + fragment_id=header.fragment_id, + fragment_offset=header.fragment_offset, + dont_fragment=header.dont_fragment, + more_fragments=header.more_fragments, + ttl=header.ttl, + ) diff --git a/contrib/python/Twisted/py2/twisted/pair/raw.py b/contrib/python/Twisted/py2/twisted/pair/raw.py new file mode 100644 index 00000000000..ed859db957d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/pair/raw.py @@ -0,0 +1,40 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +""" +Interface definitions for working with raw packets +""" + +from zope.interface import Interface + + +class IRawDatagramProtocol(Interface): + """ + An interface for protocols such as UDP, ICMP and TCP. + """ + + def addProto(): + """ + Add a protocol on top of this one. + """ + + def datagramReceived(): + """ + An IP datagram has been received. Parse and process it. + """ + + + +class IRawPacketProtocol(Interface): + """ + An interface for low-level protocols such as IP and ARP. + """ + + def addProto(): + """ + Add a protocol on top of this one. + """ + + def datagramReceived(): + """ + An IP datagram has been received. Parse and process it. + """ diff --git a/contrib/python/Twisted/py2/twisted/pair/rawudp.py b/contrib/python/Twisted/py2/twisted/pair/rawudp.py new file mode 100644 index 00000000000..f52e4174257 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/pair/rawudp.py @@ -0,0 +1,59 @@ +# -*- test-case-name: twisted.pair.test.test_rawudp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of raw packet interfaces for UDP +""" + +import struct + +from twisted.internet import protocol +from twisted.pair import raw +from zope.interface import implementer + +class UDPHeader: + def __init__(self, data): + + (self.source, self.dest, self.len, self.check) \ + = struct.unpack("!HHHH", data[:8]) + + + +@implementer(raw.IRawDatagramProtocol) +class RawUDPProtocol(protocol.AbstractDatagramProtocol): + def __init__(self): + self.udpProtos = {} + + + def addProto(self, num, proto): + if not isinstance(proto, protocol.DatagramProtocol): + raise TypeError('Added protocol must be an instance of DatagramProtocol') + if num < 0: + raise TypeError('Added protocol must be positive or zero') + if num >= 2**16: + raise TypeError('Added protocol must fit in 16 bits') + if num not in self.udpProtos: + self.udpProtos[num] = [] + self.udpProtos[num].append(proto) + + + def datagramReceived(self, + data, + partial, + source, + dest, + protocol, + version, + ihl, + tos, + tot_len, + fragment_id, + fragment_offset, + dont_fragment, + more_fragments, + ttl): + header = UDPHeader(data) + for proto in self.udpProtos.get(header.dest, ()): + proto.datagramReceived(data[8:], + (source, header.source)) diff --git a/contrib/python/Twisted/py2/twisted/pair/testing.py b/contrib/python/Twisted/py2/twisted/pair/testing.py new file mode 100644 index 00000000000..b20136bdd14 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/pair/testing.py @@ -0,0 +1,572 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Tools for automated testing of L{twisted.pair}-based applications. +""" + +import struct +import socket +from errno import ( + EPERM, EAGAIN, EWOULDBLOCK, ENOSYS, EBADF, EINVAL, EINTR, ENOBUFS) +from collections import deque +from functools import wraps + +from zope.interface import implementer + +from twisted.internet.protocol import DatagramProtocol +from twisted.pair.ethernet import EthernetProtocol +from twisted.pair.rawudp import RawUDPProtocol +from twisted.pair.ip import IPProtocol +from twisted.pair.tuntap import ( + _IFNAMSIZ, _TUNSETIFF, _IInputOutputSystem, TunnelFlags) +from twisted.python.compat import nativeString + + +# The number of bytes in the "protocol information" header that may be present +# on datagrams read from a tunnel device. This is two bytes of flags followed +# by two bytes of protocol identification. All this code does with this +# information is use it to discard the header. +_PI_SIZE = 4 + + +def _H(n): + """ + Pack an integer into a network-order two-byte string. + + @param n: The integer to pack. Only values that fit into 16 bits are + supported. + + @return: The packed representation of the integer. + @rtype: L{bytes} + """ + return struct.pack('>H', n) + + +_IPv4 = 0x0800 + + +def _ethernet(src, dst, protocol, payload): + """ + Construct an ethernet frame. + + @param src: The source ethernet address, encoded. + @type src: L{bytes} + + @param dst: The destination ethernet address, encoded. + @type dst: L{bytes} + + @param protocol: The protocol number of the payload of this datagram. + @type protocol: L{int} + + @param payload: The content of the ethernet frame (such as an IP datagram). + @type payload: L{bytes} + + @return: The full ethernet frame. + @rtype: L{bytes} + """ + return dst + src + _H(protocol) + payload + + + +def _ip(src, dst, payload): + """ + Construct an IP datagram with the given source, destination, and + application payload. + + @param src: The source IPv4 address as a dotted-quad string. + @type src: L{bytes} + + @param dst: The destination IPv4 address as a dotted-quad string. + @type dst: L{bytes} + + @param payload: The content of the IP datagram (such as a UDP datagram). + @type payload: L{bytes} + + @return: An IP datagram header and payload. + @rtype: L{bytes} + """ + ipHeader = ( + # Version and header length, 4 bits each + b'\x45' + # Differentiated services field + b'\x00' + # Total length + + _H(20 + len(payload)) + + b'\x00\x01\x00\x00\x40\x11' + # Checksum + + _H(0) + # Source address + + socket.inet_pton(socket.AF_INET, nativeString(src)) + # Destination address + + socket.inet_pton(socket.AF_INET, nativeString(dst))) + + # Total all of the 16-bit integers in the header + checksumStep1 = sum(struct.unpack('!10H', ipHeader)) + # Pull off the carry + carry = checksumStep1 >> 16 + # And add it to what was left over + checksumStep2 = (checksumStep1 & 0xFFFF) + carry + # Compute the one's complement sum + checksumStep3 = checksumStep2 ^ 0xFFFF + + # Reconstruct the IP header including the correct checksum so the platform + # IP stack, if there is one involved in this test, doesn't drop it on the + # floor as garbage. + ipHeader = ( + ipHeader[:10] + + struct.pack('!H', checksumStep3) + + ipHeader[12:]) + + return ipHeader + payload + + + +def _udp(src, dst, payload): + """ + Construct a UDP datagram with the given source, destination, and + application payload. + + @param src: The source port number. + @type src: L{int} + + @param dst: The destination port number. + @type dst: L{int} + + @param payload: The content of the UDP datagram. + @type payload: L{bytes} + + @return: A UDP datagram header and payload. + @rtype: L{bytes} + """ + udpHeader = ( + # Source port + _H(src) + # Destination port + + _H(dst) + # Length + + _H(len(payload) + 8) + # Checksum + + _H(0)) + return udpHeader + payload + + + +class Tunnel(object): + """ + An in-memory implementation of a tun or tap device. + + @cvar _DEVICE_NAME: A string representing the conventional filesystem entry + for the tunnel factory character special device. + @type _DEVICE_NAME: C{bytes} + """ + _DEVICE_NAME = b"/dev/net/tun" + + # Between POSIX and Python, there are 4 combinations. Here are two, at + # least. + EAGAIN_STYLE = IOError(EAGAIN, "Resource temporarily unavailable") + EWOULDBLOCK_STYLE = OSError(EWOULDBLOCK, "Operation would block") + + # Oh yea, and then there's the case where maybe we would've read, but + # someone sent us a signal instead. + EINTR_STYLE = IOError(EINTR, "Interrupted function call") + + nonBlockingExceptionStyle = EAGAIN_STYLE + + SEND_BUFFER_SIZE = 1024 + + def __init__(self, system, openFlags, fileMode): + """ + @param system: An L{_IInputOutputSystem} provider to use to perform I/O. + + @param openFlags: Any flags to apply when opening the tunnel device. + See C{os.O_*}. + + @type openFlags: L{int} + + @param fileMode: ignored + """ + self.system = system + + # Drop fileMode on the floor - evidence and logic suggest it is + # irrelevant with respect to /dev/net/tun + self.openFlags = openFlags + self.tunnelMode = None + self.requestedName = None + self.name = None + self.readBuffer = deque() + self.writeBuffer = deque() + self.pendingSignals = deque() + + + @property + def blocking(self): + """ + If the file descriptor for this tunnel is open in blocking mode, + C{True}. C{False} otherwise. + """ + return not (self.openFlags & self.system.O_NONBLOCK) + + + @property + def closeOnExec(self): + """ + If the file descriptor for this tunnel is marked as close-on-exec, + C{True}. C{False} otherwise. + """ + return bool(self.openFlags & self.system.O_CLOEXEC) + + + def addToReadBuffer(self, datagram): + """ + Deliver a datagram to this tunnel's read buffer. This makes it + available to be read later using the C{read} method. + + @param datagram: The IPv4 datagram to deliver. If the mode of this + tunnel is TAP then ethernet framing will be added automatically. + @type datagram: L{bytes} + """ + # TAP devices also include ethernet framing. + if self.tunnelMode & TunnelFlags.IFF_TAP.value: + datagram = _ethernet( + src=b'\x00' * 6, dst=b'\xff' * 6, protocol=_IPv4, + payload=datagram) + + self.readBuffer.append(datagram) + + + def read(self, limit): + """ + Read a datagram out of this tunnel. + + @param limit: The maximum number of bytes from the datagram to return. + If the next datagram is larger than this, extra bytes are dropped + and lost forever. + @type limit: L{int} + + @raise OSError: Any of the usual I/O problems can result in this + exception being raised with some particular error number set. + + @raise IOError: Any of the usual I/O problems can result in this + exception being raised with some particular error number set. + + @return: The datagram which was read from the tunnel. If the tunnel + mode does not include L{TunnelFlags.IFF_NO_PI} then the datagram is + prefixed with a 4 byte PI header. + @rtype: L{bytes} + """ + if self.readBuffer: + if self.tunnelMode & TunnelFlags.IFF_NO_PI.value: + header = b"" + else: + # Synthesize a PI header to include in the result. Nothing in + # twisted.pair uses the PI information yet so we can synthesize + # something incredibly boring (ie 32 bits of 0). + header = b"\x00" * _PI_SIZE + limit -= 4 + return header + self.readBuffer.popleft()[:limit] + elif self.blocking: + raise NotImplementedError() + else: + raise self.nonBlockingExceptionStyle + + + def write(self, datagram): + """ + Write a datagram into this tunnel. + + @param datagram: The datagram to write. + @type datagram: L{bytes} + + @raise IOError: Any of the usual I/O problems can result in this + exception being raised with some particular error number set. + + @return: The number of bytes of the datagram which were written. + @rtype: L{int} + """ + if self.pendingSignals: + self.pendingSignals.popleft() + raise IOError(EINTR, "Interrupted system call") + + if len(datagram) > self.SEND_BUFFER_SIZE: + raise IOError(ENOBUFS, "No buffer space available") + + self.writeBuffer.append(datagram) + return len(datagram) + + + +def _privileged(original): + """ + Wrap a L{MemoryIOSystem} method with permission-checking logic. The + returned function will check C{self.permissions} and raise L{IOError} with + L{errno.EPERM} if the function name is not listed as an available + permission. + + @param original: The L{MemoryIOSystem} instance to wrap. + + @return: A wrapper around C{original} that applies permission checks. + """ + @wraps(original) + def permissionChecker(self, *args, **kwargs): + if original.__name__ not in self.permissions: + raise IOError(EPERM, "Operation not permitted") + return original(self, *args, **kwargs) + return permissionChecker + + + +@implementer(_IInputOutputSystem) +class MemoryIOSystem(object): + """ + An in-memory implementation of basic I/O primitives, useful in the context + of unit testing as a drop-in replacement for parts of the C{os} module. + + @ivar _devices: + @ivar _openFiles: + @ivar permissions: + + @ivar _counter: + """ + _counter = 8192 + + O_RDWR = 1 << 0 + O_NONBLOCK = 1 << 1 + O_CLOEXEC = 1 << 2 + + def __init__(self): + self._devices = {} + self._openFiles = {} + self.permissions = set(['open', 'ioctl']) + + + def getTunnel(self, port): + """ + Get the L{Tunnel} object associated with the given L{TuntapPort}. + + @param port: A L{TuntapPort} previously initialized using this + L{MemoryIOSystem}. + + @return: The tunnel object created by a prior use of C{open} on this + object on the tunnel special device file. + @rtype: L{Tunnel} + """ + return self._openFiles[port.fileno()] + + + def registerSpecialDevice(self, name, cls): + """ + Specify a class which will be used to handle I/O to a device of a + particular name. + + @param name: The filesystem path name of the device. + @type name: L{bytes} + + @param cls: A class (like L{Tunnel}) to instantiated whenever this + device is opened. + """ + self._devices[name] = cls + + + @_privileged + def open(self, name, flags, mode=None): + """ + A replacement for C{os.open}. This initializes state in this + L{MemoryIOSystem} which will be reflected in the behavior of the other + file descriptor-related methods (eg L{MemoryIOSystem.read}, + L{MemoryIOSystem.write}, etc). + + @param name: A string giving the name of the file to open. + @type name: C{bytes} + + @param flags: The flags with which to open the file. + @type flags: C{int} + + @param mode: The mode with which to open the file. + @type mode: C{int} + + @raise OSError: With C{ENOSYS} if the file is not a recognized special + device file. + + @return: A file descriptor associated with the newly opened file + description. + @rtype: L{int} + """ + if name in self._devices: + fd = self._counter + self._counter += 1 + self._openFiles[fd] = self._devices[name](self, flags, mode) + return fd + raise OSError(ENOSYS, "Function not implemented") + + + def read(self, fd, limit): + """ + Try to read some bytes out of one of the in-memory buffers which may + previously have been populated by C{write}. + + @see: L{os.read} + """ + try: + return self._openFiles[fd].read(limit) + except KeyError: + raise OSError(EBADF, "Bad file descriptor") + + + def write(self, fd, data): + """ + Try to add some bytes to one of the in-memory buffers to be accessed by + a later C{read} call. + + @see: L{os.write} + """ + try: + return self._openFiles[fd].write(data) + except KeyError: + raise OSError(EBADF, "Bad file descriptor") + + + def close(self, fd): + """ + Discard the in-memory buffer and other in-memory state for the given + file descriptor. + + @see: L{os.close} + """ + try: + del self._openFiles[fd] + except KeyError: + raise OSError(EBADF, "Bad file descriptor") + + + @_privileged + def ioctl(self, fd, request, args): + """ + Perform some configuration change to the in-memory state for the given + file descriptor. + + @see: L{fcntl.ioctl} + """ + try: + tunnel = self._openFiles[fd] + except KeyError: + raise IOError(EBADF, "Bad file descriptor") + + if request != _TUNSETIFF: + raise IOError(EINVAL, "Request or args is not valid.") + + name, mode = struct.unpack('%dsH' % (_IFNAMSIZ,), args) + tunnel.tunnelMode = mode + tunnel.requestedName = name + tunnel.name = name[:_IFNAMSIZ - 3] + b"123" + + return struct.pack('%dsH' % (_IFNAMSIZ,), tunnel.name, mode) + + + def sendUDP(self, datagram, address): + """ + Write an ethernet frame containing an ip datagram containing a udp + datagram containing the given payload, addressed to the given address, + to a tunnel device previously opened on this I/O system. + + @param datagram: A UDP datagram payload to send. + @type datagram: L{bytes} + + @param address: The destination to which to send the datagram. + @type address: L{tuple} of (L{bytes}, L{int}) + + @return: A two-tuple giving the address from which gives the address + from which the datagram was sent. + @rtype: L{tuple} of (L{bytes}, L{int}) + """ + # Just make up some random thing + srcIP = '10.1.2.3' + srcPort = 21345 + + serialized = _ip( + src=srcIP, dst=address[0], payload=_udp( + src=srcPort, dst=address[1], payload=datagram)) + + openFiles = list(self._openFiles.values()) + openFiles[0].addToReadBuffer(serialized) + + return (srcIP, srcPort) + + + def receiveUDP(self, fileno, host, port): + """ + Get a socket-like object which can be used to receive a datagram sent + from the given address. + + @param fileno: A file descriptor representing a tunnel device which the + datagram will be received via. + @type fileno: L{int} + + @param host: The IPv4 address to which the datagram was sent. + @type host: L{bytes} + + @param port: The UDP port number to which the datagram was sent. + received. + @type port: L{int} + + @return: A L{socket.socket}-like object which can be used to receive + the specified datagram. + """ + return _FakePort(self, fileno) + + + +class _FakePort(object): + """ + A socket-like object which can be used to read UDP datagrams from + tunnel-like file descriptors managed by a L{MemoryIOSystem}. + """ + def __init__(self, system, fileno): + self._system = system + self._fileno = fileno + + + def recv(self, nbytes): + """ + Receive a datagram sent to this port using the L{MemoryIOSystem} which + created this object. + + This behaves like L{socket.socket.recv} but the data being I{sent} and + I{received} only passes through various memory buffers managed by this + object and L{MemoryIOSystem}. + + @see: L{socket.socket.recv} + """ + data = self._system._openFiles[self._fileno].writeBuffer.popleft() + + datagrams = [] + receiver = DatagramProtocol() + + def capture(datagram, address): + datagrams.append(datagram) + + receiver.datagramReceived = capture + + udp = RawUDPProtocol() + udp.addProto(12345, receiver) + + ip = IPProtocol() + ip.addProto(17, udp) + + mode = self._system._openFiles[self._fileno].tunnelMode + if (mode & TunnelFlags.IFF_TAP.value): + ether = EthernetProtocol() + ether.addProto(0x800, ip) + datagramReceived = ether.datagramReceived + else: + datagramReceived = lambda data: ip.datagramReceived( + data, None, None, None, None) + + dataHasPI = not (mode & TunnelFlags.IFF_NO_PI.value) + + if dataHasPI: + # datagramReceived can't handle the PI, get rid of it. + data = data[_PI_SIZE:] + + datagramReceived(data) + return datagrams[0][:nbytes] diff --git a/contrib/python/Twisted/py2/twisted/pair/tuntap.py b/contrib/python/Twisted/py2/twisted/pair/tuntap.py new file mode 100644 index 00000000000..69b3f9bfffc --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/pair/tuntap.py @@ -0,0 +1,433 @@ +# -*- test-case-name: twisted.pair.test.test_tuntap -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for Linux ethernet and IP tunnel devices. + +@see: U{https://en.wikipedia.org/wiki/TUN/TAP} +""" + +import os +import fcntl +import errno +import struct +import warnings + +from collections import namedtuple +from constantly import Flags, FlagConstant +from zope.interface import Attribute, Interface, implementer + +from twisted.python.util import FancyEqMixin, FancyStrMixin +from incremental import Version +from twisted.python.reflect import fullyQualifiedName +from twisted.python.deprecate import deprecated +from twisted.python import log +from twisted.internet import abstract, error, task, interfaces, defer +from twisted.pair import ethernet, raw + +__all__ = [ + "TunnelFlags", "TunnelAddress", "TuntapPort", + ] + + +_IFNAMSIZ = 16 +_TUNSETIFF = 0x400454ca +_TUNGETIFF = 0x800454d2 +_TUN_KO_PATH = b"/dev/net/tun" + + +class TunnelFlags(Flags): + """ + L{TunnelFlags} defines more flags which are used to configure the behavior + of a tunnel device. + + @cvar IFF_TUN: This indicates a I{tun}-type device. This type of tunnel + carries IP datagrams. This flag is mutually exclusive with C{IFF_TAP}. + + @cvar IFF_TAP: This indicates a I{tap}-type device. This type of tunnel + carries ethernet frames. This flag is mutually exclusive with C{IFF_TUN}. + + @cvar IFF_NO_PI: This indicates the I{protocol information} header will + B{not} be included in data read from the tunnel. + + @see: U{https://www.kernel.org/doc/Documentation/networking/tuntap.txt} + """ + IFF_TUN = FlagConstant(0x0001) + IFF_TAP = FlagConstant(0x0002) + + TUN_FASYNC = FlagConstant(0x0010) + TUN_NOCHECKSUM = FlagConstant(0x0020) + TUN_NO_PI = FlagConstant(0x0040) + TUN_ONE_QUEUE = FlagConstant(0x0080) + TUN_PERSIST = FlagConstant(0x0100) + TUN_VNET_HDR = FlagConstant(0x0200) + + IFF_NO_PI = FlagConstant(0x1000) + IFF_ONE_QUEUE = FlagConstant(0x2000) + IFF_VNET_HDR = FlagConstant(0x4000) + IFF_TUN_EXCL = FlagConstant(0x8000) + + + +@implementer(interfaces.IAddress) +class TunnelAddress(FancyStrMixin, FancyEqMixin, object): + """ + A L{TunnelAddress} represents the tunnel to which a L{TuntapPort} is bound. + """ + compareAttributes = ("_typeValue", "name") + showAttributes = (("type", lambda flag: flag.name), "name") + + @property + def _typeValue(self): + """ + Return the integer value of the C{type} attribute. Used to produce + correct results in the equality implementation. + """ + # Work-around for https://twistedmatrix.com/trac/ticket/6878 + return self.type.value + + + def __init__(self, type, name): + """ + @param type: Either L{TunnelFlags.IFF_TUN} or L{TunnelFlags.IFF_TAP}, + representing the type of this tunnel. + + @param name: The system name of the tunnel. + @type name: L{bytes} + """ + self.type = type + self.name = name + + + def __getitem__(self, index): + """ + Deprecated accessor for the tunnel name. Use attributes instead. + """ + warnings.warn( + "TunnelAddress.__getitem__ is deprecated since Twisted 14.0.0 " + "Use attributes instead.", category=DeprecationWarning, + stacklevel=2) + return ('TUNTAP', self.name)[index] + + + +class _TunnelDescription(namedtuple("_TunnelDescription", "fileno name")): + """ + Describe an existing tunnel. + + @ivar fileno: the file descriptor associated with the tunnel + @type fileno: L{int} + + @ivar name: the name of the tunnel + @type name: L{bytes} + """ + + + +class _IInputOutputSystem(Interface): + """ + An interface for performing some basic kinds of I/O (particularly that I/O + which might be useful for L{twisted.pair.tuntap}-using code). + """ + O_RDWR = Attribute("@see: L{os.O_RDWR}") + O_NONBLOCK = Attribute("@see: L{os.O_NONBLOCK}") + O_CLOEXEC = Attribute("@see: L{os.O_CLOEXEC}") + + def open(filename, flag, mode=0o777): + """ + @see: L{os.open} + """ + + + def ioctl(fd, opt, arg=None, mutate_flag=None): + """ + @see: L{fcntl.ioctl} + """ + + + def read(fd, limit): + """ + @see: L{os.read} + """ + + + def write(fd, data): + """ + @see: L{os.write} + """ + + + def close(fd): + """ + @see: L{os.close} + """ + + + def sendUDP(datagram, address): + """ + Send a datagram to a certain address. + + @param datagram: The payload of a UDP datagram to send. + @type datagram: L{bytes} + + @param address: The destination to which to send the datagram. + @type address: L{tuple} of (L{bytes}, L{int}) + + @return: The local address from which the datagram was sent. + @rtype: L{tuple} of (L{bytes}, L{int}) + """ + + + def receiveUDP(fileno, host, port): + """ + Return a socket which can be used to receive datagrams sent to the + given address. + + @param fileno: A file descriptor representing a tunnel device which the + datagram was either sent via or will be received via. + @type fileno: L{int} + + @param host: The IPv4 address at which the datagram will be received. + @type host: L{bytes} + + @param port: The UDP port number at which the datagram will be + received. + @type port: L{int} + + @return: A L{socket.socket} which can be used to receive the specified + datagram. + """ + + + +class _RealSystem(object): + """ + An interface to the parts of the operating system which L{TuntapPort} + relies on. This is most of an implementation of L{_IInputOutputSystem}. + """ + open = staticmethod(os.open) + read = staticmethod(os.read) + write = staticmethod(os.write) + close = staticmethod(os.close) + ioctl = staticmethod(fcntl.ioctl) + + O_RDWR = os.O_RDWR + O_NONBLOCK = os.O_NONBLOCK + # Introduced in Python 3.x + # Ubuntu 12.04, /usr/include/x86_64-linux-gnu/bits/fcntl.h + O_CLOEXEC = getattr(os, "O_CLOEXEC", 0o2000000) + + + +@implementer(interfaces.IListeningPort) +class TuntapPort(abstract.FileDescriptor): + """ + A Port that reads and writes packets from/to a TUN/TAP-device. + """ + maxThroughput = 256 * 1024 # Max bytes we read in one eventloop iteration + + def __init__(self, interface, proto, maxPacketSize=8192, reactor=None, + system=None): + if ethernet.IEthernetProtocol.providedBy(proto): + self.ethernet = 1 + self._mode = TunnelFlags.IFF_TAP + else: + self.ethernet = 0 + self._mode = TunnelFlags.IFF_TUN + assert raw.IRawPacketProtocol.providedBy(proto) + + if system is None: + system = _RealSystem() + self._system = system + + abstract.FileDescriptor.__init__(self, reactor) + self.interface = interface + self.protocol = proto + self.maxPacketSize = maxPacketSize + + logPrefix = self._getLogPrefix(self.protocol) + self.logstr = "%s (%s)" % (logPrefix, self._mode.name) + + + def __repr__(self): + args = (fullyQualifiedName(self.protocol.__class__),) + if self.connected: + args = args + ("",) + else: + args = args + ("not ",) + args = args + (self._mode.name, self.interface) + return "<%s %slistening on %s/%s>" % args + + + def startListening(self): + """ + Create and bind my socket, and begin listening on it. + + This must be called after creating a server to begin listening on the + specified tunnel. + """ + self._bindSocket() + self.protocol.makeConnection(self) + self.startReading() + + + def _openTunnel(self, name, mode): + """ + Open the named tunnel using the given mode. + + @param name: The name of the tunnel to open. + @type name: L{bytes} + + @param mode: Flags from L{TunnelFlags} with exactly one of + L{TunnelFlags.IFF_TUN} or L{TunnelFlags.IFF_TAP} set. + + @return: A L{_TunnelDescription} representing the newly opened tunnel. + """ + flags = ( + self._system.O_RDWR | self._system.O_CLOEXEC | + self._system.O_NONBLOCK) + config = struct.pack("%dsH" % (_IFNAMSIZ,), name, mode.value) + fileno = self._system.open(_TUN_KO_PATH, flags) + result = self._system.ioctl(fileno, _TUNSETIFF, config) + return _TunnelDescription(fileno, result[:_IFNAMSIZ].strip(b'\x00')) + + + def _bindSocket(self): + """ + Open the tunnel. + """ + log.msg( + format="%(protocol)s starting on %(interface)s", + protocol=self.protocol.__class__, + interface=self.interface) + try: + fileno, interface = self._openTunnel( + self.interface, self._mode | TunnelFlags.IFF_NO_PI) + except (IOError, OSError) as e: + raise error.CannotListenError(None, self.interface, e) + + self.interface = interface + self._fileno = fileno + + self.connected = 1 + + + def fileno(self): + return self._fileno + + + def doRead(self): + """ + Called when my socket is ready for reading. + """ + read = 0 + while read < self.maxThroughput: + try: + data = self._system.read(self._fileno, self.maxPacketSize) + except EnvironmentError as e: + if e.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EINTR): + return + else: + raise + except: + raise + read += len(data) + # TODO pkt.isPartial()? + try: + self.protocol.datagramReceived(data, partial=0) + except: + cls = fullyQualifiedName(self.protocol.__class__) + log.err( + None, + "Unhandled exception from %s.datagramReceived" % (cls,)) + + + def write(self, datagram): + """ + Write the given data as a single datagram. + + @param datagram: The data that will make up the complete datagram to be + written. + @type datagram: L{bytes} + """ + try: + return self._system.write(self._fileno, datagram) + except IOError as e: + if e.errno == errno.EINTR: + return self.write(datagram) + raise + + + def writeSequence(self, seq): + """ + Write a datagram constructed from a L{list} of L{bytes}. + + @param datagram: The data that will make up the complete datagram to be + written. + @type seq: L{list} of L{bytes} + """ + self.write(b"".join(seq)) + + + def stopListening(self): + """ + Stop accepting connections on this port. + + This will shut down my socket and call self.connectionLost(). + + @return: A L{Deferred} that fires when this port has stopped. + """ + self.stopReading() + if self.disconnecting: + return self._stoppedDeferred + elif self.connected: + self._stoppedDeferred = task.deferLater( + self.reactor, 0, self.connectionLost) + self.disconnecting = True + return self._stoppedDeferred + else: + return defer.succeed(None) + + + def loseConnection(self): + """ + Close this tunnel. Use L{TuntapPort.stopListening} instead. + """ + self.stopListening().addErrback(log.err) + + + def connectionLost(self, reason=None): + """ + Cleans up my socket. + + @param reason: Ignored. Do not use this. + """ + log.msg('(Tuntap %s Closed)' % self.interface) + abstract.FileDescriptor.connectionLost(self, reason) + self.protocol.doStop() + self.connected = 0 + self._system.close(self._fileno) + self._fileno = -1 + + + def logPrefix(self): + """ + Returns the name of my class, to prefix log entries with. + """ + return self.logstr + + + def getHost(self): + """ + Get the local address of this L{TuntapPort}. + + @return: A L{TunnelAddress} which describes the tunnel device to which + this object is bound. + @rtype: L{TunnelAddress} + """ + return TunnelAddress(self._mode, self.interface) + +TuntapPort.loseConnection = deprecated( + Version("Twisted", 14, 0, 0), + TuntapPort.stopListening)(TuntapPort.loseConnection) diff --git a/contrib/python/Twisted/py2/twisted/persisted/__init__.py b/contrib/python/Twisted/py2/twisted/persisted/__init__.py new file mode 100644 index 00000000000..55893b8cb9c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/persisted/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Persisted: Utilities for managing persistence. +""" diff --git a/contrib/python/Twisted/py2/twisted/persisted/aot.py b/contrib/python/Twisted/py2/twisted/persisted/aot.py new file mode 100644 index 00000000000..48978303261 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/persisted/aot.py @@ -0,0 +1,625 @@ +# -*- test-case-name: twisted.test.test_persisted -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +AOT: Abstract Object Trees +The source-code-marshallin'est abstract-object-serializin'est persister +this side of Marmalade! +""" + +from __future__ import division, absolute_import + +import types, re + +try: + from tokenize import generate_tokens as tokenize +except ImportError: + from tokenize import tokenize + + +try: + import copy_reg +except: + import copyreg as copy_reg + +from twisted.python import reflect, log +from twisted.persisted import crefutil +from twisted.python.compat import unicode, _PY3, _constructMethod + +########################### +# Abstract Object Classes # +########################### + +#"\0" in a getSource means "insert variable-width indention here". +#see `indentify'. + +class Named: + def __init__(self, name): + self.name = name + +class Class(Named): + def getSource(self): + return "Class(%r)" % self.name + +class Function(Named): + def getSource(self): + return "Function(%r)" % self.name + +class Module(Named): + def getSource(self): + return "Module(%r)" % self.name + + +class InstanceMethod: + def __init__(self, name, klass, inst): + if not (isinstance(inst, Ref) or isinstance(inst, Instance) or isinstance(inst, Deref)): + raise TypeError("%s isn't an Instance, Ref, or Deref!" % inst) + self.name = name + self.klass = klass + self.instance = inst + + def getSource(self): + return "InstanceMethod(%r, %r, \n\0%s)" % (self.name, self.klass, prettify(self.instance)) + + +class _NoStateObj: + pass +NoStateObj = _NoStateObj() + +_SIMPLE_BUILTINS = [ + bool, bytes, unicode, int, float, complex, type(None), + slice, type(Ellipsis) +] + +try: + _SIMPLE_BUILTINS.append(long) +except NameError: + pass + +class Instance: + def __init__(self, className, __stateObj__=NoStateObj, **state): + if not isinstance(className, str): + raise TypeError("%s isn't a string!" % className) + self.klass = className + if __stateObj__ is not NoStateObj: + self.state = __stateObj__ + self.stateIsDict = 0 + else: + self.state = state + self.stateIsDict = 1 + + def getSource(self): + #XXX make state be foo=bar instead of a dict. + if self.stateIsDict: + stateDict = self.state + elif isinstance(self.state, Ref) and isinstance(self.state.obj, dict): + stateDict = self.state.obj + else: + stateDict = None + if stateDict is not None: + try: + return "Instance(%r, %s)" % (self.klass, dictToKW(stateDict)) + except NonFormattableDict: + return "Instance(%r, %s)" % (self.klass, prettify(stateDict)) + return "Instance(%r, %s)" % (self.klass, prettify(self.state)) + +class Ref: + + def __init__(self, *args): + #blargh, lame. + if len(args) == 2: + self.refnum = args[0] + self.obj = args[1] + elif not args: + self.refnum = None + self.obj = None + + def setRef(self, num): + if self.refnum: + raise ValueError("Error setting id %s, I already have %s" % (num, self.refnum)) + self.refnum = num + + def setObj(self, obj): + if self.obj: + raise ValueError("Error setting obj %s, I already have %s" % (obj, self.obj)) + self.obj = obj + + def getSource(self): + if self.obj is None: + raise RuntimeError("Don't try to display me before setting an object on me!") + if self.refnum: + return "Ref(%d, \n\0%s)" % (self.refnum, prettify(self.obj)) + return prettify(self.obj) + + +class Deref: + def __init__(self, num): + self.refnum = num + + def getSource(self): + return "Deref(%d)" % self.refnum + + __repr__ = getSource + + +class Copyreg: + def __init__(self, loadfunc, state): + self.loadfunc = loadfunc + self.state = state + + def getSource(self): + return "Copyreg(%r, %s)" % (self.loadfunc, prettify(self.state)) + + + +############### +# Marshalling # +############### + + +def getSource(ao): + """Pass me an AO, I'll return a nicely-formatted source representation.""" + return indentify("app = " + prettify(ao)) + + +class NonFormattableDict(Exception): + """A dictionary was not formattable. + """ + +r = re.compile('[a-zA-Z_][a-zA-Z0-9_]*$') + +def dictToKW(d): + out = [] + items = list(d.items()) + items.sort() + for k, v in items: + if not isinstance(k, str): + raise NonFormattableDict("%r ain't a string" % k) + if not r.match(k): + raise NonFormattableDict("%r ain't an identifier" % k) + out.append( + "\n\0%s=%s," % (k, prettify(v)) + ) + return ''.join(out) + + +def prettify(obj): + if hasattr(obj, 'getSource'): + return obj.getSource() + else: + #basic type + t = type(obj) + + if t in _SIMPLE_BUILTINS: + return repr(obj) + + elif t is dict: + out = ['{'] + for k,v in obj.items(): + out.append('\n\0%s: %s,' % (prettify(k), prettify(v))) + out.append(len(obj) and '\n\0}' or '}') + return ''.join(out) + + elif t is list: + out = ["["] + for x in obj: + out.append('\n\0%s,' % prettify(x)) + out.append(len(obj) and '\n\0]' or ']') + return ''.join(out) + + elif t is tuple: + out = ["("] + for x in obj: + out.append('\n\0%s,' % prettify(x)) + out.append(len(obj) and '\n\0)' or ')') + return ''.join(out) + else: + raise TypeError("Unsupported type %s when trying to prettify %s." % (t, obj)) + +def indentify(s): + out = [] + stack = [] + l = ['', s] + for (tokenType, tokenString, (startRow, startColumn), + (endRow, endColumn), logicalLine) in tokenize(l.pop): + if tokenString in ['[', '(', '{']: + stack.append(tokenString) + elif tokenString in [']', ')', '}']: + stack.pop() + if tokenString == '\0': + out.append(' '*len(stack)) + else: + out.append(tokenString) + return ''.join(out) + + + +########### +# Unjelly # +########### + +def unjellyFromAOT(aot): + """ + Pass me an Abstract Object Tree, and I'll unjelly it for you. + """ + return AOTUnjellier().unjelly(aot) + +def unjellyFromSource(stringOrFile): + """ + Pass me a string of code or a filename that defines an 'app' variable (in + terms of Abstract Objects!), and I'll execute it and unjelly the resulting + AOT for you, returning a newly unpersisted Application object! + """ + + ns = {"Instance": Instance, + "InstanceMethod": InstanceMethod, + "Class": Class, + "Function": Function, + "Module": Module, + "Ref": Ref, + "Deref": Deref, + "Copyreg": Copyreg, + } + + if hasattr(stringOrFile, "read"): + source = stringOrFile.read() + else: + source = stringOrFile + code = compile(source, "", "exec") + eval(code, ns, ns) + + if 'app' in ns: + return unjellyFromAOT(ns['app']) + else: + raise ValueError("%s needs to define an 'app', it didn't!" % stringOrFile) + + +class AOTUnjellier: + """I handle the unjellying of an Abstract Object Tree. + See AOTUnjellier.unjellyAO + """ + def __init__(self): + self.references = {} + self.stack = [] + self.afterUnjelly = [] + + ## + # unjelly helpers (copied pretty much directly from (now deleted) marmalade) + ## + def unjellyLater(self, node): + """Unjelly a node, later. + """ + d = crefutil._Defer() + self.unjellyInto(d, 0, node) + return d + + def unjellyInto(self, obj, loc, ao): + """Utility method for unjellying one object into another. + This automates the handling of backreferences. + """ + o = self.unjellyAO(ao) + obj[loc] = o + if isinstance(o, crefutil.NotKnown): + o.addDependant(obj, loc) + return o + + def callAfter(self, callable, result): + if isinstance(result, crefutil.NotKnown): + l = [None] + result.addDependant(l, 1) + else: + l = [result] + self.afterUnjelly.append((callable, l)) + + def unjellyAttribute(self, instance, attrName, ao): + #XXX this is unused???? + """Utility method for unjellying into instances of attributes. + + Use this rather than unjellyAO unless you like surprising bugs! + Alternatively, you can use unjellyInto on your instance's __dict__. + """ + self.unjellyInto(instance.__dict__, attrName, ao) + + def unjellyAO(self, ao): + """Unjelly an Abstract Object and everything it contains. + I return the real object. + """ + self.stack.append(ao) + t = type(ao) + if t in _SIMPLE_BUILTINS: + return ao + + elif t is list: + l = [] + for x in ao: + l.append(None) + self.unjellyInto(l, len(l)-1, x) + return l + + elif t is tuple: + l = [] + tuple_ = tuple + for x in ao: + l.append(None) + if isinstance(self.unjellyInto(l, len(l)-1, x), crefutil.NotKnown): + tuple_ = crefutil._Tuple + return tuple_(l) + + elif t is dict: + d = {} + for k,v in ao.items(): + kvd = crefutil._DictKeyAndValue(d) + self.unjellyInto(kvd, 0, k) + self.unjellyInto(kvd, 1, v) + return d + else: + #Abstract Objects + c = ao.__class__ + if c is Module: + return reflect.namedModule(ao.name) + + elif c in [Class, Function] or issubclass(c, type): + return reflect.namedObject(ao.name) + + elif c is InstanceMethod: + im_name = ao.name + im_class = reflect.namedObject(ao.klass) + im_self = self.unjellyAO(ao.instance) + if im_name in im_class.__dict__: + if im_self is None: + return getattr(im_class, im_name) + elif isinstance(im_self, crefutil.NotKnown): + return crefutil._InstanceMethod(im_name, im_self, im_class) + else: + return _constructMethod(im_class, im_name, im_self) + else: + raise TypeError("instance method changed") + + elif c is Instance: + klass = reflect.namedObject(ao.klass) + state = self.unjellyAO(ao.state) + if hasattr(klass, "__new__"): + inst = klass.__new__(klass) + else: + inst = _OldStyleInstance(klass) + if hasattr(klass, "__setstate__"): + self.callAfter(inst.__setstate__, state) + else: + inst.__dict__ = state + return inst + + elif c is Ref: + o = self.unjellyAO(ao.obj) #THIS IS CHANGING THE REF OMG + refkey = ao.refnum + ref = self.references.get(refkey) + if ref is None: + self.references[refkey] = o + elif isinstance(ref, crefutil.NotKnown): + ref.resolveDependants(o) + self.references[refkey] = o + elif refkey is None: + # This happens when you're unjellying from an AOT not read from source + pass + else: + raise ValueError("Multiple references with the same ID: %s, %s, %s!" % (ref, refkey, ao)) + return o + + elif c is Deref: + num = ao.refnum + ref = self.references.get(num) + if ref is None: + der = crefutil._Dereference(num) + self.references[num] = der + return der + return ref + + elif c is Copyreg: + loadfunc = reflect.namedObject(ao.loadfunc) + d = self.unjellyLater(ao.state).addCallback( + lambda result, _l: _l(*result), loadfunc) + return d + else: + raise TypeError("Unsupported AOT type: %s" % t) + + del self.stack[-1] + + + def unjelly(self, ao): + try: + l = [None] + self.unjellyInto(l, 0, ao) + for func, v in self.afterUnjelly: + func(v[0]) + return l[0] + except: + log.msg("Error jellying object! Stacktrace follows::") + log.msg("\n".join(map(repr, self.stack))) + raise +######### +# Jelly # +######### + + +def jellyToAOT(obj): + """Convert an object to an Abstract Object Tree.""" + return AOTJellier().jelly(obj) + +def jellyToSource(obj, file=None): + """ + Pass me an object and, optionally, a file object. + I'll convert the object to an AOT either return it (if no file was + specified) or write it to the file. + """ + + aot = jellyToAOT(obj) + if file: + file.write(getSource(aot).encode("utf-8")) + else: + return getSource(aot) + + +try: + from types import (ClassType as _OldStyleClass, + InstanceType as _OldStyleInstance) +except ImportError: + _OldStyleClass = None + _OldStyleInstance = None + + + +def _classOfMethod(methodObject): + """ + Get the associated class of the given method object. + + @param methodObject: a bound method + @type methodObject: L{types.MethodType} + + @return: a class + @rtype: L{types.ClassType} or L{type} + """ + if _PY3: + return methodObject.__self__.__class__ + return methodObject.im_class + + + +def _funcOfMethod(methodObject): + """ + Get the associated function of the given method object. + + @param methodObject: a bound method + @type methodObject: L{types.MethodType} + + @return: the function implementing C{methodObject} + @rtype: L{types.FunctionType} + """ + if _PY3: + return methodObject.__func__ + return methodObject.im_func + + + +def _selfOfMethod(methodObject): + """ + Get the object that a bound method is bound to. + + @param methodObject: a bound method + @type methodObject: L{types.MethodType} + + @return: the C{self} passed to C{methodObject} + @rtype: L{object} + """ + if _PY3: + return methodObject.__self__ + return methodObject.im_self + + + +class AOTJellier: + def __init__(self): + # dict of {id(obj): (obj, node)} + self.prepared = {} + self._ref_id = 0 + self.stack = [] + + def prepareForRef(self, aoref, object): + """I prepare an object for later referencing, by storing its id() + and its _AORef in a cache.""" + self.prepared[id(object)] = aoref + + def jellyToAO(self, obj): + """I turn an object into an AOT and return it.""" + objType = type(obj) + self.stack.append(repr(obj)) + + #immutable: We don't care if these have multiple refs! + if objType in _SIMPLE_BUILTINS: + retval = obj + + elif objType is types.MethodType: + # TODO: make methods 'prefer' not to jelly the object internally, + # so that the object will show up where it's referenced first NOT + # by a method. + retval = InstanceMethod(_funcOfMethod(obj).__name__, + reflect.qual(_classOfMethod(obj)), + self.jellyToAO(_selfOfMethod(obj))) + + elif objType is types.ModuleType: + retval = Module(obj.__name__) + + elif objType is _OldStyleClass: + retval = Class(reflect.qual(obj)) + + elif issubclass(objType, type): + retval = Class(reflect.qual(obj)) + + elif objType is types.FunctionType: + retval = Function(reflect.fullFuncName(obj)) + + else: #mutable! gotta watch for refs. + +#Marmalade had the nicety of being able to just stick a 'reference' attribute +#on any Node object that was referenced, but in AOT, the referenced object +#is *inside* of a Ref call (Ref(num, obj) instead of +#). The problem is, especially for built-in types, +#I can't just assign some attribute to them to give them a refnum. So, I have +#to "wrap" a Ref(..) around them later -- that's why I put *everything* that's +#mutable inside one. The Ref() class will only print the "Ref(..)" around an +#object if it has a Reference explicitly attached. + + if id(obj) in self.prepared: + oldRef = self.prepared[id(obj)] + if oldRef.refnum: + # it's been referenced already + key = oldRef.refnum + else: + # it hasn't been referenced yet + self._ref_id = self._ref_id + 1 + key = self._ref_id + oldRef.setRef(key) + return Deref(key) + + retval = Ref() + def _stateFrom(state): + retval.setObj(Instance(reflect.qual(obj.__class__), + self.jellyToAO(state))) + self.prepareForRef(retval, obj) + + if objType is list: + retval.setObj([self.jellyToAO(o) for o in obj]) #hah! + + elif objType is tuple: + retval.setObj(tuple(map(self.jellyToAO, obj))) + + elif objType is dict: + d = {} + for k,v in obj.items(): + d[self.jellyToAO(k)] = self.jellyToAO(v) + retval.setObj(d) + + elif objType in copy_reg.dispatch_table: + unpickleFunc, state = copy_reg.dispatch_table[objType](obj) + + retval.setObj(Copyreg( reflect.fullFuncName(unpickleFunc), + self.jellyToAO(state))) + + elif hasattr(obj, "__getstate__"): + _stateFrom(obj.__getstate__()) + elif hasattr(obj, "__dict__"): + _stateFrom(obj.__dict__) + else: + raise TypeError("Unsupported type: %s" % objType.__name__) + + del self.stack[-1] + return retval + + def jelly(self, obj): + try: + ao = self.jellyToAO(obj) + return ao + except: + log.msg("Error jellying object! Stacktrace follows::") + log.msg('\n'.join(self.stack)) + raise diff --git a/contrib/python/Twisted/py2/twisted/persisted/crefutil.py b/contrib/python/Twisted/py2/twisted/persisted/crefutil.py new file mode 100644 index 00000000000..fd2e6b9bf4a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/persisted/crefutil.py @@ -0,0 +1,159 @@ +# -*- test-case-name: twisted.test.test_persisted -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Utility classes for dealing with circular references. +""" + +from __future__ import division, absolute_import + +from twisted.python import log, reflect +from twisted.python.compat import range, _constructMethod + + +class NotKnown: + def __init__(self): + self.dependants = [] + self.resolved = 0 + + def addDependant(self, mutableObject, key): + assert not self.resolved + self.dependants.append( (mutableObject, key) ) + + resolvedObject = None + + def resolveDependants(self, newObject): + self.resolved = 1 + self.resolvedObject = newObject + for mut, key in self.dependants: + mut[key] = newObject + + + def __hash__(self): + assert 0, "I am not to be used as a dictionary key." + + + +class _Container(NotKnown): + """ + Helper class to resolve circular references on container objects. + """ + + def __init__(self, l, containerType): + """ + @param l: The list of object which may contain some not yet referenced + objects. + + @param containerType: A type of container objects (e.g., C{tuple} or + C{set}). + """ + NotKnown.__init__(self) + self.containerType = containerType + self.l = l + self.locs = list(range(len(l))) + for idx in range(len(l)): + if not isinstance(l[idx], NotKnown): + self.locs.remove(idx) + else: + l[idx].addDependant(self, idx) + if not self.locs: + self.resolveDependants(self.containerType(self.l)) + + + def __setitem__(self, n, obj): + """ + Change the value of one contained objects, and resolve references if + all objects have been referenced. + """ + self.l[n] = obj + if not isinstance(obj, NotKnown): + self.locs.remove(n) + if not self.locs: + self.resolveDependants(self.containerType(self.l)) + + + +class _Tuple(_Container): + """ + Manage tuple containing circular references. Deprecated: use C{_Container} + instead. + """ + + def __init__(self, l): + """ + @param l: The list of object which may contain some not yet referenced + objects. + """ + _Container.__init__(self, l, tuple) + + + +class _InstanceMethod(NotKnown): + def __init__(self, im_name, im_self, im_class): + NotKnown.__init__(self) + self.my_class = im_class + self.name = im_name + # im_self _must_ be a NotKnown + im_self.addDependant(self, 0) + + def __call__(self, *args, **kw): + import traceback + log.msg('instance method %s.%s' % (reflect.qual(self.my_class), self.name)) + log.msg('being called with %r %r' % (args, kw)) + traceback.print_stack(file=log.logfile) + assert 0 + + def __setitem__(self, n, obj): + assert n == 0, "only zero index allowed" + if not isinstance(obj, NotKnown): + method = _constructMethod(self.my_class, self.name, obj) + self.resolveDependants(method) + +class _DictKeyAndValue: + def __init__(self, dict): + self.dict = dict + def __setitem__(self, n, obj): + if n not in (1, 0): + raise RuntimeError("DictKeyAndValue should only ever be called with 0 or 1") + if n: # value + self.value = obj + else: + self.key = obj + if hasattr(self, "key") and hasattr(self, "value"): + self.dict[self.key] = self.value + + +class _Dereference(NotKnown): + def __init__(self, id): + NotKnown.__init__(self) + self.id = id + + + +from twisted.internet.defer import Deferred + +class _Defer(Deferred, NotKnown): + def __init__(self): + Deferred.__init__(self) + NotKnown.__init__(self) + self.pause() + + wasset = 0 + + def __setitem__(self, n, obj): + if self.wasset: + raise RuntimeError('setitem should only be called once, setting %r to %r' % (n, obj)) + else: + self.wasset = 1 + self.callback(obj) + + def addDependant(self, dep, key): + # by the time I'm adding a dependant, I'm *not* adding any more + # callbacks + NotKnown.addDependant(self, dep, key) + self.unpause() + resovd = self.result + self.resolveDependants(resovd) diff --git a/contrib/python/Twisted/py2/twisted/persisted/dirdbm.py b/contrib/python/Twisted/py2/twisted/persisted/dirdbm.py new file mode 100644 index 00000000000..f97c526d09d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/persisted/dirdbm.py @@ -0,0 +1,389 @@ +# -*- test-case-name: twisted.test.test_dirdbm -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + + +""" +DBM-style interface to a directory. + +Each key is stored as a single file. This is not expected to be very fast or +efficient, but it's good for easy debugging. + +DirDBMs are *not* thread-safe, they should only be accessed by one thread at +a time. + +No files should be placed in the working directory of a DirDBM save those +created by the DirDBM itself! + +Maintainer: Itamar Shtull-Trauring +""" + + +import os +import base64 +import glob + +try: + import cPickle as pickle +except ImportError: + import pickle + +from twisted.python.filepath import FilePath + +try: + _open +except NameError: + _open = open + + + +class DirDBM: + """ + A directory with a DBM interface. + + This class presents a hash-like interface to a directory of small, + flat files. It can only use strings as keys or values. + """ + + def __init__(self, name): + """ + @type name: str + @param name: Base path to use for the directory storage. + """ + self.dname = os.path.abspath(name) + self._dnamePath = FilePath(name) + if not self._dnamePath.isdir(): + self._dnamePath.createDirectory() + else: + # Run recovery, in case we crashed. we delete all files ending + # with ".new". Then we find all files who end with ".rpl". If a + # corresponding file exists without ".rpl", we assume the write + # failed and delete the ".rpl" file. If only a ".rpl" exist we + # assume the program crashed right after deleting the old entry + # but before renaming the replacement entry. + # + # NOTE: '.' is NOT in the base64 alphabet! + for f in glob.glob(self._dnamePath.child("*.new").path): + os.remove(f) + replacements = glob.glob(self._dnamePath.child("*.rpl").path) + for f in replacements: + old = f[:-4] + if os.path.exists(old): + os.remove(f) + else: + os.rename(f, old) + + + def _encode(self, k): + """ + Encode a key so it can be used as a filename. + """ + # NOTE: '_' is NOT in the base64 alphabet! + return base64.encodestring(k).replace(b'\n', b'_').replace(b"/", b"-") + + + def _decode(self, k): + """ + Decode a filename to get the key. + """ + return base64.decodestring(k.replace(b'_', b'\n').replace(b"-", b"/")) + + + def _readFile(self, path): + """ + Read in the contents of a file. + + Override in subclasses to e.g. provide transparently encrypted dirdbm. + """ + with _open(path.path, "rb") as f: + s = f.read() + return s + + + def _writeFile(self, path, data): + """ + Write data to a file. + + Override in subclasses to e.g. provide transparently encrypted dirdbm. + """ + with _open(path.path, "wb") as f: + f.write(data) + f.flush() + + + def __len__(self): + """ + @return: The number of key/value pairs in this Shelf + """ + return len(self._dnamePath.listdir()) + + + def __setitem__(self, k, v): + """ + C{dirdbm[k] = v} + Create or modify a textfile in this directory + + @type k: bytes + @param k: key to set + + @type v: bytes + @param v: value to associate with C{k} + """ + if not type(k) == bytes: + raise TypeError("DirDBM key must be bytes") + if not type(v) == bytes: + raise TypeError("DirDBM value must be bytes") + k = self._encode(k) + + # We create a new file with extension .new, write the data to it, and + # if the write succeeds delete the old file and rename the new one. + old = self._dnamePath.child(k) + if old.exists(): + new = old.siblingExtension(".rpl") # Replacement entry + else: + new = old.siblingExtension(".new") # New entry + try: + self._writeFile(new, v) + except: + new.remove() + raise + else: + if (old.exists()): old.remove() + new.moveTo(old) + + + def __getitem__(self, k): + """ + C{dirdbm[k]} + Get the contents of a file in this directory as a string. + + @type k: bytes + @param k: key to lookup + + @return: The value associated with C{k} + @raise KeyError: Raised when there is no such key + """ + if not type(k) == bytes: + raise TypeError("DirDBM key must be bytes") + path = self._dnamePath.child(self._encode(k)) + try: + return self._readFile(path) + except (EnvironmentError): + raise KeyError(k) + + + def __delitem__(self, k): + """ + C{del dirdbm[foo]} + Delete a file in this directory. + + @type k: bytes + @param k: key to delete + + @raise KeyError: Raised when there is no such key + """ + if not type(k) == bytes: + raise TypeError("DirDBM key must be bytes") + k = self._encode(k) + try: + self._dnamePath.child(k).remove() + except (EnvironmentError): + raise KeyError(self._decode(k)) + + + def keys(self): + """ + @return: a L{list} of filenames (keys). + """ + return list(map(self._decode, self._dnamePath.asBytesMode().listdir())) + + + def values(self): + """ + @return: a L{list} of file-contents (values). + """ + vals = [] + keys = self.keys() + for key in keys: + vals.append(self[key]) + return vals + + + def items(self): + """ + @return: a L{list} of 2-tuples containing key/value pairs. + """ + items = [] + keys = self.keys() + for key in keys: + items.append((key, self[key])) + return items + + + def has_key(self, key): + """ + @type key: bytes + @param key: The key to test + + @return: A true value if this dirdbm has the specified key, a false + value otherwise. + """ + if not type(key) == bytes: + raise TypeError("DirDBM key must be bytes") + key = self._encode(key) + return self._dnamePath.child(key).isfile() + + + def setdefault(self, key, value): + """ + @type key: bytes + @param key: The key to lookup + + @param value: The value to associate with key if key is not already + associated with a value. + """ + if key not in self: + self[key] = value + return value + return self[key] + + + def get(self, key, default = None): + """ + @type key: bytes + @param key: The key to lookup + + @param default: The value to return if the given key does not exist + + @return: The value associated with C{key} or C{default} if not + L{DirDBM.has_key(key)} + """ + if key in self: + return self[key] + else: + return default + + + def __contains__(self, key): + """ + @see: L{DirDBM.has_key} + """ + return self.has_key(key) + + + def update(self, dict): + """ + Add all the key/value pairs in L{dict} to this dirdbm. Any conflicting + keys will be overwritten with the values from L{dict}. + + @type dict: mapping + @param dict: A mapping of key/value pairs to add to this dirdbm. + """ + for key, val in dict.items(): + self[key]=val + + + def copyTo(self, path): + """ + Copy the contents of this dirdbm to the dirdbm at C{path}. + + @type path: L{str} + @param path: The path of the dirdbm to copy to. If a dirdbm + exists at the destination path, it is cleared first. + + @rtype: C{DirDBM} + @return: The dirdbm this dirdbm was copied to. + """ + path = FilePath(path) + assert path != self._dnamePath + + d = self.__class__(path.path) + d.clear() + for k in self.keys(): + d[k] = self[k] + return d + + + def clear(self): + """ + Delete all key/value pairs in this dirdbm. + """ + for k in self.keys(): + del self[k] + + + def close(self): + """ + Close this dbm: no-op, for dbm-style interface compliance. + """ + + + def getModificationTime(self, key): + """ + Returns modification time of an entry. + + @return: Last modification date (seconds since epoch) of entry C{key} + @raise KeyError: Raised when there is no such key + """ + if not type(key) == bytes: + raise TypeError("DirDBM key must be bytes") + path = self._dnamePath.child(self._encode(key)) + if path.isfile(): + return path.getModificationTime() + else: + raise KeyError(key) + + + +class Shelf(DirDBM): + """ + A directory with a DBM shelf interface. + + This class presents a hash-like interface to a directory of small, + flat files. Keys must be strings, but values can be any given object. + """ + + def __setitem__(self, k, v): + """ + C{shelf[foo] = bar} + Create or modify a textfile in this directory. + + @type k: str + @param k: The key to set + + @param v: The value to associate with C{key} + """ + v = pickle.dumps(v) + DirDBM.__setitem__(self, k, v) + + + def __getitem__(self, k): + """ + C{dirdbm[foo]} + Get and unpickle the contents of a file in this directory. + + @type k: bytes + @param k: The key to lookup + + @return: The value associated with the given key + @raise KeyError: Raised if the given key does not exist + """ + return pickle.loads(DirDBM.__getitem__(self, k)) + + + +def open(file, flag = None, mode = None): + """ + This is for 'anydbm' compatibility. + + @param file: The parameter to pass to the DirDBM constructor. + + @param flag: ignored + @param mode: ignored + """ + return DirDBM(file) + + +__all__ = ["open", "DirDBM", "Shelf"] diff --git a/contrib/python/Twisted/py2/twisted/persisted/sob.py b/contrib/python/Twisted/py2/twisted/persisted/sob.py new file mode 100644 index 00000000000..cbcab16b83b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/persisted/sob.py @@ -0,0 +1,194 @@ +# -*- test-case-name: twisted.test.test_sob -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# +""" +Save and load Small OBjects to and from files, using various formats. + +Maintainer: Moshe Zadka +""" + +from __future__ import division, absolute_import + +import os +import sys + +try: + import cPickle as pickle +except ImportError: + import pickle +from twisted.python import log, runtime +from twisted.persisted import styles +from zope.interface import implementer, Interface + + + +class IPersistable(Interface): + + """An object which can be saved in several formats to a file""" + + def setStyle(style): + """Set desired format. + + @type style: string (one of 'pickle' or 'source') + """ + + def save(tag=None, filename=None, passphrase=None): + """Save object to file. + + @type tag: string + @type filename: string + @type passphrase: string + """ + + +@implementer(IPersistable) +class Persistent: + + style = "pickle" + + def __init__(self, original, name): + self.original = original + self.name = name + + def setStyle(self, style): + """Set desired format. + + @type style: string (one of 'pickle' or 'source') + """ + self.style = style + + def _getFilename(self, filename, ext, tag): + if filename: + finalname = filename + filename = finalname + "-2" + elif tag: + filename = "%s-%s-2.%s" % (self.name, tag, ext) + finalname = "%s-%s.%s" % (self.name, tag, ext) + else: + filename = "%s-2.%s" % (self.name, ext) + finalname = "%s.%s" % (self.name, ext) + return finalname, filename + + def _saveTemp(self, filename, dumpFunc): + with open(filename, 'wb') as f: + dumpFunc(self.original, f) + + def _getStyle(self): + if self.style == "source": + from twisted.persisted.aot import jellyToSource as dumpFunc + ext = "tas" + else: + def dumpFunc(obj, file): + pickle.dump(obj, file, 2) + ext = "tap" + return ext, dumpFunc + + def save(self, tag=None, filename=None, passphrase=None): + """Save object to file. + + @type tag: string + @type filename: string + @type passphrase: string + """ + ext, dumpFunc = self._getStyle() + if passphrase is not None: + raise TypeError("passphrase must be None") + ext = 'e' + ext + finalname, filename = self._getFilename(filename, ext, tag) + log.msg("Saving "+self.name+" application to "+finalname+"...") + self._saveTemp(filename, dumpFunc) + if runtime.platformType == "win32" and os.path.isfile(finalname): + os.remove(finalname) + os.rename(filename, finalname) + log.msg("Saved.") + +# "Persistant" has been present since 1.0.7, so retain it for compatibility +Persistant = Persistent + +class _EverythingEphemeral(styles.Ephemeral): + + initRun = 0 + + def __init__(self, mainMod): + """ + @param mainMod: The '__main__' module that this class will proxy. + """ + self.mainMod = mainMod + + def __getattr__(self, key): + try: + return getattr(self.mainMod, key) + except AttributeError: + if self.initRun: + raise + else: + log.msg("Warning! Loading from __main__: %s" % key) + return styles.Ephemeral() + + +def load(filename, style): + """Load an object from a file. + + Deserialize an object from a file. The file can be encrypted. + + @param filename: string + @param style: string (one of 'pickle' or 'source') + """ + mode = 'r' + if style=='source': + from twisted.persisted.aot import unjellyFromSource as _load + else: + _load, mode = pickle.load, 'rb' + + fp = open(filename, mode) + ee = _EverythingEphemeral(sys.modules['__main__']) + sys.modules['__main__'] = ee + ee.initRun = 1 + with fp: + try: + value = _load(fp) + finally: + # restore __main__ if an exception is raised. + sys.modules['__main__'] = ee.mainMod + + styles.doUpgrade() + ee.initRun = 0 + persistable = IPersistable(value, None) + if persistable is not None: + persistable.setStyle(style) + return value + + +def loadValueFromFile(filename, variable): + """Load the value of a variable in a Python file. + + Run the contents of the file in a namespace and return the result of the + variable named C{variable}. + + @param filename: string + @param variable: string + """ + with open(filename, 'r') as fileObj: + data = fileObj.read() + d = {'__file__': filename} + codeObj = compile(data, filename, "exec") + eval(codeObj, d, d) + value = d[variable] + return value + +def guessType(filename): + ext = os.path.splitext(filename)[1] + return { + '.tac': 'python', + '.etac': 'python', + '.py': 'python', + '.tap': 'pickle', + '.etap': 'pickle', + '.tas': 'source', + '.etas': 'source', + }[ext] + +__all__ = ['loadValueFromFile', 'load', 'Persistent', 'Persistant', + 'IPersistable', 'guessType'] diff --git a/contrib/python/Twisted/py2/twisted/persisted/styles.py b/contrib/python/Twisted/py2/twisted/persisted/styles.py new file mode 100644 index 00000000000..7703f370676 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/persisted/styles.py @@ -0,0 +1,423 @@ +# -*- test-case-name: twisted.test.test_persisted -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Different styles of persisted objects. +""" + +from __future__ import division, absolute_import + +# System Imports +import types +import pickle +try: + import copy_reg +except ImportError: + import copyreg as copy_reg +import copy +import inspect + +from twisted.python.compat import _PY3, _PYPY + +# Twisted Imports +from twisted.python import log +from twisted.python import reflect + +oldModules = {} + + +try: + import cPickle +except ImportError: + cPickle = None + +if cPickle is None or cPickle.PicklingError is pickle.PicklingError: + _UniversalPicklingError = pickle.PicklingError +else: + class _UniversalPicklingError(pickle.PicklingError, + cPickle.PicklingError): + """ + A PicklingError catchable by both L{cPickle.PicklingError} and + L{pickle.PicklingError} handlers. + """ + + +## First, let's register support for some stuff that really ought to +## be registerable... + +def pickleMethod(method): + 'support function for copy_reg to pickle method refs' + if _PY3: + return (unpickleMethod, (method.__name__, + method.__self__, + method.__self__.__class__)) + else: + return (unpickleMethod, (method.im_func.__name__, + method.im_self, + method.im_class)) + + + +def _methodFunction(classObject, methodName): + """ + Retrieve the function object implementing a method name given the class + it's on and a method name. + + @param classObject: A class to retrieve the method's function from. + @type classObject: L{type} or L{types.ClassType} + + @param methodName: The name of the method whose function to retrieve. + @type methodName: native L{str} + + @return: the function object corresponding to the given method name. + @rtype: L{types.FunctionType} + """ + methodObject = getattr(classObject, methodName) + if _PY3: + return methodObject + return methodObject.im_func + + + +def unpickleMethod(im_name, im_self, im_class): + """ + Support function for copy_reg to unpickle method refs. + + @param im_name: The name of the method. + @type im_name: native L{str} + + @param im_self: The instance that the method was present on. + @type im_self: L{object} + + @param im_class: The class where the method was declared. + @type im_class: L{types.ClassType} or L{type} or L{None} + """ + if im_self is None: + return getattr(im_class, im_name) + try: + methodFunction = _methodFunction(im_class, im_name) + except AttributeError: + log.msg("Method", im_name, "not on class", im_class) + assert im_self is not None, "No recourse: no instance to guess from." + # Attempt a last-ditch fix before giving up. If classes have changed + # around since we pickled this method, we may still be able to get it + # by looking on the instance's current class. + if im_self.__class__ is im_class: + raise + return unpickleMethod(im_name, im_self, im_self.__class__) + else: + if _PY3: + maybeClass = () + else: + maybeClass = tuple([im_class]) + bound = types.MethodType(methodFunction, im_self, *maybeClass) + return bound + + + +copy_reg.pickle(types.MethodType, pickleMethod, unpickleMethod) + +def _pickleFunction(f): + """ + Reduce, in the sense of L{pickle}'s C{object.__reduce__} special method, a + function object into its constituent parts. + + @param f: The function to reduce. + @type f: L{types.FunctionType} + + @return: a 2-tuple of a reference to L{_unpickleFunction} and a tuple of + its arguments, a 1-tuple of the function's fully qualified name. + @rtype: 2-tuple of C{callable, native string} + """ + if f.__name__ == '': + raise _UniversalPicklingError( + "Cannot pickle lambda function: {}".format(f)) + return (_unpickleFunction, + tuple([".".join([f.__module__, f.__qualname__])])) + + + +def _unpickleFunction(fullyQualifiedName): + """ + Convert a function name into a function by importing it. + + This is a synonym for L{twisted.python.reflect.namedAny}, but imported + locally to avoid circular imports, and also to provide a persistent name + that can be stored (and deprecated) independently of C{namedAny}. + + @param fullyQualifiedName: The fully qualified name of a function. + @type fullyQualifiedName: native C{str} + + @return: A function object imported from the given location. + @rtype: L{types.FunctionType} + """ + from twisted.python.reflect import namedAny + return namedAny(fullyQualifiedName) + + + +copy_reg.pickle(types.FunctionType, _pickleFunction, _unpickleFunction) + +def pickleModule(module): + 'support function for copy_reg to pickle module refs' + return unpickleModule, (module.__name__,) + +def unpickleModule(name): + 'support function for copy_reg to unpickle module refs' + if name in oldModules: + log.msg("Module has moved: %s" % name) + name = oldModules[name] + log.msg(name) + return __import__(name,{},{},'x') + + +copy_reg.pickle(types.ModuleType, + pickleModule, + unpickleModule) + + + +def pickleStringO(stringo): + """ + Reduce the given cStringO. + + This is only called on Python 2, because the cStringIO module only exists + on Python 2. + + @param stringo: The string output to pickle. + @type stringo: L{cStringIO.OutputType} + """ + 'support function for copy_reg to pickle StringIO.OutputTypes' + return unpickleStringO, (stringo.getvalue(), stringo.tell()) + + + +def unpickleStringO(val, sek): + """ + Convert the output of L{pickleStringO} into an appropriate type for the + current python version. This may be called on Python 3 and will convert a + cStringIO into an L{io.StringIO}. + + @param val: The content of the file. + @type val: L{bytes} + + @param sek: The seek position of the file. + @type sek: L{int} + + @return: a file-like object which you can write bytes to. + @rtype: L{cStringIO.OutputType} on Python 2, L{io.StringIO} on Python 3. + """ + x = _cStringIO() + x.write(val) + x.seek(sek) + return x + + + +def pickleStringI(stringi): + """ + Reduce the given cStringI. + + This is only called on Python 2, because the cStringIO module only exists + on Python 2. + + @param stringi: The string input to pickle. + @type stringi: L{cStringIO.InputType} + + @return: a 2-tuple of (C{unpickleStringI}, (bytes, pointer)) + @rtype: 2-tuple of (function, (bytes, int)) + """ + return unpickleStringI, (stringi.getvalue(), stringi.tell()) + + + +def unpickleStringI(val, sek): + """ + Convert the output of L{pickleStringI} into an appropriate type for the + current Python version. + + This may be called on Python 3 and will convert a cStringIO into an + L{io.StringIO}. + + @param val: The content of the file. + @type val: L{bytes} + + @param sek: The seek position of the file. + @type sek: L{int} + + @return: a file-like object which you can read bytes from. + @rtype: L{cStringIO.OutputType} on Python 2, L{io.StringIO} on Python 3. + """ + x = _cStringIO(val) + x.seek(sek) + return x + + + +try: + from cStringIO import InputType, OutputType, StringIO as _cStringIO +except ImportError: + from io import StringIO as _cStringIO +else: + copy_reg.pickle(OutputType, pickleStringO, unpickleStringO) + copy_reg.pickle(InputType, pickleStringI, unpickleStringI) + + + +class Ephemeral: + """ + This type of object is never persisted; if possible, even references to it + are eliminated. + """ + + def __reduce__(self): + """ + Serialize any subclass of L{Ephemeral} in a way which replaces it with + L{Ephemeral} itself. + """ + return (Ephemeral, ()) + + def __getstate__(self): + log.msg( "WARNING: serializing ephemeral %s" % self ) + if not _PYPY: + import gc + if getattr(gc, 'get_referrers', None): + for r in gc.get_referrers(self): + log.msg( " referred to by %s" % (r,)) + return None + + def __setstate__(self, state): + log.msg( "WARNING: unserializing ephemeral %s" % self.__class__ ) + self.__class__ = Ephemeral + + +versionedsToUpgrade = {} +upgraded = {} + +def doUpgrade(): + global versionedsToUpgrade, upgraded + for versioned in list(versionedsToUpgrade.values()): + requireUpgrade(versioned) + versionedsToUpgrade = {} + upgraded = {} + +def requireUpgrade(obj): + """Require that a Versioned instance be upgraded completely first. + """ + objID = id(obj) + if objID in versionedsToUpgrade and objID not in upgraded: + upgraded[objID] = 1 + obj.versionUpgrade() + return obj + +def _aybabtu(c): + """ + Get all of the parent classes of C{c}, not including C{c} itself, which are + strict subclasses of L{Versioned}. + + @param c: a class + @returns: list of classes + """ + # begin with two classes that should *not* be included in the + # final result + l = [c, Versioned] + for b in inspect.getmro(c): + if b not in l and issubclass(b, Versioned): + l.append(b) + # return all except the unwanted classes + return l[2:] + +class Versioned: + """ + This type of object is persisted with versioning information. + + I have a single class attribute, the int persistenceVersion. After I am + unserialized (and styles.doUpgrade() is called), self.upgradeToVersionX() + will be called for each version upgrade I must undergo. + + For example, if I serialize an instance of a Foo(Versioned) at version 4 + and then unserialize it when the code is at version 9, the calls:: + + self.upgradeToVersion5() + self.upgradeToVersion6() + self.upgradeToVersion7() + self.upgradeToVersion8() + self.upgradeToVersion9() + + will be made. If any of these methods are undefined, a warning message + will be printed. + """ + persistenceVersion = 0 + persistenceForgets = () + + def __setstate__(self, state): + versionedsToUpgrade[id(self)] = self + self.__dict__ = state + + def __getstate__(self, dict=None): + """Get state, adding a version number to it on its way out. + """ + dct = copy.copy(dict or self.__dict__) + bases = _aybabtu(self.__class__) + bases.reverse() + bases.append(self.__class__) # don't forget me!! + for base in bases: + if 'persistenceForgets' in base.__dict__: + for slot in base.persistenceForgets: + if slot in dct: + del dct[slot] + if 'persistenceVersion' in base.__dict__: + dct['%s.persistenceVersion' % reflect.qual(base)] = base.persistenceVersion + return dct + + def versionUpgrade(self): + """(internal) Do a version upgrade. + """ + bases = _aybabtu(self.__class__) + # put the bases in order so superclasses' persistenceVersion methods + # will be called first. + bases.reverse() + bases.append(self.__class__) # don't forget me!! + # first let's look for old-skool versioned's + if "persistenceVersion" in self.__dict__: + + # Hacky heuristic: if more than one class subclasses Versioned, + # we'll assume that the higher version number wins for the older + # class, so we'll consider the attribute the version of the older + # class. There are obviously possibly times when this will + # eventually be an incorrect assumption, but hopefully old-school + # persistenceVersion stuff won't make it that far into multiple + # classes inheriting from Versioned. + + pver = self.__dict__['persistenceVersion'] + del self.__dict__['persistenceVersion'] + highestVersion = 0 + highestBase = None + for base in bases: + if 'persistenceVersion' not in base.__dict__: + continue + if base.persistenceVersion > highestVersion: + highestBase = base + highestVersion = base.persistenceVersion + if highestBase: + self.__dict__['%s.persistenceVersion' % reflect.qual(highestBase)] = pver + for base in bases: + # ugly hack, but it's what the user expects, really + if (Versioned not in base.__bases__ and + 'persistenceVersion' not in base.__dict__): + continue + currentVers = base.persistenceVersion + pverName = '%s.persistenceVersion' % reflect.qual(base) + persistVers = (self.__dict__.get(pverName) or 0) + if persistVers: + del self.__dict__[pverName] + assert persistVers <= currentVers, "Sorry, can't go backwards in time." + while persistVers < currentVers: + persistVers = persistVers + 1 + method = base.__dict__.get('upgradeToVersion%s' % persistVers, None) + if method: + log.msg( "Upgrading %s (of %s @ %s) to version %s" % (reflect.qual(base), reflect.qual(self.__class__), id(self), persistVers) ) + method(self) + else: + log.msg( 'Warning: cannot upgrade %s to version %s' % (base, persistVers) ) diff --git a/contrib/python/Twisted/py2/twisted/plugin.py b/contrib/python/Twisted/py2/twisted/plugin.py new file mode 100644 index 00000000000..82522ee2fee --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugin.py @@ -0,0 +1,259 @@ +# -*- test-case-name: twisted.test.test_plugin -*- +# Copyright (c) 2005 Divmod, Inc. +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Plugin system for Twisted. + +@author: Jp Calderone +@author: Glyph Lefkowitz +""" + +from __future__ import absolute_import, division + +import os +import sys + +from zope.interface import Interface, providedBy + +def _determinePickleModule(): + """ + Determine which 'pickle' API module to use. + """ + try: + import cPickle + return cPickle + except ImportError: + import pickle + return pickle + +pickle = _determinePickleModule() + +from twisted.python.components import getAdapterFactory +from twisted.python.reflect import namedAny +from twisted.python import log +from twisted.python.modules import getModule +from twisted.python.compat import iteritems + + + +class IPlugin(Interface): + """ + Interface that must be implemented by all plugins. + + Only objects which implement this interface will be considered for return + by C{getPlugins}. To be useful, plugins should also implement some other + application-specific interface. + """ + + + +class CachedPlugin(object): + def __init__(self, dropin, name, description, provided): + self.dropin = dropin + self.name = name + self.description = description + self.provided = provided + self.dropin.plugins.append(self) + + def __repr__(self): + return '' % ( + self.name, self.dropin.moduleName, + ', '.join([i.__name__ for i in self.provided])) + + def load(self): + return namedAny(self.dropin.moduleName + '.' + self.name) + + def __conform__(self, interface, registry=None, default=None): + for providedInterface in self.provided: + if providedInterface.isOrExtends(interface): + return self.load() + if getAdapterFactory(providedInterface, interface, None) is not None: + return interface(self.load(), default) + return default + + # backwards compat HOORJ + getComponent = __conform__ + + + +class CachedDropin(object): + """ + A collection of L{CachedPlugin} instances from a particular module in a + plugin package. + + @type moduleName: C{str} + @ivar moduleName: The fully qualified name of the plugin module this + represents. + + @type description: C{str} or L{None} + @ivar description: A brief explanation of this collection of plugins + (probably the plugin module's docstring). + + @type plugins: C{list} + @ivar plugins: The L{CachedPlugin} instances which were loaded from this + dropin. + """ + def __init__(self, moduleName, description): + self.moduleName = moduleName + self.description = description + self.plugins = [] + + + +def _generateCacheEntry(provider): + dropin = CachedDropin(provider.__name__, + provider.__doc__) + for k, v in iteritems(provider.__dict__): + plugin = IPlugin(v, None) + if plugin is not None: + # Instantiated for its side-effects. + CachedPlugin(dropin, k, v.__doc__, list(providedBy(plugin))) + return dropin + +try: + fromkeys = dict.fromkeys +except AttributeError: + def fromkeys(keys, value=None): + d = {} + for k in keys: + d[k] = value + return d + + + +def getCache(module): + """ + Compute all the possible loadable plugins, while loading as few as + possible and hitting the filesystem as little as possible. + + @param module: a Python module object. This represents a package to search + for plugins. + + @return: a dictionary mapping module names to L{CachedDropin} instances. + """ + allCachesCombined = {} + mod = getModule(module.__name__) + # don't want to walk deep, only immediate children. + buckets = {} + # Fill buckets with modules by related entry on the given package's + # __path__. There's an abstraction inversion going on here, because this + # information is already represented internally in twisted.python.modules, + # but it's simple enough that I'm willing to live with it. If anyone else + # wants to fix up this iteration so that it's one path segment at a time, + # be my guest. --glyph + for plugmod in mod.iterModules(): + fpp = plugmod.filePath.parent() + if fpp not in buckets: + buckets[fpp] = [] + bucket = buckets[fpp] + bucket.append(plugmod) + for pseudoPackagePath, bucket in iteritems(buckets): + dropinPath = pseudoPackagePath.child('dropin.cache') + try: + lastCached = dropinPath.getModificationTime() + with dropinPath.open('r') as f: + dropinDotCache = pickle.load(f) + except: + dropinDotCache = {} + lastCached = 0 + + needsWrite = False + existingKeys = {} + for pluginModule in bucket: + pluginKey = pluginModule.name.split('.')[-1] + existingKeys[pluginKey] = True + if ((pluginKey not in dropinDotCache) or + (pluginModule.filePath.getModificationTime() >= lastCached)): + needsWrite = True + try: + provider = pluginModule.load() + except: + # dropinDotCache.pop(pluginKey, None) + log.err() + else: + entry = _generateCacheEntry(provider) + dropinDotCache[pluginKey] = entry + # Make sure that the cache doesn't contain any stale plugins. + for pluginKey in list(dropinDotCache.keys()): + if pluginKey not in existingKeys: + del dropinDotCache[pluginKey] + needsWrite = True + if needsWrite: + try: + dropinPath.setContent(pickle.dumps(dropinDotCache)) + except OSError as e: + log.msg( + format=( + "Unable to write to plugin cache %(path)s: error " + "number %(errno)d"), + path=dropinPath.path, errno=e.errno) + except: + log.err(None, "Unexpected error while writing cache file") + allCachesCombined.update(dropinDotCache) + return allCachesCombined + + + +def getPlugins(interface, package=None): + """ + Retrieve all plugins implementing the given interface beneath the given module. + + @param interface: An interface class. Only plugins which implement this + interface will be returned. + + @param package: A package beneath which plugins are installed. For + most uses, the default value is correct. + + @return: An iterator of plugins. + """ + if package is None: + import twisted.plugins as package + allDropins = getCache(package) + for key, dropin in iteritems(allDropins): + for plugin in dropin.plugins: + try: + adapted = interface(plugin, None) + except: + log.err() + else: + if adapted is not None: + yield adapted + + +# Old, backwards compatible name. Don't use this. +getPlugIns = getPlugins + + +def pluginPackagePaths(name): + """ + Return a list of additional directories which should be searched for + modules to be included as part of the named plugin package. + + @type name: C{str} + @param name: The fully-qualified Python name of a plugin package, eg + C{'twisted.plugins'}. + + @rtype: C{list} of C{str} + @return: The absolute paths to other directories which may contain plugin + modules for the named plugin package. + """ + package = name.split('.') + # Note that this may include directories which do not exist. It may be + # preferable to remove such directories at this point, rather than allow + # them to be searched later on. + # + # Note as well that only '__init__.py' will be considered to make a + # directory a package (and thus exclude it from this list). This means + # that if you create a master plugin package which has some other kind of + # __init__ (eg, __init__.pyc) it will be incorrectly treated as a + # supplementary plugin directory. + return [ + os.path.abspath(os.path.join(x, *package)) + for x + in sys.path + if + not os.path.exists(os.path.join(x, *package + ['__init__.py']))] + +__all__ = ['getPlugins', 'pluginPackagePaths'] diff --git a/contrib/python/Twisted/py2/twisted/plugins/__init__.py b/contrib/python/Twisted/py2/twisted/plugins/__init__.py new file mode 100644 index 00000000000..dd2d6aad026 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/__init__.py @@ -0,0 +1,19 @@ +# -*- test-case-name: twisted.test.test_plugin -*- +# Copyright (c) 2005 Divmod, Inc. +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Plugins for services implemented in Twisted. + +Plugins go in directories on your PYTHONPATH named twisted/plugins: +this is the only place where an __init__.py is necessary, thanks to +the __path__ variable. + +@author: Jp Calderone +@author: Glyph Lefkowitz +""" + +from twisted.plugin import pluginPackagePaths +__path__.extend(pluginPackagePaths(__name__)) +__all__ = [] # nothing to see here, move along, move along diff --git a/contrib/python/Twisted/py2/twisted/plugins/cred_anonymous.py b/contrib/python/Twisted/py2/twisted/plugins/cred_anonymous.py new file mode 100644 index 00000000000..8593518244d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/cred_anonymous.py @@ -0,0 +1,41 @@ +# -*- test-case-name: twisted.test.test_strcred -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Cred plugin for anonymous logins. +""" + +from __future__ import absolute_import, division + +from zope.interface import implementer + +from twisted import plugin +from twisted.cred.checkers import AllowAnonymousAccess +from twisted.cred.strcred import ICheckerFactory +from twisted.cred.credentials import IAnonymous + + +anonymousCheckerFactoryHelp = """ +This allows anonymous authentication for servers that support it. +""" + + +@implementer(ICheckerFactory, plugin.IPlugin) +class AnonymousCheckerFactory(object): + """ + Generates checkers that will authenticate an anonymous request. + """ + authType = 'anonymous' + authHelp = anonymousCheckerFactoryHelp + argStringFormat = 'No argstring required.' + credentialInterfaces = (IAnonymous,) + + + def generateChecker(self, argstring=''): + return AllowAnonymousAccess() + + + +theAnonymousCheckerFactory = AnonymousCheckerFactory() diff --git a/contrib/python/Twisted/py2/twisted/plugins/cred_file.py b/contrib/python/Twisted/py2/twisted/plugins/cred_file.py new file mode 100644 index 00000000000..66a5e334e24 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/cred_file.py @@ -0,0 +1,61 @@ +# -*- test-case-name: twisted.test.test_strcred -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Cred plugin for a file of the format 'username:password'. +""" + +from __future__ import absolute_import, division + +import sys + +from zope.interface import implementer + +from twisted import plugin +from twisted.cred.checkers import FilePasswordDB +from twisted.cred.strcred import ICheckerFactory +from twisted.cred.credentials import IUsernamePassword, IUsernameHashedPassword + + + +fileCheckerFactoryHelp = """ +This checker expects to receive the location of a file that +conforms to the FilePasswordDB format. Each line in the file +should be of the format 'username:password', in plain text. +""" + +invalidFileWarning = 'Warning: not a valid file' + + +@implementer(ICheckerFactory, plugin.IPlugin) +class FileCheckerFactory(object): + """ + A factory for instances of L{FilePasswordDB}. + """ + authType = 'file' + authHelp = fileCheckerFactoryHelp + argStringFormat = 'Location of a FilePasswordDB-formatted file.' + # Explicitly defined here because FilePasswordDB doesn't do it for us + credentialInterfaces = (IUsernamePassword, IUsernameHashedPassword) + + errorOutput = sys.stderr + + def generateChecker(self, argstring): + """ + This checker factory expects to get the location of a file. + The file should conform to the format required by + L{FilePasswordDB} (using defaults for all + initialization parameters). + """ + from twisted.python.filepath import FilePath + if not argstring.strip(): + raise ValueError('%r requires a filename' % self.authType) + elif not FilePath(argstring).isfile(): + self.errorOutput.write('%s: %s\n' % (invalidFileWarning, argstring)) + return FilePasswordDB(argstring) + + + +theFileCheckerFactory = FileCheckerFactory() diff --git a/contrib/python/Twisted/py2/twisted/plugins/cred_memory.py b/contrib/python/Twisted/py2/twisted/plugins/cred_memory.py new file mode 100644 index 00000000000..797963ca011 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/cred_memory.py @@ -0,0 +1,70 @@ +# -*- test-case-name: twisted.test.test_strcred -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Cred plugin for an in-memory user database. +""" + +from __future__ import absolute_import, division + +from zope.interface import implementer + +from twisted import plugin +from twisted.cred.strcred import ICheckerFactory +from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse +from twisted.cred.credentials import IUsernamePassword, IUsernameHashedPassword + + + +inMemoryCheckerFactoryHelp = """ +A checker that uses an in-memory user database. + +This is only of use in one-off test programs or examples which +don't want to focus too much on how credentials are verified. You +really don't want to use this for anything else. It is a toy. +""" + + + +@implementer(ICheckerFactory, plugin.IPlugin) +class InMemoryCheckerFactory(object): + """ + A factory for in-memory credentials checkers. + + This is only of use in one-off test programs or examples which don't + want to focus too much on how credentials are verified. + + You really don't want to use this for anything else. It is, at best, a + toy. If you need a simple credentials checker for a real application, + see L{cred_file.FileCheckerFactory}. + """ + authType = 'memory' + authHelp = inMemoryCheckerFactoryHelp + argStringFormat = 'A colon-separated list (name:password:...)' + credentialInterfaces = (IUsernamePassword, + IUsernameHashedPassword) + + def generateChecker(self, argstring): + """ + This checker factory expects to get a list of + username:password pairs, with each pair also separated by a + colon. For example, the string 'alice:f:bob:g' would generate + two users, one named 'alice' and one named 'bob'. + """ + checker = InMemoryUsernamePasswordDatabaseDontUse() + if argstring: + pieces = argstring.split(':') + if len(pieces) % 2: + from twisted.cred.strcred import InvalidAuthArgumentString + raise InvalidAuthArgumentString( + "argstring must be in format U:P:...") + for i in range(0, len(pieces), 2): + username, password = pieces[i], pieces[i+1] + checker.addUser(username, password) + return checker + + + +theInMemoryCheckerFactory = InMemoryCheckerFactory() diff --git a/contrib/python/Twisted/py2/twisted/plugins/cred_sshkeys.py b/contrib/python/Twisted/py2/twisted/plugins/cred_sshkeys.py new file mode 100644 index 00000000000..3d57f408626 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/cred_sshkeys.py @@ -0,0 +1,53 @@ +# -*- test-case-name: twisted.test.test_strcred -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Cred plugin for ssh key login. +""" + +from __future__ import absolute_import, division + +from zope.interface import implementer + +from twisted import plugin +from twisted.cred.strcred import ICheckerFactory + + +sshKeyCheckerFactoryHelp = """ +This allows SSH public key authentication, based on public keys listed in +authorized_keys and authorized_keys2 files in user .ssh/ directories. +""" + + +try: + from twisted.conch.checkers import ( + SSHPublicKeyChecker, UNIXAuthorizedKeysFiles) + + @implementer(ICheckerFactory, plugin.IPlugin) + class SSHKeyCheckerFactory(object): + """ + Generates checkers that will authenticate a SSH public key + """ + authType = 'sshkey' + authHelp = sshKeyCheckerFactoryHelp + argStringFormat = 'No argstring required.' + credentialInterfaces = SSHPublicKeyChecker.credentialInterfaces + + + def generateChecker(self, argstring=''): + """ + This checker factory ignores the argument string. Everything + needed to authenticate users is pulled out of the public keys + listed in user .ssh/ directories. + """ + return SSHPublicKeyChecker(UNIXAuthorizedKeysFiles()) + + + + theSSHKeyCheckerFactory = SSHKeyCheckerFactory() + +except ImportError: + # if checkers can't be imported, then there should be no SSH cred plugin + pass diff --git a/contrib/python/Twisted/py2/twisted/plugins/cred_unix.py b/contrib/python/Twisted/py2/twisted/plugins/cred_unix.py new file mode 100644 index 00000000000..7bab02b7667 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/cred_unix.py @@ -0,0 +1,185 @@ +# -*- test-case-name: twisted.test.test_strcred -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Cred plugin for UNIX user accounts. +""" + +from __future__ import absolute_import, division + +from zope.interface import implementer + +from twisted import plugin +from twisted.cred.strcred import ICheckerFactory +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer +from twisted.python.compat import StringType + + + +def verifyCryptedPassword(crypted, pw): + """ + Use L{crypt.crypt} to Verify that an unencrypted + password matches the encrypted password. + + @param crypted: The encrypted password, obtained from + the Unix password database or Unix shadow + password database. + @param pw: The unencrypted password. + @return: L{True} if there is successful match, else L{False}. + @rtype: L{bool} + """ + try: + import crypt + except ImportError: + crypt = None + + if crypt is None: + raise NotImplementedError("cred_unix not supported on this platform") + if not isinstance(pw, StringType): + pw = pw.decode('utf-8') + if not isinstance(crypted, StringType): + crypted = crypted.decode('utf-8') + return crypt.crypt(pw, crypted) == crypted + + + +@implementer(ICredentialsChecker) +class UNIXChecker(object): + """ + A credentials checker for a UNIX server. This will check that + an authenticating username/password is a valid user on the system. + + Does not work on Windows. + + Right now this supports Python's pwd and spwd modules, if they are + installed. It does not support PAM. + """ + credentialInterfaces = (IUsernamePassword,) + + + def checkPwd(self, pwd, username, password): + """ + Obtain the encrypted password for C{username} from the Unix password + database using L{pwd.getpwnam}, and see if it it matches it matches + C{password}. + + @param pwd: Module which provides functions which + access to the Unix password database. + @type pwd: C{module} + @param username: The user to look up in the Unix password database. + @type username: L{unicode}/L{str} or L{bytes} + @param password: The password to compare. + @type username: L{unicode}/L{str} or L{bytes} + """ + try: + if not isinstance(username, StringType): + username = username.decode('utf-8') + cryptedPass = pwd.getpwnam(username).pw_passwd + except KeyError: + return defer.fail(UnauthorizedLogin()) + else: + if cryptedPass in ('*', 'x'): + # Allow checkSpwd to take over + return None + elif verifyCryptedPassword(cryptedPass, password): + return defer.succeed(username) + + + def checkSpwd(self, spwd, username, password): + """ + Obtain the encrypted password for C{username} from the + Unix shadow password database using L{spwd.getspnam}, + and see if it it matches it matches C{password}. + + @param spwd: Module which provides functions which + access to the Unix shadow password database. + @type pwd: C{module} + @param username: The user to look up in the Unix password database. + @type username: L{unicode}/L{str} or L{bytes} + @param password: The password to compare. + @type username: L{unicode}/L{str} or L{bytes} + """ + try: + if not isinstance(username, StringType): + username = username.decode('utf-8') + if getattr(spwd.struct_spwd, "sp_pwdp", None): + # Python 3 + cryptedPass = spwd.getspnam(username).sp_pwdp + else: + # Python 2 + cryptedPass = spwd.getspnam(username).sp_pwd + except KeyError: + return defer.fail(UnauthorizedLogin()) + else: + if verifyCryptedPassword(cryptedPass, password): + return defer.succeed(username) + + + def requestAvatarId(self, credentials): + username, password = credentials.username, credentials.password + + try: + import pwd + except ImportError: + pwd = None + + if pwd is not None: + checked = self.checkPwd(pwd, username, password) + if checked is not None: + return checked + + try: + import spwd + except ImportError: + spwd = None + + if spwd is not None: + checked = self.checkSpwd(spwd, username, password) + if checked is not None: + return checked + # TODO: check_pam? + # TODO: check_shadow? + return defer.fail(UnauthorizedLogin()) + + + +unixCheckerFactoryHelp = """ +This checker will attempt to use every resource available to +authenticate against the list of users on the local UNIX system. +(This does not support Windows servers for very obvious reasons.) + +Right now, this includes support for: + + * Python's pwd module (which checks /etc/passwd) + * Python's spwd module (which checks /etc/shadow) + +Future versions may include support for PAM authentication. +""" + + +@implementer(ICheckerFactory, plugin.IPlugin) +class UNIXCheckerFactory(object): + """ + A factory for L{UNIXChecker}. + """ + authType = 'unix' + authHelp = unixCheckerFactoryHelp + argStringFormat = 'No argstring required.' + credentialInterfaces = UNIXChecker.credentialInterfaces + + def generateChecker(self, argstring): + """ + This checker factory ignores the argument string. Everything + needed to generate a user database is pulled out of the local + UNIX environment. + """ + return UNIXChecker() + + + +theUnixCheckerFactory = UNIXCheckerFactory() diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_conch.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_conch.py new file mode 100644 index 00000000000..4b37e0b21f6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_conch.py @@ -0,0 +1,18 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedSSH = ServiceMaker( + "Twisted Conch Server", + "twisted.conch.tap", + "A Conch SSH service.", + "conch") + +TwistedManhole = ServiceMaker( + "Twisted Manhole (new)", + "twisted.conch.manhole_tap", + ("An interactive remote debugger service accessible via telnet " + "and ssh and providing syntax coloring and basic line editing " + "functionality."), + "manhole") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_core.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_core.py new file mode 100644 index 00000000000..3fb52f0c786 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_core.py @@ -0,0 +1,18 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division + +from twisted.internet.endpoints import ( + _SystemdParser, _TCP6ServerParser, _StandardIOParser, + _TLSClientEndpointParser) + +from twisted.protocols.haproxy._parser import ( + HAProxyServerParser as _HAProxyServerParser +) + +systemdEndpointParser = _SystemdParser() +tcp6ServerEndpointParser = _TCP6ServerParser() +stdioEndpointParser = _StandardIOParser() +tlsClientEndpointParser = _TLSClientEndpointParser() +_haProxyServerEndpointParser = _HAProxyServerParser() diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_ftp.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_ftp.py new file mode 100644 index 00000000000..474a9c7cde4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_ftp.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedFTP = ServiceMaker( + "Twisted FTP", + "twisted.tap.ftp", + "An FTP server.", + "ftp") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_inet.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_inet.py new file mode 100644 index 00000000000..1196343becb --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_inet.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedINETD = ServiceMaker( + "Twisted INETD Server", + "twisted.runner.inetdtap", + "An inetd(8) replacement.", + "inetd") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_mail.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_mail.py new file mode 100644 index 00000000000..7e9a5bd57b2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_mail.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedMail = ServiceMaker( + "Twisted Mail", + "twisted.mail.tap", + "An email service", + "mail") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_names.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_names.py new file mode 100644 index 00000000000..7123bf00dfc --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_names.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedNames = ServiceMaker( + "Twisted DNS Server", + "twisted.names.tap", + "A domain name server.", + "dns") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_news.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_news.py new file mode 100644 index 00000000000..0fc88d8144f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_news.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedNews = ServiceMaker( + "Twisted News", + "twisted.news.tap", + "A news server.", + "news") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_portforward.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_portforward.py new file mode 100644 index 00000000000..1969434b752 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_portforward.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedPortForward = ServiceMaker( + "Twisted Port-Forwarding", + "twisted.tap.portforward", + "A simple port-forwarder.", + "portforward") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_reactors.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_reactors.py new file mode 100644 index 00000000000..ebb474ad7fa --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_reactors.py @@ -0,0 +1,71 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division + +from twisted.application.reactors import Reactor + +__all__ = [] + +default = Reactor( + 'default', 'twisted.internet.default', + 'A reasonable default: poll(2) if available, otherwise select(2).') +__all__.append('default') + +select = Reactor( + 'select', 'twisted.internet.selectreactor', 'select(2) based reactor.') +__all__.append('select') + +poll = Reactor( + 'poll', 'twisted.internet.pollreactor', 'poll(2) based reactor.') +__all__.append('poll') + +epoll = Reactor( + 'epoll', 'twisted.internet.epollreactor', 'epoll(4) based reactor.') +__all__.append('epoll') + +kqueue = Reactor( + 'kqueue', 'twisted.internet.kqreactor', 'kqueue(2) based reactor.') +__all__.append('kqueue') + +cf = Reactor( + 'cf' , 'twisted.internet.cfreactor', + 'CoreFoundation based reactor.') +__all__.append('cf') + +asyncio = Reactor( + 'asyncio', 'twisted.internet.asyncioreactor', + 'asyncio based reactor') +__all__.append('asyncio') + +wx = Reactor( + 'wx', 'twisted.internet.wxreactor', 'wxPython based reactor.') +__all__.append('wx') + +gi = Reactor( + 'gi', 'twisted.internet.gireactor', + 'GObject Introspection based reactor.') +__all__.append('gi') + +gtk3 = Reactor( + 'gtk3', 'twisted.internet.gtk3reactor', 'Gtk3 based reactor.') +__all__.append('gtk3') + +gtk2 = Reactor( + 'gtk2', 'twisted.internet.gtk2reactor', 'Gtk2 based reactor.') +__all__.append('gtk2') + +glib2 = Reactor( + 'glib2', 'twisted.internet.glib2reactor', + 'GLib2 based reactor.') +__all__.append('glib2') + +win32er = Reactor( + 'win32', 'twisted.internet.win32eventreactor', + 'Win32 WaitForMultipleObjects based reactor.') +__all__.append('win32er') + +iocp = Reactor( + 'iocp', 'twisted.internet.iocpreactor', + 'Win32 IO Completion Ports based reactor.') +__all__.append('iocp') diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_runner.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_runner.py new file mode 100644 index 00000000000..dc630281e20 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_runner.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedProcmon = ServiceMaker( + "Twisted Process Monitor", + "twisted.runner.procmontap", + ("A process watchdog / supervisor"), + "procmon") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_socks.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_socks.py new file mode 100644 index 00000000000..5a94f871ac9 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_socks.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedSOCKS = ServiceMaker( + "Twisted SOCKS", + "twisted.tap.socks", + "A SOCKSv4 proxy service.", + "socks") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_trial.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_trial.py new file mode 100644 index 00000000000..06c2d069360 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_trial.py @@ -0,0 +1,62 @@ +from __future__ import division, absolute_import + +from zope.interface import implementer + +from twisted.trial.itrial import IReporter +from twisted.plugin import IPlugin + + + +@implementer(IPlugin, IReporter) +class _Reporter(object): + + def __init__(self, name, module, description, longOpt, shortOpt, klass): + self.name = name + self.module = module + self.description = description + self.longOpt = longOpt + self.shortOpt = shortOpt + self.klass = klass + + +Tree = _Reporter("Tree Reporter", + "twisted.trial.reporter", + description="verbose color output (default reporter)", + longOpt="verbose", + shortOpt="v", + klass="TreeReporter") + +BlackAndWhite = _Reporter("Black-And-White Reporter", + "twisted.trial.reporter", + description="Colorless verbose output", + longOpt="bwverbose", + shortOpt="o", + klass="VerboseTextReporter") + +Minimal = _Reporter("Minimal Reporter", + "twisted.trial.reporter", + description="minimal summary output", + longOpt="summary", + shortOpt="s", + klass="MinimalReporter") + +Classic = _Reporter("Classic Reporter", + "twisted.trial.reporter", + description="terse text output", + longOpt="text", + shortOpt="t", + klass="TextReporter") + +Timing = _Reporter("Timing Reporter", + "twisted.trial.reporter", + description="Timing output", + longOpt="timing", + shortOpt=None, + klass="TimingTextReporter") + +Subunit = _Reporter("Subunit Reporter", + "twisted.trial.reporter", + description="subunit output", + longOpt="subunit", + shortOpt=None, + klass="SubunitReporter") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_web.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_web.py new file mode 100644 index 00000000000..c7655a6d074 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_web.py @@ -0,0 +1,11 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from twisted.application.service import ServiceMaker + +TwistedWeb = ServiceMaker( + "Twisted Web", + "twisted.web.tap", + ("A general-purpose web server which can serve from a " + "filesystem or application resource."), + "web") diff --git a/contrib/python/Twisted/py2/twisted/plugins/twisted_words.py b/contrib/python/Twisted/py2/twisted/plugins/twisted_words.py new file mode 100644 index 00000000000..b8dd500e32a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/plugins/twisted_words.py @@ -0,0 +1,47 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from zope.interface import provider + +from twisted.plugin import IPlugin + +from twisted.application.service import ServiceMaker +from twisted.words import iwords + + +NewTwistedWords = ServiceMaker( + "New Twisted Words", + "twisted.words.tap", + "A modern words server", + "words") + +TwistedXMPPRouter = ServiceMaker( + "XMPP Router", + "twisted.words.xmpproutertap", + "An XMPP Router server", + "xmpp-router") + + + +@provider(IPlugin, iwords.IProtocolPlugin) +class RelayChatInterface(object): + + name = 'irc' + + def getFactory(cls, realm, portal): + from twisted.words import service + return service.IRCFactory(realm, portal) + getFactory = classmethod(getFactory) + + + +@provider(IPlugin, iwords.IProtocolPlugin) +class PBChatInterface(object): + + name = 'pb' + + def getFactory(cls, realm, portal): + from twisted.spread import pb + return pb.PBServerFactory(portal, True) + getFactory = classmethod(getFactory) + diff --git a/contrib/python/Twisted/py2/twisted/positioning/__init__.py b/contrib/python/Twisted/py2/twisted/positioning/__init__.py new file mode 100644 index 00000000000..a855f41dd8e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/positioning/__init__.py @@ -0,0 +1,8 @@ +# -*- test-case-name: twisted.positioning.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +""" +Twisted Positioning: Framework for applications that make use of positioning. + +@since: 14.0 +""" diff --git a/contrib/python/Twisted/py2/twisted/positioning/_sentence.py b/contrib/python/Twisted/py2/twisted/positioning/_sentence.py new file mode 100644 index 00000000000..25c7dc19d85 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/positioning/_sentence.py @@ -0,0 +1,122 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +""" +Generic sentence handling tools: hopefully reusable. +""" + +from __future__ import absolute_import, division + + +class _BaseSentence(object): + """ + A base sentence class for a particular protocol. + + Using this base class, specific sentence classes can almost automatically + be created for a particular protocol. + To do this, fill the ALLOWED_ATTRIBUTES class attribute using + the C{getSentenceAttributes} class method of the producer:: + + class FooSentence(BaseSentence): + \"\"\" + A sentence for integalactic transmodulator sentences. + + @ivar transmogrificationConstant: The value used in the + transmogrifier while producing this sentence, corrected for + gravitational fields. + @type transmogrificationConstant: C{Tummy} + \"\"\" + ALLOWED_ATTRIBUTES = FooProtocol.getSentenceAttributes() + + @ivar presentAttributes: An iterable containing the names of the + attributes that are present in this sentence. + @type presentAttributes: iterable of C{str} + + @cvar ALLOWED_ATTRIBUTES: A set of attributes that are allowed in this + sentence. + @type ALLOWED_ATTRIBUTES: C{set} of C{str} + """ + ALLOWED_ATTRIBUTES = set() + + + def __init__(self, sentenceData): + """ + Initializes a sentence with parsed sentence data. + + @param sentenceData: The parsed sentence data. + @type sentenceData: C{dict} (C{str} -> C{str} or L{None}) + """ + self._sentenceData = sentenceData + + + @property + def presentAttributes(self): + """ + An iterable containing the names of the attributes that are present in + this sentence. + + @return: The iterable of names of present attributes. + @rtype: iterable of C{str} + """ + return iter(self._sentenceData) + + + def __getattr__(self, name): + """ + Gets an attribute of this sentence. + """ + if name in self.ALLOWED_ATTRIBUTES: + return self._sentenceData.get(name, None) + else: + className = self.__class__.__name__ + msg = "%s sentences have no %s attributes" % (className, name) + raise AttributeError(msg) + + + def __repr__(self): + """ + Returns a textual representation of this sentence. + + @return: A textual representation of this sentence. + @rtype: C{str} + """ + items = self._sentenceData.items() + data = ["%s: %s" % (k, v) for k, v in sorted(items) if k != "type"] + dataRepr = ", ".join(data) + + typeRepr = self._sentenceData.get("type") or "unknown type" + className = self.__class__.__name__ + + return "<%s (%s) {%s}>" % (className, typeRepr, dataRepr) + + + +class _PositioningSentenceProducerMixin(object): + """ + A mixin for certain protocols that produce positioning sentences. + + This mixin helps protocols that store the layout of sentences that they + consume in a C{_SENTENCE_CONTENTS} class variable provide all sentence + attributes that can ever occur. It does this by providing a class method, + C{getSentenceAttributes}, which iterates over all sentence types and + collects the possible sentence attributes. + """ + @classmethod + def getSentenceAttributes(cls): + """ + Returns a set of all attributes that might be found in the sentences + produced by this protocol. + + This is basically a set of all the attributes of all the sentences that + this protocol can produce. + + @return: The set of all possible sentence attribute names. + @rtype: C{set} of C{str} + """ + attributes = set(["type"]) + for attributeList in cls._SENTENCE_CONTENTS.values(): + for attribute in attributeList: + if attribute is None: + continue + attributes.add(attribute) + + return attributes diff --git a/contrib/python/Twisted/py2/twisted/positioning/base.py b/contrib/python/Twisted/py2/twisted/positioning/base.py new file mode 100644 index 00000000000..c659ebecafa --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/positioning/base.py @@ -0,0 +1,947 @@ +# -*- test-case-name: twisted.positioning.test.test_base,twisted.positioning.test.test_sentence -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +""" +Generic positioning base classes. + +@since: 14.0 +""" + +from __future__ import absolute_import, division + +from functools import partial +from operator import attrgetter +from zope.interface import implementer +from constantly import Names, NamedConstant + +from twisted.python.util import FancyEqMixin +from twisted.positioning import ipositioning + + +MPS_PER_KNOT = 0.5144444444444444 +MPS_PER_KPH = 0.27777777777777777 +METERS_PER_FOOT = 0.3048 + + + +class Angles(Names): + """ + The types of angles. + + @cvar LATITUDE: Angle representing a latitude of an object. + @type LATITUDE: L{NamedConstant} + + @cvar LONGITUDE: Angle representing the longitude of an object. + @type LONGITUDE: L{NamedConstant} + + @cvar HEADING: Angle representing the heading of an object. + @type HEADING: L{NamedConstant} + + @cvar VARIATION: Angle representing a magnetic variation. + @type VARIATION: L{NamedConstant} + + """ + LATITUDE = NamedConstant() + LONGITUDE = NamedConstant() + HEADING = NamedConstant() + VARIATION = NamedConstant() + + + +class Directions(Names): + """ + The four cardinal directions (north, east, south, west). + """ + NORTH = NamedConstant() + EAST = NamedConstant() + SOUTH = NamedConstant() + WEST = NamedConstant() + + + +@implementer(ipositioning.IPositioningReceiver) +class BasePositioningReceiver(object): + """ + A base positioning receiver. + + This class would be a good base class for building positioning + receivers. It implements the interface (so you don't have to) with stub + methods. + + People who want to implement positioning receivers should subclass this + class and override the specific callbacks they want to handle. + """ + def timeReceived(self, time): + """ + Implements L{IPositioningReceiver.timeReceived} stub. + """ + + + def headingReceived(self, heading): + """ + Implements L{IPositioningReceiver.headingReceived} stub. + """ + + + def speedReceived(self, speed): + """ + Implements L{IPositioningReceiver.speedReceived} stub. + """ + + + def climbReceived(self, climb): + """ + Implements L{IPositioningReceiver.climbReceived} stub. + """ + + + def positionReceived(self, latitude, longitude): + """ + Implements L{IPositioningReceiver.positionReceived} stub. + """ + + + def positionErrorReceived(self, positionError): + """ + Implements L{IPositioningReceiver.positionErrorReceived} stub. + """ + + + def altitudeReceived(self, altitude): + """ + Implements L{IPositioningReceiver.altitudeReceived} stub. + """ + + + def beaconInformationReceived(self, beaconInformation): + """ + Implements L{IPositioningReceiver.beaconInformationReceived} stub. + """ + + + +class InvalidSentence(Exception): + """ + An exception raised when a sentence is invalid. + """ + + + +class InvalidChecksum(Exception): + """ + An exception raised when the checksum of a sentence is invalid. + """ + + + +class Angle(FancyEqMixin, object): + """ + An object representing an angle. + + @cvar _RANGE_EXPRESSIONS: A collection of expressions for the allowable + range for the angular value of a particular coordinate value. + @type _RANGE_EXPRESSIONS: C{dict} of L{Angles} constants to callables + @cvar _ANGLE_TYPE_NAMES: English names for angle types. + @type _ANGLE_TYPE_NAMES: C{dict} of L{Angles} constants to C{str} + """ + _RANGE_EXPRESSIONS = { + Angles.LATITUDE: lambda latitude: -90.0 < latitude < 90.0, + Angles.LONGITUDE: lambda longitude: -180.0 < longitude < 180.0, + Angles.HEADING: lambda heading: 0 <= heading < 360, + Angles.VARIATION: lambda variation: -180 < variation <= 180, + } + + + _ANGLE_TYPE_NAMES = { + Angles.LATITUDE: "Latitude", + Angles.LONGITUDE: "Longitude", + Angles.VARIATION: "Variation", + Angles.HEADING: "Heading", + } + + + compareAttributes = 'angleType', 'inDecimalDegrees' + + + def __init__(self, angle=None, angleType=None): + """ + Initializes an angle. + + @param angle: The value of the angle in decimal degrees. (L{None} if + unknown). + @type angle: C{float} or L{None} + + @param angleType: A symbolic constant describing the angle type. Should + be one of L{Angles} or {None} if unknown. + + @raises ValueError: If the angle type is not the default argument, + but it is an unknown type (not in C{Angle._RANGE_EXPRESSIONS}), + or it is a known type but the supplied value was out of the + allowable range for said type. + """ + if angleType is not None and angleType not in self._RANGE_EXPRESSIONS: + raise ValueError("Unknown angle type") + + if angle is not None and angleType is not None: + rangeExpression = self._RANGE_EXPRESSIONS[angleType] + if not rangeExpression(angle): + template = "Angle {0} not in allowed range for type {1}" + raise ValueError(template.format(angle, angleType)) + + self.angleType = angleType + self._angle = angle + + + @property + def inDecimalDegrees(self): + """ + The value of this angle in decimal degrees. This value is immutable. + + @return: This angle expressed in decimal degrees, or L{None} if the + angle is unknown. + @rtype: C{float} (or L{None}) + """ + return self._angle + + + @property + def inDegreesMinutesSeconds(self): + """ + The value of this angle as a degrees, minutes, seconds tuple. This + value is immutable. + + @return: This angle expressed in degrees, minutes, seconds. L{None} if + the angle is unknown. + @rtype: 3-C{tuple} of C{int} (or L{None}) + """ + if self._angle is None: + return None + + degrees = abs(int(self._angle)) + fractionalDegrees = abs(self._angle - int(self._angle)) + decimalMinutes = 60 * fractionalDegrees + + minutes = int(decimalMinutes) + fractionalMinutes = decimalMinutes - int(decimalMinutes) + decimalSeconds = 60 * fractionalMinutes + + return degrees, minutes, int(decimalSeconds) + + + def setSign(self, sign): + """ + Sets the sign of this angle. + + @param sign: The new sign. C{1} for positive and C{-1} for negative + signs, respectively. + @type sign: C{int} + + @raise ValueError: If the C{sign} parameter is not C{-1} or C{1}. + """ + if sign not in (-1, 1): + raise ValueError("bad sign (got %s, expected -1 or 1)" % sign) + + self._angle = sign * abs(self._angle) + + + def __float__(self): + """ + Returns this angle as a float. + + @return: The float value of this angle, expressed in degrees. + @rtype: C{float} + """ + return self._angle + + + def __repr__(self): + """ + Returns a string representation of this angle. + + @return: The string representation. + @rtype: C{str} + """ + return "<{s._angleTypeNameRepr} ({s._angleValueRepr})>".format(s=self) + + + @property + def _angleValueRepr(self): + """ + Returns a string representation of the angular value of this angle. + + This is a helper function for the actual C{__repr__}. + + @return: The string representation. + @rtype: C{str} + """ + if self.inDecimalDegrees is not None: + return "%s degrees" % round(self.inDecimalDegrees, 2) + else: + return "unknown value" + + + @property + def _angleTypeNameRepr(self): + """ + Returns a string representation of the type of this angle. + + This is a helper function for the actual C{__repr__}. + + @return: The string representation. + @rtype: C{str} + """ + try: + return self._ANGLE_TYPE_NAMES[self.angleType] + except KeyError: + return "Angle of unknown type" + + + +class Heading(Angle): + """ + The heading of a mobile object. + + @ivar variation: The (optional) magnetic variation. + The sign of the variation is positive for variations towards the east + (clockwise from north), and negative for variations towards the west + (counterclockwise from north). + If the variation is unknown or not applicable, this is L{None}. + @type variation: C{Angle} or L{None}. + @ivar correctedHeading: The heading, corrected for variation. If the + variation is unknown (L{None}), is None. This attribute is read-only + (its value is determined by the angle and variation attributes). The + value is coerced to being between 0 (inclusive) and 360 (exclusive). + """ + def __init__(self, angle=None, variation=None): + """ + Initializes an angle with an optional variation. + """ + Angle.__init__(self, angle, Angles.HEADING) + self.variation = variation + + + @classmethod + def fromFloats(cls, angleValue=None, variationValue=None): + """ + Constructs a Heading from the float values of the angle and variation. + + @param angleValue: The angle value of this heading. + @type angleValue: C{float} + @param variationValue: The value of the variation of this heading. + @type variationValue: C{float} + @return A C{Heading } with the given values. + """ + variation = Angle(variationValue, Angles.VARIATION) + return cls(angleValue, variation) + + + @property + def correctedHeading(self): + """ + Corrects the heading by the given variation. This is sometimes known as + the true heading. + + @return: The heading, corrected by the variation. If the variation or + the angle are unknown, returns L{None}. + @rtype: C{float} or L{None} + """ + if self._angle is None or self.variation is None: + return None + + angle = (self.inDecimalDegrees - self.variation.inDecimalDegrees) % 360 + return Angle(angle, Angles.HEADING) + + + def setSign(self, sign): + """ + Sets the sign of the variation of this heading. + + @param sign: The new sign. C{1} for positive and C{-1} for negative + signs, respectively. + @type sign: C{int} + + @raise ValueError: If the C{sign} parameter is not C{-1} or C{1}. + """ + if self.variation.inDecimalDegrees is None: + raise ValueError("can't set the sign of an unknown variation") + + self.variation.setSign(sign) + + + compareAttributes = list(Angle.compareAttributes) + ["variation"] + + + def __repr__(self): + """ + Returns a string representation of this angle. + + @return: The string representation. + @rtype: C{str} + """ + if self.variation is None: + variationRepr = "unknown variation" + else: + variationRepr = repr(self.variation) + + return "<%s (%s, %s)>" % ( + self._angleTypeNameRepr, self._angleValueRepr, variationRepr) + + + +class Coordinate(Angle): + """ + A coordinate. + + @ivar angle: The value of the coordinate in decimal degrees, with the usual + rules for sign (northern and eastern hemispheres are positive, southern + and western hemispheres are negative). + @type angle: C{float} + """ + def __init__(self, angle, coordinateType=None): + """ + Initializes a coordinate. + + @param angle: The angle of this coordinate in decimal degrees. The + hemisphere is determined by the sign (north and east are positive). + If this coordinate describes a latitude, this value must be within + -90.0 and +90.0 (exclusive). If this value describes a longitude, + this value must be within -180.0 and +180.0 (exclusive). + @type angle: C{float} + @param coordinateType: The coordinate type. One of L{Angles.LATITUDE}, + L{Angles.LONGITUDE} or L{None} if unknown. + """ + if coordinateType not in [Angles.LATITUDE, Angles.LONGITUDE, None]: + raise ValueError("coordinateType must be one of Angles.LATITUDE, " + "Angles.LONGITUDE or None, was {!r}" + .format(coordinateType)) + + Angle.__init__(self, angle, coordinateType) + + + @property + def hemisphere(self): + """ + Gets the hemisphere of this coordinate. + + @return: A symbolic constant representing a hemisphere (one of + L{Angles}) + """ + + if self.angleType is Angles.LATITUDE: + if self.inDecimalDegrees < 0: + return Directions.SOUTH + else: + return Directions.NORTH + elif self.angleType is Angles.LONGITUDE: + if self.inDecimalDegrees < 0: + return Directions.WEST + else: + return Directions.EAST + else: + raise ValueError("unknown coordinate type (cant find hemisphere)") + + + +class Altitude(FancyEqMixin, object): + """ + An altitude. + + @ivar inMeters: The altitude represented by this object, in meters. This + attribute is read-only. + @type inMeters: C{float} + + @ivar inFeet: As above, but expressed in feet. + @type inFeet: C{float} + """ + compareAttributes = 'inMeters', + + def __init__(self, altitude): + """ + Initializes an altitude. + + @param altitude: The altitude in meters. + @type altitude: C{float} + """ + self._altitude = altitude + + + @property + def inFeet(self): + """ + Gets the altitude this object represents, in feet. + + @return: The altitude, expressed in feet. + @rtype: C{float} + """ + return self._altitude / METERS_PER_FOOT + + + @property + def inMeters(self): + """ + Returns the altitude this object represents, in meters. + + @return: The altitude, expressed in feet. + @rtype: C{float} + """ + return self._altitude + + + def __float__(self): + """ + Returns the altitude represented by this object expressed in meters. + + @return: The altitude represented by this object, expressed in meters. + @rtype: C{float} + """ + return self._altitude + + + def __repr__(self): + """ + Returns a string representation of this altitude. + + @return: The string representation. + @rtype: C{str} + """ + return "" % (self._altitude,) + + + +class _BaseSpeed(FancyEqMixin, object): + """ + An object representing the abstract concept of the speed (rate of + movement) of a mobile object. + + This primarily has behavior for converting between units and comparison. + """ + compareAttributes = 'inMetersPerSecond', + + def __init__(self, speed): + """ + Initializes a speed. + + @param speed: The speed that this object represents, expressed in + meters per second. + @type speed: C{float} + + @raises ValueError: Raised if value was invalid for this particular + kind of speed. Only happens in subclasses. + """ + self._speed = speed + + + @property + def inMetersPerSecond(self): + """ + The speed that this object represents, expressed in meters per second. + This attribute is immutable. + + @return: The speed this object represents, in meters per second. + @rtype: C{float} + """ + return self._speed + + + @property + def inKnots(self): + """ + Returns the speed represented by this object, expressed in knots. This + attribute is immutable. + + @return: The speed this object represents, in knots. + @rtype: C{float} + """ + return self._speed / MPS_PER_KNOT + + + def __float__(self): + """ + Returns the speed represented by this object expressed in meters per + second. + + @return: The speed represented by this object, expressed in meters per + second. + @rtype: C{float} + """ + return self._speed + + + def __repr__(self): + """ + Returns a string representation of this speed object. + + @return: The string representation. + @rtype: C{str} + """ + speedValue = round(self.inMetersPerSecond, 2) + return "<%s (%s m/s)>" % (self.__class__.__name__, speedValue) + + + +class Speed(_BaseSpeed): + """ + The speed (rate of movement) of a mobile object. + """ + def __init__(self, speed): + """ + Initializes a L{Speed} object. + + @param speed: The speed that this object represents, expressed in + meters per second. + @type speed: C{float} + + @raises ValueError: Raised if C{speed} is negative. + """ + if speed < 0: + raise ValueError("negative speed: %r" % (speed,)) + + _BaseSpeed.__init__(self, speed) + + + +class Climb(_BaseSpeed): + """ + The climb ("vertical speed") of an object. + """ + def __init__(self, climb): + """ + Initializes a L{Climb} object. + + @param climb: The climb that this object represents, expressed in + meters per second. + @type climb: C{float} + """ + _BaseSpeed.__init__(self, climb) + + + +class PositionError(FancyEqMixin, object): + """ + Position error information. + + @cvar _ALLOWABLE_THRESHOLD: The maximum allowable difference between PDOP + and the geometric mean of VDOP and HDOP. That difference is supposed + to be zero, but can be non-zero because of rounding error and limited + reporting precision. You should never have to change this value. + @type _ALLOWABLE_THRESHOLD: C{float} + @cvar _DOP_EXPRESSIONS: A mapping of DOP types (C[hvp]dop) to a list of + callables that take self and return that DOP type, or raise + C{TypeError}. This allows a DOP value to either be returned directly + if it's know, or computed from other DOP types if it isn't. + @type _DOP_EXPRESSIONS: C{dict} of C{str} to callables + @ivar pdop: The position dilution of precision. L{None} if unknown. + @type pdop: C{float} or L{None} + @ivar hdop: The horizontal dilution of precision. L{None} if unknown. + @type hdop: C{float} or L{None} + @ivar vdop: The vertical dilution of precision. L{None} if unknown. + @type vdop: C{float} or L{None} + """ + compareAttributes = 'pdop', 'hdop', 'vdop' + + def __init__(self, pdop=None, hdop=None, vdop=None, testInvariant=False): + """ + Initializes a positioning error object. + + @param pdop: The position dilution of precision. L{None} if unknown. + @type pdop: C{float} or L{None} + @param hdop: The horizontal dilution of precision. L{None} if unknown. + @type hdop: C{float} or L{None} + @param vdop: The vertical dilution of precision. L{None} if unknown. + @type vdop: C{float} or L{None} + @param testInvariant: Flag to test if the DOP invariant is valid or + not. If C{True}, the invariant (PDOP = (HDOP**2 + VDOP**2)*.5) is + checked at every mutation. By default, this is false, because the + vast majority of DOP-providing devices ignore this invariant. + @type testInvariant: c{bool} + """ + self._pdop = pdop + self._hdop = hdop + self._vdop = vdop + + self._testInvariant = testInvariant + self._testDilutionOfPositionInvariant() + + + _ALLOWABLE_TRESHOLD = 0.01 + + + def _testDilutionOfPositionInvariant(self): + """ + Tests if this positioning error object satisfies the dilution of + position invariant (PDOP = (HDOP**2 + VDOP**2)*.5), unless the + C{self._testInvariant} instance variable is C{False}. + + @return: L{None} if the invariant was not satisfied or not tested. + @raises ValueError: Raised if the invariant was tested but not + satisfied. + """ + if not self._testInvariant: + return + + for x in (self.pdop, self.hdop, self.vdop): + if x is None: + return + + delta = abs(self.pdop - (self.hdop**2 + self.vdop**2)**.5) + if delta > self._ALLOWABLE_TRESHOLD: + raise ValueError("invalid combination of dilutions of precision: " + "position: %s, horizontal: %s, vertical: %s" + % (self.pdop, self.hdop, self.vdop)) + + + _DOP_EXPRESSIONS = { + 'pdop': [ + lambda self: float(self._pdop), + lambda self: (self._hdop**2 + self._vdop**2)**.5, + ], + + 'hdop': [ + lambda self: float(self._hdop), + lambda self: (self._pdop**2 - self._vdop**2)**.5, + ], + + 'vdop': [ + lambda self: float(self._vdop), + lambda self: (self._pdop**2 - self._hdop**2)**.5, + ], + } + + + def _getDOP(self, dopType): + """ + Gets a particular dilution of position value. + + @param dopType: The type of dilution of position to get. One of + ('pdop', 'hdop', 'vdop'). + @type dopType: C{str} + @return: The DOP if it is known, L{None} otherwise. + @rtype: C{float} or L{None} + """ + for dopExpression in self._DOP_EXPRESSIONS[dopType]: + try: + return dopExpression(self) + except TypeError: + continue + + + def _setDOP(self, dopType, value): + """ + Sets a particular dilution of position value. + + @param dopType: The type of dilution of position to set. One of + ('pdop', 'hdop', 'vdop'). + @type dopType: C{str} + + @param value: The value to set the dilution of position type to. + @type value: C{float} + + If this position error tests dilution of precision invariants, + it will be checked. If the invariant is not satisfied, the + assignment will be undone and C{ValueError} is raised. + """ + attributeName = "_" + dopType + + oldValue = getattr(self, attributeName) + setattr(self, attributeName, float(value)) + + try: + self._testDilutionOfPositionInvariant() + except ValueError: + setattr(self, attributeName, oldValue) + raise + + + pdop = property(fget=lambda self: self._getDOP('pdop'), + fset=lambda self, value: self._setDOP('pdop', value)) + + + hdop = property(fget=lambda self: self._getDOP('hdop'), + fset=lambda self, value: self._setDOP('hdop', value)) + + + vdop = property(fget=lambda self: self._getDOP('vdop'), + fset=lambda self, value: self._setDOP('vdop', value)) + + + _REPR_TEMPLATE = "" + + + def __repr__(self): + """ + Returns a string representation of positioning information object. + + @return: The string representation. + @rtype: C{str} + """ + return self._REPR_TEMPLATE % (self.pdop, self.hdop, self.vdop) + + + +class BeaconInformation(object): + """ + Information about positioning beacons (a generalized term for the reference + objects that help you determine your position, such as satellites or cell + towers). + + @ivar seenBeacons: A set of visible beacons. Note that visible beacons are not + necessarily used in acquiring a positioning fix. + @type seenBeacons: C{set} of L{IPositioningBeacon} + @ivar usedBeacons: A set of the beacons that were used in obtaining a + positioning fix. This only contains beacons that are actually used, not + beacons for which it is unknown if they are used or not. + @type usedBeacons: C{set} of L{IPositioningBeacon} + """ + def __init__(self, seenBeacons=()): + """ + Initializes a beacon information object. + + @param seenBeacons: A collection of beacons that are currently seen. + @type seenBeacons: iterable of L{IPositioningBeacon}s + """ + self.seenBeacons = set(seenBeacons) + self.usedBeacons = set() + + + def __repr__(self): + """ + Returns a string representation of this beacon information object. + + The beacons are sorted by their identifier. + + @return: The string representation. + @rtype: C{str} + """ + sortedBeacons = partial(sorted, key=attrgetter("identifier")) + + usedBeacons = sortedBeacons(self.usedBeacons) + unusedBeacons = sortedBeacons(self.seenBeacons - self.usedBeacons) + + template = ("") + + formatted = template.format(numUsed=len(self.usedBeacons), + usedBeacons=usedBeacons, + unusedBeacons=unusedBeacons) + + return formatted + + + +@implementer(ipositioning.IPositioningBeacon) +class PositioningBeacon(object): + """ + A positioning beacon. + + @ivar identifier: The unique identifier for this beacon. This is usually + an integer. For GPS, this is also known as the PRN. + @type identifier: Pretty much anything that can be used as a unique + identifier. Depends on the implementation. + """ + def __init__(self, identifier): + """ + Initializes a positioning beacon. + + @param identifier: The identifier for this beacon. + @type identifier: Can be pretty much anything (see ivar documentation). + """ + self.identifier = identifier + + + def __hash__(self): + """ + Returns the hash of the identifier for this beacon. + + @return: The hash of the identifier. (C{hash(self.identifier)}) + @rtype: C{int} + """ + return hash(self.identifier) + + + def __repr__(self): + """ + Returns a string representation of this beacon. + + @return: The string representation. + @rtype: C{str} + """ + return "".format(s=self) + + + +class Satellite(PositioningBeacon): + """ + A satellite. + + @ivar azimuth: The azimuth of the satellite. This is the heading (positive + angle relative to true north) where the satellite appears to be to the + device. + @ivar elevation: The (positive) angle above the horizon where this + satellite appears to be to the device. + @ivar signalToNoiseRatio: The signal to noise ratio of the signal coming + from this satellite. + """ + def __init__(self, + identifier, + azimuth=None, + elevation=None, + signalToNoiseRatio=None): + """ + Initializes a satellite object. + + @param identifier: The PRN (unique identifier) of this satellite. + @type identifier: C{int} + @param azimuth: The azimuth of the satellite (see instance variable + documentation). + @type azimuth: C{float} + @param elevation: The elevation of the satellite (see instance variable + documentation). + @type elevation: C{float} + @param signalToNoiseRatio: The signal to noise ratio of the connection + to this satellite (see instance variable documentation). + @type signalToNoiseRatio: C{float} + """ + PositioningBeacon.__init__(self, int(identifier)) + + self.azimuth = azimuth + self.elevation = elevation + self.signalToNoiseRatio = signalToNoiseRatio + + + def __repr__(self): + """ + Returns a string representation of this Satellite. + + @return: The string representation. + @rtype: C{str} + """ + template = ("") + + return template.format(s=self) + + + +__all__ = [ + 'Altitude', + 'Angle', + 'Angles', + 'BasePositioningReceiver', + 'BeaconInformation', + 'Climb', + 'Coordinate', + 'Directions', + 'Heading', + 'InvalidChecksum', + 'InvalidSentence', + 'METERS_PER_FOOT', + 'MPS_PER_KNOT', + 'MPS_PER_KPH', + 'PositionError', + 'PositioningBeacon', + 'Satellite', + 'Speed' +] diff --git a/contrib/python/Twisted/py2/twisted/positioning/ipositioning.py b/contrib/python/Twisted/py2/twisted/positioning/ipositioning.py new file mode 100644 index 00000000000..28253248223 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/positioning/ipositioning.py @@ -0,0 +1,122 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +""" +Positioning interfaces. + +@since: 14.0 +""" + +from __future__ import absolute_import, division + +from zope.interface import Attribute, Interface + + +class IPositioningReceiver(Interface): + """ + An interface for positioning providers. + """ + def positionReceived(latitude, longitude): + """ + Method called when a position is received. + + @param latitude: The latitude of the received position. + @type latitude: L{twisted.positioning.base.Coordinate} + @param longitude: The longitude of the received position. + @type longitude: L{twisted.positioning.base.Coordinate} + """ + + + def positionErrorReceived(positionError): + """ + Method called when position error is received. + + @param positioningError: The position error. + @type positioningError: L{twisted.positioning.base.PositionError} + """ + + def timeReceived(time): + """ + Method called when time and date information arrives. + + @param time: The date and time (expressed in UTC unless otherwise + specified). + @type time: L{datetime.datetime} + """ + + + def headingReceived(heading): + """ + Method called when a true heading is received. + + @param heading: The heading. + @type heading: L{twisted.positioning.base.Heading} + """ + + + def altitudeReceived(altitude): + """ + Method called when an altitude is received. + + @param altitude: The altitude. + @type altitude: L{twisted.positioning.base.Altitude} + """ + + + def speedReceived(speed): + """ + Method called when the speed is received. + + @param speed: The speed of a mobile object. + @type speed: L{twisted.positioning.base.Speed} + """ + + + def climbReceived(climb): + """ + Method called when the climb is received. + + @param climb: The climb of the mobile object. + @type climb: L{twisted.positioning.base.Climb} + """ + + def beaconInformationReceived(beaconInformation): + """ + Method called when positioning beacon information is received. + + @param beaconInformation: The beacon information. + @type beaconInformation: L{twisted.positioning.base.BeaconInformation} + """ + + + +class IPositioningBeacon(Interface): + """ + A positioning beacon. + """ + identifier = Attribute( + """ + A unique identifier for this beacon. The type is dependent on the + implementation, but must be immutable. + """) + + + +class INMEAReceiver(Interface): + """ + An object that can receive NMEA data. + """ + def sentenceReceived(sentence): + """ + Method called when a sentence is received. + + @param sentence: The received NMEA sentence. + @type L{twisted.positioning.nmea.NMEASentence} + """ + + + +__all__ = [ + "IPositioningReceiver", + "IPositioningBeacon", + "INMEAReceiver" +] diff --git a/contrib/python/Twisted/py2/twisted/positioning/nmea.py b/contrib/python/Twisted/py2/twisted/positioning/nmea.py new file mode 100644 index 00000000000..8d1c510317e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/positioning/nmea.py @@ -0,0 +1,984 @@ +# -*- test-case-name: twisted.positioning.test.test_nmea -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +""" +Classes for working with NMEA 0183 sentence producing devices. +This standard is generally just called "NMEA", which is actually the +name of the body that produces the standard, not the standard itself.. + +For more information, read the blog post on NMEA by ESR (the gpsd +maintainer) at U{http://esr.ibiblio.org/?p=801}. Unfortunately, +official specifications on NMEA 0183 are only available at a cost. + +More information can be found on the Wikipedia page: +U{https://en.wikipedia.org/wiki/NMEA_0183}. + +The official standard may be obtained through the NMEA's website: +U{http://www.nmea.org/content/nmea_standards/nmea_0183_v_410.asp}. + +@since: 14.0 +""" + +from __future__ import absolute_import, division + +import operator +import datetime + +from zope.interface import implementer +from constantly import Values, ValueConstant + +from twisted.positioning import base, ipositioning, _sentence +from twisted.positioning.base import Angles +from twisted.protocols.basic import LineReceiver +from twisted.python.compat import reduce, izip, nativeString, iterbytes + + +class GPGGAFixQualities(Values): + """ + The possible fix quality indications for GPGGA sentences. + + @cvar INVALID_FIX: The fix is invalid. + @cvar GPS_FIX: There is a fix, acquired using GPS. + @cvar DGPS_FIX: There is a fix, acquired using differential GPS (DGPS). + @cvar PPS_FIX: There is a fix, acquired using the precise positioning + service (PPS). + @cvar RTK_FIX: There is a fix, acquired using fixed real-time + kinematics. This means that there was a sufficient number of shared + satellites with the base station, usually yielding a resolution in + the centimeter range. This was added in NMEA 0183 version 3.0. This + is also called Carrier-Phase Enhancement or CPGPS, particularly when + used in combination with GPS. + @cvar FLOAT_RTK_FIX: There is a fix, acquired using floating real-time + kinematics. The same comments apply as for a fixed real-time + kinematics fix, except that there were insufficient shared satellites + to acquire it, so instead you got a slightly less good floating fix. + Typical resolution in the decimeter range. + @cvar DEAD_RECKONING: There is currently no more fix, but this data was + computed using a previous fix and some information about motion + (either from that fix or from other sources) using simple dead + reckoning. Not particularly reliable, but better-than-nonsense data. + @cvar MANUAL: There is no real fix from this device, but the location has + been manually entered, presumably with data obtained from some other + positioning method. + @cvar SIMULATED: There is no real fix, but instead it is being simulated. + """ + INVALID_FIX = "0" + GPS_FIX = "1" + DGPS_FIX = "2" + PPS_FIX = "3" + RTK_FIX = "4" + FLOAT_RTK_FIX = "5" + DEAD_RECKONING = "6" + MANUAL = "7" + SIMULATED = "8" + + + +class GPGLLGPRMCFixQualities(Values): + """ + The possible fix quality indications in GPGLL and GPRMC sentences. + + Unfortunately, these sentences only indicate whether data is good or void. + They provide no other information, such as what went wrong if the data is + void, or how good the data is if the data is not void. + + @cvar ACTIVE: The data is okay. + @cvar VOID: The data is void, and should not be used. + """ + ACTIVE = ValueConstant("A") + VOID = ValueConstant("V") + + + +class GPGSAFixTypes(Values): + """ + The possible fix types of a GPGSA sentence. + + @cvar GSA_NO_FIX: The sentence reports no fix at all. + @cvar GSA_2D_FIX: The sentence reports a 2D fix: position but no altitude. + @cvar GSA_3D_FIX: The sentence reports a 3D fix: position with altitude. + """ + GSA_NO_FIX = ValueConstant("1") + GSA_2D_FIX = ValueConstant("2") + GSA_3D_FIX = ValueConstant("3") + + + +def _split(sentence): + """ + Returns the split version of an NMEA sentence, minus header + and checksum. + + >>> _split(b"$GPGGA,spam,eggs*00") + [b'GPGGA', b'spam', b'eggs'] + + @param sentence: The NMEA sentence to split. + @type sentence: C{bytes} + """ + if sentence[-3:-2] == b"*": # Sentence with checksum + return sentence[1:-3].split(b',') + elif sentence[-1:] == b"*": # Sentence without checksum + return sentence[1:-1].split(b',') + else: + raise base.InvalidSentence("malformed sentence %s" % (sentence,)) + + + +def _validateChecksum(sentence): + """ + Validates the checksum of an NMEA sentence. + + @param sentence: The NMEA sentence to check the checksum of. + @type sentence: C{bytes} + + @raise ValueError: If the sentence has an invalid checksum. + + Simply returns on sentences that either don't have a checksum, + or have a valid checksum. + """ + if sentence[-3:-2] == b'*': # Sentence has a checksum + reference, source = int(sentence[-2:], 16), sentence[1:-3] + computed = reduce(operator.xor, [ord(x) for x in iterbytes(source)]) + if computed != reference: + raise base.InvalidChecksum("%02x != %02x" % (computed, reference)) + + + +class NMEAProtocol(LineReceiver, _sentence._PositioningSentenceProducerMixin): + """ + A protocol that parses and verifies the checksum of an NMEA sentence (in + string form, not L{NMEASentence}), and delegates to a receiver. + + It receives lines and verifies these lines are NMEA sentences. If + they are, verifies their checksum and unpacks them into their + components. It then wraps them in L{NMEASentence} objects and + calls the appropriate receiver method with them. + + @cvar _SENTENCE_CONTENTS: Has the field names in an NMEA sentence for each + sentence type (in order, obviously). + @type _SENTENCE_CONTENTS: C{dict} of bytestrings to C{list}s of C{str} + @param _receiver: A receiver for NMEAProtocol sentence objects. + @type _receiver: L{INMEAReceiver} + @param _sentenceCallback: A function that will be called with a new + L{NMEASentence} when it is created. Useful for massaging data from + particularly misbehaving NMEA receivers. + @type _sentenceCallback: unary callable + """ + def __init__(self, receiver, sentenceCallback=None): + """ + Initializes an NMEAProtocol. + + @param receiver: A receiver for NMEAProtocol sentence objects. + @type receiver: L{INMEAReceiver} + @param sentenceCallback: A function that will be called with a new + L{NMEASentence} when it is created. Useful for massaging data from + particularly misbehaving NMEA receivers. + @type sentenceCallback: unary callable + """ + self._receiver = receiver + self._sentenceCallback = sentenceCallback + + + def lineReceived(self, rawSentence): + """ + Parses the data from the sentence and validates the checksum. + + @param rawSentence: The NMEA positioning sentence. + @type rawSentence: C{bytes} + """ + sentence = rawSentence.strip() + + _validateChecksum(sentence) + splitSentence = _split(sentence) + + sentenceType = nativeString(splitSentence[0]) + contents = [nativeString(x) for x in splitSentence[1:]] + + try: + keys = self._SENTENCE_CONTENTS[sentenceType] + except KeyError: + raise ValueError("unknown sentence type %s" % sentenceType) + + sentenceData = {"type": sentenceType} + for key, value in izip(keys, contents): + if key is not None and value != "": + sentenceData[key] = value + + sentence = NMEASentence(sentenceData) + + if self._sentenceCallback is not None: + self._sentenceCallback(sentence) + + self._receiver.sentenceReceived(sentence) + + + _SENTENCE_CONTENTS = { + 'GPGGA': [ + 'timestamp', + + 'latitudeFloat', + 'latitudeHemisphere', + 'longitudeFloat', + 'longitudeHemisphere', + + 'fixQuality', + 'numberOfSatellitesSeen', + 'horizontalDilutionOfPrecision', + + 'altitude', + 'altitudeUnits', + 'heightOfGeoidAboveWGS84', + 'heightOfGeoidAboveWGS84Units', + + # The next parts are DGPS information, currently unused. + None, # Time since last DGPS update + None, # DGPS reference source id + ], + + 'GPRMC': [ + 'timestamp', + + 'dataMode', + + 'latitudeFloat', + 'latitudeHemisphere', + 'longitudeFloat', + 'longitudeHemisphere', + + 'speedInKnots', + + 'trueHeading', + + 'datestamp', + + 'magneticVariation', + 'magneticVariationDirection', + ], + + 'GPGSV': [ + 'numberOfGSVSentences', + 'GSVSentenceIndex', + + 'numberOfSatellitesSeen', + + 'satellitePRN_0', + 'elevation_0', + 'azimuth_0', + 'signalToNoiseRatio_0', + + 'satellitePRN_1', + 'elevation_1', + 'azimuth_1', + 'signalToNoiseRatio_1', + + 'satellitePRN_2', + 'elevation_2', + 'azimuth_2', + 'signalToNoiseRatio_2', + + 'satellitePRN_3', + 'elevation_3', + 'azimuth_3', + 'signalToNoiseRatio_3', + ], + + 'GPGLL': [ + 'latitudeFloat', + 'latitudeHemisphere', + 'longitudeFloat', + 'longitudeHemisphere', + 'timestamp', + 'dataMode', + ], + + 'GPHDT': [ + 'trueHeading', + ], + + 'GPTRF': [ + 'datestamp', + 'timestamp', + + 'latitudeFloat', + 'latitudeHemisphere', + 'longitudeFloat', + 'longitudeHemisphere', + + 'elevation', + 'numberOfIterations', # Unused + 'numberOfDopplerIntervals', # Unused + 'updateDistanceInNauticalMiles', # Unused + 'satellitePRN', + ], + + 'GPGSA': [ + 'dataMode', + 'fixType', + + 'usedSatellitePRN_0', + 'usedSatellitePRN_1', + 'usedSatellitePRN_2', + 'usedSatellitePRN_3', + 'usedSatellitePRN_4', + 'usedSatellitePRN_5', + 'usedSatellitePRN_6', + 'usedSatellitePRN_7', + 'usedSatellitePRN_8', + 'usedSatellitePRN_9', + 'usedSatellitePRN_10', + 'usedSatellitePRN_11', + + 'positionDilutionOfPrecision', + 'horizontalDilutionOfPrecision', + 'verticalDilutionOfPrecision', + ] + } + + + +class NMEASentence(_sentence._BaseSentence): + """ + An object representing an NMEA sentence. + + The attributes of this objects are raw NMEA protocol data, which + are all ASCII bytestrings. + + This object contains all the raw NMEA protocol data in a single + sentence. Not all of these necessarily have to be present in the + sentence. Missing attributes are L{None} when accessed. + + @ivar type: The sentence type (C{"GPGGA"}, C{"GPGSV"}...). + @ivar numberOfGSVSentences: The total number of GSV sentences in a + sequence. + @ivar GSVSentenceIndex: The index of this GSV sentence in the GSV + sequence. + @ivar timestamp: A timestamp. (C{"123456"} -> 12:34:56Z) + @ivar datestamp: A datestamp. (C{"230394"} -> 23 Mar 1994) + @ivar latitudeFloat: Latitude value. (for example: C{"1234.567"} -> + 12 degrees, 34.567 minutes). + @ivar latitudeHemisphere: Latitudinal hemisphere (C{"N"} or C{"S"}). + @ivar longitudeFloat: Longitude value. See C{latitudeFloat} for an + example. + @ivar longitudeHemisphere: Longitudinal hemisphere (C{"E"} or C{"W"}). + @ivar altitude: The altitude above mean sea level. + @ivar altitudeUnits: Units in which altitude is expressed. (Always + C{"M"} for meters.) + @ivar heightOfGeoidAboveWGS84: The local height of the geoid above + the WGS84 ellipsoid model. + @ivar heightOfGeoidAboveWGS84Units: The units in which the height + above the geoid is expressed. (Always C{"M"} for meters.) + @ivar trueHeading: The true heading. + @ivar magneticVariation: The magnetic variation. + @ivar magneticVariationDirection: The direction of the magnetic + variation. One of C{"E"} or C{"W"}. + @ivar speedInKnots: The ground speed, expressed in knots. + @ivar fixQuality: The quality of the fix. + @type fixQuality: One of L{GPGGAFixQualities}. + @ivar dataMode: Signals if the data is usable or not. + @type dataMode: One of L{GPGLLGPRMCFixQualities}. + @ivar numberOfSatellitesSeen: The number of satellites seen by the + receiver. + @ivar numberOfSatellitesUsed: The number of satellites used in + computing the fix. + @ivar horizontalDilutionOfPrecision: The dilution of the precision of the + position on a plane tangential to the geoid. (HDOP) + @ivar verticalDilutionOfPrecision: As C{horizontalDilutionOfPrecision}, + but for a position on a plane perpendicular to the geoid. (VDOP) + @ivar positionDilutionOfPrecision: Euclidean norm of HDOP and VDOP. + @ivar satellitePRN: The unique identifcation number of a particular + satellite. Optionally suffixed with C{_N} if multiple satellites are + referenced in a sentence, where C{N in range(4)}. + @ivar elevation: The elevation of a satellite in decimal degrees. + Optionally suffixed with C{_N}, as with C{satellitePRN}. + @ivar azimuth: The azimuth of a satellite in decimal degrees. + Optionally suffixed with C{_N}, as with C{satellitePRN}. + @ivar signalToNoiseRatio: The SNR of a satellite signal, in decibels. + Optionally suffixed with C{_N}, as with C{satellitePRN}. + @ivar usedSatellitePRN_N: Where C{int(N) in range(12)}. The PRN + of a satellite used in computing the fix. + """ + ALLOWED_ATTRIBUTES = NMEAProtocol.getSentenceAttributes() + + def _isFirstGSVSentence(self): + """ + Tests if this current GSV sentence is the first one in a sequence. + + @return: C{True} if this is the first GSV sentence. + @rtype: C{bool} + """ + return self.GSVSentenceIndex == "1" + + + def _isLastGSVSentence(self): + """ + Tests if this current GSV sentence is the final one in a sequence. + + @return: C{True} if this is the last GSV sentence. + @rtype: C{bool} + """ + return self.GSVSentenceIndex == self.numberOfGSVSentences + + + +@implementer(ipositioning.INMEAReceiver) +class NMEAAdapter(object): + """ + An adapter from NMEAProtocol receivers to positioning receivers. + + @cvar _STATEFUL_UPDATE: Information on how to update partial information + in the sentence data or internal adapter state. For more information, + see C{_statefulUpdate}'s docstring. + @type _STATEFUL_UPDATE: See C{_statefulUpdate}'s docstring + @cvar _ACCEPTABLE_UNITS: A set of NMEA notations of units that are + already acceptable (metric), and therefore don't need to be converted. + @type _ACCEPTABLE_UNITS: C{frozenset} of bytestrings + @cvar _UNIT_CONVERTERS: Mapping of NMEA notations of units that are not + acceptable (not metric) to converters that take a quantity in that + unit and produce a metric quantity. + @type _UNIT_CONVERTERS: C{dict} of bytestrings to unary callables + @cvar _SPECIFIC_SENTENCE_FIXES: A mapping of sentece types to specific + fixes that are required to extract useful information from data from + those sentences. + @type _SPECIFIC_SENTENCE_FIXES: C{dict} of sentence types to callables + that take self and modify it in-place + @cvar _FIXERS: Set of unary callables that take an NMEAAdapter instance + and extract useful data from the sentence data, usually modifying the + adapter's sentence data in-place. + @type _FIXERS: C{dict} of native strings to unary callables + @ivar yearThreshold: The earliest possible year that data will be + interpreted as. For example, if this value is C{1990}, an NMEA + 0183 two-digit year of "96" will be interpreted as 1996, and + a two-digit year of "13" will be interpreted as 2013. + @type yearThreshold: L{int} + @ivar _state: The current internal state of the receiver. + @type _state: C{dict} + @ivar _sentenceData: The data present in the sentence currently being + processed. Starts empty, is filled as the sentence is parsed. + @type _sentenceData: C{dict} + @ivar _receiver: The positioning receiver that will receive parsed data. + @type _receiver: L{ipositioning.IPositioningReceiver} + """ + def __init__(self, receiver): + """ + Initializes a new NMEA adapter. + + @param receiver: The receiver for positioning sentences. + @type receiver: L{ipositioning.IPositioningReceiver} + """ + self._state = {} + self._sentenceData = {} + self._receiver = receiver + + + def _fixTimestamp(self): + """ + Turns the NMEAProtocol timestamp notation into a datetime.time object. + The time in this object is expressed as Zulu time. + """ + timestamp = self.currentSentence.timestamp.split('.')[0] + timeObject = datetime.datetime.strptime(timestamp, '%H%M%S').time() + self._sentenceData['_time'] = timeObject + + + yearThreshold = 1980 + + + def _fixDatestamp(self): + """ + Turns an NMEA datestamp format into a C{datetime.date} object. + + @raise ValueError: When the day or month value was invalid, e.g. 32nd + day, or 13th month, or 0th day or month. + """ + date = self.currentSentence.datestamp + day, month, year = map(int, [date[0:2], date[2:4], date[4:6]]) + + year += self.yearThreshold - (self.yearThreshold % 100) + if year < self.yearThreshold: + year += 100 + + self._sentenceData['_date'] = datetime.date(year, month, day) + + + def _fixCoordinateFloat(self, coordinateType): + """ + Turns the NMEAProtocol coordinate format into Python float. + + @param coordinateType: The coordinate type. + @type coordinateType: One of L{Angles.LATITUDE} or L{Angles.LONGITUDE}. + """ + if coordinateType is Angles.LATITUDE: + coordinateName = "latitude" + else: # coordinateType is Angles.LONGITUDE + coordinateName = "longitude" + nmeaCoordinate = getattr(self.currentSentence, coordinateName + "Float") + + left, right = nmeaCoordinate.split('.') + + degrees, minutes = int(left[:-2]), float("%s.%s" % (left[-2:], right)) + angle = degrees + minutes/60 + coordinate = base.Coordinate(angle, coordinateType) + self._sentenceData[coordinateName] = coordinate + + + def _fixHemisphereSign(self, coordinateType, sentenceDataKey=None): + """ + Fixes the sign for a hemisphere. + + This method must be called after the magnitude for the thing it + determines the sign of has been set. This is done by the following + functions: + + - C{self.FIXERS['magneticVariation']} + - C{self.FIXERS['latitudeFloat']} + - C{self.FIXERS['longitudeFloat']} + + @param coordinateType: Coordinate type. One of L{Angles.LATITUDE}, + L{Angles.LONGITUDE} or L{Angles.VARIATION}. + @param sentenceDataKey: The key name of the hemisphere sign being + fixed in the sentence data. If unspecified, C{coordinateType} is + used. + @type sentenceDataKey: C{str} (unless L{None}) + """ + sentenceDataKey = sentenceDataKey or coordinateType + sign = self._getHemisphereSign(coordinateType) + self._sentenceData[sentenceDataKey].setSign(sign) + + + def _getHemisphereSign(self, coordinateType): + """ + Returns the hemisphere sign for a given coordinate type. + + @param coordinateType: The coordinate type to find the hemisphere for. + @type coordinateType: L{Angles.LATITUDE}, L{Angles.LONGITUDE} or + L{Angles.VARIATION}. + @return: The sign of that hemisphere (-1 or 1). + @rtype: C{int} + """ + if coordinateType is Angles.LATITUDE: + hemisphereKey = "latitudeHemisphere" + elif coordinateType is Angles.LONGITUDE: + hemisphereKey = "longitudeHemisphere" + elif coordinateType is Angles.VARIATION: + hemisphereKey = 'magneticVariationDirection' + else: + raise ValueError("unknown coordinate type %s" % (coordinateType,)) + + hemisphere = getattr(self.currentSentence, hemisphereKey).upper() + + if hemisphere in "NE": + return 1 + elif hemisphere in "SW": + return -1 + else: + raise ValueError("bad hemisphere/direction: %s" % (hemisphere,)) + + + def _convert(self, key, converter): + """ + A simple conversion fix. + + @param key: The attribute name of the value to fix. + @type key: native string (Python identifier) + + @param converter: The function that converts the value. + @type converter: unary callable + """ + currentValue = getattr(self.currentSentence, key) + self._sentenceData[key] = converter(currentValue) + + + _STATEFUL_UPDATE = { + # sentenceKey: (stateKey, factory, attributeName, converter), + 'trueHeading': ('heading', base.Heading, '_angle', float), + 'magneticVariation': + ('heading', base.Heading, 'variation', + lambda angle: base.Angle(float(angle), Angles.VARIATION)), + + 'horizontalDilutionOfPrecision': + ('positionError', base.PositionError, 'hdop', float), + 'verticalDilutionOfPrecision': + ('positionError', base.PositionError, 'vdop', float), + 'positionDilutionOfPrecision': + ('positionError', base.PositionError, 'pdop', float), + + } + + + def _statefulUpdate(self, sentenceKey): + """ + Does a stateful update of a particular positioning attribute. + Specifically, this will mutate an object in the current sentence data. + + Using the C{sentenceKey}, this will get a tuple containing, in order, + the key name in the current state and sentence data, a factory for + new values, the attribute to update, and a converter from sentence + data (in NMEA notation) to something useful. + + If the sentence data doesn't have this data yet, it is grabbed from + the state. If that doesn't have anything useful yet either, the + factory is called to produce a new, empty object. Either way, the + object ends up in the sentence data. + + @param sentenceKey: The name of the key in the sentence attributes, + C{NMEAAdapter._STATEFUL_UPDATE} dictionary and the adapter state. + @type sentenceKey: C{str} + """ + key, factory, attr, converter = self._STATEFUL_UPDATE[sentenceKey] + + if key not in self._sentenceData: + try: + self._sentenceData[key] = self._state[key] + except KeyError: # state does not have this partial data yet + self._sentenceData[key] = factory() + + newValue = converter(getattr(self.currentSentence, sentenceKey)) + setattr(self._sentenceData[key], attr, newValue) + + + _ACCEPTABLE_UNITS = frozenset(['M']) + _UNIT_CONVERTERS = { + 'N': lambda inKnots: base.Speed(float(inKnots) * base.MPS_PER_KNOT), + 'K': lambda inKPH: base.Speed(float(inKPH) * base.MPS_PER_KPH), + } + + + def _fixUnits(self, unitKey=None, valueKey=None, sourceKey=None, + unit=None): + """ + Fixes the units of a certain value. If the units are already + acceptable (metric), does nothing. + + None of the keys are allowed to be the empty string. + + @param unit: The unit that is being converted I{from}. If unspecified + or L{None}, asks the current sentence for the C{unitKey}. If that + also fails, raises C{AttributeError}. + @type unit: C{str} + @param unitKey: The name of the key/attribute under which the unit can + be found in the current sentence. If the C{unit} parameter is set, + this parameter is not used. + @type unitKey: C{str} + @param sourceKey: The name of the key/attribute that contains the + current value to be converted (expressed in units as defined + according to the C{unit} parameter). If unset, will use the + same key as the value key. + @type sourceKey: C{str} + @param valueKey: The key name in which the data will be stored in the + C{_sentenceData} instance attribute. If unset, attempts to remove + "Units" from the end of the C{unitKey} parameter. If that fails, + raises C{ValueError}. + @type valueKey: C{str} + """ + if unit is None: + unit = getattr(self.currentSentence, unitKey) + if valueKey is None: + if unitKey is not None and unitKey.endswith("Units"): + valueKey = unitKey[:-5] + else: + raise ValueError("valueKey unspecified and couldn't be guessed") + if sourceKey is None: + sourceKey = valueKey + + if unit not in self._ACCEPTABLE_UNITS: + converter = self._UNIT_CONVERTERS[unit] + currentValue = getattr(self.currentSentence, sourceKey) + self._sentenceData[valueKey] = converter(currentValue) + + + def _fixGSV(self): + """ + Parses partial visible satellite information from a GSV sentence. + """ + # To anyone who knows NMEA, this method's name should raise a chuckle's + # worth of schadenfreude. 'Fix' GSV? Hah! Ludicrous. + beaconInformation = base.BeaconInformation() + self._sentenceData['_partialBeaconInformation'] = beaconInformation + + keys = "satellitePRN", "azimuth", "elevation", "signalToNoiseRatio" + for index in range(4): + prn, azimuth, elevation, snr = [getattr(self.currentSentence, attr) + for attr in ("%s_%i" % (key, index) for key in keys)] + + if prn is None or snr is None: + # The peephole optimizer optimizes the jump away, meaning that + # coverage.py thinks it isn't covered. It is. Replace it with + # break, and watch the test case fail. + # ML thread about this issue: http://goo.gl/1KNUi + # Related CPython bug: http://bugs.python.org/issue2506 + continue + + satellite = base.Satellite(prn, azimuth, elevation, snr) + beaconInformation.seenBeacons.add(satellite) + + + def _fixGSA(self): + """ + Extracts the information regarding which satellites were used in + obtaining the GPS fix from a GSA sentence. + + Precondition: A GSA sentence was fired. Postcondition: The current + sentence data (C{self._sentenceData} will contain a set of the + currently used PRNs (under the key C{_usedPRNs}. + """ + self._sentenceData['_usedPRNs'] = set() + for key in ("usedSatellitePRN_%d" % (x,) for x in range(12)): + prn = getattr(self.currentSentence, key, None) + if prn is not None: + self._sentenceData['_usedPRNs'].add(int(prn)) + + + _SPECIFIC_SENTENCE_FIXES = { + 'GPGSV': _fixGSV, + 'GPGSA': _fixGSA, + } + + + def _sentenceSpecificFix(self): + """ + Executes a fix for a specific type of sentence. + """ + fixer = self._SPECIFIC_SENTENCE_FIXES.get(self.currentSentence.type) + if fixer is not None: + fixer(self) + + + _FIXERS = { + 'type': + lambda self: self._sentenceSpecificFix(), + + 'timestamp': + lambda self: self._fixTimestamp(), + 'datestamp': + lambda self: self._fixDatestamp(), + + 'latitudeFloat': + lambda self: self._fixCoordinateFloat(Angles.LATITUDE), + 'latitudeHemisphere': + lambda self: self._fixHemisphereSign(Angles.LATITUDE, 'latitude'), + 'longitudeFloat': + lambda self: self._fixCoordinateFloat(Angles.LONGITUDE), + 'longitudeHemisphere': + lambda self: self._fixHemisphereSign(Angles.LONGITUDE, 'longitude'), + + 'altitude': + lambda self: self._convert('altitude', + converter=lambda strRepr: base.Altitude(float(strRepr))), + 'altitudeUnits': + lambda self: self._fixUnits(unitKey='altitudeUnits'), + + 'heightOfGeoidAboveWGS84': + lambda self: self._convert('heightOfGeoidAboveWGS84', + converter=lambda strRepr: base.Altitude(float(strRepr))), + 'heightOfGeoidAboveWGS84Units': + lambda self: self._fixUnits( + unitKey='heightOfGeoidAboveWGS84Units'), + + 'trueHeading': + lambda self: self._statefulUpdate('trueHeading'), + 'magneticVariation': + lambda self: self._statefulUpdate('magneticVariation'), + + 'magneticVariationDirection': + lambda self: self._fixHemisphereSign(Angles.VARIATION, + 'heading'), + + 'speedInKnots': + lambda self: self._fixUnits(valueKey='speed', + sourceKey='speedInKnots', + unit='N'), + + 'positionDilutionOfPrecision': + lambda self: self._statefulUpdate('positionDilutionOfPrecision'), + 'horizontalDilutionOfPrecision': + lambda self: self._statefulUpdate('horizontalDilutionOfPrecision'), + 'verticalDilutionOfPrecision': + lambda self: self._statefulUpdate('verticalDilutionOfPrecision'), + } + + + def clear(self): + """ + Resets this adapter. + + This will empty the adapter state and the current sentence data. + """ + self._state = {} + self._sentenceData = {} + + + def sentenceReceived(self, sentence): + """ + Called when a sentence is received. + + Will clean the received NMEAProtocol sentence up, and then update the + adapter's state, followed by firing the callbacks. + + If the received sentence was invalid, the state will be cleared. + + @param sentence: The sentence that is received. + @type sentence: L{NMEASentence} + """ + self.currentSentence = sentence + self._sentenceData = {} + + try: + self._validateCurrentSentence() + self._cleanCurrentSentence() + except base.InvalidSentence: + self.clear() + + self._updateState() + self._fireSentenceCallbacks() + + + def _validateCurrentSentence(self): + """ + Tests if a sentence contains a valid fix. + """ + if (self.currentSentence.fixQuality is GPGGAFixQualities.INVALID_FIX + or self.currentSentence.dataMode is GPGLLGPRMCFixQualities.VOID + or self.currentSentence.fixType is GPGSAFixTypes.GSA_NO_FIX): + raise base.InvalidSentence("bad sentence") + + + def _cleanCurrentSentence(self): + """ + Cleans the current sentence. + """ + for key in sorted(self.currentSentence.presentAttributes): + fixer = self._FIXERS.get(key, None) + + if fixer is not None: + fixer(self) + + + def _updateState(self): + """ + Updates the current state with the new information from the sentence. + """ + self._updateBeaconInformation() + self._combineDateAndTime() + self._state.update(self._sentenceData) + + + def _updateBeaconInformation(self): + """ + Updates existing beacon information state with new data. + """ + new = self._sentenceData.get('_partialBeaconInformation') + if new is None: + return + + self._updateUsedBeacons(new) + self._mergeBeaconInformation(new) + + if self.currentSentence._isLastGSVSentence(): + if not self.currentSentence._isFirstGSVSentence(): + # not a 1-sentence sequence, get rid of partial information + del self._state['_partialBeaconInformation'] + bi = self._sentenceData.pop('_partialBeaconInformation') + self._sentenceData['beaconInformation'] = bi + + + def _updateUsedBeacons(self, beaconInformation): + """ + Searches the adapter state and sentence data for information about + which beacons where used, then adds it to the provided beacon + information object. + + If no new beacon usage information is available, does nothing. + + @param beaconInformation: The beacon information object that beacon + usage information will be added to (if necessary). + @type beaconInformation: L{twisted.positioning.base.BeaconInformation} + """ + for source in [self._state, self._sentenceData]: + usedPRNs = source.get("_usedPRNs") + if usedPRNs is not None: + break + else: # No used PRN info to update + return + + for beacon in beaconInformation.seenBeacons: + if beacon.identifier in usedPRNs: + beaconInformation.usedBeacons.add(beacon) + + + def _mergeBeaconInformation(self, newBeaconInformation): + """ + Merges beacon information in the adapter state (if it exists) into + the provided beacon information. Specifically, this merges used and + seen beacons. + + If the adapter state has no beacon information, does nothing. + + @param beaconInformation: The beacon information object that beacon + information will be merged into (if necessary). + @type beaconInformation: L{twisted.positioning.base.BeaconInformation} + """ + old = self._state.get('_partialBeaconInformation') + if old is None: + return + + for attr in ["seenBeacons", "usedBeacons"]: + getattr(newBeaconInformation, attr).update(getattr(old, attr)) + + + def _combineDateAndTime(self): + """ + Combines a C{datetime.date} object and a C{datetime.time} object, + collected from one or more NMEA sentences, into a single + C{datetime.datetime} object suitable for sending to the + L{IPositioningReceiver}. + """ + if not any(k in self._sentenceData for k in ["_date", "_time"]): + # If the sentence has neither date nor time, there's + # nothing new to combine here. + return + + date, time = [self._sentenceData.get(key) or self._state.get(key) + for key in ('_date', '_time')] + + if date is None or time is None: + return + + dt = datetime.datetime.combine(date, time) + self._sentenceData['time'] = dt + + + def _fireSentenceCallbacks(self): + """ + Fires sentence callbacks for the current sentence. + + A callback will only fire if all of the keys it requires are present + in the current state and at least one such field was altered in the + current sentence. + + The callbacks will only be fired with data from L{_state}. + """ + iface = ipositioning.IPositioningReceiver + for name, method in iface.namesAndDescriptions(): + callback = getattr(self._receiver, name) + + kwargs = {} + atLeastOnePresentInSentence = False + + try: + for field in method.positional: + if field in self._sentenceData: + atLeastOnePresentInSentence = True + kwargs[field] = self._state[field] + except KeyError: + continue + + if atLeastOnePresentInSentence: + callback(**kwargs) + + + +__all__ = [ + "NMEAProtocol", + "NMEASentence", + "NMEAAdapter" +] diff --git a/contrib/python/Twisted/py2/twisted/protocols/__init__.py b/contrib/python/Twisted/py2/twisted/protocols/__init__.py new file mode 100644 index 00000000000..b04f3ec7980 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Protocols: A collection of internet protocol implementations. +""" + +from incremental import Version +from twisted.python.deprecate import deprecatedModuleAttribute + + +deprecatedModuleAttribute( + Version('Twisted', 17, 9, 0), + "There is no replacement for this module.", + "twisted.protocols", "dict") diff --git a/contrib/python/Twisted/py2/twisted/protocols/amp.py b/contrib/python/Twisted/py2/twisted/protocols/amp.py new file mode 100644 index 00000000000..322d633b686 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/amp.py @@ -0,0 +1,2897 @@ +# -*- test-case-name: twisted.test.test_amp -*- +# Copyright (c) 2005 Divmod, Inc. +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module implements AMP, the Asynchronous Messaging Protocol. + +AMP is a protocol for sending multiple asynchronous request/response pairs over +the same connection. Requests and responses are both collections of key/value +pairs. + +AMP is a very simple protocol which is not an application. This module is a +"protocol construction kit" of sorts; it attempts to be the simplest wire-level +implementation of Deferreds. AMP provides the following base-level features: + + - Asynchronous request/response handling (hence the name) + + - Requests and responses are both key/value pairs + + - Binary transfer of all data: all data is length-prefixed. Your + application will never need to worry about quoting. + + - Command dispatching (like HTTP Verbs): the protocol is extensible, and + multiple AMP sub-protocols can be grouped together easily. + +The protocol implementation also provides a few additional features which are +not part of the core wire protocol, but are nevertheless very useful: + + - Tight TLS integration, with an included StartTLS command. + + - Handshaking to other protocols: because AMP has well-defined message + boundaries and maintains all incoming and outgoing requests for you, you + can start a connection over AMP and then switch to another protocol. + This makes it ideal for firewall-traversal applications where you may + have only one forwarded port but multiple applications that want to use + it. + +Using AMP with Twisted is simple. Each message is a command, with a response. +You begin by defining a command type. Commands specify their input and output +in terms of the types that they expect to see in the request and response +key-value pairs. Here's an example of a command that adds two integers, 'a' +and 'b':: + + class Sum(amp.Command): + arguments = [('a', amp.Integer()), + ('b', amp.Integer())] + response = [('total', amp.Integer())] + +Once you have specified a command, you need to make it part of a protocol, and +define a responder for it. Here's a 'JustSum' protocol that includes a +responder for our 'Sum' command:: + + class JustSum(amp.AMP): + def sum(self, a, b): + total = a + b + print 'Did a sum: %d + %d = %d' % (a, b, total) + return {'total': total} + Sum.responder(sum) + +Later, when you want to actually do a sum, the following expression will return +a L{Deferred} which will fire with the result:: + + ClientCreator(reactor, amp.AMP).connectTCP(...).addCallback( + lambda p: p.callRemote(Sum, a=13, b=81)).addCallback( + lambda result: result['total']) + +Command responders may also return Deferreds, causing the response to be +sent only once the Deferred fires:: + + class DelayedSum(amp.AMP): + def slowSum(self, a, b): + total = a + b + result = defer.Deferred() + reactor.callLater(3, result.callback, {'total': total}) + return result + Sum.responder(slowSum) + +This is transparent to the caller. + +You can also define the propagation of specific errors in AMP. For example, +for the slightly more complicated case of division, we might have to deal with +division by zero:: + + class Divide(amp.Command): + arguments = [('numerator', amp.Integer()), + ('denominator', amp.Integer())] + response = [('result', amp.Float())] + errors = {ZeroDivisionError: 'ZERO_DIVISION'} + +The 'errors' mapping here tells AMP that if a responder to Divide emits a +L{ZeroDivisionError}, then the other side should be informed that an error of +the type 'ZERO_DIVISION' has occurred. Writing a responder which takes +advantage of this is very simple - just raise your exception normally:: + + class JustDivide(amp.AMP): + def divide(self, numerator, denominator): + result = numerator / denominator + print 'Divided: %d / %d = %d' % (numerator, denominator, total) + return {'result': result} + Divide.responder(divide) + +On the client side, the errors mapping will be used to determine what the +'ZERO_DIVISION' error means, and translated into an asynchronous exception, +which can be handled normally as any L{Deferred} would be:: + + def trapZero(result): + result.trap(ZeroDivisionError) + print "Divided by zero: returning INF" + return 1e1000 + ClientCreator(reactor, amp.AMP).connectTCP(...).addCallback( + lambda p: p.callRemote(Divide, numerator=1234, + denominator=0) + ).addErrback(trapZero) + +For a complete, runnable example of both of these commands, see the files in +the Twisted repository:: + + doc/core/examples/ampserver.py + doc/core/examples/ampclient.py + +On the wire, AMP is a protocol which uses 2-byte lengths to prefix keys and +values, and empty keys to separate messages:: + + <2-byte length><2-byte length> + <2-byte length><2-byte length> + ... + <2-byte length><2-byte length> + # Empty Key == End of Message + +And so on. Because it's tedious to refer to lengths and NULs constantly, the +documentation will refer to packets as if they were newline delimited, like +so:: + + C: _command: sum + C: _ask: ef639e5c892ccb54 + C: a: 13 + C: b: 81 + + S: _answer: ef639e5c892ccb54 + S: total: 94 + +Notes: + +In general, the order of keys is arbitrary. Specific uses of AMP may impose an +ordering requirement, but unless this is specified explicitly, any ordering may +be generated and any ordering must be accepted. This applies to the +command-related keys I{_command} and I{_ask} as well as any other keys. + +Values are limited to the maximum encodable size in a 16-bit length, 65535 +bytes. + +Keys are limited to the maximum encodable size in a 8-bit length, 255 bytes. +Note that we still use 2-byte lengths to encode keys. This small redundancy +has several features: + + - If an implementation becomes confused and starts emitting corrupt data, + or gets keys confused with values, many common errors will be signalled + immediately instead of delivering obviously corrupt packets. + + - A single NUL will separate every key, and a double NUL separates + messages. This provides some redundancy when debugging traffic dumps. + + - NULs will be present at regular intervals along the protocol, providing + some padding for otherwise braindead C implementations of the protocol, + so that string functions will see the NUL and stop. + + - This makes it possible to run an AMP server on a port also used by a + plain-text protocol, and easily distinguish between non-AMP clients (like + web browsers) which issue non-NUL as the first byte, and AMP clients, + which always issue NUL as the first byte. + +@var MAX_VALUE_LENGTH: The maximum length of a message. +@type MAX_VALUE_LENGTH: L{int} + +@var ASK: Marker for an Ask packet. +@type ASK: L{bytes} + +@var ANSWER: Marker for an Answer packet. +@type ANSWER: L{bytes} + +@var COMMAND: Marker for a Command packet. +@type COMMAND: L{bytes} + +@var ERROR: Marker for an AMP box of error type. +@type ERROR: L{bytes} + +@var ERROR_CODE: Marker for an AMP box containing the code of an error. +@type ERROR_CODE: L{bytes} + +@var ERROR_DESCRIPTION: Marker for an AMP box containing the description of the + error. +@type ERROR_DESCRIPTION: L{bytes} +""" + +from __future__ import absolute_import, division + +__metaclass__ = type + +import types, warnings + +from io import BytesIO +from struct import pack +import decimal, datetime +from functools import partial +from itertools import count + +from zope.interface import Interface, implementer + +from twisted.python.reflect import accumulateClassDict +from twisted.python.failure import Failure +from twisted.python._tzhelper import ( + FixedOffsetTimeZone as _FixedOffsetTZInfo, UTC as utc +) + +from twisted.python import log, filepath + +from twisted.internet.interfaces import IFileDescriptorReceiver +from twisted.internet.main import CONNECTION_LOST +from twisted.internet.error import PeerVerifyError, ConnectionLost +from twisted.internet.error import ConnectionClosed +from twisted.internet.defer import Deferred, maybeDeferred, fail +from twisted.protocols.basic import Int16StringReceiver, StatefulStringProtocol +from twisted.python.compat import ( + iteritems, unicode, nativeString, intToBytes, _PY3, long, +) + +try: + from twisted.internet import ssl +except ImportError: + ssl = None + +if ssl and not ssl.supported: + ssl = None + +if ssl is not None: + from twisted.internet.ssl import (CertificateOptions, Certificate, DN, + KeyPair) + + + +__all__ = [ + 'AMP', + 'ANSWER', + 'ASK', + 'AmpBox', + 'AmpError', + 'AmpList', + 'Argument', + 'BadLocalReturn', + 'BinaryBoxProtocol', + 'Boolean', + 'Box', + 'BoxDispatcher', + 'COMMAND', + 'Command', + 'CommandLocator', + 'Decimal', + 'Descriptor', + 'ERROR', + 'ERROR_CODE', + 'ERROR_DESCRIPTION', + 'Float', + 'IArgumentType', + 'IBoxReceiver', + 'IBoxSender', + 'IResponderLocator', + 'IncompatibleVersions', + 'Integer', + 'InvalidSignature', + 'ListOf', + 'MAX_KEY_LENGTH', + 'MAX_VALUE_LENGTH', + 'MalformedAmpBox', + 'NoEmptyBoxes', + 'OnlyOneTLS', + 'PROTOCOL_ERRORS', + 'PYTHON_KEYWORDS', + 'Path', + 'ProtocolSwitchCommand', + 'ProtocolSwitched', + 'QuitBox', + 'RemoteAmpError', + 'SimpleStringLocator', + 'StartTLS', + 'String', + 'TooLong', + 'UNHANDLED_ERROR_CODE', + 'UNKNOWN_ERROR_CODE', + 'UnhandledCommand', + 'utc', + 'Unicode', + 'UnknownRemoteError', + 'parse', + 'parseString', +] + + + +ASK = b'_ask' +ANSWER = b'_answer' +COMMAND = b'_command' +ERROR = b'_error' +ERROR_CODE = b'_error_code' +ERROR_DESCRIPTION = b'_error_description' +UNKNOWN_ERROR_CODE = b'UNKNOWN' +UNHANDLED_ERROR_CODE = b'UNHANDLED' + +MAX_KEY_LENGTH = 0xff +MAX_VALUE_LENGTH = 0xffff + + + +class IArgumentType(Interface): + """ + An L{IArgumentType} can serialize a Python object into an AMP box and + deserialize information from an AMP box back into a Python object. + + @since: 9.0 + """ + def fromBox(name, strings, objects, proto): + """ + Given an argument name and an AMP box containing serialized values, + extract one or more Python objects and add them to the C{objects} + dictionary. + + @param name: The name associated with this argument. Most commonly + this is the key which can be used to find a serialized value in + C{strings}. + @type name: C{bytes} + + @param strings: The AMP box from which to extract one or more + values. + @type strings: C{dict} + + @param objects: The output dictionary to populate with the value for + this argument. The key used will be derived from C{name}. It may + differ; in Python 3, for example, the key will be a Unicode/native + string. See L{_wireNameToPythonIdentifier}. + @type objects: C{dict} + + @param proto: The protocol instance which received the AMP box being + interpreted. Most likely this is an instance of L{AMP}, but + this is not guaranteed. + + @return: L{None} + """ + + + def toBox(name, strings, objects, proto): + """ + Given an argument name and a dictionary containing structured Python + objects, serialize values into one or more strings and add them to + the C{strings} dictionary. + + @param name: The name associated with this argument. Most commonly + this is the key in C{strings} to associate with a C{bytes} giving + the serialized form of that object. + @type name: C{bytes} + + @param strings: The AMP box into which to insert one or more strings. + @type strings: C{dict} + + @param objects: The input dictionary from which to extract Python + objects to serialize. The key used will be derived from C{name}. + It may differ; in Python 3, for example, the key will be a + Unicode/native string. See L{_wireNameToPythonIdentifier}. + @type objects: C{dict} + + @param proto: The protocol instance which will send the AMP box once + it is fully populated. Most likely this is an instance of + L{AMP}, but this is not guaranteed. + + @return: L{None} + """ + + + +class IBoxSender(Interface): + """ + A transport which can send L{AmpBox} objects. + """ + + def sendBox(box): + """ + Send an L{AmpBox}. + + @raise ProtocolSwitched: if the underlying protocol has been + switched. + + @raise ConnectionLost: if the underlying connection has already been + lost. + """ + + def unhandledError(failure): + """ + An unhandled error occurred in response to a box. Log it + appropriately. + + @param failure: a L{Failure} describing the error that occurred. + """ + + + +class IBoxReceiver(Interface): + """ + An application object which can receive L{AmpBox} objects and dispatch them + appropriately. + """ + + def startReceivingBoxes(boxSender): + """ + The L{IBoxReceiver.ampBoxReceived} method will start being called; + boxes may be responded to by responding to the given L{IBoxSender}. + + @param boxSender: an L{IBoxSender} provider. + """ + + + def ampBoxReceived(box): + """ + A box was received from the transport; dispatch it appropriately. + """ + + + def stopReceivingBoxes(reason): + """ + No further boxes will be received on this connection. + + @type reason: L{Failure} + """ + + + +class IResponderLocator(Interface): + """ + An application object which can look up appropriate responder methods for + AMP commands. + """ + + def locateResponder(name): + """ + Locate a responder method appropriate for the named command. + + @param name: the wire-level name (commandName) of the AMP command to be + responded to. + @type name: C{bytes} + + @return: a 1-argument callable that takes an L{AmpBox} with argument + values for the given command, and returns an L{AmpBox} containing + argument values for the named command, or a L{Deferred} that fires the + same. + """ + + + +class AmpError(Exception): + """ + Base class of all Amp-related exceptions. + """ + + + +class ProtocolSwitched(Exception): + """ + Connections which have been switched to other protocols can no longer + accept traffic at the AMP level. This is raised when you try to send it. + """ + + + +class OnlyOneTLS(AmpError): + """ + This is an implementation limitation; TLS may only be started once per + connection. + """ + + + +class NoEmptyBoxes(AmpError): + """ + You can't have empty boxes on the connection. This is raised when you + receive or attempt to send one. + """ + + + +class InvalidSignature(AmpError): + """ + You didn't pass all the required arguments. + """ + + + +class TooLong(AmpError): + """ + One of the protocol's length limitations was violated. + + @ivar isKey: true if the string being encoded in a key position, false if + it was in a value position. + + @ivar isLocal: Was the string encoded locally, or received too long from + the network? (It's only physically possible to encode "too long" values on + the network for keys.) + + @ivar value: The string that was too long. + + @ivar keyName: If the string being encoded was in a value position, what + key was it being encoded for? + """ + + def __init__(self, isKey, isLocal, value, keyName=None): + AmpError.__init__(self) + self.isKey = isKey + self.isLocal = isLocal + self.value = value + self.keyName = keyName + + + def __repr__(self): + hdr = self.isKey and "key" or "value" + if not self.isKey: + hdr += ' ' + repr(self.keyName) + lcl = self.isLocal and "local" or "remote" + return "%s %s too long: %d" % (lcl, hdr, len(self.value)) + + + +class BadLocalReturn(AmpError): + """ + A bad value was returned from a local command; we were unable to coerce it. + """ + def __init__(self, message, enclosed): + AmpError.__init__(self) + self.message = message + self.enclosed = enclosed + + + def __repr__(self): + return self.message + " " + self.enclosed.getBriefTraceback() + + __str__ = __repr__ + + + +class RemoteAmpError(AmpError): + """ + This error indicates that something went wrong on the remote end of the + connection, and the error was serialized and transmitted to you. + """ + def __init__(self, errorCode, description, fatal=False, local=None): + """Create a remote error with an error code and description. + + @param errorCode: the AMP error code of this error. + @type errorCode: C{bytes} + + @param description: some text to show to the user. + @type description: C{str} + + @param fatal: a boolean, true if this error should terminate the + connection. + + @param local: a local Failure, if one exists. + """ + if local: + localwhat = ' (local)' + othertb = local.getBriefTraceback() + else: + localwhat = '' + othertb = '' + + # Backslash-escape errorCode. Python 3.5 can do this natively + # ("backslashescape") but Python 2.7 and Python 3.4 can't. + if _PY3: + errorCodeForMessage = "".join( + "\\x%2x" % (c,) if c >= 0x80 else chr(c) + for c in errorCode) + else: + errorCodeForMessage = "".join( + "\\x%2x" % (ord(c),) if ord(c) >= 0x80 else c + for c in errorCode) + + if othertb: + message = "Code<%s>%s: %s\n%s" % ( + errorCodeForMessage, localwhat, description, othertb) + else: + message = "Code<%s>%s: %s" % ( + errorCodeForMessage, localwhat, description) + + super(RemoteAmpError, self).__init__(message) + self.local = local + self.errorCode = errorCode + self.description = description + self.fatal = fatal + + + +class UnknownRemoteError(RemoteAmpError): + """ + This means that an error whose type we can't identify was raised from the + other side. + """ + def __init__(self, description): + errorCode = UNKNOWN_ERROR_CODE + RemoteAmpError.__init__(self, errorCode, description) + + + +class MalformedAmpBox(AmpError): + """ + This error indicates that the wire-level protocol was malformed. + """ + + + +class UnhandledCommand(AmpError): + """ + A command received via amp could not be dispatched. + """ + + + +class IncompatibleVersions(AmpError): + """ + It was impossible to negotiate a compatible version of the protocol with + the other end of the connection. + """ + + +PROTOCOL_ERRORS = {UNHANDLED_ERROR_CODE: UnhandledCommand} + +class AmpBox(dict): + """ + I am a packet in the AMP protocol, much like a regular bytes:bytes dictionary. + """ + __slots__ = [] # be like a regular dictionary, don't magically + # acquire a __dict__... + + + def __init__(self, *args, **kw): + """ + Initialize a new L{AmpBox}. + + In Python 3, keyword arguments MUST be Unicode/native strings whereas + in Python 2 they could be either byte strings or Unicode strings. + + However, all keys of an L{AmpBox} MUST be byte strings, or possible to + transparently coerce into byte strings (i.e. Python 2). + + In Python 3, therefore, native string keys are coerced to byte strings + by encoding as ASCII. This can result in C{UnicodeEncodeError} being + raised. + + @param args: See C{dict}, but all keys and values should be C{bytes}. + On Python 3, native strings may be used as keys provided they + contain only ASCII characters. + + @param kw: See C{dict}, but all keys and values should be C{bytes}. + On Python 3, native strings may be used as keys provided they + contain only ASCII characters. + + @raise UnicodeEncodeError: When a native string key cannot be coerced + to an ASCII byte string (Python 3 only). + """ + super(AmpBox, self).__init__(*args, **kw) + if _PY3: + nonByteNames = [n for n in self if not isinstance(n, bytes)] + for nonByteName in nonByteNames: + byteName = nonByteName.encode("ascii") + self[byteName] = self.pop(nonByteName) + + + def copy(self): + """ + Return another AmpBox just like me. + """ + newBox = self.__class__() + newBox.update(self) + return newBox + + + def serialize(self): + """ + Convert me into a wire-encoded string. + + @return: a C{bytes} encoded according to the rules described in the + module docstring. + """ + i = sorted(iteritems(self)) + L = [] + w = L.append + for k, v in i: + if type(k) == unicode: + raise TypeError("Unicode key not allowed: %r" % k) + if type(v) == unicode: + raise TypeError( + "Unicode value for key %r not allowed: %r" % (k, v)) + if len(k) > MAX_KEY_LENGTH: + raise TooLong(True, True, k, None) + if len(v) > MAX_VALUE_LENGTH: + raise TooLong(False, True, v, k) + for kv in k, v: + w(pack("!H", len(kv))) + w(kv) + w(pack("!H", 0)) + return b''.join(L) + + + def _sendTo(self, proto): + """ + Serialize and send this box to an Amp instance. By the time it is being + sent, several keys are required. I must have exactly ONE of:: + + _ask + _answer + _error + + If the '_ask' key is set, then the '_command' key must also be + set. + + @param proto: an AMP instance. + """ + proto.sendBox(self) + + def __repr__(self): + return 'AmpBox(%s)' % (dict.__repr__(self),) + +# amp.Box => AmpBox + +Box = AmpBox + +class QuitBox(AmpBox): + """ + I am an AmpBox that, upon being sent, terminates the connection. + """ + __slots__ = [] + + + def __repr__(self): + return 'QuitBox(**%s)' % (super(QuitBox, self).__repr__(),) + + + def _sendTo(self, proto): + """ + Immediately call loseConnection after sending. + """ + super(QuitBox, self)._sendTo(proto) + proto.transport.loseConnection() + + + +class _SwitchBox(AmpBox): + """ + Implementation detail of ProtocolSwitchCommand: I am an AmpBox which sets + up state for the protocol to switch. + """ + + # DON'T set __slots__ here; we do have an attribute. + + def __init__(self, innerProto, **kw): + """ + Create a _SwitchBox with the protocol to switch to after being sent. + + @param innerProto: the protocol instance to switch to. + @type innerProto: an IProtocol provider. + """ + super(_SwitchBox, self).__init__(**kw) + self.innerProto = innerProto + + + def __repr__(self): + return '_SwitchBox(%r, **%s)' % (self.innerProto, + dict.__repr__(self),) + + + def _sendTo(self, proto): + """ + Send me; I am the last box on the connection. All further traffic will be + over the new protocol. + """ + super(_SwitchBox, self)._sendTo(proto) + proto._lockForSwitch() + proto._switchTo(self.innerProto) + + + +@implementer(IBoxReceiver) +class BoxDispatcher: + """ + A L{BoxDispatcher} dispatches '_ask', '_answer', and '_error' L{AmpBox}es, + both incoming and outgoing, to their appropriate destinations. + + Outgoing commands are converted into L{Deferred}s and outgoing boxes, and + associated tracking state to fire those L{Deferred} when '_answer' boxes + come back. Incoming '_answer' and '_error' boxes are converted into + callbacks and errbacks on those L{Deferred}s, respectively. + + Incoming '_ask' boxes are converted into method calls on a supplied method + locator. + + @ivar _outstandingRequests: a dictionary mapping request IDs to + L{Deferred}s which were returned for those requests. + + @ivar locator: an object with a L{CommandLocator.locateResponder} method + that locates a responder function that takes a Box and returns a result + (either a Box or a Deferred which fires one). + + @ivar boxSender: an object which can send boxes, via the L{_sendBoxCommand} + method, such as an L{AMP} instance. + @type boxSender: L{IBoxSender} + """ + + _failAllReason = None + _outstandingRequests = None + _counter = long(0) + boxSender = None + + def __init__(self, locator): + self._outstandingRequests = {} + self.locator = locator + + + def startReceivingBoxes(self, boxSender): + """ + The given boxSender is going to start calling boxReceived on this + L{BoxDispatcher}. + + @param boxSender: The L{IBoxSender} to send command responses to. + """ + self.boxSender = boxSender + + + def stopReceivingBoxes(self, reason): + """ + No further boxes will be received here. Terminate all currently + outstanding command deferreds with the given reason. + """ + self.failAllOutgoing(reason) + + + def failAllOutgoing(self, reason): + """ + Call the errback on all outstanding requests awaiting responses. + + @param reason: the Failure instance to pass to those errbacks. + """ + self._failAllReason = reason + OR = self._outstandingRequests.items() + self._outstandingRequests = None # we can never send another request + for key, value in OR: + value.errback(reason) + + + def _nextTag(self): + """ + Generate protocol-local serial numbers for _ask keys. + + @return: a string that has not yet been used on this connection. + """ + self._counter += 1 + return (b'%x' % (self._counter,)) + + + def _sendBoxCommand(self, command, box, requiresAnswer=True): + """ + Send a command across the wire with the given C{amp.Box}. + + Mutate the given box to give it any additional keys (_command, _ask) + required for the command and request/response machinery, then send it. + + If requiresAnswer is True, returns a C{Deferred} which fires when a + response is received. The C{Deferred} is fired with an C{amp.Box} on + success, or with an C{amp.RemoteAmpError} if an error is received. + + If the Deferred fails and the error is not handled by the caller of + this method, the failure will be logged and the connection dropped. + + @param command: a C{bytes}, the name of the command to issue. + + @param box: an AmpBox with the arguments for the command. + + @param requiresAnswer: a boolean. Defaults to True. If True, return a + Deferred which will fire when the other side responds to this command. + If False, return None and do not ask the other side for acknowledgement. + + @return: a Deferred which fires the AmpBox that holds the response to + this command, or None, as specified by requiresAnswer. + + @raise ProtocolSwitched: if the protocol has been switched. + """ + if self._failAllReason is not None: + if requiresAnswer: + return fail(self._failAllReason) + else: + return None + box[COMMAND] = command + tag = self._nextTag() + if requiresAnswer: + box[ASK] = tag + box._sendTo(self.boxSender) + if requiresAnswer: + result = self._outstandingRequests[tag] = Deferred() + else: + result = None + return result + + + def callRemoteString(self, command, requiresAnswer=True, **kw): + """ + This is a low-level API, designed only for optimizing simple messages + for which the overhead of parsing is too great. + + @param command: a C{bytes} naming the command. + + @param kw: arguments to the amp box. + + @param requiresAnswer: a boolean. Defaults to True. If True, return a + Deferred which will fire when the other side responds to this command. + If False, return None and do not ask the other side for acknowledgement. + + @return: a Deferred which fires the AmpBox that holds the response to + this command, or None, as specified by requiresAnswer. + """ + box = Box(kw) + return self._sendBoxCommand(command, box, requiresAnswer) + + + def callRemote(self, commandType, *a, **kw): + """ + This is the primary high-level API for sending messages via AMP. Invoke it + with a command and appropriate arguments to send a message to this + connection's peer. + + @param commandType: a subclass of Command. + @type commandType: L{type} + + @param a: Positional (special) parameters taken by the command. + Positional parameters will typically not be sent over the wire. The + only command included with AMP which uses positional parameters is + L{ProtocolSwitchCommand}, which takes the protocol that will be + switched to as its first argument. + + @param kw: Keyword arguments taken by the command. These are the + arguments declared in the command's 'arguments' attribute. They will + be encoded and sent to the peer as arguments for the L{commandType}. + + @return: If L{commandType} has a C{requiresAnswer} attribute set to + L{False}, then return L{None}. Otherwise, return a L{Deferred} which + fires with a dictionary of objects representing the result of this + call. Additionally, this L{Deferred} may fail with an exception + representing a connection failure, with L{UnknownRemoteError} if the + other end of the connection fails for an unknown reason, or with any + error specified as a key in L{commandType}'s C{errors} dictionary. + """ + + # XXX this takes command subclasses and not command objects on purpose. + # There's really no reason to have all this back-and-forth between + # command objects and the protocol, and the extra object being created + # (the Command instance) is pointless. Command is kind of like + # Interface, and should be more like it. + + # In other words, the fact that commandType is instantiated here is an + # implementation detail. Don't rely on it. + + try: + co = commandType(*a, **kw) + except: + return fail() + return co._doCommand(self) + + + def unhandledError(self, failure): + """ + This is a terminal callback called after application code has had a + chance to quash any errors. + """ + return self.boxSender.unhandledError(failure) + + + def _answerReceived(self, box): + """ + An AMP box was received that answered a command previously sent with + L{callRemote}. + + @param box: an AmpBox with a value for its L{ANSWER} key. + """ + question = self._outstandingRequests.pop(box[ANSWER]) + question.addErrback(self.unhandledError) + question.callback(box) + + + def _errorReceived(self, box): + """ + An AMP box was received that answered a command previously sent with + L{callRemote}, with an error. + + @param box: an L{AmpBox} with a value for its L{ERROR}, L{ERROR_CODE}, + and L{ERROR_DESCRIPTION} keys. + """ + question = self._outstandingRequests.pop(box[ERROR]) + question.addErrback(self.unhandledError) + errorCode = box[ERROR_CODE] + description = box[ERROR_DESCRIPTION] + if isinstance(description, bytes): + description = description.decode("utf-8", "replace") + if errorCode in PROTOCOL_ERRORS: + exc = PROTOCOL_ERRORS[errorCode](errorCode, description) + else: + exc = RemoteAmpError(errorCode, description) + question.errback(Failure(exc)) + + + def _commandReceived(self, box): + """ + @param box: an L{AmpBox} with a value for its L{COMMAND} and L{ASK} + keys. + """ + def formatAnswer(answerBox): + answerBox[ANSWER] = box[ASK] + return answerBox + def formatError(error): + if error.check(RemoteAmpError): + code = error.value.errorCode + desc = error.value.description + if isinstance(desc, unicode): + desc = desc.encode("utf-8", "replace") + if error.value.fatal: + errorBox = QuitBox() + else: + errorBox = AmpBox() + else: + errorBox = QuitBox() + log.err(error) # here is where server-side logging happens + # if the error isn't handled + code = UNKNOWN_ERROR_CODE + desc = b"Unknown Error" + errorBox[ERROR] = box[ASK] + errorBox[ERROR_DESCRIPTION] = desc + errorBox[ERROR_CODE] = code + return errorBox + deferred = self.dispatchCommand(box) + if ASK in box: + deferred.addCallbacks(formatAnswer, formatError) + deferred.addCallback(self._safeEmit) + deferred.addErrback(self.unhandledError) + + + def ampBoxReceived(self, box): + """ + An AmpBox was received, representing a command, or an answer to a + previously issued command (either successful or erroneous). Respond to + it according to its contents. + + @param box: an AmpBox + + @raise NoEmptyBoxes: when a box is received that does not contain an + '_answer', '_command' / '_ask', or '_error' key; i.e. one which does not + fit into the command / response protocol defined by AMP. + """ + if ANSWER in box: + self._answerReceived(box) + elif ERROR in box: + self._errorReceived(box) + elif COMMAND in box: + self._commandReceived(box) + else: + raise NoEmptyBoxes(box) + + + def _safeEmit(self, aBox): + """ + Emit a box, ignoring L{ProtocolSwitched} and L{ConnectionLost} errors + which cannot be usefully handled. + """ + try: + aBox._sendTo(self.boxSender) + except (ProtocolSwitched, ConnectionLost): + pass + + + def dispatchCommand(self, box): + """ + A box with a _command key was received. + + Dispatch it to a local handler call it. + + @param proto: an AMP instance. + @param box: an AmpBox to be dispatched. + """ + cmd = box[COMMAND] + responder = self.locator.locateResponder(cmd) + if responder is None: + description = "Unhandled Command: %r" % (cmd,) + return fail(RemoteAmpError( + UNHANDLED_ERROR_CODE, + description, + False, + local=Failure(UnhandledCommand()))) + return maybeDeferred(responder, box) + + + +@implementer(IResponderLocator) +class CommandLocator: + """ + A L{CommandLocator} is a collection of responders to AMP L{Command}s, with + the help of the L{Command.responder} decorator. + """ + + class __metaclass__(type): + """ + This metaclass keeps track of all of the Command.responder-decorated + methods defined since the last CommandLocator subclass was defined. It + assumes (usually correctly, but unfortunately not necessarily so) that + those commands responders were all declared as methods of the class + being defined. Note that this list can be incorrect if users use the + Command.responder decorator outside the context of a CommandLocator + class declaration. + + Command responders defined on subclasses are given precedence over + those inherited from a base class. + + The Command.responder decorator explicitly cooperates with this + metaclass. + """ + + _currentClassCommands = [] + def __new__(cls, name, bases, attrs): + commands = cls._currentClassCommands[:] + cls._currentClassCommands[:] = [] + cd = attrs['_commandDispatch'] = {} + subcls = type.__new__(cls, name, bases, attrs) + ancestors = list(subcls.__mro__[1:]) + ancestors.reverse() + for ancestor in ancestors: + cd.update(getattr(ancestor, '_commandDispatch', {})) + for commandClass, responderFunc in commands: + cd[commandClass.commandName] = (commandClass, responderFunc) + if (bases and ( + subcls.lookupFunction != CommandLocator.lookupFunction)): + def locateResponder(self, name): + warnings.warn( + "Override locateResponder, not lookupFunction.", + category=PendingDeprecationWarning, + stacklevel=2) + return self.lookupFunction(name) + subcls.locateResponder = locateResponder + return subcls + + + def _wrapWithSerialization(self, aCallable, command): + """ + Wrap aCallable with its command's argument de-serialization + and result serialization logic. + + @param aCallable: a callable with a 'command' attribute, designed to be + called with keyword arguments. + + @param command: the command class whose serialization to use. + + @return: a 1-arg callable which, when invoked with an AmpBox, will + deserialize the argument list and invoke appropriate user code for the + callable's command, returning a Deferred which fires with the result or + fails with an error. + """ + def doit(box): + kw = command.parseArguments(box, self) + def checkKnownErrors(error): + key = error.trap(*command.allErrors) + code = command.allErrors[key] + desc = str(error.value) + return Failure(RemoteAmpError( + code, desc, key in command.fatalErrors, local=error)) + def makeResponseFor(objects): + try: + return command.makeResponse(objects, self) + except: + # let's helpfully log this. + originalFailure = Failure() + raise BadLocalReturn( + "%r returned %r and %r could not serialize it" % ( + aCallable, + objects, + command), + originalFailure) + return maybeDeferred(aCallable, **kw).addCallback( + makeResponseFor).addErrback( + checkKnownErrors) + return doit + + + def lookupFunction(self, name): + """ + Deprecated synonym for L{CommandLocator.locateResponder} + """ + if self.__class__.lookupFunction != CommandLocator.lookupFunction: + return CommandLocator.locateResponder(self, name) + else: + warnings.warn("Call locateResponder, not lookupFunction.", + category=PendingDeprecationWarning, + stacklevel=2) + return self.locateResponder(name) + + + def locateResponder(self, name): + """ + Locate a callable to invoke when executing the named command. + + @param name: the normalized name (from the wire) of the command. + @type name: C{bytes} + + @return: a 1-argument function that takes a Box and returns a box or a + Deferred which fires a Box, for handling the command identified by the + given name, or None, if no appropriate responder can be found. + """ + # Try to find a high-level method to invoke, and if we can't find one, + # fall back to a low-level one. + cd = self._commandDispatch + if name in cd: + commandClass, responderFunc = cd[name] + if _PY3: + responderMethod = types.MethodType( + responderFunc, self) + else: + responderMethod = types.MethodType( + responderFunc, self, self.__class__) + return self._wrapWithSerialization(responderMethod, commandClass) + + + +if _PY3: + # Python 3 ignores the __metaclass__ attribute and has instead new syntax + # for setting the metaclass. Unfortunately it's not valid Python 2 syntax + # so we work-around it by recreating CommandLocator using the metaclass + # here. + CommandLocator = CommandLocator.__metaclass__( + "CommandLocator", (CommandLocator, ), {}) + + + +@implementer(IResponderLocator) +class SimpleStringLocator(object): + """ + Implement the L{AMP.locateResponder} method to do simple, string-based + dispatch. + """ + + baseDispatchPrefix = b'amp_' + + def locateResponder(self, name): + """ + Locate a callable to invoke when executing the named command. + + @return: a function with the name C{"amp_" + name} on the same + instance, or None if no such function exists. + This function will then be called with the L{AmpBox} itself as an + argument. + + @param name: the normalized name (from the wire) of the command. + @type name: C{bytes} + """ + fName = nativeString(self.baseDispatchPrefix + name.upper()) + return getattr(self, fName, None) + + + +PYTHON_KEYWORDS = [ + 'and', 'del', 'for', 'is', 'raise', 'assert', 'elif', 'from', 'lambda', + 'return', 'break', 'else', 'global', 'not', 'try', 'class', 'except', + 'if', 'or', 'while', 'continue', 'exec', 'import', 'pass', 'yield', + 'def', 'finally', 'in', 'print'] + + + +def _wireNameToPythonIdentifier(key): + """ + (Private) Normalize an argument name from the wire for use with Python + code. If the return value is going to be a python keyword it will be + capitalized. If it contains any dashes they will be replaced with + underscores. + + The rationale behind this method is that AMP should be an inherently + multi-language protocol, so message keys may contain all manner of bizarre + bytes. This is not a complete solution; there are still forms of arguments + that this implementation will be unable to parse. However, Python + identifiers share a huge raft of properties with identifiers from many + other languages, so this is a 'good enough' effort for now. We deal + explicitly with dashes because that is the most likely departure: Lisps + commonly use dashes to separate method names, so protocols initially + implemented in a lisp amp dialect may use dashes in argument or command + names. + + @param key: a C{bytes}, looking something like 'foo-bar-baz' or 'from' + @type key: C{bytes} + + @return: a native string which is a valid python identifier, looking + something like 'foo_bar_baz' or 'From'. + """ + lkey = nativeString(key.replace(b"-", b"_")) + if lkey in PYTHON_KEYWORDS: + return lkey.title() + return lkey + + + +@implementer(IArgumentType) +class Argument: + """ + Base-class of all objects that take values from Amp packets and convert + them into objects for Python functions. + + This implementation of L{IArgumentType} provides several higher-level + hooks for subclasses to override. See L{toString} and L{fromString} + which will be used to define the behavior of L{IArgumentType.toBox} and + L{IArgumentType.fromBox}, respectively. + """ + + optional = False + + + def __init__(self, optional=False): + """ + Create an Argument. + + @param optional: a boolean indicating whether this argument can be + omitted in the protocol. + """ + self.optional = optional + + + def retrieve(self, d, name, proto): + """ + Retrieve the given key from the given dictionary, removing it if found. + + @param d: a dictionary. + + @param name: a key in I{d}. + + @param proto: an instance of an AMP. + + @raise KeyError: if I am not optional and no value was found. + + @return: d[name]. + """ + if self.optional: + value = d.get(name) + if value is not None: + del d[name] + else: + value = d.pop(name) + return value + + + def fromBox(self, name, strings, objects, proto): + """ + Populate an 'out' dictionary with mapping names to Python values + decoded from an 'in' AmpBox mapping strings to string values. + + @param name: the argument name to retrieve + @type name: C{bytes} + + @param strings: The AmpBox to read string(s) from, a mapping of + argument names to string values. + @type strings: AmpBox + + @param objects: The dictionary to write object(s) to, a mapping of + names to Python objects. Keys will be native strings. + @type objects: dict + + @param proto: an AMP instance. + """ + st = self.retrieve(strings, name, proto) + nk = _wireNameToPythonIdentifier(name) + if self.optional and st is None: + objects[nk] = None + else: + objects[nk] = self.fromStringProto(st, proto) + + + def toBox(self, name, strings, objects, proto): + """ + Populate an 'out' AmpBox with strings encoded from an 'in' dictionary + mapping names to Python values. + + @param name: the argument name to retrieve + @type name: C{bytes} + + @param strings: The AmpBox to write string(s) to, a mapping of + argument names to string values. + @type strings: AmpBox + + @param objects: The dictionary to read object(s) from, a mapping of + names to Python objects. Keys should be native strings. + + @type objects: dict + + @param proto: the protocol we are converting for. + @type proto: AMP + """ + obj = self.retrieve(objects, _wireNameToPythonIdentifier(name), proto) + if self.optional and obj is None: + # strings[name] = None + pass + else: + strings[name] = self.toStringProto(obj, proto) + + + def fromStringProto(self, inString, proto): + """ + Convert a string to a Python value. + + @param inString: the string to convert. + @type inString: C{bytes} + + @param proto: the protocol we are converting for. + @type proto: AMP + + @return: a Python object. + """ + return self.fromString(inString) + + + def toStringProto(self, inObject, proto): + """ + Convert a Python object to a string. + + @param inObject: the object to convert. + + @param proto: the protocol we are converting for. + @type proto: AMP + """ + return self.toString(inObject) + + + def fromString(self, inString): + """ + Convert a string to a Python object. Subclasses must implement this. + + @param inString: the string to convert. + @type inString: C{bytes} + + @return: the decoded value from C{inString} + """ + + + def toString(self, inObject): + """ + Convert a Python object into a string for passing over the network. + + @param inObject: an object of the type that this Argument is intended + to deal with. + + @return: the wire encoding of inObject + @rtype: C{bytes} + """ + + + +class Integer(Argument): + """ + Encode any integer values of any size on the wire as the string + representation. + + Example: C{123} becomes C{"123"} + """ + fromString = int + def toString(self, inObject): + return intToBytes(inObject) + + + +class String(Argument): + """ + Don't do any conversion at all; just pass through 'str'. + """ + def toString(self, inObject): + return inObject + + def fromString(self, inString): + return inString + + + +class Float(Argument): + """ + Encode floating-point values on the wire as their repr. + """ + fromString = float + + def toString(self, inString): + if not isinstance(inString, float): + raise ValueError("Bad float value %r" % (inString,)) + return str(inString).encode('ascii') + + + +class Boolean(Argument): + """ + Encode True or False as "True" or "False" on the wire. + """ + def fromString(self, inString): + if inString == b'True': + return True + elif inString == b'False': + return False + else: + raise TypeError("Bad boolean value: %r" % (inString,)) + + + def toString(self, inObject): + if inObject: + return b'True' + else: + return b'False' + + + +class Unicode(String): + """ + Encode a unicode string on the wire as UTF-8. + """ + + def toString(self, inObject): + return String.toString(self, inObject.encode('utf-8')) + + + def fromString(self, inString): + return String.fromString(self, inString).decode('utf-8') + + + +class Path(Unicode): + """ + Encode and decode L{filepath.FilePath} instances as paths on the wire. + + This is really intended for use with subprocess communication tools: + exchanging pathnames on different machines over a network is not generally + meaningful, but neither is it disallowed; you can use this to communicate + about NFS paths, for example. + """ + def fromString(self, inString): + return filepath.FilePath(Unicode.fromString(self, inString)) + + + def toString(self, inObject): + return Unicode.toString(self, inObject.asTextMode().path) + + + +class ListOf(Argument): + """ + Encode and decode lists of instances of a single other argument type. + + For example, if you want to pass:: + + [3, 7, 9, 15] + + You can create an argument like this:: + + ListOf(Integer()) + + The serialized form of the entire list is subject to the limit imposed by + L{MAX_VALUE_LENGTH}. List elements are represented as 16-bit length + prefixed strings. The argument type passed to the L{ListOf} initializer is + responsible for producing the serialized form of each element. + + @ivar elementType: The L{Argument} instance used to encode and decode list + elements (note, not an arbitrary L{IArgumentType} implementation: + arguments must be implemented using only the C{fromString} and + C{toString} methods, not the C{fromBox} and C{toBox} methods). + + @param optional: a boolean indicating whether this argument can be + omitted in the protocol. + + @since: 10.0 + """ + def __init__(self, elementType, optional=False): + self.elementType = elementType + Argument.__init__(self, optional) + + + def fromString(self, inString): + """ + Convert the serialized form of a list of instances of some type back + into that list. + """ + strings = [] + parser = Int16StringReceiver() + parser.stringReceived = strings.append + parser.dataReceived(inString) + elementFromString = self.elementType.fromString + return [elementFromString(string) for string in strings] + + + def toString(self, inObject): + """ + Serialize the given list of objects to a single string. + """ + strings = [] + for obj in inObject: + serialized = self.elementType.toString(obj) + strings.append(pack('!H', len(serialized))) + strings.append(serialized) + return b''.join(strings) + + + +class AmpList(Argument): + """ + Convert a list of dictionaries into a list of AMP boxes on the wire. + + For example, if you want to pass:: + + [{'a': 7, 'b': u'hello'}, {'a': 9, 'b': u'goodbye'}] + + You might use an AmpList like this in your arguments or response list:: + + AmpList([('a', Integer()), + ('b', Unicode())]) + """ + def __init__(self, subargs, optional=False): + """ + Create an AmpList. + + @param subargs: a list of 2-tuples of ('name', argument) describing the + schema of the dictionaries in the sequence of amp boxes. + @type subargs: A C{list} of (C{bytes}, L{Argument}) tuples. + + @param optional: a boolean indicating whether this argument can be + omitted in the protocol. + """ + assert all(isinstance(name, bytes) for name, _ in subargs), ( + "AmpList should be defined with a list of (name, argument) " + "tuples where `name' is a byte string, got: %r" % (subargs, )) + self.subargs = subargs + Argument.__init__(self, optional) + + + def fromStringProto(self, inString, proto): + boxes = parseString(inString) + values = [_stringsToObjects(box, self.subargs, proto) + for box in boxes] + return values + + + def toStringProto(self, inObject, proto): + return b''.join([_objectsToStrings( + objects, self.subargs, Box(), proto + ).serialize() for objects in inObject]) + + + +class Descriptor(Integer): + """ + Encode and decode file descriptors for exchange over a UNIX domain socket. + + This argument type requires an AMP connection set up over an + L{IUNIXTransport} provider (for + example, the kind of connection created by + L{IReactorUNIX.connectUNIX} + and L{UNIXClientEndpoint}). + + There is no correspondence between the integer value of the file descriptor + on the sending and receiving sides, therefore an alternate approach is taken + to matching up received descriptors with particular L{Descriptor} + parameters. The argument is encoded to an ordinal (unique per connection) + for inclusion in the AMP command or response box. The descriptor itself is + sent using + L{IUNIXTransport.sendFileDescriptor}. + The receiver uses the order in which file descriptors are received and the + ordinal value to come up with the received copy of the descriptor. + """ + def fromStringProto(self, inString, proto): + """ + Take a unique identifier associated with a file descriptor which must + have been received by now and use it to look up that descriptor in a + dictionary where they are kept. + + @param inString: The base representation (as a byte string) of an + ordinal indicating which file descriptor corresponds to this usage + of this argument. + @type inString: C{str} + + @param proto: The protocol used to receive this descriptor. This + protocol must be connected via a transport providing + L{IUNIXTransport}. + @type proto: L{BinaryBoxProtocol} + + @return: The file descriptor represented by C{inString}. + @rtype: C{int} + """ + return proto._getDescriptor(int(inString)) + + + def toStringProto(self, inObject, proto): + """ + Send C{inObject}, an integer file descriptor, over C{proto}'s connection + and return a unique identifier which will allow the receiver to + associate the file descriptor with this argument. + + @param inObject: A file descriptor to duplicate over an AMP connection + as the value for this argument. + @type inObject: C{int} + + @param proto: The protocol which will be used to send this descriptor. + This protocol must be connected via a transport providing + L{IUNIXTransport}. + + @return: A byte string which can be used by the receiver to reconstruct + the file descriptor. + @type: C{str} + """ + identifier = proto._sendFileDescriptor(inObject) + outString = Integer.toStringProto(self, identifier, proto) + return outString + + + +class Command: + """ + Subclass me to specify an AMP Command. + + @cvar arguments: A list of 2-tuples of (name, Argument-subclass-instance), + specifying the names and values of the parameters which are required for + this command. + + @cvar response: A list like L{arguments}, but instead used for the return + value. + + @cvar errors: A mapping of subclasses of L{Exception} to wire-protocol tags + for errors represented as L{str}s. Responders which raise keys from + this dictionary will have the error translated to the corresponding tag + on the wire. + Invokers which receive Deferreds from invoking this command with + L{BoxDispatcher.callRemote} will potentially receive Failures with keys + from this mapping as their value. + This mapping is inherited; if you declare a command which handles + C{FooError} as 'FOO_ERROR', then subclass it and specify C{BarError} as + 'BAR_ERROR', responders to the subclass may raise either C{FooError} or + C{BarError}, and invokers must be able to deal with either of those + exceptions. + + @cvar fatalErrors: like 'errors', but errors in this list will always + terminate the connection, despite being of a recognizable error type. + + @cvar commandType: The type of Box used to issue commands; useful only for + protocol-modifying behavior like startTLS or protocol switching. Defaults + to a plain vanilla L{Box}. + + @cvar responseType: The type of Box used to respond to this command; only + useful for protocol-modifying behavior like startTLS or protocol switching. + Defaults to a plain vanilla L{Box}. + + @ivar requiresAnswer: a boolean; defaults to True. Set it to False on your + subclass if you want callRemote to return None. Note: this is a hint only + to the client side of the protocol. The return-type of a command responder + method must always be a dictionary adhering to the contract specified by + L{response}, because clients are always free to request a response if they + want one. + """ + + class __metaclass__(type): + """ + Metaclass hack to establish reverse-mappings for 'errors' and + 'fatalErrors' as class vars. + """ + def __new__(cls, name, bases, attrs): + reverseErrors = attrs['reverseErrors'] = {} + er = attrs['allErrors'] = {} + if 'commandName' not in attrs: + if _PY3: + attrs['commandName'] = name.encode("ascii") + else: + attrs['commandName'] = name + newtype = type.__new__(cls, name, bases, attrs) + + if not isinstance(newtype.commandName, bytes): + raise TypeError( + "Command names must be byte strings, got: %r" + % (newtype.commandName, )) + for name, _ in newtype.arguments: + if not isinstance(name, bytes): + raise TypeError( + "Argument names must be byte strings, got: %r" + % (name, )) + for name, _ in newtype.response: + if not isinstance(name, bytes): + raise TypeError( + "Response names must be byte strings, got: %r" + % (name, )) + + errors = {} + fatalErrors = {} + accumulateClassDict(newtype, 'errors', errors) + accumulateClassDict(newtype, 'fatalErrors', fatalErrors) + + if not isinstance(newtype.errors, dict): + newtype.errors = dict(newtype.errors) + if not isinstance(newtype.fatalErrors, dict): + newtype.fatalErrors = dict(newtype.fatalErrors) + + for v, k in iteritems(errors): + reverseErrors[k] = v + er[v] = k + for v, k in iteritems(fatalErrors): + reverseErrors[k] = v + er[v] = k + + for _, name in iteritems(newtype.errors): + if not isinstance(name, bytes): + raise TypeError( + "Error names must be byte strings, got: %r" + % (name, )) + for _, name in iteritems(newtype.fatalErrors): + if not isinstance(name, bytes): + raise TypeError( + "Fatal error names must be byte strings, got: %r" + % (name, )) + + return newtype + + arguments = [] + response = [] + extra = [] + errors = {} + fatalErrors = {} + + commandType = Box + responseType = Box + + requiresAnswer = True + + + def __init__(self, **kw): + """ + Create an instance of this command with specified values for its + parameters. + + In Python 3, keyword arguments MUST be Unicode/native strings whereas + in Python 2 they could be either byte strings or Unicode strings. + + A L{Command}'s arguments are defined in its schema using C{bytes} + names. The values for those arguments are plucked from the keyword + arguments using the name returned from L{_wireNameToPythonIdentifier}. + In other words, keyword arguments should be named using the + Python-side equivalent of the on-wire (C{bytes}) name. + + @param kw: a dict containing an appropriate value for each name + specified in the L{arguments} attribute of my class. + + @raise InvalidSignature: if you forgot any required arguments. + """ + self.structured = kw + forgotten = [] + for name, arg in self.arguments: + pythonName = _wireNameToPythonIdentifier(name) + if pythonName not in self.structured and not arg.optional: + forgotten.append(pythonName) + if forgotten: + raise InvalidSignature("forgot %s for %s" % ( + ', '.join(forgotten), self.commandName)) + forgotten = [] + + + def makeResponse(cls, objects, proto): + """ + Serialize a mapping of arguments using this L{Command}'s + response schema. + + @param objects: a dict with keys matching the names specified in + self.response, having values of the types that the Argument objects in + self.response can format. + + @param proto: an L{AMP}. + + @return: an L{AmpBox}. + """ + try: + responseType = cls.responseType() + except: + return fail() + return _objectsToStrings(objects, cls.response, responseType, proto) + makeResponse = classmethod(makeResponse) + + + def makeArguments(cls, objects, proto): + """ + Serialize a mapping of arguments using this L{Command}'s + argument schema. + + @param objects: a dict with keys similar to the names specified in + self.arguments, having values of the types that the Argument objects in + self.arguments can parse. + + @param proto: an L{AMP}. + + @return: An instance of this L{Command}'s C{commandType}. + """ + allowedNames = set() + for (argName, ignored) in cls.arguments: + allowedNames.add(_wireNameToPythonIdentifier(argName)) + + for intendedArg in objects: + if intendedArg not in allowedNames: + raise InvalidSignature( + "%s is not a valid argument" % (intendedArg,)) + return _objectsToStrings(objects, cls.arguments, cls.commandType(), + proto) + makeArguments = classmethod(makeArguments) + + + def parseResponse(cls, box, protocol): + """ + Parse a mapping of serialized arguments using this + L{Command}'s response schema. + + @param box: A mapping of response-argument names to the + serialized forms of those arguments. + @param protocol: The L{AMP} protocol. + + @return: A mapping of response-argument names to the parsed + forms. + """ + return _stringsToObjects(box, cls.response, protocol) + parseResponse = classmethod(parseResponse) + + + def parseArguments(cls, box, protocol): + """ + Parse a mapping of serialized arguments using this + L{Command}'s argument schema. + + @param box: A mapping of argument names to the seralized forms + of those arguments. + @param protocol: The L{AMP} protocol. + + @return: A mapping of argument names to the parsed forms. + """ + return _stringsToObjects(box, cls.arguments, protocol) + parseArguments = classmethod(parseArguments) + + + def responder(cls, methodfunc): + """ + Declare a method to be a responder for a particular command. + + This is a decorator. + + Use like so:: + + class MyCommand(Command): + arguments = [('a', ...), ('b', ...)] + + class MyProto(AMP): + def myFunMethod(self, a, b): + ... + MyCommand.responder(myFunMethod) + + Notes: Although decorator syntax is not used within Twisted, this + function returns its argument and is therefore safe to use with + decorator syntax. + + This is not thread safe. Don't declare AMP subclasses in other + threads. Don't declare responders outside the scope of AMP subclasses; + the behavior is undefined. + + @param methodfunc: A function which will later become a method, which + has a keyword signature compatible with this command's L{argument} list + and returns a dictionary with a set of keys compatible with this + command's L{response} list. + + @return: the methodfunc parameter. + """ + CommandLocator._currentClassCommands.append((cls, methodfunc)) + return methodfunc + responder = classmethod(responder) + + + # Our only instance method + def _doCommand(self, proto): + """ + Encode and send this Command to the given protocol. + + @param proto: an AMP, representing the connection to send to. + + @return: a Deferred which will fire or error appropriately when the + other side responds to the command (or error if the connection is lost + before it is responded to). + """ + + def _massageError(error): + error.trap(RemoteAmpError) + rje = error.value + errorType = self.reverseErrors.get(rje.errorCode, + UnknownRemoteError) + return Failure(errorType(rje.description)) + + d = proto._sendBoxCommand(self.commandName, + self.makeArguments(self.structured, proto), + self.requiresAnswer) + + if self.requiresAnswer: + d.addCallback(self.parseResponse, proto) + d.addErrback(_massageError) + + return d + + + +if _PY3: + # Python 3 ignores the __metaclass__ attribute and has instead new syntax + # for setting the metaclass. Unfortunately it's not valid Python 2 syntax + # so we work-around it by recreating Command using the metaclass here. + Command = Command.__metaclass__("Command", (Command, ), {}) + + + +class _NoCertificate: + """ + This is for peers which don't want to use a local certificate. Used by + AMP because AMP's internal language is all about certificates and this + duck-types in the appropriate place; this API isn't really stable though, + so it's not exposed anywhere public. + + For clients, it will use ephemeral DH keys, or whatever the default is for + certificate-less clients in OpenSSL. For servers, it will generate a + temporary self-signed certificate with garbage values in the DN and use + that. + """ + + def __init__(self, client): + """ + Create a _NoCertificate which either is or isn't for the client side of + the connection. + + @param client: True if we are a client and should truly have no + certificate and be anonymous, False if we are a server and actually + have to generate a temporary certificate. + + @type client: bool + """ + self.client = client + + + def options(self, *authorities): + """ + Behaves like L{twisted.internet.ssl.PrivateCertificate.options}(). + """ + if not self.client: + # do some crud with sslverify to generate a temporary self-signed + # certificate. This is SLOOOWWWWW so it is only in the absolute + # worst, most naive case. + + # We have to do this because OpenSSL will not let both the server + # and client be anonymous. + sharedDN = DN(CN='TEMPORARY CERTIFICATE') + key = KeyPair.generate() + cr = key.certificateRequest(sharedDN) + sscrd = key.signCertificateRequest(sharedDN, cr, lambda dn: True, 1) + cert = key.newCertificate(sscrd) + return cert.options(*authorities) + options = dict() + if authorities: + options.update(dict(verify=True, + requireCertificate=True, + caCerts=[auth.original for auth in authorities])) + occo = CertificateOptions(**options) + return occo + + + +class _TLSBox(AmpBox): + """ + I am an AmpBox that, upon being sent, initiates a TLS connection. + """ + __slots__ = [] + + def __init__(self): + if ssl is None: + raise RemoteAmpError(b"TLS_ERROR", "TLS not available") + AmpBox.__init__(self) + + + def _keyprop(k, default): + return property(lambda self: self.get(k, default)) + + + # These properties are described in startTLS + certificate = _keyprop(b'tls_localCertificate', _NoCertificate(False)) + verify = _keyprop(b'tls_verifyAuthorities', None) + + def _sendTo(self, proto): + """ + Send my encoded value to the protocol, then initiate TLS. + """ + ab = AmpBox(self) + for k in [b'tls_localCertificate', + b'tls_verifyAuthorities']: + ab.pop(k, None) + ab._sendTo(proto) + proto._startTLS(self.certificate, self.verify) + + + +class _LocalArgument(String): + """ + Local arguments are never actually relayed across the wire. This is just a + shim so that StartTLS can pretend to have some arguments: if arguments + acquire documentation properties, replace this with something nicer later. + """ + + def fromBox(self, name, strings, objects, proto): + pass + + + +class StartTLS(Command): + """ + Use, or subclass, me to implement a command that starts TLS. + + Callers of StartTLS may pass several special arguments, which affect the + TLS negotiation: + + - tls_localCertificate: This is a + twisted.internet.ssl.PrivateCertificate which will be used to secure + the side of the connection it is returned on. + + - tls_verifyAuthorities: This is a list of + twisted.internet.ssl.Certificate objects that will be used as the + certificate authorities to verify our peer's certificate. + + Each of those special parameters may also be present as a key in the + response dictionary. + """ + + arguments = [(b"tls_localCertificate", _LocalArgument(optional=True)), + (b"tls_verifyAuthorities", _LocalArgument(optional=True))] + + response = [(b"tls_localCertificate", _LocalArgument(optional=True)), + (b"tls_verifyAuthorities", _LocalArgument(optional=True))] + + responseType = _TLSBox + + def __init__(self, **kw): + """ + Create a StartTLS command. (This is private. Use AMP.callRemote.) + + @param tls_localCertificate: the PrivateCertificate object to use to + secure the connection. If it's None, or unspecified, an ephemeral DH + key is used instead. + + @param tls_verifyAuthorities: a list of Certificate objects which + represent root certificates to verify our peer with. + """ + if ssl is None: + raise RuntimeError("TLS not available.") + self.certificate = kw.pop('tls_localCertificate', _NoCertificate(True)) + self.authorities = kw.pop('tls_verifyAuthorities', None) + Command.__init__(self, **kw) + + + def _doCommand(self, proto): + """ + When a StartTLS command is sent, prepare to start TLS, but don't actually + do it; wait for the acknowledgement, then initiate the TLS handshake. + """ + d = Command._doCommand(self, proto) + proto._prepareTLS(self.certificate, self.authorities) + # XXX before we get back to user code we are going to start TLS... + def actuallystart(response): + proto._startTLS(self.certificate, self.authorities) + return response + d.addCallback(actuallystart) + return d + + + +class ProtocolSwitchCommand(Command): + """ + Use this command to switch from something Amp-derived to a different + protocol mid-connection. This can be useful to use amp as the + connection-startup negotiation phase. Since TLS is a different layer + entirely, you can use Amp to negotiate the security parameters of your + connection, then switch to a different protocol, and the connection will + remain secured. + """ + + def __init__(self, _protoToSwitchToFactory, **kw): + """ + Create a ProtocolSwitchCommand. + + @param _protoToSwitchToFactory: a ProtocolFactory which will generate + the Protocol to switch to. + + @param kw: Keyword arguments, encoded and handled normally as + L{Command} would. + """ + + self.protoToSwitchToFactory = _protoToSwitchToFactory + super(ProtocolSwitchCommand, self).__init__(**kw) + + + def makeResponse(cls, innerProto, proto): + return _SwitchBox(innerProto) + makeResponse = classmethod(makeResponse) + + + def _doCommand(self, proto): + """ + When we emit a ProtocolSwitchCommand, lock the protocol, but don't actually + switch to the new protocol unless an acknowledgement is received. If + an error is received, switch back. + """ + d = super(ProtocolSwitchCommand, self)._doCommand(proto) + proto._lockForSwitch() + def switchNow(ign): + innerProto = self.protoToSwitchToFactory.buildProtocol( + proto.transport.getPeer()) + proto._switchTo(innerProto, self.protoToSwitchToFactory) + return ign + def handle(ign): + proto._unlockFromSwitch() + self.protoToSwitchToFactory.clientConnectionFailed( + None, Failure(CONNECTION_LOST)) + return ign + return d.addCallbacks(switchNow, handle) + + + +@implementer(IFileDescriptorReceiver) +class _DescriptorExchanger(object): + """ + L{_DescriptorExchanger} is a mixin for L{BinaryBoxProtocol} which adds + support for receiving file descriptors, a feature offered by + L{IUNIXTransport}. + + @ivar _descriptors: Temporary storage for all file descriptors received. + Values in this dictionary are the file descriptors (as integers). Keys + in this dictionary are ordinals giving the order in which each + descriptor was received. The ordering information is used to allow + L{Descriptor} to determine which is the correct descriptor for any + particular usage of that argument type. + @type _descriptors: C{dict} + + @ivar _sendingDescriptorCounter: A no-argument callable which returns the + ordinals, starting from 0. This is used to construct values for + C{_sendFileDescriptor}. + + @ivar _receivingDescriptorCounter: A no-argument callable which returns the + ordinals, starting from 0. This is used to construct values for + C{fileDescriptorReceived}. + """ + + def __init__(self): + self._descriptors = {} + self._getDescriptor = self._descriptors.pop + self._sendingDescriptorCounter = partial(next, count()) + self._receivingDescriptorCounter = partial(next, count()) + + + def _sendFileDescriptor(self, descriptor): + """ + Assign and return the next ordinal to the given descriptor after sending + the descriptor over this protocol's transport. + """ + self.transport.sendFileDescriptor(descriptor) + return self._sendingDescriptorCounter() + + + def fileDescriptorReceived(self, descriptor): + """ + Collect received file descriptors to be claimed later by L{Descriptor}. + + @param descriptor: The received file descriptor. + @type descriptor: C{int} + """ + self._descriptors[self._receivingDescriptorCounter()] = descriptor + + + +@implementer(IBoxSender) +class BinaryBoxProtocol(StatefulStringProtocol, Int16StringReceiver, + _DescriptorExchanger): + """ + A protocol for receiving L{AmpBox}es - key/value pairs - via length-prefixed + strings. A box is composed of: + + - any number of key-value pairs, described by: + - a 2-byte network-endian packed key length (of which the first + byte must be null, and the second must be non-null: i.e. the + value of the length must be 1-255) + - a key, comprised of that many bytes + - a 2-byte network-endian unsigned value length (up to the maximum + of 65535) + - a value, comprised of that many bytes + - 2 null bytes + + In other words, an even number of strings prefixed with packed unsigned + 16-bit integers, and then a 0-length string to indicate the end of the box. + + This protocol also implements 2 extra private bits of functionality related + to the byte boundaries between messages; it can start TLS between two given + boxes or switch to an entirely different protocol. However, due to some + tricky elements of the implementation, the public interface to this + functionality is L{ProtocolSwitchCommand} and L{StartTLS}. + + @ivar _keyLengthLimitExceeded: A flag which is only true when the + connection is being closed because a key length prefix which was longer + than allowed by the protocol was received. + + @ivar boxReceiver: an L{IBoxReceiver} provider, whose + L{IBoxReceiver.ampBoxReceived} method will be invoked for each + L{AmpBox} that is received. + """ + + _justStartedTLS = False + _startingTLSBuffer = None + _locked = False + _currentKey = None + _currentBox = None + + _keyLengthLimitExceeded = False + + hostCertificate = None + noPeerCertificate = False # for tests + innerProtocol = None + innerProtocolClientFactory = None + + def __init__(self, boxReceiver): + _DescriptorExchanger.__init__(self) + self.boxReceiver = boxReceiver + + + def _switchTo(self, newProto, clientFactory=None): + """ + Switch this BinaryBoxProtocol's transport to a new protocol. You need + to do this 'simultaneously' on both ends of a connection; the easiest + way to do this is to use a subclass of ProtocolSwitchCommand. + + @param newProto: the new protocol instance to switch to. + + @param clientFactory: the ClientFactory to send the + L{twisted.internet.protocol.ClientFactory.clientConnectionLost} + notification to. + """ + # All the data that Int16Receiver has not yet dealt with belongs to our + # new protocol: luckily it's keeping that in a handy (although + # ostensibly internal) variable for us: + newProtoData = self.recvd + # We're quite possibly in the middle of a 'dataReceived' loop in + # Int16StringReceiver: let's make sure that the next iteration, the + # loop will break and not attempt to look at something that isn't a + # length prefix. + self.recvd = '' + # Finally, do the actual work of setting up the protocol and delivering + # its first chunk of data, if one is available. + self.innerProtocol = newProto + self.innerProtocolClientFactory = clientFactory + newProto.makeConnection(self.transport) + if newProtoData: + newProto.dataReceived(newProtoData) + + + def sendBox(self, box): + """ + Send a amp.Box to my peer. + + Note: transport.write is never called outside of this method. + + @param box: an AmpBox. + + @raise ProtocolSwitched: if the protocol has previously been switched. + + @raise ConnectionLost: if the connection has previously been lost. + """ + if self._locked: + raise ProtocolSwitched( + "This connection has switched: no AMP traffic allowed.") + if self.transport is None: + raise ConnectionLost() + if self._startingTLSBuffer is not None: + self._startingTLSBuffer.append(box) + else: + self.transport.write(box.serialize()) + + + def makeConnection(self, transport): + """ + Notify L{boxReceiver} that it is about to receive boxes from this + protocol by invoking L{IBoxReceiver.startReceivingBoxes}. + """ + self.transport = transport + self.boxReceiver.startReceivingBoxes(self) + self.connectionMade() + + + def dataReceived(self, data): + """ + Either parse incoming data as L{AmpBox}es or relay it to our nested + protocol. + """ + if self._justStartedTLS: + self._justStartedTLS = False + # If we already have an inner protocol, then we don't deliver data to + # the protocol parser any more; we just hand it off. + if self.innerProtocol is not None: + self.innerProtocol.dataReceived(data) + return + return Int16StringReceiver.dataReceived(self, data) + + + def connectionLost(self, reason): + """ + The connection was lost; notify any nested protocol. + """ + if self.innerProtocol is not None: + self.innerProtocol.connectionLost(reason) + if self.innerProtocolClientFactory is not None: + self.innerProtocolClientFactory.clientConnectionLost(None, reason) + if self._keyLengthLimitExceeded: + failReason = Failure(TooLong(True, False, None, None)) + elif reason.check(ConnectionClosed) and self._justStartedTLS: + # We just started TLS and haven't received any data. This means + # the other connection didn't like our cert (although they may not + # have told us why - later Twisted should make 'reason' into a TLS + # error.) + failReason = PeerVerifyError( + "Peer rejected our certificate for an unknown reason.") + else: + failReason = reason + self.boxReceiver.stopReceivingBoxes(failReason) + + + # The longest key allowed + _MAX_KEY_LENGTH = 255 + + # The longest value allowed (this is somewhat redundant, as longer values + # cannot be encoded - ah well). + _MAX_VALUE_LENGTH = 65535 + + # The first thing received is a key. + MAX_LENGTH = _MAX_KEY_LENGTH + + def proto_init(self, string): + """ + String received in the 'init' state. + """ + self._currentBox = AmpBox() + return self.proto_key(string) + + + def proto_key(self, string): + """ + String received in the 'key' state. If the key is empty, a complete + box has been received. + """ + if string: + self._currentKey = string + self.MAX_LENGTH = self._MAX_VALUE_LENGTH + return 'value' + else: + self.boxReceiver.ampBoxReceived(self._currentBox) + self._currentBox = None + return 'init' + + + def proto_value(self, string): + """ + String received in the 'value' state. + """ + self._currentBox[self._currentKey] = string + self._currentKey = None + self.MAX_LENGTH = self._MAX_KEY_LENGTH + return 'key' + + + def lengthLimitExceeded(self, length): + """ + The key length limit was exceeded. Disconnect the transport and make + sure a meaningful exception is reported. + """ + self._keyLengthLimitExceeded = True + self.transport.loseConnection() + + + def _lockForSwitch(self): + """ + Lock this binary protocol so that no further boxes may be sent. This + is used when sending a request to switch underlying protocols. You + probably want to subclass ProtocolSwitchCommand rather than calling + this directly. + """ + self._locked = True + + + def _unlockFromSwitch(self): + """ + Unlock this locked binary protocol so that further boxes may be sent + again. This is used after an attempt to switch protocols has failed + for some reason. + """ + if self.innerProtocol is not None: + raise ProtocolSwitched("Protocol already switched. Cannot unlock.") + self._locked = False + + + def _prepareTLS(self, certificate, verifyAuthorities): + """ + Used by StartTLSCommand to put us into the state where we don't + actually send things that get sent, instead we buffer them. see + L{_sendBoxCommand}. + """ + self._startingTLSBuffer = [] + if self.hostCertificate is not None: + raise OnlyOneTLS( + "Previously authenticated connection between %s and %s " + "is trying to re-establish as %s" % ( + self.hostCertificate, + self.peerCertificate, + (certificate, verifyAuthorities))) + + + def _startTLS(self, certificate, verifyAuthorities): + """ + Used by TLSBox to initiate the SSL handshake. + + @param certificate: a L{twisted.internet.ssl.PrivateCertificate} for + use locally. + + @param verifyAuthorities: L{twisted.internet.ssl.Certificate} instances + representing certificate authorities which will verify our peer. + """ + self.hostCertificate = certificate + self._justStartedTLS = True + if verifyAuthorities is None: + verifyAuthorities = () + self.transport.startTLS(certificate.options(*verifyAuthorities)) + stlsb = self._startingTLSBuffer + if stlsb is not None: + self._startingTLSBuffer = None + for box in stlsb: + self.sendBox(box) + + + def _getPeerCertificate(self): + if self.noPeerCertificate: + return None + return Certificate.peerFromTransport(self.transport) + peerCertificate = property(_getPeerCertificate) + + + def unhandledError(self, failure): + """ + The buck stops here. This error was completely unhandled, time to + terminate the connection. + """ + log.err( + failure, + "Amp server or network failure unhandled by client application. " + "Dropping connection! To avoid, add errbacks to ALL remote " + "commands!") + if self.transport is not None: + self.transport.loseConnection() + + + def _defaultStartTLSResponder(self): + """ + The default TLS responder doesn't specify any certificate or anything. + + From a security perspective, it's little better than a plain-text + connection - but it is still a *bit* better, so it's included for + convenience. + + You probably want to override this by providing your own StartTLS.responder. + """ + return {} + StartTLS.responder(_defaultStartTLSResponder) + + + +class AMP(BinaryBoxProtocol, BoxDispatcher, + CommandLocator, SimpleStringLocator): + """ + This protocol is an AMP connection. See the module docstring for protocol + details. + """ + + _ampInitialized = False + + def __init__(self, boxReceiver=None, locator=None): + # For backwards compatibility. When AMP did not separate parsing logic + # (L{BinaryBoxProtocol}), request-response logic (L{BoxDispatcher}) and + # command routing (L{CommandLocator}), it did not have a constructor. + # Now it does, so old subclasses might have defined their own that did + # not upcall. If this flag isn't set, we'll call the constructor in + # makeConnection before anything actually happens. + self._ampInitialized = True + if boxReceiver is None: + boxReceiver = self + if locator is None: + locator = self + BoxDispatcher.__init__(self, locator) + BinaryBoxProtocol.__init__(self, boxReceiver) + + + def locateResponder(self, name): + """ + Unify the implementations of L{CommandLocator} and + L{SimpleStringLocator} to perform both kinds of dispatch, preferring + L{CommandLocator}. + + @type name: C{bytes} + """ + firstResponder = CommandLocator.locateResponder(self, name) + if firstResponder is not None: + return firstResponder + secondResponder = SimpleStringLocator.locateResponder(self, name) + return secondResponder + + + def __repr__(self): + """ + A verbose string representation which gives us information about this + AMP connection. + """ + if self.innerProtocol is not None: + innerRepr = ' inner %r' % (self.innerProtocol,) + else: + innerRepr = '' + return '<%s%s at 0x%x>' % ( + self.__class__.__name__, innerRepr, id(self)) + + + def makeConnection(self, transport): + """ + Emit a helpful log message when the connection is made. + """ + if not self._ampInitialized: + # See comment in the constructor re: backward compatibility. I + # should probably emit a deprecation warning here. + AMP.__init__(self) + # Save these so we can emit a similar log message in L{connectionLost}. + self._transportPeer = transport.getPeer() + self._transportHost = transport.getHost() + log.msg("%s connection established (HOST:%s PEER:%s)" % ( + self.__class__.__name__, + self._transportHost, + self._transportPeer)) + BinaryBoxProtocol.makeConnection(self, transport) + + + def connectionLost(self, reason): + """ + Emit a helpful log message when the connection is lost. + """ + log.msg("%s connection lost (HOST:%s PEER:%s)" % + (self.__class__.__name__, + self._transportHost, + self._transportPeer)) + BinaryBoxProtocol.connectionLost(self, reason) + self.transport = None + + + +class _ParserHelper: + """ + A box receiver which records all boxes received. + """ + def __init__(self): + self.boxes = [] + + + def getPeer(self): + return 'string' + + + def getHost(self): + return 'string' + + disconnecting = False + + + def startReceivingBoxes(self, sender): + """ + No initialization is required. + """ + + + def ampBoxReceived(self, box): + self.boxes.append(box) + + + # Synchronous helpers + def parse(cls, fileObj): + """ + Parse some amp data stored in a file. + + @param fileObj: a file-like object. + + @return: a list of AmpBoxes encoded in the given file. + """ + parserHelper = cls() + bbp = BinaryBoxProtocol(boxReceiver=parserHelper) + bbp.makeConnection(parserHelper) + bbp.dataReceived(fileObj.read()) + return parserHelper.boxes + parse = classmethod(parse) + + + def parseString(cls, data): + """ + Parse some amp data stored in a string. + + @param data: a str holding some amp-encoded data. + + @return: a list of AmpBoxes encoded in the given string. + """ + return cls.parse(BytesIO(data)) + parseString = classmethod(parseString) + + + +parse = _ParserHelper.parse +parseString = _ParserHelper.parseString + +def _stringsToObjects(strings, arglist, proto): + """ + Convert an AmpBox to a dictionary of python objects, converting through a + given arglist. + + @param strings: an AmpBox (or dict of strings) + + @param arglist: a list of 2-tuples of strings and Argument objects, as + described in L{Command.arguments}. + + @param proto: an L{AMP} instance. + + @return: the converted dictionary mapping names to argument objects. + """ + objects = {} + myStrings = strings.copy() + for argname, argparser in arglist: + argparser.fromBox(argname, myStrings, objects, proto) + return objects + + + +def _objectsToStrings(objects, arglist, strings, proto): + """ + Convert a dictionary of python objects to an AmpBox, converting through a + given arglist. + + @param objects: a dict mapping names to python objects + + @param arglist: a list of 2-tuples of strings and Argument objects, as + described in L{Command.arguments}. + + @param strings: [OUT PARAMETER] An object providing the L{dict} + interface which will be populated with serialized data. + + @param proto: an L{AMP} instance. + + @return: The converted dictionary mapping names to encoded argument + strings (identical to C{strings}). + """ + myObjects = objects.copy() + for argname, argparser in arglist: + argparser.toBox(argname, strings, myObjects, proto) + return strings + + + +class Decimal(Argument): + """ + Encodes C{decimal.Decimal} instances. + + There are several ways in which a decimal value might be encoded. + + Special values are encoded as special strings:: + + - Positive infinity is encoded as C{"Infinity"} + - Negative infinity is encoded as C{"-Infinity"} + - Quiet not-a-number is encoded as either C{"NaN"} or C{"-NaN"} + - Signalling not-a-number is encoded as either C{"sNaN"} or C{"-sNaN"} + + Normal values are encoded using the base ten string representation, using + engineering notation to indicate magnitude without precision, and "normal" + digits to indicate precision. For example:: + + - C{"1"} represents the value I{1} with precision to one place. + - C{"-1"} represents the value I{-1} with precision to one place. + - C{"1.0"} represents the value I{1} with precision to two places. + - C{"10"} represents the value I{10} with precision to two places. + - C{"1E+2"} represents the value I{10} with precision to one place. + - C{"1E-1"} represents the value I{0.1} with precision to one place. + - C{"1.5E+2"} represents the value I{15} with precision to two places. + + U{http://speleotrove.com/decimal/} should be considered the authoritative + specification for the format. + """ + + def fromString(self, inString): + inString = nativeString(inString) + return decimal.Decimal(inString) + + def toString(self, inObject): + """ + Serialize a C{decimal.Decimal} instance to the specified wire format. + """ + if isinstance(inObject, decimal.Decimal): + # Hopefully decimal.Decimal.__str__ actually does what we want. + return str(inObject).encode("ascii") + raise ValueError( + "amp.Decimal can only encode instances of decimal.Decimal") + + + +class DateTime(Argument): + """ + Encodes C{datetime.datetime} instances. + + Wire format: '%04i-%02i-%02iT%02i:%02i:%02i.%06i%s%02i:%02i'. Fields in + order are: year, month, day, hour, minute, second, microsecond, timezone + direction (+ or -), timezone hour, timezone minute. Encoded string is + always exactly 32 characters long. This format is compatible with ISO 8601, + but that does not mean all ISO 8601 dates can be accepted. + + Also, note that the datetime module's notion of a "timezone" can be + complex, but the wire format includes only a fixed offset, so the + conversion is not lossless. A lossless transmission of a C{datetime} instance + is not feasible since the receiving end would require a Python interpreter. + + @ivar _positions: A sequence of slices giving the positions of various + interesting parts of the wire format. + """ + + _positions = [ + slice(0, 4), slice(5, 7), slice(8, 10), # year, month, day + slice(11, 13), slice(14, 16), slice(17, 19), # hour, minute, second + slice(20, 26), # microsecond + # intentionally skip timezone direction, as it is not an integer + slice(27, 29), slice(30, 32) # timezone hour, timezone minute + ] + + def fromString(self, s): + """ + Parse a string containing a date and time in the wire format into a + C{datetime.datetime} instance. + """ + s = nativeString(s) + + if len(s) != 32: + raise ValueError('invalid date format %r' % (s,)) + + values = [int(s[p]) for p in self._positions] + sign = s[26] + timezone = _FixedOffsetTZInfo.fromSignHoursMinutes(sign, *values[7:]) + values[7:] = [timezone] + return datetime.datetime(*values) + + + def toString(self, i): + """ + Serialize a C{datetime.datetime} instance to a string in the specified + wire format. + """ + offset = i.utcoffset() + if offset is None: + raise ValueError( + 'amp.DateTime cannot serialize naive datetime instances. ' + 'You may find amp.utc useful.') + + minutesOffset = (offset.days * 86400 + offset.seconds) // 60 + + if minutesOffset > 0: + sign = '+' + else: + sign = '-' + + # strftime has no way to format the microseconds, or put a ':' in the + # timezone. Surprise! + + # Python 3.4 cannot do % interpolation on byte strings so we pack into + # an explicitly Unicode string then encode as ASCII. + packed = u'%04i-%02i-%02iT%02i:%02i:%02i.%06i%s%02i:%02i' % ( + i.year, + i.month, + i.day, + i.hour, + i.minute, + i.second, + i.microsecond, + sign, + abs(minutesOffset) // 60, + abs(minutesOffset) % 60) + + return packed.encode("ascii") diff --git a/contrib/python/Twisted/py2/twisted/protocols/basic.py b/contrib/python/Twisted/py2/twisted/protocols/basic.py new file mode 100644 index 00000000000..adecfd30ced --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/basic.py @@ -0,0 +1,953 @@ +# -*- test-case-name: twisted.protocols.test.test_basic -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Basic protocols, such as line-oriented, netstring, and int prefixed strings. +""" + +from __future__ import absolute_import, division + +# System imports +import re +from struct import pack, unpack, calcsize +from io import BytesIO +import math + +from zope.interface import implementer + +# Twisted imports +from twisted.python.compat import _PY3 +from twisted.internet import protocol, defer, interfaces +from twisted.python import log + + +# Unfortunately we cannot use regular string formatting on Python 3; see +# http://bugs.python.org/issue3982 for details. +if _PY3: + def _formatNetstring(data): + return b''.join([str(len(data)).encode("ascii"), b':', data, b',']) +else: + def _formatNetstring(data): + return b'%d:%s,' % (len(data), data) +_formatNetstring.__doc__ = """ +Convert some C{bytes} into netstring format. + +@param data: C{bytes} that will be reformatted. +""" + + + +DEBUG = 0 + +class NetstringParseError(ValueError): + """ + The incoming data is not in valid Netstring format. + """ + + + +class IncompleteNetstring(Exception): + """ + Not enough data to complete a netstring. + """ + + +class NetstringReceiver(protocol.Protocol): + """ + A protocol that sends and receives netstrings. + + See U{http://cr.yp.to/proto/netstrings.txt} for the specification of + netstrings. Every netstring starts with digits that specify the length + of the data. This length specification is separated from the data by + a colon. The data is terminated with a comma. + + Override L{stringReceived} to handle received netstrings. This + method is called with the netstring payload as a single argument + whenever a complete netstring is received. + + Security features: + 1. Messages are limited in size, useful if you don't want + someone sending you a 500MB netstring (change C{self.MAX_LENGTH} + to the maximum length you wish to accept). + 2. The connection is lost if an illegal message is received. + + @ivar MAX_LENGTH: Defines the maximum length of netstrings that can be + received. + @type MAX_LENGTH: C{int} + + @ivar _LENGTH: A pattern describing all strings that contain a netstring + length specification. Examples for length specifications are C{b'0:'}, + C{b'12:'}, and C{b'179:'}. C{b'007:'} is not a valid length + specification, since leading zeros are not allowed. + @type _LENGTH: C{re.Match} + + @ivar _LENGTH_PREFIX: A pattern describing all strings that contain + the first part of a netstring length specification (without the + trailing comma). Examples are '0', '12', and '179'. '007' does not + start a netstring length specification, since leading zeros are + not allowed. + @type _LENGTH_PREFIX: C{re.Match} + + @ivar _PARSING_LENGTH: Indicates that the C{NetstringReceiver} is in + the state of parsing the length portion of a netstring. + @type _PARSING_LENGTH: C{int} + + @ivar _PARSING_PAYLOAD: Indicates that the C{NetstringReceiver} is in + the state of parsing the payload portion (data and trailing comma) + of a netstring. + @type _PARSING_PAYLOAD: C{int} + + @ivar brokenPeer: Indicates if the connection is still functional + @type brokenPeer: C{int} + + @ivar _state: Indicates if the protocol is consuming the length portion + (C{PARSING_LENGTH}) or the payload (C{PARSING_PAYLOAD}) of a netstring + @type _state: C{int} + + @ivar _remainingData: Holds the chunk of data that has not yet been consumed + @type _remainingData: C{string} + + @ivar _payload: Holds the payload portion of a netstring including the + trailing comma + @type _payload: C{BytesIO} + + @ivar _expectedPayloadSize: Holds the payload size plus one for the trailing + comma. + @type _expectedPayloadSize: C{int} + """ + MAX_LENGTH = 99999 + _LENGTH = re.compile(br'(0|[1-9]\d*)(:)') + + _LENGTH_PREFIX = re.compile(br'(0|[1-9]\d*)$') + + # Some error information for NetstringParseError instances. + _MISSING_LENGTH = ("The received netstring does not start with a " + "length specification.") + _OVERFLOW = ("The length specification of the received netstring " + "cannot be represented in Python - it causes an " + "OverflowError!") + _TOO_LONG = ("The received netstring is longer than the maximum %s " + "specified by self.MAX_LENGTH") + _MISSING_COMMA = "The received netstring is not terminated by a comma." + + # The following constants are used for determining if the NetstringReceiver + # is parsing the length portion of a netstring, or the payload. + _PARSING_LENGTH, _PARSING_PAYLOAD = range(2) + + def makeConnection(self, transport): + """ + Initializes the protocol. + """ + protocol.Protocol.makeConnection(self, transport) + self._remainingData = b"" + self._currentPayloadSize = 0 + self._payload = BytesIO() + self._state = self._PARSING_LENGTH + self._expectedPayloadSize = 0 + self.brokenPeer = 0 + + + def sendString(self, string): + """ + Sends a netstring. + + Wraps up C{string} by adding length information and a + trailing comma; writes the result to the transport. + + @param string: The string to send. The necessary framing (length + prefix, etc) will be added. + @type string: C{bytes} + """ + self.transport.write(_formatNetstring(string)) + + + def dataReceived(self, data): + """ + Receives some characters of a netstring. + + Whenever a complete netstring is received, this method extracts + its payload and calls L{stringReceived} to process it. + + @param data: A chunk of data representing a (possibly partial) + netstring + @type data: C{bytes} + """ + self._remainingData += data + while self._remainingData: + try: + self._consumeData() + except IncompleteNetstring: + break + except NetstringParseError: + self._handleParseError() + break + + + def stringReceived(self, string): + """ + Override this for notification when each complete string is received. + + @param string: The complete string which was received with all + framing (length prefix, etc) removed. + @type string: C{bytes} + + @raise NotImplementedError: because the method has to be implemented + by the child class. + """ + raise NotImplementedError() + + + def _maxLengthSize(self): + """ + Calculate and return the string size of C{self.MAX_LENGTH}. + + @return: The size of the string representation for C{self.MAX_LENGTH} + @rtype: C{float} + """ + return math.ceil(math.log10(self.MAX_LENGTH)) + 1 + + + def _consumeData(self): + """ + Consumes the content of C{self._remainingData}. + + @raise IncompleteNetstring: if C{self._remainingData} does not + contain enough data to complete the current netstring. + @raise NetstringParseError: if the received data do not + form a valid netstring. + """ + if self._state == self._PARSING_LENGTH: + self._consumeLength() + self._prepareForPayloadConsumption() + if self._state == self._PARSING_PAYLOAD: + self._consumePayload() + + + def _consumeLength(self): + """ + Consumes the length portion of C{self._remainingData}. + + @raise IncompleteNetstring: if C{self._remainingData} contains + a partial length specification (digits without trailing + comma). + @raise NetstringParseError: if the received data do not form a valid + netstring. + """ + lengthMatch = self._LENGTH.match(self._remainingData) + if not lengthMatch: + self._checkPartialLengthSpecification() + raise IncompleteNetstring() + self._processLength(lengthMatch) + + + def _checkPartialLengthSpecification(self): + """ + Makes sure that the received data represents a valid number. + + Checks if C{self._remainingData} represents a number smaller or + equal to C{self.MAX_LENGTH}. + + @raise NetstringParseError: if C{self._remainingData} is no + number or is too big (checked by L{_extractLength}). + """ + partialLengthMatch = self._LENGTH_PREFIX.match(self._remainingData) + if not partialLengthMatch: + raise NetstringParseError(self._MISSING_LENGTH) + lengthSpecification = (partialLengthMatch.group(1)) + self._extractLength(lengthSpecification) + + + def _processLength(self, lengthMatch): + """ + Processes the length definition of a netstring. + + Extracts and stores in C{self._expectedPayloadSize} the number + representing the netstring size. Removes the prefix + representing the length specification from + C{self._remainingData}. + + @raise NetstringParseError: if the received netstring does not + start with a number or the number is bigger than + C{self.MAX_LENGTH}. + @param lengthMatch: A regular expression match object matching + a netstring length specification + @type lengthMatch: C{re.Match} + """ + endOfNumber = lengthMatch.end(1) + startOfData = lengthMatch.end(2) + lengthString = self._remainingData[:endOfNumber] + # Expect payload plus trailing comma: + self._expectedPayloadSize = self._extractLength(lengthString) + 1 + self._remainingData = self._remainingData[startOfData:] + + + def _extractLength(self, lengthAsString): + """ + Attempts to extract the length information of a netstring. + + @raise NetstringParseError: if the number is bigger than + C{self.MAX_LENGTH}. + @param lengthAsString: A chunk of data starting with a length + specification + @type lengthAsString: C{bytes} + @return: The length of the netstring + @rtype: C{int} + """ + self._checkStringSize(lengthAsString) + length = int(lengthAsString) + if length > self.MAX_LENGTH: + raise NetstringParseError(self._TOO_LONG % (self.MAX_LENGTH,)) + return length + + + def _checkStringSize(self, lengthAsString): + """ + Checks the sanity of lengthAsString. + + Checks if the size of the length specification exceeds the + size of the string representing self.MAX_LENGTH. If this is + not the case, the number represented by lengthAsString is + certainly bigger than self.MAX_LENGTH, and a + NetstringParseError can be raised. + + This method should make sure that netstrings with extremely + long length specifications are refused before even attempting + to convert them to an integer (which might trigger a + MemoryError). + """ + if len(lengthAsString) > self._maxLengthSize(): + raise NetstringParseError(self._TOO_LONG % (self.MAX_LENGTH,)) + + + def _prepareForPayloadConsumption(self): + """ + Sets up variables necessary for consuming the payload of a netstring. + """ + self._state = self._PARSING_PAYLOAD + self._currentPayloadSize = 0 + self._payload.seek(0) + self._payload.truncate() + + + def _consumePayload(self): + """ + Consumes the payload portion of C{self._remainingData}. + + If the payload is complete, checks for the trailing comma and + processes the payload. If not, raises an L{IncompleteNetstring} + exception. + + @raise IncompleteNetstring: if the payload received so far + contains fewer characters than expected. + @raise NetstringParseError: if the payload does not end with a + comma. + """ + self._extractPayload() + if self._currentPayloadSize < self._expectedPayloadSize: + raise IncompleteNetstring() + self._checkForTrailingComma() + self._state = self._PARSING_LENGTH + self._processPayload() + + + def _extractPayload(self): + """ + Extracts payload information from C{self._remainingData}. + + Splits C{self._remainingData} at the end of the netstring. The + first part becomes C{self._payload}, the second part is stored + in C{self._remainingData}. + + If the netstring is not yet complete, the whole content of + C{self._remainingData} is moved to C{self._payload}. + """ + if self._payloadComplete(): + remainingPayloadSize = (self._expectedPayloadSize - + self._currentPayloadSize) + self._payload.write(self._remainingData[:remainingPayloadSize]) + self._remainingData = self._remainingData[remainingPayloadSize:] + self._currentPayloadSize = self._expectedPayloadSize + else: + self._payload.write(self._remainingData) + self._currentPayloadSize += len(self._remainingData) + self._remainingData = b"" + + + def _payloadComplete(self): + """ + Checks if enough data have been received to complete the netstring. + + @return: C{True} iff the received data contain at least as many + characters as specified in the length section of the + netstring + @rtype: C{bool} + """ + return (len(self._remainingData) + self._currentPayloadSize >= + self._expectedPayloadSize) + + + def _processPayload(self): + """ + Processes the actual payload with L{stringReceived}. + + Strips C{self._payload} of the trailing comma and calls + L{stringReceived} with the result. + """ + self.stringReceived(self._payload.getvalue()[:-1]) + + + def _checkForTrailingComma(self): + """ + Checks if the netstring has a trailing comma at the expected position. + + @raise NetstringParseError: if the last payload character is + anything but a comma. + """ + if self._payload.getvalue()[-1:] != b",": + raise NetstringParseError(self._MISSING_COMMA) + + + def _handleParseError(self): + """ + Terminates the connection and sets the flag C{self.brokenPeer}. + """ + self.transport.loseConnection() + self.brokenPeer = 1 + + + +class LineOnlyReceiver(protocol.Protocol): + """ + A protocol that receives only lines. + + This is purely a speed optimisation over LineReceiver, for the + cases that raw mode is known to be unnecessary. + + @cvar delimiter: The line-ending delimiter to use. By default this is + C{b'\\r\\n'}. + @cvar MAX_LENGTH: The maximum length of a line to allow (If a + sent line is longer than this, the connection is dropped). + Default is 16384. + """ + _buffer = b'' + delimiter = b'\r\n' + MAX_LENGTH = 16384 + + def dataReceived(self, data): + """ + Translates bytes into lines, and calls lineReceived. + """ + lines = (self._buffer+data).split(self.delimiter) + self._buffer = lines.pop(-1) + for line in lines: + if self.transport.disconnecting: + # this is necessary because the transport may be told to lose + # the connection by a line within a larger packet, and it is + # important to disregard all the lines in that packet following + # the one that told it to close. + return + if len(line) > self.MAX_LENGTH: + return self.lineLengthExceeded(line) + else: + self.lineReceived(line) + if len(self._buffer) > self.MAX_LENGTH: + return self.lineLengthExceeded(self._buffer) + + + def lineReceived(self, line): + """ + Override this for when each line is received. + + @param line: The line which was received with the delimiter removed. + @type line: C{bytes} + """ + raise NotImplementedError + + + def sendLine(self, line): + """ + Sends a line to the other end of the connection. + + @param line: The line to send, not including the delimiter. + @type line: C{bytes} + """ + return self.transport.writeSequence((line, self.delimiter)) + + + def lineLengthExceeded(self, line): + """ + Called when the maximum line length has been reached. + Override if it needs to be dealt with in some special way. + """ + return self.transport.loseConnection() + + + +class _PauseableMixin: + paused = False + + def pauseProducing(self): + self.paused = True + self.transport.pauseProducing() + + + def resumeProducing(self): + self.paused = False + self.transport.resumeProducing() + self.dataReceived(b'') + + + def stopProducing(self): + self.paused = True + self.transport.stopProducing() + + + +class LineReceiver(protocol.Protocol, _PauseableMixin): + """ + A protocol that receives lines and/or raw data, depending on mode. + + In line mode, each line that's received becomes a callback to + L{lineReceived}. In raw data mode, each chunk of raw data becomes a + callback to L{LineReceiver.rawDataReceived}. + The L{setLineMode} and L{setRawMode} methods switch between the two modes. + + This is useful for line-oriented protocols such as IRC, HTTP, POP, etc. + + @cvar delimiter: The line-ending delimiter to use. By default this is + C{b'\\r\\n'}. + @cvar MAX_LENGTH: The maximum length of a line to allow (If a + sent line is longer than this, the connection is dropped). + Default is 16384. + """ + line_mode = 1 + _buffer = b'' + _busyReceiving = False + delimiter = b'\r\n' + MAX_LENGTH = 16384 + + def clearLineBuffer(self): + """ + Clear buffered data. + + @return: All of the cleared buffered data. + @rtype: C{bytes} + """ + b, self._buffer = self._buffer, b"" + return b + + + def dataReceived(self, data): + """ + Protocol.dataReceived. + Translates bytes into lines, and calls lineReceived (or + rawDataReceived, depending on mode.) + """ + if self._busyReceiving: + self._buffer += data + return + + try: + self._busyReceiving = True + self._buffer += data + while self._buffer and not self.paused: + if self.line_mode: + try: + line, self._buffer = self._buffer.split( + self.delimiter, 1) + except ValueError: + if len(self._buffer) >= (self.MAX_LENGTH + + len(self.delimiter)): + line, self._buffer = self._buffer, b'' + return self.lineLengthExceeded(line) + return + else: + lineLength = len(line) + if lineLength > self.MAX_LENGTH: + exceeded = line + self.delimiter + self._buffer + self._buffer = b'' + return self.lineLengthExceeded(exceeded) + why = self.lineReceived(line) + if (why or self.transport and + self.transport.disconnecting): + return why + else: + data = self._buffer + self._buffer = b'' + why = self.rawDataReceived(data) + if why: + return why + finally: + self._busyReceiving = False + + + def setLineMode(self, extra=b''): + """ + Sets the line-mode of this receiver. + + If you are calling this from a rawDataReceived callback, + you can pass in extra unhandled data, and that data will + be parsed for lines. Further data received will be sent + to lineReceived rather than rawDataReceived. + + Do not pass extra data if calling this function from + within a lineReceived callback. + """ + self.line_mode = 1 + if extra: + return self.dataReceived(extra) + + + def setRawMode(self): + """ + Sets the raw mode of this receiver. + Further data received will be sent to rawDataReceived rather + than lineReceived. + """ + self.line_mode = 0 + + + def rawDataReceived(self, data): + """ + Override this for when raw data is received. + """ + raise NotImplementedError + + + def lineReceived(self, line): + """ + Override this for when each line is received. + + @param line: The line which was received with the delimiter removed. + @type line: C{bytes} + """ + raise NotImplementedError + + + def sendLine(self, line): + """ + Sends a line to the other end of the connection. + + @param line: The line to send, not including the delimiter. + @type line: C{bytes} + """ + return self.transport.write(line + self.delimiter) + + + def lineLengthExceeded(self, line): + """ + Called when the maximum line length has been reached. + Override if it needs to be dealt with in some special way. + + The argument 'line' contains the remainder of the buffer, starting + with (at least some part) of the line which is too long. This may + be more than one line, or may be only the initial portion of the + line. + """ + return self.transport.loseConnection() + + + +class StringTooLongError(AssertionError): + """ + Raised when trying to send a string too long for a length prefixed + protocol. + """ + + + +class _RecvdCompatHack(object): + """ + Emulates the to-be-deprecated C{IntNStringReceiver.recvd} attribute. + + The C{recvd} attribute was where the working buffer for buffering and + parsing netstrings was kept. It was updated each time new data arrived and + each time some of that data was parsed and delivered to application code. + The piecemeal updates to its string value were expensive and have been + removed from C{IntNStringReceiver} in the normal case. However, for + applications directly reading this attribute, this descriptor restores that + behavior. It only copies the working buffer when necessary (ie, when + accessed). This avoids the cost for applications not using the data. + + This is a custom descriptor rather than a property, because we still need + the default __set__ behavior in both new-style and old-style subclasses. + """ + def __get__(self, oself, type=None): + return oself._unprocessed[oself._compatibilityOffset:] + + + +class IntNStringReceiver(protocol.Protocol, _PauseableMixin): + """ + Generic class for length prefixed protocols. + + @ivar _unprocessed: bytes received, but not yet broken up into messages / + sent to stringReceived. _compatibilityOffset must be updated when this + value is updated so that the C{recvd} attribute can be generated + correctly. + @type _unprocessed: C{bytes} + + @ivar structFormat: format used for struct packing/unpacking. Define it in + subclass. + @type structFormat: C{str} + + @ivar prefixLength: length of the prefix, in bytes. Define it in subclass, + using C{struct.calcsize(structFormat)} + @type prefixLength: C{int} + + @ivar _compatibilityOffset: the offset within C{_unprocessed} to the next + message to be parsed. (used to generate the recvd attribute) + @type _compatibilityOffset: C{int} + """ + + MAX_LENGTH = 99999 + _unprocessed = b"" + _compatibilityOffset = 0 + + # Backwards compatibility support for applications which directly touch the + # "internal" parse buffer. + recvd = _RecvdCompatHack() + + def stringReceived(self, string): + """ + Override this for notification when each complete string is received. + + @param string: The complete string which was received with all + framing (length prefix, etc) removed. + @type string: C{bytes} + """ + raise NotImplementedError + + + def lengthLimitExceeded(self, length): + """ + Callback invoked when a length prefix greater than C{MAX_LENGTH} is + received. The default implementation disconnects the transport. + Override this. + + @param length: The length prefix which was received. + @type length: C{int} + """ + self.transport.loseConnection() + + + def dataReceived(self, data): + """ + Convert int prefixed strings into calls to stringReceived. + """ + # Try to minimize string copying (via slices) by keeping one buffer + # containing all the data we have so far and a separate offset into that + # buffer. + alldata = self._unprocessed + data + currentOffset = 0 + prefixLength = self.prefixLength + fmt = self.structFormat + self._unprocessed = alldata + + while len(alldata) >= (currentOffset + prefixLength) and not self.paused: + messageStart = currentOffset + prefixLength + length, = unpack(fmt, alldata[currentOffset:messageStart]) + if length > self.MAX_LENGTH: + self._unprocessed = alldata + self._compatibilityOffset = currentOffset + self.lengthLimitExceeded(length) + return + messageEnd = messageStart + length + if len(alldata) < messageEnd: + break + + # Here we have to slice the working buffer so we can send just the + # netstring into the stringReceived callback. + packet = alldata[messageStart:messageEnd] + currentOffset = messageEnd + self._compatibilityOffset = currentOffset + self.stringReceived(packet) + + # Check to see if the backwards compat "recvd" attribute got written + # to by application code. If so, drop the current data buffer and + # switch to the new buffer given by that attribute's value. + if 'recvd' in self.__dict__: + alldata = self.__dict__.pop('recvd') + self._unprocessed = alldata + self._compatibilityOffset = currentOffset = 0 + if alldata: + continue + return + + # Slice off all the data that has been processed, avoiding holding onto + # memory to store it, and update the compatibility attributes to reflect + # that change. + self._unprocessed = alldata[currentOffset:] + self._compatibilityOffset = 0 + + + def sendString(self, string): + """ + Send a prefixed string to the other end of the connection. + + @param string: The string to send. The necessary framing (length + prefix, etc) will be added. + @type string: C{bytes} + """ + if len(string) >= 2 ** (8 * self.prefixLength): + raise StringTooLongError( + "Try to send %s bytes whereas maximum is %s" % ( + len(string), 2 ** (8 * self.prefixLength))) + self.transport.write( + pack(self.structFormat, len(string)) + string) + + + +class Int32StringReceiver(IntNStringReceiver): + """ + A receiver for int32-prefixed strings. + + An int32 string is a string prefixed by 4 bytes, the 32-bit length of + the string encoded in network byte order. + + This class publishes the same interface as NetstringReceiver. + """ + structFormat = "!I" + prefixLength = calcsize(structFormat) + + + +class Int16StringReceiver(IntNStringReceiver): + """ + A receiver for int16-prefixed strings. + + An int16 string is a string prefixed by 2 bytes, the 16-bit length of + the string encoded in network byte order. + + This class publishes the same interface as NetstringReceiver. + """ + structFormat = "!H" + prefixLength = calcsize(structFormat) + + + +class Int8StringReceiver(IntNStringReceiver): + """ + A receiver for int8-prefixed strings. + + An int8 string is a string prefixed by 1 byte, the 8-bit length of + the string. + + This class publishes the same interface as NetstringReceiver. + """ + structFormat = "!B" + prefixLength = calcsize(structFormat) + + + +class StatefulStringProtocol: + """ + A stateful string protocol. + + This is a mixin for string protocols (L{Int32StringReceiver}, + L{NetstringReceiver}) which translates L{stringReceived} into a callback + (prefixed with C{'proto_'}) depending on state. + + The state C{'done'} is special; if a C{proto_*} method returns it, the + connection will be closed immediately. + + @ivar state: Current state of the protocol. Defaults to C{'init'}. + @type state: C{str} + """ + + state = 'init' + + def stringReceived(self, string): + """ + Choose a protocol phase function and call it. + + Call back to the appropriate protocol phase; this begins with + the function C{proto_init} and moves on to C{proto_*} depending on + what each C{proto_*} function returns. (For example, if + C{self.proto_init} returns 'foo', then C{self.proto_foo} will be the + next function called when a protocol message is received. + """ + try: + pto = 'proto_' + self.state + statehandler = getattr(self, pto) + except AttributeError: + log.msg('callback', self.state, 'not found') + else: + self.state = statehandler(string) + if self.state == 'done': + self.transport.loseConnection() + + + +@implementer(interfaces.IProducer) +class FileSender: + """ + A producer that sends the contents of a file to a consumer. + + This is a helper for protocols that, at some point, will take a + file-like object, read its contents, and write them out to the network, + optionally performing some transformation on the bytes in between. + """ + + CHUNK_SIZE = 2 ** 14 + + lastSent = '' + deferred = None + + def beginFileTransfer(self, file, consumer, transform=None): + """ + Begin transferring a file + + @type file: Any file-like object + @param file: The file object to read data from + + @type consumer: Any implementor of IConsumer + @param consumer: The object to write data to + + @param transform: A callable taking one string argument and returning + the same. All bytes read from the file are passed through this before + being written to the consumer. + + @rtype: C{Deferred} + @return: A deferred whose callback will be invoked when the file has + been completely written to the consumer. The last byte written to the + consumer is passed to the callback. + """ + self.file = file + self.consumer = consumer + self.transform = transform + + self.deferred = deferred = defer.Deferred() + self.consumer.registerProducer(self, False) + return deferred + + + def resumeProducing(self): + chunk = '' + if self.file: + chunk = self.file.read(self.CHUNK_SIZE) + if not chunk: + self.file = None + self.consumer.unregisterProducer() + if self.deferred: + self.deferred.callback(self.lastSent) + self.deferred = None + return + + if self.transform: + chunk = self.transform(chunk) + self.consumer.write(chunk) + self.lastSent = chunk[-1:] + + + def pauseProducing(self): + pass + + + def stopProducing(self): + if self.deferred: + self.deferred.errback( + Exception("Consumer asked us to stop producing")) + self.deferred = None diff --git a/contrib/python/Twisted/py2/twisted/protocols/dict.py b/contrib/python/Twisted/py2/twisted/protocols/dict.py new file mode 100644 index 00000000000..d7976411bc6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/dict.py @@ -0,0 +1,415 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Dict client protocol implementation. + +@author: Pavel Pergamenshchik +""" + +from twisted.protocols import basic +from twisted.internet import defer, protocol +from twisted.python import log +from io import BytesIO + +def parseParam(line): + """Chew one dqstring or atom from beginning of line and return (param, remaningline)""" + if line == b'': + return (None, b'') + elif line[0:1] != b'"': # atom + mode = 1 + else: # dqstring + mode = 2 + res = b"" + io = BytesIO(line) + if mode == 2: # skip the opening quote + io.read(1) + while 1: + a = io.read(1) + if a == b'"': + if mode == 2: + io.read(1) # skip the separating space + return (res, io.read()) + elif a == b'\\': + a = io.read(1) + if a == b'': + return (None, line) # unexpected end of string + elif a == b'': + if mode == 1: + return (res, io.read()) + else: + return (None, line) # unexpected end of string + elif a == b' ': + if mode == 1: + return (res, io.read()) + res += a + + + +def makeAtom(line): + """Munch a string into an 'atom'""" + # FIXME: proper quoting + return filter(lambda x: not (x in map(chr, range(33)+[34, 39, 92])), line) + + + +def makeWord(s): + mustquote = range(33)+[34, 39, 92] + result = [] + for c in s: + if ord(c) in mustquote: + result.append(b"\\") + result.append(c) + s = b"".join(result) + return s + + + +def parseText(line): + if len(line) == 1 and line == b'.': + return None + else: + if len(line) > 1 and line[0:2] == b'..': + line = line[1:] + return line + + + +class Definition: + """A word definition""" + def __init__(self, name, db, dbdesc, text): + self.name = name + self.db = db + self.dbdesc = dbdesc + self.text = text # list of strings not terminated by newline + + + +class DictClient(basic.LineReceiver): + """dict (RFC2229) client""" + + data = None # multiline data + MAX_LENGTH = 1024 + state = None + mode = None + result = None + factory = None + + def __init__(self): + self.data = None + self.result = None + + + def connectionMade(self): + self.state = "conn" + self.mode = "command" + + + def sendLine(self, line): + """Throw up if the line is longer than 1022 characters""" + if len(line) > self.MAX_LENGTH - 2: + raise ValueError("DictClient tried to send a too long line") + basic.LineReceiver.sendLine(self, line) + + + def lineReceived(self, line): + try: + line = line.decode("utf-8") + except UnicodeError: # garbage received, skip + return + if self.mode == "text": # we are receiving textual data + code = "text" + else: + if len(line) < 4: + log.msg("DictClient got invalid line from server -- %s" % line) + self.protocolError("Invalid line from server") + self.transport.LoseConnection() + return + code = int(line[:3]) + line = line[4:] + method = getattr(self, 'dictCode_%s_%s' % (code, self.state), self.dictCode_default) + method(line) + + + def dictCode_default(self, line): + """Unknown message""" + log.msg("DictClient got unexpected message from server -- %s" % line) + self.protocolError("Unexpected server message") + self.transport.loseConnection() + + + def dictCode_221_ready(self, line): + """We are about to get kicked off, do nothing""" + pass + + + def dictCode_220_conn(self, line): + """Greeting message""" + self.state = "ready" + self.dictConnected() + + + def dictCode_530_conn(self): + self.protocolError("Access denied") + self.transport.loseConnection() + + + def dictCode_420_conn(self): + self.protocolError("Server temporarily unavailable") + self.transport.loseConnection() + + + def dictCode_421_conn(self): + self.protocolError("Server shutting down at operator request") + self.transport.loseConnection() + + + def sendDefine(self, database, word): + """Send a dict DEFINE command""" + assert self.state == "ready", "DictClient.sendDefine called when not in ready state" + self.result = None # these two are just in case. In "ready" state, result and data + self.data = None # should be None + self.state = "define" + command = "DEFINE %s %s" % (makeAtom(database.encode("UTF-8")), makeWord(word.encode("UTF-8"))) + self.sendLine(command) + + + def sendMatch(self, database, strategy, word): + """Send a dict MATCH command""" + assert self.state == "ready", "DictClient.sendMatch called when not in ready state" + self.result = None + self.data = None + self.state = "match" + command = "MATCH %s %s %s" % (makeAtom(database), makeAtom(strategy), makeAtom(word)) + self.sendLine(command.encode("UTF-8")) + + def dictCode_550_define(self, line): + """Invalid database""" + self.mode = "ready" + self.defineFailed("Invalid database") + + + def dictCode_550_match(self, line): + """Invalid database""" + self.mode = "ready" + self.matchFailed("Invalid database") + + + def dictCode_551_match(self, line): + """Invalid strategy""" + self.mode = "ready" + self.matchFailed("Invalid strategy") + + + def dictCode_552_define(self, line): + """No match""" + self.mode = "ready" + self.defineFailed("No match") + + + def dictCode_552_match(self, line): + """No match""" + self.mode = "ready" + self.matchFailed("No match") + + + def dictCode_150_define(self, line): + """n definitions retrieved""" + self.result = [] + + + def dictCode_151_define(self, line): + """Definition text follows""" + self.mode = "text" + (word, line) = parseParam(line) + (db, line) = parseParam(line) + (dbdesc, line) = parseParam(line) + if not (word and db and dbdesc): + self.protocolError("Invalid server response") + self.transport.loseConnection() + else: + self.result.append(Definition(word, db, dbdesc, [])) + self.data = [] + + + def dictCode_152_match(self, line): + """n matches found, text follows""" + self.mode = "text" + self.result = [] + self.data = [] + + + def dictCode_text_define(self, line): + """A line of definition text received""" + res = parseText(line) + if res == None: + self.mode = "command" + self.result[-1].text = self.data + self.data = None + else: + self.data.append(line) + + + def dictCode_text_match(self, line): + """One line of match text received""" + def l(s): + p1, t = parseParam(s) + p2, t = parseParam(t) + return (p1, p2) + res = parseText(line) + if res == None: + self.mode = "command" + self.result = map(l, self.data) + self.data = None + else: + self.data.append(line) + + + def dictCode_250_define(self, line): + """ok""" + t = self.result + self.result = None + self.state = "ready" + self.defineDone(t) + + + def dictCode_250_match(self, line): + """ok""" + t = self.result + self.result = None + self.state = "ready" + self.matchDone(t) + + + def protocolError(self, reason): + """override to catch unexpected dict protocol conditions""" + pass + + + def dictConnected(self): + """override to be notified when the server is ready to accept commands""" + pass + + + def defineFailed(self, reason): + """override to catch reasonable failure responses to DEFINE""" + pass + + + def defineDone(self, result): + """override to catch successful DEFINE""" + pass + + + def matchFailed(self, reason): + """override to catch resonable failure responses to MATCH""" + pass + + + def matchDone(self, result): + """override to catch successful MATCH""" + pass + + + +class InvalidResponse(Exception): + pass + + + +class DictLookup(DictClient): + """Utility class for a single dict transaction. To be used with DictLookupFactory""" + + def protocolError(self, reason): + if not self.factory.done: + self.factory.d.errback(InvalidResponse(reason)) + self.factory.clientDone() + + + def dictConnected(self): + if self.factory.queryType == "define": + self.sendDefine(*self.factory.param) + elif self.factory.queryType == "match": + self.sendMatch(*self.factory.param) + + + def defineFailed(self, reason): + self.factory.d.callback([]) + self.factory.clientDone() + self.transport.loseConnection() + + + def defineDone(self, result): + self.factory.d.callback(result) + self.factory.clientDone() + self.transport.loseConnection() + + + def matchFailed(self, reason): + self.factory.d.callback([]) + self.factory.clientDone() + self.transport.loseConnection() + + + def matchDone(self, result): + self.factory.d.callback(result) + self.factory.clientDone() + self.transport.loseConnection() + + + +class DictLookupFactory(protocol.ClientFactory): + """Utility factory for a single dict transaction""" + protocol = DictLookup + done = None + + def __init__(self, queryType, param, d): + self.queryType = queryType + self.param = param + self.d = d + self.done = 0 + + + def clientDone(self): + """Called by client when done.""" + self.done = 1 + del self.d + + + def clientConnectionFailed(self, connector, error): + self.d.errback(error) + + + def clientConnectionLost(self, connector, error): + if not self.done: + self.d.errback(error) + + + def buildProtocol(self, addr): + p = self.protocol() + p.factory = self + return p + + + +def define(host, port, database, word): + """Look up a word using a dict server""" + d = defer.Deferred() + factory = DictLookupFactory("define", (database, word), d) + + from twisted.internet import reactor + reactor.connectTCP(host, port, factory) + return d + + + +def match(host, port, database, strategy, word): + """Match a word using a dict server""" + d = defer.Deferred() + factory = DictLookupFactory("match", (database, strategy, word), d) + + from twisted.internet import reactor + reactor.connectTCP(host, port, factory) + return d + diff --git a/contrib/python/Twisted/py2/twisted/protocols/finger.py b/contrib/python/Twisted/py2/twisted/protocols/finger.py new file mode 100644 index 00000000000..101f29b4f0e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/finger.py @@ -0,0 +1,42 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +"""The Finger User Information Protocol (RFC 1288)""" + +from twisted.protocols import basic + +class Finger(basic.LineReceiver): + + def lineReceived(self, line): + parts = line.split() + if not parts: + parts = [b''] + if len(parts) == 1: + slash_w = 0 + else: + slash_w = 1 + user = parts[-1] + if b'@' in user: + hostPlace = user.rfind(b'@') + user = user[:hostPlace] + host = user[hostPlace+1:] + return self.forwardQuery(slash_w, user, host) + if user: + return self.getUser(slash_w, user) + else: + return self.getDomain(slash_w) + + def _refuseMessage(self, message): + self.transport.write(message + b"\n") + self.transport.loseConnection() + + def forwardQuery(self, slash_w, user, host): + self._refuseMessage(b'Finger forwarding service denied') + + def getDomain(self, slash_w): + self._refuseMessage(b'Finger online list denied') + + def getUser(self, slash_w, user): + self.transport.write(b'Login: ' + user + b'\n') + self._refuseMessage(b'No such user') diff --git a/contrib/python/Twisted/py2/twisted/protocols/ftp.py b/contrib/python/Twisted/py2/twisted/protocols/ftp.py new file mode 100644 index 00000000000..0c7171a0701 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/ftp.py @@ -0,0 +1,3374 @@ +# -*- test-case-name: twisted.test.test_ftp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An FTP protocol implementation +""" + +# System Imports +import os +import time +import re +import stat +import errno +import fnmatch + +try: + import pwd + import grp +except ImportError: + pwd = grp = None + +from zope.interface import Interface, implementer + +# Twisted Imports +from twisted import copyright +from twisted.internet import reactor, interfaces, protocol, error, defer +from twisted.protocols import basic, policies + +from twisted.python import log, failure, filepath +from twisted.python.compat import range, unicode +from twisted.cred import error as cred_error, portal, credentials, checkers + +# constants +# response codes + +RESTART_MARKER_REPLY = "100" +SERVICE_READY_IN_N_MINUTES = "120" +DATA_CNX_ALREADY_OPEN_START_XFR = "125" +FILE_STATUS_OK_OPEN_DATA_CNX = "150" + +CMD_OK = "200.1" +TYPE_SET_OK = "200.2" +ENTERING_PORT_MODE = "200.3" +CMD_NOT_IMPLMNTD_SUPERFLUOUS = "202" +SYS_STATUS_OR_HELP_REPLY = "211.1" +FEAT_OK = '211.2' +DIR_STATUS = "212" +FILE_STATUS = "213" +HELP_MSG = "214" +NAME_SYS_TYPE = "215" +SVC_READY_FOR_NEW_USER = "220.1" +WELCOME_MSG = "220.2" +SVC_CLOSING_CTRL_CNX = "221.1" +GOODBYE_MSG = "221.2" +DATA_CNX_OPEN_NO_XFR_IN_PROGRESS = "225" +CLOSING_DATA_CNX = "226.1" +TXFR_COMPLETE_OK = "226.2" +ENTERING_PASV_MODE = "227" +ENTERING_EPSV_MODE = "229" +USR_LOGGED_IN_PROCEED = "230.1" # v1 of code 230 +GUEST_LOGGED_IN_PROCEED = "230.2" # v2 of code 230 +REQ_FILE_ACTN_COMPLETED_OK = "250" +PWD_REPLY = "257.1" +MKD_REPLY = "257.2" + +USR_NAME_OK_NEED_PASS = "331.1" # v1 of Code 331 +GUEST_NAME_OK_NEED_EMAIL = "331.2" # v2 of code 331 +NEED_ACCT_FOR_LOGIN = "332" +REQ_FILE_ACTN_PENDING_FURTHER_INFO = "350" + +SVC_NOT_AVAIL_CLOSING_CTRL_CNX = "421.1" +TOO_MANY_CONNECTIONS = "421.2" +CANT_OPEN_DATA_CNX = "425" +CNX_CLOSED_TXFR_ABORTED = "426" +REQ_ACTN_ABRTD_FILE_UNAVAIL = "450" +REQ_ACTN_ABRTD_LOCAL_ERR = "451" +REQ_ACTN_ABRTD_INSUFF_STORAGE = "452" + +SYNTAX_ERR = "500" +SYNTAX_ERR_IN_ARGS = "501" +CMD_NOT_IMPLMNTD = "502.1" +OPTS_NOT_IMPLEMENTED = '502.2' +BAD_CMD_SEQ = "503" +CMD_NOT_IMPLMNTD_FOR_PARAM = "504" +NOT_LOGGED_IN = "530.1" # v1 of code 530 - please log in +AUTH_FAILURE = "530.2" # v2 of code 530 - authorization failure +NEED_ACCT_FOR_STOR = "532" +FILE_NOT_FOUND = "550.1" # no such file or directory +PERMISSION_DENIED = "550.2" # permission denied +ANON_USER_DENIED = "550.3" # anonymous users can't alter filesystem +IS_NOT_A_DIR = "550.4" # rmd called on a path that is not a directory +REQ_ACTN_NOT_TAKEN = "550.5" +FILE_EXISTS = "550.6" +IS_A_DIR = "550.7" +PAGE_TYPE_UNK = "551" +EXCEEDED_STORAGE_ALLOC = "552" +FILENAME_NOT_ALLOWED = "553" + + +RESPONSE = { + # -- 100's -- + # TODO: this must be fixed + RESTART_MARKER_REPLY: '110 MARK yyyy-mmmm', + SERVICE_READY_IN_N_MINUTES: '120 service ready in %s minutes', + DATA_CNX_ALREADY_OPEN_START_XFR: '125 Data connection already open, ' + 'starting transfer', + FILE_STATUS_OK_OPEN_DATA_CNX: '150 File status okay; about to open ' + 'data connection.', + + # -- 200's -- + CMD_OK: '200 Command OK', + TYPE_SET_OK: '200 Type set to %s.', + ENTERING_PORT_MODE: '200 PORT OK', + CMD_NOT_IMPLMNTD_SUPERFLUOUS: '202 Command not implemented, ' + 'superfluous at this site', + SYS_STATUS_OR_HELP_REPLY: '211 System status reply', + FEAT_OK: ['211-Features:', '211 End'], + DIR_STATUS: '212 %s', + FILE_STATUS: '213 %s', + HELP_MSG: '214 help: %s', + NAME_SYS_TYPE: '215 UNIX Type: L8', + WELCOME_MSG: "220 %s", + SVC_READY_FOR_NEW_USER: '220 Service ready', + SVC_CLOSING_CTRL_CNX: '221 Service closing control ' + 'connection', + GOODBYE_MSG: '221 Goodbye.', + DATA_CNX_OPEN_NO_XFR_IN_PROGRESS: '225 data connection open, no ' + 'transfer in progress', + CLOSING_DATA_CNX: '226 Abort successful', + TXFR_COMPLETE_OK: '226 Transfer Complete.', + ENTERING_PASV_MODE: '227 Entering Passive Mode (%s).', + # Where is EPSV defined in the RFCs? + ENTERING_EPSV_MODE: '229 Entering Extended Passive Mode ' + '(|||%s|).', + USR_LOGGED_IN_PROCEED: '230 User logged in, proceed', + GUEST_LOGGED_IN_PROCEED: '230 Anonymous login ok, access ' + 'restrictions apply.', + # i.e. CWD completed OK + REQ_FILE_ACTN_COMPLETED_OK: '250 Requested File Action Completed ' + 'OK', + PWD_REPLY: '257 "%s"', + MKD_REPLY: '257 "%s" created', + + # -- 300's -- + USR_NAME_OK_NEED_PASS: '331 Password required for %s.', + GUEST_NAME_OK_NEED_EMAIL: '331 Guest login ok, type your email ' + 'address as password.', + NEED_ACCT_FOR_LOGIN: '332 Need account for login.', + + REQ_FILE_ACTN_PENDING_FURTHER_INFO: '350 Requested file action pending ' + 'further information.', + + # -- 400's -- + SVC_NOT_AVAIL_CLOSING_CTRL_CNX: '421 Service not available, closing ' + 'control connection.', + TOO_MANY_CONNECTIONS: '421 Too many users right now, try ' + 'again in a few minutes.', + CANT_OPEN_DATA_CNX: "425 Can't open data connection.", + CNX_CLOSED_TXFR_ABORTED: '426 Transfer aborted. Data ' + 'connection closed.', + + REQ_ACTN_ABRTD_FILE_UNAVAIL: '450 Requested action aborted. ' + 'File unavailable.', + REQ_ACTN_ABRTD_LOCAL_ERR: '451 Requested action aborted. ' + 'Local error in processing.', + REQ_ACTN_ABRTD_INSUFF_STORAGE: '452 Requested action aborted. ' + 'Insufficient storage.', + + # -- 500's -- + SYNTAX_ERR: "500 Syntax error: %s", + SYNTAX_ERR_IN_ARGS: '501 syntax error in argument(s) %s.', + CMD_NOT_IMPLMNTD: "502 Command '%s' not implemented", + OPTS_NOT_IMPLEMENTED: "502 Option '%s' not implemented.", + BAD_CMD_SEQ: '503 Incorrect sequence of commands: ' + '%s', + CMD_NOT_IMPLMNTD_FOR_PARAM: "504 Not implemented for parameter " + "'%s'.", + NOT_LOGGED_IN: '530 Please login with USER and PASS.', + AUTH_FAILURE: '530 Sorry, Authentication failed.', + NEED_ACCT_FOR_STOR: '532 Need an account for storing ' + 'files', + FILE_NOT_FOUND: '550 %s: No such file or directory.', + PERMISSION_DENIED: '550 %s: Permission denied.', + ANON_USER_DENIED: '550 Anonymous users are forbidden to ' + 'change the filesystem', + IS_NOT_A_DIR: '550 Cannot rmd, %s is not a ' + 'directory', + FILE_EXISTS: '550 %s: File exists', + IS_A_DIR: '550 %s: is a directory', + REQ_ACTN_NOT_TAKEN: '550 Requested action not taken: %s', + PAGE_TYPE_UNK: '551 Page type unknown', + EXCEEDED_STORAGE_ALLOC: '552 Requested file action aborted, ' + 'exceeded file storage allocation', + FILENAME_NOT_ALLOWED: '553 Requested action not taken, file ' + 'name not allowed' +} + + + +class InvalidPath(Exception): + """ + Internal exception used to signify an error during parsing a path. + """ + + + +def toSegments(cwd, path): + """ + Normalize a path, as represented by a list of strings each + representing one segment of the path. + """ + if path.startswith('/'): + segs = [] + else: + segs = cwd[:] + + for s in path.split('/'): + if s == '.' or s == '': + continue + elif s == '..': + if segs: + segs.pop() + else: + raise InvalidPath(cwd, path) + elif '\0' in s or '/' in s: + raise InvalidPath(cwd, path) + else: + segs.append(s) + return segs + + + +def errnoToFailure(e, path): + """ + Map C{OSError} and C{IOError} to standard FTP errors. + """ + if e == errno.ENOENT: + return defer.fail(FileNotFoundError(path)) + elif e == errno.EACCES or e == errno.EPERM: + return defer.fail(PermissionDeniedError(path)) + elif e == errno.ENOTDIR: + return defer.fail(IsNotADirectoryError(path)) + elif e == errno.EEXIST: + return defer.fail(FileExistsError(path)) + elif e == errno.EISDIR: + return defer.fail(IsADirectoryError(path)) + else: + return defer.fail() + + + +_testTranslation = fnmatch.translate('TEST') + + + +def _isGlobbingExpression(segments=None): + """ + Helper for checking if a FTPShell `segments` contains a wildcard Unix + expression. + + Only filename globbing is supported. + This means that wildcards can only be presents in the last element of + `segments`. + + @type segments: C{list} + @param segments: List of path elements as used by the FTP server protocol. + + @rtype: Boolean + @return: True if `segments` contains a globbing expression. + """ + if not segments: + return False + + # To check that something is a glob expression, we convert it to + # Regular Expression. + # We compare it to the translation of a known non-glob expression. + # If the result is the same as the original expression then it contains no + # globbing expression. + globCandidate = segments[-1] + globTranslations = fnmatch.translate(globCandidate) + nonGlobTranslations = _testTranslation.replace('TEST', globCandidate, 1) + + if nonGlobTranslations == globTranslations: + return False + else: + return True + + + +class FTPCmdError(Exception): + """ + Generic exception for FTP commands. + """ + def __init__(self, *msg): + Exception.__init__(self, *msg) + self.errorMessage = msg + + + def response(self): + """ + Generate a FTP response message for this error. + """ + return RESPONSE[self.errorCode] % self.errorMessage + + + +class FileNotFoundError(FTPCmdError): + """ + Raised when trying to access a non existent file or directory. + """ + errorCode = FILE_NOT_FOUND + + + +class AnonUserDeniedError(FTPCmdError): + """ + Raised when an anonymous user issues a command that will alter the + filesystem + """ + + errorCode = ANON_USER_DENIED + + + +class PermissionDeniedError(FTPCmdError): + """ + Raised when access is attempted to a resource to which access is + not allowed. + """ + errorCode = PERMISSION_DENIED + + + +class IsNotADirectoryError(FTPCmdError): + """ + Raised when RMD is called on a path that isn't a directory. + """ + errorCode = IS_NOT_A_DIR + + + +class FileExistsError(FTPCmdError): + """ + Raised when attempted to override an existing resource. + """ + errorCode = FILE_EXISTS + + + +class IsADirectoryError(FTPCmdError): + """ + Raised when DELE is called on a path that is a directory. + """ + errorCode = IS_A_DIR + + + +class CmdSyntaxError(FTPCmdError): + """ + Raised when a command syntax is wrong. + """ + errorCode = SYNTAX_ERR + + + +class CmdArgSyntaxError(FTPCmdError): + """ + Raised when a command is called with wrong value or a wrong number of + arguments. + """ + errorCode = SYNTAX_ERR_IN_ARGS + + + +class CmdNotImplementedError(FTPCmdError): + """ + Raised when an unimplemented command is given to the server. + """ + errorCode = CMD_NOT_IMPLMNTD + + + +class CmdNotImplementedForArgError(FTPCmdError): + """ + Raised when the handling of a parameter for a command is not implemented by + the server. + """ + errorCode = CMD_NOT_IMPLMNTD_FOR_PARAM + + + +class FTPError(Exception): + pass + + + +class PortConnectionError(Exception): + pass + + + +class BadCmdSequenceError(FTPCmdError): + """ + Raised when a client sends a series of commands in an illogical sequence. + """ + errorCode = BAD_CMD_SEQ + + + +class AuthorizationError(FTPCmdError): + """ + Raised when client authentication fails. + """ + errorCode = AUTH_FAILURE + + + +def debugDeferred(self, *_): + log.msg('debugDeferred(): %s' % str(_), debug=True) + + + +# -- DTP Protocol -- + + +_months = [ + None, + 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + + + +@implementer(interfaces.IConsumer) +class DTP(protocol.Protocol, object): + isConnected = False + + _cons = None + _onConnLost = None + _buffer = None + _encoding = 'latin-1' + + def connectionMade(self): + self.isConnected = True + self.factory.deferred.callback(None) + self._buffer = [] + + def connectionLost(self, reason): + self.isConnected = False + if self._onConnLost is not None: + self._onConnLost.callback(None) + + def sendLine(self, line): + """ + Send a line to data channel. + + @param line: The line to be sent. + @type line: L{bytes} + """ + self.transport.write(line + b'\r\n') + + + def _formatOneListResponse(self, name, size, directory, permissions, + hardlinks, modified, owner, group): + """ + Helper method to format one entry's info into a text entry like: + 'drwxrwxrwx 0 user group 0 Jan 01 1970 filename.txt' + + @param name: C{bytes} name of the entry (file or directory or link) + @param size: C{int} size of the entry + @param directory: evals to C{bool} - whether the entry is a directory + @param permissions: L{twisted.python.filepath.Permissions} object + representing that entry's permissions + @param hardlinks: C{int} number of hardlinks + @param modified: C{float} - entry's last modified time in seconds + since the epoch + @param owner: C{str} username of the owner + @param group: C{str} group name of the owner + + @return: C{str} in the requisite format + """ + def formatDate(mtime): + now = time.gmtime() + info = { + 'month': _months[mtime.tm_mon], + 'day': mtime.tm_mday, + 'year': mtime.tm_year, + 'hour': mtime.tm_hour, + 'minute': mtime.tm_min + } + if now.tm_year != mtime.tm_year: + return '%(month)s %(day)02d %(year)5d' % info + else: + return '%(month)s %(day)02d %(hour)02d:%(minute)02d' % info + + format = ('%(directory)s%(permissions)s%(hardlinks)4d ' + '%(owner)-9s %(group)-9s %(size)15d %(date)12s ' + ) + + msg = (format % { + 'directory': directory and 'd' or '-', + 'permissions': permissions.shorthand(), + 'hardlinks': hardlinks, + 'owner': owner[:8], + 'group': group[:8], + 'size': size, + 'date': formatDate(time.gmtime(modified)), + }).encode(self._encoding) + return msg + name + + + def sendListResponse(self, name, response): + self.sendLine(self._formatOneListResponse(name, *response)) + + # Proxy IConsumer to our transport + def registerProducer(self, producer, streaming): + return self.transport.registerProducer(producer, streaming) + + def unregisterProducer(self): + self.transport.unregisterProducer() + self.transport.loseConnection() + + def write(self, data): + if self.isConnected: + return self.transport.write(data) + raise Exception("Crap damn crap damn crap damn") + + + # Pretend to be a producer, too. + def _conswrite(self, bytes): + try: + self._cons.write(bytes) + except: + self._onConnLost.errback() + + def dataReceived(self, bytes): + if self._cons is not None: + self._conswrite(bytes) + else: + self._buffer.append(bytes) + + def _unregConsumer(self, ignored): + self._cons.unregisterProducer() + self._cons = None + del self._onConnLost + return ignored + + def registerConsumer(self, cons): + assert self._cons is None + self._cons = cons + self._cons.registerProducer(self, True) + for chunk in self._buffer: + self._conswrite(chunk) + self._buffer = None + if self.isConnected: + self._onConnLost = d = defer.Deferred() + d.addBoth(self._unregConsumer) + return d + else: + self._cons.unregisterProducer() + self._cons = None + return defer.succeed(None) + + def resumeProducing(self): + self.transport.resumeProducing() + + def pauseProducing(self): + self.transport.pauseProducing() + + def stopProducing(self): + self.transport.stopProducing() + + + +class DTPFactory(protocol.ClientFactory): + """ + Client factory for I{data transfer process} protocols. + + @ivar peerCheck: perform checks to make sure the ftp-pi's peer is the same + as the dtp's + @ivar pi: a reference to this factory's protocol interpreter + + @ivar _state: Indicates the current state of the DTPFactory. Initially, + this is L{_IN_PROGRESS}. If the connection fails or times out, it is + L{_FAILED}. If the connection succeeds before the timeout, it is + L{_FINISHED}. + + @cvar _IN_PROGRESS: Token to signal that connection is active. + @type _IN_PROGRESS: L{object}. + + @cvar _FAILED: Token to signal that connection has failed. + @type _FAILED: L{object}. + + @cvar _FINISHED: Token to signal that connection was successfully closed. + @type _FINISHED: L{object}. + """ + + _IN_PROGRESS = object() + _FAILED = object() + _FINISHED = object() + + _state = _IN_PROGRESS + + # -- configuration variables -- + peerCheck = False + + # -- class variables -- + def __init__(self, pi, peerHost=None, reactor=None): + """ + Constructor + + @param pi: this factory's protocol interpreter + @param peerHost: if peerCheck is True, this is the tuple that the + generated instance will use to perform security checks + """ + self.pi = pi + self.peerHost = peerHost # from FTP.transport.peerHost() + # deferred will fire when instance is connected + self.deferred = defer.Deferred() + self.delayedCall = None + if reactor is None: + from twisted.internet import reactor + self._reactor = reactor + + + def buildProtocol(self, addr): + log.msg('DTPFactory.buildProtocol', debug=True) + + if self._state is not self._IN_PROGRESS: + return None + self._state = self._FINISHED + + self.cancelTimeout() + p = DTP() + p.factory = self + p.pi = self.pi + self.pi.dtpInstance = p + return p + + + def stopFactory(self): + log.msg('dtpFactory.stopFactory', debug=True) + self.cancelTimeout() + + + def timeoutFactory(self): + log.msg('timed out waiting for DTP connection') + if self._state is not self._IN_PROGRESS: + return + self._state = self._FAILED + + d = self.deferred + self.deferred = None + d.errback( + PortConnectionError(defer.TimeoutError("DTPFactory timeout"))) + + + def cancelTimeout(self): + if self.delayedCall is not None and self.delayedCall.active(): + log.msg('cancelling DTP timeout', debug=True) + self.delayedCall.cancel() + + + def setTimeout(self, seconds): + log.msg('DTPFactory.setTimeout set to %s seconds' % seconds) + self.delayedCall = self._reactor.callLater( + seconds, self.timeoutFactory) + + + def clientConnectionFailed(self, connector, reason): + if self._state is not self._IN_PROGRESS: + return + self._state = self._FAILED + d = self.deferred + self.deferred = None + d.errback(PortConnectionError(reason)) + + + +# -- FTP-PI (Protocol Interpreter) -- + +class ASCIIConsumerWrapper(object): + def __init__(self, cons): + self.cons = cons + self.registerProducer = cons.registerProducer + self.unregisterProducer = cons.unregisterProducer + + assert os.linesep == "\r\n" or len(os.linesep) == 1, ( + "Unsupported platform (yea right like this even exists)") + + if os.linesep == "\r\n": + self.write = cons.write + + def write(self, bytes): + return self.cons.write(bytes.replace(os.linesep, "\r\n")) + + + +@implementer(interfaces.IConsumer) +class FileConsumer(object): + """ + A consumer for FTP input that writes data to a file. + + @ivar fObj: a file object opened for writing, used to write data received. + @type fObj: C{file} + """ + def __init__(self, fObj): + self.fObj = fObj + + + def registerProducer(self, producer, streaming): + self.producer = producer + assert streaming + + + def unregisterProducer(self): + self.producer = None + self.fObj.close() + + + def write(self, bytes): + self.fObj.write(bytes) + + + +class FTPOverflowProtocol(basic.LineReceiver): + """FTP mini-protocol for when there are too many connections.""" + _encoding = 'latin-1' + + def connectionMade(self): + self.sendLine(RESPONSE[TOO_MANY_CONNECTIONS].encode(self._encoding)) + self.transport.loseConnection() + + + +class FTP(basic.LineReceiver, policies.TimeoutMixin, object): + """ + Protocol Interpreter for the File Transfer Protocol + + @ivar state: The current server state. One of L{UNAUTH}, + L{INAUTH}, L{AUTHED}, L{RENAMING}. + + @ivar shell: The connected avatar + @ivar binary: The transfer mode. If false, ASCII. + @ivar dtpFactory: Generates a single DTP for this session + @ivar dtpPort: Port returned from listenTCP + @ivar listenFactory: A callable with the signature of + L{twisted.internet.interfaces.IReactorTCP.listenTCP} which will be used + to create Ports for passive connections (mainly for testing). + + @ivar passivePortRange: iterator used as source of passive port numbers. + @type passivePortRange: C{iterator} + + @cvar UNAUTH: Command channel is not yet authenticated. + @type UNAUTH: L{int} + + @cvar INAUTH: Command channel is in the process of being authenticated. + @type INAUTH: L{int} + + @cvar AUTHED: Command channel was successfully authenticated. + @type AUTHED: L{int} + + @cvar RENAMING: Command channel is between the renaming command sequence. + @type RENAMING: L{int} + """ + + disconnected = False + + # States an FTP can be in + UNAUTH, INAUTH, AUTHED, RENAMING = range(4) + + # how long the DTP waits for a connection + dtpTimeout = 10 + + portal = None + shell = None + dtpFactory = None + dtpPort = None + dtpInstance = None + binary = True + PUBLIC_COMMANDS = ['FEAT', 'QUIT'] + FEATURES = ['FEAT', 'MDTM', 'PASV', 'SIZE', 'TYPE A;I'] + + passivePortRange = range(0, 1) + + listenFactory = reactor.listenTCP + _encoding = 'latin-1' + + def reply(self, key, *args): + msg = RESPONSE[key] % args + self.sendLine(msg) + + + def sendLine(self, line): + """ + (Private) Encodes and sends a line + + @param line: L{bytes} or L{unicode} + """ + if isinstance(line, unicode): + line = line.encode(self._encoding) + super(FTP, self).sendLine(line) + + + def connectionMade(self): + self.state = self.UNAUTH + self.setTimeout(self.timeOut) + self.reply(WELCOME_MSG, self.factory.welcomeMessage) + + def connectionLost(self, reason): + # if we have a DTP protocol instance running and + # we lose connection to the client's PI, kill the + # DTP connection and close the port + if self.dtpFactory: + self.cleanupDTP() + self.setTimeout(None) + if hasattr(self.shell, 'logout') and self.shell.logout is not None: + self.shell.logout() + self.shell = None + self.transport = None + + def timeoutConnection(self): + self.transport.loseConnection() + + def lineReceived(self, line): + self.resetTimeout() + self.pauseProducing() + if bytes != str: + line = line.decode(self._encoding) + + def processFailed(err): + if err.check(FTPCmdError): + self.sendLine(err.value.response()) + elif (err.check(TypeError) and any(( + msg in err.value.args[0] for msg in ( + 'takes exactly', 'required positional argument')))): + self.reply(SYNTAX_ERR, "%s requires an argument." % (cmd,)) + else: + log.msg("Unexpected FTP error") + log.err(err) + self.reply(REQ_ACTN_NOT_TAKEN, "internal server error") + + def processSucceeded(result): + if isinstance(result, tuple): + self.reply(*result) + elif result is not None: + self.reply(result) + + def allDone(ignored): + if not self.disconnected: + self.resumeProducing() + + spaceIndex = line.find(' ') + if spaceIndex != -1: + cmd = line[:spaceIndex] + args = (line[spaceIndex + 1:],) + else: + cmd = line + args = () + d = defer.maybeDeferred(self.processCommand, cmd, *args) + d.addCallbacks(processSucceeded, processFailed) + d.addErrback(log.err) + + # XXX It burnsss + # LineReceiver doesn't let you resumeProducing inside + # lineReceived atm + from twisted.internet import reactor + reactor.callLater(0, d.addBoth, allDone) + + + def processCommand(self, cmd, *params): + + def call_ftp_command(command): + method = getattr(self, "ftp_" + command, None) + if method is not None: + return method(*params) + return defer.fail(CmdNotImplementedError(command)) + + cmd = cmd.upper() + + if cmd in self.PUBLIC_COMMANDS: + return call_ftp_command(cmd) + + elif self.state == self.UNAUTH: + if cmd == 'USER': + return self.ftp_USER(*params) + elif cmd == 'PASS': + return BAD_CMD_SEQ, "USER required before PASS" + else: + return NOT_LOGGED_IN + + elif self.state == self.INAUTH: + if cmd == 'PASS': + return self.ftp_PASS(*params) + else: + return BAD_CMD_SEQ, "PASS required after USER" + + elif self.state == self.AUTHED: + return call_ftp_command(cmd) + + elif self.state == self.RENAMING: + if cmd == 'RNTO': + return self.ftp_RNTO(*params) + else: + return BAD_CMD_SEQ, "RNTO required after RNFR" + + + def getDTPPort(self, factory): + """ + Return a port for passive access, using C{self.passivePortRange} + attribute. + """ + for portn in self.passivePortRange: + try: + dtpPort = self.listenFactory(portn, factory) + except error.CannotListenError: + continue + else: + return dtpPort + raise error.CannotListenError( + '', portn, + "No port available in range %s" % (self.passivePortRange,)) + + + def ftp_USER(self, username): + """ + First part of login. Get the username the peer wants to + authenticate as. + """ + if not username: + return defer.fail(CmdSyntaxError('USER requires an argument')) + + self._user = username + self.state = self.INAUTH + if (self.factory.allowAnonymous and + self._user == self.factory.userAnonymous): + return GUEST_NAME_OK_NEED_EMAIL + else: + return (USR_NAME_OK_NEED_PASS, username) + + # TODO: add max auth try before timeout from ip... + # TODO: need to implement minimal ABOR command + + def ftp_PASS(self, password): + """ + Second part of login. Get the password the peer wants to + authenticate with. + """ + if (self.factory.allowAnonymous and + self._user == self.factory.userAnonymous): + # anonymous login + creds = credentials.Anonymous() + reply = GUEST_LOGGED_IN_PROCEED + else: + # user login + creds = credentials.UsernamePassword(self._user, password) + reply = USR_LOGGED_IN_PROCEED + del self._user + + def _cbLogin(result): + (interface, avatar, logout) = result + assert interface is IFTPShell, "The realm is busted, jerk." + self.shell = avatar + self.logout = logout + self.workingDirectory = [] + self.state = self.AUTHED + return reply + + def _ebLogin(failure): + failure.trap( + cred_error.UnauthorizedLogin, cred_error.UnhandledCredentials) + self.state = self.UNAUTH + raise AuthorizationError + + d = self.portal.login(creds, None, IFTPShell) + d.addCallbacks(_cbLogin, _ebLogin) + return d + + + def ftp_PASV(self): + """ + Request for a passive connection + + from the rfc:: + + This command requests the server-DTP to \"listen\" on a data port + (which is not its default data port) and to wait for a connection + rather than initiate one upon receipt of a transfer command. The + response to this command includes the host and port address this + server is listening on. + """ + # if we have a DTP port set up, lose it. + if self.dtpFactory is not None: + # cleanupDTP sets dtpFactory to none. Later we'll do + # cleanup here or something. + self.cleanupDTP() + self.dtpFactory = DTPFactory(pi=self) + self.dtpFactory.setTimeout(self.dtpTimeout) + self.dtpPort = self.getDTPPort(self.dtpFactory) + + host = self.transport.getHost().host + port = self.dtpPort.getHost().port + self.reply(ENTERING_PASV_MODE, encodeHostPort(host, port)) + return self.dtpFactory.deferred.addCallback(lambda ign: None) + + + def ftp_PORT(self, address): + addr = tuple(map(int, address.split(','))) + ip = '%d.%d.%d.%d' % tuple(addr[:4]) + port = addr[4] << 8 | addr[5] + + # if we have a DTP port set up, lose it. + if self.dtpFactory is not None: + self.cleanupDTP() + + self.dtpFactory = DTPFactory( + pi=self, peerHost=self.transport.getPeer().host) + self.dtpFactory.setTimeout(self.dtpTimeout) + self.dtpPort = reactor.connectTCP(ip, port, self.dtpFactory) + + def connected(ignored): + return ENTERING_PORT_MODE + + def connFailed(err): + err.trap(PortConnectionError) + return CANT_OPEN_DATA_CNX + + return self.dtpFactory.deferred.addCallbacks(connected, connFailed) + + + def _encodeName(self, name): + """ + Encode C{name} to be sent over the wire. + + This encodes L{unicode} objects as UTF-8 and leaves L{bytes} as-is. + + As described by U{RFC 3659 section + 2.2}:: + + Various FTP commands take pathnames as arguments, or return + pathnames in responses. When the MLST command is supported, as + indicated in the response to the FEAT command, pathnames are to be + transferred in one of the following two formats. + + pathname = utf-8-name / raw + utf-8-name = + raw = + + Which format is used is at the option of the user-PI or server-PI + sending the pathname. + + @param name: Name to be encoded. + @type name: L{bytes} or L{unicode} + + @return: Wire format of C{name}. + @rtype: L{bytes} + """ + if isinstance(name, unicode): + return name.encode('utf-8') + return name + + + def ftp_LIST(self, path=''): + """ This command causes a list to be sent from the server to the + passive DTP. If the pathname specifies a directory or other + group of files, the server should transfer a list of files + in the specified directory. If the pathname specifies a + file then the server should send current information on the + file. A null argument implies the user's current working or + default directory. + """ + # XXX: why is this check different from ftp_RETR/ftp_STOR? See #4180 + if self.dtpInstance is None or not self.dtpInstance.isConnected: + return defer.fail( + BadCmdSequenceError('must send PORT or PASV before RETR')) + + # Various clients send flags like -L or -al etc. We just ignore them. + if path.lower() in ['-a', '-l', '-la', '-al']: + path = '' + + def gotListing(results): + self.reply(DATA_CNX_ALREADY_OPEN_START_XFR) + for (name, attrs) in results: + name = self._encodeName(name) + self.dtpInstance.sendListResponse(name, attrs) + self.dtpInstance.transport.loseConnection() + return (TXFR_COMPLETE_OK,) + + try: + segments = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + + d = self.shell.list( + segments, + ('size', 'directory', 'permissions', 'hardlinks', + 'modified', 'owner', 'group')) + d.addCallback(gotListing) + return d + + + def ftp_NLST(self, path): + """ + This command causes a directory listing to be sent from the server to + the client. The pathname should specify a directory or other + system-specific file group descriptor. An empty path implies the + current working directory. If the path is non-existent, send nothing. + If the path is to a file, send only the file name. + + @type path: C{str} + @param path: The path for which a directory listing should be returned. + + @rtype: L{Deferred} + @return: a L{Deferred} which will be fired when the listing request + is finished. + """ + # XXX: why is this check different from ftp_RETR/ftp_STOR? See #4180 + if self.dtpInstance is None or not self.dtpInstance.isConnected: + return defer.fail( + BadCmdSequenceError('must send PORT or PASV before RETR')) + + try: + segments = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + + def cbList(results, glob): + """ + Send, line by line, each matching file in the directory listing, + and then close the connection. + + @type results: A C{list} of C{tuple}. The first element of each + C{tuple} is a C{str} and the second element is a C{list}. + @param results: The names of the files in the directory. + + @param glob: A shell-style glob through which to filter results + (see U{http://docs.python.org/2/library/fnmatch.html}), or + L{None} for no filtering. + @type glob: L{str} or L{None} + + @return: A C{tuple} containing the status code for a successful + transfer. + @rtype: C{tuple} + """ + self.reply(DATA_CNX_ALREADY_OPEN_START_XFR) + for (name, ignored) in results: + if not glob or (glob and fnmatch.fnmatch(name, glob)): + name = self._encodeName(name) + self.dtpInstance.sendLine(name) + self.dtpInstance.transport.loseConnection() + return (TXFR_COMPLETE_OK,) + + def listErr(results): + """ + RFC 959 specifies that an NLST request may only return directory + listings. Thus, send nothing and just close the connection. + + @type results: L{Failure} + @param results: The L{Failure} wrapping a L{FileNotFoundError} that + occurred while trying to list the contents of a nonexistent + directory. + + @returns: A C{tuple} containing the status code for a successful + transfer. + @rtype: C{tuple} + """ + self.dtpInstance.transport.loseConnection() + return (TXFR_COMPLETE_OK,) + + if _isGlobbingExpression(segments): + # Remove globbing expression from path + # and keep to be used for filtering. + glob = segments.pop() + else: + glob = None + + d = self.shell.list(segments) + d.addCallback(cbList, glob) + # self.shell.list will generate an error if the path is invalid + d.addErrback(listErr) + return d + + + def ftp_CWD(self, path): + try: + segments = toSegments(self.workingDirectory, path) + except InvalidPath: + # XXX Eh, what to fail with here? + return defer.fail(FileNotFoundError(path)) + + def accessGranted(result): + self.workingDirectory = segments + return (REQ_FILE_ACTN_COMPLETED_OK,) + + return self.shell.access(segments).addCallback(accessGranted) + + + def ftp_CDUP(self): + return self.ftp_CWD('..') + + + def ftp_PWD(self): + return (PWD_REPLY, '/' + '/'.join(self.workingDirectory)) + + + def ftp_RETR(self, path): + """ + This command causes the content of a file to be sent over the data + transfer channel. If the path is to a folder, an error will be raised. + + @type path: C{str} + @param path: The path to the file which should be transferred over the + data transfer channel. + + @rtype: L{Deferred} + @return: a L{Deferred} which will be fired when the transfer is done. + """ + if self.dtpInstance is None: + raise BadCmdSequenceError('PORT or PASV required before RETR') + + try: + newsegs = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + + # XXX For now, just disable the timeout. Later we'll want to + # leave it active and have the DTP connection reset it + # periodically. + self.setTimeout(None) + + # Put it back later + def enableTimeout(result): + self.setTimeout(self.factory.timeOut) + return result + + # And away she goes + if not self.binary: + cons = ASCIIConsumerWrapper(self.dtpInstance) + else: + cons = self.dtpInstance + + def cbSent(result): + return (TXFR_COMPLETE_OK,) + + def ebSent(err): + log.msg("Unexpected error attempting to transmit file to client:") + log.err(err) + if err.check(FTPCmdError): + return err + return (CNX_CLOSED_TXFR_ABORTED,) + + def cbOpened(file): + # Tell them what to doooo + if self.dtpInstance.isConnected: + self.reply(DATA_CNX_ALREADY_OPEN_START_XFR) + else: + self.reply(FILE_STATUS_OK_OPEN_DATA_CNX) + + d = file.send(cons) + d.addCallbacks(cbSent, ebSent) + return d + + def ebOpened(err): + if not err.check( + PermissionDeniedError, FileNotFoundError, + IsADirectoryError): + log.msg( + "Unexpected error attempting to open file for " + "transmission:") + log.err(err) + if err.check(FTPCmdError): + return (err.value.errorCode, '/'.join(newsegs)) + return (FILE_NOT_FOUND, '/'.join(newsegs)) + + d = self.shell.openForReading(newsegs) + d.addCallbacks(cbOpened, ebOpened) + d.addBoth(enableTimeout) + + # Pass back Deferred that fires when the transfer is done + return d + + + def ftp_STOR(self, path): + """ + STORE (STOR) + + This command causes the server-DTP to accept the data + transferred via the data connection and to store the data as + a file at the server site. If the file specified in the + pathname exists at the server site, then its contents shall + be replaced by the data being transferred. A new file is + created at the server site if the file specified in the + pathname does not already exist. + """ + if self.dtpInstance is None: + raise BadCmdSequenceError('PORT or PASV required before STOR') + + try: + newsegs = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + + # XXX For now, just disable the timeout. Later we'll want to + # leave it active and have the DTP connection reset it + # periodically. + self.setTimeout(None) + + # Put it back later + def enableTimeout(result): + self.setTimeout(self.factory.timeOut) + return result + + def cbOpened(file): + """ + File was open for reading. Launch the data transfer channel via + the file consumer. + """ + d = file.receive() + d.addCallback(cbConsumer) + d.addCallback(lambda ignored: file.close()) + d.addCallbacks(cbSent, ebSent) + return d + + def ebOpened(err): + """ + Called when failed to open the file for reading. + + For known errors, return the FTP error code. + For all other, return a file not found error. + """ + if isinstance(err.value, FTPCmdError): + return (err.value.errorCode, '/'.join(newsegs)) + log.err(err, "Unexpected error received while opening file:") + return (FILE_NOT_FOUND, '/'.join(newsegs)) + + def cbConsumer(cons): + """ + Called after the file was opended for reading. + + Prepare the data transfer channel and send the response + to the command channel. + """ + if not self.binary: + cons = ASCIIConsumerWrapper(cons) + + d = self.dtpInstance.registerConsumer(cons) + + # Tell them what to doooo + if self.dtpInstance.isConnected: + self.reply(DATA_CNX_ALREADY_OPEN_START_XFR) + else: + self.reply(FILE_STATUS_OK_OPEN_DATA_CNX) + + return d + + def cbSent(result): + """ + Called from data transport when tranfer is done. + """ + return (TXFR_COMPLETE_OK,) + + def ebSent(err): + """ + Called from data transport when there are errors during the + transfer. + """ + log.err(err, "Unexpected error received during transfer:") + if err.check(FTPCmdError): + return err + return (CNX_CLOSED_TXFR_ABORTED,) + + d = self.shell.openForWriting(newsegs) + d.addCallbacks(cbOpened, ebOpened) + d.addBoth(enableTimeout) + + # Pass back Deferred that fires when the transfer is done + return d + + + def ftp_SIZE(self, path): + """ + File SIZE + + The FTP command, SIZE OF FILE (SIZE), is used to obtain the transfer + size of a file from the server-FTP process. This is the exact number + of octets (8 bit bytes) that would be transmitted over the data + connection should that file be transmitted. This value will change + depending on the current STRUcture, MODE, and TYPE of the data + connection or of a data connection that would be created were one + created now. Thus, the result of the SIZE command is dependent on + the currently established STRU, MODE, and TYPE parameters. + + The SIZE command returns how many octets would be transferred if the + file were to be transferred using the current transfer structure, + mode, and type. This command is normally used in conjunction with + the RESTART (REST) command when STORing a file to a remote server in + STREAM mode, to determine the restart point. The server-PI might + need to read the partially transferred file, do any appropriate + conversion, and count the number of octets that would be generated + when sending the file in order to correctly respond to this command. + Estimates of the file transfer size MUST NOT be returned; only + precise information is acceptable. + + http://tools.ietf.org/html/rfc3659 + """ + try: + newsegs = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + + def cbStat(result): + (size,) = result + return (FILE_STATUS, str(size)) + + return self.shell.stat(newsegs, ('size',)).addCallback(cbStat) + + + def ftp_MDTM(self, path): + """ + File Modification Time (MDTM) + + The FTP command, MODIFICATION TIME (MDTM), can be used to determine + when a file in the server NVFS was last modified. This command has + existed in many FTP servers for many years, as an adjunct to the REST + command for STREAM mode, thus is widely available. However, where + supported, the "modify" fact that can be provided in the result from + the new MLST command is recommended as a superior alternative. + + http://tools.ietf.org/html/rfc3659 + """ + try: + newsegs = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + + def cbStat(result): + (modified,) = result + return ( + FILE_STATUS, + time.strftime('%Y%m%d%H%M%S', time.gmtime(modified))) + + return self.shell.stat(newsegs, ('modified',)).addCallback(cbStat) + + + def ftp_TYPE(self, type): + """ + REPRESENTATION TYPE (TYPE) + + The argument specifies the representation type as described + in the Section on Data Representation and Storage. Several + types take a second parameter. The first parameter is + denoted by a single Telnet character, as is the second + Format parameter for ASCII and EBCDIC; the second parameter + for local byte is a decimal integer to indicate Bytesize. + The parameters are separated by a (Space, ASCII code + 32). + """ + p = type.upper() + if p: + f = getattr(self, 'type_' + p[0], None) + if f is not None: + return f(p[1:]) + return self.type_UNKNOWN(p) + return (SYNTAX_ERR,) + + def type_A(self, code): + if code == '' or code == 'N': + self.binary = False + return (TYPE_SET_OK, 'A' + code) + else: + return defer.fail(CmdArgSyntaxError(code)) + + def type_I(self, code): + if code == '': + self.binary = True + return (TYPE_SET_OK, 'I') + else: + return defer.fail(CmdArgSyntaxError(code)) + + def type_UNKNOWN(self, code): + return defer.fail(CmdNotImplementedForArgError(code)) + + + def ftp_SYST(self): + return NAME_SYS_TYPE + + + def ftp_STRU(self, structure): + p = structure.upper() + if p == 'F': + return (CMD_OK,) + return defer.fail(CmdNotImplementedForArgError(structure)) + + + def ftp_MODE(self, mode): + p = mode.upper() + if p == 'S': + return (CMD_OK,) + return defer.fail(CmdNotImplementedForArgError(mode)) + + + def ftp_MKD(self, path): + try: + newsegs = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + return self.shell.makeDirectory(newsegs).addCallback( + lambda ign: (MKD_REPLY, path)) + + + def ftp_RMD(self, path): + try: + newsegs = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + return self.shell.removeDirectory(newsegs).addCallback( + lambda ign: (REQ_FILE_ACTN_COMPLETED_OK,)) + + + def ftp_DELE(self, path): + try: + newsegs = toSegments(self.workingDirectory, path) + except InvalidPath: + return defer.fail(FileNotFoundError(path)) + return self.shell.removeFile(newsegs).addCallback( + lambda ign: (REQ_FILE_ACTN_COMPLETED_OK,)) + + + def ftp_NOOP(self): + return (CMD_OK,) + + + def ftp_RNFR(self, fromName): + self._fromName = fromName + self.state = self.RENAMING + return (REQ_FILE_ACTN_PENDING_FURTHER_INFO,) + + + def ftp_RNTO(self, toName): + fromName = self._fromName + del self._fromName + self.state = self.AUTHED + + try: + fromsegs = toSegments(self.workingDirectory, fromName) + tosegs = toSegments(self.workingDirectory, toName) + except InvalidPath: + return defer.fail(FileNotFoundError(fromName)) + return self.shell.rename(fromsegs, tosegs).addCallback( + lambda ign: (REQ_FILE_ACTN_COMPLETED_OK,)) + + + def ftp_FEAT(self): + """ + Advertise the features supported by the server. + + http://tools.ietf.org/html/rfc2389 + """ + self.sendLine(RESPONSE[FEAT_OK][0]) + for feature in self.FEATURES: + self.sendLine(' ' + feature) + self.sendLine(RESPONSE[FEAT_OK][1]) + + def ftp_OPTS(self, option): + """ + Handle OPTS command. + + http://tools.ietf.org/html/draft-ietf-ftpext-utf-8-option-00 + """ + return self.reply(OPTS_NOT_IMPLEMENTED, option) + + def ftp_QUIT(self): + self.reply(GOODBYE_MSG) + self.transport.loseConnection() + self.disconnected = True + + def cleanupDTP(self): + """ + Call when DTP connection exits + """ + log.msg('cleanupDTP', debug=True) + + log.msg(self.dtpPort) + dtpPort, self.dtpPort = self.dtpPort, None + if interfaces.IListeningPort.providedBy(dtpPort): + dtpPort.stopListening() + elif interfaces.IConnector.providedBy(dtpPort): + dtpPort.disconnect() + else: + assert False, ( + "dtpPort should be an IListeningPort or IConnector, " + "instead is %r" % (dtpPort,)) + + self.dtpFactory.stopFactory() + self.dtpFactory = None + + if self.dtpInstance is not None: + self.dtpInstance = None + + + +class FTPFactory(policies.LimitTotalConnectionsFactory): + """ + A factory for producing ftp protocol instances + + @ivar timeOut: the protocol interpreter's idle timeout time in seconds, + default is 600 seconds. + + @ivar passivePortRange: value forwarded to C{protocol.passivePortRange}. + @type passivePortRange: C{iterator} + """ + protocol = FTP + overflowProtocol = FTPOverflowProtocol + allowAnonymous = True + userAnonymous = 'anonymous' + timeOut = 600 + + welcomeMessage = "Twisted %s FTP Server" % (copyright.version,) + + passivePortRange = range(0, 1) + + def __init__(self, portal=None, userAnonymous='anonymous'): + self.portal = portal + self.userAnonymous = userAnonymous + self.instances = [] + + def buildProtocol(self, addr): + p = policies.LimitTotalConnectionsFactory.buildProtocol(self, addr) + if p is not None: + p.wrappedProtocol.portal = self.portal + p.wrappedProtocol.timeOut = self.timeOut + p.wrappedProtocol.passivePortRange = self.passivePortRange + return p + + def stopFactory(self): + # make sure ftp instance's timeouts are set to None + # to avoid reactor complaints + [p.setTimeout(None) for p in self.instances if p.timeOut is not None] + policies.LimitTotalConnectionsFactory.stopFactory(self) + + + +# -- Cred Objects -- + +class IFTPShell(Interface): + """ + An abstraction of the shell commands used by the FTP protocol for + a given user account. + + All path names must be absolute. + """ + + def makeDirectory(path): + """ + Create a directory. + + @param path: The path, as a list of segments, to create + @type path: C{list} of C{unicode} + + @return: A Deferred which fires when the directory has been + created, or which fails if the directory cannot be created. + """ + + + def removeDirectory(path): + """ + Remove a directory. + + @param path: The path, as a list of segments, to remove + @type path: C{list} of C{unicode} + + @return: A Deferred which fires when the directory has been + removed, or which fails if the directory cannot be removed. + """ + + + def removeFile(path): + """ + Remove a file. + + @param path: The path, as a list of segments, to remove + @type path: C{list} of C{unicode} + + @return: A Deferred which fires when the file has been + removed, or which fails if the file cannot be removed. + """ + + + def rename(fromPath, toPath): + """ + Rename a file or directory. + + @param fromPath: The current name of the path. + @type fromPath: C{list} of C{unicode} + + @param toPath: The desired new name of the path. + @type toPath: C{list} of C{unicode} + + @return: A Deferred which fires when the path has been + renamed, or which fails if the path cannot be renamed. + """ + + + def access(path): + """ + Determine whether access to the given path is allowed. + + @param path: The path, as a list of segments + + @return: A Deferred which fires with None if access is allowed + or which fails with a specific exception type if access is + denied. + """ + + + def stat(path, keys=()): + """ + Retrieve information about the given path. + + This is like list, except it will never return results about + child paths. + """ + + + def list(path, keys=()): + """ + Retrieve information about the given path. + + If the path represents a non-directory, the result list should + have only one entry with information about that non-directory. + Otherwise, the result list should have an element for each + child of the directory. + + @param path: The path, as a list of segments, to list + @type path: C{list} of C{unicode} or C{bytes} + + @param keys: A tuple of keys desired in the resulting + dictionaries. + + @return: A Deferred which fires with a list of (name, list), + where the name is the name of the entry as a unicode string or + bytes and each list contains values corresponding to the requested + keys. The following are possible elements of keys, and the + values which should be returned for them: + + - C{'size'}: size in bytes, as an integer (this is kinda required) + + - C{'directory'}: boolean indicating the type of this entry + + - C{'permissions'}: a bitvector (see os.stat(foo).st_mode) + + - C{'hardlinks'}: Number of hard links to this entry + + - C{'modified'}: number of seconds since the epoch since entry was + modified + + - C{'owner'}: string indicating the user owner of this entry + + - C{'group'}: string indicating the group owner of this entry + """ + + + def openForReading(path): + """ + @param path: The path, as a list of segments, to open + @type path: C{list} of C{unicode} + + @rtype: C{Deferred} which will fire with L{IReadFile} + """ + + + def openForWriting(path): + """ + @param path: The path, as a list of segments, to open + @type path: C{list} of C{unicode} + + @rtype: C{Deferred} which will fire with L{IWriteFile} + """ + + + +class IReadFile(Interface): + """ + A file out of which bytes may be read. + """ + + def send(consumer): + """ + Produce the contents of the given path to the given consumer. This + method may only be invoked once on each provider. + + @type consumer: C{IConsumer} + + @return: A Deferred which fires when the file has been + consumed completely. + """ + + + +class IWriteFile(Interface): + """ + A file into which bytes may be written. + """ + + def receive(): + """ + Create a consumer which will write to this file. This method may + only be invoked once on each provider. + + @rtype: C{Deferred} of C{IConsumer} + """ + + def close(): + """ + Perform any post-write work that needs to be done. This method may + only be invoked once on each provider, and will always be invoked + after receive(). + + @rtype: C{Deferred} of anything: the value is ignored. The FTP client + will not see their upload request complete until this Deferred has + been fired. + """ + + + +def _getgroups(uid): + """ + Return the primary and supplementary groups for the given UID. + + @type uid: C{int} + """ + result = [] + pwent = pwd.getpwuid(uid) + + result.append(pwent.pw_gid) + + for grent in grp.getgrall(): + if pwent.pw_name in grent.gr_mem: + result.append(grent.gr_gid) + + return result + + + +def _testPermissions(uid, gid, spath, mode='r'): + """ + checks to see if uid has proper permissions to access path with mode + + @type uid: C{int} + @param uid: numeric user id + + @type gid: C{int} + @param gid: numeric group id + + @type spath: C{str} + @param spath: the path on the server to test + + @type mode: C{str} + @param mode: 'r' or 'w' (read or write) + + @rtype: C{bool} + @return: True if the given credentials have the specified form of + access to the given path + """ + if mode == 'r': + usr = stat.S_IRUSR + grp = stat.S_IRGRP + oth = stat.S_IROTH + amode = os.R_OK + elif mode == 'w': + usr = stat.S_IWUSR + grp = stat.S_IWGRP + oth = stat.S_IWOTH + amode = os.W_OK + else: + raise ValueError("Invalid mode %r: must specify 'r' or 'w'" % (mode,)) + + access = False + if os.path.exists(spath): + if uid == 0: + access = True + else: + s = os.stat(spath) + if usr & s.st_mode and uid == s.st_uid: + access = True + elif grp & s.st_mode and gid in _getgroups(uid): + access = True + elif oth & s.st_mode: + access = True + + if access: + if not os.access(spath, amode): + access = False + log.msg( + "Filesystem grants permission to UID %d but it is " + "inaccessible to me running as UID %d" % ( + uid, os.getuid())) + return access + + + +@implementer(IFTPShell) +class FTPAnonymousShell(object): + """ + An anonymous implementation of IFTPShell + + @type filesystemRoot: L{twisted.python.filepath.FilePath} + @ivar filesystemRoot: The path which is considered the root of + this shell. + """ + def __init__(self, filesystemRoot): + self.filesystemRoot = filesystemRoot + + + def _path(self, path): + return self.filesystemRoot.descendant(path) + + + def makeDirectory(self, path): + return defer.fail(AnonUserDeniedError()) + + + def removeDirectory(self, path): + return defer.fail(AnonUserDeniedError()) + + + def removeFile(self, path): + return defer.fail(AnonUserDeniedError()) + + + def rename(self, fromPath, toPath): + return defer.fail(AnonUserDeniedError()) + + + def receive(self, path): + path = self._path(path) + return defer.fail(AnonUserDeniedError()) + + + def openForReading(self, path): + """ + Open C{path} for reading. + + @param path: The path, as a list of segments, to open. + @type path: C{list} of C{unicode} + @return: A L{Deferred} is returned that will fire with an object + implementing L{IReadFile} if the file is successfully opened. If + C{path} is a directory, or if an exception is raised while trying + to open the file, the L{Deferred} will fire with an error. + """ + p = self._path(path) + if p.isdir(): + # Normally, we would only check for EISDIR in open, but win32 + # returns EACCES in this case, so we check before + return defer.fail(IsADirectoryError(path)) + try: + f = p.open('r') + except (IOError, OSError) as e: + return errnoToFailure(e.errno, path) + except: + return defer.fail() + else: + return defer.succeed(_FileReader(f)) + + + def openForWriting(self, path): + """ + Reject write attempts by anonymous users with + L{PermissionDeniedError}. + """ + return defer.fail(PermissionDeniedError("STOR not allowed")) + + + def access(self, path): + p = self._path(path) + if not p.exists(): + # Again, win32 doesn't report a sane error after, so let's fail + # early if we can + return defer.fail(FileNotFoundError(path)) + # For now, just see if we can os.listdir() it + try: + p.listdir() + except (IOError, OSError) as e: + return errnoToFailure(e.errno, path) + except: + return defer.fail() + else: + return defer.succeed(None) + + + def stat(self, path, keys=()): + p = self._path(path) + if p.isdir(): + try: + statResult = self._statNode(p, keys) + except (IOError, OSError) as e: + return errnoToFailure(e.errno, path) + except: + return defer.fail() + else: + return defer.succeed(statResult) + else: + return self.list(path, keys).addCallback(lambda res: res[0][1]) + + + def list(self, path, keys=()): + """ + Return the list of files at given C{path}, adding C{keys} stat + informations if specified. + + @param path: the directory or file to check. + @type path: C{str} + + @param keys: the list of desired metadata + @type keys: C{list} of C{str} + """ + filePath = self._path(path) + if filePath.isdir(): + entries = filePath.listdir() + fileEntries = [filePath.child(p) for p in entries] + elif filePath.isfile(): + entries = [ + os.path.join(*filePath.segmentsFrom(self.filesystemRoot))] + fileEntries = [filePath] + else: + return defer.fail(FileNotFoundError(path)) + + results = [] + for fileName, filePath in zip(entries, fileEntries): + ent = [] + results.append((fileName, ent)) + if keys: + try: + ent.extend(self._statNode(filePath, keys)) + except (IOError, OSError) as e: + return errnoToFailure(e.errno, fileName) + except: + return defer.fail() + + return defer.succeed(results) + + + def _statNode(self, filePath, keys): + """ + Shortcut method to get stat info on a node. + + @param filePath: the node to stat. + @type filePath: C{filepath.FilePath} + + @param keys: the stat keys to get. + @type keys: C{iterable} + """ + filePath.restat() + return [getattr(self, '_stat_' + k)(filePath) for k in keys] + + + def _stat_size(self, fp): + """ + Get the filepath's size as an int + + @param fp: L{twisted.python.filepath.FilePath} + @return: C{int} representing the size + """ + return fp.getsize() + + + def _stat_permissions(self, fp): + """ + Get the filepath's permissions object + + @param fp: L{twisted.python.filepath.FilePath} + @return: L{twisted.python.filepath.Permissions} of C{fp} + """ + return fp.getPermissions() + + + def _stat_hardlinks(self, fp): + """ + Get the number of hardlinks for the filepath - if the number of + hardlinks is not yet implemented (say in Windows), just return 0 since + stat-ing a file in Windows seems to return C{st_nlink=0}. + + (Reference: + U{http://stackoverflow.com/questions/5275731/os-stat-on-windows}) + + @param fp: L{twisted.python.filepath.FilePath} + @return: C{int} representing the number of hardlinks + """ + try: + return fp.getNumberOfHardLinks() + except NotImplementedError: + return 0 + + + def _stat_modified(self, fp): + """ + Get the filepath's last modified date + + @param fp: L{twisted.python.filepath.FilePath} + @return: C{int} as seconds since the epoch + """ + return fp.getModificationTime() + + + def _stat_owner(self, fp): + """ + Get the filepath's owner's username. If this is not implemented + (say in Windows) return the string "0" since stat-ing a file in + Windows seems to return C{st_uid=0}. + + (Reference: + U{http://stackoverflow.com/questions/5275731/os-stat-on-windows}) + + @param fp: L{twisted.python.filepath.FilePath} + @return: C{str} representing the owner's username + """ + try: + userID = fp.getUserID() + except NotImplementedError: + return "0" + else: + if pwd is not None: + try: + return pwd.getpwuid(userID)[0] + except KeyError: + pass + return str(userID) + + + def _stat_group(self, fp): + """ + Get the filepath's owner's group. If this is not implemented + (say in Windows) return the string "0" since stat-ing a file in + Windows seems to return C{st_gid=0}. + + (Reference: + U{http://stackoverflow.com/questions/5275731/os-stat-on-windows}) + + @param fp: L{twisted.python.filepath.FilePath} + @return: C{str} representing the owner's group + """ + try: + groupID = fp.getGroupID() + except NotImplementedError: + return "0" + else: + if grp is not None: + try: + return grp.getgrgid(groupID)[0] + except KeyError: + pass + return str(groupID) + + + def _stat_directory(self, fp): + """ + Get whether the filepath is a directory + + @param fp: L{twisted.python.filepath.FilePath} + @return: C{bool} + """ + return fp.isdir() + + + +@implementer(IReadFile) +class _FileReader(object): + def __init__(self, fObj): + self.fObj = fObj + self._send = False + + def _close(self, passthrough): + self._send = True + self.fObj.close() + return passthrough + + def send(self, consumer): + assert not self._send, ( + "Can only call IReadFile.send *once* per instance") + self._send = True + d = basic.FileSender().beginFileTransfer(self.fObj, consumer) + d.addBoth(self._close) + return d + + + +class FTPShell(FTPAnonymousShell): + """ + An authenticated implementation of L{IFTPShell}. + """ + + def makeDirectory(self, path): + p = self._path(path) + try: + p.makedirs() + except (IOError, OSError) as e: + return errnoToFailure(e.errno, path) + except: + return defer.fail() + else: + return defer.succeed(None) + + + def removeDirectory(self, path): + p = self._path(path) + if p.isfile(): + # Win32 returns the wrong errno when rmdir is called on a file + # instead of a directory, so as we have the info here, let's fail + # early with a pertinent error + return defer.fail(IsNotADirectoryError(path)) + try: + os.rmdir(p.path) + except (IOError, OSError) as e: + return errnoToFailure(e.errno, path) + except: + return defer.fail() + else: + return defer.succeed(None) + + + def removeFile(self, path): + p = self._path(path) + if p.isdir(): + # Win32 returns the wrong errno when remove is called on a + # directory instead of a file, so as we have the info here, + # let's fail early with a pertinent error + return defer.fail(IsADirectoryError(path)) + try: + p.remove() + except (IOError, OSError) as e: + return errnoToFailure(e.errno, path) + except: + return defer.fail() + else: + return defer.succeed(None) + + + def rename(self, fromPath, toPath): + fp = self._path(fromPath) + tp = self._path(toPath) + try: + os.rename(fp.path, tp.path) + except (IOError, OSError) as e: + return errnoToFailure(e.errno, fromPath) + except: + return defer.fail() + else: + return defer.succeed(None) + + + def openForWriting(self, path): + """ + Open C{path} for writing. + + @param path: The path, as a list of segments, to open. + @type path: C{list} of C{unicode} + @return: A L{Deferred} is returned that will fire with an object + implementing L{IWriteFile} if the file is successfully opened. If + C{path} is a directory, or if an exception is raised while trying + to open the file, the L{Deferred} will fire with an error. + """ + p = self._path(path) + if p.isdir(): + # Normally, we would only check for EISDIR in open, but win32 + # returns EACCES in this case, so we check before + return defer.fail(IsADirectoryError(path)) + try: + fObj = p.open('w') + except (IOError, OSError) as e: + return errnoToFailure(e.errno, path) + except: + return defer.fail() + return defer.succeed(_FileWriter(fObj)) + + + +@implementer(IWriteFile) +class _FileWriter(object): + def __init__(self, fObj): + self.fObj = fObj + self._receive = False + + def receive(self): + assert not self._receive, ( + "Can only call IWriteFile.receive *once* per instance") + self._receive = True + # FileConsumer will close the file object + return defer.succeed(FileConsumer(self.fObj)) + + def close(self): + return defer.succeed(None) + + + +@implementer(portal.IRealm) +class BaseFTPRealm: + """ + Base class for simple FTP realms which provides an easy hook for specifying + the home directory for each user. + """ + def __init__(self, anonymousRoot): + self.anonymousRoot = filepath.FilePath(anonymousRoot) + + + def getHomeDirectory(self, avatarId): + """ + Return a L{FilePath} representing the home directory of the given + avatar. Override this in a subclass. + + @param avatarId: A user identifier returned from a credentials checker. + @type avatarId: C{str} + + @rtype: L{FilePath} + """ + raise NotImplementedError( + "%r did not override getHomeDirectory" % (self.__class__,)) + + + def requestAvatar(self, avatarId, mind, *interfaces): + for iface in interfaces: + if iface is IFTPShell: + if avatarId is checkers.ANONYMOUS: + avatar = FTPAnonymousShell(self.anonymousRoot) + else: + avatar = FTPShell(self.getHomeDirectory(avatarId)) + return (IFTPShell, avatar, + getattr(avatar, 'logout', lambda: None)) + raise NotImplementedError( + "Only IFTPShell interface is supported by this realm") + + + +class FTPRealm(BaseFTPRealm): + """ + @type anonymousRoot: L{twisted.python.filepath.FilePath} + @ivar anonymousRoot: Root of the filesystem to which anonymous + users will be granted access. + + @type userHome: L{filepath.FilePath} + @ivar userHome: Root of the filesystem containing user home directories. + """ + def __init__(self, anonymousRoot, userHome='/home'): + BaseFTPRealm.__init__(self, anonymousRoot) + self.userHome = filepath.FilePath(userHome) + + + def getHomeDirectory(self, avatarId): + """ + Use C{avatarId} as a single path segment to construct a child of + C{self.userHome} and return that child. + """ + return self.userHome.child(avatarId) + + + +class SystemFTPRealm(BaseFTPRealm): + """ + L{SystemFTPRealm} uses system user account information to decide what the + home directory for a particular avatarId is. + + This works on POSIX but probably is not reliable on Windows. + """ + def getHomeDirectory(self, avatarId): + """ + Return the system-defined home directory of the system user account + with the name C{avatarId}. + """ + path = os.path.expanduser('~' + avatarId) + if path.startswith('~'): + raise cred_error.UnauthorizedLogin() + return filepath.FilePath(path) + + + +# --- FTP CLIENT ------------------------------------------------------------- + +#### +# And now for the client... + +# Notes: +# * Reference: http://cr.yp.to/ftp.html +# * FIXME: Does not support pipelining (which is not supported by all +# servers anyway). This isn't a functionality limitation, just a +# small performance issue. +# * Only has a rudimentary understanding of FTP response codes (although +# the full response is passed to the caller if they so choose). +# * Assumes that USER and PASS should always be sent +# * Always sets TYPE I (binary mode) +# * Doesn't understand any of the weird, obscure TELNET stuff (\377...) +# * FIXME: Doesn't share any code with the FTPServer + +class ConnectionLost(FTPError): + pass + + + +class CommandFailed(FTPError): + pass + + + +class BadResponse(FTPError): + pass + + + +class UnexpectedResponse(FTPError): + pass + + + +class UnexpectedData(FTPError): + pass + + + +class FTPCommand: + def __init__(self, text=None, public=0): + self.text = text + self.deferred = defer.Deferred() + self.ready = 1 + self.public = public + self.transferDeferred = None + + def fail(self, failure): + if self.public: + self.deferred.errback(failure) + + + +class ProtocolWrapper(protocol.Protocol): + def __init__(self, original, deferred): + self.original = original + self.deferred = deferred + + def makeConnection(self, transport): + self.original.makeConnection(transport) + + def dataReceived(self, data): + self.original.dataReceived(data) + + def connectionLost(self, reason): + self.original.connectionLost(reason) + # Signal that transfer has completed + self.deferred.callback(None) + + + +class IFinishableConsumer(interfaces.IConsumer): + """ + A Consumer for producers that finish. + + @since: 11.0 + """ + + def finish(): + """ + The producer has finished producing. + """ + + + +@implementer(IFinishableConsumer) +class SenderProtocol(protocol.Protocol): + def __init__(self): + # Fired upon connection + self.connectedDeferred = defer.Deferred() + + # Fired upon disconnection + self.deferred = defer.Deferred() + + # Protocol stuff + def dataReceived(self, data): + raise UnexpectedData( + "Received data from the server on a " + "send-only data-connection" + ) + + def makeConnection(self, transport): + protocol.Protocol.makeConnection(self, transport) + self.connectedDeferred.callback(self) + + def connectionLost(self, reason): + if reason.check(error.ConnectionDone): + self.deferred.callback('connection done') + else: + self.deferred.errback(reason) + + # IFinishableConsumer stuff + def write(self, data): + self.transport.write(data) + + def registerProducer(self, producer, streaming): + """ + Register the given producer with our transport. + """ + self.transport.registerProducer(producer, streaming) + + def unregisterProducer(self): + """ + Unregister the previously registered producer. + """ + self.transport.unregisterProducer() + + def finish(self): + self.transport.loseConnection() + + + +def decodeHostPort(line): + """ + Decode an FTP response specifying a host and port. + + @return: a 2-tuple of (host, port). + """ + abcdef = re.sub('[^0-9, ]', '', line) + parsed = [int(p.strip()) for p in abcdef.split(',')] + for x in parsed: + if x < 0 or x > 255: + raise ValueError("Out of range", line, x) + a, b, c, d, e, f = parsed + host = "%s.%s.%s.%s" % (a, b, c, d) + port = (int(e) << 8) + int(f) + return host, port + + + +def encodeHostPort(host, port): + numbers = host.split('.') + [str(port >> 8), str(port % 256)] + return ','.join(numbers) + + + +def _unwrapFirstError(failure): + failure.trap(defer.FirstError) + return failure.value.subFailure + + + +class FTPDataPortFactory(protocol.ServerFactory): + """ + Factory for data connections that use the PORT command + + (i.e. "active" transfers) + """ + noisy = 0 + + def buildProtocol(self, addr): + # This is a bit hackish -- we already have a Protocol instance, + # so just return it instead of making a new one + # FIXME: Reject connections from the wrong address/port + # (potential security problem) + self.protocol.factory = self + self.port.loseConnection() + return self.protocol + + + +class FTPClientBasic(basic.LineReceiver): + """ + Foundations of an FTP client. + """ + debug = False + _encoding = 'latin-1' + + def __init__(self): + self.actionQueue = [] + self.greeting = None + self.nextDeferred = defer.Deferred().addCallback(self._cb_greeting) + self.nextDeferred.addErrback(self.fail) + self.response = [] + self._failed = 0 + + def fail(self, error): + """ + Give an error to any queued deferreds. + """ + self._fail(error) + + def _fail(self, error): + """ + Errback all queued deferreds. + """ + if self._failed: + # We're recursing; bail out here for simplicity + return error + self._failed = 1 + if self.nextDeferred: + try: + self.nextDeferred.errback(failure.Failure( + ConnectionLost('FTP connection lost', error))) + except defer.AlreadyCalledError: + pass + for ftpCommand in self.actionQueue: + ftpCommand.fail(failure.Failure( + ConnectionLost('FTP connection lost', error))) + return error + + def _cb_greeting(self, greeting): + self.greeting = greeting + + + def sendLine(self, line): + """ + Sends a line, unless line is None. + + @param line: Line to send + @type line: L{bytes} or L{unicode} + """ + if line is None: + return + elif isinstance(line, unicode): + line = line.encode(self._encoding) + basic.LineReceiver.sendLine(self, line) + + + def sendNextCommand(self): + """ + (Private) Processes the next command in the queue. + """ + ftpCommand = self.popCommandQueue() + if ftpCommand is None: + self.nextDeferred = None + return + if not ftpCommand.ready: + self.actionQueue.insert(0, ftpCommand) + reactor.callLater(1.0, self.sendNextCommand) + self.nextDeferred = None + return + + # FIXME: this if block doesn't belong in FTPClientBasic, it belongs in + # FTPClient. + if ftpCommand.text == 'PORT': + self.generatePortCommand(ftpCommand) + + if self.debug: + log.msg('<-- %s' % ftpCommand.text) + self.nextDeferred = ftpCommand.deferred + self.sendLine(ftpCommand.text) + + def queueCommand(self, ftpCommand): + """ + Add an FTPCommand object to the queue. + + If it's the only thing in the queue, and we are connected and we aren't + waiting for a response of an earlier command, the command will be sent + immediately. + + @param ftpCommand: an L{FTPCommand} + """ + self.actionQueue.append(ftpCommand) + if (len(self.actionQueue) == 1 and self.transport is not None and + self.nextDeferred is None): + self.sendNextCommand() + + def queueStringCommand(self, command, public=1): + """ + Queues a string to be issued as an FTP command + + @param command: string of an FTP command to queue + @param public: a flag intended for internal use by FTPClient. Don't + change it unless you know what you're doing. + + @return: a L{Deferred} that will be called when the response to the + command has been received. + """ + ftpCommand = FTPCommand(command, public) + self.queueCommand(ftpCommand) + return ftpCommand.deferred + + def popCommandQueue(self): + """ + Return the front element of the command queue, or None if empty. + """ + if self.actionQueue: + return self.actionQueue.pop(0) + else: + return None + + def queueLogin(self, username, password): + """ + Login: send the username, send the password. + + If the password is L{None}, the PASS command won't be sent. Also, if + the response to the USER command has a response code of 230 (User + logged in), then PASS won't be sent either. + """ + # Prepare the USER command + deferreds = [] + userDeferred = self.queueStringCommand('USER ' + username, public=0) + deferreds.append(userDeferred) + + # Prepare the PASS command (if a password is given) + if password is not None: + passwordCmd = FTPCommand('PASS ' + password, public=0) + self.queueCommand(passwordCmd) + deferreds.append(passwordCmd.deferred) + + # Avoid sending PASS if the response to USER is 230. + # (ref: http://cr.yp.to/ftp/user.html#user) + def cancelPasswordIfNotNeeded(response): + if response[0].startswith('230'): + # No password needed! + self.actionQueue.remove(passwordCmd) + return response + userDeferred.addCallback(cancelPasswordIfNotNeeded) + + # Error handling. + for deferred in deferreds: + # If something goes wrong, call fail + deferred.addErrback(self.fail) + # But also swallow the error, so we don't cause spurious errors + deferred.addErrback(lambda x: None) + + def lineReceived(self, line): + """ + (Private) Parses the response messages from the FTP server. + """ + # Add this line to the current response + if bytes != str: + line = line.decode(self._encoding) + + if self.debug: + log.msg('--> %s' % line) + self.response.append(line) + + # Bail out if this isn't the last line of a response + # The last line of response starts with 3 digits followed by a space + codeIsValid = re.match(r'\d{3} ', line) + if not codeIsValid: + return + + code = line[0:3] + + # Ignore marks + if code[0] == '1': + return + + # Check that we were expecting a response + if self.nextDeferred is None: + self.fail(UnexpectedResponse(self.response)) + return + + # Reset the response + response = self.response + self.response = [] + + # Look for a success or error code, and call the appropriate callback + if code[0] in ('2', '3'): + # Success + self.nextDeferred.callback(response) + elif code[0] in ('4', '5'): + # Failure + self.nextDeferred.errback(failure.Failure(CommandFailed(response))) + else: + # This shouldn't happen unless something screwed up. + log.msg('Server sent invalid response code %s' % (code,)) + self.nextDeferred.errback(failure.Failure(BadResponse(response))) + + # Run the next command + self.sendNextCommand() + + def connectionLost(self, reason): + self._fail(reason) + + + +class _PassiveConnectionFactory(protocol.ClientFactory): + noisy = False + + def __init__(self, protoInstance): + self.protoInstance = protoInstance + + def buildProtocol(self, ignored): + self.protoInstance.factory = self + return self.protoInstance + + def clientConnectionFailed(self, connector, reason): + e = FTPError('Connection Failed', reason) + self.protoInstance.deferred.errback(e) + + + +class FTPClient(FTPClientBasic): + """ + L{FTPClient} is a client implementation of the FTP protocol which + exposes FTP commands as methods which return L{Deferred}s. + + Each command method returns a L{Deferred} which is called back when a + successful response code (2xx or 3xx) is received from the server or + which is error backed if an error response code (4xx or 5xx) is received + from the server or if a protocol violation occurs. If an error response + code is received, the L{Deferred} fires with a L{Failure} wrapping a + L{CommandFailed} instance. The L{CommandFailed} instance is created + with a list of the response lines received from the server. + + See U{RFC 959} for error code + definitions. + + Both active and passive transfers are supported. + + @ivar passive: See description in __init__. + """ + connectFactory = reactor.connectTCP + + def __init__(self, username='anonymous', + password='twisted@twistedmatrix.com', + passive=1): + """ + Constructor. + + I will login as soon as I receive the welcome message from the server. + + @param username: FTP username + @param password: FTP password + @param passive: flag that controls if I use active or passive data + connections. You can also change this after construction by + assigning to C{self.passive}. + """ + FTPClientBasic.__init__(self) + self.queueLogin(username, password) + + self.passive = passive + + def fail(self, error): + """ + Disconnect, and also give an error to any queued deferreds. + """ + self.transport.loseConnection() + self._fail(error) + + def receiveFromConnection(self, commands, protocol): + """ + Retrieves a file or listing generated by the given command, + feeding it to the given protocol. + + @param commands: list of strings of FTP commands to execute then + receive the results of (e.g. C{LIST}, C{RETR}) + @param protocol: A L{Protocol} B{instance} e.g. an + L{FTPFileListProtocol}, or something that can be adapted to one. + Typically this will be an L{IConsumer} implementation. + + @return: L{Deferred}. + """ + protocol = interfaces.IProtocol(protocol) + wrapper = ProtocolWrapper(protocol, defer.Deferred()) + return self._openDataConnection(commands, wrapper) + + def queueLogin(self, username, password): + """ + Login: send the username, send the password, and + set retrieval mode to binary + """ + FTPClientBasic.queueLogin(self, username, password) + d = self.queueStringCommand('TYPE I', public=0) + # If something goes wrong, call fail + d.addErrback(self.fail) + # But also swallow the error, so we don't cause spurious errors + d.addErrback(lambda x: None) + + def sendToConnection(self, commands): + """ + XXX + + @return: A tuple of two L{Deferred}s: + - L{Deferred} L{IFinishableConsumer}. You must call + the C{finish} method on the IFinishableConsumer when the + file is completely transferred. + - L{Deferred} list of control-connection responses. + """ + s = SenderProtocol() + r = self._openDataConnection(commands, s) + return (s.connectedDeferred, r) + + def _openDataConnection(self, commands, protocol): + """ + This method returns a DeferredList. + """ + cmds = [FTPCommand(command, public=1) for command in commands] + cmdsDeferred = defer.DeferredList( + [cmd.deferred for cmd in cmds], + fireOnOneErrback=True, consumeErrors=True) + cmdsDeferred.addErrback(_unwrapFirstError) + + if self.passive: + # Hack: use a mutable object to sneak a variable out of the + # scope of doPassive + _mutable = [None] + + def doPassive(response): + """Connect to the port specified in the response to PASV""" + host, port = decodeHostPort(response[-1][4:]) + + f = _PassiveConnectionFactory(protocol) + _mutable[0] = self.connectFactory(host, port, f) + + pasvCmd = FTPCommand('PASV') + self.queueCommand(pasvCmd) + pasvCmd.deferred.addCallback(doPassive).addErrback(self.fail) + + results = [cmdsDeferred, pasvCmd.deferred, protocol.deferred] + d = defer.DeferredList( + results, fireOnOneErrback=True, consumeErrors=True) + d.addErrback(_unwrapFirstError) + + # Ensure the connection is always closed + def close(x, m=_mutable): + m[0] and m[0].disconnect() + return x + d.addBoth(close) + + else: + # We just place a marker command in the queue, and will fill in + # the host and port numbers later (see generatePortCommand) + portCmd = FTPCommand('PORT') + + # Ok, now we jump through a few hoops here. + # This is the problem: a transfer is not to be trusted as complete + # until we get both the "226 Transfer complete" message on the + # control connection, and the data socket is closed. Thus, we use + # a DeferredList to make sure we only fire the callback at the + # right time. + + portCmd.transferDeferred = protocol.deferred + portCmd.protocol = protocol + portCmd.deferred.addErrback(portCmd.transferDeferred.errback) + self.queueCommand(portCmd) + + # Create dummy functions for the next callback to call. + # These will also be replaced with real functions in + # generatePortCommand. + portCmd.loseConnection = lambda result: result + portCmd.fail = lambda error: error + + # Ensure that the connection always gets closed + cmdsDeferred.addErrback(lambda e, pc=portCmd: pc.fail(e) or e) + + results = [ + cmdsDeferred, portCmd.deferred, portCmd.transferDeferred] + d = defer.DeferredList( + results, fireOnOneErrback=True, consumeErrors=True) + d.addErrback(_unwrapFirstError) + + for cmd in cmds: + self.queueCommand(cmd) + return d + + def generatePortCommand(self, portCmd): + """ + (Private) Generates the text of a given PORT command. + """ + + # The problem is that we don't create the listening port until we need + # it for various reasons, and so we have to muck about to figure out + # what interface and port it's listening on, and then finally we can + # create the text of the PORT command to send to the FTP server. + + # FIXME: This method is far too ugly. + + # FIXME: The best solution is probably to only create the data port + # once per FTPClient, and just recycle it for each new download. + # This should be ok, because we don't pipeline commands. + + # Start listening on a port + factory = FTPDataPortFactory() + factory.protocol = portCmd.protocol + listener = reactor.listenTCP(0, factory) + factory.port = listener + + # Ensure we close the listening port if something goes wrong + def listenerFail(error, listener=listener): + if listener.connected: + listener.loseConnection() + return error + portCmd.fail = listenerFail + + # Construct crufty FTP magic numbers that represent host & port + host = self.transport.getHost().host + port = listener.getHost().port + portCmd.text = 'PORT ' + encodeHostPort(host, port) + + def escapePath(self, path): + """ + Returns a FTP escaped path (replace newlines with nulls). + """ + # Escape newline characters + return path.replace('\n', '\0') + + def retrieveFile(self, path, protocol, offset=0): + """ + Retrieve a file from the given path + + This method issues the 'RETR' FTP command. + + The file is fed into the given Protocol instance. The data connection + will be passive if self.passive is set. + + @param path: path to file that you wish to receive. + @param protocol: a L{Protocol} instance. + @param offset: offset to start downloading from + + @return: L{Deferred} + """ + cmds = ['RETR ' + self.escapePath(path)] + if offset: + cmds.insert(0, ('REST ' + str(offset))) + return self.receiveFromConnection(cmds, protocol) + + retr = retrieveFile + + def storeFile(self, path, offset=0): + """ + Store a file at the given path. + + This method issues the 'STOR' FTP command. + + @return: A tuple of two L{Deferred}s: + - L{Deferred} L{IFinishableConsumer}. You must call + the C{finish} method on the IFinishableConsumer when the + file is completely transferred. + - L{Deferred} list of control-connection responses. + """ + cmds = ['STOR ' + self.escapePath(path)] + if offset: + cmds.insert(0, ('REST ' + str(offset))) + return self.sendToConnection(cmds) + + stor = storeFile + + + def rename(self, pathFrom, pathTo): + """ + Rename a file. + + This method issues the I{RNFR}/I{RNTO} command sequence to rename + C{pathFrom} to C{pathTo}. + + @param: pathFrom: the absolute path to the file to be renamed + @type pathFrom: C{str} + + @param: pathTo: the absolute path to rename the file to. + @type pathTo: C{str} + + @return: A L{Deferred} which fires when the rename operation has + succeeded or failed. If it succeeds, the L{Deferred} is called + back with a two-tuple of lists. The first list contains the + responses to the I{RNFR} command. The second list contains the + responses to the I{RNTO} command. If either I{RNFR} or I{RNTO} + fails, the L{Deferred} is errbacked with L{CommandFailed} or + L{BadResponse}. + @rtype: L{Deferred} + + @since: 8.2 + """ + renameFrom = self.queueStringCommand( + 'RNFR ' + self.escapePath(pathFrom)) + renameTo = self.queueStringCommand('RNTO ' + self.escapePath(pathTo)) + + fromResponse = [] + + # Use a separate Deferred for the ultimate result so that Deferred + # chaining can't interfere with its result. + result = defer.Deferred() + # Bundle up all the responses + result.addCallback(lambda toResponse: (fromResponse, toResponse)) + + def ebFrom(failure): + # Make sure the RNTO doesn't run if the RNFR failed. + self.popCommandQueue() + result.errback(failure) + + # Save the RNFR response to pass to the result Deferred later + renameFrom.addCallbacks(fromResponse.extend, ebFrom) + + # Hook up the RNTO to the result Deferred as well + renameTo.chainDeferred(result) + + return result + + + def list(self, path, protocol): + """ + Retrieve a file listing into the given protocol instance. + + This method issues the 'LIST' FTP command. + + @param path: path to get a file listing for. + @param protocol: a L{Protocol} instance, probably a + L{FTPFileListProtocol} instance. It can cope with most common file + listing formats. + + @return: L{Deferred} + """ + if path is None: + path = '' + return self.receiveFromConnection( + ['LIST ' + self.escapePath(path)], protocol) + + + def nlst(self, path, protocol): + """ + Retrieve a short file listing into the given protocol instance. + + This method issues the 'NLST' FTP command. + + NLST (should) return a list of filenames, one per line. + + @param path: path to get short file listing for. + @param protocol: a L{Protocol} instance. + """ + if path is None: + path = '' + return self.receiveFromConnection( + ['NLST ' + self.escapePath(path)], protocol) + + + def cwd(self, path): + """ + Issues the CWD (Change Working Directory) command. + + @return: a L{Deferred} that will be called when done. + """ + return self.queueStringCommand('CWD ' + self.escapePath(path)) + + + def makeDirectory(self, path): + """ + Make a directory + + This method issues the MKD command. + + @param path: The path to the directory to create. + @type path: C{str} + + @return: A L{Deferred} which fires when the server responds. If the + directory is created, the L{Deferred} is called back with the + server response. If the server response indicates the directory + was not created, the L{Deferred} is errbacked with a L{Failure} + wrapping L{CommandFailed} or L{BadResponse}. + @rtype: L{Deferred} + + @since: 8.2 + """ + return self.queueStringCommand('MKD ' + self.escapePath(path)) + + + def removeFile(self, path): + """ + Delete a file on the server. + + L{removeFile} issues a I{DELE} command to the server to remove the + indicated file. Note that this command cannot remove a directory. + + @param path: The path to the file to delete. May be relative to the + current dir. + @type path: C{str} + + @return: A L{Deferred} which fires when the server responds. On error, + it is errbacked with either L{CommandFailed} or L{BadResponse}. On + success, it is called back with a list of response lines. + @rtype: L{Deferred} + + @since: 8.2 + """ + return self.queueStringCommand('DELE ' + self.escapePath(path)) + + + def removeDirectory(self, path): + """ + Delete a directory on the server. + + L{removeDirectory} issues a I{RMD} command to the server to remove the + indicated directory. Described in RFC959. + + @param path: The path to the directory to delete. May be relative to + the current working directory. + @type path: C{str} + + @return: A L{Deferred} which fires when the server responds. On error, + it is errbacked with either L{CommandFailed} or L{BadResponse}. On + success, it is called back with a list of response lines. + @rtype: L{Deferred} + + @since: 11.1 + """ + return self.queueStringCommand('RMD ' + self.escapePath(path)) + + + def cdup(self): + """ + Issues the CDUP (Change Directory UP) command. + + @return: a L{Deferred} that will be called when done. + """ + return self.queueStringCommand('CDUP') + + + def pwd(self): + """ + Issues the PWD (Print Working Directory) command. + + The L{getDirectory} does the same job but automatically parses the + result. + + @return: a L{Deferred} that will be called when done. It is up to the + caller to interpret the response, but the L{parsePWDResponse} + method in this module should work. + """ + return self.queueStringCommand('PWD') + + + def getDirectory(self): + """ + Returns the current remote directory. + + @return: a L{Deferred} that will be called back with a C{str} giving + the remote directory or which will errback with L{CommandFailed} + if an error response is returned. + """ + def cbParse(result): + try: + # The only valid code is 257 + if int(result[0].split(' ', 1)[0]) != 257: + raise ValueError + except (IndexError, ValueError): + return failure.Failure(CommandFailed(result)) + path = parsePWDResponse(result[0]) + if path is None: + return failure.Failure(CommandFailed(result)) + return path + return self.pwd().addCallback(cbParse) + + + def quit(self): + """ + Issues the I{QUIT} command. + + @return: A L{Deferred} that fires when the server acknowledges the + I{QUIT} command. The transport should not be disconnected until + this L{Deferred} fires. + """ + return self.queueStringCommand('QUIT') + + + +class FTPFileListProtocol(basic.LineReceiver): + """ + Parser for standard FTP file listings + + This is the evil required to match:: + + -rw-r--r-- 1 root other 531 Jan 29 03:26 README + + If you need different evil for a wacky FTP server, you can + override either C{fileLinePattern} or C{parseDirectoryLine()}. + + It populates the instance attribute self.files, which is a list containing + dicts with the following keys (examples from the above line): + - filetype: e.g. 'd' for directories, or '-' for an ordinary file + - perms: e.g. 'rw-r--r--' + - nlinks: e.g. 1 + - owner: e.g. 'root' + - group: e.g. 'other' + - size: e.g. 531 + - date: e.g. 'Jan 29 03:26' + - filename: e.g. 'README' + - linktarget: e.g. 'some/file' + + Note that the 'date' value will be formatted differently depending on the + date. Check U{http://cr.yp.to/ftp.html} if you really want to try to parse + it. + + It also matches the following:: + -rw-r--r-- 1 root other 531 Jan 29 03:26 I HAVE\\ SPACE + - filename: e.g. 'I HAVE SPACE' + + -rw-r--r-- 1 root other 531 Jan 29 03:26 LINK -> TARGET + - filename: e.g. 'LINK' + - linktarget: e.g. 'TARGET' + + -rw-r--r-- 1 root other 531 Jan 29 03:26 N S -> L S + - filename: e.g. 'N S' + - linktarget: e.g. 'L S' + + @ivar files: list of dicts describing the files in this listing + """ + fileLinePattern = re.compile( + r'^(?P.)(?P.{9})\s+(?P\d*)\s*' + r'(?P\S+)\s+(?P\S+)\s+(?P\d+)\s+' + r'(?P...\s+\d+\s+[\d:]+)\s+(?P.{1,}?)' + r'( -> (?P[^\r]*))?\r?$' + ) + delimiter = b'\n' + _encoding = 'latin-1' + + def __init__(self): + self.files = [] + + def lineReceived(self, line): + if bytes != str: + line = line.decode(self._encoding) + d = self.parseDirectoryLine(line) + if d is None: + self.unknownLine(line) + else: + self.addFile(d) + + def parseDirectoryLine(self, line): + """ + Return a dictionary of fields, or None if line cannot be parsed. + + @param line: line of text expected to contain a directory entry + @type line: str + + @return: dict + """ + match = self.fileLinePattern.match(line) + if match is None: + return None + else: + d = match.groupdict() + d['filename'] = d['filename'].replace(r'\ ', ' ') + d['nlinks'] = int(d['nlinks']) + d['size'] = int(d['size']) + if d['linktarget']: + d['linktarget'] = d['linktarget'].replace(r'\ ', ' ') + return d + + def addFile(self, info): + """ + Append file information dictionary to the list of known files. + + Subclasses can override or extend this method to handle file + information differently without affecting the parsing of data + from the server. + + @param info: dictionary containing the parsed representation + of the file information + @type info: dict + """ + self.files.append(info) + + def unknownLine(self, line): + """ + Deal with received lines which could not be parsed as file + information. + + Subclasses can override this to perform any special processing + needed. + + @param line: unparsable line as received + @type line: str + """ + pass + + + +def parsePWDResponse(response): + """ + Returns the path from a response to a PWD command. + + Responses typically look like:: + + 257 "/home/andrew" is current directory. + + For this example, I will return C{'/home/andrew'}. + + If I can't find the path, I return L{None}. + """ + match = re.search('"(.*)"', response) + if match: + return match.groups()[0] + else: + return None diff --git a/contrib/python/Twisted/py2/twisted/protocols/haproxy/__init__.py b/contrib/python/Twisted/py2/twisted/protocols/haproxy/__init__.py new file mode 100644 index 00000000000..c238b7ac1a1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/haproxy/__init__.py @@ -0,0 +1,13 @@ +# -*- test-case-name: twisted.protocols.haproxy.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HAProxy PROXY protocol implementations. +""" + +from ._wrapper import proxyEndpoint + +__all__ = [ + 'proxyEndpoint', +] diff --git a/contrib/python/Twisted/py2/twisted/protocols/haproxy/_exceptions.py b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_exceptions.py new file mode 100644 index 00000000000..8633ca4eecf --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_exceptions.py @@ -0,0 +1,52 @@ +# -*- test-case-name: twisted.protocols.haproxy.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HAProxy specific exceptions. +""" + +import contextlib +import sys + +from twisted.python import compat + + +class InvalidProxyHeader(Exception): + """ + The provided PROXY protocol header is invalid. + """ + + + +class InvalidNetworkProtocol(InvalidProxyHeader): + """ + The network protocol was not one of TCP4 TCP6 or UNKNOWN. + """ + + + +class MissingAddressData(InvalidProxyHeader): + """ + The address data is missing or incomplete. + """ + + + +@contextlib.contextmanager +def convertError(sourceType, targetType): + """ + Convert an error into a different error type. + + @param sourceType: The type of exception that should be caught and + converted. + @type sourceType: L{Exception} + + @param targetType: The type of exception to which the original should be + converted. + @type targetType: L{Exception} + """ + try: + yield None + except sourceType: + compat.reraise(targetType(), sys.exc_info()[-1]) diff --git a/contrib/python/Twisted/py2/twisted/protocols/haproxy/_info.py b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_info.py new file mode 100644 index 00000000000..489d7b2cee1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_info.py @@ -0,0 +1,36 @@ +# -*- test-case-name: twisted.protocols.haproxy.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +IProxyInfo implementation. +""" + +from zope.interface import implementer + +from ._interfaces import IProxyInfo + + +@implementer(IProxyInfo) +class ProxyInfo(object): + """ + A data container for parsed PROXY protocol information. + + @ivar header: The raw header bytes extracted from the connection. + @type header: bytes + @ivar source: The connection source address. + @type source: L{twisted.internet.interfaces.IAddress} + @ivar destination: The connection destination address. + @type destination: L{twisted.internet.interfaces.IAddress} + """ + + __slots__ = ( + 'header', + 'source', + 'destination', + ) + + def __init__(self, header, source, destination): + self.header = header + self.source = source + self.destination = destination diff --git a/contrib/python/Twisted/py2/twisted/protocols/haproxy/_interfaces.py b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_interfaces.py new file mode 100644 index 00000000000..3453ecf37b8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_interfaces.py @@ -0,0 +1,64 @@ +# -*- test-case-name: twisted.protocols.haproxy.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Interfaces used by the PROXY protocol modules. +""" + +import zope.interface + + +class IProxyInfo(zope.interface.Interface): + """ + Data container for PROXY protocol header data. + """ + + header = zope.interface.Attribute( + "The raw byestring that represents the PROXY protocol header.", + ) + source = zope.interface.Attribute( + "An L{twisted.internet.interfaces.IAddress} representing the " + "connection source." + ) + destination = zope.interface.Attribute( + "An L{twisted.internet.interfaces.IAddress} representing the " + "connection destination." + ) + + + +class IProxyParser(zope.interface.Interface): + """ + Streaming parser that handles PROXY protocol headers. + """ + + def feed(self, data): + """ + Consume a chunk of data and attempt to parse it. + + @param data: A bytestring. + @type data: bytes + + @return: A two-tuple containing, in order, an L{IProxyInfo} and any + bytes fed to the parser that followed the end of the header. Both + of these values are None until a complete header is parsed. + + @raises InvalidProxyHeader: If the bytes fed to the parser create an + invalid PROXY header. + """ + + + def parse(self, line): + """ + Parse a bytestring as a full PROXY protocol header line. + + @param line: A bytestring that represents a valid HAProxy PROXY + protocol header line. + @type line: bytes + + @return: An L{IProxyInfo} containing the parsed data. + + @raises InvalidProxyHeader: If the bytestring does not represent a + valid PROXY header. + """ diff --git a/contrib/python/Twisted/py2/twisted/protocols/haproxy/_parser.py b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_parser.py new file mode 100644 index 00000000000..35ef29fbb71 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_parser.py @@ -0,0 +1,71 @@ +# -*- test-case-name: twisted.protocols.haproxy.test.test_parser -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Parser for 'haproxy:' string endpoint. +""" + +from zope.interface import implementer +from twisted.plugin import IPlugin + +from twisted.internet.endpoints import ( + quoteStringArgument, serverFromString, IStreamServerEndpointStringParser +) +from twisted.python.compat import iteritems + +from . import proxyEndpoint + + +def unparseEndpoint(args, kwargs): + """ + Un-parse the already-parsed args and kwargs back into endpoint syntax. + + @param args: C{:}-separated arguments + @type args: L{tuple} of native L{str} + + @param kwargs: C{:} and then C{=}-separated keyword arguments + + @type arguments: L{tuple} of native L{str} + + @return: a string equivalent to the original format which this was parsed + as. + @rtype: native L{str} + """ + + description = ':'.join( + [quoteStringArgument(str(arg)) for arg in args] + + sorted(['%s=%s' % (quoteStringArgument(str(key)), + quoteStringArgument(str(value))) + for key, value in iteritems(kwargs) + ])) + return description + + + +@implementer(IPlugin, IStreamServerEndpointStringParser) +class HAProxyServerParser(object): + """ + Stream server endpoint string parser for the HAProxyServerEndpoint type. + + @ivar prefix: See L{IStreamServerEndpointStringParser.prefix}. + """ + prefix = "haproxy" + + def parseStreamServer(self, reactor, *args, **kwargs): + """ + Parse a stream server endpoint from a reactor and string-only arguments + and keyword arguments. + + @param reactor: The reactor. + + @param args: The parsed string arguments. + + @param kwargs: The parsed keyword arguments. + + @return: a stream server endpoint + @rtype: L{IStreamServerEndpoint} + """ + subdescription = unparseEndpoint(args, kwargs) + wrappedEndpoint = serverFromString(reactor, subdescription) + return proxyEndpoint(wrappedEndpoint) diff --git a/contrib/python/Twisted/py2/twisted/protocols/haproxy/_v1parser.py b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_v1parser.py new file mode 100644 index 00000000000..b17099f3cc3 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_v1parser.py @@ -0,0 +1,143 @@ +# -*- test-case-name: twisted.protocols.haproxy.test.test_v1parser -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +IProxyParser implementation for version one of the PROXY protocol. +""" + +from zope.interface import implementer +from twisted.internet import address + +from ._exceptions import ( + convertError, InvalidProxyHeader, InvalidNetworkProtocol, + MissingAddressData +) +from . import _info +from . import _interfaces + + + +@implementer(_interfaces.IProxyParser) +class V1Parser(object): + """ + PROXY protocol version one header parser. + + Version one of the PROXY protocol is a human readable format represented + by a single, newline delimited binary string that contains all of the + relevant source and destination data. + """ + + PROXYSTR = b'PROXY' + UNKNOWN_PROTO = b'UNKNOWN' + TCP4_PROTO = b'TCP4' + TCP6_PROTO = b'TCP6' + ALLOWED_NET_PROTOS = ( + TCP4_PROTO, + TCP6_PROTO, + UNKNOWN_PROTO, + ) + NEWLINE = b'\r\n' + + def __init__(self): + self.buffer = b'' + + + def feed(self, data): + """ + Consume a chunk of data and attempt to parse it. + + @param data: A bytestring. + @type data: L{bytes} + + @return: A two-tuple containing, in order, a + L{_interfaces.IProxyInfo} and any bytes fed to the + parser that followed the end of the header. Both of these values + are None until a complete header is parsed. + + @raises InvalidProxyHeader: If the bytes fed to the parser create an + invalid PROXY header. + """ + self.buffer += data + if len(self.buffer) > 107 and self.NEWLINE not in self.buffer: + raise InvalidProxyHeader() + lines = (self.buffer).split(self.NEWLINE, 1) + if not len(lines) > 1: + return (None, None) + self.buffer = b'' + remaining = lines.pop() + header = lines.pop() + info = self.parse(header) + return (info, remaining) + + + @classmethod + def parse(cls, line): + """ + Parse a bytestring as a full PROXY protocol header line. + + @param line: A bytestring that represents a valid HAProxy PROXY + protocol header line. + @type line: bytes + + @return: A L{_interfaces.IProxyInfo} containing the parsed data. + + @raises InvalidProxyHeader: If the bytestring does not represent a + valid PROXY header. + + @raises InvalidNetworkProtocol: When no protocol can be parsed or is + not one of the allowed values. + + @raises MissingAddressData: When the protocol is TCP* but the header + does not contain a complete set of addresses and ports. + """ + originalLine = line + proxyStr = None + networkProtocol = None + sourceAddr = None + sourcePort = None + destAddr = None + destPort = None + + with convertError(ValueError, InvalidProxyHeader): + proxyStr, line = line.split(b' ', 1) + + if proxyStr != cls.PROXYSTR: + raise InvalidProxyHeader() + + with convertError(ValueError, InvalidNetworkProtocol): + networkProtocol, line = line.split(b' ', 1) + + if networkProtocol not in cls.ALLOWED_NET_PROTOS: + raise InvalidNetworkProtocol() + + if networkProtocol == cls.UNKNOWN_PROTO: + + return _info.ProxyInfo(originalLine, None, None) + + with convertError(ValueError, MissingAddressData): + sourceAddr, line = line.split(b' ', 1) + + with convertError(ValueError, MissingAddressData): + destAddr, line = line.split(b' ', 1) + + with convertError(ValueError, MissingAddressData): + sourcePort, line = line.split(b' ', 1) + + with convertError(ValueError, MissingAddressData): + destPort = line.split(b' ')[0] + + if networkProtocol == cls.TCP4_PROTO: + + return _info.ProxyInfo( + originalLine, + address.IPv4Address('TCP', sourceAddr, int(sourcePort)), + address.IPv4Address('TCP', destAddr, int(destPort)), + ) + + return _info.ProxyInfo( + originalLine, + address.IPv6Address('TCP', sourceAddr, int(sourcePort)), + address.IPv6Address('TCP', destAddr, int(destPort)), + ) diff --git a/contrib/python/Twisted/py2/twisted/protocols/haproxy/_v2parser.py b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_v2parser.py new file mode 100644 index 00000000000..94c495ffe20 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_v2parser.py @@ -0,0 +1,215 @@ +# -*- test-case-name: twisted.protocols.haproxy.test.test_v2parser -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +IProxyParser implementation for version two of the PROXY protocol. +""" + +import binascii +import struct + +from constantly import Values, ValueConstant + +from zope.interface import implementer +from twisted.internet import address +from twisted.python import compat + +from ._exceptions import ( + convertError, InvalidProxyHeader, InvalidNetworkProtocol, + MissingAddressData +) +from . import _info +from . import _interfaces + +class NetFamily(Values): + """ + Values for the 'family' field. + """ + UNSPEC = ValueConstant(0x00) + INET = ValueConstant(0x10) + INET6 = ValueConstant(0x20) + UNIX = ValueConstant(0x30) + + + +class NetProtocol(Values): + """ + Values for 'protocol' field. + """ + UNSPEC = ValueConstant(0) + STREAM = ValueConstant(1) + DGRAM = ValueConstant(2) + + +_HIGH = 0b11110000 +_LOW = 0b00001111 +_LOCALCOMMAND = 'LOCAL' +_PROXYCOMMAND = 'PROXY' + +@implementer(_interfaces.IProxyParser) +class V2Parser(object): + """ + PROXY protocol version two header parser. + + Version two of the PROXY protocol is a binary format. + """ + + PREFIX = b'\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A' + VERSIONS = [32] + COMMANDS = {0: _LOCALCOMMAND, 1: _PROXYCOMMAND} + ADDRESSFORMATS = { + # TCP4 + 17: '!4s4s2H', + 18: '!4s4s2H', + # TCP6 + 33: '!16s16s2H', + 34: '!16s16s2H', + # UNIX + 49: '!108s108s', + 50: '!108s108s', + } + + def __init__(self): + self.buffer = b'' + + + def feed(self, data): + """ + Consume a chunk of data and attempt to parse it. + + @param data: A bytestring. + @type data: bytes + + @return: A two-tuple containing, in order, a L{_interfaces.IProxyInfo} + and any bytes fed to the parser that followed the end of the + header. Both of these values are None until a complete header is + parsed. + + @raises InvalidProxyHeader: If the bytes fed to the parser create an + invalid PROXY header. + """ + self.buffer += data + if len(self.buffer) < 16: + raise InvalidProxyHeader() + + size = struct.unpack('!H', self.buffer[14:16])[0] + 16 + if len(self.buffer) < size: + return (None, None) + + header, remaining = self.buffer[:size], self.buffer[size:] + self.buffer = b'' + info = self.parse(header) + return (info, remaining) + + + @staticmethod + def _bytesToIPv4(bytestring): + """ + Convert packed 32-bit IPv4 address bytes into a dotted-quad ASCII bytes + representation of that address. + + @param bytestring: 4 octets representing an IPv4 address. + @type bytestring: L{bytes} + + @return: a dotted-quad notation IPv4 address. + @rtype: L{bytes} + """ + return b'.'.join( + ('%i' % (ord(b),)).encode('ascii') + for b in compat.iterbytes(bytestring) + ) + + + @staticmethod + def _bytesToIPv6(bytestring): + """ + Convert packed 128-bit IPv6 address bytes into a colon-separated ASCII + bytes representation of that address. + + @param bytestring: 16 octets representing an IPv6 address. + @type bytestring: L{bytes} + + @return: a dotted-quad notation IPv6 address. + @rtype: L{bytes} + """ + hexString = binascii.b2a_hex(bytestring) + return b':'.join( + ('%x' % (int(hexString[b:b+4], 16),)).encode('ascii') + for b in range(0, 32, 4) + ) + + + @classmethod + def parse(cls, line): + """ + Parse a bytestring as a full PROXY protocol header. + + @param line: A bytestring that represents a valid HAProxy PROXY + protocol version 2 header. + @type line: bytes + + @return: A L{_interfaces.IProxyInfo} containing the + parsed data. + + @raises InvalidProxyHeader: If the bytestring does not represent a + valid PROXY header. + """ + prefix = line[:12] + addrInfo = None + with convertError(IndexError, InvalidProxyHeader): + # Use single value slices to ensure bytestring values are returned + # instead of int in PY3. + versionCommand = ord(line[12:13]) + familyProto = ord(line[13:14]) + + if prefix != cls.PREFIX: + raise InvalidProxyHeader() + + version, command = versionCommand & _HIGH, versionCommand & _LOW + if version not in cls.VERSIONS or command not in cls.COMMANDS: + raise InvalidProxyHeader() + + if cls.COMMANDS[command] == _LOCALCOMMAND: + return _info.ProxyInfo(line, None, None) + + family, netproto = familyProto & _HIGH, familyProto & _LOW + with convertError(ValueError, InvalidNetworkProtocol): + family = NetFamily.lookupByValue(family) + netproto = NetProtocol.lookupByValue(netproto) + if ( + family is NetFamily.UNSPEC or + netproto is NetProtocol.UNSPEC + ): + return _info.ProxyInfo(line, None, None) + + addressFormat = cls.ADDRESSFORMATS[familyProto] + addrInfo = line[16:16+struct.calcsize(addressFormat)] + if family is NetFamily.UNIX: + with convertError(struct.error, MissingAddressData): + source, dest = struct.unpack(addressFormat, addrInfo) + return _info.ProxyInfo( + line, + address.UNIXAddress(source.rstrip(b'\x00')), + address.UNIXAddress(dest.rstrip(b'\x00')), + ) + + addrType = 'TCP' + if netproto is NetProtocol.DGRAM: + addrType = 'UDP' + addrCls = address.IPv4Address + addrParser = cls._bytesToIPv4 + if family is NetFamily.INET6: + addrCls = address.IPv6Address + addrParser = cls._bytesToIPv6 + + with convertError(struct.error, MissingAddressData): + info = struct.unpack(addressFormat, addrInfo) + source, dest, sPort, dPort = info + + return _info.ProxyInfo( + line, + addrCls(addrType, addrParser(source), sPort), + addrCls(addrType, addrParser(dest), dPort), + ) diff --git a/contrib/python/Twisted/py2/twisted/protocols/haproxy/_wrapper.py b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_wrapper.py new file mode 100644 index 00000000000..a6e98892f39 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/haproxy/_wrapper.py @@ -0,0 +1,106 @@ +# -*- test-case-name: twisted.protocols.haproxy.test.test_wrapper -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Protocol wrapper that provides HAProxy PROXY protocol support. +""" + +from twisted.protocols import policies +from twisted.internet import interfaces +from twisted.internet.endpoints import _WrapperServerEndpoint + +from ._exceptions import InvalidProxyHeader +from ._v1parser import V1Parser +from ._v2parser import V2Parser + + + +class HAProxyProtocolWrapper(policies.ProtocolWrapper, object): + """ + A Protocol wrapper that provides HAProxy support. + + This protocol reads the PROXY stream header, v1 or v2, parses the provided + connection data, and modifies the behavior of getPeer and getHost to return + the data provided by the PROXY header. + """ + + def __init__(self, factory, wrappedProtocol): + policies.ProtocolWrapper.__init__(self, factory, wrappedProtocol) + self._proxyInfo = None + self._parser = None + + + def dataReceived(self, data): + if self._proxyInfo is not None: + return self.wrappedProtocol.dataReceived(data) + + if self._parser is None: + if ( + len(data) >= 16 and + data[:12] == V2Parser.PREFIX and + ord(data[12:13]) & 0b11110000 == 0x20 + ): + self._parser = V2Parser() + elif len(data) >= 8 and data[:5] == V1Parser.PROXYSTR: + self._parser = V1Parser() + else: + self.loseConnection() + return None + + try: + self._proxyInfo, remaining = self._parser.feed(data) + if remaining: + self.wrappedProtocol.dataReceived(remaining) + except InvalidProxyHeader: + self.loseConnection() + + + def getPeer(self): + if self._proxyInfo and self._proxyInfo.source: + return self._proxyInfo.source + return self.transport.getPeer() + + + def getHost(self): + if self._proxyInfo and self._proxyInfo.destination: + return self._proxyInfo.destination + return self.transport.getHost() + + + +class HAProxyWrappingFactory(policies.WrappingFactory): + """ + A Factory wrapper that adds PROXY protocol support to connections. + """ + protocol = HAProxyProtocolWrapper + + def logPrefix(self): + """ + Annotate the wrapped factory's log prefix with some text indicating + the PROXY protocol is in use. + + @rtype: C{str} + """ + if interfaces.ILoggingContext.providedBy(self.wrappedFactory): + logPrefix = self.wrappedFactory.logPrefix() + else: + logPrefix = self.wrappedFactory.__class__.__name__ + return "%s (PROXY)" % (logPrefix,) + + + +def proxyEndpoint(wrappedEndpoint): + """ + Wrap an endpoint with PROXY protocol support, so that the transport's + C{getHost} and C{getPeer} methods reflect the attributes of the proxied + connection rather than the underlying connection. + + @param wrappedEndpoint: The underlying listening endpoint. + @type wrappedEndpoint: L{IStreamServerEndpoint} + + @return: a new listening endpoint that speaks the PROXY protocol. + @rtype: L{IStreamServerEndpoint} + """ + return _WrapperServerEndpoint(wrappedEndpoint, HAProxyWrappingFactory) diff --git a/contrib/python/Twisted/py2/twisted/protocols/htb.py b/contrib/python/Twisted/py2/twisted/protocols/htb.py new file mode 100644 index 00000000000..22a9299bc6b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/htb.py @@ -0,0 +1,295 @@ +# -*- test-case-name: twisted.test.test_htb -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Hierarchical Token Bucket traffic shaping. + +Patterned after U{Martin Devera's Hierarchical Token Bucket traffic +shaper for the Linux kernel}. + +@seealso: U{HTB Linux queuing discipline manual - user guide + } +@seealso: U{Token Bucket Filter in Linux Advanced Routing & Traffic Control + HOWTO} +""" + + +# TODO: Investigate whether we should be using os.times()[-1] instead of +# time.time. time.time, it has been pointed out, can go backwards. Is +# the same true of os.times? +from time import time +from zope.interface import implementer, Interface + +from twisted.protocols import pcp + + +class Bucket: + """ + Implementation of a Token bucket. + + A bucket can hold a certain number of tokens and it drains over time. + + @cvar maxburst: The maximum number of tokens that the bucket can + hold at any given time. If this is L{None}, the bucket has + an infinite size. + @type maxburst: C{int} + @cvar rate: The rate at which the bucket drains, in number + of tokens per second. If the rate is L{None}, the bucket + drains instantaneously. + @type rate: C{int} + """ + + maxburst = None + rate = None + + _refcount = 0 + + def __init__(self, parentBucket=None): + """ + Create a L{Bucket} that may have a parent L{Bucket}. + + @param parentBucket: If a parent Bucket is specified, + all L{add} and L{drip} operations on this L{Bucket} + will be applied on the parent L{Bucket} as well. + @type parentBucket: L{Bucket} + """ + self.content = 0 + self.parentBucket = parentBucket + self.lastDrip = time() + + + def add(self, amount): + """ + Adds tokens to the L{Bucket} and its C{parentBucket}. + + This will add as many of the C{amount} tokens as will fit into both + this L{Bucket} and its C{parentBucket}. + + @param amount: The number of tokens to try to add. + @type amount: C{int} + + @returns: The number of tokens that actually fit. + @returntype: C{int} + """ + self.drip() + if self.maxburst is None: + allowable = amount + else: + allowable = min(amount, self.maxburst - self.content) + + if self.parentBucket is not None: + allowable = self.parentBucket.add(allowable) + self.content += allowable + return allowable + + + def drip(self): + """ + Let some of the bucket drain. + + The L{Bucket} drains at the rate specified by the class + variable C{rate}. + + @returns: C{True} if the bucket is empty after this drip. + @returntype: C{bool} + """ + if self.parentBucket is not None: + self.parentBucket.drip() + + if self.rate is None: + self.content = 0 + else: + now = time() + deltaTime = now - self.lastDrip + deltaTokens = deltaTime * self.rate + self.content = max(0, self.content - deltaTokens) + self.lastDrip = now + return self.content == 0 + + +class IBucketFilter(Interface): + def getBucketFor(*somethings, **some_kw): + """ + Return a L{Bucket} corresponding to the provided parameters. + + @returntype: L{Bucket} + """ + +@implementer(IBucketFilter) +class HierarchicalBucketFilter: + """ + Filter things into buckets that can be nested. + + @cvar bucketFactory: Class of buckets to make. + @type bucketFactory: L{Bucket} + @cvar sweepInterval: Seconds between sweeping out the bucket cache. + @type sweepInterval: C{int} + """ + bucketFactory = Bucket + sweepInterval = None + + def __init__(self, parentFilter=None): + self.buckets = {} + self.parentFilter = parentFilter + self.lastSweep = time() + + def getBucketFor(self, *a, **kw): + """ + Find or create a L{Bucket} corresponding to the provided parameters. + + Any parameters are passed on to L{getBucketKey}, from them it + decides which bucket you get. + + @returntype: L{Bucket} + """ + if ((self.sweepInterval is not None) + and ((time() - self.lastSweep) > self.sweepInterval)): + self.sweep() + + if self.parentFilter: + parentBucket = self.parentFilter.getBucketFor(self, *a, **kw) + else: + parentBucket = None + + key = self.getBucketKey(*a, **kw) + bucket = self.buckets.get(key) + if bucket is None: + bucket = self.bucketFactory(parentBucket) + self.buckets[key] = bucket + return bucket + + def getBucketKey(self, *a, **kw): + """ + Construct a key based on the input parameters to choose a L{Bucket}. + + The default implementation returns the same key for all + arguments. Override this method to provide L{Bucket} selection. + + @returns: Something to be used as a key in the bucket cache. + """ + return None + + def sweep(self): + """ + Remove empty buckets. + """ + for key, bucket in self.buckets.items(): + bucket_is_empty = bucket.drip() + if (bucket._refcount == 0) and bucket_is_empty: + del self.buckets[key] + + self.lastSweep = time() + + +class FilterByHost(HierarchicalBucketFilter): + """ + A Hierarchical Bucket filter with a L{Bucket} for each host. + """ + sweepInterval = 60 * 20 + + def getBucketKey(self, transport): + return transport.getPeer()[1] + + +class FilterByServer(HierarchicalBucketFilter): + """ + A Hierarchical Bucket filter with a L{Bucket} for each service. + """ + sweepInterval = None + + def getBucketKey(self, transport): + return transport.getHost()[2] + + +class ShapedConsumer(pcp.ProducerConsumerProxy): + """ + Wraps a C{Consumer} and shapes the rate at which it receives data. + """ + # Providing a Pull interface means I don't have to try to schedule + # traffic with callLaters. + iAmStreaming = False + + def __init__(self, consumer, bucket): + pcp.ProducerConsumerProxy.__init__(self, consumer) + self.bucket = bucket + self.bucket._refcount += 1 + + def _writeSomeData(self, data): + # In practice, this actually results in obscene amounts of + # overhead, as a result of generating lots and lots of packets + # with twelve-byte payloads. We may need to do a version of + # this with scheduled writes after all. + amount = self.bucket.add(len(data)) + return pcp.ProducerConsumerProxy._writeSomeData(self, data[:amount]) + + def stopProducing(self): + pcp.ProducerConsumerProxy.stopProducing(self) + self.bucket._refcount -= 1 + + +class ShapedTransport(ShapedConsumer): + """ + Wraps a C{Transport} and shapes the rate at which it receives data. + + This is a L{ShapedConsumer} with a little bit of magic to provide for + the case where the consumer it wraps is also a C{Transport} and people + will be attempting to access attributes this does not proxy as a + C{Consumer} (e.g. C{loseConnection}). + """ + # Ugh. We only wanted to filter IConsumer, not ITransport. + + iAmStreaming = False + def __getattr__(self, name): + # Because people will be doing things like .getPeer and + # .loseConnection on me. + return getattr(self.consumer, name) + + +class ShapedProtocolFactory: + """ + Dispense C{Protocols} with traffic shaping on their transports. + + Usage:: + + myserver = SomeFactory() + myserver.protocol = ShapedProtocolFactory(myserver.protocol, + bucketFilter) + + Where C{SomeServerFactory} is a L{twisted.internet.protocol.Factory}, and + C{bucketFilter} is an instance of L{HierarchicalBucketFilter}. + """ + def __init__(self, protoClass, bucketFilter): + """ + Tell me what to wrap and where to get buckets. + + @param protoClass: The class of C{Protocol} this will generate + wrapped instances of. + @type protoClass: L{Protocol} + class + @param bucketFilter: The filter which will determine how + traffic is shaped. + @type bucketFilter: L{HierarchicalBucketFilter}. + """ + # More precisely, protoClass can be any callable that will return + # instances of something that implements IProtocol. + self.protocol = protoClass + self.bucketFilter = bucketFilter + + def __call__(self, *a, **kw): + """ + Make a C{Protocol} instance with a shaped transport. + + Any parameters will be passed on to the protocol's initializer. + + @returns: A C{Protocol} instance with a L{ShapedTransport}. + """ + proto = self.protocol(*a, **kw) + origMakeConnection = proto.makeConnection + def makeConnection(transport): + bucket = self.bucketFilter.getBucketFor(transport) + shapedTransport = ShapedTransport(transport, bucket) + return origMakeConnection(shapedTransport) + proto.makeConnection = makeConnection + return proto diff --git a/contrib/python/Twisted/py2/twisted/protocols/ident.py b/contrib/python/Twisted/py2/twisted/protocols/ident.py new file mode 100644 index 00000000000..69128b4326d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/ident.py @@ -0,0 +1,255 @@ +# -*- test-case-name: twisted.test.test_ident -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Ident protocol implementation. +""" + +import struct + +from twisted.internet import defer +from twisted.protocols import basic +from twisted.python import log, failure + +_MIN_PORT = 1 +_MAX_PORT = 2 ** 16 - 1 + +class IdentError(Exception): + """ + Can't determine connection owner; reason unknown. + """ + + identDescription = 'UNKNOWN-ERROR' + + def __str__(self): + return self.identDescription + + + +class NoUser(IdentError): + """ + The connection specified by the port pair is not currently in use or + currently not owned by an identifiable entity. + """ + identDescription = 'NO-USER' + + + +class InvalidPort(IdentError): + """ + Either the local or foreign port was improperly specified. This should + be returned if either or both of the port ids were out of range (TCP + port numbers are from 1-65535), negative integers, reals or in any + fashion not recognized as a non-negative integer. + """ + identDescription = 'INVALID-PORT' + + + +class HiddenUser(IdentError): + """ + The server was able to identify the user of this port, but the + information was not returned at the request of the user. + """ + identDescription = 'HIDDEN-USER' + + + +class IdentServer(basic.LineOnlyReceiver): + """ + The Identification Protocol (a.k.a., "ident", a.k.a., "the Ident + Protocol") provides a means to determine the identity of a user of a + particular TCP connection. Given a TCP port number pair, it returns a + character string which identifies the owner of that connection on the + server's system. + + Server authors should subclass this class and override the lookup method. + The default implementation returns an UNKNOWN-ERROR response for every + query. + """ + + def lineReceived(self, line): + parts = line.split(',') + if len(parts) != 2: + self.invalidQuery() + else: + try: + portOnServer, portOnClient = map(int, parts) + except ValueError: + self.invalidQuery() + else: + if _MIN_PORT <= portOnServer <= _MAX_PORT and _MIN_PORT <= portOnClient <= _MAX_PORT: + self.validQuery(portOnServer, portOnClient) + else: + self._ebLookup(failure.Failure(InvalidPort()), portOnServer, portOnClient) + + + def invalidQuery(self): + self.transport.loseConnection() + + + def validQuery(self, portOnServer, portOnClient): + """ + Called when a valid query is received to look up and deliver the + response. + + @param portOnServer: The server port from the query. + @param portOnClient: The client port from the query. + """ + serverAddr = self.transport.getHost().host, portOnServer + clientAddr = self.transport.getPeer().host, portOnClient + defer.maybeDeferred(self.lookup, serverAddr, clientAddr + ).addCallback(self._cbLookup, portOnServer, portOnClient + ).addErrback(self._ebLookup, portOnServer, portOnClient + ) + + + def _cbLookup(self, result, sport, cport): + (sysName, userId) = result + self.sendLine('%d, %d : USERID : %s : %s' % (sport, cport, sysName, userId)) + + + def _ebLookup(self, failure, sport, cport): + if failure.check(IdentError): + self.sendLine('%d, %d : ERROR : %s' % (sport, cport, failure.value)) + else: + log.err(failure) + self.sendLine('%d, %d : ERROR : %s' % (sport, cport, IdentError(failure.value))) + + + def lookup(self, serverAddress, clientAddress): + """ + Lookup user information about the specified address pair. + + Return value should be a two-tuple of system name and username. + Acceptable values for the system name may be found online at:: + + U{http://www.iana.org/assignments/operating-system-names} + + This method may also raise any IdentError subclass (or IdentError + itself) to indicate user information will not be provided for the + given query. + + A Deferred may also be returned. + + @param serverAddress: A two-tuple representing the server endpoint + of the address being queried. The first element is a string holding + a dotted-quad IP address. The second element is an integer + representing the port. + + @param clientAddress: Like I{serverAddress}, but represents the + client endpoint of the address being queried. + """ + raise IdentError() + + + +class ProcServerMixin: + """Implements lookup() to grab entries for responses from /proc/net/tcp + """ + + SYSTEM_NAME = 'LINUX' + + try: + from pwd import getpwuid + def getUsername(self, uid, getpwuid=getpwuid): + return getpwuid(uid)[0] + del getpwuid + except ImportError: + def getUsername(self, uid): + raise IdentError() + + + def entries(self): + with open('/proc/net/tcp') as f: + f.readline() + for L in f: + yield L.strip() + + + def dottedQuadFromHexString(self, hexstr): + return '.'.join(map(str, struct.unpack('4B', struct.pack('=L', int(hexstr, 16))))) + + + def unpackAddress(self, packed): + addr, port = packed.split(':') + addr = self.dottedQuadFromHexString(addr) + port = int(port, 16) + return addr, port + + + def parseLine(self, line): + parts = line.strip().split() + localAddr, localPort = self.unpackAddress(parts[1]) + remoteAddr, remotePort = self.unpackAddress(parts[2]) + uid = int(parts[7]) + return (localAddr, localPort), (remoteAddr, remotePort), uid + + + def lookup(self, serverAddress, clientAddress): + for ent in self.entries(): + localAddr, remoteAddr, uid = self.parseLine(ent) + if remoteAddr == clientAddress and localAddr[1] == serverAddress[1]: + return (self.SYSTEM_NAME, self.getUsername(uid)) + + raise NoUser() + + + +class IdentClient(basic.LineOnlyReceiver): + + errorTypes = (IdentError, NoUser, InvalidPort, HiddenUser) + + def __init__(self): + self.queries = [] + + + def lookup(self, portOnServer, portOnClient): + """ + Lookup user information about the specified address pair. + """ + self.queries.append((defer.Deferred(), portOnServer, portOnClient)) + if len(self.queries) > 1: + return self.queries[-1][0] + + self.sendLine('%d, %d' % (portOnServer, portOnClient)) + return self.queries[-1][0] + + + def lineReceived(self, line): + if not self.queries: + log.msg("Unexpected server response: %r" % (line,)) + else: + d, _, _ = self.queries.pop(0) + self.parseResponse(d, line) + if self.queries: + self.sendLine('%d, %d' % (self.queries[0][1], self.queries[0][2])) + + + def connectionLost(self, reason): + for q in self.queries: + q[0].errback(IdentError(reason)) + self.queries = [] + + + def parseResponse(self, deferred, line): + parts = line.split(':', 2) + if len(parts) != 3: + deferred.errback(IdentError(line)) + else: + ports, type, addInfo = map(str.strip, parts) + if type == 'ERROR': + for et in self.errorTypes: + if et.identDescription == addInfo: + deferred.errback(et(line)) + return + deferred.errback(IdentError(line)) + else: + deferred.callback((type, addInfo)) + + + +__all__ = ['IdentError', 'NoUser', 'InvalidPort', 'HiddenUser', + 'IdentServer', 'IdentClient', + 'ProcServerMixin'] diff --git a/contrib/python/Twisted/py2/twisted/protocols/loopback.py b/contrib/python/Twisted/py2/twisted/protocols/loopback.py new file mode 100644 index 00000000000..9d7beb0ce44 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/loopback.py @@ -0,0 +1,385 @@ +# -*- test-case-name: twisted.test.test_loopback -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Testing support for protocols -- loopback between client and server. +""" + +from __future__ import division, absolute_import + +# system imports +import tempfile + +from zope.interface import implementer + +# Twisted Imports +from twisted.protocols import policies +from twisted.internet import interfaces, protocol, main, defer +from twisted.internet.task import deferLater +from twisted.python import failure +from twisted.internet.interfaces import IAddress + + +class _LoopbackQueue(object): + """ + Trivial wrapper around a list to give it an interface like a queue, which + the addition of also sending notifications by way of a Deferred whenever + the list has something added to it. + """ + + _notificationDeferred = None + disconnect = False + + def __init__(self): + self._queue = [] + + + def put(self, v): + self._queue.append(v) + if self._notificationDeferred is not None: + d, self._notificationDeferred = self._notificationDeferred, None + d.callback(None) + + + def __nonzero__(self): + return bool(self._queue) + __bool__ = __nonzero__ + + + def get(self): + return self._queue.pop(0) + + + +@implementer(IAddress) +class _LoopbackAddress(object): + pass + + + +@implementer(interfaces.ITransport, interfaces.IConsumer) +class _LoopbackTransport(object): + disconnecting = False + producer = None + + # ITransport + def __init__(self, q): + self.q = q + + def write(self, data): + if not isinstance(data, bytes): + raise TypeError("Can only write bytes to ITransport") + self.q.put(data) + + def writeSequence(self, iovec): + self.q.put(b''.join(iovec)) + + def loseConnection(self): + self.q.disconnect = True + self.q.put(None) + + + def abortConnection(self): + """ + Abort the connection. Same as L{loseConnection}. + """ + self.loseConnection() + + + def getPeer(self): + return _LoopbackAddress() + + def getHost(self): + return _LoopbackAddress() + + # IConsumer + def registerProducer(self, producer, streaming): + assert self.producer is None + self.producer = producer + self.streamingProducer = streaming + self._pollProducer() + + def unregisterProducer(self): + assert self.producer is not None + self.producer = None + + def _pollProducer(self): + if self.producer is not None and not self.streamingProducer: + self.producer.resumeProducing() + + + +def identityPumpPolicy(queue, target): + """ + L{identityPumpPolicy} is a policy which delivers each chunk of data written + to the given queue as-is to the target. + + This isn't a particularly realistic policy. + + @see: L{loopbackAsync} + """ + while queue: + bytes = queue.get() + if bytes is None: + break + target.dataReceived(bytes) + + + +def collapsingPumpPolicy(queue, target): + """ + L{collapsingPumpPolicy} is a policy which collapses all outstanding chunks + into a single string and delivers it to the target. + + @see: L{loopbackAsync} + """ + bytes = [] + while queue: + chunk = queue.get() + if chunk is None: + break + bytes.append(chunk) + if bytes: + target.dataReceived(b''.join(bytes)) + + + +def loopbackAsync(server, client, pumpPolicy=identityPumpPolicy): + """ + Establish a connection between C{server} and C{client} then transfer data + between them until the connection is closed. This is often useful for + testing a protocol. + + @param server: The protocol instance representing the server-side of this + connection. + + @param client: The protocol instance representing the client-side of this + connection. + + @param pumpPolicy: When either C{server} or C{client} writes to its + transport, the string passed in is added to a queue of data for the + other protocol. Eventually, C{pumpPolicy} will be called with one such + queue and the corresponding protocol object. The pump policy callable + is responsible for emptying the queue and passing the strings it + contains to the given protocol's C{dataReceived} method. The signature + of C{pumpPolicy} is C{(queue, protocol)}. C{queue} is an object with a + C{get} method which will return the next string written to the + transport, or L{None} if the transport has been disconnected, and which + evaluates to C{True} if and only if there are more items to be + retrieved via C{get}. + + @return: A L{Deferred} which fires when the connection has been closed and + both sides have received notification of this. + """ + serverToClient = _LoopbackQueue() + clientToServer = _LoopbackQueue() + + server.makeConnection(_LoopbackTransport(serverToClient)) + client.makeConnection(_LoopbackTransport(clientToServer)) + + return _loopbackAsyncBody( + server, serverToClient, client, clientToServer, pumpPolicy) + + + +def _loopbackAsyncBody(server, serverToClient, client, clientToServer, + pumpPolicy): + """ + Transfer bytes from the output queue of each protocol to the input of the other. + + @param server: The protocol instance representing the server-side of this + connection. + + @param serverToClient: The L{_LoopbackQueue} holding the server's output. + + @param client: The protocol instance representing the client-side of this + connection. + + @param clientToServer: The L{_LoopbackQueue} holding the client's output. + + @param pumpPolicy: See L{loopbackAsync}. + + @return: A L{Deferred} which fires when the connection has been closed and + both sides have received notification of this. + """ + def pump(source, q, target): + sent = False + if q: + pumpPolicy(q, target) + sent = True + if sent and not q: + # A write buffer has now been emptied. Give any producer on that + # side an opportunity to produce more data. + source.transport._pollProducer() + + return sent + + while 1: + disconnect = clientSent = serverSent = False + + # Deliver the data which has been written. + serverSent = pump(server, serverToClient, client) + clientSent = pump(client, clientToServer, server) + + if not clientSent and not serverSent: + # Neither side wrote any data. Wait for some new data to be added + # before trying to do anything further. + d = defer.Deferred() + clientToServer._notificationDeferred = d + serverToClient._notificationDeferred = d + d.addCallback( + _loopbackAsyncContinue, + server, serverToClient, client, clientToServer, pumpPolicy) + return d + if serverToClient.disconnect: + # The server wants to drop the connection. Flush any remaining + # data it has. + disconnect = True + pump(server, serverToClient, client) + elif clientToServer.disconnect: + # The client wants to drop the connection. Flush any remaining + # data it has. + disconnect = True + pump(client, clientToServer, server) + if disconnect: + # Someone wanted to disconnect, so okay, the connection is gone. + server.connectionLost(failure.Failure(main.CONNECTION_DONE)) + client.connectionLost(failure.Failure(main.CONNECTION_DONE)) + return defer.succeed(None) + + + +def _loopbackAsyncContinue(ignored, server, serverToClient, client, + clientToServer, pumpPolicy): + # Clear the Deferred from each message queue, since it has already fired + # and cannot be used again. + clientToServer._notificationDeferred = None + serverToClient._notificationDeferred = None + + # Schedule some more byte-pushing to happen. This isn't done + # synchronously because no actual transport can re-enter dataReceived as + # a result of calling write, and doing this synchronously could result + # in that. + from twisted.internet import reactor + return deferLater( + reactor, 0, + _loopbackAsyncBody, + server, serverToClient, client, clientToServer, pumpPolicy) + + + +@implementer(interfaces.ITransport, interfaces.IConsumer) +class LoopbackRelay: + buffer = b'' + shouldLose = 0 + disconnecting = 0 + producer = None + + def __init__(self, target, logFile=None): + self.target = target + self.logFile = logFile + + def write(self, data): + self.buffer = self.buffer + data + if self.logFile: + self.logFile.write("loopback writing %s\n" % repr(data)) + + def writeSequence(self, iovec): + self.write(b"".join(iovec)) + + def clearBuffer(self): + if self.shouldLose == -1: + return + + if self.producer: + self.producer.resumeProducing() + if self.buffer: + if self.logFile: + self.logFile.write("loopback receiving %s\n" % repr(self.buffer)) + buffer = self.buffer + self.buffer = b'' + self.target.dataReceived(buffer) + if self.shouldLose == 1: + self.shouldLose = -1 + self.target.connectionLost(failure.Failure(main.CONNECTION_DONE)) + + def loseConnection(self): + if self.shouldLose != -1: + self.shouldLose = 1 + + def getHost(self): + return 'loopback' + + def getPeer(self): + return 'loopback' + + def registerProducer(self, producer, streaming): + self.producer = producer + + def unregisterProducer(self): + self.producer = None + + def logPrefix(self): + return 'Loopback(%r)' % (self.target.__class__.__name__,) + + + +class LoopbackClientFactory(protocol.ClientFactory): + + def __init__(self, protocol): + self.disconnected = 0 + self.deferred = defer.Deferred() + self.protocol = protocol + + def buildProtocol(self, addr): + return self.protocol + + def clientConnectionLost(self, connector, reason): + self.disconnected = 1 + self.deferred.callback(None) + + +class _FireOnClose(policies.ProtocolWrapper): + def __init__(self, protocol, factory): + policies.ProtocolWrapper.__init__(self, protocol, factory) + self.deferred = defer.Deferred() + + def connectionLost(self, reason): + policies.ProtocolWrapper.connectionLost(self, reason) + self.deferred.callback(None) + + +def loopbackTCP(server, client, port=0, noisy=True): + """Run session between server and client protocol instances over TCP.""" + from twisted.internet import reactor + f = policies.WrappingFactory(protocol.Factory()) + serverWrapper = _FireOnClose(f, server) + f.noisy = noisy + f.buildProtocol = lambda addr: serverWrapper + serverPort = reactor.listenTCP(port, f, interface='127.0.0.1') + clientF = LoopbackClientFactory(client) + clientF.noisy = noisy + reactor.connectTCP('127.0.0.1', serverPort.getHost().port, clientF) + d = clientF.deferred + d.addCallback(lambda x: serverWrapper.deferred) + d.addCallback(lambda x: serverPort.stopListening()) + return d + + +def loopbackUNIX(server, client, noisy=True): + """Run session between server and client protocol instances over UNIX socket.""" + path = tempfile.mktemp() + from twisted.internet import reactor + f = policies.WrappingFactory(protocol.Factory()) + serverWrapper = _FireOnClose(f, server) + f.noisy = noisy + f.buildProtocol = lambda addr: serverWrapper + serverPort = reactor.listenUNIX(path, f) + clientF = LoopbackClientFactory(client) + clientF.noisy = noisy + reactor.connectUNIX(path, clientF) + d = clientF.deferred + d.addCallback(lambda x: serverWrapper.deferred) + d.addCallback(lambda x: serverPort.stopListening()) + return d diff --git a/contrib/python/Twisted/py2/twisted/protocols/memcache.py b/contrib/python/Twisted/py2/twisted/protocols/memcache.py new file mode 100644 index 00000000000..6fd666bfd08 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/memcache.py @@ -0,0 +1,766 @@ +# -*- test-case-name: twisted.test.test_memcache -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Memcache client protocol. Memcached is a caching server, storing data in the +form of pairs key/value, and memcache is the protocol to talk with it. + +To connect to a server, create a factory for L{MemCacheProtocol}:: + + from twisted.internet import reactor, protocol + from twisted.protocols.memcache import MemCacheProtocol, DEFAULT_PORT + d = protocol.ClientCreator(reactor, MemCacheProtocol + ).connectTCP("localhost", DEFAULT_PORT) + def doSomething(proto): + # Here you call the memcache operations + return proto.set("mykey", "a lot of data") + d.addCallback(doSomething) + reactor.run() + +All the operations of the memcache protocol are present, but +L{MemCacheProtocol.set} and L{MemCacheProtocol.get} are the more important. + +See U{http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt} for +more information about the protocol. +""" + +from __future__ import absolute_import, division + +from collections import deque + +from twisted.protocols.basic import LineReceiver +from twisted.protocols.policies import TimeoutMixin +from twisted.internet.defer import Deferred, fail, TimeoutError +from twisted.python import log +from twisted.python.compat import ( + intToBytes, iteritems, nativeString, networkString) + + + +DEFAULT_PORT = 11211 + + + +class NoSuchCommand(Exception): + """ + Exception raised when a non existent command is called. + """ + + + +class ClientError(Exception): + """ + Error caused by an invalid client call. + """ + + + +class ServerError(Exception): + """ + Problem happening on the server. + """ + + + +class Command(object): + """ + Wrap a client action into an object, that holds the values used in the + protocol. + + @ivar _deferred: the L{Deferred} object that will be fired when the result + arrives. + @type _deferred: L{Deferred} + + @ivar command: name of the command sent to the server. + @type command: L{bytes} + """ + + def __init__(self, command, **kwargs): + """ + Create a command. + + @param command: the name of the command. + @type command: L{bytes} + + @param kwargs: this values will be stored as attributes of the object + for future use + """ + self.command = command + self._deferred = Deferred() + for k, v in kwargs.items(): + setattr(self, k, v) + + + def success(self, value): + """ + Shortcut method to fire the underlying deferred. + """ + self._deferred.callback(value) + + + def fail(self, error): + """ + Make the underlying deferred fails. + """ + self._deferred.errback(error) + + + +class MemCacheProtocol(LineReceiver, TimeoutMixin): + """ + MemCache protocol: connect to a memcached server to store/retrieve values. + + @ivar persistentTimeOut: the timeout period used to wait for a response. + @type persistentTimeOut: L{int} + + @ivar _current: current list of requests waiting for an answer from the + server. + @type _current: L{deque} of L{Command} + + @ivar _lenExpected: amount of data expected in raw mode, when reading for + a value. + @type _lenExpected: L{int} + + @ivar _getBuffer: current buffer of data, used to store temporary data + when reading in raw mode. + @type _getBuffer: L{list} + + @ivar _bufferLength: the total amount of bytes in C{_getBuffer}. + @type _bufferLength: L{int} + + @ivar _disconnected: indicate if the connectionLost has been called or not. + @type _disconnected: L{bool} + """ + MAX_KEY_LENGTH = 250 + _disconnected = False + + def __init__(self, timeOut=60): + """ + Create the protocol. + + @param timeOut: the timeout to wait before detecting that the + connection is dead and close it. It's expressed in seconds. + @type timeOut: L{int} + """ + self._current = deque() + self._lenExpected = None + self._getBuffer = None + self._bufferLength = None + self.persistentTimeOut = self.timeOut = timeOut + + + def _cancelCommands(self, reason): + """ + Cancel all the outstanding commands, making them fail with C{reason}. + """ + while self._current: + cmd = self._current.popleft() + cmd.fail(reason) + + + def timeoutConnection(self): + """ + Close the connection in case of timeout. + """ + self._cancelCommands(TimeoutError("Connection timeout")) + self.transport.loseConnection() + + + def connectionLost(self, reason): + """ + Cause any outstanding commands to fail. + """ + self._disconnected = True + self._cancelCommands(reason) + LineReceiver.connectionLost(self, reason) + + + def sendLine(self, line): + """ + Override sendLine to add a timeout to response. + """ + if not self._current: + self.setTimeout(self.persistentTimeOut) + LineReceiver.sendLine(self, line) + + + def rawDataReceived(self, data): + """ + Collect data for a get. + """ + self.resetTimeout() + self._getBuffer.append(data) + self._bufferLength += len(data) + if self._bufferLength >= self._lenExpected + 2: + data = b"".join(self._getBuffer) + buf = data[:self._lenExpected] + rem = data[self._lenExpected + 2:] + val = buf + self._lenExpected = None + self._getBuffer = None + self._bufferLength = None + cmd = self._current[0] + if cmd.multiple: + flags, cas = cmd.values[cmd.currentKey] + cmd.values[cmd.currentKey] = (flags, cas, val) + else: + cmd.value = val + self.setLineMode(rem) + + + def cmd_STORED(self): + """ + Manage a success response to a set operation. + """ + self._current.popleft().success(True) + + + def cmd_NOT_STORED(self): + """ + Manage a specific 'not stored' response to a set operation: this is not + an error, but some condition wasn't met. + """ + self._current.popleft().success(False) + + + def cmd_END(self): + """ + This the end token to a get or a stat operation. + """ + cmd = self._current.popleft() + if cmd.command == b"get": + if cmd.multiple: + values = {key: val[::2] for key, val in iteritems(cmd.values)} + cmd.success(values) + else: + cmd.success((cmd.flags, cmd.value)) + elif cmd.command == b"gets": + if cmd.multiple: + cmd.success(cmd.values) + else: + cmd.success((cmd.flags, cmd.cas, cmd.value)) + elif cmd.command == b"stats": + cmd.success(cmd.values) + else: + raise RuntimeError( + "Unexpected END response to %s command" % + (nativeString(cmd.command),)) + + + def cmd_NOT_FOUND(self): + """ + Manage error response for incr/decr/delete. + """ + self._current.popleft().success(False) + + + def cmd_VALUE(self, line): + """ + Prepare the reading a value after a get. + """ + cmd = self._current[0] + if cmd.command == b"get": + key, flags, length = line.split() + cas = b"" + else: + key, flags, length, cas = line.split() + self._lenExpected = int(length) + self._getBuffer = [] + self._bufferLength = 0 + if cmd.multiple: + if key not in cmd.keys: + raise RuntimeError("Unexpected commands answer.") + cmd.currentKey = key + cmd.values[key] = [int(flags), cas] + else: + if cmd.key != key: + raise RuntimeError("Unexpected commands answer.") + cmd.flags = int(flags) + cmd.cas = cas + self.setRawMode() + + + def cmd_STAT(self, line): + """ + Reception of one stat line. + """ + cmd = self._current[0] + key, val = line.split(b" ", 1) + cmd.values[key] = val + + + def cmd_VERSION(self, versionData): + """ + Read version token. + """ + self._current.popleft().success(versionData) + + + def cmd_ERROR(self): + """ + A non-existent command has been sent. + """ + log.err("Non-existent command sent.") + cmd = self._current.popleft() + cmd.fail(NoSuchCommand()) + + + def cmd_CLIENT_ERROR(self, errText): + """ + An invalid input as been sent. + """ + errText = repr(errText) + log.err("Invalid input: " + errText) + cmd = self._current.popleft() + cmd.fail(ClientError(errText)) + + + def cmd_SERVER_ERROR(self, errText): + """ + An error has happened server-side. + """ + errText = repr(errText) + log.err("Server error: " + errText) + cmd = self._current.popleft() + cmd.fail(ServerError(errText)) + + + def cmd_DELETED(self): + """ + A delete command has completed successfully. + """ + self._current.popleft().success(True) + + + def cmd_OK(self): + """ + The last command has been completed. + """ + self._current.popleft().success(True) + + + def cmd_EXISTS(self): + """ + A C{checkAndSet} update has failed. + """ + self._current.popleft().success(False) + + + def lineReceived(self, line): + """ + Receive line commands from the server. + """ + self.resetTimeout() + token = line.split(b" ", 1)[0] + # First manage standard commands without space + cmd = getattr(self, "cmd_" + nativeString(token), None) + if cmd is not None: + args = line.split(b" ", 1)[1:] + if args: + cmd(args[0]) + else: + cmd() + else: + # Then manage commands with space in it + line = line.replace(b" ", b"_") + cmd = getattr(self, "cmd_" + nativeString(line), None) + if cmd is not None: + cmd() + else: + # Increment/Decrement response + cmd = self._current.popleft() + val = int(line) + cmd.success(val) + if not self._current: + # No pending request, remove timeout + self.setTimeout(None) + + + def increment(self, key, val=1): + """ + Increment the value of C{key} by given value (default to 1). + C{key} must be consistent with an int. Return the new value. + + @param key: the key to modify. + @type key: L{bytes} + + @param val: the value to increment. + @type val: L{int} + + @return: a deferred with will be called back with the new value + associated with the key (after the increment). + @rtype: L{Deferred} + """ + return self._incrdecr(b"incr", key, val) + + + def decrement(self, key, val=1): + """ + Decrement the value of C{key} by given value (default to 1). + C{key} must be consistent with an int. Return the new value, coerced to + 0 if negative. + + @param key: the key to modify. + @type key: L{bytes} + + @param val: the value to decrement. + @type val: L{int} + + @return: a deferred with will be called back with the new value + associated with the key (after the decrement). + @rtype: L{Deferred} + """ + return self._incrdecr(b"decr", key, val) + + + def _incrdecr(self, cmd, key, val): + """ + Internal wrapper for incr/decr. + """ + if self._disconnected: + return fail(RuntimeError("not connected")) + if not isinstance(key, bytes): + return fail(ClientError( + "Invalid type for key: %s, expecting bytes" % (type(key),))) + if len(key) > self.MAX_KEY_LENGTH: + return fail(ClientError("Key too long")) + fullcmd = b" ".join([cmd, key, intToBytes(int(val))]) + self.sendLine(fullcmd) + cmdObj = Command(cmd, key=key) + self._current.append(cmdObj) + return cmdObj._deferred + + + def replace(self, key, val, flags=0, expireTime=0): + """ + Replace the given C{key}. It must already exist in the server. + + @param key: the key to replace. + @type key: L{bytes} + + @param val: the new value associated with the key. + @type val: L{bytes} + + @param flags: the flags to store with the key. + @type flags: L{int} + + @param expireTime: if different from 0, the relative time in seconds + when the key will be deleted from the store. + @type expireTime: L{int} + + @return: a deferred that will fire with C{True} if the operation has + succeeded, and C{False} with the key didn't previously exist. + @rtype: L{Deferred} + """ + return self._set(b"replace", key, val, flags, expireTime, b"") + + + def add(self, key, val, flags=0, expireTime=0): + """ + Add the given C{key}. It must not exist in the server. + + @param key: the key to add. + @type key: L{bytes} + + @param val: the value associated with the key. + @type val: L{bytes} + + @param flags: the flags to store with the key. + @type flags: L{int} + + @param expireTime: if different from 0, the relative time in seconds + when the key will be deleted from the store. + @type expireTime: L{int} + + @return: a deferred that will fire with C{True} if the operation has + succeeded, and C{False} with the key already exists. + @rtype: L{Deferred} + """ + return self._set(b"add", key, val, flags, expireTime, b"") + + + def set(self, key, val, flags=0, expireTime=0): + """ + Set the given C{key}. + + @param key: the key to set. + @type key: L{bytes} + + @param val: the value associated with the key. + @type val: L{bytes} + + @param flags: the flags to store with the key. + @type flags: L{int} + + @param expireTime: if different from 0, the relative time in seconds + when the key will be deleted from the store. + @type expireTime: L{int} + + @return: a deferred that will fire with C{True} if the operation has + succeeded. + @rtype: L{Deferred} + """ + return self._set(b"set", key, val, flags, expireTime, b"") + + + def checkAndSet(self, key, val, cas, flags=0, expireTime=0): + """ + Change the content of C{key} only if the C{cas} value matches the + current one associated with the key. Use this to store a value which + hasn't been modified since last time you fetched it. + + @param key: The key to set. + @type key: L{bytes} + + @param val: The value associated with the key. + @type val: L{bytes} + + @param cas: Unique 64-bit value returned by previous call of C{get}. + @type cas: L{bytes} + + @param flags: The flags to store with the key. + @type flags: L{int} + + @param expireTime: If different from 0, the relative time in seconds + when the key will be deleted from the store. + @type expireTime: L{int} + + @return: A deferred that will fire with C{True} if the operation has + succeeded, C{False} otherwise. + @rtype: L{Deferred} + """ + return self._set(b"cas", key, val, flags, expireTime, cas) + + + def _set(self, cmd, key, val, flags, expireTime, cas): + """ + Internal wrapper for setting values. + """ + if self._disconnected: + return fail(RuntimeError("not connected")) + if not isinstance(key, bytes): + return fail(ClientError( + "Invalid type for key: %s, expecting bytes" % (type(key),))) + if len(key) > self.MAX_KEY_LENGTH: + return fail(ClientError("Key too long")) + if not isinstance(val, bytes): + return fail(ClientError( + "Invalid type for value: %s, expecting bytes" % + (type(val),))) + if cas: + cas = b" " + cas + length = len(val) + fullcmd = b" ".join([ + cmd, key, + networkString("%d %d %d" % (flags, expireTime, length))]) + cas + self.sendLine(fullcmd) + self.sendLine(val) + cmdObj = Command(cmd, key=key, flags=flags, length=length) + self._current.append(cmdObj) + return cmdObj._deferred + + + def append(self, key, val): + """ + Append given data to the value of an existing key. + + @param key: The key to modify. + @type key: L{bytes} + + @param val: The value to append to the current value associated with + the key. + @type val: L{bytes} + + @return: A deferred that will fire with C{True} if the operation has + succeeded, C{False} otherwise. + @rtype: L{Deferred} + """ + # Even if flags and expTime values are ignored, we have to pass them + return self._set(b"append", key, val, 0, 0, b"") + + + def prepend(self, key, val): + """ + Prepend given data to the value of an existing key. + + @param key: The key to modify. + @type key: L{bytes} + + @param val: The value to prepend to the current value associated with + the key. + @type val: L{bytes} + + @return: A deferred that will fire with C{True} if the operation has + succeeded, C{False} otherwise. + @rtype: L{Deferred} + """ + # Even if flags and expTime values are ignored, we have to pass them + return self._set(b"prepend", key, val, 0, 0, b"") + + + def get(self, key, withIdentifier=False): + """ + Get the given C{key}. It doesn't support multiple keys. If + C{withIdentifier} is set to C{True}, the command issued is a C{gets}, + that will return the current identifier associated with the value. This + identifier has to be used when issuing C{checkAndSet} update later, + using the corresponding method. + + @param key: The key to retrieve. + @type key: L{bytes} + + @param withIdentifier: If set to C{True}, retrieve the current + identifier along with the value and the flags. + @type withIdentifier: L{bool} + + @return: A deferred that will fire with the tuple (flags, value) if + C{withIdentifier} is C{False}, or (flags, cas identifier, value) + if C{True}. If the server indicates there is no value + associated with C{key}, the returned value will be L{None} and + the returned flags will be C{0}. + @rtype: L{Deferred} + """ + return self._get([key], withIdentifier, False) + + + def getMultiple(self, keys, withIdentifier=False): + """ + Get the given list of C{keys}. If C{withIdentifier} is set to C{True}, + the command issued is a C{gets}, that will return the identifiers + associated with each values. This identifier has to be used when + issuing C{checkAndSet} update later, using the corresponding method. + + @param keys: The keys to retrieve. + @type keys: L{list} of L{bytes} + + @param withIdentifier: If set to C{True}, retrieve the identifiers + along with the values and the flags. + @type withIdentifier: L{bool} + + @return: A deferred that will fire with a dictionary with the elements + of C{keys} as keys and the tuples (flags, value) as values if + C{withIdentifier} is C{False}, or (flags, cas identifier, value) if + C{True}. If the server indicates there is no value associated with + C{key}, the returned values will be L{None} and the returned flags + will be C{0}. + @rtype: L{Deferred} + + @since: 9.0 + """ + return self._get(keys, withIdentifier, True) + + + def _get(self, keys, withIdentifier, multiple): + """ + Helper method for C{get} and C{getMultiple}. + """ + keys = list(keys) + if self._disconnected: + return fail(RuntimeError("not connected")) + for key in keys: + if not isinstance(key, bytes): + return fail(ClientError( + "Invalid type for key: %s, expecting bytes" % + (type(key),))) + if len(key) > self.MAX_KEY_LENGTH: + return fail(ClientError("Key too long")) + if withIdentifier: + cmd = b"gets" + else: + cmd = b"get" + fullcmd = b" ".join([cmd] + keys) + self.sendLine(fullcmd) + if multiple: + values = dict([(key, (0, b"", None)) for key in keys]) + cmdObj = Command(cmd, keys=keys, values=values, multiple=True) + else: + cmdObj = Command(cmd, key=keys[0], value=None, flags=0, cas=b"", + multiple=False) + self._current.append(cmdObj) + return cmdObj._deferred + + + def stats(self, arg=None): + """ + Get some stats from the server. It will be available as a dict. + + @param arg: An optional additional string which will be sent along + with the I{stats} command. The interpretation of this value by + the server is left undefined by the memcache protocol + specification. + @type arg: L{None} or L{bytes} + + @return: a deferred that will fire with a L{dict} of the available + statistics. + @rtype: L{Deferred} + """ + if arg: + cmd = b"stats " + arg + else: + cmd = b"stats" + if self._disconnected: + return fail(RuntimeError("not connected")) + self.sendLine(cmd) + cmdObj = Command(b"stats", values={}) + self._current.append(cmdObj) + return cmdObj._deferred + + + def version(self): + """ + Get the version of the server. + + @return: a deferred that will fire with the string value of the + version. + @rtype: L{Deferred} + """ + if self._disconnected: + return fail(RuntimeError("not connected")) + self.sendLine(b"version") + cmdObj = Command(b"version") + self._current.append(cmdObj) + return cmdObj._deferred + + + def delete(self, key): + """ + Delete an existing C{key}. + + @param key: the key to delete. + @type key: L{bytes} + + @return: a deferred that will be called back with C{True} if the key + was successfully deleted, or C{False} if not. + @rtype: L{Deferred} + """ + if self._disconnected: + return fail(RuntimeError("not connected")) + if not isinstance(key, bytes): + return fail(ClientError( + "Invalid type for key: %s, expecting bytes" % (type(key),))) + self.sendLine(b"delete " + key) + cmdObj = Command(b"delete", key=key) + self._current.append(cmdObj) + return cmdObj._deferred + + + def flushAll(self): + """ + Flush all cached values. + + @return: a deferred that will be called back with C{True} when the + operation has succeeded. + @rtype: L{Deferred} + """ + if self._disconnected: + return fail(RuntimeError("not connected")) + self.sendLine(b"flush_all") + cmdObj = Command(b"flush_all") + self._current.append(cmdObj) + return cmdObj._deferred + + + +__all__ = ["MemCacheProtocol", "DEFAULT_PORT", "NoSuchCommand", "ClientError", + "ServerError"] diff --git a/contrib/python/Twisted/py2/twisted/protocols/pcp.py b/contrib/python/Twisted/py2/twisted/protocols/pcp.py new file mode 100644 index 00000000000..43f20ec4106 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/pcp.py @@ -0,0 +1,203 @@ +# -*- test-case-name: twisted.test.test_pcp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Producer-Consumer Proxy. +""" + +from zope.interface import implementer + +from twisted.internet import interfaces + + +@implementer(interfaces.IProducer, interfaces.IConsumer) +class BasicProducerConsumerProxy: + """ + I can act as a man in the middle between any Producer and Consumer. + + @ivar producer: the Producer I subscribe to. + @type producer: L{IProducer} + @ivar consumer: the Consumer I publish to. + @type consumer: L{IConsumer} + @ivar paused: As a Producer, am I paused? + @type paused: bool + """ + consumer = None + producer = None + producerIsStreaming = None + iAmStreaming = True + outstandingPull = False + paused = False + stopped = False + + def __init__(self, consumer): + self._buffer = [] + if consumer is not None: + self.consumer = consumer + consumer.registerProducer(self, self.iAmStreaming) + + # Producer methods: + + def pauseProducing(self): + self.paused = True + if self.producer: + self.producer.pauseProducing() + + def resumeProducing(self): + self.paused = False + if self._buffer: + # TODO: Check to see if consumer supports writeSeq. + self.consumer.write(''.join(self._buffer)) + self._buffer[:] = [] + else: + if not self.iAmStreaming: + self.outstandingPull = True + + if self.producer is not None: + self.producer.resumeProducing() + + def stopProducing(self): + if self.producer is not None: + self.producer.stopProducing() + if self.consumer is not None: + del self.consumer + + # Consumer methods: + + def write(self, data): + if self.paused or (not self.iAmStreaming and not self.outstandingPull): + # We could use that fifo queue here. + self._buffer.append(data) + + elif self.consumer is not None: + self.consumer.write(data) + self.outstandingPull = False + + def finish(self): + if self.consumer is not None: + self.consumer.finish() + self.unregisterProducer() + + def registerProducer(self, producer, streaming): + self.producer = producer + self.producerIsStreaming = streaming + + def unregisterProducer(self): + if self.producer is not None: + del self.producer + del self.producerIsStreaming + if self.consumer: + self.consumer.unregisterProducer() + + def __repr__(self): + return '<%s@%x around %s>' % (self.__class__, id(self), self.consumer) + + +class ProducerConsumerProxy(BasicProducerConsumerProxy): + """ProducerConsumerProxy with a finite buffer. + + When my buffer fills up, I have my parent Producer pause until my buffer + has room in it again. + """ + # Copies much from abstract.FileDescriptor + bufferSize = 2**2**2**2 + + producerPaused = False + unregistered = False + + def pauseProducing(self): + # Does *not* call up to ProducerConsumerProxy to relay the pause + # message through to my parent Producer. + self.paused = True + + def resumeProducing(self): + self.paused = False + if self._buffer: + data = ''.join(self._buffer) + bytesSent = self._writeSomeData(data) + if bytesSent < len(data): + unsent = data[bytesSent:] + assert not self.iAmStreaming, ( + "Streaming producer did not write all its data.") + self._buffer[:] = [unsent] + else: + self._buffer[:] = [] + else: + bytesSent = 0 + + if (self.unregistered and bytesSent and not self._buffer and + self.consumer is not None): + self.consumer.unregisterProducer() + + if not self.iAmStreaming: + self.outstandingPull = not bytesSent + + if self.producer is not None: + bytesBuffered = sum([len(s) for s in self._buffer]) + # TODO: You can see here the potential for high and low + # watermarks, where bufferSize would be the high mark when we + # ask the upstream producer to pause, and we wouldn't have + # it resume again until it hit the low mark. Or if producer + # is Pull, maybe we'd like to pull from it as much as necessary + # to keep our buffer full to the low mark, so we're never caught + # without something to send. + if self.producerPaused and (bytesBuffered < self.bufferSize): + # Now that our buffer is empty, + self.producerPaused = False + self.producer.resumeProducing() + elif self.outstandingPull: + # I did not have any data to write in response to a pull, + # so I'd better pull some myself. + self.producer.resumeProducing() + + def write(self, data): + if self.paused or (not self.iAmStreaming and not self.outstandingPull): + # We could use that fifo queue here. + self._buffer.append(data) + + elif self.consumer is not None: + assert not self._buffer, ( + "Writing fresh data to consumer before my buffer is empty!") + # I'm going to use _writeSomeData here so that there is only one + # path to self.consumer.write. But it doesn't actually make sense, + # if I am streaming, for some data to not be all data. But maybe I + # am not streaming, but I am writing here anyway, because there was + # an earlier request for data which was not answered. + bytesSent = self._writeSomeData(data) + self.outstandingPull = False + if not bytesSent == len(data): + assert not self.iAmStreaming, ( + "Streaming producer did not write all its data.") + self._buffer.append(data[bytesSent:]) + + if (self.producer is not None) and self.producerIsStreaming: + bytesBuffered = sum([len(s) for s in self._buffer]) + if bytesBuffered >= self.bufferSize: + + self.producer.pauseProducing() + self.producerPaused = True + + def registerProducer(self, producer, streaming): + self.unregistered = False + BasicProducerConsumerProxy.registerProducer(self, producer, streaming) + if not streaming: + producer.resumeProducing() + + def unregisterProducer(self): + if self.producer is not None: + del self.producer + del self.producerIsStreaming + self.unregistered = True + if self.consumer and not self._buffer: + self.consumer.unregisterProducer() + + def _writeSomeData(self, data): + """Write as much of this data as possible. + + @returns: The number of bytes written. + """ + if self.consumer is None: + return 0 + self.consumer.write(data) + return len(data) diff --git a/contrib/python/Twisted/py2/twisted/protocols/policies.py b/contrib/python/Twisted/py2/twisted/protocols/policies.py new file mode 100644 index 00000000000..5b8830aa86f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/policies.py @@ -0,0 +1,751 @@ +# -*- test-case-name: twisted.test.test_policies -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Resource limiting policies. + +@seealso: See also L{twisted.protocols.htb} for rate limiting. +""" + +from __future__ import division, absolute_import + +# system imports +import sys + +from zope.interface import directlyProvides, providedBy + +# twisted imports +from twisted.internet.protocol import ServerFactory, Protocol, ClientFactory +from twisted.internet import error +from twisted.internet.interfaces import ILoggingContext +from twisted.python import log + + +def _wrappedLogPrefix(wrapper, wrapped): + """ + Compute a log prefix for a wrapper and the object it wraps. + + @rtype: C{str} + """ + if ILoggingContext.providedBy(wrapped): + logPrefix = wrapped.logPrefix() + else: + logPrefix = wrapped.__class__.__name__ + return "%s (%s)" % (logPrefix, wrapper.__class__.__name__) + + + +class ProtocolWrapper(Protocol): + """ + Wraps protocol instances and acts as their transport as well. + + @ivar wrappedProtocol: An L{IProtocol} + provider to which L{IProtocol} + method calls onto this L{ProtocolWrapper} will be proxied. + + @ivar factory: The L{WrappingFactory} which created this + L{ProtocolWrapper}. + """ + + disconnecting = 0 + + def __init__(self, factory, wrappedProtocol): + self.wrappedProtocol = wrappedProtocol + self.factory = factory + + + def logPrefix(self): + """ + Use a customized log prefix mentioning both the wrapped protocol and + the current one. + """ + return _wrappedLogPrefix(self, self.wrappedProtocol) + + + def makeConnection(self, transport): + """ + When a connection is made, register this wrapper with its factory, + save the real transport, and connect the wrapped protocol to this + L{ProtocolWrapper} to intercept any transport calls it makes. + """ + directlyProvides(self, providedBy(transport)) + Protocol.makeConnection(self, transport) + self.factory.registerProtocol(self) + self.wrappedProtocol.makeConnection(self) + + + # Transport relaying + + def write(self, data): + self.transport.write(data) + + + def writeSequence(self, data): + self.transport.writeSequence(data) + + + def loseConnection(self): + self.disconnecting = 1 + self.transport.loseConnection() + + + def getPeer(self): + return self.transport.getPeer() + + + def getHost(self): + return self.transport.getHost() + + + def registerProducer(self, producer, streaming): + self.transport.registerProducer(producer, streaming) + + + def unregisterProducer(self): + self.transport.unregisterProducer() + + + def stopConsuming(self): + self.transport.stopConsuming() + + + def __getattr__(self, name): + return getattr(self.transport, name) + + + # Protocol relaying + + def dataReceived(self, data): + self.wrappedProtocol.dataReceived(data) + + + def connectionLost(self, reason): + self.factory.unregisterProtocol(self) + self.wrappedProtocol.connectionLost(reason) + + # Breaking reference cycle between self and wrappedProtocol. + self.wrappedProtocol = None + + +class WrappingFactory(ClientFactory): + """ + Wraps a factory and its protocols, and keeps track of them. + """ + + protocol = ProtocolWrapper + + def __init__(self, wrappedFactory): + self.wrappedFactory = wrappedFactory + self.protocols = {} + + + def logPrefix(self): + """ + Generate a log prefix mentioning both the wrapped factory and this one. + """ + return _wrappedLogPrefix(self, self.wrappedFactory) + + + def doStart(self): + self.wrappedFactory.doStart() + ClientFactory.doStart(self) + + + def doStop(self): + self.wrappedFactory.doStop() + ClientFactory.doStop(self) + + + def startedConnecting(self, connector): + self.wrappedFactory.startedConnecting(connector) + + + def clientConnectionFailed(self, connector, reason): + self.wrappedFactory.clientConnectionFailed(connector, reason) + + + def clientConnectionLost(self, connector, reason): + self.wrappedFactory.clientConnectionLost(connector, reason) + + + def buildProtocol(self, addr): + return self.protocol(self, self.wrappedFactory.buildProtocol(addr)) + + + def registerProtocol(self, p): + """ + Called by protocol to register itself. + """ + self.protocols[p] = 1 + + + def unregisterProtocol(self, p): + """ + Called by protocols when they go away. + """ + del self.protocols[p] + + + +class ThrottlingProtocol(ProtocolWrapper): + """ + Protocol for L{ThrottlingFactory}. + """ + + # wrap API for tracking bandwidth + + def write(self, data): + self.factory.registerWritten(len(data)) + ProtocolWrapper.write(self, data) + + + def writeSequence(self, seq): + self.factory.registerWritten(sum(map(len, seq))) + ProtocolWrapper.writeSequence(self, seq) + + + def dataReceived(self, data): + self.factory.registerRead(len(data)) + ProtocolWrapper.dataReceived(self, data) + + + def registerProducer(self, producer, streaming): + self.producer = producer + ProtocolWrapper.registerProducer(self, producer, streaming) + + + def unregisterProducer(self): + del self.producer + ProtocolWrapper.unregisterProducer(self) + + + def throttleReads(self): + self.transport.pauseProducing() + + + def unthrottleReads(self): + self.transport.resumeProducing() + + + def throttleWrites(self): + if hasattr(self, "producer"): + self.producer.pauseProducing() + + + def unthrottleWrites(self): + if hasattr(self, "producer"): + self.producer.resumeProducing() + + + +class ThrottlingFactory(WrappingFactory): + """ + Throttles bandwidth and number of connections. + + Write bandwidth will only be throttled if there is a producer + registered. + """ + + protocol = ThrottlingProtocol + + def __init__(self, wrappedFactory, maxConnectionCount=sys.maxsize, + readLimit=None, writeLimit=None): + WrappingFactory.__init__(self, wrappedFactory) + self.connectionCount = 0 + self.maxConnectionCount = maxConnectionCount + self.readLimit = readLimit # max bytes we should read per second + self.writeLimit = writeLimit # max bytes we should write per second + self.readThisSecond = 0 + self.writtenThisSecond = 0 + self.unthrottleReadsID = None + self.checkReadBandwidthID = None + self.unthrottleWritesID = None + self.checkWriteBandwidthID = None + + + def callLater(self, period, func): + """ + Wrapper around + L{reactor.callLater} + for test purpose. + """ + from twisted.internet import reactor + return reactor.callLater(period, func) + + + def registerWritten(self, length): + """ + Called by protocol to tell us more bytes were written. + """ + self.writtenThisSecond += length + + + def registerRead(self, length): + """ + Called by protocol to tell us more bytes were read. + """ + self.readThisSecond += length + + + def checkReadBandwidth(self): + """ + Checks if we've passed bandwidth limits. + """ + if self.readThisSecond > self.readLimit: + self.throttleReads() + throttleTime = (float(self.readThisSecond) / self.readLimit) - 1.0 + self.unthrottleReadsID = self.callLater(throttleTime, + self.unthrottleReads) + self.readThisSecond = 0 + self.checkReadBandwidthID = self.callLater(1, self.checkReadBandwidth) + + + def checkWriteBandwidth(self): + if self.writtenThisSecond > self.writeLimit: + self.throttleWrites() + throttleTime = (float(self.writtenThisSecond) / self.writeLimit) - 1.0 + self.unthrottleWritesID = self.callLater(throttleTime, + self.unthrottleWrites) + # reset for next round + self.writtenThisSecond = 0 + self.checkWriteBandwidthID = self.callLater(1, self.checkWriteBandwidth) + + + def throttleReads(self): + """ + Throttle reads on all protocols. + """ + log.msg("Throttling reads on %s" % self) + for p in self.protocols.keys(): + p.throttleReads() + + + def unthrottleReads(self): + """ + Stop throttling reads on all protocols. + """ + self.unthrottleReadsID = None + log.msg("Stopped throttling reads on %s" % self) + for p in self.protocols.keys(): + p.unthrottleReads() + + + def throttleWrites(self): + """ + Throttle writes on all protocols. + """ + log.msg("Throttling writes on %s" % self) + for p in self.protocols.keys(): + p.throttleWrites() + + + def unthrottleWrites(self): + """ + Stop throttling writes on all protocols. + """ + self.unthrottleWritesID = None + log.msg("Stopped throttling writes on %s" % self) + for p in self.protocols.keys(): + p.unthrottleWrites() + + + def buildProtocol(self, addr): + if self.connectionCount == 0: + if self.readLimit is not None: + self.checkReadBandwidth() + if self.writeLimit is not None: + self.checkWriteBandwidth() + + if self.connectionCount < self.maxConnectionCount: + self.connectionCount += 1 + return WrappingFactory.buildProtocol(self, addr) + else: + log.msg("Max connection count reached!") + return None + + + def unregisterProtocol(self, p): + WrappingFactory.unregisterProtocol(self, p) + self.connectionCount -= 1 + if self.connectionCount == 0: + if self.unthrottleReadsID is not None: + self.unthrottleReadsID.cancel() + if self.checkReadBandwidthID is not None: + self.checkReadBandwidthID.cancel() + if self.unthrottleWritesID is not None: + self.unthrottleWritesID.cancel() + if self.checkWriteBandwidthID is not None: + self.checkWriteBandwidthID.cancel() + + + +class SpewingProtocol(ProtocolWrapper): + def dataReceived(self, data): + log.msg("Received: %r" % data) + ProtocolWrapper.dataReceived(self,data) + + def write(self, data): + log.msg("Sending: %r" % data) + ProtocolWrapper.write(self,data) + + + +class SpewingFactory(WrappingFactory): + protocol = SpewingProtocol + + + +class LimitConnectionsByPeer(WrappingFactory): + + maxConnectionsPerPeer = 5 + + def startFactory(self): + self.peerConnections = {} + + def buildProtocol(self, addr): + peerHost = addr[0] + connectionCount = self.peerConnections.get(peerHost, 0) + if connectionCount >= self.maxConnectionsPerPeer: + return None + self.peerConnections[peerHost] = connectionCount + 1 + return WrappingFactory.buildProtocol(self, addr) + + def unregisterProtocol(self, p): + peerHost = p.getPeer()[1] + self.peerConnections[peerHost] -= 1 + if self.peerConnections[peerHost] == 0: + del self.peerConnections[peerHost] + + +class LimitTotalConnectionsFactory(ServerFactory): + """ + Factory that limits the number of simultaneous connections. + + @type connectionCount: C{int} + @ivar connectionCount: number of current connections. + @type connectionLimit: C{int} or L{None} + @cvar connectionLimit: maximum number of connections. + @type overflowProtocol: L{Protocol} or L{None} + @cvar overflowProtocol: Protocol to use for new connections when + connectionLimit is exceeded. If L{None} (the default value), excess + connections will be closed immediately. + """ + connectionCount = 0 + connectionLimit = None + overflowProtocol = None + + def buildProtocol(self, addr): + if (self.connectionLimit is None or + self.connectionCount < self.connectionLimit): + # Build the normal protocol + wrappedProtocol = self.protocol() + elif self.overflowProtocol is None: + # Just drop the connection + return None + else: + # Too many connections, so build the overflow protocol + wrappedProtocol = self.overflowProtocol() + + wrappedProtocol.factory = self + protocol = ProtocolWrapper(self, wrappedProtocol) + self.connectionCount += 1 + return protocol + + def registerProtocol(self, p): + pass + + def unregisterProtocol(self, p): + self.connectionCount -= 1 + + + +class TimeoutProtocol(ProtocolWrapper): + """ + Protocol that automatically disconnects when the connection is idle. + """ + + def __init__(self, factory, wrappedProtocol, timeoutPeriod): + """ + Constructor. + + @param factory: An L{TimeoutFactory}. + @param wrappedProtocol: A L{Protocol} to wrapp. + @param timeoutPeriod: Number of seconds to wait for activity before + timing out. + """ + ProtocolWrapper.__init__(self, factory, wrappedProtocol) + self.timeoutCall = None + self.timeoutPeriod = None + self.setTimeout(timeoutPeriod) + + + def setTimeout(self, timeoutPeriod=None): + """ + Set a timeout. + + This will cancel any existing timeouts. + + @param timeoutPeriod: If not L{None}, change the timeout period. + Otherwise, use the existing value. + """ + self.cancelTimeout() + self.timeoutPeriod = timeoutPeriod + if timeoutPeriod is not None: + self.timeoutCall = self.factory.callLater(self.timeoutPeriod, self.timeoutFunc) + + + def cancelTimeout(self): + """ + Cancel the timeout. + + If the timeout was already cancelled, this does nothing. + """ + self.timeoutPeriod = None + if self.timeoutCall: + try: + self.timeoutCall.cancel() + except (error.AlreadyCalled, error.AlreadyCancelled): + pass + self.timeoutCall = None + + + def resetTimeout(self): + """ + Reset the timeout, usually because some activity just happened. + """ + if self.timeoutCall: + self.timeoutCall.reset(self.timeoutPeriod) + + + def write(self, data): + self.resetTimeout() + ProtocolWrapper.write(self, data) + + + def writeSequence(self, seq): + self.resetTimeout() + ProtocolWrapper.writeSequence(self, seq) + + + def dataReceived(self, data): + self.resetTimeout() + ProtocolWrapper.dataReceived(self, data) + + + def connectionLost(self, reason): + self.cancelTimeout() + ProtocolWrapper.connectionLost(self, reason) + + + def timeoutFunc(self): + """ + This method is called when the timeout is triggered. + + By default it calls I{loseConnection}. Override this if you want + something else to happen. + """ + self.loseConnection() + + + +class TimeoutFactory(WrappingFactory): + """ + Factory for TimeoutWrapper. + """ + protocol = TimeoutProtocol + + + def __init__(self, wrappedFactory, timeoutPeriod=30*60): + self.timeoutPeriod = timeoutPeriod + WrappingFactory.__init__(self, wrappedFactory) + + + def buildProtocol(self, addr): + return self.protocol(self, self.wrappedFactory.buildProtocol(addr), + timeoutPeriod=self.timeoutPeriod) + + + def callLater(self, period, func): + """ + Wrapper around + L{reactor.callLater} + for test purpose. + """ + from twisted.internet import reactor + return reactor.callLater(period, func) + + + +class TrafficLoggingProtocol(ProtocolWrapper): + + def __init__(self, factory, wrappedProtocol, logfile, lengthLimit=None, + number=0): + """ + @param factory: factory which created this protocol. + @type factory: L{protocol.Factory}. + @param wrappedProtocol: the underlying protocol. + @type wrappedProtocol: C{protocol.Protocol}. + @param logfile: file opened for writing used to write log messages. + @type logfile: C{file} + @param lengthLimit: maximum size of the datareceived logged. + @type lengthLimit: C{int} + @param number: identifier of the connection. + @type number: C{int}. + """ + ProtocolWrapper.__init__(self, factory, wrappedProtocol) + self.logfile = logfile + self.lengthLimit = lengthLimit + self._number = number + + + def _log(self, line): + self.logfile.write(line + '\n') + self.logfile.flush() + + + def _mungeData(self, data): + if self.lengthLimit and len(data) > self.lengthLimit: + data = data[:self.lengthLimit - 12] + '<... elided>' + return data + + + # IProtocol + def connectionMade(self): + self._log('*') + return ProtocolWrapper.connectionMade(self) + + + def dataReceived(self, data): + self._log('C %d: %r' % (self._number, self._mungeData(data))) + return ProtocolWrapper.dataReceived(self, data) + + + def connectionLost(self, reason): + self._log('C %d: %r' % (self._number, reason)) + return ProtocolWrapper.connectionLost(self, reason) + + + # ITransport + def write(self, data): + self._log('S %d: %r' % (self._number, self._mungeData(data))) + return ProtocolWrapper.write(self, data) + + + def writeSequence(self, iovec): + self._log('SV %d: %r' % (self._number, [self._mungeData(d) for d in iovec])) + return ProtocolWrapper.writeSequence(self, iovec) + + + def loseConnection(self): + self._log('S %d: *' % (self._number,)) + return ProtocolWrapper.loseConnection(self) + + + +class TrafficLoggingFactory(WrappingFactory): + protocol = TrafficLoggingProtocol + + _counter = 0 + + def __init__(self, wrappedFactory, logfilePrefix, lengthLimit=None): + self.logfilePrefix = logfilePrefix + self.lengthLimit = lengthLimit + WrappingFactory.__init__(self, wrappedFactory) + + + def open(self, name): + return open(name, 'w') + + + def buildProtocol(self, addr): + self._counter += 1 + logfile = self.open(self.logfilePrefix + '-' + str(self._counter)) + return self.protocol(self, self.wrappedFactory.buildProtocol(addr), + logfile, self.lengthLimit, self._counter) + + + def resetCounter(self): + """ + Reset the value of the counter used to identify connections. + """ + self._counter = 0 + + + +class TimeoutMixin: + """ + Mixin for protocols which wish to timeout connections. + + Protocols that mix this in have a single timeout, set using L{setTimeout}. + When the timeout is hit, L{timeoutConnection} is called, which, by + default, closes the connection. + + @cvar timeOut: The number of seconds after which to timeout the connection. + """ + timeOut = None + + __timeoutCall = None + + def callLater(self, period, func): + """ + Wrapper around + L{reactor.callLater} + for test purpose. + """ + from twisted.internet import reactor + return reactor.callLater(period, func) + + + def resetTimeout(self): + """ + Reset the timeout count down. + + If the connection has already timed out, then do nothing. If the + timeout has been cancelled (probably using C{setTimeout(None)}), also + do nothing. + + It's often a good idea to call this when the protocol has received + some meaningful input from the other end of the connection. "I've got + some data, they're still there, reset the timeout". + """ + if self.__timeoutCall is not None and self.timeOut is not None: + self.__timeoutCall.reset(self.timeOut) + + def setTimeout(self, period): + """ + Change the timeout period + + @type period: C{int} or L{None} + @param period: The period, in seconds, to change the timeout to, or + L{None} to disable the timeout. + """ + prev = self.timeOut + self.timeOut = period + + if self.__timeoutCall is not None: + if period is None: + try: + self.__timeoutCall.cancel() + except (error.AlreadyCancelled, error.AlreadyCalled): + # Do nothing if the call was already consumed. + pass + self.__timeoutCall = None + else: + self.__timeoutCall.reset(period) + elif period is not None: + self.__timeoutCall = self.callLater(period, self.__timedOut) + + return prev + + def __timedOut(self): + self.__timeoutCall = None + self.timeoutConnection() + + def timeoutConnection(self): + """ + Called when the connection times out. + + Override to define behavior other than dropping the connection. + """ + self.transport.loseConnection() diff --git a/contrib/python/Twisted/py2/twisted/protocols/portforward.py b/contrib/python/Twisted/py2/twisted/protocols/portforward.py new file mode 100644 index 00000000000..a3c39549ae4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/portforward.py @@ -0,0 +1,99 @@ + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A simple port forwarder. +""" + +# Twisted imports +from twisted.internet import protocol +from twisted.python import log + +class Proxy(protocol.Protocol): + noisy = True + + peer = None + + def setPeer(self, peer): + self.peer = peer + + + def connectionLost(self, reason): + if self.peer is not None: + self.peer.transport.loseConnection() + self.peer = None + elif self.noisy: + log.msg("Unable to connect to peer: %s" % (reason,)) + + + def dataReceived(self, data): + self.peer.transport.write(data) + + + +class ProxyClient(Proxy): + def connectionMade(self): + self.peer.setPeer(self) + + # Wire this and the peer transport together to enable + # flow control (this stops connections from filling + # this proxy memory when one side produces data at a + # higher rate than the other can consume). + self.transport.registerProducer(self.peer.transport, True) + self.peer.transport.registerProducer(self.transport, True) + + # We're connected, everybody can read to their hearts content. + self.peer.transport.resumeProducing() + + + +class ProxyClientFactory(protocol.ClientFactory): + + protocol = ProxyClient + + def setServer(self, server): + self.server = server + + + def buildProtocol(self, *args, **kw): + prot = protocol.ClientFactory.buildProtocol(self, *args, **kw) + prot.setPeer(self.server) + return prot + + + def clientConnectionFailed(self, connector, reason): + self.server.transport.loseConnection() + + + +class ProxyServer(Proxy): + + clientProtocolFactory = ProxyClientFactory + reactor = None + + def connectionMade(self): + # Don't read anything from the connecting client until we have + # somewhere to send it to. + self.transport.pauseProducing() + + client = self.clientProtocolFactory() + client.setServer(self) + + if self.reactor is None: + from twisted.internet import reactor + self.reactor = reactor + self.reactor.connectTCP(self.factory.host, self.factory.port, client) + + + +class ProxyFactory(protocol.Factory): + """ + Factory for port forwarder. + """ + + protocol = ProxyServer + + def __init__(self, host, port): + self.host = host + self.port = port diff --git a/contrib/python/Twisted/py2/twisted/protocols/postfix.py b/contrib/python/Twisted/py2/twisted/protocols/postfix.py new file mode 100644 index 00000000000..445b88cb059 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/postfix.py @@ -0,0 +1,158 @@ +# -*- test-case-name: twisted.test.test_postfix -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Postfix mail transport agent related protocols. +""" + +import sys +try: + # Python 2 + from UserDict import UserDict +except ImportError: + # Python 3 + from collections import UserDict + +try: + # Python 2 + from urllib import quote as _quote, unquote as _unquote +except ImportError: + # Python 3 + from urllib.parse import quote as _quote, unquote as _unquote + +from twisted.protocols import basic +from twisted.protocols import policies +from twisted.internet import protocol, defer +from twisted.python import log +from twisted.python.compat import unicode + +# urllib's quote functions just happen to match +# the postfix semantics. +def quote(s): + quoted = _quote(s) + if isinstance(quoted, unicode): + quoted = quoted.encode("ascii") + return quoted + + + +def unquote(s): + if isinstance(s, bytes): + s = s.decode("ascii") + quoted = _unquote(s) + return quoted.encode("ascii") + + + +class PostfixTCPMapServer(basic.LineReceiver, policies.TimeoutMixin): + """ + Postfix mail transport agent TCP map protocol implementation. + + Receive requests for data matching given key via lineReceived, + asks it's factory for the data with self.factory.get(key), and + returns the data to the requester. None means no entry found. + + You can use postfix's postmap to test the map service:: + + /usr/sbin/postmap -q KEY tcp:localhost:4242 + + """ + + timeout = 600 + delimiter = b'\n' + + def connectionMade(self): + self.setTimeout(self.timeout) + + + + def sendCode(self, code, message=b''): + """ + Send an SMTP-like code with a message. + """ + self.sendLine(str(code).encode("ascii") + b' ' + message) + + + + def lineReceived(self, line): + self.resetTimeout() + try: + request, params = line.split(None, 1) + except ValueError: + request = line + params = None + try: + f = getattr(self, u'do_' + request.decode("ascii")) + except AttributeError: + self.sendCode(400, b'unknown command') + else: + try: + f(params) + except: + excInfo = str(sys.exc_info()[1]).encode("ascii") + self.sendCode(400, b'Command ' + request + b' failed: ' + + excInfo) + + + + def do_get(self, key): + if key is None: + self.sendCode(400, b"Command 'get' takes 1 parameters.") + else: + d = defer.maybeDeferred(self.factory.get, key) + d.addCallbacks(self._cbGot, self._cbNot) + d.addErrback(log.err) + + + + def _cbNot(self, fail): + msg = fail.getErrorMessage().encode("ascii") + self.sendCode(400, msg) + + + + def _cbGot(self, value): + if value is None: + self.sendCode(500) + else: + self.sendCode(200, quote(value)) + + + + def do_put(self, keyAndValue): + if keyAndValue is None: + self.sendCode(400, b"Command 'put' takes 2 parameters.") + else: + try: + key, value = keyAndValue.split(None, 1) + except ValueError: + self.sendCode(400, b"Command 'put' takes 2 parameters.") + else: + self.sendCode(500, b'put is not implemented yet.') + + + +class PostfixTCPMapDictServerFactory(UserDict, protocol.ServerFactory): + """ + An in-memory dictionary factory for PostfixTCPMapServer. + """ + + protocol = PostfixTCPMapServer + + + +class PostfixTCPMapDeferringDictServerFactory(protocol.ServerFactory): + """ + An in-memory dictionary factory for PostfixTCPMapServer. + """ + + protocol = PostfixTCPMapServer + + def __init__(self, data=None): + self.data = {} + if data is not None: + self.data.update(data) + + def get(self, key): + return defer.succeed(self.data.get(key)) diff --git a/contrib/python/Twisted/py2/twisted/protocols/shoutcast.py b/contrib/python/Twisted/py2/twisted/protocols/shoutcast.py new file mode 100644 index 00000000000..e2be938995f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/shoutcast.py @@ -0,0 +1,111 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Chop up shoutcast stream into MP3s and metadata, if available. +""" + +from twisted.web import http +from twisted import copyright + + +class ShoutcastClient(http.HTTPClient): + """ + Shoutcast HTTP stream. + + Modes can be 'length', 'meta' and 'mp3'. + + See U{http://www.smackfu.com/stuff/programming/shoutcast.html} + for details on the protocol. + """ + + userAgent = "Twisted Shoutcast client " + copyright.version + + def __init__(self, path="/"): + self.path = path + self.got_metadata = False + self.metaint = None + self.metamode = "mp3" + self.databuffer = "" + + def connectionMade(self): + self.sendCommand("GET", self.path) + self.sendHeader("User-Agent", self.userAgent) + self.sendHeader("Icy-MetaData", "1") + self.endHeaders() + + def lineReceived(self, line): + # fix shoutcast crappiness + if not self.firstLine and line: + if len(line.split(": ", 1)) == 1: + line = line.replace(":", ": ", 1) + http.HTTPClient.lineReceived(self, line) + + def handleHeader(self, key, value): + if key.lower() == 'icy-metaint': + self.metaint = int(value) + self.got_metadata = True + + def handleEndHeaders(self): + # Lets check if we got metadata, and set the + # appropriate handleResponsePart method. + if self.got_metadata: + # if we have metadata, then it has to be parsed out of the data stream + self.handleResponsePart = self.handleResponsePart_with_metadata + else: + # otherwise, all the data is MP3 data + self.handleResponsePart = self.gotMP3Data + + def handleResponsePart_with_metadata(self, data): + self.databuffer += data + while self.databuffer: + stop = getattr(self, "handle_%s" % self.metamode)() + if stop: + return + + def handle_length(self): + self.remaining = ord(self.databuffer[0]) * 16 + self.databuffer = self.databuffer[1:] + self.metamode = "meta" + + def handle_mp3(self): + if len(self.databuffer) > self.metaint: + self.gotMP3Data(self.databuffer[:self.metaint]) + self.databuffer = self.databuffer[self.metaint:] + self.metamode = "length" + else: + return 1 + + def handle_meta(self): + if len(self.databuffer) >= self.remaining: + if self.remaining: + data = self.databuffer[:self.remaining] + self.gotMetaData(self.parseMetadata(data)) + self.databuffer = self.databuffer[self.remaining:] + self.metamode = "mp3" + else: + return 1 + + def parseMetadata(self, data): + meta = [] + for chunk in data.split(';'): + chunk = chunk.strip().replace("\x00", "") + if not chunk: + continue + key, value = chunk.split('=', 1) + if value.startswith("'") and value.endswith("'"): + value = value[1:-1] + meta.append((key, value)) + return meta + + def gotMetaData(self, metadata): + """Called with a list of (key, value) pairs of metadata, + if metadata is available on the server. + + Will only be called on non-empty metadata. + """ + raise NotImplementedError("implement in subclass") + + def gotMP3Data(self, data): + """Called with chunk of MP3 data.""" + raise NotImplementedError("implement in subclass") diff --git a/contrib/python/Twisted/py2/twisted/protocols/sip.py b/contrib/python/Twisted/py2/twisted/protocols/sip.py new file mode 100644 index 00000000000..6a8429ed146 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/sip.py @@ -0,0 +1,1294 @@ +# -*- test-case-name: twisted.test.test_sip -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Session Initialization Protocol. + +Documented in RFC 2543. +[Superseded by 3261] +""" + +import socket +import time +import warnings + +from zope.interface import implementer, Interface +from collections import OrderedDict + +from twisted import cred +from twisted.internet import protocol, defer, reactor +from twisted.protocols import basic +from twisted.python import log +from twisted.python.compat import _PY3, iteritems, unicode + +PORT = 5060 + +# SIP headers have short forms +shortHeaders = {"call-id": "i", + "contact": "m", + "content-encoding": "e", + "content-length": "l", + "content-type": "c", + "from": "f", + "subject": "s", + "to": "t", + "via": "v", + } + +longHeaders = {} +for k, v in shortHeaders.items(): + longHeaders[v] = k +del k, v + +statusCodes = { + 100: "Trying", + 180: "Ringing", + 181: "Call Is Being Forwarded", + 182: "Queued", + 183: "Session Progress", + + 200: "OK", + + 300: "Multiple Choices", + 301: "Moved Permanently", + 302: "Moved Temporarily", + 303: "See Other", + 305: "Use Proxy", + 380: "Alternative Service", + + 400: "Bad Request", + 401: "Unauthorized", + 402: "Payment Required", + 403: "Forbidden", + 404: "Not Found", + 405: "Method Not Allowed", + 406: "Not Acceptable", + 407: "Proxy Authentication Required", + 408: "Request Timeout", + 409: "Conflict", # Not in RFC3261 + 410: "Gone", + 411: "Length Required", # Not in RFC3261 + 413: "Request Entity Too Large", + 414: "Request-URI Too Large", + 415: "Unsupported Media Type", + 416: "Unsupported URI Scheme", + 420: "Bad Extension", + 421: "Extension Required", + 423: "Interval Too Brief", + 480: "Temporarily Unavailable", + 481: "Call/Transaction Does Not Exist", + 482: "Loop Detected", + 483: "Too Many Hops", + 484: "Address Incomplete", + 485: "Ambiguous", + 486: "Busy Here", + 487: "Request Terminated", + 488: "Not Acceptable Here", + 491: "Request Pending", + 493: "Undecipherable", + + 500: "Internal Server Error", + 501: "Not Implemented", + 502: "Bad Gateway", # No donut + 503: "Service Unavailable", + 504: "Server Time-out", + 505: "SIP Version not supported", + 513: "Message Too Large", + + 600: "Busy Everywhere", + 603: "Decline", + 604: "Does not exist anywhere", + 606: "Not Acceptable", +} + +specialCases = { + 'cseq': 'CSeq', + 'call-id': 'Call-ID', + 'www-authenticate': 'WWW-Authenticate', +} + + +def dashCapitalize(s): + """ + Capitalize a string, making sure to treat '-' as a word separator + """ + return '-'.join([ x.capitalize() for x in s.split('-')]) + + + +def unq(s): + if s[0] == s[-1] == '"': + return s[1:-1] + return s + + + +_absent = object() + +class Via(object): + """ + A L{Via} is a SIP Via header, representing a segment of the path taken by + the request. + + See RFC 3261, sections 8.1.1.7, 18.2.2, and 20.42. + + @ivar transport: Network protocol used for this leg. (Probably either "TCP" + or "UDP".) + @type transport: C{str} + @ivar branch: Unique identifier for this request. + @type branch: C{str} + @ivar host: Hostname or IP for this leg. + @type host: C{str} + @ivar port: Port used for this leg. + @type port C{int}, or None. + @ivar rportRequested: Whether to request RFC 3581 client processing or not. + @type rportRequested: C{bool} + @ivar rportValue: Servers wishing to honor requests for RFC 3581 processing + should set this parameter to the source port the request was received + from. + @type rportValue: C{int}, or None. + + @ivar ttl: Time-to-live for requests on multicast paths. + @type ttl: C{int}, or None. + @ivar maddr: The destination multicast address, if any. + @type maddr: C{str}, or None. + @ivar hidden: Obsolete in SIP 2.0. + @type hidden: C{bool} + @ivar otherParams: Any other parameters in the header. + @type otherParams: C{dict} + """ + + def __init__(self, host, port=PORT, transport="UDP", ttl=None, + hidden=False, received=None, rport=_absent, branch=None, + maddr=None, **kw): + """ + Set parameters of this Via header. All arguments correspond to + attributes of the same name. + + To maintain compatibility with old SIP + code, the 'rport' argument is used to determine the values of + C{rportRequested} and C{rportValue}. If None, C{rportRequested} is set + to True. (The deprecated method for doing this is to pass True.) If an + integer, C{rportValue} is set to the given value. + + Any arguments not explicitly named here are collected into the + C{otherParams} dict. + """ + self.transport = transport + self.host = host + self.port = port + self.ttl = ttl + self.hidden = hidden + self.received = received + if rport is True: + warnings.warn( + "rport=True is deprecated since Twisted 9.0.", + DeprecationWarning, + stacklevel=2) + self.rportValue = None + self.rportRequested = True + elif rport is None: + self.rportValue = None + self.rportRequested = True + elif rport is _absent: + self.rportValue = None + self.rportRequested = False + else: + self.rportValue = rport + self.rportRequested = False + + self.branch = branch + self.maddr = maddr + self.otherParams = kw + + + def _getrport(self): + """ + Returns the rport value expected by the old SIP code. + """ + if self.rportRequested == True: + return True + elif self.rportValue is not None: + return self.rportValue + else: + return None + + + def _setrport(self, newRPort): + """ + L{Base._fixupNAT} sets C{rport} directly, so this method sets + C{rportValue} based on that. + + @param newRPort: The new rport value. + @type newRPort: C{int} + """ + self.rportValue = newRPort + self.rportRequested = False + + rport = property(_getrport, _setrport) + + def toString(self): + """ + Serialize this header for use in a request or response. + """ + s = "SIP/2.0/%s %s:%s" % (self.transport, self.host, self.port) + if self.hidden: + s += ";hidden" + for n in "ttl", "branch", "maddr", "received": + value = getattr(self, n) + if value is not None: + s += ";%s=%s" % (n, value) + if self.rportRequested: + s += ";rport" + elif self.rportValue is not None: + s += ";rport=%s" % (self.rport,) + + etc = sorted(self.otherParams.items()) + for k, v in etc: + if v is None: + s += ";" + k + else: + s += ";%s=%s" % (k, v) + return s + + + +def parseViaHeader(value): + """ + Parse a Via header. + + @return: The parsed version of this header. + @rtype: L{Via} + """ + parts = value.split(";") + sent, params = parts[0], parts[1:] + protocolinfo, by = sent.split(" ", 1) + by = by.strip() + result = {} + pname, pversion, transport = protocolinfo.split("/") + if pname != "SIP" or pversion != "2.0": + raise ValueError("wrong protocol or version: %r" % (value,)) + result["transport"] = transport + if ":" in by: + host, port = by.split(":") + result["port"] = int(port) + result["host"] = host + else: + result["host"] = by + for p in params: + # It's the comment-striping dance! + p = p.strip().split(" ", 1) + if len(p) == 1: + p, comment = p[0], "" + else: + p, comment = p + if p == "hidden": + result["hidden"] = True + continue + parts = p.split("=", 1) + if len(parts) == 1: + name, value = parts[0], None + else: + name, value = parts + if name in ("rport", "ttl"): + value = int(value) + result[name] = value + return Via(**result) + + + +class URL: + """ + A SIP URL. + """ + + def __init__(self, host, username=None, password=None, port=None, + transport=None, usertype=None, method=None, + ttl=None, maddr=None, tag=None, other=None, headers=None): + self.username = username + self.host = host + self.password = password + self.port = port + self.transport = transport + self.usertype = usertype + self.method = method + self.tag = tag + self.ttl = ttl + self.maddr = maddr + if other == None: + self.other = [] + else: + self.other = other + if headers == None: + self.headers = {} + else: + self.headers = headers + + + def toString(self): + l = []; w = l.append + w("sip:") + if self.username != None: + w(self.username) + if self.password != None: + w(":%s" % self.password) + w("@") + w(self.host) + if self.port != None: + w(":%d" % self.port) + if self.usertype != None: + w(";user=%s" % self.usertype) + for n in ("transport", "ttl", "maddr", "method", "tag"): + v = getattr(self, n) + if v != None: + w(";%s=%s" % (n, v)) + for v in self.other: + w(";%s" % v) + if self.headers: + w("?") + w("&".join([("%s=%s" % (specialCases.get(h) or dashCapitalize(h), v)) for (h, v) in self.headers.items()])) + return "".join(l) + + + def __str__(self): + return self.toString() + + + def __repr__(self): + return '' % (self.username, self.password, self.host, self.port, self.transport) + + + +def parseURL(url, host=None, port=None): + """ + Return string into URL object. + + URIs are of form 'sip:user@example.com'. + """ + d = {} + if not url.startswith("sip:"): + raise ValueError("unsupported scheme: " + url[:4]) + parts = url[4:].split(";") + userdomain, params = parts[0], parts[1:] + udparts = userdomain.split("@", 1) + if len(udparts) == 2: + userpass, hostport = udparts + upparts = userpass.split(":", 1) + if len(upparts) == 1: + d["username"] = upparts[0] + else: + d["username"] = upparts[0] + d["password"] = upparts[1] + else: + hostport = udparts[0] + hpparts = hostport.split(":", 1) + if len(hpparts) == 1: + d["host"] = hpparts[0] + else: + d["host"] = hpparts[0] + d["port"] = int(hpparts[1]) + if host != None: + d["host"] = host + if port != None: + d["port"] = port + for p in params: + if p == params[-1] and "?" in p: + d["headers"] = h = {} + p, headers = p.split("?", 1) + for header in headers.split("&"): + k, v = header.split("=") + h[k] = v + nv = p.split("=", 1) + if len(nv) == 1: + d.setdefault("other", []).append(p) + continue + name, value = nv + if name == "user": + d["usertype"] = value + elif name in ("transport", "ttl", "maddr", "method", "tag"): + if name == "ttl": + value = int(value) + d[name] = value + else: + d.setdefault("other", []).append(p) + return URL(**d) + + + +def cleanRequestURL(url): + """ + Clean a URL from a Request line. + """ + url.transport = None + url.maddr = None + url.ttl = None + url.headers = {} + + + +def parseAddress(address, host=None, port=None, clean=0): + """ + Return (name, uri, params) for From/To/Contact header. + + @param clean: remove unnecessary info, usually for From and To headers. + """ + address = address.strip() + # Simple 'sip:foo' case + if address.startswith("sip:"): + return "", parseURL(address, host=host, port=port), {} + params = {} + name, url = address.split("<", 1) + name = name.strip() + if name.startswith('"'): + name = name[1:] + if name.endswith('"'): + name = name[:-1] + url, paramstring = url.split(">", 1) + url = parseURL(url, host=host, port=port) + paramstring = paramstring.strip() + if paramstring: + for l in paramstring.split(";"): + if not l: + continue + k, v = l.split("=") + params[k] = v + if clean: + # RFC 2543 6.21 + url.ttl = None + url.headers = {} + url.transport = None + url.maddr = None + return name, url, params + + + +class SIPError(Exception): + def __init__(self, code, phrase=None): + if phrase is None: + phrase = statusCodes[code] + Exception.__init__(self, "SIP error (%d): %s" % (code, phrase)) + self.code = code + self.phrase = phrase + + + +class RegistrationError(SIPError): + """ + Registration was not possible. + """ + + + +class Message: + """ + A SIP message. + """ + + length = None + + def __init__(self): + self.headers = OrderedDict() # Map name to list of values + self.body = "" + self.finished = 0 + + + def addHeader(self, name, value): + name = name.lower() + name = longHeaders.get(name, name) + if name == "content-length": + self.length = int(value) + self.headers.setdefault(name,[]).append(value) + + + def bodyDataReceived(self, data): + self.body += data + + + def creationFinished(self): + if (self.length != None) and (self.length != len(self.body)): + raise ValueError("wrong body length") + self.finished = 1 + + + def toString(self): + s = "%s\r\n" % self._getHeaderLine() + for n, vs in self.headers.items(): + for v in vs: + s += "%s: %s\r\n" % (specialCases.get(n) or dashCapitalize(n), v) + s += "\r\n" + s += self.body + return s + + + def _getHeaderLine(self): + raise NotImplementedError + + + +class Request(Message): + """ + A Request for a URI + """ + + def __init__(self, method, uri, version="SIP/2.0"): + Message.__init__(self) + self.method = method + if isinstance(uri, URL): + self.uri = uri + else: + self.uri = parseURL(uri) + cleanRequestURL(self.uri) + + + def __repr__(self): + return "" % (id(self), self.method, self.uri.toString()) + + + def _getHeaderLine(self): + return "%s %s SIP/2.0" % (self.method, self.uri.toString()) + + + +class Response(Message): + """ + A Response to a URI Request + """ + + def __init__(self, code, phrase=None, version="SIP/2.0"): + Message.__init__(self) + self.code = code + if phrase == None: + phrase = statusCodes[code] + self.phrase = phrase + + + def __repr__(self): + return "" % (id(self), self.code) + + + def _getHeaderLine(self): + return "SIP/2.0 %s %s" % (self.code, self.phrase) + + + +class MessagesParser(basic.LineReceiver): + """ + A SIP messages parser. + + Expects dataReceived, dataDone repeatedly, + in that order. Shouldn't be connected to actual transport. + """ + + version = "SIP/2.0" + acceptResponses = 1 + acceptRequests = 1 + state = "firstline" # Or "headers", "body" or "invalid" + + debug = 0 + + def __init__(self, messageReceivedCallback): + self.messageReceived = messageReceivedCallback + self.reset() + + + def reset(self, remainingData=""): + self.state = "firstline" + self.length = None # Body length + self.bodyReceived = 0 # How much of the body we received + self.message = None + self.header = None + self.setLineMode(remainingData) + + + def invalidMessage(self): + self.state = "invalid" + self.setRawMode() + + + def dataDone(self): + """ + Clear out any buffered data that may be hanging around. + """ + self.clearLineBuffer() + if self.state == "firstline": + return + if self.state != "body": + self.reset() + return + if self.length == None: + # No content-length header, so end of data signals message done + self.messageDone() + elif self.length < self.bodyReceived: + # Aborted in the middle + self.reset() + else: + # We have enough data and message wasn't finished? something is wrong + raise RuntimeError("this should never happen") + + + def dataReceived(self, data): + try: + if isinstance(data, unicode): + data = data.encode("utf-8") + basic.LineReceiver.dataReceived(self, data) + except: + log.err() + self.invalidMessage() + + + def handleFirstLine(self, line): + """ + Expected to create self.message. + """ + raise NotImplementedError + + + def lineLengthExceeded(self, line): + self.invalidMessage() + + + def lineReceived(self, line): + if _PY3 and isinstance(line, bytes): + line = line.decode("utf-8") + + if self.state == "firstline": + while line.startswith("\n") or line.startswith("\r"): + line = line[1:] + if not line: + return + try: + a, b, c = line.split(" ", 2) + except ValueError: + self.invalidMessage() + return + if a == "SIP/2.0" and self.acceptResponses: + # Response + try: + code = int(b) + except ValueError: + self.invalidMessage() + return + self.message = Response(code, c) + elif c == "SIP/2.0" and self.acceptRequests: + self.message = Request(a, b) + else: + self.invalidMessage() + return + self.state = "headers" + return + else: + assert self.state == "headers" + if line: + # Multiline header + if line.startswith(" ") or line.startswith("\t"): + name, value = self.header + self.header = name, (value + line.lstrip()) + else: + # New header + if self.header: + self.message.addHeader(*self.header) + self.header = None + try: + name, value = line.split(":", 1) + except ValueError: + self.invalidMessage() + return + self.header = name, value.lstrip() + # XXX we assume content-length won't be multiline + if name.lower() == "content-length": + try: + self.length = int(value.lstrip()) + except ValueError: + self.invalidMessage() + return + else: + # CRLF, we now have message body until self.length bytes, + # or if no length was given, until there is no more data + # from the connection sending us data. + self.state = "body" + if self.header: + self.message.addHeader(*self.header) + self.header = None + if self.length == 0: + self.messageDone() + return + self.setRawMode() + + + def messageDone(self, remainingData=""): + assert self.state == "body" + self.message.creationFinished() + self.messageReceived(self.message) + self.reset(remainingData) + + + def rawDataReceived(self, data): + assert self.state in ("body", "invalid") + if _PY3 and isinstance(data, bytes): + data = data.decode("utf-8") + if self.state == "invalid": + return + if self.length == None: + self.message.bodyDataReceived(data) + else: + dataLen = len(data) + expectedLen = self.length - self.bodyReceived + if dataLen > expectedLen: + self.message.bodyDataReceived(data[:expectedLen]) + self.messageDone(data[expectedLen:]) + return + else: + self.bodyReceived += dataLen + self.message.bodyDataReceived(data) + if self.bodyReceived == self.length: + self.messageDone() + + + +class Base(protocol.DatagramProtocol): + """ + Base class for SIP clients and servers. + """ + + PORT = PORT + debug = False + + def __init__(self): + self.messages = [] + self.parser = MessagesParser(self.addMessage) + + + def addMessage(self, msg): + self.messages.append(msg) + + + def datagramReceived(self, data, addr): + self.parser.dataReceived(data) + self.parser.dataDone() + for m in self.messages: + self._fixupNAT(m, addr) + if self.debug: + log.msg("Received %r from %r" % (m.toString(), addr)) + if isinstance(m, Request): + self.handle_request(m, addr) + else: + self.handle_response(m, addr) + self.messages[:] = [] + + + def _fixupNAT(self, message, sourcePeer): + # RFC 2543 6.40.2, + (srcHost, srcPort) = sourcePeer + senderVia = parseViaHeader(message.headers["via"][0]) + if senderVia.host != srcHost: + senderVia.received = srcHost + if senderVia.port != srcPort: + senderVia.rport = srcPort + message.headers["via"][0] = senderVia.toString() + elif senderVia.rport == True: + senderVia.received = srcHost + senderVia.rport = srcPort + message.headers["via"][0] = senderVia.toString() + + + def deliverResponse(self, responseMessage): + """ + Deliver response. + + Destination is based on topmost Via header. + """ + destVia = parseViaHeader(responseMessage.headers["via"][0]) + # XXX we don't do multicast yet + host = destVia.received or destVia.host + port = destVia.rport or destVia.port or self.PORT + destAddr = URL(host=host, port=port) + self.sendMessage(destAddr, responseMessage) + + + def responseFromRequest(self, code, request): + """ + Create a response to a request message. + """ + response = Response(code) + for name in ("via", "to", "from", "call-id", "cseq"): + response.headers[name] = request.headers.get(name, [])[:] + + return response + + + def sendMessage(self, destURL, message): + """ + Send a message. + + @param destURL: C{URL}. This should be a *physical* URL, not a logical one. + @param message: The message to send. + """ + if destURL.transport not in ("udp", None): + raise RuntimeError("only UDP currently supported") + if self.debug: + log.msg("Sending %r to %r" % (message.toString(), destURL)) + data = message.toString() + if isinstance(data, unicode): + data = data.encode("utf-8") + self.transport.write(data, (destURL.host, destURL.port or self.PORT)) + + + def handle_request(self, message, addr): + """ + Override to define behavior for requests received + + @type message: C{Message} + @type addr: C{tuple} + """ + raise NotImplementedError + + + def handle_response(self, message, addr): + """ + Override to define behavior for responses received. + + @type message: C{Message} + @type addr: C{tuple} + """ + raise NotImplementedError + + + +class IContact(Interface): + """ + A user of a registrar or proxy + """ + + + +class Registration: + def __init__(self, secondsToExpiry, contactURL): + self.secondsToExpiry = secondsToExpiry + self.contactURL = contactURL + + + +class IRegistry(Interface): + """ + Allows registration of logical->physical URL mapping. + """ + + def registerAddress(domainURL, logicalURL, physicalURL): + """ + Register the physical address of a logical URL. + + @return: Deferred of C{Registration} or failure with RegistrationError. + """ + + + def unregisterAddress(domainURL, logicalURL, physicalURL): + """ + Unregister the physical address of a logical URL. + + @return: Deferred of C{Registration} or failure with RegistrationError. + """ + + + def getRegistrationInfo(logicalURL): + """ + Get registration info for logical URL. + + @return: Deferred of C{Registration} object or failure of LookupError. + """ + + + +class ILocator(Interface): + """ + Allow looking up physical address for logical URL. + """ + + def getAddress(logicalURL): + """ + Return physical URL of server for logical URL of user. + + @param logicalURL: a logical C{URL}. + @return: Deferred which becomes URL or fails with LookupError. + """ + + + +class Proxy(Base): + """ + SIP proxy. + """ + + PORT = PORT + + locator = None # Object implementing ILocator + + def __init__(self, host=None, port=PORT): + """ + Create new instance. + + @param host: our hostname/IP as set in Via headers. + @param port: our port as set in Via headers. + """ + self.host = host or socket.getfqdn() + self.port = port + Base.__init__(self) + + + def getVia(self): + """ + Return value of Via header for this proxy. + """ + return Via(host=self.host, port=self.port) + + + def handle_request(self, message, addr): + # Send immediate 100/trying message before processing + #self.deliverResponse(self.responseFromRequest(100, message)) + f = getattr(self, "handle_%s_request" % message.method, None) + if f is None: + f = self.handle_request_default + try: + d = f(message, addr) + except SIPError as e: + self.deliverResponse(self.responseFromRequest(e.code, message)) + except: + log.err() + self.deliverResponse(self.responseFromRequest(500, message)) + else: + if d is not None: + d.addErrback(lambda e: + self.deliverResponse(self.responseFromRequest(e.code, message)) + ) + + + def handle_request_default(self, message, sourcePeer): + """ + Default request handler. + + Default behaviour for OPTIONS and unknown methods for proxies + is to forward message on to the client. + + Since at the moment we are stateless proxy, that's basically + everything. + """ + (srcHost, srcPort) = sourcePeer + def _mungContactHeader(uri, message): + message.headers['contact'][0] = uri.toString() + return self.sendMessage(uri, message) + + viaHeader = self.getVia() + if viaHeader.toString() in message.headers["via"]: + # Must be a loop, so drop message + log.msg("Dropping looped message.") + return + + message.headers["via"].insert(0, viaHeader.toString()) + name, uri, tags = parseAddress(message.headers["to"][0], clean=1) + + # This is broken and needs refactoring to use cred + d = self.locator.getAddress(uri) + d.addCallback(self.sendMessage, message) + d.addErrback(self._cantForwardRequest, message) + + + def _cantForwardRequest(self, error, message): + error.trap(LookupError) + del message.headers["via"][0] # This'll be us + self.deliverResponse(self.responseFromRequest(404, message)) + + + def deliverResponse(self, responseMessage): + """ + Deliver response. + + Destination is based on topmost Via header. + """ + destVia = parseViaHeader(responseMessage.headers["via"][0]) + # XXX we don't do multicast yet + host = destVia.received or destVia.host + port = destVia.rport or destVia.port or self.PORT + + destAddr = URL(host=host, port=port) + self.sendMessage(destAddr, responseMessage) + + + def responseFromRequest(self, code, request): + """ + Create a response to a request message. + """ + response = Response(code) + for name in ("via", "to", "from", "call-id", "cseq"): + response.headers[name] = request.headers.get(name, [])[:] + return response + + + def handle_response(self, message, addr): + """ + Default response handler. + """ + v = parseViaHeader(message.headers["via"][0]) + if (v.host, v.port) != (self.host, self.port): + # We got a message not intended for us? + # XXX note this check breaks if we have multiple external IPs + # yay for suck protocols + log.msg("Dropping incorrectly addressed message") + return + del message.headers["via"][0] + if not message.headers["via"]: + # This message is addressed to us + self.gotResponse(message, addr) + return + self.deliverResponse(message) + + + def gotResponse(self, message, addr): + """ + Called with responses that are addressed at this server. + """ + pass + + + +class IAuthorizer(Interface): + def getChallenge(peer): + """ + Generate a challenge the client may respond to. + + @type peer: C{tuple} + @param peer: The client's address + + @rtype: C{str} + @return: The challenge string + """ + + + def decode(response): + """ + Create a credentials object from the given response. + + @type response: C{str} + """ + + + +class RegisterProxy(Proxy): + """ + A proxy that allows registration for a specific domain. + + Unregistered users won't be handled. + """ + + portal = None + + registry = None # Should implement IRegistry + + authorizers = {} + + def __init__(self, *args, **kw): + Proxy.__init__(self, *args, **kw) + self.liveChallenges = {} + + + def handle_ACK_request(self, message, host_port): + # XXX + # ACKs are a client's way of indicating they got the last message + # Responding to them is not a good idea. + # However, we should keep track of terminal messages and re-transmit + # if no ACK is received. + (host, port) = host_port + pass + + + def handle_REGISTER_request(self, message, host_port): + """ + Handle a registration request. + + Currently registration is not proxied. + """ + (host, port) = host_port + if self.portal is None: + # There is no portal. Let anyone in. + self.register(message, host, port) + else: + # There is a portal. Check for credentials. + if "authorization" not in message.headers: + return self.unauthorized(message, host, port) + else: + return self.login(message, host, port) + + + def unauthorized(self, message, host, port): + m = self.responseFromRequest(401, message) + for (scheme, auth) in iteritems(self.authorizers): + chal = auth.getChallenge((host, port)) + if chal is None: + value = '%s realm="%s"' % (scheme.title(), self.host) + else: + value = '%s %s,realm="%s"' % (scheme.title(), chal, self.host) + m.headers.setdefault('www-authenticate', []).append(value) + self.deliverResponse(m) + + + def login(self, message, host, port): + parts = message.headers['authorization'][0].split(None, 1) + a = self.authorizers.get(parts[0].lower()) + if a: + try: + c = a.decode(parts[1]) + except SIPError: + raise + except: + log.err() + self.deliverResponse(self.responseFromRequest(500, message)) + else: + c.username += '@' + self.host + self.portal.login(c, None, IContact + ).addCallback(self._cbLogin, message, host, port + ).addErrback(self._ebLogin, message, host, port + ).addErrback(log.err + ) + else: + self.deliverResponse(self.responseFromRequest(501, message)) + + + def _cbLogin(self, i_a_l, message, host, port): + # It's stateless, matey. What a joke. + (i, a, l) = i_a_l + self.register(message, host, port) + + + def _ebLogin(self, failure, message, host, port): + failure.trap(cred.error.UnauthorizedLogin) + self.unauthorized(message, host, port) + + + def register(self, message, host, port): + """ + Allow all users to register + """ + name, toURL, params = parseAddress(message.headers["to"][0], clean=1) + contact = None + if "contact" in message.headers: + contact = message.headers["contact"][0] + + if message.headers.get("expires", [None])[0] == "0": + self.unregister(message, toURL, contact) + else: + # XXX Check expires on appropriate URL, and pass it to registry + # instead of having registry hardcode it. + if contact is not None: + name, contactURL, params = parseAddress(contact, host=host, port=port) + d = self.registry.registerAddress(message.uri, toURL, contactURL) + else: + d = self.registry.getRegistrationInfo(toURL) + d.addCallbacks(self._cbRegister, self._ebRegister, + callbackArgs=(message,), + errbackArgs=(message,) + ) + + + def _cbRegister(self, registration, message): + response = self.responseFromRequest(200, message) + if registration.contactURL != None: + response.addHeader("contact", registration.contactURL.toString()) + response.addHeader("expires", "%d" % registration.secondsToExpiry) + response.addHeader("content-length", "0") + self.deliverResponse(response) + + + def _ebRegister(self, error, message): + error.trap(RegistrationError, LookupError) + # XXX return error message, and alter tests to deal with + # this, currently tests assume no message sent on failure + + + def unregister(self, message, toURL, contact): + try: + expires = int(message.headers["expires"][0]) + except ValueError: + self.deliverResponse(self.responseFromRequest(400, message)) + else: + if expires == 0: + if contact == "*": + contactURL = "*" + else: + name, contactURL, params = parseAddress(contact) + d = self.registry.unregisterAddress(message.uri, toURL, contactURL) + d.addCallback(self._cbUnregister, message + ).addErrback(self._ebUnregister, message + ) + + + def _cbUnregister(self, registration, message): + msg = self.responseFromRequest(200, message) + msg.headers.setdefault('contact', []).append(registration.contactURL.toString()) + msg.addHeader("expires", "0") + self.deliverResponse(msg) + + + def _ebUnregister(self, registration, message): + pass + + + +@implementer(IRegistry, ILocator) +class InMemoryRegistry: + """ + A simplistic registry for a specific domain. + """ + def __init__(self, domain): + self.domain = domain # The domain we handle registration for + self.users = {} # Map username to (IDelayedCall for expiry, address URI) + + + def getAddress(self, userURI): + if userURI.host != self.domain: + return defer.fail(LookupError("unknown domain")) + if userURI.username in self.users: + dc, url = self.users[userURI.username] + return defer.succeed(url) + else: + return defer.fail(LookupError("no such user")) + + + def getRegistrationInfo(self, userURI): + if userURI.host != self.domain: + return defer.fail(LookupError("unknown domain")) + if userURI.username in self.users: + dc, url = self.users[userURI.username] + return defer.succeed(Registration(int(dc.getTime() - time.time()), url)) + else: + return defer.fail(LookupError("no such user")) + + + def _expireRegistration(self, username): + try: + dc, url = self.users[username] + except KeyError: + return defer.fail(LookupError("no such user")) + else: + dc.cancel() + del self.users[username] + return defer.succeed(Registration(0, url)) + + + def registerAddress(self, domainURL, logicalURL, physicalURL): + if domainURL.host != self.domain: + log.msg("Registration for domain we don't handle.") + return defer.fail(RegistrationError(404)) + if logicalURL.host != self.domain: + log.msg("Registration for domain we don't handle.") + return defer.fail(RegistrationError(404)) + if logicalURL.username in self.users: + dc, old = self.users[logicalURL.username] + dc.reset(3600) + else: + dc = reactor.callLater(3600, self._expireRegistration, logicalURL.username) + log.msg("Registered %s at %s" % (logicalURL.toString(), physicalURL.toString())) + self.users[logicalURL.username] = (dc, physicalURL) + return defer.succeed(Registration(int(dc.getTime() - time.time()), physicalURL)) + + + def unregisterAddress(self, domainURL, logicalURL, physicalURL): + return self._expireRegistration(logicalURL.username) diff --git a/contrib/python/Twisted/py2/twisted/protocols/socks.py b/contrib/python/Twisted/py2/twisted/protocols/socks.py new file mode 100644 index 00000000000..a52c09b6697 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/socks.py @@ -0,0 +1,255 @@ +# -*- test-case-name: twisted.test.test_socks -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of the SOCKSv4 protocol. +""" + +# python imports +import struct +import string +import socket +import time + +# twisted imports +from twisted.internet import reactor, protocol, defer +from twisted.python import log + + +class SOCKSv4Outgoing(protocol.Protocol): + def __init__(self, socks): + self.socks=socks + + + def connectionMade(self): + peer = self.transport.getPeer() + self.socks.makeReply(90, 0, port=peer.port, ip=peer.host) + self.socks.otherConn=self + + + def connectionLost(self, reason): + self.socks.transport.loseConnection() + + + def dataReceived(self, data): + self.socks.write(data) + + + def write(self,data): + self.socks.log(self,data) + self.transport.write(data) + + + +class SOCKSv4Incoming(protocol.Protocol): + def __init__(self,socks): + self.socks=socks + self.socks.otherConn=self + + + def connectionLost(self, reason): + self.socks.transport.loseConnection() + + + def dataReceived(self,data): + self.socks.write(data) + + + def write(self, data): + self.socks.log(self,data) + self.transport.write(data) + + + +class SOCKSv4(protocol.Protocol): + """ + An implementation of the SOCKSv4 protocol. + + @type logging: L{str} or L{None} + @ivar logging: If not L{None}, the name of the logfile to which connection + information will be written. + + @type reactor: object providing L{twisted.internet.interfaces.IReactorTCP} + @ivar reactor: The reactor used to create connections. + + @type buf: L{str} + @ivar buf: Part of a SOCKSv4 connection request. + + @type otherConn: C{SOCKSv4Incoming}, C{SOCKSv4Outgoing} or L{None} + @ivar otherConn: Until the connection has been established, C{otherConn} is + L{None}. After that, it is the proxy-to-destination protocol instance + along which the client's connection is being forwarded. + """ + def __init__(self, logging=None, reactor=reactor): + self.logging = logging + self.reactor = reactor + + + def connectionMade(self): + self.buf = b"" + self.otherConn = None + + + def dataReceived(self, data): + """ + Called whenever data is received. + + @type data: L{bytes} + @param data: Part or all of a SOCKSv4 packet. + """ + if self.otherConn: + self.otherConn.write(data) + return + self.buf = self.buf + data + completeBuffer = self.buf + if b"\000" in self.buf[8:]: + head, self.buf = self.buf[:8], self.buf[8:] + version, code, port = struct.unpack("!BBH", head[:4]) + user, self.buf = self.buf.split(b"\000", 1) + if head[4:7] == b"\000\000\000" and head[7:8] != b"\000": + # An IP address of the form 0.0.0.X, where X is non-zero, + # signifies that this is a SOCKSv4a packet. + # If the complete packet hasn't been received, restore the + # buffer and wait for it. + if b"\000" not in self.buf: + self.buf = completeBuffer + return + server, self.buf = self.buf.split(b"\000", 1) + d = self.reactor.resolve(server) + d.addCallback(self._dataReceived2, user, + version, code, port) + d.addErrback(lambda result, self = self: self.makeReply(91)) + return + else: + server = socket.inet_ntoa(head[4:8]) + + self._dataReceived2(server, user, version, code, port) + + + def _dataReceived2(self, server, user, version, code, port): + """ + The second half of the SOCKS connection setup. For a SOCKSv4 packet this + is after the server address has been extracted from the header. For a + SOCKSv4a packet this is after the host name has been resolved. + + @type server: L{str} + @param server: The IP address of the destination, represented as a + dotted quad. + + @type user: L{str} + @param user: The username associated with the connection. + + @type version: L{int} + @param version: The SOCKS protocol version number. + + @type code: L{int} + @param code: The comand code. 1 means establish a TCP/IP stream + connection, and 2 means establish a TCP/IP port binding. + + @type port: L{int} + @param port: The port number associated with the connection. + """ + assert version == 4, "Bad version code: %s" % version + if not self.authorize(code, server, port, user): + self.makeReply(91) + return + if code == 1: # CONNECT + d = self.connectClass(server, port, SOCKSv4Outgoing, self) + d.addErrback(lambda result, self = self: self.makeReply(91)) + elif code == 2: # BIND + d = self.listenClass(0, SOCKSv4IncomingFactory, self, server) + d.addCallback(lambda x, + self = self: self.makeReply(90, 0, x[1], x[0])) + else: + raise RuntimeError("Bad Connect Code: %s" % (code,)) + assert self.buf == b"", "hmm, still stuff in buffer... %s" % repr( + self.buf) + + + def connectionLost(self, reason): + if self.otherConn: + self.otherConn.transport.loseConnection() + + + def authorize(self,code,server,port,user): + log.msg("code %s connection to %s:%s (user %s) authorized" % (code,server,port,user)) + return 1 + + + def connectClass(self, host, port, klass, *args): + return protocol.ClientCreator(reactor, klass, *args).connectTCP(host,port) + + + def listenClass(self, port, klass, *args): + serv = reactor.listenTCP(port, klass(*args)) + return defer.succeed(serv.getHost()[1:]) + + + def makeReply(self,reply,version=0,port=0,ip="0.0.0.0"): + self.transport.write(struct.pack("!BBH",version,reply,port)+socket.inet_aton(ip)) + if reply!=90: self.transport.loseConnection() + + + def write(self,data): + self.log(self,data) + self.transport.write(data) + + + def log(self,proto,data): + if not self.logging: return + peer = self.transport.getPeer() + their_peer = self.otherConn.transport.getPeer() + f=open(self.logging,"a") + f.write("%s\t%s:%d %s %s:%d\n"%(time.ctime(), + peer.host,peer.port, + ((proto==self and '<') or '>'), + their_peer.host,their_peer.port)) + while data: + p,data=data[:16],data[16:] + f.write(string.join(map(lambda x:'%02X'%ord(x),p),' ')+' ') + f.write((16-len(p))*3*' ') + for c in p: + if len(repr(c))>3: f.write('.') + else: f.write(c) + f.write('\n') + f.write('\n') + f.close() + + + +class SOCKSv4Factory(protocol.Factory): + """ + A factory for a SOCKSv4 proxy. + + Constructor accepts one argument, a log file name. + """ + def __init__(self, log): + self.logging = log + + + def buildProtocol(self, addr): + return SOCKSv4(self.logging, reactor) + + + +class SOCKSv4IncomingFactory(protocol.Factory): + """ + A utility class for building protocols for incoming connections. + """ + def __init__(self, socks, ip): + self.socks = socks + self.ip = ip + + + def buildProtocol(self, addr): + if addr[0] == self.ip: + self.ip = "" + self.socks.makeReply(90, 0) + return SOCKSv4Incoming(self.socks) + elif self.ip == "": + return None + else: + self.socks.makeReply(91, 0) + self.ip = "" + return None diff --git a/contrib/python/Twisted/py2/twisted/protocols/stateful.py b/contrib/python/Twisted/py2/twisted/protocols/stateful.py new file mode 100644 index 00000000000..cd2b7cfe707 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/stateful.py @@ -0,0 +1,49 @@ +# -*- test-case-name: twisted.test.test_stateful -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +from twisted.internet import protocol + +from io import BytesIO + +class StatefulProtocol(protocol.Protocol): + """A Protocol that stores state for you. + + state is a pair (function, num_bytes). When num_bytes bytes of data arrives + from the network, function is called. It is expected to return the next + state or None to keep same state. Initial state is returned by + getInitialState (override it). + """ + _sful_data = None, None, 0 + + def makeConnection(self, transport): + protocol.Protocol.makeConnection(self, transport) + self._sful_data = self.getInitialState(), BytesIO(), 0 + + def getInitialState(self): + raise NotImplementedError + + def dataReceived(self, data): + state, buffer, offset = self._sful_data + buffer.seek(0, 2) + buffer.write(data) + blen = buffer.tell() # how many bytes total is in the buffer + buffer.seek(offset) + while blen - offset >= state[1]: + d = buffer.read(state[1]) + offset += state[1] + next = state[0](d) + if self.transport.disconnecting: # XXX: argh stupid hack borrowed right from LineReceiver + return # dataReceived won't be called again, so who cares about consistent state + if next: + state = next + if offset != 0: + b = buffer.read() + buffer.seek(0) + buffer.truncate() + buffer.write(b) + offset = 0 + self._sful_data = state, buffer, offset + diff --git a/contrib/python/Twisted/py2/twisted/protocols/tls.py b/contrib/python/Twisted/py2/twisted/protocols/tls.py new file mode 100644 index 00000000000..52cd498aa96 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/tls.py @@ -0,0 +1,830 @@ +# -*- test-case-name: twisted.protocols.test.test_tls,twisted.internet.test.test_tls,twisted.test.test_sslverify -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of a TLS transport (L{ISSLTransport}) as an +L{IProtocol} layered on top of any +L{ITransport} implementation, based on +U{OpenSSL}'s memory BIO features. + +L{TLSMemoryBIOFactory} is a L{WrappingFactory} which wraps protocols created by +the factory it wraps with L{TLSMemoryBIOProtocol}. L{TLSMemoryBIOProtocol} +intercedes between the underlying transport and the wrapped protocol to +implement SSL and TLS. Typical usage of this module looks like this:: + + from twisted.protocols.tls import TLSMemoryBIOFactory + from twisted.internet.protocol import ServerFactory + from twisted.internet.ssl import PrivateCertificate + from twisted.internet import reactor + + from someapplication import ApplicationProtocol + + serverFactory = ServerFactory() + serverFactory.protocol = ApplicationProtocol + certificate = PrivateCertificate.loadPEM(certPEMData) + contextFactory = certificate.options() + tlsFactory = TLSMemoryBIOFactory(contextFactory, False, serverFactory) + reactor.listenTCP(12345, tlsFactory) + reactor.run() + +This API offers somewhat more flexibility than +L{twisted.internet.interfaces.IReactorSSL}; for example, a +L{TLSMemoryBIOProtocol} instance can use another instance of +L{TLSMemoryBIOProtocol} as its transport, yielding TLS over TLS - useful to +implement onion routing. It can also be used to run TLS over unusual +transports, such as UNIX sockets and stdio. +""" + +from __future__ import division, absolute_import + +from OpenSSL.SSL import Error, ZeroReturnError, WantReadError +from OpenSSL.SSL import TLSv1_METHOD, Context, Connection + +try: + Connection(Context(TLSv1_METHOD), None) +except TypeError as e: + if str(e) != "argument must be an int, or have a fileno() method.": + raise + raise ImportError("twisted.protocols.tls requires pyOpenSSL 0.10 or newer.") + +from zope.interface import implementer, providedBy, directlyProvides + +from twisted.python.compat import unicode +from twisted.python.failure import Failure +from twisted.internet.interfaces import ( + ISystemHandle, INegotiated, IPushProducer, ILoggingContext, + IOpenSSLServerConnectionCreator, IOpenSSLClientConnectionCreator, + IProtocolNegotiationFactory, IHandshakeListener +) +from twisted.internet.main import CONNECTION_LOST +from twisted.internet._producer_helpers import _PullToPush +from twisted.internet.protocol import Protocol +from twisted.internet._sslverify import _setAcceptableProtocols +from twisted.protocols.policies import ProtocolWrapper, WrappingFactory + + +@implementer(IPushProducer) +class _ProducerMembrane(object): + """ + Stand-in for producer registered with a L{TLSMemoryBIOProtocol} transport. + + Ensures that producer pause/resume events from the undelying transport are + coordinated with pause/resume events from the TLS layer. + + @ivar _producer: The application-layer producer. + """ + + _producerPaused = False + + def __init__(self, producer): + self._producer = producer + + + def pauseProducing(self): + """ + C{pauseProducing} the underlying producer, if it's not paused. + """ + if self._producerPaused: + return + self._producerPaused = True + self._producer.pauseProducing() + + + def resumeProducing(self): + """ + C{resumeProducing} the underlying producer, if it's paused. + """ + if not self._producerPaused: + return + self._producerPaused = False + self._producer.resumeProducing() + + + def stopProducing(self): + """ + C{stopProducing} the underlying producer. + + There is only a single source for this event, so it's simply passed + on. + """ + self._producer.stopProducing() + + +@implementer(ISystemHandle, INegotiated) +class TLSMemoryBIOProtocol(ProtocolWrapper): + """ + L{TLSMemoryBIOProtocol} is a protocol wrapper which uses OpenSSL via a + memory BIO to encrypt bytes written to it before sending them on to the + underlying transport and decrypts bytes received from the underlying + transport before delivering them to the wrapped protocol. + + In addition to producer events from the underlying transport, the need to + wait for reads before a write can proceed means the L{TLSMemoryBIOProtocol} + may also want to pause a producer. Pause/resume events are therefore + merged using the L{_ProducerMembrane} wrapper. Non-streaming (pull) + producers are supported by wrapping them with L{_PullToPush}. + + @ivar _tlsConnection: The L{OpenSSL.SSL.Connection} instance which is + encrypted and decrypting this connection. + + @ivar _lostTLSConnection: A flag indicating whether connection loss has + already been dealt with (C{True}) or not (C{False}). TLS disconnection + is distinct from the underlying connection being lost. + + @ivar _appSendBuffer: application-level (cleartext) data that is waiting to + be transferred to the TLS buffer, but can't be because the TLS + connection is handshaking. + @type _appSendBuffer: L{list} of L{bytes} + + @ivar _connectWrapped: A flag indicating whether or not to call + C{makeConnection} on the wrapped protocol. This is for the reactor's + L{twisted.internet.interfaces.ITLSTransport.startTLS} implementation, + since it has a protocol which it has already called C{makeConnection} + on, and which has no interest in a new transport. See #3821. + + @ivar _handshakeDone: A flag indicating whether or not the handshake is + known to have completed successfully (C{True}) or not (C{False}). This + is used to control error reporting behavior. If the handshake has not + completed, the underlying L{OpenSSL.SSL.Error} will be passed to the + application's C{connectionLost} method. If it has completed, any + unexpected L{OpenSSL.SSL.Error} will be turned into a + L{ConnectionLost}. This is weird; however, it is simply an attempt at + a faithful re-implementation of the behavior provided by + L{twisted.internet.ssl}. + + @ivar _reason: If an unexpected L{OpenSSL.SSL.Error} occurs which causes + the connection to be lost, it is saved here. If appropriate, this may + be used as the reason passed to the application protocol's + C{connectionLost} method. + + @ivar _producer: The current producer registered via C{registerProducer}, + or L{None} if no producer has been registered or a previous one was + unregistered. + + @ivar _aborted: C{abortConnection} has been called. No further data will + be received to the wrapped protocol's C{dataReceived}. + @type _aborted: L{bool} + """ + + _reason = None + _handshakeDone = False + _lostTLSConnection = False + _producer = None + _aborted = False + + def __init__(self, factory, wrappedProtocol, _connectWrapped=True): + ProtocolWrapper.__init__(self, factory, wrappedProtocol) + self._connectWrapped = _connectWrapped + + + def getHandle(self): + """ + Return the L{OpenSSL.SSL.Connection} object being used to encrypt and + decrypt this connection. + + This is done for the benefit of L{twisted.internet.ssl.Certificate}'s + C{peerFromTransport} and C{hostFromTransport} methods only. A + different system handle may be returned by future versions of this + method. + """ + return self._tlsConnection + + + def makeConnection(self, transport): + """ + Connect this wrapper to the given transport and initialize the + necessary L{OpenSSL.SSL.Connection} with a memory BIO. + """ + self._tlsConnection = self.factory._createConnection(self) + self._appSendBuffer = [] + + # Add interfaces provided by the transport we are wrapping: + for interface in providedBy(transport): + directlyProvides(self, interface) + + # Intentionally skip ProtocolWrapper.makeConnection - it might call + # wrappedProtocol.makeConnection, which we want to make conditional. + Protocol.makeConnection(self, transport) + self.factory.registerProtocol(self) + if self._connectWrapped: + # Now that the TLS layer is initialized, notify the application of + # the connection. + ProtocolWrapper.makeConnection(self, transport) + + # Now that we ourselves have a transport (initialized by the + # ProtocolWrapper.makeConnection call above), kick off the TLS + # handshake. + self._checkHandshakeStatus() + + + def _checkHandshakeStatus(self): + """ + Ask OpenSSL to proceed with a handshake in progress. + + Initially, this just sends the ClientHello; after some bytes have been + stuffed in to the C{Connection} object by C{dataReceived}, it will then + respond to any C{Certificate} or C{KeyExchange} messages. + """ + # The connection might already be aborted (eg. by a callback during + # connection setup), so don't even bother trying to handshake in that + # case. + if self._aborted: + return + try: + self._tlsConnection.do_handshake() + except WantReadError: + self._flushSendBIO() + except Error: + self._tlsShutdownFinished(Failure()) + else: + self._handshakeDone = True + if IHandshakeListener.providedBy(self.wrappedProtocol): + self.wrappedProtocol.handshakeCompleted() + + + def _flushSendBIO(self): + """ + Read any bytes out of the send BIO and write them to the underlying + transport. + """ + try: + bytes = self._tlsConnection.bio_read(2 ** 15) + except WantReadError: + # There may be nothing in the send BIO right now. + pass + else: + self.transport.write(bytes) + + + def _flushReceiveBIO(self): + """ + Try to receive any application-level bytes which are now available + because of a previous write into the receive BIO. This will take + care of delivering any application-level bytes which are received to + the protocol, as well as handling of the various exceptions which + can come from trying to get such bytes. + """ + # Keep trying this until an error indicates we should stop or we + # close the connection. Looping is necessary to make sure we + # process all of the data which was put into the receive BIO, as + # there is no guarantee that a single recv call will do it all. + while not self._lostTLSConnection: + try: + bytes = self._tlsConnection.recv(2 ** 15) + except WantReadError: + # The newly received bytes might not have been enough to produce + # any application data. + break + except ZeroReturnError: + # TLS has shut down and no more TLS data will be received over + # this connection. + self._shutdownTLS() + # Passing in None means the user protocol's connnectionLost + # will get called with reason from underlying transport: + self._tlsShutdownFinished(None) + except Error: + # Something went pretty wrong. For example, this might be a + # handshake failure during renegotiation (because there were no + # shared ciphers, because a certificate failed to verify, etc). + # TLS can no longer proceed. + failure = Failure() + self._tlsShutdownFinished(failure) + else: + if not self._aborted: + ProtocolWrapper.dataReceived(self, bytes) + + # The received bytes might have generated a response which needs to be + # sent now. For example, the handshake involves several round-trip + # exchanges without ever producing application-bytes. + self._flushSendBIO() + + + def dataReceived(self, bytes): + """ + Deliver any received bytes to the receive BIO and then read and deliver + to the application any application-level data which becomes available + as a result of this. + """ + # Let OpenSSL know some bytes were just received. + self._tlsConnection.bio_write(bytes) + + # If we are still waiting for the handshake to complete, try to + # complete the handshake with the bytes we just received. + if not self._handshakeDone: + self._checkHandshakeStatus() + + # If the handshake still isn't finished, then we've nothing left to + # do. + if not self._handshakeDone: + return + + # If we've any pending writes, this read may have un-blocked them, so + # attempt to unbuffer them into the OpenSSL layer. + if self._appSendBuffer: + self._unbufferPendingWrites() + + # Since the handshake is complete, the wire-level bytes we just + # processed might turn into some application-level bytes; try to pull + # those out. + self._flushReceiveBIO() + + + def _shutdownTLS(self): + """ + Initiate, or reply to, the shutdown handshake of the TLS layer. + """ + try: + shutdownSuccess = self._tlsConnection.shutdown() + except Error: + # Mid-handshake, a call to shutdown() can result in a + # WantWantReadError, or rather an SSL_ERR_WANT_READ; but pyOpenSSL + # doesn't allow us to get at the error. See: + # https://github.com/pyca/pyopenssl/issues/91 + shutdownSuccess = False + self._flushSendBIO() + if shutdownSuccess: + # Both sides have shutdown, so we can start closing lower-level + # transport. This will also happen if we haven't started + # negotiation at all yet, in which case shutdown succeeds + # immediately. + self.transport.loseConnection() + + + def _tlsShutdownFinished(self, reason): + """ + Called when TLS connection has gone away; tell underlying transport to + disconnect. + + @param reason: a L{Failure} whose value is an L{Exception} if we want to + report that failure through to the wrapped protocol's + C{connectionLost}, or L{None} if the C{reason} that + C{connectionLost} should receive should be coming from the + underlying transport. + @type reason: L{Failure} or L{None} + """ + if reason is not None: + # Squash an EOF in violation of the TLS protocol into + # ConnectionLost, so that applications which might run over + # multiple protocols can recognize its type. + if tuple(reason.value.args[:2]) == (-1, 'Unexpected EOF'): + reason = Failure(CONNECTION_LOST) + if self._reason is None: + self._reason = reason + self._lostTLSConnection = True + # We may need to send a TLS alert regarding the nature of the shutdown + # here (for example, why a handshake failed), so always flush our send + # buffer before telling our lower-level transport to go away. + self._flushSendBIO() + # Using loseConnection causes the application protocol's + # connectionLost method to be invoked non-reentrantly, which is always + # a nice feature. However, for error cases (reason != None) we might + # want to use abortConnection when it becomes available. The + # loseConnection call is basically tested by test_handshakeFailure. + # At least one side will need to do it or the test never finishes. + self.transport.loseConnection() + + + def connectionLost(self, reason): + """ + Handle the possible repetition of calls to this method (due to either + the underlying transport going away or due to an error at the TLS + layer) and make sure the base implementation only gets invoked once. + """ + if not self._lostTLSConnection: + # Tell the TLS connection that it's not going to get any more data + # and give it a chance to finish reading. + self._tlsConnection.bio_shutdown() + self._flushReceiveBIO() + self._lostTLSConnection = True + reason = self._reason or reason + self._reason = None + self.connected = False + ProtocolWrapper.connectionLost(self, reason) + + # Breaking reference cycle between self._tlsConnection and self. + self._tlsConnection = None + + + def loseConnection(self): + """ + Send a TLS close alert and close the underlying connection. + """ + if self.disconnecting or not self.connected: + return + # If connection setup has not finished, OpenSSL 1.0.2f+ will not shut + # down the connection until we write some data to the connection which + # allows the handshake to complete. However, since no data should be + # written after loseConnection, this means we'll be stuck forever + # waiting for shutdown to complete. Instead, we simply abort the + # connection without trying to shut down cleanly: + if not self._handshakeDone and not self._appSendBuffer: + self.abortConnection() + self.disconnecting = True + if not self._appSendBuffer and self._producer is None: + self._shutdownTLS() + + + def abortConnection(self): + """ + Tear down TLS state so that if the connection is aborted mid-handshake + we don't deliver any further data from the application. + """ + self._aborted = True + self.disconnecting = True + self._shutdownTLS() + self.transport.abortConnection() + + + def failVerification(self, reason): + """ + Abort the connection during connection setup, giving a reason that + certificate verification failed. + + @param reason: The reason that the verification failed; reported to the + application protocol's C{connectionLost} method. + @type reason: L{Failure} + """ + self._reason = reason + self.abortConnection() + + + def write(self, bytes): + """ + Process the given application bytes and send any resulting TLS traffic + which arrives in the send BIO. + + If C{loseConnection} was called, subsequent calls to C{write} will + drop the bytes on the floor. + """ + if isinstance(bytes, unicode): + raise TypeError("Must write bytes to a TLS transport, not unicode.") + # Writes after loseConnection are not supported, unless a producer has + # been registered, in which case writes can happen until the producer + # is unregistered: + if self.disconnecting and self._producer is None: + return + self._write(bytes) + + + def _bufferedWrite(self, octets): + """ + Put the given octets into L{TLSMemoryBIOProtocol._appSendBuffer}, and + tell any listening producer that it should pause because we are now + buffering. + """ + self._appSendBuffer.append(octets) + if self._producer is not None: + self._producer.pauseProducing() + + + def _unbufferPendingWrites(self): + """ + Un-buffer all waiting writes in L{TLSMemoryBIOProtocol._appSendBuffer}. + """ + pendingWrites, self._appSendBuffer = self._appSendBuffer, [] + for eachWrite in pendingWrites: + self._write(eachWrite) + + if self._appSendBuffer: + # If OpenSSL ran out of buffer space in the Connection on our way + # through the loop earlier and re-buffered any of our outgoing + # writes, then we're done; don't consider any future work. + return + + if self._producer is not None: + # If we have a registered producer, let it know that we have some + # more buffer space. + self._producer.resumeProducing() + return + + if self.disconnecting: + # Finally, if we have no further buffered data, no producer wants + # to send us more data in the future, and the application told us + # to end the stream, initiate a TLS shutdown. + self._shutdownTLS() + + + def _write(self, bytes): + """ + Process the given application bytes and send any resulting TLS traffic + which arrives in the send BIO. + + This may be called by C{dataReceived} with bytes that were buffered + before C{loseConnection} was called, which is why this function + doesn't check for disconnection but accepts the bytes regardless. + """ + if self._lostTLSConnection: + return + + # A TLS payload is 16kB max + bufferSize = 2 ** 14 + + # How far into the input we've gotten so far + alreadySent = 0 + + while alreadySent < len(bytes): + toSend = bytes[alreadySent:alreadySent + bufferSize] + try: + sent = self._tlsConnection.send(toSend) + except WantReadError: + self._bufferedWrite(bytes[alreadySent:]) + break + except Error: + # Pretend TLS connection disconnected, which will trigger + # disconnect of underlying transport. The error will be passed + # to the application protocol's connectionLost method. The + # other SSL implementation doesn't, but losing helpful + # debugging information is a bad idea. + self._tlsShutdownFinished(Failure()) + break + else: + # We've successfully handed off the bytes to the OpenSSL + # Connection object. + alreadySent += sent + # See if OpenSSL wants to hand any bytes off to the underlying + # transport as a result. + self._flushSendBIO() + + + def writeSequence(self, iovec): + """ + Write a sequence of application bytes by joining them into one string + and passing them to L{write}. + """ + self.write(b"".join(iovec)) + + + def getPeerCertificate(self): + return self._tlsConnection.get_peer_certificate() + + + @property + def negotiatedProtocol(self): + """ + @see: L{INegotiated.negotiatedProtocol} + """ + protocolName = None + + try: + # If ALPN is not implemented that's ok, NPN might be. + protocolName = self._tlsConnection.get_alpn_proto_negotiated() + except (NotImplementedError, AttributeError): + pass + + if protocolName not in (b'', None): + # A protocol was selected using ALPN. + return protocolName + + try: + protocolName = self._tlsConnection.get_next_proto_negotiated() + except (NotImplementedError, AttributeError): + pass + + if protocolName != b'': + return protocolName + + return None + + + def registerProducer(self, producer, streaming): + # If we've already disconnected, nothing to do here: + if self._lostTLSConnection: + producer.stopProducing() + return + + # If we received a non-streaming producer, wrap it so it becomes a + # streaming producer: + if not streaming: + producer = streamingProducer = _PullToPush(producer, self) + producer = _ProducerMembrane(producer) + # This will raise an exception if a producer is already registered: + self.transport.registerProducer(producer, True) + self._producer = producer + # If we received a non-streaming producer, we need to start the + # streaming wrapper: + if not streaming: + streamingProducer.startStreaming() + + + def unregisterProducer(self): + # If we have no producer, we don't need to do anything here. + if self._producer is None: + return + + # If we received a non-streaming producer, we need to stop the + # streaming wrapper: + if isinstance(self._producer._producer, _PullToPush): + self._producer._producer.stopStreaming() + self._producer = None + self._producerPaused = False + self.transport.unregisterProducer() + if self.disconnecting and not self._appSendBuffer: + self._shutdownTLS() + + + +@implementer(IOpenSSLClientConnectionCreator, IOpenSSLServerConnectionCreator) +class _ContextFactoryToConnectionFactory(object): + """ + Adapter wrapping a L{twisted.internet.interfaces.IOpenSSLContextFactory} + into a L{IOpenSSLClientConnectionCreator} or + L{IOpenSSLServerConnectionCreator}. + + See U{https://twistedmatrix.com/trac/ticket/7215} for work that should make + this unnecessary. + """ + + def __init__(self, oldStyleContextFactory): + """ + Construct a L{_ContextFactoryToConnectionFactory} with a + L{twisted.internet.interfaces.IOpenSSLContextFactory}. + + Immediately call C{getContext} on C{oldStyleContextFactory} in order to + force advance parameter checking, since old-style context factories + don't actually check that their arguments to L{OpenSSL} are correct. + + @param oldStyleContextFactory: A factory that can produce contexts. + @type oldStyleContextFactory: + L{twisted.internet.interfaces.IOpenSSLContextFactory} + """ + oldStyleContextFactory.getContext() + self._oldStyleContextFactory = oldStyleContextFactory + + + def _connectionForTLS(self, protocol): + """ + Create an L{OpenSSL.SSL.Connection} object. + + @param protocol: The protocol initiating a TLS connection. + @type protocol: L{TLSMemoryBIOProtocol} + + @return: a connection + @rtype: L{OpenSSL.SSL.Connection} + """ + context = self._oldStyleContextFactory.getContext() + return Connection(context, None) + + + def serverConnectionForTLS(self, protocol): + """ + Construct an OpenSSL server connection from the wrapped old-style + context factory. + + @note: Since old-style context factories don't distinguish between + clients and servers, this is exactly the same as + L{_ContextFactoryToConnectionFactory.clientConnectionForTLS}. + + @param protocol: The protocol initiating a TLS connection. + @type protocol: L{TLSMemoryBIOProtocol} + + @return: a connection + @rtype: L{OpenSSL.SSL.Connection} + """ + return self._connectionForTLS(protocol) + + + def clientConnectionForTLS(self, protocol): + """ + Construct an OpenSSL server connection from the wrapped old-style + context factory. + + @note: Since old-style context factories don't distinguish between + clients and servers, this is exactly the same as + L{_ContextFactoryToConnectionFactory.serverConnectionForTLS}. + + @param protocol: The protocol initiating a TLS connection. + @type protocol: L{TLSMemoryBIOProtocol} + + @return: a connection + @rtype: L{OpenSSL.SSL.Connection} + """ + return self._connectionForTLS(protocol) + + + +class TLSMemoryBIOFactory(WrappingFactory): + """ + L{TLSMemoryBIOFactory} adds TLS to connections. + + @ivar _creatorInterface: the interface which L{_connectionCreator} is + expected to implement. + @type _creatorInterface: L{zope.interface.interfaces.IInterface} + + @ivar _connectionCreator: a callable which creates an OpenSSL Connection + object. + @type _connectionCreator: 1-argument callable taking + L{TLSMemoryBIOProtocol} and returning L{OpenSSL.SSL.Connection}. + """ + protocol = TLSMemoryBIOProtocol + + noisy = False # disable unnecessary logging. + + def __init__(self, contextFactory, isClient, wrappedFactory): + """ + Create a L{TLSMemoryBIOFactory}. + + @param contextFactory: Configuration parameters used to create an + OpenSSL connection. In order of preference, what you should pass + here should be: + + 1. L{twisted.internet.ssl.CertificateOptions} (if you're + writing a server) or the result of + L{twisted.internet.ssl.optionsForClientTLS} (if you're + writing a client). If you want security you should really + use one of these. + + 2. If you really want to implement something yourself, supply a + provider of L{IOpenSSLClientConnectionCreator} or + L{IOpenSSLServerConnectionCreator}. + + 3. If you really have to, supply a + L{twisted.internet.ssl.ContextFactory}. This will likely be + deprecated at some point so please upgrade to the new + interfaces. + + @type contextFactory: L{IOpenSSLClientConnectionCreator} or + L{IOpenSSLServerConnectionCreator}, or, for compatibility with + older code, anything implementing + L{twisted.internet.interfaces.IOpenSSLContextFactory}. See + U{https://twistedmatrix.com/trac/ticket/7215} for information on + the upcoming deprecation of passing a + L{twisted.internet.ssl.ContextFactory} here. + + @param isClient: Is this a factory for TLS client connections; in other + words, those that will send a C{ClientHello} greeting? L{True} if + so, L{False} otherwise. This flag determines what interface is + expected of C{contextFactory}. If L{True}, C{contextFactory} + should provide L{IOpenSSLClientConnectionCreator}; otherwise it + should provide L{IOpenSSLServerConnectionCreator}. + @type isClient: L{bool} + + @param wrappedFactory: A factory which will create the + application-level protocol. + @type wrappedFactory: L{twisted.internet.interfaces.IProtocolFactory} + """ + WrappingFactory.__init__(self, wrappedFactory) + if isClient: + creatorInterface = IOpenSSLClientConnectionCreator + else: + creatorInterface = IOpenSSLServerConnectionCreator + self._creatorInterface = creatorInterface + if not creatorInterface.providedBy(contextFactory): + contextFactory = _ContextFactoryToConnectionFactory(contextFactory) + self._connectionCreator = contextFactory + + + def logPrefix(self): + """ + Annotate the wrapped factory's log prefix with some text indicating TLS + is in use. + + @rtype: C{str} + """ + if ILoggingContext.providedBy(self.wrappedFactory): + logPrefix = self.wrappedFactory.logPrefix() + else: + logPrefix = self.wrappedFactory.__class__.__name__ + return "%s (TLS)" % (logPrefix,) + + + def _applyProtocolNegotiation(self, connection): + """ + Applies ALPN/NPN protocol neogitation to the connection, if the factory + supports it. + + @param connection: The OpenSSL connection object to have ALPN/NPN added + to it. + @type connection: L{OpenSSL.SSL.Connection} + + @return: Nothing + @rtype: L{None} + """ + if IProtocolNegotiationFactory.providedBy(self.wrappedFactory): + protocols = self.wrappedFactory.acceptableProtocols() + context = connection.get_context() + _setAcceptableProtocols(context, protocols) + + return + + + def _createConnection(self, tlsProtocol): + """ + Create an OpenSSL connection and set it up good. + + @param tlsProtocol: The protocol which is establishing the connection. + @type tlsProtocol: L{TLSMemoryBIOProtocol} + + @return: an OpenSSL connection object for C{tlsProtocol} to use + @rtype: L{OpenSSL.SSL.Connection} + """ + connectionCreator = self._connectionCreator + if self._creatorInterface is IOpenSSLClientConnectionCreator: + connection = connectionCreator.clientConnectionForTLS(tlsProtocol) + self._applyProtocolNegotiation(connection) + connection.set_connect_state() + else: + connection = connectionCreator.serverConnectionForTLS(tlsProtocol) + self._applyProtocolNegotiation(connection) + connection.set_accept_state() + return connection diff --git a/contrib/python/Twisted/py2/twisted/protocols/wire.py b/contrib/python/Twisted/py2/twisted/protocols/wire.py new file mode 100644 index 00000000000..0e647b32352 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/protocols/wire.py @@ -0,0 +1,124 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +"""Implement standard (and unused) TCP protocols. + +These protocols are either provided by inetd, or are not provided at all. +""" + +from __future__ import absolute_import, division + +import time +import struct + +from zope.interface import implementer + +from twisted.internet import protocol, interfaces + + + +class Echo(protocol.Protocol): + """ + As soon as any data is received, write it back (RFC 862). + """ + + def dataReceived(self, data): + self.transport.write(data) + + + +class Discard(protocol.Protocol): + """ + Discard any received data (RFC 863). + """ + + def dataReceived(self, data): + # I'm ignoring you, nyah-nyah + pass + + + +@implementer(interfaces.IProducer) +class Chargen(protocol.Protocol): + """ + Generate repeating noise (RFC 864). + """ + noise = b'@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ !"#$%&?' + + def connectionMade(self): + self.transport.registerProducer(self, 0) + + + def resumeProducing(self): + self.transport.write(self.noise) + + + def pauseProducing(self): + pass + + + def stopProducing(self): + pass + + + +class QOTD(protocol.Protocol): + """ + Return a quote of the day (RFC 865). + """ + + def connectionMade(self): + self.transport.write(self.getQuote()) + self.transport.loseConnection() + + + def getQuote(self): + """ + Return a quote. May be overrriden in subclasses. + """ + return b"An apple a day keeps the doctor away.\r\n" + + + +class Who(protocol.Protocol): + """ + Return list of active users (RFC 866) + """ + + def connectionMade(self): + self.transport.write(self.getUsers()) + self.transport.loseConnection() + + + def getUsers(self): + """ + Return active users. Override in subclasses. + """ + return b"root\r\n" + + + +class Daytime(protocol.Protocol): + """ + Send back the daytime in ASCII form (RFC 867). + """ + + def connectionMade(self): + self.transport.write(time.asctime(time.gmtime(time.time())) + b'\r\n') + self.transport.loseConnection() + + + +class Time(protocol.Protocol): + """ + Send back the time in machine readable form (RFC 868). + """ + + def connectionMade(self): + # is this correct only for 32-bit machines? + result = struct.pack("!i", int(time.time())) + self.transport.write(result) + self.transport.loseConnection() + + +__all__ = ["Echo", "Discard", "Chargen", "QOTD", "Who", "Daytime", "Time"] diff --git a/contrib/python/Twisted/py2/twisted/python/__init__.py b/contrib/python/Twisted/py2/twisted/python/__init__.py new file mode 100644 index 00000000000..b6d85143f19 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Python: Utilities and Enhancements for Python. +""" + +from __future__ import absolute_import, division + +# Deprecating twisted.python.constants. +from .compat import unicode +from .versions import Version +from .deprecate import deprecatedModuleAttribute + +deprecatedModuleAttribute( + Version("Twisted", 16, 5, 0), + "Please use constantly from PyPI instead.", + "twisted.python", "constants") + + +deprecatedModuleAttribute( + Version('Twisted', 17, 5, 0), + "Please use hyperlink from PyPI instead.", + "twisted.python", "url") + + +del Version +del deprecatedModuleAttribute +del unicode diff --git a/contrib/python/Twisted/py2/twisted/python/_appdirs.py b/contrib/python/Twisted/py2/twisted/python/_appdirs.py new file mode 100644 index 00000000000..15a0ec423f1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_appdirs.py @@ -0,0 +1,32 @@ +# -*- test-case-name: twisted.python.test.test_appdirs -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Application data directory support. +""" + +from __future__ import division, absolute_import + +import appdirs +import inspect + +from twisted.python.compat import currentframe + + +def getDataDirectory(moduleName=None): + """ + Get a data directory for the caller function, or C{moduleName} if given. + + @param moduleName: The module name if you don't wish to have the caller's + module. + @type moduleName: L{str} + + @returns: A directory for putting data in. + @rtype: L{str} + """ + if not moduleName: + caller = currentframe(1) + moduleName = inspect.getmodule(caller).__name__ + + return appdirs.user_data_dir(moduleName) diff --git a/contrib/python/Twisted/py2/twisted/python/_inotify.py b/contrib/python/Twisted/py2/twisted/python/_inotify.py new file mode 100644 index 00000000000..7fb19897ee4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_inotify.py @@ -0,0 +1,110 @@ +# -*- test-case-name: twisted.internet.test.test_inotify -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Very low-level ctypes-based interface to Linux inotify(7). + +ctypes and a version of libc which supports inotify system calls are +required. +""" + +import ctypes +import ctypes.util + + + +class INotifyError(Exception): + """ + Unify all the possible exceptions that can be raised by the INotify API. + """ + + + +def init(): + """ + Create an inotify instance and return the associated file descriptor. + """ + fd = libc.inotify_init() + if fd < 0: + raise INotifyError("INotify initialization error.") + return fd + + + +def add(fd, path, mask): + """ + Add a watch for the given path to the inotify file descriptor, and return + the watch descriptor. + + @param fd: The file descriptor returned by C{libc.inotify_init}. + @type fd: L{int} + + @param path: The path to watch via inotify. + @type path: L{twisted.python.filepath.FilePath} + + @param mask: Bitmask specifying the events that inotify should monitor. + @type mask: L{int} + """ + wd = libc.inotify_add_watch(fd, path.asBytesMode().path, mask) + if wd < 0: + raise INotifyError("Failed to add watch on '%r' - (%r)" % (path, wd)) + return wd + + + +def remove(fd, wd): + """ + Remove the given watch descriptor from the inotify file descriptor. + """ + # When inotify_rm_watch returns -1 there's an error: + # The errno for this call can be either one of the following: + # EBADF: fd is not a valid file descriptor. + # EINVAL: The watch descriptor wd is not valid; or fd is + # not an inotify file descriptor. + # + # if we can't access the errno here we cannot even raise + # an exception and we need to ignore the problem, one of + # the most common cases is when you remove a directory from + # the filesystem and that directory is observed. When inotify + # tries to call inotify_rm_watch with a non existing directory + # either of the 2 errors might come up because the files inside + # it might have events generated way before they were handled. + # Unfortunately only ctypes in Python 2.6 supports accessing errno: + # http://bugs.python.org/issue1798 and in order to solve + # the problem for previous versions we need to introduce + # code that is quite complex: + # http://stackoverflow.com/questions/661017/access-to-errno-from-python + # + # See #4310 for future resolution of this issue. + libc.inotify_rm_watch(fd, wd) + + + +def initializeModule(libc): + """ + Initialize the module, checking if the expected APIs exist and setting the + argtypes and restype for C{inotify_init}, C{inotify_add_watch}, and + C{inotify_rm_watch}. + """ + for function in ("inotify_add_watch", "inotify_init", "inotify_rm_watch"): + if getattr(libc, function, None) is None: + raise ImportError("libc6 2.4 or higher needed") + libc.inotify_init.argtypes = [] + libc.inotify_init.restype = ctypes.c_int + + libc.inotify_rm_watch.argtypes = [ + ctypes.c_int, ctypes.c_int] + libc.inotify_rm_watch.restype = ctypes.c_int + + libc.inotify_add_watch.argtypes = [ + ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32] + libc.inotify_add_watch.restype = ctypes.c_int + + + +name = ctypes.util.find_library('c') +if not name: + raise ImportError("Can't find C library.") +libc = ctypes.cdll.LoadLibrary(name) +initializeModule(libc) diff --git a/contrib/python/Twisted/py2/twisted/python/_oldstyle.py b/contrib/python/Twisted/py2/twisted/python/_oldstyle.py new file mode 100644 index 00000000000..db80f2c97dd --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_oldstyle.py @@ -0,0 +1,99 @@ +# -*- test-case-name: twisted.test.test_nooldstyle -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Utilities to assist in the "flag day" new-style object transition. +""" + +from __future__ import absolute_import, division + +import types +from functools import wraps + +from twisted.python.compat import _shouldEnableNewStyle, _PY3 + + + +def _replaceIf(condition, alternative): + """ + If C{condition}, replace this function with C{alternative}. + + @param condition: A L{bool} which says whether this should be replaced. + + @param alternative: An alternative function that will be swapped in instead + of the original, if C{condition} is truthy. + + @return: A decorator. + """ + def decorator(func): + + if condition is True: + call = alternative + elif condition is False: + call = func + else: + raise ValueError(("condition argument to _replaceIf requires a " + "bool, not {}").format(repr(condition))) + + @wraps(func) + def wrapped(*args, **kwargs): + return call(*args, **kwargs) + + return wrapped + + return decorator + + + +def passthru(arg): + """ + Return C{arg}. Do nothing. + + @param arg: The arg to return. + + @return: C{arg} + """ + return arg + + + +def _ensureOldClass(cls): + """ + Ensure that C{cls} is an old-style class. + + @param cls: The class to check. + + @return: The class, if it is an old-style class. + @raises: L{ValueError} if it is a new-style class. + """ + if not type(cls) is types.ClassType: + from twisted.python.reflect import fullyQualifiedName + + raise ValueError( + ("twisted.python._oldstyle._oldStyle is being used to decorate a " + "new-style class ({cls}). This should only be used to " + "decorate old-style classes.").format( + cls=fullyQualifiedName(cls))) + + return cls + + + +@_replaceIf(_PY3, passthru) +@_replaceIf(not _shouldEnableNewStyle(), _ensureOldClass) +def _oldStyle(cls): + """ + A decorator which conditionally converts old-style classes to new-style + classes. If it is Python 3, or if the C{TWISTED_NEWSTYLE} environment + variable has a falsey (C{no}, C{false}, C{False}, or C{0}) value in the + environment, this decorator is a no-op. + + @param cls: An old-style class to convert to new-style. + @type cls: L{types.ClassType} + + @return: A new-style version of C{cls}. + """ + _ensureOldClass(cls) + _bases = cls.__bases__ + (object,) + return type(cls.__name__, _bases, cls.__dict__) diff --git a/contrib/python/Twisted/py2/twisted/python/_pydoctor.py b/contrib/python/Twisted/py2/twisted/python/_pydoctor.py new file mode 100644 index 00000000000..4964b9ddc41 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_pydoctor.py @@ -0,0 +1,269 @@ +# -*- test-case-name: twisted.python.test.test_pydoctor -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for a few things specific to documenting Twisted using pydoctor. + +FIXME: https://github.com/twisted/pydoctor/issues/106 +This documentation does not link to pydoctor API as there is no public API yet. +""" + +import urllib2 + +from compiler import ast +from pydoctor import model, zopeinterface +from pydoctor.sphinx import SphinxInventory + + +class HeadRequest(urllib2.Request, object): + """ + A request for the HEAD HTTP method. + """ + + def get_method(self): + """ + Use the HEAD HTTP method. + """ + return 'HEAD' + + + +class TwistedSphinxInventory(SphinxInventory): + """ + Custom SphinxInventory to work around broken external references to + Sphinx. + + All exceptions should be reported upstream and a comment should be created + with a link to the upstream report. + """ + + def getLink(self, name): + """ + Resolve the full URL for a cross reference. + + @param name: Value of the cross reference. + @type name: L{str} + + @return: A full URL for the I{name} reference or L{None} if no link was + found. + @rtype: L{str} or L{None} + """ + result = super(TwistedSphinxInventory, self).getLink(name) + if result is not None: + # We already got a link. Look no further. + return result + + if name.startswith('zope.interface.'): + # This is a link from zope.interface. which is not advertised in + # the Sphinx inventory. + # See if the link is a known broken link which should be handled + # as an exceptional case. + # We get the base URL from IInterface which is assume that is + # always and already well defined in the Sphinx index. + baseURL, _ = self._links.get( + 'zope.interface.interfaces.IInterface', + (None, None)) + + if baseURL is None: + # Most probably the zope.interface inventory was + # not loaded. + return None + + if name == 'zope.interface.adapter.AdapterRegistry': + # FIXME: + # https://github.com/zopefoundation/zope.interface/issues/41 + relativeLink = 'adapter.html' + else: + # Not a known exception. + relativeLink = None + + if relativeLink is None: + return None + + return '%s/%s' % (baseURL, relativeLink) + + if name.startswith('win32api'): + # This is a link to pywin32 which does not provide inter-API doc + # link capabilities + baseURL = 'http://docs.activestate.com/activepython/2.7/pywin32' + + # For now only links to methods are supported. + relativeLink = '%s_meth.html' % (name.replace('.', '__'),) + + fullURL = '%s/%s' % (baseURL, relativeLink) + + # Check if URL exists. + response = self._getURLAsHEAD(fullURL) + if response: + if response.code == 200: + return fullURL + else: + # Bad URL resolution. + print("BAD URL resolution, code: ", response.code) + + return None + + + def _getURLAsHEAD(self, url): + """ + Get are HEAD response for URL. + + Here to help with testing and allow injecting another URL getter. + + @param url: Full URL to the page which is retrieved. + @type url: L{str} + + @return: The response for the HEAD method. + @rtype: urllib2 response or L{None} + """ + try: + return urllib2.urlopen(HeadRequest(url)) + except Exception as e: + print("Error opening {}: {}".format(url, e)) + return None + + + +def getDeprecated(self, decorators): + """ + With a list of decorators, and the object it is running on, set the + C{_deprecated_info} flag if any of the decorators are a Twisted deprecation + decorator. + """ + for a in decorators: + if isinstance(a, ast.CallFunc): + decorator = a.asList() + + # Getattr is used when the decorator is @foo.bar, not @bar + if isinstance(decorator[0], ast.Getattr): + getAttr = decorator[0].asList() + name = getAttr[0].name + fn = self.expandName(name) + "." + getAttr[1] + else: + fn = self.expandName(decorator[0].name) + + if fn == "twisted.python.deprecate.deprecated": + try: + self._deprecated_info = deprecatedToUsefulText( + self.name, decorator) + except AttributeError: + # It's a reference or something that we can't figure out + # from the AST. + pass + + + +class TwistedModuleVisitor(zopeinterface.ZopeInterfaceModuleVisitor): + + def visitClass(self, node): + """ + Called when a class is visited. + """ + super(TwistedModuleVisitor, self).visitClass(node) + + cls = self.builder.current.contents[node.name] + + getDeprecated(cls, list(cls.raw_decorators)) + + + def visitFunction(self, node): + """ + Called when a class is visited. + """ + super(TwistedModuleVisitor, self).visitFunction(node) + + func = self.builder.current.contents[node.name] + + if func.decorators: + getDeprecated(func, list(func.decorators)) + + + +def versionToUsefulObject(version): + """ + Change an AST C{Version()} to a real one. + """ + from incremental import Version + + return Version(*[x.value for x in version.asList()[1:] if x]) + + + +def deprecatedToUsefulText(name, deprecated): + """ + Change a C{@deprecated} to a display string. + """ + from twisted.python.deprecate import _getDeprecationWarningString + + version = versionToUsefulObject(deprecated[1]) + if deprecated[2]: + if isinstance(deprecated[2], ast.Keyword): + replacement = deprecated[2].asList()[1].value + else: + replacement = deprecated[2].value + else: + replacement = None + + return _getDeprecationWarningString(name, version, replacement=replacement) + "." + + + +class TwistedFunction(zopeinterface.ZopeInterfaceFunction): + + def docsources(self): + + if self.decorators: + getDeprecated(self, list(self.decorators)) + + for x in super(TwistedFunction, self).docsources(): + yield x + + + +class TwistedASTBuilder(zopeinterface.ZopeInterfaceASTBuilder): + # Vistor is not a typo... + ModuleVistor = TwistedModuleVisitor + + + +class TwistedSystem(zopeinterface.ZopeInterfaceSystem): + """ + A PyDoctor "system" used to generate the docs. + """ + defaultBuilder = TwistedASTBuilder + Function = TwistedFunction + + def __init__(self, options=None): + super(TwistedSystem, self).__init__(options=options) + # Use custom SphinxInventory so that we can resolve valid L{} markup + # for which the Sphinx inventory is not published or broken. + self.intersphinx = TwistedSphinxInventory( + logger=self.msg, project_name=self.projectname) + + + def privacyClass(self, documentable): + """ + Report the privacy level for an object. + + Hide all tests with the exception of L{twisted.test.proto_helpers}. + + param obj: Object for which the privacy is reported. + type obj: C{model.Documentable} + + rtype: C{model.PrivacyClass} member + """ + if documentable.fullName() == 'twisted.test': + # Match this package exactly, so that proto_helpers + # below is visible + return model.PrivacyClass.VISIBLE + + current = documentable + while current: + if current.fullName() == 'twisted.test.proto_helpers': + return model.PrivacyClass.VISIBLE + if isinstance(current, model.Package) and current.name == 'test': + return model.PrivacyClass.HIDDEN + current = current.parent + + return super(TwistedSystem, self).privacyClass(documentable) diff --git a/contrib/python/Twisted/py2/twisted/python/_release.py b/contrib/python/Twisted/py2/twisted/python/_release.py new file mode 100644 index 00000000000..224747ed151 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_release.py @@ -0,0 +1,576 @@ +# -*- test-case-name: twisted.python.test.test_release -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted's automated release system. + +This module is only for use within Twisted's release system. If you are anyone +else, do not use it. The interface and behaviour will change without notice. + +Only Linux is supported by this code. It should not be used by any tools +which must run on multiple platforms (eg the setup.py script). +""" + +import os +import sys + +from zope.interface import Interface, implementer + +from subprocess import check_output, STDOUT, CalledProcessError + +from twisted.python.compat import execfile +from twisted.python.filepath import FilePath +from twisted.python.monkey import MonkeyPatcher + +# Types of newsfragments. +NEWSFRAGMENT_TYPES = ["doc", "bugfix", "misc", "feature", "removal"] +intersphinxURLs = [ + u"https://docs.python.org/2/objects.inv", + u"https://docs.python.org/3/objects.inv", + u"https://pyopenssl.readthedocs.io/en/stable/objects.inv", + u"https://hyperlink.readthedocs.io/en/stable/objects.inv", + u"https://twisted.github.io/constantly/docs/objects.inv", + u"https://twisted.github.io/incremental/docs/objects.inv", + u"https://hyper-h2.readthedocs.io/en/stable/objects.inv", + u"https://priority.readthedocs.io/en/stable/objects.inv", + u"https://zopeinterface.readthedocs.io/en/latest/objects.inv", + u"https://automat.readthedocs.io/en/latest/objects.inv", +] + + +def runCommand(args, **kwargs): + """Execute a vector of arguments. + + This is a wrapper around L{subprocess.check_output}, so it takes + the same arguments as L{subprocess.Popen} with one difference: all + arguments after the vector must be keyword arguments. + + @param args: arguments passed to L{subprocess.check_output} + @param kwargs: keyword arguments passed to L{subprocess.check_output} + @return: command output + @rtype: L{bytes} + """ + kwargs['stderr'] = STDOUT + return check_output(args, **kwargs) + + + +class IVCSCommand(Interface): + """ + An interface for VCS commands. + """ + def ensureIsWorkingDirectory(path): + """ + Ensure that C{path} is a working directory of this VCS. + + @type path: L{twisted.python.filepath.FilePath} + @param path: The path to check. + """ + + + def isStatusClean(path): + """ + Return the Git status of the files in the specified path. + + @type path: L{twisted.python.filepath.FilePath} + @param path: The path to get the status from (can be a directory or a + file.) + """ + + + def remove(path): + """ + Remove the specified path from a the VCS. + + @type path: L{twisted.python.filepath.FilePath} + @param path: The path to remove from the repository. + """ + + + def exportTo(fromDir, exportDir): + """ + Export the content of the VCSrepository to the specified directory. + + @type fromDir: L{twisted.python.filepath.FilePath} + @param fromDir: The path to the VCS repository to export. + + @type exportDir: L{twisted.python.filepath.FilePath} + @param exportDir: The directory to export the content of the + repository to. This directory doesn't have to exist prior to + exporting the repository. + """ + + + +@implementer(IVCSCommand) +class GitCommand(object): + """ + Subset of Git commands to release Twisted from a Git repository. + """ + @staticmethod + def ensureIsWorkingDirectory(path): + """ + Ensure that C{path} is a Git working directory. + + @type path: L{twisted.python.filepath.FilePath} + @param path: The path to check. + """ + try: + runCommand(["git", "rev-parse"], cwd=path.path) + except (CalledProcessError, OSError): + raise NotWorkingDirectory( + "%s does not appear to be a Git repository." + % (path.path,)) + + + @staticmethod + def isStatusClean(path): + """ + Return the Git status of the files in the specified path. + + @type path: L{twisted.python.filepath.FilePath} + @param path: The path to get the status from (can be a directory or a + file.) + """ + status = runCommand( + ["git", "-C", path.path, "status", "--short"]).strip() + return status == b'' + + + @staticmethod + def remove(path): + """ + Remove the specified path from a Git repository. + + @type path: L{twisted.python.filepath.FilePath} + @param path: The path to remove from the repository. + """ + runCommand(["git", "-C", path.dirname(), "rm", path.path]) + + + @staticmethod + def exportTo(fromDir, exportDir): + """ + Export the content of a Git repository to the specified directory. + + @type fromDir: L{twisted.python.filepath.FilePath} + @param fromDir: The path to the Git repository to export. + + @type exportDir: L{twisted.python.filepath.FilePath} + @param exportDir: The directory to export the content of the + repository to. This directory doesn't have to exist prior to + exporting the repository. + """ + runCommand(["git", "-C", fromDir.path, + "checkout-index", "--all", "--force", + # prefix has to end up with a "/" so that files get copied + # to a directory whose name is the prefix. + "--prefix", exportDir.path + "/"]) + + + +def getRepositoryCommand(directory): + """ + Detect the VCS used in the specified directory and return a L{GitCommand} + if the directory is a Git repository. If the directory is not git, it + raises a L{NotWorkingDirectory} exception. + + @type directory: L{FilePath} + @param directory: The directory to detect the VCS used from. + + @rtype: L{GitCommand} + + @raise NotWorkingDirectory: if no supported VCS can be found from the + specified directory. + """ + try: + GitCommand.ensureIsWorkingDirectory(directory) + return GitCommand + except (NotWorkingDirectory, OSError): + # It's not Git, but that's okay, eat the error + pass + + raise NotWorkingDirectory("No supported VCS can be found in %s" % + (directory.path,)) + + + +class Project(object): + """ + A representation of a project that has a version. + + @ivar directory: A L{twisted.python.filepath.FilePath} pointing to the base + directory of a Twisted-style Python package. The package should contain + a C{_version.py} file and a C{newsfragments} directory that contains a + C{README} file. + """ + + def __init__(self, directory): + self.directory = directory + + + def __repr__(self): + return '%s(%r)' % ( + self.__class__.__name__, self.directory) + + + def getVersion(self): + """ + @return: A L{incremental.Version} specifying the version number of the + project based on live python modules. + """ + namespace = {} + directory = self.directory + while not namespace: + if directory.path == "/": + raise Exception("Not inside a Twisted project.") + elif not directory.basename() == "twisted": + directory = directory.parent() + else: + execfile(directory.child("_version.py").path, namespace) + return namespace["__version__"] + + + +def findTwistedProjects(baseDirectory): + """ + Find all Twisted-style projects beneath a base directory. + + @param baseDirectory: A L{twisted.python.filepath.FilePath} to look inside. + @return: A list of L{Project}. + """ + projects = [] + for filePath in baseDirectory.walk(): + if filePath.basename() == 'newsfragments': + projectDirectory = filePath.parent() + projects.append(Project(projectDirectory)) + return projects + + +def replaceInFile(filename, oldToNew): + """ + I replace the text `oldstr' with `newstr' in `filename' using science. + """ + os.rename(filename, filename + '.bak') + with open(filename + '.bak') as f: + d = f.read() + for k, v in oldToNew.items(): + d = d.replace(k, v) + with open(filename + '.new', 'w') as f: + f.write(d) + os.rename(filename + '.new', filename) + os.unlink(filename + '.bak') + + + +class NoDocumentsFound(Exception): + """ + Raised when no input documents are found. + """ + + + +class APIBuilder(object): + """ + Generate API documentation from source files using + U{pydoctor}. This requires + pydoctor to be installed and usable. + """ + def build(self, projectName, projectURL, sourceURL, packagePath, + outputPath): + """ + Call pydoctor's entry point with options which will generate HTML + documentation for the specified package's API. + + @type projectName: C{str} + @param projectName: The name of the package for which to generate + documentation. + + @type projectURL: C{str} + @param projectURL: The location (probably an HTTP URL) of the project + on the web. + + @type sourceURL: C{str} + @param sourceURL: The location (probably an HTTP URL) of the root of + the source browser for the project. + + @type packagePath: L{FilePath} + @param packagePath: The path to the top-level of the package named by + C{projectName}. + + @type outputPath: L{FilePath} + @param outputPath: An existing directory to which the generated API + documentation will be written. + """ + intersphinxes = [] + + for intersphinx in intersphinxURLs: + intersphinxes.append("--intersphinx") + intersphinxes.append(intersphinx) + + # Super awful monkeypatch that will selectively use our templates. + from pydoctor.templatewriter import util + originalTemplatefile = util.templatefile + + def templatefile(filename): + + if filename in ["summary.html", "index.html", "common.html"]: + twistedPythonDir = FilePath(__file__).parent() + templatesDir = twistedPythonDir.child("_pydoctortemplates") + return templatesDir.child(filename).path + else: + return originalTemplatefile(filename) + + monkeyPatch = MonkeyPatcher((util, "templatefile", templatefile)) + monkeyPatch.patch() + + from pydoctor.driver import main + + args = [u"--project-name", projectName, + u"--project-url", projectURL, + u"--system-class", u"twisted.python._pydoctor.TwistedSystem", + u"--project-base-dir", packagePath.parent().path, + u"--html-viewsource-base", sourceURL, + u"--add-package", packagePath.path, + u"--html-output", outputPath.path, + u"--html-write-function-pages", u"--quiet", u"--make-html", + ] + intersphinxes + args = [arg.encode("utf-8") for arg in args] + main(args) + + monkeyPatch.restore() + + + +class SphinxBuilder(object): + """ + Generate HTML documentation using Sphinx. + + Generates and runs a shell command that looks something like:: + + sphinx-build -b html -d [BUILDDIR]/doctrees + [DOCDIR]/source + [BUILDDIR]/html + + where DOCDIR is a directory containing another directory called "source" + which contains the Sphinx source files, and BUILDDIR is the directory in + which the Sphinx output will be created. + """ + + def main(self, args): + """ + Build the main documentation. + + @type args: list of str + @param args: The command line arguments to process. This must contain + one string argument: the path to the root of a Twisted checkout. + Additional arguments will be ignored for compatibility with legacy + build infrastructure. + """ + output = self.build(FilePath(args[0]).child("docs")) + if output: + sys.stdout.write(u"Unclean build:\n{}\n".format(output)) + raise sys.exit(1) + + + def build(self, docDir, buildDir=None, version=''): + """ + Build the documentation in C{docDir} with Sphinx. + + @param docDir: The directory of the documentation. This is a directory + which contains another directory called "source" which contains the + Sphinx "conf.py" file and sphinx source documents. + @type docDir: L{twisted.python.filepath.FilePath} + + @param buildDir: The directory to build the documentation in. By + default this will be a child directory of {docDir} named "build". + @type buildDir: L{twisted.python.filepath.FilePath} + + @param version: The version of Twisted to set in the docs. + @type version: C{str} + + @return: the output produced by running the command + @rtype: L{str} + """ + if buildDir is None: + buildDir = docDir.parent().child('doc') + + doctreeDir = buildDir.child('doctrees') + + output = runCommand(['sphinx-build', '-q', '-b', 'html', + '-d', doctreeDir.path, docDir.path, + buildDir.path]).decode("utf-8") + + # Delete the doctrees, as we don't want them after the docs are built + doctreeDir.remove() + + for path in docDir.walk(): + if path.basename() == "man": + segments = path.segmentsFrom(docDir) + dest = buildDir + while segments: + dest = dest.child(segments.pop(0)) + if not dest.parent().isdir(): + dest.parent().makedirs() + path.copyTo(dest) + return output + + + +def filePathDelta(origin, destination): + """ + Return a list of strings that represent C{destination} as a path relative + to C{origin}. + + It is assumed that both paths represent directories, not files. That is to + say, the delta of L{twisted.python.filepath.FilePath} /foo/bar to + L{twisted.python.filepath.FilePath} /foo/baz will be C{../baz}, + not C{baz}. + + @type origin: L{twisted.python.filepath.FilePath} + @param origin: The origin of the relative path. + + @type destination: L{twisted.python.filepath.FilePath} + @param destination: The destination of the relative path. + """ + commonItems = 0 + path1 = origin.path.split(os.sep) + path2 = destination.path.split(os.sep) + for elem1, elem2 in zip(path1, path2): + if elem1 == elem2: + commonItems += 1 + else: + break + path = [".."] * (len(path1) - commonItems) + return path + path2[commonItems:] + + + +class NotWorkingDirectory(Exception): + """ + Raised when a directory does not appear to be a repository directory of a + supported VCS. + """ + + + +class BuildAPIDocsScript(object): + """ + A thing for building API documentation. See L{main}. + """ + + def buildAPIDocs(self, projectRoot, output): + """ + Build the API documentation of Twisted, with our project policy. + + @param projectRoot: A L{FilePath} representing the root of the Twisted + checkout. + @param output: A L{FilePath} pointing to the desired output directory. + """ + version = Project( + projectRoot.child("twisted")).getVersion() + versionString = version.base() + sourceURL = ("https://github.com/twisted/twisted/tree/" + "twisted-%s" % (versionString,) + "/src") + apiBuilder = APIBuilder() + apiBuilder.build( + "Twisted", + "http://twistedmatrix.com/", + sourceURL, + projectRoot.child("twisted"), + output) + + + def main(self, args): + """ + Build API documentation. + + @type args: list of str + @param args: The command line arguments to process. This must contain + two strings: the path to the root of the Twisted checkout, and a + path to an output directory. + """ + if len(args) != 2: + sys.exit("Must specify two arguments: " + "Twisted checkout and destination path") + self.buildAPIDocs(FilePath(args[0]), FilePath(args[1])) + + + +class CheckNewsfragmentScript(object): + """ + A thing for checking whether a checkout has a newsfragment. + """ + def __init__(self, _print): + self._print = _print + + + def main(self, args): + """ + Run the script. + + @type args: L{list} of L{str} + @param args: The command line arguments to process. This must contain + one string: the path to the root of the Twisted checkout. + """ + if len(args) != 1: + sys.exit("Must specify one argument: the Twisted checkout") + + encoding = sys.stdout.encoding or 'ascii' + location = os.path.abspath(args[0]) + + branch = runCommand([b"git", b"rev-parse", b"--abbrev-ref", "HEAD"], + cwd=location).decode(encoding).strip() + + # diff-filter=d to exclude deleted newsfiles (which will happen on the + # release branch) + r = runCommand( + [ + b"git", + b"diff", + b"--name-only", + b"origin/trunk...", + b"--diff-filter=d" + ], + cwd=location + ).decode(encoding).strip() + + if not r: + self._print( + "On trunk or no diffs from trunk; no need to look at this.") + sys.exit(0) + + files = r.strip().split(os.linesep) + + self._print("Looking at these files:") + for change in files: + self._print(change) + self._print("----") + + if len(files) == 1: + if files[0] == os.sep.join(["docs", "fun", "Twisted.Quotes"]): + self._print("Quotes change only; no newsfragment needed.") + sys.exit(0) + + newsfragments = [] + + for change in files: + if os.sep + "newsfragments" + os.sep in change: + if "." in change and change.rsplit(".", 1)[1] in NEWSFRAGMENT_TYPES: + newsfragments.append(change) + + if branch.startswith("release-"): + if newsfragments: + self._print("No newsfragments should be on the release branch.") + sys.exit(1) + else: + self._print("Release branch with no newsfragments, all good.") + sys.exit(0) + + for change in newsfragments: + self._print("Found " + change) + sys.exit(0) + + self._print("No newsfragment found. Have you committed it?") + sys.exit(1) diff --git a/contrib/python/Twisted/py2/twisted/python/_sendmsg.c b/contrib/python/Twisted/py2/twisted/python/_sendmsg.c new file mode 100644 index 00000000000..e84c08b51d4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_sendmsg.c @@ -0,0 +1,519 @@ +/* + * Copyright (c) Twisted Matrix Laboratories. + * See LICENSE for details. + */ + +#define PY_SSIZE_T_CLEAN 1 +#include + +#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) +/* This may cause some warnings, but if you want to get rid of them, upgrade + * your Python version. */ +typedef int Py_ssize_t; +#endif + +#include +#include +#include + +#include + +#ifdef BSD +#include +#endif + +/* + * As per + * : + * + * "To forestall portability problems, it is recommended that applications + * not use values larger than (2**31)-1 for the socklen_t type." + */ + +#define SOCKLEN_MAX 0x7FFFFFFF + +PyObject *sendmsg_socket_error; + +static PyObject *sendmsg_sendmsg(PyObject *self, PyObject *args, PyObject *keywds); +static PyObject *sendmsg_recvmsg(PyObject *self, PyObject *args, PyObject *keywds); +static PyObject *sendmsg_getsockfam(PyObject *self, PyObject *args, PyObject *keywds); + +static char sendmsg_doc[] = "\ +Bindings for sendmsg(2), recvmsg(2), and a minimal helper for inspecting\n\ +address family of a socket.\n\ +"; + +static char sendmsg_sendmsg_doc[] = "\ +Wrap the C sendmsg(2) function for sending \"messages\" on a socket.\n\ +\n\ +@param fd: The file descriptor of the socket over which to send a message.\n\ +@type fd: C{int}\n\ +\n\ +@param data: Bytes to write to the socket.\n\ +@type data: C{str}\n\ +\n\ +@param flags: Flags to affect how the message is sent. See the C{MSG_}\n\ + constants in the sendmsg(2) manual page. By default no flags are set.\n\ +@type flags: C{int}\n\ +\n\ +@param ancillary: Extra data to send over the socket outside of the normal\n\ + datagram or stream mechanism. By default no ancillary data is sent.\n\ +@type ancillary: C{list} of C{tuple} of C{int}, C{int}, and C{str}.\n\ +\n\ +@raise OverflowError: Raised if too much ancillary data is given.\n\ +@raise socket.error: Raised if the underlying syscall indicates an error.\n\ +\n\ +@return: The return value of the underlying syscall, if it succeeds.\n\ +"; + +static char sendmsg_recvmsg_doc[] = "\ +Wrap the C recvmsg(2) function for receiving \"messages\" on a socket.\n\ +\n\ +@param fd: The file descriptor of the socket over which to receive a message.\n\ +@type fd: C{int}\n\ +\n\ +@param flags: Flags to affect how the message is sent. See the C{MSG_}\n\ + constants in the sendmsg(2) manual page. By default no flags are set.\n\ +@type flags: C{int}\n\ +\n\ +@param maxsize: The maximum number of bytes to receive from the socket\n\ + using the datagram or stream mechanism. The default maximum is 8192.\n\ +@type maxsize: C{int}\n\ +\n\ +@param cmsg_size: The maximum number of bytes to receive from the socket\n\ + outside of the normal datagram or stream mechanism. The default maximum is 4096.\n\ +\n\ +@raise OverflowError: Raised if too much ancillary data is given.\n\ +@raise socket.error: Raised if the underlying syscall indicates an error.\n\ +\n\ +@return: A C{tuple} of three elements: the bytes received using the\n\ + datagram/stream mechanism, flags as an C{int} describing the data\n\ + received, and a C{list} of C{tuples} giving ancillary received data.\n\ +"; + +static char sendmsg_getsockfam_doc[] = "\ +Retrieve the address family of a given socket.\n\ +\n\ +@param fd: The file descriptor of the socket the address family of which\n\ + to retrieve.\n\ +@type fd: C{int}\n\ +\n\ +@raise socket.error: Raised if the underlying getsockname call indicates\n\ + an error.\n\ +\n\ +@return: A C{int} representing the address family of the socket. For\n\ + example, L{socket.AF_INET}, L{socket.AF_INET6}, or L{socket.AF_UNIX}.\n\ +"; + +static PyMethodDef sendmsg_methods[] = { + {"send1msg", (PyCFunction) sendmsg_sendmsg, METH_VARARGS | METH_KEYWORDS, + sendmsg_sendmsg_doc}, + {"recv1msg", (PyCFunction) sendmsg_recvmsg, METH_VARARGS | METH_KEYWORDS, + sendmsg_recvmsg_doc}, + {"getsockfam", (PyCFunction) sendmsg_getsockfam, + METH_VARARGS | METH_KEYWORDS, sendmsg_getsockfam_doc}, + {NULL, NULL, 0, NULL} +}; + + +PyMODINIT_FUNC init_sendmsg(void) { + PyObject *module; + + sendmsg_socket_error = NULL; /* Make sure that this has a known value + before doing anything that might exit. */ + + module = Py_InitModule3("_sendmsg", sendmsg_methods, sendmsg_doc); + + if (!module) { + return; + } + + /* + The following is the only value mentioned by POSIX: + http://www.opengroup.org/onlinepubs/9699919799/basedefs/sys_socket.h.html + */ + + if (-1 == PyModule_AddIntConstant(module, "SCM_RIGHTS", SCM_RIGHTS)) { + return; + } + + + /* BSD, Darwin, Hurd */ +#if defined(SCM_CREDS) + if (-1 == PyModule_AddIntConstant(module, "SCM_CREDS", SCM_CREDS)) { + return; + } +#endif + + /* Linux */ +#if defined(SCM_CREDENTIALS) + if (-1 == PyModule_AddIntConstant(module, "SCM_CREDENTIALS", SCM_CREDENTIALS)) { + return; + } +#endif + + /* Apparently everywhere, but not standardized. */ +#if defined(SCM_TIMESTAMP) + if (-1 == PyModule_AddIntConstant(module, "SCM_TIMESTAMP", SCM_TIMESTAMP)) { + return; + } +#endif + + module = PyImport_ImportModule("socket"); + if (!module) { + return; + } + + sendmsg_socket_error = PyObject_GetAttrString(module, "error"); + if (!sendmsg_socket_error) { + return; + } +} + +static PyObject *sendmsg_sendmsg(PyObject *self, PyObject *args, PyObject *keywds) { + + int fd; + int flags = 0; + Py_ssize_t sendmsg_result, iovec_length; + struct msghdr message_header; + struct iovec iov[1]; + PyObject *ancillary = NULL; + PyObject *iterator = NULL; + PyObject *item = NULL; + PyObject *result_object = NULL; + + static char *kwlist[] = {"fd", "data", "flags", "ancillary", NULL}; + + if (!PyArg_ParseTupleAndKeywords( + args, keywds, "it#|iO:sendmsg", kwlist, + &fd, + &iov[0].iov_base, + &iovec_length, + &flags, + &ancillary)) { + return NULL; + } + + iov[0].iov_len = iovec_length; + + message_header.msg_name = NULL; + message_header.msg_namelen = 0; + + message_header.msg_iov = iov; + message_header.msg_iovlen = 1; + + message_header.msg_control = NULL; + message_header.msg_controllen = 0; + + message_header.msg_flags = 0; + + if (ancillary) { + + if (!PyList_Check(ancillary)) { + PyErr_Format(PyExc_TypeError, + "send1msg argument 3 expected list, got %s", + ancillary->ob_type->tp_name); + goto finished; + } + + iterator = PyObject_GetIter(ancillary); + + if (iterator == NULL) { + goto finished; + } + + size_t all_data_len = 0; + + /* First we need to know how big the buffer needs to be in order to + have enough space for all of the messages. */ + while ( (item = PyIter_Next(iterator)) ) { + int type, level; + Py_ssize_t data_len; + size_t prev_all_data_len; + char *data; + + if (!PyTuple_Check(item)) { + PyErr_Format(PyExc_TypeError, + "send1msg argument 3 expected list of tuple, " + "got list containing %s", + item->ob_type->tp_name); + goto finished; + } + + if (!PyArg_ParseTuple( + item, "iit#:sendmsg ancillary data (level, type, data)", + &level, &type, &data, &data_len)) { + goto finished; + } + + prev_all_data_len = all_data_len; + all_data_len += CMSG_SPACE(data_len); + + Py_DECREF(item); + item = NULL; + + if (all_data_len < prev_all_data_len) { + PyErr_Format(PyExc_OverflowError, + "Too much msg_control to fit in a size_t: %zu", + prev_all_data_len); + goto finished; + } + } + + Py_DECREF(iterator); + iterator = NULL; + + /* Allocate the buffer for all of the ancillary elements, if we have + * any. */ + if (all_data_len) { + if (all_data_len > SOCKLEN_MAX) { + PyErr_Format(PyExc_OverflowError, + "Too much msg_control to fit in a socklen_t: %zu", + all_data_len); + goto finished; + } + message_header.msg_control = PyMem_Malloc(all_data_len); + if (!message_header.msg_control) { + PyErr_NoMemory(); + goto finished; + } + /* From Python 3.5.2 socketmodule.c:3891: + Need to zero out the buffer as a workaround for glibc's + CMSG_NXTHDR() implementation. After getting the pointer to + the next header, it checks its (uninitialized) cmsg_len + member to see if the "message" fits in the buffer, and + returns NULL if it doesn't. Zero-filling the buffer + ensures that this doesn't happen. */ + memset(message_header.msg_control, 0, all_data_len); + } else { + message_header.msg_control = NULL; + } + message_header.msg_controllen = (socklen_t) all_data_len; + + iterator = PyObject_GetIter(ancillary); /* again */ + + if (!iterator) { + goto finished; + } + + /* Unpack the tuples into the control message. */ + struct cmsghdr *control_message = CMSG_FIRSTHDR(&message_header); + while ( (item = PyIter_Next(iterator)) && control_message!=NULL ) { + int type, level; + Py_ssize_t data_len; + size_t data_size; + unsigned char *data, *cmsg_data; + + /* We explicitly allocated enough space for all ancillary data + above; if there isn't enough room, all bets are off. */ + assert(control_message); + + if (!PyArg_ParseTuple(item, + "iit#:sendmsg ancillary data (level, type, data)", + &level, + &type, + &data, + &data_len)) { + goto finished; + } + + control_message->cmsg_level = level; + control_message->cmsg_type = type; + data_size = CMSG_LEN(data_len); + + if (data_size > SOCKLEN_MAX) { + PyErr_Format(PyExc_OverflowError, + "CMSG_LEN(%zd) > SOCKLEN_MAX", data_len); + goto finished; + } + + control_message->cmsg_len = (socklen_t) data_size; + + cmsg_data = CMSG_DATA(control_message); + memcpy(cmsg_data, data, data_len); + + Py_DECREF(item); + item = NULL; + + control_message = CMSG_NXTHDR(&message_header, control_message); + } + Py_DECREF(iterator); + iterator = NULL; + + if (PyErr_Occurred()) { + goto finished; + } + } + + sendmsg_result = sendmsg(fd, &message_header, flags); + + if (sendmsg_result < 0) { + PyErr_SetFromErrno(sendmsg_socket_error); + goto finished; + } + + result_object = Py_BuildValue("n", sendmsg_result); + + finished: + + if (item) { + Py_DECREF(item); + item = NULL; + } + if (iterator) { + Py_DECREF(iterator); + iterator = NULL; + } + if (message_header.msg_control) { + PyMem_Free(message_header.msg_control); + message_header.msg_control = NULL; + } + return result_object; +} + +static PyObject *sendmsg_recvmsg(PyObject *self, PyObject *args, PyObject *keywds) { + int fd = -1; + int flags = 0; + int maxsize = 8192; + int cmsg_size = 4096; + size_t cmsg_space; + size_t cmsg_overhead; + Py_ssize_t recvmsg_result; + + struct msghdr message_header; + struct cmsghdr *control_message; + struct iovec iov[1]; + char *cmsgbuf; + PyObject *ancillary; + PyObject *final_result = NULL; + + static char *kwlist[] = {"fd", "flags", "maxsize", "cmsg_size", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, keywds, "i|iii:recvmsg", kwlist, + &fd, &flags, &maxsize, &cmsg_size)) { + return NULL; + } + + cmsg_space = CMSG_SPACE(cmsg_size); + + /* overflow check */ + if (cmsg_space > SOCKLEN_MAX) { + PyErr_Format(PyExc_OverflowError, + "CMSG_SPACE(cmsg_size) greater than SOCKLEN_MAX: %d", + cmsg_size); + return NULL; + } + + message_header.msg_name = NULL; + message_header.msg_namelen = 0; + + iov[0].iov_len = maxsize; + iov[0].iov_base = PyMem_Malloc(maxsize); + + if (!iov[0].iov_base) { + PyErr_NoMemory(); + return NULL; + } + + message_header.msg_iov = iov; + message_header.msg_iovlen = 1; + + cmsgbuf = PyMem_Malloc(cmsg_space); + + if (!cmsgbuf) { + PyMem_Free(iov[0].iov_base); + PyErr_NoMemory(); + return NULL; + } + + memset(cmsgbuf, 0, cmsg_space); + message_header.msg_control = cmsgbuf; + /* see above for overflow check */ + message_header.msg_controllen = (socklen_t) cmsg_space; + + recvmsg_result = recvmsg(fd, &message_header, flags); + if (recvmsg_result < 0) { + PyErr_SetFromErrno(sendmsg_socket_error); + goto finished; + } + + ancillary = PyList_New(0); + if (!ancillary) { + goto finished; + } + + for (control_message = CMSG_FIRSTHDR(&message_header); + control_message; + control_message = CMSG_NXTHDR(&message_header, + control_message)) { + PyObject *entry; + + /* Some platforms apparently always fill out the ancillary data + structure with a single bogus value if none is provided; ignore it, + if that is the case. */ + + if ((!(control_message->cmsg_level)) && + (!(control_message->cmsg_type))) { + continue; + } + + /* + * Figure out how much of the cmsg size is cmsg structure overhead - in + * other words, how much is not part of the application data. This lets + * us compute the right application data size below. There should + * really be a CMSG_ macro for this. + */ + cmsg_overhead = (char*)CMSG_DATA(control_message) - (char*)control_message; + + entry = Py_BuildValue( + "(iis#)", + control_message->cmsg_level, + control_message->cmsg_type, + CMSG_DATA(control_message), + (Py_ssize_t) (control_message->cmsg_len - cmsg_overhead)); + + if (!entry) { + Py_DECREF(ancillary); + goto finished; + } + + if (PyList_Append(ancillary, entry) < 0) { + Py_DECREF(ancillary); + Py_DECREF(entry); + goto finished; + } else { + Py_DECREF(entry); + } + } + + final_result = Py_BuildValue( + "s#iO", + iov[0].iov_base, + recvmsg_result, + message_header.msg_flags, + ancillary); + + Py_DECREF(ancillary); + + finished: + PyMem_Free(iov[0].iov_base); + PyMem_Free(cmsgbuf); + return final_result; +} + +static PyObject *sendmsg_getsockfam(PyObject *self, PyObject *args, + PyObject *keywds) { + int fd; + struct sockaddr sa; + static char *kwlist[] = {"fd", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, keywds, "i", kwlist, &fd)) { + return NULL; + } + socklen_t sz = sizeof(sa); + if (getsockname(fd, &sa, &sz)) { + PyErr_SetFromErrno(sendmsg_socket_error); + return NULL; + } + return Py_BuildValue("i", sa.sa_family); +} diff --git a/contrib/python/Twisted/py2/twisted/python/_setup.py b/contrib/python/Twisted/py2/twisted/python/_setup.py new file mode 100644 index 00000000000..dc2a280c173 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_setup.py @@ -0,0 +1,452 @@ +# -*- test-case-name: twisted.python.test.test_setup -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# pylint: disable=I0011,C0103,C9302,W9401,W9402 + +""" +Setuptools convenience functionality. + +This file must not import anything from Twisted, as it is loaded by C{exec} in +C{setup.py}. If you need compatibility functions for this code, duplicate them +here. + +@var _EXTRA_OPTIONS: These are the actual package names and versions that will + be used by C{extras_require}. This is not passed to setup directly so that + combinations of the packages can be created without the need to copy + package names multiple times. + +@var _EXTRAS_REQUIRE: C{extras_require} is a dictionary of items that can be + passed to setup.py to install optional dependencies. For example, to + install the optional dev dependencies one would type:: + + pip install -e ".[dev]" + + This has been supported by setuptools since 0.5a4. + +@var _PLATFORM_INDEPENDENT: A list of all optional cross-platform dependencies, + as setuptools version specifiers, used to populate L{_EXTRAS_REQUIRE}. + +@var _EXTENSIONS: The list of L{ConditionalExtension} used by the setup + process. + +@var notPortedModules: Modules that are not yet ported to Python 3. +""" + +import io +import os +import platform +import re +import sys + +from distutils.command import build_ext +from distutils.errors import CompileError +from setuptools import Extension, find_packages +from setuptools.command.build_py import build_py + +# Do not replace this with t.p.compat imports, this file must not import +# from Twisted. See the docstring. +if sys.version_info < (3, 0): + _PY3 = False +else: + _PY3 = True + +STATIC_PACKAGE_METADATA = dict( + name="Twisted", + description="An asynchronous networking framework written in Python", + author="Twisted Matrix Laboratories", + author_email="twisted-python@twistedmatrix.com", + maintainer="Glyph Lefkowitz", + maintainer_email="glyph@twistedmatrix.com", + url="https://twistedmatrix.com/", + project_urls={ + 'Documentation': 'https://twistedmatrix.com/documents/current/', + 'Source': 'https://github.com/twisted/twisted', + 'Issues': 'https://twistedmatrix.com/trac/report', + }, + license="MIT", + classifiers=[ + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*', +) + + +_dev = [ + 'pyflakes >= 1.0.0', + 'twisted-dev-tools >= 0.0.2', + 'python-subunit', + 'sphinx >= 1.3.1', + 'towncrier >= 17.4.0' +] + +if not _PY3: + # These modules do not yet work on Python 3. + _dev += [ + 'twistedchecker >= 0.4.0', + 'pydoctor >= 16.2.0', + ] + +_EXTRA_OPTIONS = dict( + dev=_dev, + tls=[ + 'pyopenssl >= 16.0.0', + # service_identity 18.1.0 added support for validating IP addresses in + # certificate subjectAltNames + 'service_identity >= 18.1.0', + # idna 2.3 introduced some changes that break a few things. Avoid it. + # The problems were fixed in 2.4. + 'idna >= 0.6, != 2.3', + ], + conch=[ + 'pyasn1', + 'cryptography >= 2.5', + 'appdirs >= 1.4.0', + 'bcrypt >= 3.0.0', + ], + soap=['soappy'], + serial=['pyserial >= 3.0', + 'pywin32 != 226; platform_system == "Windows"'], + macos=['pyobjc-core', + 'pyobjc-framework-CFNetwork', + 'pyobjc-framework-Cocoa'], + windows=['pywin32 != 226'], + http2=['h2 >= 3.0, < 4.0', + 'priority >= 1.1.0, < 2.0'], +) + +_PLATFORM_INDEPENDENT = ( + _EXTRA_OPTIONS['tls'] + + _EXTRA_OPTIONS['conch'] + + _EXTRA_OPTIONS['soap'] + + _EXTRA_OPTIONS['serial'] + + _EXTRA_OPTIONS['http2'] +) + +_EXTRAS_REQUIRE = { + 'dev': _EXTRA_OPTIONS['dev'], + 'tls': _EXTRA_OPTIONS['tls'], + 'conch': _EXTRA_OPTIONS['conch'], + 'soap': _EXTRA_OPTIONS['soap'], + 'serial': _EXTRA_OPTIONS['serial'], + 'http2': _EXTRA_OPTIONS['http2'], + 'all_non_platform': _PLATFORM_INDEPENDENT, + 'macos_platform': ( + _EXTRA_OPTIONS['macos'] + _PLATFORM_INDEPENDENT + ), + 'windows_platform': ( + _EXTRA_OPTIONS['windows'] + _PLATFORM_INDEPENDENT + ), +} +_EXTRAS_REQUIRE['osx_platform'] = _EXTRAS_REQUIRE['macos_platform'] + +# Scripts provided by Twisted on Python 2 and 3. +_CONSOLE_SCRIPTS = [ + "ckeygen = twisted.conch.scripts.ckeygen:run", + "cftp = twisted.conch.scripts.cftp:run", + "conch = twisted.conch.scripts.conch:run", + "mailmail = twisted.mail.scripts.mailmail:run", + "pyhtmlizer = twisted.scripts.htmlizer:run", + "tkconch = twisted.conch.scripts.tkconch:run", + "trial = twisted.scripts.trial:run", + "twist = twisted.application.twist._twist:Twist.main", + "twistd = twisted.scripts.twistd:run", + ] + + + +class ConditionalExtension(Extension, object): + """ + An extension module that will only be compiled if certain conditions are + met. + + @param condition: A callable of one argument which returns True or False to + indicate whether the extension should be built. The argument is an + instance of L{build_ext_twisted}, which has useful methods for checking + things about the platform. + """ + def __init__(self, *args, **kwargs): + self.condition = kwargs.pop("condition", lambda builder: True) + Extension.__init__(self, *args, **kwargs) + + + +# The C extensions used for Twisted. +_EXTENSIONS = [ + ConditionalExtension( + "twisted.test.raiser", + sources=["src/twisted/test/raiser.c"], + condition=lambda _: _isCPython), + + ConditionalExtension( + "twisted.internet.iocpreactor.iocpsupport", + sources=[ + "src/twisted/internet/iocpreactor/iocpsupport/iocpsupport.c", + "src/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c", + ], + libraries=["ws2_32"], + condition=lambda _: _isCPython and sys.platform == "win32"), + + ConditionalExtension( + "twisted.python._sendmsg", + sources=["src/twisted/python/_sendmsg.c"], + condition=lambda _: not _PY3 and sys.platform != "win32"), + ] + + + +def _longDescriptionArgsFromReadme(readme): + """ + Generate a PyPI long description from the readme. + + @param readme: Path to the readme reStructuredText file. + @type readme: C{str} + + @return: Keyword arguments to be passed to C{setuptools.setup()}. + @rtype: C{str} + """ + with io.open(readme, encoding='utf-8') as f: + readmeRst = f.read() + + # Munge links of the form `NEWS `_ to point at the appropriate + # location on GitHub so that they function when the long description is + # displayed on PyPI. + longDesc = re.sub( + r'`([^`]+)\s+<(?!https?://)([^>]+)>`_', + r'`\1 `_', + readmeRst, + flags=re.I, + ) + + return { + 'long_description': longDesc, + 'long_description_content_type': 'text/x-rst', + } + + + +def getSetupArgs(extensions=_EXTENSIONS, readme='README.rst'): + """ + Generate arguments for C{setuptools.setup()} + + @param extensions: C extension modules to maybe build. This argument is to + be used for testing. + @type extensions: C{list} of C{ConditionalExtension} + + @param readme: Path to the readme reStructuredText file. This argument is + to be used for testing. + @type readme: C{str} + + @return: The keyword arguments to be used by the setup method. + @rtype: L{dict} + """ + arguments = STATIC_PACKAGE_METADATA.copy() + if readme: + arguments.update(_longDescriptionArgsFromReadme(readme)) + + # This is a workaround for distutils behavior; ext_modules isn't + # actually used by our custom builder. distutils deep-down checks + # to see if there are any ext_modules defined before invoking + # the build_ext command. We need to trigger build_ext regardless + # because it is the thing that does the conditional checks to see + # if it should build any extensions. The reason we have to delay + # the conditional checks until then is that the compiler objects + # are not yet set up when this code is executed. + arguments["ext_modules"] = extensions + # Use custome class to build the extensions. + class my_build_ext(build_ext_twisted): + conditionalExtensions = extensions + command_classes = { + 'build_ext': my_build_ext, + } + + if sys.version_info[0] >= 3: + command_classes['build_py'] = BuildPy3 + + requirements = [ + "zope.interface >= 4.4.2", + "constantly >= 15.1", + "incremental >= 16.10.1", + "Automat >= 0.3.0", + "hyperlink >= 17.1.1", + # PyHamcrest 1.10.0 is Python 3 only, but lacks package metadata that + # says so. This condition can be dropped when Twisted drops support for + # Python 2.7. + "PyHamcrest >= 1.9.0, != 1.10.0", + "attrs >= 19.2.0", + ] + + arguments.update(dict( + packages=find_packages("src"), + use_incremental=True, + setup_requires=["incremental >= 16.10.1"], + install_requires=requirements, + entry_points={ + 'console_scripts': _CONSOLE_SCRIPTS + }, + cmdclass=command_classes, + include_package_data=True, + exclude_package_data={ + "": ["*.c", "*.h", "*.pxi", "*.pyx", "build.bat"], + }, + zip_safe=False, + extras_require=_EXTRAS_REQUIRE, + package_dir={"": "src"}, + )) + + return arguments + + + +class BuildPy3(build_py, object): + """ + A version of build_py that doesn't install the modules that aren't yet + ported to Python 3. + """ + def find_package_modules(self, package, package_dir): + modules = [ + module for module + in build_py.find_package_modules(self, package, package_dir) + if ".".join([module[0], module[1]]) not in notPortedModules] + return modules + + + +## Helpers and distutil tweaks + + +class build_ext_twisted(build_ext.build_ext, object): + """ + Allow subclasses to easily detect and customize Extensions to + build at install-time. + """ + + def prepare_extensions(self): + """ + Prepare the C{self.extensions} attribute (used by + L{build_ext.build_ext}) by checking which extensions in + I{conditionalExtensions} should be built. In addition, if we are + building on NT, define the WIN32 macro to 1. + """ + # always define WIN32 under Windows + if os.name == 'nt': + self.define_macros = [("WIN32", 1)] + else: + self.define_macros = [] + + # On Solaris 10, we need to define the _XOPEN_SOURCE and + # _XOPEN_SOURCE_EXTENDED macros to build in order to gain access to + # the msg_control, msg_controllen, and msg_flags members in + # sendmsg.c. (according to + # https://stackoverflow.com/questions/1034587). See the documentation + # of X/Open CAE in the standards(5) man page of Solaris. + if sys.platform.startswith('sunos'): + self.define_macros.append(('_XOPEN_SOURCE', 1)) + self.define_macros.append(('_XOPEN_SOURCE_EXTENDED', 1)) + + self.extensions = [ + x for x in self.conditionalExtensions if x.condition(self) + ] + + for ext in self.extensions: + ext.define_macros.extend(self.define_macros) + + + def build_extensions(self): + """ + Check to see which extension modules to build and then build them. + """ + self.prepare_extensions() + build_ext.build_ext.build_extensions(self) + + + def _remove_conftest(self): + for filename in ("conftest.c", "conftest.o", "conftest.obj"): + try: + os.unlink(filename) + except EnvironmentError: + pass + + + def _compile_helper(self, content): + conftest = open("conftest.c", "w") + try: + with conftest: + conftest.write(content) + + try: + self.compiler.compile(["conftest.c"], output_dir='') + except CompileError: + return False + return True + finally: + self._remove_conftest() + + + def _check_header(self, header_name): + """ + Check if the given header can be included by trying to compile a file + that contains only an #include line. + """ + self.compiler.announce("checking for {} ...".format(header_name), 0) + return self._compile_helper("#include <{}>\n".format(header_name)) + + + +def _checkCPython(sys=sys, platform=platform): + """ + Checks if this implementation is CPython. + + This uses C{platform.python_implementation}. + + This takes C{sys} and C{platform} kwargs that by default use the real + modules. You shouldn't care about these -- they are for testing purposes + only. + + @return: C{False} if the implementation is definitely not CPython, C{True} + otherwise. + """ + return platform.python_implementation() == "CPython" + + +_isCPython = _checkCPython() + +notPortedModules = [ + "twisted.mail.alias", + "twisted.mail.bounce", + "twisted.mail.mail", + "twisted.mail.maildir", + "twisted.mail.pb", + "twisted.mail.relaymanager", + "twisted.mail.scripts.__init__", + "twisted.mail.tap", + "twisted.mail.test.test_bounce", + "twisted.mail.test.test_mail", + "twisted.mail.test.test_options", + "twisted.mail.test.test_scripts", + "twisted.news.__init__", + "twisted.news.database", + "twisted.news.news", + "twisted.news.nntp", + "twisted.news.tap", + "twisted.news.test.__init__", + "twisted.news.test.test_database", + "twisted.news.test.test_news", + "twisted.news.test.test_nntp", + "twisted.plugins.twisted_mail", + "twisted.plugins.twisted_news", + "twisted.protocols.shoutcast", + "twisted.python._pydoctor", + "twisted.python.finalize", + "twisted.python.hook", + "twisted.python.test.cmodulepullpipe", + "twisted.python.test.test_pydoctor", + "twisted.python.test.test_win32", + "twisted.test.test_hook", + "twisted.web.soap", + "twisted.web.test.test_soap", +] diff --git a/contrib/python/Twisted/py2/twisted/python/_shellcomp.py b/contrib/python/Twisted/py2/twisted/python/_shellcomp.py new file mode 100644 index 00000000000..f35e8ebfe49 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_shellcomp.py @@ -0,0 +1,677 @@ +# -*- test-case-name: twisted.python.test.test_shellcomp -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +No public APIs are provided by this module. Internal use only. + +This module implements dynamic tab-completion for any command that uses +twisted.python.usage. Currently, only zsh is supported. Bash support may +be added in the future. + +Maintainer: Eric P. Mangold - twisted AT teratorn DOT org + +In order for zsh completion to take place the shell must be able to find an +appropriate "stub" file ("completion function") that invokes this code and +displays the results to the user. + +The stub used for Twisted commands is in the file C{twisted-completion.zsh}, +which is also included in the official Zsh distribution at +C{Completion/Unix/Command/_twisted}. Use this file as a basis for completion +functions for your own commands. You should only need to change the first line +to something like C{#compdef mycommand}. + +The main public documentation exists in the L{twisted.python.usage.Options} +docstring, the L{twisted.python.usage.Completions} docstring, and the +Options howto. +""" +import itertools, getopt, inspect + +from twisted.python import reflect, util, usage +from twisted.python.compat import ioType, unicode + + + +def shellComplete(config, cmdName, words, shellCompFile): + """ + Perform shell completion. + + A completion function (shell script) is generated for the requested + shell and written to C{shellCompFile}, typically C{stdout}. The result + is then eval'd by the shell to produce the desired completions. + + @type config: L{twisted.python.usage.Options} + @param config: The L{twisted.python.usage.Options} instance to generate + completions for. + + @type cmdName: C{str} + @param cmdName: The name of the command we're generating completions for. + In the case of zsh, this is used to print an appropriate + "#compdef $CMD" line at the top of the output. This is + not necessary for the functionality of the system, but it + helps in debugging, since the output we produce is properly + formed and may be saved in a file and used as a stand-alone + completion function. + + @type words: C{list} of C{str} + @param words: The raw command-line words passed to use by the shell + stub function. argv[0] has already been stripped off. + + @type shellCompFile: C{file} + @param shellCompFile: The file to write completion data to. + """ + + # If given a file with unicode semantics, such as sys.stdout on Python 3, + # we must get at the the underlying buffer which has bytes semantics. + if shellCompFile and ioType(shellCompFile) == unicode: + shellCompFile = shellCompFile.buffer + + # shellName is provided for forward-compatibility. It is not used, + # since we currently only support zsh. + shellName, position = words[-1].split(":") + position = int(position) + # zsh gives the completion position ($CURRENT) as a 1-based index, + # and argv[0] has already been stripped off, so we subtract 2 to + # get the real 0-based index. + position -= 2 + cWord = words[position] + + # since the user may hit TAB at any time, we may have been called with an + # incomplete command-line that would generate getopt errors if parsed + # verbatim. However, we must do *some* parsing in order to determine if + # there is a specific subcommand that we need to provide completion for. + # So, to make the command-line more sane we work backwards from the + # current completion position and strip off all words until we find one + # that "looks" like a subcommand. It may in fact be the argument to a + # normal command-line option, but that won't matter for our purposes. + while position >= 1: + if words[position - 1].startswith("-"): + position -= 1 + else: + break + words = words[:position] + + subCommands = getattr(config, 'subCommands', None) + if subCommands: + # OK, this command supports sub-commands, so lets see if we have been + # given one. + + # If the command-line arguments are not valid then we won't be able to + # sanely detect the sub-command, so just generate completions as if no + # sub-command was found. + args = None + try: + opts, args = getopt.getopt(words, + config.shortOpt, config.longOpt) + except getopt.error: + pass + + if args: + # yes, we have a subcommand. Try to find it. + for (cmd, short, parser, doc) in config.subCommands: + if args[0] == cmd or args[0] == short: + subOptions = parser() + subOptions.parent = config + + gen = ZshSubcommandBuilder(subOptions, config, cmdName, + shellCompFile) + gen.write() + return + + # sub-command not given, or did not match any knowns sub-command names + genSubs = True + if cWord.startswith("-"): + # optimization: if the current word being completed starts + # with a hyphen then it can't be a sub-command, so skip + # the expensive generation of the sub-command list + genSubs = False + gen = ZshBuilder(config, cmdName, shellCompFile) + gen.write(genSubs=genSubs) + else: + gen = ZshBuilder(config, cmdName, shellCompFile) + gen.write() + + + +class SubcommandAction(usage.Completer): + def _shellCode(self, optName, shellType): + if shellType == usage._ZSH: + return '*::subcmd:->subcmd' + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class ZshBuilder(object): + """ + Constructs zsh code that will complete options for a given usage.Options + instance, possibly including a list of subcommand names. + + Completions for options to subcommands won't be generated because this + class will never be used if the user is completing options for a specific + subcommand. (See L{ZshSubcommandBuilder} below) + + @type options: L{twisted.python.usage.Options} + @ivar options: The L{twisted.python.usage.Options} instance defined for this + command. + + @type cmdName: C{str} + @ivar cmdName: The name of the command we're generating completions for. + + @type file: C{file} + @ivar file: The C{file} to write the completion function to. The C{file} + must have L{bytes} I/O semantics. + """ + def __init__(self, options, cmdName, file): + self.options = options + self.cmdName = cmdName + self.file = file + + + def write(self, genSubs=True): + """ + Generate the completion function and write it to the output file + @return: L{None} + + @type genSubs: C{bool} + @param genSubs: Flag indicating whether or not completions for the list + of subcommand should be generated. Only has an effect + if the C{subCommands} attribute has been defined on the + L{twisted.python.usage.Options} instance. + """ + if genSubs and getattr(self.options, 'subCommands', None) is not None: + gen = ZshArgumentsGenerator(self.options, self.cmdName, self.file) + gen.extraActions.insert(0, SubcommandAction()) + gen.write() + self.file.write(b'local _zsh_subcmds_array\n_zsh_subcmds_array=(\n') + for (cmd, short, parser, desc) in self.options.subCommands: + self.file.write( + b'\"' + cmd.encode('utf-8') + b':' + desc.encode('utf-8') +b'\"\n') + self.file.write(b")\n\n") + self.file.write(b'_describe "sub-command" _zsh_subcmds_array\n') + else: + gen = ZshArgumentsGenerator(self.options, self.cmdName, self.file) + gen.write() + + + +class ZshSubcommandBuilder(ZshBuilder): + """ + Constructs zsh code that will complete options for a given usage.Options + instance, and also for a single sub-command. This will only be used in + the case where the user is completing options for a specific subcommand. + + @type subOptions: L{twisted.python.usage.Options} + @ivar subOptions: The L{twisted.python.usage.Options} instance defined for + the sub command. + """ + def __init__(self, subOptions, *args): + self.subOptions = subOptions + ZshBuilder.__init__(self, *args) + + + def write(self): + """ + Generate the completion function and write it to the output file + @return: L{None} + """ + gen = ZshArgumentsGenerator(self.options, self.cmdName, self.file) + gen.extraActions.insert(0, SubcommandAction()) + gen.write() + + gen = ZshArgumentsGenerator(self.subOptions, self.cmdName, self.file) + gen.write() + + + +class ZshArgumentsGenerator(object): + """ + Generate a call to the zsh _arguments completion function + based on data in a usage.Options instance + + The first three instance variables are populated based on constructor + arguments. The remaining non-constructor variables are populated by this + class with data gathered from the C{Options} instance passed in, and its + base classes. + + @type options: L{twisted.python.usage.Options} + @ivar options: The L{twisted.python.usage.Options} instance to generate for + + @type cmdName: C{str} + @ivar cmdName: The name of the command we're generating completions for. + + @type file: C{file} + @ivar file: The C{file} to write the completion function to. The C{file} + must have L{bytes} I/O semantics. + + @type descriptions: C{dict} + @ivar descriptions: A dict mapping long option names to alternate + descriptions. When this variable is defined, the descriptions + contained here will override those descriptions provided in the + optFlags and optParameters variables. + + @type multiUse: C{list} + @ivar multiUse: An iterable containing those long option names which may + appear on the command line more than once. By default, options will + only be completed one time. + + @type mutuallyExclusive: C{list} of C{tuple} + @ivar mutuallyExclusive: A sequence of sequences, with each sub-sequence + containing those long option names that are mutually exclusive. That is, + those options that cannot appear on the command line together. + + @type optActions: C{dict} + @ivar optActions: A dict mapping long option names to shell "actions". + These actions define what may be completed as the argument to the + given option, and should be given as instances of + L{twisted.python.usage.Completer}. + + Callables may instead be given for the values in this dict. The + callable should accept no arguments, and return a C{Completer} + instance used as the action. + + @type extraActions: C{list} of C{twisted.python.usage.Completer} + @ivar extraActions: Extra arguments are those arguments typically + appearing at the end of the command-line, which are not associated + with any particular named option. That is, the arguments that are + given to the parseArgs() method of your usage.Options subclass. + """ + def __init__(self, options, cmdName, file): + self.options = options + self.cmdName = cmdName + self.file = file + + self.descriptions = {} + self.multiUse = set() + self.mutuallyExclusive = [] + self.optActions = {} + self.extraActions = [] + + for cls in reversed(inspect.getmro(options.__class__)): + data = getattr(cls, 'compData', None) + if data: + self.descriptions.update(data.descriptions) + self.optActions.update(data.optActions) + self.multiUse.update(data.multiUse) + + self.mutuallyExclusive.extend(data.mutuallyExclusive) + + # I don't see any sane way to aggregate extraActions, so just + # take the one at the top of the MRO (nearest the `options' + # instance). + if data.extraActions: + self.extraActions = data.extraActions + + aCL = reflect.accumulateClassList + + optFlags = [] + optParams = [] + + aCL(options.__class__, 'optFlags', optFlags) + aCL(options.__class__, 'optParameters', optParams) + + for i, optList in enumerate(optFlags): + if len(optList) != 3: + optFlags[i] = util.padTo(3, optList) + + for i, optList in enumerate(optParams): + if len(optList) != 5: + optParams[i] = util.padTo(5, optList) + + + self.optFlags = optFlags + self.optParams = optParams + + paramNameToDefinition = {} + for optList in optParams: + paramNameToDefinition[optList[0]] = optList[1:] + self.paramNameToDefinition = paramNameToDefinition + + flagNameToDefinition = {} + for optList in optFlags: + flagNameToDefinition[optList[0]] = optList[1:] + self.flagNameToDefinition = flagNameToDefinition + + allOptionsNameToDefinition = {} + allOptionsNameToDefinition.update(paramNameToDefinition) + allOptionsNameToDefinition.update(flagNameToDefinition) + self.allOptionsNameToDefinition = allOptionsNameToDefinition + + self.addAdditionalOptions() + + # makes sure none of the Completions metadata references + # option names that don't exist. (great for catching typos) + self.verifyZshNames() + + self.excludes = self.makeExcludesDict() + + + def write(self): + """ + Write the zsh completion code to the file given to __init__ + @return: L{None} + """ + self.writeHeader() + self.writeExtras() + self.writeOptions() + self.writeFooter() + + + def writeHeader(self): + """ + This is the start of the code that calls _arguments + @return: L{None} + """ + self.file.write(b'#compdef ' + self.cmdName.encode('utf-8') + + b'\n\n' + b'_arguments -s -A "-*" \\\n') + + + def writeOptions(self): + """ + Write out zsh code for each option in this command + @return: L{None} + """ + optNames = list(self.allOptionsNameToDefinition.keys()) + optNames.sort() + for longname in optNames: + self.writeOpt(longname) + + + def writeExtras(self): + """ + Write out completion information for extra arguments appearing on the + command-line. These are extra positional arguments not associated + with a named option. That is, the stuff that gets passed to + Options.parseArgs(). + + @return: L{None} + + @raises: ValueError: if C{Completer} with C{repeat=True} is found and + is not the last item in the C{extraActions} list. + """ + for i, action in enumerate(self.extraActions): + # a repeatable action must be the last action in the list + if action._repeat and i != len(self.extraActions) - 1: + raise ValueError("Completer with repeat=True must be " + "last item in Options.extraActions") + self.file.write( + escape(action._shellCode('', usage._ZSH)).encode('utf-8')) + self.file.write(b' \\\n') + + + def writeFooter(self): + """ + Write the last bit of code that finishes the call to _arguments + @return: L{None} + """ + self.file.write(b'&& return 0\n') + + + def verifyZshNames(self): + """ + Ensure that none of the option names given in the metadata are typoed + @return: L{None} + @raise ValueError: Raised if unknown option names have been found. + """ + def err(name): + raise ValueError("Unknown option name \"%s\" found while\n" + "examining Completions instances on %s" % ( + name, self.options)) + + for name in itertools.chain(self.descriptions, self.optActions, + self.multiUse): + if name not in self.allOptionsNameToDefinition: + err(name) + + for seq in self.mutuallyExclusive: + for name in seq: + if name not in self.allOptionsNameToDefinition: + err(name) + + + def excludeStr(self, longname, buildShort=False): + """ + Generate an "exclusion string" for the given option + + @type longname: C{str} + @param longname: The long option name (e.g. "verbose" instead of "v") + + @type buildShort: C{bool} + @param buildShort: May be True to indicate we're building an excludes + string for the short option that corresponds to the given long opt. + + @return: The generated C{str} + """ + if longname in self.excludes: + exclusions = self.excludes[longname].copy() + else: + exclusions = set() + + # if longname isn't a multiUse option (can't appear on the cmd line more + # than once), then we have to exclude the short option if we're + # building for the long option, and vice versa. + if longname not in self.multiUse: + if buildShort is False: + short = self.getShortOption(longname) + if short is not None: + exclusions.add(short) + else: + exclusions.add(longname) + + if not exclusions: + return '' + + strings = [] + for optName in exclusions: + if len(optName) == 1: + # short option + strings.append("-" + optName) + else: + strings.append("--" + optName) + strings.sort() # need deterministic order for reliable unit-tests + return "(%s)" % " ".join(strings) + + + def makeExcludesDict(self): + """ + @return: A C{dict} that maps each option name appearing in + self.mutuallyExclusive to a list of those option names that is it + mutually exclusive with (can't appear on the cmd line with). + """ + + #create a mapping of long option name -> single character name + longToShort = {} + for optList in itertools.chain(self.optParams, self.optFlags): + if optList[1] != None: + longToShort[optList[0]] = optList[1] + + excludes = {} + for lst in self.mutuallyExclusive: + for i, longname in enumerate(lst): + tmp = set(lst[:i] + lst[i+1:]) + for name in tmp.copy(): + if name in longToShort: + tmp.add(longToShort[name]) + + if longname in excludes: + excludes[longname] = excludes[longname].union(tmp) + else: + excludes[longname] = tmp + return excludes + + + def writeOpt(self, longname): + """ + Write out the zsh code for the given argument. This is just part of the + one big call to _arguments + + @type longname: C{str} + @param longname: The long option name (e.g. "verbose" instead of "v") + + @return: L{None} + """ + if longname in self.flagNameToDefinition: + # It's a flag option. Not one that takes a parameter. + longField = "--%s" % longname + else: + longField = "--%s=" % longname + + short = self.getShortOption(longname) + if short != None: + shortField = "-" + short + else: + shortField = '' + + descr = self.getDescription(longname) + descriptionField = descr.replace("[", "\[") + descriptionField = descriptionField.replace("]", "\]") + descriptionField = '[%s]' % descriptionField + + actionField = self.getAction(longname) + if longname in self.multiUse: + multiField = '*' + else: + multiField = '' + + longExclusionsField = self.excludeStr(longname) + + if short: + #we have to write an extra line for the short option if we have one + shortExclusionsField = self.excludeStr(longname, buildShort=True) + self.file.write(escape('%s%s%s%s%s' % (shortExclusionsField, + multiField, shortField, descriptionField, actionField)).encode('utf-8')) + self.file.write(b' \\\n') + + self.file.write(escape('%s%s%s%s%s' % (longExclusionsField, + multiField, longField, descriptionField, actionField)).encode('utf-8')) + self.file.write(b' \\\n') + + + def getAction(self, longname): + """ + Return a zsh "action" string for the given argument + @return: C{str} + """ + if longname in self.optActions: + if callable(self.optActions[longname]): + action = self.optActions[longname]() + else: + action = self.optActions[longname] + return action._shellCode(longname, usage._ZSH) + + if longname in self.paramNameToDefinition: + return ':%s:_files' % (longname,) + return '' + + + def getDescription(self, longname): + """ + Return the description to be used for this argument + @return: C{str} + """ + #check if we have an alternate descr for this arg, and if so use it + if longname in self.descriptions: + return self.descriptions[longname] + + #otherwise we have to get it from the optFlags or optParams + try: + descr = self.flagNameToDefinition[longname][1] + except KeyError: + try: + descr = self.paramNameToDefinition[longname][2] + except KeyError: + descr = None + + if descr is not None: + return descr + + # let's try to get it from the opt_foo method doc string if there is one + longMangled = longname.replace('-', '_') # this is what t.p.usage does + obj = getattr(self.options, 'opt_%s' % longMangled, None) + if obj is not None: + descr = descrFromDoc(obj) + if descr is not None: + return descr + + return longname # we really ought to have a good description to use + + + def getShortOption(self, longname): + """ + Return the short option letter or None + @return: C{str} or L{None} + """ + optList = self.allOptionsNameToDefinition[longname] + return optList[0] or None + + + def addAdditionalOptions(self): + """ + Add additional options to the optFlags and optParams lists. + These will be defined by 'opt_foo' methods of the Options subclass + @return: L{None} + """ + methodsDict = {} + reflect.accumulateMethods(self.options, methodsDict, 'opt_') + methodToShort = {} + for name in methodsDict.copy(): + if len(name) == 1: + methodToShort[methodsDict[name]] = name + del methodsDict[name] + + for methodName, methodObj in methodsDict.items(): + longname = methodName.replace('_', '-') # t.p.usage does this + # if this option is already defined by the optFlags or + # optParameters then we don't want to override that data + if longname in self.allOptionsNameToDefinition: + continue + + descr = self.getDescription(longname) + + short = None + if methodObj in methodToShort: + short = methodToShort[methodObj] + + reqArgs = methodObj.__func__.__code__.co_argcount + if reqArgs == 2: + self.optParams.append([longname, short, None, descr]) + self.paramNameToDefinition[longname] = [short, None, descr] + self.allOptionsNameToDefinition[longname] = [short, None, descr] + else: + # reqArgs must equal 1. self.options would have failed + # to instantiate if it had opt_ methods with bad signatures. + self.optFlags.append([longname, short, descr]) + self.flagNameToDefinition[longname] = [short, descr] + self.allOptionsNameToDefinition[longname] = [short, None, descr] + + + +def descrFromDoc(obj): + """ + Generate an appropriate description from docstring of the given object + """ + if obj.__doc__ is None or obj.__doc__.isspace(): + return None + + lines = [x.strip() for x in obj.__doc__.split("\n") + if x and not x.isspace()] + return " ".join(lines) + + + +def escape(x): + """ + Shell escape the given string + + Implementation borrowed from now-deprecated commands.mkarg() in the stdlib + """ + if '\'' not in x: + return '\'' + x + '\'' + s = '"' + for c in x: + if c in '\\$"`': + s = s + '\\' + s = s + c + s = s + '"' + return s + diff --git a/contrib/python/Twisted/py2/twisted/python/_textattributes.py b/contrib/python/Twisted/py2/twisted/python/_textattributes.py new file mode 100644 index 00000000000..2cede6aae8c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_textattributes.py @@ -0,0 +1,320 @@ +# -*- test-case-name: twisted.python.test.test_textattributes -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module provides some common functionality for the manipulation of +formatting states. + +Defining the mechanism by which text containing character attributes is +constructed begins by subclassing L{CharacterAttributesMixin}. + +Defining how a single formatting state is to be serialized begins by +subclassing L{_FormattingStateMixin}. + +Serializing a formatting structure is done with L{flatten}. + +@see: L{twisted.conch.insults.helper._FormattingState} +@see: L{twisted.conch.insults.text._CharacterAttributes} +@see: L{twisted.words.protocols.irc._FormattingState} +@see: L{twisted.words.protocols.irc._CharacterAttributes} +""" + +from __future__ import print_function + +from twisted.python.util import FancyEqMixin + + + +class _Attribute(FancyEqMixin, object): + """ + A text attribute. + + Indexing a text attribute with a C{str} or another text attribute adds that + object as a child, indexing with a C{list} or C{tuple} adds the elements as + children; in either case C{self} is returned. + + @type children: C{list} + @ivar children: Child attributes. + """ + compareAttributes = ('children',) + + + def __init__(self): + self.children = [] + + + def __repr__(self): + return '<%s %r>' % (type(self).__name__, vars(self)) + + + def __getitem__(self, item): + assert isinstance(item, (list, tuple, _Attribute, str)) + if isinstance(item, (list, tuple)): + self.children.extend(item) + else: + self.children.append(item) + return self + + + def serialize(self, write, attrs=None, attributeRenderer='toVT102'): + """ + Serialize the text attribute and its children. + + @param write: C{callable}, taking one C{str} argument, called to output + a single text attribute at a time. + + @param attrs: A formatting state instance used to determine how to + serialize the attribute children. + + @type attributeRenderer: C{str} + @param attributeRenderer: Name of the method on I{attrs} that should be + called to render the attributes during serialization. Defaults to + C{'toVT102'}. + """ + if attrs is None: + attrs = DefaultFormattingState() + for ch in self.children: + if isinstance(ch, _Attribute): + ch.serialize(write, attrs.copy(), attributeRenderer) + else: + renderMeth = getattr(attrs, attributeRenderer) + write(renderMeth()) + write(ch) + + + +class _NormalAttr(_Attribute): + """ + A text attribute for normal text. + """ + def serialize(self, write, attrs, attributeRenderer): + attrs.__init__() + _Attribute.serialize(self, write, attrs, attributeRenderer) + + + +class _OtherAttr(_Attribute): + """ + A text attribute for text with formatting attributes. + + The unary minus operator returns the inverse of this attribute, where that + makes sense. + + @type attrname: C{str} + @ivar attrname: Text attribute name. + + @ivar attrvalue: Text attribute value. + """ + compareAttributes = ('attrname', 'attrvalue', 'children') + + + def __init__(self, attrname, attrvalue): + _Attribute.__init__(self) + self.attrname = attrname + self.attrvalue = attrvalue + + + def __neg__(self): + result = _OtherAttr(self.attrname, not self.attrvalue) + result.children.extend(self.children) + return result + + + def serialize(self, write, attrs, attributeRenderer): + attrs = attrs._withAttribute(self.attrname, self.attrvalue) + _Attribute.serialize(self, write, attrs, attributeRenderer) + + + +class _ColorAttr(_Attribute): + """ + Generic color attribute. + + @param color: Color value. + + @param ground: Foreground or background attribute name. + """ + compareAttributes = ('color', 'ground', 'children') + + + def __init__(self, color, ground): + _Attribute.__init__(self) + self.color = color + self.ground = ground + + + def serialize(self, write, attrs, attributeRenderer): + attrs = attrs._withAttribute(self.ground, self.color) + _Attribute.serialize(self, write, attrs, attributeRenderer) + + + +class _ForegroundColorAttr(_ColorAttr): + """ + Foreground color attribute. + """ + def __init__(self, color): + _ColorAttr.__init__(self, color, 'foreground') + + + +class _BackgroundColorAttr(_ColorAttr): + """ + Background color attribute. + """ + def __init__(self, color): + _ColorAttr.__init__(self, color, 'background') + + + +class _ColorAttribute(object): + """ + A color text attribute. + + Attribute access results in a color value lookup, by name, in + I{_ColorAttribute.attrs}. + + @type ground: L{_ColorAttr} + @param ground: Foreground or background color attribute to look color names + up from. + + @param attrs: Mapping of color names to color values. + @type attrs: Dict like object. + """ + def __init__(self, ground, attrs): + self.ground = ground + self.attrs = attrs + + + def __getattr__(self, name): + try: + return self.ground(self.attrs[name]) + except KeyError: + raise AttributeError(name) + + + +class CharacterAttributesMixin(object): + """ + Mixin for character attributes that implements a C{__getattr__} method + returning a new C{_NormalAttr} instance when attempting to access + a C{'normal'} attribute; otherwise a new C{_OtherAttr} instance is returned + for names that appears in the C{'attrs'} attribute. + """ + def __getattr__(self, name): + if name == 'normal': + return _NormalAttr() + if name in self.attrs: + return _OtherAttr(name, True) + raise AttributeError(name) + + + +class DefaultFormattingState(FancyEqMixin, object): + """ + A character attribute that does nothing, thus applying no attributes to + text. + """ + compareAttributes = ('_dummy',) + + _dummy = 0 + + + def copy(self): + """ + Make a copy of this formatting state. + + @return: A formatting state instance. + """ + return type(self)() + + + def _withAttribute(self, name, value): + """ + Add a character attribute to a copy of this formatting state. + + @param name: Attribute name to be added to formatting state. + + @param value: Attribute value. + + @return: A formatting state instance with the new attribute. + """ + return self.copy() + + + def toVT102(self): + """ + Emit a VT102 control sequence that will set up all the attributes this + formatting state has set. + + @return: A string containing VT102 control sequences that mimic this + formatting state. + """ + return '' + + + +class _FormattingStateMixin(DefaultFormattingState): + """ + Mixin for the formatting state/attributes of a single character. + """ + def copy(self): + c = DefaultFormattingState.copy(self) + c.__dict__.update(vars(self)) + return c + + + def _withAttribute(self, name, value): + if getattr(self, name) != value: + attr = self.copy() + attr._subtracting = not value + setattr(attr, name, value) + return attr + else: + return self.copy() + + + +def flatten(output, attrs, attributeRenderer='toVT102'): + """ + Serialize a sequence of characters with attribute information + + The resulting string can be interpreted by compatible software so that the + contained characters are displayed and, for those attributes which are + supported by the software, the attributes expressed. The exact result of + the serialization depends on the behavior of the method specified by + I{attributeRenderer}. + + For example, if your terminal is VT102 compatible, you might run + this for a colorful variation on the \"hello world\" theme:: + + from twisted.conch.insults.text import flatten, attributes as A + from twisted.conch.insults.helper import CharacterAttribute + print(flatten( + A.normal[A.bold[A.fg.red['He'], A.fg.green['ll'], A.fg.magenta['o'], ' ', + A.fg.yellow['Wo'], A.fg.blue['rl'], A.fg.cyan['d!']]], + CharacterAttribute())) + + @param output: Object returned by accessing attributes of the + module-level attributes object. + + @param attrs: A formatting state instance used to determine how to + serialize C{output}. + + @type attributeRenderer: C{str} + @param attributeRenderer: Name of the method on I{attrs} that should be + called to render the attributes during serialization. Defaults to + C{'toVT102'}. + + @return: A string expressing the text and display attributes specified by + L{output}. + """ + flattened = [] + output.serialize(flattened.append, attrs, attributeRenderer) + return ''.join(flattened) + + + +__all__ = [ + 'flatten', 'DefaultFormattingState', 'CharacterAttributesMixin'] diff --git a/contrib/python/Twisted/py2/twisted/python/_tzhelper.py b/contrib/python/Twisted/py2/twisted/python/_tzhelper.py new file mode 100644 index 00000000000..11ca6ce79a7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_tzhelper.py @@ -0,0 +1,119 @@ +# -*- test-case-name: twisted.python.test.test_tzhelper -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Time zone utilities. +""" + +from datetime import datetime, timedelta, tzinfo + +__all__ = [ + "FixedOffsetTimeZone", + "UTC", +] + + + +class FixedOffsetTimeZone(tzinfo): + """ + Represents a fixed timezone offset (without daylight saving time). + + @ivar name: A L{str} giving the name of this timezone; the name just + includes how much time this offset represents. + + @ivar offset: A L{timedelta} giving the amount of time this timezone is + offset. + """ + + def __init__(self, offset, name=None): + """ + Construct a L{FixedOffsetTimeZone} with a fixed offset. + + @param offset: a delta representing the offset from UTC. + @type offset: L{timedelta} + + @param name: A name to be given for this timezone. + @type name: L{str} or L{None} + """ + self.offset = offset + self.name = name + + + @classmethod + def fromSignHoursMinutes(cls, sign, hours, minutes): + """ + Construct a L{FixedOffsetTimeZone} from an offset described by sign + ('+' or '-'), hours, and minutes. + + @note: For protocol compatibility with AMP, this method never uses 'Z' + + @param sign: A string describing the positive or negative-ness of the + offset. + + @param hours: The number of hours in the offset. + @type hours: L{int} + + @param minutes: The number of minutes in the offset + @type minutes: L{int} + + @return: A time zone with the given offset, and a name describing the + offset. + @rtype: L{FixedOffsetTimeZone} + """ + name = "%s%02i:%02i" % (sign, hours, minutes) + if sign == "-": + hours = -hours + minutes = -minutes + elif sign != "+": + raise ValueError("Invalid sign for timezone %r" % (sign,)) + return cls(timedelta(hours=hours, minutes=minutes), name) + + + @classmethod + def fromLocalTimeStamp(cls, timeStamp): + """ + Create a time zone with a fixed offset corresponding to a time stamp in + the system's locally configured time zone. + + @param timeStamp: a time stamp + @type timeStamp: L{int} + + @return: a time zone + @rtype: L{FixedOffsetTimeZone} + """ + offset = ( + datetime.fromtimestamp(timeStamp) - + datetime.utcfromtimestamp(timeStamp) + ) + return cls(offset) + + + def utcoffset(self, dt): + """ + Return this timezone's offset from UTC. + """ + return self.offset + + + def dst(self, dt): + """ + Return a zero C{datetime.timedelta} for the daylight saving time + offset, since there is never one. + """ + return timedelta(0) + + + def tzname(self, dt): + """ + Return a string describing this timezone. + """ + if self.name is not None: + return self.name + # XXX this is wrong; the tests are + dt = datetime.fromtimestamp(0, self) + return dt.strftime("UTC%z") + + + +UTC = FixedOffsetTimeZone.fromSignHoursMinutes("+", 0, 0) diff --git a/contrib/python/Twisted/py2/twisted/python/_url.py b/contrib/python/Twisted/py2/twisted/python/_url.py new file mode 100644 index 00000000000..8df52b892da --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/_url.py @@ -0,0 +1,13 @@ +# -*- test-case-name: twisted.python.test.test_url -*- +# -*- coding: utf-8 -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +URL parsing, construction and rendering. +""" + +from hyperlink._url import URL + + +__all__ = ["URL"] diff --git a/contrib/python/Twisted/py2/twisted/python/compat.py b/contrib/python/Twisted/py2/twisted/python/compat.py new file mode 100644 index 00000000000..35def79041f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/compat.py @@ -0,0 +1,928 @@ +# -*- test-case-name: twisted.test.test_compat -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Compatibility module to provide backwards compatibility for useful Python +features. + +This is mainly for use of internal Twisted code. We encourage you to use +the latest version of Python directly from your code, if possible. + +@var unicode: The type of Unicode strings, C{unicode} on Python 2 and C{str} + on Python 3. + +@var NativeStringIO: An in-memory file-like object that operates on the native + string type (bytes in Python 2, unicode in Python 3). + +@var urllib_parse: a URL-parsing module (urlparse on Python 2, urllib.parse on + Python 3) +""" + +from __future__ import absolute_import, division + +import inspect +import os +import platform +import socket +import struct +import sys +import tokenize +from types import MethodType as _MethodType +import warnings + +from io import TextIOBase, IOBase + + +if sys.version_info < (3, 0): + _PY3 = False +else: + _PY3 = True + +if sys.version_info >= (3, 5, 0): + _PY35PLUS = True +else: + _PY35PLUS = False + +if sys.version_info >= (3, 7, 0): + _PY37PLUS = True +else: + _PY37PLUS = False + +if platform.python_implementation() == 'PyPy': + _PYPY = True +else: + _PYPY = False + + + +def _shouldEnableNewStyle(): + """ + Returns whether or not we should enable the new-style conversion of + old-style classes. It inspects the environment for C{TWISTED_NEWSTYLE}, + accepting an empty string, C{no}, C{false}, C{False}, and C{0} as falsey + values and everything else as a truthy value. + + @rtype: L{bool} + """ + value = os.environ.get('TWISTED_NEWSTYLE', '') + + if value in ['', 'no', 'false', 'False', '0']: + return False + else: + return True + + +_EXPECT_NEWSTYLE = _PY3 or _shouldEnableNewStyle() + + +def currentframe(n=0): + """ + In Python 3, L{inspect.currentframe} does not take a stack-level argument. + Restore that functionality from Python 2 so we don't have to re-implement + the C{f_back}-walking loop in places where it's called. + + @param n: The number of stack levels above the caller to walk. + @type n: L{int} + + @return: a frame, n levels up the stack from the caller. + @rtype: L{types.FrameType} + """ + f = inspect.currentframe() + for x in range(n + 1): + f = f.f_back + return f + + + +def inet_pton(af, addr): + """ + Emulator of L{socket.inet_pton}. + + @param af: An address family to parse; C{socket.AF_INET} or + C{socket.AF_INET6}. + @type af: L{int} + + @param addr: An address. + @type addr: native L{str} + + @return: The binary packed version of the passed address. + @rtype: L{bytes} + """ + if not addr: + raise ValueError("illegal IP address string passed to inet_pton") + if af == socket.AF_INET: + return socket.inet_aton(addr) + elif af == getattr(socket, 'AF_INET6', 'AF_INET6'): + if '%' in addr and (addr.count('%') > 1 or addr.index("%") == 0): + raise ValueError("illegal IP address string passed to inet_pton") + addr = addr.split('%')[0] + parts = addr.split(':') + elided = parts.count('') + ipv4Component = '.' in parts[-1] + + if len(parts) > (8 - ipv4Component) or elided > 3: + raise ValueError("Syntactically invalid address") + + if elided == 3: + return '\x00' * 16 + + if elided: + zeros = ['0'] * (8 - len(parts) - ipv4Component + elided) + + if addr.startswith('::'): + parts[:2] = zeros + elif addr.endswith('::'): + parts[-2:] = zeros + else: + idx = parts.index('') + parts[idx:idx+1] = zeros + + if len(parts) != 8 - ipv4Component: + raise ValueError("Syntactically invalid address") + else: + if len(parts) != (8 - ipv4Component): + raise ValueError("Syntactically invalid address") + + if ipv4Component: + if parts[-1].count('.') != 3: + raise ValueError("Syntactically invalid address") + rawipv4 = socket.inet_aton(parts[-1]) + unpackedipv4 = struct.unpack('!HH', rawipv4) + parts[-1:] = [hex(x)[2:] for x in unpackedipv4] + + parts = [int(x, 16) for x in parts] + return struct.pack('!8H', *parts) + else: + raise socket.error(97, 'Address family not supported by protocol') + + + +def inet_ntop(af, addr): + if af == socket.AF_INET: + return socket.inet_ntoa(addr) + elif af == socket.AF_INET6: + if len(addr) != 16: + raise ValueError("address length incorrect") + parts = struct.unpack('!8H', addr) + curBase = bestBase = None + for i in range(8): + if not parts[i]: + if curBase is None: + curBase = i + curLen = 0 + curLen += 1 + else: + if curBase is not None: + bestLen = None + if bestBase is None or curLen > bestLen: + bestBase = curBase + bestLen = curLen + curBase = None + if curBase is not None and (bestBase is None or curLen > bestLen): + bestBase = curBase + bestLen = curLen + parts = [hex(x)[2:] for x in parts] + if bestBase is not None: + parts[bestBase:bestBase + bestLen] = [''] + if parts[0] == '': + parts.insert(0, '') + if parts[-1] == '': + parts.insert(len(parts) - 1, '') + return ':'.join(parts) + else: + raise socket.error(97, 'Address family not supported by protocol') + +try: + socket.AF_INET6 +except AttributeError: + socket.AF_INET6 = 'AF_INET6' + +try: + socket.inet_pton(socket.AF_INET6, "::") +except (AttributeError, NameError, socket.error): + socket.inet_pton = inet_pton + socket.inet_ntop = inet_ntop + + +adict = dict + + + +if _PY3: + # These are actually useless in Python 2 as well, but we need to go + # through deprecation process there (ticket #5895): + del adict, inet_pton, inet_ntop + + +set = set +frozenset = frozenset + + +try: + from functools import reduce +except ImportError: + reduce = reduce + + + +def execfile(filename, globals, locals=None): + """ + Execute a Python script in the given namespaces. + + Similar to the execfile builtin, but a namespace is mandatory, partly + because that's a sensible thing to require, and because otherwise we'd + have to do some frame hacking. + + This is a compatibility implementation for Python 3 porting, to avoid the + use of the deprecated builtin C{execfile} function. + """ + if locals is None: + locals = globals + with open(filename, "rb") as fin: + source = fin.read() + code = compile(source, filename, "exec") + exec(code, globals, locals) + + +try: + cmp = cmp +except NameError: + def cmp(a, b): + """ + Compare two objects. + + Returns a negative number if C{a < b}, zero if they are equal, and a + positive number if C{a > b}. + """ + if a < b: + return -1 + elif a == b: + return 0 + else: + return 1 + + + +def comparable(klass): + """ + Class decorator that ensures support for the special C{__cmp__} method. + + On Python 2 this does nothing. + + On Python 3, C{__eq__}, C{__lt__}, etc. methods are added to the class, + relying on C{__cmp__} to implement their comparisons. + """ + # On Python 2, __cmp__ will just work, so no need to add extra methods: + if not _PY3: + return klass + + + def __eq__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c == 0 + + + def __ne__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c != 0 + + + def __lt__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c < 0 + + + def __le__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c <= 0 + + + def __gt__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c > 0 + + + def __ge__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c >= 0 + + klass.__lt__ = __lt__ + klass.__gt__ = __gt__ + klass.__le__ = __le__ + klass.__ge__ = __ge__ + klass.__eq__ = __eq__ + klass.__ne__ = __ne__ + return klass + + + +if _PY3: + unicode = str + long = int +else: + unicode = unicode + long = long + + + +def ioType(fileIshObject, default=unicode): + """ + Determine the type which will be returned from the given file object's + read() and accepted by its write() method as an argument. + + In other words, determine whether the given file is 'opened in text mode'. + + @param fileIshObject: Any object, but ideally one which resembles a file. + @type fileIshObject: L{object} + + @param default: A default value to return when the type of C{fileIshObject} + cannot be determined. + @type default: L{type} + + @return: There are 3 possible return values: + + 1. L{unicode}, if the file is unambiguously opened in text mode. + + 2. L{bytes}, if the file is unambiguously opened in binary mode. + + 3. L{basestring}, if we are on python 2 (the L{basestring} type + does not exist on python 3) and the file is opened in binary + mode, but has an encoding and can therefore accept both bytes + and text reliably for writing, but will return L{bytes} from + read methods. + + 4. The C{default} parameter, if the given type is not understood. + + @rtype: L{type} + """ + if isinstance(fileIshObject, TextIOBase): + # If it's for text I/O, then it's for text I/O. + return unicode + if isinstance(fileIshObject, IOBase): + # If it's for I/O but it's _not_ for text I/O, it's for bytes I/O. + return bytes + encoding = getattr(fileIshObject, 'encoding', None) + import codecs + if isinstance(fileIshObject, (codecs.StreamReader, codecs.StreamWriter)): + # On StreamReaderWriter, the 'encoding' attribute has special meaning; + # it is unambiguously unicode. + if encoding: + return unicode + else: + return bytes + if not _PY3: + # Special case: if we have an encoding file, we can *give* it unicode, + # but we can't expect to *get* unicode. + if isinstance(fileIshObject, file): + if encoding is not None: + return basestring + else: + return bytes + from cStringIO import InputType, OutputType + from StringIO import StringIO + if isinstance(fileIshObject, (StringIO, InputType, OutputType)): + return bytes + return default + + + +def nativeString(s): + """ + Convert C{bytes} or C{unicode} to the native C{str} type, using ASCII + encoding if conversion is necessary. + + @raise UnicodeError: The input string is not ASCII encodable/decodable. + @raise TypeError: The input is neither C{bytes} nor C{unicode}. + """ + if not isinstance(s, (bytes, unicode)): + raise TypeError("%r is neither bytes nor unicode" % s) + if _PY3: + if isinstance(s, bytes): + return s.decode("ascii") + else: + # Ensure we're limited to ASCII subset: + s.encode("ascii") + else: + if isinstance(s, unicode): + return s.encode("ascii") + else: + # Ensure we're limited to ASCII subset: + s.decode("ascii") + return s + + + +def _matchingString(constantString, inputString): + """ + Some functions, such as C{os.path.join}, operate on string arguments which + may be bytes or text, and wish to return a value of the same type. In + those cases you may wish to have a string constant (in the case of + C{os.path.join}, that constant would be C{os.path.sep}) involved in the + parsing or processing, that must be of a matching type in order to use + string operations on it. L{_matchingString} will take a constant string + (either L{bytes} or L{unicode}) and convert it to the same type as the + input string. C{constantString} should contain only characters from ASCII; + to ensure this, it will be encoded or decoded regardless. + + @param constantString: A string literal used in processing. + @type constantString: L{unicode} or L{bytes} + + @param inputString: A byte string or text string provided by the user. + @type inputString: L{unicode} or L{bytes} + + @return: C{constantString} converted into the same type as C{inputString} + @rtype: the type of C{inputString} + """ + if isinstance(constantString, bytes): + otherType = constantString.decode("ascii") + else: + otherType = constantString.encode("ascii") + if type(constantString) == type(inputString): + return constantString + else: + return otherType + + + +if _PY3: + def reraise(exception, traceback): + raise exception.with_traceback(traceback) +else: + exec("""def reraise(exception, traceback): + raise exception.__class__, exception, traceback""") + +reraise.__doc__ = """ +Re-raise an exception, with an optional traceback, in a way that is compatible +with both Python 2 and Python 3. + +Note that on Python 3, re-raised exceptions will be mutated, with their +C{__traceback__} attribute being set. + +@param exception: The exception instance. +@param traceback: The traceback to use, or L{None} indicating a new traceback. +""" + + + +if _PY3: + from io import StringIO as NativeStringIO +else: + from io import BytesIO as NativeStringIO + + + +# Functions for dealing with Python 3's bytes type, which is somewhat +# different than Python 2's: +if _PY3: + def iterbytes(originalBytes): + for i in range(len(originalBytes)): + yield originalBytes[i:i+1] + + + def intToBytes(i): + return ("%d" % i).encode("ascii") + + + def lazyByteSlice(object, offset=0, size=None): + """ + Return a copy of the given bytes-like object. + + If an offset is given, the copy starts at that offset. If a size is + given, the copy will only be of that length. + + @param object: C{bytes} to be copied. + + @param offset: C{int}, starting index of copy. + + @param size: Optional, if an C{int} is given limit the length of copy + to this size. + """ + view = memoryview(object) + if size is None: + return view[offset:] + else: + return view[offset:(offset + size)] + + + def networkString(s): + if not isinstance(s, unicode): + raise TypeError("Can only convert text to bytes on Python 3") + return s.encode('ascii') +else: + def iterbytes(originalBytes): + return originalBytes + + + def intToBytes(i): + return b"%d" % i + + lazyByteSlice = buffer + + def networkString(s): + if not isinstance(s, str): + raise TypeError("Can only pass-through bytes on Python 2") + # Ensure we're limited to ASCII subset: + s.decode('ascii') + return s + +iterbytes.__doc__ = """ +Return an iterable wrapper for a C{bytes} object that provides the behavior of +iterating over C{bytes} on Python 2. + +In particular, the results of iteration are the individual bytes (rather than +integers as on Python 3). + +@param originalBytes: A C{bytes} object that will be wrapped. +""" + +intToBytes.__doc__ = """ +Convert the given integer into C{bytes}, as ASCII-encoded Arab numeral. + +In other words, this is equivalent to calling C{bytes} in Python 2 on an +integer. + +@param i: The C{int} to convert to C{bytes}. +@rtype: C{bytes} +""" + +networkString.__doc__ = """ +Convert the native string type to C{bytes} if it is not already C{bytes} using +ASCII encoding if conversion is necessary. + +This is useful for sending text-like bytes that are constructed using string +interpolation. For example, this is safe on Python 2 and Python 3: + + networkString("Hello %d" % (n,)) + +@param s: A native string to convert to bytes if necessary. +@type s: C{str} + +@raise UnicodeError: The input string is not ASCII encodable/decodable. +@raise TypeError: The input is neither C{bytes} nor C{unicode}. + +@rtype: C{bytes} +""" + + +try: + StringType = basestring +except NameError: + # Python 3+ + StringType = str + +try: + from types import InstanceType +except ImportError: + # Python 3+ + InstanceType = object + +try: + from types import FileType +except ImportError: + # Python 3+ + FileType = IOBase + +if _PY3: + import urllib.parse as urllib_parse + from html import escape + from urllib.parse import quote as urlquote + from urllib.parse import unquote as urlunquote + from http import cookiejar as cookielib +else: + import urlparse as urllib_parse + from cgi import escape + from urllib import quote as urlquote + from urllib import unquote as urlunquote + import cookielib + + +# Dealing with the differences in items/iteritems +if _PY3: + def iteritems(d): + return d.items() + + + def itervalues(d): + return d.values() + + + def items(d): + return list(d.items()) + + range = range + xrange = range + izip = zip +else: + def iteritems(d): + return d.iteritems() + + + def itervalues(d): + return d.itervalues() + + + def items(d): + return d.items() + + range = xrange + xrange = xrange + from itertools import izip + izip # shh pyflakes + + +iteritems.__doc__ = """ +Return an iterable of the items of C{d}. + +@type d: L{dict} +@rtype: iterable +""" + +itervalues.__doc__ = """ +Return an iterable of the values of C{d}. + +@type d: L{dict} +@rtype: iterable +""" + +items.__doc__ = """ +Return a list of the items of C{d}. + +@type d: L{dict} +@rtype: L{list} +""" + +def _keys(d): + """ + Return a list of the keys of C{d}. + + @type d: L{dict} + @rtype: L{list} + """ + if _PY3: + return list(d.keys()) + else: + return d.keys() + + + +def bytesEnviron(): + """ + Return a L{dict} of L{os.environ} where all text-strings are encoded into + L{bytes}. + + This function is POSIX only; environment variables are always text strings + on Windows. + """ + if not _PY3: + # On py2, nothing to do. + return dict(os.environ) + + target = dict() + for x, y in os.environ.items(): + target[os.environ.encodekey(x)] = os.environ.encodevalue(y) + + return target + + + +def _constructMethod(cls, name, self): + """ + Construct a bound method. + + @param cls: The class that the method should be bound to. + @type cls: L{types.ClassType} or L{type}. + + @param name: The name of the method. + @type name: native L{str} + + @param self: The object that the method is bound to. + @type self: any object + + @return: a bound method + @rtype: L{types.MethodType} + """ + func = cls.__dict__[name] + if _PY3: + return _MethodType(func, self) + return _MethodType(func, self, cls) + + + +if _PY3: + from base64 import encodebytes as _b64encodebytes + from base64 import decodebytes as _b64decodebytes +else: + from base64 import encodestring as _b64encodebytes + from base64 import decodestring as _b64decodebytes + + + +def _bytesChr(i): + """ + Like L{chr} but always works on ASCII, returning L{bytes}. + + @param i: The ASCII code point to return. + @type i: L{int} + + @rtype: L{bytes} + """ + if _PY3: + return bytes([i]) + else: + return chr(i) + + + +try: + from sys import intern +except ImportError: + intern = intern + + + +def _coercedUnicode(s): + """ + Coerce ASCII-only byte strings into unicode for Python 2. + + In Python 2 C{unicode(b'bytes')} returns a unicode string C{'bytes'}. In + Python 3, the equivalent C{str(b'bytes')} will return C{"b'bytes'"} + instead. This function mimics the behavior for Python 2. It will decode the + byte string as ASCII. In Python 3 it simply raises a L{TypeError} when + passing a byte string. Unicode strings are returned as-is. + + @param s: The string to coerce. + @type s: L{bytes} or L{unicode} + + @raise UnicodeError: The input L{bytes} is not ASCII decodable. + @raise TypeError: The input is L{bytes} on Python 3. + """ + if isinstance(s, bytes): + if _PY3: + raise TypeError("Expected str not %r (bytes)" % (s,)) + else: + return s.decode('ascii') + else: + return s + + + +if _PY3: + unichr = chr + raw_input = input +else: + unichr = unichr + raw_input = raw_input + + + +def _bytesRepr(bytestring): + """ + Provide a repr for a byte string that begins with 'b' on both + Python 2 and 3. + + @param bytestring: The string to repr. + @type bytestring: L{bytes} + + @raise TypeError: The input is not L{bytes}. + + @return: The repr with a leading 'b'. + @rtype: L{bytes} + """ + if not isinstance(bytestring, bytes): + raise TypeError("Expected bytes not %r" % (bytestring,)) + + if _PY3: + return repr(bytestring) + else: + return 'b' + repr(bytestring) + + +if _PY3: + _tokenize = tokenize.tokenize +else: + _tokenize = tokenize.generate_tokens + +try: + from collections.abc import Sequence +except ImportError: + from collections import Sequence + + + +def _get_async_param(isAsync=None, **kwargs): + """ + Provide a backwards-compatible way to get async param value that does not + cause a syntax error under Python 3.7. + + @param isAsync: isAsync param value (should default to None) + @type isAsync: L{bool} + + @param kwargs: keyword arguments of the caller (only async is allowed) + @type kwargs: L{dict} + + @raise TypeError: Both isAsync and async specified. + + @return: Final isAsync param value + @rtype: L{bool} + """ + if 'async' in kwargs: + warnings.warn( + "'async' keyword argument is deprecated, please use isAsync", + DeprecationWarning, stacklevel=2) + if isAsync is None and 'async' in kwargs: + isAsync = kwargs.pop('async') + if kwargs: + raise TypeError + return bool(isAsync) + + + +def _pypy3BlockingHack(): + """ + Work around U{this pypy bug + } + by replacing C{socket.fromfd} with a more conservative version. + """ + try: + from fcntl import fcntl, F_GETFL, F_SETFL + except ImportError: + return + if not (_PY3 and _PYPY): + return + + def fromFDWithoutModifyingFlags(fd, family, type, proto=None): + passproto = [proto] * (proto is not None) + flags = fcntl(fd, F_GETFL) + try: + return realFromFD(fd, family, type, *passproto) + finally: + fcntl(fd, F_SETFL, flags) + realFromFD = socket.fromfd + if realFromFD.__name__ == fromFDWithoutModifyingFlags.__name__: + return + socket.fromfd = fromFDWithoutModifyingFlags + + + +_pypy3BlockingHack() + + + +__all__ = [ + "reraise", + "execfile", + "frozenset", + "reduce", + "set", + "cmp", + "comparable", + "OrderedDict", + "nativeString", + "NativeStringIO", + "networkString", + "unicode", + "iterbytes", + "intToBytes", + "lazyByteSlice", + "StringType", + "InstanceType", + "FileType", + "items", + "iteritems", + "itervalues", + "range", + "xrange", + "urllib_parse", + "bytesEnviron", + "escape", + "urlquote", + "urlunquote", + "cookielib", + "_keys", + "_b64encodebytes", + "_b64decodebytes", + "_bytesChr", + "_coercedUnicode", + "_bytesRepr", + "intern", + "unichr", + "raw_input", + "_tokenize", + "_get_async_param", + "Sequence", +] diff --git a/contrib/python/Twisted/py2/twisted/python/components.py b/contrib/python/Twisted/py2/twisted/python/components.py new file mode 100644 index 00000000000..2a359e47e71 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/components.py @@ -0,0 +1,430 @@ +# -*- test-case-name: twisted.python.test.test_components -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Component architecture for Twisted, based on Zope3 components. + +Using the Zope3 API directly is strongly recommended. Everything +you need is in the top-level of the zope.interface package, e.g.:: + + from zope.interface import Interface, implementer + + class IFoo(Interface): + pass + + @implementer(IFoo) + class Foo: + pass + + print(IFoo.implementedBy(Foo)) # True + print(IFoo.providedBy(Foo())) # True + +L{twisted.python.components.registerAdapter} from this module may be used to +add to Twisted's global adapter registry. + +L{twisted.python.components.proxyForInterface} is a factory for classes +which allow access to only the parts of another class defined by a specified +interface. +""" + +from __future__ import division, absolute_import, print_function + +# zope3 imports +from zope.interface import interface, declarations +from zope.interface.adapter import AdapterRegistry + +# twisted imports +from twisted.python.compat import NativeStringIO +from twisted.python import reflect +from twisted.python._oldstyle import _oldStyle + + + +# Twisted's global adapter registry +globalRegistry = AdapterRegistry() + +# Attribute that registerAdapter looks at. Is this supposed to be public? +ALLOW_DUPLICATES = 0 + +def registerAdapter(adapterFactory, origInterface, *interfaceClasses): + """Register an adapter class. + + An adapter class is expected to implement the given interface, by + adapting instances implementing 'origInterface'. An adapter class's + __init__ method should accept one parameter, an instance implementing + 'origInterface'. + """ + self = globalRegistry + assert interfaceClasses, "You need to pass an Interface" + global ALLOW_DUPLICATES + + # deal with class->interface adapters: + if not isinstance(origInterface, interface.InterfaceClass): + origInterface = declarations.implementedBy(origInterface) + + for interfaceClass in interfaceClasses: + factory = self.registered([origInterface], interfaceClass) + if factory is not None and not ALLOW_DUPLICATES: + raise ValueError("an adapter (%s) was already registered." % (factory, )) + for interfaceClass in interfaceClasses: + self.register([origInterface], interfaceClass, '', adapterFactory) + + +def getAdapterFactory(fromInterface, toInterface, default): + """Return registered adapter for a given class and interface. + + Note that is tied to the *Twisted* global registry, and will + thus not find adapters registered elsewhere. + """ + self = globalRegistry + if not isinstance(fromInterface, interface.InterfaceClass): + fromInterface = declarations.implementedBy(fromInterface) + factory = self.lookup1(fromInterface, toInterface) + if factory is None: + factory = default + return factory + + +def _addHook(registry): + """ + Add an adapter hook which will attempt to look up adapters in the given + registry. + + @type registry: L{zope.interface.adapter.AdapterRegistry} + + @return: The hook which was added, for later use with L{_removeHook}. + """ + lookup = registry.lookup1 + def _hook(iface, ob): + factory = lookup(declarations.providedBy(ob), iface) + if factory is None: + return None + else: + return factory(ob) + interface.adapter_hooks.append(_hook) + return _hook + + +def _removeHook(hook): + """ + Remove a previously added adapter hook. + + @param hook: An object previously returned by a call to L{_addHook}. This + will be removed from the list of adapter hooks. + """ + interface.adapter_hooks.remove(hook) + +# add global adapter lookup hook for our newly created registry +_addHook(globalRegistry) + + +def getRegistry(): + """Returns the Twisted global + C{zope.interface.adapter.AdapterRegistry} instance. + """ + return globalRegistry + +# FIXME: deprecate attribute somehow? +CannotAdapt = TypeError + +@_oldStyle +class Adapter: + """I am the default implementation of an Adapter for some interface. + + This docstring contains a limerick, by popular demand:: + + Subclassing made Zope and TR + much harder to work with by far. + So before you inherit, + be sure to declare it + Adapter, not PyObject* + + @cvar temporaryAdapter: If this is True, the adapter will not be + persisted on the Componentized. + @cvar multiComponent: If this adapter is persistent, should it be + automatically registered for all appropriate interfaces. + """ + + # These attributes are used with Componentized. + + temporaryAdapter = 0 + multiComponent = 1 + + def __init__(self, original): + """Set my 'original' attribute to be the object I am adapting. + """ + self.original = original + + def __conform__(self, interface): + """ + I forward __conform__ to self.original if it has it, otherwise I + simply return None. + """ + if hasattr(self.original, "__conform__"): + return self.original.__conform__(interface) + return None + + def isuper(self, iface, adapter): + """ + Forward isuper to self.original + """ + return self.original.isuper(iface, adapter) + + +@_oldStyle +class Componentized: + """I am a mixin to allow you to be adapted in various ways persistently. + + I define a list of persistent adapters. This is to allow adapter classes + to store system-specific state, and initialized on demand. The + getComponent method implements this. You must also register adapters for + this class for the interfaces that you wish to pass to getComponent. + + Many other classes and utilities listed here are present in Zope3; this one + is specific to Twisted. + """ + + persistenceVersion = 1 + + def __init__(self): + self._adapterCache = {} + + def locateAdapterClass(self, klass, interfaceClass, default): + return getAdapterFactory(klass, interfaceClass, default) + + def setAdapter(self, interfaceClass, adapterClass): + """ + Cache a provider for the given interface, by adapting C{self} using + the given adapter class. + """ + self.setComponent(interfaceClass, adapterClass(self)) + + def addAdapter(self, adapterClass, ignoreClass=0): + """Utility method that calls addComponent. I take an adapter class and + instantiate it with myself as the first argument. + + @return: The adapter instantiated. + """ + adapt = adapterClass(self) + self.addComponent(adapt, ignoreClass) + return adapt + + def setComponent(self, interfaceClass, component): + """ + Cache a provider of the given interface. + """ + self._adapterCache[reflect.qual(interfaceClass)] = component + + def addComponent(self, component, ignoreClass=0): + """ + Add a component to me, for all appropriate interfaces. + + In order to determine which interfaces are appropriate, the component's + provided interfaces will be scanned. + + If the argument 'ignoreClass' is True, then all interfaces are + considered appropriate. + + Otherwise, an 'appropriate' interface is one for which its class has + been registered as an adapter for my class according to the rules of + getComponent. + """ + for iface in declarations.providedBy(component): + if (ignoreClass or + (self.locateAdapterClass(self.__class__, iface, None) + == component.__class__)): + self._adapterCache[reflect.qual(iface)] = component + + def unsetComponent(self, interfaceClass): + """Remove my component specified by the given interface class.""" + del self._adapterCache[reflect.qual(interfaceClass)] + + def removeComponent(self, component): + """ + Remove the given component from me entirely, for all interfaces for which + it has been registered. + + @return: a list of the interfaces that were removed. + """ + l = [] + for k, v in list(self._adapterCache.items()): + if v is component: + del self._adapterCache[k] + l.append(reflect.namedObject(k)) + return l + + def getComponent(self, interface, default=None): + """Create or retrieve an adapter for the given interface. + + If such an adapter has already been created, retrieve it from the cache + that this instance keeps of all its adapters. Adapters created through + this mechanism may safely store system-specific state. + + If you want to register an adapter that will be created through + getComponent, but you don't require (or don't want) your adapter to be + cached and kept alive for the lifetime of this Componentized object, + set the attribute 'temporaryAdapter' to True on your adapter class. + + If you want to automatically register an adapter for all appropriate + interfaces (with addComponent), set the attribute 'multiComponent' to + True on your adapter class. + """ + k = reflect.qual(interface) + if k in self._adapterCache: + return self._adapterCache[k] + else: + adapter = interface.__adapt__(self) + if adapter is not None and not ( + hasattr(adapter, "temporaryAdapter") and + adapter.temporaryAdapter): + self._adapterCache[k] = adapter + if (hasattr(adapter, "multiComponent") and + adapter.multiComponent): + self.addComponent(adapter) + if adapter is None: + return default + return adapter + + + def __conform__(self, interface): + return self.getComponent(interface) + + +class ReprableComponentized(Componentized): + def __init__(self): + Componentized.__init__(self) + + def __repr__(self): + from pprint import pprint + sio = NativeStringIO() + pprint(self._adapterCache, sio) + return sio.getvalue() + + + +def proxyForInterface(iface, originalAttribute='original'): + """ + Create a class which proxies all method calls which adhere to an interface + to another provider of that interface. + + This function is intended for creating specialized proxies. The typical way + to use it is by subclassing the result:: + + class MySpecializedProxy(proxyForInterface(IFoo)): + def someInterfaceMethod(self, arg): + if arg == 3: + return 3 + return self.original.someInterfaceMethod(arg) + + @param iface: The Interface to which the resulting object will conform, and + which the wrapped object must provide. + + @param originalAttribute: name of the attribute used to save the original + object in the resulting class. Default to C{original}. + @type originalAttribute: C{str} + + @return: A class whose constructor takes the original object as its only + argument. Constructing the class creates the proxy. + """ + def __init__(self, original): + setattr(self, originalAttribute, original) + contents = {"__init__": __init__} + for name in iface: + contents[name] = _ProxyDescriptor(name, originalAttribute) + proxy = type("(Proxy for %s)" + % (reflect.qual(iface),), (object,), contents) + declarations.classImplements(proxy, iface) + return proxy + + + +class _ProxiedClassMethod(object): + """ + A proxied class method. + + @ivar methodName: the name of the method which this should invoke when + called. + @type methodName: L{str} + + @ivar __name__: The name of the method being proxied (the same as + C{methodName}). + @type __name__: L{str} + + @ivar originalAttribute: name of the attribute of the proxy where the + original object is stored. + @type originalAttribute: L{str} + """ + def __init__(self, methodName, originalAttribute): + self.methodName = self.__name__ = methodName + self.originalAttribute = originalAttribute + + + def __call__(self, oself, *args, **kw): + """ + Invoke the specified L{methodName} method of the C{original} attribute + for proxyForInterface. + + @param oself: an instance of a L{proxyForInterface} object. + + @return: the result of the underlying method. + """ + original = getattr(oself, self.originalAttribute) + actualMethod = getattr(original, self.methodName) + return actualMethod(*args, **kw) + + + +class _ProxyDescriptor(object): + """ + A descriptor which will proxy attribute access, mutation, and + deletion to the L{_ProxyDescriptor.originalAttribute} of the + object it is being accessed from. + + @ivar attributeName: the name of the attribute which this descriptor will + retrieve from instances' C{original} attribute. + @type attributeName: C{str} + + @ivar originalAttribute: name of the attribute of the proxy where the + original object is stored. + @type originalAttribute: C{str} + """ + def __init__(self, attributeName, originalAttribute): + self.attributeName = attributeName + self.originalAttribute = originalAttribute + + + def __get__(self, oself, type=None): + """ + Retrieve the C{self.attributeName} property from I{oself}. + """ + if oself is None: + return _ProxiedClassMethod(self.attributeName, + self.originalAttribute) + original = getattr(oself, self.originalAttribute) + return getattr(original, self.attributeName) + + + def __set__(self, oself, value): + """ + Set the C{self.attributeName} property of I{oself}. + """ + original = getattr(oself, self.originalAttribute) + setattr(original, self.attributeName, value) + + + def __delete__(self, oself): + """ + Delete the C{self.attributeName} property of I{oself}. + """ + original = getattr(oself, self.originalAttribute) + delattr(original, self.attributeName) + + + +__all__ = [ + "registerAdapter", "getAdapterFactory", + "Adapter", "Componentized", "ReprableComponentized", "getRegistry", + "proxyForInterface", +] diff --git a/contrib/python/Twisted/py2/twisted/python/constants.py b/contrib/python/Twisted/py2/twisted/python/constants.py new file mode 100644 index 00000000000..9b55ee15023 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/constants.py @@ -0,0 +1,18 @@ +# -*- test-case-name: twisted.python.test.test_constants -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Symbolic constant support, including collections and constants with text, +numeric, and bit flag values. +""" + +from __future__ import division, absolute_import + +# Import and re-export Constantly +from constantly import (NamedConstant, ValueConstant, FlagConstant, Names, + Values, Flags) + +__all__ = [ + 'NamedConstant', 'ValueConstant', 'FlagConstant', + 'Names', 'Values', 'Flags'] diff --git a/contrib/python/Twisted/py2/twisted/python/context.py b/contrib/python/Twisted/py2/twisted/python/context.py new file mode 100644 index 00000000000..9eef4a49c55 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/context.py @@ -0,0 +1,137 @@ +# -*- test-case-name: twisted.test.test_context -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Dynamic pseudo-scoping for Python. + +Call functions with context.call({key: value}, func); func and +functions that it calls will be able to use 'context.get(key)' to +retrieve 'value'. + +This is thread-safe. +""" + +from __future__ import division, absolute_import + +from threading import local + +from twisted.python._oldstyle import _oldStyle + + +defaultContextDict = {} + +setDefault = defaultContextDict.__setitem__ + +@_oldStyle +class ContextTracker: + """ + A L{ContextTracker} provides a way to pass arbitrary key/value data up and + down a call stack without passing them as parameters to the functions on + that call stack. + + This can be useful when functions on the top and bottom of the call stack + need to cooperate but the functions in between them do not allow passing the + necessary state. For example:: + + from twisted.python.context import call, get + + def handleRequest(request): + call({'request-id': request.id}, renderRequest, request.url) + + def renderRequest(url): + renderHeader(url) + renderBody(url) + + def renderHeader(url): + return "the header" + + def renderBody(url): + return "the body (request id=%r)" % (get("request-id"),) + + This should be used sparingly, since the lack of a clear connection between + the two halves can result in code which is difficult to understand and + maintain. + + @ivar contexts: A C{list} of C{dict}s tracking the context state. Each new + L{ContextTracker.callWithContext} pushes a new C{dict} onto this stack + for the duration of the call, making the data available to the function + called and restoring the previous data once it is complete.. + """ + def __init__(self): + self.contexts = [defaultContextDict] + + + def callWithContext(self, newContext, func, *args, **kw): + """ + Call C{func(*args, **kw)} such that the contents of C{newContext} will + be available for it to retrieve using L{getContext}. + + @param newContext: A C{dict} of data to push onto the context for the + duration of the call to C{func}. + + @param func: A callable which will be called. + + @param *args: Any additional positional arguments to pass to C{func}. + + @param **kw: Any additional keyword arguments to pass to C{func}. + + @return: Whatever is returned by C{func} + + @raise: Whatever is raised by C{func}. + """ + self.contexts.append(newContext) + try: + return func(*args,**kw) + finally: + self.contexts.pop() + + + def getContext(self, key, default=None): + """ + Retrieve the value for a key from the context. + + @param key: The key to look up in the context. + + @param default: The value to return if C{key} is not found in the + context. + + @return: The value most recently remembered in the context for C{key}. + """ + for ctx in reversed(self.contexts): + try: + return ctx[key] + except KeyError: + pass + return default + + + +class ThreadedContextTracker(object): + def __init__(self): + self.storage = local() + + def currentContext(self): + try: + return self.storage.ct + except AttributeError: + ct = self.storage.ct = ContextTracker() + return ct + + def callWithContext(self, ctx, func, *args, **kw): + return self.currentContext().callWithContext(ctx, func, *args, **kw) + + def getContext(self, key, default=None): + return self.currentContext().getContext(key, default) + + +def installContextTracker(ctr): + global theContextTracker + global call + global get + + theContextTracker = ctr + call = theContextTracker.callWithContext + get = theContextTracker.getContext + +installContextTracker(ThreadedContextTracker()) diff --git a/contrib/python/Twisted/py2/twisted/python/deprecate.py b/contrib/python/Twisted/py2/twisted/python/deprecate.py new file mode 100644 index 00000000000..11ba2df1b7c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/deprecate.py @@ -0,0 +1,797 @@ +# -*- test-case-name: twisted.python.test.test_deprecate -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Deprecation framework for Twisted. + +To mark a method, function, or class as being deprecated do this:: + + from incremental import Version + from twisted.python.deprecate import deprecated + + @deprecated(Version("Twisted", 8, 0, 0)) + def badAPI(self, first, second): + ''' + Docstring for badAPI. + ''' + ... + + @deprecated(Version("Twisted", 16, 0, 0)) + class BadClass(object): + ''' + Docstring for BadClass. + ''' + +The newly-decorated badAPI will issue a warning when called, and BadClass will +issue a warning when instantiated. Both will also have a deprecation notice +appended to their docstring. + +To deprecate properties you can use:: + + from incremental import Version + from twisted.python.deprecate import deprecatedProperty + + class OtherwiseUndeprecatedClass(object): + + @deprecatedProperty(Version('Twisted', 16, 0, 0)) + def badProperty(self): + ''' + Docstring for badProperty. + ''' + + @badProperty.setter + def badProperty(self, value): + ''' + Setter sill also raise the deprecation warning. + ''' + + +To mark module-level attributes as being deprecated you can use:: + + badAttribute = "someValue" + + ... + + deprecatedModuleAttribute( + Version("Twisted", 8, 0, 0), + "Use goodAttribute instead.", + "your.full.module.name", + "badAttribute") + +The deprecated attributes will issue a warning whenever they are accessed. If +the attributes being deprecated are in the same module as the +L{deprecatedModuleAttribute} call is being made from, the C{__name__} global +can be used as the C{moduleName} parameter. + +See also L{incremental.Version}. + +@type DEPRECATION_WARNING_FORMAT: C{str} +@var DEPRECATION_WARNING_FORMAT: The default deprecation warning string format + to use when one is not provided by the user. +""" + +from __future__ import division, absolute_import + +__all__ = [ + 'deprecated', + 'deprecatedProperty', + 'getDeprecationWarningString', + 'getWarningMethod', + 'setWarningMethod', + 'deprecatedModuleAttribute', + ] + + +import sys, inspect +from warnings import warn, warn_explicit +from dis import findlinestarts +from functools import wraps + +from incremental import getVersionString +from twisted.python.compat import _PY3 + +DEPRECATION_WARNING_FORMAT = '%(fqpn)s was deprecated in %(version)s' + +# Notionally, part of twisted.python.reflect, but defining it there causes a +# cyclic dependency between this module and that module. Define it here, +# instead, and let reflect import it to re-expose to the public. +def _fullyQualifiedName(obj): + """ + Return the fully qualified name of a module, class, method or function. + Classes and functions need to be module level ones to be correctly + qualified. + + @rtype: C{str}. + """ + try: + name = obj.__qualname__ + except AttributeError: + name = obj.__name__ + + if inspect.isclass(obj) or inspect.isfunction(obj): + moduleName = obj.__module__ + return "%s.%s" % (moduleName, name) + elif inspect.ismethod(obj): + try: + cls = obj.im_class + except AttributeError: + # Python 3 eliminates im_class, substitutes __module__ and + # __qualname__ to provide similar information. + return "%s.%s" % (obj.__module__, obj.__qualname__) + else: + className = _fullyQualifiedName(cls) + return "%s.%s" % (className, name) + return name +# Try to keep it looking like something in twisted.python.reflect. +_fullyQualifiedName.__module__ = 'twisted.python.reflect' +_fullyQualifiedName.__name__ = 'fullyQualifiedName' +_fullyQualifiedName.__qualname__ = 'fullyQualifiedName' + + +def _getReplacementString(replacement): + """ + Surround a replacement for a deprecated API with some polite text exhorting + the user to consider it as an alternative. + + @type replacement: C{str} or callable + + @return: a string like "please use twisted.python.modules.getModule + instead". + """ + if callable(replacement): + replacement = _fullyQualifiedName(replacement) + return "please use %s instead" % (replacement,) + + + +def _getDeprecationDocstring(version, replacement=None): + """ + Generate an addition to a deprecated object's docstring that explains its + deprecation. + + @param version: the version it was deprecated. + @type version: L{incremental.Version} + + @param replacement: The replacement, if specified. + @type replacement: C{str} or callable + + @return: a string like "Deprecated in Twisted 27.2.0; please use + twisted.timestream.tachyon.flux instead." + """ + doc = "Deprecated in %s" % (getVersionString(version),) + if replacement: + doc = "%s; %s" % (doc, _getReplacementString(replacement)) + return doc + "." + + + +def _getDeprecationWarningString(fqpn, version, format=None, replacement=None): + """ + Return a string indicating that the Python name was deprecated in the given + version. + + @param fqpn: Fully qualified Python name of the thing being deprecated + @type fqpn: C{str} + + @param version: Version that C{fqpn} was deprecated in. + @type version: L{incremental.Version} + + @param format: A user-provided format to interpolate warning values into, or + L{DEPRECATION_WARNING_FORMAT + } if L{None} is + given. + @type format: C{str} + + @param replacement: what should be used in place of C{fqpn}. Either pass in + a string, which will be inserted into the warning message, or a + callable, which will be expanded to its full import path. + @type replacement: C{str} or callable + + @return: A textual description of the deprecation + @rtype: C{str} + """ + if format is None: + format = DEPRECATION_WARNING_FORMAT + warningString = format % { + 'fqpn': fqpn, + 'version': getVersionString(version)} + if replacement: + warningString = "%s; %s" % ( + warningString, _getReplacementString(replacement)) + return warningString + + + +def getDeprecationWarningString(callableThing, version, format=None, + replacement=None): + """ + Return a string indicating that the callable was deprecated in the given + version. + + @type callableThing: C{callable} + @param callableThing: Callable object to be deprecated + + @type version: L{incremental.Version} + @param version: Version that C{callableThing} was deprecated in + + @type format: C{str} + @param format: A user-provided format to interpolate warning values into, + or L{DEPRECATION_WARNING_FORMAT + } if L{None} is + given + + @param callableThing: A callable to be deprecated. + + @param version: The L{incremental.Version} that the callable + was deprecated in. + + @param replacement: what should be used in place of the callable. Either + pass in a string, which will be inserted into the warning message, + or a callable, which will be expanded to its full import path. + @type replacement: C{str} or callable + + @return: A string describing the deprecation. + @rtype: C{str} + """ + return _getDeprecationWarningString( + _fullyQualifiedName(callableThing), version, format, replacement) + + + +def _appendToDocstring(thingWithDoc, textToAppend): + """ + Append the given text to the docstring of C{thingWithDoc}. + + If C{thingWithDoc} has no docstring, then the text just replaces the + docstring. If it has a single-line docstring then it appends a blank line + and the message text. If it has a multi-line docstring, then in appends a + blank line a the message text, and also does the indentation correctly. + """ + if thingWithDoc.__doc__: + docstringLines = thingWithDoc.__doc__.splitlines() + else: + docstringLines = [] + + if len(docstringLines) == 0: + docstringLines.append(textToAppend) + elif len(docstringLines) == 1: + docstringLines.extend(['', textToAppend, '']) + else: + spaces = docstringLines.pop() + docstringLines.extend(['', + spaces + textToAppend, + spaces]) + thingWithDoc.__doc__ = '\n'.join(docstringLines) + + + +def deprecated(version, replacement=None): + """ + Return a decorator that marks callables as deprecated. To deprecate a + property, see L{deprecatedProperty}. + + @type version: L{incremental.Version} + @param version: The version in which the callable will be marked as + having been deprecated. The decorated function will be annotated + with this version, having it set as its C{deprecatedVersion} + attribute. + + @param version: the version that the callable was deprecated in. + @type version: L{incremental.Version} + + @param replacement: what should be used in place of the callable. Either + pass in a string, which will be inserted into the warning message, + or a callable, which will be expanded to its full import path. + @type replacement: C{str} or callable + """ + def deprecationDecorator(function): + """ + Decorator that marks C{function} as deprecated. + """ + warningString = getDeprecationWarningString( + function, version, None, replacement) + + @wraps(function) + def deprecatedFunction(*args, **kwargs): + warn( + warningString, + DeprecationWarning, + stacklevel=2) + return function(*args, **kwargs) + + _appendToDocstring(deprecatedFunction, + _getDeprecationDocstring(version, replacement)) + deprecatedFunction.deprecatedVersion = version + return deprecatedFunction + + return deprecationDecorator + + + +def deprecatedProperty(version, replacement=None): + """ + Return a decorator that marks a property as deprecated. To deprecate a + regular callable or class, see L{deprecated}. + + @type version: L{incremental.Version} + @param version: The version in which the callable will be marked as + having been deprecated. The decorated function will be annotated + with this version, having it set as its C{deprecatedVersion} + attribute. + + @param version: the version that the callable was deprecated in. + @type version: L{incremental.Version} + + @param replacement: what should be used in place of the callable. + Either pass in a string, which will be inserted into the warning + message, or a callable, which will be expanded to its full import + path. + @type replacement: C{str} or callable + + @return: A new property with deprecated setter and getter. + @rtype: C{property} + + @since: 16.1.0 + """ + + class _DeprecatedProperty(property): + """ + Extension of the build-in property to allow deprecated setters. + """ + + def _deprecatedWrapper(self, function): + @wraps(function) + def deprecatedFunction(*args, **kwargs): + warn( + self.warningString, + DeprecationWarning, + stacklevel=2) + return function(*args, **kwargs) + return deprecatedFunction + + + def setter(self, function): + return property.setter(self, self._deprecatedWrapper(function)) + + + def deprecationDecorator(function): + if _PY3: + warningString = getDeprecationWarningString( + function, version, None, replacement) + else: + # Because Python 2 sucks, we need to implement our own here -- lack + # of __qualname__ means that we kinda have to stack walk. It maybe + # probably works. Probably. -Amber + functionName = function.__name__ + className = inspect.stack()[1][3] # wow hax + moduleName = function.__module__ + + fqdn = "%s.%s.%s" % (moduleName, className, functionName) + + warningString = _getDeprecationWarningString( + fqdn, version, None, replacement) + + @wraps(function) + def deprecatedFunction(*args, **kwargs): + warn( + warningString, + DeprecationWarning, + stacklevel=2) + return function(*args, **kwargs) + + _appendToDocstring(deprecatedFunction, + _getDeprecationDocstring(version, replacement)) + deprecatedFunction.deprecatedVersion = version + + result = _DeprecatedProperty(deprecatedFunction) + result.warningString = warningString + return result + + return deprecationDecorator + + + +def getWarningMethod(): + """ + Return the warning method currently used to record deprecation warnings. + """ + return warn + + + +def setWarningMethod(newMethod): + """ + Set the warning method to use to record deprecation warnings. + + The callable should take message, category and stacklevel. The return + value is ignored. + """ + global warn + warn = newMethod + + + +class _InternalState(object): + """ + An L{_InternalState} is a helper object for a L{_ModuleProxy}, so that it + can easily access its own attributes, bypassing its logic for delegating to + another object that it's proxying for. + + @ivar proxy: a L{_ModuleProxy} + """ + def __init__(self, proxy): + object.__setattr__(self, 'proxy', proxy) + + + def __getattribute__(self, name): + return object.__getattribute__(object.__getattribute__(self, 'proxy'), + name) + + + def __setattr__(self, name, value): + return object.__setattr__(object.__getattribute__(self, 'proxy'), + name, value) + + + +class _ModuleProxy(object): + """ + Python module wrapper to hook module-level attribute access. + + Access to deprecated attributes first checks + L{_ModuleProxy._deprecatedAttributes}, if the attribute does not appear + there then access falls through to L{_ModuleProxy._module}, the wrapped + module object. + + @ivar _module: Module on which to hook attribute access. + @type _module: C{module} + + @ivar _deprecatedAttributes: Mapping of attribute names to objects that + retrieve the module attribute's original value. + @type _deprecatedAttributes: C{dict} mapping C{str} to + L{_DeprecatedAttribute} + + @ivar _lastWasPath: Heuristic guess as to whether warnings about this + package should be ignored for the next call. If the last attribute + access of this module was a C{getattr} of C{__path__}, we will assume + that it was the import system doing it and we won't emit a warning for + the next access, even if it is to a deprecated attribute. The CPython + import system always tries to access C{__path__}, then the attribute + itself, then the attribute itself again, in both successful and failed + cases. + @type _lastWasPath: C{bool} + """ + def __init__(self, module): + state = _InternalState(self) + state._module = module + state._deprecatedAttributes = {} + state._lastWasPath = False + + + def __repr__(self): + """ + Get a string containing the type of the module proxy and a + representation of the wrapped module object. + """ + state = _InternalState(self) + return '<%s module=%r>' % (type(self).__name__, state._module) + + + def __setattr__(self, name, value): + """ + Set an attribute on the wrapped module object. + """ + state = _InternalState(self) + state._lastWasPath = False + setattr(state._module, name, value) + + + def __getattribute__(self, name): + """ + Get an attribute from the module object, possibly emitting a warning. + + If the specified name has been deprecated, then a warning is issued. + (Unless certain obscure conditions are met; see + L{_ModuleProxy._lastWasPath} for more information about what might quash + such a warning.) + """ + state = _InternalState(self) + if state._lastWasPath: + deprecatedAttribute = None + else: + deprecatedAttribute = state._deprecatedAttributes.get(name) + + if deprecatedAttribute is not None: + # If we have a _DeprecatedAttribute object from the earlier lookup, + # allow it to issue the warning. + value = deprecatedAttribute.get() + else: + # Otherwise, just retrieve the underlying value directly; it's not + # deprecated, there's no warning to issue. + value = getattr(state._module, name) + if name == '__path__': + state._lastWasPath = True + else: + state._lastWasPath = False + return value + + + +class _DeprecatedAttribute(object): + """ + Wrapper for deprecated attributes. + + This is intended to be used by L{_ModuleProxy}. Calling + L{_DeprecatedAttribute.get} will issue a warning and retrieve the + underlying attribute's value. + + @type module: C{module} + @ivar module: The original module instance containing this attribute + + @type fqpn: C{str} + @ivar fqpn: Fully qualified Python name for the deprecated attribute + + @type version: L{incremental.Version} + @ivar version: Version that the attribute was deprecated in + + @type message: C{str} + @ivar message: Deprecation message + """ + def __init__(self, module, name, version, message): + """ + Initialise a deprecated name wrapper. + """ + self.module = module + self.__name__ = name + self.fqpn = module.__name__ + '.' + name + self.version = version + self.message = message + + + def get(self): + """ + Get the underlying attribute value and issue a deprecation warning. + """ + # This might fail if the deprecated thing is a module inside a package. + # In that case, don't emit the warning this time. The import system + # will come back again when it's not an AttributeError and we can emit + # the warning then. + result = getattr(self.module, self.__name__) + message = _getDeprecationWarningString(self.fqpn, self.version, + DEPRECATION_WARNING_FORMAT + ': ' + self.message) + warn(message, DeprecationWarning, stacklevel=3) + return result + + + +def _deprecateAttribute(proxy, name, version, message): + """ + Mark a module-level attribute as being deprecated. + + @type proxy: L{_ModuleProxy} + @param proxy: The module proxy instance proxying the deprecated attributes + + @type name: C{str} + @param name: Attribute name + + @type version: L{incremental.Version} + @param version: Version that the attribute was deprecated in + + @type message: C{str} + @param message: Deprecation message + """ + _module = object.__getattribute__(proxy, '_module') + attr = _DeprecatedAttribute(_module, name, version, message) + # Add a deprecated attribute marker for this module's attribute. When this + # attribute is accessed via _ModuleProxy a warning is emitted. + _deprecatedAttributes = object.__getattribute__( + proxy, '_deprecatedAttributes') + _deprecatedAttributes[name] = attr + + + +def deprecatedModuleAttribute(version, message, moduleName, name): + """ + Declare a module-level attribute as being deprecated. + + @type version: L{incremental.Version} + @param version: Version that the attribute was deprecated in + + @type message: C{str} + @param message: Deprecation message + + @type moduleName: C{str} + @param moduleName: Fully-qualified Python name of the module containing + the deprecated attribute; if called from the same module as the + attributes are being deprecated in, using the C{__name__} global can + be helpful + + @type name: C{str} + @param name: Attribute name to deprecate + """ + module = sys.modules[moduleName] + if not isinstance(module, _ModuleProxy): + module = _ModuleProxy(module) + sys.modules[moduleName] = module + + _deprecateAttribute(module, name, version, message) + + +def warnAboutFunction(offender, warningString): + """ + Issue a warning string, identifying C{offender} as the responsible code. + + This function is used to deprecate some behavior of a function. It differs + from L{warnings.warn} in that it is not limited to deprecating the behavior + of a function currently on the call stack. + + @param function: The function that is being deprecated. + + @param warningString: The string that should be emitted by this warning. + @type warningString: C{str} + + @since: 11.0 + """ + # inspect.getmodule() is attractive, but somewhat + # broken in Python < 2.6. See Python bug 4845. + offenderModule = sys.modules[offender.__module__] + filename = inspect.getabsfile(offenderModule) + lineStarts = list(findlinestarts(offender.__code__)) + lastLineNo = lineStarts[-1][1] + globals = offender.__globals__ + + kwargs = dict( + category=DeprecationWarning, + filename=filename, + lineno=lastLineNo, + module=offenderModule.__name__, + registry=globals.setdefault("__warningregistry__", {}), + module_globals=None) + + warn_explicit(warningString, **kwargs) + + + +def _passedArgSpec(argspec, positional, keyword): + """ + Take an I{inspect.ArgSpec}, a tuple of positional arguments, and a dict of + keyword arguments, and return a mapping of arguments that were actually + passed to their passed values. + + @param argspec: The argument specification for the function to inspect. + @type argspec: I{inspect.ArgSpec} + + @param positional: The positional arguments that were passed. + @type positional: L{tuple} + + @param keyword: The keyword arguments that were passed. + @type keyword: L{dict} + + @return: A dictionary mapping argument names (those declared in C{argspec}) + to values that were passed explicitly by the user. + @rtype: L{dict} mapping L{str} to L{object} + """ + result = {} + unpassed = len(argspec.args) - len(positional) + if argspec.keywords is not None: + kwargs = result[argspec.keywords] = {} + if unpassed < 0: + if argspec.varargs is None: + raise TypeError("Too many arguments.") + else: + result[argspec.varargs] = positional[len(argspec.args):] + for name, value in zip(argspec.args, positional): + result[name] = value + for name, value in keyword.items(): + if name in argspec.args: + if name in result: + raise TypeError("Already passed.") + result[name] = value + elif argspec.keywords is not None: + kwargs[name] = value + else: + raise TypeError("no such param") + return result + + + +def _passedSignature(signature, positional, keyword): + """ + Take an L{inspect.Signature}, a tuple of positional arguments, and a dict of + keyword arguments, and return a mapping of arguments that were actually + passed to their passed values. + + @param signature: The signature of the function to inspect. + @type signature: L{inspect.Signature} + + @param positional: The positional arguments that were passed. + @type positional: L{tuple} + + @param keyword: The keyword arguments that were passed. + @type keyword: L{dict} + + @return: A dictionary mapping argument names (those declared in + C{signature}) to values that were passed explicitly by the user. + @rtype: L{dict} mapping L{str} to L{object} + """ + result = {} + kwargs = None + numPositional = 0 + for (n, (name, param)) in enumerate(signature.parameters.items()): + if param.kind == inspect.Parameter.VAR_POSITIONAL: + # Varargs, for example: *args + result[name] = positional[n:] + numPositional = len(result[name]) + 1 + elif param.kind == inspect.Parameter.VAR_KEYWORD: + # Variable keyword args, for example: **my_kwargs + kwargs = result[name] = {} + elif param.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.POSITIONAL_ONLY): + if n < len(positional): + result[name] = positional[n] + numPositional += 1 + elif param.kind == inspect.Parameter.KEYWORD_ONLY: + if name not in keyword: + if param.default == inspect.Parameter.empty: + raise TypeError("missing keyword arg {}".format(name)) + else: + result[name] = param.default + else: + raise TypeError("'{}' parameter is invalid kind: {}".format( + name, param.kind)) + + if len(positional) > numPositional: + raise TypeError("Too many arguments.") + for name, value in keyword.items(): + if name in signature.parameters.keys(): + if name in result: + raise TypeError("Already passed.") + result[name] = value + elif kwargs is not None: + kwargs[name] = value + else: + raise TypeError("no such param") + return result + + + +def _mutuallyExclusiveArguments(argumentPairs): + """ + Decorator which causes its decoratee to raise a L{TypeError} if two of the + given arguments are passed at the same time. + + @param argumentPairs: pairs of argument identifiers, each pair indicating + an argument that may not be passed in conjunction with another. + @type argumentPairs: sequence of 2-sequences of L{str} + + @return: A decorator, used like so:: + + @_mutuallyExclusiveArguments([["tweedledum", "tweedledee"]]) + def function(tweedledum=1, tweedledee=2): + "Don't pass tweedledum and tweedledee at the same time." + + @rtype: 1-argument callable taking a callable and returning a callable. + """ + def wrapper(wrappee): + if getattr(inspect, "signature", None): + # Python 3 + spec = inspect.signature(wrappee) + _passed = _passedSignature + else: + # Python 2 + spec = inspect.getargspec(wrappee) + _passed = _passedArgSpec + + @wraps(wrappee) + def wrapped(*args, **kwargs): + arguments = _passed(spec, args, kwargs) + for this, that in argumentPairs: + if this in arguments and that in arguments: + raise TypeError( + ("The %r and %r arguments to %s " + "are mutually exclusive.") % + (this, that, _fullyQualifiedName(wrappee))) + return wrappee(*args, **kwargs) + return wrapped + return wrapper diff --git a/contrib/python/Twisted/py2/twisted/python/failure.py b/contrib/python/Twisted/py2/twisted/python/failure.py new file mode 100644 index 00000000000..c4ad6e16276 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/failure.py @@ -0,0 +1,798 @@ +# -*- test-case-name: twisted.test.test_failure -*- +# See also test suite twisted.test.test_pbfailure + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Asynchronous-friendly error mechanism. + +See L{Failure}. +""" + +from __future__ import division, absolute_import, print_function + +# System Imports +import copy +import sys +import linecache +import inspect +import opcode +from inspect import getmro + +from twisted.python import reflect +from twisted.python.compat import _PY3, NativeStringIO as StringIO + +count = 0 +traceupLength = 4 + +class DefaultException(Exception): + pass + + + +def format_frames(frames, write, detail="default"): + """ + Format and write frames. + + @param frames: is a list of frames as used by Failure.frames, with + each frame being a list of + (funcName, fileName, lineNumber, locals.items(), globals.items()) + @type frames: list + @param write: this will be called with formatted strings. + @type write: callable + @param detail: Four detail levels are available: + default, brief, verbose, and verbose-vars-not-captured. + C{Failure.printDetailedTraceback} uses the latter when the caller asks + for verbose, but no vars were captured, so that an explicit warning + about the missing data is shown. + @type detail: string + """ + if detail not in ('default', 'brief', 'verbose', + 'verbose-vars-not-captured'): + raise ValueError( + "Detail must be default, brief, verbose, or " + "verbose-vars-not-captured. (not %r)" % (detail,)) + w = write + if detail == "brief": + for method, filename, lineno, localVars, globalVars in frames: + w('%s:%s:%s\n' % (filename, lineno, method)) + elif detail == "default": + for method, filename, lineno, localVars, globalVars in frames: + w(' File "%s", line %s, in %s\n' % (filename, lineno, method)) + w(' %s\n' % linecache.getline(filename, lineno).strip()) + elif detail == "verbose-vars-not-captured": + for method, filename, lineno, localVars, globalVars in frames: + w("%s:%d: %s(...)\n" % (filename, lineno, method)) + w(' [Capture of Locals and Globals disabled (use captureVars=True)]\n') + elif detail == "verbose": + for method, filename, lineno, localVars, globalVars in frames: + w("%s:%d: %s(...)\n" % (filename, lineno, method)) + w(' [ Locals ]\n') + # Note: the repr(val) was (self.pickled and val) or repr(val))) + for name, val in localVars: + w(" %s : %s\n" % (name, repr(val))) + w(' ( Globals )\n') + for name, val in globalVars: + w(" %s : %s\n" % (name, repr(val))) + +# slyphon: i have a need to check for this value in trial +# so I made it a module-level constant +EXCEPTION_CAUGHT_HERE = "--- ---" + + + +class NoCurrentExceptionError(Exception): + """ + Raised when trying to create a Failure from the current interpreter + exception state and there is no current exception state. + """ + + + +def _Traceback(stackFrames, tbFrames): + """ + Construct a fake traceback object using a list of frames. Note that + although frames generally include locals and globals, this information + is not kept by this method, since locals and globals are not used in + standard tracebacks. + + @param stackFrames: [(methodname, filename, lineno, locals, globals), ...] + @param tbFrames: [(methodname, filename, lineno, locals, globals), ...] + """ + assert len(tbFrames) > 0, "Must pass some frames" + # We deliberately avoid using recursion here, as the frames list may be + # long. + + # 'stackFrames' is a list of frames above (ie, older than) the point the + # exception was caught, with oldest at the start. Start by building these + # into a linked list of _Frame objects (with the f_back links pointing back + # towards the oldest frame). + stack = None + for sf in stackFrames: + stack = _Frame(sf, stack) + + # 'tbFrames' is a list of frames from the point the exception was caught, + # down to where it was thrown, with the oldest at the start. Add these to + # the linked list of _Frames, but also wrap each one with a _Traceback + # frame which is linked in the opposite direction (towards the newest + # frame). + stack = _Frame(tbFrames[0], stack) + firstTb = tb = _TracebackFrame(stack) + for sf in tbFrames[1:]: + stack = _Frame(sf, stack) + tb.tb_next = _TracebackFrame(stack) + tb = tb.tb_next + + # Return the first _TracebackFrame. + return firstTb + + + +class _TracebackFrame(object): + """ + Fake traceback object which can be passed to functions in the standard + library L{traceback} module. + """ + + def __init__(self, frame): + """ + @param frame: _Frame object + """ + self.tb_frame = frame + self.tb_lineno = frame.f_lineno + self.tb_next = None + + + +class _Frame(object): + """ + A fake frame object, used by L{_Traceback}. + + @ivar f_code: fake L{code} object + @ivar f_lineno: line number + @ivar f_globals: fake f_globals dictionary (usually empty) + @ivar f_locals: fake f_locals dictionary (usually empty) + @ivar f_back: previous stack frame (towards the caller) + """ + + def __init__(self, frameinfo, back): + """ + @param frameinfo: (methodname, filename, lineno, locals, globals) + @param back: previous (older) stack frame + @type back: C{frame} + """ + name, filename, lineno, localz, globalz = frameinfo + self.f_code = _Code(name, filename) + self.f_lineno = lineno + self.f_globals = {} + self.f_locals = {} + self.f_back = back + + + +class _Code(object): + """ + A fake code object, used by L{_Traceback} via L{_Frame}. + """ + def __init__(self, name, filename): + self.co_name = name + self.co_filename = filename + + + +_inlineCallbacksExtraneous = [] + +def _extraneous(f): + """ + Mark the given callable as extraneous to inlineCallbacks exception + reporting; don't show these functions. + + @param f: a function that you NEVER WANT TO SEE AGAIN in ANY TRACEBACK + reported by Failure. + + @type f: function + + @return: f + """ + _inlineCallbacksExtraneous.append(f.__code__) + return f + + + +class Failure(BaseException): + """ + A basic abstraction for an error that has occurred. + + This is necessary because Python's built-in error mechanisms are + inconvenient for asynchronous communication. + + The C{stack} and C{frame} attributes contain frames. Each frame is a tuple + of (funcName, fileName, lineNumber, localsItems, globalsItems), where + localsItems and globalsItems are the contents of + C{locals().items()}/C{globals().items()} for that frame, or an empty tuple + if those details were not captured. + + @ivar value: The exception instance responsible for this failure. + @ivar type: The exception's class. + @ivar stack: list of frames, innermost last, excluding C{Failure.__init__}. + @ivar frames: list of frames, innermost first. + """ + + pickled = 0 + stack = None + + # The opcode of "yield" in Python bytecode. We need this in + # _findFailure in order to identify whether an exception was + # thrown by a throwExceptionIntoGenerator. + # on PY3, b'a'[0] == 97 while in py2 b'a'[0] == b'a' opcodes + # are stored in bytes so we need to properly account for this + # difference. + if _PY3: + _yieldOpcode = opcode.opmap["YIELD_VALUE"] + else: + _yieldOpcode = chr(opcode.opmap["YIELD_VALUE"]) + + + def __init__(self, exc_value=None, exc_type=None, exc_tb=None, + captureVars=False): + """ + Initialize me with an explanation of the error. + + By default, this will use the current C{exception} + (L{sys.exc_info}()). However, if you want to specify a + particular kind of failure, you can pass an exception as an + argument. + + If no C{exc_value} is passed, then an "original" C{Failure} will + be searched for. If the current exception handler that this + C{Failure} is being constructed in is handling an exception + raised by L{raiseException}, then this C{Failure} will act like + the original C{Failure}. + + For C{exc_tb} only L{traceback} instances or L{None} are allowed. + If L{None} is supplied for C{exc_value}, the value of C{exc_tb} is + ignored, otherwise if C{exc_tb} is L{None}, it will be found from + execution context (ie, L{sys.exc_info}). + + @param captureVars: if set, capture locals and globals of stack + frames. This is pretty slow, and makes no difference unless you + are going to use L{printDetailedTraceback}. + """ + global count + count = count + 1 + self.count = count + self.type = self.value = tb = None + self.captureVars = captureVars + + if isinstance(exc_value, str) and exc_type is None: + raise TypeError("Strings are not supported by Failure") + + stackOffset = 0 + + if exc_value is None: + exc_value = self._findFailure() + + if exc_value is None: + self.type, self.value, tb = sys.exc_info() + if self.type is None: + raise NoCurrentExceptionError() + stackOffset = 1 + elif exc_type is None: + if isinstance(exc_value, Exception): + self.type = exc_value.__class__ + else: + # Allow arbitrary objects. + self.type = type(exc_value) + self.value = exc_value + else: + self.type = exc_type + self.value = exc_value + + if isinstance(self.value, Failure): + self._extrapolate(self.value) + return + + if hasattr(self.value, "__failure__"): + + # For exceptions propagated through coroutine-awaiting (see + # Deferred.send, AKA Deferred.__next__), which can't be raised as + # Failure because that would mess up the ability to except: them: + self._extrapolate(self.value.__failure__) + + # Clean up the inherently circular reference established by storing + # the failure there. This should make the common case of a Twisted + # / Deferred-returning coroutine somewhat less hard on the garbage + # collector. + del self.value.__failure__ + return + + if tb is None: + if exc_tb: + tb = exc_tb + elif getattr(self.value, "__traceback__", None): + # Python 3 + tb = self.value.__traceback__ + + frames = self.frames = [] + stack = self.stack = [] + + # Added 2003-06-23 by Chris Armstrong. Yes, I actually have a + # use case where I need this traceback object, and I've made + # sure that it'll be cleaned up. + self.tb = tb + + if tb: + f = tb.tb_frame + elif not isinstance(self.value, Failure): + # We don't do frame introspection since it's expensive, + # and if we were passed a plain exception with no + # traceback, it's not useful anyway + f = stackOffset = None + + while stackOffset and f: + # This excludes this Failure.__init__ frame from the + # stack, leaving it to start with our caller instead. + f = f.f_back + stackOffset -= 1 + + # Keeps the *full* stack. Formerly in spread.pb.print_excFullStack: + # + # The need for this function arises from the fact that several + # PB classes have the peculiar habit of discarding exceptions + # with bareword "except:"s. This premature exception + # catching means tracebacks generated here don't tend to show + # what called upon the PB object. + + while f: + if captureVars: + localz = f.f_locals.copy() + if f.f_locals is f.f_globals: + globalz = {} + else: + globalz = f.f_globals.copy() + for d in globalz, localz: + if "__builtins__" in d: + del d["__builtins__"] + localz = localz.items() + globalz = globalz.items() + else: + localz = globalz = () + stack.insert(0, ( + f.f_code.co_name, + f.f_code.co_filename, + f.f_lineno, + localz, + globalz, + )) + f = f.f_back + + while tb is not None: + f = tb.tb_frame + if captureVars: + localz = f.f_locals.copy() + if f.f_locals is f.f_globals: + globalz = {} + else: + globalz = f.f_globals.copy() + for d in globalz, localz: + if "__builtins__" in d: + del d["__builtins__"] + localz = list(localz.items()) + globalz = list(globalz.items()) + else: + localz = globalz = () + frames.append(( + f.f_code.co_name, + f.f_code.co_filename, + tb.tb_lineno, + localz, + globalz, + )) + tb = tb.tb_next + if inspect.isclass(self.type) and issubclass(self.type, Exception): + parentCs = getmro(self.type) + self.parents = list(map(reflect.qual, parentCs)) + else: + self.parents = [self.type] + + + def _extrapolate(self, otherFailure): + """ + Extrapolate from one failure into another, copying its stack frames. + + @param otherFailure: Another L{Failure}, whose traceback information, + if any, should be preserved as part of the stack presented by this + one. + @type otherFailure: L{Failure} + """ + # Copy all infos from that failure (including self.frames). + self.__dict__ = copy.copy(otherFailure.__dict__) + + # If we are re-throwing a Failure, we merge the stack-trace stored in + # the failure with the current exception's stack. This integrated with + # throwExceptionIntoGenerator and allows to provide full stack trace, + # even if we go through several layers of inlineCallbacks. + _, _, tb = sys.exc_info() + frames = [] + while tb is not None: + f = tb.tb_frame + if f.f_code not in _inlineCallbacksExtraneous: + frames.append(( + f.f_code.co_name, + f.f_code.co_filename, + tb.tb_lineno, (), () + )) + tb = tb.tb_next + # Merging current stack with stack stored in the Failure. + frames.extend(self.frames) + self.frames = frames + + + def trap(self, *errorTypes): + """ + Trap this failure if its type is in a predetermined list. + + This allows you to trap a Failure in an error callback. It will be + automatically re-raised if it is not a type that you expect. + + The reason for having this particular API is because it's very useful + in Deferred errback chains:: + + def _ebFoo(self, failure): + r = failure.trap(Spam, Eggs) + print('The Failure is due to either Spam or Eggs!') + if r == Spam: + print('Spam did it!') + elif r == Eggs: + print('Eggs did it!') + + If the failure is not a Spam or an Eggs, then the Failure will be + 'passed on' to the next errback. In Python 2 the Failure will be + raised; in Python 3 the underlying exception will be re-raised. + + @type errorTypes: L{Exception} + """ + error = self.check(*errorTypes) + if not error: + if _PY3: + self.raiseException() + else: + raise self + return error + + + def check(self, *errorTypes): + """ + Check if this failure's type is in a predetermined list. + + @type errorTypes: list of L{Exception} classes or + fully-qualified class names. + @returns: the matching L{Exception} type, or None if no match. + """ + for error in errorTypes: + err = error + if inspect.isclass(error) and issubclass(error, Exception): + err = reflect.qual(error) + if err in self.parents: + return error + return None + + # It would be nice to use twisted.python.compat.reraise, but that breaks + # the stack exploration in _findFailure; possibly this can be fixed in + # #5931. + if getattr(BaseException, "with_traceback", None): + # Python 3 + def raiseException(self): + raise self.value.with_traceback(self.tb) + else: + exec("""def raiseException(self): + raise self.type, self.value, self.tb""") + + raiseException.__doc__ = ( + """ + raise the original exception, preserving traceback + information if available. + """) + + + @_extraneous + def throwExceptionIntoGenerator(self, g): + """ + Throw the original exception into the given generator, + preserving traceback information if available. + + @return: The next value yielded from the generator. + @raise StopIteration: If there are no more values in the generator. + @raise anything else: Anything that the generator raises. + """ + # Note that the actual magic to find the traceback information + # is done in _findFailure. + return g.throw(self.type, self.value, self.tb) + + + def _findFailure(cls): + """ + Find the failure that represents the exception currently in context. + """ + tb = sys.exc_info()[-1] + if not tb: + return + + secondLastTb = None + lastTb = tb + while lastTb.tb_next: + secondLastTb = lastTb + lastTb = lastTb.tb_next + + lastFrame = lastTb.tb_frame + + # NOTE: f_locals.get('self') is used rather than + # f_locals['self'] because psyco frames do not contain + # anything in their locals() dicts. psyco makes debugging + # difficult anyhow, so losing the Failure objects (and thus + # the tracebacks) here when it is used is not that big a deal. + + # Handle raiseException-originated exceptions + if lastFrame.f_code is cls.raiseException.__code__: + return lastFrame.f_locals.get('self') + + # Handle throwExceptionIntoGenerator-originated exceptions + # this is tricky, and differs if the exception was caught + # inside the generator, or above it: + + # It is only really originating from + # throwExceptionIntoGenerator if the bottom of the traceback + # is a yield. + # Pyrex and Cython extensions create traceback frames + # with no co_code, but they can't yield so we know it's okay to + # just return here. + if ((not lastFrame.f_code.co_code) or + lastFrame.f_code.co_code[lastTb.tb_lasti] != cls._yieldOpcode): + return + + # If the exception was caught above the generator.throw + # (outside the generator), it will appear in the tb (as the + # second last item): + if secondLastTb: + frame = secondLastTb.tb_frame + if frame.f_code is cls.throwExceptionIntoGenerator.__code__: + return frame.f_locals.get('self') + + # If the exception was caught below the generator.throw + # (inside the generator), it will appear in the frames' linked + # list, above the top-level traceback item (which must be the + # generator frame itself, thus its caller is + # throwExceptionIntoGenerator). + frame = tb.tb_frame.f_back + if frame and frame.f_code is cls.throwExceptionIntoGenerator.__code__: + return frame.f_locals.get('self') + + _findFailure = classmethod(_findFailure) + + def __repr__(self): + return "<%s %s: %s>" % (reflect.qual(self.__class__), + reflect.qual(self.type), + self.getErrorMessage()) + + + def __str__(self): + return "[Failure instance: %s]" % self.getBriefTraceback() + + + def __getstate__(self): + """Avoid pickling objects in the traceback. + """ + if self.pickled: + return self.__dict__ + c = self.__dict__.copy() + + c['frames'] = [ + [ + v[0], v[1], v[2], + _safeReprVars(v[3]), + _safeReprVars(v[4]), + ] for v in self.frames + ] + + # Added 2003-06-23. See comment above in __init__ + c['tb'] = None + + if self.stack is not None: + # XXX: This is a band-aid. I can't figure out where these + # (failure.stack is None) instances are coming from. + c['stack'] = [ + [ + v[0], v[1], v[2], + _safeReprVars(v[3]), + _safeReprVars(v[4]), + ] for v in self.stack + ] + + c['pickled'] = 1 + return c + + + def cleanFailure(self): + """ + Remove references to other objects, replacing them with strings. + + On Python 3, this will also set the C{__traceback__} attribute of the + exception instance to L{None}. + """ + self.__dict__ = self.__getstate__() + if getattr(self.value, "__traceback__", None): + # Python 3 + self.value.__traceback__ = None + + + def getTracebackObject(self): + """ + Get an object that represents this Failure's stack that can be passed + to traceback.extract_tb. + + If the original traceback object is still present, return that. If this + traceback object has been lost but we still have the information, + return a fake traceback object (see L{_Traceback}). If there is no + traceback information at all, return None. + """ + if self.tb is not None: + return self.tb + elif len(self.frames) > 0: + return _Traceback(self.stack, self.frames) + else: + return None + + + def getErrorMessage(self): + """ + Get a string of the exception which caused this Failure. + """ + if isinstance(self.value, Failure): + return self.value.getErrorMessage() + return reflect.safe_str(self.value) + + + def getBriefTraceback(self): + io = StringIO() + self.printBriefTraceback(file=io) + return io.getvalue() + + + def getTraceback(self, elideFrameworkCode=0, detail='default'): + io = StringIO() + self.printTraceback(file=io, elideFrameworkCode=elideFrameworkCode, + detail=detail) + return io.getvalue() + + + def printTraceback(self, file=None, elideFrameworkCode=False, + detail='default'): + """ + Emulate Python's standard error reporting mechanism. + + @param file: If specified, a file-like object to which to write the + traceback. + + @param elideFrameworkCode: A flag indicating whether to attempt to + remove uninteresting frames from within Twisted itself from the + output. + + @param detail: A string indicating how much information to include + in the traceback. Must be one of C{'brief'}, C{'default'}, or + C{'verbose'}. + """ + if file is None: + from twisted.python import log + file = log.logerr + w = file.write + + if detail == 'verbose' and not self.captureVars: + # We don't have any locals or globals, so rather than show them as + # empty make the output explicitly say that we don't have them at + # all. + formatDetail = 'verbose-vars-not-captured' + else: + formatDetail = detail + + # Preamble + if detail == 'verbose': + w('*--- Failure #%d%s---\n' % + (self.count, + (self.pickled and ' (pickled) ') or ' ')) + elif detail == 'brief': + if self.frames: + hasFrames = 'Traceback' + else: + hasFrames = 'Traceback (failure with no frames)' + w("%s: %s: %s\n" % ( + hasFrames, + reflect.safe_str(self.type), + reflect.safe_str(self.value))) + else: + w('Traceback (most recent call last):\n') + + # Frames, formatted in appropriate style + if self.frames: + if not elideFrameworkCode: + format_frames(self.stack[-traceupLength:], w, formatDetail) + w("%s\n" % (EXCEPTION_CAUGHT_HERE,)) + format_frames(self.frames, w, formatDetail) + elif not detail == 'brief': + # Yeah, it's not really a traceback, despite looking like one... + w("Failure: ") + + # Postamble, if any + if not detail == 'brief': + w("%s: %s\n" % (reflect.qual(self.type), + reflect.safe_str(self.value))) + + # Chaining + if isinstance(self.value, Failure): + # TODO: indentation for chained failures? + file.write(" (chained Failure)\n") + self.value.printTraceback(file, elideFrameworkCode, detail) + if detail == 'verbose': + w('*--- End of Failure #%d ---\n' % self.count) + + + def printBriefTraceback(self, file=None, elideFrameworkCode=0): + """ + Print a traceback as densely as possible. + """ + self.printTraceback(file, elideFrameworkCode, detail='brief') + + + def printDetailedTraceback(self, file=None, elideFrameworkCode=0): + """ + Print a traceback with detailed locals and globals information. + """ + self.printTraceback(file, elideFrameworkCode, detail='verbose') + + + +def _safeReprVars(varsDictItems): + """ + Convert a list of (name, object) pairs into (name, repr) pairs. + + L{twisted.python.reflect.safe_repr} is used to generate the repr, so no + exceptions will be raised by faulty C{__repr__} methods. + + @param varsDictItems: a sequence of (name, value) pairs as returned by e.g. + C{locals().items()}. + @returns: a sequence of (name, repr) pairs. + """ + return [(name, reflect.safe_repr(obj)) for (name, obj) in varsDictItems] + + +# slyphon: make post-morteming exceptions tweakable + +DO_POST_MORTEM = True + +def _debuginit(self, exc_value=None, exc_type=None, exc_tb=None, + captureVars=False, + Failure__init__=Failure.__init__): + """ + Initialize failure object, possibly spawning pdb. + """ + if (exc_value, exc_type, exc_tb) == (None, None, None): + exc = sys.exc_info() + if not exc[0] == self.__class__ and DO_POST_MORTEM: + try: + strrepr = str(exc[1]) + except: + strrepr = "broken str" + print("Jumping into debugger for post-mortem of exception '%s':" % + (strrepr,)) + import pdb + pdb.post_mortem(exc[2]) + Failure__init__(self, exc_value, exc_type, exc_tb, captureVars) + + + +def startDebugMode(): + """ + Enable debug hooks for Failures. + """ + Failure.__init__ = _debuginit diff --git a/contrib/python/Twisted/py2/twisted/python/fakepwd.py b/contrib/python/Twisted/py2/twisted/python/fakepwd.py new file mode 100644 index 00000000000..389286bc5c4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/fakepwd.py @@ -0,0 +1,220 @@ +# -*- test-case-name: twisted.python.test.test_fakepwd -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +L{twisted.python.fakepwd} provides a fake implementation of the L{pwd} API. +""" + +from __future__ import absolute_import, division + +__all__ = ['UserDatabase', 'ShadowDatabase'] + + +class _UserRecord(object): + """ + L{_UserRecord} holds the user data for a single user in L{UserDatabase}. + It corresponds to L{pwd.struct_passwd}. See that class for attribute + documentation. + """ + def __init__(self, name, password, uid, gid, gecos, home, shell): + self.pw_name = name + self.pw_passwd = password + self.pw_uid = uid + self.pw_gid = gid + self.pw_gecos = gecos + self.pw_dir = home + self.pw_shell = shell + + + def __len__(self): + return 7 + + + def __getitem__(self, index): + return ( + self.pw_name, self.pw_passwd, self.pw_uid, + self.pw_gid, self.pw_gecos, self.pw_dir, self.pw_shell)[index] + + + +class UserDatabase(object): + """ + L{UserDatabase} holds a traditional POSIX user data in memory and makes it + available via the same API as L{pwd}. + + @ivar _users: A C{list} of L{_UserRecord} instances holding all user data + added to this database. + """ + def __init__(self): + self._users = [] + + + def addUser(self, username, password, uid, gid, gecos, home, shell): + """ + Add a new user record to this database. + + @param username: The value for the C{pw_name} field of the user + record to add. + @type username: C{str} + + @param password: The value for the C{pw_passwd} field of the user + record to add. + @type password: C{str} + + @param uid: The value for the C{pw_uid} field of the user record to + add. + @type uid: C{int} + + @param gid: The value for the C{pw_gid} field of the user record to + add. + @type gid: C{int} + + @param gecos: The value for the C{pw_gecos} field of the user record + to add. + @type gecos: C{str} + + @param home: The value for the C{pw_dir} field of the user record to + add. + @type home: C{str} + + @param shell: The value for the C{pw_shell} field of the user record to + add. + @type shell: C{str} + """ + self._users.append(_UserRecord( + username, password, uid, gid, gecos, home, shell)) + + + def getpwuid(self, uid): + """ + Return the user record corresponding to the given uid. + """ + for entry in self._users: + if entry.pw_uid == uid: + return entry + raise KeyError() + + + def getpwnam(self, name): + """ + Return the user record corresponding to the given username. + """ + for entry in self._users: + if entry.pw_name == name: + return entry + raise KeyError() + + + def getpwall(self): + """ + Return a list of all user records. + """ + return self._users + + + +class _ShadowRecord(object): + """ + L{_ShadowRecord} holds the shadow user data for a single user in + L{ShadowDatabase}. It corresponds to C{spwd.struct_spwd}. See that class + for attribute documentation. + """ + def __init__(self, username, password, lastChange, min, max, warn, inact, + expire, flag): + self.sp_nam = username + self.sp_pwd = password + self.sp_lstchg = lastChange + self.sp_min = min + self.sp_max = max + self.sp_warn = warn + self.sp_inact = inact + self.sp_expire = expire + self.sp_flag = flag + + + def __len__(self): + return 9 + + + def __getitem__(self, index): + return ( + self.sp_nam, self.sp_pwd, self.sp_lstchg, self.sp_min, + self.sp_max, self.sp_warn, self.sp_inact, self.sp_expire, + self.sp_flag)[index] + + + +class ShadowDatabase(object): + """ + L{ShadowDatabase} holds a shadow user database in memory and makes it + available via the same API as C{spwd}. + + @ivar _users: A C{list} of L{_ShadowRecord} instances holding all user data + added to this database. + + @since: 12.0 + """ + def __init__(self): + self._users = [] + + + def addUser(self, username, password, lastChange, min, max, warn, inact, + expire, flag): + """ + Add a new user record to this database. + + @param username: The value for the C{sp_nam} field of the user record to + add. + @type username: C{str} + + @param password: The value for the C{sp_pwd} field of the user record to + add. + @type password: C{str} + + @param lastChange: The value for the C{sp_lstchg} field of the user + record to add. + @type lastChange: C{int} + + @param min: The value for the C{sp_min} field of the user record to add. + @type min: C{int} + + @param max: The value for the C{sp_max} field of the user record to add. + @type max: C{int} + + @param warn: The value for the C{sp_warn} field of the user record to + add. + @type warn: C{int} + + @param inact: The value for the C{sp_inact} field of the user record to + add. + @type inact: C{int} + + @param expire: The value for the C{sp_expire} field of the user record + to add. + @type expire: C{int} + + @param flag: The value for the C{sp_flag} field of the user record to + add. + @type flag: C{int} + """ + self._users.append(_ShadowRecord( + username, password, lastChange, + min, max, warn, inact, expire, flag)) + + + def getspnam(self, username): + """ + Return the shadow user record corresponding to the given username. + """ + for entry in self._users: + if entry.sp_nam == username: + return entry + raise KeyError + + + def getspall(self): + """ + Return a list of all shadow user records. + """ + return self._users diff --git a/contrib/python/Twisted/py2/twisted/python/filepath.py b/contrib/python/Twisted/py2/twisted/python/filepath.py new file mode 100644 index 00000000000..1e5e640ee2b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/filepath.py @@ -0,0 +1,1766 @@ +# -*- test-case-name: twisted.test.test_paths -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Object-oriented filesystem path representation. +""" + +from __future__ import division, absolute_import + +import os +import sys +import errno +import base64 + +from os.path import isabs, exists, normpath, abspath, splitext +from os.path import basename, dirname, join as joinpath +from os import listdir, utime, stat + +from stat import S_ISREG, S_ISDIR, S_IMODE, S_ISBLK, S_ISSOCK +from stat import S_IRUSR, S_IWUSR, S_IXUSR +from stat import S_IRGRP, S_IWGRP, S_IXGRP +from stat import S_IROTH, S_IWOTH, S_IXOTH + +from zope.interface import Interface, Attribute, implementer + +# Please keep this as light as possible on other Twisted imports; many, many +# things import this module, and it would be good if it could easily be +# modified for inclusion in the standard library. --glyph + +from twisted.python.compat import comparable, cmp, unicode +from twisted.python.deprecate import deprecated +from twisted.python.runtime import platform +from incremental import Version + +from twisted.python.win32 import ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND +from twisted.python.win32 import ERROR_INVALID_NAME, ERROR_DIRECTORY, O_BINARY +from twisted.python.win32 import WindowsError + +from twisted.python.util import FancyEqMixin + +_CREATE_FLAGS = (os.O_EXCL | + os.O_CREAT | + os.O_RDWR | + O_BINARY) + + + +def _stub_islink(path): + """ + Always return C{False} if the operating system does not support symlinks. + + @param path: A path string. + @type path: L{str} + + @return: C{False} + @rtype: L{bool} + """ + return False + + +islink = getattr(os.path, 'islink', _stub_islink) +randomBytes = os.urandom +armor = base64.urlsafe_b64encode + + + +class IFilePath(Interface): + """ + File path object. + + A file path represents a location for a file-like-object and can be + organized into a hierarchy; a file path can can children which are + themselves file paths. + + A file path has a name which unique identifies it in the context of its + parent (if it has one); a file path can not have two children with the same + name. This name is referred to as the file path's "base name". + + A series of such names can be used to locate nested children of a file + path; such a series is referred to as the child's "path", relative to the + parent. In this case, each name in the path is referred to as a "path + segment"; the child's base name is the segment in the path. + + When representing a file path as a string, a "path separator" is used to + delimit the path segments within the string. For a file system path, that + would be C{os.sep}. + + Note that the values of child names may be restricted. For example, a file + system path will not allow the use of the path separator in a name, and + certain names (e.g. C{"."} and C{".."}) may be reserved or have special + meanings. + + @since: 12.1 + """ + sep = Attribute("The path separator to use in string representations") + + def child(name): + """ + Obtain a direct child of this file path. The child may or may not + exist. + + @param name: the name of a child of this path. C{name} must be a direct + child of this path and may not contain a path separator. + @return: the child of this path with the given C{name}. + @raise InsecurePath: if C{name} describes a file path that is not a + direct child of this file path. + """ + + def open(mode="r"): + """ + Opens this file path with the given mode. + + @return: a file-like object. + @raise Exception: if this file path cannot be opened. + """ + + def changed(): + """ + Clear any cached information about the state of this path on disk. + """ + + def getsize(): + """ + Retrieve the size of this file in bytes. + + @return: the size of the file at this file path in bytes. + @raise Exception: if the size cannot be obtained. + """ + + def getModificationTime(): + """ + Retrieve the time of last access from this file. + + @return: a number of seconds from the epoch. + @rtype: L{float} + """ + + def getStatusChangeTime(): + """ + Retrieve the time of the last status change for this file. + + @return: a number of seconds from the epoch. + @rtype: L{float} + """ + + def getAccessTime(): + """ + Retrieve the time that this file was last accessed. + + @return: a number of seconds from the epoch. + @rtype: L{float} + """ + + def exists(): + """ + Check if this file path exists. + + @return: C{True} if the file at this file path exists, C{False} + otherwise. + @rtype: L{bool} + """ + + def isdir(): + """ + Check if this file path refers to a directory. + + @return: C{True} if the file at this file path is a directory, C{False} + otherwise. + """ + + def isfile(): + """ + Check if this file path refers to a regular file. + + @return: C{True} if the file at this file path is a regular file, + C{False} otherwise. + """ + + def children(): + """ + List the children of this path object. + + @return: a sequence of the children of the directory at this file path. + @raise Exception: if the file at this file path is not a directory. + """ + + def basename(): + """ + Retrieve the final component of the file path's path (everything + after the final path separator). + + @return: the base name of this file path. + @rtype: L{str} + """ + + def parent(): + """ + A file path for the directory containing the file at this file path. + """ + + def sibling(name): + """ + A file path for the directory containing the file at this file path. + + @param name: the name of a sibling of this path. C{name} must be a + direct sibling of this path and may not contain a path separator. + + @return: a sibling file path of this one. + """ + + +class InsecurePath(Exception): + """ + Error that is raised when the path provided to L{FilePath} is invalid. + """ + + + +class LinkError(Exception): + """ + An error with symlinks - either that there are cyclical symlinks or that + symlink are not supported on this platform. + """ + + + +class UnlistableError(OSError): + """ + An exception which is used to distinguish between errors which mean 'this + is not a directory you can list' and other, more catastrophic errors. + + This error will try to look as much like the original error as possible, + while still being catchable as an independent type. + + @ivar originalException: the actual original exception instance, either an + L{OSError} or a L{WindowsError}. + """ + def __init__(self, originalException): + """ + Create an UnlistableError exception. + + @param originalException: an instance of OSError. + """ + self.__dict__.update(originalException.__dict__) + self.originalException = originalException + + + +class _WindowsUnlistableError(UnlistableError, WindowsError): + """ + This exception is raised on Windows, for compatibility with previous + releases of FilePath where unportable programs may have done "except + WindowsError:" around a call to children(). + + It is private because all application code may portably catch + L{UnlistableError} instead. + """ + + + +def _secureEnoughString(path): + """ + Compute a string usable as a new, temporary filename. + + @param path: The path that the new temporary filename should be able to be + concatenated with. + + @return: A pseudorandom, 16 byte string for use in secure filenames. + @rtype: the type of C{path} + """ + secureishString = armor(randomBytes(16))[:16] + return _coerceToFilesystemEncoding(path, secureishString) + + + +class AbstractFilePath(object): + """ + Abstract implementation of an L{IFilePath}; must be completed by a + subclass. + + This class primarily exists to provide common implementations of certain + methods in L{IFilePath}. It is *not* a required parent class for + L{IFilePath} implementations, just a useful starting point. + """ + + def getContent(self): + """ + Retrieve the contents of the file at this path. + + @return: the contents of the file + @rtype: L{bytes} + """ + with self.open() as fp: + return fp.read() + + + def parents(self): + """ + Retrieve an iterator of all the ancestors of this path. + + @return: an iterator of all the ancestors of this path, from the most + recent (its immediate parent) to the root of its filesystem. + """ + path = self + parent = path.parent() + # root.parent() == root, so this means "are we the root" + while path != parent: + yield parent + path = parent + parent = parent.parent() + + + def children(self): + """ + List the children of this path object. + + @raise OSError: If an error occurs while listing the directory. If the + error is 'serious', meaning that the operation failed due to an access + violation, exhaustion of some kind of resource (file descriptors or + memory), OSError or a platform-specific variant will be raised. + + @raise UnlistableError: If the inability to list the directory is due + to this path not existing or not being a directory, the more specific + OSError subclass L{UnlistableError} is raised instead. + + @return: an iterable of all currently-existing children of this object. + """ + try: + subnames = self.listdir() + except WindowsError as winErrObj: + # Under Python 3.3 and higher on Windows, WindowsError is an + # alias for OSError. OSError has a winerror attribute and an + # errno attribute. + + # Under Python 2, WindowsError is an OSError subclass. + + # Under Python 2.5 and higher on Windows, WindowsError has a + # winerror attribute and an errno attribute. + + # The winerror attribute is bound to the Windows error code while + # the errno attribute is bound to a translation of that code to a + # perhaps equivalent POSIX error number. + # + # For further details, refer to: + # https://docs.python.org/3/library/exceptions.html#OSError + + # If not for this clause OSError would be handling all of these + # errors on Windows. The errno attribute contains a POSIX error + # code while the winerror attribute contains a Windows error code. + # Windows error codes aren't the same as POSIX error codes, + # so we need to handle them differently. + + # Under Python 2.4 on Windows, WindowsError only has an errno + # attribute. It is bound to the Windows error code. + + # For simplicity of code and to keep the number of paths through + # this suite minimal, we grab the Windows error code under either + # version. + + # Furthermore, attempting to use os.listdir on a non-existent path + # in Python 2.4 will result in a Windows error code of + # ERROR_PATH_NOT_FOUND. However, in Python 2.5, + # ERROR_FILE_NOT_FOUND results instead. -exarkun + winerror = getattr(winErrObj, 'winerror', winErrObj.errno) + if winerror not in (ERROR_PATH_NOT_FOUND, + ERROR_FILE_NOT_FOUND, + ERROR_INVALID_NAME, + ERROR_DIRECTORY): + raise + raise _WindowsUnlistableError(winErrObj) + except OSError as ose: + if ose.errno not in (errno.ENOENT, errno.ENOTDIR): + # Other possible errors here, according to linux manpages: + # EACCES, EMIFLE, ENFILE, ENOMEM. None of these seem like the + # sort of thing which should be handled normally. -glyph + raise + raise UnlistableError(ose) + return [self.child(name) for name in subnames] + + def walk(self, descend=None): + """ + Yield myself, then each of my children, and each of those children's + children in turn. + + The optional argument C{descend} is a predicate that takes a FilePath, + and determines whether or not that FilePath is traversed/descended + into. It will be called with each path for which C{isdir} returns + C{True}. If C{descend} is not specified, all directories will be + traversed (including symbolic links which refer to directories). + + @param descend: A one-argument callable that will return True for + FilePaths that should be traversed, False otherwise. + + @return: a generator yielding FilePath-like objects. + """ + yield self + if self.isdir(): + for c in self.children(): + # we should first see if it's what we want, then we + # can walk through the directory + if (descend is None or descend(c)): + for subc in c.walk(descend): + if os.path.realpath(self.path).startswith( + os.path.realpath(subc.path)): + raise LinkError("Cycle in file graph.") + yield subc + else: + yield c + + + def sibling(self, path): + """ + Return a L{FilePath} with the same directory as this instance but with + a basename of C{path}. + + @param path: The basename of the L{FilePath} to return. + @type path: L{str} + + @return: The sibling path. + @rtype: L{FilePath} + """ + return self.parent().child(path) + + + def descendant(self, segments): + """ + Retrieve a child or child's child of this path. + + @param segments: A sequence of path segments as L{str} instances. + + @return: A L{FilePath} constructed by looking up the C{segments[0]} + child of this path, the C{segments[1]} child of that path, and so + on. + + @since: 10.2 + """ + path = self + for name in segments: + path = path.child(name) + return path + + + def segmentsFrom(self, ancestor): + """ + Return a list of segments between a child and its ancestor. + + For example, in the case of a path X representing /a/b/c/d and a path Y + representing /a/b, C{Y.segmentsFrom(X)} will return C{['c', + 'd']}. + + @param ancestor: an instance of the same class as self, ostensibly an + ancestor of self. + + @raise: ValueError if the 'ancestor' parameter is not actually an + ancestor, i.e. a path for /x/y/z is passed as an ancestor for /a/b/c/d. + + @return: a list of strs + """ + # this might be an unnecessarily inefficient implementation but it will + # work on win32 and for zipfiles; later I will deterimine if the + # obvious fast implemenation does the right thing too + f = self + p = f.parent() + segments = [] + while f != ancestor and p != f: + segments[0:0] = [f.basename()] + f = p + p = p.parent() + if f == ancestor and segments: + return segments + raise ValueError("%r not parent of %r" % (ancestor, self)) + + + # new in 8.0 + def __hash__(self): + """ + Hash the same as another L{FilePath} with the same path as mine. + """ + return hash((self.__class__, self.path)) + + + # pending deprecation in 8.0 + def getmtime(self): + """ + Deprecated. Use getModificationTime instead. + """ + return int(self.getModificationTime()) + + + def getatime(self): + """ + Deprecated. Use getAccessTime instead. + """ + return int(self.getAccessTime()) + + + def getctime(self): + """ + Deprecated. Use getStatusChangeTime instead. + """ + return int(self.getStatusChangeTime()) + + + +class RWX(FancyEqMixin, object): + """ + A class representing read/write/execute permissions for a single user + category (i.e. user/owner, group, or other/world). Instantiate with + three boolean values: readable? writable? executable?. + + @type read: C{bool} + @ivar read: Whether permission to read is given + + @type write: C{bool} + @ivar write: Whether permission to write is given + + @type execute: C{bool} + @ivar execute: Whether permission to execute is given + + @since: 11.1 + """ + compareAttributes = ('read', 'write', 'execute') + def __init__(self, readable, writable, executable): + self.read = readable + self.write = writable + self.execute = executable + + + def __repr__(self): + return "RWX(read=%s, write=%s, execute=%s)" % ( + self.read, self.write, self.execute) + + + def shorthand(self): + """ + Returns a short string representing the permission bits. Looks like + part of what is printed by command line utilities such as 'ls -l' + (e.g. 'rwx') + + @return: The shorthand string. + @rtype: L{str} + """ + returnval = ['r', 'w', 'x'] + i = 0 + for val in (self.read, self.write, self.execute): + if not val: + returnval[i] = '-' + i += 1 + return ''.join(returnval) + + + +class Permissions(FancyEqMixin, object): + """ + A class representing read/write/execute permissions. Instantiate with any + portion of the file's mode that includes the permission bits. + + @type user: L{RWX} + @ivar user: User/Owner permissions + + @type group: L{RWX} + @ivar group: Group permissions + + @type other: L{RWX} + @ivar other: Other/World permissions + + @since: 11.1 + """ + + compareAttributes = ('user', 'group', 'other') + + def __init__(self, statModeInt): + self.user, self.group, self.other = ( + [RWX(*[statModeInt & bit > 0 for bit in bitGroup]) for bitGroup in + [[S_IRUSR, S_IWUSR, S_IXUSR], + [S_IRGRP, S_IWGRP, S_IXGRP], + [S_IROTH, S_IWOTH, S_IXOTH]]] + ) + + + def __repr__(self): + return "[%s | %s | %s]" % ( + str(self.user), str(self.group), str(self.other)) + + + def shorthand(self): + """ + Returns a short string representing the permission bits. Looks like + what is printed by command line utilities such as 'ls -l' + (e.g. 'rwx-wx--x') + + @return: The shorthand string. + @rtype: L{str} + """ + return "".join( + [x.shorthand() for x in (self.user, self.group, self.other)]) + + +class _SpecialNoValue(object): + """ + An object that represents 'no value', to be used in deprecating statinfo. + + Please remove once statinfo is removed. + """ + pass + + + +def _asFilesystemBytes(path, encoding=None): + """ + Return C{path} as a string of L{bytes} suitable for use on this system's + filesystem. + + @param path: The path to be made suitable. + @type path: L{bytes} or L{unicode} + @param encoding: The encoding to use if coercing to L{bytes}. If none is + given, L{sys.getfilesystemencoding} is used. + + @return: L{bytes} + """ + if type(path) == bytes: + return path + else: + if encoding is None: + encoding = sys.getfilesystemencoding() + return path.encode(encoding) + + + +def _asFilesystemText(path, encoding=None): + """ + Return C{path} as a string of L{unicode} suitable for use on this system's + filesystem. + + @param path: The path to be made suitable. + @type path: L{bytes} or L{unicode} + + @param encoding: The encoding to use if coercing to L{unicode}. If none + is given, L{sys.getfilesystemencoding} is used. + + @return: L{unicode} + """ + if type(path) == unicode: + return path + else: + if encoding is None: + encoding = sys.getfilesystemencoding() + return path.decode(encoding) + + + +def _coerceToFilesystemEncoding(path, newpath, encoding=None): + """ + Return a C{newpath} that is suitable for joining to C{path}. + + @param path: The path that it should be suitable for joining to. + @param newpath: The new portion of the path to be coerced if needed. + @param encoding: If coerced, the encoding that will be used. + """ + if type(path) == bytes: + return _asFilesystemBytes(newpath, encoding=encoding) + else: + return _asFilesystemText(newpath, encoding=encoding) + + + +@comparable +@implementer(IFilePath) +class FilePath(AbstractFilePath): + """ + I am a path on the filesystem that only permits 'downwards' access. + + Instantiate me with a pathname (for example, + FilePath('/home/myuser/public_html')) and I will attempt to only provide + access to files which reside inside that path. I may be a path to a file, + a directory, or a file which does not exist. + + The correct way to use me is to instantiate me, and then do ALL filesystem + access through me. In other words, do not import the 'os' module; if you + need to open a file, call my 'open' method. If you need to list a + directory, call my 'path' method. + + Even if you pass me a relative path, I will convert that to an absolute + path internally. + + Note: although time-related methods do return floating-point results, they + may still be only second resolution depending on the platform and the last + value passed to L{os.stat_float_times}. If you want greater-than-second + precision, call C{os.stat_float_times(True)}, or use Python 2.5. + Greater-than-second precision is only available in Windows on Python2.5 and + later. + + The type of C{path} when instantiating decides the mode of the L{FilePath}. + That is, C{FilePath(b"/")} will return a L{bytes} mode L{FilePath}, and + C{FilePath(u"/")} will return a L{unicode} mode L{FilePath}. + C{FilePath("/")} will return a L{bytes} mode L{FilePath} on Python 2, and a + L{unicode} mode L{FilePath} on Python 3. + + Methods that return a new L{FilePath} use the type of the given subpath to + decide its mode. For example, C{FilePath(b"/").child(u"tmp")} will return a + L{unicode} mode L{FilePath}. + + @type alwaysCreate: L{bool} + @ivar alwaysCreate: When opening this file, only succeed if the file does + not already exist. + + @type path: L{bytes} or L{unicode} + @ivar path: The path from which 'downward' traversal is permitted. + + @ivar statinfo: (WARNING: statinfo is deprecated as of Twisted 15.0.0 and + will become a private attribute) + The currently cached status information about the file on + the filesystem that this L{FilePath} points to. This attribute is + L{None} if the file is in an indeterminate state (either this + L{FilePath} has not yet had cause to call C{stat()} yet or + L{FilePath.changed} indicated that new information is required), 0 if + C{stat()} was called and returned an error (i.e. the path did not exist + when C{stat()} was called), or a C{stat_result} object that describes + the last known status of the underlying file (or directory, as the case + may be). Trust me when I tell you that you do not want to use this + attribute. Instead, use the methods on L{FilePath} which give you + information about it, like C{getsize()}, C{isdir()}, + C{getModificationTime()}, and so on. + @type statinfo: L{int} or L{None} or L{os.stat_result} + """ + _statinfo = None + path = None + + + def __init__(self, path, alwaysCreate=False): + """ + Convert a path string to an absolute path if necessary and initialize + the L{FilePath} with the result. + """ + self.path = abspath(path) + self.alwaysCreate = alwaysCreate + + + def __getstate__(self): + """ + Support serialization by discarding cached L{os.stat} results and + returning everything else. + """ + d = self.__dict__.copy() + if '_statinfo' in d: + del d['_statinfo'] + return d + + + @property + def sep(self): + """ + Return a filesystem separator. + + @return: The native filesystem separator. + @returntype: The same type as C{self.path}. + """ + return _coerceToFilesystemEncoding(self.path, os.sep) + + + def _asBytesPath(self, encoding=None): + """ + Return the path of this L{FilePath} as bytes. + + @param encoding: The encoding to use if coercing to L{bytes}. If none is + given, L{sys.getfilesystemencoding} is used. + + @return: L{bytes} + """ + return _asFilesystemBytes(self.path, encoding=encoding) + + + def _asTextPath(self, encoding=None): + """ + Return the path of this L{FilePath} as text. + + @param encoding: The encoding to use if coercing to L{unicode}. If none + is given, L{sys.getfilesystemencoding} is used. + + @return: L{unicode} + """ + return _asFilesystemText(self.path, encoding=encoding) + + + def asBytesMode(self, encoding=None): + """ + Return this L{FilePath} in L{bytes}-mode. + + @param encoding: The encoding to use if coercing to L{bytes}. If none is + given, L{sys.getfilesystemencoding} is used. + + @return: L{bytes} mode L{FilePath} + """ + if type(self.path) == unicode: + return self.clonePath(self._asBytesPath(encoding=encoding)) + return self + + + def asTextMode(self, encoding=None): + """ + Return this L{FilePath} in L{unicode}-mode. + + @param encoding: The encoding to use if coercing to L{unicode}. If none + is given, L{sys.getfilesystemencoding} is used. + + @return: L{unicode} mode L{FilePath} + """ + if type(self.path) == bytes: + return self.clonePath(self._asTextPath(encoding=encoding)) + return self + + + def _getPathAsSameTypeAs(self, pattern): + """ + If C{pattern} is C{bytes}, return L{FilePath.path} as L{bytes}. + Otherwise, return L{FilePath.path} as L{unicode}. + + @param pattern: The new element of the path that L{FilePath.path} may + need to be coerced to match. + """ + if type(pattern) == bytes: + return self._asBytesPath() + else: + return self._asTextPath() + + + def child(self, path): + """ + Create and return a new L{FilePath} representing a path contained by + C{self}. + + @param path: The base name of the new L{FilePath}. If this contains + directory separators or parent references it will be rejected. + @type path: L{bytes} or L{unicode} + + @raise InsecurePath: If the result of combining this path with C{path} + would result in a path which is not a direct child of this path. + + @return: The child path. + @rtype: L{FilePath} with a mode equal to the type of C{path}. + """ + colon = _coerceToFilesystemEncoding(path, ":") + sep = _coerceToFilesystemEncoding(path, os.sep) + ourPath = self._getPathAsSameTypeAs(path) + + if platform.isWindows() and path.count(colon): + # Catch paths like C:blah that don't have a slash + raise InsecurePath("%r contains a colon." % (path,)) + + norm = normpath(path) + if sep in norm: + raise InsecurePath("%r contains one or more directory separators" % + (path,)) + + newpath = abspath(joinpath(ourPath, norm)) + if not newpath.startswith(ourPath): + raise InsecurePath("%r is not a child of %s" % + (newpath, ourPath)) + return self.clonePath(newpath) + + + def preauthChild(self, path): + """ + Use me if C{path} might have slashes in it, but you know they're safe. + + @param path: A relative path (ie, a path not starting with C{"/"}) + which will be interpreted as a child or descendant of this path. + @type path: L{bytes} or L{unicode} + + @return: The child path. + @rtype: L{FilePath} with a mode equal to the type of C{path}. + """ + ourPath = self._getPathAsSameTypeAs(path) + + newpath = abspath(joinpath(ourPath, normpath(path))) + if not newpath.startswith(ourPath): + raise InsecurePath("%s is not a child of %s" % + (newpath, ourPath)) + return self.clonePath(newpath) + + + def childSearchPreauth(self, *paths): + """ + Return my first existing child with a name in C{paths}. + + C{paths} is expected to be a list of *pre-secured* path fragments; + in most cases this will be specified by a system administrator and not + an arbitrary user. + + If no appropriately-named children exist, this will return L{None}. + + @return: L{None} or the child path. + @rtype: L{None} or L{FilePath} + """ + for child in paths: + p = self._getPathAsSameTypeAs(child) + jp = joinpath(p, child) + if exists(jp): + return self.clonePath(jp) + + + def siblingExtensionSearch(self, *exts): + """ + Attempt to return a path with my name, given multiple possible + extensions. + + Each extension in C{exts} will be tested and the first path which + exists will be returned. If no path exists, L{None} will be returned. + If C{''} is in C{exts}, then if the file referred to by this path + exists, C{self} will be returned. + + The extension '*' has a magic meaning, which means "any path that + begins with C{self.path + '.'} is acceptable". + """ + for ext in exts: + if not ext and self.exists(): + return self + + p = self._getPathAsSameTypeAs(ext) + star = _coerceToFilesystemEncoding(ext, "*") + dot = _coerceToFilesystemEncoding(ext, ".") + + if ext == star: + basedot = basename(p) + dot + for fn in listdir(dirname(p)): + if fn.startswith(basedot): + return self.clonePath(joinpath(dirname(p), fn)) + p2 = p + ext + if exists(p2): + return self.clonePath(p2) + + + def realpath(self): + """ + Returns the absolute target as a L{FilePath} if self is a link, self + otherwise. + + The absolute link is the ultimate file or directory the + link refers to (for instance, if the link refers to another link, and + another...). If the filesystem does not support symlinks, or + if the link is cyclical, raises a L{LinkError}. + + Behaves like L{os.path.realpath} in that it does not resolve link + names in the middle (ex. /x/y/z, y is a link to w - realpath on z + will return /x/y/z, not /x/w/z). + + @return: L{FilePath} of the target path. + @rtype: L{FilePath} + @raises LinkError: if links are not supported or links are cyclical. + """ + if self.islink(): + result = os.path.realpath(self.path) + if result == self.path: + raise LinkError("Cyclical link - will loop forever") + return self.clonePath(result) + return self + + + def siblingExtension(self, ext): + """ + Attempt to return a path with my name, given the extension at C{ext}. + + @param ext: File-extension to search for. + @type ext: L{bytes} or L{unicode} + + @return: The sibling path. + @rtype: L{FilePath} with the same mode as the type of C{ext}. + """ + ourPath = self._getPathAsSameTypeAs(ext) + return self.clonePath(ourPath + ext) + + + def linkTo(self, linkFilePath): + """ + Creates a symlink to self to at the path in the L{FilePath} + C{linkFilePath}. + + Only works on posix systems due to its dependence on + L{os.symlink}. Propagates L{OSError}s up from L{os.symlink} if + C{linkFilePath.parent()} does not exist, or C{linkFilePath} already + exists. + + @param linkFilePath: a FilePath representing the link to be created. + @type linkFilePath: L{FilePath} + """ + os.symlink(self.path, linkFilePath.path) + + + def open(self, mode='r'): + """ + Open this file using C{mode} or for writing if C{alwaysCreate} is + C{True}. + + In all cases the file is opened in binary mode, so it is not necessary + to include C{"b"} in C{mode}. + + @param mode: The mode to open the file in. Default is C{"r"}. + @type mode: L{str} + @raises AssertionError: If C{"a"} is included in the mode and + C{alwaysCreate} is C{True}. + @rtype: L{file} + @return: An open L{file} object. + """ + if self.alwaysCreate: + assert 'a' not in mode, ("Appending not supported when " + "alwaysCreate == True") + return self.create() + # This hack is necessary because of a bug in Python 2.7 on Windows: + # http://bugs.python.org/issue7686 + mode = mode.replace('b', '') + return open(self.path, mode + 'b') + + # stat methods below + + def restat(self, reraise=True): + """ + Re-calculate cached effects of 'stat'. To refresh information on this + path after you know the filesystem may have changed, call this method. + + @param reraise: a boolean. If true, re-raise exceptions from + L{os.stat}; otherwise, mark this path as not existing, and remove + any cached stat information. + + @raise Exception: If C{reraise} is C{True} and an exception occurs + while reloading metadata. + """ + try: + self._statinfo = stat(self.path) + except OSError: + self._statinfo = 0 + if reraise: + raise + + + def changed(self): + """ + Clear any cached information about the state of this path on disk. + + @since: 10.1.0 + """ + self._statinfo = None + + + def chmod(self, mode): + """ + Changes the permissions on self, if possible. Propagates errors from + L{os.chmod} up. + + @param mode: integer representing the new permissions desired (same as + the command line chmod) + @type mode: L{int} + """ + os.chmod(self.path, mode) + + + def getsize(self): + """ + Retrieve the size of this file in bytes. + + @return: The size of the file at this file path in bytes. + @raise Exception: if the size cannot be obtained. + @rtype: L{int} + """ + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return st.st_size + + + def getModificationTime(self): + """ + Retrieve the time of last access from this file. + + @return: a number of seconds from the epoch. + @rtype: L{float} + """ + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return float(st.st_mtime) + + + def getStatusChangeTime(self): + """ + Retrieve the time of the last status change for this file. + + @return: a number of seconds from the epoch. + @rtype: L{float} + """ + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return float(st.st_ctime) + + + def getAccessTime(self): + """ + Retrieve the time that this file was last accessed. + + @return: a number of seconds from the epoch. + @rtype: L{float} + """ + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return float(st.st_atime) + + + def getInodeNumber(self): + """ + Retrieve the file serial number, also called inode number, which + distinguishes this file from all other files on the same device. + + @raise NotImplementedError: if the platform is Windows, since the + inode number would be a dummy value for all files in Windows + @return: a number representing the file serial number + @rtype: L{int} + @since: 11.0 + """ + if platform.isWindows(): + raise NotImplementedError + + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return st.st_ino + + + def getDevice(self): + """ + Retrieves the device containing the file. The inode number and device + number together uniquely identify the file, but the device number is + not necessarily consistent across reboots or system crashes. + + @raise NotImplementedError: if the platform is Windows, since the + device number would be 0 for all partitions on a Windows platform + + @return: a number representing the device + @rtype: L{int} + + @since: 11.0 + """ + if platform.isWindows(): + raise NotImplementedError + + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return st.st_dev + + + def getNumberOfHardLinks(self): + """ + Retrieves the number of hard links to the file. + + This count keeps track of how many directories have entries for this + file. If the count is ever decremented to zero then the file itself is + discarded as soon as no process still holds it open. Symbolic links + are not counted in the total. + + @raise NotImplementedError: if the platform is Windows, since Windows + doesn't maintain a link count for directories, and L{os.stat} does + not set C{st_nlink} on Windows anyway. + @return: the number of hard links to the file + @rtype: L{int} + @since: 11.0 + """ + if platform.isWindows(): + raise NotImplementedError + + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return st.st_nlink + + + def getUserID(self): + """ + Returns the user ID of the file's owner. + + @raise NotImplementedError: if the platform is Windows, since the UID + is always 0 on Windows + @return: the user ID of the file's owner + @rtype: L{int} + @since: 11.0 + """ + if platform.isWindows(): + raise NotImplementedError + + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return st.st_uid + + + def getGroupID(self): + """ + Returns the group ID of the file. + + @raise NotImplementedError: if the platform is Windows, since the GID + is always 0 on windows + @return: the group ID of the file + @rtype: L{int} + @since: 11.0 + """ + if platform.isWindows(): + raise NotImplementedError + + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return st.st_gid + + + def getPermissions(self): + """ + Returns the permissions of the file. Should also work on Windows, + however, those permissions may not be what is expected in Windows. + + @return: the permissions for the file + @rtype: L{Permissions} + @since: 11.1 + """ + st = self._statinfo + if not st: + self.restat() + st = self._statinfo + return Permissions(S_IMODE(st.st_mode)) + + + def exists(self): + """ + Check if this L{FilePath} exists. + + @return: C{True} if the stats of C{path} can be retrieved successfully, + C{False} in the other cases. + @rtype: L{bool} + """ + if self._statinfo: + return True + else: + self.restat(False) + if self._statinfo: + return True + else: + return False + + + def isdir(self): + """ + Check if this L{FilePath} refers to a directory. + + @return: C{True} if this L{FilePath} refers to a directory, C{False} + otherwise. + @rtype: L{bool} + """ + st = self._statinfo + if not st: + self.restat(False) + st = self._statinfo + if not st: + return False + return S_ISDIR(st.st_mode) + + + def isfile(self): + """ + Check if this file path refers to a regular file. + + @return: C{True} if this L{FilePath} points to a regular file (not a + directory, socket, named pipe, etc), C{False} otherwise. + @rtype: L{bool} + """ + st = self._statinfo + if not st: + self.restat(False) + st = self._statinfo + if not st: + return False + return S_ISREG(st.st_mode) + + + def isBlockDevice(self): + """ + Returns whether the underlying path is a block device. + + @return: C{True} if it is a block device, C{False} otherwise + @rtype: L{bool} + @since: 11.1 + """ + st = self._statinfo + if not st: + self.restat(False) + st = self._statinfo + if not st: + return False + return S_ISBLK(st.st_mode) + + + def isSocket(self): + """ + Returns whether the underlying path is a socket. + + @return: C{True} if it is a socket, C{False} otherwise + @rtype: L{bool} + @since: 11.1 + """ + st = self._statinfo + if not st: + self.restat(False) + st = self._statinfo + if not st: + return False + return S_ISSOCK(st.st_mode) + + + def islink(self): + """ + Check if this L{FilePath} points to a symbolic link. + + @return: C{True} if this L{FilePath} points to a symbolic link, + C{False} otherwise. + @rtype: L{bool} + """ + # We can't use cached stat results here, because that is the stat of + # the destination - (see #1773) which in *every case* but this one is + # the right thing to use. We could call lstat here and use that, but + # it seems unlikely we'd actually save any work that way. -glyph + return islink(self.path) + + + def isabs(self): + """ + Check if this L{FilePath} refers to an absolute path. + + This always returns C{True}. + + @return: C{True}, always. + @rtype: L{bool} + """ + return isabs(self.path) + + + def listdir(self): + """ + List the base names of the direct children of this L{FilePath}. + + @return: A L{list} of L{bytes}/L{unicode} giving the names of the + contents of the directory this L{FilePath} refers to. These names + are relative to this L{FilePath}. + @rtype: L{list} + + @raise: Anything the platform L{os.listdir} implementation might raise + (typically L{OSError}). + """ + return listdir(self.path) + + + def splitext(self): + """ + Split the file path into a pair C{(root, ext)} such that + C{root + ext == path}. + + @return: Tuple where the first item is the filename and second item is + the file extension. See Python docs for L{os.path.splitext}. + @rtype: L{tuple} + """ + return splitext(self.path) + + + def __repr__(self): + return 'FilePath(%r)' % (self.path,) + + + def touch(self): + """ + Updates the access and last modification times of the file at this + file path to the current time. Also creates the file if it does not + already exist. + + @raise Exception: if unable to create or modify the last modification + time of the file. + """ + try: + self.open('a').close() + except IOError: + pass + utime(self.path, None) + + + def remove(self): + """ + Removes the file or directory that is represented by self. If + C{self.path} is a directory, recursively remove all its children + before removing the directory. If it's a file or link, just delete it. + """ + if self.isdir() and not self.islink(): + for child in self.children(): + child.remove() + os.rmdir(self.path) + else: + os.remove(self.path) + self.changed() + + + def makedirs(self, ignoreExistingDirectory=False): + """ + Create all directories not yet existing in C{path} segments, using + L{os.makedirs}. + + @param ignoreExistingDirectory: Don't raise L{OSError} if directory + already exists. + @type ignoreExistingDirectory: L{bool} + + @return: L{None} + """ + try: + return os.makedirs(self.path) + except OSError as e: + if not ( + e.errno == errno.EEXIST and + ignoreExistingDirectory and + self.isdir()): + raise + + + def globChildren(self, pattern): + """ + Assuming I am representing a directory, return a list of FilePaths + representing my children that match the given pattern. + + @param pattern: A glob pattern to use to match child paths. + @type pattern: L{unicode} or L{bytes} + + @return: A L{list} of matching children. + @rtype: L{list} of L{FilePath}, with the mode of C{pattern}'s type + """ + sep = _coerceToFilesystemEncoding(pattern, os.sep) + ourPath = self._getPathAsSameTypeAs(pattern) + + import glob + path = ourPath[-1] == sep and ourPath + pattern \ + or sep.join([ourPath, pattern]) + return [self.clonePath(p) for p in glob.glob(path)] + + + def basename(self): + """ + Retrieve the final component of the file path's path (everything + after the final path separator). + + @return: The final component of the L{FilePath}'s path (Everything + after the final path separator). + @rtype: the same type as this L{FilePath}'s C{path} attribute + """ + return basename(self.path) + + + def dirname(self): + """ + Retrieve all of the components of the L{FilePath}'s path except the + last one (everything up to the final path separator). + + @return: All of the components of the L{FilePath}'s path except the + last one (everything up to the final path separator). + @rtype: the same type as this L{FilePath}'s C{path} attribute + """ + return dirname(self.path) + + + def parent(self): + """ + A file path for the directory containing the file at this file path. + + @return: A L{FilePath} representing the path which directly contains + this L{FilePath}. + @rtype: L{FilePath} + """ + return self.clonePath(self.dirname()) + + + def setContent(self, content, ext=b'.new'): + """ + Replace the file at this path with a new file that contains the given + bytes, trying to avoid data-loss in the meanwhile. + + On UNIX-like platforms, this method does its best to ensure that by the + time this method returns, either the old contents I{or} the new + contents of the file will be present at this path for subsequent + readers regardless of premature device removal, program crash, or power + loss, making the following assumptions: + + - your filesystem is journaled (i.e. your filesystem will not + I{itself} lose data due to power loss) + + - your filesystem's C{rename()} is atomic + + - your filesystem will not discard new data while preserving new + metadata (see U{http://mjg59.livejournal.com/108257.html} for + more detail) + + On most versions of Windows there is no atomic C{rename()} (see + U{http://bit.ly/win32-overwrite} for more information), so this method + is slightly less helpful. There is a small window where the file at + this path may be deleted before the new file is moved to replace it: + however, the new file will be fully written and flushed beforehand so + in the unlikely event that there is a crash at that point, it should be + possible for the user to manually recover the new version of their + data. In the future, Twisted will support atomic file moves on those + versions of Windows which I{do} support them: see U{Twisted ticket + 3004}. + + This method should be safe for use by multiple concurrent processes, + but note that it is not easy to predict which process's contents will + ultimately end up on disk if they invoke this method at close to the + same time. + + @param content: The desired contents of the file at this path. + @type content: L{bytes} + + @param ext: An extension to append to the temporary filename used to + store the bytes while they are being written. This can be used to + make sure that temporary files can be identified by their suffix, + for cleanup in case of crashes. + @type ext: L{bytes} + """ + sib = self.temporarySibling(ext) + with sib.open('w') as f: + f.write(content) + if platform.isWindows() and exists(self.path): + os.unlink(self.path) + os.rename(sib.path, self.asBytesMode().path) + + + def __cmp__(self, other): + if not isinstance(other, FilePath): + return NotImplemented + return cmp(self.path, other.path) + + + def createDirectory(self): + """ + Create the directory the L{FilePath} refers to. + + @see: L{makedirs} + + @raise OSError: If the directory cannot be created. + """ + os.mkdir(self.path) + + + def requireCreate(self, val=1): + """ + Sets the C{alwaysCreate} variable. + + @param val: C{True} or C{False}, indicating whether opening this path + will be required to create the file or not. + @type val: L{bool} + + @return: L{None} + """ + self.alwaysCreate = val + + + def create(self): + """ + Exclusively create a file, only if this file previously did not exist. + + @return: A file-like object opened from this path. + """ + fdint = os.open(self.path, _CREATE_FLAGS) + + # XXX TODO: 'name' attribute of returned files is not mutable or + # settable via fdopen, so this file is slightly less functional than the + # one returned from 'open' by default. send a patch to Python... + + return os.fdopen(fdint, 'w+b') + + + def temporarySibling(self, extension=b""): + """ + Construct a path referring to a sibling of this path. + + The resulting path will be unpredictable, so that other subprocesses + should neither accidentally attempt to refer to the same path before it + is created, nor they should other processes be able to guess its name + in advance. + + @param extension: A suffix to append to the created filename. (Note + that if you want an extension with a '.' you must include the '.' + yourself.) + @type extension: L{bytes} or L{unicode} + + @return: a path object with the given extension suffix, C{alwaysCreate} + set to True. + @rtype: L{FilePath} with a mode equal to the type of C{extension} + """ + ourPath = self._getPathAsSameTypeAs(extension) + sib = self.sibling(_secureEnoughString(ourPath) + + self.clonePath(ourPath).basename() + extension) + sib.requireCreate() + return sib + + + _chunkSize = 2 ** 2 ** 2 ** 2 + + def copyTo(self, destination, followLinks=True): + """ + Copies self to destination. + + If self doesn't exist, an OSError is raised. + + If self is a directory, this method copies its children (but not + itself) recursively to destination - if destination does not exist as a + directory, this method creates it. If destination is a file, an + IOError will be raised. + + If self is a file, this method copies it to destination. If + destination is a file, this method overwrites it. If destination is a + directory, an IOError will be raised. + + If self is a link (and followLinks is False), self will be copied + over as a new symlink with the same target as returned by os.readlink. + That means that if it is absolute, both the old and new symlink will + link to the same thing. If it's relative, then perhaps not (and + it's also possible that this relative link will be broken). + + File/directory permissions and ownership will NOT be copied over. + + If followLinks is True, symlinks are followed so that they're treated + as their targets. In other words, if self is a link, the link's target + will be copied. If destination is a link, self will be copied to the + destination's target (the actual destination will be destination's + target). Symlinks under self (if self is a directory) will be + followed and its target's children be copied recursively. + + If followLinks is False, symlinks will be copied over as symlinks. + + @param destination: the destination (a FilePath) to which self + should be copied + @param followLinks: whether symlinks in self should be treated as links + or as their targets + """ + if self.islink() and not followLinks: + os.symlink(os.readlink(self.path), destination.path) + return + # XXX TODO: *thorough* audit and documentation of the exact desired + # semantics of this code. Right now the behavior of existent + # destination symlinks is convenient, and quite possibly correct, but + # its security properties need to be explained. + if self.isdir(): + if not destination.exists(): + destination.createDirectory() + for child in self.children(): + destChild = destination.child(child.basename()) + child.copyTo(destChild, followLinks) + elif self.isfile(): + with destination.open('w') as writefile, self.open() as readfile: + while 1: + # XXX TODO: optionally use os.open, os.read and + # O_DIRECT and use os.fstatvfs to determine chunk sizes + # and make *****sure**** copy is page-atomic; the + # following is good enough for 99.9% of everybody and + # won't take a week to audit though. + chunk = readfile.read(self._chunkSize) + writefile.write(chunk) + if len(chunk) < self._chunkSize: + break + elif not self.exists(): + raise OSError(errno.ENOENT, "No such file or directory") + else: + # If you see the following message because you want to copy + # symlinks, fifos, block devices, character devices, or unix + # sockets, please feel free to add support to do sensible things in + # reaction to those types! + raise NotImplementedError( + "Only copying of files and directories supported") + + + def moveTo(self, destination, followLinks=True): + """ + Move self to destination - basically renaming self to whatever + destination is named. + + If destination is an already-existing directory, + moves all children to destination if destination is empty. If + destination is a non-empty directory, or destination is a file, an + OSError will be raised. + + If moving between filesystems, self needs to be copied, and everything + that applies to copyTo applies to moveTo. + + @param destination: the destination (a FilePath) to which self + should be copied + @param followLinks: whether symlinks in self should be treated as links + or as their targets (only applicable when moving between + filesystems) + """ + try: + os.rename(self._getPathAsSameTypeAs(destination.path), + destination.path) + except OSError as ose: + if ose.errno == errno.EXDEV: + # man 2 rename, ubuntu linux 5.10 "breezy": + + # oldpath and newpath are not on the same mounted filesystem. + # (Linux permits a filesystem to be mounted at multiple + # points, but rename(2) does not work across different mount + # points, even if the same filesystem is mounted on both.) + + # that means it's time to copy trees of directories! + secsib = destination.temporarySibling() + self.copyTo(secsib, followLinks) # slow + secsib.moveTo(destination, followLinks) # visible + + # done creating new stuff. let's clean me up. + mysecsib = self.temporarySibling() + self.moveTo(mysecsib, followLinks) # visible + mysecsib.remove() # slow + else: + raise + else: + self.changed() + destination.changed() + + + def statinfo(self, value=_SpecialNoValue): + """ + FilePath.statinfo is deprecated. + + @param value: value to set statinfo to, if setting a value + @return: C{_statinfo} if getting, L{None} if setting + """ + # This is a pretty awful hack to use the deprecated decorator to + # deprecate a class attribute. Ideally, there would just be a + # statinfo property and a statinfo property setter, but the + # 'deprecated' decorator does not produce the correct FQDN on class + # methods. So the property stuff needs to be set outside the class + # definition - but the getter and setter both need the same function + # in order for the 'deprecated' decorator to produce the right + # deprecation string. + if value is _SpecialNoValue: + return self._statinfo + else: + self._statinfo = value + + +# This is all a terrible hack to get statinfo deprecated +_tmp = deprecated( + Version('Twisted', 15, 0, 0), + "other FilePath methods such as getsize(), " + "isdir(), getModificationTime(), etc.")(FilePath.statinfo) +FilePath.statinfo = property(_tmp, _tmp) + + +FilePath.clonePath = FilePath diff --git a/contrib/python/Twisted/py2/twisted/python/finalize.py b/contrib/python/Twisted/py2/twisted/python/finalize.py new file mode 100644 index 00000000000..2b2314f4596 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/finalize.py @@ -0,0 +1,48 @@ + +""" +A module for externalized finalizers. +""" + +from __future__ import print_function + +import weakref + +garbageKey = 0 + +def callbackFactory(num, fins): + def _cb(w): + del refs[num] + for fx in fins: + fx() + return _cb + +refs = {} + +def register(inst): + global garbageKey + garbageKey += 1 + r = weakref.ref(inst, callbackFactory(garbageKey, inst.__finalizers__())) + refs[garbageKey] = r + +if __name__ == '__main__': + def fin(): + print('I am _so_ dead.') + + class Finalizeable: + """ + An un-sucky __del__ + """ + + def __finalizers__(self): + """ + I'm going away. + """ + return [fin] + + f = Finalizeable() + f.f2 = f + register(f) + del f + import gc + gc.collect() + print('deled') diff --git a/contrib/python/Twisted/py2/twisted/python/formmethod.py b/contrib/python/Twisted/py2/twisted/python/formmethod.py new file mode 100644 index 00000000000..435a975d714 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/formmethod.py @@ -0,0 +1,377 @@ +# -*- test-case-name: twisted.test.test_formmethod -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Form-based method objects. + +This module contains support for descriptive method signatures that can be used +to format methods. +""" + +import calendar + +from twisted.python._oldstyle import _oldStyle + + +class FormException(Exception): + """An error occurred calling the form method. + """ + def __init__(self, *args, **kwargs): + Exception.__init__(self, *args) + self.descriptions = kwargs + + +class InputError(FormException): + """ + An error occurred with some input. + """ + + + +@_oldStyle +class Argument: + """Base class for form arguments.""" + + # default value for argument, if no other default is given + defaultDefault = None + + def __init__(self, name, default=None, shortDesc=None, + longDesc=None, hints=None, allowNone=1): + self.name = name + self.allowNone = allowNone + if default is None: + default = self.defaultDefault + self.default = default + self.shortDesc = shortDesc + self.longDesc = longDesc + if not hints: + hints = {} + self.hints = hints + + def addHints(self, **kwargs): + self.hints.update(kwargs) + + def getHint(self, name, default=None): + return self.hints.get(name, default) + + def getShortDescription(self): + return self.shortDesc or self.name.capitalize() + + def getLongDescription(self): + return self.longDesc or '' #self.shortDesc or "The %s." % self.name + + def coerce(self, val): + """Convert the value to the correct format.""" + raise NotImplementedError("implement in subclass") + + +class String(Argument): + """A single string. + """ + defaultDefault = '' + min = 0 + max = None + + def __init__(self, name, default=None, shortDesc=None, + longDesc=None, hints=None, allowNone=1, min=0, max=None): + Argument.__init__(self, name, default=default, shortDesc=shortDesc, + longDesc=longDesc, hints=hints, allowNone=allowNone) + self.min = min + self.max = max + + def coerce(self, val): + s = str(val) + if len(s) < self.min: + raise InputError("Value must be at least %s characters long" % self.min) + if self.max != None and len(s) > self.max: + raise InputError("Value must be at most %s characters long" % self.max) + return str(val) + + +class Text(String): + """A long string. + """ + + +class Password(String): + """A string which should be obscured when input. + """ + + +class VerifiedPassword(String): + """A string that should be obscured when input and needs verification.""" + + def coerce(self, vals): + if len(vals) != 2 or vals[0] != vals[1]: + raise InputError("Please enter the same password twice.") + s = str(vals[0]) + if len(s) < self.min: + raise InputError("Value must be at least %s characters long" % self.min) + if self.max != None and len(s) > self.max: + raise InputError("Value must be at most %s characters long" % self.max) + return s + + +class Hidden(String): + """A string which is not displayed. + + The passed default is used as the value. + """ + + +class Integer(Argument): + """A single integer. + """ + defaultDefault = None + + def __init__(self, name, allowNone=1, default=None, shortDesc=None, + longDesc=None, hints=None): + #although Argument now has allowNone, that was recently added, and + #putting it at the end kept things which relied on argument order + #from breaking. However, allowNone originally was in here, so + #I have to keep the same order, to prevent breaking code that + #depends on argument order only + Argument.__init__(self, name, default, shortDesc, longDesc, hints, + allowNone) + + def coerce(self, val): + if not val.strip() and self.allowNone: + return None + try: + return int(val) + except ValueError: + raise InputError("%s is not valid, please enter a whole number, e.g. 10" % val) + + +class IntegerRange(Integer): + + def __init__(self, name, min, max, allowNone=1, default=None, shortDesc=None, + longDesc=None, hints=None): + self.min = min + self.max = max + Integer.__init__(self, name, allowNone=allowNone, default=default, shortDesc=shortDesc, + longDesc=longDesc, hints=hints) + + def coerce(self, val): + result = Integer.coerce(self, val) + if self.allowNone and result == None: + return result + if result < self.min: + raise InputError("Value %s is too small, it should be at least %s" % (result, self.min)) + if result > self.max: + raise InputError("Value %s is too large, it should be at most %s" % (result, self.max)) + return result + + +class Float(Argument): + + defaultDefault = None + + def __init__(self, name, allowNone=1, default=None, shortDesc=None, + longDesc=None, hints=None): + #although Argument now has allowNone, that was recently added, and + #putting it at the end kept things which relied on argument order + #from breaking. However, allowNone originally was in here, so + #I have to keep the same order, to prevent breaking code that + #depends on argument order only + Argument.__init__(self, name, default, shortDesc, longDesc, hints, + allowNone) + + + def coerce(self, val): + if not val.strip() and self.allowNone: + return None + try: + return float(val) + except ValueError: + raise InputError("Invalid float: %s" % val) + + +class Choice(Argument): + """ + The result of a choice between enumerated types. The choices should + be a list of tuples of tag, value, and description. The tag will be + the value returned if the user hits "Submit", and the description + is the bale for the enumerated type. default is a list of all the + values (seconds element in choices). If no defaults are specified, + initially the first item will be selected. Only one item can (should) + be selected at once. + """ + def __init__(self, name, choices=[], default=[], shortDesc=None, + longDesc=None, hints=None, allowNone=1): + self.choices = choices + if choices and not default: + default.append(choices[0][1]) + Argument.__init__(self, name, default, shortDesc, longDesc, hints, allowNone=allowNone) + + def coerce(self, inIdent): + for ident, val, desc in self.choices: + if ident == inIdent: + return val + else: + raise InputError("Invalid Choice: %s" % inIdent) + + +class Flags(Argument): + """ + The result of a checkbox group or multi-menu. The flags should be a + list of tuples of tag, value, and description. The tag will be + the value returned if the user hits "Submit", and the description + is the bale for the enumerated type. default is a list of all the + values (second elements in flags). If no defaults are specified, + initially nothing will be selected. Several items may be selected at + once. + """ + def __init__(self, name, flags=(), default=(), shortDesc=None, + longDesc=None, hints=None, allowNone=1): + self.flags = flags + Argument.__init__(self, name, default, shortDesc, longDesc, hints, allowNone=allowNone) + + def coerce(self, inFlagKeys): + if not inFlagKeys: + return [] + outFlags = [] + for inFlagKey in inFlagKeys: + for flagKey, flagVal, flagDesc in self.flags: + if inFlagKey == flagKey: + outFlags.append(flagVal) + break + else: + raise InputError("Invalid Flag: %s" % inFlagKey) + return outFlags + + +class CheckGroup(Flags): + pass + + +class RadioGroup(Choice): + pass + + +class Boolean(Argument): + def coerce(self, inVal): + if not inVal: + return 0 + lInVal = str(inVal).lower() + if lInVal in ('no', 'n', 'f', 'false', '0'): + return 0 + return 1 + +class File(Argument): + def __init__(self, name, allowNone=1, shortDesc=None, longDesc=None, + hints=None): + Argument.__init__(self, name, None, shortDesc, longDesc, hints, + allowNone=allowNone) + + def coerce(self, file): + if not file and self.allowNone: + return None + elif file: + return file + else: + raise InputError("Invalid File") + +def positiveInt(x): + x = int(x) + if x <= 0: raise ValueError + return x + +class Date(Argument): + """A date -- (year, month, day) tuple.""" + + defaultDefault = None + + def __init__(self, name, allowNone=1, default=None, shortDesc=None, + longDesc=None, hints=None): + Argument.__init__(self, name, default, shortDesc, longDesc, hints) + self.allowNone = allowNone + if not allowNone: + self.defaultDefault = (1970, 1, 1) + + def coerce(self, args): + """Return tuple of ints (year, month, day).""" + if tuple(args) == ("", "", "") and self.allowNone: + return None + + try: + year, month, day = map(positiveInt, args) + except ValueError: + raise InputError("Invalid date") + if (month, day) == (2, 29): + if not calendar.isleap(year): + raise InputError("%d was not a leap year" % year) + else: + return year, month, day + try: + mdays = calendar.mdays[month] + except IndexError: + raise InputError("Invalid date") + if day > mdays: + raise InputError("Invalid date") + return year, month, day + + +class Submit(Choice): + """Submit button or a reasonable facsimile thereof.""" + + def __init__(self, name, choices=[("Submit", "submit", "Submit form")], + reset=0, shortDesc=None, longDesc=None, allowNone=0, hints=None): + Choice.__init__(self, name, choices=choices, shortDesc=shortDesc, + longDesc=longDesc, hints=hints) + self.allowNone = allowNone + self.reset = reset + + def coerce(self, value): + if self.allowNone and not value: + return None + else: + return Choice.coerce(self, value) + + + +@_oldStyle +class PresentationHint: + """ + A hint to a particular system. + """ + + + +@_oldStyle +class MethodSignature: + """ + A signature of a callable. + """ + + def __init__(self, *sigList): + """ + """ + self.methodSignature = sigList + + def getArgument(self, name): + for a in self.methodSignature: + if a.name == name: + return a + + def method(self, callable, takesRequest=False): + return FormMethod(self, callable, takesRequest) + + + +@_oldStyle +class FormMethod: + """A callable object with a signature.""" + + def __init__(self, signature, callable, takesRequest=False): + self.signature = signature + self.callable = callable + self.takesRequest = takesRequest + + def getArgs(self): + return tuple(self.signature.methodSignature) + + def call(self,*args,**kw): + return self.callable(*args,**kw) diff --git a/contrib/python/Twisted/py2/twisted/python/hook.py b/contrib/python/Twisted/py2/twisted/python/hook.py new file mode 100644 index 00000000000..c829f7320b9 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/hook.py @@ -0,0 +1,176 @@ + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + + +""" +I define support for hookable instance methods. + +These are methods which you can register pre-call and post-call external +functions to augment their functionality. People familiar with more esoteric +languages may think of these as \"method combinations\". + +This could be used to add optional preconditions, user-extensible callbacks +(a-la emacs) or a thread-safety mechanism. + +The four exported calls are: + + - L{addPre} + - L{addPost} + - L{removePre} + - L{removePost} + +All have the signature (class, methodName, callable), and the callable they +take must always have the signature (instance, *args, **kw) unless the +particular signature of the method they hook is known. + +Hooks should typically not throw exceptions, however, no effort will be made by +this module to prevent them from doing so. Pre-hooks will always be called, +but post-hooks will only be called if the pre-hooks do not raise any exceptions +(they will still be called if the main method raises an exception). The return +values and exception status of the main method will be propagated (assuming +none of the hooks raise an exception). Hooks will be executed in the order in +which they are added. +""" + + +### Public Interface + +class HookError(Exception): + "An error which will fire when an invariant is violated." + +def addPre(klass, name, func): + """hook.addPre(klass, name, func) -> None + + Add a function to be called before the method klass.name is invoked. + """ + + _addHook(klass, name, PRE, func) + +def addPost(klass, name, func): + """hook.addPost(klass, name, func) -> None + + Add a function to be called after the method klass.name is invoked. + """ + _addHook(klass, name, POST, func) + +def removePre(klass, name, func): + """hook.removePre(klass, name, func) -> None + + Remove a function (previously registered with addPre) so that it + is no longer executed before klass.name. + """ + + _removeHook(klass, name, PRE, func) + +def removePost(klass, name, func): + """hook.removePre(klass, name, func) -> None + + Remove a function (previously registered with addPost) so that it + is no longer executed after klass.name. + """ + _removeHook(klass, name, POST, func) + +### "Helper" functions. + +hooked_func = """ + +import %(module)s + +def %(name)s(*args, **kw): + klazz = %(module)s.%(klass)s + for preMethod in klazz.%(preName)s: + preMethod(*args, **kw) + try: + return klazz.%(originalName)s(*args, **kw) + finally: + for postMethod in klazz.%(postName)s: + postMethod(*args, **kw) +""" + +_PRE = '__hook_pre_%s_%s_%s__' +_POST = '__hook_post_%s_%s_%s__' +_ORIG = '__hook_orig_%s_%s_%s__' + + +def _XXX(k,n,s): + """ + String manipulation garbage. + """ + x = s % (k.__module__.replace('.', '_'), k.__name__, n) + return x + +def PRE(k,n): + "(private) munging to turn a method name into a pre-hook-method-name" + return _XXX(k,n,_PRE) + +def POST(k,n): + "(private) munging to turn a method name into a post-hook-method-name" + return _XXX(k,n,_POST) + +def ORIG(k,n): + "(private) munging to turn a method name into an `original' identifier" + return _XXX(k,n,_ORIG) + + +def _addHook(klass, name, phase, func): + "(private) adds a hook to a method on a class" + _enhook(klass, name) + + if not hasattr(klass, phase(klass, name)): + setattr(klass, phase(klass, name), []) + + phaselist = getattr(klass, phase(klass, name)) + phaselist.append(func) + + +def _removeHook(klass, name, phase, func): + "(private) removes a hook from a method on a class" + phaselistname = phase(klass, name) + if not hasattr(klass, ORIG(klass,name)): + raise HookError("no hooks present!") + + phaselist = getattr(klass, phaselistname) + try: phaselist.remove(func) + except ValueError: + raise HookError("hook %s not found in removal list for %s"% + (name,klass)) + + if not getattr(klass, PRE(klass,name)) and not getattr(klass, POST(klass, name)): + _dehook(klass, name) + +def _enhook(klass, name): + "(private) causes a certain method name to be hooked on a class" + if hasattr(klass, ORIG(klass, name)): + return + + def newfunc(*args, **kw): + for preMethod in getattr(klass, PRE(klass, name)): + preMethod(*args, **kw) + try: + return getattr(klass, ORIG(klass, name))(*args, **kw) + finally: + for postMethod in getattr(klass, POST(klass, name)): + postMethod(*args, **kw) + try: + newfunc.func_name = name + except TypeError: + # Older python's don't let you do this + pass + + oldfunc = getattr(klass, name).im_func + setattr(klass, ORIG(klass, name), oldfunc) + setattr(klass, PRE(klass, name), []) + setattr(klass, POST(klass, name), []) + setattr(klass, name, newfunc) + +def _dehook(klass, name): + "(private) causes a certain method name no longer to be hooked on a class" + + if not hasattr(klass, ORIG(klass, name)): + raise HookError("Cannot unhook!") + setattr(klass, name, getattr(klass, ORIG(klass,name))) + delattr(klass, PRE(klass,name)) + delattr(klass, POST(klass,name)) + delattr(klass, ORIG(klass,name)) diff --git a/contrib/python/Twisted/py2/twisted/python/htmlizer.py b/contrib/python/Twisted/py2/twisted/python/htmlizer.py new file mode 100644 index 00000000000..b5ff660c7b3 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/htmlizer.py @@ -0,0 +1,131 @@ +# -*- test-case-name: twisted.python.test.test_htmlizer -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HTML rendering of Python source. +""" + +from twisted.python.compat import _tokenize, escape + +import tokenize, keyword +from . import reflect +from twisted.python._oldstyle import _oldStyle + + +@_oldStyle +class TokenPrinter: + """ + Format a stream of tokens and intermediate whitespace, for pretty-printing. + """ + + currentCol, currentLine = 0, 1 + lastIdentifier = parameters = 0 + encoding = "utf-8" + + def __init__(self, writer): + """ + @param writer: A file-like object, opened in bytes mode. + """ + self.writer = writer + + + def printtoken(self, type, token, sCoordinates, eCoordinates, line): + if hasattr(tokenize, "ENCODING") and type == tokenize.ENCODING: + self.encoding = token + return + + if not isinstance(token, bytes): + token = token.encode(self.encoding) + + (srow, scol) = sCoordinates + (erow, ecol) = eCoordinates + if self.currentLine < srow: + self.writer(b'\n' * (srow-self.currentLine)) + self.currentLine, self.currentCol = srow, 0 + self.writer(b' ' * (scol-self.currentCol)) + if self.lastIdentifier: + type = "identifier" + self.parameters = 1 + elif type == tokenize.NAME: + if keyword.iskeyword(token): + type = 'keyword' + else: + if self.parameters: + type = 'parameter' + else: + type = 'variable' + else: + type = tokenize.tok_name.get(type).lower() + self.writer(token, type) + self.currentCol = ecol + self.currentLine += token.count(b'\n') + if self.currentLine != erow: + self.currentCol = 0 + self.lastIdentifier = token in (b'def', b'class') + if token == b':': + self.parameters = 0 + + + +@_oldStyle +class HTMLWriter: + """ + Write the stream of tokens and whitespace from L{TokenPrinter}, formating + tokens as HTML spans. + """ + + noSpan = [] + + def __init__(self, writer): + self.writer = writer + noSpan = [] + reflect.accumulateClassList(self.__class__, "noSpan", noSpan) + self.noSpan = noSpan + + + def write(self, token, type=None): + if isinstance(token, bytes): + token = token.decode("utf-8") + token = escape(token) + token = token.encode("utf-8") + if (type is None) or (type in self.noSpan): + self.writer(token) + else: + self.writer( + b'' + + token + b'') + + + +class SmallerHTMLWriter(HTMLWriter): + """ + HTMLWriter that doesn't generate spans for some junk. + + Results in much smaller HTML output. + """ + noSpan = ["endmarker", "indent", "dedent", "op", "newline", "nl"] + + + +def filter(inp, out, writer=HTMLWriter): + out.write(b'
')
+    printer = TokenPrinter(writer(out.write).write).printtoken
+    try:
+        for token in _tokenize(inp.readline):
+            (tokenType, string, start, end, line) = token
+            printer(tokenType, string, start, end, line)
+    except tokenize.TokenError:
+        pass
+    out.write(b'
\n') + + + +def main(): + import sys + stdout = getattr(sys.stdout, "buffer", sys.stdout) + with open(sys.argv[1], "rb") as f: + filter(f, stdout) + +if __name__ == '__main__': + main() diff --git a/contrib/python/Twisted/py2/twisted/python/lockfile.py b/contrib/python/Twisted/py2/twisted/python/lockfile.py new file mode 100644 index 00000000000..9cc42c1f79a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/lockfile.py @@ -0,0 +1,248 @@ +# -*- test-case-name: twisted.test.test_lockfile -*- +# Copyright (c) 2005 Divmod, Inc. +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Filesystem-based interprocess mutex. +""" + +from __future__ import absolute_import, division + +import errno +import os + +from time import time as _uniquefloat + +from twisted.python.runtime import platform +from twisted.python.compat import _PY3 + +def unique(): + return str(int(_uniquefloat() * 1000)) + +from os import rename + +if not platform.isWindows(): + from os import kill + from os import symlink + from os import readlink + from os import remove as rmlink + _windows = False +else: + _windows = True + + # On UNIX, a symlink can be made to a nonexistent location, and + # FilesystemLock uses this by making the target of the symlink an + # imaginary, non-existing file named that of the PID of the process with + # the lock. This has some benefits on UNIX -- making and removing this + # symlink is atomic. However, because Windows doesn't support symlinks (at + # least as how we know them), we have to fake this and actually write a + # file with the PID of the process holding the lock instead. + # These functions below perform that unenviable, probably-fraught-with- + # race-conditions duty. - hawkie + + try: + from win32api import OpenProcess + import pywintypes + except ImportError: + kill = None + else: + ERROR_ACCESS_DENIED = 5 + ERROR_INVALID_PARAMETER = 87 + + def kill(pid, signal): + try: + OpenProcess(0, 0, pid) + except pywintypes.error as e: + if e.args[0] == ERROR_ACCESS_DENIED: + return + elif e.args[0] == ERROR_INVALID_PARAMETER: + raise OSError(errno.ESRCH, None) + raise + else: + raise RuntimeError("OpenProcess is required to fail.") + + # For monkeypatching in tests + _open = open + + + def symlink(value, filename): + """ + Write a file at C{filename} with the contents of C{value}. See the + above comment block as to why this is needed. + """ + # XXX Implement an atomic thingamajig for win32 + newlinkname = filename + "." + unique() + '.newlink' + newvalname = os.path.join(newlinkname, "symlink") + os.mkdir(newlinkname) + + # Python 3 does not support the 'commit' flag of fopen in the MSVCRT + # (http://msdn.microsoft.com/en-us/library/yeby3zcb%28VS.71%29.aspx) + if _PY3: + mode = 'w' + else: + mode = 'wc' + + with _open(newvalname, mode) as f: + f.write(value) + f.flush() + + try: + rename(newlinkname, filename) + except: + os.remove(newvalname) + os.rmdir(newlinkname) + raise + + + def readlink(filename): + """ + Read the contents of C{filename}. See the above comment block as to why + this is needed. + """ + try: + fObj = _open(os.path.join(filename, 'symlink'), 'r') + except IOError as e: + if e.errno == errno.ENOENT or e.errno == errno.EIO: + raise OSError(e.errno, None) + raise + else: + with fObj: + result = fObj.read() + return result + + + def rmlink(filename): + os.remove(os.path.join(filename, 'symlink')) + os.rmdir(filename) + + + +class FilesystemLock(object): + """ + A mutex. + + This relies on the filesystem property that creating + a symlink is an atomic operation and that it will + fail if the symlink already exists. Deleting the + symlink will release the lock. + + @ivar name: The name of the file associated with this lock. + + @ivar clean: Indicates whether this lock was released cleanly by its + last owner. Only meaningful after C{lock} has been called and + returns True. + + @ivar locked: Indicates whether the lock is currently held by this + object. + """ + + clean = None + locked = False + + def __init__(self, name): + self.name = name + + + def lock(self): + """ + Acquire this lock. + + @rtype: C{bool} + @return: True if the lock is acquired, false otherwise. + + @raise: Any exception os.symlink() may raise, other than + EEXIST. + """ + clean = True + while True: + try: + symlink(str(os.getpid()), self.name) + except OSError as e: + if _windows and e.errno in (errno.EACCES, errno.EIO): + # The lock is in the middle of being deleted because we're + # on Windows where lock removal isn't atomic. Give up, we + # don't know how long this is going to take. + return False + if e.errno == errno.EEXIST: + try: + pid = readlink(self.name) + except (IOError, OSError) as e: + if e.errno == errno.ENOENT: + # The lock has vanished, try to claim it in the + # next iteration through the loop. + continue + elif _windows and e.errno == errno.EACCES: + # The lock is in the middle of being + # deleted because we're on Windows where + # lock removal isn't atomic. Give up, we + # don't know how long this is going to + # take. + return False + raise + try: + if kill is not None: + kill(int(pid), 0) + except OSError as e: + if e.errno == errno.ESRCH: + # The owner has vanished, try to claim it in the + # next iteration through the loop. + try: + rmlink(self.name) + except OSError as e: + if e.errno == errno.ENOENT: + # Another process cleaned up the lock. + # Race them to acquire it in the next + # iteration through the loop. + continue + raise + clean = False + continue + raise + return False + raise + self.locked = True + self.clean = clean + return True + + + def unlock(self): + """ + Release this lock. + + This deletes the directory with the given name. + + @raise: Any exception os.readlink() may raise, or + ValueError if the lock is not owned by this process. + """ + pid = readlink(self.name) + if int(pid) != os.getpid(): + raise ValueError( + "Lock %r not owned by this process" % (self.name,)) + rmlink(self.name) + self.locked = False + + + +def isLocked(name): + """ + Determine if the lock of the given name is held or not. + + @type name: C{str} + @param name: The filesystem path to the lock to test + + @rtype: C{bool} + @return: True if the lock is held, False otherwise. + """ + l = FilesystemLock(name) + result = None + try: + result = l.lock() + finally: + if result: + l.unlock() + return not result + + + +__all__ = ['FilesystemLock', 'isLocked'] diff --git a/contrib/python/Twisted/py2/twisted/python/log.py b/contrib/python/Twisted/py2/twisted/python/log.py new file mode 100644 index 00000000000..bcd50598bcd --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/log.py @@ -0,0 +1,767 @@ +# -*- test-case-name: twisted.test.test_log -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Logging and metrics infrastructure. +""" + +from __future__ import division, absolute_import + +import sys +import time +import warnings + +from datetime import datetime + +from zope.interface import Interface + +from twisted.python.compat import unicode, _PY3 +from twisted.python import context +from twisted.python import reflect +from twisted.python import util +from twisted.python import failure +from twisted.python._oldstyle import _oldStyle +from twisted.python.threadable import synchronize +from twisted.logger import ( + Logger as NewLogger, LogLevel as NewLogLevel, + STDLibLogObserver as NewSTDLibLogObserver, + LegacyLogObserverWrapper, LoggingFile, LogPublisher as NewPublisher, + globalLogPublisher as newGlobalLogPublisher, + globalLogBeginner as newGlobalLogBeginner, +) + +from twisted.logger._global import LogBeginner +from twisted.logger._legacy import publishToNewObserver as _publishNew + + + +@_oldStyle +class ILogContext: + """ + Actually, this interface is just a synonym for the dictionary interface, + but it serves as a key for the default information in a log. + + I do not inherit from C{Interface} because the world is a cruel place. + """ + + + +class ILogObserver(Interface): + """ + An observer which can do something with log events. + + Given that most log observers are actually bound methods, it's okay to not + explicitly declare provision of this interface. + """ + def __call__(eventDict): + """ + Log an event. + + @type eventDict: C{dict} with C{str} keys. + @param eventDict: A dictionary with arbitrary keys. However, these + keys are often available: + - C{message}: A C{tuple} of C{str} containing messages to be + logged. + - C{system}: A C{str} which indicates the "system" which is + generating this event. + - C{isError}: A C{bool} indicating whether this event represents + an error. + - C{failure}: A L{failure.Failure} instance + - C{why}: Used as header of the traceback in case of errors. + - C{format}: A string format used in place of C{message} to + customize the event. The intent is for the observer to format + a message by doing something like C{format % eventDict}. + """ + + + +context.setDefault(ILogContext, + {"system": "-"}) + + +def callWithContext(ctx, func, *args, **kw): + newCtx = context.get(ILogContext).copy() + newCtx.update(ctx) + return context.call({ILogContext: newCtx}, func, *args, **kw) + + + +def callWithLogger(logger, func, *args, **kw): + """ + Utility method which wraps a function in a try:/except:, logs a failure if + one occurs, and uses the system's logPrefix. + """ + try: + lp = logger.logPrefix() + except KeyboardInterrupt: + raise + except: + lp = '(buggy logPrefix method)' + err(system=lp) + try: + return callWithContext({"system": lp}, func, *args, **kw) + except KeyboardInterrupt: + raise + except: + err(system=lp) + + + +def err(_stuff=None, _why=None, **kw): + """ + Write a failure to the log. + + The C{_stuff} and C{_why} parameters use an underscore prefix to lessen + the chance of colliding with a keyword argument the application wishes + to pass. It is intended that they be supplied with arguments passed + positionally, not by keyword. + + @param _stuff: The failure to log. If C{_stuff} is L{None} a new + L{Failure} will be created from the current exception state. If + C{_stuff} is an C{Exception} instance it will be wrapped in a + L{Failure}. + @type _stuff: L{None}, C{Exception}, or L{Failure}. + + @param _why: The source of this failure. This will be logged along with + C{_stuff} and should describe the context in which the failure + occurred. + @type _why: C{str} + """ + if _stuff is None: + _stuff = failure.Failure() + if isinstance(_stuff, failure.Failure): + msg(failure=_stuff, why=_why, isError=1, **kw) + elif isinstance(_stuff, Exception): + msg(failure=failure.Failure(_stuff), why=_why, isError=1, **kw) + else: + msg(repr(_stuff), why=_why, isError=1, **kw) + +deferr = err + + +@_oldStyle +class Logger: + """ + This represents a class which may 'own' a log. Used by subclassing. + """ + def logPrefix(self): + """ + Override this method to insert custom logging behavior. Its + return value will be inserted in front of every line. It may + be called more times than the number of output lines. + """ + return '-' + + + +@_oldStyle +class LogPublisher: + """ + Class for singleton log message publishing. + """ + + synchronized = ['msg'] + + + def __init__(self, observerPublisher=None, publishPublisher=None, + logBeginner=None, warningsModule=warnings): + if publishPublisher is None: + publishPublisher = NewPublisher() + if observerPublisher is None: + observerPublisher = publishPublisher + if observerPublisher is None: + observerPublisher = NewPublisher() + self._observerPublisher = observerPublisher + self._publishPublisher = publishPublisher + self._legacyObservers = [] + if logBeginner is None: + # This default behavior is really only used for testing. + beginnerPublisher = NewPublisher() + beginnerPublisher.addObserver(observerPublisher) + logBeginner = LogBeginner(beginnerPublisher, NullFile(), sys, + warnings) + self._logBeginner = logBeginner + self._warningsModule = warningsModule + self._oldshowwarning = warningsModule.showwarning + self.showwarning = self._logBeginner.showwarning + + + @property + def observers(self): + """ + Property returning all observers registered on this L{LogPublisher}. + + @return: observers previously added with L{LogPublisher.addObserver} + @rtype: L{list} of L{callable} + """ + return [x.legacyObserver for x in self._legacyObservers] + + + def _startLogging(self, other, setStdout): + """ + Begin logging to the L{LogBeginner} associated with this + L{LogPublisher}. + + @param other: the observer to log to. + @type other: L{LogBeginner} + + @param setStdout: if true, send standard I/O to the observer as well. + @type setStdout: L{bool} + """ + wrapped = LegacyLogObserverWrapper(other) + self._legacyObservers.append(wrapped) + self._logBeginner.beginLoggingTo([wrapped], True, setStdout) + + + def _stopLogging(self): + """ + Clean-up hook for fixing potentially global state. Only for testing of + this module itself. If you want less global state, use the new + warnings system in L{twisted.logger}. + """ + if self._warningsModule.showwarning == self.showwarning: + self._warningsModule.showwarning = self._oldshowwarning + + + def addObserver(self, other): + """ + Add a new observer. + + @type other: Provider of L{ILogObserver} + @param other: A callable object that will be called with each new log + message (a dict). + """ + wrapped = LegacyLogObserverWrapper(other) + self._legacyObservers.append(wrapped) + self._observerPublisher.addObserver(wrapped) + + + def removeObserver(self, other): + """ + Remove an observer. + """ + for observer in self._legacyObservers: + if observer.legacyObserver == other: + self._legacyObservers.remove(observer) + self._observerPublisher.removeObserver(observer) + break + + + def msg(self, *message, **kw): + """ + Log a new message. + + The message should be a native string, i.e. bytes on Python 2 and + Unicode on Python 3. For compatibility with both use the native string + syntax, for example:: + + >>> log.msg('Hello, world.') + + You MUST avoid passing in Unicode on Python 2, and the form:: + + >>> log.msg('Hello ', 'world.') + + This form only works (sometimes) by accident. + + Keyword arguments will be converted into items in the event + dict that is passed to L{ILogObserver} implementations. + Each implementation, in turn, can define keys that are used + by it specifically, in addition to common keys listed at + L{ILogObserver.__call__}. + + For example, to set the C{system} parameter while logging + a message:: + + >>> log.msg('Started', system='Foo') + + """ + actualEventDict = (context.get(ILogContext) or {}).copy() + actualEventDict.update(kw) + actualEventDict['message'] = message + actualEventDict['time'] = time.time() + if "isError" not in actualEventDict: + actualEventDict["isError"] = 0 + + _publishNew(self._publishPublisher, actualEventDict, textFromEventDict) + + +synchronize(LogPublisher) + + + +if 'theLogPublisher' not in globals(): + def _actually(something): + """ + A decorator that returns its argument rather than the thing it is + decorating. + + This allows the documentation generator to see an alias for a method or + constant as an object with a docstring and thereby document it and + allow references to it statically. + + @param something: An object to create an alias for. + @type something: L{object} + + @return: a 1-argument callable that returns C{something} + @rtype: L{object} + """ + def decorate(thingWithADocstring): + return something + return decorate + + theLogPublisher = LogPublisher( + observerPublisher=newGlobalLogPublisher, + publishPublisher=newGlobalLogPublisher, + logBeginner=newGlobalLogBeginner, + ) + + + @_actually(theLogPublisher.addObserver) + def addObserver(observer): + """ + Add a log observer to the global publisher. + + @see: L{LogPublisher.addObserver} + + @param observer: a log observer + @type observer: L{callable} + """ + + + @_actually(theLogPublisher.removeObserver) + def removeObserver(observer): + """ + Remove a log observer from the global publisher. + + @see: L{LogPublisher.removeObserver} + + @param observer: a log observer previously added with L{addObserver} + @type observer: L{callable} + """ + + + @_actually(theLogPublisher.msg) + def msg(*message, **event): + """ + Publish a message to the global log publisher. + + @see: L{LogPublisher.msg} + + @param message: the log message + @type message: C{tuple} of L{str} (native string) + + @param event: fields for the log event + @type event: L{dict} mapping L{str} (native string) to L{object} + """ + + + @_actually(theLogPublisher.showwarning) + def showwarning(): + """ + Publish a Python warning through the global log publisher. + + @see: L{LogPublisher.showwarning} + """ + + + +def _safeFormat(fmtString, fmtDict): + """ + Try to format a string, swallowing all errors to always return a string. + + @note: For backward-compatibility reasons, this function ensures that it + returns a native string, meaning C{bytes} in Python 2 and C{unicode} in + Python 3. + + @param fmtString: a C{%}-format string + + @param fmtDict: string formatting arguments for C{fmtString} + + @return: A native string, formatted from C{fmtString} and C{fmtDict}. + @rtype: L{str} + """ + # There's a way we could make this if not safer at least more + # informative: perhaps some sort of str/repr wrapper objects + # could be wrapped around the things inside of C{fmtDict}. That way + # if the event dict contains an object with a bad __repr__, we + # can only cry about that individual object instead of the + # entire event dict. + try: + text = fmtString % fmtDict + except KeyboardInterrupt: + raise + except: + try: + text = ('Invalid format string or unformattable object in ' + 'log message: %r, %s' % (fmtString, fmtDict)) + except: + try: + text = ('UNFORMATTABLE OBJECT WRITTEN TO LOG with fmt %r, ' + 'MESSAGE LOST' % (fmtString,)) + except: + text = ('PATHOLOGICAL ERROR IN BOTH FORMAT STRING AND ' + 'MESSAGE DETAILS, MESSAGE LOST') + + # Return a native string + if _PY3: + if isinstance(text, bytes): + text = text.decode("utf-8") + else: + if isinstance(text, unicode): + text = text.encode("utf-8") + + return text + + + +def textFromEventDict(eventDict): + """ + Extract text from an event dict passed to a log observer. If it cannot + handle the dict, it returns None. + + The possible keys of eventDict are: + - C{message}: by default, it holds the final text. It's required, but can + be empty if either C{isError} or C{format} is provided (the first + having the priority). + - C{isError}: boolean indicating the nature of the event. + - C{failure}: L{failure.Failure} instance, required if the event is an + error. + - C{why}: if defined, used as header of the traceback in case of errors. + - C{format}: string format used in place of C{message} to customize + the event. It uses all keys present in C{eventDict} to format + the text. + Other keys will be used when applying the C{format}, or ignored. + """ + edm = eventDict['message'] + if not edm: + if eventDict['isError'] and 'failure' in eventDict: + why = eventDict.get('why') + if why: + why = reflect.safe_str(why) + else: + why = 'Unhandled Error' + try: + traceback = eventDict['failure'].getTraceback() + except Exception as e: + traceback = '(unable to obtain traceback): ' + str(e) + text = (why + '\n' + traceback) + elif 'format' in eventDict: + text = _safeFormat(eventDict['format'], eventDict) + else: + # We don't know how to log this + return None + else: + text = ' '.join(map(reflect.safe_str, edm)) + return text + + + +@_oldStyle +class _GlobalStartStopMixIn: + """ + Mix-in for global log observers that can start and stop. + """ + + def start(self): + """ + Start observing log events. + """ + addObserver(self.emit) + + + def stop(self): + """ + Stop observing log events. + """ + removeObserver(self.emit) + + + +class FileLogObserver(_GlobalStartStopMixIn): + """ + Log observer that writes to a file-like object. + + @type timeFormat: C{str} or L{None} + @ivar timeFormat: If not L{None}, the format string passed to strftime(). + """ + + timeFormat = None + + def __init__(self, f): + # Compatibility + self.write = f.write + self.flush = f.flush + + + def getTimezoneOffset(self, when): + """ + Return the current local timezone offset from UTC. + + @type when: C{int} + @param when: POSIX (ie, UTC) timestamp for which to find the offset. + + @rtype: C{int} + @return: The number of seconds offset from UTC. West is positive, + east is negative. + """ + offset = datetime.utcfromtimestamp(when) - datetime.fromtimestamp(when) + return offset.days * (60 * 60 * 24) + offset.seconds + + + def formatTime(self, when): + """ + Format the given UTC value as a string representing that time in the + local timezone. + + By default it's formatted as an ISO8601-like string (ISO8601 date and + ISO8601 time separated by a space). It can be customized using the + C{timeFormat} attribute, which will be used as input for the underlying + L{datetime.datetime.strftime} call. + + @type when: C{int} + @param when: POSIX (ie, UTC) timestamp for which to find the offset. + + @rtype: C{str} + """ + if self.timeFormat is not None: + return datetime.fromtimestamp(when).strftime(self.timeFormat) + + tzOffset = -self.getTimezoneOffset(when) + when = datetime.utcfromtimestamp(when + tzOffset) + tzHour = abs(int(tzOffset / 60 / 60)) + tzMin = abs(int(tzOffset / 60 % 60)) + if tzOffset < 0: + tzSign = '-' + else: + tzSign = '+' + return '%d-%02d-%02d %02d:%02d:%02d%s%02d%02d' % ( + when.year, when.month, when.day, + when.hour, when.minute, when.second, + tzSign, tzHour, tzMin) + + + def emit(self, eventDict): + """ + Format the given log event as text and write it to the output file. + + @param eventDict: a log event + @type eventDict: L{dict} mapping L{str} (native string) to L{object} + """ + text = textFromEventDict(eventDict) + if text is None: + return + + timeStr = self.formatTime(eventDict["time"]) + fmtDict = { + "system": eventDict["system"], + "text": text.replace("\n", "\n\t") + } + msgStr = _safeFormat("[%(system)s] %(text)s\n", fmtDict) + + util.untilConcludes(self.write, timeStr + " " + msgStr) + util.untilConcludes(self.flush) # Hoorj! + + + +class PythonLoggingObserver(_GlobalStartStopMixIn, object): + """ + Output twisted messages to Python standard library L{logging} module. + + WARNING: specific logging configurations (example: network) can lead to + a blocking system. Nothing is done here to prevent that, so be sure to not + use this: code within Twisted, such as twisted.web, assumes that logging + does not block. + """ + + def __init__(self, loggerName="twisted"): + """ + @param loggerName: identifier used for getting logger. + @type loggerName: C{str} + """ + self._newObserver = NewSTDLibLogObserver(loggerName) + + + def emit(self, eventDict): + """ + Receive a twisted log entry, format it and bridge it to python. + + By default the logging level used is info; log.err produces error + level, and you can customize the level by using the C{logLevel} key:: + + >>> log.msg('debugging', logLevel=logging.DEBUG) + """ + if 'log_format' in eventDict: + _publishNew(self._newObserver, eventDict, textFromEventDict) + + + +@_oldStyle +class StdioOnnaStick: + """ + Class that pretends to be stdout/err, and turns writes into log messages. + + @ivar isError: boolean indicating whether this is stderr, in which cases + log messages will be logged as errors. + + @ivar encoding: unicode encoding used to encode any unicode strings + written to this object. + """ + + closed = 0 + softspace = 0 + mode = 'wb' + name = '' + + def __init__(self, isError=0, encoding=None): + self.isError = isError + if encoding is None: + encoding = sys.getdefaultencoding() + self.encoding = encoding + self.buf = '' + + + def close(self): + pass + + + def fileno(self): + return -1 + + + def flush(self): + pass + + + def read(self): + raise IOError("can't read from the log!") + + readline = read + readlines = read + seek = read + tell = read + + + def write(self, data): + if not _PY3 and isinstance(data, unicode): + data = data.encode(self.encoding) + d = (self.buf + data).split('\n') + self.buf = d[-1] + messages = d[0:-1] + for message in messages: + msg(message, printed=1, isError=self.isError) + + + def writelines(self, lines): + for line in lines: + if not _PY3 and isinstance(line, unicode): + line = line.encode(self.encoding) + msg(line, printed=1, isError=self.isError) + + + +def startLogging(file, *a, **kw): + """ + Initialize logging to a specified file. + + @return: A L{FileLogObserver} if a new observer is added, None otherwise. + """ + if isinstance(file, LoggingFile): + return + flo = FileLogObserver(file) + startLoggingWithObserver(flo.emit, *a, **kw) + return flo + + + +def startLoggingWithObserver(observer, setStdout=1): + """ + Initialize logging to a specified observer. If setStdout is true + (defaults to yes), also redirect sys.stdout and sys.stderr + to the specified file. + """ + theLogPublisher._startLogging(observer, setStdout) + msg("Log opened.") + + + +@_oldStyle +class NullFile: + """ + A file-like object that discards everything. + """ + softspace = 0 + + def read(self): + """ + Do nothing. + """ + + + def write(self, bytes): + """ + Do nothing. + + @param bytes: data + @type bytes: L{bytes} + """ + + + def flush(self): + """ + Do nothing. + """ + + + def close(self): + """ + Do nothing. + """ + + + +def discardLogs(): + """ + Discard messages logged via the global C{logfile} object. + """ + global logfile + logfile = NullFile() + + + +# Prevent logfile from being erased on reload. This only works in cpython. +if 'logfile' not in globals(): + logfile = LoggingFile(logger=NewLogger(), + level=NewLogLevel.info, + encoding=getattr(sys.stdout, "encoding", None)) + logerr = LoggingFile(logger=NewLogger(), + level=NewLogLevel.error, + encoding=getattr(sys.stderr, "encoding", None)) + + + +class DefaultObserver(_GlobalStartStopMixIn): + """ + Default observer. + + Will ignore all non-error messages and send error messages to sys.stderr. + Will be removed when startLogging() is called for the first time. + """ + stderr = sys.stderr + + def emit(self, eventDict): + """ + Emit an event dict. + + @param eventDict: an event dict + @type eventDict: dict + """ + if eventDict["isError"]: + text = textFromEventDict(eventDict) + self.stderr.write(text) + self.stderr.flush() + + + +if 'defaultObserver' not in globals(): + defaultObserver = DefaultObserver() diff --git a/contrib/python/Twisted/py2/twisted/python/logfile.py b/contrib/python/Twisted/py2/twisted/python/logfile.py new file mode 100644 index 00000000000..378fc6c784b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/logfile.py @@ -0,0 +1,340 @@ +# -*- test-case-name: twisted.test.test_logfile -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A rotating, browsable log file. +""" + +from __future__ import division, absolute_import + +# System Imports +import os, glob, time, stat + +from twisted.python import threadable +from twisted.python._oldstyle import _oldStyle +from twisted.python.compat import unicode + + + +@_oldStyle +class BaseLogFile: + """ + The base class for a log file that can be rotated. + """ + + synchronized = ["write", "rotate"] + + def __init__(self, name, directory, defaultMode=None): + """ + Create a log file. + + @param name: name of the file + @param directory: directory holding the file + @param defaultMode: permissions used to create the file. Default to + current permissions of the file if the file exists. + """ + self.directory = directory + self.name = name + self.path = os.path.join(directory, name) + if defaultMode is None and os.path.exists(self.path): + self.defaultMode = stat.S_IMODE(os.stat(self.path)[stat.ST_MODE]) + else: + self.defaultMode = defaultMode + self._openFile() + + + def fromFullPath(cls, filename, *args, **kwargs): + """ + Construct a log file from a full file path. + """ + logPath = os.path.abspath(filename) + return cls(os.path.basename(logPath), + os.path.dirname(logPath), *args, **kwargs) + fromFullPath = classmethod(fromFullPath) + + + def shouldRotate(self): + """ + Override with a method to that returns true if the log + should be rotated. + """ + raise NotImplementedError + + + def _openFile(self): + """ + Open the log file. + + The log file is always opened in binary mode. + """ + self.closed = False + if os.path.exists(self.path): + self._file = open(self.path, "rb+", 0) + self._file.seek(0, 2) + else: + if self.defaultMode is not None: + # Set the lowest permissions + oldUmask = os.umask(0o777) + try: + self._file = open(self.path, "wb+", 0) + finally: + os.umask(oldUmask) + else: + self._file = open(self.path, "wb+", 0) + if self.defaultMode is not None: + try: + os.chmod(self.path, self.defaultMode) + except OSError: + # Probably /dev/null or something? + pass + + + def write(self, data): + """ + Write some data to the file. + + @param data: The data to write. Text will be encoded as UTF-8. + @type data: L{bytes} or L{unicode} + """ + if self.shouldRotate(): + self.flush() + self.rotate() + if isinstance(data, unicode): + data = data.encode('utf8') + self._file.write(data) + + + def flush(self): + """ + Flush the file. + """ + self._file.flush() + + + def close(self): + """ + Close the file. + + The file cannot be used once it has been closed. + """ + self.closed = True + self._file.close() + self._file = None + + + def reopen(self): + """ + Reopen the log file. This is mainly useful if you use an external log + rotation tool, which moves under your feet. + + Note that on Windows you probably need a specific API to rename the + file, as it's not supported to simply use os.rename, for example. + """ + self.close() + self._openFile() + + + def getCurrentLog(self): + """ + Return a LogReader for the current log file. + """ + return LogReader(self.path) + + +class LogFile(BaseLogFile): + """ + A log file that can be rotated. + + A rotateLength of None disables automatic log rotation. + """ + def __init__(self, name, directory, rotateLength=1000000, defaultMode=None, + maxRotatedFiles=None): + """ + Create a log file rotating on length. + + @param name: file name. + @type name: C{str} + @param directory: path of the log file. + @type directory: C{str} + @param rotateLength: size of the log file where it rotates. Default to + 1M. + @type rotateLength: C{int} + @param defaultMode: mode used to create the file. + @type defaultMode: C{int} + @param maxRotatedFiles: if not None, max number of log files the class + creates. Warning: it removes all log files above this number. + @type maxRotatedFiles: C{int} + """ + BaseLogFile.__init__(self, name, directory, defaultMode) + self.rotateLength = rotateLength + self.maxRotatedFiles = maxRotatedFiles + + def _openFile(self): + BaseLogFile._openFile(self) + self.size = self._file.tell() + + def shouldRotate(self): + """ + Rotate when the log file size is larger than rotateLength. + """ + return self.rotateLength and self.size >= self.rotateLength + + def getLog(self, identifier): + """ + Given an integer, return a LogReader for an old log file. + """ + filename = "%s.%d" % (self.path, identifier) + if not os.path.exists(filename): + raise ValueError("no such logfile exists") + return LogReader(filename) + + def write(self, data): + """ + Write some data to the file. + """ + BaseLogFile.write(self, data) + self.size += len(data) + + def rotate(self): + """ + Rotate the file and create a new one. + + If it's not possible to open new logfile, this will fail silently, + and continue logging to old logfile. + """ + if not (os.access(self.directory, os.W_OK) and os.access(self.path, os.W_OK)): + return + logs = self.listLogs() + logs.reverse() + for i in logs: + if self.maxRotatedFiles is not None and i >= self.maxRotatedFiles: + os.remove("%s.%d" % (self.path, i)) + else: + os.rename("%s.%d" % (self.path, i), "%s.%d" % (self.path, i + 1)) + self._file.close() + os.rename(self.path, "%s.1" % self.path) + self._openFile() + + def listLogs(self): + """ + Return sorted list of integers - the old logs' identifiers. + """ + result = [] + for name in glob.glob("%s.*" % self.path): + try: + counter = int(name.split('.')[-1]) + if counter: + result.append(counter) + except ValueError: + pass + result.sort() + return result + + def __getstate__(self): + state = BaseLogFile.__getstate__(self) + del state["size"] + return state + +threadable.synchronize(LogFile) + + + +class DailyLogFile(BaseLogFile): + """A log file that is rotated daily (at or after midnight localtime) + """ + def _openFile(self): + BaseLogFile._openFile(self) + self.lastDate = self.toDate(os.stat(self.path)[8]) + + def shouldRotate(self): + """Rotate when the date has changed since last write""" + return self.toDate() > self.lastDate + + def toDate(self, *args): + """Convert a unixtime to (year, month, day) localtime tuple, + or return the current (year, month, day) localtime tuple. + + This function primarily exists so you may overload it with + gmtime, or some cruft to make unit testing possible. + """ + # primarily so this can be unit tested easily + return time.localtime(*args)[:3] + + def suffix(self, tupledate): + """Return the suffix given a (year, month, day) tuple or unixtime""" + try: + return '_'.join(map(str, tupledate)) + except: + # try taking a float unixtime + return '_'.join(map(str, self.toDate(tupledate))) + + def getLog(self, identifier): + """Given a unix time, return a LogReader for an old log file.""" + if self.toDate(identifier) == self.lastDate: + return self.getCurrentLog() + filename = "%s.%s" % (self.path, self.suffix(identifier)) + if not os.path.exists(filename): + raise ValueError("no such logfile exists") + return LogReader(filename) + + def write(self, data): + """Write some data to the log file""" + BaseLogFile.write(self, data) + # Guard against a corner case where time.time() + # could potentially run backwards to yesterday. + # Primarily due to network time. + self.lastDate = max(self.lastDate, self.toDate()) + + def rotate(self): + """Rotate the file and create a new one. + + If it's not possible to open new logfile, this will fail silently, + and continue logging to old logfile. + """ + if not (os.access(self.directory, os.W_OK) and os.access(self.path, os.W_OK)): + return + newpath = "%s.%s" % (self.path, self.suffix(self.lastDate)) + if os.path.exists(newpath): + return + self._file.close() + os.rename(self.path, newpath) + self._openFile() + + def __getstate__(self): + state = BaseLogFile.__getstate__(self) + del state["lastDate"] + return state + +threadable.synchronize(DailyLogFile) + + +@_oldStyle +class LogReader: + """Read from a log file.""" + + def __init__(self, name): + """ + Open the log file for reading. + + The comments about binary-mode for L{BaseLogFile._openFile} also apply + here. + """ + self._file = open(name, "r") + + def readLines(self, lines=10): + """Read a list of lines from the log file. + + This doesn't returns all of the files lines - call it multiple times. + """ + result = [] + for i in range(lines): + line = self._file.readline() + if not line: + break + result.append(line) + return result + + def close(self): + self._file.close() diff --git a/contrib/python/Twisted/py2/twisted/python/modules.py b/contrib/python/Twisted/py2/twisted/python/modules.py new file mode 100644 index 00000000000..a8812aa7e30 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/modules.py @@ -0,0 +1,789 @@ +# -*- test-case-name: twisted.test.test_modules -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module aims to provide a unified, object-oriented view of Python's +runtime hierarchy. + +Python is a very dynamic language with wide variety of introspection utilities. +However, these utilities can be hard to use, because there is no consistent +API. The introspection API in python is made up of attributes (__name__, +__module__, func_name, etc) on instances, modules, classes and functions which +vary between those four types, utility modules such as 'inspect' which provide +some functionality, the 'imp' module, the "compiler" module, the semantics of +PEP 302 support, and setuptools, among other things. + +At the top, you have "PythonPath", an abstract representation of sys.path which +includes methods to locate top-level modules, with or without loading them. +The top-level exposed functions in this module for accessing the system path +are "walkModules", "iterModules", and "getModule". + +From most to least specific, here are the objects provided:: + + PythonPath # sys.path + | + v + PathEntry # one entry on sys.path: an importer + | + v + PythonModule # a module or package that can be loaded + | + v + PythonAttribute # an attribute of a module (function or class) + | + v + PythonAttribute # an attribute of a function or class + | + v + ... + +Here's an example of idiomatic usage: this is what you would do to list all of +the modules outside the standard library's python-files directory:: + + import os + stdlibdir = os.path.dirname(os.__file__) + + from twisted.python.modules import iterModules + + for modinfo in iterModules(): + if (modinfo.pathEntry.filePath.path != stdlibdir + and not modinfo.isPackage()): + print('unpackaged: %s: %s' % ( + modinfo.name, modinfo.filePath.path)) + +@var theSystemPath: The very top of the Python object space. +@type: L{PythonPath} +""" + +from __future__ import division, absolute_import, print_function + +__metaclass__ = type + +# let's try to keep path imports to a minimum... +from os.path import dirname, split as splitpath + +import sys +import inspect +import warnings +import zipimport + +from zope.interface import Interface, implementer + +from twisted.python.compat import nativeString +from twisted.python.components import registerAdapter +from twisted.python.filepath import FilePath, UnlistableError +from twisted.python.reflect import namedAny +from twisted.python.zippath import ZipArchive + + +_nothing = object() + +PYTHON_EXTENSIONS = ['.py'] +OPTIMIZED_MODE = __doc__ is None +if OPTIMIZED_MODE: + PYTHON_EXTENSIONS.append('.pyo') +else: + PYTHON_EXTENSIONS.append('.pyc') + +def _isPythonIdentifier(string): + """ + cheezy fake test for proper identifier-ness. + + @param string: a L{str} which might or might not be a valid python + identifier. + @return: True or False + """ + textString = nativeString(string) + return (' ' not in textString and + '.' not in textString and + '-' not in textString) + + + +def _isPackagePath(fpath): + # Determine if a FilePath-like object is a Python package. TODO: deal with + # __init__module.(so|dll|pyd)? + extless = fpath.splitext()[0] + basend = splitpath(extless)[1] + return basend == "__init__" + + + +class _ModuleIteratorHelper: + """ + This mixin provides common behavior between python module and path entries, + since the mechanism for searching sys.path and __path__ attributes is + remarkably similar. + """ + + def iterModules(self): + """ + Loop over the modules present below this entry or package on PYTHONPATH. + + For modules which are not packages, this will yield nothing. + + For packages and path entries, this will only yield modules one level + down; i.e. if there is a package a.b.c, iterModules on a will only + return a.b. If you want to descend deeply, use walkModules. + + @return: a generator which yields PythonModule instances that describe + modules which can be, or have been, imported. + """ + yielded = {} + if not self.filePath.exists(): + return + + for placeToLook in self._packagePaths(): + try: + children = sorted(placeToLook.children()) + except UnlistableError: + continue + + for potentialTopLevel in children: + ext = potentialTopLevel.splitext()[1] + potentialBasename = potentialTopLevel.basename()[:-len(ext)] + if ext in PYTHON_EXTENSIONS: + # TODO: this should be a little choosier about which path entry + # it selects first, and it should do all the .so checking and + # crud + if not _isPythonIdentifier(potentialBasename): + continue + modname = self._subModuleName(potentialBasename) + if modname.split(".")[-1] == '__init__': + # This marks the directory as a package so it can't be + # a module. + continue + if modname not in yielded: + yielded[modname] = True + pm = PythonModule(modname, potentialTopLevel, self._getEntry()) + assert pm != self + yield pm + else: + if (ext or not _isPythonIdentifier(potentialBasename) + or not potentialTopLevel.isdir()): + continue + modname = self._subModuleName(potentialTopLevel.basename()) + for ext in PYTHON_EXTENSIONS: + initpy = potentialTopLevel.child("__init__"+ext) + if initpy.exists() and modname not in yielded: + yielded[modname] = True + pm = PythonModule(modname, initpy, self._getEntry()) + assert pm != self + yield pm + break + + def walkModules(self, importPackages=False): + """ + Similar to L{iterModules}, this yields self, and then every module in my + package or entry, and every submodule in each package or entry. + + In other words, this is deep, and L{iterModules} is shallow. + """ + yield self + for package in self.iterModules(): + for module in package.walkModules(importPackages=importPackages): + yield module + + def _subModuleName(self, mn): + """ + This is a hook to provide packages with the ability to specify their names + as a prefix to submodules here. + """ + return mn + + def _packagePaths(self): + """ + Implement in subclasses to specify where to look for modules. + + @return: iterable of FilePath-like objects. + """ + raise NotImplementedError() + + def _getEntry(self): + """ + Implement in subclasses to specify what path entry submodules will come + from. + + @return: a PathEntry instance. + """ + raise NotImplementedError() + + + def __getitem__(self, modname): + """ + Retrieve a module from below this path or package. + + @param modname: a str naming a module to be loaded. For entries, this + is a top-level, undotted package name, and for packages it is the name + of the module without the package prefix. For example, if you have a + PythonModule representing the 'twisted' package, you could use:: + + twistedPackageObj['python']['modules'] + + to retrieve this module. + + @raise: KeyError if the module is not found. + + @return: a PythonModule. + """ + for module in self.iterModules(): + if module.name == self._subModuleName(modname): + return module + raise KeyError(modname) + + def __iter__(self): + """ + Implemented to raise NotImplementedError for clarity, so that attempting to + loop over this object won't call __getitem__. + + Note: in the future there might be some sensible default for iteration, + like 'walkEverything', so this is deliberately untested and undefined + behavior. + """ + raise NotImplementedError() + +class PythonAttribute: + """ + I represent a function, class, or other object that is present. + + @ivar name: the fully-qualified python name of this attribute. + + @ivar onObject: a reference to a PythonModule or other PythonAttribute that + is this attribute's logical parent. + + @ivar name: the fully qualified python name of the attribute represented by + this class. + """ + def __init__(self, name, onObject, loaded, pythonValue): + """ + Create a PythonAttribute. This is a private constructor. Do not construct + me directly, use PythonModule.iterAttributes. + + @param name: the FQPN + @param onObject: see ivar + @param loaded: always True, for now + @param pythonValue: the value of the attribute we're pointing to. + """ + self.name = name + self.onObject = onObject + self._loaded = loaded + self.pythonValue = pythonValue + + def __repr__(self): + return 'PythonAttribute<%r>'%(self.name,) + + def isLoaded(self): + """ + Return a boolean describing whether the attribute this describes has + actually been loaded into memory by importing its module. + + Note: this currently always returns true; there is no Python parser + support in this module yet. + """ + return self._loaded + + def load(self, default=_nothing): + """ + Load the value associated with this attribute. + + @return: an arbitrary Python object, or 'default' if there is an error + loading it. + """ + return self.pythonValue + + def iterAttributes(self): + for name, val in inspect.getmembers(self.load()): + yield PythonAttribute(self.name+'.'+name, self, True, val) + +class PythonModule(_ModuleIteratorHelper): + """ + Representation of a module which could be imported from sys.path. + + @ivar name: the fully qualified python name of this module. + + @ivar filePath: a FilePath-like object which points to the location of this + module. + + @ivar pathEntry: a L{PathEntry} instance which this module was located + from. + """ + + def __init__(self, name, filePath, pathEntry): + """ + Create a PythonModule. Do not construct this directly, instead inspect a + PythonPath or other PythonModule instances. + + @param name: see ivar + @param filePath: see ivar + @param pathEntry: see ivar + """ + _name = nativeString(name) + assert not _name.endswith(".__init__") + self.name = _name + self.filePath = filePath + self.parentPath = filePath.parent() + self.pathEntry = pathEntry + + def _getEntry(self): + return self.pathEntry + + def __repr__(self): + """ + Return a string representation including the module name. + """ + return 'PythonModule<%r>' % (self.name,) + + + def isLoaded(self): + """ + Determine if the module is loaded into sys.modules. + + @return: a boolean: true if loaded, false if not. + """ + return self.pathEntry.pythonPath.moduleDict.get(self.name) is not None + + + def iterAttributes(self): + """ + List all the attributes defined in this module. + + Note: Future work is planned here to make it possible to list python + attributes on a module without loading the module by inspecting ASTs or + bytecode, but currently any iteration of PythonModule objects insists + they must be loaded, and will use inspect.getmodule. + + @raise NotImplementedError: if this module is not loaded. + + @return: a generator yielding PythonAttribute instances describing the + attributes of this module. + """ + if not self.isLoaded(): + raise NotImplementedError( + "You can't load attributes from non-loaded modules yet.") + for name, val in inspect.getmembers(self.load()): + yield PythonAttribute(self.name+'.'+name, self, True, val) + + def isPackage(self): + """ + Returns true if this module is also a package, and might yield something + from iterModules. + """ + return _isPackagePath(self.filePath) + + def load(self, default=_nothing): + """ + Load this module. + + @param default: if specified, the value to return in case of an error. + + @return: a genuine python module. + + @raise: any type of exception. Importing modules is a risky business; + the erorrs of any code run at module scope may be raised from here, as + well as ImportError if something bizarre happened to the system path + between the discovery of this PythonModule object and the attempt to + import it. If you specify a default, the error will be swallowed + entirely, and not logged. + + @rtype: types.ModuleType. + """ + try: + return self.pathEntry.pythonPath.moduleLoader(self.name) + except: # this needs more thought... + if default is not _nothing: + return default + raise + + def __eq__(self, other): + """ + PythonModules with the same name are equal. + """ + if not isinstance(other, PythonModule): + return False + return other.name == self.name + + def __ne__(self, other): + """ + PythonModules with different names are not equal. + """ + if not isinstance(other, PythonModule): + return True + return other.name != self.name + + def walkModules(self, importPackages=False): + if importPackages and self.isPackage(): + self.load() + return super(PythonModule, self).walkModules(importPackages=importPackages) + + def _subModuleName(self, mn): + """ + submodules of this module are prefixed with our name. + """ + return self.name + '.' + mn + + def _packagePaths(self): + """ + Yield a sequence of FilePath-like objects which represent path segments. + """ + if not self.isPackage(): + return + if self.isLoaded(): + load = self.load() + if hasattr(load, '__path__'): + for fn in load.__path__: + if fn == self.parentPath.path: + # this should _really_ exist. + assert self.parentPath.exists() + yield self.parentPath + else: + smp = self.pathEntry.pythonPath._smartPath(fn) + if smp.exists(): + yield smp + else: + yield self.parentPath + + +class PathEntry(_ModuleIteratorHelper): + """ + I am a proxy for a single entry on sys.path. + + @ivar filePath: a FilePath-like object pointing at the filesystem location + or archive file where this path entry is stored. + + @ivar pythonPath: a PythonPath instance. + """ + def __init__(self, filePath, pythonPath): + """ + Create a PathEntry. This is a private constructor. + """ + self.filePath = filePath + self.pythonPath = pythonPath + + def _getEntry(self): + return self + + def __repr__(self): + return 'PathEntry<%r>' % (self.filePath,) + + def _packagePaths(self): + yield self.filePath + +class IPathImportMapper(Interface): + """ + This is an internal interface, used to map importers to factories for + FilePath-like objects. + """ + def mapPath(self, pathLikeString): + """ + Return a FilePath-like object. + + @param pathLikeString: a path-like string, like one that might be + passed to an import hook. + + @return: a L{FilePath}, or something like it (currently only a + L{ZipPath}, but more might be added later). + """ + + + +@implementer(IPathImportMapper) +class _DefaultMapImpl: + """ Wrapper for the default importer, i.e. None. """ + def mapPath(self, fsPathString): + return FilePath(fsPathString) +_theDefaultMapper = _DefaultMapImpl() + + +@implementer(IPathImportMapper) +class _ZipMapImpl: + """ IPathImportMapper implementation for zipimport.ZipImporter. """ + def __init__(self, importer): + self.importer = importer + + def mapPath(self, fsPathString): + """ + Map the given FS path to a ZipPath, by looking at the ZipImporter's + "archive" attribute and using it as our ZipArchive root, then walking + down into the archive from there. + + @return: a L{zippath.ZipPath} or L{zippath.ZipArchive} instance. + """ + za = ZipArchive(self.importer.archive) + myPath = FilePath(self.importer.archive) + itsPath = FilePath(fsPathString) + if myPath == itsPath: + return za + # This is NOT a general-purpose rule for sys.path or __file__: + # zipimport specifically uses regular OS path syntax in its + # pathnames, even though zip files specify that slashes are always + # the separator, regardless of platform. + segs = itsPath.segmentsFrom(myPath) + zp = za + for seg in segs: + zp = zp.child(seg) + return zp + +registerAdapter(_ZipMapImpl, zipimport.zipimporter, IPathImportMapper) + + + +def _defaultSysPathFactory(): + """ + Provide the default behavior of PythonPath's sys.path factory, which is to + return the current value of sys.path. + + @return: L{sys.path} + """ + return sys.path + + +class PythonPath: + """ + I represent the very top of the Python object-space, the module list in + C{sys.path} and the modules list in C{sys.modules}. + + @ivar _sysPath: A sequence of strings like C{sys.path}. This attribute is + read-only. + + @ivar sysPath: The current value of the module search path list. + @type sysPath: C{list} + + @ivar moduleDict: A dictionary mapping string module names to module + objects, like C{sys.modules}. + + @ivar sysPathHooks: A list of PEP-302 path hooks, like C{sys.path_hooks}. + + @ivar moduleLoader: A function that takes a fully-qualified python name and + returns a module, like L{twisted.python.reflect.namedAny}. + """ + + def __init__(self, + sysPath=None, + moduleDict=sys.modules, + sysPathHooks=sys.path_hooks, + importerCache=sys.path_importer_cache, + moduleLoader=namedAny, + sysPathFactory=None): + """ + Create a PythonPath. You almost certainly want to use + modules.theSystemPath, or its aliased methods, rather than creating a + new instance yourself, though. + + All parameters are optional, and if unspecified, will use 'system' + equivalents that makes this PythonPath like the global L{theSystemPath} + instance. + + @param sysPath: a sys.path-like list to use for this PythonPath, to + specify where to load modules from. + + @param moduleDict: a sys.modules-like dictionary to use for keeping + track of what modules this PythonPath has loaded. + + @param sysPathHooks: sys.path_hooks-like list of PEP-302 path hooks to + be used for this PythonPath, to determie which importers should be + used. + + @param importerCache: a sys.path_importer_cache-like list of PEP-302 + importers. This will be used in conjunction with the given + sysPathHooks. + + @param moduleLoader: a module loader function which takes a string and + returns a module. That is to say, it is like L{namedAny} - *not* like + L{__import__}. + + @param sysPathFactory: a 0-argument callable which returns the current + value of a sys.path-like list of strings. Specify either this, or + sysPath, not both. This alternative interface is provided because the + way the Python import mechanism works, you can re-bind the 'sys.path' + name and that is what is used for current imports, so it must be a + factory rather than a value to deal with modification by rebinding + rather than modification by mutation. Note: it is not recommended to + rebind sys.path. Although this mechanism can deal with that, it is a + subtle point which some tools that it is easy for tools which interact + with sys.path to miss. + """ + if sysPath is not None: + sysPathFactory = lambda : sysPath + elif sysPathFactory is None: + sysPathFactory = _defaultSysPathFactory + self._sysPathFactory = sysPathFactory + self._sysPath = sysPath + self.moduleDict = moduleDict + self.sysPathHooks = sysPathHooks + self.importerCache = importerCache + self.moduleLoader = moduleLoader + + + def _getSysPath(self): + """ + Retrieve the current value of the module search path list. + """ + return self._sysPathFactory() + + sysPath = property(_getSysPath) + + def _findEntryPathString(self, modobj): + """ + Determine where a given Python module object came from by looking at path + entries. + """ + topPackageObj = modobj + while '.' in topPackageObj.__name__: + topPackageObj = self.moduleDict['.'.join( + topPackageObj.__name__.split('.')[:-1])] + if _isPackagePath(FilePath(topPackageObj.__file__)): + # if package 'foo' is on sys.path at /a/b/foo, package 'foo's + # __file__ will be /a/b/foo/__init__.py, and we are looking for + # /a/b here, the path-entry; so go up two steps. + rval = dirname(dirname(topPackageObj.__file__)) + else: + # the module is completely top-level, not within any packages. The + # path entry it's on is just its dirname. + rval = dirname(topPackageObj.__file__) + + # There are probably some awful tricks that an importer could pull + # which would break this, so let's just make sure... it's a loaded + # module after all, which means that its path MUST be in + # path_importer_cache according to PEP 302 -glyph + if rval not in self.importerCache: + warnings.warn( + "%s (for module %s) not in path importer cache " + "(PEP 302 violation - check your local configuration)." % ( + rval, modobj.__name__), + stacklevel=3) + + return rval + + def _smartPath(self, pathName): + """ + Given a path entry from sys.path which may refer to an importer, + return the appropriate FilePath-like instance. + + @param pathName: a str describing the path. + + @return: a FilePath-like object. + """ + importr = self.importerCache.get(pathName, _nothing) + if importr is _nothing: + for hook in self.sysPathHooks: + try: + importr = hook(pathName) + except ImportError: + pass + if importr is _nothing: # still + importr = None + return IPathImportMapper(importr, _theDefaultMapper).mapPath(pathName) + + def iterEntries(self): + """ + Iterate the entries on my sysPath. + + @return: a generator yielding PathEntry objects + """ + for pathName in self.sysPath: + fp = self._smartPath(pathName) + yield PathEntry(fp, self) + + + def __getitem__(self, modname): + """ + Get a python module by its given fully-qualified name. + + @param modname: The fully-qualified Python module name to load. + + @type modname: C{str} + + @return: an object representing the module identified by C{modname} + + @rtype: L{PythonModule} + + @raise KeyError: if the module name is not a valid module name, or no + such module can be identified as loadable. + """ + # See if the module is already somewhere in Python-land. + moduleObject = self.moduleDict.get(modname) + if moduleObject is not None: + # we need 2 paths; one of the path entry and one for the module. + pe = PathEntry( + self._smartPath( + self._findEntryPathString(moduleObject)), + self) + mp = self._smartPath(moduleObject.__file__) + return PythonModule(modname, mp, pe) + + # Recurse if we're trying to get a submodule. + if '.' in modname: + pkg = self + for name in modname.split('.'): + pkg = pkg[name] + return pkg + + # Finally do the slowest possible thing and iterate + for module in self.iterModules(): + if module.name == modname: + return module + raise KeyError(modname) + + + def __contains__(self, module): + """ + Check to see whether or not a module exists on my import path. + + @param module: The name of the module to look for on my import path. + @type module: C{str} + """ + try: + self.__getitem__(module) + return True + except KeyError: + return False + + + def __repr__(self): + """ + Display my sysPath and moduleDict in a string representation. + """ + return "PythonPath(%r,%r)" % (self.sysPath, self.moduleDict) + + def iterModules(self): + """ + Yield all top-level modules on my sysPath. + """ + for entry in self.iterEntries(): + for module in entry.iterModules(): + yield module + + def walkModules(self, importPackages=False): + """ + Similar to L{iterModules}, this yields every module on the path, then every + submodule in each package or entry. + """ + for package in self.iterModules(): + for module in package.walkModules(importPackages=False): + yield module + +theSystemPath = PythonPath() + +def walkModules(importPackages=False): + """ + Deeply iterate all modules on the global python path. + + @param importPackages: Import packages as they are seen. + """ + return theSystemPath.walkModules(importPackages=importPackages) + +def iterModules(): + """ + Iterate all modules and top-level packages on the global Python path, but + do not descend into packages. + + @param importPackages: Import packages as they are seen. + """ + return theSystemPath.iterModules() + +def getModule(moduleName): + """ + Retrieve a module from the system path. + """ + return theSystemPath[moduleName] diff --git a/contrib/python/Twisted/py2/twisted/python/monkey.py b/contrib/python/Twisted/py2/twisted/python/monkey.py new file mode 100644 index 00000000000..4911f877a2b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/monkey.py @@ -0,0 +1,75 @@ +# -*- test-case-name: twisted.test.test_monkey -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import division, absolute_import + + +class MonkeyPatcher(object): + """ + Cover up attributes with new objects. Neat for monkey-patching things for + unit-testing purposes. + """ + + def __init__(self, *patches): + # List of patches to apply in (obj, name, value). + self._patchesToApply = [] + # List of the original values for things that have been patched. + # (obj, name, value) format. + self._originals = [] + for patch in patches: + self.addPatch(*patch) + + + def addPatch(self, obj, name, value): + """ + Add a patch so that the attribute C{name} on C{obj} will be assigned to + C{value} when C{patch} is called or during C{runWithPatches}. + + You can restore the original values with a call to restore(). + """ + self._patchesToApply.append((obj, name, value)) + + + def _alreadyPatched(self, obj, name): + """ + Has the C{name} attribute of C{obj} already been patched by this + patcher? + """ + for o, n, v in self._originals: + if (o, n) == (obj, name): + return True + return False + + + def patch(self): + """ + Apply all of the patches that have been specified with L{addPatch}. + Reverse this operation using L{restore}. + """ + for obj, name, value in self._patchesToApply: + if not self._alreadyPatched(obj, name): + self._originals.append((obj, name, getattr(obj, name))) + setattr(obj, name, value) + + + def restore(self): + """ + Restore all original values to any patched objects. + """ + while self._originals: + obj, name, value = self._originals.pop() + setattr(obj, name, value) + + + def runWithPatches(self, f, *args, **kw): + """ + Apply each patch already specified. Then run the function f with the + given args and kwargs. Restore everything when done. + """ + self.patch() + try: + return f(*args, **kw) + finally: + self.restore() diff --git a/contrib/python/Twisted/py2/twisted/python/procutils.py b/contrib/python/Twisted/py2/twisted/python/procutils.py new file mode 100644 index 00000000000..9251e7d9638 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/procutils.py @@ -0,0 +1,51 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Utilities for dealing with processes. +""" + +from __future__ import division, absolute_import + +import os + + +def which(name, flags=os.X_OK): + """ + Search PATH for executable files with the given name. + + On newer versions of MS-Windows, the PATHEXT environment variable will be + set to the list of file extensions for files considered executable. This + will normally include things like ".EXE". This function will also find files + with the given name ending with any of these extensions. + + On MS-Windows the only flag that has any meaning is os.F_OK. Any other + flags will be ignored. + + @type name: C{str} + @param name: The name for which to search. + + @type flags: C{int} + @param flags: Arguments to L{os.access}. + + @rtype: C{list} + @param: A list of the full paths to files found, in the order in which they + were found. + """ + result = [] + exts = list(filter(None, os.environ.get('PATHEXT', '').split(os.pathsep))) + path = os.environ.get('PATH', None) + + if path is None: + return [] + + for p in os.environ.get('PATH', '').split(os.pathsep): + p = os.path.join(p, name) + if os.access(p, flags): + result.append(p) + for e in exts: + pext = p + e + if os.access(pext, flags): + result.append(pext) + + return result diff --git a/contrib/python/Twisted/py2/twisted/python/randbytes.py b/contrib/python/Twisted/py2/twisted/python/randbytes.py new file mode 100644 index 00000000000..4062ed20df8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/randbytes.py @@ -0,0 +1,150 @@ +# -*- test-case-name: twisted.test.test_randbytes -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Cryptographically secure random implementation, with fallback on normal random. +""" + +from __future__ import division, absolute_import + +import warnings, os, random, string + +from twisted.python.compat import _PY3 + +getrandbits = getattr(random, 'getrandbits', None) + +if _PY3: + _fromhex = bytes.fromhex +else: + def _fromhex(hexBytes): + return hexBytes.decode('hex') + + +class SecureRandomNotAvailable(RuntimeError): + """ + Exception raised when no secure random algorithm is found. + """ + + + +class SourceNotAvailable(RuntimeError): + """ + Internal exception used when a specific random source is not available. + """ + + + +class RandomFactory(object): + """ + Factory providing L{secureRandom} and L{insecureRandom} methods. + + You shouldn't have to instantiate this class, use the module level + functions instead: it is an implementation detail and could be removed or + changed arbitrarily. + """ + + # This variable is no longer used, and will eventually be removed. + randomSources = () + + getrandbits = getrandbits + + + def _osUrandom(self, nbytes): + """ + Wrapper around C{os.urandom} that cleanly manage its absence. + """ + try: + return os.urandom(nbytes) + except (AttributeError, NotImplementedError) as e: + raise SourceNotAvailable(e) + + + def secureRandom(self, nbytes, fallback=False): + """ + Return a number of secure random bytes. + + @param nbytes: number of bytes to generate. + @type nbytes: C{int} + @param fallback: Whether the function should fallback on non-secure + random or not. Default to C{False}. + @type fallback: C{bool} + + @return: a string of random bytes. + @rtype: C{str} + """ + try: + return self._osUrandom(nbytes) + except SourceNotAvailable: + pass + + if fallback: + warnings.warn( + "urandom unavailable - " + "proceeding with non-cryptographically secure random source", + category=RuntimeWarning, + stacklevel=2) + return self.insecureRandom(nbytes) + else: + raise SecureRandomNotAvailable("No secure random source available") + + + def _randBits(self, nbytes): + """ + Wrapper around C{os.getrandbits}. + """ + if self.getrandbits is not None: + n = self.getrandbits(nbytes * 8) + hexBytes = ("%%0%dx" % (nbytes * 2)) % n + return _fromhex(hexBytes) + raise SourceNotAvailable("random.getrandbits is not available") + + + if _PY3: + _maketrans = bytes.maketrans + def _randModule(self, nbytes): + """ + Wrapper around the C{random} module. + """ + return b"".join([ + bytes([random.choice(self._BYTES)]) for i in range(nbytes)]) + else: + _maketrans = string.maketrans + def _randModule(self, nbytes): + """ + Wrapper around the C{random} module. + """ + return b"".join([ + random.choice(self._BYTES) for i in range(nbytes)]) + + _BYTES = _maketrans(b'', b'') + + + def insecureRandom(self, nbytes): + """ + Return a number of non secure random bytes. + + @param nbytes: number of bytes to generate. + @type nbytes: C{int} + + @return: a string of random bytes. + @rtype: C{str} + """ + for src in ("_randBits", "_randModule"): + try: + return getattr(self, src)(nbytes) + except SourceNotAvailable: + pass + + + +factory = RandomFactory() + +secureRandom = factory.secureRandom + +insecureRandom = factory.insecureRandom + +del factory + + +__all__ = ["secureRandom", "insecureRandom", "SecureRandomNotAvailable"] diff --git a/contrib/python/Twisted/py2/twisted/python/rebuild.py b/contrib/python/Twisted/py2/twisted/python/rebuild.py new file mode 100644 index 00000000000..74bb31f78c5 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/rebuild.py @@ -0,0 +1,310 @@ +# -*- test-case-name: twisted.test.test_rebuild -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +*Real* reloading support for Python. +""" + +# System Imports +import sys +import types +import time +import linecache + +from imp import reload + +try: + # Python 2 + from types import InstanceType +except ImportError: + # Python 3 + pass + +# Sibling Imports +from twisted.python import log, reflect +from twisted.python.compat import _PY3 + +lastRebuild = time.time() + +def _isClassType(t): + """ + Compare to types.ClassType in a py2/3-compatible way + + Python 2 used comparison to types.ClassType to check for old-style + classes Python 3 has no concept of old-style classes, so if + ClassType doesn't exist, it can't be an old-style class - return + False in that case. + + Note that the type() of new-style classes is NOT ClassType, and + so this should return False for new-style classes in python 2 + as well. + """ + _ClassType = getattr(types, 'ClassType', None) + if _ClassType is None: + return False + return t == _ClassType + + + +class Sensitive(object): + """ + A utility mixin that's sensitive to rebuilds. + + This is a mixin for classes (usually those which represent collections of + callbacks) to make sure that their code is up-to-date before running. + """ + + lastRebuild = lastRebuild + + def needRebuildUpdate(self): + yn = (self.lastRebuild < lastRebuild) + return yn + + + def rebuildUpToDate(self): + self.lastRebuild = time.time() + + + def latestVersionOf(self, anObject): + """ + Get the latest version of an object. + + This can handle just about anything callable; instances, functions, + methods, and classes. + """ + t = type(anObject) + if t == types.FunctionType: + return latestFunction(anObject) + elif t == types.MethodType: + if anObject.__self__ is None: + return getattr(anObject.im_class, anObject.__name__) + else: + return getattr(anObject.__self__, anObject.__name__) + elif not _PY3 and t == InstanceType: + # Kick it, if it's out of date. + getattr(anObject, 'nothing', None) + return anObject + elif _isClassType(t): + return latestClass(anObject) + else: + log.msg('warning returning anObject!') + return anObject + +_modDictIDMap = {} + +def latestFunction(oldFunc): + """ + Get the latest version of a function. + """ + # This may be CPython specific, since I believe jython instantiates a new + # module upon reload. + dictID = id(oldFunc.__globals__) + module = _modDictIDMap.get(dictID) + if module is None: + return oldFunc + return getattr(module, oldFunc.__name__) + + + +def latestClass(oldClass): + """ + Get the latest version of a class. + """ + module = reflect.namedModule(oldClass.__module__) + newClass = getattr(module, oldClass.__name__) + newBases = [latestClass(base) for base in newClass.__bases__] + + try: + # This makes old-style stuff work + newClass.__bases__ = tuple(newBases) + return newClass + except TypeError: + if newClass.__module__ in ("__builtin__", "builtins"): + # __builtin__ members can't be reloaded sanely + return newClass + + ctor = type(newClass) + # The value of type(newClass) is the metaclass + # in both Python 2 and 3, except if it was old-style. + if _isClassType(ctor): + ctor = getattr(newClass, '__metaclass__', type) + return ctor(newClass.__name__, tuple(newBases), + dict(newClass.__dict__)) + + + +class RebuildError(Exception): + """ + Exception raised when trying to rebuild a class whereas it's not possible. + """ + + + +def updateInstance(self): + """ + Updates an instance to be current. + """ + self.__class__ = latestClass(self.__class__) + + + +def __injectedgetattr__(self, name): + """ + A getattr method to cause a class to be refreshed. + """ + if name == '__del__': + raise AttributeError("Without this, Python segfaults.") + updateInstance(self) + log.msg("(rebuilding stale {} instance ({}))".format( + reflect.qual(self.__class__), name)) + result = getattr(self, name) + return result + + + +def rebuild(module, doLog=1): + """ + Reload a module and do as much as possible to replace its references. + """ + global lastRebuild + lastRebuild = time.time() + if hasattr(module, 'ALLOW_TWISTED_REBUILD'): + # Is this module allowed to be rebuilt? + if not module.ALLOW_TWISTED_REBUILD: + raise RuntimeError("I am not allowed to be rebuilt.") + if doLog: + log.msg('Rebuilding {}...'.format(str(module.__name__))) + + # Safely handle adapter re-registration + from twisted.python import components + components.ALLOW_DUPLICATES = True + + d = module.__dict__ + _modDictIDMap[id(d)] = module + newclasses = {} + classes = {} + functions = {} + values = {} + if doLog: + log.msg(' (scanning {}): '.format(str(module.__name__))) + for k, v in d.items(): + if _isClassType(type(v)): + # ClassType exists on Python 2.x and earlier. + # Failure condition -- instances of classes with buggy + # __hash__/__cmp__ methods referenced at the module level... + if v.__module__ == module.__name__: + classes[v] = 1 + if doLog: + log.logfile.write("c") + log.logfile.flush() + elif type(v) == types.FunctionType: + if v.__globals__ is module.__dict__: + functions[v] = 1 + if doLog: + log.logfile.write("f") + log.logfile.flush() + elif isinstance(v, type): + if v.__module__ == module.__name__: + newclasses[v] = 1 + if doLog: + log.logfile.write("o") + log.logfile.flush() + + values.update(classes) + values.update(functions) + fromOldModule = values.__contains__ + newclasses = newclasses.keys() + classes = classes.keys() + functions = functions.keys() + + if doLog: + log.msg('') + log.msg(' (reload {})'.format(str(module.__name__))) + + # Boom. + reload(module) + # Make sure that my traceback printing will at least be recent... + linecache.clearcache() + + if doLog: + log.msg(' (cleaning {}): '.format(str(module.__name__))) + + for clazz in classes: + if getattr(module, clazz.__name__) is clazz: + log.msg("WARNING: class {} not replaced by reload!".format( + reflect.qual(clazz))) + else: + if doLog: + log.logfile.write("x") + log.logfile.flush() + clazz.__bases__ = () + clazz.__dict__.clear() + clazz.__getattr__ = __injectedgetattr__ + clazz.__module__ = module.__name__ + if newclasses: + import gc + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class {} not replaced by reload!".format( + reflect.qual(nclass))) + else: + for r in gc.get_referrers(nclass): + if getattr(r, '__class__', None) is nclass: + r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing {}): '.format(str(module.__name__))) + modcount = 0 + for mk, mod in sys.modules.items(): + modcount = modcount + 1 + if mod == module or mod is None: + continue + + if not hasattr(mod, '__file__'): + # It's a builtin module; nothing to replace here. + continue + + if hasattr(mod, '__bundle__'): + # PyObjC has a few buggy objects which segfault if you hash() them. + # It doesn't make sense to try rebuilding extension modules like + # this anyway, so don't try. + continue + + changed = 0 + + for k, v in mod.__dict__.items(): + try: + hash(v) + except Exception: + continue + if fromOldModule(v): + if _isClassType(type(v)): + if doLog: + log.logfile.write("c") + log.logfile.flush() + nv = latestClass(v) + else: + if doLog: + log.logfile.write("f") + log.logfile.flush() + nv = latestFunction(v) + changed = 1 + setattr(mod, k, nv) + else: + # Replace bases of non-module classes just to be sure. + if _isClassType(type(v)): + for base in v.__bases__: + if fromOldModule(base): + latestClass(v) + if doLog and not changed and ((modcount % 10) == 0) : + log.logfile.write(".") + log.logfile.flush() + + components.ALLOW_DUPLICATES = False + if doLog: + log.msg('') + log.msg(' Rebuilt {}.'.format(str(module.__name__))) + return module diff --git a/contrib/python/Twisted/py2/twisted/python/reflect.py b/contrib/python/Twisted/py2/twisted/python/reflect.py new file mode 100644 index 00000000000..a9b4a7c3996 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/reflect.py @@ -0,0 +1,634 @@ +# -*- test-case-name: twisted.test.test_reflect -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Standardized versions of various cool and/or strange things that you can do +with Python's reflection capabilities. +""" + +from __future__ import division, absolute_import, print_function + +import sys +import types +import os +import pickle +import weakref +import re +import traceback +from collections import deque + +RegexType = type(re.compile("")) + + +from twisted.python.compat import reraise, nativeString, NativeStringIO +from twisted.python.compat import _PY3 +from twisted.python import compat +from twisted.python.deprecate import _fullyQualifiedName as fullyQualifiedName +from twisted.python._oldstyle import _oldStyle + + +def prefixedMethodNames(classObj, prefix): + """ + Given a class object C{classObj}, returns a list of method names that match + the string C{prefix}. + + @param classObj: A class object from which to collect method names. + + @param prefix: A native string giving a prefix. Each method with a name + which begins with this prefix will be returned. + @type prefix: L{str} + + @return: A list of the names of matching methods of C{classObj} (and base + classes of C{classObj}). + @rtype: L{list} of L{str} + """ + dct = {} + addMethodNamesToDict(classObj, dct, prefix) + return list(dct.keys()) + + + +def addMethodNamesToDict(classObj, dict, prefix, baseClass=None): + """ + This goes through C{classObj} (and its bases) and puts method names + starting with 'prefix' in 'dict' with a value of 1. if baseClass isn't + None, methods will only be added if classObj is-a baseClass + + If the class in question has the methods 'prefix_methodname' and + 'prefix_methodname2', the resulting dict should look something like: + {"methodname": 1, "methodname2": 1}. + + @param classObj: A class object from which to collect method names. + + @param dict: A L{dict} which will be updated with the results of the + accumulation. Items are added to this dictionary, with method names as + keys and C{1} as values. + @type dict: L{dict} + + @param prefix: A native string giving a prefix. Each method of C{classObj} + (and base classes of C{classObj}) with a name which begins with this + prefix will be returned. + @type prefix: L{str} + + @param baseClass: A class object at which to stop searching upwards for new + methods. To collect all method names, do not pass a value for this + parameter. + + @return: L{None} + """ + for base in classObj.__bases__: + addMethodNamesToDict(base, dict, prefix, baseClass) + + if baseClass is None or baseClass in classObj.__bases__: + for name, method in classObj.__dict__.items(): + optName = name[len(prefix):] + if ((type(method) is types.FunctionType) + and (name[:len(prefix)] == prefix) + and (len(optName))): + dict[optName] = 1 + + + +def prefixedMethods(obj, prefix=''): + """ + Given an object C{obj}, returns a list of method objects that match the + string C{prefix}. + + @param obj: An arbitrary object from which to collect methods. + + @param prefix: A native string giving a prefix. Each method of C{obj} with + a name which begins with this prefix will be returned. + @type prefix: L{str} + + @return: A list of the matching method objects. + @rtype: L{list} + """ + dct = {} + accumulateMethods(obj, dct, prefix) + return list(dct.values()) + + + +def accumulateMethods(obj, dict, prefix='', curClass=None): + """ + Given an object C{obj}, add all methods that begin with C{prefix}. + + @param obj: An arbitrary object to collect methods from. + + @param dict: A L{dict} which will be updated with the results of the + accumulation. Items are added to this dictionary, with method names as + keys and corresponding instance method objects as values. + @type dict: L{dict} + + @param prefix: A native string giving a prefix. Each method of C{obj} with + a name which begins with this prefix will be returned. + @type prefix: L{str} + + @param curClass: The class in the inheritance hierarchy at which to start + collecting methods. Collection proceeds up. To collect all methods + from C{obj}, do not pass a value for this parameter. + + @return: L{None} + """ + if not curClass: + curClass = obj.__class__ + for base in curClass.__bases__: + # The implementation of the object class is different on PyPy vs. + # CPython. This has the side effect of making accumulateMethods() + # pick up object methods from all new-style classes - + # such as __getattribute__, etc. + # If we ignore 'object' when accumulating methods, we can get + # consistent behavior on Pypy and CPython. + if base is not object: + accumulateMethods(obj, dict, prefix, base) + + for name, method in curClass.__dict__.items(): + optName = name[len(prefix):] + if ((type(method) is types.FunctionType) + and (name[:len(prefix)] == prefix) + and (len(optName))): + dict[optName] = getattr(obj, name) + + + +def namedModule(name): + """ + Return a module given its name. + """ + topLevel = __import__(name) + packages = name.split(".")[1:] + m = topLevel + for p in packages: + m = getattr(m, p) + return m + + + +def namedObject(name): + """ + Get a fully named module-global object. + """ + classSplit = name.split('.') + module = namedModule('.'.join(classSplit[:-1])) + return getattr(module, classSplit[-1]) + +namedClass = namedObject # backwards compat + + + +def requireModule(name, default=None): + """ + Try to import a module given its name, returning C{default} value if + C{ImportError} is raised during import. + + @param name: Module name as it would have been passed to C{import}. + @type name: C{str}. + + @param default: Value returned in case C{ImportError} is raised while + importing the module. + + @return: Module or default value. + """ + try: + return namedModule(name) + except ImportError: + return default + + + +class _NoModuleFound(Exception): + """ + No module was found because none exists. + """ + + + +class InvalidName(ValueError): + """ + The given name is not a dot-separated list of Python objects. + """ + + + +class ModuleNotFound(InvalidName): + """ + The module associated with the given name doesn't exist and it can't be + imported. + """ + + + +class ObjectNotFound(InvalidName): + """ + The object associated with the given name doesn't exist and it can't be + imported. + """ + + + +def _importAndCheckStack(importName): + """ + Import the given name as a module, then walk the stack to determine whether + the failure was the module not existing, or some code in the module (for + example a dependent import) failing. This can be helpful to determine + whether any actual application code was run. For example, to distiguish + administrative error (entering the wrong module name), from programmer + error (writing buggy code in a module that fails to import). + + @param importName: The name of the module to import. + @type importName: C{str} + @raise Exception: if something bad happens. This can be any type of + exception, since nobody knows what loading some arbitrary code might + do. + @raise _NoModuleFound: if no module was found. + """ + try: + return __import__(importName) + except ImportError: + excType, excValue, excTraceback = sys.exc_info() + while excTraceback: + execName = excTraceback.tb_frame.f_globals["__name__"] + # in Python 2 execName is None when an ImportError is encountered, + # where in Python 3 execName is equal to the importName. + if execName is None or execName == importName: + reraise(excValue, excTraceback) + excTraceback = excTraceback.tb_next + raise _NoModuleFound() + + + +def namedAny(name): + """ + Retrieve a Python object by its fully qualified name from the global Python + module namespace. The first part of the name, that describes a module, + will be discovered and imported. Each subsequent part of the name is + treated as the name of an attribute of the object specified by all of the + name which came before it. For example, the fully-qualified name of this + object is 'twisted.python.reflect.namedAny'. + + @type name: L{str} + @param name: The name of the object to return. + + @raise InvalidName: If the name is an empty string, starts or ends with + a '.', or is otherwise syntactically incorrect. + + @raise ModuleNotFound: If the name is syntactically correct but the + module it specifies cannot be imported because it does not appear to + exist. + + @raise ObjectNotFound: If the name is syntactically correct, includes at + least one '.', but the module it specifies cannot be imported because + it does not appear to exist. + + @raise AttributeError: If an attribute of an object along the way cannot be + accessed, or a module along the way is not found. + + @return: the Python object identified by 'name'. + """ + if not name: + raise InvalidName('Empty module name') + + names = name.split('.') + + # if the name starts or ends with a '.' or contains '..', the __import__ + # will raise an 'Empty module name' error. This will provide a better error + # message. + if '' in names: + raise InvalidName( + "name must be a string giving a '.'-separated list of Python " + "identifiers, not %r" % (name,)) + + topLevelPackage = None + moduleNames = names[:] + while not topLevelPackage: + if moduleNames: + trialname = '.'.join(moduleNames) + try: + topLevelPackage = _importAndCheckStack(trialname) + except _NoModuleFound: + moduleNames.pop() + else: + if len(names) == 1: + raise ModuleNotFound("No module named %r" % (name,)) + else: + raise ObjectNotFound('%r does not name an object' % (name,)) + + obj = topLevelPackage + for n in names[1:]: + obj = getattr(obj, n) + + return obj + + + +def filenameToModuleName(fn): + """ + Convert a name in the filesystem to the name of the Python module it is. + + This is aggressive about getting a module name back from a file; it will + always return a string. Aggressive means 'sometimes wrong'; it won't look + at the Python path or try to do any error checking: don't use this method + unless you already know that the filename you're talking about is a Python + module. + + @param fn: A filesystem path to a module or package; C{bytes} on Python 2, + C{bytes} or C{unicode} on Python 3. + + @return: A hopefully importable module name. + @rtype: C{str} + """ + if isinstance(fn, bytes): + initPy = b"__init__.py" + else: + initPy = "__init__.py" + fullName = os.path.abspath(fn) + base = os.path.basename(fn) + if not base: + # this happens when fn ends with a path separator, just skit it + base = os.path.basename(fn[:-1]) + modName = nativeString(os.path.splitext(base)[0]) + while 1: + fullName = os.path.dirname(fullName) + if os.path.exists(os.path.join(fullName, initPy)): + modName = "%s.%s" % ( + nativeString(os.path.basename(fullName)), + nativeString(modName)) + else: + break + return modName + + + +def qual(clazz): + """ + Return full import path of a class. + """ + return clazz.__module__ + '.' + clazz.__name__ + + + +def _determineClass(x): + try: + return x.__class__ + except: + return type(x) + + + +def _determineClassName(x): + c = _determineClass(x) + try: + return c.__name__ + except: + try: + return str(c) + except: + return '' % id(c) + + + +def _safeFormat(formatter, o): + """ + Helper function for L{safe_repr} and L{safe_str}. + + Called when C{repr} or C{str} fail. Returns a string containing info about + C{o} and the latest exception. + + @param formatter: C{str} or C{repr}. + @type formatter: C{type} + @param o: Any object. + + @rtype: C{str} + @return: A string containing information about C{o} and the raised + exception. + """ + io = NativeStringIO() + traceback.print_exc(file=io) + className = _determineClassName(o) + tbValue = io.getvalue() + return "<%s instance at 0x%x with %s error:\n %s>" % ( + className, id(o), formatter.__name__, tbValue) + + + +def safe_repr(o): + """ + Returns a string representation of an object, or a string containing a + traceback, if that object's __repr__ raised an exception. + + @param o: Any object. + + @rtype: C{str} + """ + try: + return repr(o) + except: + return _safeFormat(repr, o) + + + +def safe_str(o): + """ + Returns a string representation of an object, or a string containing a + traceback, if that object's __str__ raised an exception. + + @param o: Any object. + + @rtype: C{str} + """ + if _PY3 and isinstance(o, bytes): + # If o is bytes and seems to holds a utf-8 encoded string, + # convert it to str. + try: + return o.decode('utf-8') + except: + pass + try: + return str(o) + except: + return _safeFormat(str, o) + + + +@_oldStyle +class QueueMethod: + """ + I represent a method that doesn't exist yet. + """ + def __init__(self, name, calls): + self.name = name + self.calls = calls + def __call__(self, *args): + self.calls.append((self.name, args)) + + + +def fullFuncName(func): + qualName = (str(pickle.whichmodule(func, func.__name__)) + '.' + func.__name__) + if namedObject(qualName) is not func: + raise Exception("Couldn't find %s as %s." % (func, qualName)) + return qualName + + + +def getClass(obj): + """ + Return the class or type of object 'obj'. + Returns sensible result for oldstyle and newstyle instances and types. + """ + if hasattr(obj, '__class__'): + return obj.__class__ + else: + return type(obj) + + + +def accumulateClassDict(classObj, attr, adict, baseClass=None): + """ + Accumulate all attributes of a given name in a class hierarchy into a single dictionary. + + Assuming all class attributes of this name are dictionaries. + If any of the dictionaries being accumulated have the same key, the + one highest in the class hierarchy wins. + (XXX: If \"highest\" means \"closest to the starting class\".) + + Ex:: + + class Soy: + properties = {\"taste\": \"bland\"} + + class Plant: + properties = {\"colour\": \"green\"} + + class Seaweed(Plant): + pass + + class Lunch(Soy, Seaweed): + properties = {\"vegan\": 1 } + + dct = {} + + accumulateClassDict(Lunch, \"properties\", dct) + + print(dct) + + {\"taste\": \"bland\", \"colour\": \"green\", \"vegan\": 1} + """ + for base in classObj.__bases__: + accumulateClassDict(base, attr, adict) + if baseClass is None or baseClass in classObj.__bases__: + adict.update(classObj.__dict__.get(attr, {})) + + +def accumulateClassList(classObj, attr, listObj, baseClass=None): + """ + Accumulate all attributes of a given name in a class hierarchy into a single list. + + Assuming all class attributes of this name are lists. + """ + for base in classObj.__bases__: + accumulateClassList(base, attr, listObj) + if baseClass is None or baseClass in classObj.__bases__: + listObj.extend(classObj.__dict__.get(attr, [])) + + +def isSame(a, b): + return (a is b) + + +def isLike(a, b): + return (a == b) + + +def modgrep(goal): + return objgrep(sys.modules, goal, isLike, 'sys.modules') + + +def isOfType(start, goal): + return ((type(start) is goal) or + (isinstance(start, compat.InstanceType) and + start.__class__ is goal)) + + +def findInstances(start, t): + return objgrep(start, t, isOfType) + + +if not _PY3: + # The function objgrep() currently doesn't work on Python 3 due to some + # edge cases, as described in #6986. + # twisted.python.reflect is quite important and objgrep is not used in + # Twisted itself, so in #5929, we decided to port everything but objgrep() + # and to finish the porting in #6986 + def objgrep(start, goal, eq=isLike, path='', paths=None, seen=None, + showUnknowns=0, maxDepth=None): + """ + An insanely CPU-intensive process for finding stuff. + """ + if paths is None: + paths = [] + if seen is None: + seen = {} + if eq(start, goal): + paths.append(path) + if id(start) in seen: + if seen[id(start)] is start: + return + if maxDepth is not None: + if maxDepth == 0: + return + maxDepth -= 1 + seen[id(start)] = start + # Make an alias for those arguments which are passed recursively to + # objgrep for container objects. + args = (paths, seen, showUnknowns, maxDepth) + if isinstance(start, dict): + for k, v in start.items(): + objgrep(k, goal, eq, path+'{'+repr(v)+'}', *args) + objgrep(v, goal, eq, path+'['+repr(k)+']', *args) + elif isinstance(start, (list, tuple, deque)): + for idx, _elem in enumerate(start): + objgrep(start[idx], goal, eq, path+'['+str(idx)+']', *args) + elif isinstance(start, types.MethodType): + objgrep(start.__self__, goal, eq, path+'.__self__', *args) + objgrep(start.__func__, goal, eq, path+'.__func__', *args) + objgrep(start.__self__.__class__, goal, eq, + path+'.__self__.__class__', *args) + elif hasattr(start, '__dict__'): + for k, v in start.__dict__.items(): + objgrep(v, goal, eq, path+'.'+k, *args) + if isinstance(start, compat.InstanceType): + objgrep(start.__class__, goal, eq, path+'.__class__', *args) + elif isinstance(start, weakref.ReferenceType): + objgrep(start(), goal, eq, path+'()', *args) + elif (isinstance(start, (compat.StringType, + int, types.FunctionType, + types.BuiltinMethodType, RegexType, float, + type(None), compat.FileType)) or + type(start).__name__ in ('wrapper_descriptor', + 'method_descriptor', 'member_descriptor', + 'getset_descriptor')): + pass + elif showUnknowns: + print('unknown type', type(start), start) + return paths + + + +__all__ = [ + 'InvalidName', 'ModuleNotFound', 'ObjectNotFound', + + 'QueueMethod', + + 'namedModule', 'namedObject', 'namedClass', 'namedAny', 'requireModule', + 'safe_repr', 'safe_str', 'prefixedMethodNames', 'addMethodNamesToDict', + 'prefixedMethods', 'accumulateMethods', 'fullFuncName', 'qual', 'getClass', + 'accumulateClassDict', 'accumulateClassList', 'isSame', 'isLike', + 'modgrep', 'isOfType', 'findInstances', 'objgrep', 'filenameToModuleName', + 'fullyQualifiedName'] + + +if _PY3: + # This is to be removed when fixing #6986 + __all__.remove('objgrep') diff --git a/contrib/python/Twisted/py2/twisted/python/release.py b/contrib/python/Twisted/py2/twisted/python/release.py new file mode 100644 index 00000000000..c15b1c396f5 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/release.py @@ -0,0 +1,67 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A release-automation toolkit. + +Don't use this outside of Twisted. + +Maintainer: Christopher Armstrong +""" + +from __future__ import print_function + +import os + +from twisted.python.compat import raw_input + + +# errors + +class DirectoryExists(OSError): + """ + Some directory exists when it shouldn't. + """ + pass + + + +class DirectoryDoesntExist(OSError): + """ + Some directory doesn't exist when it should. + """ + pass + + + +class CommandFailed(OSError): + pass + + + +# utilities + +def sh(command, null=True, prompt=False): + """ + I'll try to execute C{command}, and if C{prompt} is true, I'll + ask before running it. If the command returns something other + than 0, I'll raise C{CommandFailed(command)}. + """ + print("--$", command) + + if prompt: + if raw_input("run ?? ").startswith('n'): + return + if null: + command = "%s > /dev/null" % command + if os.system(command) != 0: + raise CommandFailed(command) + + + +def runChdirSafe(f, *args, **kw): + origdir = os.path.abspath('.') + try: + return f(*args, **kw) + finally: + os.chdir(origdir) diff --git a/contrib/python/Twisted/py2/twisted/python/roots.py b/contrib/python/Twisted/py2/twisted/python/roots.py new file mode 100644 index 00000000000..d9bc3f78356 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/roots.py @@ -0,0 +1,257 @@ +# -*- test-case-name: twisted.test.test_roots -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Python Roots: an abstract hierarchy representation for Twisted. + +Maintainer: Glyph Lefkowitz +""" + +from __future__ import absolute_import, division + +from twisted.python import reflect +from twisted.python._oldstyle import _oldStyle + + + +class NotSupportedError(NotImplementedError): + """ + An exception meaning that the tree-manipulation operation + you're attempting to perform is not supported. + """ + + + +@_oldStyle +class Request: + """I am an abstract representation of a request for an entity. + + I also function as the response. The request is responded to by calling + self.write(data) until there is no data left and then calling + self.finish(). + """ + # This attribute should be set to the string name of the protocol being + # responded to (e.g. HTTP or FTP) + wireProtocol = None + def write(self, data): + """Add some data to the response to this request. + """ + raise NotImplementedError("%s.write" % reflect.qual(self.__class__)) + + def finish(self): + """The response to this request is finished; flush all data to the network stream. + """ + raise NotImplementedError("%s.finish" % reflect.qual(self.__class__)) + + + +@_oldStyle +class Entity: + """I am a terminal object in a hierarchy, with no children. + + I represent a null interface; certain non-instance objects (strings and + integers, notably) are Entities. + + Methods on this class are suggested to be implemented, but are not + required, and will be emulated on a per-protocol basis for types which do + not handle them. + """ + def render(self, request): + """ + I produce a stream of bytes for the request, by calling request.write() + and request.finish(). + """ + raise NotImplementedError("%s.render" % reflect.qual(self.__class__)) + + + +@_oldStyle +class Collection: + """I represent a static collection of entities. + + I contain methods designed to represent collections that can be dynamically + created. + """ + + def __init__(self, entities=None): + """Initialize me. + """ + if entities is not None: + self.entities = entities + else: + self.entities = {} + + def getStaticEntity(self, name): + """Get an entity that was added to me using putEntity. + + This method will return 'None' if it fails. + """ + return self.entities.get(name) + + def getDynamicEntity(self, name, request): + """Subclass this to generate an entity on demand. + + This method should return 'None' if it fails. + """ + + def getEntity(self, name, request): + """Retrieve an entity from me. + + I will first attempt to retrieve an entity statically; static entities + will obscure dynamic ones. If that fails, I will retrieve the entity + dynamically. + + If I cannot retrieve an entity, I will return 'None'. + """ + ent = self.getStaticEntity(name) + if ent is not None: + return ent + ent = self.getDynamicEntity(name, request) + if ent is not None: + return ent + return None + + def putEntity(self, name, entity): + """Store a static reference on 'name' for 'entity'. + + Raises a KeyError if the operation fails. + """ + self.entities[name] = entity + + def delEntity(self, name): + """Remove a static reference for 'name'. + + Raises a KeyError if the operation fails. + """ + del self.entities[name] + + def storeEntity(self, name, request): + """Store an entity for 'name', based on the content of 'request'. + """ + raise NotSupportedError("%s.storeEntity" % reflect.qual(self.__class__)) + + def removeEntity(self, name, request): + """Remove an entity for 'name', based on the content of 'request'. + """ + raise NotSupportedError("%s.removeEntity" % reflect.qual(self.__class__)) + + def listStaticEntities(self): + """Retrieve a list of all name, entity pairs that I store references to. + + See getStaticEntity. + """ + return self.entities.items() + + def listDynamicEntities(self, request): + """A list of all name, entity that I can generate on demand. + + See getDynamicEntity. + """ + return [] + + def listEntities(self, request): + """Retrieve a list of all name, entity pairs I contain. + + See getEntity. + """ + return self.listStaticEntities() + self.listDynamicEntities(request) + + def listStaticNames(self): + """Retrieve a list of the names of entities that I store references to. + + See getStaticEntity. + """ + return self.entities.keys() + + + def listDynamicNames(self): + """Retrieve a list of the names of entities that I store references to. + + See getDynamicEntity. + """ + return [] + + + def listNames(self, request): + """Retrieve a list of all names for entities that I contain. + + See getEntity. + """ + return self.listStaticNames() + + +class ConstraintViolation(Exception): + """An exception raised when a constraint is violated. + """ + + +class Constrained(Collection): + """A collection that has constraints on its names and/or entities.""" + + def nameConstraint(self, name): + """A method that determines whether an entity may be added to me with a given name. + + If the constraint is satisfied, return 1; if the constraint is not + satisfied, either return 0 or raise a descriptive ConstraintViolation. + """ + return 1 + + def entityConstraint(self, entity): + """A method that determines whether an entity may be added to me. + + If the constraint is satisfied, return 1; if the constraint is not + satisfied, either return 0 or raise a descriptive ConstraintViolation. + """ + return 1 + + def reallyPutEntity(self, name, entity): + Collection.putEntity(self, name, entity) + + def putEntity(self, name, entity): + """Store an entity if it meets both constraints. + + Otherwise raise a ConstraintViolation. + """ + if self.nameConstraint(name): + if self.entityConstraint(entity): + self.reallyPutEntity(name, entity) + else: + raise ConstraintViolation("Entity constraint violated.") + else: + raise ConstraintViolation("Name constraint violated.") + + +class Locked(Constrained): + """A collection that can be locked from adding entities.""" + + locked = 0 + + def lock(self): + self.locked = 1 + + def entityConstraint(self, entity): + return not self.locked + + +class Homogenous(Constrained): + """A homogenous collection of entities. + + I will only contain entities that are an instance of the class or type + specified by my 'entityType' attribute. + """ + + entityType = object + + def entityConstraint(self, entity): + if isinstance(entity, self.entityType): + return 1 + else: + raise ConstraintViolation("%s of incorrect type (%s)" % + (entity, self.entityType)) + + def getNameType(self): + return "Name" + + def getEntityType(self): + return self.entityType.__name__ diff --git a/contrib/python/Twisted/py2/twisted/python/runtime.py b/contrib/python/Twisted/py2/twisted/python/runtime.py new file mode 100644 index 00000000000..62223a7758a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/runtime.py @@ -0,0 +1,231 @@ +# -*- test-case-name: twisted.python.test.test_runtime -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import division, absolute_import + +import os +import sys +import time +import warnings + +from twisted.python._oldstyle import _oldStyle + + + +def shortPythonVersion(): + """ + Returns the Python version as a dot-separated string. + """ + return "%s.%s.%s" % sys.version_info[:3] + + + +knownPlatforms = { + 'nt': 'win32', + 'ce': 'win32', + 'posix': 'posix', + 'java': 'java', + 'org.python.modules.os': 'java', + } + + + +_timeFunctions = { + #'win32': time.clock, + 'win32': time.time, + } + + + +@_oldStyle +class Platform: + """ + Gives us information about the platform we're running on. + """ + + type = knownPlatforms.get(os.name) + seconds = staticmethod(_timeFunctions.get(type, time.time)) + _platform = sys.platform + + def __init__(self, name=None, platform=None): + if name is not None: + self.type = knownPlatforms.get(name) + self.seconds = _timeFunctions.get(self.type, time.time) + if platform is not None: + self._platform = platform + + + def isKnown(self): + """ + Do we know about this platform? + + @return: Boolean indicating whether this is a known platform or not. + @rtype: C{bool} + """ + return self.type != None + + + def getType(self): + """ + Get platform type. + + @return: Either 'posix', 'win32' or 'java' + @rtype: C{str} + """ + return self.type + + + def isMacOSX(self): + """ + Check if current platform is macOS. + + @return: C{True} if the current platform has been detected as macOS. + @rtype: C{bool} + """ + return self._platform == "darwin" + + + def isWinNT(self): + """ + Are we running in Windows NT? + + This is deprecated and always returns C{True} on win32 because + Twisted only supports Windows NT-derived platforms at this point. + + @return: C{True} if the current platform has been detected as + Windows NT. + @rtype: C{bool} + """ + warnings.warn( + "twisted.python.runtime.Platform.isWinNT was deprecated in " + "Twisted 13.0. Use Platform.isWindows instead.", + DeprecationWarning, stacklevel=2) + return self.isWindows() + + + def isWindows(self): + """ + Are we running in Windows? + + @return: C{True} if the current platform has been detected as + Windows. + @rtype: C{bool} + """ + return self.getType() == 'win32' + + + def isVista(self): + """ + Check if current platform is Windows Vista or Windows Server 2008. + + @return: C{True} if the current platform has been detected as Vista + @rtype: C{bool} + """ + if getattr(sys, "getwindowsversion", None) is not None: + return sys.getwindowsversion()[0] == 6 + else: + return False + + + def isLinux(self): + """ + Check if current platform is Linux. + + @return: C{True} if the current platform has been detected as Linux. + @rtype: C{bool} + """ + return self._platform.startswith("linux") + + + def isDocker(self, _initCGroupLocation="/proc/1/cgroup"): + """ + Check if the current platform is Linux in a Docker container. + + @return: C{True} if the current platform has been detected as Linux + inside a Docker container. + @rtype: C{bool} + """ + if not self.isLinux(): + return False + + from twisted.python.filepath import FilePath + + # Ask for the cgroups of init (pid 1) + initCGroups = FilePath(_initCGroupLocation) + if initCGroups.exists(): + # The cgroups file looks like "2:cpu:/". The third element will + # begin with /docker if it is inside a Docker container. + controlGroups = [x.split(b":") + for x in initCGroups.getContent().split(b"\n")] + + for group in controlGroups: + if len(group) == 3 and group[2].startswith(b"/docker/"): + # If it starts with /docker/, we're in a docker container + return True + + return False + + + def _supportsSymlinks(self): + """ + Check for symlink support usable for Twisted's purposes. + + @return: C{True} if symlinks are supported on the current platform, + otherwise C{False}. + @rtype: L{bool} + """ + if self.isWindows(): + # We do the isWindows() check as newer Pythons support the symlink + # support in Vista+, but only if you have some obscure permission + # (SeCreateSymbolicLinkPrivilege), which can only be given on + # platforms with msc.exe (so, Business/Enterprise editions). + # This uncommon requirement makes the Twisted test suite test fail + # in 99.99% of cases as general users don't have permission to do + # it, even if there is "symlink support". + return False + else: + # If we're not on Windows, check for existence of os.symlink. + try: + os.symlink + except AttributeError: + return False + else: + return True + + + def supportsThreads(self): + """ + Can threads be created? + + @return: C{True} if the threads are supported on the current platform. + @rtype: C{bool} + """ + try: + import threading + return threading is not None # shh pyflakes + except ImportError: + return False + + + def supportsINotify(self): + """ + Return C{True} if we can use the inotify API on this platform. + + @since: 10.1 + """ + try: + from twisted.python._inotify import INotifyError, init + except ImportError: + return False + + try: + os.close(init()) + except INotifyError: + return False + return True + + +platform = Platform() +platformType = platform.getType() +seconds = platform.seconds diff --git a/contrib/python/Twisted/py2/twisted/python/sendmsg.py b/contrib/python/Twisted/py2/twisted/python/sendmsg.py new file mode 100644 index 00000000000..7d50e4eee08 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/sendmsg.py @@ -0,0 +1,106 @@ +# -*- test-case-name: twisted.test.test_sendmsg -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +sendmsg(2) and recvmsg(2) support for Python. +""" + +from __future__ import absolute_import, division + +from collections import namedtuple +from twisted.python.compat import _PY3 + +__all__ = ["sendmsg", "recvmsg", "getSocketFamily", "SCM_RIGHTS"] + +if not _PY3: + from twisted.python._sendmsg import send1msg, recv1msg + from twisted.python._sendmsg import getsockfam, SCM_RIGHTS + __all__ += ["send1msg", "recv1msg", "getsockfam"] +else: + from socket import SCM_RIGHTS, CMSG_SPACE + +RecievedMessage = namedtuple('RecievedMessage', ['data', 'ancillary', 'flags']) + + + +def sendmsg(socket, data, ancillary=[], flags=0): + """ + Send a message on a socket. + + @param socket: The socket to send the message on. + @type socket: L{socket.socket} + + @param data: Bytes to write to the socket. + @type data: bytes + + @param ancillary: Extra data to send over the socket outside of the normal + datagram or stream mechanism. By default no ancillary data is sent. + @type ancillary: C{list} of C{tuple} of C{int}, C{int}, and C{bytes}. + + @param flags: Flags to affect how the message is sent. See the C{MSG_} + constants in the sendmsg(2) manual page. By default no flags are set. + @type flags: C{int} + + @return: The return value of the underlying syscall, if it succeeds. + """ + if _PY3: + return socket.sendmsg([data], ancillary, flags) + else: + return send1msg(socket.fileno(), data, flags, ancillary) + + + +def recvmsg(socket, maxSize=8192, cmsgSize=4096, flags=0): + """ + Receive a message on a socket. + + @param socket: The socket to receive the message on. + @type socket: L{socket.socket} + + @param maxSize: The maximum number of bytes to receive from the socket using + the datagram or stream mechanism. The default maximum is 8192. + @type maxSize: L{int} + + @param cmsgSize: The maximum number of bytes to receive from the socket + outside of the normal datagram or stream mechanism. The default maximum + is 4096. + @type cmsgSize: L{int} + + @param flags: Flags to affect how the message is sent. See the C{MSG_} + constants in the sendmsg(2) manual page. By default no flags are set. + @type flags: L{int} + + @return: A named 3-tuple of the bytes received using the datagram/stream + mechanism, a L{list} of L{tuple}s giving ancillary received data, and + flags as an L{int} describing the data received. + """ + if _PY3: + # In Twisted's sendmsg.c, the csmg_space is defined as: + # int cmsg_size = 4096; + # cmsg_space = CMSG_SPACE(cmsg_size); + # Since the default in Python 3's socket is 0, we need to define our + # own default of 4096. -hawkie + data, ancillary, flags = socket.recvmsg( + maxSize, CMSG_SPACE(cmsgSize), flags)[0:3] + else: + data, flags, ancillary = recv1msg( + socket.fileno(), flags, maxSize, cmsgSize) + + return RecievedMessage(data=data, ancillary=ancillary, flags=flags) + + + +def getSocketFamily(socket): + """ + Return the family of the given socket. + + @param socket: The socket to get the family of. + @type socket: L{socket.socket} + + @rtype: L{int} + """ + if _PY3: + return socket.family + else: + return getsockfam(socket.fileno()) diff --git a/contrib/python/Twisted/py2/twisted/python/shortcut.py b/contrib/python/Twisted/py2/twisted/python/shortcut.py new file mode 100644 index 00000000000..cb2f1fbd73c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/shortcut.py @@ -0,0 +1,85 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Creation of Windows shortcuts. + +Requires win32all. +""" + +from win32com.shell import shell +import pythoncom +import os + + +def open(filename): + """ + Open an existing shortcut for reading. + + @return: The shortcut object + @rtype: Shortcut + """ + sc = Shortcut() + sc.load(filename) + return sc + + + +class Shortcut: + """ + A shortcut on Win32. + """ + + def __init__(self, + path=None, + arguments=None, + description=None, + workingdir=None, + iconpath=None, + iconidx=0): + """ + @param path: Location of the target + @param arguments: If path points to an executable, optional arguments + to pass + @param description: Human-readable description of target + @param workingdir: Directory from which target is launched + @param iconpath: Filename that contains an icon for the shortcut + @param iconidx: If iconpath is set, optional index of the icon desired + """ + self._base = pythoncom.CoCreateInstance( + shell.CLSID_ShellLink, None, + pythoncom.CLSCTX_INPROC_SERVER, shell.IID_IShellLink + ) + if path is not None: + self.SetPath(os.path.abspath(path)) + if arguments is not None: + self.SetArguments(arguments) + if description is not None: + self.SetDescription(description) + if workingdir is not None: + self.SetWorkingDirectory(os.path.abspath(workingdir)) + if iconpath is not None: + self.SetIconLocation(os.path.abspath(iconpath), iconidx) + + + def load(self, filename): + """ + Read a shortcut file from disk. + """ + self._base.QueryInterface(pythoncom.IID_IPersistFile).Load( + os.path.abspath(filename)) + + + def save(self, filename): + """ + Write the shortcut to disk. + + The file should be named something.lnk. + """ + self._base.QueryInterface(pythoncom.IID_IPersistFile).Save( + os.path.abspath(filename), 0) + + + def __getattr__(self, name): + return getattr(self._base, name) diff --git a/contrib/python/Twisted/py2/twisted/python/syslog.py b/contrib/python/Twisted/py2/twisted/python/syslog.py new file mode 100644 index 00000000000..4ff2e1e6b2d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/syslog.py @@ -0,0 +1,109 @@ +# -*- test-case-name: twisted.python.test.test_syslog -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Classes and utility functions for integrating Twisted and syslog. + +You probably want to call L{startLogging}. +""" + +syslog = __import__('syslog') + +from twisted.python import log +from twisted.python._oldstyle import _oldStyle + +# These defaults come from the Python syslog docs. +DEFAULT_OPTIONS = 0 +DEFAULT_FACILITY = syslog.LOG_USER + + + +@_oldStyle +class SyslogObserver: + """ + A log observer for logging to syslog. + + See L{twisted.python.log} for context. + + This logObserver will automatically use LOG_ALERT priority for logged + failures (such as from C{log.err()}), but you can use any priority and + facility by setting the 'C{syslogPriority}' and 'C{syslogFacility}' keys in + the event dict. + """ + openlog = syslog.openlog + syslog = syslog.syslog + + def __init__(self, prefix, options=DEFAULT_OPTIONS, + facility=DEFAULT_FACILITY): + """ + @type prefix: C{str} + @param prefix: The syslog prefix to use. + + @type options: C{int} + @param options: A bitvector represented as an integer of the syslog + options to use. + + @type facility: C{int} + @param facility: An indication to the syslog daemon of what sort of + program this is (essentially, an additional arbitrary metadata + classification for messages sent to syslog by this observer). + """ + self.openlog(prefix, options, facility) + + + def emit(self, eventDict): + """ + Send a message event to the I{syslog}. + + @param eventDict: The event to send. If it has no C{'message'} key, it + will be ignored. Otherwise, if it has C{'syslogPriority'} and/or + C{'syslogFacility'} keys, these will be used as the syslog priority + and facility. If it has no C{'syslogPriority'} key but a true + value for the C{'isError'} key, the B{LOG_ALERT} priority will be + used; if it has a false value for C{'isError'}, B{LOG_INFO} will be + used. If the C{'message'} key is multiline, each line will be sent + to the syslog separately. + """ + # Figure out what the message-text is. + text = log.textFromEventDict(eventDict) + if text is None: + return + + # Figure out what syslog parameters we might need to use. + priority = syslog.LOG_INFO + facility = 0 + if eventDict['isError']: + priority = syslog.LOG_ALERT + if 'syslogPriority' in eventDict: + priority = int(eventDict['syslogPriority']) + if 'syslogFacility' in eventDict: + facility = int(eventDict['syslogFacility']) + + # Break the message up into lines and send them. + lines = text.split('\n') + while lines[-1:] == ['']: + lines.pop() + + firstLine = True + for line in lines: + if firstLine: + firstLine = False + else: + line = '\t' + line + self.syslog(priority | facility, + '[%s] %s' % (eventDict['system'], line)) + + + +def startLogging(prefix='Twisted', options=DEFAULT_OPTIONS, + facility=DEFAULT_FACILITY, setStdout=1): + """ + Send all Twisted logging output to syslog from now on. + + The prefix, options and facility arguments are passed to + C{syslog.openlog()}, see the Python syslog documentation for details. For + other parameters, see L{twisted.python.log.startLoggingWithObserver}. + """ + obs = SyslogObserver(prefix, options, facility) + log.startLoggingWithObserver(obs.emit, setStdout=setStdout) diff --git a/contrib/python/Twisted/py2/twisted/python/systemd.py b/contrib/python/Twisted/py2/twisted/python/systemd.py new file mode 100644 index 00000000000..045d9f6a422 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/systemd.py @@ -0,0 +1,89 @@ +# -*- test-case-name: twisted.python.test.test_systemd -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Integration with systemd. + +Currently only the minimum APIs necessary for using systemd's socket activation +feature are supported. +""" + +from __future__ import division, absolute_import + +__all__ = ['ListenFDs'] + +from os import getpid + + +class ListenFDs(object): + """ + L{ListenFDs} provides access to file descriptors inherited from systemd. + + Typically L{ListenFDs.fromEnvironment} should be used to construct a new + instance of L{ListenFDs}. + + @cvar _START: File descriptors inherited from systemd are always + consecutively numbered, with a fixed lowest "starting" descriptor. This + gives the default starting descriptor. Since this must agree with the + value systemd is using, it typically should not be overridden. + @type _START: C{int} + + @ivar _descriptors: A C{list} of C{int} giving the descriptors which were + inherited. + """ + _START = 3 + + def __init__(self, descriptors): + """ + @param descriptors: The descriptors which will be returned from calls to + C{inheritedDescriptors}. + """ + self._descriptors = descriptors + + + @classmethod + def fromEnvironment(cls, environ=None, start=None): + """ + @param environ: A dictionary-like object to inspect to discover + inherited descriptors. By default, L{None}, indicating that the + real process environment should be inspected. The default is + suitable for typical usage. + + @param start: An integer giving the lowest value of an inherited + descriptor systemd will give us. By default, L{None}, indicating + the known correct (that is, in agreement with systemd) value will be + used. The default is suitable for typical usage. + + @return: A new instance of C{cls} which can be used to look up the + descriptors which have been inherited. + """ + if environ is None: + from os import environ + if start is None: + start = cls._START + + descriptors = [] + + try: + pid = int(environ['LISTEN_PID']) + except (KeyError, ValueError): + pass + else: + if pid == getpid(): + try: + count = int(environ['LISTEN_FDS']) + except (KeyError, ValueError): + pass + else: + descriptors = range(start, start + count) + del environ['LISTEN_PID'], environ['LISTEN_FDS'] + + return cls(descriptors) + + + def inheritedDescriptors(self): + """ + @return: The configured list of descriptors. + """ + return list(self._descriptors) diff --git a/contrib/python/Twisted/py2/twisted/python/text.py b/contrib/python/Twisted/py2/twisted/python/text.py new file mode 100644 index 00000000000..37f1e37dd96 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/text.py @@ -0,0 +1,208 @@ +# -*- test-case-name: twisted.test.test_text -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Miscellany of text-munging functions. +""" + + +def stringyString(object, indentation=''): + """ + Expansive string formatting for sequence types. + + C{list.__str__} and C{dict.__str__} use C{repr()} to display their + elements. This function also turns these sequence types + into strings, but uses C{str()} on their elements instead. + + Sequence elements are also displayed on separate lines, and nested + sequences have nested indentation. + """ + braces = '' + sl = [] + + if type(object) is dict: + braces = '{}' + for key, value in object.items(): + value = stringyString(value, indentation + ' ') + if isMultiline(value): + if endsInNewline(value): + value = value[:-len('\n')] + sl.append("%s %s:\n%s" % (indentation, key, value)) + else: + # Oops. Will have to move that indentation. + sl.append("%s %s: %s" % (indentation, key, + value[len(indentation) + 3:])) + + elif type(object) is tuple or type(object) is list: + if type(object) is tuple: + braces = '()' + else: + braces = '[]' + + for element in object: + element = stringyString(element, indentation + ' ') + sl.append(element.rstrip() + ',') + else: + sl[:] = map(lambda s, i=indentation: i + s, + str(object).split('\n')) + + if not sl: + sl.append(indentation) + + if braces: + sl[0] = indentation + braces[0] + sl[0][len(indentation) + 1:] + sl[-1] = sl[-1] + braces[-1] + + s = "\n".join(sl) + + if isMultiline(s) and not endsInNewline(s): + s = s + '\n' + + return s + + +def isMultiline(s): + """ + Returns C{True} if this string has a newline in it. + """ + return (s.find('\n') != -1) + + +def endsInNewline(s): + """ + Returns C{True} if this string ends in a newline. + """ + return (s[-len('\n'):] == '\n') + + +def greedyWrap(inString, width=80): + """ + Given a string and a column width, return a list of lines. + + Caveat: I'm use a stupid greedy word-wrapping + algorythm. I won't put two spaces at the end + of a sentence. I don't do full justification. + And no, I've never even *heard* of hypenation. + """ + + outLines = [] + + #eww, evil hacks to allow paragraphs delimited by two \ns :( + if inString.find('\n\n') >= 0: + paragraphs = inString.split('\n\n') + for para in paragraphs: + outLines.extend(greedyWrap(para, width) + ['']) + return outLines + inWords = inString.split() + + column = 0 + ptr_line = 0 + while inWords: + column = column + len(inWords[ptr_line]) + ptr_line = ptr_line + 1 + + if (column > width): + if ptr_line == 1: + # This single word is too long, it will be the whole line. + pass + else: + # We've gone too far, stop the line one word back. + ptr_line = ptr_line - 1 + (l, inWords) = (inWords[0:ptr_line], inWords[ptr_line:]) + outLines.append(' '.join(l)) + + ptr_line = 0 + column = 0 + elif not (len(inWords) > ptr_line): + # Clean up the last bit. + outLines.append(' '.join(inWords)) + del inWords[:] + else: + # Space + column = column + 1 + # next word + + return outLines + + +wordWrap = greedyWrap + + +def removeLeadingBlanks(lines): + ret = [] + for line in lines: + if ret or line.strip(): + ret.append(line) + return ret + + +def removeLeadingTrailingBlanks(s): + lines = removeLeadingBlanks(s.split('\n')) + lines.reverse() + lines = removeLeadingBlanks(lines) + lines.reverse() + return '\n'.join(lines)+'\n' + + +def splitQuoted(s): + """ + Like a string split, but don't break substrings inside quotes. + + >>> splitQuoted('the "hairy monkey" likes pie') + ['the', 'hairy monkey', 'likes', 'pie'] + + Another one of those "someone must have a better solution for + this" things. This implementation is a VERY DUMB hack done too + quickly. + """ + out = [] + quot = None + phrase = None + for word in s.split(): + if phrase is None: + if word and (word[0] in ("\"", "'")): + quot = word[0] + word = word[1:] + phrase = [] + + if phrase is None: + out.append(word) + else: + if word and (word[-1] == quot): + word = word[:-1] + phrase.append(word) + out.append(" ".join(phrase)) + phrase = None + else: + phrase.append(word) + + return out + + +def strFile(p, f, caseSensitive=True): + """ + Find whether string C{p} occurs in a read()able object C{f}. + + @rtype: C{bool} + """ + buf = type(p)() + buf_len = max(len(p), 2**2**2**2) + if not caseSensitive: + p = p.lower() + while 1: + r = f.read(buf_len-len(p)) + if not caseSensitive: + r = r.lower() + bytes_read = len(r) + if bytes_read == 0: + return False + l = len(buf)+bytes_read-buf_len + if l <= 0: + buf = buf + r + else: + buf = buf[l:] + r + if buf.find(p) != -1: + return True + diff --git a/contrib/python/Twisted/py2/twisted/python/threadable.py b/contrib/python/Twisted/py2/twisted/python/threadable.py new file mode 100644 index 00000000000..2949fc4b2ba --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/threadable.py @@ -0,0 +1,141 @@ +# -*- test-case-name: twisted.python.test_threadable -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A module to provide some very basic threading primitives, such as +synchronization. +""" + +from __future__ import division, absolute_import + +from functools import wraps + +class DummyLock(object): + """ + Hack to allow locks to be unpickled on an unthreaded system. + """ + + def __reduce__(self): + return (unpickle_lock, ()) + + + +def unpickle_lock(): + if threadingmodule is not None: + return XLock() + else: + return DummyLock() +unpickle_lock.__safe_for_unpickling__ = True + + + +def _synchPre(self): + if '_threadable_lock' not in self.__dict__: + _synchLockCreator.acquire() + if '_threadable_lock' not in self.__dict__: + self.__dict__['_threadable_lock'] = XLock() + _synchLockCreator.release() + self._threadable_lock.acquire() + + + +def _synchPost(self): + self._threadable_lock.release() + + + +def _sync(klass, function): + @wraps(function) + def sync(self, *args, **kwargs): + _synchPre(self) + try: + return function(self, *args, **kwargs) + finally: + _synchPost(self) + return sync + + + +def synchronize(*klasses): + """ + Make all methods listed in each class' synchronized attribute synchronized. + + The synchronized attribute should be a list of strings, consisting of the + names of methods that must be synchronized. If we are running in threaded + mode these methods will be wrapped with a lock. + """ + if threadingmodule is not None: + for klass in klasses: + for methodName in klass.synchronized: + sync = _sync(klass, klass.__dict__[methodName]) + setattr(klass, methodName, sync) + + + +def init(with_threads=1): + """Initialize threading. + + Don't bother calling this. If it needs to happen, it will happen. + """ + global threaded, _synchLockCreator, XLock + + if with_threads: + if not threaded: + if threadingmodule is not None: + threaded = True + + class XLock(threadingmodule._RLock, object): + def __reduce__(self): + return (unpickle_lock, ()) + + _synchLockCreator = XLock() + else: + raise RuntimeError("Cannot initialize threading, platform lacks thread support") + else: + if threaded: + raise RuntimeError("Cannot uninitialize threads") + else: + pass + + + +_dummyID = object() +def getThreadID(): + if threadingmodule is None: + return _dummyID + return threadingmodule.currentThread().ident + + + +def isInIOThread(): + """Are we in the thread responsible for I/O requests (the event loop)? + """ + return ioThread == getThreadID() + + + +def registerAsIOThread(): + """Mark the current thread as responsible for I/O requests. + """ + global ioThread + ioThread = getThreadID() + + +ioThread = None +threaded = False +# Define these globals which might be overwritten in init(). +_synchLockCreator = None +XLock = None + + +try: + import threading as threadingmodule +except ImportError: + threadingmodule = None +else: + init(True) + + + +__all__ = ['isInIOThread', 'registerAsIOThread', 'getThreadID', 'XLock'] diff --git a/contrib/python/Twisted/py2/twisted/python/threadpool.py b/contrib/python/Twisted/py2/twisted/python/threadpool.py new file mode 100644 index 00000000000..351c5ccbb02 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/threadpool.py @@ -0,0 +1,320 @@ +# -*- test-case-name: twisted.test.test_threadpool -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +twisted.python.threadpool: a pool of threads to which we dispatch tasks. + +In most cases you can just use C{reactor.callInThread} and friends +instead of creating a thread pool directly. +""" + +from __future__ import division, absolute_import + +import threading + +from twisted._threads import pool as _pool +from twisted.python import log, context +from twisted.python.failure import Failure +from twisted.python._oldstyle import _oldStyle + + +WorkerStop = object() + + + +@_oldStyle +class ThreadPool: + """ + This class (hopefully) generalizes the functionality of a pool of threads + to which work can be dispatched. + + L{callInThread} and L{stop} should only be called from a single thread. + + @ivar started: Whether or not the thread pool is currently running. + @type started: L{bool} + + @ivar threads: List of workers currently running in this thread pool. + @type threads: L{list} + + @ivar _pool: A hook for testing. + @type _pool: callable compatible with L{_pool} + """ + min = 5 + max = 20 + joined = False + started = False + workers = 0 + name = None + + threadFactory = threading.Thread + currentThread = staticmethod(threading.currentThread) + _pool = staticmethod(_pool) + + def __init__(self, minthreads=5, maxthreads=20, name=None): + """ + Create a new threadpool. + + @param minthreads: minimum number of threads in the pool + @type minthreads: L{int} + + @param maxthreads: maximum number of threads in the pool + @type maxthreads: L{int} + + @param name: The name to give this threadpool; visible in log messages. + @type name: native L{str} + """ + assert minthreads >= 0, 'minimum is negative' + assert minthreads <= maxthreads, 'minimum is greater than maximum' + self.min = minthreads + self.max = maxthreads + self.name = name + self.threads = [] + + def trackingThreadFactory(*a, **kw): + thread = self.threadFactory(*a, name=self._generateName(), **kw) + self.threads.append(thread) + return thread + + def currentLimit(): + if not self.started: + return 0 + return self.max + + self._team = self._pool(currentLimit, trackingThreadFactory) + + + @property + def workers(self): + """ + For legacy compatibility purposes, return a total number of workers. + + @return: the current number of workers, both idle and busy (but not + those that have been quit by L{ThreadPool.adjustPoolsize}) + @rtype: L{int} + """ + stats = self._team.statistics() + return stats.idleWorkerCount + stats.busyWorkerCount + + + @property + def working(self): + """ + For legacy compatibility purposes, return the number of busy workers as + expressed by a list the length of that number. + + @return: the number of workers currently processing a work item. + @rtype: L{list} of L{None} + """ + return [None] * self._team.statistics().busyWorkerCount + + + @property + def waiters(self): + """ + For legacy compatibility purposes, return the number of idle workers as + expressed by a list the length of that number. + + @return: the number of workers currently alive (with an allocated + thread) but waiting for new work. + @rtype: L{list} of L{None} + """ + return [None] * self._team.statistics().idleWorkerCount + + + @property + def _queue(self): + """ + For legacy compatibility purposes, return an object with a C{qsize} + method that indicates the amount of work not yet allocated to a worker. + + @return: an object with a C{qsize} method. + """ + class NotAQueue(object): + def qsize(q): + """ + Pretend to be a Python threading Queue and return the + number of as-yet-unconsumed tasks. + + @return: the amount of backlogged work not yet dispatched to a + worker. + @rtype: L{int} + """ + return self._team.statistics().backloggedWorkCount + return NotAQueue() + + q = _queue # Yes, twistedchecker, I want a single-letter + # attribute name. + + + def start(self): + """ + Start the threadpool. + """ + self.joined = False + self.started = True + # Start some threads. + self.adjustPoolsize() + backlog = self._team.statistics().backloggedWorkCount + if backlog: + self._team.grow(backlog) + + + def startAWorker(self): + """ + Increase the number of available workers for the thread pool by 1, up + to the maximum allowed by L{ThreadPool.max}. + """ + self._team.grow(1) + + + def _generateName(self): + """ + Generate a name for a new pool thread. + + @return: A distinctive name for the thread. + @rtype: native L{str} + """ + return "PoolThread-%s-%s" % (self.name or id(self), self.workers) + + + def stopAWorker(self): + """ + Decrease the number of available workers by 1, by quitting one as soon + as it's idle. + """ + self._team.shrink(1) + + + def __setstate__(self, state): + setattr(self, "__dict__", state) + ThreadPool.__init__(self, self.min, self.max) + + + def __getstate__(self): + state = {} + state['min'] = self.min + state['max'] = self.max + return state + + + def callInThread(self, func, *args, **kw): + """ + Call a callable object in a separate thread. + + @param func: callable object to be called in separate thread + + @param args: positional arguments to be passed to C{func} + + @param kw: keyword args to be passed to C{func} + """ + self.callInThreadWithCallback(None, func, *args, **kw) + + + def callInThreadWithCallback(self, onResult, func, *args, **kw): + """ + Call a callable object in a separate thread and call C{onResult} with + the return value, or a L{twisted.python.failure.Failure} if the + callable raises an exception. + + The callable is allowed to block, but the C{onResult} function must not + block and should perform as little work as possible. + + A typical action for C{onResult} for a threadpool used with a Twisted + reactor would be to schedule a L{twisted.internet.defer.Deferred} to + fire in the main reactor thread using C{.callFromThread}. Note that + C{onResult} is called inside the separate thread, not inside the + reactor thread. + + @param onResult: a callable with the signature C{(success, result)}. + If the callable returns normally, C{onResult} is called with + C{(True, result)} where C{result} is the return value of the + callable. If the callable throws an exception, C{onResult} is + called with C{(False, failure)}. + + Optionally, C{onResult} may be L{None}, in which case it is not + called at all. + + @param func: callable object to be called in separate thread + + @param args: positional arguments to be passed to C{func} + + @param kw: keyword arguments to be passed to C{func} + """ + if self.joined: + return + ctx = context.theContextTracker.currentContext().contexts[-1] + + def inContext(): + try: + result = inContext.theWork() + ok = True + except: + result = Failure() + ok = False + + inContext.theWork = None + if inContext.onResult is not None: + inContext.onResult(ok, result) + inContext.onResult = None + elif not ok: + log.err(result) + + # Avoid closing over func, ctx, args, kw so that we can carefully + # manage their lifecycle. See + # test_threadCreationArgumentsCallInThreadWithCallback. + inContext.theWork = lambda: context.call(ctx, func, *args, **kw) + inContext.onResult = onResult + + self._team.do(inContext) + + + def stop(self): + """ + Shutdown the threads in the threadpool. + """ + self.joined = True + self.started = False + self._team.quit() + for thread in self.threads: + thread.join() + + + def adjustPoolsize(self, minthreads=None, maxthreads=None): + """ + Adjust the number of available threads by setting C{min} and C{max} to + new values. + + @param minthreads: The new value for L{ThreadPool.min}. + + @param maxthreads: The new value for L{ThreadPool.max}. + """ + if minthreads is None: + minthreads = self.min + if maxthreads is None: + maxthreads = self.max + + assert minthreads >= 0, 'minimum is negative' + assert minthreads <= maxthreads, 'minimum is greater than maximum' + + self.min = minthreads + self.max = maxthreads + if not self.started: + return + + # Kill of some threads if we have too many. + if self.workers > self.max: + self._team.shrink(self.workers - self.max) + # Start some threads if we have too few. + if self.workers < self.min: + self._team.grow(self.min - self.workers) + + + def dumpStats(self): + """ + Dump some plain-text informational messages to the log about the state + of this L{ThreadPool}. + """ + log.msg('waiters: %s' % (self.waiters,)) + log.msg('workers: %s' % (self.working,)) + log.msg('total: %s' % (self.threads,)) diff --git a/contrib/python/Twisted/py2/twisted/python/url.py b/contrib/python/Twisted/py2/twisted/python/url.py new file mode 100644 index 00000000000..f3591db8722 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/url.py @@ -0,0 +1,15 @@ +# -*- test-case-name: twisted.python.test.test_url -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +URL parsing, construction and rendering. + +@see: L{URL} +""" + +from hyperlink import URL + +__all__ = [ + "URL", +] diff --git a/contrib/python/Twisted/py2/twisted/python/urlpath.py b/contrib/python/Twisted/py2/twisted/python/urlpath.py new file mode 100644 index 00000000000..a919502a056 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/urlpath.py @@ -0,0 +1,294 @@ +# -*- test-case-name: twisted.python.test.test_urlpath -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +L{URLPath}, a representation of a URL. +""" + +from __future__ import division, absolute_import + +from twisted.python.compat import ( + nativeString, unicode, urllib_parse as urlparse, urlunquote, urlquote +) + +from hyperlink import URL as _URL + +_allascii = b"".join([chr(x).encode('ascii') for x in range(1, 128)]) + +def _rereconstituter(name): + """ + Attriute declaration to preserve mutability on L{URLPath}. + + @param name: a public attribute name + @type name: native L{str} + + @return: a descriptor which retrieves the private version of the attribute + on get and calls rerealize on set. + """ + privateName = nativeString("_") + name + return property( + lambda self: getattr(self, privateName), + lambda self, value: (setattr(self, privateName, + value if isinstance(value, bytes) + else value.encode("charmap")) or + self._reconstitute()) + ) + + + +class URLPath(object): + """ + A representation of a URL. + + @ivar scheme: The scheme of the URL (e.g. 'http'). + @type scheme: L{bytes} + + @ivar netloc: The network location ("host"). + @type netloc: L{bytes} + + @ivar path: The path on the network location. + @type path: L{bytes} + + @ivar query: The query argument (the portion after ? in the URL). + @type query: L{bytes} + + @ivar fragment: The page fragment (the portion after # in the URL). + @type fragment: L{bytes} + """ + def __init__(self, scheme=b'', netloc=b'localhost', path=b'', + query=b'', fragment=b''): + self._scheme = scheme or b'http' + self._netloc = netloc + self._path = path or b'/' + self._query = query + self._fragment = fragment + self._reconstitute() + + + def _reconstitute(self): + """ + Reconstitute this L{URLPath} from all its given attributes. + """ + urltext = urlquote( + urlparse.urlunsplit((self._scheme, self._netloc, + self._path, self._query, self._fragment)), + safe=_allascii + ) + self._url = _URL.fromText(urltext.encode("ascii").decode("ascii")) + + scheme = _rereconstituter("scheme") + netloc = _rereconstituter("netloc") + path = _rereconstituter("path") + query = _rereconstituter("query") + fragment = _rereconstituter("fragment") + + + @classmethod + def _fromURL(cls, urlInstance): + """ + Reconstruct all the public instance variables of this L{URLPath} from + its underlying L{_URL}. + + @param urlInstance: the object to base this L{URLPath} on. + @type urlInstance: L{_URL} + + @return: a new L{URLPath} + """ + self = cls.__new__(cls) + self._url = urlInstance.replace(path=urlInstance.path or [u""]) + self._scheme = self._url.scheme.encode("ascii") + self._netloc = self._url.authority().encode("ascii") + self._path = (_URL(path=self._url.path, + rooted=True).asURI().asText() + .encode("ascii")) + self._query = (_URL(query=self._url.query).asURI().asText() + .encode("ascii"))[1:] + self._fragment = self._url.fragment.encode("ascii") + return self + + + def pathList(self, unquote=False, copy=True): + """ + Split this URL's path into its components. + + @param unquote: whether to remove %-encoding from the returned strings. + + @param copy: (ignored, do not use) + + @return: The components of C{self.path} + @rtype: L{list} of L{bytes} + """ + segments = self._url.path + mapper = lambda x: x.encode("ascii") + if unquote: + mapper = (lambda x, m=mapper: m(urlunquote(x))) + return [b''] + [mapper(segment) for segment in segments] + + + @classmethod + def fromString(klass, url): + """ + Make a L{URLPath} from a L{str} or L{unicode}. + + @param url: A L{str} representation of a URL. + @type url: L{str} or L{unicode}. + + @return: a new L{URLPath} derived from the given string. + @rtype: L{URLPath} + """ + if not isinstance(url, (str, unicode)): + raise ValueError("'url' must be a str or unicode") + if isinstance(url, bytes): + # On Python 2, accepting 'str' (for compatibility) means we might + # get 'bytes'. On py3, this will not work with bytes due to the + # check above. + return klass.fromBytes(url) + return klass._fromURL(_URL.fromText(url)) + + + @classmethod + def fromBytes(klass, url): + """ + Make a L{URLPath} from a L{bytes}. + + @param url: A L{bytes} representation of a URL. + @type url: L{bytes} + + @return: a new L{URLPath} derived from the given L{bytes}. + @rtype: L{URLPath} + + @since: 15.4 + """ + if not isinstance(url, bytes): + raise ValueError("'url' must be bytes") + quoted = urlquote(url, safe=_allascii) + if isinstance(quoted, bytes): + # This will only be bytes on python 2, where we can transform it + # into unicode. On python 3, urlquote always returns str. + quoted = quoted.decode("ascii") + return klass.fromString(quoted) + + + @classmethod + def fromRequest(klass, request): + """ + Make a L{URLPath} from a L{twisted.web.http.Request}. + + @param request: A L{twisted.web.http.Request} to make the L{URLPath} + from. + + @return: a new L{URLPath} derived from the given request. + @rtype: L{URLPath} + """ + return klass.fromBytes(request.prePathURL()) + + + def _mod(self, newURL, keepQuery): + """ + Return a modified copy of C{self} using C{newURL}, keeping the query + string if C{keepQuery} is C{True}. + + @param newURL: a L{URL} to derive a new L{URLPath} from + @type newURL: L{URL} + + @param keepQuery: if C{True}, preserve the query parameters from + C{self} on the new L{URLPath}; if C{False}, give the new L{URLPath} + no query parameters. + @type keepQuery: L{bool} + + @return: a new L{URLPath} + """ + return self._fromURL(newURL.replace( + fragment=u'', query=self._url.query if keepQuery else () + )) + + + def sibling(self, path, keepQuery=False): + """ + Get the sibling of the current L{URLPath}. A sibling is a file which + is in the same directory as the current file. + + @param path: The path of the sibling. + @type path: L{bytes} + + @param keepQuery: Whether to keep the query parameters on the returned + L{URLPath}. + @type: keepQuery: L{bool} + + @return: a new L{URLPath} + """ + return self._mod(self._url.sibling(path.decode("ascii")), keepQuery) + + + def child(self, path, keepQuery=False): + """ + Get the child of this L{URLPath}. + + @param path: The path of the child. + @type path: L{bytes} + + @param keepQuery: Whether to keep the query parameters on the returned + L{URLPath}. + @type: keepQuery: L{bool} + + @return: a new L{URLPath} + """ + return self._mod(self._url.child(path.decode("ascii")), keepQuery) + + + def parent(self, keepQuery=False): + """ + Get the parent directory of this L{URLPath}. + + @param keepQuery: Whether to keep the query parameters on the returned + L{URLPath}. + @type: keepQuery: L{bool} + + @return: a new L{URLPath} + """ + return self._mod(self._url.click(u".."), keepQuery) + + + def here(self, keepQuery=False): + """ + Get the current directory of this L{URLPath}. + + @param keepQuery: Whether to keep the query parameters on the returned + L{URLPath}. + @type: keepQuery: L{bool} + + @return: a new L{URLPath} + """ + return self._mod(self._url.click(u"."), keepQuery) + + + def click(self, st): + """ + Return a path which is the URL where a browser would presumably take + you if you clicked on a link with an HREF as given. + + @param st: A relative URL, to be interpreted relative to C{self} as the + base URL. + @type st: L{bytes} + + @return: a new L{URLPath} + """ + return self._fromURL(self._url.click(st.decode("ascii"))) + + + def __str__(self): + """ + The L{str} of a L{URLPath} is its URL text. + """ + return nativeString(self._url.asURI().asText()) + + + def __repr__(self): + """ + The L{repr} of a L{URLPath} is an eval-able expression which will + construct a similar L{URLPath}. + """ + return ('URLPath(scheme=%r, netloc=%r, path=%r, query=%r, fragment=%r)' + % (self.scheme, self.netloc, self.path, self.query, + self.fragment)) diff --git a/contrib/python/Twisted/py2/twisted/python/usage.py b/contrib/python/Twisted/py2/twisted/python/usage.py new file mode 100644 index 00000000000..8114861db06 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/usage.py @@ -0,0 +1,1001 @@ +# -*- test-case-name: twisted.test.test_usage -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +twisted.python.usage is a module for parsing/handling the +command line of your program. + +For information on how to use it, see +U{http://twistedmatrix.com/projects/core/documentation/howto/options.html}, +or doc/core/howto/options.xhtml in your Twisted directory. +""" + +from __future__ import print_function +from __future__ import division, absolute_import + +# System Imports +import inspect +import os +import sys +import getopt +from os import path +import textwrap + +# Sibling Imports +from twisted.python import reflect, util +from twisted.python.compat import _PY3 + +class UsageError(Exception): + pass + + +error = UsageError + + +class CoerceParameter(object): + """ + Utility class that can corce a parameter before storing it. + """ + def __init__(self, options, coerce): + """ + @param options: parent Options object + @param coerce: callable used to coerce the value. + """ + self.options = options + self.coerce = coerce + self.doc = getattr(self.coerce, 'coerceDoc', '') + + def dispatch(self, parameterName, value): + """ + When called in dispatch, do the coerce for C{value} and save the + returned value. + """ + if value is None: + raise UsageError("Parameter '%s' requires an argument." + % (parameterName,)) + try: + value = self.coerce(value) + except ValueError as e: + raise UsageError("Parameter type enforcement failed: %s" % (e,)) + + self.options.opts[parameterName] = value + + +class Options(dict): + """ + An option list parser class + + C{optFlags} and C{optParameters} are lists of available parameters + which your program can handle. The difference between the two + is the 'flags' have an on(1) or off(0) state (off by default) + whereas 'parameters' have an assigned value, with an optional + default. (Compare '--verbose' and '--verbosity=2') + + optFlags is assigned a list of lists. Each list represents + a flag parameter, as so:: + + optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'], + ['quiet', 'q', 'Be vewy vewy quiet.']] + + As you can see, the first item is the long option name + (prefixed with '--' on the command line), followed by the + short option name (prefixed with '-'), and the description. + The description is used for the built-in handling of the + --help switch, which prints a usage summary. + + C{optParameters} is much the same, except the list also contains + a default value:: + + optParameters = [['outfile', 'O', 'outfile.log', 'Description...']] + + A coerce function can also be specified as the last element: it will be + called with the argument and should return the value that will be stored + for the option. This function can have a C{coerceDoc} attribute which + will be appended to the documentation of the option. + + subCommands is a list of 4-tuples of (command name, command shortcut, + parser class, documentation). If the first non-option argument found is + one of the given command names, an instance of the given parser class is + instantiated and given the remainder of the arguments to parse and + self.opts[command] is set to the command name. For example:: + + subCommands = [ + ['inquisition', 'inquest', InquisitionOptions, + 'Perform an inquisition'], + ['holyquest', 'quest', HolyQuestOptions, + 'Embark upon a holy quest'] + ] + + In this case, C{" holyquest --horseback --for-grail"} will cause + C{HolyQuestOptions} to be instantiated and asked to parse + C{['--horseback', '--for-grail']}. Currently, only the first sub-command + is parsed, and all options following it are passed to its parser. If a + subcommand is found, the subCommand attribute is set to its name and the + subOptions attribute is set to the Option instance that parses the + remaining options. If a subcommand is not given to parseOptions, + the subCommand attribute will be None. You can also mark one of + the subCommands to be the default:: + + defaultSubCommand = 'holyquest' + + In this case, the subCommand attribute will never be None, and + the subOptions attribute will always be set. + + If you want to handle your own options, define a method named + C{opt_paramname} that takes C{(self, option)} as arguments. C{option} + will be whatever immediately follows the parameter on the + command line. Options fully supports the mapping interface, so you + can do things like C{'self["option"] = val'} in these methods. + + Shell tab-completion is supported by this class, for zsh only at present. + Zsh ships with a stub file ("completion function") which, for Twisted + commands, performs tab-completion on-the-fly using the support provided + by this class. The stub file lives in our tree at + C{twisted/python/twisted-completion.zsh}, and in the Zsh tree at + C{Completion/Unix/Command/_twisted}. + + Tab-completion is based upon the contents of the optFlags and optParameters + lists. And, optionally, additional metadata may be provided by assigning a + special attribute, C{compData}, which should be an instance of + C{Completions}. See that class for details of what can and should be + included - and see the howto for additional help using these features - + including how third-parties may take advantage of tab-completion for their + own commands. + + Advanced functionality is covered in the howto documentation, + available at + U{http://twistedmatrix.com/projects/core/documentation/howto/options.html}, + or doc/core/howto/options.xhtml in your Twisted directory. + """ + + subCommand = None + defaultSubCommand = None + parent = None + completionData = None + _shellCompFile = sys.stdout # file to use if shell completion is requested + def __init__(self): + super(Options, self).__init__() + + self.opts = self + self.defaults = {} + + # These are strings/lists we will pass to getopt + self.longOpt = [] + self.shortOpt = '' + self.docs = {} + self.synonyms = {} + self._dispatch = {} + + + collectors = [ + self._gather_flags, + self._gather_parameters, + self._gather_handlers, + ] + + for c in collectors: + (longOpt, shortOpt, docs, settings, synonyms, dispatch) = c() + self.longOpt.extend(longOpt) + self.shortOpt = self.shortOpt + shortOpt + self.docs.update(docs) + + self.opts.update(settings) + self.defaults.update(settings) + + self.synonyms.update(synonyms) + self._dispatch.update(dispatch) + + + __hash__ = object.__hash__ + + + def opt_help(self): + """ + Display this help and exit. + """ + print(self.__str__()) + sys.exit(0) + + def opt_version(self): + """ + Display Twisted version and exit. + """ + from twisted import copyright + print("Twisted version:", copyright.version) + sys.exit(0) + + #opt_h = opt_help # this conflicted with existing 'host' options. + + def parseOptions(self, options=None): + """ + The guts of the command-line parser. + """ + + if options is None: + options = sys.argv[1:] + + # we really do need to place the shell completion check here, because + # if we used an opt_shell_completion method then it would be possible + # for other opt_* methods to be run first, and they could possibly + # raise validation errors which would result in error output on the + # terminal of the user performing shell completion. Validation errors + # would occur quite frequently, in fact, because users often initiate + # tab-completion while they are editing an unfinished command-line. + if len(options) > 1 and options[-2] == "--_shell-completion": + from twisted.python import _shellcomp + cmdName = path.basename(sys.argv[0]) + _shellcomp.shellComplete(self, cmdName, options, + self._shellCompFile) + sys.exit(0) + + try: + opts, args = getopt.getopt(options, + self.shortOpt, self.longOpt) + except getopt.error as e: + raise UsageError(str(e)) + + for opt, arg in opts: + if opt[1] == '-': + opt = opt[2:] + else: + opt = opt[1:] + + optMangled = opt + if optMangled not in self.synonyms: + optMangled = opt.replace("-", "_") + if optMangled not in self.synonyms: + raise UsageError("No such option '%s'" % (opt,)) + + optMangled = self.synonyms[optMangled] + if isinstance(self._dispatch[optMangled], CoerceParameter): + self._dispatch[optMangled].dispatch(optMangled, arg) + else: + self._dispatch[optMangled](optMangled, arg) + + if (getattr(self, 'subCommands', None) + and (args or self.defaultSubCommand is not None)): + if not args: + args = [self.defaultSubCommand] + sub, rest = args[0], args[1:] + for (cmd, short, parser, doc) in self.subCommands: + if sub == cmd or sub == short: + self.subCommand = cmd + self.subOptions = parser() + self.subOptions.parent = self + self.subOptions.parseOptions(rest) + break + else: + raise UsageError("Unknown command: %s" % sub) + else: + try: + self.parseArgs(*args) + except TypeError: + raise UsageError("Wrong number of arguments.") + + self.postOptions() + + def postOptions(self): + """ + I am called after the options are parsed. + + Override this method in your subclass to do something after + the options have been parsed and assigned, like validate that + all options are sane. + """ + + def parseArgs(self): + """ + I am called with any leftover arguments which were not options. + + Override me to do something with the remaining arguments on + the command line, those which were not flags or options. e.g. + interpret them as a list of files to operate on. + + Note that if there more arguments on the command line + than this method accepts, parseArgs will blow up with + a getopt.error. This means if you don't override me, + parseArgs will blow up if I am passed any arguments at + all! + """ + + def _generic_flag(self, flagName, value=None): + if value not in ('', None): + raise UsageError("Flag '%s' takes no argument." + " Not even \"%s\"." % (flagName, value)) + + self.opts[flagName] = 1 + + def _gather_flags(self): + """ + Gather up boolean (flag) options. + """ + + longOpt, shortOpt = [], '' + docs, settings, synonyms, dispatch = {}, {}, {}, {} + + flags = [] + reflect.accumulateClassList(self.__class__, 'optFlags', flags) + + for flag in flags: + long, short, doc = util.padTo(3, flag) + if not long: + raise ValueError("A flag cannot be without a name.") + + docs[long] = doc + settings[long] = 0 + if short: + shortOpt = shortOpt + short + synonyms[short] = long + longOpt.append(long) + synonyms[long] = long + dispatch[long] = self._generic_flag + + return longOpt, shortOpt, docs, settings, synonyms, dispatch + + def _gather_parameters(self): + """ + Gather options which take a value. + """ + longOpt, shortOpt = [], '' + docs, settings, synonyms, dispatch = {}, {}, {}, {} + + parameters = [] + + reflect.accumulateClassList(self.__class__, 'optParameters', + parameters) + + synonyms = {} + + for parameter in parameters: + long, short, default, doc, paramType = util.padTo(5, parameter) + if not long: + raise ValueError("A parameter cannot be without a name.") + + docs[long] = doc + settings[long] = default + if short: + shortOpt = shortOpt + short + ':' + synonyms[short] = long + longOpt.append(long + '=') + synonyms[long] = long + if paramType is not None: + dispatch[long] = CoerceParameter(self, paramType) + else: + dispatch[long] = CoerceParameter(self, str) + + return longOpt, shortOpt, docs, settings, synonyms, dispatch + + + def _gather_handlers(self): + """ + Gather up options with their own handler methods. + + This returns a tuple of many values. Amongst those values is a + synonyms dictionary, mapping all of the possible aliases (C{str}) + for an option to the longest spelling of that option's name + C({str}). + + Another element is a dispatch dictionary, mapping each user-facing + option name (with - substituted for _) to a callable to handle that + option. + """ + + longOpt, shortOpt = [], '' + docs, settings, synonyms, dispatch = {}, {}, {}, {} + + dct = {} + reflect.addMethodNamesToDict(self.__class__, dct, "opt_") + + for name in dct.keys(): + method = getattr(self, 'opt_'+name) + + takesArg = not flagFunction(method, name) + + prettyName = name.replace('_', '-') + doc = getattr(method, '__doc__', None) + if doc: + ## Only use the first line. + #docs[name] = doc.split('\n')[0] + docs[prettyName] = doc + else: + docs[prettyName] = self.docs.get(prettyName) + + synonyms[prettyName] = prettyName + + # A little slight-of-hand here makes dispatching much easier + # in parseOptions, as it makes all option-methods have the + # same signature. + if takesArg: + fn = lambda name, value, m=method: m(value) + else: + # XXX: This won't raise a TypeError if it's called + # with a value when it shouldn't be. + fn = lambda name, value=None, m=method: m() + + dispatch[prettyName] = fn + + if len(name) == 1: + shortOpt = shortOpt + name + if takesArg: + shortOpt = shortOpt + ':' + else: + if takesArg: + prettyName = prettyName + '=' + longOpt.append(prettyName) + + reverse_dct = {} + # Map synonyms + for name in dct.keys(): + method = getattr(self, 'opt_' + name) + if method not in reverse_dct: + reverse_dct[method] = [] + reverse_dct[method].append(name.replace('_', '-')) + + for method, names in reverse_dct.items(): + if len(names) < 2: + continue + longest = max(names, key=len) + for name in names: + synonyms[name] = longest + + return longOpt, shortOpt, docs, settings, synonyms, dispatch + + + def __str__(self): + return self.getSynopsis() + '\n' + self.getUsage(width=None) + + def getSynopsis(self): + """ + Returns a string containing a description of these options and how to + pass them to the executed file. + """ + executableName = reflect.filenameToModuleName(sys.argv[0]) + + if executableName.endswith('.__main__'): + executableName = '{} -m {}'.format(os.path.basename(sys.executable), executableName.replace('.__main__', '')) + + if self.parent is None: + default = "Usage: %s%s" % (executableName, + (self.longOpt and " [options]") or '') + else: + default = '%s' % ((self.longOpt and "[options]") or '') + synopsis = getattr(self, "synopsis", default) + + synopsis = synopsis.rstrip() + + if self.parent is not None: + synopsis = ' '.join((self.parent.getSynopsis(), + self.parent.subCommand, synopsis)) + return synopsis + + def getUsage(self, width=None): + # If subOptions exists by now, then there was probably an error while + # parsing its options. + if hasattr(self, 'subOptions'): + return self.subOptions.getUsage(width=width) + + if not width: + width = int(os.environ.get('COLUMNS', '80')) + + if hasattr(self, 'subCommands'): + cmdDicts = [] + for (cmd, short, parser, desc) in self.subCommands: + cmdDicts.append( + {'long': cmd, + 'short': short, + 'doc': desc, + 'optType': 'command', + 'default': None + }) + chunks = docMakeChunks(cmdDicts, width) + commands = 'Commands:\n' + ''.join(chunks) + else: + commands = '' + + longToShort = {} + for key, value in self.synonyms.items(): + longname = value + if (key != longname) and (len(key) == 1): + longToShort[longname] = key + else: + if longname not in longToShort: + longToShort[longname] = None + else: + pass + + optDicts = [] + for opt in self.longOpt: + if opt[-1] == '=': + optType = 'parameter' + opt = opt[:-1] + else: + optType = 'flag' + + optDicts.append( + {'long': opt, + 'short': longToShort[opt], + 'doc': self.docs[opt], + 'optType': optType, + 'default': self.defaults.get(opt, None), + 'dispatch': self._dispatch.get(opt, None) + }) + + if not (getattr(self, "longdesc", None) is None): + longdesc = self.longdesc + else: + import __main__ + if getattr(__main__, '__doc__', None): + longdesc = __main__.__doc__ + else: + longdesc = '' + + if longdesc: + longdesc = ('\n' + + '\n'.join(textwrap.wrap(longdesc, width)).strip() + + '\n') + + if optDicts: + chunks = docMakeChunks(optDicts, width) + s = "Options:\n%s" % (''.join(chunks)) + else: + s = "Options: None\n" + + return s + longdesc + commands + + #def __repr__(self): + # XXX: It'd be cool if we could return a succinct representation + # of which flags and options are set here. + + +_ZSH = 'zsh' +_BASH = 'bash' + +class Completer(object): + """ + A completion "action" - provides completion possibilities for a particular + command-line option. For example we might provide the user a fixed list of + choices, or files/dirs according to a glob. + + This class produces no completion matches itself - see the various + subclasses for specific completion functionality. + """ + _descr = None + def __init__(self, descr=None, repeat=False): + """ + @type descr: C{str} + @param descr: An optional descriptive string displayed above matches. + + @type repeat: C{bool} + @param repeat: A flag, defaulting to False, indicating whether this + C{Completer} should repeat - that is, be used to complete more + than one command-line word. This may ONLY be set to True for + actions in the C{extraActions} keyword argument to C{Completions}. + And ONLY if it is the LAST (or only) action in the C{extraActions} + list. + """ + if descr is not None: + self._descr = descr + self._repeat = repeat + + + def _getRepeatFlag(self): + if self._repeat: + return "*" + else: + return "" + _repeatFlag = property(_getRepeatFlag) + + + def _description(self, optName): + if self._descr is not None: + return self._descr + else: + return optName + + + def _shellCode(self, optName, shellType): + """ + Fetch a fragment of shell code representing this action which is + suitable for use by the completion system in _shellcomp.py + + @type optName: C{str} + @param optName: The long name of the option this action is being + used for. + + @type shellType: C{str} + @param shellType: One of the supported shell constants e.g. + C{twisted.python.usage._ZSH} + """ + if shellType == _ZSH: + return "%s:%s:" % (self._repeatFlag, + self._description(optName)) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteFiles(Completer): + """ + Completes file names based on a glob pattern + """ + def __init__(self, globPattern='*', **kw): + Completer.__init__(self, **kw) + self._globPattern = globPattern + + + def _description(self, optName): + if self._descr is not None: + return "%s (%s)" % (self._descr, self._globPattern) + else: + return "%s (%s)" % (optName, self._globPattern) + + + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + return "%s:%s:_files -g \"%s\"" % (self._repeatFlag, + self._description(optName), + self._globPattern,) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteDirs(Completer): + """ + Completes directory names + """ + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + return "%s:%s:_directories" % (self._repeatFlag, + self._description(optName)) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteList(Completer): + """ + Completes based on a fixed list of words + """ + def __init__(self, items, **kw): + Completer.__init__(self, **kw) + self._items = items + + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + return "%s:%s:(%s)" % (self._repeatFlag, + self._description(optName), + " ".join(self._items,)) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteMultiList(Completer): + """ + Completes multiple comma-separated items based on a fixed list of words + """ + def __init__(self, items, **kw): + Completer.__init__(self, **kw) + self._items = items + + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + return "%s:%s:_values -s , '%s' %s" % (self._repeatFlag, + self._description(optName), + self._description(optName), + " ".join(self._items)) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteUsernames(Completer): + """ + Complete usernames + """ + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + return "%s:%s:_users" % (self._repeatFlag, + self._description(optName)) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteGroups(Completer): + """ + Complete system group names + """ + _descr = 'group' + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + return "%s:%s:_groups" % (self._repeatFlag, + self._description(optName)) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteHostnames(Completer): + """ + Complete hostnames + """ + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + return "%s:%s:_hosts" % (self._repeatFlag, + self._description(optName)) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteUserAtHost(Completer): + """ + A completion action which produces matches in any of these forms:: + + + @ + """ + _descr = 'host | user@host' + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + # Yes this looks insane but it does work. For bonus points + # add code to grep 'Hostname' lines from ~/.ssh/config + return ('%s:%s:{_ssh;if compset -P "*@"; ' + 'then _wanted hosts expl "remote host name" _ssh_hosts ' + '&& ret=0 elif compset -S "@*"; then _wanted users ' + 'expl "login name" _ssh_users -S "" && ret=0 ' + 'else if (( $+opt_args[-l] )); then tmp=() ' + 'else tmp=( "users:login name:_ssh_users -qS@" ) fi; ' + '_alternative "hosts:remote host name:_ssh_hosts" "$tmp[@]"' + ' && ret=0 fi}' % (self._repeatFlag, + self._description(optName))) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class CompleteNetInterfaces(Completer): + """ + Complete network interface names + """ + def _shellCode(self, optName, shellType): + if shellType == _ZSH: + return "%s:%s:_net_interfaces" % (self._repeatFlag, + self._description(optName)) + raise NotImplementedError("Unknown shellType %r" % (shellType,)) + + + +class Completions(object): + """ + Extra metadata for the shell tab-completion system. + + @type descriptions: C{dict} + @ivar descriptions: ex. C{{"foo" : "use this description for foo instead"}} + A dict mapping long option names to alternate descriptions. When this + variable is defined, the descriptions contained here will override + those descriptions provided in the optFlags and optParameters + variables. + + @type multiUse: C{list} + @ivar multiUse: ex. C{ ["foo", "bar"] } + An iterable containing those long option names which may appear on the + command line more than once. By default, options will only be completed + one time. + + @type mutuallyExclusive: C{list} of C{tuple} + @ivar mutuallyExclusive: ex. C{ [("foo", "bar"), ("bar", "baz")] } + A sequence of sequences, with each sub-sequence containing those long + option names that are mutually exclusive. That is, those options that + cannot appear on the command line together. + + @type optActions: C{dict} + @ivar optActions: A dict mapping long option names to shell "actions". + These actions define what may be completed as the argument to the + given option. By default, all files/dirs will be completed if no + action is given. For example:: + + {"foo" : CompleteFiles("*.py", descr="python files"), + "bar" : CompleteList(["one", "two", "three"]), + "colors" : CompleteMultiList(["red", "green", "blue"])} + + Callables may instead be given for the values in this dict. The + callable should accept no arguments, and return a C{Completer} + instance used as the action in the same way as the literal actions in + the example above. + + As you can see in the example above. The "foo" option will have files + that end in .py completed when the user presses Tab. The "bar" + option will have either of the strings "one", "two", or "three" + completed when the user presses Tab. + + "colors" will allow multiple arguments to be completed, separated by + commas. The possible arguments are red, green, and blue. Examples:: + + my_command --foo some-file.foo --colors=red,green + my_command --colors=green + my_command --colors=green,blue + + Descriptions for the actions may be given with the optional C{descr} + keyword argument. This is separate from the description of the option + itself. + + Normally Zsh does not show these descriptions unless you have + "verbose" completion turned on. Turn on verbosity with this in your + ~/.zshrc:: + + zstyle ':completion:*' verbose yes + zstyle ':completion:*:descriptions' format '%B%d%b' + + @type extraActions: C{list} + @ivar extraActions: Extra arguments are those arguments typically + appearing at the end of the command-line, which are not associated + with any particular named option. That is, the arguments that are + given to the parseArgs() method of your usage.Options subclass. For + example:: + [CompleteFiles(descr="file to read from"), + Completer(descr="book title")] + + In the example above, the 1st non-option argument will be described as + "file to read from" and all file/dir names will be completed (*). The + 2nd non-option argument will be described as "book title", but no + actual completion matches will be produced. + + See the various C{Completer} subclasses for other types of things which + may be tab-completed (users, groups, network interfaces, etc). + + Also note the C{repeat=True} flag which may be passed to any of the + C{Completer} classes. This is set to allow the C{Completer} instance + to be re-used for subsequent command-line words. See the C{Completer} + docstring for details. + """ + def __init__(self, descriptions={}, multiUse=[], + mutuallyExclusive=[], optActions={}, extraActions=[]): + self.descriptions = descriptions + self.multiUse = multiUse + self.mutuallyExclusive = mutuallyExclusive + self.optActions = optActions + self.extraActions = extraActions + + + +def docMakeChunks(optList, width=80): + """ + Makes doc chunks for option declarations. + + Takes a list of dictionaries, each of which may have one or more + of the keys 'long', 'short', 'doc', 'default', 'optType'. + + Returns a list of strings. + The strings may be multiple lines, + all of them end with a newline. + """ + + # XXX: sanity check to make sure we have a sane combination of keys. + + # Sort the options so they always appear in the same order + optList.sort(key=lambda o: o.get('short', None) or o.get('long', None)) + + maxOptLen = 0 + for opt in optList: + optLen = len(opt.get('long', '')) + if optLen: + if opt.get('optType', None) == "parameter": + # these take up an extra character + optLen = optLen + 1 + maxOptLen = max(optLen, maxOptLen) + + colWidth1 = maxOptLen + len(" -s, -- ") + colWidth2 = width - colWidth1 + # XXX - impose some sane minimum limit. + # Then if we don't have enough room for the option and the doc + # to share one line, they can take turns on alternating lines. + + colFiller1 = " " * colWidth1 + + optChunks = [] + seen = {} + for opt in optList: + if opt.get('short', None) in seen or opt.get('long', None) in seen: + continue + for x in opt.get('short', None), opt.get('long', None): + if x is not None: + seen[x] = 1 + + optLines = [] + comma = " " + if opt.get('short', None): + short = "-%c" % (opt['short'],) + else: + short = '' + + if opt.get('long', None): + long = opt['long'] + if opt.get("optType", None) == "parameter": + long = long + '=' + + long = "%-*s" % (maxOptLen, long) + if short: + comma = "," + else: + long = " " * (maxOptLen + len('--')) + + if opt.get('optType', None) == 'command': + column1 = ' %s ' % long + else: + column1 = " %2s%c --%s " % (short, comma, long) + + if opt.get('doc', ''): + doc = opt['doc'].strip() + else: + doc = '' + + if (opt.get("optType", None) == "parameter") \ + and not (opt.get('default', None) is None): + doc = "%s [default: %s]" % (doc, opt['default']) + + if (opt.get("optType", None) == "parameter") \ + and opt.get('dispatch', None) is not None: + d = opt['dispatch'] + if isinstance(d, CoerceParameter) and d.doc: + doc = "%s. %s" % (doc, d.doc) + + if doc: + column2_l = textwrap.wrap(doc, colWidth2) + else: + column2_l = [''] + + optLines.append("%s%s\n" % (column1, column2_l.pop(0))) + + for line in column2_l: + optLines.append("%s%s\n" % (colFiller1, line)) + + optChunks.append(''.join(optLines)) + + return optChunks + + + +def flagFunction(method, name=None): + """ + Determine whether a function is an optional handler for a I{flag} or an + I{option}. + + A I{flag} handler takes no additional arguments. It is used to handle + command-line arguments like I{--nodaemon}. + + An I{option} handler takes one argument. It is used to handle command-line + arguments like I{--path=/foo/bar}. + + @param method: The bound method object to inspect. + + @param name: The name of the option for which the function is a handle. + @type name: L{str} + + @raise UsageError: If the method takes more than one argument. + + @return: If the method is a flag handler, return C{True}. Otherwise return + C{False}. + """ + if _PY3: + reqArgs = len(inspect.signature(method).parameters) + if reqArgs > 1: + raise UsageError('Invalid Option function for %s' % + (name or method.__name__)) + if reqArgs == 1: + return False + else: + reqArgs = len(inspect.getargspec(method).args) + if reqArgs > 2: + raise UsageError('Invalid Option function for %s' % + (name or method.__name__)) + if reqArgs == 2: + return False + return True + + + +def portCoerce(value): + """ + Coerce a string value to an int port number, and checks the validity. + """ + value = int(value) + if value < 0 or value > 65535: + raise ValueError("Port number not in range: %s" % (value,)) + return value +portCoerce.coerceDoc = "Must be an int between 0 and 65535." diff --git a/contrib/python/Twisted/py2/twisted/python/util.py b/contrib/python/Twisted/py2/twisted/python/util.py new file mode 100644 index 00000000000..8bcd3cbae16 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/util.py @@ -0,0 +1,1027 @@ +# -*- test-case-name: twisted.python.test.test_util -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import division, absolute_import, print_function + +import os, sys, errno, warnings +try: + import pwd, grp +except ImportError: + pwd = grp = None +try: + from os import setgroups, getgroups +except ImportError: + setgroups = getgroups = None + +from twisted.python.compat import _PY3, unicode +from incremental import Version +from twisted.python.deprecate import deprecatedModuleAttribute +from twisted.python._oldstyle import _oldStyle, _replaceIf + +# For backwards compatibility, some things import this, so just link it +from collections import OrderedDict + +deprecatedModuleAttribute( + Version("Twisted", 15, 5, 0), + "Use collections.OrderedDict instead.", + "twisted.python.util", + "OrderedDict") + + + +@_oldStyle +class InsensitiveDict: + """ + Dictionary, that has case-insensitive keys. + + Normally keys are retained in their original form when queried with + .keys() or .items(). If initialized with preserveCase=0, keys are both + looked up in lowercase and returned in lowercase by .keys() and .items(). + """ + """ + Modified recipe at + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66315 originally + contributed by Sami Hangaslammi. + """ + + def __init__(self, dict=None, preserve=1): + """ + Create an empty dictionary, or update from 'dict'. + """ + self.data = {} + self.preserve=preserve + if dict: + self.update(dict) + + + def __delitem__(self, key): + k=self._lowerOrReturn(key) + del self.data[k] + + + def _lowerOrReturn(self, key): + if isinstance(key, bytes) or isinstance(key, unicode): + return key.lower() + else: + return key + + + def __getitem__(self, key): + """ + Retrieve the value associated with 'key' (in any case). + """ + k = self._lowerOrReturn(key) + return self.data[k][1] + + + def __setitem__(self, key, value): + """ + Associate 'value' with 'key'. If 'key' already exists, but + in different case, it will be replaced. + """ + k = self._lowerOrReturn(key) + self.data[k] = (key, value) + + + def has_key(self, key): + """ + Case insensitive test whether 'key' exists. + """ + k = self._lowerOrReturn(key) + return k in self.data + + __contains__ = has_key + + + def _doPreserve(self, key): + if not self.preserve and (isinstance(key, bytes) + or isinstance(key, unicode)): + return key.lower() + else: + return key + + + def keys(self): + """ + List of keys in their original case. + """ + return list(self.iterkeys()) + + + def values(self): + """ + List of values. + """ + return list(self.itervalues()) + + + def items(self): + """ + List of (key,value) pairs. + """ + return list(self.iteritems()) + + + def get(self, key, default=None): + """ + Retrieve value associated with 'key' or return default value + if 'key' doesn't exist. + """ + try: + return self[key] + except KeyError: + return default + + + def setdefault(self, key, default): + """ + If 'key' doesn't exist, associate it with the 'default' value. + Return value associated with 'key'. + """ + if not self.has_key(key): + self[key] = default + return self[key] + + + def update(self, dict): + """ + Copy (key,value) pairs from 'dict'. + """ + for k,v in dict.items(): + self[k] = v + + + def __repr__(self): + """ + String representation of the dictionary. + """ + items = ", ".join([("%r: %r" % (k,v)) for k,v in self.items()]) + return "InsensitiveDict({%s})" % items + + + def iterkeys(self): + for v in self.data.values(): + yield self._doPreserve(v[0]) + + + def itervalues(self): + for v in self.data.values(): + yield v[1] + + + def iteritems(self): + for (k, v) in self.data.values(): + yield self._doPreserve(k), v + + + def popitem(self): + i=self.items()[0] + del self[i[0]] + return i + + + def clear(self): + for k in self.keys(): + del self[k] + + + def copy(self): + return InsensitiveDict(self, self.preserve) + + + def __len__(self): + return len(self.data) + + + def __eq__(self, other): + for k,v in self.items(): + if not (k in other) or not (other[k]==v): + return 0 + return len(self)==len(other) + + + +def uniquify(lst): + """ + Make the elements of a list unique by inserting them into a dictionary. + This must not change the order of the input lst. + """ + dct = {} + result = [] + for k in lst: + if k not in dct: + result.append(k) + dct[k] = 1 + return result + + + +def padTo(n, seq, default=None): + """ + Pads a sequence out to n elements, + + filling in with a default value if it is not long enough. + + If the input sequence is longer than n, raises ValueError. + + Details, details: + This returns a new list; it does not extend the original sequence. + The new list contains the values of the original sequence, not copies. + """ + + if len(seq) > n: + raise ValueError("%d elements is more than %d." % (len(seq), n)) + + blank = [default] * n + + blank[:len(seq)] = list(seq) + + return blank + + + +def getPluginDirs(): + warnings.warn( + "twisted.python.util.getPluginDirs is deprecated since Twisted 12.2.", + DeprecationWarning, stacklevel=2) + import twisted + systemPlugins = os.path.join(os.path.dirname(os.path.dirname( + os.path.abspath(twisted.__file__))), 'plugins') + userPlugins = os.path.expanduser("~/TwistedPlugins") + confPlugins = os.path.expanduser("~/.twisted") + allPlugins = filter(os.path.isdir, [systemPlugins, userPlugins, confPlugins]) + return allPlugins + + + +def addPluginDir(): + warnings.warn( + "twisted.python.util.addPluginDir is deprecated since Twisted 12.2.", + DeprecationWarning, stacklevel=2) + sys.path.extend(getPluginDirs()) + + + +def sibpath(path, sibling): + """ + Return the path to a sibling of a file in the filesystem. + + This is useful in conjunction with the special C{__file__} attribute + that Python provides for modules, so modules can load associated + resource files. + """ + return os.path.join(os.path.dirname(os.path.abspath(path)), sibling) + + + +def _getpass(prompt): + """ + Helper to turn IOErrors into KeyboardInterrupts. + """ + import getpass + try: + return getpass.getpass(prompt) + except IOError as e: + if e.errno == errno.EINTR: + raise KeyboardInterrupt + raise + except EOFError: + raise KeyboardInterrupt + + + +def getPassword(prompt = 'Password: ', confirm = 0, forceTTY = 0, + confirmPrompt = 'Confirm password: ', + mismatchMessage = "Passwords don't match."): + """ + Obtain a password by prompting or from stdin. + + If stdin is a terminal, prompt for a new password, and confirm (if + C{confirm} is true) by asking again to make sure the user typed the same + thing, as keystrokes will not be echoed. + + If stdin is not a terminal, and C{forceTTY} is not true, read in a line + and use it as the password, less the trailing newline, if any. If + C{forceTTY} is true, attempt to open a tty and prompt for the password + using it. Raise a RuntimeError if this is not possible. + + @returns: C{str} + """ + isaTTY = hasattr(sys.stdin, 'isatty') and sys.stdin.isatty() + + old = None + try: + if not isaTTY: + if forceTTY: + try: + old = sys.stdin, sys.stdout + sys.stdin = sys.stdout = open('/dev/tty', 'r+') + except: + raise RuntimeError("Cannot obtain a TTY") + else: + password = sys.stdin.readline() + if password[-1] == '\n': + password = password[:-1] + return password + + while 1: + try1 = _getpass(prompt) + if not confirm: + return try1 + try2 = _getpass(confirmPrompt) + if try1 == try2: + return try1 + else: + sys.stderr.write(mismatchMessage + "\n") + finally: + if old: + sys.stdin.close() + sys.stdin, sys.stdout = old + + + +def println(*a): + sys.stdout.write(' '.join(map(str, a))+'\n') + +# XXX +# This does not belong here +# But where does it belong? + + + +def str_xor(s, b): + return ''.join([chr(ord(c) ^ b) for c in s]) + + + +def makeStatBar(width, maxPosition, doneChar = '=', undoneChar = '-', currentChar = '>'): + """ + Creates a function that will return a string representing a progress bar. + """ + aValue = width / float(maxPosition) + def statBar(position, force = 0, last = ['']): + assert len(last) == 1, "Don't mess with the last parameter." + done = int(aValue * position) + toDo = width - done - 2 + result = "[%s%s%s]" % (doneChar * done, currentChar, undoneChar * toDo) + if force: + last[0] = result + return result + if result == last[0]: + return '' + last[0] = result + return result + + statBar.__doc__ = """statBar(position, force = 0) -> '[%s%s%s]'-style progress bar + + returned string is %d characters long, and the range goes from 0..%d. + The 'position' argument is where the '%s' will be drawn. If force is false, + '' will be returned instead if the resulting progress bar is identical to the + previously returned progress bar. +""" % (doneChar * 3, currentChar, undoneChar * 3, width, maxPosition, currentChar) + return statBar + + + +def spewer(frame, s, ignored): + """ + A trace function for sys.settrace that prints every function or method call. + """ + from twisted.python import reflect + if 'self' in frame.f_locals: + se = frame.f_locals['self'] + if hasattr(se, '__class__'): + k = reflect.qual(se.__class__) + else: + k = reflect.qual(type(se)) + print('method %s of %s at %s' % ( + frame.f_code.co_name, k, id(se))) + else: + print('function %s in %s, line %s' % ( + frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno)) + + + +def searchupwards(start, files=[], dirs=[]): + """ + Walk upwards from start, looking for a directory containing + all files and directories given as arguments:: + >>> searchupwards('.', ['foo.txt'], ['bar', 'bam']) + + If not found, return None + """ + start=os.path.abspath(start) + parents=start.split(os.sep) + exists=os.path.exists; join=os.sep.join; isdir=os.path.isdir + while len(parents): + candidate=join(parents)+os.sep + allpresent=1 + for f in files: + if not exists("%s%s" % (candidate, f)): + allpresent=0 + break + if allpresent: + for d in dirs: + if not isdir("%s%s" % (candidate, d)): + allpresent=0 + break + if allpresent: return candidate + parents.pop(-1) + return None + + + +@_oldStyle +class LineLog: + """ + A limited-size line-based log, useful for logging line-based + protocols such as SMTP. + + When the log fills up, old entries drop off the end. + """ + def __init__(self, size=10): + """ + Create a new log, with size lines of storage (default 10). + A log size of 0 (or less) means an infinite log. + """ + if size < 0: + size = 0 + self.log = [None] * size + self.size = size + + + def append(self,line): + if self.size: + self.log[:-1] = self.log[1:] + self.log[-1] = line + else: + self.log.append(line) + + + def str(self): + return bytes(self) + + if not _PY3: + def __str__(self): + return self.__bytes__() + + + def __bytes__(self): + return b'\n'.join(filter(None, self.log)) + + + def __getitem__(self, item): + return filter(None, self.log)[item] + + + def clear(self): + """ + Empty the log. + """ + self.log = [None] * self.size + + + +def raises(exception, f, *args, **kwargs): + """ + Determine whether the given call raises the given exception. + """ + try: + f(*args, **kwargs) + except exception: + return 1 + return 0 + + + +class IntervalDifferential(object): + """ + Given a list of intervals, generate the amount of time to sleep between + "instants". + + For example, given 7, 11 and 13, the three (infinite) sequences:: + + 7 14 21 28 35 ... + 11 22 33 44 ... + 13 26 39 52 ... + + will be generated, merged, and used to produce:: + + (7, 0) (4, 1) (2, 2) (1, 0) (7, 0) (1, 1) (4, 2) (2, 0) (5, 1) (2, 0) + + New intervals may be added or removed as iteration proceeds using the + proper methods. + """ + + def __init__(self, intervals, default=60): + """ + @type intervals: C{list} of C{int}, C{long}, or C{float} param + @param intervals: The intervals between instants. + + @type default: C{int}, C{long}, or C{float} + @param default: The duration to generate if the intervals list + becomes empty. + """ + self.intervals = intervals[:] + self.default = default + + + def __iter__(self): + return _IntervalDifferentialIterator(self.intervals, self.default) + + + +class _IntervalDifferentialIterator(object): + def __init__(self, i, d): + + self.intervals = [[e, e, n] for (e, n) in zip(i, range(len(i)))] + self.default = d + self.last = 0 + + + def __next__(self): + if not self.intervals: + return (self.default, None) + last, index = self.intervals[0][0], self.intervals[0][2] + self.intervals[0][0] += self.intervals[0][1] + self.intervals.sort() + result = last - self.last + self.last = last + return result, index + + # Iterators on Python 2 use next(), not __next__() + next = __next__ + + + def addInterval(self, i): + if self.intervals: + delay = self.intervals[0][0] - self.intervals[0][1] + self.intervals.append([delay + i, i, len(self.intervals)]) + self.intervals.sort() + else: + self.intervals.append([i, i, 0]) + + + def removeInterval(self, interval): + for i in range(len(self.intervals)): + if self.intervals[i][1] == interval: + index = self.intervals[i][2] + del self.intervals[i] + for i in self.intervals: + if i[2] > index: + i[2] -= 1 + return + raise ValueError("Specified interval not in IntervalDifferential") + + + +@_oldStyle +class FancyStrMixin: + """ + Mixin providing a flexible implementation of C{__str__}. + + C{__str__} output will begin with the name of the class, or the contents + of the attribute C{fancybasename} if it is set. + + The body of C{__str__} can be controlled by overriding C{showAttributes} in + a subclass. Set C{showAttributes} to a sequence of strings naming + attributes, or sequences of C{(attributeName, callable)}, or sequences of + C{(attributeName, displayName, formatCharacter)}. In the second case, the + callable is passed the value of the attribute and its return value used in + the output of C{__str__}. In the final case, the attribute is looked up + using C{attributeName}, but the output uses C{displayName} instead, and + renders the value of the attribute using C{formatCharacter}, e.g. C{"%.3f"} + might be used for a float. + """ + # Override in subclasses: + showAttributes = () + + + def __str__(self): + r = ['<', (hasattr(self, 'fancybasename') and self.fancybasename) + or self.__class__.__name__] + for attr in self.showAttributes: + if isinstance(attr, str): + r.append(' %s=%r' % (attr, getattr(self, attr))) + elif len(attr) == 2: + r.append((' %s=' % (attr[0],)) + attr[1](getattr(self, attr[0]))) + else: + r.append((' %s=' + attr[2]) % (attr[1], getattr(self, attr[0]))) + r.append('>') + return ''.join(r) + + __repr__ = __str__ + + + +@_oldStyle +class FancyEqMixin: + """ + Mixin that implements C{__eq__} and C{__ne__}. + + Comparison is done using the list of attributes defined in + C{compareAttributes}. + """ + compareAttributes = () + + def __eq__(self, other): + if not self.compareAttributes: + return self is other + if isinstance(self, other.__class__): + return ( + [getattr(self, name) for name in self.compareAttributes] == + [getattr(other, name) for name in self.compareAttributes]) + return NotImplemented + + + def __ne__(self, other): + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + + + +try: + # initgroups is available in Python 2.7+ on UNIX-likes + from os import initgroups as _initgroups +except ImportError: + _initgroups = None + + + +if _initgroups is None: + def initgroups(uid, primaryGid): + """ + Do nothing. + + Underlying platform support require to manipulate groups is missing. + """ +else: + def initgroups(uid, primaryGid): + """ + Initializes the group access list. + + This uses the stdlib support which calls initgroups(3) under the hood. + + If the given user is a member of more than C{NGROUPS}, arbitrary + groups will be silently discarded to bring the number below that + limit. + + @type uid: C{int} + @param uid: The UID for which to look up group information. + + @type primaryGid: C{int} + @param primaryGid: The GID to include when setting the groups. + """ + return _initgroups(pwd.getpwuid(uid).pw_name, primaryGid) + + + +def switchUID(uid, gid, euid=False): + """ + Attempts to switch the uid/euid and gid/egid for the current process. + + If C{uid} is the same value as L{os.getuid} (or L{os.geteuid}), + this function will issue a L{UserWarning} and not raise an exception. + + @type uid: C{int} or L{None} + @param uid: the UID (or EUID) to switch the current process to. This + parameter will be ignored if the value is L{None}. + + @type gid: C{int} or L{None} + @param gid: the GID (or EGID) to switch the current process to. This + parameter will be ignored if the value is L{None}. + + @type euid: C{bool} + @param euid: if True, set only effective user-id rather than real user-id. + (This option has no effect unless the process is running + as root, in which case it means not to shed all + privileges, retaining the option to regain privileges + in cases such as spawning processes. Use with caution.) + """ + if euid: + setuid = os.seteuid + setgid = os.setegid + getuid = os.geteuid + else: + setuid = os.setuid + setgid = os.setgid + getuid = os.getuid + if gid is not None: + setgid(gid) + if uid is not None: + if uid == getuid(): + uidText = (euid and "euid" or "uid") + actionText = "tried to drop privileges and set{} {}".format( + uidText, uid) + problemText = "{} is already {}".format(uidText, getuid()) + warnings.warn("{} but {}; should we be root? Continuing.".format( + actionText, problemText)) + else: + initgroups(uid, gid) + setuid(uid) + + + +class SubclassableCStringIO(object): + """ + A wrapper around cStringIO to allow for subclassing. + """ + __csio = None + + def __init__(self, *a, **kw): + from cStringIO import StringIO + self.__csio = StringIO(*a, **kw) + + + def __iter__(self): + return self.__csio.__iter__() + + + def next(self): + return self.__csio.next() + + + def close(self): + return self.__csio.close() + + + def isatty(self): + return self.__csio.isatty() + + + def seek(self, pos, mode=0): + return self.__csio.seek(pos, mode) + + + def tell(self): + return self.__csio.tell() + + + def read(self, n=-1): + return self.__csio.read(n) + + + def readline(self, length=None): + return self.__csio.readline(length) + + + def readlines(self, sizehint=0): + return self.__csio.readlines(sizehint) + + + def truncate(self, size=None): + return self.__csio.truncate(size) + + + def write(self, s): + return self.__csio.write(s) + + + def writelines(self, list): + return self.__csio.writelines(list) + + + def flush(self): + return self.__csio.flush() + + + def getvalue(self): + return self.__csio.getvalue() + + + +def untilConcludes(f, *a, **kw): + """ + Call C{f} with the given arguments, handling C{EINTR} by retrying. + + @param f: A function to call. + + @param *a: Positional arguments to pass to C{f}. + + @param **kw: Keyword arguments to pass to C{f}. + + @return: Whatever C{f} returns. + + @raise: Whatever C{f} raises, except for C{IOError} or C{OSError} with + C{errno} set to C{EINTR}. + """ + while True: + try: + return f(*a, **kw) + except (IOError, OSError) as e: + if e.args[0] == errno.EINTR: + continue + raise + + + +def mergeFunctionMetadata(f, g): + """ + Overwrite C{g}'s name and docstring with values from C{f}. Update + C{g}'s instance dictionary with C{f}'s. + + @return: A function that has C{g}'s behavior and metadata merged from + C{f}. + """ + try: + g.__name__ = f.__name__ + except TypeError: + pass + try: + g.__doc__ = f.__doc__ + except (TypeError, AttributeError): + pass + try: + g.__dict__.update(f.__dict__) + except (TypeError, AttributeError): + pass + try: + g.__module__ = f.__module__ + except TypeError: + pass + return g + + + +def nameToLabel(mname): + """ + Convert a string like a variable name into a slightly more human-friendly + string with spaces and capitalized letters. + + @type mname: C{str} + @param mname: The name to convert to a label. This must be a string + which could be used as a Python identifier. Strings which do not take + this form will result in unpredictable behavior. + + @rtype: C{str} + """ + labelList = [] + word = '' + lastWasUpper = False + for letter in mname: + if letter.isupper() == lastWasUpper: + # Continuing a word. + word += letter + else: + # breaking a word OR beginning a word + if lastWasUpper: + # could be either + if len(word) == 1: + # keep going + word += letter + else: + # acronym + # we're processing the lowercase letter after the acronym-then-capital + lastWord = word[:-1] + firstLetter = word[-1] + labelList.append(lastWord) + word = firstLetter + letter + else: + # definitely breaking: lower to upper + labelList.append(word) + word = letter + lastWasUpper = letter.isupper() + if labelList: + labelList[0] = labelList[0].capitalize() + else: + return mname.capitalize() + labelList.append(word) + return ' '.join(labelList) + + + +def uidFromString(uidString): + """ + Convert a user identifier, as a string, into an integer UID. + + @type uid: C{str} + @param uid: A string giving the base-ten representation of a UID or the + name of a user which can be converted to a UID via L{pwd.getpwnam}. + + @rtype: C{int} + @return: The integer UID corresponding to the given string. + + @raise ValueError: If the user name is supplied and L{pwd} is not + available. + """ + try: + return int(uidString) + except ValueError: + if pwd is None: + raise + return pwd.getpwnam(uidString)[2] + + + +def gidFromString(gidString): + """ + Convert a group identifier, as a string, into an integer GID. + + @type uid: C{str} + @param uid: A string giving the base-ten representation of a GID or the + name of a group which can be converted to a GID via L{grp.getgrnam}. + + @rtype: C{int} + @return: The integer GID corresponding to the given string. + + @raise ValueError: If the group name is supplied and L{grp} is not + available. + """ + try: + return int(gidString) + except ValueError: + if grp is None: + raise + return grp.getgrnam(gidString)[2] + + + +def runAsEffectiveUser(euid, egid, function, *args, **kwargs): + """ + Run the given function wrapped with seteuid/setegid calls. + + This will try to minimize the number of seteuid/setegid calls, comparing + current and wanted permissions + + @param euid: effective UID used to call the function. + @type euid: C{int} + + @type egid: effective GID used to call the function. + @param egid: C{int} + + @param function: the function run with the specific permission. + @type function: any callable + + @param *args: arguments passed to C{function} + @param **kwargs: keyword arguments passed to C{function} + """ + uid, gid = os.geteuid(), os.getegid() + if uid == euid and gid == egid: + return function(*args, **kwargs) + else: + if uid != 0 and (uid != euid or gid != egid): + os.seteuid(0) + if gid != egid: + os.setegid(egid) + if euid != 0 and (euid != uid or gid != egid): + os.seteuid(euid) + try: + return function(*args, **kwargs) + finally: + if euid != 0 and (uid != euid or gid != egid): + os.seteuid(0) + if gid != egid: + os.setegid(gid) + if uid != 0 and (uid != euid or gid != egid): + os.seteuid(uid) + + + +def runWithWarningsSuppressed(suppressedWarnings, f, *args, **kwargs): + """ + Run C{f(*args, **kwargs)}, but with some warnings suppressed. + + Unlike L{twisted.internet.utils.runWithWarningsSuppressed}, it has no + special support for L{twisted.internet.defer.Deferred}. + + @param suppressedWarnings: A list of arguments to pass to filterwarnings. + Must be a sequence of 2-tuples (args, kwargs). + + @param f: A callable. + + @param args: Arguments for C{f}. + + @param kwargs: Keyword arguments for C{f} + + @return: The result of C{f(*args, **kwargs)}. + """ + with warnings.catch_warnings(): + for a, kw in suppressedWarnings: + warnings.filterwarnings(*a, **kw) + return f(*args, **kwargs) + + + +__all__ = [ + "uniquify", "padTo", "getPluginDirs", "addPluginDir", "sibpath", + "getPassword", "println", "makeStatBar", "OrderedDict", + "InsensitiveDict", "spewer", "searchupwards", "LineLog", + "raises", "IntervalDifferential", "FancyStrMixin", "FancyEqMixin", + "switchUID", "SubclassableCStringIO", "mergeFunctionMetadata", + "nameToLabel", "uidFromString", "gidFromString", "runAsEffectiveUser", + "untilConcludes", "runWithWarningsSuppressed", "_replaceIf", +] + + +if _PY3: + __notported__ = ["SubclassableCStringIO", "makeStatBar"] + for name in __all__[:]: + if name in __notported__: + __all__.remove(name) + del globals()[name] + del name, __notported__ diff --git a/contrib/python/Twisted/py2/twisted/python/versions.py b/contrib/python/Twisted/py2/twisted/python/versions.py new file mode 100644 index 00000000000..51355258a40 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/versions.py @@ -0,0 +1,14 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Versions for Python packages. + +See L{incremental}. +""" + +from __future__ import division, absolute_import + +from incremental import IncomparableVersions, Version, getVersionString + +__all__ = ["Version", "getVersionString", "IncomparableVersions"] diff --git a/contrib/python/Twisted/py2/twisted/python/win32.py b/contrib/python/Twisted/py2/twisted/python/win32.py new file mode 100644 index 00000000000..ae62da5853e --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/win32.py @@ -0,0 +1,136 @@ +# -*- test-case-name: twisted.python.test.test_win32 -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Win32 utilities. + +See also twisted.python.shortcut. + +@var O_BINARY: the 'binary' mode flag on Windows, or 0 on other platforms, so it + may safely be OR'ed into a mask for os.open. +""" + +from __future__ import division, absolute_import + +import re +import os + + +# http://msdn.microsoft.com/library/default.asp?url=/library/en-us/debug/base/system_error_codes.asp +ERROR_FILE_NOT_FOUND = 2 +ERROR_PATH_NOT_FOUND = 3 +ERROR_INVALID_NAME = 123 +ERROR_DIRECTORY = 267 + +O_BINARY = getattr(os, "O_BINARY", 0) + +class FakeWindowsError(OSError): + """ + Stand-in for sometimes-builtin exception on platforms for which it + is missing. + """ + +try: + WindowsError = WindowsError +except NameError: + WindowsError = FakeWindowsError + + +_cmdLineQuoteRe = re.compile(r'(\\*)"') +_cmdLineQuoteRe2 = re.compile(r'(\\+)\Z') +def cmdLineQuote(s): + """ + Internal method for quoting a single command-line argument. + + @param s: an unquoted string that you want to quote so that something that + does cmd.exe-style unquoting will interpret it as a single argument, + even if it contains spaces. + @type s: C{str} + + @return: a quoted string. + @rtype: C{str} + """ + quote = ((" " in s) or ("\t" in s) or ('"' in s) or s == '') and '"' or '' + return quote + _cmdLineQuoteRe2.sub(r"\1\1", _cmdLineQuoteRe.sub(r'\1\1\\"', s)) + quote + +def quoteArguments(arguments): + """ + Quote an iterable of command-line arguments for passing to CreateProcess or + a similar API. This allows the list passed to C{reactor.spawnProcess} to + match the child process's C{sys.argv} properly. + + @param arglist: an iterable of C{str}, each unquoted. + + @return: a single string, with the given sequence quoted as necessary. + """ + return ' '.join([cmdLineQuote(a) for a in arguments]) + + +class _ErrorFormatter(object): + """ + Formatter for Windows error messages. + + @ivar winError: A callable which takes one integer error number argument + and returns an L{exceptions.WindowsError} instance for that error (like + L{ctypes.WinError}). + + @ivar formatMessage: A callable which takes one integer error number + argument and returns a C{str} giving the message for that error (like + L{win32api.FormatMessage}). + + @ivar errorTab: A mapping from integer error numbers to C{str} messages + which correspond to those erorrs (like I{socket.errorTab}). + """ + def __init__(self, WinError, FormatMessage, errorTab): + self.winError = WinError + self.formatMessage = FormatMessage + self.errorTab = errorTab + + def fromEnvironment(cls): + """ + Get as many of the platform-specific error translation objects as + possible and return an instance of C{cls} created with them. + """ + try: + from ctypes import WinError + except ImportError: + WinError = None + try: + from win32api import FormatMessage + except ImportError: + FormatMessage = None + try: + from socket import errorTab + except ImportError: + errorTab = None + return cls(WinError, FormatMessage, errorTab) + fromEnvironment = classmethod(fromEnvironment) + + + def formatError(self, errorcode): + """ + Returns the string associated with a Windows error message, such as the + ones found in socket.error. + + Attempts direct lookup against the win32 API via ctypes and then + pywin32 if available), then in the error table in the socket module, + then finally defaulting to C{os.strerror}. + + @param errorcode: the Windows error code + @type errorcode: C{int} + + @return: The error message string + @rtype: C{str} + """ + if self.winError is not None: + return self.winError(errorcode).strerror + if self.formatMessage is not None: + return self.formatMessage(errorcode) + if self.errorTab is not None: + result = self.errorTab.get(errorcode) + if result is not None: + return result + return os.strerror(errorcode) + +formatError = _ErrorFormatter.fromEnvironment().formatError diff --git a/contrib/python/Twisted/py2/twisted/python/ya.make b/contrib/python/Twisted/py2/twisted/python/ya.make new file mode 100644 index 00000000000..760fe8e8def --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/ya.make @@ -0,0 +1,31 @@ +# subset of twisted.python module to resolve cycle dependency between Automat and twisted library + +PY2_LIBRARY() + +LICENSE(MIT) + +NO_LINT() + +NO_COMPILER_WARNINGS() + +PEERDIR( + contrib/python/incremental + contrib/python/zope.interface +) + +PY_SRCS( + NAMESPACE twisted.python + _oldstyle.py + compat.py + components.py + deprecate.py + filepath.py + modules.py + reflect.py + runtime.py + util.py + win32.py + zippath.py +) + +END() diff --git a/contrib/python/Twisted/py2/twisted/python/zippath.py b/contrib/python/Twisted/py2/twisted/python/zippath.py new file mode 100644 index 00000000000..6e0e78c1d81 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/zippath.py @@ -0,0 +1,295 @@ +# -*- test-case-name: twisted.python.test.test_zippath -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module contains implementations of L{IFilePath} for zip files. + +See the constructor of L{ZipArchive} for use. +""" + +from __future__ import absolute_import, division + +import os +import time +import errno + +from zipfile import ZipFile + +from twisted.python.compat import comparable, cmp +from twisted.python.filepath import IFilePath, FilePath, AbstractFilePath +from twisted.python.filepath import _coerceToFilesystemEncoding +from twisted.python.filepath import UnlistableError + +from zope.interface import implementer + +ZIP_PATH_SEP = '/' # In zipfiles, "/" is universally used as the + # path separator, regardless of platform. + + +@comparable +@implementer(IFilePath) +class ZipPath(AbstractFilePath): + """ + I represent a file or directory contained within a zip file. + """ + + def __init__(self, archive, pathInArchive): + """ + Don't construct me directly. Use C{ZipArchive.child()}. + + @param archive: a L{ZipArchive} instance. + + @param pathInArchive: a ZIP_PATH_SEP-separated string. + """ + self.archive = archive + self.pathInArchive = pathInArchive + + # self.path pretends to be os-specific because that's the way the + # 'zipimport' module does it. + sep = _coerceToFilesystemEncoding(pathInArchive, ZIP_PATH_SEP) + archiveFilename = _coerceToFilesystemEncoding( + pathInArchive, archive.zipfile.filename) + self.path = os.path.join(archiveFilename, + *(self.pathInArchive.split(sep))) + + + def __cmp__(self, other): + if not isinstance(other, ZipPath): + return NotImplemented + return cmp((self.archive, self.pathInArchive), + (other.archive, other.pathInArchive)) + + + def __repr__(self): + parts = [_coerceToFilesystemEncoding( + self.sep, os.path.abspath(self.archive.path))] + parts.extend(self.pathInArchive.split(self.sep)) + ossep = _coerceToFilesystemEncoding(self.sep, os.sep) + return "ZipPath(%r)" % (ossep.join(parts),) + + + @property + def sep(self): + """ + Return a zip directory separator. + + @return: The zip directory separator. + @returntype: The same type as C{self.path}. + """ + return _coerceToFilesystemEncoding(self.path, ZIP_PATH_SEP) + + + def parent(self): + splitup = self.pathInArchive.split(self.sep) + if len(splitup) == 1: + return self.archive + return ZipPath(self.archive, self.sep.join(splitup[:-1])) + + + def child(self, path): + """ + Return a new ZipPath representing a path in C{self.archive} which is + a child of this path. + + @note: Requesting the C{".."} (or other special name) child will not + cause L{InsecurePath} to be raised since these names do not have + any special meaning inside a zip archive. Be particularly + careful with the C{path} attribute (if you absolutely must use + it) as this means it may include special names with special + meaning outside of the context of a zip archive. + """ + joiner = _coerceToFilesystemEncoding(path, ZIP_PATH_SEP) + pathInArchive = _coerceToFilesystemEncoding(path, self.pathInArchive) + return ZipPath(self.archive, joiner.join([pathInArchive, path])) + + + def sibling(self, path): + return self.parent().child(path) + + + def exists(self): + return self.isdir() or self.isfile() + + + def isdir(self): + return self.pathInArchive in self.archive.childmap + + + def isfile(self): + return self.pathInArchive in self.archive.zipfile.NameToInfo + + + def islink(self): + return False + + + def listdir(self): + if self.exists(): + if self.isdir(): + return list(self.archive.childmap[self.pathInArchive].keys()) + else: + raise UnlistableError( + OSError(errno.ENOTDIR, "Leaf zip entry listed")) + else: + raise UnlistableError( + OSError(errno.ENOENT, "Non-existent zip entry listed")) + + + def splitext(self): + """ + Return a value similar to that returned by C{os.path.splitext}. + """ + # This happens to work out because of the fact that we use OS-specific + # path separators in the constructor to construct our fake 'path' + # attribute. + return os.path.splitext(self.path) + + + def basename(self): + return self.pathInArchive.split(self.sep)[-1] + + + def dirname(self): + # XXX NOTE: This API isn't a very good idea on filepath, but it's even + # less meaningful here. + return self.parent().path + + + def open(self, mode="r"): + pathInArchive = _coerceToFilesystemEncoding('', self.pathInArchive) + return self.archive.zipfile.open(pathInArchive, mode=mode) + + + def changed(self): + pass + + + def getsize(self): + """ + Retrieve this file's size. + + @return: file size, in bytes + """ + pathInArchive = _coerceToFilesystemEncoding("", self.pathInArchive) + return self.archive.zipfile.NameToInfo[pathInArchive].file_size + + + def getAccessTime(self): + """ + Retrieve this file's last access-time. This is the same as the last access + time for the archive. + + @return: a number of seconds since the epoch + """ + return self.archive.getAccessTime() + + + def getModificationTime(self): + """ + Retrieve this file's last modification time. This is the time of + modification recorded in the zipfile. + + @return: a number of seconds since the epoch. + """ + pathInArchive = _coerceToFilesystemEncoding("", self.pathInArchive) + return time.mktime( + self.archive.zipfile.NameToInfo[pathInArchive].date_time + + (0, 0, 0)) + + + def getStatusChangeTime(self): + """ + Retrieve this file's last modification time. This name is provided for + compatibility, and returns the same value as getmtime. + + @return: a number of seconds since the epoch. + """ + return self.getModificationTime() + + + +class ZipArchive(ZipPath): + """ + I am a L{FilePath}-like object which can wrap a zip archive as if it were a + directory. + + It works similarly to L{FilePath} in L{bytes} and L{unicode} handling -- + instantiating with a L{bytes} will return a "bytes mode" L{ZipArchive}, + and instantiating with a L{unicode} will return a "text mode" + L{ZipArchive}. Methods that return new L{ZipArchive} or L{ZipPath} + instances will be in the mode of the argument to the creator method, + converting if required. + """ + archive = property(lambda self: self) + + def __init__(self, archivePathname): + """ + Create a ZipArchive, treating the archive at archivePathname as a zip + file. + + @param archivePathname: a L{bytes} or L{unicode}, naming a path in the + filesystem. + """ + self.path = archivePathname + self.zipfile = ZipFile(_coerceToFilesystemEncoding('', + archivePathname)) + self.pathInArchive = _coerceToFilesystemEncoding(archivePathname, '') + # zipfile is already wasting O(N) memory on cached ZipInfo instances, + # so there's no sense in trying to do this lazily or intelligently + self.childmap = {} # map parent: list of children + + for name in self.zipfile.namelist(): + name = _coerceToFilesystemEncoding(self.path, name).split(self.sep) + for x in range(len(name)): + child = name[-x] + parent = self.sep.join(name[:-x]) + if parent not in self.childmap: + self.childmap[parent] = {} + self.childmap[parent][child] = 1 + parent = _coerceToFilesystemEncoding(archivePathname, '') + + + def child(self, path): + """ + Create a ZipPath pointing at a path within the archive. + + @param path: a L{bytes} or L{unicode} with no path separators in it + (either '/' or the system path separator, if it's different). + """ + return ZipPath(self, path) + + + def exists(self): + """ + Returns C{True} if the underlying archive exists. + """ + return FilePath(self.zipfile.filename).exists() + + + def getAccessTime(self): + """ + Return the archive file's last access time. + """ + return FilePath(self.zipfile.filename).getAccessTime() + + + def getModificationTime(self): + """ + Return the archive file's modification time. + """ + return FilePath(self.zipfile.filename).getModificationTime() + + + def getStatusChangeTime(self): + """ + Return the archive file's status change time. + """ + return FilePath(self.zipfile.filename).getStatusChangeTime() + + + def __repr__(self): + return 'ZipArchive(%r)' % (os.path.abspath(self.path),) + + +__all__ = ['ZipArchive', 'ZipPath'] diff --git a/contrib/python/Twisted/py2/twisted/python/zipstream.py b/contrib/python/Twisted/py2/twisted/python/zipstream.py new file mode 100644 index 00000000000..61f3186b3e6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/python/zipstream.py @@ -0,0 +1,336 @@ +# -*- test-case-name: twisted.python.test.test_zipstream -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An incremental approach to unzipping files. This allows you to unzip a little +bit of a file at a time, which means you can report progress as a file unzips. +""" + +import zipfile +import os.path +import zlib +import struct + + +_fileHeaderSize = struct.calcsize(zipfile.structFileHeader) + +class ChunkingZipFile(zipfile.ZipFile): + """ + A L{zipfile.ZipFile} object which, with L{readfile}, also gives you access + to a file-like object for each entry. + """ + + def readfile(self, name): + """ + Return file-like object for name. + """ + if self.mode not in ("r", "a"): + raise RuntimeError('read() requires mode "r" or "a"') + if not self.fp: + raise RuntimeError( + "Attempt to read ZIP archive that was already closed") + zinfo = self.getinfo(name) + + self.fp.seek(zinfo.header_offset, 0) + + fheader = self.fp.read(_fileHeaderSize) + if fheader[0:4] != zipfile.stringFileHeader: + raise zipfile.BadZipfile("Bad magic number for file header") + + fheader = struct.unpack(zipfile.structFileHeader, fheader) + fname = self.fp.read(fheader[zipfile._FH_FILENAME_LENGTH]) + + if fheader[zipfile._FH_EXTRA_FIELD_LENGTH]: + self.fp.read(fheader[zipfile._FH_EXTRA_FIELD_LENGTH]) + + + if zinfo.flag_bits & 0x800: + # UTF-8 filename + fname_str = fname.decode("utf-8") + else: + fname_str = fname.decode("cp437") + + if fname_str != zinfo.orig_filename: + raise zipfile.BadZipfile( + 'File name in directory "%s" and header "%s" differ.' % ( + zinfo.orig_filename, fname_str)) + + if zinfo.compress_type == zipfile.ZIP_STORED: + return ZipFileEntry(self, zinfo.compress_size) + elif zinfo.compress_type == zipfile.ZIP_DEFLATED: + return DeflatedZipFileEntry(self, zinfo.compress_size) + else: + raise zipfile.BadZipfile( + "Unsupported compression method %d for file %s" % + (zinfo.compress_type, name)) + + + +class _FileEntry(object): + """ + Abstract superclass of both compressed and uncompressed variants of + file-like objects within a zip archive. + + @ivar chunkingZipFile: a chunking zip file. + @type chunkingZipFile: L{ChunkingZipFile} + + @ivar length: The number of bytes within the zip file that represent this + file. (This is the size on disk, not the number of decompressed bytes + which will result from reading it.) + + @ivar fp: the underlying file object (that contains pkzip data). Do not + touch this, please. It will quite likely move or go away. + + @ivar closed: File-like 'closed' attribute; True before this file has been + closed, False after. + @type closed: L{bool} + + @ivar finished: An older, broken synonym for 'closed'. Do not touch this, + please. + @type finished: L{int} + """ + def __init__(self, chunkingZipFile, length): + """ + Create a L{_FileEntry} from a L{ChunkingZipFile}. + """ + self.chunkingZipFile = chunkingZipFile + self.fp = self.chunkingZipFile.fp + self.length = length + self.finished = 0 + self.closed = False + + + def isatty(self): + """ + Returns false because zip files should not be ttys + """ + return False + + + def close(self): + """ + Close self (file-like object) + """ + self.closed = True + self.finished = 1 + del self.fp + + + def readline(self): + """ + Read a line. + """ + line = b"" + for byte in iter(lambda : self.read(1), b""): + line += byte + if byte == b"\n": + break + return line + + + def __next__(self): + """ + Implement next as file does (like readline, except raises StopIteration + at EOF) + """ + nextline = self.readline() + if nextline: + return nextline + raise StopIteration() + + # Iterators on Python 2 use next(), not __next__() + next = __next__ + + + def readlines(self): + """ + Returns a list of all the lines + """ + return list(self) + + + def xreadlines(self): + """ + Returns an iterator (so self) + """ + return self + + + def __iter__(self): + """ + Returns an iterator (so self) + """ + return self + + + def __enter__(self): + return self + + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + + +class ZipFileEntry(_FileEntry): + """ + File-like object used to read an uncompressed entry in a ZipFile + """ + + def __init__(self, chunkingZipFile, length): + _FileEntry.__init__(self, chunkingZipFile, length) + self.readBytes = 0 + + + def tell(self): + return self.readBytes + + + def read(self, n=None): + if n is None: + n = self.length - self.readBytes + if n == 0 or self.finished: + return b'' + data = self.chunkingZipFile.fp.read( + min(n, self.length - self.readBytes)) + self.readBytes += len(data) + if self.readBytes == self.length or len(data) < n: + self.finished = 1 + return data + + + +class DeflatedZipFileEntry(_FileEntry): + """ + File-like object used to read a deflated entry in a ZipFile + """ + + def __init__(self, chunkingZipFile, length): + _FileEntry.__init__(self, chunkingZipFile, length) + self.returnedBytes = 0 + self.readBytes = 0 + self.decomp = zlib.decompressobj(-15) + self.buffer = b"" + + + def tell(self): + return self.returnedBytes + + + def read(self, n=None): + if self.finished: + return b"" + if n is None: + result = [self.buffer,] + result.append( + self.decomp.decompress( + self.chunkingZipFile.fp.read( + self.length - self.readBytes))) + result.append(self.decomp.decompress(b"Z")) + result.append(self.decomp.flush()) + self.buffer = b"" + self.finished = 1 + result = b"".join(result) + self.returnedBytes += len(result) + return result + else: + while len(self.buffer) < n: + data = self.chunkingZipFile.fp.read( + min(n, 1024, self.length - self.readBytes)) + self.readBytes += len(data) + if not data: + result = (self.buffer + + self.decomp.decompress(b"Z") + + self.decomp.flush()) + self.finished = 1 + self.buffer = b"" + self.returnedBytes += len(result) + return result + else: + self.buffer += self.decomp.decompress(data) + result = self.buffer[:n] + self.buffer = self.buffer[n:] + self.returnedBytes += len(result) + return result + + + +DIR_BIT = 16 + + +def countZipFileChunks(filename, chunksize): + """ + Predict the number of chunks that will be extracted from the entire + zipfile, given chunksize blocks. + """ + totalchunks = 0 + zf = ChunkingZipFile(filename) + for info in zf.infolist(): + totalchunks += countFileChunks(info, chunksize) + return totalchunks + + +def countFileChunks(zipinfo, chunksize): + """ + Count the number of chunks that will result from the given C{ZipInfo}. + + @param zipinfo: a C{zipfile.ZipInfo} instance describing an entry in a zip + archive to be counted. + + @return: the number of chunks present in the zip file. (Even an empty file + counts as one chunk.) + @rtype: L{int} + """ + count, extra = divmod(zipinfo.file_size, chunksize) + if extra > 0: + count += 1 + return count or 1 + + + +def unzipIterChunky(filename, directory='.', overwrite=0, + chunksize=4096): + """ + Return a generator for the zipfile. This implementation will yield after + every chunksize uncompressed bytes, or at the end of a file, whichever + comes first. + + The value it yields is the number of chunks left to unzip. + """ + czf = ChunkingZipFile(filename, 'r') + if not os.path.exists(directory): + os.makedirs(directory) + remaining = countZipFileChunks(filename, chunksize) + names = czf.namelist() + infos = czf.infolist() + + for entry, info in zip(names, infos): + isdir = info.external_attr & DIR_BIT + f = os.path.join(directory, entry) + if isdir: + # overwrite flag only applies to files + if not os.path.exists(f): + os.makedirs(f) + remaining -= 1 + yield remaining + else: + # create the directory the file will be in first, + # since we can't guarantee it exists + fdir = os.path.split(f)[0] + if not os.path.exists(fdir): + os.makedirs(fdir) + if overwrite or not os.path.exists(f): + fp = czf.readfile(entry) + if info.file_size == 0: + remaining -= 1 + yield remaining + with open(f, 'wb') as outfile: + while fp.tell() < info.file_size: + hunk = fp.read(chunksize) + outfile.write(hunk) + remaining -= 1 + yield remaining + else: + remaining -= countFileChunks(info, chunksize) + yield remaining diff --git a/contrib/python/Twisted/py2/twisted/runner/__init__.py b/contrib/python/Twisted/py2/twisted/runner/__init__.py new file mode 100644 index 00000000000..024a284959f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Runner: Run and monitor processes. +""" diff --git a/contrib/python/Twisted/py2/twisted/runner/inetd.py b/contrib/python/Twisted/py2/twisted/runner/inetd.py new file mode 100644 index 00000000000..3402e232e43 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/inetd.py @@ -0,0 +1,70 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + +""" +Twisted inetd. + +Maintainer: Andrew Bennetts + +Future Plans: Bugfixes. Specifically for UDP and Sun-RPC, which don't work +correctly yet. +""" + +import os + +from twisted.internet import process, reactor, fdesc +from twisted.internet.protocol import Protocol, ServerFactory +from twisted.protocols import wire + +# A dict of known 'internal' services (i.e. those that don't involve spawning +# another process. +internalProtocols = { + 'echo': wire.Echo, + 'chargen': wire.Chargen, + 'discard': wire.Discard, + 'daytime': wire.Daytime, + 'time': wire.Time, +} + + +class InetdProtocol(Protocol): + """Forks a child process on connectionMade, passing the socket as fd 0.""" + def connectionMade(self): + sockFD = self.transport.fileno() + childFDs = {0: sockFD, 1: sockFD} + if self.factory.stderrFile: + childFDs[2] = self.factory.stderrFile.fileno() + + # processes run by inetd expect blocking sockets + # FIXME: maybe this should be done in process.py? are other uses of + # Process possibly affected by this? + fdesc.setBlocking(sockFD) + if 2 in childFDs: + fdesc.setBlocking(childFDs[2]) + + service = self.factory.service + uid = service.user + gid = service.group + + # don't tell Process to change our UID/GID if it's what we + # already are + if uid == os.getuid(): + uid = None + if gid == os.getgid(): + gid = None + + process.Process(None, service.program, service.programArgs, os.environ, + None, None, uid, gid, childFDs) + + reactor.removeReader(self.transport) + reactor.removeWriter(self.transport) + + +class InetdFactory(ServerFactory): + protocol = InetdProtocol + stderrFile = None + + def __init__(self, service): + self.service = service diff --git a/contrib/python/Twisted/py2/twisted/runner/inetdconf.py b/contrib/python/Twisted/py2/twisted/runner/inetdconf.py new file mode 100644 index 00000000000..391f2e0573f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/inetdconf.py @@ -0,0 +1,198 @@ +# -*- test-case-name: twisted.runner.test.test_inetdconf -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Parser for inetd.conf files +""" + +# Various exceptions +class InvalidConfError(Exception): + """ + Invalid configuration file + """ + + + +class InvalidInetdConfError(InvalidConfError): + """ + Invalid inetd.conf file + """ + + + +class InvalidServicesConfError(InvalidConfError): + """ + Invalid services file + """ + + + +class UnknownService(Exception): + """ + Unknown service name + """ + + + +class SimpleConfFile: + """ + Simple configuration file parser superclass. + + Filters out comments and empty lines (which includes lines that only + contain comments). + + To use this class, override parseLine or parseFields. + """ + + commentChar = '#' + defaultFilename = None + + def parseFile(self, file=None): + """ + Parse a configuration file + + If file is None and self.defaultFilename is set, it will open + defaultFilename and use it. + """ + close = False + if file is None and self.defaultFilename: + file = open(self.defaultFilename,'r') + close = True + + try: + for line in file.readlines(): + # Strip out comments + comment = line.find(self.commentChar) + if comment != -1: + line = line[:comment] + + # Strip whitespace + line = line.strip() + + # Skip empty lines (and lines which only contain comments) + if not line: + continue + + self.parseLine(line) + finally: + if close: + file.close() + + + def parseLine(self, line): + """ + Override this. + + By default, this will split the line on whitespace and call + self.parseFields (catching any errors). + """ + try: + self.parseFields(*line.split()) + except ValueError: + raise InvalidInetdConfError('Invalid line: ' + repr(line)) + + + def parseFields(self, *fields): + """ + Override this. + """ + + + +class InetdService: + """ + A simple description of an inetd service. + """ + name = None + port = None + socketType = None + protocol = None + wait = None + user = None + group = None + program = None + programArgs = None + + def __init__(self, name, port, socketType, protocol, wait, user, group, + program, programArgs): + self.name = name + self.port = port + self.socketType = socketType + self.protocol = protocol + self.wait = wait + self.user = user + self.group = group + self.program = program + self.programArgs = programArgs + + + +class InetdConf(SimpleConfFile): + """ + Configuration parser for a traditional UNIX inetd(8) + """ + + defaultFilename = '/etc/inetd.conf' + + def __init__(self, knownServices=None): + self.services = [] + + if knownServices is None: + knownServices = ServicesConf() + knownServices.parseFile() + self.knownServices = knownServices + + + def parseFields(self, serviceName, socketType, protocol, wait, user, + program, *programArgs): + """ + Parse an inetd.conf file. + + Implemented from the description in the Debian inetd.conf man page. + """ + # Extract user (and optional group) + user, group = (user.split('.') + [None])[:2] + + # Find the port for a service + port = self.knownServices.services.get((serviceName, protocol), None) + if not port and not protocol.startswith('rpc/'): + # FIXME: Should this be discarded/ignored, rather than throwing + # an exception? + try: + port = int(serviceName) + serviceName = 'unknown' + except: + raise UnknownService("Unknown service: %s (%s)" % ( + serviceName, protocol)) + + self.services.append(InetdService(serviceName, port, socketType, + protocol, wait, user, group, program, + programArgs)) + + + +class ServicesConf(SimpleConfFile): + """ + /etc/services parser + + @ivar services: dict mapping service names to (port, protocol) tuples. + """ + + defaultFilename = '/etc/services' + + def __init__(self): + self.services = {} + + + def parseFields(self, name, portAndProtocol, *aliases): + try: + port, protocol = portAndProtocol.split('/') + port = int(port) + except: + raise InvalidServicesConfError( + 'Invalid port/protocol: %s' % (repr(portAndProtocol),)) + + self.services[(name, protocol)] = port + for alias in aliases: + self.services[(alias, protocol)] = port diff --git a/contrib/python/Twisted/py2/twisted/runner/inetdtap.py b/contrib/python/Twisted/py2/twisted/runner/inetdtap.py new file mode 100644 index 00000000000..b959942c51d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/inetdtap.py @@ -0,0 +1,109 @@ +# -*- test-case-name: twisted.runner.test.test_inetdtap -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted inetd TAP support + +The purpose of inetdtap is to provide an inetd-like server, to allow Twisted to +invoke other programs to handle incoming sockets. +This is a useful thing as a "networking swiss army knife" tool, like netcat. +""" + +import pwd, grp, socket + +from twisted.runner import inetd, inetdconf +from twisted.python import log, usage +from twisted.internet.protocol import ServerFactory +from twisted.application import internet, service as appservice + +# Protocol map +protocolDict = {'tcp': socket.IPPROTO_TCP, 'udp': socket.IPPROTO_UDP} + + +class Options(usage.Options): + """ + To use it, create a file named `sample-inetd.conf` with: + + 8123 stream tcp wait some_user /bin/cat - + + You can then run it as in the following example and port 8123 became an + echo server. + + twistd -n inetd -f sample-inetd.conf + """ + + optParameters = [ + ['rpc', 'r', '/etc/rpc', 'DEPRECATED. RPC procedure table file'], + ['file', 'f', '/etc/inetd.conf', 'Service configuration file'] + ] + + optFlags = [['nointernal', 'i', "Don't run internal services"]] + + compData = usage.Completions( + optActions={"file": usage.CompleteFiles('*.conf')} + ) + + + +def makeService(config): + s = appservice.MultiService() + conf = inetdconf.InetdConf() + with open(config['file']) as f: + conf.parseFile(f) + + for service in conf.services: + protocol = service.protocol + + if service.protocol.startswith('rpc/'): + log.msg('Skipping rpc service due to lack of rpc support') + continue + + if (protocol, service.socketType) not in [('tcp', 'stream'), + ('udp', 'dgram')]: + log.msg('Skipping unsupported type/protocol: %s/%s' + % (service.socketType, service.protocol)) + continue + + # Convert the username into a uid (if necessary) + try: + service.user = int(service.user) + except ValueError: + try: + service.user = pwd.getpwnam(service.user)[2] + except KeyError: + log.msg('Unknown user: ' + service.user) + continue + + # Convert the group name into a gid (if necessary) + if service.group is None: + # If no group was specified, use the user's primary group + service.group = pwd.getpwuid(service.user)[3] + else: + try: + service.group = int(service.group) + except ValueError: + try: + service.group = grp.getgrnam(service.group)[2] + except KeyError: + log.msg('Unknown group: ' + service.group) + continue + + if service.program == 'internal': + if config['nointernal']: + continue + + # Internal services can use a standard ServerFactory + if service.name not in inetd.internalProtocols: + log.msg('Unknown internal service: ' + service.name) + continue + factory = ServerFactory() + factory.protocol = inetd.internalProtocols[service.name] + else: + factory = inetd.InetdFactory(service) + + if protocol == 'tcp': + internet.TCPServer(service.port, factory).setServiceParent(s) + elif protocol == 'udp': + raise RuntimeError("not supporting UDP") + return s diff --git a/contrib/python/Twisted/py2/twisted/runner/procmon.py b/contrib/python/Twisted/py2/twisted/runner/procmon.py new file mode 100644 index 00000000000..05909570b1d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/procmon.py @@ -0,0 +1,426 @@ +# -*- test-case-name: twisted.runner.test.test_procmon -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for starting, monitoring, and restarting child process. +""" +import attr +import incremental + +from twisted.python import deprecate +from twisted.internet import error, protocol, reactor as _reactor +from twisted.application import service +from twisted.protocols import basic +from twisted.logger import Logger + + + +@attr.s(frozen=True) +class _Process(object): + """ + The parameters of a process to be restarted. + + @ivar args: command-line arguments (including name of command as first one) + @type args: C{list} + + @ivar uid: user-id to run process as, or None (which means inherit uid) + @type uid: C{int} + + @ivar gid: group-id to run process as, or None (which means inherit gid) + @type gid: C{int} + + @ivar env: environment for process + @type env: C{dict} + + @ivar cwd: initial working directory for process or None + (which means inherit cwd) + @type cwd: C{str} + """ + + args = attr.ib() + uid = attr.ib(default=None) + gid = attr.ib(default=None) + env = attr.ib(default=attr.Factory(dict)) + cwd = attr.ib(default=None) + + @deprecate.deprecated(incremental.Version('Twisted', 18, 7, 0)) + def toTuple(self): + """ + Convert process to tuple. + + Convert process to tuple that looks like the legacy structure + of processes, for potential users who inspected processes + directly. + + This was only an accidental feature, and will be removed. If + you need to remember what processes were added to a process monitor, + keep track of that when they are added. The process list + inside the process monitor is no longer a public API. + + This allows changing the internal structure of the process list, + when warranted by bug fixes or additional features. + + @return: tuple representation of process + """ + return (self.args, self.uid, self.gid, self.env) + + + +class DummyTransport: + + disconnecting = 0 + + + +transport = DummyTransport() + + + +class LineLogger(basic.LineReceiver): + + tag = None + stream = None + delimiter = b'\n' + service = None + + def lineReceived(self, line): + try: + line = line.decode('utf-8') + except UnicodeDecodeError: + line = repr(line) + + self.service.log.info(u'[{tag}] {line}', + tag=self.tag, + line=line, + stream=self.stream) + + + +class LoggingProtocol(protocol.ProcessProtocol): + + service = None + name = None + + def connectionMade(self): + self._output = LineLogger() + self._output.tag = self.name + self._output.stream = 'stdout' + self._output.service = self.service + self._outputEmpty = True + + self._error = LineLogger() + self._error.tag = self.name + self._error.stream = 'stderr' + self._error.service = self.service + self._errorEmpty = True + + self._output.makeConnection(transport) + self._error.makeConnection(transport) + + + def outReceived(self, data): + self._output.dataReceived(data) + self._outputEmpty = data[-1] == b'\n' + + def errReceived(self, data): + self._error.dataReceived(data) + self._errorEmpty = data[-1] == b'\n' + + def processEnded(self, reason): + if not self._outputEmpty: + self._output.dataReceived(b'\n') + if not self._errorEmpty: + self._error.dataReceived(b'\n') + self.service.connectionLost(self.name) + + @property + def output(self): + return self._output + + @property + def empty(self): + return self._outputEmpty + + + +class ProcessMonitor(service.Service): + """ + ProcessMonitor runs processes, monitors their progress, and restarts + them when they die. + + The ProcessMonitor will not attempt to restart a process that appears to + die instantly -- with each "instant" death (less than 1 second, by + default), it will delay approximately twice as long before restarting + it. A successful run will reset the counter. + + The primary interface is L{addProcess} and L{removeProcess}. When the + service is running (that is, when the application it is attached to is + running), adding a process automatically starts it. + + Each process has a name. This name string must uniquely identify the + process. In particular, attempting to add two processes with the same + name will result in a C{KeyError}. + + @type threshold: C{float} + @ivar threshold: How long a process has to live before the death is + considered instant, in seconds. The default value is 1 second. + + @type killTime: C{float} + @ivar killTime: How long a process being killed has to get its affairs + in order before it gets killed with an unmaskable signal. The + default value is 5 seconds. + + @type minRestartDelay: C{float} + @ivar minRestartDelay: The minimum time (in seconds) to wait before + attempting to restart a process. Default 1s. + + @type maxRestartDelay: C{float} + @ivar maxRestartDelay: The maximum time (in seconds) to wait before + attempting to restart a process. Default 3600s (1h). + + @type _reactor: L{IReactorProcess} provider + @ivar _reactor: A provider of L{IReactorProcess} and L{IReactorTime} + which will be used to spawn processes and register delayed calls. + + @type log: L{Logger} + @ivar log: The logger used to propagate log messages from spawned + processes. + + """ + threshold = 1 + killTime = 5 + minRestartDelay = 1 + maxRestartDelay = 3600 + log = Logger() + + + def __init__(self, reactor=_reactor): + self._reactor = reactor + + self._processes = {} + self.protocols = {} + self.delay = {} + self.timeStarted = {} + self.murder = {} + self.restart = {} + + + @deprecate.deprecatedProperty(incremental.Version('Twisted', 18, 7, 0)) + def processes(self): + """ + Processes as dict of tuples + + @return: Dict of process name to monitored processes as tuples + """ + return {name: process.toTuple() + for name, process in self._processes.items()} + + + @deprecate.deprecated(incremental.Version('Twisted', 18, 7, 0)) + def __getstate__(self): + dct = service.Service.__getstate__(self) + del dct['_reactor'] + dct['protocols'] = {} + dct['delay'] = {} + dct['timeStarted'] = {} + dct['murder'] = {} + dct['restart'] = {} + del dct['_processes'] + dct['processes'] = self.processes + return dct + + + def addProcess(self, name, args, uid=None, gid=None, env={}, cwd=None): + """ + Add a new monitored process and start it immediately if the + L{ProcessMonitor} service is running. + + Note that args are passed to the system call, not to the shell. If + running the shell is desired, the common idiom is to use + C{ProcessMonitor.addProcess("name", ['/bin/sh', '-c', shell_script])} + + @param name: A name for this process. This value must be + unique across all processes added to this monitor. + @type name: C{str} + @param args: The argv sequence for the process to launch. + @param uid: The user ID to use to run the process. If L{None}, + the current UID is used. + @type uid: C{int} + @param gid: The group ID to use to run the process. If L{None}, + the current GID is used. + @type uid: C{int} + @param env: The environment to give to the launched process. See + L{IReactorProcess.spawnProcess}'s C{env} parameter. + @type env: C{dict} + @param cwd: The initial working directory of the launched process. + The default of C{None} means inheriting the laucnhing process's + working directory. + @type env: C{dict} + @raises: C{KeyError} if a process with the given name already + exists + """ + if name in self._processes: + raise KeyError("remove %s first" % (name,)) + self._processes[name] = _Process(args, uid, gid, env, cwd) + self.delay[name] = self.minRestartDelay + if self.running: + self.startProcess(name) + + + def removeProcess(self, name): + """ + Stop the named process and remove it from the list of monitored + processes. + + @type name: C{str} + @param name: A string that uniquely identifies the process. + """ + self.stopProcess(name) + del self._processes[name] + + + def startService(self): + """ + Start all monitored processes. + """ + service.Service.startService(self) + for name in list(self._processes): + self.startProcess(name) + + + def stopService(self): + """ + Stop all monitored processes and cancel all scheduled process restarts. + """ + service.Service.stopService(self) + + # Cancel any outstanding restarts + for name, delayedCall in list(self.restart.items()): + if delayedCall.active(): + delayedCall.cancel() + + for name in list(self._processes): + self.stopProcess(name) + + + def connectionLost(self, name): + """ + Called when a monitored processes exits. If + L{service.IService.running} is L{True} (ie the service is started), the + process will be restarted. + If the process had been running for more than + L{ProcessMonitor.threshold} seconds it will be restarted immediately. + If the process had been running for less than + L{ProcessMonitor.threshold} seconds, the restart will be delayed and + each time the process dies before the configured threshold, the restart + delay will be doubled - up to a maximum delay of maxRestartDelay sec. + + @type name: C{str} + @param name: A string that uniquely identifies the process + which exited. + """ + # Cancel the scheduled _forceStopProcess function if the process + # dies naturally + if name in self.murder: + if self.murder[name].active(): + self.murder[name].cancel() + del self.murder[name] + + del self.protocols[name] + + if self._reactor.seconds() - self.timeStarted[name] < self.threshold: + # The process died too fast - backoff + nextDelay = self.delay[name] + self.delay[name] = min(self.delay[name] * 2, self.maxRestartDelay) + + else: + # Process had been running for a significant amount of time + # restart immediately + nextDelay = 0 + self.delay[name] = self.minRestartDelay + + # Schedule a process restart if the service is running + if self.running and name in self._processes: + self.restart[name] = self._reactor.callLater(nextDelay, + self.startProcess, + name) + + + def startProcess(self, name): + """ + @param name: The name of the process to be started + """ + # If a protocol instance already exists, it means the process is + # already running + if name in self.protocols: + return + + process = self._processes[name] + + proto = LoggingProtocol() + proto.service = self + proto.name = name + self.protocols[name] = proto + self.timeStarted[name] = self._reactor.seconds() + self._reactor.spawnProcess(proto, process.args[0], process.args, + uid=process.uid, gid=process.gid, + env=process.env, path=process.cwd) + + + def _forceStopProcess(self, proc): + """ + @param proc: An L{IProcessTransport} provider + """ + try: + proc.signalProcess('KILL') + except error.ProcessExitedAlready: + pass + + + def stopProcess(self, name): + """ + @param name: The name of the process to be stopped + """ + if name not in self._processes: + raise KeyError('Unrecognized process name: %s' % (name,)) + + proto = self.protocols.get(name, None) + if proto is not None: + proc = proto.transport + try: + proc.signalProcess('TERM') + except error.ProcessExitedAlready: + pass + else: + self.murder[name] = self._reactor.callLater( + self.killTime, + self._forceStopProcess, proc) + + + def restartAll(self): + """ + Restart all processes. This is useful for third party management + services to allow a user to restart servers because of an outside change + in circumstances -- for example, a new version of a library is + installed. + """ + for name in self._processes: + self.stopProcess(name) + + + def __repr__(self): + l = [] + for name, proc in self._processes.items(): + uidgid = '' + if proc.uid is not None: + uidgid = str(proc.uid) + if proc.gid is not None: + uidgid += ':'+str(proc.gid) + + if uidgid: + uidgid = '(' + uidgid + ')' + l.append('%r%s: %r' % (name, uidgid, proc.args)) + return ('<' + self.__class__.__name__ + ' ' + + ' '.join(l) + + '>') diff --git a/contrib/python/Twisted/py2/twisted/runner/procmontap.py b/contrib/python/Twisted/py2/twisted/runner/procmontap.py new file mode 100644 index 00000000000..c0e72a45e8d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/procmontap.py @@ -0,0 +1,73 @@ +# -*- test-case-name: twisted.runner.test.test_procmontap -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for creating a service which runs a process monitor. +""" + +from twisted.python import usage +from twisted.runner.procmon import ProcessMonitor + + +class Options(usage.Options): + """ + Define the options accepted by the I{twistd procmon} plugin. + """ + + synopsis = "[procmon options] commandline" + + optParameters = [["threshold", "t", 1, "How long a process has to live " + "before the death is considered instant, in seconds.", + float], + ["killtime", "k", 5, "How long a process being killed " + "has to get its affairs in order before it gets killed " + "with an unmaskable signal.", + float], + ["minrestartdelay", "m", 1, "The minimum time (in " + "seconds) to wait before attempting to restart a " + "process", float], + ["maxrestartdelay", "M", 3600, "The maximum time (in " + "seconds) to wait before attempting to restart a " + "process", float]] + + optFlags = [] + + + longdesc = """\ +procmon runs processes, monitors their progress, and restarts them when they +die. + +procmon will not attempt to restart a process that appears to die instantly; +with each "instant" death (less than 1 second, by default), it will delay +approximately twice as long before restarting it. A successful run will reset +the counter. + +Eg twistd procmon sleep 10""" + + def parseArgs(self, *args): + """ + Grab the command line that is going to be started and monitored + """ + self['args'] = args + + + def postOptions(self): + """ + Check for dependencies. + """ + if len(self["args"]) < 1: + raise usage.UsageError("Please specify a process commandline") + + + +def makeService(config): + s = ProcessMonitor() + + s.threshold = config["threshold"] + s.killTime = config["killtime"] + s.minRestartDelay = config["minrestartdelay"] + s.maxRestartDelay = config["maxrestartdelay"] + + s.addProcess(" ".join(config["args"]), config["args"]) + return s diff --git a/contrib/python/Twisted/py2/twisted/scripts/__init__.py b/contrib/python/Twisted/py2/twisted/scripts/__init__.py new file mode 100644 index 00000000000..73d90a8eaf2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/scripts/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Subpackage containing the modules that implement the command line tools. + +Note that these are imported by top-level scripts which are intended to be +invoked directly from a shell. +""" diff --git a/contrib/python/Twisted/py2/twisted/scripts/_twistd_unix.py b/contrib/python/Twisted/py2/twisted/scripts/_twistd_unix.py new file mode 100644 index 00000000000..00488bf4299 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/scripts/_twistd_unix.py @@ -0,0 +1,453 @@ +# -*- test-case-name: twisted.test.test_twistd -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division, print_function + +import errno +import os +import pwd +import sys +import traceback + +from twisted.python import log, logfile, usage +from twisted.python.compat import (intToBytes, _bytesRepr, _PY3) +from twisted.python.util import ( + switchUID, uidFromString, gidFromString, untilConcludes) +from twisted.application import app, service +from twisted.internet.interfaces import IReactorDaemonize +from twisted import copyright, logger +from twisted.python.runtime import platformType + + + +if platformType == "win32": + raise ImportError("_twistd_unix doesn't work on Windows.") + + +def _umask(value): + return int(value, 8) + + +class ServerOptions(app.ServerOptions): + synopsis = "Usage: twistd [options]" + + optFlags = [['nodaemon', 'n', "don't daemonize, don't use default umask of 0077"], + ['originalname', None, "Don't try to change the process name"], + ['syslog', None, "Log to syslog, not to file"], + ['euid', '', + "Set only effective user-id rather than real user-id. " + "(This option has no effect unless the server is running as " + "root, in which case it means not to shed all privileges " + "after binding ports, retaining the option to regain " + "privileges in cases such as spawning processes. " + "Use with caution.)"], + ] + + optParameters = [ + ['prefix', None,'twisted', + "use the given prefix when syslogging"], + ['pidfile','','twistd.pid', + "Name of the pidfile"], + ['chroot', None, None, + 'Chroot to a supplied directory before running'], + ['uid', 'u', None, "The uid to run as.", uidFromString], + ['gid', 'g', None, + "The gid to run as. If not specified, the default gid " + "associated with the specified --uid is used.", + gidFromString], + ['umask', None, None, + "The (octal) file creation mask to apply.", _umask], + ] + + compData = usage.Completions( + optActions={"pidfile": usage.CompleteFiles("*.pid"), + "chroot": usage.CompleteDirs(descr="chroot directory"), + "gid": usage.CompleteGroups(descr="gid to run as"), + "uid": usage.CompleteUsernames(descr="uid to run as"), + "prefix": usage.Completer(descr="syslog prefix"), + }, + ) + + + def opt_version(self): + """ + Print version information and exit. + """ + print('twistd (the Twisted daemon) {}'.format(copyright.version), + file=self.stdout) + print(copyright.copyright, file=self.stdout) + sys.exit() + + + def postOptions(self): + app.ServerOptions.postOptions(self) + if self['pidfile']: + self['pidfile'] = os.path.abspath(self['pidfile']) + + +def checkPID(pidfile): + if not pidfile: + return + if os.path.exists(pidfile): + try: + with open(pidfile) as f: + pid = int(f.read()) + except ValueError: + sys.exit('Pidfile {} contains non-numeric value'.format(pidfile)) + try: + os.kill(pid, 0) + except OSError as why: + if why.errno == errno.ESRCH: + # The pid doesn't exist. + log.msg('Removing stale pidfile {}'.format(pidfile), isError=True) + os.remove(pidfile) + else: + sys.exit( + "Can't check status of PID {} from pidfile {}: {}".format( + pid, pidfile, why)) + else: + sys.exit("""\ +Another twistd server is running, PID {}\n +This could either be a previously started instance of your application or a +different application entirely. To start a new one, either run it in some other +directory, or use the --pidfile and --logfile parameters to avoid clashes. +""".format(pid)) + + + +class UnixAppLogger(app.AppLogger): + """ + A logger able to log to syslog, to files, and to stdout. + + @ivar _syslog: A flag indicating whether to use syslog instead of file + logging. + @type _syslog: C{bool} + + @ivar _syslogPrefix: If C{sysLog} is C{True}, the string prefix to use for + syslog messages. + @type _syslogPrefix: C{str} + + @ivar _nodaemon: A flag indicating the process will not be daemonizing. + @type _nodaemon: C{bool} + """ + + def __init__(self, options): + app.AppLogger.__init__(self, options) + self._syslog = options.get("syslog", False) + self._syslogPrefix = options.get("prefix", "") + self._nodaemon = options.get("nodaemon", False) + + + def _getLogObserver(self): + """ + Create and return a suitable log observer for the given configuration. + + The observer will go to syslog using the prefix C{_syslogPrefix} if + C{_syslog} is true. Otherwise, it will go to the file named + C{_logfilename} or, if C{_nodaemon} is true and C{_logfilename} is + C{"-"}, to stdout. + + @return: An object suitable to be passed to C{log.addObserver}. + """ + if self._syslog: + from twisted.python import syslog + return syslog.SyslogObserver(self._syslogPrefix).emit + + if self._logfilename == '-': + if not self._nodaemon: + sys.exit('Daemons cannot log to stdout, exiting!') + logFile = sys.stdout + elif self._nodaemon and not self._logfilename: + logFile = sys.stdout + else: + if not self._logfilename: + self._logfilename = 'twistd.log' + logFile = logfile.LogFile.fromFullPath(self._logfilename) + try: + import signal + except ImportError: + pass + else: + # Override if signal is set to None or SIG_DFL (0) + if not signal.getsignal(signal.SIGUSR1): + def rotateLog(signal, frame): + from twisted.internet import reactor + reactor.callFromThread(logFile.rotate) + signal.signal(signal.SIGUSR1, rotateLog) + return logger.textFileLogObserver(logFile) + + + +def launchWithName(name): + if name and name != sys.argv[0]: + exe = os.path.realpath(sys.executable) + log.msg('Changing process name to ' + name) + os.execv(exe, [name, sys.argv[0], '--originalname'] + sys.argv[1:]) + + + +class UnixApplicationRunner(app.ApplicationRunner): + """ + An ApplicationRunner which does Unix-specific things, like fork, + shed privileges, and maintain a PID file. + """ + loggerFactory = UnixAppLogger + + def preApplication(self): + """ + Do pre-application-creation setup. + """ + checkPID(self.config['pidfile']) + self.config['nodaemon'] = (self.config['nodaemon'] + or self.config['debug']) + self.oldstdout = sys.stdout + self.oldstderr = sys.stderr + + + def _formatChildException(self, exception): + """ + Format the C{exception} in preparation for writing to the + status pipe. This does the right thing on Python 2 if the + exception's message is Unicode, and in all cases limits the + length of the message afte* encoding to 100 bytes. + + This means the returned message may be truncated in the middle + of a unicode escape. + + @type exception: L{Exception} + @param exception: The exception to format. + + @return: The formatted message, suitable for writing to the + status pipe. + @rtype: L{bytes} + """ + # On Python 2 this will encode Unicode messages with the ascii + # codec and the backslashreplace error handler. + exceptionLine = traceback.format_exception_only(exception.__class__, + exception)[-1] + # remove the trailing newline + formattedMessage = '1 {}'.format(exceptionLine.strip()) + # On Python 3, encode the message the same way Python 2's + # format_exception_only does + if _PY3: + formattedMessage = formattedMessage.encode('ascii', + 'backslashreplace') + # By this point, the message has been encoded, if appropriate, + # with backslashreplace on both Python 2 and Python 3. + # Truncating the encoded message won't make it completely + # unreadable, and the reader should print out the repr of the + # message it receives anyway. What it will do, however, is + # ensure that only 100 bytes are written to the status pipe, + # ensuring that the child doesn't block because the pipe's + # full. This assumes PIPE_BUF > 100! + return formattedMessage[:100] + + + def postApplication(self): + """ + To be called after the application is created: start the application + and run the reactor. After the reactor stops, clean up PID files and + such. + """ + try: + self.startApplication(self.application) + except Exception as ex: + statusPipe = self.config.get("statusPipe", None) + if statusPipe is not None: + message = self._formatChildException(ex) + untilConcludes(os.write, statusPipe, message) + untilConcludes(os.close, statusPipe) + self.removePID(self.config['pidfile']) + raise + else: + statusPipe = self.config.get("statusPipe", None) + if statusPipe is not None: + untilConcludes(os.write, statusPipe, b"0") + untilConcludes(os.close, statusPipe) + self.startReactor(None, self.oldstdout, self.oldstderr) + self.removePID(self.config['pidfile']) + + + def removePID(self, pidfile): + """ + Remove the specified PID file, if possible. Errors are logged, not + raised. + + @type pidfile: C{str} + @param pidfile: The path to the PID tracking file. + """ + if not pidfile: + return + try: + os.unlink(pidfile) + except OSError as e: + if e.errno == errno.EACCES or e.errno == errno.EPERM: + log.msg("Warning: No permission to delete pid file") + else: + log.err(e, "Failed to unlink PID file:") + except: + log.err(None, "Failed to unlink PID file:") + + + def setupEnvironment(self, chroot, rundir, nodaemon, umask, pidfile): + """ + Set the filesystem root, the working directory, and daemonize. + + @type chroot: C{str} or L{None} + @param chroot: If not None, a path to use as the filesystem root (using + L{os.chroot}). + + @type rundir: C{str} + @param rundir: The path to set as the working directory. + + @type nodaemon: C{bool} + @param nodaemon: A flag which, if set, indicates that daemonization + should not be done. + + @type umask: C{int} or L{None} + @param umask: The value to which to change the process umask. + + @type pidfile: C{str} or L{None} + @param pidfile: If not L{None}, the path to a file into which to put + the PID of this process. + """ + daemon = not nodaemon + + if chroot is not None: + os.chroot(chroot) + if rundir == '.': + rundir = '/' + os.chdir(rundir) + if daemon and umask is None: + umask = 0o077 + if umask is not None: + os.umask(umask) + if daemon: + from twisted.internet import reactor + self.config["statusPipe"] = self.daemonize(reactor) + if pidfile: + with open(pidfile, 'wb') as f: + f.write(intToBytes(os.getpid())) + + + def daemonize(self, reactor): + """ + Daemonizes the application on Unix. This is done by the usual double + forking approach. + + @see: U{http://code.activestate.com/recipes/278731/} + @see: W. Richard Stevens, + "Advanced Programming in the Unix Environment", + 1992, Addison-Wesley, ISBN 0-201-56317-7 + + @param reactor: The reactor in use. If it provides + L{IReactorDaemonize}, its daemonization-related callbacks will be + invoked. + + @return: A writable pipe to be used to report errors. + @rtype: C{int} + """ + # If the reactor requires hooks to be called for daemonization, call + # them. Currently the only reactor which provides/needs that is + # KQueueReactor. + if IReactorDaemonize.providedBy(reactor): + reactor.beforeDaemonize() + r, w = os.pipe() + if os.fork(): # launch child and... + code = self._waitForStart(r) + os.close(r) + os._exit(code) # kill off parent + os.setsid() + if os.fork(): # launch child and... + os._exit(0) # kill off parent again. + null = os.open('/dev/null', os.O_RDWR) + for i in range(3): + try: + os.dup2(null, i) + except OSError as e: + if e.errno != errno.EBADF: + raise + os.close(null) + + if IReactorDaemonize.providedBy(reactor): + reactor.afterDaemonize() + + return w + + + def _waitForStart(self, readPipe): + """ + Wait for the daemonization success. + + @param readPipe: file descriptor to read start information from. + @type readPipe: C{int} + + @return: code to be passed to C{os._exit}: 0 for success, 1 for error. + @rtype: C{int} + """ + data = untilConcludes(os.read, readPipe, 100) + dataRepr = _bytesRepr(data[2:]) + if data != b"0": + msg = ("An error has occurred: {}\nPlease look at log " + "file for more information.\n".format(dataRepr)) + untilConcludes(sys.__stderr__.write, msg) + return 1 + return 0 + + + def shedPrivileges(self, euid, uid, gid): + """ + Change the UID and GID or the EUID and EGID of this process. + + @type euid: C{bool} + @param euid: A flag which, if set, indicates that only the I{effective} + UID and GID should be set. + + @type uid: C{int} or L{None} + @param uid: If not L{None}, the UID to which to switch. + + @type gid: C{int} or L{None} + @param gid: If not L{None}, the GID to which to switch. + """ + if uid is not None or gid is not None: + extra = euid and 'e' or '' + desc = '{}uid/{}gid {}/{}'.format(extra, extra, uid, gid) + try: + switchUID(uid, gid, euid) + except OSError as e: + log.msg('failed to set {}: {} (are you root?) -- ' + 'exiting.'.format(desc, e)) + sys.exit(1) + else: + log.msg('set {}'.format(desc)) + + + def startApplication(self, application): + """ + Configure global process state based on the given application and run + the application. + + @param application: An object which can be adapted to + L{service.IProcess} and L{service.IService}. + """ + process = service.IProcess(application) + if not self.config['originalname']: + launchWithName(process.processName) + self.setupEnvironment( + self.config['chroot'], self.config['rundir'], + self.config['nodaemon'], self.config['umask'], + self.config['pidfile']) + + service.IService(application).privilegedStartService() + + uid, gid = self.config['uid'], self.config['gid'] + if uid is None: + uid = process.uid + if gid is None: + gid = process.gid + if uid is not None and gid is None: + gid = pwd.getpwuid(uid).pw_gid + + self.shedPrivileges(self.config['euid'], uid, gid) + app.startApplication(application, not self.config['no_save']) diff --git a/contrib/python/Twisted/py2/twisted/scripts/_twistw.py b/contrib/python/Twisted/py2/twisted/scripts/_twistw.py new file mode 100644 index 00000000000..24db4e8a758 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/scripts/_twistw.py @@ -0,0 +1,54 @@ +# -*- test-case-name: twisted.test.test_twistd -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import print_function + +from twisted.python import log +from twisted.application import app, service, internet +from twisted import copyright +import sys, os + + + +class ServerOptions(app.ServerOptions): + synopsis = "Usage: twistd [options]" + + optFlags = [['nodaemon','n', "(for backwards compatibility)."], + ] + + def opt_version(self): + """ + Print version information and exit. + """ + print('twistd (the Twisted Windows runner) {}'.format(copyright.version), + file=self.stdout) + print(copyright.copyright, file=self.stdout) + sys.exit() + + + +class WindowsApplicationRunner(app.ApplicationRunner): + """ + An ApplicationRunner which avoids unix-specific things. No + forking, no PID files, no privileges. + """ + + def preApplication(self): + """ + Do pre-application-creation setup. + """ + self.oldstdout = sys.stdout + self.oldstderr = sys.stderr + os.chdir(self.config['rundir']) + + + def postApplication(self): + """ + Start the application and run the reactor. + """ + service.IService(self.application).privilegedStartService() + app.startApplication(self.application, not self.config['no_save']) + app.startApplication(internet.TimerService(0.1, lambda:None), 0) + self.startReactor(None, self.oldstdout, self.oldstderr) + log.msg("Server Shut Down.") diff --git a/contrib/python/Twisted/py2/twisted/scripts/htmlizer.py b/contrib/python/Twisted/py2/twisted/scripts/htmlizer.py new file mode 100644 index 00000000000..ccc1419495f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/scripts/htmlizer.py @@ -0,0 +1,74 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + +""" +HTML pretty-printing for Python source code. +""" + +from __future__ import print_function + +__version__ = '$Revision: 1.8 $'[11:-2] + +from twisted.python import htmlizer, usage +from twisted import copyright + +import os, sys + +header = ''' +%(title)s + +%(alternate)s +%(stylesheet)s + + +''' +footer = """""" + +styleLink = '' +alternateLink = '' + +class Options(usage.Options): + synopsis = """%s [options] source.py + """ % ( + os.path.basename(sys.argv[0]),) + + optParameters = [ + ('stylesheet', 's', None, "URL of stylesheet to link to."), + ] + + compData = usage.Completions( + extraActions=[usage.CompleteFiles('*.py', descr='source python file')] + ) + + + def parseArgs(self, filename): + self['filename'] = filename + + + +def run(): + options = Options() + try: + options.parseOptions() + except usage.UsageError as e: + print(str(e)) + sys.exit(1) + filename = options['filename'] + if options.get('stylesheet') is not None: + stylesheet = styleLink % (options['stylesheet'],) + else: + stylesheet = '' + + with open(filename + '.html', 'wb') as output: + outHeader = (header % { + 'title': filename, + 'generator': 'htmlizer/%s' % (copyright.longversion,), + 'alternate': alternateLink % {'source': filename}, + 'stylesheet': stylesheet + }) + output.write(outHeader.encode("utf-8")) + with open(filename, 'rb') as f: + htmlizer.filter(f, output, htmlizer.SmallerHTMLWriter) + output.write(footer.encode("utf-8")) diff --git a/contrib/python/Twisted/py2/twisted/scripts/trial.py b/contrib/python/Twisted/py2/twisted/scripts/trial.py new file mode 100644 index 00000000000..6c6e80aa167 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/scripts/trial.py @@ -0,0 +1,627 @@ +# -*- test-case-name: twisted.trial.test.test_script -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division, print_function + +import gc +import inspect +import os +import pdb +import random +import sys +import time +import warnings + +from twisted.internet import defer +from twisted.application import app +from twisted.python import usage, reflect, failure +from twisted.python.filepath import FilePath +from twisted.python.reflect import namedModule +from twisted.python.compat import long +from twisted import plugin +from twisted.trial import runner, itrial, reporter + + +# Yea, this is stupid. Leave it for command-line compatibility for a +# while, though. +TBFORMAT_MAP = { + 'plain': 'default', + 'default': 'default', + 'emacs': 'brief', + 'brief': 'brief', + 'cgitb': 'verbose', + 'verbose': 'verbose' + } + + +def _parseLocalVariables(line): + """ + Accepts a single line in Emacs local variable declaration format and + returns a dict of all the variables {name: value}. + Raises ValueError if 'line' is in the wrong format. + + See http://www.gnu.org/software/emacs/manual/html_node/File-Variables.html + """ + paren = '-*-' + start = line.find(paren) + len(paren) + end = line.rfind(paren) + if start == -1 or end == -1: + raise ValueError("%r not a valid local variable declaration" % (line,)) + items = line[start:end].split(';') + localVars = {} + for item in items: + if len(item.strip()) == 0: + continue + split = item.split(':') + if len(split) != 2: + raise ValueError("%r contains invalid declaration %r" + % (line, item)) + localVars[split[0].strip()] = split[1].strip() + return localVars + + +def loadLocalVariables(filename): + """ + Accepts a filename and attempts to load the Emacs variable declarations + from that file, simulating what Emacs does. + + See http://www.gnu.org/software/emacs/manual/html_node/File-Variables.html + """ + with open(filename, "r") as f: + lines = [f.readline(), f.readline()] + for line in lines: + try: + return _parseLocalVariables(line) + except ValueError: + pass + return {} + + +def getTestModules(filename): + testCaseVar = loadLocalVariables(filename).get('test-case-name', None) + if testCaseVar is None: + return [] + return testCaseVar.split(',') + + +def isTestFile(filename): + """ + Returns true if 'filename' looks like a file containing unit tests. + False otherwise. Doesn't care whether filename exists. + """ + basename = os.path.basename(filename) + return (basename.startswith('test_') + and os.path.splitext(basename)[1] == ('.py')) + + +def _reporterAction(): + return usage.CompleteList([p.longOpt for p in + plugin.getPlugins(itrial.IReporter)]) + + +def _maybeFindSourceLine(testThing): + """ + Try to find the source line of the given test thing. + + @param testThing: the test item to attempt to inspect + @type testThing: an L{TestCase}, test method, or module, though only the + former two have a chance to succeed + @rtype: int + @return: the starting source line, or -1 if one couldn't be found + """ + + # an instance of L{TestCase} -- locate the test it will run + method = getattr(testThing, "_testMethodName", None) + if method is not None: + testThing = getattr(testThing, method) + + # If it's a function, we can get the line number even if the source file no + # longer exists + code = getattr(testThing, "__code__", None) + if code is not None: + return code.co_firstlineno + + try: + return inspect.getsourcelines(testThing)[1] + except (IOError, TypeError): + # either testThing is a module, which raised a TypeError, or the file + # couldn't be read + return -1 + + +# orders which can be passed to trial --order +_runOrders = { + "alphabetical" : ( + "alphabetical order for test methods, arbitrary order for test cases", + runner.name), + "toptobottom" : ( + "attempt to run test cases and methods in the order they were defined", + _maybeFindSourceLine), +} + + +def _checkKnownRunOrder(order): + """ + Check that the given order is a known test running order. + + Does nothing else, since looking up the appropriate callable to sort the + tests should be done when it actually will be used, as the default argument + will not be coerced by this function. + + @param order: one of the known orders in C{_runOrders} + @return: the order unmodified + """ + if order not in _runOrders: + raise usage.UsageError( + "--order must be one of: %s. See --help-orders for details" % + (", ".join(repr(order) for order in _runOrders),)) + return order + + + +class _BasicOptions(object): + """ + Basic options shared between trial and its local workers. + """ + longdesc = ("trial loads and executes a suite of unit tests, obtained " + "from modules, packages and files listed on the command line.") + + optFlags = [["help", "h"], + ["no-recurse", "N", "Don't recurse into packages"], + ['help-orders', None, "Help on available test running orders"], + ['help-reporters', None, + "Help on available output plugins (reporters)"], + ["rterrors", "e", "realtime errors, print out tracebacks as " + "soon as they occur"], + ["unclean-warnings", None, + "Turn dirty reactor errors into warnings"], + ["force-gc", None, "Have Trial run gc.collect() before and " + "after each test case."], + ["exitfirst", "x", + "Exit after the first non-successful result (cannot be " + "specified along with --jobs)."], + ] + + optParameters = [ + ["order", "o", None, + "Specify what order to run test cases and methods. " + "See --help-orders for more info.", _checkKnownRunOrder], + ["random", "z", None, + "Run tests in random order using the specified seed"], + ['temp-directory', None, '_trial_temp', + 'Path to use as working directory for tests.'], + ['reporter', None, 'verbose', + 'The reporter to use for this test run. See --help-reporters for ' + 'more info.']] + + compData = usage.Completions( + optActions={"order": usage.CompleteList(_runOrders), + "reporter": _reporterAction, + "logfile": usage.CompleteFiles(descr="log file name"), + "random": usage.Completer(descr="random seed")}, + extraActions=[usage.CompleteFiles( + "*.py", descr="file | module | package | TestCase | testMethod", + repeat=True)], + ) + + fallbackReporter = reporter.TreeReporter + tracer = None + + def __init__(self): + self['tests'] = [] + usage.Options.__init__(self) + + def getSynopsis(self): + executableName = reflect.filenameToModuleName(sys.argv[0]) + + if executableName.endswith('.__main__'): + executableName = '{} -m {}'.format(os.path.basename(sys.executable), + executableName.replace('.__main__', '')) + + return """%s [options] [[file|package|module|TestCase|testmethod]...] + """ % (executableName,) + + def coverdir(self): + """ + Return a L{FilePath} representing the directory into which coverage + results should be written. + """ + coverdir = 'coverage' + result = FilePath(self['temp-directory']).child(coverdir) + print("Setting coverage directory to %s." % (result.path,)) + return result + + + # TODO: Some of the opt_* methods on this class have docstrings and some do + # not. This is mostly because usage.Options's currently will replace + # any intended output in optFlags and optParameters with the + # docstring. See #6427. When that is fixed, all methods should be + # given docstrings (and it should be verified that those with + # docstrings already have content suitable for printing as usage + # information). + + def opt_coverage(self): + """ + Generate coverage information in the coverage file in the + directory specified by the temp-directory option. + """ + import trace + self.tracer = trace.Trace(count=1, trace=0) + sys.settrace(self.tracer.globaltrace) + self['coverage'] = True + + + def opt_testmodule(self, filename): + """ + Filename to grep for test cases (-*- test-case-name). + """ + # If the filename passed to this parameter looks like a test module + # we just add that to the test suite. + # + # If not, we inspect it for an Emacs buffer local variable called + # 'test-case-name'. If that variable is declared, we try to add its + # value to the test suite as a module. + # + # This parameter allows automated processes (like Buildbot) to pass + # a list of files to Trial with the general expectation of "these files, + # whatever they are, will get tested" + if not os.path.isfile(filename): + sys.stderr.write("File %r doesn't exist\n" % (filename,)) + return + filename = os.path.abspath(filename) + if isTestFile(filename): + self['tests'].append(filename) + else: + self['tests'].extend(getTestModules(filename)) + + + def opt_spew(self): + """ + Print an insanely verbose log of everything that happens. Useful + when debugging freezes or locks in complex code. + """ + from twisted.python.util import spewer + sys.settrace(spewer) + + + def opt_help_orders(self): + synopsis = ("Trial can attempt to run test cases and their methods in " + "a few different orders. You can select any of the " + "following options using --order=.\n") + + print(synopsis) + for name, (description, _) in sorted(_runOrders.items()): + print(' ', name, '\t', description) + sys.exit(0) + + + def opt_help_reporters(self): + synopsis = ("Trial's output can be customized using plugins called " + "Reporters. You can\nselect any of the following " + "reporters using --reporter=\n") + print(synopsis) + for p in plugin.getPlugins(itrial.IReporter): + print(' ', p.longOpt, '\t', p.description) + sys.exit(0) + + + def opt_disablegc(self): + """ + Disable the garbage collector + """ + self["disablegc"] = True + gc.disable() + + + def opt_tbformat(self, opt): + """ + Specify the format to display tracebacks with. Valid formats are + 'plain', 'emacs', and 'cgitb' which uses the nicely verbose stdlib + cgitb.text function + """ + try: + self['tbformat'] = TBFORMAT_MAP[opt] + except KeyError: + raise usage.UsageError( + "tbformat must be 'plain', 'emacs', or 'cgitb'.") + + + def opt_recursionlimit(self, arg): + """ + see sys.setrecursionlimit() + """ + try: + sys.setrecursionlimit(int(arg)) + except (TypeError, ValueError): + raise usage.UsageError( + "argument to recursionlimit must be an integer") + else: + self["recursionlimit"] = int(arg) + + + def opt_random(self, option): + try: + self['random'] = long(option) + except ValueError: + raise usage.UsageError( + "Argument to --random must be a positive integer") + else: + if self['random'] < 0: + raise usage.UsageError( + "Argument to --random must be a positive integer") + elif self['random'] == 0: + self['random'] = long(time.time() * 100) + + + def opt_without_module(self, option): + """ + Fake the lack of the specified modules, separated with commas. + """ + self["without-module"] = option + for module in option.split(","): + if module in sys.modules: + warnings.warn("Module '%s' already imported, " + "disabling anyway." % (module,), + category=RuntimeWarning) + sys.modules[module] = None + + + def parseArgs(self, *args): + self['tests'].extend(args) + + + def _loadReporterByName(self, name): + for p in plugin.getPlugins(itrial.IReporter): + qual = "%s.%s" % (p.module, p.klass) + if p.longOpt == name: + return reflect.namedAny(qual) + raise usage.UsageError("Only pass names of Reporter plugins to " + "--reporter. See --help-reporters for " + "more info.") + + + def postOptions(self): + # Only load reporters now, as opposed to any earlier, to avoid letting + # application-defined plugins muck up reactor selecting by importing + # t.i.reactor and causing the default to be installed. + self['reporter'] = self._loadReporterByName(self['reporter']) + if 'tbformat' not in self: + self['tbformat'] = 'default' + if self['order'] is not None and self['random'] is not None: + raise usage.UsageError( + "You can't specify --random when using --order") + + + +class Options(_BasicOptions, usage.Options, app.ReactorSelectionMixin): + """ + Options to the trial command line tool. + + @ivar _workerFlags: List of flags which are accepted by trial distributed + workers. This is used by C{_getWorkerArguments} to build the command + line arguments. + @type _workerFlags: C{list} + + @ivar _workerParameters: List of parameter which are accepted by trial + distributed workers. This is used by C{_getWorkerArguments} to build + the command line arguments. + @type _workerParameters: C{list} + """ + + optFlags = [ + ["debug", "b", "Run tests in a debugger. If that debugger is " + "pdb, will load '.pdbrc' from current directory if it exists." + ], + ["debug-stacktraces", "B", "Report Deferred creation and " + "callback stack traces"], + ["nopm", None, "don't automatically jump into debugger for " + "postmorteming of exceptions"], + ["dry-run", 'n', "do everything but run the tests"], + ["profile", None, "Run tests under the Python profiler"], + ["until-failure", "u", "Repeat test until it fails"], + ] + + optParameters = [ + ["debugger", None, "pdb", "the fully qualified name of a debugger to " + "use if --debug is passed"], + ["logfile", "l", "test.log", "log file name"], + ["jobs", "j", None, "Number of local workers to run"] + ] + + compData = usage.Completions( + optActions = { + "tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]), + "reporter": _reporterAction, + }, + ) + + _workerFlags = ["disablegc", "force-gc", "coverage"] + _workerParameters = ["recursionlimit", "reactor", "without-module"] + + fallbackReporter = reporter.TreeReporter + extra = None + tracer = None + + + def opt_jobs(self, number): + """ + Number of local workers to run, a strictly positive integer. + """ + try: + number = int(number) + except ValueError: + raise usage.UsageError( + "Expecting integer argument to jobs, got '%s'" % number) + if number <= 0: + raise usage.UsageError( + "Argument to jobs must be a strictly positive integer") + self["jobs"] = number + + + def _getWorkerArguments(self): + """ + Return a list of options to pass to distributed workers. + """ + args = [] + for option in self._workerFlags: + if self.get(option) is not None: + if self[option]: + args.append("--%s" % (option,)) + for option in self._workerParameters: + if self.get(option) is not None: + args.extend(["--%s" % (option,), str(self[option])]) + return args + + + def postOptions(self): + _BasicOptions.postOptions(self) + if self['jobs']: + conflicts = ['debug', 'profile', 'debug-stacktraces', 'exitfirst'] + for option in conflicts: + if self[option]: + raise usage.UsageError( + "You can't specify --%s when using --jobs" % option) + if self['nopm']: + if not self['debug']: + raise usage.UsageError("You must specify --debug when using " + "--nopm ") + failure.DO_POST_MORTEM = False + + + +def _initialDebugSetup(config): + # do this part of debug setup first for easy debugging of import failures + if config['debug']: + failure.startDebugMode() + if config['debug'] or config['debug-stacktraces']: + defer.setDebugging(True) + + + +def _getSuite(config): + loader = _getLoader(config) + recurse = not config['no-recurse'] + return loader.loadByNames(config['tests'], recurse=recurse) + + + +def _getLoader(config): + loader = runner.TestLoader() + if config['random']: + randomer = random.Random() + randomer.seed(config['random']) + loader.sorter = lambda x : randomer.random() + print('Running tests shuffled with seed %d\n' % config['random']) + elif config['order']: + _, sorter = _runOrders[config['order']] + loader.sorter = sorter + if not config['until-failure']: + loader.suiteFactory = runner.DestructiveTestSuite + return loader + + +def _wrappedPdb(): + """ + Wrap an instance of C{pdb.Pdb} with readline support and load any .rcs. + + """ + + dbg = pdb.Pdb() + try: + namedModule('readline') + except ImportError: + print("readline module not available") + for path in ('.pdbrc', 'pdbrc'): + if os.path.exists(path): + try: + rcFile = open(path, 'r') + except IOError: + pass + else: + with rcFile: + dbg.rcLines.extend(rcFile.readlines()) + return dbg + + +class _DebuggerNotFound(Exception): + """ + A debugger import failed. + + Used to allow translating these errors into usage error messages. + + """ + + + +def _makeRunner(config): + """ + Return a trial runner class set up with the parameters extracted from + C{config}. + + @return: A trial runner instance. + @rtype: L{runner.TrialRunner} or C{DistTrialRunner} depending on the + configuration. + """ + cls = runner.TrialRunner + args = {'reporterFactory': config['reporter'], + 'tracebackFormat': config['tbformat'], + 'realTimeErrors': config['rterrors'], + 'uncleanWarnings': config['unclean-warnings'], + 'logfile': config['logfile'], + 'workingDirectory': config['temp-directory']} + if config['dry-run']: + args['mode'] = runner.TrialRunner.DRY_RUN + elif config['jobs']: + from twisted.trial._dist.disttrial import DistTrialRunner + cls = DistTrialRunner + args['workerNumber'] = config['jobs'] + args['workerArguments'] = config._getWorkerArguments() + else: + if config['debug']: + args['mode'] = runner.TrialRunner.DEBUG + debugger = config['debugger'] + + if debugger != 'pdb': + try: + args['debugger'] = reflect.namedAny(debugger) + except reflect.ModuleNotFound: + raise _DebuggerNotFound( + '%r debugger could not be found.' % (debugger,)) + else: + args['debugger'] = _wrappedPdb() + + args['exitFirst'] = config['exitfirst'] + args['profile'] = config['profile'] + args['forceGarbageCollection'] = config['force-gc'] + + return cls(**args) + + + +def run(): + if len(sys.argv) == 1: + sys.argv.append("--help") + config = Options() + try: + config.parseOptions() + except usage.error as ue: + raise SystemExit("%s: %s" % (sys.argv[0], ue)) + _initialDebugSetup(config) + + try: + trialRunner = _makeRunner(config) + except _DebuggerNotFound as e: + raise SystemExit('%s: %s' % (sys.argv[0], str(e))) + + suite = _getSuite(config) + if config['until-failure']: + test_result = trialRunner.runUntilFailure(suite) + else: + test_result = trialRunner.run(suite) + if config.tracer: + sys.settrace(None) + results = config.tracer.results() + results.write_results(show_missing=1, summary=False, + coverdir=config.coverdir().path) + sys.exit(not test_result.wasSuccessful()) diff --git a/contrib/python/Twisted/py2/twisted/scripts/twistd.py b/contrib/python/Twisted/py2/twisted/scripts/twistd.py new file mode 100644 index 00000000000..2c6439353cc --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/scripts/twistd.py @@ -0,0 +1,34 @@ +# -*- test-case-name: twisted.test.test_twistd -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +The Twisted Daemon: platform-independent interface. + +@author: Christopher Armstrong +""" + +from __future__ import absolute_import, division + +from twisted.application import app + +from twisted.python.runtime import platformType +if platformType == "win32": + from twisted.scripts._twistw import ServerOptions, \ + WindowsApplicationRunner as _SomeApplicationRunner +else: + from twisted.scripts._twistd_unix import ServerOptions, \ + UnixApplicationRunner as _SomeApplicationRunner + +def runApp(config): + runner = _SomeApplicationRunner(config) + runner.run() + if runner._exitSignal is not None: + app._exitWithSignal(runner._exitSignal) + + +def run(): + app.run(runApp, ServerOptions) + + +__all__ = ['run', 'runApp'] diff --git a/contrib/python/Twisted/py2/twisted/spread/__init__.py b/contrib/python/Twisted/py2/twisted/spread/__init__.py new file mode 100644 index 00000000000..ab3881055d7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/spread/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Spread: Spreadable (Distributed) Computing. + +@author: Glyph Lefkowitz +""" diff --git a/contrib/python/Twisted/py2/twisted/spread/banana.py b/contrib/python/Twisted/py2/twisted/spread/banana.py new file mode 100644 index 00000000000..feeb3027ac3 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/spread/banana.py @@ -0,0 +1,398 @@ +# -*- test-case-name: twisted.spread.test.test_banana -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Banana -- s-exp based protocol. + +Future Plans: This module is almost entirely stable. The same caveat applies +to it as applies to L{twisted.spread.jelly}, however. Read its future plans +for more details. + +@author: Glyph Lefkowitz +""" + +from __future__ import absolute_import, division + +import copy, struct +from io import BytesIO + +from twisted.internet import protocol +from twisted.persisted import styles +from twisted.python import log +from twisted.python.compat import iterbytes, long, _bytesChr as chr +from twisted.python.reflect import fullyQualifiedName + +class BananaError(Exception): + pass + +def int2b128(integer, stream): + if integer == 0: + stream(chr(0)) + return + assert integer > 0, "can only encode positive integers" + while integer: + stream(chr(integer & 0x7f)) + integer = integer >> 7 + + +def b1282int(st): + """ + Convert an integer represented as a base 128 string into an L{int} or + L{long}. + + @param st: The integer encoded in a byte string. + @type st: L{bytes} + + @return: The integer value extracted from the byte string. + @rtype: L{int} or L{long} + """ + e = 1 + i = 0 + for char in iterbytes(st): + n = ord(char) + i += (n * e) + e <<= 7 + return i + + +# delimiter characters. +LIST = chr(0x80) +INT = chr(0x81) +STRING = chr(0x82) +NEG = chr(0x83) +FLOAT = chr(0x84) +# "optional" -- these might be refused by a low-level implementation. +LONGINT = chr(0x85) +LONGNEG = chr(0x86) +# really optional; this is part of the 'pb' vocabulary +VOCAB = chr(0x87) + +HIGH_BIT_SET = chr(0x80) + +def setPrefixLimit(limit): + """ + Set the limit on the prefix length for all Banana connections + established after this call. + + The prefix length limit determines how many bytes of prefix a banana + decoder will allow before rejecting a potential object as too large. + + @type limit: L{int} + @param limit: The number of bytes of prefix for banana to allow when + decoding. + """ + global _PREFIX_LIMIT + _PREFIX_LIMIT = limit + +_PREFIX_LIMIT = None +setPrefixLimit(64) + +SIZE_LIMIT = 640 * 1024 # 640k is all you'll ever need :-) + +class Banana(protocol.Protocol, styles.Ephemeral): + """ + L{Banana} implements the I{Banana} s-expression protocol, client and + server. + + @ivar knownDialects: These are the profiles supported by this Banana + implementation. + @type knownDialects: L{list} of L{bytes} + """ + + # The specification calls these profiles but this implementation calls them + # dialects instead. + knownDialects = [b"pb", b"none"] + + prefixLimit = None + sizeLimit = SIZE_LIMIT + + def setPrefixLimit(self, limit): + """ + Set the prefix limit for decoding done by this protocol instance. + + @see: L{setPrefixLimit} + """ + self.prefixLimit = limit + self._smallestLongInt = -2 ** (limit * 7) + 1 + self._smallestInt = -2 ** 31 + self._largestInt = 2 ** 31 - 1 + self._largestLongInt = 2 ** (limit * 7) - 1 + + + def connectionReady(self): + """Surrogate for connectionMade + Called after protocol negotiation. + """ + + + def _selectDialect(self, dialect): + self.currentDialect = dialect + self.connectionReady() + + + def callExpressionReceived(self, obj): + if self.currentDialect: + self.expressionReceived(obj) + else: + # this is the first message we've received + if self.isClient: + # if I'm a client I have to respond + for serverVer in obj: + if serverVer in self.knownDialects: + self.sendEncoded(serverVer) + self._selectDialect(serverVer) + break + else: + # I can't speak any of those dialects. + log.msg("The client doesn't speak any of the protocols " + "offered by the server: disconnecting.") + self.transport.loseConnection() + else: + if obj in self.knownDialects: + self._selectDialect(obj) + else: + # the client just selected a protocol that I did not suggest. + log.msg("The client selected a protocol the server didn't " + "suggest and doesn't know: disconnecting.") + self.transport.loseConnection() + + + def connectionMade(self): + self.setPrefixLimit(_PREFIX_LIMIT) + self.currentDialect = None + if not self.isClient: + self.sendEncoded(self.knownDialects) + + + def gotItem(self, item): + l = self.listStack + if l: + l[-1][1].append(item) + else: + self.callExpressionReceived(item) + + buffer = b'' + + def dataReceived(self, chunk): + buffer = self.buffer + chunk + listStack = self.listStack + gotItem = self.gotItem + while buffer: + assert self.buffer != buffer, "This ain't right: %s %s" % (repr(self.buffer), repr(buffer)) + self.buffer = buffer + pos = 0 + for ch in iterbytes(buffer): + if ch >= HIGH_BIT_SET: + break + pos = pos + 1 + else: + if pos > self.prefixLimit: + raise BananaError("Security precaution: more than %d bytes of prefix" % (self.prefixLimit,)) + return + num = buffer[:pos] + typebyte = buffer[pos:pos+1] + rest = buffer[pos+1:] + if len(num) > self.prefixLimit: + raise BananaError("Security precaution: longer than %d bytes worth of prefix" % (self.prefixLimit,)) + if typebyte == LIST: + num = b1282int(num) + if num > SIZE_LIMIT: + raise BananaError("Security precaution: List too long.") + listStack.append((num, [])) + buffer = rest + elif typebyte == STRING: + num = b1282int(num) + if num > SIZE_LIMIT: + raise BananaError("Security precaution: String too long.") + if len(rest) >= num: + buffer = rest[num:] + gotItem(rest[:num]) + else: + return + elif typebyte == INT: + buffer = rest + num = b1282int(num) + gotItem(num) + elif typebyte == LONGINT: + buffer = rest + num = b1282int(num) + gotItem(num) + elif typebyte == LONGNEG: + buffer = rest + num = b1282int(num) + gotItem(-num) + elif typebyte == NEG: + buffer = rest + num = -b1282int(num) + gotItem(num) + elif typebyte == VOCAB: + buffer = rest + num = b1282int(num) + item = self.incomingVocabulary[num] + if self.currentDialect == b'pb': + # the sender issues VOCAB only for dialect pb + gotItem(item) + else: + raise NotImplementedError( + "Invalid item for pb protocol {0!r}".format(item)) + elif typebyte == FLOAT: + if len(rest) >= 8: + buffer = rest[8:] + gotItem(struct.unpack("!d", rest[:8])[0]) + else: + return + else: + raise NotImplementedError(("Invalid Type Byte %r" % (typebyte,))) + while listStack and (len(listStack[-1][1]) == listStack[-1][0]): + item = listStack.pop()[1] + gotItem(item) + self.buffer = b'' + + + def expressionReceived(self, lst): + """Called when an expression (list, string, or int) is received. + """ + raise NotImplementedError() + + + outgoingVocabulary = { + # Jelly Data Types + b'None' : 1, + b'class' : 2, + b'dereference' : 3, + b'reference' : 4, + b'dictionary' : 5, + b'function' : 6, + b'instance' : 7, + b'list' : 8, + b'module' : 9, + b'persistent' : 10, + b'tuple' : 11, + b'unpersistable' : 12, + + # PB Data Types + b'copy' : 13, + b'cache' : 14, + b'cached' : 15, + b'remote' : 16, + b'local' : 17, + b'lcache' : 18, + + # PB Protocol Messages + b'version' : 19, + b'login' : 20, + b'password' : 21, + b'challenge' : 22, + b'logged_in' : 23, + b'not_logged_in' : 24, + b'cachemessage' : 25, + b'message' : 26, + b'answer' : 27, + b'error' : 28, + b'decref' : 29, + b'decache' : 30, + b'uncache' : 31, + } + + incomingVocabulary = {} + for k, v in outgoingVocabulary.items(): + incomingVocabulary[v] = k + + + def __init__(self, isClient=1): + self.listStack = [] + self.outgoingSymbols = copy.copy(self.outgoingVocabulary) + self.outgoingSymbolCount = 0 + self.isClient = isClient + + + def sendEncoded(self, obj): + """ + Send the encoded representation of the given object: + + @param obj: An object to encode and send. + + @raise BananaError: If the given object is not an instance of one of + the types supported by Banana. + + @return: L{None} + """ + encodeStream = BytesIO() + self._encode(obj, encodeStream.write) + value = encodeStream.getvalue() + self.transport.write(value) + + + def _encode(self, obj, write): + if isinstance(obj, (list, tuple)): + if len(obj) > SIZE_LIMIT: + raise BananaError( + "list/tuple is too long to send (%d)" % (len(obj),)) + int2b128(len(obj), write) + write(LIST) + for elem in obj: + self._encode(elem, write) + elif isinstance(obj, (int, long)): + if obj < self._smallestLongInt or obj > self._largestLongInt: + raise BananaError( + "int/long is too large to send (%d)" % (obj,)) + if obj < self._smallestInt: + int2b128(-obj, write) + write(LONGNEG) + elif obj < 0: + int2b128(-obj, write) + write(NEG) + elif obj <= self._largestInt: + int2b128(obj, write) + write(INT) + else: + int2b128(obj, write) + write(LONGINT) + elif isinstance(obj, float): + write(FLOAT) + write(struct.pack("!d", obj)) + elif isinstance(obj, bytes): + # TODO: an API for extending banana... + if self.currentDialect == b"pb" and obj in self.outgoingSymbols: + symbolID = self.outgoingSymbols[obj] + int2b128(symbolID, write) + write(VOCAB) + else: + if len(obj) > SIZE_LIMIT: + raise BananaError( + "byte string is too long to send (%d)" % (len(obj),)) + int2b128(len(obj), write) + write(STRING) + write(obj) + else: + raise BananaError("Banana cannot send {0} objects: {1!r}".format( + fullyQualifiedName(type(obj)), obj)) + + +# For use from the interactive interpreter +_i = Banana() +_i.connectionMade() +_i._selectDialect(b"none") + + +def encode(lst): + """Encode a list s-expression.""" + encodeStream = BytesIO() + _i.transport = encodeStream + _i.sendEncoded(lst) + return encodeStream.getvalue() + + +def decode(st): + """ + Decode a banana-encoded string. + """ + l = [] + _i.expressionReceived = l.append + try: + _i.dataReceived(st) + finally: + _i.buffer = b'' + del _i.expressionReceived + return l[0] diff --git a/contrib/python/Twisted/py2/twisted/spread/flavors.py b/contrib/python/Twisted/py2/twisted/spread/flavors.py new file mode 100644 index 00000000000..b776577011a --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/spread/flavors.py @@ -0,0 +1,642 @@ +# -*- test-case-name: twisted.spread.test.test_pb -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module represents flavors of remotely accessible objects. + +Currently this is only objects accessible through Perspective Broker, but will +hopefully encompass all forms of remote access which can emulate subsets of PB +(such as XMLRPC or SOAP). + +Future Plans: Optimization. Exploitation of new-style object model. +Optimizations to this module should not affect external-use semantics at all, +but may have a small impact on users who subclass and override methods. + +@author: Glyph Lefkowitz +""" + +from __future__ import absolute_import, division + +# NOTE: this module should NOT import pb; it is supposed to be a module which +# abstractly defines remotely accessible types. Many of these types expect to +# be serialized by Jelly, but they ought to be accessible through other +# mechanisms (like XMLRPC) + +import sys + +from zope.interface import implementer, Interface + +from twisted.python import log, reflect +from twisted.python.compat import _PY3, unicode, comparable, cmp +from .jelly import ( + setUnjellyableForClass, setUnjellyableForClassTree, + setUnjellyableFactoryForClass, unjellyableRegistry, Jellyable, Unjellyable, + setInstanceState, getInstanceState, _createBlank +) + +# compatibility +setCopierForClass = setUnjellyableForClass +setCopierForClassTree = setUnjellyableForClassTree +setFactoryForClass = setUnjellyableFactoryForClass +copyTags = unjellyableRegistry + +copy_atom = b"copy" +cache_atom = b"cache" +cached_atom = b"cached" +remote_atom = b"remote" + + +class NoSuchMethod(AttributeError): + """Raised if there is no such remote method""" + + +class IPBRoot(Interface): + """Factory for root Referenceable objects for PB servers.""" + + def rootObject(broker): + """Return root Referenceable for broker.""" + + +class Serializable(Jellyable): + """An object that can be passed remotely. + + I am a style of object which can be serialized by Perspective + Broker. Objects which wish to be referenceable or copied remotely + have to subclass Serializable. However, clients of Perspective + Broker will probably not want to directly subclass Serializable; the + Flavors of transferable objects are listed below. + + What it means to be \"Serializable\" is that an object can be + passed to or returned from a remote method. Certain basic types + (dictionaries, lists, tuples, numbers, strings) are serializable by + default; however, classes need to choose a specific serialization + style: L{Referenceable}, L{Viewable}, L{Copyable} or L{Cacheable}. + + You may also pass C{[lists, dictionaries, tuples]} of L{Serializable} + instances to or return them from remote methods, as many levels deep + as you like. + """ + + def processUniqueID(self): + """Return an ID which uniquely represents this object for this process. + + By default, this uses the 'id' builtin, but can be overridden to + indicate that two values are identity-equivalent (such as proxies + for the same object). + """ + + return id(self) + +class Referenceable(Serializable): + perspective = None + """I am an object sent remotely as a direct reference. + + When one of my subclasses is sent as an argument to or returned + from a remote method call, I will be serialized by default as a + direct reference. + + This means that the peer will be able to call methods on me; + a method call xxx() from my peer will be resolved to methods + of the name remote_xxx. + """ + + def remoteMessageReceived(self, broker, message, args, kw): + """A remote message has been received. Dispatch it appropriately. + + The default implementation is to dispatch to a method called + 'remote_messagename' and call it with the same arguments. + """ + args = broker.unserialize(args) + kw = broker.unserialize(kw) + # Need this to interoperate with Python 2 clients + # which may try to send use keywords where keys are of type + # bytes. + if [key for key in kw.keys() if isinstance(key, bytes)]: + kw = dict((k.decode('utf8'), v) for k, v in kw.items()) + + if not isinstance(message, str): + message = message.decode('utf8') + + method = getattr(self, "remote_%s" % message, None) + if method is None: + raise NoSuchMethod("No such method: remote_%s" % (message,)) + try: + state = method(*args, **kw) + except TypeError: + log.msg("%s didn't accept %s and %s" % (method, args, kw)) + raise + return broker.serialize(state, self.perspective) + + def jellyFor(self, jellier): + """(internal) + + Return a tuple which will be used as the s-expression to + serialize this to a peer. + """ + + return [b"remote", jellier.invoker.registerReference(self)] + + +@implementer(IPBRoot) +class Root(Referenceable): + """I provide a root object to L{pb.Broker}s for a L{pb.PBClientFactory} or + L{pb.PBServerFactory}. + + When a factory produces a L{pb.Broker}, it supplies that + L{pb.Broker} with an object named \"root\". That object is obtained + by calling my rootObject method. + """ + + def rootObject(self, broker): + """A factory is requesting to publish me as a root object. + + When a factory is sending me as the root object, this + method will be invoked to allow per-broker versions of an + object. By default I return myself. + """ + return self + + +class ViewPoint(Referenceable): + """ + I act as an indirect reference to an object accessed through a + L{pb.IPerspective}. + + Simply put, I combine an object with a perspective so that when a + peer calls methods on the object I refer to, the method will be + invoked with that perspective as a first argument, so that it can + know who is calling it. + + While L{Viewable} objects will be converted to ViewPoints by default + when they are returned from or sent as arguments to a remote + method, any object may be manually proxied as well. (XXX: Now that + this class is no longer named C{Proxy}, this is the only occurrence + of the term 'proxied' in this docstring, and may be unclear.) + + This can be useful when dealing with L{pb.IPerspective}s, L{Copyable}s, + and L{Cacheable}s. It is legal to implement a method as such on + a perspective:: + + | def perspective_getViewPointForOther(self, name): + | defr = self.service.getPerspectiveRequest(name) + | defr.addCallbacks(lambda x, self=self: ViewPoint(self, x), log.msg) + | return defr + + This will allow you to have references to Perspective objects in two + different ways. One is through the initial 'attach' call -- each + peer will have a L{pb.RemoteReference} to their perspective directly. The + other is through this method; each peer can get a L{pb.RemoteReference} to + all other perspectives in the service; but that L{pb.RemoteReference} will + be to a L{ViewPoint}, not directly to the object. + + The practical offshoot of this is that you can implement 2 varieties + of remotely callable methods on this Perspective; view_xxx and + C{perspective_xxx}. C{view_xxx} methods will follow the rules for + ViewPoint methods (see ViewPoint.L{remoteMessageReceived}), and + C{perspective_xxx} methods will follow the rules for Perspective + methods. + """ + + def __init__(self, perspective, object): + """Initialize me with a Perspective and an Object. + """ + self.perspective = perspective + self.object = object + + def processUniqueID(self): + """Return an ID unique to a proxy for this perspective+object combination. + """ + return (id(self.perspective), id(self.object)) + + def remoteMessageReceived(self, broker, message, args, kw): + """A remote message has been received. Dispatch it appropriately. + + The default implementation is to dispatch to a method called + 'C{view_messagename}' to my Object and call it on my object with + the same arguments, modified by inserting my Perspective as + the first argument. + """ + args = broker.unserialize(args, self.perspective) + kw = broker.unserialize(kw, self.perspective) + + if not isinstance(message, str): + message = message.decode('utf8') + + method = getattr(self.object, "view_%s" % message) + try: + state = method(*(self.perspective,)+args, **kw) + except TypeError: + log.msg("%s didn't accept %s and %s" % (method, args, kw)) + raise + rv = broker.serialize(state, self.perspective, method, args, kw) + return rv + + +class Viewable(Serializable): + """I will be converted to a L{ViewPoint} when passed to or returned from a remote method. + + The beginning of a peer's interaction with a PB Service is always + through a perspective. However, if a C{perspective_xxx} method returns + a Viewable, it will be serialized to the peer as a response to that + method. + """ + + def jellyFor(self, jellier): + """Serialize a L{ViewPoint} for me and the perspective of the given broker. + """ + return ViewPoint(jellier.invoker.serializingPerspective, self).jellyFor(jellier) + + + +class Copyable(Serializable): + """Subclass me to get copied each time you are returned from or passed to a remote method. + + When I am returned from or passed to a remote method call, I will be + converted into data via a set of callbacks (see my methods for more + info). That data will then be serialized using Jelly, and sent to + the peer. + + The peer will then look up the type to represent this with; see + L{RemoteCopy} for details. + """ + + def getStateToCopy(self): + """Gather state to send when I am serialized for a peer. + + I will default to returning self.__dict__. Override this to + customize this behavior. + """ + + return self.__dict__ + + def getStateToCopyFor(self, perspective): + """ + Gather state to send when I am serialized for a particular + perspective. + + I will default to calling L{getStateToCopy}. Override this to + customize this behavior. + """ + + return self.getStateToCopy() + + def getTypeToCopy(self): + """Determine what type tag to send for me. + + By default, send the string representation of my class + (package.module.Class); normally this is adequate, but + you may override this to change it. + """ + + return reflect.qual(self.__class__).encode('utf-8') + + def getTypeToCopyFor(self, perspective): + """Determine what type tag to send for me. + + By default, defer to self.L{getTypeToCopy}() normally this is + adequate, but you may override this to change it. + """ + + return self.getTypeToCopy() + + def jellyFor(self, jellier): + """Assemble type tag and state to copy for this broker. + + This will call L{getTypeToCopyFor} and L{getStateToCopy}, and + return an appropriate s-expression to represent me. + """ + + if jellier.invoker is None: + return getInstanceState(self, jellier) + p = jellier.invoker.serializingPerspective + t = self.getTypeToCopyFor(p) + state = self.getStateToCopyFor(p) + sxp = jellier.prepare(self) + sxp.extend([t, jellier.jelly(state)]) + return jellier.preserve(self, sxp) + + +class Cacheable(Copyable): + """A cached instance. + + This means that it's copied; but there is some logic to make sure + that it's only copied once. Additionally, when state is retrieved, + it is passed a "proto-reference" to the state as it will exist on + the client. + + XXX: The documentation for this class needs work, but it's the most + complex part of PB and it is inherently difficult to explain. + """ + + def getStateToCacheAndObserveFor(self, perspective, observer): + """ + Get state to cache on the client and client-cache reference + to observe locally. + + This is similar to getStateToCopyFor, but it additionally + passes in a reference to the client-side RemoteCache instance + that will be created when it is unserialized. This allows + Cacheable instances to keep their RemoteCaches up to date when + they change, such that no changes can occur between the point + at which the state is initially copied and the client receives + it that are not propagated. + """ + + return self.getStateToCopyFor(perspective) + + def jellyFor(self, jellier): + """Return an appropriate tuple to serialize me. + + Depending on whether this broker has cached me or not, this may + return either a full state or a reference to an existing cache. + """ + if jellier.invoker is None: + return getInstanceState(self, jellier) + luid = jellier.invoker.cachedRemotelyAs(self, 1) + if luid is None: + luid = jellier.invoker.cacheRemotely(self) + p = jellier.invoker.serializingPerspective + type_ = self.getTypeToCopyFor(p) + observer = RemoteCacheObserver(jellier.invoker, self, p) + state = self.getStateToCacheAndObserveFor(p, observer) + l = jellier.prepare(self) + jstate = jellier.jelly(state) + l.extend([type_, luid, jstate]) + return jellier.preserve(self, l) + else: + return cached_atom, luid + + def stoppedObserving(self, perspective, observer): + """This method is called when a client has stopped observing me. + + The 'observer' argument is the same as that passed in to + getStateToCacheAndObserveFor. + """ + + + +class RemoteCopy(Unjellyable): + """I am a remote copy of a Copyable object. + + When the state from a L{Copyable} object is received, an instance will + be created based on the copy tags table (see setUnjellyableForClass) and + sent the L{setCopyableState} message. I provide a reasonable default + implementation of that message; subclass me if you wish to serve as + a copier for remote data. + + NOTE: copiers are invoked with no arguments. Do not implement a + constructor which requires args in a subclass of L{RemoteCopy}! + """ + + def setCopyableState(self, state): + """I will be invoked with the state to copy locally. + + 'state' is the data returned from the remote object's + 'getStateToCopyFor' method, which will often be the remote + object's dictionary (or a filtered approximation of it depending + on my peer's perspective). + """ + if _PY3: + state = {x.decode('utf8') if isinstance(x, bytes) + else x:y for x,y in state.items()} + self.__dict__ = state + + def unjellyFor(self, unjellier, jellyList): + if unjellier.invoker is None: + return setInstanceState(self, unjellier, jellyList) + self.setCopyableState(unjellier.unjelly(jellyList[1])) + return self + + + +class RemoteCache(RemoteCopy, Serializable): + """A cache is a local representation of a remote L{Cacheable} object. + + This represents the last known state of this object. It may + also have methods invoked on it -- in order to update caches, + the cached class generates a L{pb.RemoteReference} to this object as + it is originally sent. + + Much like copy, I will be invoked with no arguments. Do not + implement a constructor that requires arguments in one of my + subclasses. + """ + + def remoteMessageReceived(self, broker, message, args, kw): + """A remote message has been received. Dispatch it appropriately. + + The default implementation is to dispatch to a method called + 'C{observe_messagename}' and call it on my with the same arguments. + """ + if not isinstance(message, str): + message = message.decode('utf8') + + args = broker.unserialize(args) + kw = broker.unserialize(kw) + method = getattr(self, "observe_%s" % message) + try: + state = method(*args, **kw) + except TypeError: + log.msg("%s didn't accept %s and %s" % (method, args, kw)) + raise + return broker.serialize(state, None, method, args, kw) + + def jellyFor(self, jellier): + """serialize me (only for the broker I'm for) as the original cached reference + """ + if jellier.invoker is None: + return getInstanceState(self, jellier) + assert jellier.invoker is self.broker, "You cannot exchange cached proxies between brokers." + return b'lcache', self.luid + + + def unjellyFor(self, unjellier, jellyList): + if unjellier.invoker is None: + return setInstanceState(self, unjellier, jellyList) + self.broker = unjellier.invoker + self.luid = jellyList[1] + borgCopy = self._borgify() + # XXX questionable whether this was a good design idea... + init = getattr(borgCopy, "__init__", None) + if init: + init() + unjellier.invoker.cacheLocally(jellyList[1], self) + borgCopy.setCopyableState(unjellier.unjelly(jellyList[2])) + # Might have changed due to setCopyableState method; we'll assume that + # it's bad form to do so afterwards. + self.__dict__ = borgCopy.__dict__ + # chomp, chomp -- some existing code uses "self.__dict__ =", some uses + # "__dict__.update". This is here in order to handle both cases. + self.broker = unjellier.invoker + self.luid = jellyList[1] + return borgCopy + +## def __really_del__(self): +## """Final finalization call, made after all remote references have been lost. +## """ + + def __cmp__(self, other): + """Compare me [to another RemoteCache. + """ + if isinstance(other, self.__class__): + return cmp(id(self.__dict__), id(other.__dict__)) + else: + return cmp(id(self.__dict__), other) + + def __hash__(self): + """Hash me. + """ + return int(id(self.__dict__) % sys.maxsize) + + broker = None + luid = None + + def __del__(self): + """Do distributed reference counting on finalize. + """ + try: + # log.msg( ' --- decache: %s %s' % (self, self.luid) ) + if self.broker: + self.broker.decCacheRef(self.luid) + except: + log.deferr() + + + def _borgify(self): + """ + Create a new object that shares its state (i.e. its C{__dict__}) and + type with this object, but does not share its identity. + + This is an instance of U{the Borg design pattern + } originally described by + Alex Martelli, but unlike the example given there, this is not a + replacement for a Singleton. Instead, it is for lifecycle tracking + (and distributed garbage collection). The purpose of these separate + objects is to have a separate object tracking each application-level + reference to the root L{RemoteCache} object being tracked by the + broker, and to have their C{__del__} methods be invoked. + + This may be achievable via a weak value dictionary to track the root + L{RemoteCache} instances instead, but this implementation strategy + predates the availability of weak references in Python. + + @return: The new instance. + @rtype: C{self.__class__} + """ + blank = _createBlank(self.__class__) + blank.__dict__ = self.__dict__ + return blank + + + +def unjellyCached(unjellier, unjellyList): + luid = unjellyList[1] + return unjellier.invoker.cachedLocallyAs(luid)._borgify() + +setUnjellyableForClass("cached", unjellyCached) + +def unjellyLCache(unjellier, unjellyList): + luid = unjellyList[1] + obj = unjellier.invoker.remotelyCachedForLUID(luid) + return obj + +setUnjellyableForClass("lcache", unjellyLCache) + +def unjellyLocal(unjellier, unjellyList): + obj = unjellier.invoker.localObjectForID(unjellyList[1]) + return obj + +setUnjellyableForClass("local", unjellyLocal) + +@comparable +class RemoteCacheMethod: + """A method on a reference to a L{RemoteCache}. + """ + + def __init__(self, name, broker, cached, perspective): + """(internal) initialize. + """ + self.name = name + self.broker = broker + self.perspective = perspective + self.cached = cached + + def __cmp__(self, other): + return cmp((self.name, self.broker, self.perspective, self.cached), other) + + def __hash__(self): + return hash((self.name, self.broker, self.perspective, self.cached)) + + def __call__(self, *args, **kw): + """(internal) action method. + """ + cacheID = self.broker.cachedRemotelyAs(self.cached) + if cacheID is None: + from pb import ProtocolError + raise ProtocolError("You can't call a cached method when the object hasn't been given to the peer yet.") + return self.broker._sendMessage(b'cache', self.perspective, cacheID, + self.name, args, kw) + + + +@comparable +class RemoteCacheObserver: + """I am a reverse-reference to the peer's L{RemoteCache}. + + I am generated automatically when a cache is serialized. I + represent a reference to the client's L{RemoteCache} object that + will represent a particular L{Cacheable}; I am the additional + object passed to getStateToCacheAndObserveFor. + """ + + def __init__(self, broker, cached, perspective): + """(internal) Initialize me. + + @param broker: a L{pb.Broker} instance. + + @param cached: a L{Cacheable} instance that this L{RemoteCacheObserver} + corresponds to. + + @param perspective: a reference to the perspective who is observing this. + """ + + self.broker = broker + self.cached = cached + self.perspective = perspective + + def __repr__(self): + return "" % ( + self.broker, self.cached, self.perspective, id(self)) + + def __hash__(self): + """Generate a hash unique to all L{RemoteCacheObserver}s for this broker/perspective/cached triplet + """ + + return ( (hash(self.broker) % 2**10) + + (hash(self.perspective) % 2**10) + + (hash(self.cached) % 2**10)) + + def __cmp__(self, other): + """Compare me to another L{RemoteCacheObserver}. + """ + + return cmp((self.broker, self.perspective, self.cached), other) + + def callRemote(self, _name, *args, **kw): + """(internal) action method. + """ + cacheID = self.broker.cachedRemotelyAs(self.cached) + if isinstance(_name, unicode): + _name = _name.encode("utf-8") + if cacheID is None: + from pb import ProtocolError + raise ProtocolError("You can't call a cached method when the " + "object hasn't been given to the peer yet.") + return self.broker._sendMessage(b'cache', self.perspective, cacheID, + _name, args, kw) + + def remoteMethod(self, key): + """Get a L{pb.RemoteMethod} for this key. + """ + return RemoteCacheMethod(key, self.broker, self.cached, self.perspective) diff --git a/contrib/python/Twisted/py2/twisted/spread/interfaces.py b/contrib/python/Twisted/py2/twisted/spread/interfaces.py new file mode 100644 index 00000000000..a7db3ff7c51 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/spread/interfaces.py @@ -0,0 +1,31 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Spread Interfaces. +""" + +from zope.interface import Interface + + +class IJellyable(Interface): + def jellyFor(jellier): + """ + Jelly myself for jellier. + """ + + + +class IUnjellyable(Interface): + def unjellyFor(jellier, jellyList): + """ + Unjelly myself for the jellier. + + @param jellier: A stateful object which exists for the lifetime of a + single call to L{unjelly}. + + @param jellyList: The C{list} which represents the jellied state of the + object to be unjellied. + + @return: The object which results from unjellying. + """ diff --git a/contrib/python/Twisted/py2/twisted/spread/jelly.py b/contrib/python/Twisted/py2/twisted/spread/jelly.py new file mode 100644 index 00000000000..39857e3b44c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/spread/jelly.py @@ -0,0 +1,1131 @@ +# -*- test-case-name: twisted.spread.test.test_jelly -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +S-expression-based persistence of python objects. + +It does something very much like L{Pickle}; however, pickle's main goal +seems to be efficiency (both in space and time); jelly's main goals are +security, human readability, and portability to other environments. + +This is how Jelly converts various objects to s-expressions. + +Boolean:: + True --> ['boolean', 'true'] + +Integer:: + 1 --> 1 + +List:: + [1, 2] --> ['list', 1, 2] + +String:: + \"hello\" --> \"hello\" + +Float:: + 2.3 --> 2.3 + +Dictionary:: + {'a': 1, 'b': 'c'} --> ['dictionary', ['b', 'c'], ['a', 1]] + +Module:: + UserString --> ['module', 'UserString'] + +Class:: + UserString.UserString --> ['class', ['module', 'UserString'], 'UserString'] + +Function:: + string.join --> ['function', 'join', ['module', 'string']] + +Instance: s is an instance of UserString.UserString, with a __dict__ +{'data': 'hello'}:: + [\"UserString.UserString\", ['dictionary', ['data', 'hello']]] + +Class Method: UserString.UserString.center:: + ['method', 'center', ['None'], ['class', ['module', 'UserString'], + 'UserString']] + +Instance Method: s.center, where s is an instance of UserString.UserString:: + ['method', 'center', ['instance', ['reference', 1, ['class', + ['module', 'UserString'], 'UserString']], ['dictionary', ['data', 'd']]], + ['dereference', 1]] + +The C{set} builtin and the C{sets.Set} class are serialized to the same +thing, and unserialized to C{set} if available, else to C{sets.Set}. It means +that there's a possibility of type switching in the serialization process. The +solution is to always use C{set}. + +The same rule applies for C{frozenset} and C{sets.ImmutableSet}. + +@author: Glyph Lefkowitz +""" + +# System Imports +import types +import warnings +import decimal +from functools import reduce +import copy +import datetime + +try: + from types import (ClassType as _OldStyleClass, + InstanceType as _OldStyleInstance) +except ImportError: + # On Python 3 and higher, ClassType and InstanceType + # are gone. Use an empty tuple to pass to isinstance() + # tests without throwing an exception. + _OldStyleClass = () + _OldStyleInstance = () + +_SetTypes = [set] +_ImmutableSetTypes = [frozenset] + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + try: + import sets as _sets + except ImportError: + # sets module is deprecated in Python 2.6, and gone in + # Python 3 + _sets = None + else: + _SetTypes.append(_sets.Set) + _ImmutableSetTypes.append(_sets.ImmutableSet) + +from zope.interface import implementer + +# Twisted Imports +from twisted.python.compat import unicode, long, nativeString +from twisted.python.reflect import namedObject, qual, namedAny +from twisted.persisted.crefutil import NotKnown, _Tuple, _InstanceMethod +from twisted.persisted.crefutil import _DictKeyAndValue, _Dereference +from twisted.persisted.crefutil import _Container + +from twisted.spread.interfaces import IJellyable, IUnjellyable + +from twisted.python.compat import _PY3 +from twisted.python.deprecate import deprecatedModuleAttribute +from incremental import Version + +DictTypes = (dict,) + +None_atom = b"None" # N +# code +class_atom = b"class" # c +module_atom = b"module" # m +function_atom = b"function" # f + +# references +dereference_atom = b'dereference' # D +persistent_atom = b'persistent' # p +reference_atom = b'reference' # r + +# mutable collections +dictionary_atom = b"dictionary" # d +list_atom = b'list' # l +set_atom = b'set' + +# immutable collections +# (assignment to __dict__ and __class__ still might go away!) +tuple_atom = b"tuple" # t +instance_atom = b'instance' # i +frozenset_atom = b'frozenset' + + +deprecatedModuleAttribute( + Version("Twisted", 15, 0, 0), + "instance_atom is unused within Twisted.", + "twisted.spread.jelly", "instance_atom") + +# errors +unpersistable_atom = b"unpersistable"# u +unjellyableRegistry = {} +unjellyableFactoryRegistry = {} + + + +def _createBlank(cls): + """ + Given an object, if that object is a type (or a legacy old-style class), + return a new, blank instance of that type which has not had C{__init__} + called on it. If the object is not a type, return C{None}. + + @param cls: The type (or class) to create an instance of. + @type cls: L{_OldStyleClass}, L{type}, or something else that cannot be + instantiated. + + @return: a new blank instance or L{None} if C{cls} is not a class or type. + """ + if isinstance(cls, type): + return cls.__new__(cls) + if not _PY3 and isinstance(cls, _OldStyleClass): + return _OldStyleInstance(cls) + + + +def _newInstance(cls, state): + """ + Make a new instance of a class without calling its __init__ method. + Supports both new- and old-style classes. + + @param state: A C{dict} used to update C{inst.__dict__} either directly or + via C{__setstate__}, if available. + + @return: A new instance of C{cls}. + """ + instance = _createBlank(cls) + def defaultSetter(state): + instance.__dict__ = state + setter = getattr(instance, "__setstate__", defaultSetter) + setter(state) + return instance + + + +def _maybeClass(classnamep): + isObject = isinstance(classnamep, type) + + if isObject or ((not _PY3) and isinstance(classnamep, _OldStyleClass)): + classnamep = qual(classnamep) + + if not isinstance(classnamep, bytes): + classnamep = classnamep.encode('utf-8') + + return classnamep + + + +def setUnjellyableForClass(classname, unjellyable): + """ + Set which local class will represent a remote type. + + If you have written a Copyable class that you expect your client to be + receiving, write a local "copy" class to represent it, then call:: + + jellier.setUnjellyableForClass('module.package.Class', MyCopier). + + Call this at the module level immediately after its class + definition. MyCopier should be a subclass of RemoteCopy. + + The classname may be a special tag returned by + 'Copyable.getTypeToCopyFor' rather than an actual classname. + + This call is also for cached classes, since there will be no + overlap. The rules are the same. + """ + + global unjellyableRegistry + classname = _maybeClass(classname) + unjellyableRegistry[classname] = unjellyable + globalSecurity.allowTypes(classname) + + + +def setUnjellyableFactoryForClass(classname, copyFactory): + """ + Set the factory to construct a remote instance of a type:: + + jellier.setUnjellyableFactoryForClass('module.package.Class', MyFactory) + + Call this at the module level immediately after its class definition. + C{copyFactory} should return an instance or subclass of + L{RemoteCopy}. + + Similar to L{setUnjellyableForClass} except it uses a factory instead + of creating an instance. + """ + + global unjellyableFactoryRegistry + classname = _maybeClass(classname) + unjellyableFactoryRegistry[classname] = copyFactory + globalSecurity.allowTypes(classname) + + + +def setUnjellyableForClassTree(module, baseClass, prefix=None): + """ + Set all classes in a module derived from C{baseClass} as copiers for + a corresponding remote class. + + When you have a hierarchy of Copyable (or Cacheable) classes on one + side, and a mirror structure of Copied (or RemoteCache) classes on the + other, use this to setUnjellyableForClass all your Copieds for the + Copyables. + + Each copyTag (the \"classname\" argument to getTypeToCopyFor, and + what the Copyable's getTypeToCopyFor returns) is formed from + adding a prefix to the Copied's class name. The prefix defaults + to module.__name__. If you wish the copy tag to consist of solely + the classname, pass the empty string \'\'. + + @param module: a module object from which to pull the Copied classes. + (passing sys.modules[__name__] might be useful) + + @param baseClass: the base class from which all your Copied classes derive. + + @param prefix: the string prefixed to classnames to form the + unjellyableRegistry. + """ + if prefix is None: + prefix = module.__name__ + + if prefix: + prefix = "%s." % prefix + + for name in dir(module): + loaded = getattr(module, name) + try: + yes = issubclass(loaded, baseClass) + except TypeError: + "It's not a class." + else: + if yes: + setUnjellyableForClass('%s%s' % (prefix, name), loaded) + + + +def getInstanceState(inst, jellier): + """ + Utility method to default to 'normal' state rules in serialization. + """ + if hasattr(inst, "__getstate__"): + state = inst.__getstate__() + else: + state = inst.__dict__ + sxp = jellier.prepare(inst) + sxp.extend([qual(inst.__class__).encode('utf-8'), jellier.jelly(state)]) + return jellier.preserve(inst, sxp) + + + +def setInstanceState(inst, unjellier, jellyList): + """ + Utility method to default to 'normal' state rules in unserialization. + """ + state = unjellier.unjelly(jellyList[1]) + if hasattr(inst, "__setstate__"): + inst.__setstate__(state) + else: + inst.__dict__ = state + return inst + + + +class Unpersistable: + """ + This is an instance of a class that comes back when something couldn't be + unpersisted. + """ + + def __init__(self, reason): + """ + Initialize an unpersistable object with a descriptive C{reason} string. + """ + self.reason = reason + + + def __repr__(self): + return "Unpersistable(%s)" % repr(self.reason) + + + +@implementer(IJellyable) +class Jellyable: + """ + Inherit from me to Jelly yourself directly with the `getStateFor' + convenience method. + """ + + def getStateFor(self, jellier): + return self.__dict__ + + + def jellyFor(self, jellier): + """ + @see: L{twisted.spread.interfaces.IJellyable.jellyFor} + """ + sxp = jellier.prepare(self) + sxp.extend([ + qual(self.__class__).encode('utf-8'), + jellier.jelly(self.getStateFor(jellier))]) + return jellier.preserve(self, sxp) + + + +@implementer(IUnjellyable) +class Unjellyable: + """ + Inherit from me to Unjelly yourself directly with the + C{setStateFor} convenience method. + """ + + def setStateFor(self, unjellier, state): + self.__dict__ = state + + + def unjellyFor(self, unjellier, jellyList): + """ + Perform the inverse operation of L{Jellyable.jellyFor}. + + @see: L{twisted.spread.interfaces.IUnjellyable.unjellyFor} + """ + state = unjellier.unjelly(jellyList[1]) + self.setStateFor(unjellier, state) + return self + + + +class _Jellier: + """ + (Internal) This class manages state for a call to jelly() + """ + + def __init__(self, taster, persistentStore, invoker): + """ + Initialize. + """ + self.taster = taster + # `preserved' is a dict of previously seen instances. + self.preserved = {} + # `cooked' is a dict of previously backreferenced instances to their + # `ref' lists. + self.cooked = {} + self.cooker = {} + self._ref_id = 1 + self.persistentStore = persistentStore + self.invoker = invoker + + + def _cook(self, object): + """ + (internal) Backreference an object. + + Notes on this method for the hapless future maintainer: If I've already + gone through the prepare/preserve cycle on the specified object (it is + being referenced after the serializer is \"done with\" it, e.g. this + reference is NOT circular), the copy-in-place of aList is relevant, + since the list being modified is the actual, pre-existing jelly + expression that was returned for that object. If not, it's technically + superfluous, since the value in self.preserved didn't need to be set, + but the invariant that self.preserved[id(object)] is a list is + convenient because that means we don't have to test and create it or + not create it here, creating fewer code-paths. that's why + self.preserved is always set to a list. + + Sorry that this code is so hard to follow, but Python objects are + tricky to persist correctly. -glyph + """ + aList = self.preserved[id(object)] + newList = copy.copy(aList) + # make a new reference ID + refid = self._ref_id + self._ref_id = self._ref_id + 1 + # replace the old list in-place, so that we don't have to track the + # previous reference to it. + aList[:] = [reference_atom, refid, newList] + self.cooked[id(object)] = [dereference_atom, refid] + return aList + + + def prepare(self, object): + """ + (internal) Create a list for persisting an object to. This will allow + backreferences to be made internal to the object. (circular + references). + + The reason this needs to happen is that we don't generate an ID for + every object, so we won't necessarily know which ID the object will + have in the future. When it is 'cooked' ( see _cook ), it will be + assigned an ID, and the temporary placeholder list created here will be + modified in-place to create an expression that gives this object an ID: + [reference id# [object-jelly]]. + """ + + # create a placeholder list to be preserved + self.preserved[id(object)] = [] + # keep a reference to this object around, so it doesn't disappear! + # (This isn't always necessary, but for cases where the objects are + # dynamically generated by __getstate__ or getStateToCopyFor calls, it + # is; id() will return the same value for a different object if it gets + # garbage collected. This may be optimized later.) + self.cooker[id(object)] = object + return [] + + + def preserve(self, object, sexp): + """ + (internal) Mark an object's persistent list for later referral. + """ + # if I've been cooked in the meanwhile, + if id(object) in self.cooked: + # replace the placeholder empty list with the real one + self.preserved[id(object)][2] = sexp + # but give this one back. + sexp = self.preserved[id(object)] + else: + self.preserved[id(object)] = sexp + return sexp + + constantTypes = {bytes: 1, unicode: 1, int: 1, float: 1, long: 1} + + + def _checkMutable(self,obj): + objId = id(obj) + if objId in self.cooked: + return self.cooked[objId] + if objId in self.preserved: + self._cook(obj) + return self.cooked[objId] + + + def jelly(self, obj): + if isinstance(obj, Jellyable): + preRef = self._checkMutable(obj) + if preRef: + return preRef + return obj.jellyFor(self) + objType = type(obj) + if self.taster.isTypeAllowed(qual(objType).encode('utf-8')): + # "Immutable" Types + if ((objType is bytes) or + (objType is int) or + (objType is long) or + (objType is float)): + return obj + elif objType is types.MethodType: + aSelf = obj.__self__ if _PY3 else obj.im_self + aFunc = obj.__func__ if _PY3 else obj.im_func + aClass = aSelf.__class__ if _PY3 else obj.im_class + return [b"method", aFunc.__name__, self.jelly(aSelf), + self.jelly(aClass)] + elif objType is unicode: + return [b'unicode', obj.encode('UTF-8')] + elif objType is type(None): + return [b'None'] + elif objType is types.FunctionType: + return [b'function', obj.__module__ + '.' + + (obj.__qualname__ if _PY3 else obj.__name__)] + elif objType is types.ModuleType: + return [b'module', obj.__name__] + elif objType is bool: + return [b'boolean', obj and b'true' or b'false'] + elif objType is datetime.datetime: + if obj.tzinfo: + raise NotImplementedError( + "Currently can't jelly datetime objects with tzinfo") + return [b'datetime', ' '.join([unicode(x) for x in ( + obj.year, obj.month, obj.day, obj.hour, + obj.minute, obj.second, obj.microsecond)] + ).encode('utf-8')] + elif objType is datetime.time: + if obj.tzinfo: + raise NotImplementedError( + "Currently can't jelly datetime objects with tzinfo") + return [b'time', '%s %s %s %s' % (obj.hour, obj.minute, + obj.second, obj.microsecond)] + elif objType is datetime.date: + return [b'date', '%s %s %s' % (obj.year, obj.month, obj.day)] + elif objType is datetime.timedelta: + return [b'timedelta', '%s %s %s' % (obj.days, obj.seconds, + obj.microseconds)] + elif issubclass(objType, (type, _OldStyleClass)): + return [b'class', qual(obj).encode('utf-8')] + elif objType is decimal.Decimal: + return self.jelly_decimal(obj) + else: + preRef = self._checkMutable(obj) + if preRef: + return preRef + # "Mutable" Types + sxp = self.prepare(obj) + if objType is list: + sxp.extend(self._jellyIterable(list_atom, obj)) + elif objType is tuple: + sxp.extend(self._jellyIterable(tuple_atom, obj)) + elif objType in DictTypes: + sxp.append(dictionary_atom) + for key, val in obj.items(): + sxp.append([self.jelly(key), self.jelly(val)]) + elif objType in _SetTypes: + sxp.extend(self._jellyIterable(set_atom, obj)) + elif objType in _ImmutableSetTypes: + sxp.extend(self._jellyIterable(frozenset_atom, obj)) + else: + className = qual(obj.__class__).encode('utf-8') + persistent = None + if self.persistentStore: + persistent = self.persistentStore(obj, self) + if persistent is not None: + sxp.append(persistent_atom) + sxp.append(persistent) + elif self.taster.isClassAllowed(obj.__class__): + sxp.append(className) + if hasattr(obj, "__getstate__"): + state = obj.__getstate__() + else: + state = obj.__dict__ + sxp.append(self.jelly(state)) + else: + self.unpersistable( + "instance of class %s deemed insecure" % + qual(obj.__class__), sxp) + return self.preserve(obj, sxp) + else: + if objType is _OldStyleInstance: + raise InsecureJelly("Class not allowed for instance: %s %s" % + (obj.__class__, obj)) + raise InsecureJelly("Type not allowed for object: %s %s" % + (objType, obj)) + + + def _jellyIterable(self, atom, obj): + """ + Jelly an iterable object. + + @param atom: the identifier atom of the object. + @type atom: C{str} + + @param obj: any iterable object. + @type obj: C{iterable} + + @return: a generator of jellied data. + @rtype: C{generator} + """ + yield atom + for item in obj: + yield self.jelly(item) + + + def jelly_decimal(self, d): + """ + Jelly a decimal object. + + @param d: a decimal object to serialize. + @type d: C{decimal.Decimal} + + @return: jelly for the decimal object. + @rtype: C{list} + """ + sign, guts, exponent = d.as_tuple() + value = reduce(lambda left, right: left * 10 + right, guts) + if sign: + value = -value + return [b'decimal', value, exponent] + + + def unpersistable(self, reason, sxp=None): + """ + (internal) Returns an sexp: (unpersistable "reason"). Utility method + for making note that a particular object could not be serialized. + """ + if sxp is None: + sxp = [] + sxp.append(unpersistable_atom) + if isinstance(reason, unicode): + reason = reason.encode("utf-8") + sxp.append(reason) + return sxp + + + +class _Unjellier: + + def __init__(self, taster, persistentLoad, invoker): + self.taster = taster + self.persistentLoad = persistentLoad + self.references = {} + self.postCallbacks = [] + self.invoker = invoker + + + def unjellyFull(self, obj): + o = self.unjelly(obj) + for m in self.postCallbacks: + m() + return o + + + def _maybePostUnjelly(self, unjellied): + """ + If the given object has support for the C{postUnjelly} hook, set it up + to be called at the end of deserialization. + + @param unjellied: an object that has already been unjellied. + + @return: C{unjellied} + """ + if hasattr(unjellied, 'postUnjelly'): + self.postCallbacks.append(unjellied.postUnjelly) + return unjellied + + + def unjelly(self, obj): + if type(obj) is not list: + return obj + jelTypeBytes = obj[0] + if not self.taster.isTypeAllowed(jelTypeBytes): + raise InsecureJelly(jelTypeBytes) + regClass = unjellyableRegistry.get(jelTypeBytes) + if regClass is not None: + method = getattr(_createBlank(regClass), "unjellyFor", regClass) + return self._maybePostUnjelly(method(self, obj)) + regFactory = unjellyableFactoryRegistry.get(jelTypeBytes) + if regFactory is not None: + return self._maybePostUnjelly(regFactory(self.unjelly(obj[1]))) + + jelTypeText = nativeString(jelTypeBytes) + thunk = getattr(self, '_unjelly_%s' % jelTypeText, None) + if thunk is not None: + return thunk(obj[1:]) + else: + nameSplit = jelTypeText.split('.') + modName = '.'.join(nameSplit[:-1]) + if not self.taster.isModuleAllowed(modName): + raise InsecureJelly( + "Module %s not allowed (in type %s)." % (modName, jelTypeText)) + clz = namedObject(jelTypeText) + if not self.taster.isClassAllowed(clz): + raise InsecureJelly("Class %s not allowed." % jelTypeText) + return self._genericUnjelly(clz, obj[1]) + + + def _genericUnjelly(self, cls, state): + """ + Unjelly a type for which no specific unjellier is registered, but which + is nonetheless allowed. + + @param cls: the class of the instance we are unjellying. + @type cls: L{_OldStyleClass} or L{type} + + @param state: The jellied representation of the object's state; its + C{__dict__} unless it has a C{__setstate__} that takes something + else. + @type state: L{list} + + @return: the new, unjellied instance. + """ + return self._maybePostUnjelly(_newInstance(cls, self.unjelly(state))) + + + def _unjelly_None(self, exp): + return None + + + def _unjelly_unicode(self, exp): + return unicode(exp[0], "UTF-8") + + + def _unjelly_decimal(self, exp): + """ + Unjelly decimal objects. + """ + value = exp[0] + exponent = exp[1] + if value < 0: + sign = 1 + else: + sign = 0 + guts = decimal.Decimal(value).as_tuple()[1] + return decimal.Decimal((sign, guts, exponent)) + + + def _unjelly_boolean(self, exp): + if bool: + assert exp[0] in (b'true', b'false') + return exp[0] == b'true' + else: + return Unpersistable("Could not unpersist boolean: %s" % (exp[0],)) + + + def _unjelly_datetime(self, exp): + return datetime.datetime(*map(int, exp[0].split())) + + + def _unjelly_date(self, exp): + return datetime.date(*map(int, exp[0].split())) + + + def _unjelly_time(self, exp): + return datetime.time(*map(int, exp[0].split())) + + + def _unjelly_timedelta(self, exp): + days, seconds, microseconds = map(int, exp[0].split()) + return datetime.timedelta( + days=days, seconds=seconds, microseconds=microseconds) + + + def unjellyInto(self, obj, loc, jel): + o = self.unjelly(jel) + if isinstance(o, NotKnown): + o.addDependant(obj, loc) + obj[loc] = o + return o + + + def _unjelly_dereference(self, lst): + refid = lst[0] + x = self.references.get(refid) + if x is not None: + return x + der = _Dereference(refid) + self.references[refid] = der + return der + + + def _unjelly_reference(self, lst): + refid = lst[0] + exp = lst[1] + o = self.unjelly(exp) + ref = self.references.get(refid) + if (ref is None): + self.references[refid] = o + elif isinstance(ref, NotKnown): + ref.resolveDependants(o) + self.references[refid] = o + else: + assert 0, "Multiple references with same ID!" + return o + + + def _unjelly_tuple(self, lst): + l = list(range(len(lst))) + finished = 1 + for elem in l: + if isinstance(self.unjellyInto(l, elem, lst[elem]), NotKnown): + finished = 0 + if finished: + return tuple(l) + else: + return _Tuple(l) + + + def _unjelly_list(self, lst): + l = list(range(len(lst))) + for elem in l: + self.unjellyInto(l, elem, lst[elem]) + return l + + + def _unjellySetOrFrozenset(self, lst, containerType): + """ + Helper method to unjelly set or frozenset. + + @param lst: the content of the set. + @type lst: C{list} + + @param containerType: the type of C{set} to use. + """ + l = list(range(len(lst))) + finished = True + for elem in l: + data = self.unjellyInto(l, elem, lst[elem]) + if isinstance(data, NotKnown): + finished = False + if not finished: + return _Container(l, containerType) + else: + return containerType(l) + + + def _unjelly_set(self, lst): + """ + Unjelly set using the C{set} builtin. + """ + return self._unjellySetOrFrozenset(lst, set) + + + def _unjelly_frozenset(self, lst): + """ + Unjelly frozenset using the C{frozenset} builtin. + """ + return self._unjellySetOrFrozenset(lst, frozenset) + + + def _unjelly_dictionary(self, lst): + d = {} + for k, v in lst: + kvd = _DictKeyAndValue(d) + self.unjellyInto(kvd, 0, k) + self.unjellyInto(kvd, 1, v) + return d + + + def _unjelly_module(self, rest): + moduleName = nativeString(rest[0]) + if type(moduleName) != str: + raise InsecureJelly( + "Attempted to unjelly a module with a non-string name.") + if not self.taster.isModuleAllowed(moduleName): + raise InsecureJelly( + "Attempted to unjelly module named %r" % (moduleName,)) + mod = __import__(moduleName, {}, {},"x") + return mod + + + def _unjelly_class(self, rest): + cname = nativeString(rest[0]) + clist = cname.split(nativeString('.')) + modName = nativeString('.').join(clist[:-1]) + if not self.taster.isModuleAllowed(modName): + raise InsecureJelly("module %s not allowed" % modName) + klaus = namedObject(cname) + objType = type(klaus) + if objType not in (_OldStyleClass, type): + raise InsecureJelly( + "class %r unjellied to something that isn't a class: %r" % ( + cname, klaus)) + if not self.taster.isClassAllowed(klaus): + raise InsecureJelly("class not allowed: %s" % qual(klaus)) + return klaus + + + def _unjelly_function(self, rest): + fname = nativeString(rest[0]) + modSplit = fname.split(nativeString('.')) + modName = nativeString('.').join(modSplit[:-1]) + if not self.taster.isModuleAllowed(modName): + raise InsecureJelly("Module not allowed: %s" % modName) + # XXX do I need an isFunctionAllowed? + function = namedAny(fname) + return function + + + def _unjelly_persistent(self, rest): + if self.persistentLoad: + pload = self.persistentLoad(rest[0], self) + return pload + else: + return Unpersistable("Persistent callback not found") + + + def _unjelly_instance(self, rest): + """ + (internal) Unjelly an instance. + + Called to handle the deprecated I{instance} token. + + @param rest: The s-expression representing the instance. + + @return: The unjellied instance. + """ + warnings.warn_explicit( + "Unjelly support for the instance atom is deprecated since " + "Twisted 15.0.0. Upgrade peer for modern instance support.", + category=DeprecationWarning, filename="", lineno=0) + + clz = self.unjelly(rest[0]) + if not _PY3 and type(clz) is not _OldStyleClass: + raise InsecureJelly("Legacy 'instance' found with new-style class") + return self._genericUnjelly(clz, rest[1]) + + + def _unjelly_unpersistable(self, rest): + return Unpersistable("Unpersistable data: %s" % (rest[0],)) + + + def _unjelly_method(self, rest): + """ + (internal) Unjelly a method. + """ + im_name = rest[0] + im_self = self.unjelly(rest[1]) + im_class = self.unjelly(rest[2]) + if not isinstance(im_class, (type, _OldStyleClass)): + raise InsecureJelly("Method found with non-class class.") + if im_name in im_class.__dict__: + if im_self is None: + im = getattr(im_class, im_name) + elif isinstance(im_self, NotKnown): + im = _InstanceMethod(im_name, im_self, im_class) + else: + im = types.MethodType(im_class.__dict__[im_name], im_self, + *([im_class] * (not _PY3))) + else: + raise TypeError('instance method changed') + return im + + + +#### Published Interface. + + +class InsecureJelly(Exception): + """ + This exception will be raised when a jelly is deemed `insecure'; e.g. it + contains a type, class, or module disallowed by the specified `taster' + """ + + + +class DummySecurityOptions: + """ + DummySecurityOptions() -> insecure security options + Dummy security options -- this class will allow anything. + """ + + def isModuleAllowed(self, moduleName): + """ + DummySecurityOptions.isModuleAllowed(moduleName) -> boolean + returns 1 if a module by that name is allowed, 0 otherwise + """ + return 1 + + + def isClassAllowed(self, klass): + """ + DummySecurityOptions.isClassAllowed(class) -> boolean + Assumes the module has already been allowed. Returns 1 if the given + class is allowed, 0 otherwise. + """ + return 1 + + + def isTypeAllowed(self, typeName): + """ + DummySecurityOptions.isTypeAllowed(typeName) -> boolean + Returns 1 if the given type is allowed, 0 otherwise. + """ + return 1 + + + +class SecurityOptions: + """ + This will by default disallow everything, except for 'none'. + """ + + basicTypes = ["dictionary", "list", "tuple", + "reference", "dereference", "unpersistable", + "persistent", "long_int", "long", "dict"] + + def __init__(self): + """ + SecurityOptions() initialize. + """ + # I don't believe any of these types can ever pose a security hazard, + # except perhaps "reference"... + self.allowedTypes = { + b"None": 1, b"bool": 1, b"boolean": 1, b"string": 1, b"str": 1, + b"int": 1, b"float": 1, b"datetime": 1, b"time": 1, b"date": 1, + b"timedelta": 1, b"NoneType": 1, b'unicode': 1, b'decimal': 1, + b'set': 1, b'frozenset': 1, + } + self.allowedModules = {} + self.allowedClasses = {} + + + def allowBasicTypes(self): + """ + Allow all `basic' types. (Dictionary and list. Int, string, and float + are implicitly allowed.) + """ + self.allowTypes(*self.basicTypes) + + + def allowTypes(self, *types): + """ + SecurityOptions.allowTypes(typeString): Allow a particular type, by its + name. + """ + for typ in types: + if isinstance(typ, unicode): + typ = typ.encode('utf-8') + if not isinstance(typ, bytes): + typ = qual(typ) + self.allowedTypes[typ] = 1 + + + def allowInstancesOf(self, *classes): + """ + SecurityOptions.allowInstances(klass, klass, ...): allow instances + of the specified classes + + This will also allow the 'instance', 'class' (renamed 'classobj' in + Python 2.3), and 'module' types, as well as basic types. + """ + self.allowBasicTypes() + self.allowTypes("instance", "class", "classobj", "module") + for klass in classes: + self.allowTypes(qual(klass)) + self.allowModules(klass.__module__) + self.allowedClasses[klass] = 1 + + + def allowModules(self, *modules): + """ + SecurityOptions.allowModules(module, module, ...): allow modules by + name. This will also allow the 'module' type. + """ + for module in modules: + if type(module) == types.ModuleType: + module = module.__name__ + + if not isinstance(module, bytes): + module = module.encode('utf-8') + + self.allowedModules[module] = 1 + + + def isModuleAllowed(self, moduleName): + """ + SecurityOptions.isModuleAllowed(moduleName) -> boolean + returns 1 if a module by that name is allowed, 0 otherwise + """ + if not isinstance(moduleName, bytes): + moduleName = moduleName.encode('utf-8') + + return moduleName in self.allowedModules + + + def isClassAllowed(self, klass): + """ + SecurityOptions.isClassAllowed(class) -> boolean + Assumes the module has already been allowed. Returns 1 if the given + class is allowed, 0 otherwise. + """ + return klass in self.allowedClasses + + + def isTypeAllowed(self, typeName): + """ + SecurityOptions.isTypeAllowed(typeName) -> boolean + Returns 1 if the given type is allowed, 0 otherwise. + """ + if not isinstance(typeName, bytes): + typeName = typeName.encode('utf-8') + + return (typeName in self.allowedTypes or b'.' in typeName) + + +globalSecurity = SecurityOptions() +globalSecurity.allowBasicTypes() + + + +def jelly(object, taster=DummySecurityOptions(), persistentStore=None, + invoker=None): + """ + Serialize to s-expression. + + Returns a list which is the serialized representation of an object. An + optional 'taster' argument takes a SecurityOptions and will mark any + insecure objects as unpersistable rather than serializing them. + """ + return _Jellier(taster, persistentStore, invoker).jelly(object) + + + +def unjelly(sexp, taster=DummySecurityOptions(), persistentLoad=None, + invoker=None): + """ + Unserialize from s-expression. + + Takes a list that was the result from a call to jelly() and unserializes + an arbitrary object from it. The optional 'taster' argument, an instance + of SecurityOptions, will cause an InsecureJelly exception to be raised if a + disallowed type, module, or class attempted to unserialize. + """ + return _Unjellier(taster, persistentLoad, invoker).unjellyFull(sexp) diff --git a/contrib/python/Twisted/py2/twisted/spread/pb.py b/contrib/python/Twisted/py2/twisted/spread/pb.py new file mode 100644 index 00000000000..33204a2a704 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/spread/pb.py @@ -0,0 +1,1677 @@ +# -*- test-case-name: twisted.spread.test.test_pb -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Perspective Broker + +\"This isn\'t a professional opinion, but it's probably got enough +internet to kill you.\" --glyph + +Introduction +============ + +This is a broker for proxies for and copies of objects. It provides a +translucent interface layer to those proxies. + +The protocol is not opaque, because it provides objects which represent the +remote proxies and require no context (server references, IDs) to operate on. + +It is not transparent because it does I{not} attempt to make remote objects +behave identically, or even similarly, to local objects. Method calls are +invoked asynchronously, and specific rules are applied when serializing +arguments. + +To get started, begin with L{PBClientFactory} and L{PBServerFactory}. + +@author: Glyph Lefkowitz +""" + +from __future__ import absolute_import, division + +import random +from hashlib import md5 + +from zope.interface import implementer, Interface + +# Twisted Imports +from twisted.python import log, failure, reflect +from twisted.python.compat import (unicode, _bytesChr as chr, range, + comparable, cmp) +from twisted.internet import defer, protocol +from twisted.cred.portal import Portal +from twisted.cred.credentials import IAnonymous, ICredentials +from twisted.cred.credentials import IUsernameHashedPassword, Anonymous +from twisted.persisted import styles +from twisted.python.components import registerAdapter + +from twisted.spread.interfaces import IJellyable, IUnjellyable +from twisted.spread.jelly import jelly, unjelly, globalSecurity, _newInstance +from twisted.spread import banana + +from twisted.spread.flavors import Serializable +from twisted.spread.flavors import Referenceable, NoSuchMethod +from twisted.spread.flavors import Root, IPBRoot +from twisted.spread.flavors import ViewPoint +from twisted.spread.flavors import Viewable +from twisted.spread.flavors import Copyable +from twisted.spread.flavors import Jellyable +from twisted.spread.flavors import Cacheable +from twisted.spread.flavors import RemoteCopy +from twisted.spread.flavors import RemoteCache +from twisted.spread.flavors import RemoteCacheObserver +from twisted.spread.flavors import copyTags + +from twisted.spread.flavors import setUnjellyableForClass +from twisted.spread.flavors import setUnjellyableFactoryForClass +from twisted.spread.flavors import setUnjellyableForClassTree +# These three are backwards compatibility aliases for the previous three. +# Ultimately they should be deprecated. -exarkun +from twisted.spread.flavors import setCopierForClass +from twisted.spread.flavors import setFactoryForClass +from twisted.spread.flavors import setCopierForClassTree + + +MAX_BROKER_REFS = 1024 + +portno = 8787 + + + +class ProtocolError(Exception): + """ + This error is raised when an invalid protocol statement is received. + """ + + + +class DeadReferenceError(ProtocolError): + """ + This error is raised when a method is called on a dead reference (one whose + broker has been disconnected). + """ + + + +class Error(Exception): + """ + This error can be raised to generate known error conditions. + + When a PB callable method (perspective_, remote_, view_) raises + this error, it indicates that a traceback should not be printed, + but instead, the string representation of the exception should be + sent. + """ + + + +class RemoteError(Exception): + """ + This class is used to wrap a string-ified exception from the remote side to + be able to reraise it. (Raising string exceptions is no longer possible in + Python 2.6+) + + The value of this exception will be a str() representation of the remote + value. + + @ivar remoteType: The full import path of the exception class which was + raised on the remote end. + @type remoteType: C{str} + + @ivar remoteTraceback: The remote traceback. + @type remoteTraceback: C{str} + + @note: It's not possible to include the remoteTraceback if this exception is + thrown into a generator. It must be accessed as an attribute. + """ + def __init__(self, remoteType, value, remoteTraceback): + Exception.__init__(self, value) + self.remoteType = remoteType + self.remoteTraceback = remoteTraceback + + + +@comparable +class RemoteMethod: + """ + This is a translucent reference to a remote message. + """ + def __init__(self, obj, name): + """ + Initialize with a L{RemoteReference} and the name of this message. + """ + self.obj = obj + self.name = name + + + def __cmp__(self, other): + return cmp((self.obj, self.name), other) + + + def __hash__(self): + return hash((self.obj, self.name)) + + + def __call__(self, *args, **kw): + """ + Asynchronously invoke a remote method. + """ + return self.obj.broker._sendMessage(b'', self.obj.perspective, + self.obj.luid, self.name.encode("utf-8"), args, kw) + + + +class PBConnectionLost(Exception): + pass + + + +class IPerspective(Interface): + """ + per*spec*tive, n. : The relationship of aspects of a subject to each + other and to a whole: 'a perspective of history'; 'a need to view + the problem in the proper perspective'. + + This is a Perspective Broker-specific wrapper for an avatar. That + is to say, a PB-published view on to the business logic for the + system's concept of a 'user'. + + The concept of attached/detached is no longer implemented by the + framework. The realm is expected to implement such semantics if + needed. + """ + + def perspectiveMessageReceived(broker, message, args, kwargs): + """ + This method is called when a network message is received. + + @arg broker: The Perspective Broker. + + @type message: str + @arg message: The name of the method called by the other end. + + @type args: list in jelly format + @arg args: The arguments that were passed by the other end. It + is recommend that you use the `unserialize' method of the + broker to decode this. + + @type kwargs: dict in jelly format + @arg kwargs: The keyword arguments that were passed by the + other end. It is recommended that you use the + `unserialize' method of the broker to decode this. + + @rtype: A jelly list. + @return: It is recommended that you use the `serialize' method + of the broker on whatever object you need to return to + generate the return value. + """ + + + +@implementer(IPerspective) +class Avatar: + """ + A default IPerspective implementor. + + This class is intended to be subclassed, and a realm should return + an instance of such a subclass when IPerspective is requested of + it. + + A peer requesting a perspective will receive only a + L{RemoteReference} to a pb.Avatar. When a method is called on + that L{RemoteReference}, it will translate to a method on the + remote perspective named 'perspective_methodname'. (For more + information on invoking methods on other objects, see + L{flavors.ViewPoint}.) + """ + + def perspectiveMessageReceived(self, broker, message, args, kw): + """ + This method is called when a network message is received. + + This will call:: + + self.perspective_%(message)s(*broker.unserialize(args), + **broker.unserialize(kw)) + + to handle the method; subclasses of Avatar are expected to + implement methods using this naming convention. + """ + + args = broker.unserialize(args, self) + kw = broker.unserialize(kw, self) + method = getattr(self, "perspective_%s" % message) + try: + state = method(*args, **kw) + except TypeError: + log.msg("%s didn't accept %s and %s" % (method, args, kw)) + raise + return broker.serialize(state, self, method, args, kw) + + + +class AsReferenceable(Referenceable): + """ + A reference directed towards another object. + """ + + def __init__(self, object, messageType="remote"): + self.remoteMessageReceived = getattr( + object, messageType + "MessageReceived") + + + +@implementer(IUnjellyable) +@comparable +class RemoteReference(Serializable, styles.Ephemeral): + """ + A translucent reference to a remote object. + + I may be a reference to a L{flavors.ViewPoint}, a + L{flavors.Referenceable}, or an L{IPerspective} implementer (e.g., + pb.Avatar). From the client's perspective, it is not possible to + tell which except by convention. + + I am a \"translucent\" reference because although no additional + bookkeeping overhead is given to the application programmer for + manipulating a reference, return values are asynchronous. + + See also L{twisted.internet.defer}. + + @ivar broker: The broker I am obtained through. + @type broker: L{Broker} + """ + + def __init__(self, perspective, broker, luid, doRefCount): + """(internal) Initialize me with a broker and a locally-unique ID. + + The ID is unique only to the particular Perspective Broker + instance. + """ + self.luid = luid + self.broker = broker + self.doRefCount = doRefCount + self.perspective = perspective + self.disconnectCallbacks = [] + + + def notifyOnDisconnect(self, callback): + """ + Register a callback to be called if our broker gets disconnected. + + @param callback: a callable which will be called with one + argument, this instance. + """ + assert callable(callback) + self.disconnectCallbacks.append(callback) + if len(self.disconnectCallbacks) == 1: + self.broker.notifyOnDisconnect(self._disconnected) + + + def dontNotifyOnDisconnect(self, callback): + """ + Remove a callback that was registered with notifyOnDisconnect. + + @param callback: a callable + """ + self.disconnectCallbacks.remove(callback) + if not self.disconnectCallbacks: + self.broker.dontNotifyOnDisconnect(self._disconnected) + + + def _disconnected(self): + """ + Called if we are disconnected and have callbacks registered. + """ + for callback in self.disconnectCallbacks: + callback(self) + self.disconnectCallbacks = None + + + def jellyFor(self, jellier): + """ + If I am being sent back to where I came from, serialize as a local backreference. + """ + if jellier.invoker: + assert self.broker == jellier.invoker, "Can't send references to brokers other than their own." + return b"local", self.luid + else: + return b"unpersistable", "References cannot be serialized" + + + def unjellyFor(self, unjellier, unjellyList): + self.__init__(unjellier.invoker.unserializingPerspective, unjellier.invoker, unjellyList[1], 1) + return self + + + def callRemote(self, _name, *args, **kw): + """ + Asynchronously invoke a remote method. + + @type _name: L{str} + @param _name: the name of the remote method to invoke + @param args: arguments to serialize for the remote function + @param kw: keyword arguments to serialize for the remote function. + @rtype: L{twisted.internet.defer.Deferred} + @returns: a Deferred which will be fired when the result of + this remote call is received. + """ + if not isinstance(_name, bytes): + _name = _name.encode('utf8') + + # Note that we use '_name' instead of 'name' so the user can call + # remote methods with 'name' as a keyword parameter, like this: + # ref.callRemote("getPeopleNamed", count=12, name="Bob") + return self.broker._sendMessage(b'', self.perspective, self.luid, + _name, args, kw) + + + def remoteMethod(self, key): + """ + + @param key: The key. + @return: A L{RemoteMethod} for this key. + """ + return RemoteMethod(self, key) + + + def __cmp__(self, other): + """ + + @param other: another L{RemoteReference} to compare me to. + """ + if isinstance(other, RemoteReference): + if other.broker == self.broker: + return cmp(self.luid, other.luid) + return cmp(self.broker, other) + + + def __hash__(self): + """ + Hash me. + """ + return self.luid + + + def __del__(self): + """ + Do distributed reference counting on finalization. + """ + if self.doRefCount: + self.broker.sendDecRef(self.luid) + +setUnjellyableForClass("remote", RemoteReference) + +class Local: + """ + (internal) A reference to a local object. + """ + + def __init__(self, object, perspective=None): + """ + Initialize. + """ + self.object = object + self.perspective = perspective + self.refcount = 1 + + + def __repr__(self): + return "" % (self.object, self.refcount) + + + def incref(self): + """ + Increment the reference count. + + @return: the reference count after incrementing + """ + self.refcount = self.refcount + 1 + return self.refcount + + + def decref(self): + """ + Decrement the reference count. + + @return: the reference count after decrementing + """ + self.refcount = self.refcount - 1 + return self.refcount + + + +# Failure +class CopyableFailure(failure.Failure, Copyable): + """ + A L{flavors.RemoteCopy} and L{flavors.Copyable} version of + L{twisted.python.failure.Failure} for serialization. + """ + + unsafeTracebacks = 0 + + def getStateToCopy(self): + """ + Collect state related to the exception which occurred, discarding + state which cannot reasonably be serialized. + """ + state = self.__dict__.copy() + state['tb'] = None + state['frames'] = [] + state['stack'] = [] + state['value'] = str(self.value) # Exception instance + if isinstance(self.type, bytes): + state['type'] = self.type + else: + state['type'] = reflect.qual(self.type).encode('utf-8') # Exception class + if self.unsafeTracebacks: + state['traceback'] = self.getTraceback() + else: + state['traceback'] = 'Traceback unavailable\n' + return state + + + +class CopiedFailure(RemoteCopy, failure.Failure): + """ + A L{CopiedFailure} is a L{pb.RemoteCopy} of a L{failure.Failure} + transferred via PB. + + @ivar type: The full import path of the exception class which was raised on + the remote end. + @type type: C{str} + + @ivar value: A str() representation of the remote value. + @type value: L{CopiedFailure} or C{str} + + @ivar traceback: The remote traceback. + @type traceback: C{str} + """ + + def printTraceback(self, file=None, elideFrameworkCode=0, detail='default'): + if file is None: + file = log.logfile + failureType = self.type + if not isinstance(failureType, str): + failureType = failureType.decode("utf-8") + file.write("Traceback from remote host -- ") + file.write(failureType + ": " + self.value) + file.write('\n') + + + def throwExceptionIntoGenerator(self, g): + """ + Throw the original exception into the given generator, preserving + traceback information if available. In the case of a L{CopiedFailure} + where the exception type is a string, a L{pb.RemoteError} is thrown + instead. + + @return: The next value yielded from the generator. + @raise StopIteration: If there are no more values in the generator. + @raise RemoteError: The wrapped remote exception. + """ + return g.throw(RemoteError(self.type, self.value, self.traceback)) + + printBriefTraceback = printTraceback + printDetailedTraceback = printTraceback + +setUnjellyableForClass(CopyableFailure, CopiedFailure) + + + +def failure2Copyable(fail, unsafeTracebacks=0): + f = _newInstance(CopyableFailure, fail.__dict__) + f.unsafeTracebacks = unsafeTracebacks + return f + + + +class Broker(banana.Banana): + """ + I am a broker for objects. + """ + + version = 6 + username = None + factory = None + + def __init__(self, isClient=1, security=globalSecurity): + banana.Banana.__init__(self, isClient) + self.disconnected = 0 + self.disconnects = [] + self.failures = [] + self.connects = [] + self.localObjects = {} + self.security = security + self.pageProducers = [] + self.currentRequestID = 0 + self.currentLocalID = 0 + self.unserializingPerspective = None + # Some terms: + # PUID: process unique ID; return value of id() function. type "int". + # LUID: locally unique ID; an ID unique to an object mapped over this + # connection. type "int" + # GUID: (not used yet) globally unique ID; an ID for an object which + # may be on a redirected or meta server. Type as yet undecided. + # Dictionary mapping LUIDs to local objects. + # set above to allow root object to be assigned before connection is made + # self.localObjects = {} + # Dictionary mapping PUIDs to LUIDs. + self.luids = {} + # Dictionary mapping LUIDs to local (remotely cached) objects. Remotely + # cached means that they're objects which originate here, and were + # copied remotely. + self.remotelyCachedObjects = {} + # Dictionary mapping PUIDs to (cached) LUIDs + self.remotelyCachedLUIDs = {} + # Dictionary mapping (remote) LUIDs to (locally cached) objects. + self.locallyCachedObjects = {} + self.waitingForAnswers = {} + + # Mapping from LUIDs to weakref objects with callbacks for performing + # any local cleanup which may be necessary for the corresponding + # object once it no longer exists. + self._localCleanup = {} + + + def resumeProducing(self): + """ + Called when the consumer attached to me runs out of buffer. + """ + # Go backwards over the list so we can remove indexes from it as we go + for pageridx in range(len(self.pageProducers)-1, -1, -1): + pager = self.pageProducers[pageridx] + pager.sendNextPage() + if not pager.stillPaging(): + del self.pageProducers[pageridx] + if not self.pageProducers: + self.transport.unregisterProducer() + + + def pauseProducing(self): + # Streaming producer method; not necessary to implement. + pass + + + def stopProducing(self): + # Streaming producer method; not necessary to implement. + pass + + + def registerPageProducer(self, pager): + self.pageProducers.append(pager) + if len(self.pageProducers) == 1: + self.transport.registerProducer(self, 0) + + + def expressionReceived(self, sexp): + """ + Evaluate an expression as it's received. + """ + if isinstance(sexp, list): + command = sexp[0] + + if not isinstance(command, str): + command = command.decode('utf8') + + methodName = "proto_%s" % command + method = getattr(self, methodName, None) + + if method: + method(*sexp[1:]) + else: + self.sendCall(b"didNotUnderstand", command) + else: + raise ProtocolError("Non-list expression received.") + + + def proto_version(self, vnum): + """ + Protocol message: (version version-number) + + Check to make sure that both ends of the protocol are speaking + the same version dialect. + + @param vnum: The version number. + """ + + if vnum != self.version: + raise ProtocolError("Version Incompatibility: %s %s" % (self.version, vnum)) + + + def sendCall(self, *exp): + """ + Utility method to send an expression to the other side of the connection. + + @param exp: The expression. + """ + self.sendEncoded(exp) + + + def proto_didNotUnderstand(self, command): + """ + Respond to stock 'C{didNotUnderstand}' message. + + Log the command that was not understood and continue. (Note: + this will probably be changed to close the connection or raise + an exception in the future.) + + @param command: The command to log. + """ + log.msg("Didn't understand command: %r" % command) + + + def connectionReady(self): + """ + Initialize. Called after Banana negotiation is done. + """ + self.sendCall(b"version", self.version) + for notifier in self.connects: + try: + notifier() + except: + log.deferr() + self.connects = None + self.factory.clientConnectionMade(self) + + + def connectionFailed(self): + # XXX should never get called anymore? check! + for notifier in self.failures: + try: + notifier() + except: + log.deferr() + self.failures = None + + waitingForAnswers = None + + + def connectionLost(self, reason): + """ + The connection was lost. + + @param reason: message to put in L{failure.Failure} + """ + self.disconnected = 1 + # Nuke potential circular references. + self.luids = None + if self.waitingForAnswers: + for d in self.waitingForAnswers.values(): + try: + d.errback(failure.Failure(PBConnectionLost(reason))) + except: + log.deferr() + # Assure all Cacheable.stoppedObserving are called + for lobj in self.remotelyCachedObjects.values(): + cacheable = lobj.object + perspective = lobj.perspective + try: + cacheable.stoppedObserving(perspective, RemoteCacheObserver(self, cacheable, perspective)) + except: + log.deferr() + # Loop on a copy to prevent notifiers to mixup + # the list by calling dontNotifyOnDisconnect + for notifier in self.disconnects[:]: + try: + notifier() + except: + log.deferr() + self.disconnects = None + self.waitingForAnswers = None + self.localSecurity = None + self.remoteSecurity = None + self.remotelyCachedObjects = None + self.remotelyCachedLUIDs = None + self.locallyCachedObjects = None + self.localObjects = None + + + def notifyOnDisconnect(self, notifier): + """ + + @param notifier: callback to call when the Broker disconnects. + """ + assert callable(notifier) + self.disconnects.append(notifier) + + + def notifyOnFail(self, notifier): + """ + + @param notifier: callback to call if the Broker fails to connect. + """ + assert callable(notifier) + self.failures.append(notifier) + + + def notifyOnConnect(self, notifier): + """ + + @param notifier: callback to call when the Broker connects. + """ + assert callable(notifier) + if self.connects is None: + try: + notifier() + except: + log.err() + else: + self.connects.append(notifier) + + + def dontNotifyOnDisconnect(self, notifier): + """ + + @param notifier: callback to remove from list of disconnect callbacks. + """ + try: + self.disconnects.remove(notifier) + except ValueError: + pass + + + def localObjectForID(self, luid): + """ + Get a local object for a locally unique ID. + + @return: An object previously stored with L{registerReference} or + L{None} if there is no object which corresponds to the given + identifier. + """ + if isinstance(luid, unicode): + luid = luid.encode('utf8') + + lob = self.localObjects.get(luid) + if lob is None: + return + return lob.object + + maxBrokerRefsViolations = 0 + + + def registerReference(self, object): + """ + Store a persistent reference to a local object and map its + id() to a generated, session-unique ID. + + @param object: a local object + @return: the generated ID + """ + + assert object is not None + puid = object.processUniqueID() + luid = self.luids.get(puid) + if luid is None: + if len(self.localObjects) > MAX_BROKER_REFS: + self.maxBrokerRefsViolations = self.maxBrokerRefsViolations + 1 + if self.maxBrokerRefsViolations > 3: + self.transport.loseConnection() + raise Error("Maximum PB reference count exceeded. " + "Goodbye.") + raise Error("Maximum PB reference count exceeded.") + + luid = self.newLocalID() + self.localObjects[luid] = Local(object) + self.luids[puid] = luid + else: + self.localObjects[luid].incref() + return luid + + + def setNameForLocal(self, name, object): + """ + Store a special (string) ID for this object. + + This is how you specify a 'base' set of objects that the remote + protocol can connect to. + + @param name: An ID. + @param object: The object. + """ + if isinstance(name, unicode): + name = name.encode('utf8') + + assert object is not None + self.localObjects[name] = Local(object) + + + def remoteForName(self, name): + """ + Returns an object from the remote name mapping. + + Note that this does not check the validity of the name, only + creates a translucent reference for it. + + @param name: The name to look up. + @return: An object which maps to the name. + """ + if isinstance(name, unicode): + name = name.encode('utf8') + + return RemoteReference(None, self, name, 0) + + + def cachedRemotelyAs(self, instance, incref=0): + """ + + @param instance: The instance to look up. + @param incref: Flag to specify whether to increment the + reference. + @return: An ID that says what this instance is cached as + remotely, or L{None} if it's not. + """ + + puid = instance.processUniqueID() + luid = self.remotelyCachedLUIDs.get(puid) + if (luid is not None) and (incref): + self.remotelyCachedObjects[luid].incref() + return luid + + + def remotelyCachedForLUID(self, luid): + """ + + @param luid: The LUID to look up. + @return: An instance which is cached remotely. + """ + return self.remotelyCachedObjects[luid].object + + + def cacheRemotely(self, instance): + """ + XXX + + @return: A new LUID. + """ + puid = instance.processUniqueID() + luid = self.newLocalID() + if len(self.remotelyCachedObjects) > MAX_BROKER_REFS: + self.maxBrokerRefsViolations = self.maxBrokerRefsViolations + 1 + if self.maxBrokerRefsViolations > 3: + self.transport.loseConnection() + raise Error("Maximum PB cache count exceeded. " + "Goodbye.") + raise Error("Maximum PB cache count exceeded.") + + self.remotelyCachedLUIDs[puid] = luid + # This table may not be necessary -- for now, it's to make sure that no + # monkey business happens with id(instance) + self.remotelyCachedObjects[luid] = Local(instance, self.serializingPerspective) + return luid + + + def cacheLocally(self, cid, instance): + """(internal) + + Store a non-filled-out cached instance locally. + """ + self.locallyCachedObjects[cid] = instance + + + def cachedLocallyAs(self, cid): + instance = self.locallyCachedObjects[cid] + return instance + + + def serialize(self, object, perspective=None, method=None, args=None, kw=None): + """ + Jelly an object according to the remote security rules for this broker. + + @param object: The object to jelly. + @param perspective: The perspective. + @param method: The method. + @param args: Arguments. + @param kw: Keyword arguments. + """ + + if isinstance(object, defer.Deferred): + object.addCallbacks(self.serialize, lambda x: x, + callbackKeywords={ + 'perspective': perspective, + 'method': method, + 'args': args, + 'kw': kw + }) + return object + + # XXX This call is NOT REENTRANT and testing for reentrancy is just + # crazy, so it likely won't be. Don't ever write methods that call the + # broker's serialize() method recursively (e.g. sending a method call + # from within a getState (this causes concurrency problems anyway so + # you really, really shouldn't do it)) + + self.serializingPerspective = perspective + self.jellyMethod = method + self.jellyArgs = args + self.jellyKw = kw + try: + return jelly(object, self.security, None, self) + finally: + self.serializingPerspective = None + self.jellyMethod = None + self.jellyArgs = None + self.jellyKw = None + + + def unserialize(self, sexp, perspective = None): + """ + Unjelly an sexp according to the local security rules for this broker. + + @param sexp: The object to unjelly. + @param perspective: The perspective. + """ + + self.unserializingPerspective = perspective + try: + return unjelly(sexp, self.security, None, self) + finally: + self.unserializingPerspective = None + + + def newLocalID(self): + """ + + @return: A newly generated LUID. + """ + self.currentLocalID = self.currentLocalID + 1 + return self.currentLocalID + + + def newRequestID(self): + """ + + @return: A newly generated request ID. + """ + self.currentRequestID = self.currentRequestID + 1 + return self.currentRequestID + + + def _sendMessage(self, prefix, perspective, objectID, message, args, kw): + pbc = None + pbe = None + answerRequired = 1 + if 'pbcallback' in kw: + pbc = kw['pbcallback'] + del kw['pbcallback'] + if 'pberrback' in kw: + pbe = kw['pberrback'] + del kw['pberrback'] + if 'pbanswer' in kw: + assert (not pbe) and (not pbc), "You can't specify a no-answer requirement." + answerRequired = kw['pbanswer'] + del kw['pbanswer'] + if self.disconnected: + raise DeadReferenceError("Calling Stale Broker") + try: + netArgs = self.serialize(args, perspective=perspective, method=message) + netKw = self.serialize(kw, perspective=perspective, method=message) + except: + return defer.fail(failure.Failure()) + requestID = self.newRequestID() + if answerRequired: + rval = defer.Deferred() + self.waitingForAnswers[requestID] = rval + if pbc or pbe: + log.msg('warning! using deprecated "pbcallback"') + rval.addCallbacks(pbc, pbe) + else: + rval = None + self.sendCall(prefix + b"message", requestID, objectID, message, answerRequired, netArgs, netKw) + return rval + + + def proto_message(self, requestID, objectID, message, answerRequired, netArgs, netKw): + self._recvMessage(self.localObjectForID, requestID, objectID, message, answerRequired, netArgs, netKw) + + + def proto_cachemessage(self, requestID, objectID, message, answerRequired, netArgs, netKw): + self._recvMessage(self.cachedLocallyAs, requestID, objectID, message, answerRequired, netArgs, netKw) + + + def _recvMessage(self, findObjMethod, requestID, objectID, message, answerRequired, netArgs, netKw): + """ + Received a message-send. + + Look up message based on object, unserialize the arguments, and + invoke it with args, and send an 'answer' or 'error' response. + + @param findObjMethod: A callable which takes C{objectID} as argument. + @param requestID: The requiest ID. + @param objectID: The object ID. + @param message: The message. + @param answerRequired: + @param netArgs: Arguments. + @param netKw: Keyword arguments. + """ + if not isinstance(message, str): + message = message.decode('utf8') + + try: + object = findObjMethod(objectID) + if object is None: + raise Error("Invalid Object ID") + netResult = object.remoteMessageReceived(self, message, netArgs, netKw) + except Error as e: + if answerRequired: + # If the error is Jellyable or explicitly allowed via our + # security options, send it back and let the code on the + # other end deal with unjellying. If it isn't Jellyable, + # wrap it in a CopyableFailure, which ensures it can be + # unjellied on the other end. We have to do this because + # all errors must be sent back. + if isinstance(e, Jellyable) or self.security.isClassAllowed(e.__class__): + self._sendError(e, requestID) + else: + self._sendError(CopyableFailure(e), requestID) + except: + if answerRequired: + log.msg("Peer will receive following PB traceback:", isError=True) + f = CopyableFailure() + self._sendError(f, requestID) + log.err() + else: + if answerRequired: + if isinstance(netResult, defer.Deferred): + args = (requestID,) + netResult.addCallbacks(self._sendAnswer, self._sendFailureOrError, + callbackArgs=args, errbackArgs=args) + # XXX Should this be done somewhere else? + else: + self._sendAnswer(netResult, requestID) + + + def _sendAnswer(self, netResult, requestID): + """ + (internal) Send an answer to a previously sent message. + + @param netResult: The answer. + @param requestID: The request ID. + """ + self.sendCall(b"answer", requestID, netResult) + + + def proto_answer(self, requestID, netResult): + """ + (internal) Got an answer to a previously sent message. + + Look up the appropriate callback and call it. + + @param requestID: The request ID. + @param netResult: The answer. + """ + d = self.waitingForAnswers[requestID] + del self.waitingForAnswers[requestID] + d.callback(self.unserialize(netResult)) + + + def _sendFailureOrError(self, fail, requestID): + """ + Call L{_sendError} or L{_sendFailure}, depending on whether C{fail} + represents an L{Error} subclass or not. + + @param fail: The failure. + @param requestID: The request ID. + """ + if fail.check(Error) is None: + self._sendFailure(fail, requestID) + else: + self._sendError(fail, requestID) + + + def _sendFailure(self, fail, requestID): + """ + Log error and then send it. + + @param fail: The failure. + @param requestID: The request ID. + """ + log.msg("Peer will receive following PB traceback:") + log.err(fail) + self._sendError(fail, requestID) + + + def _sendError(self, fail, requestID): + """ + (internal) Send an error for a previously sent message. + + @param fail: The failure. + @param requestID: The request ID. + """ + if isinstance(fail, failure.Failure): + # If the failures value is jellyable or allowed through security, + # send the value + if (isinstance(fail.value, Jellyable) or + self.security.isClassAllowed(fail.value.__class__)): + fail = fail.value + elif not isinstance(fail, CopyableFailure): + fail = failure2Copyable(fail, self.factory.unsafeTracebacks) + if isinstance(fail, CopyableFailure): + fail.unsafeTracebacks = self.factory.unsafeTracebacks + self.sendCall(b"error", requestID, self.serialize(fail)) + + + def proto_error(self, requestID, fail): + """ + (internal) Deal with an error. + + @param requestID: The request ID. + @param fail: The failure. + """ + d = self.waitingForAnswers[requestID] + del self.waitingForAnswers[requestID] + d.errback(self.unserialize(fail)) + + + def sendDecRef(self, objectID): + """ + (internal) Send a DECREF directive. + + @param objectID: The object ID. + """ + self.sendCall(b"decref", objectID) + + + def proto_decref(self, objectID): + """ + (internal) Decrement the reference count of an object. + + If the reference count is zero, it will free the reference to this + object. + + @param objectID: The object ID. + """ + if isinstance(objectID, unicode): + objectID = objectID.encode('utf8') + refs = self.localObjects[objectID].decref() + if refs == 0: + puid = self.localObjects[objectID].object.processUniqueID() + del self.luids[puid] + del self.localObjects[objectID] + self._localCleanup.pop(puid, lambda: None)() + + + def decCacheRef(self, objectID): + """ + (internal) Send a DECACHE directive. + + @param objectID: The object ID. + """ + self.sendCall(b"decache", objectID) + + + def proto_decache(self, objectID): + """ + (internal) Decrement the reference count of a cached object. + + If the reference count is zero, free the reference, then send an + 'uncached' directive. + + @param objectID: The object ID. + """ + refs = self.remotelyCachedObjects[objectID].decref() + # log.msg('decaching: %s #refs: %s' % (objectID, refs)) + if refs == 0: + lobj = self.remotelyCachedObjects[objectID] + cacheable = lobj.object + perspective = lobj.perspective + # TODO: force_decache needs to be able to force-invalidate a + # cacheable reference. + try: + cacheable.stoppedObserving(perspective, RemoteCacheObserver(self, cacheable, perspective)) + except: + log.deferr() + puid = cacheable.processUniqueID() + del self.remotelyCachedLUIDs[puid] + del self.remotelyCachedObjects[objectID] + self.sendCall(b"uncache", objectID) + + + def proto_uncache(self, objectID): + """ + (internal) Tell the client it is now OK to uncache an object. + + @param objectID: The object ID. + """ + # log.msg("uncaching locally %d" % objectID) + obj = self.locallyCachedObjects[objectID] + obj.broker = None +## def reallyDel(obj=obj): +## obj.__really_del__() +## obj.__del__ = reallyDel + del self.locallyCachedObjects[objectID] + + + +def respond(challenge, password): + """ + Respond to a challenge. + + This is useful for challenge/response authentication. + + @param challenge: A challenge. + @param password: A password. + @return: The password hashed twice. + """ + m = md5() + m.update(password) + hashedPassword = m.digest() + m = md5() + m.update(hashedPassword) + m.update(challenge) + doubleHashedPassword = m.digest() + return doubleHashedPassword + + + +def challenge(): + """ + + @return: Some random data. + """ + crap = b'' + for x in range(random.randrange(15,25)): + crap = crap + chr(random.randint(65,90)) + crap = md5(crap).digest() + return crap + + + +class PBClientFactory(protocol.ClientFactory): + """ + Client factory for PB brokers. + + As with all client factories, use with reactor.connectTCP/SSL/etc.. + getPerspective and getRootObject can be called either before or + after the connect. + """ + + protocol = Broker + unsafeTracebacks = False + + def __init__(self, unsafeTracebacks=False, security=globalSecurity): + """ + @param unsafeTracebacks: if set, tracebacks for exceptions will be sent + over the wire. + @type unsafeTracebacks: C{bool} + + @param security: security options used by the broker, default to + C{globalSecurity}. + @type security: L{twisted.spread.jelly.SecurityOptions} + """ + self.unsafeTracebacks = unsafeTracebacks + self.security = security + self._reset() + + + def buildProtocol(self, addr): + """ + Build the broker instance, passing the security options to it. + """ + p = self.protocol(isClient=True, security=self.security) + p.factory = self + return p + + + def _reset(self): + self.rootObjectRequests = [] # list of deferred + self._broker = None + self._root = None + + + def _failAll(self, reason): + deferreds = self.rootObjectRequests + self._reset() + for d in deferreds: + d.errback(reason) + + + def clientConnectionFailed(self, connector, reason): + self._failAll(reason) + + + def clientConnectionLost(self, connector, reason, reconnecting=0): + """ + Reconnecting subclasses should call with reconnecting=1. + """ + if reconnecting: + # Any pending requests will go to next connection attempt + # so we don't fail them. + self._broker = None + self._root = None + else: + self._failAll(reason) + + + def clientConnectionMade(self, broker): + self._broker = broker + self._root = broker.remoteForName("root") + ds = self.rootObjectRequests + self.rootObjectRequests = [] + for d in ds: + d.callback(self._root) + + + def getRootObject(self): + """ + Get root object of remote PB server. + + @return: Deferred of the root object. + """ + if self._broker and not self._broker.disconnected: + return defer.succeed(self._root) + d = defer.Deferred() + self.rootObjectRequests.append(d) + return d + + + def disconnect(self): + """ + If the factory is connected, close the connection. + + Note that if you set up the factory to reconnect, you will need to + implement extra logic to prevent automatic reconnection after this + is called. + """ + if self._broker: + self._broker.transport.loseConnection() + + + def _cbSendUsername(self, root, username, password, client): + return root.callRemote("login", username).addCallback( + self._cbResponse, password, client) + + + def _cbResponse(self, challenges, password, client): + challenge, challenger = challenges + return challenger.callRemote("respond", respond(challenge, password), client) + + + def _cbLoginAnonymous(self, root, client): + """ + Attempt an anonymous login on the given remote root object. + + @type root: L{RemoteReference} + @param root: The object on which to attempt the login, most likely + returned by a call to L{PBClientFactory.getRootObject}. + + @param client: A jellyable object which will be used as the I{mind} + parameter for the login attempt. + + @rtype: L{Deferred} + @return: A L{Deferred} which will be called back with a + L{RemoteReference} to an avatar when anonymous login succeeds, or + which will errback if anonymous login fails. + """ + return root.callRemote("loginAnonymous", client) + + + def login(self, credentials, client=None): + """ + Login and get perspective from remote PB server. + + Currently the following credentials are supported:: + + L{twisted.cred.credentials.IUsernamePassword} + L{twisted.cred.credentials.IAnonymous} + + @rtype: L{Deferred} + @return: A L{Deferred} which will be called back with a + L{RemoteReference} for the avatar logged in to, or which will + errback if login fails. + """ + d = self.getRootObject() + + if IAnonymous.providedBy(credentials): + d.addCallback(self._cbLoginAnonymous, client) + else: + d.addCallback( + self._cbSendUsername, credentials.username, + credentials.password, client) + return d + + + +class PBServerFactory(protocol.ServerFactory): + """ + Server factory for perspective broker. + + Login is done using a Portal object, whose realm is expected to return + avatars implementing IPerspective. The credential checkers in the portal + should accept IUsernameHashedPassword or IUsernameMD5Password. + + Alternatively, any object providing or adaptable to L{IPBRoot} can be + used instead of a portal to provide the root object of the PB server. + """ + + unsafeTracebacks = False + + # object broker factory + protocol = Broker + + def __init__(self, root, unsafeTracebacks=False, security=globalSecurity): + """ + @param root: factory providing the root Referenceable used by the broker. + @type root: object providing or adaptable to L{IPBRoot}. + + @param unsafeTracebacks: if set, tracebacks for exceptions will be sent + over the wire. + @type unsafeTracebacks: C{bool} + + @param security: security options used by the broker, default to + C{globalSecurity}. + @type security: L{twisted.spread.jelly.SecurityOptions} + """ + self.root = IPBRoot(root) + self.unsafeTracebacks = unsafeTracebacks + self.security = security + + + def buildProtocol(self, addr): + """ + Return a Broker attached to the factory (as the service provider). + """ + proto = self.protocol(isClient=False, security=self.security) + proto.factory = self + proto.setNameForLocal("root", self.root.rootObject(proto)) + return proto + + + def clientConnectionMade(self, protocol): + # XXX does this method make any sense? + pass + + + +class IUsernameMD5Password(ICredentials): + """ + I encapsulate a username and a hashed password. + + This credential is used for username/password over PB. CredentialCheckers + which check this kind of credential must store the passwords in plaintext + form or as a MD5 digest. + + @type username: C{str} or C{Deferred} + @ivar username: The username associated with these credentials. + """ + + def checkPassword(password): + """ + Validate these credentials against the correct password. + + @type password: C{str} + @param password: The correct, plaintext password against which to + check. + + @rtype: C{bool} or L{Deferred} + @return: C{True} if the credentials represented by this object match the + given password, C{False} if they do not, or a L{Deferred} which will + be called back with one of these values. + """ + + + def checkMD5Password(password): + """ + Validate these credentials against the correct MD5 digest of the + password. + + @type password: C{str} + @param password: The correct MD5 digest of a password against which to + check. + + @rtype: C{bool} or L{Deferred} + @return: C{True} if the credentials represented by this object match the + given digest, C{False} if they do not, or a L{Deferred} which will + be called back with one of these values. + """ + + + +@implementer(IPBRoot) +class _PortalRoot: + """ + Root object, used to login to portal. + """ + + def __init__(self, portal): + self.portal = portal + + + def rootObject(self, broker): + return _PortalWrapper(self.portal, broker) + +registerAdapter(_PortalRoot, Portal, IPBRoot) + + + +class _JellyableAvatarMixin: + """ + Helper class for code which deals with avatars which PB must be capable of + sending to a peer. + """ + def _cbLogin(self, result): + """ + Ensure that the avatar to be returned to the client is jellyable and + set up disconnection notification to call the realm's logout object. + """ + (interface, avatar, logout) = result + if not IJellyable.providedBy(avatar): + avatar = AsReferenceable(avatar, "perspective") + + puid = avatar.processUniqueID() + + # only call logout once, whether the connection is dropped (disconnect) + # or a logout occurs (cleanup), and be careful to drop the reference to + # it in either case + logout = [ logout ] + def maybeLogout(): + if not logout: + return + fn = logout[0] + del logout[0] + fn() + self.broker._localCleanup[puid] = maybeLogout + self.broker.notifyOnDisconnect(maybeLogout) + + return avatar + + + +class _PortalWrapper(Referenceable, _JellyableAvatarMixin): + """ + Root Referenceable object, used to login to portal. + """ + + def __init__(self, portal, broker): + self.portal = portal + self.broker = broker + + + def remote_login(self, username): + """ + Start of username/password login. + + @param username: The username. + """ + c = challenge() + return c, _PortalAuthChallenger(self.portal, self.broker, username, c) + + + def remote_loginAnonymous(self, mind): + """ + Attempt an anonymous login. + + @param mind: An object to use as the mind parameter to the portal login + call (possibly None). + + @rtype: L{Deferred} + @return: A Deferred which will be called back with an avatar when login + succeeds or which will be errbacked if login fails somehow. + """ + d = self.portal.login(Anonymous(), mind, IPerspective) + d.addCallback(self._cbLogin) + return d + + + +@implementer(IUsernameHashedPassword, IUsernameMD5Password) +class _PortalAuthChallenger(Referenceable, _JellyableAvatarMixin): + """ + Called with response to password challenge. + """ + def __init__(self, portal, broker, username, challenge): + self.portal = portal + self.broker = broker + self.username = username + self.challenge = challenge + + + def remote_respond(self, response, mind): + self.response = response + d = self.portal.login(self, mind, IPerspective) + d.addCallback(self._cbLogin) + return d + + + def checkPassword(self, password): + """ + L{IUsernameHashedPassword} + + @param password: The password. + @return: L{_PortalAuthChallenger.checkMD5Password} + """ + return self.checkMD5Password(md5(password).digest()) + + + def checkMD5Password(self, md5Password): + """ + L{IUsernameMD5Password} + + @param md5Password: + @rtype: L{bool} + @return: L{True} if password matches. + """ + md = md5() + md.update(md5Password) + md.update(self.challenge) + correct = md.digest() + return self.response == correct + + +__all__ = [ + # Everything from flavors is exposed publicly here. + 'IPBRoot', 'Serializable', 'Referenceable', 'NoSuchMethod', 'Root', + 'ViewPoint', 'Viewable', 'Copyable', 'Jellyable', 'Cacheable', + 'RemoteCopy', 'RemoteCache', 'RemoteCacheObserver', 'copyTags', + 'setUnjellyableForClass', 'setUnjellyableFactoryForClass', + 'setUnjellyableForClassTree', + 'setCopierForClass', 'setFactoryForClass', 'setCopierForClassTree', + + 'MAX_BROKER_REFS', 'portno', + + 'ProtocolError', 'DeadReferenceError', 'Error', 'PBConnectionLost', + 'RemoteMethod', 'IPerspective', 'Avatar', 'AsReferenceable', + 'RemoteReference', 'CopyableFailure', 'CopiedFailure', 'failure2Copyable', + 'Broker', 'respond', 'challenge', 'PBClientFactory', 'PBServerFactory', + 'IUsernameMD5Password', + ] diff --git a/contrib/python/Twisted/py2/twisted/spread/publish.py b/contrib/python/Twisted/py2/twisted/spread/publish.py new file mode 100644 index 00000000000..f206063ec3b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/spread/publish.py @@ -0,0 +1,142 @@ +# -*- test-case-name: twisted.spread.test.test_pb -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Persistently cached objects for PB. + +Maintainer: Glyph Lefkowitz + +Future Plans: None known. +""" + +from __future__ import absolute_import, division + +import time + +from twisted.internet import defer +from twisted.spread import banana, jelly, flavors + + +class Publishable(flavors.Cacheable): + """An object whose cached state persists across sessions. + """ + def __init__(self, publishedID): + self.republish() + self.publishedID = publishedID + + def republish(self): + """Set the timestamp to current and (TODO) update all observers. + """ + self.timestamp = time.time() + + def view_getStateToPublish(self, perspective): + '(internal)' + return self.getStateToPublishFor(perspective) + + def getStateToPublishFor(self, perspective): + """Implement me to special-case your state for a perspective. + """ + return self.getStateToPublish() + + def getStateToPublish(self): + """Implement me to return state to copy as part of the publish phase. + """ + raise NotImplementedError("%s.getStateToPublishFor" % self.__class__) + + def getStateToCacheAndObserveFor(self, perspective, observer): + """Get all necessary metadata to keep a clientside cache. + """ + if perspective: + pname = perspective.perspectiveName + sname = perspective.getService().serviceName + else: + pname = "None" + sname = "None" + + return {"remote": flavors.ViewPoint(perspective, self), + "publishedID": self.publishedID, + "perspective": pname, + "service": sname, + "timestamp": self.timestamp} + +class RemotePublished(flavors.RemoteCache): + """The local representation of remote Publishable object. + """ + isActivated = 0 + _wasCleanWhenLoaded = 0 + def getFileName(self, ext='pub'): + return ("%s-%s-%s.%s" % + (self.service, self.perspective, str(self.publishedID), ext)) + + def setCopyableState(self, state): + self.__dict__.update(state) + self._activationListeners = [] + try: + with open(self.getFileName(), "rb") as dataFile: + data = dataFile.read() + except IOError: + recent = 0 + else: + newself = jelly.unjelly(banana.decode(data)) + recent = (newself.timestamp == self.timestamp) + if recent: + self._cbGotUpdate(newself.__dict__) + self._wasCleanWhenLoaded = 1 + else: + self.remote.callRemote('getStateToPublish').addCallbacks(self._cbGotUpdate) + + def __getstate__(self): + other = self.__dict__.copy() + # Remove PB-specific attributes + del other['broker'] + del other['remote'] + del other['luid'] + # remove my own runtime-tracking stuff + del other['_activationListeners'] + del other['isActivated'] + return other + + def _cbGotUpdate(self, newState): + self.__dict__.update(newState) + self.isActivated = 1 + # send out notifications + for listener in self._activationListeners: + listener(self) + self._activationListeners = [] + self.activated() + with open(self.getFileName(), "wb") as dataFile: + dataFile.write(banana.encode(jelly.jelly(self))) + + + def activated(self): + """Implement this method if you want to be notified when your + publishable subclass is activated. + """ + + def callWhenActivated(self, callback): + """Externally register for notification when this publishable has received all relevant data. + """ + if self.isActivated: + callback(self) + else: + self._activationListeners.append(callback) + +def whenReady(d): + """ + Wrap a deferred returned from a pb method in another deferred that + expects a RemotePublished as a result. This will allow you to wait until + the result is really available. + + Idiomatic usage would look like:: + + publish.whenReady(serverObject.getMeAPublishable()).addCallback(lookAtThePublishable) + """ + d2 = defer.Deferred() + d.addCallbacks(_pubReady, d2.errback, + callbackArgs=(d2,)) + return d2 + +def _pubReady(result, d2): + '(internal)' + result.callWhenActivated(d2.callback) diff --git a/contrib/python/Twisted/py2/twisted/spread/util.py b/contrib/python/Twisted/py2/twisted/spread/util.py new file mode 100644 index 00000000000..697f0016103 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/spread/util.py @@ -0,0 +1,215 @@ +# -*- test-case-name: twisted.test.test_pb -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Utility classes for spread. +""" + +from twisted.internet import defer +from twisted.python.failure import Failure +from twisted.spread import pb +from twisted.protocols import basic +from twisted.internet import interfaces + +from zope.interface import implementer + + +class LocalMethod: + def __init__(self, local, name): + self.local = local + self.name = name + + def __call__(self, *args, **kw): + return self.local.callRemote(self.name, *args, **kw) + + +class LocalAsRemote: + """ + A class useful for emulating the effects of remote behavior locally. + """ + reportAllTracebacks = 1 + + def callRemote(self, name, *args, **kw): + """ + Call a specially-designated local method. + + self.callRemote('x') will first try to invoke a method named + sync_x and return its result (which should probably be a + Deferred). Second, it will look for a method called async_x, + which will be called and then have its result (or Failure) + automatically wrapped in a Deferred. + """ + if hasattr(self, 'sync_'+name): + return getattr(self, 'sync_'+name)(*args, **kw) + try: + method = getattr(self, "async_" + name) + return defer.succeed(method(*args, **kw)) + except: + f = Failure() + if self.reportAllTracebacks: + f.printTraceback() + return defer.fail(f) + + def remoteMethod(self, name): + return LocalMethod(self, name) + + +class LocalAsyncForwarder: + """ + A class useful for forwarding a locally-defined interface. + """ + + def __init__(self, forwarded, interfaceClass, failWhenNotImplemented=0): + assert interfaceClass.providedBy(forwarded) + self.forwarded = forwarded + self.interfaceClass = interfaceClass + self.failWhenNotImplemented = failWhenNotImplemented + + def _callMethod(self, method, *args, **kw): + return getattr(self.forwarded, method)(*args, **kw) + + def callRemote(self, method, *args, **kw): + if self.interfaceClass.queryDescriptionFor(method): + result = defer.maybeDeferred(self._callMethod, method, *args, **kw) + return result + elif self.failWhenNotImplemented: + return defer.fail( + Failure(NotImplementedError, + "No Such Method in Interface: %s" % method)) + else: + return defer.succeed(None) + + +class Pager: + """ + I am an object which pages out information. + """ + def __init__(self, collector, callback=None, *args, **kw): + """ + Create a pager with a Reference to a remote collector and + an optional callable to invoke upon completion. + """ + if callable(callback): + self.callback = callback + self.callbackArgs = args + self.callbackKeyword = kw + else: + self.callback = None + self._stillPaging = 1 + self.collector = collector + collector.broker.registerPageProducer(self) + + def stillPaging(self): + """ + (internal) Method called by Broker. + """ + if not self._stillPaging: + self.collector.callRemote("endedPaging", pbanswer=False) + if self.callback is not None: + self.callback(*self.callbackArgs, **self.callbackKeyword) + return self._stillPaging + + def sendNextPage(self): + """ + (internal) Method called by Broker. + """ + self.collector.callRemote("gotPage", self.nextPage(), pbanswer=False) + + def nextPage(self): + """ + Override this to return an object to be sent to my collector. + """ + raise NotImplementedError() + + def stopPaging(self): + """ + Call this when you're done paging. + """ + self._stillPaging = 0 + + +class StringPager(Pager): + """ + A simple pager that splits a string into chunks. + """ + def __init__(self, collector, st, chunkSize=8192, callback=None, *args, **kw): + self.string = st + self.pointer = 0 + self.chunkSize = chunkSize + Pager.__init__(self, collector, callback, *args, **kw) + + def nextPage(self): + val = self.string[self.pointer:self.pointer+self.chunkSize] + self.pointer += self.chunkSize + if self.pointer >= len(self.string): + self.stopPaging() + return val + + +@implementer(interfaces.IConsumer) +class FilePager(Pager): + """ + Reads a file in chunks and sends the chunks as they come. + """ + + def __init__(self, collector, fd, callback=None, *args, **kw): + self.chunks = [] + Pager.__init__(self, collector, callback, *args, **kw) + self.startProducing(fd) + + def startProducing(self, fd): + self.deferred = basic.FileSender().beginFileTransfer(fd, self) + self.deferred.addBoth(lambda x : self.stopPaging()) + + def registerProducer(self, producer, streaming): + self.producer = producer + if not streaming: + self.producer.resumeProducing() + + def unregisterProducer(self): + self.producer = None + + def write(self, chunk): + self.chunks.append(chunk) + + def sendNextPage(self): + """ + Get the first chunk read and send it to collector. + """ + if not self.chunks: + return + val = self.chunks.pop(0) + self.producer.resumeProducing() + self.collector.callRemote("gotPage", val, pbanswer=False) + + +# Utility paging stuff. +class CallbackPageCollector(pb.Referenceable): + """ + I receive pages from the peer. You may instantiate a Pager with a + remote reference to me. I will call the callback with a list of pages + once they are all received. + """ + def __init__(self, callback): + self.pages = [] + self.callback = callback + + def remote_gotPage(self, page): + self.pages.append(page) + + def remote_endedPaging(self): + self.callback(self.pages) + + +def getAllPages(referenceable, methodName, *args, **kw): + """ + A utility method that will call a remote method which expects a + PageCollector as the first argument. + """ + d = defer.Deferred() + referenceable.callRemote(methodName, CallbackPageCollector(d.callback), *args, **kw) + return d + diff --git a/contrib/python/Twisted/py2/twisted/tap/__init__.py b/contrib/python/Twisted/py2/twisted/tap/__init__.py new file mode 100644 index 00000000000..cdc430f94e2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/tap/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted TAP: Twisted Application Persistence builders for other Twisted servers. +""" diff --git a/contrib/python/Twisted/py2/twisted/tap/ftp.py b/contrib/python/Twisted/py2/twisted/tap/ftp.py new file mode 100644 index 00000000000..735ab4bd2e1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/tap/ftp.py @@ -0,0 +1,69 @@ +# -*- test-case-name: twisted.test.test_ftp_options -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +I am the support module for making a ftp server with twistd. +""" + +from twisted.application import internet +from twisted.cred import portal, checkers, strcred +from twisted.protocols import ftp + +from twisted.python import usage, deprecate, versions + +import warnings + + + +class Options(usage.Options, strcred.AuthOptionMixin): + synopsis = """[options]. + WARNING: This FTP server is probably INSECURE do not use it. + """ + optParameters = [ + ["port", "p", "2121", "set the port number"], + ["root", "r", "/usr/local/ftp", "define the root of the ftp-site."], + ["userAnonymous", "", "anonymous", "Name of the anonymous user."] + ] + + compData = usage.Completions( + optActions={"root": usage.CompleteDirs(descr="root of the ftp site")} + ) + + longdesc = '' + + def __init__(self, *a, **kw): + usage.Options.__init__(self, *a, **kw) + self.addChecker(checkers.AllowAnonymousAccess()) + + + def opt_password_file(self, filename): + """ + Specify a file containing username:password login info for + authenticated connections. (DEPRECATED; see --help-auth instead) + """ + self['password-file'] = filename + msg = deprecate.getDeprecationWarningString( + self.opt_password_file, versions.Version('Twisted', 11, 1, 0)) + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + self.addChecker(checkers.FilePasswordDB(filename, cache=True)) + + + +def makeService(config): + f = ftp.FTPFactory() + + r = ftp.FTPRealm(config['root']) + p = portal.Portal(r, config.get('credCheckers', [])) + + f.tld = config['root'] + f.userAnonymous = config['userAnonymous'] + f.portal = p + f.protocol = ftp.FTP + + try: + portno = int(config['port']) + except KeyError: + portno = 2121 + return internet.TCPServer(portno, f) diff --git a/contrib/python/Twisted/py2/twisted/tap/portforward.py b/contrib/python/Twisted/py2/twisted/tap/portforward.py new file mode 100644 index 00000000000..2ad3f3612f6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/tap/portforward.py @@ -0,0 +1,27 @@ + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support module for making a port forwarder with twistd. +""" +from twisted.protocols import portforward +from twisted.python import usage +from twisted.application import strports + +class Options(usage.Options): + synopsis = "[options]" + longdesc = 'Port Forwarder.' + optParameters = [ + ["port", "p", "6666","Set the port number."], + ["host", "h", "localhost","Set the host."], + ["dest_port", "d", 6665,"Set the destination port."], + ] + + compData = usage.Completions( + optActions={"host": usage.CompleteHostnames()} + ) + +def makeService(config): + f = portforward.ProxyFactory(config['host'], int(config['dest_port'])) + return strports.service(config['port'], f) diff --git a/contrib/python/Twisted/py2/twisted/tap/socks.py b/contrib/python/Twisted/py2/twisted/tap/socks.py new file mode 100644 index 00000000000..59ea85c8551 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/tap/socks.py @@ -0,0 +1,39 @@ + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +I am a support module for making SOCKSv4 servers with twistd. +""" +from __future__ import print_function + +from twisted.protocols import socks +from twisted.python import usage +from twisted.application import internet + + +class Options(usage.Options): + synopsis = "[-i ] [-p ] [-l ]" + optParameters = [["interface", "i", "127.0.0.1", "local interface to which we listen"], + ["port", "p", 1080, "Port on which to listen"], + ["log", "l", None, "file to log connection data to"]] + + compData = usage.Completions( + optActions={"log": usage.CompleteFiles("*.log"), + "interface": usage.CompleteNetInterfaces()} + ) + + longdesc = "Makes a SOCKSv4 server." + +def makeService(config): + if config["interface"] != "127.0.0.1": + print() + print("WARNING:") + print(" You have chosen to listen on a non-local interface.") + print(" This may allow intruders to access your local network") + print(" if you run this on a firewall.") + print() + t = socks.SOCKSv4Factory(config['log']) + portno = int(config['port']) + return internet.TCPServer(portno, t, interface=config['interface']) diff --git a/contrib/python/Twisted/py2/twisted/trial/__init__.py b/contrib/python/Twisted/py2/twisted/trial/__init__.py new file mode 100644 index 00000000000..5faaa99ddd2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/__init__.py @@ -0,0 +1,50 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +# +# Maintainer: Jonathan Lange + +""" +Twisted Trial: Asynchronous unit testing framework. + +Trial extends Python's builtin C{unittest} to provide support for asynchronous +tests. + +Trial strives to be compatible with other Python xUnit testing frameworks. +"Compatibility" is a difficult things to define. In practice, it means that: + + - L{twisted.trial.unittest.TestCase} objects should be able to be used by + other test runners without those runners requiring special support for + Trial tests. + + - Tests that subclass the standard library C{TestCase} and don't do anything + "too weird" should be able to be discoverable and runnable by the Trial + test runner without the authors of those tests having to jump through + hoops. + + - Tests that implement the interface provided by the standard library + C{TestCase} should be runnable by the Trial runner. + + - The Trial test runner and Trial L{unittest.TestCase} objects ought to be + able to use standard library C{TestResult} objects, and third party + C{TestResult} objects based on the standard library. + +This list is not necessarily exhaustive -- compatibility is hard to define. +Contributors who discover more helpful ways of defining compatibility are +encouraged to update this document. + + +Examples: + +B{Timeouts} for tests should be implemented in the runner. If this is done, +then timeouts could work for third-party TestCase objects as well as for +L{twisted.trial.unittest.TestCase} objects. Further, Twisted C{TestCase} +objects will run in other runners without timing out. +See U{http://twistedmatrix.com/trac/ticket/2675}. + +Running tests in a temporary directory should be a feature of the test case, +because often tests themselves rely on this behaviour. If the feature is +implemented in the runner, then tests will change behaviour (possibly +breaking) when run in a different test runner. Further, many tests don't even +care about the filesystem. +See U{http://twistedmatrix.com/trac/ticket/2916}. +""" diff --git a/contrib/python/Twisted/py2/twisted/trial/__main__.py b/contrib/python/Twisted/py2/twisted/trial/__main__.py new file mode 100644 index 00000000000..fe5534efd48 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/__main__.py @@ -0,0 +1,10 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +if __name__ == '__main__': + from pkg_resources import load_entry_point + import sys + + sys.exit( + load_entry_point('Twisted', 'console_scripts', 'trial')() + ) diff --git a/contrib/python/Twisted/py2/twisted/trial/_asyncrunner.py b/contrib/python/Twisted/py2/twisted/trial/_asyncrunner.py new file mode 100644 index 00000000000..15e4a1add5c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_asyncrunner.py @@ -0,0 +1,185 @@ +# -*- test-case-name: twisted.trial.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Infrastructure for test running and suites. +""" + +from __future__ import division, absolute_import + +import doctest +import gc + +from twisted.python import components + +from twisted.trial import itrial, reporter +from twisted.trial._synctest import _logObserver + +pyunit = __import__('unittest') + +from zope.interface import implementer + + + +class TestSuite(pyunit.TestSuite): + """ + Extend the standard library's C{TestSuite} with a consistently overrideable + C{run} method. + """ + + def run(self, result): + """ + Call C{run} on every member of the suite. + """ + for test in self._tests: + if result.shouldStop: + break + test(result) + return result + + + +@implementer(itrial.ITestCase) +class TestDecorator(components.proxyForInterface(itrial.ITestCase, + "_originalTest")): + """ + Decorator for test cases. + + @param _originalTest: The wrapped instance of test. + @type _originalTest: A provider of L{itrial.ITestCase} + """ + + def __call__(self, result): + """ + Run the unit test. + + @param result: A TestResult object. + """ + return self.run(result) + + + def run(self, result): + """ + Run the unit test. + + @param result: A TestResult object. + """ + return self._originalTest.run( + reporter._AdaptedReporter(result, self.__class__)) + + + +def _clearSuite(suite): + """ + Clear all tests from C{suite}. + + This messes with the internals of C{suite}. In particular, it assumes that + the suite keeps all of its tests in a list in an instance variable called + C{_tests}. + """ + suite._tests = [] + + + +def decorate(test, decorator): + """ + Decorate all test cases in C{test} with C{decorator}. + + C{test} can be a test case or a test suite. If it is a test suite, then the + structure of the suite is preserved. + + L{decorate} tries to preserve the class of the test suites it finds, but + assumes the presence of the C{_tests} attribute on the suite. + + @param test: The C{TestCase} or C{TestSuite} to decorate. + + @param decorator: A unary callable used to decorate C{TestCase}s. + + @return: A decorated C{TestCase} or a C{TestSuite} containing decorated + C{TestCase}s. + """ + + try: + tests = iter(test) + except TypeError: + return decorator(test) + + # At this point, we know that 'test' is a test suite. + _clearSuite(test) + + for case in tests: + test.addTest(decorate(case, decorator)) + return test + + + +class _PyUnitTestCaseAdapter(TestDecorator): + """ + Adapt from pyunit.TestCase to ITestCase. + """ + + + +class _BrokenIDTestCaseAdapter(_PyUnitTestCaseAdapter): + """ + Adapter for pyunit-style C{TestCase} subclasses that have undesirable id() + methods. That is C{unittest.FunctionTestCase} and C{unittest.DocTestCase}. + """ + + def id(self): + """ + Return the fully-qualified Python name of the doctest. + """ + testID = self._originalTest.shortDescription() + if testID is not None: + return testID + return self._originalTest.id() + + + +class _ForceGarbageCollectionDecorator(TestDecorator): + """ + Forces garbage collection to be run before and after the test. Any errors + logged during the post-test collection are added to the test result as + errors. + """ + + def run(self, result): + gc.collect() + TestDecorator.run(self, result) + _logObserver._add() + gc.collect() + for error in _logObserver.getErrors(): + result.addError(self, error) + _logObserver.flushErrors() + _logObserver._remove() + + +components.registerAdapter( + _PyUnitTestCaseAdapter, pyunit.TestCase, itrial.ITestCase) + + +components.registerAdapter( + _BrokenIDTestCaseAdapter, pyunit.FunctionTestCase, itrial.ITestCase) + + +_docTestCase = getattr(doctest, 'DocTestCase', None) +if _docTestCase: + components.registerAdapter( + _BrokenIDTestCaseAdapter, _docTestCase, itrial.ITestCase) + + + +def _iterateTests(testSuiteOrCase): + """ + Iterate through all of the test cases in C{testSuiteOrCase}. + """ + try: + suite = iter(testSuiteOrCase) + except TypeError: + yield testSuiteOrCase + else: + for test in suite: + for subtest in _iterateTests(test): + yield subtest diff --git a/contrib/python/Twisted/py2/twisted/trial/_asynctest.py b/contrib/python/Twisted/py2/twisted/trial/_asynctest.py new file mode 100644 index 00000000000..c7a7090cf39 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_asynctest.py @@ -0,0 +1,405 @@ +# -*- test-case-name: twisted.trial.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Things likely to be used by writers of unit tests. + +Maintainer: Jonathan Lange +""" + +from __future__ import division, absolute_import + +import inspect +import warnings + +from zope.interface import implementer + +# We can't import reactor at module-level because this code runs before trial +# installs a user-specified reactor, installing the default reactor and +# breaking reactor installation. See also #6047. +from twisted.internet import defer, utils +from twisted.python import failure + +from twisted.trial import itrial, util +from twisted.trial._synctest import ( + FailTest, SkipTest, SynchronousTestCase) + +_wait_is_running = [] + +@implementer(itrial.ITestCase) +class TestCase(SynchronousTestCase): + """ + A unit test. The atom of the unit testing universe. + + This class extends L{SynchronousTestCase} which extends C{unittest.TestCase} + from the standard library. The main feature is the ability to return + C{Deferred}s from tests and fixture methods and to have the suite wait for + those C{Deferred}s to fire. Also provides new assertions such as + L{assertFailure}. + + @ivar timeout: A real number of seconds. If set, the test will + raise an error if it takes longer than C{timeout} seconds. + If not set, util.DEFAULT_TIMEOUT_DURATION is used. + """ + + def __init__(self, methodName='runTest'): + """ + Construct an asynchronous test case for C{methodName}. + + @param methodName: The name of a method on C{self}. This method should + be a unit test. That is, it should be a short method that calls some of + the assert* methods. If C{methodName} is unspecified, + L{SynchronousTestCase.runTest} will be used as the test method. This is + mostly useful for testing Trial. + """ + super(TestCase, self).__init__(methodName) + + + def assertFailure(self, deferred, *expectedFailures): + """ + Fail if C{deferred} does not errback with one of C{expectedFailures}. + Returns the original Deferred with callbacks added. You will need + to return this Deferred from your test case. + """ + def _cb(ignore): + raise self.failureException( + "did not catch an error, instead got %r" % (ignore,)) + + def _eb(failure): + if failure.check(*expectedFailures): + return failure.value + else: + output = ('\nExpected: %r\nGot:\n%s' + % (expectedFailures, str(failure))) + raise self.failureException(output) + return deferred.addCallbacks(_cb, _eb) + failUnlessFailure = assertFailure + + + def _run(self, methodName, result): + from twisted.internet import reactor + timeout = self.getTimeout() + def onTimeout(d): + e = defer.TimeoutError("%r (%s) still running at %s secs" + % (self, methodName, timeout)) + f = failure.Failure(e) + # try to errback the deferred that the test returns (for no gorram + # reason) (see issue1005 and test_errorPropagation in + # test_deferred) + try: + d.errback(f) + except defer.AlreadyCalledError: + # if the deferred has been called already but the *back chain + # is still unfinished, crash the reactor and report timeout + # error ourself. + reactor.crash() + self._timedOut = True # see self._wait + todo = self.getTodo() + if todo is not None and todo.expected(f): + result.addExpectedFailure(self, f, todo) + else: + result.addError(self, f) + onTimeout = utils.suppressWarnings( + onTimeout, util.suppress(category=DeprecationWarning)) + method = getattr(self, methodName) + if inspect.isgeneratorfunction(method): + exc = TypeError( + '%r is a generator function and therefore will never run' % ( + method,)) + return defer.fail(exc) + d = defer.maybeDeferred( + utils.runWithWarningsSuppressed, self._getSuppress(), method) + call = reactor.callLater(timeout, onTimeout, d) + d.addBoth(lambda x : call.active() and call.cancel() or x) + return d + + + def __call__(self, *args, **kwargs): + return self.run(*args, **kwargs) + + + def deferSetUp(self, ignored, result): + d = self._run('setUp', result) + d.addCallbacks(self.deferTestMethod, self._ebDeferSetUp, + callbackArgs=(result,), + errbackArgs=(result,)) + return d + + + def _ebDeferSetUp(self, failure, result): + if failure.check(SkipTest): + result.addSkip(self, self._getSkipReason(self.setUp, failure.value)) + else: + result.addError(self, failure) + if failure.check(KeyboardInterrupt): + result.stop() + return self.deferRunCleanups(None, result) + + + def deferTestMethod(self, ignored, result): + d = self._run(self._testMethodName, result) + d.addCallbacks(self._cbDeferTestMethod, self._ebDeferTestMethod, + callbackArgs=(result,), + errbackArgs=(result,)) + d.addBoth(self.deferRunCleanups, result) + d.addBoth(self.deferTearDown, result) + return d + + + def _cbDeferTestMethod(self, ignored, result): + if self.getTodo() is not None: + result.addUnexpectedSuccess(self, self.getTodo()) + else: + self._passed = True + return ignored + + + def _ebDeferTestMethod(self, f, result): + todo = self.getTodo() + if todo is not None and todo.expected(f): + result.addExpectedFailure(self, f, todo) + elif f.check(self.failureException, FailTest): + result.addFailure(self, f) + elif f.check(KeyboardInterrupt): + result.addError(self, f) + result.stop() + elif f.check(SkipTest): + result.addSkip( + self, + self._getSkipReason(getattr(self, self._testMethodName), f.value)) + else: + result.addError(self, f) + + + def deferTearDown(self, ignored, result): + d = self._run('tearDown', result) + d.addErrback(self._ebDeferTearDown, result) + return d + + + def _ebDeferTearDown(self, failure, result): + result.addError(self, failure) + if failure.check(KeyboardInterrupt): + result.stop() + self._passed = False + + + def deferRunCleanups(self, ignored, result): + """ + Run any scheduled cleanups and report errors (if any to the result + object. + """ + d = self._runCleanups() + d.addCallback(self._cbDeferRunCleanups, result) + return d + + + def _cbDeferRunCleanups(self, cleanupResults, result): + for flag, testFailure in cleanupResults: + if flag == defer.FAILURE: + result.addError(self, testFailure) + if testFailure.check(KeyboardInterrupt): + result.stop() + self._passed = False + + + def _cleanUp(self, result): + try: + clean = util._Janitor(self, result).postCaseCleanup() + if not clean: + self._passed = False + except: + result.addError(self, failure.Failure()) + self._passed = False + for error in self._observer.getErrors(): + result.addError(self, error) + self._passed = False + self.flushLoggedErrors() + self._removeObserver() + if self._passed: + result.addSuccess(self) + + + def _classCleanUp(self, result): + try: + util._Janitor(self, result).postClassCleanup() + except: + result.addError(self, failure.Failure()) + + + def _makeReactorMethod(self, name): + """ + Create a method which wraps the reactor method C{name}. The new + method issues a deprecation warning and calls the original. + """ + def _(*a, **kw): + warnings.warn("reactor.%s cannot be used inside unit tests. " + "In the future, using %s will fail the test and may " + "crash or hang the test run." + % (name, name), + stacklevel=2, category=DeprecationWarning) + return self._reactorMethods[name](*a, **kw) + return _ + + + def _deprecateReactor(self, reactor): + """ + Deprecate C{iterate}, C{crash} and C{stop} on C{reactor}. That is, + each method is wrapped in a function that issues a deprecation + warning, then calls the original. + + @param reactor: The Twisted reactor. + """ + self._reactorMethods = {} + for name in ['crash', 'iterate', 'stop']: + self._reactorMethods[name] = getattr(reactor, name) + setattr(reactor, name, self._makeReactorMethod(name)) + + + def _undeprecateReactor(self, reactor): + """ + Restore the deprecated reactor methods. Undoes what + L{_deprecateReactor} did. + + @param reactor: The Twisted reactor. + """ + for name, method in self._reactorMethods.items(): + setattr(reactor, name, method) + self._reactorMethods = {} + + + def _runCleanups(self): + """ + Run the cleanups added with L{addCleanup} in order. + + @return: A C{Deferred} that fires when all cleanups are run. + """ + def _makeFunction(f, args, kwargs): + return lambda: f(*args, **kwargs) + callables = [] + while len(self._cleanups) > 0: + f, args, kwargs = self._cleanups.pop() + callables.append(_makeFunction(f, args, kwargs)) + return util._runSequentially(callables) + + + def _runFixturesAndTest(self, result): + """ + Really run C{setUp}, the test method, and C{tearDown}. Any of these may + return L{defer.Deferred}s. After they complete, do some reactor cleanup. + + @param result: A L{TestResult} object. + """ + from twisted.internet import reactor + self._deprecateReactor(reactor) + self._timedOut = False + try: + d = self.deferSetUp(None, result) + try: + self._wait(d) + finally: + self._cleanUp(result) + self._classCleanUp(result) + finally: + self._undeprecateReactor(reactor) + + + def addCleanup(self, f, *args, **kwargs): + """ + Extend the base cleanup feature with support for cleanup functions which + return Deferreds. + + If the function C{f} returns a Deferred, C{TestCase} will wait until the + Deferred has fired before proceeding to the next function. + """ + return super(TestCase, self).addCleanup(f, *args, **kwargs) + + + def getSuppress(self): + return self._getSuppress() + + + def getTimeout(self): + """ + Returns the timeout value set on this test. Checks on the instance + first, then the class, then the module, then packages. As soon as it + finds something with a C{timeout} attribute, returns that. Returns + L{util.DEFAULT_TIMEOUT_DURATION} if it cannot find anything. See + L{TestCase} docstring for more details. + """ + timeout = util.acquireAttribute(self._parents, 'timeout', + util.DEFAULT_TIMEOUT_DURATION) + try: + return float(timeout) + except (ValueError, TypeError): + # XXX -- this is here because sometimes people will have methods + # called 'timeout', or set timeout to 'orange', or something + # Particularly, test_news.NewsTestCase and ReactorCoreTestCase + # both do this. + warnings.warn("'timeout' attribute needs to be a number.", + category=DeprecationWarning) + return util.DEFAULT_TIMEOUT_DURATION + + + def _wait(self, d, running=_wait_is_running): + """Take a Deferred that only ever callbacks. Block until it happens. + """ + if running: + raise RuntimeError("_wait is not reentrant") + + from twisted.internet import reactor + results = [] + def append(any): + if results is not None: + results.append(any) + def crash(ign): + if results is not None: + reactor.crash() + crash = utils.suppressWarnings( + crash, util.suppress(message=r'reactor\.crash cannot be used.*', + category=DeprecationWarning)) + def stop(): + reactor.crash() + stop = utils.suppressWarnings( + stop, util.suppress(message=r'reactor\.crash cannot be used.*', + category=DeprecationWarning)) + + running.append(None) + try: + d.addBoth(append) + if results: + # d might have already been fired, in which case append is + # called synchronously. Avoid any reactor stuff. + return + d.addBoth(crash) + reactor.stop = stop + try: + reactor.run() + finally: + del reactor.stop + + # If the reactor was crashed elsewhere due to a timeout, hopefully + # that crasher also reported an error. Just return. + # _timedOut is most likely to be set when d has fired but hasn't + # completed its callback chain (see self._run) + if results or self._timedOut: #defined in run() and _run() + return + + # If the timeout didn't happen, and we didn't get a result or + # a failure, then the user probably aborted the test, so let's + # just raise KeyboardInterrupt. + + # FIXME: imagine this: + # web/test/test_webclient.py: + # exc = self.assertRaises(error.Error, wait, method(url)) + # + # wait() will raise KeyboardInterrupt, and assertRaises will + # swallow it. Therefore, wait() raising KeyboardInterrupt is + # insufficient to stop trial. A suggested solution is to have + # this code set a "stop trial" flag, or otherwise notify trial + # that it should really try to stop as soon as possible. + raise KeyboardInterrupt() + finally: + results = None + running.pop() diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/__init__.py b/contrib/python/Twisted/py2/twisted/trial/_dist/__init__.py new file mode 100644 index 00000000000..502e840fef7 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/__init__.py @@ -0,0 +1,47 @@ +# -*- test-case-name: twisted.trial._dist.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This package implements the distributed Trial test runner: + + - The L{twisted.trial._dist.disttrial} module implements a test runner which + runs in a manager process and can launch additional worker processes in + which to run tests and gather up results from all of them. + + - The L{twisted.trial._dist.options} module defines command line options used + to configure the distributed test runner. + + - The L{twisted.trial._dist.managercommands} module defines AMP commands + which are sent from worker processes back to the manager process to report + the results of tests. + + - The L{twisted.trial._dist.workercommands} module defines AMP commands which + are sent from the manager process to the worker processes to control the + execution of tests there. + + - The L{twisted.trial._dist.distreporter} module defines a proxy for + L{twisted.trial.itrial.IReporter} which enforces the typical requirement + that results be passed to a reporter for only one test at a time, allowing + any reporter to be used with despite disttrial's simultaneously running + tests. + + - The L{twisted.trial._dist.workerreporter} module implements a + L{twisted.trial.itrial.IReporter} which is used by worker processes and + reports results back to the manager process using AMP commands. + + - The L{twisted.trial._dist.workertrial} module is a runnable script which is + the main point for worker processes. + + - The L{twisted.trial._dist.worker} process defines the manager's AMP + protocol for accepting results from worker processes and a process protocol + for use running workers as local child processes (as opposed to + distributing them to another host). + +@since: 12.3 +""" + +# File descriptors numbers used to set up pipes with the worker. +_WORKER_AMP_STDIN = 3 + +_WORKER_AMP_STDOUT = 4 diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/distreporter.py b/contrib/python/Twisted/py2/twisted/trial/_dist/distreporter.py new file mode 100644 index 00000000000..6648b7a0aac --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/distreporter.py @@ -0,0 +1,93 @@ +# -*- test-case-name: twisted.trial._dist.test.test_distreporter -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +The reporter is not made to support concurrent test running, so we will +hold test results in here and only send them to the reporter once the +test is over. + +@since: 12.3 +""" + +from zope.interface import implementer +from twisted.trial.itrial import IReporter +from twisted.python.components import proxyForInterface + + + +@implementer(IReporter) +class DistReporter(proxyForInterface(IReporter)): + """ + See module docstring. + """ + + def __init__(self, original): + super(DistReporter, self).__init__(original) + self.running = {} + + + def startTest(self, test): + """ + Queue test starting. + """ + self.running[test.id()] = [] + self.running[test.id()].append((self.original.startTest, test)) + + + def addFailure(self, test, fail): + """ + Queue adding a failure. + """ + self.running[test.id()].append((self.original.addFailure, + test, fail)) + + + def addError(self, test, error): + """ + Queue error adding. + """ + self.running[test.id()].append((self.original.addError, + test, error)) + + + def addSkip(self, test, reason): + """ + Queue adding a skip. + """ + self.running[test.id()].append((self.original.addSkip, + test, reason)) + + + def addUnexpectedSuccess(self, test, todo=None): + """ + Queue adding an unexpected success. + """ + self.running[test.id()].append((self.original.addUnexpectedSuccess, + test, todo)) + + + def addExpectedFailure(self, test, error, todo=None): + """ + Queue adding an unexpected failure. + """ + self.running[test.id()].append((self.original.addExpectedFailure, + test, error, todo)) + + + def addSuccess(self, test): + """ + Queue adding a success. + """ + self.running[test.id()].append((self.original.addSuccess, test)) + + + def stopTest(self, test): + """ + Queue stopping the test, then unroll the queue. + """ + self.running[test.id()].append((self.original.stopTest, test)) + for step in self.running[test.id()]: + step[0](*step[1:]) + del self.running[test.id()] diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/disttrial.py b/contrib/python/Twisted/py2/twisted/trial/_dist/disttrial.py new file mode 100644 index 00000000000..96875b4a85d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/disttrial.py @@ -0,0 +1,258 @@ +# -*- test-case-name: twisted.trial._dist.test.test_disttrial -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module contains the trial distributed runner, the management class +responsible for coordinating all of trial's behavior at the highest level. + +@since: 12.3 +""" + +import os +import sys + +from twisted.python.filepath import FilePath +from twisted.python.modules import theSystemPath +from twisted.internet.defer import DeferredList +from twisted.internet.task import cooperate + +from twisted.trial.util import _unusedTestDirectory +from twisted.trial._asyncrunner import _iterateTests +from twisted.trial._dist.worker import LocalWorker, LocalWorkerAMP +from twisted.trial._dist.distreporter import DistReporter +from twisted.trial.reporter import UncleanWarningsReporterWrapper +from twisted.trial._dist import _WORKER_AMP_STDIN, _WORKER_AMP_STDOUT + + + +class DistTrialRunner(object): + """ + A specialized runner for distributed trial. The runner launches a number of + local worker processes which will run tests. + + @ivar _workerNumber: the number of workers to be spawned. + @type _workerNumber: C{int} + + @ivar _stream: stream which the reporter will use. + + @ivar _reporterFactory: the reporter class to be used. + """ + _distReporterFactory = DistReporter + + def _makeResult(self): + """ + Make reporter factory, and wrap it with a L{DistReporter}. + """ + reporter = self._reporterFactory(self._stream, self._tbformat, + realtime=self._rterrors) + if self._uncleanWarnings: + reporter = UncleanWarningsReporterWrapper(reporter) + return self._distReporterFactory(reporter) + + + def __init__(self, reporterFactory, workerNumber, workerArguments, + stream=None, + tracebackFormat='default', + realTimeErrors=False, + uncleanWarnings=False, + logfile='test.log', + workingDirectory='_trial_temp'): + self._workerNumber = workerNumber + self._workerArguments = workerArguments + self._reporterFactory = reporterFactory + if stream is None: + stream = sys.stdout + self._stream = stream + self._tbformat = tracebackFormat + self._rterrors = realTimeErrors + self._uncleanWarnings = uncleanWarnings + self._result = None + self._workingDirectory = workingDirectory + self._logFile = logfile + self._logFileObserver = None + self._logFileObject = None + self._logWarnings = False + + + def writeResults(self, result): + """ + Write test run final outcome to result. + + @param result: A C{TestResult} which will print errors and the summary. + """ + result.done() + + + def createLocalWorkers(self, protocols, workingDirectory): + """ + Create local worker protocol instances and return them. + + @param protocols: An iterable of L{LocalWorkerAMP} instances. + + @param workingDirectory: The base path in which we should run the + workers. + @type workingDirectory: C{str} + + @return: A list of C{quantity} C{LocalWorker} instances. + """ + return [LocalWorker(protocol, + os.path.join(workingDirectory, str(x)), + self._logFile) + for x, protocol in enumerate(protocols)] + + + def launchWorkerProcesses(self, spawner, protocols, arguments): + """ + Spawn processes from a list of process protocols. + + @param spawner: A C{IReactorProcess.spawnProcess} implementation. + + @param protocols: An iterable of C{ProcessProtocol} instances. + + @param arguments: Extra arguments passed to the processes. + """ + workertrialPath = theSystemPath[ + 'twisted.trial._dist.workertrial'].filePath.path + childFDs = {0: 'w', 1: 'r', 2: 'r', _WORKER_AMP_STDIN: 'w', + _WORKER_AMP_STDOUT: 'r'} + environ = os.environ.copy() + # Add an environment variable containing the raw sys.path, to be used by + # subprocesses to make sure it's identical to the parent. See + # workertrial._setupPath. + environ['TRIAL_PYTHONPATH'] = os.pathsep.join(sys.path) + for worker in protocols: + args = [sys.executable, workertrialPath] + args.extend(arguments) + spawner(worker, sys.executable, args=args, childFDs=childFDs, + env=environ) + + + def _driveWorker(self, worker, result, testCases, cooperate): + """ + Drive a L{LocalWorkerAMP} instance, iterating the tests and calling + C{run} for every one of them. + + @param worker: The L{LocalWorkerAMP} to drive. + + @param result: The global L{DistReporter} instance. + + @param testCases: The global list of tests to iterate. + + @param cooperate: The cooperate function to use, to be customized in + tests. + @type cooperate: C{function} + + @return: A C{Deferred} firing when all the tests are finished. + """ + + def resultErrback(error, case): + result.original.addFailure(case, error) + return error + + def task(case): + d = worker.run(case, result) + d.addErrback(resultErrback, case) + return d + + return cooperate(task(case) for case in testCases).whenDone() + + + def run(self, suite, reactor=None, cooperate=cooperate, + untilFailure=False): + """ + Spawn local worker processes and load tests. After that, run them. + + @param suite: A tests suite to be run. + + @param reactor: The reactor to use, to be customized in tests. + @type reactor: A provider of + L{twisted.internet.interfaces.IReactorProcess} + + @param cooperate: The cooperate function to use, to be customized in + tests. + @type cooperate: C{function} + + @param untilFailure: If C{True}, continue to run the tests until they + fail. + @type untilFailure: C{bool}. + + @return: The test result. + @rtype: L{DistReporter} + """ + if reactor is None: + from twisted.internet import reactor + result = self._makeResult() + count = suite.countTestCases() + self._stream.write("Running %d tests.\n" % (count,)) + + if not count: + # Take a shortcut if there is no test + suite.run(result.original) + self.writeResults(result) + return result + + testDir, testDirLock = _unusedTestDirectory( + FilePath(self._workingDirectory)) + workerNumber = min(count, self._workerNumber) + ampWorkers = [LocalWorkerAMP() for x in range(workerNumber)] + workers = self.createLocalWorkers(ampWorkers, testDir.path) + processEndDeferreds = [worker.endDeferred for worker in workers] + self.launchWorkerProcesses(reactor.spawnProcess, workers, + self._workerArguments) + + def runTests(): + testCases = iter(list(_iterateTests(suite))) + + workerDeferreds = [] + for worker in ampWorkers: + workerDeferreds.append( + self._driveWorker(worker, result, testCases, + cooperate=cooperate)) + return DeferredList(workerDeferreds, consumeErrors=True, + fireOnOneErrback=True) + + stopping = [] + + def nextRun(ign): + self.writeResults(result) + if not untilFailure: + return + if not result.wasSuccessful(): + return + d = runTests() + return d.addCallback(nextRun) + + def stop(ign): + testDirLock.unlock() + if not stopping: + stopping.append(None) + reactor.stop() + + def beforeShutDown(): + if not stopping: + stopping.append(None) + d = DeferredList(processEndDeferreds, consumeErrors=True) + return d.addCallback(continueShutdown) + + def continueShutdown(ign): + self.writeResults(result) + return ign + + d = runTests() + d.addCallback(nextRun) + d.addBoth(stop) + + reactor.addSystemEventTrigger('before', 'shutdown', beforeShutDown) + reactor.run() + + return result + + + def runUntilFailure(self, suite): + """ + Run the tests with local worker processes until they fail. + + @param suite: A tests suite to be run. + """ + return self.run(suite, untilFailure=True) diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/managercommands.py b/contrib/python/Twisted/py2/twisted/trial/_dist/managercommands.py new file mode 100644 index 00000000000..4e76b81ddf1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/managercommands.py @@ -0,0 +1,86 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Commands for reporting test success of failure to the manager. + +@since: 12.3 +""" + +from twisted.protocols.amp import Command, String, Boolean, ListOf, Unicode +from twisted.python.compat import _PY3 + +NativeString = Unicode if _PY3 else String + + + +class AddSuccess(Command): + """ + Add a success. + """ + arguments = [(b'testName', NativeString())] + response = [(b'success', Boolean())] + + + +class AddError(Command): + """ + Add an error. + """ + arguments = [(b'testName', NativeString()), + (b'error', NativeString()), + (b'errorClass', NativeString()), + (b'frames', ListOf(NativeString()))] + response = [(b'success', Boolean())] + + + +class AddFailure(Command): + """ + Add a failure. + """ + arguments = [(b'testName', NativeString()), + (b'fail', NativeString()), + (b'failClass', NativeString()), + (b'frames', ListOf(NativeString()))] + response = [(b'success', Boolean())] + + + +class AddSkip(Command): + """ + Add a skip. + """ + arguments = [(b'testName', NativeString()), + (b'reason', NativeString())] + response = [(b'success', Boolean())] + + + +class AddExpectedFailure(Command): + """ + Add an expected failure. + """ + arguments = [(b'testName', NativeString()), + (b'error', NativeString()), + (b'todo', NativeString())] + response = [(b'success', Boolean())] + + + +class AddUnexpectedSuccess(Command): + """ + Add an unexpected success. + """ + arguments = [(b'testName', NativeString()), + (b'todo', NativeString())] + response = [(b'success', Boolean())] + + + +class TestWrite(Command): + """ + Write test log. + """ + arguments = [(b'out', NativeString())] + response = [(b'success', Boolean())] diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/options.py b/contrib/python/Twisted/py2/twisted/trial/_dist/options.py new file mode 100644 index 00000000000..ee5ccd887ab --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/options.py @@ -0,0 +1,30 @@ +# -*- test-case-name: twisted.trial._dist.test.test_options -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Options handling specific to trial's workers. + +@since: 12.3 +""" + +from twisted.python.filepath import FilePath +from twisted.python.usage import Options +from twisted.scripts.trial import _BasicOptions +from twisted.application.app import ReactorSelectionMixin + + + +class WorkerOptions(_BasicOptions, Options, ReactorSelectionMixin): + """ + Options forwarded to the trial distributed worker. + """ + + + def coverdir(self): + """ + Return a L{FilePath} representing the directory into which coverage + results should be written. + """ + return FilePath('coverage') diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/worker.py b/contrib/python/Twisted/py2/twisted/trial/_dist/worker.py new file mode 100644 index 00000000000..ef13c069d64 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/worker.py @@ -0,0 +1,333 @@ +# -*- test-case-name: twisted.trial._dist.test.test_worker -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +This module implements the worker classes. + +@since: 12.3 +""" + +import os + +from zope.interface import implementer + +from twisted.internet.protocol import ProcessProtocol +from twisted.internet.interfaces import ITransport, IAddress +from twisted.internet.defer import Deferred +from twisted.protocols.amp import AMP +from twisted.python.failure import Failure +from twisted.python.reflect import namedObject +from twisted.trial.unittest import Todo +from twisted.trial.runner import TrialSuite, TestLoader +from twisted.trial._dist import workercommands, managercommands +from twisted.trial._dist import _WORKER_AMP_STDIN, _WORKER_AMP_STDOUT +from twisted.trial._dist.workerreporter import WorkerReporter + + + +class WorkerProtocol(AMP): + """ + The worker-side trial distributed protocol. + """ + + def __init__(self, forceGarbageCollection=False): + self._loader = TestLoader() + self._result = WorkerReporter(self) + self._forceGarbageCollection = forceGarbageCollection + + + def run(self, testCase): + """ + Run a test case by name. + """ + case = self._loader.loadByName(testCase) + suite = TrialSuite([case], self._forceGarbageCollection) + suite.run(self._result) + return {'success': True} + + workercommands.Run.responder(run) + + + def start(self, directory): + """ + Set up the worker, moving into given directory for tests to run in + them. + """ + os.chdir(directory) + return {'success': True} + + workercommands.Start.responder(start) + + + +class LocalWorkerAMP(AMP): + """ + Local implementation of the manager commands. + """ + + def addSuccess(self, testName): + """ + Add a success to the reporter. + """ + self._result.addSuccess(self._testCase) + return {'success': True} + + managercommands.AddSuccess.responder(addSuccess) + + + def _buildFailure(self, error, errorClass, frames): + """ + Helper to build a C{Failure} with some traceback. + + @param error: An C{Exception} instance. + + @param error: The class name of the C{error} class. + + @param frames: A flat list of strings representing the information need + to approximatively rebuild C{Failure} frames. + + @return: A L{Failure} instance with enough information about a test + error. + """ + errorType = namedObject(errorClass) + failure = Failure(error, errorType) + for i in range(0, len(frames), 3): + failure.frames.append( + (frames[i], frames[i + 1], int(frames[i + 2]), [], [])) + return failure + + + def addError(self, testName, error, errorClass, frames): + """ + Add an error to the reporter. + """ + failure = self._buildFailure(error, errorClass, frames) + self._result.addError(self._testCase, failure) + return {'success': True} + + managercommands.AddError.responder(addError) + + + def addFailure(self, testName, fail, failClass, frames): + """ + Add a failure to the reporter. + """ + failure = self._buildFailure(fail, failClass, frames) + self._result.addFailure(self._testCase, failure) + return {'success': True} + + managercommands.AddFailure.responder(addFailure) + + + def addSkip(self, testName, reason): + """ + Add a skip to the reporter. + """ + self._result.addSkip(self._testCase, reason) + return {'success': True} + + managercommands.AddSkip.responder(addSkip) + + + def addExpectedFailure(self, testName, error, todo): + """ + Add an expected failure to the reporter. + """ + _todo = Todo(todo) + self._result.addExpectedFailure(self._testCase, error, _todo) + return {'success': True} + + managercommands.AddExpectedFailure.responder(addExpectedFailure) + + + def addUnexpectedSuccess(self, testName, todo): + """ + Add an unexpected success to the reporter. + """ + self._result.addUnexpectedSuccess(self._testCase, todo) + return {'success': True} + + managercommands.AddUnexpectedSuccess.responder(addUnexpectedSuccess) + + + def testWrite(self, out): + """ + Print test output from the worker. + """ + self._testStream.write(out + '\n') + self._testStream.flush() + return {'success': True} + + managercommands.TestWrite.responder(testWrite) + + + def _stopTest(self, result): + """ + Stop the current running test case, forwarding the result. + """ + self._result.stopTest(self._testCase) + return result + + + def run(self, testCase, result): + """ + Run a test. + """ + self._testCase = testCase + self._result = result + self._result.startTest(testCase) + testCaseId = testCase.id() + d = self.callRemote(workercommands.Run, testCase=testCaseId) + return d.addCallback(self._stopTest) + + + def setTestStream(self, stream): + """ + Set the stream used to log output from tests. + """ + self._testStream = stream + + + +@implementer(IAddress) +class LocalWorkerAddress(object): + """ + A L{IAddress} implementation meant to provide stub addresses for + L{ITransport.getPeer} and L{ITransport.getHost}. + """ + + + +@implementer(ITransport) +class LocalWorkerTransport(object): + """ + A stub transport implementation used to support L{AMP} over a + L{ProcessProtocol} transport. + """ + + def __init__(self, transport): + self._transport = transport + + + def write(self, data): + """ + Forward data to transport. + """ + self._transport.writeToChild(_WORKER_AMP_STDIN, data) + + + def writeSequence(self, sequence): + """ + Emulate C{writeSequence} by iterating data in the C{sequence}. + """ + for data in sequence: + self._transport.writeToChild(_WORKER_AMP_STDIN, data) + + + def loseConnection(self): + """ + Closes the transport. + """ + self._transport.loseConnection() + + + def getHost(self): + """ + Return a L{LocalWorkerAddress} instance. + """ + return LocalWorkerAddress() + + + def getPeer(self): + """ + Return a L{LocalWorkerAddress} instance. + """ + return LocalWorkerAddress() + + + +class LocalWorker(ProcessProtocol): + """ + Local process worker protocol. This worker runs as a local process and + communicates via stdin/out. + + @ivar _ampProtocol: The L{AMP} protocol instance used to communicate with + the worker. + + @ivar _logDirectory: The directory where logs will reside. + + @ivar _logFile: The name of the main log file for tests output. + """ + + def __init__(self, ampProtocol, logDirectory, logFile): + self._ampProtocol = ampProtocol + self._logDirectory = logDirectory + self._logFile = logFile + self.endDeferred = Deferred() + + + def connectionMade(self): + """ + When connection is made, create the AMP protocol instance. + """ + self._ampProtocol.makeConnection(LocalWorkerTransport(self.transport)) + if not os.path.exists(self._logDirectory): + os.makedirs(self._logDirectory) + self._outLog = open(os.path.join(self._logDirectory, 'out.log'), 'wb') + self._errLog = open(os.path.join(self._logDirectory, 'err.log'), 'wb') + self._testLog = open( + os.path.join(self._logDirectory, self._logFile), 'w') + self._ampProtocol.setTestStream(self._testLog) + logDirectory = self._logDirectory + d = self._ampProtocol.callRemote(workercommands.Start, + directory=logDirectory) + # Ignore the potential errors, the test suite will fail properly and it + # would just print garbage. + d.addErrback(lambda x: None) + + + def connectionLost(self, reason): + """ + On connection lost, close the log files that we're managing for stdin + and stdout. + """ + self._outLog.close() + self._errLog.close() + self._testLog.close() + + + def processEnded(self, reason): + """ + When the process closes, call C{connectionLost} for cleanup purposes + and forward the information to the C{_ampProtocol}. + """ + self.connectionLost(reason) + self._ampProtocol.connectionLost(reason) + self.endDeferred.callback(reason) + + + def outReceived(self, data): + """ + Send data received from stdout to log. + """ + + self._outLog.write(data) + + + def errReceived(self, data): + """ + Write error data to log. + """ + self._errLog.write(data) + + + def childDataReceived(self, childFD, data): + """ + Handle data received on the specific pipe for the C{_ampProtocol}. + """ + if childFD == _WORKER_AMP_STDOUT: + self._ampProtocol.dataReceived(data) + else: + ProcessProtocol.childDataReceived(self, childFD, data) diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/workercommands.py b/contrib/python/Twisted/py2/twisted/trial/_dist/workercommands.py new file mode 100644 index 00000000000..517cd67025c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/workercommands.py @@ -0,0 +1,31 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Commands for telling a worker to load tests or run tests. + +@since: 12.3 +""" + +from twisted.protocols.amp import Command, String, Boolean, Unicode +from twisted.python.compat import _PY3 + +NativeString = Unicode if _PY3 else String + + + +class Run(Command): + """ + Run a test. + """ + arguments = [(b'testCase', NativeString())] + response = [(b'success', Boolean())] + + + +class Start(Command): + """ + Set up the worker process, giving the running directory. + """ + arguments = [(b'directory', NativeString())] + response = [(b'success', Boolean())] diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/workerreporter.py b/contrib/python/Twisted/py2/twisted/trial/_dist/workerreporter.py new file mode 100644 index 00000000000..ce82c599941 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/workerreporter.py @@ -0,0 +1,154 @@ +# -*- test-case-name: twisted.trial._dist.test.test_workerreporter -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Test reporter forwarding test results over trial distributed AMP commands. + +@since: 12.3 +""" + +from twisted.python.failure import Failure +from twisted.python.reflect import qual +from twisted.trial.reporter import TestResult +from twisted.trial._dist import managercommands + + + +class WorkerReporter(TestResult): + """ + Reporter for trial's distributed workers. We send things not through a + stream, but through an C{AMP} protocol's C{callRemote} method. + + @ivar _DEFAULT_TODO: Default message for expected failures and + unexpected successes, used only if a C{Todo} is not provided. + """ + + _DEFAULT_TODO = 'Test expected to fail' + + def __init__(self, ampProtocol): + """ + @param ampProtocol: The communication channel with the trial + distributed manager which collects all test results. + @type ampProtocol: C{AMP} + """ + super(WorkerReporter, self).__init__() + self.ampProtocol = ampProtocol + + + def _getFailure(self, error): + """ + Convert a C{sys.exc_info()}-style tuple to a L{Failure}, if necessary. + """ + if isinstance(error, tuple): + return Failure(error[1], error[0], error[2]) + return error + + + def _getFrames(self, failure): + """ + Extract frames from a C{Failure} instance. + """ + frames = [] + for frame in failure.frames: + frames.extend([frame[0], frame[1], str(frame[2])]) + return frames + + + def addSuccess(self, test): + """ + Send a success over. + """ + super(WorkerReporter, self).addSuccess(test) + testName = test.id() + self.ampProtocol.callRemote(managercommands.AddSuccess, + testName=testName) + + + def addError(self, test, error): + """ + Send an error over. + """ + super(WorkerReporter, self).addError(test, error) + testName = test.id() + failure = self._getFailure(error) + error = failure.getErrorMessage() + errorClass = qual(failure.type) + frames = [frame for frame in self._getFrames(failure)] + self.ampProtocol.callRemote(managercommands.AddError, + testName=testName, + error=error, + errorClass=errorClass, + frames=frames) + + + def addFailure(self, test, fail): + """ + Send a Failure over. + """ + super(WorkerReporter, self).addFailure(test, fail) + testName = test.id() + failure = self._getFailure(fail) + fail = failure.getErrorMessage() + failClass = qual(failure.type) + frames = [frame for frame in self._getFrames(failure)] + self.ampProtocol.callRemote(managercommands.AddFailure, + testName=testName, + fail=fail, + failClass=failClass, + frames=frames) + + + def addSkip(self, test, reason): + """ + Send a skip over. + """ + super(WorkerReporter, self).addSkip(test, reason) + reason = str(reason) + testName = test.id() + self.ampProtocol.callRemote(managercommands.AddSkip, + testName=testName, + reason=reason) + + + def _getTodoReason(self, todo): + """ + Get the reason for a C{Todo}. + + If C{todo} is L{None}, return a sensible default. + """ + if todo is None: + return self._DEFAULT_TODO + else: + return todo.reason + + + def addExpectedFailure(self, test, error, todo=None): + """ + Send an expected failure over. + """ + super(WorkerReporter, self).addExpectedFailure(test, error, todo) + errorMessage = error.getErrorMessage() + testName = test.id() + self.ampProtocol.callRemote(managercommands.AddExpectedFailure, + testName=testName, + error=errorMessage, + todo=self._getTodoReason(todo)) + + + def addUnexpectedSuccess(self, test, todo=None): + """ + Send an unexpected success over. + """ + super(WorkerReporter, self).addUnexpectedSuccess(test, todo) + testName = test.id() + self.ampProtocol.callRemote(managercommands.AddUnexpectedSuccess, + testName=testName, + todo=self._getTodoReason(todo)) + + + def printSummary(self): + """ + I{Don't} print a summary + """ diff --git a/contrib/python/Twisted/py2/twisted/trial/_dist/workertrial.py b/contrib/python/Twisted/py2/twisted/trial/_dist/workertrial.py new file mode 100644 index 00000000000..dd14dd61eff --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_dist/workertrial.py @@ -0,0 +1,111 @@ +# -*- test-case-name: twisted.trial._dist.test.test_workertrial -*- +# +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of C{AMP} worker commands, and main executable entry point for +the workers. + +@since: 12.3 +""" + +import sys +import os +import errno + + + +def _setupPath(environ): + """ + Override C{sys.path} with what the parent passed in B{TRIAL_PYTHONPATH}. + + @see: twisted.trial._dist.disttrial.DistTrialRunner.launchWorkerProcesses + """ + if 'TRIAL_PYTHONPATH' in environ: + sys.path[:] = environ['TRIAL_PYTHONPATH'].split(os.pathsep) + + +_setupPath(os.environ) + + +from twisted.internet.protocol import FileWrapper +from twisted.python.log import startLoggingWithObserver, textFromEventDict +from twisted.trial._dist.options import WorkerOptions +from twisted.trial._dist import _WORKER_AMP_STDIN, _WORKER_AMP_STDOUT + + + +class WorkerLogObserver(object): + """ + A log observer that forward its output to a C{AMP} protocol. + """ + + def __init__(self, protocol): + """ + @param protocol: a connected C{AMP} protocol instance. + @type protocol: C{AMP} + """ + self.protocol = protocol + + + def emit(self, eventDict): + """ + Produce a log output. + """ + from twisted.trial._dist import managercommands + text = textFromEventDict(eventDict) + if text is None: + return + self.protocol.callRemote(managercommands.TestWrite, out=text) + + + +def main(_fdopen=os.fdopen): + """ + Main function to be run if __name__ == "__main__". + + @param _fdopen: If specified, the function to use in place of C{os.fdopen}. + @param _fdopen: C{callable} + """ + config = WorkerOptions() + config.parseOptions() + + from twisted.trial._dist.worker import WorkerProtocol + workerProtocol = WorkerProtocol(config['force-gc']) + + protocolIn = _fdopen(_WORKER_AMP_STDIN, 'rb') + protocolOut = _fdopen(_WORKER_AMP_STDOUT, 'wb') + workerProtocol.makeConnection(FileWrapper(protocolOut)) + + observer = WorkerLogObserver(workerProtocol) + startLoggingWithObserver(observer.emit, False) + + while True: + try: + r = protocolIn.read(1) + except IOError as e: + if e.args[0] == errno.EINTR: + if sys.version_info < (3, 0): + sys.exc_clear() + continue + else: + raise + if r == b'': + break + else: + workerProtocol.dataReceived(r) + protocolOut.flush() + sys.stdout.flush() + sys.stderr.flush() + + if config.tracer: + sys.settrace(None) + results = config.tracer.results() + results.write_results(show_missing=True, summary=False, + coverdir=config.coverdir().path) + + + +if __name__ == '__main__': + main() diff --git a/contrib/python/Twisted/py2/twisted/trial/_synctest.py b/contrib/python/Twisted/py2/twisted/trial/_synctest.py new file mode 100644 index 00000000000..2a76ca8e229 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/_synctest.py @@ -0,0 +1,1416 @@ +# -*- test-case-name: twisted.trial.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Things likely to be used by writers of unit tests. + +Maintainer: Jonathan Lange +""" + +from __future__ import division, absolute_import + +import inspect +import os, warnings, sys, tempfile, types +from dis import findlinestarts as _findlinestarts + +from twisted.python import failure, log, monkey +from twisted.python.reflect import fullyQualifiedName +from twisted.python.util import runWithWarningsSuppressed +from twisted.python.deprecate import ( + getDeprecationWarningString, warnAboutFunction +) +from twisted.internet.defer import ensureDeferred + +from twisted.trial import itrial, util + +import unittest as pyunit + +# Python 2.7 and higher has skip support built-in +SkipTest = pyunit.SkipTest + + + +class FailTest(AssertionError): + """ + Raised to indicate the current test has failed to pass. + """ + + + +class Todo(object): + """ + Internal object used to mark a L{TestCase} as 'todo'. Tests marked 'todo' + are reported differently in Trial L{TestResult}s. If todo'd tests fail, + they do not fail the suite and the errors are reported in a separate + category. If todo'd tests succeed, Trial L{TestResult}s will report an + unexpected success. + """ + + def __init__(self, reason, errors=None): + """ + @param reason: A string explaining why the test is marked 'todo' + + @param errors: An iterable of exception types that the test is + expected to raise. If one of these errors is raised by the test, it + will be trapped. Raising any other kind of error will fail the test. + If L{None} is passed, then all errors will be trapped. + """ + self.reason = reason + self.errors = errors + + + def __repr__(self): + return "" % (self.reason, self.errors) + + + def expected(self, failure): + """ + @param failure: A L{twisted.python.failure.Failure}. + + @return: C{True} if C{failure} is expected, C{False} otherwise. + """ + if self.errors is None: + return True + for error in self.errors: + if failure.check(error): + return True + return False + + + +def makeTodo(value): + """ + Return a L{Todo} object built from C{value}. + + If C{value} is a string, return a Todo that expects any exception with + C{value} as a reason. If C{value} is a tuple, the second element is used + as the reason and the first element as the excepted error(s). + + @param value: A string or a tuple of C{(errors, reason)}, where C{errors} + is either a single exception class or an iterable of exception classes. + + @return: A L{Todo} object. + """ + if isinstance(value, str): + return Todo(reason=value) + if isinstance(value, tuple): + errors, reason = value + try: + errors = list(errors) + except TypeError: + errors = [errors] + return Todo(reason=reason, errors=errors) + + + +class _Warning(object): + """ + A L{_Warning} instance represents one warning emitted through the Python + warning system (L{warnings}). This is used to insulate callers of + L{_collectWarnings} from changes to the Python warnings system which might + otherwise require changes to the warning objects that function passes to + the observer object it accepts. + + @ivar message: The string which was passed as the message parameter to + L{warnings.warn}. + + @ivar category: The L{Warning} subclass which was passed as the category + parameter to L{warnings.warn}. + + @ivar filename: The name of the file containing the definition of the code + object which was C{stacklevel} frames above the call to + L{warnings.warn}, where C{stacklevel} is the value of the C{stacklevel} + parameter passed to L{warnings.warn}. + + @ivar lineno: The source line associated with the active instruction of the + code object object which was C{stacklevel} frames above the call to + L{warnings.warn}, where C{stacklevel} is the value of the C{stacklevel} + parameter passed to L{warnings.warn}. + """ + def __init__(self, message, category, filename, lineno): + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + + + +def _setWarningRegistryToNone(modules): + """ + Disable the per-module cache for every module found in C{modules}, typically + C{sys.modules}. + + @param modules: Dictionary of modules, typically sys.module dict + """ + for v in list(modules.values()): + if v is not None: + try: + v.__warningregistry__ = None + except: + # Don't specify a particular exception type to handle in case + # some wacky object raises some wacky exception in response to + # the setattr attempt. + pass + + + +def _collectWarnings(observeWarning, f, *args, **kwargs): + """ + Call C{f} with C{args} positional arguments and C{kwargs} keyword arguments + and collect all warnings which are emitted as a result in a list. + + @param observeWarning: A callable which will be invoked with a L{_Warning} + instance each time a warning is emitted. + + @return: The return value of C{f(*args, **kwargs)}. + """ + def showWarning(message, category, filename, lineno, file=None, line=None): + assert isinstance(message, Warning) + observeWarning(_Warning( + str(message), category, filename, lineno)) + + # Disable the per-module cache for every module otherwise if the warning + # which the caller is expecting us to collect was already emitted it won't + # be re-emitted by the call to f which happens below. + _setWarningRegistryToNone(sys.modules) + + origFilters = warnings.filters[:] + origShow = warnings.showwarning + warnings.simplefilter('always') + try: + warnings.showwarning = showWarning + result = f(*args, **kwargs) + finally: + warnings.filters[:] = origFilters + warnings.showwarning = origShow + return result + + + +class UnsupportedTrialFeature(Exception): + """A feature of twisted.trial was used that pyunit cannot support.""" + + + +class PyUnitResultAdapter(object): + """ + Wrap a C{TestResult} from the standard library's C{unittest} so that it + supports the extended result types from Trial, and also supports + L{twisted.python.failure.Failure}s being passed to L{addError} and + L{addFailure}. + """ + + def __init__(self, original): + """ + @param original: A C{TestResult} instance from C{unittest}. + """ + self.original = original + + + def _exc_info(self, err): + return util.excInfoOrFailureToExcInfo(err) + + + def startTest(self, method): + self.original.startTest(method) + + + def stopTest(self, method): + self.original.stopTest(method) + + + def addFailure(self, test, fail): + self.original.addFailure(test, self._exc_info(fail)) + + + def addError(self, test, error): + self.original.addError(test, self._exc_info(error)) + + + def _unsupported(self, test, feature, info): + self.original.addFailure( + test, + (UnsupportedTrialFeature, + UnsupportedTrialFeature(feature, info), + None)) + + + def addSkip(self, test, reason): + """ + Report the skip as a failure. + """ + self.original.addSkip(test, reason) + + + def addUnexpectedSuccess(self, test, todo=None): + """ + Report the unexpected success as a failure. + """ + self._unsupported(test, 'unexpected success', todo) + + + def addExpectedFailure(self, test, error): + """ + Report the expected failure (i.e. todo) as a failure. + """ + self._unsupported(test, 'expected failure', error) + + + def addSuccess(self, test): + self.original.addSuccess(test) + + + def upDownError(self, method, error, warn, printStatus): + pass + + + +class _AssertRaisesContext(object): + """ + A helper for implementing C{assertRaises}. This is a context manager and a + helper method to support the non-context manager version of + C{assertRaises}. + + @ivar _testCase: See C{testCase} parameter of C{__init__} + + @ivar _expected: See C{expected} parameter of C{__init__} + + @ivar _returnValue: The value returned by the callable being tested (only + when not being used as a context manager). + + @ivar _expectedName: A short string describing the expected exception + (usually the name of the exception class). + + @ivar exception: The exception which was raised by the function being + tested (if it raised one). + """ + + def __init__(self, testCase, expected): + """ + @param testCase: The L{TestCase} instance which is used to raise a + test-failing exception when that is necessary. + + @param expected: The exception type expected to be raised. + """ + self._testCase = testCase + self._expected = expected + self._returnValue = None + try: + self._expectedName = self._expected.__name__ + except AttributeError: + self._expectedName = str(self._expected) + + + def _handle(self, obj): + """ + Call the given object using this object as a context manager. + + @param obj: The object to call and which is expected to raise some + exception. + @type obj: L{object} + + @return: Whatever exception is raised by C{obj()}. + @rtype: L{BaseException} + """ + with self as context: + self._returnValue = obj() + return context.exception + + + def __enter__(self): + return self + + + def __exit__(self, exceptionType, exceptionValue, traceback): + """ + Check exit exception against expected exception. + """ + # No exception raised. + if exceptionType is None: + self._testCase.fail( + "{0} not raised ({1} returned)".format( + self._expectedName, self._returnValue) + ) + + if not isinstance(exceptionValue, exceptionType): + # Support some Python 2.6 ridiculousness. Exceptions raised using + # the C API appear here as the arguments you might pass to the + # exception class to create an exception instance. So... do that + # to turn them into the instances. + if isinstance(exceptionValue, tuple): + exceptionValue = exceptionType(*exceptionValue) + else: + exceptionValue = exceptionType(exceptionValue) + + # Store exception so that it can be access from context. + self.exception = exceptionValue + + # Wrong exception raised. + if not issubclass(exceptionType, self._expected): + reason = failure.Failure(exceptionValue, exceptionType, traceback) + self._testCase.fail( + "{0} raised instead of {1}:\n {2}".format( + fullyQualifiedName(exceptionType), + self._expectedName, reason.getTraceback()), + ) + + # All good. + return True + + + +class _Assertions(pyunit.TestCase, object): + """ + Replaces many of the built-in TestCase assertions. In general, these + assertions provide better error messages and are easier to use in + callbacks. + """ + + def fail(self, msg=None): + """ + Absolutely fail the test. Do not pass go, do not collect $200. + + @param msg: the message that will be displayed as the reason for the + failure + """ + raise self.failureException(msg) + + + def assertFalse(self, condition, msg=None): + """ + Fail the test if C{condition} evaluates to True. + + @param condition: any object that defines __nonzero__ + """ + super(_Assertions, self).assertFalse(condition, msg) + return condition + assertNot = failUnlessFalse = failIf = assertFalse + + + def assertTrue(self, condition, msg=None): + """ + Fail the test if C{condition} evaluates to False. + + @param condition: any object that defines __nonzero__ + """ + super(_Assertions, self).assertTrue(condition, msg) + return condition + assert_ = failUnlessTrue = failUnless = assertTrue + + + def assertRaises(self, exception, f=None, *args, **kwargs): + """ + Fail the test unless calling the function C{f} with the given + C{args} and C{kwargs} raises C{exception}. The failure will report + the traceback and call stack of the unexpected exception. + + @param exception: exception type that is to be expected + @param f: the function to call + + @return: If C{f} is L{None}, a context manager which will make an + assertion about the exception raised from the suite it manages. If + C{f} is not L{None}, the exception raised by C{f}. + + @raise self.failureException: Raised if the function call does + not raise an exception or if it raises an exception of a + different type. + """ + context = _AssertRaisesContext(self, exception) + if f is None: + return context + + return context._handle(lambda: f(*args, **kwargs)) + failUnlessRaises = assertRaises + + + def assertEqual(self, first, second, msg=None): + """ + Fail the test if C{first} and C{second} are not equal. + + @param msg: A string describing the failure that's included in the + exception. + """ + super(_Assertions, self).assertEqual(first, second, msg) + return first + failUnlessEqual = failUnlessEquals = assertEquals = assertEqual + + + def assertIs(self, first, second, msg=None): + """ + Fail the test if C{first} is not C{second}. This is an + obect-identity-equality test, not an object equality + (i.e. C{__eq__}) test. + + @param msg: if msg is None, then the failure message will be + '%r is not %r' % (first, second) + """ + if first is not second: + raise self.failureException(msg or '%r is not %r' % (first, second)) + return first + failUnlessIdentical = assertIdentical = assertIs + + + def assertIsNot(self, first, second, msg=None): + """ + Fail the test if C{first} is C{second}. This is an + obect-identity-equality test, not an object equality + (i.e. C{__eq__}) test. + + @param msg: if msg is None, then the failure message will be + '%r is %r' % (first, second) + """ + if first is second: + raise self.failureException(msg or '%r is %r' % (first, second)) + return first + failIfIdentical = assertNotIdentical = assertIsNot + + + def assertNotEqual(self, first, second, msg=None): + """ + Fail the test if C{first} == C{second}. + + @param msg: if msg is None, then the failure message will be + '%r == %r' % (first, second) + """ + if not first != second: + raise self.failureException(msg or '%r == %r' % (first, second)) + return first + assertNotEquals = failIfEquals = failIfEqual = assertNotEqual + + + def assertIn(self, containee, container, msg=None): + """ + Fail the test if C{containee} is not found in C{container}. + + @param containee: the value that should be in C{container} + @param container: a sequence type, or in the case of a mapping type, + will follow semantics of 'if key in dict.keys()' + @param msg: if msg is None, then the failure message will be + '%r not in %r' % (first, second) + """ + if containee not in container: + raise self.failureException(msg or "%r not in %r" + % (containee, container)) + return containee + failUnlessIn = assertIn + + + def assertNotIn(self, containee, container, msg=None): + """ + Fail the test if C{containee} is found in C{container}. + + @param containee: the value that should not be in C{container} + @param container: a sequence type, or in the case of a mapping type, + will follow semantics of 'if key in dict.keys()' + @param msg: if msg is None, then the failure message will be + '%r in %r' % (first, second) + """ + if containee in container: + raise self.failureException(msg or "%r in %r" + % (containee, container)) + return containee + failIfIn = assertNotIn + + + def assertNotAlmostEqual(self, first, second, places=7, msg=None): + """ + Fail if the two objects are equal as determined by their + difference rounded to the given number of decimal places + (default 7) and comparing to zero. + + @note: decimal places (from zero) is usually not the same + as significant digits (measured from the most + significant digit). + + @note: included for compatibility with PyUnit test cases + """ + if round(second-first, places) == 0: + raise self.failureException(msg or '%r == %r within %r places' + % (first, second, places)) + return first + assertNotAlmostEquals = failIfAlmostEqual = assertNotAlmostEqual + failIfAlmostEquals = assertNotAlmostEqual + + + def assertAlmostEqual(self, first, second, places=7, msg=None): + """ + Fail if the two objects are unequal as determined by their + difference rounded to the given number of decimal places + (default 7) and comparing to zero. + + @note: decimal places (from zero) is usually not the same + as significant digits (measured from the most + significant digit). + + @note: included for compatibility with PyUnit test cases + """ + if round(second-first, places) != 0: + raise self.failureException(msg or '%r != %r within %r places' + % (first, second, places)) + return first + assertAlmostEquals = failUnlessAlmostEqual = assertAlmostEqual + failUnlessAlmostEquals = assertAlmostEqual + + + def assertApproximates(self, first, second, tolerance, msg=None): + """ + Fail if C{first} - C{second} > C{tolerance} + + @param msg: if msg is None, then the failure message will be + '%r ~== %r' % (first, second) + """ + if abs(first - second) > tolerance: + raise self.failureException(msg or "%s ~== %s" % (first, second)) + return first + failUnlessApproximates = assertApproximates + + + def assertSubstring(self, substring, astring, msg=None): + """ + Fail if C{substring} does not exist within C{astring}. + """ + return self.failUnlessIn(substring, astring, msg) + failUnlessSubstring = assertSubstring + + + def assertNotSubstring(self, substring, astring, msg=None): + """ + Fail if C{astring} contains C{substring}. + """ + return self.failIfIn(substring, astring, msg) + failIfSubstring = assertNotSubstring + + + def assertWarns(self, category, message, filename, f, + *args, **kwargs): + """ + Fail if the given function doesn't generate the specified warning when + called. It calls the function, checks the warning, and forwards the + result of the function if everything is fine. + + @param category: the category of the warning to check. + @param message: the output message of the warning to check. + @param filename: the filename where the warning should come from. + @param f: the function which is supposed to generate the warning. + @type f: any callable. + @param args: the arguments to C{f}. + @param kwargs: the keywords arguments to C{f}. + + @return: the result of the original function C{f}. + """ + warningsShown = [] + result = _collectWarnings(warningsShown.append, f, *args, **kwargs) + + if not warningsShown: + self.fail("No warnings emitted") + first = warningsShown[0] + for other in warningsShown[1:]: + if ((other.message, other.category) + != (first.message, first.category)): + self.fail("Can't handle different warnings") + self.assertEqual(first.message, message) + self.assertIdentical(first.category, category) + + # Use starts with because of .pyc/.pyo issues. + self.assertTrue( + filename.startswith(first.filename), + 'Warning in %r, expected %r' % (first.filename, filename)) + + # It would be nice to be able to check the line number as well, but + # different configurations actually end up reporting different line + # numbers (generally the variation is only 1 line, but that's enough + # to fail the test erroneously...). + # self.assertEqual(lineno, xxx) + + return result + failUnlessWarns = assertWarns + + + def assertIsInstance(self, instance, classOrTuple, message=None): + """ + Fail if C{instance} is not an instance of the given class or of + one of the given classes. + + @param instance: the object to test the type (first argument of the + C{isinstance} call). + @type instance: any. + @param classOrTuple: the class or classes to test against (second + argument of the C{isinstance} call). + @type classOrTuple: class, type, or tuple. + + @param message: Custom text to include in the exception text if the + assertion fails. + """ + if not isinstance(instance, classOrTuple): + if message is None: + suffix = "" + else: + suffix = ": " + message + self.fail("%r is not an instance of %s%s" % ( + instance, classOrTuple, suffix)) + failUnlessIsInstance = assertIsInstance + + + def assertNotIsInstance(self, instance, classOrTuple): + """ + Fail if C{instance} is an instance of the given class or of one of the + given classes. + + @param instance: the object to test the type (first argument of the + C{isinstance} call). + @type instance: any. + @param classOrTuple: the class or classes to test against (second + argument of the C{isinstance} call). + @type classOrTuple: class, type, or tuple. + """ + if isinstance(instance, classOrTuple): + self.fail("%r is an instance of %s" % (instance, classOrTuple)) + failIfIsInstance = assertNotIsInstance + + + def successResultOf(self, deferred): + """ + Return the current success result of C{deferred} or raise + C{self.failureException}. + + @param deferred: A L{Deferred} which + has a success result. This means + L{Deferred.callback} or + L{Deferred.errback} has + been called on it and it has reached the end of its callback chain + and the last callback or errback returned a non-L{failure.Failure}. + @type deferred: L{Deferred} + + @raise SynchronousTestCase.failureException: If the + L{Deferred} has no result or has a + failure result. + + @return: The result of C{deferred}. + """ + deferred = ensureDeferred(deferred) + result = [] + deferred.addBoth(result.append) + + if not result: + self.fail( + "Success result expected on {!r}, found no result instead" + .format(deferred) + ) + + result = result[0] + + if isinstance(result, failure.Failure): + self.fail( + "Success result expected on {!r}, " + "found failure result instead:\n{}" + .format(deferred, result.getTraceback()) + ) + + return result + + + def failureResultOf(self, deferred, *expectedExceptionTypes): + """ + Return the current failure result of C{deferred} or raise + C{self.failureException}. + + @param deferred: A L{Deferred} which + has a failure result. This means + L{Deferred.callback} or + L{Deferred.errback} has + been called on it and it has reached the end of its callback chain + and the last callback or errback raised an exception or returned a + L{failure.Failure}. + @type deferred: L{Deferred} + + @param expectedExceptionTypes: Exception types to expect - if + provided, and the exception wrapped by the failure result is + not one of the types provided, then this test will fail. + + @raise SynchronousTestCase.failureException: If the + L{Deferred} has no result, has a + success result, or has an unexpected failure result. + + @return: The failure result of C{deferred}. + @rtype: L{failure.Failure} + """ + deferred = ensureDeferred(deferred) + result = [] + deferred.addBoth(result.append) + + if not result: + self.fail( + "Failure result expected on {!r}, found no result instead" + .format(deferred) + ) + + result = result[0] + + if not isinstance(result, failure.Failure): + self.fail( + "Failure result expected on {!r}, " + "found success result ({!r}) instead" + .format(deferred, result) + ) + + if ( + expectedExceptionTypes and + not result.check(*expectedExceptionTypes) + ): + expectedString = " or ".join([ + ".".join((t.__module__, t.__name__)) + for t in expectedExceptionTypes + ]) + + self.fail( + "Failure of type ({}) expected on {!r}, " + "found type {!r} instead: {}" + .format( + expectedString, deferred, result.type, + result.getTraceback() + ) + ) + + return result + + + def assertNoResult(self, deferred): + """ + Assert that C{deferred} does not have a result at this point. + + If the assertion succeeds, then the result of C{deferred} is left + unchanged. Otherwise, any L{failure.Failure} result is swallowed. + + @param deferred: A L{Deferred} without + a result. This means that neither + L{Deferred.callback} nor + L{Deferred.errback} has + been called, or that the + L{Deferred} is waiting on another + L{Deferred} for a result. + @type deferred: L{Deferred} + + @raise SynchronousTestCase.failureException: If the + L{Deferred} has a result. + """ + deferred = ensureDeferred(deferred) + result = [] + + def cb(res): + result.append(res) + return res + + deferred.addBoth(cb) + + if result: + # If there is already a failure, the self.fail below will + # report it, so swallow it in the deferred + deferred.addErrback(lambda _: None) + self.fail( + "No result expected on {!r}, found {!r} instead" + .format(deferred, result[0]) + ) + + + def assertRegex(self, text, regex, msg=None): + """ + Fail the test if a C{regexp} search of C{text} fails. + + @param text: Text which is under test. + @type text: L{str} + + @param regex: A regular expression object or a string containing a + regular expression suitable for use by re.search(). + @type regex: L{str} or L{re.RegexObject} + + @param msg: Text used as the error message on failure. + @type msg: L{str} + """ + if sys.version_info[:2] > (2, 7): + super(_Assertions, self).assertRegex(text, regex, msg) + else: + # Python 2.7 has unittest.assertRegexpMatches() which was + # renamed to unittest.assertRegex() in Python 3.2 + super(_Assertions, self).assertRegexpMatches(text, regex, msg) + + + +class _LogObserver(object): + """ + Observes the Twisted logs and catches any errors. + + @ivar _errors: A C{list} of L{Failure} instances which were received as + error events from the Twisted logging system. + + @ivar _added: A C{int} giving the number of times C{_add} has been called + less the number of times C{_remove} has been called; used to only add + this observer to the Twisted logging since once, regardless of the + number of calls to the add method. + + @ivar _ignored: A C{list} of exception types which will not be recorded. + """ + + def __init__(self): + self._errors = [] + self._added = 0 + self._ignored = [] + + + def _add(self): + if self._added == 0: + log.addObserver(self.gotEvent) + self._added += 1 + + + def _remove(self): + self._added -= 1 + if self._added == 0: + log.removeObserver(self.gotEvent) + + + def _ignoreErrors(self, *errorTypes): + """ + Do not store any errors with any of the given types. + """ + self._ignored.extend(errorTypes) + + + def _clearIgnores(self): + """ + Stop ignoring any errors we might currently be ignoring. + """ + self._ignored = [] + + + def flushErrors(self, *errorTypes): + """ + Flush errors from the list of caught errors. If no arguments are + specified, remove all errors. If arguments are specified, only remove + errors of those types from the stored list. + """ + if errorTypes: + flushed = [] + remainder = [] + for f in self._errors: + if f.check(*errorTypes): + flushed.append(f) + else: + remainder.append(f) + self._errors = remainder + else: + flushed = self._errors + self._errors = [] + return flushed + + + def getErrors(self): + """ + Return a list of errors caught by this observer. + """ + return self._errors + + + def gotEvent(self, event): + """ + The actual observer method. Called whenever a message is logged. + + @param event: A dictionary containing the log message. Actual + structure undocumented (see source for L{twisted.python.log}). + """ + if event.get('isError', False) and 'failure' in event: + f = event['failure'] + if len(self._ignored) == 0 or not f.check(*self._ignored): + self._errors.append(f) + + + +_logObserver = _LogObserver() + + +class SynchronousTestCase(_Assertions): + """ + A unit test. The atom of the unit testing universe. + + This class extends C{unittest.TestCase} from the standard library. A number + of convenient testing helpers are added, including logging and warning + integration, monkey-patching support, and more. + + To write a unit test, subclass C{SynchronousTestCase} and define a method + (say, 'test_foo') on the subclass. To run the test, instantiate your + subclass with the name of the method, and call L{run} on the instance, + passing a L{TestResult} object. + + The C{trial} script will automatically find any C{SynchronousTestCase} + subclasses defined in modules beginning with 'test_' and construct test + cases for all methods beginning with 'test'. + + If an error is logged during the test run, the test will fail with an + error. See L{log.err}. + + @ivar failureException: An exception class, defaulting to C{FailTest}. If + the test method raises this exception, it will be reported as a failure, + rather than an exception. All of the assertion methods raise this if the + assertion fails. + + @ivar skip: L{None} or a string explaining why this test is to be + skipped. If defined, the test will not be run. Instead, it will be + reported to the result object as 'skipped' (if the C{TestResult} supports + skipping). + + @ivar todo: L{None}, a string or a tuple of C{(errors, reason)} where + C{errors} is either an exception class or an iterable of exception + classes, and C{reason} is a string. See L{Todo} or L{makeTodo} for more + information. + + @ivar suppress: L{None} or a list of tuples of C{(args, kwargs)} to be + passed to C{warnings.filterwarnings}. Use these to suppress warnings + raised in a test. Useful for testing deprecated code. See also + L{util.suppress}. + """ + failureException = FailTest + + def __init__(self, methodName='runTest'): + super(SynchronousTestCase, self).__init__(methodName) + self._passed = False + self._cleanups = [] + self._testMethodName = methodName + testMethod = getattr(self, methodName) + self._parents = [ + testMethod, self, sys.modules.get(self.__class__.__module__)] + + + def __eq__(self, other): + """ + Override the comparison defined by the base TestCase which considers + instances of the same class with the same _testMethodName to be + equal. Since trial puts TestCase instances into a set, that + definition of comparison makes it impossible to run the same test + method twice. Most likely, trial should stop using a set to hold + tests, but until it does, this is necessary on Python 2.6. -exarkun + """ + return self is other + + + def __ne__(self, other): + return self is not other + + + def __hash__(self): + return hash((self.__class__, self._testMethodName)) + + + def shortDescription(self): + desc = super(SynchronousTestCase, self).shortDescription() + if desc is None: + return self._testMethodName + return desc + + + def getSkip(self): + """ + Return the skip reason set on this test, if any is set. Checks on the + instance first, then the class, then the module, then packages. As + soon as it finds something with a C{skip} attribute, returns that. + Returns L{None} if it cannot find anything. See L{TestCase} docstring + for more details. + """ + return util.acquireAttribute(self._parents, 'skip', None) + + + def getTodo(self): + """ + Return a L{Todo} object if the test is marked todo. Checks on the + instance first, then the class, then the module, then packages. As + soon as it finds something with a C{todo} attribute, returns that. + Returns L{None} if it cannot find anything. See L{TestCase} docstring + for more details. + """ + todo = util.acquireAttribute(self._parents, 'todo', None) + if todo is None: + return None + return makeTodo(todo) + + + def runTest(self): + """ + If no C{methodName} argument is passed to the constructor, L{run} will + treat this method as the thing with the actual test inside. + """ + + + def run(self, result): + """ + Run the test case, storing the results in C{result}. + + First runs C{setUp} on self, then runs the test method (defined in the + constructor), then runs C{tearDown}. As with the standard library + L{unittest.TestCase}, the return value of these methods is disregarded. + In particular, returning a L{Deferred} + has no special additional consequences. + + @param result: A L{TestResult} object. + """ + log.msg("--> %s <--" % (self.id())) + new_result = itrial.IReporter(result, None) + if new_result is None: + result = PyUnitResultAdapter(result) + else: + result = new_result + result.startTest(self) + if self.getSkip(): # don't run test methods that are marked as .skip + result.addSkip(self, self.getSkip()) + result.stopTest(self) + return + + self._passed = False + self._warnings = [] + + self._installObserver() + # All the code inside _runFixturesAndTest will be run such that warnings + # emitted by it will be collected and retrievable by flushWarnings. + _collectWarnings(self._warnings.append, self._runFixturesAndTest, result) + + # Any collected warnings which the test method didn't flush get + # re-emitted so they'll be logged or show up on stdout or whatever. + for w in self.flushWarnings(): + try: + warnings.warn_explicit(**w) + except: + result.addError(self, failure.Failure()) + + result.stopTest(self) + + + def addCleanup(self, f, *args, **kwargs): + """ + Add the given function to a list of functions to be called after the + test has run, but before C{tearDown}. + + Functions will be run in reverse order of being added. This helps + ensure that tear down complements set up. + + As with all aspects of L{SynchronousTestCase}, Deferreds are not + supported in cleanup functions. + """ + self._cleanups.append((f, args, kwargs)) + + + def patch(self, obj, attribute, value): + """ + Monkey patch an object for the duration of the test. + + The monkey patch will be reverted at the end of the test using the + L{addCleanup} mechanism. + + The L{monkey.MonkeyPatcher} is returned so that users can restore and + re-apply the monkey patch within their tests. + + @param obj: The object to monkey patch. + @param attribute: The name of the attribute to change. + @param value: The value to set the attribute to. + @return: A L{monkey.MonkeyPatcher} object. + """ + monkeyPatch = monkey.MonkeyPatcher((obj, attribute, value)) + monkeyPatch.patch() + self.addCleanup(monkeyPatch.restore) + return monkeyPatch + + + def flushLoggedErrors(self, *errorTypes): + """ + Remove stored errors received from the log. + + C{TestCase} stores each error logged during the run of the test and + reports them as errors during the cleanup phase (after C{tearDown}). + + @param *errorTypes: If unspecified, flush all errors. Otherwise, only + flush errors that match the given types. + + @return: A list of failures that have been removed. + """ + return self._observer.flushErrors(*errorTypes) + + + def flushWarnings(self, offendingFunctions=None): + """ + Remove stored warnings from the list of captured warnings and return + them. + + @param offendingFunctions: If L{None}, all warnings issued during the + currently running test will be flushed. Otherwise, only warnings + which I{point} to a function included in this list will be flushed. + All warnings include a filename and source line number; if these + parts of a warning point to a source line which is part of a + function, then the warning I{points} to that function. + @type offendingFunctions: L{None} or L{list} of functions or methods. + + @raise ValueError: If C{offendingFunctions} is not L{None} and includes + an object which is not a L{types.FunctionType} or + L{types.MethodType} instance. + + @return: A C{list}, each element of which is a C{dict} giving + information about one warning which was flushed by this call. The + keys of each C{dict} are: + + - C{'message'}: The string which was passed as the I{message} + parameter to L{warnings.warn}. + + - C{'category'}: The warning subclass which was passed as the + I{category} parameter to L{warnings.warn}. + + - C{'filename'}: The name of the file containing the definition + of the code object which was C{stacklevel} frames above the + call to L{warnings.warn}, where C{stacklevel} is the value of + the C{stacklevel} parameter passed to L{warnings.warn}. + + - C{'lineno'}: The source line associated with the active + instruction of the code object object which was C{stacklevel} + frames above the call to L{warnings.warn}, where + C{stacklevel} is the value of the C{stacklevel} parameter + passed to L{warnings.warn}. + """ + if offendingFunctions is None: + toFlush = self._warnings[:] + self._warnings[:] = [] + else: + toFlush = [] + for aWarning in self._warnings: + for aFunction in offendingFunctions: + if not isinstance(aFunction, ( + types.FunctionType, types.MethodType)): + raise ValueError("%r is not a function or method" % ( + aFunction,)) + + # inspect.getabsfile(aFunction) sometimes returns a + # filename which disagrees with the filename the warning + # system generates. This seems to be because a + # function's code object doesn't deal with source files + # being renamed. inspect.getabsfile(module) seems + # better (or at least agrees with the warning system + # more often), and does some normalization for us which + # is desirable. inspect.getmodule() is attractive, but + # somewhat broken in Python < 2.6. See Python bug 4845. + aModule = sys.modules[aFunction.__module__] + filename = inspect.getabsfile(aModule) + + if filename != os.path.normcase(aWarning.filename): + continue + lineStarts = list(_findlinestarts(aFunction.__code__)) + first = lineStarts[0][1] + last = lineStarts[-1][1] + if not (first <= aWarning.lineno <= last): + continue + # The warning points to this function, flush it and move on + # to the next warning. + toFlush.append(aWarning) + break + # Remove everything which is being flushed. + list(map(self._warnings.remove, toFlush)) + + return [ + {'message': w.message, 'category': w.category, + 'filename': w.filename, 'lineno': w.lineno} + for w in toFlush] + + + def callDeprecated(self, version, f, *args, **kwargs): + """ + Call a function that should have been deprecated at a specific version + and in favor of a specific alternative, and assert that it was thusly + deprecated. + + @param version: A 2-sequence of (since, replacement), where C{since} is + a the first L{version} that C{f} + should have been deprecated since, and C{replacement} is a suggested + replacement for the deprecated functionality, as described by + L{twisted.python.deprecate.deprecated}. If there is no suggested + replacement, this parameter may also be simply a + L{version} by itself. + + @param f: The deprecated function to call. + + @param args: The arguments to pass to C{f}. + + @param kwargs: The keyword arguments to pass to C{f}. + + @return: Whatever C{f} returns. + + @raise: Whatever C{f} raises. If any exception is + raised by C{f}, though, no assertions will be made about emitted + deprecations. + + @raise FailTest: if no warnings were emitted by C{f}, or if the + L{DeprecationWarning} emitted did not produce the canonical + please-use-something-else message that is standard for Twisted + deprecations according to the given version and replacement. + """ + result = f(*args, **kwargs) + warningsShown = self.flushWarnings([self.callDeprecated]) + try: + info = list(version) + except TypeError: + since = version + replacement = None + else: + [since, replacement] = info + + if len(warningsShown) == 0: + self.fail('%r is not deprecated.' % (f,)) + + observedWarning = warningsShown[0]['message'] + expectedWarning = getDeprecationWarningString( + f, since, replacement=replacement) + self.assertEqual(expectedWarning, observedWarning) + + return result + + + def mktemp(self): + """ + Create a new path name which can be used for a new file or directory. + + The result is a relative path that is guaranteed to be unique within the + current working directory. The parent of the path will exist, but the + path will not. + + For a temporary directory call os.mkdir on the path. For a temporary + file just create the file (e.g. by opening the path for writing and then + closing it). + + @return: The newly created path + @rtype: C{str} + """ + MAX_FILENAME = 32 # some platforms limit lengths of filenames + base = os.path.join(self.__class__.__module__[:MAX_FILENAME], + self.__class__.__name__[:MAX_FILENAME], + self._testMethodName[:MAX_FILENAME]) + if not os.path.exists(base): + os.makedirs(base) + dirname = tempfile.mkdtemp('', '', base) + return os.path.join(dirname, 'temp') + + + def _getSuppress(self): + """ + Returns any warning suppressions set for this test. Checks on the + instance first, then the class, then the module, then packages. As + soon as it finds something with a C{suppress} attribute, returns that. + Returns any empty list (i.e. suppress no warnings) if it cannot find + anything. See L{TestCase} docstring for more details. + """ + return util.acquireAttribute(self._parents, 'suppress', []) + + + def _getSkipReason(self, method, skip): + """ + Return the reason to use for skipping a test method. + + @param method: The method which produced the skip. + @param skip: A L{unittest.SkipTest} instance raised by C{method}. + """ + if len(skip.args) > 0: + return skip.args[0] + + warnAboutFunction( + method, + "Do not raise unittest.SkipTest with no arguments! Give a reason " + "for skipping tests!") + return skip + + + def _run(self, suppress, todo, method, result): + """ + Run a single method, either a test method or fixture. + + @param suppress: Any warnings to suppress, as defined by the C{suppress} + attribute on this method, test case, or the module it is defined in. + + @param todo: Any expected failure or failures, as defined by the C{todo} + attribute on this method, test case, or the module it is defined in. + + @param method: The method to run. + + @param result: The TestResult instance to which to report results. + + @return: C{True} if the method fails and no further method/fixture calls + should be made, C{False} otherwise. + """ + if inspect.isgeneratorfunction(method): + exc = TypeError( + '%r is a generator function and therefore will never run' % ( + method,)) + result.addError(self, failure.Failure(exc)) + return True + try: + runWithWarningsSuppressed(suppress, method) + except SkipTest as e: + result.addSkip(self, self._getSkipReason(method, e)) + except: + reason = failure.Failure() + if todo is None or not todo.expected(reason): + if reason.check(self.failureException): + addResult = result.addFailure + else: + addResult = result.addError + addResult(self, reason) + else: + result.addExpectedFailure(self, reason, todo) + else: + return False + return True + + + def _runFixturesAndTest(self, result): + """ + Run C{setUp}, a test method, test cleanups, and C{tearDown}. + + @param result: The TestResult instance to which to report results. + """ + suppress = self._getSuppress() + try: + if self._run(suppress, None, self.setUp, result): + return + + todo = self.getTodo() + method = getattr(self, self._testMethodName) + failed = self._run(suppress, todo, method, result) + finally: + self._runCleanups(result) + + if todo and not failed: + result.addUnexpectedSuccess(self, todo) + + if self._run(suppress, None, self.tearDown, result): + failed = True + + for error in self._observer.getErrors(): + result.addError(self, error) + failed = True + self._observer.flushErrors() + self._removeObserver() + + if not (failed or todo): + result.addSuccess(self) + + + def _runCleanups(self, result): + """ + Synchronously run any cleanups which have been added. + """ + while len(self._cleanups) > 0: + f, args, kwargs = self._cleanups.pop() + try: + f(*args, **kwargs) + except: + f = failure.Failure() + result.addError(self, f) + + + def _installObserver(self): + self._observer = _logObserver + self._observer._add() + + + def _removeObserver(self): + self._observer._remove() diff --git a/contrib/python/Twisted/py2/twisted/trial/itrial.py b/contrib/python/Twisted/py2/twisted/trial/itrial.py new file mode 100644 index 00000000000..186b7e5b76b --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/itrial.py @@ -0,0 +1,259 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Interfaces for Trial. + +Maintainer: Jonathan Lange +""" + +from __future__ import division, absolute_import + +import zope.interface as zi +from zope.interface import Attribute + + +class ITestCase(zi.Interface): + """ + The interface that a test case must implement in order to be used in Trial. + """ + + failureException = zi.Attribute( + "The exception class that is raised by failed assertions") + + + def __call__(result): + """ + Run the test. Should always do exactly the same thing as run(). + """ + + + def countTestCases(): + """ + Return the number of tests in this test case. Usually 1. + """ + + + def id(): + """ + Return a unique identifier for the test, usually the fully-qualified + Python name. + """ + + + def run(result): + """ + Run the test, storing the results in C{result}. + + @param result: A L{TestResult}. + """ + + + def shortDescription(): + """ + Return a short description of the test. + """ + + + +class IReporter(zi.Interface): + """ + I report results from a run of a test suite. + """ + + stream = zi.Attribute( + "Deprecated in Twisted 8.0. " + "The io-stream that this reporter will write to") + tbformat = zi.Attribute("Either 'default', 'brief', or 'verbose'") + args = zi.Attribute( + "Additional string argument passed from the command line") + shouldStop = zi.Attribute( + """ + A boolean indicating that this reporter would like the test run to stop. + """) + separator = Attribute( + "Deprecated in Twisted 8.0. " + "A value which will occasionally be passed to the L{write} method.") + testsRun = Attribute( + """ + The number of tests that seem to have been run according to this + reporter. + """) + + + def startTest(method): + """ + Report the beginning of a run of a single test method. + + @param method: an object that is adaptable to ITestMethod + """ + + + def stopTest(method): + """ + Report the status of a single test method + + @param method: an object that is adaptable to ITestMethod + """ + + + def startSuite(name): + """ + Deprecated in Twisted 8.0. + + Suites which wish to appear in reporter output should call this + before running their tests. + """ + + + def endSuite(name): + """ + Deprecated in Twisted 8.0. + + Called at the end of a suite, if and only if that suite has called + C{startSuite}. + """ + + + def cleanupErrors(errs): + """ + Deprecated in Twisted 8.0. + + Called when the reactor has been left in a 'dirty' state + + @param errs: a list of L{twisted.python.failure.Failure}s + """ + + + def upDownError(userMeth, warn=True, printStatus=True): + """ + Deprecated in Twisted 8.0. + + Called when an error occurs in a setUp* or tearDown* method + + @param warn: indicates whether or not the reporter should emit a + warning about the error + @type warn: Boolean + @param printStatus: indicates whether or not the reporter should + print the name of the method and the status + message appropriate for the type of error + @type printStatus: Boolean + """ + + + def addSuccess(test): + """ + Record that test passed. + """ + + + def addError(test, error): + """ + Record that a test has raised an unexpected exception. + + @param test: The test that has raised an error. + @param error: The error that the test raised. It will either be a + three-tuple in the style of C{sys.exc_info()} or a + L{Failure} object. + """ + + + def addFailure(test, failure): + """ + Record that a test has failed with the given failure. + + @param test: The test that has failed. + @param failure: The failure that the test failed with. It will + either be a three-tuple in the style of C{sys.exc_info()} + or a L{Failure} object. + """ + + + def addExpectedFailure(test, failure, todo=None): + """ + Record that the given test failed, and was expected to do so. + + In Twisted 15.5 and prior, C{todo} was a mandatory parameter. + + @type test: L{unittest.TestCase} + @param test: The test which this is about. + @type error: L{failure.Failure} + @param error: The error which this test failed with. + @type todo: L{unittest.Todo} + @param todo: The reason for the test's TODO status. If L{None}, a + generic reason is used. + """ + + + def addUnexpectedSuccess(test, todo=None): + """ + Record that the given test failed, and was expected to do so. + + In Twisted 15.5 and prior, C{todo} was a mandatory parameter. + + @type test: L{unittest.TestCase} + @param test: The test which this is about. + @type todo: L{unittest.Todo} + @param todo: The reason for the test's TODO status. If L{None}, a + generic reason is used. + """ + + + def addSkip(test, reason): + """ + Record that a test has been skipped for the given reason. + + @param test: The test that has been skipped. + @param reason: An object that the test case has specified as the reason + for skipping the test. + """ + + + def printSummary(): + """ + Deprecated in Twisted 8.0, use L{done} instead. + + Present a summary of the test results. + """ + + + def printErrors(): + """ + Deprecated in Twisted 8.0, use L{done} instead. + + Present the errors that have occurred during the test run. This method + will be called after all tests have been run. + """ + + + def write(string): + """ + Deprecated in Twisted 8.0, use L{done} instead. + + Display a string to the user, without appending a new line. + """ + + + def writeln(string): + """ + Deprecated in Twisted 8.0, use L{done} instead. + + Display a string to the user, appending a new line. + """ + + def wasSuccessful(): + """ + Return a boolean indicating whether all test results that were reported + to this reporter were successful or not. + """ + + + def done(): + """ + Called when the test run is complete. + + This gives the result object an opportunity to display a summary of + information to the user. Once you have called C{done} on an + L{IReporter} object, you should assume that the L{IReporter} object is + no longer usable. + """ diff --git a/contrib/python/Twisted/py2/twisted/trial/reporter.py b/contrib/python/Twisted/py2/twisted/trial/reporter.py new file mode 100644 index 00000000000..eb48c7b99c3 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/reporter.py @@ -0,0 +1,1322 @@ +# -*- test-case-name: twisted.trial.test.test_reporter -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +# +# Maintainer: Jonathan Lange + +""" +Defines classes that handle the results of tests. +""" + +from __future__ import division, absolute_import + +import sys +import os +import time +import warnings +import unittest as pyunit + +from collections import OrderedDict + +from zope.interface import implementer + +from twisted.python import reflect, log +from twisted.python.components import proxyForInterface +from twisted.python.failure import Failure +from twisted.python.util import untilConcludes +from twisted.python.compat import _PY3, items +from twisted.trial import itrial, util + +try: + from subunit import TestProtocolClient +except ImportError: + TestProtocolClient = None + + + +def _makeTodo(value): + """ + Return a L{Todo} object built from C{value}. + + This is a synonym for L{twisted.trial.unittest.makeTodo}, but imported + locally to avoid circular imports. + + @param value: A string or a tuple of C{(errors, reason)}, where C{errors} + is either a single exception class or an iterable of exception classes. + + @return: A L{Todo} object. + """ + from twisted.trial.unittest import makeTodo + return makeTodo(value) + + + +class BrokenTestCaseWarning(Warning): + """ + Emitted as a warning when an exception occurs in one of setUp or tearDown. + """ + + + +class SafeStream(object): + """ + Wraps a stream object so that all C{write} calls are wrapped in + L{untilConcludes}. + """ + + def __init__(self, original): + self.original = original + + + def __getattr__(self, name): + return getattr(self.original, name) + + + def write(self, *a, **kw): + return untilConcludes(self.original.write, *a, **kw) + + + +@implementer(itrial.IReporter) +class TestResult(pyunit.TestResult, object): + """ + Accumulates the results of several L{twisted.trial.unittest.TestCase}s. + + @ivar successes: count the number of successes achieved by the test run. + @type successes: C{int} + """ + + # Used when no todo provided to addExpectedFailure or addUnexpectedSuccess. + _DEFAULT_TODO = 'Test expected to fail' + + def __init__(self): + super(TestResult, self).__init__() + self.skips = [] + self.expectedFailures = [] + self.unexpectedSuccesses = [] + self.successes = 0 + self._timings = [] + + + def __repr__(self): + return ('<%s run=%d errors=%d failures=%d todos=%d dones=%d skips=%d>' + % (reflect.qual(self.__class__), self.testsRun, + len(self.errors), len(self.failures), + len(self.expectedFailures), len(self.skips), + len(self.unexpectedSuccesses))) + + + def _getTime(self): + return time.time() + + + def _getFailure(self, error): + """ + Convert a C{sys.exc_info()}-style tuple to a L{Failure}, if necessary. + """ + if isinstance(error, tuple): + return Failure(error[1], error[0], error[2]) + return error + + + def startTest(self, test): + """ + This must be called before the given test is commenced. + + @type test: L{pyunit.TestCase} + """ + super(TestResult, self).startTest(test) + self._testStarted = self._getTime() + + + def stopTest(self, test): + """ + This must be called after the given test is completed. + + @type test: L{pyunit.TestCase} + """ + super(TestResult, self).stopTest(test) + self._lastTime = self._getTime() - self._testStarted + + + def addFailure(self, test, fail): + """ + Report a failed assertion for the given test. + + @type test: L{pyunit.TestCase} + @type fail: L{Failure} or L{tuple} + """ + self.failures.append((test, self._getFailure(fail))) + + + def addError(self, test, error): + """ + Report an error that occurred while running the given test. + + @type test: L{pyunit.TestCase} + @type error: L{Failure} or L{tuple} + """ + self.errors.append((test, self._getFailure(error))) + + + def addSkip(self, test, reason): + """ + Report that the given test was skipped. + + In Trial, tests can be 'skipped'. Tests are skipped mostly because + there is some platform or configuration issue that prevents them from + being run correctly. + + @type test: L{pyunit.TestCase} + @type reason: L{str} + """ + self.skips.append((test, reason)) + + + def addUnexpectedSuccess(self, test, todo=None): + """ + Report that the given test succeeded against expectations. + + In Trial, tests can be marked 'todo'. That is, they are expected to + fail. When a test that is expected to fail instead succeeds, it should + call this method to report the unexpected success. + + @type test: L{pyunit.TestCase} + @type todo: L{unittest.Todo}, or L{None}, in which case a default todo + message is provided. + """ + if todo is None: + todo = _makeTodo(self._DEFAULT_TODO) + self.unexpectedSuccesses.append((test, todo)) + + + def addExpectedFailure(self, test, error, todo=None): + """ + Report that the given test failed, and was expected to do so. + + In Trial, tests can be marked 'todo'. That is, they are expected to + fail. + + @type test: L{pyunit.TestCase} + @type error: L{Failure} + @type todo: L{unittest.Todo}, or L{None}, in which case a default todo + message is provided. + """ + if todo is None: + todo = _makeTodo(self._DEFAULT_TODO) + self.expectedFailures.append((test, error, todo)) + + + def addSuccess(self, test): + """ + Report that the given test succeeded. + + @type test: L{pyunit.TestCase} + """ + self.successes += 1 + + + def wasSuccessful(self): + """ + Report whether or not this test suite was successful or not. + + The behaviour of this method changed in L{pyunit} in Python 3.4 to + fail if there are any errors, failures, or unexpected successes. + Previous to 3.4, it was only if there were errors or failures. This + method implements the old behaviour for backwards compatibility reasons, + checking just for errors and failures. + + @rtype: L{bool} + """ + return len(self.failures) == len(self.errors) == 0 + + + def done(self): + """ + The test suite has finished running. + """ + + + +@implementer(itrial.IReporter) +class TestResultDecorator(proxyForInterface(itrial.IReporter, + "_originalReporter")): + """ + Base class for TestResult decorators. + + @ivar _originalReporter: The wrapped instance of reporter. + @type _originalReporter: A provider of L{itrial.IReporter} + """ + + + +@implementer(itrial.IReporter) +class UncleanWarningsReporterWrapper(TestResultDecorator): + """ + A wrapper for a reporter that converts L{util.DirtyReactorAggregateError}s + to warnings. + """ + + def addError(self, test, error): + """ + If the error is a L{util.DirtyReactorAggregateError}, instead of + reporting it as a normal error, throw a warning. + """ + + if (isinstance(error, Failure) + and error.check(util.DirtyReactorAggregateError)): + warnings.warn(error.getErrorMessage()) + else: + self._originalReporter.addError(test, error) + + + +@implementer(itrial.IReporter) +class _ExitWrapper(TestResultDecorator): + """ + A wrapper for a reporter that causes the reporter to stop after + unsuccessful tests. + """ + + def addError(self, *args, **kwargs): + self.shouldStop = True + return self._originalReporter.addError(*args, **kwargs) + + + def addFailure(self, *args, **kwargs): + self.shouldStop = True + return self._originalReporter.addFailure(*args, **kwargs) + + + +class _AdaptedReporter(TestResultDecorator): + """ + TestResult decorator that makes sure that addError only gets tests that + have been adapted with a particular test adapter. + """ + + def __init__(self, original, testAdapter): + """ + Construct an L{_AdaptedReporter}. + + @param original: An {itrial.IReporter}. + @param testAdapter: A callable that returns an L{itrial.ITestCase}. + """ + TestResultDecorator.__init__(self, original) + self.testAdapter = testAdapter + + + def addError(self, test, error): + """ + See L{itrial.IReporter}. + """ + test = self.testAdapter(test) + return self._originalReporter.addError(test, error) + + + def addExpectedFailure(self, test, failure, todo=None): + """ + See L{itrial.IReporter}. + + @type test: A L{pyunit.TestCase}. + @type failure: A L{failure.Failure} or L{exceptions.AssertionError} + @type todo: A L{unittest.Todo} or None + + When C{todo} is L{None} a generic C{unittest.Todo} is built. + + L{pyunit.TestCase}'s C{run()} calls this with 3 positional arguments + (without C{todo}). + """ + return self._originalReporter.addExpectedFailure( + self.testAdapter(test), failure, todo) + + + def addFailure(self, test, failure): + """ + See L{itrial.IReporter}. + """ + test = self.testAdapter(test) + return self._originalReporter.addFailure(test, failure) + + + def addSkip(self, test, skip): + """ + See L{itrial.IReporter}. + """ + test = self.testAdapter(test) + return self._originalReporter.addSkip(test, skip) + + + def addUnexpectedSuccess(self, test, todo=None): + """ + See L{itrial.IReporter}. + + @type test: A L{pyunit.TestCase}. + @type todo: A L{unittest.Todo} or None + + When C{todo} is L{None} a generic C{unittest.Todo} is built. + + L{pyunit.TestCase}'s C{run()} calls this with 2 positional arguments + (without C{todo}). + """ + test = self.testAdapter(test) + return self._originalReporter.addUnexpectedSuccess(test, todo) + + + def startTest(self, test): + """ + See L{itrial.IReporter}. + """ + return self._originalReporter.startTest(self.testAdapter(test)) + + + def stopTest(self, test): + """ + See L{itrial.IReporter}. + """ + return self._originalReporter.stopTest(self.testAdapter(test)) + + + +@implementer(itrial.IReporter) +class Reporter(TestResult): + """ + A basic L{TestResult} with support for writing to a stream. + + @ivar _startTime: The time when the first test was started. It defaults to + L{None}, which means that no test was actually launched. + @type _startTime: C{float} or L{None} + + @ivar _warningCache: A C{set} of tuples of warning message (file, line, + text, category) which have already been written to the output stream + during the currently executing test. This is used to avoid writing + duplicates of the same warning to the output stream. + @type _warningCache: C{set} + + @ivar _publisher: The log publisher which will be observed for warning + events. + @type _publisher: L{twisted.python.log.LogPublisher} + """ + + _separator = '-' * 79 + _doubleSeparator = '=' * 79 + + def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, + publisher=None): + super(Reporter, self).__init__() + self._stream = SafeStream(stream) + self.tbformat = tbformat + self.realtime = realtime + self._startTime = None + self._warningCache = set() + + # Start observing log events so as to be able to report warnings. + self._publisher = publisher + if publisher is not None: + publisher.addObserver(self._observeWarnings) + + + def _observeWarnings(self, event): + """ + Observe warning events and write them to C{self._stream}. + + This method is a log observer which will be registered with + C{self._publisher.addObserver}. + + @param event: A C{dict} from the logging system. If it has a + C{'warning'} key, a logged warning will be extracted from it and + possibly written to C{self.stream}. + """ + if 'warning' in event: + key = (event['filename'], event['lineno'], + event['category'].split('.')[-1], + str(event['warning'])) + if key not in self._warningCache: + self._warningCache.add(key) + self._stream.write('%s:%s: %s: %s\n' % key) + + + def startTest(self, test): + """ + Called when a test begins to run. Records the time when it was first + called and resets the warning cache. + + @param test: L{ITestCase} + """ + super(Reporter, self).startTest(test) + if self._startTime is None: + self._startTime = self._getTime() + self._warningCache = set() + + + def addFailure(self, test, fail): + """ + Called when a test fails. If C{realtime} is set, then it prints the + error to the stream. + + @param test: L{ITestCase} that failed. + @param fail: L{failure.Failure} containing the error. + """ + super(Reporter, self).addFailure(test, fail) + if self.realtime: + fail = self.failures[-1][1] # guarantee it's a Failure + self._write(self._formatFailureTraceback(fail)) + + + def addError(self, test, error): + """ + Called when a test raises an error. If C{realtime} is set, then it + prints the error to the stream. + + @param test: L{ITestCase} that raised the error. + @param error: L{failure.Failure} containing the error. + """ + error = self._getFailure(error) + super(Reporter, self).addError(test, error) + if self.realtime: + error = self.errors[-1][1] # guarantee it's a Failure + self._write(self._formatFailureTraceback(error)) + + + def _write(self, format, *args): + """ + Safely write to the reporter's stream. + + @param format: A format string to write. + @param *args: The arguments for the format string. + """ + s = str(format) + assert isinstance(s, type('')) + if args: + self._stream.write(s % args) + else: + self._stream.write(s) + untilConcludes(self._stream.flush) + + + def _writeln(self, format, *args): + """ + Safely write a line to the reporter's stream. Newline is appended to + the format string. + + @param format: A format string to write. + @param *args: The arguments for the format string. + """ + self._write(format, *args) + self._write('\n') + + + def upDownError(self, method, error, warn, printStatus): + super(Reporter, self).upDownError(method, error, warn, printStatus) + if warn: + tbStr = self._formatFailureTraceback(error) + log.msg(tbStr) + msg = ("caught exception in %s, your TestCase is broken\n\n%s" + % (method, tbStr)) + warnings.warn(msg, BrokenTestCaseWarning, stacklevel=2) + + + def cleanupErrors(self, errs): + super(Reporter, self).cleanupErrors(errs) + warnings.warn("%s\n%s" % ("REACTOR UNCLEAN! traceback(s) follow: ", + self._formatFailureTraceback(errs)), + BrokenTestCaseWarning) + + + def _trimFrames(self, frames): + """ + Trim frames to remove internal paths. + + When a C{SynchronousTestCase} method fails synchronously, the stack + looks like this: + - [0]: C{SynchronousTestCase._run} + - [1]: C{util.runWithWarningsSuppressed} + - [2:-2]: code in the test method which failed + - [-1]: C{_synctest.fail} + + When a C{TestCase} method fails synchronously, the stack looks like + this: + - [0]: C{defer.maybeDeferred} + - [1]: C{utils.runWithWarningsSuppressed} + - [2]: C{utils.runWithWarningsSuppressed} + - [3:-2]: code in the test method which failed + - [-1]: C{_synctest.fail} + + When a method fails inside a C{Deferred} (i.e., when the test method + returns a C{Deferred}, and that C{Deferred}'s errback fires), the stack + captured inside the resulting C{Failure} looks like this: + - [0]: C{defer.Deferred._runCallbacks} + - [1:-2]: code in the testmethod which failed + - [-1]: C{_synctest.fail} + + As a result, we want to trim either [maybeDeferred, runWWS, runWWS] or + [Deferred._runCallbacks] or [SynchronousTestCase._run, runWWS] from the + front, and trim the [unittest.fail] from the end. + + There is also another case, when the test method is badly defined and + contains extra arguments. + + If it doesn't recognize one of these cases, it just returns the + original frames. + + @param frames: The C{list} of frames from the test failure. + + @return: The C{list} of frames to display. + """ + newFrames = list(frames) + + if len(frames) < 2: + return newFrames + + firstMethod = newFrames[0][0] + firstFile = os.path.splitext(os.path.basename(newFrames[0][1]))[0] + + secondMethod = newFrames[1][0] + secondFile = os.path.splitext(os.path.basename(newFrames[1][1]))[0] + + syncCase = (("_run", "_synctest"), + ("runWithWarningsSuppressed", "util")) + asyncCase = (("maybeDeferred", "defer"), + ("runWithWarningsSuppressed", "utils")) + + twoFrames = ((firstMethod, firstFile), (secondMethod, secondFile)) + + if _PY3: + # On PY3, we have an extra frame which is reraising the exception + for frame in newFrames: + frameFile = os.path.splitext(os.path.basename(frame[1]))[0] + if frameFile == "compat" and frame[0] == "reraise": + # If it's in the compat module and is reraise, BLAM IT + newFrames.pop(newFrames.index(frame)) + + if twoFrames == syncCase: + newFrames = newFrames[2:] + elif twoFrames == asyncCase: + newFrames = newFrames[3:] + elif (firstMethod, firstFile) == ("_runCallbacks", "defer"): + newFrames = newFrames[1:] + + if not newFrames: + # The method fails before getting called, probably an argument + # problem + return newFrames + + last = newFrames[-1] + if (last[0].startswith('fail') + and os.path.splitext(os.path.basename(last[1]))[0] == '_synctest'): + newFrames = newFrames[:-1] + + return newFrames + + + def _formatFailureTraceback(self, fail): + if isinstance(fail, str): + return fail.rstrip() + '\n' + fail.frames, frames = self._trimFrames(fail.frames), fail.frames + result = fail.getTraceback(detail=self.tbformat, + elideFrameworkCode=True) + fail.frames = frames + return result + + + def _groupResults(self, results, formatter): + """ + Group tests together based on their results. + + @param results: An iterable of tuples of two or more elements. The + first element of each tuple is a test case. The remaining + elements describe the outcome of that test case. + + @param formatter: A callable which turns a test case result into a + string. The elements after the first of the tuples in + C{results} will be passed as positional arguments to + C{formatter}. + + @return: A C{list} of two-tuples. The first element of each tuple + is a unique string describing one result from at least one of + the test cases in C{results}. The second element is a list of + the test cases which had that result. + """ + groups = OrderedDict() + for content in results: + case = content[0] + outcome = content[1:] + key = formatter(*outcome) + groups.setdefault(key, []).append(case) + return items(groups) + + + def _printResults(self, flavor, errors, formatter): + """ + Print a group of errors to the stream. + + @param flavor: A string indicating the kind of error (e.g. 'TODO'). + @param errors: A list of errors, often L{failure.Failure}s, but + sometimes 'todo' errors. + @param formatter: A callable that knows how to format the errors. + """ + for reason, cases in self._groupResults(errors, formatter): + self._writeln(self._doubleSeparator) + self._writeln(flavor) + self._write(reason) + self._writeln('') + for case in cases: + self._writeln(case.id()) + + + def _printExpectedFailure(self, error, todo): + return 'Reason: %r\n%s' % (todo.reason, + self._formatFailureTraceback(error)) + + + def _printUnexpectedSuccess(self, todo): + ret = 'Reason: %r\n' % (todo.reason,) + if todo.errors: + ret += 'Expected errors: %s\n' % (', '.join(todo.errors),) + return ret + + + def _printErrors(self): + """ + Print all of the non-success results to the stream in full. + """ + self._write('\n') + self._printResults('[SKIPPED]', self.skips, lambda x: '%s\n' % x) + self._printResults('[TODO]', self.expectedFailures, + self._printExpectedFailure) + self._printResults('[FAIL]', self.failures, + self._formatFailureTraceback) + self._printResults('[ERROR]', self.errors, + self._formatFailureTraceback) + self._printResults('[SUCCESS!?!]', self.unexpectedSuccesses, + self._printUnexpectedSuccess) + + + def _getSummary(self): + """ + Return a formatted count of tests status results. + """ + summaries = [] + for stat in ("skips", "expectedFailures", "failures", "errors", + "unexpectedSuccesses"): + num = len(getattr(self, stat)) + if num: + summaries.append('%s=%d' % (stat, num)) + if self.successes: + summaries.append('successes=%d' % (self.successes,)) + summary = (summaries and ' (' + ', '.join(summaries) + ')') or '' + return summary + + + def _printSummary(self): + """ + Print a line summarising the test results to the stream. + """ + summary = self._getSummary() + if self.wasSuccessful(): + status = "PASSED" + else: + status = "FAILED" + self._write("%s%s\n", status, summary) + + + def done(self): + """ + Summarize the result of the test run. + + The summary includes a report of all of the errors, todos, skips and + so forth that occurred during the run. It also includes the number of + tests that were run and how long it took to run them (not including + load time). + + Expects that C{_printErrors}, C{_writeln}, C{_write}, C{_printSummary} + and C{_separator} are all implemented. + """ + if self._publisher is not None: + self._publisher.removeObserver(self._observeWarnings) + self._printErrors() + self._writeln(self._separator) + if self._startTime is not None: + self._writeln('Ran %d tests in %.3fs', self.testsRun, + time.time() - self._startTime) + self._write('\n') + self._printSummary() + + + +class MinimalReporter(Reporter): + """ + A minimalist reporter that prints only a summary of the test result, in + the form of (timeTaken, #tests, #tests, #errors, #failures, #skips). + """ + + def _printErrors(self): + """ + Don't print a detailed summary of errors. We only care about the + counts. + """ + + + def _printSummary(self): + """ + Print out a one-line summary of the form: + '%(runtime) %(number_of_tests) %(number_of_tests) %(num_errors) + %(num_failures) %(num_skips)' + """ + numTests = self.testsRun + if self._startTime is not None: + timing = self._getTime() - self._startTime + else: + timing = 0 + t = (timing, numTests, numTests, + len(self.errors), len(self.failures), len(self.skips)) + self._writeln(' '.join(map(str, t))) + + + +class TextReporter(Reporter): + """ + Simple reporter that prints a single character for each test as it runs, + along with the standard Trial summary text. + """ + + def addSuccess(self, test): + super(TextReporter, self).addSuccess(test) + self._write('.') + + + def addError(self, *args): + super(TextReporter, self).addError(*args) + self._write('E') + + + def addFailure(self, *args): + super(TextReporter, self).addFailure(*args) + self._write('F') + + + def addSkip(self, *args): + super(TextReporter, self).addSkip(*args) + self._write('S') + + + def addExpectedFailure(self, *args): + super(TextReporter, self).addExpectedFailure(*args) + self._write('T') + + + def addUnexpectedSuccess(self, *args): + super(TextReporter, self).addUnexpectedSuccess(*args) + self._write('!') + + + +class VerboseTextReporter(Reporter): + """ + A verbose reporter that prints the name of each test as it is running. + + Each line is printed with the name of the test, followed by the result of + that test. + """ + + # This is actually the bwverbose option + + def startTest(self, tm): + self._write('%s ... ', tm.id()) + super(VerboseTextReporter, self).startTest(tm) + + + def addSuccess(self, test): + super(VerboseTextReporter, self).addSuccess(test) + self._write('[OK]') + + + def addError(self, *args): + super(VerboseTextReporter, self).addError(*args) + self._write('[ERROR]') + + + def addFailure(self, *args): + super(VerboseTextReporter, self).addFailure(*args) + self._write('[FAILURE]') + + + def addSkip(self, *args): + super(VerboseTextReporter, self).addSkip(*args) + self._write('[SKIPPED]') + + + def addExpectedFailure(self, *args): + super(VerboseTextReporter, self).addExpectedFailure(*args) + self._write('[TODO]') + + + def addUnexpectedSuccess(self, *args): + super(VerboseTextReporter, self).addUnexpectedSuccess(*args) + self._write('[SUCCESS!?!]') + + + def stopTest(self, test): + super(VerboseTextReporter, self).stopTest(test) + self._write('\n') + + + +class TimingTextReporter(VerboseTextReporter): + """ + Prints out each test as it is running, followed by the time taken for each + test to run. + """ + + def stopTest(self, method): + """ + Mark the test as stopped, and write the time it took to run the test + to the stream. + """ + super(TimingTextReporter, self).stopTest(method) + self._write("(%.03f secs)\n" % self._lastTime) + + + +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except: + # guess false in case of error + return False + supported = classmethod(supported) + + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_OUTPUT_HANDLE, \ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_OUTPUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUTPUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + + def write(self, text, color): + self.stream.write(text) + + + +@implementer(itrial.IReporter) +class SubunitReporter(object): + """ + Reports test output via Subunit. + + @ivar _subunit: The subunit protocol client that we are wrapping. + + @ivar _successful: An internal variable, used to track whether we have + received only successful results. + + @since: 10.0 + """ + + def __init__(self, stream=sys.stdout, tbformat='default', + realtime=False, publisher=None): + """ + Construct a L{SubunitReporter}. + + @param stream: A file-like object representing the stream to print + output to. Defaults to stdout. + @param tbformat: The format for tracebacks. Ignored, since subunit + always uses Python's standard format. + @param realtime: Whether or not to print exceptions in the middle + of the test results. Ignored, since subunit always does this. + @param publisher: The log publisher which will be preserved for + reporting events. Ignored, as it's not relevant to subunit. + """ + if TestProtocolClient is None: + raise Exception("Subunit not available") + self._subunit = TestProtocolClient(stream) + self._successful = True + + + def done(self): + """ + Record that the entire test suite run is finished. + + We do nothing, since a summary clause is irrelevant to the subunit + protocol. + """ + pass + + + def shouldStop(self): + """ + Whether or not the test runner should stop running tests. + """ + return self._subunit.shouldStop + shouldStop = property(shouldStop) + + + def stop(self): + """ + Signal that the test runner should stop running tests. + """ + return self._subunit.stop() + + + def wasSuccessful(self): + """ + Has the test run been successful so far? + + @return: C{True} if we have received no reports of errors or failures, + C{False} otherwise. + """ + # Subunit has a bug in its implementation of wasSuccessful, see + # https://bugs.edge.launchpad.net/subunit/+bug/491090, so we can't + # simply forward it on. + return self._successful + + + def startTest(self, test): + """ + Record that C{test} has started. + """ + return self._subunit.startTest(test) + + + def stopTest(self, test): + """ + Record that C{test} has completed. + """ + return self._subunit.stopTest(test) + + + def addSuccess(self, test): + """ + Record that C{test} was successful. + """ + return self._subunit.addSuccess(test) + + + def addSkip(self, test, reason): + """ + Record that C{test} was skipped for C{reason}. + + Some versions of subunit don't have support for addSkip. In those + cases, the skip is reported as a success. + + @param test: A unittest-compatible C{TestCase}. + @param reason: The reason for it being skipped. The C{str()} of this + object will be included in the subunit output stream. + """ + addSkip = getattr(self._subunit, 'addSkip', None) + if addSkip is None: + self.addSuccess(test) + else: + self._subunit.addSkip(test, reason) + + + def addError(self, test, err): + """ + Record that C{test} failed with an unexpected error C{err}. + + Also marks the run as being unsuccessful, causing + L{SubunitReporter.wasSuccessful} to return C{False}. + """ + self._successful = False + return self._subunit.addError( + test, util.excInfoOrFailureToExcInfo(err)) + + + def addFailure(self, test, err): + """ + Record that C{test} failed an assertion with the error C{err}. + + Also marks the run as being unsuccessful, causing + L{SubunitReporter.wasSuccessful} to return C{False}. + """ + self._successful = False + return self._subunit.addFailure( + test, util.excInfoOrFailureToExcInfo(err)) + + + def addExpectedFailure(self, test, failure, todo): + """ + Record an expected failure from a test. + + Some versions of subunit do not implement this. For those versions, we + record a success. + """ + failure = util.excInfoOrFailureToExcInfo(failure) + addExpectedFailure = getattr(self._subunit, 'addExpectedFailure', None) + if addExpectedFailure is None: + self.addSuccess(test) + else: + addExpectedFailure(test, failure) + + + def addUnexpectedSuccess(self, test, todo=None): + """ + Record an unexpected success. + + Since subunit has no way of expressing this concept, we record a + success on the subunit stream. + """ + # Not represented in pyunit/subunit. + self.addSuccess(test) + + + +class TreeReporter(Reporter): + """ + Print out the tests in the form a tree. + + Tests are indented according to which class and module they belong. + Results are printed in ANSI color. + """ + + currentLine = '' + indent = ' ' + columns = 79 + + FAILURE = 'red' + ERROR = 'red' + TODO = 'blue' + SKIP = 'blue' + TODONE = 'red' + SUCCESS = 'green' + + def __init__(self, stream=sys.stdout, *args, **kwargs): + super(TreeReporter, self).__init__(stream, *args, **kwargs) + self._lastTest = [] + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(stream): + self._colorizer = colorizer(stream) + break + + + def getDescription(self, test): + """ + Return the name of the method which 'test' represents. This is + what gets displayed in the leaves of the tree. + + e.g. getDescription(TestCase('test_foo')) ==> test_foo + """ + return test.id().split('.')[-1] + + + def addSuccess(self, test): + super(TreeReporter, self).addSuccess(test) + self.endLine('[OK]', self.SUCCESS) + + + def addError(self, *args): + super(TreeReporter, self).addError(*args) + self.endLine('[ERROR]', self.ERROR) + + + def addFailure(self, *args): + super(TreeReporter, self).addFailure(*args) + self.endLine('[FAIL]', self.FAILURE) + + + def addSkip(self, *args): + super(TreeReporter, self).addSkip(*args) + self.endLine('[SKIPPED]', self.SKIP) + + + def addExpectedFailure(self, *args): + super(TreeReporter, self).addExpectedFailure(*args) + self.endLine('[TODO]', self.TODO) + + + def addUnexpectedSuccess(self, *args): + super(TreeReporter, self).addUnexpectedSuccess(*args) + self.endLine('[SUCCESS!?!]', self.TODONE) + + + def _write(self, format, *args): + if args: + format = format % args + self.currentLine = format + super(TreeReporter, self)._write(self.currentLine) + + + def _getPreludeSegments(self, testID): + """ + Return a list of all non-leaf segments to display in the tree. + + Normally this is the module and class name. + """ + segments = testID.split('.')[:-1] + if len(segments) == 0: + return segments + segments = [ + seg for seg in ('.'.join(segments[:-1]), segments[-1]) + if len(seg) > 0] + return segments + + + def _testPrelude(self, testID): + """ + Write the name of the test to the stream, indenting it appropriately. + + If the test is the first test in a new 'branch' of the tree, also + write all of the parents in that branch. + """ + segments = self._getPreludeSegments(testID) + indentLevel = 0 + for seg in segments: + if indentLevel < len(self._lastTest): + if seg != self._lastTest[indentLevel]: + self._write('%s%s\n' % (self.indent * indentLevel, seg)) + else: + self._write('%s%s\n' % (self.indent * indentLevel, seg)) + indentLevel += 1 + self._lastTest = segments + + + def cleanupErrors(self, errs): + self._colorizer.write(' cleanup errors', self.ERROR) + self.endLine('[ERROR]', self.ERROR) + super(TreeReporter, self).cleanupErrors(errs) + + + def upDownError(self, method, error, warn, printStatus): + self._colorizer.write(" %s" % method, self.ERROR) + if printStatus: + self.endLine('[ERROR]', self.ERROR) + super(TreeReporter, self).upDownError(method, error, warn, printStatus) + + + def startTest(self, test): + """ + Called when C{test} starts. Writes the tests name to the stream using + a tree format. + """ + self._testPrelude(test.id()) + self._write('%s%s ... ' % (self.indent * (len(self._lastTest)), + self.getDescription(test))) + super(TreeReporter, self).startTest(test) + + + def endLine(self, message, color): + """ + Print 'message' in the given color. + + @param message: A string message, usually '[OK]' or something similar. + @param color: A string color, 'red', 'green' and so forth. + """ + spaces = ' ' * (self.columns - len(self.currentLine) - len(message)) + super(TreeReporter, self)._write(spaces) + self._colorizer.write(message, color) + super(TreeReporter, self)._write("\n") + + + def _printSummary(self): + """ + Print a line summarising the test results to the stream, and color the + status result. + """ + summary = self._getSummary() + if self.wasSuccessful(): + status = "PASSED" + color = self.SUCCESS + else: + status = "FAILED" + color = self.FAILURE + self._colorizer.write(status, color) + self._write("%s\n", summary) diff --git a/contrib/python/Twisted/py2/twisted/trial/runner.py b/contrib/python/Twisted/py2/twisted/trial/runner.py new file mode 100644 index 00000000000..220c34ba706 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/runner.py @@ -0,0 +1,1064 @@ +# -*- test-case-name: twisted.trial.test.test_runner -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A miscellany of code used to run Trial tests. + +Maintainer: Jonathan Lange +""" + +from __future__ import absolute_import, division + +__all__ = [ + 'TestSuite', + + 'DestructiveTestSuite', 'ErrorHolder', 'LoggedSuite', + 'TestHolder', 'TestLoader', 'TrialRunner', 'TrialSuite', + + 'filenameToModule', 'isPackage', 'isPackageDirectory', 'isTestCase', + 'name', 'samefile', 'NOT_IN_TEST', + ] + +import doctest +import inspect +import os +import sys +import time +import types +import warnings + +from twisted.python import reflect, log, failure, modules, filepath +from twisted.python.compat import _PY3, _PY35PLUS + +from twisted.internet import defer +from twisted.trial import util, unittest +from twisted.trial.itrial import ITestCase +from twisted.trial.reporter import _ExitWrapper, UncleanWarningsReporterWrapper +from twisted.trial._asyncrunner import _ForceGarbageCollectionDecorator, _iterateTests +from twisted.trial._synctest import _logObserver + +# These are imported so that they remain in the public API for t.trial.runner +from twisted.trial.unittest import TestSuite + +from zope.interface import implementer + +pyunit = __import__('unittest') + + + +def isPackage(module): + """Given an object return True if the object looks like a package""" + if not isinstance(module, types.ModuleType): + return False + basename = os.path.splitext(os.path.basename(module.__file__))[0] + return basename == '__init__' + + +def isPackageDirectory(dirname): + """ + Is the directory at path 'dirname' a Python package directory? + Returns the name of the __init__ file (it may have a weird extension) + if dirname is a package directory. Otherwise, returns False + """ + def _getSuffixes(): + if _PY3: + import importlib + return importlib.machinery.all_suffixes() + else: + import imp + return list(zip(*imp.get_suffixes()))[0] + + + for ext in _getSuffixes(): + initFile = '__init__' + ext + if os.path.exists(os.path.join(dirname, initFile)): + return initFile + return False + + +def samefile(filename1, filename2): + """ + A hacky implementation of C{os.path.samefile}. Used by L{filenameToModule} + when the platform doesn't provide C{os.path.samefile}. Do not use this. + """ + return os.path.abspath(filename1) == os.path.abspath(filename2) + + +def filenameToModule(fn): + """ + Given a filename, do whatever possible to return a module object matching + that file. + + If the file in question is a module in Python path, properly import and + return that module. Otherwise, load the source manually. + + @param fn: A filename. + @return: A module object. + @raise ValueError: If C{fn} does not exist. + """ + if not os.path.exists(fn): + raise ValueError("%r doesn't exist" % (fn,)) + try: + ret = reflect.namedAny(reflect.filenameToModuleName(fn)) + except (ValueError, AttributeError): + # Couldn't find module. The file 'fn' is not in PYTHONPATH + return _importFromFile(fn) + + # >=3.7 has __file__ attribute as None, previously __file__ was not present + if getattr(ret, "__file__", None) is None: + # This isn't a Python module in a package, so import it from a file + return _importFromFile(fn) + + # ensure that the loaded module matches the file + retFile = os.path.splitext(ret.__file__)[0] + '.py' + # not all platforms (e.g. win32) have os.path.samefile + same = getattr(os.path, 'samefile', samefile) + if os.path.isfile(fn) and not same(fn, retFile): + del sys.modules[ret.__name__] + ret = _importFromFile(fn) + return ret + + +def _importFromFile(fn, moduleName=None): + fn = _resolveDirectory(fn) + if not moduleName: + moduleName = os.path.splitext(os.path.split(fn)[-1])[0] + if moduleName in sys.modules: + return sys.modules[moduleName] + if _PY35PLUS: + import importlib + + spec = importlib.util.spec_from_file_location(moduleName, fn) + if not spec: + raise SyntaxError(fn) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[moduleName] = module + else: + import imp + + with open(fn, 'r') as fd: + module = imp.load_source(moduleName, fn, fd) + return module + + +def _resolveDirectory(fn): + if os.path.isdir(fn): + initFile = isPackageDirectory(fn) + if initFile: + fn = os.path.join(fn, initFile) + else: + raise ValueError('%r is not a package directory' % (fn,)) + return fn + + +def _getMethodNameInClass(method): + """ + Find the attribute name on the method's class which refers to the method. + + For some methods, notably decorators which have not had __name__ set correctly: + + getattr(method.im_class, method.__name__) != method + """ + if getattr(method.im_class, method.__name__, object()) != method: + for alias in dir(method.im_class): + if getattr(method.im_class, alias, object()) == method: + return alias + return method.__name__ + + +class DestructiveTestSuite(TestSuite): + """ + A test suite which remove the tests once run, to minimize memory usage. + """ + + def run(self, result): + """ + Almost the same as L{TestSuite.run}, but with C{self._tests} being + empty at the end. + """ + while self._tests: + if result.shouldStop: + break + test = self._tests.pop(0) + test(result) + return result + + + +# When an error occurs outside of any test, the user will see this string +# in place of a test's name. +NOT_IN_TEST = "" + + + +class LoggedSuite(TestSuite): + """ + Any errors logged in this suite will be reported to the L{TestResult} + object. + """ + + def run(self, result): + """ + Run the suite, storing all errors in C{result}. If an error is logged + while no tests are running, then it will be added as an error to + C{result}. + + @param result: A L{TestResult} object. + """ + observer = _logObserver + observer._add() + super(LoggedSuite, self).run(result) + observer._remove() + for error in observer.getErrors(): + result.addError(TestHolder(NOT_IN_TEST), error) + observer.flushErrors() + + + +class TrialSuite(TestSuite): + """ + Suite to wrap around every single test in a C{trial} run. Used internally + by Trial to set up things necessary for Trial tests to work, regardless of + what context they are run in. + """ + + def __init__(self, tests=(), forceGarbageCollection=False): + if forceGarbageCollection: + newTests = [] + for test in tests: + test = unittest.decorate( + test, _ForceGarbageCollectionDecorator) + newTests.append(test) + tests = newTests + suite = LoggedSuite(tests) + super(TrialSuite, self).__init__([suite]) + + + def _bail(self): + from twisted.internet import reactor + d = defer.Deferred() + reactor.addSystemEventTrigger('after', 'shutdown', + lambda: d.callback(None)) + reactor.fireSystemEvent('shutdown') # radix's suggestion + # As long as TestCase does crap stuff with the reactor we need to + # manually shutdown the reactor here, and that requires util.wait + # :( + # so that the shutdown event completes + unittest.TestCase('mktemp')._wait(d) + + def run(self, result): + try: + TestSuite.run(self, result) + finally: + self._bail() + + +def name(thing): + """ + @param thing: an object from modules (instance of PythonModule, + PythonAttribute), a TestCase subclass, or an instance of a TestCase. + """ + if isTestCase(thing): + # TestCase subclass + theName = reflect.qual(thing) + else: + # thing from trial, or thing from modules. + # this monstrosity exists so that modules' objects do not have to + # implement id(). -jml + try: + theName = thing.id() + except AttributeError: + theName = thing.name + return theName + + +def isTestCase(obj): + """ + @return: C{True} if C{obj} is a class that contains test cases, C{False} + otherwise. Used to find all the tests in a module. + """ + try: + return issubclass(obj, pyunit.TestCase) + except TypeError: + return False + + + +@implementer(ITestCase) +class TestHolder(object): + """ + Placeholder for a L{TestCase} inside a reporter. As far as a L{TestResult} + is concerned, this looks exactly like a unit test. + """ + + failureException = None + + def __init__(self, description): + """ + @param description: A string to be displayed L{TestResult}. + """ + self.description = description + + + def __call__(self, result): + return self.run(result) + + + def id(self): + return self.description + + + def countTestCases(self): + return 0 + + + def run(self, result): + """ + This test is just a placeholder. Run the test successfully. + + @param result: The C{TestResult} to store the results in. + @type result: L{twisted.trial.itrial.IReporter}. + """ + result.startTest(self) + result.addSuccess(self) + result.stopTest(self) + + + def shortDescription(self): + return self.description + + + +class ErrorHolder(TestHolder): + """ + Used to insert arbitrary errors into a test suite run. Provides enough + methods to look like a C{TestCase}, however, when it is run, it simply adds + an error to the C{TestResult}. The most common use-case is for when a + module fails to import. + """ + + def __init__(self, description, error): + """ + @param description: A string used by C{TestResult}s to identify this + error. Generally, this is the name of a module that failed to import. + + @param error: The error to be added to the result. Can be an `exc_info` + tuple or a L{twisted.python.failure.Failure}. + """ + super(ErrorHolder, self).__init__(description) + self.error = util.excInfoOrFailureToExcInfo(error) + + + def __repr__(self): + return "" % ( + self.description, self.error[1]) + + + def run(self, result): + """ + Run the test, reporting the error. + + @param result: The C{TestResult} to store the results in. + @type result: L{twisted.trial.itrial.IReporter}. + """ + result.startTest(self) + result.addError(self, self.error) + result.stopTest(self) + + + +class TestLoader(object): + """ + I find tests inside function, modules, files -- whatever -- then return + them wrapped inside a Test (either a L{TestSuite} or a L{TestCase}). + + @ivar methodPrefix: A string prefix. C{TestLoader} will assume that all the + methods in a class that begin with C{methodPrefix} are test cases. + + @ivar modulePrefix: A string prefix. Every module in a package that begins + with C{modulePrefix} is considered a module full of tests. + + @ivar forceGarbageCollection: A flag applied to each C{TestCase} loaded. + See L{unittest.TestCase} for more information. + + @ivar sorter: A key function used to sort C{TestCase}s, test classes, + modules and packages. + + @ivar suiteFactory: A callable which is passed a list of tests (which + themselves may be suites of tests). Must return a test suite. + """ + + methodPrefix = 'test' + modulePrefix = 'test_' + + def __init__(self): + self.suiteFactory = TestSuite + self.sorter = name + self._importErrors = [] + + def sort(self, xs): + """ + Sort the given things using L{sorter}. + + @param xs: A list of test cases, class or modules. + """ + return sorted(xs, key=self.sorter) + + def findTestClasses(self, module): + """Given a module, return all Trial test classes""" + classes = [] + for name, val in inspect.getmembers(module): + if isTestCase(val): + classes.append(val) + return self.sort(classes) + + def findByName(self, name): + """ + Return a Python object given a string describing it. + + @param name: a string which may be either a filename or a + fully-qualified Python name. + + @return: If C{name} is a filename, return the module. If C{name} is a + fully-qualified Python name, return the object it refers to. + """ + if os.path.exists(name): + return filenameToModule(name) + return reflect.namedAny(name) + + def loadModule(self, module): + """ + Return a test suite with all the tests from a module. + + Included are TestCase subclasses and doctests listed in the module's + __doctests__ module. If that's not good for you, put a function named + either C{testSuite} or C{test_suite} in your module that returns a + TestSuite, and I'll use the results of that instead. + + If C{testSuite} and C{test_suite} are both present, then I'll use + C{testSuite}. + """ + ## XXX - should I add an optional parameter to disable the check for + ## a custom suite. + ## OR, should I add another method + if not isinstance(module, types.ModuleType): + raise TypeError("%r is not a module" % (module,)) + if hasattr(module, 'testSuite'): + return module.testSuite() + elif hasattr(module, 'test_suite'): + return module.test_suite() + suite = self.suiteFactory() + for testClass in self.findTestClasses(module): + suite.addTest(self.loadClass(testClass)) + if not hasattr(module, '__doctests__'): + return suite + docSuite = self.suiteFactory() + for docTest in module.__doctests__: + docSuite.addTest(self.loadDoctests(docTest)) + return self.suiteFactory([suite, docSuite]) + loadTestsFromModule = loadModule + + def loadClass(self, klass): + """ + Given a class which contains test cases, return a sorted list of + C{TestCase} instances. + """ + if not (isinstance(klass, type) or isinstance(klass, types.ClassType)): + raise TypeError("%r is not a class" % (klass,)) + if not isTestCase(klass): + raise ValueError("%r is not a test case" % (klass,)) + names = self.getTestCaseNames(klass) + tests = self.sort([self._makeCase(klass, self.methodPrefix+name) + for name in names]) + return self.suiteFactory(tests) + loadTestsFromTestCase = loadClass + + def getTestCaseNames(self, klass): + """ + Given a class that contains C{TestCase}s, return a list of names of + methods that probably contain tests. + """ + return reflect.prefixedMethodNames(klass, self.methodPrefix) + + def loadMethod(self, method): + """ + Given a method of a C{TestCase} that represents a test, return a + C{TestCase} instance for that test. + """ + if not isinstance(method, types.MethodType): + raise TypeError("%r not a method" % (method,)) + return self._makeCase(method.im_class, _getMethodNameInClass(method)) + + def _makeCase(self, klass, methodName): + return klass(methodName) + + def loadPackage(self, package, recurse=False): + """ + Load tests from a module object representing a package, and return a + TestSuite containing those tests. + + Tests are only loaded from modules whose name begins with 'test_' + (or whatever C{modulePrefix} is set to). + + @param package: a types.ModuleType object (or reasonable facsimile + obtained by importing) which may contain tests. + + @param recurse: A boolean. If True, inspect modules within packages + within the given package (and so on), otherwise, only inspect modules + in the package itself. + + @raise: TypeError if 'package' is not a package. + + @return: a TestSuite created with my suiteFactory, containing all the + tests. + """ + if not isPackage(package): + raise TypeError("%r is not a package" % (package,)) + pkgobj = modules.getModule(package.__name__) + if recurse: + discovery = pkgobj.walkModules() + else: + discovery = pkgobj.iterModules() + discovered = [] + for disco in discovery: + if disco.name.split(".")[-1].startswith(self.modulePrefix): + discovered.append(disco) + suite = self.suiteFactory() + for modinfo in self.sort(discovered): + try: + module = modinfo.load() + except: + thingToAdd = ErrorHolder(modinfo.name, failure.Failure()) + else: + thingToAdd = self.loadModule(module) + suite.addTest(thingToAdd) + return suite + + def loadDoctests(self, module): + """ + Return a suite of tests for all the doctests defined in C{module}. + + @param module: A module object or a module name. + """ + if isinstance(module, str): + try: + module = reflect.namedAny(module) + except: + return ErrorHolder(module, failure.Failure()) + if not inspect.ismodule(module): + warnings.warn("trial only supports doctesting modules") + return + extraArgs = {} + + # Work around Python issue2604: DocTestCase.tearDown clobbers globs + def saveGlobals(test): + """ + Save C{test.globs} and replace it with a copy so that if + necessary, the original will be available for the next test + run. + """ + test._savedGlobals = getattr(test, '_savedGlobals', test.globs) + test.globs = test._savedGlobals.copy() + extraArgs['setUp'] = saveGlobals + return doctest.DocTestSuite(module, **extraArgs) + + def loadAnything(self, thing, recurse=False, parent=None, qualName=None): + """ + Given a Python object, return whatever tests that are in it. Whatever + 'in' might mean. + + @param thing: A Python object. A module, method, class or package. + @param recurse: Whether or not to look in subpackages of packages. + Defaults to False. + + @param parent: For compatibility with the Python 3 loader, does + nothing. + @param qualname: For compatibility with the Python 3 loader, does + nothing. + + @return: A C{TestCase} or C{TestSuite}. + """ + if isinstance(thing, types.ModuleType): + if isPackage(thing): + return self.loadPackage(thing, recurse) + return self.loadModule(thing) + elif isinstance(thing, types.ClassType): + return self.loadClass(thing) + elif isinstance(thing, type): + return self.loadClass(thing) + elif isinstance(thing, types.MethodType): + return self.loadMethod(thing) + raise TypeError("No loader for %r. Unrecognized type" % (thing,)) + + def loadByName(self, name, recurse=False): + """ + Given a string representing a Python object, return whatever tests + are in that object. + + If C{name} is somehow inaccessible (e.g. the module can't be imported, + there is no Python object with that name etc) then return an + L{ErrorHolder}. + + @param name: The fully-qualified name of a Python object. + """ + try: + thing = self.findByName(name) + except: + return ErrorHolder(name, failure.Failure()) + return self.loadAnything(thing, recurse) + + loadTestsFromName = loadByName + + def loadByNames(self, names, recurse=False): + """ + Construct a TestSuite containing all the tests found in 'names', where + names is a list of fully qualified python names and/or filenames. The + suite returned will have no duplicate tests, even if the same object + is named twice. + """ + things = [] + errors = [] + for name in names: + try: + things.append(self.findByName(name)) + except: + errors.append(ErrorHolder(name, failure.Failure())) + suites = [self.loadAnything(thing, recurse) + for thing in self._uniqueTests(things)] + suites.extend(errors) + return self.suiteFactory(suites) + + + def _uniqueTests(self, things): + """ + Gather unique suite objects from loaded things. This will guarantee + uniqueness of inherited methods on TestCases which would otherwise hash + to same value and collapse to one test unexpectedly if using simpler + means: e.g. set(). + """ + seen = set() + for thing in things: + if isinstance(thing, types.MethodType): + thing = (thing, thing.im_class) + else: + thing = (thing,) + + if thing not in seen: + yield thing[0] + seen.add(thing) + + + +class Py3TestLoader(TestLoader): + """ + A test loader finds tests from the functions, modules, and files that is + asked to and loads them into a L{TestSuite} or L{TestCase}. + + See L{TestLoader} for further details. + """ + + def loadFile(self, fileName, recurse=False): + """ + Load a file, and then the tests in that file. + + @param fileName: The file name to load. + @param recurse: A boolean. If True, inspect modules within packages + within the given package (and so on), otherwise, only inspect + modules in the package itself. + """ + from importlib.machinery import SourceFileLoader + + name = reflect.filenameToModuleName(fileName) + try: + module = SourceFileLoader(name, fileName).load_module() + return self.loadAnything(module, recurse=recurse) + except OSError: + raise ValueError("{} is not a Python file.".format(fileName)) + + + def findByName(self, _name, recurse=False): + """ + Find and load tests, given C{name}. + + This partially duplicates the logic in C{unittest.loader.TestLoader}. + + @param name: The qualified name of the thing to load. + @param recurse: A boolean. If True, inspect modules within packages + within the given package (and so on), otherwise, only inspect + modules in the package itself. + """ + if os.sep in _name: + # It's a file, try and get the module name for this file. + name = reflect.filenameToModuleName(_name) + + try: + # Try and import it, if it's on the path. + # CAVEAT: If you have two twisteds, and you try and import the + # one NOT on your path, it'll load the one on your path. But + # that's silly, nobody should do that, and existing Trial does + # that anyway. + __import__(name) + except ImportError: + # If we can't import it, look for one NOT on the path. + return self.loadFile(_name, recurse=recurse) + + else: + name = _name + + obj = parent = remaining = None + + for searchName, remainingName in _qualNameWalker(name): + # Walk down the qualified name, trying to import a module. For + # example, `twisted.test.test_paths.FilePathTests` would try + # the full qualified name, then just up to test_paths, and then + # just up to test, and so forth. + # This gets us the highest level thing which is a module. + try: + obj = reflect.namedModule(searchName) + # If we reach here, we have successfully found a module. + # obj will be the module, and remaining will be the remaining + # part of the qualified name. + remaining = remainingName + break + + except ImportError: + # Check to see where the ImportError happened. If it happened + # in this file, ignore it. + tb = sys.exc_info()[2] + + # Walk down to the deepest frame, where it actually happened. + while tb.tb_next is not None: + tb = tb.tb_next + + # Get the filename that the ImportError originated in. + filenameWhereHappened = tb.tb_frame.f_code.co_filename + + # If it originated in the reflect file, then it's because it + # doesn't exist. If it originates elsewhere, it's because an + # ImportError happened in a module that does exist. + if filenameWhereHappened != reflect.__file__: + raise + + if remaining == "": + raise reflect.ModuleNotFound( + "The module {} does not exist.".format(name) + ) + + if obj is None: + # If it's none here, we didn't get to import anything. + # Try something drastic. + obj = reflect.namedAny(name) + remaining = name.split(".")[len(".".split(obj.__name__))+1:] + + try: + for part in remaining: + # Walk down the remaining modules. Hold on to the parent for + # methods, as on Python 3, you can no longer get the parent + # class from just holding onto the method. + parent, obj = obj, getattr(obj, part) + except AttributeError: + raise AttributeError("{} does not exist.".format(name)) + + return self.loadAnything(obj, parent=parent, qualName=remaining, + recurse=recurse) + + + def loadAnything(self, obj, recurse=False, parent=None, qualName=None): + """ + Load absolutely anything (as long as that anything is a module, + package, class, or method (with associated parent class and qualname). + + @param obj: The object to load. + @param recurse: A boolean. If True, inspect modules within packages + within the given package (and so on), otherwise, only inspect + modules in the package itself. + @param parent: If C{obj} is a method, this is the parent class of the + method. C{qualName} is also required. + @param qualName: If C{obj} is a method, this a list containing is the + qualified name of the method. C{parent} is also required. + """ + if isinstance(obj, types.ModuleType): + # It looks like a module + if isPackage(obj): + # It's a package, so recurse down it. + return self.loadPackage(obj, recurse=recurse) + # Otherwise get all the tests in the module. + return self.loadTestsFromModule(obj) + elif isinstance(obj, type) and issubclass(obj, pyunit.TestCase): + # We've found a raw test case, get the tests from it. + return self.loadTestsFromTestCase(obj) + elif (isinstance(obj, types.FunctionType) and + isinstance(parent, type) and + issubclass(parent, pyunit.TestCase)): + # We've found a method, and its parent is a TestCase. Instantiate + # it with the name of the method we want. + name = qualName[-1] + inst = parent(name) + + # Sanity check to make sure that the method we have got from the + # test case is the same one as was passed in. This doesn't actually + # use the function we passed in, because reasons. + assert getattr(inst, inst._testMethodName).__func__ == obj + + return inst + elif isinstance(obj, TestSuite): + # We've found a test suite. + return obj + else: + raise TypeError("don't know how to make test from: %s" % (obj,)) + + + def loadByName(self, name, recurse=False): + """ + Load some tests by name. + + @param name: The qualified name for the test to load. + @param recurse: A boolean. If True, inspect modules within packages + within the given package (and so on), otherwise, only inspect + modules in the package itself. + """ + try: + return self.suiteFactory([self.findByName(name, recurse=recurse)]) + except: + return self.suiteFactory([ErrorHolder(name, failure.Failure())]) + + + def loadByNames(self, names, recurse=False): + """ + Load some tests by a list of names. + + @param names: A L{list} of qualified names. + @param recurse: A boolean. If True, inspect modules within packages + within the given package (and so on), otherwise, only inspect + modules in the package itself. + """ + things = [] + errors = [] + for name in names: + try: + things.append(self.loadByName(name, recurse=recurse)) + except: + errors.append(ErrorHolder(name, failure.Failure())) + things.extend(errors) + return self.suiteFactory(self._uniqueTests(things)) + + + def loadClass(self, klass): + """ + Given a class which contains test cases, return a list of L{TestCase}s. + + @param klass: The class to load tests from. + """ + if not isinstance(klass, type): + raise TypeError("%r is not a class" % (klass,)) + if not isTestCase(klass): + raise ValueError("%r is not a test case" % (klass,)) + names = self.getTestCaseNames(klass) + tests = self.sort([self._makeCase(klass, self.methodPrefix+name) + for name in names]) + return self.suiteFactory(tests) + + + def loadMethod(self, method): + raise NotImplementedError("Can't happen on Py3") + + + def _uniqueTests(self, things): + """ + Gather unique suite objects from loaded things. This will guarantee + uniqueness of inherited methods on TestCases which would otherwise hash + to same value and collapse to one test unexpectedly if using simpler + means: e.g. set(). + """ + seen = set() + for testthing in things: + testthings = testthing._tests + for thing in testthings: + # This is horrible. + if str(thing) not in seen: + yield thing + seen.add(str(thing)) + + +def _qualNameWalker(qualName): + """ + Given a Python qualified name, this function yields a 2-tuple of the most + specific qualified name first, followed by the next-most-specific qualified + name, and so on, paired with the remainder of the qualified name. + + @param qualName: A Python qualified name. + @type qualName: L{str} + """ + # Yield what we were just given + yield (qualName, []) + + # If they want more, split the qualified name up + qualParts = qualName.split(".") + + for index in range(1, len(qualParts)): + # This code here will produce, from the example walker.texas.ranger: + # (walker.texas, ["ranger"]) + # (walker, ["texas", "ranger"]) + yield (".".join(qualParts[:-index]), qualParts[-index:]) + + +if _PY3: + del TestLoader + TestLoader = Py3TestLoader + + + +class TrialRunner(object): + """ + A specialised runner that the trial front end uses. + """ + + DEBUG = 'debug' + DRY_RUN = 'dry-run' + + def _setUpTestdir(self): + self._tearDownLogFile() + currentDir = os.getcwd() + base = filepath.FilePath(self.workingDirectory) + testdir, self._testDirLock = util._unusedTestDirectory(base) + os.chdir(testdir.path) + return currentDir + + + def _tearDownTestdir(self, oldDir): + os.chdir(oldDir) + self._testDirLock.unlock() + + + _log = log + def _makeResult(self): + reporter = self.reporterFactory(self.stream, self.tbformat, + self.rterrors, self._log) + if self._exitFirst: + reporter = _ExitWrapper(reporter) + if self.uncleanWarnings: + reporter = UncleanWarningsReporterWrapper(reporter) + return reporter + + def __init__(self, reporterFactory, + mode=None, + logfile='test.log', + stream=sys.stdout, + profile=False, + tracebackFormat='default', + realTimeErrors=False, + uncleanWarnings=False, + workingDirectory=None, + forceGarbageCollection=False, + debugger=None, + exitFirst=False): + self.reporterFactory = reporterFactory + self.logfile = logfile + self.mode = mode + self.stream = stream + self.tbformat = tracebackFormat + self.rterrors = realTimeErrors + self.uncleanWarnings = uncleanWarnings + self._result = None + self.workingDirectory = workingDirectory or '_trial_temp' + self._logFileObserver = None + self._logFileObject = None + self._forceGarbageCollection = forceGarbageCollection + self.debugger = debugger + self._exitFirst = exitFirst + if profile: + self.run = util.profiled(self.run, 'profile.data') + + def _tearDownLogFile(self): + if self._logFileObserver is not None: + log.removeObserver(self._logFileObserver.emit) + self._logFileObserver = None + if self._logFileObject is not None: + self._logFileObject.close() + self._logFileObject = None + + def _setUpLogFile(self): + self._tearDownLogFile() + if self.logfile == '-': + logFile = sys.stdout + else: + logFile = open(self.logfile, 'a') + self._logFileObject = logFile + self._logFileObserver = log.FileLogObserver(logFile) + log.startLoggingWithObserver(self._logFileObserver.emit, 0) + + + def run(self, test): + """ + Run the test or suite and return a result object. + """ + test = unittest.decorate(test, ITestCase) + return self._runWithoutDecoration(test, self._forceGarbageCollection) + + + def _runWithoutDecoration(self, test, forceGarbageCollection=False): + """ + Private helper that runs the given test but doesn't decorate it. + """ + result = self._makeResult() + # decorate the suite with reactor cleanup and log starting + # This should move out of the runner and be presumed to be + # present + suite = TrialSuite([test], forceGarbageCollection) + startTime = time.time() + if self.mode == self.DRY_RUN: + for single in _iterateTests(suite): + result.startTest(single) + result.addSuccess(single) + result.stopTest(single) + else: + if self.mode == self.DEBUG: + run = lambda: self.debugger.runcall(suite.run, result) + else: + run = lambda: suite.run(result) + + oldDir = self._setUpTestdir() + try: + self._setUpLogFile() + run() + finally: + self._tearDownLogFile() + self._tearDownTestdir(oldDir) + + endTime = time.time() + done = getattr(result, 'done', None) + if done is None: + warnings.warn( + "%s should implement done() but doesn't. Falling back to " + "printErrors() and friends." % reflect.qual(result.__class__), + category=DeprecationWarning, stacklevel=3) + result.printErrors() + result.writeln(result.separator) + result.writeln('Ran %d tests in %.3fs', result.testsRun, + endTime - startTime) + result.write('\n') + result.printSummary() + else: + result.done() + return result + + + def runUntilFailure(self, test): + """ + Repeatedly run C{test} until it fails. + """ + count = 0 + while True: + count += 1 + self.stream.write("Test Pass %d\n" % (count,)) + if count == 1: + result = self.run(test) + else: + result = self._runWithoutDecoration(test) + if result.testsRun == 0: + break + if not result.wasSuccessful(): + break + return result diff --git a/contrib/python/Twisted/py2/twisted/trial/unittest.py b/contrib/python/Twisted/py2/twisted/trial/unittest.py new file mode 100644 index 00000000000..d3f60346a63 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/unittest.py @@ -0,0 +1,35 @@ +# -*- test-case-name: twisted.trial.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Things likely to be used by writers of unit tests. +""" + +from __future__ import division, absolute_import + +# Define the public API from the two implementation modules +from twisted.trial._synctest import ( + FailTest, SkipTest, SynchronousTestCase, PyUnitResultAdapter, Todo, + makeTodo) +from twisted.trial._asynctest import TestCase +from twisted.trial._asyncrunner import ( + TestSuite, TestDecorator, decorate) + +# Further obscure the origins of these objects, to reduce surprise (and this is +# what the values were before code got shuffled around between files, but was +# otherwise unchanged). +FailTest.__module__ = SkipTest.__module__ = __name__ + +__all__ = [ + 'decorate', + 'FailTest', + 'makeTodo', + 'PyUnitResultAdapter', + 'SkipTest', + 'SynchronousTestCase', + 'TestCase', + 'TestDecorator', + 'TestSuite', + 'Todo', + ] diff --git a/contrib/python/Twisted/py2/twisted/trial/util.py b/contrib/python/Twisted/py2/twisted/trial/util.py new file mode 100644 index 00000000000..4f07112d323 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/trial/util.py @@ -0,0 +1,411 @@ +# -*- test-case-name: twisted.trial.test.test_util -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. +# + +""" +A collection of utility functions and classes, used internally by Trial. + +This code is for Trial's internal use. Do NOT use this code if you are writing +tests. It is subject to change at the Trial maintainer's whim. There is +nothing here in this module for you to use unless you are maintaining Trial. + +Any non-Trial Twisted code that uses this module will be shot. + +Maintainer: Jonathan Lange + +@var DEFAULT_TIMEOUT_DURATION: The default timeout which will be applied to + asynchronous (ie, Deferred-returning) test methods, in seconds. +""" + +from __future__ import division, absolute_import, print_function + +from random import randrange + +from twisted.internet import defer, utils, interfaces +from twisted.python.failure import Failure +from twisted.python.filepath import FilePath +from twisted.python.lockfile import FilesystemLock + +__all__ = [ + 'DEFAULT_TIMEOUT_DURATION', + + 'excInfoOrFailureToExcInfo', 'suppress', 'acquireAttribute'] + +DEFAULT_TIMEOUT = object() +DEFAULT_TIMEOUT_DURATION = 120.0 + + + +class DirtyReactorAggregateError(Exception): + """ + Passed to L{twisted.trial.itrial.IReporter.addError} when the reactor is + left in an unclean state after a test. + + @ivar delayedCalls: The L{DelayedCall} + objects which weren't cleaned up. + @ivar selectables: The selectables which weren't cleaned up. + """ + + def __init__(self, delayedCalls, selectables=None): + self.delayedCalls = delayedCalls + self.selectables = selectables + + + def __str__(self): + """ + Return a multi-line message describing all of the unclean state. + """ + msg = "Reactor was unclean." + if self.delayedCalls: + msg += ("\nDelayedCalls: (set " + "twisted.internet.base.DelayedCall.debug = True to " + "debug)\n") + msg += "\n".join(map(str, self.delayedCalls)) + if self.selectables: + msg += "\nSelectables:\n" + msg += "\n".join(map(str, self.selectables)) + return msg + + + +class _Janitor(object): + """ + The guy that cleans up after you. + + @ivar test: The L{TestCase} to report errors about. + @ivar result: The L{IReporter} to report errors to. + @ivar reactor: The reactor to use. If None, the global reactor + will be used. + """ + def __init__(self, test, result, reactor=None): + """ + @param test: See L{_Janitor.test}. + @param result: See L{_Janitor.result}. + @param reactor: See L{_Janitor.reactor}. + """ + self.test = test + self.result = result + self.reactor = reactor + + + def postCaseCleanup(self): + """ + Called by L{unittest.TestCase} after a test to catch any logged errors + or pending L{DelayedCall}s. + """ + calls = self._cleanPending() + if calls: + aggregate = DirtyReactorAggregateError(calls) + self.result.addError(self.test, Failure(aggregate)) + return False + return True + + + def postClassCleanup(self): + """ + Called by L{unittest.TestCase} after the last test in a C{TestCase} + subclass. Ensures the reactor is clean by murdering the threadpool, + catching any pending + L{DelayedCall}s, open sockets etc. + """ + selectables = self._cleanReactor() + calls = self._cleanPending() + if selectables or calls: + aggregate = DirtyReactorAggregateError(calls, selectables) + self.result.addError(self.test, Failure(aggregate)) + self._cleanThreads() + + + def _getReactor(self): + """ + Get either the passed-in reactor or the global reactor. + """ + if self.reactor is not None: + reactor = self.reactor + else: + from twisted.internet import reactor + return reactor + + + def _cleanPending(self): + """ + Cancel all pending calls and return their string representations. + """ + reactor = self._getReactor() + + # flush short-range timers + reactor.iterate(0) + reactor.iterate(0) + + delayedCallStrings = [] + for p in reactor.getDelayedCalls(): + if p.active(): + delayedString = str(p) + p.cancel() + else: + print("WEIRDNESS! pending timed call not active!") + delayedCallStrings.append(delayedString) + return delayedCallStrings + _cleanPending = utils.suppressWarnings( + _cleanPending, (('ignore',), {'category': DeprecationWarning, + 'message': + r'reactor\.iterate cannot be used.*'})) + + def _cleanThreads(self): + reactor = self._getReactor() + if interfaces.IReactorThreads.providedBy(reactor): + if reactor.threadpool is not None: + # Stop the threadpool now so that a new one is created. + # This improves test isolation somewhat (although this is a + # post class cleanup hook, so it's only isolating classes + # from each other, not methods from each other). + reactor._stopThreadPool() + + + def _cleanReactor(self): + """ + Remove all selectables from the reactor, kill any of them that were + processes, and return their string representation. + """ + reactor = self._getReactor() + selectableStrings = [] + for sel in reactor.removeAll(): + if interfaces.IProcessTransport.providedBy(sel): + sel.signalProcess('KILL') + selectableStrings.append(repr(sel)) + return selectableStrings + + + +_DEFAULT = object() +def acquireAttribute(objects, attr, default=_DEFAULT): + """ + Go through the list 'objects' sequentially until we find one which has + attribute 'attr', then return the value of that attribute. If not found, + return 'default' if set, otherwise, raise AttributeError. + """ + for obj in objects: + if hasattr(obj, attr): + return getattr(obj, attr) + if default is not _DEFAULT: + return default + raise AttributeError('attribute %r not found in %r' % (attr, objects)) + + + +def excInfoOrFailureToExcInfo(err): + """ + Coerce a Failure to an _exc_info, if err is a Failure. + + @param err: Either a tuple such as returned by L{sys.exc_info} or a + L{Failure} object. + @return: A tuple like the one returned by L{sys.exc_info}. e.g. + C{exception_type, exception_object, traceback_object}. + """ + if isinstance(err, Failure): + # Unwrap the Failure into an exc_info tuple. + err = (err.type, err.value, err.getTracebackObject()) + return err + + + +def suppress(action='ignore', **kwarg): + """ + Sets up the .suppress tuple properly, pass options to this method as you + would the stdlib warnings.filterwarnings() + + So, to use this with a .suppress magic attribute you would do the + following: + + >>> from twisted.trial import unittest, util + >>> import warnings + >>> + >>> class TestFoo(unittest.TestCase): + ... def testFooBar(self): + ... warnings.warn("i am deprecated", DeprecationWarning) + ... testFooBar.suppress = [util.suppress(message='i am deprecated')] + ... + >>> + + Note that as with the todo and timeout attributes: the module level + attribute acts as a default for the class attribute which acts as a default + for the method attribute. The suppress attribute can be overridden at any + level by specifying C{.suppress = []} + """ + return ((action,), kwarg) + + + +# This should be deleted, and replaced with twisted.application's code; see +# #6016: +def profiled(f, outputFile): + def _(*args, **kwargs): + import profile + prof = profile.Profile() + try: + result = prof.runcall(f, *args, **kwargs) + prof.dump_stats(outputFile) + except SystemExit: + pass + prof.print_stats() + return result + return _ + + + +@defer.inlineCallbacks +def _runSequentially(callables, stopOnFirstError=False): + """ + Run the given callables one after the other. If a callable returns a + Deferred, wait until it has finished before running the next callable. + + @param callables: An iterable of callables that take no parameters. + + @param stopOnFirstError: If True, then stop running callables as soon as + one raises an exception or fires an errback. False by default. + + @return: A L{Deferred} that fires a list of C{(flag, value)} tuples. Each + tuple will be either C{(SUCCESS, )} or C{(FAILURE, + )}. + """ + results = [] + for f in callables: + d = defer.maybeDeferred(f) + try: + thing = yield d + results.append((defer.SUCCESS, thing)) + except Exception: + results.append((defer.FAILURE, Failure())) + if stopOnFirstError: + break + defer.returnValue(results) + + + +class _NoTrialMarker(Exception): + """ + No trial marker file could be found. + + Raised when trial attempts to remove a trial temporary working directory + that does not contain a marker file. + """ + + + +def _removeSafely(path): + """ + Safely remove a path, recursively. + + If C{path} does not contain a node named C{_trial_marker}, a + L{_NoTrialMarker} exception is raised and the path is not removed. + """ + if not path.child(b'_trial_marker').exists(): + raise _NoTrialMarker( + '%r is not a trial temporary path, refusing to remove it' + % (path,)) + try: + path.remove() + except OSError as e: + print ("could not remove %r, caught OSError [Errno %s]: %s" + % (path, e.errno, e.strerror)) + try: + newPath = FilePath(b'_trial_temp_old' + + str(randrange(10000000)).encode("utf-8")) + path.moveTo(newPath) + except OSError as e: + print ("could not rename path, caught OSError [Errno %s]: %s" + % (e.errno,e.strerror)) + raise + + + +class _WorkingDirectoryBusy(Exception): + """ + A working directory was specified to the runner, but another test run is + currently using that directory. + """ + + + +def _unusedTestDirectory(base): + """ + Find an unused directory named similarly to C{base}. + + Once a directory is found, it will be locked and a marker dropped into it + to identify it as a trial temporary directory. + + @param base: A template path for the discovery process. If this path + exactly cannot be used, a path which varies only in a suffix of the + basename will be used instead. + @type base: L{FilePath} + + @return: A two-tuple. The first element is a L{FilePath} representing the + directory which was found and created. The second element is a locked + L{FilesystemLock}. Another + call to C{_unusedTestDirectory} will not be able to reused the + same name until the lock is released, either explicitly or by this + process exiting. + """ + counter = 0 + while True: + if counter: + testdir = base.sibling('%s-%d' % (base.basename(), counter)) + else: + testdir = base + + testDirLock = FilesystemLock(testdir.path + '.lock') + if testDirLock.lock(): + # It is not in use + if testdir.exists(): + # It exists though - delete it + _removeSafely(testdir) + + # Create it anew and mark it as ours so the next _removeSafely on + # it succeeds. + testdir.makedirs() + testdir.child(b'_trial_marker').setContent(b'') + return testdir, testDirLock + else: + # It is in use + if base.basename() == '_trial_temp': + counter += 1 + else: + raise _WorkingDirectoryBusy() + + + +def _listToPhrase(things, finalDelimiter, delimiter=', '): + """ + Produce a string containing each thing in C{things}, + separated by a C{delimiter}, with the last couple being separated + by C{finalDelimiter} + + @param things: The elements of the resulting phrase + @type things: L{list} or L{tuple} + + @param finalDelimiter: What to put between the last two things + (typically 'and' or 'or') + @type finalDelimiter: L{str} + + @param delimiter: The separator to use between each thing, + not including the last two. Should typically include a trailing space. + @type delimiter: L{str} + + @return: The resulting phrase + @rtype: L{str} + """ + if not isinstance(things, (list, tuple)): + raise TypeError("Things must be a list or a tuple") + if not things: + return '' + if len(things) == 1: + return str(things[0]) + if len(things) == 2: + return "%s %s %s" % (str(things[0]), finalDelimiter, str(things[1])) + else: + strThings = [] + for thing in things: + strThings.append(str(thing)) + return "%s%s%s %s" % (delimiter.join(strThings[:-1]), + delimiter, finalDelimiter, strThings[-1]) diff --git a/contrib/python/Twisted/py2/twisted/web/__init__.py b/contrib/python/Twisted/py2/twisted/web/__init__.py new file mode 100644 index 00000000000..806dc4a2a42 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/__init__.py @@ -0,0 +1,12 @@ +# -*- test-case-name: twisted.web.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Web: HTTP clients and servers, plus tools for implementing them. + +Contains a L{web server} (including an +L{HTTP implementation}, a +L{resource model}), and +a L{web client}. +""" diff --git a/contrib/python/Twisted/py2/twisted/web/_auth/__init__.py b/contrib/python/Twisted/py2/twisted/web/_auth/__init__.py new file mode 100644 index 00000000000..6a588700916 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_auth/__init__.py @@ -0,0 +1,7 @@ +# -*- test-case-name: twisted.web.test.test_httpauth -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HTTP header-based authentication migrated from web2 +""" diff --git a/contrib/python/Twisted/py2/twisted/web/_auth/basic.py b/contrib/python/Twisted/py2/twisted/web/_auth/basic.py new file mode 100644 index 00000000000..d539457190c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_auth/basic.py @@ -0,0 +1,61 @@ +# -*- test-case-name: twisted.web.test.test_httpauth -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HTTP BASIC authentication. + +@see: U{http://tools.ietf.org/html/rfc1945} +@see: U{http://tools.ietf.org/html/rfc2616} +@see: U{http://tools.ietf.org/html/rfc2617} +""" + +from __future__ import division, absolute_import + +import binascii + +from zope.interface import implementer + +from twisted.cred import credentials, error +from twisted.web.iweb import ICredentialFactory + + +@implementer(ICredentialFactory) +class BasicCredentialFactory(object): + """ + Credential Factory for HTTP Basic Authentication + + @type authenticationRealm: L{bytes} + @ivar authenticationRealm: The HTTP authentication realm which will be issued in + challenges. + """ + + scheme = b'basic' + + def __init__(self, authenticationRealm): + self.authenticationRealm = authenticationRealm + + + def getChallenge(self, request): + """ + Return a challenge including the HTTP authentication realm with which + this factory was created. + """ + return {'realm': self.authenticationRealm} + + + def decode(self, response, request): + """ + Parse the base64-encoded, colon-separated username and password into a + L{credentials.UsernamePassword} instance. + """ + try: + creds = binascii.a2b_base64(response + b'===') + except binascii.Error: + raise error.LoginFailed('Invalid credentials') + + creds = creds.split(b':', 1) + if len(creds) == 2: + return credentials.UsernamePassword(*creds) + else: + raise error.LoginFailed('Invalid credentials') diff --git a/contrib/python/Twisted/py2/twisted/web/_auth/digest.py b/contrib/python/Twisted/py2/twisted/web/_auth/digest.py new file mode 100644 index 00000000000..5346801f6b2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_auth/digest.py @@ -0,0 +1,56 @@ +# -*- test-case-name: twisted.web.test.test_httpauth -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementation of RFC2617: HTTP Digest Authentication + +@see: U{http://www.faqs.org/rfcs/rfc2617.html} +""" + +from __future__ import division, absolute_import + +from zope.interface import implementer +from twisted.cred import credentials +from twisted.web.iweb import ICredentialFactory + +@implementer(ICredentialFactory) +class DigestCredentialFactory(object): + """ + Wrapper for L{digest.DigestCredentialFactory} that implements the + L{ICredentialFactory} interface. + """ + + scheme = b'digest' + + def __init__(self, algorithm, authenticationRealm): + """ + Create the digest credential factory that this object wraps. + """ + self.digest = credentials.DigestCredentialFactory(algorithm, + authenticationRealm) + + + def getChallenge(self, request): + """ + Generate the challenge for use in the WWW-Authenticate header + + @param request: The L{IRequest} to with access was denied and for the + response to which this challenge is being generated. + + @return: The L{dict} that can be used to generate a WWW-Authenticate + header. + """ + return self.digest.getChallenge(request.getClientAddress().host) + + + def decode(self, response, request): + """ + Create a L{twisted.cred.credentials.DigestedCredentials} object + from the given response and request. + + @see: L{ICredentialFactory.decode} + """ + return self.digest.decode(response, + request.method, + request.getClientAddress().host) diff --git a/contrib/python/Twisted/py2/twisted/web/_auth/wrapper.py b/contrib/python/Twisted/py2/twisted/web/_auth/wrapper.py new file mode 100644 index 00000000000..1804b7a4166 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_auth/wrapper.py @@ -0,0 +1,236 @@ +# -*- test-case-name: twisted.web.test.test_httpauth -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A guard implementation which supports HTTP header-based authentication +schemes. + +If no I{Authorization} header is supplied, an anonymous login will be +attempted by using a L{Anonymous} credentials object. If such a header is +supplied and does not contain allowed credentials, or if anonymous login is +denied, a 401 will be sent in the response along with I{WWW-Authenticate} +headers for each of the allowed authentication schemes. +""" + +from __future__ import absolute_import, division + +from twisted.cred import error +from twisted.cred.credentials import Anonymous +from twisted.python.compat import unicode +from twisted.python.components import proxyForInterface +from twisted.web import util +from twisted.web.resource import ErrorPage, IResource +from twisted.logger import Logger + +from zope.interface import implementer + + +@implementer(IResource) +class UnauthorizedResource(object): + """ + Simple IResource to escape Resource dispatch + """ + isLeaf = True + + + def __init__(self, factories): + self._credentialFactories = factories + + + def render(self, request): + """ + Send www-authenticate headers to the client + """ + def ensureBytes(s): + return s.encode('ascii') if isinstance(s, unicode) else s + + def generateWWWAuthenticate(scheme, challenge): + l = [] + for k, v in challenge.items(): + k = ensureBytes(k) + v = ensureBytes(v) + l.append(k + b"=" + quoteString(v)) + return b" ".join([scheme, b", ".join(l)]) + + def quoteString(s): + return b'"' + s.replace(b'\\', b'\\\\').replace(b'"', b'\\"') + b'"' + + request.setResponseCode(401) + for fact in self._credentialFactories: + challenge = fact.getChallenge(request) + request.responseHeaders.addRawHeader( + b'www-authenticate', + generateWWWAuthenticate(fact.scheme, challenge)) + if request.method == b'HEAD': + return b'' + return b'Unauthorized' + + + def getChildWithDefault(self, path, request): + """ + Disable resource dispatch + """ + return self + + + +@implementer(IResource) +class HTTPAuthSessionWrapper(object): + """ + Wrap a portal, enforcing supported header-based authentication schemes. + + @ivar _portal: The L{Portal} which will be used to retrieve L{IResource} + avatars. + + @ivar _credentialFactories: A list of L{ICredentialFactory} providers which + will be used to decode I{Authorization} headers into L{ICredentials} + providers. + """ + isLeaf = False + _log = Logger() + + def __init__(self, portal, credentialFactories): + """ + Initialize a session wrapper + + @type portal: C{Portal} + @param portal: The portal that will authenticate the remote client + + @type credentialFactories: C{Iterable} + @param credentialFactories: The portal that will authenticate the + remote client based on one submitted C{ICredentialFactory} + """ + self._portal = portal + self._credentialFactories = credentialFactories + + + def _authorizedResource(self, request): + """ + Get the L{IResource} which the given request is authorized to receive. + If the proper authorization headers are present, the resource will be + requested from the portal. If not, an anonymous login attempt will be + made. + """ + authheader = request.getHeader(b'authorization') + if not authheader: + return util.DeferredResource(self._login(Anonymous())) + + factory, respString = self._selectParseHeader(authheader) + if factory is None: + return UnauthorizedResource(self._credentialFactories) + try: + credentials = factory.decode(respString, request) + except error.LoginFailed: + return UnauthorizedResource(self._credentialFactories) + except: + self._log.failure("Unexpected failure from credentials factory") + return ErrorPage(500, None, None) + else: + return util.DeferredResource(self._login(credentials)) + + + def render(self, request): + """ + Find the L{IResource} avatar suitable for the given request, if + possible, and render it. Otherwise, perhaps render an error page + requiring authorization or describing an internal server failure. + """ + return self._authorizedResource(request).render(request) + + + def getChildWithDefault(self, path, request): + """ + Inspect the Authorization HTTP header, and return a deferred which, + when fired after successful authentication, will return an authorized + C{Avatar}. On authentication failure, an C{UnauthorizedResource} will + be returned, essentially halting further dispatch on the wrapped + resource and all children + """ + # Don't consume any segments of the request - this class should be + # transparent! + request.postpath.insert(0, request.prepath.pop()) + return self._authorizedResource(request) + + + def _login(self, credentials): + """ + Get the L{IResource} avatar for the given credentials. + + @return: A L{Deferred} which will be called back with an L{IResource} + avatar or which will errback if authentication fails. + """ + d = self._portal.login(credentials, None, IResource) + d.addCallbacks(self._loginSucceeded, self._loginFailed) + return d + + + def _loginSucceeded(self, args): + """ + Handle login success by wrapping the resulting L{IResource} avatar + so that the C{logout} callback will be invoked when rendering is + complete. + """ + interface, avatar, logout = args + class ResourceWrapper(proxyForInterface(IResource, 'resource')): + """ + Wrap an L{IResource} so that whenever it or a child of it + completes rendering, the cred logout hook will be invoked. + + An assumption is made here that exactly one L{IResource} from + among C{avatar} and all of its children will be rendered. If + more than one is rendered, C{logout} will be invoked multiple + times and probably earlier than desired. + """ + def getChildWithDefault(self, name, request): + """ + Pass through the lookup to the wrapped resource, wrapping + the result in L{ResourceWrapper} to ensure C{logout} is + called when rendering of the child is complete. + """ + return ResourceWrapper(self.resource.getChildWithDefault(name, request)) + + def render(self, request): + """ + Hook into response generation so that when rendering has + finished completely (with or without error), C{logout} is + called. + """ + request.notifyFinish().addBoth(lambda ign: logout()) + return super(ResourceWrapper, self).render(request) + + return ResourceWrapper(avatar) + + + def _loginFailed(self, result): + """ + Handle login failure by presenting either another challenge (for + expected authentication/authorization-related failures) or a server + error page (for anything else). + """ + if result.check(error.Unauthorized, error.LoginFailed): + return UnauthorizedResource(self._credentialFactories) + else: + self._log.failure( + "HTTPAuthSessionWrapper.getChildWithDefault encountered " + "unexpected error", + failure=result, + ) + return ErrorPage(500, None, None) + + + def _selectParseHeader(self, header): + """ + Choose an C{ICredentialFactory} from C{_credentialFactories} + suitable to use to decode the given I{Authenticate} header. + + @return: A two-tuple of a factory and the remaining portion of the + header value to be decoded or a two-tuple of L{None} if no + factory can decode the header value. + """ + elements = header.split(b' ') + scheme = elements[0].lower() + for fact in self._credentialFactories: + if fact.scheme == scheme: + return (fact, b' '.join(elements[1:])) + return (None, None) diff --git a/contrib/python/Twisted/py2/twisted/web/_element.py b/contrib/python/Twisted/py2/twisted/web/_element.py new file mode 100644 index 00000000000..5c4b7e99cf2 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_element.py @@ -0,0 +1,185 @@ +# -*- test-case-name: twisted.web.test.test_template -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import division, absolute_import + +from zope.interface import implementer + +from twisted.web.iweb import IRenderable +from twisted.web.error import MissingRenderMethod, UnexposedMethodError +from twisted.web.error import MissingTemplateLoader + + +class Expose(object): + """ + Helper for exposing methods for various uses using a simple decorator-style + callable. + + Instances of this class can be called with one or more functions as + positional arguments. The names of these functions will be added to a list + on the class object of which they are methods. + + @ivar attributeName: The attribute with which exposed methods will be + tracked. + """ + def __init__(self, doc=None): + self.doc = doc + + + def __call__(self, *funcObjs): + """ + Add one or more functions to the set of exposed functions. + + This is a way to declare something about a class definition, similar to + L{zope.interface.declarations.implementer}. Use it like this:: + + magic = Expose('perform extra magic') + class Foo(Bar): + def twiddle(self, x, y): + ... + def frob(self, a, b): + ... + magic(twiddle, frob) + + Later you can query the object:: + + aFoo = Foo() + magic.get(aFoo, 'twiddle')(x=1, y=2) + + The call to C{get} will fail if the name it is given has not been + exposed using C{magic}. + + @param funcObjs: One or more function objects which will be exposed to + the client. + + @return: The first of C{funcObjs}. + """ + if not funcObjs: + raise TypeError("expose() takes at least 1 argument (0 given)") + for fObj in funcObjs: + fObj.exposedThrough = getattr(fObj, 'exposedThrough', []) + fObj.exposedThrough.append(self) + return funcObjs[0] + + + _nodefault = object() + def get(self, instance, methodName, default=_nodefault): + """ + Retrieve an exposed method with the given name from the given instance. + + @raise UnexposedMethodError: Raised if C{default} is not specified and + there is no exposed method with the given name. + + @return: A callable object for the named method assigned to the given + instance. + """ + method = getattr(instance, methodName, None) + exposedThrough = getattr(method, 'exposedThrough', []) + if self not in exposedThrough: + if default is self._nodefault: + raise UnexposedMethodError(self, methodName) + return default + return method + + + @classmethod + def _withDocumentation(cls, thunk): + """ + Slight hack to make users of this class appear to have a docstring to + documentation generators, by defining them with a decorator. (This hack + should be removed when epydoc can be convinced to use some other method + for documenting.) + """ + return cls(thunk.__doc__) + + +# Avoid exposing the ugly, private classmethod name in the docs. Luckily this +# namespace is private already so this doesn't leak further. +exposer = Expose._withDocumentation + +@exposer +def renderer(): + """ + Decorate with L{renderer} to use methods as template render directives. + + For example:: + + class Foo(Element): + @renderer + def twiddle(self, request, tag): + return tag('Hello, world.') + +
+ +
+ + Will result in this final output:: + +
+ Hello, world. +
+ """ + + + +@implementer(IRenderable) +class Element(object): + """ + Base for classes which can render part of a page. + + An Element is a renderer that can be embedded in a stan document and can + hook its template (from the loader) up to render methods. + + An Element might be used to encapsulate the rendering of a complex piece of + data which is to be displayed in multiple different contexts. The Element + allows the rendering logic to be easily re-used in different ways. + + Element returns render methods which are registered using + L{twisted.web._element.renderer}. For example:: + + class Menu(Element): + @renderer + def items(self, request, tag): + .... + + Render methods are invoked with two arguments: first, the + L{twisted.web.http.Request} being served and second, the tag object which + "invoked" the render method. + + @type loader: L{ITemplateLoader} provider + @ivar loader: The factory which will be used to load documents to + return from C{render}. + """ + loader = None + + def __init__(self, loader=None): + if loader is not None: + self.loader = loader + + + def lookupRenderMethod(self, name): + """ + Look up and return the named render method. + """ + method = renderer.get(self, name, None) + if method is None: + raise MissingRenderMethod(self, name) + return method + + + def render(self, request): + """ + Implement L{IRenderable} to allow one L{Element} to be embedded in + another's template or rendering output. + + (This will simply load the template from the C{loader}; when used in a + template, the flattening engine will keep track of this object + separately as the object to lookup renderers on and call + L{Element.renderer} to look them up. The resulting object from this + method is not directly associated with this L{Element}.) + """ + loader = self.loader + if loader is None: + raise MissingTemplateLoader(self) + return loader.load() diff --git a/contrib/python/Twisted/py2/twisted/web/_flatten.py b/contrib/python/Twisted/py2/twisted/web/_flatten.py new file mode 100644 index 00000000000..89a657dbebe --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_flatten.py @@ -0,0 +1,421 @@ +# -*- test-case-name: twisted.web.test.test_flatten -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Context-free flattener/serializer for rendering Python objects, possibly +complex or arbitrarily nested, as strings. +""" + +from __future__ import division, absolute_import + +from io import BytesIO + +from sys import exc_info +from types import GeneratorType +from traceback import extract_tb + +try: + from inspect import iscoroutine +except ImportError: + def iscoroutine(*args, **kwargs): + return False + +from twisted.python.compat import unicode, nativeString, iteritems +from twisted.internet.defer import Deferred, ensureDeferred +from twisted.web._stan import Tag, slot, voidElements, Comment, CDATA, CharRef +from twisted.web.error import UnfilledSlot, UnsupportedType, FlattenerError +from twisted.web.iweb import IRenderable + + + +def escapeForContent(data): + """ + Escape some character or UTF-8 byte data for inclusion in an HTML or XML + document, by replacing metacharacters (C{&<>}) with their entity + equivalents (C{&<>}). + + This is used as an input to L{_flattenElement}'s C{dataEscaper} parameter. + + @type data: C{bytes} or C{unicode} + @param data: The string to escape. + + @rtype: C{bytes} + @return: The quoted form of C{data}. If C{data} is unicode, return a utf-8 + encoded string. + """ + if isinstance(data, unicode): + data = data.encode('utf-8') + data = data.replace(b'&', b'&' + ).replace(b'<', b'<' + ).replace(b'>', b'>') + return data + + + +def attributeEscapingDoneOutside(data): + """ + Escape some character or UTF-8 byte data for inclusion in the top level of + an attribute. L{attributeEscapingDoneOutside} actually passes the data + through unchanged, because L{writeWithAttributeEscaping} handles the + quoting of the text within attributes outside the generator returned by + L{_flattenElement}; this is used as the C{dataEscaper} argument to that + L{_flattenElement} call so that that generator does not redundantly escape + its text output. + + @type data: C{bytes} or C{unicode} + @param data: The string to escape. + + @return: The string, unchanged, except for encoding. + @rtype: C{bytes} + """ + if isinstance(data, unicode): + return data.encode("utf-8") + return data + + + +def writeWithAttributeEscaping(write): + """ + Decorate a C{write} callable so that all output written is properly quoted + for inclusion within an XML attribute value. + + If a L{Tag } C{x} is flattened within the context + of the contents of another L{Tag } C{y}, the + metacharacters (C{<>&"}) delimiting C{x} should be passed through + unchanged, but the textual content of C{x} should still be quoted, as + usual. For example: C{&}. That is the default behavior + of L{_flattenElement} when L{escapeForContent} is passed as the + C{dataEscaper}. + + However, when a L{Tag } C{x} is flattened within + the context of an I{attribute} of another L{Tag } + C{y}, then the metacharacters delimiting C{x} should be quoted so that it + can be parsed from the attribute's value. In the DOM itself, this is not a + valid thing to do, but given that renderers and slots may be freely moved + around in a L{twisted.web.template} template, it is a condition which may + arise in a document and must be handled in a way which produces valid + output. So, for example, you should be able to get C{}. This should also be true for other XML/HTML meta-constructs such as + comments and CDATA, so if you were to serialize a L{comment + } in an attribute you should get C{}. Therefore in order to capture these + meta-characters, flattening is done with C{write} callable that is wrapped + with L{writeWithAttributeEscaping}. + + The final case, and hopefully the much more common one as compared to + serializing L{Tag } and arbitrary L{IRenderable} + objects within an attribute, is to serialize a simple string, and those + should be passed through for L{writeWithAttributeEscaping} to quote + without applying a second, redundant level of quoting. + + @param write: A callable which will be invoked with the escaped L{bytes}. + + @return: A callable that writes data with escaping. + """ + def _write(data): + write(escapeForContent(data).replace(b'"', b'"')) + return _write + + + +def escapedCDATA(data): + """ + Escape CDATA for inclusion in a document. + + @type data: L{str} or L{unicode} + @param data: The string to escape. + + @rtype: L{str} + @return: The quoted form of C{data}. If C{data} is unicode, return a utf-8 + encoded string. + """ + if isinstance(data, unicode): + data = data.encode('utf-8') + return data.replace(b']]>', b']]]]>') + + + +def escapedComment(data): + """ + Escape a comment for inclusion in a document. + + @type data: L{str} or L{unicode} + @param data: The string to escape. + + @rtype: C{str} + @return: The quoted form of C{data}. If C{data} is unicode, return a utf-8 + encoded string. + """ + if isinstance(data, unicode): + data = data.encode('utf-8') + data = data.replace(b'--', b'- - ').replace(b'>', b'>') + if data and data[-1:] == b'-': + data += b' ' + return data + + + +def _getSlotValue(name, slotData, default=None): + """ + Find the value of the named slot in the given stack of slot data. + """ + for slotFrame in slotData[::-1]: + if slotFrame is not None and name in slotFrame: + return slotFrame[name] + else: + if default is not None: + return default + raise UnfilledSlot(name) + + + +def _flattenElement(request, root, write, slotData, renderFactory, + dataEscaper): + """ + Make C{root} slightly more flat by yielding all its immediate contents as + strings, deferreds or generators that are recursive calls to itself. + + @param request: A request object which will be passed to + L{IRenderable.render}. + + @param root: An object to be made flatter. This may be of type C{unicode}, + L{str}, L{slot}, L{Tag }, L{tuple}, L{list}, + L{types.GeneratorType}, L{Deferred}, or an object that implements + L{IRenderable}. + + @param write: A callable which will be invoked with each L{bytes} produced + by flattening C{root}. + + @param slotData: A L{list} of L{dict} mapping L{str} slot names to data + with which those slots will be replaced. + + @param renderFactory: If not L{None}, an object that provides + L{IRenderable}. + + @param dataEscaper: A 1-argument callable which takes L{bytes} or + L{unicode} and returns L{bytes}, quoted as appropriate for the + rendering context. This is really only one of two values: + L{attributeEscapingDoneOutside} or L{escapeForContent}, depending on + whether the rendering context is within an attribute or not. See the + explanation in L{writeWithAttributeEscaping}. + + @return: An iterator that eventually yields L{bytes} that should be written + to the output. However it may also yield other iterators or + L{Deferred}s; if it yields another iterator, the caller will iterate + it; if it yields a L{Deferred}, the result of that L{Deferred} will + either be L{bytes}, in which case it's written, or another generator, + in which case it is iterated. See L{_flattenTree} for the trampoline + that consumes said values. + @rtype: An iterator which yields L{bytes}, L{Deferred}, and more iterators + of the same type. + """ + def keepGoing(newRoot, dataEscaper=dataEscaper, + renderFactory=renderFactory, write=write): + return _flattenElement(request, newRoot, write, slotData, + renderFactory, dataEscaper) + if isinstance(root, (bytes, unicode)): + write(dataEscaper(root)) + elif isinstance(root, slot): + slotValue = _getSlotValue(root.name, slotData, root.default) + yield keepGoing(slotValue) + elif isinstance(root, CDATA): + write(b'') + elif isinstance(root, Comment): + write(b'') + elif isinstance(root, Tag): + slotData.append(root.slotData) + if root.render is not None: + rendererName = root.render + rootClone = root.clone(False) + rootClone.render = None + renderMethod = renderFactory.lookupRenderMethod(rendererName) + result = renderMethod(request, rootClone) + yield keepGoing(result) + slotData.pop() + return + + if not root.tagName: + yield keepGoing(root.children) + return + + write(b'<') + if isinstance(root.tagName, unicode): + tagName = root.tagName.encode('ascii') + else: + tagName = root.tagName + write(tagName) + for k, v in iteritems(root.attributes): + if isinstance(k, unicode): + k = k.encode('ascii') + write(b' ' + k + b'="') + # Serialize the contents of the attribute, wrapping the results of + # that serialization so that _everything_ is quoted. + yield keepGoing( + v, + attributeEscapingDoneOutside, + write=writeWithAttributeEscaping(write)) + write(b'"') + if root.children or nativeString(tagName) not in voidElements: + write(b'>') + # Regardless of whether we're in an attribute or not, switch back + # to the escapeForContent dataEscaper. The contents of a tag must + # be quoted no matter what; in the top-level document, just so + # they're valid, and if they're within an attribute, they have to + # be quoted so that after applying the *un*-quoting required to re- + # parse the tag within the attribute, all the quoting is still + # correct. + yield keepGoing(root.children, escapeForContent) + write(b'') + else: + write(b' />') + + elif isinstance(root, (tuple, list, GeneratorType)): + for element in root: + yield keepGoing(element) + elif isinstance(root, CharRef): + escaped = '&#%d;' % (root.ordinal,) + write(escaped.encode('ascii')) + elif isinstance(root, Deferred): + yield root.addCallback(lambda result: (result, keepGoing(result))) + elif iscoroutine(root): + d = ensureDeferred(root) + yield d.addCallback(lambda result: (result, keepGoing(result))) + elif IRenderable.providedBy(root): + result = root.render(request) + yield keepGoing(result, renderFactory=root) + else: + raise UnsupportedType(root) + + + +def _flattenTree(request, root, write): + """ + Make C{root} into an iterable of L{bytes} and L{Deferred} by doing a depth + first traversal of the tree. + + @param request: A request object which will be passed to + L{IRenderable.render}. + + @param root: An object to be made flatter. This may be of type C{unicode}, + L{bytes}, L{slot}, L{Tag }, L{tuple}, + L{list}, L{types.GeneratorType}, L{Deferred}, or something providing + L{IRenderable}. + + @param write: A callable which will be invoked with each L{bytes} produced + by flattening C{root}. + + @return: An iterator which yields objects of type L{bytes} and L{Deferred}. + A L{Deferred} is only yielded when one is encountered in the process of + flattening C{root}. The returned iterator must not be iterated again + until the L{Deferred} is called back. + """ + stack = [_flattenElement(request, root, write, [], None, escapeForContent)] + while stack: + try: + frame = stack[-1].gi_frame + element = next(stack[-1]) + except StopIteration: + stack.pop() + except Exception as e: + stack.pop() + roots = [] + for generator in stack: + roots.append(generator.gi_frame.f_locals['root']) + roots.append(frame.f_locals['root']) + raise FlattenerError(e, roots, extract_tb(exc_info()[2])) + else: + if isinstance(element, Deferred): + def cbx(originalAndToFlatten): + original, toFlatten = originalAndToFlatten + stack.append(toFlatten) + return original + yield element.addCallback(cbx) + else: + stack.append(element) + + +def _writeFlattenedData(state, write, result): + """ + Take strings from an iterator and pass them to a writer function. + + @param state: An iterator of L{str} and L{Deferred}. L{str} instances will + be passed to C{write}. L{Deferred} instances will be waited on before + resuming iteration of C{state}. + + @param write: A callable which will be invoked with each L{str} + produced by iterating C{state}. + + @param result: A L{Deferred} which will be called back when C{state} has + been completely flattened into C{write} or which will be errbacked if + an exception in a generator passed to C{state} or an errback from a + L{Deferred} from state occurs. + + @return: L{None} + """ + while True: + try: + element = next(state) + except StopIteration: + result.callback(None) + except: + result.errback() + else: + def cby(original): + _writeFlattenedData(state, write, result) + return original + element.addCallbacks(cby, result.errback) + break + + + +def flatten(request, root, write): + """ + Incrementally write out a string representation of C{root} using C{write}. + + In order to create a string representation, C{root} will be decomposed into + simpler objects which will themselves be decomposed and so on until strings + or objects which can easily be converted to strings are encountered. + + @param request: A request object which will be passed to the C{render} + method of any L{IRenderable} provider which is encountered. + + @param root: An object to be made flatter. This may be of type L{unicode}, + L{bytes}, L{slot}, L{Tag }, L{tuple}, + L{list}, L{types.GeneratorType}, L{Deferred}, or something that provides + L{IRenderable}. + + @param write: A callable which will be invoked with each L{bytes} produced + by flattening C{root}. + + @return: A L{Deferred} which will be called back when C{root} has been + completely flattened into C{write} or which will be errbacked if an + unexpected exception occurs. + """ + result = Deferred() + state = _flattenTree(request, root, write) + _writeFlattenedData(state, write, result) + return result + + + +def flattenString(request, root): + """ + Collate a string representation of C{root} into a single string. + + This is basically gluing L{flatten} to an L{io.BytesIO} and returning + the results. See L{flatten} for the exact meanings of C{request} and + C{root}. + + @return: A L{Deferred} which will be called back with a single string as + its result when C{root} has been completely flattened into C{write} or + which will be errbacked if an unexpected exception occurs. + """ + io = BytesIO() + d = flatten(request, root, io.write) + d.addCallback(lambda _: io.getvalue()) + return d diff --git a/contrib/python/Twisted/py2/twisted/web/_http2.py b/contrib/python/Twisted/py2/twisted/web/_http2.py new file mode 100644 index 00000000000..fdaef007820 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_http2.py @@ -0,0 +1,1356 @@ +# -*- test-case-name: twisted.web.test.test_http2 -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HTTP2 Implementation + +This is the basic server-side protocol implementation used by the Twisted +Web server for HTTP2. This functionality is intended to be combined with the +HTTP/1.1 and HTTP/1.0 functionality in twisted.web.http to provide complete +protocol support for HTTP-type protocols. + +This API is currently considered private because it's in early draft form. When +it has stabilised, it'll be made public. +""" + +from __future__ import absolute_import, division + +import io +import warnings +import sys + +from collections import deque + +from zope.interface import implementer + +import priority +import h2.config +import h2.connection +import h2.errors +import h2.events +import h2.exceptions + +from twisted.internet.defer import Deferred +from twisted.internet.error import ConnectionLost +from twisted.internet.interfaces import ( + IProtocol, ITransport, IConsumer, IPushProducer, ISSLTransport +) +from twisted.internet._producer_helpers import _PullToPush +from twisted.internet.protocol import Protocol +from twisted.logger import Logger +from twisted.protocols.policies import TimeoutMixin +from twisted.python.failure import Failure +from twisted.web.error import ExcessiveBufferingError + + +# This API is currently considered private. +__all__ = [] + + +_END_STREAM_SENTINEL = object() + + +# Python versions 2.7.3 and older don't have a memoryview object that plays +# well with the struct module, which h2 needs. On those versions, just refuse +# to import. +if sys.version_info < (2, 7, 4): + warnings.warn( + "HTTP/2 cannot be enabled because this version of Python is too " + "old, and does not fully support memoryview objects.", + UserWarning, + stacklevel=2, + ) + raise ImportError("HTTP/2 not supported on this Python version.") + + + +@implementer(IProtocol, IPushProducer) +class H2Connection(Protocol, TimeoutMixin): + """ + A class representing a single HTTP/2 connection. + + This implementation of L{IProtocol} works hand in hand with L{H2Stream}. + This is because we have the requirement to register multiple producers for + a single HTTP/2 connection, one for each stream. The standard Twisted + interfaces don't really allow for this, so instead there's a custom + interface between the two objects that allows them to work hand-in-hand here. + + @ivar conn: The HTTP/2 connection state machine. + @type conn: L{h2.connection.H2Connection} + + @ivar streams: A mapping of stream IDs to L{H2Stream} objects, used to call + specific methods on streams when events occur. + @type streams: L{dict}, mapping L{int} stream IDs to L{H2Stream} objects. + + @ivar priority: A HTTP/2 priority tree used to ensure that responses are + prioritised appropriately. + @type priority: L{priority.PriorityTree} + + @ivar _consumerBlocked: A flag tracking whether or not the L{IConsumer} + that is consuming this data has asked us to stop producing. + @type _consumerBlocked: L{bool} + + @ivar _sendingDeferred: A L{Deferred} used to restart the data-sending loop + when more response data has been produced. Will not be present if there + is outstanding data still to send. + @type _consumerBlocked: A L{twisted.internet.defer.Deferred}, or L{None} + + @ivar _outboundStreamQueues: A map of stream IDs to queues, used to store + data blocks that are yet to be sent on the connection. These are used + both to handle producers that do not respect L{IConsumer} but also to + allow priority to multiplex data appropriately. + @type _outboundStreamQueues: A L{dict} mapping L{int} stream IDs to + L{collections.deque} queues, which contain either L{bytes} objects or + C{_END_STREAM_SENTINEL}. + + @ivar _sender: A handle to the data-sending loop, allowing it to be + terminated if needed. + @type _sender: L{twisted.internet.task.LoopingCall} + + @ivar abortTimeout: The number of seconds to wait after we attempt to shut + the transport down cleanly to give up and forcibly terminate it. This + is only used when we time a connection out, to prevent errors causing + the FD to get leaked. If this is L{None}, we will wait forever. + @type abortTimeout: L{int} + + @ivar _abortingCall: The L{twisted.internet.base.DelayedCall} that will be + used to forcibly close the transport if it doesn't close cleanly. + @type _abortingCall: L{twisted.internet.base.DelayedCall} + """ + factory = None + site = None + abortTimeout = 15 + + _log = Logger() + _abortingCall = None + + def __init__(self, reactor=None): + config = h2.config.H2Configuration( + client_side=False, header_encoding=None + ) + self.conn = h2.connection.H2Connection(config=config) + self.streams = {} + + self.priority = priority.PriorityTree() + self._consumerBlocked = None + self._sendingDeferred = None + self._outboundStreamQueues = {} + self._streamCleanupCallbacks = {} + self._stillProducing = True + + # Limit the number of buffered control frame (e.g. PING and + # SETTINGS) bytes. + self._maxBufferedControlFrameBytes = 1024 * 17 + self._bufferedControlFrames = deque() + self._bufferedControlFrameBytes = 0 + + if reactor is None: + from twisted.internet import reactor + self._reactor = reactor + + # Start the data sending function. + self._reactor.callLater(0, self._sendPrioritisedData) + + + # Implementation of IProtocol + def connectionMade(self): + """ + Called by the reactor when a connection is received. May also be called + by the L{twisted.web.http._GenericHTTPChannelProtocol} during upgrade + to HTTP/2. + """ + self.setTimeout(self.timeOut) + self.conn.initiate_connection() + self.transport.write(self.conn.data_to_send()) + + + def dataReceived(self, data): + """ + Called whenever a chunk of data is received from the transport. + + @param data: The data received from the transport. + @type data: L{bytes} + """ + try: + events = self.conn.receive_data(data) + except h2.exceptions.ProtocolError: + stillActive = self._tryToWriteControlData() + if stillActive: + self.transport.loseConnection() + self.connectionLost(Failure(), _cancelTimeouts=False) + return + + # Only reset the timeout if we've received an actual H2 + # protocol message + self.resetTimeout() + + for event in events: + if isinstance(event, h2.events.RequestReceived): + self._requestReceived(event) + elif isinstance(event, h2.events.DataReceived): + self._requestDataReceived(event) + elif isinstance(event, h2.events.StreamEnded): + self._requestEnded(event) + elif isinstance(event, h2.events.StreamReset): + self._requestAborted(event) + elif isinstance(event, h2.events.WindowUpdated): + self._handleWindowUpdate(event) + elif isinstance(event, h2.events.PriorityUpdated): + self._handlePriorityUpdate(event) + elif isinstance(event, h2.events.ConnectionTerminated): + self.transport.loseConnection() + self.connectionLost( + ConnectionLost("Remote peer sent GOAWAY"), + _cancelTimeouts=False, + ) + + self._tryToWriteControlData() + + + def timeoutConnection(self): + """ + Called when the connection has been inactive for + L{self.timeOut} + seconds. Cleanly tears the connection down, attempting to notify the + peer if needed. + + We override this method to add two extra bits of functionality: + + - We want to log the timeout. + - We want to send a GOAWAY frame indicating that the connection is + being terminated, and whether it was clean or not. We have to do this + before the connection is torn down. + """ + self._log.info( + "Timing out client {client}", client=self.transport.getPeer() + ) + + # Check whether there are open streams. If there are, we're going to + # want to use the error code PROTOCOL_ERROR. If there aren't, use + # NO_ERROR. + if (self.conn.open_outbound_streams > 0 or + self.conn.open_inbound_streams > 0): + error_code = h2.errors.ErrorCodes.PROTOCOL_ERROR + else: + error_code = h2.errors.ErrorCodes.NO_ERROR + + self.conn.close_connection(error_code=error_code) + self.transport.write(self.conn.data_to_send()) + + # Don't let the client hold this connection open too long. + if self.abortTimeout is not None: + # We use self.callLater because that's what TimeoutMixin does, even + # though we have a perfectly good reactor sitting around. See + # https://twistedmatrix.com/trac/ticket/8488. + self._abortingCall = self.callLater( + self.abortTimeout, self.forceAbortClient + ) + + # We're done, throw the connection away. + self.transport.loseConnection() + + + def forceAbortClient(self): + """ + Called if C{abortTimeout} seconds have passed since the timeout fired, + and the connection still hasn't gone away. This can really only happen + on extremely bad connections or when clients are maliciously attempting + to keep connections open. + """ + self._log.info( + "Forcibly timing out client: {client}", + client=self.transport.getPeer() + ) + # We want to lose track of the _abortingCall so that no-one tries to + # cancel it. + self._abortingCall = None + self.transport.abortConnection() + + + def connectionLost(self, reason, _cancelTimeouts=True): + """ + Called when the transport connection is lost. + + Informs all outstanding response handlers that the connection + has been lost, and cleans up all internal state. + + @param reason: See L{IProtocol.connectionLost} + + @param _cancelTimeouts: Propagate the C{reason} to this + connection's streams but don't cancel any timers, so that + peers who never read the data we've written are eventually + timed out. + """ + self._stillProducing = False + if _cancelTimeouts: + self.setTimeout(None) + + for stream in self.streams.values(): + stream.connectionLost(reason) + + for streamID in list(self.streams.keys()): + self._requestDone(streamID) + + # If we were going to force-close the transport, we don't have to now. + if _cancelTimeouts and self._abortingCall is not None: + self._abortingCall.cancel() + self._abortingCall = None + + + # Implementation of IPushProducer + # + # Here's how we handle IPushProducer. We have multiple outstanding + # H2Streams. Each of these exposes an IConsumer interface to the response + # handler that allows it to push data into the H2Stream. The H2Stream then + # writes the data into the H2Connection object. + # + # The H2Connection needs to manage these writes to account for: + # + # - flow control + # - priority + # + # We manage each of these in different ways. + # + # For flow control, we simply use the equivalent of the IPushProducer + # interface. We simply tell the H2Stream: "Hey, you can't send any data + # right now, sorry!". When that stream becomes unblocked, we free it up + # again. This allows the H2Stream to propagate this backpressure up the + # chain. + # + # For priority, we need to keep a backlog of data frames that we can send, + # and interleave them appropriately. This backlog is most sensibly kept in + # the H2Connection object itself. We keep one queue per stream, which is + # where the writes go, and then we have a loop that manages popping these + # streams off in priority order. + # + # Logically then, we go as follows: + # + # 1. Stream calls writeDataToStream(). This causes a DataFrame to be placed + # on the queue for that stream. It also informs the priority + # implementation that this stream is unblocked. + # 2. The _sendPrioritisedData() function spins in a tight loop. Each + # iteration it asks the priority implementation which stream should send + # next, and pops a data frame off that stream's queue. If, after sending + # that frame, there is no data left on that stream's queue, the function + # informs the priority implementation that the stream is blocked. + # + # If all streams are blocked, or if there are no outstanding streams, the + # _sendPrioritisedData function waits to be awoken when more data is ready + # to send. + # + # Note that all of this only applies to *data*. Headers and other control + # frames deliberately skip this processing as they are not subject to flow + # control or priority constraints. Instead, they are stored in their own buffer + # which is used primarily to detect excessive buffering. + def stopProducing(self): + """ + Stop producing data. + + This tells the L{H2Connection} that its consumer has died, so it must + stop producing data for good. + """ + self.connectionLost(ConnectionLost("Producing stopped")) + + + def pauseProducing(self): + """ + Pause producing data. + + Tells the L{H2Connection} that it has produced too much data to process + for the time being, and to stop until resumeProducing() is called. + """ + self._consumerBlocked = Deferred() + # Ensure pending control data (if any) are sent first. + self._consumerBlocked.addCallback(self._flushBufferedControlData) + + + def resumeProducing(self): + """ + Resume producing data. + + This tells the L{H2Connection} to re-add itself to the main loop and + produce more data for the consumer. + """ + if self._consumerBlocked is not None: + d = self._consumerBlocked + self._consumerBlocked = None + d.callback(None) + + + def _sendPrioritisedData(self, *args): + """ + The data sending loop. This function repeatedly calls itself, either + from L{Deferred}s or from + L{reactor.callLater} + + This function sends data on streams according to the rules of HTTP/2 + priority. It ensures that the data from each stream is interleved + according to the priority signalled by the client, making sure that the + connection is used with maximal efficiency. + + This function will execute if data is available: if all data is + exhausted, the function will place a deferred onto the L{H2Connection} + object and wait until it is called to resume executing. + """ + # If producing has stopped, we're done. Don't reschedule ourselves + if not self._stillProducing: + return + + stream = None + + while stream is None: + try: + stream = next(self.priority) + except priority.DeadlockError: + # All streams are currently blocked or not progressing. Wait + # until a new one becomes available. + assert self._sendingDeferred is None + self._sendingDeferred = Deferred() + self._sendingDeferred.addCallback(self._sendPrioritisedData) + return + + # Wait behind the transport. + if self._consumerBlocked is not None: + self._consumerBlocked.addCallback(self._sendPrioritisedData) + return + + self.resetTimeout() + + remainingWindow = self.conn.local_flow_control_window(stream) + frameData = self._outboundStreamQueues[stream].popleft() + maxFrameSize = min(self.conn.max_outbound_frame_size, remainingWindow) + + if frameData is _END_STREAM_SENTINEL: + # There's no error handling here even though this can throw + # ProtocolError because we really shouldn't encounter this problem. + # If we do, that's a nasty bug. + self.conn.end_stream(stream) + self.transport.write(self.conn.data_to_send()) + + # Clean up the stream + self._requestDone(stream) + else: + # Respect the max frame size. + if len(frameData) > maxFrameSize: + excessData = frameData[maxFrameSize:] + frameData = frameData[:maxFrameSize] + self._outboundStreamQueues[stream].appendleft(excessData) + + # There's deliberately no error handling here, because this just + # absolutely should not happen. + # If for whatever reason the max frame length is zero and so we + # have no frame data to send, don't send any. + if frameData: + self.conn.send_data(stream, frameData) + self.transport.write(self.conn.data_to_send()) + + # If there's no data left, this stream is now blocked. + if not self._outboundStreamQueues[stream]: + self.priority.block(stream) + + # Also, if the stream's flow control window is exhausted, tell it + # to stop. + if self.remainingOutboundWindow(stream) <= 0: + self.streams[stream].flowControlBlocked() + + self._reactor.callLater(0, self._sendPrioritisedData) + + + # Internal functions. + def _requestReceived(self, event): + """ + Internal handler for when a request has been received. + + @param event: The Hyper-h2 event that encodes information about the + received request. + @type event: L{h2.events.RequestReceived} + """ + stream = H2Stream( + event.stream_id, + self, event.headers, + self.requestFactory, + self.site, + self.factory + ) + self.streams[event.stream_id] = stream + self._streamCleanupCallbacks[event.stream_id] = Deferred() + self._outboundStreamQueues[event.stream_id] = deque() + + # Add the stream to the priority tree but immediately block it. + try: + self.priority.insert_stream(event.stream_id) + except priority.DuplicateStreamError: + # Stream already in the tree. This can happen if we received a + # PRIORITY frame before a HEADERS frame. Just move on: we set the + # stream up properly in _handlePriorityUpdate. + pass + else: + self.priority.block(event.stream_id) + + + def _requestDataReceived(self, event): + """ + Internal handler for when a chunk of data is received for a given + request. + + @param event: The Hyper-h2 event that encodes information about the + received data. + @type event: L{h2.events.DataReceived} + """ + stream = self.streams[event.stream_id] + stream.receiveDataChunk(event.data, event.flow_controlled_length) + + + def _requestEnded(self, event): + """ + Internal handler for when a request is complete, and we expect no + further data for that request. + + @param event: The Hyper-h2 event that encodes information about the + completed stream. + @type event: L{h2.events.StreamEnded} + """ + stream = self.streams[event.stream_id] + stream.requestComplete() + + + def _requestAborted(self, event): + """ + Internal handler for when a request is aborted by a remote peer. + + @param event: The Hyper-h2 event that encodes information about the + reset stream. + @type event: L{h2.events.StreamReset} + """ + stream = self.streams[event.stream_id] + stream.connectionLost( + ConnectionLost("Stream reset with code %s" % event.error_code) + ) + self._requestDone(event.stream_id) + + + def _handlePriorityUpdate(self, event): + """ + Internal handler for when a stream priority is updated. + + @param event: The Hyper-h2 event that encodes information about the + stream reprioritization. + @type event: L{h2.events.PriorityUpdated} + """ + try: + self.priority.reprioritize( + stream_id=event.stream_id, + depends_on=event.depends_on or None, + weight=event.weight, + exclusive=event.exclusive, + ) + except priority.MissingStreamError: + # A PRIORITY frame arrived before the HEADERS frame that would + # trigger us to insert the stream into the tree. That's fine: we + # can create the stream here and mark it as blocked. + self.priority.insert_stream( + stream_id=event.stream_id, + depends_on=event.depends_on or None, + weight=event.weight, + exclusive=event.exclusive, + ) + self.priority.block(event.stream_id) + + + def writeHeaders(self, version, code, reason, headers, streamID): + """ + Called by L{twisted.web.http.Request} objects to write a complete set + of HTTP headers to a stream. + + @param version: The HTTP version in use. Unused in HTTP/2. + @type version: L{bytes} + + @param code: The HTTP status code to write. + @type code: L{bytes} + + @param reason: The HTTP reason phrase to write. Unused in HTTP/2. + @type reason: L{bytes} + + @param headers: The headers to write to the stream. + @type headers: L{twisted.web.http_headers.Headers} + + @param streamID: The ID of the stream to write the headers to. + @type streamID: L{int} + """ + headers.insert(0, (b':status', code)) + + try: + self.conn.send_headers(streamID, headers) + except h2.exceptions.StreamClosedError: + # Stream was closed by the client at some point. We need to not + # explode here: just swallow the error. That's what write() does + # when a connection is lost, so that's what we do too. + return + else: + self._tryToWriteControlData() + + + def writeDataToStream(self, streamID, data): + """ + May be called by L{H2Stream} objects to write response data to a given + stream. Writes a single data frame. + + @param streamID: The ID of the stream to write the data to. + @type streamID: L{int} + + @param data: The data chunk to write to the stream. + @type data: L{bytes} + """ + self._outboundStreamQueues[streamID].append(data) + + # There's obviously no point unblocking this stream and the sending + # loop if the data can't actually be sent, so confirm that there's + # some room to send data. + if self.conn.local_flow_control_window(streamID) > 0: + self.priority.unblock(streamID) + if self._sendingDeferred is not None: + d = self._sendingDeferred + self._sendingDeferred = None + d.callback(streamID) + + if self.remainingOutboundWindow(streamID) <= 0: + self.streams[streamID].flowControlBlocked() + + + def endRequest(self, streamID): + """ + Called by L{H2Stream} objects to signal completion of a response. + + @param streamID: The ID of the stream to write the data to. + @type streamID: L{int} + """ + self._outboundStreamQueues[streamID].append(_END_STREAM_SENTINEL) + self.priority.unblock(streamID) + if self._sendingDeferred is not None: + d = self._sendingDeferred + self._sendingDeferred = None + d.callback(streamID) + + + def abortRequest(self, streamID): + """ + Called by L{H2Stream} objects to request early termination of a stream. + This emits a RstStream frame and then removes all stream state. + + @param streamID: The ID of the stream to write the data to. + @type streamID: L{int} + """ + self.conn.reset_stream(streamID) + stillActive = self._tryToWriteControlData() + if stillActive: + self._requestDone(streamID) + + + def _requestDone(self, streamID): + """ + Called internally by the data sending loop to clean up state that was + being used for the stream. Called when the stream is complete. + + @param streamID: The ID of the stream to clean up state for. + @type streamID: L{int} + """ + del self._outboundStreamQueues[streamID] + self.priority.remove_stream(streamID) + del self.streams[streamID] + cleanupCallback = self._streamCleanupCallbacks.pop(streamID) + cleanupCallback.callback(streamID) + + + def remainingOutboundWindow(self, streamID): + """ + Called to determine how much room is left in the send window for a + given stream. Allows us to handle blocking and unblocking producers. + + @param streamID: The ID of the stream whose flow control window we'll + check. + @type streamID: L{int} + + @return: The amount of room remaining in the send window for the given + stream, including the data queued to be sent. + @rtype: L{int} + """ + # TODO: This involves a fair bit of looping and computation for + # something that is called a lot. Consider caching values somewhere. + windowSize = self.conn.local_flow_control_window(streamID) + sendQueue = self._outboundStreamQueues[streamID] + alreadyConsumed = sum( + len(chunk) for chunk in sendQueue + if chunk is not _END_STREAM_SENTINEL + ) + + return windowSize - alreadyConsumed + + + def _handleWindowUpdate(self, event): + """ + Manage flow control windows. + + Streams that are blocked on flow control will register themselves with + the connection. This will fire deferreds that wake those streams up and + allow them to continue processing. + + @param event: The Hyper-h2 event that encodes information about the + flow control window change. + @type event: L{h2.events.WindowUpdated} + """ + streamID = event.stream_id + + if streamID: + if not self._streamIsActive(streamID): + # We may have already cleaned up our stream state, making this + # a late WINDOW_UPDATE frame. That's fine: the update is + # unnecessary but benign. We'll ignore it. + return + + # If we haven't got any data to send, don't unblock the stream. If + # we do, we'll eventually get an exception inside the + # _sendPrioritisedData loop some time later. + if self._outboundStreamQueues.get(streamID): + self.priority.unblock(streamID) + self.streams[streamID].windowUpdated() + else: + # Update strictly applies to all streams. + for stream in self.streams.values(): + stream.windowUpdated() + + # If we still have data to send for this stream, unblock it. + if self._outboundStreamQueues.get(stream.streamID): + self.priority.unblock(stream.streamID) + + + def getPeer(self): + """ + Get the remote address of this connection. + + Treat this method with caution. It is the unfortunate result of the + CGI and Jabber standards, but should not be considered reliable for + the usual host of reasons; port forwarding, proxying, firewalls, IP + masquerading, etc. + + @return: An L{IAddress} provider. + """ + return self.transport.getPeer() + + + def getHost(self): + """ + Similar to getPeer, but returns an address describing this side of the + connection. + + @return: An L{IAddress} provider. + """ + return self.transport.getHost() + + + def openStreamWindow(self, streamID, increment): + """ + Open the stream window by a given increment. + + @param streamID: The ID of the stream whose window needs to be opened. + @type streamID: L{int} + + @param increment: The amount by which the stream window must be + incremented. + @type increment: L{int} + """ + self.conn.acknowledge_received_data(increment, streamID) + self._tryToWriteControlData() + + + def _isSecure(self): + """ + Returns L{True} if this channel is using a secure transport. + + @returns: L{True} if this channel is secure. + @rtype: L{bool} + """ + # A channel is secure if its transport is ISSLTransport. + return ISSLTransport(self.transport, None) is not None + + + def _send100Continue(self, streamID): + """ + Sends a 100 Continue response, used to signal to clients that further + processing will be performed. + + @param streamID: The ID of the stream that needs the 100 Continue + response + @type streamID: L{int} + """ + headers = [(b':status', b'100')] + self.conn.send_headers(headers=headers, stream_id=streamID) + self._tryToWriteControlData() + + + def _respondToBadRequestAndDisconnect(self, streamID): + """ + This is a quick and dirty way of responding to bad requests. + + As described by HTTP standard we should be patient and accept the + whole request from the client before sending a polite bad request + response, even in the case when clients send tons of data. + + Unlike in the HTTP/1.1 case, this does not actually disconnect the + underlying transport: there's no need. This instead just sends a 400 + response and terminates the stream. + + @param streamID: The ID of the stream that needs the 100 Continue + response + @type streamID: L{int} + """ + headers = [(b':status', b'400')] + self.conn.send_headers( + headers=headers, + stream_id=streamID, + end_stream=True + ) + stillActive = self._tryToWriteControlData() + if stillActive: + stream = self.streams[streamID] + stream.connectionLost(ConnectionLost("Invalid request")) + self._requestDone(streamID) + + + def _streamIsActive(self, streamID): + """ + Checks whether Twisted has still got state for a given stream and so + can process events for that stream. + + @param streamID: The ID of the stream that needs processing. + @type streamID: L{int} + + @return: Whether the stream still has state allocated. + @rtype: L{bool} + """ + return streamID in self.streams + + def _tryToWriteControlData(self): + """ + Checks whether the connection is blocked on flow control and, + if it isn't, writes any buffered control data. + + @return: L{True} if the connection is still active and + L{False} if it was aborted because too many bytes have + been written but not consumed by the other end. + """ + bufferedBytes = self.conn.data_to_send() + if not bufferedBytes: + return True + + if self._consumerBlocked is None and not self._bufferedControlFrames: + # The consumer isn't blocked, and we don't have any buffered frames: + # write this directly. + self.transport.write(bufferedBytes) + return True + else: + # Either the consumer is blocked or we have buffered frames. If the + # consumer is blocked, we'll write this when we unblock. If we have + # buffered frames, we have presumably been re-entered from + # transport.write, and so to avoid reordering issues we'll buffer anyway. + self._bufferedControlFrames.append(bufferedBytes) + self._bufferedControlFrameBytes += len(bufferedBytes) + + if self._bufferedControlFrameBytes >= self._maxBufferedControlFrameBytes: + self._log.error( + "Maximum number of control frame bytes buffered: " + "{bufferedControlFrameBytes} > = {maxBufferedControlFrameBytes}. " + "Aborting connection to client: {client} ", + bufferedControlFrameBytes=self._bufferedControlFrameBytes, + maxBufferedControlFrameBytes=self._maxBufferedControlFrameBytes, + client=self.transport.getPeer(), + ) + # We've exceeded a reasonable buffer size for max buffered control frames. + # This is a denial of service risk, so we're going to drop this connection. + self.transport.abortConnection() + self.connectionLost(ExcessiveBufferingError()) + return False + return True + + def _flushBufferedControlData(self, *args): + """ + Called when the connection is marked writable again after being marked unwritable. + Attempts to flush buffered control data if there is any. + """ + # To respect backpressure here we send each write in order, paying attention to whether + # we got blocked + while self._consumerBlocked is None and self._bufferedControlFrames: + nextWrite = self._bufferedControlFrames.popleft() + self._bufferedControlFrameBytes -= len(nextWrite) + self.transport.write(nextWrite) + + +@implementer(ITransport, IConsumer, IPushProducer) +class H2Stream(object): + """ + A class representing a single HTTP/2 stream. + + This class works hand-in-hand with L{H2Connection}. It acts to provide an + implementation of L{ITransport}, L{IConsumer}, and L{IProducer} that work + for a single HTTP/2 connection, while tightly cleaving to the interface + provided by those interfaces. It does this by having a tight coupling to + L{H2Connection}, which allows associating many of the functions of + L{ITransport}, L{IConsumer}, and L{IProducer} to objects on a + stream-specific level. + + @ivar streamID: The numerical stream ID that this object corresponds to. + @type streamID: L{int} + + @ivar producing: Whether this stream is currently allowed to produce data + to its consumer. + @type producing: L{bool} + + @ivar command: The HTTP verb used on the request. + @type command: L{unicode} + + @ivar path: The HTTP path used on the request. + @type path: L{unicode} + + @ivar producer: The object producing the response, if any. + @type producer: L{IProducer} + + @ivar site: The L{twisted.web.server.Site} object this stream belongs to, + if any. + @type site: L{twisted.web.server.Site} + + @ivar factory: The L{twisted.web.http.HTTPFactory} object that constructed + this stream's parent connection. + @type factory: L{twisted.web.http.HTTPFactory} + + @ivar _producerProducing: Whether the producer stored in producer is + currently producing data. + @type _producerProducing: L{bool} + + @ivar _inboundDataBuffer: Any data that has been received from the network + but has not yet been received by the consumer. + @type _inboundDataBuffer: A L{collections.deque} containing L{bytes} + + @ivar _conn: A reference to the connection this stream belongs to. + @type _conn: L{H2Connection} + + @ivar _request: A request object that this stream corresponds to. + @type _request: L{twisted.web.iweb.IRequest} + + @ivar _buffer: A buffer containing data produced by the producer that could + not be sent on the network at this time. + @type _buffer: L{io.BytesIO} + """ + # We need a transport property for t.w.h.Request, but HTTP/2 doesn't want + # to expose it. So we just set it to None. + transport = None + + + def __init__(self, streamID, connection, headers, + requestFactory, site, factory): + """ + Initialize this HTTP/2 stream. + + @param streamID: The numerical stream ID that this object corresponds + to. + @type streamID: L{int} + + @param connection: The HTTP/2 connection this stream belongs to. + @type connection: L{H2Connection} + + @param headers: The HTTP/2 request headers. + @type headers: A L{list} of L{tuple}s of header name and header value, + both as L{bytes}. + + @param requestFactory: A function that builds appropriate request + request objects. + @type requestFactory: A callable that returns a + L{twisted.web.iweb.IRequest}. + + @param site: The L{twisted.web.server.Site} object this stream belongs + to, if any. + @type site: L{twisted.web.server.Site} + + @param factory: The L{twisted.web.http.HTTPFactory} object that + constructed this stream's parent connection. + @type factory: L{twisted.web.http.HTTPFactory} + """ + + self.streamID = streamID + self.site = site + self.factory = factory + self.producing = True + self.command = None + self.path = None + self.producer = None + self._producerProducing = False + self._hasStreamingProducer = None + self._inboundDataBuffer = deque() + self._conn = connection + self._request = requestFactory(self, queued=False) + self._buffer = io.BytesIO() + + self._convertHeaders(headers) + + + def _convertHeaders(self, headers): + """ + This method converts the HTTP/2 header set into something that looks + like HTTP/1.1. In particular, it strips the 'special' headers and adds + a Host: header. + + @param headers: The HTTP/2 header set. + @type headers: A L{list} of L{tuple}s of header name and header value, + both as L{bytes}. + """ + gotLength = False + + for header in headers: + if not header[0].startswith(b':'): + gotLength = ( + _addHeaderToRequest(self._request, header) or gotLength + ) + elif header[0] == b':method': + self.command = header[1] + elif header[0] == b':path': + self.path = header[1] + elif header[0] == b':authority': + # This is essentially the Host: header from HTTP/1.1 + _addHeaderToRequest(self._request, (b'host', header[1])) + + if not gotLength: + if self.command in (b'GET', b'HEAD'): + self._request.gotLength(0) + else: + self._request.gotLength(None) + + self._request.parseCookies() + expectContinue = self._request.requestHeaders.getRawHeaders(b'expect') + if expectContinue and expectContinue[0].lower() == b'100-continue': + self._send100Continue() + + + # Methods called by the H2Connection + def receiveDataChunk(self, data, flowControlledLength): + """ + Called when the connection has received a chunk of data from the + underlying transport. If the stream has been registered with a + consumer, and is currently able to push data, immediately passes it + through. Otherwise, buffers the chunk until we can start producing. + + @param data: The chunk of data that was received. + @type data: L{bytes} + + @param flowControlledLength: The total flow controlled length of this + chunk, which is used when we want to re-open the window. May be + different to C{len(data)}. + @type flowControlledLength: L{int} + """ + if not self.producing: + # Buffer data. + self._inboundDataBuffer.append((data, flowControlledLength)) + else: + self._request.handleContentChunk(data) + self._conn.openStreamWindow(self.streamID, flowControlledLength) + + + def requestComplete(self): + """ + Called by the L{H2Connection} when the all data for a request has been + received. Currently, with the legacy L{twisted.web.http.Request} + object, just calls requestReceived unless the producer wants us to be + quiet. + """ + if self.producing: + self._request.requestReceived(self.command, self.path, b'HTTP/2') + else: + self._inboundDataBuffer.append((_END_STREAM_SENTINEL, None)) + + + def connectionLost(self, reason): + """ + Called by the L{H2Connection} when a connection is lost or a stream is + reset. + + @param reason: The reason the connection was lost. + @type reason: L{str} + """ + self._request.connectionLost(reason) + + + def windowUpdated(self): + """ + Called by the L{H2Connection} when this stream's flow control window + has been opened. + """ + # If we don't have a producer, we have no-one to tell. + if not self.producer: + return + + # If we're not blocked on flow control, we don't care. + if self._producerProducing: + return + + # We check whether the stream's flow control window is actually above + # 0, and then, if a producer is registered and we still have space in + # the window, we unblock it. + remainingWindow = self._conn.remainingOutboundWindow(self.streamID) + if not remainingWindow > 0: + return + + # We have a producer and space in the window, so that producer can + # start producing again! + self._producerProducing = True + self.producer.resumeProducing() + + + def flowControlBlocked(self): + """ + Called by the L{H2Connection} when this stream's flow control window + has been exhausted. + """ + if not self.producer: + return + + if self._producerProducing: + self.producer.pauseProducing() + self._producerProducing = False + + + # Methods called by the consumer (usually an IRequest). + def writeHeaders(self, version, code, reason, headers): + """ + Called by the consumer to write headers to the stream. + + @param version: The HTTP version. + @type version: L{bytes} + + @param code: The status code. + @type code: L{int} + + @param reason: The reason phrase. Ignored in HTTP/2. + @type reason: L{bytes} + + @param headers: The HTTP response headers. + @type: Any iterable of two-tuples of L{bytes}, representing header + names and header values. + """ + self._conn.writeHeaders(version, code, reason, headers, self.streamID) + + + def requestDone(self, request): + """ + Called by a consumer to clean up whatever permanent state is in use. + + @param request: The request calling the method. + @type request: L{twisted.web.iweb.IRequest} + """ + self._conn.endRequest(self.streamID) + + + def _send100Continue(self): + """ + Sends a 100 Continue response, used to signal to clients that further + processing will be performed. + """ + self._conn._send100Continue(self.streamID) + + + def _respondToBadRequestAndDisconnect(self): + """ + This is a quick and dirty way of responding to bad requests. + + As described by HTTP standard we should be patient and accept the + whole request from the client before sending a polite bad request + response, even in the case when clients send tons of data. + + Unlike in the HTTP/1.1 case, this does not actually disconnect the + underlying transport: there's no need. This instead just sends a 400 + response and terminates the stream. + """ + self._conn._respondToBadRequestAndDisconnect(self.streamID) + + + # Implementation: ITransport + def write(self, data): + """ + Write a single chunk of data into a data frame. + + @param data: The data chunk to send. + @type data: L{bytes} + """ + self._conn.writeDataToStream(self.streamID, data) + return + + + def writeSequence(self, iovec): + """ + Write a sequence of chunks of data into data frames. + + @param iovec: A sequence of chunks to send. + @type iovec: An iterable of L{bytes} chunks. + """ + for chunk in iovec: + self.write(chunk) + + + def loseConnection(self): + """ + Close the connection after writing all pending data. + """ + self._conn.endRequest(self.streamID) + + + def abortConnection(self): + """ + Forcefully abort the connection by sending a RstStream frame. + """ + self._conn.abortRequest(self.streamID) + + + def getPeer(self): + """ + Get information about the peer. + """ + return self._conn.getPeer() + + + def getHost(self): + """ + Similar to getPeer, but for this side of the connection. + """ + return self._conn.getHost() + + + def isSecure(self): + """ + Returns L{True} if this channel is using a secure transport. + + @returns: L{True} if this channel is secure. + @rtype: L{bool} + """ + return self._conn._isSecure() + + + # Implementation: IConsumer + def registerProducer(self, producer, streaming): + """ + Register to receive data from a producer. + + This sets self to be a consumer for a producer. When this object runs + out of data (as when a send(2) call on a socket succeeds in moving the + last data from a userspace buffer into a kernelspace buffer), it will + ask the producer to resumeProducing(). + + For L{IPullProducer} providers, C{resumeProducing} will be called once + each time data is required. + + For L{IPushProducer} providers, C{pauseProducing} will be called + whenever the write buffer fills up and C{resumeProducing} will only be + called when it empties. + + @param producer: The producer to register. + @type producer: L{IProducer} provider + + @param streaming: L{True} if C{producer} provides L{IPushProducer}, + L{False} if C{producer} provides L{IPullProducer}. + @type streaming: L{bool} + + @raise RuntimeError: If a producer is already registered. + + @return: L{None} + """ + if self.producer: + raise ValueError( + "registering producer %s before previous one (%s) was " + "unregistered" % (producer, self.producer)) + + if not streaming: + self.hasStreamingProducer = False + producer = _PullToPush(producer, self) + producer.startStreaming() + else: + self.hasStreamingProducer = True + + self.producer = producer + self._producerProducing = True + + + def unregisterProducer(self): + """ + @see: L{IConsumer.unregisterProducer} + """ + # When the producer is unregistered, we're done. + if self.producer is not None and not self.hasStreamingProducer: + self.producer.stopStreaming() + + self._producerProducing = False + self.producer = None + self.hasStreamingProducer = None + + + # Implementation: IPushProducer + def stopProducing(self): + """ + @see: L{IProducer.stopProducing} + """ + self.producing = False + self.abortConnection() + + + def pauseProducing(self): + """ + @see: L{IPushProducer.pauseProducing} + """ + self.producing = False + + + def resumeProducing(self): + """ + @see: L{IPushProducer.resumeProducing} + """ + self.producing = True + consumedLength = 0 + + while self.producing and self._inboundDataBuffer: + # Allow for pauseProducing to be called in response to a call to + # resumeProducing. + chunk, flowControlledLength = self._inboundDataBuffer.popleft() + + if chunk is _END_STREAM_SENTINEL: + self.requestComplete() + else: + consumedLength += flowControlledLength + self._request.handleContentChunk(chunk) + + self._conn.openStreamWindow(self.streamID, consumedLength) + + + +def _addHeaderToRequest(request, header): + """ + Add a header tuple to a request header object. + + @param request: The request to add the header tuple to. + @type request: L{twisted.web.http.Request} + + @param header: The header tuple to add to the request. + @type header: A L{tuple} with two elements, the header name and header + value, both as L{bytes}. + + @return: If the header being added was the C{Content-Length} header. + @rtype: L{bool} + """ + requestHeaders = request.requestHeaders + name, value = header + values = requestHeaders.getRawHeaders(name) + + if values is not None: + values.append(value) + else: + requestHeaders.setRawHeaders(name, [value]) + + if name == b'content-length': + request.gotLength(int(value)) + return True + + return False diff --git a/contrib/python/Twisted/py2/twisted/web/_newclient.py b/contrib/python/Twisted/py2/twisted/web/_newclient.py new file mode 100644 index 00000000000..74a8a6c2d1f --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_newclient.py @@ -0,0 +1,1778 @@ +# -*- test-case-name: twisted.web.test.test_newclient -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An U{HTTP 1.1} client. + +The way to use the functionality provided by this module is to: + + - Connect a L{HTTP11ClientProtocol} to an HTTP server + - Create a L{Request} with the appropriate data + - Pass the request to L{HTTP11ClientProtocol.request} + - The returned Deferred will fire with a L{Response} object + - Create a L{IProtocol} provider which can handle the response body + - Connect it to the response with L{Response.deliverBody} + - When the protocol's C{connectionLost} method is called, the response is + complete. See L{Response.deliverBody} for details. + +Various other classes in this module support this usage: + + - HTTPParser is the basic HTTP parser. It can handle the parts of HTTP which + are symmetric between requests and responses. + + - HTTPClientParser extends HTTPParser to handle response-specific parts of + HTTP. One instance is created for each request to parse the corresponding + response. +""" + +from __future__ import division, absolute_import +__metaclass__ = type + +import re + +from zope.interface import implementer + +from twisted.python.compat import networkString +from twisted.python.components import proxyForInterface +from twisted.python.reflect import fullyQualifiedName +from twisted.python.failure import Failure +from twisted.internet.interfaces import IConsumer, IPushProducer +from twisted.internet.error import ConnectionDone +from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred +from twisted.internet.defer import CancelledError +from twisted.internet.protocol import Protocol +from twisted.protocols.basic import LineReceiver +from twisted.web.iweb import UNKNOWN_LENGTH, IResponse, IClientRequest +from twisted.web.http_headers import Headers +from twisted.web.http import NO_CONTENT, NOT_MODIFIED +from twisted.web.http import _DataLoss, PotentialDataLoss +from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder +from twisted.logger import Logger + +# States HTTPParser can be in +STATUS = u'STATUS' +HEADER = u'HEADER' +BODY = u'BODY' +DONE = u'DONE' +_moduleLog = Logger() + + +class BadHeaders(Exception): + """ + Headers passed to L{Request} were in some way invalid. + """ + + + +class ExcessWrite(Exception): + """ + The body L{IBodyProducer} for a request tried to write data after + indicating it had finished writing data. + """ + + +class ParseError(Exception): + """ + Some received data could not be parsed. + + @ivar data: The string which could not be parsed. + """ + def __init__(self, reason, data): + Exception.__init__(self, reason, data) + self.data = data + + + +class BadResponseVersion(ParseError): + """ + The version string in a status line was unparsable. + """ + + + +class _WrapperException(Exception): + """ + L{_WrapperException} is the base exception type for exceptions which + include one or more other exceptions as the low-level causes. + + @ivar reasons: A L{list} of one or more L{Failure} instances encountered + during an HTTP request. See subclass documentation for more details. + """ + def __init__(self, reasons): + Exception.__init__(self, reasons) + self.reasons = reasons + + + +class RequestGenerationFailed(_WrapperException): + """ + There was an error while creating the bytes which make up a request. + + @ivar reasons: A C{list} of one or more L{Failure} instances giving the + reasons the request generation was considered to have failed. + """ + + + +class RequestTransmissionFailed(_WrapperException): + """ + There was an error while sending the bytes which make up a request. + + @ivar reasons: A C{list} of one or more L{Failure} instances giving the + reasons the request transmission was considered to have failed. + """ + + + +class ConnectionAborted(Exception): + """ + The connection was explicitly aborted by application code. + """ + + + +class WrongBodyLength(Exception): + """ + An L{IBodyProducer} declared the number of bytes it was going to + produce (via its C{length} attribute) and then produced a different number + of bytes. + """ + + + +class ResponseDone(Exception): + """ + L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the + protocol passed to L{Response.deliverBody} and indicates that the entire + response has been delivered. + """ + + + +class ResponseFailed(_WrapperException): + """ + L{ResponseFailed} indicates that all of the response to a request was not + received for some reason. + + @ivar reasons: A C{list} of one or more L{Failure} instances giving the + reasons the response was considered to have failed. + + @ivar response: If specified, the L{Response} received from the server (and + in particular the status code and the headers). + """ + + def __init__(self, reasons, response=None): + _WrapperException.__init__(self, reasons) + self.response = response + + + +class ResponseNeverReceived(ResponseFailed): + """ + A L{ResponseFailed} that knows no response bytes at all have been received. + """ + + + +class RequestNotSent(Exception): + """ + L{RequestNotSent} indicates that an attempt was made to issue a request but + for reasons unrelated to the details of the request itself, the request + could not be sent. For example, this may indicate that an attempt was made + to send a request using a protocol which is no longer connected to a + server. + """ + + + +def _callAppFunction(function): + """ + Call C{function}. If it raises an exception, log it with a minimal + description of the source. + + @return: L{None} + """ + try: + function() + except: + _moduleLog.failure( + u"Unexpected exception from {name}", + name=fullyQualifiedName(function) + ) + + + +class HTTPParser(LineReceiver): + """ + L{HTTPParser} handles the parsing side of HTTP processing. With a suitable + subclass, it can parse either the client side or the server side of the + connection. + + @ivar headers: All of the non-connection control message headers yet + received. + + @ivar state: State indicator for the response parsing state machine. One + of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}. + + @ivar _partialHeader: L{None} or a C{list} of the lines of a multiline + header while that header is being received. + """ + + # NOTE: According to HTTP spec, we're supposed to eat the + # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that + # doesn't sound like a good idea to me, because it makes it impossible to + # have a non-authenticating transparent proxy in front of an authenticating + # proxy. An authenticating proxy can eat them itself. -jknight + # + # Further, quoting + # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html + # regarding the 'Proxy-Connection' header: + # + # The Proxy-Connection: header is a mistake in how some web browsers + # use HTTP. Its name is the result of a false analogy. It is not a + # standard part of the protocol. There is a different standard + # protocol mechanism for doing what it does. And its existence + # imposes a requirement upon HTTP servers such that no proxy HTTP + # server can be standards-conforming in practice. + # + # -exarkun + + # Some servers (like http://news.ycombinator.com/) return status lines and + # HTTP headers delimited by \n instead of \r\n. + delimiter = b'\n' + + CONNECTION_CONTROL_HEADERS = set([ + b'content-length', b'connection', b'keep-alive', b'te', + b'trailers', b'transfer-encoding', b'upgrade', + b'proxy-connection']) + + def connectionMade(self): + self.headers = Headers() + self.connHeaders = Headers() + self.state = STATUS + self._partialHeader = None + + + def switchToBodyMode(self, decoder): + """ + Switch to body parsing mode - interpret any more bytes delivered as + part of the message body and deliver them to the given decoder. + """ + if self.state == BODY: + raise RuntimeError(u"already in body mode") + + self.bodyDecoder = decoder + self.state = BODY + self.setRawMode() + + + def lineReceived(self, line): + """ + Handle one line from a response. + """ + # Handle the normal CR LF case. + if line[-1:] == b'\r': + line = line[:-1] + + if self.state == STATUS: + self.statusReceived(line) + self.state = HEADER + elif self.state == HEADER: + if not line or line[0] not in b' \t': + if self._partialHeader is not None: + header = b''.join(self._partialHeader) + name, value = header.split(b':', 1) + value = value.strip() + self.headerReceived(name, value) + if not line: + # Empty line means the header section is over. + self.allHeadersReceived() + else: + # Line not beginning with LWS is another header. + self._partialHeader = [line] + else: + # A line beginning with LWS is a continuation of a header + # begun on a previous line. + self._partialHeader.append(line) + + + def rawDataReceived(self, data): + """ + Pass data from the message body to the body decoder object. + """ + self.bodyDecoder.dataReceived(data) + + + def isConnectionControlHeader(self, name): + """ + Return C{True} if the given lower-cased name is the name of a + connection control header (rather than an entity header). + + According to RFC 2616, section 14.10, the tokens in the Connection + header are probably relevant here. However, I am not sure what the + practical consequences of either implementing or ignoring that are. + So I leave it unimplemented for the time being. + """ + return name in self.CONNECTION_CONTROL_HEADERS + + + def statusReceived(self, status): + """ + Callback invoked whenever the first line of a new message is received. + Override this. + + @param status: The first line of an HTTP request or response message + without trailing I{CR LF}. + @type status: C{bytes} + """ + + + def headerReceived(self, name, value): + """ + Store the given header in C{self.headers}. + """ + name = name.lower() + if self.isConnectionControlHeader(name): + headers = self.connHeaders + else: + headers = self.headers + headers.addRawHeader(name, value) + + + def allHeadersReceived(self): + """ + Callback invoked after the last header is passed to C{headerReceived}. + Override this to change to the C{BODY} or C{DONE} state. + """ + self.switchToBodyMode(None) + + + +class HTTPClientParser(HTTPParser): + """ + An HTTP parser which only handles HTTP responses. + + @ivar request: The request with which the expected response is associated. + @type request: L{Request} + + @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a + body. + + @ivar finisher: A callable to invoke when this response is fully parsed. + + @ivar _responseDeferred: A L{Deferred} which will be called back with the + response when all headers in the response have been received. + Thereafter, L{None}. + + @ivar _everReceivedData: C{True} if any bytes have been received. + """ + NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED]) + + _transferDecoders = { + b'chunked': _ChunkedTransferDecoder, + } + + bodyDecoder = None + _log = Logger() + + def __init__(self, request, finisher): + self.request = request + self.finisher = finisher + self._responseDeferred = Deferred() + self._everReceivedData = False + + + def dataReceived(self, data): + """ + Override so that we know if any response has been received. + """ + self._everReceivedData = True + HTTPParser.dataReceived(self, data) + + + def parseVersion(self, strversion): + """ + Parse version strings of the form Protocol '/' Major '.' Minor. E.g. + b'HTTP/1.1'. Returns (protocol, major, minor). Will raise ValueError + on bad syntax. + """ + try: + proto, strnumber = strversion.split(b'/') + major, minor = strnumber.split(b'.') + major, minor = int(major), int(minor) + except ValueError as e: + raise BadResponseVersion(str(e), strversion) + if major < 0 or minor < 0: + raise BadResponseVersion(u"version may not be negative", + strversion) + return (proto, major, minor) + + + def statusReceived(self, status): + """ + Parse the status line into its components and create a response object + to keep track of this response's state. + """ + parts = status.split(b' ', 2) + if len(parts) == 2: + # Some broken servers omit the required `phrase` portion of + # `status-line`. One such server identified as + # "cloudflare-nginx". Others fail to identify themselves + # entirely. Fill in an empty phrase for such cases. + version, codeBytes = parts + phrase = b"" + elif len(parts) == 3: + version, codeBytes, phrase = parts + else: + raise ParseError(u"wrong number of parts", status) + + try: + statusCode = int(codeBytes) + except ValueError: + raise ParseError(u"non-integer status code", status) + + self.response = Response._construct( + self.parseVersion(version), + statusCode, + phrase, + self.headers, + self.transport, + self.request, + ) + + + def _finished(self, rest): + """ + Called to indicate that an entire response has been received. No more + bytes will be interpreted by this L{HTTPClientParser}. Extra bytes are + passed up and the state of this L{HTTPClientParser} is set to I{DONE}. + + @param rest: A C{bytes} giving any extra bytes delivered to this + L{HTTPClientParser} which are not part of the response being + parsed. + """ + self.state = DONE + self.finisher(rest) + + + def isConnectionControlHeader(self, name): + """ + Content-Length in the response to a HEAD request is an entity header, + not a connection control header. + """ + if self.request.method == b'HEAD' and name == b'content-length': + return False + return HTTPParser.isConnectionControlHeader(self, name) + + + def allHeadersReceived(self): + """ + Figure out how long the response body is going to be by examining + headers and stuff. + """ + if 100 <= self.response.code < 200: + # RFC 7231 Section 6.2 says that if we receive a 1XX status code + # and aren't expecting it, we MAY ignore it. That's what we're + # going to do. We reset the parser here, but we leave + # _everReceivedData in its True state because we have, in fact, + # received data. + self._log.info( + "Ignoring unexpected {code} response", + code=self.response.code + ) + self.connectionMade() + del self.response + return + + if (self.response.code in self.NO_BODY_CODES + or self.request.method == b'HEAD'): + self.response.length = 0 + # The order of the next two lines might be of interest when adding + # support for pipelining. + self._finished(self.clearLineBuffer()) + self.response._bodyDataFinished() + else: + transferEncodingHeaders = self.connHeaders.getRawHeaders( + b'transfer-encoding') + if transferEncodingHeaders: + + # This could be a KeyError. However, that would mean we do not + # know how to decode the response body, so failing the request + # is as good a behavior as any. Perhaps someday we will want + # to normalize/document/test this specifically, but failing + # seems fine to me for now. + transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()] + + # If anyone ever invents a transfer encoding other than + # chunked (yea right), and that transfer encoding can predict + # the length of the response body, it might be sensible to + # allow the transfer decoder to set the response object's + # length attribute. + else: + contentLengthHeaders = self.connHeaders.getRawHeaders( + b'content-length') + if contentLengthHeaders is None: + contentLength = None + elif len(contentLengthHeaders) == 1: + contentLength = int(contentLengthHeaders[0]) + self.response.length = contentLength + else: + # "HTTP Message Splitting" or "HTTP Response Smuggling" + # potentially happening. Or it's just a buggy server. + raise ValueError(u"Too many Content-Length headers; " + u"response is invalid") + + if contentLength == 0: + self._finished(self.clearLineBuffer()) + transferDecoder = None + else: + transferDecoder = lambda x, y: _IdentityTransferDecoder( + contentLength, x, y) + + if transferDecoder is None: + self.response._bodyDataFinished() + else: + # Make sure as little data as possible from the response body + # gets delivered to the response object until the response + # object actually indicates it is ready to handle bytes + # (probably because an application gave it a way to interpret + # them). + self.transport.pauseProducing() + self.switchToBodyMode(transferDecoder( + self.response._bodyDataReceived, + self._finished)) + + # This must be last. If it were first, then application code might + # change some state (for example, registering a protocol to receive the + # response body). Then the pauseProducing above would be wrong since + # the response is ready for bytes and nothing else would ever resume + # the transport. + self._responseDeferred.callback(self.response) + del self._responseDeferred + + + def connectionLost(self, reason): + if self.bodyDecoder is not None: + try: + try: + self.bodyDecoder.noMoreData() + except PotentialDataLoss: + self.response._bodyDataFinished(Failure()) + except _DataLoss: + self.response._bodyDataFinished( + Failure(ResponseFailed([reason, Failure()], + self.response))) + else: + self.response._bodyDataFinished() + except: + # Handle exceptions from both the except suites and the else + # suite. Those functions really shouldn't raise exceptions, + # but maybe there's some buggy application code somewhere + # making things difficult. + self._log.failure('') + elif self.state != DONE: + if self._everReceivedData: + exceptionClass = ResponseFailed + else: + exceptionClass = ResponseNeverReceived + self._responseDeferred.errback(Failure(exceptionClass([reason]))) + del self._responseDeferred + + + +_VALID_METHOD = re.compile( + br"\A[%s]+\Z" % ( + bytes().join( + ( + b"!", b"#", b"$", b"%", b"&", b"'", b"*", + b"+", b"-", b".", b"^", b"_", b"`", b"|", b"~", + b"\x30-\x39", + b"\x41-\x5a", + b"\x61-\x7A", + ), + ), + ), +) + + + +def _ensureValidMethod(method): + """ + An HTTP method is an HTTP token, which consists of any visible + ASCII character that is not a delimiter (i.e. one of + C{"(),/:;<=>?@[\\]{}}.) + + @param method: the method to check + @type method: L{bytes} + + @return: the method if it is valid + @rtype: L{bytes} + + @raise ValueError: if the method is not valid + + @see: U{https://tools.ietf.org/html/rfc7230#section-3.1.1}, + U{https://tools.ietf.org/html/rfc7230#section-3.2.6}, + U{https://tools.ietf.org/html/rfc5234#appendix-B.1} + """ + if _VALID_METHOD.match(method): + return method + raise ValueError("Invalid method {!r}".format(method)) + + + +_VALID_URI = re.compile(br'\A[\x21-\x7e]+\Z') + + + +def _ensureValidURI(uri): + """ + A valid URI cannot contain control characters (i.e., characters + between 0-32, inclusive and 127) or non-ASCII characters (i.e., + characters with values between 128-255, inclusive). + + @param uri: the URI to check + @type uri: L{bytes} + + @return: the URI if it is valid + @rtype: L{bytes} + + @raise ValueError: if the URI is not valid + + @see: U{https://tools.ietf.org/html/rfc3986#section-3.3}, + U{https://tools.ietf.org/html/rfc3986#appendix-A}, + U{https://tools.ietf.org/html/rfc5234#appendix-B.1} + """ + if _VALID_URI.match(uri): + return uri + raise ValueError("Invalid URI {!r}".format(uri)) + + + +@implementer(IClientRequest) +class Request: + """ + A L{Request} instance describes an HTTP request to be sent to an HTTP + server. + + @ivar method: See L{__init__}. + @ivar uri: See L{__init__}. + @ivar headers: See L{__init__}. + @ivar bodyProducer: See L{__init__}. + @ivar persistent: See L{__init__}. + + @ivar _parsedURI: Parsed I{URI} for the request, or L{None}. + @type _parsedURI: L{twisted.web.client.URI} or L{None} + """ + _log = Logger() + + def __init__(self, method, uri, headers, bodyProducer, persistent=False): + """ + @param method: The HTTP method for this request, ex: b'GET', b'HEAD', + b'POST', etc. + @type method: L{bytes} + + @param uri: The relative URI of the resource to request. For example, + C{b'/foo/bar?baz=quux'}. + @type uri: L{bytes} + + @param headers: Headers to be sent to the server. It is important to + note that this object does not create any implicit headers. So it + is up to the HTTP Client to add required headers such as 'Host'. + @type headers: L{twisted.web.http_headers.Headers} + + @param bodyProducer: L{None} or an L{IBodyProducer} provider which + produces the content body to send to the remote HTTP server. + + @param persistent: Set to C{True} when you use HTTP persistent + connection, defaults to C{False}. + @type persistent: L{bool} + """ + self.method = _ensureValidMethod(method) + self.uri = _ensureValidURI(uri) + self.headers = headers + self.bodyProducer = bodyProducer + self.persistent = persistent + self._parsedURI = None + + + @classmethod + def _construct(cls, method, uri, headers, bodyProducer, persistent=False, + parsedURI=None): + """ + Private constructor. + + @param method: See L{__init__}. + @param uri: See L{__init__}. + @param headers: See L{__init__}. + @param bodyProducer: See L{__init__}. + @param persistent: See L{__init__}. + @param parsedURI: See L{Request._parsedURI}. + + @return: L{Request} instance. + """ + request = cls(method, uri, headers, bodyProducer, persistent) + request._parsedURI = parsedURI + return request + + + @property + def absoluteURI(self): + """ + The absolute URI of the request as C{bytes}, or L{None} if the + absolute URI cannot be determined. + """ + return getattr(self._parsedURI, 'toBytes', lambda: None)() + + + def _writeHeaders(self, transport, TEorCL): + hosts = self.headers.getRawHeaders(b'host', ()) + if len(hosts) != 1: + raise BadHeaders(u"Exactly one Host header required") + + # In the future, having the protocol version be a parameter to this + # method would probably be good. It would be nice if this method + # weren't limited to issuing HTTP/1.1 requests. + requestLines = [] + requestLines.append( + b' '.join( + [ + _ensureValidMethod(self.method), + _ensureValidURI(self.uri), + b'HTTP/1.1\r\n', + ] + ), + ) + if not self.persistent: + requestLines.append(b'Connection: close\r\n') + if TEorCL is not None: + requestLines.append(TEorCL) + for name, values in self.headers.getAllRawHeaders(): + requestLines.extend([name + b': ' + v + b'\r\n' for v in values]) + requestLines.append(b'\r\n') + transport.writeSequence(requestLines) + + + def _writeToBodyProducerChunked(self, transport): + """ + Write this request to the given transport using chunked + transfer-encoding to frame the body. + + @param transport: See L{writeTo}. + @return: See L{writeTo}. + """ + self._writeHeaders(transport, b'Transfer-Encoding: chunked\r\n') + encoder = ChunkedEncoder(transport) + encoder.registerProducer(self.bodyProducer, True) + d = self.bodyProducer.startProducing(encoder) + + def cbProduced(ignored): + encoder.unregisterProducer() + def ebProduced(err): + encoder._allowNoMoreWrites() + # Don't call the encoder's unregisterProducer because it will write + # a zero-length chunk. This would indicate to the server that the + # request body is complete. There was an error, though, so we + # don't want to do that. + transport.unregisterProducer() + return err + d.addCallbacks(cbProduced, ebProduced) + return d + + + def _writeToBodyProducerContentLength(self, transport): + """ + Write this request to the given transport using content-length to frame + the body. + + @param transport: See L{writeTo}. + @return: See L{writeTo}. + """ + self._writeHeaders( + transport, + networkString( + 'Content-Length: %d\r\n' % (self.bodyProducer.length,))) + + # This Deferred is used to signal an error in the data written to the + # encoder below. It can only errback and it will only do so before too + # many bytes have been written to the encoder and before the producer + # Deferred fires. + finishedConsuming = Deferred() + + # This makes sure the producer writes the correct number of bytes for + # the request body. + encoder = LengthEnforcingConsumer( + self.bodyProducer, transport, finishedConsuming) + + transport.registerProducer(self.bodyProducer, True) + + finishedProducing = self.bodyProducer.startProducing(encoder) + + def combine(consuming, producing): + # This Deferred is returned and will be fired when the first of + # consuming or producing fires. If it's cancelled, forward that + # cancellation to the producer. + def cancelConsuming(ign): + finishedProducing.cancel() + ultimate = Deferred(cancelConsuming) + + # Keep track of what has happened so far. This initially + # contains None, then an integer uniquely identifying what + # sequence of events happened. See the callbacks and errbacks + # defined below for the meaning of each value. + state = [None] + + def ebConsuming(err): + if state == [None]: + # The consuming Deferred failed first. This means the + # overall writeTo Deferred is going to errback now. The + # producing Deferred should not fire later (because the + # consumer should have called stopProducing on the + # producer), but if it does, a callback will be ignored + # and an errback will be logged. + state[0] = 1 + ultimate.errback(err) + else: + # The consuming Deferred errbacked after the producing + # Deferred fired. This really shouldn't ever happen. + # If it does, I goofed. Log the error anyway, just so + # there's a chance someone might notice and complain. + self._log.failure( + u"Buggy state machine in {request}/[{state}]: " + u"ebConsuming called", + failure=err, + request=repr(self), + state=state[0] + ) + + def cbProducing(result): + if state == [None]: + # The producing Deferred succeeded first. Nothing will + # ever happen to the consuming Deferred. Tell the + # encoder we're done so it can check what the producer + # wrote and make sure it was right. + state[0] = 2 + try: + encoder._noMoreWritesExpected() + except: + # Fail the overall writeTo Deferred - something the + # producer did was wrong. + ultimate.errback() + else: + # Success - succeed the overall writeTo Deferred. + ultimate.callback(None) + # Otherwise, the consuming Deferred already errbacked. The + # producing Deferred wasn't supposed to fire, but it did + # anyway. It's buggy, but there's not really anything to be + # done about it. Just ignore this result. + + def ebProducing(err): + if state == [None]: + # The producing Deferred failed first. This means the + # overall writeTo Deferred is going to errback now. + # Tell the encoder that we're done so it knows to reject + # further writes from the producer (which should not + # happen, but the producer may be buggy). + state[0] = 3 + encoder._allowNoMoreWrites() + ultimate.errback(err) + else: + # The producing Deferred failed after the consuming + # Deferred failed. It shouldn't have, so it's buggy. + # Log the exception in case anyone who can fix the code + # is watching. + self._log.failure(u"Producer is buggy", failure=err) + + consuming.addErrback(ebConsuming) + producing.addCallbacks(cbProducing, ebProducing) + + return ultimate + + d = combine(finishedConsuming, finishedProducing) + def f(passthrough): + # Regardless of what happens with the overall Deferred, once it + # fires, the producer registered way up above the definition of + # combine should be unregistered. + transport.unregisterProducer() + return passthrough + d.addBoth(f) + return d + + + def _writeToEmptyBodyContentLength(self, transport): + """ + Write this request to the given transport using content-length to frame + the (empty) body. + + @param transport: See L{writeTo}. + @return: See L{writeTo}. + """ + self._writeHeaders(transport, b"Content-Length: 0\r\n") + return succeed(None) + + + def writeTo(self, transport): + """ + Format this L{Request} as an HTTP/1.1 request and write it to the given + transport. If bodyProducer is not None, it will be associated with an + L{IConsumer}. + + @param transport: The transport to which to write. + @type transport: L{twisted.internet.interfaces.ITransport} provider + + @return: A L{Deferred} which fires with L{None} when the request has + been completely written to the transport or with a L{Failure} if + there is any problem generating the request bytes. + """ + if self.bodyProducer is None: + # If the method semantics anticipate a body, include a + # Content-Length even if it is 0. + # https://tools.ietf.org/html/rfc7230#section-3.3.2 + if self.method in (b"PUT", b"POST"): + self._writeToEmptyBodyContentLength(transport) + else: + self._writeHeaders(transport, None) + elif self.bodyProducer.length is UNKNOWN_LENGTH: + return self._writeToBodyProducerChunked(transport) + else: + return self._writeToBodyProducerContentLength(transport) + + + def stopWriting(self): + """ + Stop writing this request to the transport. This can only be called + after C{writeTo} and before the L{Deferred} returned by C{writeTo} + fires. It should cancel any asynchronous task started by C{writeTo}. + The L{Deferred} returned by C{writeTo} need not be fired if this method + is called. + """ + # If bodyProducer is None, then the Deferred returned by writeTo has + # fired already and this method cannot be called. + _callAppFunction(self.bodyProducer.stopProducing) + + + +class LengthEnforcingConsumer: + """ + An L{IConsumer} proxy which enforces an exact length requirement on the + total data written to it. + + @ivar _length: The number of bytes remaining to be written. + + @ivar _producer: The L{IBodyProducer} which is writing to this + consumer. + + @ivar _consumer: The consumer to which at most C{_length} bytes will be + forwarded. + + @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too + many bytes are written to this consumer. + """ + def __init__(self, producer, consumer, finished): + self._length = producer.length + self._producer = producer + self._consumer = consumer + self._finished = finished + + + def _allowNoMoreWrites(self): + """ + Indicate that no additional writes are allowed. Attempts to write + after calling this method will be met with an exception. + """ + self._finished = None + + + def write(self, bytes): + """ + Write C{bytes} to the underlying consumer unless + C{_noMoreWritesExpected} has been called or there are/have been too + many bytes. + """ + if self._finished is None: + # No writes are supposed to happen any more. Try to convince the + # calling code to stop calling this method by calling its + # stopProducing method and then throwing an exception at it. This + # exception isn't documented as part of the API because you're + # never supposed to expect it: only buggy code will ever receive + # it. + self._producer.stopProducing() + raise ExcessWrite() + + if len(bytes) <= self._length: + self._length -= len(bytes) + self._consumer.write(bytes) + else: + # No synchronous exception is raised in *this* error path because + # we still have _finished which we can use to report the error to a + # better place than the direct caller of this method (some + # arbitrary application code). + _callAppFunction(self._producer.stopProducing) + self._finished.errback(WrongBodyLength(u"too many bytes written")) + self._allowNoMoreWrites() + + + def _noMoreWritesExpected(self): + """ + Called to indicate no more bytes will be written to this consumer. + Check to see that the correct number have been written. + + @raise WrongBodyLength: If not enough bytes have been written. + """ + if self._finished is not None: + self._allowNoMoreWrites() + if self._length: + raise WrongBodyLength(u"too few bytes written") + + + +def makeStatefulDispatcher(name, template): + """ + Given a I{dispatch} name and a function, return a function which can be + used as a method and which, when called, will call another method defined + on the instance and return the result. The other method which is called is + determined by the value of the C{_state} attribute of the instance. + + @param name: A string which is used to construct the name of the subsidiary + method to invoke. The subsidiary method is named like C{'_%s_%s' % + (name, _state)}. + + @param template: A function object which is used to give the returned + function a docstring. + + @return: The dispatcher function. + """ + def dispatcher(self, *args, **kwargs): + func = getattr(self, '_' + name + '_' + self._state, None) + if func is None: + raise RuntimeError( + u"%r has no %s method in state %s" % (self, name, self._state)) + return func(*args, **kwargs) + dispatcher.__doc__ = template.__doc__ + return dispatcher + + + +# This proxy class is used only in the private constructor of the Response +# class below, in order to prevent users relying on any property of the +# concrete request object: they can only use what is provided by +# IClientRequest. +_ClientRequestProxy = proxyForInterface(IClientRequest) + + + +@implementer(IResponse) +class Response: + """ + A L{Response} instance describes an HTTP response received from an HTTP + server. + + L{Response} should not be subclassed or instantiated. + + @ivar _transport: See L{__init__}. + + @ivar _bodyProtocol: The L{IProtocol} provider to which the body is + delivered. L{None} before one has been registered with + C{deliverBody}. + + @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived} + before C{deliverBody} is called. L{None} afterwards. + + @ivar _state: Indicates what state this L{Response} instance is in, + particularly with respect to delivering bytes from the response body + to an application-supplied protocol object. This may be one of + C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'}, + with the following meanings: + + - INITIAL: This is the state L{Response} objects start in. No + protocol has yet been provided and the underlying transport may + still have bytes to deliver to it. + + - DEFERRED_CLOSE: If the underlying transport indicates all bytes + have been delivered but no application-provided protocol is yet + available, the L{Response} moves to this state. Data is + buffered and waiting for a protocol to be delivered to. + + - CONNECTED: If a protocol is provided when the state is INITIAL, + the L{Response} moves to this state. Any buffered data is + delivered and any data which arrives from the transport + subsequently is given directly to the protocol. + + - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state, + the L{Response} moves to this state after delivering all + buffered data to the protocol. Otherwise, if the L{Response} is + in the CONNECTED state, if the transport indicates there is no + more data, the L{Response} moves to this state. Nothing else + can happen once the L{Response} is in this state. + @type _state: C{str} + """ + + length = UNKNOWN_LENGTH + + _bodyProtocol = None + _bodyFinished = False + + def __init__(self, version, code, phrase, headers, _transport): + """ + @param version: HTTP version components protocol, major, minor. E.g. + C{(b'HTTP', 1, 1)} to mean C{b'HTTP/1.1'}. + + @param code: HTTP status code. + @type code: L{int} + + @param phrase: HTTP reason phrase, intended to give a short description + of the HTTP status code. + + @param headers: HTTP response headers. + @type headers: L{twisted.web.http_headers.Headers} + + @param _transport: The transport which is delivering this response. + """ + self.version = version + self.code = code + self.phrase = phrase + self.headers = headers + self._transport = _transport + self._bodyBuffer = [] + self._state = 'INITIAL' + self.request = None + self.previousResponse = None + + + @classmethod + def _construct(cls, version, code, phrase, headers, _transport, request): + """ + Private constructor. + + @param version: See L{__init__}. + @param code: See L{__init__}. + @param phrase: See L{__init__}. + @param headers: See L{__init__}. + @param _transport: See L{__init__}. + @param request: See L{IResponse.request}. + + @return: L{Response} instance. + """ + response = Response(version, code, phrase, headers, _transport) + response.request = _ClientRequestProxy(request) + return response + + + def setPreviousResponse(self, previousResponse): + self.previousResponse = previousResponse + + + def deliverBody(self, protocol): + """ + Dispatch the given L{IProtocol} depending of the current state of the + response. + """ + deliverBody = makeStatefulDispatcher('deliverBody', deliverBody) + + + def _deliverBody_INITIAL(self, protocol): + """ + Deliver any buffered data to C{protocol} and prepare to deliver any + future data to it. Move to the C{'CONNECTED'} state. + """ + protocol.makeConnection(self._transport) + self._bodyProtocol = protocol + for data in self._bodyBuffer: + self._bodyProtocol.dataReceived(data) + self._bodyBuffer = None + + self._state = 'CONNECTED' + + # Now that there's a protocol to consume the body, resume the + # transport. It was previously paused by HTTPClientParser to avoid + # reading too much data before it could be handled. We need to do this + # after we transition our state as it may recursively lead to more data + # being delivered, or even the body completing. + self._transport.resumeProducing() + + + def _deliverBody_CONNECTED(self, protocol): + """ + It is invalid to attempt to deliver data to a protocol when it is + already being delivered to another protocol. + """ + raise RuntimeError( + u"Response already has protocol %r, cannot deliverBody " + u"again" % (self._bodyProtocol,)) + + + def _deliverBody_DEFERRED_CLOSE(self, protocol): + """ + Deliver any buffered data to C{protocol} and then disconnect the + protocol. Move to the C{'FINISHED'} state. + """ + # Unlike _deliverBody_INITIAL, there is no need to resume the + # transport here because all of the response data has been received + # already. Some higher level code may want to resume the transport if + # that code expects further data to be received over it. + + protocol.makeConnection(self._transport) + + for data in self._bodyBuffer: + protocol.dataReceived(data) + self._bodyBuffer = None + protocol.connectionLost(self._reason) + self._state = 'FINISHED' + + + def _deliverBody_FINISHED(self, protocol): + """ + It is invalid to attempt to deliver data to a protocol after the + response body has been delivered to another protocol. + """ + raise RuntimeError( + u"Response already finished, cannot deliverBody now.") + + + def _bodyDataReceived(self, data): + """ + Called by HTTPClientParser with chunks of data from the response body. + They will be buffered or delivered to the protocol passed to + deliverBody. + """ + _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived', + _bodyDataReceived) + + + def _bodyDataReceived_INITIAL(self, data): + """ + Buffer any data received for later delivery to a protocol passed to + C{deliverBody}. + + Little or no data should be buffered by this method, since the + transport has been paused and will not be resumed until a protocol + is supplied. + """ + self._bodyBuffer.append(data) + + + def _bodyDataReceived_CONNECTED(self, data): + """ + Deliver any data received to the protocol to which this L{Response} + is connected. + """ + self._bodyProtocol.dataReceived(data) + + + def _bodyDataReceived_DEFERRED_CLOSE(self, data): + """ + It is invalid for data to be delivered after it has been indicated + that the response body has been completely delivered. + """ + raise RuntimeError(u"Cannot receive body data after _bodyDataFinished") + + + def _bodyDataReceived_FINISHED(self, data): + """ + It is invalid for data to be delivered after the response body has + been delivered to a protocol. + """ + raise RuntimeError(u"Cannot receive body data after " + u"protocol disconnected") + + + def _bodyDataFinished(self, reason=None): + """ + Called by HTTPClientParser when no more body data is available. If the + optional reason is supplied, this indicates a problem or potential + problem receiving all of the response body. + """ + _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished', + _bodyDataFinished) + + + def _bodyDataFinished_INITIAL(self, reason=None): + """ + Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to + which to deliver the response body. + """ + self._state = 'DEFERRED_CLOSE' + if reason is None: + reason = Failure(ResponseDone(u"Response body fully received")) + self._reason = reason + + + def _bodyDataFinished_CONNECTED(self, reason=None): + """ + Disconnect the protocol and move to the C{'FINISHED'} state. + """ + if reason is None: + reason = Failure(ResponseDone(u"Response body fully received")) + self._bodyProtocol.connectionLost(reason) + self._bodyProtocol = None + self._state = 'FINISHED' + + + def _bodyDataFinished_DEFERRED_CLOSE(self): + """ + It is invalid to attempt to notify the L{Response} of the end of the + response body data more than once. + """ + raise RuntimeError(u"Cannot finish body data more than once") + + + def _bodyDataFinished_FINISHED(self): + """ + It is invalid to attempt to notify the L{Response} of the end of the + response body data more than once. + """ + raise RuntimeError(u"Cannot finish body data after " + u"protocol disconnected") + + + +@implementer(IConsumer) +class ChunkedEncoder: + """ + Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol} + for streaming request bodies to the server. + """ + + def __init__(self, transport): + self.transport = transport + + + def _allowNoMoreWrites(self): + """ + Indicate that no additional writes are allowed. Attempts to write + after calling this method will be met with an exception. + """ + self.transport = None + + + def registerProducer(self, producer, streaming): + """ + Register the given producer with C{self.transport}. + """ + self.transport.registerProducer(producer, streaming) + + + def write(self, data): + """ + Write the given request body bytes to the transport using chunked + encoding. + + @type data: C{bytes} + """ + if self.transport is None: + raise ExcessWrite() + self.transport.writeSequence((networkString("%x\r\n" % len(data)), + data, b"\r\n")) + + + def unregisterProducer(self): + """ + Indicate that the request body is complete and finish the request. + """ + self.write(b'') + self.transport.unregisterProducer() + self._allowNoMoreWrites() + + + +@implementer(IPushProducer) +class TransportProxyProducer: + """ + An L{twisted.internet.interfaces.IPushProducer} implementation which + wraps another such thing and proxies calls to it until it is told to stop. + + @ivar _producer: The wrapped L{twisted.internet.interfaces.IPushProducer} + provider or L{None} after this proxy has been stopped. + """ + + # LineReceiver uses this undocumented attribute of transports to decide + # when to stop calling lineReceived or rawDataReceived (if it finds it to + # be true, it doesn't bother to deliver any more data). Set disconnecting + # to False here and never change it to true so that all data is always + # delivered to us and so that LineReceiver doesn't fail with an + # AttributeError. + disconnecting = False + + def __init__(self, producer): + self._producer = producer + + + def stopProxying(self): + """ + Stop forwarding calls of L{twisted.internet.interfaces.IPushProducer} + methods to the underlying L{twisted.internet.interfaces.IPushProducer} + provider. + """ + self._producer = None + + + def stopProducing(self): + """ + Proxy the stoppage to the underlying producer, unless this proxy has + been stopped. + """ + if self._producer is not None: + self._producer.stopProducing() + + + def resumeProducing(self): + """ + Proxy the resumption to the underlying producer, unless this proxy has + been stopped. + """ + if self._producer is not None: + self._producer.resumeProducing() + + + def pauseProducing(self): + """ + Proxy the pause to the underlying producer, unless this proxy has been + stopped. + """ + if self._producer is not None: + self._producer.pauseProducing() + + + def loseConnection(self): + """ + Proxy the request to lose the connection to the underlying producer, + unless this proxy has been stopped. + """ + if self._producer is not None: + self._producer.loseConnection() + + + +class HTTP11ClientProtocol(Protocol): + """ + L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client + protocol. It supports as few features as possible. + + @ivar _parser: After a request is issued, the L{HTTPClientParser} to + which received data making up the response to that request is + delivered. + + @ivar _finishedRequest: After a request is issued, the L{Deferred} which + will fire when a L{Response} object corresponding to that request is + available. This allows L{HTTP11ClientProtocol} to fail the request + if there is a connection or parsing problem. + + @ivar _currentRequest: After a request is issued, the L{Request} + instance used to make that request. This allows + L{HTTP11ClientProtocol} to stop request generation if necessary (for + example, if the connection is lost). + + @ivar _transportProxy: After a request is issued, the + L{TransportProxyProducer} to which C{_parser} is connected. This + allows C{_parser} to pause and resume the transport in a way which + L{HTTP11ClientProtocol} can exert some control over. + + @ivar _responseDeferred: After a request is issued, the L{Deferred} from + C{_parser} which will fire with a L{Response} when one has been + received. This is eventually chained with C{_finishedRequest}, but + only in certain cases to avoid double firing that Deferred. + + @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance + is in with respect to transmission of a request and reception of a + response. This may be one of the following strings: + + - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances + start in. Nothing is happening: no request is being sent and no + response is being received or expected. + + - TRANSMITTING: When a request is made (via L{request}), the + instance moves to this state. L{Request.writeTo} has been used + to start to send a request but it has not yet finished. + + - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a + complete response but the request has not yet been fully sent + yet. The instance will remain in this state until the request + is fully sent. + + - GENERATION_FAILED: There was an error while the request. The + request was not fully sent to the network. + + - WAITING: The request was fully sent to the network. The + instance is now waiting for the response to be fully received. + + - ABORTING: Application code has requested that the HTTP connection + be aborted. + + - CONNECTION_LOST: The connection has been lost. + @type _state: C{str} + + @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when + the connection is lost. + """ + _state = 'QUIESCENT' + _parser = None + _finishedRequest = None + _currentRequest = None + _transportProxy = None + _responseDeferred = None + _log = Logger() + + + def __init__(self, quiescentCallback=lambda c: None): + self._quiescentCallback = quiescentCallback + self._abortDeferreds = [] + + + @property + def state(self): + return self._state + + + def request(self, request): + """ + Issue C{request} over C{self.transport} and return a L{Deferred} which + will fire with a L{Response} instance or an error. + + @param request: The object defining the parameters of the request to + issue. + @type request: L{Request} + + @rtype: L{Deferred} + @return: The deferred may errback with L{RequestGenerationFailed} if + the request was not fully written to the transport due to a local + error. It may errback with L{RequestTransmissionFailed} if it was + not fully written to the transport due to a network error. It may + errback with L{ResponseFailed} if the request was sent (not + necessarily received) but some or all of the response was lost. It + may errback with L{RequestNotSent} if it is not possible to send + any more requests using this L{HTTP11ClientProtocol}. + """ + if self._state != 'QUIESCENT': + return fail(RequestNotSent()) + + self._state = 'TRANSMITTING' + _requestDeferred = maybeDeferred(request.writeTo, self.transport) + + def cancelRequest(ign): + # Explicitly cancel the request's deferred if it's still trying to + # write when this request is cancelled. + if self._state in ( + 'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'): + _requestDeferred.cancel() + else: + self.transport.abortConnection() + self._disconnectParser(Failure(CancelledError())) + self._finishedRequest = Deferred(cancelRequest) + + # Keep track of the Request object in case we need to call stopWriting + # on it. + self._currentRequest = request + + self._transportProxy = TransportProxyProducer(self.transport) + self._parser = HTTPClientParser(request, self._finishResponse) + self._parser.makeConnection(self._transportProxy) + self._responseDeferred = self._parser._responseDeferred + + def cbRequestWritten(ignored): + if self._state == 'TRANSMITTING': + self._state = 'WAITING' + self._responseDeferred.chainDeferred(self._finishedRequest) + + def ebRequestWriting(err): + if self._state == 'TRANSMITTING': + self._state = 'GENERATION_FAILED' + self.transport.abortConnection() + self._finishedRequest.errback( + Failure(RequestGenerationFailed([err]))) + else: + self._log.failure( + u'Error writing request, but not in valid state ' + u'to finalize request: {state}', + failure=err, + state=self._state + ) + + _requestDeferred.addCallbacks(cbRequestWritten, ebRequestWriting) + + return self._finishedRequest + + + def _finishResponse(self, rest): + """ + Called by an L{HTTPClientParser} to indicate that it has parsed a + complete response. + + @param rest: A C{bytes} giving any trailing bytes which were given to + the L{HTTPClientParser} which were not part of the response it + was parsing. + """ + _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse) + + + def _finishResponse_WAITING(self, rest): + # Currently the rest parameter is ignored. Don't forget to use it if + # we ever add support for pipelining. And maybe check what trailers + # mean. + if self._state == 'WAITING': + self._state = 'QUIESCENT' + else: + # The server sent the entire response before we could send the + # whole request. That sucks. Oh well. Fire the request() + # Deferred with the response. But first, make sure that if the + # request does ever finish being written that it won't try to fire + # that Deferred. + self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE' + self._responseDeferred.chainDeferred(self._finishedRequest) + + # This will happen if we're being called due to connection being lost; + # if so, no need to disconnect parser again, or to call + # _quiescentCallback. + if self._parser is None: + return + + reason = ConnectionDone(u"synthetic!") + connHeaders = self._parser.connHeaders.getRawHeaders(b'connection', ()) + if ((b'close' in connHeaders) or self._state != "QUIESCENT" or + not self._currentRequest.persistent): + self._giveUp(Failure(reason)) + else: + # Just in case we had paused the transport, resume it before + # considering it quiescent again. + self.transport.resumeProducing() + + # We call the quiescent callback first, to ensure connection gets + # added back to connection pool before we finish the request. + try: + self._quiescentCallback(self) + except: + # If callback throws exception, just log it and disconnect; + # keeping persistent connections around is an optimisation: + self._log.failure('') + self.transport.loseConnection() + self._disconnectParser(reason) + + + _finishResponse_TRANSMITTING = _finishResponse_WAITING + + + def _disconnectParser(self, reason): + """ + If there is still a parser, call its C{connectionLost} method with the + given reason. If there is not, do nothing. + + @type reason: L{Failure} + """ + if self._parser is not None: + parser = self._parser + self._parser = None + self._currentRequest = None + self._finishedRequest = None + self._responseDeferred = None + + # The parser is no longer allowed to do anything to the real + # transport. Stop proxying from the parser's transport to the real + # transport before telling the parser it's done so that it can't do + # anything. + self._transportProxy.stopProxying() + self._transportProxy = None + parser.connectionLost(reason) + + + def _giveUp(self, reason): + """ + Lose the underlying connection and disconnect the parser with the given + L{Failure}. + + Use this method instead of calling the transport's loseConnection + method directly otherwise random things will break. + """ + self.transport.loseConnection() + self._disconnectParser(reason) + + + def dataReceived(self, bytes): + """ + Handle some stuff from some place. + """ + try: + self._parser.dataReceived(bytes) + except: + self._giveUp(Failure()) + + + def connectionLost(self, reason): + """ + The underlying transport went away. If appropriate, notify the parser + object. + """ + connectionLost = makeStatefulDispatcher('connectionLost', connectionLost) + + + def _connectionLost_QUIESCENT(self, reason): + """ + Nothing is currently happening. Move to the C{'CONNECTION_LOST'} + state but otherwise do nothing. + """ + self._state = 'CONNECTION_LOST' + + + def _connectionLost_GENERATION_FAILED(self, reason): + """ + The connection was in an inconsistent state. Move to the + C{'CONNECTION_LOST'} state but otherwise do nothing. + """ + self._state = 'CONNECTION_LOST' + + + def _connectionLost_TRANSMITTING(self, reason): + """ + Fail the L{Deferred} for the current request, notify the request + object that it does not need to continue transmitting itself, and + move to the C{'CONNECTION_LOST'} state. + """ + self._state = 'CONNECTION_LOST' + self._finishedRequest.errback( + Failure(RequestTransmissionFailed([reason]))) + del self._finishedRequest + + # Tell the request that it should stop bothering now. + self._currentRequest.stopWriting() + + + def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason): + """ + Move to the C{'CONNECTION_LOST'} state. + """ + self._state = 'CONNECTION_LOST' + + + def _connectionLost_WAITING(self, reason): + """ + Disconnect the response parser so that it can propagate the event as + necessary (for example, to call an application protocol's + C{connectionLost} method, or to fail a request L{Deferred}) and move + to the C{'CONNECTION_LOST'} state. + """ + self._disconnectParser(reason) + self._state = 'CONNECTION_LOST' + + + def _connectionLost_ABORTING(self, reason): + """ + Disconnect the response parser with a L{ConnectionAborted} failure, and + move to the C{'CONNECTION_LOST'} state. + """ + self._disconnectParser(Failure(ConnectionAborted())) + self._state = 'CONNECTION_LOST' + for d in self._abortDeferreds: + d.callback(None) + self._abortDeferreds = [] + + + def abort(self): + """ + Close the connection and cause all outstanding L{request} L{Deferred}s + to fire with an error. + """ + if self._state == "CONNECTION_LOST": + return succeed(None) + self.transport.loseConnection() + self._state = 'ABORTING' + d = Deferred() + self._abortDeferreds.append(d) + return d diff --git a/contrib/python/Twisted/py2/twisted/web/_responses.py b/contrib/python/Twisted/py2/twisted/web/_responses.py new file mode 100644 index 00000000000..4f8c1cdea46 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_responses.py @@ -0,0 +1,114 @@ +# -*- test-case-name: twisted.web.test.test_http -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HTTP response code definitions. +""" + +from __future__ import division, absolute_import + +_CONTINUE = 100 +SWITCHING = 101 + +OK = 200 +CREATED = 201 +ACCEPTED = 202 +NON_AUTHORITATIVE_INFORMATION = 203 +NO_CONTENT = 204 +RESET_CONTENT = 205 +PARTIAL_CONTENT = 206 +MULTI_STATUS = 207 + +MULTIPLE_CHOICE = 300 +MOVED_PERMANENTLY = 301 +FOUND = 302 +SEE_OTHER = 303 +NOT_MODIFIED = 304 +USE_PROXY = 305 +TEMPORARY_REDIRECT = 307 + +BAD_REQUEST = 400 +UNAUTHORIZED = 401 +PAYMENT_REQUIRED = 402 +FORBIDDEN = 403 +NOT_FOUND = 404 +NOT_ALLOWED = 405 +NOT_ACCEPTABLE = 406 +PROXY_AUTH_REQUIRED = 407 +REQUEST_TIMEOUT = 408 +CONFLICT = 409 +GONE = 410 +LENGTH_REQUIRED = 411 +PRECONDITION_FAILED = 412 +REQUEST_ENTITY_TOO_LARGE = 413 +REQUEST_URI_TOO_LONG = 414 +UNSUPPORTED_MEDIA_TYPE = 415 +REQUESTED_RANGE_NOT_SATISFIABLE = 416 +EXPECTATION_FAILED = 417 + +INTERNAL_SERVER_ERROR = 500 +NOT_IMPLEMENTED = 501 +BAD_GATEWAY = 502 +SERVICE_UNAVAILABLE = 503 +GATEWAY_TIMEOUT = 504 +HTTP_VERSION_NOT_SUPPORTED = 505 +INSUFFICIENT_STORAGE_SPACE = 507 +NOT_EXTENDED = 510 + +RESPONSES = { + # 100 + _CONTINUE: b"Continue", + SWITCHING: b"Switching Protocols", + + # 200 + OK: b"OK", + CREATED: b"Created", + ACCEPTED: b"Accepted", + NON_AUTHORITATIVE_INFORMATION: b"Non-Authoritative Information", + NO_CONTENT: b"No Content", + RESET_CONTENT: b"Reset Content.", + PARTIAL_CONTENT: b"Partial Content", + MULTI_STATUS: b"Multi-Status", + + # 300 + MULTIPLE_CHOICE: b"Multiple Choices", + MOVED_PERMANENTLY: b"Moved Permanently", + FOUND: b"Found", + SEE_OTHER: b"See Other", + NOT_MODIFIED: b"Not Modified", + USE_PROXY: b"Use Proxy", + # 306 not defined?? + TEMPORARY_REDIRECT: b"Temporary Redirect", + + # 400 + BAD_REQUEST: b"Bad Request", + UNAUTHORIZED: b"Unauthorized", + PAYMENT_REQUIRED: b"Payment Required", + FORBIDDEN: b"Forbidden", + NOT_FOUND: b"Not Found", + NOT_ALLOWED: b"Method Not Allowed", + NOT_ACCEPTABLE: b"Not Acceptable", + PROXY_AUTH_REQUIRED: b"Proxy Authentication Required", + REQUEST_TIMEOUT: b"Request Time-out", + CONFLICT: b"Conflict", + GONE: b"Gone", + LENGTH_REQUIRED: b"Length Required", + PRECONDITION_FAILED: b"Precondition Failed", + REQUEST_ENTITY_TOO_LARGE: b"Request Entity Too Large", + REQUEST_URI_TOO_LONG: b"Request-URI Too Long", + UNSUPPORTED_MEDIA_TYPE: b"Unsupported Media Type", + REQUESTED_RANGE_NOT_SATISFIABLE: b"Requested Range not satisfiable", + EXPECTATION_FAILED: b"Expectation Failed", + + # 500 + INTERNAL_SERVER_ERROR: b"Internal Server Error", + NOT_IMPLEMENTED: b"Not Implemented", + BAD_GATEWAY: b"Bad Gateway", + SERVICE_UNAVAILABLE: b"Service Unavailable", + GATEWAY_TIMEOUT: b"Gateway Time-out", + HTTP_VERSION_NOT_SUPPORTED: b"HTTP Version not supported", + INSUFFICIENT_STORAGE_SPACE: b"Insufficient Storage Space", + NOT_EXTENDED: b"Not Extended" + } + diff --git a/contrib/python/Twisted/py2/twisted/web/_stan.py b/contrib/python/Twisted/py2/twisted/web/_stan.py new file mode 100644 index 00000000000..033a52c6529 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/_stan.py @@ -0,0 +1,330 @@ +# -*- test-case-name: twisted.web.test.test_stan -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An s-expression-like syntax for expressing xml in pure python. + +Stan tags allow you to build XML documents using Python. + +Stan is a DOM, or Document Object Model, implemented using basic Python types +and functions called "flatteners". A flattener is a function that knows how to +turn an object of a specific type into something that is closer to an HTML +string. Stan differs from the W3C DOM by not being as cumbersome and heavy +weight. Since the object model is built using simple python types such as lists, +strings, and dictionaries, the API is simpler and constructing a DOM less +cumbersome. + +@var voidElements: the names of HTML 'U{void + elements}'; + those which can't have contents and can therefore be self-closing in the + output. +""" + +from __future__ import absolute_import, division + +from twisted.python.compat import iteritems + + + +class slot(object): + """ + Marker for markup insertion in a template. + + @type name: C{str} + @ivar name: The name of this slot. The key which must be used in + L{Tag.fillSlots} to fill it. + + @type children: C{list} + @ivar children: The L{Tag} objects included in this L{slot}'s template. + + @type default: anything flattenable, or L{None} + @ivar default: The default contents of this slot, if it is left unfilled. + If this is L{None}, an L{UnfilledSlot} will be raised, rather than + L{None} actually being used. + + @type filename: C{str} or L{None} + @ivar filename: The name of the XML file from which this tag was parsed. + If it was not parsed from an XML file, L{None}. + + @type lineNumber: C{int} or L{None} + @ivar lineNumber: The line number on which this tag was encountered in the + XML file from which it was parsed. If it was not parsed from an XML + file, L{None}. + + @type columnNumber: C{int} or L{None} + @ivar columnNumber: The column number at which this tag was encountered in + the XML file from which it was parsed. If it was not parsed from an + XML file, L{None}. + """ + + def __init__(self, name, default=None, filename=None, lineNumber=None, + columnNumber=None): + self.name = name + self.children = [] + self.default = default + self.filename = filename + self.lineNumber = lineNumber + self.columnNumber = columnNumber + + + def __repr__(self): + return "slot(%r)" % (self.name,) + + + +class Tag(object): + """ + A L{Tag} represents an XML tags with a tag name, attributes, and children. + A L{Tag} can be constructed using the special L{twisted.web.template.tags} + object, or it may be constructed directly with a tag name. L{Tag}s have a + special method, C{__call__}, which makes representing trees of XML natural + using pure python syntax. + + @ivar tagName: The name of the represented element. For a tag like + C{
}, this would be C{"div"}. + @type tagName: C{str} + + @ivar attributes: The attributes of the element. + @type attributes: C{dict} mapping C{str} to renderable objects. + + @ivar children: The child L{Tag}s of this C{Tag}. + @type children: C{list} of renderable objects. + + @ivar render: The name of the render method to use for this L{Tag}. This + name will be looked up at render time by the + L{twisted.web.template.Element} doing the rendering, via + L{twisted.web.template.Element.lookupRenderMethod}, to determine which + method to call. + @type render: C{str} + + @type filename: C{str} or L{None} + @ivar filename: The name of the XML file from which this tag was parsed. + If it was not parsed from an XML file, L{None}. + + @type lineNumber: C{int} or L{None} + @ivar lineNumber: The line number on which this tag was encountered in the + XML file from which it was parsed. If it was not parsed from an XML + file, L{None}. + + @type columnNumber: C{int} or L{None} + @ivar columnNumber: The column number at which this tag was encountered in + the XML file from which it was parsed. If it was not parsed from an + XML file, L{None}. + + @type slotData: C{dict} or L{None} + @ivar slotData: The data which can fill slots. If present, a dictionary + mapping slot names to renderable values. The values in this dict might + be anything that can be present as the child of a L{Tag}; strings, + lists, L{Tag}s, generators, etc. + """ + + slotData = None + filename = None + lineNumber = None + columnNumber = None + + def __init__(self, tagName, attributes=None, children=None, render=None, + filename=None, lineNumber=None, columnNumber=None): + self.tagName = tagName + self.render = render + if attributes is None: + self.attributes = {} + else: + self.attributes = attributes + if children is None: + self.children = [] + else: + self.children = children + if filename is not None: + self.filename = filename + if lineNumber is not None: + self.lineNumber = lineNumber + if columnNumber is not None: + self.columnNumber = columnNumber + + + def fillSlots(self, **slots): + """ + Remember the slots provided at this position in the DOM. + + During the rendering of children of this node, slots with names in + C{slots} will be rendered as their corresponding values. + + @return: C{self}. This enables the idiom C{return tag.fillSlots(...)} in + renderers. + """ + if self.slotData is None: + self.slotData = {} + self.slotData.update(slots) + return self + + + def __call__(self, *children, **kw): + """ + Add children and change attributes on this tag. + + This is implemented using __call__ because it then allows the natural + syntax:: + + table(tr1, tr2, width="100%", height="50%", border="1") + + Children may be other tag instances, strings, functions, or any other + object which has a registered flatten. + + Attributes may be 'transparent' tag instances (so that + C{a(href=transparent(data="foo", render=myhrefrenderer))} works), + strings, functions, or any other object which has a registered + flattener. + + If the attribute is a python keyword, such as 'class', you can add an + underscore to the name, like 'class_'. + + There is one special keyword argument, 'render', which will be used as + the name of the renderer and saved as the 'render' attribute of this + instance, rather than the DOM 'render' attribute in the attributes + dictionary. + """ + self.children.extend(children) + + for k, v in iteritems(kw): + if k[-1] == '_': + k = k[:-1] + + if k == 'render': + self.render = v + else: + self.attributes[k] = v + return self + + + def _clone(self, obj, deep): + """ + Clone an arbitrary object; used by L{Tag.clone}. + + @param obj: an object with a clone method, a list or tuple, or something + which should be immutable. + + @param deep: whether to continue cloning child objects; i.e. the + contents of lists, the sub-tags within a tag. + + @return: a clone of C{obj}. + """ + if hasattr(obj, 'clone'): + return obj.clone(deep) + elif isinstance(obj, (list, tuple)): + return [self._clone(x, deep) for x in obj] + else: + return obj + + + def clone(self, deep=True): + """ + Return a clone of this tag. If deep is True, clone all of this tag's + children. Otherwise, just shallow copy the children list without copying + the children themselves. + """ + if deep: + newchildren = [self._clone(x, True) for x in self.children] + else: + newchildren = self.children[:] + newattrs = self.attributes.copy() + for key in newattrs.keys(): + newattrs[key] = self._clone(newattrs[key], True) + + newslotdata = None + if self.slotData: + newslotdata = self.slotData.copy() + for key in newslotdata: + newslotdata[key] = self._clone(newslotdata[key], True) + + newtag = Tag( + self.tagName, + attributes=newattrs, + children=newchildren, + render=self.render, + filename=self.filename, + lineNumber=self.lineNumber, + columnNumber=self.columnNumber) + newtag.slotData = newslotdata + + return newtag + + + def clear(self): + """ + Clear any existing children from this tag. + """ + self.children = [] + return self + + + def __repr__(self): + rstr = '' + if self.attributes: + rstr += ', attributes=%r' % self.attributes + if self.children: + rstr += ', children=%r' % self.children + return "Tag(%r%s)" % (self.tagName, rstr) + + + +voidElements = ('img', 'br', 'hr', 'base', 'meta', 'link', 'param', 'area', + 'input', 'col', 'basefont', 'isindex', 'frame', 'command', + 'embed', 'keygen', 'source', 'track', 'wbs') + + +class CDATA(object): + """ + A C{} block from a template. Given a separate representation in + the DOM so that they may be round-tripped through rendering without losing + information. + + @ivar data: The data between "C{}". + @type data: C{unicode} + """ + def __init__(self, data): + self.data = data + + + def __repr__(self): + return 'CDATA(%r)' % (self.data,) + + + +class Comment(object): + """ + A C{} comment from a template. Given a separate representation in + the DOM so that they may be round-tripped through rendering without losing + information. + + @ivar data: The data between "C{}". + @type data: C{unicode} + """ + + def __init__(self, data): + self.data = data + + + def __repr__(self): + return 'Comment(%r)' % (self.data,) + + + +class CharRef(object): + """ + A numeric character reference. Given a separate representation in the DOM + so that non-ASCII characters may be output as pure ASCII. + + @ivar ordinal: The ordinal value of the unicode character to which this is + object refers. + @type ordinal: C{int} + + @since: 12.0 + """ + def __init__(self, ordinal): + self.ordinal = ordinal + + + def __repr__(self): + return "CharRef(%d)" % (self.ordinal,) diff --git a/contrib/python/Twisted/py2/twisted/web/client.py b/contrib/python/Twisted/py2/twisted/web/client.py new file mode 100644 index 00000000000..7e4642ef308 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/client.py @@ -0,0 +1,2336 @@ +# -*- test-case-name: twisted.web.test.test_webclient,twisted.web.test.test_agent -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HTTP client. +""" + +from __future__ import division, absolute_import + +import os +import collections +import warnings + +try: + from urlparse import urlunparse, urljoin, urldefrag +except ImportError: + from urllib.parse import urljoin, urldefrag + from urllib.parse import urlunparse as _urlunparse + + def urlunparse(parts): + result = _urlunparse(tuple([p.decode("charmap") for p in parts])) + return result.encode("charmap") + +import zlib +from functools import wraps + +from zope.interface import implementer + +from twisted.python.compat import _PY3, networkString +from twisted.python.compat import nativeString, intToBytes, unicode, itervalues +from twisted.python.deprecate import deprecatedModuleAttribute, deprecated +from twisted.python.failure import Failure +from incremental import Version + +from twisted.web.iweb import IPolicyForHTTPS, IAgentEndpointFactory +from twisted.python.deprecate import getDeprecationWarningString +from twisted.web import http +from twisted.internet import defer, protocol, task, reactor +from twisted.internet.abstract import isIPv6Address +from twisted.internet.interfaces import IProtocol, IOpenSSLContextFactory +from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS +from twisted.python.util import InsensitiveDict +from twisted.python.components import proxyForInterface +from twisted.web import error +from twisted.web.iweb import UNKNOWN_LENGTH, IAgent, IBodyProducer, IResponse +from twisted.web.http_headers import Headers +from twisted.logger import Logger + +from twisted.web._newclient import _ensureValidURI, _ensureValidMethod + + + +class PartialDownloadError(error.Error): + """ + Page was only partially downloaded, we got disconnected in middle. + + @ivar response: All of the response body which was downloaded. + """ + + +class HTTPPageGetter(http.HTTPClient): + """ + Gets a resource via HTTP, then quits. + + Typically used with L{HTTPClientFactory}. Note that this class does not, by + itself, do anything with the response. If you want to download a resource + into a file, use L{HTTPPageDownloader} instead. + + @ivar _completelyDone: A boolean indicating whether any further requests are + necessary after this one completes in order to provide a result to + C{self.factory.deferred}. If it is C{False}, then a redirect is going + to be followed. Otherwise, this protocol's connection is the last one + before firing the result Deferred. This is used to make sure the result + Deferred is only fired after the connection is cleaned up. + """ + + quietLoss = 0 + followRedirect = True + failed = 0 + + _completelyDone = True + + _specialHeaders = set( + (b'host', b'user-agent', b'cookie', b'content-length'), + ) + + def connectionMade(self): + method = _ensureValidMethod(getattr(self.factory, 'method', b'GET')) + self.sendCommand(method, _ensureValidURI(self.factory.path)) + if self.factory.scheme == b'http' and self.factory.port != 80: + host = self.factory.host + b':' + intToBytes(self.factory.port) + elif self.factory.scheme == b'https' and self.factory.port != 443: + host = self.factory.host + b':' + intToBytes(self.factory.port) + else: + host = self.factory.host + self.sendHeader(b'Host', self.factory.headers.get(b"host", host)) + self.sendHeader(b'User-Agent', self.factory.agent) + data = getattr(self.factory, 'postdata', None) + if data is not None: + self.sendHeader(b"Content-Length", intToBytes(len(data))) + + cookieData = [] + for (key, value) in self.factory.headers.items(): + if key.lower() not in self._specialHeaders: + # we calculated it on our own + self.sendHeader(key, value) + if key.lower() == b'cookie': + cookieData.append(value) + for cookie, cookval in self.factory.cookies.items(): + cookieData.append(cookie + b'=' + cookval) + if cookieData: + self.sendHeader(b'Cookie', b'; '.join(cookieData)) + self.endHeaders() + self.headers = {} + + if data is not None: + self.transport.write(data) + + def handleHeader(self, key, value): + """ + Called every time a header is received. Stores the header information + as key-value pairs in the C{headers} attribute. + + @type key: C{str} + @param key: An HTTP header field name. + + @type value: C{str} + @param value: An HTTP header field value. + """ + key = key.lower() + l = self.headers.setdefault(key, []) + l.append(value) + + def handleStatus(self, version, status, message): + """ + Handle the HTTP status line. + + @param version: The HTTP version. + @type version: L{bytes} + @param status: The HTTP status code, an integer represented as a + bytestring. + @type status: L{bytes} + @param message: The HTTP status message. + @type message: L{bytes} + """ + self.version, self.status, self.message = version, status, message + self.factory.gotStatus(version, status, message) + + def handleEndHeaders(self): + self.factory.gotHeaders(self.headers) + m = getattr(self, 'handleStatus_' + nativeString(self.status), + self.handleStatusDefault) + m() + + def handleStatus_200(self): + pass + + handleStatus_201 = lambda self: self.handleStatus_200() + handleStatus_202 = lambda self: self.handleStatus_200() + + def handleStatusDefault(self): + self.failed = 1 + + def handleStatus_301(self): + l = self.headers.get(b'location') + if not l: + self.handleStatusDefault() + return + url = l[0] + if self.followRedirect: + self.factory._redirectCount += 1 + if self.factory._redirectCount >= self.factory.redirectLimit: + err = error.InfiniteRedirection( + self.status, + b'Infinite redirection detected', + location=url) + self.factory.noPage(Failure(err)) + self.quietLoss = True + self.transport.loseConnection() + return + + self._completelyDone = False + self.factory.setURL(url) + + if self.factory.scheme == b'https': + from twisted.internet import ssl + contextFactory = ssl.ClientContextFactory() + reactor.connectSSL(nativeString(self.factory.host), + self.factory.port, + self.factory, contextFactory) + else: + reactor.connectTCP(nativeString(self.factory.host), + self.factory.port, + self.factory) + else: + self.handleStatusDefault() + self.factory.noPage( + Failure( + error.PageRedirect( + self.status, self.message, location = url))) + self.quietLoss = True + self.transport.loseConnection() + + def handleStatus_302(self): + if self.afterFoundGet: + self.handleStatus_303() + else: + self.handleStatus_301() + + + def handleStatus_303(self): + self.factory.method = b'GET' + self.handleStatus_301() + + + def connectionLost(self, reason): + """ + When the connection used to issue the HTTP request is closed, notify the + factory if we have not already, so it can produce a result. + """ + if not self.quietLoss: + http.HTTPClient.connectionLost(self, reason) + self.factory.noPage(reason) + if self._completelyDone: + # Only if we think we're completely done do we tell the factory that + # we're "disconnected". This way when we're following redirects, + # only the last protocol used will fire the _disconnectedDeferred. + self.factory._disconnectedDeferred.callback(None) + + + def handleResponse(self, response): + if self.quietLoss: + return + if self.failed: + self.factory.noPage( + Failure( + error.Error( + self.status, self.message, response))) + if self.factory.method == b'HEAD': + # Callback with empty string, since there is never a response + # body for HEAD requests. + self.factory.page(b'') + elif self.length != None and self.length != 0: + self.factory.noPage(Failure( + PartialDownloadError(self.status, self.message, response))) + else: + self.factory.page(response) + # server might be stupid and not close connection. admittedly + # the fact we do only one request per connection is also + # stupid... + self.transport.loseConnection() + + def timeout(self): + self.quietLoss = True + self.transport.abortConnection() + self.factory.noPage(defer.TimeoutError("Getting %s took longer than %s seconds." % (self.factory.url, self.factory.timeout))) + + +class HTTPPageDownloader(HTTPPageGetter): + + transmittingPage = 0 + + def handleStatus_200(self, partialContent=0): + HTTPPageGetter.handleStatus_200(self) + self.transmittingPage = 1 + self.factory.pageStart(partialContent) + + def handleStatus_206(self): + self.handleStatus_200(partialContent=1) + + def handleResponsePart(self, data): + if self.transmittingPage: + self.factory.pagePart(data) + + def handleResponseEnd(self): + if self.length: + self.transmittingPage = 0 + self.factory.noPage( + Failure( + PartialDownloadError(self.status))) + if self.transmittingPage: + self.factory.pageEnd() + self.transmittingPage = 0 + if self.failed: + self.factory.noPage( + Failure( + error.Error( + self.status, self.message, None))) + self.transport.loseConnection() + + +class HTTPClientFactory(protocol.ClientFactory): + """Download a given URL. + + @type deferred: Deferred + @ivar deferred: A Deferred that will fire when the content has + been retrieved. Once this is fired, the ivars `status', `version', + and `message' will be set. + + @type status: bytes + @ivar status: The status of the response. + + @type version: bytes + @ivar version: The version of the response. + + @type message: bytes + @ivar message: The text message returned with the status. + + @type response_headers: dict + @ivar response_headers: The headers that were specified in the + response from the server. + + @type method: bytes + @ivar method: The HTTP method to use in the request. This should be one of + OPTIONS, GET, HEAD, POST, PUT, DELETE, TRACE, or CONNECT (case + matters). Other values may be specified if the server being contacted + supports them. + + @type redirectLimit: int + @ivar redirectLimit: The maximum number of HTTP redirects that can occur + before it is assumed that the redirection is endless. + + @type afterFoundGet: C{bool} + @ivar afterFoundGet: Deviate from the HTTP 1.1 RFC by handling redirects + the same way as most web browsers; if the request method is POST and a + 302 status is encountered, the redirect is followed with a GET method + + @type _redirectCount: int + @ivar _redirectCount: The current number of HTTP redirects encountered. + + @ivar _disconnectedDeferred: A L{Deferred} which only fires after the last + connection associated with the request (redirects may cause multiple + connections to be required) has closed. The result Deferred will only + fire after this Deferred, so that callers can be assured that there are + no more event sources in the reactor once they get the result. + """ + + protocol = HTTPPageGetter + + url = None + scheme = None + host = b'' + port = None + path = None + + def __init__(self, url, method=b'GET', postdata=None, headers=None, + agent=b"Twisted PageGetter", timeout=0, cookies=None, + followRedirect=True, redirectLimit=20, + afterFoundGet=False): + self.followRedirect = followRedirect + self.redirectLimit = redirectLimit + self._redirectCount = 0 + self.timeout = timeout + self.agent = agent + self.afterFoundGet = afterFoundGet + if cookies is None: + cookies = {} + self.cookies = cookies + if headers is not None: + self.headers = InsensitiveDict(headers) + else: + self.headers = InsensitiveDict() + if postdata is not None: + self.headers.setdefault(b'Content-Length', + intToBytes(len(postdata))) + # just in case a broken http/1.1 decides to keep connection alive + self.headers.setdefault(b"connection", b"close") + self.postdata = postdata + self.method = _ensureValidMethod(method) + + self.setURL(url) + + self.waiting = 1 + self._disconnectedDeferred = defer.Deferred() + self.deferred = defer.Deferred() + # Make sure the first callback on the result Deferred pauses the + # callback chain until the request connection is closed. + self.deferred.addBoth(self._waitForDisconnect) + self.response_headers = None + + + def _waitForDisconnect(self, passthrough): + """ + Chain onto the _disconnectedDeferred, preserving C{passthrough}, so that + the result is only available after the associated connection has been + closed. + """ + self._disconnectedDeferred.addCallback(lambda ignored: passthrough) + return self._disconnectedDeferred + + + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self.url) + + def setURL(self, url): + _ensureValidURI(url.strip()) + self.url = url + uri = URI.fromBytes(url) + if uri.scheme and uri.host: + self.scheme = uri.scheme + self.host = uri.host + self.port = uri.port + self.path = uri.originForm + + def buildProtocol(self, addr): + p = protocol.ClientFactory.buildProtocol(self, addr) + p.followRedirect = self.followRedirect + p.afterFoundGet = self.afterFoundGet + if self.timeout: + timeoutCall = reactor.callLater(self.timeout, p.timeout) + self.deferred.addBoth(self._cancelTimeout, timeoutCall) + return p + + def _cancelTimeout(self, result, timeoutCall): + if timeoutCall.active(): + timeoutCall.cancel() + return result + + def gotHeaders(self, headers): + """ + Parse the response HTTP headers. + + @param headers: The response HTTP headers. + @type headers: L{dict} + """ + self.response_headers = headers + if b'set-cookie' in headers: + for cookie in headers[b'set-cookie']: + if b'=' in cookie: + cookparts = cookie.split(b';') + cook = cookparts[0] + cook.lstrip() + k, v = cook.split(b'=', 1) + self.cookies[k.lstrip()] = v.lstrip() + + def gotStatus(self, version, status, message): + """ + Set the status of the request on us. + + @param version: The HTTP version. + @type version: L{bytes} + @param status: The HTTP status code, an integer represented as a + bytestring. + @type status: L{bytes} + @param message: The HTTP status message. + @type message: L{bytes} + """ + self.version, self.status, self.message = version, status, message + + def page(self, page): + if self.waiting: + self.waiting = 0 + self.deferred.callback(page) + + def noPage(self, reason): + if self.waiting: + self.waiting = 0 + self.deferred.errback(reason) + + def clientConnectionFailed(self, _, reason): + """ + When a connection attempt fails, the request cannot be issued. If no + result has yet been provided to the result Deferred, provide the + connection failure reason as an error result. + """ + if self.waiting: + self.waiting = 0 + # If the connection attempt failed, there is nothing more to + # disconnect, so just fire that Deferred now. + self._disconnectedDeferred.callback(None) + self.deferred.errback(reason) + + + +class HTTPDownloader(HTTPClientFactory): + """ + Download to a file. + """ + protocol = HTTPPageDownloader + value = None + _log = Logger() + + def __init__(self, url, fileOrName, + method=b'GET', postdata=None, headers=None, + agent=b"Twisted client", supportPartial=False, + timeout=0, cookies=None, followRedirect=True, + redirectLimit=20, afterFoundGet=False): + self.requestedPartial = 0 + if isinstance(fileOrName, (str, unicode)): + self.fileName = fileOrName + self.file = None + if supportPartial and os.path.exists(self.fileName): + fileLength = os.path.getsize(self.fileName) + if fileLength: + self.requestedPartial = fileLength + if headers == None: + headers = {} + headers[b"range"] = b"bytes=" + intToBytes(fileLength) + b"-" + else: + self.file = fileOrName + HTTPClientFactory.__init__( + self, url, method=method, postdata=postdata, headers=headers, + agent=agent, timeout=timeout, cookies=cookies, + followRedirect=followRedirect, redirectLimit=redirectLimit, + afterFoundGet=afterFoundGet) + + + def gotHeaders(self, headers): + HTTPClientFactory.gotHeaders(self, headers) + if self.requestedPartial: + contentRange = headers.get(b"content-range", None) + if not contentRange: + # server doesn't support partial requests, oh well + self.requestedPartial = 0 + return + start, end, realLength = http.parseContentRange(contentRange[0]) + if start != self.requestedPartial: + # server is acting weirdly + self.requestedPartial = 0 + + + def openFile(self, partialContent): + if partialContent: + file = open(self.fileName, 'rb+') + file.seek(0, 2) + else: + file = open(self.fileName, 'wb') + return file + + def pageStart(self, partialContent): + """Called on page download start. + + @param partialContent: tells us if the download is partial download we requested. + """ + if partialContent and not self.requestedPartial: + raise ValueError("we shouldn't get partial content response if we didn't want it!") + if self.waiting: + try: + if not self.file: + self.file = self.openFile(partialContent) + except IOError: + #raise + self.deferred.errback(Failure()) + + def pagePart(self, data): + if not self.file: + return + try: + self.file.write(data) + except IOError: + #raise + self.file = None + self.deferred.errback(Failure()) + + + def noPage(self, reason): + """ + Close the storage file and errback the waiting L{Deferred} with the + given reason. + """ + if self.waiting: + self.waiting = 0 + if self.file: + try: + self.file.close() + except: + self._log.failure("Error closing HTTPDownloader file") + self.deferred.errback(reason) + + + def pageEnd(self): + self.waiting = 0 + if not self.file: + return + try: + self.file.close() + except IOError: + self.deferred.errback(Failure()) + return + self.deferred.callback(self.value) + + + +class URI(object): + """ + A URI object. + + @see: U{https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21} + """ + def __init__(self, scheme, netloc, host, port, path, params, query, + fragment): + """ + @type scheme: L{bytes} + @param scheme: URI scheme specifier. + + @type netloc: L{bytes} + @param netloc: Network location component. + + @type host: L{bytes} + @param host: Host name. For IPv6 address literals the brackets are + stripped. + + @type port: L{int} + @param port: Port number. + + @type path: L{bytes} + @param path: Hierarchical path. + + @type params: L{bytes} + @param params: Parameters for last path segment. + + @type query: L{bytes} + @param query: Query string. + + @type fragment: L{bytes} + @param fragment: Fragment identifier. + """ + self.scheme = scheme + self.netloc = netloc + self.host = host.strip(b'[]') + self.port = port + self.path = path + self.params = params + self.query = query + self.fragment = fragment + + + @classmethod + def fromBytes(cls, uri, defaultPort=None): + """ + Parse the given URI into a L{URI}. + + @type uri: C{bytes} + @param uri: URI to parse. + + @type defaultPort: C{int} or L{None} + @param defaultPort: An alternate value to use as the port if the URI + does not include one. + + @rtype: L{URI} + @return: Parsed URI instance. + """ + uri = uri.strip() + scheme, netloc, path, params, query, fragment = http.urlparse(uri) + + if defaultPort is None: + if scheme == b'https': + defaultPort = 443 + else: + defaultPort = 80 + + if b':' in netloc: + host, port = netloc.rsplit(b':', 1) + try: + port = int(port) + except ValueError: + host, port = netloc, defaultPort + else: + host, port = netloc, defaultPort + return cls(scheme, netloc, host, port, path, params, query, fragment) + + + def toBytes(self): + """ + Assemble the individual parts of the I{URI} into a fully formed I{URI}. + + @rtype: C{bytes} + @return: A fully formed I{URI}. + """ + return urlunparse( + (self.scheme, self.netloc, self.path, self.params, self.query, + self.fragment)) + + + @property + def originForm(self): + """ + The absolute I{URI} path including I{URI} parameters, query string and + fragment identifier. + + @see: U{https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21#section-5.3} + + @return: The absolute path in original form. + @rtype: L{bytes} + """ + # The HTTP bis draft says the origin form should not include the + # fragment. + path = urlunparse( + (b'', b'', self.path, self.params, self.query, b'')) + if path == b'': + path = b'/' + return path + + + +def _urljoin(base, url): + """ + Construct a full ("absolute") URL by combining a "base URL" with another + URL. Informally, this uses components of the base URL, in particular the + addressing scheme, the network location and (part of) the path, to provide + missing components in the relative URL. + + Additionally, the fragment identifier is preserved according to the HTTP + 1.1 bis draft. + + @type base: C{bytes} + @param base: Base URL. + + @type url: C{bytes} + @param url: URL to combine with C{base}. + + @return: An absolute URL resulting from the combination of C{base} and + C{url}. + + @see: L{urlparse.urljoin} + + @see: U{https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-22#section-7.1.2} + """ + base, baseFrag = urldefrag(base) + url, urlFrag = urldefrag(urljoin(base, url)) + return urljoin(url, b'#' + (urlFrag or baseFrag)) + + + +def _makeGetterFactory(url, factoryFactory, contextFactory=None, + *args, **kwargs): + """ + Create and connect an HTTP page getting factory. + + Any additional positional or keyword arguments are used when calling + C{factoryFactory}. + + @param factoryFactory: Factory factory that is called with C{url}, C{args} + and C{kwargs} to produce the getter + + @param contextFactory: Context factory to use when creating a secure + connection, defaulting to L{None} + + @return: The factory created by C{factoryFactory} + """ + uri = URI.fromBytes(_ensureValidURI(url.strip())) + factory = factoryFactory(url, *args, **kwargs) + if uri.scheme == b'https': + from twisted.internet import ssl + if contextFactory is None: + contextFactory = ssl.ClientContextFactory() + reactor.connectSSL( + nativeString(uri.host), uri.port, factory, contextFactory) + else: + reactor.connectTCP(nativeString(uri.host), uri.port, factory) + return factory + + +_GETPAGE_REPLACEMENT_TEXT = "https://pypi.org/project/treq/ or twisted.web.client.Agent" + +def _deprecateGetPageClasses(): + """ + Mark the protocols and factories associated with L{getPage} and + L{downloadPage} as deprecated. + """ + for klass in [ + HTTPPageGetter, HTTPPageDownloader, + HTTPClientFactory, HTTPDownloader + ]: + deprecatedModuleAttribute( + Version("Twisted", 16, 7, 0), + getDeprecationWarningString( + klass, + Version("Twisted", 16, 7, 0), + replacement=_GETPAGE_REPLACEMENT_TEXT) + .split("; ")[1], + klass.__module__, + klass.__name__) + +_deprecateGetPageClasses() + + + +@deprecated(Version("Twisted", 16, 7, 0), + _GETPAGE_REPLACEMENT_TEXT) +def getPage(url, contextFactory=None, *args, **kwargs): + """ + Download a web page as a string. + + Download a page. Return a deferred, which will callback with a + page (as a string) or errback with a description of the error. + + See L{HTTPClientFactory} to see what extra arguments can be passed. + """ + return _makeGetterFactory( + url, + HTTPClientFactory, + contextFactory=contextFactory, + *args, **kwargs).deferred + + + +@deprecated(Version("Twisted", 16, 7, 0), + _GETPAGE_REPLACEMENT_TEXT) +def downloadPage(url, file, contextFactory=None, *args, **kwargs): + """ + Download a web page to a file. + + @param file: path to file on filesystem, or file-like object. + + See HTTPDownloader to see what extra args can be passed. + """ + factoryFactory = lambda url, *a, **kw: HTTPDownloader(url, file, *a, **kw) + return _makeGetterFactory( + url, + factoryFactory, + contextFactory=contextFactory, + *args, **kwargs).deferred + + +# The code which follows is based on the new HTTP client implementation. It +# should be significantly better than anything above, though it is not yet +# feature equivalent. + +from twisted.web.error import SchemeNotSupported +from twisted.web._newclient import ( + HTTP11ClientProtocol, + PotentialDataLoss, + Request, + RequestGenerationFailed, + RequestNotSent, + RequestTransmissionFailed, + Response, + ResponseDone, + ResponseFailed, + ResponseNeverReceived, + _WrapperException, + ) + + + +try: + from OpenSSL import SSL +except ImportError: + SSL = None +else: + from twisted.internet.ssl import (CertificateOptions, + platformTrust, + optionsForClientTLS) + + +def _requireSSL(decoratee): + """ + The decorated method requires pyOpenSSL to be present, or it raises + L{NotImplementedError}. + + @param decoratee: A function which requires pyOpenSSL. + @type decoratee: L{callable} + + @return: A function which raises L{NotImplementedError} if pyOpenSSL is not + installed; otherwise, if it is installed, simply return C{decoratee}. + @rtype: L{callable} + """ + if SSL is None: + @wraps(decoratee) + def raiseNotImplemented(*a, **kw): + """ + pyOpenSSL is not available. + + @param a: The positional arguments for C{decoratee}. + + @param kw: The keyword arguments for C{decoratee}. + + @raise NotImplementedError: Always. + """ + raise NotImplementedError("SSL support unavailable") + return raiseNotImplemented + return decoratee + + + +class WebClientContextFactory(object): + """ + This class is deprecated. Please simply use L{Agent} as-is, or if you want + to customize something, use L{BrowserLikePolicyForHTTPS}. + + A L{WebClientContextFactory} is an HTTPS policy which totally ignores the + hostname and port. It performs basic certificate verification, however the + lack of validation of service identity (e.g. hostname validation) means it + is still vulnerable to man-in-the-middle attacks. Don't use it any more. + """ + + def _getCertificateOptions(self, hostname, port): + """ + Return a L{CertificateOptions}. + + @param hostname: ignored + + @param port: ignored + + @return: A new CertificateOptions instance. + @rtype: L{CertificateOptions} + """ + return CertificateOptions( + method=SSL.SSLv23_METHOD, + trustRoot=platformTrust() + ) + + + @_requireSSL + def getContext(self, hostname, port): + """ + Return an L{OpenSSL.SSL.Context}. + + @param hostname: ignored + @param port: ignored + + @return: A new SSL context. + @rtype: L{OpenSSL.SSL.Context} + """ + return self._getCertificateOptions(hostname, port).getContext() + + + +@implementer(IPolicyForHTTPS) +class BrowserLikePolicyForHTTPS(object): + """ + SSL connection creator for web clients. + """ + def __init__(self, trustRoot=None): + self._trustRoot = trustRoot + + + @_requireSSL + def creatorForNetloc(self, hostname, port): + """ + Create a L{client connection creator + } for a + given network location. + + @param tls: The TLS protocol to create a connection for. + @type tls: L{twisted.protocols.tls.TLSMemoryBIOProtocol} + + @param hostname: The hostname part of the URI. + @type hostname: L{bytes} + + @param port: The port part of the URI. + @type port: L{int} + + @return: a connection creator with appropriate verification + restrictions set + @rtype: L{client connection creator + } + """ + return optionsForClientTLS(hostname.decode("ascii"), + trustRoot=self._trustRoot) + + + +deprecatedModuleAttribute(Version("Twisted", 14, 0, 0), + getDeprecationWarningString( + WebClientContextFactory, + Version("Twisted", 14, 0, 0), + replacement=BrowserLikePolicyForHTTPS) + .split("; ")[1], + WebClientContextFactory.__module__, + WebClientContextFactory.__name__) + + + +@implementer(IPolicyForHTTPS) +class HostnameCachingHTTPSPolicy(object): + """ + IPolicyForHTTPS that wraps a L{IPolicyForHTTPS} and caches the created + L{IOpenSSLClientConnectionCreator}. + + This policy will cache up to C{cacheSize} + L{client connection creators } for reuse in subsequent requests to + the same hostname. + + @ivar _policyForHTTPS: See C{policyforHTTPS} parameter of L{__init__}. + + @ivar _cache: A cache associating hostnames to their + L{client connection creators }. + @type _cache: L{collections.OrderedDict} + + @ivar _cacheSize: See C{cacheSize} parameter of L{__init__}. + + @since: Twisted 19.2.0 + """ + + def __init__(self, policyforHTTPS, cacheSize=20): + """ + @param policyforHTTPS: The IPolicyForHTTPS to wrap. + @type policyforHTTPS: L{IPolicyForHTTPS} + + @param cacheSize: The maximum size of the hostname cache. + @type cacheSize: L{int} + """ + self._policyForHTTPS = policyforHTTPS + self._cache = collections.OrderedDict() + self._cacheSize = cacheSize + + + def creatorForNetloc(self, hostname, port): + """ + Create a L{client connection creator + } for a + given network location and cache it for future use. + + @param hostname: The hostname part of the URI. + @type hostname: L{bytes} + + @param port: The port part of the URI. + @type port: L{int} + + @return: a connection creator with appropriate verification + restrictions set + @rtype: L{client connection creator + } + """ + host = hostname.decode("ascii") + try: + creator = self._cache.pop(host) + except KeyError: + creator = self._policyForHTTPS.creatorForNetloc(hostname, port) + + self._cache[host] = creator + if len(self._cache) > self._cacheSize: + self._cache.popitem(last=False) + + return creator + + + +@implementer(IOpenSSLContextFactory) +class _ContextFactoryWithContext(object): + """ + A L{_ContextFactoryWithContext} is like a + L{twisted.internet.ssl.ContextFactory} with a pre-created context. + + @ivar _context: A Context. + @type _context: L{OpenSSL.SSL.Context} + """ + + def __init__(self, context): + """ + Initialize a L{_ContextFactoryWithContext} with a context. + + @param context: An SSL context. + @type context: L{OpenSSL.SSL.Context} + """ + self._context = context + + + def getContext(self): + """ + Return the context created by + L{_DeprecatedToCurrentPolicyForHTTPS._webContextFactory}. + + @return: A context. + @rtype context: L{OpenSSL.SSL.Context} + """ + return self._context + + + +@implementer(IPolicyForHTTPS) +class _DeprecatedToCurrentPolicyForHTTPS(object): + """ + Adapt a web context factory to a normal context factory. + + @ivar _webContextFactory: An object providing a getContext method with + C{hostname} and C{port} arguments. + @type _webContextFactory: L{WebClientContextFactory} (or object with a + similar C{getContext} method). + """ + def __init__(self, webContextFactory): + """ + Wrap a web context factory in an L{IPolicyForHTTPS}. + + @param webContextFactory: An object providing a getContext method with + C{hostname} and C{port} arguments. + @type webContextFactory: L{WebClientContextFactory} (or object with a + similar C{getContext} method). + """ + self._webContextFactory = webContextFactory + + + def creatorForNetloc(self, hostname, port): + """ + Called the wrapped web context factory's C{getContext} method with a + hostname and port number and return the resulting context object. + + @param hostname: The hostname part of the URI. + @type hostname: L{bytes} + + @param port: The port part of the URI. + @type port: L{int} + + @return: A context factory. + @rtype: L{IOpenSSLContextFactory} + """ + context = self._webContextFactory.getContext(hostname, port) + return _ContextFactoryWithContext(context) + + + +@implementer(IBodyProducer) +class FileBodyProducer(object): + """ + L{FileBodyProducer} produces bytes from an input file object incrementally + and writes them to a consumer. + + Since file-like objects cannot be read from in an event-driven manner, + L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from + the file. This process is also paused and resumed based on notifications + from the L{IConsumer} provider being written to. + + The file is closed after it has been read, or if the producer is stopped + early. + + @ivar _inputFile: Any file-like object, bytes read from which will be + written to a consumer. + + @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to + schedule all reads. + + @ivar _readSize: The number of bytes to read from C{_inputFile} at a time. + """ + + def __init__(self, inputFile, cooperator=task, readSize=2 ** 16): + self._inputFile = inputFile + self._cooperate = cooperator.cooperate + self._readSize = readSize + self.length = self._determineLength(inputFile) + + + def _determineLength(self, fObj): + """ + Determine how many bytes can be read out of C{fObj} (assuming it is not + modified from this point on). If the determination cannot be made, + return C{UNKNOWN_LENGTH}. + """ + try: + seek = fObj.seek + tell = fObj.tell + except AttributeError: + return UNKNOWN_LENGTH + originalPosition = tell() + seek(0, os.SEEK_END) + end = tell() + seek(originalPosition, os.SEEK_SET) + return end - originalPosition + + + def stopProducing(self): + """ + Permanently stop writing bytes from the file to the consumer by + stopping the underlying L{CooperativeTask}. + """ + self._inputFile.close() + self._task.stop() + + + def startProducing(self, consumer): + """ + Start a cooperative task which will read bytes from the input file and + write them to C{consumer}. Return a L{Deferred} which fires after all + bytes have been written. If this L{Deferred} is cancelled before it is + fired, stop reading and writing bytes. + + @param consumer: Any L{IConsumer} provider + """ + self._task = self._cooperate(self._writeloop(consumer)) + d = self._task.whenDone() + def maybeStopped(reason): + if reason.check(defer.CancelledError): + self.stopProducing() + elif reason.check(task.TaskStopped): + pass + else: + return reason + # IBodyProducer.startProducing's Deferred isn't supposed to fire if + # stopProducing is called. + return defer.Deferred() + d.addCallbacks(lambda ignored: None, maybeStopped) + return d + + + def _writeloop(self, consumer): + """ + Return an iterator which reads one chunk of bytes from the input file + and writes them to the consumer for each time it is iterated. + """ + while True: + bytes = self._inputFile.read(self._readSize) + if not bytes: + self._inputFile.close() + break + consumer.write(bytes) + yield None + + + def pauseProducing(self): + """ + Temporarily suspend copying bytes from the input file to the consumer + by pausing the L{CooperativeTask} which drives that activity. + """ + self._task.pause() + + + def resumeProducing(self): + """ + Undo the effects of a previous C{pauseProducing} and resume copying + bytes to the consumer by resuming the L{CooperativeTask} which drives + the write activity. + """ + self._task.resume() + + + +class _HTTP11ClientFactory(protocol.Factory): + """ + A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}. + + @ivar _quiescentCallback: The quiescent callback to be passed to protocol + instances, used to return them to the connection pool. + + @ivar _metadata: Metadata about the low-level connection details, + used to make the repr more useful. + + @since: 11.1 + """ + def __init__(self, quiescentCallback, metadata): + self._quiescentCallback = quiescentCallback + self._metadata = metadata + + + def __repr__(self): + return '_HTTP11ClientFactory({}, {})'.format( + self._quiescentCallback, + self._metadata) + + def buildProtocol(self, addr): + return HTTP11ClientProtocol(self._quiescentCallback) + + + +class _RetryingHTTP11ClientProtocol(object): + """ + A wrapper for L{HTTP11ClientProtocol} that automatically retries requests. + + @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}. + + @ivar _newConnection: A callable that creates a new connection for a + retry. + """ + + def __init__(self, clientProtocol, newConnection): + self._clientProtocol = clientProtocol + self._newConnection = newConnection + + + def _shouldRetry(self, method, exception, bodyProducer): + """ + Indicate whether request should be retried. + + Only returns C{True} if method is idempotent, no response was + received, the reason for the failed request was not due to + user-requested cancellation, and no body was sent. The latter + requirement may be relaxed in the future, and PUT added to approved + method list. + + @param method: The method of the request. + @type method: L{bytes} + """ + if method not in (b"GET", b"HEAD", b"OPTIONS", b"DELETE", b"TRACE"): + return False + if not isinstance(exception, (RequestNotSent, + RequestTransmissionFailed, + ResponseNeverReceived)): + return False + if isinstance(exception, _WrapperException): + for aFailure in exception.reasons: + if aFailure.check(defer.CancelledError): + return False + if bodyProducer is not None: + return False + return True + + + def request(self, request): + """ + Do a request, and retry once (with a new connection) if it fails in + a retryable manner. + + @param request: A L{Request} instance that will be requested using the + wrapped protocol. + """ + d = self._clientProtocol.request(request) + + def failed(reason): + if self._shouldRetry(request.method, reason.value, + request.bodyProducer): + return self._newConnection().addCallback( + lambda connection: connection.request(request)) + else: + return reason + d.addErrback(failed) + return d + + + +class HTTPConnectionPool(object): + """ + A pool of persistent HTTP connections. + + Features: + - Cached connections will eventually time out. + - Limits on maximum number of persistent connections. + + Connections are stored using keys, which should be chosen such that any + connections stored under a given key can be used interchangeably. + + Failed requests done using previously cached connections will be retried + once if they use an idempotent method (e.g. GET), in case the HTTP server + timed them out. + + @ivar persistent: Boolean indicating whether connections should be + persistent. Connections are persistent by default. + + @ivar maxPersistentPerHost: The maximum number of cached persistent + connections for a C{host:port} destination. + @type maxPersistentPerHost: C{int} + + @ivar cachedConnectionTimeout: Number of seconds a cached persistent + connection will stay open before disconnecting. + + @ivar retryAutomatically: C{boolean} indicating whether idempotent + requests should be retried once if no response was received. + + @ivar _factory: The factory used to connect to the proxy. + + @ivar _connections: Map (scheme, host, port) to lists of + L{HTTP11ClientProtocol} instances. + + @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a + C{IDelayedCall} instance of their timeout. + + @since: 12.1 + """ + + _factory = _HTTP11ClientFactory + maxPersistentPerHost = 2 + cachedConnectionTimeout = 240 + retryAutomatically = True + _log = Logger() + + def __init__(self, reactor, persistent=True): + self._reactor = reactor + self.persistent = persistent + self._connections = {} + self._timeouts = {} + + + def getConnection(self, key, endpoint): + """ + Supply a connection, newly created or retrieved from the pool, to be + used for one HTTP request. + + The connection will remain out of the pool (not available to be + returned from future calls to this method) until one HTTP request has + been completed over it. + + Afterwards, if the connection is still open, it will automatically be + added to the pool. + + @param key: A unique key identifying connections that can be used + interchangeably. + + @param endpoint: An endpoint that can be used to open a new connection + if no cached connection is available. + + @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol} + (or a wrapper) that can be used to send a single HTTP request. + """ + # Try to get cached version: + connections = self._connections.get(key) + while connections: + connection = connections.pop(0) + # Cancel timeout: + self._timeouts[connection].cancel() + del self._timeouts[connection] + if connection.state == "QUIESCENT": + if self.retryAutomatically: + newConnection = lambda: self._newConnection(key, endpoint) + connection = _RetryingHTTP11ClientProtocol( + connection, newConnection) + return defer.succeed(connection) + + return self._newConnection(key, endpoint) + + + def _newConnection(self, key, endpoint): + """ + Create a new connection. + + This implements the new connection code path for L{getConnection}. + """ + def quiescentCallback(protocol): + self._putConnection(key, protocol) + factory = self._factory(quiescentCallback, repr(endpoint)) + return endpoint.connect(factory) + + + def _removeConnection(self, key, connection): + """ + Remove a connection from the cache and disconnect it. + """ + connection.transport.loseConnection() + self._connections[key].remove(connection) + del self._timeouts[connection] + + + def _putConnection(self, key, connection): + """ + Return a persistent connection to the pool. This will be called by + L{HTTP11ClientProtocol} when the connection becomes quiescent. + """ + if connection.state != "QUIESCENT": + # Log with traceback for debugging purposes: + try: + raise RuntimeError( + "BUG: Non-quiescent protocol added to connection pool.") + except: + self._log.failure( + "BUG: Non-quiescent protocol added to connection pool.") + return + connections = self._connections.setdefault(key, []) + if len(connections) == self.maxPersistentPerHost: + dropped = connections.pop(0) + dropped.transport.loseConnection() + self._timeouts[dropped].cancel() + del self._timeouts[dropped] + connections.append(connection) + cid = self._reactor.callLater(self.cachedConnectionTimeout, + self._removeConnection, + key, connection) + self._timeouts[connection] = cid + + + def closeCachedConnections(self): + """ + Close all persistent connections and remove them from the pool. + + @return: L{defer.Deferred} that fires when all connections have been + closed. + """ + results = [] + for protocols in itervalues(self._connections): + for p in protocols: + results.append(p.abort()) + self._connections = {} + for dc in itervalues(self._timeouts): + dc.cancel() + self._timeouts = {} + return defer.gatherResults(results).addCallback(lambda ign: None) + + + +class _AgentBase(object): + """ + Base class offering common facilities for L{Agent}-type classes. + + @ivar _reactor: The C{IReactorTime} implementation which will be used by + the pool, and perhaps by subclasses as well. + + @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections. + """ + + def __init__(self, reactor, pool): + if pool is None: + pool = HTTPConnectionPool(reactor, False) + self._reactor = reactor + self._pool = pool + + + def _computeHostValue(self, scheme, host, port): + """ + Compute the string to use for the value of the I{Host} header, based on + the given scheme, host name, and port number. + """ + if (isIPv6Address(nativeString(host))): + host = b'[' + host + b']' + if (scheme, port) in ((b'http', 80), (b'https', 443)): + return host + return host + b":" + intToBytes(port) + + + def _requestWithEndpoint(self, key, endpoint, method, parsedURI, + headers, bodyProducer, requestPath): + """ + Issue a new request, given the endpoint and the path sent as part of + the request. + """ + if not isinstance(method, bytes): + raise TypeError('method={!r} is {}, but must be bytes'.format( + method, type(method))) + + method = _ensureValidMethod(method) + + # Create minimal headers, if necessary: + if headers is None: + headers = Headers() + if not headers.hasHeader(b'host'): + headers = headers.copy() + headers.addRawHeader( + b'host', self._computeHostValue(parsedURI.scheme, + parsedURI.host, + parsedURI.port)) + + d = self._pool.getConnection(key, endpoint) + def cbConnected(proto): + return proto.request( + Request._construct(method, requestPath, headers, bodyProducer, + persistent=self._pool.persistent, + parsedURI=parsedURI)) + d.addCallback(cbConnected) + return d + + + +@implementer(IAgentEndpointFactory) +class _StandardEndpointFactory(object): + """ + Standard HTTP endpoint destinations - TCP for HTTP, TCP+TLS for HTTPS. + + @ivar _policyForHTTPS: A web context factory which will be used to create + SSL context objects for any SSL connections the agent needs to make. + + @ivar _connectTimeout: If not L{None}, the timeout passed to + L{HostnameEndpoint} for specifying the connection timeout. + + @ivar _bindAddress: If not L{None}, the address passed to + L{HostnameEndpoint} for specifying the local address to bind to. + """ + def __init__(self, reactor, contextFactory, connectTimeout, bindAddress): + """ + @param reactor: A provider to use to create endpoints. + @type reactor: see L{HostnameEndpoint.__init__} for acceptable reactor + types. + + @param contextFactory: A factory for TLS contexts, to control the + verification parameters of OpenSSL. + @type contextFactory: L{IPolicyForHTTPS}. + + @param connectTimeout: The amount of time that this L{Agent} will wait + for the peer to accept a connection. + @type connectTimeout: L{float} or L{None} + + @param bindAddress: The local address for client sockets to bind to. + @type bindAddress: L{bytes} or L{None} + """ + self._reactor = reactor + self._policyForHTTPS = contextFactory + self._connectTimeout = connectTimeout + self._bindAddress = bindAddress + + + def endpointForURI(self, uri): + """ + Connect directly over TCP for C{b'http'} scheme, and TLS for + C{b'https'}. + + @param uri: L{URI} to connect to. + + @return: Endpoint to connect to. + @rtype: L{IStreamClientEndpoint} + """ + kwargs = {} + if self._connectTimeout is not None: + kwargs['timeout'] = self._connectTimeout + kwargs['bindAddress'] = self._bindAddress + + try: + host = nativeString(uri.host) + except UnicodeDecodeError: + raise ValueError(("The host of the provided URI ({uri.host!r}) " + "contains non-ASCII octets, it should be ASCII " + "decodable.").format(uri=uri)) + + endpoint = HostnameEndpoint(self._reactor, host, uri.port, **kwargs) + if uri.scheme == b'http': + return endpoint + elif uri.scheme == b'https': + connectionCreator = self._policyForHTTPS.creatorForNetloc(uri.host, + uri.port) + return wrapClientTLS(connectionCreator, endpoint) + else: + raise SchemeNotSupported("Unsupported scheme: %r" % (uri.scheme,)) + + + +@implementer(IAgent) +class Agent(_AgentBase): + """ + L{Agent} is a very basic HTTP client. It supports I{HTTP} and I{HTTPS} + scheme URIs. + + @ivar _pool: An L{HTTPConnectionPool} instance. + + @ivar _endpointFactory: The L{IAgentEndpointFactory} which will + be used to create endpoints for outgoing connections. + + @since: 9.0 + """ + + def __init__(self, reactor, + contextFactory=BrowserLikePolicyForHTTPS(), + connectTimeout=None, bindAddress=None, + pool=None): + """ + Create an L{Agent}. + + @param reactor: A reactor for this L{Agent} to place outgoing + connections. + @type reactor: see L{HostnameEndpoint.__init__} for acceptable reactor + types. + + @param contextFactory: A factory for TLS contexts, to control the + verification parameters of OpenSSL. The default is to use a + L{BrowserLikePolicyForHTTPS}, so unless you have special + requirements you can leave this as-is. + @type contextFactory: L{IPolicyForHTTPS}. + + @param connectTimeout: The amount of time that this L{Agent} will wait + for the peer to accept a connection. + @type connectTimeout: L{float} + + @param bindAddress: The local address for client sockets to bind to. + @type bindAddress: L{bytes} + + @param pool: An L{HTTPConnectionPool} instance, or L{None}, in which + case a non-persistent L{HTTPConnectionPool} instance will be + created. + @type pool: L{HTTPConnectionPool} + """ + if not IPolicyForHTTPS.providedBy(contextFactory): + warnings.warn( + repr(contextFactory) + + " was passed as the HTTPS policy for an Agent, but it does " + "not provide IPolicyForHTTPS. Since Twisted 14.0, you must " + "pass a provider of IPolicyForHTTPS.", + stacklevel=2, category=DeprecationWarning + ) + contextFactory = _DeprecatedToCurrentPolicyForHTTPS(contextFactory) + endpointFactory = _StandardEndpointFactory( + reactor, contextFactory, connectTimeout, bindAddress) + self._init(reactor, endpointFactory, pool) + + + @classmethod + def usingEndpointFactory(cls, reactor, endpointFactory, pool=None): + """ + Create a new L{Agent} that will use the endpoint factory to figure + out how to connect to the server. + + @param reactor: A reactor for this L{Agent} to place outgoing + connections. + @type reactor: see L{HostnameEndpoint.__init__} for acceptable reactor + types. + + @param endpointFactory: Used to construct endpoints which the + HTTP client will connect with. + @type endpointFactory: an L{IAgentEndpointFactory} provider. + + @param pool: An L{HTTPConnectionPool} instance, or L{None}, in which + case a non-persistent L{HTTPConnectionPool} instance will be + created. + @type pool: L{HTTPConnectionPool} + + @return: A new L{Agent}. + """ + agent = cls.__new__(cls) + agent._init(reactor, endpointFactory, pool) + return agent + + + def _init(self, reactor, endpointFactory, pool): + """ + Initialize a new L{Agent}. + + @param reactor: A reactor for this L{Agent} to place outgoing + connections. + @type reactor: see L{HostnameEndpoint.__init__} for acceptable reactor + types. + + @param endpointFactory: Used to construct endpoints which the + HTTP client will connect with. + @type endpointFactory: an L{IAgentEndpointFactory} provider. + + @param pool: An L{HTTPConnectionPool} instance, or L{None}, in which + case a non-persistent L{HTTPConnectionPool} instance will be + created. + @type pool: L{HTTPConnectionPool} + + @return: A new L{Agent}. + """ + _AgentBase.__init__(self, reactor, pool) + self._endpointFactory = endpointFactory + + + def _getEndpoint(self, uri): + """ + Get an endpoint for the given URI, using C{self._endpointFactory}. + + @param uri: The URI of the request. + @type uri: L{URI} + + @return: An endpoint which can be used to connect to given address. + """ + return self._endpointFactory.endpointForURI(uri) + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a request to the server indicated by the given C{uri}. + + An existing connection from the connection pool may be used or a new + one may be created. + + I{HTTP} and I{HTTPS} schemes are supported in C{uri}. + + @see: L{twisted.web.iweb.IAgent.request} + """ + uri = _ensureValidURI(uri.strip()) + parsedURI = URI.fromBytes(uri) + try: + endpoint = self._getEndpoint(parsedURI) + except SchemeNotSupported: + return defer.fail(Failure()) + key = (parsedURI.scheme, parsedURI.host, parsedURI.port) + return self._requestWithEndpoint(key, endpoint, method, parsedURI, + headers, bodyProducer, + parsedURI.originForm) + + + +@implementer(IAgent) +class ProxyAgent(_AgentBase): + """ + An HTTP agent able to cross HTTP proxies. + + @ivar _proxyEndpoint: The endpoint used to connect to the proxy. + + @since: 11.1 + """ + + def __init__(self, endpoint, reactor=None, pool=None): + if reactor is None: + from twisted.internet import reactor + _AgentBase.__init__(self, reactor, pool) + self._proxyEndpoint = endpoint + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request via the configured proxy. + """ + uri = _ensureValidURI(uri.strip()) + + # Cache *all* connections under the same key, since we are only + # connecting to a single destination, the proxy: + key = ("http-proxy", self._proxyEndpoint) + + # To support proxying HTTPS via CONNECT, we will use key + # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that + # wraps _proxyEndpoint with an additional callback to do the CONNECT. + return self._requestWithEndpoint(key, self._proxyEndpoint, method, + URI.fromBytes(uri), headers, + bodyProducer, uri) + + + +class _FakeUrllib2Request(object): + """ + A fake C{urllib2.Request} object for C{cookielib} to work with. + + @see: U{http://docs.python.org/library/urllib2.html#request-objects} + + @type uri: native L{str} + @ivar uri: Request URI. + + @type headers: L{twisted.web.http_headers.Headers} + @ivar headers: Request headers. + + @type type: native L{str} + @ivar type: The scheme of the URI. + + @type host: native L{str} + @ivar host: The host[:port] of the URI. + + @since: 11.1 + """ + def __init__(self, uri): + """ + Create a fake Urllib2 request. + + @param uri: Request URI. + @type uri: L{bytes} + """ + self.uri = nativeString(uri) + self.headers = Headers() + + _uri = URI.fromBytes(uri) + self.type = nativeString(_uri.scheme) + self.host = nativeString(_uri.host) + + if (_uri.scheme, _uri.port) not in ((b'http', 80), (b'https', 443)): + # If it's not a schema on the regular port, add the port. + self.host += ":" + str(_uri.port) + + if _PY3: + self.origin_req_host = nativeString(_uri.host) + self.unverifiable = lambda _: False + + + def has_header(self, header): + return self.headers.hasHeader(networkString(header)) + + + def add_unredirected_header(self, name, value): + self.headers.addRawHeader(networkString(name), networkString(value)) + + + def get_full_url(self): + return self.uri + + + def get_header(self, name, default=None): + headers = self.headers.getRawHeaders(networkString(name), default) + if headers is not None: + headers = [nativeString(x) for x in headers] + return headers[0] + return None + + + def get_host(self): + return self.host + + + def get_type(self): + return self.type + + + def is_unverifiable(self): + # In theory this shouldn't be hardcoded. + return False + + + +class _FakeUrllib2Response(object): + """ + A fake C{urllib2.Response} object for C{cookielib} to work with. + + @type response: C{twisted.web.iweb.IResponse} + @ivar response: Underlying Twisted Web response. + + @since: 11.1 + """ + def __init__(self, response): + self.response = response + + + def info(self): + class _Meta(object): + def getheaders(zelf, name): + # PY2 + headers = self.response.headers.getRawHeaders(name, []) + return headers + def get_all(zelf, name, default): + # PY3 + headers = self.response.headers.getRawHeaders( + networkString(name), default) + h = [nativeString(x) for x in headers] + return h + return _Meta() + + + +@implementer(IAgent) +class CookieAgent(object): + """ + L{CookieAgent} extends the basic L{Agent} to add RFC-compliant + handling of HTTP cookies. Cookies are written to and extracted + from a C{cookielib.CookieJar} instance. + + The same cookie jar instance will be used for any requests through this + agent, mutating it whenever a I{Set-Cookie} header appears in a response. + + @type _agent: L{twisted.web.client.Agent} + @ivar _agent: Underlying Twisted Web agent to issue requests through. + + @type cookieJar: C{cookielib.CookieJar} + @ivar cookieJar: Initialized cookie jar to read cookies from and store + cookies to. + + @since: 11.1 + """ + def __init__(self, agent, cookieJar): + self._agent = agent + self.cookieJar = cookieJar + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request to the wrapped L{Agent}. + + Send a I{Cookie} header if a cookie for C{uri} is stored in + L{CookieAgent.cookieJar}. Cookies are automatically extracted and + stored from requests. + + If a C{'cookie'} header appears in C{headers} it will override the + automatic cookie header obtained from the cookie jar. + + @see: L{Agent.request} + """ + if headers is None: + headers = Headers() + lastRequest = _FakeUrllib2Request(uri) + # Setting a cookie header explicitly will disable automatic request + # cookies. + if not headers.hasHeader(b'cookie'): + self.cookieJar.add_cookie_header(lastRequest) + cookieHeader = lastRequest.get_header('Cookie', None) + if cookieHeader is not None: + headers = headers.copy() + headers.addRawHeader(b'cookie', networkString(cookieHeader)) + + d = self._agent.request(method, uri, headers, bodyProducer) + d.addCallback(self._extractCookies, lastRequest) + return d + + + def _extractCookies(self, response, request): + """ + Extract response cookies and store them in the cookie jar. + + @type response: L{twisted.web.iweb.IResponse} + @param response: Twisted Web response. + + @param request: A urllib2 compatible request object. + """ + resp = _FakeUrllib2Response(response) + self.cookieJar.extract_cookies(resp, request) + return response + + + +class GzipDecoder(proxyForInterface(IResponse)): + """ + A wrapper for a L{Response} instance which handles gzip'ed body. + + @ivar original: The original L{Response} object. + + @since: 11.1 + """ + + def __init__(self, response): + self.original = response + self.length = UNKNOWN_LENGTH + + + def deliverBody(self, protocol): + """ + Override C{deliverBody} to wrap the given C{protocol} with + L{_GzipProtocol}. + """ + self.original.deliverBody(_GzipProtocol(protocol, self.original)) + + + +class _GzipProtocol(proxyForInterface(IProtocol)): + """ + A L{Protocol} implementation which wraps another one, transparently + decompressing received data. + + @ivar _zlibDecompress: A zlib decompress object used to decompress the data + stream. + + @ivar _response: A reference to the original response, in case of errors. + + @since: 11.1 + """ + + def __init__(self, protocol, response): + self.original = protocol + self._response = response + self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS) + + + def dataReceived(self, data): + """ + Decompress C{data} with the zlib decompressor, forwarding the raw data + to the original protocol. + """ + try: + rawData = self._zlibDecompress.decompress(data) + except zlib.error: + raise ResponseFailed([Failure()], self._response) + if rawData: + self.original.dataReceived(rawData) + + + def connectionLost(self, reason): + """ + Forward the connection lost event, flushing remaining data from the + decompressor if any. + """ + try: + rawData = self._zlibDecompress.flush() + except zlib.error: + raise ResponseFailed([reason, Failure()], self._response) + if rawData: + self.original.dataReceived(rawData) + self.original.connectionLost(reason) + + + +@implementer(IAgent) +class ContentDecoderAgent(object): + """ + An L{Agent} wrapper to handle encoded content. + + It takes care of declaring the support for content in the + I{Accept-Encoding} header, and automatically decompresses the received data + if it's effectively using compression. + + @param decoders: A list or tuple of (name, decoder) objects. The name + declares which decoding the decoder supports, and the decoder must + return a response object when called/instantiated. For example, + C{(('gzip', GzipDecoder))}. The order determines how the decoders are + going to be advertized to the server. + + @since: 11.1 + """ + + def __init__(self, agent, decoders): + self._agent = agent + self._decoders = dict(decoders) + self._supported = b','.join([decoder[0] for decoder in decoders]) + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Send a client request which declares supporting compressed content. + + @see: L{Agent.request}. + """ + if headers is None: + headers = Headers() + else: + headers = headers.copy() + headers.addRawHeader(b'accept-encoding', self._supported) + deferred = self._agent.request(method, uri, headers, bodyProducer) + return deferred.addCallback(self._handleResponse) + + + def _handleResponse(self, response): + """ + Check if the response is encoded, and wrap it to handle decompression. + """ + contentEncodingHeaders = response.headers.getRawHeaders( + b'content-encoding', []) + contentEncodingHeaders = b','.join(contentEncodingHeaders).split(b',') + while contentEncodingHeaders: + name = contentEncodingHeaders.pop().strip() + decoder = self._decoders.get(name) + if decoder is not None: + response = decoder(response) + else: + # Add it back + contentEncodingHeaders.append(name) + break + if contentEncodingHeaders: + response.headers.setRawHeaders( + b'content-encoding', [b','.join(contentEncodingHeaders)]) + else: + response.headers.removeHeader(b'content-encoding') + return response + + + +@implementer(IAgent) +class RedirectAgent(object): + """ + An L{Agent} wrapper which handles HTTP redirects. + + The implementation is rather strict: 301 and 302 behaves like 307, not + redirecting automatically on methods different from I{GET} and I{HEAD}. + + See L{BrowserLikeRedirectAgent} for a redirecting Agent that behaves more + like a web browser. + + @param redirectLimit: The maximum number of times the agent is allowed to + follow redirects before failing with a L{error.InfiniteRedirection}. + + @cvar _redirectResponses: A L{list} of HTTP status codes to be redirected + for I{GET} and I{HEAD} methods. + + @cvar _seeOtherResponses: A L{list} of HTTP status codes to be redirected + for any method and the method altered to I{GET}. + + @since: 11.1 + """ + + _redirectResponses = [http.MOVED_PERMANENTLY, http.FOUND, + http.TEMPORARY_REDIRECT] + _seeOtherResponses = [http.SEE_OTHER] + + + def __init__(self, agent, redirectLimit=20): + self._agent = agent + self._redirectLimit = redirectLimit + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Send a client request following HTTP redirects. + + @see: L{Agent.request}. + """ + deferred = self._agent.request(method, uri, headers, bodyProducer) + return deferred.addCallback( + self._handleResponse, method, uri, headers, 0) + + + def _resolveLocation(self, requestURI, location): + """ + Resolve the redirect location against the request I{URI}. + + @type requestURI: C{bytes} + @param requestURI: The request I{URI}. + + @type location: C{bytes} + @param location: The redirect location. + + @rtype: C{bytes} + @return: Final resolved I{URI}. + """ + return _urljoin(requestURI, location) + + + def _handleRedirect(self, response, method, uri, headers, redirectCount): + """ + Handle a redirect response, checking the number of redirects already + followed, and extracting the location header fields. + """ + if redirectCount >= self._redirectLimit: + err = error.InfiniteRedirection( + response.code, + b'Infinite redirection detected', + location=uri) + raise ResponseFailed([Failure(err)], response) + locationHeaders = response.headers.getRawHeaders(b'location', []) + if not locationHeaders: + err = error.RedirectWithNoLocation( + response.code, b'No location header field', uri) + raise ResponseFailed([Failure(err)], response) + location = self._resolveLocation(uri, locationHeaders[0]) + deferred = self._agent.request(method, location, headers) + def _chainResponse(newResponse): + newResponse.setPreviousResponse(response) + return newResponse + deferred.addCallback(_chainResponse) + return deferred.addCallback( + self._handleResponse, method, uri, headers, redirectCount + 1) + + + def _handleResponse(self, response, method, uri, headers, redirectCount): + """ + Handle the response, making another request if it indicates a redirect. + """ + if response.code in self._redirectResponses: + if method not in (b'GET', b'HEAD'): + err = error.PageRedirect(response.code, location=uri) + raise ResponseFailed([Failure(err)], response) + return self._handleRedirect(response, method, uri, headers, + redirectCount) + elif response.code in self._seeOtherResponses: + return self._handleRedirect(response, b'GET', uri, headers, + redirectCount) + return response + + + +class BrowserLikeRedirectAgent(RedirectAgent): + """ + An L{Agent} wrapper which handles HTTP redirects in the same fashion as web + browsers. + + Unlike L{RedirectAgent}, the implementation is more relaxed: 301 and 302 + behave like 303, redirecting automatically on any method and altering the + redirect request to a I{GET}. + + @see: L{RedirectAgent} + + @since: 13.1 + """ + _redirectResponses = [http.TEMPORARY_REDIRECT] + _seeOtherResponses = [http.MOVED_PERMANENTLY, http.FOUND, http.SEE_OTHER] + + + +class _ReadBodyProtocol(protocol.Protocol): + """ + Protocol that collects data sent to it. + + This is a helper for L{IResponse.deliverBody}, which collects the body and + fires a deferred with it. + + @ivar deferred: See L{__init__}. + @ivar status: See L{__init__}. + @ivar message: See L{__init__}. + + @ivar dataBuffer: list of byte-strings received + @type dataBuffer: L{list} of L{bytes} + """ + + def __init__(self, status, message, deferred): + """ + @param status: Status of L{IResponse} + @ivar status: L{int} + + @param message: Message of L{IResponse} + @type message: L{bytes} + + @param deferred: deferred to fire when response is complete + @type deferred: L{Deferred} firing with L{bytes} + """ + self.deferred = deferred + self.status = status + self.message = message + self.dataBuffer = [] + + + def dataReceived(self, data): + """ + Accumulate some more bytes from the response. + """ + self.dataBuffer.append(data) + + + def connectionLost(self, reason): + """ + Deliver the accumulated response bytes to the waiting L{Deferred}, if + the response body has been completely received without error. + """ + if reason.check(ResponseDone): + self.deferred.callback(b''.join(self.dataBuffer)) + elif reason.check(PotentialDataLoss): + self.deferred.errback( + PartialDownloadError(self.status, self.message, + b''.join(self.dataBuffer))) + else: + self.deferred.errback(reason) + + + +def readBody(response): + """ + Get the body of an L{IResponse} and return it as a byte string. + + This is a helper function for clients that don't want to incrementally + receive the body of an HTTP response. + + @param response: The HTTP response for which the body will be read. + @type response: L{IResponse} provider + + @return: A L{Deferred} which will fire with the body of the response. + Cancelling it will close the connection to the server immediately. + """ + def cancel(deferred): + """ + Cancel a L{readBody} call, close the connection to the HTTP server + immediately, if it is still open. + + @param deferred: The cancelled L{defer.Deferred}. + """ + abort = getAbort() + if abort is not None: + abort() + + d = defer.Deferred(cancel) + protocol = _ReadBodyProtocol(response.code, response.phrase, d) + def getAbort(): + return getattr(protocol.transport, 'abortConnection', None) + + response.deliverBody(protocol) + + if protocol.transport is not None and getAbort() is None: + warnings.warn( + 'Using readBody with a transport that does not have an ' + 'abortConnection method', + category=DeprecationWarning, + stacklevel=2) + + return d + + + +__all__ = [ + 'Agent', + 'BrowserLikePolicyForHTTPS', + 'BrowserLikeRedirectAgent', + 'ContentDecoderAgent', + 'CookieAgent', + 'downloadPage', + 'getPage', + 'GzipDecoder', + 'HTTPClientFactory', + 'HTTPConnectionPool', + 'HTTPDownloader', + 'HTTPPageDownloader', + 'HTTPPageGetter', + 'PartialDownloadError', + 'ProxyAgent', + 'readBody', + 'RedirectAgent', + 'RequestGenerationFailed', + 'RequestTransmissionFailed', + 'Response', + 'ResponseDone', + 'ResponseFailed', + 'ResponseNeverReceived', + 'URI', + ] diff --git a/contrib/python/Twisted/py2/twisted/web/demo.py b/contrib/python/Twisted/py2/twisted/web/demo.py new file mode 100644 index 00000000000..1fe83a9e4f3 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/demo.py @@ -0,0 +1,26 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +I am a simple test resource. +""" + +from __future__ import absolute_import, division + +from twisted.web import static + + +class Test(static.Data): + isLeaf = True + def __init__(self): + static.Data.__init__( + self, + b""" + + Twisted Web Demo + + Hello! This is a Twisted Web test page. + + + """, + "text/html") diff --git a/contrib/python/Twisted/py2/twisted/web/distrib.py b/contrib/python/Twisted/py2/twisted/web/distrib.py new file mode 100644 index 00000000000..38f46a0ff9d --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/distrib.py @@ -0,0 +1,386 @@ +# -*- test-case-name: twisted.web.test.test_distrib -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Distributed web servers. + +This is going to have to be refactored so that argument parsing is done +by each subprocess and not by the main web server (i.e. GET, POST etc.). +""" + +# System Imports +import os, copy +try: + import pwd +except ImportError: + pwd = None +from io import BytesIO + +from xml.dom.minidom import getDOMImplementation + +# Twisted Imports +from twisted.spread import pb +from twisted.spread.banana import SIZE_LIMIT +from twisted.web import http, resource, server, util, static +from twisted.web.http_headers import Headers +from twisted.persisted import styles +from twisted.internet import address, reactor +from twisted.logger import Logger + + +class _ReferenceableProducerWrapper(pb.Referenceable): + def __init__(self, producer): + self.producer = producer + + def remote_resumeProducing(self): + self.producer.resumeProducing() + + def remote_pauseProducing(self): + self.producer.pauseProducing() + + def remote_stopProducing(self): + self.producer.stopProducing() + + +class Request(pb.RemoteCopy, server.Request): + """ + A request which was received by a L{ResourceSubscription} and sent via + PB to a distributed node. + """ + def setCopyableState(self, state): + """ + Initialize this L{twisted.web.distrib.Request} based on the copied + state so that it closely resembles a L{twisted.web.server.Request}. + """ + for k in 'host', 'client': + tup = state[k] + addrdesc = {'INET': 'TCP', 'UNIX': 'UNIX'}[tup[0]] + addr = {'TCP': lambda: address.IPv4Address(addrdesc, + tup[1], tup[2]), + 'UNIX': lambda: address.UNIXAddress(tup[1])}[addrdesc]() + state[k] = addr + state['requestHeaders'] = Headers(dict(state['requestHeaders'])) + pb.RemoteCopy.setCopyableState(self, state) + # Emulate the local request interface -- + self.content = BytesIO(self.content_data) + self.finish = self.remote.remoteMethod('finish') + self.setHeader = self.remote.remoteMethod('setHeader') + self.addCookie = self.remote.remoteMethod('addCookie') + self.setETag = self.remote.remoteMethod('setETag') + self.setResponseCode = self.remote.remoteMethod('setResponseCode') + self.setLastModified = self.remote.remoteMethod('setLastModified') + + # To avoid failing if a resource tries to write a very long string + # all at once, this one will be handled slightly differently. + self._write = self.remote.remoteMethod('write') + + + def write(self, bytes): + """ + Write the given bytes to the response body. + + @param bytes: The bytes to write. If this is longer than 640k, it + will be split up into smaller pieces. + """ + start = 0 + end = SIZE_LIMIT + while True: + self._write(bytes[start:end]) + start += SIZE_LIMIT + end += SIZE_LIMIT + if start >= len(bytes): + break + + + def registerProducer(self, producer, streaming): + self.remote.callRemote("registerProducer", + _ReferenceableProducerWrapper(producer), + streaming).addErrback(self.fail) + + def unregisterProducer(self): + self.remote.callRemote("unregisterProducer").addErrback(self.fail) + + def fail(self, failure): + self._log.failure('', failure=failure) + + +pb.setUnjellyableForClass(server.Request, Request) + +class Issue: + _log = Logger() + + def __init__(self, request): + self.request = request + + def finished(self, result): + if result is not server.NOT_DONE_YET: + assert isinstance(result, str), "return value not a string" + self.request.write(result) + self.request.finish() + + def failed(self, failure): + #XXX: Argh. FIXME. + failure = str(failure) + self.request.write( + resource.ErrorPage(http.INTERNAL_SERVER_ERROR, + "Server Connection Lost", + "Connection to distributed server lost:" + + util._PRE(failure)). + render(self.request)) + self.request.finish() + self._log.info(failure) + + +class ResourceSubscription(resource.Resource): + isLeaf = 1 + waiting = 0 + _log = Logger() + + def __init__(self, host, port): + resource.Resource.__init__(self) + self.host = host + self.port = port + self.pending = [] + self.publisher = None + + def __getstate__(self): + """Get persistent state for this ResourceSubscription. + """ + # When I unserialize, + state = copy.copy(self.__dict__) + # Publisher won't be connected... + state['publisher'] = None + # I won't be making a connection + state['waiting'] = 0 + # There will be no pending requests. + state['pending'] = [] + return state + + def connected(self, publisher): + """I've connected to a publisher; I'll now send all my requests. + """ + self._log.info('connected to publisher') + publisher.broker.notifyOnDisconnect(self.booted) + self.publisher = publisher + self.waiting = 0 + for request in self.pending: + self.render(request) + self.pending = [] + + def notConnected(self, msg): + """I can't connect to a publisher; I'll now reply to all pending + requests. + """ + self._log.info( + "could not connect to distributed web service: {msg}", + msg=msg + ) + self.waiting = 0 + self.publisher = None + for request in self.pending: + request.write("Unable to connect to distributed server.") + request.finish() + self.pending = [] + + def booted(self): + self.notConnected("connection dropped") + + def render(self, request): + """Render this request, from my server. + + This will always be asynchronous, and therefore return NOT_DONE_YET. + It spins off a request to the pb client, and either adds it to the list + of pending issues or requests it immediately, depending on if the + client is already connected. + """ + if not self.publisher: + self.pending.append(request) + if not self.waiting: + self.waiting = 1 + bf = pb.PBClientFactory() + timeout = 10 + if self.host == "unix": + reactor.connectUNIX(self.port, bf, timeout) + else: + reactor.connectTCP(self.host, self.port, bf, timeout) + d = bf.getRootObject() + d.addCallbacks(self.connected, self.notConnected) + + else: + i = Issue(request) + self.publisher.callRemote('request', request).addCallbacks(i.finished, i.failed) + return server.NOT_DONE_YET + + + +class ResourcePublisher(pb.Root, styles.Versioned): + """ + L{ResourcePublisher} exposes a remote API which can be used to respond + to request. + + @ivar site: The site which will be used for resource lookup. + @type site: L{twisted.web.server.Site} + """ + _log = Logger() + + def __init__(self, site): + self.site = site + + persistenceVersion = 2 + + def upgradeToVersion2(self): + self.application.authorizer.removeIdentity("web") + del self.application.services[self.serviceName] + del self.serviceName + del self.application + del self.perspectiveName + + def getPerspectiveNamed(self, name): + return self + + + def remote_request(self, request): + """ + Look up the resource for the given request and render it. + """ + res = self.site.getResourceFor(request) + self._log.info(request) + result = res.render(request) + if result is not server.NOT_DONE_YET: + request.write(result) + request.finish() + return server.NOT_DONE_YET + + + +class UserDirectory(resource.Resource): + """ + A resource which lists available user resources and serves them as + children. + + @ivar _pwd: An object like L{pwd} which is used to enumerate users and + their home directories. + """ + + userDirName = 'public_html' + userSocketName = '.twistd-web-pb' + + template = """ + + + twisted.web.distrib.UserDirectory + + + + +

twisted.web.distrib.UserDirectory

+ + %(users)s + + +""" + + def __init__(self, userDatabase=None): + resource.Resource.__init__(self) + if userDatabase is None: + userDatabase = pwd + self._pwd = userDatabase + + + def _users(self): + """ + Return a list of two-tuples giving links to user resources and text to + associate with those links. + """ + users = [] + for user in self._pwd.getpwall(): + name, passwd, uid, gid, gecos, dir, shell = user + realname = gecos.split(',')[0] + if not realname: + realname = name + if os.path.exists(os.path.join(dir, self.userDirName)): + users.append((name, realname + ' (file)')) + twistdsock = os.path.join(dir, self.userSocketName) + if os.path.exists(twistdsock): + linkName = name + '.twistd' + users.append((linkName, realname + ' (twistd)')) + return users + + + def render_GET(self, request): + """ + Render as HTML a listing of all known users with links to their + personal resources. + """ + + domImpl = getDOMImplementation() + newDoc = domImpl.createDocument(None, "ul", None) + listing = newDoc.documentElement + for link, text in self._users(): + linkElement = newDoc.createElement('a') + linkElement.setAttribute('href', link + '/') + textNode = newDoc.createTextNode(text) + linkElement.appendChild(textNode) + item = newDoc.createElement('li') + item.appendChild(linkElement) + listing.appendChild(item) + + htmlDoc = self.template % ({'users': listing.toxml()}) + return htmlDoc.encode("utf-8") + + + def getChild(self, name, request): + if name == '': + return self + + td = '.twistd' + + if name[-len(td):] == td: + username = name[:-len(td)] + sub = 1 + else: + username = name + sub = 0 + try: + pw_name, pw_passwd, pw_uid, pw_gid, pw_gecos, pw_dir, pw_shell \ + = self._pwd.getpwnam(username) + except KeyError: + return resource.NoResource() + if sub: + twistdsock = os.path.join(pw_dir, self.userSocketName) + rs = ResourceSubscription('unix',twistdsock) + self.putChild(name, rs) + return rs + else: + path = os.path.join(pw_dir, self.userDirName) + if not os.path.exists(path): + return resource.NoResource() + return static.File(path) diff --git a/contrib/python/Twisted/py2/twisted/web/domhelpers.py b/contrib/python/Twisted/py2/twisted/web/domhelpers.py new file mode 100644 index 00000000000..1ca491b470c --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/domhelpers.py @@ -0,0 +1,272 @@ +# -*- test-case-name: twisted.web.test.test_domhelpers -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +A library for performing interesting tasks with DOM objects. +""" + +from io import StringIO + +from twisted.web import microdom +from twisted.web.microdom import getElementsByTagName, escape, unescape +# These modules are imported here as a shortcut. +escape +getElementsByTagName + + + +class NodeLookupError(Exception): + pass + + +def substitute(request, node, subs): + """ + Look through the given node's children for strings, and + attempt to do string substitution with the given parameter. + """ + for child in node.childNodes: + if hasattr(child, 'nodeValue') and child.nodeValue: + child.replaceData(0, len(child.nodeValue), child.nodeValue % subs) + substitute(request, child, subs) + +def _get(node, nodeId, nodeAttrs=('id','class','model','pattern')): + """ + (internal) Get a node with the specified C{nodeId} as any of the C{class}, + C{id} or C{pattern} attributes. + """ + + if hasattr(node, 'hasAttributes') and node.hasAttributes(): + for nodeAttr in nodeAttrs: + if (str (node.getAttribute(nodeAttr)) == nodeId): + return node + if node.hasChildNodes(): + if hasattr(node.childNodes, 'length'): + length = node.childNodes.length + else: + length = len(node.childNodes) + for childNum in range(length): + result = _get(node.childNodes[childNum], nodeId) + if result: return result + +def get(node, nodeId): + """ + Get a node with the specified C{nodeId} as any of the C{class}, + C{id} or C{pattern} attributes. If there is no such node, raise + L{NodeLookupError}. + """ + result = _get(node, nodeId) + if result: return result + raise NodeLookupError(nodeId) + +def getIfExists(node, nodeId): + """ + Get a node with the specified C{nodeId} as any of the C{class}, + C{id} or C{pattern} attributes. If there is no such node, return + L{None}. + """ + return _get(node, nodeId) + +def getAndClear(node, nodeId): + """Get a node with the specified C{nodeId} as any of the C{class}, + C{id} or C{pattern} attributes. If there is no such node, raise + L{NodeLookupError}. Remove all child nodes before returning. + """ + result = get(node, nodeId) + if result: + clearNode(result) + return result + +def clearNode(node): + """ + Remove all children from the given node. + """ + node.childNodes[:] = [] + +def locateNodes(nodeList, key, value, noNesting=1): + """ + Find subnodes in the given node where the given attribute + has the given value. + """ + returnList = [] + if not isinstance(nodeList, type([])): + return locateNodes(nodeList.childNodes, key, value, noNesting) + for childNode in nodeList: + if not hasattr(childNode, 'getAttribute'): + continue + if str(childNode.getAttribute(key)) == value: + returnList.append(childNode) + if noNesting: + continue + returnList.extend(locateNodes(childNode, key, value, noNesting)) + return returnList + +def superSetAttribute(node, key, value): + if not hasattr(node, 'setAttribute'): return + node.setAttribute(key, value) + if node.hasChildNodes(): + for child in node.childNodes: + superSetAttribute(child, key, value) + +def superPrependAttribute(node, key, value): + if not hasattr(node, 'setAttribute'): return + old = node.getAttribute(key) + if old: + node.setAttribute(key, value+'/'+old) + else: + node.setAttribute(key, value) + if node.hasChildNodes(): + for child in node.childNodes: + superPrependAttribute(child, key, value) + +def superAppendAttribute(node, key, value): + if not hasattr(node, 'setAttribute'): return + old = node.getAttribute(key) + if old: + node.setAttribute(key, old + '/' + value) + else: + node.setAttribute(key, value) + if node.hasChildNodes(): + for child in node.childNodes: + superAppendAttribute(child, key, value) + +def gatherTextNodes(iNode, dounescape=0, joinWith=""): + """Visit each child node and collect its text data, if any, into a string. +For example:: + >>> doc=microdom.parseString('1234') + >>> gatherTextNodes(doc.documentElement) + '1234' +With dounescape=1, also convert entities back into normal characters. +@return: the gathered nodes as a single string +@rtype: str +""" + gathered=[] + gathered_append=gathered.append + slice=[iNode] + while len(slice)>0: + c=slice.pop(0) + if hasattr(c, 'nodeValue') and c.nodeValue is not None: + if dounescape: + val=unescape(c.nodeValue) + else: + val=c.nodeValue + gathered_append(val) + slice[:0]=c.childNodes + return joinWith.join(gathered) + +class RawText(microdom.Text): + """This is an evil and horrible speed hack. Basically, if you have a big + chunk of XML that you want to insert into the DOM, but you don't want to + incur the cost of parsing it, you can construct one of these and insert it + into the DOM. This will most certainly only work with microdom as the API + for converting nodes to xml is different in every DOM implementation. + + This could be improved by making this class a Lazy parser, so if you + inserted this into the DOM and then later actually tried to mutate this + node, it would be parsed then. + """ + + def writexml(self, writer, indent="", addindent="", newl="", strip=0, nsprefixes=None, namespace=None): + writer.write("%s%s%s" % (indent, self.data, newl)) + +def findNodes(parent, matcher, accum=None): + if accum is None: + accum = [] + if not parent.hasChildNodes(): + return accum + for child in parent.childNodes: + # print child, child.nodeType, child.nodeName + if matcher(child): + accum.append(child) + findNodes(child, matcher, accum) + return accum + + +def findNodesShallowOnMatch(parent, matcher, recurseMatcher, accum=None): + if accum is None: + accum = [] + if not parent.hasChildNodes(): + return accum + for child in parent.childNodes: + # print child, child.nodeType, child.nodeName + if matcher(child): + accum.append(child) + if recurseMatcher(child): + findNodesShallowOnMatch(child, matcher, recurseMatcher, accum) + return accum + +def findNodesShallow(parent, matcher, accum=None): + if accum is None: + accum = [] + if not parent.hasChildNodes(): + return accum + for child in parent.childNodes: + if matcher(child): + accum.append(child) + else: + findNodes(child, matcher, accum) + return accum + + +def findElementsWithAttributeShallow(parent, attribute): + """ + Return an iterable of the elements which are direct children of C{parent} + and which have the C{attribute} attribute. + """ + return findNodesShallow(parent, + lambda n: getattr(n, 'tagName', None) is not None and + n.hasAttribute(attribute)) + + +def findElements(parent, matcher): + """ + Return an iterable of the elements which are children of C{parent} for + which the predicate C{matcher} returns true. + """ + return findNodes( + parent, + lambda n, matcher=matcher: getattr(n, 'tagName', None) is not None and + matcher(n)) + +def findElementsWithAttribute(parent, attribute, value=None): + if value: + return findElements( + parent, + lambda n, attribute=attribute, value=value: + n.hasAttribute(attribute) and n.getAttribute(attribute) == value) + else: + return findElements( + parent, + lambda n, attribute=attribute: n.hasAttribute(attribute)) + + +def findNodesNamed(parent, name): + return findNodes(parent, lambda n, name=name: n.nodeName == name) + + +def writeNodeData(node, oldio): + for subnode in node.childNodes: + if hasattr(subnode, 'data'): + oldio.write(u"" + subnode.data) + else: + writeNodeData(subnode, oldio) + + +def getNodeText(node): + oldio = StringIO() + writeNodeData(node, oldio) + return oldio.getvalue() + + +def getParents(node): + l = [] + while node: + l.append(node) + node = node.parentNode + return l + +def namedChildren(parent, nodeName): + """namedChildren(parent, nodeName) -> children (not descendants) of parent + that have tagName == nodeName + """ + return [n for n in parent.childNodes if getattr(n, 'tagName', '')==nodeName] diff --git a/contrib/python/Twisted/py2/twisted/web/error.py b/contrib/python/Twisted/py2/twisted/web/error.py new file mode 100644 index 00000000000..e3456a4b4d6 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/error.py @@ -0,0 +1,407 @@ +# -*- test-case-name: twisted.web.test.test_error -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Exception definitions for L{twisted.web}. +""" + +from __future__ import division, absolute_import +try: + from future_builtins import ascii +except ImportError: + pass + +__all__ = [ + 'Error', 'PageRedirect', 'InfiniteRedirection', 'RenderError', + 'MissingRenderMethod', 'MissingTemplateLoader', 'UnexposedMethodError', + 'UnfilledSlot', 'UnsupportedType', 'FlattenerError', + 'RedirectWithNoLocation', + ] + + +from twisted.web._responses import RESPONSES +from twisted.python.compat import unicode, nativeString, intToBytes, Sequence + + + +def _codeToMessage(code): + """ + Returns the response message corresponding to an HTTP code, or None + if the code is unknown or unrecognized. + + @type code: L{bytes} + @param code: Refers to an HTTP status code, for example C{http.NOT_FOUND}. + + @return: A string message or none + @rtype: L{bytes} + """ + try: + return RESPONSES.get(int(code)) + except (ValueError, AttributeError): + return None + + +class Error(Exception): + """ + A basic HTTP error. + + @type status: L{bytes} + @ivar status: Refers to an HTTP status code, for example C{http.NOT_FOUND}. + + @type message: L{bytes} + @param message: A short error message, for example "NOT FOUND". + + @type response: L{bytes} + @ivar response: A complete HTML document for an error page. + """ + def __init__(self, code, message=None, response=None): + """ + Initializes a basic exception. + + @type code: L{bytes} or L{int} + @param code: Refers to an HTTP status code (for example, 200) either as + an integer or a bytestring representing such. If no C{message} is + given, C{code} is mapped to a descriptive bytestring that is used + instead. + + @type message: L{bytes} + @param message: A short error message, for example "NOT FOUND". + + @type response: L{bytes} + @param response: A complete HTML document for an error page. + """ + message = message or _codeToMessage(code) + + Exception.__init__(self, code, message, response) + + if isinstance(code, int): + # If we're given an int, convert it to a bytestring + # downloadPage gives a bytes, Agent gives an int, and it worked by + # accident previously, so just make it keep working. + code = intToBytes(code) + + self.status = code + self.message = message + self.response = response + + + def __str__(self): + return nativeString(self.status + b" " + self.message) + + + +class PageRedirect(Error): + """ + A request resulted in an HTTP redirect. + + @type location: L{bytes} + @ivar location: The location of the redirect which was not followed. + """ + def __init__(self, code, message=None, response=None, location=None): + """ + Initializes a page redirect exception. + + @type code: L{bytes} + @param code: Refers to an HTTP status code, for example + C{http.NOT_FOUND}. If no C{message} is given, C{code} is mapped to a + descriptive string that is used instead. + + @type message: L{bytes} + @param message: A short error message, for example "NOT FOUND". + + @type response: L{bytes} + @param response: A complete HTML document for an error page. + + @type location: L{bytes} + @param location: The location response-header field value. It is an + absolute URI used to redirect the receiver to a location other than + the Request-URI so the request can be completed. + """ + Error.__init__(self, code, message, response) + if self.message and location: + self.message = self.message + b" to " + location + self.location = location + + + +class InfiniteRedirection(Error): + """ + HTTP redirection is occurring endlessly. + + @type location: L{bytes} + @ivar location: The first URL in the series of redirections which was + not followed. + """ + def __init__(self, code, message=None, response=None, location=None): + """ + Initializes an infinite redirection exception. + + @type code: L{bytes} + @param code: Refers to an HTTP status code, for example + C{http.NOT_FOUND}. If no C{message} is given, C{code} is mapped to a + descriptive string that is used instead. + + @type message: L{bytes} + @param message: A short error message, for example "NOT FOUND". + + @type response: L{bytes} + @param response: A complete HTML document for an error page. + + @type location: L{bytes} + @param location: The location response-header field value. It is an + absolute URI used to redirect the receiver to a location other than + the Request-URI so the request can be completed. + """ + Error.__init__(self, code, message, response) + if self.message and location: + self.message = self.message + b" to " + location + self.location = location + + + +class RedirectWithNoLocation(Error): + """ + Exception passed to L{ResponseFailed} if we got a redirect without a + C{Location} header field. + + @type uri: L{bytes} + @ivar uri: The URI which failed to give a proper location header + field. + + @since: 11.1 + """ + + def __init__(self, code, message, uri): + """ + Initializes a page redirect exception when no location is given. + + @type code: L{bytes} + @param code: Refers to an HTTP status code, for example + C{http.NOT_FOUND}. If no C{message} is given, C{code} is mapped to + a descriptive string that is used instead. + + @type message: L{bytes} + @param message: A short error message. + + @type uri: L{bytes} + @param uri: The URI which failed to give a proper location header + field. + """ + Error.__init__(self, code, message) + self.message = self.message + b" to " + uri + self.uri = uri + + + +class UnsupportedMethod(Exception): + """ + Raised by a resource when faced with a strange request method. + + RFC 2616 (HTTP 1.1) gives us two choices when faced with this situation: + If the type of request is known to us, but not allowed for the requested + resource, respond with NOT_ALLOWED. Otherwise, if the request is something + we don't know how to deal with in any case, respond with NOT_IMPLEMENTED. + + When this exception is raised by a Resource's render method, the server + will make the appropriate response. + + This exception's first argument MUST be a sequence of the methods the + resource *does* support. + """ + + allowedMethods = () + + def __init__(self, allowedMethods, *args): + Exception.__init__(self, allowedMethods, *args) + self.allowedMethods = allowedMethods + + if not isinstance(allowedMethods, Sequence): + raise TypeError( + "First argument must be a sequence of supported methods, " + "but my first argument is not a sequence.") + + + def __str__(self): + return "Expected one of %r" % (self.allowedMethods,) + + + +class SchemeNotSupported(Exception): + """ + The scheme of a URI was not one of the supported values. + """ + + + +class RenderError(Exception): + """ + Base exception class for all errors which can occur during template + rendering. + """ + + + +class MissingRenderMethod(RenderError): + """ + Tried to use a render method which does not exist. + + @ivar element: The element which did not have the render method. + @ivar renderName: The name of the renderer which could not be found. + """ + def __init__(self, element, renderName): + RenderError.__init__(self, element, renderName) + self.element = element + self.renderName = renderName + + + def __repr__(self): + return '%r: %r had no render method named %r' % ( + self.__class__.__name__, self.element, self.renderName) + + + +class MissingTemplateLoader(RenderError): + """ + L{MissingTemplateLoader} is raised when trying to render an Element without + a template loader, i.e. a C{loader} attribute. + + @ivar element: The Element which did not have a document factory. + """ + def __init__(self, element): + RenderError.__init__(self, element) + self.element = element + + + def __repr__(self): + return '%r: %r had no loader' % (self.__class__.__name__, + self.element) + + + +class UnexposedMethodError(Exception): + """ + Raised on any attempt to get a method which has not been exposed. + """ + + + +class UnfilledSlot(Exception): + """ + During flattening, a slot with no associated data was encountered. + """ + + + +class UnsupportedType(Exception): + """ + During flattening, an object of a type which cannot be flattened was + encountered. + """ + + +class ExcessiveBufferingError(Exception): + """ + The HTTP/2 protocol has been forced to buffer an excessive amount of + outbound data, and has therefore closed the connection and dropped all + outbound data. + """ + + + +class FlattenerError(Exception): + """ + An error occurred while flattening an object. + + @ivar _roots: A list of the objects on the flattener's stack at the time + the unflattenable object was encountered. The first element is least + deeply nested object and the last element is the most deeply nested. + """ + def __init__(self, exception, roots, traceback): + self._exception = exception + self._roots = roots + self._traceback = traceback + Exception.__init__(self, exception, roots, traceback) + + + def _formatRoot(self, obj): + """ + Convert an object from C{self._roots} to a string suitable for + inclusion in a render-traceback (like a normal Python traceback, but + can include "frame" source locations which are not in Python source + files). + + @param obj: Any object which can be a render step I{root}. + Typically, L{Tag}s, strings, and other simple Python types. + + @return: A string representation of C{obj}. + @rtype: L{str} + """ + # There's a circular dependency between this class and 'Tag', although + # only for an isinstance() check. + from twisted.web.template import Tag + + if isinstance(obj, (bytes, str, unicode)): + # It's somewhat unlikely that there will ever be a str in the roots + # list. However, something like a MemoryError during a str.replace + # call (eg, replacing " with ") could possibly cause this. + # Likewise, UTF-8 encoding a unicode string to a byte string might + # fail like this. + if len(obj) > 40: + if isinstance(obj, unicode): + ellipsis = u'<...>' + else: + ellipsis = b'<...>' + return ascii(obj[:20] + ellipsis + obj[-20:]) + else: + return ascii(obj) + elif isinstance(obj, Tag): + if obj.filename is None: + return 'Tag <' + obj.tagName + '>' + else: + return "File \"%s\", line %d, column %d, in \"%s\"" % ( + obj.filename, obj.lineNumber, + obj.columnNumber, obj.tagName) + else: + return ascii(obj) + + + def __repr__(self): + """ + Present a string representation which includes a template traceback, so + we can tell where this error occurred in the template, as well as in + Python. + """ + # Avoid importing things unnecessarily until we actually need them; + # since this is an 'error' module we should be extra paranoid about + # that. + from traceback import format_list + if self._roots: + roots = ' ' + '\n '.join([ + self._formatRoot(r) for r in self._roots]) + '\n' + else: + roots = '' + if self._traceback: + traceback = '\n'.join([ + line + for entry in format_list(self._traceback) + for line in entry.splitlines()]) + '\n' + else: + traceback = '' + return ( + 'Exception while flattening:\n' + + roots + traceback + + self._exception.__class__.__name__ + ': ' + + str(self._exception) + '\n') + + + def __str__(self): + return repr(self) + + + +class UnsupportedSpecialHeader(Exception): + """ + A HTTP/2 request was received that contained a HTTP/2 pseudo-header field + that is not recognised by Twisted. + """ diff --git a/contrib/python/Twisted/py2/twisted/web/guard.py b/contrib/python/Twisted/py2/twisted/web/guard.py new file mode 100644 index 00000000000..0e580815edf --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/guard.py @@ -0,0 +1,20 @@ +# -*- test-case-name: twisted.web.test.test_httpauth -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Resource traversal integration with L{twisted.cred} to allow for +authentication and authorization of HTTP requests. +""" + +from __future__ import division, absolute_import + +# Expose HTTP authentication classes here. +from twisted.web._auth.wrapper import HTTPAuthSessionWrapper +from twisted.web._auth.basic import BasicCredentialFactory +from twisted.web._auth.digest import DigestCredentialFactory + +__all__ = [ + "HTTPAuthSessionWrapper", + + "BasicCredentialFactory", "DigestCredentialFactory"] diff --git a/contrib/python/Twisted/py2/twisted/web/html.py b/contrib/python/Twisted/py2/twisted/web/html.py new file mode 100644 index 00000000000..5605f5f46a1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/web/html.py @@ -0,0 +1,57 @@ +# -*- test-case-name: twisted.web.test.test_html -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +"""I hold HTML generation helpers. +""" + +from twisted.python import log +from twisted.python.compat import NativeStringIO as StringIO, escape +from twisted.python.deprecate import deprecated +from incremental import Version + + + +@deprecated(Version('Twisted', 15, 3, 0), replacement='twisted.web.template') +def PRE(text): + "Wrap 
 tags around some text and HTML-escape it."
+    return "
"+escape(text)+"
"
+
+
+
+@deprecated(Version('Twisted', 15, 3, 0), replacement='twisted.web.template')
+def UL(lst):
+    io = StringIO()
+    io.write("
    \n")
+    for el in lst:
+        io.write("
  •  %s
    • \n" % el)
+    io.write("
")
+    return io.getvalue()
+
+
+
+@deprecated(Version('Twisted', 15, 3, 0), replacement='twisted.web.template')
+def linkList(lst):
+    io = StringIO()
+    io.write("
    \n")
+    for hr, el in lst:
+        io.write('
  •  %s
    • \n' % (hr, el))
+    io.write("
")
+    return io.getvalue()
+
+
+
+@deprecated(Version('Twisted', 15, 3, 0), replacement='twisted.web.template')
+def output(func, *args, **kw):
+    """output(func, *args, **kw) -> html string
+    Either return the result of a function (which presumably returns an
+    HTML-legal string) or a sparse HTMLized error message and a message
+    in the server log.
+    """
+    try:
+        return func(*args, **kw)
+    except:
+        log.msg("Error calling %r:" % (func,))
+        log.err()
+        return PRE("An error occurred.")
diff --git a/contrib/python/Twisted/py2/twisted/web/http.py b/contrib/python/Twisted/py2/twisted/web/http.py
new file mode 100644
index 00000000000..b7afa8b0d0a
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/http.py
@@ -0,0 +1,3170 @@
+# -*- test-case-name: twisted.web.test.test_http -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HyperText Transfer Protocol implementation.
+
+This is the basic server-side protocol implementation used by the Twisted
+Web server.  It can parse HTTP 1.0 requests and supports many HTTP 1.1
+features as well.  Additionally, some functionality implemented here is
+also useful for HTTP clients (such as the chunked encoding parser).
+
+@var CACHED: A marker value to be returned from cache-related request methods
+    to indicate to the caller that a cached response will be usable and no
+    response body should be generated.
+
+@var FOUND: An HTTP response code indicating a temporary redirect.
+
+@var NOT_MODIFIED: An HTTP response code indicating that a requested
+    pre-condition (for example, the condition represented by an
+    I{If-Modified-Since} header is present in the request) has succeeded.  This
+    indicates a response body cached by the client can be used.
+
+@var PRECONDITION_FAILED: An HTTP response code indicating that a requested
+    pre-condition (for example, the condition represented by an I{If-None-Match}
+    header is present in the request) has failed.  This should typically
+    indicate that the server has not taken the requested action.
+"""
+
+from __future__ import division, absolute_import
+
+__all__ = [
+    'SWITCHING', 'OK', 'CREATED', 'ACCEPTED', 'NON_AUTHORITATIVE_INFORMATION',
+    'NO_CONTENT', 'RESET_CONTENT', 'PARTIAL_CONTENT', 'MULTI_STATUS',
+
+    'MULTIPLE_CHOICE', 'MOVED_PERMANENTLY', 'FOUND', 'SEE_OTHER',
+    'NOT_MODIFIED', 'USE_PROXY', 'TEMPORARY_REDIRECT',
+
+    'BAD_REQUEST', 'UNAUTHORIZED', 'PAYMENT_REQUIRED', 'FORBIDDEN', 'NOT_FOUND',
+    'NOT_ALLOWED', 'NOT_ACCEPTABLE', 'PROXY_AUTH_REQUIRED', 'REQUEST_TIMEOUT',
+    'CONFLICT', 'GONE', 'LENGTH_REQUIRED', 'PRECONDITION_FAILED',
+    'REQUEST_ENTITY_TOO_LARGE', 'REQUEST_URI_TOO_LONG',
+    'UNSUPPORTED_MEDIA_TYPE', 'REQUESTED_RANGE_NOT_SATISFIABLE',
+    'EXPECTATION_FAILED',
+
+    'INTERNAL_SERVER_ERROR', 'NOT_IMPLEMENTED', 'BAD_GATEWAY',
+    'SERVICE_UNAVAILABLE', 'GATEWAY_TIMEOUT', 'HTTP_VERSION_NOT_SUPPORTED',
+    'INSUFFICIENT_STORAGE_SPACE', 'NOT_EXTENDED',
+
+    'RESPONSES', 'CACHED',
+
+    'urlparse', 'parse_qs', 'datetimeToString', 'datetimeToLogString', 'timegm',
+    'stringToDatetime', 'toChunk', 'fromChunk', 'parseContentRange',
+
+    'StringTransport', 'HTTPClient', 'NO_BODY_CODES', 'Request',
+    'PotentialDataLoss', 'HTTPChannel', 'HTTPFactory',
+    ]
+
+
+# system imports
+import tempfile
+import base64, binascii
+import cgi
+import math
+import time
+import calendar
+import warnings
+import os
+from io import BytesIO as StringIO
+
+try:
+    from urlparse import (
+        ParseResult as ParseResultBytes, urlparse as _urlparse)
+    from urllib import unquote
+    from cgi import parse_header as _parseHeader
+except ImportError:
+    from urllib.parse import (
+        ParseResultBytes, urlparse as _urlparse, unquote_to_bytes as unquote)
+
+    def _parseHeader(line):
+        # cgi.parse_header requires a str
+        key, pdict = cgi.parse_header(line.decode('charmap'))
+
+        # We want the key as bytes, and cgi.parse_multipart (which consumes
+        # pdict) expects a dict of str keys but bytes values
+        key = key.encode('charmap')
+        pdict = {x:y.encode('charmap') for x, y in pdict.items()}
+        return (key, pdict)
+
+
+from zope.interface import Attribute, Interface, implementer, provider
+
+# twisted imports
+from twisted.python.compat import (
+    _PY3, long, unicode, intToBytes, networkString, nativeString, _PY37PLUS)
+from twisted.python.deprecate import deprecated
+from twisted.python import log
+from twisted.logger import Logger
+from twisted.python.failure import Failure
+from incremental import Version
+from twisted.python.components import proxyForInterface
+from twisted.internet import interfaces, protocol, address
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IProtocol
+from twisted.internet._producer_helpers import _PullToPush
+from twisted.protocols import policies, basic
+
+from twisted.web.iweb import (
+    IRequest, IAccessLogFormatter, INonQueuedRequestFactory)
+from twisted.web.http_headers import Headers, _sanitizeLinearWhitespace
+
+try:
+    from twisted.web._http2 import H2Connection
+    H2_ENABLED = True
+except ImportError:
+    H2Connection = None
+    H2_ENABLED = False
+
+
+from twisted.web._responses import (
+    SWITCHING,
+
+    OK, CREATED, ACCEPTED, NON_AUTHORITATIVE_INFORMATION, NO_CONTENT,
+    RESET_CONTENT, PARTIAL_CONTENT, MULTI_STATUS,
+
+    MULTIPLE_CHOICE, MOVED_PERMANENTLY, FOUND, SEE_OTHER, NOT_MODIFIED,
+    USE_PROXY, TEMPORARY_REDIRECT,
+
+    BAD_REQUEST, UNAUTHORIZED, PAYMENT_REQUIRED, FORBIDDEN, NOT_FOUND,
+    NOT_ALLOWED, NOT_ACCEPTABLE, PROXY_AUTH_REQUIRED, REQUEST_TIMEOUT,
+    CONFLICT, GONE, LENGTH_REQUIRED, PRECONDITION_FAILED,
+    REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG, UNSUPPORTED_MEDIA_TYPE,
+    REQUESTED_RANGE_NOT_SATISFIABLE, EXPECTATION_FAILED,
+
+    INTERNAL_SERVER_ERROR, NOT_IMPLEMENTED, BAD_GATEWAY, SERVICE_UNAVAILABLE,
+    GATEWAY_TIMEOUT, HTTP_VERSION_NOT_SUPPORTED, INSUFFICIENT_STORAGE_SPACE,
+    NOT_EXTENDED,
+
+    RESPONSES)
+
+
+_intTypes = (int, long)
+
+# A common request timeout -- 1 minute. This is roughly what nginx uses, and
+# so it seems to be a good choice for us too.
+_REQUEST_TIMEOUT = 1 * 60
+
+protocol_version = "HTTP/1.1"
+
+CACHED = """Magic constant returned by http.Request methods to set cache
+validation headers when the request is conditional and the value fails
+the condition."""
+
+# backwards compatibility
+responses = RESPONSES
+
+
+# datetime parsing and formatting
+weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
+monthname = [None,
+             'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+             'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+weekdayname_lower = [name.lower() for name in weekdayname]
+monthname_lower = [name and name.lower() for name in monthname]
+
+def urlparse(url):
+    """
+    Parse an URL into six components.
+
+    This is similar to C{urlparse.urlparse}, but rejects C{unicode} input
+    and always produces C{bytes} output.
+
+    @type url: C{bytes}
+
+    @raise TypeError: The given url was a C{unicode} string instead of a
+        C{bytes}.
+
+    @return: The scheme, net location, path, params, query string, and fragment
+        of the URL - all as C{bytes}.
+    @rtype: C{ParseResultBytes}
+    """
+    if isinstance(url, unicode):
+        raise TypeError("url must be bytes, not unicode")
+    scheme, netloc, path, params, query, fragment = _urlparse(url)
+    if isinstance(scheme, unicode):
+        scheme = scheme.encode('ascii')
+        netloc = netloc.encode('ascii')
+        path = path.encode('ascii')
+        query = query.encode('ascii')
+        fragment = fragment.encode('ascii')
+    return ParseResultBytes(scheme, netloc, path, params, query, fragment)
+
+
+
+def parse_qs(qs, keep_blank_values=0, strict_parsing=0):
+    """
+    Like C{cgi.parse_qs}, but with support for parsing byte strings on Python 3.
+
+    @type qs: C{bytes}
+    """
+    d = {}
+    items = [s2 for s1 in qs.split(b"&") for s2 in s1.split(b";")]
+    for item in items:
+        try:
+            k, v = item.split(b"=", 1)
+        except ValueError:
+            if strict_parsing:
+                raise
+            continue
+        if v or keep_blank_values:
+            k = unquote(k.replace(b"+", b" "))
+            v = unquote(v.replace(b"+", b" "))
+            if k in d:
+                d[k].append(v)
+            else:
+                d[k] = [v]
+    return d
+
+
+
+def datetimeToString(msSinceEpoch=None):
+    """
+    Convert seconds since epoch to HTTP datetime string.
+
+    @rtype: C{bytes}
+    """
+    if msSinceEpoch == None:
+        msSinceEpoch = time.time()
+    year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
+    s = networkString("%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
+            weekdayname[wd],
+            day, monthname[month], year,
+            hh, mm, ss))
+    return s
+
+
+
+def datetimeToLogString(msSinceEpoch=None):
+    """
+    Convert seconds since epoch to log datetime string.
+
+    @rtype: C{str}
+    """
+    if msSinceEpoch == None:
+        msSinceEpoch = time.time()
+    year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
+    s = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % (
+        day, monthname[month], year,
+        hh, mm, ss)
+    return s
+
+
+
+def timegm(year, month, day, hour, minute, second):
+    """
+    Convert time tuple in GMT to seconds since epoch, GMT
+    """
+    EPOCH = 1970
+    if year < EPOCH:
+        raise ValueError("Years prior to %d not supported" % (EPOCH,))
+    assert 1 <= month <= 12
+    days = 365*(year-EPOCH) + calendar.leapdays(EPOCH, year)
+    for i in range(1, month):
+        days = days + calendar.mdays[i]
+    if month > 2 and calendar.isleap(year):
+        days = days + 1
+    days = days + day - 1
+    hours = days*24 + hour
+    minutes = hours*60 + minute
+    seconds = minutes*60 + second
+    return seconds
+
+
+
+def stringToDatetime(dateString):
+    """
+    Convert an HTTP date string (one of three formats) to seconds since epoch.
+
+    @type dateString: C{bytes}
+    """
+    parts = nativeString(dateString).split()
+
+    if not parts[0][0:3].lower() in weekdayname_lower:
+        # Weekday is stupid. Might have been omitted.
+        try:
+            return stringToDatetime(b"Sun, " + dateString)
+        except ValueError:
+            # Guess not.
+            pass
+
+    partlen = len(parts)
+    if (partlen == 5 or partlen == 6) and parts[1].isdigit():
+        # 1st date format: Sun, 06 Nov 1994 08:49:37 GMT
+        # (Note: "GMT" is literal, not a variable timezone)
+        # (also handles without "GMT")
+        # This is the normal format
+        day = parts[1]
+        month = parts[2]
+        year = parts[3]
+        time = parts[4]
+    elif (partlen == 3 or partlen == 4) and parts[1].find('-') != -1:
+        # 2nd date format: Sunday, 06-Nov-94 08:49:37 GMT
+        # (Note: "GMT" is literal, not a variable timezone)
+        # (also handles without without "GMT")
+        # Two digit year, yucko.
+        day, month, year = parts[1].split('-')
+        time = parts[2]
+        year=int(year)
+        if year < 69:
+            year = year + 2000
+        elif year < 100:
+            year = year + 1900
+    elif len(parts) == 5:
+        # 3rd date format: Sun Nov  6 08:49:37 1994
+        # ANSI C asctime() format.
+        day = parts[2]
+        month = parts[1]
+        year = parts[4]
+        time = parts[3]
+    else:
+        raise ValueError("Unknown datetime format %r" % dateString)
+
+    day = int(day)
+    month = int(monthname_lower.index(month.lower()))
+    year = int(year)
+    hour, min, sec = map(int, time.split(':'))
+    return int(timegm(year, month, day, hour, min, sec))
+
+
+
+def toChunk(data):
+    """
+    Convert string to a chunk.
+
+    @type data: C{bytes}
+
+    @returns: a tuple of C{bytes} representing the chunked encoding of data
+    """
+    return (networkString('%x' % (len(data),)), b"\r\n", data, b"\r\n")
+
+
+
+def fromChunk(data):
+    """
+    Convert chunk to string.
+
+    @type data: C{bytes}
+
+    @return: tuple of (result, remaining) - both C{bytes}.
+
+    @raise ValueError: If the given data is not a correctly formatted chunked
+        byte string.
+    """
+    prefix, rest = data.split(b'\r\n', 1)
+    length = int(prefix, 16)
+    if length < 0:
+        raise ValueError("Chunk length must be >= 0, not %d" % (length,))
+    if rest[length:length + 2] != b'\r\n':
+        raise ValueError("chunk must end with CRLF")
+    return rest[:length], rest[length + 2:]
+
+
+
+def parseContentRange(header):
+    """
+    Parse a content-range header into (start, end, realLength).
+
+    realLength might be None if real length is not known ('*').
+    """
+    kind, other = header.strip().split()
+    if kind.lower() != "bytes":
+        raise ValueError("a range of type %r is not supported")
+    startend, realLength = other.split("/")
+    start, end = map(int, startend.split("-"))
+    if realLength == "*":
+        realLength = None
+    else:
+        realLength = int(realLength)
+    return (start, end, realLength)
+
+
+
+class _IDeprecatedHTTPChannelToRequestInterface(Interface):
+    """
+    The interface L{HTTPChannel} expects of L{Request}.
+    """
+
+    requestHeaders = Attribute(
+        "A L{http_headers.Headers} instance giving all received HTTP request "
+        "headers.")
+
+    responseHeaders = Attribute(
+        "A L{http_headers.Headers} instance holding all HTTP response "
+        "headers to be sent.")
+
+
+    def connectionLost(reason):
+        """
+        The underlying connection has been lost.
+
+        @param reason: A failure instance indicating the reason why
+            the connection was lost.
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+
+    def gotLength(length):
+        """
+        Called when L{HTTPChannel} has determined the length, if any,
+        of the incoming request's body.
+
+        @param length: The length of the request's body.
+        @type length: L{int} if the request declares its body's length
+            and L{None} if it does not.
+        """
+
+
+    def handleContentChunk(data):
+        """
+        Deliver a received chunk of body data to the request.  Note
+        this does not imply chunked transfer encoding.
+
+        @param data: The received chunk.
+        @type data: L{bytes}
+        """
+
+
+    def parseCookies():
+        """
+        Parse the request's cookies out of received headers.
+        """
+
+
+    def requestReceived(command, path, version):
+        """
+        Called when the entire request, including its body, has been
+        received.
+
+        @param command: The request's HTTP command.
+        @type command: L{bytes}
+
+        @param path: The request's path.  Note: this is actually what
+            RFC7320 calls the URI.
+        @type path: L{bytes}
+
+        @param version: The request's HTTP version.
+        @type version: L{bytes}
+        """
+
+
+    def __eq__(other):
+        """
+        Determines if two requests are the same object.
+
+        @param other: Another object whose identity will be compared
+            to this instance's.
+
+        @return: L{True} when the two are the same object and L{False}
+            when not.
+        @rtype: L{bool}
+        """
+
+
+    def __ne__(other):
+        """
+        Determines if two requests are not the same object.
+
+        @param other: Another object whose identity will be compared
+            to this instance's.
+
+        @return: L{True} when the two are not the same object and
+            L{False} when they are.
+        @rtype: L{bool}
+        """
+
+
+    def __hash__():
+        """
+        Generate a hash value for the request.
+
+        @return: The request's hash value.
+        @rtype: L{int}
+        """
+
+
+
+class StringTransport:
+    """
+    I am a StringIO wrapper that conforms for the transport API. I support
+    the `writeSequence' method.
+    """
+    def __init__(self):
+        self.s = StringIO()
+    def writeSequence(self, seq):
+        self.s.write(b''.join(seq))
+    def __getattr__(self, attr):
+        return getattr(self.__dict__['s'], attr)
+
+
+
+class HTTPClient(basic.LineReceiver):
+    """
+    A client for HTTP 1.0.
+
+    Notes:
+    You probably want to send a 'Host' header with the name of the site you're
+    connecting to, in order to not break name based virtual hosting.
+
+    @ivar length: The length of the request body in bytes.
+    @type length: C{int}
+
+    @ivar firstLine: Are we waiting for the first header line?
+    @type firstLine: C{bool}
+
+    @ivar __buffer: The buffer that stores the response to the HTTP request.
+    @type __buffer: A C{StringIO} object.
+
+    @ivar _header: Part or all of an HTTP request header.
+    @type _header: C{bytes}
+    """
+    length = None
+    firstLine = True
+    __buffer = None
+    _header = b""
+
+    def sendCommand(self, command, path):
+        self.transport.writeSequence([command, b' ', path, b' HTTP/1.0\r\n'])
+
+    def sendHeader(self, name, value):
+        if not isinstance(value, bytes):
+            # XXX Deprecate this case
+            value = networkString(str(value))
+        santizedName = _sanitizeLinearWhitespace(name)
+        santizedValue = _sanitizeLinearWhitespace(value)
+        self.transport.writeSequence(
+            [santizedName, b': ', santizedValue, b'\r\n'])
+
+    def endHeaders(self):
+        self.transport.write(b'\r\n')
+
+
+    def extractHeader(self, header):
+        """
+        Given a complete HTTP header, extract the field name and value and
+        process the header.
+
+        @param header: a complete HTTP request header of the form
+            'field-name: value'.
+        @type header: C{bytes}
+        """
+        key, val = header.split(b':', 1)
+        val = val.lstrip()
+        self.handleHeader(key, val)
+        if key.lower() == b'content-length':
+            self.length = int(val)
+
+
+    def lineReceived(self, line):
+        """
+        Parse the status line and headers for an HTTP request.
+
+        @param line: Part of an HTTP request header. Request bodies are parsed
+            in L{HTTPClient.rawDataReceived}.
+        @type line: C{bytes}
+        """
+        if self.firstLine:
+            self.firstLine = False
+            l = line.split(None, 2)
+            version = l[0]
+            status = l[1]
+            try:
+                message = l[2]
+            except IndexError:
+                # sometimes there is no message
+                message = b""
+            self.handleStatus(version, status, message)
+            return
+        if not line:
+            if self._header != b"":
+                # Only extract headers if there are any
+                self.extractHeader(self._header)
+            self.__buffer = StringIO()
+            self.handleEndHeaders()
+            self.setRawMode()
+            return
+
+        if line.startswith(b'\t') or line.startswith(b' '):
+            # This line is part of a multiline header. According to RFC 822, in
+            # "unfolding" multiline headers you do not strip the leading
+            # whitespace on the continuing line.
+            self._header = self._header + line
+        elif self._header:
+            # This line starts a new header, so process the previous one.
+            self.extractHeader(self._header)
+            self._header = line
+        else: # First header
+            self._header = line
+
+
+    def connectionLost(self, reason):
+        self.handleResponseEnd()
+
+    def handleResponseEnd(self):
+        """
+        The response has been completely received.
+
+        This callback may be invoked more than once per request.
+        """
+        if self.__buffer is not None:
+            b = self.__buffer.getvalue()
+            self.__buffer = None
+            self.handleResponse(b)
+
+    def handleResponsePart(self, data):
+        self.__buffer.write(data)
+
+    def connectionMade(self):
+        pass
+
+    def handleStatus(self, version, status, message):
+        """
+        Called when the status-line is received.
+
+        @param version: e.g. 'HTTP/1.0'
+        @param status: e.g. '200'
+        @type status: C{bytes}
+        @param message: e.g. 'OK'
+        """
+
+    def handleHeader(self, key, val):
+        """
+        Called every time a header is received.
+        """
+
+    def handleEndHeaders(self):
+        """
+        Called when all headers have been received.
+        """
+
+
+    def rawDataReceived(self, data):
+        if self.length is not None:
+            data, rest = data[:self.length], data[self.length:]
+            self.length -= len(data)
+        else:
+            rest = b''
+        self.handleResponsePart(data)
+        if self.length == 0:
+            self.handleResponseEnd()
+            self.setLineMode(rest)
+
+
+
+# response codes that must have empty bodies
+NO_BODY_CODES = (204, 304)
+
+
+# Sentinel object that detects people explicitly passing `queued` to Request.
+_QUEUED_SENTINEL = object()
+
+
+
+def _getContentFile(length):
+    """
+    Get a writeable file-like object to which request content can be written.
+    """
+    if length is not None and length < 100000:
+        return StringIO()
+    return tempfile.TemporaryFile()
+
+
+
+@implementer(interfaces.IConsumer,
+             _IDeprecatedHTTPChannelToRequestInterface)
+class Request:
+    """
+    A HTTP request.
+
+    Subclasses should override the process() method to determine how
+    the request will be processed.
+
+    @ivar method: The HTTP method that was used, e.g. C{b'GET'}.
+    @type method: L{bytes}
+
+    @ivar uri: The full encoded URI which was requested (including query
+        arguments), e.g. C{b'/a/b%20/c?q=v'}.
+    @type uri: L{bytes}
+
+    @ivar path: The encoded path of the request URI (not including query
+        arguments), e.g. C{b'/a/b%20/c'}.
+    @type path: L{bytes}
+
+    @ivar args: A mapping of decoded query argument names as L{bytes} to
+        corresponding query argument values as L{list}s of L{bytes}.
+        For example, for a URI with C{foo=bar&foo=baz&quux=spam}
+        as its query part C{args} will be C{{b'foo': [b'bar', b'baz'],
+        b'quux': [b'spam']}}.
+    @type args: L{dict} of L{bytes} to L{list} of L{bytes}
+
+    @ivar content: A file-like object giving the request body.  This may be
+        a file on disk, an L{io.BytesIO}, or some other type.  The
+        implementation is free to decide on a per-request basis.
+    @type content: L{typing.BinaryIO}
+
+    @ivar cookies: The cookies that will be sent in the response.
+    @type cookies: L{list} of L{bytes}
+
+    @type requestHeaders: L{http_headers.Headers}
+    @ivar requestHeaders: All received HTTP request headers.
+
+    @type responseHeaders: L{http_headers.Headers}
+    @ivar responseHeaders: All HTTP response headers to be sent.
+
+    @ivar notifications: A L{list} of L{Deferred}s which are waiting for
+        notification that the response to this request has been finished
+        (successfully or with an error).  Don't use this attribute directly,
+        instead use the L{Request.notifyFinish} method.
+
+    @ivar _disconnected: A flag which is C{False} until the connection over
+        which this request was received is closed and which is C{True} after
+        that.
+    @type _disconnected: L{bool}
+
+    @ivar _log: A logger instance for request related messages.
+    @type _log: L{twisted.logger.Logger}
+    """
+    producer = None
+    finished = 0
+    code = OK
+    code_message = RESPONSES[OK]
+    method = "(no method yet)"
+    clientproto = b"(no clientproto yet)"
+    uri = "(no uri yet)"
+    startedWriting = 0
+    chunked = 0
+    sentLength = 0 # content-length of response, or total bytes sent via chunking
+    etag = None
+    lastModified = None
+    args = None
+    path = None
+    content = None
+    _forceSSL = 0
+    _disconnected = False
+    _log = Logger()
+
+    def __init__(self, channel, queued=_QUEUED_SENTINEL):
+        """
+        @param channel: the channel we're connected to.
+        @param queued: (deprecated) are we in the request queue, or can we
+            start writing to the transport?
+        """
+        self.notifications = []
+        self.channel = channel
+
+        # Cache the client and server information, we'll need this
+        # later to be serialized and sent with the request so CGIs
+        # will work remotely
+        self.client = self.channel.getPeer()
+        self.host = self.channel.getHost()
+
+        self.requestHeaders = Headers()
+        self.received_cookies = {}
+        self.responseHeaders = Headers()
+        self.cookies = [] # outgoing cookies
+        self.transport = self.channel.transport
+
+        if queued is _QUEUED_SENTINEL:
+            queued = False
+
+        self.queued = queued
+
+
+    def _cleanup(self):
+        """
+        Called when have finished responding and are no longer queued.
+        """
+        if self.producer:
+            self._log.failure(
+                '',
+                Failure(
+                    RuntimeError(
+                        "Producer was not unregistered for %s" % (self.uri,)
+                    )
+                )
+            )
+            self.unregisterProducer()
+        self.channel.requestDone(self)
+        del self.channel
+        if self.content is not None:
+            try:
+                self.content.close()
+            except OSError:
+                # win32 suckiness, no idea why it does this
+                pass
+            del self.content
+        for d in self.notifications:
+            d.callback(None)
+        self.notifications = []
+
+    # methods for channel - end users should not use these
+
+    def noLongerQueued(self):
+        """
+        Notify the object that it is no longer queued.
+
+        We start writing whatever data we have to the transport, etc.
+
+        This method is not intended for users.
+
+        In 16.3 this method was changed to become a no-op, as L{Request}
+        objects are now never queued.
+        """
+        pass
+
+
+    def gotLength(self, length):
+        """
+        Called when HTTP channel got length of content in this request.
+
+        This method is not intended for users.
+
+        @param length: The length of the request body, as indicated by the
+            request headers.  L{None} if the request headers do not indicate a
+            length.
+        """
+        self.content = _getContentFile(length)
+
+
+    def parseCookies(self):
+        """
+        Parse cookie headers.
+
+        This method is not intended for users.
+        """
+        cookieheaders = self.requestHeaders.getRawHeaders(b"cookie")
+
+        if cookieheaders is None:
+            return
+
+        for cookietxt in cookieheaders:
+            if cookietxt:
+                for cook in cookietxt.split(b';'):
+                    cook = cook.lstrip()
+                    try:
+                        k, v = cook.split(b'=', 1)
+                        self.received_cookies[k] = v
+                    except ValueError:
+                        pass
+
+
+    def handleContentChunk(self, data):
+        """
+        Write a chunk of data.
+
+        This method is not intended for users.
+        """
+        self.content.write(data)
+
+
+    def requestReceived(self, command, path, version):
+        """
+        Called by channel when all data has been received.
+
+        This method is not intended for users.
+
+        @type command: C{bytes}
+        @param command: The HTTP verb of this request.  This has the case
+            supplied by the client (eg, it maybe "get" rather than "GET").
+
+        @type path: C{bytes}
+        @param path: The URI of this request.
+
+        @type version: C{bytes}
+        @param version: The HTTP version of this request.
+        """
+        clength = self.content.tell()
+        self.content.seek(0, 0)
+        self.args = {}
+
+        self.method, self.uri = command, path
+        self.clientproto = version
+        x = self.uri.split(b'?', 1)
+
+        if len(x) == 1:
+            self.path = self.uri
+        else:
+            self.path, argstring = x
+            self.args = parse_qs(argstring, 1)
+
+        # Argument processing
+        args = self.args
+        ctype = self.requestHeaders.getRawHeaders(b'content-type')
+        if ctype is not None:
+            ctype = ctype[0]
+
+        if self.method == b"POST" and ctype and clength:
+            mfd = b'multipart/form-data'
+            key, pdict = _parseHeader(ctype)
+            # This weird CONTENT-LENGTH param is required by
+            # cgi.parse_multipart() in some versions of Python 3.7+, see
+            # bpo-29979. It looks like this will be relaxed and backported, see
+            # https://github.com/python/cpython/pull/8530.
+            pdict["CONTENT-LENGTH"] = clength
+            if key == b'application/x-www-form-urlencoded':
+                args.update(parse_qs(self.content.read(), 1))
+            elif key == mfd:
+                try:
+                    if _PY37PLUS:
+                        cgiArgs = cgi.parse_multipart(
+                            self.content, pdict, encoding='utf8',
+                            errors="surrogateescape")
+                    else:
+                        cgiArgs = cgi.parse_multipart(self.content, pdict)
+
+                    if not _PY37PLUS and _PY3:
+                        # The parse_multipart function on Python 3
+                        # decodes the header bytes as iso-8859-1 and
+                        # returns a str key -- we want bytes so encode
+                        # it back
+                        self.args.update({x.encode('iso-8859-1'): y
+                                          for x, y in cgiArgs.items()})
+                    elif _PY37PLUS:
+                        # The parse_multipart function on Python 3.7+
+                        # decodes the header bytes as iso-8859-1 and
+                        # decodes the body bytes as utf8 with
+                        # surrogateescape -- we want bytes
+                        self.args.update({
+                            x.encode('iso-8859-1'): \
+                            [z.encode('utf8', "surrogateescape")
+                             if isinstance(z, str) else z for z in y]
+                            for x, y in cgiArgs.items()})
+
+                    else:
+                        self.args.update(cgiArgs)
+                except Exception as e:
+                    # It was a bad request, or we got a signal.
+                    self.channel._respondToBadRequestAndDisconnect()
+                    if isinstance(e, (TypeError, ValueError, KeyError)):
+                        return
+                    else:
+                        # If it's not a userspace error from CGI, reraise
+                        raise
+
+            self.content.seek(0, 0)
+
+        self.process()
+
+
+    def __repr__(self):
+        """
+        Return a string description of the request including such information
+        as the request method and request URI.
+
+        @return: A string loosely describing this L{Request} object.
+        @rtype: L{str}
+        """
+        return '<%s at 0x%x method=%s uri=%s clientproto=%s>' % (
+            self.__class__.__name__,
+            id(self),
+            nativeString(self.method),
+            nativeString(self.uri),
+            nativeString(self.clientproto))
+
+
+    def process(self):
+        """
+        Override in subclasses.
+
+        This method is not intended for users.
+        """
+        pass
+
+
+    # consumer interface
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register a producer.
+        """
+        if self.producer:
+            raise ValueError(
+                "registering producer %s before previous one (%s) was "
+                "unregistered" % (producer, self.producer))
+
+        self.streamingProducer = streaming
+        self.producer = producer
+        self.channel.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        """
+        Unregister the producer.
+        """
+        self.channel.unregisterProducer()
+        self.producer = None
+
+
+    # The following is the public interface that people should be
+    # writing to.
+    def getHeader(self, key):
+        """
+        Get an HTTP request header.
+
+        @type key: C{bytes} or C{str}
+        @param key: The name of the header to get the value of.
+
+        @rtype: C{bytes} or C{str} or L{None}
+        @return: The value of the specified header, or L{None} if that header
+            was not present in the request. The string type of the result
+            matches the type of L{key}.
+        """
+        value = self.requestHeaders.getRawHeaders(key)
+        if value is not None:
+            return value[-1]
+
+
+    def getCookie(self, key):
+        """
+        Get a cookie that was sent from the network.
+
+        @type key: C{bytes}
+        @param key: The name of the cookie to get.
+
+        @rtype: C{bytes} or C{None}
+        @returns: The value of the specified cookie, or L{None} if that cookie
+            was not present in the request.
+        """
+        return self.received_cookies.get(key)
+
+
+    def notifyFinish(self):
+        """
+        Notify when the response to this request has finished.
+
+        @note: There are some caveats around the reliability of the delivery of
+            this notification.
+
+                1. If this L{Request}'s channel is paused, the notification
+                   will not be delivered.  This can happen in one of two ways;
+                   either you can call C{request.transport.pauseProducing}
+                   yourself, or,
+
+                2. In order to deliver this notification promptly when a client
+                   disconnects, the reactor must continue reading from the
+                   transport, so that it can tell when the underlying network
+                   connection has gone away.  Twisted Web will only keep
+                   reading up until a finite (small) maximum buffer size before
+                   it gives up and pauses the transport itself.  If this
+                   occurs, you will not discover that the connection has gone
+                   away until a timeout fires or until the application attempts
+                   to send some data via L{Request.write}.
+
+                3. It is theoretically impossible to distinguish between
+                   successfully I{sending} a response and the peer successfully
+                   I{receiving} it.  There are several networking edge cases
+                   where the L{Deferred}s returned by C{notifyFinish} will
+                   indicate success, but the data will never be received.
+                   There are also edge cases where the connection will appear
+                   to fail, but in reality the response was delivered.  As a
+                   result, the information provided by the result of the
+                   L{Deferred}s returned by this method should be treated as a
+                   guess; do not make critical decisions in your applications
+                   based upon it.
+
+        @rtype: L{Deferred}
+        @return: A L{Deferred} which will be triggered when the request is
+            finished -- with a L{None} value if the request finishes
+            successfully or with an error if the request is interrupted by an
+            error (for example, the client closing the connection prematurely).
+        """
+        self.notifications.append(Deferred())
+        return self.notifications[-1]
+
+
+    def finish(self):
+        """
+        Indicate that all response data has been written to this L{Request}.
+        """
+        if self._disconnected:
+            raise RuntimeError(
+                "Request.finish called on a request after its connection was lost; "
+                "use Request.notifyFinish to keep track of this.")
+        if self.finished:
+            warnings.warn("Warning! request.finish called twice.", stacklevel=2)
+            return
+
+        if not self.startedWriting:
+            # write headers
+            self.write(b'')
+
+        if self.chunked:
+            # write last chunk and closing CRLF
+            self.channel.write(b"0\r\n\r\n")
+
+        # log request
+        if (hasattr(self.channel, "factory") and
+                self.channel.factory is not None):
+            self.channel.factory.log(self)
+
+        self.finished = 1
+        if not self.queued:
+            self._cleanup()
+
+
+    def write(self, data):
+        """
+        Write some data as a result of an HTTP request.  The first
+        time this is called, it writes out response data.
+
+        @type data: C{bytes}
+        @param data: Some bytes to be sent as part of the response body.
+        """
+        if self.finished:
+            raise RuntimeError('Request.write called on a request after '
+                               'Request.finish was called.')
+
+        if self._disconnected:
+            # Don't attempt to write any data to a disconnected client.
+            # The RuntimeError exception will be thrown as usual when
+            # request.finish is called
+            return
+
+        if not self.startedWriting:
+            self.startedWriting = 1
+            version = self.clientproto
+            code = intToBytes(self.code)
+            reason = self.code_message
+            headers = []
+
+            # if we don't have a content length, we send data in
+            # chunked mode, so that we can support pipelining in
+            # persistent connections.
+            if ((version == b"HTTP/1.1") and
+                (self.responseHeaders.getRawHeaders(b'content-length') is None) and
+                self.method != b"HEAD" and self.code not in NO_BODY_CODES):
+                headers.append((b'Transfer-Encoding', b'chunked'))
+                self.chunked = 1
+
+            if self.lastModified is not None:
+                if self.responseHeaders.hasHeader(b'last-modified'):
+                    self._log.info(
+                        "Warning: last-modified specified both in"
+                        " header list and lastModified attribute."
+                    )
+                else:
+                    self.responseHeaders.setRawHeaders(
+                        b'last-modified',
+                        [datetimeToString(self.lastModified)])
+
+            if self.etag is not None:
+                self.responseHeaders.setRawHeaders(b'ETag', [self.etag])
+
+            for name, values in self.responseHeaders.getAllRawHeaders():
+                for value in values:
+                    headers.append((name, value))
+
+            for cookie in self.cookies:
+                headers.append((b'Set-Cookie', cookie))
+
+            self.channel.writeHeaders(version, code, reason, headers)
+
+            # if this is a "HEAD" request, we shouldn't return any data
+            if self.method == b"HEAD":
+                self.write = lambda data: None
+                return
+
+            # for certain result codes, we should never return any data
+            if self.code in NO_BODY_CODES:
+                self.write = lambda data: None
+                return
+
+        self.sentLength = self.sentLength + len(data)
+        if data:
+            if self.chunked:
+                self.channel.writeSequence(toChunk(data))
+            else:
+                self.channel.write(data)
+
+    def addCookie(self, k, v, expires=None, domain=None, path=None,
+                  max_age=None, comment=None, secure=None, httpOnly=False,
+                  sameSite=None):
+        """
+        Set an outgoing HTTP cookie.
+
+        In general, you should consider using sessions instead of cookies, see
+        L{twisted.web.server.Request.getSession} and the
+        L{twisted.web.server.Session} class for details.
+
+        @param k: cookie name
+        @type k: L{bytes} or L{unicode}
+
+        @param v: cookie value
+        @type v: L{bytes} or L{unicode}
+
+        @param expires: cookie expire attribute value in
+            "Wdy, DD Mon YYYY HH:MM:SS GMT" format
+        @type expires: L{bytes} or L{unicode}
+
+        @param domain: cookie domain
+        @type domain: L{bytes} or L{unicode}
+
+        @param path: cookie path
+        @type path: L{bytes} or L{unicode}
+
+        @param max_age: cookie expiration in seconds from reception
+        @type max_age: L{bytes} or L{unicode}
+
+        @param comment: cookie comment
+        @type comment: L{bytes} or L{unicode}
+
+        @param secure: direct browser to send the cookie on encrypted
+            connections only
+        @type secure: L{bool}
+
+        @param httpOnly: direct browser not to expose cookies through channels
+            other than HTTP (and HTTPS) requests
+        @type httpOnly: L{bool}
+
+        @param sameSite: One of L{None} (default), C{'lax'} or C{'strict'}.
+            Direct browsers not to send this cookie on cross-origin requests.
+            Please see:
+            U{https://tools.ietf.org/html/draft-west-first-party-cookies-07}
+        @type sameSite: L{None}, L{bytes} or L{unicode}
+
+        @raises: L{DeprecationWarning} if an argument is not L{bytes} or
+            L{unicode}.
+            L{ValueError} if the value for C{sameSite} is not supported.
+        """
+        def _ensureBytes(val):
+            """
+            Ensure that C{val} is bytes, encoding using UTF-8 if
+            needed.
+
+            @param val: L{bytes} or L{unicode}
+
+            @return: L{bytes}
+            """
+            if val is None:
+                # It's None, so we don't want to touch it
+                return val
+
+            if isinstance(val, bytes):
+                return val
+            else:
+                return val.encode('utf8')
+
+
+        def _sanitize(val):
+            """
+            Replace linear whitespace (C{\r}, C{\n}, C{\r\n}) and
+            semicolons C{;} in C{val} with a single space.
+
+            @param val: L{bytes}
+            @return: L{bytes}
+            """
+            return _sanitizeLinearWhitespace(val).replace(b';', b' ')
+
+        cookie = (
+            _sanitize(_ensureBytes(k)) +
+            b"=" +
+            _sanitize(_ensureBytes(v)))
+        if expires is not None:
+            cookie = cookie + b"; Expires=" + _sanitize(_ensureBytes(expires))
+        if domain is not None:
+            cookie = cookie + b"; Domain=" + _sanitize(_ensureBytes(domain))
+        if path is not None:
+            cookie = cookie + b"; Path=" + _sanitize(_ensureBytes(path))
+        if max_age is not None:
+            cookie = cookie + b"; Max-Age=" + _sanitize(_ensureBytes(max_age))
+        if comment is not None:
+            cookie = cookie + b"; Comment=" + _sanitize(_ensureBytes(comment))
+        if secure:
+            cookie = cookie + b"; Secure"
+        if httpOnly:
+            cookie = cookie + b"; HttpOnly"
+        if sameSite:
+            sameSite = _ensureBytes(sameSite).lower()
+            if sameSite not in [b"lax", b"strict"]:
+                raise ValueError(
+                    "Invalid value for sameSite: " + repr(sameSite))
+            cookie += b"; SameSite=" + sameSite
+        self.cookies.append(cookie)
+
+    def setResponseCode(self, code, message=None):
+        """
+        Set the HTTP response code.
+
+        @type code: L{int}
+        @type message: L{bytes}
+        """
+        if not isinstance(code, _intTypes):
+            raise TypeError("HTTP response code must be int or long")
+        self.code = code
+        if message:
+            if not isinstance(message, bytes):
+                raise TypeError("HTTP response status message must be bytes")
+            self.code_message = message
+        else:
+            self.code_message = RESPONSES.get(code, b"Unknown Status")
+
+
+    def setHeader(self, name, value):
+        """
+        Set an HTTP response header.  Overrides any previously set values for
+        this header.
+
+        @type k: L{bytes} or L{str}
+        @param k: The name of the header for which to set the value.
+
+        @type v: L{bytes} or L{str}
+        @param v: The value to set for the named header. A L{str} will be
+            UTF-8 encoded, which may not interoperable with other
+            implementations. Avoid passing non-ASCII characters if possible.
+        """
+        self.responseHeaders.setRawHeaders(name, [value])
+
+
+    def redirect(self, url):
+        """
+        Utility function that does a redirect.
+
+        Set the response code to L{FOUND} and the I{Location} header to the
+        given URL.
+
+        The request should have C{finish()} called after this.
+
+        @param url: I{Location} header value.
+        @type url: L{bytes} or L{str}
+        """
+        self.setResponseCode(FOUND)
+        self.setHeader(b"location", url)
+
+
+    def setLastModified(self, when):
+        """
+        Set the C{Last-Modified} time for the response to this request.
+
+        If I am called more than once, I ignore attempts to set
+        Last-Modified earlier, only replacing the Last-Modified time
+        if it is to a later value.
+
+        If I am a conditional request, I may modify my response code
+        to L{NOT_MODIFIED} if appropriate for the time given.
+
+        @param when: The last time the resource being returned was
+            modified, in seconds since the epoch.
+        @type when: number
+        @return: If I am a I{If-Modified-Since} conditional request and
+            the time given is not newer than the condition, I return
+            L{http.CACHED} to indicate that you should write no
+            body.  Otherwise, I return a false value.
+        """
+        # time.time() may be a float, but the HTTP-date strings are
+        # only good for whole seconds.
+        when = int(math.ceil(when))
+        if (not self.lastModified) or (self.lastModified < when):
+            self.lastModified = when
+
+        modifiedSince = self.getHeader(b'if-modified-since')
+        if modifiedSince:
+            firstPart = modifiedSince.split(b';', 1)[0]
+            try:
+                modifiedSince = stringToDatetime(firstPart)
+            except ValueError:
+                return None
+            if modifiedSince >= self.lastModified:
+                self.setResponseCode(NOT_MODIFIED)
+                return CACHED
+        return None
+
+    def setETag(self, etag):
+        """
+        Set an C{entity tag} for the outgoing response.
+
+        That's \"entity tag\" as in the HTTP/1.1 C{ETag} header, \"used
+        for comparing two or more entities from the same requested
+        resource.\"
+
+        If I am a conditional request, I may modify my response code
+        to L{NOT_MODIFIED} or L{PRECONDITION_FAILED}, if appropriate
+        for the tag given.
+
+        @param etag: The entity tag for the resource being returned.
+        @type etag: string
+        @return: If I am a C{If-None-Match} conditional request and
+            the tag matches one in the request, I return
+            L{http.CACHED} to indicate that you should write
+            no body.  Otherwise, I return a false value.
+        """
+        if etag:
+            self.etag = etag
+
+        tags = self.getHeader(b"if-none-match")
+        if tags:
+            tags = tags.split()
+            if (etag in tags) or (b'*' in tags):
+                self.setResponseCode(((self.method in (b"HEAD", b"GET"))
+                                      and NOT_MODIFIED)
+                                     or PRECONDITION_FAILED)
+                return CACHED
+        return None
+
+
+    def getAllHeaders(self):
+        """
+        Return dictionary mapping the names of all received headers to the last
+        value received for each.
+
+        Since this method does not return all header information,
+        C{self.requestHeaders.getAllRawHeaders()} may be preferred.
+        """
+        headers = {}
+        for k, v in self.requestHeaders.getAllRawHeaders():
+            headers[k.lower()] = v[-1]
+        return headers
+
+
+    def getRequestHostname(self):
+        """
+        Get the hostname that the user passed in to the request.
+
+        This will either use the Host: header (if it is available) or the
+        host we are listening on if the header is unavailable.
+
+        @returns: the requested hostname
+        @rtype: C{bytes}
+        """
+        # XXX This method probably has no unit tests.  I changed it a ton and
+        # nothing failed.
+        host = self.getHeader(b'host')
+        if host:
+            return host.split(b':', 1)[0]
+        return networkString(self.getHost().host)
+
+
+    def getHost(self):
+        """
+        Get my originally requesting transport's host.
+
+        Don't rely on the 'transport' attribute, since Request objects may be
+        copied remotely.  For information on this method's return value, see
+        L{twisted.internet.tcp.Port}.
+        """
+        return self.host
+
+    def setHost(self, host, port, ssl=0):
+        """
+        Change the host and port the request thinks it's using.
+
+        This method is useful for working with reverse HTTP proxies (e.g.
+        both Squid and Apache's mod_proxy can do this), when the address
+        the HTTP client is using is different than the one we're listening on.
+
+        For example, Apache may be listening on https://www.example.com/, and
+        then forwarding requests to http://localhost:8080/, but we don't want
+        HTML produced by Twisted to say b'http://localhost:8080/', they should
+        say b'https://www.example.com/', so we do::
+
+           request.setHost(b'www.example.com', 443, ssl=1)
+
+        @type host: C{bytes}
+        @param host: The value to which to change the host header.
+
+        @type ssl: C{bool}
+        @param ssl: A flag which, if C{True}, indicates that the request is
+            considered secure (if C{True}, L{isSecure} will return C{True}).
+        """
+        self._forceSSL = ssl # set first so isSecure will work
+        if self.isSecure():
+            default = 443
+        else:
+            default = 80
+        if port == default:
+            hostHeader = host
+        else:
+            hostHeader = host + b":" + intToBytes(port)
+        self.requestHeaders.setRawHeaders(b"host", [hostHeader])
+        self.host = address.IPv4Address("TCP", host, port)
+
+
+    def getClientIP(self):
+        """
+        Return the IP address of the client who submitted this request.
+
+        This method is B{deprecated}.  Use L{getClientAddress} instead.
+
+        @returns: the client IP address
+        @rtype: C{str}
+        """
+        if isinstance(self.client, (address.IPv4Address, address.IPv6Address)):
+            return self.client.host
+        else:
+            return None
+
+
+    def getClientAddress(self):
+        """
+        Return the address of the client who submitted this request.
+
+        This may not be a network address (e.g., a server listening on
+        a UNIX domain socket will cause this to return
+        L{UNIXAddress}).  Callers must check the type of the returned
+        address.
+
+        @since: 18.4
+
+        @return: the client's address.
+        @rtype: L{IAddress}
+        """
+        return self.client
+
+
+    def isSecure(self):
+        """
+        Return L{True} if this request is using a secure transport.
+
+        Normally this method returns L{True} if this request's L{HTTPChannel}
+        instance is using a transport that implements
+        L{interfaces.ISSLTransport}.
+
+        This will also return L{True} if L{Request.setHost} has been called
+        with C{ssl=True}.
+
+        @returns: L{True} if this request is secure
+        @rtype: C{bool}
+        """
+        if self._forceSSL:
+            return True
+        channel = getattr(self, 'channel', None)
+        if channel is None:
+            return False
+        return channel.isSecure()
+
+
+    def _authorize(self):
+        # Authorization, (mostly) per the RFC
+        try:
+            authh = self.getHeader(b"Authorization")
+            if not authh:
+                self.user = self.password = b''
+                return
+            bas, upw = authh.split()
+            if bas.lower() != b"basic":
+                raise ValueError()
+            upw = base64.decodestring(upw)
+            self.user, self.password = upw.split(b':', 1)
+        except (binascii.Error, ValueError):
+            self.user = self.password = b''
+        except:
+            self._log.failure('')
+            self.user = self.password = b''
+
+
+    def getUser(self):
+        """
+        Return the HTTP user sent with this request, if any.
+
+        If no user was supplied, return the empty string.
+
+        @returns: the HTTP user, if any
+        @rtype: C{bytes}
+        """
+        try:
+            return self.user
+        except:
+            pass
+        self._authorize()
+        return self.user
+
+
+    def getPassword(self):
+        """
+        Return the HTTP password sent with this request, if any.
+
+        If no password was supplied, return the empty string.
+
+        @returns: the HTTP password, if any
+        @rtype: C{bytes}
+        """
+        try:
+            return self.password
+        except:
+            pass
+        self._authorize()
+        return self.password
+
+
+    def connectionLost(self, reason):
+        """
+        There is no longer a connection for this request to respond over.
+        Clean up anything which can't be useful anymore.
+        """
+        self._disconnected = True
+        self.channel = None
+        if self.content is not None:
+            self.content.close()
+        for d in self.notifications:
+            d.errback(reason)
+        self.notifications = []
+
+
+    def loseConnection(self):
+        """
+        Pass the loseConnection through to the underlying channel.
+        """
+        if self.channel is not None:
+            self.channel.loseConnection()
+
+
+    def __eq__(self, other):
+        """
+        Determines if two requests are the same object.
+
+        @param other: Another object whose identity will be compared
+            to this instance's.
+
+        @return: L{True} when the two are the same object and L{False}
+            when not.
+        @rtype: L{bool}
+        """
+        # When other is not an instance of request, return
+        # NotImplemented so that Python uses other.__eq__ to perform
+        # the comparison.  This ensures that a Request proxy generated
+        # by proxyForInterface compares equal to an actual Request
+        # instanceby turning request != proxy into proxy != request.
+        if isinstance(other, Request):
+            return self is other
+        return NotImplemented
+
+
+    def __ne__(self, other):
+        """
+        Determines if two requests are not the same object.
+
+        @param other: Another object whose identity will be compared
+            to this instance's.
+
+        @return: L{True} when the two are not the same object and
+            L{False} when they are.
+        @rtype: L{bool}
+        """
+        # When other is not an instance of request, return
+        # NotImplemented so that Python uses other.__ne__ to perform
+        # the comparison.  This ensures that a Request proxy generated
+        # by proxyForInterface can compare equal to an actual Request
+        # instance by turning request != proxy into proxy != request.
+        if isinstance(other, Request):
+            return self is not other
+        return NotImplemented
+
+
+    def __hash__(self):
+        """
+        A C{Request} is hashable so that it can be used as a mapping key.
+
+        @return: A C{int} based on the instance's identity.
+        """
+        return id(self)
+
+
+
+Request.getClientIP = deprecated(
+    Version('Twisted', 18, 4, 0),
+    replacement="getClientAddress",
+)(Request.getClientIP)
+
+Request.noLongerQueued = deprecated(
+    Version("Twisted", 16, 3, 0))(Request.noLongerQueued)
+
+
+class _DataLoss(Exception):
+    """
+    L{_DataLoss} indicates that not all of a message body was received. This
+    is only one of several possible exceptions which may indicate that data
+    was lost.  Because of this, it should not be checked for by
+    specifically; any unexpected exception should be treated as having
+    caused data loss.
+    """
+
+
+
+class PotentialDataLoss(Exception):
+    """
+    L{PotentialDataLoss} may be raised by a transfer encoding decoder's
+    C{noMoreData} method to indicate that it cannot be determined if the
+    entire response body has been delivered.  This only occurs when making
+    requests to HTTP servers which do not set I{Content-Length} or a
+    I{Transfer-Encoding} in the response because in this case the end of the
+    response is indicated by the connection being closed, an event which may
+    also be due to a transient network problem or other error.
+    """
+
+
+
+class _MalformedChunkedDataError(Exception):
+    """
+    C{_ChunkedTranferDecoder} raises L{_MalformedChunkedDataError} from its
+    C{dataReceived} method when it encounters malformed data. This exception
+    indicates a client-side error. If this exception is raised, the connection
+    should be dropped with a 400 error.
+    """
+
+
+
+class _IdentityTransferDecoder(object):
+    """
+    Protocol for accumulating bytes up to a specified length.  This handles the
+    case where no I{Transfer-Encoding} is specified.
+
+    @ivar contentLength: Counter keeping track of how many more bytes there are
+        to receive.
+
+    @ivar dataCallback: A one-argument callable which will be invoked each
+        time application data is received.
+
+    @ivar finishCallback: A one-argument callable which will be invoked when
+        the terminal chunk is received.  It will be invoked with all bytes
+        which were delivered to this protocol which came after the terminal
+        chunk.
+    """
+    def __init__(self, contentLength, dataCallback, finishCallback):
+        self.contentLength = contentLength
+        self.dataCallback = dataCallback
+        self.finishCallback = finishCallback
+
+
+    def dataReceived(self, data):
+        """
+        Interpret the next chunk of bytes received.  Either deliver them to the
+        data callback or invoke the finish callback if enough bytes have been
+        received.
+
+        @raise RuntimeError: If the finish callback has already been invoked
+            during a previous call to this methood.
+        """
+        if self.dataCallback is None:
+            raise RuntimeError(
+                "_IdentityTransferDecoder cannot decode data after finishing")
+
+        if self.contentLength is None:
+            self.dataCallback(data)
+        elif len(data) < self.contentLength:
+            self.contentLength -= len(data)
+            self.dataCallback(data)
+        else:
+            # Make the state consistent before invoking any code belonging to
+            # anyone else in case noMoreData ends up being called beneath this
+            # stack frame.
+            contentLength = self.contentLength
+            dataCallback = self.dataCallback
+            finishCallback = self.finishCallback
+            self.dataCallback = self.finishCallback = None
+            self.contentLength = 0
+
+            dataCallback(data[:contentLength])
+            finishCallback(data[contentLength:])
+
+
+    def noMoreData(self):
+        """
+        All data which will be delivered to this decoder has been.  Check to
+        make sure as much data as was expected has been received.
+
+        @raise PotentialDataLoss: If the content length is unknown.
+        @raise _DataLoss: If the content length is known and fewer than that
+            many bytes have been delivered.
+
+        @return: L{None}
+        """
+        finishCallback = self.finishCallback
+        self.dataCallback = self.finishCallback = None
+        if self.contentLength is None:
+            finishCallback(b'')
+            raise PotentialDataLoss()
+        elif self.contentLength != 0:
+            raise _DataLoss()
+
+
+
+class _ChunkedTransferDecoder(object):
+    """
+    Protocol for decoding I{chunked} Transfer-Encoding, as defined by RFC 2616,
+    section 3.6.1.  This protocol can interpret the contents of a request or
+    response body which uses the I{chunked} Transfer-Encoding.  It cannot
+    interpret any of the rest of the HTTP protocol.
+
+    It may make sense for _ChunkedTransferDecoder to be an actual IProtocol
+    implementation.  Currently, the only user of this class will only ever
+    call dataReceived on it.  However, it might be an improvement if the
+    user could connect this to a transport and deliver connection lost
+    notification.  This way, `dataCallback` becomes `self.transport.write`
+    and perhaps `finishCallback` becomes `self.transport.loseConnection()`
+    (although I'm not sure where the extra data goes in that case).  This
+    could also allow this object to indicate to the receiver of data that
+    the stream was not completely received, an error case which should be
+    noticed. -exarkun
+
+    @ivar dataCallback: A one-argument callable which will be invoked each
+        time application data is received.
+
+    @ivar finishCallback: A one-argument callable which will be invoked when
+        the terminal chunk is received.  It will be invoked with all bytes
+        which were delivered to this protocol which came after the terminal
+        chunk.
+
+    @ivar length: Counter keeping track of how many more bytes in a chunk there
+        are to receive.
+
+    @ivar state: One of C{'CHUNK_LENGTH'}, C{'CRLF'}, C{'TRAILER'},
+        C{'BODY'}, or C{'FINISHED'}.  For C{'CHUNK_LENGTH'}, data for the
+        chunk length line is currently being read.  For C{'CRLF'}, the CR LF
+        pair which follows each chunk is being read. For C{'TRAILER'}, the CR
+        LF pair which follows the terminal 0-length chunk is currently being
+        read. For C{'BODY'}, the contents of a chunk are being read. For
+        C{'FINISHED'}, the last chunk has been completely read and no more
+        input is valid.
+    """
+    state = 'CHUNK_LENGTH'
+
+    def __init__(self, dataCallback, finishCallback):
+        self.dataCallback = dataCallback
+        self.finishCallback = finishCallback
+        self._buffer = b''
+
+
+    def _dataReceived_CHUNK_LENGTH(self, data):
+        if b'\r\n' in data:
+            line, rest = data.split(b'\r\n', 1)
+            parts = line.split(b';')
+            try:
+                self.length = int(parts[0], 16)
+            except ValueError:
+                raise _MalformedChunkedDataError(
+                    "Chunk-size must be an integer.")
+            if self.length == 0:
+                self.state = 'TRAILER'
+            else:
+                self.state = 'BODY'
+            return rest
+        else:
+            self._buffer = data
+            return b''
+
+
+    def _dataReceived_CRLF(self, data):
+        if data.startswith(b'\r\n'):
+            self.state = 'CHUNK_LENGTH'
+            return data[2:]
+        else:
+            self._buffer = data
+            return b''
+
+
+    def _dataReceived_TRAILER(self, data):
+        if data.startswith(b'\r\n'):
+            data = data[2:]
+            self.state = 'FINISHED'
+            self.finishCallback(data)
+        else:
+            self._buffer = data
+        return b''
+
+
+    def _dataReceived_BODY(self, data):
+        if len(data) >= self.length:
+            chunk, data = data[:self.length], data[self.length:]
+            self.dataCallback(chunk)
+            self.state = 'CRLF'
+            return data
+        elif len(data) < self.length:
+            self.length -= len(data)
+            self.dataCallback(data)
+            return b''
+
+
+    def _dataReceived_FINISHED(self, data):
+        raise RuntimeError(
+            "_ChunkedTransferDecoder.dataReceived called after last "
+            "chunk was processed")
+
+
+    def dataReceived(self, data):
+        """
+        Interpret data from a request or response body which uses the
+        I{chunked} Transfer-Encoding.
+        """
+        data = self._buffer + data
+        self._buffer = b''
+        while data:
+            data = getattr(self, '_dataReceived_%s' % (self.state,))(data)
+
+
+    def noMoreData(self):
+        """
+        Verify that all data has been received.  If it has not been, raise
+        L{_DataLoss}.
+        """
+        if self.state != 'FINISHED':
+            raise _DataLoss(
+                "Chunked decoder in %r state, still expecting more data to "
+                "get to 'FINISHED' state." % (self.state,))
+
+
+
+@implementer(interfaces.IPushProducer)
+class _NoPushProducer(object):
+    """
+    A no-op version of L{interfaces.IPushProducer}, used to abstract over the
+    possibility that a L{HTTPChannel} transport does not provide
+    L{IPushProducer}.
+    """
+    def pauseProducing(self):
+        """
+        Pause producing data.
+
+        Tells a producer that it has produced too much data to process for
+        the time being, and to stop until resumeProducing() is called.
+        """
+        pass
+
+
+    def resumeProducing(self):
+        """
+        Resume producing data.
+
+        This tells a producer to re-add itself to the main loop and produce
+        more data for its consumer.
+        """
+        pass
+
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register to receive data from a producer.
+
+        @param producer: The producer to register.
+        @param streaming: Whether this is a streaming producer or not.
+        """
+        pass
+
+
+    def unregisterProducer(self):
+        """
+        Stop consuming data from a producer, without disconnecting.
+        """
+        pass
+
+
+
+@implementer(interfaces.ITransport,
+             interfaces.IPushProducer,
+             interfaces.IConsumer)
+class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
+    """
+    A receiver for HTTP requests.
+
+    The L{HTTPChannel} provides L{interfaces.ITransport} and
+    L{interfaces.IConsumer} to the L{Request} objects it creates.  It also
+    implements L{interfaces.IPushProducer} to C{self.transport}, allowing the
+    transport to pause it.
+
+    @ivar MAX_LENGTH: Maximum length for initial request line and each line
+        from the header.
+
+    @ivar _transferDecoder: L{None} or a decoder instance if the request body
+        uses the I{chunked} Transfer-Encoding.
+    @type _transferDecoder: L{_ChunkedTransferDecoder}
+
+    @ivar maxHeaders: Maximum number of headers allowed per request.
+    @type maxHeaders: C{int}
+
+    @ivar totalHeadersSize: Maximum bytes for request line plus all headers
+        from the request.
+    @type totalHeadersSize: C{int}
+
+    @ivar _receivedHeaderSize: Bytes received so far for the header.
+    @type _receivedHeaderSize: C{int}
+
+    @ivar _handlingRequest: Whether a request is currently being processed.
+    @type _handlingRequest: L{bool}
+
+    @ivar _dataBuffer: Any data that has been received from the connection
+        while processing an outstanding request.
+    @type _dataBuffer: L{list} of L{bytes}
+
+    @ivar _networkProducer: Either the transport, if it provides
+        L{interfaces.IPushProducer}, or a null implementation of
+        L{interfaces.IPushProducer}.  Used to attempt to prevent the transport
+        from producing excess data when we're responding to a request.
+    @type _networkProducer: L{interfaces.IPushProducer}
+
+    @ivar _requestProducer: If the L{Request} object or anything it calls
+        registers itself as an L{interfaces.IProducer}, it will be stored here.
+        This is used to create a producing pipeline: pause/resume producing
+        methods will be propagated from the C{transport}, through the
+        L{HTTPChannel} instance, to the c{_requestProducer}.
+
+        The reason we proxy through the producing methods rather than the old
+        behaviour (where we literally just set the L{Request} object as the
+        producer on the transport) is because we want to be able to exert
+        backpressure on the client to prevent it from sending in arbitrarily
+        many requests without ever reading responses.  Essentially, if the
+        client never reads our responses we will eventually stop reading its
+        requests.
+
+    @type _requestProducer: L{interfaces.IPushProducer}
+
+    @ivar _requestProducerStreaming: A boolean that tracks whether the producer
+        on the L{Request} side of this channel has registered itself as a
+        L{interfaces.IPushProducer} or an L{interfaces.IPullProducer}.
+    @type _requestProducerStreaming: L{bool} or L{None}
+
+    @ivar _waitingForTransport: A boolean that tracks whether the transport has
+        asked us to stop producing.  This is used to keep track of what we're
+        waiting for: if the transport has asked us to stop producing then we
+        don't want to unpause the transport until it asks us to produce again.
+    @type _waitingForTransport: L{bool}
+
+    @ivar abortTimeout: The number of seconds to wait after we attempt to shut
+        the transport down cleanly to give up and forcibly terminate it.  This
+        is only used when we time a connection out, to prevent errors causing
+        the FD to get leaked.  If this is L{None}, we will wait forever.
+    @type abortTimeout: L{int}
+
+    @ivar _abortingCall: The L{twisted.internet.base.DelayedCall} that will be
+        used to forcibly close the transport if it doesn't close cleanly.
+    @type _abortingCall: L{twisted.internet.base.DelayedCall}
+
+    @ivar _optimisticEagerReadSize: When a resource takes a long time to answer
+        a request (via L{twisted.web.server.NOT_DONE_YET}, hopefully one day by
+        a L{Deferred}), we would like to be able to let that resource know
+        about the underlying transport disappearing as promptly as possible,
+        via L{Request.notifyFinish}, and therefore via
+        C{self.requests[...].connectionLost()} on this L{HTTPChannel}.
+
+        However, in order to simplify application logic, we implement
+        head-of-line blocking, and do not relay pipelined requests to the
+        application until the previous request has been answered.  This means
+        that said application cannot dispose of any entity-body that comes in
+        from those subsequent requests, which may be arbitrarily large, and it
+        may need to be buffered in memory.
+
+        To implement this tradeoff between prompt notification when possible
+        (in the most frequent case of non-pipelined requests) and correct
+        behavior when not (say, if a client sends a very long-running GET
+        request followed by a PUT request with a very large body) we will
+        continue reading pipelined requests into C{self._dataBuffer} up to a
+        given limit.
+
+        C{_optimisticEagerReadSize} is the number of bytes we will accept from
+        the client and buffer before pausing the transport.
+
+        This behavior has been in place since Twisted 17.9.0 .
+
+    @type _optimisticEagerReadSize: L{int}
+    """
+
+    maxHeaders = 500
+    totalHeadersSize = 16384
+    abortTimeout = 15
+
+    length = 0
+    persistent = 1
+    __header = b''
+    __first_line = 1
+    __content = None
+
+    # set in instances or subclasses
+    requestFactory = Request
+
+    _savedTimeOut = None
+    _receivedHeaderCount = 0
+    _receivedHeaderSize = 0
+    _requestProducer = None
+    _requestProducerStreaming = None
+    _waitingForTransport = False
+    _abortingCall = None
+    _optimisticEagerReadSize = 0x4000
+    _log = Logger()
+
+    def __init__(self):
+        # the request queue
+        self.requests = []
+        self._handlingRequest = False
+        self._dataBuffer = []
+        self._transferDecoder = None
+
+
+    def connectionMade(self):
+        self.setTimeout(self.timeOut)
+        self._networkProducer = interfaces.IPushProducer(
+            self.transport, _NoPushProducer()
+        )
+        self._networkProducer.registerProducer(self, True)
+
+
+    def lineReceived(self, line):
+        """
+        Called for each line from request until the end of headers when
+        it enters binary mode.
+        """
+        self.resetTimeout()
+
+        self._receivedHeaderSize += len(line)
+        if (self._receivedHeaderSize > self.totalHeadersSize):
+            self._respondToBadRequestAndDisconnect()
+            return
+
+        if self.__first_line:
+            # if this connection is not persistent, drop any data which
+            # the client (illegally) sent after the last request.
+            if not self.persistent:
+                self.dataReceived = self.lineReceived = lambda *args: None
+                return
+
+            # IE sends an extraneous empty line (\r\n) after a POST request;
+            # eat up such a line, but only ONCE
+            if not line and self.__first_line == 1:
+                self.__first_line = 2
+                return
+
+            # create a new Request object
+            if INonQueuedRequestFactory.providedBy(self.requestFactory):
+                request = self.requestFactory(self)
+            else:
+                request = self.requestFactory(self, len(self.requests))
+            self.requests.append(request)
+
+            self.__first_line = 0
+
+            parts = line.split()
+            if len(parts) != 3:
+                self._respondToBadRequestAndDisconnect()
+                return
+            command, request, version = parts
+            try:
+                command.decode("ascii")
+            except UnicodeDecodeError:
+                self._respondToBadRequestAndDisconnect()
+                return
+
+            self._command = command
+            self._path = request
+            self._version = version
+        elif line == b'':
+            # End of headers.
+            if self.__header:
+                ok = self.headerReceived(self.__header)
+                # If the last header we got is invalid, we MUST NOT proceed
+                # with processing. We'll have sent a 400 anyway, so just stop.
+                if not ok:
+                    return
+            self.__header = b''
+            self.allHeadersReceived()
+            if self.length == 0:
+                self.allContentReceived()
+            else:
+                self.setRawMode()
+        elif line[0] in b' \t':
+            # Continuation of a multi line header.
+            self.__header = self.__header + b'\n' + line
+        # Regular header line.
+        # Processing of header line is delayed to allow accumulating multi
+        # line headers.
+        else:
+            if self.__header:
+                self.headerReceived(self.__header)
+            self.__header = line
+
+
+    def _finishRequestBody(self, data):
+        self.allContentReceived()
+        self._dataBuffer.append(data)
+
+    def _maybeChooseTransferDecoder(self, header, data):
+        """
+        If the provided header is C{content-length} or
+        C{transfer-encoding}, choose the appropriate decoder if any.
+
+        Returns L{True} if the request can proceed and L{False} if not.
+        """
+
+        def fail():
+            self._respondToBadRequestAndDisconnect()
+            self.length = None
+            return False
+
+        # Can this header determine the length?
+        if header == b'content-length':
+            try:
+                length = int(data)
+            except ValueError:
+                return fail()
+            newTransferDecoder = _IdentityTransferDecoder(
+                length, self.requests[-1].handleContentChunk,
+                self._finishRequestBody)
+        elif header == b'transfer-encoding':
+            # XXX Rather poorly tested code block, apparently only exercised by
+            # test_chunkedEncoding
+            if data.lower() == b'chunked':
+                length = None
+                newTransferDecoder = _ChunkedTransferDecoder(
+                    self.requests[-1].handleContentChunk,
+                    self._finishRequestBody)
+            elif data.lower() == b'identity':
+                return True
+            else:
+                return fail()
+        else:
+            # It's not a length related header, so exit
+            return True
+
+        if self._transferDecoder is not None:
+            return fail()
+        else:
+            self.length = length
+            self._transferDecoder = newTransferDecoder
+            return True
+
+
+    def headerReceived(self, line):
+        """
+        Do pre-processing (for content-length) and store this header away.
+        Enforce the per-request header limit.
+
+        @type line: C{bytes}
+        @param line: A line from the header section of a request, excluding the
+            line delimiter.
+
+        @return: A flag indicating whether the header was valid.
+        @rtype: L{bool}
+        """
+        try:
+            header, data = line.split(b':', 1)
+        except ValueError:
+            self._respondToBadRequestAndDisconnect()
+            return False
+
+        if not header or header[-1:].isspace():
+            self._respondToBadRequestAndDisconnect()
+            return False
+
+        header = header.lower()
+        data = data.strip()
+
+        if not self._maybeChooseTransferDecoder(header, data):
+            return False
+
+        reqHeaders = self.requests[-1].requestHeaders
+        values = reqHeaders.getRawHeaders(header)
+        if values is not None:
+            values.append(data)
+        else:
+            reqHeaders.setRawHeaders(header, [data])
+
+        self._receivedHeaderCount += 1
+        if self._receivedHeaderCount > self.maxHeaders:
+            self._respondToBadRequestAndDisconnect()
+            return False
+
+        return True
+
+
+    def allContentReceived(self):
+        command = self._command
+        path = self._path
+        version = self._version
+
+        # reset ALL state variables, so we don't interfere with next request
+        self.length = 0
+        self._receivedHeaderCount = 0
+        self._receivedHeaderSize = 0
+        self.__first_line = 1
+        self._transferDecoder = None
+        del self._command, self._path, self._version
+
+        # Disable the idle timeout, in case this request takes a long
+        # time to finish generating output.
+        if self.timeOut:
+            self._savedTimeOut = self.setTimeout(None)
+
+        self._handlingRequest = True
+
+        req = self.requests[-1]
+        req.requestReceived(command, path, version)
+
+
+    def dataReceived(self, data):
+        """
+        Data was received from the network.  Process it.
+        """
+        # If we're currently handling a request, buffer this data.
+        if self._handlingRequest:
+            self._dataBuffer.append(data)
+            if (
+                    (sum(map(len, self._dataBuffer)) >
+                     self._optimisticEagerReadSize)
+                    and not self._waitingForTransport
+            ):
+                # If we received more data than a small limit while processing
+                # the head-of-line request, apply TCP backpressure to our peer
+                # to get them to stop sending more request data until we're
+                # ready.  See docstring for _optimisticEagerReadSize above.
+                self._networkProducer.pauseProducing()
+            return
+        return basic.LineReceiver.dataReceived(self, data)
+
+
+    def rawDataReceived(self, data):
+        self.resetTimeout()
+
+        try:
+            self._transferDecoder.dataReceived(data)
+        except _MalformedChunkedDataError:
+            self._respondToBadRequestAndDisconnect()
+
+
+    def allHeadersReceived(self):
+        req = self.requests[-1]
+        req.parseCookies()
+        self.persistent = self.checkPersistence(req, self._version)
+        req.gotLength(self.length)
+        # Handle 'Expect: 100-continue' with automated 100 response code,
+        # a simplistic implementation of RFC 2686 8.2.3:
+        expectContinue = req.requestHeaders.getRawHeaders(b'expect')
+        if (expectContinue and expectContinue[0].lower() == b'100-continue' and
+            self._version == b'HTTP/1.1'):
+            self._send100Continue()
+
+
+    def checkPersistence(self, request, version):
+        """
+        Check if the channel should close or not.
+
+        @param request: The request most recently received over this channel
+            against which checks will be made to determine if this connection
+            can remain open after a matching response is returned.
+
+        @type version: C{bytes}
+        @param version: The version of the request.
+
+        @rtype: C{bool}
+        @return: A flag which, if C{True}, indicates that this connection may
+            remain open to receive another request; if C{False}, the connection
+            must be closed in order to indicate the completion of the response
+            to C{request}.
+        """
+        connection = request.requestHeaders.getRawHeaders(b'connection')
+        if connection:
+            tokens = [t.lower() for t in connection[0].split(b' ')]
+        else:
+            tokens = []
+
+        # Once any HTTP 0.9 or HTTP 1.0 request is received, the connection is
+        # no longer allowed to be persistent.  At this point in processing the
+        # request, we don't yet know if it will be possible to set a
+        # Content-Length in the response.  If it is not, then the connection
+        # will have to be closed to end an HTTP 0.9 or HTTP 1.0 response.
+
+        # If the checkPersistence call happened later, after the Content-Length
+        # has been determined (or determined not to be set), it would probably
+        # be possible to have persistent connections with HTTP 0.9 and HTTP 1.0.
+        # This may not be worth the effort, though.  Just use HTTP 1.1, okay?
+
+        if version == b"HTTP/1.1":
+            if b'close' in tokens:
+                request.responseHeaders.setRawHeaders(b'connection', [b'close'])
+                return False
+            else:
+                return True
+        else:
+            return False
+
+
+    def requestDone(self, request):
+        """
+        Called by first request in queue when it is done.
+        """
+        if request != self.requests[0]: raise TypeError
+        del self.requests[0]
+
+        # We should only resume the producer if we're not waiting for the
+        # transport.
+        if not self._waitingForTransport:
+            self._networkProducer.resumeProducing()
+
+        if self.persistent:
+            self._handlingRequest = False
+
+            if self._savedTimeOut:
+                self.setTimeout(self._savedTimeOut)
+
+            # Receive our buffered data, if any.
+            data = b''.join(self._dataBuffer)
+            self._dataBuffer = []
+            self.setLineMode(data)
+        else:
+            self.loseConnection()
+
+
+    def timeoutConnection(self):
+        self._log.info(
+            "Timing out client: {peer}",
+            peer=str(self.transport.getPeer())
+        )
+        if self.abortTimeout is not None:
+            # We use self.callLater because that's what TimeoutMixin does.
+            self._abortingCall = self.callLater(
+                self.abortTimeout, self.forceAbortClient
+            )
+        self.loseConnection()
+
+
+    def forceAbortClient(self):
+        """
+        Called if C{abortTimeout} seconds have passed since the timeout fired,
+        and the connection still hasn't gone away. This can really only happen
+        on extremely bad connections or when clients are maliciously attempting
+        to keep connections open.
+        """
+        self._log.info(
+            "Forcibly timing out client: {peer}",
+            peer=str(self.transport.getPeer())
+        )
+        # We want to lose track of the _abortingCall so that no-one tries to
+        # cancel it.
+        self._abortingCall = None
+        self.transport.abortConnection()
+
+
+    def connectionLost(self, reason):
+        self.setTimeout(None)
+        for request in self.requests:
+            request.connectionLost(reason)
+
+        # If we were going to force-close the transport, we don't have to now.
+        if self._abortingCall is not None:
+            self._abortingCall.cancel()
+            self._abortingCall = None
+
+
+    def isSecure(self):
+        """
+        Return L{True} if this channel is using a secure transport.
+
+        Normally this method returns L{True} if this instance is using a
+        transport that implements L{interfaces.ISSLTransport}.
+
+        @returns: L{True} if this request is secure
+        @rtype: C{bool}
+        """
+        if interfaces.ISSLTransport(self.transport, None) is not None:
+            return True
+        return False
+
+
+    def writeHeaders(self, version, code, reason, headers):
+        """
+        Called by L{Request} objects to write a complete set of HTTP headers to
+        a transport.
+
+        @param version: The HTTP version in use.
+        @type version: L{bytes}
+
+        @param code: The HTTP status code to write.
+        @type code: L{bytes}
+
+        @param reason: The HTTP reason phrase to write.
+        @type reason: L{bytes}
+
+        @param headers: The headers to write to the transport.
+        @type headers: L{twisted.web.http_headers.Headers}
+        """
+        sanitizedHeaders = Headers()
+        for name, value in headers:
+            sanitizedHeaders.addRawHeader(name, value)
+
+        responseLine = version + b" " + code + b" " + reason + b"\r\n"
+        headerSequence = [responseLine]
+        headerSequence.extend(
+            name + b': ' + value + b"\r\n"
+            for name, values in sanitizedHeaders.getAllRawHeaders()
+            for value in values
+        )
+        headerSequence.append(b"\r\n")
+        self.transport.writeSequence(headerSequence)
+
+
+    def write(self, data):
+        """
+        Called by L{Request} objects to write response data.
+
+        @param data: The data chunk to write to the stream.
+        @type data: L{bytes}
+
+        @return: L{None}
+        """
+        self.transport.write(data)
+
+
+    def writeSequence(self, iovec):
+        """
+        Write a list of strings to the HTTP response.
+
+        @param iovec: A list of byte strings to write to the stream.
+        @type data: L{list} of L{bytes}
+
+        @return: L{None}
+        """
+        self.transport.writeSequence(iovec)
+
+
+    def getPeer(self):
+        """
+        Get the remote address of this connection.
+
+        @return: An L{IAddress} provider.
+        """
+        return self.transport.getPeer()
+
+
+    def getHost(self):
+        """
+        Get the local address of this connection.
+
+        @return: An L{IAddress} provider.
+        """
+        return self.transport.getHost()
+
+
+    def loseConnection(self):
+        """
+        Closes the connection. Will write any data that is pending to be sent
+        on the network, but if this response has not yet been written to the
+        network will not write anything.
+
+        @return: L{None}
+        """
+        self._networkProducer.unregisterProducer()
+        return self.transport.loseConnection()
+
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register to receive data from a producer.
+
+        This sets self to be a consumer for a producer.  When this object runs
+        out of data (as when a send(2) call on a socket succeeds in moving the
+        last data from a userspace buffer into a kernelspace buffer), it will
+        ask the producer to resumeProducing().
+
+        For L{IPullProducer} providers, C{resumeProducing} will be called once
+        each time data is required.
+
+        For L{IPushProducer} providers, C{pauseProducing} will be called
+        whenever the write buffer fills up and C{resumeProducing} will only be
+        called when it empties.
+
+        @type producer: L{IProducer} provider
+        @param producer: The L{IProducer} that will be producing data.
+
+        @type streaming: L{bool}
+        @param streaming: C{True} if C{producer} provides L{IPushProducer},
+        C{False} if C{producer} provides L{IPullProducer}.
+
+        @raise RuntimeError: If a producer is already registered.
+
+        @return: L{None}
+        """
+        if self._requestProducer is not None:
+            raise RuntimeError(
+                "Cannot register producer %s, because producer %s was never "
+                "unregistered." % (producer, self._requestProducer))
+
+        if not streaming:
+            producer = _PullToPush(producer, self)
+
+        self._requestProducer = producer
+        self._requestProducerStreaming = streaming
+
+        if not streaming:
+            producer.startStreaming()
+
+
+    def unregisterProducer(self):
+        """
+        Stop consuming data from a producer, without disconnecting.
+
+        @return: L{None}
+        """
+        if self._requestProducer is None:
+            return
+
+        if not self._requestProducerStreaming:
+            self._requestProducer.stopStreaming()
+
+        self._requestProducer = None
+        self._requestProducerStreaming = None
+
+
+    def stopProducing(self):
+        """
+        Stop producing data.
+
+        The HTTPChannel doesn't *actually* implement this, beacuse the
+        assumption is that it will only be called just before C{loseConnection}
+        is called. There's nothing sensible we can do other than call
+        C{loseConnection} anyway.
+        """
+        if self._requestProducer is not None:
+            self._requestProducer.stopProducing()
+
+
+    def pauseProducing(self):
+        """
+        Pause producing data.
+
+        This will be called by the transport when the send buffers have been
+        filled up. We want to simultaneously pause the producing L{Request}
+        object and also pause our transport.
+
+        The logic behind pausing the transport is specifically to avoid issues
+        like https://twistedmatrix.com/trac/ticket/8868. In this case, our
+        inability to send does not prevent us handling more requests, which
+        means we increasingly queue up more responses in our send buffer
+        without end. The easiest way to handle this is to ensure that if we are
+        unable to send our responses, we will not read further data from the
+        connection until the client pulls some data out. This is a bit of a
+        blunt instrument, but it's ok.
+
+        Note that this potentially interacts with timeout handling in a
+        positive way. Once the transport is paused the client may run into a
+        timeout which will cause us to tear the connection down. That's a good
+        thing!
+        """
+        self._waitingForTransport = True
+
+        # The first step is to tell any producer we might currently have
+        # registered to stop producing. If we can slow our applications down
+        # we should.
+        if self._requestProducer is not None:
+            self._requestProducer.pauseProducing()
+
+        # The next step here is to pause our own transport, as discussed in the
+        # docstring.
+        if not self._handlingRequest:
+            self._networkProducer.pauseProducing()
+
+
+    def resumeProducing(self):
+        """
+        Resume producing data.
+
+        This will be called by the transport when the send buffer has dropped
+        enough to actually send more data. When this happens we can unpause any
+        outstanding L{Request} producers we have, and also unpause our
+        transport.
+        """
+        self._waitingForTransport = False
+
+        if self._requestProducer is not None:
+            self._requestProducer.resumeProducing()
+
+        # We only want to resume the network producer if we're not currently
+        # waiting for a response to show up.
+        if not self._handlingRequest:
+            self._networkProducer.resumeProducing()
+
+
+    def _send100Continue(self):
+        """
+        Sends a 100 Continue response, used to signal to clients that further
+        processing will be performed.
+        """
+        self.transport.write(b"HTTP/1.1 100 Continue\r\n\r\n")
+
+
+    def _respondToBadRequestAndDisconnect(self):
+        """
+        This is a quick and dirty way of responding to bad requests.
+
+        As described by HTTP standard we should be patient and accept the
+        whole request from the client before sending a polite bad request
+        response, even in the case when clients send tons of data.
+
+        @param transport: Transport handling connection to the client.
+        @type transport: L{interfaces.ITransport}
+        """
+        self.transport.write(b"HTTP/1.1 400 Bad Request\r\n\r\n")
+        self.loseConnection()
+
+
+
+def _escape(s):
+    """
+    Return a string like python repr, but always escaped as if surrounding
+    quotes were double quotes.
+
+    @param s: The string to escape.
+    @type s: L{bytes} or L{unicode}
+
+    @return: An escaped string.
+    @rtype: L{unicode}
+    """
+    if not isinstance(s, bytes):
+        s = s.encode("ascii")
+
+    r = repr(s)
+    if not isinstance(r, unicode):
+        r = r.decode("ascii")
+    if r.startswith(u"b"):
+        r = r[1:]
+    if r.startswith(u"'"):
+        return r[1:-1].replace(u'"', u'\\"').replace(u"\\'", u"'")
+    return r[1:-1]
+
+
+
+@provider(IAccessLogFormatter)
+def combinedLogFormatter(timestamp, request):
+    """
+    @return: A combined log formatted log line for the given request.
+
+    @see: L{IAccessLogFormatter}
+    """
+    clientAddr = request.getClientAddress()
+    if isinstance(clientAddr, (address.IPv4Address, address.IPv6Address,
+                               _XForwardedForAddress)):
+        ip = clientAddr.host
+    else:
+        ip = b'-'
+    referrer = _escape(request.getHeader(b"referer") or b"-")
+    agent = _escape(request.getHeader(b"user-agent") or b"-")
+    line = (
+        u'"%(ip)s" - - %(timestamp)s "%(method)s %(uri)s %(protocol)s" '
+        u'%(code)d %(length)s "%(referrer)s" "%(agent)s"' % dict(
+            ip=_escape(ip),
+            timestamp=timestamp,
+            method=_escape(request.method),
+            uri=_escape(request.uri),
+            protocol=_escape(request.clientproto),
+            code=request.code,
+            length=request.sentLength or u"-",
+            referrer=referrer,
+            agent=agent,
+            ))
+    return line
+
+
+
+@implementer(interfaces.IAddress)
+class _XForwardedForAddress(object):
+    """
+    L{IAddress} which represents the client IP to log for a request, as gleaned
+    from an X-Forwarded-For header.
+
+    @ivar host: An IP address or C{b"-"}.
+    @type host: L{bytes}
+
+    @see: L{proxiedLogFormatter}
+    """
+    def __init__(self, host):
+        self.host = host
+
+
+
+class _XForwardedForRequest(proxyForInterface(IRequest, "_request")):
+    """
+    Add a layer on top of another request that only uses the value of an
+    X-Forwarded-For header as the result of C{getClientAddress}.
+    """
+    def getClientAddress(self):
+        """
+        The client address (the first address) in the value of the
+        I{X-Forwarded-For header}.  If the header is not present, the IP is
+        considered to be C{b"-"}.
+
+        @return: L{_XForwardedForAddress} which wraps the client address as
+            expected by L{combinedLogFormatter}.
+        """
+        host = self._request.requestHeaders.getRawHeaders(
+            b"x-forwarded-for", [b"-"])[0].split(b",")[0].strip()
+        return _XForwardedForAddress(host)
+
+    # These are missing from the interface.  Forward them manually.
+    @property
+    def clientproto(self):
+        """
+        @return: The protocol version in the request.
+        @rtype: L{bytes}
+        """
+        return self._request.clientproto
+
+    @property
+    def code(self):
+        """
+        @return: The response code for the request.
+        @rtype: L{int}
+        """
+        return self._request.code
+
+    @property
+    def sentLength(self):
+        """
+        @return: The number of bytes sent in the response body.
+        @rtype: L{int}
+        """
+        return self._request.sentLength
+
+
+
+@provider(IAccessLogFormatter)
+def proxiedLogFormatter(timestamp, request):
+    """
+    @return: A combined log formatted log line for the given request but use
+        the value of the I{X-Forwarded-For} header as the value for the client
+        IP address.
+
+    @see: L{IAccessLogFormatter}
+    """
+    return combinedLogFormatter(timestamp, _XForwardedForRequest(request))
+
+
+
+class _GenericHTTPChannelProtocol(proxyForInterface(IProtocol, "_channel")):
+    """
+    A proxy object that wraps one of the HTTP protocol objects, and switches
+    between them depending on TLS negotiated protocol.
+
+    @ivar _negotiatedProtocol: The protocol negotiated with ALPN or NPN, if
+        any.
+    @type _negotiatedProtocol: Either a bytestring containing the ALPN token
+        for the negotiated protocol, or L{None} if no protocol has yet been
+        negotiated.
+
+    @ivar _channel: The object capable of behaving like a L{HTTPChannel} that
+        is backing this object. By default this is a L{HTTPChannel}, but if a
+        HTTP protocol upgrade takes place this may be a different channel
+        object. Must implement L{IProtocol}.
+    @type _channel: L{HTTPChannel}
+
+    @ivar _requestFactory: A callable to use to build L{IRequest} objects.
+    @type _requestFactory: L{IRequest}
+
+    @ivar _site: A reference to the creating L{twisted.web.server.Site} object.
+    @type _site: L{twisted.web.server.Site}
+
+    @ivar _factory: A reference to the creating L{HTTPFactory} object.
+    @type _factory: L{HTTPFactory}
+
+    @ivar _timeOut: A timeout value to pass to the backing channel.
+    @type _timeOut: L{int} or L{None}
+
+    @ivar _callLater: A value for the C{callLater} callback.
+    @type _callLater: L{callable}
+    """
+    _negotiatedProtocol = None
+    _requestFactory = Request
+    _factory = None
+    _site = None
+    _timeOut = None
+    _callLater = None
+
+
+    @property
+    def factory(self):
+        """
+        @see: L{_genericHTTPChannelProtocolFactory}
+        """
+        return self._channel.factory
+
+
+    @factory.setter
+    def factory(self, value):
+        self._factory = value
+        self._channel.factory = value
+
+
+    @property
+    def requestFactory(self):
+        """
+        A callable to use to build L{IRequest} objects.
+
+        Retries the object from the current backing channel.
+        """
+        return self._channel.requestFactory
+
+
+    @requestFactory.setter
+    def requestFactory(self, value):
+        """
+        A callable to use to build L{IRequest} objects.
+
+        Sets the object on the backing channel and also stores the value for
+        propagation to any new channel.
+
+        @param value: The new callable to use.
+        @type value: A L{callable} returning L{IRequest}
+        """
+        self._requestFactory = value
+        self._channel.requestFactory = value
+
+
+    @property
+    def site(self):
+        """
+        A reference to the creating L{twisted.web.server.Site} object.
+
+        Returns the site object from the backing channel.
+        """
+        return self._channel.site
+
+
+    @site.setter
+    def site(self, value):
+        """
+        A reference to the creating L{twisted.web.server.Site} object.
+
+        Sets the object on the backing channel and also stores the value for
+        propagation to any new channel.
+
+        @param value: The L{twisted.web.server.Site} object to set.
+        @type value: L{twisted.web.server.Site}
+        """
+        self._site = value
+        self._channel.site = value
+
+
+    @property
+    def timeOut(self):
+        """
+        The idle timeout for the backing channel.
+        """
+        return self._channel.timeOut
+
+
+    @timeOut.setter
+    def timeOut(self, value):
+        """
+        The idle timeout for the backing channel.
+
+        Sets the idle timeout on both the backing channel and stores it for
+        propagation to any new backing channel.
+
+        @param value: The timeout to set.
+        @type value: L{int} or L{float}
+        """
+        self._timeOut = value
+        self._channel.timeOut = value
+
+
+    @property
+    def callLater(self):
+        """
+        A value for the C{callLater} callback. This callback is used by the
+        L{twisted.protocols.policies.TimeoutMixin} to handle timeouts.
+        """
+        return self._channel.callLater
+
+
+    @callLater.setter
+    def callLater(self, value):
+        """
+        Sets the value for the C{callLater} callback. This callback is used by
+        the L{twisted.protocols.policies.TimeoutMixin} to handle timeouts.
+
+        @param value: The new callback to use.
+        @type value: L{callable}
+        """
+        self._callLater = value
+        self._channel.callLater = value
+
+
+    def dataReceived(self, data):
+        """
+        An override of L{IProtocol.dataReceived} that checks what protocol we're
+        using.
+        """
+        if self._negotiatedProtocol is None:
+            try:
+                negotiatedProtocol = self._channel.transport.negotiatedProtocol
+            except AttributeError:
+                # Plaintext HTTP, always HTTP/1.1
+                negotiatedProtocol = b'http/1.1'
+
+            if negotiatedProtocol is None:
+                negotiatedProtocol = b'http/1.1'
+
+            if negotiatedProtocol == b'h2':
+                if not H2_ENABLED:
+                    raise ValueError("Negotiated HTTP/2 without support.")
+
+                # We need to make sure that the HTTPChannel is unregistered
+                # from the transport so that the H2Connection can register
+                # itself if possible.
+                networkProducer = self._channel._networkProducer
+                networkProducer.unregisterProducer()
+
+                # Cancel the old channel's timeout.
+                self._channel.setTimeout(None)
+
+                # Cancel the old channel's timeout.
+                self._channel.setTimeout(None)
+
+                transport = self._channel.transport
+                self._channel = H2Connection()
+                self._channel.requestFactory = self._requestFactory
+                self._channel.site = self._site
+                self._channel.factory = self._factory
+                self._channel.timeOut = self._timeOut
+                self._channel.callLater = self._callLater
+                self._channel.makeConnection(transport)
+
+                # Register the H2Connection as the transport's
+                # producer, so that the transport can apply back
+                # pressure.
+                networkProducer.registerProducer(self._channel, True)
+            else:
+                # Only HTTP/2 and HTTP/1.1 are supported right now.
+                assert negotiatedProtocol == b'http/1.1', \
+                       "Unsupported protocol negotiated"
+
+            self._negotiatedProtocol = negotiatedProtocol
+
+        return self._channel.dataReceived(data)
+
+
+
+def _genericHTTPChannelProtocolFactory(self):
+    """
+    Returns an appropriately initialized _GenericHTTPChannelProtocol.
+    """
+    return _GenericHTTPChannelProtocol(HTTPChannel())
+
+
+
+class HTTPFactory(protocol.ServerFactory):
+    """
+    Factory for HTTP server.
+
+    @ivar _logDateTime: A cached datetime string for log messages, updated by
+        C{_logDateTimeCall}.
+    @type _logDateTime: C{str}
+
+    @ivar _logDateTimeCall: A delayed call for the next update to the cached
+        log datetime string.
+    @type _logDateTimeCall: L{IDelayedCall} provided
+
+    @ivar _logFormatter: See the C{logFormatter} parameter to L{__init__}
+
+    @ivar _nativeize: A flag that indicates whether the log file being written
+        to wants native strings (C{True}) or bytes (C{False}).  This is only to
+        support writing to L{twisted.python.log} which, unfortunately, works
+        with native strings.
+
+    @ivar _reactor: An L{IReactorTime} provider used to compute logging
+        timestamps.
+    """
+
+    protocol = _genericHTTPChannelProtocolFactory
+
+    logPath = None
+
+    timeOut = _REQUEST_TIMEOUT
+
+    def __init__(self, logPath=None, timeout=_REQUEST_TIMEOUT,
+                 logFormatter=None, reactor=None):
+        """
+        @param logPath: File path to which access log messages will be written
+            or C{None} to disable logging.
+        @type logPath: L{str} or L{bytes}
+
+        @param timeout: The initial value of L{timeOut}, which defines the idle
+            connection timeout in seconds, or C{None} to disable the idle
+            timeout.
+        @type timeout: L{float}
+
+        @param logFormatter: An object to format requests into log lines for
+            the access log.  L{combinedLogFormatter} when C{None} is passed.
+        @type logFormatter: L{IAccessLogFormatter} provider
+
+        @param reactor: A L{IReactorTime} provider used to manage connection
+            timeouts and compute logging timestamps.
+        """
+        if not reactor:
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+        if logPath is not None:
+            logPath = os.path.abspath(logPath)
+        self.logPath = logPath
+        self.timeOut = timeout
+        if logFormatter is None:
+            logFormatter = combinedLogFormatter
+        self._logFormatter = logFormatter
+
+        # For storing the cached log datetime and the callback to update it
+        self._logDateTime = None
+        self._logDateTimeCall = None
+
+
+    def _updateLogDateTime(self):
+        """
+        Update log datetime periodically, so we aren't always recalculating it.
+        """
+        self._logDateTime = datetimeToLogString(self._reactor.seconds())
+        self._logDateTimeCall = self._reactor.callLater(1, self._updateLogDateTime)
+
+
+    def buildProtocol(self, addr):
+        p = protocol.ServerFactory.buildProtocol(self, addr)
+
+        # This is a bit of a hack to ensure that the HTTPChannel timeouts
+        # occur on the same reactor as the one we're using here. This could
+        # ideally be resolved by passing the reactor more generally to the
+        # HTTPChannel, but that won't work for the TimeoutMixin until we fix
+        # https://twistedmatrix.com/trac/ticket/8488
+        p.callLater = self._reactor.callLater
+
+        # timeOut needs to be on the Protocol instance cause
+        # TimeoutMixin expects it there
+        p.timeOut = self.timeOut
+        return p
+
+
+    def startFactory(self):
+        """
+        Set up request logging if necessary.
+        """
+        if self._logDateTimeCall is None:
+            self._updateLogDateTime()
+
+        if self.logPath:
+            self.logFile = self._openLogFile(self.logPath)
+        else:
+            self.logFile = log.logfile
+
+
+    def stopFactory(self):
+        if hasattr(self, "logFile"):
+            if self.logFile != log.logfile:
+                self.logFile.close()
+            del self.logFile
+
+        if self._logDateTimeCall is not None and self._logDateTimeCall.active():
+            self._logDateTimeCall.cancel()
+            self._logDateTimeCall = None
+
+
+    def _openLogFile(self, path):
+        """
+        Override in subclasses, e.g. to use L{twisted.python.logfile}.
+        """
+        f = open(path, "ab", 1)
+        return f
+
+
+    def log(self, request):
+        """
+        Write a line representing C{request} to the access log file.
+
+        @param request: The request object about which to log.
+        @type request: L{Request}
+        """
+        try:
+            logFile = self.logFile
+        except AttributeError:
+            pass
+        else:
+            line = self._logFormatter(self._logDateTime, request) + u"\n"
+            logFile.write(line.encode('utf8'))
diff --git a/contrib/python/Twisted/py2/twisted/web/http_headers.py b/contrib/python/Twisted/py2/twisted/web/http_headers.py
new file mode 100644
index 00000000000..5b141ac74c0
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/http_headers.py
@@ -0,0 +1,294 @@
+# -*- test-case-name: twisted.web.test.test_http_headers -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An API for storing HTTP header names and values.
+"""
+
+from __future__ import division, absolute_import
+
+from twisted.python.compat import comparable, cmp, unicode
+
+
+def _dashCapitalize(name):
+    """
+    Return a byte string which is capitalized using '-' as a word separator.
+
+    @param name: The name of the header to capitalize.
+    @type name: L{bytes}
+
+    @return: The given header capitalized using '-' as a word separator.
+    @rtype: L{bytes}
+    """
+    return b'-'.join([word.capitalize() for word in name.split(b'-')])
+
+
+
+def _sanitizeLinearWhitespace(headerComponent):
+    r"""
+    Replace linear whitespace (C{\n}, C{\r\n}, C{\r}) in a header key
+    or value with a single space.  If C{headerComponent} is not
+    L{bytes}, it is passed through unchanged.
+
+    @param headerComponent: The header key or value to sanitize.
+    @type headerComponent: L{bytes}
+
+    @return: The sanitized header key or value.
+    @rtype: L{bytes}
+    """
+    return b' '.join(headerComponent.splitlines())
+
+
+
+@comparable
+class Headers(object):
+    """
+    Stores HTTP headers in a key and multiple value format.
+
+    Most methods accept L{bytes} and L{unicode}, with an internal L{bytes}
+    representation. When passed L{unicode}, header names (e.g. 'Content-Type')
+    are encoded using ISO-8859-1 and header values (e.g.
+    'text/html;charset=utf-8') are encoded using UTF-8. Some methods that return
+    values will return them in the same type as the name given.
+
+    If the header keys or values cannot be encoded or decoded using the rules
+    above, using just L{bytes} arguments to the methods of this class will
+    ensure no decoding or encoding is done, and L{Headers} will treat the keys
+    and values as opaque byte strings.
+
+    @cvar _caseMappings: A L{dict} that maps lowercase header names
+        to their canonicalized representation.
+
+    @ivar _rawHeaders: A L{dict} mapping header names as L{bytes} to L{list}s of
+        header values as L{bytes}.
+    """
+    _caseMappings = {
+        b'content-md5': b'Content-MD5',
+        b'dnt': b'DNT',
+        b'etag': b'ETag',
+        b'p3p': b'P3P',
+        b'te': b'TE',
+        b'www-authenticate': b'WWW-Authenticate',
+        b'x-xss-protection': b'X-XSS-Protection'}
+
+    def __init__(self, rawHeaders=None):
+        self._rawHeaders = {}
+        if rawHeaders is not None:
+            for name, values in rawHeaders.items():
+                self.setRawHeaders(name, values)
+
+
+    def __repr__(self):
+        """
+        Return a string fully describing the headers set on this object.
+        """
+        return '%s(%r)' % (self.__class__.__name__, self._rawHeaders,)
+
+
+    def __cmp__(self, other):
+        """
+        Define L{Headers} instances as being equal to each other if they have
+        the same raw headers.
+        """
+        if isinstance(other, Headers):
+            return cmp(
+                sorted(self._rawHeaders.items()),
+                sorted(other._rawHeaders.items()))
+        return NotImplemented
+
+
+    def _encodeName(self, name):
+        """
+        Encode the name of a header (eg 'Content-Type') to an ISO-8859-1 encoded
+        bytestring if required.
+
+        @param name: A HTTP header name
+        @type name: L{unicode} or L{bytes}
+
+        @return: C{name}, encoded if required, lowercased
+        @rtype: L{bytes}
+        """
+        if isinstance(name, unicode):
+            return name.lower().encode('iso-8859-1')
+        return name.lower()
+
+
+    def _encodeValue(self, value):
+        """
+        Encode a single header value to a UTF-8 encoded bytestring if required.
+
+        @param value: A single HTTP header value.
+        @type value: L{bytes} or L{unicode}
+
+        @return: C{value}, encoded if required
+        @rtype: L{bytes}
+        """
+        if isinstance(value, unicode):
+            return value.encode('utf8')
+        return value
+
+
+    def _encodeValues(self, values):
+        """
+        Encode a L{list} of header values to a L{list} of UTF-8 encoded
+        bytestrings if required.
+
+        @param values: A list of HTTP header values.
+        @type values: L{list} of L{bytes} or L{unicode} (mixed types allowed)
+
+        @return: C{values}, with each item encoded if required
+        @rtype: L{list} of L{bytes}
+        """
+        newValues = []
+
+        for value in values:
+            newValues.append(self._encodeValue(value))
+        return newValues
+
+
+    def _decodeValues(self, values):
+        """
+        Decode a L{list} of header values into a L{list} of Unicode strings.
+
+        @param values: A list of HTTP header values.
+        @type values: L{list} of UTF-8 encoded L{bytes}
+
+        @return: C{values}, with each item decoded
+        @rtype: L{list} of L{unicode}
+        """
+        newValues = []
+
+        for value in values:
+            newValues.append(value.decode('utf8'))
+        return newValues
+
+
+    def copy(self):
+        """
+        Return a copy of itself with the same headers set.
+
+        @return: A new L{Headers}
+        """
+        return self.__class__(self._rawHeaders)
+
+
+    def hasHeader(self, name):
+        """
+        Check for the existence of a given header.
+
+        @type name: L{bytes} or L{unicode}
+        @param name: The name of the HTTP header to check for.
+
+        @rtype: L{bool}
+        @return: C{True} if the header exists, otherwise C{False}.
+        """
+        return self._encodeName(name) in self._rawHeaders
+
+
+    def removeHeader(self, name):
+        """
+        Remove the named header from this header object.
+
+        @type name: L{bytes} or L{unicode}
+        @param name: The name of the HTTP header to remove.
+
+        @return: L{None}
+        """
+        self._rawHeaders.pop(self._encodeName(name), None)
+
+
+    def setRawHeaders(self, name, values):
+        """
+        Sets the raw representation of the given header.
+
+        @type name: L{bytes} or L{unicode}
+        @param name: The name of the HTTP header to set the values for.
+
+        @type values: L{list} of L{bytes} or L{unicode} strings
+        @param values: A list of strings each one being a header value of
+            the given name.
+
+        @return: L{None}
+        """
+        if not isinstance(values, list):
+            raise TypeError("Header entry %r should be list but found "
+                            "instance of %r instead" % (name, type(values)))
+
+        name = _sanitizeLinearWhitespace(self._encodeName(name))
+        encodedValues = [_sanitizeLinearWhitespace(v)
+                         for v in self._encodeValues(values)]
+
+        self._rawHeaders[name] = self._encodeValues(encodedValues)
+
+
+    def addRawHeader(self, name, value):
+        """
+        Add a new raw value for the given header.
+
+        @type name: L{bytes} or L{unicode}
+        @param name: The name of the header for which to set the value.
+
+        @type value: L{bytes} or L{unicode}
+        @param value: The value to set for the named header.
+        """
+        values = self.getRawHeaders(name)
+
+        if values is not None:
+            values.append(value)
+        else:
+            values = [value]
+
+        self.setRawHeaders(name, values)
+
+
+    def getRawHeaders(self, name, default=None):
+        """
+        Returns a list of headers matching the given name as the raw string
+        given.
+
+        @type name: L{bytes} or L{unicode}
+        @param name: The name of the HTTP header to get the values of.
+
+        @param default: The value to return if no header with the given C{name}
+            exists.
+
+        @rtype: L{list} of strings, same type as C{name} (except when
+            C{default} is returned).
+        @return: If the named header is present, a L{list} of its
+            values.  Otherwise, C{default}.
+        """
+        encodedName = self._encodeName(name)
+        values = self._rawHeaders.get(encodedName, default)
+
+        if isinstance(name, unicode) and values is not default:
+            return self._decodeValues(values)
+        return values
+
+
+    def getAllRawHeaders(self):
+        """
+        Return an iterator of key, value pairs of all headers contained in this
+        object, as L{bytes}.  The keys are capitalized in canonical
+        capitalization.
+        """
+        for k, v in self._rawHeaders.items():
+            yield self._canonicalNameCaps(k), v
+
+
+    def _canonicalNameCaps(self, name):
+        """
+        Return the canonical name for the given header.
+
+        @type name: L{bytes}
+        @param name: The all-lowercase header name to capitalize in its
+            canonical form.
+
+        @rtype: L{bytes}
+        @return: The canonical name of the header.
+        """
+        return self._caseMappings.get(name, _dashCapitalize(name))
+
+
+
+__all__ = ['Headers']
diff --git a/contrib/python/Twisted/py2/twisted/web/iweb.py b/contrib/python/Twisted/py2/twisted/web/iweb.py
new file mode 100644
index 00000000000..47dc8d3257c
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/iweb.py
@@ -0,0 +1,828 @@
+# -*- test-case-name: twisted.web.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Interface definitions for L{twisted.web}.
+
+@var UNKNOWN_LENGTH: An opaque object which may be used as the value of
+    L{IBodyProducer.length} to indicate that the length of the entity
+    body is not known in advance.
+"""
+
+from zope.interface import Interface, Attribute
+
+from twisted.internet.interfaces import IPushProducer
+from twisted.cred.credentials import IUsernameDigestHash
+
+
+class IRequest(Interface):
+    """
+    An HTTP request.
+
+    @since: 9.0
+    """
+
+    method = Attribute("A L{bytes} giving the HTTP method that was used.")
+    uri = Attribute(
+        "A L{bytes} giving the full encoded URI which was requested (including"
+        " query arguments).")
+    path = Attribute(
+        "A L{bytes} giving the encoded query path of the request URI (not "
+        "including query arguments).")
+    args = Attribute(
+        "A mapping of decoded query argument names as L{bytes} to "
+        "corresponding query argument values as L{list}s of L{bytes}.  "
+        "For example, for a URI with C{foo=bar&foo=baz&quux=spam} "
+        "for its query part, C{args} will be C{{b'foo': [b'bar', b'baz'], "
+        "b'quux': [b'spam']}}.")
+
+    prepath = Attribute(
+        "The URL path segments which have been processed during resource "
+        "traversal, as a list of {bytes}.")
+
+    postpath = Attribute(
+        "The URL path segments which have not (yet) been processed "
+        "during resource traversal, as a list of L{bytes}.")
+
+    requestHeaders = Attribute(
+        "A L{http_headers.Headers} instance giving all received HTTP request "
+        "headers.")
+
+    content = Attribute(
+        "A file-like object giving the request body.  This may be a file on "
+        "disk, an L{io.BytesIO}, or some other type.  The implementation is "
+        "free to decide on a per-request basis.")
+
+    responseHeaders = Attribute(
+        "A L{http_headers.Headers} instance holding all HTTP response "
+        "headers to be sent.")
+
+    def getHeader(key):
+        """
+        Get an HTTP request header.
+
+        @type key: L{bytes} or L{str}
+        @param key: The name of the header to get the value of.
+
+        @rtype: L{bytes} or L{str} or L{None}
+        @return: The value of the specified header, or L{None} if that header
+            was not present in the request. The string type of the result
+            matches the type of C{key}.
+        """
+
+
+    def getCookie(key):
+        """
+        Get a cookie that was sent from the network.
+
+        @type key: L{bytes}
+        @param key: The name of the cookie to get.
+
+        @rtype: L{bytes} or L{None}
+        @returns: The value of the specified cookie, or L{None} if that cookie
+            was not present in the request.
+        """
+
+
+    def getAllHeaders():
+        """
+        Return dictionary mapping the names of all received headers to the last
+        value received for each.
+
+        Since this method does not return all header information,
+        C{requestHeaders.getAllRawHeaders()} may be preferred.
+        """
+
+
+    def getRequestHostname():
+        """
+        Get the hostname that the user passed in to the request.
+
+        This will either use the Host: header (if it is available) or the
+        host we are listening on if the header is unavailable.
+
+        @returns: the requested hostname
+        @rtype: L{str}
+        """
+
+
+    def getHost():
+        """
+        Get my originally requesting transport's host.
+
+        @return: An L{IAddress}.
+        """
+
+
+    def getClientAddress():
+        """
+        Return the address of the client who submitted this request.
+
+        The address may not be a network address.  Callers must check
+        its type before using it.
+
+        @since: 18.4
+
+        @return: the client's address.
+        @rtype: an L{IAddress} provider.
+        """
+
+
+    def getClientIP():
+        """
+        Return the IP address of the client who submitted this request.
+
+        This method is B{deprecated}.  See L{getClientAddress} instead.
+
+        @returns: the client IP address or L{None} if the request was submitted
+            over a transport where IP addresses do not make sense.
+        @rtype: L{str} or L{None}
+        """
+
+
+    def getUser():
+        """
+        Return the HTTP user sent with this request, if any.
+
+        If no user was supplied, return the empty string.
+
+        @returns: the HTTP user, if any
+        @rtype: L{str}
+        """
+
+
+    def getPassword():
+        """
+        Return the HTTP password sent with this request, if any.
+
+        If no password was supplied, return the empty string.
+
+        @returns: the HTTP password, if any
+        @rtype: L{str}
+        """
+
+
+    def isSecure():
+        """
+        Return True if this request is using a secure transport.
+
+        Normally this method returns True if this request's HTTPChannel
+        instance is using a transport that implements ISSLTransport.
+
+        This will also return True if setHost() has been called
+        with ssl=True.
+
+        @returns: True if this request is secure
+        @rtype: C{bool}
+        """
+
+
+    def getSession(sessionInterface=None):
+        """
+        Look up the session associated with this request or create a new one if
+        there is not one.
+
+        @return: The L{Session} instance identified by the session cookie in
+            the request, or the C{sessionInterface} component of that session
+            if C{sessionInterface} is specified.
+        """
+
+
+    def URLPath():
+        """
+        @return: A L{URLPath} instance
+            which identifies the URL for which this request is.
+        """
+
+
+    def prePathURL():
+        """
+        At any time during resource traversal or resource rendering,
+        returns an absolute URL to the most nested resource which has
+        yet been reached.
+
+        @see: {twisted.web.server.Request.prepath}
+
+        @return: An absolute URL.
+        @type: L{bytes}
+        """
+
+
+    def rememberRootURL():
+        """
+        Remember the currently-processed part of the URL for later
+        recalling.
+        """
+
+
+    def getRootURL():
+        """
+        Get a previously-remembered URL.
+
+        @return: An absolute URL.
+        @type: L{bytes}
+        """
+
+
+    # Methods for outgoing response
+    def finish():
+        """
+        Indicate that the response to this request is complete.
+        """
+
+
+    def write(data):
+        """
+        Write some data to the body of the response to this request.  Response
+        headers are written the first time this method is called, after which
+        new response headers may not be added.
+
+        @param data: Bytes of the response body.
+        @type data: L{bytes}
+        """
+
+
+    def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
+        """
+        Set an outgoing HTTP cookie.
+
+        In general, you should consider using sessions instead of cookies, see
+        L{twisted.web.server.Request.getSession} and the
+        L{twisted.web.server.Session} class for details.
+        """
+
+
+    def setResponseCode(code, message=None):
+        """
+        Set the HTTP response code.
+
+        @type code: L{int}
+        @type message: L{bytes}
+        """
+
+
+    def setHeader(k, v):
+        """
+        Set an HTTP response header.  Overrides any previously set values for
+        this header.
+
+        @type k: L{bytes} or L{str}
+        @param k: The name of the header for which to set the value.
+
+        @type v: L{bytes} or L{str}
+        @param v: The value to set for the named header. A L{str} will be
+            UTF-8 encoded, which may not interoperable with other
+            implementations. Avoid passing non-ASCII characters if possible.
+        """
+
+
+    def redirect(url):
+        """
+        Utility function that does a redirect.
+
+        The request should have finish() called after this.
+        """
+
+
+    def setLastModified(when):
+        """
+        Set the C{Last-Modified} time for the response to this request.
+
+        If I am called more than once, I ignore attempts to set Last-Modified
+        earlier, only replacing the Last-Modified time if it is to a later
+        value.
+
+        If I am a conditional request, I may modify my response code to
+        L{NOT_MODIFIED} if appropriate for the time given.
+
+        @param when: The last time the resource being returned was modified, in
+            seconds since the epoch.
+        @type when: L{int}, L{long} or L{float}
+
+        @return: If I am a C{If-Modified-Since} conditional request and the time
+            given is not newer than the condition, I return
+            L{CACHED} to indicate that you should write no body.
+            Otherwise, I return a false value.
+        """
+
+
+    def setETag(etag):
+        """
+        Set an C{entity tag} for the outgoing response.
+
+        That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for
+        comparing two or more entities from the same requested resource."
+
+        If I am a conditional request, I may modify my response code to
+        L{NOT_MODIFIED} or
+        L{PRECONDITION_FAILED}, if appropriate for the
+        tag given.
+
+        @param etag: The entity tag for the resource being returned.
+        @type etag: L{str}
+
+        @return: If I am a C{If-None-Match} conditional request and the tag
+            matches one in the request, I return L{CACHED} to
+            indicate that you should write no body.  Otherwise, I return a
+            false value.
+        """
+
+
+    def setHost(host, port, ssl=0):
+        """
+        Change the host and port the request thinks it's using.
+
+        This method is useful for working with reverse HTTP proxies (e.g.  both
+        Squid and Apache's mod_proxy can do this), when the address the HTTP
+        client is using is different than the one we're listening on.
+
+        For example, Apache may be listening on https://www.example.com, and
+        then forwarding requests to http://localhost:8080, but we don't want
+        HTML produced by Twisted to say 'http://localhost:8080', they should
+        say 'https://www.example.com', so we do::
+
+           request.setHost('www.example.com', 443, ssl=1)
+        """
+
+
+
+class INonQueuedRequestFactory(Interface):
+    """
+    A factory of L{IRequest} objects that does not take a ``queued`` parameter.
+    """
+    def __call__(channel):
+        """
+        Create an L{IRequest} that is operating on the given channel. There
+        must only be one L{IRequest} object processing at any given time on a
+        channel.
+
+        @param channel: A L{twisted.web.http.HTTPChannel} object.
+        @type channel: L{twisted.web.http.HTTPChannel}
+
+        @return: A request object.
+        @rtype: L{IRequest}
+        """
+
+
+
+class IAccessLogFormatter(Interface):
+    """
+    An object which can represent an HTTP request as a line of text for
+    inclusion in an access log file.
+    """
+    def __call__(timestamp, request):
+        """
+        Generate a line for the access log.
+
+        @param timestamp: The time at which the request was completed in the
+            standard format for access logs.
+        @type timestamp: L{unicode}
+
+        @param request: The request object about which to log.
+        @type request: L{twisted.web.server.Request}
+
+        @return: One line describing the request without a trailing newline.
+        @rtype: L{unicode}
+        """
+
+
+
+class ICredentialFactory(Interface):
+    """
+    A credential factory defines a way to generate a particular kind of
+    authentication challenge and a way to interpret the responses to these
+    challenges.  It creates
+    L{ICredentials} providers from
+    responses.  These objects will be used with L{twisted.cred} to authenticate
+    an authorize requests.
+    """
+    scheme = Attribute(
+        "A L{str} giving the name of the authentication scheme with which "
+        "this factory is associated.  For example, C{'basic'} or C{'digest'}.")
+
+
+    def getChallenge(request):
+        """
+        Generate a new challenge to be sent to a client.
+
+        @type peer: L{twisted.web.http.Request}
+        @param peer: The request the response to which this challenge will be
+            included.
+
+        @rtype: L{dict}
+        @return: A mapping from L{str} challenge fields to associated L{str}
+            values.
+        """
+
+
+    def decode(response, request):
+        """
+        Create a credentials object from the given response.
+
+        @type response: L{str}
+        @param response: scheme specific response string
+
+        @type request: L{twisted.web.http.Request}
+        @param request: The request being processed (from which the response
+            was taken).
+
+        @raise twisted.cred.error.LoginFailed: If the response is invalid.
+
+        @rtype: L{twisted.cred.credentials.ICredentials} provider
+        @return: The credentials represented by the given response.
+        """
+
+
+
+class IBodyProducer(IPushProducer):
+    """
+    Objects which provide L{IBodyProducer} write bytes to an object which
+    provides L{IConsumer} by calling its
+    C{write} method repeatedly.
+
+    L{IBodyProducer} providers may start producing as soon as they have an
+    L{IConsumer} provider.  That is, they
+    should not wait for a C{resumeProducing} call to begin writing data.
+
+    L{IConsumer.unregisterProducer}
+    must not be called.  Instead, the
+    L{Deferred} returned from C{startProducing}
+    must be fired when all bytes have been written.
+
+    L{IConsumer.write} may
+    synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or
+    C{stopProducing}.  These methods must be implemented with this in mind.
+
+    @since: 9.0
+    """
+
+    # Despite the restrictions above and the additional requirements of
+    # stopProducing documented below, this interface still needs to be an
+    # IPushProducer subclass.  Providers of it will be passed to IConsumer
+    # providers which only know about IPushProducer and IPullProducer, not
+    # about this interface.  This interface needs to remain close enough to one
+    # of those interfaces for consumers to work with it.
+
+    length = Attribute(
+        """
+        C{length} is a L{int} indicating how many bytes in total this
+        L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH}
+        if this is not known in advance.
+        """)
+
+    def startProducing(consumer):
+        """
+        Start producing to the given
+        L{IConsumer} provider.
+
+        @return: A L{Deferred} which stops
+            production of data when L{Deferred.cancel} is called, and which
+            fires with L{None} when all bytes have been produced or with a
+            L{Failure} if there is any problem
+            before all bytes have been produced.
+        """
+
+
+    def stopProducing():
+        """
+        In addition to the standard behavior of
+        L{IProducer.stopProducing}
+        (stop producing data), make sure the
+        L{Deferred} returned by
+        C{startProducing} is never fired.
+        """
+
+
+
+class IRenderable(Interface):
+    """
+    An L{IRenderable} is an object that may be rendered by the
+    L{twisted.web.template} templating system.
+    """
+
+    def lookupRenderMethod(name):
+        """
+        Look up and return the render method associated with the given name.
+
+        @type name: L{str}
+        @param name: The value of a render directive encountered in the
+            document returned by a call to L{IRenderable.render}.
+
+        @return: A two-argument callable which will be invoked with the request
+            being responded to and the tag object on which the render directive
+            was encountered.
+        """
+
+
+    def render(request):
+        """
+        Get the document for this L{IRenderable}.
+
+        @type request: L{IRequest} provider or L{None}
+        @param request: The request in response to which this method is being
+            invoked.
+
+        @return: An object which can be flattened.
+        """
+
+
+
+class ITemplateLoader(Interface):
+    """
+    A loader for templates; something usable as a value for
+    L{twisted.web.template.Element}'s C{loader} attribute.
+    """
+
+    def load():
+        """
+        Load a template suitable for rendering.
+
+        @return: a L{list} of L{list}s, L{unicode} objects, C{Element}s and
+            other L{IRenderable} providers.
+        """
+
+
+
+class IResponse(Interface):
+    """
+    An object representing an HTTP response received from an HTTP server.
+
+    @since: 11.1
+    """
+
+    version = Attribute(
+        "A three-tuple describing the protocol and protocol version "
+        "of the response.  The first element is of type L{str}, the second "
+        "and third are of type L{int}.  For example, C{(b'HTTP', 1, 1)}.")
+
+
+    code = Attribute("The HTTP status code of this response, as a L{int}.")
+
+
+    phrase = Attribute(
+        "The HTTP reason phrase of this response, as a L{str}.")
+
+
+    headers = Attribute("The HTTP response L{Headers} of this response.")
+
+
+    length = Attribute(
+        "The L{int} number of bytes expected to be in the body of this "
+        "response or L{UNKNOWN_LENGTH} if the server did not indicate how "
+        "many bytes to expect.  For I{HEAD} responses, this will be 0; if "
+        "the response includes a I{Content-Length} header, it will be "
+        "available in C{headers}.")
+
+
+    request = Attribute(
+        "The L{IClientRequest} that resulted in this response.")
+
+
+    previousResponse = Attribute(
+        "The previous L{IResponse} from a redirect, or L{None} if there was no "
+        "previous response. This can be used to walk the response or request "
+        "history for redirections.")
+
+
+    def deliverBody(protocol):
+        """
+        Register an L{IProtocol} provider
+        to receive the response body.
+
+        The protocol will be connected to a transport which provides
+        L{IPushProducer}.  The protocol's C{connectionLost} method will be
+        called with:
+
+            - ResponseDone, which indicates that all bytes from the response
+              have been successfully delivered.
+
+            - PotentialDataLoss, which indicates that it cannot be determined
+              if the entire response body has been delivered.  This only occurs
+              when making requests to HTTP servers which do not set
+              I{Content-Length} or a I{Transfer-Encoding} in the response.
+
+            - ResponseFailed, which indicates that some bytes from the response
+              were lost.  The C{reasons} attribute of the exception may provide
+              more specific indications as to why.
+        """
+
+
+    def setPreviousResponse(response):
+        """
+        Set the reference to the previous L{IResponse}.
+
+        The value of the previous response can be read via
+        L{IResponse.previousResponse}.
+        """
+
+
+
+class _IRequestEncoder(Interface):
+    """
+    An object encoding data passed to L{IRequest.write}, for example for
+    compression purpose.
+
+    @since: 12.3
+    """
+
+    def encode(data):
+        """
+        Encode the data given and return the result.
+
+        @param data: The content to encode.
+        @type data: L{str}
+
+        @return: The encoded data.
+        @rtype: L{str}
+        """
+
+
+    def finish():
+        """
+        Callback called when the request is closing.
+
+        @return: If necessary, the pending data accumulated from previous
+            C{encode} calls.
+        @rtype: L{str}
+        """
+
+
+
+class _IRequestEncoderFactory(Interface):
+    """
+    A factory for returing L{_IRequestEncoder} instances.
+
+    @since: 12.3
+    """
+
+    def encoderForRequest(request):
+        """
+        If applicable, returns a L{_IRequestEncoder} instance which will encode
+        the request.
+        """
+
+
+
+class IClientRequest(Interface):
+    """
+    An object representing an HTTP request to make to an HTTP server.
+
+    @since: 13.1
+    """
+    method = Attribute(
+        "The HTTP method for this request, as L{bytes}. For example: "
+        "C{b'GET'}, C{b'HEAD'}, C{b'POST'}, etc.")
+
+
+    absoluteURI = Attribute(
+        "The absolute URI of the requested resource, as L{bytes}; or L{None} "
+        "if the absolute URI cannot be determined.")
+
+
+    headers = Attribute(
+        "Headers to be sent to the server, as "
+        "a L{twisted.web.http_headers.Headers} instance.")
+
+
+
+class IAgent(Interface):
+    """
+    An agent makes HTTP requests.
+
+    The way in which requests are issued is left up to each implementation.
+    Some may issue them directly to the server indicated by the net location
+    portion of the request URL.  Others may use a proxy specified by system
+    configuration.
+
+    Processing of responses is also left very widely specified.  An
+    implementation may perform no special handling of responses, or it may
+    implement redirect following or content negotiation, it may implement a
+    cookie store or automatically respond to authentication challenges.  It may
+    implement many other unforeseen behaviors as well.
+
+    It is also intended that L{IAgent} implementations be composable.  An
+    implementation which provides cookie handling features should re-use an
+    implementation that provides connection pooling and this combination could
+    be used by an implementation which adds content negotiation functionality.
+    Some implementations will be completely self-contained, such as those which
+    actually perform the network operations to send and receive requests, but
+    most or all other implementations should implement a small number of new
+    features (perhaps one new feature) and delegate the rest of the
+    request/response machinery to another implementation.
+
+    This allows for great flexibility in the behavior an L{IAgent} will
+    provide.  For example, an L{IAgent} with web browser-like behavior could be
+    obtained by combining a number of (hypothetical) implementations::
+
+        baseAgent = Agent(reactor)
+        redirect = BrowserLikeRedirectAgent(baseAgent, limit=10)
+        authenticate = AuthenticateAgent(
+            redirect, [diskStore.credentials, GtkAuthInterface()])
+        cookie = CookieAgent(authenticate, diskStore.cookie)
+        decode = ContentDecoderAgent(cookie, [(b"gzip", GzipDecoder())])
+        cache = CacheAgent(decode, diskStore.cache)
+
+        doSomeRequests(cache)
+    """
+    def request(method, uri, headers=None, bodyProducer=None):
+        """
+        Request the resource at the given location.
+
+        @param method: The request method to use, such as C{"GET"}, C{"HEAD"},
+            C{"PUT"}, C{"POST"}, etc.
+        @type method: L{bytes}
+
+        @param uri: The location of the resource to request.  This should be an
+            absolute URI but some implementations may support relative URIs
+            (with absolute or relative paths).  I{HTTP} and I{HTTPS} are the
+            schemes most likely to be supported but others may be as well.
+        @type uri: L{bytes}
+
+        @param headers: The headers to send with the request (or L{None} to
+            send no extra headers).  An implementation may add its own headers
+            to this (for example for client identification or content
+            negotiation).
+        @type headers: L{Headers} or L{None}
+
+        @param bodyProducer: An object which can generate bytes to make up the
+            body of this request (for example, the properly encoded contents of
+            a file for a file upload).  Or, L{None} if the request is to have
+            no body.
+        @type bodyProducer: L{IBodyProducer} provider
+
+        @return: A L{Deferred} that fires with an L{IResponse} provider when
+            the header of the response has been received (regardless of the
+            response status code) or with a L{Failure} if there is any problem
+            which prevents that response from being received (including
+            problems that prevent the request from being sent).
+        @rtype: L{Deferred}
+        """
+
+
+class IPolicyForHTTPS(Interface):
+    """
+    An L{IPolicyForHTTPS} provides a policy for verifying the certificates of
+    HTTPS connections, in the form of a L{client connection creator
+    } per network
+    location.
+
+    @since: 14.0
+    """
+
+    def creatorForNetloc(hostname, port):
+        """
+        Create a L{client connection creator
+        }
+        appropriate for the given URL "netloc"; i.e. hostname and port number
+        pair.
+
+        @param hostname: The name of the requested remote host.
+        @type hostname: L{bytes}
+
+        @param port: The number of the requested remote port.
+        @type port: L{int}
+
+        @return: A client connection creator expressing the security
+            requirements for the given remote host.
+        @rtype: L{client connection creator
+            }
+        """
+
+
+
+class IAgentEndpointFactory(Interface):
+    """
+    An L{IAgentEndpointFactory} provides a way of constructing an endpoint
+    used for outgoing Agent requests. This is useful in the case of needing to
+    proxy outgoing connections, or to otherwise vary the transport used.
+
+    @since: 15.0
+    """
+
+    def endpointForURI(uri):
+        """
+        Construct and return an L{IStreamClientEndpoint} for the outgoing
+        request's connection.
+
+        @param uri: The URI of the request.
+        @type uri: L{twisted.web.client.URI}
+
+        @return: An endpoint which will have its C{connect} method called to
+            issue the request.
+        @rtype: an L{IStreamClientEndpoint} provider
+
+        @raises twisted.internet.error.SchemeNotSupported: If the given
+            URI's scheme cannot be handled by this factory.
+        """
+
+
+
+UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
+
+__all__ = [
+    "IUsernameDigestHash", "ICredentialFactory", "IRequest",
+    "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",
+    "_IRequestEncoderFactory", "IClientRequest",
+
+    "UNKNOWN_LENGTH"]
diff --git a/contrib/python/Twisted/py2/twisted/web/microdom.py b/contrib/python/Twisted/py2/twisted/web/microdom.py
new file mode 100644
index 00000000000..1f522790119
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/microdom.py
@@ -0,0 +1,1145 @@
+# -*- test-case-name: twisted.web.test.test_xml -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Micro Document Object Model: a partial DOM implementation with SUX.
+
+This is an implementation of what we consider to be the useful subset of the
+DOM.  The chief advantage of this library is that, not being burdened with
+standards compliance, it can remain very stable between versions.  We can also
+implement utility 'pythonic' ways to access and mutate the XML tree.
+
+Since this has not subjected to a serious trial by fire, it is not recommended
+to use this outside of Twisted applications.  However, it seems to work just
+fine for the documentation generator, which parses a fairly representative
+sample of XML.
+
+Microdom mainly focuses on working with HTML and XHTML.
+"""
+
+# System Imports
+import re
+from io import BytesIO, StringIO
+
+
+# Twisted Imports
+from twisted.python.compat import ioType, iteritems, range, unicode
+from twisted.python.util import InsensitiveDict
+from twisted.web.sux import XMLParser, ParseError
+
+
+def getElementsByTagName(iNode, name):
+    """
+    Return a list of all child elements of C{iNode} with a name matching
+    C{name}.
+
+    Note that this implementation does not conform to the DOM Level 1 Core
+    specification because it may return C{iNode}.
+
+    @param iNode: An element at which to begin searching.  If C{iNode} has a
+        name matching C{name}, it will be included in the result.
+
+    @param name: A C{str} giving the name of the elements to return.
+
+    @return: A C{list} of direct or indirect child elements of C{iNode} with
+        the name C{name}.  This may include C{iNode}.
+    """
+    matches = []
+    matches_append = matches.append # faster lookup. don't do this at home
+    slice = [iNode]
+    while len(slice) > 0:
+        c = slice.pop(0)
+        if c.nodeName == name:
+            matches_append(c)
+        slice[:0] = c.childNodes
+    return matches
+
+
+
+def getElementsByTagNameNoCase(iNode, name):
+    name = name.lower()
+    matches = []
+    matches_append = matches.append
+    slice = [iNode]
+    while len(slice) > 0:
+        c = slice.pop(0)
+        if c.nodeName.lower() == name:
+            matches_append(c)
+        slice[:0] = c.childNodes
+    return matches
+
+
+
+def _streamWriteWrapper(stream):
+    if ioType(stream) == bytes:
+        def w(s):
+            if isinstance(s, unicode):
+                s = s.encode("utf-8")
+            stream.write(s)
+    else:
+        def w(s):
+            if isinstance(s, bytes):
+                s = s.decode("utf-8")
+            stream.write(s)
+    return w
+
+# order is important
+HTML_ESCAPE_CHARS = (('&', '&'), # don't add any entities before this one
+                    ('<', '<'),
+                    ('>', '>'),
+                    ('"', '"'))
+REV_HTML_ESCAPE_CHARS = list(HTML_ESCAPE_CHARS)
+REV_HTML_ESCAPE_CHARS.reverse()
+
+XML_ESCAPE_CHARS = HTML_ESCAPE_CHARS + (("'", '''),)
+REV_XML_ESCAPE_CHARS = list(XML_ESCAPE_CHARS)
+REV_XML_ESCAPE_CHARS.reverse()
+
+def unescape(text, chars=REV_HTML_ESCAPE_CHARS):
+    """
+    Perform the exact opposite of 'escape'.
+    """
+    for s, h in chars:
+        text = text.replace(h, s)
+    return text
+
+
+
+def escape(text, chars=HTML_ESCAPE_CHARS):
+    """
+    Escape a few XML special chars with XML entities.
+    """
+    for s, h in chars:
+        text = text.replace(s, h)
+    return text
+
+
+
+class MismatchedTags(Exception):
+
+    def __init__(self, filename, expect, got, endLine, endCol, begLine, begCol):
+        (self.filename, self.expect, self.got, self.begLine, self.begCol, self.endLine,
+         self.endCol) = filename, expect, got, begLine, begCol, endLine, endCol
+
+
+    def __str__(self):
+        return ("expected , got  line: %s col: %s, began line: %s col: %s"
+                % (self.expect, self.got, self.endLine, self.endCol, self.begLine,
+                   self.begCol))
+
+
+
+class Node(object):
+    nodeName = "Node"
+
+    def __init__(self, parentNode=None):
+        self.parentNode = parentNode
+        self.childNodes = []
+
+
+    def isEqualToNode(self, other):
+        """
+        Compare this node to C{other}.  If the nodes have the same number of
+        children and corresponding children are equal to each other, return
+        C{True}, otherwise return C{False}.
+
+        @type other: L{Node}
+        @rtype: C{bool}
+        """
+        if len(self.childNodes) != len(other.childNodes):
+            return False
+        for a, b in zip(self.childNodes, other.childNodes):
+            if not a.isEqualToNode(b):
+                return False
+        return True
+
+
+    def writexml(self, stream, indent='', addindent='', newl='', strip=0,
+                 nsprefixes={}, namespace=''):
+        raise NotImplementedError()
+
+
+    def toxml(self, indent='', addindent='', newl='', strip=0, nsprefixes={},
+              namespace=''):
+        s = StringIO()
+        self.writexml(s, indent, addindent, newl, strip, nsprefixes, namespace)
+        rv = s.getvalue()
+        return rv
+
+
+    def writeprettyxml(self, stream, indent='', addindent=' ', newl='\n', strip=0):
+        return self.writexml(stream, indent, addindent, newl, strip)
+
+
+    def toprettyxml(self, indent='', addindent=' ', newl='\n', strip=0):
+        return self.toxml(indent, addindent, newl, strip)
+
+
+    def cloneNode(self, deep=0, parent=None):
+        raise NotImplementedError()
+
+
+    def hasChildNodes(self):
+        if self.childNodes:
+            return 1
+        else:
+            return 0
+
+
+    def appendChild(self, child):
+        """
+        Make the given L{Node} the last child of this node.
+
+        @param child: The L{Node} which will become a child of this node.
+
+        @raise TypeError: If C{child} is not a C{Node} instance.
+        """
+        if not isinstance(child, Node):
+            raise TypeError("expected Node instance")
+        self.childNodes.append(child)
+        child.parentNode = self
+
+
+    def insertBefore(self, new, ref):
+        """
+        Make the given L{Node} C{new} a child of this node which comes before
+        the L{Node} C{ref}.
+
+        @param new: A L{Node} which will become a child of this node.
+
+        @param ref: A L{Node} which is already a child of this node which
+            C{new} will be inserted before.
+
+        @raise TypeError: If C{new} or C{ref} is not a C{Node} instance.
+
+        @return: C{new}
+        """
+        if not isinstance(new, Node) or not isinstance(ref, Node):
+            raise TypeError("expected Node instance")
+        i = self.childNodes.index(ref)
+        new.parentNode = self
+        self.childNodes.insert(i, new)
+        return new
+
+
+    def removeChild(self, child):
+        """
+        Remove the given L{Node} from this node's children.
+
+        @param child: A L{Node} which is a child of this node which will no
+            longer be a child of this node after this method is called.
+
+        @raise TypeError: If C{child} is not a C{Node} instance.
+
+        @return: C{child}
+        """
+        if not isinstance(child, Node):
+            raise TypeError("expected Node instance")
+        if child in self.childNodes:
+            self.childNodes.remove(child)
+            child.parentNode = None
+        return child
+
+
+    def replaceChild(self, newChild, oldChild):
+        """
+        Replace a L{Node} which is already a child of this node with a
+        different node.
+
+        @param newChild: A L{Node} which will be made a child of this node.
+
+        @param oldChild: A L{Node} which is a child of this node which will
+            give up its position to C{newChild}.
+
+        @raise TypeError: If C{newChild} or C{oldChild} is not a C{Node}
+            instance.
+
+        @raise ValueError: If C{oldChild} is not a child of this C{Node}.
+        """
+        if not isinstance(newChild, Node) or not isinstance(oldChild, Node):
+            raise TypeError("expected Node instance")
+        if oldChild.parentNode is not self:
+            raise ValueError("oldChild is not a child of this node")
+        self.childNodes[self.childNodes.index(oldChild)] = newChild
+        oldChild.parentNode = None
+        newChild.parentNode = self
+
+
+    def lastChild(self):
+        return self.childNodes[-1]
+
+
+    def firstChild(self):
+        if len(self.childNodes):
+            return self.childNodes[0]
+        return None
+
+    #def get_ownerDocument(self):
+    #   """This doesn't really get the owner document; microdom nodes
+    #   don't even have one necessarily.  This gets the root node,
+    #   which is usually what you really meant.
+    #   *NOT DOM COMPLIANT.*
+    #   """
+    #   node=self
+    #   while (node.parentNode): node=node.parentNode
+    #   return node
+    #ownerDocument=node.get_ownerDocument()
+    # leaving commented for discussion; see also domhelpers.getParents(node)
+
+
+
+class Document(Node):
+
+    def __init__(self, documentElement=None):
+        Node.__init__(self)
+        if documentElement:
+            self.appendChild(documentElement)
+
+
+    def cloneNode(self, deep=0, parent=None):
+        d = Document()
+        d.doctype = self.doctype
+        if deep:
+            newEl = self.documentElement.cloneNode(1, self)
+        else:
+            newEl = self.documentElement
+        d.appendChild(newEl)
+        return d
+
+    doctype = None
+
+    def isEqualToDocument(self, n):
+        return (self.doctype == n.doctype) and Node.isEqualToNode(self, n)
+    isEqualToNode = isEqualToDocument
+
+
+    def get_documentElement(self):
+        return self.childNodes[0]
+    documentElement = property(get_documentElement)
+
+
+    def appendChild(self, child):
+        """
+        Make the given L{Node} the I{document element} of this L{Document}.
+
+        @param child: The L{Node} to make into this L{Document}'s document
+            element.
+
+        @raise ValueError: If this document already has a document element.
+        """
+        if self.childNodes:
+            raise ValueError("Only one element per document.")
+        Node.appendChild(self, child)
+
+
+    def writexml(self, stream, indent='', addindent='', newl='', strip=0,
+                 nsprefixes={}, namespace=''):
+        w = _streamWriteWrapper(stream)
+
+        w('' + newl)
+        if self.doctype:
+            w(u"{}".format(self.doctype, newl))
+        self.documentElement.writexml(stream, indent, addindent, newl, strip,
+                                      nsprefixes, namespace)
+
+
+    # of dubious utility (?)
+    def createElement(self, name, **kw):
+        return Element(name, **kw)
+
+
+    def createTextNode(self, text):
+        return Text(text)
+
+
+    def createComment(self, text):
+        return Comment(text)
+
+
+    def getElementsByTagName(self, name):
+        if self.documentElement.caseInsensitive:
+            return getElementsByTagNameNoCase(self, name)
+        return getElementsByTagName(self, name)
+
+
+    def getElementById(self, id):
+        childNodes = self.childNodes[:]
+        while childNodes:
+            node = childNodes.pop(0)
+            if node.childNodes:
+                childNodes.extend(node.childNodes)
+            if hasattr(node, 'getAttribute') and node.getAttribute("id") == id:
+                return node
+
+
+
+class EntityReference(Node):
+
+    def __init__(self, eref, parentNode=None):
+        Node.__init__(self, parentNode)
+        self.eref = eref
+        self.nodeValue = self.data = "&" + eref + ";"
+
+
+    def isEqualToEntityReference(self, n):
+        if not isinstance(n, EntityReference):
+            return 0
+        return (self.eref == n.eref) and (self.nodeValue == n.nodeValue)
+    isEqualToNode = isEqualToEntityReference
+
+
+    def writexml(self, stream, indent='', addindent='', newl='', strip=0,
+                 nsprefixes={}, namespace=''):
+        w = _streamWriteWrapper(stream)
+        w("" + self.nodeValue)
+
+
+    def cloneNode(self, deep=0, parent=None):
+        return EntityReference(self.eref, parent)
+
+
+
+class CharacterData(Node):
+
+    def __init__(self, data, parentNode=None):
+        Node.__init__(self, parentNode)
+        self.value = self.data = self.nodeValue = data
+
+
+    def isEqualToCharacterData(self, n):
+        return self.value == n.value
+    isEqualToNode = isEqualToCharacterData
+
+
+
+class Comment(CharacterData):
+    """
+    A comment node.
+    """
+
+    def writexml(self, stream, indent='', addindent='', newl='', strip=0,
+                 nsprefixes={}, namespace=''):
+        w = _streamWriteWrapper(stream)
+        val = self.data
+        w(u"".format(val))
+
+
+    def cloneNode(self, deep=0, parent=None):
+        return Comment(self.nodeValue, parent)
+
+
+
+class Text(CharacterData):
+
+    def __init__(self, data, parentNode=None, raw=0):
+        CharacterData.__init__(self, data, parentNode)
+        self.raw = raw
+
+
+    def isEqualToNode(self, other):
+        """
+        Compare this text to C{text}.  If the underlying values and the C{raw}
+        flag are the same, return C{True}, otherwise return C{False}.
+        """
+        return (
+            CharacterData.isEqualToNode(self, other) and
+            self.raw == other.raw)
+
+
+    def cloneNode(self, deep=0, parent=None):
+        return Text(self.nodeValue, parent, self.raw)
+
+
+    def writexml(self, stream, indent='', addindent='', newl='', strip=0,
+                 nsprefixes={}, namespace=''):
+        w = _streamWriteWrapper(stream)
+        if self.raw:
+            val = self.nodeValue
+            if not isinstance(val, (str, unicode)):
+                val = str(self.nodeValue)
+        else:
+            v = self.nodeValue
+            if not isinstance(v, (str, unicode)):
+                v = str(v)
+            if strip:
+                v = ' '.join(v.split())
+            val = escape(v)
+        w(val)
+
+
+    def __repr__(self):
+        return "Text(%s" % repr(self.nodeValue) + ')'
+
+
+
+class CDATASection(CharacterData):
+    def cloneNode(self, deep=0, parent=None):
+        return CDATASection(self.nodeValue, parent)
+
+
+    def writexml(self, stream, indent='', addindent='', newl='', strip=0,
+                 nsprefixes={}, namespace=''):
+        w = _streamWriteWrapper(stream)
+        w("")
+
+
+
+def _genprefix():
+    i = 0
+    while True:
+        yield  'p' + str(i)
+        i = i + 1
+genprefix = _genprefix()
+
+
+
+class _Attr(CharacterData):
+    "Support class for getAttributeNode."
+
+
+
+class Element(Node):
+
+    preserveCase = 0
+    caseInsensitive = 1
+    nsprefixes = None
+
+    def __init__(self, tagName, attributes=None, parentNode=None,
+                 filename=None, markpos=None,
+                 caseInsensitive=1, preserveCase=0,
+                 namespace=None):
+        Node.__init__(self, parentNode)
+        self.preserveCase = preserveCase or not caseInsensitive
+        self.caseInsensitive = caseInsensitive
+        if not preserveCase:
+            tagName = tagName.lower()
+        if attributes is None:
+            self.attributes = {}
+        else:
+            self.attributes = attributes
+            for k, v in self.attributes.items():
+                self.attributes[k] = unescape(v)
+
+        if caseInsensitive:
+            self.attributes = InsensitiveDict(self.attributes,
+                                              preserve=preserveCase)
+
+        self.endTagName = self.nodeName = self.tagName = tagName
+        self._filename = filename
+        self._markpos = markpos
+        self.namespace = namespace
+
+
+    def addPrefixes(self, pfxs):
+        if self.nsprefixes is None:
+            self.nsprefixes = pfxs
+        else:
+            self.nsprefixes.update(pfxs)
+
+
+    def endTag(self, endTagName):
+        if not self.preserveCase:
+            endTagName = endTagName.lower()
+        self.endTagName = endTagName
+
+
+    def isEqualToElement(self, n):
+        if self.caseInsensitive:
+            return ((self.attributes == n.attributes)
+                    and (self.nodeName.lower() == n.nodeName.lower()))
+        return (self.attributes == n.attributes) and (self.nodeName == n.nodeName)
+
+
+    def isEqualToNode(self, other):
+        """
+        Compare this element to C{other}.  If the C{nodeName}, C{namespace},
+        C{attributes}, and C{childNodes} are all the same, return C{True},
+        otherwise return C{False}.
+        """
+        return (
+            self.nodeName.lower() == other.nodeName.lower() and
+            self.namespace == other.namespace and
+            self.attributes == other.attributes and
+            Node.isEqualToNode(self, other))
+
+
+    def cloneNode(self, deep=0, parent=None):
+        clone = Element(
+            self.tagName, parentNode=parent, namespace=self.namespace,
+            preserveCase=self.preserveCase, caseInsensitive=self.caseInsensitive)
+        clone.attributes.update(self.attributes)
+        if deep:
+            clone.childNodes = [child.cloneNode(1, clone) for child in self.childNodes]
+        else:
+            clone.childNodes = []
+        return clone
+
+
+    def getElementsByTagName(self, name):
+        if self.caseInsensitive:
+            return getElementsByTagNameNoCase(self, name)
+        return getElementsByTagName(self, name)
+
+
+    def hasAttributes(self):
+        return 1
+
+
+    def getAttribute(self, name, default=None):
+        return self.attributes.get(name, default)
+
+
+    def getAttributeNS(self, ns, name, default=None):
+        nsk = (ns, name)
+        if nsk in self.attributes:
+            return self.attributes[nsk]
+        if ns == self.namespace:
+            return self.attributes.get(name, default)
+        return default
+
+
+    def getAttributeNode(self, name):
+        return _Attr(self.getAttribute(name), self)
+
+
+    def setAttribute(self, name, attr):
+        self.attributes[name] = attr
+
+
+    def removeAttribute(self, name):
+        if name in self.attributes:
+            del self.attributes[name]
+
+
+    def hasAttribute(self, name):
+        return name in self.attributes
+
+
+    def writexml(self, stream, indent='', addindent='', newl='', strip=0,
+                 nsprefixes={}, namespace=''):
+        """
+        Serialize this L{Element} to the given stream.
+
+        @param stream: A file-like object to which this L{Element} will be
+            written.
+
+        @param nsprefixes: A C{dict} mapping namespace URIs as C{str} to
+            prefixes as C{str}.  This defines the prefixes which are already in
+            scope in the document at the point at which this L{Element} exists.
+            This is essentially an implementation detail for namespace support.
+            Applications should not try to use it.
+
+        @param namespace: The namespace URI as a C{str} which is the default at
+            the point in the document at which this L{Element} exists.  This is
+            essentially an implementation detail for namespace support.
+            Applications should not try to use it.
+        """
+        # write beginning
+        ALLOWSINGLETON = ('img', 'br', 'hr', 'base', 'meta', 'link', 'param',
+                          'area', 'input', 'col', 'basefont', 'isindex',
+                          'frame')
+        BLOCKELEMENTS = ('html', 'head', 'body', 'noscript', 'ins', 'del',
+                         'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'script',
+                         'ul', 'ol', 'dl', 'pre', 'hr', 'blockquote',
+                         'address', 'p', 'div', 'fieldset', 'table', 'tr',
+                         'form', 'object', 'fieldset', 'applet', 'map')
+        FORMATNICELY = ('tr', 'ul', 'ol', 'head')
+
+        # this should never be necessary unless people start
+        # changing .tagName on the fly(?)
+        if not self.preserveCase:
+            self.endTagName = self.tagName
+
+        w = _streamWriteWrapper(stream)
+        if self.nsprefixes:
+            newprefixes = self.nsprefixes.copy()
+            for ns in nsprefixes.keys():
+                if ns in newprefixes:
+                    del newprefixes[ns]
+        else:
+            newprefixes = {}
+
+        begin = ['<']
+        if self.tagName in BLOCKELEMENTS:
+            begin = [newl, indent] + begin
+        bext = begin.extend
+        writeattr = lambda _atr, _val: bext((' ', _atr, '="', escape(_val), '"'))
+
+        # Make a local for tracking what end tag will be used.  If namespace
+        # prefixes are involved, this will be changed to account for that
+        # before it's actually used.
+        endTagName = self.endTagName
+
+        if namespace != self.namespace and self.namespace is not None:
+            # If the current default namespace is not the namespace of this tag
+            # (and this tag has a namespace at all) then we'll write out
+            # something related to namespaces.
+            if self.namespace in nsprefixes:
+                # This tag's namespace already has a prefix bound to it.  Use
+                # that prefix.
+                prefix = nsprefixes[self.namespace]
+                bext(prefix + ':' + self.tagName)
+                # Also make sure we use it for the end tag.
+                endTagName = prefix + ':' + self.endTagName
+            else:
+                # This tag's namespace has no prefix bound to it.  Change the
+                # default namespace to this tag's namespace so we don't need
+                # prefixes.  Alternatively, we could add a new prefix binding.
+                # I'm not sure why the code was written one way rather than the
+                # other. -exarkun
+                bext(self.tagName)
+                writeattr("xmlns", self.namespace)
+                # The default namespace just changed.  Make sure any children
+                # know about this.
+                namespace = self.namespace
+        else:
+            # This tag has no namespace or its namespace is already the default
+            # namespace.  Nothing extra to do here.
+            bext(self.tagName)
+
+        j = ''.join
+        for attr, val in sorted(self.attributes.items()):
+            if isinstance(attr, tuple):
+                ns, key = attr
+                if ns in nsprefixes:
+                    prefix = nsprefixes[ns]
+                else:
+                    prefix = next(genprefix)
+                    newprefixes[ns] = prefix
+                assert val is not None
+                writeattr(prefix + ':' + key, val)
+            else:
+                assert val is not None
+                writeattr(attr, val)
+        if newprefixes:
+            for ns, prefix in iteritems(newprefixes):
+                if prefix:
+                    writeattr('xmlns:'+prefix, ns)
+            newprefixes.update(nsprefixes)
+            downprefixes = newprefixes
+        else:
+            downprefixes = nsprefixes
+        w(j(begin))
+        if self.childNodes:
+            w(">")
+            newindent = indent + addindent
+            for child in self.childNodes:
+                if self.tagName in BLOCKELEMENTS and \
+                   self.tagName in FORMATNICELY:
+                    w(j((newl, newindent)))
+                child.writexml(stream, newindent, addindent, newl, strip,
+                               downprefixes, namespace)
+            if self.tagName in BLOCKELEMENTS:
+                w(j((newl, indent)))
+            w(j(('')))
+        elif self.tagName.lower() not in ALLOWSINGLETON:
+            w(j(('>')))
+        else:
+            w(" />")
+
+
+    def __repr__(self):
+        rep = "Element(%s" % repr(self.nodeName)
+        if self.attributes:
+            rep += ", attributes=%r" % (self.attributes,)
+        if self._filename:
+            rep += ", filename=%r" % (self._filename,)
+        if self._markpos:
+            rep += ", markpos=%r" % (self._markpos,)
+        return rep + ')'
+
+
+    def __str__(self):
+        rep = "<" + self.nodeName
+        if self._filename or self._markpos:
+            rep += " ("
+        if self._filename:
+            rep += repr(self._filename)
+        if self._markpos:
+            rep += " line %s column %s" % self._markpos
+        if self._filename or self._markpos:
+            rep += ")"
+        for item in self.attributes.items():
+            rep += " %s=%r" % item
+        if self.hasChildNodes():
+            rep += " >..." % self.nodeName
+        else:
+            rep += " />"
+        return rep
+
+
+
+def _unescapeDict(d):
+    dd = {}
+    for k, v in d.items():
+        dd[k] = unescape(v)
+    return dd
+
+
+
+def _reverseDict(d):
+    dd = {}
+    for k, v in d.items():
+        dd[v] = k
+    return dd
+
+
+
+class MicroDOMParser(XMLParser):
+
+    #  glyph: a quick scan thru the DTD says BODY, AREA, LINK, IMG, HR,
+    # P, DT, DD, LI, INPUT, OPTION, THEAD, TFOOT, TBODY, COLGROUP, COL, TR, TH,
+    # TD, HEAD, BASE, META, HTML all have optional closing tags
+
+    soonClosers = 'area link br img hr input base meta'.split()
+    laterClosers = {'p': ['p', 'dt'],
+                    'dt': ['dt', 'dd'],
+                    'dd': ['dt', 'dd'],
+                    'li': ['li'],
+                    'tbody': ['thead', 'tfoot', 'tbody'],
+                    'thead': ['thead', 'tfoot', 'tbody'],
+                    'tfoot': ['thead', 'tfoot', 'tbody'],
+                    'colgroup': ['colgroup'],
+                    'col': ['col'],
+                    'tr': ['tr'],
+                    'td': ['td'],
+                    'th': ['th'],
+                    'head': ['body'],
+                    'title': ['head', 'body'], # this looks wrong...
+                    'option': ['option'],
+                    }
+
+
+    def __init__(self, beExtremelyLenient=0, caseInsensitive=1, preserveCase=0,
+                 soonClosers=soonClosers, laterClosers=laterClosers):
+        self.elementstack = []
+        d = {'xmlns': 'xmlns', '': None}
+        dr = _reverseDict(d)
+        self.nsstack = [(d, None, dr)]
+        self.documents = []
+        self._mddoctype = None
+        self.beExtremelyLenient = beExtremelyLenient
+        self.caseInsensitive = caseInsensitive
+        self.preserveCase = preserveCase or not caseInsensitive
+        self.soonClosers = soonClosers
+        self.laterClosers = laterClosers
+        # self.indentlevel = 0
+
+
+    def shouldPreserveSpace(self):
+        for edx in range(len(self.elementstack)):
+            el = self.elementstack[-edx]
+            if el.tagName == 'pre' or el.getAttribute("xml:space", '') == 'preserve':
+                return 1
+        return 0
+
+
+    def _getparent(self):
+        if self.elementstack:
+            return self.elementstack[-1]
+        else:
+            return None
+
+    COMMENT = re.compile(r"\s*/[/*]\s*")
+
+    def _fixScriptElement(self, el):
+        # this deals with case where there is comment or CDATA inside
+        # 
+            # tidy does this, for example.
+            prefix = ""
+            oldvalue = c.value
+            match = self.COMMENT.match(oldvalue)
+            if match:
+                prefix = match.group()
+                oldvalue = oldvalue[len(prefix):]
+
+            # now see if contents are actual node and comment or CDATA
+            try:
+                e = parseString("%s" % oldvalue).childNodes[0]
+            except (ParseError, MismatchedTags):
+                return
+            if len(e.childNodes) != 1:
+                return
+            e = e.firstChild()
+            if isinstance(e, (CDATASection, Comment)):
+                el.childNodes = []
+                if prefix:
+                    el.childNodes.append(Text(prefix))
+                el.childNodes.append(e)
+
+
+    def gotDoctype(self, doctype):
+        self._mddoctype = doctype
+
+
+    def gotTagStart(self, name, attributes):
+        # print ' '*self.indentlevel, 'start tag',name
+        # self.indentlevel += 1
+        parent = self._getparent()
+        if (self.beExtremelyLenient and isinstance(parent, Element)):
+            parentName = parent.tagName
+            myName = name
+            if self.caseInsensitive:
+                parentName = parentName.lower()
+                myName = myName.lower()
+            if myName in self.laterClosers.get(parentName, []):
+                self.gotTagEnd(parent.tagName)
+                parent = self._getparent()
+        attributes = _unescapeDict(attributes)
+        namespaces = self.nsstack[-1][0]
+        newspaces = {}
+        keysToDelete = []
+        for k, v in attributes.items():
+            if k.startswith('xmlns'):
+                spacenames = k.split(':', 1)
+                if len(spacenames) == 2:
+                    newspaces[spacenames[1]] = v
+                else:
+                    newspaces[''] = v
+                keysToDelete.append(k)
+        for k in keysToDelete:
+            del attributes[k]
+        if newspaces:
+            namespaces = namespaces.copy()
+            namespaces.update(newspaces)
+        keysToDelete = []
+        for k, v in attributes.items():
+            ksplit = k.split(':', 1)
+            if len(ksplit) == 2:
+                pfx, tv = ksplit
+                if pfx != 'xml' and pfx in namespaces:
+                    attributes[namespaces[pfx], tv] = v
+                    keysToDelete.append(k)
+        for k in keysToDelete:
+            del attributes[k]
+        el = Element(name, attributes, parent,
+                     self.filename, self.saveMark(),
+                     caseInsensitive=self.caseInsensitive,
+                     preserveCase=self.preserveCase,
+                     namespace=namespaces.get(''))
+        revspaces = _reverseDict(newspaces)
+        el.addPrefixes(revspaces)
+
+        if newspaces:
+            rscopy = self.nsstack[-1][2].copy()
+            rscopy.update(revspaces)
+            self.nsstack.append((namespaces, el, rscopy))
+        self.elementstack.append(el)
+        if parent:
+            parent.appendChild(el)
+        if (self.beExtremelyLenient and el.tagName in self.soonClosers):
+            self.gotTagEnd(name)
+
+
+    def _gotStandalone(self, factory, data):
+        parent = self._getparent()
+        te = factory(data, parent)
+        if parent:
+            parent.appendChild(te)
+        elif self.beExtremelyLenient:
+            self.documents.append(te)
+
+
+    def gotText(self, data):
+        if data.strip() or self.shouldPreserveSpace():
+            self._gotStandalone(Text, data)
+
+
+    def gotComment(self, data):
+        self._gotStandalone(Comment, data)
+
+
+    def gotEntityReference(self, entityRef):
+        self._gotStandalone(EntityReference, entityRef)
+
+
+    def gotCData(self, cdata):
+        self._gotStandalone(CDATASection, cdata)
+
+
+    def gotTagEnd(self, name):
+        # print ' '*self.indentlevel, 'end tag',name
+        # self.indentlevel -= 1
+        if not self.elementstack:
+            if self.beExtremelyLenient:
+                return
+            raise MismatchedTags(*((self.filename, "NOTHING", name)
+                                   + self.saveMark() + (0, 0)))
+        el = self.elementstack.pop()
+        pfxdix = self.nsstack[-1][2]
+        if self.nsstack[-1][1] is el:
+            nstuple = self.nsstack.pop()
+        else:
+            nstuple = None
+        if self.caseInsensitive:
+            tn = el.tagName.lower()
+            cname = name.lower()
+        else:
+            tn = el.tagName
+            cname = name
+
+        nsplit = name.split(':', 1)
+        if len(nsplit) == 2:
+            pfx, newname = nsplit
+            ns = pfxdix.get(pfx, None)
+            if ns is not None:
+                if el.namespace != ns:
+                    if not self.beExtremelyLenient:
+                        raise MismatchedTags(*((self.filename, el.tagName, name)
+                                               + self.saveMark() + el._markpos))
+        if not (tn == cname):
+            if self.beExtremelyLenient:
+                if self.elementstack:
+                    lastEl = self.elementstack[0]
+                    for idx in range(len(self.elementstack)):
+                        if self.elementstack[-(idx+1)].tagName == cname:
+                            self.elementstack[-(idx+1)].endTag(name)
+                            break
+                    else:
+                        # this was a garbage close tag; wait for a real one
+                        self.elementstack.append(el)
+                        if nstuple is not None:
+                            self.nsstack.append(nstuple)
+                        return
+                    del self.elementstack[-(idx+1):]
+                    if not self.elementstack:
+                        self.documents.append(lastEl)
+                        return
+            else:
+                raise MismatchedTags(*((self.filename, el.tagName, name)
+                                       + self.saveMark() + el._markpos))
+        el.endTag(name)
+        if not self.elementstack:
+            self.documents.append(el)
+        if self.beExtremelyLenient and el.tagName == "script":
+            self._fixScriptElement(el)
+
+
+    def connectionLost(self, reason):
+        XMLParser.connectionLost(self, reason) # This can cause more events!
+        if self.elementstack:
+            if self.beExtremelyLenient:
+                self.documents.append(self.elementstack[0])
+            else:
+                raise MismatchedTags(*((self.filename, self.elementstack[-1],
+                                        "END_OF_FILE")
+                                       + self.saveMark()
+                                       + self.elementstack[-1]._markpos))
+
+
+
+def parse(readable, *args, **kwargs):
+    """
+    Parse HTML or XML readable.
+    """
+    if not hasattr(readable, "read"):
+        readable = open(readable, "rb")
+    mdp = MicroDOMParser(*args, **kwargs)
+    mdp.filename = getattr(readable, "name", "")
+    mdp.makeConnection(None)
+    if hasattr(readable, "getvalue"):
+        mdp.dataReceived(readable.getvalue())
+    else:
+        r = readable.read(1024)
+        while r:
+            mdp.dataReceived(r)
+            r = readable.read(1024)
+    mdp.connectionLost(None)
+
+    if not mdp.documents:
+        raise ParseError(mdp.filename, 0, 0, "No top-level Nodes in document")
+
+    if mdp.beExtremelyLenient:
+        if len(mdp.documents) == 1:
+            d = mdp.documents[0]
+            if not isinstance(d, Element):
+                el = Element("html")
+                el.appendChild(d)
+                d = el
+        else:
+            d = Element("html")
+            for child in mdp.documents:
+                d.appendChild(child)
+    else:
+        d = mdp.documents[0]
+    doc = Document(d)
+    doc.doctype = mdp._mddoctype
+    return doc
+
+
+
+def parseString(st, *args, **kw):
+    if isinstance(st, unicode):
+        # this isn't particularly ideal, but it does work.
+        return parse(BytesIO(st.encode('UTF-16')), *args, **kw)
+    return parse(BytesIO(st), *args, **kw)
+
+
+
+def parseXML(readable):
+    """
+    Parse an XML readable object.
+    """
+    return parse(readable, caseInsensitive=0, preserveCase=1)
+
+
+
+def parseXMLString(st):
+    """
+    Parse an XML readable object.
+    """
+    return parseString(st, caseInsensitive=0, preserveCase=1)
+
+
+
+class lmx:
+    """
+    Easy creation of XML.
+    """
+
+    def __init__(self, node='div'):
+        if isinstance(node, (str, unicode)):
+            node = Element(node)
+        self.node = node
+
+
+    def __getattr__(self, name):
+        if name[0] == '_':
+            raise AttributeError("no private attrs")
+        return lambda **kw: self.add(name, **kw)
+
+
+    def __setitem__(self, key, val):
+        self.node.setAttribute(key, val)
+
+
+    def __getitem__(self, key):
+        return self.node.getAttribute(key)
+
+
+    def text(self, txt, raw=0):
+        nn = Text(txt, raw=raw)
+        self.node.appendChild(nn)
+        return self
+
+
+    def add(self, tagName, **kw):
+        newNode = Element(tagName, caseInsensitive=0, preserveCase=0)
+        self.node.appendChild(newNode)
+        xf = lmx(newNode)
+        for k, v in kw.items():
+            if k[0] == '_':
+                k = k[1:]
+            xf[k] = v
+        return xf
diff --git a/contrib/python/Twisted/py2/twisted/web/proxy.py b/contrib/python/Twisted/py2/twisted/web/proxy.py
new file mode 100644
index 00000000000..4ec677cc034
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/proxy.py
@@ -0,0 +1,303 @@
+# -*- test-case-name: twisted.web.test.test_proxy -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Simplistic HTTP proxy support.
+
+This comes in two main variants - the Proxy and the ReverseProxy.
+
+When a Proxy is in use, a browser trying to connect to a server (say,
+www.yahoo.com) will be intercepted by the Proxy, and the proxy will covertly
+connect to the server, and return the result.
+
+When a ReverseProxy is in use, the client connects directly to the ReverseProxy
+(say, www.yahoo.com) which farms off the request to one of a pool of servers,
+and returns the result.
+
+Normally, a Proxy is used on the client end of an Internet connection, while a
+ReverseProxy is used on the server end.
+"""
+from __future__ import absolute_import, division
+
+from twisted.python.compat import urllib_parse, urlquote
+from twisted.internet import reactor
+from twisted.internet.protocol import ClientFactory
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
+from twisted.web.http import HTTPClient, Request, HTTPChannel, _QUEUED_SENTINEL
+
+
+
+class ProxyClient(HTTPClient):
+    """
+    Used by ProxyClientFactory to implement a simple web proxy.
+
+    @ivar _finished: A flag which indicates whether or not the original request
+        has been finished yet.
+    """
+    _finished = False
+
+    def __init__(self, command, rest, version, headers, data, father):
+        self.father = father
+        self.command = command
+        self.rest = rest
+        if b"proxy-connection" in headers:
+            del headers[b"proxy-connection"]
+        headers[b"connection"] = b"close"
+        headers.pop(b'keep-alive', None)
+        self.headers = headers
+        self.data = data
+
+
+    def connectionMade(self):
+        self.sendCommand(self.command, self.rest)
+        for header, value in self.headers.items():
+            self.sendHeader(header, value)
+        self.endHeaders()
+        self.transport.write(self.data)
+
+
+    def handleStatus(self, version, code, message):
+        self.father.setResponseCode(int(code), message)
+
+
+    def handleHeader(self, key, value):
+        # t.web.server.Request sets default values for these headers in its
+        # 'process' method. When these headers are received from the remote
+        # server, they ought to override the defaults, rather than append to
+        # them.
+        if key.lower() in [b'server', b'date', b'content-type']:
+            self.father.responseHeaders.setRawHeaders(key, [value])
+        else:
+            self.father.responseHeaders.addRawHeader(key, value)
+
+
+    def handleResponsePart(self, buffer):
+        self.father.write(buffer)
+
+
+    def handleResponseEnd(self):
+        """
+        Finish the original request, indicating that the response has been
+        completely written to it, and disconnect the outgoing transport.
+        """
+        if not self._finished:
+            self._finished = True
+            self.father.finish()
+            self.transport.loseConnection()
+
+
+
+class ProxyClientFactory(ClientFactory):
+    """
+    Used by ProxyRequest to implement a simple web proxy.
+    """
+
+    protocol = ProxyClient
+
+
+    def __init__(self, command, rest, version, headers, data, father):
+        self.father = father
+        self.command = command
+        self.rest = rest
+        self.headers = headers
+        self.data = data
+        self.version = version
+
+
+    def buildProtocol(self, addr):
+        return self.protocol(self.command, self.rest, self.version,
+                             self.headers, self.data, self.father)
+
+
+    def clientConnectionFailed(self, connector, reason):
+        """
+        Report a connection failure in a response to the incoming request as
+        an error.
+        """
+        self.father.setResponseCode(501, b"Gateway error")
+        self.father.responseHeaders.addRawHeader(b"Content-Type", b"text/html")
+        self.father.write(b"
Could not connect
")
+        self.father.finish()
+
+
+
+class ProxyRequest(Request):
+    """
+    Used by Proxy to implement a simple web proxy.
+
+    @ivar reactor: the reactor used to create connections.
+    @type reactor: object providing L{twisted.internet.interfaces.IReactorTCP}
+    """
+
+    protocols = {b'http': ProxyClientFactory}
+    ports = {b'http': 80}
+
+    def __init__(self, channel, queued=_QUEUED_SENTINEL, reactor=reactor):
+        Request.__init__(self, channel, queued)
+        self.reactor = reactor
+
+
+    def process(self):
+        parsed = urllib_parse.urlparse(self.uri)
+        protocol = parsed[0]
+        host = parsed[1].decode('ascii')
+        port = self.ports[protocol]
+        if ':' in host:
+            host, port = host.split(':')
+            port = int(port)
+        rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
+        if not rest:
+            rest = rest + b'/'
+        class_ = self.protocols[protocol]
+        headers = self.getAllHeaders().copy()
+        if b'host' not in headers:
+            headers[b'host'] = host.encode('ascii')
+        self.content.seek(0, 0)
+        s = self.content.read()
+        clientFactory = class_(self.method, rest, self.clientproto, headers,
+                               s, self)
+        self.reactor.connectTCP(host, port, clientFactory)
+
+
+
+class Proxy(HTTPChannel):
+    """
+    This class implements a simple web proxy.
+
+    Since it inherits from L{twisted.web.http.HTTPChannel}, to use it you
+    should do something like this::
+
+        from twisted.web import http
+        f = http.HTTPFactory()
+        f.protocol = Proxy
+
+    Make the HTTPFactory a listener on a port as per usual, and you have
+    a fully-functioning web proxy!
+    """
+
+    requestFactory = ProxyRequest
+
+
+
+class ReverseProxyRequest(Request):
+    """
+    Used by ReverseProxy to implement a simple reverse proxy.
+
+    @ivar proxyClientFactoryClass: a proxy client factory class, used to create
+        new connections.
+    @type proxyClientFactoryClass: L{ClientFactory}
+
+    @ivar reactor: the reactor used to create connections.
+    @type reactor: object providing L{twisted.internet.interfaces.IReactorTCP}
+    """
+
+    proxyClientFactoryClass = ProxyClientFactory
+
+    def __init__(self, channel, queued=_QUEUED_SENTINEL, reactor=reactor):
+        Request.__init__(self, channel, queued)
+        self.reactor = reactor
+
+
+    def process(self):
+        """
+        Handle this request by connecting to the proxied server and forwarding
+        it there, then forwarding the response back as the response to this
+        request.
+        """
+        self.requestHeaders.setRawHeaders(b"host",
+                                          [self.factory.host.encode('ascii')])
+        clientFactory = self.proxyClientFactoryClass(
+            self.method, self.uri, self.clientproto, self.getAllHeaders(),
+            self.content.read(), self)
+        self.reactor.connectTCP(self.factory.host, self.factory.port,
+                                clientFactory)
+
+
+
+class ReverseProxy(HTTPChannel):
+    """
+    Implements a simple reverse proxy.
+
+    For details of usage, see the file examples/reverse-proxy.py.
+    """
+
+    requestFactory = ReverseProxyRequest
+
+
+
+class ReverseProxyResource(Resource):
+    """
+    Resource that renders the results gotten from another server
+
+    Put this resource in the tree to cause everything below it to be relayed
+    to a different server.
+
+    @ivar proxyClientFactoryClass: a proxy client factory class, used to create
+        new connections.
+    @type proxyClientFactoryClass: L{ClientFactory}
+
+    @ivar reactor: the reactor used to create connections.
+    @type reactor: object providing L{twisted.internet.interfaces.IReactorTCP}
+    """
+
+    proxyClientFactoryClass = ProxyClientFactory
+
+
+    def __init__(self, host, port, path, reactor=reactor):
+        """
+        @param host: the host of the web server to proxy.
+        @type host: C{str}
+
+        @param port: the port of the web server to proxy.
+        @type port: C{port}
+
+        @param path: the base path to fetch data from. Note that you shouldn't
+            put any trailing slashes in it, it will be added automatically in
+            request. For example, if you put B{/foo}, a request on B{/bar} will
+            be proxied to B{/foo/bar}.  Any required encoding of special
+            characters (such as " " or "/") should have been done already.
+
+        @type path: C{bytes}
+        """
+        Resource.__init__(self)
+        self.host = host
+        self.port = port
+        self.path = path
+        self.reactor = reactor
+
+
+    def getChild(self, path, request):
+        """
+        Create and return a proxy resource with the same proxy configuration
+        as this one, except that its path also contains the segment given by
+        C{path} at the end.
+        """
+        return ReverseProxyResource(
+            self.host, self.port, self.path + b'/' + urlquote(path, safe=b"").encode('utf-8'),
+            self.reactor)
+
+
+    def render(self, request):
+        """
+        Render a request by forwarding it to the proxied server.
+        """
+        # RFC 2616 tells us that we can omit the port if it's the default port,
+        # but we have to provide it otherwise
+        if self.port == 80:
+            host = self.host
+        else:
+            host = u"%s:%d" % (self.host, self.port)
+        request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')])
+        request.content.seek(0, 0)
+        qs = urllib_parse.urlparse(request.uri)[4]
+        if qs:
+            rest = self.path + b'?' + qs
+        else:
+            rest = self.path
+        clientFactory = self.proxyClientFactoryClass(
+            request.method, rest, request.clientproto,
+            request.getAllHeaders(), request.content.read(), request)
+        self.reactor.connectTCP(self.host, self.port, clientFactory)
+        return NOT_DONE_YET
diff --git a/contrib/python/Twisted/py2/twisted/web/resource.py b/contrib/python/Twisted/py2/twisted/web/resource.py
new file mode 100644
index 00000000000..147b110ef27
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/resource.py
@@ -0,0 +1,422 @@
+# -*- test-case-name: twisted.web.test.test_web -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of the lowest-level Resource class.
+"""
+
+from __future__ import division, absolute_import
+
+__all__ = [
+    'IResource', 'getChildForRequest',
+    'Resource', 'ErrorPage', 'NoResource', 'ForbiddenResource',
+    'EncodingResourceWrapper']
+
+import warnings
+
+from zope.interface import Attribute, Interface, implementer
+
+from twisted.python.compat import nativeString, unicode
+from twisted.python.reflect import prefixedMethodNames
+from twisted.python.components import proxyForInterface
+
+from twisted.web._responses import FORBIDDEN, NOT_FOUND
+from twisted.web.error import UnsupportedMethod
+
+
+
+class IResource(Interface):
+    """
+    A web resource.
+    """
+
+    isLeaf = Attribute(
+        """
+        Signal if this IResource implementor is a "leaf node" or not. If True,
+        getChildWithDefault will not be called on this Resource.
+        """)
+
+
+    def getChildWithDefault(name, request):
+        """
+        Return a child with the given name for the given request.
+        This is the external interface used by the Resource publishing
+        machinery. If implementing IResource without subclassing
+        Resource, it must be provided. However, if subclassing Resource,
+        getChild overridden instead.
+
+        @param name: A single path component from a requested URL.  For example,
+            a request for I{http://example.com/foo/bar} will result in calls to
+            this method with C{b"foo"} and C{b"bar"} as values for this
+            argument.
+        @type name: C{bytes}
+
+        @param request: A representation of all of the information about the
+            request that is being made for this child.
+        @type request: L{twisted.web.server.Request}
+        """
+
+
+    def putChild(path, child):
+        """
+        Put a child IResource implementor at the given path.
+
+        @param path: A single path component, to be interpreted relative to the
+            path this resource is found at, at which to put the given child.
+            For example, if resource A can be found at I{http://example.com/foo}
+            then a call like C{A.putChild(b"bar", B)} will make resource B
+            available at I{http://example.com/foo/bar}.
+        @type path: C{bytes}
+        """
+
+
+    def render(request):
+        """
+        Render a request. This is called on the leaf resource for a request.
+
+        @return: Either C{server.NOT_DONE_YET} to indicate an asynchronous or a
+            C{bytes} instance to write as the response to the request.  If
+            C{NOT_DONE_YET} is returned, at some point later (for example, in a
+            Deferred callback) call C{request.write(b"")} to write data to
+            the request, and C{request.finish()} to send the data to the
+            browser.
+
+        @raise twisted.web.error.UnsupportedMethod: If the HTTP verb
+            requested is not supported by this resource.
+        """
+
+
+
+def getChildForRequest(resource, request):
+    """
+    Traverse resource tree to find who will handle the request.
+    """
+    while request.postpath and not resource.isLeaf:
+        pathElement = request.postpath.pop(0)
+        request.prepath.append(pathElement)
+        resource = resource.getChildWithDefault(pathElement, request)
+    return resource
+
+
+
+@implementer(IResource)
+class Resource:
+    """
+    Define a web-accessible resource.
+
+    This serves 2 main purposes; one is to provide a standard representation
+    for what HTTP specification calls an 'entity', and the other is to provide
+    an abstract directory structure for URL retrieval.
+    """
+    entityType = IResource
+
+    server = None
+
+    def __init__(self):
+        """
+        Initialize.
+        """
+        self.children = {}
+
+    isLeaf = 0
+
+    ### Abstract Collection Interface
+
+    def listStaticNames(self):
+        return list(self.children.keys())
+
+    def listStaticEntities(self):
+        return list(self.children.items())
+
+    def listNames(self):
+        return list(self.listStaticNames()) + self.listDynamicNames()
+
+    def listEntities(self):
+        return list(self.listStaticEntities()) + self.listDynamicEntities()
+
+    def listDynamicNames(self):
+        return []
+
+    def listDynamicEntities(self, request=None):
+        return []
+
+    def getStaticEntity(self, name):
+        return self.children.get(name)
+
+    def getDynamicEntity(self, name, request):
+        if name not in self.children:
+            return self.getChild(name, request)
+        else:
+            return None
+
+    def delEntity(self, name):
+        del self.children[name]
+
+    def reallyPutEntity(self, name, entity):
+        self.children[name] = entity
+
+    # Concrete HTTP interface
+
+    def getChild(self, path, request):
+        """
+        Retrieve a 'child' resource from me.
+
+        Implement this to create dynamic resource generation -- resources which
+        are always available may be registered with self.putChild().
+
+        This will not be called if the class-level variable 'isLeaf' is set in
+        your subclass; instead, the 'postpath' attribute of the request will be
+        left as a list of the remaining path elements.
+
+        For example, the URL /foo/bar/baz will normally be::
+
+          | site.resource.getChild('foo').getChild('bar').getChild('baz').
+
+        However, if the resource returned by 'bar' has isLeaf set to true, then
+        the getChild call will never be made on it.
+
+        Parameters and return value have the same meaning and requirements as
+        those defined by L{IResource.getChildWithDefault}.
+        """
+        return NoResource("No such child resource.")
+
+
+    def getChildWithDefault(self, path, request):
+        """
+        Retrieve a static or dynamically generated child resource from me.
+
+        First checks if a resource was added manually by putChild, and then
+        call getChild to check for dynamic resources. Only override if you want
+        to affect behaviour of all child lookups, rather than just dynamic
+        ones.
+
+        This will check to see if I have a pre-registered child resource of the
+        given name, and call getChild if I do not.
+
+        @see: L{IResource.getChildWithDefault}
+        """
+        if path in self.children:
+            return self.children[path]
+        return self.getChild(path, request)
+
+
+    def getChildForRequest(self, request):
+        warnings.warn("Please use module level getChildForRequest.", DeprecationWarning, 2)
+        return getChildForRequest(self, request)
+
+
+    def putChild(self, path, child):
+        """
+        Register a static child.
+
+        You almost certainly don't want '/' in your path. If you
+        intended to have the root of a folder, e.g. /foo/, you want
+        path to be ''.
+
+        @param path: A single path component.
+        @type path: L{bytes}
+
+        @param child: The child resource to register.
+        @type child: L{IResource}
+
+        @see: L{IResource.putChild}
+        """
+        if not isinstance(path, bytes):
+            warnings.warn(
+                'Path segment must be bytes; '
+                'passing {0} has never worked, and '
+                'will raise an exception in the future.'
+                .format(type(path)),
+                category=DeprecationWarning,
+                stacklevel=2)
+
+        self.children[path] = child
+        child.server = self.server
+
+
+    def render(self, request):
+        """
+        Render a given resource. See L{IResource}'s render method.
+
+        I delegate to methods of self with the form 'render_METHOD'
+        where METHOD is the HTTP that was used to make the
+        request. Examples: render_GET, render_HEAD, render_POST, and
+        so on. Generally you should implement those methods instead of
+        overriding this one.
+
+        render_METHOD methods are expected to return a byte string which will be
+        the rendered page, unless the return value is C{server.NOT_DONE_YET}, in
+        which case it is this class's responsibility to write the results using
+        C{request.write(data)} and then call C{request.finish()}.
+
+        Old code that overrides render() directly is likewise expected
+        to return a byte string or NOT_DONE_YET.
+
+        @see: L{IResource.render}
+        """
+        m = getattr(self, 'render_' + nativeString(request.method), None)
+        if not m:
+            try:
+                allowedMethods = self.allowedMethods
+            except AttributeError:
+                allowedMethods = _computeAllowedMethods(self)
+            raise UnsupportedMethod(allowedMethods)
+        return m(request)
+
+
+    def render_HEAD(self, request):
+        """
+        Default handling of HEAD method.
+
+        I just return self.render_GET(request). When method is HEAD,
+        the framework will handle this correctly.
+        """
+        return self.render_GET(request)
+
+
+
+def _computeAllowedMethods(resource):
+    """
+    Compute the allowed methods on a C{Resource} based on defined render_FOO
+    methods. Used when raising C{UnsupportedMethod} but C{Resource} does
+    not define C{allowedMethods} attribute.
+    """
+    allowedMethods = []
+    for name in prefixedMethodNames(resource.__class__, "render_"):
+        # Potentially there should be an API for encode('ascii') in this
+        # situation - an API for taking a Python native string (bytes on Python
+        # 2, text on Python 3) and returning a socket-compatible string type.
+        allowedMethods.append(name.encode('ascii'))
+    return allowedMethods
+
+
+
+class ErrorPage(Resource):
+    """
+    L{ErrorPage} is a resource which responds with a particular
+    (parameterized) status and a body consisting of HTML containing some
+    descriptive text.  This is useful for rendering simple error pages.
+
+    @ivar template: A native string which will have a dictionary interpolated
+        into it to generate the response body.  The dictionary has the following
+        keys:
+
+          - C{"code"}: The status code passed to L{ErrorPage.__init__}.
+          - C{"brief"}: The brief description passed to L{ErrorPage.__init__}.
+          - C{"detail"}: The detailed description passed to
+            L{ErrorPage.__init__}.
+
+    @ivar code: An integer status code which will be used for the response.
+    @type code: C{int}
+
+    @ivar brief: A short string which will be included in the response body as
+        the page title.
+    @type brief: C{str}
+
+    @ivar detail: A longer string which will be included in the response body.
+    @type detail: C{str}
+    """
+
+    template = """
+
+  %(code)s - %(brief)s
+  
+    
%(brief)s

+    
%(detail)s

+  
+
+"""
+
+    def __init__(self, status, brief, detail):
+        Resource.__init__(self)
+        self.code = status
+        self.brief = brief
+        self.detail = detail
+
+
+    def render(self, request):
+        request.setResponseCode(self.code)
+        request.setHeader(b"content-type", b"text/html; charset=utf-8")
+        interpolated = self.template % dict(
+            code=self.code, brief=self.brief, detail=self.detail)
+        if isinstance(interpolated, unicode):
+            return interpolated.encode('utf-8')
+        return interpolated
+
+
+    def getChild(self, chnam, request):
+        return self
+
+
+
+class NoResource(ErrorPage):
+    """
+    L{NoResource} is a specialization of L{ErrorPage} which returns the HTTP
+    response code I{NOT FOUND}.
+    """
+    def __init__(self, message="Sorry. No luck finding that resource."):
+        ErrorPage.__init__(self, NOT_FOUND, "No Such Resource", message)
+
+
+
+class ForbiddenResource(ErrorPage):
+    """
+    L{ForbiddenResource} is a specialization of L{ErrorPage} which returns the
+    I{FORBIDDEN} HTTP response code.
+    """
+    def __init__(self, message="Sorry, resource is forbidden."):
+        ErrorPage.__init__(self, FORBIDDEN, "Forbidden Resource", message)
+
+
+
+class _IEncodingResource(Interface):
+    """
+    A resource which knows about L{_IRequestEncoderFactory}.
+
+    @since: 12.3
+    """
+
+    def getEncoder(request):
+        """
+        Parse the request and return an encoder if applicable, using
+        L{_IRequestEncoderFactory.encoderForRequest}.
+
+        @return: A L{_IRequestEncoder}, or L{None}.
+        """
+
+
+
+@implementer(_IEncodingResource)
+class EncodingResourceWrapper(proxyForInterface(IResource)):
+    """
+    Wrap a L{IResource}, potentially applying an encoding to the response body
+    generated.
+
+    Note that the returned children resources won't be wrapped, so you have to
+    explicitly wrap them if you want the encoding to be applied.
+
+    @ivar encoders: A list of
+        L{_IRequestEncoderFactory}
+        returning L{_IRequestEncoder} that
+        may transform the data passed to C{Request.write}. The list must be
+        sorted in order of priority: the first encoder factory handling the
+        request will prevent the others from doing the same.
+    @type encoders: C{list}.
+
+    @since: 12.3
+    """
+
+    def __init__(self, original, encoders):
+        super(EncodingResourceWrapper, self).__init__(original)
+        self._encoders = encoders
+
+
+    def getEncoder(self, request):
+        """
+        Browser the list of encoders looking for one applicable encoder.
+        """
+        for encoderFactory in self._encoders:
+            encoder = encoderFactory.encoderForRequest(request)
+            if encoder is not None:
+                return encoder
diff --git a/contrib/python/Twisted/py2/twisted/web/rewrite.py b/contrib/python/Twisted/py2/twisted/web/rewrite.py
new file mode 100644
index 00000000000..b5366b4eb78
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/rewrite.py
@@ -0,0 +1,52 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+from twisted.web import resource
+
+class RewriterResource(resource.Resource):
+
+    def __init__(self, orig, *rewriteRules):
+        resource.Resource.__init__(self)
+        self.resource = orig
+        self.rewriteRules = list(rewriteRules)
+
+    def _rewrite(self, request):
+        for rewriteRule in self.rewriteRules:
+            rewriteRule(request)
+
+    def getChild(self, path, request):
+        request.postpath.insert(0, path)
+        request.prepath.pop()
+        self._rewrite(request)
+        path = request.postpath.pop(0)
+        request.prepath.append(path)
+        return self.resource.getChildWithDefault(path, request)
+
+    def render(self, request):
+        self._rewrite(request)
+        return self.resource.render(request)
+
+
+def tildeToUsers(request):
+    if request.postpath and request.postpath[0][:1]=='~':
+        request.postpath[:1] = ['users', request.postpath[0][1:]]
+        request.path = '/'+'/'.join(request.prepath+request.postpath)
+
+def alias(aliasPath, sourcePath):
+    """
+    I am not a very good aliaser. But I'm the best I can be. If I'm
+    aliasing to a Resource that generates links, and it uses any parts
+    of request.prepath to do so, the links will not be relative to the
+    aliased path, but rather to the aliased-to path. That I can't
+    alias static.File directory listings that nicely. However, I can
+    still be useful, as many resources will play nice.
+    """
+    sourcePath = sourcePath.split('/')
+    aliasPath = aliasPath.split('/')
+    def rewriter(request):
+        if request.postpath[:len(aliasPath)] == aliasPath:
+            after = request.postpath[len(aliasPath):]
+            request.postpath = sourcePath + after
+            request.path = '/'+'/'.join(request.prepath+request.postpath)
+    return rewriter
diff --git a/contrib/python/Twisted/py2/twisted/web/script.py b/contrib/python/Twisted/py2/twisted/web/script.py
new file mode 100644
index 00000000000..aa030368a4a
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/script.py
@@ -0,0 +1,182 @@
+# -*- test-case-name: twisted.web.test.test_script -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+I contain PythonScript, which is a very simple python script resource.
+"""
+
+from __future__ import division, absolute_import
+
+import os, traceback
+
+from twisted import copyright
+from twisted.python.filepath import _coerceToFilesystemEncoding
+from twisted.python.compat import execfile, networkString, NativeStringIO, _PY3
+from twisted.web import http, server, static, resource, util
+
+
+rpyNoResource = """
You forgot to assign to the variable "resource" in your script. For example:

+
+# MyCoolWebApp.rpy
+
+import mygreatresource
+
+resource = mygreatresource.MyGreatResource()
+

+"""
+
+class AlreadyCached(Exception):
+    """
+    This exception is raised when a path has already been cached.
+    """
+
+class CacheScanner:
+    def __init__(self, path, registry):
+        self.path = path
+        self.registry = registry
+        self.doCache = 0
+
+    def cache(self):
+        c = self.registry.getCachedPath(self.path)
+        if c is not None:
+            raise AlreadyCached(c)
+        self.recache()
+
+    def recache(self):
+        self.doCache = 1
+
+noRsrc = resource.ErrorPage(500, "Whoops! Internal Error", rpyNoResource)
+
+def ResourceScript(path, registry):
+    """
+    I am a normal py file which must define a 'resource' global, which should
+    be an instance of (a subclass of) web.resource.Resource; it will be
+    renderred.
+    """
+    cs = CacheScanner(path, registry)
+    glob = {'__file__': _coerceToFilesystemEncoding("", path),
+            'resource': noRsrc,
+            'registry': registry,
+            'cache': cs.cache,
+            'recache': cs.recache}
+    try:
+        execfile(path, glob, glob)
+    except AlreadyCached as ac:
+        return ac.args[0]
+    rsrc = glob['resource']
+    if cs.doCache and rsrc is not noRsrc:
+        registry.cachePath(path, rsrc)
+    return rsrc
+
+
+
+def ResourceTemplate(path, registry):
+    from quixote import ptl_compile
+
+    glob = {'__file__': _coerceToFilesystemEncoding("", path),
+            'resource': resource.ErrorPage(500, "Whoops! Internal Error",
+                                           rpyNoResource),
+            'registry': registry}
+
+    with open(path) as f:  # Not closed by quixote as of 2.9.1
+        e = ptl_compile.compile_template(f, path)
+    code = compile(e, "", "exec")
+    eval(code, glob, glob)
+    return glob['resource']
+
+
+
+class ResourceScriptWrapper(resource.Resource):
+
+    def __init__(self, path, registry=None):
+        resource.Resource.__init__(self)
+        self.path = path
+        self.registry = registry or static.Registry()
+
+    def render(self, request):
+        res = ResourceScript(self.path, self.registry)
+        return res.render(request)
+
+    def getChildWithDefault(self, path, request):
+        res = ResourceScript(self.path, self.registry)
+        return res.getChildWithDefault(path, request)
+
+
+
+class ResourceScriptDirectory(resource.Resource):
+    """
+    L{ResourceScriptDirectory} is a resource which serves scripts from a
+    filesystem directory.  File children of a L{ResourceScriptDirectory} will
+    be served using L{ResourceScript}.  Directory children will be served using
+    another L{ResourceScriptDirectory}.
+
+    @ivar path: A C{str} giving the filesystem path in which children will be
+        looked up.
+
+    @ivar registry: A L{static.Registry} instance which will be used to decide
+        how to interpret scripts found as children of this resource.
+    """
+    def __init__(self, pathname, registry=None):
+        resource.Resource.__init__(self)
+        self.path = pathname
+        self.registry = registry or static.Registry()
+
+    def getChild(self, path, request):
+        fn = os.path.join(self.path, path)
+
+        if os.path.isdir(fn):
+            return ResourceScriptDirectory(fn, self.registry)
+        if os.path.exists(fn):
+            return ResourceScript(fn, self.registry)
+        return resource.NoResource()
+
+    def render(self, request):
+        return resource.NoResource().render(request)
+
+
+
+class PythonScript(resource.Resource):
+    """
+    I am an extremely simple dynamic resource; an embedded python script.
+
+    This will execute a file (usually of the extension '.epy') as Python code,
+    internal to the webserver.
+    """
+    isLeaf = True
+
+    def __init__(self, filename, registry):
+        """
+        Initialize me with a script name.
+        """
+        self.filename = filename
+        self.registry = registry
+
+    def render(self, request):
+        """
+        Render me to a web client.
+
+        Load my file, execute it in a special namespace (with 'request' and
+        '__file__' global vars) and finish the request.  Output to the web-page
+        will NOT be handled with print - standard output goes to the log - but
+        with request.write.
+        """
+        request.setHeader(b"x-powered-by", networkString("Twisted/%s" % copyright.version))
+        namespace = {'request': request,
+                     '__file__': _coerceToFilesystemEncoding("", self.filename),
+                     'registry': self.registry}
+        try:
+            execfile(self.filename, namespace, namespace)
+        except IOError as e:
+            if e.errno == 2: #file not found
+                request.setResponseCode(http.NOT_FOUND)
+                request.write(resource.NoResource("File not found.").render(request))
+        except:
+            io = NativeStringIO()
+            traceback.print_exc(file=io)
+            output = util._PRE(io.getvalue())
+            if _PY3:
+                output = output.encode("utf8")
+            request.write(output)
+        request.finish()
+        return server.NOT_DONE_YET
diff --git a/contrib/python/Twisted/py2/twisted/web/server.py b/contrib/python/Twisted/py2/twisted/web/server.py
new file mode 100644
index 00000000000..6fa488afb1a
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/server.py
@@ -0,0 +1,911 @@
+# -*- test-case-name: twisted.web.test.test_web -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This is a web server which integrates with the twisted.internet infrastructure.
+
+@var NOT_DONE_YET: A token value which L{twisted.web.resource.IResource.render}
+    implementations can return to indicate that the application will later call
+    C{.write} and C{.finish} to complete the request, and that the HTTP
+    connection should be left open.
+@type NOT_DONE_YET: Opaque; do not depend on any particular type for this
+    value.
+"""
+
+from __future__ import division, absolute_import
+
+import copy
+import os
+import re
+try:
+    from urllib import quote
+except ImportError:
+    from urllib.parse import quote as _quote
+
+    def quote(string, *args, **kwargs):
+        return _quote(
+            string.decode('charmap'), *args, **kwargs).encode('charmap')
+
+import zlib
+from binascii import hexlify
+
+from zope.interface import implementer
+
+from twisted.python.compat import networkString, nativeString, intToBytes
+from twisted.spread.pb import Copyable, ViewPoint
+from twisted.internet import address, interfaces
+from twisted.internet.error import AlreadyCalled, AlreadyCancelled
+from twisted.web import iweb, http, util
+from twisted.web.http import unquote
+from twisted.python import reflect, failure, components
+from twisted import copyright
+from twisted.web import resource
+from twisted.web.error import UnsupportedMethod
+
+from incremental import Version
+from twisted.python.deprecate import deprecatedModuleAttribute
+from twisted.python.compat import escape
+from twisted.logger import Logger
+
+NOT_DONE_YET = 1
+
+__all__ = [
+    'supportedMethods',
+    'Request',
+    'Session',
+    'Site',
+    'version',
+    'NOT_DONE_YET',
+    'GzipEncoderFactory'
+]
+
+
+# backwards compatibility
+deprecatedModuleAttribute(
+    Version("Twisted", 12, 1, 0),
+    "Please use twisted.web.http.datetimeToString instead",
+    "twisted.web.server",
+    "date_time_string")
+deprecatedModuleAttribute(
+    Version("Twisted", 12, 1, 0),
+    "Please use twisted.web.http.stringToDatetime instead",
+    "twisted.web.server",
+    "string_date_time")
+date_time_string = http.datetimeToString
+string_date_time = http.stringToDatetime
+
+# Support for other methods may be implemented on a per-resource basis.
+supportedMethods = (b'GET', b'HEAD', b'POST')
+
+
+def _addressToTuple(addr):
+    if isinstance(addr, address.IPv4Address):
+        return ('INET', addr.host, addr.port)
+    elif isinstance(addr, address.UNIXAddress):
+        return ('UNIX', addr.name)
+    else:
+        return tuple(addr)
+
+
+
+@implementer(iweb.IRequest)
+class Request(Copyable, http.Request, components.Componentized):
+    """
+    An HTTP request.
+
+    @ivar defaultContentType: A L{bytes} giving the default I{Content-Type}
+        value to send in responses if no other value is set.  L{None} disables
+        the default.
+
+    @ivar _insecureSession: The L{Session} object representing state that will
+        be transmitted over plain-text HTTP.
+
+    @ivar _secureSession: The L{Session} object representing the state that
+        will be transmitted only over HTTPS.
+    """
+
+    defaultContentType = b"text/html"
+
+    site = None
+    appRootURL = None
+    prepath = postpath = None
+    __pychecker__ = 'unusednames=issuer'
+    _inFakeHead = False
+    _encoder = None
+    _log = Logger()
+
+    def __init__(self, *args, **kw):
+        http.Request.__init__(self, *args, **kw)
+        components.Componentized.__init__(self)
+
+
+    def getStateToCopyFor(self, issuer):
+        x = self.__dict__.copy()
+        del x['transport']
+        # XXX refactor this attribute out; it's from protocol
+        # del x['server']
+        del x['channel']
+        del x['content']
+        del x['site']
+        self.content.seek(0, 0)
+        x['content_data'] = self.content.read()
+        x['remote'] = ViewPoint(issuer, self)
+
+        # Address objects aren't jellyable
+        x['host'] = _addressToTuple(x['host'])
+        x['client'] = _addressToTuple(x['client'])
+
+        # Header objects also aren't jellyable.
+        x['requestHeaders'] = list(x['requestHeaders'].getAllRawHeaders())
+
+        return x
+
+    # HTML generation helpers
+
+
+    def sibLink(self, name):
+        """
+        Return the text that links to a sibling of the requested resource.
+
+        @param name: The sibling resource
+        @type name: C{bytes}
+
+        @return: A relative URL.
+        @rtype: C{bytes}
+        """
+        if self.postpath:
+            return (len(self.postpath)*b"../") + name
+        else:
+            return name
+
+
+    def childLink(self, name):
+        """
+        Return the text that links to a child of the requested resource.
+
+        @param name: The child resource
+        @type name: C{bytes}
+
+        @return: A relative URL.
+        @rtype: C{bytes}
+        """
+        lpp = len(self.postpath)
+        if lpp > 1:
+            return ((lpp-1)*b"../") + name
+        elif lpp == 1:
+            return name
+        else:  # lpp == 0
+            if len(self.prepath) and self.prepath[-1]:
+                return self.prepath[-1] + b'/' + name
+            else:
+                return name
+
+
+    def gotLength(self, length):
+        """
+        Called when HTTP channel got length of content in this request.
+
+        This method is not intended for users.
+
+        @param length: The length of the request body, as indicated by the
+            request headers.  L{None} if the request headers do not indicate a
+            length.
+        """
+        try:
+            getContentFile = self.channel.site.getContentFile
+        except AttributeError:
+            http.Request.gotLength(self, length)
+        else:
+            self.content = getContentFile(length)
+
+
+    def process(self):
+        """
+        Process a request.
+
+        Find the addressed resource in this request's L{Site},
+        and call L{self.render()} with it.
+
+        @see: L{Site.getResourceFor()}
+        """
+
+        # get site from channel
+        self.site = self.channel.site
+
+        # set various default headers
+        self.setHeader(b'server', version)
+        self.setHeader(b'date', http.datetimeToString())
+
+        # Resource Identification
+        self.prepath = []
+        self.postpath = list(map(unquote, self.path[1:].split(b'/')))
+
+        # Short-circuit for requests whose path is '*'.
+        if self.path == b'*':
+            self._handleStar()
+            return
+
+        try:
+            resrc = self.site.getResourceFor(self)
+            if resource._IEncodingResource.providedBy(resrc):
+                encoder = resrc.getEncoder(self)
+                if encoder is not None:
+                    self._encoder = encoder
+            self.render(resrc)
+        except:
+            self.processingFailed(failure.Failure())
+
+
+    def write(self, data):
+        """
+        Write data to the transport (if not responding to a HEAD request).
+
+        @param data: A string to write to the response.
+        @type data: L{bytes}
+        """
+        if not self.startedWriting:
+            # Before doing the first write, check to see if a default
+            # Content-Type header should be supplied. We omit it on
+            # NOT_MODIFIED and NO_CONTENT responses. We also omit it if there
+            # is a Content-Length header set to 0, as empty bodies don't need
+            # a content-type.
+            needsCT = self.code not in (http.NOT_MODIFIED, http.NO_CONTENT)
+            contentType = self.responseHeaders.getRawHeaders(b'content-type')
+            contentLength = self.responseHeaders.getRawHeaders(
+                b'content-length'
+            )
+            contentLengthZero = contentLength and (contentLength[0] == b'0')
+
+            if (needsCT and contentType is None and
+                self.defaultContentType is not None and
+                not contentLengthZero
+                    ):
+                self.responseHeaders.setRawHeaders(
+                    b'content-type', [self.defaultContentType])
+
+        # Only let the write happen if we're not generating a HEAD response by
+        # faking out the request method.  Note, if we are doing that,
+        # startedWriting will never be true, and the above logic may run
+        # multiple times.  It will only actually change the responseHeaders
+        # once though, so it's still okay.
+        if not self._inFakeHead:
+            if self._encoder:
+                data = self._encoder.encode(data)
+            http.Request.write(self, data)
+
+
+    def finish(self):
+        """
+        Override C{http.Request.finish} for possible encoding.
+        """
+        if self._encoder:
+            data = self._encoder.finish()
+            if data:
+                http.Request.write(self, data)
+        return http.Request.finish(self)
+
+
+    def render(self, resrc):
+        """
+        Ask a resource to render itself.
+
+        If the resource does not support the requested method,
+        generate a C{NOT IMPLEMENTED} or C{NOT ALLOWED} response.
+
+        @param resrc: The resource to render.
+        @type resrc: L{twisted.web.resource.IResource}
+
+        @see: L{IResource.render()}
+        """
+        try:
+            body = resrc.render(self)
+        except UnsupportedMethod as e:
+            allowedMethods = e.allowedMethods
+            if (self.method == b"HEAD") and (b"GET" in allowedMethods):
+                # We must support HEAD (RFC 2616, 5.1.1).  If the
+                # resource doesn't, fake it by giving the resource
+                # a 'GET' request and then return only the headers,
+                # not the body.
+                self._log.info(
+                    "Using GET to fake a HEAD request for {resrc}",
+                    resrc=resrc
+                )
+                self.method = b"GET"
+                self._inFakeHead = True
+                body = resrc.render(self)
+
+                if body is NOT_DONE_YET:
+                    self._log.info(
+                        "Tried to fake a HEAD request for {resrc}, but "
+                        "it got away from me.", resrc=resrc
+                    )
+                    # Oh well, I guess we won't include the content length.
+                else:
+                    self.setHeader(b'content-length', intToBytes(len(body)))
+
+                self._inFakeHead = False
+                self.method = b"HEAD"
+                self.write(b'')
+                self.finish()
+                return
+
+            if self.method in (supportedMethods):
+                # We MUST include an Allow header
+                # (RFC 2616, 10.4.6 and 14.7)
+                self.setHeader(b'Allow', b', '.join(allowedMethods))
+                s = ('''Your browser approached me (at %(URI)s) with'''
+                     ''' the method "%(method)s".  I only allow'''
+                     ''' the method%(plural)s %(allowed)s here.''' % {
+                         'URI': escape(nativeString(self.uri)),
+                         'method': nativeString(self.method),
+                         'plural': ((len(allowedMethods) > 1) and 's') or '',
+                         'allowed': ', '.join(
+                            [nativeString(x) for x in allowedMethods])
+                     })
+                epage = resource.ErrorPage(http.NOT_ALLOWED,
+                                           "Method Not Allowed", s)
+                body = epage.render(self)
+            else:
+                epage = resource.ErrorPage(
+                    http.NOT_IMPLEMENTED, "Huh?",
+                    "I don't know how to treat a %s request." %
+                    (escape(self.method.decode("charmap")),))
+                body = epage.render(self)
+        # end except UnsupportedMethod
+
+        if body is NOT_DONE_YET:
+            return
+        if not isinstance(body, bytes):
+            body = resource.ErrorPage(
+                http.INTERNAL_SERVER_ERROR,
+                "Request did not return bytes",
+                "Request: " + util._PRE(reflect.safe_repr(self)) + "
" +
+                "Resource: " + util._PRE(reflect.safe_repr(resrc)) + "
" +
+                "Value: " + util._PRE(reflect.safe_repr(body))).render(self)
+
+        if self.method == b"HEAD":
+            if len(body) > 0:
+                # This is a Bad Thing (RFC 2616, 9.4)
+                self._log.info(
+                    "Warning: HEAD request {slf} for resource {resrc} is"
+                    " returning a message body. I think I'll eat it.",
+                    slf=self,
+                    resrc=resrc
+                )
+                self.setHeader(b'content-length',
+                               intToBytes(len(body)))
+            self.write(b'')
+        else:
+            self.setHeader(b'content-length',
+                           intToBytes(len(body)))
+            self.write(body)
+        self.finish()
+
+
+    def processingFailed(self, reason):
+        """
+        Finish this request with an indication that processing failed and
+        possibly display a traceback.
+
+        @param reason: Reason this request has failed.
+        @type reason: L{twisted.python.failure.Failure}
+
+        @return: The reason passed to this method.
+        @rtype: L{twisted.python.failure.Failure}
+        """
+        self._log.failure('', failure=reason)
+        if self.site.displayTracebacks:
+            body = (b"web.Server Traceback"
+                    b" (most recent call last)"
+                    b"web.Server Traceback"
+                    b" (most recent call last):\n\n" +
+                    util.formatFailure(reason) +
+                    b"\n\n\n")
+        else:
+            body = (b"Processing Failed"
+                    b""
+                    b"Processing Failed")
+
+        self.setResponseCode(http.INTERNAL_SERVER_ERROR)
+        self.setHeader(b'content-type', b"text/html")
+        self.setHeader(b'content-length', intToBytes(len(body)))
+        self.write(body)
+        self.finish()
+        return reason
+
+
+    def view_write(self, issuer, data):
+        """Remote version of write; same interface.
+        """
+        self.write(data)
+
+
+    def view_finish(self, issuer):
+        """Remote version of finish; same interface.
+        """
+        self.finish()
+
+
+    def view_addCookie(self, issuer, k, v, **kwargs):
+        """Remote version of addCookie; same interface.
+        """
+        self.addCookie(k, v, **kwargs)
+
+
+    def view_setHeader(self, issuer, k, v):
+        """Remote version of setHeader; same interface.
+        """
+        self.setHeader(k, v)
+
+
+    def view_setLastModified(self, issuer, when):
+        """Remote version of setLastModified; same interface.
+        """
+        self.setLastModified(when)
+
+
+    def view_setETag(self, issuer, tag):
+        """Remote version of setETag; same interface.
+        """
+        self.setETag(tag)
+
+
+    def view_setResponseCode(self, issuer, code, message=None):
+        """
+        Remote version of setResponseCode; same interface.
+        """
+        self.setResponseCode(code, message)
+
+
+    def view_registerProducer(self, issuer, producer, streaming):
+        """Remote version of registerProducer; same interface.
+        (requires a remote producer.)
+        """
+        self.registerProducer(_RemoteProducerWrapper(producer), streaming)
+
+
+    def view_unregisterProducer(self, issuer):
+        self.unregisterProducer()
+
+    ### these calls remain local
+
+    _secureSession = None
+    _insecureSession = None
+
+    @property
+    def session(self):
+        """
+        If a session has already been created or looked up with
+        L{Request.getSession}, this will return that object.  (This will always
+        be the session that matches the security of the request; so if
+        C{forceNotSecure} is used on a secure request, this will not return
+        that session.)
+
+        @return: the session attribute
+        @rtype: L{Session} or L{None}
+        """
+        if self.isSecure():
+            return self._secureSession
+        else:
+            return self._insecureSession
+
+
+    def getSession(self, sessionInterface=None, forceNotSecure=False):
+        """
+        Check if there is a session cookie, and if not, create it.
+
+        By default, the cookie with be secure for HTTPS requests and not secure
+        for HTTP requests.  If for some reason you need access to the insecure
+        cookie from a secure request you can set C{forceNotSecure = True}.
+
+        @param forceNotSecure: Should we retrieve a session that will be
+            transmitted over HTTP, even if this L{Request} was delivered over
+            HTTPS?
+        @type forceNotSecure: L{bool}
+        """
+        # Make sure we aren't creating a secure session on a non-secure page
+        secure = self.isSecure() and not forceNotSecure
+
+        if not secure:
+            cookieString = b"TWISTED_SESSION"
+            sessionAttribute = "_insecureSession"
+        else:
+            cookieString = b"TWISTED_SECURE_SESSION"
+            sessionAttribute = "_secureSession"
+
+        session = getattr(self, sessionAttribute)
+
+        if session is not None:
+            # We have a previously created session.
+            try:
+                # Refresh the session, to keep it alive.
+                session.touch()
+            except (AlreadyCalled, AlreadyCancelled):
+                # Session has already expired.
+                session = None
+
+        if session is None:
+            # No session was created yet for this request.
+            cookiename = b"_".join([cookieString] + self.sitepath)
+            sessionCookie = self.getCookie(cookiename)
+            if sessionCookie:
+                try:
+                    session = self.site.getSession(sessionCookie)
+                except KeyError:
+                    pass
+            # if it still hasn't been set, fix it up.
+            if not session:
+                session = self.site.makeSession()
+                self.addCookie(cookiename, session.uid, path=b"/",
+                               secure=secure)
+
+        setattr(self, sessionAttribute, session)
+
+        if sessionInterface:
+            return session.getComponent(sessionInterface)
+
+        return session
+
+
+    def _prePathURL(self, prepath):
+        port = self.getHost().port
+        if self.isSecure():
+            default = 443
+        else:
+            default = 80
+        if port == default:
+            hostport = ''
+        else:
+            hostport = ':%d' % port
+        prefix = networkString('http%s://%s%s/' % (
+            self.isSecure() and 's' or '',
+            nativeString(self.getRequestHostname()),
+            hostport))
+        path = b'/'.join([quote(segment, safe=b'') for segment in prepath])
+        return prefix + path
+
+
+    def prePathURL(self):
+        return self._prePathURL(self.prepath)
+
+
+    def URLPath(self):
+        from twisted.python import urlpath
+        return urlpath.URLPath.fromRequest(self)
+
+
+    def rememberRootURL(self):
+        """
+        Remember the currently-processed part of the URL for later
+        recalling.
+        """
+        url = self._prePathURL(self.prepath[:-1])
+        self.appRootURL = url
+
+
+    def getRootURL(self):
+        """
+        Get a previously-remembered URL.
+
+        @return: An absolute URL.
+        @rtype: L{bytes}
+        """
+        return self.appRootURL
+
+
+    def _handleStar(self):
+        """
+        Handle receiving a request whose path is '*'.
+
+        RFC 7231 defines an OPTIONS * request as being something that a client
+        can send as a low-effort way to probe server capabilities or readiness.
+        Rather than bother the user with this, we simply fast-path it back to
+        an empty 200 OK. Any non-OPTIONS verb gets a 405 Method Not Allowed
+        telling the client they can only use OPTIONS.
+        """
+        if self.method == b'OPTIONS':
+            self.setResponseCode(http.OK)
+        else:
+            self.setResponseCode(http.NOT_ALLOWED)
+            self.setHeader(b'Allow', b'OPTIONS')
+
+        # RFC 7231 says we MUST set content-length 0 when responding to this
+        # with no body.
+        self.setHeader(b'Content-Length', b'0')
+        self.finish()
+
+
+@implementer(iweb._IRequestEncoderFactory)
+class GzipEncoderFactory(object):
+    """
+    @cvar compressLevel: The compression level used by the compressor, default
+        to 9 (highest).
+
+    @since: 12.3
+    """
+    _gzipCheckRegex = re.compile(br'(:?^|[\s,])gzip(:?$|[\s,])')
+    compressLevel = 9
+
+    def encoderForRequest(self, request):
+        """
+        Check the headers if the client accepts gzip encoding, and encodes the
+        request if so.
+        """
+        acceptHeaders = b','.join(
+            request.requestHeaders.getRawHeaders(b'accept-encoding', []))
+        if self._gzipCheckRegex.search(acceptHeaders):
+            encoding = request.responseHeaders.getRawHeaders(
+                b'content-encoding')
+            if encoding:
+                encoding = b','.join(encoding + [b'gzip'])
+            else:
+                encoding = b'gzip'
+
+            request.responseHeaders.setRawHeaders(b'content-encoding',
+                                                  [encoding])
+            return _GzipEncoder(self.compressLevel, request)
+
+
+
+@implementer(iweb._IRequestEncoder)
+class _GzipEncoder(object):
+    """
+    An encoder which supports gzip.
+
+    @ivar _zlibCompressor: The zlib compressor instance used to compress the
+        stream.
+
+    @ivar _request: A reference to the originating request.
+
+    @since: 12.3
+    """
+
+    _zlibCompressor = None
+
+    def __init__(self, compressLevel, request):
+        self._zlibCompressor = zlib.compressobj(
+            compressLevel, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+        self._request = request
+
+
+    def encode(self, data):
+        """
+        Write to the request, automatically compressing data on the fly.
+        """
+        if not self._request.startedWriting:
+            # Remove the content-length header, we can't honor it
+            # because we compress on the fly.
+            self._request.responseHeaders.removeHeader(b'content-length')
+        return self._zlibCompressor.compress(data)
+
+
+    def finish(self):
+        """
+        Finish handling the request request, flushing any data from the zlib
+        buffer.
+        """
+        remain = self._zlibCompressor.flush()
+        self._zlibCompressor = None
+        return remain
+
+
+
+class _RemoteProducerWrapper:
+    def __init__(self, remote):
+        self.resumeProducing = remote.remoteMethod("resumeProducing")
+        self.pauseProducing = remote.remoteMethod("pauseProducing")
+        self.stopProducing = remote.remoteMethod("stopProducing")
+
+
+
+class Session(components.Componentized):
+    """
+    A user's session with a system.
+
+    This utility class contains no functionality, but is used to
+    represent a session.
+
+    @ivar uid: A unique identifier for the session.
+    @type uid: L{bytes}
+
+    @ivar _reactor: An object providing L{IReactorTime} to use for scheduling
+        expiration.
+    @ivar sessionTimeout: timeout of a session, in seconds.
+    """
+    sessionTimeout = 900
+
+    _expireCall = None
+
+    def __init__(self, site, uid, reactor=None):
+        """
+        Initialize a session with a unique ID for that session.
+        """
+        components.Componentized.__init__(self)
+
+        if reactor is None:
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+        self.site = site
+        self.uid = uid
+        self.expireCallbacks = []
+        self.touch()
+        self.sessionNamespaces = {}
+
+
+    def startCheckingExpiration(self):
+        """
+        Start expiration tracking.
+
+        @return: L{None}
+        """
+        self._expireCall = self._reactor.callLater(
+            self.sessionTimeout, self.expire)
+
+
+    def notifyOnExpire(self, callback):
+        """
+        Call this callback when the session expires or logs out.
+        """
+        self.expireCallbacks.append(callback)
+
+
+    def expire(self):
+        """
+        Expire/logout of the session.
+        """
+        del self.site.sessions[self.uid]
+        for c in self.expireCallbacks:
+            c()
+        self.expireCallbacks = []
+        if self._expireCall and self._expireCall.active():
+            self._expireCall.cancel()
+            # Break reference cycle.
+            self._expireCall = None
+
+
+    def touch(self):
+        """
+        Notify session modification.
+        """
+        self.lastModified = self._reactor.seconds()
+        if self._expireCall is not None:
+            self._expireCall.reset(self.sessionTimeout)
+
+
+version = networkString("TwistedWeb/%s" % (copyright.version,))
+
+
+
+@implementer(interfaces.IProtocolNegotiationFactory)
+class Site(http.HTTPFactory):
+    """
+    A web site: manage log, sessions, and resources.
+
+    @ivar counter: increment value used for generating unique sessions ID.
+    @ivar requestFactory: A factory which is called with (channel)
+        and creates L{Request} instances. Default to L{Request}.
+    @ivar displayTracebacks: If set, unhandled exceptions raised during
+        rendering are returned to the client as HTML. Default to C{False}.
+    @ivar sessionFactory: factory for sessions objects. Default to L{Session}.
+    @ivar sessionCheckTime: Deprecated.  See L{Session.sessionTimeout} instead.
+    """
+    counter = 0
+    requestFactory = Request
+    displayTracebacks = False
+    sessionFactory = Session
+    sessionCheckTime = 1800
+    _entropy = os.urandom
+
+    def __init__(self, resource, requestFactory=None, *args, **kwargs):
+        """
+        @param resource: The root of the resource hierarchy.  All request
+            traversal for requests received by this factory will begin at this
+            resource.
+        @type resource: L{IResource} provider
+        @param requestFactory: Overwrite for default requestFactory.
+        @type requestFactory: C{callable} or C{class}.
+
+        @see: L{twisted.web.http.HTTPFactory.__init__}
+        """
+        http.HTTPFactory.__init__(self, *args, **kwargs)
+        self.sessions = {}
+        self.resource = resource
+        if requestFactory is not None:
+            self.requestFactory = requestFactory
+
+
+    def _openLogFile(self, path):
+        from twisted.python import logfile
+        return logfile.LogFile(os.path.basename(path), os.path.dirname(path))
+
+
+    def __getstate__(self):
+        d = self.__dict__.copy()
+        d['sessions'] = {}
+        return d
+
+
+    def _mkuid(self):
+        """
+        (internal) Generate an opaque, unique ID for a user's session.
+        """
+        self.counter = self.counter + 1
+        return hexlify(self._entropy(32))
+
+
+    def makeSession(self):
+        """
+        Generate a new Session instance, and store it for future reference.
+        """
+        uid = self._mkuid()
+        session = self.sessions[uid] = self.sessionFactory(self, uid)
+        session.startCheckingExpiration()
+        return session
+
+
+    def getSession(self, uid):
+        """
+        Get a previously generated session.
+
+        @param uid: Unique ID of the session.
+        @type uid: L{bytes}.
+
+        @raise: L{KeyError} if the session is not found.
+        """
+        return self.sessions[uid]
+
+
+    def buildProtocol(self, addr):
+        """
+        Generate a channel attached to this site.
+        """
+        channel = http.HTTPFactory.buildProtocol(self, addr)
+        channel.requestFactory = self.requestFactory
+        channel.site = self
+        return channel
+
+    isLeaf = 0
+
+    def render(self, request):
+        """
+        Redirect because a Site is always a directory.
+        """
+        request.redirect(request.prePathURL() + b'/')
+        request.finish()
+
+
+    def getChildWithDefault(self, pathEl, request):
+        """
+        Emulate a resource's getChild method.
+        """
+        request.site = self
+        return self.resource.getChildWithDefault(pathEl, request)
+
+
+    def getResourceFor(self, request):
+        """
+        Get a resource for a request.
+
+        This iterates through the resource hierarchy, calling
+        getChildWithDefault on each resource it finds for a path element,
+        stopping when it hits an element where isLeaf is true.
+        """
+        request.site = self
+        # Sitepath is used to determine cookie names between distributed
+        # servers and disconnected sites.
+        request.sitepath = copy.copy(request.prepath)
+        return resource.getChildForRequest(self.resource, request)
+
+    # IProtocolNegotiationFactory
+    def acceptableProtocols(self):
+        """
+        Protocols this server can speak.
+        """
+        baseProtocols = [b'http/1.1']
+
+        if http.H2_ENABLED:
+            baseProtocols.insert(0, b'h2')
+
+        return baseProtocols
diff --git a/contrib/python/Twisted/py2/twisted/web/soap.py b/contrib/python/Twisted/py2/twisted/web/soap.py
new file mode 100644
index 00000000000..fc15e038fa6
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/soap.py
@@ -0,0 +1,154 @@
+# -*- test-case-name: twisted.web.test.test_soap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+SOAP support for twisted.web.
+
+Requires SOAPpy 0.10.1 or later.
+
+Maintainer: Itamar Shtull-Trauring
+
+Future plans:
+SOAPContext support of some kind.
+Pluggable method lookup policies.
+"""
+
+# SOAPpy
+import SOAPpy
+
+# twisted imports
+from twisted.web import server, resource, client
+from twisted.internet import defer
+
+
+class SOAPPublisher(resource.Resource):
+    """Publish SOAP methods.
+
+    By default, publish methods beginning with 'soap_'. If the method
+    has an attribute 'useKeywords', it well get the arguments passed
+    as keyword args.
+    """
+
+    isLeaf = 1
+
+    # override to change the encoding used for responses
+    encoding = "UTF-8"
+
+    def lookupFunction(self, functionName):
+        """Lookup published SOAP function.
+
+        Override in subclasses. Default behaviour - publish methods
+        starting with soap_.
+
+        @return: callable or None if not found.
+        """
+        return getattr(self, "soap_%s" % functionName, None)
+
+    def render(self, request):
+        """Handle a SOAP command."""
+        data = request.content.read()
+
+        p, header, body, attrs = SOAPpy.parseSOAPRPC(data, 1, 1, 1)
+
+        methodName, args, kwargs = p._name, p._aslist, p._asdict
+
+        # deal with changes in SOAPpy 0.11
+        if callable(args):
+            args = args()
+        if callable(kwargs):
+            kwargs = kwargs()
+
+        function = self.lookupFunction(methodName)
+
+        if not function:
+            self._methodNotFound(request, methodName)
+            return server.NOT_DONE_YET
+        else:
+            if hasattr(function, "useKeywords"):
+                keywords = {}
+                for k, v in kwargs.items():
+                    keywords[str(k)] = v
+                d = defer.maybeDeferred(function, **keywords)
+            else:
+                d = defer.maybeDeferred(function, *args)
+
+        d.addCallback(self._gotResult, request, methodName)
+        d.addErrback(self._gotError, request, methodName)
+        return server.NOT_DONE_YET
+
+    def _methodNotFound(self, request, methodName):
+        response = SOAPpy.buildSOAP(SOAPpy.faultType("%s:Client" %
+            SOAPpy.NS.ENV_T, "Method %s not found" % methodName),
+            encoding=self.encoding)
+        self._sendResponse(request, response, status=500)
+
+    def _gotResult(self, result, request, methodName):
+        if not isinstance(result, SOAPpy.voidType):
+            result = {"Result": result}
+        response = SOAPpy.buildSOAP(kw={'%sResponse' % methodName: result},
+                                  encoding=self.encoding)
+        self._sendResponse(request, response)
+
+    def _gotError(self, failure, request, methodName):
+        e = failure.value
+        if isinstance(e, SOAPpy.faultType):
+            fault = e
+        else:
+            fault = SOAPpy.faultType("%s:Server" % SOAPpy.NS.ENV_T,
+                "Method %s failed." % methodName)
+        response = SOAPpy.buildSOAP(fault, encoding=self.encoding)
+        self._sendResponse(request, response, status=500)
+
+    def _sendResponse(self, request, response, status=200):
+        request.setResponseCode(status)
+
+        if self.encoding is not None:
+            mimeType = 'text/xml; charset="%s"' % self.encoding
+        else:
+            mimeType = "text/xml"
+        request.setHeader("Content-type", mimeType)
+        request.setHeader("Content-length", str(len(response)))
+        request.write(response)
+        request.finish()
+
+
+class Proxy:
+    """A Proxy for making remote SOAP calls.
+
+    Pass the URL of the remote SOAP server to the constructor.
+
+    Use proxy.callRemote('foobar', 1, 2) to call remote method
+    'foobar' with args 1 and 2, proxy.callRemote('foobar', x=1)
+    will call foobar with named argument 'x'.
+    """
+
+    # at some point this should have encoding etc. kwargs
+    def __init__(self, url, namespace=None, header=None):
+        self.url = url
+        self.namespace = namespace
+        self.header = header
+
+    def _cbGotResult(self, result):
+        result = SOAPpy.parseSOAPRPC(result)
+        if hasattr(result, 'Result'):
+            return result.Result
+        elif len(result) == 1:
+            ## SOAPpy 0.11.6 wraps the return results in a containing structure.
+            ## This check added to make Proxy behaviour emulate SOAPProxy, which
+            ## flattens the structure by default.
+            ## This behaviour is OK because even singleton lists are wrapped in
+            ## another singleton structType, which is almost always useless.
+            return result[0]
+        else:
+            return result
+
+    def callRemote(self, method, *args, **kwargs):
+        payload = SOAPpy.buildSOAP(args=args, kw=kwargs, method=method,
+                                   header=self.header, namespace=self.namespace)
+        return client.getPage(self.url, postdata=payload, method="POST",
+                              headers={'content-type': 'text/xml',
+                                       'SOAPAction': method}
+                              ).addCallback(self._cbGotResult)
+
diff --git a/contrib/python/Twisted/py2/twisted/web/static.py b/contrib/python/Twisted/py2/twisted/web/static.py
new file mode 100644
index 00000000000..c603b6bbdd1
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/static.py
@@ -0,0 +1,1103 @@
+# -*- test-case-name: twisted.web.test.test_static -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Static resources for L{twisted.web}.
+"""
+
+from __future__ import division, absolute_import
+
+import errno
+import itertools
+import mimetypes
+import os
+import time
+import warnings
+
+from zope.interface import implementer
+
+from twisted.web import server
+from twisted.web import resource
+from twisted.web import http
+from twisted.web.util import redirectTo
+
+from twisted.python.compat import (_PY3, intToBytes, nativeString,
+                                   networkString)
+from twisted.python.compat import escape
+
+from twisted.python import components, filepath, log
+from twisted.internet import abstract, interfaces
+from twisted.python.util import InsensitiveDict
+from twisted.python.runtime import platformType
+from twisted.python.url import URL
+from incremental import Version
+from twisted.python.deprecate import deprecated
+
+if _PY3:
+    from urllib.parse import quote, unquote
+else:
+    from urllib import quote, unquote
+
+dangerousPathError = resource.NoResource("Invalid request URL.")
+
+def isDangerous(path):
+    return path == b'..' or b'/' in path or networkString(os.sep) in path
+
+
+class Data(resource.Resource):
+    """
+    This is a static, in-memory resource.
+    """
+
+    def __init__(self, data, type):
+        """
+        @param data: The bytes that make up this data resource.
+        @type data: L{bytes}
+
+        @param type: A native string giving the Internet media type for this
+            content.
+        @type type: L{str}
+        """
+        resource.Resource.__init__(self)
+        self.data = data
+        self.type = type
+
+
+    def render_GET(self, request):
+        request.setHeader(b"content-type", networkString(self.type))
+        request.setHeader(b"content-length", intToBytes(len(self.data)))
+        if request.method == b"HEAD":
+            return b''
+        return self.data
+    render_HEAD = render_GET
+
+
+
+@deprecated(Version("Twisted", 16, 0, 0))
+def addSlash(request):
+    """
+    Add a trailing slash to C{request}'s URI. Deprecated, do not use.
+    """
+    return _addSlash(request)
+
+
+
+def _addSlash(request):
+    """
+    Add a trailing slash to C{request}'s URI.
+
+    @param request: The incoming request to add the ending slash to.
+    @type request: An object conforming to L{twisted.web.iweb.IRequest}
+
+    @return: A URI with a trailing slash, with query and fragment preserved.
+    @rtype: L{bytes}
+    """
+    url = URL.fromText(request.uri.decode('ascii'))
+    # Add an empty path segment at the end, so that it adds a trailing slash
+    url = url.replace(path=list(url.path) + [u""])
+    return url.asText().encode('ascii')
+
+
+
+class Redirect(resource.Resource):
+    def __init__(self, request):
+        resource.Resource.__init__(self)
+        self.url = _addSlash(request)
+
+    def render(self, request):
+        return redirectTo(self.url, request)
+
+
+class Registry(components.Componentized):
+    """
+    I am a Componentized object that will be made available to internal Twisted
+    file-based dynamic web content such as .rpy and .epy scripts.
+    """
+
+    def __init__(self):
+        components.Componentized.__init__(self)
+        self._pathCache = {}
+
+    def cachePath(self, path, rsrc):
+        self._pathCache[path] = rsrc
+
+    def getCachedPath(self, path):
+        return self._pathCache.get(path)
+
+
+def loadMimeTypes(mimetype_locations=None, init=mimetypes.init):
+    """
+    Produces a mapping of extensions (with leading dot) to MIME types.
+
+    It does this by calling the C{init} function of the L{mimetypes} module.
+    This will have the side effect of modifying the global MIME types cache
+    in that module.
+
+    Multiple file locations containing mime-types can be passed as a list.
+    The files will be sourced in that order, overriding mime-types from the
+    files sourced beforehand, but only if a new entry explicitly overrides
+    the current entry.
+
+    @param mimetype_locations: Optional. List of paths to C{mime.types} style
+        files that should be used.
+    @type mimetype_locations: iterable of paths or L{None}
+    @param init: The init function to call. Defaults to the global C{init}
+        function of the C{mimetypes} module. For internal use (testing) only.
+    @type init: callable
+    """
+    init(mimetype_locations)
+    mimetypes.types_map.update(
+        {
+            '.conf':  'text/plain',
+            '.diff':  'text/plain',
+            '.flac':  'audio/x-flac',
+            '.java':  'text/plain',
+            '.oz':    'text/x-oz',
+            '.swf':   'application/x-shockwave-flash',
+            '.wml':   'text/vnd.wap.wml',
+            '.xul':   'application/vnd.mozilla.xul+xml',
+            '.patch': 'text/plain'
+        }
+    )
+    return mimetypes.types_map
+
+
+def getTypeAndEncoding(filename, types, encodings, defaultType):
+    p, ext = filepath.FilePath(filename).splitext()
+    ext = filepath._coerceToFilesystemEncoding('', ext.lower())
+    if ext in encodings:
+        enc = encodings[ext]
+        ext = os.path.splitext(p)[1].lower()
+    else:
+        enc = None
+    type = types.get(ext, defaultType)
+    return type, enc
+
+
+
+class File(resource.Resource, filepath.FilePath):
+    """
+    File is a resource that represents a plain non-interpreted file
+    (although it can look for an extension like .rpy or .cgi and hand the
+    file to a processor for interpretation if you wish). Its constructor
+    takes a file path.
+
+    Alternatively, you can give a directory path to the constructor. In this
+    case the resource will represent that directory, and its children will
+    be files underneath that directory. This provides access to an entire
+    filesystem tree with a single Resource.
+
+    If you map the URL 'http://server/FILE' to a resource created as
+    File('/tmp'), then http://server/FILE/ will return an HTML-formatted
+    listing of the /tmp/ directory, and http://server/FILE/foo/bar.html will
+    return the contents of /tmp/foo/bar.html .
+
+    @cvar childNotFound: L{Resource} used to render 404 Not Found error pages.
+    @cvar forbidden: L{Resource} used to render 403 Forbidden error pages.
+
+    @ivar contentTypes: a mapping of extensions to MIME types used to set the
+        default value for the Content-Type header.
+        It is initialized with the values returned by L{loadMimeTypes}.
+    @type contentTypes: C{dict}
+
+    @ivar contentEncodings: a mapping of extensions to encoding types used to
+        set default value for the Content-Encoding header.
+    @type contentEncodings: C{dict}
+    """
+
+    contentTypes = loadMimeTypes()
+
+    contentEncodings = {
+        ".gz" : "gzip",
+        ".bz2": "bzip2"
+        }
+
+    processors = {}
+
+    indexNames = ["index", "index.html", "index.htm", "index.rpy"]
+
+    type = None
+
+    def __init__(self, path, defaultType="text/html", ignoredExts=(), registry=None, allowExt=0):
+        """
+        Create a file with the given path.
+
+        @param path: The filename of the file from which this L{File} will
+            serve data.
+        @type path: C{str}
+
+        @param defaultType: A I{major/minor}-style MIME type specifier
+            indicating the I{Content-Type} with which this L{File}'s data
+            will be served if a MIME type cannot be determined based on
+            C{path}'s extension.
+        @type defaultType: C{str}
+
+        @param ignoredExts: A sequence giving the extensions of paths in the
+            filesystem which will be ignored for the purposes of child
+            lookup.  For example, if C{ignoredExts} is C{(".bar",)} and
+            C{path} is a directory containing a file named C{"foo.bar"}, a
+            request for the C{"foo"} child of this resource will succeed
+            with a L{File} pointing to C{"foo.bar"}.
+
+        @param registry: The registry object being used to handle this
+            request.  If L{None}, one will be created.
+        @type registry: L{Registry}
+
+        @param allowExt: Ignored parameter, only present for backwards
+            compatibility.  Do not pass a value for this parameter.
+        """
+        resource.Resource.__init__(self)
+        filepath.FilePath.__init__(self, path)
+        self.defaultType = defaultType
+        if ignoredExts in (0, 1) or allowExt:
+            warnings.warn("ignoredExts should receive a list, not a boolean")
+            if ignoredExts or allowExt:
+                self.ignoredExts = ['*']
+            else:
+                self.ignoredExts = []
+        else:
+            self.ignoredExts = list(ignoredExts)
+        self.registry = registry or Registry()
+
+
+    def ignoreExt(self, ext):
+        """Ignore the given extension.
+
+        Serve file.ext if file is requested
+        """
+        self.ignoredExts.append(ext)
+
+    childNotFound = resource.NoResource("File not found.")
+    forbidden = resource.ForbiddenResource()
+
+
+    def directoryListing(self):
+        """
+        Return a resource that generates an HTML listing of the
+        directory this path represents.
+
+        @return: A resource that renders the directory to HTML.
+        @rtype: L{DirectoryLister}
+        """
+        if _PY3:
+            path = self.path
+            names = self.listNames()
+        else:
+            # DirectoryLister works in terms of native strings, so on
+            # Python 2, ensure we have a bytes paths for this
+            # directory and its contents.  We use the asBytesMode
+            # method inherited from FilePath to ensure consistent
+            # encoding of the actual path.  This returns a FilePath
+            # instance even when called on subclasses, however, so we
+            # have to create a new File instance.
+            nativeStringPath = self.createSimilarFile(self.asBytesMode().path)
+            path = nativeStringPath.path
+            names = nativeStringPath.listNames()
+        return DirectoryLister(path,
+                               names,
+                               self.contentTypes,
+                               self.contentEncodings,
+                               self.defaultType)
+
+
+    def getChild(self, path, request):
+        """
+        If this L{File}"s path refers to a directory, return a L{File}
+        referring to the file named C{path} in that directory.
+
+        If C{path} is the empty string, return a L{DirectoryLister}
+        instead.
+
+        @param path: The current path segment.
+        @type path: L{bytes}
+
+        @param request: The incoming request.
+        @type request: An that provides L{twisted.web.iweb.IRequest}.
+
+        @return: A resource representing the requested file or
+            directory, or L{NoResource} if the path cannot be
+            accessed.
+        @rtype: An object that provides L{resource.IResource}.
+        """
+        if isinstance(path, bytes):
+            try:
+                # Request calls urllib.unquote on each path segment,
+                # leaving us with raw bytes.
+                path = path.decode('utf-8')
+            except UnicodeDecodeError:
+                log.err(None,
+                        "Could not decode path segment as utf-8: %r" % (path,))
+                return self.childNotFound
+
+        self.restat(reraise=False)
+
+        if not self.isdir():
+            return self.childNotFound
+
+        if path:
+            try:
+                fpath = self.child(path)
+            except filepath.InsecurePath:
+                return self.childNotFound
+        else:
+            fpath = self.childSearchPreauth(*self.indexNames)
+            if fpath is None:
+                return self.directoryListing()
+
+        if not fpath.exists():
+            fpath = fpath.siblingExtensionSearch(*self.ignoredExts)
+            if fpath is None:
+                return self.childNotFound
+
+        extension = fpath.splitext()[1]
+        if platformType == "win32":
+            # don't want .RPY to be different than .rpy, since that would allow
+            # source disclosure.
+            processor = InsensitiveDict(self.processors).get(extension)
+        else:
+            processor = self.processors.get(extension)
+        if processor:
+            return resource.IResource(processor(fpath.path, self.registry))
+        return self.createSimilarFile(fpath.path)
+
+
+    # methods to allow subclasses to e.g. decrypt files on the fly:
+    def openForReading(self):
+        """Open a file and return it."""
+        return self.open()
+
+
+    def getFileSize(self):
+        """Return file size."""
+        return self.getsize()
+
+
+    def _parseRangeHeader(self, range):
+        """
+        Parse the value of a Range header into (start, stop) pairs.
+
+        In a given pair, either of start or stop can be None, signifying that
+        no value was provided, but not both.
+
+        @return: A list C{[(start, stop)]} of pairs of length at least one.
+
+        @raise ValueError: if the header is syntactically invalid or if the
+            Bytes-Unit is anything other than "bytes'.
+        """
+        try:
+            kind, value = range.split(b'=', 1)
+        except ValueError:
+            raise ValueError("Missing '=' separator")
+        kind = kind.strip()
+        if kind != b'bytes':
+            raise ValueError("Unsupported Bytes-Unit: %r" % (kind,))
+        unparsedRanges = list(filter(None, map(bytes.strip, value.split(b','))))
+        parsedRanges = []
+        for byteRange in unparsedRanges:
+            try:
+                start, end = byteRange.split(b'-', 1)
+            except ValueError:
+                raise ValueError("Invalid Byte-Range: %r" % (byteRange,))
+            if start:
+                try:
+                    start = int(start)
+                except ValueError:
+                    raise ValueError("Invalid Byte-Range: %r" % (byteRange,))
+            else:
+                start = None
+            if end:
+                try:
+                    end = int(end)
+                except ValueError:
+                    raise ValueError("Invalid Byte-Range: %r" % (byteRange,))
+            else:
+                end = None
+            if start is not None:
+                if end is not None and start > end:
+                    # Start must be less than or equal to end or it is invalid.
+                    raise ValueError("Invalid Byte-Range: %r" % (byteRange,))
+            elif end is None:
+                # One or both of start and end must be specified.  Omitting
+                # both is invalid.
+                raise ValueError("Invalid Byte-Range: %r" % (byteRange,))
+            parsedRanges.append((start, end))
+        return parsedRanges
+
+
+    def _rangeToOffsetAndSize(self, start, end):
+        """
+        Convert a start and end from a Range header to an offset and size.
+
+        This method checks that the resulting range overlaps with the resource
+        being served (and so has the value of C{getFileSize()} as an indirect
+        input).
+
+        Either but not both of start or end can be L{None}:
+
+         - Omitted start means that the end value is actually a start value
+           relative to the end of the resource.
+
+         - Omitted end means the end of the resource should be the end of
+           the range.
+
+        End is interpreted as inclusive, as per RFC 2616.
+
+        If this range doesn't overlap with any of this resource, C{(0, 0)} is
+        returned, which is not otherwise a value return value.
+
+        @param start: The start value from the header, or L{None} if one was
+            not present.
+        @param end: The end value from the header, or L{None} if one was not
+            present.
+        @return: C{(offset, size)} where offset is how far into this resource
+            this resource the range begins and size is how long the range is,
+            or C{(0, 0)} if the range does not overlap this resource.
+        """
+        size = self.getFileSize()
+        if start is None:
+            start = size - end
+            end = size
+        elif end is None:
+            end = size
+        elif end < size:
+            end += 1
+        elif end > size:
+            end = size
+        if start >= size:
+            start = end = 0
+        return start, (end - start)
+
+
+    def _contentRange(self, offset, size):
+        """
+        Return a string suitable for the value of a Content-Range header for a
+        range with the given offset and size.
+
+        The offset and size are not sanity checked in any way.
+
+        @param offset: How far into this resource the range begins.
+        @param size: How long the range is.
+        @return: The value as appropriate for the value of a Content-Range
+            header.
+        """
+        return networkString('bytes %d-%d/%d' % (
+            offset, offset + size - 1, self.getFileSize()))
+
+
+    def _doSingleRangeRequest(self, request, startAndEnd):
+        """
+        Set up the response for Range headers that specify a single range.
+
+        This method checks if the request is satisfiable and sets the response
+        code and Content-Range header appropriately.  The return value
+        indicates which part of the resource to return.
+
+        @param request: The Request object.
+        @param startAndEnd: A 2-tuple of start of the byte range as specified by
+            the header and the end of the byte range as specified by the header.
+            At most one of the start and end may be L{None}.
+        @return: A 2-tuple of the offset and size of the range to return.
+            offset == size == 0 indicates that the request is not satisfiable.
+        """
+        start, end = startAndEnd
+        offset, size  = self._rangeToOffsetAndSize(start, end)
+        if offset == size == 0:
+            # This range doesn't overlap with any of this resource, so the
+            # request is unsatisfiable.
+            request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
+            request.setHeader(
+                b'content-range', networkString('bytes */%d' % (self.getFileSize(),)))
+        else:
+            request.setResponseCode(http.PARTIAL_CONTENT)
+            request.setHeader(
+                b'content-range', self._contentRange(offset, size))
+        return offset, size
+
+
+    def _doMultipleRangeRequest(self, request, byteRanges):
+        """
+        Set up the response for Range headers that specify a single range.
+
+        This method checks if the request is satisfiable and sets the response
+        code and Content-Type and Content-Length headers appropriately.  The
+        return value, which is a little complicated, indicates which parts of
+        the resource to return and the boundaries that should separate the
+        parts.
+
+        In detail, the return value is a tuple rangeInfo C{rangeInfo} is a
+        list of 3-tuples C{(partSeparator, partOffset, partSize)}.  The
+        response to this request should be, for each element of C{rangeInfo},
+        C{partSeparator} followed by C{partSize} bytes of the resource
+        starting at C{partOffset}.  Each C{partSeparator} includes the
+        MIME-style boundary and the part-specific Content-type and
+        Content-range headers.  It is convenient to return the separator as a
+        concrete string from this method, because this method needs to compute
+        the number of bytes that will make up the response to be able to set
+        the Content-Length header of the response accurately.
+
+        @param request: The Request object.
+        @param byteRanges: A list of C{(start, end)} values as specified by
+            the header.  For each range, at most one of C{start} and C{end}
+            may be L{None}.
+        @return: See above.
+        """
+        matchingRangeFound = False
+        rangeInfo = []
+        contentLength = 0
+        boundary = networkString("%x%x" % (int(time.time()*1000000), os.getpid()))
+        if self.type:
+            contentType = self.type
+        else:
+            contentType = b'bytes' # It's what Apache does...
+        for start, end in byteRanges:
+            partOffset, partSize = self._rangeToOffsetAndSize(start, end)
+            if partOffset == partSize == 0:
+                continue
+            contentLength += partSize
+            matchingRangeFound = True
+            partContentRange = self._contentRange(partOffset, partSize)
+            partSeparator = networkString((
+                "\r\n"
+                "--%s\r\n"
+                "Content-type: %s\r\n"
+                "Content-range: %s\r\n"
+                "\r\n") % (nativeString(boundary), nativeString(contentType), nativeString(partContentRange)))
+            contentLength += len(partSeparator)
+            rangeInfo.append((partSeparator, partOffset, partSize))
+        if not matchingRangeFound:
+            request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
+            request.setHeader(
+                b'content-length', b'0')
+            request.setHeader(
+                b'content-range', networkString('bytes */%d' % (self.getFileSize(),)))
+            return [], b''
+        finalBoundary = b"\r\n--" + boundary + b"--\r\n"
+        rangeInfo.append((finalBoundary, 0, 0))
+        request.setResponseCode(http.PARTIAL_CONTENT)
+        request.setHeader(
+            b'content-type', networkString('multipart/byteranges; boundary="%s"' % (nativeString(boundary),)))
+        request.setHeader(
+            b'content-length', intToBytes(contentLength + len(finalBoundary)))
+        return rangeInfo
+
+
+    def _setContentHeaders(self, request, size=None):
+        """
+        Set the Content-length and Content-type headers for this request.
+
+        This method is not appropriate for requests for multiple byte ranges;
+        L{_doMultipleRangeRequest} will set these headers in that case.
+
+        @param request: The L{twisted.web.http.Request} object.
+        @param size: The size of the response.  If not specified, default to
+            C{self.getFileSize()}.
+        """
+        if size is None:
+            size = self.getFileSize()
+        request.setHeader(b'content-length', intToBytes(size))
+        if self.type:
+            request.setHeader(b'content-type', networkString(self.type))
+        if self.encoding:
+            request.setHeader(b'content-encoding', networkString(self.encoding))
+
+
+    def makeProducer(self, request, fileForReading):
+        """
+        Make a L{StaticProducer} that will produce the body of this response.
+
+        This method will also set the response code and Content-* headers.
+
+        @param request: The L{twisted.web.http.Request} object.
+        @param fileForReading: The file object containing the resource.
+        @return: A L{StaticProducer}.  Calling C{.start()} on this will begin
+            producing the response.
+        """
+        byteRange = request.getHeader(b'range')
+        if byteRange is None:
+            self._setContentHeaders(request)
+            request.setResponseCode(http.OK)
+            return NoRangeStaticProducer(request, fileForReading)
+        try:
+            parsedRanges = self._parseRangeHeader(byteRange)
+        except ValueError:
+            log.msg("Ignoring malformed Range header %r" % (byteRange.decode(),))
+            self._setContentHeaders(request)
+            request.setResponseCode(http.OK)
+            return NoRangeStaticProducer(request, fileForReading)
+
+        if len(parsedRanges) == 1:
+            offset, size = self._doSingleRangeRequest(
+                request, parsedRanges[0])
+            self._setContentHeaders(request, size)
+            return SingleRangeStaticProducer(
+                request, fileForReading, offset, size)
+        else:
+            rangeInfo = self._doMultipleRangeRequest(request, parsedRanges)
+            return MultipleRangeStaticProducer(
+                request, fileForReading, rangeInfo)
+
+
+    def render_GET(self, request):
+        """
+        Begin sending the contents of this L{File} (or a subset of the
+        contents, based on the 'range' header) to the given request.
+        """
+        self.restat(False)
+
+        if self.type is None:
+            self.type, self.encoding = getTypeAndEncoding(self.basename(),
+                                                          self.contentTypes,
+                                                          self.contentEncodings,
+                                                          self.defaultType)
+
+        if not self.exists():
+            return self.childNotFound.render(request)
+
+        if self.isdir():
+            return self.redirect(request)
+
+        request.setHeader(b'accept-ranges', b'bytes')
+
+        try:
+            fileForReading = self.openForReading()
+        except IOError as e:
+            if e.errno == errno.EACCES:
+                return self.forbidden.render(request)
+            else:
+                raise
+
+        if request.setLastModified(self.getModificationTime()) is http.CACHED:
+            # `setLastModified` also sets the response code for us, so if the
+            # request is cached, we close the file now that we've made sure that
+            # the request would otherwise succeed and return an empty body.
+            fileForReading.close()
+            return b''
+
+        if request.method == b'HEAD':
+            # Set the content headers here, rather than making a producer.
+            self._setContentHeaders(request)
+            # We've opened the file to make sure it's accessible, so close it
+            # now that we don't need it.
+            fileForReading.close()
+            return b''
+
+        producer = self.makeProducer(request, fileForReading)
+        producer.start()
+
+        # and make sure the connection doesn't get closed
+        return server.NOT_DONE_YET
+    render_HEAD = render_GET
+
+
+    def redirect(self, request):
+        return redirectTo(_addSlash(request), request)
+
+
+    def listNames(self):
+        if not self.isdir():
+            return []
+        directory = self.listdir()
+        directory.sort()
+        return directory
+
+    def listEntities(self):
+        return list(map(lambda fileName, self=self: self.createSimilarFile(os.path.join(self.path, fileName)), self.listNames()))
+
+
+    def createSimilarFile(self, path):
+        f = self.__class__(path, self.defaultType, self.ignoredExts, self.registry)
+        # refactoring by steps, here - constructor should almost certainly take these
+        f.processors = self.processors
+        f.indexNames = self.indexNames[:]
+        f.childNotFound = self.childNotFound
+        return f
+
+
+
+@implementer(interfaces.IPullProducer)
+class StaticProducer(object):
+    """
+    Superclass for classes that implement the business of producing.
+
+    @ivar request: The L{IRequest} to write the contents of the file to.
+    @ivar fileObject: The file the contents of which to write to the request.
+    """
+
+    bufferSize = abstract.FileDescriptor.bufferSize
+
+
+    def __init__(self, request, fileObject):
+        """
+        Initialize the instance.
+        """
+        self.request = request
+        self.fileObject = fileObject
+
+
+    def start(self):
+        raise NotImplementedError(self.start)
+
+
+    def resumeProducing(self):
+        raise NotImplementedError(self.resumeProducing)
+
+
+    def stopProducing(self):
+        """
+        Stop producing data.
+
+        L{twisted.internet.interfaces.IProducer.stopProducing}
+        is called when our consumer has died, and subclasses also call this
+        method when they are done producing data.
+        """
+        self.fileObject.close()
+        self.request = None
+
+
+
+class NoRangeStaticProducer(StaticProducer):
+    """
+    A L{StaticProducer} that writes the entire file to the request.
+    """
+
+    def start(self):
+        self.request.registerProducer(self, False)
+
+
+    def resumeProducing(self):
+        if not self.request:
+            return
+        data = self.fileObject.read(self.bufferSize)
+        if data:
+            # this .write will spin the reactor, calling .doWrite and then
+            # .resumeProducing again, so be prepared for a re-entrant call
+            self.request.write(data)
+        else:
+            self.request.unregisterProducer()
+            self.request.finish()
+            self.stopProducing()
+
+
+
+class SingleRangeStaticProducer(StaticProducer):
+    """
+    A L{StaticProducer} that writes a single chunk of a file to the request.
+    """
+
+    def __init__(self, request, fileObject, offset, size):
+        """
+        Initialize the instance.
+
+        @param request: See L{StaticProducer}.
+        @param fileObject: See L{StaticProducer}.
+        @param offset: The offset into the file of the chunk to be written.
+        @param size: The size of the chunk to write.
+        """
+        StaticProducer.__init__(self, request, fileObject)
+        self.offset = offset
+        self.size = size
+
+
+    def start(self):
+        self.fileObject.seek(self.offset)
+        self.bytesWritten = 0
+        self.request.registerProducer(self, 0)
+
+
+    def resumeProducing(self):
+        if not self.request:
+            return
+        data = self.fileObject.read(
+            min(self.bufferSize, self.size - self.bytesWritten))
+        if data:
+            self.bytesWritten += len(data)
+            # this .write will spin the reactor, calling .doWrite and then
+            # .resumeProducing again, so be prepared for a re-entrant call
+            self.request.write(data)
+        if self.request and self.bytesWritten == self.size:
+            self.request.unregisterProducer()
+            self.request.finish()
+            self.stopProducing()
+
+
+
+class MultipleRangeStaticProducer(StaticProducer):
+    """
+    A L{StaticProducer} that writes several chunks of a file to the request.
+    """
+
+    def __init__(self, request, fileObject, rangeInfo):
+        """
+        Initialize the instance.
+
+        @param request: See L{StaticProducer}.
+        @param fileObject: See L{StaticProducer}.
+        @param rangeInfo: A list of tuples C{[(boundary, offset, size)]}
+            where:
+             - C{boundary} will be written to the request first.
+             - C{offset} the offset into the file of chunk to write.
+             - C{size} the size of the chunk to write.
+        """
+        StaticProducer.__init__(self, request, fileObject)
+        self.rangeInfo = rangeInfo
+
+
+    def start(self):
+        self.rangeIter = iter(self.rangeInfo)
+        self._nextRange()
+        self.request.registerProducer(self, 0)
+
+
+    def _nextRange(self):
+        self.partBoundary, partOffset, self._partSize = next(self.rangeIter)
+        self._partBytesWritten = 0
+        self.fileObject.seek(partOffset)
+
+
+    def resumeProducing(self):
+        if not self.request:
+            return
+        data = []
+        dataLength = 0
+        done = False
+        while dataLength < self.bufferSize:
+            if self.partBoundary:
+                dataLength += len(self.partBoundary)
+                data.append(self.partBoundary)
+                self.partBoundary = None
+            p = self.fileObject.read(
+                min(self.bufferSize - dataLength,
+                    self._partSize - self._partBytesWritten))
+            self._partBytesWritten += len(p)
+            dataLength += len(p)
+            data.append(p)
+            if self.request and self._partBytesWritten == self._partSize:
+                try:
+                    self._nextRange()
+                except StopIteration:
+                    done = True
+                    break
+        self.request.write(b''.join(data))
+        if done:
+            self.request.unregisterProducer()
+            self.request.finish()
+            self.stopProducing()
+
+
+
+class ASISProcessor(resource.Resource):
+    """
+    Serve files exactly as responses without generating a status-line or any
+    headers.  Inspired by Apache's mod_asis.
+    """
+
+    def __init__(self, path, registry=None):
+        resource.Resource.__init__(self)
+        self.path = path
+        self.registry = registry or Registry()
+
+
+    def render(self, request):
+        request.startedWriting = 1
+        res = File(self.path, registry=self.registry)
+        return res.render(request)
+
+
+
+def formatFileSize(size):
+    """
+    Format the given file size in bytes to human readable format.
+    """
+    if size < 1024:
+        return '%iB' % size
+    elif size < (1024 ** 2):
+        return '%iK' % (size / 1024)
+    elif size < (1024 ** 3):
+        return '%iM' % (size / (1024 ** 2))
+    else:
+        return '%iG' % (size / (1024 ** 3))
+
+
+
+class DirectoryLister(resource.Resource):
+    """
+    Print the content of a directory.
+
+    @ivar template: page template used to render the content of the directory.
+        It must contain the format keys B{header} and B{tableContent}.
+    @type template: C{str}
+
+    @ivar linePattern: template used to render one line in the listing table.
+        It must contain the format keys B{class}, B{href}, B{text}, B{size},
+        B{type} and B{encoding}.
+    @type linePattern: C{str}
+
+    @ivar contentTypes: a mapping of extensions to MIME types used to populate
+        the information of a member of this directory.
+        It is initialized with the value L{File.contentTypes}.
+    @type contentTypes: C{dict}
+
+    @ivar contentEncodings: a mapping of extensions to encoding types.
+        It is initialized with the value L{File.contentEncodings}.
+    @type contentEncodings: C{dict}
+
+    @ivar defaultType: default type used when no mimetype is detected.
+    @type defaultType: C{str}
+
+    @ivar dirs: filtered content of C{path}, if the whole content should not be
+        displayed (default to L{None}, which means the actual content of
+        C{path} is printed).
+    @type dirs: L{None} or C{list}
+
+    @ivar path: directory which content should be listed.
+    @type path: C{str}
+    """
+
+    template = """
+
+%(header)s
+
+
+
+
+
%(header)s

+
+
+    
+        
+            
+            
+            
+            
+        
+    
+    
+%(tableContent)s
+    
+
FilenameSizeContent typeContent encoding

+
+
+
+"""
+
+    linePattern = """
+    %(text)s
+    %(size)s
+    %(type)s
+    %(encoding)s
+
+"""
+
+    def __init__(self, pathname, dirs=None,
+                 contentTypes=File.contentTypes,
+                 contentEncodings=File.contentEncodings,
+                 defaultType='text/html'):
+        resource.Resource.__init__(self)
+        self.contentTypes = contentTypes
+        self.contentEncodings = contentEncodings
+        self.defaultType = defaultType
+        # dirs allows usage of the File to specify what gets listed
+        self.dirs = dirs
+        self.path = pathname
+
+
+    def _getFilesAndDirectories(self, directory):
+        """
+        Helper returning files and directories in given directory listing, with
+        attributes to be used to build a table content with
+        C{self.linePattern}.
+
+        @return: tuple of (directories, files)
+        @rtype: C{tuple} of C{list}
+        """
+        files = []
+        dirs = []
+
+        for path in directory:
+            if _PY3:
+                if isinstance(path, bytes):
+                    path = path.decode("utf8")
+
+            url = quote(path, "/")
+            escapedPath = escape(path)
+            childPath = filepath.FilePath(self.path).child(path)
+
+            if childPath.isdir():
+                dirs.append({'text': escapedPath + "/", 'href': url + "/",
+                             'size': '', 'type': '[Directory]',
+                             'encoding': ''})
+            else:
+                mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
+                                                        self.contentEncodings,
+                                                        self.defaultType)
+                try:
+                    size = childPath.getsize()
+                except OSError:
+                    continue
+                files.append({
+                    'text': escapedPath, "href": url,
+                    'type': '[%s]' % mimetype,
+                    'encoding': (encoding and '[%s]' % encoding or ''),
+                    'size': formatFileSize(size)})
+        return dirs, files
+
+
+    def _buildTableContent(self, elements):
+        """
+        Build a table content using C{self.linePattern} and giving elements odd
+        and even classes.
+        """
+        tableContent = []
+        rowClasses = itertools.cycle(['odd', 'even'])
+        for element, rowClass in zip(elements, rowClasses):
+            element["class"] = rowClass
+            tableContent.append(self.linePattern % element)
+        return tableContent
+
+
+    def render(self, request):
+        """
+        Render a listing of the content of C{self.path}.
+        """
+        request.setHeader(b"content-type", b"text/html; charset=utf-8")
+        if self.dirs is None:
+            directory = os.listdir(self.path)
+            directory.sort()
+        else:
+            directory = self.dirs
+
+        dirs, files = self._getFilesAndDirectories(directory)
+
+        tableContent = "".join(self._buildTableContent(dirs + files))
+
+        header = "Directory listing for %s" % (
+            escape(unquote(nativeString(request.uri))),)
+
+        done = self.template % {"header": header, "tableContent": tableContent}
+        if _PY3:
+            done = done.encode("utf8")
+
+        return done
+
+
+    def __repr__(self):
+        return '' % self.path
+
+    __str__ = __repr__
diff --git a/contrib/python/Twisted/py2/twisted/web/sux.py b/contrib/python/Twisted/py2/twisted/web/sux.py
new file mode 100644
index 00000000000..6d248d3aa19
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/sux.py
@@ -0,0 +1,637 @@
+# -*- test-case-name: twisted.web.test.test_xml -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+*S*mall, *U*ncomplicated *X*ML.
+
+This is a very simple implementation of XML/HTML as a network
+protocol.  It is not at all clever.  Its main features are that it
+does not:
+
+  - support namespaces
+  - mung mnemonic entity references
+  - validate
+  - perform *any* external actions (such as fetching URLs or writing files)
+    under *any* circumstances
+  - has lots and lots of horrible hacks for supporting broken HTML (as an
+    option, they're not on by default).
+"""
+
+from __future__ import print_function
+
+from twisted.internet.protocol import Protocol
+from twisted.python.compat import unicode
+from twisted.python.reflect import prefixedMethodNames
+
+
+
+# Elements of the three-tuples in the state table.
+BEGIN_HANDLER = 0
+DO_HANDLER = 1
+END_HANDLER = 2
+
+identChars = '.-_:'
+lenientIdentChars = identChars + ';+#/%~'
+
+def nop(*args, **kw):
+    "Do nothing."
+
+
+def unionlist(*args):
+    l = []
+    for x in args:
+        l.extend(x)
+    d = dict([(x, 1) for x in l])
+    return d.keys()
+
+
+def zipfndict(*args, **kw):
+    default = kw.get('default', nop)
+    d = {}
+    for key in unionlist(*[fndict.keys() for fndict in args]):
+        d[key] = tuple([x.get(key, default) for x in args])
+    return d
+
+
+def prefixedMethodClassDict(clazz, prefix):
+    return dict([(name, getattr(clazz, prefix + name)) for name in prefixedMethodNames(clazz, prefix)])
+
+
+def prefixedMethodObjDict(obj, prefix):
+    return dict([(name, getattr(obj, prefix + name)) for name in prefixedMethodNames(obj.__class__, prefix)])
+
+
+class ParseError(Exception):
+
+    def __init__(self, filename, line, col, message):
+        self.filename = filename
+        self.line = line
+        self.col = col
+        self.message = message
+
+    def __str__(self):
+       return "%s:%s:%s: %s" % (self.filename, self.line, self.col,
+                                self.message)
+
+class XMLParser(Protocol):
+
+    state = None
+    encodings = None
+    filename = ""
+    beExtremelyLenient = 0
+    _prepend = None
+
+    # _leadingBodyData will sometimes be set before switching to the
+    # 'bodydata' state, when we "accidentally" read a byte of bodydata
+    # in a different state.
+    _leadingBodyData = None
+
+    def connectionMade(self):
+        self.lineno = 1
+        self.colno = 0
+        self.encodings = []
+
+    def saveMark(self):
+        '''Get the line number and column of the last character parsed'''
+        # This gets replaced during dataReceived, restored afterwards
+        return (self.lineno, self.colno)
+
+    def _parseError(self, message):
+        raise ParseError(*((self.filename,)+self.saveMark()+(message,)))
+
+    def _buildStateTable(self):
+        '''Return a dictionary of begin, do, end state function tuples'''
+        # _buildStateTable leaves something to be desired but it does what it
+        # does.. probably slowly, so I'm doing some evil caching so it doesn't
+        # get called more than once per class.
+        stateTable = getattr(self.__class__, '__stateTable', None)
+        if stateTable is None:
+            stateTable = self.__class__.__stateTable = zipfndict(
+                *[prefixedMethodObjDict(self, prefix)
+                  for prefix in ('begin_', 'do_', 'end_')])
+        return stateTable
+
+    def _decode(self, data):
+        if 'UTF-16' in self.encodings or 'UCS-2' in self.encodings:
+            assert not len(data) & 1, 'UTF-16 must come in pairs for now'
+        if self._prepend:
+            data = self._prepend + data
+        for encoding in self.encodings:
+            data = unicode(data, encoding)
+        return data
+
+    def maybeBodyData(self):
+        if self.endtag:
+            return 'bodydata'
+
+        # Get ready for fun! We're going to allow
+        #  to work!
+        # We do this by making everything between  a Text
+        # BUT 
+        # -radix
+
+        if (self.tagName == 'script' and 'src' not in self.tagAttributes):
+            # we do this ourselves rather than having begin_waitforendscript
+            # because that can get called multiple times and we don't want
+            # bodydata to get reset other than the first time.
+            self.begin_bodydata(None)
+            return 'waitforendscript'
+        return 'bodydata'
+
+
+
+    def dataReceived(self, data):
+        stateTable = self._buildStateTable()
+        if not self.state:
+            # all UTF-16 starts with this string
+            if data.startswith((b'\xff\xfe', b'\xfe\xff')):
+                self._prepend = data[0:2]
+                self.encodings.append('UTF-16')
+                data = data[2:]
+            self.state = 'begin'
+        if self.encodings:
+            data = self._decode(data)
+        else:
+            data = data.decode("utf-8")
+        # bring state, lineno, colno into local scope
+        lineno, colno = self.lineno, self.colno
+        curState = self.state
+        # replace saveMark with a nested scope function
+        _saveMark = self.saveMark
+        def saveMark():
+            return (lineno, colno)
+        self.saveMark = saveMark
+        # fetch functions from the stateTable
+        beginFn, doFn, endFn = stateTable[curState]
+        try:
+            for byte in data:
+                # do newline stuff
+                if byte == u'\n':
+                    lineno += 1
+                    colno = 0
+                else:
+                    colno += 1
+                newState = doFn(byte)
+                if newState is not None and newState != curState:
+                    # this is the endFn from the previous state
+                    endFn()
+                    curState = newState
+                    beginFn, doFn, endFn = stateTable[curState]
+                    beginFn(byte)
+        finally:
+            self.saveMark = _saveMark
+            self.lineno, self.colno = lineno, colno
+        # state doesn't make sense if there's an exception..
+        self.state = curState
+
+
+    def connectionLost(self, reason):
+        """
+        End the last state we were in.
+        """
+        stateTable = self._buildStateTable()
+        stateTable[self.state][END_HANDLER]()
+
+
+    # state methods
+
+    def do_begin(self, byte):
+        if byte.isspace():
+            return
+        if byte != '<':
+            if self.beExtremelyLenient:
+                self._leadingBodyData = byte
+                return 'bodydata'
+            self._parseError("First char of document [%r] wasn't <" % (byte,))
+        return 'tagstart'
+
+    def begin_comment(self, byte):
+        self.commentbuf = ''
+
+    def do_comment(self, byte):
+        self.commentbuf += byte
+        if self.commentbuf.endswith('-->'):
+            self.gotComment(self.commentbuf[:-3])
+            return 'bodydata'
+
+    def begin_tagstart(self, byte):
+        self.tagName = ''               # name of the tag
+        self.tagAttributes = {}         # attributes of the tag
+        self.termtag = 0                # is the tag self-terminating
+        self.endtag = 0
+
+    def do_tagstart(self, byte):
+        if byte.isalnum() or byte in identChars:
+            self.tagName += byte
+            if self.tagName == '!--':
+                return 'comment'
+        elif byte.isspace():
+            if self.tagName:
+                if self.endtag:
+                    # properly strict thing to do here is probably to only
+                    # accept whitespace
+                    return 'waitforgt'
+                return 'attrs'
+            else:
+                self._parseError("Whitespace before tag-name")
+        elif byte == '>':
+            if self.endtag:
+                self.gotTagEnd(self.tagName)
+                return 'bodydata'
+            else:
+                self.gotTagStart(self.tagName, {})
+                return (not self.beExtremelyLenient) and 'bodydata' or self.maybeBodyData()
+        elif byte == '/':
+            if self.tagName:
+                return 'afterslash'
+            else:
+                self.endtag = 1
+        elif byte in '!?':
+            if self.tagName:
+                if not self.beExtremelyLenient:
+                    self._parseError("Invalid character in tag-name")
+            else:
+                self.tagName += byte
+                self.termtag = 1
+        elif byte == '[':
+            if self.tagName == '!':
+                return 'expectcdata'
+            else:
+                self._parseError("Invalid '[' in tag-name")
+        else:
+            if self.beExtremelyLenient:
+                self.bodydata = '<'
+                return 'unentity'
+            self._parseError('Invalid tag character: %r'% byte)
+
+    def begin_unentity(self, byte):
+        self.bodydata += byte
+
+    def do_unentity(self, byte):
+        self.bodydata += byte
+        return 'bodydata'
+
+    def end_unentity(self):
+        self.gotText(self.bodydata)
+
+    def begin_expectcdata(self, byte):
+        self.cdatabuf = byte
+
+    def do_expectcdata(self, byte):
+        self.cdatabuf += byte
+        cdb = self.cdatabuf
+        cd = '[CDATA['
+        if len(cd) > len(cdb):
+            if cd.startswith(cdb):
+                return
+            elif self.beExtremelyLenient:
+                ## WHAT THE CRAP!?  MSWord9 generates HTML that includes these
+                ## bizarre   chunks, so I've gotta ignore
+                ## 'em as best I can.  this should really be a separate parse
+                ## state but I don't even have any idea what these _are_.
+                return 'waitforgt'
+            else:
+                self._parseError("Mal-formed CDATA header")
+        if cd == cdb:
+            self.cdatabuf = ''
+            return 'cdata'
+        self._parseError("Mal-formed CDATA header")
+
+    def do_cdata(self, byte):
+        self.cdatabuf += byte
+        if self.cdatabuf.endswith("]]>"):
+            self.cdatabuf = self.cdatabuf[:-3]
+            return 'bodydata'
+
+    def end_cdata(self):
+        self.gotCData(self.cdatabuf)
+        self.cdatabuf = ''
+
+    def do_attrs(self, byte):
+        if byte.isalnum() or byte in identChars:
+            # XXX FIXME really handle !DOCTYPE at some point
+            if self.tagName == '!DOCTYPE':
+                return 'doctype'
+            if self.tagName[0] in '!?':
+                return 'waitforgt'
+            return 'attrname'
+        elif byte.isspace():
+            return
+        elif byte == '>':
+            self.gotTagStart(self.tagName, self.tagAttributes)
+            return (not self.beExtremelyLenient) and 'bodydata' or self.maybeBodyData()
+        elif byte == '/':
+            return 'afterslash'
+        elif self.beExtremelyLenient:
+            # discard and move on?  Only case I've seen of this so far was:
+            # 
+            return
+        self._parseError("Unexpected character: %r" % byte)
+
+    def begin_doctype(self, byte):
+        self.doctype = byte
+
+    def do_doctype(self, byte):
+        if byte == '>':
+            return 'bodydata'
+        self.doctype += byte
+
+    def end_doctype(self):
+        self.gotDoctype(self.doctype)
+        self.doctype = None
+
+    def do_waitforgt(self, byte):
+        if byte == '>':
+            if self.endtag or not self.beExtremelyLenient:
+                return 'bodydata'
+            return self.maybeBodyData()
+
+    def begin_attrname(self, byte):
+        self.attrname = byte
+        self._attrname_termtag = 0
+
+    def do_attrname(self, byte):
+        if byte.isalnum() or byte in identChars:
+            self.attrname += byte
+            return
+        elif byte == '=':
+            return 'beforeattrval'
+        elif byte.isspace():
+            return 'beforeeq'
+        elif self.beExtremelyLenient:
+            if byte in '"\'':
+                return 'attrval'
+            if byte in lenientIdentChars or byte.isalnum():
+                self.attrname += byte
+                return
+            if byte == '/':
+                self._attrname_termtag = 1
+                return
+            if byte == '>':
+                self.attrval = 'True'
+                self.tagAttributes[self.attrname] = self.attrval
+                self.gotTagStart(self.tagName, self.tagAttributes)
+                if self._attrname_termtag:
+                    self.gotTagEnd(self.tagName)
+                    return 'bodydata'
+                return self.maybeBodyData()
+            # something is really broken. let's leave this attribute where it
+            # is and move on to the next thing
+            return
+        self._parseError("Invalid attribute name: %r %r" % (self.attrname, byte))
+
+    def do_beforeattrval(self, byte):
+        if byte in '"\'':
+            return 'attrval'
+        elif byte.isspace():
+            return
+        elif self.beExtremelyLenient:
+            if byte in lenientIdentChars or byte.isalnum():
+                return 'messyattr'
+            if byte == '>':
+                self.attrval = 'True'
+                self.tagAttributes[self.attrname] = self.attrval
+                self.gotTagStart(self.tagName, self.tagAttributes)
+                return self.maybeBodyData()
+            if byte == '\\':
+                # I saw this in actual HTML once:
+                # SM
+                return
+        self._parseError("Invalid initial attribute value: %r; Attribute values must be quoted." % byte)
+
+    attrname = ''
+    attrval = ''
+
+    def begin_beforeeq(self,byte):
+        self._beforeeq_termtag = 0
+
+    def do_beforeeq(self, byte):
+        if byte == '=':
+            return 'beforeattrval'
+        elif byte.isspace():
+            return
+        elif self.beExtremelyLenient:
+            if byte.isalnum() or byte in identChars:
+                self.attrval = 'True'
+                self.tagAttributes[self.attrname] = self.attrval
+                return 'attrname'
+            elif byte == '>':
+                self.attrval = 'True'
+                self.tagAttributes[self.attrname] = self.attrval
+                self.gotTagStart(self.tagName, self.tagAttributes)
+                if self._beforeeq_termtag:
+                    self.gotTagEnd(self.tagName)
+                    return 'bodydata'
+                return self.maybeBodyData()
+            elif byte == '/':
+                self._beforeeq_termtag = 1
+                return
+        self._parseError("Invalid attribute")
+
+    def begin_attrval(self, byte):
+        self.quotetype = byte
+        self.attrval = ''
+
+    def do_attrval(self, byte):
+        if byte == self.quotetype:
+            return 'attrs'
+        self.attrval += byte
+
+    def end_attrval(self):
+        self.tagAttributes[self.attrname] = self.attrval
+        self.attrname = self.attrval = ''
+
+    def begin_messyattr(self, byte):
+        self.attrval = byte
+
+    def do_messyattr(self, byte):
+        if byte.isspace():
+            return 'attrs'
+        elif byte == '>':
+            endTag = 0
+            if self.attrval.endswith('/'):
+                endTag = 1
+                self.attrval = self.attrval[:-1]
+            self.tagAttributes[self.attrname] = self.attrval
+            self.gotTagStart(self.tagName, self.tagAttributes)
+            if endTag:
+                self.gotTagEnd(self.tagName)
+                return 'bodydata'
+            return self.maybeBodyData()
+        else:
+            self.attrval += byte
+
+    def end_messyattr(self):
+        if self.attrval:
+            self.tagAttributes[self.attrname] = self.attrval
+
+    def begin_afterslash(self, byte):
+        self._after_slash_closed = 0
+
+    def do_afterslash(self, byte):
+        # this state is only after a self-terminating slash, e.g. 
+        if self._after_slash_closed:
+            self._parseError("Mal-formed")#XXX When does this happen??
+        if byte != '>':
+            if self.beExtremelyLenient:
+                return
+            else:
+                self._parseError("No data allowed after '/'")
+        self._after_slash_closed = 1
+        self.gotTagStart(self.tagName, self.tagAttributes)
+        self.gotTagEnd(self.tagName)
+        # don't need maybeBodyData here because there better not be
+        # any javascript code after a , we need to
+        # remember all the data we've been through so we can append it
+        # to bodydata
+        self.temptagdata += byte
+
+        # 1
+        if byte == '/':
+            self.endtag = True
+        elif not self.endtag:
+            self.bodydata += "<" + self.temptagdata
+            return 'waitforendscript'
+        # 2
+        elif byte.isalnum() or byte in identChars:
+            self.tagName += byte
+            if not 'script'.startswith(self.tagName):
+                self.bodydata += "<" + self.temptagdata
+                return 'waitforendscript'
+            elif self.tagName == 'script':
+                self.gotText(self.bodydata)
+                self.gotTagEnd(self.tagName)
+                return 'waitforgt'
+        # 3
+        elif byte.isspace():
+            return 'waitscriptendtag'
+        # 4
+        else:
+            self.bodydata += "<" + self.temptagdata
+            return 'waitforendscript'
+
+
+    def begin_entityref(self, byte):
+        self.erefbuf = ''
+        self.erefextra = '' # extra bit for lenient mode
+
+    def do_entityref(self, byte):
+        if byte.isspace() or byte == "<":
+            if self.beExtremelyLenient:
+                # '&foo' probably was '&foo'
+                if self.erefbuf and self.erefbuf != "amp":
+                    self.erefextra = self.erefbuf
+                self.erefbuf = "amp"
+                if byte == "<":
+                    return "tagstart"
+                else:
+                    self.erefextra += byte
+                    return 'spacebodydata'
+            self._parseError("Bad entity reference")
+        elif byte != ';':
+            self.erefbuf += byte
+        else:
+            return 'bodydata'
+
+    def end_entityref(self):
+        self.gotEntityReference(self.erefbuf)
+
+    # hacky support for space after & in entityref in beExtremelyLenient
+    # state should only happen in that case
+    def begin_spacebodydata(self, byte):
+        self.bodydata = self.erefextra
+        self.erefextra = None
+    do_spacebodydata = do_bodydata
+    end_spacebodydata = end_bodydata
+
+    # Sorta SAX-ish API
+
+    def gotTagStart(self, name, attributes):
+        '''Encountered an opening tag.
+
+        Default behaviour is to print.'''
+        print('begin', name, attributes)
+
+    def gotText(self, data):
+        '''Encountered text
+
+        Default behaviour is to print.'''
+        print('text:', repr(data))
+
+    def gotEntityReference(self, entityRef):
+        '''Encountered mnemonic entity reference
+
+        Default behaviour is to print.'''
+        print('entityRef: &%s;' % entityRef)
+
+    def gotComment(self, comment):
+        '''Encountered comment.
+
+        Default behaviour is to ignore.'''
+        pass
+
+    def gotCData(self, cdata):
+        '''Encountered CDATA
+
+        Default behaviour is to call the gotText method'''
+        self.gotText(cdata)
+
+    def gotDoctype(self, doctype):
+        """Encountered DOCTYPE
+
+        This is really grotty: it basically just gives you everything between
+        '' as an argument.
+        """
+        print('!DOCTYPE', repr(doctype))
+
+    def gotTagEnd(self, name):
+        '''Encountered closing tag
+
+        Default behaviour is to print.'''
+        print('end', name)
diff --git a/contrib/python/Twisted/py2/twisted/web/tap.py b/contrib/python/Twisted/py2/twisted/web/tap.py
new file mode 100644
index 00000000000..23df64a4f44
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/tap.py
@@ -0,0 +1,316 @@
+# -*- test-case-name: twisted.web.test.test_tap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support for creating a service which runs a web server.
+"""
+
+from __future__ import absolute_import, division
+
+import os
+import warnings
+
+import incremental
+
+from twisted.application import service, strports
+from twisted.internet import interfaces, reactor
+from twisted.python import usage, reflect, threadpool, deprecate
+from twisted.spread import pb
+from twisted.web import distrib
+from twisted.web import resource, server, static, script, demo, wsgi
+from twisted.web import twcgi
+
+class Options(usage.Options):
+    """
+    Define the options accepted by the I{twistd web} plugin.
+    """
+    synopsis = "[web options]"
+
+    optParameters = [["logfile", "l", None,
+                      "Path to web CLF (Combined Log Format) log file."],
+                     ["certificate", "c", "server.pem",
+                      "(DEPRECATED: use --listen) "
+                      "SSL certificate to use for HTTPS. "],
+                     ["privkey", "k", "server.pem",
+                      "(DEPRECATED: use --listen) "
+                      "SSL certificate to use for HTTPS."],
+                     ]
+
+    optFlags = [
+        ["notracebacks", "n", (
+            "(DEPRECATED: Tracebacks are disabled by default. "
+            "See --enable-tracebacks to turn them on.")],
+        ["display-tracebacks", "", (
+            "Show uncaught exceptions during rendering tracebacks to "
+            "the client. WARNING: This may be a security risk and "
+            "expose private data!")],
+    ]
+
+    optFlags.append([
+        "personal", "",
+        "Instead of generating a webserver, generate a "
+        "ResourcePublisher which listens on  the port given by "
+        "--listen, or ~/%s " % (distrib.UserDirectory.userSocketName,) +
+        "if --listen is not specified."])
+
+    compData = usage.Completions(
+                   optActions={"logfile" : usage.CompleteFiles("*.log"),
+                               "certificate" : usage.CompleteFiles("*.pem"),
+                               "privkey" : usage.CompleteFiles("*.pem")}
+                   )
+
+    longdesc = """\
+This starts a webserver.  If you specify no arguments, it will be a
+demo webserver that has the Test class from twisted.web.demo in it."""
+
+    def __init__(self):
+        usage.Options.__init__(self)
+        self['indexes'] = []
+        self['root'] = None
+        self['extraHeaders'] = []
+        self['ports'] = []
+        self['port'] = self['https'] = None
+
+
+    def opt_port(self, port):
+        """
+        (DEPRECATED: use --listen)
+        Strports description of port to start the server on
+        """
+        msg = deprecate.getDeprecationWarningString(
+            self.opt_port, incremental.Version('Twisted', 18, 4, 0))
+        warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+        self['port'] = port
+
+    opt_p = opt_port
+
+    def opt_https(self, port):
+        """
+        (DEPRECATED: use --listen)
+        Port to listen on for Secure HTTP.
+        """
+        msg = deprecate.getDeprecationWarningString(
+            self.opt_https, incremental.Version('Twisted', 18, 4, 0))
+        warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+        self['https'] = port
+
+
+    def opt_listen(self, port):
+        """
+        Add an strports description of port to start the server on.
+        [default: tcp:8080]
+        """
+        self['ports'].append(port)
+
+
+    def opt_index(self, indexName):
+        """
+        Add the name of a file used to check for directory indexes.
+        [default: index, index.html]
+        """
+        self['indexes'].append(indexName)
+
+    opt_i = opt_index
+
+
+    def opt_user(self):
+        """
+        Makes a server with ~/public_html and ~/.twistd-web-pb support for
+        users.
+        """
+        self['root'] = distrib.UserDirectory()
+
+    opt_u = opt_user
+
+
+    def opt_path(self, path):
+        """
+         is either a specific file or a directory to be set as the root
+        of the web server. Use this if you have a directory full of HTML, cgi,
+        epy, or rpy files or any other files that you want to be served up raw.
+        """
+        self['root'] = static.File(os.path.abspath(path))
+        self['root'].processors = {
+            '.epy': script.PythonScript,
+            '.rpy': script.ResourceScript,
+        }
+        self['root'].processors['.cgi'] = twcgi.CGIScript
+
+
+    def opt_processor(self, proc):
+        """
+        `ext=class' where `class' is added as a Processor for files ending
+        with `ext'.
+        """
+        if not isinstance(self['root'], static.File):
+            raise usage.UsageError(
+                "You can only use --processor after --path.")
+        ext, klass = proc.split('=', 1)
+        self['root'].processors[ext] = reflect.namedClass(klass)
+
+
+    def opt_class(self, className):
+        """
+        Create a Resource subclass with a zero-argument constructor.
+        """
+        classObj = reflect.namedClass(className)
+        self['root'] = classObj()
+
+
+    def opt_resource_script(self, name):
+        """
+        An .rpy file to be used as the root resource of the webserver.
+        """
+        self['root'] = script.ResourceScriptWrapper(name)
+
+
+    def opt_wsgi(self, name):
+        """
+        The FQPN of a WSGI application object to serve as the root resource of
+        the webserver.
+        """
+        try:
+            application = reflect.namedAny(name)
+        except (AttributeError, ValueError):
+            raise usage.UsageError("No such WSGI application: %r" % (name,))
+        pool = threadpool.ThreadPool()
+        reactor.callWhenRunning(pool.start)
+        reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
+        self['root'] = wsgi.WSGIResource(reactor, pool, application)
+
+
+    def opt_mime_type(self, defaultType):
+        """
+        Specify the default mime-type for static files.
+        """
+        if not isinstance(self['root'], static.File):
+            raise usage.UsageError(
+                "You can only use --mime_type after --path.")
+        self['root'].defaultType = defaultType
+    opt_m = opt_mime_type
+
+
+    def opt_allow_ignore_ext(self):
+        """
+        Specify whether or not a request for 'foo' should return 'foo.ext'
+        """
+        if not isinstance(self['root'], static.File):
+            raise usage.UsageError("You can only use --allow_ignore_ext "
+                                   "after --path.")
+        self['root'].ignoreExt('*')
+
+
+    def opt_ignore_ext(self, ext):
+        """
+        Specify an extension to ignore.  These will be processed in order.
+        """
+        if not isinstance(self['root'], static.File):
+            raise usage.UsageError("You can only use --ignore_ext "
+                                   "after --path.")
+        self['root'].ignoreExt(ext)
+
+
+    def opt_add_header(self, header):
+        """
+        Specify an additional header to be included in all responses. Specified
+        as "HeaderName: HeaderValue".
+        """
+        name, value = header.split(':', 1)
+        self['extraHeaders'].append((name.strip(), value.strip()))
+
+
+    def postOptions(self):
+        """
+        Set up conditional defaults and check for dependencies.
+
+        If SSL is not available but an HTTPS server was configured, raise a
+        L{UsageError} indicating that this is not possible.
+
+        If no server port was supplied, select a default appropriate for the
+        other options supplied.
+        """
+        if self['port'] is not None:
+            self['ports'].append(self['port'])
+        if self['https'] is not None:
+            try:
+                reflect.namedModule('OpenSSL.SSL')
+            except ImportError:
+                raise usage.UsageError("SSL support not installed")
+            sslStrport = 'ssl:port={}:privateKey={}:certKey={}'.format(
+                             self['https'],
+                             self['privkey'],
+                             self['certificate'],
+                         )
+            self['ports'].append(sslStrport)
+        if len(self['ports']) == 0:
+            if self['personal']:
+                path = os.path.expanduser(
+                    os.path.join('~', distrib.UserDirectory.userSocketName))
+                self['ports'].append('unix:' + path)
+            else:
+                self['ports'].append('tcp:8080')
+
+
+
+def makePersonalServerFactory(site):
+    """
+    Create and return a factory which will respond to I{distrib} requests
+    against the given site.
+
+    @type site: L{twisted.web.server.Site}
+    @rtype: L{twisted.internet.protocol.Factory}
+    """
+    return pb.PBServerFactory(distrib.ResourcePublisher(site))
+
+
+
+class _AddHeadersResource(resource.Resource):
+    def __init__(self, originalResource, headers):
+        self._originalResource = originalResource
+        self._headers = headers
+
+
+    def getChildWithDefault(self, name, request):
+        for k, v in self._headers:
+            request.responseHeaders.addRawHeader(k, v)
+        return self._originalResource.getChildWithDefault(name, request)
+
+
+
+def makeService(config):
+    s = service.MultiService()
+    if config['root']:
+        root = config['root']
+        if config['indexes']:
+            config['root'].indexNames = config['indexes']
+    else:
+        # This really ought to be web.Admin or something
+        root = demo.Test()
+
+    if isinstance(root, static.File):
+        root.registry.setComponent(interfaces.IServiceCollection, s)
+
+    if config['extraHeaders']:
+        root = _AddHeadersResource(root, config['extraHeaders'])
+
+    if config['logfile']:
+        site = server.Site(root, logPath=config['logfile'])
+    else:
+        site = server.Site(root)
+
+    if config["display-tracebacks"]:
+        site.displayTracebacks = True
+
+    # Deprecate --notracebacks/-n
+    if config["notracebacks"]:
+        msg = deprecate._getDeprecationWarningString(
+            "--notracebacks", incremental.Version('Twisted', 19, 7, 0))
+        warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+
+    if config['personal']:
+        site = makePersonalServerFactory(site)
+    for port in config['ports']:
+        svc = strports.service(port, site)
+        svc.setServiceParent(s)
+    return s
diff --git a/contrib/python/Twisted/py2/twisted/web/template.py b/contrib/python/Twisted/py2/twisted/web/template.py
new file mode 100644
index 00000000000..1c7c9155646
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/template.py
@@ -0,0 +1,575 @@
+# -*- test-case-name: twisted.web.test.test_template -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HTML rendering for twisted.web.
+
+@var VALID_HTML_TAG_NAMES: A list of recognized HTML tag names, used by the
+    L{tag} object.
+
+@var TEMPLATE_NAMESPACE: The XML namespace used to identify attributes and
+    elements used by the templating system, which should be removed from the
+    final output document.
+
+@var tags: A convenience object which can produce L{Tag} objects on demand via
+    attribute access.  For example: C{tags.div} is equivalent to C{Tag("div")}.
+    Tags not specified in L{VALID_HTML_TAG_NAMES} will result in an
+    L{AttributeError}.
+"""
+
+from __future__ import division, absolute_import
+
+__all__ = [
+    'TEMPLATE_NAMESPACE', 'VALID_HTML_TAG_NAMES', 'Element', 'TagLoader',
+    'XMLString', 'XMLFile', 'renderer', 'flatten', 'flattenString', 'tags',
+    'Comment', 'CDATA', 'Tag', 'slot', 'CharRef', 'renderElement'
+    ]
+
+import warnings
+
+from collections import OrderedDict
+
+from zope.interface import implementer
+
+from xml.sax import make_parser, handler
+
+from twisted.python.compat import NativeStringIO, items
+from twisted.python.filepath import FilePath
+from twisted.web._stan import Tag, slot, Comment, CDATA, CharRef
+from twisted.web.iweb import ITemplateLoader
+from twisted.logger import Logger
+
+TEMPLATE_NAMESPACE = 'http://twistedmatrix.com/ns/twisted.web.template/0.1'
+
+# Go read the definition of NOT_DONE_YET. For lulz. This is totally
+# equivalent. And this turns out to be necessary, because trying to import
+# NOT_DONE_YET in this module causes a circular import which we cannot escape
+# from. From which we cannot escape. Etc. glyph is okay with this solution for
+# now, and so am I, as long as this comment stays to explain to future
+# maintainers what it means. ~ C.
+#
+# See http://twistedmatrix.com/trac/ticket/5557 for progress on fixing this.
+NOT_DONE_YET = 1
+_moduleLog = Logger()
+
+
+class _NSContext(object):
+    """
+    A mapping from XML namespaces onto their prefixes in the document.
+    """
+
+    def __init__(self, parent=None):
+        """
+        Pull out the parent's namespaces, if there's no parent then default to
+        XML.
+        """
+        self.parent = parent
+        if parent is not None:
+            self.nss = OrderedDict(parent.nss)
+        else:
+            self.nss = {'http://www.w3.org/XML/1998/namespace':'xml'}
+
+
+    def get(self, k, d=None):
+        """
+        Get a prefix for a namespace.
+
+        @param d: The default prefix value.
+        """
+        return self.nss.get(k, d)
+
+
+    def __setitem__(self, k, v):
+        """
+        Proxy through to setting the prefix for the namespace.
+        """
+        self.nss.__setitem__(k, v)
+
+
+    def __getitem__(self, k):
+        """
+        Proxy through to getting the prefix for the namespace.
+        """
+        return self.nss.__getitem__(k)
+
+
+
+class _ToStan(handler.ContentHandler, handler.EntityResolver):
+    """
+    A SAX parser which converts an XML document to the Twisted STAN
+    Document Object Model.
+    """
+
+    def __init__(self, sourceFilename):
+        """
+        @param sourceFilename: the filename to load the XML out of.
+        """
+        self.sourceFilename = sourceFilename
+        self.prefixMap = _NSContext()
+        self.inCDATA = False
+
+
+    def setDocumentLocator(self, locator):
+        """
+        Set the document locator, which knows about line and character numbers.
+        """
+        self.locator = locator
+
+
+    def startDocument(self):
+        """
+        Initialise the document.
+        """
+        self.document = []
+        self.current = self.document
+        self.stack = []
+        self.xmlnsAttrs = []
+
+
+    def endDocument(self):
+        """
+        Document ended.
+        """
+
+
+    def processingInstruction(self, target, data):
+        """
+        Processing instructions are ignored.
+        """
+
+
+    def startPrefixMapping(self, prefix, uri):
+        """
+        Set up the prefix mapping, which maps fully qualified namespace URIs
+        onto namespace prefixes.
+
+        This gets called before startElementNS whenever an C{xmlns} attribute
+        is seen.
+        """
+
+        self.prefixMap = _NSContext(self.prefixMap)
+        self.prefixMap[uri] = prefix
+
+        # Ignore the template namespace; we'll replace those during parsing.
+        if uri == TEMPLATE_NAMESPACE:
+            return
+
+        # Add to a list that will be applied once we have the element.
+        if prefix is None:
+            self.xmlnsAttrs.append(('xmlns',uri))
+        else:
+            self.xmlnsAttrs.append(('xmlns:%s'%prefix,uri))
+
+
+    def endPrefixMapping(self, prefix):
+        """
+        "Pops the stack" on the prefix mapping.
+
+        Gets called after endElementNS.
+        """
+        self.prefixMap = self.prefixMap.parent
+
+
+    def startElementNS(self, namespaceAndName, qname, attrs):
+        """
+        Gets called when we encounter a new xmlns attribute.
+
+        @param namespaceAndName: a (namespace, name) tuple, where name
+            determines which type of action to take, if the namespace matches
+            L{TEMPLATE_NAMESPACE}.
+        @param qname: ignored.
+        @param attrs: attributes on the element being started.
+        """
+
+        filename = self.sourceFilename
+        lineNumber = self.locator.getLineNumber()
+        columnNumber = self.locator.getColumnNumber()
+
+        ns, name = namespaceAndName
+        if ns == TEMPLATE_NAMESPACE:
+            if name == 'transparent':
+                name = ''
+            elif name == 'slot':
+                try:
+                    # Try to get the default value for the slot
+                    default = attrs[(None, 'default')]
+                except KeyError:
+                    # If there wasn't one, then use None to indicate no
+                    # default.
+                    default = None
+                el = slot(
+                    attrs[(None, 'name')], default=default,
+                    filename=filename, lineNumber=lineNumber,
+                    columnNumber=columnNumber)
+                self.stack.append(el)
+                self.current.append(el)
+                self.current = el.children
+                return
+
+        render = None
+
+        attrs = OrderedDict(attrs)
+        for k, v in items(attrs):
+            attrNS, justTheName = k
+            if attrNS != TEMPLATE_NAMESPACE:
+                continue
+            if justTheName == 'render':
+                render = v
+                del attrs[k]
+
+        # nonTemplateAttrs is a dictionary mapping attributes that are *not* in
+        # TEMPLATE_NAMESPACE to their values.  Those in TEMPLATE_NAMESPACE were
+        # just removed from 'attrs' in the loop immediately above.  The key in
+        # nonTemplateAttrs is either simply the attribute name (if it was not
+        # specified as having a namespace in the template) or prefix:name,
+        # preserving the xml namespace prefix given in the document.
+
+        nonTemplateAttrs = OrderedDict()
+        for (attrNs, attrName), v in items(attrs):
+            nsPrefix = self.prefixMap.get(attrNs)
+            if nsPrefix is None:
+                attrKey = attrName
+            else:
+                attrKey = '%s:%s' % (nsPrefix, attrName)
+            nonTemplateAttrs[attrKey] = v
+
+        if ns == TEMPLATE_NAMESPACE and name == 'attr':
+            if not self.stack:
+                # TODO: define a better exception for this?
+                raise AssertionError(
+                    '<{%s}attr> as top-level element' % (TEMPLATE_NAMESPACE,))
+            if 'name' not in nonTemplateAttrs:
+                # TODO: same here
+                raise AssertionError(
+                    '<{%s}attr> requires a name attribute' % (TEMPLATE_NAMESPACE,))
+            el = Tag('', render=render, filename=filename,
+                     lineNumber=lineNumber, columnNumber=columnNumber)
+            self.stack[-1].attributes[nonTemplateAttrs['name']] = el
+            self.stack.append(el)
+            self.current = el.children
+            return
+
+        # Apply any xmlns attributes
+        if self.xmlnsAttrs:
+            nonTemplateAttrs.update(OrderedDict(self.xmlnsAttrs))
+            self.xmlnsAttrs = []
+
+        # Add the prefix that was used in the parsed template for non-template
+        # namespaces (which will not be consumed anyway).
+        if ns != TEMPLATE_NAMESPACE and ns is not None:
+            prefix = self.prefixMap[ns]
+            if prefix is not None:
+                name = '%s:%s' % (self.prefixMap[ns],name)
+        el = Tag(
+            name, attributes=OrderedDict(nonTemplateAttrs), render=render,
+            filename=filename, lineNumber=lineNumber,
+            columnNumber=columnNumber)
+        self.stack.append(el)
+        self.current.append(el)
+        self.current = el.children
+
+
+    def characters(self, ch):
+        """
+        Called when we receive some characters.  CDATA characters get passed
+        through as is.
+
+        @type ch: C{string}
+        """
+        if self.inCDATA:
+            self.stack[-1].append(ch)
+            return
+        self.current.append(ch)
+
+
+    def endElementNS(self, name, qname):
+        """
+        A namespace tag is closed.  Pop the stack, if there's anything left in
+        it, otherwise return to the document's namespace.
+        """
+        self.stack.pop()
+        if self.stack:
+            self.current = self.stack[-1].children
+        else:
+            self.current = self.document
+
+
+    def startDTD(self, name, publicId, systemId):
+        """
+        DTDs are ignored.
+        """
+
+
+    def endDTD(self, *args):
+        """
+        DTDs are ignored.
+        """
+
+
+    def startCDATA(self):
+        """
+        We're starting to be in a CDATA element, make a note of this.
+        """
+        self.inCDATA = True
+        self.stack.append([])
+
+
+    def endCDATA(self):
+        """
+        We're no longer in a CDATA element.  Collect up the characters we've
+        parsed and put them in a new CDATA object.
+        """
+        self.inCDATA = False
+        comment = ''.join(self.stack.pop())
+        self.current.append(CDATA(comment))
+
+
+    def comment(self, content):
+        """
+        Add an XML comment which we've encountered.
+        """
+        self.current.append(Comment(content))
+
+
+
+def _flatsaxParse(fl):
+    """
+    Perform a SAX parse of an XML document with the _ToStan class.
+
+    @param fl: The XML document to be parsed.
+    @type fl: A file object or filename.
+
+    @return: a C{list} of Stan objects.
+    """
+    parser = make_parser()
+    parser.setFeature(handler.feature_validation, 0)
+    parser.setFeature(handler.feature_namespaces, 1)
+    parser.setFeature(handler.feature_external_ges, 0)
+    parser.setFeature(handler.feature_external_pes, 0)
+
+    s = _ToStan(getattr(fl, "name", None))
+    parser.setContentHandler(s)
+    parser.setEntityResolver(s)
+    parser.setProperty(handler.property_lexical_handler, s)
+
+    parser.parse(fl)
+
+    return s.document
+
+
+@implementer(ITemplateLoader)
+class TagLoader(object):
+    """
+    An L{ITemplateLoader} that loads existing L{IRenderable} providers.
+
+    @ivar tag: The object which will be loaded.
+    @type tag: An L{IRenderable} provider.
+    """
+
+    def __init__(self, tag):
+        """
+        @param tag: The object which will be loaded.
+        @type tag: An L{IRenderable} provider.
+        """
+        self.tag = tag
+
+
+    def load(self):
+        return [self.tag]
+
+
+
+@implementer(ITemplateLoader)
+class XMLString(object):
+    """
+    An L{ITemplateLoader} that loads and parses XML from a string.
+
+    @ivar _loadedTemplate: The loaded document.
+    @type _loadedTemplate: a C{list} of Stan objects.
+    """
+
+    def __init__(self, s):
+        """
+        Run the parser on a L{NativeStringIO} copy of the string.
+
+        @param s: The string from which to load the XML.
+        @type s: C{str}, or a UTF-8 encoded L{bytes}.
+        """
+        if not isinstance(s, str):
+            s = s.decode('utf8')
+
+        self._loadedTemplate = _flatsaxParse(NativeStringIO(s))
+
+
+    def load(self):
+        """
+        Return the document.
+
+        @return: the loaded document.
+        @rtype: a C{list} of Stan objects.
+        """
+        return self._loadedTemplate
+
+
+
+@implementer(ITemplateLoader)
+class XMLFile(object):
+    """
+    An L{ITemplateLoader} that loads and parses XML from a file.
+
+    @ivar _loadedTemplate: The loaded document, or L{None}, if not loaded.
+    @type _loadedTemplate: a C{list} of Stan objects, or L{None}.
+
+    @ivar _path: The L{FilePath}, file object, or filename that is being
+        loaded from.
+    """
+
+    def __init__(self, path):
+        """
+        Run the parser on a file.
+
+        @param path: The file from which to load the XML.
+        @type path: L{FilePath}
+        """
+        if not isinstance(path, FilePath):
+            warnings.warn(
+                "Passing filenames or file objects to XMLFile is deprecated "
+                "since Twisted 12.1.  Pass a FilePath instead.",
+                category=DeprecationWarning, stacklevel=2)
+        self._loadedTemplate = None
+        self._path = path
+
+
+    def _loadDoc(self):
+        """
+        Read and parse the XML.
+
+        @return: the loaded document.
+        @rtype: a C{list} of Stan objects.
+        """
+        if not isinstance(self._path, FilePath):
+            return _flatsaxParse(self._path)
+        else:
+            with self._path.open('r') as f:
+                return _flatsaxParse(f)
+
+
+    def __repr__(self):
+        return '' % (self._path,)
+
+
+    def load(self):
+        """
+        Return the document, first loading it if necessary.
+
+        @return: the loaded document.
+        @rtype: a C{list} of Stan objects.
+        """
+        if self._loadedTemplate is None:
+            self._loadedTemplate = self._loadDoc()
+        return self._loadedTemplate
+
+
+
+# Last updated October 2011, using W3Schools as a reference. Link:
+# http://www.w3schools.com/html5/html5_reference.asp
+# Note that  is explicitly omitted; its semantics do not work with
+# t.w.template and it is officially deprecated.
+VALID_HTML_TAG_NAMES = set([
+    'a', 'abbr', 'acronym', 'address', 'applet', 'area', 'article', 'aside',
+    'audio', 'b', 'base', 'basefont', 'bdi', 'bdo', 'big', 'blockquote',
+    'body', 'br', 'button', 'canvas', 'caption', 'center', 'cite', 'code',
+    'col', 'colgroup', 'command', 'datalist', 'dd', 'del', 'details', 'dfn',
+    'dir', 'div', 'dl', 'dt', 'em', 'embed', 'fieldset', 'figcaption',
+    'figure', 'font', 'footer', 'form', 'frame', 'frameset', 'h1', 'h2', 'h3',
+    'h4', 'h5', 'h6', 'head', 'header', 'hgroup', 'hr', 'html', 'i', 'iframe',
+    'img', 'input', 'ins', 'isindex', 'keygen', 'kbd', 'label', 'legend',
+    'li', 'link', 'map', 'mark', 'menu', 'meta', 'meter', 'nav', 'noframes',
+    'noscript', 'object', 'ol', 'optgroup', 'option', 'output', 'p', 'param',
+    'pre', 'progress', 'q', 'rp', 'rt', 'ruby', 's', 'samp', 'script',
+    'section', 'select', 'small', 'source', 'span', 'strike', 'strong',
+    'style', 'sub', 'summary', 'sup', 'table', 'tbody', 'td', 'textarea',
+    'tfoot', 'th', 'thead', 'time', 'title', 'tr', 'tt', 'u', 'ul', 'var',
+    'video', 'wbr',
+])
+
+
+
+class _TagFactory(object):
+    """
+    A factory for L{Tag} objects; the implementation of the L{tags} object.
+
+    This allows for the syntactic convenience of C{from twisted.web.html import
+    tags; tags.a(href="linked-page.html")}, where 'a' can be basically any HTML
+    tag.
+
+    The class is not exposed publicly because you only ever need one of these,
+    and we already made it for you.
+
+    @see: L{tags}
+    """
+    def __getattr__(self, tagName):
+        if tagName == 'transparent':
+            return Tag('')
+        # allow for E.del as E.del_
+        tagName = tagName.rstrip('_')
+        if tagName not in VALID_HTML_TAG_NAMES:
+            raise AttributeError('unknown tag %r' % (tagName,))
+        return Tag(tagName)
+
+
+
+tags = _TagFactory()
+
+
+
+def renderElement(request, element,
+                  doctype=b'<!DOCTYPE html>', _failElement=None):
+    """
+    Render an element or other C{IRenderable}.
+
+    @param request: The C{Request} being rendered to.
+    @param element: An C{IRenderable} which will be rendered.
+    @param doctype: A C{bytes} which will be written as the first line of
+        the request, or L{None} to disable writing of a doctype.  The C{string}
+        should not include a trailing newline and will default to the HTML5
+        doctype C{'<!DOCTYPE html>'}.
+
+    @returns: NOT_DONE_YET
+
+    @since: 12.1
+    """
+    if doctype is not None:
+        request.write(doctype)
+        request.write(b'\n')
+
+    if _failElement is None:
+        _failElement = twisted.web.util.FailureElement
+
+    d = flatten(request, element, request.write)
+
+    def eb(failure):
+        _moduleLog.failure(
+            "An error occurred while rendering the response.",
+            failure=failure
+        )
+        if request.site.displayTracebacks:
+            return flatten(request, _failElement(failure),
+                           request.write).encode('utf8')
+        else:
+            request.write(
+                (b'<div style="font-size:800%;'
+                 b'background-color:#FFF;'
+                 b'color:#F00'
+                 b'">An error occurred while rendering the response.</div>'))
+
+    d.addErrback(eb)
+    d.addBoth(lambda _: request.finish())
+    return NOT_DONE_YET
+
+
+
+from twisted.web._element import Element, renderer
+from twisted.web._flatten import flatten, flattenString
+import twisted.web.util
diff --git a/contrib/python/Twisted/py2/twisted/web/test/requesthelper.py b/contrib/python/Twisted/py2/twisted/web/test/requesthelper.py
new file mode 100644
index 00000000000..7e16477ce39
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/test/requesthelper.py
@@ -0,0 +1,486 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Helpers related to HTTP requests, used by tests.
+"""
+
+from __future__ import division, absolute_import
+
+__all__ = ['DummyChannel', 'DummyRequest']
+
+from io import BytesIO
+
+from zope.interface import implementer, verify
+
+from twisted.python.compat import intToBytes
+from twisted.python.deprecate import deprecated
+from incremental import Version
+from twisted.internet.defer import Deferred
+from twisted.internet.address import IPv4Address, IPv6Address
+from twisted.internet.interfaces import ISSLTransport, IAddress
+
+from twisted.trial import unittest
+
+from twisted.web.http_headers import Headers
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET, Session, Site
+from twisted.web._responses import FOUND
+
+
+
+textLinearWhitespaceComponents = [
+    u"Foo%sbar" % (lw,) for lw in
+    [u'\r', u'\n', u'\r\n']
+]
+
+sanitizedText = "Foo bar"
+bytesLinearWhitespaceComponents = [
+    component.encode('ascii') for component in
+    textLinearWhitespaceComponents
+]
+sanitizedBytes = sanitizedText.encode('ascii')
+
+
+
+@implementer(IAddress)
+class NullAddress(object):
+    """
+    A null implementation of L{IAddress}.
+    """
+
+
+
+class DummyChannel:
+    class TCP:
+        port = 80
+        disconnected = False
+
+        def __init__(self, peer=None):
+            if peer is None:
+                peer = IPv4Address("TCP", '192.168.1.1', 12344)
+            self._peer = peer
+            self.written = BytesIO()
+            self.producers = []
+
+        def getPeer(self):
+            return self._peer
+
+        def write(self, data):
+            if not isinstance(data, bytes):
+                raise TypeError("Can only write bytes to a transport, not %r" % (data,))
+            self.written.write(data)
+
+        def writeSequence(self, iovec):
+            for data in iovec:
+                self.write(data)
+
+        def getHost(self):
+            return IPv4Address("TCP", '10.0.0.1', self.port)
+
+        def registerProducer(self, producer, streaming):
+            self.producers.append((producer, streaming))
+
+        def unregisterProducer(self):
+            pass
+
+        def loseConnection(self):
+            self.disconnected = True
+
+
+    @implementer(ISSLTransport)
+    class SSL(TCP):
+        pass
+
+    site = Site(Resource())
+
+    def __init__(self, peer=None):
+        self.transport = self.TCP(peer)
+
+
+    def requestDone(self, request):
+        pass
+
+
+    def writeHeaders(self, version, code, reason, headers):
+        response_line = version + b" " + code + b" " + reason + b"\r\n"
+        headerSequence = [response_line]
+        headerSequence.extend(
+            name + b': ' + value + b"\r\n" for name, value in headers
+        )
+        headerSequence.append(b"\r\n")
+        self.transport.writeSequence(headerSequence)
+
+
+    def getPeer(self):
+        return self.transport.getPeer()
+
+
+    def getHost(self):
+        return self.transport.getHost()
+
+
+    def registerProducer(self, producer, streaming):
+        self.transport.registerProducer(producer, streaming)
+
+
+    def unregisterProducer(self):
+        self.transport.unregisterProducer()
+
+
+    def write(self, data):
+        self.transport.write(data)
+
+
+    def writeSequence(self, iovec):
+        self.transport.writeSequence(iovec)
+
+
+    def loseConnection(self):
+        self.transport.loseConnection()
+
+
+    def endRequest(self):
+        pass
+
+
+    def isSecure(self):
+        return isinstance(self.transport, self.SSL)
+
+
+
+class DummyRequest(object):
+    """
+    Represents a dummy or fake request. See L{twisted.web.server.Request}.
+
+    @ivar _finishedDeferreds: L{None} or a C{list} of L{Deferreds} which will
+        be called back with L{None} when C{finish} is called or which will be
+        errbacked if C{processingFailed} is called.
+
+    @type requestheaders: C{Headers}
+    @ivar requestheaders: A Headers instance that stores values for all request
+        headers.
+
+    @type responseHeaders: C{Headers}
+    @ivar responseHeaders: A Headers instance that stores values for all
+        response headers.
+
+    @type responseCode: C{int}
+    @ivar responseCode: The response code which was passed to
+        C{setResponseCode}.
+
+    @type written: C{list} of C{bytes}
+    @ivar written: The bytes which have been written to the request.
+    """
+    uri = b'http://dummy/'
+    method = b'GET'
+    client = None
+
+
+    def registerProducer(self, prod, s):
+        """
+        Call an L{IPullProducer}'s C{resumeProducing} method in a
+        loop until it unregisters itself.
+
+        @param prod: The producer.
+        @type prod: L{IPullProducer}
+
+        @param s: Whether or not the producer is streaming.
+        """
+        # XXX: Handle IPushProducers
+        self.go = 1
+        while self.go:
+            prod.resumeProducing()
+
+
+    def unregisterProducer(self):
+        self.go = 0
+
+
+    def __init__(self, postpath, session=None, client=None):
+        self.sitepath = []
+        self.written = []
+        self.finished = 0
+        self.postpath = postpath
+        self.prepath = []
+        self.session = None
+        self.protoSession = session or Session(0, self)
+        self.args = {}
+        self.requestHeaders = Headers()
+        self.responseHeaders = Headers()
+        self.responseCode = None
+        self._finishedDeferreds = []
+        self._serverName = b"dummy"
+        self.clientproto = b"HTTP/1.0"
+
+
+    def getAllHeaders(self):
+        """
+        Return dictionary mapping the names of all received headers to the last
+        value received for each.
+
+        Since this method does not return all header information,
+        C{self.requestHeaders.getAllRawHeaders()} may be preferred.
+
+        NOTE: This function is a direct copy of
+        C{twisted.web.http.Request.getAllRawHeaders}.
+        """
+        headers = {}
+        for k, v in self.requestHeaders.getAllRawHeaders():
+            headers[k.lower()] = v[-1]
+        return headers
+
+
+    def getHeader(self, name):
+        """
+        Retrieve the value of a request header.
+
+        @type name: C{bytes}
+        @param name: The name of the request header for which to retrieve the
+            value.  Header names are compared case-insensitively.
+
+        @rtype: C{bytes} or L{None}
+        @return: The value of the specified request header.
+        """
+        return self.requestHeaders.getRawHeaders(name.lower(), [None])[0]
+
+
+    def setHeader(self, name, value):
+        """TODO: make this assert on write() if the header is content-length
+        """
+        self.responseHeaders.addRawHeader(name, value)
+
+
+    def getSession(self):
+        if self.session:
+            return self.session
+        assert not self.written, "Session cannot be requested after data has been written."
+        self.session = self.protoSession
+        return self.session
+
+
+    def render(self, resource):
+        """
+        Render the given resource as a response to this request.
+
+        This implementation only handles a few of the most common behaviors of
+        resources.  It can handle a render method that returns a string or
+        C{NOT_DONE_YET}.  It doesn't know anything about the semantics of
+        request methods (eg HEAD) nor how to set any particular headers.
+        Basically, it's largely broken, but sufficient for some tests at least.
+        It should B{not} be expanded to do all the same stuff L{Request} does.
+        Instead, L{DummyRequest} should be phased out and L{Request} (or some
+        other real code factored in a different way) used.
+        """
+        result = resource.render(self)
+        if result is NOT_DONE_YET:
+            return
+        self.write(result)
+        self.finish()
+
+
+    def write(self, data):
+        if not isinstance(data, bytes):
+            raise TypeError("write() only accepts bytes")
+        self.written.append(data)
+
+
+    def notifyFinish(self):
+        """
+        Return a L{Deferred} which is called back with L{None} when the request
+        is finished.  This will probably only work if you haven't called
+        C{finish} yet.
+        """
+        finished = Deferred()
+        self._finishedDeferreds.append(finished)
+        return finished
+
+
+    def finish(self):
+        """
+        Record that the request is finished and callback and L{Deferred}s
+        waiting for notification of this.
+        """
+        self.finished = self.finished + 1
+        if self._finishedDeferreds is not None:
+            observers = self._finishedDeferreds
+            self._finishedDeferreds = None
+            for obs in observers:
+                obs.callback(None)
+
+
+    def processingFailed(self, reason):
+        """
+        Errback and L{Deferreds} waiting for finish notification.
+        """
+        if self._finishedDeferreds is not None:
+            observers = self._finishedDeferreds
+            self._finishedDeferreds = None
+            for obs in observers:
+                obs.errback(reason)
+
+
+    def addArg(self, name, value):
+        self.args[name] = [value]
+
+
+    def setResponseCode(self, code, message=None):
+        """
+        Set the HTTP status response code, but takes care that this is called
+        before any data is written.
+        """
+        assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join(self.written)
+        self.responseCode = code
+        self.responseMessage = message
+
+
+    def setLastModified(self, when):
+        assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join(self.written)
+
+
+    def setETag(self, tag):
+        assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join(self.written)
+
+
+    def getClientIP(self):
+        """
+        Return the IPv4 address of the client which made this request, if there
+        is one, otherwise L{None}.
+        """
+        if isinstance(self.client, (IPv4Address, IPv6Address)):
+            return self.client.host
+        return None
+
+
+    def getClientAddress(self):
+        """
+        Return the L{IAddress} of the client that made this request.
+
+        @return: an address.
+        @rtype: an L{IAddress} provider.
+        """
+        if self.client is None:
+            return NullAddress()
+        return self.client
+
+
+    def getRequestHostname(self):
+        """
+        Get a dummy hostname associated to the HTTP request.
+
+        @rtype: C{bytes}
+        @returns: a dummy hostname
+        """
+        return self._serverName
+
+
+    def getHost(self):
+        """
+        Get a dummy transport's host.
+
+        @rtype: C{IPv4Address}
+        @returns: a dummy transport's host
+        """
+        return IPv4Address('TCP', '127.0.0.1', 80)
+
+
+    def setHost(self, host, port, ssl=0):
+        """
+        Change the host and port the request thinks it's using.
+
+        @type host: C{bytes}
+        @param host: The value to which to change the host header.
+
+        @type ssl: C{bool}
+        @param ssl: A flag which, if C{True}, indicates that the request is
+            considered secure (if C{True}, L{isSecure} will return C{True}).
+        """
+        self._forceSSL = ssl # set first so isSecure will work
+        if self.isSecure():
+            default = 443
+        else:
+            default = 80
+        if port == default:
+            hostHeader = host
+        else:
+            hostHeader = host + b":" + intToBytes(port)
+        self.requestHeaders.addRawHeader(b"host", hostHeader)
+
+
+    def redirect(self, url):
+        """
+        Utility function that does a redirect.
+
+        The request should have finish() called after this.
+        """
+        self.setResponseCode(FOUND)
+        self.setHeader(b"location", url)
+
+
+
+DummyRequest.getClientIP = deprecated(
+    Version('Twisted', 18, 4, 0),
+    replacement="getClientAddress",
+)(DummyRequest.getClientIP)
+
+
+
+class DummyRequestTests(unittest.SynchronousTestCase):
+    """
+    Tests for L{DummyRequest}.
+    """
+
+    def test_getClientIPDeprecated(self):
+        """
+        L{DummyRequest.getClientIP} is deprecated in favor of
+        L{DummyRequest.getClientAddress}
+        """
+
+        request = DummyRequest([])
+        request.getClientIP()
+
+        warnings = self.flushWarnings(
+            offendingFunctions=[self.test_getClientIPDeprecated])
+
+        self.assertEqual(1, len(warnings))
+        [warning] = warnings
+        self.assertEqual(warning.get("category"), DeprecationWarning)
+        self.assertEqual(
+            warning.get("message"),
+            ("twisted.web.test.requesthelper.DummyRequest.getClientIP "
+             "was deprecated in Twisted 18.4.0; "
+             "please use getClientAddress instead"),
+        )
+
+
+    def test_getClientIPSupportsIPv6(self):
+        """
+        L{DummyRequest.getClientIP} supports IPv6 addresses, just like
+        L{twisted.web.http.Request.getClientIP}.
+        """
+        request = DummyRequest([])
+        client = IPv6Address("TCP", "::1", 12345)
+        request.client = client
+
+        self.assertEqual("::1", request.getClientIP())
+
+
+    def test_getClientAddressWithoutClient(self):
+        """
+        L{DummyRequest.getClientAddress} returns an L{IAddress}
+        provider no C{client} has been set.
+        """
+        request = DummyRequest([])
+        null = request.getClientAddress()
+        verify.verifyObject(IAddress, null)
+
+
+    def test_getClientAddress(self):
+        """
+        L{DummyRequest.getClientAddress} returns the C{client}.
+        """
+        request = DummyRequest([])
+        client = IPv4Address("TCP", "127.0.0.1", 12345)
+        request.client = client
+        address = request.getClientAddress()
+        self.assertIs(address, client)
diff --git a/contrib/python/Twisted/py2/twisted/web/twcgi.py b/contrib/python/Twisted/py2/twisted/web/twcgi.py
new file mode 100644
index 00000000000..1c92960cfc3
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/twcgi.py
@@ -0,0 +1,321 @@
+# -*- test-case-name: twisted.web.test.test_cgi -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+I hold resource classes and helper classes that deal with CGI scripts.
+"""
+
+# System Imports
+import os
+import urllib
+
+# Twisted Imports
+from twisted.internet import protocol
+from twisted.logger import Logger
+from twisted.python import filepath
+from twisted.spread import pb
+from twisted.web import http, resource, server, static
+
+
+class CGIDirectory(resource.Resource, filepath.FilePath):
+    def __init__(self, pathname):
+        resource.Resource.__init__(self)
+        filepath.FilePath.__init__(self, pathname)
+
+
+    def getChild(self, path, request):
+        fnp = self.child(path)
+        if not fnp.exists():
+            return static.File.childNotFound
+        elif fnp.isdir():
+            return CGIDirectory(fnp.path)
+        else:
+            return CGIScript(fnp.path)
+        return resource.NoResource()
+
+
+    def render(self, request):
+        notFound = resource.NoResource(
+            "CGI directories do not support directory listing.")
+        return notFound.render(request)
+
+
+
+class CGIScript(resource.Resource):
+    """
+    L{CGIScript} is a resource which runs child processes according to the CGI
+    specification.
+
+    The implementation is complex due to the fact that it requires asynchronous
+    IPC with an external process with an unpleasant protocol.
+    """
+    isLeaf = 1
+    def __init__(self, filename, registry=None, reactor=None):
+        """
+        Initialize, with the name of a CGI script file.
+        """
+        self.filename = filename
+        if reactor is None:
+            # This installs a default reactor, if None was installed before.
+            # We do a late import here, so that importing the current module
+            # won't directly trigger installing a default reactor.
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+
+    def render(self, request):
+        """
+        Do various things to conform to the CGI specification.
+
+        I will set up the usual slew of environment variables, then spin off a
+        process.
+
+        @type request: L{twisted.web.http.Request}
+        @param request: An HTTP request.
+        """
+        scriptName = b"/" + b"/".join(request.prepath)
+        serverName = request.getRequestHostname().split(b':')[0]
+        env = {"SERVER_SOFTWARE":   server.version,
+               "SERVER_NAME":       serverName,
+               "GATEWAY_INTERFACE": "CGI/1.1",
+               "SERVER_PROTOCOL":   request.clientproto,
+               "SERVER_PORT":       str(request.getHost().port),
+               "REQUEST_METHOD":    request.method,
+               "SCRIPT_NAME":       scriptName,
+               "SCRIPT_FILENAME":   self.filename,
+               "REQUEST_URI":       request.uri}
+
+        ip = request.getClientAddress().host
+        if ip is not None:
+            env['REMOTE_ADDR'] = ip
+        pp = request.postpath
+        if pp:
+            env["PATH_INFO"] = "/" + "/".join(pp)
+
+        if hasattr(request, "content"):
+            # 'request.content' is either a StringIO or a TemporaryFile, and
+            # the file pointer is sitting at the beginning (seek(0,0))
+            request.content.seek(0, 2)
+            length = request.content.tell()
+            request.content.seek(0, 0)
+            env['CONTENT_LENGTH'] = str(length)
+
+        try:
+            qindex = request.uri.index(b'?')
+        except ValueError:
+            env['QUERY_STRING'] = ''
+            qargs = []
+        else:
+            qs = env['QUERY_STRING'] = request.uri[qindex+1:]
+            if '=' in qs:
+                qargs = []
+            else:
+                qargs = [urllib.unquote(x) for x in qs.split('+')]
+
+        # Propagate HTTP headers
+        for title, header in request.getAllHeaders().items():
+            envname = title.replace(b'-', b'_').upper()
+            if title not in (b'content-type', b'content-length', b'proxy'):
+                envname = b"HTTP_" + envname
+            env[envname] = header
+        # Propagate our environment
+        for key, value in os.environ.items():
+            if key not in env:
+                env[key] = value
+        # And they're off!
+        self.runProcess(env, request, qargs)
+        return server.NOT_DONE_YET
+
+
+    def runProcess(self, env, request, qargs=[]):
+        """
+        Run the cgi script.
+
+        @type env: A L{dict} of L{str}, or L{None}
+        @param env: The environment variables to pass to the process that will
+            get spawned. See
+            L{twisted.internet.interfaces.IReactorProcess.spawnProcess} for
+            more information about environments and process creation.
+
+        @type request: L{twisted.web.http.Request}
+        @param request: An HTTP request.
+
+        @type qargs: A L{list} of L{str}
+        @param qargs: The command line arguments to pass to the process that
+            will get spawned.
+        """
+        p = CGIProcessProtocol(request)
+        self._reactor.spawnProcess(p, self.filename, [self.filename] + qargs,
+                                   env, os.path.dirname(self.filename))
+
+
+
+class FilteredScript(CGIScript):
+    """
+    I am a special version of a CGI script, that uses a specific executable.
+
+    This is useful for interfacing with other scripting languages that adhere
+    to the CGI standard. My C{filter} attribute specifies what executable to
+    run, and my C{filename} init parameter describes which script to pass to
+    the first argument of that script.
+
+    To customize me for a particular location of a CGI interpreter, override
+    C{filter}.
+
+    @type filter: L{str}
+    @ivar filter: The absolute path to the executable.
+    """
+
+    filter = '/usr/bin/cat'
+
+
+    def runProcess(self, env, request, qargs=[]):
+        """
+        Run a script through the C{filter} executable.
+
+        @type env: A L{dict} of L{str}, or L{None}
+        @param env: The environment variables to pass to the process that will
+            get spawned. See
+            L{twisted.internet.interfaces.IReactorProcess.spawnProcess}
+            for more information about environments and process creation.
+
+        @type request: L{twisted.web.http.Request}
+        @param request: An HTTP request.
+
+        @type qargs: A L{list} of L{str}
+        @param qargs: The command line arguments to pass to the process that
+            will get spawned.
+        """
+        p = CGIProcessProtocol(request)
+        self._reactor.spawnProcess(p, self.filter,
+                                   [self.filter, self.filename] + qargs, env,
+                                   os.path.dirname(self.filename))
+
+
+
+class CGIProcessProtocol(protocol.ProcessProtocol, pb.Viewable):
+    handling_headers = 1
+    headers_written = 0
+    headertext = b''
+    errortext = b''
+    _log = Logger()
+
+    # Remotely relay producer interface.
+
+    def view_resumeProducing(self, issuer):
+        self.resumeProducing()
+
+
+    def view_pauseProducing(self, issuer):
+        self.pauseProducing()
+
+
+    def view_stopProducing(self, issuer):
+        self.stopProducing()
+
+
+    def resumeProducing(self):
+        self.transport.resumeProducing()
+
+
+    def pauseProducing(self):
+        self.transport.pauseProducing()
+
+
+    def stopProducing(self):
+        self.transport.loseConnection()
+
+
+    def __init__(self, request):
+        self.request = request
+
+
+    def connectionMade(self):
+        self.request.registerProducer(self, 1)
+        self.request.content.seek(0, 0)
+        content = self.request.content.read()
+        if content:
+            self.transport.write(content)
+        self.transport.closeStdin()
+
+
+    def errReceived(self, error):
+        self.errortext = self.errortext + error
+
+
+    def outReceived(self, output):
+        """
+        Handle a chunk of input
+        """
+        # First, make sure that the headers from the script are sorted
+        # out (we'll want to do some parsing on these later.)
+        if self.handling_headers:
+            text = self.headertext + output
+            headerEnds = []
+            for delimiter in b'\n\n', b'\r\n\r\n', b'\r\r', b'\n\r\n':
+                headerend = text.find(delimiter)
+                if headerend != -1:
+                    headerEnds.append((headerend, delimiter))
+            if headerEnds:
+                # The script is entirely in control of response headers;
+                # disable the default Content-Type value normally provided by
+                # twisted.web.server.Request.
+                self.request.defaultContentType = None
+
+                headerEnds.sort()
+                headerend, delimiter = headerEnds[0]
+                self.headertext = text[:headerend]
+                # This is a final version of the header text.
+                linebreak = delimiter[:len(delimiter)//2]
+                headers = self.headertext.split(linebreak)
+                for header in headers:
+                    br = header.find(b': ')
+                    if br == -1:
+                        self._log.error(
+                            'ignoring malformed CGI header: {header!r}',
+                            header=header)
+                    else:
+                        headerName = header[:br].lower()
+                        headerText = header[br+2:]
+                        if headerName == b'location':
+                            self.request.setResponseCode(http.FOUND)
+                        if headerName == b'status':
+                            try:
+                                # "XXX <description>" sometimes happens.
+                                statusNum = int(headerText[:3])
+                            except:
+                                self._log.error("malformed status header")
+                            else:
+                                self.request.setResponseCode(statusNum)
+                        else:
+                            # Don't allow the application to control
+                            # these required headers.
+                            if headerName.lower() not in (b'server', b'date'):
+                                self.request.responseHeaders.addRawHeader(
+                                    headerName, headerText)
+                output = text[headerend+len(delimiter):]
+                self.handling_headers = 0
+            if self.handling_headers:
+                self.headertext = text
+        if not self.handling_headers:
+            self.request.write(output)
+
+
+    def processEnded(self, reason):
+        if reason.value.exitCode != 0:
+            self._log.error("CGI {uri} exited with exit code {exitCode}",
+                    uri=self.request.uri, exitCode=reason.value.exitCode)
+        if self.errortext:
+            self._log.error("Errors from CGI {uri}: {errorText}",
+                uri=self.request.uri, errorText=self.errortext)
+        if self.handling_headers:
+            self._log.error("Premature end of headers in {uri}: {headerText}",
+                uri=self.request.uri, headerText=self.headertext)
+            self.request.write(
+                resource.ErrorPage(http.INTERNAL_SERVER_ERROR,
+                    "CGI Script Error",
+                    "Premature end of script headers.").render(self.request))
+        self.request.unregisterProducer()
+        self.request.finish()
diff --git a/contrib/python/Twisted/py2/twisted/web/util.py b/contrib/python/Twisted/py2/twisted/web/util.py
new file mode 100644
index 00000000000..3fffac1eacc
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/util.py
@@ -0,0 +1,443 @@
+# -*- test-case-name: twisted.web.test.test_util -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An assortment of web server-related utilities.
+"""
+
+from __future__ import division, absolute_import
+
+import linecache
+
+from twisted.python import urlpath
+from twisted.python.compat import _PY3, unicode, nativeString, escape
+from twisted.python.reflect import fullyQualifiedName
+
+from twisted.web import resource
+
+from twisted.web.template import TagLoader, XMLString, Element, renderer
+from twisted.web.template import flattenString
+
+
+
+def _PRE(text):
+    """
+    Wraps <pre> tags around some text and HTML-escape it.
+
+    This is here since once twisted.web.html was deprecated it was hard to
+    migrate the html.PRE from current code to twisted.web.template.
+
+    For new code consider using twisted.web.template.
+
+    @return: Escaped text wrapped in <pre> tags.
+    @rtype: C{str}
+    """
+    return '<pre>%s</pre>' % (escape(text),)
+
+
+
+def redirectTo(URL, request):
+    """
+    Generate a redirect to the given location.
+
+    @param URL: A L{bytes} giving the location to which to redirect.
+    @type URL: L{bytes}
+
+    @param request: The request object to use to generate the redirect.
+    @type request: L{IRequest<twisted.web.iweb.IRequest>} provider
+
+    @raise TypeError: If the type of C{URL} a L{unicode} instead of L{bytes}.
+
+    @return: A C{bytes} containing HTML which tries to convince the client agent
+        to visit the new location even if it doesn't respect the I{FOUND}
+        response code.  This is intended to be returned from a render method,
+        eg::
+
+            def render_GET(self, request):
+                return redirectTo(b"http://example.com/", request)
+    """
+    if isinstance(URL, unicode) :
+        raise TypeError("Unicode object not allowed as URL")
+    request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+    request.redirect(URL)
+    content =  """
+<html>
+    <head>
+        <meta http-equiv=\"refresh\" content=\"0;URL=%(url)s\">
+    </head>
+    <body bgcolor=\"#FFFFFF\" text=\"#000000\">
+    <a href=\"%(url)s\">click here</a>
+    </body>
+</html>
+""" % {'url': nativeString(URL)}
+    if _PY3:
+        content = content.encode("utf8")
+    return content
+
+
+class Redirect(resource.Resource):
+    isLeaf = True
+
+    def __init__(self, url):
+        resource.Resource.__init__(self)
+        self.url = url
+
+    def render(self, request):
+        return redirectTo(self.url, request)
+
+    def getChild(self, name, request):
+        return self
+
+
+class ChildRedirector(Redirect):
+    isLeaf = 0
+    def __init__(self, url):
+        # XXX is this enough?
+        if ((url.find('://') == -1)
+            and (not url.startswith('..'))
+            and (not url.startswith('/'))):
+            raise ValueError("It seems you've given me a redirect (%s) that is a child of myself! That's not good, it'll cause an infinite redirect." % url)
+        Redirect.__init__(self, url)
+
+    def getChild(self, name, request):
+        newUrl = self.url
+        if not newUrl.endswith('/'):
+            newUrl += '/'
+        newUrl += name
+        return ChildRedirector(newUrl)
+
+
+class ParentRedirect(resource.Resource):
+    """
+    I redirect to URLPath.here().
+    """
+    isLeaf = 1
+    def render(self, request):
+        return redirectTo(urlpath.URLPath.fromRequest(request).here(), request)
+
+    def getChild(self, request):
+        return self
+
+
+class DeferredResource(resource.Resource):
+    """
+    I wrap up a Deferred that will eventually result in a Resource
+    object.
+    """
+    isLeaf = 1
+
+    def __init__(self, d):
+        resource.Resource.__init__(self)
+        self.d = d
+
+    def getChild(self, name, request):
+        return self
+
+    def render(self, request):
+        self.d.addCallback(self._cbChild, request).addErrback(
+            self._ebChild,request)
+        from twisted.web.server import NOT_DONE_YET
+        return NOT_DONE_YET
+
+    def _cbChild(self, child, request):
+        request.render(resource.getChildForRequest(child, request))
+
+    def _ebChild(self, reason, request):
+        request.processingFailed(reason)
+
+
+
+class _SourceLineElement(Element):
+    """
+    L{_SourceLineElement} is an L{IRenderable} which can render a single line of
+    source code.
+
+    @ivar number: A C{int} giving the line number of the source code to be
+        rendered.
+    @ivar source: A C{str} giving the source code to be rendered.
+    """
+    def __init__(self, loader, number, source):
+        Element.__init__(self, loader)
+        self.number = number
+        self.source = source
+
+
+    @renderer
+    def sourceLine(self, request, tag):
+        """
+        Render the line of source as a child of C{tag}.
+        """
+        return tag(self.source.replace('  ', u' \N{NO-BREAK SPACE}'))
+
+
+    @renderer
+    def lineNumber(self, request, tag):
+        """
+        Render the line number as a child of C{tag}.
+        """
+        return tag(str(self.number))
+
+
+
+class _SourceFragmentElement(Element):
+    """
+    L{_SourceFragmentElement} is an L{IRenderable} which can render several lines
+    of source code near the line number of a particular frame object.
+
+    @ivar frame: A L{Failure<twisted.python.failure.Failure>}-style frame object
+        for which to load a source line to render.  This is really a tuple
+        holding some information from a frame object.  See
+        L{Failure.frames<twisted.python.failure.Failure>} for specifics.
+    """
+    def __init__(self, loader, frame):
+        Element.__init__(self, loader)
+        self.frame = frame
+
+
+    def _getSourceLines(self):
+        """
+        Find the source line references by C{self.frame} and yield, in source
+        line order, it and the previous and following lines.
+
+        @return: A generator which yields two-tuples.  Each tuple gives a source
+            line number and the contents of that source line.
+        """
+        filename = self.frame[1]
+        lineNumber = self.frame[2]
+        for snipLineNumber in range(lineNumber - 1, lineNumber + 2):
+            yield (snipLineNumber,
+                   linecache.getline(filename, snipLineNumber).rstrip())
+
+
+    @renderer
+    def sourceLines(self, request, tag):
+        """
+        Render the source line indicated by C{self.frame} and several
+        surrounding lines.  The active line will be given a I{class} of
+        C{"snippetHighlightLine"}.  Other lines will be given a I{class} of
+        C{"snippetLine"}.
+        """
+        for (lineNumber, sourceLine) in self._getSourceLines():
+            newTag = tag.clone()
+            if lineNumber == self.frame[2]:
+                cssClass = "snippetHighlightLine"
+            else:
+                cssClass = "snippetLine"
+            loader = TagLoader(newTag(**{"class": cssClass}))
+            yield _SourceLineElement(loader, lineNumber, sourceLine)
+
+
+
+class _FrameElement(Element):
+    """
+    L{_FrameElement} is an L{IRenderable} which can render details about one
+    frame from a L{Failure<twisted.python.failure.Failure>}.
+
+    @ivar frame: A L{Failure<twisted.python.failure.Failure>}-style frame object
+        for which to load a source line to render.  This is really a tuple
+        holding some information from a frame object.  See
+        L{Failure.frames<twisted.python.failure.Failure>} for specifics.
+    """
+    def __init__(self, loader, frame):
+        Element.__init__(self, loader)
+        self.frame = frame
+
+
+    @renderer
+    def filename(self, request, tag):
+        """
+        Render the name of the file this frame references as a child of C{tag}.
+        """
+        return tag(self.frame[1])
+
+
+    @renderer
+    def lineNumber(self, request, tag):
+        """
+        Render the source line number this frame references as a child of
+        C{tag}.
+        """
+        return tag(str(self.frame[2]))
+
+
+    @renderer
+    def function(self, request, tag):
+        """
+        Render the function name this frame references as a child of C{tag}.
+        """
+        return tag(self.frame[0])
+
+
+    @renderer
+    def source(self, request, tag):
+        """
+        Render the source code surrounding the line this frame references,
+        replacing C{tag}.
+        """
+        return _SourceFragmentElement(TagLoader(tag), self.frame)
+
+
+
+class _StackElement(Element):
+    """
+    L{_StackElement} renders an L{IRenderable} which can render a list of frames.
+    """
+    def __init__(self, loader, stackFrames):
+        Element.__init__(self, loader)
+        self.stackFrames = stackFrames
+
+
+    @renderer
+    def frames(self, request, tag):
+        """
+        Render the list of frames in this L{_StackElement}, replacing C{tag}.
+        """
+        return [
+            _FrameElement(TagLoader(tag.clone()), frame)
+            for frame
+            in self.stackFrames]
+
+
+
+class FailureElement(Element):
+    """
+    L{FailureElement} is an L{IRenderable} which can render detailed information
+    about a L{Failure<twisted.python.failure.Failure>}.
+
+    @ivar failure: The L{Failure<twisted.python.failure.Failure>} instance which
+        will be rendered.
+
+    @since: 12.1
+    """
+    loader = XMLString("""
+<div xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
+  <style type="text/css">
+    div.error {
+      color: red;
+      font-family: Verdana, Arial, helvetica, sans-serif;
+      font-weight: bold;
+    }
+
+    div {
+      font-family: Verdana, Arial, helvetica, sans-serif;
+    }
+
+    div.stackTrace {
+    }
+
+    div.frame {
+      padding: 1em;
+      background: white;
+      border-bottom: thin black dashed;
+    }
+
+    div.frame:first-child {
+      padding: 1em;
+      background: white;
+      border-top: thin black dashed;
+      border-bottom: thin black dashed;
+    }
+
+    div.location {
+    }
+
+    span.function {
+      font-weight: bold;
+      font-family: "Courier New", courier, monospace;
+    }
+
+    div.snippet {
+      margin-bottom: 0.5em;
+      margin-left: 1em;
+      background: #FFFFDD;
+    }
+
+    div.snippetHighlightLine {
+      color: red;
+    }
+
+    span.code {
+      font-family: "Courier New", courier, monospace;
+    }
+  </style>
+
+  <div class="error">
+    <span t:render="type" />: <span t:render="value" />
+  </div>
+  <div class="stackTrace" t:render="traceback">
+    <div class="frame" t:render="frames">
+      <div class="location">
+        <span t:render="filename" />:<span t:render="lineNumber" /> in
+        <span class="function" t:render="function" />
+      </div>
+      <div class="snippet" t:render="source">
+        <div t:render="sourceLines">
+          <span class="lineno" t:render="lineNumber" />
+          <code class="code" t:render="sourceLine" />
+        </div>
+      </div>
+    </div>
+  </div>
+  <div class="error">
+    <span t:render="type" />: <span t:render="value" />
+  </div>
+</div>
+""")
+
+    def __init__(self, failure, loader=None):
+        Element.__init__(self, loader)
+        self.failure = failure
+
+
+    @renderer
+    def type(self, request, tag):
+        """
+        Render the exception type as a child of C{tag}.
+        """
+        return tag(fullyQualifiedName(self.failure.type))
+
+
+    @renderer
+    def value(self, request, tag):
+        """
+        Render the exception value as a child of C{tag}.
+        """
+        return tag(unicode(self.failure.value).encode('utf8'))
+
+
+    @renderer
+    def traceback(self, request, tag):
+        """
+        Render all the frames in the wrapped
+        L{Failure<twisted.python.failure.Failure>}'s traceback stack, replacing
+        C{tag}.
+        """
+        return _StackElement(TagLoader(tag), self.failure.frames)
+
+
+
+def formatFailure(myFailure):
+    """
+    Construct an HTML representation of the given failure.
+
+    Consider using L{FailureElement} instead.
+
+    @type myFailure: L{Failure<twisted.python.failure.Failure>}
+
+    @rtype: C{bytes}
+    @return: A string containing the HTML representation of the given failure.
+    """
+    result = []
+    flattenString(None, FailureElement(myFailure)).addBoth(result.append)
+    if isinstance(result[0], bytes):
+        # Ensure the result string is all ASCII, for compatibility with the
+        # default encoding expected by browsers.
+        return result[0].decode('utf-8').encode('ascii', 'xmlcharrefreplace')
+    result[0].raiseException()
+
+
+
+__all__ = [
+    "redirectTo", "Redirect", "ChildRedirector", "ParentRedirect",
+    "DeferredResource", "FailureElement", "formatFailure"]
diff --git a/contrib/python/Twisted/py2/twisted/web/vhost.py b/contrib/python/Twisted/py2/twisted/web/vhost.py
new file mode 100644
index 00000000000..3751b768ae6
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/vhost.py
@@ -0,0 +1,138 @@
+# -*- test-case-name: twisted.web.
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+I am a virtual hosts implementation.
+"""
+
+from __future__ import division, absolute_import
+
+# Twisted Imports
+from twisted.python import roots
+from twisted.web import resource
+
+
+class VirtualHostCollection(roots.Homogenous):
+    """Wrapper for virtual hosts collection.
+
+    This exists for configuration purposes.
+    """
+    entityType = resource.Resource
+
+    def __init__(self, nvh):
+        self.nvh = nvh
+
+    def listStaticEntities(self):
+        return self.nvh.hosts.items()
+
+    def getStaticEntity(self, name):
+        return self.nvh.hosts.get(self)
+
+    def reallyPutEntity(self, name, entity):
+        self.nvh.addHost(name, entity)
+
+    def delEntity(self, name):
+        self.nvh.removeHost(name)
+
+
+class NameVirtualHost(resource.Resource):
+    """I am a resource which represents named virtual hosts.
+    """
+
+    default = None
+
+    def __init__(self):
+        """Initialize.
+        """
+        resource.Resource.__init__(self)
+        self.hosts = {}
+
+    def listStaticEntities(self):
+        return resource.Resource.listStaticEntities(self) + [("Virtual Hosts", VirtualHostCollection(self))]
+
+    def getStaticEntity(self, name):
+        if name == "Virtual Hosts":
+            return VirtualHostCollection(self)
+        else:
+            return resource.Resource.getStaticEntity(self, name)
+
+    def addHost(self, name, resrc):
+        """Add a host to this virtual host.
+
+        This will take a host named `name', and map it to a resource
+        `resrc'.  For example, a setup for our virtual hosts would be::
+
+            nvh.addHost('divunal.com', divunalDirectory)
+            nvh.addHost('www.divunal.com', divunalDirectory)
+            nvh.addHost('twistedmatrix.com', twistedMatrixDirectory)
+            nvh.addHost('www.twistedmatrix.com', twistedMatrixDirectory)
+        """
+        self.hosts[name] = resrc
+
+    def removeHost(self, name):
+        """Remove a host."""
+        del self.hosts[name]
+
+    def _getResourceForRequest(self, request):
+        """(Internal) Get the appropriate resource for the given host.
+        """
+        hostHeader = request.getHeader(b'host')
+        if hostHeader == None:
+            return self.default or resource.NoResource()
+        else:
+            host = hostHeader.lower().split(b':', 1)[0]
+        return (self.hosts.get(host, self.default)
+                or resource.NoResource("host %s not in vhost map" % repr(host)))
+
+    def render(self, request):
+        """Implementation of resource.Resource's render method.
+        """
+        resrc = self._getResourceForRequest(request)
+        return resrc.render(request)
+
+    def getChild(self, path, request):
+        """Implementation of resource.Resource's getChild method.
+        """
+        resrc = self._getResourceForRequest(request)
+        if resrc.isLeaf:
+            request.postpath.insert(0,request.prepath.pop(-1))
+            return resrc
+        else:
+            return resrc.getChildWithDefault(path, request)
+
+class _HostResource(resource.Resource):
+
+    def getChild(self, path, request):
+        if b':' in path:
+            host, port = path.split(b':', 1)
+            port = int(port)
+        else:
+            host, port = path, 80
+        request.setHost(host, port)
+        prefixLen = (3 + request.isSecure() + 4 + len(path) +
+                     len(request.prepath[-3]))
+        request.path = b'/' + b'/'.join(request.postpath)
+        request.uri = request.uri[prefixLen:]
+        del request.prepath[:3]
+        return request.site.getResourceFor(request)
+
+
+class VHostMonsterResource(resource.Resource):
+
+    """
+    Use this to be able to record the hostname and method (http vs. https)
+    in the URL without disturbing your web site. If you put this resource
+    in a URL http://foo.com/bar then requests to
+    http://foo.com/bar/http/baz.com/something will be equivalent to
+    http://foo.com/something, except that the hostname the request will
+    appear to be accessing will be "baz.com". So if "baz.com" is redirecting
+    all requests for to foo.com, while foo.com is inaccessible from the outside,
+    then redirect and url generation will work correctly
+    """
+    def getChild(self, path, request):
+        if path == b'http':
+            request.isSecure = lambda: 0
+        elif path == b'https':
+            request.isSecure = lambda: 1
+        return _HostResource()
diff --git a/contrib/python/Twisted/py2/twisted/web/wsgi.py b/contrib/python/Twisted/py2/twisted/web/wsgi.py
new file mode 100644
index 00000000000..311050f2338
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/wsgi.py
@@ -0,0 +1,596 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An implementation of
+U{Python Web Server Gateway Interface v1.0.1<http://www.python.org/dev/peps/pep-3333/>}.
+"""
+
+__metaclass__ = type
+
+from sys import exc_info
+from warnings import warn
+
+from zope.interface import implementer
+
+from twisted.internet.threads import blockingCallFromThread
+from twisted.python.compat import reraise, Sequence
+from twisted.python.failure import Failure
+from twisted.web.resource import IResource
+from twisted.web.server import NOT_DONE_YET
+from twisted.web.http import INTERNAL_SERVER_ERROR
+from twisted.logger import Logger
+
+
+
+# PEP-3333 -- which has superseded PEP-333 -- states that, in both Python 2
+# and Python 3, text strings MUST be represented using the platform's native
+# string type, limited to characters defined in ISO-8859-1. Byte strings are
+# used only for values read from wsgi.input, passed to write() or yielded by
+# the application.
+#
+# Put another way:
+#
+# - In Python 2, all text strings and binary data are of type str/bytes and
+#   NEVER of type unicode. Whether the strings contain binary data or
+#   ISO-8859-1 text depends on context.
+#
+# - In Python 3, all text strings are of type str, and all binary data are of
+#   type bytes. Text MUST always be limited to that which can be encoded as
+#   ISO-8859-1, U+0000 to U+00FF inclusive.
+#
+# The following pair of functions -- _wsgiString() and _wsgiStringToBytes() --
+# are used to make Twisted's WSGI support compliant with the standard.
+if str is bytes:
+    def _wsgiString(string):  # Python 2.
+        """
+        Convert C{string} to an ISO-8859-1 byte string, if it is not already.
+
+        @type string: C{str}/C{bytes} or C{unicode}
+        @rtype: C{str}/C{bytes}
+
+        @raise UnicodeEncodeError: If C{string} contains non-ISO-8859-1 chars.
+        """
+        if isinstance(string, str):
+            return string
+        else:
+            return string.encode('iso-8859-1')
+
+    def _wsgiStringToBytes(string):  # Python 2.
+        """
+        Return C{string} as is; a WSGI string is a byte string in Python 2.
+
+        @type string: C{str}/C{bytes}
+        @rtype: C{str}/C{bytes}
+        """
+        return string
+
+else:
+    def _wsgiString(string):  # Python 3.
+        """
+        Convert C{string} to a WSGI "bytes-as-unicode" string.
+
+        If it's a byte string, decode as ISO-8859-1. If it's a Unicode string,
+        round-trip it to bytes and back using ISO-8859-1 as the encoding.
+
+        @type string: C{str} or C{bytes}
+        @rtype: C{str}
+
+        @raise UnicodeEncodeError: If C{string} contains non-ISO-8859-1 chars.
+        """
+        if isinstance(string, str):
+            return string.encode("iso-8859-1").decode('iso-8859-1')
+        else:
+            return string.decode("iso-8859-1")
+
+    def _wsgiStringToBytes(string):  # Python 3.
+        """
+        Convert C{string} from a WSGI "bytes-as-unicode" string to an
+        ISO-8859-1 byte string.
+
+        @type string: C{str}
+        @rtype: C{bytes}
+
+        @raise UnicodeEncodeError: If C{string} contains non-ISO-8859-1 chars.
+        """
+        return string.encode("iso-8859-1")
+
+
+
+class _ErrorStream:
+    """
+    File-like object instances of which are used as the value for the
+    C{'wsgi.errors'} key in the C{environ} dictionary passed to the application
+    object.
+
+    This simply passes writes on to L{logging<twisted.logger>} system as
+    error events from the C{'wsgi'} system.  In the future, it may be desirable
+    to expose more information in the events it logs, such as the application
+    object which generated the message.
+    """
+    _log = Logger()
+
+    def write(self, data):
+        """
+        Generate an event for the logging system with the given bytes as the
+        message.
+
+        This is called in a WSGI application thread, not the I/O thread.
+
+        @type data: str
+
+        @raise TypeError: On Python 3, if C{data} is not a native string. On
+            Python 2 a warning will be issued.
+        """
+        if not isinstance(data, str):
+            if str is bytes:
+                warn("write() argument should be str, not %r (%s)" % (
+                    data, type(data).__name__), category=UnicodeWarning)
+            else:
+                raise TypeError(
+                    "write() argument must be str, not %r (%s)"
+                    % (data, type(data).__name__))
+
+        # Note that in old style, message was a tuple.  logger._legacy
+        # will overwrite this value if it is not properly formatted here.
+        self._log.error(
+            data,
+            system='wsgi',
+            isError=True,
+            message=(data,)
+        )
+
+
+    def writelines(self, iovec):
+        """
+        Join the given lines and pass them to C{write} to be handled in the
+        usual way.
+
+        This is called in a WSGI application thread, not the I/O thread.
+
+        @param iovec: A C{list} of C{'\\n'}-terminated C{str} which will be
+            logged.
+
+        @raise TypeError: On Python 3, if C{iovec} contains any non-native
+            strings. On Python 2 a warning will be issued.
+        """
+        self.write(''.join(iovec))
+
+
+    def flush(self):
+        """
+        Nothing is buffered, so flushing does nothing.  This method is required
+        to exist by PEP 333, though.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+
+
+
+class _InputStream:
+    """
+    File-like object instances of which are used as the value for the
+    C{'wsgi.input'} key in the C{environ} dictionary passed to the application
+    object.
+
+    This only exists to make the handling of C{readline(-1)} consistent across
+    different possible underlying file-like object implementations.  The other
+    supported methods pass through directly to the wrapped object.
+    """
+    def __init__(self, input):
+        """
+        Initialize the instance.
+
+        This is called in the I/O thread, not a WSGI application thread.
+        """
+        self._wrapped = input
+
+
+    def read(self, size=None):
+        """
+        Pass through to the underlying C{read}.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+        # Avoid passing None because cStringIO and file don't like it.
+        if size is None:
+            return self._wrapped.read()
+        return self._wrapped.read(size)
+
+
+    def readline(self, size=None):
+        """
+        Pass through to the underlying C{readline}, with a size of C{-1} replaced
+        with a size of L{None}.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+        # Check for -1 because StringIO doesn't handle it correctly.  Check for
+        # None because files and tempfiles don't accept that.
+        if size == -1 or size is None:
+            return self._wrapped.readline()
+        return self._wrapped.readline(size)
+
+
+    def readlines(self, size=None):
+        """
+        Pass through to the underlying C{readlines}.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+        # Avoid passing None because cStringIO and file don't like it.
+        if size is None:
+            return self._wrapped.readlines()
+        return self._wrapped.readlines(size)
+
+
+    def __iter__(self):
+        """
+        Pass through to the underlying C{__iter__}.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+        return iter(self._wrapped)
+
+
+
+class _WSGIResponse:
+    """
+    Helper for L{WSGIResource} which drives the WSGI application using a
+    threadpool and hooks it up to the L{http.Request}.
+
+    @ivar started: A L{bool} indicating whether or not the response status and
+        headers have been written to the request yet.  This may only be read or
+        written in the WSGI application thread.
+
+    @ivar reactor: An L{IReactorThreads} provider which is used to call methods
+        on the request in the I/O thread.
+
+    @ivar threadpool: A L{ThreadPool} which is used to call the WSGI
+        application object in a non-I/O thread.
+
+    @ivar application: The WSGI application object.
+
+    @ivar request: The L{http.Request} upon which the WSGI environment is
+        based and to which the application's output will be sent.
+
+    @ivar environ: The WSGI environment L{dict}.
+
+    @ivar status: The HTTP response status L{str} supplied to the WSGI
+        I{start_response} callable by the application.
+
+    @ivar headers: A list of HTTP response headers supplied to the WSGI
+        I{start_response} callable by the application.
+
+    @ivar _requestFinished: A flag which indicates whether it is possible to
+        generate more response data or not.  This is L{False} until
+        L{http.Request.notifyFinish} tells us the request is done,
+        then L{True}.
+    """
+
+    _requestFinished = False
+    _log = Logger()
+
+    def __init__(self, reactor, threadpool, application, request):
+        self.started = False
+        self.reactor = reactor
+        self.threadpool = threadpool
+        self.application = application
+        self.request = request
+        self.request.notifyFinish().addBoth(self._finished)
+
+        if request.prepath:
+            scriptName = b'/' + b'/'.join(request.prepath)
+        else:
+            scriptName = b''
+
+        if request.postpath:
+            pathInfo = b'/' + b'/'.join(request.postpath)
+        else:
+            pathInfo = b''
+
+        parts = request.uri.split(b'?', 1)
+        if len(parts) == 1:
+            queryString = b''
+        else:
+            queryString = parts[1]
+
+        # All keys and values need to be native strings, i.e. of type str in
+        # *both* Python 2 and Python 3, so says PEP-3333.
+        self.environ = {
+            'REQUEST_METHOD': _wsgiString(request.method),
+            'REMOTE_ADDR': _wsgiString(request.getClientAddress().host),
+            'SCRIPT_NAME': _wsgiString(scriptName),
+            'PATH_INFO': _wsgiString(pathInfo),
+            'QUERY_STRING': _wsgiString(queryString),
+            'CONTENT_TYPE': _wsgiString(
+                request.getHeader(b'content-type') or ''),
+            'CONTENT_LENGTH': _wsgiString(
+                request.getHeader(b'content-length') or ''),
+            'SERVER_NAME': _wsgiString(request.getRequestHostname()),
+            'SERVER_PORT': _wsgiString(str(request.getHost().port)),
+            'SERVER_PROTOCOL': _wsgiString(request.clientproto)}
+
+        # The application object is entirely in control of response headers;
+        # disable the default Content-Type value normally provided by
+        # twisted.web.server.Request.
+        self.request.defaultContentType = None
+
+        for name, values in request.requestHeaders.getAllRawHeaders():
+            name = 'HTTP_' + _wsgiString(name).upper().replace('-', '_')
+            # It might be preferable for http.HTTPChannel to clear out
+            # newlines.
+            self.environ[name] = ','.join(
+                _wsgiString(v) for v in values).replace('\n', ' ')
+
+        self.environ.update({
+                'wsgi.version': (1, 0),
+                'wsgi.url_scheme': request.isSecure() and 'https' or 'http',
+                'wsgi.run_once': False,
+                'wsgi.multithread': True,
+                'wsgi.multiprocess': False,
+                'wsgi.errors': _ErrorStream(),
+                # Attend: request.content was owned by the I/O thread up until
+                # this point.  By wrapping it and putting the result into the
+                # environment dictionary, it is effectively being given to
+                # another thread.  This means that whatever it is, it has to be
+                # safe to access it from two different threads.  The access
+                # *should* all be serialized (first the I/O thread writes to
+                # it, then the WSGI thread reads from it, then the I/O thread
+                # closes it).  However, since the request is made available to
+                # arbitrary application code during resource traversal, it's
+                # possible that some other code might decide to use it in the
+                # I/O thread concurrently with its use in the WSGI thread.
+                # More likely than not, this will break.  This seems like an
+                # unlikely possibility to me, but if it is to be allowed,
+                # something here needs to change. -exarkun
+                'wsgi.input': _InputStream(request.content)})
+
+
+    def _finished(self, ignored):
+        """
+        Record the end of the response generation for the request being
+        serviced.
+        """
+        self._requestFinished = True
+
+
+    def startResponse(self, status, headers, excInfo=None):
+        """
+        The WSGI I{start_response} callable.  The given values are saved until
+        they are needed to generate the response.
+
+        This will be called in a non-I/O thread.
+        """
+        if self.started and excInfo is not None:
+            reraise(excInfo[1], excInfo[2])
+
+        # PEP-3333 mandates that status should be a native string. In practice
+        # this is mandated by Twisted's HTTP implementation too, so we enforce
+        # on both Python 2 and Python 3.
+        if not isinstance(status, str):
+            raise TypeError(
+                "status must be str, not %r (%s)"
+                % (status, type(status).__name__))
+
+        # PEP-3333 mandates that headers should be a plain list, but in
+        # practice we work with any sequence type and only warn when it's not
+        # a plain list.
+        if isinstance(headers, list):
+            pass  # This is okay.
+        elif isinstance(headers, Sequence):
+            warn("headers should be a list, not %r (%s)" % (
+                headers, type(headers).__name__), category=RuntimeWarning)
+        else:
+            raise TypeError(
+                "headers must be a list, not %r (%s)"
+                % (headers, type(headers).__name__))
+
+        # PEP-3333 mandates that each header should be a (str, str) tuple, but
+        # in practice we work with any sequence type and only warn when it's
+        # not a plain list.
+        for header in headers:
+            if isinstance(header, tuple):
+                pass  # This is okay.
+            elif isinstance(header, Sequence):
+                warn("header should be a (str, str) tuple, not %r (%s)" % (
+                    header, type(header).__name__), category=RuntimeWarning)
+            else:
+                raise TypeError(
+                    "header must be a (str, str) tuple, not %r (%s)"
+                    % (header, type(header).__name__))
+
+            # However, the sequence MUST contain only 2 elements.
+            if len(header) != 2:
+                raise TypeError(
+                    "header must be a (str, str) tuple, not %r"
+                    % (header, ))
+
+            # Both elements MUST be native strings. Non-native strings will be
+            # rejected by the underlying HTTP machinery in any case, but we
+            # reject them here in order to provide a more informative error.
+            for elem in header:
+                if not isinstance(elem, str):
+                    raise TypeError(
+                        "header must be (str, str) tuple, not %r"
+                        % (header, ))
+
+        self.status = status
+        self.headers = headers
+        return self.write
+
+
+    def write(self, data):
+        """
+        The WSGI I{write} callable returned by the I{start_response} callable.
+        The given bytes will be written to the response body, possibly flushing
+        the status and headers first.
+
+        This will be called in a non-I/O thread.
+        """
+        # PEP-3333 states:
+        #
+        #   The server or gateway must transmit the yielded bytestrings to the
+        #   client in an unbuffered fashion, completing the transmission of
+        #   each bytestring before requesting another one.
+        #
+        # This write() method is used for the imperative and (indirectly) for
+        # the more familiar iterable-of-bytestrings WSGI mechanism. It uses
+        # C{blockingCallFromThread} to schedule writes. This allows exceptions
+        # to propagate up from the underlying HTTP implementation. However,
+        # that underlying implementation does not, as yet, provide any way to
+        # know if the written data has been transmitted, so this method
+        # violates the above part of PEP-3333.
+        #
+        # PEP-3333 also says that a server may:
+        #
+        #   Use a different thread to ensure that the block continues to be
+        #   transmitted while the application produces the next block.
+        #
+        # Which suggests that this is actually compliant with PEP-3333,
+        # because writes are done in the reactor thread.
+        #
+        # However, providing some back-pressure may nevertheless be a Good
+        # Thing at some point in the future.
+
+        def wsgiWrite(started):
+            if not started:
+                self._sendResponseHeaders()
+            self.request.write(data)
+
+        try:
+            return blockingCallFromThread(
+                self.reactor, wsgiWrite, self.started)
+        finally:
+            self.started = True
+
+
+    def _sendResponseHeaders(self):
+        """
+        Set the response code and response headers on the request object, but
+        do not flush them.  The caller is responsible for doing a write in
+        order for anything to actually be written out in response to the
+        request.
+
+        This must be called in the I/O thread.
+        """
+        code, message = self.status.split(None, 1)
+        code = int(code)
+        self.request.setResponseCode(code, _wsgiStringToBytes(message))
+
+        for name, value in self.headers:
+            # Don't allow the application to control these required headers.
+            if name.lower() not in ('server', 'date'):
+                self.request.responseHeaders.addRawHeader(
+                    _wsgiStringToBytes(name), _wsgiStringToBytes(value))
+
+
+    def start(self):
+        """
+        Start the WSGI application in the threadpool.
+
+        This must be called in the I/O thread.
+        """
+        self.threadpool.callInThread(self.run)
+
+
+    def run(self):
+        """
+        Call the WSGI application object, iterate it, and handle its output.
+
+        This must be called in a non-I/O thread (ie, a WSGI application
+        thread).
+        """
+        try:
+            appIterator = self.application(self.environ, self.startResponse)
+            for elem in appIterator:
+                if elem:
+                    self.write(elem)
+                if self._requestFinished:
+                    break
+            close = getattr(appIterator, 'close', None)
+            if close is not None:
+                close()
+        except:
+            def wsgiError(started, type, value, traceback):
+                self._log.failure(
+                    "WSGI application error",
+                    failure=Failure(value, type, traceback)
+                )
+                if started:
+                    self.request.loseConnection()
+                else:
+                    self.request.setResponseCode(INTERNAL_SERVER_ERROR)
+                    self.request.finish()
+            self.reactor.callFromThread(wsgiError, self.started, *exc_info())
+        else:
+            def wsgiFinish(started):
+                if not self._requestFinished:
+                    if not started:
+                        self._sendResponseHeaders()
+                    self.request.finish()
+            self.reactor.callFromThread(wsgiFinish, self.started)
+        self.started = True
+
+
+
+@implementer(IResource)
+class WSGIResource:
+    """
+    An L{IResource} implementation which delegates responsibility for all
+    resources hierarchically inferior to it to a WSGI application.
+
+    @ivar _reactor: An L{IReactorThreads} provider which will be passed on to
+        L{_WSGIResponse} to schedule calls in the I/O thread.
+
+    @ivar _threadpool: A L{ThreadPool} which will be passed on to
+        L{_WSGIResponse} to run the WSGI application object.
+
+    @ivar _application: The WSGI application object.
+    """
+
+    # Further resource segments are left up to the WSGI application object to
+    # handle.
+    isLeaf = True
+
+    def __init__(self, reactor, threadpool, application):
+        self._reactor = reactor
+        self._threadpool = threadpool
+        self._application = application
+
+
+    def render(self, request):
+        """
+        Turn the request into the appropriate C{environ} C{dict} suitable to be
+        passed to the WSGI application object and then pass it on.
+
+        The WSGI application object is given almost complete control of the
+        rendering process.  C{NOT_DONE_YET} will always be returned in order
+        and response completion will be dictated by the application object, as
+        will the status, headers, and the response body.
+        """
+        response = _WSGIResponse(
+            self._reactor, self._threadpool, self._application, request)
+        response.start()
+        return NOT_DONE_YET
+
+
+    def getChildWithDefault(self, name, request):
+        """
+        Reject attempts to retrieve a child resource.  All path segments beyond
+        the one which refers to this resource are handled by the WSGI
+        application object.
+        """
+        raise RuntimeError("Cannot get IResource children from WSGIResource")
+
+
+    def putChild(self, path, child):
+        """
+        Reject attempts to add a child resource to this resource.  The WSGI
+        application object handles all path segments beneath this resource, so
+        L{IResource} children can never be found.
+        """
+        raise RuntimeError("Cannot put IResource children under WSGIResource")
+
+
+__all__ = ['WSGIResource']
diff --git a/contrib/python/Twisted/py2/twisted/web/xmlrpc.py b/contrib/python/Twisted/py2/twisted/web/xmlrpc.py
new file mode 100644
index 00000000000..4a9f3e0afc2
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/web/xmlrpc.py
@@ -0,0 +1,591 @@
+# -*- test-case-name: twisted.web.test.test_xmlrpc -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A generic resource for publishing objects via XML-RPC.
+
+Maintainer: Itamar Shtull-Trauring
+
+@var Fault: See L{xmlrpclib.Fault}
+@type Fault: L{xmlrpclib.Fault}
+"""
+
+from __future__ import division, absolute_import
+
+from twisted.python.compat import _PY3, intToBytes, nativeString, urllib_parse
+from twisted.python.compat import unicode
+
+# System Imports
+import base64
+if _PY3:
+    import xmlrpc.client as xmlrpclib
+else:
+    import xmlrpclib
+
+# Sibling Imports
+from twisted.web import resource, server, http
+from twisted.internet import defer, protocol, reactor
+from twisted.python import reflect, failure
+from twisted.logger import Logger
+
+# These are deprecated, use the class level definitions
+NOT_FOUND = 8001
+FAILURE = 8002
+
+
+# Useful so people don't need to import xmlrpclib directly
+Fault = xmlrpclib.Fault
+Binary = xmlrpclib.Binary
+Boolean = xmlrpclib.Boolean
+DateTime = xmlrpclib.DateTime
+
+
+def withRequest(f):
+    """
+    Decorator to cause the request to be passed as the first argument
+    to the method.
+
+    If an I{xmlrpc_} method is wrapped with C{withRequest}, the
+    request object is passed as the first argument to that method.
+    For example::
+
+        @withRequest
+        def xmlrpc_echo(self, request, s):
+            return s
+
+    @since: 10.2
+    """
+    f.withRequest = True
+    return f
+
+
+
+class NoSuchFunction(Fault):
+    """
+    There is no function by the given name.
+    """
+
+
+class Handler:
+    """
+    Handle a XML-RPC request and store the state for a request in progress.
+
+    Override the run() method and return result using self.result,
+    a Deferred.
+
+    We require this class since we're not using threads, so we can't
+    encapsulate state in a running function if we're going  to have
+    to wait for results.
+
+    For example, lets say we want to authenticate against twisted.cred,
+    run a LDAP query and then pass its result to a database query, all
+    as a result of a single XML-RPC command. We'd use a Handler instance
+    to store the state of the running command.
+    """
+
+    def __init__(self, resource, *args):
+        self.resource = resource # the XML-RPC resource we are connected to
+        self.result = defer.Deferred()
+        self.run(*args)
+
+    def run(self, *args):
+        # event driven equivalent of 'raise UnimplementedError'
+        self.result.errback(
+            NotImplementedError("Implement run() in subclasses"))
+
+
+class XMLRPC(resource.Resource):
+    """
+    A resource that implements XML-RPC.
+
+    You probably want to connect this to '/RPC2'.
+
+    Methods published can return XML-RPC serializable results, Faults,
+    Binary, Boolean, DateTime, Deferreds, or Handler instances.
+
+    By default methods beginning with 'xmlrpc_' are published.
+
+    Sub-handlers for prefixed methods (e.g., system.listMethods)
+    can be added with putSubHandler. By default, prefixes are
+    separated with a '.'. Override self.separator to change this.
+
+    @ivar allowNone: Permit XML translating of Python constant None.
+    @type allowNone: C{bool}
+
+    @ivar useDateTime: Present C{datetime} values as C{datetime.datetime}
+        objects?
+    @type useDateTime: C{bool}
+    """
+
+    # Error codes for Twisted, if they conflict with yours then
+    # modify them at runtime.
+    NOT_FOUND = 8001
+    FAILURE = 8002
+
+    isLeaf = 1
+    separator = '.'
+    allowedMethods = (b'POST',)
+    _log = Logger()
+
+    def __init__(self, allowNone=False, useDateTime=False):
+        resource.Resource.__init__(self)
+        self.subHandlers = {}
+        self.allowNone = allowNone
+        self.useDateTime = useDateTime
+
+
+    def __setattr__(self, name, value):
+        self.__dict__[name] = value
+
+
+    def putSubHandler(self, prefix, handler):
+        self.subHandlers[prefix] = handler
+
+    def getSubHandler(self, prefix):
+        return self.subHandlers.get(prefix, None)
+
+    def getSubHandlerPrefixes(self):
+        return list(self.subHandlers.keys())
+
+    def render_POST(self, request):
+        request.content.seek(0, 0)
+        request.setHeader(b"content-type", b"text/xml; charset=utf-8")
+        try:
+            args, functionPath = xmlrpclib.loads(request.content.read(),
+                use_datetime=self.useDateTime)
+        except Exception as e:
+            f = Fault(self.FAILURE, "Can't deserialize input: %s" % (e,))
+            self._cbRender(f, request)
+        else:
+            try:
+                function = self.lookupProcedure(functionPath)
+            except Fault as f:
+                self._cbRender(f, request)
+            else:
+                # Use this list to track whether the response has failed or not.
+                # This will be used later on to decide if the result of the
+                # Deferred should be written out and Request.finish called.
+                responseFailed = []
+                request.notifyFinish().addErrback(responseFailed.append)
+                if getattr(function, 'withRequest', False):
+                    d = defer.maybeDeferred(function, request, *args)
+                else:
+                    d = defer.maybeDeferred(function, *args)
+                d.addErrback(self._ebRender)
+                d.addCallback(self._cbRender, request, responseFailed)
+        return server.NOT_DONE_YET
+
+
+    def _cbRender(self, result, request, responseFailed=None):
+        if responseFailed:
+            return
+
+        if isinstance(result, Handler):
+            result = result.result
+        if not isinstance(result, Fault):
+            result = (result,)
+        try:
+            try:
+                content = xmlrpclib.dumps(
+                    result, methodresponse=True,
+                    allow_none=self.allowNone)
+            except Exception as e:
+                f = Fault(self.FAILURE, "Can't serialize output: %s" % (e,))
+                content = xmlrpclib.dumps(f, methodresponse=True,
+                                          allow_none=self.allowNone)
+
+            if isinstance(content, unicode):
+                content = content.encode('utf8')
+            request.setHeader(
+                b"content-length", intToBytes(len(content)))
+            request.write(content)
+        except:
+            self._log.failure('')
+        request.finish()
+
+
+    def _ebRender(self, failure):
+        if isinstance(failure.value, Fault):
+            return failure.value
+        self._log.failure('', failure)
+        return Fault(self.FAILURE, "error")
+
+
+    def lookupProcedure(self, procedurePath):
+        """
+        Given a string naming a procedure, return a callable object for that
+        procedure or raise NoSuchFunction.
+
+        The returned object will be called, and should return the result of the
+        procedure, a Deferred, or a Fault instance.
+
+        Override in subclasses if you want your own policy.  The base
+        implementation that given C{'foo'}, C{self.xmlrpc_foo} will be returned.
+        If C{procedurePath} contains C{self.separator}, the sub-handler for the
+        initial prefix is used to search for the remaining path.
+
+        If you override C{lookupProcedure}, you may also want to override
+        C{listProcedures} to accurately report the procedures supported by your
+        resource, so that clients using the I{system.listMethods} procedure
+        receive accurate results.
+
+        @since: 11.1
+        """
+        if procedurePath.find(self.separator) != -1:
+            prefix, procedurePath = procedurePath.split(self.separator, 1)
+            handler = self.getSubHandler(prefix)
+            if handler is None:
+                raise NoSuchFunction(self.NOT_FOUND,
+                    "no such subHandler %s" % prefix)
+            return handler.lookupProcedure(procedurePath)
+
+        f = getattr(self, "xmlrpc_%s" % procedurePath, None)
+        if not f:
+            raise NoSuchFunction(self.NOT_FOUND,
+                "procedure %s not found" % procedurePath)
+        elif not callable(f):
+            raise NoSuchFunction(self.NOT_FOUND,
+                "procedure %s not callable" % procedurePath)
+        else:
+            return f
+
+    def listProcedures(self):
+        """
+        Return a list of the names of all xmlrpc procedures.
+
+        @since: 11.1
+        """
+        return reflect.prefixedMethodNames(self.__class__, 'xmlrpc_')
+
+
+class XMLRPCIntrospection(XMLRPC):
+    """
+    Implement the XML-RPC Introspection API.
+
+    By default, the methodHelp method returns the 'help' method attribute,
+    if it exists, otherwise the __doc__ method attribute, if it exists,
+    otherwise the empty string.
+
+    To enable the methodSignature method, add a 'signature' method attribute
+    containing a list of lists. See methodSignature's documentation for the
+    format. Note the type strings should be XML-RPC types, not Python types.
+    """
+
+    def __init__(self, parent):
+        """
+        Implement Introspection support for an XMLRPC server.
+
+        @param parent: the XMLRPC server to add Introspection support to.
+        @type parent: L{XMLRPC}
+        """
+        XMLRPC.__init__(self)
+        self._xmlrpc_parent = parent
+
+    def xmlrpc_listMethods(self):
+        """
+        Return a list of the method names implemented by this server.
+        """
+        functions = []
+        todo = [(self._xmlrpc_parent, '')]
+        while todo:
+            obj, prefix = todo.pop(0)
+            functions.extend([prefix + name for name in obj.listProcedures()])
+            todo.extend([ (obj.getSubHandler(name),
+                           prefix + name + obj.separator)
+                          for name in obj.getSubHandlerPrefixes() ])
+        return functions
+
+    xmlrpc_listMethods.signature = [['array']]
+
+    def xmlrpc_methodHelp(self, method):
+        """
+        Return a documentation string describing the use of the given method.
+        """
+        method = self._xmlrpc_parent.lookupProcedure(method)
+        return (getattr(method, 'help', None)
+                or getattr(method, '__doc__', None) or '')
+
+    xmlrpc_methodHelp.signature = [['string', 'string']]
+
+    def xmlrpc_methodSignature(self, method):
+        """
+        Return a list of type signatures.
+
+        Each type signature is a list of the form [rtype, type1, type2, ...]
+        where rtype is the return type and typeN is the type of the Nth
+        argument. If no signature information is available, the empty
+        string is returned.
+        """
+        method = self._xmlrpc_parent.lookupProcedure(method)
+        return getattr(method, 'signature', None) or ''
+
+    xmlrpc_methodSignature.signature = [['array', 'string'],
+                                        ['string', 'string']]
+
+
+def addIntrospection(xmlrpc):
+    """
+    Add Introspection support to an XMLRPC server.
+
+    @param parent: the XMLRPC server to add Introspection support to.
+    @type parent: L{XMLRPC}
+    """
+    xmlrpc.putSubHandler('system', XMLRPCIntrospection(xmlrpc))
+
+
+class QueryProtocol(http.HTTPClient):
+    def connectionMade(self):
+        self._response = None
+        self.sendCommand(b'POST', self.factory.path)
+        self.sendHeader(b'User-Agent', b'Twisted/XMLRPClib')
+        self.sendHeader(b'Host', self.factory.host)
+        self.sendHeader(b'Content-type', b'text/xml; charset=utf-8')
+        payload = self.factory.payload
+        self.sendHeader(b'Content-length', intToBytes(len(payload)))
+
+        if self.factory.user:
+            auth = b':'.join([self.factory.user, self.factory.password])
+            authHeader = b''.join([b'Basic ', base64.b64encode(auth)])
+            self.sendHeader(b'Authorization', authHeader)
+        self.endHeaders()
+        self.transport.write(payload)
+
+    def handleStatus(self, version, status, message):
+        if status != b'200':
+            self.factory.badStatus(status, message)
+
+    def handleResponse(self, contents):
+        """
+        Handle the XML-RPC response received from the server.
+
+        Specifically, disconnect from the server and store the XML-RPC
+        response so that it can be properly handled when the disconnect is
+        finished.
+        """
+        self.transport.loseConnection()
+        self._response = contents
+
+    def connectionLost(self, reason):
+        """
+        The connection to the server has been lost.
+
+        If we have a full response from the server, then parse it and fired a
+        Deferred with the return value or C{Fault} that the server gave us.
+        """
+        http.HTTPClient.connectionLost(self, reason)
+        if self._response is not None:
+            response, self._response = self._response, None
+            self.factory.parseResponse(response)
+
+
+payloadTemplate = """<?xml version="1.0"?>
+<methodCall>
+<methodName>%s</methodName>
+%s
+</methodCall>
+"""
+
+
+class _QueryFactory(protocol.ClientFactory):
+    """
+    XML-RPC Client Factory
+
+    @ivar path: The path portion of the URL to which to post method calls.
+    @type path: L{bytes}
+
+    @ivar host: The value to use for the Host HTTP header.
+    @type host: L{bytes}
+
+    @ivar user: The username with which to authenticate with the server
+        when making calls.
+    @type user: L{bytes} or L{None}
+
+    @ivar password: The password with which to authenticate with the server
+        when making calls.
+    @type password: L{bytes} or L{None}
+
+    @ivar useDateTime: Accept datetime values as datetime.datetime objects.
+        also passed to the underlying xmlrpclib implementation.  Defaults to
+        C{False}.
+    @type useDateTime: C{bool}
+    """
+
+    deferred = None
+    protocol = QueryProtocol
+
+    def __init__(self, path, host, method, user=None, password=None,
+                 allowNone=False, args=(), canceller=None, useDateTime=False):
+        """
+        @param method: The name of the method to call.
+        @type method: C{str}
+
+        @param allowNone: allow the use of None values in parameters. It's
+            passed to the underlying xmlrpclib implementation. Defaults to
+            C{False}.
+        @type allowNone: C{bool} or L{None}
+
+        @param args: the arguments to pass to the method.
+        @type args: C{tuple}
+
+        @param canceller: A 1-argument callable passed to the deferred as the
+            canceller callback.
+        @type canceller: callable or L{None}
+        """
+        self.path, self.host = path, host
+        self.user, self.password = user, password
+        self.payload = payloadTemplate % (method,
+            xmlrpclib.dumps(args, allow_none=allowNone))
+        if isinstance(self.payload, unicode):
+            self.payload = self.payload.encode('utf8')
+        self.deferred = defer.Deferred(canceller)
+        self.useDateTime = useDateTime
+
+    def parseResponse(self, contents):
+        if not self.deferred:
+            return
+        try:
+            response = xmlrpclib.loads(contents,
+                use_datetime=self.useDateTime)[0][0]
+        except:
+            deferred, self.deferred = self.deferred, None
+            deferred.errback(failure.Failure())
+        else:
+            deferred, self.deferred = self.deferred, None
+            deferred.callback(response)
+
+    def clientConnectionLost(self, _, reason):
+        if self.deferred is not None:
+            deferred, self.deferred = self.deferred, None
+            deferred.errback(reason)
+
+    clientConnectionFailed = clientConnectionLost
+
+    def badStatus(self, status, message):
+        deferred, self.deferred = self.deferred, None
+        deferred.errback(ValueError(status, message))
+
+
+
+class Proxy:
+    """
+    A Proxy for making remote XML-RPC calls.
+
+    Pass the URL of the remote XML-RPC server to the constructor.
+
+    Use C{proxy.callRemote('foobar', *args)} to call remote method
+    'foobar' with *args.
+
+    @ivar user: The username with which to authenticate with the server
+        when making calls.  If specified, overrides any username information
+        embedded in C{url}.  If not specified, a value may be taken from
+        C{url} if present.
+    @type user: L{bytes} or L{None}
+
+    @ivar password: The password with which to authenticate with the server
+        when making calls.  If specified, overrides any password information
+        embedded in C{url}.  If not specified, a value may be taken from
+        C{url} if present.
+    @type password: L{bytes} or L{None}
+
+    @ivar allowNone: allow the use of None values in parameters. It's
+        passed to the underlying L{xmlrpclib} implementation. Defaults to
+        C{False}.
+    @type allowNone: C{bool} or L{None}
+
+    @ivar useDateTime: Accept datetime values as datetime.datetime objects.
+        also passed to the underlying L{xmlrpclib} implementation. Defaults to
+        C{False}.
+    @type useDateTime: C{bool}
+
+    @ivar connectTimeout: Number of seconds to wait before assuming the
+        connection has failed.
+    @type connectTimeout: C{float}
+
+    @ivar _reactor: The reactor used to create connections.
+    @type _reactor: Object providing L{twisted.internet.interfaces.IReactorTCP}
+
+    @ivar queryFactory: Object returning a factory for XML-RPC protocol. Mainly
+        useful for tests.
+    """
+    queryFactory = _QueryFactory
+
+    def __init__(self, url, user=None, password=None, allowNone=False,
+                 useDateTime=False, connectTimeout=30.0, reactor=reactor):
+        """
+        @param url: The URL to which to post method calls.  Calls will be made
+            over SSL if the scheme is HTTPS.  If netloc contains username or
+            password information, these will be used to authenticate, as long as
+            the C{user} and C{password} arguments are not specified.
+        @type url: L{bytes}
+
+        """
+        scheme, netloc, path, params, query, fragment = urllib_parse.urlparse(
+            url)
+        netlocParts = netloc.split(b'@')
+        if len(netlocParts) == 2:
+            userpass = netlocParts.pop(0).split(b':')
+            self.user = userpass.pop(0)
+            try:
+                self.password = userpass.pop(0)
+            except:
+                self.password = None
+        else:
+            self.user = self.password = None
+        hostport = netlocParts[0].split(b':')
+        self.host = hostport.pop(0)
+        try:
+            self.port = int(hostport.pop(0))
+        except:
+            self.port = None
+        self.path = path
+        if self.path in [b'', None]:
+            self.path = b'/'
+        self.secure = (scheme == b'https')
+        if user is not None:
+            self.user = user
+        if password is not None:
+            self.password = password
+        self.allowNone = allowNone
+        self.useDateTime = useDateTime
+        self.connectTimeout = connectTimeout
+        self._reactor = reactor
+
+
+    def callRemote(self, method, *args):
+        """
+        Call remote XML-RPC C{method} with given arguments.
+
+        @return: a L{defer.Deferred} that will fire with the method response,
+            or a failure if the method failed. Generally, the failure type will
+            be L{Fault}, but you can also have an C{IndexError} on some buggy
+            servers giving empty responses.
+
+            If the deferred is cancelled before the request completes, the
+            connection is closed and the deferred will fire with a
+            L{defer.CancelledError}.
+        """
+        def cancel(d):
+            factory.deferred = None
+            connector.disconnect()
+        factory = self.queryFactory(
+            self.path, self.host, method, self.user,
+            self.password, self.allowNone, args, cancel, self.useDateTime)
+
+        if self.secure:
+            from twisted.internet import ssl
+            connector = self._reactor.connectSSL(
+                nativeString(self.host), self.port or 443,
+                factory, ssl.ClientContextFactory(),
+                timeout=self.connectTimeout)
+        else:
+            connector = self._reactor.connectTCP(
+                nativeString(self.host), self.port or 80, factory,
+                timeout=self.connectTimeout)
+        return factory.deferred
+
+
+__all__ = [
+    "XMLRPC", "Handler", "NoSuchFunction", "Proxy",
+
+    "Fault", "Binary", "Boolean", "DateTime"]
diff --git a/contrib/python/Twisted/py2/twisted/words/__init__.py b/contrib/python/Twisted/py2/twisted/words/__init__.py
new file mode 100644
index 00000000000..454119778f7
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/__init__.py
@@ -0,0 +1,8 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Words: Client and server implementations for IRC, XMPP, and other chat
+services.
+"""
diff --git a/contrib/python/Twisted/py2/twisted/words/ewords.py b/contrib/python/Twisted/py2/twisted/words/ewords.py
new file mode 100644
index 00000000000..7621a71bdfe
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/ewords.py
@@ -0,0 +1,34 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""Exception definitions for Words
+"""
+
+class WordsError(Exception):
+    def __str__(self):
+        return self.__class__.__name__ + ': ' + Exception.__str__(self)
+
+class NoSuchUser(WordsError):
+    pass
+
+
+class DuplicateUser(WordsError):
+    pass
+
+
+class NoSuchGroup(WordsError):
+    pass
+
+
+class DuplicateGroup(WordsError):
+    pass
+
+
+class AlreadyLoggedIn(WordsError):
+    pass
+
+__all__ = [
+    'WordsError', 'NoSuchUser', 'DuplicateUser',
+    'NoSuchGroup', 'DuplicateGroup', 'AlreadyLoggedIn',
+    ]
diff --git a/contrib/python/Twisted/py2/twisted/words/im/__init__.py b/contrib/python/Twisted/py2/twisted/words/im/__init__.py
new file mode 100644
index 00000000000..f2ef3b83bf6
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/im/__init__.py
@@ -0,0 +1,8 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Instance Messenger, Pan-protocol chat client.
+"""
+
diff --git a/contrib/python/Twisted/py2/twisted/words/im/baseaccount.py b/contrib/python/Twisted/py2/twisted/words/im/baseaccount.py
new file mode 100644
index 00000000000..0261dbf1e33
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/im/baseaccount.py
@@ -0,0 +1,62 @@
+# -*- Python -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+
+class AccountManager:
+    """I am responsible for managing a user's accounts.
+
+    That is, remembering what accounts are available, their settings,
+    adding and removal of accounts, etc.
+
+    @ivar accounts: A collection of available accounts.
+    @type accounts: mapping of strings to L{Account<interfaces.IAccount>}s.
+    """
+    def __init__(self):
+        self.accounts = {}
+
+    def getSnapShot(self):
+        """A snapshot of all the accounts and their status.
+
+        @returns: A list of tuples, each of the form
+            (string:accountName, boolean:isOnline,
+            boolean:autoLogin, string:gatewayType)
+        """
+        data = []
+        for account in self.accounts.values():
+            data.append((account.accountName, account.isOnline(),
+                         account.autoLogin, account.gatewayType))
+        return data
+
+    def isEmpty(self):
+        return len(self.accounts) == 0
+
+    def getConnectionInfo(self):
+        connectioninfo = []
+        for account in self.accounts.values():
+            connectioninfo.append(account.isOnline())
+        return connectioninfo
+
+    def addAccount(self, account):
+        self.accounts[account.accountName] = account
+
+    def delAccount(self, accountName):
+        del self.accounts[accountName]
+
+    def connect(self, accountName, chatui):
+        """
+        @returntype: Deferred L{interfaces.IClient}
+        """
+        return self.accounts[accountName].logOn(chatui)
+
+    def disconnect(self, accountName):
+        pass
+        #self.accounts[accountName].logOff()  - not yet implemented
+
+    def quit(self):
+        pass
+        #for account in self.accounts.values():
+        #    account.logOff()  - not yet implemented
diff --git a/contrib/python/Twisted/py2/twisted/words/im/basechat.py b/contrib/python/Twisted/py2/twisted/words/im/basechat.py
new file mode 100644
index 00000000000..ef4c25af9c5
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/im/basechat.py
@@ -0,0 +1,512 @@
+# -*- test-case-name: twisted.words.test.test_basechat -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Base classes for Instance Messenger clients.
+"""
+
+from twisted.words.im.locals import OFFLINE, ONLINE, AWAY
+
+
+class ContactsList:
+    """
+    A GUI object that displays a contacts list.
+
+    @ivar chatui: The GUI chat client associated with this contacts list.
+    @type chatui: L{ChatUI}
+
+    @ivar contacts: The contacts.
+    @type contacts: C{dict} mapping C{str} to a L{IPerson<interfaces.IPerson>}
+        provider
+
+    @ivar onlineContacts: The contacts who are currently online (have a status
+        that is not C{OFFLINE}).
+    @type onlineContacts: C{dict} mapping C{str} to a
+        L{IPerson<interfaces.IPerson>} provider
+
+    @ivar clients: The signed-on clients.
+    @type clients: C{list} of L{IClient<interfaces.IClient>} providers
+    """
+    def __init__(self, chatui):
+        """
+        @param chatui: The GUI chat client associated with this contacts list.
+        @type chatui: L{ChatUI}
+        """
+        self.chatui = chatui
+        self.contacts = {}
+        self.onlineContacts = {}
+        self.clients = []
+
+
+    def setContactStatus(self, person):
+        """
+        Inform the user that a person's status has changed.
+
+        @param person: The person whose status has changed.
+        @type person: L{IPerson<interfaces.IPerson>} provider
+        """
+        if person.name not in self.contacts:
+            self.contacts[person.name] = person
+        if person.name not in self.onlineContacts and \
+                (person.status == ONLINE or person.status == AWAY):
+            self.onlineContacts[person.name] = person
+        if person.name in self.onlineContacts and \
+                person.status == OFFLINE:
+            del self.onlineContacts[person.name]
+
+
+    def registerAccountClient(self, client):
+        """
+        Notify the user that an account client has been signed on to.
+
+        @param client: The client being added to your list of account clients.
+        @type client: L{IClient<interfaces.IClient>} provider
+        """
+        if not client in self.clients:
+            self.clients.append(client)
+
+
+    def unregisterAccountClient(self, client):
+        """
+        Notify the user that an account client has been signed off or
+        disconnected from.
+
+        @param client: The client being removed from the list of account
+            clients.
+        @type client: L{IClient<interfaces.IClient>} provider
+        """
+        if client in self.clients:
+            self.clients.remove(client)
+
+
+    def contactChangedNick(self, person, newnick):
+        """
+        Update your contact information to reflect a change to a contact's
+        nickname.
+
+        @param person: The person in your contacts list whose nickname is
+            changing.
+        @type person: L{IPerson<interfaces.IPerson>} provider
+
+        @param newnick: The new nickname for this person.
+        @type newnick: C{str}
+        """
+        oldname = person.name
+        if oldname in self.contacts:
+            del self.contacts[oldname]
+            person.name = newnick
+            self.contacts[newnick] = person
+            if oldname in self.onlineContacts:
+                del self.onlineContacts[oldname]
+                self.onlineContacts[newnick] = person
+
+
+
+class Conversation:
+    """
+    A GUI window of a conversation with a specific person.
+
+    @ivar person: The person who you're having this conversation with.
+    @type person: L{IPerson<interfaces.IPerson>} provider
+
+    @ivar chatui: The GUI chat client associated with this conversation.
+    @type chatui: L{ChatUI}
+    """
+    def __init__(self, person, chatui):
+        """
+        @param person: The person who you're having this conversation with.
+        @type person: L{IPerson<interfaces.IPerson>} provider
+
+        @param chatui: The GUI chat client associated with this conversation.
+        @type chatui: L{ChatUI}
+        """
+        self.chatui = chatui
+        self.person = person
+
+
+    def show(self):
+        """
+        Display the ConversationWindow.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def hide(self):
+        """
+        Hide the ConversationWindow.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def sendText(self, text):
+        """
+        Send text to the person with whom the user is conversing.
+
+        @param text: The text to be sent.
+        @type text: C{str}
+        """
+        self.person.sendMessage(text, None)
+
+
+    def showMessage(self, text, metadata=None):
+        """
+        Display a message sent from the person with whom the user is conversing.
+
+        @param text: The sent message.
+        @type text: C{str}
+
+        @param metadata: Metadata associated with this message.
+        @type metadata: C{dict}
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def contactChangedNick(self, person, newnick):
+        """
+        Change a person's name.
+
+        @param person: The person whose nickname is changing.
+        @type person: L{IPerson<interfaces.IPerson>} provider
+
+        @param newnick: The new nickname for this person.
+        @type newnick: C{str}
+        """
+        self.person.name = newnick
+
+
+
+class GroupConversation:
+    """
+    A GUI window of a conversation with a group of people.
+
+    @ivar chatui: The GUI chat client associated with this conversation.
+    @type chatui: L{ChatUI}
+
+    @ivar group: The group of people that are having this conversation.
+    @type group: L{IGroup<interfaces.IGroup>} provider
+
+    @ivar members: The names of the people in this conversation.
+    @type members: C{list} of C{str}
+    """
+    def __init__(self, group, chatui):
+        """
+        @param chatui: The GUI chat client associated with this conversation.
+        @type chatui: L{ChatUI}
+
+        @param group: The group of people that are having this conversation.
+        @type group: L{IGroup<interfaces.IGroup>} provider
+        """
+        self.chatui = chatui
+        self.group = group
+        self.members = []
+
+
+    def show(self):
+        """
+        Display the GroupConversationWindow.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def hide(self):
+        """
+        Hide the GroupConversationWindow.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def sendText(self, text):
+        """
+        Send text to the group.
+
+        @param: The text to be sent.
+        @type text: C{str}
+        """
+        self.group.sendGroupMessage(text, None)
+
+
+    def showGroupMessage(self, sender, text, metadata=None):
+        """
+        Display to the user a message sent to this group from the given sender.
+
+        @param sender: The person sending the message.
+        @type sender: C{str}
+
+        @param text: The sent message.
+        @type text: C{str}
+
+        @param metadata: Metadata associated with this message.
+        @type metadata: C{dict}
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def setGroupMembers(self, members):
+        """
+        Set the list of members in the group.
+
+        @param members: The names of the people that will be in this group.
+        @type members: C{list} of C{str}
+        """
+        self.members = members
+
+
+    def setTopic(self, topic, author):
+        """
+        Change the topic for the group conversation window and display this
+        change to the user.
+
+        @param topic: This group's topic.
+        @type topic: C{str}
+
+        @param author: The person changing the topic.
+        @type author: C{str}
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def memberJoined(self, member):
+        """
+        Add the given member to the list of members in the group conversation
+        and displays this to the user.
+
+        @param member: The person joining the group conversation.
+        @type member: C{str}
+        """
+        if not member in self.members:
+            self.members.append(member)
+
+
+    def memberChangedNick(self, oldnick, newnick):
+        """
+        Change the nickname for a member of the group conversation and displays
+        this change to the user.
+
+        @param oldnick: The old nickname.
+        @type oldnick: C{str}
+
+        @param newnick: The new nickname.
+        @type newnick: C{str}
+        """
+        if oldnick in self.members:
+            self.members.remove(oldnick)
+            self.members.append(newnick)
+
+
+    def memberLeft(self, member):
+        """
+        Delete the given member from the list of members in the group
+        conversation and displays the change to the user.
+
+        @param member: The person leaving the group conversation.
+        @type member: C{str}
+        """
+        if member in self.members:
+            self.members.remove(member)
+
+
+
+class ChatUI:
+    """
+    A GUI chat client.
+
+    @type conversations: C{dict} of L{Conversation}
+    @ivar conversations: A cache of all the direct windows.
+
+    @type groupConversations: C{dict} of L{GroupConversation}
+    @ivar groupConversations: A cache of all the group windows.
+
+    @type persons: C{dict} with keys that are a C{tuple} of (C{str},
+       L{IAccount<interfaces.IAccount>} provider) and values that are
+       L{IPerson<interfaces.IPerson>} provider
+    @ivar persons: A cache of all the users associated with this client.
+
+    @type groups: C{dict} with keys that are a C{tuple} of (C{str},
+        L{IAccount<interfaces.IAccount>} provider) and values that are
+        L{IGroup<interfaces.IGroup>} provider
+    @ivar groups: A cache of all the groups associated with this client.
+
+    @type onlineClients: C{list} of L{IClient<interfaces.IClient>} providers
+    @ivar onlineClients: A list of message sources currently online.
+
+    @type contactsList: L{ContactsList}
+    @ivar contactsList: A contacts list.
+    """
+    def __init__(self):
+        self.conversations = {}
+        self.groupConversations = {}
+        self.persons = {}
+        self.groups = {}
+        self.onlineClients = []
+        self.contactsList = ContactsList(self)
+
+
+    def registerAccountClient(self, client):
+        """
+        Notify the user that an account has been signed on to.
+
+        @type client: L{IClient<interfaces.IClient>} provider
+        @param client: The client account for the person who has just signed on.
+
+        @rtype client: L{IClient<interfaces.IClient>} provider
+        @return: The client, so that it may be used in a callback chain.
+        """
+        self.onlineClients.append(client)
+        self.contactsList.registerAccountClient(client)
+        return client
+
+
+    def unregisterAccountClient(self, client):
+        """
+        Notify the user that an account has been signed off or disconnected.
+
+        @type client: L{IClient<interfaces.IClient>} provider
+        @param client: The client account for the person who has just signed
+            off.
+        """
+        self.onlineClients.remove(client)
+        self.contactsList.unregisterAccountClient(client)
+
+
+    def getContactsList(self):
+        """
+        Get the contacts list associated with this chat window.
+
+        @rtype: L{ContactsList}
+        @return: The contacts list associated with this chat window.
+        """
+        return self.contactsList
+
+
+    def getConversation(self, person, Class=Conversation, stayHidden=False):
+        """
+        For the given person object, return the conversation window or create
+        and return a new conversation window if one does not exist.
+
+        @type person: L{IPerson<interfaces.IPerson>} provider
+        @param person: The person whose conversation window we want to get.
+
+        @type Class: L{IConversation<interfaces.IConversation>} implementor
+        @param: The kind of conversation window we want. If the conversation
+            window for this person didn't already exist, create one of this type.
+
+        @type stayHidden: C{bool}
+        @param stayHidden: Whether or not the conversation window should stay
+            hidden.
+
+        @rtype: L{IConversation<interfaces.IConversation>} provider
+        @return: The conversation window.
+        """
+        conv = self.conversations.get(person)
+        if not conv:
+            conv = Class(person, self)
+            self.conversations[person] = conv
+        if stayHidden:
+            conv.hide()
+        else:
+            conv.show()
+        return conv
+
+
+    def getGroupConversation(self, group, Class=GroupConversation,
+                             stayHidden=False):
+        """
+        For the given group object, return the group conversation window or
+        create and return a new group conversation window if it doesn't exist.
+
+        @type group: L{IGroup<interfaces.IGroup>} provider
+        @param group: The group whose conversation window we want to get.
+
+        @type Class: L{IConversation<interfaces.IConversation>} implementor
+        @param: The kind of conversation window we want. If the conversation
+            window for this person didn't already exist, create one of this type.
+
+        @type stayHidden: C{bool}
+        @param stayHidden: Whether or not the conversation window should stay
+            hidden.
+
+        @rtype: L{IGroupConversation<interfaces.IGroupConversation>} provider
+        @return: The group conversation window.
+        """
+        conv = self.groupConversations.get(group)
+        if not conv:
+            conv = Class(group, self)
+            self.groupConversations[group] = conv
+        if stayHidden:
+            conv.hide()
+        else:
+            conv.show()
+        return conv
+
+
+    def getPerson(self, name, client):
+        """
+        For the given name and account client, return an instance of a
+        L{IGroup<interfaces.IPerson>} provider or create and return a new
+        instance of a L{IGroup<interfaces.IPerson>} provider.
+
+        @type name: C{str}
+        @param name: The name of the person of interest.
+
+        @type client: L{IClient<interfaces.IClient>} provider
+        @param client: The client account of interest.
+
+        @rtype: L{IPerson<interfaces.IPerson>} provider
+        @return: The person with that C{name}.
+        """
+        account = client.account
+        p = self.persons.get((name, account))
+        if not p:
+            p = account.getPerson(name)
+            self.persons[name, account] = p
+        return p
+
+
+    def getGroup(self, name, client):
+        """
+        For the given name and account client, return an instance of a
+        L{IGroup<interfaces.IGroup>} provider or create and return a new instance
+        of a L{IGroup<interfaces.IGroup>} provider.
+
+        @type name: C{str}
+        @param name: The name of the group of interest.
+
+        @type client: L{IClient<interfaces.IClient>} provider
+        @param client: The client account of interest.
+
+        @rtype: L{IGroup<interfaces.IGroup>} provider
+        @return: The group with that C{name}.
+        """
+        # I accept 'client' instead of 'account' in my signature for
+        # backwards compatibility.  (Groups changed to be Account-oriented
+        # in CVS revision 1.8.)
+        account = client.account
+        g = self.groups.get((name, account))
+        if not g:
+            g = account.getGroup(name)
+            self.groups[name, account] = g
+        return g
+
+
+    def contactChangedNick(self, person, newnick):
+        """
+        For the given C{person}, change the C{person}'s C{name} to C{newnick}
+        and tell the contact list and any conversation windows with that
+        C{person} to change as well.
+
+        @type person: L{IPerson<interfaces.IPerson>} provider
+        @param person: The person whose nickname will get changed.
+
+        @type newnick: C{str}
+        @param newnick: The new C{name} C{person} will take.
+        """
+        oldnick = person.name
+        if (oldnick, person.account) in self.persons:
+            conv = self.conversations.get(person)
+            if conv:
+                conv.contactChangedNick(person, newnick)
+            self.contactsList.contactChangedNick(person, newnick)
+            del self.persons[oldnick, person.account]
+            person.name = newnick
+            self.persons[person.name, person.account] = person
diff --git a/contrib/python/Twisted/py2/twisted/words/im/basesupport.py b/contrib/python/Twisted/py2/twisted/words/im/basesupport.py
new file mode 100644
index 00000000000..43b4c1ac37f
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/im/basesupport.py
@@ -0,0 +1,269 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+"""Instance Messenger base classes for protocol support.
+
+You will find these useful if you're adding a new protocol to IM.
+"""
+
+# Abstract representation of chat "model" classes
+
+from twisted.words.im.locals import OFFLINE, OfflineError
+
+from twisted.internet.protocol import Protocol
+
+from twisted.python.reflect import prefixedMethods
+from twisted.persisted import styles
+
+from twisted.internet import error
+
+class AbstractGroup:
+    def __init__(self, name, account):
+        self.name = name
+        self.account = account
+
+    def getGroupCommands(self):
+        """finds group commands
+
+        these commands are methods on me that start with imgroup_; they are
+        called with no arguments
+        """
+        return prefixedMethods(self, "imgroup_")
+
+    def getTargetCommands(self, target):
+        """finds group commands
+
+        these commands are methods on me that start with imgroup_; they are
+        called with a user present within this room as an argument
+
+        you may want to override this in your group in order to filter for
+        appropriate commands on the given user
+        """
+        return prefixedMethods(self, "imtarget_")
+
+    def join(self):
+        if not self.account.client:
+            raise OfflineError
+        self.account.client.joinGroup(self.name)
+
+    def leave(self):
+        if not self.account.client:
+            raise OfflineError
+        self.account.client.leaveGroup(self.name)
+
+    def __repr__(self):
+        return '<%s %r>' % (self.__class__, self.name)
+
+    def __str__(self):
+        return '%s@%s' % (self.name, self.account.accountName)
+
+class AbstractPerson:
+    def __init__(self, name, baseAccount):
+        self.name = name
+        self.account = baseAccount
+        self.status = OFFLINE
+
+    def getPersonCommands(self):
+        """finds person commands
+
+        these commands are methods on me that start with imperson_; they are
+        called with no arguments
+        """
+        return prefixedMethods(self, "imperson_")
+
+    def getIdleTime(self):
+        """
+        Returns a string.
+        """
+        return '--'
+
+    def __repr__(self):
+        return '<%s %r/%s>' % (self.__class__, self.name, self.status)
+
+    def __str__(self):
+        return '%s@%s' % (self.name, self.account.accountName)
+
+class AbstractClientMixin:
+    """Designed to be mixed in to a Protocol implementing class.
+
+    Inherit from me first.
+
+    @ivar _logonDeferred: Fired when I am done logging in.
+    """
+    def __init__(self, account, chatui, logonDeferred):
+        for base in self.__class__.__bases__:
+            if issubclass(base, Protocol):
+                self.__class__._protoBase = base
+                break
+        else:
+            pass
+        self.account = account
+        self.chat = chatui
+        self._logonDeferred = logonDeferred
+
+    def connectionMade(self):
+        self._protoBase.connectionMade(self)
+
+    def connectionLost(self, reason):
+        self.account._clientLost(self, reason)
+        self.unregisterAsAccountClient()
+        return self._protoBase.connectionLost(self, reason)
+
+    def unregisterAsAccountClient(self):
+        """Tell the chat UI that I have `signed off'.
+        """
+        self.chat.unregisterAccountClient(self)
+
+
+class AbstractAccount(styles.Versioned):
+    """Base class for Accounts.
+
+    I am the start of an implementation of L{IAccount<interfaces.IAccount>}, I
+    implement L{isOnline} and most of L{logOn}, though you'll need to implement
+    L{_startLogOn} in a subclass.
+
+    @cvar _groupFactory: A Callable that will return a L{IGroup} appropriate
+        for this account type.
+    @cvar _personFactory: A Callable that will return a L{IPerson} appropriate
+        for this account type.
+
+    @type _isConnecting: boolean
+    @ivar _isConnecting: Whether I am in the process of establishing a
+    connection to the server.
+    @type _isOnline: boolean
+    @ivar _isOnline: Whether I am currently on-line with the server.
+
+    @ivar accountName:
+    @ivar autoLogin:
+    @ivar username:
+    @ivar password:
+    @ivar host:
+    @ivar port:
+    """
+
+    _isOnline = 0
+    _isConnecting = 0
+    client = None
+
+    _groupFactory = AbstractGroup
+    _personFactory = AbstractPerson
+
+    persistanceVersion = 2
+
+    def __init__(self, accountName, autoLogin, username, password, host, port):
+        self.accountName = accountName
+        self.autoLogin = autoLogin
+        self.username = username
+        self.password = password
+        self.host = host
+        self.port = port
+
+        self._groups = {}
+        self._persons = {}
+
+    def upgrateToVersion2(self):
+        # Added in CVS revision 1.16.
+        for k in ('_groups', '_persons'):
+            if not hasattr(self, k):
+                setattr(self, k, {})
+
+    def __getstate__(self):
+        state = styles.Versioned.__getstate__(self)
+        for k in ('client', '_isOnline', '_isConnecting'):
+            try:
+                del state[k]
+            except KeyError:
+                pass
+        return state
+
+    def isOnline(self):
+        return self._isOnline
+
+    def logOn(self, chatui):
+        """Log on to this account.
+
+        Takes care to not start a connection if a connection is
+        already in progress.  You will need to implement
+        L{_startLogOn} for this to work, and it would be a good idea
+        to override L{_loginFailed} too.
+
+        @returntype: Deferred L{interfaces.IClient}
+        """
+        if (not self._isConnecting) and (not self._isOnline):
+            self._isConnecting = 1
+            d = self._startLogOn(chatui)
+            d.addCallback(self._cb_logOn)
+            # if chatui is not None:
+            # (I don't particularly like having to pass chatUI to this function,
+            # but we haven't factored it out yet.)
+            d.addCallback(chatui.registerAccountClient)
+            d.addErrback(self._loginFailed)
+            return d
+        else:
+            raise error.ConnectError("Connection in progress")
+
+    def getGroup(self, name):
+        """Group factory.
+
+        @param name: Name of the group on this account.
+        @type name: string
+        """
+        group = self._groups.get(name)
+        if group is None:
+            group = self._groupFactory(name, self)
+            self._groups[name] = group
+        return group
+
+    def getPerson(self, name):
+        """Person factory.
+
+        @param name: Name of the person on this account.
+        @type name: string
+        """
+        person = self._persons.get(name)
+        if person is None:
+            person = self._personFactory(name, self)
+            self._persons[name] = person
+        return person
+
+    def _startLogOn(self, chatui):
+        """Start the sign on process.
+
+        Factored out of L{logOn}.
+
+        @returntype: Deferred L{interfaces.IClient}
+        """
+        raise NotImplementedError()
+
+    def _cb_logOn(self, client):
+        self._isConnecting = 0
+        self._isOnline = 1
+        self.client = client
+        return client
+
+    def _loginFailed(self, reason):
+        """Errorback for L{logOn}.
+
+        @type reason: Failure
+
+        @returns: I{reason}, for further processing in the callback chain.
+        @returntype: Failure
+        """
+        self._isConnecting = 0
+        self._isOnline = 0 # just in case
+        return reason
+
+    def _clientLost(self, client, reason):
+        self.client = None
+        self._isConnecting = 0
+        self._isOnline = 0
+        return reason
+
+    def __repr__(self):
+        return "<%s: %s (%s@%s:%s)>" % (self.__class__,
+                                        self.accountName,
+                                        self.username,
+                                        self.host,
+                                        self.port)
diff --git a/contrib/python/Twisted/py2/twisted/words/im/interfaces.py b/contrib/python/Twisted/py2/twisted/words/im/interfaces.py
new file mode 100644
index 00000000000..fa2b0658593
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/im/interfaces.py
@@ -0,0 +1,398 @@
+# -*- Python -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Pan-protocol chat client.
+"""
+
+from zope.interface import Interface, Attribute
+
+# (Random musings, may not reflect on current state of code:)
+#
+# Accounts have Protocol components (clients)
+# Persons have Conversation components
+# Groups have GroupConversation components
+# Persons and Groups are associated with specific Accounts
+# At run-time, Clients/Accounts are slaved to a User Interface
+#   (Note: User may be a bot, so don't assume all UIs are built on gui toolkits)
+
+
+class IAccount(Interface):
+    """
+    I represent a user's account with a chat service.
+    """
+
+    client = Attribute('The L{IClient} currently connecting to this account, if any.')
+    gatewayType = Attribute('A C{str} that identifies the protocol used by this account.')
+
+    def __init__(accountName, autoLogin, username, password, host, port):
+        """
+        @type accountName: string
+        @param accountName: A name to refer to the account by locally.
+        @type autoLogin: boolean
+        @type username: string
+        @type password: string
+        @type host: string
+        @type port: integer
+        """
+
+    def isOnline():
+        """
+        Am I online?
+
+        @rtype: boolean
+        """
+
+    def logOn(chatui):
+        """
+        Go on-line.
+
+        @type chatui: Implementor of C{IChatUI}
+
+        @rtype: L{Deferred} with an eventual L{IClient} result.
+        """
+
+    def logOff():
+        """
+        Sign off.
+        """
+
+    def getGroup(groupName):
+        """
+        @rtype: L{Group<IGroup>}
+        """
+
+    def getPerson(personName):
+        """
+        @rtype: L{Person<IPerson>}
+        """
+
+
+
+class IClient(Interface):
+
+    account = Attribute('The L{IAccount} I am a Client for')
+
+    def __init__(account, chatui, logonDeferred):
+        """
+        @type account: L{IAccount}
+        @type chatui: L{IChatUI}
+        @param logonDeferred: Will be called back once I am logged on.
+        @type logonDeferred: L{Deferred<twisted.internet.defer.Deferred>}
+        """
+
+    def joinGroup(groupName):
+        """
+        @param groupName: The name of the group to join.
+        @type groupName: string
+        """
+
+    def leaveGroup(groupName):
+        """
+        @param groupName: The name of the group to leave.
+        @type groupName: string
+        """
+
+    def getGroupConversation(name, hide=0):
+        pass
+
+
+    def getPerson(name):
+        pass
+
+
+
+class IPerson(Interface):
+
+    def __init__(name, account):
+        """
+        Initialize me.
+
+        @param name: My name, as the server knows me.
+        @type name: string
+        @param account: The account I am accessed through.
+        @type account: I{Account}
+        """
+
+
+    def isOnline():
+        """
+        Am I online right now?
+
+        @rtype: boolean
+        """
+
+
+    def getStatus():
+        """
+        What is my on-line status?
+
+        @return: L{locals.StatusEnum}
+        """
+
+
+    def getIdleTime():
+        """
+        @rtype: string (XXX: How about a scalar?)
+        """
+
+
+    def sendMessage(text, metadata=None):
+        """
+        Send a message to this person.
+
+        @type text: string
+        @type metadata: dict
+        """
+
+
+
+class IGroup(Interface):
+    """
+    A group which you may have a conversation with.
+
+    Groups generally have a loosely-defined set of members, who may
+    leave and join at any time.
+    """
+
+    name = Attribute('My C{str} name, as the server knows me.')
+    account = Attribute('The L{Account<IAccount>} I am accessed through.')
+
+    def __init__(name, account):
+        """
+        Initialize me.
+
+        @param name: My name, as the server knows me.
+        @type name: str
+        @param account: The account I am accessed through.
+        @type account: L{Account<IAccount>}
+        """
+
+
+    def setTopic(text):
+        """
+        Set this Groups topic on the server.
+
+        @type text: string
+        """
+
+
+    def sendGroupMessage(text, metadata=None):
+        """
+        Send a message to this group.
+
+        @type text: str
+
+        @type metadata: dict
+        @param metadata: Valid keys for this dictionary include:
+
+            - C{'style'}: associated with one of:
+                - C{'emote'}: indicates this is an action
+        """
+
+
+    def join():
+        """
+        Join this group.
+        """
+
+
+    def leave():
+        """
+        Depart this group.
+        """
+
+
+
+class IConversation(Interface):
+    """
+    A conversation with a specific person.
+    """
+
+    def __init__(person, chatui):
+        """
+        @type person: L{IPerson}
+        """
+
+
+    def show():
+        """
+        doesn't seem like it belongs in this interface.
+        """
+
+
+    def hide():
+        """
+        nor this neither.
+        """
+
+
+    def sendText(text, metadata):
+        pass
+
+
+    def showMessage(text, metadata):
+        pass
+
+
+    def changedNick(person, newnick):
+        """
+        @param person: XXX Shouldn't this always be Conversation.person?
+        """
+
+
+
+class IGroupConversation(Interface):
+
+    def show():
+        """
+        doesn't seem like it belongs in this interface.
+        """
+
+
+    def hide():
+        """
+        nor this neither.
+        """
+
+
+    def sendText(text, metadata):
+        pass
+
+
+    def showGroupMessage(sender, text, metadata):
+        pass
+
+
+    def setGroupMembers(members):
+        """
+        Sets the list of members in the group and displays it to the user.
+        """
+
+
+    def setTopic(topic, author):
+        """
+        Displays the topic (from the server) for the group conversation window.
+
+        @type topic: string
+        @type author: string (XXX: Not Person?)
+        """
+
+
+    def memberJoined(member):
+        """
+        Adds the given member to the list of members in the group conversation
+        and displays this to the user,
+
+        @type member: string (XXX: Not Person?)
+        """
+
+
+    def memberChangedNick(oldnick, newnick):
+        """
+        Changes the oldnick in the list of members to C{newnick} and displays this
+        change to the user,
+
+        @type oldnick: string (XXX: Not Person?)
+        @type newnick: string
+        """
+
+
+    def memberLeft(member):
+        """
+        Deletes the given member from the list of members in the group
+        conversation and displays the change to the user.
+
+        @type member: string (XXX: Not Person?)
+        """
+
+
+
+class IChatUI(Interface):
+
+    def registerAccountClient(client):
+        """
+        Notifies user that an account has been signed on to.
+
+        @type client: L{Client<IClient>}
+        """
+
+
+    def unregisterAccountClient(client):
+        """
+        Notifies user that an account has been signed off or disconnected.
+
+        @type client: L{Client<IClient>}
+        """
+
+
+    def getContactsList():
+        """
+        @rtype: L{ContactsList}
+        """
+
+    # WARNING: You'll want to be polymorphed into something with
+    # intrinsic stoning resistance before continuing.
+
+    def getConversation(person, Class, stayHidden=0):
+        """
+        For the given person object, returns the conversation window
+        or creates and returns a new conversation window if one does not exist.
+
+        @type person: L{Person<IPerson>}
+        @type Class: L{Conversation<IConversation>} class
+        @type stayHidden: boolean
+
+        @rtype: L{Conversation<IConversation>}
+        """
+
+
+    def getGroupConversation(group, Class, stayHidden=0):
+        """
+        For the given group object, returns the group conversation window or
+        creates and returns a new group conversation window if it doesn't exist.
+
+        @type group: L{Group<interfaces.IGroup>}
+        @type Class: L{Conversation<interfaces.IConversation>} class
+        @type stayHidden: boolean
+
+        @rtype: L{GroupConversation<interfaces.IGroupConversation>}
+        """
+
+
+    def getPerson(name, client):
+        """
+        Get a Person for a client.
+
+        Duplicates L{IAccount.getPerson}.
+
+        @type name: string
+        @type client: L{Client<IClient>}
+
+        @rtype: L{Person<IPerson>}
+        """
+
+
+    def getGroup(name, client):
+        """
+        Get a Group for a client.
+
+        Duplicates L{IAccount.getGroup}.
+
+        @type name: string
+        @type client: L{Client<IClient>}
+
+        @rtype: L{Group<IGroup>}
+        """
+
+
+    def contactChangedNick(oldnick, newnick):
+        """
+        For the given person, changes the person's name to newnick, and
+        tells the contact list and any conversation windows with that person
+        to change as well.
+
+        @type oldnick: string
+        @type newnick: string
+        """
diff --git a/contrib/python/Twisted/py2/twisted/words/im/ircsupport.py b/contrib/python/Twisted/py2/twisted/words/im/ircsupport.py
new file mode 100644
index 00000000000..e16bfcd0d77
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/im/ircsupport.py
@@ -0,0 +1,293 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+IRC support for Instance Messenger.
+"""
+
+from twisted.words.protocols import irc
+from twisted.words.im.locals import ONLINE
+from twisted.internet import defer, reactor, protocol
+from twisted.internet.defer import succeed
+from twisted.words.im import basesupport, interfaces, locals
+from zope.interface import implementer
+
+
+class IRCPerson(basesupport.AbstractPerson):
+
+    def imperson_whois(self):
+        if self.account.client is None:
+            raise locals.OfflineError
+        self.account.client.sendLine("WHOIS %s" % self.name)
+
+
+    ### interface impl
+    def isOnline(self):
+        return ONLINE
+
+
+    def getStatus(self):
+        return ONLINE
+
+
+    def setStatus(self,status):
+        self.status=status
+        self.chat.getContactsList().setContactStatus(self)
+
+
+    def sendMessage(self, text, meta=None):
+        if self.account.client is None:
+            raise locals.OfflineError
+        for line in text.split('\n'):
+            if meta and meta.get("style", None) == "emote":
+                self.account.client.ctcpMakeQuery(self.name,[('ACTION', line)])
+            else:
+                self.account.client.msg(self.name, line)
+        return succeed(text)
+
+
+
+@implementer(interfaces.IGroup)
+class IRCGroup(basesupport.AbstractGroup):
+    def imgroup_testAction(self):
+        pass
+
+
+    def imtarget_kick(self, target):
+        if self.account.client is None:
+            raise locals.OfflineError
+        reason = "for great justice!"
+        self.account.client.sendLine("KICK #%s %s :%s" % (
+            self.name, target.name, reason))
+
+
+    ### Interface Implementation
+    def setTopic(self, topic):
+        if self.account.client is None:
+            raise locals.OfflineError
+        self.account.client.topic(self.name, topic)
+
+
+    def sendGroupMessage(self, text, meta={}):
+        if self.account.client is None:
+            raise locals.OfflineError
+        if meta and meta.get("style", None) == "emote":
+            self.account.client.ctcpMakeQuery(self.name,[('ACTION', text)])
+            return succeed(text)
+        #standard shmandard, clients don't support plain escaped newlines!
+        for line in text.split('\n'):
+            self.account.client.say(self.name, line)
+        return succeed(text)
+
+
+    def leave(self):
+        if self.account.client is None:
+            raise locals.OfflineError
+        self.account.client.leave(self.name)
+        self.account.client.getGroupConversation(self.name,1)
+
+
+
+class IRCProto(basesupport.AbstractClientMixin, irc.IRCClient):
+    def __init__(self, account, chatui, logonDeferred=None):
+        basesupport.AbstractClientMixin.__init__(self, account, chatui,
+                                                 logonDeferred)
+        self._namreplies={}
+        self._ingroups={}
+        self._groups={}
+        self._topics={}
+
+
+    def getGroupConversation(self, name, hide=0):
+        name = name.lower()
+        return self.chat.getGroupConversation(self.chat.getGroup(name, self),
+                                              stayHidden=hide)
+
+
+    def getPerson(self,name):
+        return self.chat.getPerson(name, self)
+
+
+    def connectionMade(self):
+        # XXX: Why do I duplicate code in IRCClient.register?
+        try:
+            self.performLogin = True
+            self.nickname = self.account.username
+            self.password = self.account.password
+            self.realname = "Twisted-IM user"
+
+            irc.IRCClient.connectionMade(self)
+
+            for channel in self.account.channels:
+                self.joinGroup(channel)
+            self.account._isOnline=1
+            if self._logonDeferred is not None:
+                self._logonDeferred.callback(self)
+            self.chat.getContactsList()
+        except:
+            import traceback
+            traceback.print_exc()
+
+
+    def setNick(self,nick):
+        self.name=nick
+        self.accountName="%s (IRC)"%nick
+        irc.IRCClient.setNick(self,nick)
+
+
+    def kickedFrom(self, channel, kicker, message):
+        """
+        Called when I am kicked from a channel.
+        """
+        return self.chat.getGroupConversation(
+            self.chat.getGroup(channel[1:], self), 1)
+
+
+    def userKicked(self, kickee, channel, kicker, message):
+        pass
+
+
+    def noticed(self, username, channel, message):
+        self.privmsg(username, channel, message, {"dontAutoRespond": 1})
+
+
+    def privmsg(self, username, channel, message, metadata=None):
+        if metadata is None:
+            metadata = {}
+        username = username.split('!',1)[0]
+        if username==self.name: return
+        if channel[0]=='#':
+            group=channel[1:]
+            self.getGroupConversation(group).showGroupMessage(username, message, metadata)
+            return
+        self.chat.getConversation(self.getPerson(username)).showMessage(message, metadata)
+
+
+    def action(self,username,channel,emote):
+        username = username.split('!',1)[0]
+        if username==self.name: return
+        meta={'style':'emote'}
+        if channel[0]=='#':
+            group=channel[1:]
+            self.getGroupConversation(group).showGroupMessage(username, emote, meta)
+            return
+        self.chat.getConversation(self.getPerson(username)).showMessage(emote,meta)
+
+
+    def irc_RPL_NAMREPLY(self,prefix,params):
+        """
+        RPL_NAMREPLY
+        >> NAMES #bnl
+        << :Arlington.VA.US.Undernet.Org 353 z3p = #bnl :pSwede Dan-- SkOyg AG
+        """
+        group = params[2][1:].lower()
+        users = params[3].split()
+        for ui in range(len(users)):
+            while users[ui][0] in ["@","+"]: # channel modes
+                users[ui]=users[ui][1:]
+        if group not in self._namreplies:
+            self._namreplies[group]=[]
+        self._namreplies[group].extend(users)
+        for nickname in users:
+            try:
+                self._ingroups[nickname].append(group)
+            except:
+                self._ingroups[nickname]=[group]
+
+
+    def irc_RPL_ENDOFNAMES(self,prefix,params):
+        group=params[1][1:]
+        self.getGroupConversation(group).setGroupMembers(self._namreplies[group.lower()])
+        del self._namreplies[group.lower()]
+
+
+    def irc_RPL_TOPIC(self,prefix,params):
+        self._topics[params[1][1:]]=params[2]
+
+
+    def irc_333(self,prefix,params):
+        group=params[1][1:]
+        self.getGroupConversation(group).setTopic(self._topics[group],params[2])
+        del self._topics[group]
+
+
+    def irc_TOPIC(self,prefix,params):
+        nickname = prefix.split("!")[0]
+        group = params[0][1:]
+        topic = params[1]
+        self.getGroupConversation(group).setTopic(topic,nickname)
+
+
+    def irc_JOIN(self,prefix,params):
+        nickname = prefix.split("!")[0]
+        group = params[0][1:].lower()
+        if nickname!=self.nickname:
+            try:
+                self._ingroups[nickname].append(group)
+            except:
+                self._ingroups[nickname]=[group]
+            self.getGroupConversation(group).memberJoined(nickname)
+
+
+    def irc_PART(self,prefix,params):
+        nickname = prefix.split("!")[0]
+        group = params[0][1:].lower()
+        if nickname!=self.nickname:
+            if group in self._ingroups[nickname]:
+                self._ingroups[nickname].remove(group)
+                self.getGroupConversation(group).memberLeft(nickname)
+
+
+    def irc_QUIT(self,prefix,params):
+        nickname = prefix.split("!")[0]
+        if nickname in self._ingroups:
+            for group in self._ingroups[nickname]:
+                self.getGroupConversation(group).memberLeft(nickname)
+            self._ingroups[nickname]=[]
+
+
+    def irc_NICK(self, prefix, params):
+        fromNick = prefix.split("!")[0]
+        toNick = params[0]
+        if fromNick not in self._ingroups:
+            return
+        for group in self._ingroups[fromNick]:
+            self.getGroupConversation(group).memberChangedNick(fromNick, toNick)
+        self._ingroups[toNick] = self._ingroups[fromNick]
+        del self._ingroups[fromNick]
+
+
+    def irc_unknown(self, prefix, command, params):
+        pass
+
+
+    # GTKIM calls
+    def joinGroup(self,name):
+        self.join(name)
+        self.getGroupConversation(name)
+
+
+
+@implementer(interfaces.IAccount)
+class IRCAccount(basesupport.AbstractAccount):
+    gatewayType = "IRC"
+
+    _groupFactory = IRCGroup
+    _personFactory = IRCPerson
+
+    def __init__(self, accountName, autoLogin, username, password, host, port,
+                 channels=''):
+        basesupport.AbstractAccount.__init__(self, accountName, autoLogin,
+                                             username, password, host, port)
+        self.channels = [channel.strip() for channel in channels.split(',')]
+        if self.channels == ['']:
+            self.channels = []
+
+
+    def _startLogOn(self, chatui):
+        logonDeferred = defer.Deferred()
+        cc = protocol.ClientCreator(reactor, IRCProto, self, chatui,
+                                    logonDeferred)
+        d = cc.connectTCP(self.host, self.port)
+        d.addErrback(logonDeferred.errback)
+        return logonDeferred
diff --git a/contrib/python/Twisted/py2/twisted/words/im/locals.py b/contrib/python/Twisted/py2/twisted/words/im/locals.py
new file mode 100644
index 00000000000..a63547a3036
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/im/locals.py
@@ -0,0 +1,26 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+class Enum:
+    group = None
+
+    def __init__(self, label):
+        self.label = label
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.group, self.label)
+
+    def __str__(self):
+        return self.label
+
+
+class StatusEnum(Enum):
+    group = 'Status'
+
+OFFLINE = Enum('Offline')
+ONLINE = Enum('Online')
+AWAY = Enum('Away')
+
+class OfflineError(Exception):
+    """The requested action can't happen while offline."""
diff --git a/contrib/python/Twisted/py2/twisted/words/im/pbsupport.py b/contrib/python/Twisted/py2/twisted/words/im/pbsupport.py
new file mode 100644
index 00000000000..b30123ccfca
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/im/pbsupport.py
@@ -0,0 +1,262 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+L{twisted.words} support for Instance Messenger.
+"""
+
+from __future__ import print_function
+
+from twisted.internet import defer
+from twisted.internet import error
+from twisted.python import log
+from twisted.python.failure import Failure
+from twisted.spread import pb
+
+from twisted.words.im.locals import ONLINE, OFFLINE, AWAY
+
+from twisted.words.im import basesupport, interfaces
+from zope.interface import implementer
+
+
+class TwistedWordsPerson(basesupport.AbstractPerson):
+    """I a facade for a person you can talk to through a twisted.words service.
+    """
+    def __init__(self, name, wordsAccount):
+        basesupport.AbstractPerson.__init__(self, name, wordsAccount)
+        self.status = OFFLINE
+
+    def isOnline(self):
+        return ((self.status == ONLINE) or
+                (self.status == AWAY))
+
+    def getStatus(self):
+        return self.status
+
+    def sendMessage(self, text, metadata):
+        """Return a deferred...
+        """
+        if metadata:
+            d=self.account.client.perspective.directMessage(self.name,
+                                                            text, metadata)
+            d.addErrback(self.metadataFailed, "* "+text)
+            return d
+        else:
+            return self.account.client.perspective.callRemote('directMessage',self.name, text)
+
+    def metadataFailed(self, result, text):
+        print("result:",result,"text:",text)
+        return self.account.client.perspective.directMessage(self.name, text)
+
+    def setStatus(self, status):
+        self.status = status
+        self.chat.getContactsList().setContactStatus(self)
+
+@implementer(interfaces.IGroup)
+class TwistedWordsGroup(basesupport.AbstractGroup):
+    def __init__(self, name, wordsClient):
+        basesupport.AbstractGroup.__init__(self, name, wordsClient)
+        self.joined = 0
+
+    def sendGroupMessage(self, text, metadata=None):
+        """Return a deferred.
+        """
+        #for backwards compatibility with older twisted.words servers.
+        if metadata:
+            d=self.account.client.perspective.callRemote(
+                'groupMessage', self.name, text, metadata)
+            d.addErrback(self.metadataFailed, "* "+text)
+            return d
+        else:
+            return self.account.client.perspective.callRemote('groupMessage',
+                                                              self.name, text)
+
+    def setTopic(self, text):
+        self.account.client.perspective.callRemote(
+            'setGroupMetadata',
+            {'topic': text, 'topic_author': self.client.name},
+            self.name)
+
+    def metadataFailed(self, result, text):
+        print("result:",result,"text:",text)
+        return self.account.client.perspective.callRemote('groupMessage',
+                                                          self.name, text)
+
+    def joining(self):
+        self.joined = 1
+
+    def leaving(self):
+        self.joined = 0
+
+    def leave(self):
+        return self.account.client.perspective.callRemote('leaveGroup',
+                                                          self.name)
+
+
+
+class TwistedWordsClient(pb.Referenceable, basesupport.AbstractClientMixin):
+    """In some cases, this acts as an Account, since it a source of text
+    messages (multiple Words instances may be on a single PB connection)
+    """
+    def __init__(self, acct, serviceName, perspectiveName, chatui,
+                 _logonDeferred=None):
+        self.accountName = "%s (%s:%s)" % (acct.accountName, serviceName, perspectiveName)
+        self.name = perspectiveName
+        print("HELLO I AM A PB SERVICE", serviceName, perspectiveName)
+        self.chat = chatui
+        self.account = acct
+        self._logonDeferred = _logonDeferred
+
+    def getPerson(self, name):
+        return self.chat.getPerson(name, self)
+
+    def getGroup(self, name):
+        return self.chat.getGroup(name, self)
+
+    def getGroupConversation(self, name):
+        return self.chat.getGroupConversation(self.getGroup(name))
+
+    def addContact(self, name):
+        self.perspective.callRemote('addContact', name)
+
+    def remote_receiveGroupMembers(self, names, group):
+        print('received group members:', names, group)
+        self.getGroupConversation(group).setGroupMembers(names)
+
+    def remote_receiveGroupMessage(self, sender, group, message, metadata=None):
+        print('received a group message', sender, group, message, metadata)
+        self.getGroupConversation(group).showGroupMessage(sender, message, metadata)
+
+    def remote_memberJoined(self, member, group):
+        print('member joined', member, group)
+        self.getGroupConversation(group).memberJoined(member)
+
+    def remote_memberLeft(self, member, group):
+        print('member left')
+        self.getGroupConversation(group).memberLeft(member)
+
+    def remote_notifyStatusChanged(self, name, status):
+        self.chat.getPerson(name, self).setStatus(status)
+
+    def remote_receiveDirectMessage(self, name, message, metadata=None):
+        self.chat.getConversation(self.chat.getPerson(name, self)).showMessage(message, metadata)
+
+    def remote_receiveContactList(self, clist):
+        for name, status in clist:
+            self.chat.getPerson(name, self).setStatus(status)
+
+    def remote_setGroupMetadata(self, dict_, groupName):
+        if "topic" in dict_:
+            self.getGroupConversation(groupName).setTopic(dict_["topic"], dict_.get("topic_author", None))
+
+    def joinGroup(self, name):
+        self.getGroup(name).joining()
+        return self.perspective.callRemote('joinGroup', name).addCallback(self._cbGroupJoined, name)
+
+    def leaveGroup(self, name):
+        self.getGroup(name).leaving()
+        return self.perspective.callRemote('leaveGroup', name).addCallback(self._cbGroupLeft, name)
+
+    def _cbGroupJoined(self, result, name):
+        groupConv = self.chat.getGroupConversation(self.getGroup(name))
+        groupConv.showGroupMessage("sys", "you joined")
+        self.perspective.callRemote('getGroupMembers', name)
+
+    def _cbGroupLeft(self, result, name):
+        print('left',name)
+        groupConv = self.chat.getGroupConversation(self.getGroup(name), 1)
+        groupConv.showGroupMessage("sys", "you left")
+
+    def connected(self, perspective):
+        print('Connected Words Client!', perspective)
+        if self._logonDeferred is not None:
+            self._logonDeferred.callback(self)
+        self.perspective = perspective
+        self.chat.getContactsList()
+
+
+pbFrontEnds = {
+    "twisted.words": TwistedWordsClient,
+    "twisted.reality": None
+    }
+
+
+@implementer(interfaces.IAccount)
+class PBAccount(basesupport.AbstractAccount):
+    gatewayType = "PB"
+    _groupFactory = TwistedWordsGroup
+    _personFactory = TwistedWordsPerson
+
+    def __init__(self, accountName, autoLogin, username, password, host, port,
+                 services=None):
+        """
+        @param username: The name of your PB Identity.
+        @type username: string
+        """
+        basesupport.AbstractAccount.__init__(self, accountName, autoLogin,
+                                             username, password, host, port)
+        self.services = []
+        if not services:
+            services = [('twisted.words', 'twisted.words', username)]
+        for serviceType, serviceName, perspectiveName in services:
+            self.services.append([pbFrontEnds[serviceType], serviceName,
+                                  perspectiveName])
+
+    def logOn(self, chatui):
+        """
+        @returns: this breaks with L{interfaces.IAccount}
+        @returntype: DeferredList of L{interfaces.IClient}s
+        """
+        # Overriding basesupport's implementation on account of the
+        # fact that _startLogOn tends to return a deferredList rather
+        # than a simple Deferred, and we need to do registerAccountClient.
+        if (not self._isConnecting) and (not self._isOnline):
+            self._isConnecting = 1
+            d = self._startLogOn(chatui)
+            d.addErrback(self._loginFailed)
+            def registerMany(results):
+                for success, result in results:
+                    if success:
+                        chatui.registerAccountClient(result)
+                        self._cb_logOn(result)
+                    else:
+                        log.err(result)
+            d.addCallback(registerMany)
+            return d
+        else:
+            raise error.ConnectionError("Connection in progress")
+
+
+    def _startLogOn(self, chatui):
+        print('Connecting...', end=' ')
+        d = pb.getObjectAt(self.host, self.port)
+        d.addCallbacks(self._cbConnected, self._ebConnected,
+                       callbackArgs=(chatui,))
+        return d
+
+    def _cbConnected(self, root, chatui):
+        print('Connected!')
+        print('Identifying...', end=' ')
+        d = pb.authIdentity(root, self.username, self.password)
+        d.addCallbacks(self._cbIdent, self._ebConnected,
+                       callbackArgs=(chatui,))
+        return d
+
+    def _cbIdent(self, ident, chatui):
+        if not ident:
+            print('falsely identified.')
+            return self._ebConnected(Failure(Exception("username or password incorrect")))
+        print('Identified!')
+        dl = []
+        for handlerClass, sname, pname in self.services:
+            d = defer.Deferred()
+            dl.append(d)
+            handler = handlerClass(self, sname, pname, chatui, d)
+            ident.callRemote('attach', sname, pname, handler).addCallback(handler.connected)
+        return defer.DeferredList(dl)
+
+    def _ebConnected(self, error):
+        print('Not connected.')
+        return error
+
diff --git a/contrib/python/Twisted/py2/twisted/words/iwords.py b/contrib/python/Twisted/py2/twisted/words/iwords.py
new file mode 100644
index 00000000000..c5959378a4e
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/iwords.py
@@ -0,0 +1,267 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from zope.interface import Interface, Attribute
+
+
+class IProtocolPlugin(Interface):
+    """Interface for plugins providing an interface to a Words service
+    """
+
+    name = Attribute("A single word describing what kind of interface this is (eg, irc or web)")
+
+    def getFactory(realm, portal):
+        """Retrieve a C{twisted.internet.interfaces.IServerFactory} provider
+
+        @param realm: An object providing C{twisted.cred.portal.IRealm} and
+        L{IChatService}, with which service information should be looked up.
+
+        @param portal: An object providing C{twisted.cred.portal.IPortal},
+        through which logins should be performed.
+        """
+
+
+class IGroup(Interface):
+    name = Attribute("A short string, unique among groups.")
+
+    def add(user):
+        """Include the given user in this group.
+
+        @type user: L{IUser}
+        """
+
+    def remove(user, reason=None):
+        """Remove the given user from this group.
+
+        @type user: L{IUser}
+        @type reason: C{unicode}
+        """
+
+    def size():
+        """Return the number of participants in this group.
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with an C{int} representing the
+        number of participants in this group.
+        """
+
+    def receive(sender, recipient, message):
+        """
+        Broadcast the given message from the given sender to other
+        users in group.
+
+        The message is not re-transmitted to the sender.
+
+        @param sender: L{IUser}
+
+        @type recipient: L{IGroup}
+        @param recipient: This is probably a wart.  Maybe it will be removed
+        in the future.  For now, it should be the group object the message
+        is being delivered to.
+
+        @param message: C{dict}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with None when delivery has been
+        attempted for all users.
+        """
+
+    def setMetadata(meta):
+        """Change the metadata associated with this group.
+
+        @type meta: C{dict}
+        """
+
+    def iterusers():
+        """Return an iterator of all users in this group.
+        """
+
+
+class IChatClient(Interface):
+    """Interface through which IChatService interacts with clients.
+    """
+
+    name = Attribute("A short string, unique among users.  This will be set by the L{IChatService} at login time.")
+
+    def receive(sender, recipient, message):
+        """
+        Callback notifying this user of the given message sent by the
+        given user.
+
+        This will be invoked whenever another user sends a message to a
+        group this user is participating in, or whenever another user sends
+        a message directly to this user.  In the former case, C{recipient}
+        will be the group to which the message was sent; in the latter, it
+        will be the same object as the user who is receiving the message.
+
+        @type sender: L{IUser}
+        @type recipient: L{IUser} or L{IGroup}
+        @type message: C{dict}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires when the message has been delivered,
+        or which fails in some way.  If the Deferred fails and the message
+        was directed at a group, this user will be removed from that group.
+        """
+
+    def groupMetaUpdate(group, meta):
+        """
+        Callback notifying this user that the metadata for the given
+        group has changed.
+
+        @type group: L{IGroup}
+        @type meta: C{dict}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+    def userJoined(group, user):
+        """
+        Callback notifying this user that the given user has joined
+        the given group.
+
+        @type group: L{IGroup}
+        @type user: L{IUser}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+    def userLeft(group, user, reason=None):
+        """
+        Callback notifying this user that the given user has left the
+        given group for the given reason.
+
+        @type group: L{IGroup}
+        @type user: L{IUser}
+        @type reason: C{unicode}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+
+class IUser(Interface):
+    """Interface through which clients interact with IChatService.
+    """
+
+    realm = Attribute("A reference to the Realm to which this user belongs.  Set if and only if the user is logged in.")
+    mind = Attribute("A reference to the mind which logged in to this user.  Set if and only if the user is logged in.")
+    name = Attribute("A short string, unique among users.")
+
+    lastMessage = Attribute("A POSIX timestamp indicating the time of the last message received from this user.")
+    signOn = Attribute("A POSIX timestamp indicating this user's most recent sign on time.")
+
+    def loggedIn(realm, mind):
+        """Invoked by the associated L{IChatService} when login occurs.
+
+        @param realm: The L{IChatService} through which login is occurring.
+        @param mind: The mind object used for cred login.
+        """
+
+    def send(recipient, message):
+        """Send the given message to the given user or group.
+
+        @type recipient: Either L{IUser} or L{IGroup}
+        @type message: C{dict}
+        """
+
+    def join(group):
+        """Attempt to join the given group.
+
+        @type group: L{IGroup}
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+    def leave(group):
+        """Discontinue participation in the given group.
+
+        @type group: L{IGroup}
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+    def itergroups():
+        """
+        Return an iterator of all groups of which this user is a
+        member.
+        """
+
+
+class IChatService(Interface):
+    name = Attribute("A short string identifying this chat service (eg, a hostname)")
+
+    createGroupOnRequest = Attribute(
+        "A boolean indicating whether L{getGroup} should implicitly "
+        "create groups which are requested but which do not yet exist.")
+
+    createUserOnRequest = Attribute(
+        "A boolean indicating whether L{getUser} should implicitly "
+        "create users which are requested but which do not yet exist.")
+
+    def itergroups():
+        """Return all groups available on this service.
+
+        @rtype: C{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with a list of C{IGroup} providers.
+        """
+
+    def getGroup(name):
+        """Retrieve the group by the given name.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the group with the given
+        name if one exists (or if one is created due to the setting of
+        L{IChatService.createGroupOnRequest}, or which fails with
+        L{twisted.words.ewords.NoSuchGroup} if no such group exists.
+        """
+
+    def createGroup(name):
+        """Create a new group with the given name.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the created group, or
+        with fails with L{twisted.words.ewords.DuplicateGroup} if a
+        group by that name exists already.
+        """
+
+    def lookupGroup(name):
+        """Retrieve a group by name.
+
+        Unlike C{getGroup}, this will never implicitly create a group.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the group by the given
+        name, or which fails with L{twisted.words.ewords.NoSuchGroup}.
+        """
+
+    def getUser(name):
+        """Retrieve the user by the given name.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the user with the given
+        name if one exists (or if one is created due to the setting of
+        L{IChatService.createUserOnRequest}, or which fails with
+        L{twisted.words.ewords.NoSuchUser} if no such user exists.
+        """
+
+    def createUser(name):
+        """Create a new user with the given name.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the created user, or
+        with fails with L{twisted.words.ewords.DuplicateUser} if a
+        user by that name exists already.
+        """
+
+__all__ = [
+    'IGroup', 'IChatClient', 'IUser', 'IChatService',
+    ]
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/__init__.py b/contrib/python/Twisted/py2/twisted/words/protocols/__init__.py
new file mode 100644
index 00000000000..59dc76c0f6a
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Chat protocols.
+"""
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/irc.py b/contrib/python/Twisted/py2/twisted/words/protocols/irc.py
new file mode 100644
index 00000000000..389e10b9bab
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/irc.py
@@ -0,0 +1,4074 @@
+# -*- test-case-name: twisted.words.test.test_irc -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Internet Relay Chat protocol for client and server.
+
+Future Plans
+============
+
+The way the IRCClient class works here encourages people to implement
+IRC clients by subclassing the ephemeral protocol class, and it tends
+to end up with way more state than it should for an object which will
+be destroyed as soon as the TCP transport drops.  Someone oughta do
+something about that, ya know?
+
+The DCC support needs to have more hooks for the client for it to be
+able to ask the user things like "Do you want to accept this session?"
+and "Transfer #2 is 67% done." and otherwise manage the DCC sessions.
+
+Test coverage needs to be better.
+
+@var MAX_COMMAND_LENGTH: The maximum length of a command, as defined by RFC
+    2812 section 2.3.
+
+@var attributes: Singleton instance of L{_CharacterAttributes}, used for
+    constructing formatted text information.
+
+@author: Kevin Turner
+
+@see: RFC 1459: Internet Relay Chat Protocol
+@see: RFC 2812: Internet Relay Chat: Client Protocol
+@see: U{The Client-To-Client-Protocol
+<http://www.irchelp.org/irchelp/rfc/ctcpspec.html>}
+"""
+
+import errno, os, random, re, stat, struct, sys, time, traceback
+import operator
+import string, socket
+import textwrap
+import shlex
+from functools import reduce
+from os import path
+
+from twisted.internet import reactor, protocol, task
+from twisted.persisted import styles
+from twisted.protocols import basic
+from twisted.python import log, reflect, _textattributes
+from twisted.python.compat import unicode, range
+
+NUL = chr(0)
+CR = chr(0o15)
+NL = chr(0o12)
+LF = NL
+SPC = chr(0o40)
+
+# This includes the CRLF terminator characters.
+MAX_COMMAND_LENGTH = 512
+
+CHANNEL_PREFIXES = '&#!+'
+
+class IRCBadMessage(Exception):
+    pass
+
+class IRCPasswordMismatch(Exception):
+    pass
+
+
+
+class IRCBadModes(ValueError):
+    """
+    A malformed mode was encountered while attempting to parse a mode string.
+    """
+
+
+
+def parsemsg(s):
+    """
+    Breaks a message from an IRC server into its prefix, command, and
+    arguments.
+
+    @param s: The message to break.
+    @type s: L{bytes}
+
+    @return: A tuple of (prefix, command, args).
+    @rtype: L{tuple}
+    """
+    prefix = ''
+    trailing = []
+    if not s:
+        raise IRCBadMessage("Empty line.")
+    if s[0:1] == ':':
+        prefix, s = s[1:].split(' ', 1)
+    if s.find(' :') != -1:
+        s, trailing = s.split(' :', 1)
+        args = s.split()
+        args.append(trailing)
+    else:
+        args = s.split()
+    command = args.pop(0)
+    return prefix, command, args
+
+
+
+def split(str, length=80):
+    """
+    Split a string into multiple lines.
+
+    Whitespace near C{str[length]} will be preferred as a breaking point.
+    C{"\\n"} will also be used as a breaking point.
+
+    @param str: The string to split.
+    @type str: C{str}
+
+    @param length: The maximum length which will be allowed for any string in
+        the result.
+    @type length: C{int}
+
+    @return: C{list} of C{str}
+    """
+    return [chunk
+            for line in str.split('\n')
+            for chunk in textwrap.wrap(line, length)]
+
+
+def _intOrDefault(value, default=None):
+    """
+    Convert a value to an integer if possible.
+
+    @rtype: C{int} or type of L{default}
+    @return: An integer when C{value} can be converted to an integer,
+        otherwise return C{default}
+    """
+    if value:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            pass
+    return default
+
+
+
+class UnhandledCommand(RuntimeError):
+    """
+    A command dispatcher could not locate an appropriate command handler.
+    """
+
+
+
+class _CommandDispatcherMixin(object):
+    """
+    Dispatch commands to handlers based on their name.
+
+    Command handler names should be of the form C{prefix_commandName},
+    where C{prefix} is the value specified by L{prefix}, and must
+    accept the parameters as given to L{dispatch}.
+
+    Attempting to mix this in more than once for a single class will cause
+    strange behaviour, due to L{prefix} being overwritten.
+
+    @type prefix: C{str}
+    @ivar prefix: Command handler prefix, used to locate handler attributes
+    """
+    prefix = None
+
+    def dispatch(self, commandName, *args):
+        """
+        Perform actual command dispatch.
+        """
+        def _getMethodName(command):
+            return '%s_%s' % (self.prefix, command)
+
+        def _getMethod(name):
+            return getattr(self, _getMethodName(name), None)
+
+        method = _getMethod(commandName)
+        if method is not None:
+            return method(*args)
+
+        method = _getMethod('unknown')
+        if method is None:
+            raise UnhandledCommand("No handler for %r could be found" % (_getMethodName(commandName),))
+        return method(commandName, *args)
+
+
+
+
+
+def parseModes(modes, params, paramModes=('', '')):
+    """
+    Parse an IRC mode string.
+
+    The mode string is parsed into two lists of mode changes (added and
+    removed), with each mode change represented as C{(mode, param)} where mode
+    is the mode character, and param is the parameter passed for that mode, or
+    L{None} if no parameter is required.
+
+    @type modes: C{str}
+    @param modes: Modes string to parse.
+
+    @type params: C{list}
+    @param params: Parameters specified along with L{modes}.
+
+    @type paramModes: C{(str, str)}
+    @param paramModes: A pair of strings (C{(add, remove)}) that indicate which modes take
+        parameters when added or removed.
+
+    @returns: Two lists of mode changes, one for modes added and the other for
+        modes removed respectively, mode changes in each list are represented as
+        C{(mode, param)}.
+    """
+    if len(modes) == 0:
+        raise IRCBadModes('Empty mode string')
+
+    if modes[0] not in '+-':
+        raise IRCBadModes('Malformed modes string: %r' % (modes,))
+
+    changes = ([], [])
+
+    direction = None
+    count = -1
+    for ch in modes:
+        if ch in '+-':
+            if count == 0:
+                raise IRCBadModes('Empty mode sequence: %r' % (modes,))
+            direction = '+-'.index(ch)
+            count = 0
+        else:
+            param = None
+            if ch in paramModes[direction]:
+                try:
+                    param = params.pop(0)
+                except IndexError:
+                    raise IRCBadModes('Not enough parameters: %r' % (ch,))
+            changes[direction].append((ch, param))
+            count += 1
+
+    if len(params) > 0:
+        raise IRCBadModes('Too many parameters: %r %r' % (modes, params))
+
+    if count == 0:
+        raise IRCBadModes('Empty mode sequence: %r' % (modes,))
+
+    return changes
+
+
+
+class IRC(protocol.Protocol):
+    """
+    Internet Relay Chat server protocol.
+    """
+
+    buffer = ""
+    hostname = None
+
+    encoding = None
+
+    def connectionMade(self):
+        self.channels = []
+        if self.hostname is None:
+            self.hostname = socket.getfqdn()
+
+
+    def sendLine(self, line):
+        line = line + CR + LF
+        if isinstance(line, unicode):
+            useEncoding = self.encoding if self.encoding else "utf-8"
+            line = line.encode(useEncoding)
+        self.transport.write(line)
+
+
+    def sendMessage(self, command, *parameter_list, **prefix):
+        """
+        Send a line formatted as an IRC message.
+
+        First argument is the command, all subsequent arguments are parameters
+        to that command.  If a prefix is desired, it may be specified with the
+        keyword argument 'prefix'.
+
+        The L{sendCommand} method is generally preferred over this one.
+        Notably, this method does not support sending message tags, while the
+        L{sendCommand} method does.
+        """
+        if not command:
+            raise ValueError("IRC message requires a command.")
+
+        if ' ' in command or command[0] == ':':
+            # Not the ONLY way to screw up, but provides a little
+            # sanity checking to catch likely dumb mistakes.
+            raise ValueError("Somebody screwed up, 'cuz this doesn't" \
+                  " look like a command to me: %s" % command)
+
+        line = ' '.join([command] + list(parameter_list))
+        if 'prefix' in prefix:
+            line = ":%s %s" % (prefix['prefix'], line)
+        self.sendLine(line)
+
+        if len(parameter_list) > 15:
+            log.msg("Message has %d parameters (RFC allows 15):\n%s" %
+                    (len(parameter_list), line))
+
+
+    def sendCommand(self, command, parameters, prefix=None, tags=None):
+        """
+        Send to the remote peer a line formatted as an IRC message.
+
+        @param command: The command or numeric to send.
+        @type command: L{unicode}
+
+        @param parameters: The parameters to send with the command.
+        @type parameters: A L{tuple} or L{list} of L{unicode} parameters
+
+        @param prefix: The prefix to send with the command.  If not
+            given, no prefix is sent.
+        @type prefix: L{unicode}
+
+        @param tags: A dict of message tags.  If not given, no message
+            tags are sent.  The dict key should be the name of the tag
+            to send as a string; the value should be the unescaped value
+            to send with the tag, or either None or "" if no value is to
+            be sent with the tag.
+        @type tags: L{dict} of tags (L{unicode}) => values (L{unicode})
+        @see: U{https://ircv3.net/specs/core/message-tags-3.2.html}
+        """
+        if not command:
+            raise ValueError("IRC message requires a command.")
+
+        if " " in command or command[0] == ":":
+            # Not the ONLY way to screw up, but provides a little
+            # sanity checking to catch likely dumb mistakes.
+            raise ValueError('Invalid command: "%s"' % (command,))
+
+        if tags is None:
+            tags = {}
+
+        line = " ".join([command] + list(parameters))
+        if prefix:
+            line = ":%s %s" % (prefix, line)
+        if tags:
+            tagStr = self._stringTags(tags)
+            line = "@%s %s" % (tagStr, line)
+        self.sendLine(line)
+
+        if len(parameters) > 15:
+            log.msg("Message has %d parameters (RFC allows 15):\n%s" %
+                    (len(parameters), line))
+
+
+    def _stringTags(self, tags):
+        """
+        Converts a tag dictionary to a string.
+
+        @param tags: The tag dict passed to sendMsg.
+
+        @rtype: L{unicode}
+        @return: IRCv3-format tag string
+        """
+        self._validateTags(tags)
+        tagStrings = []
+        for tag, value in tags.items():
+            if value:
+                tagStrings.append("%s=%s" % (tag, self._escapeTagValue(value)))
+            else:
+                tagStrings.append(tag)
+        return ";".join(tagStrings)
+
+
+    def _validateTags(self, tags):
+        """
+        Checks the tag dict for errors and raises L{ValueError} if an
+        error is found.
+
+        @param tags: The tag dict passed to sendMsg.
+        """
+        for tag, value in tags.items():
+            if not tag:
+                raise ValueError("A tag name is required.")
+            for char in tag:
+                if not char.isalnum() and char not in ("-", "/", "."):
+                    raise ValueError("Tag contains invalid characters.")
+
+
+    def _escapeTagValue(self, value):
+        """
+        Escape the given tag value according to U{escaping rules in IRCv3
+        <https://ircv3.net/specs/core/message-tags-3.2.html>}.
+
+        @param value: The string value to escape.
+        @type value: L{str}
+
+        @return: The escaped string for sending as a message value
+        @rtype: L{str}
+        """
+        return (value.replace("\\", "\\\\")
+            .replace(";", "\\:")
+            .replace(" ", "\\s")
+            .replace("\r", "\\r")
+            .replace("\n", "\\n")
+            )
+
+
+    def dataReceived(self, data):
+        """
+        This hack is to support mIRC, which sends LF only, even though the RFC
+        says CRLF.  (Also, the flexibility of LineReceiver to turn "line mode"
+        on and off was not required.)
+        """
+        if isinstance(data, bytes):
+            data = data.decode("utf-8")
+        lines = (self.buffer + data).split(LF)
+        # Put the (possibly empty) element after the last LF back in the
+        # buffer
+        self.buffer = lines.pop()
+
+        for line in lines:
+            if len(line) <= 2:
+                # This is a blank line, at best.
+                continue
+            if line[-1] == CR:
+                line = line[:-1]
+            prefix, command, params = parsemsg(line)
+            # mIRC is a big pile of doo-doo
+            command = command.upper()
+            # DEBUG: log.msg( "%s %s %s" % (prefix, command, params))
+
+            self.handleCommand(command, prefix, params)
+
+
+    def handleCommand(self, command, prefix, params):
+        """
+        Determine the function to call for the given command and call it with
+        the given arguments.
+
+        @param command: The IRC command to determine the function for.
+        @type command: L{bytes}
+
+        @param prefix: The prefix of the IRC message (as returned by
+            L{parsemsg}).
+        @type prefix: L{bytes}
+
+        @param params: A list of parameters to call the function with.
+        @type params: L{list}
+        """
+        method = getattr(self, "irc_%s" % command, None)
+        try:
+            if method is not None:
+                method(prefix, params)
+            else:
+                self.irc_unknown(prefix, command, params)
+        except:
+            log.deferr()
+
+
+    def irc_unknown(self, prefix, command, params):
+        """
+        Called by L{handleCommand} on a command that doesn't have a defined
+        handler. Subclasses should override this method.
+        """
+        raise NotImplementedError(command, prefix, params)
+
+
+    # Helper methods
+    def privmsg(self, sender, recip, message):
+        """
+        Send a message to a channel or user
+
+        @type sender: C{str} or C{unicode}
+        @param sender: Who is sending this message.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type recip: C{str} or C{unicode}
+        @param recip: The recipient of this message.  If a channel, it must
+            start with a channel prefix.
+
+        @type message: C{str} or C{unicode}
+        @param message: The message being sent.
+        """
+        self.sendCommand("PRIVMSG", (recip, ":%s" % (lowQuote(message),)), sender)
+
+
+    def notice(self, sender, recip, message):
+        """
+        Send a "notice" to a channel or user.
+
+        Notices differ from privmsgs in that the RFC claims they are different.
+        Robots are supposed to send notices and not respond to them.  Clients
+        typically display notices differently from privmsgs.
+
+        @type sender: C{str} or C{unicode}
+        @param sender: Who is sending this message.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type recip: C{str} or C{unicode}
+        @param recip: The recipient of this message.  If a channel, it must
+            start with a channel prefix.
+
+        @type message: C{str} or C{unicode}
+        @param message: The message being sent.
+        """
+        self.sendCommand("NOTICE", (recip, ":%s" % (message,)), sender)
+
+
+    def action(self, sender, recip, message):
+        """
+        Send an action to a channel or user.
+
+        @type sender: C{str} or C{unicode}
+        @param sender: Who is sending this message.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type recip: C{str} or C{unicode}
+        @param recip: The recipient of this message.  If a channel, it must
+            start with a channel prefix.
+
+        @type message: C{str} or C{unicode}
+        @param message: The action being sent.
+        """
+        self.sendLine(":%s ACTION %s :%s" % (sender, recip, message))
+
+
+    def topic(self, user, channel, topic, author=None):
+        """
+        Send the topic to a user.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving the topic.  Only their nickname, not
+            the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this is the topic.
+
+        @type topic: C{str} or C{unicode} or L{None}
+        @param topic: The topic string, unquoted, or None if there is no topic.
+
+        @type author: C{str} or C{unicode}
+        @param author: If the topic is being changed, the full username and
+            hostmask of the person changing it.
+        """
+        if author is None:
+            if topic is None:
+                self.sendLine(':%s %s %s %s :%s' % (
+                    self.hostname, RPL_NOTOPIC, user, channel, 'No topic is set.'))
+            else:
+                self.sendLine(":%s %s %s %s :%s" % (
+                    self.hostname, RPL_TOPIC, user, channel, lowQuote(topic)))
+        else:
+            self.sendLine(":%s TOPIC %s :%s" % (author, channel, lowQuote(topic)))
+
+
+    def topicAuthor(self, user, channel, author, date):
+        """
+        Send the author of and time at which a topic was set for the given
+        channel.
+
+        This sends a 333 reply message, which is not part of the IRC RFC.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving the topic.  Only their nickname, not
+            the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this information is relevant.
+
+        @type author: C{str} or C{unicode}
+        @param author: The nickname (without hostmask) of the user who last set
+            the topic.
+
+        @type date: C{int}
+        @param date: A POSIX timestamp (number of seconds since the epoch) at
+            which the topic was last set.
+        """
+        self.sendLine(':%s %d %s %s %s %d' % (
+            self.hostname, 333, user, channel, author, date))
+
+
+    def names(self, user, channel, names):
+        """
+        Send the names of a channel's participants to a user.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving the name list.  Only their nickname,
+            not the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this is the namelist.
+
+        @type names: C{list} of C{str} or C{unicode}
+        @param names: The names to send.
+        """
+        # XXX If unicode is given, these limits are not quite correct
+        prefixLength = len(channel) + len(user) + 10
+        namesLength = 512 - prefixLength
+
+        L = []
+        count = 0
+        for n in names:
+            if count + len(n) + 1 > namesLength:
+                self.sendLine(":%s %s %s = %s :%s" % (
+                    self.hostname, RPL_NAMREPLY, user, channel, ' '.join(L)))
+                L = [n]
+                count = len(n)
+            else:
+                L.append(n)
+                count += len(n) + 1
+        if L:
+            self.sendLine(":%s %s %s = %s :%s" % (
+                self.hostname, RPL_NAMREPLY, user, channel, ' '.join(L)))
+        self.sendLine(":%s %s %s %s :End of /NAMES list" % (
+            self.hostname, RPL_ENDOFNAMES, user, channel))
+
+
+    def who(self, user, channel, memberInfo):
+        """
+        Send a list of users participating in a channel.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving this member information.  Only their
+            nickname, not the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this is the member information.
+
+        @type memberInfo: C{list} of C{tuples}
+        @param memberInfo: For each member of the given channel, a 7-tuple
+            containing their username, their hostmask, the server to which they
+            are connected, their nickname, the letter "H" or "G" (standing for
+            "Here" or "Gone"), the hopcount from C{user} to this member, and
+            this member's real name.
+        """
+        for info in memberInfo:
+            (username, hostmask, server, nickname, flag, hops, realName) = info
+            assert flag in ("H", "G")
+            self.sendLine(":%s %s %s %s %s %s %s %s %s :%d %s" % (
+                self.hostname, RPL_WHOREPLY, user, channel,
+                username, hostmask, server, nickname, flag, hops, realName))
+
+        self.sendLine(":%s %s %s %s :End of /WHO list." % (
+            self.hostname, RPL_ENDOFWHO, user, channel))
+
+
+    def whois(self, user, nick, username, hostname, realName, server, serverInfo, oper, idle, signOn, channels):
+        """
+        Send information about the state of a particular user.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving this information.  Only their nickname,
+            not the full hostmask.
+
+        @type nick: C{str} or C{unicode}
+        @param nick: The nickname of the user this information describes.
+
+        @type username: C{str} or C{unicode}
+        @param username: The user's username (eg, ident response)
+
+        @type hostname: C{str}
+        @param hostname: The user's hostmask
+
+        @type realName: C{str} or C{unicode}
+        @param realName: The user's real name
+
+        @type server: C{str} or C{unicode}
+        @param server: The name of the server to which the user is connected
+
+        @type serverInfo: C{str} or C{unicode}
+        @param serverInfo: A descriptive string about that server
+
+        @type oper: C{bool}
+        @param oper: Indicates whether the user is an IRC operator
+
+        @type idle: C{int}
+        @param idle: The number of seconds since the user last sent a message
+
+        @type signOn: C{int}
+        @param signOn: A POSIX timestamp (number of seconds since the epoch)
+            indicating the time the user signed on
+
+        @type channels: C{list} of C{str} or C{unicode}
+        @param channels: A list of the channels which the user is participating in
+        """
+        self.sendLine(":%s %s %s %s %s %s * :%s" % (
+            self.hostname, RPL_WHOISUSER, user, nick, username, hostname, realName))
+        self.sendLine(":%s %s %s %s %s :%s" % (
+            self.hostname, RPL_WHOISSERVER, user, nick, server, serverInfo))
+        if oper:
+            self.sendLine(":%s %s %s %s :is an IRC operator" % (
+                self.hostname, RPL_WHOISOPERATOR, user, nick))
+        self.sendLine(":%s %s %s %s %d %d :seconds idle, signon time" % (
+            self.hostname, RPL_WHOISIDLE, user, nick, idle, signOn))
+        self.sendLine(":%s %s %s %s :%s" % (
+            self.hostname, RPL_WHOISCHANNELS, user, nick, ' '.join(channels)))
+        self.sendLine(":%s %s %s %s :End of WHOIS list." % (
+            self.hostname, RPL_ENDOFWHOIS, user, nick))
+
+
+    def join(self, who, where):
+        """
+        Send a join message.
+
+        @type who: C{str} or C{unicode}
+        @param who: The name of the user joining.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type where: C{str} or C{unicode}
+        @param where: The channel the user is joining.
+        """
+        self.sendLine(":%s JOIN %s" % (who, where))
+
+
+    def part(self, who, where, reason=None):
+        """
+        Send a part message.
+
+        @type who: C{str} or C{unicode}
+        @param who: The name of the user joining.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type where: C{str} or C{unicode}
+        @param where: The channel the user is joining.
+
+        @type reason: C{str} or C{unicode}
+        @param reason: A string describing the misery which caused this poor
+            soul to depart.
+        """
+        if reason:
+            self.sendLine(":%s PART %s :%s" % (who, where, reason))
+        else:
+            self.sendLine(":%s PART %s" % (who, where))
+
+
+    def channelMode(self, user, channel, mode, *args):
+        """
+        Send information about the mode of a channel.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving the name list.  Only their nickname,
+            not the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this is the namelist.
+
+        @type mode: C{str}
+        @param mode: A string describing this channel's modes.
+
+        @param args: Any additional arguments required by the modes.
+        """
+        self.sendLine(":%s %s %s %s %s %s" % (
+            self.hostname, RPL_CHANNELMODEIS, user, channel, mode, ' '.join(args)))
+
+
+
+class ServerSupportedFeatures(_CommandDispatcherMixin):
+    """
+    Handle ISUPPORT messages.
+
+    Feature names match those in the ISUPPORT RFC draft identically.
+
+    Information regarding the specifics of ISUPPORT was gleaned from
+    <http://www.irc.org/tech_docs/draft-brocklesby-irc-isupport-03.txt>.
+    """
+    prefix = 'isupport'
+
+    def __init__(self):
+        self._features = {
+            'CHANNELLEN': 200,
+            'CHANTYPES': tuple('#&'),
+            'MODES': 3,
+            'NICKLEN': 9,
+            'PREFIX': self._parsePrefixParam('(ovh)@+%'),
+            # The ISUPPORT draft explicitly says that there is no default for
+            # CHANMODES, but we're defaulting it here to handle the case where
+            # the IRC server doesn't send us any ISUPPORT information, since
+            # IRCClient.getChannelModeParams relies on this value.
+            'CHANMODES': self._parseChanModesParam(['b', '', 'lk', ''])}
+
+
+    @classmethod
+    def _splitParamArgs(cls, params, valueProcessor=None):
+        """
+        Split ISUPPORT parameter arguments.
+
+        Values can optionally be processed by C{valueProcessor}.
+
+        For example::
+
+            >>> ServerSupportedFeatures._splitParamArgs(['A:1', 'B:2'])
+            (('A', '1'), ('B', '2'))
+
+        @type params: C{iterable} of C{str}
+
+        @type valueProcessor: C{callable} taking {str}
+        @param valueProcessor: Callable to process argument values, or L{None}
+            to perform no processing
+
+        @rtype: C{list} of C{(str, object)}
+        @return: Sequence of C{(name, processedValue)}
+        """
+        if valueProcessor is None:
+            valueProcessor = lambda x: x
+
+        def _parse():
+            for param in params:
+                if ':' not in param:
+                    param += ':'
+                a, b = param.split(':', 1)
+                yield a, valueProcessor(b)
+        return list(_parse())
+
+
+    @classmethod
+    def _unescapeParamValue(cls, value):
+        """
+        Unescape an ISUPPORT parameter.
+
+        The only form of supported escape is C{\\xHH}, where HH must be a valid
+        2-digit hexadecimal number.
+
+        @rtype: C{str}
+        """
+        def _unescape():
+            parts = value.split('\\x')
+            # The first part can never be preceded by the escape.
+            yield parts.pop(0)
+            for s in parts:
+                octet, rest = s[:2], s[2:]
+                try:
+                    octet = int(octet, 16)
+                except ValueError:
+                    raise ValueError('Invalid hex octet: %r' % (octet,))
+                yield chr(octet) + rest
+
+        if '\\x' not in value:
+            return value
+        return ''.join(_unescape())
+
+
+    @classmethod
+    def _splitParam(cls, param):
+        """
+        Split an ISUPPORT parameter.
+
+        @type param: C{str}
+
+        @rtype: C{(str, list)}
+        @return C{(key, arguments)}
+        """
+        if '=' not in param:
+            param += '='
+        key, value = param.split('=', 1)
+        return key, [cls._unescapeParamValue(v) for v in value.split(',')]
+
+
+    @classmethod
+    def _parsePrefixParam(cls, prefix):
+        """
+        Parse the ISUPPORT "PREFIX" parameter.
+
+        The order in which the parameter arguments appear is significant, the
+        earlier a mode appears the more privileges it gives.
+
+        @rtype: C{dict} mapping C{str} to C{(str, int)}
+        @return: A dictionary mapping a mode character to a two-tuple of
+            C({symbol, priority)}, the lower a priority (the lowest being
+            C{0}) the more privileges it gives
+        """
+        if not prefix:
+            return None
+        if prefix[0] != '(' and ')' not in prefix:
+            raise ValueError('Malformed PREFIX parameter')
+        modes, symbols = prefix.split(')', 1)
+        symbols = zip(symbols, range(len(symbols)))
+        modes = modes[1:]
+        return dict(zip(modes, symbols))
+
+
+    @classmethod
+    def _parseChanModesParam(self, params):
+        """
+        Parse the ISUPPORT "CHANMODES" parameter.
+
+        See L{isupport_CHANMODES} for a detailed explanation of this parameter.
+        """
+        names = ('addressModes', 'param', 'setParam', 'noParam')
+        if len(params) > len(names):
+            raise ValueError(
+                'Expecting a maximum of %d channel mode parameters, got %d' % (
+                    len(names), len(params)))
+        items = map(lambda key, value: (key, value or ''), names, params)
+        return dict(items)
+
+
+    def getFeature(self, feature, default=None):
+        """
+        Get a server supported feature's value.
+
+        A feature with the value L{None} is equivalent to the feature being
+        unsupported.
+
+        @type feature: C{str}
+        @param feature: Feature name
+
+        @type default: C{object}
+        @param default: The value to default to, assuming that C{feature}
+            is not supported
+
+        @return: Feature value
+        """
+        return self._features.get(feature, default)
+
+
+    def hasFeature(self, feature):
+        """
+        Determine whether a feature is supported or not.
+
+        @rtype: C{bool}
+        """
+        return self.getFeature(feature) is not None
+
+
+    def parse(self, params):
+        """
+        Parse ISUPPORT parameters.
+
+        If an unknown parameter is encountered, it is simply added to the
+        dictionary, keyed by its name, as a tuple of the parameters provided.
+
+        @type params: C{iterable} of C{str}
+        @param params: Iterable of ISUPPORT parameters to parse
+        """
+        for param in params:
+            key, value = self._splitParam(param)
+            if key.startswith('-'):
+                self._features.pop(key[1:], None)
+            else:
+                self._features[key] = self.dispatch(key, value)
+
+
+    def isupport_unknown(self, command, params):
+        """
+        Unknown ISUPPORT parameter.
+        """
+        return tuple(params)
+
+
+    def isupport_CHANLIMIT(self, params):
+        """
+        The maximum number of each channel type a user may join.
+        """
+        return self._splitParamArgs(params, _intOrDefault)
+
+
+    def isupport_CHANMODES(self, params):
+        """
+        Available channel modes.
+
+        There are 4 categories of channel mode::
+
+            addressModes - Modes that add or remove an address to or from a
+            list, these modes always take a parameter.
+
+            param - Modes that change a setting on a channel, these modes
+            always take a parameter.
+
+            setParam - Modes that change a setting on a channel, these modes
+            only take a parameter when being set.
+
+            noParam - Modes that change a setting on a channel, these modes
+            never take a parameter.
+        """
+        try:
+            return self._parseChanModesParam(params)
+        except ValueError:
+            return self.getFeature('CHANMODES')
+
+
+    def isupport_CHANNELLEN(self, params):
+        """
+        Maximum length of a channel name a client may create.
+        """
+        return _intOrDefault(params[0], self.getFeature('CHANNELLEN'))
+
+
+    def isupport_CHANTYPES(self, params):
+        """
+        Valid channel prefixes.
+        """
+        return tuple(params[0])
+
+
+    def isupport_EXCEPTS(self, params):
+        """
+        Mode character for "ban exceptions".
+
+        The presence of this parameter indicates that the server supports
+        this functionality.
+        """
+        return params[0] or 'e'
+
+
+    def isupport_IDCHAN(self, params):
+        """
+        Safe channel identifiers.
+
+        The presence of this parameter indicates that the server supports
+        this functionality.
+        """
+        return self._splitParamArgs(params)
+
+
+    def isupport_INVEX(self, params):
+        """
+        Mode character for "invite exceptions".
+
+        The presence of this parameter indicates that the server supports
+        this functionality.
+        """
+        return params[0] or 'I'
+
+
+    def isupport_KICKLEN(self, params):
+        """
+        Maximum length of a kick message a client may provide.
+        """
+        return _intOrDefault(params[0])
+
+
+    def isupport_MAXLIST(self, params):
+        """
+        Maximum number of "list modes" a client may set on a channel at once.
+
+        List modes are identified by the "addressModes" key in CHANMODES.
+        """
+        return self._splitParamArgs(params, _intOrDefault)
+
+
+    def isupport_MODES(self, params):
+        """
+        Maximum number of modes accepting parameters that may be sent, by a
+        client, in a single MODE command.
+        """
+        return _intOrDefault(params[0])
+
+
+    def isupport_NETWORK(self, params):
+        """
+        IRC network name.
+        """
+        return params[0]
+
+
+    def isupport_NICKLEN(self, params):
+        """
+        Maximum length of a nickname the client may use.
+        """
+        return _intOrDefault(params[0], self.getFeature('NICKLEN'))
+
+
+    def isupport_PREFIX(self, params):
+        """
+        Mapping of channel modes that clients may have to status flags.
+        """
+        try:
+            return self._parsePrefixParam(params[0])
+        except ValueError:
+            return self.getFeature('PREFIX')
+
+
+    def isupport_SAFELIST(self, params):
+        """
+        Flag indicating that a client may request a LIST without being
+        disconnected due to the large amount of data generated.
+        """
+        return True
+
+
+    def isupport_STATUSMSG(self, params):
+        """
+        The server supports sending messages to only to clients on a channel
+        with a specific status.
+        """
+        return params[0]
+
+
+    def isupport_TARGMAX(self, params):
+        """
+        Maximum number of targets allowable for commands that accept multiple
+        targets.
+        """
+        return dict(self._splitParamArgs(params, _intOrDefault))
+
+
+    def isupport_TOPICLEN(self, params):
+        """
+        Maximum length of a topic that may be set.
+        """
+        return _intOrDefault(params[0])
+
+
+
+class IRCClient(basic.LineReceiver):
+    """
+    Internet Relay Chat client protocol, with sprinkles.
+
+    In addition to providing an interface for an IRC client protocol,
+    this class also contains reasonable implementations of many common
+    CTCP methods.
+
+    TODO
+    ====
+     - Limit the length of messages sent (because the IRC server probably
+       does).
+     - Add flood protection/rate limiting for my CTCP replies.
+     - NickServ cooperation.  (a mix-in?)
+
+    @ivar nickname: Nickname the client will use.
+    @ivar password: Password used to log on to the server.  May be L{None}.
+    @ivar realname: Supplied to the server during login as the "Real name"
+        or "ircname".  May be L{None}.
+    @ivar username: Supplied to the server during login as the "User name".
+        May be L{None}
+
+    @ivar userinfo: Sent in reply to a C{USERINFO} CTCP query.  If L{None}, no
+        USERINFO reply will be sent.
+        "This is used to transmit a string which is settable by
+        the user (and never should be set by the client)."
+    @ivar fingerReply: Sent in reply to a C{FINGER} CTCP query.  If L{None}, no
+        FINGER reply will be sent.
+    @type fingerReply: Callable or String
+
+    @ivar versionName: CTCP VERSION reply, client name.  If L{None}, no VERSION
+        reply will be sent.
+    @type versionName: C{str}, or None.
+    @ivar versionNum: CTCP VERSION reply, client version.
+    @type versionNum: C{str}, or None.
+    @ivar versionEnv: CTCP VERSION reply, environment the client is running in.
+    @type versionEnv: C{str}, or None.
+
+    @ivar sourceURL: CTCP SOURCE reply, a URL where the source code of this
+        client may be found.  If L{None}, no SOURCE reply will be sent.
+
+    @ivar lineRate: Minimum delay between lines sent to the server.  If
+        L{None}, no delay will be imposed.
+    @type lineRate: Number of Seconds.
+
+    @ivar motd: Either L{None} or, between receipt of I{RPL_MOTDSTART} and
+        I{RPL_ENDOFMOTD}, a L{list} of L{str}, each of which is the content
+        of an I{RPL_MOTD} message.
+
+    @ivar erroneousNickFallback: Default nickname assigned when an unregistered
+        client triggers an C{ERR_ERRONEUSNICKNAME} while trying to register
+        with an illegal nickname.
+    @type erroneousNickFallback: C{str}
+
+    @ivar _registered: Whether or not the user is registered. It becomes True
+        once a welcome has been received from the server.
+    @type _registered: C{bool}
+
+    @ivar _attemptedNick: The nickname that will try to get registered. It may
+        change if it is illegal or already taken. L{nickname} becomes the
+        L{_attemptedNick} that is successfully registered.
+    @type _attemptedNick:  C{str}
+
+    @type supported: L{ServerSupportedFeatures}
+    @ivar supported: Available ISUPPORT features on the server
+
+    @type hostname: C{str}
+    @ivar hostname: Host name of the IRC server the client is connected to.
+        Initially the host name is L{None} and later is set to the host name
+        from which the I{RPL_WELCOME} message is received.
+
+    @type _heartbeat: L{task.LoopingCall}
+    @ivar _heartbeat: Looping call to perform the keepalive by calling
+        L{IRCClient._sendHeartbeat} every L{heartbeatInterval} seconds, or
+        L{None} if there is no heartbeat.
+
+    @type heartbeatInterval: C{float}
+    @ivar heartbeatInterval: Interval, in seconds, to send I{PING} messages to
+        the server as a form of keepalive, defaults to 120 seconds. Use L{None}
+        to disable the heartbeat.
+    """
+    hostname = None
+    motd = None
+    nickname = 'irc'
+    password = None
+    realname = None
+    username = None
+    ### Responses to various CTCP queries.
+
+    userinfo = None
+    # fingerReply is a callable returning a string, or a str()able object.
+    fingerReply = None
+    versionName = None
+    versionNum = None
+    versionEnv = None
+
+    sourceURL = "http://twistedmatrix.com/downloads/"
+
+    dcc_destdir = '.'
+    dcc_sessions = None
+
+    # If this is false, no attempt will be made to identify
+    # ourself to the server.
+    performLogin = 1
+
+    lineRate = None
+    _queue = None
+    _queueEmptying = None
+
+    delimiter = b'\n' # b'\r\n' will also work (see dataReceived)
+
+    __pychecker__ = 'unusednames=params,prefix,channel'
+
+    _registered = False
+    _attemptedNick = ''
+    erroneousNickFallback = 'defaultnick'
+
+    _heartbeat = None
+    heartbeatInterval = 120
+
+
+    def _reallySendLine(self, line):
+        quoteLine = lowQuote(line)
+        if isinstance(quoteLine, unicode):
+            quoteLine = quoteLine.encode("utf-8")
+        quoteLine += b'\r'
+        return basic.LineReceiver.sendLine(self, quoteLine)
+
+    def sendLine(self, line):
+        if self.lineRate is None:
+            self._reallySendLine(line)
+        else:
+            self._queue.append(line)
+            if not self._queueEmptying:
+                self._sendLine()
+
+    def _sendLine(self):
+        if self._queue:
+            self._reallySendLine(self._queue.pop(0))
+            self._queueEmptying = reactor.callLater(self.lineRate,
+                                                    self._sendLine)
+        else:
+            self._queueEmptying = None
+
+
+    def connectionLost(self, reason):
+        basic.LineReceiver.connectionLost(self, reason)
+        self.stopHeartbeat()
+
+
+    def _createHeartbeat(self):
+        """
+        Create the heartbeat L{LoopingCall}.
+        """
+        return task.LoopingCall(self._sendHeartbeat)
+
+
+    def _sendHeartbeat(self):
+        """
+        Send a I{PING} message to the IRC server as a form of keepalive.
+        """
+        self.sendLine('PING ' + self.hostname)
+
+
+    def stopHeartbeat(self):
+        """
+        Stop sending I{PING} messages to keep the connection to the server
+        alive.
+
+        @since: 11.1
+        """
+        if self._heartbeat is not None:
+            self._heartbeat.stop()
+            self._heartbeat = None
+
+
+    def startHeartbeat(self):
+        """
+        Start sending I{PING} messages every L{IRCClient.heartbeatInterval}
+        seconds to keep the connection to the server alive during periods of no
+        activity.
+
+        @since: 11.1
+        """
+        self.stopHeartbeat()
+        if self.heartbeatInterval is None:
+            return
+        self._heartbeat = self._createHeartbeat()
+        self._heartbeat.start(self.heartbeatInterval, now=False)
+
+
+    ### Interface level client->user output methods
+    ###
+    ### You'll want to override these.
+
+    ### Methods relating to the server itself
+
+    def created(self, when):
+        """
+        Called with creation date information about the server, usually at logon.
+
+        @type when: C{str}
+        @param when: A string describing when the server was created, probably.
+        """
+
+    def yourHost(self, info):
+        """
+        Called with daemon information about the server, usually at logon.
+
+        @type info: C{str}
+        @param when: A string describing what software the server is running, probably.
+        """
+
+    def myInfo(self, servername, version, umodes, cmodes):
+        """
+        Called with information about the server, usually at logon.
+
+        @type servername: C{str}
+        @param servername: The hostname of this server.
+
+        @type version: C{str}
+        @param version: A description of what software this server runs.
+
+        @type umodes: C{str}
+        @param umodes: All the available user modes.
+
+        @type cmodes: C{str}
+        @param cmodes: All the available channel modes.
+        """
+
+    def luserClient(self, info):
+        """
+        Called with information about the number of connections, usually at logon.
+
+        @type info: C{str}
+        @param info: A description of the number of clients and servers
+        connected to the network, probably.
+        """
+
+    def bounce(self, info):
+        """
+        Called with information about where the client should reconnect.
+
+        @type info: C{str}
+        @param info: A plaintext description of the address that should be
+        connected to.
+        """
+
+    def isupport(self, options):
+        """
+        Called with various information about what the server supports.
+
+        @type options: C{list} of C{str}
+        @param options: Descriptions of features or limits of the server, possibly
+        in the form "NAME=VALUE".
+        """
+
+    def luserChannels(self, channels):
+        """
+        Called with the number of channels existent on the server.
+
+        @type channels: C{int}
+        """
+
+    def luserOp(self, ops):
+        """
+        Called with the number of ops logged on to the server.
+
+        @type ops: C{int}
+        """
+
+    def luserMe(self, info):
+        """
+        Called with information about the server connected to.
+
+        @type info: C{str}
+        @param info: A plaintext string describing the number of users and servers
+        connected to this server.
+        """
+
+    ### Methods involving me directly
+
+    def privmsg(self, user, channel, message):
+        """
+        Called when I have a message from a user to me or a channel.
+        """
+        pass
+
+    def joined(self, channel):
+        """
+        Called when I finish joining a channel.
+
+        channel has the starting character (C{'#'}, C{'&'}, C{'!'}, or C{'+'})
+        intact.
+        """
+
+    def left(self, channel):
+        """
+        Called when I have left a channel.
+
+        channel has the starting character (C{'#'}, C{'&'}, C{'!'}, or C{'+'})
+        intact.
+        """
+
+
+    def noticed(self, user, channel, message):
+        """
+        Called when I have a notice from a user to me or a channel.
+
+        If the client makes any automated replies, it must not do so in
+        response to a NOTICE message, per the RFC::
+
+            The difference between NOTICE and PRIVMSG is that
+            automatic replies MUST NEVER be sent in response to a
+            NOTICE message. [...] The object of this rule is to avoid
+            loops between clients automatically sending something in
+            response to something it received.
+        """
+
+
+    def modeChanged(self, user, channel, set, modes, args):
+        """
+        Called when users or channel's modes are changed.
+
+        @type user: C{str}
+        @param user: The user and hostmask which instigated this change.
+
+        @type channel: C{str}
+        @param channel: The channel where the modes are changed. If args is
+        empty the channel for which the modes are changing. If the changes are
+        at server level it could be equal to C{user}.
+
+        @type set: C{bool} or C{int}
+        @param set: True if the mode(s) is being added, False if it is being
+        removed. If some modes are added and others removed at the same time
+        this function will be called twice, the first time with all the added
+        modes, the second with the removed ones. (To change this behaviour
+        override the irc_MODE method)
+
+        @type modes: C{str}
+        @param modes: The mode or modes which are being changed.
+
+        @type args: C{tuple}
+        @param args: Any additional information required for the mode
+        change.
+        """
+
+    def pong(self, user, secs):
+        """
+        Called with the results of a CTCP PING query.
+        """
+        pass
+
+    def signedOn(self):
+        """
+        Called after successfully signing on to the server.
+        """
+        pass
+
+    def kickedFrom(self, channel, kicker, message):
+        """
+        Called when I am kicked from a channel.
+        """
+        pass
+
+    def nickChanged(self, nick):
+        """
+        Called when my nick has been changed.
+        """
+        self.nickname = nick
+
+
+    ### Things I observe other people doing in a channel.
+
+    def userJoined(self, user, channel):
+        """
+        Called when I see another user joining a channel.
+        """
+        pass
+
+    def userLeft(self, user, channel):
+        """
+        Called when I see another user leaving a channel.
+        """
+        pass
+
+    def userQuit(self, user, quitMessage):
+        """
+        Called when I see another user disconnect from the network.
+        """
+        pass
+
+    def userKicked(self, kickee, channel, kicker, message):
+        """
+        Called when I observe someone else being kicked from a channel.
+        """
+        pass
+
+    def action(self, user, channel, data):
+        """
+        Called when I see a user perform an ACTION on a channel.
+        """
+        pass
+
+    def topicUpdated(self, user, channel, newTopic):
+        """
+        In channel, user changed the topic to newTopic.
+
+        Also called when first joining a channel.
+        """
+        pass
+
+    def userRenamed(self, oldname, newname):
+        """
+        A user changed their name from oldname to newname.
+        """
+        pass
+
+    ### Information from the server.
+
+    def receivedMOTD(self, motd):
+        """
+        I received a message-of-the-day banner from the server.
+
+        motd is a list of strings, where each string was sent as a separate
+        message from the server. To display, you might want to use::
+
+            '\\n'.join(motd)
+
+        to get a nicely formatted string.
+        """
+        pass
+
+    ### user input commands, client->server
+    ### Your client will want to invoke these.
+
+    def join(self, channel, key=None):
+        """
+        Join a channel.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to join. If it has no prefix,
+            C{'#'} will be prepended to it.
+        @type key: C{str}
+        @param key: If specified, the key used to join the channel.
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = '#' + channel
+        if key:
+            self.sendLine("JOIN %s %s" % (channel, key))
+        else:
+            self.sendLine("JOIN %s" % (channel,))
+
+    def leave(self, channel, reason=None):
+        """
+        Leave a channel.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to leave. If it has no prefix,
+            C{'#'} will be prepended to it.
+        @type reason: C{str}
+        @param reason: If given, the reason for leaving.
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = '#' + channel
+        if reason:
+            self.sendLine("PART %s :%s" % (channel, reason))
+        else:
+            self.sendLine("PART %s" % (channel,))
+
+    def kick(self, channel, user, reason=None):
+        """
+        Attempt to kick a user from a channel.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to kick the user from. If it has
+            no prefix, C{'#'} will be prepended to it.
+        @type user: C{str}
+        @param user: The nick of the user to kick.
+        @type reason: C{str}
+        @param reason: If given, the reason for kicking the user.
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = '#' + channel
+        if reason:
+            self.sendLine("KICK %s %s :%s" % (channel, user, reason))
+        else:
+            self.sendLine("KICK %s %s" % (channel, user))
+
+    part = leave
+
+
+    def invite(self, user, channel):
+        """
+        Attempt to invite user to channel
+
+        @type user: C{str}
+        @param user: The user to invite
+        @type channel: C{str}
+        @param channel: The channel to invite the user too
+
+        @since: 11.0
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = '#' + channel
+        self.sendLine("INVITE %s %s" % (user, channel))
+
+
+    def topic(self, channel, topic=None):
+        """
+        Attempt to set the topic of the given channel, or ask what it is.
+
+        If topic is None, then I sent a topic query instead of trying to set the
+        topic. The server should respond with a TOPIC message containing the
+        current topic of the given channel.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to change the topic on. If it
+            has no prefix, C{'#'} will be prepended to it.
+        @type topic: C{str}
+        @param topic: If specified, what to set the topic to.
+        """
+        # << TOPIC #xtestx :fff
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = '#' + channel
+        if topic != None:
+            self.sendLine("TOPIC %s :%s" % (channel, topic))
+        else:
+            self.sendLine("TOPIC %s" % (channel,))
+
+
+    def mode(self, chan, set, modes, limit = None, user = None, mask = None):
+        """
+        Change the modes on a user or channel.
+
+        The C{limit}, C{user}, and C{mask} parameters are mutually exclusive.
+
+        @type chan: C{str}
+        @param chan: The name of the channel to operate on.
+        @type set: C{bool}
+        @param set: True to give the user or channel permissions and False to
+            remove them.
+        @type modes: C{str}
+        @param modes: The mode flags to set on the user or channel.
+        @type limit: C{int}
+        @param limit: In conjunction with the C{'l'} mode flag, limits the
+             number of users on the channel.
+        @type user: C{str}
+        @param user: The user to change the mode on.
+        @type mask: C{str}
+        @param mask: In conjunction with the C{'b'} mode flag, sets a mask of
+            users to be banned from the channel.
+        """
+        if set:
+            line = 'MODE %s +%s' % (chan, modes)
+        else:
+            line = 'MODE %s -%s' % (chan, modes)
+        if limit is not None:
+            line = '%s %d' % (line, limit)
+        elif user is not None:
+            line = '%s %s' % (line, user)
+        elif mask is not None:
+            line = '%s %s' % (line, mask)
+        self.sendLine(line)
+
+
+    def say(self, channel, message, length=None):
+        """
+        Send a message to a channel
+
+        @type channel: C{str}
+        @param channel: The channel to say the message on. If it has no prefix,
+            C{'#'} will be prepended to it.
+        @type message: C{str}
+        @param message: The message to say.
+        @type length: C{int}
+        @param length: The maximum number of octets to send at a time.  This has
+            the effect of turning a single call to C{msg()} into multiple
+            commands to the server.  This is useful when long messages may be
+            sent that would otherwise cause the server to kick us off or
+            silently truncate the text we are sending.  If None is passed, the
+            entire message is always send in one command.
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = '#' + channel
+        self.msg(channel, message, length)
+
+
+    def _safeMaximumLineLength(self, command):
+        """
+        Estimate a safe maximum line length for the given command.
+
+        This is done by assuming the maximum values for nickname length,
+        realname and hostname combined with the command that needs to be sent
+        and some guessing. A theoretical maximum value is used because it is
+        possible that our nickname, username or hostname changes (on the server
+        side) while the length is still being calculated.
+        """
+        # :nickname!realname@hostname COMMAND ...
+        theoretical = ':%s!%s@%s %s' % (
+            'a' * self.supported.getFeature('NICKLEN'),
+            # This value is based on observation.
+            'b' * 10,
+            # See <http://tools.ietf.org/html/rfc2812#section-2.3.1>.
+            'c' * 63,
+            command)
+        # Fingers crossed.
+        fudge = 10
+        return MAX_COMMAND_LENGTH - len(theoretical) - fudge
+
+
+    def msg(self, user, message, length=None):
+        """
+        Send a message to a user or channel.
+
+        The message will be split into multiple commands to the server if:
+         - The message contains any newline characters
+         - Any span between newline characters is longer than the given
+           line-length.
+
+        @param user: Username or channel name to which to direct the
+            message.
+        @type user: C{str}
+
+        @param message: Text to send.
+        @type message: C{str}
+
+        @param length: Maximum number of octets to send in a single
+            command, including the IRC protocol framing. If L{None} is given
+            then L{IRCClient._safeMaximumLineLength} is used to determine a
+            value.
+        @type length: C{int}
+        """
+        fmt = 'PRIVMSG %s :' % (user,)
+
+        if length is None:
+            length = self._safeMaximumLineLength(fmt)
+
+        # Account for the line terminator.
+        minimumLength = len(fmt) + 2
+        if length <= minimumLength:
+            raise ValueError("Maximum length must exceed %d for message "
+                             "to %s" % (minimumLength, user))
+        for line in split(message, length - minimumLength):
+            self.sendLine(fmt + line)
+
+
+    def notice(self, user, message):
+        """
+        Send a notice to a user.
+
+        Notices are like normal message, but should never get automated
+        replies.
+
+        @type user: C{str}
+        @param user: The user to send a notice to.
+        @type message: C{str}
+        @param message: The contents of the notice to send.
+        """
+        self.sendLine("NOTICE %s :%s" % (user, message))
+
+
+    def away(self, message=''):
+        """
+        Mark this client as away.
+
+        @type message: C{str}
+        @param message: If specified, the away message.
+        """
+        self.sendLine("AWAY :%s" % message)
+
+
+    def back(self):
+        """
+        Clear the away status.
+        """
+        # An empty away marks us as back
+        self.away()
+
+
+    def whois(self, nickname, server=None):
+        """
+        Retrieve user information about the given nickname.
+
+        @type nickname: C{str}
+        @param nickname: The nickname about which to retrieve information.
+
+        @since: 8.2
+        """
+        if server is None:
+            self.sendLine('WHOIS ' + nickname)
+        else:
+            self.sendLine('WHOIS %s %s' % (server, nickname))
+
+
+    def register(self, nickname, hostname='foo', servername='bar'):
+        """
+        Login to the server.
+
+        @type nickname: C{str}
+        @param nickname: The nickname to register.
+        @type hostname: C{str}
+        @param hostname: If specified, the hostname to logon as.
+        @type servername: C{str}
+        @param servername: If specified, the servername to logon as.
+        """
+        if self.password is not None:
+            self.sendLine("PASS %s" % self.password)
+        self.setNick(nickname)
+        if self.username is None:
+            self.username = nickname
+        self.sendLine("USER %s %s %s :%s" % (self.username, hostname, servername, self.realname))
+
+
+    def setNick(self, nickname):
+        """
+        Set this client's nickname.
+
+        @type nickname: C{str}
+        @param nickname: The nickname to change to.
+        """
+        self._attemptedNick = nickname
+        self.sendLine("NICK %s" % nickname)
+
+
+    def quit(self, message = ''):
+        """
+        Disconnect from the server
+
+        @type message: C{str}
+
+        @param message: If specified, the message to give when quitting the
+            server.
+        """
+        self.sendLine("QUIT :%s" % message)
+
+    ### user input commands, client->client
+
+    def describe(self, channel, action):
+        """
+        Strike a pose.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to have an action on. If it
+            has no prefix, it is sent to the user of that name.
+        @type action: C{str}
+        @param action: The action to preform.
+        @since: 9.0
+        """
+        self.ctcpMakeQuery(channel, [('ACTION', action)])
+
+
+    _pings = None
+    _MAX_PINGRING = 12
+
+    def ping(self, user, text = None):
+        """
+        Measure round-trip delay to another IRC client.
+        """
+        if self._pings is None:
+            self._pings = {}
+
+        if text is None:
+            chars = string.ascii_letters + string.digits + string.punctuation
+            key = ''.join([random.choice(chars) for i in range(12)])
+        else:
+            key = str(text)
+        self._pings[(user, key)] = time.time()
+        self.ctcpMakeQuery(user, [('PING', key)])
+
+        if len(self._pings) > self._MAX_PINGRING:
+            # Remove some of the oldest entries.
+            byValue = [(v, k) for (k, v) in self._pings.items()]
+            byValue.sort()
+            excess = len(self._pings) - self._MAX_PINGRING
+            for i in range(excess):
+                del self._pings[byValue[i][1]]
+
+
+    def dccSend(self, user, file):
+        """
+        This is supposed to send a user a file directly.  This generally
+        doesn't work on any client, and this method is included only for
+        backwards compatibility and completeness.
+
+        @param user: C{str} representing the user
+        @param file: an open file (unknown, since this is not implemented)
+        """
+        raise NotImplementedError(
+            "XXX!!! Help!  I need to bind a socket, have it listen, and tell me its address.  "
+            "(and stop accepting once we've made a single connection.)")
+
+
+    def dccResume(self, user, fileName, port, resumePos):
+        """
+        Send a DCC RESUME request to another user.
+        """
+        self.ctcpMakeQuery(user, [
+            ('DCC', ['RESUME', fileName, port, resumePos])])
+
+
+    def dccAcceptResume(self, user, fileName, port, resumePos):
+        """
+        Send a DCC ACCEPT response to clients who have requested a resume.
+        """
+        self.ctcpMakeQuery(user, [
+            ('DCC', ['ACCEPT', fileName, port, resumePos])])
+
+    ### server->client messages
+    ### You might want to fiddle with these,
+    ### but it is safe to leave them alone.
+
+    def irc_ERR_NICKNAMEINUSE(self, prefix, params):
+        """
+        Called when we try to register or change to a nickname that is already
+        taken.
+        """
+        self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
+        self.setNick(self._attemptedNick)
+
+
+    def alterCollidedNick(self, nickname):
+        """
+        Generate an altered version of a nickname that caused a collision in an
+        effort to create an unused related name for subsequent registration.
+
+        @param nickname: The nickname a user is attempting to register.
+        @type nickname: C{str}
+
+        @returns: A string that is in some way different from the nickname.
+        @rtype: C{str}
+        """
+        return nickname + '_'
+
+
+    def irc_ERR_ERRONEUSNICKNAME(self, prefix, params):
+        """
+        Called when we try to register or change to an illegal nickname.
+
+        The server should send this reply when the nickname contains any
+        disallowed characters.  The bot will stall, waiting for RPL_WELCOME, if
+        we don't handle this during sign-on.
+
+        @note: The method uses the spelling I{erroneus}, as it appears in
+            the RFC, section 6.1.
+        """
+        if not self._registered:
+            self.setNick(self.erroneousNickFallback)
+
+
+    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
+        """
+        Called when the login was incorrect.
+        """
+        raise IRCPasswordMismatch("Password Incorrect.")
+
+
+    def irc_RPL_WELCOME(self, prefix, params):
+        """
+        Called when we have received the welcome from the server.
+        """
+        self.hostname = prefix
+        self._registered = True
+        self.nickname = self._attemptedNick
+        self.signedOn()
+        self.startHeartbeat()
+
+
+    def irc_JOIN(self, prefix, params):
+        """
+        Called when a user joins a channel.
+        """
+        nick = prefix.split('!')[0]
+        channel = params[-1]
+        if nick == self.nickname:
+            self.joined(channel)
+        else:
+            self.userJoined(nick, channel)
+
+    def irc_PART(self, prefix, params):
+        """
+        Called when a user leaves a channel.
+        """
+        nick = prefix.split('!')[0]
+        channel = params[0]
+        if nick == self.nickname:
+            self.left(channel)
+        else:
+            self.userLeft(nick, channel)
+
+    def irc_QUIT(self, prefix, params):
+        """
+        Called when a user has quit.
+        """
+        nick = prefix.split('!')[0]
+        self.userQuit(nick, params[0])
+
+
+    def irc_MODE(self, user, params):
+        """
+        Parse a server mode change message.
+        """
+        channel, modes, args = params[0], params[1], params[2:]
+
+        if modes[0] not in '-+':
+            modes = '+' + modes
+
+        if channel == self.nickname:
+            # This is a mode change to our individual user, not a channel mode
+            # that involves us.
+            paramModes = self.getUserModeParams()
+        else:
+            paramModes = self.getChannelModeParams()
+
+        try:
+            added, removed = parseModes(modes, args, paramModes)
+        except IRCBadModes:
+            log.err(None, 'An error occurred while parsing the following '
+                          'MODE message: MODE %s' % (' '.join(params),))
+        else:
+            if added:
+                modes, params = zip(*added)
+                self.modeChanged(user, channel, True, ''.join(modes), params)
+
+            if removed:
+                modes, params = zip(*removed)
+                self.modeChanged(user, channel, False, ''.join(modes), params)
+
+
+    def irc_PING(self, prefix, params):
+        """
+        Called when some has pinged us.
+        """
+        self.sendLine("PONG %s" % params[-1])
+
+    def irc_PRIVMSG(self, prefix, params):
+        """
+        Called when we get a message.
+        """
+        user = prefix
+        channel = params[0]
+        message = params[-1]
+
+        if not message:
+            # Don't raise an exception if we get blank message.
+            return
+
+        if message[0] == X_DELIM:
+            m = ctcpExtract(message)
+            if m['extended']:
+                self.ctcpQuery(user, channel, m['extended'])
+
+            if not m['normal']:
+                return
+
+            message = ' '.join(m['normal'])
+
+        self.privmsg(user, channel, message)
+
+    def irc_NOTICE(self, prefix, params):
+        """
+        Called when a user gets a notice.
+        """
+        user = prefix
+        channel = params[0]
+        message = params[-1]
+
+        if message[0]==X_DELIM:
+            m = ctcpExtract(message)
+            if m['extended']:
+                self.ctcpReply(user, channel, m['extended'])
+
+            if not m['normal']:
+                return
+
+            message = ' '.join(m['normal'])
+
+        self.noticed(user, channel, message)
+
+    def irc_NICK(self, prefix, params):
+        """
+        Called when a user changes their nickname.
+        """
+        nick = prefix.split('!', 1)[0]
+        if nick == self.nickname:
+            self.nickChanged(params[0])
+        else:
+            self.userRenamed(nick, params[0])
+
+    def irc_KICK(self, prefix, params):
+        """
+        Called when a user is kicked from a channel.
+        """
+        kicker = prefix.split('!')[0]
+        channel = params[0]
+        kicked = params[1]
+        message = params[-1]
+        if kicked.lower() == self.nickname.lower():
+            # Yikes!
+            self.kickedFrom(channel, kicker, message)
+        else:
+            self.userKicked(kicked, channel, kicker, message)
+
+    def irc_TOPIC(self, prefix, params):
+        """
+        Someone in the channel set the topic.
+        """
+        user = prefix.split('!')[0]
+        channel = params[0]
+        newtopic = params[1]
+        self.topicUpdated(user, channel, newtopic)
+
+    def irc_RPL_TOPIC(self, prefix, params):
+        """
+        Called when the topic for a channel is initially reported or when it
+        subsequently changes.
+        """
+        user = prefix.split('!')[0]
+        channel = params[1]
+        newtopic = params[2]
+        self.topicUpdated(user, channel, newtopic)
+
+    def irc_RPL_NOTOPIC(self, prefix, params):
+        user = prefix.split('!')[0]
+        channel = params[1]
+        newtopic = ""
+        self.topicUpdated(user, channel, newtopic)
+
+    def irc_RPL_MOTDSTART(self, prefix, params):
+        if params[-1].startswith("- "):
+            params[-1] = params[-1][2:]
+        self.motd = [params[-1]]
+
+    def irc_RPL_MOTD(self, prefix, params):
+        if params[-1].startswith("- "):
+            params[-1] = params[-1][2:]
+        if self.motd is None:
+            self.motd = []
+        self.motd.append(params[-1])
+
+
+    def irc_RPL_ENDOFMOTD(self, prefix, params):
+        """
+        I{RPL_ENDOFMOTD} indicates the end of the message of the day
+        messages.  Deliver the accumulated lines to C{receivedMOTD}.
+        """
+        motd = self.motd
+        self.motd = None
+        self.receivedMOTD(motd)
+
+
+    def irc_RPL_CREATED(self, prefix, params):
+        self.created(params[1])
+
+    def irc_RPL_YOURHOST(self, prefix, params):
+        self.yourHost(params[1])
+
+    def irc_RPL_MYINFO(self, prefix, params):
+        info = params[1].split(None, 3)
+        while len(info) < 4:
+            info.append(None)
+        self.myInfo(*info)
+
+    def irc_RPL_BOUNCE(self, prefix, params):
+        self.bounce(params[1])
+
+    def irc_RPL_ISUPPORT(self, prefix, params):
+        args = params[1:-1]
+        # Several ISUPPORT messages, in no particular order, may be sent
+        # to the client at any given point in time (usually only on connect,
+        # though.) For this reason, ServerSupportedFeatures.parse is intended
+        # to mutate the supported feature list.
+        self.supported.parse(args)
+        self.isupport(args)
+
+    def irc_RPL_LUSERCLIENT(self, prefix, params):
+        self.luserClient(params[1])
+
+    def irc_RPL_LUSEROP(self, prefix, params):
+        try:
+            self.luserOp(int(params[1]))
+        except ValueError:
+            pass
+
+    def irc_RPL_LUSERCHANNELS(self, prefix, params):
+        try:
+            self.luserChannels(int(params[1]))
+        except ValueError:
+            pass
+
+    def irc_RPL_LUSERME(self, prefix, params):
+        self.luserMe(params[1])
+
+    def irc_unknown(self, prefix, command, params):
+        pass
+
+    ### Receiving a CTCP query from another party
+    ### It is safe to leave these alone.
+
+
+    def ctcpQuery(self, user, channel, messages):
+        """
+        Dispatch method for any CTCP queries received.
+
+        Duplicated CTCP queries are ignored and no dispatch is
+        made. Unrecognized CTCP queries invoke L{IRCClient.ctcpUnknownQuery}.
+        """
+        seen = set()
+        for tag, data in messages:
+            method = getattr(self, 'ctcpQuery_%s' % tag, None)
+            if tag not in seen:
+                if method is not None:
+                    method(user, channel, data)
+                else:
+                    self.ctcpUnknownQuery(user, channel, tag, data)
+            seen.add(tag)
+
+
+    def ctcpUnknownQuery(self, user, channel, tag, data):
+        """
+        Fallback handler for unrecognized CTCP queries.
+
+        No CTCP I{ERRMSG} reply is made to remove a potential denial of service
+        avenue.
+        """
+        log.msg('Unknown CTCP query from %r: %r %r' % (user, tag, data))
+
+
+    def ctcpQuery_ACTION(self, user, channel, data):
+        self.action(user, channel, data)
+
+    def ctcpQuery_PING(self, user, channel, data):
+        nick = user.split('!')[0]
+        self.ctcpMakeReply(nick, [("PING", data)])
+
+    def ctcpQuery_FINGER(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage("Why did %s send '%s' with a FINGER query?"
+                               % (user, data))
+        if not self.fingerReply:
+            return
+
+        if callable(self.fingerReply):
+            reply = self.fingerReply()
+        else:
+            reply = str(self.fingerReply)
+
+        nick = user.split('!')[0]
+        self.ctcpMakeReply(nick, [('FINGER', reply)])
+
+    def ctcpQuery_VERSION(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage("Why did %s send '%s' with a VERSION query?"
+                               % (user, data))
+
+        if self.versionName:
+            nick = user.split('!')[0]
+            self.ctcpMakeReply(nick, [('VERSION', '%s:%s:%s' %
+                                       (self.versionName,
+                                        self.versionNum or '',
+                                        self.versionEnv or ''))])
+
+    def ctcpQuery_SOURCE(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage("Why did %s send '%s' with a SOURCE query?"
+                               % (user, data))
+        if self.sourceURL:
+            nick = user.split('!')[0]
+            # The CTCP document (Zeuge, Rollo, Mesander 1994) says that SOURCE
+            # replies should be responded to with the location of an anonymous
+            # FTP server in host:directory:file format.  I'm taking the liberty
+            # of bringing it into the 21st century by sending a URL instead.
+            self.ctcpMakeReply(nick, [('SOURCE', self.sourceURL),
+                                      ('SOURCE', None)])
+
+    def ctcpQuery_USERINFO(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage("Why did %s send '%s' with a USERINFO query?"
+                               % (user, data))
+        if self.userinfo:
+            nick = user.split('!')[0]
+            self.ctcpMakeReply(nick, [('USERINFO', self.userinfo)])
+
+    def ctcpQuery_CLIENTINFO(self, user, channel, data):
+        """
+        A master index of what CTCP tags this client knows.
+
+        If no arguments are provided, respond with a list of known tags, sorted
+        in alphabetical order.
+        If an argument is provided, provide human-readable help on
+        the usage of that tag.
+        """
+        nick = user.split('!')[0]
+        if not data:
+            # XXX: prefixedMethodNames gets methods from my *class*,
+            # but it's entirely possible that this *instance* has more
+            # methods.
+            names = sorted(reflect.prefixedMethodNames(self.__class__,
+                                                       'ctcpQuery_'))
+
+            self.ctcpMakeReply(nick, [('CLIENTINFO', ' '.join(names))])
+        else:
+            args = data.split()
+            method = getattr(self, 'ctcpQuery_%s' % (args[0],), None)
+            if not method:
+                self.ctcpMakeReply(nick, [('ERRMSG',
+                                           "CLIENTINFO %s :"
+                                           "Unknown query '%s'"
+                                           % (data, args[0]))])
+                return
+            doc = getattr(method, '__doc__', '')
+            self.ctcpMakeReply(nick, [('CLIENTINFO', doc)])
+
+
+    def ctcpQuery_ERRMSG(self, user, channel, data):
+        # Yeah, this seems strange, but that's what the spec says to do
+        # when faced with an ERRMSG query (not a reply).
+        nick = user.split('!')[0]
+        self.ctcpMakeReply(nick, [('ERRMSG',
+                                   "%s :No error has occurred." % data)])
+
+    def ctcpQuery_TIME(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage("Why did %s send '%s' with a TIME query?"
+                               % (user, data))
+        nick = user.split('!')[0]
+        self.ctcpMakeReply(nick,
+                           [('TIME', ':%s' %
+                             time.asctime(time.localtime(time.time())))])
+
+    def ctcpQuery_DCC(self, user, channel, data):
+        """
+        Initiate a Direct Client Connection
+
+        @param user: The hostmask of the user/client.
+        @type user: L{bytes}
+
+        @param channel: The name of the IRC channel.
+        @type channel: L{bytes}
+
+        @param data: The DCC request message.
+        @type data: L{bytes}
+        """
+
+        if not data: return
+        dcctype = data.split(None, 1)[0].upper()
+        handler = getattr(self, "dcc_" + dcctype, None)
+        if handler:
+            if self.dcc_sessions is None:
+                self.dcc_sessions = []
+            data = data[len(dcctype)+1:]
+            handler(user, channel, data)
+        else:
+            nick = user.split('!')[0]
+            self.ctcpMakeReply(nick, [('ERRMSG',
+                                       "DCC %s :Unknown DCC type '%s'"
+                                       % (data, dcctype))])
+            self.quirkyMessage("%s offered unknown DCC type %s"
+                               % (user, dcctype))
+
+
+    def dcc_SEND(self, user, channel, data):
+        # Use shlex.split for those who send files with spaces in the names.
+        data = shlex.split(data)
+        if len(data) < 3:
+            raise IRCBadMessage("malformed DCC SEND request: %r" % (data,))
+
+        (filename, address, port) = data[:3]
+
+        address = dccParseAddress(address)
+        try:
+            port = int(port)
+        except ValueError:
+            raise IRCBadMessage("Indecipherable port %r" % (port,))
+
+        size = -1
+        if len(data) >= 4:
+            try:
+                size = int(data[3])
+            except ValueError:
+                pass
+
+        # XXX Should we bother passing this data?
+        self.dccDoSend(user, address, port, filename, size, data)
+
+
+    def dcc_ACCEPT(self, user, channel, data):
+        data = shlex.split(data)
+        if len(data) < 3:
+            raise IRCBadMessage("malformed DCC SEND ACCEPT request: %r" % (
+                data,))
+        (filename, port, resumePos) = data[:3]
+        try:
+            port = int(port)
+            resumePos = int(resumePos)
+        except ValueError:
+            return
+
+        self.dccDoAcceptResume(user, filename, port, resumePos)
+
+
+    def dcc_RESUME(self, user, channel, data):
+        data = shlex.split(data)
+        if len(data) < 3:
+            raise IRCBadMessage("malformed DCC SEND RESUME request: %r" % (
+                data,))
+        (filename, port, resumePos) = data[:3]
+        try:
+            port = int(port)
+            resumePos = int(resumePos)
+        except ValueError:
+            return
+
+        self.dccDoResume(user, filename, port, resumePos)
+
+
+    def dcc_CHAT(self, user, channel, data):
+        data = shlex.split(data)
+        if len(data) < 3:
+            raise IRCBadMessage("malformed DCC CHAT request: %r" % (data,))
+
+        (filename, address, port) = data[:3]
+
+        address = dccParseAddress(address)
+        try:
+            port = int(port)
+        except ValueError:
+            raise IRCBadMessage("Indecipherable port %r" % (port,))
+
+        self.dccDoChat(user, channel, address, port, data)
+
+    ### The dccDo methods are the slightly higher-level siblings of
+    ### common dcc_ methods; the arguments have been parsed for them.
+
+    def dccDoSend(self, user, address, port, fileName, size, data):
+        """
+        Called when I receive a DCC SEND offer from a client.
+
+        By default, I do nothing here.
+
+        @param user: The hostmask of the requesting user.
+        @type user: L{bytes}
+
+        @param address: The IP address of the requesting user.
+        @type address: L{bytes}
+
+        @param port: An integer representing the port of the requesting user.
+        @type port: L{int}
+
+        @param fileName: The name of the file to be transferred.
+        @type fileName: L{bytes}
+
+        @param size: The size of the file to be transferred, which may be C{-1}
+            if the size of the file was not specified in the DCC SEND request.
+        @type size: L{int}
+
+        @param data: A 3-list of [fileName, address, port].
+        @type data: L{list}
+        """
+
+
+    def dccDoResume(self, user, file, port, resumePos):
+        """
+        Called when a client is trying to resume an offered file via DCC send.
+        It should be either replied to with a DCC ACCEPT or ignored (default).
+
+        @param user: The hostmask of the user who wants to resume the transfer
+            of a file previously offered via DCC send.
+        @type user: L{bytes}
+
+        @param file: The name of the file to resume the transfer of.
+        @type file: L{bytes}
+
+        @param port: An integer representing the port of the requesting user.
+        @type port: L{int}
+
+        @param resumePos: The position in the file from where the transfer
+            should resume.
+        @type resumePos: L{int}
+        """
+        pass
+
+
+    def dccDoAcceptResume(self, user, file, port, resumePos):
+        """
+        Called when a client has verified and accepted a DCC resume request
+        made by us.  By default it will do nothing.
+
+        @param user: The hostmask of the user who has accepted the DCC resume
+            request.
+        @type user: L{bytes}
+
+        @param file: The name of the file to resume the transfer of.
+        @type file: L{bytes}
+
+        @param port: An integer representing the port of the accepting user.
+        @type port: L{int}
+
+        @param resumePos: The position in the file from where the transfer
+            should resume.
+        @type resumePos: L{int}
+        """
+        pass
+
+
+    def dccDoChat(self, user, channel, address, port, data):
+        pass
+        #factory = DccChatFactory(self, queryData=(user, channel, data))
+        #reactor.connectTCP(address, port, factory)
+        #self.dcc_sessions.append(factory)
+
+    #def ctcpQuery_SED(self, user, data):
+    #    """Simple Encryption Doodoo
+    #
+    #    Feel free to implement this, but no specification is available.
+    #    """
+    #    raise NotImplementedError
+
+
+    def ctcpMakeReply(self, user, messages):
+        """
+        Send one or more C{extended messages} as a CTCP reply.
+
+        @type messages: a list of extended messages.  An extended
+        message is a (tag, data) tuple, where 'data' may be L{None}.
+        """
+        self.notice(user, ctcpStringify(messages))
+
+    ### client CTCP query commands
+
+    def ctcpMakeQuery(self, user, messages):
+        """
+        Send one or more C{extended messages} as a CTCP query.
+
+        @type messages: a list of extended messages.  An extended
+        message is a (tag, data) tuple, where 'data' may be L{None}.
+        """
+        self.msg(user, ctcpStringify(messages))
+
+    ### Receiving a response to a CTCP query (presumably to one we made)
+    ### You may want to add methods here, or override UnknownReply.
+
+    def ctcpReply(self, user, channel, messages):
+        """
+        Dispatch method for any CTCP replies received.
+        """
+        for m in messages:
+            method = getattr(self, "ctcpReply_%s" % m[0], None)
+            if method:
+                method(user, channel, m[1])
+            else:
+                self.ctcpUnknownReply(user, channel, m[0], m[1])
+
+    def ctcpReply_PING(self, user, channel, data):
+        nick = user.split('!', 1)[0]
+        if (not self._pings) or ((nick, data) not in self._pings):
+            raise IRCBadMessage(
+                "Bogus PING response from %s: %s" % (user, data))
+
+        t0 = self._pings[(nick, data)]
+        self.pong(user, time.time() - t0)
+
+    def ctcpUnknownReply(self, user, channel, tag, data):
+        """
+        Called when a fitting ctcpReply_ method is not found.
+
+        @param user: The hostmask of the user.
+        @type user: L{bytes}
+
+        @param channel: The name of the IRC channel.
+        @type channel: L{bytes}
+
+        @param tag: The CTCP request tag for which no fitting method is found.
+        @type tag: L{bytes}
+
+        @param data: The CTCP message.
+        @type data: L{bytes}
+        """
+        # FIXME:7560:
+        # Add code for handling arbitrary queries and not treat them as
+        # anomalies.
+
+        log.msg("Unknown CTCP reply from %s: %s %s\n"
+                 % (user, tag, data))
+
+    ### Error handlers
+    ### You may override these with something more appropriate to your UI.
+
+    def badMessage(self, line, excType, excValue, tb):
+        """
+        When I get a message that's so broken I can't use it.
+
+        @param line: The indecipherable message.
+        @type line: L{bytes}
+
+        @param excType: The exception type of the exception raised by the
+            message.
+        @type excType: L{type}
+
+        @param excValue: The exception parameter of excType or its associated
+            value(the second argument to C{raise}).
+        @type excValue: L{BaseException}
+
+        @param tb: The Traceback as a traceback object.
+        @type tb: L{traceback}
+        """
+        log.msg(line)
+        log.msg(''.join(traceback.format_exception(excType, excValue, tb)))
+
+
+    def quirkyMessage(self, s):
+        """
+        This is called when I receive a message which is peculiar, but not
+        wholly indecipherable.
+
+        @param s: The peculiar message.
+        @type s: L{bytes}
+        """
+        log.msg(s + '\n')
+
+    ### Protocol methods
+
+    def connectionMade(self):
+        self.supported = ServerSupportedFeatures()
+        self._queue = []
+        if self.performLogin:
+            self.register(self.nickname)
+
+    def dataReceived(self, data):
+        if isinstance(data, unicode):
+            data = data.encode("utf-8")
+        data = data.replace(b'\r', b'')
+        basic.LineReceiver.dataReceived(self, data)
+
+
+    def lineReceived(self, line):
+        if bytes != str and isinstance(line, bytes):
+            # decode bytes from transport to unicode
+            line = line.decode("utf-8")
+
+        line = lowDequote(line)
+        try:
+            prefix, command, params = parsemsg(line)
+            if command in numeric_to_symbolic:
+                command = numeric_to_symbolic[command]
+            self.handleCommand(command, prefix, params)
+        except IRCBadMessage:
+            self.badMessage(line, *sys.exc_info())
+
+
+    def getUserModeParams(self):
+        """
+        Get user modes that require parameters for correct parsing.
+
+        @rtype: C{[str, str]}
+        @return C{[add, remove]}
+        """
+        return ['', '']
+
+
+    def getChannelModeParams(self):
+        """
+        Get channel modes that require parameters for correct parsing.
+
+        @rtype: C{[str, str]}
+        @return C{[add, remove]}
+        """
+        # PREFIX modes are treated as "type B" CHANMODES, they always take
+        # parameter.
+        params = ['', '']
+        prefixes = self.supported.getFeature('PREFIX', {})
+        params[0] = params[1] = ''.join(prefixes.keys())
+
+        chanmodes = self.supported.getFeature('CHANMODES')
+        if chanmodes is not None:
+            params[0] += chanmodes.get('addressModes', '')
+            params[0] += chanmodes.get('param', '')
+            params[1] = params[0]
+            params[0] += chanmodes.get('setParam', '')
+        return params
+
+
+    def handleCommand(self, command, prefix, params):
+        """
+        Determine the function to call for the given command and call it with
+        the given arguments.
+
+        @param command: The IRC command to determine the function for.
+        @type command: L{bytes}
+
+        @param prefix: The prefix of the IRC message (as returned by
+            L{parsemsg}).
+        @type prefix: L{bytes}
+
+        @param params: A list of parameters to call the function with.
+        @type params: L{list}
+        """
+        method = getattr(self, "irc_%s" % command, None)
+        try:
+            if method is not None:
+                method(prefix, params)
+            else:
+                self.irc_unknown(prefix, command, params)
+        except:
+            log.deferr()
+
+
+    def __getstate__(self):
+        dct = self.__dict__.copy()
+        dct['dcc_sessions'] = None
+        dct['_pings'] = None
+        return dct
+
+
+def dccParseAddress(address):
+    if '.' in address:
+        pass
+    else:
+        try:
+            address = int(address)
+        except ValueError:
+            raise IRCBadMessage("Indecipherable address %r" % (address,))
+        else:
+            address = (
+                (address >> 24) & 0xFF,
+                (address >> 16) & 0xFF,
+                (address >> 8) & 0xFF,
+                address & 0xFF,
+                )
+            address = '.'.join(map(str,address))
+    return address
+
+
+class DccFileReceiveBasic(protocol.Protocol, styles.Ephemeral):
+    """
+    Bare protocol to receive a Direct Client Connection SEND stream.
+
+    This does enough to keep the other guy talking, but you'll want to extend
+    my dataReceived method to *do* something with the data I get.
+
+    @ivar bytesReceived: An integer representing the number of bytes of data
+        received.
+    @type bytesReceived: L{int}
+    """
+
+    bytesReceived = 0
+
+    def __init__(self, resumeOffset=0):
+        """
+        @param resumeOffset: An integer representing the amount of bytes from
+            where the transfer of data should be resumed.
+        @type resumeOffset: L{int}
+        """
+        self.bytesReceived = resumeOffset
+        self.resume = (resumeOffset != 0)
+
+    def dataReceived(self, data):
+        """
+        See: L{protocol.Protocol.dataReceived}
+
+        Warning: This just acknowledges to the remote host that the data has
+        been received; it doesn't I{do} anything with the data, so you'll want
+        to override this.
+        """
+        self.bytesReceived = self.bytesReceived + len(data)
+        self.transport.write(struct.pack('!i', self.bytesReceived))
+
+
+class DccSendProtocol(protocol.Protocol, styles.Ephemeral):
+    """
+    Protocol for an outgoing Direct Client Connection SEND.
+
+    @ivar blocksize: An integer representing the size of an individual block of
+        data.
+    @type blocksize: L{int}
+
+    @ivar file: The file to be sent.  This can be either a file object or
+        simply the name of the file.
+    @type file: L{file} or L{bytes}
+
+    @ivar bytesSent: An integer representing the number of bytes sent.
+    @type bytesSent: L{int}
+
+    @ivar completed: An integer representing whether the transfer has been
+        completed or not.
+    @type completed: L{int}
+
+    @ivar connected: An integer representing whether the connection has been
+        established or not.
+    @type connected: L{int}
+    """
+
+    blocksize = 1024
+    file = None
+    bytesSent = 0
+    completed = 0
+    connected = 0
+
+    def __init__(self, file):
+        if type(file) is str:
+            self.file = open(file, 'r')
+
+    def connectionMade(self):
+        self.connected = 1
+        self.sendBlock()
+
+    def dataReceived(self, data):
+        # XXX: Do we need to check to see if len(data) != fmtsize?
+
+        bytesShesGot = struct.unpack("!I", data)
+        if bytesShesGot < self.bytesSent:
+            # Wait for her.
+            # XXX? Add some checks to see if we've stalled out?
+            return
+        elif bytesShesGot > self.bytesSent:
+            # self.transport.log("DCC SEND %s: She says she has %d bytes "
+            #                    "but I've only sent %d.  I'm stopping "
+            #                    "this screwy transfer."
+            #                    % (self.file,
+            #                       bytesShesGot, self.bytesSent))
+            self.transport.loseConnection()
+            return
+
+        self.sendBlock()
+
+    def sendBlock(self):
+        block = self.file.read(self.blocksize)
+        if block:
+            self.transport.write(block)
+            self.bytesSent = self.bytesSent + len(block)
+        else:
+            # Nothing more to send, transfer complete.
+            self.transport.loseConnection()
+            self.completed = 1
+
+    def connectionLost(self, reason):
+        self.connected = 0
+        if hasattr(self.file, "close"):
+            self.file.close()
+
+
+class DccSendFactory(protocol.Factory):
+    protocol = DccSendProtocol
+    def __init__(self, file):
+        self.file = file
+
+    def buildProtocol(self, connection):
+        p = self.protocol(self.file)
+        p.factory = self
+        return p
+
+
+def fileSize(file):
+    """
+    I'll try my damndest to determine the size of this file object.
+
+    @param file: The file object to determine the size of.
+    @type file: L{file}
+
+    @rtype: L{int} or L{None}
+    @return: The size of the file object as an integer if it can be determined,
+        otherwise return L{None}.
+    """
+    size = None
+    if hasattr(file, "fileno"):
+        fileno = file.fileno()
+        try:
+            stat_ = os.fstat(fileno)
+            size = stat_[stat.ST_SIZE]
+        except:
+            pass
+        else:
+            return size
+
+    if hasattr(file, "name") and path.exists(file.name):
+        try:
+            size = path.getsize(file.name)
+        except:
+            pass
+        else:
+            return size
+
+    if hasattr(file, "seek") and hasattr(file, "tell"):
+        try:
+            try:
+                file.seek(0, 2)
+                size = file.tell()
+            finally:
+                file.seek(0, 0)
+        except:
+            pass
+        else:
+            return size
+
+    return size
+
+class DccChat(basic.LineReceiver, styles.Ephemeral):
+    """
+    Direct Client Connection protocol type CHAT.
+
+    DCC CHAT is really just your run o' the mill basic.LineReceiver
+    protocol.  This class only varies from that slightly, accepting
+    either LF or CR LF for a line delimeter for incoming messages
+    while always using CR LF for outgoing.
+
+    The lineReceived method implemented here uses the DCC connection's
+    'client' attribute (provided upon construction) to deliver incoming
+    lines from the DCC chat via IRCClient's normal privmsg interface.
+    That's something of a spoof, which you may well want to override.
+    """
+
+    queryData = None
+    delimiter = CR + NL
+    client = None
+    remoteParty = None
+    buffer = b""
+
+    def __init__(self, client, queryData=None):
+        """
+        Initialize a new DCC CHAT session.
+
+        queryData is a 3-tuple of
+        (fromUser, targetUserOrChannel, data)
+        as received by the CTCP query.
+
+        (To be honest, fromUser is the only thing that's currently
+        used here. targetUserOrChannel is potentially useful, while
+        the 'data' argument is solely for informational purposes.)
+        """
+        self.client = client
+        if queryData:
+            self.queryData = queryData
+            self.remoteParty = self.queryData[0]
+
+    def dataReceived(self, data):
+        self.buffer = self.buffer + data
+        lines = self.buffer.split(LF)
+        # Put the (possibly empty) element after the last LF back in the
+        # buffer
+        self.buffer = lines.pop()
+
+        for line in lines:
+            if line[-1] == CR:
+                line = line[:-1]
+            self.lineReceived(line)
+
+    def lineReceived(self, line):
+        log.msg("DCC CHAT<%s> %s" % (self.remoteParty, line))
+        self.client.privmsg(self.remoteParty,
+                            self.client.nickname, line)
+
+
+class DccChatFactory(protocol.ClientFactory):
+    protocol = DccChat
+    noisy = 0
+    def __init__(self, client, queryData):
+        self.client = client
+        self.queryData = queryData
+
+
+    def buildProtocol(self, addr):
+        p = self.protocol(client=self.client, queryData=self.queryData)
+        p.factory = self
+        return p
+
+
+    def clientConnectionFailed(self, unused_connector, unused_reason):
+        self.client.dcc_sessions.remove(self)
+
+    def clientConnectionLost(self, unused_connector, unused_reason):
+        self.client.dcc_sessions.remove(self)
+
+
+def dccDescribe(data):
+    """
+    Given the data chunk from a DCC query, return a descriptive string.
+
+    @param data: The data from a DCC query.
+    @type data: L{bytes}
+
+    @rtype: L{bytes}
+    @return: A descriptive string.
+    """
+
+    orig_data = data
+    data = data.split()
+    if len(data) < 4:
+        return orig_data
+
+    (dcctype, arg, address, port) = data[:4]
+
+    if '.' in address:
+        pass
+    else:
+        try:
+            address = int(address)
+        except ValueError:
+            pass
+        else:
+            address = (
+                (address >> 24) & 0xFF,
+                (address >> 16) & 0xFF,
+                (address >> 8) & 0xFF,
+                address & 0xFF,
+                )
+            address = '.'.join(map(str, address))
+
+    if dcctype == 'SEND':
+        filename = arg
+
+        size_txt = ''
+        if len(data) >= 5:
+            try:
+                size = int(data[4])
+                size_txt = ' of size %d bytes' % (size,)
+            except ValueError:
+                pass
+
+        dcc_text = ("SEND for file '%s'%s at host %s, port %s"
+                    % (filename, size_txt, address, port))
+    elif dcctype == 'CHAT':
+        dcc_text = ("CHAT for host %s, port %s"
+                    % (address, port))
+    else:
+        dcc_text = orig_data
+
+    return dcc_text
+
+
+class DccFileReceive(DccFileReceiveBasic):
+    """
+    Higher-level coverage for getting a file from DCC SEND.
+
+    I allow you to change the file's name and destination directory.  I won't
+    overwrite an existing file unless I've been told it's okay to do so.  If
+    passed the resumeOffset keyword argument I will attempt to resume the file
+    from that amount of bytes.
+
+    XXX: I need to let the client know when I am finished.
+    XXX: I need to decide how to keep a progress indicator updated.
+    XXX: Client needs a way to tell me "Do not finish until I say so."
+    XXX: I need to make sure the client understands if the file cannot be written.
+
+    @ivar filename: The name of the file to get.
+    @type filename: L{bytes}
+
+    @ivar fileSize: The size of the file to get, which has a default value of
+        C{-1} if the size of the file was not specified in the DCC SEND
+        request.
+    @type fileSize: L{int}
+
+    @ivar destDir: The destination directory for the file to be received.
+    @type destDir: L{bytes}
+
+    @ivar overwrite: An integer representing whether an existing file should be
+        overwritten or not.  This initially is an L{int} but can be modified to
+        be a L{bool} using the L{set_overwrite} method.
+    @type overwrite: L{int} or L{bool}
+
+    @ivar queryData: queryData is a 3-tuple of (user, channel, data).
+    @type queryData: L{tuple}
+
+    @ivar fromUser: This is the hostmask of the requesting user and is found at
+        index 0 of L{queryData}.
+    @type fromUser: L{bytes}
+    """
+
+    filename = 'dcc'
+    fileSize = -1
+    destDir = '.'
+    overwrite = 0
+    fromUser = None
+    queryData = None
+
+    def __init__(self, filename, fileSize=-1, queryData=None,
+                 destDir='.', resumeOffset=0):
+        DccFileReceiveBasic.__init__(self, resumeOffset=resumeOffset)
+        self.filename = filename
+        self.destDir = destDir
+        self.fileSize = fileSize
+        self._resumeOffset = resumeOffset
+
+        if queryData:
+            self.queryData = queryData
+            self.fromUser = self.queryData[0]
+
+    def set_directory(self, directory):
+        """
+        Set the directory where the downloaded file will be placed.
+
+        May raise OSError if the supplied directory path is not suitable.
+
+        @param directory: The directory where the file to be received will be
+            placed.
+        @type directory: L{bytes}
+        """
+        if not path.exists(directory):
+            raise OSError(errno.ENOENT, "You see no directory there.",
+                          directory)
+        if not path.isdir(directory):
+            raise OSError(errno.ENOTDIR, "You cannot put a file into "
+                          "something which is not a directory.",
+                          directory)
+        if not os.access(directory, os.X_OK | os.W_OK):
+            raise OSError(errno.EACCES,
+                          "This directory is too hard to write in to.",
+                          directory)
+        self.destDir = directory
+
+    def set_filename(self, filename):
+        """
+        Change the name of the file being transferred.
+
+        This replaces the file name provided by the sender.
+
+        @param filename: The new name for the file.
+        @type filename: L{bytes}
+        """
+        self.filename = filename
+
+    def set_overwrite(self, boolean):
+        """
+        May I overwrite existing files?
+
+        @param boolean: A boolean value representing whether existing files
+            should be overwritten or not.
+        @type boolean: L{bool}
+        """
+        self.overwrite = boolean
+
+
+    # Protocol-level methods.
+
+    def connectionMade(self):
+        dst = path.abspath(path.join(self.destDir,self.filename))
+        exists = path.exists(dst)
+        if self.resume and exists:
+            # I have been told I want to resume, and a file already
+            # exists - Here we go
+            self.file = open(dst, 'rb+')
+            self.file.seek(self._resumeOffset)
+            self.file.truncate()
+            log.msg("Attempting to resume %s - starting from %d bytes" %
+                    (self.file, self.file.tell()))
+        elif self.resume and not exists:
+            raise OSError(errno.ENOENT,
+                          "You cannot resume writing to a file "
+                          "that does not exist!",
+                          dst)
+        elif self.overwrite or not exists:
+            self.file = open(dst, 'wb')
+        else:
+            raise OSError(errno.EEXIST,
+                          "There's a file in the way.  "
+                          "Perhaps that's why you cannot open it.",
+                          dst)
+
+    def dataReceived(self, data):
+        self.file.write(data)
+        DccFileReceiveBasic.dataReceived(self, data)
+
+        # XXX: update a progress indicator here?
+
+    def connectionLost(self, reason):
+        """
+        When the connection is lost, I close the file.
+
+        @param reason: The reason why the connection was lost.
+        @type reason: L{Failure}
+        """
+        self.connected = 0
+        logmsg = ("%s closed." % (self,))
+        if self.fileSize > 0:
+            logmsg = ("%s  %d/%d bytes received"
+                      % (logmsg, self.bytesReceived, self.fileSize))
+            if self.bytesReceived == self.fileSize:
+                pass # Hooray!
+            elif self.bytesReceived < self.fileSize:
+                logmsg = ("%s (Warning: %d bytes short)"
+                          % (logmsg, self.fileSize - self.bytesReceived))
+            else:
+                logmsg = ("%s (file larger than expected)"
+                          % (logmsg,))
+        else:
+            logmsg = ("%s  %d bytes received"
+                      % (logmsg, self.bytesReceived))
+
+        if hasattr(self, 'file'):
+            logmsg = "%s and written to %s.\n" % (logmsg, self.file.name)
+            if hasattr(self.file, 'close'): self.file.close()
+
+        # self.transport.log(logmsg)
+
+    def __str__(self):
+        if not self.connected:
+            return "<Unconnected DccFileReceive object at %x>" % (id(self),)
+        from_ = self.transport.getPeer()
+        if self.fromUser:
+            from_ = "%s (%s)" % (self.fromUser, from_)
+
+        s = ("DCC transfer of '%s' from %s" % (self.filename, from_))
+        return s
+
+    def __repr__(self):
+        s = ("<%s at %x: GET %s>"
+             % (self.__class__, id(self), self.filename))
+        return s
+
+
+
+_OFF = '\x0f'
+_BOLD = '\x02'
+_COLOR = '\x03'
+_REVERSE_VIDEO = '\x16'
+_UNDERLINE = '\x1f'
+
+# Mapping of IRC color names to their color values.
+_IRC_COLORS = dict(
+    zip(['white', 'black', 'blue', 'green', 'lightRed', 'red', 'magenta',
+         'orange', 'yellow', 'lightGreen', 'cyan', 'lightCyan', 'lightBlue',
+         'lightMagenta', 'gray', 'lightGray'], range(16)))
+
+# Mapping of IRC color values to their color names.
+_IRC_COLOR_NAMES = dict((code, name) for name, code in _IRC_COLORS.items())
+
+
+
+class _CharacterAttributes(_textattributes.CharacterAttributesMixin):
+    """
+    Factory for character attributes, including foreground and background color
+    and non-color attributes such as bold, reverse video and underline.
+
+    Character attributes are applied to actual text by using object
+    indexing-syntax (C{obj['abc']}) after accessing a factory attribute, for
+    example::
+
+        attributes.bold['Some text']
+
+    These can be nested to mix attributes::
+
+        attributes.bold[attributes.underline['Some text']]
+
+    And multiple values can be passed::
+
+        attributes.normal[attributes.bold['Some'], ' text']
+
+    Non-color attributes can be accessed by attribute name, available
+    attributes are:
+
+        - bold
+        - reverseVideo
+        - underline
+
+    Available colors are:
+
+        0. white
+        1. black
+        2. blue
+        3. green
+        4. light red
+        5. red
+        6. magenta
+        7. orange
+        8. yellow
+        9. light green
+        10. cyan
+        11. light cyan
+        12. light blue
+        13. light magenta
+        14. gray
+        15. light gray
+
+    @ivar fg: Foreground colors accessed by attribute name, see above
+        for possible names.
+
+    @ivar bg: Background colors accessed by attribute name, see above
+        for possible names.
+
+    @since: 13.1
+    """
+    fg = _textattributes._ColorAttribute(
+        _textattributes._ForegroundColorAttr, _IRC_COLORS)
+    bg = _textattributes._ColorAttribute(
+        _textattributes._BackgroundColorAttr, _IRC_COLORS)
+
+    attrs = {
+        'bold': _BOLD,
+        'reverseVideo': _REVERSE_VIDEO,
+        'underline': _UNDERLINE}
+
+
+
+attributes = _CharacterAttributes()
+
+
+
+class _FormattingState(_textattributes._FormattingStateMixin):
+    """
+    Formatting state/attributes of a single character.
+
+    Attributes include:
+        - Formatting nullifier
+        - Bold
+        - Underline
+        - Reverse video
+        - Foreground color
+        - Background color
+
+    @since: 13.1
+    """
+    compareAttributes = (
+        'off', 'bold', 'underline', 'reverseVideo', 'foreground', 'background')
+
+
+    def __init__(self, off=False, bold=False, underline=False,
+                 reverseVideo=False, foreground=None, background=None):
+        self.off = off
+        self.bold = bold
+        self.underline = underline
+        self.reverseVideo = reverseVideo
+        self.foreground = foreground
+        self.background = background
+
+
+    def toMIRCControlCodes(self):
+        """
+        Emit a mIRC control sequence that will set up all the attributes this
+        formatting state has set.
+
+        @return: A string containing mIRC control sequences that mimic this
+            formatting state.
+        """
+        attrs = []
+        if self.bold:
+            attrs.append(_BOLD)
+        if self.underline:
+            attrs.append(_UNDERLINE)
+        if self.reverseVideo:
+            attrs.append(_REVERSE_VIDEO)
+        if self.foreground is not None or self.background is not None:
+            c = ''
+            if self.foreground is not None:
+                c += '%02d' % (self.foreground,)
+            if self.background is not None:
+                c += ',%02d' % (self.background,)
+            attrs.append(_COLOR + c)
+        return _OFF + ''.join(map(str, attrs))
+
+
+
+def _foldr(f, z, xs):
+    """
+    Apply a function of two arguments cumulatively to the items of
+    a sequence, from right to left, so as to reduce the sequence to
+    a single value.
+
+    @type f: C{callable} taking 2 arguments
+
+    @param z: Initial value.
+
+    @param xs: Sequence to reduce.
+
+    @return: Single value resulting from reducing C{xs}.
+    """
+    return reduce(lambda x, y: f(y, x), reversed(xs), z)
+
+
+
+class _FormattingParser(_CommandDispatcherMixin):
+    """
+    A finite-state machine that parses formatted IRC text.
+
+    Currently handled formatting includes: bold, reverse, underline,
+    mIRC color codes and the ability to remove all current formatting.
+
+    @see: U{http://www.mirc.co.uk/help/color.txt}
+
+    @type _formatCodes: C{dict} mapping C{str} to C{str}
+    @cvar _formatCodes: Mapping of format code values to names.
+
+    @type state: C{str}
+    @ivar state: Current state of the finite-state machine.
+
+    @type _buffer: C{str}
+    @ivar _buffer: Buffer, containing the text content, of the formatting
+        sequence currently being parsed, the buffer is used as the content for
+        L{_attrs} before being added to L{_result} and emptied upon calling
+        L{emit}.
+
+    @type _attrs: C{set}
+    @ivar _attrs: Set of the applicable formatting states (bold, underline,
+        etc.) for the current L{_buffer}, these are applied to L{_buffer} when
+        calling L{emit}.
+
+    @type foreground: L{_ForegroundColorAttr}
+    @ivar foreground: Current foreground color attribute, or L{None}.
+
+    @type background: L{_BackgroundColorAttr}
+    @ivar background: Current background color attribute, or L{None}.
+
+    @ivar _result: Current parse result.
+    """
+    prefix = 'state'
+
+
+    _formatCodes = {
+        _OFF: 'off',
+        _BOLD: 'bold',
+        _COLOR: 'color',
+        _REVERSE_VIDEO: 'reverseVideo',
+        _UNDERLINE: 'underline'}
+
+
+    def __init__(self):
+        self.state = 'TEXT'
+        self._buffer = ''
+        self._attrs = set()
+        self._result = None
+        self.foreground = None
+        self.background = None
+
+
+    def process(self, ch):
+        """
+        Handle input.
+
+        @type ch: C{str}
+        @param ch: A single character of input to process
+        """
+        self.dispatch(self.state, ch)
+
+
+    def complete(self):
+        """
+        Flush the current buffer and return the final parsed result.
+
+        @return: Structured text and attributes.
+        """
+        self.emit()
+        if self._result is None:
+            self._result = attributes.normal
+        return self._result
+
+
+    def emit(self):
+        """
+        Add the currently parsed input to the result.
+        """
+        if self._buffer:
+            attrs = [getattr(attributes, name) for name in self._attrs]
+            attrs.extend(filter(None, [self.foreground, self.background]))
+            if not attrs:
+                attrs.append(attributes.normal)
+            attrs.append(self._buffer)
+
+            attr = _foldr(operator.getitem, attrs.pop(), attrs)
+            if self._result is None:
+                self._result = attr
+            else:
+                self._result[attr]
+            self._buffer = ''
+
+
+    def state_TEXT(self, ch):
+        """
+        Handle the "text" state.
+
+        Along with regular text, single token formatting codes are handled
+        in this state too.
+
+        @param ch: The character being processed.
+        """
+        formatName = self._formatCodes.get(ch)
+        if formatName == 'color':
+            self.emit()
+            self.state = 'COLOR_FOREGROUND'
+        else:
+            if formatName is None:
+                self._buffer += ch
+            else:
+                self.emit()
+                if formatName == 'off':
+                    self._attrs = set()
+                    self.foreground = self.background = None
+                else:
+                    self._attrs.symmetric_difference_update([formatName])
+
+
+    def state_COLOR_FOREGROUND(self, ch):
+        """
+        Handle the foreground color state.
+
+        Foreground colors can consist of up to two digits and may optionally
+        end in a I{,}. Any non-digit or non-comma characters are treated as
+        invalid input and result in the state being reset to "text".
+
+        @param ch: The character being processed.
+        """
+        # Color codes may only be a maximum of two characters.
+        if ch.isdigit() and len(self._buffer) < 2:
+            self._buffer += ch
+        else:
+            if self._buffer:
+                # Wrap around for color numbers higher than we support, like
+                # most other IRC clients.
+                col = int(self._buffer) % len(_IRC_COLORS)
+                self.foreground = getattr(attributes.fg, _IRC_COLOR_NAMES[col])
+            else:
+                # If there were no digits, then this has been an empty color
+                # code and we can reset the color state.
+                self.foreground = self.background = None
+
+            if ch == ',' and self._buffer:
+                # If there's a comma and it's not the first thing, move on to
+                # the background state.
+                self._buffer = ''
+                self.state = 'COLOR_BACKGROUND'
+            else:
+                # Otherwise, this is a bogus color code, fall back to text.
+                self._buffer = ''
+                self.state = 'TEXT'
+                self.emit()
+                self.process(ch)
+
+
+    def state_COLOR_BACKGROUND(self, ch):
+        """
+        Handle the background color state.
+
+        Background colors can consist of up to two digits and must occur after
+        a foreground color and must be preceded by a I{,}. Any non-digit
+        character is treated as invalid input and results in the state being
+        set to "text".
+
+        @param ch: The character being processed.
+        """
+        # Color codes may only be a maximum of two characters.
+        if ch.isdigit() and len(self._buffer) < 2:
+            self._buffer += ch
+        else:
+            if self._buffer:
+                # Wrap around for color numbers higher than we support, like
+                # most other IRC clients.
+                col = int(self._buffer) % len(_IRC_COLORS)
+                self.background = getattr(attributes.bg, _IRC_COLOR_NAMES[col])
+                self._buffer = ''
+
+            self.emit()
+            self.state = 'TEXT'
+            self.process(ch)
+
+
+
+def parseFormattedText(text):
+    """
+    Parse text containing IRC formatting codes into structured information.
+
+    Color codes are mapped from 0 to 15 and wrap around if greater than 15.
+
+    @type text: C{str}
+    @param text: Formatted text to parse.
+
+    @return: Structured text and attributes.
+
+    @since: 13.1
+    """
+    state = _FormattingParser()
+    for ch in text:
+        state.process(ch)
+    return state.complete()
+
+
+
+def assembleFormattedText(formatted):
+    """
+    Assemble formatted text from structured information.
+
+    Currently handled formatting includes: bold, reverse, underline,
+    mIRC color codes and the ability to remove all current formatting.
+
+    It is worth noting that assembled text will always begin with the control
+    code to disable other attributes for the sake of correctness.
+
+    For example::
+
+        from twisted.words.protocols.irc import attributes as A
+        assembleFormattedText(
+            A.normal[A.bold['Time: '], A.fg.lightRed['Now!']])
+
+    Would produce "Time: " in bold formatting, followed by "Now!" with a
+    foreground color of light red and without any additional formatting.
+
+    Available attributes are:
+        - bold
+        - reverseVideo
+        - underline
+
+    Available colors are:
+        0. white
+        1. black
+        2. blue
+        3. green
+        4. light red
+        5. red
+        6. magenta
+        7. orange
+        8. yellow
+        9. light green
+        10. cyan
+        11. light cyan
+        12. light blue
+        13. light magenta
+        14. gray
+        15. light gray
+
+    @see: U{http://www.mirc.co.uk/help/color.txt}
+
+    @param formatted: Structured text and attributes.
+
+    @rtype: C{str}
+    @return: String containing mIRC control sequences that mimic those
+        specified by I{formatted}.
+
+    @since: 13.1
+    """
+    return _textattributes.flatten(
+        formatted, _FormattingState(), 'toMIRCControlCodes')
+
+
+
+def stripFormatting(text):
+    """
+    Remove all formatting codes from C{text}, leaving only the text.
+
+    @type text: C{str}
+    @param text: Formatted text to parse.
+
+    @rtype: C{str}
+    @return: Plain text without any control sequences.
+
+    @since: 13.1
+    """
+    formatted = parseFormattedText(text)
+    return _textattributes.flatten(
+        formatted, _textattributes.DefaultFormattingState())
+
+
+
+# CTCP constants and helper functions
+
+X_DELIM = chr(0o01)
+
+def ctcpExtract(message):
+    """
+    Extract CTCP data from a string.
+
+    @return: A C{dict} containing two keys:
+       - C{'extended'}: A list of CTCP (tag, data) tuples.
+       - C{'normal'}: A list of strings which were not inside a CTCP delimiter.
+    """
+    extended_messages = []
+    normal_messages = []
+    retval = {'extended': extended_messages,
+              'normal': normal_messages }
+
+    messages = message.split(X_DELIM)
+    odd = 0
+
+    # X1 extended data X2 nomal data X3 extended data X4 normal...
+    while messages:
+        if odd:
+            extended_messages.append(messages.pop(0))
+        else:
+            normal_messages.append(messages.pop(0))
+        odd = not odd
+
+    extended_messages[:] = filter(None, extended_messages)
+    normal_messages[:] = filter(None, normal_messages)
+
+    extended_messages[:] = map(ctcpDequote, extended_messages)
+    for i in range(len(extended_messages)):
+        m = extended_messages[i].split(SPC, 1)
+        tag = m[0]
+        if len(m) > 1:
+            data = m[1]
+        else:
+            data = None
+
+        extended_messages[i] = (tag, data)
+
+    return retval
+
+# CTCP escaping
+
+M_QUOTE= chr(0o20)
+
+mQuoteTable = {
+    NUL: M_QUOTE + '0',
+    NL: M_QUOTE + 'n',
+    CR: M_QUOTE + 'r',
+    M_QUOTE: M_QUOTE + M_QUOTE
+    }
+
+mDequoteTable = {}
+for k, v in mQuoteTable.items():
+    mDequoteTable[v[-1]] = k
+del k, v
+
+mEscape_re = re.compile('%s.' % (re.escape(M_QUOTE),), re.DOTALL)
+
+def lowQuote(s):
+    for c in (M_QUOTE, NUL, NL, CR):
+        s = s.replace(c, mQuoteTable[c])
+    return s
+
+def lowDequote(s):
+    def sub(matchobj, mDequoteTable=mDequoteTable):
+        s = matchobj.group()[1]
+        try:
+            s = mDequoteTable[s]
+        except KeyError:
+            s = s
+        return s
+
+    return mEscape_re.sub(sub, s)
+
+X_QUOTE = '\\'
+
+xQuoteTable = {
+    X_DELIM: X_QUOTE + 'a',
+    X_QUOTE: X_QUOTE + X_QUOTE
+    }
+
+xDequoteTable = {}
+
+for k, v in xQuoteTable.items():
+    xDequoteTable[v[-1]] = k
+
+xEscape_re = re.compile('%s.' % (re.escape(X_QUOTE),), re.DOTALL)
+
+def ctcpQuote(s):
+    for c in (X_QUOTE, X_DELIM):
+        s = s.replace(c, xQuoteTable[c])
+    return s
+
+def ctcpDequote(s):
+    def sub(matchobj, xDequoteTable=xDequoteTable):
+        s = matchobj.group()[1]
+        try:
+            s = xDequoteTable[s]
+        except KeyError:
+            s = s
+        return s
+
+    return xEscape_re.sub(sub, s)
+
+def ctcpStringify(messages):
+    """
+    @type messages: a list of extended messages.  An extended
+    message is a (tag, data) tuple, where 'data' may be L{None}, a
+    string, or a list of strings to be joined with whitespace.
+
+    @returns: String
+    """
+    coded_messages = []
+    for (tag, data) in messages:
+        if data:
+            if not isinstance(data, str):
+                try:
+                    # data as list-of-strings
+                    data = " ".join(map(str, data))
+                except TypeError:
+                    # No?  Then use it's %s representation.
+                    pass
+            m = "%s %s" % (tag, data)
+        else:
+            m = str(tag)
+        m = ctcpQuote(m)
+        m = "%s%s%s" % (X_DELIM, m, X_DELIM)
+        coded_messages.append(m)
+
+    line = ''.join(coded_messages)
+    return line
+
+
+# Constants (from RFC 2812)
+RPL_WELCOME = '001'
+RPL_YOURHOST = '002'
+RPL_CREATED = '003'
+RPL_MYINFO = '004'
+RPL_ISUPPORT = '005'
+RPL_BOUNCE = '010'
+RPL_USERHOST = '302'
+RPL_ISON = '303'
+RPL_AWAY = '301'
+RPL_UNAWAY = '305'
+RPL_NOWAWAY = '306'
+RPL_WHOISUSER = '311'
+RPL_WHOISSERVER = '312'
+RPL_WHOISOPERATOR = '313'
+RPL_WHOISIDLE = '317'
+RPL_ENDOFWHOIS = '318'
+RPL_WHOISCHANNELS = '319'
+RPL_WHOWASUSER = '314'
+RPL_ENDOFWHOWAS = '369'
+RPL_LISTSTART = '321'
+RPL_LIST = '322'
+RPL_LISTEND = '323'
+RPL_UNIQOPIS = '325'
+RPL_CHANNELMODEIS = '324'
+RPL_NOTOPIC = '331'
+RPL_TOPIC = '332'
+RPL_INVITING = '341'
+RPL_SUMMONING = '342'
+RPL_INVITELIST = '346'
+RPL_ENDOFINVITELIST = '347'
+RPL_EXCEPTLIST = '348'
+RPL_ENDOFEXCEPTLIST = '349'
+RPL_VERSION = '351'
+RPL_WHOREPLY = '352'
+RPL_ENDOFWHO = '315'
+RPL_NAMREPLY = '353'
+RPL_ENDOFNAMES = '366'
+RPL_LINKS = '364'
+RPL_ENDOFLINKS = '365'
+RPL_BANLIST = '367'
+RPL_ENDOFBANLIST = '368'
+RPL_INFO = '371'
+RPL_ENDOFINFO = '374'
+RPL_MOTDSTART = '375'
+RPL_MOTD = '372'
+RPL_ENDOFMOTD = '376'
+RPL_YOUREOPER = '381'
+RPL_REHASHING = '382'
+RPL_YOURESERVICE = '383'
+RPL_TIME = '391'
+RPL_USERSSTART = '392'
+RPL_USERS = '393'
+RPL_ENDOFUSERS = '394'
+RPL_NOUSERS = '395'
+RPL_TRACELINK = '200'
+RPL_TRACECONNECTING = '201'
+RPL_TRACEHANDSHAKE = '202'
+RPL_TRACEUNKNOWN = '203'
+RPL_TRACEOPERATOR = '204'
+RPL_TRACEUSER = '205'
+RPL_TRACESERVER = '206'
+RPL_TRACESERVICE = '207'
+RPL_TRACENEWTYPE = '208'
+RPL_TRACECLASS = '209'
+RPL_TRACERECONNECT = '210'
+RPL_TRACELOG = '261'
+RPL_TRACEEND = '262'
+RPL_STATSLINKINFO = '211'
+RPL_STATSCOMMANDS = '212'
+RPL_ENDOFSTATS = '219'
+RPL_STATSUPTIME = '242'
+RPL_STATSOLINE = '243'
+RPL_UMODEIS = '221'
+RPL_SERVLIST = '234'
+RPL_SERVLISTEND = '235'
+RPL_LUSERCLIENT = '251'
+RPL_LUSEROP = '252'
+RPL_LUSERUNKNOWN = '253'
+RPL_LUSERCHANNELS = '254'
+RPL_LUSERME = '255'
+RPL_ADMINME = '256'
+RPL_ADMINLOC1 = '257'
+RPL_ADMINLOC2 = '258'
+RPL_ADMINEMAIL = '259'
+RPL_TRYAGAIN = '263'
+ERR_NOSUCHNICK = '401'
+ERR_NOSUCHSERVER = '402'
+ERR_NOSUCHCHANNEL = '403'
+ERR_CANNOTSENDTOCHAN = '404'
+ERR_TOOMANYCHANNELS = '405'
+ERR_WASNOSUCHNICK = '406'
+ERR_TOOMANYTARGETS = '407'
+ERR_NOSUCHSERVICE = '408'
+ERR_NOORIGIN = '409'
+ERR_NORECIPIENT = '411'
+ERR_NOTEXTTOSEND = '412'
+ERR_NOTOPLEVEL = '413'
+ERR_WILDTOPLEVEL = '414'
+ERR_BADMASK = '415'
+# Defined in errata.
+# https://www.rfc-editor.org/errata_search.php?rfc=2812&eid=2822
+ERR_TOOMANYMATCHES = '416'
+ERR_UNKNOWNCOMMAND = '421'
+ERR_NOMOTD = '422'
+ERR_NOADMININFO = '423'
+ERR_FILEERROR = '424'
+ERR_NONICKNAMEGIVEN = '431'
+ERR_ERRONEUSNICKNAME = '432'
+ERR_NICKNAMEINUSE = '433'
+ERR_NICKCOLLISION = '436'
+ERR_UNAVAILRESOURCE = '437'
+ERR_USERNOTINCHANNEL = '441'
+ERR_NOTONCHANNEL = '442'
+ERR_USERONCHANNEL = '443'
+ERR_NOLOGIN = '444'
+ERR_SUMMONDISABLED = '445'
+ERR_USERSDISABLED = '446'
+ERR_NOTREGISTERED = '451'
+ERR_NEEDMOREPARAMS = '461'
+ERR_ALREADYREGISTRED = '462'
+ERR_NOPERMFORHOST = '463'
+ERR_PASSWDMISMATCH = '464'
+ERR_YOUREBANNEDCREEP = '465'
+ERR_YOUWILLBEBANNED = '466'
+ERR_KEYSET = '467'
+ERR_CHANNELISFULL = '471'
+ERR_UNKNOWNMODE = '472'
+ERR_INVITEONLYCHAN = '473'
+ERR_BANNEDFROMCHAN = '474'
+ERR_BADCHANNELKEY = '475'
+ERR_BADCHANMASK = '476'
+ERR_NOCHANMODES = '477'
+ERR_BANLISTFULL = '478'
+ERR_NOPRIVILEGES = '481'
+ERR_CHANOPRIVSNEEDED = '482'
+ERR_CANTKILLSERVER = '483'
+ERR_RESTRICTED = '484'
+ERR_UNIQOPPRIVSNEEDED = '485'
+ERR_NOOPERHOST = '491'
+ERR_NOSERVICEHOST = '492'
+ERR_UMODEUNKNOWNFLAG = '501'
+ERR_USERSDONTMATCH = '502'
+
+# And hey, as long as the strings are already intern'd...
+symbolic_to_numeric = {
+    "RPL_WELCOME": '001',
+    "RPL_YOURHOST": '002',
+    "RPL_CREATED": '003',
+    "RPL_MYINFO": '004',
+    "RPL_ISUPPORT": '005',
+    "RPL_BOUNCE": '010',
+    "RPL_USERHOST": '302',
+    "RPL_ISON": '303',
+    "RPL_AWAY": '301',
+    "RPL_UNAWAY": '305',
+    "RPL_NOWAWAY": '306',
+    "RPL_WHOISUSER": '311',
+    "RPL_WHOISSERVER": '312',
+    "RPL_WHOISOPERATOR": '313',
+    "RPL_WHOISIDLE": '317',
+    "RPL_ENDOFWHOIS": '318',
+    "RPL_WHOISCHANNELS": '319',
+    "RPL_WHOWASUSER": '314',
+    "RPL_ENDOFWHOWAS": '369',
+    "RPL_LISTSTART": '321',
+    "RPL_LIST": '322',
+    "RPL_LISTEND": '323',
+    "RPL_UNIQOPIS": '325',
+    "RPL_CHANNELMODEIS": '324',
+    "RPL_NOTOPIC": '331',
+    "RPL_TOPIC": '332',
+    "RPL_INVITING": '341',
+    "RPL_SUMMONING": '342',
+    "RPL_INVITELIST": '346',
+    "RPL_ENDOFINVITELIST": '347',
+    "RPL_EXCEPTLIST": '348',
+    "RPL_ENDOFEXCEPTLIST": '349',
+    "RPL_VERSION": '351',
+    "RPL_WHOREPLY": '352',
+    "RPL_ENDOFWHO": '315',
+    "RPL_NAMREPLY": '353',
+    "RPL_ENDOFNAMES": '366',
+    "RPL_LINKS": '364',
+    "RPL_ENDOFLINKS": '365',
+    "RPL_BANLIST": '367',
+    "RPL_ENDOFBANLIST": '368',
+    "RPL_INFO": '371',
+    "RPL_ENDOFINFO": '374',
+    "RPL_MOTDSTART": '375',
+    "RPL_MOTD": '372',
+    "RPL_ENDOFMOTD": '376',
+    "RPL_YOUREOPER": '381',
+    "RPL_REHASHING": '382',
+    "RPL_YOURESERVICE": '383',
+    "RPL_TIME": '391',
+    "RPL_USERSSTART": '392',
+    "RPL_USERS": '393',
+    "RPL_ENDOFUSERS": '394',
+    "RPL_NOUSERS": '395',
+    "RPL_TRACELINK": '200',
+    "RPL_TRACECONNECTING": '201',
+    "RPL_TRACEHANDSHAKE": '202',
+    "RPL_TRACEUNKNOWN": '203',
+    "RPL_TRACEOPERATOR": '204',
+    "RPL_TRACEUSER": '205',
+    "RPL_TRACESERVER": '206',
+    "RPL_TRACESERVICE": '207',
+    "RPL_TRACENEWTYPE": '208',
+    "RPL_TRACECLASS": '209',
+    "RPL_TRACERECONNECT": '210',
+    "RPL_TRACELOG": '261',
+    "RPL_TRACEEND": '262',
+    "RPL_STATSLINKINFO": '211',
+    "RPL_STATSCOMMANDS": '212',
+    "RPL_ENDOFSTATS": '219',
+    "RPL_STATSUPTIME": '242',
+    "RPL_STATSOLINE": '243',
+    "RPL_UMODEIS": '221',
+    "RPL_SERVLIST": '234',
+    "RPL_SERVLISTEND": '235',
+    "RPL_LUSERCLIENT": '251',
+    "RPL_LUSEROP": '252',
+    "RPL_LUSERUNKNOWN": '253',
+    "RPL_LUSERCHANNELS": '254',
+    "RPL_LUSERME": '255',
+    "RPL_ADMINME": '256',
+    "RPL_ADMINLOC1": '257',
+    "RPL_ADMINLOC2": '258',
+    "RPL_ADMINEMAIL": '259',
+    "RPL_TRYAGAIN": '263',
+    "ERR_NOSUCHNICK": '401',
+    "ERR_NOSUCHSERVER": '402',
+    "ERR_NOSUCHCHANNEL": '403',
+    "ERR_CANNOTSENDTOCHAN": '404',
+    "ERR_TOOMANYCHANNELS": '405',
+    "ERR_WASNOSUCHNICK": '406',
+    "ERR_TOOMANYTARGETS": '407',
+    "ERR_NOSUCHSERVICE": '408',
+    "ERR_NOORIGIN": '409',
+    "ERR_NORECIPIENT": '411',
+    "ERR_NOTEXTTOSEND": '412',
+    "ERR_NOTOPLEVEL": '413',
+    "ERR_WILDTOPLEVEL": '414',
+    "ERR_BADMASK": '415',
+    "ERR_TOOMANYMATCHES": '416',
+    "ERR_UNKNOWNCOMMAND": '421',
+    "ERR_NOMOTD": '422',
+    "ERR_NOADMININFO": '423',
+    "ERR_FILEERROR": '424',
+    "ERR_NONICKNAMEGIVEN": '431',
+    "ERR_ERRONEUSNICKNAME": '432',
+    "ERR_NICKNAMEINUSE": '433',
+    "ERR_NICKCOLLISION": '436',
+    "ERR_UNAVAILRESOURCE": '437',
+    "ERR_USERNOTINCHANNEL": '441',
+    "ERR_NOTONCHANNEL": '442',
+    "ERR_USERONCHANNEL": '443',
+    "ERR_NOLOGIN": '444',
+    "ERR_SUMMONDISABLED": '445',
+    "ERR_USERSDISABLED": '446',
+    "ERR_NOTREGISTERED": '451',
+    "ERR_NEEDMOREPARAMS": '461',
+    "ERR_ALREADYREGISTRED": '462',
+    "ERR_NOPERMFORHOST": '463',
+    "ERR_PASSWDMISMATCH": '464',
+    "ERR_YOUREBANNEDCREEP": '465',
+    "ERR_YOUWILLBEBANNED": '466',
+    "ERR_KEYSET": '467',
+    "ERR_CHANNELISFULL": '471',
+    "ERR_UNKNOWNMODE": '472',
+    "ERR_INVITEONLYCHAN": '473',
+    "ERR_BANNEDFROMCHAN": '474',
+    "ERR_BADCHANNELKEY": '475',
+    "ERR_BADCHANMASK": '476',
+    "ERR_NOCHANMODES": '477',
+    "ERR_BANLISTFULL": '478',
+    "ERR_NOPRIVILEGES": '481',
+    "ERR_CHANOPRIVSNEEDED": '482',
+    "ERR_CANTKILLSERVER": '483',
+    "ERR_RESTRICTED": '484',
+    "ERR_UNIQOPPRIVSNEEDED": '485',
+    "ERR_NOOPERHOST": '491',
+    "ERR_NOSERVICEHOST": '492',
+    "ERR_UMODEUNKNOWNFLAG": '501',
+    "ERR_USERSDONTMATCH": '502',
+}
+
+numeric_to_symbolic = {}
+for k, v in symbolic_to_numeric.items():
+    numeric_to_symbolic[v] = k
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/__init__.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/__init__.py
new file mode 100644
index 00000000000..ad95b6853ec
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/__init__.py
@@ -0,0 +1,8 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Twisted Jabber: Jabber Protocol Helpers
+"""
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/client.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/client.py
new file mode 100644
index 00000000000..8f197cdafe1
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/client.py
@@ -0,0 +1,408 @@
+# -*- test-case-name: twisted.words.test.test_jabberclient -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from __future__ import absolute_import, division
+
+from twisted.python.compat import _coercedUnicode, unicode
+from twisted.words.protocols.jabber import xmlstream, sasl, error
+from twisted.words.protocols.jabber.jid import JID
+from twisted.words.xish import domish, xpath, utility
+
+NS_XMPP_STREAMS = 'urn:ietf:params:xml:ns:xmpp-streams'
+NS_XMPP_BIND = 'urn:ietf:params:xml:ns:xmpp-bind'
+NS_XMPP_SESSION = 'urn:ietf:params:xml:ns:xmpp-session'
+NS_IQ_AUTH_FEATURE = 'http://jabber.org/features/iq-auth'
+
+DigestAuthQry = xpath.internQuery("/iq/query/digest")
+PlaintextAuthQry = xpath.internQuery("/iq/query/password")
+
+def basicClientFactory(jid, secret):
+    a = BasicAuthenticator(jid, secret)
+    return xmlstream.XmlStreamFactory(a)
+
+class IQ(domish.Element):
+    """
+    Wrapper for a Info/Query packet.
+
+    This provides the necessary functionality to send IQs and get notified when
+    a result comes back. It's a subclass from L{domish.Element}, so you can use
+    the standard DOM manipulation calls to add data to the outbound request.
+
+    @type callbacks: L{utility.CallbackList}
+    @cvar callbacks: Callback list to be notified when response comes back
+
+    """
+    def __init__(self, xmlstream, type = "set"):
+        """
+        @type xmlstream: L{xmlstream.XmlStream}
+        @param xmlstream: XmlStream to use for transmission of this IQ
+
+        @type type: C{str}
+        @param type: IQ type identifier ('get' or 'set')
+        """
+
+        domish.Element.__init__(self, ("jabber:client", "iq"))
+        self.addUniqueId()
+        self["type"] = type
+        self._xmlstream = xmlstream
+        self.callbacks = utility.CallbackList()
+
+    def addCallback(self, fn, *args, **kwargs):
+        """
+        Register a callback for notification when the IQ result is available.
+        """
+
+        self.callbacks.addCallback(True, fn, *args, **kwargs)
+
+    def send(self, to = None):
+        """
+        Call this method to send this IQ request via the associated XmlStream.
+
+        @param to: Jabber ID of the entity to send the request to
+        @type to: C{str}
+
+        @returns: Callback list for this IQ. Any callbacks added to this list
+                  will be fired when the result comes back.
+        """
+        if to != None:
+            self["to"] = to
+        self._xmlstream.addOnetimeObserver("/iq[@id='%s']" % self["id"], \
+                                                             self._resultEvent)
+        self._xmlstream.send(self)
+
+    def _resultEvent(self, iq):
+        self.callbacks.callback(iq)
+        self.callbacks = None
+
+
+
+class IQAuthInitializer(object):
+    """
+    Non-SASL Authentication initializer for the initiating entity.
+
+    This protocol is defined in
+    U{JEP-0078<http://www.jabber.org/jeps/jep-0078.html>} and mainly serves for
+    compatibility with pre-XMPP-1.0 server implementations.
+
+    @cvar INVALID_USER_EVENT: Token to signal that authentication failed, due
+        to invalid username.
+    @type INVALID_USER_EVENT: L{str}
+
+    @cvar AUTH_FAILED_EVENT: Token to signal that authentication failed, due to
+        invalid password.
+    @type AUTH_FAILED_EVENT: L{str}
+    """
+
+    INVALID_USER_EVENT    = "//event/client/basicauth/invaliduser"
+    AUTH_FAILED_EVENT     = "//event/client/basicauth/authfailed"
+
+    def __init__(self, xs):
+        self.xmlstream = xs
+
+
+    def initialize(self):
+        # Send request for auth fields
+        iq = xmlstream.IQ(self.xmlstream, "get")
+        iq.addElement(("jabber:iq:auth", "query"))
+        jid = self.xmlstream.authenticator.jid
+        iq.query.addElement("username", content = jid.user)
+
+        d = iq.send()
+        d.addCallbacks(self._cbAuthQuery, self._ebAuthQuery)
+        return d
+
+
+    def _cbAuthQuery(self, iq):
+        jid = self.xmlstream.authenticator.jid
+        password = _coercedUnicode(self.xmlstream.authenticator.password)
+
+        # Construct auth request
+        reply = xmlstream.IQ(self.xmlstream, "set")
+        reply.addElement(("jabber:iq:auth", "query"))
+        reply.query.addElement("username", content = jid.user)
+        reply.query.addElement("resource", content = jid.resource)
+
+        # Prefer digest over plaintext
+        if DigestAuthQry.matches(iq):
+            digest = xmlstream.hashPassword(self.xmlstream.sid, password)
+            reply.query.addElement("digest", content=unicode(digest))
+        else:
+            reply.query.addElement("password", content = password)
+
+        d = reply.send()
+        d.addCallbacks(self._cbAuth, self._ebAuth)
+        return d
+
+
+    def _ebAuthQuery(self, failure):
+        failure.trap(error.StanzaError)
+        e = failure.value
+        if e.condition == 'not-authorized':
+            self.xmlstream.dispatch(e.stanza, self.INVALID_USER_EVENT)
+        else:
+            self.xmlstream.dispatch(e.stanza, self.AUTH_FAILED_EVENT)
+
+        return failure
+
+
+    def _cbAuth(self, iq):
+        pass
+
+
+    def _ebAuth(self, failure):
+        failure.trap(error.StanzaError)
+        self.xmlstream.dispatch(failure.value.stanza, self.AUTH_FAILED_EVENT)
+        return failure
+
+
+
+class BasicAuthenticator(xmlstream.ConnectAuthenticator):
+    """
+    Authenticates an XmlStream against a Jabber server as a Client.
+
+    This only implements non-SASL authentication, per
+    U{JEP-0078<http://www.jabber.org/jeps/jep-0078.html>}. Additionally, this
+    authenticator provides the ability to perform inline registration, per
+    U{JEP-0077<http://www.jabber.org/jeps/jep-0077.html>}.
+
+    Under normal circumstances, the BasicAuthenticator generates the
+    L{xmlstream.STREAM_AUTHD_EVENT} once the stream has authenticated. However,
+    it can also generate other events, such as:
+      - L{INVALID_USER_EVENT} : Authentication failed, due to invalid username
+      - L{AUTH_FAILED_EVENT} : Authentication failed, due to invalid password
+      - L{REGISTER_FAILED_EVENT} : Registration failed
+
+    If authentication fails for any reason, you can attempt to register by
+    calling the L{registerAccount} method. If the registration succeeds, a
+    L{xmlstream.STREAM_AUTHD_EVENT} will be fired. Otherwise, one of the above
+    errors will be generated (again).
+
+
+    @cvar INVALID_USER_EVENT: See L{IQAuthInitializer.INVALID_USER_EVENT}.
+    @type INVALID_USER_EVENT: L{str}
+
+    @cvar AUTH_FAILED_EVENT: See L{IQAuthInitializer.AUTH_FAILED_EVENT}.
+    @type AUTH_FAILED_EVENT: L{str}
+
+    @cvar REGISTER_FAILED_EVENT: Token to signal that registration failed.
+    @type REGISTER_FAILED_EVENT: L{str}
+
+    """
+
+    namespace = "jabber:client"
+
+    INVALID_USER_EVENT = IQAuthInitializer.INVALID_USER_EVENT
+    AUTH_FAILED_EVENT = IQAuthInitializer.AUTH_FAILED_EVENT
+    REGISTER_FAILED_EVENT = "//event/client/basicauth/registerfailed"
+
+    def __init__(self, jid, password):
+        xmlstream.ConnectAuthenticator.__init__(self, jid.host)
+        self.jid = jid
+        self.password = password
+
+    def associateWithStream(self, xs):
+        xs.version = (0, 0)
+        xmlstream.ConnectAuthenticator.associateWithStream(self, xs)
+
+        xs.initializers = [
+            xmlstream.TLSInitiatingInitializer(xs, required=False),
+            IQAuthInitializer(xs),
+        ]
+
+    # TODO: move registration into an Initializer?
+
+    def registerAccount(self, username = None, password = None):
+        if username:
+            self.jid.user = username
+        if password:
+            self.password = password
+
+        iq = IQ(self.xmlstream, "set")
+        iq.addElement(("jabber:iq:register", "query"))
+        iq.query.addElement("username", content = self.jid.user)
+        iq.query.addElement("password", content = self.password)
+
+        iq.addCallback(self._registerResultEvent)
+
+        iq.send()
+
+    def _registerResultEvent(self, iq):
+        if iq["type"] == "result":
+            # Registration succeeded -- go ahead and auth
+            self.streamStarted()
+        else:
+            # Registration failed
+            self.xmlstream.dispatch(iq, self.REGISTER_FAILED_EVENT)
+
+
+
+class CheckVersionInitializer(object):
+    """
+    Initializer that checks if the minimum common stream version number is 1.0.
+    """
+
+    def __init__(self, xs):
+        self.xmlstream = xs
+
+
+    def initialize(self):
+        if self.xmlstream.version < (1, 0):
+            raise error.StreamError('unsupported-version')
+
+
+
+class BindInitializer(xmlstream.BaseFeatureInitiatingInitializer):
+    """
+    Initializer that implements Resource Binding for the initiating entity.
+
+    This protocol is documented in U{RFC 3920, section
+    7<http://www.xmpp.org/specs/rfc3920.html#bind>}.
+    """
+
+    feature = (NS_XMPP_BIND, 'bind')
+
+    def start(self):
+        iq = xmlstream.IQ(self.xmlstream, 'set')
+        bind = iq.addElement((NS_XMPP_BIND, 'bind'))
+        resource = self.xmlstream.authenticator.jid.resource
+        if resource:
+            bind.addElement('resource', content=resource)
+        d = iq.send()
+        d.addCallback(self.onBind)
+        return d
+
+
+    def onBind(self, iq):
+        if iq.bind:
+            self.xmlstream.authenticator.jid = JID(unicode(iq.bind.jid))
+
+
+
+class SessionInitializer(xmlstream.BaseFeatureInitiatingInitializer):
+    """
+    Initializer that implements session establishment for the initiating
+    entity.
+
+    This protocol is defined in U{RFC 3921, section
+    3<http://www.xmpp.org/specs/rfc3921.html#session>}.
+    """
+
+    feature = (NS_XMPP_SESSION, 'session')
+
+    def start(self):
+        iq = xmlstream.IQ(self.xmlstream, 'set')
+        iq.addElement((NS_XMPP_SESSION, 'session'))
+        return iq.send()
+
+
+
+def XMPPClientFactory(jid, password, configurationForTLS=None):
+    """
+    Client factory for XMPP 1.0 (only).
+
+    This returns a L{xmlstream.XmlStreamFactory} with an L{XMPPAuthenticator}
+    object to perform the stream initialization steps (such as authentication).
+
+    @see: The notes at L{XMPPAuthenticator} describe how the L{jid} and
+    L{password} parameters are to be used.
+
+    @param jid: Jabber ID to connect with.
+    @type jid: L{jid.JID}
+
+    @param password: password to authenticate with.
+    @type password: L{unicode}
+
+    @param configurationForTLS: An object which creates appropriately
+        configured TLS connections. This is passed to C{startTLS} on the
+        transport and is preferably created using
+        L{twisted.internet.ssl.optionsForClientTLS}. If C{None}, the default is
+        to verify the server certificate against the trust roots as provided by
+        the platform. See L{twisted.internet._sslverify.platformTrust}.
+    @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or C{None}
+
+    @return: XML stream factory.
+    @rtype: L{xmlstream.XmlStreamFactory}
+    """
+    a = XMPPAuthenticator(jid, password,
+                          configurationForTLS=configurationForTLS)
+    return xmlstream.XmlStreamFactory(a)
+
+
+
+class XMPPAuthenticator(xmlstream.ConnectAuthenticator):
+    """
+    Initializes an XmlStream connecting to an XMPP server as a Client.
+
+    This authenticator performs the initialization steps needed to start
+    exchanging XML stanzas with an XMPP server as an XMPP client. It checks if
+    the server advertises XML stream version 1.0, negotiates TLS (when
+    available), performs SASL authentication, binds a resource and establishes
+    a session.
+
+    Upon successful stream initialization, the L{xmlstream.STREAM_AUTHD_EVENT}
+    event will be dispatched through the XML stream object. Otherwise, the
+    L{xmlstream.INIT_FAILED_EVENT} event will be dispatched with a failure
+    object.
+
+    After inspection of the failure, initialization can then be restarted by
+    calling L{ConnectAuthenticator.initializeStream}. For example, in case of
+    authentication failure, a user may be given the opportunity to input the
+    correct password.  By setting the L{password} instance variable and restarting
+    initialization, the stream authentication step is then retried, and subsequent
+    steps are performed if successful.
+
+    @ivar jid: Jabber ID to authenticate with. This may contain a resource
+               part, as a suggestion to the server for resource binding. A
+               server may override this, though. If the resource part is left
+               off, the server will generate a unique resource identifier.
+               The server will always return the full Jabber ID in the
+               resource binding step, and this is stored in this instance
+               variable.
+    @type jid: L{jid.JID}
+
+    @ivar password: password to be used during SASL authentication.
+    @type password: L{unicode}
+    """
+
+    namespace = 'jabber:client'
+
+    def __init__(self, jid, password, configurationForTLS=None):
+        """
+        @param configurationForTLS: An object which creates appropriately
+            configured TLS connections. This is passed to C{startTLS} on the
+            transport and is preferably created using
+            L{twisted.internet.ssl.optionsForClientTLS}. If C{None}, the
+            default is to verify the server certificate against the trust roots
+            as provided by the platform. See
+            L{twisted.internet._sslverify.platformTrust}.
+        @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or
+            C{None}
+        """
+        xmlstream.ConnectAuthenticator.__init__(self, jid.host)
+        self.jid = jid
+        self.password = password
+        self._configurationForTLS = configurationForTLS
+
+
+    def associateWithStream(self, xs):
+        """
+        Register with the XML stream.
+
+        Populates stream's list of initializers, along with their
+        requiredness. This list is used by
+        L{ConnectAuthenticator.initializeStream} to perform the initialization
+        steps.
+        """
+        xmlstream.ConnectAuthenticator.associateWithStream(self, xs)
+
+        xs.initializers = [
+                CheckVersionInitializer(xs),
+                xmlstream.TLSInitiatingInitializer(
+                    xs, required=True,
+                    configurationForTLS=self._configurationForTLS),
+                sasl.SASLInitiatingInitializer(xs, required=True),
+                BindInitializer(xs, required=True),
+                SessionInitializer(xs, required=False),
+                ]
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/component.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/component.py
new file mode 100644
index 00000000000..796550577a6
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/component.py
@@ -0,0 +1,475 @@
+# -*- test-case-name: twisted.words.test.test_jabbercomponent -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+External server-side components.
+
+Most Jabber server implementations allow for add-on components that act as a
+separate entity on the Jabber network, but use the server-to-server
+functionality of a regular Jabber IM server. These so-called 'external
+components' are connected to the Jabber server using the Jabber Component
+Protocol as defined in U{JEP-0114<http://www.jabber.org/jeps/jep-0114.html>}.
+
+This module allows for writing external server-side component by assigning one
+or more services implementing L{ijabber.IService} up to L{ServiceManager}. The
+ServiceManager connects to the Jabber server and is responsible for the
+corresponding XML stream.
+"""
+
+from zope.interface import implementer
+
+from twisted.application import service
+from twisted.internet import defer
+from twisted.python import log
+from twisted.python.compat import _coercedUnicode, unicode
+from twisted.words.xish import domish
+from twisted.words.protocols.jabber import error, ijabber, jstrports, xmlstream
+from twisted.words.protocols.jabber.jid import internJID as JID
+
+NS_COMPONENT_ACCEPT = 'jabber:component:accept'
+
+def componentFactory(componentid, password):
+    """
+    XML stream factory for external server-side components.
+
+    @param componentid: JID of the component.
+    @type componentid: L{unicode}
+    @param password: password used to authenticate to the server.
+    @type password: C{str}
+    """
+    a = ConnectComponentAuthenticator(componentid, password)
+    return xmlstream.XmlStreamFactory(a)
+
+class ComponentInitiatingInitializer(object):
+    """
+    External server-side component authentication initializer for the
+    initiating entity.
+
+    @ivar xmlstream: XML stream between server and component.
+    @type xmlstream: L{xmlstream.XmlStream}
+    """
+
+    def __init__(self, xs):
+        self.xmlstream = xs
+        self._deferred = None
+
+    def initialize(self):
+        xs = self.xmlstream
+        hs = domish.Element((self.xmlstream.namespace, "handshake"))
+        digest = xmlstream.hashPassword(
+            xs.sid,
+            _coercedUnicode(xs.authenticator.password))
+        hs.addContent(unicode(digest))
+
+        # Setup observer to watch for handshake result
+        xs.addOnetimeObserver("/handshake", self._cbHandshake)
+        xs.send(hs)
+        self._deferred = defer.Deferred()
+        return self._deferred
+
+    def _cbHandshake(self, _):
+        # we have successfully shaken hands and can now consider this
+        # entity to represent the component JID.
+        self.xmlstream.thisEntity = self.xmlstream.otherEntity
+        self._deferred.callback(None)
+
+
+
+class ConnectComponentAuthenticator(xmlstream.ConnectAuthenticator):
+    """
+    Authenticator to permit an XmlStream to authenticate against a Jabber
+    server as an external component (where the Authenticator is initiating the
+    stream).
+    """
+    namespace = NS_COMPONENT_ACCEPT
+
+    def __init__(self, componentjid, password):
+        """
+        @type componentjid: C{str}
+        @param componentjid: Jabber ID that this component wishes to bind to.
+
+        @type password: C{str}
+        @param password: Password/secret this component uses to authenticate.
+        """
+        # Note that we are sending 'to' our desired component JID.
+        xmlstream.ConnectAuthenticator.__init__(self, componentjid)
+        self.password = password
+
+    def associateWithStream(self, xs):
+        xs.version = (0, 0)
+        xmlstream.ConnectAuthenticator.associateWithStream(self, xs)
+
+        xs.initializers = [ComponentInitiatingInitializer(xs)]
+
+
+
+class ListenComponentAuthenticator(xmlstream.ListenAuthenticator):
+    """
+    Authenticator for accepting components.
+
+    @since: 8.2
+    @ivar secret: The shared secret used to authorized incoming component
+                  connections.
+    @type secret: C{unicode}.
+    """
+
+    namespace = NS_COMPONENT_ACCEPT
+
+    def __init__(self, secret):
+        self.secret = secret
+        xmlstream.ListenAuthenticator.__init__(self)
+
+
+    def associateWithStream(self, xs):
+        """
+        Associate the authenticator with a stream.
+
+        This sets the stream's version to 0.0, because the XEP-0114 component
+        protocol was not designed for XMPP 1.0.
+        """
+        xs.version = (0, 0)
+        xmlstream.ListenAuthenticator.associateWithStream(self, xs)
+
+
+    def streamStarted(self, rootElement):
+        """
+        Called by the stream when it has started.
+
+        This examines the default namespace of the incoming stream and whether
+        there is a requested hostname for the component. Then it generates a
+        stream identifier, sends a response header and adds an observer for
+        the first incoming element, triggering L{onElement}.
+        """
+
+        xmlstream.ListenAuthenticator.streamStarted(self, rootElement)
+
+        if rootElement.defaultUri != self.namespace:
+            exc = error.StreamError('invalid-namespace')
+            self.xmlstream.sendStreamError(exc)
+            return
+
+        # self.xmlstream.thisEntity is set to the address the component
+        # wants to assume.
+        if not self.xmlstream.thisEntity:
+            exc = error.StreamError('improper-addressing')
+            self.xmlstream.sendStreamError(exc)
+            return
+
+        self.xmlstream.sendHeader()
+        self.xmlstream.addOnetimeObserver('/*', self.onElement)
+
+
+    def onElement(self, element):
+        """
+        Called on incoming XML Stanzas.
+
+        The very first element received should be a request for handshake.
+        Otherwise, the stream is dropped with a 'not-authorized' error. If a
+        handshake request was received, the hash is extracted and passed to
+        L{onHandshake}.
+        """
+        if (element.uri, element.name) == (self.namespace, 'handshake'):
+            self.onHandshake(unicode(element))
+        else:
+            exc = error.StreamError('not-authorized')
+            self.xmlstream.sendStreamError(exc)
+
+
+    def onHandshake(self, handshake):
+        """
+        Called upon receiving the handshake request.
+
+        This checks that the given hash in C{handshake} is equal to a
+        calculated hash, responding with a handshake reply or a stream error.
+        If the handshake was ok, the stream is authorized, and  XML Stanzas may
+        be exchanged.
+        """
+        calculatedHash = xmlstream.hashPassword(self.xmlstream.sid,
+                                                unicode(self.secret))
+        if handshake != calculatedHash:
+            exc = error.StreamError('not-authorized', text='Invalid hash')
+            self.xmlstream.sendStreamError(exc)
+        else:
+            self.xmlstream.send('<handshake/>')
+            self.xmlstream.dispatch(self.xmlstream,
+                                    xmlstream.STREAM_AUTHD_EVENT)
+
+
+
+@implementer(ijabber.IService)
+class Service(service.Service):
+    """
+    External server-side component service.
+    """
+
+    def componentConnected(self, xs):
+        pass
+
+    def componentDisconnected(self):
+        pass
+
+    def transportConnected(self, xs):
+        pass
+
+    def send(self, obj):
+        """
+        Send data over service parent's XML stream.
+
+        @note: L{ServiceManager} maintains a queue for data sent using this
+        method when there is no current established XML stream. This data is
+        then sent as soon as a new stream has been established and initialized.
+        Subsequently, L{componentConnected} will be called again. If this
+        queueing is not desired, use C{send} on the XmlStream object (passed to
+        L{componentConnected}) directly.
+
+        @param obj: data to be sent over the XML stream. This is usually an
+        object providing L{domish.IElement}, or serialized XML. See
+        L{xmlstream.XmlStream} for details.
+        """
+
+        self.parent.send(obj)
+
+class ServiceManager(service.MultiService):
+    """
+    Business logic for a managed component connection to a Jabber router.
+
+    This service maintains a single connection to a Jabber router and provides
+    facilities for packet routing and transmission. Business logic modules are
+    services implementing L{ijabber.IService} (like subclasses of L{Service}),
+    and added as sub-service.
+    """
+
+    def __init__(self, jid, password):
+        service.MultiService.__init__(self)
+
+        # Setup defaults
+        self.jabberId = jid
+        self.xmlstream = None
+
+        # Internal buffer of packets
+        self._packetQueue = []
+
+        # Setup the xmlstream factory
+        self._xsFactory = componentFactory(self.jabberId, password)
+
+        # Register some lambda functions to keep the self.xmlstream var up to
+        # date
+        self._xsFactory.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT,
+                                     self._connected)
+        self._xsFactory.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self._authd)
+        self._xsFactory.addBootstrap(xmlstream.STREAM_END_EVENT,
+                                     self._disconnected)
+
+        # Map addBootstrap and removeBootstrap to the underlying factory -- is
+        # this right? I have no clue...but it'll work for now, until i can
+        # think about it more.
+        self.addBootstrap = self._xsFactory.addBootstrap
+        self.removeBootstrap = self._xsFactory.removeBootstrap
+
+    def getFactory(self):
+        return self._xsFactory
+
+    def _connected(self, xs):
+        self.xmlstream = xs
+        for c in self:
+            if ijabber.IService.providedBy(c):
+                c.transportConnected(xs)
+
+    def _authd(self, xs):
+        # Flush all pending packets
+        for p in self._packetQueue:
+            self.xmlstream.send(p)
+        self._packetQueue = []
+
+        # Notify all child services which implement the IService interface
+        for c in self:
+            if ijabber.IService.providedBy(c):
+                c.componentConnected(xs)
+
+    def _disconnected(self, _):
+        self.xmlstream = None
+
+        # Notify all child services which implement
+        # the IService interface
+        for c in self:
+            if ijabber.IService.providedBy(c):
+                c.componentDisconnected()
+
+    def send(self, obj):
+        """
+        Send data over the XML stream.
+
+        When there is no established XML stream, the data is queued and sent
+        out when a new XML stream has been established and initialized.
+
+        @param obj: data to be sent over the XML stream. This is usually an
+        object providing L{domish.IElement}, or serialized XML. See
+        L{xmlstream.XmlStream} for details.
+        """
+
+        if self.xmlstream != None:
+            self.xmlstream.send(obj)
+        else:
+            self._packetQueue.append(obj)
+
+def buildServiceManager(jid, password, strport):
+    """
+    Constructs a pre-built L{ServiceManager}, using the specified strport
+    string.
+    """
+
+    svc = ServiceManager(jid, password)
+    client_svc = jstrports.client(strport, svc.getFactory())
+    client_svc.setServiceParent(svc)
+    return svc
+
+
+
+class Router(object):
+    """
+    XMPP Server's Router.
+
+    A router connects the different components of the XMPP service and routes
+    messages between them based on the given routing table.
+
+    Connected components are trusted to have correct addressing in the
+    stanzas they offer for routing.
+
+    A route destination of L{None} adds a default route. Traffic for which no
+    specific route exists, will be routed to this default route.
+
+    @since: 8.2
+    @ivar routes: Routes based on the host part of JIDs. Maps host names to the
+                  L{EventDispatcher<utility.EventDispatcher>}s that should
+                  receive the traffic. A key of L{None} means the default
+                  route.
+    @type routes: C{dict}
+    """
+
+    def __init__(self):
+        self.routes = {}
+
+
+    def addRoute(self, destination, xs):
+        """
+        Add a new route.
+
+        The passed XML Stream C{xs} will have an observer for all stanzas
+        added to route its outgoing traffic. In turn, traffic for
+        C{destination} will be passed to this stream.
+
+        @param destination: Destination of the route to be added as a host name
+                            or L{None} for the default route.
+        @type destination: C{str} or L{None}.
+        @param xs: XML Stream to register the route for.
+        @type xs: L{EventDispatcher<utility.EventDispatcher>}.
+        """
+        self.routes[destination] = xs
+        xs.addObserver('/*', self.route)
+
+
+    def removeRoute(self, destination, xs):
+        """
+        Remove a route.
+
+        @param destination: Destination of the route that should be removed.
+        @type destination: C{str}.
+        @param xs: XML Stream to remove the route for.
+        @type xs: L{EventDispatcher<utility.EventDispatcher>}.
+        """
+        xs.removeObserver('/*', self.route)
+        if (xs == self.routes[destination]):
+            del self.routes[destination]
+
+
+    def route(self, stanza):
+        """
+        Route a stanza.
+
+        @param stanza: The stanza to be routed.
+        @type stanza: L{domish.Element}.
+        """
+        destination = JID(stanza['to'])
+
+        log.msg("Routing to %s: %r" % (destination.full(), stanza.toXml()))
+
+        if destination.host in self.routes:
+            self.routes[destination.host].send(stanza)
+        else:
+            self.routes[None].send(stanza)
+
+
+
+class XMPPComponentServerFactory(xmlstream.XmlStreamServerFactory):
+    """
+    XMPP Component Server factory.
+
+    This factory accepts XMPP external component connections and makes
+    the router service route traffic for a component's bound domain
+    to that component.
+
+    @since: 8.2
+    """
+
+    logTraffic = False
+
+    def __init__(self, router, secret='secret'):
+        self.router = router
+        self.secret = secret
+
+        def authenticatorFactory():
+            return ListenComponentAuthenticator(self.secret)
+
+        xmlstream.XmlStreamServerFactory.__init__(self, authenticatorFactory)
+        self.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT,
+                          self.onConnectionMade)
+        self.addBootstrap(xmlstream.STREAM_AUTHD_EVENT,
+                          self.onAuthenticated)
+
+        self.serial = 0
+
+
+    def onConnectionMade(self, xs):
+        """
+        Called when a component connection was made.
+
+        This enables traffic debugging on incoming streams.
+        """
+        xs.serial = self.serial
+        self.serial += 1
+
+        def logDataIn(buf):
+            log.msg("RECV (%d): %r" % (xs.serial, buf))
+
+        def logDataOut(buf):
+            log.msg("SEND (%d): %r" % (xs.serial, buf))
+
+        if self.logTraffic:
+            xs.rawDataInFn = logDataIn
+            xs.rawDataOutFn = logDataOut
+
+        xs.addObserver(xmlstream.STREAM_ERROR_EVENT, self.onError)
+
+
+    def onAuthenticated(self, xs):
+        """
+        Called when a component has successfully authenticated.
+
+        Add the component to the routing table and establish a handler
+        for a closed connection.
+        """
+        destination = xs.thisEntity.host
+
+        self.router.addRoute(destination, xs)
+        xs.addObserver(xmlstream.STREAM_END_EVENT, self.onConnectionLost, 0,
+                                                   destination, xs)
+
+
+    def onError(self, reason):
+        log.err(reason, "Stream Error")
+
+
+    def onConnectionLost(self, destination, xs, reason):
+        self.router.removeRoute(destination, xs)
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/error.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/error.py
new file mode 100644
index 00000000000..481ae1de7e0
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/error.py
@@ -0,0 +1,331 @@
+# -*- test-case-name: twisted.words.test.test_jabbererror -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XMPP Error support.
+"""
+
+from __future__ import absolute_import, division
+
+import copy
+
+from twisted.python.compat import unicode
+from twisted.words.xish import domish
+
+NS_XML = "http://www.w3.org/XML/1998/namespace"
+NS_XMPP_STREAMS = "urn:ietf:params:xml:ns:xmpp-streams"
+NS_XMPP_STANZAS = "urn:ietf:params:xml:ns:xmpp-stanzas"
+
+STANZA_CONDITIONS = {
+    'bad-request':              {'code': '400', 'type': 'modify'},
+    'conflict':                 {'code': '409', 'type': 'cancel'},
+    'feature-not-implemented':  {'code': '501', 'type': 'cancel'},
+    'forbidden':                {'code': '403', 'type': 'auth'},
+    'gone':                     {'code': '302', 'type': 'modify'},
+    'internal-server-error':    {'code': '500', 'type': 'wait'},
+    'item-not-found':           {'code': '404', 'type': 'cancel'},
+    'jid-malformed':            {'code': '400', 'type': 'modify'},
+    'not-acceptable':           {'code': '406', 'type': 'modify'},
+    'not-allowed':              {'code': '405', 'type': 'cancel'},
+    'not-authorized':           {'code': '401', 'type': 'auth'},
+    'payment-required':         {'code': '402', 'type': 'auth'},
+    'recipient-unavailable':    {'code': '404', 'type': 'wait'},
+    'redirect':                 {'code': '302', 'type': 'modify'},
+    'registration-required':    {'code': '407', 'type': 'auth'},
+    'remote-server-not-found':  {'code': '404', 'type': 'cancel'},
+    'remote-server-timeout':    {'code': '504', 'type': 'wait'},
+    'resource-constraint':      {'code': '500', 'type': 'wait'},
+    'service-unavailable':      {'code': '503', 'type': 'cancel'},
+    'subscription-required':    {'code': '407', 'type': 'auth'},
+    'undefined-condition':      {'code': '500', 'type': None},
+    'unexpected-request':       {'code': '400', 'type': 'wait'},
+}
+
+CODES_TO_CONDITIONS = {
+    '302': ('gone', 'modify'),
+    '400': ('bad-request', 'modify'),
+    '401': ('not-authorized', 'auth'),
+    '402': ('payment-required', 'auth'),
+    '403': ('forbidden', 'auth'),
+    '404': ('item-not-found', 'cancel'),
+    '405': ('not-allowed', 'cancel'),
+    '406': ('not-acceptable', 'modify'),
+    '407': ('registration-required', 'auth'),
+    '408': ('remote-server-timeout', 'wait'),
+    '409': ('conflict', 'cancel'),
+    '500': ('internal-server-error', 'wait'),
+    '501': ('feature-not-implemented', 'cancel'),
+    '502': ('service-unavailable', 'wait'),
+    '503': ('service-unavailable', 'cancel'),
+    '504': ('remote-server-timeout', 'wait'),
+    '510': ('service-unavailable', 'cancel'),
+}
+
+class BaseError(Exception):
+    """
+    Base class for XMPP error exceptions.
+
+    @cvar namespace: The namespace of the C{error} element generated by
+                     C{getElement}.
+    @type namespace: C{str}
+    @ivar condition: The error condition. The valid values are defined by
+                     subclasses of L{BaseError}.
+    @type contition: C{str}
+    @ivar text: Optional text message to supplement the condition or application
+                specific condition.
+    @type text: C{unicode}
+    @ivar textLang: Identifier of the language used for the message in C{text}.
+                    Values are as described in RFC 3066.
+    @type textLang: C{str}
+    @ivar appCondition: Application specific condition element, supplementing
+                        the error condition in C{condition}.
+    @type appCondition: object providing L{domish.IElement}.
+    """
+
+    namespace = None
+
+    def __init__(self, condition, text=None, textLang=None, appCondition=None):
+        Exception.__init__(self)
+        self.condition = condition
+        self.text = text
+        self.textLang = textLang
+        self.appCondition = appCondition
+
+
+    def __str__(self):
+        message = "%s with condition %r" % (self.__class__.__name__,
+                                            self.condition)
+
+        if self.text:
+            message += ': ' + self.text
+
+        return message
+
+
+    def getElement(self):
+        """
+        Get XML representation from self.
+
+        The method creates an L{domish} representation of the
+        error data contained in this exception.
+
+        @rtype: L{domish.Element}
+        """
+        error = domish.Element((None, 'error'))
+        error.addElement((self.namespace, self.condition))
+        if self.text:
+            text = error.addElement((self.namespace, 'text'),
+                                    content=self.text)
+            if self.textLang:
+                text[(NS_XML, 'lang')] = self.textLang
+        if self.appCondition:
+            error.addChild(self.appCondition)
+        return error
+
+
+
+class StreamError(BaseError):
+    """
+    Stream Error exception.
+
+    Refer to RFC 3920, section 4.7.3, for the allowed values for C{condition}.
+    """
+
+    namespace = NS_XMPP_STREAMS
+
+    def getElement(self):
+        """
+        Get XML representation from self.
+
+        Overrides the base L{BaseError.getElement} to make sure the returned
+        element is in the XML Stream namespace.
+
+        @rtype: L{domish.Element}
+        """
+        from twisted.words.protocols.jabber.xmlstream import NS_STREAMS
+
+        error = BaseError.getElement(self)
+        error.uri = NS_STREAMS
+        return error
+
+
+
+class StanzaError(BaseError):
+    """
+    Stanza Error exception.
+
+    Refer to RFC 3920, section 9.3, for the allowed values for C{condition} and
+    C{type}.
+
+    @ivar type: The stanza error type. Gives a suggestion to the recipient
+                of the error on how to proceed.
+    @type type: C{str}
+    @ivar code: A numeric identifier for the error condition for backwards
+                compatibility with pre-XMPP Jabber implementations.
+    """
+
+    namespace = NS_XMPP_STANZAS
+
+    def __init__(self, condition, type=None, text=None, textLang=None,
+                       appCondition=None):
+        BaseError.__init__(self, condition, text, textLang, appCondition)
+
+        if type is None:
+            try:
+                type = STANZA_CONDITIONS[condition]['type']
+            except KeyError:
+                pass
+        self.type = type
+
+        try:
+            self.code = STANZA_CONDITIONS[condition]['code']
+        except KeyError:
+            self.code = None
+
+        self.children = []
+        self.iq = None
+
+
+    def getElement(self):
+        """
+        Get XML representation from self.
+
+        Overrides the base L{BaseError.getElement} to make sure the returned
+        element has a C{type} attribute and optionally a legacy C{code}
+        attribute.
+
+        @rtype: L{domish.Element}
+        """
+        error = BaseError.getElement(self)
+        error['type'] = self.type
+        if self.code:
+            error['code'] = self.code
+        return error
+
+
+    def toResponse(self, stanza):
+        """
+        Construct error response stanza.
+
+        The C{stanza} is transformed into an error response stanza by
+        swapping the C{to} and C{from} addresses and inserting an error
+        element.
+
+        @note: This creates a shallow copy of the list of child elements of the
+               stanza. The child elements themselves are not copied themselves,
+               and references to their parent element will still point to the
+               original stanza element.
+
+               The serialization of an element does not use the reference to
+               its parent, so the typical use case of immediately sending out
+               the constructed error response is not affected.
+
+        @param stanza: the stanza to respond to
+        @type stanza: L{domish.Element}
+        """
+        from twisted.words.protocols.jabber.xmlstream import toResponse
+        response = toResponse(stanza, stanzaType='error')
+        response.children = copy.copy(stanza.children)
+        response.addChild(self.getElement())
+        return response
+
+
+
+def _parseError(error, errorNamespace):
+    """
+    Parses an error element.
+
+    @param error: The error element to be parsed
+    @type error: L{domish.Element}
+    @param errorNamespace: The namespace of the elements that hold the error
+                           condition and text.
+    @type errorNamespace: C{str}
+    @return: Dictionary with extracted error information. If present, keys
+             C{condition}, C{text}, C{textLang} have a string value,
+             and C{appCondition} has an L{domish.Element} value.
+    @rtype: C{dict}
+    """
+    condition = None
+    text = None
+    textLang = None
+    appCondition = None
+
+    for element in error.elements():
+        if element.uri == errorNamespace:
+            if element.name == 'text':
+                text = unicode(element)
+                textLang = element.getAttribute((NS_XML, 'lang'))
+            else:
+                condition = element.name
+        else:
+            appCondition = element
+
+    return {
+        'condition': condition,
+        'text': text,
+        'textLang': textLang,
+        'appCondition': appCondition,
+    }
+
+
+
+def exceptionFromStreamError(element):
+    """
+    Build an exception object from a stream error.
+
+    @param element: the stream error
+    @type element: L{domish.Element}
+    @return: the generated exception object
+    @rtype: L{StreamError}
+    """
+    error = _parseError(element, NS_XMPP_STREAMS)
+
+    exception = StreamError(error['condition'],
+                            error['text'],
+                            error['textLang'],
+                            error['appCondition'])
+
+    return exception
+
+
+
+def exceptionFromStanza(stanza):
+    """
+    Build an exception object from an error stanza.
+
+    @param stanza: the error stanza
+    @type stanza: L{domish.Element}
+    @return: the generated exception object
+    @rtype: L{StanzaError}
+    """
+    children = []
+    condition = text = textLang = appCondition = type = code = None
+
+    for element in stanza.elements():
+        if element.name == 'error' and element.uri == stanza.uri:
+            code = element.getAttribute('code')
+            type = element.getAttribute('type')
+            error = _parseError(element, NS_XMPP_STANZAS)
+            condition = error['condition']
+            text = error['text']
+            textLang = error['textLang']
+            appCondition = error['appCondition']
+
+            if not condition and code:
+               condition, type = CODES_TO_CONDITIONS[code]
+               text = unicode(stanza.error)
+        else:
+            children.append(element)
+
+    if condition is None:
+        # TODO: raise exception instead?
+        return StanzaError(None)
+
+    exception = StanzaError(condition, type, text, textLang, appCondition)
+
+    exception.children = children
+    exception.stanza = stanza
+
+    return exception
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/ijabber.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/ijabber.py
new file mode 100644
index 00000000000..e6745b10a47
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/ijabber.py
@@ -0,0 +1,201 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Public Jabber Interfaces.
+"""
+
+from zope.interface import Attribute, Interface
+
+class IInitializer(Interface):
+    """
+    Interface for XML stream initializers.
+
+    Initializers perform a step in getting the XML stream ready to be
+    used for the exchange of XML stanzas.
+    """
+
+
+
+class IInitiatingInitializer(IInitializer):
+    """
+    Interface for XML stream initializers for the initiating entity.
+    """
+
+    xmlstream = Attribute("""The associated XML stream""")
+
+    def initialize():
+        """
+        Initiate the initialization step.
+
+        May return a deferred when the initialization is done asynchronously.
+        """
+
+
+
+class IIQResponseTracker(Interface):
+    """
+    IQ response tracker interface.
+
+    The XMPP stanza C{iq} has a request-response nature that fits
+    naturally with deferreds. You send out a request and when the response
+    comes back a deferred is fired.
+
+    The L{twisted.words.protocols.jabber.client.IQ} class implements a C{send}
+    method that returns a deferred. This deferred is put in a dictionary that
+    is kept in an L{XmlStream} object, keyed by the request stanzas C{id}
+    attribute.
+
+    An object providing this interface (usually an instance of L{XmlStream}),
+    keeps the said dictionary and sets observers on the iq stanzas of type
+    C{result} and C{error} and lets the callback fire the associated deferred.
+    """
+    iqDeferreds = Attribute("Dictionary of deferreds waiting for an iq "
+                             "response")
+
+
+
+class IXMPPHandler(Interface):
+    """
+    Interface for XMPP protocol handlers.
+
+    Objects that provide this interface can be added to a stream manager to
+    handle of (part of) an XMPP extension protocol.
+    """
+
+    parent = Attribute("""XML stream manager for this handler""")
+    xmlstream = Attribute("""The managed XML stream""")
+
+    def setHandlerParent(parent):
+        """
+        Set the parent of the handler.
+
+        @type parent: L{IXMPPHandlerCollection}
+        """
+
+
+    def disownHandlerParent(parent):
+        """
+        Remove the parent of the handler.
+
+        @type parent: L{IXMPPHandlerCollection}
+        """
+
+
+    def makeConnection(xs):
+        """
+        A connection over the underlying transport of the XML stream has been
+        established.
+
+        At this point, no traffic has been exchanged over the XML stream
+        given in C{xs}.
+
+        This should setup L{xmlstream} and call L{connectionMade}.
+
+        @type xs:
+               L{twisted.words.protocols.jabber.xmlstream.XmlStream}
+        """
+
+
+    def connectionMade():
+        """
+        Called after a connection has been established.
+
+        This method can be used to change properties of the XML Stream, its
+        authenticator or the stream manager prior to stream initialization
+        (including authentication).
+        """
+
+
+    def connectionInitialized():
+        """
+        The XML stream has been initialized.
+
+        At this point, authentication was successful, and XML stanzas can be
+        exchanged over the XML stream L{xmlstream}. This method can be
+        used to setup observers for incoming stanzas.
+        """
+
+
+    def connectionLost(reason):
+        """
+        The XML stream has been closed.
+
+        Subsequent use of C{parent.send} will result in data being queued
+        until a new connection has been established.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+
+
+class IXMPPHandlerCollection(Interface):
+    """
+    Collection of handlers.
+
+    Contain several handlers and manage their connection.
+    """
+
+    def __iter__():
+        """
+        Get an iterator over all child handlers.
+        """
+
+
+    def addHandler(handler):
+        """
+        Add a child handler.
+
+        @type handler: L{IXMPPHandler}
+        """
+
+
+    def removeHandler(handler):
+        """
+        Remove a child handler.
+
+        @type handler: L{IXMPPHandler}
+        """
+
+
+
+class IService(Interface):
+    """
+    External server-side component service interface.
+
+    Services that provide this interface can be added to L{ServiceManager} to
+    implement (part of) the functionality of the server-side component.
+    """
+
+    def componentConnected(xs):
+        """
+        Parent component has established a connection.
+
+        At this point, authentication was successful, and XML stanzas
+        can be exchanged over the XML stream C{xs}. This method can be used
+        to setup observers for incoming stanzas.
+
+        @param xs: XML Stream that represents the established connection.
+        @type xs: L{xmlstream.XmlStream}
+        """
+
+
+    def componentDisconnected():
+        """
+        Parent component has lost the connection to the Jabber server.
+
+        Subsequent use of C{self.parent.send} will result in data being
+        queued until a new connection has been established.
+        """
+
+
+    def transportConnected(xs):
+        """
+        Parent component has established a connection over the underlying
+        transport.
+
+        At this point, no traffic has been exchanged over the XML stream. This
+        method can be used to change properties of the XML Stream (in C{xs}),
+        the service manager or it's authenticator prior to stream
+        initialization (including authentication).
+        """
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/jid.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/jid.py
new file mode 100644
index 00000000000..a5b012c682b
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/jid.py
@@ -0,0 +1,253 @@
+# -*- test-case-name: twisted.words.test.test_jabberjid -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Jabber Identifier support.
+
+This module provides an object to represent Jabber Identifiers (JIDs) and
+parse string representations into them with proper checking for illegal
+characters, case folding and canonicalisation through L{stringprep<twisted.words.protocols.jabber.xmpp_stringprep>}.
+"""
+
+from twisted.python.compat import _PY3, unicode
+from twisted.words.protocols.jabber.xmpp_stringprep import nodeprep, resourceprep, nameprep
+
+class InvalidFormat(Exception):
+    """
+    The given string could not be parsed into a valid Jabber Identifier (JID).
+    """
+
+def parse(jidstring):
+    """
+    Parse given JID string into its respective parts and apply stringprep.
+
+    @param jidstring: string representation of a JID.
+    @type jidstring: L{unicode}
+    @return: tuple of (user, host, resource), each of type L{unicode} as
+             the parsed and stringprep'd parts of the given JID. If the
+             given string did not have a user or resource part, the respective
+             field in the tuple will hold L{None}.
+    @rtype: L{tuple}
+    """
+    user = None
+    host = None
+    resource = None
+
+    # Search for delimiters
+    user_sep = jidstring.find("@")
+    res_sep  = jidstring.find("/")
+
+    if user_sep == -1:
+        if res_sep == -1:
+            # host
+            host = jidstring
+        else:
+            # host/resource
+            host = jidstring[0:res_sep]
+            resource = jidstring[res_sep + 1:] or None
+    else:
+        if res_sep == -1:
+            # user@host
+            user = jidstring[0:user_sep] or None
+            host = jidstring[user_sep + 1:]
+        else:
+            if user_sep < res_sep:
+                # user@host/resource
+                user = jidstring[0:user_sep] or None
+                host = jidstring[user_sep + 1:user_sep + (res_sep - user_sep)]
+                resource = jidstring[res_sep + 1:] or None
+            else:
+                # host/resource (with an @ in resource)
+                host = jidstring[0:res_sep]
+                resource = jidstring[res_sep + 1:] or None
+
+    return prep(user, host, resource)
+
+def prep(user, host, resource):
+    """
+    Perform stringprep on all JID fragments.
+
+    @param user: The user part of the JID.
+    @type user: L{unicode}
+    @param host: The host part of the JID.
+    @type host: L{unicode}
+    @param resource: The resource part of the JID.
+    @type resource: L{unicode}
+    @return: The given parts with stringprep applied.
+    @rtype: L{tuple}
+    """
+
+    if user:
+        try:
+            user = nodeprep.prepare(unicode(user))
+        except UnicodeError:
+            raise InvalidFormat("Invalid character in username")
+    else:
+        user = None
+
+    if not host:
+        raise InvalidFormat("Server address required.")
+    else:
+        try:
+            host = nameprep.prepare(unicode(host))
+        except UnicodeError:
+            raise InvalidFormat("Invalid character in hostname")
+
+    if resource:
+        try:
+            resource = resourceprep.prepare(unicode(resource))
+        except UnicodeError:
+            raise InvalidFormat("Invalid character in resource")
+    else:
+        resource = None
+
+    return (user, host, resource)
+
+__internJIDs = {}
+
+def internJID(jidstring):
+    """
+    Return interned JID.
+
+    @rtype: L{JID}
+    """
+
+    if jidstring in __internJIDs:
+        return __internJIDs[jidstring]
+    else:
+        j = JID(jidstring)
+        __internJIDs[jidstring] = j
+        return j
+
+class JID(object):
+    """
+    Represents a stringprep'd Jabber ID.
+
+    JID objects are hashable so they can be used in sets and as keys in
+    dictionaries.
+    """
+
+    def __init__(self, str=None, tuple=None):
+        if not (str or tuple):
+            raise RuntimeError("You must provide a value for either 'str' or "
+                               "'tuple' arguments.")
+
+        if str:
+            user, host, res = parse(str)
+        else:
+            user, host, res = prep(*tuple)
+
+        self.user = user
+        self.host = host
+        self.resource = res
+
+    def userhost(self):
+        """
+        Extract the bare JID as a unicode string.
+
+        A bare JID does not have a resource part, so this returns either
+        C{user@host} or just C{host}.
+
+        @rtype: L{unicode}
+        """
+        if self.user:
+            return u"%s@%s" % (self.user, self.host)
+        else:
+            return self.host
+
+    def userhostJID(self):
+        """
+        Extract the bare JID.
+
+        A bare JID does not have a resource part, so this returns a
+        L{JID} object representing either C{user@host} or just C{host}.
+
+        If the object this method is called upon doesn't have a resource
+        set, it will return itself. Otherwise, the bare JID object will
+        be created, interned using L{internJID}.
+
+        @rtype: L{JID}
+        """
+        if self.resource:
+            return internJID(self.userhost())
+        else:
+            return self
+
+    def full(self):
+        """
+        Return the string representation of this JID.
+
+        @rtype: L{unicode}
+        """
+        if self.user:
+            if self.resource:
+                return u"%s@%s/%s" % (self.user, self.host, self.resource)
+            else:
+                return u"%s@%s" % (self.user, self.host)
+        else:
+            if self.resource:
+                return u"%s/%s" % (self.host, self.resource)
+            else:
+                return self.host
+
+    def __eq__(self, other):
+        """
+        Equality comparison.
+
+        L{JID}s compare equal if their user, host and resource parts all
+        compare equal.  When comparing against instances of other types, it
+        uses the default comparison.
+        """
+        if isinstance(other, JID):
+            return (self.user == other.user and
+                    self.host == other.host and
+                    self.resource == other.resource)
+        else:
+            return NotImplemented
+
+    def __ne__(self, other):
+        """
+        Inequality comparison.
+
+        This negates L{__eq__} for comparison with JIDs and uses the default
+        comparison for other types.
+        """
+        result = self.__eq__(other)
+        if result is NotImplemented:
+            return result
+        else:
+            return not result
+
+    def __hash__(self):
+        """
+        Calculate hash.
+
+        L{JID}s with identical constituent user, host and resource parts have
+        equal hash values.  In combination with the comparison defined on JIDs,
+        this allows for using L{JID}s in sets and as dictionary keys.
+        """
+        return hash((self.user, self.host, self.resource))
+
+    def __unicode__(self):
+        """
+        Get unicode representation.
+
+        Return the string representation of this JID as a unicode string.
+        @see: L{full}
+        """
+
+        return self.full()
+
+    if _PY3:
+        __str__ = __unicode__
+
+    def __repr__(self):
+        """
+        Get object representation.
+
+        Returns a string that would create a new JID object that compares equal
+        to this one.
+        """
+        return 'JID(%r)' % self.full()
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/jstrports.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/jstrports.py
new file mode 100644
index 00000000000..61f4cc89532
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/jstrports.py
@@ -0,0 +1,33 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+""" A temporary placeholder for client-capable strports, until we
+sufficient use cases get identified """
+
+from __future__ import absolute_import, division
+
+from twisted.internet.endpoints import _parse
+
+def _parseTCPSSL(factory, domain, port):
+    """ For the moment, parse TCP or SSL connections the same """
+    return (domain, int(port), factory), {}
+
+def _parseUNIX(factory, address):
+    return (address, factory), {}
+
+
+_funcs = { "tcp"  : _parseTCPSSL,
+           "unix" : _parseUNIX,
+           "ssl"  : _parseTCPSSL }
+
+
+def parse(description, factory):
+    args, kw = _parse(description)
+    return (args[0].upper(),) + _funcs[args[0]](factory, *args[1:], **kw)
+
+def client(description, factory):
+    from twisted.application import internet
+    name, args, kw = parse(description, factory)
+    return getattr(internet, name + 'Client')(*args, **kw)
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/sasl.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/sasl.py
new file mode 100644
index 00000000000..3dc92d87d37
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/sasl.py
@@ -0,0 +1,233 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XMPP-specific SASL profile.
+"""
+
+from __future__ import absolute_import, division
+
+from base64 import b64decode, b64encode
+import re
+from twisted.internet import defer
+from twisted.python.compat import unicode
+from twisted.words.protocols.jabber import sasl_mechanisms, xmlstream
+from twisted.words.xish import domish
+
+NS_XMPP_SASL = 'urn:ietf:params:xml:ns:xmpp-sasl'
+
+def get_mechanisms(xs):
+    """
+    Parse the SASL feature to extract the available mechanism names.
+    """
+    mechanisms = []
+    for element in xs.features[(NS_XMPP_SASL, 'mechanisms')].elements():
+        if element.name == 'mechanism':
+            mechanisms.append(unicode(element))
+
+    return mechanisms
+
+
+class SASLError(Exception):
+    """
+    SASL base exception.
+    """
+
+
+class SASLNoAcceptableMechanism(SASLError):
+    """
+    The server did not present an acceptable SASL mechanism.
+    """
+
+
+class SASLAuthError(SASLError):
+    """
+    SASL Authentication failed.
+    """
+    def __init__(self, condition=None):
+        self.condition = condition
+
+
+    def __str__(self):
+        return "SASLAuthError with condition %r" % self.condition
+
+
+class SASLIncorrectEncodingError(SASLError):
+    """
+    SASL base64 encoding was incorrect.
+
+    RFC 3920 specifies that any characters not in the base64 alphabet
+    and padding characters present elsewhere than at the end of the string
+    MUST be rejected. See also L{fromBase64}.
+
+    This exception is raised whenever the encoded string does not adhere
+    to these additional restrictions or when the decoding itself fails.
+
+    The recommended behaviour for so-called receiving entities (like servers in
+    client-to-server connections, see RFC 3920 for terminology) is to fail the
+    SASL negotiation with a C{'incorrect-encoding'} condition. For initiating
+    entities, one should assume the receiving entity to be either buggy or
+    malevolent. The stream should be terminated and reconnecting is not
+    advised.
+    """
+
+base64Pattern = re.compile("^[0-9A-Za-z+/]*[0-9A-Za-z+/=]{,2}$")
+
+def fromBase64(s):
+    """
+    Decode base64 encoded string.
+
+    This helper performs regular decoding of a base64 encoded string, but also
+    rejects any characters that are not in the base64 alphabet and padding
+    occurring elsewhere from the last or last two characters, as specified in
+    section 14.9 of RFC 3920. This safeguards against various attack vectors
+    among which the creation of a covert channel that "leaks" information.
+    """
+
+    if base64Pattern.match(s) is None:
+        raise SASLIncorrectEncodingError()
+
+    try:
+        return b64decode(s)
+    except Exception as e:
+        raise SASLIncorrectEncodingError(str(e))
+
+
+
+class SASLInitiatingInitializer(xmlstream.BaseFeatureInitiatingInitializer):
+    """
+    Stream initializer that performs SASL authentication.
+
+    The supported mechanisms by this initializer are C{DIGEST-MD5}, C{PLAIN}
+    and C{ANONYMOUS}. The C{ANONYMOUS} SASL mechanism is used when the JID, set
+    on the authenticator, does not have a localpart (username), requesting an
+    anonymous session where the username is generated by the server.
+    Otherwise, C{DIGEST-MD5} and C{PLAIN} are attempted, in that order.
+    """
+
+    feature = (NS_XMPP_SASL, 'mechanisms')
+    _deferred = None
+
+    def setMechanism(self):
+        """
+        Select and setup authentication mechanism.
+
+        Uses the authenticator's C{jid} and C{password} attribute for the
+        authentication credentials. If no supported SASL mechanisms are
+        advertized by the receiving party, a failing deferred is returned with
+        a L{SASLNoAcceptableMechanism} exception.
+        """
+
+        jid = self.xmlstream.authenticator.jid
+        password = self.xmlstream.authenticator.password
+
+        mechanisms = get_mechanisms(self.xmlstream)
+        if jid.user is not None:
+            if 'DIGEST-MD5' in mechanisms:
+                self.mechanism = sasl_mechanisms.DigestMD5('xmpp', jid.host, None,
+                                                           jid.user, password)
+            elif 'PLAIN' in mechanisms:
+                self.mechanism = sasl_mechanisms.Plain(None, jid.user, password)
+            else:
+                raise SASLNoAcceptableMechanism()
+        else:
+            if 'ANONYMOUS' in mechanisms:
+                self.mechanism = sasl_mechanisms.Anonymous()
+            else:
+                raise SASLNoAcceptableMechanism()
+
+
+    def start(self):
+        """
+        Start SASL authentication exchange.
+        """
+
+        self.setMechanism()
+        self._deferred = defer.Deferred()
+        self.xmlstream.addObserver('/challenge', self.onChallenge)
+        self.xmlstream.addOnetimeObserver('/success', self.onSuccess)
+        self.xmlstream.addOnetimeObserver('/failure', self.onFailure)
+        self.sendAuth(self.mechanism.getInitialResponse())
+        return self._deferred
+
+
+    def sendAuth(self, data=None):
+        """
+        Initiate authentication protocol exchange.
+
+        If an initial client response is given in C{data}, it will be
+        sent along.
+
+        @param data: initial client response.
+        @type data: C{str} or L{None}.
+        """
+
+        auth = domish.Element((NS_XMPP_SASL, 'auth'))
+        auth['mechanism'] = self.mechanism.name
+        if data is not None:
+            auth.addContent(b64encode(data).decode('ascii') or u'=')
+        self.xmlstream.send(auth)
+
+
+    def sendResponse(self, data=b''):
+        """
+        Send response to a challenge.
+
+        @param data: client response.
+        @type data: L{bytes}.
+        """
+
+        response = domish.Element((NS_XMPP_SASL, 'response'))
+        if data:
+            response.addContent(b64encode(data).decode('ascii'))
+        self.xmlstream.send(response)
+
+
+    def onChallenge(self, element):
+        """
+        Parse challenge and send response from the mechanism.
+
+        @param element: the challenge protocol element.
+        @type element: L{domish.Element}.
+        """
+
+        try:
+            challenge = fromBase64(unicode(element))
+        except SASLIncorrectEncodingError:
+            self._deferred.errback()
+        else:
+            self.sendResponse(self.mechanism.getResponse(challenge))
+
+
+    def onSuccess(self, success):
+        """
+        Clean up observers, reset the XML stream and send a new header.
+
+        @param success: the success protocol element. For now unused, but
+                        could hold additional data.
+        @type success: L{domish.Element}
+        """
+
+        self.xmlstream.removeObserver('/challenge', self.onChallenge)
+        self.xmlstream.removeObserver('/failure', self.onFailure)
+        self.xmlstream.reset()
+        self.xmlstream.sendHeader()
+        self._deferred.callback(xmlstream.Reset)
+
+
+    def onFailure(self, failure):
+        """
+        Clean up observers, parse the failure and errback the deferred.
+
+        @param failure: the failure protocol element. Holds details on
+                        the error condition.
+        @type failure: L{domish.Element}
+        """
+
+        self.xmlstream.removeObserver('/challenge', self.onChallenge)
+        self.xmlstream.removeObserver('/success', self.onSuccess)
+        try:
+            condition = failure.firstChildElement().name
+        except AttributeError:
+            condition = None
+        self._deferred.errback(SASLAuthError(condition))
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/sasl_mechanisms.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/sasl_mechanisms.py
new file mode 100644
index 00000000000..cbc7a90a5af
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/sasl_mechanisms.py
@@ -0,0 +1,293 @@
+# -*- test-case-name: twisted.words.test.test_jabbersaslmechanisms -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Protocol agnostic implementations of SASL authentication mechanisms.
+"""
+
+from __future__ import absolute_import, division
+
+import binascii, random, time, os
+from hashlib import md5
+
+from zope.interface import Interface, Attribute, implementer
+
+from twisted.python.compat import iteritems, networkString
+
+
+class ISASLMechanism(Interface):
+    name = Attribute("""Common name for the SASL Mechanism.""")
+
+    def getInitialResponse():
+        """
+        Get the initial client response, if defined for this mechanism.
+
+        @return: initial client response string.
+        @rtype: C{str}.
+        """
+
+
+    def getResponse(challenge):
+        """
+        Get the response to a server challenge.
+
+        @param challenge: server challenge.
+        @type challenge: C{str}.
+        @return: client response.
+        @rtype: C{str}.
+        """
+
+
+
+@implementer(ISASLMechanism)
+class Anonymous(object):
+    """
+    Implements the ANONYMOUS SASL authentication mechanism.
+
+    This mechanism is defined in RFC 2245.
+    """
+    name = 'ANONYMOUS'
+
+    def getInitialResponse(self):
+        return None
+
+
+
+@implementer(ISASLMechanism)
+class Plain(object):
+    """
+    Implements the PLAIN SASL authentication mechanism.
+
+    The PLAIN SASL authentication mechanism is defined in RFC 2595.
+    """
+    name = 'PLAIN'
+
+    def __init__(self, authzid, authcid, password):
+        """
+        @param authzid: The authorization identity.
+        @type authzid: L{unicode}
+
+        @param authcid: The authentication identity.
+        @type authcid: L{unicode}
+
+        @param password: The plain-text password.
+        @type password: L{unicode}
+        """
+
+        self.authzid = authzid or u''
+        self.authcid = authcid or u''
+        self.password = password or u''
+
+
+    def getInitialResponse(self):
+        return (self.authzid.encode('utf-8') + b"\x00" +
+                self.authcid.encode('utf-8') + b"\x00" +
+                self.password.encode('utf-8'))
+
+
+
+@implementer(ISASLMechanism)
+class DigestMD5(object):
+    """
+    Implements the DIGEST-MD5 SASL authentication mechanism.
+
+    The DIGEST-MD5 SASL authentication mechanism is defined in RFC 2831.
+    """
+    name = 'DIGEST-MD5'
+
+    def __init__(self, serv_type, host, serv_name, username, password):
+        """
+        @param serv_type: An indication of what kind of server authentication
+            is being attempted against.  For example, C{u"xmpp"}.
+        @type serv_type: C{unicode}
+
+        @param host: The authentication hostname.  Also known as the realm.
+            This is used as a scope to help select the right credentials.
+        @type host: C{unicode}
+
+        @param serv_name: An additional identifier for the server.
+        @type serv_name: C{unicode}
+
+        @param username: The authentication username to use to respond to a
+            challenge.
+        @type username: C{unicode}
+
+        @param username: The authentication password to use to respond to a
+            challenge.
+        @type password: C{unicode}
+        """
+        self.username = username
+        self.password = password
+        self.defaultRealm = host
+
+        self.digest_uri = u'%s/%s' % (serv_type, host)
+        if serv_name is not None:
+            self.digest_uri += u'/%s' % (serv_name,)
+
+
+    def getInitialResponse(self):
+        return None
+
+
+    def getResponse(self, challenge):
+        directives = self._parse(challenge)
+
+        # Compat for implementations that do not send this along with
+        # a successful authentication.
+        if b'rspauth' in directives:
+            return b''
+
+        charset = directives[b'charset'].decode('ascii')
+
+        try:
+            realm = directives[b'realm']
+        except KeyError:
+            realm = self.defaultRealm.encode(charset)
+
+        return self._genResponse(charset,
+                                 realm,
+                                 directives[b'nonce'])
+
+
+    def _parse(self, challenge):
+        """
+        Parses the server challenge.
+
+        Splits the challenge into a dictionary of directives with values.
+
+        @return: challenge directives and their values.
+        @rtype: C{dict} of C{str} to C{str}.
+        """
+        s = challenge
+        paramDict = {}
+        cur = 0
+        remainingParams = True
+        while remainingParams:
+            # Parse a param. We can't just split on commas, because there can
+            # be some commas inside (quoted) param values, e.g.:
+            # qop="auth,auth-int"
+
+            middle = s.index(b"=", cur)
+            name = s[cur:middle].lstrip()
+            middle += 1
+            if s[middle:middle+1] == b'"':
+                middle += 1
+                end = s.index(b'"', middle)
+                value = s[middle:end]
+                cur = s.find(b',', end) + 1
+                if cur == 0:
+                    remainingParams = False
+            else:
+                end = s.find(b',', middle)
+                if end == -1:
+                    value = s[middle:].rstrip()
+                    remainingParams = False
+                else:
+                    value = s[middle:end].rstrip()
+                cur = end + 1
+            paramDict[name] = value
+
+        for param in (b'qop', b'cipher'):
+            if param in paramDict:
+                paramDict[param] = paramDict[param].split(b',')
+
+        return paramDict
+
+    def _unparse(self, directives):
+        """
+        Create message string from directives.
+
+        @param directives: dictionary of directives (names to their values).
+                           For certain directives, extra quotes are added, as
+                           needed.
+        @type directives: C{dict} of C{str} to C{str}
+        @return: message string.
+        @rtype: C{str}.
+        """
+
+        directive_list = []
+        for name, value in iteritems(directives):
+            if name in (b'username', b'realm', b'cnonce',
+                        b'nonce', b'digest-uri', b'authzid', b'cipher'):
+                directive = name + b'=' + value
+            else:
+                directive = name + b'=' + value
+
+            directive_list.append(directive)
+
+        return b','.join(directive_list)
+
+
+    def _calculateResponse(self, cnonce, nc, nonce,
+                            username, password, realm, uri):
+        """
+        Calculates response with given encoded parameters.
+
+        @return: The I{response} field of a response to a Digest-MD5 challenge
+            of the given parameters.
+        @rtype: L{bytes}
+        """
+        def H(s):
+            return md5(s).digest()
+
+        def HEX(n):
+            return binascii.b2a_hex(n)
+
+        def KD(k, s):
+            return H(k + b':' + s)
+
+        a1 = (H(username + b":" + realm + b":" + password) + b":" +
+              nonce + b":" +
+              cnonce)
+        a2 = b"AUTHENTICATE:" + uri
+
+        response = HEX(KD(HEX(H(a1)),
+                       nonce + b":" + nc + b":" + cnonce + b":" +
+                       b"auth" + b":" + HEX(H(a2))))
+        return response
+
+
+    def _genResponse(self, charset, realm, nonce):
+        """
+        Generate response-value.
+
+        Creates a response to a challenge according to section 2.1.2.1 of
+        RFC 2831 using the C{charset}, C{realm} and C{nonce} directives
+        from the challenge.
+        """
+        try:
+            username = self.username.encode(charset)
+            password = self.password.encode(charset)
+            digest_uri = self.digest_uri.encode(charset)
+        except UnicodeError:
+            # TODO - add error checking
+            raise
+
+        nc = networkString('%08x' % (1,)) # TODO: support subsequent auth.
+        cnonce = self._gen_nonce()
+        qop = b'auth'
+
+        # TODO - add support for authzid
+        response = self._calculateResponse(cnonce, nc, nonce,
+                                           username, password, realm,
+                                           digest_uri)
+
+        directives = {b'username': username,
+                      b'realm' : realm,
+                      b'nonce' : nonce,
+                      b'cnonce' : cnonce,
+                      b'nc' : nc,
+                      b'qop' : qop,
+                      b'digest-uri': digest_uri,
+                      b'response': response,
+                      b'charset': charset.encode('ascii')}
+
+        return self._unparse(directives)
+
+
+    def _gen_nonce(self):
+        nonceString = "%f:%f:%d" % (random.random(), time.time(), os.getpid())
+        nonceBytes = networkString(nonceString)
+        return md5(nonceBytes).hexdigest().encode('ascii')
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/xmlstream.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/xmlstream.py
new file mode 100644
index 00000000000..20948c6d3be
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/xmlstream.py
@@ -0,0 +1,1170 @@
+# -*- test-case-name: twisted.words.test.test_jabberxmlstream -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XMPP XML Streams
+
+Building blocks for setting up XML Streams, including helping classes for
+doing authentication on either client or server side, and working with XML
+Stanzas.
+
+@var STREAM_AUTHD_EVENT: Token dispatched by L{Authenticator} when the
+    stream has been completely initialized
+@type STREAM_AUTHD_EVENT: L{str}.
+
+@var INIT_FAILED_EVENT: Token dispatched by L{Authenticator} when the
+    stream has failed to be initialized
+@type INIT_FAILED_EVENT: L{str}.
+
+@var Reset: Token to signal that the XML stream has been reset.
+@type Reset: Basic object.
+"""
+
+from __future__ import absolute_import, division
+
+from binascii import hexlify
+from hashlib import sha1
+from zope.interface import directlyProvides, implementer
+
+from twisted.internet import defer, protocol
+from twisted.internet.error import ConnectionLost
+from twisted.python import failure, log, randbytes
+from twisted.python.compat import intern, iteritems, itervalues, unicode
+from twisted.words.protocols.jabber import error, ijabber, jid
+from twisted.words.xish import domish, xmlstream
+from twisted.words.xish.xmlstream import STREAM_CONNECTED_EVENT
+from twisted.words.xish.xmlstream import STREAM_START_EVENT
+from twisted.words.xish.xmlstream import STREAM_END_EVENT
+from twisted.words.xish.xmlstream import STREAM_ERROR_EVENT
+
+try:
+    from twisted.internet import ssl
+except ImportError:
+    ssl = None
+if ssl and not ssl.supported:
+    ssl = None
+
+STREAM_AUTHD_EVENT = intern("//event/stream/authd")
+INIT_FAILED_EVENT = intern("//event/xmpp/initfailed")
+
+NS_STREAMS = 'http://etherx.jabber.org/streams'
+NS_XMPP_TLS = 'urn:ietf:params:xml:ns:xmpp-tls'
+
+Reset = object()
+
+def hashPassword(sid, password):
+    """
+    Create a SHA1-digest string of a session identifier and password.
+
+    @param sid: The stream session identifier.
+    @type sid: C{unicode}.
+    @param password: The password to be hashed.
+    @type password: C{unicode}.
+    """
+    if not isinstance(sid, unicode):
+        raise TypeError("The session identifier must be a unicode object")
+    if not isinstance(password, unicode):
+        raise TypeError("The password must be a unicode object")
+    input = u"%s%s" % (sid, password)
+    return sha1(input.encode('utf-8')).hexdigest()
+
+
+
+class Authenticator:
+    """
+    Base class for business logic of initializing an XmlStream
+
+    Subclass this object to enable an XmlStream to initialize and authenticate
+    to different types of stream hosts (such as clients, components, etc.).
+
+    Rules:
+      1. The Authenticator MUST dispatch a L{STREAM_AUTHD_EVENT} when the
+         stream has been completely initialized.
+      2. The Authenticator SHOULD reset all state information when
+         L{associateWithStream} is called.
+      3. The Authenticator SHOULD override L{streamStarted}, and start
+         initialization there.
+
+    @type xmlstream: L{XmlStream}
+    @ivar xmlstream: The XmlStream that needs authentication
+
+    @note: the term authenticator is historical. Authenticators perform
+           all steps required to prepare the stream for the exchange
+           of XML stanzas.
+    """
+
+    def __init__(self):
+        self.xmlstream = None
+
+
+    def connectionMade(self):
+        """
+        Called by the XmlStream when the underlying socket connection is
+        in place.
+
+        This allows the Authenticator to send an initial root element, if it's
+        connecting, or wait for an inbound root from the peer if it's accepting
+        the connection.
+
+        Subclasses can use self.xmlstream.send() to send any initial data to
+        the peer.
+        """
+
+
+    def streamStarted(self, rootElement):
+        """
+        Called by the XmlStream when the stream has started.
+
+        A stream is considered to have started when the start tag of the root
+        element has been received.
+
+        This examines C{rootElement} to see if there is a version attribute.
+        If absent, C{0.0} is assumed per RFC 3920. Subsequently, the
+        minimum of the version from the received stream header and the
+        value stored in L{xmlstream} is taken and put back in L{xmlstream}.
+
+        Extensions of this method can extract more information from the
+        stream header and perform checks on them, optionally sending
+        stream errors and closing the stream.
+        """
+        if rootElement.hasAttribute("version"):
+            version = rootElement["version"].split(".")
+            try:
+                version = (int(version[0]), int(version[1]))
+            except (IndexError, ValueError):
+                version = (0, 0)
+        else:
+            version = (0, 0)
+
+        self.xmlstream.version = min(self.xmlstream.version, version)
+
+
+    def associateWithStream(self, xmlstream):
+        """
+        Called by the XmlStreamFactory when a connection has been made
+        to the requested peer, and an XmlStream object has been
+        instantiated.
+
+        The default implementation just saves a handle to the new
+        XmlStream.
+
+        @type xmlstream: L{XmlStream}
+        @param xmlstream: The XmlStream that will be passing events to this
+                          Authenticator.
+
+        """
+        self.xmlstream = xmlstream
+
+
+
+class ConnectAuthenticator(Authenticator):
+    """
+    Authenticator for initiating entities.
+    """
+
+    namespace = None
+
+    def __init__(self, otherHost):
+        self.otherHost = otherHost
+
+
+    def connectionMade(self):
+        self.xmlstream.namespace = self.namespace
+        self.xmlstream.otherEntity = jid.internJID(self.otherHost)
+        self.xmlstream.sendHeader()
+
+
+    def initializeStream(self):
+        """
+        Perform stream initialization procedures.
+
+        An L{XmlStream} holds a list of initializer objects in its
+        C{initializers} attribute. This method calls these initializers in
+        order and dispatches the L{STREAM_AUTHD_EVENT} event when the list has
+        been successfully processed. Otherwise it dispatches the
+        C{INIT_FAILED_EVENT} event with the failure.
+
+        Initializers may return the special L{Reset} object to halt the
+        initialization processing. It signals that the current initializer was
+        successfully processed, but that the XML Stream has been reset. An
+        example is the TLSInitiatingInitializer.
+        """
+
+        def remove_first(result):
+            self.xmlstream.initializers.pop(0)
+
+            return result
+
+        def do_next(result):
+            """
+            Take the first initializer and process it.
+
+            On success, the initializer is removed from the list and
+            then next initializer will be tried.
+            """
+
+            if result is Reset:
+                return None
+
+            try:
+                init = self.xmlstream.initializers[0]
+            except IndexError:
+                self.xmlstream.dispatch(self.xmlstream, STREAM_AUTHD_EVENT)
+                return None
+            else:
+                d = defer.maybeDeferred(init.initialize)
+                d.addCallback(remove_first)
+                d.addCallback(do_next)
+                return d
+
+        d = defer.succeed(None)
+        d.addCallback(do_next)
+        d.addErrback(self.xmlstream.dispatch, INIT_FAILED_EVENT)
+
+
+    def streamStarted(self, rootElement):
+        """
+        Called by the XmlStream when the stream has started.
+
+        This extends L{Authenticator.streamStarted} to extract further stream
+        headers from C{rootElement}, optionally wait for stream features being
+        received and then call C{initializeStream}.
+        """
+
+        Authenticator.streamStarted(self, rootElement)
+
+        self.xmlstream.sid = rootElement.getAttribute("id")
+
+        if rootElement.hasAttribute("from"):
+            self.xmlstream.otherEntity = jid.internJID(rootElement["from"])
+
+        # Setup observer for stream features, if applicable
+        if self.xmlstream.version >= (1, 0):
+            def onFeatures(element):
+                features = {}
+                for feature in element.elements():
+                    features[(feature.uri, feature.name)] = feature
+
+                self.xmlstream.features = features
+                self.initializeStream()
+
+            self.xmlstream.addOnetimeObserver('/features[@xmlns="%s"]' %
+                                                  NS_STREAMS,
+                                              onFeatures)
+        else:
+            self.initializeStream()
+
+
+
+class ListenAuthenticator(Authenticator):
+    """
+    Authenticator for receiving entities.
+    """
+
+    namespace = None
+
+    def associateWithStream(self, xmlstream):
+        """
+        Called by the XmlStreamFactory when a connection has been made.
+
+        Extend L{Authenticator.associateWithStream} to set the L{XmlStream}
+        to be non-initiating.
+        """
+        Authenticator.associateWithStream(self, xmlstream)
+        self.xmlstream.initiating = False
+
+
+    def streamStarted(self, rootElement):
+        """
+        Called by the XmlStream when the stream has started.
+
+        This extends L{Authenticator.streamStarted} to extract further
+        information from the stream headers from C{rootElement}.
+        """
+        Authenticator.streamStarted(self, rootElement)
+
+        self.xmlstream.namespace = rootElement.defaultUri
+
+        if rootElement.hasAttribute("to"):
+            self.xmlstream.thisEntity = jid.internJID(rootElement["to"])
+
+        self.xmlstream.prefixes = {}
+        for prefix, uri in iteritems(rootElement.localPrefixes):
+            self.xmlstream.prefixes[uri] = prefix
+
+        self.xmlstream.sid = hexlify(randbytes.secureRandom(8)).decode('ascii')
+
+
+
+class FeatureNotAdvertized(Exception):
+    """
+    Exception indicating a stream feature was not advertized, while required by
+    the initiating entity.
+    """
+
+
+
+@implementer(ijabber.IInitiatingInitializer)
+class BaseFeatureInitiatingInitializer(object):
+    """
+    Base class for initializers with a stream feature.
+
+    This assumes the associated XmlStream represents the initiating entity
+    of the connection.
+
+    @cvar feature: tuple of (uri, name) of the stream feature root element.
+    @type feature: tuple of (C{str}, C{str})
+
+    @ivar required: whether the stream feature is required to be advertized
+                    by the receiving entity.
+    @type required: C{bool}
+    """
+
+    feature = None
+
+    def __init__(self, xs, required=False):
+        self.xmlstream = xs
+        self.required = required
+
+
+    def initialize(self):
+        """
+        Initiate the initialization.
+
+        Checks if the receiving entity advertizes the stream feature. If it
+        does, the initialization is started. If it is not advertized, and the
+        C{required} instance variable is C{True}, it raises
+        L{FeatureNotAdvertized}. Otherwise, the initialization silently
+        succeeds.
+        """
+
+        if self.feature in self.xmlstream.features:
+            return self.start()
+        elif self.required:
+            raise FeatureNotAdvertized
+        else:
+            return None
+
+
+    def start(self):
+        """
+        Start the actual initialization.
+
+        May return a deferred for asynchronous initialization.
+        """
+
+
+
+class TLSError(Exception):
+    """
+    TLS base exception.
+    """
+
+
+
+class TLSFailed(TLSError):
+    """
+    Exception indicating failed TLS negotiation
+    """
+
+
+
+class TLSRequired(TLSError):
+    """
+    Exception indicating required TLS negotiation.
+
+    This exception is raised when the receiving entity requires TLS
+    negotiation and the initiating does not desire to negotiate TLS.
+    """
+
+
+
+class TLSNotSupported(TLSError):
+    """
+    Exception indicating missing TLS support.
+
+    This exception is raised when the initiating entity wants and requires to
+    negotiate TLS when the OpenSSL library is not available.
+    """
+
+
+
+class TLSInitiatingInitializer(BaseFeatureInitiatingInitializer):
+    """
+    TLS stream initializer for the initiating entity.
+
+    It is strongly required to include this initializer in the list of
+    initializers for an XMPP stream. By default it will try to negotiate TLS.
+    An XMPP server may indicate that TLS is required. If TLS is not desired,
+    set the C{wanted} attribute to False instead of removing it from the list
+    of initializers, so a proper exception L{TLSRequired} can be raised.
+
+    @ivar wanted: indicates if TLS negotiation is wanted.
+    @type wanted: C{bool}
+    """
+
+    feature = (NS_XMPP_TLS, 'starttls')
+    wanted = True
+    _deferred = None
+    _configurationForTLS = None
+
+    def __init__(self, xs, required=True, configurationForTLS=None):
+        """
+        @param configurationForTLS: An object which creates appropriately
+            configured TLS connections. This is passed to C{startTLS} on the
+            transport and is preferably created using
+            L{twisted.internet.ssl.optionsForClientTLS}.  If C{None}, the
+            default is to verify the server certificate against the trust roots
+            as provided by the platform. See
+            L{twisted.internet._sslverify.platformTrust}.
+        @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or
+            C{None}
+        """
+        super(TLSInitiatingInitializer, self).__init__(
+                xs, required=required)
+        self._configurationForTLS = configurationForTLS
+
+
+    def onProceed(self, obj):
+        """
+        Proceed with TLS negotiation and reset the XML stream.
+        """
+
+        self.xmlstream.removeObserver('/failure', self.onFailure)
+        if self._configurationForTLS:
+            ctx = self._configurationForTLS
+        else:
+            ctx = ssl.optionsForClientTLS(self.xmlstream.otherEntity.host)
+        self.xmlstream.transport.startTLS(ctx)
+        self.xmlstream.reset()
+        self.xmlstream.sendHeader()
+        self._deferred.callback(Reset)
+
+
+    def onFailure(self, obj):
+        self.xmlstream.removeObserver('/proceed', self.onProceed)
+        self._deferred.errback(TLSFailed())
+
+
+    def start(self):
+        """
+        Start TLS negotiation.
+
+        This checks if the receiving entity requires TLS, the SSL library is
+        available and uses the C{required} and C{wanted} instance variables to
+        determine what to do in the various different cases.
+
+        For example, if the SSL library is not available, and wanted and
+        required by the user, it raises an exception. However if it is not
+        required by both parties, initialization silently succeeds, moving
+        on to the next step.
+        """
+        if self.wanted:
+            if ssl is None:
+                if self.required:
+                    return defer.fail(TLSNotSupported())
+                else:
+                    return defer.succeed(None)
+            else:
+                pass
+        elif self.xmlstream.features[self.feature].required:
+            return defer.fail(TLSRequired())
+        else:
+            return defer.succeed(None)
+
+        self._deferred = defer.Deferred()
+        self.xmlstream.addOnetimeObserver("/proceed", self.onProceed)
+        self.xmlstream.addOnetimeObserver("/failure", self.onFailure)
+        self.xmlstream.send(domish.Element((NS_XMPP_TLS, "starttls")))
+        return self._deferred
+
+
+
+class XmlStream(xmlstream.XmlStream):
+    """
+    XMPP XML Stream protocol handler.
+
+    @ivar version: XML stream version as a tuple (major, minor). Initially,
+                   this is set to the minimally supported version. Upon
+                   receiving the stream header of the peer, it is set to the
+                   minimum of that value and the version on the received
+                   header.
+    @type version: (C{int}, C{int})
+    @ivar namespace: default namespace URI for stream
+    @type namespace: C{unicode}
+    @ivar thisEntity: JID of this entity
+    @type thisEntity: L{JID}
+    @ivar otherEntity: JID of the peer entity
+    @type otherEntity: L{JID}
+    @ivar sid: session identifier
+    @type sid: C{unicode}
+    @ivar initiating: True if this is the initiating stream
+    @type initiating: C{bool}
+    @ivar features: map of (uri, name) to stream features element received from
+                    the receiving entity.
+    @type features: C{dict} of (C{unicode}, C{unicode}) to L{domish.Element}.
+    @ivar prefixes: map of URI to prefixes that are to appear on stream
+                    header.
+    @type prefixes: C{dict} of C{unicode} to C{unicode}
+    @ivar initializers: list of stream initializer objects
+    @type initializers: C{list} of objects that provide L{IInitializer}
+    @ivar authenticator: associated authenticator that uses C{initializers} to
+                         initialize the XML stream.
+    """
+
+    version = (1, 0)
+    namespace = 'invalid'
+    thisEntity = None
+    otherEntity = None
+    sid = None
+    initiating = True
+
+    _headerSent = False     # True if the stream header has been sent
+
+    def __init__(self, authenticator):
+        xmlstream.XmlStream.__init__(self)
+
+        self.prefixes = {NS_STREAMS: 'stream'}
+        self.authenticator = authenticator
+        self.initializers = []
+        self.features = {}
+
+        # Reset the authenticator
+        authenticator.associateWithStream(self)
+
+
+    def _callLater(self, *args, **kwargs):
+        from twisted.internet import reactor
+        return reactor.callLater(*args, **kwargs)
+
+
+    def reset(self):
+        """
+        Reset XML Stream.
+
+        Resets the XML Parser for incoming data. This is to be used after
+        successfully negotiating a new layer, e.g. TLS and SASL. Note that
+        registered event observers will continue to be in place.
+        """
+        self._headerSent = False
+        self._initializeStream()
+
+
+    def onStreamError(self, errelem):
+        """
+        Called when a stream:error element has been received.
+
+        Dispatches a L{STREAM_ERROR_EVENT} event with the error element to
+        allow for cleanup actions and drops the connection.
+
+        @param errelem: The received error element.
+        @type errelem: L{domish.Element}
+        """
+        self.dispatch(failure.Failure(error.exceptionFromStreamError(errelem)),
+                      STREAM_ERROR_EVENT)
+        self.transport.loseConnection()
+
+
+    def sendHeader(self):
+        """
+        Send stream header.
+        """
+        # set up optional extra namespaces
+        localPrefixes = {}
+        for uri, prefix in iteritems(self.prefixes):
+            if uri != NS_STREAMS:
+                localPrefixes[prefix] = uri
+
+        rootElement = domish.Element((NS_STREAMS, 'stream'), self.namespace,
+                                     localPrefixes=localPrefixes)
+
+        if self.otherEntity:
+            rootElement['to'] = self.otherEntity.userhost()
+
+        if self.thisEntity:
+            rootElement['from'] = self.thisEntity.userhost()
+
+        if not self.initiating and self.sid:
+            rootElement['id'] = self.sid
+
+        if self.version >= (1, 0):
+            rootElement['version'] = "%d.%d" % self.version
+
+        self.send(rootElement.toXml(prefixes=self.prefixes, closeElement=0))
+        self._headerSent = True
+
+
+    def sendFooter(self):
+        """
+        Send stream footer.
+        """
+        self.send('</stream:stream>')
+
+
+    def sendStreamError(self, streamError):
+        """
+        Send stream level error.
+
+        If we are the receiving entity, and haven't sent the header yet,
+        we sent one first.
+
+        After sending the stream error, the stream is closed and the transport
+        connection dropped.
+
+        @param streamError: stream error instance
+        @type streamError: L{error.StreamError}
+        """
+        if not self._headerSent and not self.initiating:
+            self.sendHeader()
+
+        if self._headerSent:
+            self.send(streamError.getElement())
+            self.sendFooter()
+
+        self.transport.loseConnection()
+
+
+    def send(self, obj):
+        """
+        Send data over the stream.
+
+        This overrides L{xmlstream.XmlStream.send} to use the default namespace
+        of the stream header when serializing L{domish.IElement}s. It is
+        assumed that if you pass an object that provides L{domish.IElement},
+        it represents a direct child of the stream's root element.
+        """
+        if domish.IElement.providedBy(obj):
+            obj = obj.toXml(prefixes=self.prefixes,
+                            defaultUri=self.namespace,
+                            prefixesInScope=list(self.prefixes.values()))
+
+        xmlstream.XmlStream.send(self, obj)
+
+
+    def connectionMade(self):
+        """
+        Called when a connection is made.
+
+        Notifies the authenticator when a connection has been made.
+        """
+        xmlstream.XmlStream.connectionMade(self)
+        self.authenticator.connectionMade()
+
+
+    def onDocumentStart(self, rootElement):
+        """
+        Called when the stream header has been received.
+
+        Extracts the header's C{id} and C{version} attributes from the root
+        element. The C{id} attribute is stored in our C{sid} attribute and the
+        C{version} attribute is parsed and the minimum of the version we sent
+        and the parsed C{version} attribute is stored as a tuple (major, minor)
+        in this class' C{version} attribute. If no C{version} attribute was
+        present, we assume version 0.0.
+
+        If appropriate (we are the initiating stream and the minimum of our and
+        the other party's version is at least 1.0), a one-time observer is
+        registered for getting the stream features. The registered function is
+        C{onFeatures}.
+
+        Ultimately, the authenticator's C{streamStarted} method will be called.
+
+        @param rootElement: The root element.
+        @type rootElement: L{domish.Element}
+        """
+        xmlstream.XmlStream.onDocumentStart(self, rootElement)
+
+        # Setup observer for stream errors
+        self.addOnetimeObserver("/error[@xmlns='%s']" % NS_STREAMS,
+                                self.onStreamError)
+
+        self.authenticator.streamStarted(rootElement)
+
+
+
+class XmlStreamFactory(xmlstream.XmlStreamFactory):
+    """
+    Factory for Jabber XmlStream objects as a reconnecting client.
+
+    Note that this differs from L{xmlstream.XmlStreamFactory} in that
+    it generates Jabber specific L{XmlStream} instances that have
+    authenticators.
+    """
+
+    protocol = XmlStream
+
+    def __init__(self, authenticator):
+        xmlstream.XmlStreamFactory.__init__(self, authenticator)
+        self.authenticator = authenticator
+
+
+
+class XmlStreamServerFactory(xmlstream.BootstrapMixin,
+                             protocol.ServerFactory):
+    """
+    Factory for Jabber XmlStream objects as a server.
+
+    @since: 8.2.
+    @ivar authenticatorFactory: Factory callable that takes no arguments, to
+                                create a fresh authenticator to be associated
+                                with the XmlStream.
+    """
+
+    protocol = XmlStream
+
+    def __init__(self, authenticatorFactory):
+        xmlstream.BootstrapMixin.__init__(self)
+        self.authenticatorFactory = authenticatorFactory
+
+
+    def buildProtocol(self, addr):
+        """
+        Create an instance of XmlStream.
+
+        A new authenticator instance will be created and passed to the new
+        XmlStream. Registered bootstrap event observers are installed as well.
+        """
+        authenticator = self.authenticatorFactory()
+        xs = self.protocol(authenticator)
+        xs.factory = self
+        self.installBootstraps(xs)
+        return xs
+
+
+
+class TimeoutError(Exception):
+    """
+    Exception raised when no IQ response has been received before the
+    configured timeout.
+    """
+
+
+
+def upgradeWithIQResponseTracker(xs):
+    """
+    Enhances an XmlStream for iq response tracking.
+
+    This makes an L{XmlStream} object provide L{IIQResponseTracker}. When a
+    response is an error iq stanza, the deferred has its errback invoked with a
+    failure that holds a L{StanzaError<error.StanzaError>} that is
+    easier to examine.
+    """
+    def callback(iq):
+        """
+        Handle iq response by firing associated deferred.
+        """
+        if getattr(iq, 'handled', False):
+            return
+
+        try:
+            d = xs.iqDeferreds[iq["id"]]
+        except KeyError:
+            pass
+        else:
+            del xs.iqDeferreds[iq["id"]]
+            iq.handled = True
+            if iq['type'] == 'error':
+                d.errback(error.exceptionFromStanza(iq))
+            else:
+                d.callback(iq)
+
+
+    def disconnected(_):
+        """
+        Make sure deferreds do not linger on after disconnect.
+
+        This errbacks all deferreds of iq's for which no response has been
+        received with a L{ConnectionLost} failure. Otherwise, the deferreds
+        will never be fired.
+        """
+        iqDeferreds = xs.iqDeferreds
+        xs.iqDeferreds = {}
+        for d in itervalues(iqDeferreds):
+            d.errback(ConnectionLost())
+
+    xs.iqDeferreds = {}
+    xs.iqDefaultTimeout = getattr(xs, 'iqDefaultTimeout', None)
+    xs.addObserver(xmlstream.STREAM_END_EVENT, disconnected)
+    xs.addObserver('/iq[@type="result"]', callback)
+    xs.addObserver('/iq[@type="error"]', callback)
+    directlyProvides(xs, ijabber.IIQResponseTracker)
+
+
+
+class IQ(domish.Element):
+    """
+    Wrapper for an iq stanza.
+
+    Iq stanzas are used for communications with a request-response behaviour.
+    Each iq request is associated with an XML stream and has its own unique id
+    to be able to track the response.
+
+    @ivar timeout: if set, a timeout period after which the deferred returned
+                   by C{send} will have its errback called with a
+                   L{TimeoutError} failure.
+    @type timeout: C{float}
+    """
+
+    timeout = None
+
+    def __init__(self, xmlstream, stanzaType="set"):
+        """
+        @type xmlstream: L{xmlstream.XmlStream}
+        @param xmlstream: XmlStream to use for transmission of this IQ
+
+        @type stanzaType: C{str}
+        @param stanzaType: IQ type identifier ('get' or 'set')
+        """
+        domish.Element.__init__(self, (None, "iq"))
+        self.addUniqueId()
+        self["type"] = stanzaType
+        self._xmlstream = xmlstream
+
+
+    def send(self, to=None):
+        """
+        Send out this iq.
+
+        Returns a deferred that is fired when an iq response with the same id
+        is received. Result responses will be passed to the deferred callback.
+        Error responses will be transformed into a
+        L{StanzaError<error.StanzaError>} and result in the errback of the
+        deferred being invoked.
+
+        @rtype: L{defer.Deferred}
+        """
+        if to is not None:
+            self["to"] = to
+
+        if not ijabber.IIQResponseTracker.providedBy(self._xmlstream):
+            upgradeWithIQResponseTracker(self._xmlstream)
+
+        d = defer.Deferred()
+        self._xmlstream.iqDeferreds[self['id']] = d
+
+        timeout = self.timeout or self._xmlstream.iqDefaultTimeout
+        if timeout is not None:
+            def onTimeout():
+                del self._xmlstream.iqDeferreds[self['id']]
+                d.errback(TimeoutError("IQ timed out"))
+
+            call = self._xmlstream._callLater(timeout, onTimeout)
+
+            def cancelTimeout(result):
+                if call.active():
+                    call.cancel()
+
+                return result
+
+            d.addBoth(cancelTimeout)
+
+        self._xmlstream.send(self)
+        return d
+
+
+
+def toResponse(stanza, stanzaType=None):
+    """
+    Create a response stanza from another stanza.
+
+    This takes the addressing and id attributes from a stanza to create a (new,
+    empty) response stanza. The addressing attributes are swapped and the id
+    copied. Optionally, the stanza type of the response can be specified.
+
+    @param stanza: the original stanza
+    @type stanza: L{domish.Element}
+    @param stanzaType: optional response stanza type
+    @type stanzaType: C{str}
+    @return: the response stanza.
+    @rtype: L{domish.Element}
+    """
+
+    toAddr = stanza.getAttribute('from')
+    fromAddr = stanza.getAttribute('to')
+    stanzaID = stanza.getAttribute('id')
+
+    response = domish.Element((None, stanza.name))
+    if toAddr:
+        response['to'] = toAddr
+    if fromAddr:
+        response['from'] = fromAddr
+    if stanzaID:
+        response['id'] = stanzaID
+    if stanzaType:
+        response['type'] = stanzaType
+
+    return response
+
+
+
+@implementer(ijabber.IXMPPHandler)
+class XMPPHandler(object):
+    """
+    XMPP protocol handler.
+
+    Classes derived from this class implement (part of) one or more XMPP
+    extension protocols, and are referred to as a subprotocol implementation.
+    """
+
+    def __init__(self):
+        self.parent = None
+        self.xmlstream = None
+
+
+    def setHandlerParent(self, parent):
+        self.parent = parent
+        self.parent.addHandler(self)
+
+
+    def disownHandlerParent(self, parent):
+        self.parent.removeHandler(self)
+        self.parent = None
+
+
+    def makeConnection(self, xs):
+        self.xmlstream = xs
+        self.connectionMade()
+
+
+    def connectionMade(self):
+        """
+        Called after a connection has been established.
+
+        Can be overridden to perform work before stream initialization.
+        """
+
+
+    def connectionInitialized(self):
+        """
+        The XML stream has been initialized.
+
+        Can be overridden to perform work after stream initialization, e.g. to
+        set up observers and start exchanging XML stanzas.
+        """
+
+
+    def connectionLost(self, reason):
+        """
+        The XML stream has been closed.
+
+        This method can be extended to inspect the C{reason} argument and
+        act on it.
+        """
+        self.xmlstream = None
+
+
+    def send(self, obj):
+        """
+        Send data over the managed XML stream.
+
+        @note: The stream manager maintains a queue for data sent using this
+               method when there is no current initialized XML stream. This
+               data is then sent as soon as a new stream has been established
+               and initialized. Subsequently, L{connectionInitialized} will be
+               called again. If this queueing is not desired, use C{send} on
+               C{self.xmlstream}.
+
+        @param obj: data to be sent over the XML stream. This is usually an
+                    object providing L{domish.IElement}, or serialized XML. See
+                    L{xmlstream.XmlStream} for details.
+        """
+        self.parent.send(obj)
+
+
+
+@implementer(ijabber.IXMPPHandlerCollection)
+class XMPPHandlerCollection(object):
+    """
+    Collection of XMPP subprotocol handlers.
+
+    This allows for grouping of subprotocol handlers, but is not an
+    L{XMPPHandler} itself, so this is not recursive.
+
+    @ivar handlers: List of protocol handlers.
+    @type handlers: C{list} of objects providing
+                      L{IXMPPHandler}
+    """
+
+    def __init__(self):
+        self.handlers = []
+
+
+    def __iter__(self):
+        """
+        Act as a container for handlers.
+        """
+        return iter(self.handlers)
+
+
+    def addHandler(self, handler):
+        """
+        Add protocol handler.
+
+        Protocol handlers are expected to provide L{ijabber.IXMPPHandler}.
+        """
+        self.handlers.append(handler)
+
+
+    def removeHandler(self, handler):
+        """
+        Remove protocol handler.
+        """
+        self.handlers.remove(handler)
+
+
+
+class StreamManager(XMPPHandlerCollection):
+    """
+    Business logic representing a managed XMPP connection.
+
+    This maintains a single XMPP connection and provides facilities for packet
+    routing and transmission. Business logic modules are objects providing
+    L{ijabber.IXMPPHandler} (like subclasses of L{XMPPHandler}), and added
+    using L{addHandler}.
+
+    @ivar xmlstream: currently managed XML stream
+    @type xmlstream: L{XmlStream}
+    @ivar logTraffic: if true, log all traffic.
+    @type logTraffic: C{bool}
+    @ivar _initialized: Whether the stream represented by L{xmlstream} has
+                        been initialized. This is used when caching outgoing
+                        stanzas.
+    @type _initialized: C{bool}
+    @ivar _packetQueue: internal buffer of unsent data. See L{send} for details.
+    @type _packetQueue: C{list}
+    """
+
+    logTraffic = False
+
+    def __init__(self, factory):
+        XMPPHandlerCollection.__init__(self)
+        self.xmlstream = None
+        self._packetQueue = []
+        self._initialized = False
+
+        factory.addBootstrap(STREAM_CONNECTED_EVENT, self._connected)
+        factory.addBootstrap(STREAM_AUTHD_EVENT, self._authd)
+        factory.addBootstrap(INIT_FAILED_EVENT, self.initializationFailed)
+        factory.addBootstrap(STREAM_END_EVENT, self._disconnected)
+        self.factory = factory
+
+
+    def addHandler(self, handler):
+        """
+        Add protocol handler.
+
+        When an XML stream has already been established, the handler's
+        C{connectionInitialized} will be called to get it up to speed.
+        """
+        XMPPHandlerCollection.addHandler(self, handler)
+
+        # get protocol handler up to speed when a connection has already
+        # been established
+        if self.xmlstream and self._initialized:
+            handler.makeConnection(self.xmlstream)
+            handler.connectionInitialized()
+
+
+    def _connected(self, xs):
+        """
+        Called when the transport connection has been established.
+
+        Here we optionally set up traffic logging (depending on L{logTraffic})
+        and call each handler's C{makeConnection} method with the L{XmlStream}
+        instance.
+        """
+        def logDataIn(buf):
+            log.msg("RECV: %r" % buf)
+
+        def logDataOut(buf):
+            log.msg("SEND: %r" % buf)
+
+        if self.logTraffic:
+            xs.rawDataInFn = logDataIn
+            xs.rawDataOutFn = logDataOut
+
+        self.xmlstream = xs
+
+        for e in self:
+            e.makeConnection(xs)
+
+
+    def _authd(self, xs):
+        """
+        Called when the stream has been initialized.
+
+        Send out cached stanzas and call each handler's
+        C{connectionInitialized} method.
+        """
+        # Flush all pending packets
+        for p in self._packetQueue:
+            xs.send(p)
+        self._packetQueue = []
+        self._initialized = True
+
+        # Notify all child services which implement
+        # the IService interface
+        for e in self:
+            e.connectionInitialized()
+
+
+    def initializationFailed(self, reason):
+        """
+        Called when stream initialization has failed.
+
+        Stream initialization has halted, with the reason indicated by
+        C{reason}. It may be retried by calling the authenticator's
+        C{initializeStream}. See the respective authenticators for details.
+
+        @param reason: A failure instance indicating why stream initialization
+                       failed.
+        @type reason: L{failure.Failure}
+        """
+
+
+    def _disconnected(self, reason):
+        """
+        Called when the stream has been closed.
+
+        From this point on, the manager doesn't interact with the
+        L{XmlStream} anymore and notifies each handler that the connection
+        was lost by calling its C{connectionLost} method.
+        """
+        self.xmlstream = None
+        self._initialized = False
+
+        # Notify all child services which implement
+        # the IService interface
+        for e in self:
+            e.connectionLost(reason)
+
+
+    def send(self, obj):
+        """
+        Send data over the XML stream.
+
+        When there is no established XML stream, the data is queued and sent
+        out when a new XML stream has been established and initialized.
+
+        @param obj: data to be sent over the XML stream. See
+                    L{xmlstream.XmlStream.send} for details.
+        """
+        if self._initialized:
+            self.xmlstream.send(obj)
+        else:
+            self._packetQueue.append(obj)
+
+
+
+__all__ = ['Authenticator', 'BaseFeatureInitiatingInitializer',
+           'ConnectAuthenticator', 'FeatureNotAdvertized',
+           'INIT_FAILED_EVENT', 'IQ', 'ListenAuthenticator', 'NS_STREAMS',
+           'NS_XMPP_TLS', 'Reset', 'STREAM_AUTHD_EVENT',
+           'STREAM_CONNECTED_EVENT', 'STREAM_END_EVENT', 'STREAM_ERROR_EVENT',
+           'STREAM_START_EVENT', 'StreamManager', 'TLSError', 'TLSFailed',
+           'TLSInitiatingInitializer', 'TLSNotSupported', 'TLSRequired',
+           'TimeoutError', 'XMPPHandler', 'XMPPHandlerCollection', 'XmlStream',
+           'XmlStreamFactory', 'XmlStreamServerFactory', 'hashPassword',
+           'toResponse', 'upgradeWithIQResponseTracker']
diff --git a/contrib/python/Twisted/py2/twisted/words/protocols/jabber/xmpp_stringprep.py b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/xmpp_stringprep.py
new file mode 100644
index 00000000000..1723856a091
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/protocols/jabber/xmpp_stringprep.py
@@ -0,0 +1,244 @@
+# -*- test-case-name: twisted.words.test.test_jabberxmppstringprep -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from encodings import idna
+from itertools import chain
+import stringprep
+
+# We require Unicode version 3.2.
+from unicodedata import ucd_3_2_0 as unicodedata
+
+from twisted.python.compat import unichr
+from twisted.python.deprecate import deprecatedModuleAttribute
+from incremental import Version
+
+from zope.interface import Interface, implementer
+
+
+crippled = False
+deprecatedModuleAttribute(
+    Version("Twisted", 13, 1, 0),
+    "crippled is always False",
+    __name__,
+    "crippled")
+
+
+
+class ILookupTable(Interface):
+    """
+    Interface for character lookup classes.
+    """
+
+    def lookup(c):
+        """
+        Return whether character is in this table.
+        """
+
+
+
+class IMappingTable(Interface):
+    """
+    Interface for character mapping classes.
+    """
+
+    def map(c):
+        """
+        Return mapping for character.
+        """
+
+
+
+@implementer(ILookupTable)
+class LookupTableFromFunction:
+
+    def __init__(self, in_table_function):
+        self.lookup = in_table_function
+
+
+
+@implementer(ILookupTable)
+class LookupTable:
+
+    def __init__(self, table):
+        self._table = table
+
+    def lookup(self, c):
+        return c in self._table
+
+
+
+@implementer(IMappingTable)
+class MappingTableFromFunction:
+
+    def __init__(self, map_table_function):
+        self.map = map_table_function
+
+
+
+@implementer(IMappingTable)
+class EmptyMappingTable:
+
+    def __init__(self, in_table_function):
+        self._in_table_function = in_table_function
+
+    def map(self, c):
+        if self._in_table_function(c):
+            return None
+        else:
+            return c
+
+
+
+class Profile:
+    def __init__(self, mappings=[],  normalize=True, prohibiteds=[],
+                       check_unassigneds=True, check_bidi=True):
+        self.mappings = mappings
+        self.normalize = normalize
+        self.prohibiteds = prohibiteds
+        self.do_check_unassigneds = check_unassigneds
+        self.do_check_bidi = check_bidi
+
+    def prepare(self, string):
+        result = self.map(string)
+        if self.normalize:
+            result = unicodedata.normalize("NFKC", result)
+        self.check_prohibiteds(result)
+        if self.do_check_unassigneds:
+            self.check_unassigneds(result)
+        if self.do_check_bidi:
+            self.check_bidirectionals(result)
+        return result
+
+    def map(self, string):
+        result = []
+
+        for c in string:
+            result_c = c
+
+            for mapping in self.mappings:
+                result_c = mapping.map(c)
+                if result_c != c:
+                    break
+
+            if result_c is not None:
+                result.append(result_c)
+
+        return u"".join(result)
+
+    def check_prohibiteds(self, string):
+        for c in string:
+            for table in self.prohibiteds:
+                if table.lookup(c):
+                    raise UnicodeError("Invalid character %s" % repr(c))
+
+    def check_unassigneds(self, string):
+        for c in string:
+            if stringprep.in_table_a1(c):
+                raise UnicodeError("Unassigned code point %s" % repr(c))
+
+    def check_bidirectionals(self, string):
+        found_LCat = False
+        found_RandALCat = False
+
+        for c in string:
+            if stringprep.in_table_d1(c):
+                found_RandALCat = True
+            if stringprep.in_table_d2(c):
+                found_LCat = True
+
+        if found_LCat and found_RandALCat:
+            raise UnicodeError("Violation of BIDI Requirement 2")
+
+        if found_RandALCat and not (stringprep.in_table_d1(string[0]) and
+                                    stringprep.in_table_d1(string[-1])):
+            raise UnicodeError("Violation of BIDI Requirement 3")
+
+
+class NamePrep:
+    """ Implements preparation of internationalized domain names.
+
+    This class implements preparing internationalized domain names using the
+    rules defined in RFC 3491, section 4 (Conversion operations).
+
+    We do not perform step 4 since we deal with unicode representations of
+    domain names and do not convert from or to ASCII representations using
+    punycode encoding. When such a conversion is needed, the C{idna} standard
+    library provides the C{ToUnicode()} and C{ToASCII()} functions. Note that
+    C{idna} itself assumes UseSTD3ASCIIRules to be false.
+
+    The following steps are performed by C{prepare()}:
+
+      - Split the domain name in labels at the dots (RFC 3490, 3.1)
+      - Apply nameprep proper on each label (RFC 3491)
+      - Enforce the restrictions on ASCII characters in host names by
+        assuming STD3ASCIIRules to be true. (STD 3)
+      - Rejoin the labels using the label separator U+002E (full stop).
+
+    """
+
+    # Prohibited characters.
+    prohibiteds = [unichr(n) for n in chain(range(0x00, 0x2c + 1),
+                                            range(0x2e, 0x2f + 1),
+                                            range(0x3a, 0x40 + 1),
+                                            range(0x5b, 0x60 + 1),
+                                            range(0x7b, 0x7f + 1))]
+
+    def prepare(self, string):
+        result = []
+
+        labels = idna.dots.split(string)
+
+        if labels and len(labels[-1]) == 0:
+            trailing_dot = u'.'
+            del labels[-1]
+        else:
+            trailing_dot = u''
+
+        for label in labels:
+            result.append(self.nameprep(label))
+
+        return u".".join(result) + trailing_dot
+
+    def check_prohibiteds(self, string):
+        for c in string:
+           if c in self.prohibiteds:
+               raise UnicodeError("Invalid character %s" % repr(c))
+
+    def nameprep(self, label):
+        label = idna.nameprep(label)
+        self.check_prohibiteds(label)
+        if label[0] == u'-':
+            raise UnicodeError("Invalid leading hyphen-minus")
+        if label[-1] == u'-':
+            raise UnicodeError("Invalid trailing hyphen-minus")
+        return label
+
+
+C_11 = LookupTableFromFunction(stringprep.in_table_c11)
+C_12 = LookupTableFromFunction(stringprep.in_table_c12)
+C_21 = LookupTableFromFunction(stringprep.in_table_c21)
+C_22 = LookupTableFromFunction(stringprep.in_table_c22)
+C_3 = LookupTableFromFunction(stringprep.in_table_c3)
+C_4 = LookupTableFromFunction(stringprep.in_table_c4)
+C_5 = LookupTableFromFunction(stringprep.in_table_c5)
+C_6 = LookupTableFromFunction(stringprep.in_table_c6)
+C_7 = LookupTableFromFunction(stringprep.in_table_c7)
+C_8 = LookupTableFromFunction(stringprep.in_table_c8)
+C_9 = LookupTableFromFunction(stringprep.in_table_c9)
+
+B_1 = EmptyMappingTable(stringprep.in_table_b1)
+B_2 = MappingTableFromFunction(stringprep.map_table_b2)
+
+nodeprep = Profile(mappings=[B_1, B_2],
+                   prohibiteds=[C_11, C_12, C_21, C_22,
+                                C_3, C_4, C_5, C_6, C_7, C_8, C_9,
+                                LookupTable([u'"', u'&', u"'", u'/',
+                                             u':', u'<', u'>', u'@'])])
+
+resourceprep = Profile(mappings=[B_1,],
+                       prohibiteds=[C_12, C_21, C_22,
+                                    C_3, C_4, C_5, C_6, C_7, C_8, C_9])
+
+nameprep = NamePrep()
diff --git a/contrib/python/Twisted/py2/twisted/words/service.py b/contrib/python/Twisted/py2/twisted/words/service.py
new file mode 100644
index 00000000000..8720b5175d9
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/service.py
@@ -0,0 +1,1269 @@
+# -*- test-case-name: twisted.words.test.test_service -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A module that needs a better name.
+
+Implements new cred things for words.
+
+How does this thing work?
+
+  - Network connection on some port expecting to speak some protocol
+
+  - Protocol-specific authentication, resulting in some kind of credentials object
+
+  - twisted.cred.portal login using those credentials for the interface
+    IUser and with something implementing IChatClient as the mind
+
+  - successful login results in an IUser avatar the protocol can call
+    methods on, and state added to the realm such that the mind will have
+    methods called on it as is necessary
+
+  - protocol specific actions lead to calls onto the avatar; remote events
+    lead to calls onto the mind
+
+  - protocol specific hangup, realm is notified, user is removed from active
+    play, the end.
+"""
+
+from time import time, ctime
+
+from zope.interface import implementer
+
+from twisted import copyright
+from twisted.cred import portal, credentials, error as ecred
+from twisted.internet import defer, protocol
+from twisted.python import log, failure, reflect
+from twisted.python.compat import itervalues, unicode
+from twisted.python.components import registerAdapter
+from twisted.spread import pb
+from twisted.words import iwords, ewords
+from twisted.words.protocols import irc
+
+
+@implementer(iwords.IGroup)
+class Group(object):
+    def __init__(self, name):
+        self.name = name
+        self.users = {}
+        self.meta = {
+            "topic": "",
+            "topic_author": "",
+            }
+
+
+    def _ebUserCall(self, err, p):
+        return failure.Failure(Exception(p, err))
+
+
+    def _cbUserCall(self, results):
+        for (success, result) in results:
+            if not success:
+                user, err = result.value # XXX
+                self.remove(user, err.getErrorMessage())
+
+
+    def add(self, user):
+        assert iwords.IChatClient.providedBy(user), "%r is not a chat client" % (user,)
+        if user.name not in self.users:
+            additions = []
+            self.users[user.name] = user
+            for p in itervalues(self.users):
+                if p is not user:
+                    d = defer.maybeDeferred(p.userJoined, self, user)
+                    d.addErrback(self._ebUserCall, p=p)
+                    additions.append(d)
+            defer.DeferredList(additions).addCallback(self._cbUserCall)
+        return defer.succeed(None)
+
+
+    def remove(self, user, reason=None):
+        try:
+            del self.users[user.name]
+        except KeyError:
+            pass
+        else:
+            removals = []
+            for p in itervalues(self.users):
+                if p is not user:
+                    d = defer.maybeDeferred(p.userLeft, self, user, reason)
+                    d.addErrback(self._ebUserCall, p=p)
+                    removals.append(d)
+            defer.DeferredList(removals).addCallback(self._cbUserCall)
+        return defer.succeed(None)
+
+
+    def size(self):
+        return defer.succeed(len(self.users))
+
+
+    def receive(self, sender, recipient, message):
+        assert recipient is self
+        receives = []
+        for p in itervalues(self.users):
+            if p is not sender:
+                d = defer.maybeDeferred(p.receive, sender, self, message)
+                d.addErrback(self._ebUserCall, p=p)
+                receives.append(d)
+        defer.DeferredList(receives).addCallback(self._cbUserCall)
+        return defer.succeed(None)
+
+
+    def setMetadata(self, meta):
+        self.meta = meta
+        sets = []
+        for p in itervalues(self.users):
+            d = defer.maybeDeferred(p.groupMetaUpdate, self, meta)
+            d.addErrback(self._ebUserCall, p=p)
+            sets.append(d)
+        defer.DeferredList(sets).addCallback(self._cbUserCall)
+        return defer.succeed(None)
+
+
+    def iterusers(self):
+        # XXX Deferred?
+        return iter(self.users.values())
+
+
+@implementer(iwords.IUser)
+class User(object):
+    realm = None
+    mind = None
+
+    def __init__(self, name):
+        self.name = name
+        self.groups = []
+        self.lastMessage = time()
+
+
+    def loggedIn(self, realm, mind):
+        self.realm = realm
+        self.mind = mind
+        self.signOn = time()
+
+
+    def join(self, group):
+        def cbJoin(result):
+            self.groups.append(group)
+            return result
+        return group.add(self.mind).addCallback(cbJoin)
+
+
+    def leave(self, group, reason=None):
+        def cbLeave(result):
+            self.groups.remove(group)
+            return result
+        return group.remove(self.mind, reason).addCallback(cbLeave)
+
+
+    def send(self, recipient, message):
+        self.lastMessage = time()
+        return recipient.receive(self.mind, recipient, message)
+
+
+    def itergroups(self):
+        return iter(self.groups)
+
+
+    def logout(self):
+        for g in self.groups[:]:
+            self.leave(g)
+
+
+NICKSERV = 'NickServ!NickServ@services'
+
+
+@implementer(iwords.IChatClient)
+class IRCUser(irc.IRC):
+    """
+    Protocol instance representing an IRC user connected to the server.
+    """
+    # A list of IGroups in which I am participating
+    groups = None
+
+    # A no-argument callable I should invoke when I go away
+    logout = None
+
+    # An IUser we use to interact with the chat service
+    avatar = None
+
+    # To whence I belong
+    realm = None
+
+    # How to handle unicode (TODO: Make this customizable on a per-user basis)
+    encoding = 'utf-8'
+
+    # Twisted callbacks
+    def connectionMade(self):
+        self.irc_PRIVMSG = self.irc_NICKSERV_PRIVMSG
+        self.realm = self.factory.realm
+        self.hostname = self.realm.name
+
+
+    def connectionLost(self, reason):
+        if self.logout is not None:
+            self.logout()
+            self.avatar = None
+
+
+    # Make sendMessage a bit more useful to us
+    def sendMessage(self, command, *parameter_list, **kw):
+        if 'prefix' not in kw:
+            kw['prefix'] = self.hostname
+        if 'to' not in kw:
+            kw['to'] = self.name.encode(self.encoding)
+
+        arglist = [self, command, kw['to']] + list(parameter_list)
+        arglistUnicode  = []
+        for arg in arglist:
+            if isinstance(arg, bytes):
+                arg = arg.decode("utf-8")
+            arglistUnicode.append(arg)
+        irc.IRC.sendMessage(*arglistUnicode, **kw)
+
+
+    # IChatClient implementation
+    def userJoined(self, group, user):
+        self.join(
+            "%s!%s@%s" % (user.name, user.name, self.hostname),
+            '#' + group.name)
+
+
+    def userLeft(self, group, user, reason=None):
+        self.part(
+            "%s!%s@%s" % (user.name, user.name, self.hostname),
+            '#' + group.name,
+            (reason or u"leaving"))
+
+
+    def receive(self, sender, recipient, message):
+        #>> :glyph!glyph@adsl-64-123-27-108.dsl.austtx.swbell.net PRIVMSG glyph_ :hello
+
+        # omg???????????
+        if iwords.IGroup.providedBy(recipient):
+            recipientName = '#' + recipient.name
+        else:
+            recipientName = recipient.name
+
+        text = message.get('text', '<an unrepresentable message>')
+        for L in text.splitlines():
+            self.privmsg(
+                '%s!%s@%s' % (sender.name, sender.name, self.hostname),
+                recipientName,
+                L)
+
+
+    def groupMetaUpdate(self, group, meta):
+        if 'topic' in meta:
+            topic = meta['topic']
+            author = meta.get('topic_author', '')
+            self.topic(
+                self.name,
+                '#' + group.name,
+                topic,
+                '%s!%s@%s' % (author, author, self.hostname)
+                )
+
+    # irc.IRC callbacks - starting with login related stuff.
+    nickname = None
+    password = None
+
+    def irc_PASS(self, prefix, params):
+        """
+        Password message -- Register a password.
+
+        Parameters: <password>
+
+        [REQUIRED]
+
+        Note that IRC requires the client send this *before* NICK
+        and USER.
+        """
+        self.password = params[-1]
+
+
+    def irc_NICK(self, prefix, params):
+        """
+        Nick message -- Set your nickname.
+
+        Parameters: <nickname>
+
+        [REQUIRED]
+        """
+        nickname = params[0]
+        try:
+            if isinstance(nickname, bytes):
+                nickname = nickname.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.privmsg(
+                NICKSERV,
+                repr(nickname),
+                'Your nickname cannot be decoded. Please use ASCII or UTF-8.')
+            self.transport.loseConnection()
+            return
+
+        self.nickname = nickname
+        self.name = nickname
+
+        for code, text in self._motdMessages:
+            self.sendMessage(code, text % self.factory._serverInfo)
+
+        if self.password is None:
+            self.privmsg(
+                NICKSERV,
+                nickname,
+                'Password?')
+        else:
+            password = self.password
+            self.password = None
+            self.logInAs(nickname, password)
+
+
+    def irc_USER(self, prefix, params):
+        """
+        User message -- Set your realname.
+
+        Parameters: <user> <mode> <unused> <realname>
+        """
+        # Note: who gives a crap about this?  The IUser has the real
+        # information we care about.  Save it anyway, I guess, just
+        # for fun.
+        self.realname = params[-1]
+
+
+    def irc_NICKSERV_PRIVMSG(self, prefix, params):
+        """
+        Send a (private) message.
+
+        Parameters: <msgtarget> <text to be sent>
+        """
+        target = params[0]
+        password = params[-1]
+
+        if self.nickname is None:
+            # XXX Send an error response here
+            self.transport.loseConnection()
+        elif target.lower() != "nickserv":
+            self.privmsg(
+                NICKSERV,
+                self.nickname,
+                "Denied.  Please send me (NickServ) your password.")
+        else:
+            nickname = self.nickname
+            self.nickname = None
+            self.logInAs(nickname, password)
+
+
+    def logInAs(self, nickname, password):
+        d = self.factory.portal.login(
+            credentials.UsernamePassword(nickname, password),
+            self,
+            iwords.IUser)
+        d.addCallbacks(self._cbLogin, self._ebLogin, errbackArgs=(nickname,))
+
+
+    _welcomeMessages = [
+        (irc.RPL_WELCOME,
+         ":connected to Twisted IRC"),
+        (irc.RPL_YOURHOST,
+         ":Your host is %(serviceName)s, running version %(serviceVersion)s"),
+        (irc.RPL_CREATED,
+         ":This server was created on %(creationDate)s"),
+
+        # "Bummer.  This server returned a worthless 004 numeric.
+        #  I'll have to guess at all the values"
+        #    -- epic
+        (irc.RPL_MYINFO,
+         # w and n are the currently supported channel and user modes
+         # -- specify this better
+         "%(serviceName)s %(serviceVersion)s w n")
+        ]
+
+    _motdMessages = [
+        (irc.RPL_MOTDSTART,
+         ":- %(serviceName)s Message of the Day - "),
+        (irc.RPL_ENDOFMOTD,
+         ":End of /MOTD command.")
+        ]
+
+    def _cbLogin(self, result):
+        (iface, avatar, logout) = result
+        assert iface is iwords.IUser, "Realm is buggy, got %r" % (iface,)
+
+        # Let them send messages to the world
+        del self.irc_PRIVMSG
+
+        self.avatar = avatar
+        self.logout = logout
+        for code, text in self._welcomeMessages:
+            self.sendMessage(code, text % self.factory._serverInfo)
+
+
+    def _ebLogin(self, err, nickname):
+        if err.check(ewords.AlreadyLoggedIn):
+            self.privmsg(
+                NICKSERV,
+                nickname,
+                "Already logged in.  No pod people allowed!")
+        elif err.check(ecred.UnauthorizedLogin):
+            self.privmsg(
+                NICKSERV,
+                nickname,
+                "Login failed.  Goodbye.")
+        else:
+            log.msg("Unhandled error during login:")
+            log.err(err)
+            self.privmsg(
+                NICKSERV,
+                nickname,
+                "Server error during login.  Sorry.")
+        self.transport.loseConnection()
+
+
+    # Great, now that's out of the way, here's some of the interesting
+    # bits
+    def irc_PING(self, prefix, params):
+        """
+        Ping message
+
+        Parameters: <server1> [ <server2> ]
+        """
+        if self.realm is not None:
+            self.sendMessage('PONG', self.hostname)
+
+
+    def irc_QUIT(self, prefix, params):
+        """
+        Quit
+
+        Parameters: [ <Quit Message> ]
+        """
+        self.transport.loseConnection()
+
+
+    def _channelMode(self, group, modes=None, *args):
+        if modes:
+            self.sendMessage(
+                irc.ERR_UNKNOWNMODE,
+                ":Unknown MODE flag.")
+        else:
+            self.channelMode(self.name, '#' + group.name, '+')
+
+
+    def _userMode(self, user, modes=None):
+        if modes:
+            self.sendMessage(
+                irc.ERR_UNKNOWNMODE,
+                ":Unknown MODE flag.")
+        elif user is self.avatar:
+            self.sendMessage(
+                irc.RPL_UMODEIS,
+                "+")
+        else:
+            self.sendMessage(
+                irc.ERR_USERSDONTMATCH,
+                ":You can't look at someone else's modes.")
+
+
+    def irc_MODE(self, prefix, params):
+        """
+        User mode message
+
+        Parameters: <nickname>
+        *( ( "+" / "-" ) *( "i" / "w" / "o" / "O" / "r" ) )
+
+        """
+        try:
+            channelOrUser = params[0]
+            if isinstance(channelOrUser, bytes):
+                channelOrUser = channelOrUser.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHNICK, params[0],
+                ":No such nickname (could not decode your unicode!)")
+            return
+
+        if channelOrUser.startswith('#'):
+            def ebGroup(err):
+                err.trap(ewords.NoSuchGroup)
+                self.sendMessage(
+                    irc.ERR_NOSUCHCHANNEL, params[0],
+                    ":That channel doesn't exist.")
+            d = self.realm.lookupGroup(channelOrUser[1:])
+            d.addCallbacks(
+                self._channelMode,
+                ebGroup,
+                callbackArgs=tuple(params[1:]))
+        else:
+            def ebUser(err):
+                self.sendMessage(
+                    irc.ERR_NOSUCHNICK,
+                    ":No such nickname.")
+
+            d = self.realm.lookupUser(channelOrUser)
+            d.addCallbacks(
+                self._userMode,
+                ebUser,
+                callbackArgs=tuple(params[1:]))
+
+
+    def irc_USERHOST(self, prefix, params):
+        """
+        Userhost message
+
+        Parameters: <nickname> *( SPACE <nickname> )
+
+        [Optional]
+        """
+        pass
+
+
+    def irc_PRIVMSG(self, prefix, params):
+        """
+        Send a (private) message.
+
+        Parameters: <msgtarget> <text to be sent>
+        """
+        try:
+            targetName = params[0]
+            if isinstance(targetName, bytes):
+                targetName = targetName.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHNICK, params[0],
+                ":No such nick/channel (could not decode your unicode!)")
+            return
+
+        messageText = params[-1]
+        if targetName.startswith('#'):
+            target = self.realm.lookupGroup(targetName[1:])
+        else:
+            target = self.realm.lookupUser(targetName).addCallback(lambda user: user.mind)
+
+        def cbTarget(targ):
+            if targ is not None:
+                return self.avatar.send(targ, {"text": messageText})
+
+        def ebTarget(err):
+            self.sendMessage(
+                irc.ERR_NOSUCHNICK, targetName,
+                ":No such nick/channel.")
+
+        target.addCallbacks(cbTarget, ebTarget)
+
+
+    def irc_JOIN(self, prefix, params):
+        """
+        Join message
+
+        Parameters: ( <channel> *( "," <channel> ) [ <key> *( "," <key> ) ] )
+        """
+        try:
+            groupName = params[0]
+            if isinstance(groupName, bytes):
+                groupName = groupName.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL, params[0],
+                ":No such channel (could not decode your unicode!)")
+            return
+
+        if groupName.startswith('#'):
+            groupName = groupName[1:]
+
+        def cbGroup(group):
+            def cbJoin(ign):
+                self.userJoined(group, self)
+                self.names(
+                    self.name,
+                    '#' + group.name,
+                    [user.name for user in group.iterusers()])
+                self._sendTopic(group)
+            return self.avatar.join(group).addCallback(cbJoin)
+
+        def ebGroup(err):
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL, '#' + groupName,
+                ":No such channel.")
+
+        self.realm.getGroup(groupName).addCallbacks(cbGroup, ebGroup)
+
+
+    def irc_PART(self, prefix, params):
+        """
+        Part message
+
+        Parameters: <channel> *( "," <channel> ) [ <Part Message> ]
+        """
+        try:
+            groupName = params[0]
+            if isinstance(params[0], bytes):
+                groupName = params[0].decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOTONCHANNEL, params[0],
+                ":Could not decode your unicode!")
+            return
+
+        if groupName.startswith('#'):
+            groupName = groupName[1:]
+
+        if len(params) > 1:
+            reason = params[1]
+            if isinstance(reason, bytes):
+                reason = reason.decode('utf-8')
+        else:
+            reason = None
+
+        def cbGroup(group):
+            def cbLeave(result):
+                self.userLeft(group, self, reason)
+            return self.avatar.leave(group, reason).addCallback(cbLeave)
+
+        def ebGroup(err):
+            err.trap(ewords.NoSuchGroup)
+            self.sendMessage(
+                irc.ERR_NOTONCHANNEL,
+                '#' + groupName,
+                ":" + err.getErrorMessage())
+
+        self.realm.lookupGroup(groupName).addCallbacks(cbGroup, ebGroup)
+
+
+    def irc_NAMES(self, prefix, params):
+        """
+        Names message
+
+        Parameters: [ <channel> *( "," <channel> ) [ <target> ] ]
+        """
+        #<< NAMES #python
+        #>> :benford.openprojects.net 353 glyph = #python :Orban ... @glyph ... Zymurgy skreech
+        #>> :benford.openprojects.net 366 glyph #python :End of /NAMES list.
+        try:
+            channel = params[-1]
+            if isinstance(channel, bytes):
+                channel = channel.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL, params[-1],
+                ":No such channel (could not decode your unicode!)")
+            return
+
+        if channel.startswith('#'):
+            channel = channel[1:]
+
+        def cbGroup(group):
+            self.names(
+                self.name,
+                '#' + group.name,
+                [user.name for user in group.iterusers()])
+
+        def ebGroup(err):
+            err.trap(ewords.NoSuchGroup)
+            # No group?  Fine, no names!
+            self.names(
+                self.name,
+                '#' + channel,
+                [])
+
+        self.realm.lookupGroup(channel).addCallbacks(cbGroup, ebGroup)
+
+
+    def irc_TOPIC(self, prefix, params):
+        """
+        Topic message
+
+        Parameters: <channel> [ <topic> ]
+        """
+        try:
+            channel = params[0]
+            if isinstance(params[0], bytes):
+                channel = channel.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL,
+                ":That channel doesn't exist (could not decode your unicode!)")
+            return
+
+        if channel.startswith('#'):
+            channel = channel[1:]
+
+        if len(params) > 1:
+            self._setTopic(channel, params[1])
+        else:
+            self._getTopic(channel)
+
+
+    def _sendTopic(self, group):
+        """
+        Send the topic of the given group to this user, if it has one.
+        """
+        topic = group.meta.get("topic")
+        if topic:
+            author = group.meta.get("topic_author") or "<noone>"
+            date = group.meta.get("topic_date", 0)
+            self.topic(self.name, '#' + group.name, topic)
+            self.topicAuthor(self.name, '#' + group.name, author, date)
+
+
+    def _getTopic(self, channel):
+        #<< TOPIC #python
+        #>> :benford.openprojects.net 332 glyph #python :<churchr> I really did. I sprained all my toes.
+        #>> :benford.openprojects.net 333 glyph #python itamar|nyc 994713482
+        def ebGroup(err):
+            err.trap(ewords.NoSuchGroup)
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL, '=', channel,
+                ":That channel doesn't exist.")
+
+        self.realm.lookupGroup(channel).addCallbacks(self._sendTopic, ebGroup)
+
+
+    def _setTopic(self, channel, topic):
+        #<< TOPIC #divunal :foo
+        #>> :glyph!glyph@adsl-64-123-27-108.dsl.austtx.swbell.net TOPIC #divunal :foo
+
+        def cbGroup(group):
+            newMeta = group.meta.copy()
+            newMeta['topic'] = topic
+            newMeta['topic_author'] = self.name
+            newMeta['topic_date'] = int(time())
+
+            def ebSet(err):
+                self.sendMessage(
+                    irc.ERR_CHANOPRIVSNEEDED,
+                    "#" + group.name,
+                    ":You need to be a channel operator to do that.")
+
+            return group.setMetadata(newMeta).addErrback(ebSet)
+
+        def ebGroup(err):
+            err.trap(ewords.NoSuchGroup)
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL, '=', channel,
+                ":That channel doesn't exist.")
+
+        self.realm.lookupGroup(channel).addCallbacks(cbGroup, ebGroup)
+
+
+    def list(self, channels):
+        """
+        Send a group of LIST response lines
+
+        @type channel: C{list} of C{(str, int, str)}
+        @param channel: Information about the channels being sent:
+        their name, the number of participants, and their topic.
+        """
+        for (name, size, topic) in channels:
+            self.sendMessage(irc.RPL_LIST, name, str(size), ":" + topic)
+        self.sendMessage(irc.RPL_LISTEND, ":End of /LIST")
+
+
+    def irc_LIST(self, prefix, params):
+        """
+        List query
+
+        Return information about the indicated channels, or about all
+        channels if none are specified.
+
+        Parameters: [ <channel> *( "," <channel> ) [ <target> ] ]
+        """
+        #<< list #python
+        #>> :orwell.freenode.net 321 exarkun Channel :Users  Name
+        #>> :orwell.freenode.net 322 exarkun #python 358 :The Python programming language
+        #>> :orwell.freenode.net 323 exarkun :End of /LIST
+        if params:
+            # Return information about indicated channels
+            try:
+                allChannels = params[0]
+                if isinstance(allChannels, bytes):
+                    allChannels = allChannels.decode(self.encoding)
+                channels = allChannels.split(',')
+            except UnicodeDecodeError:
+                self.sendMessage(
+                    irc.ERR_NOSUCHCHANNEL, params[0],
+                    ":No such channel (could not decode your unicode!)")
+                return
+
+            groups = []
+            for ch in channels:
+                if ch.startswith('#'):
+                    ch = ch[1:]
+                groups.append(self.realm.lookupGroup(ch))
+
+            groups = defer.DeferredList(groups, consumeErrors=True)
+            groups.addCallback(lambda gs: [r for (s, r) in gs if s])
+        else:
+            # Return information about all channels
+            groups = self.realm.itergroups()
+
+        def cbGroups(groups):
+            def gotSize(size, group):
+                return group.name, size, group.meta.get('topic')
+            d = defer.DeferredList([
+                group.size().addCallback(gotSize, group) for group in groups])
+            d.addCallback(lambda results: self.list([r for (s, r) in results if s]))
+            return d
+        groups.addCallback(cbGroups)
+
+
+    def _channelWho(self, group):
+        self.who(self.name, '#' + group.name,
+            [(m.name, self.hostname, self.realm.name, m.name, "H", 0, m.name) for m in group.iterusers()])
+
+
+    def _userWho(self, user):
+        self.sendMessage(irc.RPL_ENDOFWHO,
+                         ":User /WHO not implemented")
+
+
+    def irc_WHO(self, prefix, params):
+        """
+        Who query
+
+        Parameters: [ <mask> [ "o" ] ]
+        """
+        #<< who #python
+        #>> :x.opn 352 glyph #python aquarius pc-62-31-193-114-du.blueyonder.co.uk y.opn Aquarius H :3 Aquarius
+        # ...
+        #>> :x.opn 352 glyph #python foobar europa.tranquility.net z.opn skreech H :0 skreech
+        #>> :x.opn 315 glyph #python :End of /WHO list.
+        ### also
+        #<< who glyph
+        #>> :x.opn 352 glyph #python glyph adsl-64-123-27-108.dsl.austtx.swbell.net x.opn glyph H :0 glyph
+        #>> :x.opn 315 glyph glyph :End of /WHO list.
+        if not params:
+            self.sendMessage(irc.RPL_ENDOFWHO, ":/WHO not supported.")
+            return
+
+        try:
+            channelOrUser = params[0]
+            if isinstance(channelOrUser, bytes):
+                channelOrUser = channelOrUser.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.RPL_ENDOFWHO, params[0],
+                ":End of /WHO list (could not decode your unicode!)")
+            return
+
+        if channelOrUser.startswith('#'):
+            def ebGroup(err):
+                err.trap(ewords.NoSuchGroup)
+                self.sendMessage(
+                    irc.RPL_ENDOFWHO, channelOrUser,
+                    ":End of /WHO list.")
+            d = self.realm.lookupGroup(channelOrUser[1:])
+            d.addCallbacks(self._channelWho, ebGroup)
+        else:
+            def ebUser(err):
+                err.trap(ewords.NoSuchUser)
+                self.sendMessage(
+                    irc.RPL_ENDOFWHO, channelOrUser,
+                    ":End of /WHO list.")
+            d = self.realm.lookupUser(channelOrUser)
+            d.addCallbacks(self._userWho, ebUser)
+
+
+
+    def irc_WHOIS(self, prefix, params):
+        """
+        Whois query
+
+        Parameters: [ <target> ] <mask> *( "," <mask> )
+        """
+        def cbUser(user):
+            self.whois(
+                self.name,
+                user.name, user.name, self.realm.name,
+                user.name, self.realm.name, 'Hi mom!', False,
+                int(time() - user.lastMessage), user.signOn,
+                ['#' + group.name for group in user.itergroups()])
+
+        def ebUser(err):
+            err.trap(ewords.NoSuchUser)
+            self.sendMessage(
+                irc.ERR_NOSUCHNICK,
+                params[0],
+                ":No such nick/channel")
+
+        try:
+            user = params[0]
+            if isinstance(user, bytes):
+                user = user.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHNICK,
+                params[0],
+                ":No such nick/channel")
+            return
+
+        self.realm.lookupUser(user).addCallbacks(cbUser, ebUser)
+
+
+    # Unsupported commands, here for legacy compatibility
+    def irc_OPER(self, prefix, params):
+        """
+        Oper message
+
+        Parameters: <name> <password>
+        """
+        self.sendMessage(irc.ERR_NOOPERHOST, ":O-lines not applicable")
+
+
+class IRCFactory(protocol.ServerFactory):
+    """
+    IRC server that creates instances of the L{IRCUser} protocol.
+
+    @ivar _serverInfo: A dictionary mapping:
+        "serviceName" to the name of the server,
+        "serviceVersion" to the copyright version,
+        "creationDate" to the time that the server was started.
+    """
+    protocol = IRCUser
+
+    def __init__(self, realm, portal):
+        self.realm = realm
+        self.portal = portal
+        self._serverInfo = {
+            "serviceName": self.realm.name,
+            "serviceVersion": copyright.version,
+            "creationDate": ctime()
+            }
+
+
+
+class PBMind(pb.Referenceable):
+    def __init__(self):
+        pass
+
+    def jellyFor(self, jellier):
+        qual = reflect.qual(PBMind)
+        if isinstance(qual, unicode):
+            qual = qual.encode("utf-8")
+        return qual, jellier.invoker.registerReference(self)
+
+    def remote_userJoined(self, user, group):
+        pass
+
+    def remote_userLeft(self, user, group, reason):
+        pass
+
+    def remote_receive(self, sender, recipient, message):
+        pass
+
+    def remote_groupMetaUpdate(self, group, meta):
+        pass
+
+
+@implementer(iwords.IChatClient)
+class PBMindReference(pb.RemoteReference):
+    def receive(self, sender, recipient, message):
+        if iwords.IGroup.providedBy(recipient):
+            rec = PBGroup(self.realm, self.avatar, recipient)
+        else:
+            rec = PBUser(self.realm, self.avatar, recipient)
+        return self.callRemote(
+            'receive',
+            PBUser(self.realm, self.avatar, sender),
+            rec,
+            message)
+
+    def groupMetaUpdate(self, group, meta):
+        return self.callRemote(
+            'groupMetaUpdate',
+            PBGroup(self.realm, self.avatar, group),
+            meta)
+
+    def userJoined(self, group, user):
+        return self.callRemote(
+            'userJoined',
+            PBGroup(self.realm, self.avatar, group),
+            PBUser(self.realm, self.avatar, user))
+
+    def userLeft(self, group, user, reason=None):
+        return self.callRemote(
+            'userLeft',
+            PBGroup(self.realm, self.avatar, group),
+            PBUser(self.realm, self.avatar, user),
+            reason)
+pb.setUnjellyableForClass(PBMind, PBMindReference)
+
+
+class PBGroup(pb.Referenceable):
+    def __init__(self, realm, avatar, group):
+        self.realm = realm
+        self.avatar = avatar
+        self.group = group
+
+
+    def processUniqueID(self):
+        return hash((self.realm.name, self.avatar.name, self.group.name))
+
+
+    def jellyFor(self, jellier):
+        qual = reflect.qual(self.__class__)
+        if isinstance(qual, unicode):
+            qual = qual.encode("utf-8")
+        group = self.group.name
+        if isinstance(group, unicode):
+            group = group.encode("utf-8")
+        return qual, group, jellier.invoker.registerReference(self)
+
+
+    def remote_leave(self, reason=None):
+        return self.avatar.leave(self.group, reason)
+
+
+    def remote_send(self, message):
+        return self.avatar.send(self.group, message)
+
+
+@implementer(iwords.IGroup)
+class PBGroupReference(pb.RemoteReference):
+    def unjellyFor(self, unjellier, unjellyList):
+        clsName, name, ref = unjellyList
+        self.name = name
+        if bytes != str and isinstance(self.name, bytes):
+            self.name = self.name.decode('utf-8')
+        return pb.RemoteReference.unjellyFor(self, unjellier, [clsName, ref])
+
+    def leave(self, reason=None):
+        return self.callRemote("leave", reason)
+
+    def send(self, message):
+        return self.callRemote("send", message)
+pb.setUnjellyableForClass(PBGroup, PBGroupReference)
+
+class PBUser(pb.Referenceable):
+    def __init__(self, realm, avatar, user):
+        self.realm = realm
+        self.avatar = avatar
+        self.user = user
+
+    def processUniqueID(self):
+        return hash((self.realm.name, self.avatar.name, self.user.name))
+
+
+@implementer(iwords.IChatClient)
+class ChatAvatar(pb.Referenceable):
+    def __init__(self, avatar):
+        self.avatar = avatar
+
+
+    def jellyFor(self, jellier):
+        qual = reflect.qual(self.__class__)
+        if isinstance(qual, unicode):
+            qual = qual.encode("utf-8")
+        return qual, jellier.invoker.registerReference(self)
+
+
+    def remote_join(self, groupName):
+        def cbGroup(group):
+            def cbJoin(ignored):
+                return PBGroup(self.avatar.realm, self.avatar, group)
+            d = self.avatar.join(group)
+            d.addCallback(cbJoin)
+            return d
+        d = self.avatar.realm.getGroup(groupName)
+        d.addCallback(cbGroup)
+        return d
+registerAdapter(ChatAvatar, iwords.IUser, pb.IPerspective)
+
+class AvatarReference(pb.RemoteReference):
+    def join(self, groupName):
+        return self.callRemote('join', groupName)
+
+    def quit(self):
+        d = defer.Deferred()
+        self.broker.notifyOnDisconnect(lambda: d.callback(None))
+        self.broker.transport.loseConnection()
+        return d
+
+pb.setUnjellyableForClass(ChatAvatar, AvatarReference)
+
+
+@implementer(portal.IRealm, iwords.IChatService)
+class WordsRealm(object):
+    _encoding = 'utf-8'
+
+    def __init__(self, name):
+        self.name = name
+
+
+    def userFactory(self, name):
+        return User(name)
+
+
+    def groupFactory(self, name):
+        return Group(name)
+
+
+    def logoutFactory(self, avatar, facet):
+        def logout():
+            # XXX Deferred support here
+            getattr(facet, 'logout', lambda: None)()
+            avatar.realm = avatar.mind = None
+        return logout
+
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        if isinstance(avatarId, bytes):
+            avatarId = avatarId.decode(self._encoding)
+
+        def gotAvatar(avatar):
+            if avatar.realm is not None:
+                raise ewords.AlreadyLoggedIn()
+            for iface in interfaces:
+                facet = iface(avatar, None)
+                if facet is not None:
+                    avatar.loggedIn(self, mind)
+                    mind.name = avatarId
+                    mind.realm = self
+                    mind.avatar = avatar
+                    return iface, facet, self.logoutFactory(avatar, facet)
+            raise NotImplementedError(self, interfaces)
+
+        return self.getUser(avatarId).addCallback(gotAvatar)
+
+
+    # IChatService, mostly.
+    createGroupOnRequest = False
+    createUserOnRequest = True
+
+    def lookupUser(self, name):
+        raise NotImplementedError
+
+
+    def lookupGroup(self, group):
+        raise NotImplementedError
+
+
+    def addUser(self, user):
+        """
+        Add the given user to this service.
+
+        This is an internal method intended to be overridden by
+        L{WordsRealm} subclasses, not called by external code.
+
+        @type user: L{IUser}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with L{None} when the user is
+        added, or which fails with
+        L{twisted.words.ewords.DuplicateUser} if a user with the
+        same name exists already.
+        """
+        raise NotImplementedError
+
+
+    def addGroup(self, group):
+        """
+        Add the given group to this service.
+
+        @type group: L{IGroup}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with L{None} when the group is
+        added, or which fails with
+        L{twisted.words.ewords.DuplicateGroup} if a group with the
+        same name exists already.
+        """
+        raise NotImplementedError
+
+
+    def getGroup(self, name):
+        if self.createGroupOnRequest:
+            def ebGroup(err):
+                err.trap(ewords.DuplicateGroup)
+                return self.lookupGroup(name)
+            return self.createGroup(name).addErrback(ebGroup)
+        return self.lookupGroup(name)
+
+
+    def getUser(self, name):
+        if self.createUserOnRequest:
+            def ebUser(err):
+                err.trap(ewords.DuplicateUser)
+                return self.lookupUser(name)
+            return self.createUser(name).addErrback(ebUser)
+        return self.lookupUser(name)
+
+
+    def createUser(self, name):
+        def cbLookup(user):
+            return failure.Failure(ewords.DuplicateUser(name))
+        def ebLookup(err):
+            err.trap(ewords.NoSuchUser)
+            return self.userFactory(name)
+
+        name = name.lower()
+        d = self.lookupUser(name)
+        d.addCallbacks(cbLookup, ebLookup)
+        d.addCallback(self.addUser)
+        return d
+
+
+    def createGroup(self, name):
+        def cbLookup(group):
+            return failure.Failure(ewords.DuplicateGroup(name))
+        def ebLookup(err):
+            err.trap(ewords.NoSuchGroup)
+            return self.groupFactory(name)
+
+        name = name.lower()
+        d = self.lookupGroup(name)
+        d.addCallbacks(cbLookup, ebLookup)
+        d.addCallback(self.addGroup)
+        return d
+
+
+class InMemoryWordsRealm(WordsRealm):
+    def __init__(self, *a, **kw):
+        super(InMemoryWordsRealm, self).__init__(*a, **kw)
+        self.users = {}
+        self.groups = {}
+
+
+    def itergroups(self):
+        return defer.succeed(itervalues(self.groups))
+
+
+    def addUser(self, user):
+        if user.name in self.users:
+            return defer.fail(failure.Failure(ewords.DuplicateUser()))
+        self.users[user.name] = user
+        return defer.succeed(user)
+
+
+    def addGroup(self, group):
+        if group.name in self.groups:
+            return defer.fail(failure.Failure(ewords.DuplicateGroup()))
+        self.groups[group.name] = group
+        return defer.succeed(group)
+
+
+    def lookupUser(self, name):
+        name = name.lower()
+        try:
+            user = self.users[name]
+        except KeyError:
+            return defer.fail(failure.Failure(ewords.NoSuchUser(name)))
+        else:
+            return defer.succeed(user)
+
+
+    def lookupGroup(self, name):
+        name = name.lower()
+        try:
+            group = self.groups[name]
+        except KeyError:
+            return defer.fail(failure.Failure(ewords.NoSuchGroup(name)))
+        else:
+            return defer.succeed(group)
+
+__all__ = [
+    'Group', 'User',
+
+    'WordsRealm', 'InMemoryWordsRealm',
+    ]
diff --git a/contrib/python/Twisted/py2/twisted/words/tap.py b/contrib/python/Twisted/py2/twisted/words/tap.py
new file mode 100644
index 00000000000..c0ba9fd0d7a
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/tap.py
@@ -0,0 +1,74 @@
+# -*- test-case-name: twisted.words.test.test_tap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Shiny new words service maker
+"""
+
+import sys, socket
+
+from twisted.application import strports
+from twisted.application.service import MultiService
+from twisted.python import usage
+from twisted import plugin
+
+from twisted.words import iwords, service
+from twisted.cred import checkers, credentials, portal, strcred
+
+class Options(usage.Options, strcred.AuthOptionMixin):
+    supportedInterfaces = [credentials.IUsernamePassword]
+    optParameters = [
+        ('hostname', None, socket.gethostname(),
+         'Name of this server; purely an informative')]
+
+    compData = usage.Completions(multiUse=["group"])
+
+    interfacePlugins = {}
+    plg = None
+    for plg in plugin.getPlugins(iwords.IProtocolPlugin):
+        assert plg.name not in interfacePlugins
+        interfacePlugins[plg.name] = plg
+        optParameters.append((
+            plg.name + '-port',
+            None, None,
+            'strports description of the port to bind for the  ' + plg.name + ' server'))
+    del plg
+
+    def __init__(self, *a, **kw):
+        usage.Options.__init__(self, *a, **kw)
+        self['groups'] = []
+
+    def opt_group(self, name):
+        """Specify a group which should exist
+        """
+        self['groups'].append(name.decode(sys.stdin.encoding))
+
+    def opt_passwd(self, filename):
+        """
+        Name of a passwd-style file. (This is for
+        backwards-compatibility only; you should use the --auth
+        command instead.)
+        """
+        self.addChecker(checkers.FilePasswordDB(filename))
+
+def makeService(config):
+    credCheckers = config.get('credCheckers', [])
+    wordsRealm = service.InMemoryWordsRealm(config['hostname'])
+    wordsPortal = portal.Portal(wordsRealm, credCheckers)
+
+    msvc = MultiService()
+
+    # XXX Attribute lookup on config is kind of bad - hrm.
+    for plgName in config.interfacePlugins:
+        port = config.get(plgName + '-port')
+        if port is not None:
+            factory = config.interfacePlugins[plgName].getFactory(wordsRealm, wordsPortal)
+            svc = strports.service(port, factory)
+            svc.setServiceParent(msvc)
+
+    # This is bogus.  createGroup is async.  makeService must be
+    # allowed to return a Deferred or some crap.
+    for g in config['groups']:
+        wordsRealm.createGroup(g)
+
+    return msvc
diff --git a/contrib/python/Twisted/py2/twisted/words/xish/__init__.py b/contrib/python/Twisted/py2/twisted/words/xish/__init__.py
new file mode 100644
index 00000000000..1d2469fe303
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/xish/__init__.py
@@ -0,0 +1,10 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+
+Twisted X-ish: XML-ish DOM and XPath-ish engine
+
+"""
diff --git a/contrib/python/Twisted/py2/twisted/words/xish/domish.py b/contrib/python/Twisted/py2/twisted/words/xish/domish.py
new file mode 100644
index 00000000000..fc49285f58e
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/xish/domish.py
@@ -0,0 +1,899 @@
+# -*- test-case-name: twisted.words.test.test_domish -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+DOM-like XML processing support.
+
+This module provides support for parsing XML into DOM-like object structures
+and serializing such structures to an XML string representation, optimized
+for use in streaming XML applications.
+"""
+
+from __future__ import absolute_import, division
+
+from zope.interface import implementer, Interface, Attribute
+
+from twisted.python.compat import (_PY3, StringType, _coercedUnicode,
+                                   iteritems, itervalues, unicode)
+
+def _splitPrefix(name):
+    """ Internal method for splitting a prefixed Element name into its
+        respective parts """
+    ntok = name.split(":", 1)
+    if len(ntok) == 2:
+        return ntok
+    else:
+        return (None, ntok[0])
+
+# Global map of prefixes that always get injected
+# into the serializers prefix map (note, that doesn't
+# mean they're always _USED_)
+G_PREFIXES = { "http://www.w3.org/XML/1998/namespace":"xml" }
+
+class _ListSerializer:
+    """ Internal class which serializes an Element tree into a buffer """
+    def __init__(self, prefixes=None, prefixesInScope=None):
+        self.writelist = []
+        self.prefixes = {}
+        if prefixes:
+            self.prefixes.update(prefixes)
+        self.prefixes.update(G_PREFIXES)
+        self.prefixStack = [G_PREFIXES.values()] + (prefixesInScope or [])
+        self.prefixCounter = 0
+
+    def getValue(self):
+        return u"".join(self.writelist)
+
+    def getPrefix(self, uri):
+        if uri not in self.prefixes:
+            self.prefixes[uri] = "xn%d" % (self.prefixCounter)
+            self.prefixCounter = self.prefixCounter + 1
+        return self.prefixes[uri]
+
+    def prefixInScope(self, prefix):
+        stack = self.prefixStack
+        for i in range(-1, (len(self.prefixStack)+1) * -1, -1):
+            if prefix in stack[i]:
+                return True
+        return False
+
+    def serialize(self, elem, closeElement=1, defaultUri=''):
+        # Optimization shortcuts
+        write = self.writelist.append
+
+        # Shortcut, check to see if elem is actually a chunk o' serialized XML
+        if isinstance(elem, SerializedXML):
+            write(elem)
+            return
+
+        # Shortcut, check to see if elem is actually a string (aka Cdata)
+        if isinstance(elem, StringType):
+            write(escapeToXml(elem))
+            return
+
+        # Further optimizations
+        name = elem.name
+        uri = elem.uri
+        defaultUri, currentDefaultUri = elem.defaultUri, defaultUri
+
+        for p, u in iteritems(elem.localPrefixes):
+            self.prefixes[u] = p
+        self.prefixStack.append(list(elem.localPrefixes.keys()))
+
+        # Inherit the default namespace
+        if defaultUri is None:
+            defaultUri = currentDefaultUri
+
+        if uri is None:
+            uri = defaultUri
+
+        prefix = None
+        if uri != defaultUri or uri in self.prefixes:
+            prefix = self.getPrefix(uri)
+            inScope = self.prefixInScope(prefix)
+
+        # Create the starttag
+
+        if not prefix:
+            write("<%s" % (name))
+        else:
+            write("<%s:%s" % (prefix, name))
+
+            if not inScope:
+                write(" xmlns:%s='%s'" % (prefix, uri))
+                self.prefixStack[-1].append(prefix)
+                inScope = True
+
+        if defaultUri != currentDefaultUri and \
+           (uri != defaultUri or not prefix or not inScope):
+            write(" xmlns='%s'" % (defaultUri))
+
+        for p, u in iteritems(elem.localPrefixes):
+            write(" xmlns:%s='%s'" % (p, u))
+
+        # Serialize attributes
+        for k,v in elem.attributes.items():
+            # If the attribute name is a tuple, it's a qualified attribute
+            if isinstance(k, tuple):
+                attr_uri, attr_name = k
+                attr_prefix = self.getPrefix(attr_uri)
+
+                if not self.prefixInScope(attr_prefix):
+                    write(" xmlns:%s='%s'" % (attr_prefix, attr_uri))
+                    self.prefixStack[-1].append(attr_prefix)
+
+                write(" %s:%s='%s'" % (attr_prefix, attr_name,
+                                       escapeToXml(v, 1)))
+            else:
+                write((" %s='%s'" % ( k, escapeToXml(v, 1))))
+
+        # Shortcut out if this is only going to return
+        # the element (i.e. no children)
+        if closeElement == 0:
+            write(">")
+            return
+
+        # Serialize children
+        if len(elem.children) > 0:
+            write(">")
+            for c in elem.children:
+                self.serialize(c, defaultUri=defaultUri)
+            # Add closing tag
+            if not prefix:
+                write("</%s>" % (name))
+            else:
+                write("</%s:%s>" % (prefix, name))
+        else:
+            write("/>")
+
+        self.prefixStack.pop()
+
+
+SerializerClass = _ListSerializer
+
+def escapeToXml(text, isattrib = 0):
+    """ Escape text to proper XML form, per section 2.3 in the XML specification.
+
+    @type text: C{str}
+    @param text: Text to escape
+
+    @type isattrib: C{bool}
+    @param isattrib: Triggers escaping of characters necessary for use as
+                     attribute values
+    """
+    text = text.replace("&", "&amp;")
+    text = text.replace("<", "&lt;")
+    text = text.replace(">", "&gt;")
+    if isattrib == 1:
+        text = text.replace("'", "&apos;")
+        text = text.replace("\"", "&quot;")
+    return text
+
+def unescapeFromXml(text):
+    text = text.replace("&lt;", "<")
+    text = text.replace("&gt;", ">")
+    text = text.replace("&apos;", "'")
+    text = text.replace("&quot;", "\"")
+    text = text.replace("&amp;", "&")
+    return text
+
+def generateOnlyInterface(list, int):
+    """ Filters items in a list by class
+    """
+    for n in list:
+        if int.providedBy(n):
+            yield n
+
+def generateElementsQNamed(list, name, uri):
+    """ Filters Element items in a list with matching name and URI. """
+    for n in list:
+        if IElement.providedBy(n) and n.name == name and n.uri == uri:
+            yield n
+
+def generateElementsNamed(list, name):
+    """ Filters Element items in a list with matching name, regardless of URI.
+    """
+    for n in list:
+        if IElement.providedBy(n) and n.name == name:
+            yield n
+
+
+class SerializedXML(unicode):
+    """ Marker class for pre-serialized XML in the DOM. """
+    pass
+
+
+class Namespace:
+    """ Convenience object for tracking namespace declarations. """
+    def __init__(self, uri):
+        self._uri = uri
+    def __getattr__(self, n):
+        return (self._uri, n)
+    def __getitem__(self, n):
+        return (self._uri, n)
+
+class IElement(Interface):
+    """
+    Interface to XML element nodes.
+
+    See L{Element} for a detailed example of its general use.
+
+    Warning: this Interface is not yet complete!
+    """
+
+    uri = Attribute(""" Element's namespace URI """)
+    name = Attribute(""" Element's local name """)
+    defaultUri = Attribute(""" Default namespace URI of child elements """)
+    attributes = Attribute(""" Dictionary of element attributes """)
+    children = Attribute(""" List of child nodes """)
+    parent = Attribute(""" Reference to element's parent element """)
+    localPrefixes = Attribute(""" Dictionary of local prefixes """)
+
+    def toXml(prefixes=None, closeElement=1, defaultUri='',
+              prefixesInScope=None):
+        """ Serializes object to a (partial) XML document
+
+        @param prefixes: dictionary that maps namespace URIs to suggested
+                         prefix names.
+        @type prefixes: L{dict}
+
+        @param closeElement: flag that determines whether to include the
+            closing tag of the element in the serialized string. A value of
+            C{0} only generates the element's start tag. A value of C{1} yields
+            a complete serialization.
+        @type closeElement: L{int}
+
+        @param defaultUri: Initial default namespace URI. This is most useful
+            for partial rendering, where the logical parent element (of which
+            the starttag was already serialized) declares a default namespace
+            that should be inherited.
+        @type defaultUri: L{unicode}
+
+        @param prefixesInScope: list of prefixes that are assumed to be
+            declared by ancestors.
+        @type prefixesInScope: C{list}
+
+        @return: (partial) serialized XML
+        @rtype: C{unicode}
+        """
+
+    def addElement(name, defaultUri=None, content=None):
+        """
+        Create an element and add as child.
+
+        The new element is added to this element as a child, and will have
+        this element as its parent.
+
+        @param name: element name. This can be either a L{unicode} object that
+            contains the local name, or a tuple of (uri, local_name) for a
+            fully qualified name. In the former case, the namespace URI is
+            inherited from this element.
+        @type name: L{unicode} or L{tuple} of (L{unicode}, L{unicode})
+
+        @param defaultUri: default namespace URI for child elements. If
+            L{None}, this is inherited from this element.
+        @type defaultUri: L{unicode}
+
+        @param content: text contained by the new element.
+        @type content: L{unicode}
+
+        @return: the created element
+        @rtype: object providing L{IElement}
+        """
+
+    def addChild(node):
+        """
+        Adds a node as child of this element.
+
+        The C{node} will be added to the list of childs of this element, and
+        will have this element set as its parent when C{node} provides
+        L{IElement}. If C{node} is a L{unicode} and the current last child is
+        character data (L{unicode}), the text from C{node} is appended to the
+        existing last child.
+
+        @param node: the child node.
+        @type node: L{unicode} or object implementing L{IElement}
+        """
+
+    def addContent(text):
+        """
+        Adds character data to this element.
+
+        If the current last child of this element is a string, the text will
+        be appended to that string. Otherwise, the text will be added as a new
+        child.
+
+        @param text: The character data to be added to this element.
+        @type text: L{unicode}
+        """
+
+
+@implementer(IElement)
+class Element(object):
+    """ Represents an XML element node.
+
+    An Element contains a series of attributes (name/value pairs), content
+    (character data), and other child Element objects. When building a document
+    with markup (such as HTML or XML), use this object as the starting point.
+
+    Element objects fully support XML Namespaces. The fully qualified name of
+    the XML Element it represents is stored in the C{uri} and C{name}
+    attributes, where C{uri} holds the namespace URI. There is also a default
+    namespace, for child elements. This is stored in the C{defaultUri}
+    attribute. Note that C{''} means the empty namespace.
+
+    Serialization of Elements through C{toXml()} will use these attributes
+    for generating proper serialized XML. When both C{uri} and C{defaultUri}
+    are not None in the Element and all of its descendents, serialization
+    proceeds as expected:
+
+      >>> from twisted.words.xish import domish
+      >>> root = domish.Element(('myns', 'root'))
+      >>> root.addElement('child', content='test')
+      <twisted.words.xish.domish.Element object at 0x83002ac>
+      >>> root.toXml()
+      u"<root xmlns='myns'><child>test</child></root>"
+
+    For partial serialization, needed for streaming XML, a special value for
+    namespace URIs can be used: L{None}.
+
+    Using L{None} as the value for C{uri} means: this element is in whatever
+    namespace inherited by the closest logical ancestor when the complete XML
+    document has been serialized. The serialized start tag will have a
+    non-prefixed name, and no xmlns declaration will be generated.
+
+    Similarly, L{None} for C{defaultUri} means: the default namespace for my
+    child elements is inherited from the logical ancestors of this element,
+    when the complete XML document has been serialized.
+
+    To illustrate, an example from a Jabber stream. Assume the start tag of the
+    root element of the stream has already been serialized, along with several
+    complete child elements, and sent off, looking like this::
+
+      <stream:stream xmlns:stream='http://etherx.jabber.org/streams'
+                     xmlns='jabber:client' to='example.com'>
+        ...
+
+    Now suppose we want to send a complete element represented by an
+    object C{message} created like:
+
+      >>> message = domish.Element((None, 'message'))
+      >>> message['to'] = 'user@example.com'
+      >>> message.addElement('body', content='Hi!')
+      <twisted.words.xish.domish.Element object at 0x8276e8c>
+      >>> message.toXml()
+      u"<message to='user@example.com'><body>Hi!</body></message>"
+
+    As, you can see, this XML snippet has no xmlns declaration. When sent
+    off, it inherits the C{jabber:client} namespace from the root element.
+    Note that this renders the same as using C{''} instead of L{None}:
+
+      >>> presence = domish.Element(('', 'presence'))
+      >>> presence.toXml()
+      u"<presence/>"
+
+    However, if this object has a parent defined, the difference becomes
+    clear:
+
+      >>> child = message.addElement(('http://example.com/', 'envelope'))
+      >>> child.addChild(presence)
+      <twisted.words.xish.domish.Element object at 0x8276fac>
+      >>> message.toXml()
+      u"<message to='user@example.com'><body>Hi!</body><envelope xmlns='http://example.com/'><presence xmlns=''/></envelope></message>"
+
+    As, you can see, the <presence/> element is now in the empty namespace, not
+    in the default namespace of the parent or the streams'.
+
+    @type uri: C{unicode} or None
+    @ivar uri: URI of this Element's name
+
+    @type name: C{unicode}
+    @ivar name: Name of this Element
+
+    @type defaultUri: C{unicode} or None
+    @ivar defaultUri: URI this Element exists within
+
+    @type children: C{list}
+    @ivar children: List of child Elements and content
+
+    @type parent: L{Element}
+    @ivar parent: Reference to the parent Element, if any.
+
+    @type attributes: L{dict}
+    @ivar attributes: Dictionary of attributes associated with this Element.
+
+    @type localPrefixes: L{dict}
+    @ivar localPrefixes: Dictionary of namespace declarations on this
+                         element. The key is the prefix to bind the
+                         namespace uri to.
+    """
+
+    _idCounter = 0
+
+    def __init__(self, qname, defaultUri=None, attribs=None,
+                       localPrefixes=None):
+        """
+        @param qname: Tuple of (uri, name)
+        @param defaultUri: The default URI of the element; defaults to the URI
+                           specified in C{qname}
+        @param attribs: Dictionary of attributes
+        @param localPrefixes: Dictionary of namespace declarations on this
+                              element. The key is the prefix to bind the
+                              namespace uri to.
+        """
+        self.localPrefixes = localPrefixes or {}
+        self.uri, self.name = qname
+        if defaultUri is None and \
+           self.uri not in itervalues(self.localPrefixes):
+            self.defaultUri = self.uri
+        else:
+            self.defaultUri = defaultUri
+        self.attributes = attribs or {}
+        self.children = []
+        self.parent = None
+
+    def __getattr__(self, key):
+        # Check child list for first Element with a name matching the key
+        for n in self.children:
+            if IElement.providedBy(n) and n.name == key:
+                return n
+
+        # Tweak the behaviour so that it's more friendly about not
+        # finding elements -- we need to document this somewhere :)
+        if key.startswith('_'):
+            raise AttributeError(key)
+        else:
+            return None
+
+    def __getitem__(self, key):
+        return self.attributes[self._dqa(key)]
+
+    def __delitem__(self, key):
+        del self.attributes[self._dqa(key)];
+
+    def __setitem__(self, key, value):
+        self.attributes[self._dqa(key)] = value
+
+    def __unicode__(self):
+        """
+        Retrieve the first CData (content) node
+        """
+        for n in self.children:
+            if isinstance(n, StringType):
+                return n
+        return u""
+
+    def __bytes__(self):
+        """
+        Retrieve the first character data node as UTF-8 bytes.
+        """
+        return unicode(self).encode('utf-8')
+
+    if _PY3:
+        __str__ = __unicode__
+    else:
+        __str__ = __bytes__
+
+    def _dqa(self, attr):
+        """ Dequalify an attribute key as needed """
+        if isinstance(attr, tuple) and not attr[0]:
+            return attr[1]
+        else:
+            return attr
+
+    def getAttribute(self, attribname, default = None):
+        """ Retrieve the value of attribname, if it exists """
+        return self.attributes.get(attribname, default)
+
+    def hasAttribute(self, attrib):
+        """ Determine if the specified attribute exists """
+        return self._dqa(attrib) in self.attributes
+
+    def compareAttribute(self, attrib, value):
+        """ Safely compare the value of an attribute against a provided value.
+
+        L{None}-safe.
+        """
+        return self.attributes.get(self._dqa(attrib), None) == value
+
+    def swapAttributeValues(self, left, right):
+        """ Swap the values of two attribute. """
+        d = self.attributes
+        l = d[left]
+        d[left] = d[right]
+        d[right] = l
+
+    def addChild(self, node):
+        """ Add a child to this Element. """
+        if IElement.providedBy(node):
+            node.parent = self
+        self.children.append(node)
+        return node
+
+    def addContent(self, text):
+        """ Add some text data to this Element. """
+        text = _coercedUnicode(text)
+        c = self.children
+        if len(c) > 0 and isinstance(c[-1], unicode):
+            c[-1] = c[-1] + text
+        else:
+            c.append(text)
+        return c[-1]
+
+    def addElement(self, name, defaultUri = None, content = None):
+        if isinstance(name, tuple):
+            if defaultUri is None:
+                defaultUri = name[0]
+            child = Element(name, defaultUri)
+        else:
+            if defaultUri is None:
+                defaultUri = self.defaultUri
+            child = Element((defaultUri, name), defaultUri)
+
+        self.addChild(child)
+
+        if content:
+            child.addContent(content)
+
+        return child
+
+    def addRawXml(self, rawxmlstring):
+        """ Add a pre-serialized chunk o' XML as a child of this Element. """
+        self.children.append(SerializedXML(rawxmlstring))
+
+    def addUniqueId(self):
+        """ Add a unique (across a given Python session) id attribute to this
+            Element.
+        """
+        self.attributes["id"] = "H_%d" % Element._idCounter
+        Element._idCounter = Element._idCounter + 1
+
+
+    def elements(self, uri=None, name=None):
+        """
+        Iterate across all children of this Element that are Elements.
+
+        Returns a generator over the child elements. If both the C{uri} and
+        C{name} parameters are set, the returned generator will only yield
+        on elements matching the qualified name.
+
+        @param uri: Optional element URI.
+        @type uri: C{unicode}
+        @param name: Optional element name.
+        @type name: C{unicode}
+        @return: Iterator that yields objects implementing L{IElement}.
+        """
+        if name is None:
+            return generateOnlyInterface(self.children, IElement)
+        else:
+            return generateElementsQNamed(self.children, name, uri)
+
+
+    def toXml(self, prefixes=None, closeElement=1, defaultUri='',
+                    prefixesInScope=None):
+        """ Serialize this Element and all children to a string. """
+        s = SerializerClass(prefixes=prefixes, prefixesInScope=prefixesInScope)
+        s.serialize(self, closeElement=closeElement, defaultUri=defaultUri)
+        return s.getValue()
+
+    def firstChildElement(self):
+        for c in self.children:
+            if IElement.providedBy(c):
+                return c
+        return None
+
+
+class ParserError(Exception):
+    """ Exception thrown when a parsing error occurs """
+    pass
+
+def elementStream():
+    """ Preferred method to construct an ElementStream
+
+    Uses Expat-based stream if available, and falls back to Sux if necessary.
+    """
+    try:
+        es = ExpatElementStream()
+        return es
+    except ImportError:
+        if SuxElementStream is None:
+            raise Exception("No parsers available :(")
+        es = SuxElementStream()
+        return es
+
+try:
+    from twisted.web import sux
+except:
+    SuxElementStream = None
+else:
+    class SuxElementStream(sux.XMLParser):
+        def __init__(self):
+            self.connectionMade()
+            self.DocumentStartEvent = None
+            self.ElementEvent = None
+            self.DocumentEndEvent = None
+            self.currElem = None
+            self.rootElem = None
+            self.documentStarted = False
+            self.defaultNsStack = []
+            self.prefixStack = []
+
+        def parse(self, buffer):
+            try:
+                self.dataReceived(buffer)
+            except sux.ParseError as e:
+                raise ParserError(str(e))
+
+
+        def findUri(self, prefix):
+            # Walk prefix stack backwards, looking for the uri
+            # matching the specified prefix
+            stack = self.prefixStack
+            for i in range(-1, (len(self.prefixStack)+1) * -1, -1):
+                if prefix in stack[i]:
+                    return stack[i][prefix]
+            return None
+
+        def gotTagStart(self, name, attributes):
+            defaultUri = None
+            localPrefixes = {}
+            attribs = {}
+            uri = None
+
+            # Pass 1 - Identify namespace decls
+            for k, v in list(attributes.items()):
+                if k.startswith("xmlns"):
+                    x, p = _splitPrefix(k)
+                    if (x is None): # I.e.  default declaration
+                        defaultUri = v
+                    else:
+                        localPrefixes[p] = v
+                    del attributes[k]
+
+            # Push namespace decls onto prefix stack
+            self.prefixStack.append(localPrefixes)
+
+            # Determine default namespace for this element; if there
+            # is one
+            if defaultUri is None:
+                if len(self.defaultNsStack) > 0:
+                    defaultUri = self.defaultNsStack[-1]
+                else:
+                    defaultUri = ''
+
+            # Fix up name
+            prefix, name = _splitPrefix(name)
+            if prefix is None: # This element is in the default namespace
+                uri = defaultUri
+            else:
+                # Find the URI for the prefix
+                uri = self.findUri(prefix)
+
+            # Pass 2 - Fix up and escape attributes
+            for k, v in attributes.items():
+                p, n = _splitPrefix(k)
+                if p is None:
+                    attribs[n] = v
+                else:
+                    attribs[(self.findUri(p)), n] = unescapeFromXml(v)
+
+            # Construct the actual Element object
+            e = Element((uri, name), defaultUri, attribs, localPrefixes)
+
+            # Save current default namespace
+            self.defaultNsStack.append(defaultUri)
+
+            # Document already started
+            if self.documentStarted:
+                # Starting a new packet
+                if self.currElem is None:
+                    self.currElem = e
+                # Adding to existing element
+                else:
+                    self.currElem = self.currElem.addChild(e)
+            # New document
+            else:
+                self.rootElem = e
+                self.documentStarted = True
+                self.DocumentStartEvent(e)
+
+        def gotText(self, data):
+            if self.currElem != None:
+                if isinstance(data, bytes):
+                    data = data.decode('ascii')
+                self.currElem.addContent(data)
+
+        def gotCData(self, data):
+            if self.currElem != None:
+                if isinstance(data, bytes):
+                    data = data.decode('ascii')
+                self.currElem.addContent(data)
+
+        def gotComment(self, data):
+            # Ignore comments for the moment
+            pass
+
+        entities = { "amp" : "&",
+                     "lt"  : "<",
+                     "gt"  : ">",
+                     "apos": "'",
+                     "quot": "\"" }
+
+        def gotEntityReference(self, entityRef):
+            # If this is an entity we know about, add it as content
+            # to the current element
+            if entityRef in SuxElementStream.entities:
+                data = SuxElementStream.entities[entityRef]
+                if isinstance(data, bytes):
+                    data = data.decode('ascii')
+                self.currElem.addContent(data)
+
+        def gotTagEnd(self, name):
+            # Ensure the document hasn't already ended
+            if self.rootElem is None:
+                # XXX: Write more legible explanation
+                raise ParserError("Element closed after end of document.")
+
+            # Fix up name
+            prefix, name = _splitPrefix(name)
+            if prefix is None:
+                uri = self.defaultNsStack[-1]
+            else:
+                uri = self.findUri(prefix)
+
+            # End of document
+            if self.currElem is None:
+                # Ensure element name and uri matches
+                if self.rootElem.name != name or self.rootElem.uri != uri:
+                    raise ParserError("Mismatched root elements")
+                self.DocumentEndEvent()
+                self.rootElem = None
+
+            # Other elements
+            else:
+                # Ensure the tag being closed matches the name of the current
+                # element
+                if self.currElem.name != name or self.currElem.uri != uri:
+                    # XXX: Write more legible explanation
+                    raise ParserError("Malformed element close")
+
+                # Pop prefix and default NS stack
+                self.prefixStack.pop()
+                self.defaultNsStack.pop()
+
+                # Check for parent null parent of current elem;
+                # that's the top of the stack
+                if self.currElem.parent is None:
+                    self.currElem.parent = self.rootElem
+                    self.ElementEvent(self.currElem)
+                    self.currElem = None
+
+                # Anything else is just some element wrapping up
+                else:
+                    self.currElem = self.currElem.parent
+
+
+class ExpatElementStream:
+    def __init__(self):
+        import pyexpat
+        self.DocumentStartEvent = None
+        self.ElementEvent = None
+        self.DocumentEndEvent = None
+        self.error = pyexpat.error
+        self.parser = pyexpat.ParserCreate("UTF-8", " ")
+        self.parser.StartElementHandler = self._onStartElement
+        self.parser.EndElementHandler = self._onEndElement
+        self.parser.CharacterDataHandler = self._onCdata
+        self.parser.StartNamespaceDeclHandler = self._onStartNamespace
+        self.parser.EndNamespaceDeclHandler = self._onEndNamespace
+        self.currElem = None
+        self.defaultNsStack = ['']
+        self.documentStarted = 0
+        self.localPrefixes = {}
+
+    def parse(self, buffer):
+        try:
+            self.parser.Parse(buffer)
+        except self.error as e:
+            raise ParserError(str(e))
+
+    def _onStartElement(self, name, attrs):
+        # Generate a qname tuple from the provided name.  See
+        # http://docs.python.org/library/pyexpat.html#xml.parsers.expat.ParserCreate
+        # for an explanation of the formatting of name.
+        qname = name.rsplit(" ", 1)
+        if len(qname) == 1:
+            qname = ('', name)
+
+        # Process attributes
+        newAttrs = {}
+        toDelete = []
+        for k, v in attrs.items():
+            if " " in k:
+                aqname = k.rsplit(" ", 1)
+                newAttrs[(aqname[0], aqname[1])] = v
+                toDelete.append(k)
+
+        attrs.update(newAttrs)
+
+        for k in toDelete:
+            del attrs[k]
+
+        # Construct the new element
+        e = Element(qname, self.defaultNsStack[-1], attrs, self.localPrefixes)
+        self.localPrefixes = {}
+
+        # Document already started
+        if self.documentStarted == 1:
+            if self.currElem != None:
+                self.currElem.children.append(e)
+                e.parent = self.currElem
+            self.currElem = e
+
+        # New document
+        else:
+            self.documentStarted = 1
+            self.DocumentStartEvent(e)
+
+    def _onEndElement(self, _):
+        # Check for null current elem; end of doc
+        if self.currElem is None:
+            self.DocumentEndEvent()
+
+        # Check for parent that is None; that's
+        # the top of the stack
+        elif self.currElem.parent is None:
+            self.ElementEvent(self.currElem)
+            self.currElem = None
+
+        # Anything else is just some element in the current
+        # packet wrapping up
+        else:
+            self.currElem = self.currElem.parent
+
+    def _onCdata(self, data):
+        if self.currElem != None:
+            self.currElem.addContent(data)
+
+    def _onStartNamespace(self, prefix, uri):
+        # If this is the default namespace, put
+        # it on the stack
+        if prefix is None:
+            self.defaultNsStack.append(uri)
+        else:
+            self.localPrefixes[prefix] = uri
+
+    def _onEndNamespace(self, prefix):
+        # Remove last element on the stack
+        if prefix is None:
+            self.defaultNsStack.pop()
+
+## class FileParser(ElementStream):
+##     def __init__(self):
+##         ElementStream.__init__(self)
+##         self.DocumentStartEvent = self.docStart
+##         self.ElementEvent = self.elem
+##         self.DocumentEndEvent = self.docEnd
+##         self.done = 0
+
+##     def docStart(self, elem):
+##         self.document = elem
+
+##     def elem(self, elem):
+##         self.document.addChild(elem)
+
+##     def docEnd(self):
+##         self.done = 1
+
+##     def parse(self, filename):
+##         with open(filename) as f:
+##             for l in f.readlines():
+##                 self.parser.Parse(l)
+##         assert self.done == 1
+##         return self.document
+
+## def parseFile(filename):
+##     return FileParser().parse(filename)
+
+
diff --git a/contrib/python/Twisted/py2/twisted/words/xish/utility.py b/contrib/python/Twisted/py2/twisted/words/xish/utility.py
new file mode 100644
index 00000000000..6f8a11527da
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/xish/utility.py
@@ -0,0 +1,375 @@
+# -*- test-case-name: twisted.words.test.test_xishutil -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Event Dispatching and Callback utilities.
+"""
+
+from __future__ import absolute_import, division
+
+from twisted.python import log
+from twisted.python.compat import iteritems
+from twisted.words.xish import xpath
+
+class _MethodWrapper(object):
+    """
+    Internal class for tracking method calls.
+    """
+    def __init__(self, method, *args, **kwargs):
+        self.method = method
+        self.args = args
+        self.kwargs = kwargs
+
+
+    def __call__(self, *args, **kwargs):
+        nargs = self.args + args
+        nkwargs = self.kwargs.copy()
+        nkwargs.update(kwargs)
+        self.method(*nargs, **nkwargs)
+
+
+
+class CallbackList:
+    """
+    Container for callbacks.
+
+    Event queries are linked to lists of callables. When a matching event
+    occurs, these callables are called in sequence. One-time callbacks
+    are removed from the list after the first time the event was triggered.
+
+    Arguments to callbacks are split spread across two sets. The first set,
+    callback specific, is passed to C{addCallback} and is used for all
+    subsequent event triggers.  The second set is passed to C{callback} and is
+    event specific. Positional arguments in the second set come after the
+    positional arguments of the first set. Keyword arguments in the second set
+    override those in the first set.
+
+    @ivar callbacks: The registered callbacks as mapping from the callable to a
+                     tuple of a wrapper for that callable that keeps the
+                     callback specific arguments and a boolean that signifies
+                     if it is to be called only once.
+    @type callbacks: C{dict}
+    """
+
+    def __init__(self):
+        self.callbacks = {}
+
+
+    def addCallback(self, onetime, method, *args, **kwargs):
+        """
+        Add callback.
+
+        The arguments passed are used as callback specific arguments.
+
+        @param onetime: If C{True}, this callback is called at most once.
+        @type onetime: C{bool}
+        @param method: The callback callable to be added.
+        @param args: Positional arguments to the callable.
+        @type args: C{list}
+        @param kwargs: Keyword arguments to the callable.
+        @type kwargs: C{dict}
+        """
+
+        if not method in self.callbacks:
+            self.callbacks[method] = (_MethodWrapper(method, *args, **kwargs),
+                                      onetime)
+
+
+    def removeCallback(self, method):
+        """
+        Remove callback.
+
+        @param method: The callable to be removed.
+        """
+
+        if method in self.callbacks:
+            del self.callbacks[method]
+
+
+    def callback(self, *args, **kwargs):
+        """
+        Call all registered callbacks.
+
+        The passed arguments are event specific and augment and override
+        the callback specific arguments as described above.
+
+        @note: Exceptions raised by callbacks are trapped and logged. They will
+               not propagate up to make sure other callbacks will still be
+               called, and the event dispatching always succeeds.
+
+        @param args: Positional arguments to the callable.
+        @type args: C{list}
+        @param kwargs: Keyword arguments to the callable.
+        @type kwargs: C{dict}
+        """
+
+        for key, (methodwrapper, onetime) in list(self.callbacks.items()):
+            try:
+                methodwrapper(*args, **kwargs)
+            except:
+                log.err()
+
+            if onetime:
+                del self.callbacks[key]
+
+
+    def isEmpty(self):
+        """
+        Return if list of registered callbacks is empty.
+
+        @rtype: C{bool}
+        """
+
+        return len(self.callbacks) == 0
+
+
+
+class EventDispatcher:
+    """
+    Event dispatching service.
+
+    The C{EventDispatcher} allows observers to be registered for certain events
+    that are dispatched. There are two types of events: XPath events and Named
+    events.
+
+    Every dispatch is triggered by calling L{dispatch} with a data object and,
+    for named events, the name of the event.
+
+    When an XPath type event is dispatched, the associated object is assumed to
+    be an L{Element<twisted.words.xish.domish.Element>} instance, which is
+    matched against all registered XPath queries. For every match, the
+    respective observer will be called with the data object.
+
+    A named event will simply call each registered observer for that particular
+    event name, with the data object. Unlike XPath type events, the data object
+    is not restricted to L{Element<twisted.words.xish.domish.Element>}, but can
+    be anything.
+
+    When registering observers, the event that is to be observed is specified
+    using an L{xpath.XPathQuery} instance or a string. In the latter case, the
+    string can also contain the string representation of an XPath expression.
+    To distinguish these from named events, each named event should start with
+    a special prefix that is stored in C{self.prefix}. It defaults to
+    C{//event/}.
+
+    Observers registered using L{addObserver} are persistent: after the
+    observer has been triggered by a dispatch, it remains registered for a
+    possible next dispatch. If instead L{addOnetimeObserver} was used to
+    observe an event, the observer is removed from the list of observers after
+    the first observed event.
+
+    Observers can also be prioritized, by providing an optional C{priority}
+    parameter to the L{addObserver} and L{addOnetimeObserver} methods. Higher
+    priority observers are then called before lower priority observers.
+
+    Finally, observers can be unregistered by using L{removeObserver}.
+    """
+
+    def __init__(self, eventprefix="//event/"):
+        self.prefix = eventprefix
+        self._eventObservers = {}
+        self._xpathObservers = {}
+        self._dispatchDepth = 0  # Flag indicating levels of dispatching
+                                 # in progress
+        self._updateQueue = [] # Queued updates for observer ops
+
+
+    def _getEventAndObservers(self, event):
+        if isinstance(event, xpath.XPathQuery):
+            # Treat as xpath
+            observers = self._xpathObservers
+        else:
+            if self.prefix == event[:len(self.prefix)]:
+                # Treat as event
+                observers = self._eventObservers
+            else:
+                # Treat as xpath
+                event = xpath.internQuery(event)
+                observers = self._xpathObservers
+
+        return event, observers
+
+
+    def addOnetimeObserver(self, event, observerfn, priority=0, *args, **kwargs):
+        """
+        Register a one-time observer for an event.
+
+        Like L{addObserver}, but is only triggered at most once. See there
+        for a description of the parameters.
+        """
+        self._addObserver(True, event, observerfn, priority, *args, **kwargs)
+
+
+    def addObserver(self, event, observerfn, priority=0, *args, **kwargs):
+        """
+        Register an observer for an event.
+
+        Each observer will be registered with a certain priority. Higher
+        priority observers get called before lower priority observers.
+
+        @param event: Name or XPath query for the event to be monitored.
+        @type event: C{str} or L{xpath.XPathQuery}.
+        @param observerfn: Function to be called when the specified event
+                           has been triggered. This callable takes
+                           one parameter: the data object that triggered
+                           the event. When specified, the C{*args} and
+                           C{**kwargs} parameters to addObserver are being used
+                           as additional parameters to the registered observer
+                           callable.
+        @param priority: (Optional) priority of this observer in relation to
+                         other observer that match the same event. Defaults to
+                         C{0}.
+        @type priority: C{int}
+        """
+        self._addObserver(False, event, observerfn, priority, *args, **kwargs)
+
+
+    def _addObserver(self, onetime, event, observerfn, priority, *args, **kwargs):
+        # If this is happening in the middle of the dispatch, queue
+        # it up for processing after the dispatch completes
+        if self._dispatchDepth > 0:
+            self._updateQueue.append(lambda:self._addObserver(onetime, event, observerfn, priority, *args, **kwargs))
+            return
+
+        event, observers = self._getEventAndObservers(event)
+
+        if priority not in observers:
+            cbl = CallbackList()
+            observers[priority] = {event: cbl}
+        else:
+            priorityObservers = observers[priority]
+            if event not in priorityObservers:
+                cbl = CallbackList()
+                observers[priority][event] = cbl
+            else:
+                cbl = priorityObservers[event]
+
+        cbl.addCallback(onetime, observerfn, *args, **kwargs)
+
+
+    def removeObserver(self, event, observerfn):
+        """
+        Remove callable as observer for an event.
+
+        The observer callable is removed for all priority levels for the
+        specified event.
+
+        @param event: Event for which the observer callable was registered.
+        @type event: C{str} or L{xpath.XPathQuery}
+        @param observerfn: Observer callable to be unregistered.
+        """
+
+        # If this is happening in the middle of the dispatch, queue
+        # it up for processing after the dispatch completes
+        if self._dispatchDepth > 0:
+            self._updateQueue.append(lambda:self.removeObserver(event, observerfn))
+            return
+
+        event, observers = self._getEventAndObservers(event)
+
+        emptyLists = []
+        for priority, priorityObservers in iteritems(observers):
+            for query, callbacklist in iteritems(priorityObservers):
+                if event == query:
+                    callbacklist.removeCallback(observerfn)
+                    if callbacklist.isEmpty():
+                        emptyLists.append((priority, query))
+
+        for priority, query in emptyLists:
+            del observers[priority][query]
+
+
+    def dispatch(self, obj, event=None):
+        """
+        Dispatch an event.
+
+        When C{event} is L{None}, an XPath type event is triggered, and
+        C{obj} is assumed to be an instance of
+        L{Element<twisted.words.xish.domish.Element>}. Otherwise, C{event}
+        holds the name of the named event being triggered. In the latter case,
+        C{obj} can be anything.
+
+        @param obj: The object to be dispatched.
+        @param event: Optional event name.
+        @type event: C{str}
+        """
+
+        foundTarget = False
+
+        self._dispatchDepth += 1
+
+        if event != None:
+            # Named event
+            observers = self._eventObservers
+            match = lambda query, obj: query == event
+        else:
+            # XPath event
+            observers = self._xpathObservers
+            match = lambda query, obj: query.matches(obj)
+
+        priorities = list(observers.keys())
+        priorities.sort()
+        priorities.reverse()
+
+        emptyLists = []
+        for priority in priorities:
+            for query, callbacklist in iteritems(observers[priority]):
+                if match(query, obj):
+                    callbacklist.callback(obj)
+                    foundTarget = True
+                    if callbacklist.isEmpty():
+                        emptyLists.append((priority, query))
+
+        for priority, query in emptyLists:
+            del observers[priority][query]
+
+        self._dispatchDepth -= 1
+
+        # If this is a dispatch within a dispatch, don't
+        # do anything with the updateQueue -- it needs to
+        # wait until we've back all the way out of the stack
+        if self._dispatchDepth == 0:
+            # Deal with pending update operations
+            for f in self._updateQueue:
+                f()
+            self._updateQueue = []
+
+        return foundTarget
+
+
+
+class XmlPipe(object):
+    """
+    XML stream pipe.
+
+    Connects two objects that communicate stanzas through an XML stream like
+    interface. Each of the ends of the pipe (sink and source) can be used to
+    send XML stanzas to the other side, or add observers to process XML stanzas
+    that were sent from the other side.
+
+    XML pipes are usually used in place of regular XML streams that are
+    transported over TCP. This is the reason for the use of the names source
+    and sink for both ends of the pipe. The source side corresponds with the
+    entity that initiated the TCP connection, whereas the sink corresponds with
+    the entity that accepts that connection. In this object, though, the source
+    and sink are treated equally.
+
+    Unlike Jabber
+    L{XmlStream<twisted.words.protocols.jabber.xmlstream.XmlStream>}s, the sink
+    and source objects are assumed to represent an eternal connected and
+    initialized XML stream. As such, events corresponding to connection,
+    disconnection, initialization and stream errors are not dispatched or
+    processed.
+
+    @since: 8.2
+    @ivar source: Source XML stream.
+    @ivar sink: Sink XML stream.
+    """
+
+    def __init__(self):
+        self.source = EventDispatcher()
+        self.sink = EventDispatcher()
+        self.source.send = lambda obj: self.sink.dispatch(obj)
+        self.sink.send = lambda obj: self.source.dispatch(obj)
diff --git a/contrib/python/Twisted/py2/twisted/words/xish/xmlstream.py b/contrib/python/Twisted/py2/twisted/words/xish/xmlstream.py
new file mode 100644
index 00000000000..e3f09c53da5
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/xish/xmlstream.py
@@ -0,0 +1,279 @@
+# -*- test-case-name: twisted.words.test.test_xmlstream -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XML Stream processing.
+
+An XML Stream is defined as a connection over which two XML documents are
+exchanged during the lifetime of the connection, one for each direction. The
+unit of interaction is a direct child element of the root element (stanza).
+
+The most prominent use of XML Streams is Jabber, but this module is generically
+usable. See Twisted Words for Jabber specific protocol support.
+
+Maintainer: Ralph Meijer
+
+@var STREAM_CONNECTED_EVENT: This event signals that the connection has been
+    established.
+@type STREAM_CONNECTED_EVENT: L{str}.
+
+@var STREAM_END_EVENT: This event signals that the connection has been closed.
+@type STREAM_END_EVENT: L{str}.
+
+@var STREAM_ERROR_EVENT: This event signals that a parse error occurred.
+@type STREAM_ERROR_EVENT: L{str}.
+
+@var STREAM_START_EVENT: This event signals that the root element of the XML
+    Stream has been received.
+    For XMPP, this would be the C{<stream:stream ...>} opening tag.
+@type STREAM_START_EVENT: L{str}.
+"""
+
+from __future__ import absolute_import, division
+
+from twisted.python import failure
+from twisted.python.compat import intern, unicode
+from twisted.internet import protocol
+from twisted.words.xish import domish, utility
+
+STREAM_CONNECTED_EVENT = intern("//event/stream/connected")
+STREAM_START_EVENT = intern("//event/stream/start")
+STREAM_END_EVENT = intern("//event/stream/end")
+STREAM_ERROR_EVENT = intern("//event/stream/error")
+
+class XmlStream(protocol.Protocol, utility.EventDispatcher):
+    """ Generic Streaming XML protocol handler.
+
+    This protocol handler will parse incoming data as XML and dispatch events
+    accordingly. Incoming stanzas can be handled by registering observers using
+    XPath-like expressions that are matched against each stanza. See
+    L{utility.EventDispatcher} for details.
+    """
+    def __init__(self):
+        utility.EventDispatcher.__init__(self)
+        self.stream = None
+        self.rawDataOutFn = None
+        self.rawDataInFn = None
+
+    def _initializeStream(self):
+        """ Sets up XML Parser. """
+        self.stream = domish.elementStream()
+        self.stream.DocumentStartEvent = self.onDocumentStart
+        self.stream.ElementEvent = self.onElement
+        self.stream.DocumentEndEvent = self.onDocumentEnd
+
+    ### --------------------------------------------------------------
+    ###
+    ### Protocol events
+    ###
+    ### --------------------------------------------------------------
+
+    def connectionMade(self):
+        """ Called when a connection is made.
+
+        Sets up the XML parser and dispatches the L{STREAM_CONNECTED_EVENT}
+        event indicating the connection has been established.
+        """
+        self._initializeStream()
+        self.dispatch(self, STREAM_CONNECTED_EVENT)
+
+    def dataReceived(self, data):
+        """ Called whenever data is received.
+
+        Passes the data to the XML parser. This can result in calls to the
+        DOM handlers. If a parse error occurs, the L{STREAM_ERROR_EVENT} event
+        is called to allow for cleanup actions, followed by dropping the
+        connection.
+        """
+        try:
+            if self.rawDataInFn:
+                self.rawDataInFn(data)
+            self.stream.parse(data)
+        except domish.ParserError:
+            self.dispatch(failure.Failure(), STREAM_ERROR_EVENT)
+            self.transport.loseConnection()
+
+    def connectionLost(self, reason):
+        """ Called when the connection is shut down.
+
+        Dispatches the L{STREAM_END_EVENT}.
+        """
+        self.dispatch(reason, STREAM_END_EVENT)
+        self.stream = None
+
+    ### --------------------------------------------------------------
+    ###
+    ### DOM events
+    ###
+    ### --------------------------------------------------------------
+
+    def onDocumentStart(self, rootElement):
+        """ Called whenever the start tag of a root element has been received.
+
+        Dispatches the L{STREAM_START_EVENT}.
+        """
+        self.dispatch(self, STREAM_START_EVENT)
+
+    def onElement(self, element):
+        """ Called whenever a direct child element of the root element has
+        been received.
+
+        Dispatches the received element.
+        """
+        self.dispatch(element)
+
+    def onDocumentEnd(self):
+        """ Called whenever the end tag of the root element has been received.
+
+        Closes the connection. This causes C{connectionLost} being called.
+        """
+        self.transport.loseConnection()
+
+    def setDispatchFn(self, fn):
+        """ Set another function to handle elements. """
+        self.stream.ElementEvent = fn
+
+    def resetDispatchFn(self):
+        """ Set the default function (C{onElement}) to handle elements. """
+        self.stream.ElementEvent = self.onElement
+
+    def send(self, obj):
+        """ Send data over the stream.
+
+        Sends the given C{obj} over the connection. C{obj} may be instances of
+        L{domish.Element}, C{unicode} and C{str}. The first two will be
+        properly serialized and/or encoded. C{str} objects must be in UTF-8
+        encoding.
+
+        Note: because it is easy to make mistakes in maintaining a properly
+        encoded C{str} object, it is advised to use C{unicode} objects
+        everywhere when dealing with XML Streams.
+
+        @param obj: Object to be sent over the stream.
+        @type obj: L{domish.Element}, L{domish} or C{str}
+
+        """
+        if domish.IElement.providedBy(obj):
+            obj = obj.toXml()
+
+        if isinstance(obj, unicode):
+            obj = obj.encode('utf-8')
+
+        if self.rawDataOutFn:
+            self.rawDataOutFn(obj)
+
+        self.transport.write(obj)
+
+
+
+class BootstrapMixin(object):
+    """
+    XmlStream factory mixin to install bootstrap event observers.
+
+    This mixin is for factories providing
+    L{IProtocolFactory<twisted.internet.interfaces.IProtocolFactory>} to make
+    sure bootstrap event observers are set up on protocols, before incoming
+    data is processed. Such protocols typically derive from
+    L{utility.EventDispatcher}, like L{XmlStream}.
+
+    You can set up bootstrap event observers using C{addBootstrap}. The
+    C{event} and C{fn} parameters correspond with the C{event} and
+    C{observerfn} arguments to L{utility.EventDispatcher.addObserver}.
+
+    @since: 8.2.
+    @ivar bootstraps: The list of registered bootstrap event observers.
+    @type bootstrap: C{list}
+    """
+
+    def __init__(self):
+        self.bootstraps = []
+
+
+    def installBootstraps(self, dispatcher):
+        """
+        Install registered bootstrap observers.
+
+        @param dispatcher: Event dispatcher to add the observers to.
+        @type dispatcher: L{utility.EventDispatcher}
+        """
+        for event, fn in self.bootstraps:
+            dispatcher.addObserver(event, fn)
+
+
+    def addBootstrap(self, event, fn):
+        """
+        Add a bootstrap event handler.
+
+        @param event: The event to register an observer for.
+        @type event: C{str} or L{xpath.XPathQuery}
+        @param fn: The observer callable to be registered.
+        """
+        self.bootstraps.append((event, fn))
+
+
+    def removeBootstrap(self, event, fn):
+        """
+        Remove a bootstrap event handler.
+
+        @param event: The event the observer is registered for.
+        @type event: C{str} or L{xpath.XPathQuery}
+        @param fn: The registered observer callable.
+        """
+        self.bootstraps.remove((event, fn))
+
+
+
+class XmlStreamFactoryMixin(BootstrapMixin):
+    """
+    XmlStream factory mixin that takes care of event handlers.
+
+    All positional and keyword arguments passed to create this factory are
+    passed on as-is to the protocol.
+
+    @ivar args: Positional arguments passed to the protocol upon instantiation.
+    @type args: C{tuple}.
+    @ivar kwargs: Keyword arguments passed to the protocol upon instantiation.
+    @type kwargs: C{dict}.
+    """
+
+    def __init__(self, *args, **kwargs):
+        BootstrapMixin.__init__(self)
+        self.args = args
+        self.kwargs = kwargs
+
+
+    def buildProtocol(self, addr):
+        """
+        Create an instance of XmlStream.
+
+        The returned instance will have bootstrap event observers registered
+        and will proceed to handle input on an incoming connection.
+        """
+        xs = self.protocol(*self.args, **self.kwargs)
+        xs.factory = self
+        self.installBootstraps(xs)
+        return xs
+
+
+
+class XmlStreamFactory(XmlStreamFactoryMixin,
+                       protocol.ReconnectingClientFactory):
+    """
+    Factory for XmlStream protocol objects as a reconnection client.
+    """
+
+    protocol = XmlStream
+
+    def buildProtocol(self, addr):
+        """
+        Create a protocol instance.
+
+        Overrides L{XmlStreamFactoryMixin.buildProtocol} to work with
+        a L{ReconnectingClientFactory}. As this is called upon having an
+        connection established, we are resetting the delay for reconnection
+        attempts when the connection is lost again.
+        """
+        self.resetDelay()
+        return XmlStreamFactoryMixin.buildProtocol(self, addr)
diff --git a/contrib/python/Twisted/py2/twisted/words/xish/xpath.py b/contrib/python/Twisted/py2/twisted/words/xish/xpath.py
new file mode 100644
index 00000000000..85361c1c55e
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/xish/xpath.py
@@ -0,0 +1,337 @@
+# -*- test-case-name: twisted.words.test.test_xpath -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XPath query support.
+
+This module provides L{XPathQuery} to match
+L{domish.Element<twisted.words.xish.domish.Element>} instances against
+XPath-like expressions.
+"""
+
+from __future__ import absolute_import, division
+
+from io import StringIO
+
+from twisted.python.compat import StringType, unicode
+
+class LiteralValue(unicode):
+    def value(self, elem):
+        return self
+
+
+class IndexValue:
+    def __init__(self, index):
+        self.index = int(index) - 1
+
+    def value(self, elem):
+        return elem.children[self.index]
+
+
+class AttribValue:
+    def __init__(self, attribname):
+        self.attribname = attribname
+        if self.attribname == "xmlns":
+            self.value = self.value_ns
+
+    def value_ns(self, elem):
+        return elem.uri
+
+    def value(self, elem):
+        if self.attribname in elem.attributes:
+            return elem.attributes[self.attribname]
+        else:
+            return None
+
+
+class CompareValue:
+    def __init__(self, lhs, op, rhs):
+        self.lhs = lhs
+        self.rhs = rhs
+        if op == "=":
+            self.value = self._compareEqual
+        else:
+            self.value = self._compareNotEqual
+
+    def _compareEqual(self, elem):
+        return self.lhs.value(elem) == self.rhs.value(elem)
+
+    def _compareNotEqual(self, elem):
+        return self.lhs.value(elem) != self.rhs.value(elem)
+
+
+class BooleanValue:
+    """
+    Provide boolean XPath expression operators.
+
+    @ivar lhs: Left hand side expression of the operator.
+    @ivar op: The operator. One of C{'and'}, C{'or'}.
+    @ivar rhs: Right hand side expression of the operator.
+    @ivar value: Reference to the method that will calculate the value of
+                 this expression given an element.
+    """
+    def __init__(self, lhs, op, rhs):
+        self.lhs = lhs
+        self.rhs = rhs
+        if op == "and":
+            self.value = self._booleanAnd
+        else:
+            self.value = self._booleanOr
+
+    def _booleanAnd(self, elem):
+        """
+        Calculate boolean and of the given expressions given an element.
+
+        @param elem: The element to calculate the value of the expression from.
+        """
+        return self.lhs.value(elem) and self.rhs.value(elem)
+
+    def _booleanOr(self, elem):
+        """
+        Calculate boolean or of the given expressions given an element.
+
+        @param elem: The element to calculate the value of the expression from.
+        """
+        return self.lhs.value(elem) or self.rhs.value(elem)
+
+
+def Function(fname):
+    """
+    Internal method which selects the function object
+    """
+    klassname = "_%s_Function" % fname
+    c = globals()[klassname]()
+    return c
+
+
+class _not_Function:
+    def __init__(self):
+        self.baseValue = None
+
+    def setParams(self, baseValue):
+        self.baseValue = baseValue
+
+    def value(self, elem):
+        return not self.baseValue.value(elem)
+
+
+class _text_Function:
+    def setParams(self):
+        pass
+
+    def value(self, elem):
+        return unicode(elem)
+
+
+class _Location:
+    def __init__(self):
+        self.predicates = []
+        self.elementName  = None
+        self.childLocation = None
+
+    def matchesPredicates(self, elem):
+        if self.elementName != None and self.elementName != elem.name:
+            return 0
+
+        for p in self.predicates:
+            if not p.value(elem):
+                return 0
+
+        return 1
+
+    def matches(self, elem):
+        if not self.matchesPredicates(elem):
+            return 0
+
+        if self.childLocation != None:
+            for c in elem.elements():
+                if self.childLocation.matches(c):
+                    return 1
+        else:
+            return 1
+
+        return 0
+
+    def queryForString(self, elem, resultbuf):
+        if not self.matchesPredicates(elem):
+            return
+
+        if self.childLocation != None:
+            for c in elem.elements():
+                self.childLocation.queryForString(c, resultbuf)
+        else:
+            resultbuf.write(unicode(elem))
+
+    def queryForNodes(self, elem, resultlist):
+        if not self.matchesPredicates(elem):
+            return
+
+        if self.childLocation != None:
+            for c in elem.elements():
+                self.childLocation.queryForNodes(c, resultlist)
+        else:
+            resultlist.append(elem)
+
+    def queryForStringList(self, elem, resultlist):
+        if not self.matchesPredicates(elem):
+            return
+
+        if self.childLocation != None:
+            for c in elem.elements():
+                self.childLocation.queryForStringList(c, resultlist)
+        else:
+            for c in elem.children:
+                if isinstance(c, StringType):
+                    resultlist.append(c)
+
+
+class _AnyLocation:
+    def __init__(self):
+        self.predicates = []
+        self.elementName = None
+        self.childLocation = None
+
+    def matchesPredicates(self, elem):
+        for p in self.predicates:
+            if not p.value(elem):
+                return 0
+        return 1
+
+    def listParents(self, elem, parentlist):
+        if elem.parent != None:
+            self.listParents(elem.parent, parentlist)
+        parentlist.append(elem.name)
+
+    def isRootMatch(self, elem):
+        if (self.elementName == None or self.elementName == elem.name) and \
+           self.matchesPredicates(elem):
+            if self.childLocation != None:
+                for c in elem.elements():
+                    if self.childLocation.matches(c):
+                        return True
+            else:
+                return True
+        return False
+
+    def findFirstRootMatch(self, elem):
+        if (self.elementName == None or self.elementName == elem.name) and \
+           self.matchesPredicates(elem):
+            # Thus far, the name matches and the predicates match,
+            # now check into the children and find the first one
+            # that matches the rest of the structure
+            # the rest of the structure
+            if self.childLocation != None:
+                for c in elem.elements():
+                    if self.childLocation.matches(c):
+                        return c
+                return None
+            else:
+                # No children locations; this is a match!
+                return elem
+        else:
+            # Ok, predicates or name didn't match, so we need to start
+            # down each child and treat it as the root and try
+            # again
+            for c in elem.elements():
+                if self.matches(c):
+                    return c
+            # No children matched...
+            return None
+
+    def matches(self, elem):
+        if self.isRootMatch(elem):
+            return True
+        else:
+            # Ok, initial element isn't an exact match, walk
+            # down each child and treat it as the root and try
+            # again
+            for c in elem.elements():
+                if self.matches(c):
+                    return True
+            # No children matched...
+            return False
+
+    def queryForString(self, elem, resultbuf):
+        raise NotImplementedError(
+            "queryForString is not implemented for any location")
+
+    def queryForNodes(self, elem, resultlist):
+        # First check to see if _this_ element is a root
+        if self.isRootMatch(elem):
+            resultlist.append(elem)
+
+        # Now check each child
+        for c in elem.elements():
+            self.queryForNodes(c, resultlist)
+
+
+    def queryForStringList(self, elem, resultlist):
+        if self.isRootMatch(elem):
+            for c in elem.children:
+                if isinstance(c, StringType):
+                    resultlist.append(c)
+        for c in elem.elements():
+            self.queryForStringList(c, resultlist)
+
+
+class XPathQuery:
+    def __init__(self, queryStr):
+        self.queryStr = queryStr
+        # Prevent a circular import issue, as xpathparser imports this module.
+        from twisted.words.xish.xpathparser import (XPathParser,
+                                                    XPathParserScanner)
+        parser = XPathParser(XPathParserScanner(queryStr))
+        self.baseLocation = getattr(parser, 'XPATH')()
+
+    def __hash__(self):
+        return self.queryStr.__hash__()
+
+    def matches(self, elem):
+        return self.baseLocation.matches(elem)
+
+    def queryForString(self, elem):
+        result = StringIO()
+        self.baseLocation.queryForString(elem, result)
+        return result.getvalue()
+
+    def queryForNodes(self, elem):
+        result = []
+        self.baseLocation.queryForNodes(elem, result)
+        if len(result) == 0:
+            return None
+        else:
+            return result
+
+    def queryForStringList(self, elem):
+        result = []
+        self.baseLocation.queryForStringList(elem, result)
+        if len(result) == 0:
+            return None
+        else:
+            return result
+
+
+__internedQueries = {}
+
+def internQuery(queryString):
+    if queryString not in __internedQueries:
+        __internedQueries[queryString] = XPathQuery(queryString)
+    return __internedQueries[queryString]
+
+
+def matches(xpathstr, elem):
+    return internQuery(xpathstr).matches(elem)
+
+
+def queryForStringList(xpathstr, elem):
+    return internQuery(xpathstr).queryForStringList(elem)
+
+
+def queryForString(xpathstr, elem):
+    return internQuery(xpathstr).queryForString(elem)
+
+
+def queryForNodes(xpathstr, elem):
+    return internQuery(xpathstr).queryForNodes(elem)
diff --git a/contrib/python/Twisted/py2/twisted/words/xish/xpathparser.py b/contrib/python/Twisted/py2/twisted/words/xish/xpathparser.py
new file mode 100644
index 00000000000..83994a737d3
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/xish/xpathparser.py
@@ -0,0 +1,650 @@
+# -*- test-case-name: twisted.words.test.test_xpath -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+# pylint: disable=W9401,W9402
+
+# DO NOT EDIT xpathparser.py!
+#
+# It is generated from xpathparser.g using Yapps. Make needed changes there.
+# This also means that the generated Python may not conform to Twisted's coding
+# standards, so it is wrapped in exec to prevent automated checkers from
+# complaining.
+
+# HOWTO Generate me:
+#
+# 1.) Grab a copy of yapps2:
+#         https://github.com/smurfix/yapps
+#
+#     Note: Do NOT use the package in debian/ubuntu as it has incompatible
+#     modifications. The original at http://theory.stanford.edu/~amitp/yapps/
+#     hasn't been touched since 2003 and has not been updated to work with
+#     Python 3.
+#
+# 2.) Generate the grammar:
+#
+#         yapps2 xpathparser.g xpathparser.py.proto
+#
+# 3.) Edit the output to depend on the embedded runtime, and remove extraneous
+#     imports:
+#
+#         sed -e '/^# Begin/,${/^[^ ].*mport/d}' -e '/^[^#]/s/runtime\.//g' \
+#             -e "s/^\(from __future\)/exec(r'''\n\1/" -e"\$a''')"
+#             xpathparser.py.proto > xpathparser.py
+
+"""
+XPath Parser.
+
+Besides the parser code produced by Yapps, this module also defines the
+parse-time exception classes, a scanner class, a base class for parsers
+produced by Yapps, and a context class that keeps track of the parse stack.
+These have been copied from the Yapps runtime module.
+"""
+
+exec(r'''
+from __future__ import print_function
+import sys, re
+
+MIN_WINDOW=4096
+# File lookup window
+
+class SyntaxError(Exception):
+    """When we run into an unexpected token, this is the exception to use"""
+    def __init__(self, pos=None, msg="Bad Token", context=None):
+        Exception.__init__(self)
+        self.pos = pos
+        self.msg = msg
+        self.context = context
+
+    def __str__(self):
+        if not self.pos: return 'SyntaxError'
+        else: return 'SyntaxError@%s(%s)' % (repr(self.pos), self.msg)
+
+class NoMoreTokens(Exception):
+    """Another exception object, for when we run out of tokens"""
+    pass
+
+class Token(object):
+    """Yapps token.
+
+    This is a container for a scanned token.
+    """
+
+    def __init__(self, type,value, pos=None):
+        """Initialize a token."""
+        self.type = type
+        self.value = value
+        self.pos = pos
+
+    def __repr__(self):
+        output = '<%s: %s' % (self.type, repr(self.value))
+        if self.pos:
+            output += " @ "
+            if self.pos[0]:
+                output += "%s:" % self.pos[0]
+            if self.pos[1]:
+                output += "%d" % self.pos[1]
+            if self.pos[2] is not None:
+                output += ".%d" % self.pos[2]
+        output += ">"
+        return output
+
+in_name=0
+class Scanner(object):
+    """Yapps scanner.
+
+    The Yapps scanner can work in context sensitive or context
+    insensitive modes.  The token(i) method is used to retrieve the
+    i-th token.  It takes a restrict set that limits the set of tokens
+    it is allowed to return.  In context sensitive mode, this restrict
+    set guides the scanner.  In context insensitive mode, there is no
+    restriction (the set is always the full set of tokens).
+
+    """
+
+    def __init__(self, patterns, ignore, input="",
+            file=None,filename=None,stacked=False):
+        """Initialize the scanner.
+
+        Parameters:
+          patterns : [(terminal, uncompiled regex), ...] or None
+          ignore : {terminal:None, ...}
+          input : string
+
+        If patterns is None, we assume that the subclass has
+        defined self.patterns : [(terminal, compiled regex), ...].
+        Note that the patterns parameter expects uncompiled regexes,
+        whereas the self.patterns field expects compiled regexes.
+
+        The 'ignore' value is either None or a callable, which is called
+        with the scanner and the to-be-ignored match object; this can
+        be used for include file or comment handling.
+        """
+
+        if not filename:
+            global in_name
+            filename="<f.%d>" % in_name
+            in_name += 1
+
+        self.input = input
+        self.ignore = ignore
+        self.file = file
+        self.filename = filename
+        self.pos = 0
+        self.del_pos = 0 # skipped
+        self.line = 1
+        self.del_line = 0 # skipped
+        self.col = 0
+        self.tokens = []
+        self.stack = None
+        self.stacked = stacked
+
+        self.last_read_token = None
+        self.last_token = None
+        self.last_types = None
+
+        if patterns is not None:
+            # Compile the regex strings into regex objects
+            self.patterns = []
+            for terminal, regex in patterns:
+                self.patterns.append( (terminal, re.compile(regex)) )
+
+    def stack_input(self, input="", file=None, filename=None):
+        """Temporarily parse from a second file."""
+
+        # Already reading from somewhere else: Go on top of that, please.
+        if self.stack:
+            # autogenerate a recursion-level-identifying filename
+            if not filename:
+                filename = 1
+            else:
+                try:
+                    filename += 1
+                except TypeError:
+                    pass
+                # now pass off to the include file
+            self.stack.stack_input(input,file,filename)
+        else:
+
+            try:
+                filename += 0
+            except TypeError:
+                pass
+            else:
+                filename = "<str_%d>" % filename
+
+#			self.stack = object.__new__(self.__class__)
+#			Scanner.__init__(self.stack,self.patterns,self.ignore,input,file,filename, stacked=True)
+
+            # Note that the pattern+ignore are added by the generated
+            # scanner code
+            self.stack = self.__class__(input,file,filename, stacked=True)
+
+    def get_pos(self):
+        """Return a file/line/char tuple."""
+        if self.stack: return self.stack.get_pos()
+
+        return (self.filename, self.line+self.del_line, self.col)
+
+#	def __repr__(self):
+#		"""Print the last few tokens that have been scanned in"""
+#		output = ''
+#		for t in self.tokens:
+#			output += '%s\n' % (repr(t),)
+#		return output
+
+    def print_line_with_pointer(self, pos, length=0, out=sys.stderr):
+        """Print the line of 'text' that includes position 'p',
+        along with a second line with a single caret (^) at position p"""
+
+        file,line,p = pos
+        if file != self.filename:
+            if self.stack: return self.stack.print_line_with_pointer(pos,length=length,out=out)
+            print >>out, "(%s: not in input buffer)" % file
+            return
+
+        text = self.input
+        p += length-1 # starts at pos 1
+
+        origline=line
+        line -= self.del_line
+        spos=0
+        if line > 0:
+            while 1:
+                line = line - 1
+                try:
+                    cr = text.index("\n",spos)
+                except ValueError:
+                    if line:
+                        text = ""
+                    break
+                if line == 0:
+                    text = text[spos:cr]
+                    break
+                spos = cr+1
+        else:
+            print >>out, "(%s:%d not in input buffer)" % (file,origline)
+            return
+
+        # Now try printing part of the line
+        text = text[max(p-80, 0):p+80]
+        p = p - max(p-80, 0)
+
+        # Strip to the left
+        i = text[:p].rfind('\n')
+        j = text[:p].rfind('\r')
+        if i < 0 or (0 <= j < i): i = j
+        if 0 <= i < p:
+            p = p - i - 1
+            text = text[i+1:]
+
+        # Strip to the right
+        i = text.find('\n', p)
+        j = text.find('\r', p)
+        if i < 0 or (0 <= j < i): i = j
+        if i >= 0:
+            text = text[:i]
+
+        # Now shorten the text
+        while len(text) > 70 and p > 60:
+            # Cut off 10 chars
+            text = "..." + text[10:]
+            p = p - 7
+
+        # Now print the string, along with an indicator
+        print >>out, '> ',text
+        print >>out, '> ',' '*p + '^'
+
+    def grab_input(self):
+        """Get more input if possible."""
+        if not self.file: return
+        if len(self.input) - self.pos >= MIN_WINDOW: return
+
+        data = self.file.read(MIN_WINDOW)
+        if data is None or data == "":
+            self.file = None
+
+        # Drop bytes from the start, if necessary.
+        if self.pos > 2*MIN_WINDOW:
+            self.del_pos += MIN_WINDOW
+            self.del_line += self.input[:MIN_WINDOW].count("\n")
+            self.pos -= MIN_WINDOW
+            self.input = self.input[MIN_WINDOW:] + data
+        else:
+            self.input = self.input + data
+
+    def getchar(self):
+        """Return the next character."""
+        self.grab_input()
+
+        c = self.input[self.pos]
+        self.pos += 1
+        return c
+
+    def token(self, restrict, context=None):
+        """Scan for another token."""
+
+        while 1:
+            if self.stack:
+                try:
+                    return self.stack.token(restrict, context)
+                except StopIteration:
+                    self.stack = None
+
+        # Keep looking for a token, ignoring any in self.ignore
+            self.grab_input()
+
+            # special handling for end-of-file
+            if self.stacked and self.pos==len(self.input):
+                raise StopIteration
+
+            # Search the patterns for the longest match, with earlier
+            # tokens in the list having preference
+            best_match = -1
+            best_pat = '(error)'
+            best_m = None
+            for p, regexp in self.patterns:
+                # First check to see if we're ignoring this token
+                if restrict and p not in restrict and p not in self.ignore:
+                    continue
+                m = regexp.match(self.input, self.pos)
+                if m and m.end()-m.start() > best_match:
+                    # We got a match that's better than the previous one
+                    best_pat = p
+                    best_match = m.end()-m.start()
+                    best_m = m
+
+            # If we didn't find anything, raise an error
+            if best_pat == '(error)' and best_match < 0:
+                msg = 'Bad Token'
+                if restrict:
+                    msg = 'Trying to find one of '+', '.join(restrict)
+                raise SyntaxError(self.get_pos(), msg, context=context)
+
+            ignore = best_pat in self.ignore
+            value = self.input[self.pos:self.pos+best_match]
+            if not ignore:
+                tok=Token(type=best_pat, value=value, pos=self.get_pos())
+
+            self.pos += best_match
+
+            npos = value.rfind("\n")
+            if npos > -1:
+                self.col = best_match-npos
+                self.line += value.count("\n")
+            else:
+                self.col += best_match
+
+            # If we found something that isn't to be ignored, return it
+            if not ignore:
+                if len(self.tokens) >= 10:
+                    del self.tokens[0]
+                self.tokens.append(tok)
+                self.last_read_token = tok
+                # print repr(tok)
+                return tok
+            else:
+                ignore = self.ignore[best_pat]
+                if ignore:
+                    ignore(self, best_m)
+
+    def peek(self, *types, **kw):
+        """Returns the token type for lookahead; if there are any args
+        then the list of args is the set of token types to allow"""
+        context = kw.get("context",None)
+        if self.last_token is None:
+            self.last_types = types
+            self.last_token = self.token(types,context)
+        elif self.last_types:
+            for t in types:
+                if t not in self.last_types:
+                    raise NotImplementedError("Unimplemented: restriction set changed")
+        return self.last_token.type
+
+    def scan(self, type, **kw):
+        """Returns the matched text, and moves to the next token"""
+        context = kw.get("context",None)
+
+        if self.last_token is None:
+            tok = self.token([type],context)
+        else:
+            if self.last_types and type not in self.last_types:
+                raise NotImplementedError("Unimplemented: restriction set changed")
+
+            tok = self.last_token
+            self.last_token = None
+        if tok.type != type:
+            if not self.last_types: self.last_types=[]
+            raise SyntaxError(tok.pos, 'Trying to find '+type+': '+ ', '.join(self.last_types)+", got "+tok.type, context=context)
+        return tok.value
+
+class Parser(object):
+    """Base class for Yapps-generated parsers.
+
+    """
+
+    def __init__(self, scanner):
+        self._scanner = scanner
+
+    def _stack(self, input="",file=None,filename=None):
+        """Temporarily read from someplace else"""
+        self._scanner.stack_input(input,file,filename)
+        self._tok = None
+
+    def _peek(self, *types, **kw):
+        """Returns the token type for lookahead; if there are any args
+        then the list of args is the set of token types to allow"""
+        return self._scanner.peek(*types, **kw)
+
+    def _scan(self, type, **kw):
+        """Returns the matched text, and moves to the next token"""
+        return self._scanner.scan(type, **kw)
+
+class Context(object):
+    """Class to represent the parser's call stack.
+
+    Every rule creates a Context that links to its parent rule.  The
+    contexts can be used for debugging.
+
+    """
+
+    def __init__(self, parent, scanner, rule, args=()):
+        """Create a new context.
+
+        Args:
+        parent: Context object or None
+        scanner: Scanner object
+        rule: string (name of the rule)
+        args: tuple listing parameters to the rule
+
+        """
+        self.parent = parent
+        self.scanner = scanner
+        self.rule = rule
+        self.args = args
+        while scanner.stack: scanner = scanner.stack
+        self.token = scanner.last_read_token
+
+    def __str__(self):
+        output = ''
+        if self.parent: output = str(self.parent) + ' > '
+        output += self.rule
+        return output
+
+def print_error(err, scanner, max_ctx=None):
+    """Print error messages, the parser stack, and the input text -- for human-readable error messages."""
+    # NOTE: this function assumes 80 columns :-(
+    # Figure out the line number
+    pos = err.pos
+    if not pos:
+        pos = scanner.get_pos()
+
+    file_name, line_number, column_number = pos
+    print('%s:%d:%d: %s' % (file_name, line_number, column_number, err.msg), file=sys.stderr)
+
+    scanner.print_line_with_pointer(pos)
+
+    context = err.context
+    token = None
+    while context:
+        print('while parsing %s%s:' % (context.rule, tuple(context.args)), file=sys.stderr)
+        if context.token:
+            token = context.token
+        if token:
+            scanner.print_line_with_pointer(token.pos, length=len(token.value))
+        context = context.parent
+        if max_ctx:
+            max_ctx = max_ctx-1
+            if not max_ctx:
+                break
+
+def wrap_error_reporter(parser, rule, *args,**kw):
+    try:
+        return getattr(parser, rule)(*args,**kw)
+    except SyntaxError as e:
+        print_error(e, parser._scanner)
+    except NoMoreTokens:
+        print('Could not complete parsing; stopped around here:', file=sys.stderr)
+        print(parser._scanner, file=sys.stderr)
+
+from twisted.words.xish.xpath import AttribValue, BooleanValue, CompareValue
+from twisted.words.xish.xpath import Function, IndexValue, LiteralValue
+from twisted.words.xish.xpath import _AnyLocation, _Location
+
+
+# Begin -- grammar generated by Yapps
+
+class XPathParserScanner(Scanner):
+    patterns = [
+        ('","', re.compile(',')),
+        ('"@"', re.compile('@')),
+        ('"\\)"', re.compile('\\)')),
+        ('"\\("', re.compile('\\(')),
+        ('"\\]"', re.compile('\\]')),
+        ('"\\["', re.compile('\\[')),
+        ('"//"', re.compile('//')),
+        ('"/"', re.compile('/')),
+        ('\\s+', re.compile('\\s+')),
+        ('INDEX', re.compile('[0-9]+')),
+        ('WILDCARD', re.compile('\\*')),
+        ('IDENTIFIER', re.compile('[a-zA-Z][a-zA-Z0-9_\\-]*')),
+        ('ATTRIBUTE', re.compile('\\@[a-zA-Z][a-zA-Z0-9_\\-]*')),
+        ('FUNCNAME', re.compile('[a-zA-Z][a-zA-Z0-9_]*')),
+        ('CMP_EQ', re.compile('\\=')),
+        ('CMP_NE', re.compile('\\!\\=')),
+        ('STR_DQ', re.compile('"([^"]|(\\"))*?"')),
+        ('STR_SQ', re.compile("'([^']|(\\'))*?'")),
+        ('OP_AND', re.compile('and')),
+        ('OP_OR', re.compile('or')),
+        ('END', re.compile('$')),
+    ]
+    def __init__(self, str,*args,**kw):
+        Scanner.__init__(self,None,{'\\s+':None,},str,*args,**kw)
+
+class XPathParser(Parser):
+    Context = Context
+    def XPATH(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'XPATH', [])
+        PATH = self.PATH(_context)
+        result = PATH; current = result
+        while self._peek('END', '"/"', '"//"', context=_context) != 'END':
+            PATH = self.PATH(_context)
+            current.childLocation = PATH; current = current.childLocation
+        END = self._scan('END', context=_context)
+        return  result
+
+    def PATH(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'PATH', [])
+        _token = self._peek('"/"', '"//"', context=_context)
+        if _token == '"/"':
+            self._scan('"/"', context=_context)
+            result = _Location()
+        else: # == '"//"'
+            self._scan('"//"', context=_context)
+            result = _AnyLocation()
+        _token = self._peek('IDENTIFIER', 'WILDCARD', context=_context)
+        if _token == 'IDENTIFIER':
+            IDENTIFIER = self._scan('IDENTIFIER', context=_context)
+            result.elementName = IDENTIFIER
+        else: # == 'WILDCARD'
+            WILDCARD = self._scan('WILDCARD', context=_context)
+            result.elementName = None
+        while self._peek('"\\["', 'END', '"/"', '"//"', context=_context) == '"\\["':
+            self._scan('"\\["', context=_context)
+            PREDICATE = self.PREDICATE(_context)
+            result.predicates.append(PREDICATE)
+            self._scan('"\\]"', context=_context)
+        return result
+
+    def PREDICATE(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'PREDICATE', [])
+        _token = self._peek('INDEX', '"\\("', '"@"', 'FUNCNAME', 'STR_DQ', 'STR_SQ', context=_context)
+        if _token != 'INDEX':
+            EXPR = self.EXPR(_context)
+            return EXPR
+        else: # == 'INDEX'
+            INDEX = self._scan('INDEX', context=_context)
+            return IndexValue(INDEX)
+
+    def EXPR(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'EXPR', [])
+        FACTOR = self.FACTOR(_context)
+        e = FACTOR
+        while self._peek('OP_AND', 'OP_OR', '"\\)"', '"\\]"', context=_context) in ['OP_AND', 'OP_OR']:
+            BOOLOP = self.BOOLOP(_context)
+            FACTOR = self.FACTOR(_context)
+            e = BooleanValue(e, BOOLOP, FACTOR)
+        return e
+
+    def BOOLOP(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'BOOLOP', [])
+        _token = self._peek('OP_AND', 'OP_OR', context=_context)
+        if _token == 'OP_AND':
+            OP_AND = self._scan('OP_AND', context=_context)
+            return OP_AND
+        else: # == 'OP_OR'
+            OP_OR = self._scan('OP_OR', context=_context)
+            return OP_OR
+
+    def FACTOR(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'FACTOR', [])
+        _token = self._peek('"\\("', '"@"', 'FUNCNAME', 'STR_DQ', 'STR_SQ', context=_context)
+        if _token != '"\\("':
+            TERM = self.TERM(_context)
+            return TERM
+        else: # == '"\\("'
+            self._scan('"\\("', context=_context)
+            EXPR = self.EXPR(_context)
+            self._scan('"\\)"', context=_context)
+            return EXPR
+
+    def TERM(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'TERM', [])
+        VALUE = self.VALUE(_context)
+        t = VALUE
+        if self._peek('CMP_EQ', 'CMP_NE', 'OP_AND', 'OP_OR', '"\\)"', '"\\]"', context=_context) in ['CMP_EQ', 'CMP_NE']:
+            CMP = self.CMP(_context)
+            VALUE = self.VALUE(_context)
+            t = CompareValue(t, CMP, VALUE)
+        return t
+
+    def VALUE(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'VALUE', [])
+        _token = self._peek('"@"', 'FUNCNAME', 'STR_DQ', 'STR_SQ', context=_context)
+        if _token == '"@"':
+            self._scan('"@"', context=_context)
+            IDENTIFIER = self._scan('IDENTIFIER', context=_context)
+            return AttribValue(IDENTIFIER)
+        elif _token == 'FUNCNAME':
+            FUNCNAME = self._scan('FUNCNAME', context=_context)
+            f = Function(FUNCNAME); args = []
+            self._scan('"\\("', context=_context)
+            if self._peek('"\\)"', '"@"', 'FUNCNAME', '","', 'STR_DQ', 'STR_SQ', context=_context) not in ['"\\)"', '","']:
+                VALUE = self.VALUE(_context)
+                args.append(VALUE)
+                while self._peek('","', '"\\)"', context=_context) == '","':
+                    self._scan('","', context=_context)
+                    VALUE = self.VALUE(_context)
+                    args.append(VALUE)
+            self._scan('"\\)"', context=_context)
+            f.setParams(*args); return f
+        else: # in ['STR_DQ', 'STR_SQ']
+            STR = self.STR(_context)
+            return LiteralValue(STR[1:len(STR)-1])
+
+    def CMP(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'CMP', [])
+        _token = self._peek('CMP_EQ', 'CMP_NE', context=_context)
+        if _token == 'CMP_EQ':
+            CMP_EQ = self._scan('CMP_EQ', context=_context)
+            return CMP_EQ
+        else: # == 'CMP_NE'
+            CMP_NE = self._scan('CMP_NE', context=_context)
+            return CMP_NE
+
+    def STR(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'STR', [])
+        _token = self._peek('STR_DQ', 'STR_SQ', context=_context)
+        if _token == 'STR_DQ':
+            STR_DQ = self._scan('STR_DQ', context=_context)
+            return STR_DQ
+        else: # == 'STR_SQ'
+            STR_SQ = self._scan('STR_SQ', context=_context)
+            return STR_SQ
+
+
+def parse(rule, text):
+    P = XPathParser(XPathParserScanner(text))
+    return wrap_error_reporter(P, rule)
+
+if __name__ == '__main__':
+    from sys import argv, stdin
+    if len(argv) >= 2:
+        if len(argv) >= 3:
+            f = open(argv[2],'r')
+        else:
+            f = stdin
+        print(parse(argv[1], f.read()))
+    else: print ('Args:  <rule> [<filename>]', file=sys.stderr)
+# End -- grammar generated by Yapps
+''')
diff --git a/contrib/python/Twisted/py2/twisted/words/xmpproutertap.py b/contrib/python/Twisted/py2/twisted/words/xmpproutertap.py
new file mode 100644
index 00000000000..0bdae0ae4ef
--- /dev/null
+++ b/contrib/python/Twisted/py2/twisted/words/xmpproutertap.py
@@ -0,0 +1,30 @@
+# -*- test-case-name: twisted.words.test.test_xmpproutertap -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application import strports
+from twisted.python import usage
+from twisted.words.protocols.jabber import component
+
+class Options(usage.Options):
+    optParameters = [
+            ('port', None, 'tcp:5347:interface=127.0.0.1',
+                           'Port components connect to'),
+            ('secret', None, 'secret', 'Router secret'),
+    ]
+
+    optFlags = [
+            ('verbose', 'v', 'Log traffic'),
+    ]
+
+
+
+def makeService(config):
+    router = component.Router()
+    factory = component.XMPPComponentServerFactory(router, config['secret'])
+
+    if config['verbose']:
+        factory.logTraffic = True
+
+    return strports.service(config['port'], factory)
diff --git a/contrib/python/Twisted/py2/ya.make b/contrib/python/Twisted/py2/ya.make
new file mode 100644
index 00000000000..81d7a3aa06b
--- /dev/null
+++ b/contrib/python/Twisted/py2/ya.make
@@ -0,0 +1,472 @@
+PY2_LIBRARY()
+
+LICENSE(MIT)
+
+VERSION(20.3.0)
+
+NO_CHECK_IMPORTS(
+    twisted.*
+)
+
+PEERDIR(
+    contrib/python/Automat
+    contrib/python/attrs
+    contrib/python/constantly
+    contrib/python/hyperlink
+    contrib/python/incremental
+    contrib/python/pyOpenSSL
+    contrib/python/Twisted/py2/twisted/python
+    contrib/python/zope.interface
+)
+
+NO_LINT()
+
+NO_COMPILER_WARNINGS()
+
+PY_SRCS(
+    TOP_LEVEL
+    twisted/__init__.py
+    twisted/_threads/__init__.py
+    twisted/_threads/_convenience.py
+    twisted/_threads/_ithreads.py
+    twisted/_threads/_memory.py
+    twisted/_threads/_pool.py
+    twisted/_threads/_team.py
+    twisted/_threads/_threadworker.py
+    twisted/_version.py
+    twisted/application/__init__.py
+    twisted/application/app.py
+    twisted/application/internet.py
+    twisted/application/reactors.py
+    twisted/application/runner/__init__.py
+    twisted/application/runner/_exit.py
+    twisted/application/runner/_pidfile.py
+    twisted/application/runner/_runner.py
+    twisted/application/service.py
+    twisted/application/strports.py
+    twisted/application/twist/__init__.py
+    twisted/application/twist/_options.py
+    twisted/application/twist/_twist.py
+    twisted/conch/__init__.py
+    twisted/conch/avatar.py
+    twisted/conch/checkers.py
+    twisted/conch/client/__init__.py
+    twisted/conch/client/agent.py
+    twisted/conch/client/connect.py
+    twisted/conch/client/default.py
+    twisted/conch/client/direct.py
+    twisted/conch/client/knownhosts.py
+    twisted/conch/client/options.py
+    twisted/conch/endpoints.py
+    twisted/conch/error.py
+    twisted/conch/insults/__init__.py
+    twisted/conch/insults/helper.py
+    twisted/conch/insults/insults.py
+    twisted/conch/insults/text.py
+    twisted/conch/insults/window.py
+    twisted/conch/interfaces.py
+    twisted/conch/ls.py
+    twisted/conch/manhole.py
+    twisted/conch/manhole_ssh.py
+    twisted/conch/manhole_tap.py
+    twisted/conch/mixin.py
+    twisted/conch/openssh_compat/__init__.py
+    twisted/conch/openssh_compat/factory.py
+    twisted/conch/openssh_compat/primes.py
+    twisted/conch/recvline.py
+    twisted/conch/ssh/__init__.py
+    twisted/conch/ssh/_kex.py
+    twisted/conch/ssh/address.py
+    twisted/conch/ssh/agent.py
+    twisted/conch/ssh/channel.py
+    twisted/conch/ssh/common.py
+    twisted/conch/ssh/connection.py
+    twisted/conch/ssh/factory.py
+    twisted/conch/ssh/filetransfer.py
+    twisted/conch/ssh/forwarding.py
+    twisted/conch/ssh/keys.py
+    twisted/conch/ssh/service.py
+    twisted/conch/ssh/session.py
+    twisted/conch/ssh/sexpy.py
+    twisted/conch/ssh/transport.py
+    twisted/conch/ssh/userauth.py
+    twisted/conch/stdio.py
+    twisted/conch/tap.py
+    twisted/conch/telnet.py
+    twisted/conch/ttymodes.py
+    twisted/conch/ui/__init__.py
+    twisted/conch/ui/ansi.py
+    twisted/conch/ui/tkvt100.py
+    twisted/conch/unix.py
+    twisted/copyright.py
+    twisted/cred/__init__.py
+    twisted/cred/_digest.py
+    twisted/cred/checkers.py
+    twisted/cred/credentials.py
+    twisted/cred/error.py
+    twisted/cred/portal.py
+    twisted/cred/strcred.py
+    twisted/enterprise/__init__.py
+    twisted/enterprise/adbapi.py
+    twisted/internet/__init__.py
+    twisted/internet/_baseprocess.py
+    twisted/internet/_dumbwin32proc.py
+    twisted/internet/_glibbase.py
+    twisted/internet/_idna.py
+    twisted/internet/_newtls.py
+    twisted/internet/_pollingfile.py
+    twisted/internet/_posixserialport.py
+    twisted/internet/_posixstdio.py
+    twisted/internet/_producer_helpers.py
+    twisted/internet/_resolver.py
+    twisted/internet/_signals.py
+    twisted/internet/_sslverify.py
+    twisted/internet/_threadedselect.py
+    twisted/internet/_win32serialport.py
+    twisted/internet/_win32stdio.py
+    twisted/internet/abstract.py
+    twisted/internet/address.py
+    twisted/internet/asyncioreactor.py
+    twisted/internet/base.py
+    twisted/internet/cfreactor.py
+    twisted/internet/default.py
+    twisted/internet/defer.py
+    twisted/internet/endpoints.py
+    twisted/internet/epollreactor.py
+    twisted/internet/error.py
+    twisted/internet/fdesc.py
+    twisted/internet/gireactor.py
+    twisted/internet/glib2reactor.py
+    twisted/internet/gtk2reactor.py
+    twisted/internet/gtk3reactor.py
+    twisted/internet/inotify.py
+    twisted/internet/interfaces.py
+    twisted/internet/iocpreactor/__init__.py
+    twisted/internet/iocpreactor/abstract.py
+    twisted/internet/iocpreactor/const.py
+    twisted/internet/iocpreactor/interfaces.py
+    twisted/internet/iocpreactor/reactor.py
+    twisted/internet/iocpreactor/setup.py
+    twisted/internet/iocpreactor/tcp.py
+    twisted/internet/iocpreactor/udp.py
+    twisted/internet/kqreactor.py
+    twisted/internet/main.py
+    twisted/internet/pollreactor.py
+    twisted/internet/posixbase.py
+    twisted/internet/process.py
+    twisted/internet/protocol.py
+    twisted/internet/pyuisupport.py
+    twisted/internet/reactor.py
+    twisted/internet/selectreactor.py
+    twisted/internet/serialport.py
+    twisted/internet/ssl.py
+    twisted/internet/stdio.py
+    twisted/internet/task.py
+    twisted/internet/tcp.py
+    twisted/internet/testing.py
+    twisted/internet/threads.py
+    twisted/internet/tksupport.py
+    twisted/internet/udp.py
+    twisted/internet/unix.py
+    twisted/internet/utils.py
+    twisted/internet/win32eventreactor.py
+    twisted/internet/wxreactor.py
+    twisted/internet/wxsupport.py
+    twisted/logger/__init__.py
+    twisted/logger/_buffer.py
+    twisted/logger/_capture.py
+    twisted/logger/_file.py
+    twisted/logger/_filter.py
+    twisted/logger/_flatten.py
+    twisted/logger/_format.py
+    twisted/logger/_global.py
+    twisted/logger/_io.py
+    twisted/logger/_json.py
+    twisted/logger/_legacy.py
+    twisted/logger/_levels.py
+    twisted/logger/_logger.py
+    twisted/logger/_observer.py
+    twisted/logger/_stdlib.py
+    twisted/logger/_util.py
+    twisted/mail/__init__.py
+    twisted/mail/_cred.py
+    twisted/mail/_except.py
+    twisted/mail/alias.py
+    twisted/mail/bounce.py
+    twisted/mail/imap4.py
+    twisted/mail/interfaces.py
+    twisted/mail/mail.py
+    twisted/mail/maildir.py
+    twisted/mail/pb.py
+    twisted/mail/pop3.py
+    twisted/mail/pop3client.py
+    twisted/mail/protocols.py
+    twisted/mail/relay.py
+    twisted/mail/relaymanager.py
+    twisted/mail/smtp.py
+    twisted/mail/tap.py
+    twisted/names/__init__.py
+    twisted/names/_rfc1982.py
+    twisted/names/authority.py
+    twisted/names/cache.py
+    twisted/names/client.py
+    twisted/names/common.py
+    twisted/names/dns.py
+    twisted/names/error.py
+    twisted/names/hosts.py
+    twisted/names/resolve.py
+    twisted/names/root.py
+    twisted/names/secondary.py
+    twisted/names/server.py
+    twisted/names/srvconnect.py
+    twisted/names/tap.py
+    twisted/news/__init__.py
+    twisted/news/database.py
+    twisted/news/news.py
+    twisted/news/nntp.py
+    twisted/news/tap.py
+    twisted/pair/__init__.py
+    twisted/pair/ethernet.py
+    twisted/pair/ip.py
+    twisted/pair/raw.py
+    twisted/pair/rawudp.py
+    twisted/pair/testing.py
+    twisted/pair/tuntap.py
+    twisted/persisted/__init__.py
+    twisted/persisted/aot.py
+    twisted/persisted/crefutil.py
+    twisted/persisted/dirdbm.py
+    twisted/persisted/sob.py
+    twisted/persisted/styles.py
+    twisted/plugin.py
+    twisted/plugins/__init__.py
+    twisted/plugins/cred_anonymous.py
+    twisted/plugins/cred_file.py
+    twisted/plugins/cred_memory.py
+    twisted/plugins/cred_sshkeys.py
+    twisted/plugins/cred_unix.py
+    twisted/plugins/twisted_conch.py
+    twisted/plugins/twisted_core.py
+    twisted/plugins/twisted_ftp.py
+    twisted/plugins/twisted_inet.py
+    twisted/plugins/twisted_mail.py
+    twisted/plugins/twisted_names.py
+    twisted/plugins/twisted_news.py
+    twisted/plugins/twisted_portforward.py
+    twisted/plugins/twisted_reactors.py
+    twisted/plugins/twisted_runner.py
+    twisted/plugins/twisted_socks.py
+    twisted/plugins/twisted_trial.py
+    twisted/plugins/twisted_web.py
+    twisted/plugins/twisted_words.py
+    twisted/positioning/__init__.py
+    twisted/positioning/_sentence.py
+    twisted/positioning/base.py
+    twisted/positioning/ipositioning.py
+    twisted/positioning/nmea.py
+    twisted/protocols/__init__.py
+    twisted/protocols/amp.py
+    twisted/protocols/basic.py
+    twisted/protocols/dict.py
+    twisted/protocols/finger.py
+    twisted/protocols/ftp.py
+    twisted/protocols/haproxy/__init__.py
+    twisted/protocols/haproxy/_exceptions.py
+    twisted/protocols/haproxy/_info.py
+    twisted/protocols/haproxy/_interfaces.py
+    twisted/protocols/haproxy/_parser.py
+    twisted/protocols/haproxy/_v1parser.py
+    twisted/protocols/haproxy/_v2parser.py
+    twisted/protocols/haproxy/_wrapper.py
+    twisted/protocols/htb.py
+    twisted/protocols/ident.py
+    twisted/protocols/loopback.py
+    twisted/protocols/memcache.py
+    twisted/protocols/pcp.py
+    twisted/protocols/policies.py
+    twisted/protocols/portforward.py
+    twisted/protocols/postfix.py
+    twisted/protocols/shoutcast.py
+    twisted/protocols/sip.py
+    twisted/protocols/socks.py
+    twisted/protocols/stateful.py
+    twisted/protocols/tls.py
+    twisted/protocols/wire.py
+    twisted/python/__init__.py
+    twisted/python/_appdirs.py
+    twisted/python/_inotify.py
+    twisted/python/_pydoctor.py
+    twisted/python/_release.py
+    twisted/python/_setup.py
+    twisted/python/_shellcomp.py
+    twisted/python/_textattributes.py
+    twisted/python/_tzhelper.py
+    twisted/python/_url.py
+    twisted/python/constants.py
+    twisted/python/context.py
+    twisted/python/failure.py
+    twisted/python/fakepwd.py
+    twisted/python/finalize.py
+    twisted/python/formmethod.py
+    twisted/python/hook.py
+    twisted/python/htmlizer.py
+    twisted/python/lockfile.py
+    twisted/python/log.py
+    twisted/python/logfile.py
+    twisted/python/monkey.py
+    twisted/python/procutils.py
+    twisted/python/randbytes.py
+    twisted/python/rebuild.py
+    twisted/python/release.py
+    twisted/python/roots.py
+    twisted/python/sendmsg.py
+    twisted/python/shortcut.py
+    twisted/python/syslog.py
+    twisted/python/systemd.py
+    twisted/python/text.py
+    twisted/python/threadable.py
+    twisted/python/threadpool.py
+    twisted/python/url.py
+    twisted/python/urlpath.py
+    twisted/python/usage.py
+    twisted/python/versions.py
+    twisted/python/zipstream.py
+    twisted/runner/__init__.py
+    twisted/runner/inetd.py
+    twisted/runner/inetdconf.py
+    twisted/runner/inetdtap.py
+    twisted/runner/procmon.py
+    twisted/runner/procmontap.py
+    twisted/scripts/__init__.py
+    twisted/scripts/_twistd_unix.py
+    twisted/scripts/_twistw.py
+    twisted/scripts/htmlizer.py
+    twisted/scripts/trial.py
+    twisted/scripts/twistd.py
+    twisted/spread/__init__.py
+    twisted/spread/banana.py
+    twisted/spread/flavors.py
+    twisted/spread/interfaces.py
+    twisted/spread/jelly.py
+    twisted/spread/pb.py
+    twisted/spread/publish.py
+    twisted/spread/util.py
+    twisted/tap/__init__.py
+    twisted/tap/ftp.py
+    twisted/tap/portforward.py
+    twisted/tap/socks.py
+    twisted/trial/__init__.py
+    twisted/trial/__main__.py
+    twisted/trial/_asyncrunner.py
+    twisted/trial/_asynctest.py
+    twisted/trial/_dist/__init__.py
+    twisted/trial/_dist/distreporter.py
+    twisted/trial/_dist/disttrial.py
+    twisted/trial/_dist/managercommands.py
+    twisted/trial/_dist/options.py
+    twisted/trial/_dist/worker.py
+    twisted/trial/_dist/workercommands.py
+    twisted/trial/_dist/workerreporter.py
+    twisted/trial/_dist/workertrial.py
+    twisted/trial/_synctest.py
+    twisted/trial/itrial.py
+    twisted/trial/reporter.py
+    twisted/trial/runner.py
+    twisted/trial/unittest.py
+    twisted/trial/util.py
+    twisted/web/__init__.py
+    twisted/web/_auth/__init__.py
+    twisted/web/_auth/basic.py
+    twisted/web/_auth/digest.py
+    twisted/web/_auth/wrapper.py
+    twisted/web/_element.py
+    twisted/web/_flatten.py
+    twisted/web/_http2.py
+    twisted/web/_newclient.py
+    twisted/web/_responses.py
+    twisted/web/_stan.py
+    twisted/web/client.py
+    twisted/web/demo.py
+    twisted/web/distrib.py
+    twisted/web/domhelpers.py
+    twisted/web/error.py
+    twisted/web/guard.py
+    twisted/web/html.py
+    twisted/web/http.py
+    twisted/web/http_headers.py
+    twisted/web/iweb.py
+    twisted/web/microdom.py
+    twisted/web/proxy.py
+    twisted/web/resource.py
+    twisted/web/rewrite.py
+    twisted/web/script.py
+    twisted/web/server.py
+    twisted/web/soap.py
+    twisted/web/static.py
+    twisted/web/sux.py
+    twisted/web/tap.py
+    twisted/web/template.py
+    twisted/web/test/requesthelper.py
+    twisted/web/twcgi.py
+    twisted/web/util.py
+    twisted/web/vhost.py
+    twisted/web/wsgi.py
+    twisted/web/xmlrpc.py
+    twisted/words/__init__.py
+    twisted/words/ewords.py
+    twisted/words/im/__init__.py
+    twisted/words/im/baseaccount.py
+    twisted/words/im/basechat.py
+    twisted/words/im/basesupport.py
+    twisted/words/im/interfaces.py
+    twisted/words/im/ircsupport.py
+    twisted/words/im/locals.py
+    twisted/words/im/pbsupport.py
+    twisted/words/iwords.py
+    twisted/words/protocols/__init__.py
+    twisted/words/protocols/irc.py
+    twisted/words/protocols/jabber/__init__.py
+    twisted/words/protocols/jabber/client.py
+    twisted/words/protocols/jabber/component.py
+    twisted/words/protocols/jabber/error.py
+    twisted/words/protocols/jabber/ijabber.py
+    twisted/words/protocols/jabber/jid.py
+    twisted/words/protocols/jabber/jstrports.py
+    twisted/words/protocols/jabber/sasl.py
+    twisted/words/protocols/jabber/sasl_mechanisms.py
+    twisted/words/protocols/jabber/xmlstream.py
+    twisted/words/protocols/jabber/xmpp_stringprep.py
+    twisted/words/service.py
+    twisted/words/tap.py
+    twisted/words/xish/__init__.py
+    twisted/words/xish/domish.py
+    twisted/words/xish/utility.py
+    twisted/words/xish/xmlstream.py
+    twisted/words/xish/xpath.py
+    twisted/words/xish/xpathparser.py
+    twisted/words/xmpproutertap.py
+)
+
+IF (OS_WINDOWS)
+    SRCS(
+        twisted/internet/iocpreactor/iocpsupport/iocpsupport.c
+        twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c
+    )
+
+    PY_REGISTER(iocpsupport)
+ELSE()
+    SRCS(
+        twisted/python/_sendmsg.c
+    )
+
+    PY_REGISTER(_sendmsg)
+ENDIF ()
+
+RESOURCE_FILES(
+    PREFIX contrib/python/Twisted/py2/
+    .dist-info/METADATA
+    .dist-info/entry_points.txt
+    .dist-info/top_level.txt
+)
+
+END()
diff --git a/contrib/python/Twisted/py3/.dist-info/METADATA b/contrib/python/Twisted/py3/.dist-info/METADATA
new file mode 100644
index 00000000000..6f2beda880b
--- /dev/null
+++ b/contrib/python/Twisted/py3/.dist-info/METADATA
@@ -0,0 +1,224 @@
+Metadata-Version: 2.1
+Name: Twisted
+Version: 23.10.0
+Summary: An asynchronous networking framework written in Python
+Project-URL: Changelog, https://github.com/twisted/twisted/blob/HEAD/NEWS.rst
+Project-URL: Documentation, https://docs.twistedmatrix.com/
+Project-URL: Homepage, https://twistedmatrix.com/
+Project-URL: Issues, https://twistedmatrix.com/trac/report
+Project-URL: Source, https://github.com/twisted/twisted
+Project-URL: Twitter, https://twitter.com/twistedmatrix
+Author-email: Twisted Matrix Laboratories <twisted-python@twistedmatrix.com>
+License: MIT License
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8.0
+Requires-Dist: attrs>=21.3.0
+Requires-Dist: automat>=0.8.0
+Requires-Dist: constantly>=15.1
+Requires-Dist: hyperlink>=17.1.1
+Requires-Dist: incremental>=22.10.0
+Requires-Dist: twisted-iocpsupport<2,>=1.0.2; platform_system == 'Windows'
+Requires-Dist: typing-extensions>=4.2.0
+Requires-Dist: zope-interface>=5
+Provides-Extra: all-non-platform
+Requires-Dist: twisted[conch,http2,serial,test,tls]; extra == 'all-non-platform'
+Provides-Extra: all_non_platform
+Requires-Dist: twisted[conch,http2,serial,test,tls]; extra == 'all_non_platform'
+Provides-Extra: conch
+Requires-Dist: appdirs>=1.4.0; extra == 'conch'
+Requires-Dist: bcrypt>=3.1.3; extra == 'conch'
+Requires-Dist: cryptography>=3.3; extra == 'conch'
+Provides-Extra: dev
+Requires-Dist: coverage<7,>=6b1; extra == 'dev'
+Requires-Dist: pyflakes~=2.2; extra == 'dev'
+Requires-Dist: python-subunit~=1.4; extra == 'dev'
+Requires-Dist: twisted[dev-release]; extra == 'dev'
+Requires-Dist: twistedchecker~=0.7; extra == 'dev'
+Provides-Extra: dev-release
+Requires-Dist: pydoctor~=23.9.0; extra == 'dev-release'
+Requires-Dist: sphinx-rtd-theme~=1.3; extra == 'dev-release'
+Requires-Dist: sphinx<7,>=6; extra == 'dev-release'
+Requires-Dist: towncrier~=23.6; extra == 'dev-release'
+Provides-Extra: dev_release
+Requires-Dist: pydoctor~=23.9.0; extra == 'dev_release'
+Requires-Dist: sphinx-rtd-theme~=1.3; extra == 'dev_release'
+Requires-Dist: sphinx<7,>=6; extra == 'dev_release'
+Requires-Dist: towncrier~=23.6; extra == 'dev_release'
+Provides-Extra: gtk-platform
+Requires-Dist: pygobject; extra == 'gtk-platform'
+Requires-Dist: twisted[all-non-platform]; extra == 'gtk-platform'
+Provides-Extra: gtk_platform
+Requires-Dist: pygobject; extra == 'gtk_platform'
+Requires-Dist: twisted[all-non-platform]; extra == 'gtk_platform'
+Provides-Extra: http2
+Requires-Dist: h2<5.0,>=3.0; extra == 'http2'
+Requires-Dist: priority<2.0,>=1.1.0; extra == 'http2'
+Provides-Extra: macos-platform
+Requires-Dist: pyobjc-core; extra == 'macos-platform'
+Requires-Dist: pyobjc-framework-cfnetwork; extra == 'macos-platform'
+Requires-Dist: pyobjc-framework-cocoa; extra == 'macos-platform'
+Requires-Dist: twisted[all-non-platform]; extra == 'macos-platform'
+Provides-Extra: macos_platform
+Requires-Dist: pyobjc-core; extra == 'macos_platform'
+Requires-Dist: pyobjc-framework-cfnetwork; extra == 'macos_platform'
+Requires-Dist: pyobjc-framework-cocoa; extra == 'macos_platform'
+Requires-Dist: twisted[all-non-platform]; extra == 'macos_platform'
+Provides-Extra: mypy
+Requires-Dist: mypy-zope~=1.0.1; extra == 'mypy'
+Requires-Dist: mypy~=1.5.1; extra == 'mypy'
+Requires-Dist: twisted[all-non-platform,dev]; extra == 'mypy'
+Requires-Dist: types-pyopenssl; extra == 'mypy'
+Requires-Dist: types-setuptools; extra == 'mypy'
+Provides-Extra: osx-platform
+Requires-Dist: twisted[macos-platform]; extra == 'osx-platform'
+Provides-Extra: osx_platform
+Requires-Dist: twisted[macos-platform]; extra == 'osx_platform'
+Provides-Extra: serial
+Requires-Dist: pyserial>=3.0; extra == 'serial'
+Requires-Dist: pywin32!=226; platform_system == 'Windows' and extra == 'serial'
+Provides-Extra: test
+Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'test'
+Requires-Dist: hypothesis>=6.56; extra == 'test'
+Requires-Dist: pyhamcrest>=2; extra == 'test'
+Provides-Extra: tls
+Requires-Dist: idna>=2.4; extra == 'tls'
+Requires-Dist: pyopenssl>=21.0.0; extra == 'tls'
+Requires-Dist: service-identity>=18.1.0; extra == 'tls'
+Provides-Extra: windows-platform
+Requires-Dist: pywin32!=226; extra == 'windows-platform'
+Requires-Dist: twisted[all-non-platform]; extra == 'windows-platform'
+Provides-Extra: windows_platform
+Requires-Dist: pywin32!=226; extra == 'windows_platform'
+Requires-Dist: twisted[all-non-platform]; extra == 'windows_platform'
+Description-Content-Type: text/x-rst
+
+Twisted
+#######
+
+|gitter|_
+|rtd|_
+|pypi|_
+|ci|_
+
+For information on changes in this release, see the `NEWS <https://github.com/twisted/twisted/blob/trunk/NEWS.rst>`_ file.
+
+
+What is this?
+-------------
+
+Twisted is an event-based framework for internet applications, supporting Python 3.6+.
+It includes modules for many different purposes, including the following:
+
+- ``twisted.web``: HTTP clients and servers, HTML templating, and a WSGI server
+- ``twisted.conch``: SSHv2 and Telnet clients and servers and terminal emulators
+- ``twisted.words``: Clients and servers for IRC, XMPP, and other IM protocols
+- ``twisted.mail``: IMAPv4, POP3, SMTP clients and servers
+- ``twisted.positioning``: Tools for communicating with NMEA-compatible GPS receivers
+- ``twisted.names``: DNS client and tools for making your own DNS servers
+- ``twisted.trial``: A unit testing framework that integrates well with Twisted-based code.
+
+Twisted supports all major system event loops -- ``select`` (all platforms), ``poll`` (most POSIX platforms), ``epoll`` (Linux), ``kqueue`` (FreeBSD, macOS), IOCP (Windows), and various GUI event loops (GTK+2/3, Qt, wxWidgets).
+Third-party reactors can plug into Twisted, and provide support for additional event loops.
+
+
+Installing
+----------
+
+To install the latest version of Twisted using pip::
+
+  $ pip install twisted
+
+Additional instructions for installing this software are in `the installation instructions <https://github.com/twisted/twisted/blob/trunk/INSTALL.rst>`_.
+
+
+Documentation and Support
+-------------------------
+
+Twisted's documentation is available from the `Twisted Matrix website <https://twistedmatrix.com/documents/current/>`_.
+This documentation contains how-tos, code examples, and an API reference.
+
+Help is also available on the `Twisted mailing list <https://mail.python.org/mailman3/lists/twisted.python.org/>`_.
+
+There is also an IRC channel, ``#twisted``,
+on the `Libera.Chat <https://libera.chat/>`_ network.
+A web client is available at `web.libera.chat <https://web.libera.chat/>`_.
+
+
+Unit Tests
+----------
+
+Twisted has a comprehensive test suite, which can be run by ``tox``::
+
+  $ tox -l                       # to view all test environments
+  $ tox -e nocov                 # to run all the tests without coverage
+  $ tox -e withcov               # to run all the tests with coverage
+  $ tox -e alldeps-withcov-posix # install all dependencies, run tests with coverage on POSIX platform
+
+
+You can test running the test suite under the different reactors with the ``TWISTED_REACTOR`` environment variable::
+
+  $ env TWISTED_REACTOR=epoll tox -e alldeps-withcov-posix
+
+Some of these tests may fail if you:
+
+* don't have the dependencies required for a particular subsystem installed,
+* have a firewall blocking some ports (or things like Multicast, which Linux NAT has shown itself to do), or
+* run them as root.
+
+
+Static Code Checkers
+--------------------
+
+You can ensure that code complies to Twisted `coding standards <https://twistedmatrix.com/documents/current/core/development/policy/coding-standard.html>`_::
+
+  $ tox -e lint   # run pre-commit to check coding stanards
+  $ tox -e mypy   # run MyPy static type checker to check for type errors
+
+Or, for speed, use pre-commit directly::
+
+  $ pipx run pre-commit run
+
+
+Copyright
+---------
+
+All of the code in this distribution is Copyright (c) 2001-2023 Twisted Matrix Laboratories.
+
+Twisted is made available under the MIT license.
+The included `LICENSE <https://github.com/twisted/twisted/blob/trunk/LICENSE>`_ file describes this in detail.
+
+
+Warranty
+--------
+
+  THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER
+  EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+  TO THE USE OF THIS SOFTWARE IS WITH YOU.
+
+  IN NO EVENT WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+  AND/OR REDISTRIBUTE THE LIBRARY, BE LIABLE TO YOU FOR ANY DAMAGES, EVEN IF
+  SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+  DAMAGES.
+
+Again, see the included `LICENSE <https://github.com/twisted/twisted/blob/trunk/LICENSE>`_ file for specific legal details.
+
+
+.. |pypi| image:: https://img.shields.io/pypi/v/twisted.svg
+.. _pypi: https://pypi.python.org/pypi/twisted
+
+.. |gitter| image:: https://img.shields.io/gitter/room/twisted/twisted.svg
+.. _gitter: https://gitter.im/twisted/twisted
+
+.. |ci| image:: https://github.com/twisted/twisted/actions/workflows/test.yaml/badge.svg
+.. _ci: https://github.com/twisted/twisted
+
+.. |rtd| image:: https://readthedocs.org/projects/twisted/badge/?version=latest&style=flat
+.. _rtd: https://docs.twistedmatrix.com
diff --git a/contrib/python/Twisted/py3/.dist-info/entry_points.txt b/contrib/python/Twisted/py3/.dist-info/entry_points.txt
new file mode 100644
index 00000000000..c5448a75075
--- /dev/null
+++ b/contrib/python/Twisted/py3/.dist-info/entry_points.txt
@@ -0,0 +1,10 @@
+[console_scripts]
+cftp = twisted.conch.scripts.cftp:run
+ckeygen = twisted.conch.scripts.ckeygen:run
+conch = twisted.conch.scripts.conch:run
+mailmail = twisted.mail.scripts.mailmail:run
+pyhtmlizer = twisted.scripts.htmlizer:run
+tkconch = twisted.conch.scripts.tkconch:run
+trial = twisted.scripts.trial:run
+twist = twisted.application.twist._twist:Twist.main
+twistd = twisted.scripts.twistd:run
diff --git a/contrib/python/Twisted/py3/.dist-info/top_level.txt b/contrib/python/Twisted/py3/.dist-info/top_level.txt
new file mode 100644
index 00000000000..3eb29f049fb
--- /dev/null
+++ b/contrib/python/Twisted/py3/.dist-info/top_level.txt
@@ -0,0 +1 @@
+twisted
diff --git a/contrib/python/Twisted/py3/LICENSE b/contrib/python/Twisted/py3/LICENSE
new file mode 100644
index 00000000000..f3811313cd5
--- /dev/null
+++ b/contrib/python/Twisted/py3/LICENSE
@@ -0,0 +1,74 @@
+Copyright (c) 2001-2023
+Allen Short
+Amber Hawkie Brown
+Andrew Bennetts
+Andy Gayton
+Antoine Pitrou
+Apple Computer, Inc.
+Ashwini Oruganti
+Benjamin Bruheim
+Bob Ippolito
+Canonical Limited
+Christopher Armstrong
+Ciena Corporation
+David Reid
+Divmod Inc.
+Donovan Preston
+Eric Mangold
+Eyal Lotem
+Google Inc.
+Hybrid Logic Ltd.
+Hynek Schlawack
+Itamar Turner-Trauring
+James Knight
+Jason A. Mobarak
+Jean-Paul Calderone
+Jessica McKellar
+Jonathan D. Simms
+Jonathan Jacobs
+Jonathan Lange
+Julian Berman
+Jürgen Hermann
+Kevin Horn
+Kevin Turner
+Laurens Van Houtven
+Mary Gardiner
+Massachusetts Institute of Technology
+Matthew Lefkowitz
+Moshe Zadka
+Paul Swartz
+Pavel Pergamenshchik
+Rackspace, US Inc.
+Ralph Meijer
+Richard Wall
+Sean Riley
+Software Freedom Conservancy
+Tavendo GmbH
+Thijs Triemstra
+Thomas Grainger
+Thomas Herve
+Timothy Allen
+Tom Most
+Tom Prince
+Travis B. Hartwell
+
+and others that have contributed code to the public domain.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/python/Twisted/py3/README.rst b/contrib/python/Twisted/py3/README.rst
new file mode 100644
index 00000000000..73783ca21ab
--- /dev/null
+++ b/contrib/python/Twisted/py3/README.rst
@@ -0,0 +1,123 @@
+Twisted
+#######
+
+|gitter|_
+|rtd|_
+|pypi|_
+|ci|_
+
+For information on changes in this release, see the `NEWS <NEWS.rst>`_ file.
+
+
+What is this?
+-------------
+
+Twisted is an event-based framework for internet applications, supporting Python 3.6+.
+It includes modules for many different purposes, including the following:
+
+- ``twisted.web``: HTTP clients and servers, HTML templating, and a WSGI server
+- ``twisted.conch``: SSHv2 and Telnet clients and servers and terminal emulators
+- ``twisted.words``: Clients and servers for IRC, XMPP, and other IM protocols
+- ``twisted.mail``: IMAPv4, POP3, SMTP clients and servers
+- ``twisted.positioning``: Tools for communicating with NMEA-compatible GPS receivers
+- ``twisted.names``: DNS client and tools for making your own DNS servers
+- ``twisted.trial``: A unit testing framework that integrates well with Twisted-based code.
+
+Twisted supports all major system event loops -- ``select`` (all platforms), ``poll`` (most POSIX platforms), ``epoll`` (Linux), ``kqueue`` (FreeBSD, macOS), IOCP (Windows), and various GUI event loops (GTK+2/3, Qt, wxWidgets).
+Third-party reactors can plug into Twisted, and provide support for additional event loops.
+
+
+Installing
+----------
+
+To install the latest version of Twisted using pip::
+
+  $ pip install twisted
+
+Additional instructions for installing this software are in `the installation instructions <INSTALL.rst>`_.
+
+
+Documentation and Support
+-------------------------
+
+Twisted's documentation is available from the `Twisted Matrix website <https://twistedmatrix.com/documents/current/>`_.
+This documentation contains how-tos, code examples, and an API reference.
+
+Help is also available on the `Twisted mailing list <https://mail.python.org/mailman3/lists/twisted.python.org/>`_.
+
+There is also an IRC channel, ``#twisted``,
+on the `Libera.Chat <https://libera.chat/>`_ network.
+A web client is available at `web.libera.chat <https://web.libera.chat/>`_.
+
+
+Unit Tests
+----------
+
+Twisted has a comprehensive test suite, which can be run by ``tox``::
+
+  $ tox -l                       # to view all test environments
+  $ tox -e nocov                 # to run all the tests without coverage
+  $ tox -e withcov               # to run all the tests with coverage
+  $ tox -e alldeps-withcov-posix # install all dependencies, run tests with coverage on POSIX platform
+
+
+You can test running the test suite under the different reactors with the ``TWISTED_REACTOR`` environment variable::
+
+  $ env TWISTED_REACTOR=epoll tox -e alldeps-withcov-posix
+
+Some of these tests may fail if you:
+
+* don't have the dependencies required for a particular subsystem installed,
+* have a firewall blocking some ports (or things like Multicast, which Linux NAT has shown itself to do), or
+* run them as root.
+
+
+Static Code Checkers
+--------------------
+
+You can ensure that code complies to Twisted `coding standards <https://twistedmatrix.com/documents/current/core/development/policy/coding-standard.html>`_::
+
+  $ tox -e lint   # run pre-commit to check coding stanards
+  $ tox -e mypy   # run MyPy static type checker to check for type errors
+
+Or, for speed, use pre-commit directly::
+
+  $ pipx run pre-commit run
+
+
+Copyright
+---------
+
+All of the code in this distribution is Copyright (c) 2001-2023 Twisted Matrix Laboratories.
+
+Twisted is made available under the MIT license.
+The included `LICENSE <LICENSE>`_ file describes this in detail.
+
+
+Warranty
+--------
+
+  THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER
+  EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+  TO THE USE OF THIS SOFTWARE IS WITH YOU.
+
+  IN NO EVENT WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+  AND/OR REDISTRIBUTE THE LIBRARY, BE LIABLE TO YOU FOR ANY DAMAGES, EVEN IF
+  SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+  DAMAGES.
+
+Again, see the included `LICENSE <LICENSE>`_ file for specific legal details.
+
+
+.. |pypi| image:: https://img.shields.io/pypi/v/twisted.svg
+.. _pypi: https://pypi.python.org/pypi/twisted
+
+.. |gitter| image:: https://img.shields.io/gitter/room/twisted/twisted.svg
+.. _gitter: https://gitter.im/twisted/twisted
+
+.. |ci| image:: https://github.com/twisted/twisted/actions/workflows/test.yaml/badge.svg
+.. _ci: https://github.com/twisted/twisted
+
+.. |rtd| image:: https://readthedocs.org/projects/twisted/badge/?version=latest&style=flat
+.. _rtd: https://docs.twistedmatrix.com
diff --git a/contrib/python/Twisted/py3/twisted/11715.misc b/contrib/python/Twisted/py3/twisted/11715.misc
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/Twisted/py3/twisted/__init__.py b/contrib/python/Twisted/py3/twisted/__init__.py
new file mode 100644
index 00000000000..1ecb96a511c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/__init__.py
@@ -0,0 +1,12 @@
+# -*- test-case-name: twisted -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted: The Framework Of Your Internet.
+"""
+
+from twisted._version import __version__ as version
+
+__version__ = version.short()
diff --git a/contrib/python/Twisted/py3/twisted/__main__.py b/contrib/python/Twisted/py3/twisted/__main__.py
new file mode 100644
index 00000000000..7f60ea2afb8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/__main__.py
@@ -0,0 +1,14 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+# Make the twisted module executable with the default behaviour of
+# running twist.
+# This is not a docstring to avoid changing the string output of twist.
+
+
+import sys
+
+if __name__ == "__main__":
+    from twisted.application.twist._twist import Twist
+
+    sys.exit(Twist.main())
diff --git a/contrib/python/Twisted/py3/twisted/_threads/__init__.py b/contrib/python/Twisted/py3/twisted/_threads/__init__.py
new file mode 100644
index 00000000000..13c7c7dda5a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/_threads/__init__.py
@@ -0,0 +1,24 @@
+# -*- test-case-name: twisted.test.test_paths -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted integration with operating system threads.
+"""
+
+
+from ._ithreads import AlreadyQuit, IWorker
+from ._memory import createMemoryWorker
+from ._pool import pool
+from ._team import Team
+from ._threadworker import LockWorker, ThreadWorker
+
+__all__ = [
+    "ThreadWorker",
+    "LockWorker",
+    "IWorker",
+    "AlreadyQuit",
+    "Team",
+    "createMemoryWorker",
+    "pool",
+]
diff --git a/contrib/python/Twisted/py3/twisted/_threads/_convenience.py b/contrib/python/Twisted/py3/twisted/_threads/_convenience.py
new file mode 100644
index 00000000000..deff5764624
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/_threads/_convenience.py
@@ -0,0 +1,43 @@
+# -*- test-case-name: twisted._threads.test.test_convenience -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Common functionality used within the implementation of various workers.
+"""
+
+
+from ._ithreads import AlreadyQuit
+
+
+class Quit:
+    """
+    A flag representing whether a worker has been quit.
+
+    @ivar isSet: Whether this flag is set.
+    @type isSet: L{bool}
+    """
+
+    def __init__(self):
+        """
+        Create a L{Quit} un-set.
+        """
+        self.isSet = False
+
+    def set(self):
+        """
+        Set the flag if it has not been set.
+
+        @raise AlreadyQuit: If it has been set.
+        """
+        self.check()
+        self.isSet = True
+
+    def check(self):
+        """
+        Check if the flag has been set.
+
+        @raise AlreadyQuit: If it has been set.
+        """
+        if self.isSet:
+            raise AlreadyQuit()
diff --git a/contrib/python/Twisted/py3/twisted/_threads/_ithreads.py b/contrib/python/Twisted/py3/twisted/_threads/_ithreads.py
new file mode 100644
index 00000000000..cab9135f874
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/_threads/_ithreads.py
@@ -0,0 +1,61 @@
+# -*- test-case-name: twisted._threads.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Interfaces related to threads.
+"""
+
+
+from typing import Callable
+
+from zope.interface import Interface
+
+
+class AlreadyQuit(Exception):
+    """
+    This worker worker is dead and cannot execute more instructions.
+    """
+
+
+class IWorker(Interface):
+    """
+    A worker that can perform some work concurrently.
+
+    All methods on this interface must be thread-safe.
+    """
+
+    def do(task: Callable[[], None]) -> None:
+        """
+        Perform the given task.
+
+        As an interface, this method makes no specific claims about concurrent
+        execution.  An L{IWorker}'s C{do} implementation may defer execution
+        for later on the same thread, immediately on a different thread, or
+        some combination of the two.  It is valid for a C{do} method to
+        schedule C{task} in such a way that it may never be executed.
+
+        It is important for some implementations to provide specific properties
+        with respect to where C{task} is executed, of course, and client code
+        may rely on a more specific implementation of C{do} than L{IWorker}.
+
+        @param task: a task to call in a thread or other concurrent context.
+        @type task: 0-argument callable
+
+        @raise AlreadyQuit: if C{quit} has been called.
+        """
+
+    def quit():
+        """
+        Free any resources associated with this L{IWorker} and cause it to
+        reject all future work.
+
+        @raise AlreadyQuit: if this method has already been called.
+        """
+
+
+class IExclusiveWorker(IWorker):
+    """
+    Like L{IWorker}, but with the additional guarantee that the callables
+    passed to C{do} will not be called exclusively with each other.
+    """
diff --git a/contrib/python/Twisted/py3/twisted/_threads/_memory.py b/contrib/python/Twisted/py3/twisted/_threads/_memory.py
new file mode 100644
index 00000000000..4c56db02ae9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/_threads/_memory.py
@@ -0,0 +1,70 @@
+# -*- test-case-name: twisted._threads.test.test_memory -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of an in-memory worker that defers execution.
+"""
+
+
+from zope.interface import implementer
+
+from . import IWorker
+from ._convenience import Quit
+
+NoMoreWork = object()
+
+
+@implementer(IWorker)
+class MemoryWorker:
+    """
+    An L{IWorker} that queues work for later performance.
+
+    @ivar _quit: a flag indicating
+    @type _quit: L{Quit}
+    """
+
+    def __init__(self, pending=list):
+        """
+        Create a L{MemoryWorker}.
+        """
+        self._quit = Quit()
+        self._pending = pending()
+
+    def do(self, work):
+        """
+        Queue some work for to perform later; see L{createMemoryWorker}.
+
+        @param work: The work to perform.
+        """
+        self._quit.check()
+        self._pending.append(work)
+
+    def quit(self):
+        """
+        Quit this worker.
+        """
+        self._quit.set()
+        self._pending.append(NoMoreWork)
+
+
+def createMemoryWorker():
+    """
+    Create an L{IWorker} that does nothing but defer work, to be performed
+    later.
+
+    @return: a worker that will enqueue work to perform later, and a callable
+        that will perform one element of that work.
+    @rtype: 2-L{tuple} of (L{IWorker}, L{callable})
+    """
+
+    def perform():
+        if not worker._pending:
+            return False
+        if worker._pending[0] is NoMoreWork:
+            return False
+        worker._pending.pop(0)()
+        return True
+
+    worker = MemoryWorker()
+    return (worker, perform)
diff --git a/contrib/python/Twisted/py3/twisted/_threads/_pool.py b/contrib/python/Twisted/py3/twisted/_threads/_pool.py
new file mode 100644
index 00000000000..99c055d2404
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/_threads/_pool.py
@@ -0,0 +1,73 @@
+# -*- test-case-name: twisted._threads.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Top level thread pool interface, used to implement
+L{twisted.python.threadpool}.
+"""
+
+
+from queue import Queue
+from threading import Lock, Thread, local as LocalStorage
+from typing import Callable, Optional
+
+from typing_extensions import Protocol
+
+from twisted.python.log import err
+from ._ithreads import IWorker
+from ._team import Team
+from ._threadworker import LockWorker, ThreadWorker
+
+
+class _ThreadFactory(Protocol):
+    def __call__(self, *, target: Callable[..., object]) -> Thread:
+        ...
+
+
+def pool(
+    currentLimit: Callable[[], int], threadFactory: _ThreadFactory = Thread
+) -> Team:
+    """
+    Construct a L{Team} that spawns threads as a thread pool, with the given
+    limiting function.
+
+    @note: Future maintainers: while the public API for the eventual move to
+        twisted.threads should look I{something} like this, and while this
+        function is necessary to implement the API described by
+        L{twisted.python.threadpool}, I am starting to think the idea of a hard
+        upper limit on threadpool size is just bad (turning memory performance
+        issues into correctness issues well before we run into memory
+        pressure), and instead we should build something with reactor
+        integration for slowly releasing idle threads when they're not needed
+        and I{rate} limiting the creation of new threads rather than just
+        hard-capping it.
+
+    @param currentLimit: a callable that returns the current limit on the
+        number of workers that the returned L{Team} should create; if it
+        already has more workers than that value, no new workers will be
+        created.
+    @type currentLimit: 0-argument callable returning L{int}
+
+    @param threadFactory: Factory that, when given a C{target} keyword argument,
+        returns a L{threading.Thread} that will run that target.
+    @type threadFactory: callable returning a L{threading.Thread}
+
+    @return: a new L{Team}.
+    """
+
+    def startThread(target: Callable[..., object]) -> None:
+        return threadFactory(target=target).start()
+
+    def limitedWorkerCreator() -> Optional[IWorker]:
+        stats = team.statistics()
+        if stats.busyWorkerCount + stats.idleWorkerCount >= currentLimit():
+            return None
+        return ThreadWorker(startThread, Queue())
+
+    team = Team(
+        coordinator=LockWorker(Lock(), LocalStorage()),
+        createWorker=limitedWorkerCreator,
+        logException=err,
+    )
+    return team
diff --git a/contrib/python/Twisted/py3/twisted/_threads/_team.py b/contrib/python/Twisted/py3/twisted/_threads/_team.py
new file mode 100644
index 00000000000..d15ae04242d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/_threads/_team.py
@@ -0,0 +1,232 @@
+# -*- test-case-name: twisted._threads.test.test_team -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of a L{Team} of workers; a thread-pool that can allocate work to
+workers.
+"""
+from __future__ import annotations
+
+from collections import deque
+from typing import Callable, Optional, Set
+
+from zope.interface import implementer
+
+from . import IWorker
+from ._convenience import Quit
+from ._ithreads import IExclusiveWorker
+
+
+class Statistics:
+    """
+    Statistics about a L{Team}'s current activity.
+
+    @ivar idleWorkerCount: The number of idle workers.
+    @type idleWorkerCount: L{int}
+
+    @ivar busyWorkerCount: The number of busy workers.
+    @type busyWorkerCount: L{int}
+
+    @ivar backloggedWorkCount: The number of work items passed to L{Team.do}
+        which have not yet been sent to a worker to be performed because not
+        enough workers are available.
+    @type backloggedWorkCount: L{int}
+    """
+
+    def __init__(
+        self, idleWorkerCount: int, busyWorkerCount: int, backloggedWorkCount: int
+    ) -> None:
+        self.idleWorkerCount = idleWorkerCount
+        self.busyWorkerCount = busyWorkerCount
+        self.backloggedWorkCount = backloggedWorkCount
+
+
+@implementer(IWorker)
+class Team:
+    """
+    A composite L{IWorker} implementation.
+
+    @ivar _quit: A L{Quit} flag indicating whether this L{Team} has been quit
+        yet.  This may be set by an arbitrary thread since L{Team.quit} may be
+        called from anywhere.
+
+    @ivar _coordinator: the L{IExclusiveWorker} coordinating access to this
+        L{Team}'s internal resources.
+
+    @ivar _createWorker: a callable that will create new workers.
+
+    @ivar _logException: a 0-argument callable called in an exception context
+        when there is an unhandled error from a task passed to L{Team.do}
+
+    @ivar _idle: a L{set} of idle workers.
+
+    @ivar _busyCount: the number of workers currently busy.
+
+    @ivar _pending: a C{deque} of tasks - that is, 0-argument callables passed
+        to L{Team.do} - that are outstanding.
+
+    @ivar _shouldQuitCoordinator: A flag indicating that the coordinator should
+        be quit at the next available opportunity.  Unlike L{Team._quit}, this
+        flag is only set by the coordinator.
+
+    @ivar _toShrink: the number of workers to shrink this L{Team} by at the
+        next available opportunity; set in the coordinator.
+    """
+
+    def __init__(
+        self,
+        coordinator: IExclusiveWorker,
+        createWorker: Callable[[], Optional[IWorker]],
+        logException: Callable[[], None],
+    ):
+        """
+        @param coordinator: an L{IExclusiveWorker} which will coordinate access
+            to resources on this L{Team}; that is to say, an
+            L{IExclusiveWorker} whose C{do} method ensures that its given work
+            will be executed in a mutually exclusive context, not in parallel
+            with other work enqueued by C{do} (although possibly in parallel
+            with the caller).
+
+        @param createWorker: A 0-argument callable that will create an
+            L{IWorker} to perform work.
+
+        @param logException: A 0-argument callable called in an exception
+            context when the work passed to C{do} raises an exception.
+        """
+        self._quit = Quit()
+        self._coordinator = coordinator
+        self._createWorker = createWorker
+        self._logException = logException
+
+        # Don't touch these except from the coordinator.
+        self._idle: Set[IWorker] = set()
+        self._busyCount = 0
+        self._pending: "deque[Callable[..., object]]" = deque()
+        self._shouldQuitCoordinator = False
+        self._toShrink = 0
+
+    def statistics(self) -> Statistics:
+        """
+        Gather information on the current status of this L{Team}.
+
+        @return: a L{Statistics} describing the current state of this L{Team}.
+        """
+        return Statistics(len(self._idle), self._busyCount, len(self._pending))
+
+    def grow(self, n: int) -> None:
+        """
+        Increase the the number of idle workers by C{n}.
+
+        @param n: The number of new idle workers to create.
+        @type n: L{int}
+        """
+        self._quit.check()
+
+        @self._coordinator.do
+        def createOneWorker() -> None:
+            for x in range(n):
+                worker = self._createWorker()
+                if worker is None:
+                    return
+                self._recycleWorker(worker)
+
+    def shrink(self, n: Optional[int] = None) -> None:
+        """
+        Decrease the number of idle workers by C{n}.
+
+        @param n: The number of idle workers to shut down, or L{None} (or
+            unspecified) to shut down all workers.
+        @type n: L{int} or L{None}
+        """
+        self._quit.check()
+        self._coordinator.do(lambda: self._quitIdlers(n))
+
+    def _quitIdlers(self, n: Optional[int] = None) -> None:
+        """
+        The implmentation of C{shrink}, performed by the coordinator worker.
+
+        @param n: see L{Team.shrink}
+        """
+        if n is None:
+            n = len(self._idle) + self._busyCount
+        for x in range(n):
+            if self._idle:
+                self._idle.pop().quit()
+            else:
+                self._toShrink += 1
+        if self._shouldQuitCoordinator and self._busyCount == 0:
+            self._coordinator.quit()
+
+    def do(self, task: Callable[[], None]) -> None:
+        """
+        Perform some work in a worker created by C{createWorker}.
+
+        @param task: the callable to run
+        """
+        self._quit.check()
+        self._coordinator.do(lambda: self._coordinateThisTask(task))
+
+    def _coordinateThisTask(self, task: Callable[..., object]) -> None:
+        """
+        Select a worker to dispatch to, either an idle one or a new one, and
+        perform it.
+
+        This method should run on the coordinator worker.
+
+        @param task: the task to dispatch
+        @type task: 0-argument callable
+        """
+        worker = self._idle.pop() if self._idle else self._createWorker()
+        if worker is None:
+            # The createWorker method may return None if we're out of resources
+            # to create workers.
+            self._pending.append(task)
+            return
+        not_none_worker = worker
+        self._busyCount += 1
+
+        @worker.do
+        def doWork() -> None:
+            try:
+                task()
+            except BaseException:
+                self._logException()
+
+            @self._coordinator.do
+            def idleAndPending() -> None:
+                self._busyCount -= 1
+                self._recycleWorker(not_none_worker)
+
+    def _recycleWorker(self, worker: IWorker) -> None:
+        """
+        Called only from coordinator.
+
+        Recycle the given worker into the idle pool.
+
+        @param worker: a worker created by C{createWorker} and now idle.
+        @type worker: L{IWorker}
+        """
+        self._idle.add(worker)
+        if self._pending:
+            # Re-try the first enqueued thing.
+            # (Explicitly do _not_ honor _quit.)
+            self._coordinateThisTask(self._pending.popleft())
+        elif self._shouldQuitCoordinator:
+            self._quitIdlers()
+        elif self._toShrink > 0:
+            self._toShrink -= 1
+            self._idle.remove(worker)
+            worker.quit()
+
+    def quit(self) -> None:
+        """
+        Stop doing work and shut down all idle workers.
+        """
+        self._quit.set()
+        # In case all the workers are idle when we do this.
+
+        @self._coordinator.do
+        def startFinishing() -> None:
+            self._shouldQuitCoordinator = True
+            self._quitIdlers()
diff --git a/contrib/python/Twisted/py3/twisted/_threads/_threadworker.py b/contrib/python/Twisted/py3/twisted/_threads/_threadworker.py
new file mode 100644
index 00000000000..e7ffc097580
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/_threads/_threadworker.py
@@ -0,0 +1,121 @@
+# -*- test-case-name: twisted._threads.test.test_threadworker -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of an L{IWorker} based on native threads and queues.
+"""
+
+
+from typing import Callable
+
+from zope.interface import implementer
+
+from ._convenience import Quit
+from ._ithreads import IExclusiveWorker
+
+_stop = object()
+
+
+@implementer(IExclusiveWorker)
+class ThreadWorker:
+    """
+    An L{IExclusiveWorker} implemented based on a single thread and a queue.
+
+    This worker ensures exclusivity (i.e. it is an L{IExclusiveWorker} and not
+    an L{IWorker}) by performing all of the work passed to C{do} on the I{same}
+    thread.
+    """
+
+    def __init__(self, startThread, queue):
+        """
+        Create a L{ThreadWorker} with a function to start a thread and a queue
+        to use to communicate with that thread.
+
+        @param startThread: a callable that takes a callable to run in another
+            thread.
+        @type startThread: callable taking a 0-argument callable and returning
+            nothing.
+
+        @param queue: A L{Queue} to use to give tasks to the thread created by
+            C{startThread}.
+        @type queue: L{Queue}
+        """
+        self._q = queue
+        self._hasQuit = Quit()
+
+        def work():
+            for task in iter(queue.get, _stop):
+                task()
+
+        startThread(work)
+
+    def do(self, task: Callable[[], None]) -> None:
+        """
+        Perform the given task on the thread owned by this L{ThreadWorker}.
+
+        @param task: the function to call on a thread.
+        """
+        self._hasQuit.check()
+        self._q.put(task)
+
+    def quit(self):
+        """
+        Reject all future work and stop the thread started by C{__init__}.
+        """
+        # Reject all future work.  Set this _before_ enqueueing _stop, so
+        # that no work is ever enqueued _after_ _stop.
+        self._hasQuit.set()
+        self._q.put(_stop)
+
+
+@implementer(IExclusiveWorker)
+class LockWorker:
+    """
+    An L{IWorker} implemented based on a mutual-exclusion lock.
+    """
+
+    def __init__(self, lock, local):
+        """
+        @param lock: A mutual-exclusion lock, with C{acquire} and C{release}
+            methods.
+        @type lock: L{threading.Lock}
+
+        @param local: Local storage.
+        @type local: L{threading.local}
+        """
+        self._quit = Quit()
+        self._lock = lock
+        self._local = local
+
+    def do(self, work: Callable[[], None]) -> None:
+        """
+        Do the given work on this thread, with the mutex acquired.  If this is
+        called re-entrantly, return and wait for the outer invocation to do the
+        work.
+
+        @param work: the work to do with the lock held.
+        """
+        lock = self._lock
+        local = self._local
+        self._quit.check()
+        working = getattr(local, "working", None)
+        if working is None:
+            working = local.working = []
+            working.append(work)
+            lock.acquire()
+            try:
+                while working:
+                    working.pop(0)()
+            finally:
+                lock.release()
+                local.working = None
+        else:
+            working.append(work)
+
+    def quit(self):
+        """
+        Quit this L{LockWorker}.
+        """
+        self._quit.set()
+        self._lock = None
diff --git a/contrib/python/Twisted/py3/twisted/_version.py b/contrib/python/Twisted/py3/twisted/_version.py
new file mode 100644
index 00000000000..3f2476fe9ca
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/_version.py
@@ -0,0 +1,11 @@
+"""
+Provides Twisted version information.
+"""
+
+# This file is auto-generated! Do not edit!
+# Use `python -m incremental.update Twisted` to change this file.
+
+from incremental import Version
+
+__version__ = Version("Twisted", 23, 10, 0)
+__all__ = ["__version__"]
diff --git a/contrib/python/Twisted/py3/twisted/application/__init__.py b/contrib/python/Twisted/py3/twisted/application/__init__.py
new file mode 100644
index 00000000000..a462fa6b24a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Configuration objects for Twisted Applications.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/application/app.py b/contrib/python/Twisted/py3/twisted/application/app.py
new file mode 100644
index 00000000000..e07e594d574
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/app.py
@@ -0,0 +1,706 @@
+# -*- test-case-name: twisted.test.test_application,twisted.test.test_twistd -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+import getpass
+import os
+import pdb
+import signal
+import sys
+import traceback
+import warnings
+from operator import attrgetter
+
+from twisted import copyright, logger, plugin
+from twisted.application import reactors, service
+
+# Expose the new implementation of installReactor at the old location.
+from twisted.application.reactors import NoSuchReactor, installReactor
+from twisted.internet import defer
+from twisted.internet.interfaces import _ISupportsExitSignalCapturing
+from twisted.persisted import sob
+from twisted.python import failure, log, logfile, runtime, usage, util
+from twisted.python.reflect import namedAny, namedModule, qual
+
+
+class _BasicProfiler:
+    """
+    @ivar saveStats: if C{True}, save the stats information instead of the
+        human readable format
+    @type saveStats: C{bool}
+
+    @ivar profileOutput: the name of the file use to print profile data.
+    @type profileOutput: C{str}
+    """
+
+    def __init__(self, profileOutput, saveStats):
+        self.profileOutput = profileOutput
+        self.saveStats = saveStats
+
+    def _reportImportError(self, module, e):
+        """
+        Helper method to report an import error with a profile module. This
+        has to be explicit because some of these modules are removed by
+        distributions due to them being non-free.
+        """
+        s = f"Failed to import module {module}: {e}"
+        s += """
+This is most likely caused by your operating system not including
+the module due to it being non-free. Either do not use the option
+--profile, or install the module; your operating system vendor
+may provide it in a separate package.
+"""
+        raise SystemExit(s)
+
+
+class ProfileRunner(_BasicProfiler):
+    """
+    Runner for the standard profile module.
+    """
+
+    def run(self, reactor):
+        """
+        Run reactor under the standard profiler.
+        """
+        try:
+            import profile
+        except ImportError as e:
+            self._reportImportError("profile", e)
+
+        p = profile.Profile()
+        p.runcall(reactor.run)
+        if self.saveStats:
+            p.dump_stats(self.profileOutput)
+        else:
+            tmp, sys.stdout = sys.stdout, open(self.profileOutput, "a")
+            try:
+                p.print_stats()
+            finally:
+                sys.stdout, tmp = tmp, sys.stdout
+                tmp.close()
+
+
+class CProfileRunner(_BasicProfiler):
+    """
+    Runner for the cProfile module.
+    """
+
+    def run(self, reactor):
+        """
+        Run reactor under the cProfile profiler.
+        """
+        try:
+            import cProfile
+            import pstats
+        except ImportError as e:
+            self._reportImportError("cProfile", e)
+
+        p = cProfile.Profile()
+        p.runcall(reactor.run)
+        if self.saveStats:
+            p.dump_stats(self.profileOutput)
+        else:
+            with open(self.profileOutput, "w") as stream:
+                s = pstats.Stats(p, stream=stream)
+                s.strip_dirs()
+                s.sort_stats(-1)
+                s.print_stats()
+
+
+class AppProfiler:
+    """
+    Class which selects a specific profile runner based on configuration
+    options.
+
+    @ivar profiler: the name of the selected profiler.
+    @type profiler: C{str}
+    """
+
+    profilers = {"profile": ProfileRunner, "cprofile": CProfileRunner}
+
+    def __init__(self, options):
+        saveStats = options.get("savestats", False)
+        profileOutput = options.get("profile", None)
+        self.profiler = options.get("profiler", "cprofile").lower()
+        if self.profiler in self.profilers:
+            profiler = self.profilers[self.profiler](profileOutput, saveStats)
+            self.run = profiler.run
+        else:
+            raise SystemExit(f"Unsupported profiler name: {self.profiler}")
+
+
+class AppLogger:
+    """
+    An L{AppLogger} attaches the configured log observer specified on the
+    commandline to a L{ServerOptions} object, a custom L{logger.ILogObserver},
+    or a legacy custom {log.ILogObserver}.
+
+    @ivar _logfilename: The name of the file to which to log, if other than the
+        default.
+    @type _logfilename: C{str}
+
+    @ivar _observerFactory: Callable object that will create a log observer, or
+        None.
+
+    @ivar _observer: log observer added at C{start} and removed at C{stop}.
+    @type _observer: a callable that implements L{logger.ILogObserver} or
+        L{log.ILogObserver}.
+    """
+
+    _observer = None
+
+    def __init__(self, options):
+        """
+        Initialize an L{AppLogger} with a L{ServerOptions}.
+        """
+        self._logfilename = options.get("logfile", "")
+        self._observerFactory = options.get("logger") or None
+
+    def start(self, application):
+        """
+        Initialize the global logging system for the given application.
+
+        If a custom logger was specified on the command line it will be used.
+        If not, and an L{logger.ILogObserver} or legacy L{log.ILogObserver}
+        component has been set on C{application}, then it will be used as the
+        log observer. Otherwise a log observer will be created based on the
+        command line options for built-in loggers (e.g. C{--logfile}).
+
+        @param application: The application on which to check for an
+            L{logger.ILogObserver} or legacy L{log.ILogObserver}.
+        @type application: L{twisted.python.components.Componentized}
+        """
+        if self._observerFactory is not None:
+            observer = self._observerFactory()
+        else:
+            observer = application.getComponent(logger.ILogObserver, None)
+            if observer is None:
+                # If there's no new ILogObserver, try the legacy one
+                observer = application.getComponent(log.ILogObserver, None)
+
+        if observer is None:
+            observer = self._getLogObserver()
+        self._observer = observer
+
+        if logger.ILogObserver.providedBy(self._observer):
+            observers = [self._observer]
+        elif log.ILogObserver.providedBy(self._observer):
+            observers = [logger.LegacyLogObserverWrapper(self._observer)]
+        else:
+            warnings.warn(
+                (
+                    "Passing a logger factory which makes log observers which do "
+                    "not implement twisted.logger.ILogObserver or "
+                    "twisted.python.log.ILogObserver to "
+                    "twisted.application.app.AppLogger was deprecated in "
+                    "Twisted 16.2. Please use a factory that produces "
+                    "twisted.logger.ILogObserver (or the legacy "
+                    "twisted.python.log.ILogObserver) implementing objects "
+                    "instead."
+                ),
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            observers = [logger.LegacyLogObserverWrapper(self._observer)]
+
+        logger.globalLogBeginner.beginLoggingTo(observers)
+        self._initialLog()
+
+    def _initialLog(self):
+        """
+        Print twistd start log message.
+        """
+        from twisted.internet import reactor
+
+        logger._loggerFor(self).info(
+            "twistd {version} ({exe} {pyVersion}) starting up.",
+            version=copyright.version,
+            exe=sys.executable,
+            pyVersion=runtime.shortPythonVersion(),
+        )
+        logger._loggerFor(self).info(
+            "reactor class: {reactor}.", reactor=qual(reactor.__class__)
+        )
+
+    def _getLogObserver(self):
+        """
+        Create a log observer to be added to the logging system before running
+        this application.
+        """
+        if self._logfilename == "-" or not self._logfilename:
+            logFile = sys.stdout
+        else:
+            logFile = logfile.LogFile.fromFullPath(self._logfilename)
+        return logger.textFileLogObserver(logFile)
+
+    def stop(self):
+        """
+        Remove all log observers previously set up by L{AppLogger.start}.
+        """
+        logger._loggerFor(self).info("Server Shut Down.")
+        if self._observer is not None:
+            logger.globalLogPublisher.removeObserver(self._observer)
+            self._observer = None
+
+
+def fixPdb():
+    def do_stop(self, arg):
+        self.clear_all_breaks()
+        self.set_continue()
+        from twisted.internet import reactor
+
+        reactor.callLater(0, reactor.stop)
+        return 1
+
+    def help_stop(self):
+        print(
+            "stop - Continue execution, then cleanly shutdown the twisted " "reactor."
+        )
+
+    def set_quit(self):
+        os._exit(0)
+
+    pdb.Pdb.set_quit = set_quit
+    pdb.Pdb.do_stop = do_stop
+    pdb.Pdb.help_stop = help_stop
+
+
+def runReactorWithLogging(config, oldstdout, oldstderr, profiler=None, reactor=None):
+    """
+    Start the reactor, using profiling if specified by the configuration, and
+    log any error happening in the process.
+
+    @param config: configuration of the twistd application.
+    @type config: L{ServerOptions}
+
+    @param oldstdout: initial value of C{sys.stdout}.
+    @type oldstdout: C{file}
+
+    @param oldstderr: initial value of C{sys.stderr}.
+    @type oldstderr: C{file}
+
+    @param profiler: object used to run the reactor with profiling.
+    @type profiler: L{AppProfiler}
+
+    @param reactor: The reactor to use.  If L{None}, the global reactor will
+        be used.
+    """
+    if reactor is None:
+        from twisted.internet import reactor
+    try:
+        if config["profile"]:
+            if profiler is not None:
+                profiler.run(reactor)
+        elif config["debug"]:
+            sys.stdout = oldstdout
+            sys.stderr = oldstderr
+            if runtime.platformType == "posix":
+                signal.signal(signal.SIGUSR2, lambda *args: pdb.set_trace())
+                signal.signal(signal.SIGINT, lambda *args: pdb.set_trace())
+            fixPdb()
+            pdb.runcall(reactor.run)
+        else:
+            reactor.run()
+    except BaseException:
+        close = False
+        if config["nodaemon"]:
+            file = oldstdout
+        else:
+            file = open("TWISTD-CRASH.log", "a")
+            close = True
+        try:
+            traceback.print_exc(file=file)
+            file.flush()
+        finally:
+            if close:
+                file.close()
+
+
+def getPassphrase(needed):
+    if needed:
+        return getpass.getpass("Passphrase: ")
+    else:
+        return None
+
+
+def getSavePassphrase(needed):
+    if needed:
+        return util.getPassword("Encryption passphrase: ")
+    else:
+        return None
+
+
+class ApplicationRunner:
+    """
+    An object which helps running an application based on a config object.
+
+    Subclass me and implement preApplication and postApplication
+    methods. postApplication generally will want to run the reactor
+    after starting the application.
+
+    @ivar config: The config object, which provides a dict-like interface.
+
+    @ivar application: Available in postApplication, but not
+       preApplication. This is the application object.
+
+    @ivar profilerFactory: Factory for creating a profiler object, able to
+        profile the application if options are set accordingly.
+
+    @ivar profiler: Instance provided by C{profilerFactory}.
+
+    @ivar loggerFactory: Factory for creating object responsible for logging.
+
+    @ivar logger: Instance provided by C{loggerFactory}.
+    """
+
+    profilerFactory = AppProfiler
+    loggerFactory = AppLogger
+
+    def __init__(self, config):
+        self.config = config
+        self.profiler = self.profilerFactory(config)
+        self.logger = self.loggerFactory(config)
+
+    def run(self):
+        """
+        Run the application.
+        """
+        self.preApplication()
+        self.application = self.createOrGetApplication()
+
+        self.logger.start(self.application)
+
+        self.postApplication()
+        self.logger.stop()
+
+    def startReactor(self, reactor, oldstdout, oldstderr):
+        """
+        Run the reactor with the given configuration.  Subclasses should
+        probably call this from C{postApplication}.
+
+        @see: L{runReactorWithLogging}
+        """
+        if reactor is None:
+            from twisted.internet import reactor
+        runReactorWithLogging(self.config, oldstdout, oldstderr, self.profiler, reactor)
+
+        if _ISupportsExitSignalCapturing.providedBy(reactor):
+            self._exitSignal = reactor._exitSignal
+        else:
+            self._exitSignal = None
+
+    def preApplication(self):
+        """
+        Override in subclass.
+
+        This should set up any state necessary before loading and
+        running the Application.
+        """
+        raise NotImplementedError()
+
+    def postApplication(self):
+        """
+        Override in subclass.
+
+        This will be called after the application has been loaded (so
+        the C{application} attribute will be set). Generally this
+        should start the application and run the reactor.
+        """
+        raise NotImplementedError()
+
+    def createOrGetApplication(self):
+        """
+        Create or load an Application based on the parameters found in the
+        given L{ServerOptions} instance.
+
+        If a subcommand was used, the L{service.IServiceMaker} that it
+        represents will be used to construct a service to be added to
+        a newly-created Application.
+
+        Otherwise, an application will be loaded based on parameters in
+        the config.
+        """
+        if self.config.subCommand:
+            # If a subcommand was given, it's our responsibility to create
+            # the application, instead of load it from a file.
+
+            # loadedPlugins is set up by the ServerOptions.subCommands
+            # property, which is iterated somewhere in the bowels of
+            # usage.Options.
+            plg = self.config.loadedPlugins[self.config.subCommand]
+            ser = plg.makeService(self.config.subOptions)
+            application = service.Application(plg.tapname)
+            ser.setServiceParent(application)
+        else:
+            passphrase = getPassphrase(self.config["encrypted"])
+            application = getApplication(self.config, passphrase)
+        return application
+
+
+def getApplication(config, passphrase):
+    s = [(config[t], t) for t in ["python", "source", "file"] if config[t]][0]
+    filename, style = s[0], {"file": "pickle"}.get(s[1], s[1])
+    try:
+        log.msg("Loading %s..." % filename)
+        application = service.loadApplication(filename, style, passphrase)
+        log.msg("Loaded.")
+    except Exception as e:
+        s = "Failed to load application: %s" % e
+        if isinstance(e, KeyError) and e.args[0] == "application":
+            s += """
+Could not find 'application' in the file. To use 'twistd -y', your .tac
+file must create a suitable object (e.g., by calling service.Application())
+and store it in a variable named 'application'. twistd loads your .tac file
+and scans the global variables for one of this name.
+
+Please read the 'Using Application' HOWTO for details.
+"""
+        traceback.print_exc(file=log.logfile)
+        log.msg(s)
+        log.deferr()
+        sys.exit("\n" + s + "\n")
+    return application
+
+
+def _reactorAction():
+    return usage.CompleteList([r.shortName for r in reactors.getReactorTypes()])
+
+
+class ReactorSelectionMixin:
+    """
+    Provides options for selecting a reactor to install.
+
+    If a reactor is installed, the short name which was used to locate it is
+    saved as the value for the C{"reactor"} key.
+    """
+
+    compData = usage.Completions(optActions={"reactor": _reactorAction})
+
+    messageOutput = sys.stdout
+    _getReactorTypes = staticmethod(reactors.getReactorTypes)
+
+    def opt_help_reactors(self):
+        """
+        Display a list of possibly available reactor names.
+        """
+        rcts = sorted(self._getReactorTypes(), key=attrgetter("shortName"))
+        notWorkingReactors = ""
+        for r in rcts:
+            try:
+                namedModule(r.moduleName)
+                self.messageOutput.write(f"    {r.shortName:<4}\t{r.description}\n")
+            except ImportError as e:
+                notWorkingReactors += "    !{:<4}\t{} ({})\n".format(
+                    r.shortName,
+                    r.description,
+                    e.args[0],
+                )
+
+        if notWorkingReactors:
+            self.messageOutput.write("\n")
+            self.messageOutput.write(
+                "    reactors not available " "on this platform:\n\n"
+            )
+            self.messageOutput.write(notWorkingReactors)
+        raise SystemExit(0)
+
+    def opt_reactor(self, shortName):
+        """
+        Which reactor to use (see --help-reactors for a list of possibilities)
+        """
+        # Actually actually actually install the reactor right at this very
+        # moment, before any other code (for example, a sub-command plugin)
+        # runs and accidentally imports and installs the default reactor.
+        #
+        # This could probably be improved somehow.
+        try:
+            installReactor(shortName)
+        except NoSuchReactor:
+            msg = (
+                "The specified reactor does not exist: '%s'.\n"
+                "See the list of available reactors with "
+                "--help-reactors" % (shortName,)
+            )
+            raise usage.UsageError(msg)
+        except Exception as e:
+            msg = (
+                "The specified reactor cannot be used, failed with error: "
+                "%s.\nSee the list of available reactors with "
+                "--help-reactors" % (e,)
+            )
+            raise usage.UsageError(msg)
+        else:
+            self["reactor"] = shortName
+
+    opt_r = opt_reactor
+
+
+class ServerOptions(usage.Options, ReactorSelectionMixin):
+    longdesc = (
+        "twistd reads a twisted.application.service.Application out "
+        "of a file and runs it."
+    )
+
+    optFlags = [
+        [
+            "savestats",
+            None,
+            "save the Stats object rather than the text output of " "the profiler.",
+        ],
+        ["no_save", "o", "do not save state on shutdown"],
+        ["encrypted", "e", "The specified tap/aos file is encrypted."],
+    ]
+
+    optParameters = [
+        ["logfile", "l", None, "log to a specified file, - for stdout"],
+        [
+            "logger",
+            None,
+            None,
+            "A fully-qualified name to a log observer factory to "
+            "use for the initial log observer.  Takes precedence "
+            "over --logfile and --syslog (when available).",
+        ],
+        [
+            "profile",
+            "p",
+            None,
+            "Run in profile mode, dumping results to specified " "file.",
+        ],
+        [
+            "profiler",
+            None,
+            "cprofile",
+            "Name of the profiler to use (%s)." % ", ".join(AppProfiler.profilers),
+        ],
+        ["file", "f", "twistd.tap", "read the given .tap file"],
+        [
+            "python",
+            "y",
+            None,
+            "read an application from within a Python file " "(implies -o)",
+        ],
+        ["source", "s", None, "Read an application from a .tas file (AOT format)."],
+        ["rundir", "d", ".", "Change to a supplied directory before running"],
+    ]
+
+    compData = usage.Completions(
+        mutuallyExclusive=[("file", "python", "source")],
+        optActions={
+            "file": usage.CompleteFiles("*.tap"),
+            "python": usage.CompleteFiles("*.(tac|py)"),
+            "source": usage.CompleteFiles("*.tas"),
+            "rundir": usage.CompleteDirs(),
+        },
+    )
+
+    _getPlugins = staticmethod(plugin.getPlugins)
+
+    def __init__(self, *a, **kw):
+        self["debug"] = False
+        if "stdout" in kw:
+            self.stdout = kw["stdout"]
+        else:
+            self.stdout = sys.stdout
+        usage.Options.__init__(self)
+
+    def opt_debug(self):
+        """
+        Run the application in the Python Debugger (implies nodaemon),
+        sending SIGUSR2 will drop into debugger
+        """
+        defer.setDebugging(True)
+        failure.startDebugMode()
+        self["debug"] = True
+
+    opt_b = opt_debug
+
+    def opt_spew(self):
+        """
+        Print an insanely verbose log of everything that happens.
+        Useful when debugging freezes or locks in complex code.
+        """
+        sys.settrace(util.spewer)
+        try:
+            import threading
+        except ImportError:
+            return
+        threading.settrace(util.spewer)
+
+    def parseOptions(self, options=None):
+        if options is None:
+            options = sys.argv[1:] or ["--help"]
+        usage.Options.parseOptions(self, options)
+
+    def postOptions(self):
+        if self.subCommand or self["python"]:
+            self["no_save"] = True
+        if self["logger"] is not None:
+            try:
+                self["logger"] = namedAny(self["logger"])
+            except Exception as e:
+                raise usage.UsageError(
+                    "Logger '{}' could not be imported: {}".format(self["logger"], e)
+                )
+
+    @property
+    def subCommands(self):
+        plugins = self._getPlugins(service.IServiceMaker)
+        self.loadedPlugins = {}
+        for plug in sorted(plugins, key=attrgetter("tapname")):
+            self.loadedPlugins[plug.tapname] = plug
+            yield (
+                plug.tapname,
+                None,
+                # Avoid resolving the options attribute right away, in case
+                # it's a property with a non-trivial getter (eg, one which
+                # imports modules).
+                lambda plug=plug: plug.options(),
+                plug.description,
+            )
+
+
+def run(runApp, ServerOptions):
+    config = ServerOptions()
+    try:
+        config.parseOptions()
+    except usage.error as ue:
+        commstr = " ".join(sys.argv[0:2])
+        print(config)
+        print(f"{commstr}: {ue}")
+    else:
+        runApp(config)
+
+
+def convertStyle(filein, typein, passphrase, fileout, typeout, encrypt):
+    application = service.loadApplication(filein, typein, passphrase)
+    sob.IPersistable(application).setStyle(typeout)
+    passphrase = getSavePassphrase(encrypt)
+    if passphrase:
+        fileout = None
+    sob.IPersistable(application).save(filename=fileout, passphrase=passphrase)
+
+
+def startApplication(application, save):
+    from twisted.internet import reactor
+
+    service.IService(application).startService()
+    if save:
+        p = sob.IPersistable(application)
+        reactor.addSystemEventTrigger("after", "shutdown", p.save, "shutdown")
+    reactor.addSystemEventTrigger(
+        "before", "shutdown", service.IService(application).stopService
+    )
+
+
+def _exitWithSignal(sig):
+    """
+    Force the application to terminate with the specified signal by replacing
+    the signal handler with the default and sending the signal to ourselves.
+
+    @param sig:  Signal to use to terminate the process with C{os.kill}.
+    @type sig:  C{int}
+    """
+    signal.signal(sig, signal.SIG_DFL)
+    os.kill(os.getpid(), sig)
diff --git a/contrib/python/Twisted/py3/twisted/application/internet.py b/contrib/python/Twisted/py3/twisted/application/internet.py
new file mode 100644
index 00000000000..194a4dd8a70
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/internet.py
@@ -0,0 +1,1205 @@
+# -*- test-case-name: twisted.application.test.test_internet,twisted.test.test_application,twisted.test.test_cooperator -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Reactor-based Services
+
+Here are services to run clients, servers and periodic services using
+the reactor.
+
+If you want to run a server service, L{StreamServerEndpointService} defines a
+service that can wrap an arbitrary L{IStreamServerEndpoint
+<twisted.internet.interfaces.IStreamServerEndpoint>}
+as an L{IService}. See also L{twisted.application.strports.service} for
+constructing one of these directly from a descriptive string.
+
+Additionally, this module (dynamically) defines various Service subclasses that
+let you represent clients and servers in a Service hierarchy.  Endpoints APIs
+should be preferred for stream server services, but since those APIs do not yet
+exist for clients or datagram services, many of these are still useful.
+
+They are as follows::
+
+  TCPServer, TCPClient,
+  UNIXServer, UNIXClient,
+  SSLServer, SSLClient,
+  UDPServer,
+  UNIXDatagramServer, UNIXDatagramClient,
+  MulticastServer
+
+These classes take arbitrary arguments in their constructors and pass
+them straight on to their respective reactor.listenXXX or
+reactor.connectXXX calls.
+
+For example, the following service starts a web server on port 8080:
+C{TCPServer(8080, server.Site(r))}.  See the documentation for the
+reactor.listen/connect* methods for more information.
+"""
+
+
+from random import random as _goodEnoughRandom
+from typing import List
+
+from automat import MethodicalMachine  # type: ignore[import]
+
+from twisted.application import service
+from twisted.internet import task
+from twisted.internet.defer import (
+    CancelledError,
+    Deferred,
+    fail,
+    maybeDeferred,
+    succeed,
+)
+from twisted.logger import Logger
+from twisted.python import log
+from twisted.python.failure import Failure
+
+
+def _maybeGlobalReactor(maybeReactor):
+    """
+    @return: the argument, or the global reactor if the argument is L{None}.
+    """
+    if maybeReactor is None:
+        from twisted.internet import reactor
+
+        return reactor
+    else:
+        return maybeReactor
+
+
+class _VolatileDataService(service.Service):
+    volatile: List[str] = []
+
+    def __getstate__(self):
+        d = service.Service.__getstate__(self)
+        for attr in self.volatile:
+            if attr in d:
+                del d[attr]
+        return d
+
+
+class _AbstractServer(_VolatileDataService):
+    """
+    @cvar volatile: list of attribute to remove from pickling.
+    @type volatile: C{list}
+
+    @ivar method: the type of method to call on the reactor, one of B{TCP},
+        B{UDP}, B{SSL} or B{UNIX}.
+    @type method: C{str}
+
+    @ivar reactor: the current running reactor.
+    @type reactor: a provider of C{IReactorTCP}, C{IReactorUDP},
+        C{IReactorSSL} or C{IReactorUnix}.
+
+    @ivar _port: instance of port set when the service is started.
+    @type _port: a provider of L{twisted.internet.interfaces.IListeningPort}.
+    """
+
+    volatile = ["_port"]
+    method: str = ""
+    reactor = None
+
+    _port = None
+
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        if "reactor" in kwargs:
+            self.reactor = kwargs.pop("reactor")
+        self.kwargs = kwargs
+
+    def privilegedStartService(self):
+        service.Service.privilegedStartService(self)
+        self._port = self._getPort()
+
+    def startService(self):
+        service.Service.startService(self)
+        if self._port is None:
+            self._port = self._getPort()
+
+    def stopService(self):
+        service.Service.stopService(self)
+        # TODO: if startup failed, should shutdown skip stopListening?
+        # _port won't exist
+        if self._port is not None:
+            d = self._port.stopListening()
+            del self._port
+            return d
+
+    def _getPort(self):
+        """
+        Wrapper around the appropriate listen method of the reactor.
+
+        @return: the port object returned by the listen method.
+        @rtype: an object providing
+            L{twisted.internet.interfaces.IListeningPort}.
+        """
+        return getattr(
+            _maybeGlobalReactor(self.reactor),
+            "listen{}".format(
+                self.method,
+            ),
+        )(*self.args, **self.kwargs)
+
+
+class _AbstractClient(_VolatileDataService):
+    """
+    @cvar volatile: list of attribute to remove from pickling.
+    @type volatile: C{list}
+
+    @ivar method: the type of method to call on the reactor, one of B{TCP},
+        B{UDP}, B{SSL} or B{UNIX}.
+    @type method: C{str}
+
+    @ivar reactor: the current running reactor.
+    @type reactor: a provider of C{IReactorTCP}, C{IReactorUDP},
+        C{IReactorSSL} or C{IReactorUnix}.
+
+    @ivar _connection: instance of connection set when the service is started.
+    @type _connection: a provider of L{twisted.internet.interfaces.IConnector}.
+    """
+
+    volatile = ["_connection"]
+    method: str = ""
+    reactor = None
+
+    _connection = None
+
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        if "reactor" in kwargs:
+            self.reactor = kwargs.pop("reactor")
+        self.kwargs = kwargs
+
+    def startService(self):
+        service.Service.startService(self)
+        self._connection = self._getConnection()
+
+    def stopService(self):
+        service.Service.stopService(self)
+        if self._connection is not None:
+            self._connection.disconnect()
+            del self._connection
+
+    def _getConnection(self):
+        """
+        Wrapper around the appropriate connect method of the reactor.
+
+        @return: the port object returned by the connect method.
+        @rtype: an object providing L{twisted.internet.interfaces.IConnector}.
+        """
+        return getattr(_maybeGlobalReactor(self.reactor), f"connect{self.method}")(
+            *self.args, **self.kwargs
+        )
+
+
+_clientDoc = """Connect to {tran}
+
+Call reactor.connect{tran} when the service starts, with the
+arguments given to the constructor.
+"""
+
+_serverDoc = """Serve {tran} clients
+
+Call reactor.listen{tran} when the service starts, with the
+arguments given to the constructor. When the service stops,
+stop listening. See twisted.internet.interfaces for documentation
+on arguments to the reactor method.
+"""
+
+
+class TCPServer(_AbstractServer):
+    __doc__ = _serverDoc.format(tran="TCP")
+    method = "TCP"
+
+
+class TCPClient(_AbstractClient):
+    __doc__ = _clientDoc.format(tran="TCP")
+    method = "TCP"
+
+
+class UNIXServer(_AbstractServer):
+    __doc__ = _serverDoc.format(tran="UNIX")
+    method = "UNIX"
+
+
+class UNIXClient(_AbstractClient):
+    __doc__ = _clientDoc.format(tran="UNIX")
+    method = "UNIX"
+
+
+class SSLServer(_AbstractServer):
+    __doc__ = _serverDoc.format(tran="SSL")
+    method = "SSL"
+
+
+class SSLClient(_AbstractClient):
+    __doc__ = _clientDoc.format(tran="SSL")
+    method = "SSL"
+
+
+class UDPServer(_AbstractServer):
+    __doc__ = _serverDoc.format(tran="UDP")
+    method = "UDP"
+
+
+class UNIXDatagramServer(_AbstractServer):
+    __doc__ = _serverDoc.format(tran="UNIXDatagram")
+    method = "UNIXDatagram"
+
+
+class UNIXDatagramClient(_AbstractClient):
+    __doc__ = _clientDoc.format(tran="UNIXDatagram")
+    method = "UNIXDatagram"
+
+
+class MulticastServer(_AbstractServer):
+    __doc__ = _serverDoc.format(tran="Multicast")
+    method = "Multicast"
+
+
+class TimerService(_VolatileDataService):
+    """
+    Service to periodically call a function
+
+    Every C{step} seconds call the given function with the given arguments.
+    The service starts the calls when it starts, and cancels them
+    when it stops.
+
+    @ivar clock: Source of time. This defaults to L{None} which is
+        causes L{twisted.internet.reactor} to be used.
+        Feel free to set this to something else, but it probably ought to be
+        set *before* calling L{startService}.
+    @type clock: L{IReactorTime<twisted.internet.interfaces.IReactorTime>}
+
+    @ivar call: Function and arguments to call periodically.
+    @type call: L{tuple} of C{(callable, args, kwargs)}
+    """
+
+    volatile = ["_loop", "_loopFinished"]
+
+    def __init__(self, step, callable, *args, **kwargs):
+        """
+        @param step: The number of seconds between calls.
+        @type step: L{float}
+
+        @param callable: Function to call
+        @type callable: L{callable}
+
+        @param args: Positional arguments to pass to function
+        @param kwargs: Keyword arguments to pass to function
+        """
+        self.step = step
+        self.call = (callable, args, kwargs)
+        self.clock = None
+
+    def startService(self):
+        service.Service.startService(self)
+        callable, args, kwargs = self.call
+        # we have to make a new LoopingCall each time we're started, because
+        # an active LoopingCall remains active when serialized. If
+        # LoopingCall were a _VolatileDataService, we wouldn't need to do
+        # this.
+        self._loop = task.LoopingCall(callable, *args, **kwargs)
+        self._loop.clock = _maybeGlobalReactor(self.clock)
+        self._loopFinished = self._loop.start(self.step, now=True)
+        self._loopFinished.addErrback(self._failed)
+
+    def _failed(self, why):
+        # make a note that the LoopingCall is no longer looping, so we don't
+        # try to shut it down a second time in stopService. I think this
+        # should be in LoopingCall. -warner
+        self._loop.running = False
+        log.err(why)
+
+    def stopService(self):
+        """
+        Stop the service.
+
+        @rtype: L{Deferred<defer.Deferred>}
+        @return: a L{Deferred<defer.Deferred>} which is fired when the
+            currently running call (if any) is finished.
+        """
+        if self._loop.running:
+            self._loop.stop()
+        self._loopFinished.addCallback(lambda _: service.Service.stopService(self))
+        return self._loopFinished
+
+
+class CooperatorService(service.Service):
+    """
+    Simple L{service.IService} which starts and stops a L{twisted.internet.task.Cooperator}.
+    """
+
+    def __init__(self):
+        self.coop = task.Cooperator(started=False)
+
+    def coiterate(self, iterator):
+        return self.coop.coiterate(iterator)
+
+    def startService(self):
+        self.coop.start()
+
+    def stopService(self):
+        self.coop.stop()
+
+
+class StreamServerEndpointService(service.Service):
+    """
+    A L{StreamServerEndpointService} is an L{IService} which runs a server on a
+    listening port described by an L{IStreamServerEndpoint
+    <twisted.internet.interfaces.IStreamServerEndpoint>}.
+
+    @ivar factory: A server factory which will be used to listen on the
+        endpoint.
+
+    @ivar endpoint: An L{IStreamServerEndpoint
+        <twisted.internet.interfaces.IStreamServerEndpoint>} provider
+        which will be used to listen when the service starts.
+
+    @ivar _waitingForPort: a Deferred, if C{listen} has yet been invoked on the
+        endpoint, otherwise None.
+
+    @ivar _raiseSynchronously: Defines error-handling behavior for the case
+        where C{listen(...)} raises an exception before C{startService} or
+        C{privilegedStartService} have completed.
+
+    @type _raiseSynchronously: C{bool}
+
+    @since: 10.2
+    """
+
+    _raiseSynchronously = False
+
+    def __init__(self, endpoint, factory):
+        self.endpoint = endpoint
+        self.factory = factory
+        self._waitingForPort = None
+
+    def privilegedStartService(self):
+        """
+        Start listening on the endpoint.
+        """
+        service.Service.privilegedStartService(self)
+        self._waitingForPort = self.endpoint.listen(self.factory)
+        raisedNow = []
+
+        def handleIt(err):
+            if self._raiseSynchronously:
+                raisedNow.append(err)
+            elif not err.check(CancelledError):
+                log.err(err)
+
+        self._waitingForPort.addErrback(handleIt)
+        if raisedNow:
+            raisedNow[0].raiseException()
+        self._raiseSynchronously = False
+
+    def startService(self):
+        """
+        Start listening on the endpoint, unless L{privilegedStartService} got
+        around to it already.
+        """
+        service.Service.startService(self)
+        if self._waitingForPort is None:
+            self.privilegedStartService()
+
+    def stopService(self):
+        """
+        Stop listening on the port if it is already listening, otherwise,
+        cancel the attempt to listen.
+
+        @return: a L{Deferred<twisted.internet.defer.Deferred>} which fires
+            with L{None} when the port has stopped listening.
+        """
+        self._waitingForPort.cancel()
+
+        def stopIt(port):
+            if port is not None:
+                return port.stopListening()
+
+        d = self._waitingForPort.addCallback(stopIt)
+
+        def stop(passthrough):
+            self.running = False
+            return passthrough
+
+        d.addBoth(stop)
+        return d
+
+
+class _ReconnectingProtocolProxy:
+    """
+    A proxy for a Protocol to provide connectionLost notification to a client
+    connection service, in support of reconnecting when connections are lost.
+    """
+
+    def __init__(self, protocol, lostNotification):
+        """
+        Create a L{_ReconnectingProtocolProxy}.
+
+        @param protocol: the application-provided L{interfaces.IProtocol}
+            provider.
+        @type protocol: provider of L{interfaces.IProtocol} which may
+            additionally provide L{interfaces.IHalfCloseableProtocol} and
+            L{interfaces.IFileDescriptorReceiver}.
+
+        @param lostNotification: a 1-argument callable to invoke with the
+            C{reason} when the connection is lost.
+        """
+        self._protocol = protocol
+        self._lostNotification = lostNotification
+
+    def connectionLost(self, reason):
+        """
+        The connection was lost.  Relay this information.
+
+        @param reason: The reason the connection was lost.
+
+        @return: the underlying protocol's result
+        """
+        try:
+            return self._protocol.connectionLost(reason)
+        finally:
+            self._lostNotification(reason)
+
+    def __getattr__(self, item):
+        return getattr(self._protocol, item)
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} wrapping {self._protocol!r}>"
+
+
+class _DisconnectFactory:
+    """
+    A L{_DisconnectFactory} is a proxy for L{IProtocolFactory} that catches
+    C{connectionLost} notifications and relays them.
+    """
+
+    def __init__(self, protocolFactory, protocolDisconnected):
+        self._protocolFactory = protocolFactory
+        self._protocolDisconnected = protocolDisconnected
+
+    def buildProtocol(self, addr):
+        """
+        Create a L{_ReconnectingProtocolProxy} with the disconnect-notification
+        callback we were called with.
+
+        @param addr: The address the connection is coming from.
+
+        @return: a L{_ReconnectingProtocolProxy} for a protocol produced by
+            C{self._protocolFactory}
+        """
+        return _ReconnectingProtocolProxy(
+            self._protocolFactory.buildProtocol(addr), self._protocolDisconnected
+        )
+
+    def __getattr__(self, item):
+        return getattr(self._protocolFactory, item)
+
+    def __repr__(self) -> str:
+        return "<{} wrapping {!r}>".format(
+            self.__class__.__name__, self._protocolFactory
+        )
+
+
+def backoffPolicy(
+    initialDelay=1.0, maxDelay=60.0, factor=1.5, jitter=_goodEnoughRandom
+):
+    """
+    A timeout policy for L{ClientService} which computes an exponential backoff
+    interval with configurable parameters.
+
+    @since: 16.1.0
+
+    @param initialDelay: Delay for the first reconnection attempt (default
+        1.0s).
+    @type initialDelay: L{float}
+
+    @param maxDelay: Maximum number of seconds between connection attempts
+        (default 60 seconds, or one minute).  Note that this value is before
+        jitter is applied, so the actual maximum possible delay is this value
+        plus the maximum possible result of C{jitter()}.
+    @type maxDelay: L{float}
+
+    @param factor: A multiplicative factor by which the delay grows on each
+        failed reattempt.  Default: 1.5.
+    @type factor: L{float}
+
+    @param jitter: A 0-argument callable that introduces noise into the delay.
+        By default, C{random.random}, i.e. a pseudorandom floating-point value
+        between zero and one.
+    @type jitter: 0-argument callable returning L{float}
+
+    @return: a 1-argument callable that, given an attempt count, returns a
+        floating point number; the number of seconds to delay.
+    @rtype: see L{ClientService.__init__}'s C{retryPolicy} argument.
+    """
+
+    def policy(attempt):
+        try:
+            delay = min(initialDelay * (factor ** min(100, attempt)), maxDelay)
+        except OverflowError:
+            delay = maxDelay
+        return delay + jitter()
+
+    return policy
+
+
+_defaultPolicy = backoffPolicy()
+
+
+def _firstResult(gen):
+    """
+    Return the first element of a generator and exhaust it.
+
+    C{MethodicalMachine.upon}'s C{collector} argument takes a generator of
+    output results. If the generator is exhausted, the later outputs aren't
+    actually run.
+
+    @param gen: Generator to extract values from
+
+    @return: The first element of the generator.
+    """
+    return list(gen)[0]
+
+
+class _ClientMachine:
+    """
+    State machine for maintaining a single outgoing connection to an endpoint.
+
+    @ivar _awaitingConnected: notifications to make when connection
+        succeeds, fails, or is cancelled
+    @type _awaitingConnected: list of (Deferred, count) tuples
+
+    @see: L{ClientService}
+    """
+
+    _machine = MethodicalMachine()
+
+    def __init__(self, endpoint, factory, retryPolicy, clock, prepareConnection, log):
+        """
+        @see: L{ClientService.__init__}
+
+        @param log: The logger for the L{ClientService} instance this state
+            machine is associated to.
+        @type log: L{Logger}
+        """
+        self._endpoint = endpoint
+        self._failedAttempts = 0
+        self._stopped = False
+        self._factory = factory
+        self._timeoutForAttempt = retryPolicy
+        self._clock = clock
+        self._prepareConnection = prepareConnection
+        self._connectionInProgress = succeed(None)
+
+        self._awaitingConnected = []
+
+        self._stopWaiters = []
+        self._log = log
+
+    @_machine.state(initial=True)
+    def _init(self):
+        """
+        The service has not been started.
+        """
+
+    @_machine.state()
+    def _connecting(self):
+        """
+        The service has started connecting.
+        """
+
+    @_machine.state()
+    def _waiting(self):
+        """
+        The service is waiting for the reconnection period
+        before reconnecting.
+        """
+
+    @_machine.state()
+    def _connected(self):
+        """
+        The service is connected.
+        """
+
+    @_machine.state()
+    def _disconnecting(self):
+        """
+        The service is disconnecting after being asked to shutdown.
+        """
+
+    @_machine.state()
+    def _restarting(self):
+        """
+        The service is disconnecting and has been asked to restart.
+        """
+
+    @_machine.state()
+    def _stopped(self):
+        """
+        The service has been stopped and is disconnected.
+        """
+
+    @_machine.input()
+    def start(self):
+        """
+        Start this L{ClientService}, initiating the connection retry loop.
+        """
+
+    @_machine.output()
+    def _connect(self):
+        """
+        Start a connection attempt.
+        """
+        factoryProxy = _DisconnectFactory(
+            self._factory, lambda _: self._clientDisconnected()
+        )
+
+        self._connectionInProgress = (
+            self._endpoint.connect(factoryProxy)
+            .addCallback(self._runPrepareConnection)
+            .addCallback(self._connectionMade)
+            .addErrback(self._connectionFailed)
+        )
+
+    def _runPrepareConnection(self, protocol):
+        """
+        Run any C{prepareConnection} callback with the connected protocol,
+        ignoring its return value but propagating any failure.
+
+        @param protocol: The protocol of the connection.
+        @type protocol: L{IProtocol}
+
+        @return: Either:
+
+            - A L{Deferred} that succeeds with the protocol when the
+              C{prepareConnection} callback has executed successfully.
+
+            - A L{Deferred} that fails when the C{prepareConnection} callback
+              throws or returns a failed L{Deferred}.
+
+            - The protocol, when no C{prepareConnection} callback is defined.
+        """
+        if self._prepareConnection:
+            return maybeDeferred(self._prepareConnection, protocol).addCallback(
+                lambda _: protocol
+            )
+        return protocol
+
+    @_machine.output()
+    def _resetFailedAttempts(self):
+        """
+        Reset the number of failed attempts.
+        """
+        self._failedAttempts = 0
+
+    @_machine.input()
+    def stop(self):
+        """
+        Stop trying to connect and disconnect any current connection.
+
+        @return: a L{Deferred} that fires when all outstanding connections are
+            closed and all in-progress connection attempts halted.
+        """
+
+    @_machine.output()
+    def _waitForStop(self):
+        """
+        Return a deferred that will fire when the service has finished
+        disconnecting.
+
+        @return: L{Deferred} that fires when the service has finished
+            disconnecting.
+        """
+        self._stopWaiters.append(Deferred())
+        return self._stopWaiters[-1]
+
+    @_machine.output()
+    def _stopConnecting(self):
+        """
+        Stop pending connection attempt.
+        """
+        self._connectionInProgress.cancel()
+
+    @_machine.output()
+    def _stopRetrying(self):
+        """
+        Stop pending attempt to reconnect.
+        """
+        self._retryCall.cancel()
+        del self._retryCall
+
+    @_machine.output()
+    def _disconnect(self):
+        """
+        Disconnect the current connection.
+        """
+        self._currentConnection.transport.loseConnection()
+
+    @_machine.input()
+    def _connectionMade(self, protocol):
+        """
+        A connection has been made.
+
+        @param protocol: The protocol of the connection.
+        @type protocol: L{IProtocol}
+        """
+
+    @_machine.output()
+    def _notifyWaiters(self, protocol):
+        """
+        Notify all pending requests for a connection that a connection has been
+        made.
+
+        @param protocol: The protocol of the connection.
+        @type protocol: L{IProtocol}
+        """
+        # This should be in _resetFailedAttempts but the signature doesn't
+        # match.
+        self._failedAttempts = 0
+
+        self._currentConnection = protocol._protocol
+        self._unawait(self._currentConnection)
+
+    @_machine.input()
+    def _connectionFailed(self, f):
+        """
+        The current connection attempt failed.
+        """
+
+    @_machine.output()
+    def _wait(self):
+        """
+        Schedule a retry attempt.
+        """
+        self._doWait()
+
+    @_machine.output()
+    def _ignoreAndWait(self, f):
+        """
+        Schedule a retry attempt, and ignore the Failure passed in.
+        """
+        return self._doWait()
+
+    def _doWait(self):
+        self._failedAttempts += 1
+        delay = self._timeoutForAttempt(self._failedAttempts)
+        self._log.info(
+            "Scheduling retry {attempt} to connect {endpoint} " "in {delay} seconds.",
+            attempt=self._failedAttempts,
+            endpoint=self._endpoint,
+            delay=delay,
+        )
+        self._retryCall = self._clock.callLater(delay, self._reconnect)
+
+    @_machine.input()
+    def _reconnect(self):
+        """
+        The wait between connection attempts is done.
+        """
+
+    @_machine.input()
+    def _clientDisconnected(self):
+        """
+        The current connection has been disconnected.
+        """
+
+    @_machine.output()
+    def _forgetConnection(self):
+        """
+        Forget the current connection.
+        """
+        del self._currentConnection
+
+    @_machine.output()
+    def _cancelConnectWaiters(self):
+        """
+        Notify all pending requests for a connection that no more connections
+        are expected.
+        """
+        self._unawait(Failure(CancelledError()))
+
+    @_machine.output()
+    def _ignoreAndCancelConnectWaiters(self, f):
+        """
+        Notify all pending requests for a connection that no more connections
+        are expected, after ignoring the Failure passed in.
+        """
+        self._unawait(Failure(CancelledError()))
+
+    @_machine.output()
+    def _finishStopping(self):
+        """
+        Notify all deferreds waiting on the service stopping.
+        """
+        self._doFinishStopping()
+
+    @_machine.output()
+    def _ignoreAndFinishStopping(self, f):
+        """
+        Notify all deferreds waiting on the service stopping, and ignore the
+        Failure passed in.
+        """
+        self._doFinishStopping()
+
+    def _doFinishStopping(self):
+        self._stopWaiters, waiting = [], self._stopWaiters
+        for w in waiting:
+            w.callback(None)
+
+    @_machine.input()
+    def whenConnected(self, failAfterFailures=None):
+        """
+        Retrieve the currently-connected L{Protocol}, or the next one to
+        connect.
+
+        @param failAfterFailures: number of connection failures after which
+            the Deferred will deliver a Failure (None means the Deferred will
+            only fail if/when the service is stopped).  Set this to 1 to make
+            the very first connection failure signal an error.  Use 2 to
+            allow one failure but signal an error if the subsequent retry
+            then fails.
+        @type failAfterFailures: L{int} or None
+
+        @return: a Deferred that fires with a protocol produced by the
+            factory passed to C{__init__}
+        @rtype: L{Deferred} that may:
+
+            - fire with L{IProtocol}
+
+            - fail with L{CancelledError} when the service is stopped
+
+            - fail with e.g.
+              L{DNSLookupError<twisted.internet.error.DNSLookupError>} or
+              L{ConnectionRefusedError<twisted.internet.error.ConnectionRefusedError>}
+              when the number of consecutive failed connection attempts
+              equals the value of "failAfterFailures"
+        """
+
+    @_machine.output()
+    def _currentConnection(self, failAfterFailures=None):
+        """
+        Return the currently connected protocol.
+
+        @return: L{Deferred} that is fired with currently connected protocol.
+        """
+        return succeed(self._currentConnection)
+
+    @_machine.output()
+    def _noConnection(self, failAfterFailures=None):
+        """
+        Notify the caller that no connection is expected.
+
+        @return: L{Deferred} that is fired with L{CancelledError}.
+        """
+        return fail(CancelledError())
+
+    @_machine.output()
+    def _awaitingConnection(self, failAfterFailures=None):
+        """
+        Return a deferred that will fire with the next connected protocol.
+
+        @return: L{Deferred} that will fire with the next connected protocol.
+        """
+        result = Deferred()
+        self._awaitingConnected.append((result, failAfterFailures))
+        return result
+
+    @_machine.output()
+    def _deferredSucceededWithNone(self):
+        """
+        Return a deferred that has already fired with L{None}.
+
+        @return: A L{Deferred} that has already fired with L{None}.
+        """
+        return succeed(None)
+
+    def _unawait(self, value):
+        """
+        Fire all outstanding L{ClientService.whenConnected} L{Deferred}s.
+
+        @param value: the value to fire the L{Deferred}s with.
+        """
+        self._awaitingConnected, waiting = [], self._awaitingConnected
+        for w, remaining in waiting:
+            w.callback(value)
+
+    @_machine.output()
+    def _deliverConnectionFailure(self, f):
+        """
+        Deliver connection failures to any L{ClientService.whenConnected}
+        L{Deferred}s that have met their failAfterFailures threshold.
+
+        @param f: the Failure to fire the L{Deferred}s with.
+        """
+        ready = []
+        notReady = []
+        for w, remaining in self._awaitingConnected:
+            if remaining is None:
+                notReady.append((w, remaining))
+            elif remaining <= 1:
+                ready.append(w)
+            else:
+                notReady.append((w, remaining - 1))
+        self._awaitingConnected = notReady
+        for w in ready:
+            w.callback(f)
+
+    # State Transitions
+
+    _init.upon(start, enter=_connecting, outputs=[_connect])
+    _init.upon(
+        stop,
+        enter=_stopped,
+        outputs=[_deferredSucceededWithNone],
+        collector=_firstResult,
+    )
+
+    _connecting.upon(start, enter=_connecting, outputs=[])
+    # Note that this synchonously triggers _connectionFailed in the
+    # _disconnecting state.
+    _connecting.upon(
+        stop,
+        enter=_disconnecting,
+        outputs=[_waitForStop, _stopConnecting],
+        collector=_firstResult,
+    )
+    _connecting.upon(_connectionMade, enter=_connected, outputs=[_notifyWaiters])
+    _connecting.upon(
+        _connectionFailed,
+        enter=_waiting,
+        outputs=[_ignoreAndWait, _deliverConnectionFailure],
+    )
+
+    _waiting.upon(start, enter=_waiting, outputs=[])
+    _waiting.upon(
+        stop,
+        enter=_stopped,
+        outputs=[_waitForStop, _cancelConnectWaiters, _stopRetrying, _finishStopping],
+        collector=_firstResult,
+    )
+    _waiting.upon(_reconnect, enter=_connecting, outputs=[_connect])
+
+    _connected.upon(start, enter=_connected, outputs=[])
+    _connected.upon(
+        stop,
+        enter=_disconnecting,
+        outputs=[_waitForStop, _disconnect],
+        collector=_firstResult,
+    )
+    _connected.upon(
+        _clientDisconnected, enter=_waiting, outputs=[_forgetConnection, _wait]
+    )
+
+    _disconnecting.upon(start, enter=_restarting, outputs=[_resetFailedAttempts])
+    _disconnecting.upon(
+        stop, enter=_disconnecting, outputs=[_waitForStop], collector=_firstResult
+    )
+    _disconnecting.upon(
+        _clientDisconnected,
+        enter=_stopped,
+        outputs=[_cancelConnectWaiters, _finishStopping, _forgetConnection],
+    )
+    # Note that this is triggered synchonously with the transition from
+    # _connecting
+    _disconnecting.upon(
+        _connectionFailed,
+        enter=_stopped,
+        outputs=[_ignoreAndCancelConnectWaiters, _ignoreAndFinishStopping],
+    )
+
+    _restarting.upon(start, enter=_restarting, outputs=[])
+    _restarting.upon(
+        stop, enter=_disconnecting, outputs=[_waitForStop], collector=_firstResult
+    )
+    _restarting.upon(
+        _clientDisconnected, enter=_connecting, outputs=[_finishStopping, _connect]
+    )
+
+    _stopped.upon(start, enter=_connecting, outputs=[_connect])
+    _stopped.upon(
+        stop,
+        enter=_stopped,
+        outputs=[_deferredSucceededWithNone],
+        collector=_firstResult,
+    )
+
+    _init.upon(
+        whenConnected,
+        enter=_init,
+        outputs=[_awaitingConnection],
+        collector=_firstResult,
+    )
+    _connecting.upon(
+        whenConnected,
+        enter=_connecting,
+        outputs=[_awaitingConnection],
+        collector=_firstResult,
+    )
+    _waiting.upon(
+        whenConnected,
+        enter=_waiting,
+        outputs=[_awaitingConnection],
+        collector=_firstResult,
+    )
+    _connected.upon(
+        whenConnected,
+        enter=_connected,
+        outputs=[_currentConnection],
+        collector=_firstResult,
+    )
+    _disconnecting.upon(
+        whenConnected,
+        enter=_disconnecting,
+        outputs=[_awaitingConnection],
+        collector=_firstResult,
+    )
+    _restarting.upon(
+        whenConnected,
+        enter=_restarting,
+        outputs=[_awaitingConnection],
+        collector=_firstResult,
+    )
+    _stopped.upon(
+        whenConnected, enter=_stopped, outputs=[_noConnection], collector=_firstResult
+    )
+
+
+class ClientService(service.Service):
+    """
+    A L{ClientService} maintains a single outgoing connection to a client
+    endpoint, reconnecting after a configurable timeout when a connection
+    fails, either before or after connecting.
+
+    @since: 16.1.0
+    """
+
+    _log = Logger()
+
+    def __init__(
+        self, endpoint, factory, retryPolicy=None, clock=None, prepareConnection=None
+    ):
+        """
+        @param endpoint: A L{stream client endpoint
+            <interfaces.IStreamClientEndpoint>} provider which will be used to
+            connect when the service starts.
+
+        @param factory: A L{protocol factory <interfaces.IProtocolFactory>}
+            which will be used to create clients for the endpoint.
+
+        @param retryPolicy: A policy configuring how long L{ClientService} will
+            wait between attempts to connect to C{endpoint}.
+        @type retryPolicy: callable taking (the number of failed connection
+            attempts made in a row (L{int})) and returning the number of
+            seconds to wait before making another attempt.
+
+        @param clock: The clock used to schedule reconnection.  It's mainly
+            useful to be parametrized in tests.  If the factory is serialized,
+            this attribute will not be serialized, and the default value (the
+            reactor) will be restored when deserialized.
+        @type clock: L{IReactorTime}
+
+        @param prepareConnection: A single argument L{callable} that may return
+            a L{Deferred}. It will be called once with the L{protocol
+            <interfaces.IProtocol>} each time a new connection is made.  It may
+            call methods on the protocol to prepare it for use (e.g.
+            authenticate) or validate it (check its health).
+
+            The C{prepareConnection} callable may raise an exception or return
+            a L{Deferred} which fails to reject the connection.  A rejected
+            connection is not used to fire an L{Deferred} returned by
+            L{whenConnected}.  Instead, L{ClientService} handles the failure
+            and continues as if the connection attempt were a failure
+            (incrementing the counter passed to C{retryPolicy}).
+
+            L{Deferred}s returned by L{whenConnected} will not fire until
+            any L{Deferred} returned by the C{prepareConnection} callable
+            fire. Otherwise its successful return value is consumed, but
+            ignored.
+
+            Present Since Twisted 18.7.0
+
+        @type prepareConnection: L{callable}
+
+        """
+        clock = _maybeGlobalReactor(clock)
+        retryPolicy = _defaultPolicy if retryPolicy is None else retryPolicy
+
+        self._machine = _ClientMachine(
+            endpoint,
+            factory,
+            retryPolicy,
+            clock,
+            prepareConnection=prepareConnection,
+            log=self._log,
+        )
+
+    def whenConnected(self, failAfterFailures=None):
+        """
+        Retrieve the currently-connected L{Protocol}, or the next one to
+        connect.
+
+        @param failAfterFailures: number of connection failures after which
+            the Deferred will deliver a Failure (None means the Deferred will
+            only fail if/when the service is stopped).  Set this to 1 to make
+            the very first connection failure signal an error.  Use 2 to
+            allow one failure but signal an error if the subsequent retry
+            then fails.
+        @type failAfterFailures: L{int} or None
+
+        @return: a Deferred that fires with a protocol produced by the
+            factory passed to C{__init__}
+        @rtype: L{Deferred} that may:
+
+            - fire with L{IProtocol}
+
+            - fail with L{CancelledError} when the service is stopped
+
+            - fail with e.g.
+              L{DNSLookupError<twisted.internet.error.DNSLookupError>} or
+              L{ConnectionRefusedError<twisted.internet.error.ConnectionRefusedError>}
+              when the number of consecutive failed connection attempts
+              equals the value of "failAfterFailures"
+        """
+        return self._machine.whenConnected(failAfterFailures)
+
+    def startService(self):
+        """
+        Start this L{ClientService}, initiating the connection retry loop.
+        """
+        if self.running:
+            self._log.warn("Duplicate ClientService.startService {log_source}")
+            return
+        super().startService()
+        self._machine.start()
+
+    def stopService(self):
+        """
+        Stop attempting to reconnect and close any existing connections.
+
+        @return: a L{Deferred} that fires when all outstanding connections are
+            closed and all in-progress connection attempts halted.
+        """
+        super().stopService()
+        return self._machine.stop()
+
+
+__all__ = [
+    "TimerService",
+    "CooperatorService",
+    "MulticastServer",
+    "StreamServerEndpointService",
+    "UDPServer",
+    "ClientService",
+    "TCPServer",
+    "TCPClient",
+    "UNIXServer",
+    "UNIXClient",
+    "SSLServer",
+    "SSLClient",
+    "UNIXDatagramServer",
+    "UNIXDatagramClient",
+]
diff --git a/contrib/python/Twisted/py3/twisted/application/newsfragments/10146.misc b/contrib/python/Twisted/py3/twisted/application/newsfragments/10146.misc
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/Twisted/py3/twisted/application/newsfragments/9746.misc b/contrib/python/Twisted/py3/twisted/application/newsfragments/9746.misc
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/newsfragments/9746.misc
@@ -0,0 +1 @@
+
diff --git a/contrib/python/Twisted/py3/twisted/application/reactors.py b/contrib/python/Twisted/py3/twisted/application/reactors.py
new file mode 100644
index 00000000000..a476ca98b79
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/reactors.py
@@ -0,0 +1,87 @@
+# -*- test-case-name: twisted.test.test_application -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Plugin-based system for enumerating available reactors and installing one of
+them.
+"""
+from typing import Iterable, cast
+
+from zope.interface import Attribute, Interface, implementer
+
+from twisted.internet.interfaces import IReactorCore
+from twisted.plugin import IPlugin, getPlugins
+from twisted.python.reflect import namedAny
+
+
+class IReactorInstaller(Interface):
+    """
+    Definition of a reactor which can probably be installed.
+    """
+
+    shortName = Attribute(
+        """
+    A brief string giving the user-facing name of this reactor.
+    """
+    )
+
+    description = Attribute(
+        """
+    A longer string giving a user-facing description of this reactor.
+    """
+    )
+
+    def install() -> None:
+        """
+        Install this reactor.
+        """
+
+    # TODO - A method which provides a best-guess as to whether this reactor
+    # can actually be used in the execution environment.
+
+
+class NoSuchReactor(KeyError):
+    """
+    Raised when an attempt is made to install a reactor which cannot be found.
+    """
+
+
+@implementer(IPlugin, IReactorInstaller)
+class Reactor:
+    """
+    @ivar moduleName: The fully-qualified Python name of the module of which
+    the install callable is an attribute.
+    """
+
+    def __init__(self, shortName: str, moduleName: str, description: str):
+        self.shortName = shortName
+        self.moduleName = moduleName
+        self.description = description
+
+    def install(self) -> None:
+        namedAny(self.moduleName).install()
+
+
+def getReactorTypes() -> Iterable[IReactorInstaller]:
+    """
+    Return an iterator of L{IReactorInstaller} plugins.
+    """
+    return getPlugins(IReactorInstaller)
+
+
+def installReactor(shortName: str) -> IReactorCore:
+    """
+    Install the reactor with the given C{shortName} attribute.
+
+    @raise NoSuchReactor: If no reactor is found with a matching C{shortName}.
+
+    @raise Exception: Anything that the specified reactor can raise when installed.
+    """
+    for installer in getReactorTypes():
+        if installer.shortName == shortName:
+            installer.install()
+            from twisted.internet import reactor
+
+            return cast(IReactorCore, reactor)
+    raise NoSuchReactor(shortName)
diff --git a/contrib/python/Twisted/py3/twisted/application/runner/__init__.py b/contrib/python/Twisted/py3/twisted/application/runner/__init__.py
new file mode 100644
index 00000000000..6da0ac04e7c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/runner/__init__.py
@@ -0,0 +1,7 @@
+# -*- test-case-name: twisted.application.runner.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Facilities for running a Twisted application.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/application/runner/_exit.py b/contrib/python/Twisted/py3/twisted/application/runner/_exit.py
new file mode 100644
index 00000000000..3d6bc10f6f9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/runner/_exit.py
@@ -0,0 +1,99 @@
+# -*- test-case-name: twisted.application.runner.test.test_exit -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+System exit support.
+"""
+
+import typing
+from enum import IntEnum
+from sys import exit as sysexit, stderr, stdout
+from typing import Union
+
+try:
+    import posix as Status
+except ImportError:
+
+    class Status:  # type: ignore[no-redef]
+        """
+        Object to hang C{EX_*} values off of as a substitute for L{posix}.
+        """
+
+        EX__BASE = 64
+
+        EX_OK = 0
+        EX_USAGE = EX__BASE
+        EX_DATAERR = EX__BASE + 1
+        EX_NOINPUT = EX__BASE + 2
+        EX_NOUSER = EX__BASE + 3
+        EX_NOHOST = EX__BASE + 4
+        EX_UNAVAILABLE = EX__BASE + 5
+        EX_SOFTWARE = EX__BASE + 6
+        EX_OSERR = EX__BASE + 7
+        EX_OSFILE = EX__BASE + 8
+        EX_CANTCREAT = EX__BASE + 9
+        EX_IOERR = EX__BASE + 10
+        EX_TEMPFAIL = EX__BASE + 11
+        EX_PROTOCOL = EX__BASE + 12
+        EX_NOPERM = EX__BASE + 13
+        EX_CONFIG = EX__BASE + 14
+
+
+class ExitStatus(IntEnum):
+    """
+    Standard exit status codes for system programs.
+
+    @cvar EX_OK: Successful termination.
+    @cvar EX_USAGE: Command line usage error.
+    @cvar EX_DATAERR: Data format error.
+    @cvar EX_NOINPUT: Cannot open input.
+    @cvar EX_NOUSER: Addressee unknown.
+    @cvar EX_NOHOST: Host name unknown.
+    @cvar EX_UNAVAILABLE: Service unavailable.
+    @cvar EX_SOFTWARE: Internal software error.
+    @cvar EX_OSERR: System error (e.g., can't fork).
+    @cvar EX_OSFILE: Critical OS file missing.
+    @cvar EX_CANTCREAT: Can't create (user) output file.
+    @cvar EX_IOERR: Input/output error.
+    @cvar EX_TEMPFAIL: Temporary failure; the user is invited to retry.
+    @cvar EX_PROTOCOL: Remote error in protocol.
+    @cvar EX_NOPERM: Permission denied.
+    @cvar EX_CONFIG: Configuration error.
+    """
+
+    EX_OK = Status.EX_OK
+    EX_USAGE = Status.EX_USAGE
+    EX_DATAERR = Status.EX_DATAERR
+    EX_NOINPUT = Status.EX_NOINPUT
+    EX_NOUSER = Status.EX_NOUSER
+    EX_NOHOST = Status.EX_NOHOST
+    EX_UNAVAILABLE = Status.EX_UNAVAILABLE
+    EX_SOFTWARE = Status.EX_SOFTWARE
+    EX_OSERR = Status.EX_OSERR
+    EX_OSFILE = Status.EX_OSFILE
+    EX_CANTCREAT = Status.EX_CANTCREAT
+    EX_IOERR = Status.EX_IOERR
+    EX_TEMPFAIL = Status.EX_TEMPFAIL
+    EX_PROTOCOL = Status.EX_PROTOCOL
+    EX_NOPERM = Status.EX_NOPERM
+    EX_CONFIG = Status.EX_CONFIG
+
+
+def exit(status: Union[int, ExitStatus], message: str = "") -> "typing.NoReturn":
+    """
+    Exit the python interpreter with the given status and an optional message.
+
+    @param status: An exit status. An appropriate value from L{ExitStatus} is
+        recommended.
+    @param message: An optional message to print.
+    """
+    if message:
+        if status == ExitStatus.EX_OK:
+            out = stdout
+        else:
+            out = stderr
+        out.write(message)
+        out.write("\n")
+
+    sysexit(status)
diff --git a/contrib/python/Twisted/py3/twisted/application/runner/_pidfile.py b/contrib/python/Twisted/py3/twisted/application/runner/_pidfile.py
new file mode 100644
index 00000000000..b6aab1499fb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/runner/_pidfile.py
@@ -0,0 +1,282 @@
+# -*- test-case-name: twisted.application.runner.test.test_pidfile -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+PID file.
+"""
+from __future__ import annotations
+
+import errno
+from os import getpid, kill, name as SYSTEM_NAME
+from types import TracebackType
+from typing import Any, Optional, Type
+
+from zope.interface import Interface, implementer
+
+from twisted.logger import Logger
+from twisted.python.filepath import FilePath
+
+
+class IPIDFile(Interface):
+    """
+    Manages a file that remembers a process ID.
+    """
+
+    def read() -> int:
+        """
+        Read the process ID stored in this PID file.
+
+        @return: The contained process ID.
+
+        @raise NoPIDFound: If this PID file does not exist.
+        @raise EnvironmentError: If this PID file cannot be read.
+        @raise ValueError: If this PID file's content is invalid.
+        """
+
+    def writeRunningPID() -> None:
+        """
+        Store the PID of the current process in this PID file.
+
+        @raise EnvironmentError: If this PID file cannot be written.
+        """
+
+    def remove() -> None:
+        """
+        Remove this PID file.
+
+        @raise EnvironmentError: If this PID file cannot be removed.
+        """
+
+    def isRunning() -> bool:
+        """
+        Determine whether there is a running process corresponding to the PID
+        in this PID file.
+
+        @return: True if this PID file contains a PID and a process with that
+            PID is currently running; false otherwise.
+
+        @raise EnvironmentError: If this PID file cannot be read.
+        @raise InvalidPIDFileError: If this PID file's content is invalid.
+        @raise StalePIDFileError: If this PID file's content refers to a PID
+            for which there is no corresponding running process.
+        """
+
+    def __enter__() -> "IPIDFile":
+        """
+        Enter a context using this PIDFile.
+
+        Writes the PID file with the PID of the running process.
+
+        @raise AlreadyRunningError: A process corresponding to the PID in this
+            PID file is already running.
+        """
+
+    def __exit__(
+        excType: Optional[Type[BaseException]],
+        excValue: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Optional[bool]:
+        """
+        Exit a context using this PIDFile.
+
+        Removes the PID file.
+        """
+
+
+@implementer(IPIDFile)
+class PIDFile:
+    """
+    Concrete implementation of L{IPIDFile}.
+
+    This implementation is presently not supported on non-POSIX platforms.
+    Specifically, calling L{PIDFile.isRunning} will raise
+    L{NotImplementedError}.
+    """
+
+    _log = Logger()
+
+    @staticmethod
+    def _format(pid: int) -> bytes:
+        """
+        Format a PID file's content.
+
+        @param pid: A process ID.
+
+        @return: Formatted PID file contents.
+        """
+        return f"{int(pid)}\n".encode()
+
+    def __init__(self, filePath: FilePath[Any]) -> None:
+        """
+        @param filePath: The path to the PID file on disk.
+        """
+        self.filePath = filePath
+
+    def read(self) -> int:
+        pidString = b""
+        try:
+            with self.filePath.open() as fh:
+                for pidString in fh:
+                    break
+        except OSError as e:
+            if e.errno == errno.ENOENT:  # No such file
+                raise NoPIDFound("PID file does not exist")
+            raise
+
+        try:
+            return int(pidString)
+        except ValueError:
+            raise InvalidPIDFileError(
+                f"non-integer PID value in PID file: {pidString!r}"
+            )
+
+    def _write(self, pid: int) -> None:
+        """
+        Store a PID in this PID file.
+
+        @param pid: A PID to store.
+
+        @raise EnvironmentError: If this PID file cannot be written.
+        """
+        self.filePath.setContent(self._format(pid=pid))
+
+    def writeRunningPID(self) -> None:
+        self._write(getpid())
+
+    def remove(self) -> None:
+        self.filePath.remove()
+
+    def isRunning(self) -> bool:
+        try:
+            pid = self.read()
+        except NoPIDFound:
+            return False
+
+        if SYSTEM_NAME == "posix":
+            return self._pidIsRunningPOSIX(pid)
+        else:
+            raise NotImplementedError(f"isRunning is not implemented on {SYSTEM_NAME}")
+
+    @staticmethod
+    def _pidIsRunningPOSIX(pid: int) -> bool:
+        """
+        POSIX implementation for running process check.
+
+        Determine whether there is a running process corresponding to the given
+        PID.
+
+        @param pid: The PID to check.
+
+        @return: True if the given PID is currently running; false otherwise.
+
+        @raise EnvironmentError: If this PID file cannot be read.
+        @raise InvalidPIDFileError: If this PID file's content is invalid.
+        @raise StalePIDFileError: If this PID file's content refers to a PID
+            for which there is no corresponding running process.
+        """
+        try:
+            kill(pid, 0)
+        except OSError as e:
+            if e.errno == errno.ESRCH:  # No such process
+                raise StalePIDFileError("PID file refers to non-existing process")
+            elif e.errno == errno.EPERM:  # Not permitted to kill
+                return True
+            else:
+                raise
+        else:
+            return True
+
+    def __enter__(self) -> "PIDFile":
+        try:
+            if self.isRunning():
+                raise AlreadyRunningError()
+        except StalePIDFileError:
+            self._log.info("Replacing stale PID file: {log_source}")
+        self.writeRunningPID()
+        return self
+
+    def __exit__(
+        self,
+        excType: Optional[Type[BaseException]],
+        excValue: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None:
+        self.remove()
+        return None
+
+
+@implementer(IPIDFile)
+class NonePIDFile:
+    """
+    PID file implementation that does nothing.
+
+    This is meant to be used as a "active None" object in place of a PID file
+    when no PID file is desired.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    def read(self) -> int:
+        raise NoPIDFound("PID file does not exist")
+
+    def _write(self, pid: int) -> None:
+        """
+        Store a PID in this PID file.
+
+        @param pid: A PID to store.
+
+        @raise EnvironmentError: If this PID file cannot be written.
+
+        @note: This implementation always raises an L{EnvironmentError}.
+        """
+        raise OSError(errno.EPERM, "Operation not permitted")
+
+    def writeRunningPID(self) -> None:
+        self._write(0)
+
+    def remove(self) -> None:
+        raise OSError(errno.ENOENT, "No such file or directory")
+
+    def isRunning(self) -> bool:
+        return False
+
+    def __enter__(self) -> "NonePIDFile":
+        return self
+
+    def __exit__(
+        self,
+        excType: Optional[Type[BaseException]],
+        excValue: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None:
+        return None
+
+
+nonePIDFile: IPIDFile = NonePIDFile()
+
+
+class AlreadyRunningError(Exception):
+    """
+    Process is already running.
+    """
+
+
+class InvalidPIDFileError(Exception):
+    """
+    PID file contents are invalid.
+    """
+
+
+class StalePIDFileError(Exception):
+    """
+    PID file contents are valid, but there is no process with the referenced
+    PID.
+    """
+
+
+class NoPIDFound(Exception):
+    """
+    No PID found in PID file.
+    """
diff --git a/contrib/python/Twisted/py3/twisted/application/runner/_runner.py b/contrib/python/Twisted/py3/twisted/application/runner/_runner.py
new file mode 100644
index 00000000000..17b21962ae6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/runner/_runner.py
@@ -0,0 +1,166 @@
+# -*- test-case-name: twisted.application.runner.test.test_runner -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted application runner.
+"""
+
+from os import kill
+from signal import SIGTERM
+from sys import stderr
+from typing import Any, Callable, Mapping, TextIO
+
+from attr import Factory, attrib, attrs
+from constantly import NamedConstant  # type: ignore[import]
+
+from twisted.internet.interfaces import IReactorCore
+from twisted.logger import (
+    FileLogObserver,
+    FilteringLogObserver,
+    Logger,
+    LogLevel,
+    LogLevelFilterPredicate,
+    globalLogBeginner,
+    textFileLogObserver,
+)
+from ._exit import ExitStatus, exit
+from ._pidfile import AlreadyRunningError, InvalidPIDFileError, IPIDFile, nonePIDFile
+
+
+@attrs(frozen=True)
+class Runner:
+    """
+    Twisted application runner.
+
+    @cvar _log: The logger attached to this class.
+
+    @ivar _reactor: The reactor to start and run the application in.
+    @ivar _pidFile: The file to store the running process ID in.
+    @ivar _kill: Whether this runner should kill an existing running
+        instance of the application.
+    @ivar _defaultLogLevel: The default log level to start the logging
+        system with.
+    @ivar _logFile: A file stream to write logging output to.
+    @ivar _fileLogObserverFactory: A factory for the file log observer to
+        use when starting the logging system.
+    @ivar _whenRunning: Hook to call after the reactor is running;
+        this is where the application code that relies on the reactor gets
+        called.
+    @ivar _whenRunningArguments: Keyword arguments to pass to
+        C{whenRunning} when it is called.
+    @ivar _reactorExited: Hook to call after the reactor exits.
+    @ivar _reactorExitedArguments: Keyword arguments to pass to
+        C{reactorExited} when it is called.
+    """
+
+    _log = Logger()
+
+    _reactor = attrib(type=IReactorCore)
+    _pidFile = attrib(type=IPIDFile, default=nonePIDFile)
+    _kill = attrib(type=bool, default=False)
+    _defaultLogLevel = attrib(type=NamedConstant, default=LogLevel.info)
+    _logFile = attrib(type=TextIO, default=stderr)
+    _fileLogObserverFactory = attrib(
+        type=Callable[[TextIO], FileLogObserver], default=textFileLogObserver
+    )
+    _whenRunning = attrib(type=Callable[..., None], default=lambda **_: None)
+    _whenRunningArguments = attrib(type=Mapping[str, Any], default=Factory(dict))
+    _reactorExited = attrib(type=Callable[..., None], default=lambda **_: None)
+    _reactorExitedArguments = attrib(type=Mapping[str, Any], default=Factory(dict))
+
+    def run(self) -> None:
+        """
+        Run this command.
+        """
+        pidFile = self._pidFile
+
+        self.killIfRequested()
+
+        try:
+            with pidFile:
+                self.startLogging()
+                self.startReactor()
+                self.reactorExited()
+
+        except AlreadyRunningError:
+            exit(ExitStatus.EX_CONFIG, "Already running.")
+            # When testing, patched exit doesn't exit
+            return  # type: ignore[unreachable]
+
+    def killIfRequested(self) -> None:
+        """
+        If C{self._kill} is true, attempt to kill a running instance of the
+        application.
+        """
+        pidFile = self._pidFile
+
+        if self._kill:
+            if pidFile is nonePIDFile:
+                exit(ExitStatus.EX_USAGE, "No PID file specified.")
+                # When testing, patched exit doesn't exit
+                return  # type: ignore[unreachable]
+
+            try:
+                pid = pidFile.read()
+            except OSError:
+                exit(ExitStatus.EX_IOERR, "Unable to read PID file.")
+                # When testing, patched exit doesn't exit
+                return  # type: ignore[unreachable]
+            except InvalidPIDFileError:
+                exit(ExitStatus.EX_DATAERR, "Invalid PID file.")
+                # When testing, patched exit doesn't exit
+                return  # type: ignore[unreachable]
+
+            self.startLogging()
+            self._log.info("Terminating process: {pid}", pid=pid)
+
+            kill(pid, SIGTERM)
+
+            exit(ExitStatus.EX_OK)
+            # When testing, patched exit doesn't exit
+            return  # type: ignore[unreachable]
+
+    def startLogging(self) -> None:
+        """
+        Start the L{twisted.logger} logging system.
+        """
+        logFile = self._logFile
+
+        fileLogObserverFactory = self._fileLogObserverFactory
+
+        fileLogObserver = fileLogObserverFactory(logFile)
+
+        logLevelPredicate = LogLevelFilterPredicate(
+            defaultLogLevel=self._defaultLogLevel
+        )
+
+        filteringObserver = FilteringLogObserver(fileLogObserver, [logLevelPredicate])
+
+        globalLogBeginner.beginLoggingTo([filteringObserver])
+
+    def startReactor(self) -> None:
+        """
+        Register C{self._whenRunning} with the reactor so that it is called
+        once the reactor is running, then start the reactor.
+        """
+        self._reactor.callWhenRunning(self.whenRunning)
+
+        self._log.info("Starting reactor...")
+        self._reactor.run()
+
+    def whenRunning(self) -> None:
+        """
+        Call C{self._whenRunning} with C{self._whenRunningArguments}.
+
+        @note: This method is called after the reactor starts running.
+        """
+        self._whenRunning(**self._whenRunningArguments)
+
+    def reactorExited(self) -> None:
+        """
+        Call C{self._reactorExited} with C{self._reactorExitedArguments}.
+
+        @note: This method is called after the reactor exits.
+        """
+        self._reactorExited(**self._reactorExitedArguments)
diff --git a/contrib/python/Twisted/py3/twisted/application/service.py b/contrib/python/Twisted/py3/twisted/application/service.py
new file mode 100644
index 00000000000..58c3df2edbe
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/service.py
@@ -0,0 +1,420 @@
+# -*- test-case-name: twisted.application.test.test_service -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Service architecture for Twisted.
+
+Services are arranged in a hierarchy. At the leafs of the hierarchy,
+the services which actually interact with the outside world are started.
+Services can be named or anonymous -- usually, they will be named if
+there is need to access them through the hierarchy (from a parent or
+a sibling).
+
+Maintainer: Moshe Zadka
+"""
+
+
+from zope.interface import Attribute, Interface, implementer
+
+from twisted.internet import defer
+from twisted.persisted import sob
+from twisted.plugin import IPlugin
+from twisted.python import components
+from twisted.python.reflect import namedAny
+
+
+class IServiceMaker(Interface):
+    """
+    An object which can be used to construct services in a flexible
+    way.
+
+    This interface should most often be implemented along with
+    L{twisted.plugin.IPlugin}, and will most often be used by the
+    'twistd' command.
+    """
+
+    tapname = Attribute(
+        "A short string naming this Twisted plugin, for example 'web' or "
+        "'pencil'. This name will be used as the subcommand of 'twistd'."
+    )
+
+    description = Attribute(
+        "A brief summary of the features provided by this "
+        "Twisted application plugin."
+    )
+
+    options = Attribute(
+        "A C{twisted.python.usage.Options} subclass defining the "
+        "configuration options for this application."
+    )
+
+    def makeService(options):
+        """
+        Create and return an object providing
+        L{twisted.application.service.IService}.
+
+        @param options: A mapping (typically a C{dict} or
+        L{twisted.python.usage.Options} instance) of configuration
+        options to desired configuration values.
+        """
+
+
+@implementer(IPlugin, IServiceMaker)
+class ServiceMaker:
+    """
+    Utility class to simplify the definition of L{IServiceMaker} plugins.
+    """
+
+    def __init__(self, name, module, description, tapname):
+        self.name = name
+        self.module = module
+        self.description = description
+        self.tapname = tapname
+
+    @property
+    def options(self):
+        return namedAny(self.module).Options
+
+    @property
+    def makeService(self):
+        return namedAny(self.module).makeService
+
+
+class IService(Interface):
+    """
+    A service.
+
+    Run start-up and shut-down code at the appropriate times.
+    """
+
+    name = Attribute("A C{str} which is the name of the service or C{None}.")
+
+    running = Attribute("A C{boolean} which indicates whether the service is running.")
+
+    parent = Attribute("An C{IServiceCollection} which is the parent or C{None}.")
+
+    def setName(name):
+        """
+        Set the name of the service.
+
+        @type name: C{str}
+        @raise RuntimeError: Raised if the service already has a parent.
+        """
+
+    def setServiceParent(parent):
+        """
+        Set the parent of the service.  This method is responsible for setting
+        the C{parent} attribute on this service (the child service).
+
+        @type parent: L{IServiceCollection}
+        @raise RuntimeError: Raised if the service already has a parent
+            or if the service has a name and the parent already has a child
+            by that name.
+        """
+
+    def disownServiceParent():
+        """
+        Use this API to remove an L{IService} from an L{IServiceCollection}.
+
+        This method is used symmetrically with L{setServiceParent} in that it
+        sets the C{parent} attribute on the child.
+
+        @rtype: L{Deferred<defer.Deferred>}
+        @return: a L{Deferred<defer.Deferred>} which is triggered when the
+            service has finished shutting down. If shutting down is immediate,
+            a value can be returned (usually, L{None}).
+        """
+
+    def startService():
+        """
+        Start the service.
+        """
+
+    def stopService():
+        """
+        Stop the service.
+
+        @rtype: L{Deferred<defer.Deferred>}
+        @return: a L{Deferred<defer.Deferred>} which is triggered when the
+            service has finished shutting down. If shutting down is immediate,
+            a value can be returned (usually, L{None}).
+        """
+
+    def privilegedStartService():
+        """
+        Do preparation work for starting the service.
+
+        Here things which should be done before changing directory,
+        root or shedding privileges are done.
+        """
+
+
+@implementer(IService)
+class Service:
+    """
+    Base class for services.
+
+    Most services should inherit from this class. It handles the
+    book-keeping responsibilities of starting and stopping, as well
+    as not serializing this book-keeping information.
+    """
+
+    running = 0
+    name = None
+    parent = None
+
+    def __getstate__(self):
+        dict = self.__dict__.copy()
+        if "running" in dict:
+            del dict["running"]
+        return dict
+
+    def setName(self, name):
+        if self.parent is not None:
+            raise RuntimeError("cannot change name when parent exists")
+        self.name = name
+
+    def setServiceParent(self, parent):
+        if self.parent is not None:
+            self.disownServiceParent()
+        parent = IServiceCollection(parent, parent)
+        self.parent = parent
+        self.parent.addService(self)
+
+    def disownServiceParent(self):
+        d = self.parent.removeService(self)
+        self.parent = None
+        return d
+
+    def privilegedStartService(self):
+        pass
+
+    def startService(self):
+        self.running = 1
+
+    def stopService(self):
+        self.running = 0
+
+
+class IServiceCollection(Interface):
+    """
+    Collection of services.
+
+    Contain several services, and manage their start-up/shut-down.
+    Services can be accessed by name if they have a name, and it
+    is always possible to iterate over them.
+    """
+
+    def getServiceNamed(name):
+        """
+        Get the child service with a given name.
+
+        @type name: C{str}
+        @rtype: L{IService}
+        @raise KeyError: Raised if the service has no child with the
+            given name.
+        """
+
+    def __iter__():
+        """
+        Get an iterator over all child services.
+        """
+
+    def addService(service):
+        """
+        Add a child service.
+
+        Only implementations of L{IService.setServiceParent} should use this
+        method.
+
+        @type service: L{IService}
+        @raise RuntimeError: Raised if the service has a child with
+            the given name.
+        """
+
+    def removeService(service):
+        """
+        Remove a child service.
+
+        Only implementations of L{IService.disownServiceParent} should
+        use this method.
+
+        @type service: L{IService}
+        @raise ValueError: Raised if the given service is not a child.
+        @rtype: L{Deferred<defer.Deferred>}
+        @return: a L{Deferred<defer.Deferred>} which is triggered when the
+            service has finished shutting down. If shutting down is immediate,
+            a value can be returned (usually, L{None}).
+        """
+
+
+@implementer(IServiceCollection)
+class MultiService(Service):
+    """
+    Straightforward Service Container.
+
+    Hold a collection of services, and manage them in a simplistic
+    way. No service will wait for another, but this object itself
+    will not finish shutting down until all of its child services
+    will finish.
+    """
+
+    def __init__(self):
+        self.services = []
+        self.namedServices = {}
+        self.parent = None
+
+    def privilegedStartService(self):
+        Service.privilegedStartService(self)
+        for service in self:
+            service.privilegedStartService()
+
+    def startService(self):
+        Service.startService(self)
+        for service in self:
+            service.startService()
+
+    def stopService(self):
+        Service.stopService(self)
+        l = []
+        services = list(self)
+        services.reverse()
+        for service in services:
+            l.append(defer.maybeDeferred(service.stopService))
+        return defer.DeferredList(l)
+
+    def getServiceNamed(self, name):
+        return self.namedServices[name]
+
+    def __iter__(self):
+        return iter(self.services)
+
+    def addService(self, service):
+        if service.name is not None:
+            if service.name in self.namedServices:
+                raise RuntimeError(
+                    "cannot have two services with same name" " '%s'" % service.name
+                )
+            self.namedServices[service.name] = service
+        self.services.append(service)
+        if self.running:
+            # It may be too late for that, but we will do our best
+            service.privilegedStartService()
+            service.startService()
+
+    def removeService(self, service):
+        if service.name:
+            del self.namedServices[service.name]
+        self.services.remove(service)
+        if self.running:
+            # Returning this so as not to lose information from the
+            # MultiService.stopService deferred.
+            return service.stopService()
+        else:
+            return None
+
+
+class IProcess(Interface):
+    """
+    Process running parameters.
+
+    Represents parameters for how processes should be run.
+    """
+
+    processName = Attribute(
+        """
+        A C{str} giving the name the process should have in ps (or L{None}
+        to leave the name alone).
+        """
+    )
+
+    uid = Attribute(
+        """
+        An C{int} giving the user id as which the process should run (or
+        L{None} to leave the UID alone).
+        """
+    )
+
+    gid = Attribute(
+        """
+        An C{int} giving the group id as which the process should run (or
+        L{None} to leave the GID alone).
+        """
+    )
+
+
+@implementer(IProcess)
+class Process:
+    """
+    Process running parameters.
+
+    Sets up uid/gid in the constructor, and has a default
+    of L{None} as C{processName}.
+    """
+
+    processName = None
+
+    def __init__(self, uid=None, gid=None):
+        """
+        Set uid and gid.
+
+        @param uid: The user ID as whom to execute the process.  If
+            this is L{None}, no attempt will be made to change the UID.
+
+        @param gid: The group ID as whom to execute the process.  If
+            this is L{None}, no attempt will be made to change the GID.
+        """
+        self.uid = uid
+        self.gid = gid
+
+
+def Application(name, uid=None, gid=None):
+    """
+    Return a compound class.
+
+    Return an object supporting the L{IService}, L{IServiceCollection},
+    L{IProcess} and L{sob.IPersistable} interfaces, with the given
+    parameters. Always access the return value by explicit casting to
+    one of the interfaces.
+    """
+    ret = components.Componentized()
+    availableComponents = [MultiService(), Process(uid, gid), sob.Persistent(ret, name)]
+
+    for comp in availableComponents:
+        ret.addComponent(comp, ignoreClass=1)
+    IService(ret).setName(name)
+    return ret
+
+
+def loadApplication(filename, kind, passphrase=None):
+    """
+    Load Application from a given file.
+
+    The serialization format it was saved in should be given as
+    C{kind}, and is one of C{pickle}, C{source}, C{xml} or C{python}. If
+    C{passphrase} is given, the application was encrypted with the
+    given passphrase.
+
+    @type filename: C{str}
+    @type kind: C{str}
+    @type passphrase: C{str}
+    """
+    if kind == "python":
+        application = sob.loadValueFromFile(filename, "application")
+    else:
+        application = sob.load(filename, kind)
+    return application
+
+
+__all__ = [
+    "IServiceMaker",
+    "IService",
+    "Service",
+    "IServiceCollection",
+    "MultiService",
+    "IProcess",
+    "Process",
+    "Application",
+    "loadApplication",
+]
diff --git a/contrib/python/Twisted/py3/twisted/application/strports.py b/contrib/python/Twisted/py3/twisted/application/strports.py
new file mode 100644
index 00000000000..3c96ee335d4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/strports.py
@@ -0,0 +1,83 @@
+# -*- test-case-name: twisted.test.test_strports -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Construct listening port services from a simple string description.
+
+@see: L{twisted.internet.endpoints.serverFromString}
+@see: L{twisted.internet.endpoints.clientFromString}
+"""
+from typing import Optional, cast
+
+from twisted.application.internet import StreamServerEndpointService
+from twisted.internet import endpoints, interfaces
+
+
+def _getReactor() -> interfaces.IReactorCore:
+    from twisted.internet import reactor
+
+    return cast(interfaces.IReactorCore, reactor)
+
+
+def service(
+    description: str,
+    factory: interfaces.IProtocolFactory,
+    reactor: Optional[interfaces.IReactorCore] = None,
+) -> StreamServerEndpointService:
+    """
+    Return the service corresponding to a description.
+
+    @param description: The description of the listening port, in the syntax
+        described by L{twisted.internet.endpoints.serverFromString}.
+    @type description: C{str}
+
+    @param factory: The protocol factory which will build protocols for
+        connections to this service.
+    @type factory: L{twisted.internet.interfaces.IProtocolFactory}
+
+    @rtype: C{twisted.application.service.IService}
+    @return: the service corresponding to a description of a reliable stream
+        server.
+
+    @see: L{twisted.internet.endpoints.serverFromString}
+    """
+    if reactor is None:
+        reactor = _getReactor()
+
+    svc = StreamServerEndpointService(
+        endpoints.serverFromString(reactor, description), factory
+    )
+    svc._raiseSynchronously = True
+    return svc
+
+
+def listen(
+    description: str, factory: interfaces.IProtocolFactory
+) -> interfaces.IListeningPort:
+    """
+    Listen on a port corresponding to a description.
+
+    @param description: The description of the connecting port, in the syntax
+        described by L{twisted.internet.endpoints.serverFromString}.
+    @type description: L{str}
+
+    @param factory: The protocol factory which will build protocols on
+        connection.
+    @type factory: L{twisted.internet.interfaces.IProtocolFactory}
+
+    @rtype: L{twisted.internet.interfaces.IListeningPort}
+    @return: the port corresponding to a description of a reliable virtual
+        circuit server.
+
+    @see: L{twisted.internet.endpoints.serverFromString}
+    """
+    from twisted.internet import reactor
+
+    name, args, kw = endpoints._parseServer(description, factory)
+    return cast(
+        interfaces.IListeningPort, getattr(reactor, "listen" + name)(*args, **kw)
+    )
+
+
+__all__ = ["service", "listen"]
diff --git a/contrib/python/Twisted/py3/twisted/application/twist/__init__.py b/contrib/python/Twisted/py3/twisted/application/twist/__init__.py
new file mode 100644
index 00000000000..ea7c5d29ce7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/twist/__init__.py
@@ -0,0 +1,7 @@
+# -*- test-case-name: twisted.application.twist.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+C{twist} command line tool.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/application/twist/_options.py b/contrib/python/Twisted/py3/twisted/application/twist/_options.py
new file mode 100644
index 00000000000..2bcd207bf4c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/twist/_options.py
@@ -0,0 +1,207 @@
+# -*- test-case-name: twisted.application.twist.test.test_options -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Command line options for C{twist}.
+"""
+
+import typing
+from sys import stderr, stdout
+from textwrap import dedent
+from typing import Callable, Iterable, Mapping, Optional, Sequence, Tuple, cast
+
+from twisted.copyright import version
+from twisted.internet.interfaces import IReactorCore
+from twisted.logger import (
+    InvalidLogLevelError,
+    LogLevel,
+    jsonFileLogObserver,
+    textFileLogObserver,
+)
+from twisted.plugin import getPlugins
+from twisted.python.usage import Options, UsageError
+from ..reactors import NoSuchReactor, getReactorTypes, installReactor
+from ..runner._exit import ExitStatus, exit
+from ..service import IServiceMaker
+
+openFile = open
+
+
+def _update_doc(opt: Callable[["TwistOptions", str], None], **kwargs: str) -> None:
+    """
+    Update the docstring of a method that implements an option.
+    The string is dedented and the given keyword arguments are substituted.
+    """
+    opt.__doc__ = dedent(opt.__doc__ or "").format(**kwargs)
+
+
+class TwistOptions(Options):
+    """
+    Command line options for C{twist}.
+    """
+
+    defaultReactorName = "default"
+    defaultLogLevel = LogLevel.info
+
+    def __init__(self) -> None:
+        Options.__init__(self)
+
+        self["reactorName"] = self.defaultReactorName
+        self["logLevel"] = self.defaultLogLevel
+        self["logFile"] = stdout
+        # An empty long description is explicitly set here as otherwise
+        # when executing from distributed trial twisted.python.usage will
+        # pull the description from `__main__` which is another entry point.
+        self.longdesc = ""
+
+    def getSynopsis(self) -> str:
+        return f"{Options.getSynopsis(self)} plugin [plugin_options]"
+
+    def opt_version(self) -> "typing.NoReturn":
+        """
+        Print version and exit.
+        """
+        exit(ExitStatus.EX_OK, f"{version}")
+
+    def opt_reactor(self, name: str) -> None:
+        """
+        The name of the reactor to use.
+        (options: {options})
+        """
+        # Actually actually actually install the reactor right at this very
+        # moment, before any other code (for example, a sub-command plugin)
+        # runs and accidentally imports and installs the default reactor.
+        try:
+            self["reactor"] = self.installReactor(name)
+        except NoSuchReactor:
+            raise UsageError(f"Unknown reactor: {name}")
+        else:
+            self["reactorName"] = name
+
+    _update_doc(
+        opt_reactor,
+        options=", ".join(f'"{rt.shortName}"' for rt in getReactorTypes()),
+    )
+
+    def installReactor(self, name: str) -> IReactorCore:
+        """
+        Install the reactor.
+        """
+        if name == self.defaultReactorName:
+            from twisted.internet import reactor
+
+            return cast(IReactorCore, reactor)
+        else:
+            return installReactor(name)
+
+    def opt_log_level(self, levelName: str) -> None:
+        """
+        Set default log level.
+        (options: {options}; default: "{default}")
+        """
+        try:
+            self["logLevel"] = LogLevel.levelWithName(levelName)
+        except InvalidLogLevelError:
+            raise UsageError(f"Invalid log level: {levelName}")
+
+    _update_doc(
+        opt_log_level,
+        options=", ".join(
+            f'"{constant.name}"' for constant in LogLevel.iterconstants()
+        ),
+        default=defaultLogLevel.name,
+    )
+
+    def opt_log_file(self, fileName: str) -> None:
+        """
+        Log to file. ("-" for stdout, "+" for stderr; default: "-")
+        """
+        if fileName == "-":
+            self["logFile"] = stdout
+            return
+
+        if fileName == "+":
+            self["logFile"] = stderr
+            return
+
+        try:
+            self["logFile"] = openFile(fileName, "a")
+        except OSError as e:
+            exit(
+                ExitStatus.EX_IOERR,
+                f"Unable to open log file {fileName!r}: {e}",
+            )
+
+    def opt_log_format(self, format: str) -> None:
+        """
+        Log file format.
+        (options: "text", "json"; default: "text" if the log file is a tty,
+        otherwise "json")
+        """
+        format = format.lower()
+
+        if format == "text":
+            self["fileLogObserverFactory"] = textFileLogObserver
+        elif format == "json":
+            self["fileLogObserverFactory"] = jsonFileLogObserver
+        else:
+            raise UsageError(f"Invalid log format: {format}")
+        self["logFormat"] = format
+
+    _update_doc(opt_log_format)
+
+    def selectDefaultLogObserver(self) -> None:
+        """
+        Set C{fileLogObserverFactory} to the default appropriate for the
+        chosen C{logFile}.
+        """
+        if "fileLogObserverFactory" not in self:
+            logFile = self["logFile"]
+
+            if hasattr(logFile, "isatty") and logFile.isatty():
+                self["fileLogObserverFactory"] = textFileLogObserver
+                self["logFormat"] = "text"
+            else:
+                self["fileLogObserverFactory"] = jsonFileLogObserver
+                self["logFormat"] = "json"
+
+    def parseOptions(self, options: Optional[Sequence[str]] = None) -> None:
+        self.selectDefaultLogObserver()
+
+        Options.parseOptions(self, options=options)
+
+        if "reactor" not in self:
+            self["reactor"] = self.installReactor(self["reactorName"])
+
+    @property
+    def plugins(self) -> Mapping[str, IServiceMaker]:
+        if "plugins" not in self:
+            plugins = {}
+            for plugin in getPlugins(IServiceMaker):
+                plugins[plugin.tapname] = plugin
+            self["plugins"] = plugins
+
+        return cast(Mapping[str, IServiceMaker], self["plugins"])
+
+    @property
+    def subCommands(
+        self,
+    ) -> Iterable[Tuple[str, None, Callable[[IServiceMaker], Options], str]]:
+        plugins = self.plugins
+        for name in sorted(plugins):
+            plugin = plugins[name]
+
+            # Don't pass plugin.options along in order to avoid resolving the
+            # options attribute right away, in case it's a property with a
+            # non-trivial getter (eg, one which imports modules).
+            def options(plugin: IServiceMaker = plugin) -> Options:
+                return cast(Options, plugin.options())
+
+            yield (plugin.tapname, None, options, plugin.description)
+
+    def postOptions(self) -> None:
+        Options.postOptions(self)
+
+        if self.subCommand is None:
+            raise UsageError("No plugin specified.")
diff --git a/contrib/python/Twisted/py3/twisted/application/twist/_twist.py b/contrib/python/Twisted/py3/twisted/application/twist/_twist.py
new file mode 100644
index 00000000000..80cf4470f11
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/application/twist/_twist.py
@@ -0,0 +1,114 @@
+# -*- test-case-name: twisted.application.twist.test.test_twist -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Run a Twisted application.
+"""
+
+import sys
+from typing import Sequence
+
+from twisted.application.app import _exitWithSignal
+from twisted.internet.interfaces import IReactorCore, _ISupportsExitSignalCapturing
+from twisted.python.usage import Options, UsageError
+from ..runner._exit import ExitStatus, exit
+from ..runner._runner import Runner
+from ..service import Application, IService, IServiceMaker
+from ._options import TwistOptions
+
+
+class Twist:
+    """
+    Run a Twisted application.
+    """
+
+    @staticmethod
+    def options(argv: Sequence[str]) -> TwistOptions:
+        """
+        Parse command line options.
+
+        @param argv: Command line arguments.
+        @return: The parsed options.
+        """
+        options = TwistOptions()
+
+        try:
+            options.parseOptions(argv[1:])
+        except UsageError as e:
+            exit(ExitStatus.EX_USAGE, f"Error: {e}\n\n{options}")
+
+        return options
+
+    @staticmethod
+    def service(plugin: IServiceMaker, options: Options) -> IService:
+        """
+        Create the application service.
+
+        @param plugin: The name of the plugin that implements the service
+            application to run.
+        @param options: Options to pass to the application.
+        @return: The created application service.
+        """
+        service = plugin.makeService(options)
+        application = Application(plugin.tapname)
+        service.setServiceParent(application)
+
+        return IService(application)
+
+    @staticmethod
+    def startService(reactor: IReactorCore, service: IService) -> None:
+        """
+        Start the application service.
+
+        @param reactor: The reactor to run the service with.
+        @param service: The application service to run.
+        """
+        service.startService()
+
+        # Ask the reactor to stop the service before shutting down
+        reactor.addSystemEventTrigger("before", "shutdown", service.stopService)
+
+    @staticmethod
+    def run(twistOptions: TwistOptions) -> None:
+        """
+        Run the application service.
+
+        @param twistOptions: Command line options to convert to runner
+            arguments.
+        """
+        runner = Runner(
+            reactor=twistOptions["reactor"],
+            defaultLogLevel=twistOptions["logLevel"],
+            logFile=twistOptions["logFile"],
+            fileLogObserverFactory=twistOptions["fileLogObserverFactory"],
+        )
+        runner.run()
+        reactor = twistOptions["reactor"]
+        if _ISupportsExitSignalCapturing.providedBy(reactor):
+            if reactor._exitSignal is not None:
+                _exitWithSignal(reactor._exitSignal)
+
+    @classmethod
+    def main(cls, argv: Sequence[str] = sys.argv) -> None:
+        """
+        Executable entry point for L{Twist}.
+        Processes options and run a twisted reactor with a service.
+
+        @param argv: Command line arguments.
+        @type argv: L{list}
+        """
+        options = cls.options(argv)
+
+        reactor = options["reactor"]
+        # If subCommand is None, TwistOptions.parseOptions() raises UsageError
+        # and Twist.options() will exit the runner, so we'll never get here.
+        subCommand = options.subCommand
+        assert subCommand is not None
+        service = cls.service(
+            plugin=options.plugins[subCommand],
+            options=options.subOptions,
+        )
+
+        cls.startService(reactor, service)
+        cls.run(options)
diff --git a/contrib/python/Twisted/py3/twisted/conch/__init__.py b/contrib/python/Twisted/py3/twisted/conch/__init__.py
new file mode 100644
index 00000000000..adc49d01e64
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/__init__.py
@@ -0,0 +1,7 @@
+# -*- test-case-name: twisted.conch.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Conch: The Twisted Shell. Terminal emulation, SSHv2 and telnet.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/conch/avatar.py b/contrib/python/Twisted/py3/twisted/conch/avatar.py
new file mode 100644
index 00000000000..c9a106ef52d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/avatar.py
@@ -0,0 +1,56 @@
+# -*- test-case-name: twisted.conch.test.test_conch -*-
+
+
+from zope.interface import implementer
+
+from twisted.conch.error import ConchError
+from twisted.conch.interfaces import IConchUser
+from twisted.conch.ssh.connection import OPEN_UNKNOWN_CHANNEL_TYPE
+from twisted.logger import Logger
+from twisted.python.compat import nativeString
+
+
+@implementer(IConchUser)
+class ConchUser:
+    _log = Logger()
+
+    def __init__(self):
+        self.channelLookup = {}
+        self.subsystemLookup = {}
+
+    @property
+    def conn(self):
+        return self._conn
+
+    @conn.setter
+    def conn(self, value):
+        self._conn = value
+
+    def lookupChannel(self, channelType, windowSize, maxPacket, data):
+        klass = self.channelLookup.get(channelType, None)
+        if not klass:
+            raise ConchError(OPEN_UNKNOWN_CHANNEL_TYPE, "unknown channel")
+        else:
+            return klass(
+                remoteWindow=windowSize,
+                remoteMaxPacket=maxPacket,
+                data=data,
+                avatar=self,
+            )
+
+    def lookupSubsystem(self, subsystem, data):
+        self._log.debug(
+            "Subsystem lookup: {subsystem!r}", subsystem=self.subsystemLookup
+        )
+        klass = self.subsystemLookup.get(subsystem, None)
+        if not klass:
+            return False
+        return klass(data, avatar=self)
+
+    def gotGlobalRequest(self, requestType, data):
+        # XXX should this use method dispatch?
+        requestType = nativeString(requestType.replace(b"-", b"_"))
+        f = getattr(self, "global_%s" % requestType, None)
+        if not f:
+            return 0
+        return f(data)
diff --git a/contrib/python/Twisted/py3/twisted/conch/checkers.py b/contrib/python/Twisted/py3/twisted/conch/checkers.py
new file mode 100644
index 00000000000..3ade2d8eeb8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/checkers.py
@@ -0,0 +1,640 @@
+# -*- test-case-name: twisted.conch.test.test_checkers -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Provide L{ICredentialsChecker} implementations to be used in Conch protocols.
+"""
+
+
+import binascii
+import errno
+import sys
+from base64 import decodebytes
+from typing import IO, Any, Callable, Iterable, Iterator, Mapping, Optional, Tuple, cast
+
+from zope.interface import Interface, implementer, providedBy
+
+from incremental import Version
+from typing_extensions import Literal, Protocol
+
+from twisted.conch import error
+from twisted.conch.ssh import keys
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.cred.credentials import ISSHPrivateKey, IUsernamePassword
+from twisted.cred.error import UnauthorizedLogin, UnhandledCredentials
+from twisted.internet import defer
+from twisted.logger import Logger
+from twisted.plugins.cred_unix import verifyCryptedPassword
+from twisted.python import failure, reflect
+from twisted.python.deprecate import deprecatedModuleAttribute
+from twisted.python.filepath import FilePath
+from twisted.python.util import runAsEffectiveUser
+
+_log = Logger()
+
+
+class UserRecord(Tuple[str, str, int, int, str, str, str]):
+    """
+    A record in a UNIX-style password database. See L{pwd} for field details.
+
+    This corresponds to the undocumented type L{pwd.struct_passwd}, but lacks named
+    field accessors.
+    """
+
+    @property
+    def pw_dir(self) -> str:  # type: ignore[empty-body]
+        ...
+
+
+class UserDB(Protocol):
+    """
+    A database of users by name, like the stdlib L{pwd} module.
+
+    See L{twisted.python.fakepwd} for an in-memory implementation.
+    """
+
+    def getpwnam(self, username: str) -> UserRecord:
+        """
+        Lookup a user record by name.
+
+        @raises KeyError: when no such user exists
+        """
+
+
+pwd: Optional[UserDB]
+try:
+    import pwd as _pwd
+except ImportError:
+    pwd = None
+else:
+    pwd = cast(UserDB, _pwd)
+
+
+try:
+    import spwd as _spwd
+except ImportError:
+    spwd = None
+else:
+    spwd = _spwd
+
+
+class CryptedPasswordRecord(Protocol):
+    """
+    A sequence where the item at index 1 may be a crypted password.
+
+    Both L{pwd.struct_passwd} and L{spwd.struct_spwd} conform to this protocol.
+    """
+
+    def __getitem__(self, index: Literal[1]) -> str:
+        """
+        Get the crypted password.
+        """
+
+
+def _lookupUser(userdb: UserDB, username: bytes) -> UserRecord:
+    """
+    Lookup a user by name in a L{pwd}-style database.
+
+    @param userdb: The user database.
+
+    @param username: Identifying name in bytes. This will be decoded according
+    to the filesystem encoding, as the L{pwd} module does internally.
+
+    @raises KeyError: when the user doesn't exist
+    """
+    return userdb.getpwnam(username.decode(sys.getfilesystemencoding()))
+
+
+def _pwdGetByName(username: str) -> Optional[CryptedPasswordRecord]:
+    """
+    Look up a user in the /etc/passwd database using the pwd module.  If the
+    pwd module is not available, return None.
+
+    @param username: the username of the user to return the passwd database
+        information for.
+
+    @returns: A L{pwd.struct_passwd}, where field 1 may contain a crypted
+        password, or L{None} when the L{pwd} database is unavailable.
+
+    @raises KeyError: when no such user exists
+    """
+    if pwd is None:
+        return None
+    return cast(CryptedPasswordRecord, pwd.getpwnam(username))
+
+
+def _shadowGetByName(username: str) -> Optional[CryptedPasswordRecord]:
+    """
+    Look up a user in the /etc/shadow database using the spwd module. If it is
+    not available, return L{None}.
+
+    @param username: the username of the user to return the shadow database
+        information for.
+    @type username: L{str}
+
+    @returns: A L{spwd.struct_spwd}, where field 1 may contain a crypted
+        password, or L{None} when the L{spwd} database is unavailable.
+
+    @raises KeyError: when no such user exists
+    """
+    if spwd is not None:
+        f = spwd.getspnam
+    else:
+        return None
+    return cast(CryptedPasswordRecord, runAsEffectiveUser(0, 0, f, username))
+
+
+@implementer(ICredentialsChecker)
+class UNIXPasswordDatabase:
+    """
+    A checker which validates users out of the UNIX password databases, or
+    databases of a compatible format.
+
+    @ivar _getByNameFunctions: a C{list} of functions which are called in order
+        to validate a user.  The default value is such that the C{/etc/passwd}
+        database will be tried first, followed by the C{/etc/shadow} database.
+    """
+
+    credentialInterfaces = (IUsernamePassword,)
+
+    def __init__(self, getByNameFunctions=None):
+        if getByNameFunctions is None:
+            getByNameFunctions = [_pwdGetByName, _shadowGetByName]
+        self._getByNameFunctions = getByNameFunctions
+
+    def requestAvatarId(self, credentials):
+        # We get bytes, but the Py3 pwd module uses str. So attempt to decode
+        # it using the same method that CPython does for the file on disk.
+        username = credentials.username.decode(sys.getfilesystemencoding())
+        password = credentials.password.decode(sys.getfilesystemencoding())
+
+        for func in self._getByNameFunctions:
+            try:
+                pwnam = func(username)
+            except KeyError:
+                return defer.fail(UnauthorizedLogin("invalid username"))
+            else:
+                if pwnam is not None:
+                    crypted = pwnam[1]
+                    if crypted == "":
+                        continue
+
+                    if verifyCryptedPassword(crypted, password):
+                        return defer.succeed(credentials.username)
+        # fallback
+        return defer.fail(UnauthorizedLogin("unable to verify password"))
+
+
+@implementer(ICredentialsChecker)
+class SSHPublicKeyDatabase:
+    """
+    Checker that authenticates SSH public keys, based on public keys listed in
+    authorized_keys and authorized_keys2 files in user .ssh/ directories.
+    """
+
+    credentialInterfaces = (ISSHPrivateKey,)
+
+    _userdb: UserDB = cast(UserDB, pwd)
+
+    def requestAvatarId(self, credentials):
+        d = defer.maybeDeferred(self.checkKey, credentials)
+        d.addCallback(self._cbRequestAvatarId, credentials)
+        d.addErrback(self._ebRequestAvatarId)
+        return d
+
+    def _cbRequestAvatarId(self, validKey, credentials):
+        """
+        Check whether the credentials themselves are valid, now that we know
+        if the key matches the user.
+
+        @param validKey: A boolean indicating whether or not the public key
+            matches a key in the user's authorized_keys file.
+
+        @param credentials: The credentials offered by the user.
+        @type credentials: L{ISSHPrivateKey} provider
+
+        @raise UnauthorizedLogin: (as a failure) if the key does not match the
+            user in C{credentials}. Also raised if the user provides an invalid
+            signature.
+
+        @raise ValidPublicKey: (as a failure) if the key matches the user but
+            the credentials do not include a signature. See
+            L{error.ValidPublicKey} for more information.
+
+        @return: The user's username, if authentication was successful.
+        """
+        if not validKey:
+            return failure.Failure(UnauthorizedLogin("invalid key"))
+        if not credentials.signature:
+            return failure.Failure(error.ValidPublicKey())
+        else:
+            try:
+                pubKey = keys.Key.fromString(credentials.blob)
+                if pubKey.verify(credentials.signature, credentials.sigData):
+                    return credentials.username
+            except Exception:  # any error should be treated as a failed login
+                _log.failure("Error while verifying key")
+                return failure.Failure(UnauthorizedLogin("error while verifying key"))
+        return failure.Failure(UnauthorizedLogin("unable to verify key"))
+
+    def getAuthorizedKeysFiles(self, credentials):
+        """
+        Return a list of L{FilePath} instances for I{authorized_keys} files
+        which might contain information about authorized keys for the given
+        credentials.
+
+        On OpenSSH servers, the default location of the file containing the
+        list of authorized public keys is
+        U{$HOME/.ssh/authorized_keys<http://www.openbsd.org/cgi-bin/man.cgi?query=sshd_config>}.
+
+        I{$HOME/.ssh/authorized_keys2} is also returned, though it has been
+        U{deprecated by OpenSSH since
+        2001<http://marc.info/?m=100508718416162>}.
+
+        @return: A list of L{FilePath} instances to files with the authorized keys.
+        """
+        pwent = _lookupUser(self._userdb, credentials.username)
+        root = FilePath(pwent.pw_dir).child(".ssh")
+        files = ["authorized_keys", "authorized_keys2"]
+        return [root.child(f) for f in files]
+
+    def checkKey(self, credentials):
+        """
+        Retrieve files containing authorized keys and check against user
+        credentials.
+        """
+        ouid, ogid = _lookupUser(self._userdb, credentials.username)[2:4]
+        for filepath in self.getAuthorizedKeysFiles(credentials):
+            if not filepath.exists():
+                continue
+            try:
+                lines = filepath.open()
+            except OSError as e:
+                if e.errno == errno.EACCES:
+                    lines = runAsEffectiveUser(ouid, ogid, filepath.open)
+                else:
+                    raise
+            with lines:
+                for l in lines:
+                    l2 = l.split()
+                    if len(l2) < 2:
+                        continue
+                    try:
+                        if decodebytes(l2[1]) == credentials.blob:
+                            return True
+                    except binascii.Error:
+                        continue
+        return False
+
+    def _ebRequestAvatarId(self, f):
+        if not f.check(UnauthorizedLogin):
+            _log.error(
+                "Unauthorized login due to internal error: {error}", error=f.value
+            )
+            return failure.Failure(UnauthorizedLogin("unable to get avatar id"))
+        return f
+
+
+@implementer(ICredentialsChecker)
+class SSHProtocolChecker:
+    """
+    SSHProtocolChecker is a checker that requires multiple authentications
+    to succeed.  To add a checker, call my registerChecker method with
+    the checker and the interface.
+
+    After each successful authenticate, I call my areDone method with the
+    avatar id.  To get a list of the successful credentials for an avatar id,
+    use C{SSHProcotolChecker.successfulCredentials[avatarId]}.  If L{areDone}
+    returns True, the authentication has succeeded.
+    """
+
+    def __init__(self):
+        self.checkers = {}
+        self.successfulCredentials = {}
+
+    @property
+    def credentialInterfaces(self):
+        return list(self.checkers.keys())
+
+    def registerChecker(self, checker, *credentialInterfaces):
+        if not credentialInterfaces:
+            credentialInterfaces = checker.credentialInterfaces
+        for credentialInterface in credentialInterfaces:
+            self.checkers[credentialInterface] = checker
+
+    def requestAvatarId(self, credentials):
+        """
+        Part of the L{ICredentialsChecker} interface.  Called by a portal with
+        some credentials to check if they'll authenticate a user.  We check the
+        interfaces that the credentials provide against our list of acceptable
+        checkers.  If one of them matches, we ask that checker to verify the
+        credentials.  If they're valid, we call our L{_cbGoodAuthentication}
+        method to continue.
+
+        @param credentials: the credentials the L{Portal} wants us to verify
+        """
+        ifac = providedBy(credentials)
+        for i in ifac:
+            c = self.checkers.get(i)
+            if c is not None:
+                d = defer.maybeDeferred(c.requestAvatarId, credentials)
+                return d.addCallback(self._cbGoodAuthentication, credentials)
+        return defer.fail(
+            UnhandledCredentials(
+                "No checker for %s" % ", ".join(map(reflect.qual, ifac))
+            )
+        )
+
+    def _cbGoodAuthentication(self, avatarId, credentials):
+        """
+        Called if a checker has verified the credentials.  We call our
+        L{areDone} method to see if the whole of the successful authentications
+        are enough.  If they are, we return the avatar ID returned by the first
+        checker.
+        """
+        if avatarId not in self.successfulCredentials:
+            self.successfulCredentials[avatarId] = []
+        self.successfulCredentials[avatarId].append(credentials)
+        if self.areDone(avatarId):
+            del self.successfulCredentials[avatarId]
+            return avatarId
+        else:
+            raise error.NotEnoughAuthentication()
+
+    def areDone(self, avatarId):
+        """
+        Override to determine if the authentication is finished for a given
+        avatarId.
+
+        @param avatarId: the avatar returned by the first checker.  For
+            this checker to function correctly, all the checkers must
+            return the same avatar ID.
+        """
+        return True
+
+
+deprecatedModuleAttribute(
+    Version("Twisted", 15, 0, 0),
+    (
+        "Please use twisted.conch.checkers.SSHPublicKeyChecker, "
+        "initialized with an instance of "
+        "twisted.conch.checkers.UNIXAuthorizedKeysFiles instead."
+    ),
+    __name__,
+    "SSHPublicKeyDatabase",
+)
+
+
+class IAuthorizedKeysDB(Interface):
+    """
+    An object that provides valid authorized ssh keys mapped to usernames.
+
+    @since: 15.0
+    """
+
+    def getAuthorizedKeys(avatarId):
+        """
+        Gets an iterable of authorized keys that are valid for the given
+        C{avatarId}.
+
+        @param avatarId: the ID of the avatar
+        @type avatarId: valid return value of
+            L{twisted.cred.checkers.ICredentialsChecker.requestAvatarId}
+
+        @return: an iterable of L{twisted.conch.ssh.keys.Key}
+        """
+
+
+def readAuthorizedKeyFile(
+    fileobj: IO[bytes], parseKey: Callable[[bytes], keys.Key] = keys.Key.fromString
+) -> Iterator[keys.Key]:
+    """
+    Reads keys from an authorized keys file.  Any non-comment line that cannot
+    be parsed as a key will be ignored, although that particular line will
+    be logged.
+
+    @param fileobj: something from which to read lines which can be parsed
+        as keys
+    @param parseKey: a callable that takes bytes and returns a
+        L{twisted.conch.ssh.keys.Key}, mainly to be used for testing.  The
+        default is L{twisted.conch.ssh.keys.Key.fromString}.
+    @return: an iterable of L{twisted.conch.ssh.keys.Key}
+    @since: 15.0
+    """
+    for line in fileobj:
+        line = line.strip()
+        if line and not line.startswith(b"#"):  # for comments
+            try:
+                yield parseKey(line)
+            except keys.BadKeyError as e:
+                _log.error(
+                    "Unable to parse line {line!r} as a key: {error!s}",
+                    line=line,
+                    error=e,
+                )
+
+
+def _keysFromFilepaths(
+    filepaths: Iterable[FilePath[Any]], parseKey: Callable[[bytes], keys.Key]
+) -> Iterable[keys.Key]:
+    """
+    Helper function that turns an iterable of filepaths into a generator of
+    keys.  If any file cannot be read, a message is logged but it is
+    otherwise ignored.
+
+    @param filepaths: iterable of L{twisted.python.filepath.FilePath}.
+    @type filepaths: iterable
+
+    @param parseKey: a callable that takes a string and returns a
+        L{twisted.conch.ssh.keys.Key}
+    @type parseKey: L{callable}
+
+    @return: generator of L{twisted.conch.ssh.keys.Key}
+
+    @since: 15.0
+    """
+    for fp in filepaths:
+        if fp.exists():
+            try:
+                with fp.open() as f:
+                    yield from readAuthorizedKeyFile(f, parseKey)
+            except OSError as e:
+                _log.error("Unable to read {path!r}: {error!s}", path=fp.path, error=e)
+
+
+@implementer(IAuthorizedKeysDB)
+class InMemorySSHKeyDB:
+    """
+    Object that provides SSH public keys based on a dictionary of usernames
+    mapped to L{twisted.conch.ssh.keys.Key}s.
+
+    @since: 15.0
+    """
+
+    def __init__(self, mapping: Mapping[bytes, Iterable[keys.Key]]) -> None:
+        """
+        Initializes a new L{InMemorySSHKeyDB}.
+
+        @param mapping: mapping of usernames to iterables of
+            L{twisted.conch.ssh.keys.Key}s
+
+        """
+        self._mapping = mapping
+
+    def getAuthorizedKeys(self, username: bytes) -> Iterable[keys.Key]:
+        """
+        Look up the authorized keys for a user.
+
+        @param username: Name of the user
+        """
+        return self._mapping.get(username, [])
+
+
+@implementer(IAuthorizedKeysDB)
+class UNIXAuthorizedKeysFiles:
+    """
+    Object that provides SSH public keys based on public keys listed in
+    authorized_keys and authorized_keys2 files in UNIX user .ssh/ directories.
+    If any of the files cannot be read, a message is logged but that file is
+    otherwise ignored.
+
+    @since: 15.0
+    """
+
+    _userdb: UserDB
+
+    def __init__(
+        self,
+        userdb: Optional[UserDB] = None,
+        parseKey: Callable[[bytes], keys.Key] = keys.Key.fromString,
+    ):
+        """
+        Initializes a new L{UNIXAuthorizedKeysFiles}.
+
+        @param userdb: access to the Unix user account and password database
+            (default is the Python module L{pwd}, if available)
+
+        @param parseKey: a callable that takes a string and returns a
+            L{twisted.conch.ssh.keys.Key}, mainly to be used for testing.  The
+            default is L{twisted.conch.ssh.keys.Key.fromString}.
+        """
+        if userdb is not None:
+            self._userdb = userdb
+        elif pwd is not None:
+            self._userdb = pwd
+        else:
+            raise ValueError("No pwd module found, and no userdb argument passed.")
+        self._parseKey = parseKey
+
+    def getAuthorizedKeys(self, username: bytes) -> Iterable[keys.Key]:
+        try:
+            passwd = _lookupUser(self._userdb, username)
+        except KeyError:
+            return ()
+
+        root = FilePath(passwd.pw_dir).child(".ssh")
+        files = ["authorized_keys", "authorized_keys2"]
+        return _keysFromFilepaths((root.child(f) for f in files), self._parseKey)
+
+
+@implementer(ICredentialsChecker)
+class SSHPublicKeyChecker:
+    """
+    Checker that authenticates SSH public keys, based on public keys listed in
+    authorized_keys and authorized_keys2 files in user .ssh/ directories.
+
+    Initializing this checker with a L{UNIXAuthorizedKeysFiles} should be
+    used instead of L{twisted.conch.checkers.SSHPublicKeyDatabase}.
+
+    @since: 15.0
+    """
+
+    credentialInterfaces = (ISSHPrivateKey,)
+
+    def __init__(self, keydb: IAuthorizedKeysDB) -> None:
+        """
+        Initializes a L{SSHPublicKeyChecker}.
+
+        @param keydb: a provider of L{IAuthorizedKeysDB}
+        """
+        self._keydb = keydb
+
+    def requestAvatarId(self, credentials):
+        d = defer.execute(self._sanityCheckKey, credentials)
+        d.addCallback(self._checkKey, credentials)
+        d.addCallback(self._verifyKey, credentials)
+        return d
+
+    def _sanityCheckKey(self, credentials):
+        """
+        Checks whether the provided credentials are a valid SSH key with a
+        signature (does not actually verify the signature).
+
+        @param credentials: the credentials offered by the user
+        @type credentials: L{ISSHPrivateKey} provider
+
+        @raise ValidPublicKey: the credentials do not include a signature. See
+            L{error.ValidPublicKey} for more information.
+
+        @raise BadKeyError: The key included with the credentials is not
+            recognized as a key.
+
+        @return: the key in the credentials
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        """
+        if not credentials.signature:
+            raise error.ValidPublicKey()
+
+        return keys.Key.fromString(credentials.blob)
+
+    def _checkKey(self, pubKey, credentials):
+        """
+        Checks the public key against all authorized keys (if any) for the
+        user.
+
+        @param pubKey: the key in the credentials (just to prevent it from
+            having to be calculated again)
+        @type pubKey:
+
+        @param credentials: the credentials offered by the user
+        @type credentials: L{ISSHPrivateKey} provider
+
+        @raise UnauthorizedLogin: If the key is not authorized, or if there
+            was any error obtaining a list of authorized keys for the user.
+
+        @return: C{pubKey} if the key is authorized
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        """
+        if any(
+            key == pubKey for key in self._keydb.getAuthorizedKeys(credentials.username)
+        ):
+            return pubKey
+
+        raise UnauthorizedLogin("Key not authorized")
+
+    def _verifyKey(self, pubKey, credentials):
+        """
+        Checks whether the credentials themselves are valid, now that we know
+        if the key matches the user.
+
+        @param pubKey: the key in the credentials (just to prevent it from
+            having to be calculated again)
+        @type pubKey: L{twisted.conch.ssh.keys.Key}
+
+        @param credentials: the credentials offered by the user
+        @type credentials: L{ISSHPrivateKey} provider
+
+        @raise UnauthorizedLogin: If the key signature is invalid or there
+            was any error verifying the signature.
+
+        @return: The user's username, if authentication was successful
+        @rtype: L{bytes}
+        """
+        try:
+            if pubKey.verify(credentials.signature, credentials.sigData):
+                return credentials.username
+        except Exception as e:  # Any error should be treated as a failed login
+            raise UnauthorizedLogin("Error while verifying key") from e
+
+        raise UnauthorizedLogin("Key signature invalid.")
diff --git a/contrib/python/Twisted/py3/twisted/conch/client/__init__.py b/contrib/python/Twisted/py3/twisted/conch/client/__init__.py
new file mode 100644
index 00000000000..f55d474db4c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/client/__init__.py
@@ -0,0 +1,9 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+"""
+Client support code for Conch.
+
+Maintainer: Paul Swartz
+"""
diff --git a/contrib/python/Twisted/py3/twisted/conch/client/agent.py b/contrib/python/Twisted/py3/twisted/conch/client/agent.py
new file mode 100644
index 00000000000..7a1afca0a05
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/client/agent.py
@@ -0,0 +1,65 @@
+# -*- test-case-name: twisted.conch.test.test_default -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Accesses the key agent for user authentication.
+
+Maintainer: Paul Swartz
+"""
+
+import os
+
+from twisted.conch.ssh import agent, channel, keys
+from twisted.internet import protocol, reactor
+from twisted.logger import Logger
+
+
+class SSHAgentClient(agent.SSHAgentClient):
+    _log = Logger()
+
+    def __init__(self):
+        agent.SSHAgentClient.__init__(self)
+        self.blobs = []
+
+    def getPublicKeys(self):
+        return self.requestIdentities().addCallback(self._cbPublicKeys)
+
+    def _cbPublicKeys(self, blobcomm):
+        self._log.debug("got {num_keys} public keys", num_keys=len(blobcomm))
+        self.blobs = [x[0] for x in blobcomm]
+
+    def getPublicKey(self):
+        """
+        Return a L{Key} from the first blob in C{self.blobs}, if any, or
+        return L{None}.
+        """
+        if self.blobs:
+            return keys.Key.fromString(self.blobs.pop(0))
+        return None
+
+
+class SSHAgentForwardingChannel(channel.SSHChannel):
+    def channelOpen(self, specificData):
+        cc = protocol.ClientCreator(reactor, SSHAgentForwardingLocal)
+        d = cc.connectUNIX(os.environ["SSH_AUTH_SOCK"])
+        d.addCallback(self._cbGotLocal)
+        d.addErrback(lambda x: self.loseConnection())
+        self.buf = ""
+
+    def _cbGotLocal(self, local):
+        self.local = local
+        self.dataReceived = self.local.transport.write
+        self.local.dataReceived = self.write
+
+    def dataReceived(self, data):
+        self.buf += data
+
+    def closed(self):
+        if self.local:
+            self.local.loseConnection()
+            self.local = None
+
+
+class SSHAgentForwardingLocal(protocol.Protocol):
+    pass
diff --git a/contrib/python/Twisted/py3/twisted/conch/client/connect.py b/contrib/python/Twisted/py3/twisted/conch/client/connect.py
new file mode 100644
index 00000000000..f21f16768bb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/client/connect.py
@@ -0,0 +1,24 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+from twisted.conch.client import direct
+
+connectTypes = {"direct": direct.connect}
+
+
+def connect(host, port, options, verifyHostKey, userAuthObject):
+    useConnects = ["direct"]
+    return _ebConnect(
+        None, useConnects, host, port, options, verifyHostKey, userAuthObject
+    )
+
+
+def _ebConnect(f, useConnects, host, port, options, vhk, uao):
+    if not useConnects:
+        return f
+    connectType = useConnects.pop(0)
+    f = connectTypes[connectType]
+    d = f(host, port, options, vhk, uao)
+    d.addErrback(_ebConnect, useConnects, host, port, options, vhk, uao)
+    return d
diff --git a/contrib/python/Twisted/py3/twisted/conch/client/default.py b/contrib/python/Twisted/py3/twisted/conch/client/default.py
new file mode 100644
index 00000000000..daf4cf33719
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/client/default.py
@@ -0,0 +1,331 @@
+# -*- test-case-name: twisted.conch.test.test_knownhosts,twisted.conch.test.test_default -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Various classes and functions for implementing user-interaction in the
+command-line conch client.
+
+You probably shouldn't use anything in this module directly, since it assumes
+you are sitting at an interactive terminal.  For example, to programmatically
+interact with a known_hosts database, use L{twisted.conch.client.knownhosts}.
+"""
+
+import contextlib
+import getpass
+import io
+import os
+import sys
+from base64 import decodebytes
+
+from twisted.conch.client import agent
+from twisted.conch.client.knownhosts import ConsoleUI, KnownHostsFile
+from twisted.conch.error import ConchError
+from twisted.conch.ssh import common, keys, userauth
+from twisted.internet import defer, protocol, reactor
+from twisted.python.compat import nativeString
+from twisted.python.filepath import FilePath
+
+# The default location of the known hosts file (probably should be parsed out
+# of an ssh config file someday).
+_KNOWN_HOSTS = "~/.ssh/known_hosts"
+
+
+# This name is bound so that the unit tests can use 'patch' to override it.
+_open = open
+_input = input
+
+
+def verifyHostKey(transport, host, pubKey, fingerprint):
+    """
+    Verify a host's key.
+
+    This function is a gross vestige of some bad factoring in the client
+    internals.  The actual implementation, and a better signature of this logic
+    is in L{KnownHostsFile.verifyHostKey}.  This function is not deprecated yet
+    because the callers have not yet been rehabilitated, but they should
+    eventually be changed to call that method instead.
+
+    However, this function does perform two functions not implemented by
+    L{KnownHostsFile.verifyHostKey}.  It determines the path to the user's
+    known_hosts file based on the options (which should really be the options
+    object's job), and it provides an opener to L{ConsoleUI} which opens
+    '/dev/tty' so that the user will be prompted on the tty of the process even
+    if the input and output of the process has been redirected.  This latter
+    part is, somewhat obviously, not portable, but I don't know of a portable
+    equivalent that could be used.
+
+    @param host: Due to a bug in L{SSHClientTransport.verifyHostKey}, this is
+    always the dotted-quad IP address of the host being connected to.
+    @type host: L{str}
+
+    @param transport: the client transport which is attempting to connect to
+    the given host.
+    @type transport: L{SSHClientTransport}
+
+    @param fingerprint: the fingerprint of the given public key, in
+    xx:xx:xx:... format.  This is ignored in favor of getting the fingerprint
+    from the key itself.
+    @type fingerprint: L{str}
+
+    @param pubKey: The public key of the server being connected to.
+    @type pubKey: L{str}
+
+    @return: a L{Deferred} which fires with C{1} if the key was successfully
+    verified, or fails if the key could not be successfully verified.  Failure
+    types may include L{HostKeyChanged}, L{UserRejectedKey}, L{IOError} or
+    L{KeyboardInterrupt}.
+    """
+    actualHost = transport.factory.options["host"]
+    actualKey = keys.Key.fromString(pubKey)
+    kh = KnownHostsFile.fromPath(
+        FilePath(
+            transport.factory.options["known-hosts"] or os.path.expanduser(_KNOWN_HOSTS)
+        )
+    )
+    ui = ConsoleUI(lambda: _open("/dev/tty", "r+b", buffering=0))
+    return kh.verifyHostKey(ui, actualHost, host, actualKey)
+
+
+def isInKnownHosts(host, pubKey, options):
+    """
+    Checks to see if host is in the known_hosts file for the user.
+
+    @return: 0 if it isn't, 1 if it is and is the same, 2 if it's changed.
+    @rtype: L{int}
+    """
+    keyType = common.getNS(pubKey)[0]
+    retVal = 0
+
+    if not options["known-hosts"] and not os.path.exists(os.path.expanduser("~/.ssh/")):
+        print("Creating ~/.ssh directory...")
+        os.mkdir(os.path.expanduser("~/.ssh"))
+    kh_file = options["known-hosts"] or _KNOWN_HOSTS
+    try:
+        known_hosts = open(os.path.expanduser(kh_file), "rb")
+    except OSError:
+        return 0
+    with known_hosts:
+        for line in known_hosts.readlines():
+            split = line.split()
+            if len(split) < 3:
+                continue
+            hosts, hostKeyType, encodedKey = split[:3]
+            if host not in hosts.split(b","):  # incorrect host
+                continue
+            if hostKeyType != keyType:  # incorrect type of key
+                continue
+            try:
+                decodedKey = decodebytes(encodedKey)
+            except BaseException:
+                continue
+            if decodedKey == pubKey:
+                return 1
+            else:
+                retVal = 2
+    return retVal
+
+
+def getHostKeyAlgorithms(host, options):
+    """
+    Look in known_hosts for a key corresponding to C{host}.
+    This can be used to change the order of supported key types
+    in the KEXINIT packet.
+
+    @type host: L{str}
+    @param host: the host to check in known_hosts
+    @type options: L{twisted.conch.client.options.ConchOptions}
+    @param options: options passed to client
+    @return: L{list} of L{str} representing key types or L{None}.
+    """
+    knownHosts = KnownHostsFile.fromPath(
+        FilePath(options["known-hosts"] or os.path.expanduser(_KNOWN_HOSTS))
+    )
+    keyTypes = []
+    for entry in knownHosts.iterentries():
+        if entry.matchesHost(host):
+            if entry.keyType not in keyTypes:
+                keyTypes.append(entry.keyType)
+    return keyTypes or None
+
+
+class SSHUserAuthClient(userauth.SSHUserAuthClient):
+    def __init__(self, user, options, *args):
+        userauth.SSHUserAuthClient.__init__(self, user, *args)
+        self.keyAgent = None
+        self.options = options
+        self.usedFiles = []
+        if not options.identitys:
+            options.identitys = ["~/.ssh/id_rsa", "~/.ssh/id_dsa"]
+
+    def serviceStarted(self):
+        if "SSH_AUTH_SOCK" in os.environ and not self.options["noagent"]:
+            self._log.debug(
+                "using SSH agent {authSock!r}", authSock=os.environ["SSH_AUTH_SOCK"]
+            )
+            cc = protocol.ClientCreator(reactor, agent.SSHAgentClient)
+            d = cc.connectUNIX(os.environ["SSH_AUTH_SOCK"])
+            d.addCallback(self._setAgent)
+            d.addErrback(self._ebSetAgent)
+        else:
+            userauth.SSHUserAuthClient.serviceStarted(self)
+
+    def serviceStopped(self):
+        if self.keyAgent:
+            self.keyAgent.transport.loseConnection()
+            self.keyAgent = None
+
+    def _setAgent(self, a):
+        self.keyAgent = a
+        d = self.keyAgent.getPublicKeys()
+        d.addBoth(self._ebSetAgent)
+        return d
+
+    def _ebSetAgent(self, f):
+        userauth.SSHUserAuthClient.serviceStarted(self)
+
+    def _getPassword(self, prompt):
+        """
+        Prompt for a password using L{getpass.getpass}.
+
+        @param prompt: Written on tty to ask for the input.
+        @type prompt: L{str}
+        @return: The input.
+        @rtype: L{str}
+        """
+        with self._replaceStdoutStdin():
+            try:
+                p = getpass.getpass(prompt)
+                return p
+            except (KeyboardInterrupt, OSError):
+                print()
+                raise ConchError("PEBKAC")
+
+    def getPassword(self, prompt=None):
+        if prompt:
+            prompt = nativeString(prompt)
+        else:
+            prompt = "{}@{}'s password: ".format(
+                nativeString(self.user),
+                self.transport.transport.getPeer().host,
+            )
+        try:
+            # We don't know the encoding the other side is using,
+            # signaling that is not part of the SSH protocol. But
+            # using our defaultencoding is better than just going for
+            # ASCII.
+            p = self._getPassword(prompt).encode(sys.getdefaultencoding())
+            return defer.succeed(p)
+        except ConchError:
+            return defer.fail()
+
+    def getPublicKey(self):
+        """
+        Get a public key from the key agent if possible, otherwise look in
+        the next configured identity file for one.
+        """
+        if self.keyAgent:
+            key = self.keyAgent.getPublicKey()
+            if key is not None:
+                return key
+        files = [x for x in self.options.identitys if x not in self.usedFiles]
+        self._log.debug(
+            "public key identities: {identities}\n{files}",
+            identities=self.options.identitys,
+            files=files,
+        )
+        if not files:
+            return None
+        file = files[0]
+        self.usedFiles.append(file)
+        file = os.path.expanduser(file)
+        file += ".pub"
+        if not os.path.exists(file):
+            return self.getPublicKey()  # try again
+        try:
+            return keys.Key.fromFile(file)
+        except keys.BadKeyError:
+            return self.getPublicKey()  # try again
+
+    def signData(self, publicKey, signData):
+        """
+        Extend the base signing behavior by using an SSH agent to sign the
+        data, if one is available.
+
+        @type publicKey: L{Key}
+        @type signData: L{bytes}
+        """
+        if not self.usedFiles:  # agent key
+            return self.keyAgent.signData(publicKey.blob(), signData)
+        else:
+            return userauth.SSHUserAuthClient.signData(self, publicKey, signData)
+
+    def getPrivateKey(self):
+        """
+        Try to load the private key from the last used file identified by
+        C{getPublicKey}, potentially asking for the passphrase if the key is
+        encrypted.
+        """
+        file = os.path.expanduser(self.usedFiles[-1])
+        if not os.path.exists(file):
+            return None
+        try:
+            return defer.succeed(keys.Key.fromFile(file))
+        except keys.EncryptedKeyError:
+            for i in range(3):
+                prompt = "Enter passphrase for key '%s': " % self.usedFiles[-1]
+                try:
+                    p = self._getPassword(prompt).encode(sys.getfilesystemencoding())
+                    return defer.succeed(keys.Key.fromFile(file, passphrase=p))
+                except (keys.BadKeyError, ConchError):
+                    pass
+                return defer.fail(ConchError("bad password"))
+            raise
+        except KeyboardInterrupt:
+            print()
+            reactor.stop()
+
+    def getGenericAnswers(self, name, instruction, prompts):
+        responses = []
+        with self._replaceStdoutStdin():
+            if name:
+                print(name.decode("utf-8"))
+            if instruction:
+                print(instruction.decode("utf-8"))
+            for prompt, echo in prompts:
+                prompt = prompt.decode("utf-8")
+                if echo:
+                    responses.append(_input(prompt))
+                else:
+                    responses.append(getpass.getpass(prompt))
+        return defer.succeed(responses)
+
+    @classmethod
+    def _openTty(cls):
+        """
+        Open /dev/tty as two streams one in read, one in write mode,
+        and return them.
+
+        @return: File objects for reading and writing to /dev/tty,
+                 corresponding to standard input and standard output.
+        @rtype: A L{tuple} of L{io.TextIOWrapper} on Python 3.
+        """
+        stdin = io.TextIOWrapper(open("/dev/tty", "rb"))
+        stdout = io.TextIOWrapper(open("/dev/tty", "wb"))
+        return stdin, stdout
+
+    @classmethod
+    @contextlib.contextmanager
+    def _replaceStdoutStdin(cls):
+        """
+        Contextmanager that replaces stdout and stdin with /dev/tty
+        and resets them when it is done.
+        """
+        oldout, oldin = sys.stdout, sys.stdin
+        sys.stdin, sys.stdout = cls._openTty()
+        try:
+            yield
+        finally:
+            sys.stdout.close()
+            sys.stdin.close()
+            sys.stdout, sys.stdin = oldout, oldin
diff --git a/contrib/python/Twisted/py3/twisted/conch/client/direct.py b/contrib/python/Twisted/py3/twisted/conch/client/direct.py
new file mode 100644
index 00000000000..d9f4828ec5f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/client/direct.py
@@ -0,0 +1,98 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+from twisted.conch import error
+from twisted.conch.ssh import transport
+from twisted.internet import defer, protocol, reactor
+
+
+class SSHClientFactory(protocol.ClientFactory):
+    def __init__(self, d, options, verifyHostKey, userAuthObject):
+        self.d = d
+        self.options = options
+        self.verifyHostKey = verifyHostKey
+        self.userAuthObject = userAuthObject
+
+    def clientConnectionLost(self, connector, reason):
+        if self.options["reconnect"]:
+            connector.connect()
+
+    def clientConnectionFailed(self, connector, reason):
+        if self.d is None:
+            return
+        d, self.d = self.d, None
+        d.errback(reason)
+
+    def buildProtocol(self, addr):
+        trans = SSHClientTransport(self)
+        if self.options["ciphers"]:
+            trans.supportedCiphers = self.options["ciphers"]
+        if self.options["macs"]:
+            trans.supportedMACs = self.options["macs"]
+        if self.options["compress"]:
+            trans.supportedCompressions[0:1] = ["zlib"]
+        if self.options["host-key-algorithms"]:
+            trans.supportedPublicKeys = self.options["host-key-algorithms"]
+        return trans
+
+
+class SSHClientTransport(transport.SSHClientTransport):
+    def __init__(self, factory):
+        self.factory = factory
+        self.unixServer = None
+
+    def connectionLost(self, reason):
+        if self.unixServer:
+            d = self.unixServer.stopListening()
+            self.unixServer = None
+        else:
+            d = defer.succeed(None)
+        d.addCallback(
+            lambda x: transport.SSHClientTransport.connectionLost(self, reason)
+        )
+
+    def receiveError(self, code, desc):
+        if self.factory.d is None:
+            return
+        d, self.factory.d = self.factory.d, None
+        d.errback(error.ConchError(desc, code))
+
+    def sendDisconnect(self, code, reason):
+        if self.factory.d is None:
+            return
+        d, self.factory.d = self.factory.d, None
+        transport.SSHClientTransport.sendDisconnect(self, code, reason)
+        d.errback(error.ConchError(reason, code))
+
+    def receiveDebug(self, alwaysDisplay, message, lang):
+        self._log.debug(
+            "Received Debug Message: {message}",
+            message=message,
+            alwaysDisplay=alwaysDisplay,
+            lang=lang,
+        )
+        if alwaysDisplay:  # XXX what should happen here?
+            print(message)
+
+    def verifyHostKey(self, pubKey, fingerprint):
+        return self.factory.verifyHostKey(
+            self, self.transport.getPeer().host, pubKey, fingerprint
+        )
+
+    def setService(self, service):
+        self._log.info("setting client server to {service}", service=service)
+        transport.SSHClientTransport.setService(self, service)
+        if service.name != "ssh-userauth" and self.factory.d is not None:
+            d, self.factory.d = self.factory.d, None
+            d.callback(None)
+
+    def connectionSecure(self):
+        self.requestService(self.factory.userAuthObject)
+
+
+def connect(host, port, options, verifyHostKey, userAuthObject):
+    d = defer.Deferred()
+    factory = SSHClientFactory(d, options, verifyHostKey, userAuthObject)
+    reactor.connectTCP(host, port, factory)
+    return d
diff --git a/contrib/python/Twisted/py3/twisted/conch/client/knownhosts.py b/contrib/python/Twisted/py3/twisted/conch/client/knownhosts.py
new file mode 100644
index 00000000000..39bf10ba79f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/client/knownhosts.py
@@ -0,0 +1,620 @@
+# -*- test-case-name: twisted.conch.test.test_knownhosts -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An implementation of the OpenSSH known_hosts database.
+
+@since: 8.2
+"""
+
+
+import hmac
+import sys
+from binascii import Error as DecodeError, a2b_base64, b2a_base64
+from contextlib import closing
+from hashlib import sha1
+
+from zope.interface import implementer
+
+from twisted.conch.error import HostKeyChanged, InvalidEntry, UserRejectedKey
+from twisted.conch.interfaces import IKnownHostEntry
+from twisted.conch.ssh.keys import BadKeyError, FingerprintFormats, Key
+from twisted.internet import defer
+from twisted.logger import Logger
+from twisted.python.compat import nativeString
+from twisted.python.randbytes import secureRandom
+from twisted.python.util import FancyEqMixin
+
+log = Logger()
+
+
+def _b64encode(s):
+    """
+    Encode a binary string as base64 with no trailing newline.
+
+    @param s: The string to encode.
+    @type s: L{bytes}
+
+    @return: The base64-encoded string.
+    @rtype: L{bytes}
+    """
+    return b2a_base64(s).strip()
+
+
+def _extractCommon(string):
+    """
+    Extract common elements of base64 keys from an entry in a hosts file.
+
+    @param string: A known hosts file entry (a single line).
+    @type string: L{bytes}
+
+    @return: a 4-tuple of hostname data (L{bytes}), ssh key type (L{bytes}), key
+        (L{Key}), and comment (L{bytes} or L{None}).  The hostname data is
+        simply the beginning of the line up to the first occurrence of
+        whitespace.
+    @rtype: L{tuple}
+    """
+    elements = string.split(None, 2)
+    if len(elements) != 3:
+        raise InvalidEntry()
+    hostnames, keyType, keyAndComment = elements
+    splitkey = keyAndComment.split(None, 1)
+    if len(splitkey) == 2:
+        keyString, comment = splitkey
+        comment = comment.rstrip(b"\n")
+    else:
+        keyString = splitkey[0]
+        comment = None
+    key = Key.fromString(a2b_base64(keyString))
+    return hostnames, keyType, key, comment
+
+
+class _BaseEntry:
+    """
+    Abstract base of both hashed and non-hashed entry objects, since they
+    represent keys and key types the same way.
+
+    @ivar keyType: The type of the key; either ssh-dss or ssh-rsa.
+    @type keyType: L{bytes}
+
+    @ivar publicKey: The server public key indicated by this line.
+    @type publicKey: L{twisted.conch.ssh.keys.Key}
+
+    @ivar comment: Trailing garbage after the key line.
+    @type comment: L{bytes}
+    """
+
+    def __init__(self, keyType, publicKey, comment):
+        self.keyType = keyType
+        self.publicKey = publicKey
+        self.comment = comment
+
+    def matchesKey(self, keyObject):
+        """
+        Check to see if this entry matches a given key object.
+
+        @param keyObject: A public key object to check.
+        @type keyObject: L{Key}
+
+        @return: C{True} if this entry's key matches C{keyObject}, C{False}
+            otherwise.
+        @rtype: L{bool}
+        """
+        return self.publicKey == keyObject
+
+
+@implementer(IKnownHostEntry)
+class PlainEntry(_BaseEntry):
+    """
+    A L{PlainEntry} is a representation of a plain-text entry in a known_hosts
+    file.
+
+    @ivar _hostnames: the list of all host-names associated with this entry.
+    @type _hostnames: L{list} of L{bytes}
+    """
+
+    def __init__(self, hostnames, keyType, publicKey, comment):
+        self._hostnames = hostnames
+        super().__init__(keyType, publicKey, comment)
+
+    @classmethod
+    def fromString(cls, string):
+        """
+        Parse a plain-text entry in a known_hosts file, and return a
+        corresponding L{PlainEntry}.
+
+        @param string: a space-separated string formatted like "hostname
+        key-type base64-key-data comment".
+
+        @type string: L{bytes}
+
+        @raise DecodeError: if the key is not valid encoded as valid base64.
+
+        @raise InvalidEntry: if the entry does not have the right number of
+        elements and is therefore invalid.
+
+        @raise BadKeyError: if the key, once decoded from base64, is not
+        actually an SSH key.
+
+        @return: an IKnownHostEntry representing the hostname and key in the
+        input line.
+
+        @rtype: L{PlainEntry}
+        """
+        hostnames, keyType, key, comment = _extractCommon(string)
+        self = cls(hostnames.split(b","), keyType, key, comment)
+        return self
+
+    def matchesHost(self, hostname):
+        """
+        Check to see if this entry matches a given hostname.
+
+        @param hostname: A hostname or IP address literal to check against this
+            entry.
+        @type hostname: L{bytes}
+
+        @return: C{True} if this entry is for the given hostname or IP address,
+            C{False} otherwise.
+        @rtype: L{bool}
+        """
+        if isinstance(hostname, str):
+            hostname = hostname.encode("utf-8")
+        return hostname in self._hostnames
+
+    def toString(self):
+        """
+        Implement L{IKnownHostEntry.toString} by recording the comma-separated
+        hostnames, key type, and base-64 encoded key.
+
+        @return: The string representation of this entry, with unhashed hostname
+            information.
+        @rtype: L{bytes}
+        """
+        fields = [
+            b",".join(self._hostnames),
+            self.keyType,
+            _b64encode(self.publicKey.blob()),
+        ]
+        if self.comment is not None:
+            fields.append(self.comment)
+        return b" ".join(fields)
+
+
+@implementer(IKnownHostEntry)
+class UnparsedEntry:
+    """
+    L{UnparsedEntry} is an entry in a L{KnownHostsFile} which can't actually be
+    parsed; therefore it matches no keys and no hosts.
+    """
+
+    def __init__(self, string):
+        """
+        Create an unparsed entry from a line in a known_hosts file which cannot
+        otherwise be parsed.
+        """
+        self._string = string
+
+    def matchesHost(self, hostname):
+        """
+        Always returns False.
+        """
+        return False
+
+    def matchesKey(self, key):
+        """
+        Always returns False.
+        """
+        return False
+
+    def toString(self):
+        """
+        Returns the input line, without its newline if one was given.
+
+        @return: The string representation of this entry, almost exactly as was
+            used to initialize this entry but without a trailing newline.
+        @rtype: L{bytes}
+        """
+        return self._string.rstrip(b"\n")
+
+
+def _hmacedString(key, string):
+    """
+    Return the SHA-1 HMAC hash of the given key and string.
+
+    @param key: The HMAC key.
+    @type key: L{bytes}
+
+    @param string: The string to be hashed.
+    @type string: L{bytes}
+
+    @return: The keyed hash value.
+    @rtype: L{bytes}
+    """
+    hash = hmac.HMAC(key, digestmod=sha1)
+    if isinstance(string, str):
+        string = string.encode("utf-8")
+    hash.update(string)
+    return hash.digest()
+
+
+@implementer(IKnownHostEntry)
+class HashedEntry(_BaseEntry, FancyEqMixin):
+    """
+    A L{HashedEntry} is a representation of an entry in a known_hosts file
+    where the hostname has been hashed and salted.
+
+    @ivar _hostSalt: the salt to combine with a hostname for hashing.
+
+    @ivar _hostHash: the hashed representation of the hostname.
+
+    @cvar MAGIC: the 'hash magic' string used to identify a hashed line in a
+    known_hosts file as opposed to a plaintext one.
+    """
+
+    MAGIC = b"|1|"
+
+    compareAttributes = ("_hostSalt", "_hostHash", "keyType", "publicKey", "comment")
+
+    def __init__(self, hostSalt, hostHash, keyType, publicKey, comment):
+        self._hostSalt = hostSalt
+        self._hostHash = hostHash
+        super().__init__(keyType, publicKey, comment)
+
+    @classmethod
+    def fromString(cls, string):
+        """
+        Load a hashed entry from a string representing a line in a known_hosts
+        file.
+
+        @param string: A complete single line from a I{known_hosts} file,
+            formatted as defined by OpenSSH.
+        @type string: L{bytes}
+
+        @raise DecodeError: if the key, the hostname, or the is not valid
+            encoded as valid base64
+
+        @raise InvalidEntry: if the entry does not have the right number of
+            elements and is therefore invalid, or the host/hash portion contains
+            more items than just the host and hash.
+
+        @raise BadKeyError: if the key, once decoded from base64, is not
+            actually an SSH key.
+
+        @return: The newly created L{HashedEntry} instance, initialized with the
+            information from C{string}.
+        """
+        stuff, keyType, key, comment = _extractCommon(string)
+        saltAndHash = stuff[len(cls.MAGIC) :].split(b"|")
+        if len(saltAndHash) != 2:
+            raise InvalidEntry()
+        hostSalt, hostHash = saltAndHash
+        self = cls(a2b_base64(hostSalt), a2b_base64(hostHash), keyType, key, comment)
+        return self
+
+    def matchesHost(self, hostname):
+        """
+        Implement L{IKnownHostEntry.matchesHost} to compare the hash of the
+        input to the stored hash.
+
+        @param hostname: A hostname or IP address literal to check against this
+            entry.
+        @type hostname: L{bytes}
+
+        @return: C{True} if this entry is for the given hostname or IP address,
+            C{False} otherwise.
+        @rtype: L{bool}
+        """
+        return hmac.compare_digest(
+            _hmacedString(self._hostSalt, hostname), self._hostHash
+        )
+
+    def toString(self):
+        """
+        Implement L{IKnownHostEntry.toString} by base64-encoding the salt, host
+        hash, and key.
+
+        @return: The string representation of this entry, with the hostname part
+            hashed.
+        @rtype: L{bytes}
+        """
+        fields = [
+            self.MAGIC
+            + b"|".join([_b64encode(self._hostSalt), _b64encode(self._hostHash)]),
+            self.keyType,
+            _b64encode(self.publicKey.blob()),
+        ]
+        if self.comment is not None:
+            fields.append(self.comment)
+        return b" ".join(fields)
+
+
+class KnownHostsFile:
+    """
+    A structured representation of an OpenSSH-format ~/.ssh/known_hosts file.
+
+    @ivar _added: A list of L{IKnownHostEntry} providers which have been added
+        to this instance in memory but not yet saved.
+
+    @ivar _clobber: A flag indicating whether the current contents of the save
+        path will be disregarded and potentially overwritten or not.  If
+        C{True}, this will be done.  If C{False}, entries in the save path will
+        be read and new entries will be saved by appending rather than
+        overwriting.
+    @type _clobber: L{bool}
+
+    @ivar _savePath: See C{savePath} parameter of L{__init__}.
+    """
+
+    def __init__(self, savePath):
+        """
+        Create a new, empty KnownHostsFile.
+
+        Unless you want to erase the current contents of C{savePath}, you want
+        to use L{KnownHostsFile.fromPath} instead.
+
+        @param savePath: The L{FilePath} to which to save new entries.
+        @type savePath: L{FilePath}
+        """
+        self._added = []
+        self._savePath = savePath
+        self._clobber = True
+
+    @property
+    def savePath(self):
+        """
+        @see: C{savePath} parameter of L{__init__}
+        """
+        return self._savePath
+
+    def iterentries(self):
+        """
+        Iterate over the host entries in this file.
+
+        @return: An iterable the elements of which provide L{IKnownHostEntry}.
+            There is an element for each entry in the file as well as an element
+            for each added but not yet saved entry.
+        @rtype: iterable of L{IKnownHostEntry} providers
+        """
+        for entry in self._added:
+            yield entry
+
+        if self._clobber:
+            return
+
+        try:
+            fp = self._savePath.open()
+        except OSError:
+            return
+
+        with fp:
+            for line in fp:
+                try:
+                    if line.startswith(HashedEntry.MAGIC):
+                        entry = HashedEntry.fromString(line)
+                    else:
+                        entry = PlainEntry.fromString(line)
+                except (DecodeError, InvalidEntry, BadKeyError):
+                    entry = UnparsedEntry(line)
+                yield entry
+
+    def hasHostKey(self, hostname, key):
+        """
+        Check for an entry with matching hostname and key.
+
+        @param hostname: A hostname or IP address literal to check for.
+        @type hostname: L{bytes}
+
+        @param key: The public key to check for.
+        @type key: L{Key}
+
+        @return: C{True} if the given hostname and key are present in this file,
+            C{False} if they are not.
+        @rtype: L{bool}
+
+        @raise HostKeyChanged: if the host key found for the given hostname
+            does not match the given key.
+        """
+        for lineidx, entry in enumerate(self.iterentries(), -len(self._added)):
+            if entry.matchesHost(hostname) and entry.keyType == key.sshType():
+                if entry.matchesKey(key):
+                    return True
+                else:
+                    # Notice that lineidx is 0-based but HostKeyChanged.lineno
+                    # is 1-based.
+                    if lineidx < 0:
+                        line = None
+                        path = None
+                    else:
+                        line = lineidx + 1
+                        path = self._savePath
+                    raise HostKeyChanged(entry, path, line)
+        return False
+
+    def verifyHostKey(self, ui, hostname, ip, key):
+        """
+        Verify the given host key for the given IP and host, asking for
+        confirmation from, and notifying, the given UI about changes to this
+        file.
+
+        @param ui: The user interface to request an IP address from.
+
+        @param hostname: The hostname that the user requested to connect to.
+
+        @param ip: The string representation of the IP address that is actually
+        being connected to.
+
+        @param key: The public key of the server.
+
+        @return: a L{Deferred} that fires with True when the key has been
+            verified, or fires with an errback when the key either cannot be
+            verified or has changed.
+        @rtype: L{Deferred}
+        """
+        hhk = defer.execute(self.hasHostKey, hostname, key)
+
+        def gotHasKey(result):
+            if result:
+                if not self.hasHostKey(ip, key):
+                    ui.warn(
+                        "Warning: Permanently added the %s host key for "
+                        "IP address '%s' to the list of known hosts."
+                        % (key.type(), nativeString(ip))
+                    )
+                    self.addHostKey(ip, key)
+                    self.save()
+                return result
+            else:
+
+                def promptResponse(response):
+                    if response:
+                        self.addHostKey(hostname, key)
+                        self.addHostKey(ip, key)
+                        self.save()
+                        return response
+                    else:
+                        raise UserRejectedKey()
+
+                keytype = key.type()
+
+                if keytype == "EC":
+                    keytype = "ECDSA"
+
+                prompt = (
+                    "The authenticity of host '%s (%s)' "
+                    "can't be established.\n"
+                    "%s key fingerprint is SHA256:%s.\n"
+                    "Are you sure you want to continue connecting (yes/no)? "
+                    % (
+                        nativeString(hostname),
+                        nativeString(ip),
+                        keytype,
+                        key.fingerprint(format=FingerprintFormats.SHA256_BASE64),
+                    )
+                )
+                proceed = ui.prompt(prompt.encode(sys.getdefaultencoding()))
+                return proceed.addCallback(promptResponse)
+
+        return hhk.addCallback(gotHasKey)
+
+    def addHostKey(self, hostname, key):
+        """
+        Add a new L{HashedEntry} to the key database.
+
+        Note that you still need to call L{KnownHostsFile.save} if you wish
+        these changes to be persisted.
+
+        @param hostname: A hostname or IP address literal to associate with the
+            new entry.
+        @type hostname: L{bytes}
+
+        @param key: The public key to associate with the new entry.
+        @type key: L{Key}
+
+        @return: The L{HashedEntry} that was added.
+        @rtype: L{HashedEntry}
+        """
+        salt = secureRandom(20)
+        keyType = key.sshType()
+        entry = HashedEntry(salt, _hmacedString(salt, hostname), keyType, key, None)
+        self._added.append(entry)
+        return entry
+
+    def save(self):
+        """
+        Save this L{KnownHostsFile} to the path it was loaded from.
+        """
+        p = self._savePath.parent()
+        if not p.isdir():
+            p.makedirs()
+
+        if self._clobber:
+            mode = "wb"
+        else:
+            mode = "ab"
+
+        with self._savePath.open(mode) as hostsFileObj:
+            if self._added:
+                hostsFileObj.write(
+                    b"\n".join([entry.toString() for entry in self._added]) + b"\n"
+                )
+                self._added = []
+        self._clobber = False
+
+    @classmethod
+    def fromPath(cls, path):
+        """
+        Create a new L{KnownHostsFile}, potentially reading existing known
+        hosts information from the given file.
+
+        @param path: A path object to use for both reading contents from and
+            later saving to.  If no file exists at this path, it is not an
+            error; a L{KnownHostsFile} with no entries is returned.
+        @type path: L{FilePath}
+
+        @return: A L{KnownHostsFile} initialized with entries from C{path}.
+        @rtype: L{KnownHostsFile}
+        """
+        knownHosts = cls(path)
+        knownHosts._clobber = False
+        return knownHosts
+
+
+class ConsoleUI:
+    """
+    A UI object that can ask true/false questions and post notifications on the
+    console, to be used during key verification.
+    """
+
+    def __init__(self, opener):
+        """
+        @param opener: A no-argument callable which should open a console
+            binary-mode file-like object to be used for reading and writing.
+            This initializes the C{opener} attribute.
+        @type opener: callable taking no arguments and returning a read/write
+            file-like object
+        """
+        self.opener = opener
+
+    def prompt(self, text):
+        """
+        Write the given text as a prompt to the console output, then read a
+        result from the console input.
+
+        @param text: Something to present to a user to solicit a yes or no
+            response.
+        @type text: L{bytes}
+
+        @return: a L{Deferred} which fires with L{True} when the user answers
+            'yes' and L{False} when the user answers 'no'.  It may errback if
+            there were any I/O errors.
+        """
+        d = defer.succeed(None)
+
+        def body(ignored):
+            with closing(self.opener()) as f:
+                f.write(text)
+                while True:
+                    answer = f.readline().strip().lower()
+                    if answer == b"yes":
+                        return True
+                    elif answer == b"no":
+                        return False
+                    else:
+                        f.write(b"Please type 'yes' or 'no': ")
+
+        return d.addCallback(body)
+
+    def warn(self, text):
+        """
+        Notify the user (non-interactively) of the provided text, by writing it
+        to the console.
+
+        @param text: Some information the user is to be made aware of.
+        @type text: L{bytes}
+        """
+        try:
+            with closing(self.opener()) as f:
+                f.write(text)
+        except Exception:
+            log.failure("Failed to write to console")
diff --git a/contrib/python/Twisted/py3/twisted/conch/client/options.py b/contrib/python/Twisted/py3/twisted/conch/client/options.py
new file mode 100644
index 00000000000..2ab2455099a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/client/options.py
@@ -0,0 +1,109 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+import sys
+from typing import List, Optional, Union
+
+#
+from twisted.conch.ssh.transport import SSHCiphers, SSHClientTransport
+from twisted.python import usage
+
+
+class ConchOptions(usage.Options):
+    optParameters: List[List[Optional[Union[str, int]]]] = [
+        ["user", "l", None, "Log in using this user name."],
+        ["identity", "i", None],
+        ["ciphers", "c", None],
+        ["macs", "m", None],
+        ["port", "p", None, "Connect to this port.  Server must be on the same port."],
+        ["option", "o", None, "Ignored OpenSSH options"],
+        ["host-key-algorithms", "", None],
+        ["known-hosts", "", None, "File to check for host keys"],
+        ["user-authentications", "", None, "Types of user authentications to use."],
+        ["logfile", "", None, "File to log to, or - for stdout"],
+    ]
+
+    optFlags = [
+        ["version", "V", "Display version number only."],
+        ["compress", "C", "Enable compression."],
+        ["log", "v", "Enable logging (defaults to stderr)"],
+        ["nox11", "x", "Disable X11 connection forwarding (default)"],
+        ["agent", "A", "Enable authentication agent forwarding"],
+        ["noagent", "a", "Disable authentication agent forwarding (default)"],
+        ["reconnect", "r", "Reconnect to the server if the connection is lost."],
+    ]
+
+    compData = usage.Completions(
+        mutuallyExclusive=[("agent", "noagent")],
+        optActions={
+            "user": usage.CompleteUsernames(),
+            "ciphers": usage.CompleteMultiList(
+                [v.decode() for v in SSHCiphers.cipherMap.keys()],
+                descr="ciphers to choose from",
+            ),
+            "macs": usage.CompleteMultiList(
+                [v.decode() for v in SSHCiphers.macMap.keys()],
+                descr="macs to choose from",
+            ),
+            "host-key-algorithms": usage.CompleteMultiList(
+                [v.decode() for v in SSHClientTransport.supportedPublicKeys],
+                descr="host key algorithms to choose from",
+            ),
+            # "user-authentications": usage.CompleteMultiList(?
+            # descr='user authentication types' ),
+        },
+        extraActions=[
+            usage.CompleteUserAtHost(),
+            usage.Completer(descr="command"),
+            usage.Completer(descr="argument", repeat=True),
+        ],
+    )
+
+    def __init__(self, *args, **kw):
+        usage.Options.__init__(self, *args, **kw)
+        self.identitys = []
+        self.conns = None
+
+    def opt_identity(self, i):
+        """Identity for public-key authentication"""
+        self.identitys.append(i)
+
+    def opt_ciphers(self, ciphers):
+        "Select encryption algorithms"
+        ciphers = ciphers.split(",")
+        for cipher in ciphers:
+            if cipher not in SSHCiphers.cipherMap:
+                sys.exit("Unknown cipher type '%s'" % cipher)
+        self["ciphers"] = ciphers
+
+    def opt_macs(self, macs):
+        "Specify MAC algorithms"
+        if isinstance(macs, str):
+            macs = macs.encode("utf-8")
+        macs = macs.split(b",")
+        for mac in macs:
+            if mac not in SSHCiphers.macMap:
+                sys.exit("Unknown mac type '%r'" % mac)
+        self["macs"] = macs
+
+    def opt_host_key_algorithms(self, hkas):
+        "Select host key algorithms"
+        if isinstance(hkas, str):
+            hkas = hkas.encode("utf-8")
+        hkas = hkas.split(b",")
+        for hka in hkas:
+            if hka not in SSHClientTransport.supportedPublicKeys:
+                sys.exit("Unknown host key type '%r'" % hka)
+        self["host-key-algorithms"] = hkas
+
+    def opt_user_authentications(self, uas):
+        "Choose how to authenticate to the remote server"
+        if isinstance(uas, str):
+            uas = uas.encode("utf-8")
+        self["user-authentications"] = uas.split(b",")
+
+
+#    def opt_compress(self):
+#        "Enable compression"
+#        self.enableCompression = 1
+#        SSHClientTransport.supportedCompressions[0:1] = ['zlib']
diff --git a/contrib/python/Twisted/py3/twisted/conch/endpoints.py b/contrib/python/Twisted/py3/twisted/conch/endpoints.py
new file mode 100644
index 00000000000..f2ab315848a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/endpoints.py
@@ -0,0 +1,875 @@
+# -*- test-case-name: twisted.conch.test.test_endpoints -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Endpoint implementations of various SSH interactions.
+"""
+
+__all__ = ["AuthenticationFailed", "SSHCommandAddress", "SSHCommandClientEndpoint"]
+
+import signal
+from os.path import expanduser
+from struct import unpack
+
+from zope.interface import Interface, implementer
+
+from twisted.conch.client.agent import SSHAgentClient
+from twisted.conch.client.default import _KNOWN_HOSTS
+from twisted.conch.client.knownhosts import ConsoleUI, KnownHostsFile
+from twisted.conch.ssh.channel import SSHChannel
+from twisted.conch.ssh.common import NS, getNS
+from twisted.conch.ssh.connection import SSHConnection
+from twisted.conch.ssh.keys import Key
+from twisted.conch.ssh.transport import SSHClientTransport
+from twisted.conch.ssh.userauth import SSHUserAuthClient
+from twisted.internet.defer import CancelledError, Deferred, succeed
+from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol
+from twisted.internet.error import ConnectionDone, ProcessTerminated
+from twisted.internet.interfaces import IStreamClientEndpoint
+from twisted.internet.protocol import Factory
+from twisted.logger import Logger
+from twisted.python.compat import nativeString, networkString
+from twisted.python.failure import Failure
+from twisted.python.filepath import FilePath
+
+
+class AuthenticationFailed(Exception):
+    """
+    An SSH session could not be established because authentication was not
+    successful.
+    """
+
+
+# This should be public.  See #6541.
+class _ISSHConnectionCreator(Interface):
+    """
+    An L{_ISSHConnectionCreator} knows how to create SSH connections somehow.
+    """
+
+    def secureConnection():
+        """
+        Return a new, connected, secured, but not yet authenticated instance of
+        L{twisted.conch.ssh.transport.SSHServerTransport} or
+        L{twisted.conch.ssh.transport.SSHClientTransport}.
+        """
+
+    def cleanupConnection(connection, immediate):
+        """
+        Perform cleanup necessary for a connection object previously returned
+        from this creator's C{secureConnection} method.
+
+        @param connection: An L{twisted.conch.ssh.transport.SSHServerTransport}
+            or L{twisted.conch.ssh.transport.SSHClientTransport} returned by a
+            previous call to C{secureConnection}.  It is no longer needed by
+            the caller of that method and may be closed or otherwise cleaned up
+            as necessary.
+
+        @param immediate: If C{True} don't wait for any network communication,
+            just close the connection immediately and as aggressively as
+            necessary.
+        """
+
+
+class SSHCommandAddress:
+    """
+    An L{SSHCommandAddress} instance represents the address of an SSH server, a
+    username which was used to authenticate with that server, and a command
+    which was run there.
+
+    @ivar server: See L{__init__}
+    @ivar username: See L{__init__}
+    @ivar command: See L{__init__}
+    """
+
+    def __init__(self, server, username, command):
+        """
+        @param server: The address of the SSH server on which the command is
+            running.
+        @type server: L{IAddress} provider
+
+        @param username: An authentication username which was used to
+            authenticate against the server at the given address.
+        @type username: L{bytes}
+
+        @param command: A command which was run in a session channel on the
+            server at the given address.
+        @type command: L{bytes}
+        """
+        self.server = server
+        self.username = username
+        self.command = command
+
+
+class _CommandChannel(SSHChannel):
+    """
+    A L{_CommandChannel} executes a command in a session channel and connects
+    its input and output to an L{IProtocol} provider.
+
+    @ivar _creator: See L{__init__}
+    @ivar _command: See L{__init__}
+    @ivar _protocolFactory:  See L{__init__}
+    @ivar _commandConnected:  See L{__init__}
+    @ivar _protocol: An L{IProtocol} provider created using C{_protocolFactory}
+        which is hooked up to the running command's input and output streams.
+    """
+
+    name = b"session"
+    _log = Logger()
+
+    def __init__(self, creator, command, protocolFactory, commandConnected):
+        """
+        @param creator: The L{_ISSHConnectionCreator} provider which was used
+            to get the connection which this channel exists on.
+        @type creator: L{_ISSHConnectionCreator} provider
+
+        @param command: The command to be executed.
+        @type command: L{bytes}
+
+        @param protocolFactory: A client factory to use to build a L{IProtocol}
+            provider to use to associate with the running command.
+
+        @param commandConnected: A L{Deferred} to use to signal that execution
+            of the command has failed or that it has succeeded and the command
+            is now running.
+        @type commandConnected: L{Deferred}
+        """
+        SSHChannel.__init__(self)
+        self._creator = creator
+        self._command = command
+        self._protocolFactory = protocolFactory
+        self._commandConnected = commandConnected
+        self._reason = None
+
+    def openFailed(self, reason):
+        """
+        When the request to open a new channel to run this command in fails,
+        fire the C{commandConnected} deferred with a failure indicating that.
+        """
+        self._commandConnected.errback(reason)
+
+    def channelOpen(self, ignored):
+        """
+        When the request to open a new channel to run this command in succeeds,
+        issue an C{"exec"} request to run the command.
+        """
+        command = self.conn.sendRequest(
+            self, b"exec", NS(self._command), wantReply=True
+        )
+        command.addCallbacks(self._execSuccess, self._execFailure)
+
+    def _execFailure(self, reason):
+        """
+        When the request to execute the command in this channel fails, fire the
+        C{commandConnected} deferred with a failure indicating this.
+
+        @param reason: The cause of the command execution failure.
+        @type reason: L{Failure}
+        """
+        self._commandConnected.errback(reason)
+
+    def _execSuccess(self, ignored):
+        """
+        When the request to execute the command in this channel succeeds, use
+        C{protocolFactory} to build a protocol to handle the command's input
+        and output and connect the protocol to a transport representing those
+        streams.
+
+        Also fire C{commandConnected} with the created protocol after it is
+        connected to its transport.
+
+        @param ignored: The (ignored) result of the execute request
+        """
+        self._protocol = self._protocolFactory.buildProtocol(
+            SSHCommandAddress(
+                self.conn.transport.transport.getPeer(),
+                self.conn.transport.creator.username,
+                self.conn.transport.creator.command,
+            )
+        )
+        self._protocol.makeConnection(self)
+        self._commandConnected.callback(self._protocol)
+
+    def dataReceived(self, data):
+        """
+        When the command's stdout data arrives over the channel, deliver it to
+        the protocol instance.
+
+        @param data: The bytes from the command's stdout.
+        @type data: L{bytes}
+        """
+        self._protocol.dataReceived(data)
+
+    def request_exit_status(self, data):
+        """
+        When the server sends the command's exit status, record it for later
+        delivery to the protocol.
+
+        @param data: The network-order four byte representation of the exit
+            status of the command.
+        @type data: L{bytes}
+        """
+        (status,) = unpack(">L", data)
+        if status != 0:
+            self._reason = ProcessTerminated(status, None, None)
+
+    def request_exit_signal(self, data):
+        """
+        When the server sends the command's exit status, record it for later
+        delivery to the protocol.
+
+        @param data: The network-order four byte representation of the exit
+            signal of the command.
+        @type data: L{bytes}
+        """
+        shortSignalName, data = getNS(data)
+        coreDumped, data = bool(ord(data[0:1])), data[1:]
+        errorMessage, data = getNS(data)
+        languageTag, data = getNS(data)
+        signalName = f"SIG{nativeString(shortSignalName)}"
+        signalID = getattr(signal, signalName, -1)
+        self._log.info(
+            "Process exited with signal {shortSignalName!r};"
+            " core dumped: {coreDumped};"
+            " error message: {errorMessage};"
+            " language: {languageTag!r}",
+            shortSignalName=shortSignalName,
+            coreDumped=coreDumped,
+            errorMessage=errorMessage.decode("utf-8"),
+            languageTag=languageTag,
+        )
+        self._reason = ProcessTerminated(None, signalID, None)
+
+    def closed(self):
+        """
+        When the channel closes, deliver disconnection notification to the
+        protocol.
+        """
+        self._creator.cleanupConnection(self.conn, False)
+        if self._reason is None:
+            reason = ConnectionDone("ssh channel closed")
+        else:
+            reason = self._reason
+        self._protocol.connectionLost(Failure(reason))
+
+
+class _ConnectionReady(SSHConnection):
+    """
+    L{_ConnectionReady} is an L{SSHConnection} (an SSH service) which only
+    propagates the I{serviceStarted} event to a L{Deferred} to be handled
+    elsewhere.
+    """
+
+    def __init__(self, ready):
+        """
+        @param ready: A L{Deferred} which should be fired when
+            I{serviceStarted} happens.
+        """
+        SSHConnection.__init__(self)
+        self._ready = ready
+
+    def serviceStarted(self):
+        """
+        When the SSH I{connection} I{service} this object represents is ready
+        to be used, fire the C{connectionReady} L{Deferred} to publish that
+        event to some other interested party.
+
+        """
+        self._ready.callback(self)
+        del self._ready
+
+
+class _UserAuth(SSHUserAuthClient):
+    """
+    L{_UserAuth} implements the client part of SSH user authentication in the
+    convenient way a user might expect if they are familiar with the
+    interactive I{ssh} command line client.
+
+    L{_UserAuth} supports key-based authentication, password-based
+    authentication, and delegating authentication to an agent.
+    """
+
+    password = None
+    keys = None
+    agent = None
+
+    def getPublicKey(self):
+        """
+        Retrieve the next public key object to offer to the server, possibly
+        delegating to an authentication agent if there is one.
+
+        @return: The public part of a key pair that could be used to
+            authenticate with the server, or L{None} if there are no more
+            public keys to try.
+        @rtype: L{twisted.conch.ssh.keys.Key} or L{None}
+        """
+        if self.agent is not None:
+            return self.agent.getPublicKey()
+
+        if self.keys:
+            self.key = self.keys.pop(0)
+        else:
+            self.key = None
+        return self.key.public()
+
+    def signData(self, publicKey, signData):
+        """
+        Extend the base signing behavior by using an SSH agent to sign the
+        data, if one is available.
+
+        @type publicKey: L{Key}
+        @type signData: L{str}
+        """
+        if self.agent is not None:
+            return self.agent.signData(publicKey.blob(), signData)
+        else:
+            return SSHUserAuthClient.signData(self, publicKey, signData)
+
+    def getPrivateKey(self):
+        """
+        Get the private part of a key pair to use for authentication.  The key
+        corresponds to the public part most recently returned from
+        C{getPublicKey}.
+
+        @return: A L{Deferred} which fires with the private key.
+        @rtype: L{Deferred}
+        """
+        return succeed(self.key)
+
+    def getPassword(self):
+        """
+        Get the password to use for authentication.
+
+        @return: A L{Deferred} which fires with the password, or L{None} if the
+            password was not specified.
+        """
+        if self.password is None:
+            return
+        return succeed(self.password)
+
+    def ssh_USERAUTH_SUCCESS(self, packet):
+        """
+        Handle user authentication success in the normal way, but also make a
+        note of the state change on the L{_CommandTransport}.
+        """
+        self.transport._state = b"CHANNELLING"
+        return SSHUserAuthClient.ssh_USERAUTH_SUCCESS(self, packet)
+
+    def connectToAgent(self, endpoint):
+        """
+        Set up a connection to the authentication agent and trigger its
+        initialization.
+
+        @param endpoint: An endpoint which can be used to connect to the
+            authentication agent.
+        @type endpoint: L{IStreamClientEndpoint} provider
+
+        @return: A L{Deferred} which fires when the agent connection is ready
+            for use.
+        """
+        factory = Factory()
+        factory.protocol = SSHAgentClient
+        d = endpoint.connect(factory)
+
+        def connected(agent):
+            self.agent = agent
+            return agent.getPublicKeys()
+
+        d.addCallback(connected)
+        return d
+
+    def loseAgentConnection(self):
+        """
+        Disconnect the agent.
+        """
+        if self.agent is None:
+            return
+        self.agent.transport.loseConnection()
+
+
+class _CommandTransport(SSHClientTransport):
+    """
+    L{_CommandTransport} is an SSH client I{transport} which includes a host
+    key verification step before it will proceed to secure the connection.
+
+    L{_CommandTransport} also knows how to set up a connection to an
+    authentication agent if it is told where it can connect to one.
+
+    @ivar _userauth: The L{_UserAuth} instance which is in charge of the
+        overall authentication process or L{None} if the SSH connection has not
+        reach yet the C{user-auth} service.
+    @type _userauth: L{_UserAuth}
+    """
+
+    # STARTING -> SECURING -> AUTHENTICATING -> CHANNELLING -> RUNNING
+    _state = b"STARTING"
+
+    _hostKeyFailure = None
+
+    _userauth = None
+
+    def __init__(self, creator):
+        """
+        @param creator: The L{_NewConnectionHelper} that created this
+            connection.
+
+        @type creator: L{_NewConnectionHelper}.
+        """
+        self.connectionReady = Deferred(lambda d: self.transport.abortConnection())
+        # Clear the reference to that deferred to help the garbage collector
+        # and to signal to other parts of this implementation (in particular
+        # connectionLost) that it has already been fired and does not need to
+        # be fired again.
+
+        def readyFired(result):
+            self.connectionReady = None
+            return result
+
+        self.connectionReady.addBoth(readyFired)
+        self.creator = creator
+
+    def verifyHostKey(self, hostKey, fingerprint):
+        """
+        Ask the L{KnownHostsFile} provider available on the factory which
+        created this protocol this protocol to verify the given host key.
+
+        @return: A L{Deferred} which fires with the result of
+            L{KnownHostsFile.verifyHostKey}.
+        """
+        hostname = self.creator.hostname
+        ip = networkString(self.transport.getPeer().host)
+
+        self._state = b"SECURING"
+        d = self.creator.knownHosts.verifyHostKey(
+            self.creator.ui, hostname, ip, Key.fromString(hostKey)
+        )
+        d.addErrback(self._saveHostKeyFailure)
+        return d
+
+    def _saveHostKeyFailure(self, reason):
+        """
+        When host key verification fails, record the reason for the failure in
+        order to fire a L{Deferred} with it later.
+
+        @param reason: The cause of the host key verification failure.
+        @type reason: L{Failure}
+
+        @return: C{reason}
+        @rtype: L{Failure}
+        """
+        self._hostKeyFailure = reason
+        return reason
+
+    def connectionSecure(self):
+        """
+        When the connection is secure, start the authentication process.
+        """
+        self._state = b"AUTHENTICATING"
+
+        command = _ConnectionReady(self.connectionReady)
+
+        self._userauth = _UserAuth(self.creator.username, command)
+        self._userauth.password = self.creator.password
+        if self.creator.keys:
+            self._userauth.keys = list(self.creator.keys)
+
+        if self.creator.agentEndpoint is not None:
+            d = self._userauth.connectToAgent(self.creator.agentEndpoint)
+        else:
+            d = succeed(None)
+
+        def maybeGotAgent(ignored):
+            self.requestService(self._userauth)
+
+        d.addBoth(maybeGotAgent)
+
+    def connectionLost(self, reason):
+        """
+        When the underlying connection to the SSH server is lost, if there were
+        any connection setup errors, propagate them. Also, clean up the
+        connection to the ssh agent if one was created.
+        """
+        if self._userauth:
+            self._userauth.loseAgentConnection()
+
+        if self._state == b"RUNNING" or self.connectionReady is None:
+            return
+        if self._state == b"SECURING" and self._hostKeyFailure is not None:
+            reason = self._hostKeyFailure
+        elif self._state == b"AUTHENTICATING":
+            reason = Failure(
+                AuthenticationFailed("Connection lost while authenticating")
+            )
+        self.connectionReady.errback(reason)
+
+
+@implementer(IStreamClientEndpoint)
+class SSHCommandClientEndpoint:
+    """
+    L{SSHCommandClientEndpoint} exposes the command-executing functionality of
+    SSH servers.
+
+    L{SSHCommandClientEndpoint} can set up a new SSH connection, authenticate
+    it in any one of a number of different ways (keys, passwords, agents),
+    launch a command over that connection and then associate its input and
+    output with a protocol.
+
+    It can also re-use an existing, already-authenticated SSH connection
+    (perhaps one which already has some SSH channels being used for other
+    purposes).  In this case it creates a new SSH channel to use to execute the
+    command.  Notably this means it supports multiplexing several different
+    command invocations over a single SSH connection.
+    """
+
+    def __init__(self, creator, command):
+        """
+        @param creator: An L{_ISSHConnectionCreator} provider which will be
+            used to set up the SSH connection which will be used to run a
+            command.
+        @type creator: L{_ISSHConnectionCreator} provider
+
+        @param command: The command line to execute on the SSH server.  This
+            byte string is interpreted by a shell on the SSH server, so it may
+            have a value like C{"ls /"}.  Take care when trying to run a
+            command like C{"/Volumes/My Stuff/a-program"} - spaces (and other
+            special bytes) may require escaping.
+        @type command: L{bytes}
+
+        """
+        self._creator = creator
+        self._command = command
+
+    @classmethod
+    def newConnection(
+        cls,
+        reactor,
+        command,
+        username,
+        hostname,
+        port=None,
+        keys=None,
+        password=None,
+        agentEndpoint=None,
+        knownHosts=None,
+        ui=None,
+    ):
+        """
+        Create and return a new endpoint which will try to create a new
+        connection to an SSH server and run a command over it.  It will also
+        close the connection if there are problems leading up to the command
+        being executed, after the command finishes, or if the connection
+        L{Deferred} is cancelled.
+
+        @param reactor: The reactor to use to establish the connection.
+        @type reactor: L{IReactorTCP} provider
+
+        @param command: See L{__init__}'s C{command} argument.
+
+        @param username: The username with which to authenticate to the SSH
+            server.
+        @type username: L{bytes}
+
+        @param hostname: The hostname of the SSH server.
+        @type hostname: L{bytes}
+
+        @param port: The port number of the SSH server.  By default, the
+            standard SSH port number is used.
+        @type port: L{int}
+
+        @param keys: Private keys with which to authenticate to the SSH server,
+            if key authentication is to be attempted (otherwise L{None}).
+        @type keys: L{list} of L{Key}
+
+        @param password: The password with which to authenticate to the SSH
+            server, if password authentication is to be attempted (otherwise
+            L{None}).
+        @type password: L{bytes} or L{None}
+
+        @param agentEndpoint: An L{IStreamClientEndpoint} provider which may be
+            used to connect to an SSH agent, if one is to be used to help with
+            authentication.
+        @type agentEndpoint: L{IStreamClientEndpoint} provider
+
+        @param knownHosts: The currently known host keys, used to check the
+            host key presented by the server we actually connect to.
+        @type knownHosts: L{KnownHostsFile}
+
+        @param ui: An object for interacting with users to make decisions about
+            whether to accept the server host keys.  If L{None}, a L{ConsoleUI}
+            connected to /dev/tty will be used; if /dev/tty is unavailable, an
+            object which answers C{b"no"} to all prompts will be used.
+        @type ui: L{None} or L{ConsoleUI}
+
+        @return: A new instance of C{cls} (probably
+            L{SSHCommandClientEndpoint}).
+        """
+        helper = _NewConnectionHelper(
+            reactor,
+            hostname,
+            port,
+            command,
+            username,
+            keys,
+            password,
+            agentEndpoint,
+            knownHosts,
+            ui,
+        )
+        return cls(helper, command)
+
+    @classmethod
+    def existingConnection(cls, connection, command):
+        """
+        Create and return a new endpoint which will try to open a new channel
+        on an existing SSH connection and run a command over it.  It will
+        B{not} close the connection if there is a problem executing the command
+        or after the command finishes.
+
+        @param connection: An existing connection to an SSH server.
+        @type connection: L{SSHConnection}
+
+        @param command: See L{SSHCommandClientEndpoint.newConnection}'s
+            C{command} parameter.
+        @type command: L{bytes}
+
+        @return: A new instance of C{cls} (probably
+            L{SSHCommandClientEndpoint}).
+        """
+        helper = _ExistingConnectionHelper(connection)
+        return cls(helper, command)
+
+    def connect(self, protocolFactory):
+        """
+        Set up an SSH connection, use a channel from that connection to launch
+        a command, and hook the stdin and stdout of that command up as a
+        transport for a protocol created by the given factory.
+
+        @param protocolFactory: A L{Factory} to use to create the protocol
+            which will be connected to the stdin and stdout of the command on
+            the SSH server.
+
+        @return: A L{Deferred} which will fire with an error if the connection
+            cannot be set up for any reason or with the protocol instance
+            created by C{protocolFactory} once it has been connected to the
+            command.
+        """
+        d = self._creator.secureConnection()
+        d.addCallback(self._executeCommand, protocolFactory)
+        return d
+
+    def _executeCommand(self, connection, protocolFactory):
+        """
+        Given a secured SSH connection, try to execute a command in a new
+        channel created on it and associate the result with a protocol from the
+        given factory.
+
+        @param connection: See L{SSHCommandClientEndpoint.existingConnection}'s
+            C{connection} parameter.
+
+        @param protocolFactory: See L{SSHCommandClientEndpoint.connect}'s
+            C{protocolFactory} parameter.
+
+        @return: See L{SSHCommandClientEndpoint.connect}'s return value.
+        """
+        commandConnected = Deferred()
+
+        def disconnectOnFailure(passthrough):
+            # Close the connection immediately in case of cancellation, since
+            # that implies user wants it gone immediately (e.g. a timeout):
+            immediate = passthrough.check(CancelledError)
+            self._creator.cleanupConnection(connection, immediate)
+            return passthrough
+
+        commandConnected.addErrback(disconnectOnFailure)
+
+        channel = _CommandChannel(
+            self._creator, self._command, protocolFactory, commandConnected
+        )
+        connection.openChannel(channel)
+        return commandConnected
+
+
+class _ReadFile:
+    """
+    A weakly file-like object which can be used with L{KnownHostsFile} to
+    respond in the negative to all prompts for decisions.
+    """
+
+    def __init__(self, contents):
+        """
+        @param contents: L{bytes} which will be returned from every C{readline}
+            call.
+        """
+        self._contents = contents
+
+    def write(self, data):
+        """
+        No-op.
+
+        @param data: ignored
+        """
+
+    def readline(self, count=-1):
+        """
+        Always give back the byte string that this L{_ReadFile} was initialized
+        with.
+
+        @param count: ignored
+
+        @return: A fixed byte-string.
+        @rtype: L{bytes}
+        """
+        return self._contents
+
+    def close(self):
+        """
+        No-op.
+        """
+
+
+@implementer(_ISSHConnectionCreator)
+class _NewConnectionHelper:
+    """
+    L{_NewConnectionHelper} implements L{_ISSHConnectionCreator} by
+    establishing a brand new SSH connection, securing it, and authenticating.
+    """
+
+    _KNOWN_HOSTS = _KNOWN_HOSTS
+    port = 22
+
+    def __init__(
+        self,
+        reactor,
+        hostname,
+        port,
+        command,
+        username,
+        keys,
+        password,
+        agentEndpoint,
+        knownHosts,
+        ui,
+        tty=FilePath(b"/dev/tty"),
+    ):
+        """
+        @param tty: The path of the tty device to use in case C{ui} is L{None}.
+        @type tty: L{FilePath}
+
+        @see: L{SSHCommandClientEndpoint.newConnection}
+        """
+        self.reactor = reactor
+        self.hostname = hostname
+        if port is not None:
+            self.port = port
+        self.command = command
+        self.username = username
+        self.keys = keys
+        self.password = password
+        self.agentEndpoint = agentEndpoint
+        if knownHosts is None:
+            knownHosts = self._knownHosts()
+        self.knownHosts = knownHosts
+
+        if ui is None:
+            ui = ConsoleUI(self._opener)
+        self.ui = ui
+        self.tty = tty
+
+    def _opener(self):
+        """
+        Open the tty if possible, otherwise give back a file-like object from
+        which C{b"no"} can be read.
+
+        For use as the opener argument to L{ConsoleUI}.
+        """
+        try:
+            return self.tty.open("rb+")
+        except BaseException:
+            # Give back a file-like object from which can be read a byte string
+            # that KnownHostsFile recognizes as rejecting some option (b"no").
+            return _ReadFile(b"no")
+
+    @classmethod
+    def _knownHosts(cls):
+        """
+
+        @return: A L{KnownHostsFile} instance pointed at the user's personal
+            I{known hosts} file.
+        @rtype: L{KnownHostsFile}
+        """
+        return KnownHostsFile.fromPath(FilePath(expanduser(cls._KNOWN_HOSTS)))
+
+    def secureConnection(self):
+        """
+        Create and return a new SSH connection which has been secured and on
+        which authentication has already happened.
+
+        @return: A L{Deferred} which fires with the ready-to-use connection or
+            with a failure if something prevents the connection from being
+            setup, secured, or authenticated.
+        """
+        protocol = _CommandTransport(self)
+        ready = protocol.connectionReady
+
+        sshClient = TCP4ClientEndpoint(
+            self.reactor, nativeString(self.hostname), self.port
+        )
+
+        d = connectProtocol(sshClient, protocol)
+        d.addCallback(lambda ignored: ready)
+        return d
+
+    def cleanupConnection(self, connection, immediate):
+        """
+        Clean up the connection by closing it.  The command running on the
+        endpoint has ended so the connection is no longer needed.
+
+        @param connection: The L{SSHConnection} to close.
+        @type connection: L{SSHConnection}
+
+        @param immediate: Whether to close connection immediately.
+        @type immediate: L{bool}.
+        """
+        if immediate:
+            # We're assuming the underlying connection is an ITCPTransport,
+            # which is what the current implementation is restricted to:
+            connection.transport.transport.abortConnection()
+        else:
+            connection.transport.loseConnection()
+
+
+@implementer(_ISSHConnectionCreator)
+class _ExistingConnectionHelper:
+    """
+    L{_ExistingConnectionHelper} implements L{_ISSHConnectionCreator} by
+    handing out an existing SSH connection which is supplied to its
+    initializer.
+    """
+
+    def __init__(self, connection):
+        """
+        @param connection: See L{SSHCommandClientEndpoint.existingConnection}'s
+            C{connection} parameter.
+        """
+        self.connection = connection
+
+    def secureConnection(self):
+        """
+
+        @return: A L{Deferred} that fires synchronously with the
+            already-established connection object.
+        """
+        return succeed(self.connection)
+
+    def cleanupConnection(self, connection, immediate):
+        """
+        Do not do any cleanup on the connection.  Leave that responsibility to
+        whatever code created it in the first place.
+
+        @param connection: The L{SSHConnection} which will not be modified in
+            any way.
+        @type connection: L{SSHConnection}
+
+        @param immediate: An argument which will be ignored.
+        @type immediate: L{bool}.
+        """
diff --git a/contrib/python/Twisted/py3/twisted/conch/error.py b/contrib/python/Twisted/py3/twisted/conch/error.py
new file mode 100644
index 00000000000..a923b9a4c4a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/error.py
@@ -0,0 +1,96 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An error to represent bad things happening in Conch.
+
+Maintainer: Paul Swartz
+"""
+
+
+from twisted.cred.error import UnauthorizedLogin
+
+
+class ConchError(Exception):
+    def __init__(self, value, data=None):
+        Exception.__init__(self, value, data)
+        self.value = value
+        self.data = data
+
+
+class NotEnoughAuthentication(Exception):
+    """
+    This is thrown if the authentication is valid, but is not enough to
+    successfully verify the user.  i.e. don't retry this type of
+    authentication, try another one.
+    """
+
+
+class ValidPublicKey(UnauthorizedLogin):
+    """
+    Raised by public key checkers when they receive public key credentials
+    that don't contain a signature at all, but are valid in every other way.
+    (e.g. the public key matches one in the user's authorized_keys file).
+
+    Protocol code (eg
+    L{SSHUserAuthServer<twisted.conch.ssh.userauth.SSHUserAuthServer>}) which
+    attempts to log in using
+    L{ISSHPrivateKey<twisted.cred.credentials.ISSHPrivateKey>} credentials
+    should be prepared to handle a failure of this type by telling the user to
+    re-authenticate using the same key and to include a signature with the new
+    attempt.
+
+    See U{http://www.ietf.org/rfc/rfc4252.txt} section 7 for more details.
+    """
+
+
+class IgnoreAuthentication(Exception):
+    """
+    This is thrown to let the UserAuthServer know it doesn't need to handle the
+    authentication anymore.
+    """
+
+
+class MissingKeyStoreError(Exception):
+    """
+    Raised if an SSHAgentServer starts receiving data without its factory
+    providing a keys dict on which to read/write key data.
+    """
+
+
+class UserRejectedKey(Exception):
+    """
+    The user interactively rejected a key.
+    """
+
+
+class InvalidEntry(Exception):
+    """
+    An entry in a known_hosts file could not be interpreted as a valid entry.
+    """
+
+
+class HostKeyChanged(Exception):
+    """
+    The host key of a remote host has changed.
+
+    @ivar offendingEntry: The entry which contains the persistent host key that
+    disagrees with the given host key.
+
+    @type offendingEntry: L{twisted.conch.interfaces.IKnownHostEntry}
+
+    @ivar path: a reference to the known_hosts file that the offending entry
+    was loaded from
+
+    @type path: L{twisted.python.filepath.FilePath}
+
+    @ivar lineno: The line number of the offending entry in the given path.
+
+    @type lineno: L{int}
+    """
+
+    def __init__(self, offendingEntry, path, lineno):
+        Exception.__init__(self)
+        self.offendingEntry = offendingEntry
+        self.path = path
+        self.lineno = lineno
diff --git a/contrib/python/Twisted/py3/twisted/conch/insults/__init__.py b/contrib/python/Twisted/py3/twisted/conch/insults/__init__.py
new file mode 100644
index 00000000000..3d838766989
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/insults/__init__.py
@@ -0,0 +1,4 @@
+"""
+Insults: a replacement for Curses/S-Lang.
+
+Very basic at the moment."""
diff --git a/contrib/python/Twisted/py3/twisted/conch/insults/helper.py b/contrib/python/Twisted/py3/twisted/conch/insults/helper.py
new file mode 100644
index 00000000000..92eca1d89eb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/insults/helper.py
@@ -0,0 +1,556 @@
+# -*- test-case-name: twisted.conch.test.test_helper -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Partial in-memory terminal emulator
+
+@author: Jp Calderone
+"""
+
+
+import re
+import string
+
+from zope.interface import implementer
+
+from incremental import Version
+
+from twisted.conch.insults import insults
+from twisted.internet import defer, protocol, reactor
+from twisted.logger import Logger
+from twisted.python import _textattributes
+from twisted.python.compat import iterbytes
+from twisted.python.deprecate import deprecated, deprecatedModuleAttribute
+
+FOREGROUND = 30
+BACKGROUND = 40
+BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, N_COLORS = range(9)
+
+
+class _FormattingState(_textattributes._FormattingStateMixin):
+    """
+    Represents the formatting state/attributes of a single character.
+
+    Character set, intensity, underlinedness, blinkitude, video
+    reversal, as well as foreground and background colors made up a
+    character's attributes.
+    """
+
+    compareAttributes = (
+        "charset",
+        "bold",
+        "underline",
+        "blink",
+        "reverseVideo",
+        "foreground",
+        "background",
+        "_subtracting",
+    )
+
+    def __init__(
+        self,
+        charset=insults.G0,
+        bold=False,
+        underline=False,
+        blink=False,
+        reverseVideo=False,
+        foreground=WHITE,
+        background=BLACK,
+        _subtracting=False,
+    ):
+        self.charset = charset
+        self.bold = bold
+        self.underline = underline
+        self.blink = blink
+        self.reverseVideo = reverseVideo
+        self.foreground = foreground
+        self.background = background
+        self._subtracting = _subtracting
+
+    @deprecated(Version("Twisted", 13, 1, 0))
+    def wantOne(self, **kw):
+        """
+        Add a character attribute to a copy of this formatting state.
+
+        @param kw: An optional attribute name and value can be provided with
+            a keyword argument.
+
+        @return: A formatting state instance with the new attribute.
+
+        @see: L{DefaultFormattingState._withAttribute}.
+        """
+        k, v = kw.popitem()
+        return self._withAttribute(k, v)
+
+    def toVT102(self):
+        # Spit out a vt102 control sequence that will set up
+        # all the attributes set here.  Except charset.
+        attrs = []
+        if self._subtracting:
+            attrs.append(0)
+        if self.bold:
+            attrs.append(insults.BOLD)
+        if self.underline:
+            attrs.append(insults.UNDERLINE)
+        if self.blink:
+            attrs.append(insults.BLINK)
+        if self.reverseVideo:
+            attrs.append(insults.REVERSE_VIDEO)
+        if self.foreground != WHITE:
+            attrs.append(FOREGROUND + self.foreground)
+        if self.background != BLACK:
+            attrs.append(BACKGROUND + self.background)
+        if attrs:
+            return "\x1b[" + ";".join(map(str, attrs)) + "m"
+        return ""
+
+
+CharacterAttribute = _FormattingState
+
+deprecatedModuleAttribute(
+    Version("Twisted", 13, 1, 0),
+    "Use twisted.conch.insults.text.assembleFormattedText instead.",
+    "twisted.conch.insults.helper",
+    "CharacterAttribute",
+)
+
+
+# XXX - need to support scroll regions and scroll history
+@implementer(insults.ITerminalTransport)
+class TerminalBuffer(protocol.Protocol):
+    """
+    An in-memory terminal emulator.
+    """
+
+    for keyID in (
+        b"UP_ARROW",
+        b"DOWN_ARROW",
+        b"RIGHT_ARROW",
+        b"LEFT_ARROW",
+        b"HOME",
+        b"INSERT",
+        b"DELETE",
+        b"END",
+        b"PGUP",
+        b"PGDN",
+        b"F1",
+        b"F2",
+        b"F3",
+        b"F4",
+        b"F5",
+        b"F6",
+        b"F7",
+        b"F8",
+        b"F9",
+        b"F10",
+        b"F11",
+        b"F12",
+    ):
+        execBytes = keyID + b" = object()"
+        execStr = execBytes.decode("ascii")
+        exec(execStr)
+
+    TAB = b"\t"
+    BACKSPACE = b"\x7f"
+
+    width = 80
+    height = 24
+
+    fill = b" "
+    void = object()
+    _log = Logger()
+
+    def getCharacter(self, x, y):
+        return self.lines[y][x]
+
+    def connectionMade(self):
+        self.reset()
+
+    def write(self, data):
+        """
+        Add the given printable bytes to the terminal.
+
+        Line feeds in L{bytes} will be replaced with carriage return / line
+        feed pairs.
+        """
+        for b in iterbytes(data.replace(b"\n", b"\r\n")):
+            self.insertAtCursor(b)
+
+    def _currentFormattingState(self):
+        return _FormattingState(self.activeCharset, **self.graphicRendition)
+
+    def insertAtCursor(self, b):
+        """
+        Add one byte to the terminal at the cursor and make consequent state
+        updates.
+
+        If b is a carriage return, move the cursor to the beginning of the
+        current row.
+
+        If b is a line feed, move the cursor to the next row or scroll down if
+        the cursor is already in the last row.
+
+        Otherwise, if b is printable, put it at the cursor position (inserting
+        or overwriting as dictated by the current mode) and move the cursor.
+        """
+        if b == b"\r":
+            self.x = 0
+        elif b == b"\n":
+            self._scrollDown()
+        elif b in string.printable.encode("ascii"):
+            if self.x >= self.width:
+                self.nextLine()
+            ch = (b, self._currentFormattingState())
+            if self.modes.get(insults.modes.IRM):
+                self.lines[self.y][self.x : self.x] = [ch]
+                self.lines[self.y].pop()
+            else:
+                self.lines[self.y][self.x] = ch
+            self.x += 1
+
+    def _emptyLine(self, width):
+        return [(self.void, self._currentFormattingState()) for i in range(width)]
+
+    def _scrollDown(self):
+        self.y += 1
+        if self.y >= self.height:
+            self.y -= 1
+            del self.lines[0]
+            self.lines.append(self._emptyLine(self.width))
+
+    def _scrollUp(self):
+        self.y -= 1
+        if self.y < 0:
+            self.y = 0
+            del self.lines[-1]
+            self.lines.insert(0, self._emptyLine(self.width))
+
+    def cursorUp(self, n=1):
+        self.y = max(0, self.y - n)
+
+    def cursorDown(self, n=1):
+        self.y = min(self.height - 1, self.y + n)
+
+    def cursorBackward(self, n=1):
+        self.x = max(0, self.x - n)
+
+    def cursorForward(self, n=1):
+        self.x = min(self.width, self.x + n)
+
+    def cursorPosition(self, column, line):
+        self.x = column
+        self.y = line
+
+    def cursorHome(self):
+        self.x = self.home.x
+        self.y = self.home.y
+
+    def index(self):
+        self._scrollDown()
+
+    def reverseIndex(self):
+        self._scrollUp()
+
+    def nextLine(self):
+        """
+        Update the cursor position attributes and scroll down if appropriate.
+        """
+        self.x = 0
+        self._scrollDown()
+
+    def saveCursor(self):
+        self._savedCursor = (self.x, self.y)
+
+    def restoreCursor(self):
+        self.x, self.y = self._savedCursor
+        del self._savedCursor
+
+    def setModes(self, modes):
+        for m in modes:
+            self.modes[m] = True
+
+    def resetModes(self, modes):
+        for m in modes:
+            try:
+                del self.modes[m]
+            except KeyError:
+                pass
+
+    def setPrivateModes(self, modes):
+        """
+        Enable the given modes.
+
+        Track which modes have been enabled so that the implementations of
+        other L{insults.ITerminalTransport} methods can be properly implemented
+        to respect these settings.
+
+        @see: L{resetPrivateModes}
+        @see: L{insults.ITerminalTransport.setPrivateModes}
+        """
+        for m in modes:
+            self.privateModes[m] = True
+
+    def resetPrivateModes(self, modes):
+        """
+        Disable the given modes.
+
+        @see: L{setPrivateModes}
+        @see: L{insults.ITerminalTransport.resetPrivateModes}
+        """
+        for m in modes:
+            try:
+                del self.privateModes[m]
+            except KeyError:
+                pass
+
+    def applicationKeypadMode(self):
+        self.keypadMode = "app"
+
+    def numericKeypadMode(self):
+        self.keypadMode = "num"
+
+    def selectCharacterSet(self, charSet, which):
+        self.charsets[which] = charSet
+
+    def shiftIn(self):
+        self.activeCharset = insults.G0
+
+    def shiftOut(self):
+        self.activeCharset = insults.G1
+
+    def singleShift2(self):
+        oldActiveCharset = self.activeCharset
+        self.activeCharset = insults.G2
+        f = self.insertAtCursor
+
+        def insertAtCursor(b):
+            f(b)
+            del self.insertAtCursor
+            self.activeCharset = oldActiveCharset
+
+        self.insertAtCursor = insertAtCursor
+
+    def singleShift3(self):
+        oldActiveCharset = self.activeCharset
+        self.activeCharset = insults.G3
+        f = self.insertAtCursor
+
+        def insertAtCursor(b):
+            f(b)
+            del self.insertAtCursor
+            self.activeCharset = oldActiveCharset
+
+        self.insertAtCursor = insertAtCursor
+
+    def selectGraphicRendition(self, *attributes):
+        for a in attributes:
+            if a == insults.NORMAL:
+                self.graphicRendition = {
+                    "bold": False,
+                    "underline": False,
+                    "blink": False,
+                    "reverseVideo": False,
+                    "foreground": WHITE,
+                    "background": BLACK,
+                }
+            elif a == insults.BOLD:
+                self.graphicRendition["bold"] = True
+            elif a == insults.UNDERLINE:
+                self.graphicRendition["underline"] = True
+            elif a == insults.BLINK:
+                self.graphicRendition["blink"] = True
+            elif a == insults.REVERSE_VIDEO:
+                self.graphicRendition["reverseVideo"] = True
+            else:
+                try:
+                    v = int(a)
+                except ValueError:
+                    self._log.error(
+                        "Unknown graphic rendition attribute: {attr!r}", attr=a
+                    )
+                else:
+                    if FOREGROUND <= v <= FOREGROUND + N_COLORS:
+                        self.graphicRendition["foreground"] = v - FOREGROUND
+                    elif BACKGROUND <= v <= BACKGROUND + N_COLORS:
+                        self.graphicRendition["background"] = v - BACKGROUND
+                    else:
+                        self._log.error(
+                            "Unknown graphic rendition attribute: {attr!r}", attr=a
+                        )
+
+    def eraseLine(self):
+        self.lines[self.y] = self._emptyLine(self.width)
+
+    def eraseToLineEnd(self):
+        width = self.width - self.x
+        self.lines[self.y][self.x :] = self._emptyLine(width)
+
+    def eraseToLineBeginning(self):
+        self.lines[self.y][: self.x + 1] = self._emptyLine(self.x + 1)
+
+    def eraseDisplay(self):
+        self.lines = [self._emptyLine(self.width) for i in range(self.height)]
+
+    def eraseToDisplayEnd(self):
+        self.eraseToLineEnd()
+        height = self.height - self.y - 1
+        self.lines[self.y + 1 :] = [self._emptyLine(self.width) for i in range(height)]
+
+    def eraseToDisplayBeginning(self):
+        self.eraseToLineBeginning()
+        self.lines[: self.y] = [self._emptyLine(self.width) for i in range(self.y)]
+
+    def deleteCharacter(self, n=1):
+        del self.lines[self.y][self.x : self.x + n]
+        self.lines[self.y].extend(self._emptyLine(min(self.width - self.x, n)))
+
+    def insertLine(self, n=1):
+        self.lines[self.y : self.y] = [self._emptyLine(self.width) for i in range(n)]
+        del self.lines[self.height :]
+
+    def deleteLine(self, n=1):
+        del self.lines[self.y : self.y + n]
+        self.lines.extend([self._emptyLine(self.width) for i in range(n)])
+
+    def reportCursorPosition(self):
+        return (self.x, self.y)
+
+    def reset(self):
+        self.home = insults.Vector(0, 0)
+        self.x = self.y = 0
+        self.modes = {}
+        self.privateModes = {}
+        self.setPrivateModes(
+            [insults.privateModes.AUTO_WRAP, insults.privateModes.CURSOR_MODE]
+        )
+        self.numericKeypad = "app"
+        self.activeCharset = insults.G0
+        self.graphicRendition = {
+            "bold": False,
+            "underline": False,
+            "blink": False,
+            "reverseVideo": False,
+            "foreground": WHITE,
+            "background": BLACK,
+        }
+        self.charsets = {
+            insults.G0: insults.CS_US,
+            insults.G1: insults.CS_US,
+            insults.G2: insults.CS_ALTERNATE,
+            insults.G3: insults.CS_ALTERNATE_SPECIAL,
+        }
+        self.eraseDisplay()
+
+    def unhandledControlSequence(self, buf):
+        print("Could not handle", repr(buf))
+
+    def __bytes__(self):
+        lines = []
+        for L in self.lines:
+            buf = []
+            length = 0
+            for ch, attr in L:
+                if ch is not self.void:
+                    buf.append(ch)
+                    length = len(buf)
+                else:
+                    buf.append(self.fill)
+            lines.append(b"".join(buf[:length]))
+        return b"\n".join(lines)
+
+    def getHost(self):
+        # ITransport.getHost
+        raise NotImplementedError("Unimplemented: TerminalBuffer.getHost")
+
+    def getPeer(self):
+        # ITransport.getPeer
+        raise NotImplementedError("Unimplemented: TerminalBuffer.getPeer")
+
+    def loseConnection(self):
+        # ITransport.loseConnection
+        raise NotImplementedError("Unimplemented: TerminalBuffer.loseConnection")
+
+    def writeSequence(self, data):
+        # ITransport.writeSequence
+        raise NotImplementedError("Unimplemented: TerminalBuffer.writeSequence")
+
+    def horizontalTabulationSet(self):
+        # ITerminalTransport.horizontalTabulationSet
+        raise NotImplementedError(
+            "Unimplemented: TerminalBuffer.horizontalTabulationSet"
+        )
+
+    def tabulationClear(self):
+        # TerminalTransport.tabulationClear
+        raise NotImplementedError("Unimplemented: TerminalBuffer.tabulationClear")
+
+    def tabulationClearAll(self):
+        # TerminalTransport.tabulationClearAll
+        raise NotImplementedError("Unimplemented: TerminalBuffer.tabulationClearAll")
+
+    def doubleHeightLine(self, top=True):
+        # ITerminalTransport.doubleHeightLine
+        raise NotImplementedError("Unimplemented: TerminalBuffer.doubleHeightLine")
+
+    def singleWidthLine(self):
+        # ITerminalTransport.singleWidthLine
+        raise NotImplementedError("Unimplemented: TerminalBuffer.singleWidthLine")
+
+    def doubleWidthLine(self):
+        # ITerminalTransport.doubleWidthLine
+        raise NotImplementedError("Unimplemented: TerminalBuffer.doubleWidthLine")
+
+
+class ExpectationTimeout(Exception):
+    pass
+
+
+class ExpectableBuffer(TerminalBuffer):
+    _mark = 0
+
+    def connectionMade(self):
+        TerminalBuffer.connectionMade(self)
+        self._expecting = []
+
+    def write(self, data):
+        TerminalBuffer.write(self, data)
+        self._checkExpected()
+
+    def cursorHome(self):
+        TerminalBuffer.cursorHome(self)
+        self._mark = 0
+
+    def _timeoutExpected(self, d):
+        d.errback(ExpectationTimeout())
+        self._checkExpected()
+
+    def _checkExpected(self):
+        s = self.__bytes__()[self._mark :]
+        while self._expecting:
+            expr, timer, deferred = self._expecting[0]
+            if timer and not timer.active():
+                del self._expecting[0]
+                continue
+            for match in expr.finditer(s):
+                if timer:
+                    timer.cancel()
+                del self._expecting[0]
+                self._mark += match.end()
+                s = s[match.end() :]
+                deferred.callback(match)
+                break
+            else:
+                return
+
+    def expect(self, expression, timeout=None, scheduler=reactor):
+        d = defer.Deferred()
+        timer = None
+        if timeout:
+            timer = scheduler.callLater(timeout, self._timeoutExpected, d)
+        self._expecting.append((re.compile(expression), timer, d))
+        self._checkExpected()
+        return d
+
+
+__all__ = ["CharacterAttribute", "TerminalBuffer", "ExpectableBuffer"]
diff --git a/contrib/python/Twisted/py3/twisted/conch/insults/insults.py b/contrib/python/Twisted/py3/twisted/conch/insults/insults.py
new file mode 100644
index 00000000000..4640aab3689
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/insults/insults.py
@@ -0,0 +1,1223 @@
+# -*- test-case-name: twisted.conch.test.test_insults -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+VT102 and VT220 terminal manipulation.
+
+@author: Jp Calderone
+"""
+
+from zope.interface import Interface, implementer
+
+from twisted.internet import defer, interfaces as iinternet, protocol
+from twisted.python.compat import iterbytes, networkString
+
+
+class ITerminalProtocol(Interface):
+    def makeConnection(transport):
+        """
+        Called with an L{ITerminalTransport} when a connection is established.
+        """
+
+    def keystrokeReceived(keyID, modifier):
+        """
+        A keystroke was received.
+
+        Each keystroke corresponds to one invocation of this method.
+        keyID is a string identifier for that key.  Printable characters
+        are represented by themselves.  Control keys, such as arrows and
+        function keys, are represented with symbolic constants on
+        L{ServerProtocol}.
+        """
+
+    def terminalSize(width, height):
+        """
+        Called to indicate the size of the terminal.
+
+        A terminal of 80x24 should be assumed if this method is not
+        called.  This method might not be called for real terminals.
+        """
+
+    def unhandledControlSequence(seq):
+        """
+        Called when an unsupported control sequence is received.
+
+        @type seq: L{str}
+        @param seq: The whole control sequence which could not be interpreted.
+        """
+
+    def connectionLost(reason):
+        """
+        Called when the connection has been lost.
+
+        reason is a Failure describing why.
+        """
+
+
+@implementer(ITerminalProtocol)
+class TerminalProtocol:
+    def makeConnection(self, terminal):
+        # assert ITerminalTransport.providedBy(transport), "TerminalProtocol.makeConnection must be passed an ITerminalTransport implementor"
+        self.terminal = terminal
+        self.connectionMade()
+
+    def connectionMade(self):
+        """
+        Called after a connection has been established.
+        """
+
+    def keystrokeReceived(self, keyID, modifier):
+        pass
+
+    def terminalSize(self, width, height):
+        pass
+
+    def unhandledControlSequence(self, seq):
+        pass
+
+    def connectionLost(self, reason):
+        pass
+
+
+class ITerminalTransport(iinternet.ITransport):
+    def cursorUp(n=1):
+        """
+        Move the cursor up n lines.
+        """
+
+    def cursorDown(n=1):
+        """
+        Move the cursor down n lines.
+        """
+
+    def cursorForward(n=1):
+        """
+        Move the cursor right n columns.
+        """
+
+    def cursorBackward(n=1):
+        """
+        Move the cursor left n columns.
+        """
+
+    def cursorPosition(column, line):
+        """
+        Move the cursor to the given line and column.
+        """
+
+    def cursorHome():
+        """
+        Move the cursor home.
+        """
+
+    def index():
+        """
+        Move the cursor down one line, performing scrolling if necessary.
+        """
+
+    def reverseIndex():
+        """
+        Move the cursor up one line, performing scrolling if necessary.
+        """
+
+    def nextLine():
+        """
+        Move the cursor to the first position on the next line, performing scrolling if necessary.
+        """
+
+    def saveCursor():
+        """
+        Save the cursor position, character attribute, character set, and origin mode selection.
+        """
+
+    def restoreCursor():
+        """
+        Restore the previously saved cursor position, character attribute, character set, and origin mode selection.
+
+        If no cursor state was previously saved, move the cursor to the home position.
+        """
+
+    def setModes(modes):
+        """
+        Set the given modes on the terminal.
+        """
+
+    def resetModes(mode):
+        """
+        Reset the given modes on the terminal.
+        """
+
+    def setPrivateModes(modes):
+        """
+        Set the given DEC private modes on the terminal.
+        """
+
+    def resetPrivateModes(modes):
+        """
+        Reset the given DEC private modes on the terminal.
+        """
+
+    def applicationKeypadMode():
+        """
+        Cause keypad to generate control functions.
+
+        Cursor key mode selects the type of characters generated by cursor keys.
+        """
+
+    def numericKeypadMode():
+        """
+        Cause keypad to generate normal characters.
+        """
+
+    def selectCharacterSet(charSet, which):
+        """
+        Select a character set.
+
+        charSet should be one of CS_US, CS_UK, CS_DRAWING, CS_ALTERNATE, or
+        CS_ALTERNATE_SPECIAL.
+
+        which should be one of G0 or G1.
+        """
+
+    def shiftIn():
+        """
+        Activate the G0 character set.
+        """
+
+    def shiftOut():
+        """
+        Activate the G1 character set.
+        """
+
+    def singleShift2():
+        """
+        Shift to the G2 character set for a single character.
+        """
+
+    def singleShift3():
+        """
+        Shift to the G3 character set for a single character.
+        """
+
+    def selectGraphicRendition(*attributes):
+        """
+        Enabled one or more character attributes.
+
+        Arguments should be one or more of UNDERLINE, REVERSE_VIDEO, BLINK, or BOLD.
+        NORMAL may also be specified to disable all character attributes.
+        """
+
+    def horizontalTabulationSet():
+        """
+        Set a tab stop at the current cursor position.
+        """
+
+    def tabulationClear():
+        """
+        Clear the tab stop at the current cursor position.
+        """
+
+    def tabulationClearAll():
+        """
+        Clear all tab stops.
+        """
+
+    def doubleHeightLine(top=True):
+        """
+        Make the current line the top or bottom half of a double-height, double-width line.
+
+        If top is True, the current line is the top half.  Otherwise, it is the bottom half.
+        """
+
+    def singleWidthLine():
+        """
+        Make the current line a single-width, single-height line.
+        """
+
+    def doubleWidthLine():
+        """
+        Make the current line a double-width line.
+        """
+
+    def eraseToLineEnd():
+        """
+        Erase from the cursor to the end of line, including cursor position.
+        """
+
+    def eraseToLineBeginning():
+        """
+        Erase from the cursor to the beginning of the line, including the cursor position.
+        """
+
+    def eraseLine():
+        """
+        Erase the entire cursor line.
+        """
+
+    def eraseToDisplayEnd():
+        """
+        Erase from the cursor to the end of the display, including the cursor position.
+        """
+
+    def eraseToDisplayBeginning():
+        """
+        Erase from the cursor to the beginning of the display, including the cursor position.
+        """
+
+    def eraseDisplay():
+        """
+        Erase the entire display.
+        """
+
+    def deleteCharacter(n=1):
+        """
+        Delete n characters starting at the cursor position.
+
+        Characters to the right of deleted characters are shifted to the left.
+        """
+
+    def insertLine(n=1):
+        """
+        Insert n lines at the cursor position.
+
+        Lines below the cursor are shifted down.  Lines moved past the bottom margin are lost.
+        This command is ignored when the cursor is outside the scroll region.
+        """
+
+    def deleteLine(n=1):
+        """
+        Delete n lines starting at the cursor position.
+
+        Lines below the cursor are shifted up.  This command is ignored when the cursor is outside
+        the scroll region.
+        """
+
+    def reportCursorPosition():
+        """
+        Return a Deferred that fires with a two-tuple of (x, y) indicating the cursor position.
+        """
+
+    def reset():
+        """
+        Reset the terminal to its initial state.
+        """
+
+    def unhandledControlSequence(seq):
+        """
+        Called when an unsupported control sequence is received.
+
+        @type seq: L{str}
+        @param seq: The whole control sequence which could not be interpreted.
+        """
+
+
+CSI = b"\x1b"
+CST = {b"~": b"tilde"}
+
+
+class modes:
+    """
+    ECMA 48 standardized modes
+    """
+
+    # BREAKS YOPUR KEYBOARD MOFO
+    KEYBOARD_ACTION = KAM = 2
+
+    # When set, enables character insertion. New display characters
+    # move old display characters to the right. Characters moved past
+    # the right margin are lost.
+
+    # When reset, enables replacement mode (disables character
+    # insertion). New display characters replace old display
+    # characters at cursor position. The old character is erased.
+    INSERTION_REPLACEMENT = IRM = 4
+
+    # Set causes a received linefeed, form feed, or vertical tab to
+    # move cursor to first column of next line. RETURN transmits both
+    # a carriage return and linefeed. This selection is also called
+    # new line option.
+
+    # Reset causes a received linefeed, form feed, or vertical tab to
+    # move cursor to next line in current column. RETURN transmits a
+    # carriage return.
+    LINEFEED_NEWLINE = LNM = 20
+
+
+class privateModes:
+    """
+    ANSI-Compatible Private Modes
+    """
+
+    ERROR = 0
+    CURSOR_KEY = 1
+    ANSI_VT52 = 2
+    COLUMN = 3
+    SCROLL = 4
+    SCREEN = 5
+    ORIGIN = 6
+    AUTO_WRAP = 7
+    AUTO_REPEAT = 8
+    PRINTER_FORM_FEED = 18
+    PRINTER_EXTENT = 19
+
+    # Toggle cursor visibility (reset hides it)
+    CURSOR_MODE = 25
+
+
+# Character sets
+CS_US = b"CS_US"
+CS_UK = b"CS_UK"
+CS_DRAWING = b"CS_DRAWING"
+CS_ALTERNATE = b"CS_ALTERNATE"
+CS_ALTERNATE_SPECIAL = b"CS_ALTERNATE_SPECIAL"
+
+# Groupings (or something?? These are like variables that can be bound to character sets)
+G0 = b"G0"
+G1 = b"G1"
+
+# G2 and G3 cannot be changed, but they can be shifted to.
+G2 = b"G2"
+G3 = b"G3"
+
+# Character attributes
+
+NORMAL = 0
+BOLD = 1
+UNDERLINE = 4
+BLINK = 5
+REVERSE_VIDEO = 7
+
+
+class Vector:
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+
+def log(s):
+    with open("log", "a") as f:
+        f.write(str(s) + "\n")
+
+
+# XXX TODO - These attributes are really part of the
+# ITerminalTransport interface, I think.
+_KEY_NAMES = (
+    "UP_ARROW",
+    "DOWN_ARROW",
+    "RIGHT_ARROW",
+    "LEFT_ARROW",
+    "HOME",
+    "INSERT",
+    "DELETE",
+    "END",
+    "PGUP",
+    "PGDN",
+    "NUMPAD_MIDDLE",
+    "F1",
+    "F2",
+    "F3",
+    "F4",
+    "F5",
+    "F6",
+    "F7",
+    "F8",
+    "F9",
+    "F10",
+    "F11",
+    "F12",
+    "ALT",
+    "SHIFT",
+    "CONTROL",
+)
+
+
+class _const:
+    """
+    @ivar name: A string naming this constant
+    """
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+    def __repr__(self) -> str:
+        return "[" + self.name + "]"
+
+    def __bytes__(self) -> bytes:
+        return ("[" + self.name + "]").encode("ascii")
+
+
+FUNCTION_KEYS = [_const(_name).__bytes__() for _name in _KEY_NAMES]
+
+
+@implementer(ITerminalTransport)
+class ServerProtocol(protocol.Protocol):
+    protocolFactory = None
+    terminalProtocol = None
+
+    TAB = b"\t"
+    BACKSPACE = b"\x7f"
+    ##
+
+    lastWrite = b""
+
+    state = b"data"
+
+    termSize = Vector(80, 24)
+    cursorPos = Vector(0, 0)
+    scrollRegion = None
+
+    # Factory who instantiated me
+    factory = None
+
+    def __init__(self, protocolFactory=None, *a, **kw):
+        """
+        @param protocolFactory: A callable which will be invoked with
+        *a, **kw and should return an ITerminalProtocol implementor.
+        This will be invoked when a connection to this ServerProtocol
+        is established.
+
+        @param a: Any positional arguments to pass to protocolFactory.
+        @param kw: Any keyword arguments to pass to protocolFactory.
+        """
+        # assert protocolFactory is None or ITerminalProtocol.implementedBy(protocolFactory), "ServerProtocol.__init__ must be passed an ITerminalProtocol implementor"
+        if protocolFactory is not None:
+            self.protocolFactory = protocolFactory
+        self.protocolArgs = a
+        self.protocolKwArgs = kw
+
+        self._cursorReports = []
+
+    def getHost(self):
+        # ITransport.getHost
+        raise NotImplementedError("Unimplemented: ServerProtocol.getHost")
+
+    def getPeer(self):
+        # ITransport.getPeer
+        raise NotImplementedError("Unimplemented: ServerProtocol.getPeer")
+
+    def connectionMade(self):
+        if self.protocolFactory is not None:
+            self.terminalProtocol = self.protocolFactory(
+                *self.protocolArgs, **self.protocolKwArgs
+            )
+
+            try:
+                factory = self.factory
+            except AttributeError:
+                pass
+            else:
+                self.terminalProtocol.factory = factory
+
+            self.terminalProtocol.makeConnection(self)
+
+    def dataReceived(self, data):
+        for ch in iterbytes(data):
+            if self.state == b"data":
+                if ch == b"\x1b":
+                    self.state = b"escaped"
+                else:
+                    self.terminalProtocol.keystrokeReceived(ch, None)
+            elif self.state == b"escaped":
+                if ch == b"[":
+                    self.state = b"bracket-escaped"
+                    self.escBuf = []
+                elif ch == b"O":
+                    self.state = b"low-function-escaped"
+                else:
+                    self.state = b"data"
+                    self._handleShortControlSequence(ch)
+            elif self.state == b"bracket-escaped":
+                if ch == b"O":
+                    self.state = b"low-function-escaped"
+                elif ch.isalpha() or ch == b"~":
+                    self._handleControlSequence(b"".join(self.escBuf) + ch)
+                    del self.escBuf
+                    self.state = b"data"
+                else:
+                    self.escBuf.append(ch)
+            elif self.state == b"low-function-escaped":
+                self._handleLowFunctionControlSequence(ch)
+                self.state = b"data"
+            else:
+                raise ValueError("Illegal state")
+
+    def _handleShortControlSequence(self, ch):
+        self.terminalProtocol.keystrokeReceived(ch, self.ALT)
+
+    def _handleControlSequence(self, buf):
+        buf = b"\x1b[" + buf
+        f = getattr(
+            self.controlSequenceParser,
+            CST.get(buf[-1:], buf[-1:]).decode("ascii"),
+            None,
+        )
+        if f is None:
+            self.unhandledControlSequence(buf)
+        else:
+            f(self, self.terminalProtocol, buf[:-1])
+
+    def unhandledControlSequence(self, buf):
+        self.terminalProtocol.unhandledControlSequence(buf)
+
+    def _handleLowFunctionControlSequence(self, ch):
+        functionKeys = {b"P": self.F1, b"Q": self.F2, b"R": self.F3, b"S": self.F4}
+        keyID = functionKeys.get(ch)
+        if keyID is not None:
+            self.terminalProtocol.keystrokeReceived(keyID, None)
+        else:
+            self.terminalProtocol.unhandledControlSequence(b"\x1b[O" + ch)
+
+    class ControlSequenceParser:
+        def A(self, proto, handler, buf):
+            if buf == b"\x1b[":
+                handler.keystrokeReceived(proto.UP_ARROW, None)
+            else:
+                handler.unhandledControlSequence(buf + b"A")
+
+        def B(self, proto, handler, buf):
+            if buf == b"\x1b[":
+                handler.keystrokeReceived(proto.DOWN_ARROW, None)
+            else:
+                handler.unhandledControlSequence(buf + b"B")
+
+        def C(self, proto, handler, buf):
+            if buf == b"\x1b[":
+                handler.keystrokeReceived(proto.RIGHT_ARROW, None)
+            else:
+                handler.unhandledControlSequence(buf + b"C")
+
+        def D(self, proto, handler, buf):
+            if buf == b"\x1b[":
+                handler.keystrokeReceived(proto.LEFT_ARROW, None)
+            else:
+                handler.unhandledControlSequence(buf + b"D")
+
+        def E(self, proto, handler, buf):
+            if buf == b"\x1b[":
+                handler.keystrokeReceived(proto.NUMPAD_MIDDLE, None)
+            else:
+                handler.unhandledControlSequence(buf + b"E")
+
+        def F(self, proto, handler, buf):
+            if buf == b"\x1b[":
+                handler.keystrokeReceived(proto.END, None)
+            else:
+                handler.unhandledControlSequence(buf + b"F")
+
+        def H(self, proto, handler, buf):
+            if buf == b"\x1b[":
+                handler.keystrokeReceived(proto.HOME, None)
+            else:
+                handler.unhandledControlSequence(buf + b"H")
+
+        def R(self, proto, handler, buf):
+            if not proto._cursorReports:
+                handler.unhandledControlSequence(buf + b"R")
+            elif buf.startswith(b"\x1b["):
+                report = buf[2:]
+                parts = report.split(b";")
+                if len(parts) != 2:
+                    handler.unhandledControlSequence(buf + b"R")
+                else:
+                    Pl, Pc = parts
+                    try:
+                        Pl, Pc = int(Pl), int(Pc)
+                    except ValueError:
+                        handler.unhandledControlSequence(buf + b"R")
+                    else:
+                        d = proto._cursorReports.pop(0)
+                        d.callback((Pc - 1, Pl - 1))
+            else:
+                handler.unhandledControlSequence(buf + b"R")
+
+        def Z(self, proto, handler, buf):
+            if buf == b"\x1b[":
+                handler.keystrokeReceived(proto.TAB, proto.SHIFT)
+            else:
+                handler.unhandledControlSequence(buf + b"Z")
+
+        def tilde(self, proto, handler, buf):
+            map = {
+                1: proto.HOME,
+                2: proto.INSERT,
+                3: proto.DELETE,
+                4: proto.END,
+                5: proto.PGUP,
+                6: proto.PGDN,
+                15: proto.F5,
+                17: proto.F6,
+                18: proto.F7,
+                19: proto.F8,
+                20: proto.F9,
+                21: proto.F10,
+                23: proto.F11,
+                24: proto.F12,
+            }
+
+            if buf.startswith(b"\x1b["):
+                ch = buf[2:]
+                try:
+                    v = int(ch)
+                except ValueError:
+                    handler.unhandledControlSequence(buf + b"~")
+                else:
+                    symbolic = map.get(v)
+                    if symbolic is not None:
+                        handler.keystrokeReceived(map[v], None)
+                    else:
+                        handler.unhandledControlSequence(buf + b"~")
+            else:
+                handler.unhandledControlSequence(buf + b"~")
+
+    controlSequenceParser = ControlSequenceParser()
+
+    # ITerminalTransport
+    def cursorUp(self, n=1):
+        assert n >= 1
+        self.cursorPos.y = max(self.cursorPos.y - n, 0)
+        self.write(b"\x1b[%dA" % (n,))
+
+    def cursorDown(self, n=1):
+        assert n >= 1
+        self.cursorPos.y = min(self.cursorPos.y + n, self.termSize.y - 1)
+        self.write(b"\x1b[%dB" % (n,))
+
+    def cursorForward(self, n=1):
+        assert n >= 1
+        self.cursorPos.x = min(self.cursorPos.x + n, self.termSize.x - 1)
+        self.write(b"\x1b[%dC" % (n,))
+
+    def cursorBackward(self, n=1):
+        assert n >= 1
+        self.cursorPos.x = max(self.cursorPos.x - n, 0)
+        self.write(b"\x1b[%dD" % (n,))
+
+    def cursorPosition(self, column, line):
+        self.write(b"\x1b[%d;%dH" % (line + 1, column + 1))
+
+    def cursorHome(self):
+        self.cursorPos.x = self.cursorPos.y = 0
+        self.write(b"\x1b[H")
+
+    def index(self):
+        # ECMA48 5th Edition removes this
+        self.cursorPos.y = min(self.cursorPos.y + 1, self.termSize.y - 1)
+        self.write(b"\x1bD")
+
+    def reverseIndex(self):
+        self.cursorPos.y = max(self.cursorPos.y - 1, 0)
+        self.write(b"\x1bM")
+
+    def nextLine(self):
+        self.cursorPos.x = 0
+        self.cursorPos.y = min(self.cursorPos.y + 1, self.termSize.y - 1)
+        self.write(b"\n")
+
+    def saveCursor(self):
+        self._savedCursorPos = Vector(self.cursorPos.x, self.cursorPos.y)
+        self.write(b"\x1b7")
+
+    def restoreCursor(self):
+        self.cursorPos = self._savedCursorPos
+        del self._savedCursorPos
+        self.write(b"\x1b8")
+
+    def setModes(self, modes):
+        # XXX Support ANSI-Compatible private modes
+        modesBytes = b";".join(b"%d" % (mode,) for mode in modes)
+        self.write(b"\x1b[" + modesBytes + b"h")
+
+    def setPrivateModes(self, modes):
+        modesBytes = b";".join(b"%d" % (mode,) for mode in modes)
+        self.write(b"\x1b[?" + modesBytes + b"h")
+
+    def resetModes(self, modes):
+        # XXX Support ANSI-Compatible private modes
+        modesBytes = b";".join(b"%d" % (mode,) for mode in modes)
+        self.write(b"\x1b[" + modesBytes + b"l")
+
+    def resetPrivateModes(self, modes):
+        modesBytes = b";".join(b"%d" % (mode,) for mode in modes)
+        self.write(b"\x1b[?" + modesBytes + b"l")
+
+    def applicationKeypadMode(self):
+        self.write(b"\x1b=")
+
+    def numericKeypadMode(self):
+        self.write(b"\x1b>")
+
+    def selectCharacterSet(self, charSet, which):
+        # XXX Rewrite these as dict lookups
+        if which == G0:
+            which = b"("
+        elif which == G1:
+            which = b")"
+        else:
+            raise ValueError("`which' argument to selectCharacterSet must be G0 or G1")
+        if charSet == CS_UK:
+            charSet = b"A"
+        elif charSet == CS_US:
+            charSet = b"B"
+        elif charSet == CS_DRAWING:
+            charSet = b"0"
+        elif charSet == CS_ALTERNATE:
+            charSet = b"1"
+        elif charSet == CS_ALTERNATE_SPECIAL:
+            charSet = b"2"
+        else:
+            raise ValueError("Invalid `charSet' argument to selectCharacterSet")
+        self.write(b"\x1b" + which + charSet)
+
+    def shiftIn(self):
+        self.write(b"\x15")
+
+    def shiftOut(self):
+        self.write(b"\x14")
+
+    def singleShift2(self):
+        self.write(b"\x1bN")
+
+    def singleShift3(self):
+        self.write(b"\x1bO")
+
+    def selectGraphicRendition(self, *attributes):
+        # each member of attributes must be a native string
+        attrs = []
+        for a in attributes:
+            attrs.append(networkString(a))
+        self.write(b"\x1b[" + b";".join(attrs) + b"m")
+
+    def horizontalTabulationSet(self):
+        self.write(b"\x1bH")
+
+    def tabulationClear(self):
+        self.write(b"\x1b[q")
+
+    def tabulationClearAll(self):
+        self.write(b"\x1b[3q")
+
+    def doubleHeightLine(self, top=True):
+        if top:
+            self.write(b"\x1b#3")
+        else:
+            self.write(b"\x1b#4")
+
+    def singleWidthLine(self):
+        self.write(b"\x1b#5")
+
+    def doubleWidthLine(self):
+        self.write(b"\x1b#6")
+
+    def eraseToLineEnd(self):
+        self.write(b"\x1b[K")
+
+    def eraseToLineBeginning(self):
+        self.write(b"\x1b[1K")
+
+    def eraseLine(self):
+        self.write(b"\x1b[2K")
+
+    def eraseToDisplayEnd(self):
+        self.write(b"\x1b[J")
+
+    def eraseToDisplayBeginning(self):
+        self.write(b"\x1b[1J")
+
+    def eraseDisplay(self):
+        self.write(b"\x1b[2J")
+
+    def deleteCharacter(self, n=1):
+        self.write(b"\x1b[%dP" % (n,))
+
+    def insertLine(self, n=1):
+        self.write(b"\x1b[%dL" % (n,))
+
+    def deleteLine(self, n=1):
+        self.write(b"\x1b[%dM" % (n,))
+
+    def setScrollRegion(self, first=None, last=None):
+        if first is not None:
+            first = b"%d" % (first,)
+        else:
+            first = b""
+        if last is not None:
+            last = b"%d" % (last,)
+        else:
+            last = b""
+        self.write(b"\x1b[%b;%br" % (first, last))
+
+    def resetScrollRegion(self):
+        self.setScrollRegion()
+
+    def reportCursorPosition(self):
+        d = defer.Deferred()
+        self._cursorReports.append(d)
+        self.write(b"\x1b[6n")
+        return d
+
+    def reset(self):
+        self.cursorPos.x = self.cursorPos.y = 0
+        try:
+            del self._savedCursorPos
+        except AttributeError:
+            pass
+        self.write(b"\x1bc")
+
+    # ITransport
+    def write(self, data):
+        if data:
+            if not isinstance(data, bytes):
+                data = data.encode("utf-8")
+            self.lastWrite = data
+            self.transport.write(b"\r\n".join(data.split(b"\n")))
+
+    def writeSequence(self, data):
+        self.write(b"".join(data))
+
+    def loseConnection(self):
+        self.reset()
+        self.transport.loseConnection()
+
+    def connectionLost(self, reason):
+        if self.terminalProtocol is not None:
+            try:
+                self.terminalProtocol.connectionLost(reason)
+            finally:
+                self.terminalProtocol = None
+
+
+# Add symbolic names for function keys
+for name, const in zip(_KEY_NAMES, FUNCTION_KEYS):
+    setattr(ServerProtocol, name, const)
+
+
+class ClientProtocol(protocol.Protocol):
+    terminalFactory = None
+    terminal = None
+
+    state = b"data"
+
+    _escBuf = None
+
+    _shorts = {
+        b"D": b"index",
+        b"M": b"reverseIndex",
+        b"E": b"nextLine",
+        b"7": b"saveCursor",
+        b"8": b"restoreCursor",
+        b"=": b"applicationKeypadMode",
+        b">": b"numericKeypadMode",
+        b"N": b"singleShift2",
+        b"O": b"singleShift3",
+        b"H": b"horizontalTabulationSet",
+        b"c": b"reset",
+    }
+
+    _longs = {
+        b"[": b"bracket-escape",
+        b"(": b"select-g0",
+        b")": b"select-g1",
+        b"#": b"select-height-width",
+    }
+
+    _charsets = {
+        b"A": CS_UK,
+        b"B": CS_US,
+        b"0": CS_DRAWING,
+        b"1": CS_ALTERNATE,
+        b"2": CS_ALTERNATE_SPECIAL,
+    }
+
+    # Factory who instantiated me
+    factory = None
+
+    def __init__(self, terminalFactory=None, *a, **kw):
+        """
+        @param terminalFactory: A callable which will be invoked with
+        *a, **kw and should return an ITerminalTransport provider.
+        This will be invoked when this ClientProtocol establishes a
+        connection.
+
+        @param a: Any positional arguments to pass to terminalFactory.
+        @param kw: Any keyword arguments to pass to terminalFactory.
+        """
+        # assert terminalFactory is None or ITerminalTransport.implementedBy(terminalFactory), "ClientProtocol.__init__ must be passed an ITerminalTransport implementor"
+        if terminalFactory is not None:
+            self.terminalFactory = terminalFactory
+        self.terminalArgs = a
+        self.terminalKwArgs = kw
+
+    def connectionMade(self):
+        if self.terminalFactory is not None:
+            self.terminal = self.terminalFactory(
+                *self.terminalArgs, **self.terminalKwArgs
+            )
+            self.terminal.factory = self.factory
+            self.terminal.makeConnection(self)
+
+    def connectionLost(self, reason):
+        if self.terminal is not None:
+            try:
+                self.terminal.connectionLost(reason)
+            finally:
+                del self.terminal
+
+    def dataReceived(self, data):
+        """
+        Parse the given data from a terminal server, dispatching to event
+        handlers defined by C{self.terminal}.
+        """
+        toWrite = []
+        for b in iterbytes(data):
+            if self.state == b"data":
+                if b == b"\x1b":
+                    if toWrite:
+                        self.terminal.write(b"".join(toWrite))
+                        del toWrite[:]
+                    self.state = b"escaped"
+                elif b == b"\x14":
+                    if toWrite:
+                        self.terminal.write(b"".join(toWrite))
+                        del toWrite[:]
+                    self.terminal.shiftOut()
+                elif b == b"\x15":
+                    if toWrite:
+                        self.terminal.write(b"".join(toWrite))
+                        del toWrite[:]
+                    self.terminal.shiftIn()
+                elif b == b"\x08":
+                    if toWrite:
+                        self.terminal.write(b"".join(toWrite))
+                        del toWrite[:]
+                    self.terminal.cursorBackward()
+                else:
+                    toWrite.append(b)
+            elif self.state == b"escaped":
+                fName = self._shorts.get(b)
+                if fName is not None:
+                    self.state = b"data"
+                    getattr(self.terminal, fName.decode("ascii"))()
+                else:
+                    state = self._longs.get(b)
+                    if state is not None:
+                        self.state = state
+                    else:
+                        self.terminal.unhandledControlSequence(b"\x1b" + b)
+                        self.state = b"data"
+            elif self.state == b"bracket-escape":
+                if self._escBuf is None:
+                    self._escBuf = []
+                if b.isalpha() or b == b"~":
+                    self._handleControlSequence(b"".join(self._escBuf), b)
+                    del self._escBuf
+                    self.state = b"data"
+                else:
+                    self._escBuf.append(b)
+            elif self.state == b"select-g0":
+                self.terminal.selectCharacterSet(self._charsets.get(b, b), G0)
+                self.state = b"data"
+            elif self.state == b"select-g1":
+                self.terminal.selectCharacterSet(self._charsets.get(b, b), G1)
+                self.state = b"data"
+            elif self.state == b"select-height-width":
+                self._handleHeightWidth(b)
+                self.state = b"data"
+            else:
+                raise ValueError("Illegal state")
+        if toWrite:
+            self.terminal.write(b"".join(toWrite))
+
+    def _handleControlSequence(self, buf, terminal):
+        f = getattr(
+            self.controlSequenceParser,
+            CST.get(terminal, terminal).decode("ascii"),
+            None,
+        )
+        if f is None:
+            self.terminal.unhandledControlSequence(b"\x1b[" + buf + terminal)
+        else:
+            f(self, self.terminal, buf)
+
+    class ControlSequenceParser:
+        def _makeSimple(ch, fName):
+            n = "cursor" + fName
+
+            def simple(self, proto, handler, buf):
+                if not buf:
+                    getattr(handler, n)(1)
+                else:
+                    try:
+                        m = int(buf)
+                    except ValueError:
+                        handler.unhandledControlSequence(b"\x1b[" + buf + ch)
+                    else:
+                        getattr(handler, n)(m)
+
+            return simple
+
+        for ch, fName in (
+            ("A", "Up"),
+            ("B", "Down"),
+            ("C", "Forward"),
+            ("D", "Backward"),
+        ):
+            exec(ch + " = _makeSimple(ch, fName)")
+        del _makeSimple
+
+        def h(self, proto, handler, buf):
+            # XXX - Handle '?' to introduce ANSI-Compatible private modes.
+            try:
+                modes = [int(mode) for mode in buf.split(b";")]
+            except ValueError:
+                handler.unhandledControlSequence(b"\x1b[" + buf + b"h")
+            else:
+                handler.setModes(modes)
+
+        def l(self, proto, handler, buf):
+            # XXX - Handle '?' to introduce ANSI-Compatible private modes.
+            try:
+                modes = [int(mode) for mode in buf.split(b";")]
+            except ValueError:
+                handler.unhandledControlSequence(b"\x1b[" + buf + "l")
+            else:
+                handler.resetModes(modes)
+
+        def r(self, proto, handler, buf):
+            parts = buf.split(b";")
+            if len(parts) == 1:
+                handler.setScrollRegion(None, None)
+            elif len(parts) == 2:
+                try:
+                    if parts[0]:
+                        pt = int(parts[0])
+                    else:
+                        pt = None
+                    if parts[1]:
+                        pb = int(parts[1])
+                    else:
+                        pb = None
+                except ValueError:
+                    handler.unhandledControlSequence(b"\x1b[" + buf + b"r")
+                else:
+                    handler.setScrollRegion(pt, pb)
+            else:
+                handler.unhandledControlSequence(b"\x1b[" + buf + b"r")
+
+        def K(self, proto, handler, buf):
+            if not buf:
+                handler.eraseToLineEnd()
+            elif buf == b"1":
+                handler.eraseToLineBeginning()
+            elif buf == b"2":
+                handler.eraseLine()
+            else:
+                handler.unhandledControlSequence(b"\x1b[" + buf + b"K")
+
+        def H(self, proto, handler, buf):
+            handler.cursorHome()
+
+        def J(self, proto, handler, buf):
+            if not buf:
+                handler.eraseToDisplayEnd()
+            elif buf == b"1":
+                handler.eraseToDisplayBeginning()
+            elif buf == b"2":
+                handler.eraseDisplay()
+            else:
+                handler.unhandledControlSequence(b"\x1b[" + buf + b"J")
+
+        def P(self, proto, handler, buf):
+            if not buf:
+                handler.deleteCharacter(1)
+            else:
+                try:
+                    n = int(buf)
+                except ValueError:
+                    handler.unhandledControlSequence(b"\x1b[" + buf + b"P")
+                else:
+                    handler.deleteCharacter(n)
+
+        def L(self, proto, handler, buf):
+            if not buf:
+                handler.insertLine(1)
+            else:
+                try:
+                    n = int(buf)
+                except ValueError:
+                    handler.unhandledControlSequence(b"\x1b[" + buf + b"L")
+                else:
+                    handler.insertLine(n)
+
+        def M(self, proto, handler, buf):
+            if not buf:
+                handler.deleteLine(1)
+            else:
+                try:
+                    n = int(buf)
+                except ValueError:
+                    handler.unhandledControlSequence(b"\x1b[" + buf + b"M")
+                else:
+                    handler.deleteLine(n)
+
+        def n(self, proto, handler, buf):
+            if buf == b"6":
+                x, y = handler.reportCursorPosition()
+                proto.transport.write(b"\x1b[%d;%dR" % (x + 1, y + 1))
+            else:
+                handler.unhandledControlSequence(b"\x1b[" + buf + b"n")
+
+        def m(self, proto, handler, buf):
+            if not buf:
+                handler.selectGraphicRendition(NORMAL)
+            else:
+                attrs = []
+                for a in buf.split(b";"):
+                    try:
+                        a = int(a)
+                    except ValueError:
+                        pass
+                    attrs.append(a)
+                handler.selectGraphicRendition(*attrs)
+
+    controlSequenceParser = ControlSequenceParser()
+
+    def _handleHeightWidth(self, b):
+        if b == b"3":
+            self.terminal.doubleHeightLine(True)
+        elif b == b"4":
+            self.terminal.doubleHeightLine(False)
+        elif b == b"5":
+            self.terminal.singleWidthLine()
+        elif b == b"6":
+            self.terminal.doubleWidthLine()
+        else:
+            self.terminal.unhandledControlSequence(b"\x1b#" + b)
+
+
+__all__ = [
+    # Interfaces
+    "ITerminalProtocol",
+    "ITerminalTransport",
+    # Symbolic constants
+    "modes",
+    "privateModes",
+    "FUNCTION_KEYS",
+    "CS_US",
+    "CS_UK",
+    "CS_DRAWING",
+    "CS_ALTERNATE",
+    "CS_ALTERNATE_SPECIAL",
+    "G0",
+    "G1",
+    "G2",
+    "G3",
+    "UNDERLINE",
+    "REVERSE_VIDEO",
+    "BLINK",
+    "BOLD",
+    "NORMAL",
+    # Protocol classes
+    "ServerProtocol",
+    "ClientProtocol",
+]
diff --git a/contrib/python/Twisted/py3/twisted/conch/insults/text.py b/contrib/python/Twisted/py3/twisted/conch/insults/text.py
new file mode 100644
index 00000000000..37a736f4f60
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/insults/text.py
@@ -0,0 +1,176 @@
+# -*- test-case-name: twisted.conch.test.test_text -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Character attribute manipulation API.
+
+This module provides a domain-specific language (using Python syntax)
+for the creation of text with additional display attributes associated
+with it.  It is intended as an alternative to manually building up
+strings containing ECMA 48 character attribute control codes.  It
+currently supports foreground and background colors (black, red,
+green, yellow, blue, magenta, cyan, and white), intensity selection,
+underlining, blinking and reverse video.  Character set selection
+support is planned.
+
+Character attributes are specified by using two Python operations:
+attribute lookup and indexing.  For example, the string \"Hello
+world\" with red foreground and all other attributes set to their
+defaults, assuming the name twisted.conch.insults.text.attributes has
+been imported and bound to the name \"A\" (with the statement C{from
+twisted.conch.insults.text import attributes as A}, for example) one
+uses this expression::
+
+    A.fg.red[\"Hello world\"]
+
+Other foreground colors are set by substituting their name for
+\"red\".  To set both a foreground and a background color, this
+expression is used::
+
+    A.fg.red[A.bg.green[\"Hello world\"]]
+
+Note that either A.bg.green can be nested within A.fg.red or vice
+versa.  Also note that multiple items can be nested within a single
+index operation by separating them with commas::
+
+    A.bg.green[A.fg.red[\"Hello\"], " ", A.fg.blue[\"world\"]]
+
+Other character attributes are set in a similar fashion.  To specify a
+blinking version of the previous expression::
+
+    A.blink[A.bg.green[A.fg.red[\"Hello\"], " ", A.fg.blue[\"world\"]]]
+
+C{A.reverseVideo}, C{A.underline}, and C{A.bold} are also valid.
+
+A third operation is actually supported: unary negation.  This turns
+off an attribute when an enclosing expression would otherwise have
+caused it to be on.  For example::
+
+    A.underline[A.fg.red[\"Hello\", -A.underline[\" world\"]]]
+
+A formatting structure can then be serialized into a string containing the
+necessary VT102 control codes with L{assembleFormattedText}.
+
+@see: L{twisted.conch.insults.text._CharacterAttributes}
+@author: Jp Calderone
+"""
+
+from incremental import Version
+
+from twisted.conch.insults import helper, insults
+from twisted.python import _textattributes
+from twisted.python.deprecate import deprecatedModuleAttribute
+
+flatten = _textattributes.flatten
+
+deprecatedModuleAttribute(
+    Version("Twisted", 13, 1, 0),
+    "Use twisted.conch.insults.text.assembleFormattedText instead.",
+    "twisted.conch.insults.text",
+    "flatten",
+)
+
+_TEXT_COLORS = {
+    "black": helper.BLACK,
+    "red": helper.RED,
+    "green": helper.GREEN,
+    "yellow": helper.YELLOW,
+    "blue": helper.BLUE,
+    "magenta": helper.MAGENTA,
+    "cyan": helper.CYAN,
+    "white": helper.WHITE,
+}
+
+
+class _CharacterAttributes(_textattributes.CharacterAttributesMixin):
+    """
+    Factory for character attributes, including foreground and background color
+    and non-color attributes such as bold, reverse video and underline.
+
+    Character attributes are applied to actual text by using object
+    indexing-syntax (C{obj['abc']}) after accessing a factory attribute, for
+    example::
+
+        attributes.bold['Some text']
+
+    These can be nested to mix attributes::
+
+        attributes.bold[attributes.underline['Some text']]
+
+    And multiple values can be passed::
+
+        attributes.normal[attributes.bold['Some'], ' text']
+
+    Non-color attributes can be accessed by attribute name, available
+    attributes are:
+
+        - bold
+        - blink
+        - reverseVideo
+        - underline
+
+    Available colors are:
+
+        0. black
+        1. red
+        2. green
+        3. yellow
+        4. blue
+        5. magenta
+        6. cyan
+        7. white
+
+    @ivar fg: Foreground colors accessed by attribute name, see above
+        for possible names.
+
+    @ivar bg: Background colors accessed by attribute name, see above
+        for possible names.
+    """
+
+    fg = _textattributes._ColorAttribute(
+        _textattributes._ForegroundColorAttr, _TEXT_COLORS
+    )
+    bg = _textattributes._ColorAttribute(
+        _textattributes._BackgroundColorAttr, _TEXT_COLORS
+    )
+
+    attrs = {
+        "bold": insults.BOLD,
+        "blink": insults.BLINK,
+        "underline": insults.UNDERLINE,
+        "reverseVideo": insults.REVERSE_VIDEO,
+    }
+
+
+def assembleFormattedText(formatted):
+    """
+    Assemble formatted text from structured information.
+
+    Currently handled formatting includes: bold, blink, reverse, underline and
+    color codes.
+
+    For example::
+
+        from twisted.conch.insults.text import attributes as A
+        assembleFormattedText(
+            A.normal[A.bold['Time: '], A.fg.lightRed['Now!']])
+
+    Would produce "Time: " in bold formatting, followed by "Now!" with a
+    foreground color of light red and without any additional formatting.
+
+    @param formatted: Structured text and attributes.
+
+    @rtype: L{str}
+    @return: String containing VT102 control sequences that mimic those
+        specified by C{formatted}.
+
+    @see: L{twisted.conch.insults.text._CharacterAttributes}
+    @since: 13.1
+    """
+    return _textattributes.flatten(formatted, helper._FormattingState(), "toVT102")
+
+
+attributes = _CharacterAttributes()
+
+__all__ = ["attributes", "flatten"]
diff --git a/contrib/python/Twisted/py3/twisted/conch/insults/window.py b/contrib/python/Twisted/py3/twisted/conch/insults/window.py
new file mode 100644
index 00000000000..c93fae7b21b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/insults/window.py
@@ -0,0 +1,928 @@
+# -*- test-case-name: twisted.conch.test.test_window -*-
+
+"""
+Simple insults-based widget library
+
+@author: Jp Calderone
+"""
+
+import array
+
+from twisted.conch.insults import helper, insults
+from twisted.python import text as tptext
+
+
+class YieldFocus(Exception):
+    """
+    Input focus manipulation exception
+    """
+
+
+class BoundedTerminalWrapper:
+    def __init__(self, terminal, width, height, xoff, yoff):
+        self.width = width
+        self.height = height
+        self.xoff = xoff
+        self.yoff = yoff
+        self.terminal = terminal
+        self.cursorForward = terminal.cursorForward
+        self.selectCharacterSet = terminal.selectCharacterSet
+        self.selectGraphicRendition = terminal.selectGraphicRendition
+        self.saveCursor = terminal.saveCursor
+        self.restoreCursor = terminal.restoreCursor
+
+    def cursorPosition(self, x, y):
+        return self.terminal.cursorPosition(
+            self.xoff + min(self.width, x), self.yoff + min(self.height, y)
+        )
+
+    def cursorHome(self):
+        return self.terminal.cursorPosition(self.xoff, self.yoff)
+
+    def write(self, data):
+        return self.terminal.write(data)
+
+
+class Widget:
+    focused = False
+    parent = None
+    dirty = False
+    width = height = None
+
+    def repaint(self):
+        if not self.dirty:
+            self.dirty = True
+        if self.parent is not None and not self.parent.dirty:
+            self.parent.repaint()
+
+    def filthy(self):
+        self.dirty = True
+
+    def redraw(self, width, height, terminal):
+        self.filthy()
+        self.draw(width, height, terminal)
+
+    def draw(self, width, height, terminal):
+        if width != self.width or height != self.height or self.dirty:
+            self.width = width
+            self.height = height
+            self.dirty = False
+            self.render(width, height, terminal)
+
+    def render(self, width, height, terminal):
+        pass
+
+    def sizeHint(self):
+        return None
+
+    def keystrokeReceived(self, keyID, modifier):
+        if keyID == b"\t":
+            self.tabReceived(modifier)
+        elif keyID == b"\x7f":
+            self.backspaceReceived()
+        elif keyID in insults.FUNCTION_KEYS:
+            self.functionKeyReceived(keyID, modifier)
+        else:
+            self.characterReceived(keyID, modifier)
+
+    def tabReceived(self, modifier):
+        # XXX TODO - Handle shift+tab
+        raise YieldFocus()
+
+    def focusReceived(self):
+        """
+        Called when focus is being given to this widget.
+
+        May raise YieldFocus is this widget does not want focus.
+        """
+        self.focused = True
+        self.repaint()
+
+    def focusLost(self):
+        self.focused = False
+        self.repaint()
+
+    def backspaceReceived(self):
+        pass
+
+    def functionKeyReceived(self, keyID, modifier):
+        name = keyID
+        if not isinstance(keyID, str):
+            name = name.decode("utf-8")
+        func = getattr(self, "func_" + name, None)
+        if func is not None:
+            func(modifier)
+
+    def characterReceived(self, keyID, modifier):
+        pass
+
+
+class ContainerWidget(Widget):
+    """
+    @ivar focusedChild: The contained widget which currently has
+    focus, or None.
+    """
+
+    focusedChild = None
+    focused = False
+
+    def __init__(self):
+        Widget.__init__(self)
+        self.children = []
+
+    def addChild(self, child):
+        assert child.parent is None
+        child.parent = self
+        self.children.append(child)
+        if self.focusedChild is None and self.focused:
+            try:
+                child.focusReceived()
+            except YieldFocus:
+                pass
+            else:
+                self.focusedChild = child
+        self.repaint()
+
+    def remChild(self, child):
+        assert child.parent is self
+        child.parent = None
+        self.children.remove(child)
+        self.repaint()
+
+    def filthy(self):
+        for ch in self.children:
+            ch.filthy()
+        Widget.filthy(self)
+
+    def render(self, width, height, terminal):
+        for ch in self.children:
+            ch.draw(width, height, terminal)
+
+    def changeFocus(self):
+        self.repaint()
+
+        if self.focusedChild is not None:
+            self.focusedChild.focusLost()
+            focusedChild = self.focusedChild
+            self.focusedChild = None
+            try:
+                curFocus = self.children.index(focusedChild) + 1
+            except ValueError:
+                raise YieldFocus()
+        else:
+            curFocus = 0
+        while curFocus < len(self.children):
+            try:
+                self.children[curFocus].focusReceived()
+            except YieldFocus:
+                curFocus += 1
+            else:
+                self.focusedChild = self.children[curFocus]
+                return
+        # None of our children wanted focus
+        raise YieldFocus()
+
+    def focusReceived(self):
+        self.changeFocus()
+        self.focused = True
+
+    def keystrokeReceived(self, keyID, modifier):
+        if self.focusedChild is not None:
+            try:
+                self.focusedChild.keystrokeReceived(keyID, modifier)
+            except YieldFocus:
+                self.changeFocus()
+                self.repaint()
+        else:
+            Widget.keystrokeReceived(self, keyID, modifier)
+
+
+class TopWindow(ContainerWidget):
+    """
+    A top-level container object which provides focus wrap-around and paint
+    scheduling.
+
+    @ivar painter: A no-argument callable which will be invoked when this
+    widget needs to be redrawn.
+
+    @ivar scheduler: A one-argument callable which will be invoked with a
+    no-argument callable and should arrange for it to invoked at some point in
+    the near future.  The no-argument callable will cause this widget and all
+    its children to be redrawn.  It is typically beneficial for the no-argument
+    callable to be invoked at the end of handling for whatever event is
+    currently active; for example, it might make sense to call it at the end of
+    L{twisted.conch.insults.insults.ITerminalProtocol.keystrokeReceived}.
+    Note, however, that since calls to this may also be made in response to no
+    apparent event, arrangements should be made for the function to be called
+    even if an event handler such as C{keystrokeReceived} is not on the call
+    stack (eg, using
+    L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>}
+    with a short timeout).
+    """
+
+    focused = True
+
+    def __init__(self, painter, scheduler):
+        ContainerWidget.__init__(self)
+        self.painter = painter
+        self.scheduler = scheduler
+
+    _paintCall = None
+
+    def repaint(self):
+        if self._paintCall is None:
+            self._paintCall = object()
+            self.scheduler(self._paint)
+        ContainerWidget.repaint(self)
+
+    def _paint(self):
+        self._paintCall = None
+        self.painter()
+
+    def changeFocus(self):
+        try:
+            ContainerWidget.changeFocus(self)
+        except YieldFocus:
+            try:
+                ContainerWidget.changeFocus(self)
+            except YieldFocus:
+                pass
+
+    def keystrokeReceived(self, keyID, modifier):
+        try:
+            ContainerWidget.keystrokeReceived(self, keyID, modifier)
+        except YieldFocus:
+            self.changeFocus()
+
+
+class AbsoluteBox(ContainerWidget):
+    def moveChild(self, child, x, y):
+        for n in range(len(self.children)):
+            if self.children[n][0] is child:
+                self.children[n] = (child, x, y)
+                break
+        else:
+            raise ValueError("No such child", child)
+
+    def render(self, width, height, terminal):
+        for ch, x, y in self.children:
+            wrap = BoundedTerminalWrapper(terminal, width - x, height - y, x, y)
+            ch.draw(width, height, wrap)
+
+
+class _Box(ContainerWidget):
+    TOP, CENTER, BOTTOM = range(3)
+
+    def __init__(self, gravity=CENTER):
+        ContainerWidget.__init__(self)
+        self.gravity = gravity
+
+    def sizeHint(self):
+        height = 0
+        width = 0
+        for ch in self.children:
+            hint = ch.sizeHint()
+            if hint is None:
+                hint = (None, None)
+
+            if self.variableDimension == 0:
+                if hint[0] is None:
+                    width = None
+                elif width is not None:
+                    width += hint[0]
+                if hint[1] is None:
+                    height = None
+                elif height is not None:
+                    height = max(height, hint[1])
+            else:
+                if hint[0] is None:
+                    width = None
+                elif width is not None:
+                    width = max(width, hint[0])
+                if hint[1] is None:
+                    height = None
+                elif height is not None:
+                    height += hint[1]
+
+        return width, height
+
+    def render(self, width, height, terminal):
+        if not self.children:
+            return
+
+        greedy = 0
+        wants = []
+        for ch in self.children:
+            hint = ch.sizeHint()
+            if hint is None:
+                hint = (None, None)
+            if hint[self.variableDimension] is None:
+                greedy += 1
+            wants.append(hint[self.variableDimension])
+
+        length = (width, height)[self.variableDimension]
+        totalWant = sum(w for w in wants if w is not None)
+        if greedy:
+            leftForGreedy = int((length - totalWant) / greedy)
+
+        widthOffset = heightOffset = 0
+
+        for want, ch in zip(wants, self.children):
+            if want is None:
+                want = leftForGreedy
+
+            subWidth, subHeight = width, height
+            if self.variableDimension == 0:
+                subWidth = want
+            else:
+                subHeight = want
+
+            wrap = BoundedTerminalWrapper(
+                terminal,
+                subWidth,
+                subHeight,
+                widthOffset,
+                heightOffset,
+            )
+            ch.draw(subWidth, subHeight, wrap)
+            if self.variableDimension == 0:
+                widthOffset += want
+            else:
+                heightOffset += want
+
+
+class HBox(_Box):
+    variableDimension = 0
+
+
+class VBox(_Box):
+    variableDimension = 1
+
+
+class Packer(ContainerWidget):
+    def render(self, width, height, terminal):
+        if not self.children:
+            return
+
+        root = int(len(self.children) ** 0.5 + 0.5)
+        boxes = [VBox() for n in range(root)]
+        for n, ch in enumerate(self.children):
+            boxes[n % len(boxes)].addChild(ch)
+        h = HBox()
+        map(h.addChild, boxes)
+        h.render(width, height, terminal)
+
+
+class Canvas(Widget):
+    focused = False
+
+    contents = None
+
+    def __init__(self):
+        Widget.__init__(self)
+        self.resize(1, 1)
+
+    def resize(self, width, height):
+        contents = array.array("B", b" " * width * height)
+        if self.contents is not None:
+            for x in range(min(width, self._width)):
+                for y in range(min(height, self._height)):
+                    contents[width * y + x] = self[x, y]
+        self.contents = contents
+        self._width = width
+        self._height = height
+        if self.x >= width:
+            self.x = width - 1
+        if self.y >= height:
+            self.y = height - 1
+
+    def __getitem__(self, index):
+        (x, y) = index
+        return self.contents[(self._width * y) + x]
+
+    def __setitem__(self, index, value):
+        (x, y) = index
+        self.contents[(self._width * y) + x] = value
+
+    def clear(self):
+        self.contents = array.array("B", b" " * len(self.contents))
+
+    def render(self, width, height, terminal):
+        if not width or not height:
+            return
+
+        if width != self._width or height != self._height:
+            self.resize(width, height)
+        for i in range(height):
+            terminal.cursorPosition(0, i)
+            text = self.contents[
+                self._width * i : self._width * i + self._width
+            ].tobytes()
+            text = text[:width]
+            terminal.write(text)
+
+
+def horizontalLine(terminal, y, left, right):
+    terminal.selectCharacterSet(insults.CS_DRAWING, insults.G0)
+    terminal.cursorPosition(left, y)
+    terminal.write(b"\161" * (right - left))
+    terminal.selectCharacterSet(insults.CS_US, insults.G0)
+
+
+def verticalLine(terminal, x, top, bottom):
+    terminal.selectCharacterSet(insults.CS_DRAWING, insults.G0)
+    for n in range(top, bottom):
+        terminal.cursorPosition(x, n)
+        terminal.write(b"\170")
+    terminal.selectCharacterSet(insults.CS_US, insults.G0)
+
+
+def rectangle(terminal, position, dimension):
+    """
+    Draw a rectangle
+
+    @type position: L{tuple}
+    @param position: A tuple of the (top, left) coordinates of the rectangle.
+    @type dimension: L{tuple}
+    @param dimension: A tuple of the (width, height) size of the rectangle.
+    """
+    (top, left) = position
+    (width, height) = dimension
+    terminal.selectCharacterSet(insults.CS_DRAWING, insults.G0)
+
+    terminal.cursorPosition(top, left)
+    terminal.write(b"\154")
+    terminal.write(b"\161" * (width - 2))
+    terminal.write(b"\153")
+    for n in range(height - 2):
+        terminal.cursorPosition(left, top + n + 1)
+        terminal.write(b"\170")
+        terminal.cursorForward(width - 2)
+        terminal.write(b"\170")
+    terminal.cursorPosition(0, top + height - 1)
+    terminal.write(b"\155")
+    terminal.write(b"\161" * (width - 2))
+    terminal.write(b"\152")
+
+    terminal.selectCharacterSet(insults.CS_US, insults.G0)
+
+
+class Border(Widget):
+    def __init__(self, containee):
+        Widget.__init__(self)
+        self.containee = containee
+        self.containee.parent = self
+
+    def focusReceived(self):
+        return self.containee.focusReceived()
+
+    def focusLost(self):
+        return self.containee.focusLost()
+
+    def keystrokeReceived(self, keyID, modifier):
+        return self.containee.keystrokeReceived(keyID, modifier)
+
+    def sizeHint(self):
+        hint = self.containee.sizeHint()
+        if hint is None:
+            hint = (None, None)
+        if hint[0] is None:
+            x = None
+        else:
+            x = hint[0] + 2
+        if hint[1] is None:
+            y = None
+        else:
+            y = hint[1] + 2
+        return x, y
+
+    def filthy(self):
+        self.containee.filthy()
+        Widget.filthy(self)
+
+    def render(self, width, height, terminal):
+        if self.containee.focused:
+            terminal.write(b"\x1b[31m")
+        rectangle(terminal, (0, 0), (width, height))
+        terminal.write(b"\x1b[0m")
+        wrap = BoundedTerminalWrapper(terminal, width - 2, height - 2, 1, 1)
+        self.containee.draw(width - 2, height - 2, wrap)
+
+
+class Button(Widget):
+    def __init__(self, label, onPress):
+        Widget.__init__(self)
+        self.label = label
+        self.onPress = onPress
+
+    def sizeHint(self):
+        return len(self.label), 1
+
+    def characterReceived(self, keyID, modifier):
+        if keyID == b"\r":
+            self.onPress()
+
+    def render(self, width, height, terminal):
+        terminal.cursorPosition(0, 0)
+        if self.focused:
+            terminal.write(b"\x1b[1m" + self.label + b"\x1b[0m")
+        else:
+            terminal.write(self.label)
+
+
+class TextInput(Widget):
+    def __init__(self, maxwidth, onSubmit):
+        Widget.__init__(self)
+        self.onSubmit = onSubmit
+        self.maxwidth = maxwidth
+        self.buffer = b""
+        self.cursor = 0
+
+    def setText(self, text):
+        self.buffer = text[: self.maxwidth]
+        self.cursor = len(self.buffer)
+        self.repaint()
+
+    def func_LEFT_ARROW(self, modifier):
+        if self.cursor > 0:
+            self.cursor -= 1
+            self.repaint()
+
+    def func_RIGHT_ARROW(self, modifier):
+        if self.cursor < len(self.buffer):
+            self.cursor += 1
+            self.repaint()
+
+    def backspaceReceived(self):
+        if self.cursor > 0:
+            self.buffer = self.buffer[: self.cursor - 1] + self.buffer[self.cursor :]
+            self.cursor -= 1
+            self.repaint()
+
+    def characterReceived(self, keyID, modifier):
+        if keyID == b"\r":
+            self.onSubmit(self.buffer)
+        else:
+            if len(self.buffer) < self.maxwidth:
+                self.buffer = (
+                    self.buffer[: self.cursor] + keyID + self.buffer[self.cursor :]
+                )
+                self.cursor += 1
+                self.repaint()
+
+    def sizeHint(self):
+        return self.maxwidth + 1, 1
+
+    def render(self, width, height, terminal):
+        currentText = self._renderText()
+        terminal.cursorPosition(0, 0)
+        if self.focused:
+            terminal.write(currentText[: self.cursor])
+            cursor(terminal, currentText[self.cursor : self.cursor + 1] or b" ")
+            terminal.write(currentText[self.cursor + 1 :])
+            terminal.write(b" " * (self.maxwidth - len(currentText) + 1))
+        else:
+            more = self.maxwidth - len(currentText)
+            terminal.write(currentText + b"_" * more)
+
+    def _renderText(self):
+        return self.buffer
+
+
+class PasswordInput(TextInput):
+    def _renderText(self):
+        return "*" * len(self.buffer)
+
+
+class TextOutput(Widget):
+    text = b""
+
+    def __init__(self, size=None):
+        Widget.__init__(self)
+        self.size = size
+
+    def sizeHint(self):
+        return self.size
+
+    def render(self, width, height, terminal):
+        terminal.cursorPosition(0, 0)
+        text = self.text[:width]
+        terminal.write(text + b" " * (width - len(text)))
+
+    def setText(self, text):
+        self.text = text
+        self.repaint()
+
+    def focusReceived(self):
+        raise YieldFocus()
+
+
+class TextOutputArea(TextOutput):
+    WRAP, TRUNCATE = range(2)
+
+    def __init__(self, size=None, longLines=WRAP):
+        TextOutput.__init__(self, size)
+        self.longLines = longLines
+
+    def render(self, width, height, terminal):
+        n = 0
+        inputLines = self.text.splitlines()
+        outputLines = []
+        while inputLines:
+            if self.longLines == self.WRAP:
+                line = inputLines.pop(0)
+                if not isinstance(line, str):
+                    line = line.decode("utf-8")
+                wrappedLines = []
+                for wrappedLine in tptext.greedyWrap(line, width):
+                    if not isinstance(wrappedLine, bytes):
+                        wrappedLine = wrappedLine.encode("utf-8")
+                    wrappedLines.append(wrappedLine)
+                outputLines.extend(wrappedLines or [b""])
+            else:
+                outputLines.append(inputLines.pop(0)[:width])
+            if len(outputLines) >= height:
+                break
+        for n, L in enumerate(outputLines[:height]):
+            terminal.cursorPosition(0, n)
+            terminal.write(L)
+
+
+class Viewport(Widget):
+    _xOffset = 0
+    _yOffset = 0
+
+    @property
+    def xOffset(self):
+        return self._xOffset
+
+    @xOffset.setter
+    def xOffset(self, value):
+        if self._xOffset != value:
+            self._xOffset = value
+            self.repaint()
+
+    @property
+    def yOffset(self):
+        return self._yOffset
+
+    @yOffset.setter
+    def yOffset(self, value):
+        if self._yOffset != value:
+            self._yOffset = value
+            self.repaint()
+
+    _width = 160
+    _height = 24
+
+    def __init__(self, containee):
+        Widget.__init__(self)
+        self.containee = containee
+        self.containee.parent = self
+
+        self._buf = helper.TerminalBuffer()
+        self._buf.width = self._width
+        self._buf.height = self._height
+        self._buf.connectionMade()
+
+    def filthy(self):
+        self.containee.filthy()
+        Widget.filthy(self)
+
+    def render(self, width, height, terminal):
+        self.containee.draw(self._width, self._height, self._buf)
+
+        # XXX /Lame/
+        for y, line in enumerate(
+            self._buf.lines[self._yOffset : self._yOffset + height]
+        ):
+            terminal.cursorPosition(0, y)
+            n = 0
+            for n, (ch, attr) in enumerate(line[self._xOffset : self._xOffset + width]):
+                if ch is self._buf.void:
+                    ch = b" "
+                terminal.write(ch)
+            if n < width:
+                terminal.write(b" " * (width - n - 1))
+
+
+class _Scrollbar(Widget):
+    def __init__(self, onScroll):
+        Widget.__init__(self)
+        self.onScroll = onScroll
+        self.percent = 0.0
+
+    def smaller(self):
+        self.percent = min(1.0, max(0.0, self.onScroll(-1)))
+        self.repaint()
+
+    def bigger(self):
+        self.percent = min(1.0, max(0.0, self.onScroll(+1)))
+        self.repaint()
+
+
+class HorizontalScrollbar(_Scrollbar):
+    def sizeHint(self):
+        return (None, 1)
+
+    def func_LEFT_ARROW(self, modifier):
+        self.smaller()
+
+    def func_RIGHT_ARROW(self, modifier):
+        self.bigger()
+
+    _left = "\N{BLACK LEFT-POINTING TRIANGLE}"
+    _right = "\N{BLACK RIGHT-POINTING TRIANGLE}"
+    _bar = "\N{LIGHT SHADE}"
+    _slider = "\N{DARK SHADE}"
+
+    def render(self, width, height, terminal):
+        terminal.cursorPosition(0, 0)
+        n = width - 3
+        before = int(n * self.percent)
+        after = n - before
+        me = (
+            self._left
+            + (self._bar * before)
+            + self._slider
+            + (self._bar * after)
+            + self._right
+        )
+        terminal.write(me.encode("utf-8"))
+
+
+class VerticalScrollbar(_Scrollbar):
+    def sizeHint(self):
+        return (1, None)
+
+    def func_UP_ARROW(self, modifier):
+        self.smaller()
+
+    def func_DOWN_ARROW(self, modifier):
+        self.bigger()
+
+    _up = "\N{BLACK UP-POINTING TRIANGLE}"
+    _down = "\N{BLACK DOWN-POINTING TRIANGLE}"
+    _bar = "\N{LIGHT SHADE}"
+    _slider = "\N{DARK SHADE}"
+
+    def render(self, width, height, terminal):
+        terminal.cursorPosition(0, 0)
+        knob = int(self.percent * (height - 2))
+        terminal.write(self._up.encode("utf-8"))
+        for i in range(1, height - 1):
+            terminal.cursorPosition(0, i)
+            if i != (knob + 1):
+                terminal.write(self._bar.encode("utf-8"))
+            else:
+                terminal.write(self._slider.encode("utf-8"))
+        terminal.cursorPosition(0, height - 1)
+        terminal.write(self._down.encode("utf-8"))
+
+
+class ScrolledArea(Widget):
+    """
+    A L{ScrolledArea} contains another widget wrapped in a viewport and
+    vertical and horizontal scrollbars for moving the viewport around.
+    """
+
+    def __init__(self, containee):
+        Widget.__init__(self)
+        self._viewport = Viewport(containee)
+        self._horiz = HorizontalScrollbar(self._horizScroll)
+        self._vert = VerticalScrollbar(self._vertScroll)
+
+        for w in self._viewport, self._horiz, self._vert:
+            w.parent = self
+
+    def _horizScroll(self, n):
+        self._viewport.xOffset += n
+        self._viewport.xOffset = max(0, self._viewport.xOffset)
+        return self._viewport.xOffset / 25.0
+
+    def _vertScroll(self, n):
+        self._viewport.yOffset += n
+        self._viewport.yOffset = max(0, self._viewport.yOffset)
+        return self._viewport.yOffset / 25.0
+
+    def func_UP_ARROW(self, modifier):
+        self._vert.smaller()
+
+    def func_DOWN_ARROW(self, modifier):
+        self._vert.bigger()
+
+    def func_LEFT_ARROW(self, modifier):
+        self._horiz.smaller()
+
+    def func_RIGHT_ARROW(self, modifier):
+        self._horiz.bigger()
+
+    def filthy(self):
+        self._viewport.filthy()
+        self._horiz.filthy()
+        self._vert.filthy()
+        Widget.filthy(self)
+
+    def render(self, width, height, terminal):
+        wrapper = BoundedTerminalWrapper(terminal, width - 2, height - 2, 1, 1)
+        self._viewport.draw(width - 2, height - 2, wrapper)
+        if self.focused:
+            terminal.write(b"\x1b[31m")
+        horizontalLine(terminal, 0, 1, width - 1)
+        verticalLine(terminal, 0, 1, height - 1)
+        self._vert.draw(
+            1, height - 1, BoundedTerminalWrapper(terminal, 1, height - 1, width - 1, 0)
+        )
+        self._horiz.draw(
+            width, 1, BoundedTerminalWrapper(terminal, width, 1, 0, height - 1)
+        )
+        terminal.write(b"\x1b[0m")
+
+
+def cursor(terminal, ch):
+    terminal.saveCursor()
+    terminal.selectGraphicRendition(str(insults.REVERSE_VIDEO))
+    terminal.write(ch)
+    terminal.restoreCursor()
+    terminal.cursorForward()
+
+
+class Selection(Widget):
+    # Index into the sequence
+    focusedIndex = 0
+
+    # Offset into the displayed subset of the sequence
+    renderOffset = 0
+
+    def __init__(self, sequence, onSelect, minVisible=None):
+        Widget.__init__(self)
+        self.sequence = sequence
+        self.onSelect = onSelect
+        self.minVisible = minVisible
+        if minVisible is not None:
+            self._width = max(map(len, self.sequence))
+
+    def sizeHint(self):
+        if self.minVisible is not None:
+            return self._width, self.minVisible
+
+    def func_UP_ARROW(self, modifier):
+        if self.focusedIndex > 0:
+            self.focusedIndex -= 1
+            if self.renderOffset > 0:
+                self.renderOffset -= 1
+            self.repaint()
+
+    def func_PGUP(self, modifier):
+        if self.renderOffset != 0:
+            self.focusedIndex -= self.renderOffset
+            self.renderOffset = 0
+        else:
+            self.focusedIndex = max(0, self.focusedIndex - self.height)
+        self.repaint()
+
+    def func_DOWN_ARROW(self, modifier):
+        if self.focusedIndex < len(self.sequence) - 1:
+            self.focusedIndex += 1
+            if self.renderOffset < self.height - 1:
+                self.renderOffset += 1
+            self.repaint()
+
+    def func_PGDN(self, modifier):
+        if self.renderOffset != self.height - 1:
+            change = self.height - self.renderOffset - 1
+            if change + self.focusedIndex >= len(self.sequence):
+                change = len(self.sequence) - self.focusedIndex - 1
+            self.focusedIndex += change
+            self.renderOffset = self.height - 1
+        else:
+            self.focusedIndex = min(
+                len(self.sequence) - 1, self.focusedIndex + self.height
+            )
+        self.repaint()
+
+    def characterReceived(self, keyID, modifier):
+        if keyID == b"\r":
+            self.onSelect(self.sequence[self.focusedIndex])
+
+    def render(self, width, height, terminal):
+        self.height = height
+        start = self.focusedIndex - self.renderOffset
+        if start > len(self.sequence) - height:
+            start = max(0, len(self.sequence) - height)
+
+        elements = self.sequence[start : start + height]
+
+        for n, ele in enumerate(elements):
+            terminal.cursorPosition(0, n)
+            if n == self.renderOffset:
+                terminal.saveCursor()
+                if self.focused:
+                    modes = str(insults.REVERSE_VIDEO), str(insults.BOLD)
+                else:
+                    modes = (str(insults.REVERSE_VIDEO),)
+                terminal.selectGraphicRendition(*modes)
+            text = ele[:width]
+            terminal.write(text + (b" " * (width - len(text))))
+            if n == self.renderOffset:
+                terminal.restoreCursor()
diff --git a/contrib/python/Twisted/py3/twisted/conch/interfaces.py b/contrib/python/Twisted/py3/twisted/conch/interfaces.py
new file mode 100644
index 00000000000..749fad2f96d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/interfaces.py
@@ -0,0 +1,454 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module contains interfaces defined for the L{twisted.conch} package.
+"""
+
+from zope.interface import Attribute, Interface
+
+
+class IConchUser(Interface):
+    """
+    A user who has been authenticated to Cred through Conch.  This is
+    the interface between the SSH connection and the user.
+    """
+
+    conn = Attribute("The SSHConnection object for this user.")
+
+    def lookupChannel(channelType, windowSize, maxPacket, data):
+        """
+        The other side requested a channel of some sort.
+
+        C{channelType} is the type of channel being requested,
+        as an ssh connection protocol channel type.
+        C{data} is any other packet data (often nothing).
+
+        We return a subclass of L{SSHChannel<ssh.channel.SSHChannel>}.  If
+        the channel type is unknown, we return C{None}.
+
+        For other failures, we raise an exception. If a
+        L{ConchError<error.ConchError>} is raised, the C{.value} will
+        be the message, and the C{.data} will be the error code.
+
+        @param channelType: The requested channel type
+        @type channelType:  L{bytes}
+        @param windowSize:  The initial size of the remote window
+        @type windowSize:   L{int}
+        @param maxPacket:   The largest packet we should send
+        @type maxPacket:    L{int}
+        @param data:        Additional request data
+        @type data:         L{bytes}
+        @rtype:             a subclass of L{SSHChannel} or L{None}
+        """
+
+    def lookupSubsystem(subsystem, data):
+        """
+        The other side requested a subsystem.
+
+        We return a L{Protocol} implementing the requested subsystem.
+        If the subsystem is not available, we return C{None}.
+
+        @param subsystem: The name of the subsystem being requested
+        @type subsystem: L{bytes}
+        @param data:     Additional request data (often nothing)
+        @type data:      L{bytes}
+        @rtype:          L{Protocol} or L{None}
+        """
+
+    def gotGlobalRequest(requestType, data):
+        """
+        A global request was sent from the other side.
+
+        We return a true value on success or a false value on failure.
+        If we indicate success by returning a tuple, its second item
+        will be sent to the other side as additional response data.
+
+        @param requestType: The type of the request
+        @type requestType:  L{bytes}
+        @param data:        Additional request data
+        @type data:         L{bytes}
+        @rtype:             boolean or L{tuple}
+        """
+
+
+class ISession(Interface):
+    def getPty(term, windowSize, modes):
+        """
+        Get a pseudo-terminal for use by a shell or command.
+
+        If a pseudo-terminal is not available, or the request otherwise
+        fails, raise an exception.
+        """
+
+    def openShell(proto):
+        """
+        Open a shell and connect it to proto.
+
+        @param proto: a L{ProcessProtocol} instance.
+        """
+
+    def execCommand(proto, command):
+        """
+        Execute a command.
+
+        @param proto: a L{ProcessProtocol} instance.
+        """
+
+    def windowChanged(newWindowSize):
+        """
+        Called when the size of the remote screen has changed.
+        """
+
+    def eofReceived():
+        """
+        Called when the other side has indicated no more data will be sent.
+        """
+
+    def closed():
+        """
+        Called when the session is closed.
+        """
+
+
+class EnvironmentVariableNotPermitted(ValueError):
+    """Setting this environment variable in this session is not permitted."""
+
+
+class ISessionSetEnv(Interface):
+    """A session that can set environment variables."""
+
+    def setEnv(name, value):
+        """
+        Set an environment variable for the shell or command to be started.
+
+        From U{RFC 4254, section 6.4
+        <https://tools.ietf.org/html/rfc4254#section-6.4>}: "Uncontrolled
+        setting of environment variables in a privileged process can be a
+        security hazard.  It is recommended that implementations either
+        maintain a list of allowable variable names or only set environment
+        variables after the server process has dropped sufficient
+        privileges."
+
+        (OpenSSH refuses all environment variables by default, but has an
+        C{AcceptEnv} configuration option to select specific variables to
+        accept.)
+
+        @param name: The name of the environment variable to set.
+        @type name: L{bytes}
+        @param value: The value of the environment variable to set.
+        @type value: L{bytes}
+        @raise EnvironmentVariableNotPermitted: if setting this environment
+            variable is not permitted.
+        """
+
+
+class ISFTPServer(Interface):
+    """
+    SFTP subsystem for server-side communication.
+
+    Each method should check to verify that the user has permission for
+    their actions.
+    """
+
+    avatar = Attribute(
+        """
+        The avatar returned by the Realm that we are authenticated with,
+        and represents the logged-in user.
+        """
+    )
+
+    def gotVersion(otherVersion, extData):
+        """
+        Called when the client sends their version info.
+
+        otherVersion is an integer representing the version of the SFTP
+        protocol they are claiming.
+        extData is a dictionary of extended_name : extended_data items.
+        These items are sent by the client to indicate additional features.
+
+        This method should return a dictionary of extended_name : extended_data
+        items.  These items are the additional features (if any) supported
+        by the server.
+        """
+        return {}
+
+    def openFile(filename, flags, attrs):
+        """
+        Called when the clients asks to open a file.
+
+        @param filename: a string representing the file to open.
+
+        @param flags: an integer of the flags to open the file with, ORed
+        together.  The flags and their values are listed at the bottom of
+        L{twisted.conch.ssh.filetransfer} as FXF_*.
+
+        @param attrs: a list of attributes to open the file with.  It is a
+        dictionary, consisting of 0 or more keys.  The possible keys are::
+
+            size: the size of the file in bytes
+            uid: the user ID of the file as an integer
+            gid: the group ID of the file as an integer
+            permissions: the permissions of the file with as an integer.
+            the bit representation of this field is defined by POSIX.
+            atime: the access time of the file as seconds since the epoch.
+            mtime: the modification time of the file as seconds since the epoch.
+            ext_*: extended attributes.  The server is not required to
+            understand this, but it may.
+
+        NOTE: there is no way to indicate text or binary files.  it is up
+        to the SFTP client to deal with this.
+
+        This method returns an object that meets the ISFTPFile interface.
+        Alternatively, it can return a L{Deferred} that will be called back
+        with the object.
+        """
+
+    def removeFile(filename):
+        """
+        Remove the given file.
+
+        This method returns when the remove succeeds, or a Deferred that is
+        called back when it succeeds.
+
+        @param filename: the name of the file as a string.
+        """
+
+    def renameFile(oldpath, newpath):
+        """
+        Rename the given file.
+
+        This method returns when the rename succeeds, or a L{Deferred} that is
+        called back when it succeeds. If the rename fails, C{renameFile} will
+        raise an implementation-dependent exception.
+
+        @param oldpath: the current location of the file.
+        @param newpath: the new file name.
+        """
+
+    def makeDirectory(path, attrs):
+        """
+        Make a directory.
+
+        This method returns when the directory is created, or a Deferred that
+        is called back when it is created.
+
+        @param path: the name of the directory to create as a string.
+        @param attrs: a dictionary of attributes to create the directory with.
+        Its meaning is the same as the attrs in the L{openFile} method.
+        """
+
+    def removeDirectory(path):
+        """
+        Remove a directory (non-recursively)
+
+        It is an error to remove a directory that has files or directories in
+        it.
+
+        This method returns when the directory is removed, or a Deferred that
+        is called back when it is removed.
+
+        @param path: the directory to remove.
+        """
+
+    def openDirectory(path):
+        """
+        Open a directory for scanning.
+
+        This method returns an iterable object that has a close() method,
+        or a Deferred that is called back with same.
+
+        The close() method is called when the client is finished reading
+        from the directory.  At this point, the iterable will no longer
+        be used.
+
+        The iterable should return triples of the form (filename,
+        longname, attrs) or Deferreds that return the same.  The
+        sequence must support __getitem__, but otherwise may be any
+        'sequence-like' object.
+
+        filename is the name of the file relative to the directory.
+        logname is an expanded format of the filename.  The recommended format
+        is:
+        -rwxr-xr-x   1 mjos     staff      348911 Mar 25 14:29 t-filexfer
+        1234567890 123 12345678 12345678 12345678 123456789012
+
+        The first line is sample output, the second is the length of the field.
+        The fields are: permissions, link count, user owner, group owner,
+        size in bytes, modification time.
+
+        attrs is a dictionary in the format of the attrs argument to openFile.
+
+        @param path: the directory to open.
+        """
+
+    def getAttrs(path, followLinks):
+        """
+        Return the attributes for the given path.
+
+        This method returns a dictionary in the same format as the attrs
+        argument to openFile or a Deferred that is called back with same.
+
+        @param path: the path to return attributes for as a string.
+        @param followLinks: a boolean.  If it is True, follow symbolic links
+        and return attributes for the real path at the base.  If it is False,
+        return attributes for the specified path.
+        """
+
+    def setAttrs(path, attrs):
+        """
+        Set the attributes for the path.
+
+        This method returns when the attributes are set or a Deferred that is
+        called back when they are.
+
+        @param path: the path to set attributes for as a string.
+        @param attrs: a dictionary in the same format as the attrs argument to
+        L{openFile}.
+        """
+
+    def readLink(path):
+        """
+        Find the root of a set of symbolic links.
+
+        This method returns the target of the link, or a Deferred that
+        returns the same.
+
+        @param path: the path of the symlink to read.
+        """
+
+    def makeLink(linkPath, targetPath):
+        """
+        Create a symbolic link.
+
+        This method returns when the link is made, or a Deferred that
+        returns the same.
+
+        @param linkPath: the pathname of the symlink as a string.
+        @param targetPath: the path of the target of the link as a string.
+        """
+
+    def realPath(path):
+        """
+        Convert any path to an absolute path.
+
+        This method returns the absolute path as a string, or a Deferred
+        that returns the same.
+
+        @param path: the path to convert as a string.
+        """
+
+    def extendedRequest(extendedName, extendedData):
+        """
+        This is the extension mechanism for SFTP.  The other side can send us
+        arbitrary requests.
+
+        If we don't implement the request given by extendedName, raise
+        NotImplementedError.
+
+        The return value is a string, or a Deferred that will be called
+        back with a string.
+
+        @param extendedName: the name of the request as a string.
+        @param extendedData: the data the other side sent with the request,
+        as a string.
+        """
+
+
+class IKnownHostEntry(Interface):
+    """
+    A L{IKnownHostEntry} is an entry in an OpenSSH-formatted C{known_hosts}
+    file.
+
+    @since: 8.2
+    """
+
+    def matchesKey(key):
+        """
+        Return True if this entry matches the given Key object, False
+        otherwise.
+
+        @param key: The key object to match against.
+        @type key: L{twisted.conch.ssh.keys.Key}
+        """
+
+    def matchesHost(hostname):
+        """
+        Return True if this entry matches the given hostname, False otherwise.
+
+        Note that this does no name resolution; if you want to match an IP
+        address, you have to resolve it yourself, and pass it in as a dotted
+        quad string.
+
+        @param hostname: The hostname to match against.
+        @type hostname: L{str}
+        """
+
+    def toString():
+        """
+
+        @return: a serialized string representation of this entry, suitable for
+        inclusion in a known_hosts file.  (Newline not included.)
+
+        @rtype: L{str}
+        """
+
+
+class ISFTPFile(Interface):
+    """
+    This represents an open file on the server.  An object adhering to this
+    interface should be returned from L{openFile}().
+    """
+
+    def close():
+        """
+        Close the file.
+
+        This method returns nothing if the close succeeds immediately, or a
+        Deferred that is called back when the close succeeds.
+        """
+
+    def readChunk(offset, length):
+        """
+        Read from the file.
+
+        If EOF is reached before any data is read, raise EOFError.
+
+        This method returns the data as a string, or a Deferred that is
+        called back with same.
+
+        @param offset: an integer that is the index to start from in the file.
+        @param length: the maximum length of data to return.  The actual amount
+        returned may less than this.  For normal disk files, however,
+        this should read the requested number (up to the end of the file).
+        """
+
+    def writeChunk(offset, data):
+        """
+        Write to the file.
+
+        This method returns when the write completes, or a Deferred that is
+        called when it completes.
+
+        @param offset: an integer that is the index to start from in the file.
+        @param data: a string that is the data to write.
+        """
+
+    def getAttrs():
+        """
+        Return the attributes for the file.
+
+        This method returns a dictionary in the same format as the attrs
+        argument to L{openFile} or a L{Deferred} that is called back with same.
+        """
+
+    def setAttrs(attrs):
+        """
+        Set the attributes for the file.
+
+        This method returns when the attributes are set or a Deferred that is
+        called back when they are.
+
+        @param attrs: a dictionary in the same format as the attrs argument to
+        L{openFile}.
+        """
diff --git a/contrib/python/Twisted/py3/twisted/conch/ls.py b/contrib/python/Twisted/py3/twisted/conch/ls.py
new file mode 100644
index 00000000000..63fd5a0eb7a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ls.py
@@ -0,0 +1,104 @@
+# -*- test-case-name: twisted.conch.test.test_cftp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+import array
+import stat
+from time import localtime, strftime, time
+
+# Locale-independent month names to use instead of strftime's
+_MONTH_NAMES = dict(
+    list(
+        zip(
+            list(range(1, 13)),
+            "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split(),
+        )
+    )
+)
+
+
+def lsLine(name, s):
+    """
+    Build an 'ls' line for a file ('file' in its generic sense, it
+    can be of any type).
+    """
+    mode = s.st_mode
+    perms = array.array("B", b"-" * 10)
+    ft = stat.S_IFMT(mode)
+    if stat.S_ISDIR(ft):
+        perms[0] = ord("d")
+    elif stat.S_ISCHR(ft):
+        perms[0] = ord("c")
+    elif stat.S_ISBLK(ft):
+        perms[0] = ord("b")
+    elif stat.S_ISREG(ft):
+        perms[0] = ord("-")
+    elif stat.S_ISFIFO(ft):
+        perms[0] = ord("f")
+    elif stat.S_ISLNK(ft):
+        perms[0] = ord("l")
+    elif stat.S_ISSOCK(ft):
+        perms[0] = ord("s")
+    else:
+        perms[0] = ord("!")
+    # User
+    if mode & stat.S_IRUSR:
+        perms[1] = ord("r")
+    if mode & stat.S_IWUSR:
+        perms[2] = ord("w")
+    if mode & stat.S_IXUSR:
+        perms[3] = ord("x")
+    # Group
+    if mode & stat.S_IRGRP:
+        perms[4] = ord("r")
+    if mode & stat.S_IWGRP:
+        perms[5] = ord("w")
+    if mode & stat.S_IXGRP:
+        perms[6] = ord("x")
+    # Other
+    if mode & stat.S_IROTH:
+        perms[7] = ord("r")
+    if mode & stat.S_IWOTH:
+        perms[8] = ord("w")
+    if mode & stat.S_IXOTH:
+        perms[9] = ord("x")
+    # Suid/sgid
+    if mode & stat.S_ISUID:
+        if perms[3] == ord("x"):
+            perms[3] = ord("s")
+        else:
+            perms[3] = ord("S")
+    if mode & stat.S_ISGID:
+        if perms[6] == ord("x"):
+            perms[6] = ord("s")
+        else:
+            perms[6] = ord("S")
+
+    if isinstance(name, bytes):
+        name = name.decode("utf-8")
+    lsPerms = perms.tobytes()
+    lsPerms = lsPerms.decode("utf-8")
+
+    lsresult = [
+        lsPerms,
+        str(s.st_nlink).rjust(5),
+        " ",
+        str(s.st_uid).ljust(9),
+        str(s.st_gid).ljust(9),
+        str(s.st_size).rjust(8),
+        " ",
+    ]
+    # Need to specify the month manually, as strftime depends on locale
+    ttup = localtime(s.st_mtime)
+    sixmonths = 60 * 60 * 24 * 7 * 26
+    if s.st_mtime + sixmonths < time():  # Last edited more than 6mo ago
+        strtime = strftime("%%s %d  %Y ", ttup)
+    else:
+        strtime = strftime("%%s %d %H:%M ", ttup)
+    lsresult.append(strtime % (_MONTH_NAMES[ttup[1]],))
+
+    lsresult.append(name)
+    return "".join(lsresult)
+
+
+__all__ = ["lsLine"]
diff --git a/contrib/python/Twisted/py3/twisted/conch/manhole.py b/contrib/python/Twisted/py3/twisted/conch/manhole.py
new file mode 100644
index 00000000000..5bf2f817a4e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/manhole.py
@@ -0,0 +1,392 @@
+# -*- test-case-name: twisted.conch.test.test_manhole -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Line-input oriented interactive interpreter loop.
+
+Provides classes for handling Python source input and arbitrary output
+interactively from a Twisted application.  Also included is syntax coloring
+code with support for VT102 terminals, control code handling (^C, ^D, ^Q),
+and reasonable handling of Deferreds.
+
+@author: Jp Calderone
+"""
+
+import code
+import sys
+import tokenize
+from io import BytesIO
+from traceback import format_exception
+from types import TracebackType
+from typing import Type
+
+from twisted.conch import recvline
+from twisted.internet import defer
+from twisted.python.compat import _get_async_param
+from twisted.python.htmlizer import TokenPrinter
+from twisted.python.monkey import MonkeyPatcher
+
+
+class FileWrapper:
+    """
+    Minimal write-file-like object.
+
+    Writes are translated into addOutput calls on an object passed to
+    __init__.  Newlines are also converted from network to local style.
+    """
+
+    softspace = 0
+    state = "normal"
+
+    def __init__(self, o):
+        self.o = o
+
+    def flush(self):
+        pass
+
+    def write(self, data):
+        self.o.addOutput(data.replace("\r\n", "\n"))
+
+    def writelines(self, lines):
+        self.write("".join(lines))
+
+
+class ManholeInterpreter(code.InteractiveInterpreter):
+    """
+    Interactive Interpreter with special output and Deferred support.
+
+    Aside from the features provided by L{code.InteractiveInterpreter}, this
+    class captures sys.stdout output and redirects it to the appropriate
+    location (the Manhole protocol instance).  It also treats Deferreds
+    which reach the top-level specially: each is formatted to the user with
+    a unique identifier and a new callback and errback added to it, each of
+    which will format the unique identifier and the result with which the
+    Deferred fires and then pass it on to the next participant in the
+    callback chain.
+    """
+
+    numDeferreds = 0
+
+    def __init__(self, handler, locals=None, filename="<console>"):
+        code.InteractiveInterpreter.__init__(self, locals)
+        self._pendingDeferreds = {}
+        self.handler = handler
+        self.filename = filename
+        self.resetBuffer()
+
+        self.monkeyPatcher = MonkeyPatcher()
+        self.monkeyPatcher.addPatch(sys, "displayhook", self.displayhook)
+        self.monkeyPatcher.addPatch(sys, "excepthook", self.excepthook)
+        self.monkeyPatcher.addPatch(sys, "stdout", FileWrapper(self.handler))
+
+    def resetBuffer(self):
+        """
+        Reset the input buffer.
+        """
+        self.buffer = []
+
+    def push(self, line):
+        """
+        Push a line to the interpreter.
+
+        The line should not have a trailing newline; it may have
+        internal newlines.  The line is appended to a buffer and the
+        interpreter's runsource() method is called with the
+        concatenated contents of the buffer as source.  If this
+        indicates that the command was executed or invalid, the buffer
+        is reset; otherwise, the command is incomplete, and the buffer
+        is left as it was after the line was appended.  The return
+        value is 1 if more input is required, 0 if the line was dealt
+        with in some way (this is the same as runsource()).
+
+        @param line: line of text
+        @type line: L{bytes}
+        @return: L{bool} from L{code.InteractiveInterpreter.runsource}
+        """
+        self.buffer.append(line)
+        source = b"\n".join(self.buffer)
+        source = source.decode("utf-8")
+        more = self.runsource(source, self.filename)
+        if not more:
+            self.resetBuffer()
+        return more
+
+    def runcode(self, *a, **kw):
+        with self.monkeyPatcher:
+            code.InteractiveInterpreter.runcode(self, *a, **kw)
+
+    def excepthook(
+        self,
+        excType: Type[BaseException],
+        excValue: BaseException,
+        excTraceback: TracebackType,
+    ) -> None:
+        """
+        Format exception tracebacks and write them to the output handler.
+        """
+        lines = format_exception(excType, excValue, excTraceback.tb_next)
+        self.write("".join(lines))
+
+    def displayhook(self, obj):
+        self.locals["_"] = obj
+        if isinstance(obj, defer.Deferred):
+            # XXX Ick, where is my "hasFired()" interface?
+            if hasattr(obj, "result"):
+                self.write(repr(obj))
+            elif id(obj) in self._pendingDeferreds:
+                self.write("<Deferred #%d>" % (self._pendingDeferreds[id(obj)][0],))
+            else:
+                d = self._pendingDeferreds
+                k = self.numDeferreds
+                d[id(obj)] = (k, obj)
+                self.numDeferreds += 1
+                obj.addCallbacks(
+                    self._cbDisplayDeferred,
+                    self._ebDisplayDeferred,
+                    callbackArgs=(k, obj),
+                    errbackArgs=(k, obj),
+                )
+                self.write("<Deferred #%d>" % (k,))
+        elif obj is not None:
+            self.write(repr(obj))
+
+    def _cbDisplayDeferred(self, result, k, obj):
+        self.write("Deferred #%d called back: %r" % (k, result), True)
+        del self._pendingDeferreds[id(obj)]
+        return result
+
+    def _ebDisplayDeferred(self, failure, k, obj):
+        self.write("Deferred #%d failed: %r" % (k, failure.getErrorMessage()), True)
+        del self._pendingDeferreds[id(obj)]
+        return failure
+
+    def write(self, data, isAsync=None, **kwargs):
+        isAsync = _get_async_param(isAsync, **kwargs)
+        self.handler.addOutput(data, isAsync)
+
+
+CTRL_C = b"\x03"
+CTRL_D = b"\x04"
+CTRL_BACKSLASH = b"\x1c"
+CTRL_L = b"\x0c"
+CTRL_A = b"\x01"
+CTRL_E = b"\x05"
+
+
+class Manhole(recvline.HistoricRecvLine):
+    r"""
+    Mediator between a fancy line source and an interactive interpreter.
+
+    This accepts lines from its transport and passes them on to a
+    L{ManholeInterpreter}.  Control commands (^C, ^D, ^\) are also handled
+    with something approximating their normal terminal-mode behavior.  It
+    can optionally be constructed with a dict which will be used as the
+    local namespace for any code executed.
+    """
+
+    namespace = None
+
+    def __init__(self, namespace=None):
+        recvline.HistoricRecvLine.__init__(self)
+        if namespace is not None:
+            self.namespace = namespace.copy()
+
+    def connectionMade(self):
+        recvline.HistoricRecvLine.connectionMade(self)
+        self.interpreter = ManholeInterpreter(self, self.namespace)
+        self.keyHandlers[CTRL_C] = self.handle_INT
+        self.keyHandlers[CTRL_D] = self.handle_EOF
+        self.keyHandlers[CTRL_L] = self.handle_FF
+        self.keyHandlers[CTRL_A] = self.handle_HOME
+        self.keyHandlers[CTRL_E] = self.handle_END
+        self.keyHandlers[CTRL_BACKSLASH] = self.handle_QUIT
+
+    def handle_INT(self):
+        """
+        Handle ^C as an interrupt keystroke by resetting the current input
+        variables to their initial state.
+        """
+        self.pn = 0
+        self.lineBuffer = []
+        self.lineBufferIndex = 0
+        self.interpreter.resetBuffer()
+
+        self.terminal.nextLine()
+        self.terminal.write(b"KeyboardInterrupt")
+        self.terminal.nextLine()
+        self.terminal.write(self.ps[self.pn])
+
+    def handle_EOF(self):
+        if self.lineBuffer:
+            self.terminal.write(b"\a")
+        else:
+            self.handle_QUIT()
+
+    def handle_FF(self):
+        """
+        Handle a 'form feed' byte - generally used to request a screen
+        refresh/redraw.
+        """
+        self.terminal.eraseDisplay()
+        self.terminal.cursorHome()
+        self.drawInputLine()
+
+    def handle_QUIT(self):
+        self.terminal.loseConnection()
+
+    def _needsNewline(self):
+        w = self.terminal.lastWrite
+        return not w.endswith(b"\n") and not w.endswith(b"\x1bE")
+
+    def addOutput(self, data, isAsync=None, **kwargs):
+        isAsync = _get_async_param(isAsync, **kwargs)
+        if isAsync:
+            self.terminal.eraseLine()
+            self.terminal.cursorBackward(len(self.lineBuffer) + len(self.ps[self.pn]))
+
+        self.terminal.write(data)
+
+        if isAsync:
+            if self._needsNewline():
+                self.terminal.nextLine()
+
+            self.terminal.write(self.ps[self.pn])
+
+            if self.lineBuffer:
+                oldBuffer = self.lineBuffer
+                self.lineBuffer = []
+                self.lineBufferIndex = 0
+
+                self._deliverBuffer(oldBuffer)
+
+    def lineReceived(self, line):
+        more = self.interpreter.push(line)
+        self.pn = bool(more)
+        if self._needsNewline():
+            self.terminal.nextLine()
+        self.terminal.write(self.ps[self.pn])
+
+
+class VT102Writer:
+    """
+    Colorizer for Python tokens.
+
+    A series of tokens are written to instances of this object.  Each is
+    colored in a particular way.  The final line of the result of this is
+    generally added to the output.
+    """
+
+    typeToColor = {
+        "identifier": b"\x1b[31m",
+        "keyword": b"\x1b[32m",
+        "parameter": b"\x1b[33m",
+        "variable": b"\x1b[1;33m",
+        "string": b"\x1b[35m",
+        "number": b"\x1b[36m",
+        "op": b"\x1b[37m",
+    }
+
+    normalColor = b"\x1b[0m"
+
+    def __init__(self):
+        self.written = []
+
+    def color(self, type):
+        r = self.typeToColor.get(type, b"")
+        return r
+
+    def write(self, token, type=None):
+        if token and token != b"\r":
+            c = self.color(type)
+            if c:
+                self.written.append(c)
+            self.written.append(token)
+            if c:
+                self.written.append(self.normalColor)
+
+    def __bytes__(self):
+        s = b"".join(self.written)
+        return s.strip(b"\n").splitlines()[-1]
+
+    if bytes == str:
+        # Compat with Python 2.7
+        __str__ = __bytes__
+
+
+def lastColorizedLine(source):
+    """
+    Tokenize and colorize the given Python source.
+
+    Returns a VT102-format colorized version of the last line of C{source}.
+
+    @param source: Python source code
+    @type source: L{str} or L{bytes}
+    @return: L{bytes} of colorized source
+    """
+    if not isinstance(source, bytes):
+        source = source.encode("utf-8")
+    w = VT102Writer()
+    p = TokenPrinter(w.write).printtoken
+    s = BytesIO(source)
+
+    for token in tokenize.tokenize(s.readline):
+        (tokenType, string, start, end, line) = token
+        p(tokenType, string, start, end, line)
+
+    return bytes(w)
+
+
+class ColoredManhole(Manhole):
+    """
+    A REPL which syntax colors input as users type it.
+    """
+
+    def getSource(self):
+        """
+        Return a string containing the currently entered source.
+
+        This is only the code which will be considered for execution
+        next.
+        """
+        return b"\n".join(self.interpreter.buffer) + b"\n" + b"".join(self.lineBuffer)
+
+    def characterReceived(self, ch, moreCharactersComing):
+        if self.mode == "insert":
+            self.lineBuffer.insert(self.lineBufferIndex, ch)
+        else:
+            self.lineBuffer[self.lineBufferIndex : self.lineBufferIndex + 1] = [ch]
+        self.lineBufferIndex += 1
+
+        if moreCharactersComing:
+            # Skip it all, we'll get called with another character in
+            # like 2 femtoseconds.
+            return
+
+        if ch == b" ":
+            # Don't bother to try to color whitespace
+            self.terminal.write(ch)
+            return
+
+        source = self.getSource()
+
+        # Try to write some junk
+        try:
+            coloredLine = lastColorizedLine(source)
+        except tokenize.TokenError:
+            # We couldn't do it.  Strange.  Oh well, just add the character.
+            self.terminal.write(ch)
+        else:
+            # Success!  Clear the source on this line.
+            self.terminal.eraseLine()
+            self.terminal.cursorBackward(
+                len(self.lineBuffer) + len(self.ps[self.pn]) - 1
+            )
+
+            # And write a new, colorized one.
+            self.terminal.write(self.ps[self.pn] + coloredLine)
+
+            # And move the cursor to where it belongs
+            n = len(self.lineBuffer) - self.lineBufferIndex
+            if n:
+                self.terminal.cursorBackward(n)
diff --git a/contrib/python/Twisted/py3/twisted/conch/manhole_ssh.py b/contrib/python/Twisted/py3/twisted/conch/manhole_ssh.py
new file mode 100644
index 00000000000..8ac3b6d4be1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/manhole_ssh.py
@@ -0,0 +1,148 @@
+# -*- test-case-name: twisted.conch.test.test_manhole -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+insults/SSH integration support.
+
+@author: Jp Calderone
+"""
+
+from typing import Dict
+
+from zope.interface import implementer
+
+from twisted.conch import avatar, error as econch, interfaces as iconch
+from twisted.conch.insults import insults
+from twisted.conch.ssh import factory, session
+from twisted.python import components
+
+
+class _Glue:
+    """
+    A feeble class for making one attribute look like another.
+
+    This should be replaced with a real class at some point, probably.
+    Try not to write new code that uses it.
+    """
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
+    def __getattr__(self, name):
+        raise AttributeError(self.name, "has no attribute", name)
+
+
+class TerminalSessionTransport:
+    def __init__(self, proto, chainedProtocol, avatar, width, height):
+        self.proto = proto
+        self.avatar = avatar
+        self.chainedProtocol = chainedProtocol
+
+        protoSession = self.proto.session
+
+        self.proto.makeConnection(
+            _Glue(
+                write=self.chainedProtocol.dataReceived,
+                loseConnection=lambda: avatar.conn.sendClose(protoSession),
+                name="SSH Proto Transport",
+            )
+        )
+
+        def loseConnection():
+            self.proto.loseConnection()
+
+        self.chainedProtocol.makeConnection(
+            _Glue(
+                write=self.proto.write,
+                loseConnection=loseConnection,
+                name="Chained Proto Transport",
+            )
+        )
+
+        # XXX TODO
+        # chainedProtocol is supposed to be an ITerminalTransport,
+        # maybe.  That means perhaps its terminalProtocol attribute is
+        # an ITerminalProtocol, it could be.  So calling terminalSize
+        # on that should do the right thing But it'd be nice to clean
+        # this bit up.
+        self.chainedProtocol.terminalProtocol.terminalSize(width, height)
+
+
+@implementer(iconch.ISession)
+class TerminalSession(components.Adapter):
+    transportFactory = TerminalSessionTransport
+    chainedProtocolFactory = insults.ServerProtocol
+
+    def getPty(self, term, windowSize, attrs):
+        self.height, self.width = windowSize[:2]
+
+    def openShell(self, proto):
+        self.transportFactory(
+            proto,
+            self.chainedProtocolFactory(),
+            iconch.IConchUser(self.original),
+            self.width,
+            self.height,
+        )
+
+    def execCommand(self, proto, cmd):
+        raise econch.ConchError("Cannot execute commands")
+
+    def windowChanged(self, newWindowSize):
+        # ISession.windowChanged
+        raise NotImplementedError("Unimplemented: TerminalSession.windowChanged")
+
+    def eofReceived(self):
+        # ISession.eofReceived
+        raise NotImplementedError("Unimplemented: TerminalSession.eofReceived")
+
+    def closed(self):
+        # ISession.closed
+        pass
+
+
+class TerminalUser(avatar.ConchUser, components.Adapter):
+    def __init__(self, original, avatarId):
+        components.Adapter.__init__(self, original)
+        avatar.ConchUser.__init__(self)
+        self.channelLookup[b"session"] = session.SSHSession
+
+
+class TerminalRealm:
+    userFactory = TerminalUser
+    sessionFactory = TerminalSession
+
+    transportFactory = TerminalSessionTransport
+    chainedProtocolFactory = insults.ServerProtocol
+
+    def _getAvatar(self, avatarId):
+        comp = components.Componentized()
+        user = self.userFactory(comp, avatarId)
+        sess = self.sessionFactory(comp)
+
+        sess.transportFactory = self.transportFactory
+        sess.chainedProtocolFactory = self.chainedProtocolFactory
+
+        comp.setComponent(iconch.IConchUser, user)
+        comp.setComponent(iconch.ISession, sess)
+
+        return user
+
+    def __init__(self, transportFactory=None):
+        if transportFactory is not None:
+            self.transportFactory = transportFactory
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        for i in interfaces:
+            if i is iconch.IConchUser:
+                return (iconch.IConchUser, self._getAvatar(avatarId), lambda: None)
+        raise NotImplementedError()
+
+
+class ConchFactory(factory.SSHFactory):
+    publicKeys: Dict[bytes, bytes] = {}
+    privateKeys: Dict[bytes, bytes] = {}
+
+    def __init__(self, portal):
+        self.portal = portal
diff --git a/contrib/python/Twisted/py3/twisted/conch/manhole_tap.py b/contrib/python/Twisted/py3/twisted/conch/manhole_tap.py
new file mode 100644
index 00000000000..24c79a5e7f4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/manhole_tap.py
@@ -0,0 +1,180 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+TAP plugin for creating telnet- and ssh-accessible manhole servers.
+
+@author: Jp Calderone
+"""
+
+from zope.interface import implementer
+
+from twisted.application import service, strports
+from twisted.conch import manhole, manhole_ssh, telnet
+from twisted.conch.insults import insults
+from twisted.conch.ssh import keys
+from twisted.cred import checkers, portal
+from twisted.internet import protocol
+from twisted.python import filepath, usage
+
+
+class makeTelnetProtocol:
+    def __init__(self, portal):
+        self.portal = portal
+
+    def __call__(self):
+        auth = telnet.AuthenticatingTelnetProtocol
+        args = (self.portal,)
+        return telnet.TelnetTransport(auth, *args)
+
+
+class chainedProtocolFactory:
+    def __init__(self, namespace):
+        self.namespace = namespace
+
+    def __call__(self):
+        return insults.ServerProtocol(manhole.ColoredManhole, self.namespace)
+
+
+@implementer(portal.IRealm)
+class _StupidRealm:
+    def __init__(self, proto, *a, **kw):
+        self.protocolFactory = proto
+        self.protocolArgs = a
+        self.protocolKwArgs = kw
+
+    def requestAvatar(self, avatarId, *interfaces):
+        if telnet.ITelnetProtocol in interfaces:
+            return (
+                telnet.ITelnetProtocol,
+                self.protocolFactory(*self.protocolArgs, **self.protocolKwArgs),
+                lambda: None,
+            )
+        raise NotImplementedError()
+
+
+class Options(usage.Options):
+    optParameters = [
+        [
+            "telnetPort",
+            "t",
+            None,
+            (
+                "strports description of the address on which to listen for telnet "
+                "connections"
+            ),
+        ],
+        [
+            "sshPort",
+            "s",
+            None,
+            (
+                "strports description of the address on which to listen for ssh "
+                "connections"
+            ),
+        ],
+        [
+            "passwd",
+            "p",
+            "/etc/passwd",
+            "name of a passwd(5)-format username/password file",
+        ],
+        [
+            "sshKeyDir",
+            None,
+            "<USER DATA DIR>",
+            "Directory where the autogenerated SSH key is kept.",
+        ],
+        ["sshKeyName", None, "server.key", "Filename of the autogenerated SSH key."],
+        ["sshKeySize", None, 4096, "Size of the automatically generated SSH key."],
+    ]
+
+    def __init__(self):
+        usage.Options.__init__(self)
+        self["namespace"] = None
+
+    def postOptions(self):
+        if self["telnetPort"] is None and self["sshPort"] is None:
+            raise usage.UsageError(
+                "At least one of --telnetPort and --sshPort must be specified"
+            )
+
+
+def makeService(options):
+    """
+    Create a manhole server service.
+
+    @type options: L{dict}
+    @param options: A mapping describing the configuration of
+    the desired service.  Recognized key/value pairs are::
+
+        "telnetPort": strports description of the address on which
+                      to listen for telnet connections.  If None,
+                      no telnet service will be started.
+
+        "sshPort": strports description of the address on which to
+                   listen for ssh connections.  If None, no ssh
+                   service will be started.
+
+        "namespace": dictionary containing desired initial locals
+                     for manhole connections.  If None, an empty
+                     dictionary will be used.
+
+        "passwd": Name of a passwd(5)-format username/password file.
+
+        "sshKeyDir": The folder that the SSH server key will be kept in.
+
+        "sshKeyName": The filename of the key.
+
+        "sshKeySize": The size of the key, in bits. Default is 4096.
+
+    @rtype: L{twisted.application.service.IService}
+    @return: A manhole service.
+    """
+    svc = service.MultiService()
+
+    namespace = options["namespace"]
+    if namespace is None:
+        namespace = {}
+
+    checker = checkers.FilePasswordDB(options["passwd"])
+
+    if options["telnetPort"]:
+        telnetRealm = _StupidRealm(
+            telnet.TelnetBootstrapProtocol,
+            insults.ServerProtocol,
+            manhole.ColoredManhole,
+            namespace,
+        )
+
+        telnetPortal = portal.Portal(telnetRealm, [checker])
+
+        telnetFactory = protocol.ServerFactory()
+        telnetFactory.protocol = makeTelnetProtocol(telnetPortal)
+        telnetService = strports.service(options["telnetPort"], telnetFactory)
+        telnetService.setServiceParent(svc)
+
+    if options["sshPort"]:
+        sshRealm = manhole_ssh.TerminalRealm()
+        sshRealm.chainedProtocolFactory = chainedProtocolFactory(namespace)
+
+        sshPortal = portal.Portal(sshRealm, [checker])
+        sshFactory = manhole_ssh.ConchFactory(sshPortal)
+
+        if options["sshKeyDir"] != "<USER DATA DIR>":
+            keyDir = options["sshKeyDir"]
+        else:
+            from twisted.python._appdirs import getDataDirectory
+
+            keyDir = getDataDirectory()
+
+        keyLocation = filepath.FilePath(keyDir).child(options["sshKeyName"])
+
+        sshKey = keys._getPersistentRSAKey(keyLocation, int(options["sshKeySize"]))
+        sshFactory.publicKeys[b"ssh-rsa"] = sshKey
+        sshFactory.privateKeys[b"ssh-rsa"] = sshKey
+
+        sshService = strports.service(options["sshPort"], sshFactory)
+        sshService.setServiceParent(svc)
+
+    return svc
diff --git a/contrib/python/Twisted/py3/twisted/conch/mixin.py b/contrib/python/Twisted/py3/twisted/conch/mixin.py
new file mode 100644
index 00000000000..6cf6531a6a3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/mixin.py
@@ -0,0 +1,54 @@
+# -*- test-case-name: twisted.conch.test.test_mixin -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Experimental optimization
+
+This module provides a single mixin class which allows protocols to
+collapse numerous small writes into a single larger one.
+
+@author: Jp Calderone
+"""
+
+from twisted.internet import reactor
+
+
+class BufferingMixin:
+    """
+    Mixin which adds write buffering.
+    """
+
+    _delayedWriteCall = None
+    data = None
+
+    DELAY = 0.0
+
+    def schedule(self):
+        return reactor.callLater(self.DELAY, self.flush)
+
+    def reschedule(self, token):
+        token.reset(self.DELAY)
+
+    def write(self, data):
+        """
+        Buffer some bytes to be written soon.
+
+        Every call to this function delays the real write by C{self.DELAY}
+        seconds.  When the delay expires, all collected bytes are written
+        to the underlying transport using L{ITransport.writeSequence}.
+        """
+        if self._delayedWriteCall is None:
+            self.data = []
+            self._delayedWriteCall = self.schedule()
+        else:
+            self.reschedule(self._delayedWriteCall)
+        self.data.append(data)
+
+    def flush(self):
+        """
+        Flush the buffer immediately.
+        """
+        self._delayedWriteCall = None
+        self.transport.writeSequence(self.data)
+        self.data = None
diff --git a/contrib/python/Twisted/py3/twisted/conch/newsfragments/.gitignore b/contrib/python/Twisted/py3/twisted/conch/newsfragments/.gitignore
new file mode 100644
index 00000000000..f935021a8f8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/newsfragments/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/contrib/python/Twisted/py3/twisted/conch/openssh_compat/__init__.py b/contrib/python/Twisted/py3/twisted/conch/openssh_compat/__init__.py
new file mode 100644
index 00000000000..4b4e023c1b7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/openssh_compat/__init__.py
@@ -0,0 +1,10 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+"""
+Support for OpenSSH configuration files.
+
+Maintainer: Paul Swartz
+"""
diff --git a/contrib/python/Twisted/py3/twisted/conch/openssh_compat/factory.py b/contrib/python/Twisted/py3/twisted/conch/openssh_compat/factory.py
new file mode 100644
index 00000000000..20051fc89f7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/openssh_compat/factory.py
@@ -0,0 +1,74 @@
+# -*- test-case-name: twisted.conch.test.test_openssh_compat -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Factory for reading openssh configuration files: public keys, private keys, and
+moduli file.
+"""
+
+import errno
+import os
+from typing import Dict, List, Optional, Tuple
+
+from twisted.conch.openssh_compat import primes
+from twisted.conch.ssh import common, factory, keys
+from twisted.python.util import runAsEffectiveUser
+
+
+class OpenSSHFactory(factory.SSHFactory):
+    dataRoot = "/usr/local/etc"
+    # For openbsd which puts moduli in a different directory from keys.
+    moduliRoot = "/usr/local/etc"
+
+    def getPublicKeys(self):
+        """
+        Return the server public keys.
+        """
+        ks = {}
+        for filename in os.listdir(self.dataRoot):
+            if filename[:9] == "ssh_host_" and filename[-8:] == "_key.pub":
+                try:
+                    k = keys.Key.fromFile(os.path.join(self.dataRoot, filename))
+                    t = common.getNS(k.blob())[0]
+                    ks[t] = k
+                except Exception as e:
+                    self._log.error(
+                        "bad public key file {filename}: {error}",
+                        filename=filename,
+                        error=e,
+                    )
+        return ks
+
+    def getPrivateKeys(self):
+        """
+        Return the server private keys.
+        """
+        privateKeys = {}
+        for filename in os.listdir(self.dataRoot):
+            if filename[:9] == "ssh_host_" and filename[-4:] == "_key":
+                fullPath = os.path.join(self.dataRoot, filename)
+                try:
+                    key = keys.Key.fromFile(fullPath)
+                except OSError as e:
+                    if e.errno == errno.EACCES:
+                        # Not allowed, let's switch to root
+                        key = runAsEffectiveUser(0, 0, keys.Key.fromFile, fullPath)
+                        privateKeys[key.sshType()] = key
+                    else:
+                        raise
+                except Exception as e:
+                    self._log.error(
+                        "bad public key file {filename}: {error}",
+                        filename=filename,
+                        error=e,
+                    )
+                else:
+                    privateKeys[key.sshType()] = key
+        return privateKeys
+
+    def getPrimes(self) -> Optional[Dict[int, List[Tuple[int, int]]]]:
+        try:
+            return primes.parseModuliFile(self.moduliRoot + "/moduli")
+        except OSError:
+            return None
diff --git a/contrib/python/Twisted/py3/twisted/conch/openssh_compat/primes.py b/contrib/python/Twisted/py3/twisted/conch/openssh_compat/primes.py
new file mode 100644
index 00000000000..9e2070e19aa
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/openssh_compat/primes.py
@@ -0,0 +1,31 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+"""
+Parsing for the moduli file, which contains Diffie-Hellman prime groups.
+
+Maintainer: Paul Swartz
+"""
+
+
+from typing import Dict, List, Tuple
+
+
+def parseModuliFile(filename: str) -> Dict[int, List[Tuple[int, int]]]:
+    with open(filename) as f:
+        lines = f.readlines()
+    primes: Dict[int, List[Tuple[int, int]]] = {}
+    for l in lines:
+        l = l.strip()
+        if not l or l[0] == "#":
+            continue
+        tim, typ, tst, tri, sizestr, genstr, modstr = l.split()
+        size = int(sizestr) + 1
+        gen = int(genstr)
+        mod = int(modstr, 16)
+        if size not in primes:
+            primes[size] = []
+        primes[size].append((gen, mod))
+    return primes
diff --git a/contrib/python/Twisted/py3/twisted/conch/recvline.py b/contrib/python/Twisted/py3/twisted/conch/recvline.py
new file mode 100644
index 00000000000..aa8115bdc7a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/recvline.py
@@ -0,0 +1,569 @@
+# -*- test-case-name: twisted.conch.test.test_recvline -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Basic line editing support.
+
+@author: Jp Calderone
+"""
+
+import string
+from typing import Dict
+
+from zope.interface import implementer
+
+from twisted.conch.insults import helper, insults
+from twisted.logger import Logger
+from twisted.python import reflect
+from twisted.python.compat import iterbytes
+
+_counters: Dict[str, int] = {}
+
+
+class Logging:
+    """
+    Wrapper which logs attribute lookups.
+
+    This was useful in debugging something, I guess.  I forget what.
+    It can probably be deleted or moved somewhere more appropriate.
+    Nothing special going on here, really.
+    """
+
+    def __init__(self, original):
+        self.original = original
+        key = reflect.qual(original.__class__)
+        count = _counters.get(key, 0)
+        _counters[key] = count + 1
+        self._logFile = open(key + "-" + str(count), "w")
+
+    def __str__(self) -> str:
+        return str(super().__getattribute__("original"))
+
+    def __repr__(self) -> str:
+        return repr(super().__getattribute__("original"))
+
+    def __getattribute__(self, name):
+        original = super().__getattribute__("original")
+        logFile = super().__getattribute__("_logFile")
+        logFile.write(name + "\n")
+        return getattr(original, name)
+
+
+@implementer(insults.ITerminalTransport)
+class TransportSequence:
+    """
+    An L{ITerminalTransport} implementation which forwards calls to
+    one or more other L{ITerminalTransport}s.
+
+    This is a cheap way for servers to keep track of the state they
+    expect the client to see, since all terminal manipulations can be
+    send to the real client and to a terminal emulator that lives in
+    the server process.
+    """
+
+    for keyID in (
+        b"UP_ARROW",
+        b"DOWN_ARROW",
+        b"RIGHT_ARROW",
+        b"LEFT_ARROW",
+        b"HOME",
+        b"INSERT",
+        b"DELETE",
+        b"END",
+        b"PGUP",
+        b"PGDN",
+        b"F1",
+        b"F2",
+        b"F3",
+        b"F4",
+        b"F5",
+        b"F6",
+        b"F7",
+        b"F8",
+        b"F9",
+        b"F10",
+        b"F11",
+        b"F12",
+    ):
+        execBytes = keyID + b" = object()"
+        execStr = execBytes.decode("ascii")
+        exec(execStr)
+
+    TAB = b"\t"
+    BACKSPACE = b"\x7f"
+
+    def __init__(self, *transports):
+        assert transports, "Cannot construct a TransportSequence with no transports"
+        self.transports = transports
+
+    for method in insults.ITerminalTransport:
+        exec(
+            """\
+def %s(self, *a, **kw):
+    for tpt in self.transports:
+        result = tpt.%s(*a, **kw)
+    return result
+"""
+            % (method, method)
+        )
+
+    def getHost(self):
+        # ITransport.getHost
+        raise NotImplementedError("Unimplemented: TransportSequence.getHost")
+
+    def getPeer(self):
+        # ITransport.getPeer
+        raise NotImplementedError("Unimplemented: TransportSequence.getPeer")
+
+    def loseConnection(self):
+        # ITransport.loseConnection
+        raise NotImplementedError("Unimplemented: TransportSequence.loseConnection")
+
+    def write(self, data):
+        # ITransport.write
+        raise NotImplementedError("Unimplemented: TransportSequence.write")
+
+    def writeSequence(self, data):
+        # ITransport.writeSequence
+        raise NotImplementedError("Unimplemented: TransportSequence.writeSequence")
+
+    def cursorUp(self, n=1):
+        # ITerminalTransport.cursorUp
+        raise NotImplementedError("Unimplemented: TransportSequence.cursorUp")
+
+    def cursorDown(self, n=1):
+        # ITerminalTransport.cursorDown
+        raise NotImplementedError("Unimplemented: TransportSequence.cursorDown")
+
+    def cursorForward(self, n=1):
+        # ITerminalTransport.cursorForward
+        raise NotImplementedError("Unimplemented: TransportSequence.cursorForward")
+
+    def cursorBackward(self, n=1):
+        # ITerminalTransport.cursorBackward
+        raise NotImplementedError("Unimplemented: TransportSequence.cursorBackward")
+
+    def cursorPosition(self, column, line):
+        # ITerminalTransport.cursorPosition
+        raise NotImplementedError("Unimplemented: TransportSequence.cursorPosition")
+
+    def cursorHome(self):
+        # ITerminalTransport.cursorHome
+        raise NotImplementedError("Unimplemented: TransportSequence.cursorHome")
+
+    def index(self):
+        # ITerminalTransport.index
+        raise NotImplementedError("Unimplemented: TransportSequence.index")
+
+    def reverseIndex(self):
+        # ITerminalTransport.reverseIndex
+        raise NotImplementedError("Unimplemented: TransportSequence.reverseIndex")
+
+    def nextLine(self):
+        # ITerminalTransport.nextLine
+        raise NotImplementedError("Unimplemented: TransportSequence.nextLine")
+
+    def saveCursor(self):
+        # ITerminalTransport.saveCursor
+        raise NotImplementedError("Unimplemented: TransportSequence.saveCursor")
+
+    def restoreCursor(self):
+        # ITerminalTransport.restoreCursor
+        raise NotImplementedError("Unimplemented: TransportSequence.restoreCursor")
+
+    def setModes(self, modes):
+        # ITerminalTransport.setModes
+        raise NotImplementedError("Unimplemented: TransportSequence.setModes")
+
+    def resetModes(self, mode):
+        # ITerminalTransport.resetModes
+        raise NotImplementedError("Unimplemented: TransportSequence.resetModes")
+
+    def setPrivateModes(self, modes):
+        # ITerminalTransport.setPrivateModes
+        raise NotImplementedError("Unimplemented: TransportSequence.setPrivateModes")
+
+    def resetPrivateModes(self, modes):
+        # ITerminalTransport.resetPrivateModes
+        raise NotImplementedError("Unimplemented: TransportSequence.resetPrivateModes")
+
+    def applicationKeypadMode(self):
+        # ITerminalTransport.applicationKeypadMode
+        raise NotImplementedError(
+            "Unimplemented: TransportSequence.applicationKeypadMode"
+        )
+
+    def numericKeypadMode(self):
+        # ITerminalTransport.numericKeypadMode
+        raise NotImplementedError("Unimplemented: TransportSequence.numericKeypadMode")
+
+    def selectCharacterSet(self, charSet, which):
+        # ITerminalTransport.selectCharacterSet
+        raise NotImplementedError("Unimplemented: TransportSequence.selectCharacterSet")
+
+    def shiftIn(self):
+        # ITerminalTransport.shiftIn
+        raise NotImplementedError("Unimplemented: TransportSequence.shiftIn")
+
+    def shiftOut(self):
+        # ITerminalTransport.shiftOut
+        raise NotImplementedError("Unimplemented: TransportSequence.shiftOut")
+
+    def singleShift2(self):
+        # ITerminalTransport.singleShift2
+        raise NotImplementedError("Unimplemented: TransportSequence.singleShift2")
+
+    def singleShift3(self):
+        # ITerminalTransport.singleShift3
+        raise NotImplementedError("Unimplemented: TransportSequence.singleShift3")
+
+    def selectGraphicRendition(self, *attributes):
+        # ITerminalTransport.selectGraphicRendition
+        raise NotImplementedError(
+            "Unimplemented: TransportSequence.selectGraphicRendition"
+        )
+
+    def horizontalTabulationSet(self):
+        # ITerminalTransport.horizontalTabulationSet
+        raise NotImplementedError(
+            "Unimplemented: TransportSequence.horizontalTabulationSet"
+        )
+
+    def tabulationClear(self):
+        # ITerminalTransport.tabulationClear
+        raise NotImplementedError("Unimplemented: TransportSequence.tabulationClear")
+
+    def tabulationClearAll(self):
+        # ITerminalTransport.tabulationClearAll
+        raise NotImplementedError("Unimplemented: TransportSequence.tabulationClearAll")
+
+    def doubleHeightLine(self, top=True):
+        # ITerminalTransport.doubleHeightLine
+        raise NotImplementedError("Unimplemented: TransportSequence.doubleHeightLine")
+
+    def singleWidthLine(self):
+        # ITerminalTransport.singleWidthLine
+        raise NotImplementedError("Unimplemented: TransportSequence.singleWidthLine")
+
+    def doubleWidthLine(self):
+        # ITerminalTransport.doubleWidthLine
+        raise NotImplementedError("Unimplemented: TransportSequence.doubleWidthLine")
+
+    def eraseToLineEnd(self):
+        # ITerminalTransport.eraseToLineEnd
+        raise NotImplementedError("Unimplemented: TransportSequence.eraseToLineEnd")
+
+    def eraseToLineBeginning(self):
+        # ITerminalTransport.eraseToLineBeginning
+        raise NotImplementedError(
+            "Unimplemented: TransportSequence.eraseToLineBeginning"
+        )
+
+    def eraseLine(self):
+        # ITerminalTransport.eraseLine
+        raise NotImplementedError("Unimplemented: TransportSequence.eraseLine")
+
+    def eraseToDisplayEnd(self):
+        # ITerminalTransport.eraseToDisplayEnd
+        raise NotImplementedError("Unimplemented: TransportSequence.eraseToDisplayEnd")
+
+    def eraseToDisplayBeginning(self):
+        # ITerminalTransport.eraseToDisplayBeginning
+        raise NotImplementedError(
+            "Unimplemented: TransportSequence.eraseToDisplayBeginning"
+        )
+
+    def eraseDisplay(self):
+        # ITerminalTransport.eraseDisplay
+        raise NotImplementedError("Unimplemented: TransportSequence.eraseDisplay")
+
+    def deleteCharacter(self, n=1):
+        # ITerminalTransport.deleteCharacter
+        raise NotImplementedError("Unimplemented: TransportSequence.deleteCharacter")
+
+    def insertLine(self, n=1):
+        # ITerminalTransport.insertLine
+        raise NotImplementedError("Unimplemented: TransportSequence.insertLine")
+
+    def deleteLine(self, n=1):
+        # ITerminalTransport.deleteLine
+        raise NotImplementedError("Unimplemented: TransportSequence.deleteLine")
+
+    def reportCursorPosition(self):
+        # ITerminalTransport.reportCursorPosition
+        raise NotImplementedError(
+            "Unimplemented: TransportSequence.reportCursorPosition"
+        )
+
+    def reset(self):
+        # ITerminalTransport.reset
+        raise NotImplementedError("Unimplemented: TransportSequence.reset")
+
+    def unhandledControlSequence(self, seq):
+        # ITerminalTransport.unhandledControlSequence
+        raise NotImplementedError(
+            "Unimplemented: TransportSequence.unhandledControlSequence"
+        )
+
+
+class LocalTerminalBufferMixin:
+    """
+    A mixin for RecvLine subclasses which records the state of the terminal.
+
+    This is accomplished by performing all L{ITerminalTransport} operations on both
+    the transport passed to makeConnection and an instance of helper.TerminalBuffer.
+
+    @ivar terminalCopy: A L{helper.TerminalBuffer} instance which efforts
+    will be made to keep up to date with the actual terminal
+    associated with this protocol instance.
+    """
+
+    def makeConnection(self, transport):
+        self.terminalCopy = helper.TerminalBuffer()
+        self.terminalCopy.connectionMade()
+        return super().makeConnection(TransportSequence(transport, self.terminalCopy))
+
+    def __str__(self) -> str:
+        return str(self.terminalCopy)
+
+
+class RecvLine(insults.TerminalProtocol):
+    """
+    L{TerminalProtocol} which adds line editing features.
+
+    Clients will be prompted for lines of input with all the usual
+    features: character echoing, left and right arrow support for
+    moving the cursor to different areas of the line buffer, backspace
+    and delete for removing characters, and insert for toggling
+    between typeover and insert mode.  Tabs will be expanded to enough
+    spaces to move the cursor to the next tabstop (every four
+    characters by default).  Enter causes the line buffer to be
+    cleared and the line to be passed to the lineReceived() method
+    which, by default, does nothing.  Subclasses are responsible for
+    redrawing the input prompt (this will probably change).
+    """
+
+    width = 80
+    height = 24
+
+    TABSTOP = 4
+
+    ps = (b">>> ", b"... ")
+    pn = 0
+    _printableChars = string.printable.encode("ascii")
+
+    _log = Logger()
+
+    def connectionMade(self):
+        # A list containing the characters making up the current line
+        self.lineBuffer = []
+
+        # A zero-based (wtf else?) index into self.lineBuffer.
+        # Indicates the current cursor position.
+        self.lineBufferIndex = 0
+
+        t = self.terminal
+        # A map of keyIDs to bound instance methods.
+        self.keyHandlers = {
+            t.LEFT_ARROW: self.handle_LEFT,
+            t.RIGHT_ARROW: self.handle_RIGHT,
+            t.TAB: self.handle_TAB,
+            # Both of these should not be necessary, but figuring out
+            # which is necessary is a huge hassle.
+            b"\r": self.handle_RETURN,
+            b"\n": self.handle_RETURN,
+            t.BACKSPACE: self.handle_BACKSPACE,
+            t.DELETE: self.handle_DELETE,
+            t.INSERT: self.handle_INSERT,
+            t.HOME: self.handle_HOME,
+            t.END: self.handle_END,
+        }
+
+        self.initializeScreen()
+
+    def initializeScreen(self):
+        # Hmm, state sucks.  Oh well.
+        # For now we will just take over the whole terminal.
+        self.terminal.reset()
+        self.terminal.write(self.ps[self.pn])
+        # XXX Note: I would prefer to default to starting in insert
+        # mode, however this does not seem to actually work!  I do not
+        # know why.  This is probably of interest to implementors
+        # subclassing RecvLine.
+
+        # XXX XXX Note: But the unit tests all expect the initial mode
+        # to be insert right now.  Fuck, there needs to be a way to
+        # query the current mode or something.
+        # self.setTypeoverMode()
+        self.setInsertMode()
+
+    def currentLineBuffer(self):
+        s = b"".join(self.lineBuffer)
+        return s[: self.lineBufferIndex], s[self.lineBufferIndex :]
+
+    def setInsertMode(self):
+        self.mode = "insert"
+        self.terminal.setModes([insults.modes.IRM])
+
+    def setTypeoverMode(self):
+        self.mode = "typeover"
+        self.terminal.resetModes([insults.modes.IRM])
+
+    def drawInputLine(self):
+        """
+        Write a line containing the current input prompt and the current line
+        buffer at the current cursor position.
+        """
+        self.terminal.write(self.ps[self.pn] + b"".join(self.lineBuffer))
+
+    def terminalSize(self, width, height):
+        # XXX - Clear the previous input line, redraw it at the new
+        # cursor position
+        self.terminal.eraseDisplay()
+        self.terminal.cursorHome()
+        self.width = width
+        self.height = height
+        self.drawInputLine()
+
+    def unhandledControlSequence(self, seq):
+        pass
+
+    def keystrokeReceived(self, keyID, modifier):
+        m = self.keyHandlers.get(keyID)
+        if m is not None:
+            m()
+        elif keyID in self._printableChars:
+            self.characterReceived(keyID, False)
+        else:
+            self._log.warn("Received unhandled keyID: {keyID!r}", keyID=keyID)
+
+    def characterReceived(self, ch, moreCharactersComing):
+        if self.mode == "insert":
+            self.lineBuffer.insert(self.lineBufferIndex, ch)
+        else:
+            self.lineBuffer[self.lineBufferIndex : self.lineBufferIndex + 1] = [ch]
+        self.lineBufferIndex += 1
+        self.terminal.write(ch)
+
+    def handle_TAB(self):
+        n = self.TABSTOP - (len(self.lineBuffer) % self.TABSTOP)
+        self.terminal.cursorForward(n)
+        self.lineBufferIndex += n
+        self.lineBuffer.extend(iterbytes(b" " * n))
+
+    def handle_LEFT(self):
+        if self.lineBufferIndex > 0:
+            self.lineBufferIndex -= 1
+            self.terminal.cursorBackward()
+
+    def handle_RIGHT(self):
+        if self.lineBufferIndex < len(self.lineBuffer):
+            self.lineBufferIndex += 1
+            self.terminal.cursorForward()
+
+    def handle_HOME(self):
+        if self.lineBufferIndex:
+            self.terminal.cursorBackward(self.lineBufferIndex)
+            self.lineBufferIndex = 0
+
+    def handle_END(self):
+        offset = len(self.lineBuffer) - self.lineBufferIndex
+        if offset:
+            self.terminal.cursorForward(offset)
+            self.lineBufferIndex = len(self.lineBuffer)
+
+    def handle_BACKSPACE(self):
+        if self.lineBufferIndex > 0:
+            self.lineBufferIndex -= 1
+            del self.lineBuffer[self.lineBufferIndex]
+            self.terminal.cursorBackward()
+            self.terminal.deleteCharacter()
+
+    def handle_DELETE(self):
+        if self.lineBufferIndex < len(self.lineBuffer):
+            del self.lineBuffer[self.lineBufferIndex]
+            self.terminal.deleteCharacter()
+
+    def handle_RETURN(self):
+        line = b"".join(self.lineBuffer)
+        self.lineBuffer = []
+        self.lineBufferIndex = 0
+        self.terminal.nextLine()
+        self.lineReceived(line)
+
+    def handle_INSERT(self):
+        assert self.mode in ("typeover", "insert")
+        if self.mode == "typeover":
+            self.setInsertMode()
+        else:
+            self.setTypeoverMode()
+
+    def lineReceived(self, line):
+        pass
+
+
+class HistoricRecvLine(RecvLine):
+    """
+    L{TerminalProtocol} which adds both basic line-editing features and input history.
+
+    Everything supported by L{RecvLine} is also supported by this class.  In addition, the
+    up and down arrows traverse the input history.  Each received line is automatically
+    added to the end of the input history.
+    """
+
+    def connectionMade(self):
+        RecvLine.connectionMade(self)
+
+        self.historyLines = []
+        self.historyPosition = 0
+
+        t = self.terminal
+        self.keyHandlers.update(
+            {t.UP_ARROW: self.handle_UP, t.DOWN_ARROW: self.handle_DOWN}
+        )
+
+    def currentHistoryBuffer(self):
+        b = tuple(self.historyLines)
+        return b[: self.historyPosition], b[self.historyPosition :]
+
+    def _deliverBuffer(self, buf):
+        if buf:
+            for ch in iterbytes(buf[:-1]):
+                self.characterReceived(ch, True)
+            self.characterReceived(buf[-1:], False)
+
+    def handle_UP(self):
+        if self.lineBuffer and self.historyPosition == len(self.historyLines):
+            self.historyLines.append(b"".join(self.lineBuffer))
+        if self.historyPosition > 0:
+            self.handle_HOME()
+            self.terminal.eraseToLineEnd()
+
+            self.historyPosition -= 1
+            self.lineBuffer = []
+
+            self._deliverBuffer(self.historyLines[self.historyPosition])
+
+    def handle_DOWN(self):
+        if self.historyPosition < len(self.historyLines) - 1:
+            self.handle_HOME()
+            self.terminal.eraseToLineEnd()
+
+            self.historyPosition += 1
+            self.lineBuffer = []
+
+            self._deliverBuffer(self.historyLines[self.historyPosition])
+        else:
+            self.handle_HOME()
+            self.terminal.eraseToLineEnd()
+
+            self.historyPosition = len(self.historyLines)
+            self.lineBuffer = []
+            self.lineBufferIndex = 0
+
+    def handle_RETURN(self):
+        if self.lineBuffer:
+            self.historyLines.append(b"".join(self.lineBuffer))
+        self.historyPosition = len(self.historyLines)
+        return RecvLine.handle_RETURN(self)
diff --git a/contrib/python/Twisted/py3/twisted/conch/scripts/__init__.py b/contrib/python/Twisted/py3/twisted/conch/scripts/__init__.py
new file mode 100644
index 00000000000..b39c2b748a9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/scripts/__init__.py
@@ -0,0 +1 @@
+"conch scripts"
diff --git a/contrib/python/Twisted/py3/twisted/conch/scripts/cftp.py b/contrib/python/Twisted/py3/twisted/conch/scripts/cftp.py
new file mode 100644
index 00000000000..e8241fdc3ad
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/scripts/cftp.py
@@ -0,0 +1,1002 @@
+# -*- test-case-name: twisted.conch.test.test_cftp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation module for the I{cftp} command.
+"""
+import fcntl
+import fnmatch
+import getpass
+import glob
+import os
+import pwd
+import stat
+import struct
+import sys
+import tty
+from typing import List, Optional, TextIO, Union
+
+from twisted.conch.client import connect, default, options
+from twisted.conch.ssh import channel, common, connection, filetransfer
+from twisted.internet import defer, reactor, stdio, utils
+from twisted.protocols import basic
+from twisted.python import failure, log, usage
+from twisted.python.filepath import FilePath
+
+
+class ClientOptions(options.ConchOptions):
+    synopsis = """Usage:   cftp [options] [user@]host
+         cftp [options] [user@]host[:dir[/]]
+         cftp [options] [user@]host[:file [localfile]]
+"""
+    longdesc = (
+        "cftp is a client for logging into a remote machine and "
+        "executing commands to send and receive file information"
+    )
+
+    optParameters: List[List[Optional[Union[str, int]]]] = [
+        ["buffersize", "B", 32768, "Size of the buffer to use for sending/receiving."],
+        ["batchfile", "b", None, "File to read commands from, or '-' for stdin."],
+        ["requests", "R", 5, "Number of requests to make before waiting for a reply."],
+        ["subsystem", "s", "sftp", "Subsystem/server program to connect to."],
+    ]
+
+    compData = usage.Completions(
+        descriptions={"buffersize": "Size of send/receive buffer (default: 32768)"},
+        extraActions=[
+            usage.CompleteUserAtHost(),
+            usage.CompleteFiles(descr="local file"),
+        ],
+    )
+
+    def parseArgs(self, host, localPath=None):
+        self["remotePath"] = ""
+        if ":" in host:
+            host, self["remotePath"] = host.split(":", 1)
+            self["remotePath"].rstrip("/")
+        self["host"] = host
+        self["localPath"] = localPath
+
+
+def run():
+    args = sys.argv[1:]
+    if "-l" in args:  # cvs is an idiot
+        i = args.index("-l")
+        args = args[i : i + 2] + args
+        del args[i + 2 : i + 4]
+    options = ClientOptions()
+    try:
+        options.parseOptions(args)
+    except usage.UsageError as u:
+        print("ERROR: %s" % u)
+        sys.exit(1)
+    if options["log"]:
+        realout = sys.stdout
+        log.startLogging(sys.stderr)
+        sys.stdout = realout
+    else:
+        log.discardLogs()
+    doConnect(options)
+    reactor.run()
+
+
+def handleError():
+    global exitStatus
+    exitStatus = 2
+    try:
+        reactor.stop()
+    except BaseException:
+        pass
+    log.err(failure.Failure())
+    raise
+
+
+def doConnect(options):
+    if "@" in options["host"]:
+        options["user"], options["host"] = options["host"].split("@", 1)
+    host = options["host"]
+    if not options["user"]:
+        options["user"] = getpass.getuser()
+    if not options["port"]:
+        options["port"] = 22
+    else:
+        options["port"] = int(options["port"])
+    host = options["host"]
+    port = options["port"]
+    conn = SSHConnection()
+    conn.options = options
+    vhk = default.verifyHostKey
+    uao = default.SSHUserAuthClient(options["user"], options, conn)
+    connect.connect(host, port, options, vhk, uao).addErrback(_ebExit)
+
+
+def _ebExit(f):
+    if hasattr(f.value, "value"):
+        s = f.value.value
+    else:
+        s = str(f)
+    print(s)
+    try:
+        reactor.stop()
+    except BaseException:
+        pass
+
+
+def _ignore(*args):
+    pass
+
+
+class FileWrapper:
+    def __init__(self, f):
+        self.f = f
+        self.total = 0.0
+        f.seek(0, 2)  # seek to the end
+        self.size = f.tell()
+
+    def __getattr__(self, attr):
+        return getattr(self.f, attr)
+
+
+class StdioClient(basic.LineReceiver):
+    _pwd = pwd
+
+    ps = "cftp> "
+    delimiter = b"\n"
+
+    reactor = reactor
+
+    def __init__(self, client, f=None):
+        self.client = client
+        self.currentDirectory = ""
+        self.file = f
+        self.useProgressBar = (not f and 1) or 0
+
+    def connectionMade(self):
+        self.client.realPath("").addCallback(self._cbSetCurDir)
+
+    def _cbSetCurDir(self, path):
+        self.currentDirectory = path
+        self._newLine()
+
+    def _writeToTransport(self, msg):
+        if isinstance(msg, str):
+            msg = msg.encode("utf-8")
+        return self.transport.write(msg)
+
+    def lineReceived(self, line):
+        if self.client.transport.localClosed:
+            return
+        if isinstance(line, bytes):
+            line = line.decode("utf-8")
+        log.msg("got line %s" % line)
+        line = line.lstrip()
+        if not line:
+            self._newLine()
+            return
+        if self.file and line.startswith("-"):
+            self.ignoreErrors = 1
+            line = line[1:]
+        else:
+            self.ignoreErrors = 0
+        d = self._dispatchCommand(line)
+        if d is not None:
+            d.addCallback(self._cbCommand)
+            d.addErrback(self._ebCommand)
+
+    def _dispatchCommand(self, line):
+        if " " in line:
+            command, rest = line.split(" ", 1)
+            rest = rest.lstrip()
+        else:
+            command, rest = line, ""
+        if command.startswith("!"):  # command
+            f = self.cmd_EXEC
+            rest = (command[1:] + " " + rest).strip()
+        else:
+            command = command.upper()
+            log.msg("looking up cmd %s" % command)
+            f = getattr(self, "cmd_%s" % command, None)
+        if f is not None:
+            return defer.maybeDeferred(f, rest)
+        else:
+            errMsg = "No command called `%s'" % (command)
+            self._ebCommand(failure.Failure(NotImplementedError(errMsg)))
+            self._newLine()
+
+    def _printFailure(self, f):
+        log.msg(f)
+        e = f.trap(NotImplementedError, filetransfer.SFTPError, OSError, IOError)
+        if e == NotImplementedError:
+            self._writeToTransport(self.cmd_HELP(""))
+        elif e == filetransfer.SFTPError:
+            errMsg = "remote error %i: %s\n" % (f.value.code, f.value.message)
+            self._writeToTransport(errMsg)
+        elif e in (OSError, IOError):
+            errMsg = "local error %i: %s\n" % (f.value.errno, f.value.strerror)
+            self._writeToTransport(errMsg)
+
+    def _newLine(self):
+        if self.client.transport.localClosed:
+            return
+        self._writeToTransport(self.ps)
+        self.ignoreErrors = 0
+        if self.file:
+            l = self.file.readline()
+            if not l:
+                self.client.transport.loseConnection()
+            else:
+                self._writeToTransport(l)
+                self.lineReceived(l.strip())
+
+    def _cbCommand(self, result):
+        if result is not None:
+            if isinstance(result, str):
+                result = result.encode("utf-8")
+            self._writeToTransport(result)
+            if not result.endswith(b"\n"):
+                self._writeToTransport(b"\n")
+        self._newLine()
+
+    def _ebCommand(self, f):
+        self._printFailure(f)
+        if self.file and not self.ignoreErrors:
+            self.client.transport.loseConnection()
+        self._newLine()
+
+    def cmd_CD(self, path):
+        path, rest = self._getFilename(path)
+        if not path.endswith("/"):
+            path += "/"
+        newPath = path and os.path.join(self.currentDirectory, path) or ""
+        d = self.client.openDirectory(newPath)
+        d.addCallback(self._cbCd)
+        d.addErrback(self._ebCommand)
+        return d
+
+    def _cbCd(self, directory):
+        directory.close()
+        d = self.client.realPath(directory.name)
+        d.addCallback(self._cbCurDir)
+        return d
+
+    def _cbCurDir(self, path):
+        self.currentDirectory = path
+
+    def cmd_CHGRP(self, rest):
+        grp, rest = rest.split(None, 1)
+        path, rest = self._getFilename(rest)
+        grp = int(grp)
+        d = self.client.getAttrs(path)
+        d.addCallback(self._cbSetUsrGrp, path, grp=grp)
+        return d
+
+    def cmd_CHMOD(self, rest):
+        mod, rest = rest.split(None, 1)
+        path, rest = self._getFilename(rest)
+        mod = int(mod, 8)
+        d = self.client.setAttrs(path, {"permissions": mod})
+        d.addCallback(_ignore)
+        return d
+
+    def cmd_CHOWN(self, rest):
+        usr, rest = rest.split(None, 1)
+        path, rest = self._getFilename(rest)
+        usr = int(usr)
+        d = self.client.getAttrs(path)
+        d.addCallback(self._cbSetUsrGrp, path, usr=usr)
+        return d
+
+    def _cbSetUsrGrp(self, attrs, path, usr=None, grp=None):
+        new = {}
+        new["uid"] = (usr is not None) and usr or attrs["uid"]
+        new["gid"] = (grp is not None) and grp or attrs["gid"]
+        d = self.client.setAttrs(path, new)
+        d.addCallback(_ignore)
+        return d
+
+    def cmd_GET(self, rest):
+        remote, rest = self._getFilename(rest)
+        if "*" in remote or "?" in remote:  # wildcard
+            if rest:
+                local, rest = self._getFilename(rest)
+                if not os.path.isdir(local):
+                    return "Wildcard get with non-directory target."
+            else:
+                local = b""
+            d = self._remoteGlob(remote)
+            d.addCallback(self._cbGetMultiple, local)
+            return d
+        if rest:
+            local, rest = self._getFilename(rest)
+        else:
+            local = os.path.split(remote)[1]
+        log.msg((remote, local))
+        lf = open(local, "wb", 0)
+        path = FilePath(self.currentDirectory).child(remote)
+        d = self.client.openFile(path.path, filetransfer.FXF_READ, {})
+        d.addCallback(self._cbGetOpenFile, lf)
+        d.addErrback(self._ebCloseLf, lf)
+        return d
+
+    def _cbGetMultiple(self, files, local):
+        # XXX this can be optimized for times w/o progress bar
+        return self._cbGetMultipleNext(None, files, local)
+
+    def _cbGetMultipleNext(self, res, files, local):
+        if isinstance(res, failure.Failure):
+            self._printFailure(res)
+        elif res:
+            self._writeToTransport(res)
+            if not res.endswith("\n"):
+                self._writeToTransport("\n")
+        if not files:
+            return
+        f = files.pop(0)[0]
+        lf = open(os.path.join(local, os.path.split(f)[1]), "wb", 0)
+        path = FilePath(self.currentDirectory).child(f)
+        d = self.client.openFile(path.path, filetransfer.FXF_READ, {})
+        d.addCallback(self._cbGetOpenFile, lf)
+        d.addErrback(self._ebCloseLf, lf)
+        d.addBoth(self._cbGetMultipleNext, files, local)
+        return d
+
+    def _ebCloseLf(self, f, lf):
+        lf.close()
+        return f
+
+    def _cbGetOpenFile(self, rf, lf):
+        return rf.getAttrs().addCallback(self._cbGetFileSize, rf, lf)
+
+    def _cbGetFileSize(self, attrs, rf, lf):
+        if not stat.S_ISREG(attrs["permissions"]):
+            rf.close()
+            lf.close()
+            return "Can't get non-regular file: %s" % rf.name
+        rf.size = attrs["size"]
+        bufferSize = self.client.transport.conn.options["buffersize"]
+        numRequests = self.client.transport.conn.options["requests"]
+        rf.total = 0.0
+        dList = []
+        chunks = []
+        startTime = self.reactor.seconds()
+        for i in range(numRequests):
+            d = self._cbGetRead("", rf, lf, chunks, 0, bufferSize, startTime)
+            dList.append(d)
+        dl = defer.DeferredList(dList, fireOnOneErrback=1)
+        dl.addCallback(self._cbGetDone, rf, lf)
+        return dl
+
+    def _getNextChunk(self, chunks):
+        end = 0
+        for chunk in chunks:
+            if end == "eof":
+                return  # nothing more to get
+            if end != chunk[0]:
+                i = chunks.index(chunk)
+                chunks.insert(i, (end, chunk[0]))
+                return (end, chunk[0] - end)
+            end = chunk[1]
+        bufSize = int(self.client.transport.conn.options["buffersize"])
+        chunks.append((end, end + bufSize))
+        return (end, bufSize)
+
+    def _cbGetRead(self, data, rf, lf, chunks, start, size, startTime):
+        if data and isinstance(data, failure.Failure):
+            log.msg("get read err: %s" % data)
+            reason = data
+            reason.trap(EOFError)
+            i = chunks.index((start, start + size))
+            del chunks[i]
+            chunks.insert(i, (start, "eof"))
+        elif data:
+            log.msg("get read data: %i" % len(data))
+            lf.seek(start)
+            lf.write(data)
+            if len(data) != size:
+                log.msg("got less than we asked for: %i < %i" % (len(data), size))
+                i = chunks.index((start, start + size))
+                del chunks[i]
+                chunks.insert(i, (start, start + len(data)))
+            rf.total += len(data)
+        if self.useProgressBar:
+            self._printProgressBar(rf, startTime)
+        chunk = self._getNextChunk(chunks)
+        if not chunk:
+            return
+        else:
+            start, length = chunk
+        log.msg("asking for %i -> %i" % (start, start + length))
+        d = rf.readChunk(start, length)
+        d.addBoth(self._cbGetRead, rf, lf, chunks, start, length, startTime)
+        return d
+
+    def _cbGetDone(self, ignored, rf, lf):
+        log.msg("get done")
+        rf.close()
+        lf.close()
+        if self.useProgressBar:
+            self._writeToTransport("\n")
+        return f"Transferred {rf.name} to {lf.name}"
+
+    def cmd_PUT(self, rest):
+        """
+        Do an upload request for a single local file or a globing expression.
+
+        @param rest: Requested command line for the PUT command.
+        @type rest: L{str}
+
+        @return: A deferred which fires with L{None} when transfer is done.
+        @rtype: L{defer.Deferred}
+        """
+        local, rest = self._getFilename(rest)
+
+        # FIXME: https://twistedmatrix.com/trac/ticket/7241
+        # Use a better check for globbing expression.
+        if "*" in local or "?" in local:
+            if rest:
+                remote, rest = self._getFilename(rest)
+                remote = os.path.join(self.currentDirectory, remote)
+            else:
+                remote = ""
+
+            files = glob.glob(local)
+            return self._putMultipleFiles(files, remote)
+
+        else:
+            if rest:
+                remote, rest = self._getFilename(rest)
+            else:
+                remote = os.path.split(local)[1]
+            return self._putSingleFile(local, remote)
+
+    def _putSingleFile(self, local, remote):
+        """
+        Perform an upload for a single file.
+
+        @param local: Path to local file.
+        @type local: L{str}.
+
+        @param remote: Remote path for the request relative to current working
+            directory.
+        @type remote: L{str}
+
+        @return: A deferred which fires when transfer is done.
+        """
+        return self._cbPutMultipleNext(None, [local], remote, single=True)
+
+    def _putMultipleFiles(self, files, remote):
+        """
+        Perform an upload for a list of local files.
+
+        @param files: List of local files.
+        @type files: C{list} of L{str}.
+
+        @param remote: Remote path for the request relative to current working
+            directory.
+        @type remote: L{str}
+
+        @return: A deferred which fires when transfer is done.
+        """
+        return self._cbPutMultipleNext(None, files, remote)
+
+    def _cbPutMultipleNext(self, previousResult, files, remotePath, single=False):
+        """
+        Perform an upload for the next file in the list of local files.
+
+        @param previousResult: Result form previous file form the list.
+        @type previousResult: L{str}
+
+        @param files: List of local files.
+        @type files: C{list} of L{str}
+
+        @param remotePath: Remote path for the request relative to current
+            working directory.
+        @type remotePath: L{str}
+
+        @param single: A flag which signals if this is a transfer for a single
+            file in which case we use the exact remote path
+        @type single: L{bool}
+
+        @return: A deferred which fires when transfer is done.
+        """
+        if isinstance(previousResult, failure.Failure):
+            self._printFailure(previousResult)
+        elif previousResult:
+            if isinstance(previousResult, str):
+                previousResult = previousResult.encode("utf-8")
+            self._writeToTransport(previousResult)
+            if not previousResult.endswith(b"\n"):
+                self._writeToTransport(b"\n")
+
+        currentFile = None
+        while files and not currentFile:
+            try:
+                currentFile = files.pop(0)
+                localStream = open(currentFile, "rb")
+            except BaseException:
+                self._printFailure(failure.Failure())
+                currentFile = None
+
+        # No more files to transfer.
+        if not currentFile:
+            return None
+
+        if single:
+            remote = remotePath
+        else:
+            name = os.path.split(currentFile)[1]
+            remote = os.path.join(remotePath, name)
+            log.msg((name, remote, remotePath))
+
+        d = self._putRemoteFile(localStream, remote)
+        d.addBoth(self._cbPutMultipleNext, files, remotePath)
+        return d
+
+    def _putRemoteFile(self, localStream, remotePath):
+        """
+        Do an upload request.
+
+        @param localStream: Local stream from where data is read.
+        @type localStream: File like object.
+
+        @param remotePath: Remote path for the request relative to current working directory.
+        @type remotePath: L{str}
+
+        @return: A deferred which fires when transfer is done.
+        """
+        remote = os.path.join(self.currentDirectory, remotePath)
+        flags = filetransfer.FXF_WRITE | filetransfer.FXF_CREAT | filetransfer.FXF_TRUNC
+        d = self.client.openFile(remote, flags, {})
+        d.addCallback(self._cbPutOpenFile, localStream)
+        d.addErrback(self._ebCloseLf, localStream)
+        return d
+
+    def _cbPutOpenFile(self, rf, lf):
+        numRequests = self.client.transport.conn.options["requests"]
+        if self.useProgressBar:
+            lf = FileWrapper(lf)
+        dList = []
+        chunks = []
+        startTime = self.reactor.seconds()
+        for i in range(numRequests):
+            d = self._cbPutWrite(None, rf, lf, chunks, startTime)
+            if d:
+                dList.append(d)
+        dl = defer.DeferredList(dList, fireOnOneErrback=1)
+        dl.addCallback(self._cbPutDone, rf, lf)
+        return dl
+
+    def _cbPutWrite(self, ignored, rf, lf, chunks, startTime):
+        chunk = self._getNextChunk(chunks)
+        start, size = chunk
+        lf.seek(start)
+        data = lf.read(size)
+        if self.useProgressBar:
+            lf.total += len(data)
+            self._printProgressBar(lf, startTime)
+        if data:
+            d = rf.writeChunk(start, data)
+            d.addCallback(self._cbPutWrite, rf, lf, chunks, startTime)
+            return d
+        else:
+            return
+
+    def _cbPutDone(self, ignored, rf, lf):
+        lf.close()
+        rf.close()
+        if self.useProgressBar:
+            self._writeToTransport("\n")
+        return f"Transferred {lf.name} to {rf.name}"
+
+    def cmd_LCD(self, path):
+        os.chdir(path)
+
+    def cmd_LN(self, rest):
+        linkpath, rest = self._getFilename(rest)
+        targetpath, rest = self._getFilename(rest)
+        linkpath, targetpath = map(
+            lambda x: os.path.join(self.currentDirectory, x), (linkpath, targetpath)
+        )
+        return self.client.makeLink(linkpath, targetpath).addCallback(_ignore)
+
+    def cmd_LS(self, rest):
+        # possible lines:
+        # ls                    current directory
+        # ls name_of_file       that file
+        # ls name_of_directory  that directory
+        # ls some_glob_string   current directory, globbed for that string
+        options = []
+        rest = rest.split()
+        while rest and rest[0] and rest[0][0] == "-":
+            opts = rest.pop(0)[1:]
+            for o in opts:
+                if o == "l":
+                    options.append("verbose")
+                elif o == "a":
+                    options.append("all")
+        rest = " ".join(rest)
+        path, rest = self._getFilename(rest)
+        if not path:
+            fullPath = self.currentDirectory + "/"
+        else:
+            fullPath = os.path.join(self.currentDirectory, path)
+        d = self._remoteGlob(fullPath)
+        d.addCallback(self._cbDisplayFiles, options)
+        return d
+
+    def _cbDisplayFiles(self, files, options):
+        files.sort()
+        if "all" not in options:
+            files = [f for f in files if not f[0].startswith(b".")]
+        if "verbose" in options:
+            lines = [f[1] for f in files]
+        else:
+            lines = [f[0] for f in files]
+        if not lines:
+            return None
+        else:
+            return b"\n".join(lines)
+
+    def cmd_MKDIR(self, path):
+        path, rest = self._getFilename(path)
+        path = os.path.join(self.currentDirectory, path)
+        return self.client.makeDirectory(path, {}).addCallback(_ignore)
+
+    def cmd_RMDIR(self, path):
+        path, rest = self._getFilename(path)
+        path = os.path.join(self.currentDirectory, path)
+        return self.client.removeDirectory(path).addCallback(_ignore)
+
+    def cmd_LMKDIR(self, path):
+        os.system("mkdir %s" % path)
+
+    def cmd_RM(self, path):
+        path, rest = self._getFilename(path)
+        path = os.path.join(self.currentDirectory, path)
+        return self.client.removeFile(path).addCallback(_ignore)
+
+    def cmd_LLS(self, rest):
+        os.system("ls %s" % rest)
+
+    def cmd_RENAME(self, rest):
+        oldpath, rest = self._getFilename(rest)
+        newpath, rest = self._getFilename(rest)
+        oldpath, newpath = map(
+            lambda x: os.path.join(self.currentDirectory, x), (oldpath, newpath)
+        )
+        return self.client.renameFile(oldpath, newpath).addCallback(_ignore)
+
+    def cmd_EXIT(self, ignored):
+        self.client.transport.loseConnection()
+
+    cmd_QUIT = cmd_EXIT
+
+    def cmd_VERSION(self, ignored):
+        version = "SFTP version %i" % self.client.version
+        if isinstance(version, str):
+            version = version.encode("utf-8")
+        return version
+
+    def cmd_HELP(self, ignored):
+        return """Available commands:
+cd path                         Change remote directory to 'path'.
+chgrp gid path                  Change gid of 'path' to 'gid'.
+chmod mode path                 Change mode of 'path' to 'mode'.
+chown uid path                  Change uid of 'path' to 'uid'.
+exit                            Disconnect from the server.
+get remote-path [local-path]    Get remote file.
+help                            Get a list of available commands.
+lcd path                        Change local directory to 'path'.
+lls [ls-options] [path]         Display local directory listing.
+lmkdir path                     Create local directory.
+ln linkpath targetpath          Symlink remote file.
+lpwd                            Print the local working directory.
+ls [-l] [path]                  Display remote directory listing.
+mkdir path                      Create remote directory.
+progress                        Toggle progress bar.
+put local-path [remote-path]    Put local file.
+pwd                             Print the remote working directory.
+quit                            Disconnect from the server.
+rename oldpath newpath          Rename remote file.
+rmdir path                      Remove remote directory.
+rm path                         Remove remote file.
+version                         Print the SFTP version.
+?                               Synonym for 'help'.
+"""
+
+    def cmd_PWD(self, ignored):
+        return self.currentDirectory
+
+    def cmd_LPWD(self, ignored):
+        return os.getcwd()
+
+    def cmd_PROGRESS(self, ignored):
+        self.useProgressBar = not self.useProgressBar
+        return "%ssing progess bar." % (self.useProgressBar and "U" or "Not u")
+
+    def cmd_EXEC(self, rest):
+        """
+        Run C{rest} using the user's shell (or /bin/sh if they do not have
+        one).
+        """
+        shell = self._pwd.getpwnam(getpass.getuser())[6]
+        if not shell:
+            shell = "/bin/sh"
+        if rest:
+            cmds = ["-c", rest]
+            return utils.getProcessOutput(shell, cmds, errortoo=1)
+        else:
+            os.system(shell)
+
+    # accessory functions
+
+    def _remoteGlob(self, fullPath):
+        log.msg("looking up %s" % fullPath)
+        head, tail = os.path.split(fullPath)
+        if "*" in tail or "?" in tail:
+            glob = 1
+        else:
+            glob = 0
+        if tail and not glob:  # could be file or directory
+            # try directory first
+            d = self.client.openDirectory(fullPath)
+            d.addCallback(self._cbOpenList, "")
+            d.addErrback(self._ebNotADirectory, head, tail)
+        else:
+            d = self.client.openDirectory(head)
+            d.addCallback(self._cbOpenList, tail)
+        return d
+
+    def _cbOpenList(self, directory, glob):
+        files = []
+        d = directory.read()
+        d.addBoth(self._cbReadFile, files, directory, glob)
+        return d
+
+    def _ebNotADirectory(self, reason, path, glob):
+        d = self.client.openDirectory(path)
+        d.addCallback(self._cbOpenList, glob)
+        return d
+
+    def _cbReadFile(self, files, matchedFiles, directory, glob):
+        if not isinstance(files, failure.Failure):
+            if glob:
+                glob = glob.encode("utf-8")
+                matchedFiles.extend([f for f in files if fnmatch.fnmatch(f[0], glob)])
+            else:
+                matchedFiles.extend(files)
+            d = directory.read()
+            d.addBoth(self._cbReadFile, matchedFiles, directory, glob)
+            return d
+        else:
+            reason = files
+            reason.trap(EOFError)
+            directory.close()
+            return matchedFiles
+
+    def _abbrevSize(self, size):
+        # from http://mail.python.org/pipermail/python-list/1999-December/018395.html
+        _abbrevs = [
+            (1 << 50, "PB"),
+            (1 << 40, "TB"),
+            (1 << 30, "GB"),
+            (1 << 20, "MB"),
+            (1 << 10, "kB"),
+            (1, "B"),
+        ]
+
+        for factor, suffix in _abbrevs:
+            if size > factor:
+                break
+        return "%.1f" % (size / factor) + suffix
+
+    def _abbrevTime(self, t):
+        if t > 3600:  # 1 hour
+            hours = int(t / 3600)
+            t -= 3600 * hours
+            mins = int(t / 60)
+            t -= 60 * mins
+            return "%i:%02i:%02i" % (hours, mins, t)
+        else:
+            mins = int(t / 60)
+            t -= 60 * mins
+            return "%02i:%02i" % (mins, t)
+
+    def _printProgressBar(self, f, startTime):
+        """
+        Update a console progress bar on this L{StdioClient}'s transport, based
+        on the difference between the start time of the operation and the
+        current time according to the reactor, and appropriate to the size of
+        the console window.
+
+        @param f: a wrapper around the file which is being written or read
+        @type f: L{FileWrapper}
+
+        @param startTime: The time at which the operation being tracked began.
+        @type startTime: L{float}
+        """
+        diff = self.reactor.seconds() - startTime
+        total = f.total
+        try:
+            winSize = struct.unpack("4H", fcntl.ioctl(0, tty.TIOCGWINSZ, "12345679"))
+        except OSError:
+            winSize = [None, 80]
+        if diff == 0.0:
+            speed = 0.0
+        else:
+            speed = total / diff
+        if speed:
+            timeLeft = (f.size - total) / speed
+        else:
+            timeLeft = 0
+        front = f.name
+        if f.size:
+            percentage = (total / f.size) * 100
+        else:
+            percentage = 100
+        back = "%3i%% %s %sps %s " % (
+            percentage,
+            self._abbrevSize(total),
+            self._abbrevSize(speed),
+            self._abbrevTime(timeLeft),
+        )
+        spaces = (winSize[1] - (len(front) + len(back) + 1)) * " "
+        command = f"\r{front}{spaces}{back}"
+        self._writeToTransport(command)
+
+    def _getFilename(self, line):
+        """
+        Parse line received as command line input and return first filename
+        together with the remaining line.
+
+        @param line: Arguments received from command line input.
+        @type line: L{str}
+
+        @return: Tupple with filename and rest. Return empty values when no path was not found.
+        @rtype: C{tupple}
+        """
+        line = line.strip()
+        if not line:
+            return "", ""
+        if line[0] in "'\"":
+            ret = []
+            line = list(line)
+            try:
+                for i in range(1, len(line)):
+                    c = line[i]
+                    if c == line[0]:
+                        return "".join(ret), "".join(line[i + 1 :]).lstrip()
+                    elif c == "\\":  # quoted character
+                        del line[i]
+                        if line[i] not in "'\"\\":
+                            raise IndexError(f"bad quote: \\{line[i]}")
+                        ret.append(line[i])
+                    else:
+                        ret.append(line[i])
+            except IndexError:
+                raise IndexError("unterminated quote")
+        ret = line.split(None, 1)
+        if len(ret) == 1:
+            return ret[0], ""
+        else:
+            return ret[0], ret[1]
+
+
+setattr(StdioClient, "cmd_?", StdioClient.cmd_HELP)
+
+
+class SSHConnection(connection.SSHConnection):
+    def serviceStarted(self):
+        self.openChannel(SSHSession())
+
+
+class SSHSession(channel.SSHChannel):
+    name: bytes = b"session"
+    stderr: TextIO = sys.stderr
+
+    def channelOpen(self, foo):
+        log.msg("session %s open" % self.id)
+        if self.conn.options["subsystem"].startswith("/"):
+            request = "exec"
+        else:
+            request = "subsystem"
+        d = self.conn.sendRequest(
+            self, request, common.NS(self.conn.options["subsystem"]), wantReply=1
+        )
+        d.addCallback(self._cbSubsystem)
+        d.addErrback(_ebExit)
+
+    def _cbSubsystem(self, result):
+        self.client = filetransfer.FileTransferClient()
+        self.client.makeConnection(self)
+        self.dataReceived = self.client.dataReceived
+        f = None
+        if self.conn.options["batchfile"]:
+            fn = self.conn.options["batchfile"]
+            if fn != "-":
+                f = open(fn)
+        self.stdio = stdio.StandardIO(StdioClient(self.client, f))
+
+    def extReceived(self, t: int, data: bytes) -> None:
+        if t == connection.EXTENDED_DATA_STDERR:
+            log.msg("got %s stderr data" % len(data))
+            # RFC 4251
+            # ========
+            # Strings are also used to store text.  In that case, US-ASCII is
+            # used for internal names, and ISO-10646 UTF-8 for text that might
+            # be displayed to the user.  The terminating null character SHOULD
+            # NOT normally be stored in the string.  For example: the US-ASCII
+            # string "testing" is represented as 00 00 00 07 t e s t i n g.
+            # The UTF-8 mapping does not alter the encoding of US-ASCII
+            # characters.
+            #
+            # RFC 4254
+            # ========
+            # Additionally, some channels can transfer several types of data.  An
+            # example of this is stderr data from interactive sessions.  Such data
+            # can be passed with SSH_MSG_CHANNEL_EXTENDED_DATA messages, where a
+            # separate integer specifies the type of data.  The available types and
+            # their interpretation depend on the type of channel.
+            #
+            #    byte      SSH_MSG_CHANNEL_EXTENDED_DATA
+            #    uint32    recipient channel
+            #    uint32    data_type_code
+            #    string    data
+            #
+            # Data sent with these messages consumes the same window as ordinary
+            # data.
+            #
+            # Currently, only the following type is defined.  Note that the value
+            # for the 'data_type_code' is given in decimal format for readability,
+            # but the values are actually uint32 values.
+            #
+            #             Symbolic name                  data_type_code
+            #             -------------                  --------------
+            #           SSH_EXTENDED_DATA_STDERR               1
+            #
+            # (end of RFC quotations)
+            #
+            # Here we decode the stderr bytes as UTF-8 and handle errors by
+            # representing undecodeable bytes with a certain escape scheme.
+            # There is no guarantee that the peer is sending UTF-8 encoded
+            # bytes but if they are not it is complex to determine what
+            # encoding they _are_ sending.  The standard says nothing about
+            # how these bytes should be decoded because the standard probably
+            # doesn't think they should be decoded at all - just handle them
+            # as bytes!  However, our stderr is a text-mode file so we *must*
+            # decode them to be able to write them out at all.  And even if we
+            # had a binary-mode file we would still /probably/ want to write
+            # bytes in a *known* encoding to it.
+            #
+            # Perhaps in the future we can somehow inspect LANG or LC_* in the
+            # remote execution environment (but I'm not sure how) and use that
+            # as a hint about which encoding to use for decoding here.
+            # Meanwhile, UTF-8 is the de facto universal interoperable
+            # encoding so: use it.
+            self.stderr.write(data.decode("utf-8", "backslashreplace"))
+            self.stderr.flush()
+
+    def eofReceived(self):
+        log.msg("got eof")
+        self.stdio.loseWriteConnection()
+
+    def closeReceived(self):
+        log.msg("remote side closed %s" % self)
+        self.conn.sendClose(self)
+
+    def closed(self):
+        try:
+            reactor.stop()
+        except BaseException:
+            pass
+
+    def stopWriting(self):
+        self.stdio.pauseProducing()
+
+    def startWriting(self):
+        self.stdio.resumeProducing()
+
+
+if __name__ == "__main__":
+    run()
diff --git a/contrib/python/Twisted/py3/twisted/conch/scripts/ckeygen.py b/contrib/python/Twisted/py3/twisted/conch/scripts/ckeygen.py
new file mode 100644
index 00000000000..728dc6430c7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/scripts/ckeygen.py
@@ -0,0 +1,400 @@
+# -*- test-case-name: twisted.conch.test.test_ckeygen -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation module for the `ckeygen` command.
+"""
+from __future__ import annotations
+
+import getpass
+import os
+import platform
+import socket
+import sys
+from collections.abc import Callable
+from functools import wraps
+from importlib import reload
+from typing import Any, Dict, Optional
+
+from twisted.conch.ssh import keys
+from twisted.python import failure, filepath, log, usage
+
+if getpass.getpass == getpass.unix_getpass:  # type: ignore[attr-defined]
+    try:
+        import termios  # hack around broken termios
+
+        termios.tcgetattr, termios.tcsetattr
+    except (ImportError, AttributeError):
+        sys.modules["termios"] = None  # type: ignore[assignment]
+        reload(getpass)
+
+supportedKeyTypes = dict()
+
+
+def _keyGenerator(keyType):
+    def assignkeygenerator(keygenerator):
+        @wraps(keygenerator)
+        def wrapper(*args, **kwargs):
+            return keygenerator(*args, **kwargs)
+
+        supportedKeyTypes[keyType] = wrapper
+        return wrapper
+
+    return assignkeygenerator
+
+
+class GeneralOptions(usage.Options):
+    synopsis = """Usage:    ckeygen [options]
+ """
+
+    longdesc = "ckeygen manipulates public/private keys in various ways."
+
+    optParameters = [
+        ["bits", "b", None, "Number of bits in the key to create."],
+        ["filename", "f", None, "Filename of the key file."],
+        ["type", "t", None, "Specify type of key to create."],
+        ["comment", "C", None, "Provide new comment."],
+        ["newpass", "N", None, "Provide new passphrase."],
+        ["pass", "P", None, "Provide old passphrase."],
+        ["format", "o", "sha256-base64", "Fingerprint format of key file."],
+        [
+            "private-key-subtype",
+            None,
+            None,
+            'OpenSSH private key subtype to write ("PEM" or "v1").',
+        ],
+    ]
+
+    optFlags = [
+        ["fingerprint", "l", "Show fingerprint of key file."],
+        ["changepass", "p", "Change passphrase of private key file."],
+        ["quiet", "q", "Quiet."],
+        ["no-passphrase", None, "Create the key with no passphrase."],
+        ["showpub", "y", "Read private key file and print public key."],
+    ]
+
+    compData = usage.Completions(
+        optActions={
+            "type": usage.CompleteList(list(supportedKeyTypes.keys())),
+            "private-key-subtype": usage.CompleteList(["PEM", "v1"]),
+        }
+    )
+
+
+def run():
+    options = GeneralOptions()
+    try:
+        options.parseOptions(sys.argv[1:])
+    except usage.UsageError as u:
+        print("ERROR: %s" % u)
+        options.opt_help()
+        sys.exit(1)
+    log.discardLogs()
+    log.deferr = handleError  # HACK
+    if options["type"]:
+        if options["type"].lower() in supportedKeyTypes:
+            print("Generating public/private %s key pair." % (options["type"]))
+            supportedKeyTypes[options["type"].lower()](options)
+        else:
+            sys.exit(
+                "Key type was %s, must be one of %s"
+                % (options["type"], ", ".join(supportedKeyTypes.keys()))
+            )
+    elif options["fingerprint"]:
+        printFingerprint(options)
+    elif options["changepass"]:
+        changePassPhrase(options)
+    elif options["showpub"]:
+        displayPublicKey(options)
+    else:
+        options.opt_help()
+        sys.exit(1)
+
+
+def enumrepresentation(options):
+    if options["format"] == "md5-hex":
+        options["format"] = keys.FingerprintFormats.MD5_HEX
+        return options
+    elif options["format"] == "sha256-base64":
+        options["format"] = keys.FingerprintFormats.SHA256_BASE64
+        return options
+    else:
+        raise keys.BadFingerPrintFormat(
+            f"Unsupported fingerprint format: {options['format']}"
+        )
+
+
+def handleError():
+    global exitStatus
+    exitStatus = 2
+    log.err(failure.Failure())
+    raise
+
+
+@_keyGenerator("rsa")
+def generateRSAkey(options):
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives.asymmetric import rsa
+
+    if not options["bits"]:
+        options["bits"] = 2048
+    keyPrimitive = rsa.generate_private_key(
+        key_size=int(options["bits"]),
+        public_exponent=65537,
+        backend=default_backend(),
+    )
+    key = keys.Key(keyPrimitive)
+    _saveKey(key, options)
+
+
+@_keyGenerator("dsa")
+def generateDSAkey(options):
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives.asymmetric import dsa
+
+    if not options["bits"]:
+        options["bits"] = 1024
+    keyPrimitive = dsa.generate_private_key(
+        key_size=int(options["bits"]),
+        backend=default_backend(),
+    )
+    key = keys.Key(keyPrimitive)
+    _saveKey(key, options)
+
+
+@_keyGenerator("ecdsa")
+def generateECDSAkey(options):
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives.asymmetric import ec
+
+    if not options["bits"]:
+        options["bits"] = 256
+    # OpenSSH supports only mandatory sections of RFC5656.
+    # See https://www.openssh.com/txt/release-5.7
+    curve = b"ecdsa-sha2-nistp" + str(options["bits"]).encode("ascii")
+    keyPrimitive = ec.generate_private_key(
+        curve=keys._curveTable[curve], backend=default_backend()
+    )
+    key = keys.Key(keyPrimitive)
+    _saveKey(key, options)
+
+
+@_keyGenerator("ed25519")
+def generateEd25519key(options):
+    keyPrimitive = keys.Ed25519PrivateKey.generate()
+    key = keys.Key(keyPrimitive)
+    _saveKey(key, options)
+
+
+def _defaultPrivateKeySubtype(keyType):
+    """
+    Return a reasonable default private key subtype for a given key type.
+
+    @type keyType: L{str}
+    @param keyType: A key type, as returned by
+        L{twisted.conch.ssh.keys.Key.type}.
+
+    @rtype: L{str}
+    @return: A private OpenSSH key subtype (C{'PEM'} or C{'v1'}).
+    """
+    if keyType == "Ed25519":
+        # No PEM format is defined for Ed25519 keys.
+        return "v1"
+    else:
+        return "PEM"
+
+
+def _getKeyOrDefault(
+    options: Dict[Any, Any],
+    inputCollector: Optional[Callable[[str], str]] = None,
+    keyTypeName: str = "rsa",
+) -> str:
+    """
+    If C{options["filename"]} is None, prompt the user to enter a path
+    or attempt to set it to .ssh/id_rsa
+    @param options: command line options
+    @param inputCollector: dependency injection for testing
+    @param keyTypeName: key type or "rsa"
+    """
+    if inputCollector is None:
+        inputCollector = input
+    filename = options["filename"]
+    if not filename:
+        filename = os.path.expanduser(f"~/.ssh/id_{keyTypeName}")
+        if platform.system() == "Windows":
+            filename = os.path.expanduser(Rf"%HOMEPATH %\.ssh\id_{keyTypeName}")
+        filename = (
+            inputCollector("Enter file in which the key is (%s): " % filename)
+            or filename
+        )
+    return str(filename)
+
+
+def printFingerprint(options: Dict[Any, Any]) -> None:
+    filename = _getKeyOrDefault(options)
+    if os.path.exists(filename + ".pub"):
+        filename += ".pub"
+    options = enumrepresentation(options)
+    try:
+        key = keys.Key.fromFile(filename)
+        print(
+            "%s %s %s"
+            % (
+                key.size(),
+                key.fingerprint(options["format"]),
+                os.path.basename(filename),
+            )
+        )
+    except keys.BadKeyError:
+        sys.exit("bad key")
+    except FileNotFoundError:
+        sys.exit(f"{filename} could not be opened, please specify a file.")
+
+
+def changePassPhrase(options):
+    filename = _getKeyOrDefault(options)
+    try:
+        key = keys.Key.fromFile(filename)
+    except keys.EncryptedKeyError:
+        # Raised if password not supplied for an encrypted key
+        if not options.get("pass"):
+            options["pass"] = getpass.getpass("Enter old passphrase: ")
+        try:
+            key = keys.Key.fromFile(filename, passphrase=options["pass"])
+        except keys.BadKeyError:
+            sys.exit("Could not change passphrase: old passphrase error")
+        except keys.EncryptedKeyError as e:
+            sys.exit(f"Could not change passphrase: {e}")
+    except keys.BadKeyError as e:
+        sys.exit(f"Could not change passphrase: {e}")
+    except FileNotFoundError:
+        sys.exit(f"{filename} could not be opened, please specify a file.")
+
+    if not options.get("newpass"):
+        while 1:
+            p1 = getpass.getpass("Enter new passphrase (empty for no passphrase): ")
+            p2 = getpass.getpass("Enter same passphrase again: ")
+            if p1 == p2:
+                break
+            print("Passphrases do not match.  Try again.")
+        options["newpass"] = p1
+
+    if options.get("private-key-subtype") is None:
+        options["private-key-subtype"] = _defaultPrivateKeySubtype(key.type())
+
+    try:
+        newkeydata = key.toString(
+            "openssh",
+            subtype=options["private-key-subtype"],
+            passphrase=options["newpass"],
+        )
+    except Exception as e:
+        sys.exit(f"Could not change passphrase: {e}")
+
+    try:
+        keys.Key.fromString(newkeydata, passphrase=options["newpass"])
+    except (keys.EncryptedKeyError, keys.BadKeyError) as e:
+        sys.exit(f"Could not change passphrase: {e}")
+
+    with open(filename, "wb") as fd:
+        fd.write(newkeydata)
+
+    print("Your identification has been saved with the new passphrase.")
+
+
+def displayPublicKey(options):
+    filename = _getKeyOrDefault(options)
+    try:
+        key = keys.Key.fromFile(filename)
+    except FileNotFoundError:
+        sys.exit(f"{filename} could not be opened, please specify a file.")
+    except keys.EncryptedKeyError:
+        if not options.get("pass"):
+            options["pass"] = getpass.getpass("Enter passphrase: ")
+        key = keys.Key.fromFile(filename, passphrase=options["pass"])
+    displayKey = key.public().toString("openssh").decode("ascii")
+    print(displayKey)
+
+
+def _inputSaveFile(prompt: str) -> str:
+    """
+    Ask the user where to save the key.
+
+    This needs to be a separate function so the unit test can patch it.
+    """
+    return input(prompt)
+
+
+def _saveKey(
+    key: keys.Key,
+    options: Dict[Any, Any],
+    inputCollector: Optional[Callable[[str], str]] = None,
+) -> None:
+    """
+    Persist a SSH key on local filesystem.
+
+    @param key: Key which is persisted on local filesystem.
+
+    @param options:
+
+    @param inputCollector: Dependency injection for testing.
+    """
+    if inputCollector is None:
+        inputCollector = input
+    KeyTypeMapping = {"EC": "ecdsa", "Ed25519": "ed25519", "RSA": "rsa", "DSA": "dsa"}
+    keyTypeName = KeyTypeMapping[key.type()]
+    filename = options["filename"]
+    if not filename:
+        defaultPath = _getKeyOrDefault(options, inputCollector, keyTypeName)
+        newPath = _inputSaveFile(
+            f"Enter file in which to save the key ({defaultPath}): "
+        )
+
+        filename = newPath.strip() or defaultPath
+
+    if os.path.exists(filename):
+        print(f"{filename} already exists.")
+        yn = inputCollector("Overwrite (y/n)? ")
+        if yn[0].lower() != "y":
+            sys.exit()
+
+    if options.get("no-passphrase"):
+        options["pass"] = b""
+    elif not options["pass"]:
+        while 1:
+            p1 = getpass.getpass("Enter passphrase (empty for no passphrase): ")
+            p2 = getpass.getpass("Enter same passphrase again: ")
+            if p1 == p2:
+                break
+            print("Passphrases do not match.  Try again.")
+        options["pass"] = p1
+
+    if options.get("private-key-subtype") is None:
+        options["private-key-subtype"] = _defaultPrivateKeySubtype(key.type())
+
+    comment = f"{getpass.getuser()}@{socket.gethostname()}"
+
+    fp = filepath.FilePath(filename)
+    fp.setContent(
+        key.toString(
+            "openssh",
+            subtype=options["private-key-subtype"],
+            passphrase=options["pass"],
+        )
+    )
+    fp.chmod(0o100600)
+
+    filepath.FilePath(filename + ".pub").setContent(
+        key.public().toString("openssh", comment=comment)
+    )
+    options = enumrepresentation(options)
+
+    print(f"Your identification has been saved in {filename}")
+    print(f"Your public key has been saved in {filename}.pub")
+    print(f"The key fingerprint in {options['format']} is:")
+    print(key.fingerprint(options["format"]))
+
+
+if __name__ == "__main__":
+    run()
diff --git a/contrib/python/Twisted/py3/twisted/conch/scripts/conch.py b/contrib/python/Twisted/py3/twisted/conch/scripts/conch.py
new file mode 100644
index 00000000000..f3e5479bd91
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/scripts/conch.py
@@ -0,0 +1,578 @@
+# -*- test-case-name: twisted.conch.test.test_conch -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+# $Id: conch.py,v 1.65 2004/03/11 00:29:14 z3p Exp $
+
+# Implementation module for the `conch` command.
+#
+
+import fcntl
+import getpass
+import os
+import signal
+import struct
+import sys
+import tty
+from typing import List, Tuple
+
+from twisted.conch.client import connect, default
+from twisted.conch.client.options import ConchOptions
+from twisted.conch.error import ConchError
+from twisted.conch.ssh import channel, common, connection, forwarding, session
+from twisted.internet import reactor, stdio, task
+from twisted.python import log, usage
+from twisted.python.compat import ioType, networkString
+
+
+class ClientOptions(ConchOptions):
+    synopsis = """Usage:   conch [options] host [command]
+"""
+    longdesc = (
+        "conch is a SSHv2 client that allows logging into a remote "
+        "machine and executing commands."
+    )
+
+    optParameters = [
+        ["escape", "e", "~"],
+        [
+            "localforward",
+            "L",
+            None,
+            "listen-port:host:port   Forward local port to remote address",
+        ],
+        [
+            "remoteforward",
+            "R",
+            None,
+            "listen-port:host:port   Forward remote port to local address",
+        ],
+    ]
+
+    optFlags = [
+        ["null", "n", "Redirect input from /dev/null."],
+        ["fork", "f", "Fork to background after authentication."],
+        ["tty", "t", "Tty; allocate a tty even if command is given."],
+        ["notty", "T", "Do not allocate a tty."],
+        ["noshell", "N", "Do not execute a shell or command."],
+        ["subsystem", "s", "Invoke command (mandatory) as SSH2 subsystem."],
+    ]
+
+    compData = usage.Completions(
+        mutuallyExclusive=[("tty", "notty")],
+        optActions={
+            "localforward": usage.Completer(descr="listen-port:host:port"),
+            "remoteforward": usage.Completer(descr="listen-port:host:port"),
+        },
+        extraActions=[
+            usage.CompleteUserAtHost(),
+            usage.Completer(descr="command"),
+            usage.Completer(descr="argument", repeat=True),
+        ],
+    )
+
+    localForwards: List[Tuple[int, Tuple[int, int]]] = []
+    remoteForwards: List[Tuple[int, Tuple[int, int]]] = []
+
+    def opt_escape(self, esc):
+        """
+        Set escape character; ``none'' = disable
+        """
+        if esc == "none":
+            self["escape"] = None
+        elif esc[0] == "^" and len(esc) == 2:
+            self["escape"] = chr(ord(esc[1]) - 64)
+        elif len(esc) == 1:
+            self["escape"] = esc
+        else:
+            sys.exit(f"Bad escape character '{esc}'.")
+
+    def opt_localforward(self, f):
+        """
+        Forward local port to remote address (lport:host:port)
+        """
+        localPort, remoteHost, remotePort = f.split(":")  # Doesn't do v6 yet
+        localPort = int(localPort)
+        remotePort = int(remotePort)
+        self.localForwards.append((localPort, (remoteHost, remotePort)))
+
+    def opt_remoteforward(self, f):
+        """
+        Forward remote port to local address (rport:host:port)
+        """
+        remotePort, connHost, connPort = f.split(":")  # Doesn't do v6 yet
+        remotePort = int(remotePort)
+        connPort = int(connPort)
+        self.remoteForwards.append((remotePort, (connHost, connPort)))
+
+    def parseArgs(self, host, *command):
+        self["host"] = host
+        self["command"] = " ".join(command)
+
+
+# Rest of code in "run"
+options = None
+conn = None
+exitStatus = 0
+old = None
+_inRawMode = 0
+_savedRawMode = None
+
+
+def run():
+    global options, old
+    args = sys.argv[1:]
+    if "-l" in args:  # CVS is an idiot
+        i = args.index("-l")
+        args = args[i : i + 2] + args
+        del args[i + 2 : i + 4]
+    for arg in args[:]:
+        try:
+            i = args.index(arg)
+            if arg[:2] == "-o" and args[i + 1][0] != "-":
+                args[i : i + 2] = []  # Suck on it scp
+        except ValueError:
+            pass
+    options = ClientOptions()
+    try:
+        options.parseOptions(args)
+    except usage.UsageError as u:
+        print(f"ERROR: {u}")
+        options.opt_help()
+        sys.exit(1)
+    if options["log"]:
+        if options["logfile"]:
+            if options["logfile"] == "-":
+                f = sys.stdout
+            else:
+                f = open(options["logfile"], "a+")
+        else:
+            f = sys.stderr
+        realout = sys.stdout
+        log.startLogging(f)
+        sys.stdout = realout
+    else:
+        log.discardLogs()
+    doConnect()
+    fd = sys.stdin.fileno()
+    try:
+        old = tty.tcgetattr(fd)
+    except BaseException:
+        old = None
+    try:
+        oldUSR1 = signal.signal(
+            signal.SIGUSR1, lambda *a: reactor.callLater(0, reConnect)
+        )
+    except BaseException:
+        oldUSR1 = None
+    try:
+        reactor.run()
+    finally:
+        if old:
+            tty.tcsetattr(fd, tty.TCSANOW, old)
+        if oldUSR1:
+            signal.signal(signal.SIGUSR1, oldUSR1)
+        if (options["command"] and options["tty"]) or not options["notty"]:
+            signal.signal(signal.SIGWINCH, signal.SIG_DFL)
+    if sys.stdout.isatty() and not options["command"]:
+        print("Connection to {} closed.".format(options["host"]))
+    sys.exit(exitStatus)
+
+
+def handleError():
+    from twisted.python import failure
+
+    global exitStatus
+    exitStatus = 2
+    reactor.callLater(0.01, _stopReactor)
+    log.err(failure.Failure())
+    raise
+
+
+def _stopReactor():
+    try:
+        reactor.stop()
+    except BaseException:
+        pass
+
+
+def doConnect():
+    if "@" in options["host"]:
+        options["user"], options["host"] = options["host"].split("@", 1)
+    if not options.identitys:
+        options.identitys = ["~/.ssh/id_rsa", "~/.ssh/id_dsa"]
+    host = options["host"]
+    if not options["user"]:
+        options["user"] = getpass.getuser()
+    if not options["port"]:
+        options["port"] = 22
+    else:
+        options["port"] = int(options["port"])
+    host = options["host"]
+    port = options["port"]
+    vhk = default.verifyHostKey
+    if not options["host-key-algorithms"]:
+        options["host-key-algorithms"] = default.getHostKeyAlgorithms(host, options)
+    uao = default.SSHUserAuthClient(options["user"], options, SSHConnection())
+    connect.connect(host, port, options, vhk, uao).addErrback(_ebExit)
+
+
+def _ebExit(f):
+    global exitStatus
+    exitStatus = f"conch: exiting with error {f}"
+    reactor.callLater(0.1, _stopReactor)
+
+
+def onConnect():
+    #    if keyAgent and options['agent']:
+    #        cc = protocol.ClientCreator(reactor, SSHAgentForwardingLocal, conn)
+    #        cc.connectUNIX(os.environ['SSH_AUTH_SOCK'])
+    if hasattr(conn.transport, "sendIgnore"):
+        _KeepAlive(conn)
+    if options.localForwards:
+        for localPort, hostport in options.localForwards:
+            s = reactor.listenTCP(
+                localPort,
+                forwarding.SSHListenForwardingFactory(
+                    conn, hostport, SSHListenClientForwardingChannel
+                ),
+            )
+            conn.localForwards.append(s)
+    if options.remoteForwards:
+        for remotePort, hostport in options.remoteForwards:
+            log.msg(f"asking for remote forwarding for {remotePort}:{hostport}")
+            conn.requestRemoteForwarding(remotePort, hostport)
+        reactor.addSystemEventTrigger("before", "shutdown", beforeShutdown)
+    if not options["noshell"] or options["agent"]:
+        conn.openChannel(SSHSession())
+    if options["fork"]:
+        if os.fork():
+            os._exit(0)
+        os.setsid()
+        for i in range(3):
+            try:
+                os.close(i)
+            except OSError as e:
+                import errno
+
+                if e.errno != errno.EBADF:
+                    raise
+
+
+def reConnect():
+    beforeShutdown()
+    conn.transport.transport.loseConnection()
+
+
+def beforeShutdown():
+    remoteForwards = options.remoteForwards
+    for remotePort, hostport in remoteForwards:
+        log.msg(f"cancelling {remotePort}:{hostport}")
+        conn.cancelRemoteForwarding(remotePort)
+
+
+def stopConnection():
+    if not options["reconnect"]:
+        reactor.callLater(0.1, _stopReactor)
+
+
+class _KeepAlive:
+    def __init__(self, conn):
+        self.conn = conn
+        self.globalTimeout = None
+        self.lc = task.LoopingCall(self.sendGlobal)
+        self.lc.start(300)
+
+    def sendGlobal(self):
+        d = self.conn.sendGlobalRequest(
+            b"conch-keep-alive@twistedmatrix.com", b"", wantReply=1
+        )
+        d.addBoth(self._cbGlobal)
+        self.globalTimeout = reactor.callLater(30, self._ebGlobal)
+
+    def _cbGlobal(self, res):
+        if self.globalTimeout:
+            self.globalTimeout.cancel()
+            self.globalTimeout = None
+
+    def _ebGlobal(self):
+        if self.globalTimeout:
+            self.globalTimeout = None
+            self.conn.transport.loseConnection()
+
+
+class SSHConnection(connection.SSHConnection):
+    def serviceStarted(self):
+        global conn
+        conn = self
+        self.localForwards = []
+        self.remoteForwards = {}
+        onConnect()
+
+    def serviceStopped(self):
+        lf = self.localForwards
+        self.localForwards = []
+        for s in lf:
+            s.loseConnection()
+        stopConnection()
+
+    def requestRemoteForwarding(self, remotePort, hostport):
+        data = forwarding.packGlobal_tcpip_forward(("0.0.0.0", remotePort))
+        d = self.sendGlobalRequest(b"tcpip-forward", data, wantReply=1)
+        log.msg(f"requesting remote forwarding {remotePort}:{hostport}")
+        d.addCallback(self._cbRemoteForwarding, remotePort, hostport)
+        d.addErrback(self._ebRemoteForwarding, remotePort, hostport)
+
+    def _cbRemoteForwarding(self, result, remotePort, hostport):
+        log.msg(f"accepted remote forwarding {remotePort}:{hostport}")
+        self.remoteForwards[remotePort] = hostport
+        log.msg(repr(self.remoteForwards))
+
+    def _ebRemoteForwarding(self, f, remotePort, hostport):
+        log.msg(f"remote forwarding {remotePort}:{hostport} failed")
+        log.msg(f)
+
+    def cancelRemoteForwarding(self, remotePort):
+        data = forwarding.packGlobal_tcpip_forward(("0.0.0.0", remotePort))
+        self.sendGlobalRequest(b"cancel-tcpip-forward", data)
+        log.msg(f"cancelling remote forwarding {remotePort}")
+        try:
+            del self.remoteForwards[remotePort]
+        except Exception:
+            pass
+        log.msg(repr(self.remoteForwards))
+
+    def channel_forwarded_tcpip(self, windowSize, maxPacket, data):
+        log.msg(f"FTCP {data!r}")
+        remoteHP, origHP = forwarding.unpackOpen_forwarded_tcpip(data)
+        log.msg(self.remoteForwards)
+        log.msg(remoteHP)
+        if remoteHP[1] in self.remoteForwards:
+            connectHP = self.remoteForwards[remoteHP[1]]
+            log.msg(f"connect forwarding {connectHP}")
+            return SSHConnectForwardingChannel(
+                connectHP, remoteWindow=windowSize, remoteMaxPacket=maxPacket, conn=self
+            )
+        else:
+            raise ConchError(
+                connection.OPEN_CONNECT_FAILED, "don't know about that port"
+            )
+
+    def channelClosed(self, channel):
+        log.msg(f"connection closing {channel}")
+        log.msg(self.channels)
+        if len(self.channels) == 1:  # Just us left
+            log.msg("stopping connection")
+            stopConnection()
+        else:
+            # Because of the unix thing
+            self.__class__.__bases__[0].channelClosed(self, channel)
+
+
+class SSHSession(channel.SSHChannel):
+    name = b"session"
+
+    def channelOpen(self, foo):
+        log.msg(f"session {self.id} open")
+        if options["agent"]:
+            d = self.conn.sendRequest(
+                self, b"auth-agent-req@openssh.com", b"", wantReply=1
+            )
+            d.addBoth(lambda x: log.msg(x))
+        if options["noshell"]:
+            return
+        if (options["command"] and options["tty"]) or not options["notty"]:
+            _enterRawMode()
+        c = session.SSHSessionClient()
+        if options["escape"] and not options["notty"]:
+            self.escapeMode = 1
+            c.dataReceived = self.handleInput
+        else:
+            c.dataReceived = self.write
+        c.connectionLost = lambda x: self.sendEOF()
+        self.stdio = stdio.StandardIO(c)
+        fd = 0
+        if options["subsystem"]:
+            self.conn.sendRequest(self, b"subsystem", common.NS(options["command"]))
+        elif options["command"]:
+            if options["tty"]:
+                term = os.environ["TERM"]
+                winsz = fcntl.ioctl(fd, tty.TIOCGWINSZ, "12345678")
+                winSize = struct.unpack("4H", winsz)
+                ptyReqData = session.packRequest_pty_req(term, winSize, "")
+                self.conn.sendRequest(self, b"pty-req", ptyReqData)
+                signal.signal(signal.SIGWINCH, self._windowResized)
+            self.conn.sendRequest(self, b"exec", common.NS(options["command"]))
+        else:
+            if not options["notty"]:
+                term = os.environ["TERM"]
+                winsz = fcntl.ioctl(fd, tty.TIOCGWINSZ, "12345678")
+                winSize = struct.unpack("4H", winsz)
+                ptyReqData = session.packRequest_pty_req(term, winSize, "")
+                self.conn.sendRequest(self, b"pty-req", ptyReqData)
+                signal.signal(signal.SIGWINCH, self._windowResized)
+            self.conn.sendRequest(self, b"shell", b"")
+            # if hasattr(conn.transport, 'transport'):
+            #    conn.transport.transport.setTcpNoDelay(1)
+
+    def handleInput(self, char):
+        if char in (b"\n", b"\r"):
+            self.escapeMode = 1
+            self.write(char)
+        elif self.escapeMode == 1 and char == options["escape"]:
+            self.escapeMode = 2
+        elif self.escapeMode == 2:
+            self.escapeMode = 1  # So we can chain escapes together
+            if char == b".":  # Disconnect
+                log.msg("disconnecting from escape")
+                stopConnection()
+                return
+            elif char == b"\x1a":  # ^Z, suspend
+
+                def _():
+                    _leaveRawMode()
+                    sys.stdout.flush()
+                    sys.stdin.flush()
+                    os.kill(os.getpid(), signal.SIGTSTP)
+                    _enterRawMode()
+
+                reactor.callLater(0, _)
+                return
+            elif char == b"R":  # Rekey connection
+                log.msg("rekeying connection")
+                self.conn.transport.sendKexInit()
+                return
+            elif char == b"#":  # Display connections
+                self.stdio.write(b"\r\nThe following connections are open:\r\n")
+                channels = self.conn.channels.keys()
+                channels.sort()
+                for channelId in channels:
+                    self.stdio.write(
+                        networkString(
+                            "  #{} {}\r\n".format(
+                                channelId, self.conn.channels[channelId]
+                            )
+                        )
+                    )
+                return
+            self.write(b"~" + char)
+        else:
+            self.escapeMode = 0
+            self.write(char)
+
+    def dataReceived(self, data):
+        self.stdio.write(data)
+
+    def extReceived(self, t, data):
+        if t == connection.EXTENDED_DATA_STDERR:
+            log.msg(f"got {len(data)} stderr data")
+            if ioType(sys.stderr) == str:
+                sys.stderr.buffer.write(data)
+            else:
+                sys.stderr.write(data)
+
+    def eofReceived(self):
+        log.msg("got eof")
+        self.stdio.loseWriteConnection()
+
+    def closeReceived(self):
+        log.msg(f"remote side closed {self}")
+        self.conn.sendClose(self)
+
+    def closed(self):
+        global old
+        log.msg(f"closed {self}")
+        log.msg(repr(self.conn.channels))
+
+    def request_exit_status(self, data):
+        global exitStatus
+        exitStatus = int(struct.unpack(">L", data)[0])
+        log.msg(f"exit status: {exitStatus}")
+
+    def sendEOF(self):
+        self.conn.sendEOF(self)
+
+    def stopWriting(self):
+        self.stdio.pauseProducing()
+
+    def startWriting(self):
+        self.stdio.resumeProducing()
+
+    def _windowResized(self, *args):
+        winsz = fcntl.ioctl(0, tty.TIOCGWINSZ, "12345678")
+        winSize = struct.unpack("4H", winsz)
+        newSize = winSize[1], winSize[0], winSize[2], winSize[3]
+        self.conn.sendRequest(self, b"window-change", struct.pack("!4L", *newSize))
+
+
+class SSHListenClientForwardingChannel(forwarding.SSHListenClientForwardingChannel):
+    pass
+
+
+class SSHConnectForwardingChannel(forwarding.SSHConnectForwardingChannel):
+    pass
+
+
+def _leaveRawMode():
+    global _inRawMode
+    if not _inRawMode:
+        return
+    fd = sys.stdin.fileno()
+    tty.tcsetattr(fd, tty.TCSANOW, _savedRawMode)
+    _inRawMode = 0
+
+
+def _enterRawMode():
+    global _inRawMode, _savedRawMode
+    if _inRawMode:
+        return
+    fd = sys.stdin.fileno()
+    try:
+        old = tty.tcgetattr(fd)
+        new = old[:]
+    except BaseException:
+        log.msg("not a typewriter!")
+    else:
+        # iflage
+        new[0] = new[0] | tty.IGNPAR
+        new[0] = new[0] & ~(
+            tty.ISTRIP
+            | tty.INLCR
+            | tty.IGNCR
+            | tty.ICRNL
+            | tty.IXON
+            | tty.IXANY
+            | tty.IXOFF
+        )
+        if hasattr(tty, "IUCLC"):
+            new[0] = new[0] & ~tty.IUCLC
+
+        # lflag
+        new[3] = new[3] & ~(
+            tty.ISIG
+            | tty.ICANON
+            | tty.ECHO
+            | tty.ECHO
+            | tty.ECHOE
+            | tty.ECHOK
+            | tty.ECHONL
+        )
+        if hasattr(tty, "IEXTEN"):
+            new[3] = new[3] & ~tty.IEXTEN
+
+        # oflag
+        new[1] = new[1] & ~tty.OPOST
+
+        new[6][tty.VMIN] = 1
+        new[6][tty.VTIME] = 0
+
+        _savedRawMode = old
+        tty.tcsetattr(fd, tty.TCSANOW, new)
+        # tty.setraw(fd)
+        _inRawMode = 1
+
+
+if __name__ == "__main__":
+    run()
diff --git a/contrib/python/Twisted/py3/twisted/conch/scripts/tkconch.py b/contrib/python/Twisted/py3/twisted/conch/scripts/tkconch.py
new file mode 100644
index 00000000000..e6738403daa
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/scripts/tkconch.py
@@ -0,0 +1,673 @@
+# -*- test-case-name: twisted.conch.test.test_scripts -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation module for the `tkconch` command.
+"""
+
+
+import base64
+import getpass
+import os
+import signal
+import struct
+import sys
+import tkinter as Tkinter
+import tkinter.filedialog as tkFileDialog
+import tkinter.messagebox as tkMessageBox
+from typing import List, Tuple
+
+from twisted.conch import error
+from twisted.conch.client.default import isInKnownHosts
+from twisted.conch.ssh import (
+    channel,
+    common,
+    connection,
+    forwarding,
+    keys,
+    session,
+    transport,
+    userauth,
+)
+from twisted.conch.ui import tkvt100
+from twisted.internet import defer, protocol, reactor, tksupport
+from twisted.python import log, usage
+
+
+class TkConchMenu(Tkinter.Frame):
+    def __init__(self, *args, **params):
+        ## Standard heading: initialization
+        Tkinter.Frame.__init__(self, *args, **params)
+
+        self.master.title("TkConch")
+        self.localRemoteVar = Tkinter.StringVar()
+        self.localRemoteVar.set("local")
+
+        Tkinter.Label(self, anchor="w", justify="left", text="Hostname").grid(
+            column=1, row=1, sticky="w"
+        )
+        self.host = Tkinter.Entry(self)
+        self.host.grid(column=2, columnspan=2, row=1, sticky="nesw")
+
+        Tkinter.Label(self, anchor="w", justify="left", text="Port").grid(
+            column=1, row=2, sticky="w"
+        )
+        self.port = Tkinter.Entry(self)
+        self.port.grid(column=2, columnspan=2, row=2, sticky="nesw")
+
+        Tkinter.Label(self, anchor="w", justify="left", text="Username").grid(
+            column=1, row=3, sticky="w"
+        )
+        self.user = Tkinter.Entry(self)
+        self.user.grid(column=2, columnspan=2, row=3, sticky="nesw")
+
+        Tkinter.Label(self, anchor="w", justify="left", text="Command").grid(
+            column=1, row=4, sticky="w"
+        )
+        self.command = Tkinter.Entry(self)
+        self.command.grid(column=2, columnspan=2, row=4, sticky="nesw")
+
+        Tkinter.Label(self, anchor="w", justify="left", text="Identity").grid(
+            column=1, row=5, sticky="w"
+        )
+        self.identity = Tkinter.Entry(self)
+        self.identity.grid(column=2, row=5, sticky="nesw")
+        Tkinter.Button(self, command=self.getIdentityFile, text="Browse").grid(
+            column=3, row=5, sticky="nesw"
+        )
+
+        Tkinter.Label(self, text="Port Forwarding").grid(column=1, row=6, sticky="w")
+        self.forwards = Tkinter.Listbox(self, height=0, width=0)
+        self.forwards.grid(column=2, columnspan=2, row=6, sticky="nesw")
+        Tkinter.Button(self, text="Add", command=self.addForward).grid(column=1, row=7)
+        Tkinter.Button(self, text="Remove", command=self.removeForward).grid(
+            column=1, row=8
+        )
+        self.forwardPort = Tkinter.Entry(self)
+        self.forwardPort.grid(column=2, row=7, sticky="nesw")
+        Tkinter.Label(self, text="Port").grid(column=3, row=7, sticky="nesw")
+        self.forwardHost = Tkinter.Entry(self)
+        self.forwardHost.grid(column=2, row=8, sticky="nesw")
+        Tkinter.Label(self, text="Host").grid(column=3, row=8, sticky="nesw")
+        self.localForward = Tkinter.Radiobutton(
+            self, text="Local", variable=self.localRemoteVar, value="local"
+        )
+        self.localForward.grid(column=2, row=9)
+        self.remoteForward = Tkinter.Radiobutton(
+            self, text="Remote", variable=self.localRemoteVar, value="remote"
+        )
+        self.remoteForward.grid(column=3, row=9)
+
+        Tkinter.Label(self, text="Advanced Options").grid(
+            column=1, columnspan=3, row=10, sticky="nesw"
+        )
+
+        Tkinter.Label(self, anchor="w", justify="left", text="Cipher").grid(
+            column=1, row=11, sticky="w"
+        )
+        self.cipher = Tkinter.Entry(self, name="cipher")
+        self.cipher.grid(column=2, columnspan=2, row=11, sticky="nesw")
+
+        Tkinter.Label(self, anchor="w", justify="left", text="MAC").grid(
+            column=1, row=12, sticky="w"
+        )
+        self.mac = Tkinter.Entry(self, name="mac")
+        self.mac.grid(column=2, columnspan=2, row=12, sticky="nesw")
+
+        Tkinter.Label(self, anchor="w", justify="left", text="Escape Char").grid(
+            column=1, row=13, sticky="w"
+        )
+        self.escape = Tkinter.Entry(self, name="escape")
+        self.escape.grid(column=2, columnspan=2, row=13, sticky="nesw")
+        Tkinter.Button(self, text="Connect!", command=self.doConnect).grid(
+            column=1, columnspan=3, row=14, sticky="nesw"
+        )
+
+        # Resize behavior(s)
+        self.grid_rowconfigure(6, weight=1, minsize=64)
+        self.grid_columnconfigure(2, weight=1, minsize=2)
+
+        self.master.protocol("WM_DELETE_WINDOW", sys.exit)
+
+    def getIdentityFile(self):
+        r = tkFileDialog.askopenfilename()
+        if r:
+            self.identity.delete(0, Tkinter.END)
+            self.identity.insert(Tkinter.END, r)
+
+    def addForward(self):
+        port = self.forwardPort.get()
+        self.forwardPort.delete(0, Tkinter.END)
+        host = self.forwardHost.get()
+        self.forwardHost.delete(0, Tkinter.END)
+        if self.localRemoteVar.get() == "local":
+            self.forwards.insert(Tkinter.END, f"L:{port}:{host}")
+        else:
+            self.forwards.insert(Tkinter.END, f"R:{port}:{host}")
+
+    def removeForward(self):
+        cur = self.forwards.curselection()
+        if cur:
+            self.forwards.remove(cur[0])
+
+    def doConnect(self):
+        finished = 1
+        options["host"] = self.host.get()
+        options["port"] = self.port.get()
+        options["user"] = self.user.get()
+        options["command"] = self.command.get()
+        cipher = self.cipher.get()
+        mac = self.mac.get()
+        escape = self.escape.get()
+        if cipher:
+            if cipher in SSHClientTransport.supportedCiphers:
+                SSHClientTransport.supportedCiphers = [cipher]
+            else:
+                tkMessageBox.showerror("TkConch", "Bad cipher.")
+                finished = 0
+
+        if mac:
+            if mac in SSHClientTransport.supportedMACs:
+                SSHClientTransport.supportedMACs = [mac]
+            elif finished:
+                tkMessageBox.showerror("TkConch", "Bad MAC.")
+                finished = 0
+
+        if escape:
+            if escape == "none":
+                options["escape"] = None
+            elif escape[0] == "^" and len(escape) == 2:
+                options["escape"] = chr(ord(escape[1]) - 64)
+            elif len(escape) == 1:
+                options["escape"] = escape
+            elif finished:
+                tkMessageBox.showerror("TkConch", "Bad escape character '%s'." % escape)
+                finished = 0
+
+        if self.identity.get():
+            options.identitys.append(self.identity.get())
+
+        for line in self.forwards.get(0, Tkinter.END):
+            if line[0] == "L":
+                options.opt_localforward(line[2:])
+            else:
+                options.opt_remoteforward(line[2:])
+
+        if "@" in options["host"]:
+            options["user"], options["host"] = options["host"].split("@", 1)
+
+        if (not options["host"] or not options["user"]) and finished:
+            tkMessageBox.showerror("TkConch", "Missing host or username.")
+            finished = 0
+        if finished:
+            self.master.quit()
+            self.master.destroy()
+            if options["log"]:
+                realout = sys.stdout
+                log.startLogging(sys.stderr)
+                sys.stdout = realout
+            else:
+                log.discardLogs()
+            log.deferr = handleError  # HACK
+            if not options.identitys:
+                options.identitys = ["~/.ssh/id_rsa", "~/.ssh/id_dsa"]
+            host = options["host"]
+            port = int(options["port"] or 22)
+            log.msg((host, port))
+            reactor.connectTCP(host, port, SSHClientFactory())
+            frame.master.deiconify()
+            frame.master.title(
+                "{}@{} - TkConch".format(options["user"], options["host"])
+            )
+        else:
+            self.focus()
+
+
+class GeneralOptions(usage.Options):
+    synopsis = """Usage:    tkconch [options] host [command]
+ """
+
+    optParameters = [
+        ["user", "l", None, "Log in using this user name."],
+        ["identity", "i", "~/.ssh/identity", "Identity for public key authentication"],
+        ["escape", "e", "~", "Set escape character; ``none'' = disable"],
+        ["cipher", "c", None, "Select encryption algorithm."],
+        ["macs", "m", None, "Specify MAC algorithms for protocol version 2."],
+        ["port", "p", None, "Connect to this port.  Server must be on the same port."],
+        [
+            "localforward",
+            "L",
+            None,
+            "listen-port:host:port   Forward local port to remote address",
+        ],
+        [
+            "remoteforward",
+            "R",
+            None,
+            "listen-port:host:port   Forward remote port to local address",
+        ],
+    ]
+
+    optFlags = [
+        ["tty", "t", "Tty; allocate a tty even if command is given."],
+        ["notty", "T", "Do not allocate a tty."],
+        ["version", "V", "Display version number only."],
+        ["compress", "C", "Enable compression."],
+        ["noshell", "N", "Do not execute a shell or command."],
+        ["subsystem", "s", "Invoke command (mandatory) as SSH2 subsystem."],
+        ["log", "v", "Log to stderr"],
+        ["ansilog", "a", "Print the received data to stdout"],
+    ]
+
+    _ciphers = transport.SSHClientTransport.supportedCiphers
+    _macs = transport.SSHClientTransport.supportedMACs
+
+    compData = usage.Completions(
+        mutuallyExclusive=[("tty", "notty")],
+        optActions={
+            "cipher": usage.CompleteList([v.decode() for v in _ciphers]),
+            "macs": usage.CompleteList([v.decode() for v in _macs]),
+            "localforward": usage.Completer(descr="listen-port:host:port"),
+            "remoteforward": usage.Completer(descr="listen-port:host:port"),
+        },
+        extraActions=[
+            usage.CompleteUserAtHost(),
+            usage.Completer(descr="command"),
+            usage.Completer(descr="argument", repeat=True),
+        ],
+    )
+
+    identitys: List[str] = []
+    localForwards: List[Tuple[int, Tuple[int, int]]] = []
+    remoteForwards: List[Tuple[int, Tuple[int, int]]] = []
+
+    def opt_identity(self, i):
+        self.identitys.append(i)
+
+    def opt_localforward(self, f):
+        localPort, remoteHost, remotePort = f.split(":")  # doesn't do v6 yet
+        localPort = int(localPort)
+        remotePort = int(remotePort)
+        self.localForwards.append((localPort, (remoteHost, remotePort)))
+
+    def opt_remoteforward(self, f):
+        remotePort, connHost, connPort = f.split(":")  # doesn't do v6 yet
+        remotePort = int(remotePort)
+        connPort = int(connPort)
+        self.remoteForwards.append((remotePort, (connHost, connPort)))
+
+    def opt_compress(self):
+        SSHClientTransport.supportedCompressions[0:1] = ["zlib"]
+
+    def parseArgs(self, *args):
+        if args:
+            self["host"] = args[0]
+            self["command"] = " ".join(args[1:])
+        else:
+            self["host"] = ""
+            self["command"] = ""
+
+
+# Rest of code in "run"
+options = None
+menu = None
+exitStatus = 0
+frame = None
+
+
+def deferredAskFrame(question, echo):
+    if frame.callback:
+        raise ValueError("can't ask 2 questions at once!")
+    d = defer.Deferred()
+    resp = []
+
+    def gotChar(ch, resp=resp):
+        if not ch:
+            return
+        if ch == "\x03":  # C-c
+            reactor.stop()
+        if ch == "\r":
+            frame.write("\r\n")
+            stresp = "".join(resp)
+            del resp
+            frame.callback = None
+            d.callback(stresp)
+            return
+        elif 32 <= ord(ch) < 127:
+            resp.append(ch)
+            if echo:
+                frame.write(ch)
+        elif ord(ch) == 8 and resp:  # BS
+            if echo:
+                frame.write("\x08 \x08")
+            resp.pop()
+
+    frame.callback = gotChar
+    frame.write(question)
+    frame.canvas.focus_force()
+    return d
+
+
+def run():
+    global menu, options, frame
+    args = sys.argv[1:]
+    if "-l" in args:  # cvs is an idiot
+        i = args.index("-l")
+        args = args[i : i + 2] + args
+        del args[i + 2 : i + 4]
+    for arg in args[:]:
+        try:
+            i = args.index(arg)
+            if arg[:2] == "-o" and args[i + 1][0] != "-":
+                args[i : i + 2] = []  # suck on it scp
+        except ValueError:
+            pass
+    root = Tkinter.Tk()
+    root.withdraw()
+    top = Tkinter.Toplevel()
+    menu = TkConchMenu(top)
+    menu.pack(side=Tkinter.TOP, fill=Tkinter.BOTH, expand=1)
+    options = GeneralOptions()
+    try:
+        options.parseOptions(args)
+    except usage.UsageError as u:
+        print("ERROR: %s" % u)
+        options.opt_help()
+        sys.exit(1)
+    for k, v in options.items():
+        if v and hasattr(menu, k):
+            getattr(menu, k).insert(Tkinter.END, v)
+    for p, (rh, rp) in options.localForwards:
+        menu.forwards.insert(Tkinter.END, f"L:{p}:{rh}:{rp}")
+    options.localForwards = []
+    for p, (rh, rp) in options.remoteForwards:
+        menu.forwards.insert(Tkinter.END, f"R:{p}:{rh}:{rp}")
+    options.remoteForwards = []
+    frame = tkvt100.VT100Frame(root, callback=None)
+    root.geometry(
+        "%dx%d"
+        % (tkvt100.fontWidth * frame.width + 3, tkvt100.fontHeight * frame.height + 3)
+    )
+    frame.pack(side=Tkinter.TOP)
+    tksupport.install(root)
+    root.withdraw()
+    if (options["host"] and options["user"]) or "@" in options["host"]:
+        menu.doConnect()
+    else:
+        top.mainloop()
+    reactor.run()
+    sys.exit(exitStatus)
+
+
+def handleError():
+    from twisted.python import failure
+
+    global exitStatus
+    exitStatus = 2
+    log.err(failure.Failure())
+    reactor.stop()
+    raise
+
+
+class SSHClientFactory(protocol.ClientFactory):
+    noisy = True
+
+    def stopFactory(self):
+        reactor.stop()
+
+    def buildProtocol(self, addr):
+        return SSHClientTransport()
+
+    def clientConnectionFailed(self, connector, reason):
+        tkMessageBox.showwarning(
+            "TkConch",
+            f"Connection Failed, Reason:\n {reason.type}: {reason.value}",
+        )
+
+
+class SSHClientTransport(transport.SSHClientTransport):
+    def receiveError(self, code, desc):
+        global exitStatus
+        exitStatus = (
+            "conch:\tRemote side disconnected with error code %i\nconch:\treason: %s"
+            % (code, desc)
+        )
+
+    def sendDisconnect(self, code, reason):
+        global exitStatus
+        exitStatus = (
+            "conch:\tSending disconnect with error code %i\nconch:\treason: %s"
+            % (code, reason)
+        )
+        transport.SSHClientTransport.sendDisconnect(self, code, reason)
+
+    def receiveDebug(self, alwaysDisplay, message, lang):
+        global options
+        if alwaysDisplay or options["log"]:
+            log.msg("Received Debug Message: %s" % message)
+
+    def verifyHostKey(self, pubKey, fingerprint):
+        # d = defer.Deferred()
+        # d.addCallback(lambda x:defer.succeed(1))
+        # d.callback(2)
+        # return d
+        goodKey = isInKnownHosts(options["host"], pubKey, {"known-hosts": None})
+        if goodKey == 1:  # good key
+            return defer.succeed(1)
+        elif goodKey == 2:  # AAHHHHH changed
+            return defer.fail(error.ConchError("bad host key"))
+        else:
+            if options["host"] == self.transport.getPeer().host:
+                host = options["host"]
+                khHost = options["host"]
+            else:
+                host = "{} ({})".format(options["host"], self.transport.getPeer().host)
+                khHost = "{},{}".format(options["host"], self.transport.getPeer().host)
+            keyType = common.getNS(pubKey)[0]
+            ques = """The authenticity of host '{}' can't be established.\r
+{} key fingerprint is {}.""".format(
+                host,
+                {b"ssh-dss": "DSA", b"ssh-rsa": "RSA"}[keyType],
+                fingerprint,
+            )
+            ques += "\r\nAre you sure you want to continue connecting (yes/no)? "
+            return deferredAskFrame(ques, 1).addCallback(
+                self._cbVerifyHostKey, pubKey, khHost, keyType
+            )
+
+    def _cbVerifyHostKey(self, ans, pubKey, khHost, keyType):
+        if ans.lower() not in ("yes", "no"):
+            return deferredAskFrame("Please type  'yes' or 'no': ", 1).addCallback(
+                self._cbVerifyHostKey, pubKey, khHost, keyType
+            )
+        if ans.lower() == "no":
+            frame.write("Host key verification failed.\r\n")
+            raise error.ConchError("bad host key")
+        try:
+            frame.write(
+                "Warning: Permanently added '%s' (%s) to the list of "
+                "known hosts.\r\n"
+                % (khHost, {b"ssh-dss": "DSA", b"ssh-rsa": "RSA"}[keyType])
+            )
+            with open(os.path.expanduser("~/.ssh/known_hosts"), "a") as known_hosts:
+                encodedKey = base64.b64encode(pubKey)
+                known_hosts.write(f"\n{khHost} {keyType} {encodedKey}")
+        except BaseException:
+            log.deferr()
+            raise error.ConchError
+
+    def connectionSecure(self):
+        if options["user"]:
+            user = options["user"]
+        else:
+            user = getpass.getuser()
+        self.requestService(SSHUserAuthClient(user, SSHConnection()))
+
+
+class SSHUserAuthClient(userauth.SSHUserAuthClient):
+    usedFiles: List[str] = []
+
+    def getPassword(self, prompt=None):
+        if not prompt:
+            prompt = "{}@{}'s password: ".format(self.user, options["host"])
+        return deferredAskFrame(prompt, 0)
+
+    def getPublicKey(self):
+        files = [x for x in options.identitys if x not in self.usedFiles]
+        if not files:
+            return None
+        file = files[0]
+        log.msg(file)
+        self.usedFiles.append(file)
+        file = os.path.expanduser(file)
+        file += ".pub"
+        if not os.path.exists(file):
+            return
+        try:
+            return keys.Key.fromFile(file).blob()
+        except BaseException:
+            return self.getPublicKey()  # try again
+
+    def getPrivateKey(self):
+        file = os.path.expanduser(self.usedFiles[-1])
+        if not os.path.exists(file):
+            return None
+        try:
+            return defer.succeed(keys.Key.fromFile(file).keyObject)
+        except keys.BadKeyError as e:
+            if e.args[0] == "encrypted key with no password":
+                prompt = "Enter passphrase for key '%s': " % self.usedFiles[-1]
+                return deferredAskFrame(prompt, 0).addCallback(self._cbGetPrivateKey, 0)
+
+    def _cbGetPrivateKey(self, ans, count):
+        file = os.path.expanduser(self.usedFiles[-1])
+        try:
+            return keys.Key.fromFile(file, password=ans).keyObject
+        except keys.BadKeyError:
+            if count == 2:
+                raise
+            prompt = "Enter passphrase for key '%s': " % self.usedFiles[-1]
+            return deferredAskFrame(prompt, 0).addCallback(
+                self._cbGetPrivateKey, count + 1
+            )
+
+
+class SSHConnection(connection.SSHConnection):
+    def serviceStarted(self):
+        if not options["noshell"]:
+            self.openChannel(SSHSession())
+        if options.localForwards:
+            for localPort, hostport in options.localForwards:
+                reactor.listenTCP(
+                    localPort,
+                    forwarding.SSHListenForwardingFactory(
+                        self, hostport, forwarding.SSHListenClientForwardingChannel
+                    ),
+                )
+        if options.remoteForwards:
+            for remotePort, hostport in options.remoteForwards:
+                log.msg(
+                    "asking for remote forwarding for {}:{}".format(
+                        remotePort, hostport
+                    )
+                )
+                data = forwarding.packGlobal_tcpip_forward(("0.0.0.0", remotePort))
+                self.sendGlobalRequest("tcpip-forward", data)
+                self.remoteForwards[remotePort] = hostport
+
+
+class SSHSession(channel.SSHChannel):
+    name = b"session"
+
+    def channelOpen(self, foo):
+        # global globalSession
+        # globalSession = self
+        # turn off local echo
+        self.escapeMode = 1
+        c = session.SSHSessionClient()
+        if options["escape"]:
+            c.dataReceived = self.handleInput
+        else:
+            c.dataReceived = self.write
+        c.connectionLost = self.sendEOF
+        frame.callback = c.dataReceived
+        frame.canvas.focus_force()
+        if options["subsystem"]:
+            self.conn.sendRequest(self, b"subsystem", common.NS(options["command"]))
+        elif options["command"]:
+            if options["tty"]:
+                term = os.environ.get("TERM", "xterm")
+                # winsz = fcntl.ioctl(fd, tty.TIOCGWINSZ, '12345678')
+                winSize = (25, 80, 0, 0)  # struct.unpack('4H', winsz)
+                ptyReqData = session.packRequest_pty_req(term, winSize, "")
+                self.conn.sendRequest(self, b"pty-req", ptyReqData)
+            self.conn.sendRequest(self, "exec", common.NS(options["command"]))
+        else:
+            if not options["notty"]:
+                term = os.environ.get("TERM", "xterm")
+                # winsz = fcntl.ioctl(fd, tty.TIOCGWINSZ, '12345678')
+                winSize = (25, 80, 0, 0)  # struct.unpack('4H', winsz)
+                ptyReqData = session.packRequest_pty_req(term, winSize, "")
+                self.conn.sendRequest(self, b"pty-req", ptyReqData)
+            self.conn.sendRequest(self, b"shell", b"")
+        self.conn.transport.transport.setTcpNoDelay(1)
+
+    def handleInput(self, char):
+        # log.msg('handling %s' % repr(char))
+        if char in ("\n", "\r"):
+            self.escapeMode = 1
+            self.write(char)
+        elif self.escapeMode == 1 and char == options["escape"]:
+            self.escapeMode = 2
+        elif self.escapeMode == 2:
+            self.escapeMode = 1  # so we can chain escapes together
+            if char == ".":  # disconnect
+                log.msg("disconnecting from escape")
+                reactor.stop()
+                return
+            elif char == "\x1a":  # ^Z, suspend
+                # following line courtesy of Erwin@freenode
+                os.kill(os.getpid(), signal.SIGSTOP)
+                return
+            elif char == "R":  # rekey connection
+                log.msg("rekeying connection")
+                self.conn.transport.sendKexInit()
+                return
+            self.write("~" + char)
+        else:
+            self.escapeMode = 0
+            self.write(char)
+
+    def dataReceived(self, data):
+        data = data.decode("utf-8")
+        if options["ansilog"]:
+            print(repr(data))
+        frame.write(data)
+
+    def extReceived(self, t, data):
+        if t == connection.EXTENDED_DATA_STDERR:
+            log.msg("got %s stderr data" % len(data))
+            sys.stderr.write(data)
+            sys.stderr.flush()
+
+    def eofReceived(self):
+        log.msg("got eof")
+        sys.stdin.close()
+
+    def closed(self):
+        log.msg("closed %s" % self)
+        if len(self.conn.channels) == 1:  # just us left
+            reactor.stop()
+
+    def request_exit_status(self, data):
+        global exitStatus
+        exitStatus = int(struct.unpack(">L", data)[0])
+        log.msg("exit status: %s" % exitStatus)
+
+    def sendEOF(self):
+        self.conn.sendEOF(self)
+
+
+if __name__ == "__main__":
+    run()
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/__init__.py b/contrib/python/Twisted/py3/twisted/conch/ssh/__init__.py
new file mode 100644
index 00000000000..66ee847aa2b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/__init__.py
@@ -0,0 +1,10 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+"""
+An SSHv2 implementation for Twisted.  Part of the Twisted.Conch package.
+
+Maintainer: Paul Swartz
+"""
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/_kex.py b/contrib/python/Twisted/py3/twisted/conch/ssh/_kex.py
new file mode 100644
index 00000000000..c23acec219c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/_kex.py
@@ -0,0 +1,293 @@
+# -*- test-case-name: twisted.conch.test.test_transport -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+SSH key exchange handling.
+"""
+
+
+from hashlib import sha1, sha256, sha384, sha512
+
+from zope.interface import Attribute, Interface, implementer
+
+from twisted.conch import error
+
+
+class _IKexAlgorithm(Interface):
+    """
+    An L{_IKexAlgorithm} describes a key exchange algorithm.
+    """
+
+    preference = Attribute(
+        "An L{int} giving the preference of the algorithm when negotiating "
+        "key exchange. Algorithms with lower precedence values are more "
+        "preferred."
+    )
+
+    hashProcessor = Attribute(
+        "A callable hash algorithm constructor (e.g. C{hashlib.sha256}) "
+        "suitable for use with this key exchange algorithm."
+    )
+
+
+class _IFixedGroupKexAlgorithm(_IKexAlgorithm):
+    """
+    An L{_IFixedGroupKexAlgorithm} describes a key exchange algorithm with a
+    fixed prime / generator group.
+    """
+
+    prime = Attribute(
+        "An L{int} giving the prime number used in Diffie-Hellman key "
+        "exchange, or L{None} if not applicable."
+    )
+
+    generator = Attribute(
+        "An L{int} giving the generator number used in Diffie-Hellman key "
+        "exchange, or L{None} if not applicable. (This is not related to "
+        "Python generator functions.)"
+    )
+
+
+class _IEllipticCurveExchangeKexAlgorithm(_IKexAlgorithm):
+    """
+    An L{_IEllipticCurveExchangeKexAlgorithm} describes a key exchange algorithm
+    that uses an elliptic curve exchange between the client and server.
+    """
+
+
+class _IGroupExchangeKexAlgorithm(_IKexAlgorithm):
+    """
+    An L{_IGroupExchangeKexAlgorithm} describes a key exchange algorithm
+    that uses group exchange between the client and server.
+
+    A prime / generator group should be chosen at run time based on the
+    requested size. See RFC 4419.
+    """
+
+
+@implementer(_IEllipticCurveExchangeKexAlgorithm)
+class _Curve25519SHA256:
+    """
+    Elliptic Curve Key Exchange using Curve25519 and SHA256. Defined in
+    U{https://datatracker.ietf.org/doc/draft-ietf-curdle-ssh-curves/}.
+    """
+
+    preference = 1
+    hashProcessor = sha256
+
+
+@implementer(_IEllipticCurveExchangeKexAlgorithm)
+class _Curve25519SHA256LibSSH:
+    """
+    As L{_Curve25519SHA256}, but with a pre-standardized algorithm name.
+    """
+
+    preference = 2
+    hashProcessor = sha256
+
+
+@implementer(_IEllipticCurveExchangeKexAlgorithm)
+class _ECDH256:
+    """
+    Elliptic Curve Key Exchange with SHA-256 as HASH. Defined in
+    RFC 5656.
+
+    Note that C{ecdh-sha2-nistp256} takes priority over nistp384 or nistp512.
+    This is the same priority from OpenSSH.
+
+    C{ecdh-sha2-nistp256} is considered preety good cryptography.
+    If you need something better consider using C{curve25519-sha256}.
+    """
+
+    preference = 3
+    hashProcessor = sha256
+
+
+@implementer(_IEllipticCurveExchangeKexAlgorithm)
+class _ECDH384:
+    """
+    Elliptic Curve Key Exchange with SHA-384 as HASH. Defined in
+    RFC 5656.
+    """
+
+    preference = 4
+    hashProcessor = sha384
+
+
+@implementer(_IEllipticCurveExchangeKexAlgorithm)
+class _ECDH512:
+    """
+    Elliptic Curve Key Exchange with SHA-512 as HASH. Defined in
+    RFC 5656.
+    """
+
+    preference = 5
+    hashProcessor = sha512
+
+
+@implementer(_IGroupExchangeKexAlgorithm)
+class _DHGroupExchangeSHA256:
+    """
+    Diffie-Hellman Group and Key Exchange with SHA-256 as HASH. Defined in
+    RFC 4419, 4.2.
+    """
+
+    preference = 6
+    hashProcessor = sha256
+
+
+@implementer(_IGroupExchangeKexAlgorithm)
+class _DHGroupExchangeSHA1:
+    """
+    Diffie-Hellman Group and Key Exchange with SHA-1 as HASH. Defined in
+    RFC 4419, 4.1.
+    """
+
+    preference = 7
+    hashProcessor = sha1
+
+
+@implementer(_IFixedGroupKexAlgorithm)
+class _DHGroup14SHA1:
+    """
+    Diffie-Hellman key exchange with SHA-1 as HASH and Oakley Group 14
+    (2048-bit MODP Group). Defined in RFC 4253, 8.2.
+    """
+
+    preference = 8
+    hashProcessor = sha1
+    # Diffie-Hellman primes from Oakley Group 14 (RFC 3526, 3).
+    prime = int(
+        "323170060713110073003389139264238282488179412411402391128420"
+        "097514007417066343542226196894173635693471179017379097041917"
+        "546058732091950288537589861856221532121754125149017745202702"
+        "357960782362488842461894775876411059286460994117232454266225"
+        "221932305409190376805242355191256797158701170010580558776510"
+        "388618472802579760549035697325615261670813393617995413364765"
+        "591603683178967290731783845896806396719009772021941686472258"
+        "710314113364293195361934716365332097170774482279885885653692"
+        "086452966360772502689555059283627511211740969729980684105543"
+        "595848665832916421362182310789909994486524682624169720359118"
+        "52507045361090559"
+    )
+    generator = 2
+
+
+# Which ECDH hash function to use is dependent on the size.
+_kexAlgorithms = {
+    b"curve25519-sha256": _Curve25519SHA256(),
+    b"curve25519-sha256@libssh.org": _Curve25519SHA256LibSSH(),
+    b"diffie-hellman-group-exchange-sha256": _DHGroupExchangeSHA256(),
+    b"diffie-hellman-group-exchange-sha1": _DHGroupExchangeSHA1(),
+    b"diffie-hellman-group14-sha1": _DHGroup14SHA1(),
+    b"ecdh-sha2-nistp256": _ECDH256(),
+    b"ecdh-sha2-nistp384": _ECDH384(),
+    b"ecdh-sha2-nistp521": _ECDH512(),
+}
+
+
+def getKex(kexAlgorithm):
+    """
+    Get a description of a named key exchange algorithm.
+
+    @param kexAlgorithm: The key exchange algorithm name.
+    @type kexAlgorithm: L{bytes}
+
+    @return: A description of the key exchange algorithm named by
+        C{kexAlgorithm}.
+    @rtype: L{_IKexAlgorithm}
+
+    @raises ConchError: if the key exchange algorithm is not found.
+    """
+    if kexAlgorithm not in _kexAlgorithms:
+        raise error.ConchError(f"Unsupported key exchange algorithm: {kexAlgorithm}")
+    return _kexAlgorithms[kexAlgorithm]
+
+
+def isEllipticCurve(kexAlgorithm):
+    """
+    Returns C{True} if C{kexAlgorithm} is an elliptic curve.
+
+    @param kexAlgorithm: The key exchange algorithm name.
+    @type kexAlgorithm: C{str}
+
+    @return: C{True} if C{kexAlgorithm} is an elliptic curve,
+        otherwise C{False}.
+    @rtype: C{bool}
+    """
+    return _IEllipticCurveExchangeKexAlgorithm.providedBy(getKex(kexAlgorithm))
+
+
+def isFixedGroup(kexAlgorithm):
+    """
+    Returns C{True} if C{kexAlgorithm} has a fixed prime / generator group.
+
+    @param kexAlgorithm: The key exchange algorithm name.
+    @type kexAlgorithm: L{bytes}
+
+    @return: C{True} if C{kexAlgorithm} has a fixed prime / generator group,
+        otherwise C{False}.
+    @rtype: L{bool}
+    """
+    return _IFixedGroupKexAlgorithm.providedBy(getKex(kexAlgorithm))
+
+
+def getHashProcessor(kexAlgorithm):
+    """
+    Get the hash algorithm callable to use in key exchange.
+
+    @param kexAlgorithm: The key exchange algorithm name.
+    @type kexAlgorithm: L{bytes}
+
+    @return: A callable hash algorithm constructor (e.g. C{hashlib.sha256}).
+    @rtype: C{callable}
+    """
+    kex = getKex(kexAlgorithm)
+    return kex.hashProcessor
+
+
+def getDHGeneratorAndPrime(kexAlgorithm):
+    """
+    Get the generator and the prime to use in key exchange.
+
+    @param kexAlgorithm: The key exchange algorithm name.
+    @type kexAlgorithm: L{bytes}
+
+    @return: A L{tuple} containing L{int} generator and L{int} prime.
+    @rtype: L{tuple}
+    """
+    kex = getKex(kexAlgorithm)
+    return kex.generator, kex.prime
+
+
+def getSupportedKeyExchanges():
+    """
+    Get a list of supported key exchange algorithm names in order of
+    preference.
+
+    @return: A C{list} of supported key exchange algorithm names.
+    @rtype: C{list} of L{bytes}
+    """
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives.asymmetric import ec
+
+    from twisted.conch.ssh.keys import _curveTable
+
+    backend = default_backend()
+    kexAlgorithms = _kexAlgorithms.copy()
+    for keyAlgorithm in list(kexAlgorithms):
+        if keyAlgorithm.startswith(b"ecdh"):
+            keyAlgorithmDsa = keyAlgorithm.replace(b"ecdh", b"ecdsa")
+            supported = backend.elliptic_curve_exchange_algorithm_supported(
+                ec.ECDH(), _curveTable[keyAlgorithmDsa]
+            )
+        elif keyAlgorithm.startswith(b"curve25519-sha256"):
+            supported = backend.x25519_supported()
+        else:
+            supported = True
+        if not supported:
+            kexAlgorithms.pop(keyAlgorithm)
+    return sorted(
+        kexAlgorithms, key=lambda kexAlgorithm: kexAlgorithms[kexAlgorithm].preference
+    )
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/address.py b/contrib/python/Twisted/py3/twisted/conch/ssh/address.py
new file mode 100644
index 00000000000..88ce0d1c7fb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/address.py
@@ -0,0 +1,43 @@
+# -*- test-case-name: twisted.conch.test.test_address -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Address object for SSH network connections.
+
+Maintainer: Paul Swartz
+
+@since: 12.1
+"""
+
+
+from zope.interface import implementer
+
+from twisted.internet.interfaces import IAddress
+from twisted.python import util
+
+
+@implementer(IAddress)
+class SSHTransportAddress(util.FancyEqMixin):
+    """
+    Object representing an SSH Transport endpoint.
+
+    This is used to ensure that any code inspecting this address and
+    attempting to construct a similar connection based upon it is not
+    mislead into creating a transport which is not similar to the one it is
+    indicating.
+
+    @ivar address: An instance of an object which implements I{IAddress} to
+        which this transport address is connected.
+    """
+
+    compareAttributes = ("address",)
+
+    def __init__(self, address):
+        self.address = address
+
+    def __repr__(self) -> str:
+        return f"SSHTransportAddress({self.address!r})"
+
+    def __hash__(self):
+        return hash(("SSH", self.address))
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/agent.py b/contrib/python/Twisted/py3/twisted/conch/ssh/agent.py
new file mode 100644
index 00000000000..359b266fd6f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/agent.py
@@ -0,0 +1,278 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implements the SSH v2 key agent protocol.  This protocol is documented in the
+SSH source code, in the file
+U{PROTOCOL.agent<http://www.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL.agent>}.
+
+Maintainer: Paul Swartz
+"""
+
+
+import struct
+
+from twisted.conch.error import ConchError, MissingKeyStoreError
+from twisted.conch.ssh import keys
+from twisted.conch.ssh.common import NS, getMP, getNS
+from twisted.internet import defer, protocol
+
+
+class SSHAgentClient(protocol.Protocol):
+    """
+    The client side of the SSH agent protocol.  This is equivalent to
+    ssh-add(1) and can be used with either ssh-agent(1) or the SSHAgentServer
+    protocol, also in this package.
+    """
+
+    def __init__(self):
+        self.buf = b""
+        self.deferreds = []
+
+    def dataReceived(self, data):
+        self.buf += data
+        while 1:
+            if len(self.buf) <= 4:
+                return
+            packLen = struct.unpack("!L", self.buf[:4])[0]
+            if len(self.buf) < 4 + packLen:
+                return
+            packet, self.buf = self.buf[4 : 4 + packLen], self.buf[4 + packLen :]
+            reqType = ord(packet[0:1])
+            d = self.deferreds.pop(0)
+            if reqType == AGENT_FAILURE:
+                d.errback(ConchError("agent failure"))
+            elif reqType == AGENT_SUCCESS:
+                d.callback(b"")
+            else:
+                d.callback(packet)
+
+    def sendRequest(self, reqType, data):
+        pack = struct.pack("!LB", len(data) + 1, reqType) + data
+        self.transport.write(pack)
+        d = defer.Deferred()
+        self.deferreds.append(d)
+        return d
+
+    def requestIdentities(self):
+        """
+        @return: A L{Deferred} which will fire with a list of all keys found in
+            the SSH agent. The list of keys is comprised of (public key blob,
+            comment) tuples.
+        """
+        d = self.sendRequest(AGENTC_REQUEST_IDENTITIES, b"")
+        d.addCallback(self._cbRequestIdentities)
+        return d
+
+    def _cbRequestIdentities(self, data):
+        """
+        Unpack a collection of identities into a list of tuples comprised of
+        public key blobs and comments.
+        """
+        if ord(data[0:1]) != AGENT_IDENTITIES_ANSWER:
+            raise ConchError("unexpected response: %i" % ord(data[0:1]))
+        numKeys = struct.unpack("!L", data[1:5])[0]
+        result = []
+        data = data[5:]
+        for i in range(numKeys):
+            blob, data = getNS(data)
+            comment, data = getNS(data)
+            result.append((blob, comment))
+        return result
+
+    def addIdentity(self, blob, comment=b""):
+        """
+        Add a private key blob to the agent's collection of keys.
+        """
+        req = blob
+        req += NS(comment)
+        return self.sendRequest(AGENTC_ADD_IDENTITY, req)
+
+    def signData(self, blob, data):
+        """
+        Request that the agent sign the given C{data} with the private key
+        which corresponds to the public key given by C{blob}.  The private
+        key should have been added to the agent already.
+
+        @type blob: L{bytes}
+        @type data: L{bytes}
+        @return: A L{Deferred} which fires with a signature for given data
+            created with the given key.
+        """
+        req = NS(blob)
+        req += NS(data)
+        req += b"\000\000\000\000"  # flags
+        return self.sendRequest(AGENTC_SIGN_REQUEST, req).addCallback(self._cbSignData)
+
+    def _cbSignData(self, data):
+        if ord(data[0:1]) != AGENT_SIGN_RESPONSE:
+            raise ConchError("unexpected data: %i" % ord(data[0:1]))
+        signature = getNS(data[1:])[0]
+        return signature
+
+    def removeIdentity(self, blob):
+        """
+        Remove the private key corresponding to the public key in blob from the
+        running agent.
+        """
+        req = NS(blob)
+        return self.sendRequest(AGENTC_REMOVE_IDENTITY, req)
+
+    def removeAllIdentities(self):
+        """
+        Remove all keys from the running agent.
+        """
+        return self.sendRequest(AGENTC_REMOVE_ALL_IDENTITIES, b"")
+
+
+class SSHAgentServer(protocol.Protocol):
+    """
+    The server side of the SSH agent protocol.  This is equivalent to
+    ssh-agent(1) and can be used with either ssh-add(1) or the SSHAgentClient
+    protocol, also in this package.
+    """
+
+    def __init__(self):
+        self.buf = b""
+
+    def dataReceived(self, data):
+        self.buf += data
+        while 1:
+            if len(self.buf) <= 4:
+                return
+            packLen = struct.unpack("!L", self.buf[:4])[0]
+            if len(self.buf) < 4 + packLen:
+                return
+            packet, self.buf = self.buf[4 : 4 + packLen], self.buf[4 + packLen :]
+            reqType = ord(packet[0:1])
+            reqName = messages.get(reqType, None)
+            if not reqName:
+                self.sendResponse(AGENT_FAILURE, b"")
+            else:
+                f = getattr(self, "agentc_%s" % reqName)
+                if getattr(self.factory, "keys", None) is None:
+                    self.sendResponse(AGENT_FAILURE, b"")
+                    raise MissingKeyStoreError()
+                f(packet[1:])
+
+    def sendResponse(self, reqType, data):
+        pack = struct.pack("!LB", len(data) + 1, reqType) + data
+        self.transport.write(pack)
+
+    def agentc_REQUEST_IDENTITIES(self, data):
+        """
+        Return all of the identities that have been added to the server
+        """
+        assert data == b""
+        numKeys = len(self.factory.keys)
+        resp = []
+
+        resp.append(struct.pack("!L", numKeys))
+        for key, comment in self.factory.keys.values():
+            resp.append(NS(key.blob()))  # yes, wrapped in an NS
+            resp.append(NS(comment))
+        self.sendResponse(AGENT_IDENTITIES_ANSWER, b"".join(resp))
+
+    def agentc_SIGN_REQUEST(self, data):
+        """
+        Data is a structure with a reference to an already added key object and
+        some data that the clients wants signed with that key.  If the key
+        object wasn't loaded, return AGENT_FAILURE, else return the signature.
+        """
+        blob, data = getNS(data)
+        if blob not in self.factory.keys:
+            return self.sendResponse(AGENT_FAILURE, b"")
+        signData, data = getNS(data)
+        assert data == b"\000\000\000\000"
+        self.sendResponse(
+            AGENT_SIGN_RESPONSE, NS(self.factory.keys[blob][0].sign(signData))
+        )
+
+    def agentc_ADD_IDENTITY(self, data):
+        """
+        Adds a private key to the agent's collection of identities.  On
+        subsequent interactions, the private key can be accessed using only the
+        corresponding public key.
+        """
+
+        # need to pre-read the key data so we can get past it to the comment string
+        keyType, rest = getNS(data)
+        if keyType == b"ssh-rsa":
+            nmp = 6
+        elif keyType == b"ssh-dss":
+            nmp = 5
+        else:
+            raise keys.BadKeyError("unknown blob type: %s" % keyType)
+
+        rest = getMP(rest, nmp)[
+            -1
+        ]  # ignore the key data for now, we just want the comment
+        comment, rest = getNS(rest)  # the comment, tacked onto the end of the key blob
+
+        k = keys.Key.fromString(data, type="private_blob")  # not wrapped in NS here
+        self.factory.keys[k.blob()] = (k, comment)
+        self.sendResponse(AGENT_SUCCESS, b"")
+
+    def agentc_REMOVE_IDENTITY(self, data):
+        """
+        Remove a specific key from the agent's collection of identities.
+        """
+        blob, _ = getNS(data)
+        k = keys.Key.fromString(blob, type="blob")
+        del self.factory.keys[k.blob()]
+        self.sendResponse(AGENT_SUCCESS, b"")
+
+    def agentc_REMOVE_ALL_IDENTITIES(self, data):
+        """
+        Remove all keys from the agent's collection of identities.
+        """
+        assert data == b""
+        self.factory.keys = {}
+        self.sendResponse(AGENT_SUCCESS, b"")
+
+    # v1 messages that we ignore because we don't keep v1 keys
+    # open-ssh sends both v1 and v2 commands, so we have to
+    # do no-ops for v1 commands or we'll get "bad request" errors
+
+    def agentc_REQUEST_RSA_IDENTITIES(self, data):
+        """
+        v1 message for listing RSA1 keys; superseded by
+        agentc_REQUEST_IDENTITIES, which handles different key types.
+        """
+        self.sendResponse(AGENT_RSA_IDENTITIES_ANSWER, struct.pack("!L", 0))
+
+    def agentc_REMOVE_RSA_IDENTITY(self, data):
+        """
+        v1 message for removing RSA1 keys; superseded by
+        agentc_REMOVE_IDENTITY, which handles different key types.
+        """
+        self.sendResponse(AGENT_SUCCESS, b"")
+
+    def agentc_REMOVE_ALL_RSA_IDENTITIES(self, data):
+        """
+        v1 message for removing all RSA1 keys; superseded by
+        agentc_REMOVE_ALL_IDENTITIES, which handles different key types.
+        """
+        self.sendResponse(AGENT_SUCCESS, b"")
+
+
+AGENTC_REQUEST_RSA_IDENTITIES = 1
+AGENT_RSA_IDENTITIES_ANSWER = 2
+AGENT_FAILURE = 5
+AGENT_SUCCESS = 6
+
+AGENTC_REMOVE_RSA_IDENTITY = 8
+AGENTC_REMOVE_ALL_RSA_IDENTITIES = 9
+
+AGENTC_REQUEST_IDENTITIES = 11
+AGENT_IDENTITIES_ANSWER = 12
+AGENTC_SIGN_REQUEST = 13
+AGENT_SIGN_RESPONSE = 14
+AGENTC_ADD_IDENTITY = 17
+AGENTC_REMOVE_IDENTITY = 18
+AGENTC_REMOVE_ALL_IDENTITIES = 19
+
+messages = {}
+for name, value in locals().copy().items():
+    if name[:7] == "AGENTC_":
+        messages[value] = name[7:]  # doesn't handle doubles
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/channel.py b/contrib/python/Twisted/py3/twisted/conch/ssh/channel.py
new file mode 100644
index 00000000000..ab54f1f85cc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/channel.py
@@ -0,0 +1,312 @@
+# -*- test-case-name: twisted.conch.test.test_channel -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+The parent class for all the SSH Channels.  Currently implemented channels
+are session, direct-tcp, and forwarded-tcp.
+
+Maintainer: Paul Swartz
+"""
+
+
+from zope.interface import implementer
+
+from twisted.internet import interfaces
+from twisted.logger import Logger
+from twisted.python import log
+
+
+@implementer(interfaces.ITransport)
+class SSHChannel(log.Logger):
+    """
+    A class that represents a multiplexed channel over an SSH connection.
+    The channel has a local window which is the maximum amount of data it will
+    receive, and a remote which is the maximum amount of data the remote side
+    will accept.  There is also a maximum packet size for any individual data
+    packet going each way.
+
+    @ivar name: the name of the channel.
+    @type name: L{bytes}
+    @ivar localWindowSize: the maximum size of the local window in bytes.
+    @type localWindowSize: L{int}
+    @ivar localWindowLeft: how many bytes are left in the local window.
+    @type localWindowLeft: L{int}
+    @ivar localMaxPacket: the maximum size of packet we will accept in bytes.
+    @type localMaxPacket: L{int}
+    @ivar remoteWindowLeft: how many bytes are left in the remote window.
+    @type remoteWindowLeft: L{int}
+    @ivar remoteMaxPacket: the maximum size of a packet the remote side will
+        accept in bytes.
+    @type remoteMaxPacket: L{int}
+    @ivar conn: the connection this channel is multiplexed through.
+    @type conn: L{SSHConnection}
+    @ivar data: any data to send to the other side when the channel is
+        requested.
+    @type data: L{bytes}
+    @ivar avatar: an avatar for the logged-in user (if a server channel)
+    @ivar localClosed: True if we aren't accepting more data.
+    @type localClosed: L{bool}
+    @ivar remoteClosed: True if the other side isn't accepting more data.
+    @type remoteClosed: L{bool}
+    """
+
+    _log = Logger()
+    name: bytes = None  # type: ignore[assignment]  # only needed for client channels
+
+    def __init__(
+        self,
+        localWindow=0,
+        localMaxPacket=0,
+        remoteWindow=0,
+        remoteMaxPacket=0,
+        conn=None,
+        data=None,
+        avatar=None,
+    ):
+        self.localWindowSize = localWindow or 131072
+        self.localWindowLeft = self.localWindowSize
+        self.localMaxPacket = localMaxPacket or 32768
+        self.remoteWindowLeft = remoteWindow
+        self.remoteMaxPacket = remoteMaxPacket
+        self.areWriting = 1
+        self.conn = conn
+        self.data = data
+        self.avatar = avatar
+        self.specificData = b""
+        self.buf = b""
+        self.extBuf = []
+        self.closing = 0
+        self.localClosed = 0
+        self.remoteClosed = 0
+        self.id = None  # gets set later by SSHConnection
+
+    def __str__(self) -> str:
+        return self.__bytes__().decode("ascii")
+
+    def __bytes__(self) -> bytes:
+        """
+        Return a byte string representation of the channel
+        """
+        name = self.name
+        if not name:
+            name = b"None"
+
+        return b"<SSHChannel %b (lw %d rw %d)>" % (
+            name,
+            self.localWindowLeft,
+            self.remoteWindowLeft,
+        )
+
+    def logPrefix(self):
+        id = (self.id is not None and str(self.id)) or "unknown"
+        if self.name:
+            name = self.name.decode("ascii")
+        else:
+            name = "None"
+        return f"SSHChannel {name} ({id}) on {self.conn.logPrefix()}"
+
+    def channelOpen(self, specificData):
+        """
+        Called when the channel is opened.  specificData is any data that the
+        other side sent us when opening the channel.
+
+        @type specificData: L{bytes}
+        """
+        self._log.info("channel open")
+
+    def openFailed(self, reason):
+        """
+        Called when the open failed for some reason.
+        reason.desc is a string descrption, reason.code the SSH error code.
+
+        @type reason: L{error.ConchError}
+        """
+        self._log.error("other side refused open\nreason: {reason}", reason=reason)
+
+    def addWindowBytes(self, data):
+        """
+        Called when bytes are added to the remote window.  By default it clears
+        the data buffers.
+
+        @type data:    L{bytes}
+        """
+        self.remoteWindowLeft = self.remoteWindowLeft + data
+        if not self.areWriting and not self.closing:
+            self.areWriting = True
+            self.startWriting()
+        if self.buf:
+            b = self.buf
+            self.buf = b""
+            self.write(b)
+        if self.extBuf:
+            b = self.extBuf
+            self.extBuf = []
+            for type, data in b:
+                self.writeExtended(type, data)
+
+    def requestReceived(self, requestType, data):
+        """
+        Called when a request is sent to this channel.  By default it delegates
+        to self.request_<requestType>.
+        If this function returns true, the request succeeded, otherwise it
+        failed.
+
+        @type requestType:  L{bytes}
+        @type data:         L{bytes}
+        @rtype:             L{bool}
+        """
+        foo = requestType.replace(b"-", b"_").decode("ascii")
+        f = getattr(self, "request_" + foo, None)
+        if f:
+            return f(data)
+        self._log.info("unhandled request for {requestType}", requestType=requestType)
+        return 0
+
+    def dataReceived(self, data):
+        """
+        Called when we receive data.
+
+        @type data: L{bytes}
+        """
+        self._log.debug("got data {data}", data=data)
+
+    def extReceived(self, dataType, data):
+        """
+        Called when we receive extended data (usually standard error).
+
+        @type dataType: L{int}
+        @type data:     L{str}
+        """
+        self._log.debug(
+            "got extended data {dataType} {data!r}", dataType=dataType, data=data
+        )
+
+    def eofReceived(self):
+        """
+        Called when the other side will send no more data.
+        """
+        self._log.info("remote eof")
+
+    def closeReceived(self):
+        """
+        Called when the other side has closed the channel.
+        """
+        self._log.info("remote close")
+        self.loseConnection()
+
+    def closed(self):
+        """
+        Called when the channel is closed.  This means that both our side and
+        the remote side have closed the channel.
+        """
+        self._log.info("closed")
+
+    def write(self, data):
+        """
+        Write some data to the channel.  If there is not enough remote window
+        available, buffer until it is.  Otherwise, split the data into
+        packets of length remoteMaxPacket and send them.
+
+        @type data: L{bytes}
+        """
+        if self.buf:
+            self.buf += data
+            return
+        top = len(data)
+        if top > self.remoteWindowLeft:
+            data, self.buf = (
+                data[: self.remoteWindowLeft],
+                data[self.remoteWindowLeft :],
+            )
+            self.areWriting = 0
+            self.stopWriting()
+            top = self.remoteWindowLeft
+        rmp = self.remoteMaxPacket
+        write = self.conn.sendData
+        r = range(0, top, rmp)
+        for offset in r:
+            write(self, data[offset : offset + rmp])
+        self.remoteWindowLeft -= top
+        if self.closing and not self.buf:
+            self.loseConnection()  # try again
+
+    def writeExtended(self, dataType, data):
+        """
+        Send extended data to this channel.  If there is not enough remote
+        window available, buffer until there is.  Otherwise, split the data
+        into packets of length remoteMaxPacket and send them.
+
+        @type dataType: L{int}
+        @type data:     L{bytes}
+        """
+        if self.extBuf:
+            if self.extBuf[-1][0] == dataType:
+                self.extBuf[-1][1] += data
+            else:
+                self.extBuf.append([dataType, data])
+            return
+        if len(data) > self.remoteWindowLeft:
+            data, self.extBuf = (
+                data[: self.remoteWindowLeft],
+                [[dataType, data[self.remoteWindowLeft :]]],
+            )
+            self.areWriting = 0
+            self.stopWriting()
+        while len(data) > self.remoteMaxPacket:
+            self.conn.sendExtendedData(self, dataType, data[: self.remoteMaxPacket])
+            data = data[self.remoteMaxPacket :]
+            self.remoteWindowLeft -= self.remoteMaxPacket
+        if data:
+            self.conn.sendExtendedData(self, dataType, data)
+            self.remoteWindowLeft -= len(data)
+        if self.closing:
+            self.loseConnection()  # try again
+
+    def writeSequence(self, data):
+        """
+        Part of the Transport interface.  Write a list of strings to the
+        channel.
+
+        @type data: C{list} of L{str}
+        """
+        self.write(b"".join(data))
+
+    def loseConnection(self):
+        """
+        Close the channel if there is no buferred data.  Otherwise, note the
+        request and return.
+        """
+        self.closing = 1
+        if not self.buf and not self.extBuf:
+            self.conn.sendClose(self)
+
+    def getPeer(self):
+        """
+        See: L{ITransport.getPeer}
+
+        @return: The remote address of this connection.
+        @rtype: L{SSHTransportAddress}.
+        """
+        return self.conn.transport.getPeer()
+
+    def getHost(self):
+        """
+        See: L{ITransport.getHost}
+
+        @return: An address describing this side of the connection.
+        @rtype: L{SSHTransportAddress}.
+        """
+        return self.conn.transport.getHost()
+
+    def stopWriting(self):
+        """
+        Called when the remote buffer is full, as a hint to stop writing.
+        This can be ignored, but it can be helpful.
+        """
+
+    def startWriting(self):
+        """
+        Called when the remote buffer has more room, as a hint to continue
+        writing.
+        """
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/common.py b/contrib/python/Twisted/py3/twisted/conch/ssh/common.py
new file mode 100644
index 00000000000..8bb6a286c3b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/common.py
@@ -0,0 +1,85 @@
+# -*- test-case-name: twisted.conch.test.test_ssh -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Common functions for the SSH classes.
+
+Maintainer: Paul Swartz
+"""
+
+
+import struct
+
+from cryptography.utils import int_to_bytes
+
+from twisted.python.deprecate import deprecated
+from twisted.python.versions import Version
+
+__all__ = ["NS", "getNS", "MP", "getMP", "ffs"]
+
+
+def NS(t):
+    """
+    net string
+    """
+    if isinstance(t, str):
+        t = t.encode("utf-8")
+    return struct.pack("!L", len(t)) + t
+
+
+def getNS(s, count=1):
+    """
+    get net string
+    """
+    ns = []
+    c = 0
+    for i in range(count):
+        (l,) = struct.unpack("!L", s[c : c + 4])
+        ns.append(s[c + 4 : 4 + l + c])
+        c += 4 + l
+    return tuple(ns) + (s[c:],)
+
+
+def MP(number):
+    if number == 0:
+        return b"\000" * 4
+    assert number > 0
+    bn = int_to_bytes(number)
+    if ord(bn[0:1]) & 128:
+        bn = b"\000" + bn
+    return struct.pack(">L", len(bn)) + bn
+
+
+def getMP(data, count=1):
+    """
+    Get multiple precision integer out of the string.  A multiple precision
+    integer is stored as a 4-byte length followed by length bytes of the
+    integer.  If count is specified, get count integers out of the string.
+    The return value is a tuple of count integers followed by the rest of
+    the data.
+    """
+    mp = []
+    c = 0
+    for i in range(count):
+        (length,) = struct.unpack(">L", data[c : c + 4])
+        mp.append(int.from_bytes(data[c + 4 : c + 4 + length], "big"))
+        c += 4 + length
+    return tuple(mp) + (data[c:],)
+
+
+def ffs(c, s):
+    """
+    first from second
+    goes through the first list, looking for items in the second, returns the first one
+    """
+    for i in c:
+        if i in s:
+            return i
+
+
+@deprecated(Version("Twisted", 16, 5, 0))
+def install():
+    # This used to install gmpy, but is technically public API, so just do
+    # nothing.
+    pass
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/connection.py b/contrib/python/Twisted/py3/twisted/conch/ssh/connection.py
new file mode 100644
index 00000000000..c740a545810
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/connection.py
@@ -0,0 +1,679 @@
+# -*- test-case-name: twisted.conch.test.test_connection -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module contains the implementation of the ssh-connection service, which
+allows access to the shell and port-forwarding.
+
+Maintainer: Paul Swartz
+"""
+
+import string
+import struct
+
+import twisted.internet.error
+from twisted.conch import error
+from twisted.conch.ssh import common, service
+from twisted.internet import defer
+from twisted.logger import Logger
+from twisted.python.compat import nativeString, networkString
+
+
+class SSHConnection(service.SSHService):
+    """
+    An implementation of the 'ssh-connection' service.  It is used to
+    multiplex multiple channels over the single SSH connection.
+
+    @ivar localChannelID: the next number to use as a local channel ID.
+    @type localChannelID: L{int}
+    @ivar channels: a L{dict} mapping a local channel ID to C{SSHChannel}
+        subclasses.
+    @type channels: L{dict}
+    @ivar localToRemoteChannel: a L{dict} mapping a local channel ID to a
+        remote channel ID.
+    @type localToRemoteChannel: L{dict}
+    @ivar channelsToRemoteChannel: a L{dict} mapping a C{SSHChannel} subclass
+        to remote channel ID.
+    @type channelsToRemoteChannel: L{dict}
+    @ivar deferreds: a L{dict} mapping a local channel ID to a C{list} of
+        C{Deferreds} for outstanding channel requests.  Also, the 'global'
+        key stores the C{list} of pending global request C{Deferred}s.
+    """
+
+    name = b"ssh-connection"
+    _log = Logger()
+
+    def __init__(self):
+        self.localChannelID = 0  # this is the current # to use for channel ID
+        # local channel ID -> remote channel ID
+        self.localToRemoteChannel = {}
+        # local channel ID -> subclass of SSHChannel
+        self.channels = {}
+        # subclass of SSHChannel -> remote channel ID
+        self.channelsToRemoteChannel = {}
+        # local channel -> list of deferreds for pending requests
+        # or 'global' -> list of deferreds for global requests
+        self.deferreds = {"global": []}
+
+        self.transport = None  # gets set later
+
+    def serviceStarted(self):
+        if hasattr(self.transport, "avatar"):
+            self.transport.avatar.conn = self
+
+    def serviceStopped(self):
+        """
+        Called when the connection is stopped.
+        """
+        # Close any fully open channels
+        for channel in list(self.channelsToRemoteChannel.keys()):
+            self.channelClosed(channel)
+        # Indicate failure to any channels that were in the process of
+        # opening but not yet open.
+        while self.channels:
+            (_, channel) = self.channels.popitem()
+            channel.openFailed(twisted.internet.error.ConnectionLost())
+        # Errback any unfinished global requests.
+        self._cleanupGlobalDeferreds()
+
+    def _cleanupGlobalDeferreds(self):
+        """
+        All pending requests that have returned a deferred must be errbacked
+        when this service is stopped, otherwise they might be left uncalled and
+        uncallable.
+        """
+        for d in self.deferreds["global"]:
+            d.errback(error.ConchError("Connection stopped."))
+        del self.deferreds["global"][:]
+
+    # packet methods
+    def ssh_GLOBAL_REQUEST(self, packet):
+        """
+        The other side has made a global request.  Payload::
+            string  request type
+            bool    want reply
+            <request specific data>
+
+        This dispatches to self.gotGlobalRequest.
+        """
+        requestType, rest = common.getNS(packet)
+        wantReply, rest = ord(rest[0:1]), rest[1:]
+        ret = self.gotGlobalRequest(requestType, rest)
+        if wantReply:
+            reply = MSG_REQUEST_FAILURE
+            data = b""
+            if ret:
+                reply = MSG_REQUEST_SUCCESS
+                if isinstance(ret, (tuple, list)):
+                    data = ret[1]
+            self.transport.sendPacket(reply, data)
+
+    def ssh_REQUEST_SUCCESS(self, packet):
+        """
+        Our global request succeeded.  Get the appropriate Deferred and call
+        it back with the packet we received.
+        """
+        self._log.debug("global request success")
+        self.deferreds["global"].pop(0).callback(packet)
+
+    def ssh_REQUEST_FAILURE(self, packet):
+        """
+        Our global request failed.  Get the appropriate Deferred and errback
+        it with the packet we received.
+        """
+        self._log.debug("global request failure")
+        self.deferreds["global"].pop(0).errback(
+            error.ConchError("global request failed", packet)
+        )
+
+    def ssh_CHANNEL_OPEN(self, packet):
+        """
+        The other side wants to get a channel.  Payload::
+            string  channel name
+            uint32  remote channel number
+            uint32  remote window size
+            uint32  remote maximum packet size
+            <channel specific data>
+
+        We get a channel from self.getChannel(), give it a local channel number
+        and notify the other side.  Then notify the channel by calling its
+        channelOpen method.
+        """
+        channelType, rest = common.getNS(packet)
+        senderChannel, windowSize, maxPacket = struct.unpack(">3L", rest[:12])
+        packet = rest[12:]
+        try:
+            channel = self.getChannel(channelType, windowSize, maxPacket, packet)
+            localChannel = self.localChannelID
+            self.localChannelID += 1
+            channel.id = localChannel
+            self.channels[localChannel] = channel
+            self.channelsToRemoteChannel[channel] = senderChannel
+            self.localToRemoteChannel[localChannel] = senderChannel
+            openConfirmPacket = (
+                struct.pack(
+                    ">4L",
+                    senderChannel,
+                    localChannel,
+                    channel.localWindowSize,
+                    channel.localMaxPacket,
+                )
+                + channel.specificData
+            )
+            self.transport.sendPacket(MSG_CHANNEL_OPEN_CONFIRMATION, openConfirmPacket)
+            channel.channelOpen(packet)
+        except Exception as e:
+            self._log.failure("channel open failed")
+            if isinstance(e, error.ConchError):
+                textualInfo, reason = e.args
+                if isinstance(textualInfo, int):
+                    # See #3657 and #3071
+                    textualInfo, reason = reason, textualInfo
+            else:
+                reason = OPEN_CONNECT_FAILED
+                textualInfo = "unknown failure"
+            self.transport.sendPacket(
+                MSG_CHANNEL_OPEN_FAILURE,
+                struct.pack(">2L", senderChannel, reason)
+                + common.NS(networkString(textualInfo))
+                + common.NS(b""),
+            )
+
+    def ssh_CHANNEL_OPEN_CONFIRMATION(self, packet):
+        """
+        The other side accepted our MSG_CHANNEL_OPEN request.  Payload::
+            uint32  local channel number
+            uint32  remote channel number
+            uint32  remote window size
+            uint32  remote maximum packet size
+            <channel specific data>
+
+        Find the channel using the local channel number and notify its
+        channelOpen method.
+        """
+        (localChannel, remoteChannel, windowSize, maxPacket) = struct.unpack(
+            ">4L", packet[:16]
+        )
+        specificData = packet[16:]
+        channel = self.channels[localChannel]
+        channel.conn = self
+        self.localToRemoteChannel[localChannel] = remoteChannel
+        self.channelsToRemoteChannel[channel] = remoteChannel
+        channel.remoteWindowLeft = windowSize
+        channel.remoteMaxPacket = maxPacket
+        channel.channelOpen(specificData)
+
+    def ssh_CHANNEL_OPEN_FAILURE(self, packet):
+        """
+        The other side did not accept our MSG_CHANNEL_OPEN request.  Payload::
+            uint32  local channel number
+            uint32  reason code
+            string  reason description
+
+        Find the channel using the local channel number and notify it by
+        calling its openFailed() method.
+        """
+        localChannel, reasonCode = struct.unpack(">2L", packet[:8])
+        reasonDesc = common.getNS(packet[8:])[0]
+        channel = self.channels[localChannel]
+        del self.channels[localChannel]
+        channel.conn = self
+        reason = error.ConchError(reasonDesc, reasonCode)
+        channel.openFailed(reason)
+
+    def ssh_CHANNEL_WINDOW_ADJUST(self, packet):
+        """
+        The other side is adding bytes to its window.  Payload::
+            uint32  local channel number
+            uint32  bytes to add
+
+        Call the channel's addWindowBytes() method to add new bytes to the
+        remote window.
+        """
+        localChannel, bytesToAdd = struct.unpack(">2L", packet[:8])
+        channel = self.channels[localChannel]
+        channel.addWindowBytes(bytesToAdd)
+
+    def ssh_CHANNEL_DATA(self, packet):
+        """
+        The other side is sending us data.  Payload::
+            uint32 local channel number
+            string data
+
+        Check to make sure the other side hasn't sent too much data (more
+        than what's in the window, or more than the maximum packet size).  If
+        they have, close the channel.  Otherwise, decrease the available
+        window and pass the data to the channel's dataReceived().
+        """
+        localChannel, dataLength = struct.unpack(">2L", packet[:8])
+        channel = self.channels[localChannel]
+        # XXX should this move to dataReceived to put client in charge?
+        if (
+            dataLength > channel.localWindowLeft or dataLength > channel.localMaxPacket
+        ):  # more data than we want
+            self._log.error("too much data")
+            self.sendClose(channel)
+            return
+            # packet = packet[:channel.localWindowLeft+4]
+        data = common.getNS(packet[4:])[0]
+        channel.localWindowLeft -= dataLength
+        if channel.localWindowLeft < channel.localWindowSize // 2:
+            self.adjustWindow(
+                channel, channel.localWindowSize - channel.localWindowLeft
+            )
+        channel.dataReceived(data)
+
+    def ssh_CHANNEL_EXTENDED_DATA(self, packet):
+        """
+        The other side is sending us exteneded data.  Payload::
+            uint32  local channel number
+            uint32  type code
+            string  data
+
+        Check to make sure the other side hasn't sent too much data (more
+        than what's in the window, or than the maximum packet size).  If
+        they have, close the channel.  Otherwise, decrease the available
+        window and pass the data and type code to the channel's
+        extReceived().
+        """
+        localChannel, typeCode, dataLength = struct.unpack(">3L", packet[:12])
+        channel = self.channels[localChannel]
+        if dataLength > channel.localWindowLeft or dataLength > channel.localMaxPacket:
+            self._log.error("too much extdata")
+            self.sendClose(channel)
+            return
+        data = common.getNS(packet[8:])[0]
+        channel.localWindowLeft -= dataLength
+        if channel.localWindowLeft < channel.localWindowSize // 2:
+            self.adjustWindow(
+                channel, channel.localWindowSize - channel.localWindowLeft
+            )
+        channel.extReceived(typeCode, data)
+
+    def ssh_CHANNEL_EOF(self, packet):
+        """
+        The other side is not sending any more data.  Payload::
+            uint32  local channel number
+
+        Notify the channel by calling its eofReceived() method.
+        """
+        localChannel = struct.unpack(">L", packet[:4])[0]
+        channel = self.channels[localChannel]
+        channel.eofReceived()
+
+    def ssh_CHANNEL_CLOSE(self, packet):
+        """
+        The other side is closing its end; it does not want to receive any
+        more data.  Payload::
+            uint32  local channel number
+
+        Notify the channnel by calling its closeReceived() method.  If
+        the channel has also sent a close message, call self.channelClosed().
+        """
+        localChannel = struct.unpack(">L", packet[:4])[0]
+        channel = self.channels[localChannel]
+        channel.closeReceived()
+        channel.remoteClosed = True
+        if channel.localClosed and channel.remoteClosed:
+            self.channelClosed(channel)
+
+    def ssh_CHANNEL_REQUEST(self, packet):
+        """
+        The other side is sending a request to a channel.  Payload::
+            uint32  local channel number
+            string  request name
+            bool    want reply
+            <request specific data>
+
+        Pass the message to the channel's requestReceived method.  If the
+        other side wants a reply, add callbacks which will send the
+        reply.
+        """
+        localChannel = struct.unpack(">L", packet[:4])[0]
+        requestType, rest = common.getNS(packet[4:])
+        wantReply = ord(rest[0:1])
+        channel = self.channels[localChannel]
+        d = defer.maybeDeferred(channel.requestReceived, requestType, rest[1:])
+        if wantReply:
+            d.addCallback(self._cbChannelRequest, localChannel)
+            d.addErrback(self._ebChannelRequest, localChannel)
+            return d
+
+    def _cbChannelRequest(self, result, localChannel):
+        """
+        Called back if the other side wanted a reply to a channel request.  If
+        the result is true, send a MSG_CHANNEL_SUCCESS.  Otherwise, raise
+        a C{error.ConchError}
+
+        @param result: the value returned from the channel's requestReceived()
+            method.  If it's False, the request failed.
+        @type result: L{bool}
+        @param localChannel: the local channel ID of the channel to which the
+            request was made.
+        @type localChannel: L{int}
+        @raises ConchError: if the result is False.
+        """
+        if not result:
+            raise error.ConchError("failed request")
+        self.transport.sendPacket(
+            MSG_CHANNEL_SUCCESS,
+            struct.pack(">L", self.localToRemoteChannel[localChannel]),
+        )
+
+    def _ebChannelRequest(self, result, localChannel):
+        """
+        Called if the other wisde wanted a reply to the channel requeset and
+        the channel request failed.
+
+        @param result: a Failure, but it's not used.
+        @param localChannel: the local channel ID of the channel to which the
+            request was made.
+        @type localChannel: L{int}
+        """
+        self.transport.sendPacket(
+            MSG_CHANNEL_FAILURE,
+            struct.pack(">L", self.localToRemoteChannel[localChannel]),
+        )
+
+    def ssh_CHANNEL_SUCCESS(self, packet):
+        """
+        Our channel request to the other side succeeded.  Payload::
+            uint32  local channel number
+
+        Get the C{Deferred} out of self.deferreds and call it back.
+        """
+        localChannel = struct.unpack(">L", packet[:4])[0]
+        if self.deferreds.get(localChannel):
+            d = self.deferreds[localChannel].pop(0)
+            d.callback("")
+
+    def ssh_CHANNEL_FAILURE(self, packet):
+        """
+        Our channel request to the other side failed.  Payload::
+            uint32  local channel number
+
+        Get the C{Deferred} out of self.deferreds and errback it with a
+        C{error.ConchError}.
+        """
+        localChannel = struct.unpack(">L", packet[:4])[0]
+        if self.deferreds.get(localChannel):
+            d = self.deferreds[localChannel].pop(0)
+            d.errback(error.ConchError("channel request failed"))
+
+    # methods for users of the connection to call
+
+    def sendGlobalRequest(self, request, data, wantReply=0):
+        """
+        Send a global request for this connection.  Current this is only used
+        for remote->local TCP forwarding.
+
+        @type request:      L{bytes}
+        @type data:         L{bytes}
+        @type wantReply:    L{bool}
+        @rtype:             C{Deferred}/L{None}
+        """
+        self.transport.sendPacket(
+            MSG_GLOBAL_REQUEST,
+            common.NS(request) + (wantReply and b"\xff" or b"\x00") + data,
+        )
+        if wantReply:
+            d = defer.Deferred()
+            self.deferreds["global"].append(d)
+            return d
+
+    def openChannel(self, channel, extra=b""):
+        """
+        Open a new channel on this connection.
+
+        @type channel:  subclass of C{SSHChannel}
+        @type extra:    L{bytes}
+        """
+        self._log.info(
+            "opening channel {id} with {localWindowSize} {localMaxPacket}",
+            id=self.localChannelID,
+            localWindowSize=channel.localWindowSize,
+            localMaxPacket=channel.localMaxPacket,
+        )
+        self.transport.sendPacket(
+            MSG_CHANNEL_OPEN,
+            common.NS(channel.name)
+            + struct.pack(
+                ">3L",
+                self.localChannelID,
+                channel.localWindowSize,
+                channel.localMaxPacket,
+            )
+            + extra,
+        )
+        channel.id = self.localChannelID
+        self.channels[self.localChannelID] = channel
+        self.localChannelID += 1
+
+    def sendRequest(self, channel, requestType, data, wantReply=0):
+        """
+        Send a request to a channel.
+
+        @type channel:      subclass of C{SSHChannel}
+        @type requestType:  L{bytes}
+        @type data:         L{bytes}
+        @type wantReply:    L{bool}
+        @rtype:             C{Deferred}/L{None}
+        """
+        if channel.localClosed:
+            return
+        self._log.debug("sending request {requestType}", requestType=requestType)
+        self.transport.sendPacket(
+            MSG_CHANNEL_REQUEST,
+            struct.pack(">L", self.channelsToRemoteChannel[channel])
+            + common.NS(requestType)
+            + (b"\1" if wantReply else b"\0")
+            + data,
+        )
+        if wantReply:
+            d = defer.Deferred()
+            self.deferreds.setdefault(channel.id, []).append(d)
+            return d
+
+    def adjustWindow(self, channel, bytesToAdd):
+        """
+        Tell the other side that we will receive more data.  This should not
+        normally need to be called as it is managed automatically.
+
+        @type channel:      subclass of L{SSHChannel}
+        @type bytesToAdd:   L{int}
+        """
+        if channel.localClosed:
+            return  # we're already closed
+        packet = struct.pack(">2L", self.channelsToRemoteChannel[channel], bytesToAdd)
+        self.transport.sendPacket(MSG_CHANNEL_WINDOW_ADJUST, packet)
+        self._log.debug(
+            "adding {bytesToAdd} to {localWindowLeft} in channel {id}",
+            bytesToAdd=bytesToAdd,
+            localWindowLeft=channel.localWindowLeft,
+            id=channel.id,
+        )
+        channel.localWindowLeft += bytesToAdd
+
+    def sendData(self, channel, data):
+        """
+        Send data to a channel.  This should not normally be used: instead use
+        channel.write(data) as it manages the window automatically.
+
+        @type channel:  subclass of L{SSHChannel}
+        @type data:     L{bytes}
+        """
+        if channel.localClosed:
+            return  # we're already closed
+        self.transport.sendPacket(
+            MSG_CHANNEL_DATA,
+            struct.pack(">L", self.channelsToRemoteChannel[channel]) + common.NS(data),
+        )
+
+    def sendExtendedData(self, channel, dataType, data):
+        """
+        Send extended data to a channel.  This should not normally be used:
+        instead use channel.writeExtendedData(data, dataType) as it manages
+        the window automatically.
+
+        @type channel:  subclass of L{SSHChannel}
+        @type dataType: L{int}
+        @type data:     L{bytes}
+        """
+        if channel.localClosed:
+            return  # we're already closed
+        self.transport.sendPacket(
+            MSG_CHANNEL_EXTENDED_DATA,
+            struct.pack(">2L", self.channelsToRemoteChannel[channel], dataType)
+            + common.NS(data),
+        )
+
+    def sendEOF(self, channel):
+        """
+        Send an EOF (End of File) for a channel.
+
+        @type channel:  subclass of L{SSHChannel}
+        """
+        if channel.localClosed:
+            return  # we're already closed
+        self._log.debug("sending eof")
+        self.transport.sendPacket(
+            MSG_CHANNEL_EOF, struct.pack(">L", self.channelsToRemoteChannel[channel])
+        )
+
+    def sendClose(self, channel):
+        """
+        Close a channel.
+
+        @type channel:  subclass of L{SSHChannel}
+        """
+        if channel.localClosed:
+            return  # we're already closed
+        self._log.info("sending close {id}", id=channel.id)
+        self.transport.sendPacket(
+            MSG_CHANNEL_CLOSE, struct.pack(">L", self.channelsToRemoteChannel[channel])
+        )
+        channel.localClosed = True
+        if channel.localClosed and channel.remoteClosed:
+            self.channelClosed(channel)
+
+    # methods to override
+    def getChannel(self, channelType, windowSize, maxPacket, data):
+        """
+        The other side requested a channel of some sort.
+        channelType is the type of channel being requested,
+        windowSize is the initial size of the remote window,
+        maxPacket is the largest packet we should send,
+        data is any other packet data (often nothing).
+
+        We return a subclass of L{SSHChannel}.
+
+        By default, this dispatches to a method 'channel_channelType' with any
+        non-alphanumerics in the channelType replace with _'s.  If it cannot
+        find a suitable method, it returns an OPEN_UNKNOWN_CHANNEL_TYPE error.
+        The method is called with arguments of windowSize, maxPacket, data.
+
+        @type channelType:  L{bytes}
+        @type windowSize:   L{int}
+        @type maxPacket:    L{int}
+        @type data:         L{bytes}
+        @rtype:             subclass of L{SSHChannel}/L{tuple}
+        """
+        self._log.debug("got channel {channelType!r} request", channelType=channelType)
+        if hasattr(self.transport, "avatar"):  # this is a server!
+            chan = self.transport.avatar.lookupChannel(
+                channelType, windowSize, maxPacket, data
+            )
+        else:
+            channelType = channelType.translate(TRANSLATE_TABLE)
+            attr = "channel_%s" % nativeString(channelType)
+            f = getattr(self, attr, None)
+            if f is not None:
+                chan = f(windowSize, maxPacket, data)
+            else:
+                chan = None
+        if chan is None:
+            raise error.ConchError("unknown channel", OPEN_UNKNOWN_CHANNEL_TYPE)
+        else:
+            chan.conn = self
+            return chan
+
+    def gotGlobalRequest(self, requestType, data):
+        """
+        We got a global request.  pretty much, this is just used by the client
+        to request that we forward a port from the server to the client.
+        Returns either:
+            - 1: request accepted
+            - 1, <data>: request accepted with request specific data
+            - 0: request denied
+
+        By default, this dispatches to a method 'global_requestType' with
+        -'s in requestType replaced with _'s.  The found method is passed data.
+        If this method cannot be found, this method returns 0.  Otherwise, it
+        returns the return value of that method.
+
+        @type requestType:  L{bytes}
+        @type data:         L{bytes}
+        @rtype:             L{int}/L{tuple}
+        """
+        self._log.debug("got global {requestType} request", requestType=requestType)
+        if hasattr(self.transport, "avatar"):  # this is a server!
+            return self.transport.avatar.gotGlobalRequest(requestType, data)
+
+        requestType = nativeString(requestType.replace(b"-", b"_"))
+        f = getattr(self, "global_%s" % requestType, None)
+        if not f:
+            return 0
+        return f(data)
+
+    def channelClosed(self, channel):
+        """
+        Called when a channel is closed.
+        It clears the local state related to the channel, and calls
+        channel.closed().
+        MAKE SURE YOU CALL THIS METHOD, even if you subclass L{SSHConnection}.
+        If you don't, things will break mysteriously.
+
+        @type channel: L{SSHChannel}
+        """
+        if channel in self.channelsToRemoteChannel:  # actually open
+            channel.localClosed = channel.remoteClosed = True
+            del self.localToRemoteChannel[channel.id]
+            del self.channels[channel.id]
+            del self.channelsToRemoteChannel[channel]
+            for d in self.deferreds.pop(channel.id, []):
+                d.errback(error.ConchError("Channel closed."))
+            channel.closed()
+
+
+MSG_GLOBAL_REQUEST = 80
+MSG_REQUEST_SUCCESS = 81
+MSG_REQUEST_FAILURE = 82
+MSG_CHANNEL_OPEN = 90
+MSG_CHANNEL_OPEN_CONFIRMATION = 91
+MSG_CHANNEL_OPEN_FAILURE = 92
+MSG_CHANNEL_WINDOW_ADJUST = 93
+MSG_CHANNEL_DATA = 94
+MSG_CHANNEL_EXTENDED_DATA = 95
+MSG_CHANNEL_EOF = 96
+MSG_CHANNEL_CLOSE = 97
+MSG_CHANNEL_REQUEST = 98
+MSG_CHANNEL_SUCCESS = 99
+MSG_CHANNEL_FAILURE = 100
+
+OPEN_ADMINISTRATIVELY_PROHIBITED = 1
+OPEN_CONNECT_FAILED = 2
+OPEN_UNKNOWN_CHANNEL_TYPE = 3
+OPEN_RESOURCE_SHORTAGE = 4
+
+# From RFC 4254
+EXTENDED_DATA_STDERR = 1
+
+messages = {}
+for name, value in locals().copy().items():
+    if name[:4] == "MSG_":
+        messages[value] = name  # Doesn't handle doubles
+
+alphanums = networkString(string.ascii_letters + string.digits)
+TRANSLATE_TABLE = bytes(i if i in alphanums else ord("_") for i in range(256))
+SSHConnection.protocolMessages = messages
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/factory.py b/contrib/python/Twisted/py3/twisted/conch/ssh/factory.py
new file mode 100644
index 00000000000..da1c3a8f9ea
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/factory.py
@@ -0,0 +1,129 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A Factory for SSH servers.
+
+See also L{twisted.conch.openssh_compat.factory} for OpenSSH compatibility.
+
+Maintainer: Paul Swartz
+"""
+
+
+import random
+from itertools import chain
+from typing import Dict, List, Optional, Tuple
+
+from twisted.conch import error
+from twisted.conch.ssh import _kex, connection, transport, userauth
+from twisted.internet import protocol
+from twisted.logger import Logger
+
+
+class SSHFactory(protocol.Factory):
+    """
+    A Factory for SSH servers.
+    """
+
+    primes: Optional[Dict[int, List[Tuple[int, int]]]]
+
+    _log = Logger()
+    protocol = transport.SSHServerTransport
+
+    services = {
+        b"ssh-userauth": userauth.SSHUserAuthServer,
+        b"ssh-connection": connection.SSHConnection,
+    }
+
+    def startFactory(self) -> None:
+        """
+        Check for public and private keys.
+        """
+        if not hasattr(self, "publicKeys"):
+            self.publicKeys = self.getPublicKeys()
+        if not hasattr(self, "privateKeys"):
+            self.privateKeys = self.getPrivateKeys()
+        if not self.publicKeys or not self.privateKeys:
+            raise error.ConchError("no host keys, failing")
+        if not hasattr(self, "primes"):
+            self.primes = self.getPrimes()
+
+    def buildProtocol(self, addr):
+        """
+        Create an instance of the server side of the SSH protocol.
+
+        @type addr: L{twisted.internet.interfaces.IAddress} provider
+        @param addr: The address at which the server will listen.
+
+        @rtype: L{twisted.conch.ssh.transport.SSHServerTransport}
+        @return: The built transport.
+        """
+        t = protocol.Factory.buildProtocol(self, addr)
+        t.supportedPublicKeys = list(
+            chain.from_iterable(
+                key.supportedSignatureAlgorithms() for key in self.privateKeys.values()
+            )
+        )
+        if not self.primes:
+            self._log.info(
+                "disabling non-fixed-group key exchange algorithms "
+                "because we cannot find moduli file"
+            )
+            t.supportedKeyExchanges = [
+                kexAlgorithm
+                for kexAlgorithm in t.supportedKeyExchanges
+                if _kex.isFixedGroup(kexAlgorithm) or _kex.isEllipticCurve(kexAlgorithm)
+            ]
+        return t
+
+    def getPublicKeys(self):
+        """
+        Called when the factory is started to get the public portions of the
+        servers host keys.  Returns a dictionary mapping SSH key types to
+        public key strings.
+
+        @rtype: L{dict}
+        """
+        raise NotImplementedError("getPublicKeys unimplemented")
+
+    def getPrivateKeys(self):
+        """
+        Called when the factory is started to get the  private portions of the
+        servers host keys.  Returns a dictionary mapping SSH key types to
+        L{twisted.conch.ssh.keys.Key} objects.
+
+        @rtype: L{dict}
+        """
+        raise NotImplementedError("getPrivateKeys unimplemented")
+
+    def getPrimes(self) -> Optional[Dict[int, List[Tuple[int, int]]]]:
+        """
+        Called when the factory is started to get Diffie-Hellman generators and
+        primes to use.  Returns a dictionary mapping number of bits to lists of
+        tuple of (generator, prime).
+        """
+
+    def getDHPrime(self, bits: int) -> Tuple[int, int]:
+        """
+        Return a tuple of (g, p) for a Diffe-Hellman process, with p being as
+        close to C{bits} bits as possible.
+        """
+
+        def keyfunc(i: int) -> int:
+            return abs(i - bits)
+
+        assert self.primes is not None, "Factory should have been started by now."
+        primesKeys = sorted(self.primes.keys(), key=keyfunc)
+        realBits = primesKeys[0]
+        return random.choice(self.primes[realBits])
+
+    def getService(self, transport, service):
+        """
+        Return a class to use as a service for the given transport.
+
+        @type transport:    L{transport.SSHServerTransport}
+        @type service:      L{bytes}
+        @rtype:             subclass of L{service.SSHService}
+        """
+        if service == b"ssh-userauth" or hasattr(transport, "avatar"):
+            return self.services[service]
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/filetransfer.py b/contrib/python/Twisted/py3/twisted/conch/ssh/filetransfer.py
new file mode 100644
index 00000000000..830ff711ddd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/filetransfer.py
@@ -0,0 +1,1069 @@
+# -*- test-case-name: twisted.conch.test.test_filetransfer -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+import errno
+import os
+import struct
+import warnings
+from typing import Dict
+
+from zope.interface import implementer
+
+from twisted.conch.interfaces import ISFTPFile, ISFTPServer
+from twisted.conch.ssh.common import NS, getNS
+from twisted.internet import defer, error, protocol
+from twisted.logger import Logger
+from twisted.python import failure
+from twisted.python.compat import nativeString, networkString
+
+
+class FileTransferBase(protocol.Protocol):
+    _log = Logger()
+
+    versions = (3,)
+
+    packetTypes: Dict[int, str] = {}
+
+    def __init__(self):
+        self.buf = b""
+        self.otherVersion = None  # This gets set
+
+    def sendPacket(self, kind, data):
+        self.transport.write(struct.pack("!LB", len(data) + 1, kind) + data)
+
+    def dataReceived(self, data):
+        self.buf += data
+
+        # Continue processing the input buffer as long as there is a chance it
+        # could contain a complete request.  The "General Packet Format"
+        # (format all requests follow) is a 4 byte length prefix, a 1 byte
+        # type field, and a 4 byte request id.  If we have fewer than 4 + 1 +
+        # 4 == 9 bytes we cannot possibly have a complete request.
+        while len(self.buf) >= 9:
+            header = self.buf[:9]
+            length, kind, reqId = struct.unpack("!LBL", header)
+            # From draft-ietf-secsh-filexfer-13 (the draft we implement):
+            #
+            #   The `length' is the length of the data area [including the
+            #   kind byte], and does not include the `length' field itself.
+            #
+            # If the input buffer doesn't have enough bytes to satisfy the
+            # full length then we cannot process it now.  Wait until we have
+            # more bytes.
+            if len(self.buf) < 4 + length:
+                return
+
+            # We parsed the request id out of the input buffer above but the
+            # interface to the `packet_TYPE` methods involves passing them a
+            # data buffer which still includes the request id ... So leave
+            # those bytes in the `data` we slice off here.
+            data, self.buf = self.buf[5 : 4 + length], self.buf[4 + length :]
+
+            packetType = self.packetTypes.get(kind, None)
+            if not packetType:
+                self._log.info("no packet type for {kind}", kind=kind)
+                continue
+
+            f = getattr(self, f"packet_{packetType}", None)
+            if not f:
+                self._log.info(
+                    "not implemented: {packetType} data={data!r}",
+                    packetType=packetType,
+                    data=data[4:],
+                )
+                self._sendStatus(
+                    reqId, FX_OP_UNSUPPORTED, f"don't understand {packetType}"
+                )
+                # XXX not implemented
+                continue
+            self._log.info(
+                "dispatching: {packetType} requestId={reqId}",
+                packetType=packetType,
+                reqId=reqId,
+            )
+            try:
+                f(data)
+            except Exception:
+                self._log.failure(
+                    "Failed to handle packet of type {packetType}",
+                    packetType=packetType,
+                )
+                continue
+
+    def _parseAttributes(self, data):
+        (flags,) = struct.unpack("!L", data[:4])
+        attrs = {}
+        data = data[4:]
+        if flags & FILEXFER_ATTR_SIZE == FILEXFER_ATTR_SIZE:
+            (size,) = struct.unpack("!Q", data[:8])
+            attrs["size"] = size
+            data = data[8:]
+        if flags & FILEXFER_ATTR_OWNERGROUP == FILEXFER_ATTR_OWNERGROUP:
+            uid, gid = struct.unpack("!2L", data[:8])
+            attrs["uid"] = uid
+            attrs["gid"] = gid
+            data = data[8:]
+        if flags & FILEXFER_ATTR_PERMISSIONS == FILEXFER_ATTR_PERMISSIONS:
+            (perms,) = struct.unpack("!L", data[:4])
+            attrs["permissions"] = perms
+            data = data[4:]
+        if flags & FILEXFER_ATTR_ACMODTIME == FILEXFER_ATTR_ACMODTIME:
+            atime, mtime = struct.unpack("!2L", data[:8])
+            attrs["atime"] = atime
+            attrs["mtime"] = mtime
+            data = data[8:]
+        if flags & FILEXFER_ATTR_EXTENDED == FILEXFER_ATTR_EXTENDED:
+            (extendedCount,) = struct.unpack("!L", data[:4])
+            data = data[4:]
+            for i in range(extendedCount):
+                (extendedType, data) = getNS(data)
+                (extendedData, data) = getNS(data)
+                attrs[f"ext_{nativeString(extendedType)}"] = extendedData
+        return attrs, data
+
+    def _packAttributes(self, attrs):
+        flags = 0
+        data = b""
+        if "size" in attrs:
+            data += struct.pack("!Q", attrs["size"])
+            flags |= FILEXFER_ATTR_SIZE
+        if "uid" in attrs and "gid" in attrs:
+            data += struct.pack("!2L", attrs["uid"], attrs["gid"])
+            flags |= FILEXFER_ATTR_OWNERGROUP
+        if "permissions" in attrs:
+            data += struct.pack("!L", attrs["permissions"])
+            flags |= FILEXFER_ATTR_PERMISSIONS
+        if "atime" in attrs and "mtime" in attrs:
+            data += struct.pack("!2L", attrs["atime"], attrs["mtime"])
+            flags |= FILEXFER_ATTR_ACMODTIME
+        extended = []
+        for k in attrs:
+            if k.startswith("ext_"):
+                extType = NS(networkString(k[4:]))
+                extData = NS(attrs[k])
+                extended.append(extType + extData)
+        if extended:
+            data += struct.pack("!L", len(extended))
+            data += b"".join(extended)
+            flags |= FILEXFER_ATTR_EXTENDED
+        return struct.pack("!L", flags) + data
+
+    def connectionLost(self, reason):
+        """
+        Called when connection to the remote subsystem was lost.
+        """
+
+        super().connectionLost(reason)
+        self.connected = False
+
+
+class FileTransferServer(FileTransferBase):
+    def __init__(self, data=None, avatar=None):
+        FileTransferBase.__init__(self)
+        self.client = ISFTPServer(avatar)  # yay interfaces
+        self.openFiles = {}
+        self.openDirs = {}
+
+    def packet_INIT(self, data):
+        (version,) = struct.unpack("!L", data[:4])
+        self.version = min(list(self.versions) + [version])
+        data = data[4:]
+        ext = {}
+        while data:
+            extName, data = getNS(data)
+            extData, data = getNS(data)
+            ext[extName] = extData
+        ourExt = self.client.gotVersion(version, ext)
+        ourExtData = b""
+        for k, v in ourExt.items():
+            ourExtData += NS(k) + NS(v)
+        self.sendPacket(FXP_VERSION, struct.pack("!L", self.version) + ourExtData)
+
+    def packet_OPEN(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        filename, data = getNS(data)
+        (flags,) = struct.unpack("!L", data[:4])
+        data = data[4:]
+        attrs, data = self._parseAttributes(data)
+        assert data == b"", f"still have data in OPEN: {data!r}"
+        d = defer.maybeDeferred(self.client.openFile, filename, flags, attrs)
+        d.addCallback(self._cbOpenFile, requestId)
+        d.addErrback(self._ebStatus, requestId, b"open failed")
+
+    def _cbOpenFile(self, fileObj, requestId):
+        fileId = networkString(str(hash(fileObj)))
+        if fileId in self.openFiles:
+            raise KeyError("id already open")
+        self.openFiles[fileId] = fileObj
+        self.sendPacket(FXP_HANDLE, requestId + NS(fileId))
+
+    def packet_CLOSE(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        handle, data = getNS(data)
+        self._log.info(
+            "closing: {requestId!r} {handle!r}",
+            requestId=requestId,
+            handle=handle,
+        )
+        assert data == b"", f"still have data in CLOSE: {data!r}"
+        if handle in self.openFiles:
+            fileObj = self.openFiles[handle]
+            d = defer.maybeDeferred(fileObj.close)
+            d.addCallback(self._cbClose, handle, requestId)
+            d.addErrback(self._ebStatus, requestId, b"close failed")
+        elif handle in self.openDirs:
+            dirObj = self.openDirs[handle][0]
+            d = defer.maybeDeferred(dirObj.close)
+            d.addCallback(self._cbClose, handle, requestId, 1)
+            d.addErrback(self._ebStatus, requestId, b"close failed")
+        else:
+            code = errno.ENOENT
+            text = os.strerror(code)
+            err = OSError(code, text)
+            self._ebStatus(failure.Failure(err), requestId)
+
+    def _cbClose(self, result, handle, requestId, isDir=0):
+        if isDir:
+            del self.openDirs[handle]
+        else:
+            del self.openFiles[handle]
+        self._sendStatus(requestId, FX_OK, b"file closed")
+
+    def packet_READ(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        handle, data = getNS(data)
+        (offset, length), data = struct.unpack("!QL", data[:12]), data[12:]
+        assert data == b"", f"still have data in READ: {data!r}"
+        if handle not in self.openFiles:
+            self._ebRead(failure.Failure(KeyError()), requestId)
+        else:
+            fileObj = self.openFiles[handle]
+            d = defer.maybeDeferred(fileObj.readChunk, offset, length)
+            d.addCallback(self._cbRead, requestId)
+            d.addErrback(self._ebStatus, requestId, b"read failed")
+
+    def _cbRead(self, result, requestId):
+        if result == b"":  # Python's read will return this for EOF
+            raise EOFError()
+        self.sendPacket(FXP_DATA, requestId + NS(result))
+
+    def packet_WRITE(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        handle, data = getNS(data)
+        (offset,) = struct.unpack("!Q", data[:8])
+        data = data[8:]
+        writeData, data = getNS(data)
+        assert data == b"", f"still have data in WRITE: {data!r}"
+        if handle not in self.openFiles:
+            self._ebWrite(failure.Failure(KeyError()), requestId)
+        else:
+            fileObj = self.openFiles[handle]
+            d = defer.maybeDeferred(fileObj.writeChunk, offset, writeData)
+            d.addCallback(self._cbStatus, requestId, b"write succeeded")
+            d.addErrback(self._ebStatus, requestId, b"write failed")
+
+    def packet_REMOVE(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        filename, data = getNS(data)
+        assert data == b"", f"still have data in REMOVE: {data!r}"
+        d = defer.maybeDeferred(self.client.removeFile, filename)
+        d.addCallback(self._cbStatus, requestId, b"remove succeeded")
+        d.addErrback(self._ebStatus, requestId, b"remove failed")
+
+    def packet_RENAME(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        oldPath, data = getNS(data)
+        newPath, data = getNS(data)
+        assert data == b"", f"still have data in RENAME: {data!r}"
+        d = defer.maybeDeferred(self.client.renameFile, oldPath, newPath)
+        d.addCallback(self._cbStatus, requestId, b"rename succeeded")
+        d.addErrback(self._ebStatus, requestId, b"rename failed")
+
+    def packet_MKDIR(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        path, data = getNS(data)
+        attrs, data = self._parseAttributes(data)
+        assert data == b"", f"still have data in MKDIR: {data!r}"
+        d = defer.maybeDeferred(self.client.makeDirectory, path, attrs)
+        d.addCallback(self._cbStatus, requestId, b"mkdir succeeded")
+        d.addErrback(self._ebStatus, requestId, b"mkdir failed")
+
+    def packet_RMDIR(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        path, data = getNS(data)
+        assert data == b"", f"still have data in RMDIR: {data!r}"
+        d = defer.maybeDeferred(self.client.removeDirectory, path)
+        d.addCallback(self._cbStatus, requestId, b"rmdir succeeded")
+        d.addErrback(self._ebStatus, requestId, b"rmdir failed")
+
+    def packet_OPENDIR(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        path, data = getNS(data)
+        assert data == b"", f"still have data in OPENDIR: {data!r}"
+        d = defer.maybeDeferred(self.client.openDirectory, path)
+        d.addCallback(self._cbOpenDirectory, requestId)
+        d.addErrback(self._ebStatus, requestId, b"opendir failed")
+
+    def _cbOpenDirectory(self, dirObj, requestId):
+        handle = networkString(str(hash(dirObj)))
+        if handle in self.openDirs:
+            raise KeyError("already opened this directory")
+        self.openDirs[handle] = [dirObj, iter(dirObj)]
+        self.sendPacket(FXP_HANDLE, requestId + NS(handle))
+
+    def packet_READDIR(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        handle, data = getNS(data)
+        assert data == b"", f"still have data in READDIR: {data!r}"
+        if handle not in self.openDirs:
+            self._ebStatus(failure.Failure(KeyError()), requestId)
+        else:
+            dirObj, dirIter = self.openDirs[handle]
+            d = defer.maybeDeferred(self._scanDirectory, dirIter, [])
+            d.addCallback(self._cbSendDirectory, requestId)
+            d.addErrback(self._ebStatus, requestId, b"scan directory failed")
+
+    def _scanDirectory(self, dirIter, f):
+        while len(f) < 250:
+            try:
+                info = next(dirIter)
+            except StopIteration:
+                if not f:
+                    raise EOFError
+                return f
+            if isinstance(info, defer.Deferred):
+                info.addCallback(self._cbScanDirectory, dirIter, f)
+                return
+            else:
+                f.append(info)
+        return f
+
+    def _cbScanDirectory(self, result, dirIter, f):
+        f.append(result)
+        return self._scanDirectory(dirIter, f)
+
+    def _cbSendDirectory(self, result, requestId):
+        data = b""
+        for filename, longname, attrs in result:
+            data += NS(filename)
+            data += NS(longname)
+            data += self._packAttributes(attrs)
+        self.sendPacket(FXP_NAME, requestId + struct.pack("!L", len(result)) + data)
+
+    def packet_STAT(self, data, followLinks=1):
+        requestId = data[:4]
+        data = data[4:]
+        path, data = getNS(data)
+        assert data == b"", f"still have data in STAT/LSTAT: {data!r}"
+        d = defer.maybeDeferred(self.client.getAttrs, path, followLinks)
+        d.addCallback(self._cbStat, requestId)
+        d.addErrback(self._ebStatus, requestId, b"stat/lstat failed")
+
+    def packet_LSTAT(self, data):
+        self.packet_STAT(data, 0)
+
+    def packet_FSTAT(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        handle, data = getNS(data)
+        assert data == b"", f"still have data in FSTAT: {data!r}"
+        if handle not in self.openFiles:
+            self._ebStatus(
+                failure.Failure(KeyError(f"{handle} not in self.openFiles")),
+                requestId,
+            )
+        else:
+            fileObj = self.openFiles[handle]
+            d = defer.maybeDeferred(fileObj.getAttrs)
+            d.addCallback(self._cbStat, requestId)
+            d.addErrback(self._ebStatus, requestId, b"fstat failed")
+
+    def _cbStat(self, result, requestId):
+        data = requestId + self._packAttributes(result)
+        self.sendPacket(FXP_ATTRS, data)
+
+    def packet_SETSTAT(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        path, data = getNS(data)
+        attrs, data = self._parseAttributes(data)
+        if data != b"":
+            self._log.warn("Still have data in SETSTAT: {data!r}", data=data)
+        d = defer.maybeDeferred(self.client.setAttrs, path, attrs)
+        d.addCallback(self._cbStatus, requestId, b"setstat succeeded")
+        d.addErrback(self._ebStatus, requestId, b"setstat failed")
+
+    def packet_FSETSTAT(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        handle, data = getNS(data)
+        attrs, data = self._parseAttributes(data)
+        assert data == b"", f"still have data in FSETSTAT: {data!r}"
+        if handle not in self.openFiles:
+            self._ebStatus(failure.Failure(KeyError()), requestId)
+        else:
+            fileObj = self.openFiles[handle]
+            d = defer.maybeDeferred(fileObj.setAttrs, attrs)
+            d.addCallback(self._cbStatus, requestId, b"fsetstat succeeded")
+            d.addErrback(self._ebStatus, requestId, b"fsetstat failed")
+
+    def packet_READLINK(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        path, data = getNS(data)
+        assert data == b"", f"still have data in READLINK: {data!r}"
+        d = defer.maybeDeferred(self.client.readLink, path)
+        d.addCallback(self._cbReadLink, requestId)
+        d.addErrback(self._ebStatus, requestId, b"readlink failed")
+
+    def _cbReadLink(self, result, requestId):
+        self._cbSendDirectory([(result, b"", {})], requestId)
+
+    def packet_SYMLINK(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        linkPath, data = getNS(data)
+        targetPath, data = getNS(data)
+        d = defer.maybeDeferred(self.client.makeLink, linkPath, targetPath)
+        d.addCallback(self._cbStatus, requestId, b"symlink succeeded")
+        d.addErrback(self._ebStatus, requestId, b"symlink failed")
+
+    def packet_REALPATH(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        path, data = getNS(data)
+        assert data == b"", f"still have data in REALPATH: {data!r}"
+        d = defer.maybeDeferred(self.client.realPath, path)
+        d.addCallback(self._cbReadLink, requestId)  # Same return format
+        d.addErrback(self._ebStatus, requestId, b"realpath failed")
+
+    def packet_EXTENDED(self, data):
+        requestId = data[:4]
+        data = data[4:]
+        extName, extData = getNS(data)
+        d = defer.maybeDeferred(self.client.extendedRequest, extName, extData)
+        d.addCallback(self._cbExtended, requestId)
+        d.addErrback(self._ebStatus, requestId, b"extended " + extName + b" failed")
+
+    def _cbExtended(self, data, requestId):
+        self.sendPacket(FXP_EXTENDED_REPLY, requestId + data)
+
+    def _cbStatus(self, result, requestId, msg=b"request succeeded"):
+        self._sendStatus(requestId, FX_OK, msg)
+
+    def _ebStatus(self, reason, requestId, msg=b"request failed"):
+        code = FX_FAILURE
+        message = msg
+        if isinstance(reason.value, (IOError, OSError)):
+            if reason.value.errno == errno.ENOENT:  # No such file
+                code = FX_NO_SUCH_FILE
+                message = networkString(reason.value.strerror)
+            elif reason.value.errno == errno.EACCES:  # Permission denied
+                code = FX_PERMISSION_DENIED
+                message = networkString(reason.value.strerror)
+            elif reason.value.errno == errno.EEXIST:
+                code = FX_FILE_ALREADY_EXISTS
+            else:
+                self._log.failure(
+                    "Request {requestId} failed: {message}",
+                    failure=reason,
+                    requestId=requestId,
+                    message=message,
+                )
+        elif isinstance(reason.value, EOFError):  # EOF
+            code = FX_EOF
+            if reason.value.args:
+                message = networkString(reason.value.args[0])
+        elif isinstance(reason.value, NotImplementedError):
+            code = FX_OP_UNSUPPORTED
+            if reason.value.args:
+                message = networkString(reason.value.args[0])
+        elif isinstance(reason.value, SFTPError):
+            code = reason.value.code
+            message = networkString(reason.value.message)
+        else:
+            self._log.failure(
+                "Request {requestId} failed with unknown error: {message}",
+                failure=reason,
+                requestId=requestId,
+                message=message,
+            )
+        self._sendStatus(requestId, code, message)
+
+    def _sendStatus(self, requestId, code, message, lang=b""):
+        """
+        Helper method to send a FXP_STATUS message.
+        """
+        data = requestId + struct.pack("!L", code)
+        data += NS(message)
+        data += NS(lang)
+        self.sendPacket(FXP_STATUS, data)
+
+    def connectionLost(self, reason):
+        """
+        Called when connection to the remote subsystem was lost.
+
+        Clean all opened files and directories.
+        """
+
+        FileTransferBase.connectionLost(self, reason)
+
+        for fileObj in self.openFiles.values():
+            fileObj.close()
+        self.openFiles = {}
+        for dirObj, dirIter in self.openDirs.values():
+            dirObj.close()
+        self.openDirs = {}
+
+
+class FileTransferClient(FileTransferBase):
+    def __init__(self, extData={}):
+        """
+        @param extData: a dict of extended_name : extended_data items
+        to be sent to the server.
+        """
+        FileTransferBase.__init__(self)
+        self.extData = {}
+        self.counter = 0
+        self.openRequests = {}  # id -> Deferred
+
+    def connectionMade(self):
+        data = struct.pack("!L", max(self.versions))
+        for k, v in self.extData.values():
+            data += NS(k) + NS(v)
+        self.sendPacket(FXP_INIT, data)
+
+    def connectionLost(self, reason):
+        """
+        Called when connection to the remote subsystem was lost.
+
+        Any pending requests are aborted.
+        """
+
+        FileTransferBase.connectionLost(self, reason)
+
+        # If there are still requests waiting for responses when the
+        # connection is lost, fail them.
+        if self.openRequests:
+            # Even if our transport was lost "cleanly", our
+            # requests were still not cancelled "cleanly".
+            requestError = error.ConnectionLost()
+            requestError.__cause__ = reason.value
+            requestFailure = failure.Failure(requestError)
+            while self.openRequests:
+                _, deferred = self.openRequests.popitem()
+                deferred.errback(requestFailure)
+
+    def _sendRequest(self, msg, data):
+        """
+        Send a request and return a deferred which waits for the result.
+
+        @type msg: L{int}
+        @param msg: The request type (e.g., C{FXP_READ}).
+
+        @type data: L{bytes}
+        @param data: The body of the request.
+        """
+        if not self.connected:
+            return defer.fail(error.ConnectionLost())
+
+        data = struct.pack("!L", self.counter) + data
+        d = defer.Deferred()
+        self.openRequests[self.counter] = d
+        self.counter += 1
+        self.sendPacket(msg, data)
+        return d
+
+    def _parseRequest(self, data):
+        (id,) = struct.unpack("!L", data[:4])
+        d = self.openRequests[id]
+        del self.openRequests[id]
+        return d, data[4:]
+
+    def openFile(self, filename, flags, attrs):
+        """
+        Open a file.
+
+        This method returns a L{Deferred} that is called back with an object
+        that provides the L{ISFTPFile} interface.
+
+        @type filename: L{bytes}
+        @param filename: a string representing the file to open.
+
+        @param flags: an integer of the flags to open the file with, ORed together.
+        The flags and their values are listed at the bottom of this file.
+
+        @param attrs: a list of attributes to open the file with.  It is a
+        dictionary, consisting of 0 or more keys.  The possible keys are::
+
+            size: the size of the file in bytes
+            uid: the user ID of the file as an integer
+            gid: the group ID of the file as an integer
+            permissions: the permissions of the file with as an integer.
+            the bit representation of this field is defined by POSIX.
+            atime: the access time of the file as seconds since the epoch.
+            mtime: the modification time of the file as seconds since the epoch.
+            ext_*: extended attributes.  The server is not required to
+            understand this, but it may.
+
+        NOTE: there is no way to indicate text or binary files.  it is up
+        to the SFTP client to deal with this.
+        """
+        data = NS(filename) + struct.pack("!L", flags) + self._packAttributes(attrs)
+        d = self._sendRequest(FXP_OPEN, data)
+        d.addCallback(self._cbOpenHandle, ClientFile, filename)
+        return d
+
+    def _cbOpenHandle(self, handle, handleClass, name):
+        """
+        Callback invoked when an OPEN or OPENDIR request succeeds.
+
+        @param handle: The handle returned by the server
+        @type handle: L{bytes}
+        @param handleClass: The class that will represent the
+        newly-opened file or directory to the user (either L{ClientFile} or
+        L{ClientDirectory}).
+        @param name: The name of the file or directory represented
+        by C{handle}.
+        @type name: L{bytes}
+        """
+        cb = handleClass(self, handle)
+        cb.name = name
+        return cb
+
+    def removeFile(self, filename):
+        """
+        Remove the given file.
+
+        This method returns a Deferred that is called back when it succeeds.
+
+        @type filename: L{bytes}
+        @param filename: the name of the file as a string.
+        """
+        return self._sendRequest(FXP_REMOVE, NS(filename))
+
+    def renameFile(self, oldpath, newpath):
+        """
+        Rename the given file.
+
+        This method returns a Deferred that is called back when it succeeds.
+
+        @type oldpath: L{bytes}
+        @param oldpath: the current location of the file.
+        @type newpath: L{bytes}
+        @param newpath: the new file name.
+        """
+        return self._sendRequest(FXP_RENAME, NS(oldpath) + NS(newpath))
+
+    def makeDirectory(self, path, attrs):
+        """
+        Make a directory.
+
+        This method returns a Deferred that is called back when it is
+        created.
+
+        @type path: L{bytes}
+        @param path: the name of the directory to create as a string.
+
+        @param attrs: a dictionary of attributes to create the directory
+        with.  Its meaning is the same as the attrs in the openFile method.
+        """
+        return self._sendRequest(FXP_MKDIR, NS(path) + self._packAttributes(attrs))
+
+    def removeDirectory(self, path):
+        """
+        Remove a directory (non-recursively)
+
+        It is an error to remove a directory that has files or directories in
+        it.
+
+        This method returns a Deferred that is called back when it is removed.
+
+        @type path: L{bytes}
+        @param path: the directory to remove.
+        """
+        return self._sendRequest(FXP_RMDIR, NS(path))
+
+    def openDirectory(self, path):
+        """
+        Open a directory for scanning.
+
+        This method returns a Deferred that is called back with an iterable
+        object that has a close() method.
+
+        The close() method is called when the client is finished reading
+        from the directory.  At this point, the iterable will no longer
+        be used.
+
+        The iterable returns triples of the form (filename, longname, attrs)
+        or a Deferred that returns the same.  The sequence must support
+        __getitem__, but otherwise may be any 'sequence-like' object.
+
+        filename is the name of the file relative to the directory.
+        logname is an expanded format of the filename.  The recommended format
+        is:
+        -rwxr-xr-x   1 mjos     staff      348911 Mar 25 14:29 t-filexfer
+        1234567890 123 12345678 12345678 12345678 123456789012
+
+        The first line is sample output, the second is the length of the field.
+        The fields are: permissions, link count, user owner, group owner,
+        size in bytes, modification time.
+
+        attrs is a dictionary in the format of the attrs argument to openFile.
+
+        @type path: L{bytes}
+        @param path: the directory to open.
+        """
+        d = self._sendRequest(FXP_OPENDIR, NS(path))
+        d.addCallback(self._cbOpenHandle, ClientDirectory, path)
+        return d
+
+    def getAttrs(self, path, followLinks=0):
+        """
+        Return the attributes for the given path.
+
+        This method returns a dictionary in the same format as the attrs
+        argument to openFile or a Deferred that is called back with same.
+
+        @type path: L{bytes}
+        @param path: the path to return attributes for as a string.
+        @param followLinks: a boolean.  if it is True, follow symbolic links
+        and return attributes for the real path at the base.  if it is False,
+        return attributes for the specified path.
+        """
+        if followLinks:
+            m = FXP_STAT
+        else:
+            m = FXP_LSTAT
+        return self._sendRequest(m, NS(path))
+
+    def setAttrs(self, path, attrs):
+        """
+        Set the attributes for the path.
+
+        This method returns when the attributes are set or a Deferred that is
+        called back when they are.
+
+        @type path: L{bytes}
+        @param path: the path to set attributes for as a string.
+        @param attrs: a dictionary in the same format as the attrs argument to
+        openFile.
+        """
+        data = NS(path) + self._packAttributes(attrs)
+        return self._sendRequest(FXP_SETSTAT, data)
+
+    def readLink(self, path):
+        """
+        Find the root of a set of symbolic links.
+
+        This method returns the target of the link, or a Deferred that
+        returns the same.
+
+        @type path: L{bytes}
+        @param path: the path of the symlink to read.
+        """
+        d = self._sendRequest(FXP_READLINK, NS(path))
+        return d.addCallback(self._cbRealPath)
+
+    def makeLink(self, linkPath, targetPath):
+        """
+        Create a symbolic link.
+
+        This method returns when the link is made, or a Deferred that
+        returns the same.
+
+        @type linkPath: L{bytes}
+        @param linkPath: the pathname of the symlink as a string
+        @type targetPath: L{bytes}
+        @param targetPath: the path of the target of the link as a string.
+        """
+        return self._sendRequest(FXP_SYMLINK, NS(linkPath) + NS(targetPath))
+
+    def realPath(self, path):
+        """
+        Convert any path to an absolute path.
+
+        This method returns the absolute path as a string, or a Deferred
+        that returns the same.
+
+        @type path: L{bytes}
+        @param path: the path to convert as a string.
+        """
+        d = self._sendRequest(FXP_REALPATH, NS(path))
+        return d.addCallback(self._cbRealPath)
+
+    def _cbRealPath(self, result):
+        name, longname, attrs = result[0]
+        name = name.decode("utf-8")
+        return name
+
+    def extendedRequest(self, request, data):
+        """
+        Make an extended request of the server.
+
+        The method returns a Deferred that is called back with
+        the result of the extended request.
+
+        @type request: L{bytes}
+        @param request: the name of the extended request to make.
+        @type data: L{bytes}
+        @param data: any other data that goes along with the request.
+        """
+        return self._sendRequest(FXP_EXTENDED, NS(request) + data)
+
+    def packet_VERSION(self, data):
+        (version,) = struct.unpack("!L", data[:4])
+        data = data[4:]
+        d = {}
+        while data:
+            k, data = getNS(data)
+            v, data = getNS(data)
+            d[k] = v
+        self.version = version
+        self.gotServerVersion(version, d)
+
+    def packet_STATUS(self, data):
+        d, data = self._parseRequest(data)
+        (code,) = struct.unpack("!L", data[:4])
+        data = data[4:]
+        if len(data) >= 4:
+            msg, data = getNS(data)
+            if len(data) >= 4:
+                lang, data = getNS(data)
+            else:
+                lang = b""
+        else:
+            msg = b""
+            lang = b""
+        if code == FX_OK:
+            d.callback((msg, lang))
+        elif code == FX_EOF:
+            d.errback(EOFError(msg))
+        elif code == FX_OP_UNSUPPORTED:
+            d.errback(NotImplementedError(msg))
+        else:
+            d.errback(SFTPError(code, nativeString(msg), lang))
+
+    def packet_HANDLE(self, data):
+        d, data = self._parseRequest(data)
+        handle, _ = getNS(data)
+        d.callback(handle)
+
+    def packet_DATA(self, data):
+        d, data = self._parseRequest(data)
+        d.callback(getNS(data)[0])
+
+    def packet_NAME(self, data):
+        d, data = self._parseRequest(data)
+        (count,) = struct.unpack("!L", data[:4])
+        data = data[4:]
+        files = []
+        for i in range(count):
+            filename, data = getNS(data)
+            longname, data = getNS(data)
+            attrs, data = self._parseAttributes(data)
+            files.append((filename, longname, attrs))
+        d.callback(files)
+
+    def packet_ATTRS(self, data):
+        d, data = self._parseRequest(data)
+        d.callback(self._parseAttributes(data)[0])
+
+    def packet_EXTENDED_REPLY(self, data):
+        d, data = self._parseRequest(data)
+        d.callback(data)
+
+    def gotServerVersion(self, serverVersion, extData):
+        """
+        Called when the client sends their version info.
+
+        @param serverVersion: an integer representing the version of the SFTP
+        protocol they are claiming.
+        @param extData: a dictionary of extended_name : extended_data items.
+        These items are sent by the client to indicate additional features.
+        """
+
+
+@implementer(ISFTPFile)
+class ClientFile:
+    def __init__(self, parent, handle):
+        self.parent = parent
+        self.handle = NS(handle)
+
+    def close(self):
+        return self.parent._sendRequest(FXP_CLOSE, self.handle)
+
+    def readChunk(self, offset, length):
+        data = self.handle + struct.pack("!QL", offset, length)
+        return self.parent._sendRequest(FXP_READ, data)
+
+    def writeChunk(self, offset, chunk):
+        data = self.handle + struct.pack("!Q", offset) + NS(chunk)
+        return self.parent._sendRequest(FXP_WRITE, data)
+
+    def getAttrs(self):
+        return self.parent._sendRequest(FXP_FSTAT, self.handle)
+
+    def setAttrs(self, attrs):
+        data = self.handle + self.parent._packAttributes(attrs)
+        return self.parent._sendRequest(FXP_FSTAT, data)
+
+
+class ClientDirectory:
+    def __init__(self, parent, handle):
+        self.parent = parent
+        self.handle = NS(handle)
+        self.filesCache = []
+
+    def read(self):
+        return self.parent._sendRequest(FXP_READDIR, self.handle)
+
+    def close(self):
+        if self.handle is None:
+            return defer.succeed(None)
+        d = self.parent._sendRequest(FXP_CLOSE, self.handle)
+        self.handle = None
+        return d
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        warnings.warn(
+            (
+                "Using twisted.conch.ssh.filetransfer.ClientDirectory "
+                "as an iterator was deprecated in Twisted 18.9.0."
+            ),
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        if self.filesCache:
+            return self.filesCache.pop(0)
+        if self.filesCache is None:
+            raise StopIteration()
+        d = self.read()
+        d.addCallbacks(self._cbReadDir, self._ebReadDir)
+        return d
+
+    next = __next__
+
+    def _cbReadDir(self, names):
+        self.filesCache = names[1:]
+        return names[0]
+
+    def _ebReadDir(self, reason):
+        reason.trap(EOFError)
+        self.filesCache = None
+        return failure.Failure(StopIteration())
+
+
+class SFTPError(Exception):
+    def __init__(self, errorCode, errorMessage, lang=""):
+        Exception.__init__(self)
+        self.code = errorCode
+        self._message = errorMessage
+        self.lang = lang
+
+    @property
+    def message(self):
+        """
+        A string received over the network that explains the error to a human.
+        """
+        # Python 2.6 deprecates assigning to the 'message' attribute of an
+        # exception. We define this read-only property here in order to
+        # prevent the warning about deprecation while maintaining backwards
+        # compatibility with object clients that rely on the 'message'
+        # attribute being set correctly. See bug #3897.
+        return self._message
+
+    def __str__(self) -> str:
+        return f"SFTPError {self.code}: {self.message}"
+
+
+FXP_INIT = 1
+FXP_VERSION = 2
+FXP_OPEN = 3
+FXP_CLOSE = 4
+FXP_READ = 5
+FXP_WRITE = 6
+FXP_LSTAT = 7
+FXP_FSTAT = 8
+FXP_SETSTAT = 9
+FXP_FSETSTAT = 10
+FXP_OPENDIR = 11
+FXP_READDIR = 12
+FXP_REMOVE = 13
+FXP_MKDIR = 14
+FXP_RMDIR = 15
+FXP_REALPATH = 16
+FXP_STAT = 17
+FXP_RENAME = 18
+FXP_READLINK = 19
+FXP_SYMLINK = 20
+FXP_STATUS = 101
+FXP_HANDLE = 102
+FXP_DATA = 103
+FXP_NAME = 104
+FXP_ATTRS = 105
+FXP_EXTENDED = 200
+FXP_EXTENDED_REPLY = 201
+
+FILEXFER_ATTR_SIZE = 0x00000001
+FILEXFER_ATTR_UIDGID = 0x00000002
+FILEXFER_ATTR_OWNERGROUP = FILEXFER_ATTR_UIDGID
+FILEXFER_ATTR_PERMISSIONS = 0x00000004
+FILEXFER_ATTR_ACMODTIME = 0x00000008
+FILEXFER_ATTR_EXTENDED = 0x80000000
+
+FILEXFER_TYPE_REGULAR = 1
+FILEXFER_TYPE_DIRECTORY = 2
+FILEXFER_TYPE_SYMLINK = 3
+FILEXFER_TYPE_SPECIAL = 4
+FILEXFER_TYPE_UNKNOWN = 5
+
+FXF_READ = 0x00000001
+FXF_WRITE = 0x00000002
+FXF_APPEND = 0x00000004
+FXF_CREAT = 0x00000008
+FXF_TRUNC = 0x00000010
+FXF_EXCL = 0x00000020
+FXF_TEXT = 0x00000040
+
+FX_OK = 0
+FX_EOF = 1
+FX_NO_SUCH_FILE = 2
+FX_PERMISSION_DENIED = 3
+FX_FAILURE = 4
+FX_BAD_MESSAGE = 5
+FX_NO_CONNECTION = 6
+FX_CONNECTION_LOST = 7
+FX_OP_UNSUPPORTED = 8
+FX_FILE_ALREADY_EXISTS = 11
+# http://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/ defines more
+# useful error codes, but so far OpenSSH doesn't implement them. We use them
+# internally for clarity, but for now define them all as FX_FAILURE to be
+# compatible with existing software.
+FX_NOT_A_DIRECTORY = FX_FAILURE
+FX_FILE_IS_A_DIRECTORY = FX_FAILURE
+
+
+# initialize FileTransferBase.packetTypes:
+g = globals()
+for name in list(g.keys()):
+    if name.startswith("FXP_"):
+        value = g[name]
+        FileTransferBase.packetTypes[value] = name[4:]
+del g, name, value
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/forwarding.py b/contrib/python/Twisted/py3/twisted/conch/ssh/forwarding.py
new file mode 100644
index 00000000000..4068cdc3ffd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/forwarding.py
@@ -0,0 +1,272 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module contains the implementation of the TCP forwarding, which allows
+clients and servers to forward arbitrary TCP data across the connection.
+
+Maintainer: Paul Swartz
+"""
+
+
+import struct
+
+from twisted.conch.ssh import channel, common
+from twisted.internet import protocol, reactor
+from twisted.internet.endpoints import HostnameEndpoint, connectProtocol
+
+
+class SSHListenForwardingFactory(protocol.Factory):
+    def __init__(self, connection, hostport, klass):
+        self.conn = connection
+        self.hostport = hostport  # tuple
+        self.klass = klass
+
+    def buildProtocol(self, addr):
+        channel = self.klass(conn=self.conn)
+        client = SSHForwardingClient(channel)
+        channel.client = client
+        addrTuple = (addr.host, addr.port)
+        channelOpenData = packOpen_direct_tcpip(self.hostport, addrTuple)
+        self.conn.openChannel(channel, channelOpenData)
+        return client
+
+
+class SSHListenForwardingChannel(channel.SSHChannel):
+    def channelOpen(self, specificData):
+        self._log.info("opened forwarding channel {id}", id=self.id)
+        if len(self.client.buf) > 1:
+            b = self.client.buf[1:]
+            self.write(b)
+        self.client.buf = b""
+
+    def openFailed(self, reason):
+        self.closed()
+
+    def dataReceived(self, data):
+        self.client.transport.write(data)
+
+    def eofReceived(self):
+        self.client.transport.loseConnection()
+
+    def closed(self):
+        if hasattr(self, "client"):
+            self._log.info("closing local forwarding channel {id}", id=self.id)
+            self.client.transport.loseConnection()
+            del self.client
+
+
+class SSHListenClientForwardingChannel(SSHListenForwardingChannel):
+    name = b"direct-tcpip"
+
+
+class SSHListenServerForwardingChannel(SSHListenForwardingChannel):
+    name = b"forwarded-tcpip"
+
+
+class SSHConnectForwardingChannel(channel.SSHChannel):
+    """
+    Channel used for handling server side forwarding request.
+    It acts as a client for the remote forwarding destination.
+
+    @ivar hostport: C{(host, port)} requested by client as forwarding
+        destination.
+    @type hostport: L{tuple} or a C{sequence}
+
+    @ivar client: Protocol connected to the forwarding destination.
+    @type client: L{protocol.Protocol}
+
+    @ivar clientBuf: Data received while forwarding channel is not yet
+        connected.
+    @type clientBuf: L{bytes}
+
+    @var  _reactor: Reactor used for TCP connections.
+    @type _reactor: A reactor.
+
+    @ivar _channelOpenDeferred: Deferred used in testing to check the
+        result of C{channelOpen}.
+    @type _channelOpenDeferred: L{twisted.internet.defer.Deferred}
+    """
+
+    _reactor = reactor
+
+    def __init__(self, hostport, *args, **kw):
+        channel.SSHChannel.__init__(self, *args, **kw)
+        self.hostport = hostport
+        self.client = None
+        self.clientBuf = b""
+
+    def channelOpen(self, specificData):
+        """
+        See: L{channel.SSHChannel}
+        """
+        self._log.info(
+            "connecting to {host}:{port}", host=self.hostport[0], port=self.hostport[1]
+        )
+        ep = HostnameEndpoint(self._reactor, self.hostport[0], self.hostport[1])
+        d = connectProtocol(ep, SSHForwardingClient(self))
+        d.addCallbacks(self._setClient, self._close)
+        self._channelOpenDeferred = d
+
+    def _setClient(self, client):
+        """
+        Called when the connection was established to the forwarding
+        destination.
+
+        @param client: Client protocol connected to the forwarding destination.
+        @type  client: L{protocol.Protocol}
+        """
+        self.client = client
+        self._log.info(
+            "connected to {host}:{port}", host=self.hostport[0], port=self.hostport[1]
+        )
+        if self.clientBuf:
+            self.client.transport.write(self.clientBuf)
+            self.clientBuf = None
+        if self.client.buf[1:]:
+            self.write(self.client.buf[1:])
+        self.client.buf = b""
+
+    def _close(self, reason):
+        """
+        Called when failed to connect to the forwarding destination.
+
+        @param reason: Reason why connection failed.
+        @type  reason: L{twisted.python.failure.Failure}
+        """
+        self._log.error(
+            "failed to connect to {host}:{port}: {reason}",
+            host=self.hostport[0],
+            port=self.hostport[1],
+            reason=reason,
+        )
+        self.loseConnection()
+
+    def dataReceived(self, data):
+        """
+        See: L{channel.SSHChannel}
+        """
+        if self.client:
+            self.client.transport.write(data)
+        else:
+            self.clientBuf += data
+
+    def closed(self):
+        """
+        See: L{channel.SSHChannel}
+        """
+        if self.client:
+            self._log.info("closed remote forwarding channel {id}", id=self.id)
+            if self.client.channel:
+                self.loseConnection()
+            self.client.transport.loseConnection()
+            del self.client
+
+
+def openConnectForwardingClient(remoteWindow, remoteMaxPacket, data, avatar):
+    remoteHP, origHP = unpackOpen_direct_tcpip(data)
+    return SSHConnectForwardingChannel(
+        remoteHP,
+        remoteWindow=remoteWindow,
+        remoteMaxPacket=remoteMaxPacket,
+        avatar=avatar,
+    )
+
+
+class SSHForwardingClient(protocol.Protocol):
+    def __init__(self, channel):
+        self.channel = channel
+        self.buf = b"\000"
+
+    def dataReceived(self, data):
+        if self.buf:
+            self.buf += data
+        else:
+            self.channel.write(data)
+
+    def connectionLost(self, reason):
+        if self.channel:
+            self.channel.loseConnection()
+            self.channel = None
+
+
+def packOpen_direct_tcpip(destination, source):
+    """
+    Pack the data suitable for sending in a CHANNEL_OPEN packet.
+
+    @type destination: L{tuple}
+    @param destination: A tuple of the (host, port) of the destination host.
+
+    @type source: L{tuple}
+    @param source: A tuple of the (host, port) of the source host.
+    """
+    (connHost, connPort) = destination
+    (origHost, origPort) = source
+    if isinstance(connHost, str):
+        connHost = connHost.encode("utf-8")
+    if isinstance(origHost, str):
+        origHost = origHost.encode("utf-8")
+    conn = common.NS(connHost) + struct.pack(">L", connPort)
+    orig = common.NS(origHost) + struct.pack(">L", origPort)
+    return conn + orig
+
+
+packOpen_forwarded_tcpip = packOpen_direct_tcpip
+
+
+def unpackOpen_direct_tcpip(data):
+    """Unpack the data to a usable format."""
+    connHost, rest = common.getNS(data)
+    if isinstance(connHost, bytes):
+        connHost = connHost.decode("utf-8")
+    connPort = int(struct.unpack(">L", rest[:4])[0])
+    origHost, rest = common.getNS(rest[4:])
+    if isinstance(origHost, bytes):
+        origHost = origHost.decode("utf-8")
+    origPort = int(struct.unpack(">L", rest[:4])[0])
+    return (connHost, connPort), (origHost, origPort)
+
+
+unpackOpen_forwarded_tcpip = unpackOpen_direct_tcpip
+
+
+def packGlobal_tcpip_forward(peer):
+    """
+    Pack the data for tcpip forwarding.
+
+    @param peer: A tuple of the (host, port) .
+    @type peer: L{tuple}
+    """
+    (host, port) = peer
+    return common.NS(host) + struct.pack(">L", port)
+
+
+def unpackGlobal_tcpip_forward(data):
+    host, rest = common.getNS(data)
+    if isinstance(host, bytes):
+        host = host.decode("utf-8")
+    port = int(struct.unpack(">L", rest[:4])[0])
+    return host, port
+
+
+"""This is how the data -> eof -> close stuff /should/ work.
+
+debug3: channel 1: waiting for connection
+debug1: channel 1: connected
+debug1: channel 1: read<=0 rfd 7 len 0
+debug1: channel 1: read failed
+debug1: channel 1: close_read
+debug1: channel 1: input open -> drain
+debug1: channel 1: ibuf empty
+debug1: channel 1: send eof
+debug1: channel 1: input drain -> closed
+debug1: channel 1: rcvd eof
+debug1: channel 1: output open -> drain
+debug1: channel 1: obuf empty
+debug1: channel 1: close_write
+debug1: channel 1: output drain -> closed
+debug1: channel 1: rcvd close
+debug3: channel 1: will not send data after close
+debug1: channel 1: send close
+debug1: channel 1: is dead
+"""
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/keys.py b/contrib/python/Twisted/py3/twisted/conch/ssh/keys.py
new file mode 100644
index 00000000000..e959f022a0b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/keys.py
@@ -0,0 +1,1818 @@
+# -*- test-case-name: twisted.conch.test.test_keys -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Handling of RSA, DSA, ECDSA, and Ed25519 keys.
+"""
+from __future__ import annotations
+
+import binascii
+import struct
+import unicodedata
+import warnings
+from base64 import b64encode, decodebytes, encodebytes
+from hashlib import md5, sha256
+from typing import Any
+
+import bcrypt
+from cryptography import utils
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import dsa, ec, ed25519, padding, rsa
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.primitives.serialization import (
+    load_pem_private_key,
+    load_ssh_public_key,
+)
+from typing_extensions import Literal
+
+from twisted.conch.ssh import common, sexpy
+from twisted.conch.ssh.common import int_to_bytes
+from twisted.python import randbytes
+from twisted.python.compat import iterbytes, nativeString
+from twisted.python.constants import NamedConstant, Names
+from twisted.python.deprecate import _mutuallyExclusiveArguments
+
+try:
+    from cryptography.hazmat.primitives.asymmetric.utils import (
+        decode_dss_signature,
+        encode_dss_signature,
+    )
+except ImportError:
+    from cryptography.hazmat.primitives.asymmetric.utils import (  # type: ignore[no-redef,attr-defined]
+        decode_rfc6979_signature as decode_dss_signature,
+        encode_rfc6979_signature as encode_dss_signature,
+    )
+
+
+# Curve lookup table
+_curveTable = {
+    b"ecdsa-sha2-nistp256": ec.SECP256R1(),
+    b"ecdsa-sha2-nistp384": ec.SECP384R1(),
+    b"ecdsa-sha2-nistp521": ec.SECP521R1(),
+}
+
+_secToNist = {
+    b"secp256r1": b"nistp256",
+    b"secp384r1": b"nistp384",
+    b"secp521r1": b"nistp521",
+}
+
+
+Ed25519PublicKey = ed25519.Ed25519PublicKey
+Ed25519PrivateKey = ed25519.Ed25519PrivateKey
+
+
+class BadKeyError(Exception):
+    """
+    Raised when a key isn't what we expected from it.
+
+    XXX: we really need to check for bad keys
+    """
+
+
+class BadSignatureAlgorithmError(Exception):
+    """
+    Raised when a public key signature algorithm name isn't defined for this
+    public key format.
+    """
+
+
+class EncryptedKeyError(Exception):
+    """
+    Raised when an encrypted key is presented to fromString/fromFile without
+    a password.
+    """
+
+
+class BadFingerPrintFormat(Exception):
+    """
+    Raises when unsupported fingerprint formats are presented to fingerprint.
+    """
+
+
+class FingerprintFormats(Names):
+    """
+    Constants representing the supported formats of key fingerprints.
+
+    @cvar MD5_HEX: Named constant representing fingerprint format generated
+        using md5[RFC1321] algorithm in hexadecimal encoding.
+    @type MD5_HEX: L{twisted.python.constants.NamedConstant}
+
+    @cvar SHA256_BASE64: Named constant representing fingerprint format
+        generated using sha256[RFC4634] algorithm in base64 encoding
+    @type SHA256_BASE64: L{twisted.python.constants.NamedConstant}
+    """
+
+    MD5_HEX = NamedConstant()
+    SHA256_BASE64 = NamedConstant()
+
+
+class PassphraseNormalizationError(Exception):
+    """
+    Raised when a passphrase contains Unicode characters that cannot be
+    normalized using the available Unicode character database.
+    """
+
+
+def _normalizePassphrase(passphrase):
+    """
+    Normalize a passphrase, which may be Unicode.
+
+    If the passphrase is Unicode, this follows the requirements of U{NIST
+    800-63B, section
+    5.1.1.2<https://pages.nist.gov/800-63-3/sp800-63b.html#memsecretver>}
+    for Unicode characters in memorized secrets: it applies the
+    Normalization Process for Stabilized Strings using NFKC normalization.
+    The passphrase is then encoded using UTF-8.
+
+    @type passphrase: L{bytes} or L{unicode} or L{None}
+    @param passphrase: The passphrase to normalize.
+
+    @return: The normalized passphrase, if any.
+    @rtype: L{bytes} or L{None}
+    @raises PassphraseNormalizationError: if the passphrase is Unicode and
+    cannot be normalized using the available Unicode character database.
+    """
+    if isinstance(passphrase, str):
+        # The Normalization Process for Stabilized Strings requires aborting
+        # with an error if the string contains any unassigned code point.
+        if any(unicodedata.category(c) == "Cn" for c in passphrase):
+            # Perhaps not very helpful, but we don't want to leak any other
+            # information about the passphrase.
+            raise PassphraseNormalizationError()
+        return unicodedata.normalize("NFKC", passphrase).encode("UTF-8")
+    else:
+        return passphrase
+
+
+class Key:
+    """
+    An object representing a key.  A key can be either a public or
+    private key.  A public key can verify a signature; a private key can
+    create or verify a signature.  To generate a string that can be stored
+    on disk, use the toString method.  If you have a private key, but want
+    the string representation of the public key, use Key.public().toString().
+    """
+
+    @classmethod
+    def fromFile(cls, filename, type=None, passphrase=None):
+        """
+        Load a key from a file.
+
+        @param filename: The path to load key data from.
+
+        @type type: L{str} or L{None}
+        @param type: A string describing the format the key data is in, or
+        L{None} to attempt detection of the type.
+
+        @type passphrase: L{bytes} or L{None}
+        @param passphrase: The passphrase the key is encrypted with, or L{None}
+        if there is no encryption.
+
+        @rtype: L{Key}
+        @return: The loaded key.
+        """
+        with open(filename, "rb") as f:
+            return cls.fromString(f.read(), type, passphrase)
+
+    @classmethod
+    def fromString(cls, data, type=None, passphrase=None):
+        """
+        Return a Key object corresponding to the string data.
+        type is optionally the type of string, matching a _fromString_*
+        method.  Otherwise, the _guessStringType() classmethod will be used
+        to guess a type.  If the key is encrypted, passphrase is used as
+        the decryption key.
+
+        @type data: L{bytes}
+        @param data: The key data.
+
+        @type type: L{str} or L{None}
+        @param type: A string describing the format the key data is in, or
+        L{None} to attempt detection of the type.
+
+        @type passphrase: L{bytes} or L{None}
+        @param passphrase: The passphrase the key is encrypted with, or L{None}
+        if there is no encryption.
+
+        @rtype: L{Key}
+        @return: The loaded key.
+        """
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        passphrase = _normalizePassphrase(passphrase)
+        if type is None:
+            type = cls._guessStringType(data)
+        if type is None:
+            raise BadKeyError(f"cannot guess the type of {data!r}")
+        method = getattr(cls, f"_fromString_{type.upper()}", None)
+        if method is None:
+            raise BadKeyError(f"no _fromString method for {type}")
+        if method.__code__.co_argcount == 2:  # No passphrase
+            if passphrase:
+                raise BadKeyError("key not encrypted")
+            return method(data)
+        else:
+            return method(data, passphrase)
+
+    @classmethod
+    def _fromString_BLOB(cls, blob):
+        """
+        Return a public key object corresponding to this public key blob.
+        The format of a RSA public key blob is::
+            string 'ssh-rsa'
+            integer e
+            integer n
+
+        The format of a DSA public key blob is::
+            string 'ssh-dss'
+            integer p
+            integer q
+            integer g
+            integer y
+
+        The format of ECDSA-SHA2-* public key blob is::
+            string 'ecdsa-sha2-[identifier]'
+            integer x
+            integer y
+
+            identifier is the standard NIST curve name.
+
+        The format of an Ed25519 public key blob is::
+            string 'ssh-ed25519'
+            string a
+
+        @type blob: L{bytes}
+        @param blob: The key data.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if the key type (the first string) is unknown.
+        """
+        keyType, rest = common.getNS(blob)
+        if keyType == b"ssh-rsa":
+            e, n, rest = common.getMP(rest, 2)
+            return cls(rsa.RSAPublicNumbers(e, n).public_key(default_backend()))
+        elif keyType == b"ssh-dss":
+            p, q, g, y, rest = common.getMP(rest, 4)
+            return cls(
+                dsa.DSAPublicNumbers(
+                    y=y, parameter_numbers=dsa.DSAParameterNumbers(p=p, q=q, g=g)
+                ).public_key(default_backend())
+            )
+        elif keyType in _curveTable:
+            return cls(
+                ec.EllipticCurvePublicKey.from_encoded_point(
+                    _curveTable[keyType], common.getNS(rest, 2)[1]
+                )
+            )
+        elif keyType == b"ssh-ed25519":
+            a, rest = common.getNS(rest)
+            return cls._fromEd25519Components(a)
+        else:
+            raise BadKeyError(f"unknown blob type: {keyType}")
+
+    @classmethod
+    def _fromString_PRIVATE_BLOB(cls, blob):
+        """
+        Return a private key object corresponding to this private key blob.
+        The blob formats are as follows:
+
+        RSA keys::
+            string 'ssh-rsa'
+            integer n
+            integer e
+            integer d
+            integer u
+            integer p
+            integer q
+
+        DSA keys::
+            string 'ssh-dss'
+            integer p
+            integer q
+            integer g
+            integer y
+            integer x
+
+        EC keys::
+            string 'ecdsa-sha2-[identifier]'
+            string identifier
+            string q
+            integer privateValue
+
+            identifier is the standard NIST curve name.
+
+        Ed25519 keys::
+            string 'ssh-ed25519'
+            string a
+            string k || a
+
+
+        @type blob: L{bytes}
+        @param blob: The key data.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if
+            * the key type (the first string) is unknown
+            * the curve name of an ECDSA key does not match the key type
+        """
+        keyType, rest = common.getNS(blob)
+
+        if keyType == b"ssh-rsa":
+            n, e, d, u, p, q, rest = common.getMP(rest, 6)
+            return cls._fromRSAComponents(n=n, e=e, d=d, p=p, q=q)
+        elif keyType == b"ssh-dss":
+            p, q, g, y, x, rest = common.getMP(rest, 5)
+            return cls._fromDSAComponents(y=y, g=g, p=p, q=q, x=x)
+        elif keyType in _curveTable:
+            curve = _curveTable[keyType]
+            curveName, q, rest = common.getNS(rest, 2)
+            if curveName != _secToNist[curve.name.encode("ascii")]:
+                raise BadKeyError(
+                    "ECDSA curve name %r does not match key "
+                    "type %r" % (curveName, keyType)
+                )
+            privateValue, rest = common.getMP(rest)
+            return cls._fromECEncodedPoint(
+                encodedPoint=q, curve=keyType, privateValue=privateValue
+            )
+        elif keyType == b"ssh-ed25519":
+            # OpenSSH's format repeats the public key bytes for some reason.
+            # We're only interested in the private key here anyway.
+            a, combined, rest = common.getNS(rest, 2)
+            k = combined[:32]
+            return cls._fromEd25519Components(a, k=k)
+        else:
+            raise BadKeyError(f"unknown blob type: {keyType}")
+
+    @classmethod
+    def _fromString_PUBLIC_OPENSSH(cls, data):
+        """
+        Return a public key object corresponding to this OpenSSH public key
+        string.  The format of an OpenSSH public key string is::
+            <key type> <base64-encoded public key blob>
+
+        @type data: L{bytes}
+        @param data: The key data.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if the blob type is unknown.
+        """
+        # ECDSA keys don't need base64 decoding which is required
+        # for RSA or DSA key.
+        if data.startswith(b"ecdsa-sha2"):
+            return cls(load_ssh_public_key(data, default_backend()))
+        blob = decodebytes(data.split()[1])
+        return cls._fromString_BLOB(blob)
+
+    @classmethod
+    def _fromPrivateOpenSSH_v1(cls, data, passphrase):
+        """
+        Return a private key object corresponding to this OpenSSH private key
+        string, in the "openssh-key-v1" format introduced in OpenSSH 6.5.
+
+        The format of an openssh-key-v1 private key string is::
+            -----BEGIN OPENSSH PRIVATE KEY-----
+            <base64-encoded SSH protocol string>
+            -----END OPENSSH PRIVATE KEY-----
+
+        The SSH protocol string is as described in
+        U{PROTOCOL.key<https://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL.key>}.
+
+        @type data: L{bytes}
+        @param data: The key data.
+
+        @type passphrase: L{bytes} or L{None}
+        @param passphrase: The passphrase the key is encrypted with, or L{None}
+        if it is not encrypted.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if
+            * a passphrase is provided for an unencrypted key
+            * the SSH protocol encoding is incorrect
+        @raises EncryptedKeyError: if
+            * a passphrase is not provided for an encrypted key
+        """
+        lines = data.strip().splitlines()
+        keyList = decodebytes(b"".join(lines[1:-1]))
+        if not keyList.startswith(b"openssh-key-v1\0"):
+            raise BadKeyError("unknown OpenSSH private key format")
+        keyList = keyList[len(b"openssh-key-v1\0") :]
+        cipher, kdf, kdfOptions, rest = common.getNS(keyList, 3)
+        n = struct.unpack("!L", rest[:4])[0]
+        if n != 1:
+            raise BadKeyError(
+                "only OpenSSH private key files containing "
+                "a single key are supported"
+            )
+        # Ignore public key
+        _, encPrivKeyList, _ = common.getNS(rest[4:], 2)
+        if cipher != b"none":
+            if not passphrase:
+                raise EncryptedKeyError(
+                    "Passphrase must be provided " "for an encrypted key"
+                )
+            # Determine cipher
+            if cipher in (b"aes128-ctr", b"aes192-ctr", b"aes256-ctr"):
+                algorithmClass = algorithms.AES
+                blockSize = 16
+                keySize = int(cipher[3:6]) // 8
+                ivSize = blockSize
+            else:
+                raise BadKeyError(f"unknown encryption type {cipher!r}")
+            if kdf == b"bcrypt":
+                salt, rest = common.getNS(kdfOptions)
+                rounds = struct.unpack("!L", rest[:4])[0]
+                decKey = bcrypt.kdf(
+                    passphrase,
+                    salt,
+                    keySize + ivSize,
+                    rounds,
+                    # We can only use the number of rounds that OpenSSH used.
+                    ignore_few_rounds=True,
+                )
+            else:
+                raise BadKeyError(f"unknown KDF type {kdf!r}")
+            if (len(encPrivKeyList) % blockSize) != 0:
+                raise BadKeyError("bad padding")
+            decryptor = Cipher(
+                algorithmClass(decKey[:keySize]),
+                modes.CTR(decKey[keySize : keySize + ivSize]),
+                backend=default_backend(),
+            ).decryptor()
+            privKeyList = decryptor.update(encPrivKeyList) + decryptor.finalize()
+        else:
+            if kdf != b"none":
+                raise BadKeyError(
+                    "private key specifies KDF %r but no " "cipher" % (kdf,)
+                )
+            privKeyList = encPrivKeyList
+        check1 = struct.unpack("!L", privKeyList[:4])[0]
+        check2 = struct.unpack("!L", privKeyList[4:8])[0]
+        if check1 != check2:
+            raise BadKeyError("check values do not match: %d != %d" % (check1, check2))
+        return cls._fromString_PRIVATE_BLOB(privKeyList[8:])
+
+    @classmethod
+    def _fromPrivateOpenSSH_PEM(cls, data, passphrase):
+        """
+        Return a private key object corresponding to this OpenSSH private key
+        string, in the old PEM-based format.
+
+        The format of a PEM-based OpenSSH private key string is::
+            -----BEGIN <key type> PRIVATE KEY-----
+            [Proc-Type: 4,ENCRYPTED
+            DEK-Info: DES-EDE3-CBC,<initialization value>]
+            <base64-encoded ASN.1 structure>
+            ------END <key type> PRIVATE KEY------
+
+        The ASN.1 structure of a RSA key is::
+            (0, n, e, d, p, q)
+
+        The ASN.1 structure of a DSA key is::
+            (0, p, q, g, y, x)
+
+        The ASN.1 structure of a ECDSA key is::
+            (ECParameters, OID, NULL)
+
+        @type data: L{bytes}
+        @param data: The key data.
+
+        @type passphrase: L{bytes} or L{None}
+        @param passphrase: The passphrase the key is encrypted with, or L{None}
+        if it is not encrypted.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if
+            * a passphrase is provided for an unencrypted key
+            * the ASN.1 encoding is incorrect
+        @raises EncryptedKeyError: if
+            * a passphrase is not provided for an encrypted key
+        """
+        lines = data.strip().splitlines()
+        kind = lines[0][11:-17]
+        # cryptography considers an empty byte string a passphrase, but
+        # twisted considers that to be "no password". So we need to convert
+        # to None on empty.
+        if not passphrase:
+            passphrase = None
+        if kind in (b"EC", b"RSA", b"DSA"):
+            try:
+                key = load_pem_private_key(data, passphrase, default_backend())
+            except TypeError:
+                raise EncryptedKeyError(
+                    "Passphrase must be provided for an encrypted key"
+                )
+            except ValueError:
+                raise BadKeyError("Failed to decode key (Bad Passphrase?)")
+            return cls(key)
+        else:
+            raise BadKeyError(f"unknown key type {kind}")
+
+    @classmethod
+    def _fromString_PRIVATE_OPENSSH(cls, data, passphrase):
+        """
+        Return a private key object corresponding to this OpenSSH private key
+        string.  If the key is encrypted, passphrase MUST be provided.
+        Providing a passphrase for an unencrypted key is an error.
+
+        @type data: L{bytes}
+        @param data: The key data.
+
+        @type passphrase: L{bytes} or L{None}
+        @param passphrase: The passphrase the key is encrypted with, or L{None}
+        if it is not encrypted.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if
+            * a passphrase is provided for an unencrypted key
+            * the encoding is incorrect
+        @raises EncryptedKeyError: if
+            * a passphrase is not provided for an encrypted key
+        """
+        if data.strip().splitlines()[0][11:-17] == b"OPENSSH":
+            # New-format (openssh-key-v1) key
+            return cls._fromPrivateOpenSSH_v1(data, passphrase)
+        else:
+            # Old-format (PEM) key
+            return cls._fromPrivateOpenSSH_PEM(data, passphrase)
+
+    @classmethod
+    def _fromString_PUBLIC_LSH(cls, data):
+        """
+        Return a public key corresponding to this LSH public key string.
+        The LSH public key string format is::
+            <s-expression: ('public-key', (<key type>, (<name, <value>)+))>
+
+        The names for a RSA (key type 'rsa-pkcs1-sha1') key are: n, e.
+        The names for a DSA (key type 'dsa') key are: y, g, p, q.
+
+        @type data: L{bytes}
+        @param data: The key data.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if the key type is unknown
+        """
+        sexp = sexpy.parse(decodebytes(data[1:-1]))
+        assert sexp[0] == b"public-key"
+        kd = {}
+        for name, data in sexp[1][1:]:
+            kd[name] = common.getMP(common.NS(data))[0]
+        if sexp[1][0] == b"dsa":
+            return cls._fromDSAComponents(
+                y=kd[b"y"], g=kd[b"g"], p=kd[b"p"], q=kd[b"q"]
+            )
+
+        elif sexp[1][0] == b"rsa-pkcs1-sha1":
+            return cls._fromRSAComponents(n=kd[b"n"], e=kd[b"e"])
+        else:
+            raise BadKeyError(f"unknown lsh key type {sexp[1][0]}")
+
+    @classmethod
+    def _fromString_PRIVATE_LSH(cls, data):
+        """
+        Return a private key corresponding to this LSH private key string.
+        The LSH private key string format is::
+            <s-expression: ('private-key', (<key type>, (<name>, <value>)+))>
+
+        The names for a RSA (key type 'rsa-pkcs1-sha1') key are: n, e, d, p, q.
+        The names for a DSA (key type 'dsa') key are: y, g, p, q, x.
+
+        @type data: L{bytes}
+        @param data: The key data.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if the key type is unknown
+        """
+        sexp = sexpy.parse(data)
+        assert sexp[0] == b"private-key"
+        kd = {}
+        for name, data in sexp[1][1:]:
+            kd[name] = common.getMP(common.NS(data))[0]
+        if sexp[1][0] == b"dsa":
+            assert len(kd) == 5, len(kd)
+            return cls._fromDSAComponents(
+                y=kd[b"y"], g=kd[b"g"], p=kd[b"p"], q=kd[b"q"], x=kd[b"x"]
+            )
+        elif sexp[1][0] == b"rsa-pkcs1":
+            assert len(kd) == 8, len(kd)
+            if kd[b"p"] > kd[b"q"]:  # Make p smaller than q
+                kd[b"p"], kd[b"q"] = kd[b"q"], kd[b"p"]
+            return cls._fromRSAComponents(
+                n=kd[b"n"], e=kd[b"e"], d=kd[b"d"], p=kd[b"p"], q=kd[b"q"]
+            )
+
+        else:
+            raise BadKeyError(f"unknown lsh key type {sexp[1][0]}")
+
+    @classmethod
+    def _fromString_AGENTV3(cls, data):
+        """
+        Return a private key object corresponsing to the Secure Shell Key
+        Agent v3 format.
+
+        The SSH Key Agent v3 format for a RSA key is::
+            string 'ssh-rsa'
+            integer e
+            integer d
+            integer n
+            integer u
+            integer p
+            integer q
+
+        The SSH Key Agent v3 format for a DSA key is::
+            string 'ssh-dss'
+            integer p
+            integer q
+            integer g
+            integer y
+            integer x
+
+        @type data: L{bytes}
+        @param data: The key data.
+
+        @return: A new key.
+        @rtype: L{twisted.conch.ssh.keys.Key}
+        @raises BadKeyError: if the key type (the first string) is unknown
+        """
+        keyType, data = common.getNS(data)
+        if keyType == b"ssh-dss":
+            p, data = common.getMP(data)
+            q, data = common.getMP(data)
+            g, data = common.getMP(data)
+            y, data = common.getMP(data)
+            x, data = common.getMP(data)
+            return cls._fromDSAComponents(y=y, g=g, p=p, q=q, x=x)
+        elif keyType == b"ssh-rsa":
+            e, data = common.getMP(data)
+            d, data = common.getMP(data)
+            n, data = common.getMP(data)
+            u, data = common.getMP(data)
+            p, data = common.getMP(data)
+            q, data = common.getMP(data)
+            return cls._fromRSAComponents(n=n, e=e, d=d, p=p, q=q, u=u)
+        else:
+            raise BadKeyError(f"unknown key type {keyType}")
+
+    @classmethod
+    def _guessStringType(cls, data):
+        """
+        Guess the type of key in data.  The types map to _fromString_*
+        methods.
+
+        @type data: L{bytes}
+        @param data: The key data.
+        """
+        if data.startswith(b"ssh-") or data.startswith(b"ecdsa-sha2-"):
+            return "public_openssh"
+        elif data.startswith(b"-----BEGIN"):
+            return "private_openssh"
+        elif data.startswith(b"{"):
+            return "public_lsh"
+        elif data.startswith(b"("):
+            return "private_lsh"
+        elif (
+            data.startswith(b"\x00\x00\x00\x07ssh-")
+            or data.startswith(b"\x00\x00\x00\x13ecdsa-")
+            or data.startswith(b"\x00\x00\x00\x0bssh-ed25519")
+        ):
+            ignored, rest = common.getNS(data)
+            count = 0
+            while rest:
+                count += 1
+                ignored, rest = common.getMP(rest)
+            if count > 4:
+                return "agentv3"
+            else:
+                return "blob"
+
+    @classmethod
+    def _fromRSAComponents(cls, n, e, d=None, p=None, q=None, u=None):
+        """
+        Build a key from RSA numerical components.
+
+        @type n: L{int}
+        @param n: The 'n' RSA variable.
+
+        @type e: L{int}
+        @param e: The 'e' RSA variable.
+
+        @type d: L{int} or L{None}
+        @param d: The 'd' RSA variable (optional for a public key).
+
+        @type p: L{int} or L{None}
+        @param p: The 'p' RSA variable (optional for a public key).
+
+        @type q: L{int} or L{None}
+        @param q: The 'q' RSA variable (optional for a public key).
+
+        @type u: L{int} or L{None}
+        @param u: The 'u' RSA variable. Ignored, as its value is determined by
+        p and q.
+
+        @rtype: L{Key}
+        @return: An RSA key constructed from the values as given.
+        """
+        publicNumbers = rsa.RSAPublicNumbers(e=e, n=n)
+        if d is None:
+            # We have public components.
+            keyObject = publicNumbers.public_key(default_backend())
+        else:
+            privateNumbers = rsa.RSAPrivateNumbers(
+                p=p,
+                q=q,
+                d=d,
+                dmp1=rsa.rsa_crt_dmp1(d, p),
+                dmq1=rsa.rsa_crt_dmq1(d, q),
+                iqmp=rsa.rsa_crt_iqmp(p, q),
+                public_numbers=publicNumbers,
+            )
+            keyObject = privateNumbers.private_key(default_backend())
+
+        return cls(keyObject)
+
+    @classmethod
+    def _fromDSAComponents(cls, y, p, q, g, x=None):
+        """
+        Build a key from DSA numerical components.
+
+        @type y: L{int}
+        @param y: The 'y' DSA variable.
+
+        @type p: L{int}
+        @param p: The 'p' DSA variable.
+
+        @type q: L{int}
+        @param q: The 'q' DSA variable.
+
+        @type g: L{int}
+        @param g: The 'g' DSA variable.
+
+        @type x: L{int} or L{None}
+        @param x: The 'x' DSA variable (optional for a public key)
+
+        @rtype: L{Key}
+        @return: A DSA key constructed from the values as given.
+        """
+        publicNumbers = dsa.DSAPublicNumbers(
+            y=y, parameter_numbers=dsa.DSAParameterNumbers(p=p, q=q, g=g)
+        )
+        if x is None:
+            # We have public components.
+            keyObject = publicNumbers.public_key(default_backend())
+        else:
+            privateNumbers = dsa.DSAPrivateNumbers(x=x, public_numbers=publicNumbers)
+            keyObject = privateNumbers.private_key(default_backend())
+
+        return cls(keyObject)
+
+    @classmethod
+    def _fromECComponents(cls, x, y, curve, privateValue=None):
+        """
+        Build a key from EC components.
+
+        @param x: The affine x component of the public point used for verifying.
+        @type x: L{int}
+
+        @param y: The affine y component of the public point used for verifying.
+        @type y: L{int}
+
+        @param curve: NIST name of elliptic curve.
+        @type curve: L{bytes}
+
+        @param privateValue: The private value.
+        @type privateValue: L{int}
+        """
+
+        publicNumbers = ec.EllipticCurvePublicNumbers(
+            x=x, y=y, curve=_curveTable[curve]
+        )
+        if privateValue is None:
+            # We have public components.
+            keyObject = publicNumbers.public_key(default_backend())
+        else:
+            privateNumbers = ec.EllipticCurvePrivateNumbers(
+                private_value=privateValue, public_numbers=publicNumbers
+            )
+            keyObject = privateNumbers.private_key(default_backend())
+
+        return cls(keyObject)
+
+    @classmethod
+    def _fromECEncodedPoint(cls, encodedPoint, curve, privateValue=None):
+        """
+        Build a key from an EC encoded point.
+
+        @param encodedPoint: The public point encoded as in SEC 1 v2.0
+        section 2.3.3.
+        @type encodedPoint: L{bytes}
+
+        @param curve: NIST name of elliptic curve.
+        @type curve: L{bytes}
+
+        @param privateValue: The private value.
+        @type privateValue: L{int}
+        """
+
+        if privateValue is None:
+            # We have public components.
+            keyObject = ec.EllipticCurvePublicKey.from_encoded_point(
+                _curveTable[curve], encodedPoint
+            )
+        else:
+            keyObject = ec.derive_private_key(
+                privateValue, _curveTable[curve], default_backend()
+            )
+
+        return cls(keyObject)
+
+    @classmethod
+    def _fromEd25519Components(cls, a, k=None):
+        """Build a key from Ed25519 components.
+
+        @param a: The Ed25519 public key, as defined in RFC 8032 section
+            5.1.5.
+        @type a: L{bytes}
+
+        @param k: The Ed25519 private key, as defined in RFC 8032 section
+            5.1.5.
+        @type k: L{bytes}
+        """
+
+        if Ed25519PublicKey is None or Ed25519PrivateKey is None:
+            raise BadKeyError("Ed25519 keys not supported on this system")
+
+        if k is None:
+            keyObject = Ed25519PublicKey.from_public_bytes(a)
+        else:
+            keyObject = Ed25519PrivateKey.from_private_bytes(k)
+
+        return cls(keyObject)
+
+    def __init__(self, keyObject):
+        """
+        Initialize with a private or public
+        C{cryptography.hazmat.primitives.asymmetric} key.
+
+        @param keyObject: Low level key.
+        @type keyObject: C{cryptography.hazmat.primitives.asymmetric} key.
+        """
+        self._keyObject = keyObject
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Return True if other represents an object with the same key.
+        """
+        if isinstance(other, Key):
+            return self.type() == other.type() and self.data() == other.data()
+        else:
+            return NotImplemented
+
+    def __repr__(self) -> str:
+        """
+        Return a pretty representation of this object.
+        """
+        if self.type() == "EC":
+            data = self.data()
+            name = data["curve"].decode("utf-8")
+
+            if self.isPublic():
+                out = f"<Elliptic Curve Public Key ({name[-3:]} bits)"
+            else:
+                out = f"<Elliptic Curve Private Key ({name[-3:]} bits)"
+
+            for k, v in sorted(data.items()):
+                if k == "curve":
+                    out += f"\ncurve:\n\t{name}"
+                else:
+                    out += f"\n{k}:\n\t{v}"
+
+            return out + ">\n"
+        else:
+            lines = [
+                "<%s %s (%s bits)"
+                % (
+                    nativeString(self.type()),
+                    self.isPublic() and "Public Key" or "Private Key",
+                    self.size(),
+                )
+            ]
+            for k, v in sorted(self.data().items()):
+                lines.append(f"attr {k}:")
+                by = v if self.type() == "Ed25519" else common.MP(v)[4:]
+                while by:
+                    m = by[:15]
+                    by = by[15:]
+                    o = ""
+                    for c in iterbytes(m):
+                        o = o + f"{ord(c):02x}:"
+                    if len(m) < 15:
+                        o = o[:-1]
+                    lines.append("\t" + o)
+            lines[-1] = lines[-1] + ">"
+            return "\n".join(lines)
+
+    def isPublic(self):
+        """
+        Check if this instance is a public key.
+
+        @return: C{True} if this is a public key.
+        """
+        return isinstance(
+            self._keyObject,
+            (
+                rsa.RSAPublicKey,
+                dsa.DSAPublicKey,
+                ec.EllipticCurvePublicKey,
+                ed25519.Ed25519PublicKey,
+            ),
+        )
+
+    def public(self):
+        """
+        Returns a version of this key containing only the public key data.
+        If this is a public key, this may or may not be the same object
+        as self.
+
+        @rtype: L{Key}
+        @return: A public key.
+        """
+        if self.isPublic():
+            return self
+        else:
+            return Key(self._keyObject.public_key())
+
+    def fingerprint(self, format=FingerprintFormats.MD5_HEX):
+        """
+        The fingerprint of a public key consists of the output of the
+        message-digest algorithm in the specified format.
+        Supported formats include L{FingerprintFormats.MD5_HEX} and
+        L{FingerprintFormats.SHA256_BASE64}
+
+        The input to the algorithm is the public key data as specified by [RFC4253].
+
+        The output of sha256[RFC4634] algorithm is presented to the
+        user in the form of base64 encoded sha256 hashes.
+        Example: C{US5jTUa0kgX5ZxdqaGF0yGRu8EgKXHNmoT8jHKo1StM=}
+
+        The output of the MD5[RFC1321](default) algorithm is presented to the user as
+        a sequence of 16 octets printed as hexadecimal with lowercase letters
+        and separated by colons.
+        Example: C{c1:b1:30:29:d7:b8:de:6c:97:77:10:d7:46:41:63:87}
+
+        @param format: Format for fingerprint generation. Consists
+            hash function and representation format.
+            Default is L{FingerprintFormats.MD5_HEX}
+
+        @since: 8.2
+
+        @return: the user presentation of this L{Key}'s fingerprint, as a
+        string.
+
+        @rtype: L{str}
+        """
+        if format is FingerprintFormats.SHA256_BASE64:
+            return nativeString(b64encode(sha256(self.blob()).digest()))
+        elif format is FingerprintFormats.MD5_HEX:
+            return nativeString(
+                b":".join(
+                    [binascii.hexlify(x) for x in iterbytes(md5(self.blob()).digest())]
+                )
+            )
+        else:
+            raise BadFingerPrintFormat(f"Unsupported fingerprint format: {format}")
+
+    def type(self) -> Literal["RSA", "DSA", "EC", "Ed25519"]:
+        """
+        Return the type of the object we wrap.  Currently this can only be
+        'RSA', 'DSA', 'EC', or 'Ed25519'.
+
+        @rtype: L{str}
+        @raises RuntimeError: If the object type is unknown.
+        """
+        if isinstance(self._keyObject, (rsa.RSAPublicKey, rsa.RSAPrivateKey)):
+            return "RSA"
+        elif isinstance(self._keyObject, (dsa.DSAPublicKey, dsa.DSAPrivateKey)):
+            return "DSA"
+        elif isinstance(
+            self._keyObject, (ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey)
+        ):
+            return "EC"
+        elif isinstance(
+            self._keyObject, (ed25519.Ed25519PublicKey, ed25519.Ed25519PrivateKey)
+        ):
+            return "Ed25519"
+        else:
+            raise RuntimeError(f"unknown type of object: {self._keyObject!r}")
+
+    def sshType(self):
+        """
+        Get the type of the object we wrap as defined in the SSH protocol,
+        defined in RFC 4253, Section 6.6 and RFC 8332, section 4 (this is a
+        public key format name, not a public key algorithm name). Currently
+        this can only be b'ssh-rsa', b'ssh-dss', b'ecdsa-sha2-[identifier]'
+        or b'ssh-ed25519'.
+
+        identifier is the standard NIST curve name
+
+        @return: The key type format.
+        @rtype: L{bytes}
+        """
+        if self.type() == "EC":
+            return (
+                b"ecdsa-sha2-" + _secToNist[self._keyObject.curve.name.encode("ascii")]
+            )
+        else:
+            return {
+                "RSA": b"ssh-rsa",
+                "DSA": b"ssh-dss",
+                "Ed25519": b"ssh-ed25519",
+            }[self.type()]
+
+    def supportedSignatureAlgorithms(self):
+        """
+        Get the public key signature algorithms supported by this key.
+
+        @return: A list of supported public key signature algorithm names.
+        @rtype: L{list} of L{bytes}
+        """
+        if self.type() == "RSA":
+            return [b"rsa-sha2-512", b"rsa-sha2-256", b"ssh-rsa"]
+        else:
+            return [self.sshType()]
+
+    def _getHashAlgorithm(self, signatureType):
+        """
+        Return a hash algorithm for this key type given an SSH signature
+        algorithm name, or L{None} if no such hash algorithm is defined for
+        this key type.
+        """
+        if self.type() == "EC":
+            # Hash algorithm depends on key size
+            if signatureType == self.sshType():
+                keySize = self.size()
+                if keySize <= 256:
+                    return hashes.SHA256()
+                elif keySize <= 384:
+                    return hashes.SHA384()
+                else:
+                    return hashes.SHA512()
+            else:
+                return None
+        else:
+            return {
+                ("RSA", b"ssh-rsa"): hashes.SHA1(),
+                ("RSA", b"rsa-sha2-256"): hashes.SHA256(),
+                ("RSA", b"rsa-sha2-512"): hashes.SHA512(),
+                ("DSA", b"ssh-dss"): hashes.SHA1(),
+                ("Ed25519", b"ssh-ed25519"): hashes.SHA512(),
+            }.get((self.type(), signatureType))
+
+    def size(self):
+        """
+        Return the size of the object we wrap.
+
+        @return: The size of the key.
+        @rtype: L{int}
+        """
+        if self._keyObject is None:
+            return 0
+        elif self.type() == "EC":
+            return self._keyObject.curve.key_size
+        elif self.type() == "Ed25519":
+            return 256
+        return self._keyObject.key_size
+
+    def data(self) -> dict[str, Any]:
+        """
+        Return the values of the public key as a dictionary.
+
+        @rtype: L{dict}
+        """
+        if isinstance(self._keyObject, rsa.RSAPublicKey):
+            rsa_pub_numbers = self._keyObject.public_numbers()
+            return {
+                "n": rsa_pub_numbers.n,
+                "e": rsa_pub_numbers.e,
+            }
+        elif isinstance(self._keyObject, rsa.RSAPrivateKey):
+            rsa_priv_numbers = self._keyObject.private_numbers()
+            return {
+                "n": rsa_priv_numbers.public_numbers.n,
+                "e": rsa_priv_numbers.public_numbers.e,
+                "d": rsa_priv_numbers.d,
+                "p": rsa_priv_numbers.p,
+                "q": rsa_priv_numbers.q,
+                # Use a trick: iqmp is q^-1 % p, u is p^-1 % q
+                "u": rsa.rsa_crt_iqmp(rsa_priv_numbers.q, rsa_priv_numbers.p),
+            }
+        elif isinstance(self._keyObject, dsa.DSAPublicKey):
+            dsa_pub_numbers = self._keyObject.public_numbers()
+            return {
+                "y": dsa_pub_numbers.y,
+                "g": dsa_pub_numbers.parameter_numbers.g,
+                "p": dsa_pub_numbers.parameter_numbers.p,
+                "q": dsa_pub_numbers.parameter_numbers.q,
+            }
+        elif isinstance(self._keyObject, dsa.DSAPrivateKey):
+            dsa_priv_numbers = self._keyObject.private_numbers()
+            return {
+                "x": dsa_priv_numbers.x,
+                "y": dsa_priv_numbers.public_numbers.y,
+                "g": dsa_priv_numbers.public_numbers.parameter_numbers.g,
+                "p": dsa_priv_numbers.public_numbers.parameter_numbers.p,
+                "q": dsa_priv_numbers.public_numbers.parameter_numbers.q,
+            }
+        elif isinstance(self._keyObject, ec.EllipticCurvePublicKey):
+            ec_pub_numbers = self._keyObject.public_numbers()
+            return {
+                "x": ec_pub_numbers.x,
+                "y": ec_pub_numbers.y,
+                "curve": self.sshType(),
+            }
+        elif isinstance(self._keyObject, ec.EllipticCurvePrivateKey):
+            ec_priv_numbers = self._keyObject.private_numbers()
+            return {
+                "x": ec_priv_numbers.public_numbers.x,
+                "y": ec_priv_numbers.public_numbers.y,
+                "privateValue": ec_priv_numbers.private_value,
+                "curve": self.sshType(),
+            }
+        elif isinstance(self._keyObject, ed25519.Ed25519PublicKey):
+            return {
+                "a": self._keyObject.public_bytes(
+                    serialization.Encoding.Raw, serialization.PublicFormat.Raw
+                ),
+            }
+        elif isinstance(self._keyObject, ed25519.Ed25519PrivateKey):
+            return {
+                "a": self._keyObject.public_key().public_bytes(
+                    serialization.Encoding.Raw, serialization.PublicFormat.Raw
+                ),
+                "k": self._keyObject.private_bytes(
+                    serialization.Encoding.Raw,
+                    serialization.PrivateFormat.Raw,
+                    serialization.NoEncryption(),
+                ),
+            }
+
+        else:
+            raise RuntimeError(f"Unexpected key type: {self._keyObject}")
+
+    def blob(self):
+        """
+        Return the public key blob for this key. The blob is the
+        over-the-wire format for public keys.
+
+        SECSH-TRANS RFC 4253 Section 6.6.
+
+        RSA keys::
+            string 'ssh-rsa'
+            integer e
+            integer n
+
+        DSA keys::
+            string 'ssh-dss'
+            integer p
+            integer q
+            integer g
+            integer y
+
+        EC keys::
+            string 'ecdsa-sha2-[identifier]'
+            integer x
+            integer y
+
+            identifier is the standard NIST curve name
+
+        Ed25519 keys::
+            string 'ssh-ed25519'
+            string a
+
+        @rtype: L{bytes}
+        """
+        type = self.type()
+        data = self.data()
+        if type == "RSA":
+            return common.NS(b"ssh-rsa") + common.MP(data["e"]) + common.MP(data["n"])
+        elif type == "DSA":
+            return (
+                common.NS(b"ssh-dss")
+                + common.MP(data["p"])
+                + common.MP(data["q"])
+                + common.MP(data["g"])
+                + common.MP(data["y"])
+            )
+        elif type == "EC":
+            byteLength = (self._keyObject.curve.key_size + 7) // 8
+            return (
+                common.NS(data["curve"])
+                + common.NS(data["curve"][-8:])
+                + common.NS(
+                    b"\x04"
+                    + utils.int_to_bytes(data["x"], byteLength)
+                    + utils.int_to_bytes(data["y"], byteLength)
+                )
+            )
+        elif type == "Ed25519":
+            return common.NS(b"ssh-ed25519") + common.NS(data["a"])
+        else:
+            raise BadKeyError(f"unknown key type: {type}")
+
+    def privateBlob(self):
+        """
+        Return the private key blob for this key. The blob is the
+        over-the-wire format for private keys:
+
+        Specification in OpenSSH PROTOCOL.agent
+
+        RSA keys::
+
+            string 'ssh-rsa'
+            integer n
+            integer e
+            integer d
+            integer u
+            integer p
+            integer q
+
+        DSA keys::
+
+            string 'ssh-dss'
+            integer p
+            integer q
+            integer g
+            integer y
+            integer x
+
+        EC keys::
+
+            string 'ecdsa-sha2-[identifier]'
+            integer x
+            integer y
+            integer privateValue
+
+            identifier is the NIST standard curve name.
+
+        Ed25519 keys::
+
+            string 'ssh-ed25519'
+            string a
+            string k || a
+        """
+        type = self.type()
+        data = self.data()
+        if type == "RSA":
+            iqmp = rsa.rsa_crt_iqmp(data["p"], data["q"])
+            return (
+                common.NS(b"ssh-rsa")
+                + common.MP(data["n"])
+                + common.MP(data["e"])
+                + common.MP(data["d"])
+                + common.MP(iqmp)
+                + common.MP(data["p"])
+                + common.MP(data["q"])
+            )
+        elif type == "DSA":
+            return (
+                common.NS(b"ssh-dss")
+                + common.MP(data["p"])
+                + common.MP(data["q"])
+                + common.MP(data["g"])
+                + common.MP(data["y"])
+                + common.MP(data["x"])
+            )
+        elif type == "EC":
+            encPub = self._keyObject.public_key().public_bytes(
+                serialization.Encoding.X962,
+                serialization.PublicFormat.UncompressedPoint,
+            )
+            return (
+                common.NS(data["curve"])
+                + common.NS(data["curve"][-8:])
+                + common.NS(encPub)
+                + common.MP(data["privateValue"])
+            )
+        elif type == "Ed25519":
+            return (
+                common.NS(b"ssh-ed25519")
+                + common.NS(data["a"])
+                + common.NS(data["k"] + data["a"])
+            )
+        else:
+            raise BadKeyError(f"unknown key type: {type}")
+
+    @_mutuallyExclusiveArguments(
+        [
+            ["extra", "comment"],
+            ["extra", "passphrase"],
+        ]
+    )
+    def toString(self, type, extra=None, subtype=None, comment=None, passphrase=None):
+        """
+        Create a string representation of this key.  If the key is a private
+        key and you want the representation of its public key, use
+        C{key.public().toString()}.  type maps to a _toString_* method.
+
+        @param type: The type of string to emit.  Currently supported values
+            are C{'OPENSSH'}, C{'LSH'}, and C{'AGENTV3'}.
+        @type type: L{str}
+
+        @param extra: Any extra data supported by the selected format which
+            is not part of the key itself.  For public OpenSSH keys, this is
+            a comment.  For private OpenSSH keys, this is a passphrase to
+            encrypt with.  (Deprecated since Twisted 20.3.0; use C{comment}
+            or C{passphrase} as appropriate instead.)
+        @type extra: L{bytes} or L{unicode} or L{None}
+
+        @param subtype: A subtype of the requested C{type} to emit.  Only
+            supported for private OpenSSH keys, for which the currently
+            supported subtypes are C{'PEM'} and C{'v1'}.  If not given, an
+            appropriate default is used.
+        @type subtype: L{str} or L{None}
+
+        @param comment: A comment to include with the key.  Only supported
+            for OpenSSH keys.
+
+            Present since Twisted 20.3.0.
+
+        @type comment: L{bytes} or L{unicode} or L{None}
+
+        @param passphrase: A passphrase to encrypt the key with.  Only
+            supported for private OpenSSH keys.
+
+            Present since Twisted 20.3.0.
+
+        @type passphrase: L{bytes} or L{unicode} or L{None}
+
+        @rtype: L{bytes}
+        """
+        if extra is not None:
+            # Compatibility with old parameter format.
+            warnings.warn(
+                "The 'extra' argument to "
+                "twisted.conch.ssh.keys.Key.toString was deprecated in "
+                "Twisted 20.3.0; use 'comment' or 'passphrase' instead.",
+                DeprecationWarning,
+                stacklevel=3,
+            )
+            if self.isPublic():
+                comment = extra
+            else:
+                passphrase = extra
+        if isinstance(comment, str):
+            comment = comment.encode("utf-8")
+        passphrase = _normalizePassphrase(passphrase)
+        method = getattr(self, f"_toString_{type.upper()}", None)
+        if method is None:
+            raise BadKeyError(f"unknown key type: {type}")
+        return method(subtype=subtype, comment=comment, passphrase=passphrase)
+
+    def _toPublicOpenSSH(self, comment=None):
+        """
+        Return a public OpenSSH key string.
+
+        See _fromString_PUBLIC_OPENSSH for the string format.
+
+        @type comment: L{bytes} or L{None}
+        @param comment: A comment to include with the key, or L{None} to
+        omit the comment.
+        """
+        if self.type() == "EC":
+            if not comment:
+                comment = b""
+            return (
+                self._keyObject.public_bytes(
+                    serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH
+                )
+                + b" "
+                + comment
+            ).strip()
+
+        b64Data = encodebytes(self.blob()).replace(b"\n", b"")
+        if not comment:
+            comment = b""
+        return (self.sshType() + b" " + b64Data + b" " + comment).strip()
+
+    def _toPrivateOpenSSH_v1(self, comment=None, passphrase=None):
+        """
+        Return a private OpenSSH key string, in the "openssh-key-v1" format
+        introduced in OpenSSH 6.5.
+
+        See _fromPrivateOpenSSH_v1 for the string format.
+
+        @type passphrase: L{bytes} or L{None}
+        @param passphrase: The passphrase to encrypt the key with, or L{None}
+        if it is not encrypted.
+        """
+        if passphrase:
+            # For now we just hardcode the cipher to the one used by
+            # OpenSSH.  We could make this configurable later if it's
+            # needed.
+            cipher = algorithms.AES
+            cipherName = b"aes256-ctr"
+            kdfName = b"bcrypt"
+            blockSize = cipher.block_size // 8
+            keySize = 32
+            ivSize = blockSize
+            salt = randbytes.secureRandom(ivSize)
+            rounds = 100
+            kdfOptions = common.NS(salt) + struct.pack("!L", rounds)
+        else:
+            cipherName = b"none"
+            kdfName = b"none"
+            blockSize = 8
+            kdfOptions = b""
+        check = randbytes.secureRandom(4)
+        privKeyList = check + check + self.privateBlob() + common.NS(comment or b"")
+        padByte = 0
+        while len(privKeyList) % blockSize:
+            padByte += 1
+            privKeyList += bytes((padByte & 0xFF,))
+        if passphrase:
+            encKey = bcrypt.kdf(passphrase, salt, keySize + ivSize, 100)
+            encryptor = Cipher(
+                cipher(encKey[:keySize]),
+                modes.CTR(encKey[keySize : keySize + ivSize]),
+                backend=default_backend(),
+            ).encryptor()
+            encPrivKeyList = encryptor.update(privKeyList) + encryptor.finalize()
+        else:
+            encPrivKeyList = privKeyList
+        blob = (
+            b"openssh-key-v1\0"
+            + common.NS(cipherName)
+            + common.NS(kdfName)
+            + common.NS(kdfOptions)
+            + struct.pack("!L", 1)
+            + common.NS(self.blob())
+            + common.NS(encPrivKeyList)
+        )
+        b64Data = encodebytes(blob).replace(b"\n", b"")
+        lines = (
+            [b"-----BEGIN OPENSSH PRIVATE KEY-----"]
+            + [b64Data[i : i + 64] for i in range(0, len(b64Data), 64)]
+            + [b"-----END OPENSSH PRIVATE KEY-----"]
+        )
+        return b"\n".join(lines) + b"\n"
+
+    def _toPrivateOpenSSH_PEM(self, passphrase=None):
+        """
+        Return a private OpenSSH key string, in the old PEM-based format.
+
+        See _fromPrivateOpenSSH_PEM for the string format.
+
+        @type passphrase: L{bytes} or L{None}
+        @param passphrase: The passphrase to encrypt the key with, or L{None}
+        if it is not encrypted.
+        """
+        if not passphrase:
+            # unencrypted private key
+            encryptor = serialization.NoEncryption()
+        else:
+            encryptor = serialization.BestAvailableEncryption(passphrase)
+        if self.type() != "Ed25519":
+            return self._keyObject.private_bytes(
+                serialization.Encoding.PEM,
+                serialization.PrivateFormat.TraditionalOpenSSL,
+                encryptor,
+            )
+        else:
+            # TODO: why not just support serialization here
+            assert self.type() == "Ed25519"
+            raise ValueError(
+                "cannot serialize Ed25519 key to OpenSSH PEM format; use v1 " "instead"
+            )
+
+    def _toString_OPENSSH(self, subtype=None, comment=None, passphrase=None):
+        """
+        Return a public or private OpenSSH string.  See
+        L{_fromString_PUBLIC_OPENSSH} and L{_fromPrivateOpenSSH_PEM} for the
+        string formats.
+
+        @param subtype: A subtype to emit.  Only supported for private keys,
+            for which the currently supported subtypes are C{'PEM'} and C{'v1'}.
+            If not given, an appropriate default is used.
+        @type subtype: L{str} or L{None}
+
+        @param comment: Comment for a public key.
+        @type comment: L{bytes}
+
+        @param passphrase: Passphrase for a private key.
+        @type passphrase: L{bytes}
+
+        @rtype: L{bytes}
+        """
+        if self.isPublic():
+            return self._toPublicOpenSSH(comment=comment)
+        # No pre-v1 format is defined for Ed25519 keys.
+        elif subtype == "v1" or (subtype is None and self.type() == "Ed25519"):
+            return self._toPrivateOpenSSH_v1(comment=comment, passphrase=passphrase)
+        elif subtype is None or subtype == "PEM":
+            return self._toPrivateOpenSSH_PEM(passphrase=passphrase)
+        else:
+            raise ValueError(f"unknown subtype {subtype}")
+
+    def _toString_LSH(self, **kwargs):
+        """
+        Return a public or private LSH key.  See _fromString_PUBLIC_LSH and
+        _fromString_PRIVATE_LSH for the key formats.
+
+        @rtype: L{bytes}
+        """
+        data = self.data()
+        type = self.type()
+        if self.isPublic():
+            if type == "RSA":
+                keyData = sexpy.pack(
+                    [
+                        [
+                            b"public-key",
+                            [
+                                b"rsa-pkcs1-sha1",
+                                [b"n", common.MP(data["n"])[4:]],
+                                [b"e", common.MP(data["e"])[4:]],
+                            ],
+                        ]
+                    ]
+                )
+            elif type == "DSA":
+                keyData = sexpy.pack(
+                    [
+                        [
+                            b"public-key",
+                            [
+                                b"dsa",
+                                [b"p", common.MP(data["p"])[4:]],
+                                [b"q", common.MP(data["q"])[4:]],
+                                [b"g", common.MP(data["g"])[4:]],
+                                [b"y", common.MP(data["y"])[4:]],
+                            ],
+                        ]
+                    ]
+                )
+            else:
+                raise BadKeyError(f"unknown key type {type}")
+            return b"{" + encodebytes(keyData).replace(b"\n", b"") + b"}"
+        else:
+            if type == "RSA":
+                p, q = data["p"], data["q"]
+                iqmp = rsa.rsa_crt_iqmp(p, q)
+                return sexpy.pack(
+                    [
+                        [
+                            b"private-key",
+                            [
+                                b"rsa-pkcs1",
+                                [b"n", common.MP(data["n"])[4:]],
+                                [b"e", common.MP(data["e"])[4:]],
+                                [b"d", common.MP(data["d"])[4:]],
+                                [b"p", common.MP(q)[4:]],
+                                [b"q", common.MP(p)[4:]],
+                                [b"a", common.MP(data["d"] % (q - 1))[4:]],
+                                [b"b", common.MP(data["d"] % (p - 1))[4:]],
+                                [b"c", common.MP(iqmp)[4:]],
+                            ],
+                        ]
+                    ]
+                )
+            elif type == "DSA":
+                return sexpy.pack(
+                    [
+                        [
+                            b"private-key",
+                            [
+                                b"dsa",
+                                [b"p", common.MP(data["p"])[4:]],
+                                [b"q", common.MP(data["q"])[4:]],
+                                [b"g", common.MP(data["g"])[4:]],
+                                [b"y", common.MP(data["y"])[4:]],
+                                [b"x", common.MP(data["x"])[4:]],
+                            ],
+                        ]
+                    ]
+                )
+            else:
+                raise BadKeyError(f"unknown key type {type}'")
+
+    def _toString_AGENTV3(self, **kwargs):
+        """
+        Return a private Secure Shell Agent v3 key.  See
+        _fromString_AGENTV3 for the key format.
+
+        @rtype: L{bytes}
+        """
+        data = self.data()
+        if not self.isPublic():
+            if self.type() == "RSA":
+                values = (
+                    data["e"],
+                    data["d"],
+                    data["n"],
+                    data["u"],
+                    data["p"],
+                    data["q"],
+                )
+            elif self.type() == "DSA":
+                values = (data["p"], data["q"], data["g"], data["y"], data["x"])
+            return common.NS(self.sshType()) + b"".join(map(common.MP, values))
+
+    def sign(self, data, signatureType=None):
+        """
+        Sign some data with this key.
+
+        SECSH-TRANS RFC 4253 Section 6.6.
+
+        @type data: L{bytes}
+        @param data: The data to sign.
+
+        @type signatureType: L{bytes}
+        @param signatureType: The SSH public key algorithm name to sign this
+        data with, or L{None} to use a reasonable default for the key.
+
+        @rtype: L{bytes}
+        @return: A signature for the given data.
+        """
+        keyType = self.type()
+        if signatureType is None:
+            # Use the SSH public key type name by default, since for all
+            # current key types this can also be used as a public key
+            # algorithm name.  (This exists for compatibility; new code
+            # should explicitly specify a public key algorithm name.)
+            signatureType = self.sshType()
+
+        hashAlgorithm = self._getHashAlgorithm(signatureType)
+        if hashAlgorithm is None:
+            raise BadSignatureAlgorithmError(
+                f"public key signature algorithm {signatureType} is not "
+                f"defined for {keyType} keys"
+            )
+
+        if keyType == "RSA":
+            sig = self._keyObject.sign(data, padding.PKCS1v15(), hashAlgorithm)
+            ret = common.NS(sig)
+
+        elif keyType == "DSA":
+            sig = self._keyObject.sign(data, hashAlgorithm)
+            (r, s) = decode_dss_signature(sig)
+            # SSH insists that the DSS signature blob be two 160-bit integers
+            # concatenated together. The sig[0], [1] numbers from obj.sign
+            # are just numbers, and could be any length from 0 to 160 bits.
+            # Make sure they are padded out to 160 bits (20 bytes each)
+            ret = common.NS(int_to_bytes(r, 20) + int_to_bytes(s, 20))
+
+        elif keyType == "EC":  # Pragma: no branch
+            signature = self._keyObject.sign(data, ec.ECDSA(hashAlgorithm))
+            (r, s) = decode_dss_signature(signature)
+
+            rb = int_to_bytes(r)
+            sb = int_to_bytes(s)
+
+            # Int_to_bytes returns rb[0] as a str in python2
+            # and an as int in python3
+            if type(rb[0]) is str:
+                rcomp = ord(rb[0])
+            else:
+                rcomp = rb[0]
+
+            # If the MSB is set, prepend a null byte for correct formatting.
+            if rcomp & 0x80:
+                rb = b"\x00" + rb
+
+            if type(sb[0]) is str:
+                scomp = ord(sb[0])
+            else:
+                scomp = sb[0]
+
+            if scomp & 0x80:
+                sb = b"\x00" + sb
+
+            ret = common.NS(common.NS(rb) + common.NS(sb))
+
+        elif keyType == "Ed25519":
+            ret = common.NS(self._keyObject.sign(data))
+        return common.NS(signatureType) + ret
+
+    def verify(self, signature, data):
+        """
+        Verify a signature using this key.
+
+        @type signature: L{bytes}
+        @param signature: The signature to verify.
+
+        @type data: L{bytes}
+        @param data: The signed data.
+
+        @rtype: L{bool}
+        @return: C{True} if the signature is valid.
+        """
+        if len(signature) == 40:
+            # DSA key with no padding
+            signatureType, signature = b"ssh-dss", common.NS(signature)
+        else:
+            signatureType, signature = common.getNS(signature)
+
+        hashAlgorithm = self._getHashAlgorithm(signatureType)
+        if hashAlgorithm is None:
+            return False
+
+        keyType = self.type()
+        if keyType == "RSA":
+            k = self._keyObject
+            if not self.isPublic():
+                k = k.public_key()
+            args = (
+                common.getNS(signature)[0],
+                data,
+                padding.PKCS1v15(),
+                hashAlgorithm,
+            )
+        elif keyType == "DSA":
+            concatenatedSignature = common.getNS(signature)[0]
+            r = int.from_bytes(concatenatedSignature[:20], "big")
+            s = int.from_bytes(concatenatedSignature[20:], "big")
+            signature = encode_dss_signature(r, s)
+            k = self._keyObject
+            if not self.isPublic():
+                k = k.public_key()
+            args = (signature, data, hashAlgorithm)
+
+        elif keyType == "EC":  # Pragma: no branch
+            concatenatedSignature = common.getNS(signature)[0]
+            rstr, sstr, rest = common.getNS(concatenatedSignature, 2)
+            r = int.from_bytes(rstr, "big")
+            s = int.from_bytes(sstr, "big")
+            signature = encode_dss_signature(r, s)
+
+            k = self._keyObject
+            if not self.isPublic():
+                k = k.public_key()
+
+            args = (signature, data, ec.ECDSA(hashAlgorithm))
+
+        elif keyType == "Ed25519":
+            k = self._keyObject
+            if not self.isPublic():
+                k = k.public_key()
+            args = (common.getNS(signature)[0], data)
+
+        try:
+            k.verify(*args)
+        except InvalidSignature:
+            return False
+        else:
+            return True
+
+
+def _getPersistentRSAKey(location, keySize=4096):
+    """
+    This function returns a persistent L{Key}.
+
+    The key is loaded from a PEM file in C{location}. If it does not exist, a
+    key with the key size of C{keySize} is generated and saved.
+
+    @param location: Where the key is stored.
+    @type location: L{twisted.python.filepath.FilePath}
+
+    @param keySize: The size of the key, if it needs to be generated.
+    @type keySize: L{int}
+
+    @returns: A persistent key.
+    @rtype: L{Key}
+    """
+    location.parent().makedirs(ignoreExistingDirectory=True)
+
+    # If it doesn't exist, we want to generate a new key and save it
+    if not location.exists():
+        privateKey = rsa.generate_private_key(
+            public_exponent=65537, key_size=keySize, backend=default_backend()
+        )
+
+        pem = privateKey.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+
+        location.setContent(pem)
+
+    # By this point (save any hilarious race conditions) we should have a
+    # working PEM file. Load it!
+    # (Future archaeological readers: I chose not to short circuit above,
+    # because then there's two exit paths to this code!)
+    with location.open("rb") as keyFile:
+        privateKey = serialization.load_pem_private_key(
+            keyFile.read(), password=None, backend=default_backend()
+        )
+        return Key(privateKey)
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/service.py b/contrib/python/Twisted/py3/twisted/conch/ssh/service.py
new file mode 100644
index 00000000000..7d0d41c4aed
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/service.py
@@ -0,0 +1,56 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+The parent class for all the SSH services.  Currently implemented services
+are ssh-userauth and ssh-connection.
+
+Maintainer: Paul Swartz
+"""
+
+from typing import Dict
+
+from twisted.logger import Logger
+
+
+class SSHService:
+    # this is the ssh name for the service:
+    name: bytes = None  # type:ignore[assignment]
+
+    protocolMessages: Dict[int, str] = {}  # map #'s -> protocol names
+    transport = None  # gets set later
+
+    _log = Logger()
+
+    def serviceStarted(self):
+        """
+        called when the service is active on the transport.
+        """
+
+    def serviceStopped(self):
+        """
+        called when the service is stopped, either by the connection ending
+        or by another service being started
+        """
+
+    def logPrefix(self):
+        return "SSHService {!r} on {}".format(
+            self.name, self.transport.transport.logPrefix()
+        )
+
+    def packetReceived(self, messageNum, packet):
+        """
+        called when we receive a packet on the transport
+        """
+        # print self.protocolMessages
+        if messageNum in self.protocolMessages:
+            messageType = self.protocolMessages[messageNum]
+            f = getattr(self, "ssh_%s" % messageType[4:], None)
+            if f is not None:
+                return f(packet)
+        self._log.info(
+            "couldn't handle {messageNum} {packet!r}",
+            messageNum=messageNum,
+            packet=packet,
+        )
+        self.transport.sendUnimplemented()
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/session.py b/contrib/python/Twisted/py3/twisted/conch/ssh/session.py
new file mode 100644
index 00000000000..6bea03b81e8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/session.py
@@ -0,0 +1,440 @@
+# -*- test-case-name: twisted.conch.test.test_session -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module contains the implementation of SSHSession, which (by default)
+allows access to a shell and a python interpreter over SSH.
+
+Maintainer: Paul Swartz
+"""
+
+
+import os
+import signal
+import struct
+import sys
+
+from zope.interface import implementer
+
+from twisted.conch.interfaces import (
+    EnvironmentVariableNotPermitted,
+    ISession,
+    ISessionSetEnv,
+)
+from twisted.conch.ssh import channel, common, connection
+from twisted.internet import interfaces, protocol
+from twisted.logger import Logger
+from twisted.python.compat import networkString
+
+log = Logger()
+
+
+class SSHSession(channel.SSHChannel):
+    """
+    A generalized implementation of an SSH session.
+
+    See RFC 4254, section 6.
+
+    The precise implementation of the various operations that the remote end
+    can send is left up to the avatar, usually via an adapter to an
+    interface such as L{ISession}.
+
+    @ivar buf: a buffer for data received before making a connection to a
+        client.
+    @type buf: L{bytes}
+    @ivar client: a protocol for communication with a shell, an application
+        program, or a subsystem (see RFC 4254, section 6.5).
+    @type client: L{SSHSessionProcessProtocol}
+    @ivar session: an object providing concrete implementations of session
+        operations.
+    @type session: L{ISession}
+    """
+
+    name = b"session"
+
+    def __init__(self, *args, **kw):
+        channel.SSHChannel.__init__(self, *args, **kw)
+        self.buf = b""
+        self.client = None
+        self.session = None
+
+    def request_subsystem(self, data):
+        subsystem, ignored = common.getNS(data)
+        log.info('Asking for subsystem "{subsystem}"', subsystem=subsystem)
+        client = self.avatar.lookupSubsystem(subsystem, data)
+        if client:
+            pp = SSHSessionProcessProtocol(self)
+            proto = wrapProcessProtocol(pp)
+            client.makeConnection(proto)
+            pp.makeConnection(wrapProtocol(client))
+            self.client = pp
+            return 1
+        else:
+            log.error("Failed to get subsystem")
+            return 0
+
+    def request_shell(self, data):
+        log.info("Getting shell")
+        if not self.session:
+            self.session = ISession(self.avatar)
+        try:
+            pp = SSHSessionProcessProtocol(self)
+            self.session.openShell(pp)
+        except Exception:
+            log.failure("Error getting shell")
+            return 0
+        else:
+            self.client = pp
+            return 1
+
+    def request_exec(self, data):
+        if not self.session:
+            self.session = ISession(self.avatar)
+        f, data = common.getNS(data)
+        log.info('Executing command "{f}"', f=f)
+        try:
+            pp = SSHSessionProcessProtocol(self)
+            self.session.execCommand(pp, f)
+        except Exception:
+            log.failure('Error executing command "{f}"', f=f)
+            return 0
+        else:
+            self.client = pp
+            return 1
+
+    def request_pty_req(self, data):
+        if not self.session:
+            self.session = ISession(self.avatar)
+        term, windowSize, modes = parseRequest_pty_req(data)
+        log.info(
+            "Handling pty request: {term!r} {windowSize!r}",
+            term=term,
+            windowSize=windowSize,
+        )
+        try:
+            self.session.getPty(term, windowSize, modes)
+        except Exception:
+            log.failure("Error handling pty request")
+            return 0
+        else:
+            return 1
+
+    def request_env(self, data):
+        """
+        Process a request to pass an environment variable.
+
+        @param data: The environment variable name and value, each encoded
+            as an SSH protocol string and concatenated.
+        @type data: L{bytes}
+        @return: A true value if the request to pass this environment
+            variable was accepted, otherwise a false value.
+        """
+        if not self.session:
+            self.session = ISession(self.avatar)
+        if not ISessionSetEnv.providedBy(self.session):
+            return 0
+        name, value, data = common.getNS(data, 2)
+        try:
+            self.session.setEnv(name, value)
+        except EnvironmentVariableNotPermitted:
+            return 0
+        except Exception:
+            log.failure("Error setting environment variable {name}", name=name)
+            return 0
+        else:
+            return 1
+
+    def request_window_change(self, data):
+        if not self.session:
+            self.session = ISession(self.avatar)
+        winSize = parseRequest_window_change(data)
+        try:
+            self.session.windowChanged(winSize)
+        except Exception:
+            log.failure("Error changing window size")
+            return 0
+        else:
+            return 1
+
+    def dataReceived(self, data):
+        if not self.client:
+            # self.conn.sendClose(self)
+            self.buf += data
+            return
+        self.client.transport.write(data)
+
+    def extReceived(self, dataType, data):
+        if dataType == connection.EXTENDED_DATA_STDERR:
+            if self.client and hasattr(self.client.transport, "writeErr"):
+                self.client.transport.writeErr(data)
+        else:
+            log.warn("Weird extended data: {dataType}", dataType=dataType)
+
+    def eofReceived(self):
+        # If we have a session, tell it that EOF has been received and
+        # expect it to send a close message (it may need to send other
+        # messages such as exit-status or exit-signal first).  If we don't
+        # have a session, then just send a close message directly.
+        if self.session:
+            self.session.eofReceived()
+        elif self.client:
+            self.conn.sendClose(self)
+
+    def closed(self):
+        if self.client and self.client.transport:
+            self.client.transport.loseConnection()
+        if self.session:
+            self.session.closed()
+
+    # def closeReceived(self):
+    #    self.loseConnection() # don't know what to do with this
+
+    def loseConnection(self):
+        if self.client:
+            self.client.transport.loseConnection()
+        channel.SSHChannel.loseConnection(self)
+
+
+class _ProtocolWrapper(protocol.ProcessProtocol):
+    """
+    This class wraps a L{Protocol} instance in a L{ProcessProtocol} instance.
+    """
+
+    def __init__(self, proto):
+        self.proto = proto
+
+    def connectionMade(self):
+        self.proto.connectionMade()
+
+    def outReceived(self, data):
+        self.proto.dataReceived(data)
+
+    def processEnded(self, reason):
+        self.proto.connectionLost(reason)
+
+
+class _DummyTransport:
+    def __init__(self, proto):
+        self.proto = proto
+
+    def dataReceived(self, data):
+        self.proto.transport.write(data)
+
+    def write(self, data):
+        self.proto.dataReceived(data)
+
+    def writeSequence(self, seq):
+        self.write(b"".join(seq))
+
+    def loseConnection(self):
+        self.proto.connectionLost(protocol.connectionDone)
+
+
+def wrapProcessProtocol(inst):
+    if isinstance(inst, protocol.Protocol):
+        return _ProtocolWrapper(inst)
+    else:
+        return inst
+
+
+def wrapProtocol(proto):
+    return _DummyTransport(proto)
+
+
+# SUPPORTED_SIGNALS is a list of signals that every session channel is supposed
+# to accept.  See RFC 4254
+SUPPORTED_SIGNALS = [
+    "ABRT",
+    "ALRM",
+    "FPE",
+    "HUP",
+    "ILL",
+    "INT",
+    "KILL",
+    "PIPE",
+    "QUIT",
+    "SEGV",
+    "TERM",
+    "USR1",
+    "USR2",
+]
+
+
+@implementer(interfaces.ITransport)
+class SSHSessionProcessProtocol(protocol.ProcessProtocol):
+    """I am both an L{IProcessProtocol} and an L{ITransport}.
+
+    I am a transport to the remote endpoint and a process protocol to the
+    local subsystem.
+    """
+
+    # once initialized, a dictionary mapping signal values to strings
+    # that follow RFC 4254.
+    _signalValuesToNames = None
+
+    def __init__(self, session):
+        self.session = session
+        self.lostOutOrErrFlag = False
+
+    def connectionMade(self):
+        if self.session.buf:
+            self.transport.write(self.session.buf)
+            self.session.buf = None
+
+    def outReceived(self, data):
+        self.session.write(data)
+
+    def errReceived(self, err):
+        self.session.writeExtended(connection.EXTENDED_DATA_STDERR, err)
+
+    def outConnectionLost(self):
+        """
+        EOF should only be sent when both STDOUT and STDERR have been closed.
+        """
+        if self.lostOutOrErrFlag:
+            self.session.conn.sendEOF(self.session)
+        else:
+            self.lostOutOrErrFlag = True
+
+    def errConnectionLost(self):
+        """
+        See outConnectionLost().
+        """
+        self.outConnectionLost()
+
+    def connectionLost(self, reason=None):
+        self.session.loseConnection()
+
+    def _getSignalName(self, signum):
+        """
+        Get a signal name given a signal number.
+        """
+        if self._signalValuesToNames is None:
+            self._signalValuesToNames = {}
+            # make sure that the POSIX ones are the defaults
+            for signame in SUPPORTED_SIGNALS:
+                signame = "SIG" + signame
+                sigvalue = getattr(signal, signame, None)
+                if sigvalue is not None:
+                    self._signalValuesToNames[sigvalue] = signame
+            for k, v in signal.__dict__.items():
+                # Check for platform specific signals, ignoring Python specific
+                # SIG_DFL and SIG_IGN
+                if k.startswith("SIG") and not k.startswith("SIG_"):
+                    if v not in self._signalValuesToNames:
+                        self._signalValuesToNames[v] = k + "@" + sys.platform
+        return self._signalValuesToNames[signum]
+
+    def processEnded(self, reason=None):
+        """
+        When we are told the process ended, try to notify the other side about
+        how the process ended using the exit-signal or exit-status requests.
+        Also, close the channel.
+        """
+        if reason is not None:
+            err = reason.value
+            if err.signal is not None:
+                signame = self._getSignalName(err.signal)
+                if getattr(os, "WCOREDUMP", None) is not None and os.WCOREDUMP(
+                    err.status
+                ):
+                    log.info("exitSignal: {signame} (core dumped)", signame=signame)
+                    coreDumped = True
+                else:
+                    log.info("exitSignal: {}", signame=signame)
+                    coreDumped = False
+                self.session.conn.sendRequest(
+                    self.session,
+                    b"exit-signal",
+                    common.NS(networkString(signame[3:]))
+                    + (b"\1" if coreDumped else b"\0")
+                    + common.NS(b"")
+                    + common.NS(b""),
+                )
+            elif err.exitCode is not None:
+                log.info("exitCode: {exitCode!r}", exitCode=err.exitCode)
+                self.session.conn.sendRequest(
+                    self.session, b"exit-status", struct.pack(">L", err.exitCode)
+                )
+        self.session.loseConnection()
+
+    def getHost(self):
+        """
+        Return the host from my session's transport.
+        """
+        return self.session.conn.transport.getHost()
+
+    def getPeer(self):
+        """
+        Return the peer from my session's transport.
+        """
+        return self.session.conn.transport.getPeer()
+
+    def write(self, data):
+        self.session.write(data)
+
+    def writeSequence(self, seq):
+        self.session.write(b"".join(seq))
+
+    def loseConnection(self):
+        self.session.loseConnection()
+
+
+class SSHSessionClient(protocol.Protocol):
+    def dataReceived(self, data):
+        if self.transport:
+            self.transport.write(data)
+
+
+# methods factored out to make live easier on server writers
+def parseRequest_pty_req(data):
+    """Parse the data from a pty-req request into usable data.
+
+    @returns: a tuple of (terminal type, (rows, cols, xpixel, ypixel), modes)
+    """
+    term, rest = common.getNS(data)
+    cols, rows, xpixel, ypixel = struct.unpack(">4L", rest[:16])
+    modes, ignored = common.getNS(rest[16:])
+    winSize = (rows, cols, xpixel, ypixel)
+    modes = [
+        (ord(modes[i : i + 1]), struct.unpack(">L", modes[i + 1 : i + 5])[0])
+        for i in range(0, len(modes) - 1, 5)
+    ]
+    return term, winSize, modes
+
+
+def packRequest_pty_req(term, geometry, modes):
+    """
+    Pack a pty-req request so that it is suitable for sending.
+
+    NOTE: modes must be packed before being sent here.
+
+    @type geometry: L{tuple}
+    @param geometry: A tuple of (rows, columns, xpixel, ypixel)
+    """
+    (rows, cols, xpixel, ypixel) = geometry
+    termPacked = common.NS(term)
+    winSizePacked = struct.pack(">4L", cols, rows, xpixel, ypixel)
+    modesPacked = common.NS(modes)  # depend on the client packing modes
+    return termPacked + winSizePacked + modesPacked
+
+
+def parseRequest_window_change(data):
+    """Parse the data from a window-change request into usuable data.
+
+    @returns: a tuple of (rows, cols, xpixel, ypixel)
+    """
+    cols, rows, xpixel, ypixel = struct.unpack(">4L", data)
+    return rows, cols, xpixel, ypixel
+
+
+def packRequest_window_change(geometry):
+    """
+    Pack a window-change request so that it is suitable for sending.
+
+    @type geometry: L{tuple}
+    @param geometry: A tuple of (rows, columns, xpixel, ypixel)
+    """
+    (rows, cols, xpixel, ypixel) = geometry
+    return struct.pack(">4L", cols, rows, xpixel, ypixel)
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/sexpy.py b/contrib/python/Twisted/py3/twisted/conch/ssh/sexpy.py
new file mode 100644
index 00000000000..961611d364f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/sexpy.py
@@ -0,0 +1,40 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+def parse(s):
+    s = s.strip()
+    expr = []
+    while s:
+        if s[0:1] == b"(":
+            newSexp = []
+            if expr:
+                expr[-1].append(newSexp)
+            expr.append(newSexp)
+            s = s[1:]
+            continue
+        if s[0:1] == b")":
+            aList = expr.pop()
+            s = s[1:]
+            if not expr:
+                assert not s
+                return aList
+            continue
+        i = 0
+        while s[i : i + 1].isdigit():
+            i += 1
+        assert i
+        length = int(s[:i])
+        data = s[i + 1 : i + 1 + length]
+        expr[-1].append(data)
+        s = s[i + 1 + length :]
+    assert False, "this should not happen"
+
+
+def pack(sexp):
+    return b"".join(
+        b"(%b)" % (pack(o),)
+        if type(o) in (type(()), type([]))
+        else b"%d:%b" % (len(o), o)
+        for o in sexp
+    )
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/transport.py b/contrib/python/Twisted/py3/twisted/conch/ssh/transport.py
new file mode 100644
index 00000000000..d46f093dff9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/transport.py
@@ -0,0 +1,2266 @@
+# -*- test-case-name: twisted.conch.test.test_transport -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+The lowest level SSH protocol.  This handles the key negotiation, the
+encryption and the compression.  The transport layer is described in
+RFC 4253.
+
+Maintainer: Paul Swartz
+"""
+from __future__ import annotations
+
+import binascii
+import hmac
+import struct
+import types
+import zlib
+from hashlib import md5, sha1, sha256, sha384, sha512
+from typing import Any, Callable, Dict, Tuple, Union
+
+from cryptography.exceptions import UnsupportedAlgorithm
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import dh, ec, x25519
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from typing_extensions import Literal
+
+from twisted import __version__ as twisted_version
+from twisted.conch.ssh import _kex, address, keys
+from twisted.conch.ssh.common import MP, NS, ffs, getMP, getNS
+from twisted.internet import defer, protocol
+from twisted.logger import Logger
+from twisted.python import randbytes
+from twisted.python.compat import iterbytes, networkString
+
+# This import is needed if SHA256 hashing is used.
+# from twisted.python.compat import nativeString
+
+
+def _mpFromBytes(data):
+    """Make an SSH multiple-precision integer from big-endian L{bytes}.
+
+    Used in ECDH key exchange.
+
+    @type data: L{bytes}
+    @param data: The input data, interpreted as a big-endian octet string.
+
+    @rtype: L{bytes}
+    @return: The given data encoded as an SSH multiple-precision integer.
+    """
+    return MP(int.from_bytes(data, "big"))
+
+
+# from https://github.com/python/typeshed/blob/703ed36d5a5c9505c903ea2182e6eed679d9bddb/stdlib/hmac.pyi#L9-L10
+_Hash = Any
+_DigestMod = Union[str, Callable[[], _Hash], types.ModuleType]
+
+
+class _MACParams(Tuple[_DigestMod, bytes, bytes, int]):
+    """
+    L{_MACParams} represents the parameters necessary to compute SSH MAC
+    (Message Authenticate Codes).
+
+    L{_MACParams} is a L{tuple} subclass to maintain compatibility with older
+    versions of the code.  The elements of a L{_MACParams} are::
+
+        0. The digest object used for the MAC
+        1. The inner pad ("ipad") string
+        2. The outer pad ("opad") string
+        3. The size of the digest produced by the digest object
+
+    L{_MACParams} is also an object lesson in why tuples are a bad type for
+    public APIs.
+
+    @ivar key: The HMAC key which will be used.
+    """
+
+    key: bytes
+
+
+class SSHCiphers:
+    """
+    SSHCiphers represents all the encryption operations that need to occur
+    to encrypt and authenticate the SSH connection.
+
+    @cvar cipherMap: A dictionary mapping SSH encryption names to 3-tuples of
+        (<cryptography.hazmat.primitives.interfaces.CipherAlgorithm>,
+        <block size>, <cryptography.hazmat.primitives.interfaces.Mode>)
+    @cvar macMap: A dictionary mapping SSH MAC names to hash modules.
+
+    @ivar outCipType: the string type of the outgoing cipher.
+    @ivar inCipType: the string type of the incoming cipher.
+    @ivar outMACType: the string type of the incoming MAC.
+    @ivar inMACType: the string type of the incoming MAC.
+    @ivar encBlockSize: the block size of the outgoing cipher.
+    @ivar decBlockSize: the block size of the incoming cipher.
+    @ivar verifyDigestSize: the size of the incoming MAC.
+    @ivar outMAC: a tuple of (<hash module>, <inner key>, <outer key>,
+        <digest size>) representing the outgoing MAC.
+    @ivar inMAc: see outMAC, but for the incoming MAC.
+    """
+
+    cipherMap = {
+        b"3des-cbc": (algorithms.TripleDES, 24, modes.CBC),
+        b"blowfish-cbc": (algorithms.Blowfish, 16, modes.CBC),
+        b"aes256-cbc": (algorithms.AES, 32, modes.CBC),
+        b"aes192-cbc": (algorithms.AES, 24, modes.CBC),
+        b"aes128-cbc": (algorithms.AES, 16, modes.CBC),
+        b"cast128-cbc": (algorithms.CAST5, 16, modes.CBC),
+        b"aes128-ctr": (algorithms.AES, 16, modes.CTR),
+        b"aes192-ctr": (algorithms.AES, 24, modes.CTR),
+        b"aes256-ctr": (algorithms.AES, 32, modes.CTR),
+        b"3des-ctr": (algorithms.TripleDES, 24, modes.CTR),
+        b"blowfish-ctr": (algorithms.Blowfish, 16, modes.CTR),
+        b"cast128-ctr": (algorithms.CAST5, 16, modes.CTR),
+        b"none": (None, 0, modes.CBC),
+    }
+    macMap = {
+        b"hmac-sha2-512": sha512,
+        b"hmac-sha2-384": sha384,
+        b"hmac-sha2-256": sha256,
+        b"hmac-sha1": sha1,
+        b"hmac-md5": md5,
+        b"none": None,
+    }
+
+    def __init__(self, outCip, inCip, outMac, inMac):
+        self.outCipType = outCip
+        self.inCipType = inCip
+        self.outMACType = outMac
+        self.inMACType = inMac
+        self.encBlockSize = 0
+        self.decBlockSize = 0
+        self.verifyDigestSize = 0
+        self.outMAC = (None, b"", b"", 0)
+        self.inMAC = (None, b"", b"", 0)
+
+    def setKeys(self, outIV, outKey, inIV, inKey, outInteg, inInteg):
+        """
+        Set up the ciphers and hashes using the given keys,
+
+        @param outIV: the outgoing initialization vector
+        @param outKey: the outgoing encryption key
+        @param inIV: the incoming initialization vector
+        @param inKey: the incoming encryption key
+        @param outInteg: the outgoing integrity key
+        @param inInteg: the incoming integrity key.
+        """
+        o = self._getCipher(self.outCipType, outIV, outKey)
+        self.encryptor = o.encryptor()
+        self.encBlockSize = o.algorithm.block_size // 8
+        o = self._getCipher(self.inCipType, inIV, inKey)
+        self.decryptor = o.decryptor()
+        self.decBlockSize = o.algorithm.block_size // 8
+        self.outMAC = self._getMAC(self.outMACType, outInteg)
+        self.inMAC = self._getMAC(self.inMACType, inInteg)
+        if self.inMAC:
+            self.verifyDigestSize = self.inMAC[3]
+
+    def _getCipher(self, cip, iv, key):
+        """
+        Creates an initialized cipher object.
+
+        @param cip: the name of the cipher, maps into cipherMap
+        @param iv: the initialzation vector
+        @param key: the encryption key
+
+        @return: the cipher object.
+        """
+        algorithmClass, keySize, modeClass = self.cipherMap[cip]
+        if algorithmClass is None:
+            return _DummyCipher()
+
+        return Cipher(
+            algorithmClass(key[:keySize]),
+            modeClass(iv[: algorithmClass.block_size // 8]),
+            backend=default_backend(),
+        )
+
+    def _getMAC(
+        self, mac: bytes, key: bytes
+    ) -> tuple[None, Literal[b""], Literal[b""], Literal[0]] | _MACParams:
+        """
+        Gets a 4-tuple representing the message authentication code.
+        (<hash module>, <inner hash value>, <outer hash value>,
+        <digest size>)
+
+        @type mac: L{bytes}
+        @param mac: a key mapping into macMap
+
+        @type key: L{bytes}
+        @param key: the MAC key.
+
+        @rtype: L{bytes}
+        @return: The MAC components.
+        """
+        mod = self.macMap[mac]
+        if not mod:
+            return (None, b"", b"", 0)
+
+        # With stdlib we can only get attributes fron an instantiated object.
+        hashObject = mod()
+        digestSize = hashObject.digest_size
+        blockSize = hashObject.block_size
+
+        # Truncation here appears to contravene RFC 2104, section 2.  However,
+        # implementing the hashing behavior prescribed by the RFC breaks
+        # interoperability with OpenSSH (at least version 5.5p1).
+        key = key[:digestSize] + (b"\x00" * (blockSize - digestSize))
+        i = key.translate(hmac.trans_36)
+        o = key.translate(hmac.trans_5C)
+        result = _MACParams((mod, i, o, digestSize))
+        result.key = key
+        return result
+
+    def encrypt(self, blocks):
+        """
+        Encrypt some data.
+
+        @type blocks: L{bytes}
+        @param blocks: The data to encrypt.
+
+        @rtype: L{bytes}
+        @return: The encrypted data.
+        """
+        return self.encryptor.update(blocks)
+
+    def decrypt(self, blocks):
+        """
+        Decrypt some data.
+
+        @type blocks: L{bytes}
+        @param blocks: The data to decrypt.
+
+        @rtype: L{bytes}
+        @return: The decrypted data.
+        """
+        return self.decryptor.update(blocks)
+
+    def makeMAC(self, seqid, data):
+        """
+        Create a message authentication code (MAC) for the given packet using
+        the outgoing MAC values.
+
+        @type seqid: L{int}
+        @param seqid: The sequence ID of the outgoing packet.
+
+        @type data: L{bytes}
+        @param data: The data to create a MAC for.
+
+        @rtype: L{str}
+        @return: The serialized MAC.
+        """
+        if not self.outMAC[0]:
+            return b""
+        data = struct.pack(">L", seqid) + data
+        return hmac.HMAC(self.outMAC.key, data, self.outMAC[0]).digest()
+
+    def verify(self, seqid, data, mac):
+        """
+        Verify an incoming MAC using the incoming MAC values.
+
+        @type seqid: L{int}
+        @param seqid: The sequence ID of the incoming packet.
+
+        @type data: L{bytes}
+        @param data: The packet data to verify.
+
+        @type mac: L{bytes}
+        @param mac: The MAC sent with the packet.
+
+        @rtype: L{bool}
+        @return: C{True} if the MAC is valid.
+        """
+        if not self.inMAC[0]:
+            return mac == b""
+        data = struct.pack(">L", seqid) + data
+        outer = hmac.HMAC(self.inMAC.key, data, self.inMAC[0]).digest()
+        return hmac.compare_digest(mac, outer)
+
+
+def _getSupportedCiphers():
+    """
+    Build a list of ciphers that are supported by the backend in use.
+
+    @return: a list of supported ciphers.
+    @rtype: L{list} of L{str}
+    """
+    supportedCiphers = []
+    cs = [
+        b"aes256-ctr",
+        b"aes256-cbc",
+        b"aes192-ctr",
+        b"aes192-cbc",
+        b"aes128-ctr",
+        b"aes128-cbc",
+        b"cast128-ctr",
+        b"cast128-cbc",
+        b"blowfish-ctr",
+        b"blowfish-cbc",
+        b"3des-ctr",
+        b"3des-cbc",
+    ]
+    for cipher in cs:
+        algorithmClass, keySize, modeClass = SSHCiphers.cipherMap[cipher]
+        try:
+            Cipher(
+                algorithmClass(b" " * keySize),
+                modeClass(b" " * (algorithmClass.block_size // 8)),
+                backend=default_backend(),
+            ).encryptor()
+        except UnsupportedAlgorithm:
+            pass
+        else:
+            supportedCiphers.append(cipher)
+    return supportedCiphers
+
+
+class SSHTransportBase(protocol.Protocol):
+    """
+    Protocol supporting basic SSH functionality: sending/receiving packets
+    and message dispatch.  To connect to or run a server, you must use
+    SSHClientTransport or SSHServerTransport.
+
+    @ivar protocolVersion: A string representing the version of the SSH
+        protocol we support.  Currently defaults to '2.0'.
+
+    @ivar version: A string representing the version of the server or client.
+        Currently defaults to 'Twisted'.
+
+    @ivar comment: An optional string giving more information about the
+        server or client.
+
+    @ivar supportedCiphers: A list of strings representing the encryption
+        algorithms supported, in order from most-preferred to least.
+
+    @ivar supportedMACs: A list of strings representing the message
+        authentication codes (hashes) supported, in order from most-preferred
+        to least.  Both this and supportedCiphers can include 'none' to use
+        no encryption or authentication, but that must be done manually,
+
+    @ivar supportedKeyExchanges: A list of strings representing the
+        key exchanges supported, in order from most-preferred to least.
+
+    @ivar supportedPublicKeys:  A list of strings representing the
+        public key algorithms supported, in order from most-preferred to
+        least.
+
+    @ivar supportedCompressions: A list of strings representing compression
+        types supported, from most-preferred to least.
+
+    @ivar supportedLanguages: A list of strings representing languages
+        supported, from most-preferred to least.
+
+    @ivar supportedVersions: A container of strings representing supported ssh
+        protocol version numbers.
+
+    @ivar isClient: A boolean indicating whether this is a client or server.
+
+    @ivar gotVersion: A boolean indicating whether we have received the
+        version string from the other side.
+
+    @ivar buf: Data we've received but hasn't been parsed into a packet.
+
+    @ivar outgoingPacketSequence: the sequence number of the next packet we
+        will send.
+
+    @ivar incomingPacketSequence: the sequence number of the next packet we
+        are expecting from the other side.
+
+    @ivar outgoingCompression: an object supporting the .compress(str) and
+        .flush() methods, or None if there is no outgoing compression.  Used to
+        compress outgoing data.
+
+    @ivar outgoingCompressionType: A string representing the outgoing
+        compression type.
+
+    @ivar incomingCompression: an object supporting the .decompress(str)
+        method, or None if there is no incoming compression.  Used to
+        decompress incoming data.
+
+    @ivar incomingCompressionType: A string representing the incoming
+        compression type.
+
+    @ivar ourVersionString: the version string that we sent to the other side.
+        Used in the key exchange.
+
+    @ivar otherVersionString: the version string sent by the other side.  Used
+        in the key exchange.
+
+    @ivar ourKexInitPayload: the MSG_KEXINIT payload we sent.  Used in the key
+        exchange.
+
+    @ivar otherKexInitPayload: the MSG_KEXINIT payload we received.  Used in
+        the key exchange
+
+    @ivar sessionID: a string that is unique to this SSH session.  Created as
+        part of the key exchange, sessionID is used to generate the various
+        encryption and authentication keys.
+
+    @ivar service: an SSHService instance, or None.  If it's set to an object,
+        it's the currently running service.
+
+    @ivar kexAlg: the agreed-upon key exchange algorithm.
+
+    @ivar keyAlg: the agreed-upon public key type for the key exchange.
+
+    @ivar currentEncryptions: an SSHCiphers instance.  It represents the
+        current encryption and authentication options for the transport.
+
+    @ivar nextEncryptions: an SSHCiphers instance.  Held here until the
+        MSG_NEWKEYS messages are exchanged, when nextEncryptions is
+        transitioned to currentEncryptions.
+
+    @ivar first: the first bytes of the next packet.  In order to avoid
+        decrypting data twice, the first bytes are decrypted and stored until
+        the whole packet is available.
+
+    @ivar _keyExchangeState: The current protocol state with respect to key
+        exchange.  This is either C{_KEY_EXCHANGE_NONE} if no key exchange is
+        in progress (and returns to this value after any key exchange
+        completqes), C{_KEY_EXCHANGE_REQUESTED} if this side of the connection
+        initiated a key exchange, and C{_KEY_EXCHANGE_PROGRESSING} if the other
+        side of the connection initiated a key exchange.  C{_KEY_EXCHANGE_NONE}
+        is the initial value (however SSH connections begin with key exchange,
+        so it will quickly change to another state).
+
+    @ivar _blockedByKeyExchange: Whenever C{_keyExchangeState} is not
+        C{_KEY_EXCHANGE_NONE}, this is a C{list} of pending messages which were
+        passed to L{sendPacket} but could not be sent because it is not legal
+        to send them while a key exchange is in progress.  When the key
+        exchange completes, another attempt is made to send these messages.
+
+    @ivar _peerSupportsExtensions: a boolean indicating whether the other side
+        of the connection supports RFC 8308 extension negotiation.
+
+    @ivar peerExtensions: a dict of extensions supported by the other side of
+        the connection.
+    """
+
+    _log = Logger()
+
+    protocolVersion = b"2.0"
+    version = b"Twisted_" + twisted_version.encode("ascii")
+    comment = b""
+    ourVersionString = (
+        b"SSH-" + protocolVersion + b"-" + version + b" " + comment
+    ).strip()
+
+    # L{None} is supported as cipher and hmac. For security they are disabled
+    # by default. To enable them, subclass this class and add it, or do:
+    # SSHTransportBase.supportedCiphers.append('none')
+    # List ordered by preference.
+    supportedCiphers = _getSupportedCiphers()
+    supportedMACs = [
+        b"hmac-sha2-512",
+        b"hmac-sha2-384",
+        b"hmac-sha2-256",
+        b"hmac-sha1",
+        b"hmac-md5",
+        # `none`,
+    ]
+
+    supportedKeyExchanges = _kex.getSupportedKeyExchanges()
+    supportedPublicKeys = []
+
+    # Add the supported EC keys, and change the name from ecdh* to ecdsa*
+    for eckey in supportedKeyExchanges:
+        if eckey.find(b"ecdh") != -1:
+            supportedPublicKeys += [eckey.replace(b"ecdh", b"ecdsa")]
+
+    supportedPublicKeys += [b"rsa-sha2-512", b"rsa-sha2-256", b"ssh-rsa", b"ssh-dss"]
+    if default_backend().ed25519_supported():
+        supportedPublicKeys.append(b"ssh-ed25519")
+
+    supportedCompressions = [b"none", b"zlib"]
+    supportedLanguages = ()
+    supportedVersions = (b"1.99", b"2.0")
+    isClient = False
+    gotVersion = False
+    buf = b""
+    outgoingPacketSequence = 0
+    incomingPacketSequence = 0
+    outgoingCompression = None
+    incomingCompression = None
+    sessionID = None
+    service = None
+
+    # There is no key exchange activity in progress.
+    _KEY_EXCHANGE_NONE = "_KEY_EXCHANGE_NONE"
+
+    # Key exchange is in progress and we started it.
+    _KEY_EXCHANGE_REQUESTED = "_KEY_EXCHANGE_REQUESTED"
+
+    # Key exchange is in progress and both sides have sent KEXINIT messages.
+    _KEY_EXCHANGE_PROGRESSING = "_KEY_EXCHANGE_PROGRESSING"
+
+    # There is a fourth conceptual state not represented here: KEXINIT received
+    # but not sent.  Since we always send a KEXINIT as soon as we get it, we
+    # can't ever be in that state.
+
+    # The current key exchange state.
+    _keyExchangeState = _KEY_EXCHANGE_NONE
+    _blockedByKeyExchange = None
+
+    # Added to key exchange algorithms by a client to indicate support for
+    # extension negotiation.
+    _EXT_INFO_C = b"ext-info-c"
+
+    # Added to key exchange algorithms by a server to indicate support for
+    # extension negotiation.
+    _EXT_INFO_S = b"ext-info-s"
+
+    _peerSupportsExtensions = False
+    peerExtensions: Dict[bytes, bytes] = {}
+
+    def connectionLost(self, reason):
+        """
+        When the underlying connection is closed, stop the running service (if
+        any), and log out the avatar (if any).
+
+        @type reason: L{twisted.python.failure.Failure}
+        @param reason: The cause of the connection being closed.
+        """
+        if self.service:
+            self.service.serviceStopped()
+        if hasattr(self, "avatar"):
+            self.logoutFunction()
+        self._log.info("connection lost")
+
+    def connectionMade(self):
+        """
+        Called when the connection is made to the other side.  We sent our
+        version and the MSG_KEXINIT packet.
+        """
+        self.transport.write(self.ourVersionString + b"\r\n")
+        self.currentEncryptions = SSHCiphers(b"none", b"none", b"none", b"none")
+        self.currentEncryptions.setKeys(b"", b"", b"", b"", b"", b"")
+        self.sendKexInit()
+
+    def sendKexInit(self):
+        """
+        Send a I{KEXINIT} message to initiate key exchange or to respond to a
+        key exchange initiated by the peer.
+
+        @raise RuntimeError: If a key exchange has already been started and it
+            is not appropriate to send a I{KEXINIT} message at this time.
+
+        @return: L{None}
+        """
+        if self._keyExchangeState != self._KEY_EXCHANGE_NONE:
+            raise RuntimeError(
+                "Cannot send KEXINIT while key exchange state is %r"
+                % (self._keyExchangeState,)
+            )
+
+        supportedKeyExchanges = list(self.supportedKeyExchanges)
+        # Advertise extension negotiation (RFC 8308, section 2.1).  At
+        # present, the Conch client processes the "server-sig-algs"
+        # extension (section 3.1), and the Conch server sends that but
+        # ignores any extensions sent by the client, so strictly speaking at
+        # the moment we only need to send this in the client case; however,
+        # there's nothing to forbid the server from sending it as well, and
+        # doing so makes things easier if it needs to process extensions
+        # sent by clients in future.
+        supportedKeyExchanges.append(
+            self._EXT_INFO_C if self.isClient else self._EXT_INFO_S
+        )
+
+        self.ourKexInitPayload = b"".join(
+            [
+                bytes((MSG_KEXINIT,)),
+                randbytes.secureRandom(16),
+                NS(b",".join(supportedKeyExchanges)),
+                NS(b",".join(self.supportedPublicKeys)),
+                NS(b",".join(self.supportedCiphers)),
+                NS(b",".join(self.supportedCiphers)),
+                NS(b",".join(self.supportedMACs)),
+                NS(b",".join(self.supportedMACs)),
+                NS(b",".join(self.supportedCompressions)),
+                NS(b",".join(self.supportedCompressions)),
+                NS(b",".join(self.supportedLanguages)),
+                NS(b",".join(self.supportedLanguages)),
+                b"\000\000\000\000\000",
+            ]
+        )
+        self.sendPacket(MSG_KEXINIT, self.ourKexInitPayload[1:])
+        self._keyExchangeState = self._KEY_EXCHANGE_REQUESTED
+        self._blockedByKeyExchange = []
+
+    def _allowedKeyExchangeMessageType(self, messageType):
+        """
+        Determine if the given message type may be sent while key exchange is
+        in progress.
+
+        @param messageType: The type of message
+        @type messageType: L{int}
+
+        @return: C{True} if the given type of message may be sent while key
+            exchange is in progress, C{False} if it may not.
+        @rtype: L{bool}
+
+        @see: U{http://tools.ietf.org/html/rfc4253#section-7.1}
+        """
+        # Written somewhat peculularly to reflect the way the specification
+        # defines the allowed message types.
+        if 1 <= messageType <= 19:
+            return messageType not in (
+                MSG_SERVICE_REQUEST,
+                MSG_SERVICE_ACCEPT,
+                MSG_EXT_INFO,
+            )
+        if 20 <= messageType <= 29:
+            return messageType not in (MSG_KEXINIT,)
+        return 30 <= messageType <= 49
+
+    def sendPacket(self, messageType, payload):
+        """
+        Sends a packet.  If it's been set up, compress the data, encrypt it,
+        and authenticate it before sending.  If key exchange is in progress and
+        the message is not part of key exchange, queue it to be sent later.
+
+        @param messageType: The type of the packet; generally one of the
+                            MSG_* values.
+        @type messageType: L{int}
+        @param payload: The payload for the message.
+        @type payload: L{str}
+        """
+        if self._keyExchangeState != self._KEY_EXCHANGE_NONE:
+            if not self._allowedKeyExchangeMessageType(messageType):
+                self._blockedByKeyExchange.append((messageType, payload))
+                return
+
+        payload = bytes((messageType,)) + payload
+        if self.outgoingCompression:
+            payload = self.outgoingCompression.compress(
+                payload
+            ) + self.outgoingCompression.flush(2)
+        bs = self.currentEncryptions.encBlockSize
+        # 4 for the packet length and 1 for the padding length
+        totalSize = 5 + len(payload)
+        lenPad = bs - (totalSize % bs)
+        if lenPad < 4:
+            lenPad = lenPad + bs
+        packet = (
+            struct.pack("!LB", totalSize + lenPad - 4, lenPad)
+            + payload
+            + randbytes.secureRandom(lenPad)
+        )
+        encPacket = self.currentEncryptions.encrypt(
+            packet
+        ) + self.currentEncryptions.makeMAC(self.outgoingPacketSequence, packet)
+        self.transport.write(encPacket)
+        self.outgoingPacketSequence += 1
+
+    def getPacket(self):
+        """
+        Try to return a decrypted, authenticated, and decompressed packet
+        out of the buffer.  If there is not enough data, return None.
+
+        @rtype: L{str} or L{None}
+        @return: The decoded packet, if any.
+        """
+        bs = self.currentEncryptions.decBlockSize
+        ms = self.currentEncryptions.verifyDigestSize
+        if len(self.buf) < bs:
+            # Not enough data for a block
+            return
+        if not hasattr(self, "first"):
+            first = self.currentEncryptions.decrypt(self.buf[:bs])
+        else:
+            first = self.first
+            del self.first
+        packetLen, paddingLen = struct.unpack("!LB", first[:5])
+        if packetLen > 1048576:  # 1024 ** 2
+            self.sendDisconnect(
+                DISCONNECT_PROTOCOL_ERROR,
+                networkString(f"bad packet length {packetLen}"),
+            )
+            return
+        if len(self.buf) < packetLen + 4 + ms:
+            # Not enough data for a packet
+            self.first = first
+            return
+        if (packetLen + 4) % bs != 0:
+            self.sendDisconnect(
+                DISCONNECT_PROTOCOL_ERROR,
+                networkString(
+                    "bad packet mod (%i%%%i == %i)"
+                    % (packetLen + 4, bs, (packetLen + 4) % bs)
+                ),
+            )
+            return
+        encData, self.buf = self.buf[: 4 + packetLen], self.buf[4 + packetLen :]
+        packet = first + self.currentEncryptions.decrypt(encData[bs:])
+        if len(packet) != 4 + packetLen:
+            self.sendDisconnect(DISCONNECT_PROTOCOL_ERROR, b"bad decryption")
+            return
+        if ms:
+            macData, self.buf = self.buf[:ms], self.buf[ms:]
+            if not self.currentEncryptions.verify(
+                self.incomingPacketSequence, packet, macData
+            ):
+                self.sendDisconnect(DISCONNECT_MAC_ERROR, b"bad MAC")
+                return
+        payload = packet[5:-paddingLen]
+        if self.incomingCompression:
+            try:
+                payload = self.incomingCompression.decompress(payload)
+            except Exception:
+                # Tolerate any errors in decompression
+                self._log.failure("Error decompressing payload")
+                self.sendDisconnect(DISCONNECT_COMPRESSION_ERROR, b"compression error")
+                return
+        self.incomingPacketSequence += 1
+        return payload
+
+    def _unsupportedVersionReceived(self, remoteVersion):
+        """
+        Called when an unsupported version of the ssh protocol is received from
+        the remote endpoint.
+
+        @param remoteVersion: remote ssh protocol version which is unsupported
+            by us.
+        @type remoteVersion: L{str}
+        """
+        self.sendDisconnect(
+            DISCONNECT_PROTOCOL_VERSION_NOT_SUPPORTED, b"bad version " + remoteVersion
+        )
+
+    def dataReceived(self, data):
+        """
+        First, check for the version string (SSH-2.0-*).  After that has been
+        received, this method adds data to the buffer, and pulls out any
+        packets.
+
+        @type data: L{bytes}
+        @param data: The data that was received.
+        """
+        self.buf = self.buf + data
+        if not self.gotVersion:
+            if len(self.buf) > 4096:
+                self.sendDisconnect(
+                    DISCONNECT_CONNECTION_LOST,
+                    b"Peer version string longer than 4KB. "
+                    b"Preventing a denial of service attack.",
+                )
+                return
+
+            if self.buf.find(b"\n", self.buf.find(b"SSH-")) == -1:
+                return
+
+            # RFC 4253 section 4.2 ask for strict `\r\n` line ending.
+            # Here we are a bit more relaxed and accept implementations ending
+            # only in '\n'.
+            # https://tools.ietf.org/html/rfc4253#section-4.2
+            lines = self.buf.split(b"\n")
+            for p in lines:
+                if p.startswith(b"SSH-"):
+                    self.gotVersion = True
+                    # Since the line was split on '\n' and most of the time
+                    # it uses '\r\n' we may get an extra '\r'.
+                    self.otherVersionString = p.rstrip(b"\r")
+                    remoteVersion = p.split(b"-")[1]
+                    if remoteVersion not in self.supportedVersions:
+                        self._unsupportedVersionReceived(remoteVersion)
+                        return
+                    i = lines.index(p)
+                    self.buf = b"\n".join(lines[i + 1 :])
+        packet = self.getPacket()
+        while packet:
+            messageNum = ord(packet[0:1])
+            self.dispatchMessage(messageNum, packet[1:])
+            packet = self.getPacket()
+
+    def dispatchMessage(self, messageNum, payload):
+        """
+        Send a received message to the appropriate method.
+
+        @type messageNum: L{int}
+        @param messageNum: The message number.
+
+        @type payload: L{bytes}
+        @param payload: The message payload.
+        """
+        if messageNum < 50 and messageNum in messages:
+            messageType = messages[messageNum][4:]
+            f = getattr(self, f"ssh_{messageType}", None)
+            if f is not None:
+                f(payload)
+            else:
+                self._log.debug(
+                    "couldn't handle {messageType}: {payload!r}",
+                    messageType=messageType,
+                    payload=payload,
+                )
+                self.sendUnimplemented()
+        elif self.service:
+            self.service.packetReceived(messageNum, payload)
+        else:
+            self._log.debug(
+                "couldn't handle {messageNum}: {payload!r}",
+                messageNum=messageNum,
+                payload=payload,
+            )
+            self.sendUnimplemented()
+
+    def getPeer(self):
+        """
+        Returns an L{SSHTransportAddress} corresponding to the other (peer)
+        side of this transport.
+
+        @return: L{SSHTransportAddress} for the peer
+        @rtype: L{SSHTransportAddress}
+        @since: 12.1
+        """
+        return address.SSHTransportAddress(self.transport.getPeer())
+
+    def getHost(self):
+        """
+        Returns an L{SSHTransportAddress} corresponding to the this side of
+        transport.
+
+        @return: L{SSHTransportAddress} for the peer
+        @rtype: L{SSHTransportAddress}
+        @since: 12.1
+        """
+        return address.SSHTransportAddress(self.transport.getHost())
+
+    @property
+    def kexAlg(self):
+        """
+        The key exchange algorithm name agreed between client and server.
+        """
+        return self._kexAlg
+
+    @kexAlg.setter
+    def kexAlg(self, value):
+        """
+        Set the key exchange algorithm name.
+        """
+        self._kexAlg = value
+
+    # Client-initiated rekeying looks like this:
+    #
+    #  C> MSG_KEXINIT
+    #  S> MSG_KEXINIT
+    #  C> MSG_KEX_DH_GEX_REQUEST  or   MSG_KEXDH_INIT
+    #  S> MSG_KEX_DH_GEX_GROUP    or   MSG_KEXDH_REPLY
+    #  C> MSG_KEX_DH_GEX_INIT     or   --
+    #  S> MSG_KEX_DH_GEX_REPLY    or   --
+    #  C> MSG_NEWKEYS
+    #  S> MSG_NEWKEYS
+    #
+    # Server-initiated rekeying is the same, only the first two messages are
+    # switched.
+
+    def ssh_KEXINIT(self, packet):
+        """
+        Called when we receive a MSG_KEXINIT message.  Payload::
+            bytes[16] cookie
+            string keyExchangeAlgorithms
+            string keyAlgorithms
+            string incomingEncryptions
+            string outgoingEncryptions
+            string incomingAuthentications
+            string outgoingAuthentications
+            string incomingCompressions
+            string outgoingCompressions
+            string incomingLanguages
+            string outgoingLanguages
+            bool firstPacketFollows
+            unit32 0 (reserved)
+
+        Starts setting up the key exchange, keys, encryptions, and
+        authentications.  Extended by ssh_KEXINIT in SSHServerTransport and
+        SSHClientTransport.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+
+        @return: A L{tuple} of negotiated key exchange algorithms, key
+        algorithms, and unhandled data, or L{None} if something went wrong.
+        """
+        self.otherKexInitPayload = bytes((MSG_KEXINIT,)) + packet
+        # This is useless to us:
+        # cookie = packet[: 16]
+        k = getNS(packet[16:], 10)
+        strings, rest = k[:-1], k[-1]
+        (
+            kexAlgs,
+            keyAlgs,
+            encCS,
+            encSC,
+            macCS,
+            macSC,
+            compCS,
+            compSC,
+            langCS,
+            langSC,
+        ) = (s.split(b",") for s in strings)
+        # These are the server directions
+        outs = [encSC, macSC, compSC]
+        ins = [encCS, macCS, compCS]
+        if self.isClient:
+            outs, ins = ins, outs  # Switch directions
+        server = (
+            self.supportedKeyExchanges,
+            self.supportedPublicKeys,
+            self.supportedCiphers,
+            self.supportedCiphers,
+            self.supportedMACs,
+            self.supportedMACs,
+            self.supportedCompressions,
+            self.supportedCompressions,
+        )
+        client = (kexAlgs, keyAlgs, outs[0], ins[0], outs[1], ins[1], outs[2], ins[2])
+        if self.isClient:
+            server, client = client, server
+        self.kexAlg = ffs(client[0], server[0])
+        self.keyAlg = ffs(client[1], server[1])
+        self.nextEncryptions = SSHCiphers(
+            ffs(client[2], server[2]),
+            ffs(client[3], server[3]),
+            ffs(client[4], server[4]),
+            ffs(client[5], server[5]),
+        )
+        self.outgoingCompressionType = ffs(client[6], server[6])
+        self.incomingCompressionType = ffs(client[7], server[7])
+        if (
+            None
+            in (
+                self.kexAlg,
+                self.keyAlg,
+                self.outgoingCompressionType,
+                self.incomingCompressionType,
+            )
+            # We MUST disconnect if an extension negotiation indication ends
+            # up being negotiated as a key exchange method (RFC 8308,
+            # section 2.2).
+            or self.kexAlg in (self._EXT_INFO_C, self._EXT_INFO_S)
+        ):
+            self.sendDisconnect(
+                DISCONNECT_KEY_EXCHANGE_FAILED, b"couldn't match all kex parts"
+            )
+            return
+        if None in self.nextEncryptions.__dict__.values():
+            self.sendDisconnect(
+                DISCONNECT_KEY_EXCHANGE_FAILED, b"couldn't match all kex parts"
+            )
+            return
+        self._peerSupportsExtensions = (
+            self._EXT_INFO_S if self.isClient else self._EXT_INFO_C
+        ) in kexAlgs
+        self._log.debug(
+            "kex alg={kexAlg!r} key alg={keyAlg!r}",
+            kexAlg=self.kexAlg,
+            keyAlg=self.keyAlg,
+        )
+        self._log.debug(
+            "outgoing: {cip!r} {mac!r} {compression!r}",
+            cip=self.nextEncryptions.outCipType,
+            mac=self.nextEncryptions.outMACType,
+            compression=self.outgoingCompressionType,
+        )
+        self._log.debug(
+            "incoming: {cip!r} {mac!r} {compression!r}",
+            cip=self.nextEncryptions.inCipType,
+            mac=self.nextEncryptions.inMACType,
+            compression=self.incomingCompressionType,
+        )
+
+        if self._keyExchangeState == self._KEY_EXCHANGE_REQUESTED:
+            self._keyExchangeState = self._KEY_EXCHANGE_PROGRESSING
+        else:
+            self.sendKexInit()
+
+        return kexAlgs, keyAlgs, rest  # For SSHServerTransport to use
+
+    def ssh_DISCONNECT(self, packet):
+        """
+        Called when we receive a MSG_DISCONNECT message.  Payload::
+            long code
+            string description
+
+        This means that the other side has disconnected.  Pass the message up
+        and disconnect ourselves.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        reasonCode = struct.unpack(">L", packet[:4])[0]
+        description, foo = getNS(packet[4:])
+        self.receiveError(reasonCode, description)
+        self.transport.loseConnection()
+
+    def ssh_IGNORE(self, packet):
+        """
+        Called when we receive a MSG_IGNORE message.  No payload.
+        This means nothing; we simply return.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+
+    def ssh_UNIMPLEMENTED(self, packet):
+        """
+        Called when we receive a MSG_UNIMPLEMENTED message.  Payload::
+            long packet
+
+        This means that the other side did not implement one of our packets.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        (seqnum,) = struct.unpack(">L", packet)
+        self.receiveUnimplemented(seqnum)
+
+    def ssh_DEBUG(self, packet):
+        """
+        Called when we receive a MSG_DEBUG message.  Payload::
+            bool alwaysDisplay
+            string message
+            string language
+
+        This means the other side has passed along some debugging info.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        alwaysDisplay = bool(ord(packet[0:1]))
+        message, lang, foo = getNS(packet[1:], 2)
+        self.receiveDebug(alwaysDisplay, message, lang)
+
+    def ssh_EXT_INFO(self, packet):
+        """
+        Called when we get a MSG_EXT_INFO message.  Payload::
+            uint32 nr-extensions
+            repeat the following 2 fields "nr-extensions" times:
+              string extension-name
+              string extension-value (binary)
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        (numExtensions,) = struct.unpack(">L", packet[:4])
+        packet = packet[4:]
+        extensions = {}
+        for _ in range(numExtensions):
+            extName, extValue, packet = getNS(packet, 2)
+            extensions[extName] = extValue
+        self.peerExtensions = extensions
+
+    def setService(self, service):
+        """
+        Set our service to service and start it running.  If we were
+        running a service previously, stop it first.
+
+        @type service: C{SSHService}
+        @param service: The service to attach.
+        """
+        self._log.debug("starting service {service!r}", service=service.name)
+        if self.service:
+            self.service.serviceStopped()
+        self.service = service
+        service.transport = self
+        self.service.serviceStarted()
+
+    def sendDebug(self, message, alwaysDisplay=False, language=b""):
+        """
+        Send a debug message to the other side.
+
+        @param message: the message to send.
+        @type message: L{str}
+        @param alwaysDisplay: if True, tell the other side to always
+                              display this message.
+        @type alwaysDisplay: L{bool}
+        @param language: optionally, the language the message is in.
+        @type language: L{str}
+        """
+        self.sendPacket(
+            MSG_DEBUG, (b"\1" if alwaysDisplay else b"\0") + NS(message) + NS(language)
+        )
+
+    def sendIgnore(self, message):
+        """
+        Send a message that will be ignored by the other side.  This is
+        useful to fool attacks based on guessing packet sizes in the
+        encrypted stream.
+
+        @param message: data to send with the message
+        @type message: L{str}
+        """
+        self.sendPacket(MSG_IGNORE, NS(message))
+
+    def sendUnimplemented(self):
+        """
+        Send a message to the other side that the last packet was not
+        understood.
+        """
+        seqnum = self.incomingPacketSequence
+        self.sendPacket(MSG_UNIMPLEMENTED, struct.pack("!L", seqnum))
+
+    def sendDisconnect(self, reason, desc):
+        """
+        Send a disconnect message to the other side and then disconnect.
+
+        @param reason: the reason for the disconnect.  Should be one of the
+                       DISCONNECT_* values.
+        @type reason: L{int}
+        @param desc: a descrption of the reason for the disconnection.
+        @type desc: L{str}
+        """
+        self.sendPacket(MSG_DISCONNECT, struct.pack(">L", reason) + NS(desc) + NS(b""))
+        self._log.info(
+            "Disconnecting with error, code {code}\nreason: {description}",
+            code=reason,
+            description=desc,
+        )
+        self.transport.loseConnection()
+
+    def sendExtInfo(self, extensions):
+        """
+        Send an RFC 8308 extension advertisement to the remote peer.
+
+        Nothing is sent if the peer doesn't support negotiations.
+        @type extensions: L{list} of (L{bytes}, L{bytes})
+        @param extensions: a list of (extension-name, extension-value) pairs.
+        """
+        if self._peerSupportsExtensions:
+            payload = b"".join(
+                [struct.pack(">L", len(extensions))]
+                + [NS(name) + NS(value) for name, value in extensions]
+            )
+            self.sendPacket(MSG_EXT_INFO, payload)
+
+    def _startEphemeralDH(self):
+        """
+        Prepares for a Diffie-Hellman key agreement exchange.
+
+        Creates an ephemeral keypair in the group defined by (self.g,
+        self.p) and stores it.
+        """
+
+        numbers = dh.DHParameterNumbers(self.p, self.g)
+        parameters = numbers.parameters(default_backend())
+        self.dhSecretKey = parameters.generate_private_key()
+        y = self.dhSecretKey.public_key().public_numbers().y
+        self.dhSecretKeyPublicMP = MP(y)
+
+    def _finishEphemeralDH(self, remoteDHpublicKey):
+        """
+        Completes the Diffie-Hellman key agreement started by
+        _startEphemeralDH, and forgets the ephemeral secret key.
+
+        @type remoteDHpublicKey: L{int}
+        @rtype: L{bytes}
+        @return: The new shared secret, in SSH C{mpint} format.
+
+        """
+
+        remoteKey = dh.DHPublicNumbers(
+            remoteDHpublicKey, dh.DHParameterNumbers(self.p, self.g)
+        ).public_key(default_backend())
+        secret = self.dhSecretKey.exchange(remoteKey)
+        del self.dhSecretKey
+
+        # The result of a Diffie-Hellman exchange is an integer, but
+        # the Cryptography module returns it as bytes in a form that
+        # is only vaguely documented. We fix it up to match the SSH
+        # MP-integer format as described in RFC4251.
+        secret = secret.lstrip(b"\x00")
+        ch = ord(secret[0:1])
+        if ch & 0x80:  # High bit set?
+            # Make room for the sign bit
+            prefix = struct.pack(">L", len(secret) + 1) + b"\x00"
+        else:
+            prefix = struct.pack(">L", len(secret))
+        return prefix + secret
+
+    def _getKey(self, c, sharedSecret, exchangeHash):
+        """
+        Get one of the keys for authentication/encryption.
+
+        @type c: L{bytes}
+        @param c: The letter identifying which key this is.
+
+        @type sharedSecret: L{bytes}
+        @param sharedSecret: The shared secret K.
+
+        @type exchangeHash: L{bytes}
+        @param exchangeHash: The hash H from key exchange.
+
+        @rtype: L{bytes}
+        @return: The derived key.
+        """
+        hashProcessor = _kex.getHashProcessor(self.kexAlg)
+        k1 = hashProcessor(sharedSecret + exchangeHash + c + self.sessionID)
+        k1 = k1.digest()
+        k2 = hashProcessor(sharedSecret + exchangeHash + k1).digest()
+        k3 = hashProcessor(sharedSecret + exchangeHash + k1 + k2).digest()
+        k4 = hashProcessor(sharedSecret + exchangeHash + k1 + k2 + k3).digest()
+        return k1 + k2 + k3 + k4
+
+    def _keySetup(self, sharedSecret, exchangeHash):
+        """
+        Set up the keys for the connection and sends MSG_NEWKEYS when
+        finished,
+
+        @param sharedSecret: a secret string agreed upon using a Diffie-
+                             Hellman exchange, so it is only shared between
+                             the server and the client.
+        @type sharedSecret: L{str}
+        @param exchangeHash: A hash of various data known by both sides.
+        @type exchangeHash: L{str}
+        """
+        if not self.sessionID:
+            self.sessionID = exchangeHash
+        initIVCS = self._getKey(b"A", sharedSecret, exchangeHash)
+        initIVSC = self._getKey(b"B", sharedSecret, exchangeHash)
+        encKeyCS = self._getKey(b"C", sharedSecret, exchangeHash)
+        encKeySC = self._getKey(b"D", sharedSecret, exchangeHash)
+        integKeyCS = self._getKey(b"E", sharedSecret, exchangeHash)
+        integKeySC = self._getKey(b"F", sharedSecret, exchangeHash)
+        outs = [initIVSC, encKeySC, integKeySC]
+        ins = [initIVCS, encKeyCS, integKeyCS]
+        if self.isClient:  # Reverse for the client
+            outs, ins = ins, outs
+        self.nextEncryptions.setKeys(outs[0], outs[1], ins[0], ins[1], outs[2], ins[2])
+        self.sendPacket(MSG_NEWKEYS, b"")
+
+    def _newKeys(self):
+        """
+        Called back by a subclass once a I{MSG_NEWKEYS} message has been
+        received.  This indicates key exchange has completed and new encryption
+        and compression parameters should be adopted.  Any messages which were
+        queued during key exchange will also be flushed.
+        """
+        self._log.debug("NEW KEYS")
+        self.currentEncryptions = self.nextEncryptions
+        if self.outgoingCompressionType == b"zlib":
+            self.outgoingCompression = zlib.compressobj(6)
+        if self.incomingCompressionType == b"zlib":
+            self.incomingCompression = zlib.decompressobj()
+
+        self._keyExchangeState = self._KEY_EXCHANGE_NONE
+        messages = self._blockedByKeyExchange
+        self._blockedByKeyExchange = None
+        for messageType, payload in messages:
+            self.sendPacket(messageType, payload)
+
+    def isEncrypted(self, direction="out"):
+        """
+        Check if the connection is encrypted in the given direction.
+
+        @type direction: L{str}
+        @param direction: The direction: one of 'out', 'in', or 'both'.
+
+        @rtype: L{bool}
+        @return: C{True} if it is encrypted.
+        """
+        if direction == "out":
+            return self.currentEncryptions.outCipType != b"none"
+        elif direction == "in":
+            return self.currentEncryptions.inCipType != b"none"
+        elif direction == "both":
+            return self.isEncrypted("in") and self.isEncrypted("out")
+        else:
+            raise TypeError('direction must be "out", "in", or "both"')
+
+    def isVerified(self, direction="out"):
+        """
+        Check if the connection is verified/authentication in the given direction.
+
+        @type direction: L{str}
+        @param direction: The direction: one of 'out', 'in', or 'both'.
+
+        @rtype: L{bool}
+        @return: C{True} if it is verified.
+        """
+        if direction == "out":
+            return self.currentEncryptions.outMACType != b"none"
+        elif direction == "in":
+            return self.currentEncryptions.inMACType != b"none"
+        elif direction == "both":
+            return self.isVerified("in") and self.isVerified("out")
+        else:
+            raise TypeError('direction must be "out", "in", or "both"')
+
+    def loseConnection(self):
+        """
+        Lose the connection to the other side, sending a
+        DISCONNECT_CONNECTION_LOST message.
+        """
+        self.sendDisconnect(DISCONNECT_CONNECTION_LOST, b"user closed connection")
+
+    # Client methods
+
+    def receiveError(self, reasonCode, description):
+        """
+        Called when we receive a disconnect error message from the other
+        side.
+
+        @param reasonCode: the reason for the disconnect, one of the
+                           DISCONNECT_ values.
+        @type reasonCode: L{int}
+        @param description: a human-readable description of the
+                            disconnection.
+        @type description: L{str}
+        """
+        self._log.error(
+            "Got remote error, code {code}\nreason: {description}",
+            code=reasonCode,
+            description=description,
+        )
+
+    def receiveUnimplemented(self, seqnum):
+        """
+        Called when we receive an unimplemented packet message from the other
+        side.
+
+        @param seqnum: the sequence number that was not understood.
+        @type seqnum: L{int}
+        """
+        self._log.warn("other side unimplemented packet #{seqnum}", seqnum=seqnum)
+
+    def receiveDebug(self, alwaysDisplay, message, lang):
+        """
+        Called when we receive a debug message from the other side.
+
+        @param alwaysDisplay: if True, this message should always be
+                              displayed.
+        @type alwaysDisplay: L{bool}
+        @param message: the debug message
+        @type message: L{str}
+        @param lang: optionally the language the message is in.
+        @type lang: L{str}
+        """
+        if alwaysDisplay:
+            self._log.debug("Remote Debug Message: {message}", message=message)
+
+    def _generateECPrivateKey(self):
+        """
+        Generate an private key for ECDH key exchange.
+
+        @rtype: The appropriate private key type matching C{self.kexAlg}:
+            L{ec.EllipticCurvePrivateKey} for C{ecdh-sha2-nistp*}, or
+            L{x25519.X25519PrivateKey} for C{curve25519-sha256}.
+        @return: The generated private key.
+        """
+        if self.kexAlg.startswith(b"ecdh-sha2-nistp"):
+            try:
+                curve = keys._curveTable[b"ecdsa" + self.kexAlg[4:]]
+            except KeyError:
+                raise UnsupportedAlgorithm("unused-key")
+
+            return ec.generate_private_key(curve, default_backend())
+        elif self.kexAlg in (b"curve25519-sha256", b"curve25519-sha256@libssh.org"):
+            return x25519.X25519PrivateKey.generate()
+        else:
+            raise UnsupportedAlgorithm(
+                "Cannot generate elliptic curve private key for {!r}".format(
+                    self.kexAlg
+                )
+            )
+
+    def _encodeECPublicKey(self, ecPub):
+        """
+        Encode an elliptic curve public key to bytes.
+
+        @type ecPub: The appropriate public key type matching
+            C{self.kexAlg}: L{ec.EllipticCurvePublicKey} for
+            C{ecdh-sha2-nistp*}, or L{x25519.X25519PublicKey} for
+            C{curve25519-sha256}.
+        @param ecPub: The public key to encode.
+
+        @rtype: L{bytes}
+        @return: The encoded public key.
+        """
+        if self.kexAlg.startswith(b"ecdh-sha2-nistp"):
+            return ecPub.public_bytes(
+                serialization.Encoding.X962,
+                serialization.PublicFormat.UncompressedPoint,
+            )
+        elif self.kexAlg in (b"curve25519-sha256", b"curve25519-sha256@libssh.org"):
+            return ecPub.public_bytes(
+                serialization.Encoding.Raw, serialization.PublicFormat.Raw
+            )
+        else:
+            raise UnsupportedAlgorithm(
+                f"Cannot encode elliptic curve public key for {self.kexAlg!r}"
+            )
+
+    def _generateECSharedSecret(self, ecPriv, theirECPubBytes):
+        """
+        Generate a shared secret for ECDH key exchange.
+
+        @type ecPriv: The appropriate private key type matching
+            C{self.kexAlg}: L{ec.EllipticCurvePrivateKey} for
+            C{ecdh-sha2-nistp*}, or L{x25519.X25519PrivateKey} for
+            C{curve25519-sha256}.
+        @param ecPriv: Our private key.
+
+        @rtype: L{bytes}
+        @return: The generated shared secret, as an SSH multiple-precision
+            integer.
+        """
+        if self.kexAlg.startswith(b"ecdh-sha2-nistp"):
+            try:
+                curve = keys._curveTable[b"ecdsa" + self.kexAlg[4:]]
+            except KeyError:
+                raise UnsupportedAlgorithm("unused-key")
+
+            theirECPub = ec.EllipticCurvePublicKey.from_encoded_point(
+                curve, theirECPubBytes
+            )
+            sharedSecret = ecPriv.exchange(ec.ECDH(), theirECPub)
+        elif self.kexAlg in (b"curve25519-sha256", b"curve25519-sha256@libssh.org"):
+            theirECPub = x25519.X25519PublicKey.from_public_bytes(theirECPubBytes)
+            sharedSecret = ecPriv.exchange(theirECPub)
+        else:
+            raise UnsupportedAlgorithm(
+                "Cannot generate elliptic curve shared secret for {!r}".format(
+                    self.kexAlg
+                )
+            )
+
+        return _mpFromBytes(sharedSecret)
+
+
+class SSHServerTransport(SSHTransportBase):
+    """
+    SSHServerTransport implements the server side of the SSH protocol.
+
+    @ivar isClient: since we are never the client, this is always False.
+
+    @ivar ignoreNextPacket: if True, ignore the next key exchange packet.  This
+        is set when the client sends a guessed key exchange packet but with
+        an incorrect guess.
+
+    @ivar dhGexRequest: the KEX_DH_GEX_REQUEST(_OLD) that the client sent.
+        The key generation needs this to be stored.
+
+    @ivar g: the Diffie-Hellman group generator.
+
+    @ivar p: the Diffie-Hellman group prime.
+    """
+
+    isClient = False
+    ignoreNextPacket = 0
+
+    def _getHostKeys(self, keyAlg):
+        """
+        Get the public and private host keys corresponding to the given
+        public key signature algorithm.
+
+        The factory stores public and private host keys by their key format,
+        which is not quite the same as the key signature algorithm: for
+        example, an ssh-rsa key can sign using any of the ssh-rsa,
+        rsa-sha2-256, or rsa-sha2-512 algorithms.
+
+        @type keyAlg: L{bytes}
+        @param keyAlg: A public key signature algorithm name.
+
+        @rtype: 2-L{tuple} of L{keys.Key}
+        @return: The public and private host keys.
+
+        @raises KeyError: if the factory does not have both a public and a
+        private host key for this signature algorithm.
+        """
+        if keyAlg in {b"rsa-sha2-256", b"rsa-sha2-512"}:
+            keyFormat = b"ssh-rsa"
+        else:
+            keyFormat = keyAlg
+        return self.factory.publicKeys[keyFormat], self.factory.privateKeys[keyFormat]
+
+    def ssh_KEXINIT(self, packet):
+        """
+        Called when we receive a MSG_KEXINIT message.  For a description
+        of the packet, see SSHTransportBase.ssh_KEXINIT().  Additionally,
+        this method checks if a guessed key exchange packet was sent.  If
+        it was sent, and it guessed incorrectly, the next key exchange
+        packet MUST be ignored.
+        """
+        retval = SSHTransportBase.ssh_KEXINIT(self, packet)
+        if not retval:  # Disconnected
+            return
+        else:
+            kexAlgs, keyAlgs, rest = retval
+        if ord(rest[0:1]):  # Flag first_kex_packet_follows?
+            if (
+                kexAlgs[0] != self.supportedKeyExchanges[0]
+                or keyAlgs[0] != self.supportedPublicKeys[0]
+            ):
+                self.ignoreNextPacket = True  # Guess was wrong
+
+    def _ssh_KEX_ECDH_INIT(self, packet):
+        """
+        Called from L{ssh_KEX_DH_GEX_REQUEST_OLD} to handle
+        elliptic curve key exchanges.
+
+        Payload::
+
+            string client Elliptic Curve Diffie-Hellman public key
+
+        Just like L{_ssh_KEXDH_INIT} this message type is also not dispatched
+        directly. Extra check to determine if this is really KEX_ECDH_INIT
+        is required.
+
+        First we load the host's public/private keys.
+        Then we generate the ECDH public/private keypair for the given curve.
+        With that we generate the shared secret key.
+        Then we compute the hash to sign and send back to the client
+        Along with the server's public key and the ECDH public key.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+
+        @return: None.
+        """
+        # Get the raw client public key.
+        pktPub, packet = getNS(packet)
+
+        # Get the host's public and private keys
+        pubHostKey, privHostKey = self._getHostKeys(self.keyAlg)
+
+        # Generate the private key
+        ecPriv = self._generateECPrivateKey()
+
+        # Get the public key
+        self.ecPub = ecPriv.public_key()
+        encPub = self._encodeECPublicKey(self.ecPub)
+
+        # Generate the shared secret
+        sharedSecret = self._generateECSharedSecret(ecPriv, pktPub)
+
+        # Finish update and digest
+        h = _kex.getHashProcessor(self.kexAlg)()
+        h.update(NS(self.otherVersionString))
+        h.update(NS(self.ourVersionString))
+        h.update(NS(self.otherKexInitPayload))
+        h.update(NS(self.ourKexInitPayload))
+        h.update(NS(pubHostKey.blob()))
+        h.update(NS(pktPub))
+        h.update(NS(encPub))
+        h.update(sharedSecret)
+        exchangeHash = h.digest()
+
+        self.sendPacket(
+            MSG_KEXDH_REPLY,
+            NS(pubHostKey.blob())
+            + NS(encPub)
+            + NS(privHostKey.sign(exchangeHash, signatureType=self.keyAlg)),
+        )
+        self._keySetup(sharedSecret, exchangeHash)
+
+    def _ssh_KEXDH_INIT(self, packet):
+        """
+        Called to handle the beginning of a non-group key exchange.
+
+        Unlike other message types, this is not dispatched automatically.  It
+        is called from C{ssh_KEX_DH_GEX_REQUEST_OLD} because an extra check is
+        required to determine if this is really a KEXDH_INIT message or if it
+        is a KEX_DH_GEX_REQUEST_OLD message.
+
+        The KEXDH_INIT payload::
+
+                integer e (the client's Diffie-Hellman public key)
+
+        We send the KEXDH_REPLY with our host key and signature.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        clientDHpublicKey, foo = getMP(packet)
+        pubHostKey, privHostKey = self._getHostKeys(self.keyAlg)
+        self.g, self.p = _kex.getDHGeneratorAndPrime(self.kexAlg)
+        self._startEphemeralDH()
+        sharedSecret = self._finishEphemeralDH(clientDHpublicKey)
+        h = _kex.getHashProcessor(self.kexAlg)()
+        h.update(NS(self.otherVersionString))
+        h.update(NS(self.ourVersionString))
+        h.update(NS(self.otherKexInitPayload))
+        h.update(NS(self.ourKexInitPayload))
+        h.update(NS(pubHostKey.blob()))
+        h.update(MP(clientDHpublicKey))
+        h.update(self.dhSecretKeyPublicMP)
+        h.update(sharedSecret)
+        exchangeHash = h.digest()
+        self.sendPacket(
+            MSG_KEXDH_REPLY,
+            NS(pubHostKey.blob())
+            + self.dhSecretKeyPublicMP
+            + NS(privHostKey.sign(exchangeHash, signatureType=self.keyAlg)),
+        )
+        self._keySetup(sharedSecret, exchangeHash)
+
+    def ssh_KEX_DH_GEX_REQUEST_OLD(self, packet):
+        """
+        This represents different key exchange methods that share the same
+        integer value.  If the message is determined to be a KEXDH_INIT,
+        L{_ssh_KEXDH_INIT} is called to handle it. If it is a KEX_ECDH_INIT,
+        L{_ssh_KEX_ECDH_INIT} is called.
+        Otherwise, for KEX_DH_GEX_REQUEST_OLD payload::
+
+                integer ideal (ideal size for the Diffie-Hellman prime)
+
+            We send the KEX_DH_GEX_GROUP message with the group that is
+            closest in size to ideal.
+
+        If we were told to ignore the next key exchange packet by ssh_KEXINIT,
+        drop it on the floor and return.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        if self.ignoreNextPacket:
+            self.ignoreNextPacket = 0
+            return
+
+        # KEXDH_INIT, KEX_ECDH_INIT, and KEX_DH_GEX_REQUEST_OLD
+        # have the same value, so use another cue
+        # to decide what kind of message the peer sent us.
+        if _kex.isFixedGroup(self.kexAlg):
+            return self._ssh_KEXDH_INIT(packet)
+        elif _kex.isEllipticCurve(self.kexAlg):
+            return self._ssh_KEX_ECDH_INIT(packet)
+        else:
+            self.dhGexRequest = packet
+            ideal = struct.unpack(">L", packet)[0]
+            self.g, self.p = self.factory.getDHPrime(ideal)
+            self._startEphemeralDH()
+            self.sendPacket(MSG_KEX_DH_GEX_GROUP, MP(self.p) + MP(self.g))
+
+    def ssh_KEX_DH_GEX_REQUEST(self, packet):
+        """
+        Called when we receive a MSG_KEX_DH_GEX_REQUEST message.  Payload::
+            integer minimum
+            integer ideal
+            integer maximum
+
+        The client is asking for a Diffie-Hellman group between minimum and
+        maximum size, and close to ideal if possible.  We reply with a
+        MSG_KEX_DH_GEX_GROUP message.
+
+        If we were told to ignore the next key exchange packet by ssh_KEXINIT,
+        drop it on the floor and return.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        if self.ignoreNextPacket:
+            self.ignoreNextPacket = 0
+            return
+        self.dhGexRequest = packet
+        min, ideal, max = struct.unpack(">3L", packet)
+        self.g, self.p = self.factory.getDHPrime(ideal)
+        self._startEphemeralDH()
+        self.sendPacket(MSG_KEX_DH_GEX_GROUP, MP(self.p) + MP(self.g))
+
+    def ssh_KEX_DH_GEX_INIT(self, packet):
+        """
+        Called when we get a MSG_KEX_DH_GEX_INIT message.  Payload::
+            integer e (client DH public key)
+
+        We send the MSG_KEX_DH_GEX_REPLY message with our host key and
+        signature.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        clientDHpublicKey, foo = getMP(packet)
+        pubHostKey, privHostKey = self._getHostKeys(self.keyAlg)
+        # TODO: we should also look at the value they send to us and reject
+        # insecure values of f (if g==2 and f has a single '1' bit while the
+        # rest are '0's, then they must have used a small y also).
+
+        # TODO: This could be computed when self.p is set up
+        #  or do as openssh does and scan f for a single '1' bit instead
+
+        sharedSecret = self._finishEphemeralDH(clientDHpublicKey)
+        h = _kex.getHashProcessor(self.kexAlg)()
+        h.update(NS(self.otherVersionString))
+        h.update(NS(self.ourVersionString))
+        h.update(NS(self.otherKexInitPayload))
+        h.update(NS(self.ourKexInitPayload))
+        h.update(NS(pubHostKey.blob()))
+        h.update(self.dhGexRequest)
+        h.update(MP(self.p))
+        h.update(MP(self.g))
+        h.update(MP(clientDHpublicKey))
+        h.update(self.dhSecretKeyPublicMP)
+        h.update(sharedSecret)
+        exchangeHash = h.digest()
+        self.sendPacket(
+            MSG_KEX_DH_GEX_REPLY,
+            NS(pubHostKey.blob())
+            + self.dhSecretKeyPublicMP
+            + NS(privHostKey.sign(exchangeHash, signatureType=self.keyAlg)),
+        )
+        self._keySetup(sharedSecret, exchangeHash)
+
+    def _keySetup(self, sharedSecret, exchangeHash):
+        """
+        See SSHTransportBase._keySetup().
+        """
+        firstKey = self.sessionID is None
+        SSHTransportBase._keySetup(self, sharedSecret, exchangeHash)
+        # RFC 8308 section 2.4 says that the server MAY send EXT_INFO at
+        # zero, one, or both of the following opportunities: the next packet
+        # following the server's first MSG_NEWKEYS, or immediately preceding
+        # the server's MSG_USERAUTH_SUCCESS.  We have no need for the
+        # latter, so make sure we only send it in the former case.
+        if firstKey:
+            self.sendExtInfo(
+                [(b"server-sig-algs", b",".join(self.supportedPublicKeys))]
+            )
+
+    def ssh_NEWKEYS(self, packet):
+        """
+        Called when we get a MSG_NEWKEYS message.  No payload.
+        When we get this, the keys have been set on both sides, and we
+        start using them to encrypt and authenticate the connection.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        if packet != b"":
+            self.sendDisconnect(DISCONNECT_PROTOCOL_ERROR, b"NEWKEYS takes no data")
+            return
+        self._newKeys()
+
+    def ssh_SERVICE_REQUEST(self, packet):
+        """
+        Called when we get a MSG_SERVICE_REQUEST message.  Payload::
+            string serviceName
+
+        The client has requested a service.  If we can start the service,
+        start it; otherwise, disconnect with
+        DISCONNECT_SERVICE_NOT_AVAILABLE.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        service, rest = getNS(packet)
+        cls = self.factory.getService(self, service)
+        if not cls:
+            self.sendDisconnect(
+                DISCONNECT_SERVICE_NOT_AVAILABLE, b"don't have service " + service
+            )
+            return
+        else:
+            self.sendPacket(MSG_SERVICE_ACCEPT, NS(service))
+            self.setService(cls())
+
+
+class SSHClientTransport(SSHTransportBase):
+    """
+    SSHClientTransport implements the client side of the SSH protocol.
+
+    @ivar isClient: since we are always the client, this is always True.
+
+    @ivar _gotNewKeys: if we receive a MSG_NEWKEYS message before we are
+        ready to transition to the new keys, this is set to True so we
+        can transition when the keys are ready locally.
+
+    @ivar x: our Diffie-Hellman private key.
+
+    @ivar e: our Diffie-Hellman public key.
+
+    @ivar g: the Diffie-Hellman group generator.
+
+    @ivar p: the Diffie-Hellman group prime
+
+    @ivar instance: the SSHService object we are requesting.
+
+    @ivar _dhMinimalGroupSize: Minimal acceptable group size advertised by the
+        client in MSG_KEX_DH_GEX_REQUEST.
+    @type _dhMinimalGroupSize: int
+
+    @ivar _dhMaximalGroupSize: Maximal acceptable group size advertised by the
+        client in MSG_KEX_DH_GEX_REQUEST.
+    @type _dhMaximalGroupSize: int
+
+    @ivar _dhPreferredGroupSize: Preferred group size advertised by the client
+        in MSG_KEX_DH_GEX_REQUEST.
+    @type _dhPreferredGroupSize: int
+    """
+
+    isClient = True
+
+    # Recommended minimal and maximal values from RFC 4419, 3.
+    _dhMinimalGroupSize = 1024
+    _dhMaximalGroupSize = 8192
+    # FIXME: https://twistedmatrix.com/trac/ticket/8103
+    # This may need to be more dynamic; compare kexgex_client in
+    # OpenSSH.
+    _dhPreferredGroupSize = 2048
+
+    def connectionMade(self):
+        """
+        Called when the connection is started with the server.  Just sets
+        up a private instance variable.
+        """
+        SSHTransportBase.connectionMade(self)
+        self._gotNewKeys = 0
+
+    def ssh_KEXINIT(self, packet):
+        """
+        Called when we receive a MSG_KEXINIT message.  For a description
+        of the packet, see SSHTransportBase.ssh_KEXINIT().  Additionally,
+        this method sends the first key exchange packet.
+
+        If the agreed-upon exchange is ECDH, generate a key pair for the
+        corresponding curve and send the public key.
+
+        If the agreed-upon exchange has a fixed prime/generator group,
+        generate a public key and send it in a MSG_KEXDH_INIT message.
+        Otherwise, ask for a 2048 bit group with a MSG_KEX_DH_GEX_REQUEST
+        message.
+        """
+        if SSHTransportBase.ssh_KEXINIT(self, packet) is None:
+            # Connection was disconnected while doing base processing.
+            # Maybe no common protocols were agreed.
+            return
+        # Are we using ECDH?
+        if _kex.isEllipticCurve(self.kexAlg):
+            # Generate the keys
+            self.ecPriv = self._generateECPrivateKey()
+            self.ecPub = self.ecPriv.public_key()
+
+            # DH_GEX_REQUEST_OLD is the same number we need.
+            self.sendPacket(
+                MSG_KEX_DH_GEX_REQUEST_OLD, NS(self._encodeECPublicKey(self.ecPub))
+            )
+        elif _kex.isFixedGroup(self.kexAlg):
+            # We agreed on a fixed group key exchange algorithm.
+            self.g, self.p = _kex.getDHGeneratorAndPrime(self.kexAlg)
+            self._startEphemeralDH()
+            self.sendPacket(MSG_KEXDH_INIT, self.dhSecretKeyPublicMP)
+        else:
+            # We agreed on a dynamic group. Tell the server what range of
+            # group sizes we accept, and what size we prefer; the server
+            # will then select a group.
+            self.sendPacket(
+                MSG_KEX_DH_GEX_REQUEST,
+                struct.pack(
+                    "!LLL",
+                    self._dhMinimalGroupSize,
+                    self._dhPreferredGroupSize,
+                    self._dhMaximalGroupSize,
+                ),
+            )
+
+    def _ssh_KEX_ECDH_REPLY(self, packet):
+        """
+        Called to handle a reply to a ECDH exchange message(KEX_ECDH_INIT).
+
+        Like the handler for I{KEXDH_INIT}, this message type has an
+        overlapping value.  This method is called from C{ssh_KEX_DH_GEX_GROUP}
+        if that method detects a non-group key exchange is in progress.
+
+        Payload::
+
+            string serverHostKey
+            string server Elliptic Curve Diffie-Hellman public key
+            string signature
+
+        We verify the host key and continue if it passes verificiation.
+        Otherwise raise an exception and return.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+
+        @return: A deferred firing when key exchange is complete.
+        """
+
+        def _continue_KEX_ECDH_REPLY(ignored, hostKey, pubKey, signature):
+            # Save off the host public key.
+            theirECHost = hostKey
+
+            sharedSecret = self._generateECSharedSecret(self.ecPriv, pubKey)
+
+            h = _kex.getHashProcessor(self.kexAlg)()
+            h.update(NS(self.ourVersionString))
+            h.update(NS(self.otherVersionString))
+            h.update(NS(self.ourKexInitPayload))
+            h.update(NS(self.otherKexInitPayload))
+            h.update(NS(theirECHost))
+            h.update(NS(self._encodeECPublicKey(self.ecPub)))
+            h.update(NS(pubKey))
+            h.update(sharedSecret)
+
+            exchangeHash = h.digest()
+
+            if not keys.Key.fromString(theirECHost).verify(signature, exchangeHash):
+                self.sendDisconnect(DISCONNECT_KEY_EXCHANGE_FAILED, b"bad signature")
+            else:
+                self._keySetup(sharedSecret, exchangeHash)
+
+        # Get the host public key,
+        # the raw ECDH public key bytes and the signature
+        hostKey, pubKey, signature, packet = getNS(packet, 3)
+
+        # Easier to comment this out for now than to update all of the tests.
+        # fingerprint = nativeString(base64.b64encode(
+        #        sha256(hostKey).digest()))
+
+        fingerprint = b":".join(
+            [binascii.hexlify(ch) for ch in iterbytes(md5(hostKey).digest())]
+        )
+        d = self.verifyHostKey(hostKey, fingerprint)
+        d.addCallback(_continue_KEX_ECDH_REPLY, hostKey, pubKey, signature)
+        d.addErrback(
+            lambda unused: self.sendDisconnect(
+                DISCONNECT_HOST_KEY_NOT_VERIFIABLE, b"bad host key"
+            )
+        )
+        return d
+
+    def _ssh_KEXDH_REPLY(self, packet):
+        """
+        Called to handle a reply to a non-group key exchange message
+        (KEXDH_INIT).
+
+        Like the handler for I{KEXDH_INIT}, this message type has an
+        overlapping value.  This method is called from C{ssh_KEX_DH_GEX_GROUP}
+        if that method detects a non-group key exchange is in progress.
+
+        Payload::
+
+            string serverHostKey
+            integer f (server Diffie-Hellman public key)
+            string signature
+
+        We verify the host key by calling verifyHostKey, then continue in
+        _continueKEXDH_REPLY.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+
+        @return: A deferred firing when key exchange is complete.
+        """
+        pubKey, packet = getNS(packet)
+        f, packet = getMP(packet)
+        signature, packet = getNS(packet)
+        fingerprint = b":".join(
+            [binascii.hexlify(ch) for ch in iterbytes(md5(pubKey).digest())]
+        )
+        d = self.verifyHostKey(pubKey, fingerprint)
+        d.addCallback(self._continueKEXDH_REPLY, pubKey, f, signature)
+        d.addErrback(
+            lambda unused: self.sendDisconnect(
+                DISCONNECT_HOST_KEY_NOT_VERIFIABLE, b"bad host key"
+            )
+        )
+        return d
+
+    def ssh_KEX_DH_GEX_GROUP(self, packet):
+        """
+        This handles different messages which share an integer value.
+
+        If the key exchange does not have a fixed prime/generator group,
+        we generate a Diffie-Hellman public key and send it in a
+        MSG_KEX_DH_GEX_INIT message.
+
+        Payload::
+            string g (group generator)
+            string p (group prime)
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        if _kex.isFixedGroup(self.kexAlg):
+            return self._ssh_KEXDH_REPLY(packet)
+        elif _kex.isEllipticCurve(self.kexAlg):
+            return self._ssh_KEX_ECDH_REPLY(packet)
+        else:
+            self.p, rest = getMP(packet)
+            self.g, rest = getMP(rest)
+            self._startEphemeralDH()
+            self.sendPacket(MSG_KEX_DH_GEX_INIT, self.dhSecretKeyPublicMP)
+
+    def _continueKEXDH_REPLY(self, ignored, pubKey, f, signature):
+        """
+        The host key has been verified, so we generate the keys.
+
+        @param ignored: Ignored.
+
+        @param pubKey: the public key blob for the server's public key.
+        @type pubKey: L{str}
+        @param f: the server's Diffie-Hellman public key.
+        @type f: L{int}
+        @param signature: the server's signature, verifying that it has the
+            correct private key.
+        @type signature: L{str}
+        """
+        serverKey = keys.Key.fromString(pubKey)
+        sharedSecret = self._finishEphemeralDH(f)
+        h = _kex.getHashProcessor(self.kexAlg)()
+        h.update(NS(self.ourVersionString))
+        h.update(NS(self.otherVersionString))
+        h.update(NS(self.ourKexInitPayload))
+        h.update(NS(self.otherKexInitPayload))
+        h.update(NS(pubKey))
+        h.update(self.dhSecretKeyPublicMP)
+        h.update(MP(f))
+        h.update(sharedSecret)
+        exchangeHash = h.digest()
+        if not serverKey.verify(signature, exchangeHash):
+            self.sendDisconnect(DISCONNECT_KEY_EXCHANGE_FAILED, b"bad signature")
+            return
+        self._keySetup(sharedSecret, exchangeHash)
+
+    def ssh_KEX_DH_GEX_REPLY(self, packet):
+        """
+        Called when we receive a MSG_KEX_DH_GEX_REPLY message.  Payload::
+            string server host key
+            integer f (server DH public key)
+
+        We verify the host key by calling verifyHostKey, then continue in
+        _continueGEX_REPLY.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+
+        @return: A deferred firing once key exchange is complete.
+        """
+        pubKey, packet = getNS(packet)
+        f, packet = getMP(packet)
+        signature, packet = getNS(packet)
+        fingerprint = b":".join(
+            [binascii.hexlify(c) for c in iterbytes(md5(pubKey).digest())]
+        )
+        d = self.verifyHostKey(pubKey, fingerprint)
+        d.addCallback(self._continueGEX_REPLY, pubKey, f, signature)
+        d.addErrback(
+            lambda unused: self.sendDisconnect(
+                DISCONNECT_HOST_KEY_NOT_VERIFIABLE, b"bad host key"
+            )
+        )
+        return d
+
+    def _continueGEX_REPLY(self, ignored, pubKey, f, signature):
+        """
+        The host key has been verified, so we generate the keys.
+
+        @param ignored: Ignored.
+
+        @param pubKey: the public key blob for the server's public key.
+        @type pubKey: L{str}
+        @param f: the server's Diffie-Hellman public key.
+        @type f: L{int}
+        @param signature: the server's signature, verifying that it has the
+            correct private key.
+        @type signature: L{str}
+        """
+        serverKey = keys.Key.fromString(pubKey)
+        sharedSecret = self._finishEphemeralDH(f)
+        h = _kex.getHashProcessor(self.kexAlg)()
+        h.update(NS(self.ourVersionString))
+        h.update(NS(self.otherVersionString))
+        h.update(NS(self.ourKexInitPayload))
+        h.update(NS(self.otherKexInitPayload))
+        h.update(NS(pubKey))
+        h.update(
+            struct.pack(
+                "!LLL",
+                self._dhMinimalGroupSize,
+                self._dhPreferredGroupSize,
+                self._dhMaximalGroupSize,
+            )
+        )
+        h.update(MP(self.p))
+        h.update(MP(self.g))
+        h.update(self.dhSecretKeyPublicMP)
+        h.update(MP(f))
+        h.update(sharedSecret)
+        exchangeHash = h.digest()
+        if not serverKey.verify(signature, exchangeHash):
+            self.sendDisconnect(DISCONNECT_KEY_EXCHANGE_FAILED, b"bad signature")
+            return
+        self._keySetup(sharedSecret, exchangeHash)
+
+    def _keySetup(self, sharedSecret, exchangeHash):
+        """
+        See SSHTransportBase._keySetup().
+        """
+        SSHTransportBase._keySetup(self, sharedSecret, exchangeHash)
+        if self._gotNewKeys:
+            self.ssh_NEWKEYS(b"")
+
+    def ssh_NEWKEYS(self, packet):
+        """
+        Called when we receive a MSG_NEWKEYS message.  No payload.
+        If we've finished setting up our own keys, start using them.
+        Otherwise, remember that we've received this message.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        if packet != b"":
+            self.sendDisconnect(DISCONNECT_PROTOCOL_ERROR, b"NEWKEYS takes no data")
+            return
+        if not self.nextEncryptions.encBlockSize:
+            self._gotNewKeys = 1
+            return
+        self._newKeys()
+        self.connectionSecure()
+
+    def ssh_SERVICE_ACCEPT(self, packet):
+        """
+        Called when we receive a MSG_SERVICE_ACCEPT message.  Payload::
+            string service name
+
+        Start the service we requested.
+
+        @type packet: L{bytes}
+        @param packet: The message data.
+        """
+        if packet == b"":
+            self._log.info("got SERVICE_ACCEPT without payload")
+        else:
+            name = getNS(packet)[0]
+            if name != self.instance.name:
+                self.sendDisconnect(
+                    DISCONNECT_PROTOCOL_ERROR,
+                    b"received accept for service we did not request",
+                )
+        self.setService(self.instance)
+
+    def requestService(self, instance):
+        """
+        Request that a service be run over this transport.
+
+        @type instance: subclass of L{twisted.conch.ssh.service.SSHService}
+        @param instance: The service to run.
+        """
+        self.sendPacket(MSG_SERVICE_REQUEST, NS(instance.name))
+        self.instance = instance
+
+    # Client methods
+
+    def verifyHostKey(self, hostKey, fingerprint):
+        """
+        Returns a Deferred that gets a callback if it is a valid key, or
+        an errback if not.
+
+        @type hostKey: L{bytes}
+        @param hostKey: The host key to verify.
+
+        @type fingerprint: L{bytes}
+        @param fingerprint: The fingerprint of the key.
+
+        @return: A deferred firing with C{True} if the key is valid.
+        """
+        return defer.fail(NotImplementedError())
+
+    def connectionSecure(self):
+        """
+        Called when the encryption has been set up.  Generally,
+        requestService() is called to run another service over the transport.
+        """
+        raise NotImplementedError()
+
+
+class _NullEncryptionContext:
+    """
+    An encryption context that does not actually encrypt anything.
+    """
+
+    def update(self, data):
+        """
+        'Encrypt' new data by doing nothing.
+
+        @type data: L{bytes}
+        @param data: The data to 'encrypt'.
+
+        @rtype: L{bytes}
+        @return: The 'encrypted' data.
+        """
+        return data
+
+
+class _DummyAlgorithm:
+    """
+    An encryption algorithm that does not actually encrypt anything.
+    """
+
+    block_size = 64
+
+
+class _DummyCipher:
+    """
+    A cipher for the none encryption method.
+
+    @ivar block_size: the block size of the encryption.  In the case of the
+    none cipher, this is 8 bytes.
+    """
+
+    algorithm = _DummyAlgorithm()
+
+    def encryptor(self):
+        """
+        Construct a noop encryptor.
+
+        @return: The encryptor.
+        """
+        return _NullEncryptionContext()
+
+    def decryptor(self):
+        """
+        Construct a noop decryptor.
+
+        @return: The decryptor.
+        """
+        return _NullEncryptionContext()
+
+
+DH_GENERATOR, DH_PRIME = _kex.getDHGeneratorAndPrime(b"diffie-hellman-group14-sha1")
+
+
+MSG_DISCONNECT = 1
+MSG_IGNORE = 2
+MSG_UNIMPLEMENTED = 3
+MSG_DEBUG = 4
+MSG_SERVICE_REQUEST = 5
+MSG_SERVICE_ACCEPT = 6
+MSG_EXT_INFO = 7
+MSG_KEXINIT = 20
+MSG_NEWKEYS = 21
+MSG_KEXDH_INIT = 30
+MSG_KEXDH_REPLY = 31
+MSG_KEX_DH_GEX_REQUEST_OLD = 30
+MSG_KEX_DH_GEX_REQUEST = 34
+MSG_KEX_DH_GEX_GROUP = 31
+MSG_KEX_DH_GEX_INIT = 32
+MSG_KEX_DH_GEX_REPLY = 33
+
+
+DISCONNECT_HOST_NOT_ALLOWED_TO_CONNECT = 1
+DISCONNECT_PROTOCOL_ERROR = 2
+DISCONNECT_KEY_EXCHANGE_FAILED = 3
+DISCONNECT_RESERVED = 4
+DISCONNECT_MAC_ERROR = 5
+DISCONNECT_COMPRESSION_ERROR = 6
+DISCONNECT_SERVICE_NOT_AVAILABLE = 7
+DISCONNECT_PROTOCOL_VERSION_NOT_SUPPORTED = 8
+DISCONNECT_HOST_KEY_NOT_VERIFIABLE = 9
+DISCONNECT_CONNECTION_LOST = 10
+DISCONNECT_BY_APPLICATION = 11
+DISCONNECT_TOO_MANY_CONNECTIONS = 12
+DISCONNECT_AUTH_CANCELLED_BY_USER = 13
+DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE = 14
+DISCONNECT_ILLEGAL_USER_NAME = 15
+
+
+messages = {}
+for name, value in list(globals().items()):
+    # Avoid legacy messages which overlap with never ones
+    if name.startswith("MSG_") and not name.startswith("MSG_KEXDH_"):
+        messages[value] = name
+# Check for regressions (#5352)
+if "MSG_KEXDH_INIT" in messages or "MSG_KEXDH_REPLY" in messages:
+    raise RuntimeError("legacy SSH mnemonics should not end up in messages dict")
diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/userauth.py b/contrib/python/Twisted/py3/twisted/conch/ssh/userauth.py
new file mode 100644
index 00000000000..310f5f09f2e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ssh/userauth.py
@@ -0,0 +1,764 @@
+# -*- test-case-name: twisted.conch.test.test_userauth -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of the ssh-userauth service.
+Currently implemented authentication types are public-key and password.
+
+Maintainer: Paul Swartz
+"""
+
+
+import struct
+
+from twisted.conch import error, interfaces
+from twisted.conch.ssh import keys, service, transport
+from twisted.conch.ssh.common import NS, getNS
+from twisted.cred import credentials
+from twisted.cred.error import UnauthorizedLogin
+from twisted.internet import defer, reactor
+from twisted.logger import Logger
+from twisted.python import failure
+from twisted.python.compat import nativeString
+
+
+class SSHUserAuthServer(service.SSHService):
+    """
+    A service implementing the server side of the 'ssh-userauth' service.  It
+    is used to authenticate the user on the other side as being able to access
+    this server.
+
+    @ivar name: the name of this service: 'ssh-userauth'
+    @type name: L{bytes}
+    @ivar authenticatedWith: a list of authentication methods that have
+        already been used.
+    @type authenticatedWith: L{list}
+    @ivar loginTimeout: the number of seconds we wait before disconnecting
+        the user for taking too long to authenticate
+    @type loginTimeout: L{int}
+    @ivar attemptsBeforeDisconnect: the number of failed login attempts we
+        allow before disconnecting.
+    @type attemptsBeforeDisconnect: L{int}
+    @ivar loginAttempts: the number of login attempts that have been made
+    @type loginAttempts: L{int}
+    @ivar passwordDelay: the number of seconds to delay when the user gives
+        an incorrect password
+    @type passwordDelay: L{int}
+    @ivar interfaceToMethod: a L{dict} mapping credential interfaces to
+        authentication methods.  The server checks to see which of the
+        cred interfaces have checkers and tells the client that those methods
+        are valid for authentication.
+    @type interfaceToMethod: L{dict}
+    @ivar supportedAuthentications: A list of the supported authentication
+        methods.
+    @type supportedAuthentications: L{list} of L{bytes}
+    @ivar user: the last username the client tried to authenticate with
+    @type user: L{bytes}
+    @ivar method: the current authentication method
+    @type method: L{bytes}
+    @ivar nextService: the service the user wants started after authentication
+        has been completed.
+    @type nextService: L{bytes}
+    @ivar portal: the L{twisted.cred.portal.Portal} we are using for
+        authentication
+    @type portal: L{twisted.cred.portal.Portal}
+    @ivar clock: an object with a callLater method.  Stubbed out for testing.
+    """
+
+    name = b"ssh-userauth"
+    loginTimeout = 10 * 60 * 60
+    # 10 minutes before we disconnect them
+    attemptsBeforeDisconnect = 20
+    # 20 login attempts before a disconnect
+    passwordDelay = 1  # number of seconds to delay on a failed password
+    clock = reactor
+    interfaceToMethod = {
+        credentials.ISSHPrivateKey: b"publickey",
+        credentials.IUsernamePassword: b"password",
+    }
+    _log = Logger()
+
+    def serviceStarted(self):
+        """
+        Called when the userauth service is started.  Set up instance
+        variables, check if we should allow password authentication (only
+        allow if the outgoing connection is encrypted) and set up a login
+        timeout.
+        """
+        self.authenticatedWith = []
+        self.loginAttempts = 0
+        self.user = None
+        self.nextService = None
+        self.portal = self.transport.factory.portal
+
+        self.supportedAuthentications = []
+        for i in self.portal.listCredentialsInterfaces():
+            if i in self.interfaceToMethod:
+                self.supportedAuthentications.append(self.interfaceToMethod[i])
+
+        if not self.transport.isEncrypted("in"):
+            # don't let us transport password in plaintext
+            if b"password" in self.supportedAuthentications:
+                self.supportedAuthentications.remove(b"password")
+        self._cancelLoginTimeout = self.clock.callLater(
+            self.loginTimeout, self.timeoutAuthentication
+        )
+
+    def serviceStopped(self):
+        """
+        Called when the userauth service is stopped.  Cancel the login timeout
+        if it's still going.
+        """
+        if self._cancelLoginTimeout:
+            self._cancelLoginTimeout.cancel()
+            self._cancelLoginTimeout = None
+
+    def timeoutAuthentication(self):
+        """
+        Called when the user has timed out on authentication.  Disconnect
+        with a DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE message.
+        """
+        self._cancelLoginTimeout = None
+        self.transport.sendDisconnect(
+            transport.DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, b"you took too long"
+        )
+
+    def tryAuth(self, kind, user, data):
+        """
+        Try to authenticate the user with the given method.  Dispatches to a
+        auth_* method.
+
+        @param kind: the authentication method to try.
+        @type kind: L{bytes}
+        @param user: the username the client is authenticating with.
+        @type user: L{bytes}
+        @param data: authentication specific data sent by the client.
+        @type data: L{bytes}
+        @return: A Deferred called back if the method succeeded, or erred back
+            if it failed.
+        @rtype: C{defer.Deferred}
+        """
+        self._log.debug("{user!r} trying auth {kind!r}", user=user, kind=kind)
+        if kind not in self.supportedAuthentications:
+            return defer.fail(error.ConchError("unsupported authentication, failing"))
+        kind = nativeString(kind.replace(b"-", b"_"))
+        f = getattr(self, f"auth_{kind}", None)
+        if f:
+            ret = f(data)
+            if not ret:
+                return defer.fail(
+                    error.ConchError(f"{kind} return None instead of a Deferred")
+                )
+            else:
+                return ret
+        return defer.fail(error.ConchError(f"bad auth type: {kind}"))
+
+    def ssh_USERAUTH_REQUEST(self, packet):
+        """
+        The client has requested authentication.  Payload::
+            string user
+            string next service
+            string method
+            <authentication specific data>
+
+        @type packet: L{bytes}
+        """
+        user, nextService, method, rest = getNS(packet, 3)
+        if user != self.user or nextService != self.nextService:
+            self.authenticatedWith = []  # clear auth state
+        self.user = user
+        self.nextService = nextService
+        self.method = method
+        d = self.tryAuth(method, user, rest)
+        if not d:
+            self._ebBadAuth(failure.Failure(error.ConchError("auth returned none")))
+            return
+        d.addCallback(self._cbFinishedAuth)
+        d.addErrback(self._ebMaybeBadAuth)
+        d.addErrback(self._ebBadAuth)
+        return d
+
+    def _cbFinishedAuth(self, result):
+        """
+        The callback when user has successfully been authenticated.  For a
+        description of the arguments, see L{twisted.cred.portal.Portal.login}.
+        We start the service requested by the user.
+        """
+        (interface, avatar, logout) = result
+        self.transport.avatar = avatar
+        self.transport.logoutFunction = logout
+        service = self.transport.factory.getService(self.transport, self.nextService)
+        if not service:
+            raise error.ConchError(f"could not get next service: {self.nextService}")
+        self._log.debug(
+            "{user!r} authenticated with {method!r}", user=self.user, method=self.method
+        )
+        self.transport.sendPacket(MSG_USERAUTH_SUCCESS, b"")
+        self.transport.setService(service())
+
+    def _ebMaybeBadAuth(self, reason):
+        """
+        An intermediate errback.  If the reason is
+        error.NotEnoughAuthentication, we send a MSG_USERAUTH_FAILURE, but
+        with the partial success indicator set.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+        reason.trap(error.NotEnoughAuthentication)
+        self.transport.sendPacket(
+            MSG_USERAUTH_FAILURE, NS(b",".join(self.supportedAuthentications)) + b"\xff"
+        )
+
+    def _ebBadAuth(self, reason):
+        """
+        The final errback in the authentication chain.  If the reason is
+        error.IgnoreAuthentication, we simply return; the authentication
+        method has sent its own response.  Otherwise, send a failure message
+        and (if the method is not 'none') increment the number of login
+        attempts.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+        if reason.check(error.IgnoreAuthentication):
+            return
+        if self.method != b"none":
+            self._log.debug(
+                "{user!r} failed auth {method!r}", user=self.user, method=self.method
+            )
+            if reason.check(UnauthorizedLogin):
+                self._log.debug(
+                    "unauthorized login: {message}", message=reason.getErrorMessage()
+                )
+            elif reason.check(error.ConchError):
+                self._log.debug("reason: {reason}", reason=reason.getErrorMessage())
+            else:
+                self._log.failure(
+                    "Error checking auth for user {user}",
+                    failure=reason,
+                    user=self.user,
+                )
+            self.loginAttempts += 1
+            if self.loginAttempts > self.attemptsBeforeDisconnect:
+                self.transport.sendDisconnect(
+                    transport.DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE,
+                    b"too many bad auths",
+                )
+                return
+        self.transport.sendPacket(
+            MSG_USERAUTH_FAILURE, NS(b",".join(self.supportedAuthentications)) + b"\x00"
+        )
+
+    def auth_publickey(self, packet):
+        """
+        Public key authentication.  Payload::
+            byte has signature
+            string algorithm name
+            string key blob
+            [string signature] (if has signature is True)
+
+        Create a SSHPublicKey credential and verify it using our portal.
+        """
+        hasSig = ord(packet[0:1])
+        algName, blob, rest = getNS(packet[1:], 2)
+
+        try:
+            keys.Key.fromString(blob)
+        except keys.BadKeyError:
+            error = "Unsupported key type {} or bad key".format(algName.decode("ascii"))
+            self._log.error(error)
+            return defer.fail(UnauthorizedLogin(error))
+
+        signature = hasSig and getNS(rest)[0] or None
+        if hasSig:
+            b = (
+                NS(self.transport.sessionID)
+                + bytes((MSG_USERAUTH_REQUEST,))
+                + NS(self.user)
+                + NS(self.nextService)
+                + NS(b"publickey")
+                + bytes((hasSig,))
+                + NS(algName)
+                + NS(blob)
+            )
+            c = credentials.SSHPrivateKey(self.user, algName, blob, b, signature)
+            return self.portal.login(c, None, interfaces.IConchUser)
+        else:
+            c = credentials.SSHPrivateKey(self.user, algName, blob, None, None)
+            return self.portal.login(c, None, interfaces.IConchUser).addErrback(
+                self._ebCheckKey, packet[1:]
+            )
+
+    def _ebCheckKey(self, reason, packet):
+        """
+        Called back if the user did not sent a signature.  If reason is
+        error.ValidPublicKey then this key is valid for the user to
+        authenticate with.  Send MSG_USERAUTH_PK_OK.
+        """
+        reason.trap(error.ValidPublicKey)
+        # if we make it here, it means that the publickey is valid
+        self.transport.sendPacket(MSG_USERAUTH_PK_OK, packet)
+        return failure.Failure(error.IgnoreAuthentication())
+
+    def auth_password(self, packet):
+        """
+        Password authentication.  Payload::
+            string password
+
+        Make a UsernamePassword credential and verify it with our portal.
+        """
+        password = getNS(packet[1:])[0]
+        c = credentials.UsernamePassword(self.user, password)
+        return self.portal.login(c, None, interfaces.IConchUser).addErrback(
+            self._ebPassword
+        )
+
+    def _ebPassword(self, f):
+        """
+        If the password is invalid, wait before sending the failure in order
+        to delay brute-force password guessing.
+        """
+        d = defer.Deferred()
+        self.clock.callLater(self.passwordDelay, d.callback, f)
+        return d
+
+
+class SSHUserAuthClient(service.SSHService):
+    """
+    A service implementing the client side of 'ssh-userauth'.
+
+    This service will try all authentication methods provided by the server,
+    making callbacks for more information when necessary.
+
+    @ivar name: the name of this service: 'ssh-userauth'
+    @type name: L{str}
+    @ivar preferredOrder: a list of authentication methods that should be used
+        first, in order of preference, if supported by the server
+    @type preferredOrder: L{list}
+    @ivar user: the name of the user to authenticate as
+    @type user: L{bytes}
+    @ivar instance: the service to start after authentication has finished
+    @type instance: L{service.SSHService}
+    @ivar authenticatedWith: a list of strings of authentication methods we've tried
+    @type authenticatedWith: L{list} of L{bytes}
+    @ivar triedPublicKeys: a list of public key objects that we've tried to
+        authenticate with
+    @type triedPublicKeys: L{list} of L{Key}
+    @ivar lastPublicKey: the last public key object we've tried to authenticate
+        with
+    @type lastPublicKey: L{Key}
+    """
+
+    name = b"ssh-userauth"
+    preferredOrder = [b"publickey", b"password", b"keyboard-interactive"]
+
+    def __init__(self, user, instance):
+        self.user = user
+        self.instance = instance
+
+    def serviceStarted(self):
+        self.authenticatedWith = []
+        self.triedPublicKeys = []
+        self.lastPublicKey = None
+        self.askForAuth(b"none", b"")
+
+    def askForAuth(self, kind, extraData):
+        """
+        Send a MSG_USERAUTH_REQUEST.
+
+        @param kind: the authentication method to try.
+        @type kind: L{bytes}
+        @param extraData: method-specific data to go in the packet
+        @type extraData: L{bytes}
+        """
+        self.lastAuth = kind
+        self.transport.sendPacket(
+            MSG_USERAUTH_REQUEST,
+            NS(self.user) + NS(self.instance.name) + NS(kind) + extraData,
+        )
+
+    def tryAuth(self, kind):
+        """
+        Dispatch to an authentication method.
+
+        @param kind: the authentication method
+        @type kind: L{bytes}
+        """
+        kind = nativeString(kind.replace(b"-", b"_"))
+        self._log.debug("trying to auth with {kind}", kind=kind)
+        f = getattr(self, "auth_" + kind, None)
+        if f:
+            return f()
+
+    def _ebAuth(self, ignored, *args):
+        """
+        Generic callback for a failed authentication attempt.  Respond by
+        asking for the list of accepted methods (the 'none' method)
+        """
+        self.askForAuth(b"none", b"")
+
+    def ssh_USERAUTH_SUCCESS(self, packet):
+        """
+        We received a MSG_USERAUTH_SUCCESS.  The server has accepted our
+        authentication, so start the next service.
+        """
+        self.transport.setService(self.instance)
+
+    def ssh_USERAUTH_FAILURE(self, packet):
+        """
+        We received a MSG_USERAUTH_FAILURE.  Payload::
+            string methods
+            byte partial success
+
+        If partial success is C{True}, then the previous method succeeded but is
+        not sufficient for authentication. C{methods} is a comma-separated list
+        of accepted authentication methods.
+
+        We sort the list of methods by their position in C{self.preferredOrder},
+        removing methods that have already succeeded. We then call
+        C{self.tryAuth} with the most preferred method.
+
+        @param packet: the C{MSG_USERAUTH_FAILURE} payload.
+        @type packet: L{bytes}
+
+        @return: a L{defer.Deferred} that will be callbacked with L{None} as
+            soon as all authentication methods have been tried, or L{None} if no
+            more authentication methods are available.
+        @rtype: C{defer.Deferred} or L{None}
+        """
+        canContinue, partial = getNS(packet)
+        partial = ord(partial)
+        if partial:
+            self.authenticatedWith.append(self.lastAuth)
+
+        def orderByPreference(meth):
+            """
+            Invoked once per authentication method in order to extract a
+            comparison key which is then used for sorting.
+
+            @param meth: the authentication method.
+            @type meth: L{bytes}
+
+            @return: the comparison key for C{meth}.
+            @rtype: L{int}
+            """
+            if meth in self.preferredOrder:
+                return self.preferredOrder.index(meth)
+            else:
+                # put the element at the end of the list.
+                return len(self.preferredOrder)
+
+        canContinue = sorted(
+            (
+                meth
+                for meth in canContinue.split(b",")
+                if meth not in self.authenticatedWith
+            ),
+            key=orderByPreference,
+        )
+
+        self._log.debug("can continue with: {methods}", methods=canContinue)
+        return self._cbUserauthFailure(None, iter(canContinue))
+
+    def _cbUserauthFailure(self, result, iterator):
+        if result:
+            return
+        try:
+            method = next(iterator)
+        except StopIteration:
+            self.transport.sendDisconnect(
+                transport.DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE,
+                b"no more authentication methods available",
+            )
+        else:
+            d = defer.maybeDeferred(self.tryAuth, method)
+            d.addCallback(self._cbUserauthFailure, iterator)
+            return d
+
+    def ssh_USERAUTH_PK_OK(self, packet):
+        """
+        This message (number 60) can mean several different messages depending
+        on the current authentication type.  We dispatch to individual methods
+        in order to handle this request.
+        """
+        func = getattr(
+            self,
+            "ssh_USERAUTH_PK_OK_%s" % nativeString(self.lastAuth.replace(b"-", b"_")),
+            None,
+        )
+        if func is not None:
+            return func(packet)
+        else:
+            self.askForAuth(b"none", b"")
+
+    def ssh_USERAUTH_PK_OK_publickey(self, packet):
+        """
+        This is MSG_USERAUTH_PK.  Our public key is valid, so we create a
+        signature and try to authenticate with it.
+        """
+        publicKey = self.lastPublicKey
+        b = (
+            NS(self.transport.sessionID)
+            + bytes((MSG_USERAUTH_REQUEST,))
+            + NS(self.user)
+            + NS(self.instance.name)
+            + NS(b"publickey")
+            + b"\x01"
+            + NS(publicKey.sshType())
+            + NS(publicKey.blob())
+        )
+        d = self.signData(publicKey, b)
+        if not d:
+            self.askForAuth(b"none", b"")
+            # this will fail, we'll move on
+            return
+        d.addCallback(self._cbSignedData)
+        d.addErrback(self._ebAuth)
+
+    def ssh_USERAUTH_PK_OK_password(self, packet):
+        """
+        This is MSG_USERAUTH_PASSWD_CHANGEREQ.  The password given has expired.
+        We ask for an old password and a new password, then send both back to
+        the server.
+        """
+        prompt, language, rest = getNS(packet, 2)
+        self._oldPass = self._newPass = None
+        d = self.getPassword(b"Old Password: ")
+        d = d.addCallbacks(self._setOldPass, self._ebAuth)
+        d.addCallback(lambda ignored: self.getPassword(prompt))
+        d.addCallbacks(self._setNewPass, self._ebAuth)
+
+    def ssh_USERAUTH_PK_OK_keyboard_interactive(self, packet):
+        """
+        This is MSG_USERAUTH_INFO_RESPONSE.  The server has sent us the
+        questions it wants us to answer, so we ask the user and sent the
+        responses.
+        """
+        name, instruction, lang, data = getNS(packet, 3)
+        numPrompts = struct.unpack("!L", data[:4])[0]
+        data = data[4:]
+        prompts = []
+        for i in range(numPrompts):
+            prompt, data = getNS(data)
+            echo = bool(ord(data[0:1]))
+            data = data[1:]
+            prompts.append((prompt, echo))
+        d = self.getGenericAnswers(name, instruction, prompts)
+        d.addCallback(self._cbGenericAnswers)
+        d.addErrback(self._ebAuth)
+
+    def _cbSignedData(self, signedData):
+        """
+        Called back out of self.signData with the signed data.  Send the
+        authentication request with the signature.
+
+        @param signedData: the data signed by the user's private key.
+        @type signedData: L{bytes}
+        """
+        publicKey = self.lastPublicKey
+        self.askForAuth(
+            b"publickey",
+            b"\x01" + NS(publicKey.sshType()) + NS(publicKey.blob()) + NS(signedData),
+        )
+
+    def _setOldPass(self, op):
+        """
+        Called back when we are choosing a new password.  Simply store the old
+        password for now.
+
+        @param op: the old password as entered by the user
+        @type op: L{bytes}
+        """
+        self._oldPass = op
+
+    def _setNewPass(self, np):
+        """
+        Called back when we are choosing a new password.  Get the old password
+        and send the authentication message with both.
+
+        @param np: the new password as entered by the user
+        @type np: L{bytes}
+        """
+        op = self._oldPass
+        self._oldPass = None
+        self.askForAuth(b"password", b"\xff" + NS(op) + NS(np))
+
+    def _cbGenericAnswers(self, responses):
+        """
+        Called back when we are finished answering keyboard-interactive
+        questions.  Send the info back to the server in a
+        MSG_USERAUTH_INFO_RESPONSE.
+
+        @param responses: a list of L{bytes} responses
+        @type responses: L{list}
+        """
+        data = struct.pack("!L", len(responses))
+        for r in responses:
+            data += NS(r.encode("UTF8"))
+        self.transport.sendPacket(MSG_USERAUTH_INFO_RESPONSE, data)
+
+    def auth_publickey(self):
+        """
+        Try to authenticate with a public key.  Ask the user for a public key;
+        if the user has one, send the request to the server and return True.
+        Otherwise, return False.
+
+        @rtype: L{bool}
+        """
+        d = defer.maybeDeferred(self.getPublicKey)
+        d.addBoth(self._cbGetPublicKey)
+        return d
+
+    def _cbGetPublicKey(self, publicKey):
+        if not isinstance(publicKey, keys.Key):  # failure or None
+            publicKey = None
+        if publicKey is not None:
+            self.lastPublicKey = publicKey
+            self.triedPublicKeys.append(publicKey)
+            self._log.debug("using key of type {keyType}", keyType=publicKey.type())
+            self.askForAuth(
+                b"publickey", b"\x00" + NS(publicKey.sshType()) + NS(publicKey.blob())
+            )
+            return True
+        else:
+            return False
+
+    def auth_password(self):
+        """
+        Try to authenticate with a password.  Ask the user for a password.
+        If the user will return a password, return True.  Otherwise, return
+        False.
+
+        @rtype: L{bool}
+        """
+        d = self.getPassword()
+        if d:
+            d.addCallbacks(self._cbPassword, self._ebAuth)
+            return True
+        else:  # returned None, don't do password auth
+            return False
+
+    def auth_keyboard_interactive(self):
+        """
+        Try to authenticate with keyboard-interactive authentication.  Send
+        the request to the server and return True.
+
+        @rtype: L{bool}
+        """
+        self._log.debug("authing with keyboard-interactive")
+        self.askForAuth(b"keyboard-interactive", NS(b"") + NS(b""))
+        return True
+
+    def _cbPassword(self, password):
+        """
+        Called back when the user gives a password.  Send the request to the
+        server.
+
+        @param password: the password the user entered
+        @type password: L{bytes}
+        """
+        self.askForAuth(b"password", b"\x00" + NS(password))
+
+    def signData(self, publicKey, signData):
+        """
+        Sign the given data with the given public key.
+
+        By default, this will call getPrivateKey to get the private key,
+        then sign the data using Key.sign().
+
+        This method is factored out so that it can be overridden to use
+        alternate methods, such as a key agent.
+
+        @param publicKey: The public key object returned from L{getPublicKey}
+        @type publicKey: L{keys.Key}
+
+        @param signData: the data to be signed by the private key.
+        @type signData: L{bytes}
+        @return: a Deferred that's called back with the signature
+        @rtype: L{defer.Deferred}
+        """
+        key = self.getPrivateKey()
+        if not key:
+            return
+        return key.addCallback(self._cbSignData, signData)
+
+    def _cbSignData(self, privateKey, signData):
+        """
+        Called back when the private key is returned.  Sign the data and
+        return the signature.
+
+        @param privateKey: the private key object
+        @type privateKey: L{keys.Key}
+        @param signData: the data to be signed by the private key.
+        @type signData: L{bytes}
+        @return: the signature
+        @rtype: L{bytes}
+        """
+        return privateKey.sign(signData)
+
+    def getPublicKey(self):
+        """
+        Return a public key for the user.  If no more public keys are
+        available, return L{None}.
+
+        This implementation always returns L{None}.  Override it in a
+        subclass to actually find and return a public key object.
+
+        @rtype: L{Key} or L{None}
+        """
+        return None
+
+    def getPrivateKey(self):
+        """
+        Return a L{Deferred} that will be called back with the private key
+        object corresponding to the last public key from getPublicKey().
+        If the private key is not available, errback on the Deferred.
+
+        @rtype: L{Deferred} called back with L{Key}
+        """
+        return defer.fail(NotImplementedError())
+
+    def getPassword(self, prompt=None):
+        """
+        Return a L{Deferred} that will be called back with a password.
+        prompt is a string to display for the password, or None for a generic
+        'user@hostname's password: '.
+
+        @type prompt: L{bytes}/L{None}
+        @rtype: L{defer.Deferred}
+        """
+        return defer.fail(NotImplementedError())
+
+    def getGenericAnswers(self, name, instruction, prompts):
+        """
+        Returns a L{Deferred} with the responses to the promopts.
+
+        @param name: The name of the authentication currently in progress.
+        @param instruction: Describes what the authentication wants.
+        @param prompts: A list of (prompt, echo) pairs, where prompt is a
+        string to display and echo is a boolean indicating whether the
+        user's response should be echoed as they type it.
+        """
+        return defer.fail(NotImplementedError())
+
+
+MSG_USERAUTH_REQUEST = 50
+MSG_USERAUTH_FAILURE = 51
+MSG_USERAUTH_SUCCESS = 52
+MSG_USERAUTH_BANNER = 53
+MSG_USERAUTH_INFO_RESPONSE = 61
+MSG_USERAUTH_PK_OK = 60
+
+messages = {}
+for k, v in list(locals().items()):
+    if k[:4] == "MSG_":
+        messages[v] = k
+
+SSHUserAuthServer.protocolMessages = messages
+SSHUserAuthClient.protocolMessages = messages
+del messages
+del v
+
+# Doubles, not included in the protocols' mappings
+MSG_USERAUTH_PASSWD_CHANGEREQ = 60
+MSG_USERAUTH_INFO_REQUEST = 60
diff --git a/contrib/python/Twisted/py3/twisted/conch/stdio.py b/contrib/python/Twisted/py3/twisted/conch/stdio.py
new file mode 100644
index 00000000000..8b51087d6d8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/stdio.py
@@ -0,0 +1,114 @@
+# -*- test-case-name: twisted.conch.test.test_manhole -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Asynchronous local terminal input handling
+
+@author: Jp Calderone
+"""
+
+import os
+import sys
+import termios
+import tty
+
+from twisted.conch.insults.insults import ServerProtocol
+from twisted.conch.manhole import ColoredManhole
+from twisted.internet import defer, protocol, reactor, stdio
+from twisted.python import failure, log, reflect
+
+
+class UnexpectedOutputError(Exception):
+    pass
+
+
+class TerminalProcessProtocol(protocol.ProcessProtocol):
+    def __init__(self, proto):
+        self.proto = proto
+        self.onConnection = defer.Deferred()
+
+    def connectionMade(self):
+        self.proto.makeConnection(self)
+        self.onConnection.callback(None)
+        self.onConnection = None
+
+    def write(self, data):
+        """
+        Write to the terminal.
+
+        @param data: Data to write.
+        @type data: L{bytes}
+        """
+        self.transport.write(data)
+
+    def outReceived(self, data):
+        """
+        Receive data from the terminal.
+
+        @param data: Data received.
+        @type data: L{bytes}
+        """
+        self.proto.dataReceived(data)
+
+    def errReceived(self, data):
+        """
+        Report an error.
+
+        @param data: Data to include in L{Failure}.
+        @type data: L{bytes}
+        """
+        self.transport.loseConnection()
+        if self.proto is not None:
+            self.proto.connectionLost(failure.Failure(UnexpectedOutputError(data)))
+            self.proto = None
+
+    def childConnectionLost(self, childFD):
+        if self.proto is not None:
+            self.proto.childConnectionLost(childFD)
+
+    def processEnded(self, reason):
+        if self.proto is not None:
+            self.proto.connectionLost(reason)
+            self.proto = None
+
+
+class ConsoleManhole(ColoredManhole):
+    """
+    A manhole protocol specifically for use with L{stdio.StandardIO}.
+    """
+
+    def connectionLost(self, reason):
+        """
+        When the connection is lost, there is nothing more to do.  Stop the
+        reactor so that the process can exit.
+        """
+        reactor.stop()
+
+
+def runWithProtocol(klass):
+    fd = sys.__stdin__.fileno()
+    oldSettings = termios.tcgetattr(fd)
+    tty.setraw(fd)
+    try:
+        stdio.StandardIO(ServerProtocol(klass))
+        reactor.run()
+    finally:
+        termios.tcsetattr(fd, termios.TCSANOW, oldSettings)
+        os.write(fd, b"\r\x1bc\r")
+
+
+def main(argv=None):
+    log.startLogging(open("child.log", "w"))
+
+    if argv is None:
+        argv = sys.argv[1:]
+    if argv:
+        klass = reflect.namedClass(argv[0])
+    else:
+        klass = ConsoleManhole
+    runWithProtocol(klass)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/contrib/python/Twisted/py3/twisted/conch/tap.py b/contrib/python/Twisted/py3/twisted/conch/tap.py
new file mode 100644
index 00000000000..935ad9bf0c9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/tap.py
@@ -0,0 +1,91 @@
+# -*- test-case-name: twisted.conch.test.test_tap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support module for making SSH servers with twistd.
+"""
+
+from twisted.application import strports
+from twisted.conch import checkers as conch_checkers, unix
+from twisted.conch.openssh_compat import factory
+from twisted.cred import portal, strcred
+from twisted.python import usage
+
+
+class Options(usage.Options, strcred.AuthOptionMixin):
+    synopsis = "[-i <interface>] [-p <port>] [-d <dir>] "
+    longdesc = (
+        "Makes a Conch SSH server.  If no authentication methods are "
+        "specified, the default authentication methods are UNIX passwords "
+        "and SSH public keys.  If --auth options are "
+        "passed, only the measures specified will be used."
+    )
+    optParameters = [
+        ["interface", "i", "", "local interface to which we listen"],
+        ["port", "p", "tcp:22", "Port on which to listen"],
+        ["data", "d", "/etc", "directory to look for host keys in"],
+        [
+            "moduli",
+            "",
+            None,
+            "directory to look for moduli in " "(if different from --data)",
+        ],
+    ]
+    compData = usage.Completions(
+        optActions={
+            "data": usage.CompleteDirs(descr="data directory"),
+            "moduli": usage.CompleteDirs(descr="moduli directory"),
+            "interface": usage.CompleteNetInterfaces(),
+        }
+    )
+
+    def __init__(self, *a, **kw):
+        usage.Options.__init__(self, *a, **kw)
+
+        # Call the default addCheckers (for backwards compatibility) that will
+        # be used if no --auth option is provided - note that conch's
+        # UNIXPasswordDatabase is used, instead of twisted.plugins.cred_unix's
+        # checker
+        super().addChecker(conch_checkers.UNIXPasswordDatabase())
+        super().addChecker(
+            conch_checkers.SSHPublicKeyChecker(conch_checkers.UNIXAuthorizedKeysFiles())
+        )
+        self._usingDefaultAuth = True
+
+    def addChecker(self, checker):
+        """
+        Add the checker specified.  If any checkers are added, the default
+        checkers are automatically cleared and the only checkers will be the
+        specified one(s).
+        """
+        if self._usingDefaultAuth:
+            self["credCheckers"] = []
+            self["credInterfaces"] = {}
+            self._usingDefaultAuth = False
+        super().addChecker(checker)
+
+
+def makeService(config):
+    """
+    Construct a service for operating a SSH server.
+
+    @param config: An L{Options} instance specifying server options, including
+        where server keys are stored and what authentication methods to use.
+
+    @return: A L{twisted.application.service.IService} provider which contains
+        the requested SSH server.
+    """
+
+    t = factory.OpenSSHFactory()
+
+    r = unix.UnixSSHRealm()
+    t.portal = portal.Portal(r, config.get("credCheckers", []))
+    t.dataRoot = config["data"]
+    t.moduliRoot = config["moduli"] or config["data"]
+
+    port = config["port"]
+    if config["interface"]:
+        # Add warning here
+        port += ":interface=" + config["interface"]
+    return strports.service(port, t)
diff --git a/contrib/python/Twisted/py3/twisted/conch/telnet.py b/contrib/python/Twisted/py3/twisted/conch/telnet.py
new file mode 100644
index 00000000000..4221bf90acd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/telnet.py
@@ -0,0 +1,1144 @@
+# -*- test-case-name: twisted.conch.test.test_telnet -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Telnet protocol implementation.
+
+@author: Jean-Paul Calderone
+"""
+
+
+import struct
+
+from zope.interface import implementer
+
+from twisted.internet import defer, interfaces as iinternet, protocol
+from twisted.logger import Logger
+from twisted.python.compat import iterbytes
+
+
+def _chr(i: int) -> bytes:
+    """Create a byte sequence of length 1.
+
+    U{RFC 854<https://tools.ietf.org/html/rfc854>} specifies codes in decimal,
+    but Python can only handle L{bytes} literals in octal or hexadecimal.
+    This helper function bridges that gap.
+
+    @param i: The value of the only byte in the sequence.
+    """
+    return bytes((i,))
+
+
+MODE = _chr(1)
+EDIT = 1
+TRAPSIG = 2
+MODE_ACK = 4
+SOFT_TAB = 8
+LIT_ECHO = 16
+
+# Characters gleaned from the various (and conflicting) RFCs. Not all of these
+# are correct.
+
+NULL = _chr(0)  # No operation.
+BEL = _chr(7)  # Produces an audible or visible signal (which does NOT move the
+# print head).
+BS = _chr(8)  # Moves the print head one character position towards the left
+# margin.
+HT = _chr(9)  # Moves the printer to the next horizontal tab stop. It remains
+# unspecified how either party determines or establishes where such tab stops
+# are located.
+LF = _chr(10)  # Moves the printer to the next print line, keeping the same
+# horizontal position.
+VT = _chr(11)  # Moves the printer to the next vertical tab stop. It remains
+# unspecified how either party determines or establishes where such tab stops
+# are located.
+FF = _chr(12)  # Moves the printer to the top of the next page, keeping the same
+# horizontal position.
+CR = _chr(13)  # Moves the printer to the left margin of the current line.
+
+ECHO = _chr(1)  # User-to-Server:  Asks the server to send Echos of the
+# transmitted data.
+SGA = _chr(3)  # Suppress Go Ahead.  Go Ahead is silly and most modern servers
+# should suppress it.
+NAWS = _chr(31)  # Negotiate About Window Size.  Indicate that information about
+# the size of the terminal can be communicated.
+LINEMODE = _chr(34)  # Allow line buffering to be negotiated about.
+
+EOR = _chr(239)  # End of Record (RFC 885)
+SE = _chr(240)  # End of subnegotiation parameters.
+NOP = _chr(241)  # No operation.
+DM = _chr(242)  # "Data Mark": The data stream portion of a Synch. This should
+# always be accompanied by a TCP Urgent notification.
+BRK = _chr(243)  # NVT character Break.
+IP = _chr(244)  # The function Interrupt Process.
+AO = _chr(245)  # The function Abort Output
+AYT = _chr(246)  # The function Are You There.
+EC = _chr(247)  # The function Erase Character.
+EL = _chr(248)  # The function Erase Line
+GA = _chr(249)  # The Go Ahead signal.
+SB = _chr(250)  # Indicates that what follows is subnegotiation of the indicated
+# option.
+WILL = _chr(251)  # Indicates the desire to begin performing, or confirmation
+# that you are now performing, the indicated option.
+WONT = _chr(252)  # Indicates the refusal to perform, or continue performing,
+# the indicated option.
+DO = _chr(253)  # Indicates the request that the other party perform, or
+# confirmation that you are expecting the other party to perform, the indicated
+# option.
+DONT = _chr(254)  # Indicates the demand that the other party stop performing,
+# or confirmation that you are no longer expecting the other party to perform,
+# the indicated option.
+IAC = _chr(255)  # Data Byte 255. Introduces a telnet command.
+
+LINEMODE_MODE = _chr(1)
+LINEMODE_EDIT = _chr(1)
+LINEMODE_TRAPSIG = _chr(2)
+LINEMODE_MODE_ACK = _chr(4)
+LINEMODE_SOFT_TAB = _chr(8)
+LINEMODE_LIT_ECHO = _chr(16)
+LINEMODE_FORWARDMASK = _chr(2)
+LINEMODE_SLC = _chr(3)
+LINEMODE_SLC_SYNCH = _chr(1)
+LINEMODE_SLC_BRK = _chr(2)
+LINEMODE_SLC_IP = _chr(3)
+LINEMODE_SLC_AO = _chr(4)
+LINEMODE_SLC_AYT = _chr(5)
+LINEMODE_SLC_EOR = _chr(6)
+LINEMODE_SLC_ABORT = _chr(7)
+LINEMODE_SLC_EOF = _chr(8)
+LINEMODE_SLC_SUSP = _chr(9)
+LINEMODE_SLC_EC = _chr(10)
+LINEMODE_SLC_EL = _chr(11)
+
+LINEMODE_SLC_EW = _chr(12)
+LINEMODE_SLC_RP = _chr(13)
+LINEMODE_SLC_LNEXT = _chr(14)
+LINEMODE_SLC_XON = _chr(15)
+LINEMODE_SLC_XOFF = _chr(16)
+LINEMODE_SLC_FORW1 = _chr(17)
+LINEMODE_SLC_FORW2 = _chr(18)
+LINEMODE_SLC_MCL = _chr(19)
+LINEMODE_SLC_MCR = _chr(20)
+LINEMODE_SLC_MCWL = _chr(21)
+LINEMODE_SLC_MCWR = _chr(22)
+LINEMODE_SLC_MCBOL = _chr(23)
+LINEMODE_SLC_MCEOL = _chr(24)
+LINEMODE_SLC_INSRT = _chr(25)
+LINEMODE_SLC_OVER = _chr(26)
+LINEMODE_SLC_ECR = _chr(27)
+LINEMODE_SLC_EWR = _chr(28)
+LINEMODE_SLC_EBOL = _chr(29)
+LINEMODE_SLC_EEOL = _chr(30)
+
+LINEMODE_SLC_DEFAULT = _chr(3)
+LINEMODE_SLC_VALUE = _chr(2)
+LINEMODE_SLC_CANTCHANGE = _chr(1)
+LINEMODE_SLC_NOSUPPORT = _chr(0)
+LINEMODE_SLC_LEVELBITS = _chr(3)
+
+LINEMODE_SLC_ACK = _chr(128)
+LINEMODE_SLC_FLUSHIN = _chr(64)
+LINEMODE_SLC_FLUSHOUT = _chr(32)
+LINEMODE_EOF = _chr(236)
+LINEMODE_SUSP = _chr(237)
+LINEMODE_ABORT = _chr(238)
+
+
+class ITelnetProtocol(iinternet.IProtocol):
+    def unhandledCommand(command, argument):
+        """
+        A command was received but not understood.
+
+        @param command: the command received.
+        @type command: L{str}, a single character.
+        @param argument: the argument to the received command.
+        @type argument: L{str}, a single character, or None if the command that
+            was unhandled does not provide an argument.
+        """
+
+    def unhandledSubnegotiation(command, data):
+        """
+        A subnegotiation command was received but not understood.
+
+        @param command: the command being subnegotiated. That is, the first
+            byte after the SB command.
+        @type command: L{str}, a single character.
+        @param data: all other bytes of the subneogation. That is, all but the
+            first bytes between SB and SE, with IAC un-escaping applied.
+        @type data: L{bytes}, each a single character
+        """
+
+    def enableLocal(option):
+        """
+        Enable the given option locally.
+
+        This should enable the given option on this side of the
+        telnet connection and return True.  If False is returned,
+        the option will be treated as still disabled and the peer
+        will be notified.
+
+        @param option: the option to be enabled.
+        @type option: L{bytes}, a single character.
+        """
+
+    def enableRemote(option):
+        """
+        Indicate whether the peer should be allowed to enable this option.
+
+        Returns True if the peer should be allowed to enable this option,
+        False otherwise.
+
+        @param option: the option to be enabled.
+        @type option: L{bytes}, a single character.
+        """
+
+    def disableLocal(option):
+        """
+        Disable the given option locally.
+
+        Unlike enableLocal, this method cannot fail.  The option must be
+        disabled.
+
+        @param option: the option to be disabled.
+        @type option: L{bytes}, a single character.
+        """
+
+    def disableRemote(option):
+        """
+        Indicate that the peer has disabled this option.
+
+        @param option: the option to be disabled.
+        @type option: L{bytes}, a single character.
+        """
+
+
+class ITelnetTransport(iinternet.ITransport):
+    def do(option):
+        """
+        Indicate a desire for the peer to begin performing the given option.
+
+        Returns a Deferred that fires with True when the peer begins performing
+        the option, or fails with L{OptionRefused} when the peer refuses to
+        perform it.  If the peer is already performing the given option, the
+        Deferred will fail with L{AlreadyEnabled}.  If a negotiation regarding
+        this option is already in progress, the Deferred will fail with
+        L{AlreadyNegotiating}.
+
+        Note: It is currently possible that this Deferred will never fire,
+        if the peer never responds, or if the peer believes the option to
+        already be enabled.
+        """
+
+    def dont(option):
+        """
+        Indicate a desire for the peer to cease performing the given option.
+
+        Returns a Deferred that fires with True when the peer ceases performing
+        the option.  If the peer is not performing the given option, the
+        Deferred will fail with L{AlreadyDisabled}.  If negotiation regarding
+        this option is already in progress, the Deferred will fail with
+        L{AlreadyNegotiating}.
+
+        Note: It is currently possible that this Deferred will never fire,
+        if the peer never responds, or if the peer believes the option to
+        already be disabled.
+        """
+
+    def will(option):
+        """
+        Indicate our willingness to begin performing this option locally.
+
+        Returns a Deferred that fires with True when the peer agrees to allow us
+        to begin performing this option, or fails with L{OptionRefused} if the
+        peer refuses to allow us to begin performing it.  If the option is
+        already enabled locally, the Deferred will fail with L{AlreadyEnabled}.
+        If negotiation regarding this option is already in progress, the
+        Deferred will fail with L{AlreadyNegotiating}.
+
+        Note: It is currently possible that this Deferred will never fire,
+        if the peer never responds, or if the peer believes the option to
+        already be enabled.
+        """
+
+    def wont(option):
+        """
+        Indicate that we will stop performing the given option.
+
+        Returns a Deferred that fires with True when the peer acknowledges
+        we have stopped performing this option.  If the option is already
+        disabled locally, the Deferred will fail with L{AlreadyDisabled}.
+        If negotiation regarding this option is already in progress,
+        the Deferred will fail with L{AlreadyNegotiating}.
+
+        Note: It is currently possible that this Deferred will never fire,
+        if the peer never responds, or if the peer believes the option to
+        already be disabled.
+        """
+
+    def requestNegotiation(about, data):
+        """
+        Send a subnegotiation request.
+
+        @param about: A byte indicating the feature being negotiated.
+        @param data: Any number of L{bytes} containing specific information
+        about the negotiation being requested.  No values in this string
+        need to be escaped, as this function will escape any value which
+        requires it.
+        """
+
+
+class TelnetError(Exception):
+    pass
+
+
+class NegotiationError(TelnetError):
+    def __str__(self) -> str:
+        return (
+            self.__class__.__module__
+            + "."
+            + self.__class__.__name__
+            + ":"
+            + repr(self.args[0])
+        )
+
+
+class OptionRefused(NegotiationError):
+    pass
+
+
+class AlreadyEnabled(NegotiationError):
+    pass
+
+
+class AlreadyDisabled(NegotiationError):
+    pass
+
+
+class AlreadyNegotiating(NegotiationError):
+    pass
+
+
+@implementer(ITelnetProtocol)
+class TelnetProtocol(protocol.Protocol):
+    _log = Logger()
+
+    def unhandledCommand(self, command, argument):
+        pass
+
+    def unhandledSubnegotiation(self, command, data):
+        pass
+
+    def enableLocal(self, option):
+        pass
+
+    def enableRemote(self, option):
+        pass
+
+    def disableLocal(self, option):
+        pass
+
+    def disableRemote(self, option):
+        pass
+
+
+class Telnet(protocol.Protocol):
+    """
+    @ivar commandMap: A mapping of bytes to callables.  When a
+    telnet command is received, the command byte (the first byte
+    after IAC) is looked up in this dictionary.  If a callable is
+    found, it is invoked with the argument of the command, or None
+    if the command takes no argument.  Values should be added to
+    this dictionary if commands wish to be handled.  By default,
+    only WILL, WONT, DO, and DONT are handled.  These should not
+    be overridden, as this class handles them correctly and
+    provides an API for interacting with them.
+
+    @ivar negotiationMap: A mapping of bytes to callables.  When
+    a subnegotiation command is received, the command byte (the
+    first byte after SB) is looked up in this dictionary.  If
+    a callable is found, it is invoked with the argument of the
+    subnegotiation.  Values should be added to this dictionary if
+    subnegotiations are to be handled.  By default, no values are
+    handled.
+
+    @ivar options: A mapping of option bytes to their current
+    state.  This state is likely of little use to user code.
+    Changes should not be made to it.
+
+    @ivar state: A string indicating the current parse state.  It
+    can take on the values "data", "escaped", "command", "newline",
+    "subnegotiation", and "subnegotiation-escaped".  Changes
+    should not be made to it.
+
+    @ivar transport: This protocol's transport object.
+    """
+
+    # One of a lot of things
+    state = "data"
+
+    def __init__(self):
+        self.options = {}
+        self.negotiationMap = {}
+        self.commandMap = {
+            WILL: self.telnet_WILL,
+            WONT: self.telnet_WONT,
+            DO: self.telnet_DO,
+            DONT: self.telnet_DONT,
+        }
+
+    def _write(self, data):
+        self.transport.write(data)
+
+    class _OptionState:
+        """
+        Represents the state of an option on both sides of a telnet
+        connection.
+
+        @ivar us: The state of the option on this side of the connection.
+
+        @ivar him: The state of the option on the other side of the
+            connection.
+        """
+
+        class _Perspective:
+            """
+            Represents the state of an option on side of the telnet
+            connection.  Some options can be enabled on a particular side of
+            the connection (RFC 1073 for example: only the client can have
+            NAWS enabled).  Other options can be enabled on either or both
+            sides (such as RFC 1372: each side can have its own flow control
+            state).
+
+            @ivar state: C{'yes'} or C{'no'} indicating whether or not this
+                option is enabled on one side of the connection.
+
+            @ivar negotiating: A boolean tracking whether negotiation about
+                this option is in progress.
+
+            @ivar onResult: When negotiation about this option has been
+                initiated by this side of the connection, a L{Deferred}
+                which will fire with the result of the negotiation.  L{None}
+                at other times.
+            """
+
+            state = "no"
+            negotiating = False
+            onResult = None
+
+            def __str__(self) -> str:
+                return self.state + ("*" * self.negotiating)
+
+        def __init__(self):
+            self.us = self._Perspective()
+            self.him = self._Perspective()
+
+        def __repr__(self) -> str:
+            return f"<_OptionState us={self.us} him={self.him}>"
+
+    def getOptionState(self, opt):
+        return self.options.setdefault(opt, self._OptionState())
+
+    def _do(self, option):
+        self._write(IAC + DO + option)
+
+    def _dont(self, option):
+        self._write(IAC + DONT + option)
+
+    def _will(self, option):
+        self._write(IAC + WILL + option)
+
+    def _wont(self, option):
+        self._write(IAC + WONT + option)
+
+    def will(self, option):
+        """
+        Indicate our willingness to enable an option.
+        """
+        s = self.getOptionState(option)
+        if s.us.negotiating or s.him.negotiating:
+            return defer.fail(AlreadyNegotiating(option))
+        elif s.us.state == "yes":
+            return defer.fail(AlreadyEnabled(option))
+        else:
+            s.us.negotiating = True
+            s.us.onResult = d = defer.Deferred()
+            self._will(option)
+            return d
+
+    def wont(self, option):
+        """
+        Indicate we are not willing to enable an option.
+        """
+        s = self.getOptionState(option)
+        if s.us.negotiating or s.him.negotiating:
+            return defer.fail(AlreadyNegotiating(option))
+        elif s.us.state == "no":
+            return defer.fail(AlreadyDisabled(option))
+        else:
+            s.us.negotiating = True
+            s.us.onResult = d = defer.Deferred()
+            self._wont(option)
+            return d
+
+    def do(self, option):
+        s = self.getOptionState(option)
+        if s.us.negotiating or s.him.negotiating:
+            return defer.fail(AlreadyNegotiating(option))
+        elif s.him.state == "yes":
+            return defer.fail(AlreadyEnabled(option))
+        else:
+            s.him.negotiating = True
+            s.him.onResult = d = defer.Deferred()
+            self._do(option)
+            return d
+
+    def dont(self, option):
+        s = self.getOptionState(option)
+        if s.us.negotiating or s.him.negotiating:
+            return defer.fail(AlreadyNegotiating(option))
+        elif s.him.state == "no":
+            return defer.fail(AlreadyDisabled(option))
+        else:
+            s.him.negotiating = True
+            s.him.onResult = d = defer.Deferred()
+            self._dont(option)
+            return d
+
+    def requestNegotiation(self, about, data):
+        """
+        Send a negotiation message for the option C{about} with C{data} as the
+        payload.
+
+        @param data: the payload
+        @type data: L{bytes}
+        @see: L{ITelnetTransport.requestNegotiation}
+        """
+        data = data.replace(IAC, IAC * 2)
+        self._write(IAC + SB + about + data + IAC + SE)
+
+    def dataReceived(self, data):
+        appDataBuffer = []
+
+        for b in iterbytes(data):
+            if self.state == "data":
+                if b == IAC:
+                    self.state = "escaped"
+                elif b == b"\r":
+                    self.state = "newline"
+                else:
+                    appDataBuffer.append(b)
+            elif self.state == "escaped":
+                if b == IAC:
+                    appDataBuffer.append(b)
+                    self.state = "data"
+                elif b == SB:
+                    self.state = "subnegotiation"
+                    self.commands = []
+                elif b in (EOR, NOP, DM, BRK, IP, AO, AYT, EC, EL, GA):
+                    self.state = "data"
+                    if appDataBuffer:
+                        self.applicationDataReceived(b"".join(appDataBuffer))
+                        del appDataBuffer[:]
+                    self.commandReceived(b, None)
+                elif b in (WILL, WONT, DO, DONT):
+                    self.state = "command"
+                    self.command = b
+                else:
+                    raise ValueError("Stumped", b)
+            elif self.state == "command":
+                self.state = "data"
+                command = self.command
+                del self.command
+                if appDataBuffer:
+                    self.applicationDataReceived(b"".join(appDataBuffer))
+                    del appDataBuffer[:]
+                self.commandReceived(command, b)
+            elif self.state == "newline":
+                self.state = "data"
+                if b == b"\n":
+                    appDataBuffer.append(b"\n")
+                elif b == b"\0":
+                    appDataBuffer.append(b"\r")
+                elif b == IAC:
+                    # IAC isn't really allowed after \r, according to the
+                    # RFC, but handling it this way is less surprising than
+                    # delivering the IAC to the app as application data.
+                    # The purpose of the restriction is to allow terminals
+                    # to unambiguously interpret the behavior of the CR
+                    # after reading only one more byte.  CR LF is supposed
+                    # to mean one thing (cursor to next line, first column),
+                    # CR NUL another (cursor to first column).  Absent the
+                    # NUL, it still makes sense to interpret this as CR and
+                    # then apply all the usual interpretation to the IAC.
+                    appDataBuffer.append(b"\r")
+                    self.state = "escaped"
+                else:
+                    appDataBuffer.append(b"\r" + b)
+            elif self.state == "subnegotiation":
+                if b == IAC:
+                    self.state = "subnegotiation-escaped"
+                else:
+                    self.commands.append(b)
+            elif self.state == "subnegotiation-escaped":
+                if b == SE:
+                    self.state = "data"
+                    commands = self.commands
+                    del self.commands
+                    if appDataBuffer:
+                        self.applicationDataReceived(b"".join(appDataBuffer))
+                        del appDataBuffer[:]
+                    self.negotiate(commands)
+                else:
+                    self.state = "subnegotiation"
+                    self.commands.append(b)
+            else:
+                raise ValueError("How'd you do this?")
+
+        if appDataBuffer:
+            self.applicationDataReceived(b"".join(appDataBuffer))
+
+    def connectionLost(self, reason):
+        for state in self.options.values():
+            if state.us.onResult is not None:
+                d = state.us.onResult
+                state.us.onResult = None
+                d.errback(reason)
+            if state.him.onResult is not None:
+                d = state.him.onResult
+                state.him.onResult = None
+                d.errback(reason)
+
+    def applicationDataReceived(self, data):
+        """
+        Called with application-level data.
+        """
+
+    def unhandledCommand(self, command, argument):
+        """
+        Called for commands for which no handler is installed.
+        """
+
+    def commandReceived(self, command, argument):
+        cmdFunc = self.commandMap.get(command)
+        if cmdFunc is None:
+            self.unhandledCommand(command, argument)
+        else:
+            cmdFunc(argument)
+
+    def unhandledSubnegotiation(self, command, data):
+        """
+        Called for subnegotiations for which no handler is installed.
+        """
+
+    def negotiate(self, data):
+        command, data = data[0], data[1:]
+        cmdFunc = self.negotiationMap.get(command)
+        if cmdFunc is None:
+            self.unhandledSubnegotiation(command, data)
+        else:
+            cmdFunc(data)
+
+    def telnet_WILL(self, option):
+        s = self.getOptionState(option)
+        self.willMap[s.him.state, s.him.negotiating](self, s, option)
+
+    def will_no_false(self, state, option):
+        # He is unilaterally offering to enable an option.
+        if self.enableRemote(option):
+            state.him.state = "yes"
+            self._do(option)
+        else:
+            self._dont(option)
+
+    def will_no_true(self, state, option):
+        # Peer agreed to enable an option in response to our request.
+        state.him.state = "yes"
+        state.him.negotiating = False
+        d = state.him.onResult
+        state.him.onResult = None
+        d.callback(True)
+        assert self.enableRemote(
+            option
+        ), "enableRemote must return True in this context (for option {!r})".format(
+            option
+        )
+
+    def will_yes_false(self, state, option):
+        # He is unilaterally offering to enable an already-enabled option.
+        # Ignore this.
+        pass
+
+    def will_yes_true(self, state, option):
+        # This is a bogus state.  It is here for completeness.  It will
+        # never be entered.
+        assert (
+            False
+        ), "will_yes_true can never be entered, but was called with {!r}, {!r}".format(
+            state,
+            option,
+        )
+
+    willMap = {
+        ("no", False): will_no_false,
+        ("no", True): will_no_true,
+        ("yes", False): will_yes_false,
+        ("yes", True): will_yes_true,
+    }
+
+    def telnet_WONT(self, option):
+        s = self.getOptionState(option)
+        self.wontMap[s.him.state, s.him.negotiating](self, s, option)
+
+    def wont_no_false(self, state, option):
+        # He is unilaterally demanding that an already-disabled option be/remain disabled.
+        # Ignore this (although we could record it and refuse subsequent enable attempts
+        # from our side - he can always refuse them again though, so we won't)
+        pass
+
+    def wont_no_true(self, state, option):
+        # Peer refused to enable an option in response to our request.
+        state.him.negotiating = False
+        d = state.him.onResult
+        state.him.onResult = None
+        d.errback(OptionRefused(option))
+
+    def wont_yes_false(self, state, option):
+        # Peer is unilaterally demanding that an option be disabled.
+        state.him.state = "no"
+        self.disableRemote(option)
+        self._dont(option)
+
+    def wont_yes_true(self, state, option):
+        # Peer agreed to disable an option at our request.
+        state.him.state = "no"
+        state.him.negotiating = False
+        d = state.him.onResult
+        state.him.onResult = None
+        d.callback(True)
+        self.disableRemote(option)
+
+    wontMap = {
+        ("no", False): wont_no_false,
+        ("no", True): wont_no_true,
+        ("yes", False): wont_yes_false,
+        ("yes", True): wont_yes_true,
+    }
+
+    def telnet_DO(self, option):
+        s = self.getOptionState(option)
+        self.doMap[s.us.state, s.us.negotiating](self, s, option)
+
+    def do_no_false(self, state, option):
+        # Peer is unilaterally requesting that we enable an option.
+        if self.enableLocal(option):
+            state.us.state = "yes"
+            self._will(option)
+        else:
+            self._wont(option)
+
+    def do_no_true(self, state, option):
+        # Peer agreed to allow us to enable an option at our request.
+        state.us.state = "yes"
+        state.us.negotiating = False
+        d = state.us.onResult
+        state.us.onResult = None
+        d.callback(True)
+        self.enableLocal(option)
+
+    def do_yes_false(self, state, option):
+        # Peer is unilaterally requesting us to enable an already-enabled option.
+        # Ignore this.
+        pass
+
+    def do_yes_true(self, state, option):
+        # This is a bogus state.  It is here for completeness.  It will never be
+        # entered.
+        assert (
+            False
+        ), "do_yes_true can never be entered, but was called with {!r}, {!r}".format(
+            state,
+            option,
+        )
+
+    doMap = {
+        ("no", False): do_no_false,
+        ("no", True): do_no_true,
+        ("yes", False): do_yes_false,
+        ("yes", True): do_yes_true,
+    }
+
+    def telnet_DONT(self, option):
+        s = self.getOptionState(option)
+        self.dontMap[s.us.state, s.us.negotiating](self, s, option)
+
+    def dont_no_false(self, state, option):
+        # Peer is unilaterally demanding us to disable an already-disabled option.
+        # Ignore this.
+        pass
+
+    def dont_no_true(self, state, option):
+        # Offered option was refused.  Fail the Deferred returned by the
+        # previous will() call.
+        state.us.negotiating = False
+        d = state.us.onResult
+        state.us.onResult = None
+        d.errback(OptionRefused(option))
+
+    def dont_yes_false(self, state, option):
+        # Peer is unilaterally demanding we disable an option.
+        state.us.state = "no"
+        self.disableLocal(option)
+        self._wont(option)
+
+    def dont_yes_true(self, state, option):
+        # Peer acknowledged our notice that we will disable an option.
+        state.us.state = "no"
+        state.us.negotiating = False
+        d = state.us.onResult
+        state.us.onResult = None
+        d.callback(True)
+        self.disableLocal(option)
+
+    dontMap = {
+        ("no", False): dont_no_false,
+        ("no", True): dont_no_true,
+        ("yes", False): dont_yes_false,
+        ("yes", True): dont_yes_true,
+    }
+
+    def enableLocal(self, option):
+        """
+        Reject all attempts to enable options.
+        """
+        return False
+
+    def enableRemote(self, option):
+        """
+        Reject all attempts to enable options.
+        """
+        return False
+
+    def disableLocal(self, option):
+        """
+        Signal a programming error by raising an exception.
+
+        L{enableLocal} must return true for the given value of C{option} in
+        order for this method to be called.  If a subclass of L{Telnet}
+        overrides enableLocal to allow certain options to be enabled, it must
+        also override disableLocal to disable those options.
+
+        @raise NotImplementedError: Always raised.
+        """
+        raise NotImplementedError(
+            f"Don't know how to disable local telnet option {option!r}"
+        )
+
+    def disableRemote(self, option):
+        """
+        Signal a programming error by raising an exception.
+
+        L{enableRemote} must return true for the given value of C{option} in
+        order for this method to be called.  If a subclass of L{Telnet}
+        overrides enableRemote to allow certain options to be enabled, it must
+        also override disableRemote tto disable those options.
+
+        @raise NotImplementedError: Always raised.
+        """
+        raise NotImplementedError(
+            f"Don't know how to disable remote telnet option {option!r}"
+        )
+
+
+class ProtocolTransportMixin:
+    def write(self, data):
+        self.transport.write(data.replace(b"\n", b"\r\n"))
+
+    def writeSequence(self, seq):
+        self.transport.writeSequence(seq)
+
+    def loseConnection(self):
+        self.transport.loseConnection()
+
+    def getHost(self):
+        return self.transport.getHost()
+
+    def getPeer(self):
+        return self.transport.getPeer()
+
+
+class TelnetTransport(Telnet, ProtocolTransportMixin):
+    """
+    @ivar protocol: An instance of the protocol to which this
+    transport is connected, or None before the connection is
+    established and after it is lost.
+
+    @ivar protocolFactory: A callable which returns protocol instances
+    which provide L{ITelnetProtocol}.  This will be invoked when a
+    connection is established.  It is passed *protocolArgs and
+    **protocolKwArgs.
+
+    @ivar protocolArgs: A tuple of additional arguments to
+    pass to protocolFactory.
+
+    @ivar protocolKwArgs: A dictionary of additional arguments
+    to pass to protocolFactory.
+    """
+
+    disconnecting = False
+
+    protocolFactory = None
+    protocol = None
+
+    def __init__(self, protocolFactory=None, *a, **kw):
+        Telnet.__init__(self)
+        if protocolFactory is not None:
+            self.protocolFactory = protocolFactory
+            self.protocolArgs = a
+            self.protocolKwArgs = kw
+
+    def connectionMade(self):
+        if self.protocolFactory is not None:
+            self.protocol = self.protocolFactory(
+                *self.protocolArgs, **self.protocolKwArgs
+            )
+            assert ITelnetProtocol.providedBy(self.protocol)
+            try:
+                factory = self.factory
+            except AttributeError:
+                pass
+            else:
+                self.protocol.factory = factory
+            self.protocol.makeConnection(self)
+
+    def connectionLost(self, reason):
+        Telnet.connectionLost(self, reason)
+        if self.protocol is not None:
+            try:
+                self.protocol.connectionLost(reason)
+            finally:
+                del self.protocol
+
+    def enableLocal(self, option):
+        return self.protocol.enableLocal(option)
+
+    def enableRemote(self, option):
+        return self.protocol.enableRemote(option)
+
+    def disableLocal(self, option):
+        return self.protocol.disableLocal(option)
+
+    def disableRemote(self, option):
+        return self.protocol.disableRemote(option)
+
+    def unhandledSubnegotiation(self, command, data):
+        self.protocol.unhandledSubnegotiation(command, data)
+
+    def unhandledCommand(self, command, argument):
+        self.protocol.unhandledCommand(command, argument)
+
+    def applicationDataReceived(self, data):
+        self.protocol.dataReceived(data)
+
+    def write(self, data):
+        ProtocolTransportMixin.write(self, data.replace(b"\xff", b"\xff\xff"))
+
+
+class TelnetBootstrapProtocol(TelnetProtocol, ProtocolTransportMixin):
+    protocol = None
+
+    def __init__(self, protocolFactory, *args, **kw):
+        self.protocolFactory = protocolFactory
+        self.protocolArgs = args
+        self.protocolKwArgs = kw
+
+    def connectionMade(self):
+        self.transport.negotiationMap[NAWS] = self.telnet_NAWS
+        self.transport.negotiationMap[LINEMODE] = self.telnet_LINEMODE
+
+        for opt in (LINEMODE, NAWS, SGA):
+            self.transport.do(opt).addErrback(
+                lambda f: self._log.failure("Error do {opt!r}", f, opt=opt)
+            )
+        for opt in (ECHO,):
+            self.transport.will(opt).addErrback(
+                lambda f: self._log.failure("Error setting will {opt!r}", f, opt=opt)
+            )
+
+        self.protocol = self.protocolFactory(*self.protocolArgs, **self.protocolKwArgs)
+
+        try:
+            factory = self.factory
+        except AttributeError:
+            pass
+        else:
+            self.protocol.factory = factory
+
+        self.protocol.makeConnection(self)
+
+    def connectionLost(self, reason):
+        if self.protocol is not None:
+            try:
+                self.protocol.connectionLost(reason)
+            finally:
+                del self.protocol
+
+    def dataReceived(self, data):
+        self.protocol.dataReceived(data)
+
+    def enableLocal(self, opt):
+        if opt == ECHO:
+            return True
+        elif opt == SGA:
+            return True
+        else:
+            return False
+
+    def enableRemote(self, opt):
+        if opt == LINEMODE:
+            self.transport.requestNegotiation(LINEMODE, MODE + LINEMODE_TRAPSIG)
+            return True
+        elif opt == NAWS:
+            return True
+        elif opt == SGA:
+            return True
+        else:
+            return False
+
+    def telnet_NAWS(self, data):
+        # NAWS is client -> server *only*.  self.protocol will
+        # therefore be an ITerminalTransport, the `.protocol'
+        # attribute of which will be an ITerminalProtocol.  Maybe.
+        # You know what, XXX TODO clean this up.
+        if len(data) == 4:
+            width, height = struct.unpack("!HH", b"".join(data))
+            self.protocol.terminalProtocol.terminalSize(width, height)
+        else:
+            self._log.error("Wrong number of NAWS bytes: {nbytes}", nbytes=len(data))
+
+    linemodeSubcommands = {LINEMODE_SLC: "SLC"}
+
+    def telnet_LINEMODE(self, data):
+        # linemodeSubcommand = data[0]
+        # # XXX TODO: This should be enabled to parse linemode subnegotiation.
+        # getattr(self, "linemode_" + self.linemodeSubcommands[linemodeSubcommand])(
+        #     data[1:]
+        # )
+        pass
+
+    def linemode_SLC(self, data):
+        chunks = zip(*[iter(data)] * 3)
+        for slcFunction, slcValue, slcWhat in chunks:
+            # Later, we should parse stuff.
+            "SLC", ord(slcFunction), ord(slcValue), ord(slcWhat)
+
+
+from twisted.protocols import basic
+
+
+class StatefulTelnetProtocol(basic.LineReceiver, TelnetProtocol):
+    delimiter = b"\n"
+
+    state = "Discard"
+
+    def connectionLost(self, reason):
+        basic.LineReceiver.connectionLost(self, reason)
+        TelnetProtocol.connectionLost(self, reason)
+
+    def lineReceived(self, line):
+        oldState = self.state
+        newState = getattr(self, "telnet_" + oldState)(line)
+        if newState is not None:
+            if self.state == oldState:
+                self.state = newState
+            else:
+                self._log.warn("state changed and new state returned")
+
+    def telnet_Discard(self, line):
+        pass
+
+
+from twisted.cred import credentials
+
+
+class AuthenticatingTelnetProtocol(StatefulTelnetProtocol):
+    """
+    A protocol which prompts for credentials and attempts to authenticate them.
+
+    Username and password prompts are given (the password is obscured).  When the
+    information is collected, it is passed to a portal and an avatar implementing
+    L{ITelnetProtocol} is requested.  If an avatar is returned, it connected to this
+    protocol's transport, and this protocol's transport is connected to it.
+    Otherwise, the user is re-prompted for credentials.
+    """
+
+    state = "User"
+    protocol = None
+
+    def __init__(self, portal):
+        self.portal = portal
+
+    def connectionMade(self):
+        self.transport.write(b"Username: ")
+
+    def connectionLost(self, reason):
+        StatefulTelnetProtocol.connectionLost(self, reason)
+        if self.protocol is not None:
+            try:
+                self.protocol.connectionLost(reason)
+                self.logout()
+            finally:
+                del self.protocol, self.logout
+
+    def telnet_User(self, line):
+        self.username = line
+        self.transport.will(ECHO)
+        self.transport.write(b"Password: ")
+        return "Password"
+
+    def telnet_Password(self, line):
+        username, password = self.username, line
+        del self.username
+
+        def login(ignored):
+            creds = credentials.UsernamePassword(username, password)
+            d = self.portal.login(creds, None, ITelnetProtocol)
+            d.addCallback(self._cbLogin)
+            d.addErrback(self._ebLogin)
+
+        self.transport.wont(ECHO).addCallback(login)
+        return "Discard"
+
+    def _cbLogin(self, ial):
+        interface, protocol, logout = ial
+        assert interface is ITelnetProtocol
+        self.protocol = protocol
+        self.logout = logout
+        self.state = "Command"
+
+        protocol.makeConnection(self.transport)
+        self.transport.protocol = protocol
+
+    def _ebLogin(self, failure):
+        self.transport.write(b"\nAuthentication failed\n")
+        self.transport.write(b"Username: ")
+        self.state = "User"
+
+
+__all__ = [
+    # Exceptions
+    "TelnetError",
+    "NegotiationError",
+    "OptionRefused",
+    "AlreadyNegotiating",
+    "AlreadyEnabled",
+    "AlreadyDisabled",
+    # Interfaces
+    "ITelnetProtocol",
+    "ITelnetTransport",
+    # Other stuff, protocols, etc.
+    "Telnet",
+    "TelnetProtocol",
+    "TelnetTransport",
+    "TelnetBootstrapProtocol",
+]
diff --git a/contrib/python/Twisted/py3/twisted/conch/ttymodes.py b/contrib/python/Twisted/py3/twisted/conch/ttymodes.py
new file mode 100644
index 00000000000..e929d49590f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ttymodes.py
@@ -0,0 +1,122 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+import tty
+
+# this module was autogenerated.
+
+VINTR = 1
+VQUIT = 2
+VERASE = 3
+VKILL = 4
+VEOF = 5
+VEOL = 6
+VEOL2 = 7
+VSTART = 8
+VSTOP = 9
+VSUSP = 10
+VDSUSP = 11
+VREPRINT = 12
+VWERASE = 13
+VLNEXT = 14
+VFLUSH = 15
+VSWTCH = 16
+VSTATUS = 17
+VDISCARD = 18
+IGNPAR = 30
+PARMRK = 31
+INPCK = 32
+ISTRIP = 33
+INLCR = 34
+IGNCR = 35
+ICRNL = 36
+IUCLC = 37
+IXON = 38
+IXANY = 39
+IXOFF = 40
+IMAXBEL = 41
+ISIG = 50
+ICANON = 51
+XCASE = 52
+ECHO = 53
+ECHOE = 54
+ECHOK = 55
+ECHONL = 56
+NOFLSH = 57
+TOSTOP = 58
+IEXTEN = 59
+ECHOCTL = 60
+ECHOKE = 61
+PENDIN = 62
+OPOST = 70
+OLCUC = 71
+ONLCR = 72
+OCRNL = 73
+ONOCR = 74
+ONLRET = 75
+CS7 = 90
+CS8 = 91
+PARENB = 92
+PARODD = 93
+TTY_OP_ISPEED = 128
+TTY_OP_OSPEED = 129
+
+TTYMODES = {
+    1: "VINTR",
+    2: "VQUIT",
+    3: "VERASE",
+    4: "VKILL",
+    5: "VEOF",
+    6: "VEOL",
+    7: "VEOL2",
+    8: "VSTART",
+    9: "VSTOP",
+    10: "VSUSP",
+    11: "VDSUSP",
+    12: "VREPRINT",
+    13: "VWERASE",
+    14: "VLNEXT",
+    15: "VFLUSH",
+    16: "VSWTCH",
+    17: "VSTATUS",
+    18: "VDISCARD",
+    30: (tty.IFLAG, "IGNPAR"),
+    31: (tty.IFLAG, "PARMRK"),
+    32: (tty.IFLAG, "INPCK"),
+    33: (tty.IFLAG, "ISTRIP"),
+    34: (tty.IFLAG, "INLCR"),
+    35: (tty.IFLAG, "IGNCR"),
+    36: (tty.IFLAG, "ICRNL"),
+    37: (tty.IFLAG, "IUCLC"),
+    38: (tty.IFLAG, "IXON"),
+    39: (tty.IFLAG, "IXANY"),
+    40: (tty.IFLAG, "IXOFF"),
+    41: (tty.IFLAG, "IMAXBEL"),
+    50: (tty.LFLAG, "ISIG"),
+    51: (tty.LFLAG, "ICANON"),
+    52: (tty.LFLAG, "XCASE"),
+    53: (tty.LFLAG, "ECHO"),
+    54: (tty.LFLAG, "ECHOE"),
+    55: (tty.LFLAG, "ECHOK"),
+    56: (tty.LFLAG, "ECHONL"),
+    57: (tty.LFLAG, "NOFLSH"),
+    58: (tty.LFLAG, "TOSTOP"),
+    59: (tty.LFLAG, "IEXTEN"),
+    60: (tty.LFLAG, "ECHOCTL"),
+    61: (tty.LFLAG, "ECHOKE"),
+    62: (tty.LFLAG, "PENDIN"),
+    70: (tty.OFLAG, "OPOST"),
+    71: (tty.OFLAG, "OLCUC"),
+    72: (tty.OFLAG, "ONLCR"),
+    73: (tty.OFLAG, "OCRNL"),
+    74: (tty.OFLAG, "ONOCR"),
+    75: (tty.OFLAG, "ONLRET"),
+    #   90 : (tty.CFLAG, 'CS7'),
+    #   91 : (tty.CFLAG, 'CS8'),
+    92: (tty.CFLAG, "PARENB"),
+    93: (tty.CFLAG, "PARODD"),
+    128: "ISPEED",
+    129: "OSPEED",
+}
diff --git a/contrib/python/Twisted/py3/twisted/conch/ui/__init__.py b/contrib/python/Twisted/py3/twisted/conch/ui/__init__.py
new file mode 100644
index 00000000000..ea0eea83183
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ui/__init__.py
@@ -0,0 +1,11 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+
+"""
+twisted.conch.ui is home to the UI elements for tkconch.
+
+Maintainer: Paul Swartz
+"""
diff --git a/contrib/python/Twisted/py3/twisted/conch/ui/ansi.py b/contrib/python/Twisted/py3/twisted/conch/ui/ansi.py
new file mode 100644
index 00000000000..60f1666b616
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ui/ansi.py
@@ -0,0 +1,253 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+"""Module to parse ANSI escape sequences
+
+Maintainer: Jean-Paul Calderone
+"""
+
+import string
+
+# Twisted imports
+from twisted.logger import Logger
+
+_log = Logger()
+
+
+class ColorText:
+    """
+    Represents an element of text along with the texts colors and
+    additional attributes.
+    """
+
+    # The colors to use
+    COLORS = ("b", "r", "g", "y", "l", "m", "c", "w")
+    BOLD_COLORS = tuple(x.upper() for x in COLORS)
+    BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(len(COLORS))
+
+    # Color names
+    COLOR_NAMES = (
+        "Black",
+        "Red",
+        "Green",
+        "Yellow",
+        "Blue",
+        "Magenta",
+        "Cyan",
+        "White",
+    )
+
+    def __init__(self, text, fg, bg, display, bold, underline, flash, reverse):
+        self.text, self.fg, self.bg = text, fg, bg
+        self.display = display
+        self.bold = bold
+        self.underline = underline
+        self.flash = flash
+        self.reverse = reverse
+        if self.reverse:
+            self.fg, self.bg = self.bg, self.fg
+
+
+class AnsiParser:
+    """
+    Parser class for ANSI codes.
+    """
+
+    # Terminators for cursor movement ansi controls - unsupported
+    CURSOR_SET = ("H", "f", "A", "B", "C", "D", "R", "s", "u", "d", "G")
+
+    # Terminators for erasure ansi controls - unsupported
+    ERASE_SET = ("J", "K", "P")
+
+    # Terminators for mode change ansi controls - unsupported
+    MODE_SET = ("h", "l")
+
+    # Terminators for keyboard assignment ansi controls - unsupported
+    ASSIGN_SET = ("p",)
+
+    # Terminators for color change ansi controls - supported
+    COLOR_SET = ("m",)
+
+    SETS = (CURSOR_SET, ERASE_SET, MODE_SET, ASSIGN_SET, COLOR_SET)
+
+    def __init__(self, defaultFG, defaultBG):
+        self.defaultFG, self.defaultBG = defaultFG, defaultBG
+        self.currentFG, self.currentBG = self.defaultFG, self.defaultBG
+        self.bold, self.flash, self.underline, self.reverse = 0, 0, 0, 0
+        self.display = 1
+        self.prepend = ""
+
+    def stripEscapes(self, string):
+        """
+        Remove all ANSI color escapes from the given string.
+        """
+        result = ""
+        show = 1
+        i = 0
+        L = len(string)
+        while i < L:
+            if show == 0 and string[i] in _sets:
+                show = 1
+            elif show:
+                n = string.find("\x1B", i)
+                if n == -1:
+                    return result + string[i:]
+                else:
+                    result = result + string[i:n]
+                    i = n
+                    show = 0
+            i = i + 1
+        return result
+
+    def writeString(self, colorstr):
+        pass
+
+    def parseString(self, str):
+        """
+        Turn a string input into a list of L{ColorText} elements.
+        """
+
+        if self.prepend:
+            str = self.prepend + str
+            self.prepend = ""
+        parts = str.split("\x1B")
+
+        if len(parts) == 1:
+            self.writeString(self.formatText(parts[0]))
+        else:
+            self.writeString(self.formatText(parts[0]))
+            for s in parts[1:]:
+                L = len(s)
+                i = 0
+                type = None
+                while i < L:
+                    if s[i] not in string.digits + "[;?":
+                        break
+                    i += 1
+                if not s:
+                    self.prepend = "\x1b"
+                    return
+                if s[0] != "[":
+                    self.writeString(self.formatText(s[i + 1 :]))
+                    continue
+                else:
+                    s = s[1:]
+                    i -= 1
+                if i == L - 1:
+                    self.prepend = "\x1b["
+                    return
+                type = _setmap.get(s[i], None)
+                if type is None:
+                    continue
+
+                if type == AnsiParser.COLOR_SET:
+                    self.parseColor(s[: i + 1])
+                    s = s[i + 1 :]
+                    self.writeString(self.formatText(s))
+                elif type == AnsiParser.CURSOR_SET:
+                    cursor, s = s[: i + 1], s[i + 1 :]
+                    self.parseCursor(cursor)
+                    self.writeString(self.formatText(s))
+                elif type == AnsiParser.ERASE_SET:
+                    erase, s = s[: i + 1], s[i + 1 :]
+                    self.parseErase(erase)
+                    self.writeString(self.formatText(s))
+                elif type == AnsiParser.MODE_SET:
+                    s = s[i + 1 :]
+                    # self.parseErase('2J')
+                    self.writeString(self.formatText(s))
+                elif i == L:
+                    self.prepend = "\x1B[" + s
+                else:
+                    _log.warn(
+                        "Unhandled ANSI control type: {control_type}", control_type=s[i]
+                    )
+                    s = s[i + 1 :]
+                    self.writeString(self.formatText(s))
+
+    def parseColor(self, str):
+        """
+        Handle a single ANSI color sequence
+        """
+        # Drop the trailing 'm'
+        str = str[:-1]
+
+        if not str:
+            str = "0"
+
+        try:
+            parts = map(int, str.split(";"))
+        except ValueError:
+            _log.error("Invalid ANSI color sequence: {sequence!r}", sequence=str)
+            self.currentFG, self.currentBG = self.defaultFG, self.defaultBG
+            return
+
+        for x in parts:
+            if x == 0:
+                self.currentFG, self.currentBG = self.defaultFG, self.defaultBG
+                self.bold, self.flash, self.underline, self.reverse = 0, 0, 0, 0
+                self.display = 1
+            elif x == 1:
+                self.bold = 1
+            elif 30 <= x <= 37:
+                self.currentFG = x - 30
+            elif 40 <= x <= 47:
+                self.currentBG = x - 40
+            elif x == 39:
+                self.currentFG = self.defaultFG
+            elif x == 49:
+                self.currentBG = self.defaultBG
+            elif x == 4:
+                self.underline = 1
+            elif x == 5:
+                self.flash = 1
+            elif x == 7:
+                self.reverse = 1
+            elif x == 8:
+                self.display = 0
+            elif x == 22:
+                self.bold = 0
+            elif x == 24:
+                self.underline = 0
+            elif x == 25:
+                self.blink = 0
+            elif x == 27:
+                self.reverse = 0
+            elif x == 28:
+                self.display = 1
+            else:
+                _log.error("Unrecognised ANSI color command: {command}", command=x)
+
+    def parseCursor(self, cursor):
+        pass
+
+    def parseErase(self, erase):
+        pass
+
+    def pickColor(self, value, mode, BOLD=ColorText.BOLD_COLORS):
+        if mode:
+            return ColorText.COLORS[value]
+        else:
+            return self.bold and BOLD[value] or ColorText.COLORS[value]
+
+    def formatText(self, text):
+        return ColorText(
+            text,
+            self.pickColor(self.currentFG, 0),
+            self.pickColor(self.currentBG, 1),
+            self.display,
+            self.bold,
+            self.underline,
+            self.flash,
+            self.reverse,
+        )
+
+
+_sets = "".join(map("".join, AnsiParser.SETS))
+
+_setmap = {}
+for s in AnsiParser.SETS:
+    for r in s:
+        _setmap[r] = s
+del s
diff --git a/contrib/python/Twisted/py3/twisted/conch/ui/tkvt100.py b/contrib/python/Twisted/py3/twisted/conch/ui/tkvt100.py
new file mode 100644
index 00000000000..336780e36a8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/ui/tkvt100.py
@@ -0,0 +1,249 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+"""Module to emulate a VT100 terminal in Tkinter.
+
+Maintainer: Paul Swartz
+"""
+
+import string
+import tkinter as Tkinter
+import tkinter.font as tkFont
+
+from . import ansi
+
+ttyFont = None  # tkFont.Font(family = 'Courier', size = 10)
+fontWidth, fontHeight = (
+    None,
+    None,
+)  # max(map(ttyFont.measure, string.letters+string.digits)), int(ttyFont.metrics()['linespace'])
+
+colorKeys = (
+    "b",
+    "r",
+    "g",
+    "y",
+    "l",
+    "m",
+    "c",
+    "w",
+    "B",
+    "R",
+    "G",
+    "Y",
+    "L",
+    "M",
+    "C",
+    "W",
+)
+
+colorMap = {
+    "b": "#000000",
+    "r": "#c40000",
+    "g": "#00c400",
+    "y": "#c4c400",
+    "l": "#000080",
+    "m": "#c400c4",
+    "c": "#00c4c4",
+    "w": "#c4c4c4",
+    "B": "#626262",
+    "R": "#ff0000",
+    "G": "#00ff00",
+    "Y": "#ffff00",
+    "L": "#0000ff",
+    "M": "#ff00ff",
+    "C": "#00ffff",
+    "W": "#ffffff",
+}
+
+
+class VT100Frame(Tkinter.Frame):
+    def __init__(self, *args, **kw):
+        global ttyFont, fontHeight, fontWidth
+        ttyFont = tkFont.Font(family="Courier", size=10)
+        fontWidth = max(map(ttyFont.measure, string.ascii_letters + string.digits))
+        fontHeight = int(ttyFont.metrics()["linespace"])
+        self.width = kw.get("width", 80)
+        self.height = kw.get("height", 25)
+        self.callback = kw["callback"]
+        del kw["callback"]
+        kw["width"] = w = fontWidth * self.width
+        kw["height"] = h = fontHeight * self.height
+        Tkinter.Frame.__init__(self, *args, **kw)
+        self.canvas = Tkinter.Canvas(bg="#000000", width=w, height=h)
+        self.canvas.pack(side=Tkinter.TOP, fill=Tkinter.BOTH, expand=1)
+        self.canvas.bind("<Key>", self.keyPressed)
+        self.canvas.bind("<1>", lambda x: "break")
+        self.canvas.bind("<Up>", self.upPressed)
+        self.canvas.bind("<Down>", self.downPressed)
+        self.canvas.bind("<Left>", self.leftPressed)
+        self.canvas.bind("<Right>", self.rightPressed)
+        self.canvas.focus()
+
+        self.ansiParser = ansi.AnsiParser(ansi.ColorText.WHITE, ansi.ColorText.BLACK)
+        self.ansiParser.writeString = self.writeString
+        self.ansiParser.parseCursor = self.parseCursor
+        self.ansiParser.parseErase = self.parseErase
+        # for (a, b) in colorMap.items():
+        #    self.canvas.tag_config(a, foreground=b)
+        #    self.canvas.tag_config('b'+a, background=b)
+        # self.canvas.tag_config('underline', underline=1)
+
+        self.x = 0
+        self.y = 0
+        self.cursor = self.canvas.create_rectangle(
+            0, 0, fontWidth - 1, fontHeight - 1, fill="green", outline="green"
+        )
+
+    def _delete(self, sx, sy, ex, ey):
+        csx = sx * fontWidth + 1
+        csy = sy * fontHeight + 1
+        cex = ex * fontWidth + 3
+        cey = ey * fontHeight + 3
+        items = self.canvas.find_overlapping(csx, csy, cex, cey)
+        for item in items:
+            self.canvas.delete(item)
+
+    def _write(self, ch, fg, bg):
+        if self.x == self.width:
+            self.x = 0
+            self.y += 1
+            if self.y == self.height:
+                [self.canvas.move(x, 0, -fontHeight) for x in self.canvas.find_all()]
+                self.y -= 1
+        canvasX = self.x * fontWidth + 1
+        canvasY = self.y * fontHeight + 1
+        items = self.canvas.find_overlapping(canvasX, canvasY, canvasX + 2, canvasY + 2)
+        if items:
+            [self.canvas.delete(item) for item in items]
+        if bg:
+            self.canvas.create_rectangle(
+                canvasX,
+                canvasY,
+                canvasX + fontWidth - 1,
+                canvasY + fontHeight - 1,
+                fill=bg,
+                outline=bg,
+            )
+        self.canvas.create_text(
+            canvasX, canvasY, anchor=Tkinter.NW, font=ttyFont, text=ch, fill=fg
+        )
+        self.x += 1
+
+    def write(self, data):
+        self.ansiParser.parseString(data)
+        self.canvas.delete(self.cursor)
+        canvasX = self.x * fontWidth + 1
+        canvasY = self.y * fontHeight + 1
+        self.cursor = self.canvas.create_rectangle(
+            canvasX,
+            canvasY,
+            canvasX + fontWidth - 1,
+            canvasY + fontHeight - 1,
+            fill="green",
+            outline="green",
+        )
+        self.canvas.lower(self.cursor)
+
+    def writeString(self, i):
+        if not i.display:
+            return
+        fg = colorMap[i.fg]
+        bg = i.bg != "b" and colorMap[i.bg]
+        for ch in i.text:
+            b = ord(ch)
+            if b == 7:  # bell
+                self.bell()
+            elif b == 8:  # BS
+                if self.x:
+                    self.x -= 1
+            elif b == 9:  # TAB
+                [self._write(" ", fg, bg) for index in range(8)]
+            elif b == 10:
+                if self.y == self.height - 1:
+                    self._delete(0, 0, self.width, 0)
+                    [
+                        self.canvas.move(x, 0, -fontHeight)
+                        for x in self.canvas.find_all()
+                    ]
+                else:
+                    self.y += 1
+            elif b == 13:
+                self.x = 0
+            elif 32 <= b < 127:
+                self._write(ch, fg, bg)
+
+    def parseErase(self, erase):
+        if ";" in erase:
+            end = erase[-1]
+            parts = erase[:-1].split(";")
+            [self.parseErase(x + end) for x in parts]
+            return
+        start = 0
+        x, y = self.x, self.y
+        if len(erase) > 1:
+            start = int(erase[:-1])
+        if erase[-1] == "J":
+            if start == 0:
+                self._delete(x, y, self.width, self.height)
+            else:
+                self._delete(0, 0, self.width, self.height)
+                self.x = 0
+                self.y = 0
+        elif erase[-1] == "K":
+            if start == 0:
+                self._delete(x, y, self.width, y)
+            elif start == 1:
+                self._delete(0, y, x, y)
+                self.x = 0
+            else:
+                self._delete(0, y, self.width, y)
+                self.x = 0
+        elif erase[-1] == "P":
+            self._delete(x, y, x + start, y)
+
+    def parseCursor(self, cursor):
+        # if ';' in cursor and cursor[-1]!='H':
+        #    end = cursor[-1]
+        #    parts = cursor[:-1].split(';')
+        #    [self.parseCursor(x+end) for x in parts]
+        #    return
+        start = 1
+        if len(cursor) > 1 and cursor[-1] != "H":
+            start = int(cursor[:-1])
+        if cursor[-1] == "C":
+            self.x += start
+        elif cursor[-1] == "D":
+            self.x -= start
+        elif cursor[-1] == "d":
+            self.y = start - 1
+        elif cursor[-1] == "G":
+            self.x = start - 1
+        elif cursor[-1] == "H":
+            if len(cursor) > 1:
+                y, x = map(int, cursor[:-1].split(";"))
+                y -= 1
+                x -= 1
+            else:
+                x, y = 0, 0
+            self.x = x
+            self.y = y
+
+    def keyPressed(self, event):
+        if self.callback and event.char:
+            self.callback(event.char)
+        return "break"
+
+    def upPressed(self, event):
+        self.callback("\x1bOA")
+
+    def downPressed(self, event):
+        self.callback("\x1bOB")
+
+    def rightPressed(self, event):
+        self.callback("\x1bOC")
+
+    def leftPressed(self, event):
+        self.callback("\x1bOD")
diff --git a/contrib/python/Twisted/py3/twisted/conch/unix.py b/contrib/python/Twisted/py3/twisted/conch/unix.py
new file mode 100644
index 00000000000..e314e3cd72d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/conch/unix.py
@@ -0,0 +1,524 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A UNIX SSH server.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import grp
+import os
+import pty
+import pwd
+import socket
+import struct
+import time
+import tty
+from typing import Callable, Dict, Tuple
+
+from zope.interface import implementer
+
+from twisted.conch import ttymodes
+from twisted.conch.avatar import ConchUser
+from twisted.conch.error import ConchError
+from twisted.conch.interfaces import ISession, ISFTPFile, ISFTPServer
+from twisted.conch.ls import lsLine
+from twisted.conch.ssh import filetransfer, forwarding, session
+from twisted.conch.ssh.filetransfer import (
+    FXF_APPEND,
+    FXF_CREAT,
+    FXF_EXCL,
+    FXF_READ,
+    FXF_TRUNC,
+    FXF_WRITE,
+)
+from twisted.cred import portal
+from twisted.cred.error import LoginDenied
+from twisted.internet.error import ProcessExitedAlready
+from twisted.internet.interfaces import IListeningPort
+from twisted.logger import Logger
+from twisted.python import components
+from twisted.python.compat import nativeString
+
+try:
+    import utmp  # type: ignore[import]
+except ImportError:
+    utmp = None
+
+
+@implementer(portal.IRealm)
+class UnixSSHRealm:
+    def requestAvatar(
+        self,
+        username: bytes | Tuple[()],
+        mind: object,
+        *interfaces: portal._InterfaceItself,
+    ) -> Tuple[portal._InterfaceItself, UnixConchUser, Callable[[], None]]:
+        if not isinstance(username, bytes):
+            raise LoginDenied("UNIX SSH realm does not authorize anonymous sessions.")
+        user = UnixConchUser(username.decode())
+        return interfaces[0], user, user.logout
+
+
+class UnixConchUser(ConchUser):
+    def __init__(self, username: str) -> None:
+        ConchUser.__init__(self)
+        self.username = username
+        self.pwdData = pwd.getpwnam(self.username)
+        l = [self.pwdData[3]]
+        for groupname, password, gid, userlist in grp.getgrall():
+            if username in userlist:
+                l.append(gid)
+        self.otherGroups = l
+        self.listeners: Dict[
+            str, IListeningPort
+        ] = {}  # Dict mapping (interface, port) -> listener
+        self.channelLookup.update(
+            {
+                b"session": session.SSHSession,
+                b"direct-tcpip": forwarding.openConnectForwardingClient,
+            }
+        )
+
+        self.subsystemLookup.update({b"sftp": filetransfer.FileTransferServer})
+
+    def getUserGroupId(self):
+        return self.pwdData[2:4]
+
+    def getOtherGroups(self):
+        return self.otherGroups
+
+    def getHomeDir(self):
+        return self.pwdData[5]
+
+    def getShell(self):
+        return self.pwdData[6]
+
+    def global_tcpip_forward(self, data):
+        hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data)
+        from twisted.internet import reactor
+
+        try:
+            listener = self._runAsUser(
+                reactor.listenTCP,
+                portToBind,
+                forwarding.SSHListenForwardingFactory(
+                    self.conn,
+                    (hostToBind, portToBind),
+                    forwarding.SSHListenServerForwardingChannel,
+                ),
+                interface=hostToBind,
+            )
+        except BaseException:
+            return 0
+        else:
+            self.listeners[(hostToBind, portToBind)] = listener
+            if portToBind == 0:
+                portToBind = listener.getHost()[2]  # The port
+                return 1, struct.pack(">L", portToBind)
+            else:
+                return 1
+
+    def global_cancel_tcpip_forward(self, data):
+        hostToBind, portToBind = forwarding.unpackGlobal_tcpip_forward(data)
+        listener = self.listeners.get((hostToBind, portToBind), None)
+        if not listener:
+            return 0
+        del self.listeners[(hostToBind, portToBind)]
+        self._runAsUser(listener.stopListening)
+        return 1
+
+    def logout(self) -> None:
+        # Remove all listeners.
+        for listener in self.listeners.values():
+            self._runAsUser(listener.stopListening)
+        self._log.info(
+            "avatar {username} logging out ({nlisteners})",
+            username=self.username,
+            nlisteners=len(self.listeners),
+        )
+
+    def _runAsUser(self, f, *args, **kw):
+        euid = os.geteuid()
+        egid = os.getegid()
+        groups = os.getgroups()
+        uid, gid = self.getUserGroupId()
+        os.setegid(0)
+        os.seteuid(0)
+        os.setgroups(self.getOtherGroups())
+        os.setegid(gid)
+        os.seteuid(uid)
+        try:
+            f = iter(f)
+        except TypeError:
+            f = [(f, args, kw)]
+        try:
+            for i in f:
+                func = i[0]
+                args = len(i) > 1 and i[1] or ()
+                kw = len(i) > 2 and i[2] or {}
+                r = func(*args, **kw)
+        finally:
+            os.setegid(0)
+            os.seteuid(0)
+            os.setgroups(groups)
+            os.setegid(egid)
+            os.seteuid(euid)
+        return r
+
+
+@implementer(ISession)
+class SSHSessionForUnixConchUser:
+    _log = Logger()
+
+    def __init__(self, avatar, reactor=None):
+        """
+        Construct an C{SSHSessionForUnixConchUser}.
+
+        @param avatar: The L{UnixConchUser} for whom this is an SSH session.
+        @param reactor: An L{IReactorProcess} used to handle shell and exec
+            requests. Uses the default reactor if None.
+        """
+        if reactor is None:
+            from twisted.internet import reactor
+        self._reactor = reactor
+        self.avatar = avatar
+        self.environ = {"PATH": "/bin:/usr/bin:/usr/local/bin"}
+        self.pty = None
+        self.ptyTuple = 0
+
+    def addUTMPEntry(self, loggedIn=1):
+        if not utmp:
+            return
+        ipAddress = self.avatar.conn.transport.transport.getPeer().host
+        (packedIp,) = struct.unpack("L", socket.inet_aton(ipAddress))
+        ttyName = self.ptyTuple[2][5:]
+        t = time.time()
+        t1 = int(t)
+        t2 = int((t - t1) * 1e6)
+        entry = utmp.UtmpEntry()
+        entry.ut_type = loggedIn and utmp.USER_PROCESS or utmp.DEAD_PROCESS
+        entry.ut_pid = self.pty.pid
+        entry.ut_line = ttyName
+        entry.ut_id = ttyName[-4:]
+        entry.ut_tv = (t1, t2)
+        if loggedIn:
+            entry.ut_user = self.avatar.username
+            entry.ut_host = socket.gethostbyaddr(ipAddress)[0]
+            entry.ut_addr_v6 = (packedIp, 0, 0, 0)
+        a = utmp.UtmpRecord(utmp.UTMP_FILE)
+        a.pututline(entry)
+        a.endutent()
+        b = utmp.UtmpRecord(utmp.WTMP_FILE)
+        b.pututline(entry)
+        b.endutent()
+
+    def getPty(self, term, windowSize, modes):
+        self.environ["TERM"] = term
+        self.winSize = windowSize
+        self.modes = modes
+        master, slave = pty.openpty()
+        ttyname = os.ttyname(slave)
+        self.environ["SSH_TTY"] = ttyname
+        self.ptyTuple = (master, slave, ttyname)
+
+    def openShell(self, proto):
+        if not self.ptyTuple:  # We didn't get a pty-req.
+            self._log.error("tried to get shell without pty, failing")
+            raise ConchError("no pty")
+        uid, gid = self.avatar.getUserGroupId()
+        homeDir = self.avatar.getHomeDir()
+        shell = self.avatar.getShell()
+        self.environ["USER"] = self.avatar.username
+        self.environ["HOME"] = homeDir
+        self.environ["SHELL"] = shell
+        shellExec = os.path.basename(shell)
+        peer = self.avatar.conn.transport.transport.getPeer()
+        host = self.avatar.conn.transport.transport.getHost()
+        self.environ["SSH_CLIENT"] = f"{peer.host} {peer.port} {host.port}"
+        self.getPtyOwnership()
+        self.pty = self._reactor.spawnProcess(
+            proto,
+            shell,
+            [f"-{shellExec}"],
+            self.environ,
+            homeDir,
+            uid,
+            gid,
+            usePTY=self.ptyTuple,
+        )
+        self.addUTMPEntry()
+        fcntl.ioctl(self.pty.fileno(), tty.TIOCSWINSZ, struct.pack("4H", *self.winSize))
+        if self.modes:
+            self.setModes()
+        self.oldWrite = proto.transport.write
+        proto.transport.write = self._writeHack
+        self.avatar.conn.transport.transport.setTcpNoDelay(1)
+
+    def execCommand(self, proto, cmd):
+        uid, gid = self.avatar.getUserGroupId()
+        homeDir = self.avatar.getHomeDir()
+        shell = self.avatar.getShell() or "/bin/sh"
+        self.environ["HOME"] = homeDir
+        command = (shell, "-c", cmd)
+        peer = self.avatar.conn.transport.transport.getPeer()
+        host = self.avatar.conn.transport.transport.getHost()
+        self.environ["SSH_CLIENT"] = f"{peer.host} {peer.port} {host.port}"
+        if self.ptyTuple:
+            self.getPtyOwnership()
+        self.pty = self._reactor.spawnProcess(
+            proto,
+            shell,
+            command,
+            self.environ,
+            homeDir,
+            uid,
+            gid,
+            usePTY=self.ptyTuple or 0,
+        )
+        if self.ptyTuple:
+            self.addUTMPEntry()
+            if self.modes:
+                self.setModes()
+        self.avatar.conn.transport.transport.setTcpNoDelay(1)
+
+    def getPtyOwnership(self):
+        ttyGid = os.stat(self.ptyTuple[2])[5]
+        uid, gid = self.avatar.getUserGroupId()
+        euid, egid = os.geteuid(), os.getegid()
+        os.setegid(0)
+        os.seteuid(0)
+        try:
+            os.chown(self.ptyTuple[2], uid, ttyGid)
+        finally:
+            os.setegid(egid)
+            os.seteuid(euid)
+
+    def setModes(self):
+        pty = self.pty
+        attr = tty.tcgetattr(pty.fileno())
+        for mode, modeValue in self.modes:
+            if mode not in ttymodes.TTYMODES:
+                continue
+            ttyMode = ttymodes.TTYMODES[mode]
+            if len(ttyMode) == 2:  # Flag.
+                flag, ttyAttr = ttyMode
+                if not hasattr(tty, ttyAttr):
+                    continue
+                ttyval = getattr(tty, ttyAttr)
+                if modeValue:
+                    attr[flag] = attr[flag] | ttyval
+                else:
+                    attr[flag] = attr[flag] & ~ttyval
+            elif ttyMode == "OSPEED":
+                attr[tty.OSPEED] = getattr(tty, f"B{modeValue}")
+            elif ttyMode == "ISPEED":
+                attr[tty.ISPEED] = getattr(tty, f"B{modeValue}")
+            else:
+                if not hasattr(tty, ttyMode):
+                    continue
+                ttyval = getattr(tty, ttyMode)
+                attr[tty.CC][ttyval] = bytes((modeValue,))
+        tty.tcsetattr(pty.fileno(), tty.TCSANOW, attr)
+
+    def eofReceived(self):
+        if self.pty:
+            self.pty.closeStdin()
+
+    def closed(self):
+        if self.ptyTuple and os.path.exists(self.ptyTuple[2]):
+            ttyGID = os.stat(self.ptyTuple[2])[5]
+            os.chown(self.ptyTuple[2], 0, ttyGID)
+        if self.pty:
+            try:
+                self.pty.signalProcess("HUP")
+            except (OSError, ProcessExitedAlready):
+                pass
+            self.pty.loseConnection()
+            self.addUTMPEntry(0)
+        self._log.info("shell closed")
+
+    def windowChanged(self, winSize):
+        self.winSize = winSize
+        fcntl.ioctl(self.pty.fileno(), tty.TIOCSWINSZ, struct.pack("4H", *self.winSize))
+
+    def _writeHack(self, data):
+        """
+        Hack to send ignore messages when we aren't echoing.
+        """
+        if self.pty is not None:
+            attr = tty.tcgetattr(self.pty.fileno())[3]
+            if not attr & tty.ECHO and attr & tty.ICANON:  # No echo.
+                self.avatar.conn.transport.sendIgnore("\x00" * (8 + len(data)))
+        self.oldWrite(data)
+
+
+@implementer(ISFTPServer)
+class SFTPServerForUnixConchUser:
+    def __init__(self, avatar):
+        self.avatar = avatar
+
+    def _setAttrs(self, path, attrs):
+        """
+        NOTE: this function assumes it runs as the logged-in user:
+        i.e. under _runAsUser()
+        """
+        if "uid" in attrs and "gid" in attrs:
+            os.chown(path, attrs["uid"], attrs["gid"])
+        if "permissions" in attrs:
+            os.chmod(path, attrs["permissions"])
+        if "atime" in attrs and "mtime" in attrs:
+            os.utime(path, (attrs["atime"], attrs["mtime"]))
+
+    def _getAttrs(self, s):
+        return {
+            "size": s.st_size,
+            "uid": s.st_uid,
+            "gid": s.st_gid,
+            "permissions": s.st_mode,
+            "atime": int(s.st_atime),
+            "mtime": int(s.st_mtime),
+        }
+
+    def _absPath(self, path):
+        home = self.avatar.getHomeDir()
+        return os.path.join(nativeString(home.path), nativeString(path))
+
+    def gotVersion(self, otherVersion, extData):
+        return {}
+
+    def openFile(self, filename, flags, attrs):
+        return UnixSFTPFile(self, self._absPath(filename), flags, attrs)
+
+    def removeFile(self, filename):
+        filename = self._absPath(filename)
+        return self.avatar._runAsUser(os.remove, filename)
+
+    def renameFile(self, oldpath, newpath):
+        oldpath = self._absPath(oldpath)
+        newpath = self._absPath(newpath)
+        return self.avatar._runAsUser(os.rename, oldpath, newpath)
+
+    def makeDirectory(self, path, attrs):
+        path = self._absPath(path)
+        return self.avatar._runAsUser(
+            [(os.mkdir, (path,)), (self._setAttrs, (path, attrs))]
+        )
+
+    def removeDirectory(self, path):
+        path = self._absPath(path)
+        self.avatar._runAsUser(os.rmdir, path)
+
+    def openDirectory(self, path):
+        return UnixSFTPDirectory(self, self._absPath(path))
+
+    def getAttrs(self, path, followLinks):
+        path = self._absPath(path)
+        if followLinks:
+            s = self.avatar._runAsUser(os.stat, path)
+        else:
+            s = self.avatar._runAsUser(os.lstat, path)
+        return self._getAttrs(s)
+
+    def setAttrs(self, path, attrs):
+        path = self._absPath(path)
+        self.avatar._runAsUser(self._setAttrs, path, attrs)
+
+    def readLink(self, path):
+        path = self._absPath(path)
+        return self.avatar._runAsUser(os.readlink, path)
+
+    def makeLink(self, linkPath, targetPath):
+        linkPath = self._absPath(linkPath)
+        targetPath = self._absPath(targetPath)
+        return self.avatar._runAsUser(os.symlink, targetPath, linkPath)
+
+    def realPath(self, path):
+        return os.path.realpath(self._absPath(path))
+
+    def extendedRequest(self, extName, extData):
+        raise NotImplementedError
+
+
+@implementer(ISFTPFile)
+class UnixSFTPFile:
+    def __init__(self, server, filename, flags, attrs):
+        self.server = server
+        openFlags = 0
+        if flags & FXF_READ == FXF_READ and flags & FXF_WRITE == 0:
+            openFlags = os.O_RDONLY
+        if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == 0:
+            openFlags = os.O_WRONLY
+        if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == FXF_READ:
+            openFlags = os.O_RDWR
+        if flags & FXF_APPEND == FXF_APPEND:
+            openFlags |= os.O_APPEND
+        if flags & FXF_CREAT == FXF_CREAT:
+            openFlags |= os.O_CREAT
+        if flags & FXF_TRUNC == FXF_TRUNC:
+            openFlags |= os.O_TRUNC
+        if flags & FXF_EXCL == FXF_EXCL:
+            openFlags |= os.O_EXCL
+        if "permissions" in attrs:
+            mode = attrs["permissions"]
+            del attrs["permissions"]
+        else:
+            mode = 0o777
+        fd = server.avatar._runAsUser(os.open, filename, openFlags, mode)
+        if attrs:
+            server.avatar._runAsUser(server._setAttrs, filename, attrs)
+        self.fd = fd
+
+    def close(self):
+        return self.server.avatar._runAsUser(os.close, self.fd)
+
+    def readChunk(self, offset, length):
+        return self.server.avatar._runAsUser(
+            [(os.lseek, (self.fd, offset, 0)), (os.read, (self.fd, length))]
+        )
+
+    def writeChunk(self, offset, data):
+        return self.server.avatar._runAsUser(
+            [(os.lseek, (self.fd, offset, 0)), (os.write, (self.fd, data))]
+        )
+
+    def getAttrs(self):
+        s = self.server.avatar._runAsUser(os.fstat, self.fd)
+        return self.server._getAttrs(s)
+
+    def setAttrs(self, attrs):
+        raise NotImplementedError
+
+
+class UnixSFTPDirectory:
+    def __init__(self, server, directory):
+        self.server = server
+        self.files = server.avatar._runAsUser(os.listdir, directory)
+        self.dir = directory
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            f = self.files.pop(0)
+        except IndexError:
+            raise StopIteration
+        else:
+            s = self.server.avatar._runAsUser(os.lstat, os.path.join(self.dir, f))
+            longname = lsLine(f, s)
+            attrs = self.server._getAttrs(s)
+            return (f, longname, attrs)
+
+    next = __next__
+
+    def close(self):
+        self.files = []
+
+
+components.registerAdapter(
+    SFTPServerForUnixConchUser, UnixConchUser, filetransfer.ISFTPServer
+)
+components.registerAdapter(SSHSessionForUnixConchUser, UnixConchUser, session.ISession)
diff --git a/contrib/python/Twisted/py3/twisted/copyright.py b/contrib/python/Twisted/py3/twisted/copyright.py
new file mode 100644
index 00000000000..9aed452d3be
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/copyright.py
@@ -0,0 +1,44 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Copyright information for Twisted.
+"""
+
+
+__all__ = ["copyright", "disclaimer", "longversion", "version"]
+
+from twisted import __version__ as version, version as _longversion
+
+longversion = str(_longversion)
+
+copyright = """\
+Copyright (c) 2001-2023 Twisted Matrix Laboratories.
+See LICENSE for details."""
+
+disclaimer = """
+Twisted, the Framework of Your Internet
+{}
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+""".format(
+    copyright,
+)
diff --git a/contrib/python/Twisted/py3/twisted/cred/__init__.py b/contrib/python/Twisted/py3/twisted/cred/__init__.py
new file mode 100644
index 00000000000..2ee268c5e90
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/cred/__init__.py
@@ -0,0 +1,7 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Cred: Support for verifying credentials, and providing services to user
+based on those credentials.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/cred/_digest.py b/contrib/python/Twisted/py3/twisted/cred/_digest.py
new file mode 100644
index 00000000000..3505e155a29
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/cred/_digest.py
@@ -0,0 +1,132 @@
+# -*- test-case-name: twisted.cred.test.test_digestauth -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Calculations for HTTP Digest authentication.
+
+@see: U{http://www.faqs.org/rfcs/rfc2617.html}
+"""
+
+
+from binascii import hexlify
+from hashlib import md5, sha1
+
+# The digest math
+
+algorithms = {
+    b"md5": md5,
+    # md5-sess is more complicated than just another algorithm.  It requires
+    # H(A1) state to be remembered from the first WWW-Authenticate challenge
+    # issued and re-used to process any Authorization header in response to
+    # that WWW-Authenticate challenge.  It is *not* correct to simply
+    # recalculate H(A1) each time an Authorization header is received.  Read
+    # RFC 2617, section 3.2.2.2 and do not try to make DigestCredentialFactory
+    # support this unless you completely understand it. -exarkun
+    b"md5-sess": md5,
+    b"sha": sha1,
+}
+
+# DigestCalcHA1
+
+
+def calcHA1(
+    pszAlg, pszUserName, pszRealm, pszPassword, pszNonce, pszCNonce, preHA1=None
+):
+    """
+    Compute H(A1) from RFC 2617.
+
+    @param pszAlg: The name of the algorithm to use to calculate the digest.
+        Currently supported are md5, md5-sess, and sha.
+    @param pszUserName: The username
+    @param pszRealm: The realm
+    @param pszPassword: The password
+    @param pszNonce: The nonce
+    @param pszCNonce: The cnonce
+
+    @param preHA1: If available this is a str containing a previously
+       calculated H(A1) as a hex string.  If this is given then the values for
+       pszUserName, pszRealm, and pszPassword must be L{None} and are ignored.
+    """
+
+    if preHA1 and (pszUserName or pszRealm or pszPassword):
+        raise TypeError(
+            "preHA1 is incompatible with the pszUserName, "
+            "pszRealm, and pszPassword arguments"
+        )
+
+    if preHA1 is None:
+        # We need to calculate the HA1 from the username:realm:password
+        m = algorithms[pszAlg]()
+        m.update(pszUserName)
+        m.update(b":")
+        m.update(pszRealm)
+        m.update(b":")
+        m.update(pszPassword)
+        HA1 = hexlify(m.digest())
+    else:
+        # We were given a username:realm:password
+        HA1 = preHA1
+
+    if pszAlg == b"md5-sess":
+        m = algorithms[pszAlg]()
+        m.update(HA1)
+        m.update(b":")
+        m.update(pszNonce)
+        m.update(b":")
+        m.update(pszCNonce)
+        HA1 = hexlify(m.digest())
+
+    return HA1
+
+
+def calcHA2(algo, pszMethod, pszDigestUri, pszQop, pszHEntity):
+    """
+    Compute H(A2) from RFC 2617.
+
+    @param algo: The name of the algorithm to use to calculate the digest.
+        Currently supported are md5, md5-sess, and sha.
+    @param pszMethod: The request method.
+    @param pszDigestUri: The request URI.
+    @param pszQop: The Quality-of-Protection value.
+    @param pszHEntity: The hash of the entity body or L{None} if C{pszQop} is
+        not C{'auth-int'}.
+    @return: The hash of the A2 value for the calculation of the response
+        digest.
+    """
+    m = algorithms[algo]()
+    m.update(pszMethod)
+    m.update(b":")
+    m.update(pszDigestUri)
+    if pszQop == b"auth-int":
+        m.update(b":")
+        m.update(pszHEntity)
+    return hexlify(m.digest())
+
+
+def calcResponse(HA1, HA2, algo, pszNonce, pszNonceCount, pszCNonce, pszQop):
+    """
+    Compute the digest for the given parameters.
+
+    @param HA1: The H(A1) value, as computed by L{calcHA1}.
+    @param HA2: The H(A2) value, as computed by L{calcHA2}.
+    @param pszNonce: The challenge nonce.
+    @param pszNonceCount: The (client) nonce count value for this response.
+    @param pszCNonce: The client nonce.
+    @param pszQop: The Quality-of-Protection value.
+    """
+    m = algorithms[algo]()
+    m.update(HA1)
+    m.update(b":")
+    m.update(pszNonce)
+    m.update(b":")
+    if pszNonceCount and pszCNonce:
+        m.update(pszNonceCount)
+        m.update(b":")
+        m.update(pszCNonce)
+        m.update(b":")
+        m.update(pszQop)
+        m.update(b":")
+    m.update(HA2)
+    respHash = hexlify(m.digest())
+    return respHash
diff --git a/contrib/python/Twisted/py3/twisted/cred/checkers.py b/contrib/python/Twisted/py3/twisted/cred/checkers.py
new file mode 100644
index 00000000000..24dd40ff173
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/cred/checkers.py
@@ -0,0 +1,334 @@
+# -*- test-case-name: twisted.cred.test.test_cred -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Basic credential checkers
+
+@var ANONYMOUS: An empty tuple used to represent the anonymous avatar ID.
+"""
+
+
+import os
+from typing import Any, Dict, Optional, Tuple, Union
+
+from zope.interface import Attribute, Interface, implementer
+
+from twisted.cred import error
+from twisted.cred.credentials import (
+    IAnonymous,
+    IUsernameHashedPassword,
+    IUsernamePassword,
+)
+from twisted.internet import defer
+from twisted.internet.defer import Deferred
+from twisted.logger import Logger
+from twisted.python import failure
+
+# A note on anonymity - We do not want None as the value for anonymous
+# because it is too easy to accidentally return it.  We do not want the
+# empty string, because it is too easy to mistype a password file.  For
+# example, an .htpasswd file may contain the lines: ['hello:asdf',
+# 'world:asdf', 'goodbye', ':world'].  This misconfiguration will have an
+# ill effect in any case, but accidentally granting anonymous access is a
+# worse failure mode than simply granting access to an untypeable
+# username.  We do not want an instance of 'object', because that would
+# create potential problems with persistence.
+
+ANONYMOUS: Tuple[()] = ()
+
+
+class ICredentialsChecker(Interface):
+    """
+    An object that can check sub-interfaces of L{ICredentials}.
+    """
+
+    credentialInterfaces = Attribute(
+        "A list of sub-interfaces of L{ICredentials} which specifies which I "
+        "may check."
+    )
+
+    def requestAvatarId(credentials: Any) -> Deferred[Union[bytes, Tuple[()]]]:
+        """
+        Validate credentials and produce an avatar ID.
+
+        @param credentials: something which implements one of the interfaces in
+            C{credentialInterfaces}.
+
+        @return: a L{Deferred} which will fire with a L{bytes} that identifies
+            an avatar, an empty tuple to specify an authenticated anonymous
+            user (provided as L{twisted.cred.checkers.ANONYMOUS}) or fail with
+            L{UnauthorizedLogin}.  Alternatively, return the result itself.
+
+        @see: L{twisted.cred.credentials}
+        """
+
+
+@implementer(ICredentialsChecker)
+class AllowAnonymousAccess:
+    """
+    A credentials checker that unconditionally grants anonymous access.
+
+    @cvar credentialInterfaces: Tuple containing L{IAnonymous}.
+    """
+
+    credentialInterfaces = (IAnonymous,)
+
+    def requestAvatarId(self, credentials):
+        """
+        Succeed with the L{ANONYMOUS} avatar ID.
+
+        @return: L{Deferred} that fires with L{twisted.cred.checkers.ANONYMOUS}
+        """
+        return defer.succeed(ANONYMOUS)
+
+
+@implementer(ICredentialsChecker)
+class InMemoryUsernamePasswordDatabaseDontUse:
+    """
+    An extremely simple credentials checker.
+
+    This is only of use in one-off test programs or examples which don't
+    want to focus too much on how credentials are verified.
+
+    You really don't want to use this for anything else.  It is, at best, a
+    toy.  If you need a simple credentials checker for a real application,
+    see L{FilePasswordDB}.
+
+    @cvar credentialInterfaces: Tuple of L{IUsernamePassword} and
+    L{IUsernameHashedPassword}.
+
+    @ivar users: Mapping of usernames to passwords.
+    @type users: L{dict} mapping L{bytes} to L{bytes}
+    """
+
+    credentialInterfaces = (
+        IUsernamePassword,
+        IUsernameHashedPassword,
+    )
+
+    def __init__(self, **users: bytes) -> None:
+        """
+        Initialize the in-memory database.
+
+        For example::
+
+            db = InMemoryUsernamePasswordDatabaseDontUse(
+                user1=b'sesame',
+                user2=b'hunter2',
+            )
+
+        @param users: Usernames and passwords to seed the database with.
+        Each username given as a keyword is encoded to L{bytes} as ASCII.
+        Passwords must be given as L{bytes}.
+        @type users: L{dict} of L{str} to L{bytes}
+        """
+        self.users = {x.encode("ascii"): y for x, y in users.items()}
+
+    def addUser(self, username: bytes, password: bytes) -> None:
+        """
+        Set a user's password.
+
+        @param username: Name of the user.
+        @type username: L{bytes}
+
+        @param password: Password to associate with the username.
+        @type password: L{bytes}
+        """
+        self.users[username] = password
+
+    def _cbPasswordMatch(self, matched, username):
+        if matched:
+            return username
+        else:
+            return failure.Failure(error.UnauthorizedLogin())
+
+    def requestAvatarId(self, credentials):
+        if credentials.username in self.users:
+            return defer.maybeDeferred(
+                credentials.checkPassword, self.users[credentials.username]
+            ).addCallback(self._cbPasswordMatch, credentials.username)
+        else:
+            return defer.fail(error.UnauthorizedLogin())
+
+
+@implementer(ICredentialsChecker)
+class FilePasswordDB:
+    """
+    A file-based, text-based username/password database.
+
+    Records in the datafile for this class are delimited by a particular
+    string.  The username appears in a fixed field of the columns delimited
+    by this string, as does the password.  Both fields are specifiable.  If
+    the passwords are not stored plaintext, a hash function must be supplied
+    to convert plaintext passwords to the form stored on disk and this
+    CredentialsChecker will only be able to check L{IUsernamePassword}
+    credentials.  If the passwords are stored plaintext,
+    L{IUsernameHashedPassword} credentials will be checkable as well.
+    """
+
+    cache = False
+    _credCache: Optional[Dict[bytes, bytes]] = None
+    _cacheTimestamp: float = 0
+    _log = Logger()
+
+    def __init__(
+        self,
+        filename,
+        delim=b":",
+        usernameField=0,
+        passwordField=1,
+        caseSensitive=True,
+        hash=None,
+        cache=False,
+    ):
+        """
+        @type filename: L{str}
+        @param filename: The name of the file from which to read username and
+        password information.
+
+        @type delim: L{bytes}
+        @param delim: The field delimiter used in the file.
+
+        @type usernameField: L{int}
+        @param usernameField: The index of the username after splitting a
+        line on the delimiter.
+
+        @type passwordField: L{int}
+        @param passwordField: The index of the password after splitting a
+        line on the delimiter.
+
+        @type caseSensitive: L{bool}
+        @param caseSensitive: If true, consider the case of the username when
+        performing a lookup.  Ignore it otherwise.
+
+        @type hash: Three-argument callable or L{None}
+        @param hash: A function used to transform the plaintext password
+        received over the network to a format suitable for comparison
+        against the version stored on disk.  The arguments to the callable
+        are the username, the network-supplied password, and the in-file
+        version of the password.  If the return value compares equal to the
+        version stored on disk, the credentials are accepted.
+
+        @type cache: L{bool}
+        @param cache: If true, maintain an in-memory cache of the
+        contents of the password file.  On lookups, the mtime of the
+        file will be checked, and the file will only be re-parsed if
+        the mtime is newer than when the cache was generated.
+        """
+        self.filename = filename
+        self.delim = delim
+        self.ufield = usernameField
+        self.pfield = passwordField
+        self.caseSensitive = caseSensitive
+        self.hash = hash
+        self.cache = cache
+
+        if self.hash is None:
+            # The passwords are stored plaintext.  We can support both
+            # plaintext and hashed passwords received over the network.
+            self.credentialInterfaces = (
+                IUsernamePassword,
+                IUsernameHashedPassword,
+            )
+        else:
+            # The passwords are hashed on disk.  We can support only
+            # plaintext passwords received over the network.
+            self.credentialInterfaces = (IUsernamePassword,)
+
+    def __getstate__(self):
+        d = dict(vars(self))
+        for k in "_credCache", "_cacheTimestamp":
+            try:
+                del d[k]
+            except KeyError:
+                pass
+        return d
+
+    def _cbPasswordMatch(self, matched, username):
+        if matched:
+            return username
+        else:
+            return failure.Failure(error.UnauthorizedLogin())
+
+    def _loadCredentials(self):
+        """
+        Loads the credentials from the configured file.
+
+        @return: An iterable of C{username, password} couples.
+        @rtype: C{iterable}
+
+        @raise UnauthorizedLogin: when failing to read the credentials from the
+            file.
+        """
+        try:
+            with open(self.filename, "rb") as f:
+                for line in f:
+                    line = line.rstrip()
+                    parts = line.split(self.delim)
+
+                    if self.ufield >= len(parts) or self.pfield >= len(parts):
+                        continue
+                    if self.caseSensitive:
+                        yield parts[self.ufield], parts[self.pfield]
+                    else:
+                        yield parts[self.ufield].lower(), parts[self.pfield]
+        except OSError as e:
+            self._log.error("Unable to load credentials db: {e!r}", e=e)
+            raise error.UnauthorizedLogin()
+
+    def getUser(self, username: bytes) -> Tuple[bytes, bytes]:
+        """
+        Look up the credentials for a username.
+
+        @param username: The username to look up.
+        @type username: L{bytes}
+
+        @returns: Two-tuple of the canonicalicalized username (i.e. lowercase
+        if the database is not case sensitive) and the associated password
+        value, both L{bytes}.
+        @rtype: L{tuple}
+
+        @raises KeyError: When lookup of the username fails.
+        """
+        if not self.caseSensitive:
+            username = username.lower()
+
+        if self.cache:
+            if (
+                self._credCache is None
+                or os.path.getmtime(self.filename) > self._cacheTimestamp
+            ):
+                self._cacheTimestamp = os.path.getmtime(self.filename)
+                self._credCache = dict(self._loadCredentials())
+            return username, self._credCache[username]
+        else:
+            for u, p in self._loadCredentials():
+                if u == username:
+                    return u, p
+            raise KeyError(username)
+
+    def requestAvatarId(
+        self, credentials: IUsernamePassword
+    ) -> Deferred[Union[bytes, Tuple[()]]]:
+        try:
+            u, p = self.getUser(credentials.username)
+        except KeyError:
+            return defer.fail(error.UnauthorizedLogin())
+        else:
+            up = IUsernamePassword(credentials, None)
+            if self.hash:
+                if up is not None:
+                    h = self.hash(up.username, up.password, p)
+                    if h == p:
+                        return defer.succeed(u)
+                return defer.fail(error.UnauthorizedLogin())
+            else:
+                return defer.maybeDeferred(credentials.checkPassword, p).addCallback(
+                    self._cbPasswordMatch, u
+                )
+
+
+# For backwards compatibility
+# Allow access as the old name.
+OnDiskUsernamePasswordDatabase = FilePasswordDB
diff --git a/contrib/python/Twisted/py3/twisted/cred/credentials.py b/contrib/python/Twisted/py3/twisted/cred/credentials.py
new file mode 100644
index 00000000000..662913951c3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/cred/credentials.py
@@ -0,0 +1,508 @@
+# -*- test-case-name: twisted.cred.test.test_cred-*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module defines L{ICredentials}, an interface for objects that represent
+authentication credentials to provide, and also includes a number of useful
+implementations of that interface.
+"""
+
+
+import base64
+import hmac
+import random
+import re
+import time
+from binascii import hexlify
+from hashlib import md5
+
+from zope.interface import Interface, implementer
+
+from twisted.cred import error
+from twisted.cred._digest import calcHA1, calcHA2, calcResponse
+from twisted.python.compat import nativeString, networkString
+from twisted.python.deprecate import deprecatedModuleAttribute
+from twisted.python.randbytes import secureRandom
+from twisted.python.versions import Version
+
+
+class ICredentials(Interface):
+    """
+    I check credentials.
+
+    Implementors I{must} specify the sub-interfaces of ICredentials
+    to which it conforms, using L{zope.interface.implementer}.
+    """
+
+
+class IUsernameDigestHash(ICredentials):
+    """
+    This credential is used when a CredentialChecker has access to the hash
+    of the username:realm:password as in an Apache .htdigest file.
+    """
+
+    def checkHash(digestHash):
+        """
+        @param digestHash: The hashed username:realm:password to check against.
+
+        @return: C{True} if the credentials represented by this object match
+            the given hash, C{False} if they do not, or a L{Deferred} which
+            will be called back with one of these values.
+        """
+
+
+class IUsernameHashedPassword(ICredentials):
+    """
+    I encapsulate a username and a hashed password.
+
+    This credential is used when a hashed password is received from the
+    party requesting authentication.  CredentialCheckers which check this
+    kind of credential must store the passwords in plaintext (or as
+    password-equivalent hashes) form so that they can be hashed in a manner
+    appropriate for the particular credentials class.
+
+    @type username: L{bytes}
+    @ivar username: The username associated with these credentials.
+    """
+
+    def checkPassword(password):
+        """
+        Validate these credentials against the correct password.
+
+        @type password: L{bytes}
+        @param password: The correct, plaintext password against which to
+        check.
+
+        @rtype: C{bool} or L{Deferred}
+        @return: C{True} if the credentials represented by this object match the
+            given password, C{False} if they do not, or a L{Deferred} which will
+            be called back with one of these values.
+        """
+
+
+class IUsernamePassword(ICredentials):
+    """
+    I encapsulate a username and a plaintext password.
+
+    This encapsulates the case where the password received over the network
+    has been hashed with the identity function (That is, not at all).  The
+    CredentialsChecker may store the password in whatever format it desires,
+    it need only transform the stored password in a similar way before
+    performing the comparison.
+
+    @type username: L{bytes}
+    @ivar username: The username associated with these credentials.
+
+    @type password: L{bytes}
+    @ivar password: The password associated with these credentials.
+    """
+
+    username: bytes
+    password: bytes
+
+    def checkPassword(password: bytes) -> bool:
+        """
+        Validate these credentials against the correct password.
+
+        @type password: L{bytes}
+        @param password: The correct, plaintext password against which to
+        check.
+
+        @rtype: C{bool} or L{Deferred}
+        @return: C{True} if the credentials represented by this object match the
+            given password, C{False} if they do not, or a L{Deferred} which will
+            be called back with one of these values.
+        """
+
+
+class IAnonymous(ICredentials):
+    """
+    I am an explicitly anonymous request for access.
+
+    @see: L{twisted.cred.checkers.AllowAnonymousAccess}
+    """
+
+
+@implementer(IUsernameHashedPassword, IUsernameDigestHash)
+class DigestedCredentials:
+    """
+    Yet Another Simple HTTP Digest authentication scheme.
+    """
+
+    def __init__(self, username, method, realm, fields):
+        self.username = username
+        self.method = method
+        self.realm = realm
+        self.fields = fields
+
+    def checkPassword(self, password):
+        """
+        Verify that the credentials represented by this object agree with the
+        given plaintext C{password} by hashing C{password} in the same way the
+        response hash represented by this object was generated and comparing
+        the results.
+        """
+        response = self.fields.get("response")
+        uri = self.fields.get("uri")
+        nonce = self.fields.get("nonce")
+        cnonce = self.fields.get("cnonce")
+        nc = self.fields.get("nc")
+        algo = self.fields.get("algorithm", b"md5").lower()
+        qop = self.fields.get("qop", b"auth")
+
+        expected = calcResponse(
+            calcHA1(algo, self.username, self.realm, password, nonce, cnonce),
+            calcHA2(algo, self.method, uri, qop, None),
+            algo,
+            nonce,
+            nc,
+            cnonce,
+            qop,
+        )
+
+        return expected == response
+
+    def checkHash(self, digestHash):
+        """
+        Verify that the credentials represented by this object agree with the
+        credentials represented by the I{H(A1)} given in C{digestHash}.
+
+        @param digestHash: A precomputed H(A1) value based on the username,
+            realm, and password associate with this credentials object.
+        """
+        response = self.fields.get("response")
+        uri = self.fields.get("uri")
+        nonce = self.fields.get("nonce")
+        cnonce = self.fields.get("cnonce")
+        nc = self.fields.get("nc")
+        algo = self.fields.get("algorithm", b"md5").lower()
+        qop = self.fields.get("qop", b"auth")
+
+        expected = calcResponse(
+            calcHA1(algo, None, None, None, nonce, cnonce, preHA1=digestHash),
+            calcHA2(algo, self.method, uri, qop, None),
+            algo,
+            nonce,
+            nc,
+            cnonce,
+            qop,
+        )
+
+        return expected == response
+
+
+class DigestCredentialFactory:
+    """
+    Support for RFC2617 HTTP Digest Authentication
+
+    @cvar CHALLENGE_LIFETIME_SECS: The number of seconds for which an
+        opaque should be valid.
+
+    @type privateKey: L{bytes}
+    @ivar privateKey: A random string used for generating the secure opaque.
+
+    @type algorithm: L{bytes}
+    @param algorithm: Case insensitive string specifying the hash algorithm to
+        use.  Must be either C{'md5'} or C{'sha'}.  C{'md5-sess'} is B{not}
+        supported.
+
+    @type authenticationRealm: L{bytes}
+    @param authenticationRealm: case sensitive string that specifies the realm
+        portion of the challenge
+    """
+
+    _parseparts = re.compile(
+        b"([^= ]+)"  # The key
+        b"="  # Conventional key/value separator (literal)
+        b"(?:"  # Group together a couple options
+        b'"([^"]*)"'  # A quoted string of length 0 or more
+        b"|"  # The other option in the group is coming
+        b"([^,]+)"  # An unquoted string of length 1 or more, up to a comma
+        b")"  # That non-matching group ends
+        b",?"
+    )  # There might be a comma at the end (none on last pair)
+
+    CHALLENGE_LIFETIME_SECS = 15 * 60  # 15 minutes
+
+    scheme = b"digest"
+
+    def __init__(self, algorithm, authenticationRealm):
+        self.algorithm = algorithm
+        self.authenticationRealm = authenticationRealm
+        self.privateKey = secureRandom(12)
+
+    def getChallenge(self, address):
+        """
+        Generate the challenge for use in the WWW-Authenticate header.
+
+        @param address: The client address to which this challenge is being
+            sent.
+
+        @return: The L{dict} that can be used to generate a WWW-Authenticate
+            header.
+        """
+        c = self._generateNonce()
+        o = self._generateOpaque(c, address)
+
+        return {
+            "nonce": c,
+            "opaque": o,
+            "qop": b"auth",
+            "algorithm": self.algorithm,
+            "realm": self.authenticationRealm,
+        }
+
+    def _generateNonce(self):
+        """
+        Create a random value suitable for use as the nonce parameter of a
+        WWW-Authenticate challenge.
+
+        @rtype: L{bytes}
+        """
+        return hexlify(secureRandom(12))
+
+    def _getTime(self):
+        """
+        Parameterize the time based seed used in C{_generateOpaque}
+        so we can deterministically unittest it's behavior.
+        """
+        return time.time()
+
+    def _generateOpaque(self, nonce, clientip):
+        """
+        Generate an opaque to be returned to the client.  This is a unique
+        string that can be returned to us and verified.
+        """
+        # Now, what we do is encode the nonce, client ip and a timestamp in the
+        # opaque value with a suitable digest.
+        now = b"%d" % (int(self._getTime()),)
+
+        if not clientip:
+            clientip = b""
+        elif isinstance(clientip, str):
+            clientip = clientip.encode("ascii")
+
+        key = b",".join((nonce, clientip, now))
+        digest = hexlify(md5(key + self.privateKey).digest())
+        ekey = base64.b64encode(key)
+        return b"-".join((digest, ekey.replace(b"\n", b"")))
+
+    def _verifyOpaque(self, opaque, nonce, clientip):
+        """
+        Given the opaque and nonce from the request, as well as the client IP
+        that made the request, verify that the opaque was generated by us.
+        And that it's not too old.
+
+        @param opaque: The opaque value from the Digest response
+        @param nonce: The nonce value from the Digest response
+        @param clientip: The remote IP address of the client making the request
+            or L{None} if the request was submitted over a channel where this
+            does not make sense.
+
+        @return: C{True} if the opaque was successfully verified.
+
+        @raise error.LoginFailed: if C{opaque} could not be parsed or
+            contained the wrong values.
+        """
+        # First split the digest from the key
+        opaqueParts = opaque.split(b"-")
+        if len(opaqueParts) != 2:
+            raise error.LoginFailed("Invalid response, invalid opaque value")
+
+        if not clientip:
+            clientip = b""
+        elif isinstance(clientip, str):
+            clientip = clientip.encode("ascii")
+
+        # Verify the key
+        key = base64.b64decode(opaqueParts[1])
+        keyParts = key.split(b",")
+
+        if len(keyParts) != 3:
+            raise error.LoginFailed("Invalid response, invalid opaque value")
+
+        if keyParts[0] != nonce:
+            raise error.LoginFailed(
+                "Invalid response, incompatible opaque/nonce values"
+            )
+
+        if keyParts[1] != clientip:
+            raise error.LoginFailed(
+                "Invalid response, incompatible opaque/client values"
+            )
+
+        try:
+            when = int(keyParts[2])
+        except ValueError:
+            raise error.LoginFailed("Invalid response, invalid opaque/time values")
+
+        if (
+            int(self._getTime()) - when
+            > DigestCredentialFactory.CHALLENGE_LIFETIME_SECS
+        ):
+            raise error.LoginFailed(
+                "Invalid response, incompatible opaque/nonce too old"
+            )
+
+        # Verify the digest
+        digest = hexlify(md5(key + self.privateKey).digest())
+        if digest != opaqueParts[0]:
+            raise error.LoginFailed("Invalid response, invalid opaque value")
+
+        return True
+
+    def decode(self, response, method, host):
+        """
+        Decode the given response and attempt to generate a
+        L{DigestedCredentials} from it.
+
+        @type response: L{bytes}
+        @param response: A string of comma separated key=value pairs
+
+        @type method: L{bytes}
+        @param method: The action requested to which this response is addressed
+            (GET, POST, INVITE, OPTIONS, etc).
+
+        @type host: L{bytes}
+        @param host: The address the request was sent from.
+
+        @raise error.LoginFailed: If the response does not contain a username,
+            a nonce, an opaque, or if the opaque is invalid.
+
+        @return: L{DigestedCredentials}
+        """
+        response = b" ".join(response.splitlines())
+        parts = self._parseparts.findall(response)
+        auth = {}
+        for key, bare, quoted in parts:
+            value = (quoted or bare).strip()
+            auth[nativeString(key.strip())] = value
+
+        username = auth.get("username")
+        if not username:
+            raise error.LoginFailed("Invalid response, no username given.")
+
+        if "opaque" not in auth:
+            raise error.LoginFailed("Invalid response, no opaque given.")
+
+        if "nonce" not in auth:
+            raise error.LoginFailed("Invalid response, no nonce given.")
+
+        # Now verify the nonce/opaque values for this client
+        if self._verifyOpaque(auth.get("opaque"), auth.get("nonce"), host):
+            return DigestedCredentials(username, method, self.authenticationRealm, auth)
+
+
+@implementer(IUsernameHashedPassword)
+class CramMD5Credentials:
+    """
+    An encapsulation of some CramMD5 hashed credentials.
+
+    @ivar challenge: The challenge to be sent to the client.
+    @type challenge: L{bytes}
+
+    @ivar response: The hashed response from the client.
+    @type response: L{bytes}
+
+    @ivar username: The username from the response from the client.
+    @type username: L{bytes} or L{None} if not yet provided.
+    """
+
+    username = None
+    challenge = b""
+    response = b""
+
+    def __init__(self, host=None):
+        self.host = host
+
+    def getChallenge(self):
+        if self.challenge:
+            return self.challenge
+        # The data encoded in the first ready response contains an
+        # presumptively arbitrary string of random digits, a timestamp, and
+        # the fully-qualified primary host name of the server.  The syntax of
+        # the unencoded form must correspond to that of an RFC 822 'msg-id'
+        # [RFC822] as described in [POP3].
+        #   -- RFC 2195
+        r = random.randrange(0x7FFFFFFF)
+        t = time.time()
+        self.challenge = networkString(
+            "<%d.%d@%s>" % (r, t, nativeString(self.host) if self.host else None)
+        )
+        return self.challenge
+
+    def setResponse(self, response):
+        self.username, self.response = response.split(None, 1)
+
+    def moreChallenges(self):
+        return False
+
+    def checkPassword(self, password):
+        verify = hexlify(hmac.HMAC(password, self.challenge, digestmod=md5).digest())
+        return verify == self.response
+
+
+@implementer(IUsernameHashedPassword)
+class UsernameHashedPassword:
+    deprecatedModuleAttribute(
+        Version("Twisted", 21, 2, 0),
+        "Use twisted.cred.credentials.UsernamePassword instead.",
+        "twisted.cred.credentials",
+        "UsernameHashedPassword",
+    )
+
+    def __init__(self, username, hashed):
+        self.username = username
+        self.hashed = hashed
+
+    def checkPassword(self, password):
+        return self.hashed == password
+
+
+@implementer(IUsernamePassword)
+class UsernamePassword:
+    def __init__(self, username: bytes, password: bytes) -> None:
+        self.username = username
+        self.password = password
+
+    def checkPassword(self, password: bytes) -> bool:
+        return self.password == password
+
+
+@implementer(IAnonymous)
+class Anonymous:
+    pass
+
+
+class ISSHPrivateKey(ICredentials):
+    """
+    L{ISSHPrivateKey} credentials encapsulate an SSH public key to be checked
+    against a user's private key.
+
+    @ivar username: The username associated with these credentials.
+    @type username: L{bytes}
+
+    @ivar algName: The algorithm name for the blob.
+    @type algName: L{bytes}
+
+    @ivar blob: The public key blob as sent by the client.
+    @type blob: L{bytes}
+
+    @ivar sigData: The data the signature was made from.
+    @type sigData: L{bytes}
+
+    @ivar signature: The signed data.  This is checked to verify that the user
+        owns the private key.
+    @type signature: L{bytes} or L{None}
+    """
+
+
+@implementer(ISSHPrivateKey)
+class SSHPrivateKey:
+    def __init__(self, username, algName, blob, sigData, signature):
+        self.username = username
+        self.algName = algName
+        self.blob = blob
+        self.sigData = sigData
+        self.signature = signature
diff --git a/contrib/python/Twisted/py3/twisted/cred/error.py b/contrib/python/Twisted/py3/twisted/cred/error.py
new file mode 100644
index 00000000000..8e13d730839
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/cred/error.py
@@ -0,0 +1,38 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Cred errors.
+"""
+
+
+class Unauthorized(Exception):
+    """Standard unauthorized error."""
+
+
+class LoginFailed(Exception):
+    """
+    The user's request to log in failed for some reason.
+    """
+
+
+class UnauthorizedLogin(LoginFailed, Unauthorized):
+    """The user was not authorized to log in."""
+
+
+class UnhandledCredentials(LoginFailed):
+    """A type of credentials were passed in with no knowledge of how to check
+    them.  This is a server configuration error - it means that a protocol was
+    connected to a Portal without a CredentialChecker that can check all of its
+    potential authentication strategies.
+    """
+
+
+class LoginDenied(LoginFailed):
+    """
+    The realm rejected this login for some reason.
+
+    Examples of reasons this might be raised include an avatar logging in
+    too frequently, a quota having been fully used, or the overall server
+    load being too high.
+    """
diff --git a/contrib/python/Twisted/py3/twisted/cred/portal.py b/contrib/python/Twisted/py3/twisted/cred/portal.py
new file mode 100644
index 00000000000..3e5abde918d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/cred/portal.py
@@ -0,0 +1,154 @@
+# -*- test-case-name: twisted.cred.test.test_cred -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+The point of integration of application and authentication.
+"""
+
+
+from typing import Callable, Dict, Iterable, List, Tuple, Type, Union
+
+from zope.interface import Interface, providedBy
+
+from twisted.cred import error
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.cred.credentials import ICredentials
+from twisted.internet import defer
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.python import failure, reflect
+
+# To say 'we need an Interface object', we have to say Type[Interface];
+# although zope.interface has no type/instance distinctions within the
+# implementation of Interface itself (subclassing it actually instantiates it),
+# since mypy-zope treats Interface objects *as* types, this is how you have to
+# treat it.
+_InterfaceItself = Type[Interface]
+
+# This is the result shape for both IRealm.requestAvatar and Portal.login,
+# although the former is optionally allowed to return synchronously and the
+# latter must be Deferred.
+_requestResult = Tuple[_InterfaceItself, object, Callable[[], None]]
+
+
+class IRealm(Interface):
+    """
+    The realm connects application-specific objects to the
+    authentication system.
+    """
+
+    def requestAvatar(
+        avatarId: Union[bytes, Tuple[()]], mind: object, *interfaces: _InterfaceItself
+    ) -> Union[Deferred[_requestResult], _requestResult]:
+        """
+        Return avatar which provides one of the given interfaces.
+
+        @param avatarId: a string that identifies an avatar, as returned by
+            L{ICredentialsChecker.requestAvatarId<twisted.cred.checkers.ICredentialsChecker.requestAvatarId>}
+            (via a Deferred).  Alternatively, it may be
+            C{twisted.cred.checkers.ANONYMOUS}.
+        @param mind: usually None.  See the description of mind in
+            L{Portal.login}.
+        @param interfaces: the interface(s) the returned avatar should
+            implement, e.g.  C{IMailAccount}.  See the description of
+            L{Portal.login}.
+
+        @returns: a deferred which will fire a tuple of (interface,
+            avatarAspect, logout), or the tuple itself.  The interface will be
+            one of the interfaces passed in the 'interfaces' argument.  The
+            'avatarAspect' will implement that interface.  The 'logout' object
+            is a callable which will detach the mind from the avatar.
+        """
+
+
+class Portal:
+    """
+    A mediator between clients and a realm.
+
+    A portal is associated with one Realm and zero or more credentials checkers.
+    When a login is attempted, the portal finds the appropriate credentials
+    checker for the credentials given, invokes it, and if the credentials are
+    valid, retrieves the appropriate avatar from the Realm.
+
+    This class is not intended to be subclassed.  Customization should be done
+    in the realm object and in the credentials checker objects.
+    """
+
+    checkers: Dict[Type[Interface], ICredentialsChecker]
+
+    def __init__(
+        self, realm: IRealm, checkers: Iterable[ICredentialsChecker] = ()
+    ) -> None:
+        """
+        Create a Portal to a L{IRealm}.
+        """
+        self.realm = realm
+        self.checkers = {}
+        for checker in checkers:
+            self.registerChecker(checker)
+
+    def listCredentialsInterfaces(self) -> List[Type[Interface]]:
+        """
+        Return list of credentials interfaces that can be used to login.
+        """
+        return list(self.checkers.keys())
+
+    def registerChecker(
+        self, checker: ICredentialsChecker, *credentialInterfaces: Type[Interface]
+    ) -> None:
+        if not credentialInterfaces:
+            credentialInterfaces = checker.credentialInterfaces
+        for credentialInterface in credentialInterfaces:
+            self.checkers[credentialInterface] = checker
+
+    def login(
+        self, credentials: ICredentials, mind: object, *interfaces: Type[Interface]
+    ) -> Deferred[_requestResult]:
+        """
+        @param credentials: an implementor of
+            L{twisted.cred.credentials.ICredentials}
+
+        @param mind: an object which implements a client-side interface for
+            your particular realm.  In many cases, this may be None, so if the
+            word 'mind' confuses you, just ignore it.
+
+        @param interfaces: list of interfaces for the perspective that the mind
+            wishes to attach to. Usually, this will be only one interface, for
+            example IMailAccount. For highly dynamic protocols, however, this
+            may be a list like (IMailAccount, IUserChooser, IServiceInfo).  To
+            expand: if we are speaking to the system over IMAP, any information
+            that will be relayed to the user MUST be returned as an
+            IMailAccount implementor; IMAP clients would not be able to
+            understand anything else. Any information about unusual status
+            would have to be relayed as a single mail message in an
+            otherwise-empty mailbox. However, in a web-based mail system, or a
+            PB-based client, the ``mind'' object inside the web server
+            (implemented with a dynamic page-viewing mechanism such as a
+            Twisted Web Resource) or on the user's client program may be
+            intelligent enough to respond to several ``server''-side
+            interfaces.
+
+        @return: A deferred which will fire a tuple of (interface,
+            avatarAspect, logout).  The interface will be one of the interfaces
+            passed in the 'interfaces' argument.  The 'avatarAspect' will
+            implement that interface. The 'logout' object is a callable which
+            will detach the mind from the avatar. It must be called when the
+            user has conceptually disconnected from the service. Although in
+            some cases this will not be in connectionLost (such as in a
+            web-based session), it will always be at the end of a user's
+            interactive session.
+        """
+        for i in self.checkers:
+            if i.providedBy(credentials):
+                return maybeDeferred(
+                    self.checkers[i].requestAvatarId, credentials
+                ).addCallback(self.realm.requestAvatar, mind, *interfaces)
+        ifac = providedBy(credentials)
+        return defer.fail(
+            failure.Failure(
+                error.UnhandledCredentials(
+                    "No checker for %s" % ", ".join(map(reflect.qual, ifac))
+                )
+            )
+        )
diff --git a/contrib/python/Twisted/py3/twisted/cred/strcred.py b/contrib/python/Twisted/py3/twisted/cred/strcred.py
new file mode 100644
index 00000000000..574cd8e38e5
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/cred/strcred.py
@@ -0,0 +1,250 @@
+# -*- test-case-name: twisted.cred.test.test_strcred -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+#
+
+"""
+Support for resolving command-line strings that represent different
+checkers available to cred.
+
+Examples:
+ - passwd:/etc/passwd
+ - memory:admin:asdf:user:lkj
+ - unix
+"""
+
+
+import sys
+from typing import Optional, Sequence, Type
+
+from zope.interface import Attribute, Interface
+
+from twisted.plugin import getPlugins
+from twisted.python import usage
+
+
+class ICheckerFactory(Interface):
+    """
+    A factory for objects which provide
+    L{twisted.cred.checkers.ICredentialsChecker}.
+
+    It's implemented by twistd plugins creating checkers.
+    """
+
+    authType = Attribute("A tag that identifies the authentication method.")
+
+    authHelp = Attribute(
+        "A detailed (potentially multi-line) description of precisely "
+        "what functionality this CheckerFactory provides."
+    )
+
+    argStringFormat = Attribute(
+        "A short (one-line) description of the argument string format."
+    )
+
+    credentialInterfaces = Attribute(
+        "A list of credentials interfaces that this factory will support."
+    )
+
+    def generateChecker(argstring):
+        """
+        Return an L{twisted.cred.checkers.ICredentialsChecker} provider using the supplied
+        argument string.
+        """
+
+
+class StrcredException(Exception):
+    """
+    Base exception class for strcred.
+    """
+
+
+class InvalidAuthType(StrcredException):
+    """
+    Raised when a user provides an invalid identifier for the
+    authentication plugin (known as the authType).
+    """
+
+
+class InvalidAuthArgumentString(StrcredException):
+    """
+    Raised by an authentication plugin when the argument string
+    provided is formatted incorrectly.
+    """
+
+
+class UnsupportedInterfaces(StrcredException):
+    """
+    Raised when an application is given a checker to use that does not
+    provide any of the application's supported credentials interfaces.
+    """
+
+
+# This will be used to warn the users whenever they view help for an
+# authType that is not supported by the application.
+notSupportedWarning = "WARNING: This authType is not supported by " "this application."
+
+
+def findCheckerFactories():
+    """
+    Find all objects that implement L{ICheckerFactory}.
+    """
+    return getPlugins(ICheckerFactory)
+
+
+def findCheckerFactory(authType):
+    """
+    Find the first checker factory that supports the given authType.
+    """
+    for factory in findCheckerFactories():
+        if factory.authType == authType:
+            return factory
+    raise InvalidAuthType(authType)
+
+
+def makeChecker(description):
+    """
+    Returns an L{twisted.cred.checkers.ICredentialsChecker} based on the
+    contents of a descriptive string. Similar to
+    L{twisted.application.strports}.
+    """
+    if ":" in description:
+        authType, argstring = description.split(":", 1)
+    else:
+        authType = description
+        argstring = ""
+    return findCheckerFactory(authType).generateChecker(argstring)
+
+
+class AuthOptionMixin:
+    """
+    Defines helper methods that can be added on to any
+    L{usage.Options} subclass that needs authentication.
+
+    This mixin implements three new options methods:
+
+    The opt_auth method (--auth) will write two new values to the
+    'self' dictionary: C{credInterfaces} (a dict of lists) and
+    C{credCheckers} (a list).
+
+    The opt_help_auth method (--help-auth) will search for all
+    available checker plugins and list them for the user; it will exit
+    when finished.
+
+    The opt_help_auth_type method (--help-auth-type) will display
+    detailed help for a particular checker plugin.
+
+    @cvar supportedInterfaces: An iterable object that returns
+       credential interfaces which this application is able to support.
+
+    @cvar authOutput: A writeable object to which this options class
+        will send all help-related output. Default: L{sys.stdout}
+    """
+
+    supportedInterfaces: Optional[Sequence[Type[Interface]]] = None
+    authOutput = sys.stdout
+
+    def supportsInterface(self, interface):
+        """
+        Returns whether a particular credentials interface is supported.
+        """
+        return self.supportedInterfaces is None or interface in self.supportedInterfaces
+
+    def supportsCheckerFactory(self, factory):
+        """
+        Returns whether a checker factory will provide at least one of
+        the credentials interfaces that we care about.
+        """
+        for interface in factory.credentialInterfaces:
+            if self.supportsInterface(interface):
+                return True
+        return False
+
+    def addChecker(self, checker):
+        """
+        Supply a supplied credentials checker to the Options class.
+        """
+        # First figure out which interfaces we're willing to support.
+        supported = []
+        if self.supportedInterfaces is None:
+            supported = checker.credentialInterfaces
+        else:
+            for interface in checker.credentialInterfaces:
+                if self.supportsInterface(interface):
+                    supported.append(interface)
+        if not supported:
+            raise UnsupportedInterfaces(checker.credentialInterfaces)
+        # If we get this far, then we know we can use this checker.
+        if "credInterfaces" not in self:
+            self["credInterfaces"] = {}
+        if "credCheckers" not in self:
+            self["credCheckers"] = []
+        self["credCheckers"].append(checker)
+        for interface in supported:
+            self["credInterfaces"].setdefault(interface, []).append(checker)
+
+    def opt_auth(self, description):
+        """
+        Specify an authentication method for the server.
+        """
+        try:
+            self.addChecker(makeChecker(description))
+        except UnsupportedInterfaces as e:
+            raise usage.UsageError("Auth plugin not supported: %s" % e.args[0])
+        except InvalidAuthType as e:
+            raise usage.UsageError("Auth plugin not recognized: %s" % e.args[0])
+        except Exception as e:
+            raise usage.UsageError("Unexpected error: %s" % e)
+
+    def _checkerFactoriesForOptHelpAuth(self):
+        """
+        Return a list of which authTypes will be displayed by --help-auth.
+        This makes it a lot easier to test this module.
+        """
+        for factory in findCheckerFactories():
+            for interface in factory.credentialInterfaces:
+                if self.supportsInterface(interface):
+                    yield factory
+                    break
+
+    def opt_help_auth(self):
+        """
+        Show all authentication methods available.
+        """
+        self.authOutput.write("Usage: --auth AuthType[:ArgString]\n")
+        self.authOutput.write("For detailed help: --help-auth-type AuthType\n")
+        self.authOutput.write("\n")
+        # Figure out the right width for our columns
+        firstLength = 0
+        for factory in self._checkerFactoriesForOptHelpAuth():
+            if len(factory.authType) > firstLength:
+                firstLength = len(factory.authType)
+        formatString = "  %%-%is\t%%s\n" % firstLength
+        self.authOutput.write(formatString % ("AuthType", "ArgString format"))
+        self.authOutput.write(formatString % ("========", "================"))
+        for factory in self._checkerFactoriesForOptHelpAuth():
+            self.authOutput.write(
+                formatString % (factory.authType, factory.argStringFormat)
+            )
+        self.authOutput.write("\n")
+        raise SystemExit(0)
+
+    def opt_help_auth_type(self, authType):
+        """
+        Show help for a particular authentication type.
+        """
+        try:
+            cf = findCheckerFactory(authType)
+        except InvalidAuthType:
+            raise usage.UsageError("Invalid auth type: %s" % authType)
+        self.authOutput.write("Usage: --auth %s[:ArgString]\n" % authType)
+        self.authOutput.write("ArgString format: %s\n" % cf.argStringFormat)
+        self.authOutput.write("\n")
+        for line in cf.authHelp.strip().splitlines():
+            self.authOutput.write("  %s\n" % line.rstrip())
+        self.authOutput.write("\n")
+        if not self.supportsCheckerFactory(cf):
+            self.authOutput.write("  %s\n" % notSupportedWarning)
+            self.authOutput.write("\n")
+        raise SystemExit(0)
diff --git a/contrib/python/Twisted/py3/twisted/enterprise/__init__.py b/contrib/python/Twisted/py3/twisted/enterprise/__init__.py
new file mode 100644
index 00000000000..55d809aefcb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/enterprise/__init__.py
@@ -0,0 +1,8 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Enterprise: Database support for Twisted services.
+"""
+
+__all__ = ["adbapi"]
diff --git a/contrib/python/Twisted/py3/twisted/enterprise/adbapi.py b/contrib/python/Twisted/py3/twisted/enterprise/adbapi.py
new file mode 100644
index 00000000000..06a7c5838cf
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/enterprise/adbapi.py
@@ -0,0 +1,478 @@
+# -*- test-case-name: twisted.test.test_adbapi -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An asynchronous mapping to U{DB-API
+2.0<http://www.python.org/topics/database/DatabaseAPI-2.0.html>}.
+"""
+
+
+from twisted.internet import threads
+from twisted.python import log, reflect
+
+
+class ConnectionLost(Exception):
+    """
+    This exception means that a db connection has been lost.  Client code may
+    try again.
+    """
+
+
+class Connection:
+    """
+    A wrapper for a DB-API connection instance.
+
+    The wrapper passes almost everything to the wrapped connection and so has
+    the same API. However, the L{Connection} knows about its pool and also
+    handle reconnecting should when the real connection dies.
+    """
+
+    def __init__(self, pool):
+        self._pool = pool
+        self._connection = None
+        self.reconnect()
+
+    def close(self):
+        # The way adbapi works right now means that closing a connection is
+        # a really bad thing  as it leaves a dead connection associated with
+        # a thread in the thread pool.
+        # Really, I think closing a pooled connection should return it to the
+        # pool but that's handled by the runWithConnection method already so,
+        # rather than upsetting anyone by raising an exception, let's ignore
+        # the request
+        pass
+
+    def rollback(self):
+        if not self._pool.reconnect:
+            self._connection.rollback()
+            return
+
+        try:
+            self._connection.rollback()
+            curs = self._connection.cursor()
+            curs.execute(self._pool.good_sql)
+            curs.close()
+            self._connection.commit()
+            return
+        except BaseException:
+            log.err(None, "Rollback failed")
+
+        self._pool.disconnect(self._connection)
+
+        if self._pool.noisy:
+            log.msg("Connection lost.")
+
+        raise ConnectionLost()
+
+    def reconnect(self):
+        if self._connection is not None:
+            self._pool.disconnect(self._connection)
+        self._connection = self._pool.connect()
+
+    def __getattr__(self, name):
+        return getattr(self._connection, name)
+
+
+class Transaction:
+    """
+    A lightweight wrapper for a DB-API 'cursor' object.
+
+    Relays attribute access to the DB cursor. That is, you can call
+    C{execute()}, C{fetchall()}, etc., and they will be called on the
+    underlying DB-API cursor object. Attributes will also be retrieved from
+    there.
+    """
+
+    _cursor = None
+
+    def __init__(self, pool, connection):
+        self._pool = pool
+        self._connection = connection
+        self.reopen()
+
+    def close(self):
+        _cursor = self._cursor
+        self._cursor = None
+        _cursor.close()
+
+    def reopen(self):
+        if self._cursor is not None:
+            self.close()
+
+        try:
+            self._cursor = self._connection.cursor()
+            return
+        except BaseException:
+            if not self._pool.reconnect:
+                raise
+            else:
+                log.err(None, "Cursor creation failed")
+
+        if self._pool.noisy:
+            log.msg("Connection lost, reconnecting")
+
+        self.reconnect()
+        self._cursor = self._connection.cursor()
+
+    def reconnect(self):
+        self._connection.reconnect()
+        self._cursor = None
+
+    def __getattr__(self, name):
+        return getattr(self._cursor, name)
+
+
+class ConnectionPool:
+    """
+    Represent a pool of connections to a DB-API 2.0 compliant database.
+
+    @ivar connectionFactory: factory for connections, default to L{Connection}.
+    @type connectionFactory: any callable.
+
+    @ivar transactionFactory: factory for transactions, default to
+        L{Transaction}.
+    @type transactionFactory: any callable
+
+    @ivar shutdownID: L{None} or a handle on the shutdown event trigger which
+        will be used to stop the connection pool workers when the reactor
+        stops.
+
+    @ivar _reactor: The reactor which will be used to schedule startup and
+        shutdown events.
+    @type _reactor: L{IReactorCore} provider
+    """
+
+    CP_ARGS = "min max name noisy openfun reconnect good_sql".split()
+
+    noisy = False  # If true, generate informational log messages
+    min = 3  # Minimum number of connections in pool
+    max = 5  # Maximum number of connections in pool
+    name = None  # Name to assign to thread pool for debugging
+    openfun = None  # A function to call on new connections
+    reconnect = False  # Reconnect when connections fail
+    good_sql = "select 1"  # A query which should always succeed
+
+    running = False  # True when the pool is operating
+    connectionFactory = Connection
+    transactionFactory = Transaction
+
+    # Initialize this to None so it's available in close() even if start()
+    # never runs.
+    shutdownID = None
+
+    def __init__(self, dbapiName, *connargs, **connkw):
+        """
+        Create a new L{ConnectionPool}.
+
+        Any positional or keyword arguments other than those documented here
+        are passed to the DB-API object when connecting. Use these arguments to
+        pass database names, usernames, passwords, etc.
+
+        @param dbapiName: an import string to use to obtain a DB-API compatible
+            module (e.g. C{'pyPgSQL.PgSQL'})
+
+        @keyword cp_min: the minimum number of connections in pool (default 3)
+
+        @keyword cp_max: the maximum number of connections in pool (default 5)
+
+        @keyword cp_noisy: generate informational log messages during operation
+            (default C{False})
+
+        @keyword cp_openfun: a callback invoked after every C{connect()} on the
+            underlying DB-API object. The callback is passed a new DB-API
+            connection object. This callback can setup per-connection state
+            such as charset, timezone, etc.
+
+        @keyword cp_reconnect: detect connections which have failed and reconnect
+            (default C{False}). Failed connections may result in
+            L{ConnectionLost} exceptions, which indicate the query may need to
+            be re-sent.
+
+        @keyword cp_good_sql: an sql query which should always succeed and change
+            no state (default C{'select 1'})
+
+        @keyword cp_reactor: use this reactor instead of the global reactor
+            (added in Twisted 10.2).
+        @type cp_reactor: L{IReactorCore} provider
+        """
+        self.dbapiName = dbapiName
+        self.dbapi = reflect.namedModule(dbapiName)
+
+        if getattr(self.dbapi, "apilevel", None) != "2.0":
+            log.msg("DB API module not DB API 2.0 compliant.")
+
+        if getattr(self.dbapi, "threadsafety", 0) < 1:
+            log.msg("DB API module not sufficiently thread-safe.")
+
+        reactor = connkw.pop("cp_reactor", None)
+        if reactor is None:
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+        self.connargs = connargs
+        self.connkw = connkw
+
+        for arg in self.CP_ARGS:
+            cpArg = f"cp_{arg}"
+            if cpArg in connkw:
+                setattr(self, arg, connkw[cpArg])
+                del connkw[cpArg]
+
+        self.min = min(self.min, self.max)
+        self.max = max(self.min, self.max)
+
+        # All connections, hashed on thread id
+        self.connections = {}
+
+        # These are optional so import them here
+        from twisted.python import threadable, threadpool
+
+        self.threadID = threadable.getThreadID
+        self.threadpool = threadpool.ThreadPool(self.min, self.max)
+        self.startID = self._reactor.callWhenRunning(self._start)
+
+    def _start(self):
+        self.startID = None
+        return self.start()
+
+    def start(self):
+        """
+        Start the connection pool.
+
+        If you are using the reactor normally, this function does *not*
+        need to be called.
+        """
+        if not self.running:
+            self.threadpool.start()
+            self.shutdownID = self._reactor.addSystemEventTrigger(
+                "during", "shutdown", self.finalClose
+            )
+            self.running = True
+
+    def runWithConnection(self, func, *args, **kw):
+        """
+        Execute a function with a database connection and return the result.
+
+        @param func: A callable object of one argument which will be executed
+            in a thread with a connection from the pool. It will be passed as
+            its first argument a L{Connection} instance (whose interface is
+            mostly identical to that of a connection object for your DB-API
+            module of choice), and its results will be returned as a
+            L{Deferred}. If the method raises an exception the transaction will
+            be rolled back. Otherwise, the transaction will be committed.
+            B{Note} that this function is B{not} run in the main thread: it
+            must be threadsafe.
+
+        @param args: positional arguments to be passed to func
+
+        @param kw: keyword arguments to be passed to func
+
+        @return: a L{Deferred} which will fire the return value of
+            C{func(Transaction(...), *args, **kw)}, or a
+            L{twisted.python.failure.Failure}.
+        """
+        return threads.deferToThreadPool(
+            self._reactor, self.threadpool, self._runWithConnection, func, *args, **kw
+        )
+
+    def _runWithConnection(self, func, *args, **kw):
+        conn = self.connectionFactory(self)
+        try:
+            result = func(conn, *args, **kw)
+            conn.commit()
+            return result
+        except BaseException:
+            try:
+                conn.rollback()
+            except BaseException:
+                log.err(None, "Rollback failed")
+            raise
+
+    def runInteraction(self, interaction, *args, **kw):
+        """
+        Interact with the database and return the result.
+
+        The 'interaction' is a callable object which will be executed in a
+        thread using a pooled connection. It will be passed an L{Transaction}
+        object as an argument (whose interface is identical to that of the
+        database cursor for your DB-API module of choice), and its results will
+        be returned as a L{Deferred}. If running the method raises an
+        exception, the transaction will be rolled back. If the method returns a
+        value, the transaction will be committed.
+
+        NOTE that the function you pass is *not* run in the main thread: you
+        may have to worry about thread-safety in the function you pass to this
+        if it tries to use non-local objects.
+
+        @param interaction: a callable object whose first argument is an
+            L{adbapi.Transaction}.
+
+        @param args: additional positional arguments to be passed to
+            interaction
+
+        @param kw: keyword arguments to be passed to interaction
+
+        @return: a Deferred which will fire the return value of
+            C{interaction(Transaction(...), *args, **kw)}, or a
+            L{twisted.python.failure.Failure}.
+        """
+        return threads.deferToThreadPool(
+            self._reactor,
+            self.threadpool,
+            self._runInteraction,
+            interaction,
+            *args,
+            **kw,
+        )
+
+    def runQuery(self, *args, **kw):
+        """
+        Execute an SQL query and return the result.
+
+        A DB-API cursor which will be invoked with C{cursor.execute(*args,
+        **kw)}. The exact nature of the arguments will depend on the specific
+        flavor of DB-API being used, but the first argument in C{*args} be an
+        SQL statement. The result of a subsequent C{cursor.fetchall()} will be
+        fired to the L{Deferred} which is returned. If either the 'execute' or
+        'fetchall' methods raise an exception, the transaction will be rolled
+        back and a L{twisted.python.failure.Failure} returned.
+
+        The C{*args} and C{**kw} arguments will be passed to the DB-API
+        cursor's 'execute' method.
+
+        @return: a L{Deferred} which will fire the return value of a DB-API
+            cursor's 'fetchall' method, or a L{twisted.python.failure.Failure}.
+        """
+        return self.runInteraction(self._runQuery, *args, **kw)
+
+    def runOperation(self, *args, **kw):
+        """
+        Execute an SQL query and return L{None}.
+
+        A DB-API cursor which will be invoked with C{cursor.execute(*args,
+        **kw)}. The exact nature of the arguments will depend on the specific
+        flavor of DB-API being used, but the first argument in C{*args} will be
+        an SQL statement. This method will not attempt to fetch any results
+        from the query and is thus suitable for C{INSERT}, C{DELETE}, and other
+        SQL statements which do not return values. If the 'execute' method
+        raises an exception, the transaction will be rolled back and a
+        L{Failure} returned.
+
+        The C{*args} and C{*kw} arguments will be passed to the DB-API cursor's
+        'execute' method.
+
+        @return: a L{Deferred} which will fire with L{None} or a
+            L{twisted.python.failure.Failure}.
+        """
+        return self.runInteraction(self._runOperation, *args, **kw)
+
+    def close(self):
+        """
+        Close all pool connections and shutdown the pool.
+        """
+        if self.shutdownID:
+            self._reactor.removeSystemEventTrigger(self.shutdownID)
+            self.shutdownID = None
+        if self.startID:
+            self._reactor.removeSystemEventTrigger(self.startID)
+            self.startID = None
+        self.finalClose()
+
+    def finalClose(self):
+        """
+        This should only be called by the shutdown trigger.
+        """
+        self.shutdownID = None
+        self.threadpool.stop()
+        self.running = False
+        for conn in self.connections.values():
+            self._close(conn)
+        self.connections.clear()
+
+    def connect(self):
+        """
+        Return a database connection when one becomes available.
+
+        This method blocks and should be run in a thread from the internal
+        threadpool. Don't call this method directly from non-threaded code.
+        Using this method outside the external threadpool may exceed the
+        maximum number of connections in the pool.
+
+        @return: a database connection from the pool.
+        """
+
+        tid = self.threadID()
+        conn = self.connections.get(tid)
+        if conn is None:
+            if self.noisy:
+                log.msg(f"adbapi connecting: {self.dbapiName}")
+            conn = self.dbapi.connect(*self.connargs, **self.connkw)
+            if self.openfun is not None:
+                self.openfun(conn)
+            self.connections[tid] = conn
+        return conn
+
+    def disconnect(self, conn):
+        """
+        Disconnect a database connection associated with this pool.
+
+        Note: This function should only be used by the same thread which called
+        L{ConnectionPool.connect}. As with C{connect}, this function is not
+        used in normal non-threaded Twisted code.
+        """
+        tid = self.threadID()
+        if conn is not self.connections.get(tid):
+            raise Exception("wrong connection for thread")
+        if conn is not None:
+            self._close(conn)
+            del self.connections[tid]
+
+    def _close(self, conn):
+        if self.noisy:
+            log.msg(f"adbapi closing: {self.dbapiName}")
+        try:
+            conn.close()
+        except BaseException:
+            log.err(None, "Connection close failed")
+
+    def _runInteraction(self, interaction, *args, **kw):
+        conn = self.connectionFactory(self)
+        trans = self.transactionFactory(self, conn)
+        try:
+            result = interaction(trans, *args, **kw)
+            trans.close()
+            conn.commit()
+            return result
+        except BaseException:
+            try:
+                conn.rollback()
+            except BaseException:
+                log.err(None, "Rollback failed")
+            raise
+
+    def _runQuery(self, trans, *args, **kw):
+        trans.execute(*args, **kw)
+        return trans.fetchall()
+
+    def _runOperation(self, trans, *args, **kw):
+        trans.execute(*args, **kw)
+
+    def __getstate__(self):
+        return {
+            "dbapiName": self.dbapiName,
+            "min": self.min,
+            "max": self.max,
+            "noisy": self.noisy,
+            "reconnect": self.reconnect,
+            "good_sql": self.good_sql,
+            "connargs": self.connargs,
+            "connkw": self.connkw,
+        }
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+        self.__init__(self.dbapiName, *self.connargs, **self.connkw)
+
+
+__all__ = ["Transaction", "ConnectionPool"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/__init__.py b/contrib/python/Twisted/py3/twisted/internet/__init__.py
new file mode 100644
index 00000000000..a3d851d1983
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/__init__.py
@@ -0,0 +1,12 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Internet: Asynchronous I/O and Events.
+
+Twisted Internet is a collection of compatible event-loops for Python. It contains
+the code to dispatch events to interested observers and a portable API so that
+observers need not care about which event loop is running. Thus, it is possible
+to use the same code for different loops, from Twisted's basic, yet portable,
+select-based loop to the loops of various GUI toolkits like GTK+ or Tk.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/internet/_baseprocess.py b/contrib/python/Twisted/py3/twisted/internet/_baseprocess.py
new file mode 100644
index 00000000000..83bc08fdc0b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_baseprocess.py
@@ -0,0 +1,68 @@
+# -*- test-case-name: twisted.test.test_process -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Cross-platform process-related functionality used by different
+L{IReactorProcess} implementations.
+"""
+
+from typing import Optional
+
+from twisted.python.deprecate import getWarningMethod
+from twisted.python.failure import Failure
+from twisted.python.log import err
+from twisted.python.reflect import qual
+
+_missingProcessExited = (
+    "Since Twisted 8.2, IProcessProtocol.processExited "
+    "is required.  %s must implement it."
+)
+
+
+class BaseProcess:
+    pid: Optional[int] = None
+    status: Optional[int] = None
+    lostProcess = 0
+    proto = None
+
+    def __init__(self, protocol):
+        self.proto = protocol
+
+    def _callProcessExited(self, reason):
+        default = object()
+        processExited = getattr(self.proto, "processExited", default)
+        if processExited is default:
+            getWarningMethod()(
+                _missingProcessExited % (qual(self.proto.__class__),),
+                DeprecationWarning,
+                stacklevel=0,
+            )
+        else:
+            try:
+                processExited(Failure(reason))
+            except BaseException:
+                err(None, "unexpected error in processExited")
+
+    def processEnded(self, status):
+        """
+        This is called when the child terminates.
+        """
+        self.status = status
+        self.lostProcess += 1
+        self.pid = None
+        self._callProcessExited(self._getReason(status))
+        self.maybeCallProcessEnded()
+
+    def maybeCallProcessEnded(self):
+        """
+        Call processEnded on protocol after final cleanup.
+        """
+        if self.proto is not None:
+            reason = self._getReason(self.status)
+            proto = self.proto
+            self.proto = None
+            try:
+                proto.processEnded(Failure(reason))
+            except BaseException:
+                err(None, "unexpected error in processEnded")
diff --git a/contrib/python/Twisted/py3/twisted/internet/_deprecate.py b/contrib/python/Twisted/py3/twisted/internet/_deprecate.py
new file mode 100644
index 00000000000..6eb1e3855f9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_deprecate.py
@@ -0,0 +1,25 @@
+"""
+Support similar deprecation of several reactors.
+"""
+
+import warnings
+
+from incremental import Version, getVersionString
+
+from twisted.python.deprecate import DEPRECATION_WARNING_FORMAT
+
+
+def deprecatedGnomeReactor(name: str, version: Version) -> None:
+    """
+    Emit a deprecation warning about a gnome-related reactor.
+
+    @param name: The name of the reactor.  For example, C{"gtk2reactor"}.
+
+    @param version: The version in which the deprecation was introduced.
+    """
+    stem = DEPRECATION_WARNING_FORMAT % {
+        "fqpn": "twisted.internet." + name,
+        "version": getVersionString(version),
+    }
+    msg = stem + ".  Please use twisted.internet.gireactor instead."
+    warnings.warn(msg, category=DeprecationWarning)
diff --git a/contrib/python/Twisted/py3/twisted/internet/_dumbwin32proc.py b/contrib/python/Twisted/py3/twisted/internet/_dumbwin32proc.py
new file mode 100644
index 00000000000..678f54e59b2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_dumbwin32proc.py
@@ -0,0 +1,397 @@
+# -*- test-case-name: twisted.test.test_process -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Windows Process Management, used with reactor.spawnProcess
+"""
+
+
+import os
+import sys
+
+from zope.interface import implementer
+
+import pywintypes  # type: ignore[import]
+
+# Win32 imports
+import win32api  # type: ignore[import]
+import win32con  # type: ignore[import]
+import win32event  # type: ignore[import]
+import win32file  # type: ignore[import]
+import win32pipe  # type: ignore[import]
+import win32process  # type: ignore[import]
+import win32security  # type: ignore[import]
+
+from twisted.internet import _pollingfile, error
+from twisted.internet._baseprocess import BaseProcess
+from twisted.internet.interfaces import IConsumer, IProcessTransport, IProducer
+from twisted.python.win32 import quoteArguments
+
+# Security attributes for pipes
+PIPE_ATTRS_INHERITABLE = win32security.SECURITY_ATTRIBUTES()
+PIPE_ATTRS_INHERITABLE.bInheritHandle = 1
+
+
+def debug(msg):
+    print(msg)
+    sys.stdout.flush()
+
+
+class _Reaper(_pollingfile._PollableResource):
+    def __init__(self, proc):
+        self.proc = proc
+
+    def checkWork(self):
+        if (
+            win32event.WaitForSingleObject(self.proc.hProcess, 0)
+            != win32event.WAIT_OBJECT_0
+        ):
+            return 0
+        exitCode = win32process.GetExitCodeProcess(self.proc.hProcess)
+        self.deactivate()
+        self.proc.processEnded(exitCode)
+        return 0
+
+
+def _findShebang(filename):
+    """
+    Look for a #! line, and return the value following the #! if one exists, or
+    None if this file is not a script.
+
+    I don't know if there are any conventions for quoting in Windows shebang
+    lines, so this doesn't support any; therefore, you may not pass any
+    arguments to scripts invoked as filters.  That's probably wrong, so if
+    somebody knows more about the cultural expectations on Windows, please feel
+    free to fix.
+
+    This shebang line support was added in support of the CGI tests;
+    appropriately enough, I determined that shebang lines are culturally
+    accepted in the Windows world through this page::
+
+        http://www.cgi101.com/learn/connect/winxp.html
+
+    @param filename: str representing a filename
+
+    @return: a str representing another filename.
+    """
+    with open(filename) as f:
+        if f.read(2) == "#!":
+            exe = f.readline(1024).strip("\n")
+            return exe
+
+
+def _invalidWin32App(pywinerr):
+    """
+    Determine if a pywintypes.error is telling us that the given process is
+    'not a valid win32 application', i.e. not a PE format executable.
+
+    @param pywinerr: a pywintypes.error instance raised by CreateProcess
+
+    @return: a boolean
+    """
+
+    # Let's do this better in the future, but I have no idea what this error
+    # is; MSDN doesn't mention it, and there is no symbolic constant in
+    # win32process module that represents 193.
+
+    return pywinerr.args[0] == 193
+
+
+@implementer(IProcessTransport, IConsumer, IProducer)
+class Process(_pollingfile._PollingTimer, BaseProcess):
+    """
+    A process that integrates with the Twisted event loop.
+
+    If your subprocess is a python program, you need to:
+
+     - Run python.exe with the '-u' command line option - this turns on
+       unbuffered I/O. Buffering stdout/err/in can cause problems, see e.g.
+       http://support.microsoft.com/default.aspx?scid=kb;EN-US;q1903
+
+     - If you don't want Windows messing with data passed over
+       stdin/out/err, set the pipes to be in binary mode::
+
+        import os, sys, mscvrt
+        msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
+        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+        msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
+
+    """
+
+    closedNotifies = 0
+
+    def __init__(self, reactor, protocol, command, args, environment, path):
+        """
+        Create a new child process.
+        """
+        _pollingfile._PollingTimer.__init__(self, reactor)
+        BaseProcess.__init__(self, protocol)
+
+        # security attributes for pipes
+        sAttrs = win32security.SECURITY_ATTRIBUTES()
+        sAttrs.bInheritHandle = 1
+
+        # create the pipes which will connect to the secondary process
+        self.hStdoutR, hStdoutW = win32pipe.CreatePipe(sAttrs, 0)
+        self.hStderrR, hStderrW = win32pipe.CreatePipe(sAttrs, 0)
+        hStdinR, self.hStdinW = win32pipe.CreatePipe(sAttrs, 0)
+
+        win32pipe.SetNamedPipeHandleState(
+            self.hStdinW, win32pipe.PIPE_NOWAIT, None, None
+        )
+
+        # set the info structure for the new process.
+        StartupInfo = win32process.STARTUPINFO()
+        StartupInfo.hStdOutput = hStdoutW
+        StartupInfo.hStdError = hStderrW
+        StartupInfo.hStdInput = hStdinR
+        StartupInfo.dwFlags = win32process.STARTF_USESTDHANDLES
+
+        # Create new handles whose inheritance property is false
+        currentPid = win32api.GetCurrentProcess()
+
+        tmp = win32api.DuplicateHandle(
+            currentPid, self.hStdoutR, currentPid, 0, 0, win32con.DUPLICATE_SAME_ACCESS
+        )
+        win32file.CloseHandle(self.hStdoutR)
+        self.hStdoutR = tmp
+
+        tmp = win32api.DuplicateHandle(
+            currentPid, self.hStderrR, currentPid, 0, 0, win32con.DUPLICATE_SAME_ACCESS
+        )
+        win32file.CloseHandle(self.hStderrR)
+        self.hStderrR = tmp
+
+        tmp = win32api.DuplicateHandle(
+            currentPid, self.hStdinW, currentPid, 0, 0, win32con.DUPLICATE_SAME_ACCESS
+        )
+        win32file.CloseHandle(self.hStdinW)
+        self.hStdinW = tmp
+
+        # Add the specified environment to the current environment - this is
+        # necessary because certain operations are only supported on Windows
+        # if certain environment variables are present.
+
+        env = os.environ.copy()
+        env.update(environment or {})
+        env = {os.fsdecode(key): os.fsdecode(value) for key, value in env.items()}
+
+        # Make sure all the arguments are Unicode.
+        args = [os.fsdecode(x) for x in args]
+
+        cmdline = quoteArguments(args)
+
+        # The command, too, needs to be Unicode, if it is a value.
+        command = os.fsdecode(command) if command else command
+        path = os.fsdecode(path) if path else path
+
+        # TODO: error detection here.  See #2787 and #4184.
+        def doCreate():
+            flags = win32con.CREATE_NO_WINDOW
+            self.hProcess, self.hThread, self.pid, dwTid = win32process.CreateProcess(
+                command, cmdline, None, None, 1, flags, env, path, StartupInfo
+            )
+
+        try:
+            doCreate()
+        except pywintypes.error as pwte:
+            if not _invalidWin32App(pwte):
+                # This behavior isn't _really_ documented, but let's make it
+                # consistent with the behavior that is documented.
+                raise OSError(pwte)
+            else:
+                # look for a shebang line.  Insert the original 'command'
+                # (actually a script) into the new arguments list.
+                sheb = _findShebang(command)
+                if sheb is None:
+                    raise OSError(
+                        "%r is neither a Windows executable, "
+                        "nor a script with a shebang line" % command
+                    )
+                else:
+                    args = list(args)
+                    args.insert(0, command)
+                    cmdline = quoteArguments(args)
+                    origcmd = command
+                    command = sheb
+                    try:
+                        # Let's try again.
+                        doCreate()
+                    except pywintypes.error as pwte2:
+                        # d'oh, failed again!
+                        if _invalidWin32App(pwte2):
+                            raise OSError(
+                                "%r has an invalid shebang line: "
+                                "%r is not a valid executable" % (origcmd, sheb)
+                            )
+                        raise OSError(pwte2)
+
+        # close handles which only the child will use
+        win32file.CloseHandle(hStderrW)
+        win32file.CloseHandle(hStdoutW)
+        win32file.CloseHandle(hStdinR)
+
+        # set up everything
+        self.stdout = _pollingfile._PollableReadPipe(
+            self.hStdoutR,
+            lambda data: self.proto.childDataReceived(1, data),
+            self.outConnectionLost,
+        )
+
+        self.stderr = _pollingfile._PollableReadPipe(
+            self.hStderrR,
+            lambda data: self.proto.childDataReceived(2, data),
+            self.errConnectionLost,
+        )
+
+        self.stdin = _pollingfile._PollableWritePipe(
+            self.hStdinW, self.inConnectionLost
+        )
+
+        for pipewatcher in self.stdout, self.stderr, self.stdin:
+            self._addPollableResource(pipewatcher)
+
+        # notify protocol
+        self.proto.makeConnection(self)
+
+        self._addPollableResource(_Reaper(self))
+
+    def signalProcess(self, signalID):
+        if self.pid is None:
+            raise error.ProcessExitedAlready()
+        if signalID in ("INT", "TERM", "KILL"):
+            win32process.TerminateProcess(self.hProcess, 1)
+
+    def _getReason(self, status):
+        if status == 0:
+            return error.ProcessDone(status)
+        return error.ProcessTerminated(status)
+
+    def write(self, data):
+        """
+        Write data to the process' stdin.
+
+        @type data: C{bytes}
+        """
+        self.stdin.write(data)
+
+    def writeSequence(self, seq):
+        """
+        Write data to the process' stdin.
+
+        @type seq: C{list} of C{bytes}
+        """
+        self.stdin.writeSequence(seq)
+
+    def writeToChild(self, fd, data):
+        """
+        Similar to L{ITransport.write} but also allows the file descriptor in
+        the child process which will receive the bytes to be specified.
+
+        This implementation is limited to writing to the child's standard input.
+
+        @param fd: The file descriptor to which to write.  Only stdin (C{0}) is
+            supported.
+        @type fd: C{int}
+
+        @param data: The bytes to write.
+        @type data: C{bytes}
+
+        @return: L{None}
+
+        @raise KeyError: If C{fd} is anything other than the stdin file
+            descriptor (C{0}).
+        """
+        if fd == 0:
+            self.stdin.write(data)
+        else:
+            raise KeyError(fd)
+
+    def closeChildFD(self, fd):
+        if fd == 0:
+            self.closeStdin()
+        elif fd == 1:
+            self.closeStdout()
+        elif fd == 2:
+            self.closeStderr()
+        else:
+            raise NotImplementedError(
+                "Only standard-IO file descriptors available on win32"
+            )
+
+    def closeStdin(self):
+        """Close the process' stdin."""
+        self.stdin.close()
+
+    def closeStderr(self):
+        self.stderr.close()
+
+    def closeStdout(self):
+        self.stdout.close()
+
+    def loseConnection(self):
+        """
+        Close the process' stdout, in and err.
+        """
+        self.closeStdin()
+        self.closeStdout()
+        self.closeStderr()
+
+    def outConnectionLost(self):
+        self.proto.childConnectionLost(1)
+        self.connectionLostNotify()
+
+    def errConnectionLost(self):
+        self.proto.childConnectionLost(2)
+        self.connectionLostNotify()
+
+    def inConnectionLost(self):
+        self.proto.childConnectionLost(0)
+        self.connectionLostNotify()
+
+    def connectionLostNotify(self):
+        """
+        Will be called 3 times, by stdout/err threads and process handle.
+        """
+        self.closedNotifies += 1
+        self.maybeCallProcessEnded()
+
+    def maybeCallProcessEnded(self):
+        if self.closedNotifies == 3 and self.lostProcess:
+            win32file.CloseHandle(self.hProcess)
+            win32file.CloseHandle(self.hThread)
+            self.hProcess = None
+            self.hThread = None
+            BaseProcess.maybeCallProcessEnded(self)
+
+    # IConsumer
+    def registerProducer(self, producer, streaming):
+        self.stdin.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        self.stdin.unregisterProducer()
+
+    # IProducer
+    def pauseProducing(self):
+        self._pause()
+
+    def resumeProducing(self):
+        self._unpause()
+
+    def stopProducing(self):
+        self.loseConnection()
+
+    def getHost(self):
+        # ITransport.getHost
+        raise NotImplementedError("Unimplemented: Process.getHost")
+
+    def getPeer(self):
+        # ITransport.getPeer
+        raise NotImplementedError("Unimplemented: Process.getPeer")
+
+    def __repr__(self) -> str:
+        """
+        Return a string representation of the process.
+        """
+        return f"<{self.__class__.__name__} pid={self.pid}>"
diff --git a/contrib/python/Twisted/py3/twisted/internet/_glibbase.py b/contrib/python/Twisted/py3/twisted/internet/_glibbase.py
new file mode 100644
index 00000000000..4a6d1323ab9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_glibbase.py
@@ -0,0 +1,369 @@
+# -*- test-case-name: twisted.internet.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module provides base support for Twisted to interact with the glib/gtk
+mainloops.
+
+The classes in this module should not be used directly, but rather you should
+import gireactor or gtk3reactor for GObject Introspection based applications,
+or glib2reactor or gtk2reactor for applications using legacy static bindings.
+"""
+
+
+import sys
+from typing import Any, Callable, Dict, Set
+
+from zope.interface import implementer
+
+from twisted.internet import posixbase
+from twisted.internet.abstract import FileDescriptor
+from twisted.internet.interfaces import IReactorFDSet, IReadDescriptor, IWriteDescriptor
+from twisted.python import log
+from twisted.python.monkey import MonkeyPatcher
+from ._signals import _UnixWaker
+
+
+def ensureNotImported(moduleNames, errorMessage, preventImports=[]):
+    """
+    Check whether the given modules were imported, and if requested, ensure
+    they will not be importable in the future.
+
+    @param moduleNames: A list of module names we make sure aren't imported.
+    @type moduleNames: C{list} of C{str}
+
+    @param preventImports: A list of module name whose future imports should
+        be prevented.
+    @type preventImports: C{list} of C{str}
+
+    @param errorMessage: Message to use when raising an C{ImportError}.
+    @type errorMessage: C{str}
+
+    @raise ImportError: with given error message if a given module name
+        has already been imported.
+    """
+    for name in moduleNames:
+        if sys.modules.get(name) is not None:
+            raise ImportError(errorMessage)
+
+    # Disable module imports to avoid potential problems.
+    for name in preventImports:
+        sys.modules[name] = None
+
+
+class GlibWaker(_UnixWaker):
+    """
+    Run scheduled events after waking up.
+    """
+
+    def __init__(self, reactor):
+        super().__init__()
+        self.reactor = reactor
+
+    def doRead(self) -> None:
+        super().doRead()
+        self.reactor._simulate()
+
+
+def _signalGlue():
+    """
+    Integrate glib's wakeup file descriptor usage and our own.
+
+    Python supports only one wakeup file descriptor at a time and both Twisted
+    and glib want to use it.
+
+    This is a context manager that can be wrapped around the whole glib
+    reactor main loop which makes our signal handling work with glib's signal
+    handling.
+    """
+    from gi import _ossighelper as signalGlue  # type: ignore[import]
+
+    patcher = MonkeyPatcher()
+    patcher.addPatch(signalGlue, "_wakeup_fd_is_active", True)
+    return patcher
+
+
+def _loopQuitter(
+    idleAdd: Callable[[Callable[[], None]], None], loopQuit: Callable[[], None]
+) -> Callable[[], None]:
+    """
+    Combine the C{glib.idle_add} and C{glib.MainLoop.quit} functions into a
+    function suitable for crashing the reactor.
+    """
+    return lambda: idleAdd(loopQuit)
+
+
+@implementer(IReactorFDSet)
+class GlibReactorBase(posixbase.PosixReactorBase, posixbase._PollLikeMixin):
+    """
+    Base class for GObject event loop reactors.
+
+    Notification for I/O events (reads and writes on file descriptors) is done
+    by the gobject-based event loop. File descriptors are registered with
+    gobject with the appropriate flags for read/write/disconnect notification.
+
+    Time-based events, the results of C{callLater} and C{callFromThread}, are
+    handled differently. Rather than registering each event with gobject, a
+    single gobject timeout is registered for the earliest scheduled event, the
+    output of C{reactor.timeout()}. For example, if there are timeouts in 1, 2
+    and 3.4 seconds, a single timeout is registered for 1 second in the
+    future. When this timeout is hit, C{_simulate} is called, which calls the
+    appropriate Twisted-level handlers, and a new timeout is added to gobject
+    by the C{_reschedule} method.
+
+    To handle C{callFromThread} events, we use a custom waker that calls
+    C{_simulate} whenever it wakes up.
+
+    @ivar _sources: A dictionary mapping L{FileDescriptor} instances to
+        GSource handles.
+
+    @ivar _reads: A set of L{FileDescriptor} instances currently monitored for
+        reading.
+
+    @ivar _writes: A set of L{FileDescriptor} instances currently monitored for
+        writing.
+
+    @ivar _simtag: A GSource handle for the next L{simulate} call.
+    """
+
+    # Install a waker that knows it needs to call C{_simulate} in order to run
+    # callbacks queued from a thread:
+    def _wakerFactory(self) -> GlibWaker:
+        return GlibWaker(self)
+
+    def __init__(self, glib_module: Any, gtk_module: Any, useGtk: bool = False) -> None:
+        self._simtag = None
+        self._reads: Set[IReadDescriptor] = set()
+        self._writes: Set[IWriteDescriptor] = set()
+        self._sources: Dict[FileDescriptor, int] = {}
+        self._glib = glib_module
+
+        self._POLL_DISCONNECTED = (
+            glib_module.IOCondition.HUP
+            | glib_module.IOCondition.ERR
+            | glib_module.IOCondition.NVAL
+        )
+        self._POLL_IN = glib_module.IOCondition.IN
+        self._POLL_OUT = glib_module.IOCondition.OUT
+
+        # glib's iochannel sources won't tell us about any events that we haven't
+        # asked for, even if those events aren't sensible inputs to the poll()
+        # call.
+        self.INFLAGS = self._POLL_IN | self._POLL_DISCONNECTED
+        self.OUTFLAGS = self._POLL_OUT | self._POLL_DISCONNECTED
+
+        super().__init__()
+
+        self._source_remove = self._glib.source_remove
+        self._timeout_add = self._glib.timeout_add
+
+        self.context = self._glib.main_context_default()
+        self._pending = self.context.pending
+        self._iteration = self.context.iteration
+        self.loop = self._glib.MainLoop()
+        self._crash = _loopQuitter(self._glib.idle_add, self.loop.quit)
+        self._run = self.loop.run
+
+    def _reallyStartRunning(self):
+        """
+        Make sure the reactor's signal handlers are installed despite any
+        outside interference.
+        """
+        # First, install SIGINT and friends:
+        super()._reallyStartRunning()
+
+        # Next, since certain versions of gtk will clobber our signal handler,
+        # set all signal handlers again after the event loop has started to
+        # ensure they're *really* set.
+        #
+        # We don't actually know which versions of gtk do this so this might
+        # be obsolete.  If so, that would be great and this whole method can
+        # go away.  Someone needs to find out, though.
+        #
+        # https://github.com/twisted/twisted/issues/11762
+
+        def reinitSignals():
+            self._signals.uninstall()
+            self._signals.install()
+
+        self.callLater(0, reinitSignals)
+
+    # The input_add function in pygtk1 checks for objects with a
+    # 'fileno' method and, if present, uses the result of that method
+    # as the input source. The pygtk2 input_add does not do this. The
+    # function below replicates the pygtk1 functionality.
+
+    # In addition, pygtk maps gtk.input_add to _gobject.io_add_watch, and
+    # g_io_add_watch() takes different condition bitfields than
+    # gtk_input_add(). We use g_io_add_watch() here in case pygtk fixes this
+    # bug.
+    def input_add(self, source, condition, callback):
+        if hasattr(source, "fileno"):
+            # handle python objects
+            def wrapper(ignored, condition):
+                return callback(source, condition)
+
+            fileno = source.fileno()
+        else:
+            fileno = source
+            wrapper = callback
+        return self._glib.io_add_watch(
+            fileno,
+            self._glib.PRIORITY_DEFAULT_IDLE,
+            condition,
+            wrapper,
+        )
+
+    def _ioEventCallback(self, source, condition):
+        """
+        Called by event loop when an I/O event occurs.
+        """
+        log.callWithLogger(source, self._doReadOrWrite, source, source, condition)
+        return True  # True = don't auto-remove the source
+
+    def _add(self, source, primary, other, primaryFlag, otherFlag):
+        """
+        Add the given L{FileDescriptor} for monitoring either for reading or
+        writing. If the file is already monitored for the other operation, we
+        delete the previous registration and re-register it for both reading
+        and writing.
+        """
+        if source in primary:
+            return
+        flags = primaryFlag
+        if source in other:
+            self._source_remove(self._sources[source])
+            flags |= otherFlag
+        self._sources[source] = self.input_add(source, flags, self._ioEventCallback)
+        primary.add(source)
+
+    def addReader(self, reader):
+        """
+        Add a L{FileDescriptor} for monitoring of data available to read.
+        """
+        self._add(reader, self._reads, self._writes, self.INFLAGS, self.OUTFLAGS)
+
+    def addWriter(self, writer):
+        """
+        Add a L{FileDescriptor} for monitoring ability to write data.
+        """
+        self._add(writer, self._writes, self._reads, self.OUTFLAGS, self.INFLAGS)
+
+    def getReaders(self):
+        """
+        Retrieve the list of current L{FileDescriptor} monitored for reading.
+        """
+        return list(self._reads)
+
+    def getWriters(self):
+        """
+        Retrieve the list of current L{FileDescriptor} monitored for writing.
+        """
+        return list(self._writes)
+
+    def removeAll(self):
+        """
+        Remove monitoring for all registered L{FileDescriptor}s.
+        """
+        return self._removeAll(self._reads, self._writes)
+
+    def _remove(self, source, primary, other, flags):
+        """
+        Remove monitoring the given L{FileDescriptor} for either reading or
+        writing. If it's still monitored for the other operation, we
+        re-register the L{FileDescriptor} for only that operation.
+        """
+        if source not in primary:
+            return
+        self._source_remove(self._sources[source])
+        primary.remove(source)
+        if source in other:
+            self._sources[source] = self.input_add(source, flags, self._ioEventCallback)
+        else:
+            self._sources.pop(source)
+
+    def removeReader(self, reader):
+        """
+        Stop monitoring the given L{FileDescriptor} for reading.
+        """
+        self._remove(reader, self._reads, self._writes, self.OUTFLAGS)
+
+    def removeWriter(self, writer):
+        """
+        Stop monitoring the given L{FileDescriptor} for writing.
+        """
+        self._remove(writer, self._writes, self._reads, self.INFLAGS)
+
+    def iterate(self, delay=0):
+        """
+        One iteration of the event loop, for trial's use.
+
+        This is not used for actual reactor runs.
+        """
+        self.runUntilCurrent()
+        while self._pending():
+            self._iteration(0)
+
+    def crash(self):
+        """
+        Crash the reactor.
+        """
+        posixbase.PosixReactorBase.crash(self)
+        self._crash()
+
+    def stop(self):
+        """
+        Stop the reactor.
+        """
+        posixbase.PosixReactorBase.stop(self)
+        # The base implementation only sets a flag, to ensure shutting down is
+        # not reentrant. Unfortunately, this flag is not meaningful to the
+        # gobject event loop. We therefore call wakeUp() to ensure the event
+        # loop will call back into Twisted once this iteration is done. This
+        # will result in self.runUntilCurrent() being called, where the stop
+        # flag will trigger the actual shutdown process, eventually calling
+        # crash() which will do the actual gobject event loop shutdown.
+        self.wakeUp()
+
+    def run(self, installSignalHandlers=True):
+        """
+        Run the reactor.
+        """
+        with _signalGlue():
+            self.callWhenRunning(self._reschedule)
+            self.startRunning(installSignalHandlers=installSignalHandlers)
+            if self._started:
+                self._run()
+
+    def callLater(self, *args, **kwargs):
+        """
+        Schedule a C{DelayedCall}.
+        """
+        result = posixbase.PosixReactorBase.callLater(self, *args, **kwargs)
+        # Make sure we'll get woken up at correct time to handle this new
+        # scheduled call:
+        self._reschedule()
+        return result
+
+    def _reschedule(self):
+        """
+        Schedule a glib timeout for C{_simulate}.
+        """
+        if self._simtag is not None:
+            self._source_remove(self._simtag)
+            self._simtag = None
+        timeout = self.timeout()
+        if timeout is not None:
+            self._simtag = self._timeout_add(
+                int(timeout * 1000),
+                self._simulate,
+                priority=self._glib.PRIORITY_DEFAULT_IDLE,
+            )
+
+    def _simulate(self):
+        """
+        Run timers, and then reschedule glib timeout for next scheduled event.
+        """
+        self.runUntilCurrent()
+        self._reschedule()
diff --git a/contrib/python/Twisted/py3/twisted/internet/_idna.py b/contrib/python/Twisted/py3/twisted/internet/_idna.py
new file mode 100644
index 00000000000..852d8a6be8f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_idna.py
@@ -0,0 +1,51 @@
+# -*- test-case-name: twisted.test.test_sslverify -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Shared interface to IDNA encoding and decoding, using the C{idna} PyPI package
+if available, otherwise the stdlib implementation.
+"""
+
+
+def _idnaBytes(text: str) -> bytes:
+    """
+    Convert some text typed by a human into some ASCII bytes.
+
+    This is provided to allow us to use the U{partially-broken IDNA
+    implementation in the standard library <http://bugs.python.org/issue17305>}
+    if the more-correct U{idna <https://pypi.python.org/pypi/idna>} package is
+    not available; C{service_identity} is somewhat stricter about this.
+
+    @param text: A domain name, hopefully.
+    @type text: L{unicode}
+
+    @return: The domain name's IDNA representation, encoded as bytes.
+    @rtype: L{bytes}
+    """
+    try:
+        import idna
+    except ImportError:
+        return text.encode("idna")
+    else:
+        return idna.encode(text)
+
+
+def _idnaText(octets: bytes) -> str:
+    """
+    Convert some IDNA-encoded octets into some human-readable text.
+
+    Currently only used by the tests.
+
+    @param octets: Some bytes representing a hostname.
+    @type octets: L{bytes}
+
+    @return: A human-readable domain name.
+    @rtype: L{unicode}
+    """
+    try:
+        import idna
+    except ImportError:
+        return octets.decode("idna")
+    else:
+        return idna.decode(octets)
diff --git a/contrib/python/Twisted/py3/twisted/internet/_newtls.py b/contrib/python/Twisted/py3/twisted/internet/_newtls.py
new file mode 100644
index 00000000000..5c27f31eb9d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_newtls.py
@@ -0,0 +1,256 @@
+# -*- test-case-name: twisted.test.test_ssl -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module implements memory BIO based TLS support.  It is the preferred
+implementation and will be used whenever pyOpenSSL 0.10 or newer is installed
+(whenever L{twisted.protocols.tls} is importable).
+
+@since: 11.1
+"""
+
+
+from zope.interface import directlyProvides
+
+from twisted.internet.abstract import FileDescriptor
+from twisted.internet.interfaces import ISSLTransport
+from twisted.protocols.tls import TLSMemoryBIOFactory
+
+
+class _BypassTLS:
+    """
+    L{_BypassTLS} is used as the transport object for the TLS protocol object
+    used to implement C{startTLS}.  Its methods skip any TLS logic which
+    C{startTLS} enables.
+
+    @ivar _base: A transport class L{_BypassTLS} has been mixed in with to which
+        methods will be forwarded.  This class is only responsible for sending
+        bytes over the connection, not doing TLS.
+
+    @ivar _connection: A L{Connection} which TLS has been started on which will
+        be proxied to by this object.  Any method which has its behavior
+        altered after C{startTLS} will be skipped in favor of the base class's
+        implementation.  This allows the TLS protocol object to have direct
+        access to the transport, necessary to actually implement TLS.
+    """
+
+    def __init__(self, base, connection):
+        self._base = base
+        self._connection = connection
+
+    def __getattr__(self, name):
+        """
+        Forward any extra attribute access to the original transport object.
+        For example, this exposes C{getHost}, the behavior of which does not
+        change after TLS is enabled.
+        """
+        return getattr(self._connection, name)
+
+    def write(self, data):
+        """
+        Write some bytes directly to the connection.
+        """
+        return self._base.write(self._connection, data)
+
+    def writeSequence(self, iovec):
+        """
+        Write a some bytes directly to the connection.
+        """
+        return self._base.writeSequence(self._connection, iovec)
+
+    def loseConnection(self, *args, **kwargs):
+        """
+        Close the underlying connection.
+        """
+        return self._base.loseConnection(self._connection, *args, **kwargs)
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register a producer with the underlying connection.
+        """
+        return self._base.registerProducer(self._connection, producer, streaming)
+
+    def unregisterProducer(self):
+        """
+        Unregister a producer with the underlying connection.
+        """
+        return self._base.unregisterProducer(self._connection)
+
+
+def startTLS(transport, contextFactory, normal, bypass):
+    """
+    Add a layer of SSL to a transport.
+
+    @param transport: The transport which will be modified.  This can either by
+        a L{FileDescriptor<twisted.internet.abstract.FileDescriptor>} or a
+        L{FileHandle<twisted.internet.iocpreactor.abstract.FileHandle>}.  The
+        actual requirements of this instance are that it have:
+
+          - a C{_tlsClientDefault} attribute indicating whether the transport is
+            a client (C{True}) or a server (C{False})
+          - a settable C{TLS} attribute which can be used to mark the fact
+            that SSL has been started
+          - settable C{getHandle} and C{getPeerCertificate} attributes so
+            these L{ISSLTransport} methods can be added to it
+          - a C{protocol} attribute referring to the L{IProtocol} currently
+            connected to the transport, which can also be set to a new
+            L{IProtocol} for the transport to deliver data to
+
+    @param contextFactory: An SSL context factory defining SSL parameters for
+        the new SSL layer.
+    @type contextFactory: L{twisted.internet.interfaces.IOpenSSLContextFactory}
+
+    @param normal: A flag indicating whether SSL will go in the same direction
+        as the underlying transport goes.  That is, if the SSL client will be
+        the underlying client and the SSL server will be the underlying server.
+        C{True} means it is the same, C{False} means they are switched.
+    @type normal: L{bool}
+
+    @param bypass: A transport base class to call methods on to bypass the new
+        SSL layer (so that the SSL layer itself can send its bytes).
+    @type bypass: L{type}
+    """
+    # Figure out which direction the SSL goes in.  If normal is True,
+    # we'll go in the direction indicated by the subclass.  Otherwise,
+    # we'll go the other way (client = not normal ^ _tlsClientDefault,
+    # in other words).
+    if normal:
+        client = transport._tlsClientDefault
+    else:
+        client = not transport._tlsClientDefault
+
+    # If we have a producer, unregister it, and then re-register it below once
+    # we've switched to TLS mode, so it gets hooked up correctly:
+    producer, streaming = None, None
+    if transport.producer is not None:
+        producer, streaming = transport.producer, transport.streamingProducer
+        transport.unregisterProducer()
+
+    tlsFactory = TLSMemoryBIOFactory(contextFactory, client, None)
+    tlsProtocol = tlsFactory.protocol(tlsFactory, transport.protocol, False)
+    # Hook up the new TLS protocol to the transport:
+    transport.protocol = tlsProtocol
+
+    transport.getHandle = tlsProtocol.getHandle
+    transport.getPeerCertificate = tlsProtocol.getPeerCertificate
+
+    # Mark the transport as secure.
+    directlyProvides(transport, ISSLTransport)
+
+    # Remember we did this so that write and writeSequence can send the
+    # data to the right place.
+    transport.TLS = True
+
+    # Hook it up
+    transport.protocol.makeConnection(_BypassTLS(bypass, transport))
+
+    # Restore producer if necessary:
+    if producer:
+        transport.registerProducer(producer, streaming)
+
+
+class ConnectionMixin:
+    """
+    A mixin for L{twisted.internet.abstract.FileDescriptor} which adds an
+    L{ITLSTransport} implementation.
+
+    @ivar TLS: A flag indicating whether TLS is currently in use on this
+        transport.  This is not a good way for applications to check for TLS,
+        instead use L{twisted.internet.interfaces.ISSLTransport}.
+    """
+
+    TLS = False
+
+    def startTLS(self, ctx, normal=True):
+        """
+        @see: L{ITLSTransport.startTLS}
+        """
+        startTLS(self, ctx, normal, FileDescriptor)
+
+    def write(self, bytes):
+        """
+        Write some bytes to this connection, passing them through a TLS layer if
+        necessary, or discarding them if the connection has already been lost.
+        """
+        if self.TLS:
+            if self.connected:
+                self.protocol.write(bytes)
+        else:
+            FileDescriptor.write(self, bytes)
+
+    def writeSequence(self, iovec):
+        """
+        Write some bytes to this connection, scatter/gather-style, passing them
+        through a TLS layer if necessary, or discarding them if the connection
+        has already been lost.
+        """
+        if self.TLS:
+            if self.connected:
+                self.protocol.writeSequence(iovec)
+        else:
+            FileDescriptor.writeSequence(self, iovec)
+
+    def loseConnection(self):
+        """
+        Close this connection after writing all pending data.
+
+        If TLS has been negotiated, perform a TLS shutdown.
+        """
+        if self.TLS:
+            if self.connected and not self.disconnecting:
+                self.protocol.loseConnection()
+        else:
+            FileDescriptor.loseConnection(self)
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register a producer.
+
+        If TLS is enabled, the TLS connection handles this.
+        """
+        if self.TLS:
+            # Registering a producer before we're connected shouldn't be a
+            # problem. If we end up with a write(), that's already handled in
+            # the write() code above, and there are no other potential
+            # side-effects.
+            self.protocol.registerProducer(producer, streaming)
+        else:
+            FileDescriptor.registerProducer(self, producer, streaming)
+
+    def unregisterProducer(self):
+        """
+        Unregister a producer.
+
+        If TLS is enabled, the TLS connection handles this.
+        """
+        if self.TLS:
+            self.protocol.unregisterProducer()
+        else:
+            FileDescriptor.unregisterProducer(self)
+
+
+class ClientMixin:
+    """
+    A mixin for L{twisted.internet.tcp.Client} which just marks it as a client
+    for the purposes of the default TLS handshake.
+
+    @ivar _tlsClientDefault: Always C{True}, indicating that this is a client
+        connection, and by default when TLS is negotiated this class will act as
+        a TLS client.
+    """
+
+    _tlsClientDefault = True
+
+
+class ServerMixin:
+    """
+    A mixin for L{twisted.internet.tcp.Server} which just marks it as a server
+    for the purposes of the default TLS handshake.
+
+    @ivar _tlsClientDefault: Always C{False}, indicating that this is a server
+        connection, and by default when TLS is negotiated this class will act as
+        a TLS server.
+    """
+
+    _tlsClientDefault = False
diff --git a/contrib/python/Twisted/py3/twisted/internet/_pollingfile.py b/contrib/python/Twisted/py3/twisted/internet/_pollingfile.py
new file mode 100644
index 00000000000..758a4cecb72
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_pollingfile.py
@@ -0,0 +1,291 @@
+# -*- test-case-name: twisted.internet.test.test_pollingfile -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implements a simple polling interface for file descriptors that don't work with
+select() - this is pretty much only useful on Windows.
+"""
+
+
+from zope.interface import implementer
+
+from twisted.internet.interfaces import IConsumer, IPushProducer
+
+MIN_TIMEOUT = 0.000000001
+MAX_TIMEOUT = 0.1
+
+
+class _PollableResource:
+    active = True
+
+    def activate(self):
+        self.active = True
+
+    def deactivate(self):
+        self.active = False
+
+
+class _PollingTimer:
+    # Everything is private here because it is really an implementation detail.
+
+    def __init__(self, reactor):
+        self.reactor = reactor
+        self._resources = []
+        self._pollTimer = None
+        self._currentTimeout = MAX_TIMEOUT
+        self._paused = False
+
+    def _addPollableResource(self, res):
+        self._resources.append(res)
+        self._checkPollingState()
+
+    def _checkPollingState(self):
+        for resource in self._resources:
+            if resource.active:
+                self._startPolling()
+                break
+        else:
+            self._stopPolling()
+
+    def _startPolling(self):
+        if self._pollTimer is None:
+            self._pollTimer = self._reschedule()
+
+    def _stopPolling(self):
+        if self._pollTimer is not None:
+            self._pollTimer.cancel()
+            self._pollTimer = None
+
+    def _pause(self):
+        self._paused = True
+
+    def _unpause(self):
+        self._paused = False
+        self._checkPollingState()
+
+    def _reschedule(self):
+        if not self._paused:
+            return self.reactor.callLater(self._currentTimeout, self._pollEvent)
+
+    def _pollEvent(self):
+        workUnits = 0.0
+        anyActive = []
+        for resource in self._resources:
+            if resource.active:
+                workUnits += resource.checkWork()
+                # Check AFTER work has been done
+                if resource.active:
+                    anyActive.append(resource)
+
+        newTimeout = self._currentTimeout
+        if workUnits:
+            newTimeout = self._currentTimeout / (workUnits + 1.0)
+            if newTimeout < MIN_TIMEOUT:
+                newTimeout = MIN_TIMEOUT
+        else:
+            newTimeout = self._currentTimeout * 2.0
+            if newTimeout > MAX_TIMEOUT:
+                newTimeout = MAX_TIMEOUT
+        self._currentTimeout = newTimeout
+        if anyActive:
+            self._pollTimer = self._reschedule()
+
+
+# If we ever (let's hope not) need the above functionality on UNIX, this could
+# be factored into a different module.
+
+import pywintypes  # type: ignore[import]
+import win32api  # type: ignore[import]
+import win32file  # type: ignore[import]
+import win32pipe  # type: ignore[import]
+
+
+@implementer(IPushProducer)
+class _PollableReadPipe(_PollableResource):
+    def __init__(self, pipe, receivedCallback, lostCallback):
+        # security attributes for pipes
+        self.pipe = pipe
+        self.receivedCallback = receivedCallback
+        self.lostCallback = lostCallback
+
+    def checkWork(self):
+        finished = 0
+        fullDataRead = []
+
+        while 1:
+            try:
+                buffer, bytesToRead, result = win32pipe.PeekNamedPipe(self.pipe, 1)
+                # finished = (result == -1)
+                if not bytesToRead:
+                    break
+                hr, data = win32file.ReadFile(self.pipe, bytesToRead, None)
+                fullDataRead.append(data)
+            except win32api.error:
+                finished = 1
+                break
+
+        dataBuf = b"".join(fullDataRead)
+        if dataBuf:
+            self.receivedCallback(dataBuf)
+        if finished:
+            self.cleanup()
+        return len(dataBuf)
+
+    def cleanup(self):
+        self.deactivate()
+        self.lostCallback()
+
+    def close(self):
+        try:
+            win32api.CloseHandle(self.pipe)
+        except pywintypes.error:
+            # You can't close std handles...?
+            pass
+
+    def stopProducing(self):
+        self.close()
+
+    def pauseProducing(self):
+        self.deactivate()
+
+    def resumeProducing(self):
+        self.activate()
+
+
+FULL_BUFFER_SIZE = 64 * 1024
+
+
+@implementer(IConsumer)
+class _PollableWritePipe(_PollableResource):
+    def __init__(self, writePipe, lostCallback):
+        self.disconnecting = False
+        self.producer = None
+        self.producerPaused = False
+        self.streamingProducer = 0
+        self.outQueue = []
+        self.writePipe = writePipe
+        self.lostCallback = lostCallback
+        try:
+            win32pipe.SetNamedPipeHandleState(
+                writePipe, win32pipe.PIPE_NOWAIT, None, None
+            )
+        except pywintypes.error:
+            # Maybe it's an invalid handle.  Who knows.
+            pass
+
+    def close(self):
+        self.disconnecting = True
+
+    def bufferFull(self):
+        if self.producer is not None:
+            self.producerPaused = True
+            self.producer.pauseProducing()
+
+    def bufferEmpty(self):
+        if self.producer is not None and (
+            (not self.streamingProducer) or self.producerPaused
+        ):
+            self.producer.producerPaused = False
+            self.producer.resumeProducing()
+            return True
+        return False
+
+    # almost-but-not-quite-exact copy-paste from abstract.FileDescriptor... ugh
+
+    def registerProducer(self, producer, streaming):
+        """Register to receive data from a producer.
+
+        This sets this selectable to be a consumer for a producer.  When this
+        selectable runs out of data on a write() call, it will ask the producer
+        to resumeProducing(). A producer should implement the IProducer
+        interface.
+
+        FileDescriptor provides some infrastructure for producer methods.
+        """
+        if self.producer is not None:
+            raise RuntimeError(
+                "Cannot register producer %s, because producer %s was never "
+                "unregistered." % (producer, self.producer)
+            )
+        if not self.active:
+            producer.stopProducing()
+        else:
+            self.producer = producer
+            self.streamingProducer = streaming
+            if not streaming:
+                producer.resumeProducing()
+
+    def unregisterProducer(self):
+        """Stop consuming data from a producer, without disconnecting."""
+        self.producer = None
+
+    def writeConnectionLost(self):
+        self.deactivate()
+        try:
+            win32api.CloseHandle(self.writePipe)
+        except pywintypes.error:
+            # OMG what
+            pass
+        self.lostCallback()
+
+    def writeSequence(self, seq):
+        """
+        Append a C{list} or C{tuple} of bytes to the output buffer.
+
+        @param seq: C{list} or C{tuple} of C{str} instances to be appended to
+            the output buffer.
+
+        @raise TypeError: If C{seq} contains C{unicode}.
+        """
+        if str in map(type, seq):
+            raise TypeError("Unicode not allowed in output buffer.")
+        self.outQueue.extend(seq)
+
+    def write(self, data):
+        """
+        Append some bytes to the output buffer.
+
+        @param data: C{str} to be appended to the output buffer.
+        @type data: C{str}.
+
+        @raise TypeError: If C{data} is C{unicode} instead of C{str}.
+        """
+        if isinstance(data, str):
+            raise TypeError("Unicode not allowed in output buffer.")
+        if self.disconnecting:
+            return
+        self.outQueue.append(data)
+        if sum(map(len, self.outQueue)) > FULL_BUFFER_SIZE:
+            self.bufferFull()
+
+    def checkWork(self):
+        numBytesWritten = 0
+        if not self.outQueue:
+            if self.disconnecting:
+                self.writeConnectionLost()
+                return 0
+            try:
+                win32file.WriteFile(self.writePipe, b"", None)
+            except pywintypes.error:
+                self.writeConnectionLost()
+                return numBytesWritten
+        while self.outQueue:
+            data = self.outQueue.pop(0)
+            errCode = 0
+            try:
+                errCode, nBytesWritten = win32file.WriteFile(self.writePipe, data, None)
+            except win32api.error:
+                self.writeConnectionLost()
+                break
+            else:
+                # assert not errCode, "wtf an error code???"
+                numBytesWritten += nBytesWritten
+                if len(data) > nBytesWritten:
+                    self.outQueue.insert(0, data[nBytesWritten:])
+                    break
+        else:
+            resumed = self.bufferEmpty()
+            if not resumed and self.disconnecting:
+                self.writeConnectionLost()
+        return numBytesWritten
diff --git a/contrib/python/Twisted/py3/twisted/internet/_posixserialport.py b/contrib/python/Twisted/py3/twisted/internet/_posixserialport.py
new file mode 100644
index 00000000000..636aefa170f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_posixserialport.py
@@ -0,0 +1,81 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Serial Port Protocol
+"""
+
+
+# dependent on pyserial ( http://pyserial.sf.net/ )
+# only tested w/ 1.18 (5 Dec 2002)
+from serial import PARITY_NONE  # type: ignore[import]
+from serial import EIGHTBITS, STOPBITS_ONE
+
+from twisted.internet import abstract, fdesc
+from twisted.internet.serialport import BaseSerialPort
+
+
+class SerialPort(BaseSerialPort, abstract.FileDescriptor):
+    """
+    A select()able serial device, acting as a transport.
+    """
+
+    connected = 1
+
+    def __init__(
+        self,
+        protocol,
+        deviceNameOrPortNumber,
+        reactor,
+        baudrate=9600,
+        bytesize=EIGHTBITS,
+        parity=PARITY_NONE,
+        stopbits=STOPBITS_ONE,
+        timeout=0,
+        xonxoff=0,
+        rtscts=0,
+    ):
+        abstract.FileDescriptor.__init__(self, reactor)
+        self._serial = self._serialFactory(
+            deviceNameOrPortNumber,
+            baudrate=baudrate,
+            bytesize=bytesize,
+            parity=parity,
+            stopbits=stopbits,
+            timeout=timeout,
+            xonxoff=xonxoff,
+            rtscts=rtscts,
+        )
+        self.reactor = reactor
+        self.flushInput()
+        self.flushOutput()
+        self.protocol = protocol
+        self.protocol.makeConnection(self)
+        self.startReading()
+
+    def fileno(self):
+        return self._serial.fd
+
+    def writeSomeData(self, data):
+        """
+        Write some data to the serial device.
+        """
+        return fdesc.writeToFD(self.fileno(), data)
+
+    def doRead(self):
+        """
+        Some data's readable from serial device.
+        """
+        return fdesc.readFromFD(self.fileno(), self.protocol.dataReceived)
+
+    def connectionLost(self, reason):
+        """
+        Called when the serial port disconnects.
+
+        Will call C{connectionLost} on the protocol that is handling the
+        serial data.
+        """
+        abstract.FileDescriptor.connectionLost(self, reason)
+        self._serial.close()
+        self.protocol.connectionLost(reason)
diff --git a/contrib/python/Twisted/py3/twisted/internet/_posixstdio.py b/contrib/python/Twisted/py3/twisted/internet/_posixstdio.py
new file mode 100644
index 00000000000..b7ef9cdac39
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_posixstdio.py
@@ -0,0 +1,178 @@
+# -*- test-case-name: twisted.test.test_stdio -*-
+
+"""Standard input/out/err support.
+
+Future Plans::
+
+    support for stderr, perhaps
+    Rewrite to use the reactor instead of an ad-hoc mechanism for connecting
+        protocols to transport.
+
+Maintainer: James Y Knight
+"""
+
+from zope.interface import implementer
+
+from twisted.internet import error, interfaces, process
+from twisted.python import failure, log
+
+
+@implementer(interfaces.IAddress)
+class PipeAddress:
+    pass
+
+
+@implementer(
+    interfaces.ITransport,
+    interfaces.IProducer,
+    interfaces.IConsumer,
+    interfaces.IHalfCloseableDescriptor,
+)
+class StandardIO:
+    _reader = None
+    _writer = None
+    disconnected = False
+    disconnecting = False
+
+    def __init__(self, proto, stdin=0, stdout=1, reactor=None):
+        if reactor is None:
+            from twisted.internet import reactor
+        self.protocol = proto
+
+        self._writer = process.ProcessWriter(reactor, self, "write", stdout)
+        self._reader = process.ProcessReader(reactor, self, "read", stdin)
+        self._reader.startReading()
+        self.protocol.makeConnection(self)
+
+    # ITransport
+
+    # XXX Actually, see #3597.
+    def loseWriteConnection(self):
+        if self._writer is not None:
+            self._writer.loseConnection()
+
+    def write(self, data):
+        if self._writer is not None:
+            self._writer.write(data)
+
+    def writeSequence(self, data):
+        if self._writer is not None:
+            self._writer.writeSequence(data)
+
+    def loseConnection(self):
+        self.disconnecting = True
+
+        if self._writer is not None:
+            self._writer.loseConnection()
+        if self._reader is not None:
+            # Don't loseConnection, because we don't want to SIGPIPE it.
+            self._reader.stopReading()
+
+    def getPeer(self):
+        return PipeAddress()
+
+    def getHost(self):
+        return PipeAddress()
+
+    # Callbacks from process.ProcessReader/ProcessWriter
+    def childDataReceived(self, fd, data):
+        self.protocol.dataReceived(data)
+
+    def childConnectionLost(self, fd, reason):
+        if self.disconnected:
+            return
+
+        if reason.value.__class__ == error.ConnectionDone:
+            # Normal close
+            if fd == "read":
+                self._readConnectionLost(reason)
+            else:
+                self._writeConnectionLost(reason)
+        else:
+            self.connectionLost(reason)
+
+    def connectionLost(self, reason):
+        self.disconnected = True
+
+        # Make sure to cleanup the other half
+        _reader = self._reader
+        _writer = self._writer
+        protocol = self.protocol
+        self._reader = self._writer = None
+        self.protocol = None
+
+        if _writer is not None and not _writer.disconnected:
+            _writer.connectionLost(reason)
+
+        if _reader is not None and not _reader.disconnected:
+            _reader.connectionLost(reason)
+
+        try:
+            protocol.connectionLost(reason)
+        except BaseException:
+            log.err()
+
+    def _writeConnectionLost(self, reason):
+        self._writer = None
+        if self.disconnecting:
+            self.connectionLost(reason)
+            return
+
+        p = interfaces.IHalfCloseableProtocol(self.protocol, None)
+        if p:
+            try:
+                p.writeConnectionLost()
+            except BaseException:
+                log.err()
+                self.connectionLost(failure.Failure())
+
+    def _readConnectionLost(self, reason):
+        self._reader = None
+        p = interfaces.IHalfCloseableProtocol(self.protocol, None)
+        if p:
+            try:
+                p.readConnectionLost()
+            except BaseException:
+                log.err()
+                self.connectionLost(failure.Failure())
+        else:
+            self.connectionLost(reason)
+
+    # IConsumer
+    def registerProducer(self, producer, streaming):
+        if self._writer is None:
+            producer.stopProducing()
+        else:
+            self._writer.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        if self._writer is not None:
+            self._writer.unregisterProducer()
+
+    # IProducer
+    def stopProducing(self):
+        self.loseConnection()
+
+    def pauseProducing(self):
+        if self._reader is not None:
+            self._reader.pauseProducing()
+
+    def resumeProducing(self):
+        if self._reader is not None:
+            self._reader.resumeProducing()
+
+    def stopReading(self):
+        """Compatibility only, don't use. Call pauseProducing."""
+        self.pauseProducing()
+
+    def startReading(self):
+        """Compatibility only, don't use. Call resumeProducing."""
+        self.resumeProducing()
+
+    def readConnectionLost(self, reason):
+        # L{IHalfCloseableDescriptor.readConnectionLost}
+        raise NotImplementedError()
+
+    def writeConnectionLost(self, reason):
+        # L{IHalfCloseableDescriptor.writeConnectionLost}
+        raise NotImplementedError()
diff --git a/contrib/python/Twisted/py3/twisted/internet/_producer_helpers.py b/contrib/python/Twisted/py3/twisted/internet/_producer_helpers.py
new file mode 100644
index 00000000000..c2136e05091
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_producer_helpers.py
@@ -0,0 +1,124 @@
+# -*- test-case-name: twisted.test.test_producer_helpers -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Helpers for working with producers.
+"""
+
+from typing import List
+
+from zope.interface import implementer
+
+from twisted.internet.interfaces import IPushProducer
+from twisted.internet.task import cooperate
+from twisted.python import log
+from twisted.python.reflect import safe_str
+
+# This module exports nothing public, it's for internal Twisted use only.
+__all__: List[str] = []
+
+
+@implementer(IPushProducer)
+class _PullToPush:
+    """
+    An adapter that converts a non-streaming to a streaming producer.
+
+    Because of limitations of the producer API, this adapter requires the
+    cooperation of the consumer. When the consumer's C{registerProducer} is
+    called with a non-streaming producer, it must wrap it with L{_PullToPush}
+    and then call C{startStreaming} on the resulting object. When the
+    consumer's C{unregisterProducer} is called, it must call
+    C{stopStreaming} on the L{_PullToPush} instance.
+
+    If the underlying producer throws an exception from C{resumeProducing},
+    the producer will be unregistered from the consumer.
+
+    @ivar _producer: the underling non-streaming producer.
+
+    @ivar _consumer: the consumer with which the underlying producer was
+                     registered.
+
+    @ivar _finished: C{bool} indicating whether the producer has finished.
+
+    @ivar _coopTask: the result of calling L{cooperate}, the task driving the
+                     streaming producer.
+    """
+
+    _finished = False
+
+    def __init__(self, pullProducer, consumer):
+        self._producer = pullProducer
+        self._consumer = consumer
+
+    def _pull(self):
+        """
+        A generator that calls C{resumeProducing} on the underlying producer
+        forever.
+
+        If C{resumeProducing} throws an exception, the producer is
+        unregistered, which should result in streaming stopping.
+        """
+        while True:
+            try:
+                self._producer.resumeProducing()
+            except BaseException:
+                log.err(
+                    None,
+                    "%s failed, producing will be stopped:"
+                    % (safe_str(self._producer),),
+                )
+                try:
+                    self._consumer.unregisterProducer()
+                    # The consumer should now call stopStreaming() on us,
+                    # thus stopping the streaming.
+                except BaseException:
+                    # Since the consumer blew up, we may not have had
+                    # stopStreaming() called, so we just stop on our own:
+                    log.err(
+                        None,
+                        "%s failed to unregister producer:"
+                        % (safe_str(self._consumer),),
+                    )
+                    self._finished = True
+                    return
+            yield None
+
+    def startStreaming(self):
+        """
+        This should be called by the consumer when the producer is registered.
+
+        Start streaming data to the consumer.
+        """
+        self._coopTask = cooperate(self._pull())
+
+    def stopStreaming(self):
+        """
+        This should be called by the consumer when the producer is
+        unregistered.
+
+        Stop streaming data to the consumer.
+        """
+        if self._finished:
+            return
+        self._finished = True
+        self._coopTask.stop()
+
+    def pauseProducing(self):
+        """
+        @see: C{IPushProducer.pauseProducing}
+        """
+        self._coopTask.pause()
+
+    def resumeProducing(self):
+        """
+        @see: C{IPushProducer.resumeProducing}
+        """
+        self._coopTask.resume()
+
+    def stopProducing(self):
+        """
+        @see: C{IPushProducer.stopProducing}
+        """
+        self.stopStreaming()
+        self._producer.stopProducing()
diff --git a/contrib/python/Twisted/py3/twisted/internet/_resolver.py b/contrib/python/Twisted/py3/twisted/internet/_resolver.py
new file mode 100644
index 00000000000..f4a56b4808f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_resolver.py
@@ -0,0 +1,342 @@
+# -*- test-case-name: twisted.internet.test.test_resolver -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+IPv6-aware hostname resolution.
+
+@see: L{IHostnameResolver}
+"""
+
+
+from socket import (
+    AF_INET,
+    AF_INET6,
+    AF_UNSPEC,
+    SOCK_DGRAM,
+    SOCK_STREAM,
+    AddressFamily,
+    SocketKind,
+    gaierror,
+    getaddrinfo,
+)
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    List,
+    NoReturn,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+)
+
+from zope.interface import implementer
+
+from twisted.internet._idna import _idnaBytes
+from twisted.internet.address import IPv4Address, IPv6Address
+from twisted.internet.defer import Deferred
+from twisted.internet.error import DNSLookupError
+from twisted.internet.interfaces import (
+    IAddress,
+    IHostnameResolver,
+    IHostResolution,
+    IReactorThreads,
+    IResolutionReceiver,
+    IResolverSimple,
+)
+from twisted.internet.threads import deferToThreadPool
+from twisted.logger import Logger
+from twisted.python.compat import nativeString
+
+if TYPE_CHECKING:
+    from twisted.python.threadpool import ThreadPool
+
+
+@implementer(IHostResolution)
+class HostResolution:
+    """
+    The in-progress resolution of a given hostname.
+    """
+
+    def __init__(self, name: str):
+        """
+        Create a L{HostResolution} with the given name.
+        """
+        self.name = name
+
+    def cancel(self) -> NoReturn:
+        # IHostResolution.cancel
+        raise NotImplementedError()
+
+
+_any = frozenset([IPv4Address, IPv6Address])
+
+_typesToAF = {
+    frozenset([IPv4Address]): AF_INET,
+    frozenset([IPv6Address]): AF_INET6,
+    _any: AF_UNSPEC,
+}
+
+_afToType = {
+    AF_INET: IPv4Address,
+    AF_INET6: IPv6Address,
+}
+
+_transportToSocket = {
+    "TCP": SOCK_STREAM,
+    "UDP": SOCK_DGRAM,
+}
+
+_socktypeToType = {
+    SOCK_STREAM: "TCP",
+    SOCK_DGRAM: "UDP",
+}
+
+
+_GETADDRINFO_RESULT = List[
+    Tuple[
+        AddressFamily,
+        SocketKind,
+        int,
+        str,
+        Union[Tuple[str, int], Tuple[str, int, int, int]],
+    ]
+]
+
+
+@implementer(IHostnameResolver)
+class GAIResolver:
+    """
+    L{IHostnameResolver} implementation that resolves hostnames by calling
+    L{getaddrinfo} in a thread.
+    """
+
+    def __init__(
+        self,
+        reactor: IReactorThreads,
+        getThreadPool: Optional[Callable[[], "ThreadPool"]] = None,
+        getaddrinfo: Callable[[str, int, int, int], _GETADDRINFO_RESULT] = getaddrinfo,
+    ):
+        """
+        Create a L{GAIResolver}.
+
+        @param reactor: the reactor to schedule result-delivery on
+        @type reactor: L{IReactorThreads}
+
+        @param getThreadPool: a function to retrieve the thread pool to use for
+            scheduling name resolutions.  If not supplied, the use the given
+            C{reactor}'s thread pool.
+        @type getThreadPool: 0-argument callable returning a
+            L{twisted.python.threadpool.ThreadPool}
+
+        @param getaddrinfo: a reference to the L{getaddrinfo} to use - mainly
+            parameterized for testing.
+        @type getaddrinfo: callable with the same signature as L{getaddrinfo}
+        """
+        self._reactor = reactor
+        self._getThreadPool = (
+            reactor.getThreadPool if getThreadPool is None else getThreadPool
+        )
+        self._getaddrinfo = getaddrinfo
+
+    def resolveHostName(
+        self,
+        resolutionReceiver: IResolutionReceiver,
+        hostName: str,
+        portNumber: int = 0,
+        addressTypes: Optional[Sequence[Type[IAddress]]] = None,
+        transportSemantics: str = "TCP",
+    ) -> IHostResolution:
+        """
+        See L{IHostnameResolver.resolveHostName}
+
+        @param resolutionReceiver: see interface
+
+        @param hostName: see interface
+
+        @param portNumber: see interface
+
+        @param addressTypes: see interface
+
+        @param transportSemantics: see interface
+
+        @return: see interface
+        """
+        pool = self._getThreadPool()
+        addressFamily = _typesToAF[
+            _any if addressTypes is None else frozenset(addressTypes)
+        ]
+        socketType = _transportToSocket[transportSemantics]
+
+        def get() -> _GETADDRINFO_RESULT:
+            try:
+                return self._getaddrinfo(
+                    hostName, portNumber, addressFamily, socketType
+                )
+            except gaierror:
+                return []
+
+        d = deferToThreadPool(self._reactor, pool, get)
+        resolution = HostResolution(hostName)
+        resolutionReceiver.resolutionBegan(resolution)
+
+        @d.addCallback
+        def deliverResults(result: _GETADDRINFO_RESULT) -> None:
+            for family, socktype, proto, cannoname, sockaddr in result:
+                addrType = _afToType[family]
+                resolutionReceiver.addressResolved(
+                    addrType(_socktypeToType.get(socktype, "TCP"), *sockaddr)
+                )
+            resolutionReceiver.resolutionComplete()
+
+        return resolution
+
+
+@implementer(IHostnameResolver)
+class SimpleResolverComplexifier:
+    """
+    A converter from L{IResolverSimple} to L{IHostnameResolver}.
+    """
+
+    _log = Logger()
+
+    def __init__(self, simpleResolver: IResolverSimple):
+        """
+        Construct a L{SimpleResolverComplexifier} with an L{IResolverSimple}.
+        """
+        self._simpleResolver = simpleResolver
+
+    def resolveHostName(
+        self,
+        resolutionReceiver: IResolutionReceiver,
+        hostName: str,
+        portNumber: int = 0,
+        addressTypes: Optional[Sequence[Type[IAddress]]] = None,
+        transportSemantics: str = "TCP",
+    ) -> IHostResolution:
+        """
+        See L{IHostnameResolver.resolveHostName}
+
+        @param resolutionReceiver: see interface
+
+        @param hostName: see interface
+
+        @param portNumber: see interface
+
+        @param addressTypes: see interface
+
+        @param transportSemantics: see interface
+
+        @return: see interface
+        """
+        # If it's str, we need to make sure that it's just ASCII.
+        try:
+            hostName_bytes = hostName.encode("ascii")
+        except UnicodeEncodeError:
+            # If it's not just ASCII, IDNA it. We don't want to give a Unicode
+            # string with non-ASCII in it to Python 3, as if anyone passes that
+            # to a Python 3 stdlib function, it will probably use the wrong
+            # IDNA version and break absolutely everything
+            hostName_bytes = _idnaBytes(hostName)
+
+        # Make sure it's passed down as a native str, to maintain the interface
+        hostName = nativeString(hostName_bytes)
+
+        resolution = HostResolution(hostName)
+        resolutionReceiver.resolutionBegan(resolution)
+        (
+            self._simpleResolver.getHostByName(hostName)
+            .addCallback(
+                lambda address: resolutionReceiver.addressResolved(
+                    IPv4Address("TCP", address, portNumber)
+                )
+            )
+            .addErrback(
+                lambda error: None
+                if error.check(DNSLookupError)
+                else self._log.failure(
+                    "while looking up {name} with {resolver}",
+                    error,
+                    name=hostName,
+                    resolver=self._simpleResolver,
+                )
+            )
+            .addCallback(lambda nothing: resolutionReceiver.resolutionComplete())
+        )
+        return resolution
+
+
+@implementer(IResolutionReceiver)
+class FirstOneWins:
+    """
+    An L{IResolutionReceiver} which fires a L{Deferred} with its first result.
+    """
+
+    def __init__(self, deferred: "Deferred[str]"):
+        """
+        @param deferred: The L{Deferred} to fire when the first resolution
+            result arrives.
+        """
+        self._deferred = deferred
+        self._resolved = False
+
+    def resolutionBegan(self, resolution: IHostResolution) -> None:
+        """
+        See L{IResolutionReceiver.resolutionBegan}
+
+        @param resolution: See L{IResolutionReceiver.resolutionBegan}
+        """
+        self._resolution = resolution
+
+    def addressResolved(self, address: IAddress) -> None:
+        """
+        See L{IResolutionReceiver.addressResolved}
+
+        @param address: See L{IResolutionReceiver.addressResolved}
+        """
+        if self._resolved:
+            return
+        self._resolved = True
+        # This is used by ComplexResolverSimplifier which specifies only results
+        # of IPv4Address.
+        assert isinstance(address, IPv4Address)
+        self._deferred.callback(address.host)
+
+    def resolutionComplete(self) -> None:
+        """
+        See L{IResolutionReceiver.resolutionComplete}
+        """
+        if self._resolved:
+            return
+        self._deferred.errback(DNSLookupError(self._resolution.name))
+
+
+@implementer(IResolverSimple)
+class ComplexResolverSimplifier:
+    """
+    A converter from L{IHostnameResolver} to L{IResolverSimple}
+    """
+
+    def __init__(self, nameResolver: IHostnameResolver):
+        """
+        Create a L{ComplexResolverSimplifier} with an L{IHostnameResolver}.
+
+        @param nameResolver: The L{IHostnameResolver} to use.
+        """
+        self._nameResolver = nameResolver
+
+    def getHostByName(self, name: str, timeouts: Sequence[int] = ()) -> "Deferred[str]":
+        """
+        See L{IResolverSimple.getHostByName}
+
+        @param name: see L{IResolverSimple.getHostByName}
+
+        @param timeouts: see L{IResolverSimple.getHostByName}
+
+        @return: see L{IResolverSimple.getHostByName}
+        """
+        result: "Deferred[str]" = Deferred()
+        self._nameResolver.resolveHostName(FirstOneWins(result), name, 0, [IPv4Address])
+        return result
diff --git a/contrib/python/Twisted/py3/twisted/internet/_signals.py b/contrib/python/Twisted/py3/twisted/internet/_signals.py
new file mode 100644
index 00000000000..fa878f6cbad
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_signals.py
@@ -0,0 +1,445 @@
+# -*- test-case-name: twisted.internet.test.test_sigchld -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module is used to integrate child process termination into a
+reactor event loop.  This is a challenging feature to provide because
+most platforms indicate process termination via SIGCHLD and do not
+provide a way to wait for that signal and arbitrary I/O events at the
+same time.  The naive implementation involves installing a Python
+SIGCHLD handler; unfortunately this leads to other syscalls being
+interrupted (whenever SIGCHLD is received) and failing with EINTR
+(which almost no one is prepared to handle).  This interruption can be
+disabled via siginterrupt(2) (or one of the equivalent mechanisms);
+however, if the SIGCHLD is delivered by the platform to a non-main
+thread (not a common occurrence, but difficult to prove impossible),
+the main thread (waiting on select() or another event notification
+API) may not wake up leading to an arbitrary delay before the child
+termination is noticed.
+
+The basic solution to all these issues involves enabling SA_RESTART (ie,
+disabling system call interruption) and registering a C signal handler which
+writes a byte to a pipe.  The other end of the pipe is registered with the
+event loop, allowing it to wake up shortly after SIGCHLD is received.  See
+L{_SIGCHLDWaker} for the implementation of the event loop side of this
+solution.  The use of a pipe this way is known as the U{self-pipe
+trick<http://cr.yp.to/docs/selfpipe.html>}.
+
+From Python version 2.6, C{signal.siginterrupt} and C{signal.set_wakeup_fd}
+provide the necessary C signal handler which writes to the pipe to be
+registered with C{SA_RESTART}.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import errno
+import os
+import signal
+import socket
+from types import FrameType
+from typing import Callable, Optional, Sequence
+
+from zope.interface import Attribute, Interface, implementer
+
+from attrs import define, frozen
+from typing_extensions import Protocol, TypeAlias
+
+from twisted.internet.interfaces import IReadDescriptor
+from twisted.python import failure, log, util
+from twisted.python.runtime import platformType
+
+if platformType == "posix":
+    from . import fdesc, process
+
+SignalHandler: TypeAlias = Callable[[int, Optional[FrameType]], None]
+
+
+def installHandler(fd: int) -> int:
+    """
+    Install a signal handler which will write a byte to C{fd} when
+    I{SIGCHLD} is received.
+
+    This is implemented by installing a SIGCHLD handler that does nothing,
+    setting the I{SIGCHLD} handler as not allowed to interrupt system calls,
+    and using L{signal.set_wakeup_fd} to do the actual writing.
+
+    @param fd: The file descriptor to which to write when I{SIGCHLD} is
+        received.
+
+    @return: The file descriptor previously configured for this use.
+    """
+    if fd == -1:
+        signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+    else:
+
+        def noopSignalHandler(*args):
+            pass
+
+        signal.signal(signal.SIGCHLD, noopSignalHandler)
+        signal.siginterrupt(signal.SIGCHLD, False)
+    return signal.set_wakeup_fd(fd)
+
+
+def isDefaultHandler():
+    """
+    Determine whether the I{SIGCHLD} handler is the default or not.
+    """
+    return signal.getsignal(signal.SIGCHLD) == signal.SIG_DFL
+
+
+class SignalHandling(Protocol):
+    """
+    The L{SignalHandling} protocol enables customizable signal-handling
+    behaviors for reactors.
+
+    A value that conforms to L{SignalHandling} has install and uninstall hooks
+    that are called by a reactor at the correct times to have the (typically)
+    process-global effects necessary for dealing with signals.
+    """
+
+    def install(self) -> None:
+        """
+        Install the signal handlers.
+        """
+
+    def uninstall(self) -> None:
+        """
+        Restore signal handlers to their original state.
+        """
+
+
+@frozen
+class _WithoutSignalHandling:
+    """
+    A L{SignalHandling} implementation that does no signal handling.
+
+    This is the implementation of C{installSignalHandlers=False}.
+    """
+
+    def install(self) -> None:
+        """
+        Do not install any signal handlers.
+        """
+
+    def uninstall(self) -> None:
+        """
+        Do nothing because L{install} installed nothing.
+        """
+
+
+@frozen
+class _WithSignalHandling:
+    """
+    A reactor core helper that can manage signals: it installs signal handlers
+    at start time.
+    """
+
+    _sigInt: SignalHandler
+    _sigBreak: SignalHandler
+    _sigTerm: SignalHandler
+
+    def install(self) -> None:
+        """
+        Install the signal handlers for the Twisted event loop.
+        """
+        if signal.getsignal(signal.SIGINT) == signal.default_int_handler:
+            # only handle if there isn't already a handler, e.g. for Pdb.
+            signal.signal(signal.SIGINT, self._sigInt)
+        signal.signal(signal.SIGTERM, self._sigTerm)
+
+        # Catch Ctrl-Break in windows
+        SIGBREAK = getattr(signal, "SIGBREAK", None)
+        if SIGBREAK is not None:
+            signal.signal(SIGBREAK, self._sigBreak)
+
+    def uninstall(self) -> None:
+        """
+        At the moment, do nothing (for historical reasons).
+        """
+        # This should really do something.
+        # https://github.com/twisted/twisted/issues/11761
+
+
+@define
+class _MultiSignalHandling:
+    """
+    An implementation of L{SignalHandling} which propagates protocol
+    method calls to a number of other implementations.
+
+    This supports composition of multiple signal handling implementations into
+    a single object so the reactor doesn't have to be concerned with how those
+    implementations are factored.
+
+    @ivar _signalHandlings: The other C{SignalHandling} implementations to
+        which to propagate calls.
+
+    @ivar _installed: If L{install} has been called but L{uninstall} has not.
+        This is used to avoid double cleanup which otherwise results (at least
+        during test suite runs) because twisted.internet.reactormixins doesn't
+        keep track of whether a reactor has run or not but always invokes its
+        cleanup logic.
+    """
+
+    _signalHandlings: Sequence[SignalHandling]
+    _installed: bool = False
+
+    def install(self) -> None:
+        for d in self._signalHandlings:
+            d.install()
+        self._installed = True
+
+    def uninstall(self) -> None:
+        if self._installed:
+            for d in self._signalHandlings:
+                d.uninstall()
+            self._installed = False
+
+
+@define
+class _ChildSignalHandling:
+    """
+    Signal handling behavior which supports I{SIGCHLD} for notification about
+    changes to child process state.
+
+    @ivar _childWaker: L{None} or a reference to the L{_SIGCHLDWaker} which is
+        used to properly notice child process termination.  This is L{None}
+        when this handling behavior is not installed and non-C{None}
+        otherwise.  This is mostly an unfortunate implementation detail due to
+        L{_SIGCHLDWaker} allocating file descriptors as a side-effect of its
+        initializer.
+    """
+
+    _addInternalReader: Callable[[IReadDescriptor], object]
+    _removeInternalReader: Callable[[IReadDescriptor], object]
+    _childWaker: Optional[_SIGCHLDWaker] = None
+
+    def install(self) -> None:
+        """
+        Extend the basic signal handling logic to also support handling
+        SIGCHLD to know when to try to reap child processes.
+        """
+        # This conditional should probably not be necessary.
+        # https://github.com/twisted/twisted/issues/11763
+        if self._childWaker is None:
+            self._childWaker = _SIGCHLDWaker()
+            self._addInternalReader(self._childWaker)
+        self._childWaker.install()
+
+        # Also reap all processes right now, in case we missed any
+        # signals before we installed the SIGCHLD waker/handler.
+        # This should only happen if someone used spawnProcess
+        # before calling reactor.run (and the process also exited
+        # already).
+        process.reapAllProcesses()
+
+    def uninstall(self) -> None:
+        """
+        If a child waker was created and installed, uninstall it now.
+
+        Since this disables reactor functionality and is only called when the
+        reactor is stopping, it doesn't provide any directly useful
+        functionality, but the cleanup of reactor-related process-global state
+        that it does helps in unit tests involving multiple reactors and is
+        generally just a nice thing.
+        """
+        assert self._childWaker is not None
+
+        # XXX This would probably be an alright place to put all of the
+        # cleanup code for all internal readers (here and in the base class,
+        # anyway).  See #3063 for that cleanup task.
+        self._removeInternalReader(self._childWaker)
+        self._childWaker.uninstall()
+        self._childWaker.connectionLost(failure.Failure(Exception("uninstalled")))
+
+        # We just spoiled the current _childWaker so throw it away.  We can
+        # make a new one later if need be.
+        self._childWaker = None
+
+
+class _IWaker(Interface):
+    """
+    Interface to wake up the event loop based on the self-pipe trick.
+
+    The U{I{self-pipe trick}<http://cr.yp.to/docs/selfpipe.html>}, used to wake
+    up the main loop from another thread or a signal handler.
+    This is why we have wakeUp together with doRead
+
+    This is used by threads or signals to wake up the event loop.
+    """
+
+    disconnected = Attribute("")
+
+    def wakeUp():
+        """
+        Called when the event should be wake up.
+        """
+
+    def doRead():
+        """
+        Read some data from my connection and discard it.
+        """
+
+    def connectionLost(reason: failure.Failure) -> None:
+        """
+        Called when connection was closed and the pipes.
+        """
+
+
+@implementer(_IWaker)
+class _SocketWaker(log.Logger):
+    """
+    The I{self-pipe trick<http://cr.yp.to/docs/selfpipe.html>}, implemented
+    using a pair of sockets rather than pipes (due to the lack of support in
+    select() on Windows for pipes), used to wake up the main loop from
+    another thread.
+    """
+
+    disconnected = 0
+
+    def __init__(self) -> None:
+        """Initialize."""
+        # Following select_trigger (from asyncore)'s example;
+        client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        with contextlib.closing(
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        ) as server:
+            server.bind(("127.0.0.1", 0))
+            server.listen(1)
+            client.connect(server.getsockname())
+            reader, clientaddr = server.accept()
+        client.setblocking(False)
+        reader.setblocking(False)
+        self.r = reader
+        self.w = client
+        self.fileno = self.r.fileno
+
+    def wakeUp(self):
+        """Send a byte to my connection."""
+        try:
+            util.untilConcludes(self.w.send, b"x")
+        except OSError as e:
+            if e.args[0] != errno.WSAEWOULDBLOCK:
+                raise
+
+    def doRead(self):
+        """
+        Read some data from my connection.
+        """
+        try:
+            self.r.recv(8192)
+        except OSError:
+            pass
+
+    def connectionLost(self, reason):
+        self.r.close()
+        self.w.close()
+
+
+@implementer(IReadDescriptor)
+class _FDWaker(log.Logger):
+    """
+    The I{self-pipe trick<http://cr.yp.to/docs/selfpipe.html>}, used to wake
+    up the main loop from another thread or a signal handler.
+
+    L{_FDWaker} is a base class for waker implementations based on
+    writing to a pipe being monitored by the reactor.
+
+    @ivar o: The file descriptor for the end of the pipe which can be
+        written to wake up a reactor monitoring this waker.
+
+    @ivar i: The file descriptor which should be monitored in order to
+        be awoken by this waker.
+    """
+
+    disconnected = 0
+
+    i: int
+    o: int
+
+    def __init__(self) -> None:
+        """Initialize."""
+        self.i, self.o = os.pipe()
+        fdesc.setNonBlocking(self.i)
+        fdesc._setCloseOnExec(self.i)
+        fdesc.setNonBlocking(self.o)
+        fdesc._setCloseOnExec(self.o)
+        self.fileno = lambda: self.i
+
+    def doRead(self) -> None:
+        """
+        Read some bytes from the pipe and discard them.
+        """
+        fdesc.readFromFD(self.fileno(), lambda data: None)
+
+    def connectionLost(self, reason):
+        """Close both ends of my pipe."""
+        if not hasattr(self, "o"):
+            return
+        for fd in self.i, self.o:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+        del self.i, self.o
+
+
+@implementer(_IWaker)
+class _UnixWaker(_FDWaker):
+    """
+    This class provides a simple interface to wake up the event loop.
+
+    This is used by threads or signals to wake up the event loop.
+    """
+
+    def wakeUp(self):
+        """Write one byte to the pipe, and flush it."""
+        # We don't use fdesc.writeToFD since we need to distinguish
+        # between EINTR (try again) and EAGAIN (do nothing).
+        if self.o is not None:
+            try:
+                util.untilConcludes(os.write, self.o, b"x")
+            except OSError as e:
+                # XXX There is no unit test for raising the exception
+                # for other errnos. See #4285.
+                if e.errno != errno.EAGAIN:
+                    raise
+
+
+if platformType == "posix":
+    _Waker = _UnixWaker
+else:
+    # Primarily Windows and Jython.
+    _Waker = _SocketWaker  # type: ignore[misc,assignment]
+
+
+class _SIGCHLDWaker(_FDWaker):
+    """
+    L{_SIGCHLDWaker} can wake up a reactor whenever C{SIGCHLD} is received.
+    """
+
+    def install(self) -> None:
+        """
+        Install the handler necessary to make this waker active.
+        """
+        installHandler(self.o)
+
+    def uninstall(self) -> None:
+        """
+        Remove the handler which makes this waker active.
+        """
+        installHandler(-1)
+
+    def doRead(self) -> None:
+        """
+        Having woken up the reactor in response to receipt of
+        C{SIGCHLD}, reap the process which exited.
+
+        This is called whenever the reactor notices the waker pipe is
+        writeable, which happens soon after any call to the C{wakeUp}
+        method.
+        """
+        super().doRead()
+        process.reapAllProcesses()
diff --git a/contrib/python/Twisted/py3/twisted/internet/_sslverify.py b/contrib/python/Twisted/py3/twisted/internet/_sslverify.py
new file mode 100644
index 00000000000..552c30bbf0e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_sslverify.py
@@ -0,0 +1,2017 @@
+# -*- test-case-name: twisted.test.test_sslverify -*-
+# Copyright (c) 2005 Divmod, Inc.
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+from __future__ import annotations
+
+import warnings
+from binascii import hexlify
+from functools import lru_cache
+from hashlib import md5
+from typing import Dict
+
+from zope.interface import Interface, implementer
+
+from OpenSSL import SSL, crypto
+from OpenSSL._util import lib as pyOpenSSLlib  # type: ignore[import]
+
+import attr
+from constantly import FlagConstant, Flags, NamedConstant, Names  # type: ignore[import]
+from incremental import Version
+
+from twisted.internet.abstract import isIPAddress, isIPv6Address
+from twisted.internet.defer import Deferred
+from twisted.internet.error import CertificateError, VerifyError
+from twisted.internet.interfaces import (
+    IAcceptableCiphers,
+    ICipher,
+    IOpenSSLClientConnectionCreator,
+    IOpenSSLContextFactory,
+)
+from twisted.python import log, util
+from twisted.python.compat import nativeString
+from twisted.python.deprecate import _mutuallyExclusiveArguments, deprecated
+from twisted.python.failure import Failure
+from twisted.python.randbytes import secureRandom
+from ._idna import _idnaBytes
+
+
+class TLSVersion(Names):
+    """
+    TLS versions that we can negotiate with the client/server.
+    """
+
+    SSLv3 = NamedConstant()
+    TLSv1_0 = NamedConstant()
+    TLSv1_1 = NamedConstant()
+    TLSv1_2 = NamedConstant()
+    TLSv1_3 = NamedConstant()
+
+
+_tlsDisableFlags = {
+    TLSVersion.SSLv3: SSL.OP_NO_SSLv3,
+    TLSVersion.TLSv1_0: SSL.OP_NO_TLSv1,
+    TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1,
+    TLSVersion.TLSv1_2: SSL.OP_NO_TLSv1_2,
+    # If we don't have TLS v1.3 yet, we can't disable it -- this is just so
+    # when it makes it into OpenSSL, connections knowingly bracketed to v1.2
+    # don't end up going to v1.3
+    TLSVersion.TLSv1_3: getattr(SSL, "OP_NO_TLSv1_3", 0x00),
+}
+
+
+def _getExcludedTLSProtocols(oldest, newest):
+    """
+    Given a pair of L{TLSVersion} constants, figure out what versions we want
+    to disable (as OpenSSL is an exclusion based API).
+
+    @param oldest: The oldest L{TLSVersion} we want to allow.
+    @type oldest: L{TLSVersion} constant
+
+    @param newest: The newest L{TLSVersion} we want to allow, or L{None} for no
+        upper limit.
+    @type newest: L{TLSVersion} constant or L{None}
+
+    @return: The versions we want to disable.
+    @rtype: L{list} of L{TLSVersion} constants.
+    """
+    versions = list(TLSVersion.iterconstants())
+    excludedVersions = [x for x in versions[: versions.index(oldest)]]
+
+    if newest:
+        excludedVersions.extend([x for x in versions[versions.index(newest) :]])
+
+    return excludedVersions
+
+
+class SimpleVerificationError(Exception):
+    """
+    Not a very useful verification error.
+    """
+
+
+def simpleVerifyHostname(connection, hostname):
+    """
+    Check only the common name in the certificate presented by the peer and
+    only for an exact match.
+
+    This is to provide I{something} in the way of hostname verification to
+    users who haven't installed C{service_identity}. This check is overly
+    strict, relies on a deprecated TLS feature (you're supposed to ignore the
+    commonName if the subjectAlternativeName extensions are present, I
+    believe), and lots of valid certificates will fail.
+
+    @param connection: the OpenSSL connection to verify.
+    @type connection: L{OpenSSL.SSL.Connection}
+
+    @param hostname: The hostname expected by the user.
+    @type hostname: L{unicode}
+
+    @raise twisted.internet.ssl.VerificationError: if the common name and
+        hostname don't match.
+    """
+    commonName = connection.get_peer_certificate().get_subject().commonName
+    if commonName != hostname:
+        raise SimpleVerificationError(repr(commonName) + "!=" + repr(hostname))
+
+
+def simpleVerifyIPAddress(connection, hostname):
+    """
+    Always fails validation of IP addresses
+
+    @param connection: the OpenSSL connection to verify.
+    @type connection: L{OpenSSL.SSL.Connection}
+
+    @param hostname: The hostname expected by the user.
+    @type hostname: L{unicode}
+
+    @raise twisted.internet.ssl.VerificationError: Always raised
+    """
+    raise SimpleVerificationError("Cannot verify certificate IP addresses")
+
+
+def _usablePyOpenSSL(version):
+    """
+    Check pyOpenSSL version string whether we can use it for host verification.
+
+    @param version: A pyOpenSSL version string.
+    @type version: L{str}
+
+    @rtype: L{bool}
+    """
+    major, minor = (int(part) for part in version.split(".")[:2])
+    return (major, minor) >= (0, 12)
+
+
+def _selectVerifyImplementation():
+    """
+    Determine if C{service_identity} is installed. If so, use it. If not, use
+    simplistic and incorrect checking as implemented in
+    L{simpleVerifyHostname}.
+
+    @return: 2-tuple of (C{verify_hostname}, C{VerificationError})
+    @rtype: L{tuple}
+    """
+
+    whatsWrong = (
+        "Without the service_identity module, Twisted can perform only "
+        "rudimentary TLS client hostname verification.  Many valid "
+        "certificate/hostname mappings may be rejected."
+    )
+
+    try:
+        from service_identity import VerificationError
+        from service_identity.pyopenssl import verify_hostname, verify_ip_address
+
+        return verify_hostname, verify_ip_address, VerificationError
+    except ImportError as e:
+        warnings.warn_explicit(
+            "You do not have a working installation of the "
+            "service_identity module: '" + str(e) + "'.  "
+            "Please install it from "
+            "<https://pypi.python.org/pypi/service_identity> and make "
+            "sure all of its dependencies are satisfied.  " + whatsWrong,
+            # Unfortunately the lineno is required.
+            category=UserWarning,
+            filename="",
+            lineno=0,
+        )
+
+    return simpleVerifyHostname, simpleVerifyIPAddress, SimpleVerificationError
+
+
+verifyHostname, verifyIPAddress, VerificationError = _selectVerifyImplementation()
+
+
+class ProtocolNegotiationSupport(Flags):
+    """
+    L{ProtocolNegotiationSupport} defines flags which are used to indicate the
+    level of NPN/ALPN support provided by the TLS backend.
+
+    @cvar NOSUPPORT: There is no support for NPN or ALPN. This is exclusive
+        with both L{NPN} and L{ALPN}.
+    @cvar NPN: The implementation supports Next Protocol Negotiation.
+    @cvar ALPN: The implementation supports Application Layer Protocol
+        Negotiation.
+    """
+
+    NPN = FlagConstant(0x0001)
+    ALPN = FlagConstant(0x0002)
+
+
+# FIXME: https://twistedmatrix.com/trac/ticket/8074
+# Currently flags with literal zero values behave incorrectly. However,
+# creating a flag by NOTing a flag with itself appears to work totally fine, so
+# do that instead.
+ProtocolNegotiationSupport.NOSUPPORT = (
+    ProtocolNegotiationSupport.NPN ^ ProtocolNegotiationSupport.NPN
+)
+
+
+def protocolNegotiationMechanisms():
+    """
+    Checks whether your versions of PyOpenSSL and OpenSSL are recent enough to
+    support protocol negotiation, and if they are, what kind of protocol
+    negotiation is supported.
+
+    @return: A combination of flags from L{ProtocolNegotiationSupport} that
+        indicate which mechanisms for protocol negotiation are supported.
+    @rtype: L{constantly.FlagConstant}
+    """
+    support = ProtocolNegotiationSupport.NOSUPPORT
+    ctx = SSL.Context(SSL.SSLv23_METHOD)
+
+    try:
+        ctx.set_npn_advertise_callback(lambda c: None)
+    except (AttributeError, NotImplementedError):
+        pass
+    else:
+        support |= ProtocolNegotiationSupport.NPN
+
+    try:
+        ctx.set_alpn_select_callback(lambda c: None)
+    except (AttributeError, NotImplementedError):
+        pass
+    else:
+        support |= ProtocolNegotiationSupport.ALPN
+
+    return support
+
+
+_x509names = {
+    "CN": "commonName",
+    "commonName": "commonName",
+    "O": "organizationName",
+    "organizationName": "organizationName",
+    "OU": "organizationalUnitName",
+    "organizationalUnitName": "organizationalUnitName",
+    "L": "localityName",
+    "localityName": "localityName",
+    "ST": "stateOrProvinceName",
+    "stateOrProvinceName": "stateOrProvinceName",
+    "C": "countryName",
+    "countryName": "countryName",
+    "emailAddress": "emailAddress",
+}
+
+
+class DistinguishedName(Dict[str, bytes]):
+    """
+    Identify and describe an entity.
+
+    Distinguished names are used to provide a minimal amount of identifying
+    information about a certificate issuer or subject.  They are commonly
+    created with one or more of the following fields::
+
+        commonName (CN)
+        organizationName (O)
+        organizationalUnitName (OU)
+        localityName (L)
+        stateOrProvinceName (ST)
+        countryName (C)
+        emailAddress
+
+    A L{DistinguishedName} should be constructed using keyword arguments whose
+    keys can be any of the field names above (as a native string), and the
+    values are either Unicode text which is encodable to ASCII, or L{bytes}
+    limited to the ASCII subset. Any fields passed to the constructor will be
+    set as attributes, accessible using both their extended name and their
+    shortened acronym. The attribute values will be the ASCII-encoded
+    bytes. For example::
+
+        >>> dn = DistinguishedName(commonName=b'www.example.com',
+        ...                        C='US')
+        >>> dn.C
+        b'US'
+        >>> dn.countryName
+        b'US'
+        >>> hasattr(dn, "organizationName")
+        False
+
+    L{DistinguishedName} instances can also be used as dictionaries; the keys
+    are extended name of the fields::
+
+        >>> dn.keys()
+        ['countryName', 'commonName']
+        >>> dn['countryName']
+        b'US'
+
+    """
+
+    __slots__ = ()
+
+    def __init__(self, **kw):
+        for k, v in kw.items():
+            setattr(self, k, v)
+
+    def _copyFrom(self, x509name):
+        for name in _x509names:
+            value = getattr(x509name, name, None)
+            if value is not None:
+                setattr(self, name, value)
+
+    def _copyInto(self, x509name):
+        for k, v in self.items():
+            setattr(x509name, k, nativeString(v))
+
+    def __repr__(self) -> str:
+        return "<DN %s>" % (dict.__repr__(self)[1:-1])
+
+    def __getattr__(self, attr):
+        try:
+            return self[_x509names[attr]]
+        except KeyError:
+            raise AttributeError(attr)
+
+    def __setattr__(self, attr, value):
+        if attr not in _x509names:
+            raise AttributeError(f"{attr} is not a valid OpenSSL X509 name field")
+        realAttr = _x509names[attr]
+        if not isinstance(value, bytes):
+            value = value.encode("ascii")
+        self[realAttr] = value
+
+    def inspect(self):
+        """
+        Return a multi-line, human-readable representation of this DN.
+
+        @rtype: L{str}
+        """
+        l = []
+        lablen = 0
+
+        def uniqueValues(mapping):
+            return set(mapping.values())
+
+        for k in sorted(uniqueValues(_x509names)):
+            label = util.nameToLabel(k)
+            lablen = max(len(label), lablen)
+            v = getattr(self, k, None)
+            if v is not None:
+                l.append((label, nativeString(v)))
+        lablen += 2
+        for n, (label, attrib) in enumerate(l):
+            l[n] = label.rjust(lablen) + ": " + attrib
+        return "\n".join(l)
+
+
+DN = DistinguishedName
+
+
+class CertBase:
+    """
+    Base class for public (certificate only) and private (certificate + key
+    pair) certificates.
+
+    @ivar original: The underlying OpenSSL certificate object.
+    @type original: L{OpenSSL.crypto.X509}
+    """
+
+    def __init__(self, original):
+        self.original = original
+
+    def _copyName(self, suffix):
+        dn = DistinguishedName()
+        dn._copyFrom(getattr(self.original, "get_" + suffix)())
+        return dn
+
+    def getSubject(self):
+        """
+        Retrieve the subject of this certificate.
+
+        @return: A copy of the subject of this certificate.
+        @rtype: L{DistinguishedName}
+        """
+        return self._copyName("subject")
+
+    def __conform__(self, interface):
+        """
+        Convert this L{CertBase} into a provider of the given interface.
+
+        @param interface: The interface to conform to.
+        @type interface: L{zope.interface.interfaces.IInterface}
+
+        @return: an L{IOpenSSLTrustRoot} provider or L{NotImplemented}
+        @rtype: L{IOpenSSLTrustRoot} or L{NotImplemented}
+        """
+        if interface is IOpenSSLTrustRoot:
+            return OpenSSLCertificateAuthorities([self.original])
+        return NotImplemented
+
+
+def _handleattrhelper(Class, transport, methodName):
+    """
+    (private) Helper for L{Certificate.peerFromTransport} and
+    L{Certificate.hostFromTransport} which checks for incompatible handle types
+    and null certificates and raises the appropriate exception or returns the
+    appropriate certificate object.
+    """
+    method = getattr(transport.getHandle(), f"get_{methodName}_certificate", None)
+    if method is None:
+        raise CertificateError(
+            "non-TLS transport {!r} did not have {} certificate".format(
+                transport, methodName
+            )
+        )
+    cert = method()
+    if cert is None:
+        raise CertificateError(
+            "TLS transport {!r} did not have {} certificate".format(
+                transport, methodName
+            )
+        )
+    return Class(cert)
+
+
+class Certificate(CertBase):
+    """
+    An x509 certificate.
+    """
+
+    def __repr__(self) -> str:
+        return "<{} Subject={} Issuer={}>".format(
+            self.__class__.__name__,
+            self.getSubject().commonName,
+            self.getIssuer().commonName,
+        )
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Certificate):
+            return self.dump() == other.dump()
+        return NotImplemented
+
+    @classmethod
+    def load(Class, requestData, format=crypto.FILETYPE_ASN1, args=()):
+        """
+        Load a certificate from an ASN.1- or PEM-format string.
+
+        @rtype: C{Class}
+        """
+        return Class(crypto.load_certificate(format, requestData), *args)
+
+    # We can't use super() because it is old style still, so we have to hack
+    # around things wanting to call the parent function
+    _load = load
+
+    def dumpPEM(self):
+        """
+        Dump this certificate to a PEM-format data string.
+
+        @rtype: L{str}
+        """
+        return self.dump(crypto.FILETYPE_PEM)
+
+    @classmethod
+    def loadPEM(Class, data):
+        """
+        Load a certificate from a PEM-format data string.
+
+        @rtype: C{Class}
+        """
+        return Class.load(data, crypto.FILETYPE_PEM)
+
+    @classmethod
+    def peerFromTransport(Class, transport):
+        """
+        Get the certificate for the remote end of the given transport.
+
+        @param transport: an L{ISystemHandle} provider
+
+        @rtype: C{Class}
+
+        @raise CertificateError: if the given transport does not have a peer
+            certificate.
+        """
+        return _handleattrhelper(Class, transport, "peer")
+
+    @classmethod
+    def hostFromTransport(Class, transport):
+        """
+        Get the certificate for the local end of the given transport.
+
+        @param transport: an L{ISystemHandle} provider; the transport we will
+
+        @rtype: C{Class}
+
+        @raise CertificateError: if the given transport does not have a host
+            certificate.
+        """
+        return _handleattrhelper(Class, transport, "host")
+
+    def getPublicKey(self):
+        """
+        Get the public key for this certificate.
+
+        @rtype: L{PublicKey}
+        """
+        return PublicKey(self.original.get_pubkey())
+
+    def dump(self, format: int = crypto.FILETYPE_ASN1) -> bytes:
+        return crypto.dump_certificate(format, self.original)
+
+    def serialNumber(self):
+        """
+        Retrieve the serial number of this certificate.
+
+        @rtype: L{int}
+        """
+        return self.original.get_serial_number()
+
+    def digest(self, method="md5"):
+        """
+        Return a digest hash of this certificate using the specified hash
+        algorithm.
+
+        @param method: One of C{'md5'} or C{'sha'}.
+
+        @return: The digest of the object, formatted as b":"-delimited hex
+            pairs
+        @rtype: L{bytes}
+        """
+        return self.original.digest(method)
+
+    def _inspect(self):
+        return "\n".join(
+            [
+                "Certificate For Subject:",
+                self.getSubject().inspect(),
+                "\nIssuer:",
+                self.getIssuer().inspect(),
+                "\nSerial Number: %d" % self.serialNumber(),
+                "Digest: %s" % nativeString(self.digest()),
+            ]
+        )
+
+    def inspect(self):
+        """
+        Return a multi-line, human-readable representation of this
+        Certificate, including information about the subject, issuer, and
+        public key.
+        """
+        return "\n".join((self._inspect(), self.getPublicKey().inspect()))
+
+    def getIssuer(self):
+        """
+        Retrieve the issuer of this certificate.
+
+        @rtype: L{DistinguishedName}
+        @return: A copy of the issuer of this certificate.
+        """
+        return self._copyName("issuer")
+
+    def options(self, *authorities):
+        raise NotImplementedError("Possible, but doubtful we need this yet")
+
+
+class CertificateRequest(CertBase):
+    """
+    An x509 certificate request.
+
+    Certificate requests are given to certificate authorities to be signed and
+    returned resulting in an actual certificate.
+    """
+
+    @classmethod
+    def load(Class, requestData, requestFormat=crypto.FILETYPE_ASN1):
+        req = crypto.load_certificate_request(requestFormat, requestData)
+        dn = DistinguishedName()
+        dn._copyFrom(req.get_subject())
+        if not req.verify(req.get_pubkey()):
+            raise VerifyError(f"Can't verify that request for {dn!r} is self-signed.")
+        return Class(req)
+
+    def dump(self, format=crypto.FILETYPE_ASN1):
+        return crypto.dump_certificate_request(format, self.original)
+
+
+class PrivateCertificate(Certificate):
+    """
+    An x509 certificate and private key.
+    """
+
+    def __repr__(self) -> str:
+        return Certificate.__repr__(self) + " with " + repr(self.privateKey)
+
+    def _setPrivateKey(self, privateKey):
+        if not privateKey.matches(self.getPublicKey()):
+            raise VerifyError("Certificate public and private keys do not match.")
+        self.privateKey = privateKey
+        return self
+
+    def newCertificate(self, newCertData, format=crypto.FILETYPE_ASN1):
+        """
+        Create a new L{PrivateCertificate} from the given certificate data and
+        this instance's private key.
+        """
+        return self.load(newCertData, self.privateKey, format)
+
+    @classmethod
+    def load(Class, data, privateKey, format=crypto.FILETYPE_ASN1):
+        return Class._load(data, format)._setPrivateKey(privateKey)
+
+    def inspect(self):
+        return "\n".join([Certificate._inspect(self), self.privateKey.inspect()])
+
+    def dumpPEM(self):
+        """
+        Dump both public and private parts of a private certificate to
+        PEM-format data.
+        """
+        return self.dump(crypto.FILETYPE_PEM) + self.privateKey.dump(
+            crypto.FILETYPE_PEM
+        )
+
+    @classmethod
+    def loadPEM(Class, data):
+        """
+        Load both private and public parts of a private certificate from a
+        chunk of PEM-format data.
+        """
+        return Class.load(
+            data, KeyPair.load(data, crypto.FILETYPE_PEM), crypto.FILETYPE_PEM
+        )
+
+    @classmethod
+    def fromCertificateAndKeyPair(Class, certificateInstance, privateKey):
+        privcert = Class(certificateInstance.original)
+        return privcert._setPrivateKey(privateKey)
+
+    def options(self, *authorities):
+        """
+        Create a context factory using this L{PrivateCertificate}'s certificate
+        and private key.
+
+        @param authorities: A list of L{Certificate} object
+
+        @return: A context factory.
+        @rtype: L{CertificateOptions <twisted.internet.ssl.CertificateOptions>}
+        """
+        options = dict(privateKey=self.privateKey.original, certificate=self.original)
+        if authorities:
+            options.update(
+                dict(
+                    trustRoot=OpenSSLCertificateAuthorities(
+                        [auth.original for auth in authorities]
+                    )
+                )
+            )
+        return OpenSSLCertificateOptions(**options)
+
+    def certificateRequest(self, format=crypto.FILETYPE_ASN1, digestAlgorithm="sha256"):
+        return self.privateKey.certificateRequest(
+            self.getSubject(), format, digestAlgorithm
+        )
+
+    def signCertificateRequest(
+        self,
+        requestData,
+        verifyDNCallback,
+        serialNumber,
+        requestFormat=crypto.FILETYPE_ASN1,
+        certificateFormat=crypto.FILETYPE_ASN1,
+    ):
+        issuer = self.getSubject()
+        return self.privateKey.signCertificateRequest(
+            issuer,
+            requestData,
+            verifyDNCallback,
+            serialNumber,
+            requestFormat,
+            certificateFormat,
+        )
+
+    def signRequestObject(
+        self,
+        certificateRequest,
+        serialNumber,
+        secondsToExpiry=60 * 60 * 24 * 365,  # One year
+        digestAlgorithm="sha256",
+    ):
+        return self.privateKey.signRequestObject(
+            self.getSubject(),
+            certificateRequest,
+            serialNumber,
+            secondsToExpiry,
+            digestAlgorithm,
+        )
+
+
+class PublicKey:
+    """
+    A L{PublicKey} is a representation of the public part of a key pair.
+
+    You can't do a whole lot with it aside from comparing it to other
+    L{PublicKey} objects.
+
+    @note: If constructing a L{PublicKey} manually, be sure to pass only a
+        L{OpenSSL.crypto.PKey} that does not contain a private key!
+
+    @ivar original: The original private key.
+    """
+
+    def __init__(self, osslpkey):
+        """
+        @param osslpkey: The underlying pyOpenSSL key object.
+        @type osslpkey: L{OpenSSL.crypto.PKey}
+        """
+        self.original = osslpkey
+
+    def matches(self, otherKey):
+        """
+        Does this L{PublicKey} contain the same value as another L{PublicKey}?
+
+        @param otherKey: The key to compare C{self} to.
+        @type otherKey: L{PublicKey}
+
+        @return: L{True} if these keys match, L{False} if not.
+        @rtype: L{bool}
+        """
+        return self.keyHash() == otherKey.keyHash()
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.keyHash()}>"
+
+    def keyHash(self):
+        """
+        Compute a hash of the underlying PKey object.
+
+        The purpose of this method is to allow you to determine if two
+        certificates share the same public key; it is not really useful for
+        anything else.
+
+        In versions of Twisted prior to 15.0, C{keyHash} used a technique
+        involving certificate requests for computing the hash that was not
+        stable in the face of changes to the underlying OpenSSL library.
+
+        @return: Return a 32-character hexadecimal string uniquely identifying
+            this public key, I{for this version of Twisted}.
+        @rtype: native L{str}
+        """
+        raw = crypto.dump_publickey(crypto.FILETYPE_ASN1, self.original)
+        h = md5()
+        h.update(raw)
+        return h.hexdigest()
+
+    def inspect(self):
+        return f"Public Key with Hash: {self.keyHash()}"
+
+
+class KeyPair(PublicKey):
+    @classmethod
+    def load(Class, data, format=crypto.FILETYPE_ASN1):
+        return Class(crypto.load_privatekey(format, data))
+
+    def dump(self, format=crypto.FILETYPE_ASN1):
+        return crypto.dump_privatekey(format, self.original)
+
+    @deprecated(Version("Twisted", 15, 0, 0), "a real persistence system")
+    def __getstate__(self):
+        return self.dump()
+
+    @deprecated(Version("Twisted", 15, 0, 0), "a real persistence system")
+    def __setstate__(self, state):
+        self.__init__(crypto.load_privatekey(crypto.FILETYPE_ASN1, state))
+
+    def inspect(self):
+        t = self.original.type()
+        if t == crypto.TYPE_RSA:
+            ts = "RSA"
+        elif t == crypto.TYPE_DSA:
+            ts = "DSA"
+        else:
+            ts = "(Unknown Type!)"
+        L = (self.original.bits(), ts, self.keyHash())
+        return "%s-bit %s Key Pair with Hash: %s" % L
+
+    @classmethod
+    def generate(Class, kind=crypto.TYPE_RSA, size=2048):
+        pkey = crypto.PKey()
+        pkey.generate_key(kind, size)
+        return Class(pkey)
+
+    def newCertificate(self, newCertData, format=crypto.FILETYPE_ASN1):
+        return PrivateCertificate.load(newCertData, self, format)
+
+    def requestObject(self, distinguishedName, digestAlgorithm="sha256"):
+        req = crypto.X509Req()
+        req.set_pubkey(self.original)
+        distinguishedName._copyInto(req.get_subject())
+        req.sign(self.original, digestAlgorithm)
+        return CertificateRequest(req)
+
+    def certificateRequest(
+        self, distinguishedName, format=crypto.FILETYPE_ASN1, digestAlgorithm="sha256"
+    ):
+        """
+        Create a certificate request signed with this key.
+
+        @return: a string, formatted according to the 'format' argument.
+        """
+        return self.requestObject(distinguishedName, digestAlgorithm).dump(format)
+
+    def signCertificateRequest(
+        self,
+        issuerDistinguishedName,
+        requestData,
+        verifyDNCallback,
+        serialNumber,
+        requestFormat=crypto.FILETYPE_ASN1,
+        certificateFormat=crypto.FILETYPE_ASN1,
+        secondsToExpiry=60 * 60 * 24 * 365,  # One year
+        digestAlgorithm="sha256",
+    ):
+        """
+        Given a blob of certificate request data and a certificate authority's
+        DistinguishedName, return a blob of signed certificate data.
+
+        If verifyDNCallback returns a Deferred, I will return a Deferred which
+        fires the data when that Deferred has completed.
+        """
+        hlreq = CertificateRequest.load(requestData, requestFormat)
+
+        dn = hlreq.getSubject()
+        vval = verifyDNCallback(dn)
+
+        def verified(value):
+            if not value:
+                raise VerifyError(
+                    "DN callback {!r} rejected request DN {!r}".format(
+                        verifyDNCallback, dn
+                    )
+                )
+            return self.signRequestObject(
+                issuerDistinguishedName,
+                hlreq,
+                serialNumber,
+                secondsToExpiry,
+                digestAlgorithm,
+            ).dump(certificateFormat)
+
+        if isinstance(vval, Deferred):
+            return vval.addCallback(verified)
+        else:
+            return verified(vval)
+
+    def signRequestObject(
+        self,
+        issuerDistinguishedName,
+        requestObject,
+        serialNumber,
+        secondsToExpiry=60 * 60 * 24 * 365,  # One year
+        digestAlgorithm="sha256",
+    ):
+        """
+        Sign a CertificateRequest instance, returning a Certificate instance.
+        """
+        req = requestObject.original
+        cert = crypto.X509()
+        issuerDistinguishedName._copyInto(cert.get_issuer())
+        cert.set_subject(req.get_subject())
+        cert.set_pubkey(req.get_pubkey())
+        cert.gmtime_adj_notBefore(0)
+        cert.gmtime_adj_notAfter(secondsToExpiry)
+        cert.set_serial_number(serialNumber)
+        cert.sign(self.original, digestAlgorithm)
+        return Certificate(cert)
+
+    def selfSignedCert(self, serialNumber, **kw):
+        dn = DN(**kw)
+        return PrivateCertificate.fromCertificateAndKeyPair(
+            self.signRequestObject(dn, self.requestObject(dn), serialNumber), self
+        )
+
+
+class IOpenSSLTrustRoot(Interface):
+    """
+    Trust settings for an OpenSSL context.
+
+    Note that this interface's methods are private, so things outside of
+    Twisted shouldn't implement it.
+    """
+
+    def _addCACertsToContext(context):
+        """
+        Add certificate-authority certificates to an SSL context whose
+        connections should trust those authorities.
+
+        @param context: An SSL context for a connection which should be
+            verified by some certificate authority.
+        @type context: L{OpenSSL.SSL.Context}
+
+        @return: L{None}
+        """
+
+
+@implementer(IOpenSSLTrustRoot)
+class OpenSSLCertificateAuthorities:
+    """
+    Trust an explicitly specified set of certificates, represented by a list of
+    L{OpenSSL.crypto.X509} objects.
+    """
+
+    def __init__(self, caCerts):
+        """
+        @param caCerts: The certificate authorities to trust when using this
+            object as a C{trustRoot} for L{OpenSSLCertificateOptions}.
+        @type caCerts: L{list} of L{OpenSSL.crypto.X509}
+        """
+        self._caCerts = caCerts
+
+    def _addCACertsToContext(self, context):
+        store = context.get_cert_store()
+        for cert in self._caCerts:
+            store.add_cert(cert)
+
+
+def trustRootFromCertificates(certificates):
+    """
+    Builds an object that trusts multiple root L{Certificate}s.
+
+    When passed to L{optionsForClientTLS}, connections using those options will
+    reject any server certificate not signed by at least one of the
+    certificates in the `certificates` list.
+
+    @since: 16.0
+
+    @param certificates: All certificates which will be trusted.
+    @type certificates: C{iterable} of L{CertBase}
+
+    @rtype: L{IOpenSSLTrustRoot}
+    @return: an object suitable for use as the trustRoot= keyword argument to
+        L{optionsForClientTLS}
+    """
+
+    certs = []
+    for cert in certificates:
+        # PrivateCertificate or Certificate are both okay
+        if isinstance(cert, CertBase):
+            cert = cert.original
+        else:
+            raise TypeError(
+                "certificates items must be twisted.internet.ssl.CertBase" " instances"
+            )
+        certs.append(cert)
+    return OpenSSLCertificateAuthorities(certs)
+
+
+@implementer(IOpenSSLTrustRoot)
+class OpenSSLDefaultPaths:
+    """
+    Trust the set of default verify paths that OpenSSL was built with, as
+    specified by U{SSL_CTX_set_default_verify_paths
+    <https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_load_verify_locations.html>}.
+    """
+
+    def _addCACertsToContext(self, context):
+        context.set_default_verify_paths()
+
+
+def platformTrust():
+    """
+    Attempt to discover a set of trusted certificate authority certificates
+    (or, in other words: trust roots, or root certificates) whose trust is
+    managed and updated by tools outside of Twisted.
+
+    If you are writing any client-side TLS code with Twisted, you should use
+    this as the C{trustRoot} argument to L{CertificateOptions
+    <twisted.internet.ssl.CertificateOptions>}.
+
+    The result of this function should be like the up-to-date list of
+    certificates in a web browser.  When developing code that uses
+    C{platformTrust}, you can think of it that way.  However, the choice of
+    which certificate authorities to trust is never Twisted's responsibility.
+    Unless you're writing a very unusual application or library, it's not your
+    code's responsibility either.  The user may use platform-specific tools for
+    defining which server certificates should be trusted by programs using TLS.
+    The purpose of using this API is to respect that decision as much as
+    possible.
+
+    This should be a set of trust settings most appropriate for I{client} TLS
+    connections; i.e. those which need to verify a server's authenticity.  You
+    should probably use this by default for any client TLS connection that you
+    create.  For servers, however, client certificates are typically not
+    verified; or, if they are, their verification will depend on a custom,
+    application-specific certificate authority.
+
+    @since: 14.0
+
+    @note: Currently, L{platformTrust} depends entirely upon your OpenSSL build
+        supporting a set of "L{default verify paths <OpenSSLDefaultPaths>}"
+        which correspond to certificate authority trust roots.  Unfortunately,
+        whether this is true of your system is both outside of Twisted's
+        control and difficult (if not impossible) for Twisted to detect
+        automatically.
+
+        Nevertheless, this ought to work as desired by default on:
+
+            - Ubuntu Linux machines with the U{ca-certificates
+              <https://launchpad.net/ubuntu/+source/ca-certificates>} package
+              installed,
+
+            - macOS when using the system-installed version of OpenSSL (i.e.
+              I{not} one installed via MacPorts or Homebrew),
+
+            - any build of OpenSSL which has had certificate authority
+              certificates installed into its default verify paths (by default,
+              C{/usr/local/ssl/certs} if you've built your own OpenSSL), or
+
+            - any process where the C{SSL_CERT_FILE} environment variable is
+              set to the path of a file containing your desired CA certificates
+              bundle.
+
+        Hopefully soon, this API will be updated to use more sophisticated
+        trust-root discovery mechanisms.  Until then, you can follow tickets in
+        the Twisted tracker for progress on this implementation on U{Microsoft
+        Windows <https://twistedmatrix.com/trac/ticket/6371>}, U{macOS
+        <https://twistedmatrix.com/trac/ticket/6372>}, and U{a fallback for
+        other platforms which do not have native trust management tools
+        <https://twistedmatrix.com/trac/ticket/6934>}.
+
+    @return: an appropriate trust settings object for your platform.
+    @rtype: L{IOpenSSLTrustRoot}
+
+    @raise NotImplementedError: if this platform is not yet supported by
+        Twisted.  At present, only OpenSSL is supported.
+    """
+    return OpenSSLDefaultPaths()
+
+
+def _tolerateErrors(wrapped):
+    """
+    Wrap up an C{info_callback} for pyOpenSSL so that if something goes wrong
+    the error is immediately logged and the connection is dropped if possible.
+
+    This wrapper exists because some versions of pyOpenSSL don't handle errors
+    from callbacks at I{all}, and those which do write tracebacks directly to
+    stderr rather than to a supplied logging system.  This reports unexpected
+    errors to the Twisted logging system.
+
+    Also, this terminates the connection immediately if possible because if
+    you've got bugs in your verification logic it's much safer to just give up.
+
+    @param wrapped: A valid C{info_callback} for pyOpenSSL.
+    @type wrapped: L{callable}
+
+    @return: A valid C{info_callback} for pyOpenSSL that handles any errors in
+        C{wrapped}.
+    @rtype: L{callable}
+    """
+
+    def infoCallback(connection, where, ret):
+        try:
+            return wrapped(connection, where, ret)
+        except BaseException:
+            f = Failure()
+            log.err(f, "Error during info_callback")
+            connection.get_app_data().failVerification(f)
+
+    return infoCallback
+
+
+@implementer(IOpenSSLClientConnectionCreator)
+class ClientTLSOptions:
+    """
+    Client creator for TLS.
+
+    Private implementation type (not exposed to applications) for public
+    L{optionsForClientTLS} API.
+
+    @ivar _ctx: The context to use for new connections.
+    @type _ctx: L{OpenSSL.SSL.Context}
+
+    @ivar _hostname: The hostname to verify, as specified by the application,
+        as some human-readable text.
+    @type _hostname: L{unicode}
+
+    @ivar _hostnameBytes: The hostname to verify, decoded into IDNA-encoded
+        bytes.  This is passed to APIs which think that hostnames are bytes,
+        such as OpenSSL's SNI implementation.
+    @type _hostnameBytes: L{bytes}
+
+    @ivar _hostnameASCII: The hostname, as transcoded into IDNA ASCII-range
+        unicode code points.  This is pre-transcoded because the
+        C{service_identity} package is rather strict about requiring the
+        C{idna} package from PyPI for internationalized domain names, rather
+        than working with Python's built-in (but sometimes broken) IDNA
+        encoding.  ASCII values, however, will always work.
+    @type _hostnameASCII: L{unicode}
+
+    @ivar _hostnameIsDnsName: Whether or not the C{_hostname} is a DNSName.
+        Will be L{False} if C{_hostname} is an IP address or L{True} if
+        C{_hostname} is a DNSName
+    @type _hostnameIsDnsName: L{bool}
+    """
+
+    def __init__(self, hostname, ctx):
+        """
+        Initialize L{ClientTLSOptions}.
+
+        @param hostname: The hostname to verify as input by a human.
+        @type hostname: L{unicode}
+
+        @param ctx: an L{OpenSSL.SSL.Context} to use for new connections.
+        @type ctx: L{OpenSSL.SSL.Context}.
+        """
+        self._ctx = ctx
+        self._hostname = hostname
+
+        if isIPAddress(hostname) or isIPv6Address(hostname):
+            self._hostnameBytes = hostname.encode("ascii")
+            self._hostnameIsDnsName = False
+        else:
+            self._hostnameBytes = _idnaBytes(hostname)
+            self._hostnameIsDnsName = True
+
+        self._hostnameASCII = self._hostnameBytes.decode("ascii")
+        ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
+
+    def clientConnectionForTLS(self, tlsProtocol):
+        """
+        Create a TLS connection for a client.
+
+        @note: This will call C{set_app_data} on its connection.  If you're
+            delegating to this implementation of this method, don't ever call
+            C{set_app_data} or C{set_info_callback} on the returned connection,
+            or you'll break the implementation of various features of this
+            class.
+
+        @param tlsProtocol: the TLS protocol initiating the connection.
+        @type tlsProtocol: L{twisted.protocols.tls.TLSMemoryBIOProtocol}
+
+        @return: the configured client connection.
+        @rtype: L{OpenSSL.SSL.Connection}
+        """
+        context = self._ctx
+        connection = SSL.Connection(context, None)
+        connection.set_app_data(tlsProtocol)
+        return connection
+
+    def _identityVerifyingInfoCallback(self, connection, where, ret):
+        """
+        U{info_callback
+        <http://pythonhosted.org/pyOpenSSL/api/ssl.html#OpenSSL.SSL.Context.set_info_callback>
+        } for pyOpenSSL that verifies the hostname in the presented certificate
+        matches the one passed to this L{ClientTLSOptions}.
+
+        @param connection: the connection which is handshaking.
+        @type connection: L{OpenSSL.SSL.Connection}
+
+        @param where: flags indicating progress through a TLS handshake.
+        @type where: L{int}
+
+        @param ret: ignored
+        @type ret: ignored
+        """
+        # Literal IPv4 and IPv6 addresses are not permitted
+        # as host names according to the RFCs
+        if where & SSL.SSL_CB_HANDSHAKE_START and self._hostnameIsDnsName:
+            connection.set_tlsext_host_name(self._hostnameBytes)
+        elif where & SSL.SSL_CB_HANDSHAKE_DONE:
+            try:
+                if self._hostnameIsDnsName:
+                    verifyHostname(connection, self._hostnameASCII)
+                else:
+                    verifyIPAddress(connection, self._hostnameASCII)
+            except VerificationError:
+                f = Failure()
+                transport = connection.get_app_data()
+                transport.failVerification(f)
+
+
+def optionsForClientTLS(
+    hostname,
+    trustRoot=None,
+    clientCertificate=None,
+    acceptableProtocols=None,
+    *,
+    extraCertificateOptions=None,
+):
+    """
+    Create a L{client connection creator <IOpenSSLClientConnectionCreator>} for
+    use with APIs such as L{SSL4ClientEndpoint
+    <twisted.internet.endpoints.SSL4ClientEndpoint>}, L{connectSSL
+    <twisted.internet.interfaces.IReactorSSL.connectSSL>}, and L{startTLS
+    <twisted.internet.interfaces.ITLSTransport.startTLS>}.
+
+    @since: 14.0
+
+    @param hostname: The expected name of the remote host. This serves two
+        purposes: first, and most importantly, it verifies that the certificate
+        received from the server correctly identifies the specified hostname.
+        The second purpose is to use the U{Server Name Indication extension
+        <https://en.wikipedia.org/wiki/Server_Name_Indication>} to indicate to
+        the server which certificate should be used.
+    @type hostname: L{unicode}
+
+    @param trustRoot: Specification of trust requirements of peers. This may be
+        a L{Certificate} or the result of L{platformTrust}. By default it is
+        L{platformTrust} and you probably shouldn't adjust it unless you really
+        know what you're doing. Be aware that clients using this interface
+        I{must} verify the server; you cannot explicitly pass L{None} since
+        that just means to use L{platformTrust}.
+    @type trustRoot: L{IOpenSSLTrustRoot}
+
+    @param clientCertificate: The certificate and private key that the client
+        will use to authenticate to the server. If unspecified, the client will
+        not authenticate.
+    @type clientCertificate: L{PrivateCertificate}
+
+    @param acceptableProtocols: The protocols this peer is willing to speak
+        after the TLS negotiation has completed, advertised over both ALPN and
+        NPN. If this argument is specified, and no overlap can be found with
+        the other peer, the connection will fail to be established. If the
+        remote peer does not offer NPN or ALPN, the connection will be
+        established, but no protocol wil be negotiated. Protocols earlier in
+        the list are preferred over those later in the list.
+    @type acceptableProtocols: L{list} of L{bytes}
+
+    @param extraCertificateOptions: A dictionary of additional keyword arguments
+        to be presented to L{CertificateOptions}. Please avoid using this unless
+        you absolutely need to; any time you need to pass an option here that is
+        a bug in this interface.
+    @type extraCertificateOptions: L{dict}
+
+    @return: A client connection creator.
+    @rtype: L{IOpenSSLClientConnectionCreator}
+    """
+    if extraCertificateOptions is None:
+        extraCertificateOptions = {}
+    if trustRoot is None:
+        trustRoot = platformTrust()
+    if not isinstance(hostname, str):
+        raise TypeError(
+            "optionsForClientTLS requires text for host names, not "
+            + hostname.__class__.__name__
+        )
+    if clientCertificate:
+        extraCertificateOptions.update(
+            privateKey=clientCertificate.privateKey.original,
+            certificate=clientCertificate.original,
+        )
+    certificateOptions = OpenSSLCertificateOptions(
+        trustRoot=trustRoot,
+        acceptableProtocols=acceptableProtocols,
+        **extraCertificateOptions,
+    )
+    return ClientTLSOptions(hostname, certificateOptions.getContext())
+
+
+@implementer(IOpenSSLContextFactory)
+class OpenSSLCertificateOptions:
+    """
+    A L{CertificateOptions <twisted.internet.ssl.CertificateOptions>} specifies
+    the security properties for a client or server TLS connection used with
+    OpenSSL.
+
+    @ivar _options: Any option flags to set on the L{OpenSSL.SSL.Context}
+        object that will be created.
+    @type _options: L{int}
+
+    @ivar _cipherString: An OpenSSL-specific cipher string.
+    @type _cipherString: L{unicode}
+
+    @ivar _defaultMinimumTLSVersion: The default TLS version that will be
+        negotiated.  This should be a "safe default", with wide client and
+        server support, vs an optimally secure one that excludes a large number
+        of users.  As of May 2022, TLSv1.2 is that safe default.
+    @type _defaultMinimumTLSVersion: L{TLSVersion} constant
+    """
+
+    # Factory for creating contexts.  Configurable for testability.
+    _contextFactory = SSL.Context
+    _context = None
+
+    _OP_NO_TLSv1_3 = _tlsDisableFlags[TLSVersion.TLSv1_3]
+
+    _defaultMinimumTLSVersion = TLSVersion.TLSv1_2
+
+    @_mutuallyExclusiveArguments(
+        [
+            ["trustRoot", "requireCertificate"],
+            ["trustRoot", "verify"],
+            ["trustRoot", "caCerts"],
+            ["method", "insecurelyLowerMinimumTo"],
+            ["method", "raiseMinimumTo"],
+            ["raiseMinimumTo", "insecurelyLowerMinimumTo"],
+            ["method", "lowerMaximumSecurityTo"],
+        ]
+    )
+    def __init__(
+        self,
+        privateKey=None,
+        certificate=None,
+        method=None,
+        verify=False,
+        caCerts=None,
+        verifyDepth=9,
+        requireCertificate=True,
+        verifyOnce=True,
+        enableSingleUseKeys=True,
+        enableSessions=False,
+        fixBrokenPeers=False,
+        enableSessionTickets=False,
+        extraCertChain=None,
+        acceptableCiphers=None,
+        dhParameters=None,
+        trustRoot=None,
+        acceptableProtocols=None,
+        raiseMinimumTo=None,
+        insecurelyLowerMinimumTo=None,
+        lowerMaximumSecurityTo=None,
+    ):
+        """
+        Create an OpenSSL context SSL connection context factory.
+
+        @param privateKey: A PKey object holding the private key.
+
+        @param certificate: An X509 object holding the certificate.
+
+        @param method: Deprecated, use a combination of
+            C{insecurelyLowerMinimumTo}, C{raiseMinimumTo}, or
+            C{lowerMaximumSecurityTo} instead.  The SSL protocol to use, one of
+            C{TLS_METHOD}, C{TLSv1_2_METHOD}, or C{TLSv1_2_METHOD} (or any
+            future method constants provided by pyOpenSSL).  By default, a
+            setting will be used which allows TLSv1.2 and TLSv1.3.  Can not be
+            used with C{insecurelyLowerMinimumTo}, C{raiseMinimumTo}, or
+            C{lowerMaximumSecurityTo}.
+
+        @param verify: Please use a C{trustRoot} keyword argument instead,
+            since it provides the same functionality in a less error-prone way.
+            By default this is L{False}.
+
+            If L{True}, verify certificates received from the peer and fail the
+            handshake if verification fails.  Otherwise, allow anonymous
+            sessions and sessions with certificates which fail validation.
+
+        @param caCerts: Please use a C{trustRoot} keyword argument instead,
+            since it provides the same functionality in a less error-prone way.
+
+            List of certificate authority certificate objects to use to verify
+            the peer's certificate.  Only used if verify is L{True} and will be
+            ignored otherwise.  Since verify is L{False} by default, this is
+            L{None} by default.
+
+        @type caCerts: L{list} of L{OpenSSL.crypto.X509}
+
+        @param verifyDepth: Depth in certificate chain down to which to verify.
+            If unspecified, use the underlying default (9).
+
+        @param requireCertificate: Please use a C{trustRoot} keyword argument
+            instead, since it provides the same functionality in a less
+            error-prone way.
+
+            If L{True}, do not allow anonymous sessions; defaults to L{True}.
+
+        @param verifyOnce: If True, do not re-verify the certificate on session
+            resumption.
+
+        @param enableSingleUseKeys: If L{True}, generate a new key whenever
+            ephemeral DH and ECDH parameters are used to prevent small subgroup
+            attacks and to ensure perfect forward secrecy.
+
+        @param enableSessions: This allows a shortened handshake to be used
+            when a known client reconnects to the same process.  If True,
+            enable OpenSSL's session caching.  Note that session caching only
+            works on a single Twisted node at once.  Also, it is currently
+            somewhat risky due to U{a crashing bug when using OpenSSL 1.1.1
+            <https://twistedmatrix.com/trac/ticket/9764>}.
+
+        @param fixBrokenPeers: If True, enable various non-spec protocol fixes
+            for broken SSL implementations.  This should be entirely safe,
+            according to the OpenSSL documentation, but YMMV.  This option is
+            now off by default, because it causes problems with connections
+            between peers using OpenSSL 0.9.8a.
+
+        @param enableSessionTickets: If L{True}, enable session ticket
+            extension for session resumption per RFC 5077.  Note there is no
+            support for controlling session tickets.  This option is off by
+            default, as some server implementations don't correctly process
+            incoming empty session ticket extensions in the hello.
+
+        @param extraCertChain: List of certificates that I{complete} your
+            verification chain if the certificate authority that signed your
+            C{certificate} isn't widely supported.  Do I{not} add
+            C{certificate} to it.
+        @type extraCertChain: C{list} of L{OpenSSL.crypto.X509}
+
+        @param acceptableCiphers: Ciphers that are acceptable for connections.
+            Uses a secure default if left L{None}.
+        @type acceptableCiphers: L{IAcceptableCiphers}
+
+        @param dhParameters: Key generation parameters that are required for
+            Diffie-Hellman key exchange.  If this argument is left L{None},
+            C{EDH} ciphers are I{disabled} regardless of C{acceptableCiphers}.
+        @type dhParameters: L{DiffieHellmanParameters
+            <twisted.internet.ssl.DiffieHellmanParameters>}
+
+        @param trustRoot: Specification of trust requirements of peers.  If
+            this argument is specified, the peer is verified.  It requires a
+            certificate, and that certificate must be signed by one of the
+            certificate authorities specified by this object.
+
+            Note that since this option specifies the same information as
+            C{caCerts}, C{verify}, and C{requireCertificate}, specifying any of
+            those options in combination with this one will raise a
+            L{TypeError}.
+
+        @type trustRoot: L{IOpenSSLTrustRoot}
+
+        @param acceptableProtocols: The protocols this peer is willing to speak
+            after the TLS negotiation has completed, advertised over both ALPN
+            and NPN.  If this argument is specified, and no overlap can be
+            found with the other peer, the connection will fail to be
+            established.  If the remote peer does not offer NPN or ALPN, the
+            connection will be established, but no protocol wil be negotiated.
+            Protocols earlier in the list are preferred over those later in the
+            list.
+        @type acceptableProtocols: L{list} of L{bytes}
+
+        @param raiseMinimumTo: The minimum TLS version that you want to use, or
+            Twisted's default if it is higher.  Use this if you want to make
+            your client/server more secure than Twisted's default, but will
+            accept Twisted's default instead if it moves higher than this
+            value.  You probably want to use this over
+            C{insecurelyLowerMinimumTo}.
+        @type raiseMinimumTo: L{TLSVersion} constant
+
+        @param insecurelyLowerMinimumTo: The minimum TLS version to use,
+            possibly lower than Twisted's default.  If not specified, it is a
+            generally considered safe default (TLSv1.0).  If you want to raise
+            your minimum TLS version to above that of this default, use
+            C{raiseMinimumTo}.  DO NOT use this argument unless you are
+            absolutely sure this is what you want.
+        @type insecurelyLowerMinimumTo: L{TLSVersion} constant
+
+        @param lowerMaximumSecurityTo: The maximum TLS version to use.  If not
+            specified, it is the most recent your OpenSSL supports.  You only
+            want to set this if the peer that you are communicating with has
+            problems with more recent TLS versions, it lowers your security
+            when communicating with newer peers.  DO NOT use this argument
+            unless you are absolutely sure this is what you want.
+        @type lowerMaximumSecurityTo: L{TLSVersion} constant
+
+        @raise ValueError: when C{privateKey} or C{certificate} are set without
+            setting the respective other.
+        @raise ValueError: when C{verify} is L{True} but C{caCerts} doesn't
+            specify any CA certificates.
+        @raise ValueError: when C{extraCertChain} is passed without specifying
+            C{privateKey} or C{certificate}.
+        @raise ValueError: when C{acceptableCiphers} doesn't yield any usable
+            ciphers for the current platform.
+
+        @raise TypeError: if C{trustRoot} is passed in combination with
+            C{caCert}, C{verify}, or C{requireCertificate}.  Please prefer
+            C{trustRoot} in new code, as its semantics are less tricky.
+        @raise TypeError: if C{method} is passed in combination with
+            C{tlsProtocols}.  Please prefer the more explicit C{tlsProtocols}
+            in new code.
+
+        @raises NotImplementedError: If acceptableProtocols were provided but
+            no negotiation mechanism is available.
+        """
+
+        if (privateKey is None) != (certificate is None):
+            raise ValueError("Specify neither or both of privateKey and certificate")
+        self.privateKey = privateKey
+        self.certificate = certificate
+
+        # Set basic security options: disallow insecure SSLv2, disallow TLS
+        # compression to avoid CRIME attack, make the server choose the
+        # ciphers.
+        self._options = (
+            SSL.OP_NO_SSLv2 | SSL.OP_NO_COMPRESSION | SSL.OP_CIPHER_SERVER_PREFERENCE
+        )
+
+        # Set the mode to Release Buffers, which demallocs send/recv buffers on
+        # idle TLS connections to save memory
+        self._mode = SSL.MODE_RELEASE_BUFFERS
+
+        if method is None:
+            self.method = SSL.TLS_METHOD
+
+            if raiseMinimumTo:
+                if lowerMaximumSecurityTo and raiseMinimumTo > lowerMaximumSecurityTo:
+                    raise ValueError(
+                        "raiseMinimumTo needs to be lower than "
+                        "lowerMaximumSecurityTo"
+                    )
+
+                if raiseMinimumTo > self._defaultMinimumTLSVersion:
+                    insecurelyLowerMinimumTo = raiseMinimumTo
+
+            if insecurelyLowerMinimumTo is None:
+                insecurelyLowerMinimumTo = self._defaultMinimumTLSVersion
+
+                # If you set the max lower than the default, but don't set the
+                # minimum, pull it down to that
+                if (
+                    lowerMaximumSecurityTo
+                    and insecurelyLowerMinimumTo > lowerMaximumSecurityTo
+                ):
+                    insecurelyLowerMinimumTo = lowerMaximumSecurityTo
+
+            if (
+                lowerMaximumSecurityTo
+                and insecurelyLowerMinimumTo > lowerMaximumSecurityTo
+            ):
+                raise ValueError(
+                    "insecurelyLowerMinimumTo needs to be lower than "
+                    "lowerMaximumSecurityTo"
+                )
+
+            excludedVersions = _getExcludedTLSProtocols(
+                insecurelyLowerMinimumTo, lowerMaximumSecurityTo
+            )
+
+            for version in excludedVersions:
+                self._options |= _tlsDisableFlags[version]
+        else:
+            warnings.warn(
+                (
+                    "Passing method to twisted.internet.ssl.CertificateOptions "
+                    "was deprecated in Twisted 17.1.0. Please use a combination "
+                    "of insecurelyLowerMinimumTo, raiseMinimumTo, and "
+                    "lowerMaximumSecurityTo instead, as Twisted will correctly "
+                    "configure the method."
+                ),
+                DeprecationWarning,
+                stacklevel=3,
+            )
+
+            # Otherwise respect the application decision.
+            self.method = method
+
+        if verify and not caCerts:
+            raise ValueError(
+                "Specify client CA certificate information if and"
+                " only if enabling certificate verification"
+            )
+        self.verify = verify
+        if extraCertChain is not None and None in (privateKey, certificate):
+            raise ValueError(
+                "A private key and a certificate are required "
+                "when adding a supplemental certificate chain."
+            )
+        if extraCertChain is not None:
+            self.extraCertChain = extraCertChain
+        else:
+            self.extraCertChain = []
+
+        self.caCerts = caCerts
+        self.verifyDepth = verifyDepth
+        self.requireCertificate = requireCertificate
+        self.verifyOnce = verifyOnce
+        self.enableSingleUseKeys = enableSingleUseKeys
+        if enableSingleUseKeys:
+            self._options |= SSL.OP_SINGLE_DH_USE | SSL.OP_SINGLE_ECDH_USE
+        self.enableSessions = enableSessions
+        self.fixBrokenPeers = fixBrokenPeers
+        if fixBrokenPeers:
+            self._options |= SSL.OP_ALL
+        self.enableSessionTickets = enableSessionTickets
+
+        if not enableSessionTickets:
+            self._options |= SSL.OP_NO_TICKET
+        self.dhParameters = dhParameters
+
+        self._ecChooser = _ChooseDiffieHellmanEllipticCurve(
+            SSL.OPENSSL_VERSION_NUMBER,
+            openSSLlib=pyOpenSSLlib,
+            openSSLcrypto=crypto,
+        )
+
+        if acceptableCiphers is None:
+            acceptableCiphers = defaultCiphers
+        # This needs to run when method and _options are finalized.
+        self._cipherString = ":".join(
+            c.fullName
+            for c in acceptableCiphers.selectCiphers(
+                _expandCipherString("ALL", self.method, self._options)
+            )
+        )
+        if self._cipherString == "":
+            raise ValueError(
+                "Supplied IAcceptableCiphers yielded no usable ciphers "
+                "on this platform."
+            )
+
+        if trustRoot is None:
+            if self.verify:
+                trustRoot = OpenSSLCertificateAuthorities(caCerts)
+        else:
+            self.verify = True
+            self.requireCertificate = True
+            trustRoot = IOpenSSLTrustRoot(trustRoot)
+        self.trustRoot = trustRoot
+
+        if acceptableProtocols is not None and not protocolNegotiationMechanisms():
+            raise NotImplementedError(
+                "No support for protocol negotiation on this platform."
+            )
+
+        self._acceptableProtocols = acceptableProtocols
+
+    def __getstate__(self):
+        d = self.__dict__.copy()
+        try:
+            del d["_context"]
+        except KeyError:
+            pass
+        return d
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+
+    def getContext(self):
+        """
+        Return an L{OpenSSL.SSL.Context} object.
+        """
+        if self._context is None:
+            self._context = self._makeContext()
+        return self._context
+
+    def _makeContext(self):
+        ctx = self._contextFactory(self.method)
+        ctx.set_options(self._options)
+        ctx.set_mode(self._mode)
+
+        if self.certificate is not None and self.privateKey is not None:
+            ctx.use_certificate(self.certificate)
+            ctx.use_privatekey(self.privateKey)
+            for extraCert in self.extraCertChain:
+                ctx.add_extra_chain_cert(extraCert)
+            # Sanity check
+            ctx.check_privatekey()
+
+        verifyFlags = SSL.VERIFY_NONE
+        if self.verify:
+            verifyFlags = SSL.VERIFY_PEER
+            if self.requireCertificate:
+                verifyFlags |= SSL.VERIFY_FAIL_IF_NO_PEER_CERT
+            if self.verifyOnce:
+                verifyFlags |= SSL.VERIFY_CLIENT_ONCE
+            self.trustRoot._addCACertsToContext(ctx)
+
+        ctx.set_verify(verifyFlags)
+        if self.verifyDepth is not None:
+            ctx.set_verify_depth(self.verifyDepth)
+
+        # Until we know what's going on with
+        # https://twistedmatrix.com/trac/ticket/9764 let's be conservative
+        # in naming this; ASCII-only, short, as the recommended value (a
+        # hostname) might be:
+        sessionIDContext = hexlify(secureRandom(7))
+        # Note that this doesn't actually set the session ID (which had
+        # better be per-connection anyway!):
+        # https://github.com/pyca/pyopenssl/issues/845
+
+        # This is set unconditionally because it's apparently required for
+        # client certificates to work:
+        # https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_session_id_context.html
+        ctx.set_session_id(sessionIDContext)
+
+        if self.enableSessions:
+            ctx.set_session_cache_mode(SSL.SESS_CACHE_SERVER)
+        else:
+            ctx.set_session_cache_mode(SSL.SESS_CACHE_OFF)
+
+        if self.dhParameters:
+            ctx.load_tmp_dh(self.dhParameters._dhFile.path)
+        ctx.set_cipher_list(self._cipherString.encode("ascii"))
+
+        self._ecChooser.configureECDHCurve(ctx)
+
+        if self._acceptableProtocols:
+            # Try to set NPN and ALPN. _acceptableProtocols cannot be set by
+            # the constructor unless at least one mechanism is supported.
+            _setAcceptableProtocols(ctx, self._acceptableProtocols)
+
+        return ctx
+
+
+OpenSSLCertificateOptions.__getstate__ = deprecated(
+    Version("Twisted", 15, 0, 0), "a real persistence system"
+)(OpenSSLCertificateOptions.__getstate__)
+OpenSSLCertificateOptions.__setstate__ = deprecated(
+    Version("Twisted", 15, 0, 0), "a real persistence system"
+)(OpenSSLCertificateOptions.__setstate__)
+
+
+@implementer(ICipher)
+@attr.s(frozen=True, auto_attribs=True)
+class OpenSSLCipher:
+    """
+    A representation of an OpenSSL cipher.
+
+    @ivar fullName: The full name of the cipher. For example
+        C{u"ECDHE-RSA-AES256-GCM-SHA384"}.
+    @type fullName: L{unicode}
+    """
+
+    fullName: str
+
+
+@lru_cache(maxsize=32)
+def _expandCipherString(cipherString, method, options):
+    """
+    Expand C{cipherString} according to C{method} and C{options} to a tuple of
+    explicit ciphers that are supported by the current platform.
+
+    @param cipherString: An OpenSSL cipher string to expand.
+    @type cipherString: L{unicode}
+
+    @param method: An OpenSSL method like C{SSL.TLS_METHOD} used for
+        determining the effective ciphers.
+
+    @param options: OpenSSL options like C{SSL.OP_NO_SSLv3} ORed together.
+    @type options: L{int}
+
+    @return: The effective list of explicit ciphers that results from the
+        arguments on the current platform.
+    @rtype: L{tuple} of L{ICipher}
+    """
+    ctx = SSL.Context(method)
+    ctx.set_options(options)
+    try:
+        ctx.set_cipher_list(cipherString.encode("ascii"))
+    except SSL.Error as e:
+        # OpenSSL 1.1.1 turns an invalid cipher list into TLS 1.3
+        # ciphers, so pyOpenSSL >= 19.0.0 raises an artificial Error
+        # that lacks a corresponding OpenSSL error if the cipher list
+        # consists only of these after a call to set_cipher_list.
+        if not e.args[0]:
+            return tuple()
+        if e.args[0][0][2] == "no cipher match":
+            return tuple()
+        else:
+            raise
+    conn = SSL.Connection(ctx, None)
+    ciphers = conn.get_cipher_list()
+    if isinstance(ciphers[0], str):
+        return tuple(OpenSSLCipher(cipher) for cipher in ciphers)
+    else:
+        return tuple(OpenSSLCipher(cipher.decode("ascii")) for cipher in ciphers)
+
+
+@lru_cache(maxsize=128)
+def _selectCiphers(wantedCiphers, availableCiphers):
+    """
+    Caclulate the acceptable list of ciphers from the ciphers we want and the
+    ciphers we have support for.
+
+    @param wantedCiphers: The ciphers we want to use.
+    @type wantedCiphers: L{tuple} of L{OpenSSLCipher}
+
+    @param availableCiphers: The ciphers we have available to use.
+    @type availableCiphers: L{tuple} of L{OpenSSLCipher}
+
+    @rtype: L{tuple} of L{OpenSSLCipher}
+    """
+    return tuple(cipher for cipher in wantedCiphers if cipher in availableCiphers)
+
+
+@implementer(IAcceptableCiphers)
+class OpenSSLAcceptableCiphers:
+    """
+    A representation of ciphers that are acceptable for TLS connections.
+    """
+
+    def __init__(self, ciphers):
+        self._ciphers = tuple(ciphers)
+
+    def selectCiphers(self, availableCiphers):
+        return _selectCiphers(self._ciphers, tuple(availableCiphers))
+
+    @classmethod
+    def fromOpenSSLCipherString(cls, cipherString):
+        """
+        Create a new instance using an OpenSSL cipher string.
+
+        @param cipherString: An OpenSSL cipher string that describes what
+            cipher suites are acceptable.
+            See the documentation of U{OpenSSL
+            <http://www.openssl.org/docs/apps/ciphers.html#CIPHER_STRINGS>} or
+            U{Apache
+            <http://httpd.apache.org/docs/2.4/mod/mod_ssl.html#sslciphersuite>}
+            for details.
+        @type cipherString: L{unicode}
+
+        @return: Instance representing C{cipherString}.
+        @rtype: L{twisted.internet.ssl.AcceptableCiphers}
+        """
+        return cls(
+            _expandCipherString(
+                nativeString(cipherString),
+                SSL.TLS_METHOD,
+                SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3,
+            )
+        )
+
+
+# A secure default.
+# Sources for more information on TLS ciphers:
+#
+# - https://wiki.mozilla.org/Security/Server_Side_TLS
+# - https://www.ssllabs.com/projects/best-practices/index.html
+# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
+#
+# The general intent is:
+# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
+# - prefer ECDHE over DHE for better performance,
+# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and
+#   security,
+# - prefer AES-GCM to ChaCha20 because AES hardware support is common,
+# - disable NULL authentication, MD5 MACs and DSS for security reasons.
+#
+defaultCiphers = OpenSSLAcceptableCiphers.fromOpenSSLCipherString(
+    "TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:"
+    "TLS13-AES-128-GCM-SHA256:"
+    "ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:DH+CHACHA20:ECDH+AES256:DH+AES256:"
+    "ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:"
+    "!aNULL:!MD5:!DSS"
+)
+_defaultCurveName = "prime256v1"
+
+
+class _ChooseDiffieHellmanEllipticCurve:
+    """
+    Chooses the best elliptic curve for Elliptic Curve Diffie-Hellman
+    key exchange, and provides a C{configureECDHCurve} method to set
+    the curve, when appropriate, on a new L{OpenSSL.SSL.Context}.
+
+    The C{configureECDHCurve} method will be set to one of the
+    following based on the provided OpenSSL version and configuration:
+
+        - L{_configureOpenSSL110}
+
+        - L{_configureOpenSSL102}
+
+        - L{_configureOpenSSL101}
+
+        - L{_configureOpenSSL101NoCurves}.
+
+    @param openSSLVersion: The OpenSSL version number.
+    @type openSSLVersion: L{int}
+
+    @see: L{OpenSSL.SSL.OPENSSL_VERSION_NUMBER}
+
+    @param openSSLlib: The OpenSSL C{cffi} library module.
+    @param openSSLcrypto: The OpenSSL L{crypto} module.
+
+    @see: L{crypto}
+    """
+
+    def __init__(self, openSSLVersion, openSSLlib, openSSLcrypto):
+        self._openSSLlib = openSSLlib
+        self._openSSLcrypto = openSSLcrypto
+        if openSSLVersion >= 0x10100000:
+            self.configureECDHCurve = self._configureOpenSSL110
+        elif openSSLVersion >= 0x10002000:
+            self.configureECDHCurve = self._configureOpenSSL102
+        else:
+            try:
+                self._ecCurve = openSSLcrypto.get_elliptic_curve(_defaultCurveName)
+            except ValueError:
+                # The get_elliptic_curve method raises a ValueError
+                # when the curve does not exist.
+                self.configureECDHCurve = self._configureOpenSSL101NoCurves
+            else:
+                self.configureECDHCurve = self._configureOpenSSL101
+
+    def _configureOpenSSL110(self, ctx):
+        """
+        OpenSSL 1.1.0 Contexts are preconfigured with an optimal set
+        of ECDH curves.  This method does nothing.
+
+        @param ctx: L{OpenSSL.SSL.Context}
+        """
+
+    def _configureOpenSSL102(self, ctx):
+        """
+        Have the context automatically choose elliptic curves for
+        ECDH.  Run on OpenSSL 1.0.2 and OpenSSL 1.1.0+, but only has
+        an effect on OpenSSL 1.0.2.
+
+        @param ctx: The context which .
+        @type ctx: L{OpenSSL.SSL.Context}
+        """
+        ctxPtr = ctx._context
+        try:
+            self._openSSLlib.SSL_CTX_set_ecdh_auto(ctxPtr, True)
+        except BaseException:
+            pass
+
+    def _configureOpenSSL101(self, ctx):
+        """
+        Set the default elliptic curve for ECDH on the context.  Only
+        run on OpenSSL 1.0.1.
+
+        @param ctx: The context on which to set the ECDH curve.
+        @type ctx: L{OpenSSL.SSL.Context}
+        """
+        try:
+            ctx.set_tmp_ecdh(self._ecCurve)
+        except BaseException:
+            pass
+
+    def _configureOpenSSL101NoCurves(self, ctx):
+        """
+        No elliptic curves are available on OpenSSL 1.0.1. We can't
+        set anything, so do nothing.
+
+        @param ctx: The context on which to set the ECDH curve.
+        @type ctx: L{OpenSSL.SSL.Context}
+        """
+
+
+class OpenSSLDiffieHellmanParameters:
+    """
+    A representation of key generation parameters that are required for
+    Diffie-Hellman key exchange.
+    """
+
+    def __init__(self, parameters):
+        self._dhFile = parameters
+
+    @classmethod
+    def fromFile(cls, filePath):
+        """
+        Load parameters from a file.
+
+        Such a file can be generated using the C{openssl} command line tool as
+        following:
+
+        C{openssl dhparam -out dh_param_2048.pem -2 2048}
+
+        Please refer to U{OpenSSL's C{dhparam} documentation
+        <http://www.openssl.org/docs/apps/dhparam.html>} for further details.
+
+        @param filePath: A file containing parameters for Diffie-Hellman key
+            exchange.
+        @type filePath: L{FilePath <twisted.python.filepath.FilePath>}
+
+        @return: An instance that loads its parameters from C{filePath}.
+        @rtype: L{DiffieHellmanParameters
+            <twisted.internet.ssl.DiffieHellmanParameters>}
+        """
+        return cls(filePath)
+
+
+def _setAcceptableProtocols(context, acceptableProtocols):
+    """
+    Called to set up the L{OpenSSL.SSL.Context} for doing NPN and/or ALPN
+    negotiation.
+
+    @param context: The context which is set up.
+    @type context: L{OpenSSL.SSL.Context}
+
+    @param acceptableProtocols: The protocols this peer is willing to speak
+        after the TLS negotiation has completed, advertised over both ALPN and
+        NPN. If this argument is specified, and no overlap can be found with
+        the other peer, the connection will fail to be established. If the
+        remote peer does not offer NPN or ALPN, the connection will be
+        established, but no protocol wil be negotiated. Protocols earlier in
+        the list are preferred over those later in the list.
+    @type acceptableProtocols: L{list} of L{bytes}
+    """
+
+    def protoSelectCallback(conn, protocols):
+        """
+        NPN client-side and ALPN server-side callback used to select
+        the next protocol. Prefers protocols found earlier in
+        C{_acceptableProtocols}.
+
+        @param conn: The context which is set up.
+        @type conn: L{OpenSSL.SSL.Connection}
+
+        @param conn: Protocols advertised by the other side.
+        @type conn: L{list} of L{bytes}
+        """
+        overlap = set(protocols) & set(acceptableProtocols)
+
+        for p in acceptableProtocols:
+            if p in overlap:
+                return p
+        else:
+            return b""
+
+    # If we don't actually have protocols to negotiate, don't set anything up.
+    # Depending on OpenSSL version, failing some of the selection callbacks can
+    # cause the handshake to fail, which is presumably not what was intended
+    # here.
+    if not acceptableProtocols:
+        return
+
+    supported = protocolNegotiationMechanisms()
+
+    if supported & ProtocolNegotiationSupport.NPN:
+
+        def npnAdvertiseCallback(conn):
+            return acceptableProtocols
+
+        context.set_npn_advertise_callback(npnAdvertiseCallback)
+        context.set_npn_select_callback(protoSelectCallback)
+
+    if supported & ProtocolNegotiationSupport.ALPN:
+        context.set_alpn_select_callback(protoSelectCallback)
+        context.set_alpn_protos(acceptableProtocols)
diff --git a/contrib/python/Twisted/py3/twisted/internet/_threadedselect.py b/contrib/python/Twisted/py3/twisted/internet/_threadedselect.py
new file mode 100644
index 00000000000..8a53e4ca962
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_threadedselect.py
@@ -0,0 +1,337 @@
+# -*- test-case-name: twisted.test.test_internet -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Threaded select reactor
+
+The threadedselectreactor is a specialized reactor for integrating with
+arbitrary foreign event loop, such as those you find in GUI toolkits.
+
+There are three things you'll need to do to use this reactor.
+
+Install the reactor at the beginning of your program, before importing
+the rest of Twisted::
+
+    | from twisted.internet import _threadedselect
+    | _threadedselect.install()
+
+Interleave this reactor with your foreign event loop, at some point after
+your event loop is initialized::
+
+    | from twisted.internet import reactor
+    | reactor.interleave(foreignEventLoopWakerFunction)
+    | self.addSystemEventTrigger('after', 'shutdown', foreignEventLoopStop)
+
+Instead of shutting down the foreign event loop directly, shut down the
+reactor::
+
+    | from twisted.internet import reactor
+    | reactor.stop()
+
+In order for Twisted to do its work in the main thread (the thread that
+interleave is called from), a waker function is necessary.  The waker function
+will be called from a "background" thread with one argument: func.
+The waker function's purpose is to call func() from the main thread.
+Many GUI toolkits ship with appropriate waker functions.
+Some examples of this are wxPython's wx.callAfter (may be wxCallAfter in
+older versions of wxPython) or PyObjC's PyObjCTools.AppHelper.callAfter.
+These would be used in place of "foreignEventLoopWakerFunction" in the above
+example.
+
+The other integration point at which the foreign event loop and this reactor
+must integrate is shutdown.  In order to ensure clean shutdown of Twisted,
+you must allow for Twisted to come to a complete stop before quitting the
+application.  Typically, you will do this by setting up an after shutdown
+trigger to stop your foreign event loop, and call reactor.stop() where you
+would normally have initiated the shutdown procedure for the foreign event
+loop.  Shutdown functions that could be used in place of
+"foreignEventloopStop" would be the ExitMainLoop method of the wxApp instance
+with wxPython, or the PyObjCTools.AppHelper.stopEventLoop function.
+"""
+
+import select
+import sys
+from errno import EBADF, EINTR
+from functools import partial
+from queue import Empty, Queue
+from threading import Thread
+
+from zope.interface import implementer
+
+from twisted.internet import posixbase
+from twisted.internet.interfaces import IReactorFDSet
+from twisted.internet.posixbase import _NO_FILEDESC, _NO_FILENO
+from twisted.internet.selectreactor import _select
+from twisted.python import failure, log, threadable
+
+
+def dictRemove(dct, value):
+    try:
+        del dct[value]
+    except KeyError:
+        pass
+
+
+def raiseException(e):
+    raise e
+
+
+@implementer(IReactorFDSet)
+class ThreadedSelectReactor(posixbase.PosixReactorBase):
+    """A threaded select() based reactor - runs on all POSIX platforms and on
+    Win32.
+    """
+
+    def __init__(self):
+        threadable.init(1)
+        self.reads = {}
+        self.writes = {}
+        self.toThreadQueue = Queue()
+        self.toMainThread = Queue()
+        self.workerThread = None
+        self.mainWaker = None
+        posixbase.PosixReactorBase.__init__(self)
+        self.addSystemEventTrigger("after", "shutdown", self._mainLoopShutdown)
+
+    def wakeUp(self):
+        # we want to wake up from any thread
+        self.waker.wakeUp()
+
+    def callLater(self, *args, **kw):
+        tple = posixbase.PosixReactorBase.callLater(self, *args, **kw)
+        self.wakeUp()
+        return tple
+
+    def _sendToMain(self, msg, *args):
+        self.toMainThread.put((msg, args))
+        if self.mainWaker is not None:
+            self.mainWaker()
+
+    def _sendToThread(self, fn, *args):
+        self.toThreadQueue.put((fn, args))
+
+    def _preenDescriptorsInThread(self):
+        log.msg("Malformed file descriptor found.  Preening lists.")
+        readers = self.reads.keys()
+        writers = self.writes.keys()
+        self.reads.clear()
+        self.writes.clear()
+        for selDict, selList in ((self.reads, readers), (self.writes, writers)):
+            for selectable in selList:
+                try:
+                    select.select([selectable], [selectable], [selectable], 0)
+                except BaseException:
+                    log.msg("bad descriptor %s" % selectable)
+                else:
+                    selDict[selectable] = 1
+
+    def _workerInThread(self):
+        try:
+            while 1:
+                fn, args = self.toThreadQueue.get()
+                fn(*args)
+        except SystemExit:
+            pass  # Exception indicates this thread should exit
+        except BaseException:
+            f = failure.Failure()
+            self._sendToMain("Failure", f)
+
+    def _doSelectInThread(self, timeout):
+        """Run one iteration of the I/O monitor loop.
+
+        This will run all selectables who had input or output readiness
+        waiting for them.
+        """
+        reads = self.reads
+        writes = self.writes
+        while 1:
+            try:
+                r, w, ignored = _select(reads.keys(), writes.keys(), [], timeout)
+                break
+            except ValueError:
+                # Possibly a file descriptor has gone negative?
+                log.err()
+                self._preenDescriptorsInThread()
+            except TypeError:
+                # Something *totally* invalid (object w/o fileno, non-integral
+                # result) was passed
+                log.err()
+                self._preenDescriptorsInThread()
+            except OSError as se:
+                # select(2) encountered an error
+                if se.args[0] in (0, 2):
+                    # windows does this if it got an empty list
+                    if (not reads) and (not writes):
+                        return
+                    else:
+                        raise
+                elif se.args[0] == EINTR:
+                    return
+                elif se.args[0] == EBADF:
+                    self._preenDescriptorsInThread()
+                else:
+                    # OK, I really don't know what's going on.  Blow up.
+                    raise
+        self._sendToMain("Notify", r, w)
+
+    def _process_Notify(self, r, w):
+        reads = self.reads
+        writes = self.writes
+
+        _drdw = self._doReadOrWrite
+        _logrun = log.callWithLogger
+        for selectables, method, dct in ((r, "doRead", reads), (w, "doWrite", writes)):
+            for selectable in selectables:
+                # if this was disconnected in another thread, kill it.
+                if selectable not in dct:
+                    continue
+                # This for pausing input when we're not ready for more.
+                _logrun(selectable, _drdw, selectable, method, dct)
+
+    def _process_Failure(self, f):
+        f.raiseException()
+
+    _doIterationInThread = _doSelectInThread
+
+    def ensureWorkerThread(self):
+        if self.workerThread is None or not self.workerThread.isAlive():
+            self.workerThread = Thread(target=self._workerInThread)
+            self.workerThread.start()
+
+    def doThreadIteration(self, timeout):
+        self._sendToThread(self._doIterationInThread, timeout)
+        self.ensureWorkerThread()
+        msg, args = self.toMainThread.get()
+        getattr(self, "_process_" + msg)(*args)
+
+    doIteration = doThreadIteration
+
+    def _interleave(self):
+        while self.running:
+            self.runUntilCurrent()
+            t2 = self.timeout()
+            t = self.running and t2
+            self._sendToThread(self._doIterationInThread, t)
+            yield None
+            msg, args = self.toMainThread.get_nowait()
+            getattr(self, "_process_" + msg)(*args)
+
+    def interleave(self, waker, *args, **kw):
+        """
+        interleave(waker) interleaves this reactor with the
+        current application by moving the blocking parts of
+        the reactor (select() in this case) to a separate
+        thread.  This is typically useful for integration with
+        GUI applications which have their own event loop
+        already running.
+
+        See the module docstring for more information.
+        """
+        self.startRunning(*args, **kw)
+        loop = self._interleave()
+
+        def mainWaker(waker=waker, loop=loop):
+            waker(partial(next, loop))
+
+        self.mainWaker = mainWaker
+        next(loop)
+        self.ensureWorkerThread()
+
+    def _mainLoopShutdown(self):
+        self.mainWaker = None
+        if self.workerThread is not None:
+            self._sendToThread(raiseException, SystemExit)
+            self.wakeUp()
+            try:
+                while 1:
+                    msg, args = self.toMainThread.get_nowait()
+            except Empty:
+                pass
+            self.workerThread.join()
+            self.workerThread = None
+        try:
+            while 1:
+                fn, args = self.toThreadQueue.get_nowait()
+                if fn is self._doIterationInThread:
+                    log.msg("Iteration is still in the thread queue!")
+                elif fn is raiseException and args[0] is SystemExit:
+                    pass
+                else:
+                    fn(*args)
+        except Empty:
+            pass
+
+    def _doReadOrWrite(self, selectable, method, dict):
+        try:
+            why = getattr(selectable, method)()
+            handfn = getattr(selectable, "fileno", None)
+            if not handfn:
+                why = _NO_FILENO
+            elif handfn() == -1:
+                why = _NO_FILEDESC
+        except BaseException:
+            why = sys.exc_info()[1]
+            log.err()
+        if why:
+            self._disconnectSelectable(selectable, why, method == "doRead")
+
+    def addReader(self, reader):
+        """Add a FileDescriptor for notification of data available to read."""
+        self._sendToThread(self.reads.__setitem__, reader, 1)
+        self.wakeUp()
+
+    def addWriter(self, writer):
+        """Add a FileDescriptor for notification of data available to write."""
+        self._sendToThread(self.writes.__setitem__, writer, 1)
+        self.wakeUp()
+
+    def removeReader(self, reader):
+        """Remove a Selectable for notification of data available to read."""
+        self._sendToThread(dictRemove, self.reads, reader)
+
+    def removeWriter(self, writer):
+        """Remove a Selectable for notification of data available to write."""
+        self._sendToThread(dictRemove, self.writes, writer)
+
+    def removeAll(self):
+        return self._removeAll(self.reads, self.writes)
+
+    def getReaders(self):
+        return list(self.reads.keys())
+
+    def getWriters(self):
+        return list(self.writes.keys())
+
+    def stop(self):
+        """
+        Extend the base stop implementation to also wake up the select thread so
+        that C{runUntilCurrent} notices the reactor should stop.
+        """
+        posixbase.PosixReactorBase.stop(self)
+        self.wakeUp()
+
+    def run(self, installSignalHandlers=True):
+        self.startRunning(installSignalHandlers=installSignalHandlers)
+        self.mainLoop()
+
+    def mainLoop(self):
+        q = Queue()
+        self.interleave(q.put)
+        while self.running:
+            try:
+                q.get()()
+            except StopIteration:
+                break
+
+
+def install():
+    """Configure the twisted mainloop to be run using the select() reactor."""
+    reactor = ThreadedSelectReactor()
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
+    return reactor
+
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/_win32serialport.py b/contrib/python/Twisted/py3/twisted/internet/_win32serialport.py
new file mode 100644
index 00000000000..2dda4b9816f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_win32serialport.py
@@ -0,0 +1,156 @@
+# -*- test-case-name: twisted.internet.test.test_win32serialport -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Serial port support for Windows.
+
+Requires PySerial and pywin32.
+"""
+
+
+import win32event  # type: ignore[import]
+import win32file  # type: ignore[import]
+
+# system imports
+from serial import PARITY_NONE  # type: ignore[import]
+from serial import EIGHTBITS, STOPBITS_ONE
+from serial.serialutil import to_bytes  # type: ignore[import]
+
+# twisted imports
+from twisted.internet import abstract
+
+# sibling imports
+from twisted.internet.serialport import BaseSerialPort
+
+
+class SerialPort(BaseSerialPort, abstract.FileDescriptor):
+    """A serial device, acting as a transport, that uses a win32 event."""
+
+    connected = 1
+
+    def __init__(
+        self,
+        protocol,
+        deviceNameOrPortNumber,
+        reactor,
+        baudrate=9600,
+        bytesize=EIGHTBITS,
+        parity=PARITY_NONE,
+        stopbits=STOPBITS_ONE,
+        xonxoff=0,
+        rtscts=0,
+    ):
+        self._serial = self._serialFactory(
+            deviceNameOrPortNumber,
+            baudrate=baudrate,
+            bytesize=bytesize,
+            parity=parity,
+            stopbits=stopbits,
+            timeout=None,
+            xonxoff=xonxoff,
+            rtscts=rtscts,
+        )
+        self.flushInput()
+        self.flushOutput()
+        self.reactor = reactor
+        self.protocol = protocol
+        self.outQueue = []
+        self.closed = 0
+        self.closedNotifies = 0
+        self.writeInProgress = 0
+
+        self.protocol = protocol
+        self._overlappedRead = win32file.OVERLAPPED()
+        self._overlappedRead.hEvent = win32event.CreateEvent(None, 1, 0, None)
+        self._overlappedWrite = win32file.OVERLAPPED()
+        self._overlappedWrite.hEvent = win32event.CreateEvent(None, 0, 0, None)
+
+        self.reactor.addEvent(self._overlappedRead.hEvent, self, "serialReadEvent")
+        self.reactor.addEvent(self._overlappedWrite.hEvent, self, "serialWriteEvent")
+
+        self.protocol.makeConnection(self)
+        self._finishPortSetup()
+
+    def _finishPortSetup(self):
+        """
+        Finish setting up the serial port.
+
+        This is a separate method to facilitate testing.
+        """
+        flags, comstat = self._clearCommError()
+        rc, self.read_buf = win32file.ReadFile(
+            self._serial._port_handle,
+            win32file.AllocateReadBuffer(1),
+            self._overlappedRead,
+        )
+
+    def _clearCommError(self):
+        return win32file.ClearCommError(self._serial._port_handle)
+
+    def serialReadEvent(self):
+        # get that character we set up
+        n = win32file.GetOverlappedResult(
+            self._serial._port_handle, self._overlappedRead, 0
+        )
+        first = to_bytes(self.read_buf[:n])
+        # now we should get everything that is already in the buffer
+        flags, comstat = self._clearCommError()
+        if comstat.cbInQue:
+            win32event.ResetEvent(self._overlappedRead.hEvent)
+            rc, buf = win32file.ReadFile(
+                self._serial._port_handle,
+                win32file.AllocateReadBuffer(comstat.cbInQue),
+                self._overlappedRead,
+            )
+            n = win32file.GetOverlappedResult(
+                self._serial._port_handle, self._overlappedRead, 1
+            )
+            # handle all the received data:
+            self.protocol.dataReceived(first + to_bytes(buf[:n]))
+        else:
+            # handle all the received data:
+            self.protocol.dataReceived(first)
+
+        # set up next one
+        win32event.ResetEvent(self._overlappedRead.hEvent)
+        rc, self.read_buf = win32file.ReadFile(
+            self._serial._port_handle,
+            win32file.AllocateReadBuffer(1),
+            self._overlappedRead,
+        )
+
+    def write(self, data):
+        if data:
+            if self.writeInProgress:
+                self.outQueue.append(data)
+            else:
+                self.writeInProgress = 1
+                win32file.WriteFile(
+                    self._serial._port_handle, data, self._overlappedWrite
+                )
+
+    def serialWriteEvent(self):
+        try:
+            dataToWrite = self.outQueue.pop(0)
+        except IndexError:
+            self.writeInProgress = 0
+            return
+        else:
+            win32file.WriteFile(
+                self._serial._port_handle, dataToWrite, self._overlappedWrite
+            )
+
+    def connectionLost(self, reason):
+        """
+        Called when the serial port disconnects.
+
+        Will call C{connectionLost} on the protocol that is handling the
+        serial data.
+        """
+        self.reactor.removeEvent(self._overlappedRead.hEvent)
+        self.reactor.removeEvent(self._overlappedWrite.hEvent)
+        abstract.FileDescriptor.connectionLost(self, reason)
+        self._serial.close()
+        self.protocol.connectionLost(reason)
diff --git a/contrib/python/Twisted/py3/twisted/internet/_win32stdio.py b/contrib/python/Twisted/py3/twisted/internet/_win32stdio.py
new file mode 100644
index 00000000000..f1ac920f606
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/_win32stdio.py
@@ -0,0 +1,127 @@
+# -*- test-case-name: twisted.test.test_stdio -*-
+
+"""
+Windows-specific implementation of the L{twisted.internet.stdio} interface.
+"""
+
+
+import msvcrt
+import os
+
+from zope.interface import implementer
+
+import win32api  # type: ignore[import]
+
+from twisted.internet import _pollingfile, main
+from twisted.internet.interfaces import (
+    IAddress,
+    IConsumer,
+    IHalfCloseableProtocol,
+    IPushProducer,
+    ITransport,
+)
+from twisted.python.failure import Failure
+
+
+@implementer(IAddress)
+class Win32PipeAddress:
+    pass
+
+
+@implementer(ITransport, IConsumer, IPushProducer)
+class StandardIO(_pollingfile._PollingTimer):
+    disconnecting = False
+    disconnected = False
+
+    def __init__(self, proto, reactor=None):
+        """
+        Start talking to standard IO with the given protocol.
+
+        Also, put it stdin/stdout/stderr into binary mode.
+        """
+        if reactor is None:
+            from twisted.internet import reactor
+
+        for stdfd in range(0, 1, 2):
+            msvcrt.setmode(stdfd, os.O_BINARY)
+
+        _pollingfile._PollingTimer.__init__(self, reactor)
+        self.proto = proto
+
+        hstdin = win32api.GetStdHandle(win32api.STD_INPUT_HANDLE)
+        hstdout = win32api.GetStdHandle(win32api.STD_OUTPUT_HANDLE)
+
+        self.stdin = _pollingfile._PollableReadPipe(
+            hstdin, self.dataReceived, self.readConnectionLost
+        )
+
+        self.stdout = _pollingfile._PollableWritePipe(hstdout, self.writeConnectionLost)
+
+        self._addPollableResource(self.stdin)
+        self._addPollableResource(self.stdout)
+
+        self.proto.makeConnection(self)
+
+    def dataReceived(self, data):
+        self.proto.dataReceived(data)
+
+    def readConnectionLost(self):
+        if IHalfCloseableProtocol.providedBy(self.proto):
+            self.proto.readConnectionLost()
+        self.checkConnLost()
+
+    def writeConnectionLost(self):
+        if IHalfCloseableProtocol.providedBy(self.proto):
+            self.proto.writeConnectionLost()
+        self.checkConnLost()
+
+    connsLost = 0
+
+    def checkConnLost(self):
+        self.connsLost += 1
+        if self.connsLost >= 2:
+            self.disconnecting = True
+            self.disconnected = True
+            self.proto.connectionLost(Failure(main.CONNECTION_DONE))
+
+    # ITransport
+
+    def write(self, data):
+        self.stdout.write(data)
+
+    def writeSequence(self, seq):
+        self.stdout.write(b"".join(seq))
+
+    def loseConnection(self):
+        self.disconnecting = True
+        self.stdin.close()
+        self.stdout.close()
+
+    def getPeer(self):
+        return Win32PipeAddress()
+
+    def getHost(self):
+        return Win32PipeAddress()
+
+    # IConsumer
+
+    def registerProducer(self, producer, streaming):
+        return self.stdout.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        return self.stdout.unregisterProducer()
+
+    # def write() above
+
+    # IProducer
+
+    def stopProducing(self):
+        self.stdin.stopProducing()
+
+    # IPushProducer
+
+    def pauseProducing(self):
+        self.stdin.pauseProducing()
+
+    def resumeProducing(self):
+        self.stdin.resumeProducing()
diff --git a/contrib/python/Twisted/py3/twisted/internet/abstract.py b/contrib/python/Twisted/py3/twisted/internet/abstract.py
new file mode 100644
index 00000000000..45a80383c2c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/abstract.py
@@ -0,0 +1,542 @@
+# -*- test-case-name: twisted.test.test_abstract -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support for generic select()able objects.
+"""
+
+
+from socket import AF_INET, AF_INET6, inet_pton
+from typing import Iterable, List, Optional
+
+from zope.interface import implementer
+
+from twisted.internet import interfaces, main
+from twisted.python import failure, reflect
+
+# Twisted Imports
+from twisted.python.compat import lazyByteSlice
+
+
+def _dataMustBeBytes(obj):
+    if not isinstance(obj, bytes):  # no, really, I mean it
+        raise TypeError("Data must be bytes")
+
+
+# Python 3.4+ can join bytes and memoryviews; using a
+# memoryview prevents the slice from copying
+def _concatenate(bObj, offset, bArray):
+    return b"".join([memoryview(bObj)[offset:]] + bArray)
+
+
+class _ConsumerMixin:
+    """
+    L{IConsumer} implementations can mix this in to get C{registerProducer} and
+    C{unregisterProducer} methods which take care of keeping track of a
+    producer's state.
+
+    Subclasses must provide three attributes which L{_ConsumerMixin} will read
+    but not write:
+
+      - connected: A C{bool} which is C{True} as long as the consumer has
+        someplace to send bytes (for example, a TCP connection), and then
+        C{False} when it no longer does.
+
+      - disconnecting: A C{bool} which is C{False} until something like
+        L{ITransport.loseConnection} is called, indicating that the send buffer
+        should be flushed and the connection lost afterwards.  Afterwards,
+        C{True}.
+
+      - disconnected: A C{bool} which is C{False} until the consumer no longer
+        has a place to send bytes, then C{True}.
+
+    Subclasses must also override the C{startWriting} method.
+
+    @ivar producer: L{None} if no producer is registered, otherwise the
+        registered producer.
+
+    @ivar producerPaused: A flag indicating whether the producer is currently
+        paused.
+    @type producerPaused: L{bool}
+
+    @ivar streamingProducer: A flag indicating whether the producer was
+        registered as a streaming (ie push) producer or not (ie a pull
+        producer).  This will determine whether the consumer may ever need to
+        pause and resume it, or if it can merely call C{resumeProducing} on it
+        when buffer space is available.
+    @ivar streamingProducer: C{bool} or C{int}
+
+    """
+
+    producer = None
+    producerPaused = False
+    streamingProducer = False
+
+    def startWriting(self):
+        """
+        Override in a subclass to cause the reactor to monitor this selectable
+        for write events.  This will be called once in C{unregisterProducer} if
+        C{loseConnection} has previously been called, so that the connection can
+        actually close.
+        """
+        raise NotImplementedError("%r did not implement startWriting")
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register to receive data from a producer.
+
+        This sets this selectable to be a consumer for a producer.  When this
+        selectable runs out of data on a write() call, it will ask the producer
+        to resumeProducing(). When the FileDescriptor's internal data buffer is
+        filled, it will ask the producer to pauseProducing(). If the connection
+        is lost, FileDescriptor calls producer's stopProducing() method.
+
+        If streaming is true, the producer should provide the IPushProducer
+        interface. Otherwise, it is assumed that producer provides the
+        IPullProducer interface. In this case, the producer won't be asked to
+        pauseProducing(), but it has to be careful to write() data only when its
+        resumeProducing() method is called.
+        """
+        if self.producer is not None:
+            raise RuntimeError(
+                "Cannot register producer %s, because producer %s was never "
+                "unregistered." % (producer, self.producer)
+            )
+        if self.disconnected:
+            producer.stopProducing()
+        else:
+            self.producer = producer
+            self.streamingProducer = streaming
+            if not streaming:
+                producer.resumeProducing()
+
+    def unregisterProducer(self):
+        """
+        Stop consuming data from a producer, without disconnecting.
+        """
+        self.producer = None
+        if self.connected and self.disconnecting:
+            self.startWriting()
+
+
+@implementer(interfaces.ILoggingContext)
+class _LogOwner:
+    """
+    Mixin to help implement L{interfaces.ILoggingContext} for transports which
+    have a protocol, the log prefix of which should also appear in the
+    transport's log prefix.
+    """
+
+    def _getLogPrefix(self, applicationObject: object) -> str:
+        """
+        Determine the log prefix to use for messages related to
+        C{applicationObject}, which may or may not be an
+        L{interfaces.ILoggingContext} provider.
+
+        @return: A C{str} giving the log prefix to use.
+        """
+        if interfaces.ILoggingContext.providedBy(applicationObject):
+            return applicationObject.logPrefix()
+        return applicationObject.__class__.__name__
+
+    def logPrefix(self):
+        """
+        Override this method to insert custom logging behavior.  Its
+        return value will be inserted in front of every line.  It may
+        be called more times than the number of output lines.
+        """
+        return "-"
+
+
+@implementer(
+    interfaces.IPushProducer,
+    interfaces.IReadWriteDescriptor,
+    interfaces.IConsumer,
+    interfaces.ITransport,
+    interfaces.IHalfCloseableDescriptor,
+)
+class FileDescriptor(_ConsumerMixin, _LogOwner):
+    """
+    An object which can be operated on by select().
+
+    This is an abstract superclass of all objects which may be notified when
+    they are readable or writable; e.g. they have a file-descriptor that is
+    valid to be passed to select(2).
+    """
+
+    connected = 0
+    disconnected = 0
+    disconnecting = 0
+    _writeDisconnecting = False
+    _writeDisconnected = False
+    dataBuffer = b""
+    offset = 0
+
+    SEND_LIMIT = 128 * 1024
+
+    def __init__(self, reactor: Optional[interfaces.IReactorFDSet] = None):
+        """
+        @param reactor: An L{IReactorFDSet} provider which this descriptor will
+            use to get readable and writeable event notifications.  If no value
+            is given, the global reactor will be used.
+        """
+        if not reactor:
+            from twisted.internet import reactor as _reactor
+
+            reactor = _reactor  # type: ignore[assignment]
+        self.reactor = reactor
+        # will be added to dataBuffer in doWrite
+        self._tempDataBuffer: List[bytes] = []
+        self._tempDataLen = 0
+
+    def connectionLost(self, reason):
+        """The connection was lost.
+
+        This is called when the connection on a selectable object has been
+        lost.  It will be called whether the connection was closed explicitly,
+        an exception occurred in an event handler, or the other end of the
+        connection closed it first.
+
+        Clean up state here, but make sure to call back up to FileDescriptor.
+        """
+        self.disconnected = 1
+        self.connected = 0
+        if self.producer is not None:
+            self.producer.stopProducing()
+            self.producer = None
+        self.stopReading()
+        self.stopWriting()
+
+    def writeSomeData(self, data: bytes) -> None:
+        """
+        Write as much as possible of the given data, immediately.
+
+        This is called to invoke the lower-level writing functionality, such
+        as a socket's send() method, or a file's write(); this method
+        returns an integer or an exception.  If an integer, it is the number
+        of bytes written (possibly zero); if an exception, it indicates the
+        connection was lost.
+        """
+        raise NotImplementedError(
+            "%s does not implement writeSomeData" % reflect.qual(self.__class__)
+        )
+
+    def doRead(self):
+        """
+        Called when data is available for reading.
+
+        Subclasses must override this method. The result will be interpreted
+        in the same way as a result of doWrite().
+        """
+        raise NotImplementedError(
+            "%s does not implement doRead" % reflect.qual(self.__class__)
+        )
+
+    def doWrite(self):
+        """
+        Called when data can be written.
+
+        @return: L{None} on success, an exception or a negative integer on
+            failure.
+
+        @see: L{twisted.internet.interfaces.IWriteDescriptor.doWrite}.
+        """
+        if len(self.dataBuffer) - self.offset < self.SEND_LIMIT:
+            # If there is currently less than SEND_LIMIT bytes left to send
+            # in the string, extend it with the array data.
+            self.dataBuffer = _concatenate(
+                self.dataBuffer, self.offset, self._tempDataBuffer
+            )
+            self.offset = 0
+            self._tempDataBuffer = []
+            self._tempDataLen = 0
+
+        # Send as much data as you can.
+        if self.offset:
+            l = self.writeSomeData(lazyByteSlice(self.dataBuffer, self.offset))
+        else:
+            l = self.writeSomeData(self.dataBuffer)
+
+        # There is no writeSomeData implementation in Twisted which returns
+        # < 0, but the documentation for writeSomeData used to claim negative
+        # integers meant connection lost.  Keep supporting this here,
+        # although it may be worth deprecating and removing at some point.
+        if isinstance(l, Exception) or l < 0:
+            return l
+        self.offset += l
+        # If there is nothing left to send,
+        if self.offset == len(self.dataBuffer) and not self._tempDataLen:
+            self.dataBuffer = b""
+            self.offset = 0
+            # stop writing.
+            self.stopWriting()
+            # If I've got a producer who is supposed to supply me with data,
+            if self.producer is not None and (
+                (not self.streamingProducer) or self.producerPaused
+            ):
+                # tell them to supply some more.
+                self.producerPaused = False
+                self.producer.resumeProducing()
+            elif self.disconnecting:
+                # But if I was previously asked to let the connection die, do
+                # so.
+                return self._postLoseConnection()
+            elif self._writeDisconnecting:
+                # I was previously asked to half-close the connection.  We
+                # set _writeDisconnected before calling handler, in case the
+                # handler calls loseConnection(), which will want to check for
+                # this attribute.
+                self._writeDisconnected = True
+                result = self._closeWriteConnection()
+                return result
+        return None
+
+    def _postLoseConnection(self):
+        """Called after a loseConnection(), when all data has been written.
+
+        Whatever this returns is then returned by doWrite.
+        """
+        # default implementation, telling reactor we're finished
+        return main.CONNECTION_DONE
+
+    def _closeWriteConnection(self):
+        # override in subclasses
+        pass
+
+    def writeConnectionLost(self, reason):
+        # in current code should never be called
+        self.connectionLost(reason)
+
+    def readConnectionLost(self, reason: failure.Failure) -> None:
+        # override in subclasses
+        self.connectionLost(reason)
+
+    def getHost(self):
+        # ITransport.getHost
+        raise NotImplementedError()
+
+    def getPeer(self):
+        # ITransport.getPeer
+        raise NotImplementedError()
+
+    def _isSendBufferFull(self):
+        """
+        Determine whether the user-space send buffer for this transport is full
+        or not.
+
+        When the buffer contains more than C{self.bufferSize} bytes, it is
+        considered full.  This might be improved by considering the size of the
+        kernel send buffer and how much of it is free.
+
+        @return: C{True} if it is full, C{False} otherwise.
+        """
+        return len(self.dataBuffer) + self._tempDataLen > self.bufferSize
+
+    def _maybePauseProducer(self):
+        """
+        Possibly pause a producer, if there is one and the send buffer is full.
+        """
+        # If we are responsible for pausing our producer,
+        if self.producer is not None and self.streamingProducer:
+            # and our buffer is full,
+            if self._isSendBufferFull():
+                # pause it.
+                self.producerPaused = True
+                self.producer.pauseProducing()
+
+    def write(self, data: bytes) -> None:
+        """Reliably write some data.
+
+        The data is buffered until the underlying file descriptor is ready
+        for writing. If there is more than C{self.bufferSize} data in the
+        buffer and this descriptor has a registered streaming producer, its
+        C{pauseProducing()} method will be called.
+        """
+        _dataMustBeBytes(data)
+        if not self.connected or self._writeDisconnected:
+            return
+        if data:
+            self._tempDataBuffer.append(data)
+            self._tempDataLen += len(data)
+            self._maybePauseProducer()
+            self.startWriting()
+
+    def writeSequence(self, iovec: Iterable[bytes]) -> None:
+        """
+        Reliably write a sequence of data.
+
+        Currently, this is a convenience method roughly equivalent to::
+
+            for chunk in iovec:
+                fd.write(chunk)
+
+        It may have a more efficient implementation at a later time or in a
+        different reactor.
+
+        As with the C{write()} method, if a buffer size limit is reached and a
+        streaming producer is registered, it will be paused until the buffered
+        data is written to the underlying file descriptor.
+        """
+        for i in iovec:
+            _dataMustBeBytes(i)
+        if not self.connected or not iovec or self._writeDisconnected:
+            return
+        self._tempDataBuffer.extend(iovec)
+        for i in iovec:
+            self._tempDataLen += len(i)
+        self._maybePauseProducer()
+        self.startWriting()
+
+    def loseConnection(self):
+        """Close the connection at the next available opportunity.
+
+        Call this to cause this FileDescriptor to lose its connection.  It will
+        first write any data that it has buffered.
+
+        If there is data buffered yet to be written, this method will cause the
+        transport to lose its connection as soon as it's done flushing its
+        write buffer.  If you have a producer registered, the connection won't
+        be closed until the producer is finished. Therefore, make sure you
+        unregister your producer when it's finished, or the connection will
+        never close.
+        """
+
+        if self.connected and not self.disconnecting:
+            if self._writeDisconnected:
+                # doWrite won't trigger the connection close anymore
+                self.stopReading()
+                self.stopWriting()
+                self.connectionLost(failure.Failure(main.CONNECTION_DONE))
+            else:
+                self.stopReading()
+                self.startWriting()
+                self.disconnecting = 1
+
+    def loseWriteConnection(self):
+        self._writeDisconnecting = True
+        self.startWriting()
+
+    def stopReading(self):
+        """Stop waiting for read availability.
+
+        Call this to remove this selectable from being notified when it is
+        ready for reading.
+        """
+        self.reactor.removeReader(self)
+
+    def stopWriting(self):
+        """Stop waiting for write availability.
+
+        Call this to remove this selectable from being notified when it is ready
+        for writing.
+        """
+        self.reactor.removeWriter(self)
+
+    def startReading(self):
+        """Start waiting for read availability."""
+        self.reactor.addReader(self)
+
+    def startWriting(self):
+        """Start waiting for write availability.
+
+        Call this to have this FileDescriptor be notified whenever it is ready for
+        writing.
+        """
+        self.reactor.addWriter(self)
+
+    # Producer/consumer implementation
+
+    # first, the consumer stuff.  This requires no additional work, as
+    # any object you can write to can be a consumer, really.
+
+    producer = None
+    bufferSize = 2**2**2**2
+
+    def stopConsuming(self):
+        """Stop consuming data.
+
+        This is called when a producer has lost its connection, to tell the
+        consumer to go lose its connection (and break potential circular
+        references).
+        """
+        self.unregisterProducer()
+        self.loseConnection()
+
+    # producer interface implementation
+
+    def resumeProducing(self):
+        if self.connected and not self.disconnecting:
+            self.startReading()
+
+    def pauseProducing(self):
+        self.stopReading()
+
+    def stopProducing(self):
+        self.loseConnection()
+
+    def fileno(self):
+        """File Descriptor number for select().
+
+        This method must be overridden or assigned in subclasses to
+        indicate a valid file descriptor for the operating system.
+        """
+        return -1
+
+
+def isIPAddress(addr: str, family: int = AF_INET) -> bool:
+    """
+    Determine whether the given string represents an IP address of the given
+    family; by default, an IPv4 address.
+
+    @param addr: A string which may or may not be the decimal dotted
+        representation of an IPv4 address.
+    @param family: The address family to test for; one of the C{AF_*} constants
+        from the L{socket} module.  (This parameter has only been available
+        since Twisted 17.1.0; previously L{isIPAddress} could only test for IPv4
+        addresses.)
+
+    @return: C{True} if C{addr} represents an IPv4 address, C{False} otherwise.
+    """
+    if isinstance(addr, bytes):  # type: ignore[unreachable]
+        try:  # type: ignore[unreachable]
+            addr = addr.decode("ascii")
+        except UnicodeDecodeError:
+            return False
+    if family == AF_INET6:
+        # On some platforms, inet_ntop fails unless the scope ID is valid; this
+        # is a test for whether the given string *is* an IP address, so strip
+        # any potential scope ID before checking.
+        addr = addr.split("%", 1)[0]
+    elif family == AF_INET:
+        # On Windows, where 3.5+ implement inet_pton, "0" is considered a valid
+        # IPv4 address, but we want to ensure we have all 4 segments.
+        if addr.count(".") != 3:
+            return False
+    else:
+        raise ValueError(f"unknown address family {family!r}")
+    try:
+        # This might be a native implementation or the one from
+        # twisted.python.compat.
+        inet_pton(family, addr)
+    except (ValueError, OSError):
+        return False
+    return True
+
+
+def isIPv6Address(addr: str) -> bool:
+    """
+    Determine whether the given string represents an IPv6 address.
+
+    @param addr: A string which may or may not be the hex
+        representation of an IPv6 address.
+    @type addr: C{str}
+
+    @return: C{True} if C{addr} represents an IPv6 address, C{False}
+        otherwise.
+    @rtype: C{bool}
+    """
+    return isIPAddress(addr, AF_INET6)
+
+
+__all__ = ["FileDescriptor", "isIPAddress", "isIPv6Address"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/address.py b/contrib/python/Twisted/py3/twisted/internet/address.py
new file mode 100644
index 00000000000..10fa85241e5
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/address.py
@@ -0,0 +1,182 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Address objects for network connections.
+"""
+
+
+import os
+from typing import Optional, Union
+from warnings import warn
+
+from zope.interface import implementer
+
+import attr
+from typing_extensions import Literal
+
+from twisted.internet.interfaces import IAddress
+from twisted.python.filepath import _asFilesystemBytes, _coerceToFilesystemEncoding
+from twisted.python.runtime import platform
+
+
+@implementer(IAddress)
+@attr.s(hash=True, auto_attribs=True)
+class IPv4Address:
+    """
+    An L{IPv4Address} represents the address of an IPv4 socket endpoint.
+
+    @ivar type: A string describing the type of transport, either 'TCP' or
+        'UDP'.
+
+    @ivar host: A string containing a dotted-quad IPv4 address; for example,
+        "127.0.0.1".
+    @type host: C{str}
+
+    @ivar port: An integer representing the port number.
+    @type port: C{int}
+    """
+
+    type: Union[Literal["TCP"], Literal["UDP"]] = attr.ib(
+        validator=attr.validators.in_(["TCP", "UDP"])
+    )
+    host: str
+    port: int
+
+
+@implementer(IAddress)
+@attr.s(hash=True, auto_attribs=True)
+class IPv6Address:
+    """
+    An L{IPv6Address} represents the address of an IPv6 socket endpoint.
+
+    @ivar type: A string describing the type of transport, either 'TCP' or
+        'UDP'.
+
+    @ivar host: A string containing a colon-separated, hexadecimal formatted
+        IPv6 address; for example, "::1".
+    @type host: C{str}
+
+    @ivar port: An integer representing the port number.
+    @type port: C{int}
+
+    @ivar flowInfo: the IPv6 flow label.  This can be used by QoS routers to
+        identify flows of traffic; you may generally safely ignore it.
+    @type flowInfo: L{int}
+
+    @ivar scopeID: the IPv6 scope identifier - roughly analagous to what
+        interface traffic destined for this address must be transmitted over.
+    @type scopeID: L{int} or L{str}
+    """
+
+    type: Union[Literal["TCP"], Literal["UDP"]] = attr.ib(
+        validator=attr.validators.in_(["TCP", "UDP"])
+    )
+    host: str
+    port: int
+    flowInfo: int = 0
+    scopeID: Union[str, int] = 0
+
+
+@implementer(IAddress)
+class _ProcessAddress:
+    """
+    An L{interfaces.IAddress} provider for process transports.
+    """
+
+
+@attr.s(hash=True, auto_attribs=True)
+@implementer(IAddress)
+class HostnameAddress:
+    """
+    A L{HostnameAddress} represents the address of a L{HostnameEndpoint}.
+
+    @ivar hostname: A hostname byte string; for example, b"example.com".
+    @type hostname: L{bytes}
+
+    @ivar port: An integer representing the port number.
+    @type port: L{int}
+    """
+
+    hostname: bytes
+    port: int
+
+
+@attr.s(hash=False, repr=False, eq=False, auto_attribs=True)
+@implementer(IAddress)
+class UNIXAddress:
+    """
+    Object representing a UNIX socket endpoint.
+
+    @ivar name: The filename associated with this socket.
+    @type name: C{bytes}
+    """
+
+    name: Optional[bytes] = attr.ib(
+        converter=attr.converters.optional(_asFilesystemBytes)
+    )
+
+    if getattr(os.path, "samefile", None) is not None:
+
+        def __eq__(self, other: object) -> bool:
+            """
+            Overriding C{attrs} to ensure the os level samefile
+            check is done if the name attributes do not match.
+            """
+            if not isinstance(other, self.__class__):
+                return NotImplemented
+            res = self.name == other.name
+            if not res and self.name and other.name:
+                try:
+                    return os.path.samefile(self.name, other.name)
+                except OSError:
+                    pass
+                except (TypeError, ValueError) as e:
+                    # On Linux, abstract namespace UNIX sockets start with a
+                    # \0, which os.path doesn't like.
+                    if not platform.isLinux():
+                        raise e
+            return res
+
+    else:
+
+        def __eq__(self, other: object) -> bool:
+            if isinstance(other, self.__class__):
+                return self.name == other.name
+            return NotImplemented
+
+    def __repr__(self) -> str:
+        name = self.name
+        show = _coerceToFilesystemEncoding("", name) if name is not None else None
+        return f"UNIXAddress({show!r})"
+
+    def __hash__(self):
+        if self.name is None:
+            return hash((self.__class__, None))
+        try:
+            s1 = os.stat(self.name)
+            return hash((s1.st_ino, s1.st_dev))
+        except OSError:
+            return hash(self.name)
+
+
+# These are for buildFactory backwards compatibility due to
+# stupidity-induced inconsistency.
+
+
+class _ServerFactoryIPv4Address(IPv4Address):
+    """Backwards compatibility hack. Just like IPv4Address in practice."""
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, tuple):
+            warn(
+                "IPv4Address.__getitem__ is deprecated.  " "Use attributes instead.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            return (self.host, self.port) == other
+        elif isinstance(other, IPv4Address):
+            a = (self.type, self.host, self.port)
+            b = (other.type, other.host, other.port)
+            return a == b
+        return NotImplemented
diff --git a/contrib/python/Twisted/py3/twisted/internet/asyncioreactor.py b/contrib/python/Twisted/py3/twisted/internet/asyncioreactor.py
new file mode 100644
index 00000000000..cd1cf65f05d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/asyncioreactor.py
@@ -0,0 +1,307 @@
+# -*- test-case-name: twisted.test.test_internet -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+asyncio-based reactor implementation.
+"""
+
+
+import errno
+import sys
+from asyncio import AbstractEventLoop, get_event_loop
+from typing import Dict, Optional, Type
+
+from zope.interface import implementer
+
+from twisted.internet.abstract import FileDescriptor
+from twisted.internet.interfaces import IReactorFDSet
+from twisted.internet.posixbase import (
+    _NO_FILEDESC,
+    PosixReactorBase,
+    _ContinuousPolling,
+)
+from twisted.logger import Logger
+from twisted.python.log import callWithLogger
+
+
+@implementer(IReactorFDSet)
+class AsyncioSelectorReactor(PosixReactorBase):
+    """
+    Reactor running on top of L{asyncio.SelectorEventLoop}.
+
+    On POSIX platforms, the default event loop is
+    L{asyncio.SelectorEventLoop}.
+    On Windows, the default event loop on Python 3.7 and older
+    is C{asyncio.WindowsSelectorEventLoop}, but on Python 3.8 and newer
+    the default event loop is C{asyncio.WindowsProactorEventLoop} which
+    is incompatible with L{AsyncioSelectorReactor}.
+    Applications that use L{AsyncioSelectorReactor} on Windows
+    with Python 3.8+ must call
+    C{asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())}
+    before instantiating and running L{AsyncioSelectorReactor}.
+    """
+
+    _asyncClosed = False
+    _log = Logger()
+
+    def __init__(self, eventloop: Optional[AbstractEventLoop] = None):
+        if eventloop is None:
+            _eventloop: AbstractEventLoop = get_event_loop()
+        else:
+            _eventloop = eventloop
+
+        # On Python 3.8+, asyncio.get_event_loop() on
+        # Windows was changed to return a ProactorEventLoop
+        # unless the loop policy has been changed.
+        if sys.platform == "win32":
+            from asyncio import ProactorEventLoop
+
+            if isinstance(_eventloop, ProactorEventLoop):
+                raise TypeError(
+                    f"ProactorEventLoop is not supported, got: {_eventloop}"
+                )
+
+        self._asyncioEventloop: AbstractEventLoop = _eventloop
+        self._writers: Dict[Type[FileDescriptor], int] = {}
+        self._readers: Dict[Type[FileDescriptor], int] = {}
+        self._continuousPolling = _ContinuousPolling(self)
+
+        self._scheduledAt = None
+        self._timerHandle = None
+
+        super().__init__()
+
+    def _unregisterFDInAsyncio(self, fd):
+        """
+        Compensate for a bug in asyncio where it will not unregister a FD that
+        it cannot handle in the epoll loop. It touches internal asyncio code.
+
+        A description of the bug by markrwilliams:
+
+        The C{add_writer} method of asyncio event loops isn't atomic because
+        all the Selector classes in the selector module internally record a
+        file object before passing it to the platform's selector
+        implementation. If the platform's selector decides the file object
+        isn't acceptable, the resulting exception doesn't cause the Selector to
+        un-track the file object.
+
+        The failing/hanging stdio test goes through the following sequence of
+        events (roughly):
+
+        * The first C{connection.write(intToByte(value))} call hits the asyncio
+        reactor's C{addWriter} method.
+
+        * C{addWriter} calls the asyncio loop's C{add_writer} method, which
+        happens to live on C{_BaseSelectorEventLoop}.
+
+        * The asyncio loop's C{add_writer} method checks if the file object has
+        been registered before via the selector's C{get_key} method.
+
+        * It hasn't, so the KeyError block runs and calls the selector's
+        register method
+
+        * Code examples that follow use EpollSelector, but the code flow holds
+        true for any other selector implementation. The selector's register
+        method first calls through to the next register method in the MRO
+
+        * That next method is always C{_BaseSelectorImpl.register} which
+        creates a C{SelectorKey} instance for the file object, stores it under
+        the file object's file descriptor, and then returns it.
+
+        * Control returns to the concrete selector implementation, which asks
+        the operating system to track the file descriptor using the right API.
+
+        * The operating system refuses! An exception is raised that, in this
+        case, the asyncio reactor handles by creating a C{_ContinuousPolling}
+        object to watch the file descriptor.
+
+        * The second C{connection.write(intToByte(value))} call hits the
+        asyncio reactor's C{addWriter} method, which hits the C{add_writer}
+        method. But the loop's selector's get_key method now returns a
+        C{SelectorKey}! Now the asyncio reactor's C{addWriter} method thinks
+        the asyncio loop will watch the file descriptor, even though it won't.
+        """
+        try:
+            self._asyncioEventloop._selector.unregister(fd)
+        except BaseException:
+            pass
+
+    def _readOrWrite(self, selectable, read):
+        method = selectable.doRead if read else selectable.doWrite
+
+        if selectable.fileno() == -1:
+            self._disconnectSelectable(selectable, _NO_FILEDESC, read)
+            return
+
+        try:
+            why = method()
+        except Exception as e:
+            why = e
+            self._log.failure(None)
+        if why:
+            self._disconnectSelectable(selectable, why, read)
+
+    def addReader(self, reader):
+        if reader in self._readers.keys() or reader in self._continuousPolling._readers:
+            return
+
+        fd = reader.fileno()
+        try:
+            self._asyncioEventloop.add_reader(
+                fd, callWithLogger, reader, self._readOrWrite, reader, True
+            )
+            self._readers[reader] = fd
+        except OSError as e:
+            self._unregisterFDInAsyncio(fd)
+            if e.errno == errno.EPERM:
+                # epoll(7) doesn't support certain file descriptors,
+                # e.g. filesystem files, so for those we just poll
+                # continuously:
+                self._continuousPolling.addReader(reader)
+            else:
+                raise
+
+    def addWriter(self, writer):
+        if writer in self._writers.keys() or writer in self._continuousPolling._writers:
+            return
+
+        fd = writer.fileno()
+        try:
+            self._asyncioEventloop.add_writer(
+                fd, callWithLogger, writer, self._readOrWrite, writer, False
+            )
+            self._writers[writer] = fd
+        except PermissionError:
+            self._unregisterFDInAsyncio(fd)
+            # epoll(7) doesn't support certain file descriptors,
+            # e.g. filesystem files, so for those we just poll
+            # continuously:
+            self._continuousPolling.addWriter(writer)
+        except BrokenPipeError:
+            # The kqueuereactor will raise this if there is a broken pipe
+            self._unregisterFDInAsyncio(fd)
+        except BaseException:
+            self._unregisterFDInAsyncio(fd)
+            raise
+
+    def removeReader(self, reader):
+        # First, see if they're trying to remove a reader that we don't have.
+        if not (
+            reader in self._readers.keys() or self._continuousPolling.isReading(reader)
+        ):
+            # We don't have it, so just return OK.
+            return
+
+        # If it was a cont. polling reader, check there first.
+        if self._continuousPolling.isReading(reader):
+            self._continuousPolling.removeReader(reader)
+            return
+
+        fd = reader.fileno()
+        if fd == -1:
+            # If the FD is -1, we want to know what its original FD was, to
+            # remove it.
+            fd = self._readers.pop(reader)
+        else:
+            self._readers.pop(reader)
+
+        self._asyncioEventloop.remove_reader(fd)
+
+    def removeWriter(self, writer):
+        # First, see if they're trying to remove a writer that we don't have.
+        if not (
+            writer in self._writers.keys() or self._continuousPolling.isWriting(writer)
+        ):
+            # We don't have it, so just return OK.
+            return
+
+        # If it was a cont. polling writer, check there first.
+        if self._continuousPolling.isWriting(writer):
+            self._continuousPolling.removeWriter(writer)
+            return
+
+        fd = writer.fileno()
+
+        if fd == -1:
+            # If the FD is -1, we want to know what its original FD was, to
+            # remove it.
+            fd = self._writers.pop(writer)
+        else:
+            self._writers.pop(writer)
+
+        self._asyncioEventloop.remove_writer(fd)
+
+    def removeAll(self):
+        return (
+            self._removeAll(self._readers.keys(), self._writers.keys())
+            + self._continuousPolling.removeAll()
+        )
+
+    def getReaders(self):
+        return list(self._readers.keys()) + self._continuousPolling.getReaders()
+
+    def getWriters(self):
+        return list(self._writers.keys()) + self._continuousPolling.getWriters()
+
+    def iterate(self, timeout):
+        self._asyncioEventloop.call_later(timeout + 0.01, self._asyncioEventloop.stop)
+        self._asyncioEventloop.run_forever()
+
+    def run(self, installSignalHandlers=True):
+        self.startRunning(installSignalHandlers=installSignalHandlers)
+        self._asyncioEventloop.run_forever()
+        if self._justStopped:
+            self._justStopped = False
+
+    def stop(self):
+        super().stop()
+        # This will cause runUntilCurrent which in its turn
+        # will call fireSystemEvent("shutdown")
+        self.callLater(0, lambda: None)
+
+    def crash(self):
+        super().crash()
+        self._asyncioEventloop.stop()
+
+    def _onTimer(self):
+        self._scheduledAt = None
+        self.runUntilCurrent()
+        self._reschedule()
+
+    def _reschedule(self):
+        timeout = self.timeout()
+        if timeout is not None:
+            abs_time = self._asyncioEventloop.time() + timeout
+            self._scheduledAt = abs_time
+            if self._timerHandle is not None:
+                self._timerHandle.cancel()
+            self._timerHandle = self._asyncioEventloop.call_at(abs_time, self._onTimer)
+
+    def _moveCallLaterSooner(self, tple):
+        PosixReactorBase._moveCallLaterSooner(self, tple)
+        self._reschedule()
+
+    def callLater(self, seconds, f, *args, **kwargs):
+        dc = PosixReactorBase.callLater(self, seconds, f, *args, **kwargs)
+        abs_time = self._asyncioEventloop.time() + self.timeout()
+        if self._scheduledAt is None or abs_time < self._scheduledAt:
+            self._reschedule()
+        return dc
+
+    def callFromThread(self, f, *args, **kwargs):
+        g = lambda: self.callLater(0, f, *args, **kwargs)
+        self._asyncioEventloop.call_soon_threadsafe(g)
+
+
+def install(eventloop=None):
+    """
+    Install an asyncio-based reactor.
+
+    @param eventloop: The asyncio eventloop to wrap. If default, the global one
+        is selected.
+    """
+    reactor = AsyncioSelectorReactor(eventloop)
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
diff --git a/contrib/python/Twisted/py3/twisted/internet/base.py b/contrib/python/Twisted/py3/twisted/internet/base.py
new file mode 100644
index 00000000000..f039dfe5c4e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/base.py
@@ -0,0 +1,1345 @@
+# -*- test-case-name: twisted.test.test_internet,twisted.internet.test.test_core -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Very basic functionality for a Reactor implementation.
+"""
+
+
+import builtins
+import socket  # needed only for sync-dns
+import warnings
+from abc import ABC, abstractmethod
+from heapq import heapify, heappop, heappush
+from traceback import format_stack
+from types import FrameType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    List,
+    NewType,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
+
+from zope.interface import classImplements, implementer
+
+from twisted.internet import abstract, defer, error, fdesc, main, threads
+from twisted.internet._resolver import (
+    ComplexResolverSimplifier as _ComplexResolverSimplifier,
+    GAIResolver as _GAIResolver,
+    SimpleResolverComplexifier as _SimpleResolverComplexifier,
+)
+from twisted.internet.defer import Deferred, DeferredList
+from twisted.internet.interfaces import (
+    IAddress,
+    IConnector,
+    IDelayedCall,
+    IHostnameResolver,
+    IProtocol,
+    IReactorCore,
+    IReactorFromThreads,
+    IReactorPluggableNameResolver,
+    IReactorPluggableResolver,
+    IReactorThreads,
+    IReactorTime,
+    IReadDescriptor,
+    IResolverSimple,
+    IWriteDescriptor,
+    _ISupportsExitSignalCapturing,
+)
+from twisted.internet.protocol import ClientFactory
+from twisted.python import log, reflect
+from twisted.python.failure import Failure
+from twisted.python.runtime import platform, seconds as runtimeSeconds
+from ._signals import SignalHandling, _WithoutSignalHandling, _WithSignalHandling
+
+if TYPE_CHECKING:
+    from twisted.internet.tcp import Client
+
+# This import is for side-effects!  Even if you don't see any code using it
+# in this module, don't delete it.
+from twisted.python import threadable
+
+if platform.supportsThreads():
+    from twisted.python.threadpool import ThreadPool
+else:
+    ThreadPool = None  # type: ignore[misc, assignment]
+
+
+@implementer(IDelayedCall)
+class DelayedCall:
+    # enable .debug to record creator call stack, and it will be logged if
+    # an exception occurs while the function is being run
+    debug = False
+    _repr: Optional[str] = None
+
+    # In debug mode, the call stack at the time of instantiation.
+    creator: Optional[Sequence[str]] = None
+
+    def __init__(
+        self,
+        time: float,
+        func: Callable[..., Any],
+        args: Sequence[object],
+        kw: Dict[str, object],
+        cancel: Callable[["DelayedCall"], None],
+        reset: Callable[["DelayedCall"], None],
+        seconds: Callable[[], float] = runtimeSeconds,
+    ) -> None:
+        """
+        @param time: Seconds from the epoch at which to call C{func}.
+        @param func: The callable to call.
+        @param args: The positional arguments to pass to the callable.
+        @param kw: The keyword arguments to pass to the callable.
+        @param cancel: A callable which will be called with this
+            DelayedCall before cancellation.
+        @param reset: A callable which will be called with this
+            DelayedCall after changing this DelayedCall's scheduled
+            execution time. The callable should adjust any necessary
+            scheduling details to ensure this DelayedCall is invoked
+            at the new appropriate time.
+        @param seconds: If provided, a no-argument callable which will be
+            used to determine the current time any time that information is
+            needed.
+        """
+        self.time, self.func, self.args, self.kw = time, func, args, kw
+        self.resetter = reset
+        self.canceller = cancel
+        self.seconds = seconds
+        self.cancelled = self.called = 0
+        self.delayed_time = 0.0
+        if self.debug:
+            self.creator = format_stack()[:-2]
+
+    def getTime(self) -> float:
+        """
+        Return the time at which this call will fire
+
+        @return: The number of seconds after the epoch at which this call is
+            scheduled to be made.
+        """
+        return self.time + self.delayed_time
+
+    def cancel(self) -> None:
+        """
+        Unschedule this call
+
+        @raise AlreadyCancelled: Raised if this call has already been
+            unscheduled.
+
+        @raise AlreadyCalled: Raised if this call has already been made.
+        """
+        if self.cancelled:
+            raise error.AlreadyCancelled
+        elif self.called:
+            raise error.AlreadyCalled
+        else:
+            self.canceller(self)
+            self.cancelled = 1
+            if self.debug:
+                self._repr = repr(self)
+            del self.func, self.args, self.kw
+
+    def reset(self, secondsFromNow: float) -> None:
+        """
+        Reschedule this call for a different time
+
+        @param secondsFromNow: The number of seconds from the time of the
+            C{reset} call at which this call will be scheduled.
+
+        @raise AlreadyCancelled: Raised if this call has been cancelled.
+        @raise AlreadyCalled: Raised if this call has already been made.
+        """
+        if self.cancelled:
+            raise error.AlreadyCancelled
+        elif self.called:
+            raise error.AlreadyCalled
+        else:
+            newTime = self.seconds() + secondsFromNow
+            if newTime < self.time:
+                self.delayed_time = 0.0
+                self.time = newTime
+                self.resetter(self)
+            else:
+                self.delayed_time = newTime - self.time
+
+    def delay(self, secondsLater: float) -> None:
+        """
+        Reschedule this call for a later time
+
+        @param secondsLater: The number of seconds after the originally
+            scheduled time for which to reschedule this call.
+
+        @raise AlreadyCancelled: Raised if this call has been cancelled.
+        @raise AlreadyCalled: Raised if this call has already been made.
+        """
+        if self.cancelled:
+            raise error.AlreadyCancelled
+        elif self.called:
+            raise error.AlreadyCalled
+        else:
+            self.delayed_time += secondsLater
+            if self.delayed_time < 0.0:
+                self.activate_delay()
+                self.resetter(self)
+
+    def activate_delay(self) -> None:
+        self.time += self.delayed_time
+        self.delayed_time = 0.0
+
+    def active(self) -> bool:
+        """Determine whether this call is still pending
+
+        @return: True if this call has not yet been made or cancelled,
+            False otherwise.
+        """
+        return not (self.cancelled or self.called)
+
+    def __le__(self, other: object) -> bool:
+        """
+        Implement C{<=} operator between two L{DelayedCall} instances.
+
+        Comparison is based on the C{time} attribute (unadjusted by the
+        delayed time).
+        """
+        if isinstance(other, DelayedCall):
+            return self.time <= other.time
+        else:
+            return NotImplemented
+
+    def __lt__(self, other: object) -> bool:
+        """
+        Implement C{<} operator between two L{DelayedCall} instances.
+
+        Comparison is based on the C{time} attribute (unadjusted by the
+        delayed time).
+        """
+        if isinstance(other, DelayedCall):
+            return self.time < other.time
+        else:
+            return NotImplemented
+
+    def __repr__(self) -> str:
+        """
+        Implement C{repr()} for L{DelayedCall} instances.
+
+        @returns: String containing details of the L{DelayedCall}.
+        """
+        if self._repr is not None:
+            return self._repr
+        if hasattr(self, "func"):
+            # This code should be replaced by a utility function in reflect;
+            # see ticket #6066:
+            func = getattr(self.func, "__qualname__", None)
+            if func is None:
+                func = getattr(self.func, "__name__", None)
+                if func is not None:
+                    imClass = getattr(self.func, "im_class", None)
+                    if imClass is not None:
+                        func = f"{imClass}.{func}"
+            if func is None:
+                func = reflect.safe_repr(self.func)
+        else:
+            func = None
+
+        now = self.seconds()
+        L = [
+            "<DelayedCall 0x%x [%ss] called=%s cancelled=%s"
+            % (id(self), self.time - now, self.called, self.cancelled)
+        ]
+        if func is not None:
+            L.extend((" ", func, "("))
+            if self.args:
+                L.append(", ".join([reflect.safe_repr(e) for e in self.args]))
+                if self.kw:
+                    L.append(", ")
+            if self.kw:
+                L.append(
+                    ", ".join(
+                        [f"{k}={reflect.safe_repr(v)}" for (k, v) in self.kw.items()]
+                    )
+                )
+            L.append(")")
+
+        if self.creator is not None:
+            L.append("\n\ntraceback at creation: \n\n%s" % ("    ".join(self.creator)))
+        L.append(">")
+
+        return "".join(L)
+
+
+@implementer(IResolverSimple)
+class ThreadedResolver:
+    """
+    L{ThreadedResolver} uses a reactor, a threadpool, and
+    L{socket.gethostbyname} to perform name lookups without blocking the
+    reactor thread.  It also supports timeouts indepedently from whatever
+    timeout logic L{socket.gethostbyname} might have.
+
+    @ivar reactor: The reactor the threadpool of which will be used to call
+        L{socket.gethostbyname} and the I/O thread of which the result will be
+        delivered.
+    """
+
+    def __init__(self, reactor: "ReactorBase") -> None:
+        self.reactor = reactor
+        self._runningQueries: Dict[
+            Deferred[str], Tuple[Deferred[str], IDelayedCall]
+        ] = {}
+
+    def _fail(self, name: str, err: str) -> Failure:
+        lookupError = error.DNSLookupError(f"address {name!r} not found: {err}")
+        return Failure(lookupError)
+
+    def _cleanup(self, name: str, lookupDeferred: Deferred[str]) -> None:
+        userDeferred, cancelCall = self._runningQueries[lookupDeferred]
+        del self._runningQueries[lookupDeferred]
+        userDeferred.errback(self._fail(name, "timeout error"))
+
+    def _checkTimeout(
+        self, result: Union[str, Failure], name: str, lookupDeferred: Deferred[str]
+    ) -> None:
+        try:
+            userDeferred, cancelCall = self._runningQueries[lookupDeferred]
+        except KeyError:
+            pass
+        else:
+            del self._runningQueries[lookupDeferred]
+            cancelCall.cancel()
+
+            if isinstance(result, Failure):
+                userDeferred.errback(self._fail(name, result.getErrorMessage()))
+            else:
+                userDeferred.callback(result)
+
+    def getHostByName(
+        self, name: str, timeout: Sequence[int] = (1, 3, 11, 45)
+    ) -> Deferred[str]:
+        """
+        See L{twisted.internet.interfaces.IResolverSimple.getHostByName}.
+
+        Note that the elements of C{timeout} are summed and the result is used
+        as a timeout for the lookup.  Any intermediate timeout or retry logic
+        is left up to the platform via L{socket.gethostbyname}.
+        """
+        if timeout:
+            timeoutDelay = sum(timeout)
+        else:
+            timeoutDelay = 60
+        userDeferred: Deferred[str] = Deferred()
+        lookupDeferred = threads.deferToThreadPool(
+            cast(IReactorFromThreads, self.reactor),
+            cast(IReactorThreads, self.reactor).getThreadPool(),
+            socket.gethostbyname,
+            name,
+        )
+        cancelCall = cast(IReactorTime, self.reactor).callLater(
+            timeoutDelay, self._cleanup, name, lookupDeferred
+        )
+        self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
+        _: Deferred[None] = lookupDeferred.addBoth(
+            self._checkTimeout, name, lookupDeferred
+        )
+        return userDeferred
+
+
+@implementer(IResolverSimple)
+class BlockingResolver:
+    def getHostByName(
+        self, name: str, timeout: Sequence[int] = (1, 3, 11, 45)
+    ) -> Deferred[str]:
+        try:
+            address = socket.gethostbyname(name)
+        except OSError:
+            msg = f"address {name!r} not found"
+            err = error.DNSLookupError(msg)
+            return defer.fail(err)
+        else:
+            return defer.succeed(address)
+
+
+_ThreePhaseEventTriggerCallable = Callable[..., Any]
+_ThreePhaseEventTrigger = Tuple[
+    _ThreePhaseEventTriggerCallable, Tuple[object, ...], Dict[str, object]
+]
+_ThreePhaseEventTriggerHandle = NewType(
+    "_ThreePhaseEventTriggerHandle",
+    Tuple[str, _ThreePhaseEventTriggerCallable, Tuple[object, ...], Dict[str, object]],
+)
+
+
+class _ThreePhaseEvent:
+    """
+    Collection of callables (with arguments) which can be invoked as a group in
+    a particular order.
+
+    This provides the underlying implementation for the reactor's system event
+    triggers.  An instance of this class tracks triggers for all phases of a
+    single type of event.
+
+    @ivar before: A list of the before-phase triggers containing three-tuples
+        of a callable, a tuple of positional arguments, and a dict of keyword
+        arguments
+
+    @ivar finishedBefore: A list of the before-phase triggers which have
+        already been executed.  This is only populated in the C{'BEFORE'} state.
+
+    @ivar during: A list of the during-phase triggers containing three-tuples
+        of a callable, a tuple of positional arguments, and a dict of keyword
+        arguments
+
+    @ivar after: A list of the after-phase triggers containing three-tuples
+        of a callable, a tuple of positional arguments, and a dict of keyword
+        arguments
+
+    @ivar state: A string indicating what is currently going on with this
+        object.  One of C{'BASE'} (for when nothing in particular is happening;
+        this is the initial value), C{'BEFORE'} (when the before-phase triggers
+        are in the process of being executed).
+    """
+
+    def __init__(self) -> None:
+        self.before: List[_ThreePhaseEventTrigger] = []
+        self.during: List[_ThreePhaseEventTrigger] = []
+        self.after: List[_ThreePhaseEventTrigger] = []
+        self.state = "BASE"
+
+    def addTrigger(
+        self,
+        phase: str,
+        callable: _ThreePhaseEventTriggerCallable,
+        *args: object,
+        **kwargs: object,
+    ) -> _ThreePhaseEventTriggerHandle:
+        """
+        Add a trigger to the indicate phase.
+
+        @param phase: One of C{'before'}, C{'during'}, or C{'after'}.
+
+        @param callable: An object to be called when this event is triggered.
+        @param args: Positional arguments to pass to C{callable}.
+        @param kwargs: Keyword arguments to pass to C{callable}.
+
+        @return: An opaque handle which may be passed to L{removeTrigger} to
+            reverse the effects of calling this method.
+        """
+        if phase not in ("before", "during", "after"):
+            raise KeyError("invalid phase")
+        getattr(self, phase).append((callable, args, kwargs))
+        return _ThreePhaseEventTriggerHandle((phase, callable, args, kwargs))
+
+    def removeTrigger(self, handle: _ThreePhaseEventTriggerHandle) -> None:
+        """
+        Remove a previously added trigger callable.
+
+        @param handle: An object previously returned by L{addTrigger}.  The
+            trigger added by that call will be removed.
+
+        @raise ValueError: If the trigger associated with C{handle} has already
+            been removed or if C{handle} is not a valid handle.
+        """
+        getattr(self, "removeTrigger_" + self.state)(handle)
+
+    def removeTrigger_BASE(self, handle: _ThreePhaseEventTriggerHandle) -> None:
+        """
+        Just try to remove the trigger.
+
+        @see: removeTrigger
+        """
+        try:
+            phase, callable, args, kwargs = handle
+        except (TypeError, ValueError):
+            raise ValueError("invalid trigger handle")
+        else:
+            if phase not in ("before", "during", "after"):
+                raise KeyError("invalid phase")
+            getattr(self, phase).remove((callable, args, kwargs))
+
+    def removeTrigger_BEFORE(self, handle: _ThreePhaseEventTriggerHandle) -> None:
+        """
+        Remove the trigger if it has yet to be executed, otherwise emit a
+        warning that in the future an exception will be raised when removing an
+        already-executed trigger.
+
+        @see: removeTrigger
+        """
+        phase, callable, args, kwargs = handle
+        if phase != "before":
+            return self.removeTrigger_BASE(handle)
+        if (callable, args, kwargs) in self.finishedBefore:
+            warnings.warn(
+                "Removing already-fired system event triggers will raise an "
+                "exception in a future version of Twisted.",
+                category=DeprecationWarning,
+                stacklevel=3,
+            )
+        else:
+            self.removeTrigger_BASE(handle)
+
+    def fireEvent(self) -> None:
+        """
+        Call the triggers added to this event.
+        """
+        self.state = "BEFORE"
+        self.finishedBefore = []
+        beforeResults: List[Deferred[object]] = []
+        while self.before:
+            callable, args, kwargs = self.before.pop(0)
+            self.finishedBefore.append((callable, args, kwargs))
+            try:
+                result = callable(*args, **kwargs)
+            except BaseException:
+                log.err()
+            else:
+                if isinstance(result, Deferred):
+                    beforeResults.append(result)
+        DeferredList(beforeResults).addCallback(self._continueFiring)
+
+    def _continueFiring(self, ignored: object) -> None:
+        """
+        Call the during and after phase triggers for this event.
+        """
+        self.state = "BASE"
+        self.finishedBefore = []
+        for phase in self.during, self.after:
+            while phase:
+                callable, args, kwargs = phase.pop(0)
+                try:
+                    callable(*args, **kwargs)
+                except BaseException:
+                    log.err()
+
+
+@implementer(IReactorPluggableNameResolver, IReactorPluggableResolver)
+class PluggableResolverMixin:
+    """
+    A mixin which implements the pluggable resolver reactor interfaces.
+
+    @ivar resolver: The installed L{IResolverSimple}.
+    @ivar _nameResolver: The installed L{IHostnameResolver}.
+    """
+
+    resolver: IResolverSimple = BlockingResolver()
+    _nameResolver: IHostnameResolver = _SimpleResolverComplexifier(resolver)
+
+    # IReactorPluggableResolver
+    def installResolver(self, resolver: IResolverSimple) -> IResolverSimple:
+        """
+        See L{IReactorPluggableResolver}.
+
+        @param resolver: see L{IReactorPluggableResolver}.
+
+        @return: see L{IReactorPluggableResolver}.
+        """
+        assert IResolverSimple.providedBy(resolver)
+        oldResolver = self.resolver
+        self.resolver = resolver
+        self._nameResolver = _SimpleResolverComplexifier(resolver)
+        return oldResolver
+
+    # IReactorPluggableNameResolver
+    def installNameResolver(self, resolver: IHostnameResolver) -> IHostnameResolver:
+        """
+        See L{IReactorPluggableNameResolver}.
+
+        @param resolver: See L{IReactorPluggableNameResolver}.
+
+        @return: see L{IReactorPluggableNameResolver}.
+        """
+        previousNameResolver = self._nameResolver
+        self._nameResolver = resolver
+        self.resolver = _ComplexResolverSimplifier(resolver)
+        return previousNameResolver
+
+    @property
+    def nameResolver(self) -> IHostnameResolver:
+        """
+        Implementation of read-only
+        L{IReactorPluggableNameResolver.nameResolver}.
+        """
+        return self._nameResolver
+
+
+_SystemEventID = NewType("_SystemEventID", Tuple[str, _ThreePhaseEventTriggerHandle])
+_ThreadCall = Tuple[Callable[..., Any], Tuple[object, ...], Dict[str, object]]
+
+
+@implementer(IReactorCore, IReactorTime, _ISupportsExitSignalCapturing)
+class ReactorBase(PluggableResolverMixin):
+    """
+    Default base class for Reactors.
+
+    @ivar _stopped: A flag which is true between paired calls to C{reactor.run}
+        and C{reactor.stop}.  This should be replaced with an explicit state
+        machine.
+    @ivar _justStopped: A flag which is true between the time C{reactor.stop}
+        is called and the time the shutdown system event is fired.  This is
+        used to determine whether that event should be fired after each
+        iteration through the mainloop.  This should be replaced with an
+        explicit state machine.
+    @ivar _started: A flag which is true from the time C{reactor.run} is called
+        until the time C{reactor.run} returns.  This is used to prevent calls
+        to C{reactor.run} on a running reactor.  This should be replaced with
+        an explicit state machine.
+    @ivar running: See L{IReactorCore.running}
+    @ivar _registerAsIOThread: A flag controlling whether the reactor will
+        register the thread it is running in as the I/O thread when it starts.
+        If C{True}, registration will be done, otherwise it will not be.
+    @ivar _exitSignal: See L{_ISupportsExitSignalCapturing._exitSignal}
+
+    @ivar _installSignalHandlers: A flag which indicates whether any signal
+        handlers will be installed during startup.  This includes handlers for
+        SIGCHLD to monitor child processes, and SIGINT, SIGTERM, and SIGBREAK
+
+    @ivar _signals: An object which knows how to install and uninstall the
+        reactor's signal-handling behavior.
+    """
+
+    _registerAsIOThread = True
+
+    _stopped = True
+    installed = False
+    usingThreads = False
+    _exitSignal = None
+
+    # Set to something meaningful between startRunning and shortly before run
+    # returns.  We don't know the value to be used by `run` until that method
+    # itself is called and we learn the value of installSignalHandlers.
+    # However, we can use a no-op implementation until then.
+    _signals: SignalHandling = _WithoutSignalHandling()
+
+    __name__ = "twisted.internet.reactor"
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.threadCallQueue: List[_ThreadCall] = []
+        self._eventTriggers: Dict[str, _ThreePhaseEvent] = {}
+        self._pendingTimedCalls: List[DelayedCall] = []
+        self._newTimedCalls: List[DelayedCall] = []
+        self._cancellations = 0
+        self.running = False
+        self._started = False
+        self._justStopped = False
+        self._startedBefore = False
+        # reactor internal readers, e.g. the waker.
+        # Using Any as the type here… unable to find a suitable defined interface
+        self._internalReaders: Set[Any] = set()
+        self.waker: Any = None
+
+        # Arrange for the running attribute to change to True at the right time
+        # and let a subclass possibly do other things at that time (eg install
+        # signal handlers).
+        self.addSystemEventTrigger("during", "startup", self._reallyStartRunning)
+        self.addSystemEventTrigger("during", "shutdown", self.crash)
+        self.addSystemEventTrigger("during", "shutdown", self.disconnectAll)
+
+        if platform.supportsThreads():
+            self._initThreads()
+        self.installWaker()
+
+    # Signal handling pieces
+    _installSignalHandlers: bool = False
+
+    def _makeSignalHandling(self, installSignalHandlers: bool) -> SignalHandling:
+        """
+        Get an appropriate signal handling object.
+
+        @param installSignalHandlers: Indicate whether to even try to do any
+            signal handling.  If C{False} then the result will be a no-op
+            implementation.
+        """
+        if installSignalHandlers:
+            return self._signalsFactory()
+        return _WithoutSignalHandling()
+
+    def _signalsFactory(self) -> SignalHandling:
+        """
+        Get a signal handling object that implements the basic behavior of
+        stopping the reactor on SIGINT, SIGBREAK, and SIGTERM.
+        """
+        return _WithSignalHandling(
+            self.sigInt,
+            self.sigBreak,
+            self.sigTerm,
+        )
+
+    def _addInternalReader(self, reader: IReadDescriptor) -> None:
+        """
+        Add a read descriptor which is part of the implementation of the
+        reactor itself.
+
+        The read descriptor will not be removed by L{IReactorFDSet.removeAll}.
+        """
+        self._internalReaders.add(reader)
+        self.addReader(reader)
+
+    def _removeInternalReader(self, reader: IReadDescriptor) -> None:
+        """
+        Remove a read descriptor which is part of the implementation of the
+        reactor itself.
+        """
+        self._internalReaders.remove(reader)
+        self.removeReader(reader)
+
+    def run(self, installSignalHandlers: bool = True) -> None:
+        self.startRunning(installSignalHandlers=installSignalHandlers)
+        try:
+            self.mainLoop()
+        finally:
+            self._signals.uninstall()
+
+    def mainLoop(self) -> None:
+        while self._started:
+            try:
+                while self._started:
+                    # Advance simulation time in delayed event
+                    # processors.
+                    self.runUntilCurrent()
+                    t2 = self.timeout()
+                    t = self.running and t2
+                    self.doIteration(t)
+            except BaseException:
+                log.msg("Unexpected error in main loop.")
+                log.err()
+            else:
+                log.msg("Main loop terminated.")  # type:ignore[unreachable]
+
+    # override in subclasses
+
+    _lock = None
+
+    def installWaker(self) -> None:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement installWaker"
+        )
+
+    def wakeUp(self) -> None:
+        """
+        Wake up the event loop.
+        """
+        if self.waker:
+            self.waker.wakeUp()
+        # if the waker isn't installed, the reactor isn't running, and
+        # therefore doesn't need to be woken up
+
+    def doIteration(self, delay: Optional[float]) -> None:
+        """
+        Do one iteration over the readers and writers which have been added.
+        """
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement doIteration"
+        )
+
+    def addReader(self, reader: IReadDescriptor) -> None:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement addReader"
+        )
+
+    def addWriter(self, writer: IWriteDescriptor) -> None:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement addWriter"
+        )
+
+    def removeReader(self, reader: IReadDescriptor) -> None:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement removeReader"
+        )
+
+    def removeWriter(self, writer: IWriteDescriptor) -> None:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement removeWriter"
+        )
+
+    def removeAll(self) -> List[Union[IReadDescriptor, IWriteDescriptor]]:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement removeAll"
+        )
+
+    def getReaders(self) -> List[IReadDescriptor]:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement getReaders"
+        )
+
+    def getWriters(self) -> List[IWriteDescriptor]:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement getWriters"
+        )
+
+    # IReactorCore
+    def resolve(
+        self, name: str, timeout: Sequence[int] = (1, 3, 11, 45)
+    ) -> Deferred[str]:
+        """
+        Return a Deferred that will resolve a hostname."""
+        if not name:
+            # XXX - This is *less than* '::', and will screw up IPv6 servers
+            return defer.succeed("0.0.0.0")
+        if abstract.isIPAddress(name):
+            return defer.succeed(name)
+        return self.resolver.getHostByName(name, timeout)
+
+    def stop(self) -> None:
+        """
+        See twisted.internet.interfaces.IReactorCore.stop.
+        """
+        if self._stopped:
+            raise error.ReactorNotRunning("Can't stop reactor that isn't running.")
+        self._stopped = True
+        self._justStopped = True
+        self._startedBefore = True
+
+    def crash(self) -> None:
+        """
+        See twisted.internet.interfaces.IReactorCore.crash.
+
+        Reset reactor state tracking attributes and re-initialize certain
+        state-transition helpers which were set up in C{__init__} but later
+        destroyed (through use).
+        """
+        self._started = False
+        self.running = False
+        self.addSystemEventTrigger("during", "startup", self._reallyStartRunning)
+
+    def sigInt(self, number: int, frame: Optional[FrameType] = None) -> None:
+        """
+        Handle a SIGINT interrupt.
+
+        @param number: See handler specification in L{signal.signal}
+        @param frame: See handler specification in L{signal.signal}
+        """
+        log.msg("Received SIGINT, shutting down.")
+        self.callFromThread(self.stop)
+        self._exitSignal = number
+
+    def sigBreak(self, number: int, frame: Optional[FrameType] = None) -> None:
+        """
+        Handle a SIGBREAK interrupt.
+
+        @param number: See handler specification in L{signal.signal}
+        @param frame: See handler specification in L{signal.signal}
+        """
+        log.msg("Received SIGBREAK, shutting down.")
+        self.callFromThread(self.stop)
+        self._exitSignal = number
+
+    def sigTerm(self, number: int, frame: Optional[FrameType] = None) -> None:
+        """
+        Handle a SIGTERM interrupt.
+
+        @param number: See handler specification in L{signal.signal}
+        @param frame: See handler specification in L{signal.signal}
+        """
+        log.msg("Received SIGTERM, shutting down.")
+        self.callFromThread(self.stop)
+        self._exitSignal = number
+
+    def disconnectAll(self) -> None:
+        """Disconnect every reader, and writer in the system."""
+        selectables = self.removeAll()
+        for reader in selectables:
+            log.callWithLogger(
+                reader, reader.connectionLost, Failure(main.CONNECTION_LOST)
+            )
+
+    def iterate(self, delay: float = 0.0) -> None:
+        """
+        See twisted.internet.interfaces.IReactorCore.iterate.
+        """
+        self.runUntilCurrent()
+        self.doIteration(delay)
+
+    def fireSystemEvent(self, eventType: str) -> None:
+        """
+        See twisted.internet.interfaces.IReactorCore.fireSystemEvent.
+        """
+        event = self._eventTriggers.get(eventType)
+        if event is not None:
+            event.fireEvent()
+
+    def addSystemEventTrigger(
+        self,
+        phase: str,
+        eventType: str,
+        callable: Callable[..., Any],
+        *args: object,
+        **kwargs: object,
+    ) -> _SystemEventID:
+        """
+        See twisted.internet.interfaces.IReactorCore.addSystemEventTrigger.
+        """
+        assert builtins.callable(callable), f"{callable} is not callable"
+        if eventType not in self._eventTriggers:
+            self._eventTriggers[eventType] = _ThreePhaseEvent()
+        return _SystemEventID(
+            (
+                eventType,
+                self._eventTriggers[eventType].addTrigger(
+                    phase, callable, *args, **kwargs
+                ),
+            )
+        )
+
+    def removeSystemEventTrigger(self, triggerID: _SystemEventID) -> None:
+        """
+        See twisted.internet.interfaces.IReactorCore.removeSystemEventTrigger.
+        """
+        eventType, handle = triggerID
+        self._eventTriggers[eventType].removeTrigger(handle)
+
+    def callWhenRunning(
+        self, callable: Callable[..., Any], *args: object, **kwargs: object
+    ) -> Optional[_SystemEventID]:
+        """
+        See twisted.internet.interfaces.IReactorCore.callWhenRunning.
+        """
+        if self.running:
+            callable(*args, **kwargs)
+            return None
+        else:
+            return self.addSystemEventTrigger(
+                "after", "startup", callable, *args, **kwargs
+            )
+
+    def startRunning(self, installSignalHandlers: bool = True) -> None:
+        """
+        Method called when reactor starts: do some initialization and fire
+        startup events.
+
+        Don't call this directly, call reactor.run() instead: it should take
+        care of calling this.
+
+        This method is somewhat misnamed.  The reactor will not necessarily be
+        in the running state by the time this method returns.  The only
+        guarantee is that it will be on its way to the running state.
+
+        @param installSignalHandlers: A flag which, if set, indicates that
+            handlers for a number of (implementation-defined) signals should be
+            installed during startup.
+        """
+        if self._started:
+            raise error.ReactorAlreadyRunning()
+        if self._startedBefore:
+            raise error.ReactorNotRestartable()
+
+        self._signals.uninstall()
+        self._installSignalHandlers = installSignalHandlers
+        self._signals = self._makeSignalHandling(installSignalHandlers)
+
+        self._started = True
+        self._stopped = False
+        if self._registerAsIOThread:
+            threadable.registerAsIOThread()
+        self.fireSystemEvent("startup")
+
+    def _reallyStartRunning(self) -> None:
+        """
+        Method called to transition to the running state.  This should happen
+        in the I{during startup} event trigger phase.
+        """
+        self.running = True
+        if self._installSignalHandlers:
+            # Make sure this happens before after-startup events, since the
+            # expectation of after-startup is that the reactor is fully
+            # initialized.  Don't do it right away for historical reasons
+            # (perhaps some before-startup triggers don't want there to be a
+            # custom SIGCHLD handler so that they can run child processes with
+            # some blocking api).
+            self._signals.install()
+
+    # IReactorTime
+
+    seconds = staticmethod(runtimeSeconds)
+
+    def callLater(
+        self, delay: float, callable: Callable[..., Any], *args: object, **kw: object
+    ) -> DelayedCall:
+        """
+        See twisted.internet.interfaces.IReactorTime.callLater.
+        """
+        assert builtins.callable(callable), f"{callable} is not callable"
+        assert delay >= 0, f"{delay} is not greater than or equal to 0 seconds"
+        delayedCall = DelayedCall(
+            self.seconds() + delay,
+            callable,
+            args,
+            kw,
+            self._cancelCallLater,
+            self._moveCallLaterSooner,
+            seconds=self.seconds,
+        )
+        self._newTimedCalls.append(delayedCall)
+        return delayedCall
+
+    def _moveCallLaterSooner(self, delayedCall: DelayedCall) -> None:
+        # Linear time find: slow.
+        heap = self._pendingTimedCalls
+        try:
+            pos = heap.index(delayedCall)
+
+            # Move elt up the heap until it rests at the right place.
+            elt = heap[pos]
+            while pos != 0:
+                parent = (pos - 1) // 2
+                if heap[parent] <= elt:
+                    break
+                # move parent down
+                heap[pos] = heap[parent]
+                pos = parent
+            heap[pos] = elt
+        except ValueError:
+            # element was not found in heap - oh well...
+            pass
+
+    def _cancelCallLater(self, delayedCall: DelayedCall) -> None:
+        self._cancellations += 1
+
+    def getDelayedCalls(self) -> Sequence[IDelayedCall]:
+        """
+        See L{twisted.internet.interfaces.IReactorTime.getDelayedCalls}
+        """
+        return [
+            x
+            for x in (self._pendingTimedCalls + self._newTimedCalls)
+            if not x.cancelled
+        ]
+
+    def _insertNewDelayedCalls(self) -> None:
+        for call in self._newTimedCalls:
+            if call.cancelled:
+                self._cancellations -= 1
+            else:
+                call.activate_delay()
+                heappush(self._pendingTimedCalls, call)
+        self._newTimedCalls = []
+
+    def timeout(self) -> Optional[float]:
+        """
+        Determine the longest time the reactor may sleep (waiting on I/O
+        notification, perhaps) before it must wake up to service a time-related
+        event.
+
+        @return: The maximum number of seconds the reactor may sleep.
+        """
+        # insert new delayed calls to make sure to include them in timeout value
+        self._insertNewDelayedCalls()
+
+        if not self._pendingTimedCalls:
+            return None
+
+        delay = self._pendingTimedCalls[0].time - self.seconds()
+
+        # Pick a somewhat arbitrary maximum possible value for the timeout.
+        # This value is 2 ** 31 / 1000, which is the number of seconds which can
+        # be represented as an integer number of milliseconds in a signed 32 bit
+        # integer.  This particular limit is imposed by the epoll_wait(3)
+        # interface which accepts a timeout as a C "int" type and treats it as
+        # representing a number of milliseconds.
+        longest = 2147483
+
+        # Don't let the delay be in the past (negative) or exceed a plausible
+        # maximum (platform-imposed) interval.
+        return max(0, min(longest, delay))
+
+    def runUntilCurrent(self) -> None:
+        """
+        Run all pending timed calls.
+        """
+        if self.threadCallQueue:
+            # Keep track of how many calls we actually make, as we're
+            # making them, in case another call is added to the queue
+            # while we're in this loop.
+            count = 0
+            total = len(self.threadCallQueue)
+            for f, a, kw in self.threadCallQueue:
+                try:
+                    f(*a, **kw)
+                except BaseException:
+                    log.err()
+                count += 1
+                if count == total:
+                    break
+            del self.threadCallQueue[:count]
+            if self.threadCallQueue:
+                self.wakeUp()
+
+        # insert new delayed calls now
+        self._insertNewDelayedCalls()
+
+        now = self.seconds()
+        while self._pendingTimedCalls and (self._pendingTimedCalls[0].time <= now):
+            call = heappop(self._pendingTimedCalls)
+            if call.cancelled:
+                self._cancellations -= 1
+                continue
+
+            if call.delayed_time > 0.0:
+                call.activate_delay()
+                heappush(self._pendingTimedCalls, call)
+                continue
+
+            try:
+                call.called = 1
+                call.func(*call.args, **call.kw)
+            except BaseException:
+                log.err()
+                if call.creator is not None:
+                    e = "\n"
+                    e += (
+                        " C: previous exception occurred in "
+                        + "a DelayedCall created here:\n"
+                    )
+                    e += " C:"
+                    e += "".join(call.creator).rstrip().replace("\n", "\n C:")
+                    e += "\n"
+                    log.msg(e)
+
+        if (
+            self._cancellations > 50
+            and self._cancellations > len(self._pendingTimedCalls) >> 1
+        ):
+            self._cancellations = 0
+            self._pendingTimedCalls = [
+                x for x in self._pendingTimedCalls if not x.cancelled
+            ]
+            heapify(self._pendingTimedCalls)
+
+        if self._justStopped:
+            self._justStopped = False
+            self.fireSystemEvent("shutdown")
+
+    # IReactorThreads
+    if platform.supportsThreads():
+        assert ThreadPool is not None
+
+        threadpool = None
+        # ID of the trigger starting the threadpool
+        _threadpoolStartupID = None
+        # ID of the trigger stopping the threadpool
+        threadpoolShutdownID = None
+
+        def _initThreads(self) -> None:
+            self.installNameResolver(
+                _GAIResolver(cast(IReactorThreads, self), self.getThreadPool)
+            )
+            self.usingThreads = True
+
+        # `IReactorFromThreads` defines the first named argument as
+        # `callable: Callable[..., Any]` but this defines it as `f`
+        # really both should be defined using py3.8 positional only
+        def callFromThread(  # type: ignore[override]
+            self, f: Callable[..., Any], *args: object, **kwargs: object
+        ) -> None:
+            """
+            See
+            L{twisted.internet.interfaces.IReactorFromThreads.callFromThread}.
+            """
+            assert callable(f), f"{f} is not callable"
+            # lists are thread-safe in CPython, but not in Jython
+            # this is probably a bug in Jython, but until fixed this code
+            # won't work in Jython.
+            self.threadCallQueue.append((f, args, kwargs))
+            self.wakeUp()
+
+        def _initThreadPool(self) -> None:
+            """
+            Create the threadpool accessible with callFromThread.
+            """
+            self.threadpool = ThreadPool(0, 10, "twisted.internet.reactor")
+            self._threadpoolStartupID = self.callWhenRunning(self.threadpool.start)
+            self.threadpoolShutdownID = self.addSystemEventTrigger(
+                "during", "shutdown", self._stopThreadPool
+            )
+
+        def _uninstallHandler(self) -> None:
+            self._signals.uninstall()
+
+        def _stopThreadPool(self) -> None:
+            """
+            Stop the reactor threadpool.  This method is only valid if there
+            is currently a threadpool (created by L{_initThreadPool}).  It
+            is not intended to be called directly; instead, it will be
+            called by a shutdown trigger created in L{_initThreadPool}.
+            """
+            triggers = [self._threadpoolStartupID, self.threadpoolShutdownID]
+            for trigger in filter(None, triggers):
+                try:
+                    self.removeSystemEventTrigger(trigger)
+                except ValueError:
+                    pass
+            self._threadpoolStartupID = None
+            self.threadpoolShutdownID = None
+            assert self.threadpool is not None
+            self.threadpool.stop()
+            self.threadpool = None
+
+        def getThreadPool(self) -> ThreadPool:
+            """
+            See L{twisted.internet.interfaces.IReactorThreads.getThreadPool}.
+            """
+            if self.threadpool is None:
+                self._initThreadPool()
+                assert self.threadpool is not None
+            return self.threadpool
+
+        # `IReactorInThreads` defines the first named argument as
+        # `callable: Callable[..., Any]` but this defines it as `_callable`
+        # really both should be defined using py3.8 positional only
+        def callInThread(  # type: ignore[override]
+            self, _callable: Callable[..., Any], *args: object, **kwargs: object
+        ) -> None:
+            """
+            See L{twisted.internet.interfaces.IReactorInThreads.callInThread}.
+            """
+            self.getThreadPool().callInThread(_callable, *args, **kwargs)
+
+        def suggestThreadPoolSize(self, size: int) -> None:
+            """
+            See L{twisted.internet.interfaces.IReactorThreads.suggestThreadPoolSize}.
+            """
+            self.getThreadPool().adjustPoolsize(maxthreads=size)
+
+    else:
+        # This is for signal handlers.
+        def callFromThread(
+            self, f: Callable[..., Any], *args: object, **kwargs: object
+        ) -> None:
+            assert callable(f), f"{f} is not callable"
+            # See comment in the other callFromThread implementation.
+            self.threadCallQueue.append((f, args, kwargs))
+
+
+if platform.supportsThreads():
+    classImplements(ReactorBase, IReactorThreads)
+
+
+@implementer(IConnector)
+class BaseConnector(ABC):
+    """
+    Basic implementation of L{IConnector}.
+
+    State can be: "connecting", "connected", "disconnected"
+    """
+
+    timeoutID = None
+    factoryStarted = 0
+
+    def __init__(
+        self, factory: ClientFactory, timeout: float, reactor: ReactorBase
+    ) -> None:
+        self.state = "disconnected"
+        self.reactor = reactor
+        self.factory = factory
+        self.timeout = timeout
+
+    def disconnect(self) -> None:
+        """Disconnect whatever our state is."""
+        if self.state == "connecting":
+            self.stopConnecting()
+        elif self.state == "connected":
+            assert self.transport is not None
+            self.transport.loseConnection()
+
+    @abstractmethod
+    def _makeTransport(self) -> "Client":
+        pass
+
+    def connect(self) -> None:
+        """Start connection to remote server."""
+        if self.state != "disconnected":
+            raise RuntimeError("can't connect in this state")
+
+        self.state = "connecting"
+        if not self.factoryStarted:
+            self.factory.doStart()
+            self.factoryStarted = 1
+        self.transport: Optional[Client] = self._makeTransport()
+        if self.timeout is not None:
+            self.timeoutID = self.reactor.callLater(
+                self.timeout, self.transport.failIfNotConnected, error.TimeoutError()
+            )
+        self.factory.startedConnecting(self)
+
+    def stopConnecting(self) -> None:
+        """Stop attempting to connect."""
+        if self.state != "connecting":
+            raise error.NotConnectingError("we're not trying to connect")
+
+        assert self.transport is not None
+        self.state = "disconnected"
+        self.transport.failIfNotConnected(error.UserError())
+        del self.transport
+
+    def cancelTimeout(self) -> None:
+        if self.timeoutID is not None:
+            try:
+                self.timeoutID.cancel()
+            except ValueError:
+                pass
+            del self.timeoutID
+
+    def buildProtocol(self, addr: IAddress) -> Optional[IProtocol]:
+        self.state = "connected"
+        self.cancelTimeout()
+        return self.factory.buildProtocol(addr)
+
+    def connectionFailed(self, reason: Failure) -> None:
+        self.cancelTimeout()
+        self.transport = None
+        self.state = "disconnected"
+        self.factory.clientConnectionFailed(self, reason)
+        if self.state == "disconnected":
+            # factory hasn't called our connect() method
+            self.factory.doStop()
+            self.factoryStarted = 0
+
+    def connectionLost(self, reason: Failure) -> None:
+        self.state = "disconnected"
+        self.factory.clientConnectionLost(self, reason)
+        if self.state == "disconnected":
+            # factory hasn't called our connect() method
+            self.factory.doStop()
+            self.factoryStarted = 0
+
+    def getDestination(self) -> IAddress:
+        raise NotImplementedError(
+            reflect.qual(self.__class__) + " did not implement " "getDestination"
+        )
+
+    def __repr__(self) -> str:
+        return "<{} instance at 0x{:x} {} {}>".format(
+            reflect.qual(self.__class__),
+            id(self),
+            self.state,
+            self.getDestination(),
+        )
+
+
+class BasePort(abstract.FileDescriptor):
+    """Basic implementation of a ListeningPort.
+
+    Note: This does not actually implement IListeningPort.
+    """
+
+    addressFamily: socket.AddressFamily = None  # type: ignore[assignment]
+    socketType: socket.SocketKind = None  # type: ignore[assignment]
+
+    def createInternetSocket(self) -> socket.socket:
+        s = socket.socket(self.addressFamily, self.socketType)
+        s.setblocking(False)
+        fdesc._setCloseOnExec(s.fileno())
+        return s
+
+    def doWrite(self) -> Optional[Failure]:
+        """Raises a RuntimeError"""
+        raise RuntimeError("doWrite called on a %s" % reflect.qual(self.__class__))
+
+
+__all__: List[str] = []
diff --git a/contrib/python/Twisted/py3/twisted/internet/cfreactor.py b/contrib/python/Twisted/py3/twisted/internet/cfreactor.py
new file mode 100644
index 00000000000..142c0472ef4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/cfreactor.py
@@ -0,0 +1,593 @@
+# -*- test-case-name: twisted.internet.test.test_core -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A reactor for integrating with U{CFRunLoop<http://bit.ly/cfrunloop>}, the
+CoreFoundation main loop used by macOS.
+
+This is useful for integrating Twisted with U{PyObjC<http://pyobjc.sf.net/>}
+applications.
+"""
+from __future__ import annotations
+
+__all__ = ["install", "CFReactor"]
+
+import sys
+
+from zope.interface import implementer
+
+from CFNetwork import (  # type: ignore[import]
+    CFSocketCreateRunLoopSource,
+    CFSocketCreateWithNative,
+    CFSocketDisableCallBacks,
+    CFSocketEnableCallBacks,
+    CFSocketInvalidate,
+    CFSocketSetSocketFlags,
+    kCFSocketAutomaticallyReenableReadCallBack,
+    kCFSocketAutomaticallyReenableWriteCallBack,
+    kCFSocketConnectCallBack,
+    kCFSocketReadCallBack,
+    kCFSocketWriteCallBack,
+)
+from CoreFoundation import (  # type: ignore[import]
+    CFAbsoluteTimeGetCurrent,
+    CFRunLoopAddSource,
+    CFRunLoopAddTimer,
+    CFRunLoopGetCurrent,
+    CFRunLoopRemoveSource,
+    CFRunLoopRun,
+    CFRunLoopStop,
+    CFRunLoopTimerCreate,
+    CFRunLoopTimerInvalidate,
+    kCFAllocatorDefault,
+    kCFRunLoopCommonModes,
+)
+
+from twisted.internet.interfaces import IReactorFDSet
+from twisted.internet.posixbase import _NO_FILEDESC, PosixReactorBase
+from twisted.python import log
+
+# We know that we're going to run on macOS so we can just pick the
+# POSIX-appropriate waker.  This also avoids having a dynamic base class and
+# so lets more things get type checked.
+from ._signals import _UnixWaker
+
+_READ = 0
+_WRITE = 1
+_preserveSOError = 1 << 6
+
+
+class _WakerPlus(_UnixWaker):
+    """
+    The normal Twisted waker will simply wake up the main loop, which causes an
+    iteration to run, which in turn causes L{ReactorBase.runUntilCurrent}
+    to get invoked.
+
+    L{CFReactor} has a slightly different model of iteration, though: rather
+    than have each iteration process the thread queue, then timed calls, then
+    file descriptors, each callback is run as it is dispatched by the CFRunLoop
+    observer which triggered it.
+
+    So this waker needs to not only unblock the loop, but also make sure the
+    work gets done; so, it reschedules the invocation of C{runUntilCurrent} to
+    be immediate (0 seconds from now) even if there is no timed call work to
+    do.
+    """
+
+    def __init__(self, reactor):
+        super().__init__()
+        self.reactor = reactor
+
+    def doRead(self):
+        """
+        Wake up the loop and force C{runUntilCurrent} to run immediately in the
+        next timed iteration.
+        """
+        result = super().doRead()
+        self.reactor._scheduleSimulate(True)
+        return result
+
+
+@implementer(IReactorFDSet)
+class CFReactor(PosixReactorBase):
+    """
+    The CoreFoundation reactor.
+
+    You probably want to use this via the L{install} API.
+
+    @ivar _fdmap: a dictionary, mapping an integer (a file descriptor) to a
+        4-tuple of:
+
+            - source: a C{CFRunLoopSource}; the source associated with this
+              socket.
+            - socket: a C{CFSocket} wrapping the file descriptor.
+            - descriptor: an L{IReadDescriptor} and/or L{IWriteDescriptor}
+              provider.
+            - read-write: a 2-C{list} of booleans: respectively, whether this
+              descriptor is currently registered for reading or registered for
+              writing.
+
+    @ivar _idmap: a dictionary, mapping the id() of an L{IReadDescriptor} or
+        L{IWriteDescriptor} to a C{fd} in L{_fdmap}.  Implemented in this
+        manner so that we don't have to rely (even more) on the hashability of
+        L{IReadDescriptor} providers, and we know that they won't be collected
+        since these are kept in sync with C{_fdmap}.  Necessary because the
+        .fileno() of a file descriptor may change at will, so we need to be
+        able to look up what its file descriptor I{used} to be, so that we can
+        look it up in C{_fdmap}
+
+    @ivar _cfrunloop: the C{CFRunLoop} pyobjc object wrapped
+        by this reactor.
+
+    @ivar _inCFLoop: Is C{CFRunLoopRun} currently running?
+
+    @type _inCFLoop: L{bool}
+
+    @ivar _currentSimulator: if a CFTimer is currently scheduled with the CF
+        run loop to run Twisted callLater calls, this is a reference to it.
+        Otherwise, it is L{None}
+    """
+
+    def __init__(self, runLoop=None, runner=None):
+        self._fdmap = {}
+        self._idmap = {}
+        if runner is None:
+            runner = CFRunLoopRun
+        self._runner = runner
+
+        if runLoop is None:
+            runLoop = CFRunLoopGetCurrent()
+        self._cfrunloop = runLoop
+        PosixReactorBase.__init__(self)
+
+    def _wakerFactory(self) -> _WakerPlus:
+        return _WakerPlus(self)
+
+    def _socketCallback(
+        self, cfSocket, callbackType, ignoredAddress, ignoredData, context
+    ):
+        """
+        The socket callback issued by CFRunLoop.  This will issue C{doRead} or
+        C{doWrite} calls to the L{IReadDescriptor} and L{IWriteDescriptor}
+        registered with the file descriptor that we are being notified of.
+
+        @param cfSocket: The C{CFSocket} which has got some activity.
+
+        @param callbackType: The type of activity that we are being notified
+            of.  Either C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack}.
+
+        @param ignoredAddress: Unused, because this is not used for either of
+            the callback types we register for.
+
+        @param ignoredData: Unused, because this is not used for either of the
+            callback types we register for.
+
+        @param context: The data associated with this callback by
+            C{CFSocketCreateWithNative} (in C{CFReactor._watchFD}).  A 2-tuple
+            of C{(int, CFRunLoopSource)}.
+        """
+        (fd, smugglesrc) = context
+        if fd not in self._fdmap:
+            # Spurious notifications seem to be generated sometimes if you
+            # CFSocketDisableCallBacks in the middle of an event.  I don't know
+            # about this FD, any more, so let's get rid of it.
+            CFRunLoopRemoveSource(self._cfrunloop, smugglesrc, kCFRunLoopCommonModes)
+            return
+
+        src, skt, readWriteDescriptor, rw = self._fdmap[fd]
+
+        def _drdw():
+            why = None
+            isRead = False
+
+            try:
+                if readWriteDescriptor.fileno() == -1:
+                    why = _NO_FILEDESC
+                else:
+                    isRead = callbackType == kCFSocketReadCallBack
+                    # CFSocket seems to deliver duplicate read/write
+                    # notifications sometimes, especially a duplicate
+                    # writability notification when first registering the
+                    # socket.  This bears further investigation, since I may
+                    # have been mis-interpreting the behavior I was seeing.
+                    # (Running the full Twisted test suite, while thorough, is
+                    # not always entirely clear.) Until this has been more
+                    # thoroughly investigated , we consult our own
+                    # reading/writing state flags to determine whether we
+                    # should actually attempt a doRead/doWrite first.  -glyph
+                    if isRead:
+                        if rw[_READ]:
+                            why = readWriteDescriptor.doRead()
+                    else:
+                        if rw[_WRITE]:
+                            why = readWriteDescriptor.doWrite()
+            except BaseException:
+                why = sys.exc_info()[1]
+                log.err()
+            if why:
+                self._disconnectSelectable(readWriteDescriptor, why, isRead)
+
+        log.callWithLogger(readWriteDescriptor, _drdw)
+
+    def _watchFD(self, fd, descr, flag):
+        """
+        Register a file descriptor with the C{CFRunLoop}, or modify its state
+        so that it's listening for both notifications (read and write) rather
+        than just one; used to implement C{addReader} and C{addWriter}.
+
+        @param fd: The file descriptor.
+
+        @type fd: L{int}
+
+        @param descr: the L{IReadDescriptor} or L{IWriteDescriptor}
+
+        @param flag: the flag to register for callbacks on, either
+            C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack}
+        """
+        if fd == -1:
+            raise RuntimeError("Invalid file descriptor.")
+        if fd in self._fdmap:
+            src, cfs, gotdescr, rw = self._fdmap[fd]
+            # do I need to verify that it's the same descr?
+        else:
+            ctx = []
+            ctx.append(fd)
+            cfs = CFSocketCreateWithNative(
+                kCFAllocatorDefault,
+                fd,
+                kCFSocketReadCallBack
+                | kCFSocketWriteCallBack
+                | kCFSocketConnectCallBack,
+                self._socketCallback,
+                ctx,
+            )
+            CFSocketSetSocketFlags(
+                cfs,
+                kCFSocketAutomaticallyReenableReadCallBack
+                | kCFSocketAutomaticallyReenableWriteCallBack
+                |
+                # This extra flag is to ensure that CF doesn't (destructively,
+                # because destructively is the only way to do it) retrieve
+                # SO_ERROR and thereby break twisted.internet.tcp.BaseClient,
+                # which needs SO_ERROR to tell it whether or not it needs to
+                # call connect_ex a second time.
+                _preserveSOError,
+            )
+            src = CFSocketCreateRunLoopSource(kCFAllocatorDefault, cfs, 0)
+            ctx.append(src)
+            CFRunLoopAddSource(self._cfrunloop, src, kCFRunLoopCommonModes)
+            CFSocketDisableCallBacks(
+                cfs,
+                kCFSocketReadCallBack
+                | kCFSocketWriteCallBack
+                | kCFSocketConnectCallBack,
+            )
+            rw = [False, False]
+            self._idmap[id(descr)] = fd
+            self._fdmap[fd] = src, cfs, descr, rw
+        rw[self._flag2idx(flag)] = True
+        CFSocketEnableCallBacks(cfs, flag)
+
+    def _flag2idx(self, flag):
+        """
+        Convert a C{kCFSocket...} constant to an index into the read/write
+        state list (C{_READ} or C{_WRITE}) (the 4th element of the value of
+        C{self._fdmap}).
+
+        @param flag: C{kCFSocketReadCallBack} or C{kCFSocketWriteCallBack}
+
+        @return: C{_READ} or C{_WRITE}
+        """
+        return {kCFSocketReadCallBack: _READ, kCFSocketWriteCallBack: _WRITE}[flag]
+
+    def _unwatchFD(self, fd, descr, flag):
+        """
+        Unregister a file descriptor with the C{CFRunLoop}, or modify its state
+        so that it's listening for only one notification (read or write) as
+        opposed to both; used to implement C{removeReader} and C{removeWriter}.
+
+        @param fd: a file descriptor
+
+        @type fd: C{int}
+
+        @param descr: an L{IReadDescriptor} or L{IWriteDescriptor}
+
+        @param flag: C{kCFSocketWriteCallBack} C{kCFSocketReadCallBack}
+        """
+        if id(descr) not in self._idmap:
+            return
+        if fd == -1:
+            # need to deal with it in this case, I think.
+            realfd = self._idmap[id(descr)]
+        else:
+            realfd = fd
+        src, cfs, descr, rw = self._fdmap[realfd]
+        CFSocketDisableCallBacks(cfs, flag)
+        rw[self._flag2idx(flag)] = False
+        if not rw[_READ] and not rw[_WRITE]:
+            del self._idmap[id(descr)]
+            del self._fdmap[realfd]
+            CFRunLoopRemoveSource(self._cfrunloop, src, kCFRunLoopCommonModes)
+            CFSocketInvalidate(cfs)
+
+    def addReader(self, reader):
+        """
+        Implement L{IReactorFDSet.addReader}.
+        """
+        self._watchFD(reader.fileno(), reader, kCFSocketReadCallBack)
+
+    def addWriter(self, writer):
+        """
+        Implement L{IReactorFDSet.addWriter}.
+        """
+        self._watchFD(writer.fileno(), writer, kCFSocketWriteCallBack)
+
+    def removeReader(self, reader):
+        """
+        Implement L{IReactorFDSet.removeReader}.
+        """
+        self._unwatchFD(reader.fileno(), reader, kCFSocketReadCallBack)
+
+    def removeWriter(self, writer):
+        """
+        Implement L{IReactorFDSet.removeWriter}.
+        """
+        self._unwatchFD(writer.fileno(), writer, kCFSocketWriteCallBack)
+
+    def removeAll(self):
+        """
+        Implement L{IReactorFDSet.removeAll}.
+        """
+        allDesc = {descr for src, cfs, descr, rw in self._fdmap.values()}
+        allDesc -= set(self._internalReaders)
+        for desc in allDesc:
+            self.removeReader(desc)
+            self.removeWriter(desc)
+        return list(allDesc)
+
+    def getReaders(self):
+        """
+        Implement L{IReactorFDSet.getReaders}.
+        """
+        return [descr for src, cfs, descr, rw in self._fdmap.values() if rw[_READ]]
+
+    def getWriters(self):
+        """
+        Implement L{IReactorFDSet.getWriters}.
+        """
+        return [descr for src, cfs, descr, rw in self._fdmap.values() if rw[_WRITE]]
+
+    def _moveCallLaterSooner(self, tple):
+        """
+        Override L{PosixReactorBase}'s implementation of L{IDelayedCall.reset}
+        so that it will immediately reschedule.  Normally
+        C{_moveCallLaterSooner} depends on the fact that C{runUntilCurrent} is
+        always run before the mainloop goes back to sleep, so this forces it to
+        immediately recompute how long the loop needs to stay asleep.
+        """
+        result = PosixReactorBase._moveCallLaterSooner(self, tple)
+        self._scheduleSimulate()
+        return result
+
+    def startRunning(self, installSignalHandlers: bool = True) -> None:
+        """
+        Start running the reactor, then kick off the timer that advances
+        Twisted's clock to keep pace with CFRunLoop's.
+        """
+        super().startRunning(installSignalHandlers)
+
+        # Before 'startRunning' is called, the reactor is not attached to the
+        # CFRunLoop[1]; specifically, the CFTimer that runs all of Twisted's
+        # timers is not active and will not have been added to the loop by any
+        # application code.  Now that _running is probably[2] True, we need to
+        # ensure that timed calls will actually run on the main loop.  This
+        # call needs to be here, rather than at the top of mainLoop, because
+        # it's possible to use startRunning to *attach* a reactor to an
+        # already-running CFRunLoop, i.e. within a plugin for an application
+        # that doesn't otherwise use Twisted, rather than calling it via run().
+        self._scheduleSimulate(force=True)
+
+        # [1]: readers & writers are still active in the loop, but arguably
+        #      they should not be.
+
+        # [2]: application code within a 'startup' system event trigger *may*
+        #      have already crashed the reactor and thus set _started to False,
+        #      but that specific case is handled by mainLoop, since that case
+        #      is inherently irrelevant in an attach-to-application case and is
+        #      only necessary to handle mainLoop spuriously blocking.
+
+    _inCFLoop = False
+
+    def mainLoop(self) -> None:
+        """
+        Run the runner (C{CFRunLoopRun} or something that calls it), which runs
+        the run loop until C{crash()} is called.
+        """
+        if not self._started:
+            # If we arrive here, we were crashed by application code in a
+            # 'startup' system event trigger, (or crashed manually before the
+            # application calls 'mainLoop' directly for whatever reason; sigh,
+            # this method should not be public).  However, application code
+            # doing obscure things will expect an invocation of this loop to
+            # have at least *one* pass over ready readers, writers, and delayed
+            # calls.  iterate(), in particular, is emulated in exactly this way
+            # in this reactor implementation.  In order to ensure that we enter
+            # the real implementation of the mainloop and do all of those
+            # things, we need to set _started back to True so that callLater
+            # actually schedules itself against the CFRunLoop, but immediately
+            # crash once we are in the context of the loop where we've run
+            # ready I/O and timers.
+
+            def docrash() -> None:
+                self.crash()
+
+            self._started = True
+            self.callLater(0, docrash)
+        already = False
+        try:
+            while self._started:
+                if already:
+                    # Sometimes CFRunLoopRun (or its equivalents) may exit
+                    # without CFRunLoopStop being called.
+
+                    # This is really only *supposed* to happen when it runs out
+                    # of sources & timers to process.  However, in full Twisted
+                    # test-suite runs we have observed, extremely rarely (once
+                    # in every 3000 tests or so) CFRunLoopRun exiting in cases
+                    # where it seems as though there *is* still some work to
+                    # do.  However, given the difficulty of reproducing the
+                    # race conditions necessary to make this happen, it's
+                    # possible that we have missed some nuance of when
+                    # CFRunLoop considers the list of work "empty" and various
+                    # callbacks and timers to be "invalidated".  Therefore we
+                    # are not fully confident that this is a platform bug, but
+                    # it is nevertheless unexpected behavior from our reading
+                    # of the documentation.
+
+                    # To accommodate this rare and slightly ambiguous stress
+                    # case, we make extra sure that our scheduled timer is
+                    # re-created on the loop as a CFRunLoopTimer, which
+                    # reliably gives the loop some work to do and 'fixes' it if
+                    # it exited due to having no active sources or timers.
+                    self._scheduleSimulate()
+
+                    # At this point, there may be a little more code that we
+                    # would need to put here for full correctness for a very
+                    # peculiar type of application: if you're writing a
+                    # command-line tool using CFReactor, adding *nothing* to
+                    # the reactor itself, disabling even the internal Waker
+                    # file descriptors, then there's a possibility that
+                    # CFRunLoopRun will exit early, and if we have no timers,
+                    # we might busy-loop here.  Because we cannot seem to force
+                    # this to happen under normal circumstances, we're leaving
+                    # that code out.
+
+                already = True
+                self._inCFLoop = True
+                try:
+                    self._runner()
+                finally:
+                    self._inCFLoop = False
+        finally:
+            self._stopSimulating()
+
+    _currentSimulator: object | None = None
+
+    def _stopSimulating(self) -> None:
+        """
+        If we have a CFRunLoopTimer registered with the CFRunLoop, invalidate
+        it and set it to None.
+        """
+        if self._currentSimulator is None:
+            return
+        CFRunLoopTimerInvalidate(self._currentSimulator)
+        self._currentSimulator = None
+
+    def _scheduleSimulate(self, force: bool = False) -> None:
+        """
+        Schedule a call to C{self.runUntilCurrent}.  This will cancel the
+        currently scheduled call if it is already scheduled.
+
+        @param force: Even if there are no timed calls, make sure that
+            C{runUntilCurrent} runs immediately (in a 0-seconds-from-now
+            C{CFRunLoopTimer}).  This is necessary for calls which need to
+            trigger behavior of C{runUntilCurrent} other than running timed
+            calls, such as draining the thread call queue or calling C{crash()}
+            when the appropriate flags are set.
+
+        @type force: C{bool}
+        """
+        self._stopSimulating()
+        if not self._started:
+            # If the reactor is not running (e.g. we are scheduling callLater
+            # calls before starting the reactor) we should not be scheduling
+            # CFRunLoopTimers against the global CFRunLoop.
+            return
+
+        timeout = 0.0 if force else self.timeout()
+        if timeout is None:
+            return
+
+        fireDate = CFAbsoluteTimeGetCurrent() + timeout
+
+        def simulate(cftimer, extra):
+            self._currentSimulator = None
+            self.runUntilCurrent()
+            self._scheduleSimulate()
+
+        c = self._currentSimulator = CFRunLoopTimerCreate(
+            kCFAllocatorDefault, fireDate, 0, 0, 0, simulate, None
+        )
+        CFRunLoopAddTimer(self._cfrunloop, c, kCFRunLoopCommonModes)
+
+    def callLater(self, _seconds, _f, *args, **kw):
+        """
+        Implement L{IReactorTime.callLater}.
+        """
+        delayedCall = PosixReactorBase.callLater(self, _seconds, _f, *args, **kw)
+        self._scheduleSimulate()
+        return delayedCall
+
+    def stop(self):
+        """
+        Implement L{IReactorCore.stop}.
+        """
+        PosixReactorBase.stop(self)
+        self._scheduleSimulate(True)
+
+    def crash(self):
+        """
+        Implement L{IReactorCore.crash}
+        """
+        PosixReactorBase.crash(self)
+        if not self._inCFLoop:
+            return
+        CFRunLoopStop(self._cfrunloop)
+
+    def iterate(self, delay=0):
+        """
+        Emulate the behavior of C{iterate()} for things that want to call it,
+        by letting the loop run for a little while and then scheduling a timed
+        call to exit it.
+        """
+        self._started = True
+        # Since the CoreFoundation loop doesn't have the concept of "iterate"
+        # we can't ask it to do this.  Instead we will make arrangements to
+        # crash it *very* soon and then make it run.  This is a rough
+        # approximation of "an iteration".  Using crash and mainLoop here
+        # means that it's safe (as safe as anything using "iterate" can be) to
+        # do this repeatedly.
+        self.callLater(0, self.crash)
+        self.mainLoop()
+
+
+def install(runLoop=None, runner=None):
+    """
+    Configure the twisted mainloop to be run inside CFRunLoop.
+
+    @param runLoop: the run loop to use.
+
+    @param runner: the function to call in order to actually invoke the main
+        loop.  This will default to C{CFRunLoopRun} if not specified.  However,
+        this is not an appropriate choice for GUI applications, as you need to
+        run NSApplicationMain (or something like it).  For example, to run the
+        Twisted mainloop in a PyObjC application, your C{main.py} should look
+        something like this::
+
+            from PyObjCTools import AppHelper
+            from twisted.internet.cfreactor import install
+            install(runner=AppHelper.runEventLoop)
+            # initialize your application
+            reactor.run()
+
+    @return: The installed reactor.
+
+    @rtype: C{CFReactor}
+    """
+
+    reactor = CFReactor(runLoop=runLoop, runner=runner)
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
+    return reactor
diff --git a/contrib/python/Twisted/py3/twisted/internet/default.py b/contrib/python/Twisted/py3/twisted/internet/default.py
new file mode 100644
index 00000000000..5dfc7ee8191
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/default.py
@@ -0,0 +1,55 @@
+# -*- test-case-name: twisted.internet.test.test_default -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+The most suitable default reactor for the current platform.
+
+Depending on a specific application's needs, some other reactor may in
+fact be better.
+"""
+
+
+__all__ = ["install"]
+
+from twisted.python.runtime import platform
+
+
+def _getInstallFunction(platform):
+    """
+    Return a function to install the reactor most suited for the given platform.
+
+    @param platform: The platform for which to select a reactor.
+    @type platform: L{twisted.python.runtime.Platform}
+
+    @return: A zero-argument callable which will install the selected
+        reactor.
+    """
+    # Linux: epoll(7) is the default, since it scales well.
+    #
+    # macOS: poll(2) is not exposed by Python because it doesn't support all
+    # file descriptors (in particular, lack of PTY support is a problem) --
+    # see <http://bugs.python.org/issue5154>. kqueue has the same restrictions
+    # as poll(2) as far PTY support goes.
+    #
+    # Windows: IOCP should eventually be default, but still has some serious
+    # bugs, e.g. <http://twistedmatrix.com/trac/ticket/4667>.
+    #
+    # We therefore choose epoll(7) on Linux, poll(2) on other non-macOS POSIX
+    # platforms, and select(2) everywhere else.
+    try:
+        if platform.isLinux():
+            try:
+                from twisted.internet.epollreactor import install
+            except ImportError:
+                from twisted.internet.pollreactor import install
+        elif platform.getType() == "posix" and not platform.isMacOSX():
+            from twisted.internet.pollreactor import install
+        else:
+            from twisted.internet.selectreactor import install
+    except ImportError:
+        from twisted.internet.selectreactor import install
+    return install
+
+
+install = _getInstallFunction(platform)
diff --git a/contrib/python/Twisted/py3/twisted/internet/defer.py b/contrib/python/Twisted/py3/twisted/internet/defer.py
new file mode 100644
index 00000000000..17e717cad28
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/defer.py
@@ -0,0 +1,2697 @@
+# -*- test-case-name: twisted.test.test_defer -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support for results that aren't immediately available.
+
+Maintainer: Glyph Lefkowitz
+"""
+from __future__ import annotations
+
+import inspect
+import traceback
+import warnings
+from abc import ABC, abstractmethod
+from asyncio import AbstractEventLoop, Future, iscoroutine
+from contextvars import Context as _Context, copy_context as _copy_context
+from enum import Enum
+from functools import wraps
+from sys import exc_info
+from types import CoroutineType, GeneratorType, MappingProxyType, TracebackType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Coroutine,
+    Generator,
+    Generic,
+    Iterable,
+    List,
+    Mapping,
+    NoReturn,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+    overload,
+)
+
+import attr
+from incremental import Version
+from typing_extensions import Concatenate, Literal, ParamSpec, Self
+
+from twisted.internet.interfaces import IDelayedCall, IReactorTime
+from twisted.logger import Logger
+from twisted.python import lockfile
+from twisted.python.compat import _PYPY, cmp, comparable
+from twisted.python.deprecate import deprecated, warnAboutFunction
+from twisted.python.failure import Failure, _extraneous
+
+log = Logger()
+
+
+_T = TypeVar("_T")
+_P = ParamSpec("_P")
+
+
+class AlreadyCalledError(Exception):
+    """
+    This error is raised when one of L{Deferred.callback} or L{Deferred.errback}
+    is called after one of the two had already been called.
+    """
+
+
+class CancelledError(Exception):
+    """
+    This error is raised by default when a L{Deferred} is cancelled.
+    """
+
+
+class TimeoutError(Exception):
+    """
+    This error is raised by default when a L{Deferred} times out.
+    """
+
+
+class NotACoroutineError(TypeError):
+    """
+    This error is raised when a coroutine is expected and something else is
+    encountered.
+    """
+
+
+def logError(err: Failure) -> Failure:
+    """
+    Log and return failure.
+
+    This method can be used as an errback that passes the failure on to the
+    next errback unmodified. Note that if this is the last errback, and the
+    deferred gets garbage collected after being this errback has been called,
+    the clean up code logs it again.
+    """
+    log.failure("", err)
+    return err
+
+
+def succeed(result: _T) -> "Deferred[_T]":
+    """
+    Return a L{Deferred} that has already had C{.callback(result)} called.
+
+    This is useful when you're writing synchronous code to an
+    asynchronous interface: i.e., some code is calling you expecting a
+    L{Deferred} result, but you don't actually need to do anything
+    asynchronous. Just return C{defer.succeed(theResult)}.
+
+    See L{fail} for a version of this function that uses a failing
+    L{Deferred} rather than a successful one.
+
+    @param result: The result to give to the Deferred's 'callback'
+           method.
+    """
+    d: Deferred[_T] = Deferred()
+    d.callback(result)
+    return d
+
+
+def fail(result: Optional[Union[Failure, BaseException]] = None) -> "Deferred[Any]":
+    """
+    Return a L{Deferred} that has already had C{.errback(result)} called.
+
+    See L{succeed}'s docstring for rationale.
+
+    @param result: The same argument that L{Deferred.errback} takes.
+
+    @raise NoCurrentExceptionError: If C{result} is L{None} but there is no
+        current exception state.
+    """
+    d: Deferred[Any] = Deferred()
+    d.errback(result)
+    return d
+
+
+def execute(
+    callable: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
+) -> "Deferred[_T]":
+    """
+    Create a L{Deferred} from a callable and arguments.
+
+    Call the given function with the given arguments.  Return a L{Deferred}
+    which has been fired with its callback as the result of that invocation
+    or its C{errback} with a L{Failure} for the exception thrown.
+    """
+    try:
+        result = callable(*args, **kwargs)
+    except BaseException:
+        return fail()
+    else:
+        return succeed(result)
+
+
+@overload
+def maybeDeferred(
+    f: Callable[_P, Deferred[_T]], *args: _P.args, **kwargs: _P.kwargs
+) -> "Deferred[_T]":
+    ...
+
+
+@overload
+def maybeDeferred(
+    f: Callable[_P, Coroutine[Deferred[Any], Any, _T]],
+    *args: _P.args,
+    **kwargs: _P.kwargs,
+) -> "Deferred[_T]":
+    ...
+
+
+@overload
+def maybeDeferred(
+    f: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
+) -> "Deferred[_T]":
+    ...
+
+
+def maybeDeferred(
+    f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]],
+    *args: _P.args,
+    **kwargs: _P.kwargs,
+) -> "Deferred[_T]":
+    """
+    Invoke a function that may or may not return a L{Deferred} or coroutine.
+
+    Call the given function with the given arguments.  Then:
+
+      - If the returned object is a L{Deferred}, return it.
+
+      - If the returned object is a L{Failure}, wrap it with L{fail} and
+        return it.
+
+      - If the returned object is a L{types.CoroutineType}, wrap it with
+        L{Deferred.fromCoroutine} and return it.
+
+      - Otherwise, wrap it in L{succeed} and return it.
+
+      - If an exception is raised, convert it to a L{Failure}, wrap it in
+        L{fail}, and then return it.
+
+    @param f: The callable to invoke
+    @param args: The arguments to pass to C{f}
+    @param kwargs: The keyword arguments to pass to C{f}
+
+    @return: The result of the function call, wrapped in a L{Deferred} if
+    necessary.
+    """
+    try:
+        result = f(*args, **kwargs)
+    except BaseException:
+        return fail(Failure(captureVars=Deferred.debug))
+
+    if isinstance(result, Deferred):
+        return result
+    elif isinstance(result, Failure):
+        return fail(result)
+    elif type(result) is CoroutineType:
+        # A note on how we identify this case ...
+        #
+        # inspect.iscoroutinefunction(f) should be the simplest and easiest
+        # way to determine if we want to apply coroutine handling.  However,
+        # the value may be returned by a regular function that calls a
+        # coroutine function and returns its result.  It would be confusing if
+        # cases like this led to different handling of the coroutine (even
+        # though it is a mistake to have a regular function call a coroutine
+        # function to return its result - doing so immediately destroys a
+        # large part of the value of coroutine functions: that they can only
+        # have a coroutine result).
+        #
+        # There are many ways we could inspect ``result`` to determine if it
+        # is a "coroutine" but most of these are mistakes.  The goal is only
+        # to determine whether the value came from ``async def`` or not
+        # because these are the only values we're trying to handle with this
+        # case.  Such values always have exactly one type: CoroutineType.
+        return Deferred.fromCoroutine(result)
+    else:
+        returned: _T = result  # type: ignore
+        return succeed(returned)
+
+
+@deprecated(
+    Version("Twisted", 17, 1, 0),
+    replacement="twisted.internet.defer.Deferred.addTimeout",
+)
+def timeout(deferred: "Deferred[object]") -> None:
+    deferred.errback(Failure(TimeoutError("Callback timed out")))
+
+
+def passthru(arg: _T) -> _T:
+    return arg
+
+
+def _failthru(arg: Failure) -> Failure:
+    return arg
+
+
+def setDebugging(on: bool) -> None:
+    """
+    Enable or disable L{Deferred} debugging.
+
+    When debugging is on, the call stacks from creation and invocation are
+    recorded, and added to any L{AlreadyCalledError}s we raise.
+    """
+    Deferred.debug = bool(on)
+
+
+def getDebugging() -> bool:
+    """
+    Determine whether L{Deferred} debugging is enabled.
+    """
+    return Deferred.debug
+
+
+def _cancelledToTimedOutError(value: _T, timeout: float) -> _T:
+    """
+    A default translation function that translates L{Failure}s that are
+    L{CancelledError}s to L{TimeoutError}s.
+
+    @param value: Anything
+    @param timeout: The timeout
+
+    @raise TimeoutError: If C{value} is a L{Failure} that is a L{CancelledError}.
+    @raise Exception: If C{value} is a L{Failure} that is not a L{CancelledError},
+        it is re-raised.
+
+    @since: 16.5
+    """
+    if isinstance(value, Failure):
+        value.trap(CancelledError)
+        raise TimeoutError(timeout, "Deferred")
+    return value
+
+
+class _Sentinel(Enum):
+    """
+    @cvar _NO_RESULT:
+        The result used to represent the fact that there is no result.
+        B{Never ever ever use this as an actual result for a Deferred}.
+        You have been warned.
+    @cvar _CONTINUE:
+        A marker left in L{Deferred.callback}s to indicate a Deferred chain.
+        Always accompanied by a Deferred instance in the args tuple pointing at
+        the Deferred which is chained to the Deferred which has this marker.
+    """
+
+    _NO_RESULT = object()
+    _CONTINUE = object()
+
+
+# Cache these values for use without the extra lookup in deferred hot code paths
+_NO_RESULT = _Sentinel._NO_RESULT
+_CONTINUE = _Sentinel._CONTINUE
+
+
+# type note: this should be Callable[[object, ...], object] but mypy doesn't allow.
+#     Callable[[object], object] is next best, but disallows valid callback signatures
+DeferredCallback = Callable[..., object]
+# type note: this should be Callable[[Failure, ...], object] but mypy doesn't allow.
+#     Callable[[Failure], object] is next best, but disallows valid callback signatures
+DeferredErrback = Callable[..., object]
+
+_CallbackOrderedArguments = Tuple[object, ...]
+_CallbackKeywordArguments = Mapping[str, object]
+_CallbackChain = Tuple[
+    Tuple[
+        Union[DeferredCallback, Literal[_Sentinel._CONTINUE]],
+        _CallbackOrderedArguments,
+        _CallbackKeywordArguments,
+    ],
+    Tuple[
+        Union[DeferredErrback, DeferredCallback, Literal[_Sentinel._CONTINUE]],
+        _CallbackOrderedArguments,
+        _CallbackKeywordArguments,
+    ],
+]
+
+_NONE_KWARGS: _CallbackKeywordArguments = MappingProxyType({})
+
+
+_SelfResultT = TypeVar("_SelfResultT")
+_NextResultT = TypeVar("_NextResultT")
+
+
+class DebugInfo:
+    """
+    Deferred debug helper.
+    """
+
+    failResult: Optional[Failure] = None
+    creator: Optional[List[str]] = None
+    invoker: Optional[List[str]] = None
+
+    def _getDebugTracebacks(self) -> str:
+        info = ""
+        if self.creator is not None:
+            info += " C: Deferred was created:\n C:"
+            info += "".join(self.creator).rstrip().replace("\n", "\n C:")
+            info += "\n"
+        if self.invoker is not None:
+            info += " I: First Invoker was:\n I:"
+            info += "".join(self.invoker).rstrip().replace("\n", "\n I:")
+            info += "\n"
+        return info
+
+    def __del__(self) -> None:
+        """
+        Print tracebacks and die.
+
+        If the *last* (and I do mean *last*) callback leaves me in an error
+        state, print a traceback (if said errback is a L{Failure}).
+        """
+        if self.failResult is not None:
+            # Note: this is two separate messages for compatibility with
+            # earlier tests; arguably it should be a single error message.
+            log.critical("Unhandled error in Deferred:", isError=True)
+
+            debugInfo = self._getDebugTracebacks()
+            if debugInfo:
+                format = "(debug: {debugInfo})"
+            else:
+                format = ""
+
+            log.failure(format, self.failResult, debugInfo=debugInfo)
+
+
+class Deferred(Awaitable[_SelfResultT]):
+    """
+    This is a callback which will be put off until later.
+
+    Why do we want this? Well, in cases where a function in a threaded
+    program would block until it gets a result, for Twisted it should
+    not block. Instead, it should return a L{Deferred}.
+
+    This can be implemented for protocols that run over the network by
+    writing an asynchronous protocol for L{twisted.internet}. For methods
+    that come from outside packages that are not under our control, we use
+    threads (see for example L{twisted.enterprise.adbapi}).
+
+    For more information about Deferreds, see doc/core/howto/defer.html or
+    U{http://twistedmatrix.com/documents/current/core/howto/defer.html}
+
+    When creating a Deferred, you may provide a canceller function, which
+    will be called by d.cancel() to let you do any clean-up necessary if the
+    user decides not to wait for the deferred to complete.
+
+    @ivar called: A flag which is C{False} until either C{callback} or
+        C{errback} is called and afterwards always C{True}.
+    @ivar paused: A counter of how many unmatched C{pause} calls have been made
+        on this instance.
+    @ivar _suppressAlreadyCalled: A flag used by the cancellation mechanism
+        which is C{True} if the Deferred has no canceller and has been
+        cancelled, C{False} otherwise.  If C{True}, it can be expected that
+        C{callback} or C{errback} will eventually be called and the result
+        should be silently discarded.
+    @ivar _runningCallbacks: A flag which is C{True} while this instance is
+        executing its callback chain, used to stop recursive execution of
+        L{_runCallbacks}
+    @ivar _chainedTo: If this L{Deferred} is waiting for the result of another
+        L{Deferred}, this is a reference to the other Deferred.  Otherwise,
+        L{None}.
+    """
+
+    called = False
+    paused = 0
+    _debugInfo: Optional[DebugInfo] = None
+    _suppressAlreadyCalled = False
+
+    # Are we currently running a user-installed callback?  Meant to prevent
+    # recursive running of callbacks when a reentrant call to add a callback is
+    # used.
+    _runningCallbacks = False
+
+    # Keep this class attribute for now, for compatibility with code that
+    # sets it directly.
+    debug = False
+
+    _chainedTo: "Optional[Deferred[Any]]" = None
+
+    def __init__(
+        self, canceller: Optional[Callable[["Deferred[Any]"], None]] = None
+    ) -> None:
+        """
+        Initialize a L{Deferred}.
+
+        @param canceller: a callable used to stop the pending operation
+            scheduled by this L{Deferred} when L{Deferred.cancel} is invoked.
+            The canceller will be passed the deferred whose cancellation is
+            requested (i.e., C{self}).
+
+            If a canceller is not given, or does not invoke its argument's
+            C{callback} or C{errback} method, L{Deferred.cancel} will
+            invoke L{Deferred.errback} with a L{CancelledError}.
+
+            Note that if a canceller is not given, C{callback} or
+            C{errback} may still be invoked exactly once, even though
+            defer.py will have already invoked C{errback}, as described
+            above.  This allows clients of code which returns a L{Deferred}
+            to cancel it without requiring the L{Deferred} instantiator to
+            provide any specific implementation support for cancellation.
+            New in 10.1.
+
+        @type canceller: a 1-argument callable which takes a L{Deferred}. The
+            return result is ignored.
+        """
+        self.callbacks: List[_CallbackChain] = []
+        self._canceller = canceller
+        if self.debug:
+            self._debugInfo = DebugInfo()
+            self._debugInfo.creator = traceback.format_stack()[:-1]
+
+    def addCallbacks(
+        self,
+        callback: Union[
+            Callable[..., _NextResultT],
+            Callable[..., Deferred[_NextResultT]],
+            Callable[..., Failure],
+            Callable[
+                ...,
+                Union[_NextResultT, Deferred[_NextResultT], Failure],
+            ],
+        ],
+        errback: Union[
+            Callable[..., _NextResultT],
+            Callable[..., Deferred[_NextResultT]],
+            Callable[..., Failure],
+            Callable[
+                ...,
+                Union[_NextResultT, Deferred[_NextResultT], Failure],
+            ],
+            None,
+        ] = None,
+        callbackArgs: Tuple[Any, ...] = (),
+        callbackKeywords: Mapping[str, Any] = _NONE_KWARGS,
+        errbackArgs: _CallbackOrderedArguments = (),
+        errbackKeywords: _CallbackKeywordArguments = _NONE_KWARGS,
+    ) -> "Deferred[_NextResultT]":
+        """
+        Add a pair of callbacks (success and error) to this L{Deferred}.
+
+        These will be executed when the 'master' callback is run.
+
+        @note: The signature of this function was designed many years before
+            PEP 612; ParamSpec provides no mechanism to annotate parameters
+            like C{callbackArgs}; this is therefore inherently less type-safe
+            than calling C{addCallback} and C{addErrback} separately.
+
+        @return: C{self}.
+        """
+        if errback is None:
+            errback = _failthru
+
+        # Default value used to be None and callers may be using None
+        if callbackArgs is None:
+            callbackArgs = ()  # type: ignore[unreachable]
+        if callbackKeywords is None:
+            callbackKeywords = {}  # type: ignore[unreachable]
+        if errbackArgs is None:
+            errbackArgs = ()  # type: ignore[unreachable]
+        if errbackKeywords is None:
+            errbackKeywords = {}  # type: ignore[unreachable]
+
+        assert callable(callback)
+        assert callable(errback)
+
+        self.callbacks.append(
+            (
+                (callback, callbackArgs, callbackKeywords),
+                (errback, errbackArgs, errbackKeywords),
+            )
+        )
+
+        if self.called:
+            self._runCallbacks()
+
+        # type note: The Deferred's type has changed here, but *idiomatically*
+        #     the caller should treat the result as the new type, consistently.
+        return self  # type:ignore[return-value]
+
+    # BEGIN way too many @overload-s for addCallback, addErrback, and addBoth:
+    # these must be accomplished with @overloads, rather than a big Union on
+    # the result type as you might expect, because the fact that
+    # _NextResultT has no bound makes mypy get confused and require the
+    # return types of functions to be combinations of Deferred and Failure
+    # rather than the actual return type.  I'm not entirely sure what about the
+    # semantics of <nothing> create this overzealousness on the part of trying
+    # to assign a type; there *might* be a mypy bug in there somewhere.
+    # Possibly https://github.com/python/typing/issues/548 is implicated here
+    # because TypeVar for the *callable* with a variadic bound might express to
+    # Mypy the actual constraint that we want on its type.
+
+    @overload
+    def addCallback(
+        self,
+        callback: Callable[Concatenate[_SelfResultT, _P], Failure],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addCallback(
+        self,
+        callback: Callable[
+            Concatenate[_SelfResultT, _P],
+            Union[Failure, Deferred[_NextResultT]],
+        ],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addCallback(
+        self,
+        callback: Callable[Concatenate[_SelfResultT, _P], Union[Failure, _NextResultT]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addCallback(
+        self,
+        callback: Callable[Concatenate[_SelfResultT, _P], Deferred[_NextResultT]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addCallback(
+        self,
+        callback: Callable[
+            Concatenate[_SelfResultT, _P],
+            Union[Deferred[_NextResultT], _NextResultT],
+        ],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addCallback(
+        self,
+        callback: Callable[Concatenate[_SelfResultT, _P], _NextResultT],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    def addCallback(self, callback: Any, *args: Any, **kwargs: Any) -> "Deferred[Any]":
+        """
+        Convenience method for adding just a callback.
+
+        See L{addCallbacks}.
+        """
+        # Implementation Note: Any annotations for brevity; the overloads above
+        # handle specifying the actual signature, and there's nothing worth
+        # type-checking in this implementation.
+        return self.addCallbacks(callback, callbackArgs=args, callbackKeywords=kwargs)
+
+    @overload
+    def addErrback(
+        self,
+        errback: Callable[Concatenate[Failure, _P], Deferred[_NextResultT]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> "Deferred[Union[_SelfResultT, _NextResultT]]":
+        ...
+
+    @overload
+    def addErrback(
+        self,
+        errback: Callable[Concatenate[Failure, _P], Failure],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> "Deferred[Union[_SelfResultT]]":
+        ...
+
+    @overload
+    def addErrback(
+        self,
+        errback: Callable[Concatenate[Failure, _P], _NextResultT],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> "Deferred[Union[_SelfResultT, _NextResultT]]":
+        ...
+
+    def addErrback(self, errback: Any, *args: Any, **kwargs: Any) -> "Deferred[Any]":
+        """
+        Convenience method for adding just an errback.
+
+        See L{addCallbacks}.
+        """
+        # See implementation note in addCallbacks about Any arguments
+        return self.addCallbacks(
+            passthru, errback, errbackArgs=args, errbackKeywords=kwargs
+        )
+
+    @overload
+    def addBoth(
+        self,
+        callback: Callable[Concatenate[Union[_SelfResultT, Failure], _P], Failure],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addBoth(
+        self,
+        callback: Callable[
+            Concatenate[Union[_SelfResultT, Failure], _P],
+            Union[Failure, Deferred[_NextResultT]],
+        ],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addBoth(
+        self,
+        callback: Callable[
+            Concatenate[Union[_SelfResultT, Failure], _P], Union[Failure, _NextResultT]
+        ],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addBoth(
+        self,
+        callback: Callable[
+            Concatenate[Union[_SelfResultT, Failure], _P], Deferred[_NextResultT]
+        ],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addBoth(
+        self,
+        callback: Callable[
+            Concatenate[Union[_SelfResultT, Failure], _P],
+            Union[Deferred[_NextResultT], _NextResultT],
+        ],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addBoth(
+        self,
+        callback: Callable[Concatenate[Union[_SelfResultT, Failure], _P], _NextResultT],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_NextResultT]:
+        ...
+
+    @overload
+    def addBoth(
+        self,
+        callback: Callable[Concatenate[_T, _P], _T],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_SelfResultT]:
+        ...
+
+    def addBoth(self, callback: Any, *args: Any, **kwargs: Any) -> "Deferred[Any]":
+        """
+        Convenience method for adding a single callable as both a callback
+        and an errback.
+
+        See L{addCallbacks}.
+        """
+        # See implementation note in addCallbacks about Any arguments
+        return self.addCallbacks(
+            callback,
+            callback,
+            callbackArgs=args,
+            errbackArgs=args,
+            callbackKeywords=kwargs,
+            errbackKeywords=kwargs,
+        )
+
+    # END way too many overloads
+
+    def addTimeout(
+        self,
+        timeout: float,
+        clock: IReactorTime,
+        onTimeoutCancel: Optional[
+            Callable[
+                [Union[_SelfResultT, Failure], float],
+                Union[_NextResultT, Failure],
+            ]
+        ] = None,
+    ) -> "Deferred[Union[_SelfResultT, _NextResultT]]":
+        """
+        Time out this L{Deferred} by scheduling it to be cancelled after
+        C{timeout} seconds.
+
+        The timeout encompasses all the callbacks and errbacks added to this
+        L{defer.Deferred} before the call to L{addTimeout}, and none added
+        after the call.
+
+        If this L{Deferred} gets timed out, it errbacks with a L{TimeoutError},
+        unless a cancelable function was passed to its initialization or unless
+        a different C{onTimeoutCancel} callable is provided.
+
+        @param timeout: number of seconds to wait before timing out this
+            L{Deferred}
+        @param clock: The object which will be used to schedule the timeout.
+        @param onTimeoutCancel: A callable which is called immediately after
+            this L{Deferred} times out, and not if this L{Deferred} is
+            otherwise cancelled before the timeout. It takes an arbitrary
+            value, which is the value of this L{Deferred} at that exact point
+            in time (probably a L{CancelledError} L{Failure}), and the
+            C{timeout}.  The default callable (if C{None} is provided) will
+            translate a L{CancelledError} L{Failure} into a L{TimeoutError}.
+
+        @return: C{self}.
+
+        @since: 16.5
+        """
+
+        timedOut = [False]
+
+        def timeItOut() -> None:
+            timedOut[0] = True
+            self.cancel()
+
+        delayedCall = clock.callLater(timeout, timeItOut)
+
+        def convertCancelled(
+            result: Union[_SelfResultT, Failure],
+        ) -> Union[_SelfResultT, _NextResultT, Failure]:
+            # if C{deferred} was timed out, call the translation function,
+            # if provided, otherwise just use L{cancelledToTimedOutError}
+            if timedOut[0]:
+                toCall = onTimeoutCancel or _cancelledToTimedOutError
+                return toCall(result, timeout)
+            return result
+
+        def cancelTimeout(result: _T) -> _T:
+            # stop the pending call to cancel the deferred if it's been fired
+            if delayedCall.active():
+                delayedCall.cancel()
+            return result
+
+        # Note: Mypy cannot infer this type, apparently thanks to the ambiguity
+        # of _SelfResultT / _NextResultT both being unbound.  Explicitly
+        # annotating it seems to do the trick though.
+        converted: Deferred[Union[_SelfResultT, _NextResultT]] = self.addBoth(
+            convertCancelled
+        )
+        return converted.addBoth(cancelTimeout)
+
+    def chainDeferred(self, d: "Deferred[_SelfResultT]") -> "Deferred[None]":
+        """
+        Chain another L{Deferred} to this L{Deferred}.
+
+        This method adds callbacks to this L{Deferred} to call C{d}'s callback
+        or errback, as appropriate. It is merely a shorthand way of performing
+        the following::
+
+            d1.addCallbacks(d2.callback, d2.errback)
+
+        When you chain a deferred C{d2} to another deferred C{d1} with
+        C{d1.chainDeferred(d2)}, you are making C{d2} participate in the
+        callback chain of C{d1}.
+        Thus any event that fires C{d1} will also fire C{d2}.
+        However, the converse is B{not} true; if C{d2} is fired, C{d1} will not
+        be affected.
+
+        Note that unlike the case where chaining is caused by a L{Deferred}
+        being returned from a callback, it is possible to cause the call
+        stack size limit to be exceeded by chaining many L{Deferred}s
+        together with C{chainDeferred}.
+
+        @return: C{self}.
+        """
+        d._chainedTo = self
+        return self.addCallbacks(d.callback, d.errback)
+
+    def callback(self, result: Union[_SelfResultT, Failure]) -> None:
+        """
+        Run all success callbacks that have been added to this L{Deferred}.
+
+        Each callback will have its result passed as the first argument to
+        the next; this way, the callbacks act as a 'processing chain'.  If
+        the success-callback returns a L{Failure} or raises an L{Exception},
+        processing will continue on the *error* callback chain.  If a
+        callback (or errback) returns another L{Deferred}, this L{Deferred}
+        will be chained to it (and further callbacks will not run until that
+        L{Deferred} has a result).
+
+        An instance of L{Deferred} may only have either L{callback} or
+        L{errback} called on it, and only once.
+
+        @param result: The object which will be passed to the first callback
+            added to this L{Deferred} (via L{addCallback}), unless C{result} is
+            a L{Failure}, in which case the behavior is the same as calling
+            C{errback(result)}.
+
+        @raise AlreadyCalledError: If L{callback} or L{errback} has already been
+            called on this L{Deferred}.
+        """
+        assert not isinstance(result, Deferred)
+        self._startRunCallbacks(result)
+
+    def errback(self, fail: Optional[Union[Failure, BaseException]] = None) -> None:
+        """
+        Run all error callbacks that have been added to this L{Deferred}.
+
+        Each callback will have its result passed as the first
+        argument to the next; this way, the callbacks act as a
+        'processing chain'. Also, if the error-callback returns a non-Failure
+        or doesn't raise an L{Exception}, processing will continue on the
+        *success*-callback chain.
+
+        If the argument that's passed to me is not a L{Failure} instance,
+        it will be embedded in one. If no argument is passed, a
+        L{Failure} instance will be created based on the current
+        traceback stack.
+
+        Passing a string as `fail' is deprecated, and will be punished with
+        a warning message.
+
+        An instance of L{Deferred} may only have either L{callback} or
+        L{errback} called on it, and only once.
+
+        @param fail: The L{Failure} object which will be passed to the first
+            errback added to this L{Deferred} (via L{addErrback}).
+            Alternatively, a L{Exception} instance from which a L{Failure} will
+            be constructed (with no traceback) or L{None} to create a L{Failure}
+            instance from the current exception state (with a traceback).
+
+        @raise AlreadyCalledError: If L{callback} or L{errback} has already been
+            called on this L{Deferred}.
+        @raise NoCurrentExceptionError: If C{fail} is L{None} but there is
+            no current exception state.
+        """
+        if fail is None:
+            fail = Failure(captureVars=self.debug)
+        elif not isinstance(fail, Failure):
+            fail = Failure(fail)
+
+        self._startRunCallbacks(fail)
+
+    def pause(self) -> None:
+        """
+        Stop processing on a L{Deferred} until L{unpause}() is called.
+        """
+        self.paused = self.paused + 1
+
+    def unpause(self) -> None:
+        """
+        Process all callbacks made since L{pause}() was called.
+        """
+        self.paused = self.paused - 1
+        if self.paused:
+            return
+        if self.called:
+            self._runCallbacks()
+
+    def cancel(self) -> None:
+        """
+        Cancel this L{Deferred}.
+
+        If the L{Deferred} has not yet had its C{errback} or C{callback} method
+        invoked, call the canceller function provided to the constructor. If
+        that function does not invoke C{callback} or C{errback}, or if no
+        canceller function was provided, errback with L{CancelledError}.
+
+        If this L{Deferred} is waiting on another L{Deferred}, forward the
+        cancellation to the other L{Deferred}.
+        """
+        if not self.called:
+            canceller = self._canceller
+            if canceller:
+                canceller(self)
+            else:
+                # Arrange to eat the callback that will eventually be fired
+                # since there was no real canceller.
+                self._suppressAlreadyCalled = True
+            if not self.called:
+                # There was no canceller, or the canceller didn't call
+                # callback or errback.
+                self.errback(Failure(CancelledError()))
+        elif isinstance(self.result, Deferred):
+            # Waiting for another deferred -- cancel it instead.
+            self.result.cancel()
+
+    def _startRunCallbacks(self, result: object) -> None:
+        if self.called:
+            if self._suppressAlreadyCalled:
+                self._suppressAlreadyCalled = False
+                return
+            if self.debug:
+                if self._debugInfo is None:
+                    self._debugInfo = DebugInfo()
+                extra = "\n" + self._debugInfo._getDebugTracebacks()
+                raise AlreadyCalledError(extra)
+            raise AlreadyCalledError
+        if self.debug:
+            if self._debugInfo is None:
+                self._debugInfo = DebugInfo()
+            self._debugInfo.invoker = traceback.format_stack()[:-2]
+        self.called = True
+
+        # Clear the canceller to avoid any circular references. This is safe to
+        # do as the canceller does not get called after the deferred has fired
+        self._canceller = None
+
+        self.result = result
+        self._runCallbacks()
+
+    def _continuation(self) -> _CallbackChain:
+        """
+        Build a tuple of callback and errback with L{_Sentinel._CONTINUE}.
+        """
+        return (
+            (_Sentinel._CONTINUE, (self,), _NONE_KWARGS),
+            (_Sentinel._CONTINUE, (self,), _NONE_KWARGS),
+        )
+
+    def _runCallbacks(self) -> None:
+        """
+        Run the chain of callbacks once a result is available.
+
+        This consists of a simple loop over all of the callbacks, calling each
+        with the current result and making the current result equal to the
+        return value (or raised exception) of that call.
+
+        If L{_runningCallbacks} is true, this loop won't run at all, since
+        it is already running above us on the call stack.  If C{self.paused} is
+        true, the loop also won't run, because that's what it means to be
+        paused.
+
+        The loop will terminate before processing all of the callbacks if a
+        L{Deferred} without a result is encountered.
+
+        If a L{Deferred} I{with} a result is encountered, that result is taken
+        and the loop proceeds.
+
+        @note: The implementation is complicated slightly by the fact that
+            chaining (associating two L{Deferred}s with each other such that one
+            will wait for the result of the other, as happens when a Deferred is
+            returned from a callback on another L{Deferred}) is supported
+            iteratively rather than recursively, to avoid running out of stack
+            frames when processing long chains.
+        """
+        if self._runningCallbacks:
+            # Don't recursively run callbacks
+            return
+
+        # Keep track of all the Deferreds encountered while propagating results
+        # up a chain.  The way a Deferred gets onto this stack is by having
+        # added its _continuation() to the callbacks list of a second Deferred
+        # and then that second Deferred being fired.  ie, if ever had _chainedTo
+        # set to something other than None, you might end up on this stack.
+        chain: List[Deferred[Any]] = [self]
+
+        while chain:
+            current = chain[-1]
+
+            if current.paused:
+                # This Deferred isn't going to produce a result at all.  All the
+                # Deferreds up the chain waiting on it will just have to...
+                # wait.
+                return
+
+            finished = True
+            current._chainedTo = None
+            while current.callbacks:
+                item = current.callbacks.pop(0)
+                if not isinstance(current.result, Failure):
+                    callback, args, kwargs = item[0]
+                else:
+                    # type note: Callback signature also works for Errbacks in
+                    #     this context.
+                    callback, args, kwargs = item[1]
+
+                # Avoid recursion if we can.
+                if callback is _CONTINUE:
+                    # Give the waiting Deferred our current result and then
+                    # forget about that result ourselves.
+                    chainee = cast(Deferred[object], args[0])
+                    chainee.result = current.result
+                    current.result = None
+                    # Making sure to update _debugInfo
+                    if current._debugInfo is not None:
+                        current._debugInfo.failResult = None
+                    chainee.paused -= 1
+                    chain.append(chainee)
+                    # Delay cleaning this Deferred and popping it from the chain
+                    # until after we've dealt with chainee.
+                    finished = False
+                    break
+
+                try:
+                    current._runningCallbacks = True
+                    try:
+                        # type note: mypy sees `callback is _CONTINUE` above and
+                        #    then decides that `callback` is not callable.
+                        #    This goes away when we use `_Sentinel._CONTINUE`
+                        #    instead, but we don't want to do that attribute
+                        #    lookup in this hot code path, so we ignore the mypy
+                        #    complaint here.
+                        current.result = callback(  # type: ignore[misc]
+                            current.result, *args, **kwargs
+                        )
+
+                        if current.result is current:
+                            warnAboutFunction(
+                                callback,
+                                "Callback returned the Deferred "
+                                "it was attached to; this breaks the "
+                                "callback chain and will raise an "
+                                "exception in the future.",
+                            )
+                    finally:
+                        current._runningCallbacks = False
+                except BaseException:
+                    # Including full frame information in the Failure is quite
+                    # expensive, so we avoid it unless self.debug is set.
+                    current.result = Failure(captureVars=self.debug)
+                else:
+                    if isinstance(current.result, Deferred):
+                        # The result is another Deferred.  If it has a result,
+                        # we can take it and keep going.
+                        resultResult = getattr(current.result, "result", _NO_RESULT)
+                        if (
+                            resultResult is _NO_RESULT
+                            or isinstance(resultResult, Deferred)
+                            or current.result.paused
+                        ):
+                            # Nope, it didn't.  Pause and chain.
+                            current.pause()
+                            current._chainedTo = current.result
+                            # Note: current.result has no result, so it's not
+                            # running its callbacks right now.  Therefore we can
+                            # append to the callbacks list directly instead of
+                            # using addCallbacks.
+                            current.result.callbacks.append(current._continuation())
+                            break
+                        else:
+                            # Yep, it did.  Steal it.
+                            current.result.result = None
+                            # Make sure _debugInfo's failure state is updated.
+                            if current.result._debugInfo is not None:
+                                current.result._debugInfo.failResult = None
+                            current.result = resultResult
+
+            if finished:
+                # As much of the callback chain - perhaps all of it - as can be
+                # processed right now has been.  The current Deferred is waiting on
+                # another Deferred or for more callbacks.  Before finishing with it,
+                # make sure its _debugInfo is in the proper state.
+                if isinstance(current.result, Failure):
+                    # Stash the Failure in the _debugInfo for unhandled error
+                    # reporting.
+                    current.result.cleanFailure()
+                    if current._debugInfo is None:
+                        current._debugInfo = DebugInfo()
+                    current._debugInfo.failResult = current.result
+                else:
+                    # Clear out any Failure in the _debugInfo, since the result
+                    # is no longer a Failure.
+                    if current._debugInfo is not None:
+                        current._debugInfo.failResult = None
+
+                # This Deferred is done, pop it from the chain and move back up
+                # to the Deferred which supplied us with our result.
+                chain.pop()
+
+    def __str__(self) -> str:
+        """
+        Return a string representation of this L{Deferred}.
+        """
+        cname = self.__class__.__name__
+        result = getattr(self, "result", _NO_RESULT)
+        myID = id(self)
+        if self._chainedTo is not None:
+            result = f" waiting on Deferred at 0x{id(self._chainedTo):x}"
+        elif result is _NO_RESULT:
+            result = ""
+        else:
+            result = f" current result: {result!r}"
+        return f"<{cname} at 0x{myID:x}{result}>"
+
+    __repr__ = __str__
+
+    def __iter__(self) -> "Deferred[_SelfResultT]":
+        return self
+
+    @_extraneous
+    def send(self, value: object = None) -> "Deferred[_SelfResultT]":
+        if self.paused:
+            # If we're paused, we have no result to give
+            return self
+
+        result = getattr(self, "result", _NO_RESULT)
+        if result is _NO_RESULT:
+            return self
+        if isinstance(result, Failure):
+            # Clear the failure on debugInfo so it doesn't raise "unhandled
+            # exception"
+            assert self._debugInfo is not None
+            self._debugInfo.failResult = None
+            result.value.__failure__ = result
+            raise result.value
+        else:
+            raise StopIteration(result)
+
+    # For PEP-492 support (async/await)
+    # type note: base class "Awaitable" defined the type as:
+    #     Callable[[], Generator[Any, None, _SelfResultT]]
+    #     See: https://github.com/python/typeshed/issues/5125
+    #     When the typeshed patch is included in a mypy release,
+    #     this method can be replaced by `__await__ = __iter__`.
+    def __await__(self) -> Generator[Any, None, _SelfResultT]:
+        return self.__iter__()  # type: ignore[return-value]
+
+    __next__ = send
+
+    def asFuture(self, loop: AbstractEventLoop) -> "Future[_SelfResultT]":
+        """
+        Adapt this L{Deferred} into a L{Future} which is bound to C{loop}.
+
+        @note: converting a L{Deferred} to an L{Future} consumes both
+            its result and its errors, so this method implicitly converts
+            C{self} into a L{Deferred} firing with L{None}, regardless of what
+            its result previously would have been.
+
+        @since: Twisted 17.5.0
+
+        @param loop: The L{asyncio} event loop to bind the L{Future} to.
+
+        @return: A L{Future} which will fire when the L{Deferred} fires.
+        """
+        future = loop.create_future()
+
+        def checkCancel(futureAgain: "Future[_SelfResultT]") -> None:
+            if futureAgain.cancelled():
+                self.cancel()
+
+        def maybeFail(failure: Failure) -> None:
+            if not future.cancelled():
+                future.set_exception(failure.value)
+
+        def maybeSucceed(result: object) -> None:
+            if not future.cancelled():
+                future.set_result(result)
+
+        self.addCallbacks(maybeSucceed, maybeFail)
+        future.add_done_callback(checkCancel)
+
+        return future
+
+    @classmethod
+    def fromFuture(cls, future: "Future[_SelfResultT]") -> "Deferred[_SelfResultT]":
+        """
+        Adapt a L{Future} to a L{Deferred}.
+
+        @note: This creates a L{Deferred} from a L{Future}, I{not} from
+            a C{coroutine}; in other words, you will need to call
+            L{asyncio.ensure_future}, L{asyncio.loop.create_task} or create an
+            L{asyncio.Task} yourself to get from a C{coroutine} to a
+            L{Future} if what you have is an awaitable coroutine and
+            not a L{Future}.  (The length of this list of techniques is
+            exactly why we have left it to the caller!)
+
+        @since: Twisted 17.5.0
+
+        @param future: The L{Future} to adapt.
+
+        @return: A L{Deferred} which will fire when the L{Future} fires.
+        """
+
+        def adapt(result: Future[_SelfResultT]) -> None:
+            try:
+                extracted: _SelfResultT | Failure = result.result()
+            except BaseException:
+                extracted = Failure()
+            actual.callback(extracted)
+
+        futureCancel = object()
+
+        def cancel(reself: Deferred[object]) -> None:
+            future.cancel()
+            reself.callback(futureCancel)
+
+        self = cls(cancel)
+        actual = self
+
+        def uncancel(
+            result: _SelfResultT,
+        ) -> Union[_SelfResultT, Deferred[_SelfResultT]]:
+            if result is futureCancel:
+                nonlocal actual
+                actual = Deferred()
+                return actual
+            return result
+
+        self.addCallback(uncancel)
+        future.add_done_callback(adapt)
+
+        return self
+
+    @classmethod
+    def fromCoroutine(
+        cls,
+        coro: Union[
+            Coroutine[Deferred[Any], Any, _T],
+            Generator[Deferred[Any], Any, _T],
+        ],
+    ) -> "Deferred[_T]":
+        """
+        Schedule the execution of a coroutine that awaits on L{Deferred}s,
+        wrapping it in a L{Deferred} that will fire on success/failure of the
+        coroutine.
+
+        Coroutine functions return a coroutine object, similar to how
+        generators work. This function turns that coroutine into a Deferred,
+        meaning that it can be used in regular Twisted code. For example::
+
+            import treq
+            from twisted.internet.defer import Deferred
+            from twisted.internet.task import react
+
+            async def crawl(pages):
+                results = {}
+                for page in pages:
+                    results[page] = await treq.content(await treq.get(page))
+                return results
+
+            def main(reactor):
+                pages = [
+                    "http://localhost:8080"
+                ]
+                d = Deferred.fromCoroutine(crawl(pages))
+                d.addCallback(print)
+                return d
+
+            react(main)
+
+        @since: Twisted 21.2.0
+
+        @param coro: The coroutine object to schedule.
+
+        @raise ValueError: If C{coro} is not a coroutine or generator.
+        """
+        # asyncio.iscoroutine <3.12 identifies generators as coroutines, too.
+        # for >=3.12 we need to check isgenerator also
+        # see https://github.com/python/cpython/issues/102748
+        if iscoroutine(coro) or inspect.isgenerator(coro):
+            return _cancellableInlineCallbacks(coro)
+        raise NotACoroutineError(f"{coro!r} is not a coroutine")
+
+
+def ensureDeferred(
+    coro: Union[
+        Coroutine[Deferred[Any], Any, _T],
+        Generator[Deferred[Any], Any, _T],
+        Deferred[_T],
+    ]
+) -> Deferred[_T]:
+    """
+    Schedule the execution of a coroutine that awaits/yields from L{Deferred}s,
+    wrapping it in a L{Deferred} that will fire on success/failure of the
+    coroutine. If a Deferred is passed to this function, it will be returned
+    directly (mimicking the L{asyncio.ensure_future} function).
+
+    See L{Deferred.fromCoroutine} for examples of coroutines.
+
+    @param coro: The coroutine object to schedule, or a L{Deferred}.
+    """
+    if isinstance(coro, Deferred):
+        return coro
+    else:
+        try:
+            return Deferred.fromCoroutine(coro)
+        except NotACoroutineError:
+            # It's not a coroutine. Raise an exception, but say that it's also
+            # not a Deferred so the error makes sense.
+            raise NotACoroutineError(f"{coro!r} is not a coroutine or a Deferred")
+
+
+@comparable
+class FirstError(Exception):
+    """
+    First error to occur in a L{DeferredList} if C{fireOnOneErrback} is set.
+
+    @ivar subFailure: The L{Failure} that occurred.
+    @ivar index: The index of the L{Deferred} in the L{DeferredList} where
+        it happened.
+    """
+
+    def __init__(self, failure: Failure, index: int) -> None:
+        Exception.__init__(self, failure, index)
+        self.subFailure = failure
+        self.index = index
+
+    def __repr__(self) -> str:
+        """
+        The I{repr} of L{FirstError} instances includes the repr of the
+        wrapped failure's exception and the index of the L{FirstError}.
+        """
+        return "FirstError[#%d, %r]" % (self.index, self.subFailure.value)
+
+    def __str__(self) -> str:
+        """
+        The I{str} of L{FirstError} instances includes the I{str} of the
+        entire wrapped failure (including its traceback and exception) and
+        the index of the L{FirstError}.
+        """
+        return "FirstError[#%d, %s]" % (self.index, self.subFailure)
+
+    def __cmp__(self, other: object) -> int:
+        """
+        Comparison between L{FirstError} and other L{FirstError} instances
+        is defined as the comparison of the index and sub-failure of each
+        instance.  L{FirstError} instances don't compare equal to anything
+        that isn't a L{FirstError} instance.
+
+        @since: 8.2
+        """
+        if isinstance(other, FirstError):
+            return cmp((self.index, self.subFailure), (other.index, other.subFailure))
+        return -1
+
+
+_DeferredListSingleResultT = Tuple[_SelfResultT, int]
+_DeferredListResultItemT = Tuple[bool, _SelfResultT]
+_DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]]
+
+if TYPE_CHECKING:
+    # The result type is different depending on whether fireOnOneCallback
+    # is True or False.  The type system is not flexible enough to handle
+    # that in a class definition, so instead we pretend that DeferredList
+    # is a function that returns a Deferred.
+
+    @overload
+    def _DeferredList(
+        deferredList: Iterable[Deferred[_SelfResultT]],
+        fireOnOneCallback: Literal[True],
+        fireOnOneErrback: bool = False,
+        consumeErrors: bool = False,
+    ) -> Deferred[_DeferredListSingleResultT[_SelfResultT]]:
+        ...
+
+    @overload
+    def _DeferredList(
+        deferredList: Iterable[Deferred[_SelfResultT]],
+        fireOnOneCallback: Literal[False] = False,
+        fireOnOneErrback: bool = False,
+        consumeErrors: bool = False,
+    ) -> Deferred[_DeferredListResultListT[_SelfResultT]]:
+        ...
+
+    def _DeferredList(
+        deferredList: Iterable[Deferred[_SelfResultT]],
+        fireOnOneCallback: bool = False,
+        fireOnOneErrback: bool = False,
+        consumeErrors: bool = False,
+    ) -> Union[
+        Deferred[_DeferredListSingleResultT[_SelfResultT]],
+        Deferred[_DeferredListResultListT[_SelfResultT]],
+    ]:
+        ...
+
+    DeferredList = _DeferredList
+
+
+class DeferredList(  # type: ignore[no-redef] # noqa:F811
+    Deferred[_DeferredListResultListT[Any]]
+):
+    """
+    L{DeferredList} is a tool for collecting the results of several Deferreds.
+
+    This tracks a list of L{Deferred}s for their results, and makes a single
+    callback when they have all completed.  By default, the ultimate result is a
+    list of (success, result) tuples, 'success' being a boolean.
+    L{DeferredList} exposes the same API that L{Deferred} does, so callbacks and
+    errbacks can be added to it in the same way.
+
+    L{DeferredList} is implemented by adding callbacks and errbacks to each
+    L{Deferred} in the list passed to it.  This means callbacks and errbacks
+    added to the Deferreds before they are passed to L{DeferredList} will change
+    the result that L{DeferredList} sees (i.e., L{DeferredList} is not special).
+    Callbacks and errbacks can also be added to the Deferreds after they are
+    passed to L{DeferredList} and L{DeferredList} may change the result that
+    they see.
+
+    See the documentation for the C{__init__} arguments for more information.
+
+    @ivar _deferredList: The L{list} of L{Deferred}s to track.
+    """
+
+    fireOnOneCallback = False
+    fireOnOneErrback = False
+
+    def __init__(
+        self,
+        deferredList: Iterable[Deferred[_SelfResultT]],
+        fireOnOneCallback: bool = False,
+        fireOnOneErrback: bool = False,
+        consumeErrors: bool = False,
+    ):
+        """
+        Initialize a DeferredList.
+
+        @param deferredList: The deferreds to track.
+        @param fireOnOneCallback: (keyword param) a flag indicating that this
+            L{DeferredList} will fire when the first L{Deferred} in
+            C{deferredList} fires with a non-failure result without waiting for
+            any of the other Deferreds.  When this flag is set, the DeferredList
+            will fire with a two-tuple: the first element is the result of the
+            Deferred which fired; the second element is the index in
+            C{deferredList} of that Deferred.
+        @param fireOnOneErrback: (keyword param) a flag indicating that this
+            L{DeferredList} will fire when the first L{Deferred} in
+            C{deferredList} fires with a failure result without waiting for any
+            of the other Deferreds.  When this flag is set, if a Deferred in the
+            list errbacks, the DeferredList will errback with a L{FirstError}
+            failure wrapping the failure of that Deferred.
+        @param consumeErrors: (keyword param) a flag indicating that failures in
+            any of the included L{Deferred}s should not be propagated to
+            errbacks added to the individual L{Deferred}s after this
+            L{DeferredList} is constructed.  After constructing the
+            L{DeferredList}, any errors in the individual L{Deferred}s will be
+            converted to a callback result of L{None}.  This is useful to
+            prevent spurious 'Unhandled error in Deferred' messages from being
+            logged.  This does not prevent C{fireOnOneErrback} from working.
+        """
+        self._deferredList = list(deferredList)
+
+        # Note this contains optional result values as the DeferredList is
+        # processing its results, even though the callback result will not,
+        # which is why we aren't using _DeferredListResultListT here.
+        self.resultList: List[Optional[_DeferredListResultItemT[Any]]] = [None] * len(
+            self._deferredList
+        )
+        """
+        The final result, in progress.
+        Each item in the list corresponds to the L{Deferred} at the same
+        position in L{_deferredList}. It will be L{None} if the L{Deferred}
+        did not complete yet, or a C{(success, result)} pair if it did.
+        """
+
+        Deferred.__init__(self)
+        if len(self._deferredList) == 0 and not fireOnOneCallback:
+            self.callback([])
+
+        # These flags need to be set *before* attaching callbacks to the
+        # deferreds, because the callbacks use these flags, and will run
+        # synchronously if any of the deferreds are already fired.
+        self.fireOnOneCallback = fireOnOneCallback
+        self.fireOnOneErrback = fireOnOneErrback
+        self.consumeErrors = consumeErrors
+        self.finishedCount = 0
+
+        index = 0
+        for deferred in self._deferredList:
+            deferred.addCallbacks(
+                self._cbDeferred,
+                self._cbDeferred,
+                callbackArgs=(index, SUCCESS),
+                errbackArgs=(index, FAILURE),
+            )
+            index = index + 1
+
+    def _cbDeferred(
+        self, result: _SelfResultT, index: int, succeeded: bool
+    ) -> Optional[_SelfResultT]:
+        """
+        (internal) Callback for when one of my deferreds fires.
+        """
+        self.resultList[index] = (succeeded, result)
+
+        self.finishedCount += 1
+        if not self.called:
+            if succeeded == SUCCESS and self.fireOnOneCallback:
+                self.callback((result, index))  # type: ignore[arg-type]
+            elif succeeded == FAILURE and self.fireOnOneErrback:
+                assert isinstance(result, Failure)
+                self.errback(Failure(FirstError(result, index)))
+            elif self.finishedCount == len(self.resultList):
+                # At this point, None values in self.resultList have been
+                # replaced by result values, so we cast it to
+                # _DeferredListResultListT to match the callback result type.
+                self.callback(cast(_DeferredListResultListT[Any], self.resultList))
+
+        if succeeded == FAILURE and self.consumeErrors:
+            return None
+
+        return result
+
+    def cancel(self) -> None:
+        """
+        Cancel this L{DeferredList}.
+
+        If the L{DeferredList} hasn't fired yet, cancel every L{Deferred} in
+        the list.
+
+        If the L{DeferredList} has fired, including the case where the
+        C{fireOnOneCallback}/C{fireOnOneErrback} flag is set and the
+        L{DeferredList} fires because one L{Deferred} in the list fires with a
+        non-failure/failure result, do nothing in the C{cancel} method.
+        """
+        if not self.called:
+            for deferred in self._deferredList:
+                try:
+                    deferred.cancel()
+                except BaseException:
+                    log.failure("Exception raised from user supplied canceller")
+
+
+def _parseDeferredListResult(
+    resultList: List[_DeferredListResultItemT[_T]], fireOnOneErrback: bool = False
+) -> List[_T]:
+    if __debug__:
+        for result in resultList:
+            assert result is not None
+            success, value = result
+            assert success
+    return [x[1] for x in resultList]
+
+
+def gatherResults(
+    deferredList: Iterable[Deferred[_T]], consumeErrors: bool = False
+) -> Deferred[List[_T]]:
+    """
+    Returns, via a L{Deferred}, a list with the results of the given
+    L{Deferred}s - in effect, a "join" of multiple deferred operations.
+
+    The returned L{Deferred} will fire when I{all} of the provided L{Deferred}s
+    have fired, or when any one of them has failed.
+
+    This method can be cancelled by calling the C{cancel} method of the
+    L{Deferred}, all the L{Deferred}s in the list will be cancelled.
+
+    This differs from L{DeferredList} in that you don't need to parse
+    the result for success/failure.
+
+    @param consumeErrors: (keyword param) a flag, defaulting to False,
+        indicating that failures in any of the given L{Deferred}s should not be
+        propagated to errbacks added to the individual L{Deferred}s after this
+        L{gatherResults} invocation.  Any such errors in the individual
+        L{Deferred}s will be converted to a callback result of L{None}.  This
+        is useful to prevent spurious 'Unhandled error in Deferred' messages
+        from being logged.  This parameter is available since 11.1.0.
+    """
+    d = DeferredList(deferredList, fireOnOneErrback=True, consumeErrors=consumeErrors)
+    d.addCallback(_parseDeferredListResult)
+    return cast(Deferred[List[_T]], d)
+
+
+class FailureGroup(Exception):
+    """
+    More than one failure occurred.
+    """
+
+    def __init__(self, failures: Sequence[Failure]) -> None:
+        super(FailureGroup, self).__init__()
+        self.failures = failures
+
+
+def race(ds: Sequence[Deferred[_T]]) -> Deferred[tuple[int, _T]]:
+    """
+    Select the first available result from the sequence of Deferreds and
+    cancel the rest.
+
+    @return: A cancellable L{Deferred} that fires with the index and output of
+        the element of C{ds} to have a success result first, or that fires
+        with L{FailureGroup} holding a list of their failures if they all
+        fail.
+    """
+    # Keep track of the Deferred for the action which completed first.  When
+    # it completes, all of the other Deferreds will get cancelled but this one
+    # shouldn't be.  Even though it "completed" it isn't really done - the
+    # caller will still be using it for something.  If we cancelled it,
+    # cancellation could propagate down to them.
+    winner: Optional[Deferred[_T]] = None
+
+    # The cancellation function for the Deferred this function returns.
+    def cancel(result: Deferred[_T]) -> None:
+        # If it is cancelled then we cancel all of the Deferreds for the
+        # individual actions because there is no longer the possibility of
+        # delivering any of their results anywhere.  We don't have to fire
+        # `result` because the Deferred will do that for us.
+        for d in to_cancel:
+            d.cancel()
+
+    # The Deferred that this function will return.  It will fire with the
+    # index and output of the action that completes first, or errback if all
+    # of the actions fail.  If it is cancelled, all of the actions will be
+    # cancelled.
+    final_result: Deferred[tuple[int, _T]] = Deferred(canceller=cancel)
+
+    # A callback for an individual action.
+    def succeeded(this_output: _T, this_index: int) -> None:
+        # If it is the first action to succeed then it becomes the "winner",
+        # its index/output become the externally visible result, and the rest
+        # of the action Deferreds get cancelled.  If it is not the first
+        # action to succeed (because some action did not support
+        # cancellation), just ignore the result.  It is uncommon for this
+        # callback to be entered twice.  The only way it can happen is if one
+        # of the input Deferreds has a cancellation function that fires the
+        # Deferred with a success result.
+        nonlocal winner
+        if winner is None:
+            # This is the first success.  Act on it.
+            winner = to_cancel[this_index]
+
+            # Cancel the rest.
+            for d in to_cancel:
+                if d is not winner:
+                    d.cancel()
+
+            # Fire our Deferred
+            final_result.callback((this_index, this_output))
+
+    # Keep track of how many actions have failed.  If they all fail we need to
+    # deliver failure notification on our externally visible result.
+    failure_state = []
+
+    def failed(failure: Failure, this_index: int) -> None:
+        failure_state.append((this_index, failure))
+        if len(failure_state) == len(to_cancel):
+            # Every operation failed.
+            failure_state.sort()
+            failures = [f for (ignored, f) in failure_state]
+            final_result.errback(FailureGroup(failures))
+
+    # Copy the sequence of Deferreds so we know it doesn't get mutated out
+    # from under us.
+    to_cancel = list(ds)
+    for index, d in enumerate(ds):
+        # Propagate the position of this action as well as the argument to f
+        # to the success callback so we can cancel the right Deferreds and
+        # propagate the result outwards.
+        d.addCallbacks(succeeded, failed, callbackArgs=(index,), errbackArgs=(index,))
+
+    return final_result
+
+
+# Constants for use with DeferredList
+SUCCESS = True
+FAILURE = False
+
+
+## deferredGenerator
+class waitForDeferred:
+    """
+    See L{deferredGenerator}.
+    """
+
+    result: Any = _NO_RESULT
+
+    def __init__(self, d: Deferred[object]) -> None:
+        warnings.warn(
+            "twisted.internet.defer.waitForDeferred was deprecated in "
+            "Twisted 15.0.0; please use twisted.internet.defer.inlineCallbacks "
+            "instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        if not isinstance(d, Deferred):
+            raise TypeError(
+                f"You must give waitForDeferred a Deferred. You gave it {d!r}."
+            )
+        self.d = d
+
+    def getResult(self) -> Any:
+        if isinstance(self.result, Failure):
+            self.result.raiseException()
+        self.result is not _NO_RESULT
+        return self.result
+
+
+_DeferableGenerator = Generator[object, None, None]
+
+
+def _deferGenerator(
+    g: _DeferableGenerator, deferred: Deferred[object]
+) -> Deferred[Any]:
+    """
+    See L{deferredGenerator}.
+    """
+
+    result = None
+
+    # This function is complicated by the need to prevent unbounded recursion
+    # arising from repeatedly yielding immediately ready deferreds.  This while
+    # loop and the waiting variable solve that by manually unfolding the
+    # recursion.
+
+    # defgen is waiting for result?  # result
+    # type note: List[Any] because you can't annotate List items by index.
+    #     …better fix would be to create a class, but we need to jettison
+    #     deferredGenerator anyway.
+    waiting: List[Any] = [True, None]
+
+    while 1:
+        try:
+            result = next(g)
+        except StopIteration:
+            deferred.callback(result)
+            return deferred
+        except BaseException:
+            deferred.errback()
+            return deferred
+
+        # Deferred.callback(Deferred) raises an error; we catch this case
+        # early here and give a nicer error message to the user in case
+        # they yield a Deferred.
+        if isinstance(result, Deferred):
+            return fail(TypeError("Yield waitForDeferred(d), not d!"))
+
+        if isinstance(result, waitForDeferred):
+            # a waitForDeferred was yielded, get the result.
+            # Pass result in so it don't get changed going around the loop
+            # This isn't a problem for waiting, as it's only reused if
+            # gotResult has already been executed.
+            def gotResult(
+                r: object, result: waitForDeferred = cast(waitForDeferred, result)
+            ) -> None:
+                result.result = r
+                if waiting[0]:
+                    waiting[0] = False
+                    waiting[1] = r
+                else:
+                    _deferGenerator(g, deferred)
+
+            result.d.addBoth(gotResult)
+            if waiting[0]:
+                # Haven't called back yet, set flag so that we get reinvoked
+                # and return from the loop
+                waiting[0] = False
+                return deferred
+            # Reset waiting to initial values for next loop
+            waiting[0] = True
+            waiting[1] = None
+
+            result = None
+
+
+@deprecated(Version("Twisted", 15, 0, 0), "twisted.internet.defer.inlineCallbacks")
+def deferredGenerator(
+    f: Callable[..., _DeferableGenerator]
+) -> Callable[..., Deferred[object]]:
+    """
+    L{deferredGenerator} and L{waitForDeferred} help you write
+    L{Deferred}-using code that looks like a regular sequential function.
+    Consider the use of L{inlineCallbacks} instead, which can accomplish
+    the same thing in a more concise manner.
+
+    There are two important functions involved: L{waitForDeferred}, and
+    L{deferredGenerator}.  They are used together, like this::
+
+        @deferredGenerator
+        def thingummy():
+            thing = waitForDeferred(makeSomeRequestResultingInDeferred())
+            yield thing
+            thing = thing.getResult()
+            print(thing) #the result! hoorj!
+
+    L{waitForDeferred} returns something that you should immediately yield; when
+    your generator is resumed, calling C{thing.getResult()} will either give you
+    the result of the L{Deferred} if it was a success, or raise an exception if it
+    was a failure.  Calling C{getResult} is B{absolutely mandatory}.  If you do
+    not call it, I{your program will not work}.
+
+    L{deferredGenerator} takes one of these waitForDeferred-using generator
+    functions and converts it into a function that returns a L{Deferred}. The
+    result of the L{Deferred} will be the last value that your generator yielded
+    unless the last value is a L{waitForDeferred} instance, in which case the
+    result will be L{None}.  If the function raises an unhandled exception, the
+    L{Deferred} will errback instead.  Remember that C{return result} won't work;
+    use C{yield result; return} in place of that.
+
+    Note that not yielding anything from your generator will make the L{Deferred}
+    result in L{None}. Yielding a L{Deferred} from your generator is also an error
+    condition; always yield C{waitForDeferred(d)} instead.
+
+    The L{Deferred} returned from your deferred generator may also errback if your
+    generator raised an exception.  For example::
+
+        @deferredGenerator
+        def thingummy():
+            thing = waitForDeferred(makeSomeRequestResultingInDeferred())
+            yield thing
+            thing = thing.getResult()
+            if thing == 'I love Twisted':
+                # will become the result of the Deferred
+                yield 'TWISTED IS GREAT!'
+                return
+            else:
+                # will trigger an errback
+                raise Exception('DESTROY ALL LIFE')
+
+    Put succinctly, these functions connect deferred-using code with this 'fake
+    blocking' style in both directions: L{waitForDeferred} converts from a
+    L{Deferred} to the 'blocking' style, and L{deferredGenerator} converts from the
+    'blocking' style to a L{Deferred}.
+    """
+
+    @wraps(f)
+    def unwindGenerator(*args: object, **kwargs: object) -> Deferred[object]:
+        return _deferGenerator(f(*args, **kwargs), Deferred())
+
+    return unwindGenerator
+
+
+## inlineCallbacks
+
+
+class _DefGen_Return(BaseException):
+    def __init__(self, value: object) -> None:
+        self.value = value
+
+
+def returnValue(val: object) -> NoReturn:
+    """
+    Return val from a L{inlineCallbacks} generator.
+
+    Note: this is currently implemented by raising an exception
+    derived from L{BaseException}.  You might want to change any
+    'except:' clauses to an 'except Exception:' clause so as not to
+    catch this exception.
+
+    Also: while this function currently will work when called from
+    within arbitrary functions called from within the generator, do
+    not rely upon this behavior.
+    """
+    raise _DefGen_Return(val)
+
+
+@attr.s(auto_attribs=True)
+class _CancellationStatus(Generic[_SelfResultT]):
+    """
+    Cancellation status of an L{inlineCallbacks} invocation.
+
+    @ivar deferred: the L{Deferred} to callback or errback when the generator
+        invocation has finished.
+    @ivar waitingOn: the L{Deferred} being waited upon (which
+        L{_inlineCallbacks} must fill out before returning)
+    """
+
+    deferred: Deferred[_SelfResultT]
+    waitingOn: Optional[Deferred[_SelfResultT]] = None
+
+
+def _gotResultInlineCallbacks(
+    r: object,
+    waiting: List[Any],
+    gen: Union[
+        Generator[Deferred[Any], Any, _T],
+        Coroutine[Deferred[Any], Any, _T],
+    ],
+    status: _CancellationStatus[_T],
+    context: _Context,
+) -> None:
+    """
+    Helper for L{_inlineCallbacks} to handle a nested L{Deferred} firing.
+
+    @param r: The result of the L{Deferred}
+    @param waiting: Whether the L{_inlineCallbacks} was waiting, and the result.
+    @param gen: a generator object returned by calling a function or method
+        decorated with C{@}L{inlineCallbacks}
+    @param status: a L{_CancellationStatus} tracking the current status of C{gen}
+    @param context: the contextvars context to run `gen` in
+    """
+    if waiting[0]:
+        waiting[0] = False
+        waiting[1] = r
+    else:
+        _inlineCallbacks(r, gen, status, context)
+
+
+@_extraneous
+def _inlineCallbacks(
+    result: object,
+    gen: Union[
+        Generator[Deferred[Any], Any, _T],
+        Coroutine[Deferred[Any], Any, _T],
+    ],
+    status: _CancellationStatus[_T],
+    context: _Context,
+) -> None:
+    """
+    Carry out the work of L{inlineCallbacks}.
+
+    Iterate the generator produced by an C{@}L{inlineCallbacks}-decorated
+    function, C{gen}, C{send()}ing it the results of each value C{yield}ed by
+    that generator, until a L{Deferred} is yielded, at which point a callback
+    is added to that L{Deferred} to call this function again.
+
+    @param result: The last result seen by this generator.  Note that this is
+        never a L{Deferred} - by the time this function is invoked, the
+        L{Deferred} has been called back and this will be a particular result
+        at a point in its callback chain.
+
+    @param gen: a generator object returned by calling a function or method
+        decorated with C{@}L{inlineCallbacks}
+
+    @param status: a L{_CancellationStatus} tracking the current status of C{gen}
+
+    @param context: the contextvars context to run `gen` in
+    """
+    # This function is complicated by the need to prevent unbounded recursion
+    # arising from repeatedly yielding immediately ready deferreds.  This while
+    # loop and the waiting variable solve that by manually unfolding the
+    # recursion.
+
+    # waiting for result?  # result
+    waiting: List[Any] = [True, None]
+
+    stopIteration: bool = False
+    callbackValue: Any = None
+
+    while 1:
+        try:
+            # Send the last result back as the result of the yield expression.
+            isFailure = isinstance(result, Failure)
+
+            if isFailure:
+                result = context.run(
+                    cast(Failure, result).throwExceptionIntoGenerator, gen
+                )
+            else:
+                result = context.run(gen.send, result)
+        except StopIteration as e:
+            # fell off the end, or "return" statement
+            stopIteration = True
+            callbackValue = getattr(e, "value", None)
+
+        except _DefGen_Return as e:
+            # returnValue() was called; time to give a result to the original
+            # Deferred.  First though, let's try to identify the potentially
+            # confusing situation which results when returnValue() is
+            # accidentally invoked from a different function, one that wasn't
+            # decorated with @inlineCallbacks.
+
+            # The traceback starts in this frame (the one for
+            # _inlineCallbacks); the next one down should be the application
+            # code.
+            excInfo = exc_info()
+            assert excInfo is not None
+
+            traceback = excInfo[2]
+            assert traceback is not None
+
+            appCodeTrace = traceback.tb_next
+            assert appCodeTrace is not None
+
+            if _PYPY:
+                # PyPy as of 3.7 adds an extra frame.
+                appCodeTrace = appCodeTrace.tb_next
+                assert appCodeTrace is not None
+
+            if isFailure:
+                # If we invoked this generator frame by throwing an exception
+                # into it, then throwExceptionIntoGenerator will consume an
+                # additional stack frame itself, so we need to skip that too.
+                appCodeTrace = appCodeTrace.tb_next
+                assert appCodeTrace is not None
+
+            # Now that we've identified the frame being exited by the
+            # exception, let's figure out if returnValue was called from it
+            # directly.  returnValue itself consumes a stack frame, so the
+            # application code will have a tb_next, but it will *not* have a
+            # second tb_next.
+            assert appCodeTrace.tb_next is not None
+            if appCodeTrace.tb_next.tb_next:
+                # If returnValue was invoked non-local to the frame which it is
+                # exiting, identify the frame that ultimately invoked
+                # returnValue so that we can warn the user, as this behavior is
+                # confusing.
+                ultimateTrace = appCodeTrace
+
+                assert ultimateTrace is not None
+                assert ultimateTrace.tb_next is not None
+                while ultimateTrace.tb_next.tb_next:
+                    ultimateTrace = ultimateTrace.tb_next
+                    assert ultimateTrace is not None
+
+                filename = ultimateTrace.tb_frame.f_code.co_filename
+                lineno = ultimateTrace.tb_lineno
+
+                assert ultimateTrace.tb_frame is not None
+                assert appCodeTrace.tb_frame is not None
+                warnings.warn_explicit(
+                    "returnValue() in %r causing %r to exit: "
+                    "returnValue should only be invoked by functions decorated "
+                    "with inlineCallbacks"
+                    % (
+                        ultimateTrace.tb_frame.f_code.co_name,
+                        appCodeTrace.tb_frame.f_code.co_name,
+                    ),
+                    DeprecationWarning,
+                    filename,
+                    lineno,
+                )
+
+            stopIteration = True
+            callbackValue = e.value
+
+        except BaseException:
+            status.deferred.errback()
+            return
+
+        if stopIteration:
+            # Call the callback outside of the exception handler to avoid inappropriate/confusing
+            # "During handling of the above exception, another exception occurred:" if the callback
+            # itself throws an exception.
+            status.deferred.callback(callbackValue)
+            return
+
+        if isinstance(result, Deferred):
+            # a deferred was yielded, get the result.
+            result.addBoth(_gotResultInlineCallbacks, waiting, gen, status, context)
+            if waiting[0]:
+                # Haven't called back yet, set flag so that we get reinvoked
+                # and return from the loop
+                waiting[0] = False
+                status.waitingOn = result
+                return
+
+            result = waiting[1]
+            # Reset waiting to initial values for next loop.  gotResult uses
+            # waiting, but this isn't a problem because gotResult is only
+            # executed once, and if it hasn't been executed yet, the return
+            # branch above would have been taken.
+
+            waiting[0] = True
+            waiting[1] = None
+
+
+def _addCancelCallbackToDeferred(
+    it: Deferred[_T], status: _CancellationStatus[_T]
+) -> None:
+    """
+    Helper for L{_cancellableInlineCallbacks} to add
+    L{_handleCancelInlineCallbacks} as the first errback.
+
+    @param it: The L{Deferred} to add the errback to.
+    @param status: a L{_CancellationStatus} tracking the current status of C{gen}
+    """
+    it.callbacks, tmp = [], it.callbacks
+    it.addErrback(_handleCancelInlineCallbacks, status)
+    it.callbacks.extend(tmp)
+    it.errback(_InternalInlineCallbacksCancelledError())
+
+
+def _handleCancelInlineCallbacks(
+    result: Failure,
+    status: _CancellationStatus[_T],
+) -> Deferred[_T]:
+    """
+    Propagate the cancellation of an C{@}L{inlineCallbacks} to the
+    L{Deferred} it is waiting on.
+
+    @param result: An L{_InternalInlineCallbacksCancelledError} from
+        C{cancel()}.
+    @param status: a L{_CancellationStatus} tracking the current status of C{gen}
+    @return: A new L{Deferred} that the C{@}L{inlineCallbacks} generator
+        can callback or errback through.
+    """
+    result.trap(_InternalInlineCallbacksCancelledError)
+    status.deferred = Deferred(lambda d: _addCancelCallbackToDeferred(d, status))
+
+    # We would only end up here if the inlineCallback is waiting on
+    # another Deferred.  It needs to be cancelled.
+    awaited = status.waitingOn
+    assert awaited is not None
+    awaited.cancel()
+
+    return status.deferred
+
+
+def _cancellableInlineCallbacks(
+    gen: Union[
+        Generator[Deferred[Any], object, _T],
+        Coroutine[Deferred[Any], object, _T],
+    ]
+) -> Deferred[_T]:
+    """
+    Make an C{@}L{inlineCallbacks} cancellable.
+
+    @param gen: a generator object returned by calling a function or method
+        decorated with C{@}L{inlineCallbacks}
+
+    @return: L{Deferred} for the C{@}L{inlineCallbacks} that is cancellable.
+    """
+
+    deferred: Deferred[_T] = Deferred(lambda d: _addCancelCallbackToDeferred(d, status))
+    status = _CancellationStatus(deferred)
+
+    _inlineCallbacks(None, gen, status, _copy_context())
+
+    return deferred
+
+
+class _InternalInlineCallbacksCancelledError(Exception):
+    """
+    A unique exception used only in L{_cancellableInlineCallbacks} to verify
+    that an L{inlineCallbacks} is being cancelled as expected.
+    """
+
+
+def inlineCallbacks(
+    f: Callable[_P, Generator[Deferred[Any], Any, _T]]
+) -> Callable[_P, Deferred[_T]]:
+    """
+    L{inlineCallbacks} helps you write L{Deferred}-using code that looks like a
+    regular sequential function. For example::
+
+        @inlineCallbacks
+        def thingummy():
+            thing = yield makeSomeRequestResultingInDeferred()
+            print(thing)  # the result! hoorj!
+
+    When you call anything that results in a L{Deferred}, you can simply yield it;
+    your generator will automatically be resumed when the Deferred's result is
+    available. The generator will be sent the result of the L{Deferred} with the
+    'send' method on generators, or if the result was a failure, 'throw'.
+
+    Things that are not L{Deferred}s may also be yielded, and your generator
+    will be resumed with the same object sent back. This means C{yield}
+    performs an operation roughly equivalent to L{maybeDeferred}.
+
+    Your inlineCallbacks-enabled generator will return a L{Deferred} object, which
+    will result in the return value of the generator (or will fail with a
+    failure object if your generator raises an unhandled exception). Note that
+    you can't use C{return result} to return a value; use C{returnValue(result)}
+    instead. Falling off the end of the generator, or simply using C{return}
+    will cause the L{Deferred} to have a result of L{None}.
+
+    Be aware that L{returnValue} will not accept a L{Deferred} as a parameter.
+    If you believe the thing you'd like to return could be a L{Deferred}, do
+    this::
+
+        result = yield result
+        returnValue(result)
+
+    The L{Deferred} returned from your deferred generator may errback if your
+    generator raised an exception::
+
+        @inlineCallbacks
+        def thingummy():
+            thing = yield makeSomeRequestResultingInDeferred()
+            if thing == 'I love Twisted':
+                # will become the result of the Deferred
+                returnValue('TWISTED IS GREAT!')
+            else:
+                # will trigger an errback
+                raise Exception('DESTROY ALL LIFE')
+
+    It is possible to use the C{return} statement instead of L{returnValue}::
+
+        @inlineCallbacks
+        def loadData(url):
+            response = yield makeRequest(url)
+            return json.loads(response)
+
+    You can cancel the L{Deferred} returned from your L{inlineCallbacks}
+    generator before it is fired by your generator completing (either by
+    reaching its end, a C{return} statement, or by calling L{returnValue}).
+    A C{CancelledError} will be raised from the C{yield}ed L{Deferred} that
+    has been cancelled if that C{Deferred} does not otherwise suppress it.
+    """
+
+    @wraps(f)
+    def unwindGenerator(*args: _P.args, **kwargs: _P.kwargs) -> Deferred[_T]:
+        try:
+            gen = f(*args, **kwargs)
+        except _DefGen_Return:
+            raise TypeError(
+                "inlineCallbacks requires %r to produce a generator; instead"
+                "caught returnValue being used in a non-generator" % (f,)
+            )
+        if not isinstance(gen, GeneratorType):
+            raise TypeError(
+                "inlineCallbacks requires %r to produce a generator; "
+                "instead got %r" % (f, gen)
+            )
+        return _cancellableInlineCallbacks(gen)
+
+    return unwindGenerator
+
+
+## DeferredLock/DeferredQueue
+
+
+class _ConcurrencyPrimitive(ABC):
+    def __init__(self: Self) -> None:
+        self.waiting: List[Deferred[Self]] = []
+
+    def _releaseAndReturn(self, r: _T) -> _T:
+        self.release()
+        return r
+
+    @overload
+    def run(
+        self: Self,
+        /,
+        f: Callable[_P, Deferred[_T]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_T]:
+        ...
+
+    @overload
+    def run(
+        self: Self,
+        /,
+        f: Callable[_P, Coroutine[Deferred[Any], Any, _T]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_T]:
+        ...
+
+    @overload
+    def run(
+        self: Self, /, f: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
+    ) -> Deferred[_T]:
+        ...
+
+    def run(
+        self: Self,
+        /,
+        f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_T]:
+        """
+        Acquire, run, release.
+
+        This method takes a callable as its first argument and any
+        number of other positional and keyword arguments.  When the
+        lock or semaphore is acquired, the callable will be invoked
+        with those arguments.
+
+        The callable may return a L{Deferred}; if it does, the lock or
+        semaphore won't be released until that L{Deferred} fires.
+
+        @return: L{Deferred} of function result.
+        """
+
+        def execute(ignoredResult: object) -> Deferred[_T]:
+            # maybeDeferred arg type requires one of the possible union members
+            # and won't accept all possible union members
+            return maybeDeferred(f, *args, **kwargs).addBoth(
+                self._releaseAndReturn
+            )  # type: ignore[return-value]
+
+        return self.acquire().addCallback(execute)
+
+    def __aenter__(self: Self) -> Deferred[Self]:
+        """
+        We can be used as an asynchronous context manager.
+        """
+        return self.acquire()
+
+    def __aexit__(
+        self,
+        __exc_type: Optional[Type[BaseException]],
+        __exc_value: Optional[BaseException],
+        __traceback: Optional[TracebackType],
+    ) -> Deferred[Literal[False]]:
+        self.release()
+        # We return False to indicate that we have not consumed the
+        # exception, if any.
+        return succeed(False)
+
+    @abstractmethod
+    def acquire(self: Self) -> Deferred[Self]:
+        pass
+
+    @abstractmethod
+    def release(self) -> None:
+        pass
+
+
+class DeferredLock(_ConcurrencyPrimitive):
+    """
+    A lock for event driven systems.
+
+    @ivar locked: C{True} when this Lock has been acquired, false at all other
+        times.  Do not change this value, but it is useful to examine for the
+        equivalent of a "non-blocking" acquisition.
+    """
+
+    locked = False
+
+    def _cancelAcquire(self: Self, d: Deferred[Self]) -> None:
+        """
+        Remove a deferred d from our waiting list, as the deferred has been
+        canceled.
+
+        Note: We do not need to wrap this in a try/except to catch d not
+        being in self.waiting because this canceller will not be called if
+        d has fired. release() pops a deferred out of self.waiting and
+        calls it, so the canceller will no longer be called.
+
+        @param d: The deferred that has been canceled.
+        """
+        self.waiting.remove(d)
+
+    def acquire(self: Self) -> Deferred[Self]:
+        """
+        Attempt to acquire the lock.  Returns a L{Deferred} that fires on
+        lock acquisition with the L{DeferredLock} as the value.  If the lock
+        is locked, then the Deferred is placed at the end of a waiting list.
+
+        @return: a L{Deferred} which fires on lock acquisition.
+        @rtype: a L{Deferred}
+        """
+        d: Deferred[Self] = Deferred(canceller=self._cancelAcquire)
+        if self.locked:
+            self.waiting.append(d)
+        else:
+            self.locked = True
+            d.callback(self)
+        return d
+
+    def release(self: Self) -> None:
+        """
+        Release the lock.  If there is a waiting list, then the first
+        L{Deferred} in that waiting list will be called back.
+
+        Should be called by whomever did the L{acquire}() when the shared
+        resource is free.
+        """
+        assert self.locked, "Tried to release an unlocked lock"
+        self.locked = False
+        if self.waiting:
+            # someone is waiting to acquire lock
+            self.locked = True
+            d = self.waiting.pop(0)
+            d.callback(self)
+
+
+class DeferredSemaphore(_ConcurrencyPrimitive):
+    """
+    A semaphore for event driven systems.
+
+    If you are looking into this as a means of limiting parallelism, you might
+    find L{twisted.internet.task.Cooperator} more useful.
+
+    @ivar limit: At most this many users may acquire this semaphore at
+        once.
+    @ivar tokens: The difference between C{limit} and the number of users
+        which have currently acquired this semaphore.
+    """
+
+    def __init__(self, tokens: int) -> None:
+        """
+        @param tokens: initial value of L{tokens} and L{limit}
+        @type tokens: L{int}
+        """
+        _ConcurrencyPrimitive.__init__(self)
+        if tokens < 1:
+            raise ValueError("DeferredSemaphore requires tokens >= 1")
+        self.tokens = tokens
+        self.limit = tokens
+
+    def _cancelAcquire(self: Self, d: Deferred[Self]) -> None:
+        """
+        Remove a deferred d from our waiting list, as the deferred has been
+        canceled.
+
+        Note: We do not need to wrap this in a try/except to catch d not
+        being in self.waiting because this canceller will not be called if
+        d has fired. release() pops a deferred out of self.waiting and
+        calls it, so the canceller will no longer be called.
+
+        @param d: The deferred that has been canceled.
+        """
+        self.waiting.remove(d)
+
+    def acquire(self: Self) -> Deferred[Self]:
+        """
+        Attempt to acquire the token.
+
+        @return: a L{Deferred} which fires on token acquisition.
+        """
+        assert (
+            self.tokens >= 0
+        ), "Internal inconsistency??  tokens should never be negative"
+        d: Deferred[Self] = Deferred(canceller=self._cancelAcquire)
+        if not self.tokens:
+            self.waiting.append(d)
+        else:
+            self.tokens = self.tokens - 1
+            d.callback(self)
+        return d
+
+    def release(self: Self) -> None:
+        """
+        Release the token.
+
+        Should be called by whoever did the L{acquire}() when the shared
+        resource is free.
+        """
+        assert (
+            self.tokens < self.limit
+        ), "Someone released me too many times: too many tokens!"
+        self.tokens = self.tokens + 1
+        if self.waiting:
+            # someone is waiting to acquire token
+            self.tokens = self.tokens - 1
+            d = self.waiting.pop(0)
+            d.callback(self)
+
+
+class QueueOverflow(Exception):
+    pass
+
+
+class QueueUnderflow(Exception):
+    pass
+
+
+class DeferredQueue(Generic[_T]):
+    """
+    An event driven queue.
+
+    Objects may be added as usual to this queue.  When an attempt is
+    made to retrieve an object when the queue is empty, a L{Deferred} is
+    returned which will fire when an object becomes available.
+
+    @ivar size: The maximum number of objects to allow into the queue
+        at a time.  When an attempt to add a new object would exceed this
+        limit, L{QueueOverflow} is raised synchronously.  L{None} for no limit.
+    @ivar backlog: The maximum number of L{Deferred} gets to allow at
+        one time.  When an attempt is made to get an object which would
+        exceed this limit, L{QueueUnderflow} is raised synchronously.  L{None}
+        for no limit.
+    """
+
+    def __init__(
+        self, size: Optional[int] = None, backlog: Optional[int] = None
+    ) -> None:
+        self.waiting: List[Deferred[_T]] = []
+        self.pending: List[_T] = []
+        self.size = size
+        self.backlog = backlog
+
+    def _cancelGet(self, d: Deferred[_T]) -> None:
+        """
+        Remove a deferred d from our waiting list, as the deferred has been
+        canceled.
+
+        Note: We do not need to wrap this in a try/except to catch d not
+        being in self.waiting because this canceller will not be called if
+        d has fired. put() pops a deferred out of self.waiting and calls
+        it, so the canceller will no longer be called.
+
+        @param d: The deferred that has been canceled.
+        """
+        self.waiting.remove(d)
+
+    def put(self, obj: _T) -> None:
+        """
+        Add an object to this queue.
+
+        @raise QueueOverflow: Too many objects are in this queue.
+        """
+        if self.waiting:
+            self.waiting.pop(0).callback(obj)
+        elif self.size is None or len(self.pending) < self.size:
+            self.pending.append(obj)
+        else:
+            raise QueueOverflow()
+
+    def get(self) -> Deferred[_T]:
+        """
+        Attempt to retrieve and remove an object from the queue.
+
+        @return: a L{Deferred} which fires with the next object available in
+        the queue.
+
+        @raise QueueUnderflow: Too many (more than C{backlog})
+        L{Deferred}s are already waiting for an object from this queue.
+        """
+        if self.pending:
+            return succeed(self.pending.pop(0))
+        elif self.backlog is None or len(self.waiting) < self.backlog:
+            d: Deferred[_T] = Deferred(canceller=self._cancelGet)
+            self.waiting.append(d)
+            return d
+        else:
+            raise QueueUnderflow()
+
+
+class AlreadyTryingToLockError(Exception):
+    """
+    Raised when L{DeferredFilesystemLock.deferUntilLocked} is called twice on a
+    single L{DeferredFilesystemLock}.
+    """
+
+
+class DeferredFilesystemLock(lockfile.FilesystemLock):
+    """
+    A L{FilesystemLock} that allows for a L{Deferred} to be fired when the lock is
+    acquired.
+
+    @ivar _scheduler: The object in charge of scheduling retries. In this
+        implementation this is parameterized for testing.
+    @ivar _interval: The retry interval for an L{IReactorTime} based scheduler.
+    @ivar _tryLockCall: An L{IDelayedCall} based on C{_interval} that will manage
+        the next retry for acquiring the lock.
+    @ivar _timeoutCall: An L{IDelayedCall} based on C{deferUntilLocked}'s timeout
+        argument.  This is in charge of timing out our attempt to acquire the
+        lock.
+    """
+
+    _interval = 1
+    _tryLockCall: Optional[IDelayedCall] = None
+    _timeoutCall: Optional[IDelayedCall] = None
+
+    def __init__(self, name: str, scheduler: Optional[IReactorTime] = None) -> None:
+        """
+        @param name: The name of the lock to acquire
+        @param scheduler: An object which provides L{IReactorTime}
+        """
+        lockfile.FilesystemLock.__init__(self, name)
+
+        if scheduler is None:
+            from twisted.internet import reactor
+
+            scheduler = cast(IReactorTime, reactor)
+
+        self._scheduler = scheduler
+
+    def deferUntilLocked(self, timeout: Optional[float] = None) -> Deferred[None]:
+        """
+        Wait until we acquire this lock.  This method is not safe for
+        concurrent use.
+
+        @param timeout: the number of seconds after which to time out if the
+            lock has not been acquired.
+
+        @return: a L{Deferred} which will callback when the lock is acquired, or
+            errback with a L{TimeoutError} after timing out or an
+            L{AlreadyTryingToLockError} if the L{deferUntilLocked} has already
+            been called and not successfully locked the file.
+        """
+        if self._tryLockCall is not None:
+            return fail(
+                AlreadyTryingToLockError(
+                    "deferUntilLocked isn't safe for concurrent use."
+                )
+            )
+
+        def _cancelLock(reason: Union[Failure, Exception]) -> None:
+            """
+            Cancel a L{DeferredFilesystemLock.deferUntilLocked} call.
+
+            @type reason: L{Failure}
+            @param reason: The reason why the call is cancelled.
+            """
+            assert self._tryLockCall is not None
+            self._tryLockCall.cancel()
+            self._tryLockCall = None
+            if self._timeoutCall is not None and self._timeoutCall.active():
+                self._timeoutCall.cancel()
+                self._timeoutCall = None
+
+            if self.lock():
+                d.callback(None)
+            else:
+                d.errback(reason)
+
+        d: Deferred[None] = Deferred(lambda deferred: _cancelLock(CancelledError()))
+
+        def _tryLock() -> None:
+            if self.lock():
+                if self._timeoutCall is not None:
+                    self._timeoutCall.cancel()
+                    self._timeoutCall = None
+
+                self._tryLockCall = None
+
+                d.callback(None)
+            else:
+                if timeout is not None and self._timeoutCall is None:
+                    reason = Failure(
+                        TimeoutError(
+                            "Timed out acquiring lock: %s after %fs"
+                            % (self.name, timeout)
+                        )
+                    )
+                    self._timeoutCall = self._scheduler.callLater(
+                        timeout, _cancelLock, reason
+                    )
+
+                self._tryLockCall = self._scheduler.callLater(self._interval, _tryLock)
+
+        _tryLock()
+
+        return d
+
+
+__all__ = [
+    "Deferred",
+    "DeferredList",
+    "succeed",
+    "fail",
+    "FAILURE",
+    "SUCCESS",
+    "AlreadyCalledError",
+    "TimeoutError",
+    "gatherResults",
+    "maybeDeferred",
+    "ensureDeferred",
+    "waitForDeferred",
+    "deferredGenerator",
+    "inlineCallbacks",
+    "returnValue",
+    "DeferredLock",
+    "DeferredSemaphore",
+    "DeferredQueue",
+    "DeferredFilesystemLock",
+    "AlreadyTryingToLockError",
+    "CancelledError",
+]
diff --git a/contrib/python/Twisted/py3/twisted/internet/endpoints.py b/contrib/python/Twisted/py3/twisted/internet/endpoints.py
new file mode 100644
index 00000000000..4a4cf55e8e9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/endpoints.py
@@ -0,0 +1,2338 @@
+# -*- test-case-name: twisted.internet.test.test_endpoints -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementations of L{IStreamServerEndpoint} and L{IStreamClientEndpoint} that
+wrap the L{IReactorTCP}, L{IReactorSSL}, and L{IReactorUNIX} interfaces.
+
+This also implements an extensible mini-language for describing endpoints,
+parsed by the L{clientFromString} and L{serverFromString} functions.
+
+@since: 10.1
+"""
+
+
+import os
+import re
+import socket
+import warnings
+from typing import Optional, Sequence, Type
+from unicodedata import normalize
+
+from zope.interface import directlyProvides, implementer, provider
+
+from constantly import NamedConstant, Names  # type: ignore[import]
+from incremental import Version
+
+from twisted.internet import defer, error, fdesc, interfaces, threads
+from twisted.internet.abstract import isIPAddress, isIPv6Address
+from twisted.internet.address import (
+    HostnameAddress,
+    IPv4Address,
+    IPv6Address,
+    _ProcessAddress,
+)
+from twisted.internet.interfaces import (
+    IAddress,
+    IHostnameResolver,
+    IHostResolution,
+    IReactorPluggableNameResolver,
+    IReactorSocket,
+    IResolutionReceiver,
+    IStreamClientEndpointStringParserWithReactor,
+    IStreamServerEndpointStringParser,
+)
+from twisted.internet.protocol import ClientFactory, Factory, ProcessProtocol, Protocol
+
+try:
+    from twisted.internet.stdio import PipeAddress, StandardIO
+except ImportError:
+    # fallback if pywin32 is not installed
+    StandardIO = None  # type: ignore[assignment,misc]
+    PipeAddress = None  # type: ignore[assignment,misc]
+
+from twisted.internet._resolver import HostResolution
+from twisted.internet.defer import Deferred
+from twisted.internet.task import LoopingCall
+from twisted.logger import Logger
+from twisted.plugin import IPlugin, getPlugins
+from twisted.python import deprecate, log
+from twisted.python.compat import _matchingString, iterbytes, nativeString
+from twisted.python.components import proxyForInterface
+from twisted.python.failure import Failure
+from twisted.python.filepath import FilePath
+from twisted.python.systemd import ListenFDs
+from ._idna import _idnaBytes, _idnaText
+
+try:
+    from OpenSSL.SSL import Error as SSLError
+
+    from twisted.internet.ssl import (
+        Certificate,
+        CertificateOptions,
+        KeyPair,
+        PrivateCertificate,
+        optionsForClientTLS,
+        trustRootFromCertificates,
+    )
+    from twisted.protocols.tls import TLSMemoryBIOFactory as _TLSMemoryBIOFactory
+except ImportError:
+    TLSMemoryBIOFactory = None
+else:
+    TLSMemoryBIOFactory = _TLSMemoryBIOFactory
+
+__all__ = [
+    "clientFromString",
+    "serverFromString",
+    "TCP4ServerEndpoint",
+    "TCP6ServerEndpoint",
+    "TCP4ClientEndpoint",
+    "TCP6ClientEndpoint",
+    "UNIXServerEndpoint",
+    "UNIXClientEndpoint",
+    "SSL4ServerEndpoint",
+    "SSL4ClientEndpoint",
+    "AdoptedStreamServerEndpoint",
+    "StandardIOEndpoint",
+    "ProcessEndpoint",
+    "HostnameEndpoint",
+    "StandardErrorBehavior",
+    "connectProtocol",
+    "wrapClientTLS",
+]
+
+
+class _WrappingProtocol(Protocol):
+    """
+    Wrap another protocol in order to notify my user when a connection has
+    been made.
+    """
+
+    def __init__(self, connectedDeferred, wrappedProtocol):
+        """
+        @param connectedDeferred: The L{Deferred} that will callback
+            with the C{wrappedProtocol} when it is connected.
+
+        @param wrappedProtocol: An L{IProtocol} provider that will be
+            connected.
+        """
+        self._connectedDeferred = connectedDeferred
+        self._wrappedProtocol = wrappedProtocol
+
+        for iface in [
+            interfaces.IHalfCloseableProtocol,
+            interfaces.IFileDescriptorReceiver,
+            interfaces.IHandshakeListener,
+        ]:
+            if iface.providedBy(self._wrappedProtocol):
+                directlyProvides(self, iface)
+
+    def logPrefix(self):
+        """
+        Transparently pass through the wrapped protocol's log prefix.
+        """
+        if interfaces.ILoggingContext.providedBy(self._wrappedProtocol):
+            return self._wrappedProtocol.logPrefix()
+        return self._wrappedProtocol.__class__.__name__
+
+    def connectionMade(self):
+        """
+        Connect the C{self._wrappedProtocol} to our C{self.transport} and
+        callback C{self._connectedDeferred} with the C{self._wrappedProtocol}
+        """
+        self._wrappedProtocol.makeConnection(self.transport)
+        self._connectedDeferred.callback(self._wrappedProtocol)
+
+    def dataReceived(self, data):
+        """
+        Proxy C{dataReceived} calls to our C{self._wrappedProtocol}
+        """
+        return self._wrappedProtocol.dataReceived(data)
+
+    def fileDescriptorReceived(self, descriptor):
+        """
+        Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol}
+        """
+        return self._wrappedProtocol.fileDescriptorReceived(descriptor)
+
+    def connectionLost(self, reason):
+        """
+        Proxy C{connectionLost} calls to our C{self._wrappedProtocol}
+        """
+        return self._wrappedProtocol.connectionLost(reason)
+
+    def readConnectionLost(self):
+        """
+        Proxy L{IHalfCloseableProtocol.readConnectionLost} to our
+        C{self._wrappedProtocol}
+        """
+        self._wrappedProtocol.readConnectionLost()
+
+    def writeConnectionLost(self):
+        """
+        Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our
+        C{self._wrappedProtocol}
+        """
+        self._wrappedProtocol.writeConnectionLost()
+
+    def handshakeCompleted(self):
+        """
+        Proxy L{interfaces.IHandshakeListener} to our
+        C{self._wrappedProtocol}.
+        """
+        self._wrappedProtocol.handshakeCompleted()
+
+
+class _WrappingFactory(ClientFactory):
+    """
+    Wrap a factory in order to wrap the protocols it builds.
+
+    @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol
+        method will be called and whose resulting protocol will be wrapped.
+
+    @ivar _onConnection: A L{Deferred} that fires when the protocol is
+        connected
+
+    @ivar _connector: A L{connector <twisted.internet.interfaces.IConnector>}
+        that is managing the current or previous connection attempt.
+    """
+
+    # Type is wrong.  See https://twistedmatrix.com/trac/ticket/10005#ticket
+    protocol = _WrappingProtocol  # type: ignore[assignment]
+
+    def __init__(self, wrappedFactory):
+        """
+        @param wrappedFactory: A provider of I{IProtocolFactory} whose
+            buildProtocol method will be called and whose resulting protocol
+            will be wrapped.
+        """
+        self._wrappedFactory = wrappedFactory
+        self._onConnection = defer.Deferred(canceller=self._canceller)
+
+    def startedConnecting(self, connector):
+        """
+        A connection attempt was started.  Remember the connector which started
+        said attempt, for use later.
+        """
+        self._connector = connector
+
+    def _canceller(self, deferred):
+        """
+        The outgoing connection attempt was cancelled.  Fail that L{Deferred}
+        with an L{error.ConnectingCancelledError}.
+
+        @param deferred: The L{Deferred <defer.Deferred>} that was cancelled;
+            should be the same as C{self._onConnection}.
+        @type deferred: L{Deferred <defer.Deferred>}
+
+        @note: This relies on startedConnecting having been called, so it may
+            seem as though there's a race condition where C{_connector} may not
+            have been set.  However, using public APIs, this condition is
+            impossible to catch, because a connection API
+            (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a
+            L{_WrappingFactory}'s L{Deferred <defer.Deferred>} is returned to
+            C{connect()}'s caller.
+
+        @return: L{None}
+        """
+        deferred.errback(
+            error.ConnectingCancelledError(self._connector.getDestination())
+        )
+        self._connector.stopConnecting()
+
+    def doStart(self):
+        """
+        Start notifications are passed straight through to the wrapped factory.
+        """
+        self._wrappedFactory.doStart()
+
+    def doStop(self):
+        """
+        Stop notifications are passed straight through to the wrapped factory.
+        """
+        self._wrappedFactory.doStop()
+
+    def buildProtocol(self, addr):
+        """
+        Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback the
+        C{self._onConnection} L{Deferred} if the wrapped factory raises an
+        exception or returns L{None}.
+
+        @return: An instance of L{_WrappingProtocol} or L{None}
+        """
+        try:
+            proto = self._wrappedFactory.buildProtocol(addr)
+            if proto is None:
+                raise error.NoProtocol()
+        except BaseException:
+            self._onConnection.errback()
+        else:
+            return self.protocol(self._onConnection, proto)
+
+    def clientConnectionFailed(self, connector, reason):
+        """
+        Errback the C{self._onConnection} L{Deferred} when the
+        client connection fails.
+        """
+        if not self._onConnection.called:
+            self._onConnection.errback(reason)
+
+
+@implementer(interfaces.IStreamServerEndpoint)
+class StandardIOEndpoint:
+    """
+    A Standard Input/Output endpoint
+
+    @ivar _stdio: a callable, like L{stdio.StandardIO}, which takes an
+        L{IProtocol} provider and a C{reactor} keyword argument (interface
+        dependent upon your platform).
+    """
+
+    _stdio = StandardIO
+
+    def __init__(self, reactor):
+        """
+        @param reactor: The reactor for the endpoint.
+        """
+        self._reactor = reactor
+
+    def listen(self, stdioProtocolFactory):
+        """
+        Implement L{IStreamServerEndpoint.listen} to listen on stdin/stdout
+        """
+        return defer.execute(
+            self._stdio,
+            stdioProtocolFactory.buildProtocol(PipeAddress()),
+            reactor=self._reactor,
+        )
+
+
+class _IProcessTransportWithConsumerAndProducer(
+    interfaces.IProcessTransport, interfaces.IConsumer, interfaces.IPushProducer
+):
+    """
+    An L{_IProcessTransportWithConsumerAndProducer} combines various interfaces
+    to work around the issue that L{interfaces.IProcessTransport} is
+    incompletely defined and doesn't specify flow-control interfaces, and that
+    L{proxyForInterface} doesn't allow for multiple interfaces.
+    """
+
+
+class _ProcessEndpointTransport(
+    proxyForInterface(  # type: ignore[misc]
+        _IProcessTransportWithConsumerAndProducer,
+        "_process",
+    )
+):
+    """
+    An L{ITransport}, L{IProcessTransport}, L{IConsumer}, and L{IPushProducer}
+    provider for the L{IProtocol} instance passed to the process endpoint.
+
+    @ivar _process: An active process transport which will be used by write
+        methods on this object to write data to a child process.
+    @type _process: L{interfaces.IProcessTransport} provider
+    """
+
+
+class _WrapIProtocol(ProcessProtocol):
+    """
+    An L{IProcessProtocol} provider that wraps an L{IProtocol}.
+
+    @ivar transport: A L{_ProcessEndpointTransport} provider that is hooked to
+        the wrapped L{IProtocol} provider.
+
+    @see: L{protocol.ProcessProtocol}
+    """
+
+    def __init__(self, proto, executable, errFlag):
+        """
+        @param proto: An L{IProtocol} provider.
+        @param errFlag: A constant belonging to L{StandardErrorBehavior}
+            that determines if stderr is logged or dropped.
+        @param executable: The file name (full path) to spawn.
+        """
+        self.protocol = proto
+        self.errFlag = errFlag
+        self.executable = executable
+
+    def makeConnection(self, process):
+        """
+        Call L{IProtocol} provider's makeConnection method with an
+        L{ITransport} provider.
+
+        @param process: An L{IProcessTransport} provider.
+        """
+        self.transport = _ProcessEndpointTransport(process)
+        return self.protocol.makeConnection(self.transport)
+
+    def childDataReceived(self, childFD, data):
+        """
+        This is called with data from the process's stdout or stderr pipes. It
+        checks the status of the errFlag to setermine if stderr should be
+        logged (default) or dropped.
+        """
+        if childFD == 1:
+            return self.protocol.dataReceived(data)
+        elif childFD == 2 and self.errFlag == StandardErrorBehavior.LOG:
+            log.msg(
+                format="Process %(executable)r wrote stderr unhandled by "
+                "%(protocol)s: %(data)s",
+                executable=self.executable,
+                protocol=self.protocol,
+                data=data,
+            )
+
+    def processEnded(self, reason):
+        """
+        If the process ends with L{error.ProcessDone}, this method calls the
+        L{IProtocol} provider's L{connectionLost} with a
+        L{error.ConnectionDone}
+
+        @see: L{ProcessProtocol.processEnded}
+        """
+        if (reason.check(error.ProcessDone) == error.ProcessDone) and (
+            reason.value.status == 0
+        ):
+            return self.protocol.connectionLost(Failure(error.ConnectionDone()))
+        else:
+            return self.protocol.connectionLost(reason)
+
+
+class StandardErrorBehavior(Names):
+    """
+    Constants used in ProcessEndpoint to decide what to do with stderr.
+
+    @cvar LOG: Indicates that stderr is to be logged.
+    @cvar DROP: Indicates that stderr is to be dropped (and not logged).
+
+    @since: 13.1
+    """
+
+    LOG = NamedConstant()
+    DROP = NamedConstant()
+
+
+@implementer(interfaces.IStreamClientEndpoint)
+class ProcessEndpoint:
+    """
+    An endpoint for child processes
+
+    @ivar _spawnProcess: A hook used for testing the spawning of child process.
+
+    @since: 13.1
+    """
+
+    def __init__(
+        self,
+        reactor,
+        executable,
+        args=(),
+        env={},
+        path=None,
+        uid=None,
+        gid=None,
+        usePTY=0,
+        childFDs=None,
+        errFlag=StandardErrorBehavior.LOG,
+    ):
+        """
+        See L{IReactorProcess.spawnProcess}.
+
+        @param errFlag: Determines if stderr should be logged.
+        @type errFlag: L{endpoints.StandardErrorBehavior}
+        """
+        self._reactor = reactor
+        self._executable = executable
+        self._args = args
+        self._env = env
+        self._path = path
+        self._uid = uid
+        self._gid = gid
+        self._usePTY = usePTY
+        self._childFDs = childFDs
+        self._errFlag = errFlag
+        self._spawnProcess = self._reactor.spawnProcess
+
+    def connect(self, protocolFactory):
+        """
+        Implement L{IStreamClientEndpoint.connect} to launch a child process
+        and connect it to a protocol created by C{protocolFactory}.
+
+        @param protocolFactory: A factory for an L{IProtocol} provider which
+            will be notified of all events related to the created process.
+        """
+        proto = protocolFactory.buildProtocol(_ProcessAddress())
+        try:
+            self._spawnProcess(
+                _WrapIProtocol(proto, self._executable, self._errFlag),
+                self._executable,
+                self._args,
+                self._env,
+                self._path,
+                self._uid,
+                self._gid,
+                self._usePTY,
+                self._childFDs,
+            )
+        except BaseException:
+            return defer.fail()
+        else:
+            return defer.succeed(proto)
+
+
+@implementer(interfaces.IStreamServerEndpoint)
+class _TCPServerEndpoint:
+    """
+    A TCP server endpoint interface
+    """
+
+    def __init__(self, reactor, port, backlog, interface):
+        """
+        @param reactor: An L{IReactorTCP} provider.
+
+        @param port: The port number used for listening
+        @type port: int
+
+        @param backlog: Size of the listen queue
+        @type backlog: int
+
+        @param interface: The hostname to bind to
+        @type interface: str
+        """
+        self._reactor = reactor
+        self._port = port
+        self._backlog = backlog
+        self._interface = interface
+
+    def listen(self, protocolFactory):
+        """
+        Implement L{IStreamServerEndpoint.listen} to listen on a TCP
+        socket
+        """
+        return defer.execute(
+            self._reactor.listenTCP,
+            self._port,
+            protocolFactory,
+            backlog=self._backlog,
+            interface=self._interface,
+        )
+
+
+class TCP4ServerEndpoint(_TCPServerEndpoint):
+    """
+    Implements TCP server endpoint with an IPv4 configuration
+    """
+
+    def __init__(self, reactor, port, backlog=50, interface=""):
+        """
+        @param reactor: An L{IReactorTCP} provider.
+
+        @param port: The port number used for listening
+        @type port: int
+
+        @param backlog: Size of the listen queue
+        @type backlog: int
+
+        @param interface: The hostname to bind to, defaults to '' (all)
+        @type interface: str
+        """
+        _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface)
+
+
+class TCP6ServerEndpoint(_TCPServerEndpoint):
+    """
+    Implements TCP server endpoint with an IPv6 configuration
+    """
+
+    def __init__(self, reactor, port, backlog=50, interface="::"):
+        """
+        @param reactor: An L{IReactorTCP} provider.
+
+        @param port: The port number used for listening
+        @type port: int
+
+        @param backlog: Size of the listen queue
+        @type backlog: int
+
+        @param interface: The hostname to bind to, defaults to C{::} (all)
+        @type interface: str
+        """
+        _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface)
+
+
+@implementer(interfaces.IStreamClientEndpoint)
+class TCP4ClientEndpoint:
+    """
+    TCP client endpoint with an IPv4 configuration.
+    """
+
+    def __init__(self, reactor, host, port, timeout=30, bindAddress=None):
+        """
+        @param reactor: An L{IReactorTCP} provider
+
+        @param host: A hostname, used when connecting
+        @type host: str
+
+        @param port: The port number, used when connecting
+        @type port: int
+
+        @param timeout: The number of seconds to wait before assuming the
+            connection has failed.
+        @type timeout: L{float} or L{int}
+
+        @param bindAddress: A (host, port) tuple of local address to bind to,
+            or None.
+        @type bindAddress: tuple
+        """
+        self._reactor = reactor
+        self._host = host
+        self._port = port
+        self._timeout = timeout
+        self._bindAddress = bindAddress
+
+    def connect(self, protocolFactory):
+        """
+        Implement L{IStreamClientEndpoint.connect} to connect via TCP.
+        """
+        try:
+            wf = _WrappingFactory(protocolFactory)
+            self._reactor.connectTCP(
+                self._host,
+                self._port,
+                wf,
+                timeout=self._timeout,
+                bindAddress=self._bindAddress,
+            )
+            return wf._onConnection
+        except BaseException:
+            return defer.fail()
+
+
+@implementer(interfaces.IStreamClientEndpoint)
+class TCP6ClientEndpoint:
+    """
+    TCP client endpoint with an IPv6 configuration.
+
+    @ivar _getaddrinfo: A hook used for testing name resolution.
+
+    @ivar _deferToThread: A hook used for testing deferToThread.
+
+    @ivar _GAI_ADDRESS: Index of the address portion in result of
+        getaddrinfo to be used.
+
+    @ivar _GAI_ADDRESS_HOST: Index of the actual host-address in the
+        5-tuple L{_GAI_ADDRESS}.
+    """
+
+    _getaddrinfo = staticmethod(socket.getaddrinfo)
+    _deferToThread = staticmethod(threads.deferToThread)
+    _GAI_ADDRESS = 4
+    _GAI_ADDRESS_HOST = 0
+
+    def __init__(self, reactor, host, port, timeout=30, bindAddress=None):
+        """
+        @param host: An IPv6 address literal or a hostname with an
+            IPv6 address
+
+        @see: L{twisted.internet.interfaces.IReactorTCP.connectTCP}
+        """
+        self._reactor = reactor
+        self._host = host
+        self._port = port
+        self._timeout = timeout
+        self._bindAddress = bindAddress
+
+    def connect(self, protocolFactory):
+        """
+        Implement L{IStreamClientEndpoint.connect} to connect via TCP,
+        once the hostname resolution is done.
+        """
+        if isIPv6Address(self._host):
+            d = self._resolvedHostConnect(self._host, protocolFactory)
+        else:
+            d = self._nameResolution(self._host)
+            d.addCallback(
+                lambda result: result[0][self._GAI_ADDRESS][self._GAI_ADDRESS_HOST]
+            )
+            d.addCallback(self._resolvedHostConnect, protocolFactory)
+        return d
+
+    def _nameResolution(self, host):
+        """
+        Resolve the hostname string into a tuple containing the host
+        IPv6 address.
+        """
+        return self._deferToThread(self._getaddrinfo, host, 0, socket.AF_INET6)
+
+    def _resolvedHostConnect(self, resolvedHost, protocolFactory):
+        """
+        Connect to the server using the resolved hostname.
+        """
+        try:
+            wf = _WrappingFactory(protocolFactory)
+            self._reactor.connectTCP(
+                resolvedHost,
+                self._port,
+                wf,
+                timeout=self._timeout,
+                bindAddress=self._bindAddress,
+            )
+            return wf._onConnection
+        except BaseException:
+            return defer.fail()
+
+
+@implementer(IHostnameResolver)
+class _SimpleHostnameResolver:
+    """
+    An L{IHostnameResolver} provider that invokes a provided callable
+    to resolve hostnames.
+
+    @ivar _nameResolution: the callable L{resolveHostName} invokes to
+        resolve hostnames.
+    @type _nameResolution: A L{callable} that accepts two arguments:
+        the host to resolve and the port number to include in the
+        result.
+    """
+
+    _log = Logger()
+
+    def __init__(self, nameResolution):
+        """
+        Create a L{_SimpleHostnameResolver} instance.
+        """
+        self._nameResolution = nameResolution
+
+    def resolveHostName(
+        self,
+        resolutionReceiver: IResolutionReceiver,
+        hostName: str,
+        portNumber: int = 0,
+        addressTypes: Optional[Sequence[Type[IAddress]]] = None,
+        transportSemantics: str = "TCP",
+    ) -> IHostResolution:
+        """
+        Initiate a hostname resolution.
+
+        @param resolutionReceiver: an object that will receive each resolved
+            address as it arrives.
+        @type resolutionReceiver: L{IResolutionReceiver}
+
+        @param hostName: see interface
+
+        @param portNumber: see interface
+
+        @param addressTypes: Ignored in this implementation.
+
+        @param transportSemantics: Ignored in this implementation.
+
+        @return: The resolution in progress.
+        @rtype: L{IResolutionReceiver}
+        """
+        resolution = HostResolution(hostName)
+        resolutionReceiver.resolutionBegan(resolution)
+        d = self._nameResolution(hostName, portNumber)
+
+        def cbDeliver(gairesult):
+            for family, socktype, proto, canonname, sockaddr in gairesult:
+                if family == socket.AF_INET6:
+                    resolutionReceiver.addressResolved(IPv6Address("TCP", *sockaddr))
+                elif family == socket.AF_INET:
+                    resolutionReceiver.addressResolved(IPv4Address("TCP", *sockaddr))
+
+        def ebLog(error):
+            self._log.failure(
+                "while looking up {name} with {callable}",
+                error,
+                name=hostName,
+                callable=self._nameResolution,
+            )
+
+        d.addCallback(cbDeliver)
+        d.addErrback(ebLog)
+        d.addBoth(lambda ignored: resolutionReceiver.resolutionComplete())
+        return resolution
+
+
+@implementer(interfaces.IStreamClientEndpoint)
+class HostnameEndpoint:
+    """
+    A name-based endpoint that connects to the fastest amongst the resolved
+    host addresses.
+
+    @cvar _DEFAULT_ATTEMPT_DELAY: The default time to use between attempts, in
+        seconds, when no C{attemptDelay} is given to
+        L{HostnameEndpoint.__init__}.
+
+    @ivar _hostText: the textual representation of the hostname passed to the
+        constructor.  Used to pass to the reactor's hostname resolver.
+    @type _hostText: L{unicode}
+
+    @ivar _hostBytes: the encoded bytes-representation of the hostname passed
+        to the constructor.  Used to construct the L{HostnameAddress}
+        associated with this endpoint.
+    @type _hostBytes: L{bytes}
+
+    @ivar _hostStr: the native-string representation of the hostname passed to
+        the constructor, used for exception construction
+    @type _hostStr: native L{str}
+
+    @ivar _badHostname: a flag - hopefully false!  - indicating that an invalid
+        hostname was passed to the constructor.  This might be a textual
+        hostname that isn't valid IDNA, or non-ASCII bytes.
+    @type _badHostname: L{bool}
+    """
+
+    _getaddrinfo = staticmethod(socket.getaddrinfo)
+    _deferToThread = staticmethod(threads.deferToThread)
+    _DEFAULT_ATTEMPT_DELAY = 0.3
+
+    def __init__(
+        self, reactor, host, port, timeout=30, bindAddress=None, attemptDelay=None
+    ):
+        """
+        Create a L{HostnameEndpoint}.
+
+        @param reactor: The reactor to use for connections and delayed calls.
+        @type reactor: provider of L{IReactorTCP}, L{IReactorTime} and either
+            L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}.
+
+        @param host: A hostname to connect to.
+        @type host: L{bytes} or L{unicode}
+
+        @param port: The port number to connect to.
+        @type port: L{int}
+
+        @param timeout: For each individual connection attempt, the number of
+            seconds to wait before assuming the connection has failed.
+        @type timeout: L{float} or L{int}
+
+        @param bindAddress: the local address of the network interface to make
+            the connections from.
+        @type bindAddress: L{bytes}
+
+        @param attemptDelay: The number of seconds to delay between connection
+            attempts.
+        @type attemptDelay: L{float}
+
+        @see: L{twisted.internet.interfaces.IReactorTCP.connectTCP}
+        """
+
+        self._reactor = reactor
+        self._nameResolver = self._getNameResolverAndMaybeWarn(reactor)
+        [self._badHostname, self._hostBytes, self._hostText] = self._hostAsBytesAndText(
+            host
+        )
+        self._hostStr = self._hostBytes if bytes is str else self._hostText
+        self._port = port
+        self._timeout = timeout
+        self._bindAddress = bindAddress
+        if attemptDelay is None:
+            attemptDelay = self._DEFAULT_ATTEMPT_DELAY
+        self._attemptDelay = attemptDelay
+
+    def __repr__(self) -> str:
+        """
+        Produce a string representation of the L{HostnameEndpoint}.
+
+        @return: A L{str}
+        """
+        if self._badHostname:
+            # Use the backslash-encoded version of the string passed to the
+            # constructor, which is already a native string.
+            host = self._hostStr
+        elif isIPv6Address(self._hostStr):
+            host = f"[{self._hostStr}]"
+        else:
+            # Convert the bytes representation to a native string to ensure
+            # that we display the punycoded version of the hostname, which is
+            # more useful than any IDN version as it can be easily copy-pasted
+            # into debugging tools.
+            host = nativeString(self._hostBytes)
+        return "".join(["<HostnameEndpoint ", host, ":", str(self._port), ">"])
+
+    def _getNameResolverAndMaybeWarn(self, reactor):
+        """
+        Retrieve a C{nameResolver} callable and warn the caller's
+        caller that using a reactor which doesn't provide
+        L{IReactorPluggableNameResolver} is deprecated.
+
+        @param reactor: The reactor to check.
+
+        @return: A L{IHostnameResolver} provider.
+        """
+        if not IReactorPluggableNameResolver.providedBy(reactor):
+            warningString = deprecate.getDeprecationWarningString(
+                reactor.__class__,
+                Version("Twisted", 17, 5, 0),
+                format=(
+                    "Passing HostnameEndpoint a reactor that does not"
+                    " provide IReactorPluggableNameResolver (%(fqpn)s)"
+                    " was deprecated in %(version)s"
+                ),
+                replacement=(
+                    "a reactor that provides" " IReactorPluggableNameResolver"
+                ),
+            )
+            warnings.warn(warningString, DeprecationWarning, stacklevel=3)
+            return _SimpleHostnameResolver(self._fallbackNameResolution)
+        return reactor.nameResolver
+
+    @staticmethod
+    def _hostAsBytesAndText(host):
+        """
+        For various reasons (documented in the C{@ivar}'s in the class
+        docstring) we need both a textual and a binary representation of the
+        hostname given to the constructor.  For compatibility and convenience,
+        we accept both textual and binary representations of the hostname, save
+        the form that was passed, and convert into the other form.  This is
+        mostly just because L{HostnameAddress} chose somewhat poorly to define
+        its attribute as bytes; hopefully we can find a compatible way to clean
+        this up in the future and just operate in terms of text internally.
+
+        @param host: A hostname to convert.
+        @type host: L{bytes} or C{str}
+
+        @return: a 3-tuple of C{(invalid, bytes, text)} where C{invalid} is a
+            boolean indicating the validity of the hostname, C{bytes} is a
+            binary representation of C{host}, and C{text} is a textual
+            representation of C{host}.
+        """
+        if isinstance(host, bytes):
+            if isIPAddress(host) or isIPv6Address(host):
+                return False, host, host.decode("ascii")
+            else:
+                try:
+                    return False, host, _idnaText(host)
+                except UnicodeError:
+                    # Convert the host to _some_ kind of text, to handle below.
+                    host = host.decode("charmap")
+        else:
+            host = normalize("NFC", host)
+            if isIPAddress(host) or isIPv6Address(host):
+                return False, host.encode("ascii"), host
+            else:
+                try:
+                    return False, _idnaBytes(host), host
+                except UnicodeError:
+                    pass
+        # `host` has been converted to text by this point either way; it's
+        # invalid as a hostname, and so may contain unprintable characters and
+        # such. escape it with backslashes so the user can get _some_ guess as
+        # to what went wrong.
+        asciibytes = host.encode("ascii", "backslashreplace")
+        return True, asciibytes, asciibytes.decode("ascii")
+
+    def connect(self, protocolFactory):
+        """
+        Attempts a connection to each resolved address, and returns a
+        connection which is established first.
+
+        @param protocolFactory: The protocol factory whose protocol
+            will be connected.
+        @type protocolFactory:
+            L{IProtocolFactory<twisted.internet.interfaces.IProtocolFactory>}
+
+        @return: A L{Deferred} that fires with the connected protocol
+            or fails a connection-related error.
+        """
+        if self._badHostname:
+            return defer.fail(ValueError(f"invalid hostname: {self._hostStr}"))
+
+        d = Deferred()
+        addresses = []
+
+        @provider(IResolutionReceiver)
+        class EndpointReceiver:
+            @staticmethod
+            def resolutionBegan(resolutionInProgress):
+                pass
+
+            @staticmethod
+            def addressResolved(address):
+                addresses.append(address)
+
+            @staticmethod
+            def resolutionComplete():
+                d.callback(addresses)
+
+        self._nameResolver.resolveHostName(
+            EndpointReceiver, self._hostText, portNumber=self._port
+        )
+
+        d.addErrback(
+            lambda ignored: defer.fail(
+                error.DNSLookupError(f"Couldn't find the hostname '{self._hostStr}'")
+            )
+        )
+
+        @d.addCallback
+        def resolvedAddressesToEndpoints(addresses):
+            # Yield an endpoint for every address resolved from the name.
+            for eachAddress in addresses:
+                if isinstance(eachAddress, IPv6Address):
+                    yield TCP6ClientEndpoint(
+                        self._reactor,
+                        eachAddress.host,
+                        eachAddress.port,
+                        self._timeout,
+                        self._bindAddress,
+                    )
+                if isinstance(eachAddress, IPv4Address):
+                    yield TCP4ClientEndpoint(
+                        self._reactor,
+                        eachAddress.host,
+                        eachAddress.port,
+                        self._timeout,
+                        self._bindAddress,
+                    )
+
+        d.addCallback(list)
+
+        def _canceller(d):
+            # This canceller must remain defined outside of
+            # `startConnectionAttempts`, because Deferred should not
+            # participate in cycles with their cancellers; that would create a
+            # potentially problematic circular reference and possibly
+            # gc.garbage.
+            d.errback(
+                error.ConnectingCancelledError(
+                    HostnameAddress(self._hostBytes, self._port)
+                )
+            )
+
+        @d.addCallback
+        def startConnectionAttempts(endpoints):
+            """
+            Given a sequence of endpoints obtained via name resolution, start
+            connecting to a new one every C{self._attemptDelay} seconds until
+            one of the connections succeeds, all of them fail, or the attempt
+            is cancelled.
+
+            @param endpoints: a list of all the endpoints we might try to
+                connect to, as determined by name resolution.
+            @type endpoints: L{list} of L{IStreamServerEndpoint}
+
+            @return: a Deferred that fires with the result of the
+                C{endpoint.connect} method that completes the fastest, or fails
+                with the first connection error it encountered if none of them
+                succeed.
+            @rtype: L{Deferred} failing with L{error.ConnectingCancelledError}
+                or firing with L{IProtocol}
+            """
+            if not endpoints:
+                raise error.DNSLookupError(
+                    f"no results for hostname lookup: {self._hostStr}"
+                )
+            iterEndpoints = iter(endpoints)
+            pending = []
+            failures = []
+            winner = defer.Deferred(canceller=_canceller)
+
+            def checkDone():
+                if pending or checkDone.completed or checkDone.endpointsLeft:
+                    return
+                winner.errback(failures.pop())
+
+            checkDone.completed = False
+            checkDone.endpointsLeft = True
+
+            @LoopingCall
+            def iterateEndpoint():
+                endpoint = next(iterEndpoints, None)
+                if endpoint is None:
+                    # The list of endpoints ends.
+                    checkDone.endpointsLeft = False
+                    checkDone()
+                    return
+
+                eachAttempt = endpoint.connect(protocolFactory)
+                pending.append(eachAttempt)
+
+                @eachAttempt.addBoth
+                def noLongerPending(result):
+                    pending.remove(eachAttempt)
+                    return result
+
+                @eachAttempt.addCallback
+                def succeeded(result):
+                    winner.callback(result)
+
+                @eachAttempt.addErrback
+                def failed(reason):
+                    failures.append(reason)
+                    checkDone()
+
+            iterateEndpoint.clock = self._reactor
+            iterateEndpoint.start(self._attemptDelay)
+
+            @winner.addBoth
+            def cancelRemainingPending(result):
+                checkDone.completed = True
+                for remaining in pending[:]:
+                    remaining.cancel()
+                if iterateEndpoint.running:
+                    iterateEndpoint.stop()
+                return result
+
+            return winner
+
+        return d
+
+    def _fallbackNameResolution(self, host, port):
+        """
+        Resolve the hostname string into a tuple containing the host
+        address.  This is method is only used when the reactor does
+        not provide L{IReactorPluggableNameResolver}.
+
+        @param host: A unicode hostname to resolve.
+
+        @param port: The port to include in the resolution.
+
+        @return: A L{Deferred} that fires with L{_getaddrinfo}'s
+            return value.
+        """
+        return self._deferToThread(self._getaddrinfo, host, port, 0, socket.SOCK_STREAM)
+
+
+@implementer(interfaces.IStreamServerEndpoint)
+class SSL4ServerEndpoint:
+    """
+    SSL secured TCP server endpoint with an IPv4 configuration.
+    """
+
+    def __init__(self, reactor, port, sslContextFactory, backlog=50, interface=""):
+        """
+        @param reactor: An L{IReactorSSL} provider.
+
+        @param port: The port number used for listening
+        @type port: int
+
+        @param sslContextFactory: An instance of
+            L{interfaces.IOpenSSLContextFactory}.
+
+        @param backlog: Size of the listen queue
+        @type backlog: int
+
+        @param interface: The hostname to bind to, defaults to '' (all)
+        @type interface: str
+        """
+        self._reactor = reactor
+        self._port = port
+        self._sslContextFactory = sslContextFactory
+        self._backlog = backlog
+        self._interface = interface
+
+    def listen(self, protocolFactory):
+        """
+        Implement L{IStreamServerEndpoint.listen} to listen for SSL on a
+        TCP socket.
+        """
+        return defer.execute(
+            self._reactor.listenSSL,
+            self._port,
+            protocolFactory,
+            contextFactory=self._sslContextFactory,
+            backlog=self._backlog,
+            interface=self._interface,
+        )
+
+
+@implementer(interfaces.IStreamClientEndpoint)
+class SSL4ClientEndpoint:
+    """
+    SSL secured TCP client endpoint with an IPv4 configuration
+    """
+
+    def __init__(
+        self, reactor, host, port, sslContextFactory, timeout=30, bindAddress=None
+    ):
+        """
+        @param reactor: An L{IReactorSSL} provider.
+
+        @param host: A hostname, used when connecting
+        @type host: str
+
+        @param port: The port number, used when connecting
+        @type port: int
+
+        @param sslContextFactory: SSL Configuration information as an instance
+            of L{interfaces.IOpenSSLContextFactory}.
+
+        @param timeout: Number of seconds to wait before assuming the
+            connection has failed.
+        @type timeout: int
+
+        @param bindAddress: A (host, port) tuple of local address to bind to,
+            or None.
+        @type bindAddress: tuple
+        """
+        self._reactor = reactor
+        self._host = host
+        self._port = port
+        self._sslContextFactory = sslContextFactory
+        self._timeout = timeout
+        self._bindAddress = bindAddress
+
+    def connect(self, protocolFactory):
+        """
+        Implement L{IStreamClientEndpoint.connect} to connect with SSL over
+        TCP.
+        """
+        try:
+            wf = _WrappingFactory(protocolFactory)
+            self._reactor.connectSSL(
+                self._host,
+                self._port,
+                wf,
+                self._sslContextFactory,
+                timeout=self._timeout,
+                bindAddress=self._bindAddress,
+            )
+            return wf._onConnection
+        except BaseException:
+            return defer.fail()
+
+
+@implementer(interfaces.IStreamServerEndpoint)
+class UNIXServerEndpoint:
+    """
+    UnixSocket server endpoint.
+    """
+
+    def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0):
+        """
+        @param reactor: An L{IReactorUNIX} provider.
+        @param address: The path to the Unix socket file, used when listening
+        @param backlog: number of connections to allow in backlog.
+        @param mode: mode to set on the unix socket.  This parameter is
+            deprecated.  Permissions should be set on the directory which
+            contains the UNIX socket.
+        @param wantPID: If True, create a pidfile for the socket.
+        """
+        self._reactor = reactor
+        self._address = address
+        self._backlog = backlog
+        self._mode = mode
+        self._wantPID = wantPID
+
+    def listen(self, protocolFactory):
+        """
+        Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket.
+        """
+        return defer.execute(
+            self._reactor.listenUNIX,
+            self._address,
+            protocolFactory,
+            backlog=self._backlog,
+            mode=self._mode,
+            wantPID=self._wantPID,
+        )
+
+
+@implementer(interfaces.IStreamClientEndpoint)
+class UNIXClientEndpoint:
+    """
+    UnixSocket client endpoint.
+    """
+
+    def __init__(self, reactor, path, timeout=30, checkPID=0):
+        """
+        @param reactor: An L{IReactorUNIX} provider.
+
+        @param path: The path to the Unix socket file, used when connecting
+        @type path: str
+
+        @param timeout: Number of seconds to wait before assuming the
+            connection has failed.
+        @type timeout: int
+
+        @param checkPID: If True, check for a pid file to verify that a server
+            is listening.
+        @type checkPID: bool
+        """
+        self._reactor = reactor
+        self._path = path
+        self._timeout = timeout
+        self._checkPID = checkPID
+
+    def connect(self, protocolFactory):
+        """
+        Implement L{IStreamClientEndpoint.connect} to connect via a
+        UNIX Socket
+        """
+        try:
+            wf = _WrappingFactory(protocolFactory)
+            self._reactor.connectUNIX(
+                self._path, wf, timeout=self._timeout, checkPID=self._checkPID
+            )
+            return wf._onConnection
+        except BaseException:
+            return defer.fail()
+
+
+@implementer(interfaces.IStreamServerEndpoint)
+class AdoptedStreamServerEndpoint:
+    """
+    An endpoint for listening on a file descriptor initialized outside of
+    Twisted.
+
+    @ivar _used: A C{bool} indicating whether this endpoint has been used to
+        listen with a factory yet.  C{True} if so.
+    """
+
+    _close = os.close
+    _setNonBlocking = staticmethod(fdesc.setNonBlocking)
+
+    def __init__(self, reactor, fileno, addressFamily):
+        """
+        @param reactor: An L{IReactorSocket} provider.
+
+        @param fileno: An integer file descriptor corresponding to a listening
+            I{SOCK_STREAM} socket.
+
+        @param addressFamily: The address family of the socket given by
+            C{fileno}.
+        """
+        self.reactor = reactor
+        self.fileno = fileno
+        self.addressFamily = addressFamily
+        self._used = False
+
+    def listen(self, factory):
+        """
+        Implement L{IStreamServerEndpoint.listen} to start listening on, and
+        then close, C{self._fileno}.
+        """
+        if self._used:
+            return defer.fail(error.AlreadyListened())
+        self._used = True
+
+        try:
+            self._setNonBlocking(self.fileno)
+            port = self.reactor.adoptStreamPort(
+                self.fileno, self.addressFamily, factory
+            )
+            self._close(self.fileno)
+        except BaseException:
+            return defer.fail()
+        return defer.succeed(port)
+
+
+def _parseTCP(factory, port, interface="", backlog=50):
+    """
+    Internal parser function for L{_parseServer} to convert the string
+    arguments for a TCP(IPv4) stream endpoint into the structured arguments.
+
+    @param factory: the protocol factory being parsed, or L{None}.  (This was a
+        leftover argument from when this code was in C{strports}, and is now
+        mostly None and unused.)
+
+    @type factory: L{IProtocolFactory} or L{None}
+
+    @param port: the integer port number to bind
+    @type port: C{str}
+
+    @param interface: the interface IP to listen on
+    @param backlog: the length of the listen queue
+    @type backlog: C{str}
+
+    @return: a 2-tuple of (args, kwargs), describing  the parameters to
+        L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments
+        to L{TCP4ServerEndpoint}.
+    """
+    return (int(port), factory), {"interface": interface, "backlog": int(backlog)}
+
+
+def _parseUNIX(factory, address, mode="666", backlog=50, lockfile=True):
+    """
+    Internal parser function for L{_parseServer} to convert the string
+    arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the
+    structured arguments.
+
+    @param factory: the protocol factory being parsed, or L{None}.  (This was a
+        leftover argument from when this code was in C{strports}, and is now
+        mostly None and unused.)
+
+    @type factory: L{IProtocolFactory} or L{None}
+
+    @param address: the pathname of the unix socket
+    @type address: C{str}
+
+    @param backlog: the length of the listen queue
+    @type backlog: C{str}
+
+    @param lockfile: A string '0' or '1', mapping to True and False
+        respectively.  See the C{wantPID} argument to C{listenUNIX}
+
+    @return: a 2-tuple of (args, kwargs), describing  the parameters to
+        L{twisted.internet.interfaces.IReactorUNIX.listenUNIX} (or,
+        modulo argument 2, the factory, arguments to L{UNIXServerEndpoint}.
+    """
+    return (
+        (address, factory),
+        {"mode": int(mode, 8), "backlog": int(backlog), "wantPID": bool(int(lockfile))},
+    )
+
+
+def _parseSSL(
+    factory,
+    port,
+    privateKey="server.pem",
+    certKey=None,
+    sslmethod=None,
+    interface="",
+    backlog=50,
+    extraCertChain=None,
+    dhParameters=None,
+):
+    """
+    Internal parser function for L{_parseServer} to convert the string
+    arguments for an SSL (over TCP/IPv4) stream endpoint into the structured
+    arguments.
+
+    @param factory: the protocol factory being parsed, or L{None}.  (This was a
+        leftover argument from when this code was in C{strports}, and is now
+        mostly None and unused.)
+    @type factory: L{IProtocolFactory} or L{None}
+
+    @param port: the integer port number to bind
+    @type port: C{str}
+
+    @param interface: the interface IP to listen on
+    @param backlog: the length of the listen queue
+    @type backlog: C{str}
+
+    @param privateKey: The file name of a PEM format private key file.
+    @type privateKey: C{str}
+
+    @param certKey: The file name of a PEM format certificate file.
+    @type certKey: C{str}
+
+    @param sslmethod: The string name of an SSL method, based on the name of a
+        constant in C{OpenSSL.SSL}.
+    @type sslmethod: C{str}
+
+    @param extraCertChain: The path of a file containing one or more
+        certificates in PEM format that establish the chain from a root CA to
+        the CA that signed your C{certKey}.
+    @type extraCertChain: L{str}
+
+    @param dhParameters: The file name of a file containing parameters that are
+        required for Diffie-Hellman key exchange.  If this is not specified,
+        the forward secret C{DHE} ciphers aren't available for servers.
+    @type dhParameters: L{str}
+
+    @return: a 2-tuple of (args, kwargs), describing  the parameters to
+        L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments
+        to L{SSL4ServerEndpoint}.
+    """
+    from twisted.internet import ssl
+
+    if certKey is None:
+        certKey = privateKey
+    kw = {}
+    if sslmethod is not None:
+        kw["method"] = getattr(ssl.SSL, sslmethod)
+    certPEM = FilePath(certKey).getContent()
+    keyPEM = FilePath(privateKey).getContent()
+    privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + b"\n" + keyPEM)
+    if extraCertChain is not None:
+        matches = re.findall(
+            r"(-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----)",
+            nativeString(FilePath(extraCertChain).getContent()),
+            flags=re.DOTALL,
+        )
+        chainCertificates = [
+            ssl.Certificate.loadPEM(chainCertPEM).original for chainCertPEM in matches
+        ]
+        if not chainCertificates:
+            raise ValueError(
+                "Specified chain file '%s' doesn't contain any valid "
+                "certificates in PEM format." % (extraCertChain,)
+            )
+    else:
+        chainCertificates = None
+    if dhParameters is not None:
+        dhParameters = ssl.DiffieHellmanParameters.fromFile(
+            FilePath(dhParameters),
+        )
+
+    cf = ssl.CertificateOptions(
+        privateKey=privateCertificate.privateKey.original,
+        certificate=privateCertificate.original,
+        extraCertChain=chainCertificates,
+        dhParameters=dhParameters,
+        **kw,
+    )
+    return ((int(port), factory, cf), {"interface": interface, "backlog": int(backlog)})
+
+
+@implementer(IPlugin, IStreamServerEndpointStringParser)
+class _StandardIOParser:
+    """
+    Stream server endpoint string parser for the Standard I/O type.
+
+    @ivar prefix: See L{IStreamServerEndpointStringParser.prefix}.
+    """
+
+    prefix = "stdio"
+
+    def _parseServer(self, reactor):
+        """
+        Internal parser function for L{_parseServer} to convert the string
+        arguments into structured arguments for the L{StandardIOEndpoint}
+
+        @param reactor: Reactor for the endpoint
+        """
+        return StandardIOEndpoint(reactor)
+
+    def parseStreamServer(self, reactor, *args, **kwargs):
+        # Redirects to another function (self._parseServer), tricks zope.interface
+        # into believing the interface is correctly implemented.
+        return self._parseServer(reactor)
+
+
+@implementer(IPlugin, IStreamServerEndpointStringParser)
+class _SystemdParser:
+    """
+    Stream server endpoint string parser for the I{systemd} endpoint type.
+
+    @ivar prefix: See L{IStreamServerEndpointStringParser.prefix}.
+
+    @ivar _sddaemon: A L{ListenFDs} instance used to translate an index into an
+        actual file descriptor.
+    """
+
+    _sddaemon = ListenFDs.fromEnvironment()
+
+    prefix = "systemd"
+
+    def _parseServer(
+        self,
+        reactor: IReactorSocket,
+        domain: str,
+        index: Optional[str] = None,
+        name: Optional[str] = None,
+    ) -> AdoptedStreamServerEndpoint:
+        """
+        Internal parser function for L{_parseServer} to convert the string
+        arguments for a systemd server endpoint into structured arguments for
+        L{AdoptedStreamServerEndpoint}.
+
+        @param reactor: An L{IReactorSocket} provider.
+
+        @param domain: The domain (or address family) of the socket inherited
+            from systemd.  This is a string like C{"INET"} or C{"UNIX"}, ie
+            the name of an address family from the L{socket} module, without
+            the C{"AF_"} prefix.
+
+        @param index: If given, the decimal representation of an integer
+            giving the offset into the list of file descriptors inherited from
+            systemd.  Since the order of descriptors received from systemd is
+            hard to predict, this option should only be used if only one
+            descriptor is being inherited.  Even in that case, C{name} is
+            probably a better idea.  Either C{index} or C{name} must be given.
+
+        @param name: If given, the name (as defined by C{FileDescriptorName}
+            in the C{[Socket]} section of a systemd service definition) of an
+            inherited file descriptor.  Either C{index} or C{name} must be
+            given.
+
+        @return: An L{AdoptedStreamServerEndpoint} which will adopt the
+            inherited listening port when it is used to listen.
+        """
+        if (index is None) == (name is None):
+            raise ValueError("Specify exactly one of descriptor index or name")
+
+        if index is not None:
+            fileno = self._sddaemon.inheritedDescriptors()[int(index)]
+        else:
+            assert name is not None
+            fileno = self._sddaemon.inheritedNamedDescriptors()[name]
+
+        addressFamily = getattr(socket, "AF_" + domain)
+        return AdoptedStreamServerEndpoint(reactor, fileno, addressFamily)
+
+    def parseStreamServer(self, reactor, *args, **kwargs):
+        # Delegate to another function with a sane signature.  This function has
+        # an insane signature to trick zope.interface into believing the
+        # interface is correctly implemented.
+        return self._parseServer(reactor, *args, **kwargs)
+
+
+@implementer(IPlugin, IStreamServerEndpointStringParser)
+class _TCP6ServerParser:
+    """
+    Stream server endpoint string parser for the TCP6ServerEndpoint type.
+
+    @ivar prefix: See L{IStreamServerEndpointStringParser.prefix}.
+    """
+
+    prefix = (
+        "tcp6"  # Used in _parseServer to identify the plugin with the endpoint type
+    )
+
+    def _parseServer(self, reactor, port, backlog=50, interface="::"):
+        """
+        Internal parser function for L{_parseServer} to convert the string
+        arguments into structured arguments for the L{TCP6ServerEndpoint}
+
+        @param reactor: An L{IReactorTCP} provider.
+
+        @param port: The port number used for listening
+        @type port: int
+
+        @param backlog: Size of the listen queue
+        @type backlog: int
+
+        @param interface: The hostname to bind to
+        @type interface: str
+        """
+        port = int(port)
+        backlog = int(backlog)
+        return TCP6ServerEndpoint(reactor, port, backlog, interface)
+
+    def parseStreamServer(self, reactor, *args, **kwargs):
+        # Redirects to another function (self._parseServer), tricks zope.interface
+        # into believing the interface is correctly implemented.
+        return self._parseServer(reactor, *args, **kwargs)
+
+
+_serverParsers = {
+    "tcp": _parseTCP,
+    "unix": _parseUNIX,
+    "ssl": _parseSSL,
+}
+
+_OP, _STRING = range(2)
+
+
+def _tokenize(description):
+    """
+    Tokenize a strports string and yield each token.
+
+    @param description: a string as described by L{serverFromString} or
+        L{clientFromString}.
+    @type description: L{str} or L{bytes}
+
+    @return: an iterable of 2-tuples of (C{_OP} or C{_STRING}, string).  Tuples
+        starting with C{_OP} will contain a second element of either ':' (i.e.
+        'next parameter') or '=' (i.e. 'assign parameter value').  For example,
+        the string 'hello:greeting=world' would result in a generator yielding
+        these values::
+
+            _STRING, 'hello'
+            _OP, ':'
+            _STRING, 'greet=ing'
+            _OP, '='
+            _STRING, 'world'
+    """
+    empty = _matchingString("", description)
+    colon = _matchingString(":", description)
+    equals = _matchingString("=", description)
+    backslash = _matchingString("\x5c", description)
+    current = empty
+
+    ops = colon + equals
+    nextOps = {colon: colon + equals, equals: colon}
+    iterdesc = iter(iterbytes(description))
+    for n in iterdesc:
+        if n in iterbytes(ops):
+            yield _STRING, current
+            yield _OP, n
+            current = empty
+            ops = nextOps[n]
+        elif n == backslash:
+            current += next(iterdesc)
+        else:
+            current += n
+    yield _STRING, current
+
+
+def _parse(description):
+    """
+    Convert a description string into a list of positional and keyword
+    parameters, using logic vaguely like what Python does.
+
+    @param description: a string as described by L{serverFromString} or
+        L{clientFromString}.
+
+    @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all
+        ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of
+        all C{str}s which do contain an '='.  For example, the result of
+        C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}.
+    """
+    args, kw = [], {}
+    colon = _matchingString(":", description)
+
+    def add(sofar):
+        if len(sofar) == 1:
+            args.append(sofar[0])
+        else:
+            kw[nativeString(sofar[0])] = sofar[1]
+
+    sofar = ()
+    for type, value in _tokenize(description):
+        if type is _STRING:
+            sofar += (value,)
+        elif value == colon:
+            add(sofar)
+            sofar = ()
+    add(sofar)
+    return args, kw
+
+
+# Mappings from description "names" to endpoint constructors.
+_endpointServerFactories = {
+    "TCP": TCP4ServerEndpoint,
+    "SSL": SSL4ServerEndpoint,
+    "UNIX": UNIXServerEndpoint,
+}
+
+_endpointClientFactories = {
+    "TCP": TCP4ClientEndpoint,
+    "SSL": SSL4ClientEndpoint,
+    "UNIX": UNIXClientEndpoint,
+}
+
+
+def _parseServer(description, factory):
+    """
+    Parse a strports description into a 2-tuple of arguments and keyword
+    values.
+
+    @param description: A description in the format explained by
+        L{serverFromString}.
+    @type description: C{str}
+
+    @param factory: A 'factory' argument; this is left-over from
+        twisted.application.strports, it's not really used.
+    @type factory: L{IProtocolFactory} or L{None}
+
+    @return: a 3-tuple of (plugin or name, arguments, keyword arguments)
+    """
+    args, kw = _parse(description)
+    endpointType = args[0]
+    parser = _serverParsers.get(endpointType)
+    if parser is None:
+        # If the required parser is not found in _server, check if
+        # a plugin exists for the endpointType
+        plugin = _matchPluginToPrefix(
+            getPlugins(IStreamServerEndpointStringParser), endpointType
+        )
+        return (plugin, args[1:], kw)
+    return (endpointType.upper(),) + parser(factory, *args[1:], **kw)
+
+
+def _matchPluginToPrefix(plugins, endpointType):
+    """
+    Match plugin to prefix.
+    """
+    endpointType = endpointType.lower()
+    for plugin in plugins:
+        if _matchingString(plugin.prefix.lower(), endpointType) == endpointType:
+            return plugin
+    raise ValueError(f"Unknown endpoint type: '{endpointType}'")
+
+
+def serverFromString(reactor, description):
+    """
+    Construct a stream server endpoint from an endpoint description string.
+
+    The format for server endpoint descriptions is a simple byte string.  It is
+    a prefix naming the type of endpoint, then a colon, then the arguments for
+    that endpoint.
+
+    For example, you can call it like this to create an endpoint that will
+    listen on TCP port 80::
+
+        serverFromString(reactor, "tcp:80")
+
+    Additional arguments may be specified as keywords, separated with colons.
+    For example, you can specify the interface for a TCP server endpoint to
+    bind to like this::
+
+        serverFromString(reactor, "tcp:80:interface=127.0.0.1")
+
+    SSL server endpoints may be specified with the 'ssl' prefix, and the
+    private key and certificate files may be specified by the C{privateKey} and
+    C{certKey} arguments::
+
+        serverFromString(
+            reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem")
+
+    If a private key file name (C{privateKey}) isn't provided, a "server.pem"
+    file is assumed to exist which contains the private key. If the certificate
+    file name (C{certKey}) isn't provided, the private key file is assumed to
+    contain the certificate as well.
+
+    You may escape colons in arguments with a backslash, which you will need to
+    use if you want to specify a full pathname argument on Windows::
+
+        serverFromString(reactor,
+            "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem")
+
+    finally, the 'unix' prefix may be used to specify a filesystem UNIX socket,
+    optionally with a 'mode' argument to specify the mode of the socket file
+    created by C{listen}::
+
+        serverFromString(reactor, "unix:/var/run/finger")
+        serverFromString(reactor, "unix:/var/run/finger:mode=660")
+
+    This function is also extensible; new endpoint types may be registered as
+    L{IStreamServerEndpointStringParser} plugins.  See that interface for more
+    information.
+
+    @param reactor: The server endpoint will be constructed with this reactor.
+
+    @param description: The strports description to parse.
+    @type description: L{str}
+
+    @return: A new endpoint which can be used to listen with the parameters
+        given by C{description}.
+
+    @rtype: L{IStreamServerEndpoint<twisted.internet.interfaces.IStreamServerEndpoint>}
+
+    @raise ValueError: when the 'description' string cannot be parsed.
+
+    @since: 10.2
+    """
+    nameOrPlugin, args, kw = _parseServer(description, None)
+    if type(nameOrPlugin) is not str:
+        plugin = nameOrPlugin
+        return plugin.parseStreamServer(reactor, *args, **kw)
+    else:
+        name = nameOrPlugin
+    # Chop out the factory.
+    args = args[:1] + args[2:]
+    return _endpointServerFactories[name](reactor, *args, **kw)
+
+
+def quoteStringArgument(argument):
+    """
+    Quote an argument to L{serverFromString} and L{clientFromString}.  Since
+    arguments are separated with colons and colons are escaped with
+    backslashes, some care is necessary if, for example, you have a pathname,
+    you may be tempted to interpolate into a string like this::
+
+        serverFromString(reactor, "ssl:443:privateKey=%s" % (myPathName,))
+
+    This may appear to work, but will have portability issues (Windows
+    pathnames, for example).  Usually you should just construct the appropriate
+    endpoint type rather than interpolating strings, which in this case would
+    be L{SSL4ServerEndpoint}.  There are some use-cases where you may need to
+    generate such a string, though; for example, a tool to manipulate a
+    configuration file which has strports descriptions in it.  To be correct in
+    those cases, do this instead::
+
+        serverFromString(reactor, "ssl:443:privateKey=%s" %
+                         (quoteStringArgument(myPathName),))
+
+    @param argument: The part of the endpoint description string you want to
+        pass through.
+
+    @type argument: C{str}
+
+    @return: The quoted argument.
+
+    @rtype: C{str}
+    """
+    backslash, colon = "\\:"
+    for c in backslash, colon:
+        argument = argument.replace(c, backslash + c)
+    return argument
+
+
+def _parseClientTCP(*args, **kwargs):
+    """
+    Perform any argument value coercion necessary for TCP client parameters.
+
+    Valid positional arguments to this function are host and port.
+
+    Valid keyword arguments to this function are all L{IReactorTCP.connectTCP}
+    arguments.
+
+    @return: The coerced values as a C{dict}.
+    """
+
+    if len(args) == 2:
+        kwargs["port"] = int(args[1])
+        kwargs["host"] = args[0]
+    elif len(args) == 1:
+        if "host" in kwargs:
+            kwargs["port"] = int(args[0])
+        else:
+            kwargs["host"] = args[0]
+
+    try:
+        kwargs["port"] = int(kwargs["port"])
+    except KeyError:
+        pass
+
+    try:
+        kwargs["timeout"] = int(kwargs["timeout"])
+    except KeyError:
+        pass
+
+    try:
+        kwargs["bindAddress"] = (kwargs["bindAddress"], 0)
+    except KeyError:
+        pass
+
+    return kwargs
+
+
+def _loadCAsFromDir(directoryPath):
+    """
+    Load certificate-authority certificate objects in a given directory.
+
+    @param directoryPath: a L{unicode} or L{bytes} pointing at a directory to
+        load .pem files from, or L{None}.
+
+    @return: an L{IOpenSSLTrustRoot} provider.
+    """
+    caCerts = {}
+    for child in directoryPath.children():
+        if not child.asTextMode().basename().split(".")[-1].lower() == "pem":
+            continue
+        try:
+            data = child.getContent()
+        except OSError:
+            # Permission denied, corrupt disk, we don't care.
+            continue
+        try:
+            theCert = Certificate.loadPEM(data)
+        except SSLError:
+            # Duplicate certificate, invalid certificate, etc.  We don't care.
+            pass
+        else:
+            caCerts[theCert.digest()] = theCert
+    return trustRootFromCertificates(caCerts.values())
+
+
+def _parseTrustRootPath(pathName):
+    """
+    Parse a string referring to a directory full of certificate authorities
+    into a trust root.
+
+    @param pathName: path name
+    @type pathName: L{unicode} or L{bytes} or L{None}
+
+    @return: L{None} or L{IOpenSSLTrustRoot}
+    """
+    if pathName is None:
+        return None
+    return _loadCAsFromDir(FilePath(pathName))
+
+
+def _privateCertFromPaths(certificatePath, keyPath):
+    """
+    Parse a certificate path and key path, either or both of which might be
+    L{None}, into a certificate object.
+
+    @param certificatePath: the certificate path
+    @type certificatePath: L{bytes} or L{unicode} or L{None}
+
+    @param keyPath: the private key path
+    @type keyPath: L{bytes} or L{unicode} or L{None}
+
+    @return: a L{PrivateCertificate} or L{None}
+    """
+    if certificatePath is None:
+        return None
+    certBytes = FilePath(certificatePath).getContent()
+    if keyPath is None:
+        return PrivateCertificate.loadPEM(certBytes)
+    else:
+        return PrivateCertificate.fromCertificateAndKeyPair(
+            Certificate.loadPEM(certBytes),
+            KeyPair.load(FilePath(keyPath).getContent(), 1),
+        )
+
+
+def _parseClientSSLOptions(kwargs):
+    """
+    Parse common arguments for SSL endpoints, creating an L{CertificateOptions}
+    instance.
+
+    @param kwargs: A dict of keyword arguments to be parsed, potentially
+        containing keys C{certKey}, C{privateKey}, C{caCertsDir}, and
+        C{hostname}.  See L{_parseClientSSL}.
+    @type kwargs: L{dict}
+
+    @return: The remaining arguments, including a new key C{sslContextFactory}.
+    """
+    hostname = kwargs.pop("hostname", None)
+    clientCertificate = _privateCertFromPaths(
+        kwargs.pop("certKey", None), kwargs.pop("privateKey", None)
+    )
+    trustRoot = _parseTrustRootPath(kwargs.pop("caCertsDir", None))
+    if hostname is not None:
+        configuration = optionsForClientTLS(
+            _idnaText(hostname),
+            trustRoot=trustRoot,
+            clientCertificate=clientCertificate,
+        )
+    else:
+        # _really_ though, you should specify a hostname.
+        if clientCertificate is not None:
+            privateKeyOpenSSL = clientCertificate.privateKey.original
+            certificateOpenSSL = clientCertificate.original
+        else:
+            privateKeyOpenSSL = None
+            certificateOpenSSL = None
+        configuration = CertificateOptions(
+            trustRoot=trustRoot,
+            privateKey=privateKeyOpenSSL,
+            certificate=certificateOpenSSL,
+        )
+    kwargs["sslContextFactory"] = configuration
+    return kwargs
+
+
+def _parseClientSSL(*args, **kwargs):
+    """
+    Perform any argument value coercion necessary for SSL client parameters.
+
+    Valid keyword arguments to this function are all L{IReactorSSL.connectSSL}
+    arguments except for C{contextFactory}.  Instead, C{certKey} (the path name
+    of the certificate file) C{privateKey} (the path name of the private key
+    associated with the certificate) are accepted and used to construct a
+    context factory.
+
+    Valid positional arguments to this function are host and port.
+
+    @keyword caCertsDir: The one parameter which is not part of
+        L{IReactorSSL.connectSSL}'s signature, this is a path name used to
+        construct a list of certificate authority certificates.  The directory
+        will be scanned for files ending in C{.pem}, all of which will be
+        considered valid certificate authorities for this connection.
+    @type caCertsDir: L{str}
+
+    @keyword hostname: The hostname to use for validating the server's
+        certificate.
+    @type hostname: L{unicode}
+
+    @return: The coerced values as a L{dict}.
+    """
+    kwargs = _parseClientTCP(*args, **kwargs)
+    return _parseClientSSLOptions(kwargs)
+
+
+def _parseClientUNIX(*args, **kwargs):
+    """
+    Perform any argument value coercion necessary for UNIX client parameters.
+
+    Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX}
+    keyword arguments except for C{checkPID}.  Instead, C{lockfile} is accepted
+    and has the same meaning.  Also C{path} is used instead of C{address}.
+
+    Valid positional arguments to this function are C{path}.
+
+    @return: The coerced values as a C{dict}.
+    """
+    if len(args) == 1:
+        kwargs["path"] = args[0]
+
+    try:
+        kwargs["checkPID"] = bool(int(kwargs.pop("lockfile")))
+    except KeyError:
+        pass
+    try:
+        kwargs["timeout"] = int(kwargs["timeout"])
+    except KeyError:
+        pass
+    return kwargs
+
+
+_clientParsers = {
+    "TCP": _parseClientTCP,
+    "SSL": _parseClientSSL,
+    "UNIX": _parseClientUNIX,
+}
+
+
+def clientFromString(reactor, description):
+    """
+    Construct a client endpoint from a description string.
+
+    Client description strings are much like server description strings,
+    although they take all of their arguments as keywords, aside from host and
+    port.
+
+    You can create a TCP client endpoint with the 'host' and 'port' arguments,
+    like so::
+
+        clientFromString(reactor, "tcp:host=www.example.com:port=80")
+
+    or, without specifying host and port keywords::
+
+        clientFromString(reactor, "tcp:www.example.com:80")
+
+    Or you can specify only one or the other, as in the following 2 examples::
+
+        clientFromString(reactor, "tcp:host=www.example.com:80")
+        clientFromString(reactor, "tcp:www.example.com:port=80")
+
+    or an SSL client endpoint with those arguments, plus the arguments used by
+    the server SSL, for a client certificate::
+
+        clientFromString(reactor, "ssl:web.example.com:443:"
+                                  "privateKey=foo.pem:certKey=foo.pem")
+
+    to specify your certificate trust roots, you can identify a directory with
+    PEM files in it with the C{caCertsDir} argument::
+
+        clientFromString(reactor, "ssl:host=web.example.com:port=443:"
+                                  "caCertsDir=/etc/ssl/certs")
+
+    Both TCP and SSL client endpoint description strings can include a
+    'bindAddress' keyword argument, whose value should be a local IPv4
+    address. This fixes the client socket to that IP address::
+
+        clientFromString(reactor, "tcp:www.example.com:80:"
+                                  "bindAddress=192.0.2.100")
+
+    NB: Fixed client ports are not currently supported in TCP or SSL
+    client endpoints. The client socket will always use an ephemeral
+    port assigned by the operating system
+
+    You can create a UNIX client endpoint with the 'path' argument and optional
+    'lockfile' and 'timeout' arguments::
+
+        clientFromString(
+            reactor, b"unix:path=/var/foo/bar:lockfile=1:timeout=9")
+
+    or, with the path as a positional argument with or without optional
+    arguments as in the following 2 examples::
+
+        clientFromString(reactor, "unix:/var/foo/bar")
+        clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9")
+
+    This function is also extensible; new endpoint types may be registered as
+    L{IStreamClientEndpointStringParserWithReactor} plugins.  See that
+    interface for more information.
+
+    @param reactor: The client endpoint will be constructed with this reactor.
+
+    @param description: The strports description to parse.
+    @type description: L{str}
+
+    @return: A new endpoint which can be used to connect with the parameters
+        given by C{description}.
+    @rtype: L{IStreamClientEndpoint<twisted.internet.interfaces.IStreamClientEndpoint>}
+
+    @since: 10.2
+    """
+    args, kwargs = _parse(description)
+    aname = args.pop(0)
+    name = aname.upper()
+    if name not in _clientParsers:
+        plugin = _matchPluginToPrefix(
+            getPlugins(IStreamClientEndpointStringParserWithReactor), name
+        )
+        return plugin.parseStreamClient(reactor, *args, **kwargs)
+    kwargs = _clientParsers[name](*args, **kwargs)
+    return _endpointClientFactories[name](reactor, **kwargs)
+
+
+def connectProtocol(endpoint, protocol):
+    """
+    Connect a protocol instance to an endpoint.
+
+    This allows using a client endpoint without having to create a factory.
+
+    @param endpoint: A client endpoint to connect to.
+
+    @param protocol: A protocol instance.
+
+    @return: The result of calling C{connect} on the endpoint, i.e. a
+        L{Deferred} that will fire with the protocol when connected, or an
+        appropriate error.
+
+    @since: 13.1
+    """
+
+    class OneShotFactory(Factory):
+        def buildProtocol(self, addr):
+            return protocol
+
+    return endpoint.connect(OneShotFactory())
+
+
+@implementer(interfaces.IStreamClientEndpoint)
+class _WrapperEndpoint:
+    """
+    An endpoint that wraps another endpoint.
+    """
+
+    def __init__(self, wrappedEndpoint, wrapperFactory):
+        """
+        Construct a L{_WrapperEndpoint}.
+        """
+        self._wrappedEndpoint = wrappedEndpoint
+        self._wrapperFactory = wrapperFactory
+
+    def connect(self, protocolFactory):
+        """
+        Connect the given protocol factory and unwrap its result.
+        """
+        return self._wrappedEndpoint.connect(
+            self._wrapperFactory(protocolFactory)
+        ).addCallback(lambda protocol: protocol.wrappedProtocol)
+
+
+@implementer(interfaces.IStreamServerEndpoint)
+class _WrapperServerEndpoint:
+    """
+    A server endpoint that wraps another server endpoint.
+    """
+
+    def __init__(self, wrappedEndpoint, wrapperFactory):
+        """
+        Construct a L{_WrapperServerEndpoint}.
+        """
+        self._wrappedEndpoint = wrappedEndpoint
+        self._wrapperFactory = wrapperFactory
+
+    def listen(self, protocolFactory):
+        """
+        Connect the given protocol factory and unwrap its result.
+        """
+        return self._wrappedEndpoint.listen(self._wrapperFactory(protocolFactory))
+
+
+def wrapClientTLS(connectionCreator, wrappedEndpoint):
+    """
+    Wrap an endpoint which upgrades to TLS as soon as the connection is
+    established.
+
+    @since: 16.0
+
+    @param connectionCreator: The TLS options to use when connecting; see
+        L{twisted.internet.ssl.optionsForClientTLS} for how to construct this.
+    @type connectionCreator:
+        L{twisted.internet.interfaces.IOpenSSLClientConnectionCreator}
+
+    @param wrappedEndpoint: The endpoint to wrap.
+    @type wrappedEndpoint: An L{IStreamClientEndpoint} provider.
+
+    @return: an endpoint that provides transport level encryption layered on
+        top of C{wrappedEndpoint}
+    @rtype: L{twisted.internet.interfaces.IStreamClientEndpoint}
+    """
+    if TLSMemoryBIOFactory is None:
+        raise NotImplementedError(
+            "OpenSSL not available. Try `pip install twisted[tls]`."
+        )
+    return _WrapperEndpoint(
+        wrappedEndpoint,
+        lambda protocolFactory: TLSMemoryBIOFactory(
+            connectionCreator, True, protocolFactory
+        ),
+    )
+
+
+def _parseClientTLS(
+    reactor,
+    host,
+    port,
+    timeout=b"30",
+    bindAddress=None,
+    certificate=None,
+    privateKey=None,
+    trustRoots=None,
+    endpoint=None,
+    **kwargs,
+):
+    """
+    Internal method to construct an endpoint from string parameters.
+
+    @param reactor: The reactor passed to L{clientFromString}.
+
+    @param host: The hostname to connect to.
+    @type host: L{bytes} or L{unicode}
+
+    @param port: The port to connect to.
+    @type port: L{bytes} or L{unicode}
+
+    @param timeout: For each individual connection attempt, the number of
+        seconds to wait before assuming the connection has failed.
+    @type timeout: L{bytes} or L{unicode}
+
+    @param bindAddress: The address to which to bind outgoing connections.
+    @type bindAddress: L{bytes} or L{unicode}
+
+    @param certificate: a string representing a filesystem path to a
+        PEM-encoded certificate.
+    @type certificate: L{bytes} or L{unicode}
+
+    @param privateKey: a string representing a filesystem path to a PEM-encoded
+        certificate.
+    @type privateKey: L{bytes} or L{unicode}
+
+    @param endpoint: an optional string endpoint description of an endpoint to
+        wrap; if this is passed then C{host} is used only for certificate
+        verification.
+    @type endpoint: L{bytes} or L{unicode}
+
+    @return: a client TLS endpoint
+    @rtype: L{IStreamClientEndpoint}
+    """
+    if kwargs:
+        raise TypeError("unrecognized keyword arguments present", list(kwargs.keys()))
+    host = host if isinstance(host, str) else host.decode("utf-8")
+    bindAddress = (
+        bindAddress
+        if isinstance(bindAddress, str) or bindAddress is None
+        else bindAddress.decode("utf-8")
+    )
+    port = int(port)
+    timeout = int(timeout)
+    return wrapClientTLS(
+        optionsForClientTLS(
+            host,
+            trustRoot=_parseTrustRootPath(trustRoots),
+            clientCertificate=_privateCertFromPaths(certificate, privateKey),
+        ),
+        clientFromString(reactor, endpoint)
+        if endpoint is not None
+        else HostnameEndpoint(reactor, _idnaBytes(host), port, timeout, bindAddress),
+    )
+
+
+@implementer(IPlugin, IStreamClientEndpointStringParserWithReactor)
+class _TLSClientEndpointParser:
+    """
+    Stream client endpoint string parser for L{wrapClientTLS} with
+    L{HostnameEndpoint}.
+
+    @ivar prefix: See
+        L{IStreamClientEndpointStringParserWithReactor.prefix}.
+    """
+
+    prefix = "tls"
+
+    @staticmethod
+    def parseStreamClient(reactor, *args, **kwargs):
+        """
+        Redirects to another function L{_parseClientTLS}; tricks zope.interface
+        into believing the interface is correctly implemented, since the
+        signature is (C{reactor}, C{*args}, C{**kwargs}).  See
+        L{_parseClientTLS} for the specific signature description for this
+        endpoint parser.
+
+        @param reactor: The reactor passed to L{clientFromString}.
+
+        @param args: The positional arguments in the endpoint description.
+        @type args: L{tuple}
+
+        @param kwargs: The named arguments in the endpoint description.
+        @type kwargs: L{dict}
+
+        @return: a client TLS endpoint
+        @rtype: L{IStreamClientEndpoint}
+        """
+        return _parseClientTLS(reactor, *args, **kwargs)
diff --git a/contrib/python/Twisted/py3/twisted/internet/epollreactor.py b/contrib/python/Twisted/py3/twisted/internet/epollreactor.py
new file mode 100644
index 00000000000..c13e50be013
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/epollreactor.py
@@ -0,0 +1,259 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An epoll() based implementation of the twisted main loop.
+
+To install the event loop (and you should do this before any connections,
+listeners or connectors are added)::
+
+    from twisted.internet import epollreactor
+    epollreactor.install()
+"""
+
+import errno
+import select
+
+from zope.interface import implementer
+
+from twisted.internet import posixbase
+from twisted.internet.interfaces import IReactorFDSet
+from twisted.python import log
+
+try:
+    # This is to keep mypy from complaining
+    # We don't use type: ignore[attr-defined] on import, because mypy only complains
+    # on on some platforms, and then the unused ignore is an issue if the undefined
+    # attribute isn't.
+    epoll = getattr(select, "epoll")
+    EPOLLHUP = getattr(select, "EPOLLHUP")
+    EPOLLERR = getattr(select, "EPOLLERR")
+    EPOLLIN = getattr(select, "EPOLLIN")
+    EPOLLOUT = getattr(select, "EPOLLOUT")
+except AttributeError as e:
+    raise ImportError(e)
+
+
+@implementer(IReactorFDSet)
+class EPollReactor(posixbase.PosixReactorBase, posixbase._PollLikeMixin):
+    """
+    A reactor that uses epoll(7).
+
+    @ivar _poller: A C{epoll} which will be used to check for I/O
+        readiness.
+
+    @ivar _selectables: A dictionary mapping integer file descriptors to
+        instances of C{FileDescriptor} which have been registered with the
+        reactor.  All C{FileDescriptors} which are currently receiving read or
+        write readiness notifications will be present as values in this
+        dictionary.
+
+    @ivar _reads: A set containing integer file descriptors.  Values in this
+        set will be registered with C{_poller} for read readiness notifications
+        which will be dispatched to the corresponding C{FileDescriptor}
+        instances in C{_selectables}.
+
+    @ivar _writes: A set containing integer file descriptors.  Values in this
+        set will be registered with C{_poller} for write readiness
+        notifications which will be dispatched to the corresponding
+        C{FileDescriptor} instances in C{_selectables}.
+
+    @ivar _continuousPolling: A L{_ContinuousPolling} instance, used to handle
+        file descriptors (e.g. filesystem files) that are not supported by
+        C{epoll(7)}.
+    """
+
+    # Attributes for _PollLikeMixin
+    _POLL_DISCONNECTED = EPOLLHUP | EPOLLERR
+    _POLL_IN = EPOLLIN
+    _POLL_OUT = EPOLLOUT
+
+    def __init__(self):
+        """
+        Initialize epoll object, file descriptor tracking dictionaries, and the
+        base class.
+        """
+        # Create the poller we're going to use.  The 1024 here is just a hint
+        # to the kernel, it is not a hard maximum.  After Linux 2.6.8, the size
+        # argument is completely ignored.
+        self._poller = epoll(1024)
+        self._reads = set()
+        self._writes = set()
+        self._selectables = {}
+        self._continuousPolling = posixbase._ContinuousPolling(self)
+        posixbase.PosixReactorBase.__init__(self)
+
+    def _add(self, xer, primary, other, selectables, event, antievent):
+        """
+        Private method for adding a descriptor from the event loop.
+
+        It takes care of adding it if  new or modifying it if already added
+        for another state (read -> read/write for example).
+        """
+        fd = xer.fileno()
+        if fd not in primary:
+            flags = event
+            # epoll_ctl can raise all kinds of IOErrors, and every one
+            # indicates a bug either in the reactor or application-code.
+            # Let them all through so someone sees a traceback and fixes
+            # something.  We'll do the same thing for every other call to
+            # this method in this file.
+            if fd in other:
+                flags |= antievent
+                self._poller.modify(fd, flags)
+            else:
+                self._poller.register(fd, flags)
+
+            # Update our own tracking state *only* after the epoll call has
+            # succeeded.  Otherwise we may get out of sync.
+            primary.add(fd)
+            selectables[fd] = xer
+
+    def addReader(self, reader):
+        """
+        Add a FileDescriptor for notification of data available to read.
+        """
+        try:
+            self._add(
+                reader, self._reads, self._writes, self._selectables, EPOLLIN, EPOLLOUT
+            )
+        except OSError as e:
+            if e.errno == errno.EPERM:
+                # epoll(7) doesn't support certain file descriptors,
+                # e.g. filesystem files, so for those we just poll
+                # continuously:
+                self._continuousPolling.addReader(reader)
+            else:
+                raise
+
+    def addWriter(self, writer):
+        """
+        Add a FileDescriptor for notification of data available to write.
+        """
+        try:
+            self._add(
+                writer, self._writes, self._reads, self._selectables, EPOLLOUT, EPOLLIN
+            )
+        except OSError as e:
+            if e.errno == errno.EPERM:
+                # epoll(7) doesn't support certain file descriptors,
+                # e.g. filesystem files, so for those we just poll
+                # continuously:
+                self._continuousPolling.addWriter(writer)
+            else:
+                raise
+
+    def _remove(self, xer, primary, other, selectables, event, antievent):
+        """
+        Private method for removing a descriptor from the event loop.
+
+        It does the inverse job of _add, and also add a check in case of the fd
+        has gone away.
+        """
+        fd = xer.fileno()
+        if fd == -1:
+            for fd, fdes in selectables.items():
+                if xer is fdes:
+                    break
+            else:
+                return
+        if fd in primary:
+            if fd in other:
+                flags = antievent
+                # See comment above modify call in _add.
+                self._poller.modify(fd, flags)
+            else:
+                del selectables[fd]
+                # See comment above _control call in _add.
+                self._poller.unregister(fd)
+            primary.remove(fd)
+
+    def removeReader(self, reader):
+        """
+        Remove a Selectable for notification of data available to read.
+        """
+        if self._continuousPolling.isReading(reader):
+            self._continuousPolling.removeReader(reader)
+            return
+        self._remove(
+            reader, self._reads, self._writes, self._selectables, EPOLLIN, EPOLLOUT
+        )
+
+    def removeWriter(self, writer):
+        """
+        Remove a Selectable for notification of data available to write.
+        """
+        if self._continuousPolling.isWriting(writer):
+            self._continuousPolling.removeWriter(writer)
+            return
+        self._remove(
+            writer, self._writes, self._reads, self._selectables, EPOLLOUT, EPOLLIN
+        )
+
+    def removeAll(self):
+        """
+        Remove all selectables, and return a list of them.
+        """
+        return (
+            self._removeAll(
+                [self._selectables[fd] for fd in self._reads],
+                [self._selectables[fd] for fd in self._writes],
+            )
+            + self._continuousPolling.removeAll()
+        )
+
+    def getReaders(self):
+        return [
+            self._selectables[fd] for fd in self._reads
+        ] + self._continuousPolling.getReaders()
+
+    def getWriters(self):
+        return [
+            self._selectables[fd] for fd in self._writes
+        ] + self._continuousPolling.getWriters()
+
+    def doPoll(self, timeout):
+        """
+        Poll the poller for new events.
+        """
+        if timeout is None:
+            timeout = -1  # Wait indefinitely.
+
+        try:
+            # Limit the number of events to the number of io objects we're
+            # currently tracking (because that's maybe a good heuristic) and
+            # the amount of time we block to the value specified by our
+            # caller.
+            l = self._poller.poll(timeout, len(self._selectables))
+        except OSError as err:
+            if err.errno == errno.EINTR:
+                return
+            # See epoll_wait(2) for documentation on the other conditions
+            # under which this can fail.  They can only be due to a serious
+            # programming error on our part, so let's just announce them
+            # loudly.
+            raise
+
+        _drdw = self._doReadOrWrite
+        for fd, event in l:
+            try:
+                selectable = self._selectables[fd]
+            except KeyError:
+                pass
+            else:
+                log.callWithLogger(selectable, _drdw, selectable, fd, event)
+
+    doIteration = doPoll
+
+
+def install():
+    """
+    Install the epoll() reactor.
+    """
+    p = EPollReactor()
+    from twisted.internet.main import installReactor
+
+    installReactor(p)
+
+
+__all__ = ["EPollReactor", "install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/error.py b/contrib/python/Twisted/py3/twisted/internet/error.py
new file mode 100644
index 00000000000..e66a194f149
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/error.py
@@ -0,0 +1,510 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Exceptions and errors for use in twisted.internet modules.
+"""
+
+
+import socket
+
+from incremental import Version
+
+from twisted.python import deprecate
+
+
+class BindError(Exception):
+    __doc__ = MESSAGE = "An error occurred binding to an interface"
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.args:
+            s = "{}: {}".format(s, " ".join(self.args))
+        s = "%s." % s
+        return s
+
+
+class CannotListenError(BindError):
+    """
+    This gets raised by a call to startListening, when the object cannotstart
+    listening.
+
+    @ivar interface: the interface I tried to listen on
+    @ivar port: the port I tried to listen on
+    @ivar socketError: the exception I got when I tried to listen
+    @type socketError: L{socket.error}
+    """
+
+    def __init__(self, interface, port, socketError):
+        BindError.__init__(self, interface, port, socketError)
+        self.interface = interface
+        self.port = port
+        self.socketError = socketError
+
+    def __str__(self) -> str:
+        iface = self.interface or "any"
+        return "Couldn't listen on {}:{}: {}.".format(
+            iface, self.port, self.socketError
+        )
+
+
+class MulticastJoinError(Exception):
+    """
+    An attempt to join a multicast group failed.
+    """
+
+
+class MessageLengthError(Exception):
+    __doc__ = MESSAGE = "Message is too long to send"
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.args:
+            s = "{}: {}".format(s, " ".join(self.args))
+        s = "%s." % s
+        return s
+
+
+class DNSLookupError(IOError):
+    __doc__ = MESSAGE = "DNS lookup failed"
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.args:
+            s = "{}: {}".format(s, " ".join(self.args))
+        s = "%s." % s
+        return s
+
+
+class ConnectInProgressError(Exception):
+    """A connect operation was started and isn't done yet."""
+
+
+# connection errors
+
+
+class ConnectError(Exception):
+    __doc__ = MESSAGE = "An error occurred while connecting"
+
+    def __init__(self, osError=None, string=""):
+        self.osError = osError
+        Exception.__init__(self, string)
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.osError:
+            s = f"{s}: {self.osError}"
+        if self.args[0]:
+            s = f"{s}: {self.args[0]}"
+        s = "%s." % s
+        return s
+
+
+class ConnectBindError(ConnectError):
+    __doc__ = MESSAGE = "Couldn't bind"
+
+
+class UnknownHostError(ConnectError):
+    __doc__ = MESSAGE = "Hostname couldn't be looked up"
+
+
+class NoRouteError(ConnectError):
+    __doc__ = MESSAGE = "No route to host"
+
+
+class ConnectionRefusedError(ConnectError):
+    __doc__ = MESSAGE = "Connection was refused by other side"
+
+
+class TCPTimedOutError(ConnectError):
+    __doc__ = MESSAGE = "TCP connection timed out"
+
+
+class BadFileError(ConnectError):
+    __doc__ = MESSAGE = "File used for UNIX socket is no good"
+
+
+class ServiceNameUnknownError(ConnectError):
+    __doc__ = MESSAGE = "Service name given as port is unknown"
+
+
+class UserError(ConnectError):
+    __doc__ = MESSAGE = "User aborted connection"
+
+
+class TimeoutError(UserError):
+    __doc__ = MESSAGE = "User timeout caused connection failure"
+
+
+class SSLError(ConnectError):
+    __doc__ = MESSAGE = "An SSL error occurred"
+
+
+class VerifyError(Exception):
+    __doc__ = MESSAGE = "Could not verify something that was supposed to be signed."
+
+
+class PeerVerifyError(VerifyError):
+    __doc__ = MESSAGE = "The peer rejected our verify error."
+
+
+class CertificateError(Exception):
+    __doc__ = MESSAGE = "We did not find a certificate where we expected to find one."
+
+
+try:
+    import errno
+
+    errnoMapping = {
+        errno.ENETUNREACH: NoRouteError,
+        errno.ECONNREFUSED: ConnectionRefusedError,
+        errno.ETIMEDOUT: TCPTimedOutError,
+    }
+    if hasattr(errno, "WSAECONNREFUSED"):
+        errnoMapping[errno.WSAECONNREFUSED] = ConnectionRefusedError
+        errnoMapping[errno.WSAENETUNREACH] = NoRouteError  # type: ignore[attr-defined]
+except ImportError:
+    errnoMapping = {}
+
+
+def getConnectError(e):
+    """Given a socket exception, return connection error."""
+    if isinstance(e, Exception):
+        args = e.args
+    else:
+        args = e
+    try:
+        number, string = args
+    except ValueError:
+        return ConnectError(string=e)
+
+    if hasattr(socket, "gaierror") and isinstance(e, socket.gaierror):
+        # Only works in 2.2 in newer. Really that means always; #5978 covers
+        # this and other weirdnesses in this function.
+        klass = UnknownHostError
+    else:
+        klass = errnoMapping.get(number, ConnectError)
+    return klass(number, string)
+
+
+class ConnectionClosed(Exception):
+    """
+    Connection was closed, whether cleanly or non-cleanly.
+    """
+
+
+class ConnectionLost(ConnectionClosed):
+    __doc__ = MESSAGE = """
+    Connection to the other side was lost in a non-clean fashion
+    """
+
+    def __str__(self) -> str:
+        s = self.MESSAGE.strip().splitlines()[:1]
+        if self.args:
+            s.append(": ")
+            s.append(" ".join(self.args))
+        s.append(".")
+        return "".join(s)
+
+
+class ConnectionAborted(ConnectionLost):
+    """
+    Connection was aborted locally, using
+    L{twisted.internet.interfaces.ITCPTransport.abortConnection}.
+
+    @since: 11.1
+    """
+
+    MESSAGE = "Connection was aborted locally using " "ITCPTransport.abortConnection"
+
+
+class ConnectionDone(ConnectionClosed):
+    __doc__ = MESSAGE = "Connection was closed cleanly"
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.args:
+            s = "{}: {}".format(s, " ".join(self.args))
+        s = "%s." % s
+        return s
+
+
+class FileDescriptorOverrun(ConnectionLost):
+    """
+    A mis-use of L{IUNIXTransport.sendFileDescriptor} caused the connection to
+    be closed.
+
+    Each file descriptor sent using C{sendFileDescriptor} must be associated
+    with at least one byte sent using L{ITransport.write}.  If at any point
+    fewer bytes have been written than file descriptors have been sent, the
+    connection is closed with this exception.
+    """
+
+    MESSAGE = (
+        "A mis-use of IUNIXTransport.sendFileDescriptor caused "
+        "the connection to be closed."
+    )
+
+
+class ConnectionFdescWentAway(ConnectionLost):
+    __doc__ = MESSAGE = "Uh"  # TODO
+
+
+class AlreadyCalled(ValueError):
+    __doc__ = MESSAGE = "Tried to cancel an already-called event"
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.args:
+            s = "{}: {}".format(s, " ".join(self.args))
+        s = "%s." % s
+        return s
+
+
+class AlreadyCancelled(ValueError):
+    __doc__ = MESSAGE = "Tried to cancel an already-cancelled event"
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.args:
+            s = "{}: {}".format(s, " ".join(self.args))
+        s = "%s." % s
+        return s
+
+
+class PotentialZombieWarning(Warning):
+    """
+    Emitted when L{IReactorProcess.spawnProcess} is called in a way which may
+    result in termination of the created child process not being reported.
+
+    Deprecated in Twisted 10.0.
+    """
+
+    MESSAGE = (
+        "spawnProcess called, but the SIGCHLD handler is not "
+        "installed. This probably means you have not yet "
+        "called reactor.run, or called "
+        "reactor.run(installSignalHandler=0). You will probably "
+        "never see this process finish, and it may become a "
+        "zombie process."
+    )
+
+
+deprecate.deprecatedModuleAttribute(
+    Version("Twisted", 10, 0, 0),
+    "There is no longer any potential for zombie process.",
+    __name__,
+    "PotentialZombieWarning",
+)
+
+
+class ProcessDone(ConnectionDone):
+    __doc__ = MESSAGE = "A process has ended without apparent errors"
+
+    def __init__(self, status):
+        Exception.__init__(self, "process finished with exit code 0")
+        self.exitCode = 0
+        self.signal = None
+        self.status = status
+
+
+class ProcessTerminated(ConnectionLost):
+    __doc__ = MESSAGE = """
+    A process has ended with a probable error condition
+
+    @ivar exitCode: See L{__init__}
+    @ivar signal: See L{__init__}
+    @ivar status: See L{__init__}
+    """
+
+    def __init__(self, exitCode=None, signal=None, status=None):
+        """
+        @param exitCode: The exit status of the process.  This is roughly like
+            the value you might pass to L{os._exit}.  This is L{None} if the
+            process exited due to a signal.
+        @type exitCode: L{int} or L{None}
+
+        @param signal: The exit signal of the process.  This is L{None} if the
+            process did not exit due to a signal.
+        @type signal: L{int} or L{None}
+
+        @param status: The exit code of the process.  This is a platform
+            specific combination of the exit code and the exit signal.  See
+            L{os.WIFEXITED} and related functions.
+        @type status: L{int}
+        """
+        self.exitCode = exitCode
+        self.signal = signal
+        self.status = status
+        s = "process ended"
+        if exitCode is not None:
+            s = s + " with exit code %s" % exitCode
+        if signal is not None:
+            s = s + " by signal %s" % signal
+        Exception.__init__(self, s)
+
+
+class ProcessExitedAlready(Exception):
+    """
+    The process has already exited and the operation requested can no longer
+    be performed.
+    """
+
+
+class NotConnectingError(RuntimeError):
+    __doc__ = (
+        MESSAGE
+    ) = "The Connector was not connecting when it was asked to stop connecting"
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.args:
+            s = "{}: {}".format(s, " ".join(self.args))
+        s = "%s." % s
+        return s
+
+
+class NotListeningError(RuntimeError):
+    __doc__ = MESSAGE = "The Port was not listening when it was asked to stop listening"
+
+    def __str__(self) -> str:
+        s = self.MESSAGE
+        if self.args:
+            s = "{}: {}".format(s, " ".join(self.args))
+        s = "%s." % s
+        return s
+
+
+class ReactorNotRunning(RuntimeError):
+    """
+    Error raised when trying to stop a reactor which is not running.
+    """
+
+
+class ReactorNotRestartable(RuntimeError):
+    """
+    Error raised when trying to run a reactor which was stopped.
+    """
+
+
+class ReactorAlreadyRunning(RuntimeError):
+    """
+    Error raised when trying to start the reactor multiple times.
+    """
+
+
+class ReactorAlreadyInstalledError(AssertionError):
+    """
+    Could not install reactor because one is already installed.
+    """
+
+
+class ConnectingCancelledError(Exception):
+    """
+    An C{Exception} that will be raised when an L{IStreamClientEndpoint} is
+    cancelled before it connects.
+
+    @ivar address: The L{IAddress} that is the destination of the
+        cancelled L{IStreamClientEndpoint}.
+    """
+
+    def __init__(self, address):
+        """
+        @param address: The L{IAddress} that is the destination of the
+            L{IStreamClientEndpoint} that was cancelled.
+        """
+        Exception.__init__(self, address)
+        self.address = address
+
+
+class NoProtocol(Exception):
+    """
+    An C{Exception} that will be raised when the factory given to a
+    L{IStreamClientEndpoint} returns L{None} from C{buildProtocol}.
+    """
+
+
+class UnsupportedAddressFamily(Exception):
+    """
+    An attempt was made to use a socket with an address family (eg I{AF_INET},
+    I{AF_INET6}, etc) which is not supported by the reactor.
+    """
+
+
+class UnsupportedSocketType(Exception):
+    """
+    An attempt was made to use a socket of a type (eg I{SOCK_STREAM},
+    I{SOCK_DGRAM}, etc) which is not supported by the reactor.
+    """
+
+
+class AlreadyListened(Exception):
+    """
+    An attempt was made to listen on a file descriptor which can only be
+    listened on once.
+    """
+
+
+class InvalidAddressError(ValueError):
+    """
+    An invalid address was specified (i.e. neither IPv4 or IPv6, or expected
+    one and got the other).
+
+    @ivar address: See L{__init__}
+    @ivar message: See L{__init__}
+    """
+
+    def __init__(self, address, message):
+        """
+        @param address: The address that was provided.
+        @type address: L{bytes}
+        @param message: A native string of additional information provided by
+            the calling context.
+        @type address: L{str}
+        """
+        self.address = address
+        self.message = message
+
+
+__all__ = [
+    "BindError",
+    "CannotListenError",
+    "MulticastJoinError",
+    "MessageLengthError",
+    "DNSLookupError",
+    "ConnectInProgressError",
+    "ConnectError",
+    "ConnectBindError",
+    "UnknownHostError",
+    "NoRouteError",
+    "ConnectionRefusedError",
+    "TCPTimedOutError",
+    "BadFileError",
+    "ServiceNameUnknownError",
+    "UserError",
+    "TimeoutError",
+    "SSLError",
+    "VerifyError",
+    "PeerVerifyError",
+    "CertificateError",
+    "getConnectError",
+    "ConnectionClosed",
+    "ConnectionLost",
+    "ConnectionDone",
+    "ConnectionFdescWentAway",
+    "AlreadyCalled",
+    "AlreadyCancelled",
+    "PotentialZombieWarning",
+    "ProcessDone",
+    "ProcessTerminated",
+    "ProcessExitedAlready",
+    "NotConnectingError",
+    "NotListeningError",
+    "ReactorNotRunning",
+    "ReactorAlreadyRunning",
+    "ReactorAlreadyInstalledError",
+    "ConnectingCancelledError",
+    "UnsupportedAddressFamily",
+    "UnsupportedSocketType",
+    "InvalidAddressError",
+]
diff --git a/contrib/python/Twisted/py3/twisted/internet/fdesc.py b/contrib/python/Twisted/py3/twisted/internet/fdesc.py
new file mode 100644
index 00000000000..b95755222dd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/fdesc.py
@@ -0,0 +1,121 @@
+# -*- test-case-name: twisted.test.test_fdesc -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Utility functions for dealing with POSIX file descriptors.
+"""
+
+import errno
+import os
+
+try:
+    import fcntl as _fcntl
+except ImportError:
+    fcntl = None
+else:
+    fcntl = _fcntl
+
+# twisted imports
+from twisted.internet.main import CONNECTION_DONE, CONNECTION_LOST
+
+
+def setNonBlocking(fd):
+    """
+    Set the file description of the given file descriptor to non-blocking.
+    """
+    flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+    flags = flags | os.O_NONBLOCK
+    fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+
+def setBlocking(fd):
+    """
+    Set the file description of the given file descriptor to blocking.
+    """
+    flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+    flags = flags & ~os.O_NONBLOCK
+    fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+
+if fcntl is None:
+    # fcntl isn't available on Windows.  By default, handles aren't
+    # inherited on Windows, so we can do nothing here.
+    _setCloseOnExec = _unsetCloseOnExec = lambda fd: None
+else:
+
+    def _setCloseOnExec(fd):
+        """
+        Make a file descriptor close-on-exec.
+        """
+        flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+        flags = flags | fcntl.FD_CLOEXEC
+        fcntl.fcntl(fd, fcntl.F_SETFD, flags)
+
+    def _unsetCloseOnExec(fd):
+        """
+        Make a file descriptor close-on-exec.
+        """
+        flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+        flags = flags & ~fcntl.FD_CLOEXEC
+        fcntl.fcntl(fd, fcntl.F_SETFD, flags)
+
+
+def readFromFD(fd, callback):
+    """
+    Read from file descriptor, calling callback with resulting data.
+
+    If successful, call 'callback' with a single argument: the
+    resulting data.
+
+    Returns same thing FileDescriptor.doRead would: CONNECTION_LOST,
+    CONNECTION_DONE, or None.
+
+    @type fd: C{int}
+    @param fd: non-blocking file descriptor to be read from.
+    @param callback: a callable which accepts a single argument. If
+    data is read from the file descriptor it will be called with this
+    data. Handling exceptions from calling the callback is up to the
+    caller.
+
+    Note that if the descriptor is still connected but no data is read,
+    None will be returned but callback will not be called.
+
+    @return: CONNECTION_LOST on error, CONNECTION_DONE when fd is
+    closed, otherwise None.
+    """
+    try:
+        output = os.read(fd, 8192)
+    except OSError as ioe:
+        if ioe.args[0] in (errno.EAGAIN, errno.EINTR):
+            return
+        else:
+            return CONNECTION_LOST
+    if not output:
+        return CONNECTION_DONE
+    callback(output)
+
+
+def writeToFD(fd, data):
+    """
+    Write data to file descriptor.
+
+    Returns same thing FileDescriptor.writeSomeData would.
+
+    @type fd: C{int}
+    @param fd: non-blocking file descriptor to be written to.
+    @type data: C{str} or C{buffer}
+    @param data: bytes to write to fd.
+
+    @return: number of bytes written, or CONNECTION_LOST.
+    """
+    try:
+        return os.write(fd, data)
+    except OSError as io:
+        if io.errno in (errno.EAGAIN, errno.EINTR):
+            return 0
+        return CONNECTION_LOST
+
+
+__all__ = ["setNonBlocking", "setBlocking", "readFromFD", "writeToFD"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/gireactor.py b/contrib/python/Twisted/py3/twisted/internet/gireactor.py
new file mode 100644
index 00000000000..e9c072f41a6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/gireactor.py
@@ -0,0 +1,122 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module provides support for Twisted to interact with the glib
+mainloop via GObject Introspection.
+
+In order to use this support, simply do the following::
+
+    from twisted.internet import gireactor
+    gireactor.install()
+
+If you wish to use a GApplication, register it with the reactor::
+
+    from twisted.internet import reactor
+    reactor.registerGApplication(app)
+
+Then use twisted.internet APIs as usual.
+
+On Python 3, pygobject v3.4 or later is required.
+"""
+
+
+from typing import Union
+
+from gi.repository import GLib  # type:ignore[import]
+
+from twisted.internet import _glibbase
+from twisted.internet.error import ReactorAlreadyRunning
+from twisted.python import runtime
+
+if getattr(GLib, "threads_init", None) is not None:
+    GLib.threads_init()
+
+
+class GIReactor(_glibbase.GlibReactorBase):
+    """
+    GObject-introspection event loop reactor.
+
+    @ivar _gapplication: A C{Gio.Application} instance that was registered
+        with C{registerGApplication}.
+    """
+
+    # By default no Application is registered:
+    _gapplication = None
+
+    def __init__(self, useGtk=False):
+        _glibbase.GlibReactorBase.__init__(self, GLib, None)
+
+    def registerGApplication(self, app):
+        """
+        Register a C{Gio.Application} or C{Gtk.Application}, whose main loop
+        will be used instead of the default one.
+
+        We will C{hold} the application so it doesn't exit on its own. In
+        versions of C{python-gi} 3.2 and later, we exit the event loop using
+        the C{app.quit} method which overrides any holds. Older versions are
+        not supported.
+        """
+        if self._gapplication is not None:
+            raise RuntimeError("Can't register more than one application instance.")
+        if self._started:
+            raise ReactorAlreadyRunning(
+                "Can't register application after reactor was started."
+            )
+        if not hasattr(app, "quit"):
+            raise RuntimeError(
+                "Application registration is not supported in"
+                " versions of PyGObject prior to 3.2."
+            )
+        self._gapplication = app
+
+        def run():
+            app.hold()
+            app.run(None)
+
+        self._run = run
+
+        self._crash = app.quit
+
+
+class PortableGIReactor(_glibbase.GlibReactorBase):
+    """
+    Portable GObject Introspection event loop reactor.
+    """
+
+    def __init__(self, useGtk=False):
+        super().__init__(GLib, None, useGtk=useGtk)
+
+    def registerGApplication(self, app):
+        """
+        Register a C{Gio.Application} or C{Gtk.Application}, whose main loop
+        will be used instead of the default one.
+        """
+        raise NotImplementedError("GApplication is not currently supported on Windows.")
+
+    def simulate(self) -> None:
+        """
+        For compatibility only. Do nothing.
+        """
+
+
+def install(useGtk: bool = False) -> Union[GIReactor, PortableGIReactor]:
+    """
+    Configure the twisted mainloop to be run inside the glib mainloop.
+
+    @param useGtk: A hint that the Gtk GUI will or will not be used.  Currently
+        does not modify any behavior.
+    """
+    reactor: Union[GIReactor, PortableGIReactor]
+    if runtime.platform.getType() == "posix":
+        reactor = GIReactor(useGtk=useGtk)
+    else:
+        reactor = PortableGIReactor(useGtk=useGtk)
+
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
+    return reactor
+
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/glib2reactor.py b/contrib/python/Twisted/py3/twisted/internet/glib2reactor.py
new file mode 100644
index 00000000000..9a11bec02ad
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/glib2reactor.py
@@ -0,0 +1,50 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module provides support for Twisted to interact with the glib mainloop.
+This is like gtk2, but slightly faster and does not require a working
+$DISPLAY. However, you cannot run GUIs under this reactor: for that you must
+use the gtk2reactor instead.
+
+In order to use this support, simply do the following::
+
+    from twisted.internet import glib2reactor
+    glib2reactor.install()
+
+Then use twisted.internet APIs as usual.  The other methods here are not
+intended to be called directly.
+"""
+
+from incremental import Version
+
+from ._deprecate import deprecatedGnomeReactor
+
+deprecatedGnomeReactor("glib2reactor", Version("Twisted", 23, 8, 0))
+
+from twisted.internet import gtk2reactor
+
+
+class Glib2Reactor(gtk2reactor.Gtk2Reactor):
+    """
+    The reactor using the glib mainloop.
+    """
+
+    def __init__(self):
+        """
+        Override init to set the C{useGtk} flag.
+        """
+        gtk2reactor.Gtk2Reactor.__init__(self, useGtk=False)
+
+
+def install():
+    """
+    Configure the twisted mainloop to be run inside the glib mainloop.
+    """
+    reactor = Glib2Reactor()
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
+
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/gtk2reactor.py b/contrib/python/Twisted/py3/twisted/internet/gtk2reactor.py
new file mode 100644
index 00000000000..b4e0c4c4e1c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/gtk2reactor.py
@@ -0,0 +1,119 @@
+# -*- test-case-name: twisted.internet.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+This module provides support for Twisted to interact with the glib/gtk2
+mainloop.
+
+In order to use this support, simply do the following::
+
+    from twisted.internet import gtk2reactor
+    gtk2reactor.install()
+
+Then use twisted.internet APIs as usual.  The other methods here are not
+intended to be called directly.
+"""
+
+from incremental import Version
+
+from ._deprecate import deprecatedGnomeReactor
+
+deprecatedGnomeReactor("gtk2reactor", Version("Twisted", 23, 8, 0))
+
+# System Imports
+import sys
+
+# Twisted Imports
+from twisted.internet import _glibbase
+from twisted.python import runtime
+
+# Certain old versions of pygtk and gi crash if imported at the same
+# time. This is a problem when running Twisted's unit tests, since they will
+# attempt to run both gtk2 and gtk3/gi tests. However, gireactor makes sure
+# that if we are in such an old version, and gireactor was imported,
+# gtk2reactor will not be importable. So we don't *need* to enforce that here
+# as well; whichever is imported first will still win. Moreover, additional
+# enforcement in this module is unnecessary in modern versions, and downright
+# problematic in certain versions where for some reason importing gtk also
+# imports some subset of gi. So we do nothing here, relying on gireactor to
+# prevent the crash.
+
+try:
+    if not hasattr(sys, "frozen"):
+        # Don't want to check this for py2exe
+        import pygtk  # type: ignore[import]
+
+        pygtk.require("2.0")
+except (ImportError, AttributeError):
+    pass  # maybe we're using pygtk before this hack existed.
+
+import gobject  # type: ignore[import]
+
+if not hasattr(gobject, "IO_HUP"):
+    # gi.repository's legacy compatibility helper raises an AttributeError with
+    # a custom error message rather than a useful ImportError, so things tend
+    # to fail loudly.  Things that import this module expect an ImportError if,
+    # well, something failed to import, and treat an AttributeError as an
+    # arbitrary application code failure, so we satisfy that expectation here.
+    raise ImportError("pygobject 2.x is not installed. Use the `gi` reactor.")
+
+if hasattr(gobject, "threads_init"):
+    # recent versions of python-gtk expose this. python-gtk=2.4.1
+    # (wrapping glib-2.4.7) does. python-gtk=2.0.0 (wrapping
+    # glib-2.2.3) does not.
+    gobject.threads_init()
+
+
+class Gtk2Reactor(_glibbase.GlibReactorBase):
+    """
+    PyGTK+ 2 event loop reactor.
+    """
+
+    def __init__(self, useGtk=True):
+        _gtk = None
+        if useGtk is True:
+            import gtk as _gtk  # type: ignore[import]
+
+        _glibbase.GlibReactorBase.__init__(self, gobject, _gtk, useGtk=useGtk)
+
+
+# We don't bother deprecating the PortableGtkReactor.
+# The original code was removed and replaced with the
+# backward compatible generic GTK reactor.
+PortableGtkReactor = Gtk2Reactor
+
+
+def install(useGtk=True):
+    """
+    Configure the twisted mainloop to be run inside the gtk mainloop.
+
+    @param useGtk: should glib rather than GTK+ event loop be
+        used (this will be slightly faster but does not support GUI).
+    """
+    reactor = Gtk2Reactor(useGtk)
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
+    return reactor
+
+
+def portableInstall(useGtk=True):
+    """
+    Configure the twisted mainloop to be run inside the gtk mainloop.
+    """
+    reactor = PortableGtkReactor()
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
+    return reactor
+
+
+if runtime.platform.getType() == "posix":
+    install = install
+else:
+    install = portableInstall
+
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/gtk3reactor.py b/contrib/python/Twisted/py3/twisted/internet/gtk3reactor.py
new file mode 100644
index 00000000000..a2c60f9feca
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/gtk3reactor.py
@@ -0,0 +1,22 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module is a legacy compatibility alias for L{twisted.internet.gireactor}.
+See that module instead.
+"""
+
+from incremental import Version
+
+from ._deprecate import deprecatedGnomeReactor
+
+deprecatedGnomeReactor("gtk3reactor", Version("Twisted", 23, 8, 0))
+
+from twisted.internet import gireactor
+
+Gtk3Reactor = gireactor.GIReactor
+PortableGtk3Reactor = gireactor.PortableGIReactor
+
+install = gireactor.install
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/inotify.py b/contrib/python/Twisted/py3/twisted/internet/inotify.py
new file mode 100644
index 00000000000..0fd8fd681cc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/inotify.py
@@ -0,0 +1,426 @@
+# -*- test-case-name: twisted.internet.test.test_inotify -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module provides support for Twisted to linux inotify API.
+
+In order to use this support, simply do the following (and start a reactor
+at some point)::
+
+    from twisted.internet import inotify
+    from twisted.python import filepath
+
+    def notify(ignored, filepath, mask):
+        \"""
+        For historical reasons, an opaque handle is passed as first
+        parameter. This object should never be used.
+
+        @param filepath: FilePath on which the event happened.
+        @param mask: inotify event as hexadecimal masks
+        \"""
+        print("event %s on %s" % (
+            ', '.join(inotify.humanReadableMask(mask)), filepath))
+
+    notifier = inotify.INotify()
+    notifier.startReading()
+    notifier.watch(filepath.FilePath("/some/directory"), callbacks=[notify])
+    notifier.watch(filepath.FilePath(b"/some/directory2"), callbacks=[notify])
+
+Note that in the above example, a L{FilePath} which is a L{bytes} path name
+or L{str} path name may be used.  However, no matter what type of
+L{FilePath} is passed to this module, internally the L{FilePath} is
+converted to L{bytes} according to L{sys.getfilesystemencoding}.
+For any L{FilePath} returned by this module, the caller is responsible for
+converting from a L{bytes} path name to a L{str} path name.
+
+@since: 10.1
+"""
+
+
+import os
+import struct
+
+from twisted.internet import fdesc
+from twisted.internet.abstract import FileDescriptor
+from twisted.python import _inotify, log
+
+# from /usr/src/linux/include/linux/inotify.h
+
+IN_ACCESS = 0x00000001  # File was accessed
+IN_MODIFY = 0x00000002  # File was modified
+IN_ATTRIB = 0x00000004  # Metadata changed
+IN_CLOSE_WRITE = 0x00000008  # Writeable file was closed
+IN_CLOSE_NOWRITE = 0x00000010  # Unwriteable file closed
+IN_OPEN = 0x00000020  # File was opened
+IN_MOVED_FROM = 0x00000040  # File was moved from X
+IN_MOVED_TO = 0x00000080  # File was moved to Y
+IN_CREATE = 0x00000100  # Subfile was created
+IN_DELETE = 0x00000200  # Subfile was delete
+IN_DELETE_SELF = 0x00000400  # Self was deleted
+IN_MOVE_SELF = 0x00000800  # Self was moved
+IN_UNMOUNT = 0x00002000  # Backing fs was unmounted
+IN_Q_OVERFLOW = 0x00004000  # Event queued overflowed
+IN_IGNORED = 0x00008000  # File was ignored
+
+IN_ONLYDIR = 0x01000000  # only watch the path if it is a directory
+IN_DONT_FOLLOW = 0x02000000  # don't follow a sym link
+IN_MASK_ADD = 0x20000000  # add to the mask of an already existing watch
+IN_ISDIR = 0x40000000  # event occurred against dir
+IN_ONESHOT = 0x80000000  # only send event once
+
+IN_CLOSE = IN_CLOSE_WRITE | IN_CLOSE_NOWRITE  # closes
+IN_MOVED = IN_MOVED_FROM | IN_MOVED_TO  # moves
+IN_CHANGED = IN_MODIFY | IN_ATTRIB  # changes
+
+IN_WATCH_MASK = (
+    IN_MODIFY
+    | IN_ATTRIB
+    | IN_CREATE
+    | IN_DELETE
+    | IN_DELETE_SELF
+    | IN_MOVE_SELF
+    | IN_UNMOUNT
+    | IN_MOVED_FROM
+    | IN_MOVED_TO
+)
+
+
+_FLAG_TO_HUMAN = [
+    (IN_ACCESS, "access"),
+    (IN_MODIFY, "modify"),
+    (IN_ATTRIB, "attrib"),
+    (IN_CLOSE_WRITE, "close_write"),
+    (IN_CLOSE_NOWRITE, "close_nowrite"),
+    (IN_OPEN, "open"),
+    (IN_MOVED_FROM, "moved_from"),
+    (IN_MOVED_TO, "moved_to"),
+    (IN_CREATE, "create"),
+    (IN_DELETE, "delete"),
+    (IN_DELETE_SELF, "delete_self"),
+    (IN_MOVE_SELF, "move_self"),
+    (IN_UNMOUNT, "unmount"),
+    (IN_Q_OVERFLOW, "queue_overflow"),
+    (IN_IGNORED, "ignored"),
+    (IN_ONLYDIR, "only_dir"),
+    (IN_DONT_FOLLOW, "dont_follow"),
+    (IN_MASK_ADD, "mask_add"),
+    (IN_ISDIR, "is_dir"),
+    (IN_ONESHOT, "one_shot"),
+]
+
+
+def humanReadableMask(mask):
+    """
+    Auxiliary function that converts a hexadecimal mask into a series
+    of human readable flags.
+    """
+    s = []
+    for k, v in _FLAG_TO_HUMAN:
+        if k & mask:
+            s.append(v)
+    return s
+
+
+class _Watch:
+    """
+    Watch object that represents a Watch point in the filesystem. The
+    user should let INotify to create these objects
+
+    @ivar path: The path over which this watch point is monitoring
+    @ivar mask: The events monitored by this watchpoint
+    @ivar autoAdd: Flag that determines whether this watch point
+        should automatically add created subdirectories
+    @ivar callbacks: L{list} of callback functions that will be called
+        when an event occurs on this watch.
+    """
+
+    def __init__(self, path, mask=IN_WATCH_MASK, autoAdd=False, callbacks=None):
+        self.path = path.asBytesMode()
+        self.mask = mask
+        self.autoAdd = autoAdd
+        if callbacks is None:
+            callbacks = []
+        self.callbacks = callbacks
+
+    def _notify(self, filepath, events):
+        """
+        Callback function used by L{INotify} to dispatch an event.
+        """
+        filepath = filepath.asBytesMode()
+        for callback in self.callbacks:
+            callback(self, filepath, events)
+
+
+class INotify(FileDescriptor):
+    """
+    The INotify file descriptor, it basically does everything related
+    to INotify, from reading to notifying watch points.
+
+    @ivar _buffer: a L{bytes} containing the data read from the inotify fd.
+
+    @ivar _watchpoints: a L{dict} that maps from inotify watch ids to
+        watchpoints objects
+
+    @ivar _watchpaths: a L{dict} that maps from watched paths to the
+        inotify watch ids
+    """
+
+    _inotify = _inotify
+
+    def __init__(self, reactor=None):
+        FileDescriptor.__init__(self, reactor=reactor)
+
+        # Smart way to allow parametrization of libc so I can override
+        # it and test for the system errors.
+        self._fd = self._inotify.init()
+
+        fdesc.setNonBlocking(self._fd)
+        fdesc._setCloseOnExec(self._fd)
+
+        # The next 2 lines are needed to have self.loseConnection()
+        # to call connectionLost() on us. Since we already created the
+        # fd that talks to inotify we want to be notified even if we
+        # haven't yet started reading.
+        self.connected = 1
+        self._writeDisconnected = True
+
+        self._buffer = b""
+        self._watchpoints = {}
+        self._watchpaths = {}
+
+    def _addWatch(self, path, mask, autoAdd, callbacks):
+        """
+        Private helper that abstracts the use of ctypes.
+
+        Calls the internal inotify API and checks for any errors after the
+        call. If there's an error L{INotify._addWatch} can raise an
+        INotifyError. If there's no error it proceeds creating a watchpoint and
+        adding a watchpath for inverse lookup of the file descriptor from the
+        path.
+        """
+        path = path.asBytesMode()
+        wd = self._inotify.add(self._fd, path, mask)
+
+        iwp = _Watch(path, mask, autoAdd, callbacks)
+
+        self._watchpoints[wd] = iwp
+        self._watchpaths[path] = wd
+
+        return wd
+
+    def _rmWatch(self, wd):
+        """
+        Private helper that abstracts the use of ctypes.
+
+        Calls the internal inotify API to remove an fd from inotify then
+        removes the corresponding watchpoint from the internal mapping together
+        with the file descriptor from the watchpath.
+        """
+        self._inotify.remove(self._fd, wd)
+        iwp = self._watchpoints.pop(wd)
+        self._watchpaths.pop(iwp.path)
+
+    def connectionLost(self, reason):
+        """
+        Release the inotify file descriptor and do the necessary cleanup
+        """
+        FileDescriptor.connectionLost(self, reason)
+        if self._fd >= 0:
+            try:
+                os.close(self._fd)
+            except OSError as e:
+                log.err(e, "Couldn't close INotify file descriptor.")
+
+    def fileno(self):
+        """
+        Get the underlying file descriptor from this inotify observer.
+        Required by L{abstract.FileDescriptor} subclasses.
+        """
+        return self._fd
+
+    def doRead(self):
+        """
+        Read some data from the observed file descriptors
+        """
+        fdesc.readFromFD(self._fd, self._doRead)
+
+    def _doRead(self, in_):
+        """
+        Work on the data just read from the file descriptor.
+        """
+        self._buffer += in_
+        while len(self._buffer) >= 16:
+            wd, mask, cookie, size = struct.unpack("=LLLL", self._buffer[0:16])
+
+            if size:
+                name = self._buffer[16 : 16 + size].rstrip(b"\0")
+            else:
+                name = None
+
+            self._buffer = self._buffer[16 + size :]
+
+            try:
+                iwp = self._watchpoints[wd]
+            except KeyError:
+                continue
+
+            path = iwp.path.asBytesMode()
+            if name:
+                path = path.child(name)
+            iwp._notify(path, mask)
+
+            if iwp.autoAdd and mask & IN_ISDIR and mask & IN_CREATE:
+                # mask & IN_ISDIR already guarantees that the path is a
+                # directory. There's no way you can get here without a
+                # directory anyway, so no point in checking for that again.
+                new_wd = self.watch(
+                    path, mask=iwp.mask, autoAdd=True, callbacks=iwp.callbacks
+                )
+                # This is very very very hacky and I'd rather not do this but
+                # we have no other alternative that is less hacky other than
+                # surrender.  We use callLater because we don't want to have
+                # too many events waiting while we process these subdirs, we
+                # must always answer events as fast as possible or the overflow
+                # might come.
+                self.reactor.callLater(0, self._addChildren, self._watchpoints[new_wd])
+            if mask & IN_DELETE_SELF:
+                self._rmWatch(wd)
+                self.loseConnection()
+
+    def _addChildren(self, iwp):
+        """
+        This is a very private method, please don't even think about using it.
+
+        Note that this is a fricking hack... it's because we cannot be fast
+        enough in adding a watch to a directory and so we basically end up
+        getting here too late if some operations have already been going on in
+        the subdir, we basically need to catchup.  This eventually ends up
+        meaning that we generate double events, your app must be resistant.
+        """
+        try:
+            listdir = iwp.path.children()
+        except OSError:
+            # Somebody or something (like a test) removed this directory while
+            # we were in the callLater(0...) waiting. It doesn't make sense to
+            # process it anymore
+            return
+
+        # note that it's true that listdir will only see the subdirs inside
+        # path at the moment of the call but path is monitored already so if
+        # something is created we will receive an event.
+        for f in listdir:
+            # It's a directory, watch it and then add its children
+            if f.isdir():
+                wd = self.watch(f, mask=iwp.mask, autoAdd=True, callbacks=iwp.callbacks)
+                iwp._notify(f, IN_ISDIR | IN_CREATE)
+                # now f is watched, we can add its children the callLater is to
+                # avoid recursion
+                self.reactor.callLater(0, self._addChildren, self._watchpoints[wd])
+
+            # It's a file and we notify it.
+            if f.isfile():
+                iwp._notify(f, IN_CREATE | IN_CLOSE_WRITE)
+
+    def watch(
+        self, path, mask=IN_WATCH_MASK, autoAdd=False, callbacks=None, recursive=False
+    ):
+        """
+        Watch the 'mask' events in given path. Can raise C{INotifyError} when
+        there's a problem while adding a directory.
+
+        @param path: The path needing monitoring
+        @type path: L{FilePath}
+
+        @param mask: The events that should be watched
+        @type mask: L{int}
+
+        @param autoAdd: if True automatically add newly created
+                        subdirectories
+        @type autoAdd: L{bool}
+
+        @param callbacks: A list of callbacks that should be called
+                          when an event happens in the given path.
+                          The callback should accept 3 arguments:
+                          (ignored, filepath, mask)
+        @type callbacks: L{list} of callables
+
+        @param recursive: Also add all the subdirectories in this path
+        @type recursive: L{bool}
+        """
+        if recursive:
+            # This behavior is needed to be compatible with the windows
+            # interface for filesystem changes:
+            # http://msdn.microsoft.com/en-us/library/aa365465(VS.85).aspx
+            # ReadDirectoryChangesW can do bWatchSubtree so it doesn't
+            # make sense to implement this at a higher abstraction
+            # level when other platforms support it already
+            for child in path.walk():
+                if child.isdir():
+                    self.watch(child, mask, autoAdd, callbacks, recursive=False)
+        else:
+            wd = self._isWatched(path)
+            if wd:
+                return wd
+
+            mask = mask | IN_DELETE_SELF  # need this to remove the watch
+
+            return self._addWatch(path, mask, autoAdd, callbacks)
+
+    def ignore(self, path):
+        """
+        Remove the watch point monitoring the given path
+
+        @param path: The path that should be ignored
+        @type path: L{FilePath}
+        """
+        path = path.asBytesMode()
+        wd = self._isWatched(path)
+        if wd is None:
+            raise KeyError(f"{path!r} is not watched")
+        else:
+            self._rmWatch(wd)
+
+    def _isWatched(self, path):
+        """
+        Helper function that checks if the path is already monitored
+        and returns its watchdescriptor if so or None otherwise.
+
+        @param path: The path that should be checked
+        @type path: L{FilePath}
+        """
+        path = path.asBytesMode()
+        return self._watchpaths.get(path, None)
+
+
+INotifyError = _inotify.INotifyError
+
+
+__all__ = [
+    "INotify",
+    "humanReadableMask",
+    "IN_WATCH_MASK",
+    "IN_ACCESS",
+    "IN_MODIFY",
+    "IN_ATTRIB",
+    "IN_CLOSE_NOWRITE",
+    "IN_CLOSE_WRITE",
+    "IN_OPEN",
+    "IN_MOVED_FROM",
+    "IN_MOVED_TO",
+    "IN_CREATE",
+    "IN_DELETE",
+    "IN_DELETE_SELF",
+    "IN_MOVE_SELF",
+    "IN_UNMOUNT",
+    "IN_Q_OVERFLOW",
+    "IN_IGNORED",
+    "IN_ONLYDIR",
+    "IN_DONT_FOLLOW",
+    "IN_MASK_ADD",
+    "IN_ISDIR",
+    "IN_ONESHOT",
+    "IN_CLOSE",
+    "IN_MOVED",
+    "IN_CHANGED",
+]
diff --git a/contrib/python/Twisted/py3/twisted/internet/interfaces.py b/contrib/python/Twisted/py3/twisted/internet/interfaces.py
new file mode 100644
index 00000000000..78380ccc397
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/interfaces.py
@@ -0,0 +1,2756 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Interface documentation.
+
+Maintainer: Itamar Shtull-Trauring
+"""
+from __future__ import annotations
+
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AnyStr,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+)
+
+from zope.interface import Attribute, Interface
+
+from twisted.python.failure import Failure
+
+if TYPE_CHECKING:
+    from socket import AddressFamily
+
+    try:
+        from OpenSSL.SSL import (
+            Connection as OpenSSLConnection,
+            Context as OpenSSLContext,
+        )
+    except ImportError:
+        OpenSSLConnection = OpenSSLContext = object  # type: ignore[misc,assignment]
+
+    from twisted.internet.abstract import FileDescriptor
+    from twisted.internet.address import IPv4Address, IPv6Address, UNIXAddress
+    from twisted.internet.defer import Deferred
+    from twisted.internet.protocol import (
+        ClientFactory,
+        ConnectedDatagramProtocol,
+        DatagramProtocol,
+        Factory,
+        ServerFactory,
+    )
+    from twisted.internet.ssl import ClientContextFactory
+    from twisted.names.dns import Query, RRHeader
+    from twisted.protocols.tls import TLSMemoryBIOProtocol
+    from twisted.python.runtime import platform
+
+    if platform.supportsThreads():
+        from twisted.python.threadpool import ThreadPool
+    else:
+        ThreadPool = object  # type: ignore[misc, assignment]
+
+
+class IAddress(Interface):
+    """
+    An address, e.g. a TCP C{(host, port)}.
+
+    Default implementations are in L{twisted.internet.address}.
+    """
+
+
+### Reactor Interfaces
+
+
+class IConnector(Interface):
+    """
+    Object used to interface between connections and protocols.
+
+    Each L{IConnector} manages one connection.
+    """
+
+    def stopConnecting() -> None:
+        """
+        Stop attempting to connect.
+        """
+
+    def disconnect() -> None:
+        """
+        Disconnect regardless of the connection state.
+
+        If we are connected, disconnect, if we are trying to connect,
+        stop trying.
+        """
+
+    def connect() -> None:
+        """
+        Try to connect to remote address.
+        """
+
+    def getDestination() -> IAddress:
+        """
+        Return destination this will try to connect to.
+
+        @return: An object which provides L{IAddress}.
+        """
+
+
+class IResolverSimple(Interface):
+    def getHostByName(name: str, timeout: Sequence[int] = ()) -> "Deferred[str]":
+        """
+        Resolve the domain name C{name} into an IP address.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: The callback of the Deferred that is returned will be
+            passed a string that represents the IP address of the
+            specified name, or the errback will be called if the
+            lookup times out.  If multiple types of address records
+            are associated with the name, A6 records will be returned
+            in preference to AAAA records, which will be returned in
+            preference to A records.  If there are multiple records of
+            the type to be returned, one will be selected at random.
+
+        @raise twisted.internet.defer.TimeoutError: Raised
+            (asynchronously) if the name cannot be resolved within the
+            specified timeout period.
+        """
+
+
+class IHostResolution(Interface):
+    """
+    An L{IHostResolution} represents represents an in-progress recursive query
+    for a DNS name.
+
+    @since: Twisted 17.1.0
+    """
+
+    name = Attribute(
+        """
+        L{unicode}; the name of the host being resolved.
+        """
+    )
+
+    def cancel() -> None:
+        """
+        Stop the hostname resolution in progress.
+        """
+
+
+class IResolutionReceiver(Interface):
+    """
+    An L{IResolutionReceiver} receives the results of a hostname resolution in
+    progress, initiated by an L{IHostnameResolver}.
+
+    @since: Twisted 17.1.0
+    """
+
+    def resolutionBegan(resolutionInProgress: IHostResolution) -> None:
+        """
+        A hostname resolution began.
+
+        @param resolutionInProgress: an L{IHostResolution}.
+        """
+
+    def addressResolved(address: IAddress) -> None:
+        """
+        An internet address.  This is called when an address for the given name
+        is discovered.  In the current implementation this practically means
+        L{IPv4Address} or L{IPv6Address}, but implementations of this interface
+        should be lenient to other types being passed to this interface as
+        well, for future-proofing.
+
+        @param address: An address object.
+        """
+
+    def resolutionComplete() -> None:
+        """
+        Resolution has completed; no further addresses will be relayed to
+        L{IResolutionReceiver.addressResolved}.
+        """
+
+
+class IHostnameResolver(Interface):
+    """
+    An L{IHostnameResolver} can resolve a host name and port number into a
+    series of L{IAddress} objects.
+
+    @since: Twisted 17.1.0
+    """
+
+    def resolveHostName(
+        resolutionReceiver: IResolutionReceiver,
+        hostName: str,
+        portNumber: int = 0,
+        addressTypes: Optional[Sequence[Type[IAddress]]] = None,
+        transportSemantics: str = "TCP",
+    ) -> IHostResolution:
+        """
+        Initiate a hostname resolution.
+
+        @param resolutionReceiver: an object that will receive each resolved
+            address as it arrives.
+        @param hostName: The name of the host to resolve.  If this contains
+            non-ASCII code points, they will be converted to IDNA first.
+        @param portNumber: The port number that the returned addresses should
+            include.
+        @param addressTypes: An iterable of implementors of L{IAddress} that
+            are acceptable values for C{resolutionReceiver} to receive to its
+            L{addressResolved <IResolutionReceiver.addressResolved>}.  In
+            practice, this means an iterable containing
+            L{twisted.internet.address.IPv4Address},
+            L{twisted.internet.address.IPv6Address}, both, or neither.
+        @param transportSemantics: A string describing the semantics of the
+            transport; either C{'TCP'} for stream-oriented transports or
+            C{'UDP'} for datagram-oriented; see
+            L{twisted.internet.address.IPv6Address.type} and
+            L{twisted.internet.address.IPv4Address.type}.
+
+        @return: The resolution in progress.
+        """
+
+
+class IResolver(IResolverSimple):
+    def query(
+        query: "Query", timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Dispatch C{query} to the method which can handle its type.
+
+        @param query: The DNS query being issued, to which a response is to be
+            generated.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupAddress(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an A record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupAddress6(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an A6 record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupIPV6Address(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an AAAA record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupMailExchange(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an MX record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupNameservers(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an NS record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupCanonicalName(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform a CNAME record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupMailBox(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an MB record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupMailGroup(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an MG record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupMailRename(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an MR record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupPointer(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform a PTR record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupAuthority(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an SOA record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupNull(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform a NULL record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupWellKnownServices(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform a WKS record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupHostInfo(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform a HINFO record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupMailboxInfo(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an MINFO record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupText(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform a TXT record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupResponsibility(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an RP record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupAFSDatabase(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an AFSDB record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupService(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an SRV record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupAllRecords(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an ALL_RECORD lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupSenderPolicy(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform a SPF record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupNamingAuthorityPointer(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform a NAPTR record lookup.
+
+        @param name: DNS name to resolve.
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+    def lookupZone(
+        name: str, timeout: Sequence[int]
+    ) -> "Deferred[Tuple[RRHeader, RRHeader, RRHeader]]":
+        """
+        Perform an AXFR record lookup.
+
+        NB This is quite different from other DNS requests. See
+        U{http://cr.yp.to/djbdns/axfr-notes.html} for more
+        information.
+
+        NB Unlike other C{lookup*} methods, the timeout here is not a
+        list of ints, it is a single int.
+
+        @param name: DNS name to resolve.
+        @param timeout: When this timeout expires, the query is
+            considered failed.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.
+            The first element of the tuple gives answers.
+            The second and third elements are always empty.
+            The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+
+
+class IReactorTCP(Interface):
+    def listenTCP(
+        port: int, factory: "ServerFactory", backlog: int, interface: str
+    ) -> "IListeningPort":
+        """
+        Connects a given protocol factory to the given numeric TCP/IP port.
+
+        @param port: a port number on which to listen
+        @param factory: a L{twisted.internet.protocol.ServerFactory} instance
+        @param backlog: size of the listen queue
+        @param interface: The local IPv4 or IPv6 address to which to bind;
+            defaults to '', ie all IPv4 addresses.  To bind to all IPv4 and IPv6
+            addresses, you must call this method twice.
+
+        @return: an object that provides L{IListeningPort}.
+
+        @raise CannotListenError: as defined here
+                                  L{twisted.internet.error.CannotListenError},
+                                  if it cannot listen on this port (e.g., it
+                                  cannot bind to the required port number)
+        """
+
+    def connectTCP(
+        host: str,
+        port: int,
+        factory: "ClientFactory",
+        timeout: float,
+        bindAddress: Optional[Tuple[str, int]],
+    ) -> IConnector:
+        """
+        Connect a TCP client.
+
+        @param host: A hostname or an IPv4 or IPv6 address literal.
+        @param port: a port number
+        @param factory: a L{twisted.internet.protocol.ClientFactory} instance
+        @param timeout: number of seconds to wait before assuming the
+                        connection has failed.
+        @param bindAddress: a (host, port) tuple of local address to bind
+                            to, or None.
+
+        @return: An object which provides L{IConnector}. This connector will
+                 call various callbacks on the factory when a connection is
+                 made, failed, or lost - see
+                 L{ClientFactory<twisted.internet.protocol.ClientFactory>}
+                 docs for details.
+        """
+
+
+class IReactorSSL(Interface):
+    def connectSSL(
+        host: str,
+        port: int,
+        factory: "ClientFactory",
+        contextFactory: "ClientContextFactory",
+        timeout: float,
+        bindAddress: Optional[Tuple[str, int]],
+    ) -> IConnector:
+        """
+        Connect a client Protocol to a remote SSL socket.
+
+        @param host: a host name
+        @param port: a port number
+        @param factory: a L{twisted.internet.protocol.ClientFactory} instance
+        @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object.
+        @param timeout: number of seconds to wait before assuming the
+                        connection has failed.
+        @param bindAddress: a (host, port) tuple of local address to bind to,
+                            or L{None}.
+
+        @return: An object which provides L{IConnector}.
+        """
+
+    def listenSSL(
+        port: int,
+        factory: "ServerFactory",
+        contextFactory: "IOpenSSLContextFactory",
+        backlog: int,
+        interface: str,
+    ) -> "IListeningPort":
+        """
+        Connects a given protocol factory to the given numeric TCP/IP port.
+        The connection is a SSL one, using contexts created by the context
+        factory.
+
+        @param port: a port number on which to listen
+        @param factory: a L{twisted.internet.protocol.ServerFactory} instance
+        @param contextFactory: an implementor of L{IOpenSSLContextFactory}
+        @param backlog: size of the listen queue
+        @param interface: the hostname to bind to, defaults to '' (all)
+        """
+
+
+class IReactorUNIX(Interface):
+    """
+    UNIX socket methods.
+    """
+
+    def connectUNIX(
+        address: str, factory: "ClientFactory", timeout: float, checkPID: bool
+    ) -> IConnector:
+        """
+        Connect a client protocol to a UNIX socket.
+
+        @param address: a path to a unix socket on the filesystem.
+        @param factory: a L{twisted.internet.protocol.ClientFactory} instance
+        @param timeout: number of seconds to wait before assuming the connection
+            has failed.
+        @param checkPID: if True, check for a pid file to verify that a server
+            is listening.  If C{address} is a Linux abstract namespace path,
+            this must be C{False}.
+
+        @return: An object which provides L{IConnector}.
+        """
+
+    def listenUNIX(
+        address: str, factory: "Factory", backlog: int, mode: int, wantPID: bool
+    ) -> "IListeningPort":
+        """
+        Listen on a UNIX socket.
+
+        @param address: a path to a unix socket on the filesystem.
+        @param factory: a L{twisted.internet.protocol.Factory} instance.
+        @param backlog: number of connections to allow in backlog.
+        @param mode: The mode (B{not} umask) to set on the unix socket.  See
+            platform specific documentation for information about how this
+            might affect connection attempts.
+        @param wantPID: if True, create a pidfile for the socket.  If C{address}
+            is a Linux abstract namespace path, this must be C{False}.
+
+        @return: An object which provides L{IListeningPort}.
+        """
+
+
+class IReactorUNIXDatagram(Interface):
+    """
+    Datagram UNIX socket methods.
+    """
+
+    def connectUNIXDatagram(
+        address: str,
+        protocol: "ConnectedDatagramProtocol",
+        maxPacketSize: int,
+        mode: int,
+        bindAddress: Optional[Tuple[str, int]],
+    ) -> IConnector:
+        """
+        Connect a client protocol to a datagram UNIX socket.
+
+        @param address: a path to a unix socket on the filesystem.
+        @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance
+        @param maxPacketSize: maximum packet size to accept
+        @param mode: The mode (B{not} umask) to set on the unix socket.  See
+            platform specific documentation for information about how this
+            might affect connection attempts.
+
+        @param bindAddress: address to bind to
+
+        @return: An object which provides L{IConnector}.
+        """
+
+    def listenUNIXDatagram(
+        address: str, protocol: "DatagramProtocol", maxPacketSize: int, mode: int
+    ) -> "IListeningPort":
+        """
+        Listen on a datagram UNIX socket.
+
+        @param address: a path to a unix socket on the filesystem.
+        @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance.
+        @param maxPacketSize: maximum packet size to accept
+        @param mode: The mode (B{not} umask) to set on the unix socket.  See
+            platform specific documentation for information about how this
+            might affect connection attempts.
+
+        @return: An object which provides L{IListeningPort}.
+        """
+
+
+class IReactorWin32Events(Interface):
+    """
+    Win32 Event API methods
+
+    @since: 10.2
+    """
+
+    def addEvent(event: object, fd: "FileDescriptor", action: str) -> None:
+        """
+        Add a new win32 event to the event loop.
+
+        @param event: a Win32 event object created using win32event.CreateEvent()
+        @param fd: an instance of L{twisted.internet.abstract.FileDescriptor}
+        @param action: a string that is a method name of the fd instance.
+                       This method is called in response to the event.
+        """
+
+    def removeEvent(event: object) -> None:
+        """
+        Remove an event.
+
+        @param event: a Win32 event object added using L{IReactorWin32Events.addEvent}
+
+        @return: None
+        """
+
+
+class IReactorUDP(Interface):
+    """
+    UDP socket methods.
+    """
+
+    def listenUDP(
+        port: int, protocol: "DatagramProtocol", interface: str, maxPacketSize: int
+    ) -> "IListeningPort":
+        """
+        Connects a given L{DatagramProtocol} to the given numeric UDP port.
+
+        @param port: A port number on which to listen.
+        @param protocol: A L{DatagramProtocol} instance which will be
+            connected to the given C{port}.
+        @param interface: The local IPv4 or IPv6 address to which to bind;
+            defaults to '', ie all IPv4 addresses.
+        @param maxPacketSize: The maximum packet size to accept.
+
+        @return: object which provides L{IListeningPort}.
+        """
+
+
+class IReactorMulticast(Interface):
+    """
+    UDP socket methods that support multicast.
+
+    IMPORTANT: This is an experimental new interface. It may change
+    without backwards compatibility. Suggestions are welcome.
+    """
+
+    def listenMulticast(
+        port: int,
+        protocol: "DatagramProtocol",
+        interface: str,
+        maxPacketSize: int,
+        listenMultiple: bool,
+    ) -> "IListeningPort":
+        """
+        Connects a given
+        L{DatagramProtocol<twisted.internet.protocol.DatagramProtocol>} to the
+        given numeric UDP port.
+
+        @param listenMultiple: If set to True, allows multiple sockets to
+            bind to the same address and port number at the same time.
+
+        @returns: An object which provides L{IListeningPort}.
+
+        @see: L{twisted.internet.interfaces.IMulticastTransport}
+        @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
+        """
+
+
+class IReactorSocket(Interface):
+    """
+    Methods which allow a reactor to use externally created sockets.
+
+    For example, to use C{adoptStreamPort} to implement behavior equivalent
+    to that of L{IReactorTCP.listenTCP}, you might write code like this::
+
+        from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket
+        portSocket = socket(AF_INET, SOCK_STREAM)
+        # Set FD_CLOEXEC on port, left as an exercise.  Then make it into a
+        # non-blocking listening port:
+        portSocket.setblocking(False)
+        portSocket.bind(('192.168.1.2', 12345))
+        portSocket.listen(SOMAXCONN)
+
+        # Now have the reactor use it as a TCP port
+        port = reactor.adoptStreamPort(
+            portSocket.fileno(), AF_INET, YourFactory())
+
+        # portSocket itself is no longer necessary, and needs to be cleaned
+        # up by us.
+        portSocket.close()
+
+        # Whenever the server is no longer needed, stop it as usual.
+        stoppedDeferred = port.stopListening()
+
+    Another potential use is to inherit a listening descriptor from a parent
+    process (for example, systemd or launchd), or to receive one over a UNIX
+    domain socket.
+
+    Some plans for extending this interface exist.  See:
+
+        - U{http://twistedmatrix.com/trac/ticket/6594}: AF_UNIX SOCK_DGRAM ports
+    """
+
+    def adoptStreamPort(
+        fileDescriptor: int, addressFamily: "AddressFamily", factory: "ServerFactory"
+    ) -> "IListeningPort":
+        """
+        Add an existing listening I{SOCK_STREAM} socket to the reactor to
+        monitor for new connections to accept and handle.
+
+        @param fileDescriptor: A file descriptor associated with a socket which
+            is already bound to an address and marked as listening.  The socket
+            must be set non-blocking.  Any additional flags (for example,
+            close-on-exec) must also be set by application code.  Application
+            code is responsible for closing the file descriptor, which may be
+            done as soon as C{adoptStreamPort} returns.
+        @param addressFamily: The address family (or I{domain}) of the socket.
+            For example, L{socket.AF_INET6}.
+        @param factory: A L{ServerFactory} instance to use to create new
+            protocols to handle connections accepted via this socket.
+
+        @return: An object providing L{IListeningPort}.
+
+        @raise twisted.internet.error.UnsupportedAddressFamily: If the
+            given address family is not supported by this reactor, or
+            not supported with the given socket type.
+        @raise twisted.internet.error.UnsupportedSocketType: If the
+            given socket type is not supported by this reactor, or not
+            supported with the given socket type.
+        """
+
+    def adoptStreamConnection(
+        fileDescriptor: int, addressFamily: "AddressFamily", factory: "ServerFactory"
+    ) -> None:
+        """
+        Add an existing connected I{SOCK_STREAM} socket to the reactor to
+        monitor for data.
+
+        Note that the given factory won't have its C{startFactory} and
+        C{stopFactory} methods called, as there is no sensible time to call
+        them in this situation.
+
+        @param fileDescriptor: A file descriptor associated with a socket which
+            is already connected.  The socket must be set non-blocking.  Any
+            additional flags (for example, close-on-exec) must also be set by
+            application code.  Application code is responsible for closing the
+            file descriptor, which may be done as soon as
+            C{adoptStreamConnection} returns.
+        @param addressFamily: The address family (or I{domain}) of the socket.
+            For example, L{socket.AF_INET6}.
+        @param factory: A L{ServerFactory} instance to use to create a new
+            protocol to handle the connection via this socket.
+
+        @raise UnsupportedAddressFamily: If the given address family is not
+            supported by this reactor, or not supported with the given socket
+            type.
+        @raise UnsupportedSocketType: If the given socket type is not supported
+            by this reactor, or not supported with the given socket type.
+        """
+
+    def adoptDatagramPort(
+        fileDescriptor: int,
+        addressFamily: "AddressFamily",
+        protocol: "DatagramProtocol",
+        maxPacketSize: int,
+    ) -> "IListeningPort":
+        """
+        Add an existing listening I{SOCK_DGRAM} socket to the reactor to
+        monitor for read and write readiness.
+
+        @param fileDescriptor: A file descriptor associated with a socket which
+            is already bound to an address and marked as listening.  The socket
+            must be set non-blocking.  Any additional flags (for example,
+            close-on-exec) must also be set by application code.  Application
+            code is responsible for closing the file descriptor, which may be
+            done as soon as C{adoptDatagramPort} returns.
+        @param addressFamily: The address family or I{domain} of the socket.
+            For example, L{socket.AF_INET6}.
+        @param protocol: A L{DatagramProtocol} instance to connect to
+            a UDP transport.
+        @param maxPacketSize: The maximum packet size to accept.
+
+        @return: An object providing L{IListeningPort}.
+
+        @raise UnsupportedAddressFamily: If the given address family is not
+            supported by this reactor, or not supported with the given socket
+            type.
+        @raise UnsupportedSocketType: If the given socket type is not supported
+            by this reactor, or not supported with the given socket type.
+        """
+
+
+class IReactorProcess(Interface):
+    def spawnProcess(
+        processProtocol: "IProcessProtocol",
+        executable: Union[bytes, str],
+        args: Sequence[Union[bytes, str]],
+        env: Optional[Mapping[AnyStr, AnyStr]] = None,
+        path: Union[None, bytes, str] = None,
+        uid: Optional[int] = None,
+        gid: Optional[int] = None,
+        usePTY: bool = False,
+        childFDs: Optional[Mapping[int, Union[int, str]]] = None,
+    ) -> "IProcessTransport":
+        """
+        Spawn a process, with a process protocol.
+
+        Arguments given to this function that are listed as L{bytes} or
+        L{unicode} may be encoded or decoded depending on the platform and the
+        argument type given.  On UNIX systems (Linux, FreeBSD, macOS) and
+        Python 2 on Windows, L{unicode} arguments will be encoded down to
+        L{bytes} using the encoding given by L{sys.getfilesystemencoding}, to be
+        used with the "narrow" OS APIs.  On Python 3 on Windows, L{bytes}
+        arguments will be decoded up to L{unicode} using the encoding given by
+        L{sys.getfilesystemencoding} (C{utf8}) and given to Windows's native "wide" APIs.
+
+        @param processProtocol: An object which will be notified of all events
+            related to the created process.
+
+        @param executable: the file name to spawn - the full path should be
+            used.
+
+        @param args: the command line arguments to pass to the process; a
+            sequence of strings.  The first string should be the executable's
+            name.
+
+        @param env: the environment variables to pass to the child process.
+            The resulting behavior varies between platforms.  If:
+
+                - C{env} is not set:
+                  - On POSIX: pass an empty environment.
+                  - On Windows: pass L{os.environ}.
+                - C{env} is L{None}:
+                  - On POSIX: pass L{os.environ}.
+                  - On Windows: pass L{os.environ}.
+                - C{env} is a L{dict}:
+                  - On POSIX: pass the key/value pairs in C{env} as the
+                    complete environment.
+                  - On Windows: update L{os.environ} with the key/value
+                    pairs in the L{dict} before passing it. As a
+                    consequence of U{bug #1640
+                    <http://twistedmatrix.com/trac/ticket/1640>}, passing
+                    keys with empty values in an effort to unset
+                    environment variables I{won't} unset them.
+
+        @param path: the path to run the subprocess in - defaults to the
+            current directory.
+
+        @param uid: user ID to run the subprocess as.  (Only available on POSIX
+            systems.)
+
+        @param gid: group ID to run the subprocess as.  (Only available on
+            POSIX systems.)
+
+        @param usePTY: if true, run this process in a pseudo-terminal.
+            optionally a tuple of C{(masterfd, slavefd, ttyname)}, in which
+            case use those file descriptors.  (Not available on all systems.)
+
+        @param childFDs: A dictionary mapping file descriptors in the new child
+            process to an integer or to the string 'r' or 'w'.
+
+            If the value is an integer, it specifies a file descriptor in the
+            parent process which will be mapped to a file descriptor (specified
+            by the key) in the child process.  This is useful for things like
+            inetd and shell-like file redirection.
+
+            If it is the string 'r', a pipe will be created and attached to the
+            child at that file descriptor: the child will be able to write to
+            that file descriptor and the parent will receive read notification
+            via the L{IProcessProtocol.childDataReceived} callback.  This is
+            useful for the child's stdout and stderr.
+
+            If it is the string 'w', similar setup to the previous case will
+            occur, with the pipe being readable by the child instead of
+            writeable.  The parent process can write to that file descriptor
+            using L{IProcessTransport.writeToChild}.  This is useful for the
+            child's stdin.
+
+            If childFDs is not passed, the default behaviour is to use a
+            mapping that opens the usual stdin/stdout/stderr pipes.
+
+        @see: L{twisted.internet.protocol.ProcessProtocol}
+
+        @return: An object which provides L{IProcessTransport}.
+
+        @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are
+            insufficient system resources to create a new process.
+        """
+
+
+class IReactorTime(Interface):
+    """
+    Time methods that a Reactor should implement.
+    """
+
+    def seconds() -> float:
+        """
+        Get the current time in seconds.
+
+        @return: A number-like object of some sort.
+        """
+
+    def callLater(
+        delay: float, callable: Callable[..., Any], *args: object, **kwargs: object
+    ) -> "IDelayedCall":
+        """
+        Call a function later.
+
+        @param delay: the number of seconds to wait.
+        @param callable: the callable object to call later.
+        @param args: the arguments to call it with.
+        @param kwargs: the keyword arguments to call it with.
+
+        @return: An object which provides L{IDelayedCall} and can be used to
+                 cancel the scheduled call, by calling its C{cancel()} method.
+                 It also may be rescheduled by calling its C{delay()} or
+                 C{reset()} methods.
+        """
+
+    def getDelayedCalls() -> Sequence["IDelayedCall"]:
+        """
+        See L{twisted.internet.interfaces.IReactorTime.getDelayedCalls}
+        """
+
+
+class IDelayedCall(Interface):
+    """
+    A scheduled call.
+
+    There are probably other useful methods we can add to this interface;
+    suggestions are welcome.
+    """
+
+    def getTime() -> float:
+        """
+        Get time when delayed call will happen.
+
+        @return: time in seconds since epoch (a float).
+        """
+
+    def cancel() -> None:
+        """
+        Cancel the scheduled call.
+
+        @raises twisted.internet.error.AlreadyCalled: if the call has already
+            happened.
+        @raises twisted.internet.error.AlreadyCancelled: if the call has already
+            been cancelled.
+        """
+
+    def delay(secondsLater: float) -> None:
+        """
+        Delay the scheduled call.
+
+        @param secondsLater: how many seconds from its current firing time to delay
+
+        @raises twisted.internet.error.AlreadyCalled: if the call has already
+            happened.
+        @raises twisted.internet.error.AlreadyCancelled: if the call has already
+            been cancelled.
+        """
+
+    def reset(secondsFromNow: float) -> None:
+        """
+        Reset the scheduled call's timer.
+
+        @param secondsFromNow: how many seconds from now it should fire,
+            equivalent to C{.cancel()} and then doing another
+            C{reactor.callLater(secondsLater, ...)}
+
+        @raises twisted.internet.error.AlreadyCalled: if the call has already
+            happened.
+        @raises twisted.internet.error.AlreadyCancelled: if the call has already
+            been cancelled.
+        """
+
+    def active() -> bool:
+        """
+        @return: True if this call is still active, False if it has been
+                 called or cancelled.
+        """
+
+
+class IReactorFromThreads(Interface):
+    """
+    This interface is the set of thread-safe methods which may be invoked on
+    the reactor from other threads.
+
+    @since: 15.4
+    """
+
+    def callFromThread(
+        callable: Callable[..., Any], *args: object, **kwargs: object
+    ) -> None:
+        """
+        Cause a function to be executed by the reactor thread.
+
+        Use this method when you want to run a function in the reactor's thread
+        from another thread.  Calling L{callFromThread} should wake up the main
+        thread (where L{reactor.run() <IReactorCore.run>} is executing) and run
+        the given callable in that thread.
+
+        If you're writing a multi-threaded application the C{callable}
+        may need to be thread safe, but this method doesn't require it as such.
+        If you want to call a function in the next mainloop iteration, but
+        you're in the same thread, use L{callLater} with a delay of 0.
+        """
+
+
+class IReactorInThreads(Interface):
+    """
+    This interface contains the methods exposed by a reactor which will let you
+    run functions in another thread.
+
+    @since: 15.4
+    """
+
+    def callInThread(
+        callable: Callable[..., Any], *args: object, **kwargs: object
+    ) -> None:
+        """
+        Run the given callable object in a separate thread, with the given
+        arguments and keyword arguments.
+        """
+
+
+class IReactorThreads(IReactorFromThreads, IReactorInThreads):
+    """
+    Dispatch methods to be run in threads.
+
+    Internally, this should use a thread pool and dispatch methods to them.
+    """
+
+    def getThreadPool() -> "ThreadPool":
+        """
+        Return the threadpool used by L{IReactorInThreads.callInThread}.
+        Create it first if necessary.
+        """
+
+    def suggestThreadPoolSize(size: int) -> None:
+        """
+        Suggest the size of the internal threadpool used to dispatch functions
+        passed to L{IReactorInThreads.callInThread}.
+        """
+
+
+class IReactorCore(Interface):
+    """
+    Core methods that a Reactor must implement.
+    """
+
+    running = Attribute(
+        "A C{bool} which is C{True} from I{during startup} to "
+        "I{during shutdown} and C{False} the rest of the time."
+    )
+
+    def resolve(name: str, timeout: Sequence[int]) -> "Deferred[str]":
+        """
+        Return a L{twisted.internet.defer.Deferred} that will resolve
+        a hostname.
+        """
+
+    def run() -> None:
+        """
+        Fire 'startup' System Events, move the reactor to the 'running'
+        state, then run the main loop until it is stopped with C{stop()} or
+        C{crash()}.
+        """
+
+    def stop() -> None:
+        """
+        Fire 'shutdown' System Events, which will move the reactor to the
+        'stopped' state and cause C{reactor.run()} to exit.
+        """
+
+    def crash() -> None:
+        """
+        Stop the main loop *immediately*, without firing any system events.
+
+        This is named as it is because this is an extremely "rude" thing to do;
+        it is possible to lose data and put your system in an inconsistent
+        state by calling this.  However, it is necessary, as sometimes a system
+        can become wedged in a pre-shutdown call.
+        """
+
+    def iterate(delay: float) -> None:
+        """
+        Run the main loop's I/O polling function for a period of time.
+
+        This is most useful in applications where the UI is being drawn "as
+        fast as possible", such as games. All pending L{IDelayedCall}s will
+        be called.
+
+        The reactor must have been started (via the C{run()} method) prior to
+        any invocations of this method.  It must also be stopped manually
+        after the last call to this method (via the C{stop()} method).  This
+        method is not re-entrant: you must not call it recursively; in
+        particular, you must not call it while the reactor is running.
+        """
+
+    def fireSystemEvent(eventType: str) -> None:
+        """
+        Fire a system-wide event.
+
+        System-wide events are things like 'startup', 'shutdown', and
+        'persist'.
+        """
+
+    def addSystemEventTrigger(
+        phase: str,
+        eventType: str,
+        callable: Callable[..., Any],
+        *args: object,
+        **kwargs: object,
+    ) -> Any:
+        """
+        Add a function to be called when a system event occurs.
+
+        Each "system event" in Twisted, such as 'startup', 'shutdown', and
+        'persist', has 3 phases: 'before', 'during', and 'after' (in that
+        order, of course).  These events will be fired internally by the
+        Reactor.
+
+        An implementor of this interface must only implement those events
+        described here.
+
+        Callbacks registered for the "before" phase may return either None or a
+        Deferred.  The "during" phase will not execute until all of the
+        Deferreds from the "before" phase have fired.
+
+        Once the "during" phase is running, all of the remaining triggers must
+        execute; their return values must be ignored.
+
+        @param phase: a time to call the event -- either the string 'before',
+                      'after', or 'during', describing when to call it
+                      relative to the event's execution.
+        @param eventType: this is a string describing the type of event.
+        @param callable: the object to call before shutdown.
+        @param args: the arguments to call it with.
+        @param kwargs: the keyword arguments to call it with.
+
+        @return: an ID that can be used to remove this call with
+                 removeSystemEventTrigger.
+        """
+
+    def removeSystemEventTrigger(triggerID: Any) -> None:
+        """
+        Removes a trigger added with addSystemEventTrigger.
+
+        @param triggerID: a value returned from addSystemEventTrigger.
+
+        @raise KeyError: If there is no system event trigger for the given
+            C{triggerID}.
+        @raise ValueError: If there is no system event trigger for the given
+            C{triggerID}.
+        @raise TypeError: If there is no system event trigger for the given
+            C{triggerID}.
+        """
+
+    def callWhenRunning(
+        callable: Callable[..., Any], *args: object, **kwargs: object
+    ) -> Optional[Any]:
+        """
+        Call a function when the reactor is running.
+
+        If the reactor has not started, the callable will be scheduled
+        to run when it does start. Otherwise, the callable will be invoked
+        immediately.
+
+        @param callable: the callable object to call later.
+        @param args: the arguments to call it with.
+        @param kwargs: the keyword arguments to call it with.
+
+        @return: None if the callable was invoked, otherwise a system
+                 event id for the scheduled call.
+        """
+
+
+class IReactorPluggableResolver(Interface):
+    """
+    An L{IReactorPluggableResolver} is a reactor which can be customized with
+    an L{IResolverSimple}.  This is a fairly limited interface, that supports
+    only IPv4; you should use L{IReactorPluggableNameResolver} instead.
+
+    @see: L{IReactorPluggableNameResolver}
+    """
+
+    def installResolver(resolver: IResolverSimple) -> IResolverSimple:
+        """
+        Set the internal resolver to use to for name lookups.
+
+        @param resolver: The new resolver to use.
+
+        @return: The previously installed resolver.
+        """
+
+
+class IReactorPluggableNameResolver(Interface):
+    """
+    An L{IReactorPluggableNameResolver} is a reactor whose name resolver can be
+    set to a user-supplied object.
+    """
+
+    nameResolver = Attribute(
+        """
+        Read-only attribute; the resolver installed with L{installResolver}.
+        An L{IHostnameResolver}.
+        """
+    )
+
+    def installNameResolver(resolver: IHostnameResolver) -> IHostnameResolver:
+        """
+        Set the internal resolver to use for name lookups.
+
+        @param resolver: The new resolver to use.
+
+        @return: The previously installed resolver.
+        """
+
+
+class IReactorDaemonize(Interface):
+    """
+    A reactor which provides hooks that need to be called before and after
+    daemonization.
+
+    Notes:
+       - This interface SHOULD NOT be called by applications.
+       - This interface should only be implemented by reactors as a workaround
+         (in particular, it's implemented currently only by kqueue()).
+         For details please see the comments on ticket #1918.
+    """
+
+    def beforeDaemonize() -> None:
+        """
+        Hook to be called immediately before daemonization. No reactor methods
+        may be called until L{afterDaemonize} is called.
+        """
+
+    def afterDaemonize() -> None:
+        """
+        Hook to be called immediately after daemonization. This may only be
+        called after L{beforeDaemonize} had been called previously.
+        """
+
+
+class IReactorFDSet(Interface):
+    """
+    Implement me to be able to use L{IFileDescriptor} type resources.
+
+    This assumes that your main-loop uses UNIX-style numeric file descriptors
+    (or at least similarly opaque IDs returned from a .fileno() method)
+    """
+
+    def addReader(reader: "IReadDescriptor") -> None:
+        """
+        I add reader to the set of file descriptors to get read events for.
+
+        @param reader: An L{IReadDescriptor} provider that will be checked for
+                       read events until it is removed from the reactor with
+                       L{removeReader}.
+        """
+
+    def addWriter(writer: "IWriteDescriptor") -> None:
+        """
+        I add writer to the set of file descriptors to get write events for.
+
+        @param writer: An L{IWriteDescriptor} provider that will be checked for
+                       write events until it is removed from the reactor with
+                       L{removeWriter}.
+        """
+
+    def removeReader(reader: "IReadDescriptor") -> None:
+        """
+        Removes an object previously added with L{addReader}.
+        """
+
+    def removeWriter(writer: "IWriteDescriptor") -> None:
+        """
+        Removes an object previously added with L{addWriter}.
+        """
+
+    def removeAll() -> List[Union["IReadDescriptor", "IWriteDescriptor"]]:
+        """
+        Remove all readers and writers.
+
+        Should not remove reactor internal reactor connections (like a waker).
+
+        @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers
+                 which were removed.
+        """
+
+    def getReaders() -> List["IReadDescriptor"]:
+        """
+        Return the list of file descriptors currently monitored for input
+        events by the reactor.
+
+        @return: the list of file descriptors monitored for input events.
+        """
+
+    def getWriters() -> List["IWriteDescriptor"]:
+        """
+        Return the list file descriptors currently monitored for output events
+        by the reactor.
+
+        @return: the list of file descriptors monitored for output events.
+        """
+
+
+class IListeningPort(Interface):
+    """
+    A listening port.
+    """
+
+    def startListening() -> None:
+        """
+        Start listening on this port.
+
+        @raise CannotListenError: If it cannot listen on this port (e.g., it is
+                                  a TCP port and it cannot bind to the required
+                                  port number).
+        """
+
+    def stopListening() -> Optional["Deferred[None]"]:
+        """
+        Stop listening on this port.
+
+        If it does not complete immediately, will return Deferred that fires
+        upon completion.
+        """
+
+    def getHost() -> IAddress:
+        """
+        Get the host that this port is listening for.
+
+        @return: An L{IAddress} provider.
+        """
+
+
+class ILoggingContext(Interface):
+    """
+    Give context information that will be used to log events generated by
+    this item.
+    """
+
+    def logPrefix() -> str:
+        """
+        @return: Prefix used during log formatting to indicate context.
+        """
+
+
+class IFileDescriptor(ILoggingContext):
+    """
+    An interface representing a UNIX-style numeric file descriptor.
+    """
+
+    def fileno() -> object:
+        """
+        @return: The platform-specified representation of a file descriptor
+            number.  Or C{-1} if the descriptor no longer has a valid file
+            descriptor number associated with it.  As long as the descriptor
+            is valid, calls to this method on a particular instance must
+            return the same value.
+        """
+
+    def connectionLost(reason: Failure) -> None:
+        """
+        Called when the connection was lost.
+
+        This is called when the connection on a selectable object has been
+        lost.  It will be called whether the connection was closed explicitly,
+        an exception occurred in an event handler, or the other end of the
+        connection closed it first.
+
+        See also L{IHalfCloseableDescriptor} if your descriptor wants to be
+        notified separately of the two halves of the connection being closed.
+
+        @param reason: A failure instance indicating the reason why the
+                       connection was lost.  L{error.ConnectionLost} and
+                       L{error.ConnectionDone} are of special note, but the
+                       failure may be of other classes as well.
+        """
+
+
+class IReadDescriptor(IFileDescriptor):
+    """
+    An L{IFileDescriptor} that can read.
+
+    This interface is generally used in conjunction with L{IReactorFDSet}.
+    """
+
+    def doRead() -> Optional[Failure]:
+        """
+        Some data is available for reading on your descriptor.
+
+        @return: If an error is encountered which causes the descriptor to
+            no longer be valid, a L{Failure} should be returned.  Otherwise,
+            L{None}.
+        """
+
+
+class IWriteDescriptor(IFileDescriptor):
+    """
+    An L{IFileDescriptor} that can write.
+
+    This interface is generally used in conjunction with L{IReactorFDSet}.
+    """
+
+    def doWrite() -> Optional[Failure]:
+        """
+        Some data can be written to your descriptor.
+
+        @return: If an error is encountered which causes the descriptor to
+            no longer be valid, a L{Failure} should be returned.  Otherwise,
+            L{None}.
+        """
+
+
+class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor):
+    """
+    An L{IFileDescriptor} that can both read and write.
+    """
+
+
+class IHalfCloseableDescriptor(Interface):
+    """
+    A descriptor that can be half-closed.
+    """
+
+    def writeConnectionLost(reason: Failure) -> None:
+        """
+        Indicates write connection was lost.
+        """
+
+    def readConnectionLost(reason: Failure) -> None:
+        """
+        Indicates read connection was lost.
+        """
+
+
+class ISystemHandle(Interface):
+    """
+    An object that wraps a networking OS-specific handle.
+    """
+
+    def getHandle() -> object:
+        """
+        Return a system- and reactor-specific handle.
+
+        This might be a socket.socket() object, or some other type of
+        object, depending on which reactor is being used. Use and
+        manipulate at your own risk.
+
+        This might be used in cases where you want to set specific
+        options not exposed by the Twisted APIs.
+        """
+
+
+class IConsumer(Interface):
+    """
+    A consumer consumes data from a producer.
+    """
+
+    def registerProducer(producer: "IProducer", streaming: bool) -> None:
+        """
+        Register to receive data from a producer.
+
+        This sets self to be a consumer for a producer.  When this object runs
+        out of data (as when a send(2) call on a socket succeeds in moving the
+        last data from a userspace buffer into a kernelspace buffer), it will
+        ask the producer to resumeProducing().
+
+        For L{IPullProducer} providers, C{resumeProducing} will be called once
+        each time data is required.
+
+        For L{IPushProducer} providers, C{pauseProducing} will be called
+        whenever the write buffer fills up and C{resumeProducing} will only be
+        called when it empties.  The consumer will only call C{resumeProducing}
+        to balance a previous C{pauseProducing} call; the producer is assumed
+        to start in an un-paused state.
+
+        @param streaming: C{True} if C{producer} provides L{IPushProducer},
+            C{False} if C{producer} provides L{IPullProducer}.
+
+        @raise RuntimeError: If a producer is already registered.
+        """
+
+    def unregisterProducer() -> None:
+        """
+        Stop consuming data from a producer, without disconnecting.
+        """
+
+    def write(data: bytes) -> None:
+        """
+        The producer will write data by calling this method.
+
+        The implementation must be non-blocking and perform whatever
+        buffering is necessary.  If the producer has provided enough data
+        for now and it is a L{IPushProducer}, the consumer may call its
+        C{pauseProducing} method.
+        """
+
+
+class IProducer(Interface):
+    """
+    A producer produces data for a consumer.
+
+    Typically producing is done by calling the C{write} method of a class
+    implementing L{IConsumer}.
+    """
+
+    def stopProducing() -> None:
+        """
+        Stop producing data.
+
+        This tells a producer that its consumer has died, so it must stop
+        producing data for good.
+        """
+
+
+class IPushProducer(IProducer):
+    """
+    A push producer, also known as a streaming producer is expected to
+    produce (write to this consumer) data on a continuous basis, unless
+    it has been paused. A paused push producer will resume producing
+    after its C{resumeProducing()} method is called.   For a push producer
+    which is not pauseable, these functions may be noops.
+    """
+
+    def pauseProducing() -> None:
+        """
+        Pause producing data.
+
+        Tells a producer that it has produced too much data to process for
+        the time being, and to stop until C{resumeProducing()} is called.
+        """
+
+    def resumeProducing() -> None:
+        """
+        Resume producing data.
+
+        This tells a producer to re-add itself to the main loop and produce
+        more data for its consumer.
+        """
+
+
+class IPullProducer(IProducer):
+    """
+    A pull producer, also known as a non-streaming producer, is
+    expected to produce data each time L{resumeProducing()} is called.
+    """
+
+    def resumeProducing() -> None:
+        """
+        Produce data for the consumer a single time.
+
+        This tells a producer to produce data for the consumer once
+        (not repeatedly, once only). Typically this will be done
+        by calling the consumer's C{write} method a single time with
+        produced data. The producer should produce data before returning
+        from C{resumeProducing()}, that is, it should not schedule a deferred
+        write.
+        """
+
+
+class IProtocol(Interface):
+    def dataReceived(data: bytes) -> None:
+        """
+        Called whenever data is received.
+
+        Use this method to translate to a higher-level message.  Usually, some
+        callback will be made upon the receipt of each complete protocol
+        message.
+
+        Please keep in mind that you will probably need to buffer some data
+        as partial (or multiple) protocol messages may be received!  We
+        recommend that unit tests for protocols call through to this method
+        with differing chunk sizes, down to one byte at a time.
+
+        @param data: bytes of indeterminate length
+        """
+
+    def connectionLost(reason: Failure) -> None:
+        """
+        Called when the connection is shut down.
+
+        Clear any circular references here, and any external references
+        to this Protocol.  The connection has been closed. The C{reason}
+        Failure wraps a L{twisted.internet.error.ConnectionDone} or
+        L{twisted.internet.error.ConnectionLost} instance (or a subclass
+        of one of those).
+        """
+
+    def makeConnection(transport: "ITransport") -> None:
+        """
+        Make a connection to a transport and a server.
+        """
+
+    def connectionMade() -> None:
+        """
+        Called when a connection is made.
+
+        This may be considered the initializer of the protocol, because
+        it is called when the connection is completed.  For clients,
+        this is called once the connection to the server has been
+        established; for servers, this is called after an accept() call
+        stops blocking and a socket has been received.  If you need to
+        send any greeting or initial message, do it here.
+        """
+
+
+class IProcessProtocol(Interface):
+    """
+    Interface for process-related event handlers.
+    """
+
+    def makeConnection(process: "IProcessTransport") -> None:
+        """
+        Called when the process has been created.
+
+        @param process: An object representing the process which has been
+            created and associated with this protocol.
+        """
+
+    def childDataReceived(childFD: int, data: bytes) -> None:
+        """
+        Called when data arrives from the child process.
+
+        @param childFD: The file descriptor from which the data was
+            received.
+        @param data: The data read from the child's file descriptor.
+        """
+
+    def childConnectionLost(childFD: int) -> None:
+        """
+        Called when a file descriptor associated with the child process is
+        closed.
+
+        @param childFD: The file descriptor which was closed.
+        """
+
+    def processExited(reason: Failure) -> None:
+        """
+        Called when the child process exits.
+
+        @param reason: A failure giving the reason the child process
+            terminated.  The type of exception for this failure is either
+            L{twisted.internet.error.ProcessDone} or
+            L{twisted.internet.error.ProcessTerminated}.
+
+        @since: 8.2
+        """
+
+    def processEnded(reason: Failure) -> None:
+        """
+        Called when the child process exits and all file descriptors associated
+        with it have been closed.
+
+        @param reason: A failure giving the reason the child process
+            terminated.  The type of exception for this failure is either
+            L{twisted.internet.error.ProcessDone} or
+            L{twisted.internet.error.ProcessTerminated}.
+        """
+
+
+class IHalfCloseableProtocol(Interface):
+    """
+    Implemented to indicate they want notification of half-closes.
+
+    TCP supports the notion of half-closing the connection, e.g.
+    closing the write side but still not stopping reading. A protocol
+    that implements this interface will be notified of such events,
+    instead of having connectionLost called.
+    """
+
+    def readConnectionLost() -> None:
+        """
+        Notification of the read connection being closed.
+
+        This indicates peer did half-close of write side. It is now
+        the responsibility of the this protocol to call
+        loseConnection().  In addition, the protocol MUST make sure a
+        reference to it still exists (i.e. by doing a callLater with
+        one of its methods, etc.)  as the reactor will only have a
+        reference to it if it is writing.
+
+        If the protocol does not do so, it might get garbage collected
+        without the connectionLost method ever being called.
+        """
+
+    def writeConnectionLost() -> None:
+        """
+        Notification of the write connection being closed.
+
+        This will never be called for TCP connections as TCP does not
+        support notification of this type of half-close.
+        """
+
+
+class IHandshakeListener(Interface):
+    """
+    An interface implemented by a L{IProtocol} to indicate that it would like
+    to be notified when TLS handshakes complete when run over a TLS-based
+    transport.
+
+    This interface is only guaranteed to be called when run over a TLS-based
+    transport: non TLS-based transports will not respect this interface.
+    """
+
+    def handshakeCompleted() -> None:
+        """
+        Notification of the TLS handshake being completed.
+
+        This notification fires when OpenSSL has completed the TLS handshake.
+        At this point the TLS connection is established, and the protocol can
+        interrogate its transport (usually an L{ISSLTransport}) for details of
+        the TLS connection.
+
+        This notification *also* fires whenever the TLS session is
+        renegotiated. As a result, protocols that have certain minimum security
+        requirements should implement this interface to ensure that they are
+        able to re-evaluate the security of the TLS session if it changes.
+        """
+
+
+class IFileDescriptorReceiver(Interface):
+    """
+    Protocols may implement L{IFileDescriptorReceiver} to receive file
+    descriptors sent to them.  This is useful in conjunction with
+    L{IUNIXTransport}, which allows file descriptors to be sent between
+    processes on a single host.
+    """
+
+    def fileDescriptorReceived(descriptor: int) -> None:
+        """
+        Called when a file descriptor is received over the connection.
+
+        @param descriptor: The descriptor which was received.
+
+        @return: L{None}
+        """
+
+
+class IProtocolFactory(Interface):
+    """
+    Interface for protocol factories.
+    """
+
+    def buildProtocol(addr: IAddress) -> Optional[IProtocol]:
+        """
+        Called when a connection has been established to addr.
+
+        If None is returned, the connection is assumed to have been refused,
+        and the Port will close the connection.
+
+        @param addr: The address of the newly-established connection
+
+        @return: None if the connection was refused, otherwise an object
+                 providing L{IProtocol}.
+        """
+
+    def doStart() -> None:
+        """
+        Called every time this is connected to a Port or Connector.
+        """
+
+    def doStop() -> None:
+        """
+        Called every time this is unconnected from a Port or Connector.
+        """
+
+
+class ITransport(Interface):
+    """
+    I am a transport for bytes.
+
+    I represent (and wrap) the physical connection and synchronicity
+    of the framework which is talking to the network.  I make no
+    representations about whether calls to me will happen immediately
+    or require returning to a control loop, or whether they will happen
+    in the same or another thread.  Consider methods of this class
+    (aside from getPeer) to be 'thrown over the wall', to happen at some
+    indeterminate time.
+    """
+
+    def write(data: bytes) -> None:
+        """
+        Write some data to the physical connection, in sequence, in a
+        non-blocking fashion.
+
+        If possible, make sure that it is all written.  No data will
+        ever be lost, although (obviously) the connection may be closed
+        before it all gets through.
+
+        @param data: The data to write.
+        """
+
+    def writeSequence(data: Iterable[bytes]) -> None:
+        """
+        Write an iterable of byte strings to the physical connection.
+
+        If possible, make sure that all of the data is written to
+        the socket at once, without first copying it all into a
+        single byte string.
+
+        @param data: The data to write.
+        """
+
+    def loseConnection() -> None:
+        """
+        Close my connection, after writing all pending data.
+
+        Note that if there is a registered producer on a transport it
+        will not be closed until the producer has been unregistered.
+        """
+
+    def getPeer() -> IAddress:
+        """
+        Get the remote address of this connection.
+
+        Treat this method with caution.  It is the unfortunate result of the
+        CGI and Jabber standards, but should not be considered reliable for
+        the usual host of reasons; port forwarding, proxying, firewalls, IP
+        masquerading, etc.
+
+        @return: An L{IAddress} provider.
+        """
+
+    def getHost() -> IAddress:
+        """
+        Similar to getPeer, but returns an address describing this side of the
+        connection.
+
+        @return: An L{IAddress} provider.
+        """
+
+
+class ITCPTransport(ITransport):
+    """
+    A TCP based transport.
+    """
+
+    def loseWriteConnection() -> None:
+        """
+        Half-close the write side of a TCP connection.
+
+        If the protocol instance this is attached to provides
+        IHalfCloseableProtocol, it will get notified when the operation is
+        done. When closing write connection, as with loseConnection this will
+        only happen when buffer has emptied and there is no registered
+        producer.
+        """
+
+    def abortConnection() -> None:
+        """
+        Close the connection abruptly.
+
+        Discards any buffered data, stops any registered producer,
+        and, if possible, notifies the other end of the unclean
+        closure.
+
+        @since: 11.1
+        """
+
+    def getTcpNoDelay() -> bool:
+        """
+        Return if C{TCP_NODELAY} is enabled.
+        """
+
+    def setTcpNoDelay(enabled: bool) -> None:
+        """
+        Enable/disable C{TCP_NODELAY}.
+
+        Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are
+        sent sooner, possibly at the expense of overall throughput.
+        """
+
+    def getTcpKeepAlive() -> bool:
+        """
+        Return if C{SO_KEEPALIVE} is enabled.
+        """
+
+    def setTcpKeepAlive(enabled: bool) -> None:
+        """
+        Enable/disable C{SO_KEEPALIVE}.
+
+        Enabling C{SO_KEEPALIVE} sends packets periodically when the connection
+        is otherwise idle, usually once every two hours. They are intended
+        to allow detection of lost peers in a non-infinite amount of time.
+        """
+
+    def getHost() -> Union["IPv4Address", "IPv6Address"]:
+        """
+        Returns L{IPv4Address} or L{IPv6Address}.
+        """
+
+    def getPeer() -> Union["IPv4Address", "IPv6Address"]:
+        """
+        Returns L{IPv4Address} or L{IPv6Address}.
+        """
+
+
+class IUNIXTransport(ITransport):
+    """
+    Transport for stream-oriented unix domain connections.
+    """
+
+    def sendFileDescriptor(descriptor: int) -> None:
+        """
+        Send a duplicate of this (file, socket, pipe, etc) descriptor to the
+        other end of this connection.
+
+        The send is non-blocking and will be queued if it cannot be performed
+        immediately.  The send will be processed in order with respect to other
+        C{sendFileDescriptor} calls on this transport, but not necessarily with
+        respect to C{write} calls on this transport.  The send can only be
+        processed if there are also bytes in the normal connection-oriented send
+        buffer (ie, you must call C{write} at least as many times as you call
+        C{sendFileDescriptor}).
+
+        @param descriptor: An C{int} giving a valid file descriptor in this
+            process.  Note that a I{file descriptor} may actually refer to a
+            socket, a pipe, or anything else POSIX tries to treat in the same
+            way as a file.
+        """
+
+
+class IOpenSSLServerConnectionCreator(Interface):
+    """
+    A provider of L{IOpenSSLServerConnectionCreator} can create
+    L{OpenSSL.SSL.Connection} objects for TLS servers.
+
+    @see: L{twisted.internet.ssl}
+
+    @note: Creating OpenSSL connection objects is subtle, error-prone, and
+        security-critical.  Before implementing this interface yourself,
+        consider using L{twisted.internet.ssl.CertificateOptions} as your
+        C{contextFactory}.  (For historical reasons, that class does not
+        actually I{implement} this interface; nevertheless it is usable in all
+        Twisted APIs which require a provider of this interface.)
+    """
+
+    def serverConnectionForTLS(
+        tlsProtocol: "TLSMemoryBIOProtocol",
+    ) -> "OpenSSLConnection":
+        """
+        Create a connection for the given server protocol.
+
+        @return: an OpenSSL connection object configured appropriately for the
+            given Twisted protocol.
+        """
+
+
+class IOpenSSLClientConnectionCreator(Interface):
+    """
+    A provider of L{IOpenSSLClientConnectionCreator} can create
+    L{OpenSSL.SSL.Connection} objects for TLS clients.
+
+    @see: L{twisted.internet.ssl}
+
+    @note: Creating OpenSSL connection objects is subtle, error-prone, and
+        security-critical.  Before implementing this interface yourself,
+        consider using L{twisted.internet.ssl.optionsForClientTLS} as your
+        C{contextFactory}.
+    """
+
+    def clientConnectionForTLS(
+        tlsProtocol: "TLSMemoryBIOProtocol",
+    ) -> "OpenSSLConnection":
+        """
+        Create a connection for the given client protocol.
+
+        @param tlsProtocol: the client protocol making the request.
+
+        @return: an OpenSSL connection object configured appropriately for the
+            given Twisted protocol.
+        """
+
+
+class IProtocolNegotiationFactory(Interface):
+    """
+    A provider of L{IProtocolNegotiationFactory} can provide information about
+    the various protocols that the factory can create implementations of. This
+    can be used, for example, to provide protocol names for Next Protocol
+    Negotiation and Application Layer Protocol Negotiation.
+
+    @see: L{twisted.internet.ssl}
+    """
+
+    def acceptableProtocols() -> List[bytes]:
+        """
+        Returns a list of protocols that can be spoken by the connection
+        factory in the form of ALPN tokens, as laid out in the IANA registry
+        for ALPN tokens.
+
+        @return: a list of ALPN tokens in order of preference.
+        """
+
+
+class IOpenSSLContextFactory(Interface):
+    """
+    A provider of L{IOpenSSLContextFactory} is capable of generating
+    L{OpenSSL.SSL.Context} classes suitable for configuring TLS on a
+    connection. A provider will store enough state to be able to generate these
+    contexts as needed for individual connections.
+
+    @see: L{twisted.internet.ssl}
+    """
+
+    def getContext() -> "OpenSSLContext":
+        """
+        Returns a TLS context object, suitable for securing a TLS connection.
+        This context object will be appropriately customized for the connection
+        based on the state in this object.
+
+        @return: A TLS context object.
+        """
+
+
+class ITLSTransport(ITCPTransport):
+    """
+    A TCP transport that supports switching to TLS midstream.
+
+    Once TLS mode is started the transport will implement L{ISSLTransport}.
+    """
+
+    def startTLS(
+        contextFactory: Union[
+            IOpenSSLClientConnectionCreator, IOpenSSLServerConnectionCreator
+        ]
+    ) -> None:
+        """
+        Initiate TLS negotiation.
+
+        @param contextFactory: An object which creates appropriately configured
+            TLS connections.
+
+            For clients, use L{twisted.internet.ssl.optionsForClientTLS}; for
+            servers, use L{twisted.internet.ssl.CertificateOptions}.
+
+        @type contextFactory: L{IOpenSSLClientConnectionCreator} or
+            L{IOpenSSLServerConnectionCreator}, depending on whether this
+            L{ITLSTransport} is a server or not.  If the appropriate interface
+            is not provided by the value given for C{contextFactory}, it must
+            be an implementor of L{IOpenSSLContextFactory}.
+        """
+
+
+class ISSLTransport(ITCPTransport):
+    """
+    A SSL/TLS based transport.
+    """
+
+    def getPeerCertificate() -> object:
+        """
+        Return an object with the peer's certificate info.
+        """
+
+
+class INegotiated(ISSLTransport):
+    """
+    A TLS based transport that supports using ALPN/NPN to negotiate the
+    protocol to be used inside the encrypted tunnel.
+    """
+
+    negotiatedProtocol = Attribute(
+        """
+        The protocol selected to be spoken using ALPN/NPN. The result from ALPN
+        is preferred to the result from NPN if both were used. If the remote
+        peer does not support ALPN or NPN, or neither NPN or ALPN are available
+        on this machine, will be L{None}. Otherwise, will be the name of the
+        selected protocol as C{bytes}. Note that until the handshake has
+        completed this property may incorrectly return L{None}: wait until data
+        has been received before trusting it (see
+        https://twistedmatrix.com/trac/ticket/6024).
+        """
+    )
+
+
+class ICipher(Interface):
+    """
+    A TLS cipher.
+    """
+
+    fullName = Attribute("The fully qualified name of the cipher in L{unicode}.")
+
+
+class IAcceptableCiphers(Interface):
+    """
+    A list of acceptable ciphers for a TLS context.
+    """
+
+    def selectCiphers(availableCiphers: Tuple[ICipher]) -> Tuple[ICipher]:
+        """
+        Choose which ciphers to allow to be negotiated on a TLS connection.
+
+        @param availableCiphers: A L{tuple} of L{ICipher} which gives the names
+            of all ciphers supported by the TLS implementation in use.
+
+        @return: A L{tuple} of L{ICipher} which represents the ciphers
+            which may be negotiated on the TLS connection.  The result is
+            ordered by preference with more preferred ciphers appearing
+            earlier.
+        """
+
+
+class IProcessTransport(ITransport):
+    """
+    A process transport.
+    """
+
+    pid = Attribute(
+        "From before L{IProcessProtocol.makeConnection} is called to before "
+        "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} "
+        "giving the platform process ID of this process.  C{pid} is L{None} "
+        "at all other times."
+    )
+
+    def closeStdin() -> None:
+        """
+        Close stdin after all data has been written out.
+        """
+
+    def closeStdout() -> None:
+        """
+        Close stdout.
+        """
+
+    def closeStderr() -> None:
+        """
+        Close stderr.
+        """
+
+    def closeChildFD(descriptor: int) -> None:
+        """
+        Close a file descriptor which is connected to the child process, identified
+        by its FD in the child process.
+        """
+
+    def writeToChild(childFD: int, data: bytes) -> None:
+        """
+        Similar to L{ITransport.write} but also allows the file descriptor in
+        the child process which will receive the bytes to be specified.
+
+        @param childFD: The file descriptor to which to write.
+        @param data: The bytes to write.
+
+        @raise KeyError: If C{childFD} is not a file descriptor that was mapped
+            in the child when L{IReactorProcess.spawnProcess} was used to create
+            it.
+        """
+
+    def loseConnection() -> None:
+        """
+        Close stdin, stderr and stdout.
+        """
+
+    def signalProcess(signalID: Union[str, int]) -> None:
+        """
+        Send a signal to the process.
+
+        @param signalID: can be
+          - one of C{"KILL"}, C{"TERM"}, or C{"INT"}.
+              These will be implemented in a
+              cross-platform manner, and so should be used
+              if possible.
+          - an integer, where it represents a POSIX
+              signal ID.
+
+        @raise twisted.internet.error.ProcessExitedAlready: If the process has
+            already exited.
+        @raise OSError: If the C{os.kill} call fails with an errno different
+            from C{ESRCH}.
+        """
+
+
+class IServiceCollection(Interface):
+    """
+    An object which provides access to a collection of services.
+    """
+
+    def getServiceNamed(serviceName: str) -> object:
+        """
+        Retrieve the named service from this application.
+
+        Raise a C{KeyError} if there is no such service name.
+        """
+
+    def addService(service: object) -> None:
+        """
+        Add a service to this collection.
+        """
+
+    def removeService(service: object) -> None:
+        """
+        Remove a service from this collection.
+        """
+
+
+class IUDPTransport(Interface):
+    """
+    Transport for UDP DatagramProtocols.
+    """
+
+    def write(packet: bytes, addr: Optional[Tuple[str, int]]) -> None:
+        """
+        Write packet to given address.
+
+        @param addr: a tuple of (ip, port). For connected transports must
+                     be the address the transport is connected to, or None.
+                     In non-connected mode this is mandatory.
+
+        @raise twisted.internet.error.MessageLengthError: C{packet} was too
+        long.
+        """
+
+    def connect(host: str, port: int) -> None:
+        """
+        Connect the transport to an address.
+
+        This changes it to connected mode. Datagrams can only be sent to
+        this address, and will only be received from this address. In addition
+        the protocol's connectionRefused method might get called if destination
+        is not receiving datagrams.
+
+        @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost')
+        @param port: port to connect to.
+        """
+
+    def getHost() -> Union["IPv4Address", "IPv6Address"]:
+        """
+        Get this port's host address.
+
+        @return: an address describing the listening port.
+        """
+
+    def stopListening() -> Optional["Deferred[None]"]:
+        """
+        Stop listening on this port.
+
+        If it does not complete immediately, will return L{Deferred} that fires
+        upon completion.
+        """
+
+    def setBroadcastAllowed(enabled: bool) -> None:
+        """
+        Set whether this port may broadcast.
+
+        @param enabled: Whether the port may broadcast.
+        """
+
+    def getBroadcastAllowed() -> bool:
+        """
+        Checks if broadcast is currently allowed on this port.
+
+        @return: Whether this port may broadcast.
+        """
+
+
+class IUNIXDatagramTransport(Interface):
+    """
+    Transport for UDP PacketProtocols.
+    """
+
+    def write(packet: bytes, addr: str) -> None:
+        """
+        Write packet to given address.
+        """
+
+    def getHost() -> "UNIXAddress":
+        """
+        Returns L{UNIXAddress}.
+        """
+
+
+class IUNIXDatagramConnectedTransport(Interface):
+    """
+    Transport for UDP ConnectedPacketProtocols.
+    """
+
+    def write(packet: bytes) -> None:
+        """
+        Write packet to address we are connected to.
+        """
+
+    def getHost() -> "UNIXAddress":
+        """
+        Returns L{UNIXAddress}.
+        """
+
+    def getPeer() -> "UNIXAddress":
+        """
+        Returns L{UNIXAddress}.
+        """
+
+
+class IMulticastTransport(Interface):
+    """
+    Additional functionality for multicast UDP.
+    """
+
+    def getOutgoingInterface() -> str:
+        """
+        Return interface of outgoing multicast packets.
+        """
+
+    def setOutgoingInterface(addr: str) -> None:
+        """
+        Set interface for outgoing multicast packets.
+
+        Returns Deferred of success.
+        """
+
+    def getLoopbackMode() -> bool:
+        """
+        Return if loopback mode is enabled.
+        """
+
+    def setLoopbackMode(mode: bool) -> None:
+        """
+        Set if loopback mode is enabled.
+        """
+
+    def getTTL() -> int:
+        """
+        Get time to live for multicast packets.
+        """
+
+    def setTTL(ttl: int) -> None:
+        """
+        Set time to live on multicast packets.
+        """
+
+    def joinGroup(addr: str, interface: str) -> "Deferred[None]":
+        """
+        Join a multicast group. Returns L{Deferred} of success or failure.
+
+        If an error occurs, the returned L{Deferred} will fail with
+        L{error.MulticastJoinError}.
+        """
+
+    def leaveGroup(addr: str, interface: str) -> "Deferred[None]":
+        """
+        Leave multicast group, return L{Deferred} of success.
+        """
+
+
+class IStreamClientEndpoint(Interface):
+    """
+    A stream client endpoint is a place that L{ClientFactory} can connect to.
+    For example, a remote TCP host/port pair would be a TCP client endpoint.
+
+    @since: 10.1
+    """
+
+    def connect(protocolFactory: IProtocolFactory) -> "Deferred[IProtocol]":
+        """
+        Connect the C{protocolFactory} to the location specified by this
+        L{IStreamClientEndpoint} provider.
+
+        @param protocolFactory: A provider of L{IProtocolFactory}
+
+        @return: A L{Deferred} that results in an L{IProtocol} upon successful
+            connection otherwise a L{Failure} wrapping L{ConnectError} or
+            L{NoProtocol <twisted.internet.error.NoProtocol>}.
+        """
+
+
+class IStreamServerEndpoint(Interface):
+    """
+    A stream server endpoint is a place that a L{Factory} can listen for
+    incoming connections.
+
+    @since: 10.1
+    """
+
+    def listen(protocolFactory: IProtocolFactory) -> "Deferred[IListeningPort]":
+        """
+        Listen with C{protocolFactory} at the location specified by this
+        L{IStreamServerEndpoint} provider.
+
+        @param protocolFactory: A provider of L{IProtocolFactory}
+
+        @return: A L{Deferred} that results in an L{IListeningPort} or an
+            L{CannotListenError}
+        """
+
+
+class IStreamServerEndpointStringParser(Interface):
+    """
+    An L{IStreamServerEndpointStringParser} is like an
+    L{IStreamClientEndpointStringParserWithReactor}, except for
+    L{IStreamServerEndpoint}s instead of clients.  It integrates with
+    L{endpoints.serverFromString} in much the same way.
+    """
+
+    prefix = Attribute(
+        """
+        A C{str}, the description prefix to respond to.  For example, an
+        L{IStreamServerEndpointStringParser} plugin which had C{"foo"} for its
+        C{prefix} attribute would be called for endpoint descriptions like
+        C{"foo:bar:baz"} or C{"foo:"}.
+        """
+    )
+
+    def parseStreamServer(
+        reactor: IReactorCore, *args: object, **kwargs: object
+    ) -> IStreamServerEndpoint:
+        """
+        Parse a stream server endpoint from a reactor and string-only arguments
+        and keyword arguments.
+
+        @see: L{IStreamClientEndpointStringParserWithReactor.parseStreamClient}
+
+        @return: a stream server endpoint
+        """
+
+
+class IStreamClientEndpointStringParserWithReactor(Interface):
+    """
+    An L{IStreamClientEndpointStringParserWithReactor} is a parser which can
+    convert a set of string C{*args} and C{**kwargs} into an
+    L{IStreamClientEndpoint} provider.
+
+    This interface is really only useful in the context of the plugin system
+    for L{endpoints.clientFromString}.  See the document entitled "I{The
+    Twisted Plugin System}" for more details on how to write a plugin.
+
+    If you place an L{IStreamClientEndpointStringParserWithReactor} plugin in
+    the C{twisted.plugins} package, that plugin's C{parseStreamClient} method
+    will be used to produce endpoints for any description string that begins
+    with the result of that L{IStreamClientEndpointStringParserWithReactor}'s
+    prefix attribute.
+    """
+
+    prefix = Attribute(
+        """
+        L{bytes}, the description prefix to respond to.  For example, an
+        L{IStreamClientEndpointStringParserWithReactor} plugin which had
+        C{b"foo"} for its C{prefix} attribute would be called for endpoint
+        descriptions like C{b"foo:bar:baz"} or C{b"foo:"}.
+        """
+    )
+
+    def parseStreamClient(
+        reactor: IReactorCore, *args: object, **kwargs: object
+    ) -> IStreamClientEndpoint:
+        """
+        This method is invoked by L{endpoints.clientFromString}, if the type of
+        endpoint matches the return value from this
+        L{IStreamClientEndpointStringParserWithReactor}'s C{prefix} method.
+
+        @param reactor: The reactor passed to L{endpoints.clientFromString}.
+        @param args: The byte string arguments, minus the endpoint type, in the
+            endpoint description string, parsed according to the rules
+            described in L{endpoints.quoteStringArgument}.  For example, if the
+            description were C{b"my-type:foo:bar:baz=qux"}, C{args} would be
+            C{(b'foo', b'bar')}
+        @param kwargs: The byte string arguments from the endpoint description
+            passed as keyword arguments.  For example, if the description were
+            C{b"my-type:foo:bar:baz=qux"}, C{kwargs} would be
+            C{dict(baz=b'qux')}.
+
+        @return: a client endpoint
+        """
+
+
+class _ISupportsExitSignalCapturing(Interface):
+    """
+    An implementor of L{_ISupportsExitSignalCapturing} will capture the
+    value of any delivered exit signal (SIGINT, SIGTERM, SIGBREAK) for which
+    it has installed a handler.  The caught signal number is made available in
+    the _exitSignal attribute.
+    """
+
+    _exitSignal = Attribute(
+        """
+        C{int} or C{None}, the integer exit signal delivered to the
+        application, or None if no signal was delivered.
+        """
+    )
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/__init__.py b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/__init__.py
new file mode 100644
index 00000000000..d1881d4fe3c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/__init__.py
@@ -0,0 +1,10 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+I/O Completion Ports reactor
+"""
+
+from twisted.internet.iocpreactor.reactor import install
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/abstract.py b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/abstract.py
new file mode 100644
index 00000000000..818c86068d5
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/abstract.py
@@ -0,0 +1,387 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Abstract file handle class
+"""
+
+import errno
+
+from zope.interface import implementer
+
+from twisted.internet import error, interfaces, main
+from twisted.internet.abstract import _ConsumerMixin, _dataMustBeBytes, _LogOwner
+from twisted.internet.iocpreactor import iocpsupport as _iocp
+from twisted.internet.iocpreactor.const import ERROR_HANDLE_EOF, ERROR_IO_PENDING
+from twisted.python import failure
+
+
+@implementer(
+    interfaces.IPushProducer,
+    interfaces.IConsumer,
+    interfaces.ITransport,
+    interfaces.IHalfCloseableDescriptor,
+)
+class FileHandle(_ConsumerMixin, _LogOwner):
+    """
+    File handle that can read and write asynchronously
+    """
+
+    # read stuff
+    maxReadBuffers = 16
+    readBufferSize = 4096
+    reading = False
+    dynamicReadBuffers = True  # set this to false if subclass doesn't do iovecs
+    _readNextBuffer = 0
+    _readSize = 0  # how much data we have in the read buffer
+    _readScheduled = None
+    _readScheduledInOS = False
+
+    def startReading(self):
+        self.reactor.addActiveHandle(self)
+        if not self._readScheduled and not self.reading:
+            self.reading = True
+            self._readScheduled = self.reactor.callLater(0, self._resumeReading)
+
+    def stopReading(self):
+        if self._readScheduled:
+            self._readScheduled.cancel()
+            self._readScheduled = None
+        self.reading = False
+
+    def _resumeReading(self):
+        self._readScheduled = None
+        if self._dispatchData() and not self._readScheduledInOS:
+            self.doRead()
+
+    def _dispatchData(self):
+        """
+        Dispatch previously read data. Return True if self.reading and we don't
+        have any more data
+        """
+        if not self._readSize:
+            return self.reading
+        size = self._readSize
+        full_buffers = size // self.readBufferSize
+        while self._readNextBuffer < full_buffers:
+            self.dataReceived(self._readBuffers[self._readNextBuffer])
+            self._readNextBuffer += 1
+            if not self.reading:
+                return False
+        remainder = size % self.readBufferSize
+        if remainder:
+            self.dataReceived(self._readBuffers[full_buffers][0:remainder])
+        if self.dynamicReadBuffers:
+            total_buffer_size = self.readBufferSize * len(self._readBuffers)
+            # we have one buffer too many
+            if size < total_buffer_size - self.readBufferSize:
+                del self._readBuffers[-1]
+            # we filled all buffers, so allocate one more
+            elif (
+                size == total_buffer_size
+                and len(self._readBuffers) < self.maxReadBuffers
+            ):
+                self._readBuffers.append(bytearray(self.readBufferSize))
+        self._readNextBuffer = 0
+        self._readSize = 0
+        return self.reading
+
+    def _cbRead(self, rc, data, evt):
+        self._readScheduledInOS = False
+        if self._handleRead(rc, data, evt):
+            self.doRead()
+
+    def _handleRead(self, rc, data, evt):
+        """
+        Returns False if we should stop reading for now
+        """
+        if self.disconnected:
+            return False
+        # graceful disconnection
+        if (not (rc or data)) or rc in (errno.WSAEDISCON, ERROR_HANDLE_EOF):
+            self.reactor.removeActiveHandle(self)
+            self.readConnectionLost(failure.Failure(main.CONNECTION_DONE))
+            return False
+        # XXX: not handling WSAEWOULDBLOCK
+        # ("too many outstanding overlapped I/O requests")
+        elif rc:
+            self.connectionLost(
+                failure.Failure(
+                    error.ConnectionLost(
+                        "read error -- %s (%s)"
+                        % (errno.errorcode.get(rc, "unknown"), rc)
+                    )
+                )
+            )
+            return False
+        else:
+            assert self._readSize == 0
+            assert self._readNextBuffer == 0
+            self._readSize = data
+            return self._dispatchData()
+
+    def doRead(self):
+        evt = _iocp.Event(self._cbRead, self)
+
+        evt.buff = buff = self._readBuffers
+        rc, numBytesRead = self.readFromHandle(buff, evt)
+
+        if not rc or rc == ERROR_IO_PENDING:
+            self._readScheduledInOS = True
+        else:
+            self._handleRead(rc, numBytesRead, evt)
+
+    def readFromHandle(self, bufflist, evt):
+        raise NotImplementedError()  # TODO: this should default to ReadFile
+
+    def dataReceived(self, data):
+        raise NotImplementedError
+
+    def readConnectionLost(self, reason):
+        self.connectionLost(reason)
+
+    # write stuff
+    dataBuffer = b""
+    offset = 0
+    writing = False
+    _writeScheduled = None
+    _writeDisconnecting = False
+    _writeDisconnected = False
+    writeBufferSize = 2**2**2**2
+
+    def loseWriteConnection(self):
+        self._writeDisconnecting = True
+        self.startWriting()
+
+    def _closeWriteConnection(self):
+        # override in subclasses
+        pass
+
+    def writeConnectionLost(self, reason):
+        # in current code should never be called
+        self.connectionLost(reason)
+
+    def startWriting(self):
+        self.reactor.addActiveHandle(self)
+
+        if not self._writeScheduled and not self.writing:
+            self.writing = True
+            self._writeScheduled = self.reactor.callLater(0, self._resumeWriting)
+
+    def stopWriting(self):
+        if self._writeScheduled:
+            self._writeScheduled.cancel()
+            self._writeScheduled = None
+        self.writing = False
+
+    def _resumeWriting(self):
+        self._writeScheduled = None
+        self.doWrite()
+
+    def _cbWrite(self, rc, numBytesWritten, evt):
+        if self._handleWrite(rc, numBytesWritten, evt):
+            self.doWrite()
+
+    def _handleWrite(self, rc, numBytesWritten, evt):
+        """
+        Returns false if we should stop writing for now
+        """
+        if self.disconnected or self._writeDisconnected:
+            return False
+        # XXX: not handling WSAEWOULDBLOCK
+        # ("too many outstanding overlapped I/O requests")
+        if rc:
+            self.connectionLost(
+                failure.Failure(
+                    error.ConnectionLost(
+                        "write error -- %s (%s)"
+                        % (errno.errorcode.get(rc, "unknown"), rc)
+                    )
+                )
+            )
+            return False
+        else:
+            self.offset += numBytesWritten
+            # If there is nothing left to send,
+            if self.offset == len(self.dataBuffer) and not self._tempDataLen:
+                self.dataBuffer = b""
+                self.offset = 0
+                # stop writing
+                self.stopWriting()
+                # If I've got a producer who is supposed to supply me with data
+                if self.producer is not None and (
+                    (not self.streamingProducer) or self.producerPaused
+                ):
+                    # tell them to supply some more.
+                    self.producerPaused = True
+                    self.producer.resumeProducing()
+                elif self.disconnecting:
+                    # But if I was previously asked to let the connection die,
+                    # do so.
+                    self.connectionLost(failure.Failure(main.CONNECTION_DONE))
+                elif self._writeDisconnecting:
+                    # I was previously asked to half-close the connection.
+                    self._writeDisconnected = True
+                    self._closeWriteConnection()
+                return False
+            else:
+                return True
+
+    def doWrite(self):
+        if len(self.dataBuffer) - self.offset < self.SEND_LIMIT:
+            # If there is currently less than SEND_LIMIT bytes left to send
+            # in the string, extend it with the array data.
+            self.dataBuffer = self.dataBuffer[self.offset :] + b"".join(
+                self._tempDataBuffer
+            )
+            self.offset = 0
+            self._tempDataBuffer = []
+            self._tempDataLen = 0
+
+        evt = _iocp.Event(self._cbWrite, self)
+
+        # Send as much data as you can.
+        if self.offset:
+            sendView = memoryview(self.dataBuffer)
+            evt.buff = buff = sendView[self.offset :]
+        else:
+            evt.buff = buff = self.dataBuffer
+        rc, data = self.writeToHandle(buff, evt)
+        if rc and rc != ERROR_IO_PENDING:
+            self._handleWrite(rc, data, evt)
+
+    def writeToHandle(self, buff, evt):
+        raise NotImplementedError()  # TODO: this should default to WriteFile
+
+    def write(self, data):
+        """Reliably write some data.
+
+        The data is buffered until his file descriptor is ready for writing.
+        """
+        _dataMustBeBytes(data)
+        if not self.connected or self._writeDisconnected:
+            return
+        if data:
+            self._tempDataBuffer.append(data)
+            self._tempDataLen += len(data)
+            if self.producer is not None and self.streamingProducer:
+                if len(self.dataBuffer) + self._tempDataLen > self.writeBufferSize:
+                    self.producerPaused = True
+                    self.producer.pauseProducing()
+            self.startWriting()
+
+    def writeSequence(self, iovec):
+        for i in iovec:
+            _dataMustBeBytes(i)
+        if not self.connected or not iovec or self._writeDisconnected:
+            return
+        self._tempDataBuffer.extend(iovec)
+        for i in iovec:
+            self._tempDataLen += len(i)
+        if self.producer is not None and self.streamingProducer:
+            if len(self.dataBuffer) + self._tempDataLen > self.writeBufferSize:
+                self.producerPaused = True
+                self.producer.pauseProducing()
+        self.startWriting()
+
+    # general stuff
+    connected = False
+    disconnected = False
+    disconnecting = False
+    logstr = "Uninitialized"
+
+    SEND_LIMIT = 128 * 1024
+
+    def __init__(self, reactor=None):
+        if not reactor:
+            from twisted.internet import reactor
+        self.reactor = reactor
+        self._tempDataBuffer = []  # will be added to dataBuffer in doWrite
+        self._tempDataLen = 0
+        self._readBuffers = [bytearray(self.readBufferSize)]
+
+    def connectionLost(self, reason):
+        """
+        The connection was lost.
+
+        This is called when the connection on a selectable object has been
+        lost.  It will be called whether the connection was closed explicitly,
+        an exception occurred in an event handler, or the other end of the
+        connection closed it first.
+
+        Clean up state here, but make sure to call back up to FileDescriptor.
+        """
+
+        self.disconnected = True
+        self.connected = False
+        if self.producer is not None:
+            self.producer.stopProducing()
+            self.producer = None
+        self.stopReading()
+        self.stopWriting()
+        self.reactor.removeActiveHandle(self)
+
+    def getFileHandle(self):
+        return -1
+
+    def loseConnection(self, _connDone=failure.Failure(main.CONNECTION_DONE)):
+        """
+        Close the connection at the next available opportunity.
+
+        Call this to cause this FileDescriptor to lose its connection.  It will
+        first write any data that it has buffered.
+
+        If there is data buffered yet to be written, this method will cause the
+        transport to lose its connection as soon as it's done flushing its
+        write buffer.  If you have a producer registered, the connection won't
+        be closed until the producer is finished. Therefore, make sure you
+        unregister your producer when it's finished, or the connection will
+        never close.
+        """
+
+        if self.connected and not self.disconnecting:
+            if self._writeDisconnected:
+                # doWrite won't trigger the connection close anymore
+                self.stopReading()
+                self.stopWriting
+                self.connectionLost(_connDone)
+            else:
+                self.stopReading()
+                self.startWriting()
+                self.disconnecting = 1
+
+    # Producer/consumer implementation
+
+    def stopConsuming(self):
+        """
+        Stop consuming data.
+
+        This is called when a producer has lost its connection, to tell the
+        consumer to go lose its connection (and break potential circular
+        references).
+        """
+        self.unregisterProducer()
+        self.loseConnection()
+
+    # producer interface implementation
+
+    def resumeProducing(self):
+        if self.connected and not self.disconnecting:
+            self.startReading()
+
+    def pauseProducing(self):
+        self.stopReading()
+
+    def stopProducing(self):
+        self.loseConnection()
+
+    def getHost(self):
+        # ITransport.getHost
+        raise NotImplementedError()
+
+    def getPeer(self):
+        # ITransport.getPeer
+        raise NotImplementedError()
+
+
+__all__ = ["FileHandle"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/const.py b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/const.py
new file mode 100644
index 00000000000..4814425af9e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/const.py
@@ -0,0 +1,25 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Windows constants for IOCP
+"""
+
+
+# this stuff should really be gotten from Windows headers via pyrex, but it
+# probably is not going to change
+
+ERROR_PORT_UNREACHABLE = 1234
+ERROR_NETWORK_UNREACHABLE = 1231
+ERROR_CONNECTION_REFUSED = 1225
+ERROR_IO_PENDING = 997
+ERROR_OPERATION_ABORTED = 995
+WAIT_TIMEOUT = 258
+ERROR_NETNAME_DELETED = 64
+ERROR_HANDLE_EOF = 38
+
+INFINITE = -1
+
+SO_UPDATE_CONNECT_CONTEXT = 0x7010
+SO_UPDATE_ACCEPT_CONTEXT = 0x700B
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/interfaces.py b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/interfaces.py
new file mode 100644
index 00000000000..b161341efa6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/interfaces.py
@@ -0,0 +1,42 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Interfaces for iocpreactor
+"""
+
+
+from zope.interface import Interface
+
+
+class IReadHandle(Interface):
+    def readFromHandle(bufflist, evt):
+        """
+        Read into the given buffers from this handle.
+
+        @param bufflist: the buffers to read into
+        @type bufflist: list of objects implementing the read/write buffer protocol
+
+        @param evt: an IOCP Event object
+
+        @return: tuple (return code, number of bytes read)
+        """
+
+
+class IWriteHandle(Interface):
+    def writeToHandle(buff, evt):
+        """
+        Write the given buffer to this handle.
+
+        @param buff: the buffer to write
+        @type buff: any object implementing the buffer protocol
+
+        @param evt: an IOCP Event object
+
+        @return: tuple (return code, number of bytes written)
+        """
+
+
+class IReadWriteHandle(IReadHandle, IWriteHandle):
+    pass
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/iocpsupport.py b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/iocpsupport.py
new file mode 100644
index 00000000000..826c976487f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/iocpsupport.py
@@ -0,0 +1,27 @@
+__all__ = [
+    "CompletionPort",
+    "Event",
+    "accept",
+    "connect",
+    "get_accept_addrs",
+    "have_connectex",
+    "makesockaddr",
+    "maxAddrLen",
+    "recv",
+    "recvfrom",
+    "send",
+]
+
+from twisted_iocpsupport.iocpsupport import (  # type: ignore[import]
+    CompletionPort,
+    Event,
+    accept,
+    connect,
+    get_accept_addrs,
+    have_connectex,
+    makesockaddr,
+    maxAddrLen,
+    recv,
+    recvfrom,
+    send,
+)
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/notes.txt b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/notes.txt
new file mode 100644
index 00000000000..4caffb882f1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/notes.txt
@@ -0,0 +1,24 @@
+test specifically:
+failed accept error message -- similar to test_tcp_internals
+immediate success on accept/connect/recv, including Event.ignore
+parametrize iocpsupport somehow -- via reactor?
+
+do:
+break handling -- WaitForSingleObject on the IOCP handle?
+iovecs for write buffer
+do not wait for a mainloop iteration if resumeProducing (in _handleWrite) does startWriting
+don't addActiveHandle in every call to startWriting/startReading
+iocpified process support
+  win32er-in-a-thread (or run GQCS in a thread -- it can't receive SIGBREAK)
+blocking in sendto() -- I think Windows can do that, especially with local UDP
+
+buildbot:
+run in vmware
+start from a persistent snapshot
+
+use a stub inside the vm to svnup/run tests/collect stdio
+lift logs through SMB? or ship them via tcp beams to the VM host
+
+have a timeout on the test run
+if we time out, take a screenshot, save it, kill the VM
+
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/reactor.py b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/reactor.py
new file mode 100644
index 00000000000..e9c3716219f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/reactor.py
@@ -0,0 +1,285 @@
+# -*- test-case-name: twisted.internet.test.test_iocp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Reactor that uses IO completion ports
+"""
+
+
+import socket
+import sys
+import warnings
+from typing import Tuple, Type
+
+from zope.interface import implementer
+
+from twisted.internet import base, error, interfaces, main
+from twisted.internet._dumbwin32proc import Process
+from twisted.internet.iocpreactor import iocpsupport as _iocp, tcp, udp
+from twisted.internet.iocpreactor.const import WAIT_TIMEOUT
+from twisted.internet.win32eventreactor import _ThreadedWin32EventsMixin
+from twisted.python import failure, log
+
+try:
+    from twisted.protocols.tls import TLSMemoryBIOFactory as _TLSMemoryBIOFactory
+except ImportError:
+    TLSMemoryBIOFactory = None
+    # Either pyOpenSSL isn't installed, or it is too old for this code to work.
+    # The reactor won't provide IReactorSSL.
+    _extraInterfaces: Tuple[Type[interfaces.IReactorSSL], ...] = ()
+    warnings.warn(
+        "pyOpenSSL 0.10 or newer is required for SSL support in iocpreactor. "
+        "It is missing, so the reactor will not support SSL APIs."
+    )
+else:
+    TLSMemoryBIOFactory = _TLSMemoryBIOFactory
+    _extraInterfaces = (interfaces.IReactorSSL,)
+
+MAX_TIMEOUT = 2000  # 2 seconds, see doIteration for explanation
+
+EVENTS_PER_LOOP = 1000  # XXX: what's a good value here?
+
+# keys to associate with normal and waker events
+KEY_NORMAL, KEY_WAKEUP = range(2)
+
+_NO_GETHANDLE = error.ConnectionFdescWentAway("Handler has no getFileHandle method")
+_NO_FILEDESC = error.ConnectionFdescWentAway("Filedescriptor went away")
+
+
+@implementer(
+    interfaces.IReactorTCP,
+    interfaces.IReactorUDP,
+    interfaces.IReactorMulticast,
+    interfaces.IReactorProcess,
+    *_extraInterfaces,
+)
+class IOCPReactor(base.ReactorBase, _ThreadedWin32EventsMixin):
+    port = None
+
+    def __init__(self):
+        base.ReactorBase.__init__(self)
+        self.port = _iocp.CompletionPort()
+        self.handles = set()
+
+    def addActiveHandle(self, handle):
+        self.handles.add(handle)
+
+    def removeActiveHandle(self, handle):
+        self.handles.discard(handle)
+
+    def doIteration(self, timeout):
+        """
+        Poll the IO completion port for new events.
+        """
+        # This function sits and waits for an IO completion event.
+        #
+        # There are two requirements: process IO events as soon as they arrive
+        # and process ctrl-break from the user in a reasonable amount of time.
+        #
+        # There are three kinds of waiting.
+        # 1) GetQueuedCompletionStatus (self.port.getEvent) to wait for IO
+        # events only.
+        # 2) Msg* family of wait functions that can stop waiting when
+        # ctrl-break is detected (then, I think, Python converts it into a
+        # KeyboardInterrupt)
+        # 3) *Ex family of wait functions that put the thread into an
+        # "alertable" wait state which is supposedly triggered by IO completion
+        #
+        # 2) and 3) can be combined. Trouble is, my IO completion is not
+        # causing 3) to trigger, possibly because I do not use an IO completion
+        # callback. Windows is weird.
+        # There are two ways to handle this. I could use MsgWaitForSingleObject
+        # here and GetQueuedCompletionStatus in a thread. Or I could poll with
+        # a reasonable interval. Guess what! Threads are hard.
+
+        processed_events = 0
+        if timeout is None:
+            timeout = MAX_TIMEOUT
+        else:
+            timeout = min(MAX_TIMEOUT, int(1000 * timeout))
+        rc, numBytes, key, evt = self.port.getEvent(timeout)
+        while 1:
+            if rc == WAIT_TIMEOUT:
+                break
+            if key != KEY_WAKEUP:
+                assert key == KEY_NORMAL
+                log.callWithLogger(
+                    evt.owner, self._callEventCallback, rc, numBytes, evt
+                )
+                processed_events += 1
+            if processed_events >= EVENTS_PER_LOOP:
+                break
+            rc, numBytes, key, evt = self.port.getEvent(0)
+
+    def _callEventCallback(self, rc, numBytes, evt):
+        owner = evt.owner
+        why = None
+        try:
+            evt.callback(rc, numBytes, evt)
+            handfn = getattr(owner, "getFileHandle", None)
+            if not handfn:
+                why = _NO_GETHANDLE
+            elif handfn() == -1:
+                why = _NO_FILEDESC
+            if why:
+                return  # ignore handles that were closed
+        except BaseException:
+            why = sys.exc_info()[1]
+            log.err()
+        if why:
+            owner.loseConnection(failure.Failure(why))
+
+    def installWaker(self):
+        pass
+
+    def wakeUp(self):
+        self.port.postEvent(0, KEY_WAKEUP, None)
+
+    def registerHandle(self, handle):
+        self.port.addHandle(handle, KEY_NORMAL)
+
+    def createSocket(self, af, stype):
+        skt = socket.socket(af, stype)
+        self.registerHandle(skt.fileno())
+        return skt
+
+    def listenTCP(self, port, factory, backlog=50, interface=""):
+        """
+        @see: twisted.internet.interfaces.IReactorTCP.listenTCP
+        """
+        p = tcp.Port(port, factory, backlog, interface, self)
+        p.startListening()
+        return p
+
+    def connectTCP(self, host, port, factory, timeout=30, bindAddress=None):
+        """
+        @see: twisted.internet.interfaces.IReactorTCP.connectTCP
+        """
+        c = tcp.Connector(host, port, factory, timeout, bindAddress, self)
+        c.connect()
+        return c
+
+    if TLSMemoryBIOFactory is not None:
+
+        def listenSSL(self, port, factory, contextFactory, backlog=50, interface=""):
+            """
+            @see: twisted.internet.interfaces.IReactorSSL.listenSSL
+            """
+            port = self.listenTCP(
+                port,
+                TLSMemoryBIOFactory(contextFactory, False, factory),
+                backlog,
+                interface,
+            )
+            port._type = "TLS"
+            return port
+
+        def connectSSL(
+            self, host, port, factory, contextFactory, timeout=30, bindAddress=None
+        ):
+            """
+            @see: twisted.internet.interfaces.IReactorSSL.connectSSL
+            """
+            return self.connectTCP(
+                host,
+                port,
+                TLSMemoryBIOFactory(contextFactory, True, factory),
+                timeout,
+                bindAddress,
+            )
+
+    else:
+
+        def listenSSL(self, port, factory, contextFactory, backlog=50, interface=""):
+            """
+            Non-implementation of L{IReactorSSL.listenSSL}.  Some dependency
+            is not satisfied.  This implementation always raises
+            L{NotImplementedError}.
+            """
+            raise NotImplementedError(
+                "pyOpenSSL 0.10 or newer is required for SSL support in "
+                "iocpreactor. It is missing, so the reactor does not support "
+                "SSL APIs."
+            )
+
+        def connectSSL(
+            self, host, port, factory, contextFactory, timeout=30, bindAddress=None
+        ):
+            """
+            Non-implementation of L{IReactorSSL.connectSSL}.  Some dependency
+            is not satisfied.  This implementation always raises
+            L{NotImplementedError}.
+            """
+            raise NotImplementedError(
+                "pyOpenSSL 0.10 or newer is required for SSL support in "
+                "iocpreactor. It is missing, so the reactor does not support "
+                "SSL APIs."
+            )
+
+    def listenUDP(self, port, protocol, interface="", maxPacketSize=8192):
+        """
+        Connects a given L{DatagramProtocol} to the given numeric UDP port.
+
+        @returns: object conforming to L{IListeningPort}.
+        """
+        p = udp.Port(port, protocol, interface, maxPacketSize, self)
+        p.startListening()
+        return p
+
+    def listenMulticast(
+        self, port, protocol, interface="", maxPacketSize=8192, listenMultiple=False
+    ):
+        """
+        Connects a given DatagramProtocol to the given numeric UDP port.
+
+        EXPERIMENTAL.
+
+        @returns: object conforming to IListeningPort.
+        """
+        p = udp.MulticastPort(
+            port, protocol, interface, maxPacketSize, self, listenMultiple
+        )
+        p.startListening()
+        return p
+
+    def spawnProcess(
+        self,
+        processProtocol,
+        executable,
+        args=(),
+        env={},
+        path=None,
+        uid=None,
+        gid=None,
+        usePTY=0,
+        childFDs=None,
+    ):
+        """
+        Spawn a process.
+        """
+        if uid is not None:
+            raise ValueError("Setting UID is unsupported on this platform.")
+        if gid is not None:
+            raise ValueError("Setting GID is unsupported on this platform.")
+        if usePTY:
+            raise ValueError("PTYs are unsupported on this platform.")
+        if childFDs is not None:
+            raise ValueError(
+                "Custom child file descriptor mappings are unsupported on "
+                "this platform."
+            )
+        return Process(self, processProtocol, executable, args, env, path)
+
+    def removeAll(self):
+        res = list(self.handles)
+        self.handles.clear()
+        return res
+
+
+def install():
+    r = IOCPReactor()
+    main.installReactor(r)
+
+
+__all__ = ["IOCPReactor", "install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/tcp.py b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/tcp.py
new file mode 100644
index 00000000000..aadd685269c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/tcp.py
@@ -0,0 +1,608 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+TCP support for IOCP reactor
+"""
+
+import errno
+import socket
+import struct
+from typing import Optional
+
+from zope.interface import classImplements, implementer
+
+from twisted.internet import address, defer, error, interfaces, main
+from twisted.internet.abstract import _LogOwner, isIPv6Address
+from twisted.internet.iocpreactor import abstract, iocpsupport as _iocp
+from twisted.internet.iocpreactor.const import (
+    ERROR_CONNECTION_REFUSED,
+    ERROR_IO_PENDING,
+    ERROR_NETWORK_UNREACHABLE,
+    SO_UPDATE_ACCEPT_CONTEXT,
+    SO_UPDATE_CONNECT_CONTEXT,
+)
+from twisted.internet.iocpreactor.interfaces import IReadWriteHandle
+from twisted.internet.protocol import Protocol
+from twisted.internet.tcp import (
+    Connector as TCPConnector,
+    _AbortingMixin,
+    _BaseBaseClient,
+    _BaseTCPClient,
+    _getsockname,
+    _resolveIPv6,
+    _SocketCloser,
+)
+from twisted.python import failure, log, reflect
+
+try:
+    from twisted.internet._newtls import startTLS as __startTLS
+except ImportError:
+    _startTLS = None
+else:
+    _startTLS = __startTLS
+
+
+# ConnectEx returns these. XXX: find out what it does for timeout
+connectExErrors = {
+    ERROR_CONNECTION_REFUSED: errno.WSAECONNREFUSED,  # type: ignore[attr-defined]
+    ERROR_NETWORK_UNREACHABLE: errno.WSAENETUNREACH,  # type: ignore[attr-defined]
+}
+
+
+@implementer(IReadWriteHandle, interfaces.ITCPTransport, interfaces.ISystemHandle)
+class Connection(abstract.FileHandle, _SocketCloser, _AbortingMixin):
+    """
+    @ivar TLS: C{False} to indicate the connection is in normal TCP mode,
+        C{True} to indicate that TLS has been started and that operations must
+        be routed through the L{TLSMemoryBIOProtocol} instance.
+    """
+
+    TLS = False
+
+    def __init__(self, sock, proto, reactor=None):
+        abstract.FileHandle.__init__(self, reactor)
+        self.socket = sock
+        self.getFileHandle = sock.fileno
+        self.protocol = proto
+
+    def getHandle(self):
+        return self.socket
+
+    def dataReceived(self, rbuffer):
+        """
+        @param rbuffer: Data received.
+        @type rbuffer: L{bytes} or L{bytearray}
+        """
+        if isinstance(rbuffer, bytes):
+            pass
+        elif isinstance(rbuffer, bytearray):
+            # XXX: some day, we'll have protocols that can handle raw buffers
+            rbuffer = bytes(rbuffer)
+        else:
+            raise TypeError("data must be bytes or bytearray, not " + type(rbuffer))
+
+        self.protocol.dataReceived(rbuffer)
+
+    def readFromHandle(self, bufflist, evt):
+        return _iocp.recv(self.getFileHandle(), bufflist, evt)
+
+    def writeToHandle(self, buff, evt):
+        """
+        Send C{buff} to current file handle using C{_iocp.send}. The buffer
+        sent is limited to a size of C{self.SEND_LIMIT}.
+        """
+        writeView = memoryview(buff)
+        return _iocp.send(
+            self.getFileHandle(), writeView[0 : self.SEND_LIMIT].tobytes(), evt
+        )
+
+    def _closeWriteConnection(self):
+        try:
+            self.socket.shutdown(1)
+        except OSError:
+            pass
+        p = interfaces.IHalfCloseableProtocol(self.protocol, None)
+        if p:
+            try:
+                p.writeConnectionLost()
+            except BaseException:
+                f = failure.Failure()
+                log.err()
+                self.connectionLost(f)
+
+    def readConnectionLost(self, reason):
+        p = interfaces.IHalfCloseableProtocol(self.protocol, None)
+        if p:
+            try:
+                p.readConnectionLost()
+            except BaseException:
+                log.err()
+                self.connectionLost(failure.Failure())
+        else:
+            self.connectionLost(reason)
+
+    def connectionLost(self, reason):
+        if self.disconnected:
+            return
+        abstract.FileHandle.connectionLost(self, reason)
+        isClean = reason is None or not reason.check(error.ConnectionAborted)
+        self._closeSocket(isClean)
+        protocol = self.protocol
+        del self.protocol
+        del self.socket
+        del self.getFileHandle
+        protocol.connectionLost(reason)
+
+    def logPrefix(self):
+        """
+        Return the prefix to log with when I own the logging thread.
+        """
+        return self.logstr
+
+    def getTcpNoDelay(self):
+        return bool(self.socket.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY))
+
+    def setTcpNoDelay(self, enabled):
+        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, enabled)
+
+    def getTcpKeepAlive(self):
+        return bool(self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE))
+
+    def setTcpKeepAlive(self, enabled):
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, enabled)
+
+    if _startTLS is not None:
+
+        def startTLS(self, contextFactory, normal=True):
+            """
+            @see: L{ITLSTransport.startTLS}
+            """
+            _startTLS(self, contextFactory, normal, abstract.FileHandle)
+
+    def write(self, data):
+        """
+        Write some data, either directly to the underlying handle or, if TLS
+        has been started, to the L{TLSMemoryBIOProtocol} for it to encrypt and
+        send.
+
+        @see: L{twisted.internet.interfaces.ITransport.write}
+        """
+        if self.disconnected:
+            return
+        if self.TLS:
+            self.protocol.write(data)
+        else:
+            abstract.FileHandle.write(self, data)
+
+    def writeSequence(self, iovec):
+        """
+        Write some data, either directly to the underlying handle or, if TLS
+        has been started, to the L{TLSMemoryBIOProtocol} for it to encrypt and
+        send.
+
+        @see: L{twisted.internet.interfaces.ITransport.writeSequence}
+        """
+        if self.disconnected:
+            return
+        if self.TLS:
+            self.protocol.writeSequence(iovec)
+        else:
+            abstract.FileHandle.writeSequence(self, iovec)
+
+    def loseConnection(self, reason=None):
+        """
+        Close the underlying handle or, if TLS has been started, first shut it
+        down.
+
+        @see: L{twisted.internet.interfaces.ITransport.loseConnection}
+        """
+        if self.TLS:
+            if self.connected and not self.disconnecting:
+                self.protocol.loseConnection()
+        else:
+            abstract.FileHandle.loseConnection(self, reason)
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register a producer.
+
+        If TLS is enabled, the TLS connection handles this.
+        """
+        if self.TLS:
+            # Registering a producer before we're connected shouldn't be a
+            # problem. If we end up with a write(), that's already handled in
+            # the write() code above, and there are no other potential
+            # side-effects.
+            self.protocol.registerProducer(producer, streaming)
+        else:
+            abstract.FileHandle.registerProducer(self, producer, streaming)
+
+    def unregisterProducer(self):
+        """
+        Unregister a producer.
+
+        If TLS is enabled, the TLS connection handles this.
+        """
+        if self.TLS:
+            self.protocol.unregisterProducer()
+        else:
+            abstract.FileHandle.unregisterProducer(self)
+
+    def getHost(self):
+        # ITCPTransport.getHost
+        pass
+
+    def getPeer(self):
+        # ITCPTransport.getPeer
+        pass
+
+
+if _startTLS is not None:
+    classImplements(Connection, interfaces.ITLSTransport)
+
+
+class Client(_BaseBaseClient, _BaseTCPClient, Connection):
+    """
+    @ivar _tlsClientDefault: Always C{True}, indicating that this is a client
+        connection, and by default when TLS is negotiated this class will act as
+        a TLS client.
+    """
+
+    addressFamily = socket.AF_INET
+    socketType = socket.SOCK_STREAM
+
+    _tlsClientDefault = True
+    _commonConnection = Connection
+
+    def __init__(self, host, port, bindAddress, connector, reactor):
+        # ConnectEx documentation says socket _has_ to be bound
+        if bindAddress is None:
+            bindAddress = ("", 0)
+        self.reactor = reactor  # createInternetSocket needs this
+        _BaseTCPClient.__init__(self, host, port, bindAddress, connector, reactor)
+
+    def createInternetSocket(self):
+        """
+        Create a socket registered with the IOCP reactor.
+
+        @see: L{_BaseTCPClient}
+        """
+        return self.reactor.createSocket(self.addressFamily, self.socketType)
+
+    def _collectSocketDetails(self):
+        """
+        Clean up potentially circular references to the socket and to its
+        C{getFileHandle} method.
+
+        @see: L{_BaseBaseClient}
+        """
+        del self.socket, self.getFileHandle
+
+    def _stopReadingAndWriting(self):
+        """
+        Remove the active handle from the reactor.
+
+        @see: L{_BaseBaseClient}
+        """
+        self.reactor.removeActiveHandle(self)
+
+    def cbConnect(self, rc, data, evt):
+        if rc:
+            rc = connectExErrors.get(rc, rc)
+            self.failIfNotConnected(
+                error.getConnectError((rc, errno.errorcode.get(rc, "Unknown error")))
+            )
+        else:
+            self.socket.setsockopt(
+                socket.SOL_SOCKET,
+                SO_UPDATE_CONNECT_CONTEXT,
+                struct.pack("P", self.socket.fileno()),
+            )
+            self.protocol = self.connector.buildProtocol(self.getPeer())
+            self.connected = True
+            logPrefix = self._getLogPrefix(self.protocol)
+            self.logstr = logPrefix + ",client"
+            if self.protocol is None:
+                # Factory.buildProtocol is allowed to return None.  In that
+                # case, make up a protocol to satisfy the rest of the
+                # implementation; connectionLost is going to be called on
+                # something, for example.  This is easier than adding special
+                # case support for a None protocol throughout the rest of the
+                # transport implementation.
+                self.protocol = Protocol()
+                # But dispose of the connection quickly.
+                self.loseConnection()
+            else:
+                self.protocol.makeConnection(self)
+                self.startReading()
+
+    def doConnect(self):
+        if not hasattr(self, "connector"):
+            # this happens if we connector.stopConnecting in
+            # factory.startedConnecting
+            return
+        assert _iocp.have_connectex
+        self.reactor.addActiveHandle(self)
+        evt = _iocp.Event(self.cbConnect, self)
+
+        rc = _iocp.connect(self.socket.fileno(), self.realAddress, evt)
+        if rc and rc != ERROR_IO_PENDING:
+            self.cbConnect(rc, 0, evt)
+
+
+class Server(Connection):
+    """
+    Serverside socket-stream connection class.
+
+    I am a serverside network connection transport; a socket which came from an
+    accept() on a server.
+
+    @ivar _tlsClientDefault: Always C{False}, indicating that this is a server
+        connection, and by default when TLS is negotiated this class will act as
+        a TLS server.
+    """
+
+    _tlsClientDefault = False
+
+    def __init__(self, sock, protocol, clientAddr, serverAddr, sessionno, reactor):
+        """
+        Server(sock, protocol, client, server, sessionno)
+
+        Initialize me with a socket, a protocol, a descriptor for my peer (a
+        tuple of host, port describing the other end of the connection), an
+        instance of Port, and a session number.
+        """
+        Connection.__init__(self, sock, protocol, reactor)
+        self.serverAddr = serverAddr
+        self.clientAddr = clientAddr
+        self.sessionno = sessionno
+        logPrefix = self._getLogPrefix(self.protocol)
+        self.logstr = f"{logPrefix},{sessionno},{self.clientAddr.host}"
+        self.repstr: str = "<{} #{} on {}>".format(
+            self.protocol.__class__.__name__,
+            self.sessionno,
+            self.serverAddr.port,
+        )
+        self.connected = True
+        self.startReading()
+
+    def __repr__(self) -> str:
+        """
+        A string representation of this connection.
+        """
+        return self.repstr
+
+    def getHost(self):
+        """
+        Returns an IPv4Address.
+
+        This indicates the server's address.
+        """
+        return self.serverAddr
+
+    def getPeer(self):
+        """
+        Returns an IPv4Address.
+
+        This indicates the client's address.
+        """
+        return self.clientAddr
+
+
+class Connector(TCPConnector):
+    def _makeTransport(self):
+        return Client(self.host, self.port, self.bindAddress, self, self.reactor)
+
+
+@implementer(interfaces.IListeningPort)
+class Port(_SocketCloser, _LogOwner):
+    connected = False
+    disconnected = False
+    disconnecting = False
+    addressFamily = socket.AF_INET
+    socketType = socket.SOCK_STREAM
+    _addressType = address.IPv4Address
+    sessionno = 0
+
+    # Actual port number being listened on, only set to a non-None
+    # value when we are actually listening.
+    _realPortNumber: Optional[int] = None
+
+    # A string describing the connections which will be created by this port.
+    # Normally this is C{"TCP"}, since this is a TCP port, but when the TLS
+    # implementation re-uses this class it overrides the value with C{"TLS"}.
+    # Only used for logging.
+    _type = "TCP"
+
+    def __init__(self, port, factory, backlog=50, interface="", reactor=None):
+        self.port = port
+        self.factory = factory
+        self.backlog = backlog
+        self.interface = interface
+        self.reactor = reactor
+        if isIPv6Address(interface):
+            self.addressFamily = socket.AF_INET6
+            self._addressType = address.IPv6Address
+
+    def __repr__(self) -> str:
+        if self._realPortNumber is not None:
+            return "<{} of {} on {}>".format(
+                self.__class__,
+                self.factory.__class__,
+                self._realPortNumber,
+            )
+        else:
+            return "<{} of {} (not listening)>".format(
+                self.__class__,
+                self.factory.__class__,
+            )
+
+    def startListening(self):
+        try:
+            skt = self.reactor.createSocket(self.addressFamily, self.socketType)
+            # TODO: resolve self.interface if necessary
+            if self.addressFamily == socket.AF_INET6:
+                addr = _resolveIPv6(self.interface, self.port)
+            else:
+                addr = (self.interface, self.port)
+            skt.bind(addr)
+        except OSError as le:
+            raise error.CannotListenError(self.interface, self.port, le)
+
+        self.addrLen = _iocp.maxAddrLen(skt.fileno())
+
+        # Make sure that if we listened on port 0, we update that to
+        # reflect what the OS actually assigned us.
+        self._realPortNumber = skt.getsockname()[1]
+
+        log.msg(
+            "%s starting on %s"
+            % (self._getLogPrefix(self.factory), self._realPortNumber)
+        )
+
+        self.factory.doStart()
+        skt.listen(self.backlog)
+        self.connected = True
+        self.disconnected = False
+        self.reactor.addActiveHandle(self)
+        self.socket = skt
+        self.getFileHandle = self.socket.fileno
+        self.doAccept()
+
+    def loseConnection(self, connDone=failure.Failure(main.CONNECTION_DONE)):
+        """
+        Stop accepting connections on this port.
+
+        This will shut down my socket and call self.connectionLost().
+        It returns a deferred which will fire successfully when the
+        port is actually closed.
+        """
+        self.disconnecting = True
+        if self.connected:
+            self.deferred = defer.Deferred()
+            self.reactor.callLater(0, self.connectionLost, connDone)
+            return self.deferred
+
+    stopListening = loseConnection
+
+    def _logConnectionLostMsg(self):
+        """
+        Log message for closing port
+        """
+        log.msg(f"({self._type} Port {self._realPortNumber} Closed)")
+
+    def connectionLost(self, reason):
+        """
+        Cleans up the socket.
+        """
+        self._logConnectionLostMsg()
+        self._realPortNumber = None
+        d = None
+        if hasattr(self, "deferred"):
+            d = self.deferred
+            del self.deferred
+
+        self.disconnected = True
+        self.reactor.removeActiveHandle(self)
+        self.connected = False
+        self._closeSocket(True)
+        del self.socket
+        del self.getFileHandle
+
+        try:
+            self.factory.doStop()
+        except BaseException:
+            self.disconnecting = False
+            if d is not None:
+                d.errback(failure.Failure())
+            else:
+                raise
+        else:
+            self.disconnecting = False
+            if d is not None:
+                d.callback(None)
+
+    def logPrefix(self):
+        """
+        Returns the name of my class, to prefix log entries with.
+        """
+        return reflect.qual(self.factory.__class__)
+
+    def getHost(self):
+        """
+        Returns an IPv4Address or IPv6Address.
+
+        This indicates the server's address.
+        """
+        return self._addressType("TCP", *_getsockname(self.socket))
+
+    def cbAccept(self, rc, data, evt):
+        self.handleAccept(rc, evt)
+        if not (self.disconnecting or self.disconnected):
+            self.doAccept()
+
+    def handleAccept(self, rc, evt):
+        if self.disconnecting or self.disconnected:
+            return False
+
+        # possible errors:
+        # (WSAEMFILE, WSAENOBUFS, WSAENFILE, WSAENOMEM, WSAECONNABORTED)
+        if rc:
+            log.msg(
+                "Could not accept new connection -- %s (%s)"
+                % (errno.errorcode.get(rc, "unknown error"), rc)
+            )
+            return False
+        else:
+            # Inherit the properties from the listening port socket as
+            # documented in the `Remarks` section of AcceptEx.
+            # https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-acceptex
+            # In this way we can call getsockname and getpeername on the
+            # accepted socket.
+            evt.newskt.setsockopt(
+                socket.SOL_SOCKET,
+                SO_UPDATE_ACCEPT_CONTEXT,
+                struct.pack("P", self.socket.fileno()),
+            )
+            family, lAddr, rAddr = _iocp.get_accept_addrs(evt.newskt.fileno(), evt.buff)
+            assert family == self.addressFamily
+
+            # Build an IPv6 address that includes the scopeID, if necessary
+            if "%" in lAddr[0]:
+                scope = int(lAddr[0].split("%")[1])
+                lAddr = (lAddr[0], lAddr[1], 0, scope)
+            if "%" in rAddr[0]:
+                scope = int(rAddr[0].split("%")[1])
+                rAddr = (rAddr[0], rAddr[1], 0, scope)
+
+            protocol = self.factory.buildProtocol(self._addressType("TCP", *rAddr))
+            if protocol is None:
+                evt.newskt.close()
+            else:
+                s = self.sessionno
+                self.sessionno = s + 1
+                transport = Server(
+                    evt.newskt,
+                    protocol,
+                    self._addressType("TCP", *rAddr),
+                    self._addressType("TCP", *lAddr),
+                    s,
+                    self.reactor,
+                )
+                protocol.makeConnection(transport)
+            return True
+
+    def doAccept(self):
+        evt = _iocp.Event(self.cbAccept, self)
+
+        # see AcceptEx documentation
+        evt.buff = buff = bytearray(2 * (self.addrLen + 16))
+
+        evt.newskt = newskt = self.reactor.createSocket(
+            self.addressFamily, self.socketType
+        )
+        rc = _iocp.accept(self.socket.fileno(), newskt.fileno(), buff, evt)
+
+        if rc and rc != ERROR_IO_PENDING:
+            self.handleAccept(rc, evt)
diff --git a/contrib/python/Twisted/py3/twisted/internet/iocpreactor/udp.py b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/udp.py
new file mode 100644
index 00000000000..59c5fefb4b2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/iocpreactor/udp.py
@@ -0,0 +1,428 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+UDP support for IOCP reactor
+"""
+
+import errno
+import socket
+import struct
+import warnings
+from typing import Optional
+
+from zope.interface import implementer
+
+from twisted.internet import address, defer, error, interfaces
+from twisted.internet.abstract import isIPAddress, isIPv6Address
+from twisted.internet.iocpreactor import abstract, iocpsupport as _iocp
+from twisted.internet.iocpreactor.const import (
+    ERROR_CONNECTION_REFUSED,
+    ERROR_IO_PENDING,
+    ERROR_PORT_UNREACHABLE,
+)
+from twisted.internet.iocpreactor.interfaces import IReadWriteHandle
+from twisted.python import failure, log
+
+
+@implementer(
+    IReadWriteHandle,
+    interfaces.IListeningPort,
+    interfaces.IUDPTransport,
+    interfaces.ISystemHandle,
+)
+class Port(abstract.FileHandle):
+    """
+    UDP port, listening for packets.
+
+    @ivar addressFamily: L{socket.AF_INET} or L{socket.AF_INET6}, depending on
+        whether this port is listening on an IPv4 address or an IPv6 address.
+    """
+
+    addressFamily = socket.AF_INET
+    socketType = socket.SOCK_DGRAM
+    dynamicReadBuffers = False
+
+    # Actual port number being listened on, only set to a non-None
+    # value when we are actually listening.
+    _realPortNumber: Optional[int] = None
+
+    def __init__(self, port, proto, interface="", maxPacketSize=8192, reactor=None):
+        """
+        Initialize with a numeric port to listen on.
+        """
+        self.port = port
+        self.protocol = proto
+        self.readBufferSize = maxPacketSize
+        self.interface = interface
+        self.setLogStr()
+        self._connectedAddr = None
+        self._setAddressFamily()
+
+        abstract.FileHandle.__init__(self, reactor)
+
+        skt = socket.socket(self.addressFamily, self.socketType)
+        addrLen = _iocp.maxAddrLen(skt.fileno())
+        self.addressBuffer = bytearray(addrLen)
+        # WSARecvFrom takes an int
+        self.addressLengthBuffer = bytearray(struct.calcsize("i"))
+
+    def _setAddressFamily(self):
+        """
+        Resolve address family for the socket.
+        """
+        if isIPv6Address(self.interface):
+            self.addressFamily = socket.AF_INET6
+        elif isIPAddress(self.interface):
+            self.addressFamily = socket.AF_INET
+        elif self.interface:
+            raise error.InvalidAddressError(
+                self.interface, "not an IPv4 or IPv6 address"
+            )
+
+    def __repr__(self) -> str:
+        if self._realPortNumber is not None:
+            return f"<{self.protocol.__class__} on {self._realPortNumber}>"
+        else:
+            return f"<{self.protocol.__class__} not connected>"
+
+    def getHandle(self):
+        """
+        Return a socket object.
+        """
+        return self.socket
+
+    def startListening(self):
+        """
+        Create and bind my socket, and begin listening on it.
+
+        This is called on unserialization, and must be called after creating a
+        server to begin listening on the specified port.
+        """
+        self._bindSocket()
+        self._connectToProtocol()
+
+    def createSocket(self):
+        return self.reactor.createSocket(self.addressFamily, self.socketType)
+
+    def _bindSocket(self):
+        try:
+            skt = self.createSocket()
+            skt.bind((self.interface, self.port))
+        except OSError as le:
+            raise error.CannotListenError(self.interface, self.port, le)
+
+        # Make sure that if we listened on port 0, we update that to
+        # reflect what the OS actually assigned us.
+        self._realPortNumber = skt.getsockname()[1]
+
+        log.msg(
+            "%s starting on %s"
+            % (self._getLogPrefix(self.protocol), self._realPortNumber)
+        )
+
+        self.connected = True
+        self.socket = skt
+        self.getFileHandle = self.socket.fileno
+
+    def _connectToProtocol(self):
+        self.protocol.makeConnection(self)
+        self.startReading()
+        self.reactor.addActiveHandle(self)
+
+    def cbRead(self, rc, data, evt):
+        if self.reading:
+            self.handleRead(rc, data, evt)
+            self.doRead()
+
+    def handleRead(self, rc, data, evt):
+        if rc in (
+            errno.WSAECONNREFUSED,
+            errno.WSAECONNRESET,
+            ERROR_CONNECTION_REFUSED,
+            ERROR_PORT_UNREACHABLE,
+        ):
+            if self._connectedAddr:
+                self.protocol.connectionRefused()
+        elif rc:
+            log.msg(
+                "error in recvfrom -- %s (%s)"
+                % (errno.errorcode.get(rc, "unknown error"), rc)
+            )
+        else:
+            try:
+                self.protocol.datagramReceived(
+                    bytes(evt.buff[:data]), _iocp.makesockaddr(evt.addr_buff)
+                )
+            except BaseException:
+                log.err()
+
+    def doRead(self):
+        evt = _iocp.Event(self.cbRead, self)
+
+        evt.buff = buff = self._readBuffers[0]
+        evt.addr_buff = addr_buff = self.addressBuffer
+        evt.addr_len_buff = addr_len_buff = self.addressLengthBuffer
+        rc, data = _iocp.recvfrom(
+            self.getFileHandle(), buff, addr_buff, addr_len_buff, evt
+        )
+
+        if rc and rc != ERROR_IO_PENDING:
+            # If the error was not 0 or IO_PENDING then that means recvfrom() hit a
+            # failure condition. In this situation recvfrom() gives us our response
+            # right away and we don't need to wait for Windows to call the callback
+            # on our event. In fact, windows will not call it for us so we must call it
+            # ourselves manually
+            self.reactor.callLater(0, self.cbRead, rc, data, evt)
+
+    def write(self, datagram, addr=None):
+        """
+        Write a datagram.
+
+        @param addr: should be a tuple (ip, port), can be None in connected
+        mode.
+        """
+        if self._connectedAddr:
+            assert addr in (None, self._connectedAddr)
+            try:
+                return self.socket.send(datagram)
+            except OSError as se:
+                no = se.args[0]
+                if no == errno.WSAEINTR:
+                    return self.write(datagram)
+                elif no == errno.WSAEMSGSIZE:
+                    raise error.MessageLengthError("message too long")
+                elif no in (
+                    errno.WSAECONNREFUSED,
+                    errno.WSAECONNRESET,
+                    ERROR_CONNECTION_REFUSED,
+                    ERROR_PORT_UNREACHABLE,
+                ):
+                    self.protocol.connectionRefused()
+                else:
+                    raise
+        else:
+            assert addr != None
+            if (
+                not isIPAddress(addr[0])
+                and not isIPv6Address(addr[0])
+                and addr[0] != "<broadcast>"
+            ):
+                raise error.InvalidAddressError(
+                    addr[0], "write() only accepts IP addresses, not hostnames"
+                )
+            if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6:
+                raise error.InvalidAddressError(
+                    addr[0], "IPv6 port write() called with IPv4 address"
+                )
+            if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET:
+                raise error.InvalidAddressError(
+                    addr[0], "IPv4 port write() called with IPv6 address"
+                )
+            try:
+                return self.socket.sendto(datagram, addr)
+            except OSError as se:
+                no = se.args[0]
+                if no == errno.WSAEINTR:
+                    return self.write(datagram, addr)
+                elif no == errno.WSAEMSGSIZE:
+                    raise error.MessageLengthError("message too long")
+                elif no in (
+                    errno.WSAECONNREFUSED,
+                    errno.WSAECONNRESET,
+                    ERROR_CONNECTION_REFUSED,
+                    ERROR_PORT_UNREACHABLE,
+                ):
+                    # in non-connected UDP ECONNREFUSED is platform dependent,
+                    # I think and the info is not necessarily useful.
+                    # Nevertheless maybe we should call connectionRefused? XXX
+                    return
+                else:
+                    raise
+
+    def writeSequence(self, seq, addr):
+        self.write(b"".join(seq), addr)
+
+    def connect(self, host, port):
+        """
+        'Connect' to remote server.
+        """
+        if self._connectedAddr:
+            raise RuntimeError(
+                "already connected, reconnecting is not currently supported "
+                "(talk to itamar if you want this)"
+            )
+        if not isIPAddress(host) and not isIPv6Address(host):
+            raise error.InvalidAddressError(host, "not an IPv4 or IPv6 address.")
+        self._connectedAddr = (host, port)
+        self.socket.connect((host, port))
+
+    def _loseConnection(self):
+        self.stopReading()
+        self.reactor.removeActiveHandle(self)
+        if self.connected:  # actually means if we are *listening*
+            self.reactor.callLater(0, self.connectionLost)
+
+    def stopListening(self):
+        if self.connected:
+            result = self.d = defer.Deferred()
+        else:
+            result = None
+        self._loseConnection()
+        return result
+
+    def loseConnection(self):
+        warnings.warn(
+            "Please use stopListening() to disconnect port",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.stopListening()
+
+    def connectionLost(self, reason=None):
+        """
+        Cleans up my socket.
+        """
+        log.msg("(UDP Port %s Closed)" % self._realPortNumber)
+        self._realPortNumber = None
+        abstract.FileHandle.connectionLost(self, reason)
+        self.protocol.doStop()
+        self.socket.close()
+        del self.socket
+        del self.getFileHandle
+        if hasattr(self, "d"):
+            self.d.callback(None)
+            del self.d
+
+    def setLogStr(self):
+        """
+        Initialize the C{logstr} attribute to be used by C{logPrefix}.
+        """
+        logPrefix = self._getLogPrefix(self.protocol)
+        self.logstr = "%s (UDP)" % logPrefix
+
+    def logPrefix(self):
+        """
+        Returns the name of my class, to prefix log entries with.
+        """
+        return self.logstr
+
+    def getHost(self):
+        """
+        Return the local address of the UDP connection
+
+        @returns: the local address of the UDP connection
+        @rtype: L{IPv4Address} or L{IPv6Address}
+        """
+        addr = self.socket.getsockname()
+        if self.addressFamily == socket.AF_INET:
+            return address.IPv4Address("UDP", *addr)
+        elif self.addressFamily == socket.AF_INET6:
+            return address.IPv6Address("UDP", *(addr[:2]))
+
+    def setBroadcastAllowed(self, enabled):
+        """
+        Set whether this port may broadcast. This is disabled by default.
+
+        @param enabled: Whether the port may broadcast.
+        @type enabled: L{bool}
+        """
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, enabled)
+
+    def getBroadcastAllowed(self):
+        """
+        Checks if broadcast is currently allowed on this port.
+
+        @return: Whether this port may broadcast.
+        @rtype: L{bool}
+        """
+        return bool(self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST))
+
+
+class MulticastMixin:
+    """
+    Implement multicast functionality.
+    """
+
+    def getOutgoingInterface(self):
+        i = self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF)
+        return socket.inet_ntoa(struct.pack("@i", i))
+
+    def setOutgoingInterface(self, addr):
+        """
+        Returns Deferred of success.
+        """
+        return self.reactor.resolve(addr).addCallback(self._setInterface)
+
+    def _setInterface(self, addr):
+        i = socket.inet_aton(addr)
+        self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, i)
+        return 1
+
+    def getLoopbackMode(self):
+        return self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP)
+
+    def setLoopbackMode(self, mode):
+        mode = struct.pack("b", bool(mode))
+        self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, mode)
+
+    def getTTL(self):
+        return self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL)
+
+    def setTTL(self, ttl):
+        ttl = struct.pack("B", ttl)
+        self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl)
+
+    def joinGroup(self, addr, interface=""):
+        """
+        Join a multicast group. Returns Deferred of success.
+        """
+        return self.reactor.resolve(addr).addCallback(self._joinAddr1, interface, 1)
+
+    def _joinAddr1(self, addr, interface, join):
+        return self.reactor.resolve(interface).addCallback(self._joinAddr2, addr, join)
+
+    def _joinAddr2(self, interface, addr, join):
+        addr = socket.inet_aton(addr)
+        interface = socket.inet_aton(interface)
+        if join:
+            cmd = socket.IP_ADD_MEMBERSHIP
+        else:
+            cmd = socket.IP_DROP_MEMBERSHIP
+        try:
+            self.socket.setsockopt(socket.IPPROTO_IP, cmd, addr + interface)
+        except OSError as e:
+            return failure.Failure(error.MulticastJoinError(addr, interface, *e.args))
+
+    def leaveGroup(self, addr, interface=""):
+        """
+        Leave multicast group, return Deferred of success.
+        """
+        return self.reactor.resolve(addr).addCallback(self._joinAddr1, interface, 0)
+
+
+@implementer(interfaces.IMulticastTransport)
+class MulticastPort(MulticastMixin, Port):
+    """
+    UDP Port that supports multicasting.
+    """
+
+    def __init__(
+        self,
+        port,
+        proto,
+        interface="",
+        maxPacketSize=8192,
+        reactor=None,
+        listenMultiple=False,
+    ):
+        Port.__init__(self, port, proto, interface, maxPacketSize, reactor)
+        self.listenMultiple = listenMultiple
+
+    def createSocket(self):
+        skt = Port.createSocket(self)
+        if self.listenMultiple:
+            skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            if hasattr(socket, "SO_REUSEPORT"):
+                skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        return skt
diff --git a/contrib/python/Twisted/py3/twisted/internet/kqreactor.py b/contrib/python/Twisted/py3/twisted/internet/kqreactor.py
new file mode 100644
index 00000000000..a4863b183cd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/kqreactor.py
@@ -0,0 +1,324 @@
+# -*- test-case-name: twisted.test.test_kqueuereactor -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A kqueue()/kevent() based implementation of the Twisted main loop.
+
+To use this reactor, start your application specifying the kqueue reactor::
+
+   twistd --reactor kqueue ...
+
+To install the event loop from code (and you should do this before any
+connections, listeners or connectors are added)::
+
+   from twisted.internet import kqreactor
+   kqreactor.install()
+"""
+
+import errno
+import select
+
+from zope.interface import Attribute, Interface, declarations, implementer
+
+from twisted.internet import main, posixbase
+from twisted.internet.interfaces import IReactorDaemonize, IReactorFDSet
+from twisted.python import failure, log
+
+try:
+    # This is to keep mypy from complaining
+    # We don't use type: ignore[attr-defined] on import, because mypy only complains
+    # on on some platforms, and then the unused ignore is an issue if the undefined
+    # attribute isn't.
+    KQ_EV_ADD = getattr(select, "KQ_EV_ADD")
+    KQ_EV_DELETE = getattr(select, "KQ_EV_DELETE")
+    KQ_EV_EOF = getattr(select, "KQ_EV_EOF")
+    KQ_FILTER_READ = getattr(select, "KQ_FILTER_READ")
+    KQ_FILTER_WRITE = getattr(select, "KQ_FILTER_WRITE")
+except AttributeError as e:
+    raise ImportError(e)
+
+
+class _IKQueue(Interface):
+    """
+    An interface for KQueue implementations.
+    """
+
+    kqueue = Attribute("An implementation of kqueue(2).")
+    kevent = Attribute("An implementation of kevent(2).")
+
+
+declarations.directlyProvides(select, _IKQueue)
+
+
+@implementer(IReactorFDSet, IReactorDaemonize)
+class KQueueReactor(posixbase.PosixReactorBase):
+    """
+    A reactor that uses kqueue(2)/kevent(2) and relies on Python 2.6 or higher
+    which has built in support for kqueue in the select module.
+
+    @ivar _kq: A C{kqueue} which will be used to check for I/O readiness.
+
+    @ivar _impl: The implementation of L{_IKQueue} to use.
+
+    @ivar _selectables: A dictionary mapping integer file descriptors to
+        instances of L{FileDescriptor} which have been registered with the
+        reactor.  All L{FileDescriptor}s which are currently receiving read or
+        write readiness notifications will be present as values in this
+        dictionary.
+
+    @ivar _reads: A set containing integer file descriptors.  Values in this
+        set will be registered with C{_kq} for read readiness notifications
+        which will be dispatched to the corresponding L{FileDescriptor}
+        instances in C{_selectables}.
+
+    @ivar _writes: A set containing integer file descriptors.  Values in this
+        set will be registered with C{_kq} for write readiness notifications
+        which will be dispatched to the corresponding L{FileDescriptor}
+        instances in C{_selectables}.
+    """
+
+    def __init__(self, _kqueueImpl=select):
+        """
+        Initialize kqueue object, file descriptor tracking dictionaries, and
+        the base class.
+
+        See:
+            - http://docs.python.org/library/select.html
+            - www.freebsd.org/cgi/man.cgi?query=kqueue
+            - people.freebsd.org/~jlemon/papers/kqueue.pdf
+
+        @param _kqueueImpl: The implementation of L{_IKQueue} to use. A
+            hook for testing.
+        """
+        self._impl = _kqueueImpl
+        self._kq = self._impl.kqueue()
+        self._reads = set()
+        self._writes = set()
+        self._selectables = {}
+        posixbase.PosixReactorBase.__init__(self)
+
+    def _updateRegistration(self, fd, filter, op):
+        """
+        Private method for changing kqueue registration on a given FD
+        filtering for events given filter/op. This will never block and
+        returns nothing.
+        """
+        self._kq.control([self._impl.kevent(fd, filter, op)], 0, 0)
+
+    def beforeDaemonize(self):
+        """
+        Implement L{IReactorDaemonize.beforeDaemonize}.
+        """
+        # Twisted-internal method called during daemonization (when application
+        # is started via twistd). This is called right before the magic double
+        # forking done for daemonization. We cleanly close the kqueue() and later
+        # recreate it. This is needed since a) kqueue() are not inherited across
+        # forks and b) twistd will create the reactor already before daemonization
+        # (and will also add at least 1 reader to the reactor, an instance of
+        # twisted.internet.posixbase._UnixWaker).
+        #
+        # See: twisted.scripts._twistd_unix.daemonize()
+        self._kq.close()
+        self._kq = None
+
+    def afterDaemonize(self):
+        """
+        Implement L{IReactorDaemonize.afterDaemonize}.
+        """
+        # Twisted-internal method called during daemonization. This is called right
+        # after daemonization and recreates the kqueue() and any readers/writers
+        # that were added before. Note that you MUST NOT call any reactor methods
+        # in between beforeDaemonize() and afterDaemonize()!
+        self._kq = self._impl.kqueue()
+        for fd in self._reads:
+            self._updateRegistration(fd, KQ_FILTER_READ, KQ_EV_ADD)
+        for fd in self._writes:
+            self._updateRegistration(fd, KQ_FILTER_WRITE, KQ_EV_ADD)
+
+    def addReader(self, reader):
+        """
+        Implement L{IReactorFDSet.addReader}.
+        """
+        fd = reader.fileno()
+        if fd not in self._reads:
+            try:
+                self._updateRegistration(fd, KQ_FILTER_READ, KQ_EV_ADD)
+            except OSError:
+                pass
+            finally:
+                self._selectables[fd] = reader
+                self._reads.add(fd)
+
+    def addWriter(self, writer):
+        """
+        Implement L{IReactorFDSet.addWriter}.
+        """
+        fd = writer.fileno()
+        if fd not in self._writes:
+            try:
+                self._updateRegistration(fd, KQ_FILTER_WRITE, KQ_EV_ADD)
+            except OSError:
+                pass
+            finally:
+                self._selectables[fd] = writer
+                self._writes.add(fd)
+
+    def removeReader(self, reader):
+        """
+        Implement L{IReactorFDSet.removeReader}.
+        """
+        wasLost = False
+        try:
+            fd = reader.fileno()
+        except BaseException:
+            fd = -1
+        if fd == -1:
+            for fd, fdes in self._selectables.items():
+                if reader is fdes:
+                    wasLost = True
+                    break
+            else:
+                return
+        if fd in self._reads:
+            self._reads.remove(fd)
+            if fd not in self._writes:
+                del self._selectables[fd]
+            if not wasLost:
+                try:
+                    self._updateRegistration(fd, KQ_FILTER_READ, KQ_EV_DELETE)
+                except OSError:
+                    pass
+
+    def removeWriter(self, writer):
+        """
+        Implement L{IReactorFDSet.removeWriter}.
+        """
+        wasLost = False
+        try:
+            fd = writer.fileno()
+        except BaseException:
+            fd = -1
+        if fd == -1:
+            for fd, fdes in self._selectables.items():
+                if writer is fdes:
+                    wasLost = True
+                    break
+            else:
+                return
+        if fd in self._writes:
+            self._writes.remove(fd)
+            if fd not in self._reads:
+                del self._selectables[fd]
+            if not wasLost:
+                try:
+                    self._updateRegistration(fd, KQ_FILTER_WRITE, KQ_EV_DELETE)
+                except OSError:
+                    pass
+
+    def removeAll(self):
+        """
+        Implement L{IReactorFDSet.removeAll}.
+        """
+        return self._removeAll(
+            [self._selectables[fd] for fd in self._reads],
+            [self._selectables[fd] for fd in self._writes],
+        )
+
+    def getReaders(self):
+        """
+        Implement L{IReactorFDSet.getReaders}.
+        """
+        return [self._selectables[fd] for fd in self._reads]
+
+    def getWriters(self):
+        """
+        Implement L{IReactorFDSet.getWriters}.
+        """
+        return [self._selectables[fd] for fd in self._writes]
+
+    def doKEvent(self, timeout):
+        """
+        Poll the kqueue for new events.
+        """
+        if timeout is None:
+            timeout = 1
+
+        try:
+            events = self._kq.control([], len(self._selectables), timeout)
+        except OSError as e:
+            # Since this command blocks for potentially a while, it's possible
+            # EINTR can be raised for various reasons (for example, if the user
+            # hits ^C).
+            if e.errno == errno.EINTR:
+                return
+            else:
+                raise
+
+        _drdw = self._doWriteOrRead
+        for event in events:
+            fd = event.ident
+            try:
+                selectable = self._selectables[fd]
+            except KeyError:
+                # Handles the infrequent case where one selectable's
+                # handler disconnects another.
+                continue
+            else:
+                log.callWithLogger(selectable, _drdw, selectable, fd, event)
+
+    def _doWriteOrRead(self, selectable, fd, event):
+        """
+        Private method called when a FD is ready for reading, writing or was
+        lost. Do the work and raise errors where necessary.
+        """
+        why = None
+        inRead = False
+        (filter, flags, data, fflags) = (
+            event.filter,
+            event.flags,
+            event.data,
+            event.fflags,
+        )
+
+        if flags & KQ_EV_EOF and data and fflags:
+            why = main.CONNECTION_LOST
+        else:
+            try:
+                if selectable.fileno() == -1:
+                    inRead = False
+                    why = posixbase._NO_FILEDESC
+                else:
+                    if filter == KQ_FILTER_READ:
+                        inRead = True
+                        why = selectable.doRead()
+                    if filter == KQ_FILTER_WRITE:
+                        inRead = False
+                        why = selectable.doWrite()
+            except BaseException:
+                # Any exception from application code gets logged and will
+                # cause us to disconnect the selectable.
+                why = failure.Failure()
+                log.err(
+                    why,
+                    "An exception was raised from application code"
+                    " while processing a reactor selectable",
+                )
+
+        if why:
+            self._disconnectSelectable(selectable, why, inRead)
+
+    doIteration = doKEvent
+
+
+def install():
+    """
+    Install the kqueue() reactor.
+    """
+    p = KQueueReactor()
+    from twisted.internet.main import installReactor
+
+    installReactor(p)
+
+
+__all__ = ["KQueueReactor", "install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/main.py b/contrib/python/Twisted/py3/twisted/internet/main.py
new file mode 100644
index 00000000000..2a05ac9c6dd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/main.py
@@ -0,0 +1,37 @@
+# -*- test-case-name: twisted.internet.test.test_main -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Backwards compatibility, and utility functions.
+
+In general, this module should not be used, other than by reactor authors
+who need to use the 'installReactor' method.
+"""
+
+
+from twisted.internet import error
+
+CONNECTION_DONE = error.ConnectionDone("Connection done")
+CONNECTION_LOST = error.ConnectionLost("Connection lost")
+
+
+def installReactor(reactor):
+    """
+    Install reactor C{reactor}.
+
+    @param reactor: An object that provides one or more IReactor* interfaces.
+    """
+    # this stuff should be common to all reactors.
+    import sys
+
+    import twisted.internet
+
+    if "twisted.internet.reactor" in sys.modules:
+        raise error.ReactorAlreadyInstalledError("reactor already installed")
+    twisted.internet.reactor = reactor
+    sys.modules["twisted.internet.reactor"] = reactor
+
+
+__all__ = ["CONNECTION_LOST", "CONNECTION_DONE", "installReactor"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/pollreactor.py b/contrib/python/Twisted/py3/twisted/internet/pollreactor.py
new file mode 100644
index 00000000000..b9f1fb84025
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/pollreactor.py
@@ -0,0 +1,189 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A poll() based implementation of the twisted main loop.
+
+To install the event loop (and you should do this before any connections,
+listeners or connectors are added)::
+
+    from twisted.internet import pollreactor
+    pollreactor.install()
+"""
+
+
+# System imports
+import errno
+from select import (
+    POLLERR,
+    POLLHUP,
+    POLLIN,
+    POLLNVAL,
+    POLLOUT,
+    error as SelectError,
+    poll,
+)
+
+from zope.interface import implementer
+
+from twisted.internet import posixbase
+from twisted.internet.interfaces import IReactorFDSet
+
+# Twisted imports
+from twisted.python import log
+
+
+@implementer(IReactorFDSet)
+class PollReactor(posixbase.PosixReactorBase, posixbase._PollLikeMixin):
+    """
+    A reactor that uses poll(2).
+
+    @ivar _poller: A L{select.poll} which will be used to check for I/O
+        readiness.
+
+    @ivar _selectables: A dictionary mapping integer file descriptors to
+        instances of L{FileDescriptor} which have been registered with the
+        reactor.  All L{FileDescriptor}s which are currently receiving read or
+        write readiness notifications will be present as values in this
+        dictionary.
+
+    @ivar _reads: A dictionary mapping integer file descriptors to arbitrary
+        values (this is essentially a set).  Keys in this dictionary will be
+        registered with C{_poller} for read readiness notifications which will
+        be dispatched to the corresponding L{FileDescriptor} instances in
+        C{_selectables}.
+
+    @ivar _writes: A dictionary mapping integer file descriptors to arbitrary
+        values (this is essentially a set).  Keys in this dictionary will be
+        registered with C{_poller} for write readiness notifications which will
+        be dispatched to the corresponding L{FileDescriptor} instances in
+        C{_selectables}.
+    """
+
+    _POLL_DISCONNECTED = POLLHUP | POLLERR | POLLNVAL
+    _POLL_IN = POLLIN
+    _POLL_OUT = POLLOUT
+
+    def __init__(self):
+        """
+        Initialize polling object, file descriptor tracking dictionaries, and
+        the base class.
+        """
+        self._poller = poll()
+        self._selectables = {}
+        self._reads = {}
+        self._writes = {}
+        posixbase.PosixReactorBase.__init__(self)
+
+    def _updateRegistration(self, fd):
+        """Register/unregister an fd with the poller."""
+        try:
+            self._poller.unregister(fd)
+        except KeyError:
+            pass
+
+        mask = 0
+        if fd in self._reads:
+            mask = mask | POLLIN
+        if fd in self._writes:
+            mask = mask | POLLOUT
+        if mask != 0:
+            self._poller.register(fd, mask)
+        else:
+            if fd in self._selectables:
+                del self._selectables[fd]
+
+    def _dictRemove(self, selectable, mdict):
+        try:
+            # the easy way
+            fd = selectable.fileno()
+            # make sure the fd is actually real.  In some situations we can get
+            # -1 here.
+            mdict[fd]
+        except BaseException:
+            # the hard way: necessary because fileno() may disappear at any
+            # moment, thanks to python's underlying sockets impl
+            for fd, fdes in self._selectables.items():
+                if selectable is fdes:
+                    break
+            else:
+                # Hmm, maybe not the right course of action?  This method can't
+                # fail, because it happens inside error detection...
+                return
+        if fd in mdict:
+            del mdict[fd]
+            self._updateRegistration(fd)
+
+    def addReader(self, reader):
+        """Add a FileDescriptor for notification of data available to read."""
+        fd = reader.fileno()
+        if fd not in self._reads:
+            self._selectables[fd] = reader
+            self._reads[fd] = 1
+            self._updateRegistration(fd)
+
+    def addWriter(self, writer):
+        """Add a FileDescriptor for notification of data available to write."""
+        fd = writer.fileno()
+        if fd not in self._writes:
+            self._selectables[fd] = writer
+            self._writes[fd] = 1
+            self._updateRegistration(fd)
+
+    def removeReader(self, reader):
+        """Remove a Selectable for notification of data available to read."""
+        return self._dictRemove(reader, self._reads)
+
+    def removeWriter(self, writer):
+        """Remove a Selectable for notification of data available to write."""
+        return self._dictRemove(writer, self._writes)
+
+    def removeAll(self):
+        """
+        Remove all selectables, and return a list of them.
+        """
+        return self._removeAll(
+            [self._selectables[fd] for fd in self._reads],
+            [self._selectables[fd] for fd in self._writes],
+        )
+
+    def doPoll(self, timeout):
+        """Poll the poller for new events."""
+        if timeout is not None:
+            timeout = int(timeout * 1000)  # convert seconds to milliseconds
+
+        try:
+            l = self._poller.poll(timeout)
+        except SelectError as e:
+            if e.args[0] == errno.EINTR:
+                return
+            else:
+                raise
+        _drdw = self._doReadOrWrite
+        for fd, event in l:
+            try:
+                selectable = self._selectables[fd]
+            except KeyError:
+                # Handles the infrequent case where one selectable's
+                # handler disconnects another.
+                continue
+            log.callWithLogger(selectable, _drdw, selectable, fd, event)
+
+    doIteration = doPoll
+
+    def getReaders(self):
+        return [self._selectables[fd] for fd in self._reads]
+
+    def getWriters(self):
+        return [self._selectables[fd] for fd in self._writes]
+
+
+def install():
+    """Install the poll() reactor."""
+    p = PollReactor()
+    from twisted.internet.main import installReactor
+
+    installReactor(p)
+
+
+__all__ = ["PollReactor", "install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/posixbase.py b/contrib/python/Twisted/py3/twisted/internet/posixbase.py
new file mode 100644
index 00000000000..bd160ec8656
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/posixbase.py
@@ -0,0 +1,653 @@
+# -*- test-case-name: twisted.test.test_internet,twisted.internet.test.test_posixbase -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Posix reactor base class
+"""
+
+
+import socket
+import sys
+from typing import Sequence
+
+from zope.interface import classImplements, implementer
+
+from twisted.internet import error, tcp, udp
+from twisted.internet.base import ReactorBase
+from twisted.internet.interfaces import (
+    IHalfCloseableDescriptor,
+    IReactorFDSet,
+    IReactorMulticast,
+    IReactorProcess,
+    IReactorSocket,
+    IReactorSSL,
+    IReactorTCP,
+    IReactorUDP,
+    IReactorUNIX,
+    IReactorUNIXDatagram,
+)
+from twisted.internet.main import CONNECTION_DONE, CONNECTION_LOST
+from twisted.python import failure, log
+from twisted.python.runtime import platform, platformType
+from ._signals import (
+    SignalHandling,
+    _ChildSignalHandling,
+    _IWaker,
+    _MultiSignalHandling,
+    _Waker,
+)
+
+# Exceptions that doSelect might return frequently
+_NO_FILENO = error.ConnectionFdescWentAway("Handler has no fileno method")
+_NO_FILEDESC = error.ConnectionFdescWentAway("File descriptor lost")
+
+
+try:
+    from twisted.protocols import tls as _tls
+except ImportError:
+    tls = None
+else:
+    tls = _tls
+
+try:
+    from twisted.internet import ssl as _ssl
+except ImportError:
+    ssl = None
+else:
+    ssl = _ssl
+
+unixEnabled = platformType == "posix"
+
+processEnabled = False
+if unixEnabled:
+    from twisted.internet import process, unix
+
+    processEnabled = True
+
+
+if platform.isWindows():
+    try:
+        import win32process  # type: ignore[import]
+
+        processEnabled = True
+    except ImportError:
+        win32process = None
+
+
+class _DisconnectSelectableMixin:
+    """
+    Mixin providing the C{_disconnectSelectable} method.
+    """
+
+    def _disconnectSelectable(
+        self,
+        selectable,
+        why,
+        isRead,
+        faildict={
+            error.ConnectionDone: failure.Failure(error.ConnectionDone()),
+            error.ConnectionLost: failure.Failure(error.ConnectionLost()),
+        },
+    ):
+        """
+        Utility function for disconnecting a selectable.
+
+        Supports half-close notification, isRead should be boolean indicating
+        whether error resulted from doRead().
+        """
+        self.removeReader(selectable)
+        f = faildict.get(why.__class__)
+        if f:
+            if (
+                isRead
+                and why.__class__ == error.ConnectionDone
+                and IHalfCloseableDescriptor.providedBy(selectable)
+            ):
+                selectable.readConnectionLost(f)
+            else:
+                self.removeWriter(selectable)
+                selectable.connectionLost(f)
+        else:
+            self.removeWriter(selectable)
+            selectable.connectionLost(failure.Failure(why))
+
+
+@implementer(IReactorTCP, IReactorUDP, IReactorMulticast)
+class PosixReactorBase(_DisconnectSelectableMixin, ReactorBase):
+    """
+    A basis for reactors that use file descriptors.
+
+    @ivar _childWaker: L{None} or a reference to the L{_SIGCHLDWaker}
+        which is used to properly notice child process termination.
+    """
+
+    _childWaker = None
+
+    # Callable that creates a waker, overrideable so that subclasses can
+    # substitute their own implementation:
+    def _wakerFactory(self) -> _IWaker:
+        return _Waker()
+
+    def installWaker(self):
+        """
+        Install a `waker' to allow threads and signals to wake up the IO thread.
+
+        We use the self-pipe trick (http://cr.yp.to/docs/selfpipe.html) to wake
+        the reactor. On Windows we use a pair of sockets.
+        """
+        if not self.waker:
+            self.waker = self._wakerFactory()
+            self._internalReaders.add(self.waker)
+            self.addReader(self.waker)
+
+    def _signalsFactory(self) -> SignalHandling:
+        """
+        Customize reactor signal handling to support child processes on POSIX
+        platforms.
+        """
+        baseHandling = super()._signalsFactory()
+        # If we're on a platform that uses signals for process event signaling
+        if platformType == "posix":
+            # Compose ...
+            return _MultiSignalHandling(
+                (
+                    # the base signal handling behavior ...
+                    baseHandling,
+                    # with our extra SIGCHLD handling behavior.
+                    _ChildSignalHandling(
+                        self._addInternalReader,
+                        self._removeInternalReader,
+                    ),
+                )
+            )
+
+        # Otherwise just use the base behavior
+        return baseHandling
+
+    # IReactorProcess
+
+    def spawnProcess(
+        self,
+        processProtocol,
+        executable,
+        args=(),
+        env={},
+        path=None,
+        uid=None,
+        gid=None,
+        usePTY=0,
+        childFDs=None,
+    ):
+        if platformType == "posix":
+            if usePTY:
+                if childFDs is not None:
+                    raise ValueError(
+                        "Using childFDs is not supported with usePTY=True."
+                    )
+                return process.PTYProcess(
+                    self, executable, args, env, path, processProtocol, uid, gid, usePTY
+                )
+            else:
+                return process.Process(
+                    self,
+                    executable,
+                    args,
+                    env,
+                    path,
+                    processProtocol,
+                    uid,
+                    gid,
+                    childFDs,
+                )
+        elif platformType == "win32":
+            if uid is not None:
+                raise ValueError("Setting UID is unsupported on this platform.")
+            if gid is not None:
+                raise ValueError("Setting GID is unsupported on this platform.")
+            if usePTY:
+                raise ValueError("The usePTY parameter is not supported on Windows.")
+            if childFDs:
+                raise ValueError("Customizing childFDs is not supported on Windows.")
+
+            if win32process:
+                from twisted.internet._dumbwin32proc import Process
+
+                return Process(self, processProtocol, executable, args, env, path)
+            else:
+                raise NotImplementedError(
+                    "spawnProcess not available since pywin32 is not installed."
+                )
+        else:
+            raise NotImplementedError(
+                "spawnProcess only available on Windows or POSIX."
+            )
+
+    # IReactorUDP
+
+    def listenUDP(self, port, protocol, interface="", maxPacketSize=8192):
+        """Connects a given L{DatagramProtocol} to the given numeric UDP port.
+
+        @returns: object conforming to L{IListeningPort}.
+        """
+        p = udp.Port(port, protocol, interface, maxPacketSize, self)
+        p.startListening()
+        return p
+
+    # IReactorMulticast
+
+    def listenMulticast(
+        self, port, protocol, interface="", maxPacketSize=8192, listenMultiple=False
+    ):
+        """Connects a given DatagramProtocol to the given numeric UDP port.
+
+        EXPERIMENTAL.
+
+        @returns: object conforming to IListeningPort.
+        """
+        p = udp.MulticastPort(
+            port, protocol, interface, maxPacketSize, self, listenMultiple
+        )
+        p.startListening()
+        return p
+
+    # IReactorUNIX
+
+    def connectUNIX(self, address, factory, timeout=30, checkPID=0):
+        assert unixEnabled, "UNIX support is not present"
+        c = unix.Connector(address, factory, timeout, self, checkPID)
+        c.connect()
+        return c
+
+    def listenUNIX(self, address, factory, backlog=50, mode=0o666, wantPID=0):
+        assert unixEnabled, "UNIX support is not present"
+        p = unix.Port(address, factory, backlog, mode, self, wantPID)
+        p.startListening()
+        return p
+
+    # IReactorUNIXDatagram
+
+    def listenUNIXDatagram(self, address, protocol, maxPacketSize=8192, mode=0o666):
+        """
+        Connects a given L{DatagramProtocol} to the given path.
+
+        EXPERIMENTAL.
+
+        @returns: object conforming to L{IListeningPort}.
+        """
+        assert unixEnabled, "UNIX support is not present"
+        p = unix.DatagramPort(address, protocol, maxPacketSize, mode, self)
+        p.startListening()
+        return p
+
+    def connectUNIXDatagram(
+        self, address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None
+    ):
+        """
+        Connects a L{ConnectedDatagramProtocol} instance to a path.
+
+        EXPERIMENTAL.
+        """
+        assert unixEnabled, "UNIX support is not present"
+        p = unix.ConnectedDatagramPort(
+            address, protocol, maxPacketSize, mode, bindAddress, self
+        )
+        p.startListening()
+        return p
+
+    # IReactorSocket (no AF_UNIX on Windows)
+
+    if unixEnabled:
+        _supportedAddressFamilies: Sequence[socket.AddressFamily] = (
+            socket.AF_INET,
+            socket.AF_INET6,
+            socket.AF_UNIX,
+        )
+    else:
+        _supportedAddressFamilies = (
+            socket.AF_INET,
+            socket.AF_INET6,
+        )
+
+    def adoptStreamPort(self, fileDescriptor, addressFamily, factory):
+        """
+        Create a new L{IListeningPort} from an already-initialized socket.
+
+        This just dispatches to a suitable port implementation (eg from
+        L{IReactorTCP}, etc) based on the specified C{addressFamily}.
+
+        @see: L{twisted.internet.interfaces.IReactorSocket.adoptStreamPort}
+        """
+        if addressFamily not in self._supportedAddressFamilies:
+            raise error.UnsupportedAddressFamily(addressFamily)
+
+        if unixEnabled and addressFamily == socket.AF_UNIX:
+            p = unix.Port._fromListeningDescriptor(self, fileDescriptor, factory)
+        else:
+            p = tcp.Port._fromListeningDescriptor(
+                self, fileDescriptor, addressFamily, factory
+            )
+        p.startListening()
+        return p
+
+    def adoptStreamConnection(self, fileDescriptor, addressFamily, factory):
+        """
+        @see:
+            L{twisted.internet.interfaces.IReactorSocket.adoptStreamConnection}
+        """
+        if addressFamily not in self._supportedAddressFamilies:
+            raise error.UnsupportedAddressFamily(addressFamily)
+
+        if unixEnabled and addressFamily == socket.AF_UNIX:
+            return unix.Server._fromConnectedSocket(fileDescriptor, factory, self)
+        else:
+            return tcp.Server._fromConnectedSocket(
+                fileDescriptor, addressFamily, factory, self
+            )
+
+    def adoptDatagramPort(
+        self, fileDescriptor, addressFamily, protocol, maxPacketSize=8192
+    ):
+        if addressFamily not in (socket.AF_INET, socket.AF_INET6):
+            raise error.UnsupportedAddressFamily(addressFamily)
+
+        p = udp.Port._fromListeningDescriptor(
+            self, fileDescriptor, addressFamily, protocol, maxPacketSize=maxPacketSize
+        )
+        p.startListening()
+        return p
+
+    # IReactorTCP
+
+    def listenTCP(self, port, factory, backlog=50, interface=""):
+        p = tcp.Port(port, factory, backlog, interface, self)
+        p.startListening()
+        return p
+
+    def connectTCP(self, host, port, factory, timeout=30, bindAddress=None):
+        c = tcp.Connector(host, port, factory, timeout, bindAddress, self)
+        c.connect()
+        return c
+
+    # IReactorSSL (sometimes, not implemented)
+
+    def connectSSL(
+        self, host, port, factory, contextFactory, timeout=30, bindAddress=None
+    ):
+        if tls is not None:
+            tlsFactory = tls.TLSMemoryBIOFactory(contextFactory, True, factory)
+            return self.connectTCP(host, port, tlsFactory, timeout, bindAddress)
+        elif ssl is not None:
+            c = ssl.Connector(
+                host, port, factory, contextFactory, timeout, bindAddress, self
+            )
+            c.connect()
+            return c
+        else:
+            assert False, "SSL support is not present"
+
+    def listenSSL(self, port, factory, contextFactory, backlog=50, interface=""):
+        if tls is not None:
+            tlsFactory = tls.TLSMemoryBIOFactory(contextFactory, False, factory)
+            port = self.listenTCP(port, tlsFactory, backlog, interface)
+            port._type = "TLS"
+            return port
+        elif ssl is not None:
+            p = ssl.Port(port, factory, contextFactory, backlog, interface, self)
+            p.startListening()
+            return p
+        else:
+            assert False, "SSL support is not present"
+
+    def _removeAll(self, readers, writers):
+        """
+        Remove all readers and writers, and list of removed L{IReadDescriptor}s
+        and L{IWriteDescriptor}s.
+
+        Meant for calling from subclasses, to implement removeAll, like::
+
+          def removeAll(self):
+              return self._removeAll(self._reads, self._writes)
+
+        where C{self._reads} and C{self._writes} are iterables.
+        """
+        removedReaders = set(readers) - self._internalReaders
+        for reader in removedReaders:
+            self.removeReader(reader)
+
+        removedWriters = set(writers)
+        for writer in removedWriters:
+            self.removeWriter(writer)
+
+        return list(removedReaders | removedWriters)
+
+
+class _PollLikeMixin:
+    """
+    Mixin for poll-like reactors.
+
+    Subclasses must define the following attributes::
+
+      - _POLL_DISCONNECTED - Bitmask for events indicating a connection was
+        lost.
+      - _POLL_IN - Bitmask for events indicating there is input to read.
+      - _POLL_OUT - Bitmask for events indicating output can be written.
+
+    Must be mixed in to a subclass of PosixReactorBase (for
+    _disconnectSelectable).
+    """
+
+    def _doReadOrWrite(self, selectable, fd, event):
+        """
+        fd is available for read or write, do the work and raise errors if
+        necessary.
+        """
+        why = None
+        inRead = False
+        if event & self._POLL_DISCONNECTED and not (event & self._POLL_IN):
+            # Handle disconnection.  But only if we finished processing all
+            # the pending input.
+            if fd in self._reads:
+                # If we were reading from the descriptor then this is a
+                # clean shutdown.  We know there are no read events pending
+                # because we just checked above.  It also might be a
+                # half-close (which is why we have to keep track of inRead).
+                inRead = True
+                why = CONNECTION_DONE
+            else:
+                # If we weren't reading, this is an error shutdown of some
+                # sort.
+                why = CONNECTION_LOST
+        else:
+            # Any non-disconnect event turns into a doRead or a doWrite.
+            try:
+                # First check to see if the descriptor is still valid.  This
+                # gives fileno() a chance to raise an exception, too.
+                # Ideally, disconnection would always be indicated by the
+                # return value of doRead or doWrite (or an exception from
+                # one of those methods), but calling fileno here helps make
+                # buggy applications more transparent.
+                if selectable.fileno() == -1:
+                    # -1 is sort of a historical Python artifact.  Python
+                    # files and sockets used to change their file descriptor
+                    # to -1 when they closed.  For the time being, we'll
+                    # continue to support this anyway in case applications
+                    # replicated it, plus abstract.FileDescriptor.fileno
+                    # returns -1.  Eventually it'd be good to deprecate this
+                    # case.
+                    why = _NO_FILEDESC
+                else:
+                    if event & self._POLL_IN:
+                        # Handle a read event.
+                        why = selectable.doRead()
+                        inRead = True
+                    if not why and event & self._POLL_OUT:
+                        # Handle a write event, as long as doRead didn't
+                        # disconnect us.
+                        why = selectable.doWrite()
+                        inRead = False
+            except BaseException:
+                # Any exception from application code gets logged and will
+                # cause us to disconnect the selectable.
+                why = sys.exc_info()[1]
+                log.err()
+        if why:
+            self._disconnectSelectable(selectable, why, inRead)
+
+
+@implementer(IReactorFDSet)
+class _ContinuousPolling(_PollLikeMixin, _DisconnectSelectableMixin):
+    """
+    Schedule reads and writes based on the passage of time, rather than
+    notification.
+
+    This is useful for supporting polling filesystem files, which C{epoll(7)}
+    does not support.
+
+    The implementation uses L{_PollLikeMixin}, which is a bit hacky, but
+    re-implementing and testing the relevant code yet again is unappealing.
+
+    @ivar _reactor: The L{EPollReactor} that is using this instance.
+
+    @ivar _loop: A C{LoopingCall} that drives the polling, or L{None}.
+
+    @ivar _readers: A C{set} of C{FileDescriptor} objects that should be read
+        from.
+
+    @ivar _writers: A C{set} of C{FileDescriptor} objects that should be
+        written to.
+    """
+
+    # Attributes for _PollLikeMixin
+    _POLL_DISCONNECTED = 1
+    _POLL_IN = 2
+    _POLL_OUT = 4
+
+    def __init__(self, reactor):
+        self._reactor = reactor
+        self._loop = None
+        self._readers = set()
+        self._writers = set()
+
+    def _checkLoop(self):
+        """
+        Start or stop a C{LoopingCall} based on whether there are readers and
+        writers.
+        """
+        if self._readers or self._writers:
+            if self._loop is None:
+                from twisted.internet.task import _EPSILON, LoopingCall
+
+                self._loop = LoopingCall(self.iterate)
+                self._loop.clock = self._reactor
+                # LoopingCall seems unhappy with timeout of 0, so use very
+                # small number:
+                self._loop.start(_EPSILON, now=False)
+        elif self._loop:
+            self._loop.stop()
+            self._loop = None
+
+    def iterate(self):
+        """
+        Call C{doRead} and C{doWrite} on all readers and writers respectively.
+        """
+        for reader in list(self._readers):
+            self._doReadOrWrite(reader, reader, self._POLL_IN)
+        for writer in list(self._writers):
+            self._doReadOrWrite(writer, writer, self._POLL_OUT)
+
+    def addReader(self, reader):
+        """
+        Add a C{FileDescriptor} for notification of data available to read.
+        """
+        self._readers.add(reader)
+        self._checkLoop()
+
+    def addWriter(self, writer):
+        """
+        Add a C{FileDescriptor} for notification of data available to write.
+        """
+        self._writers.add(writer)
+        self._checkLoop()
+
+    def removeReader(self, reader):
+        """
+        Remove a C{FileDescriptor} from notification of data available to read.
+        """
+        try:
+            self._readers.remove(reader)
+        except KeyError:
+            return
+        self._checkLoop()
+
+    def removeWriter(self, writer):
+        """
+        Remove a C{FileDescriptor} from notification of data available to
+        write.
+        """
+        try:
+            self._writers.remove(writer)
+        except KeyError:
+            return
+        self._checkLoop()
+
+    def removeAll(self):
+        """
+        Remove all readers and writers.
+        """
+        result = list(self._readers | self._writers)
+        # Don't reset to new value, since self.isWriting and .isReading refer
+        # to the existing instance:
+        self._readers.clear()
+        self._writers.clear()
+        return result
+
+    def getReaders(self):
+        """
+        Return a list of the readers.
+        """
+        return list(self._readers)
+
+    def getWriters(self):
+        """
+        Return a list of the writers.
+        """
+        return list(self._writers)
+
+    def isReading(self, fd):
+        """
+        Checks if the file descriptor is currently being observed for read
+        readiness.
+
+        @param fd: The file descriptor being checked.
+        @type fd: L{twisted.internet.abstract.FileDescriptor}
+        @return: C{True} if the file descriptor is being observed for read
+            readiness, C{False} otherwise.
+        @rtype: C{bool}
+        """
+        return fd in self._readers
+
+    def isWriting(self, fd):
+        """
+        Checks if the file descriptor is currently being observed for write
+        readiness.
+
+        @param fd: The file descriptor being checked.
+        @type fd: L{twisted.internet.abstract.FileDescriptor}
+        @return: C{True} if the file descriptor is being observed for write
+            readiness, C{False} otherwise.
+        @rtype: C{bool}
+        """
+        return fd in self._writers
+
+
+if tls is not None or ssl is not None:
+    classImplements(PosixReactorBase, IReactorSSL)
+if unixEnabled:
+    classImplements(PosixReactorBase, IReactorUNIX, IReactorUNIXDatagram)
+if processEnabled:
+    classImplements(PosixReactorBase, IReactorProcess)
+if getattr(socket, "fromfd", None) is not None:
+    classImplements(PosixReactorBase, IReactorSocket)
+
+__all__ = ["PosixReactorBase"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/process.py b/contrib/python/Twisted/py3/twisted/internet/process.py
new file mode 100644
index 00000000000..ef3b88d9f19
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/process.py
@@ -0,0 +1,1293 @@
+# -*- test-case-name: twisted.test.test_process -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+UNIX Process management.
+
+Do NOT use this module directly - use reactor.spawnProcess() instead.
+
+Maintainer: Itamar Shtull-Trauring
+"""
+from __future__ import annotations
+
+import errno
+import gc
+import io
+import os
+import signal
+import stat
+import sys
+import traceback
+from collections import defaultdict
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+
+_PS_CLOSE: int
+_PS_DUP2: int
+
+if not TYPE_CHECKING:
+    try:
+        from os import POSIX_SPAWN_CLOSE as _PS_CLOSE, POSIX_SPAWN_DUP2 as _PS_DUP2
+    except ImportError:
+        pass
+
+from zope.interface import implementer
+
+from twisted.internet import abstract, error, fdesc
+from twisted.internet._baseprocess import BaseProcess
+from twisted.internet.interfaces import IProcessTransport
+from twisted.internet.main import CONNECTION_DONE, CONNECTION_LOST
+from twisted.python import failure, log
+from twisted.python.runtime import platform
+from twisted.python.util import switchUID
+
+if platform.isWindows():
+    raise ImportError(
+        "twisted.internet.process does not work on Windows. "
+        "Use the reactor.spawnProcess() API instead."
+    )
+
+try:
+    import pty as _pty
+except ImportError:
+    pty = None
+else:
+    pty = _pty
+
+try:
+    import fcntl as _fcntl
+    import termios
+except ImportError:
+    fcntl = None
+else:
+    fcntl = _fcntl
+
+# Some people were importing this, which is incorrect, just keeping it
+# here for backwards compatibility:
+ProcessExitedAlready = error.ProcessExitedAlready
+
+reapProcessHandlers: Dict[int, _BaseProcess] = {}
+
+
+def reapAllProcesses() -> None:
+    """
+    Reap all registered processes.
+    """
+    # Coerce this to a list, as reaping the process changes the dictionary and
+    # causes a "size changed during iteration" exception
+    for process in list(reapProcessHandlers.values()):
+        process.reapProcess()
+
+
+def registerReapProcessHandler(pid, process):
+    """
+    Register a process handler for the given pid, in case L{reapAllProcesses}
+    is called.
+
+    @param pid: the pid of the process.
+    @param process: a process handler.
+    """
+    if pid in reapProcessHandlers:
+        raise RuntimeError("Try to register an already registered process.")
+    try:
+        auxPID, status = os.waitpid(pid, os.WNOHANG)
+    except BaseException:
+        log.msg(f"Failed to reap {pid}:")
+        log.err()
+
+        if pid is None:
+            return
+
+        auxPID = None
+    if auxPID:
+        process.processEnded(status)
+    else:
+        # if auxPID is 0, there are children but none have exited
+        reapProcessHandlers[pid] = process
+
+
+def unregisterReapProcessHandler(pid, process):
+    """
+    Unregister a process handler previously registered with
+    L{registerReapProcessHandler}.
+    """
+    if not (pid in reapProcessHandlers and reapProcessHandlers[pid] == process):
+        raise RuntimeError("Try to unregister a process not registered.")
+    del reapProcessHandlers[pid]
+
+
+class ProcessWriter(abstract.FileDescriptor):
+    """
+    (Internal) Helper class to write into a Process's input pipe.
+
+    I am a helper which describes a selectable asynchronous writer to a
+    process's input pipe, including stdin.
+
+    @ivar enableReadHack: A flag which determines how readability on this
+        write descriptor will be handled.  If C{True}, then readability may
+        indicate the reader for this write descriptor has been closed (ie,
+        the connection has been lost).  If C{False}, then readability events
+        are ignored.
+    """
+
+    connected = 1
+    ic = 0
+    enableReadHack = False
+
+    def __init__(self, reactor, proc, name, fileno, forceReadHack=False):
+        """
+        Initialize, specifying a Process instance to connect to.
+        """
+        abstract.FileDescriptor.__init__(self, reactor)
+        fdesc.setNonBlocking(fileno)
+        self.proc = proc
+        self.name = name
+        self.fd = fileno
+
+        if not stat.S_ISFIFO(os.fstat(self.fileno()).st_mode):
+            # If the fd is not a pipe, then the read hack is never
+            # applicable.  This case arises when ProcessWriter is used by
+            # StandardIO and stdout is redirected to a normal file.
+            self.enableReadHack = False
+        elif forceReadHack:
+            self.enableReadHack = True
+        else:
+            # Detect if this fd is actually a write-only fd. If it's
+            # valid to read, don't try to detect closing via read.
+            # This really only means that we cannot detect a TTY's write
+            # pipe being closed.
+            try:
+                os.read(self.fileno(), 0)
+            except OSError:
+                # It's a write-only pipe end, enable hack
+                self.enableReadHack = True
+
+        if self.enableReadHack:
+            self.startReading()
+
+    def fileno(self):
+        """
+        Return the fileno() of my process's stdin.
+        """
+        return self.fd
+
+    def writeSomeData(self, data):
+        """
+        Write some data to the open process.
+        """
+        rv = fdesc.writeToFD(self.fd, data)
+        if rv == len(data) and self.enableReadHack:
+            # If the send buffer is now empty and it is necessary to monitor
+            # this descriptor for readability to detect close, try detecting
+            # readability now.
+            self.startReading()
+        return rv
+
+    def write(self, data):
+        self.stopReading()
+        abstract.FileDescriptor.write(self, data)
+
+    def doRead(self):
+        """
+        The only way a write pipe can become "readable" is at EOF, because the
+        child has closed it, and we're using a reactor which doesn't
+        distinguish between readable and closed (such as the select reactor).
+
+        Except that's not true on linux < 2.6.11. It has the following
+        characteristics: write pipe is completely empty => POLLOUT (writable in
+        select), write pipe is not completely empty => POLLIN (readable in
+        select), write pipe's reader closed => POLLIN|POLLERR (readable and
+        writable in select)
+
+        That's what this funky code is for. If linux was not broken, this
+        function could be simply "return CONNECTION_LOST".
+        """
+        if self.enableReadHack:
+            return CONNECTION_LOST
+        else:
+            self.stopReading()
+
+    def connectionLost(self, reason):
+        """
+        See abstract.FileDescriptor.connectionLost.
+        """
+        # At least on macOS 10.4, exiting while stdout is non-blocking can
+        # result in data loss.  For some reason putting the file descriptor
+        # back into blocking mode seems to resolve this issue.
+        fdesc.setBlocking(self.fd)
+
+        abstract.FileDescriptor.connectionLost(self, reason)
+        self.proc.childConnectionLost(self.name, reason)
+
+
+class ProcessReader(abstract.FileDescriptor):
+    """
+    ProcessReader
+
+    I am a selectable representation of a process's output pipe, such as
+    stdout and stderr.
+    """
+
+    connected = True
+
+    def __init__(self, reactor, proc, name, fileno):
+        """
+        Initialize, specifying a process to connect to.
+        """
+        abstract.FileDescriptor.__init__(self, reactor)
+        fdesc.setNonBlocking(fileno)
+        self.proc = proc
+        self.name = name
+        self.fd = fileno
+        self.startReading()
+
+    def fileno(self):
+        """
+        Return the fileno() of my process's stderr.
+        """
+        return self.fd
+
+    def writeSomeData(self, data):
+        # the only time this is actually called is after .loseConnection Any
+        # actual write attempt would fail, so we must avoid that. This hack
+        # allows us to use .loseConnection on both readers and writers.
+        assert data == b""
+        return CONNECTION_LOST
+
+    def doRead(self):
+        """
+        This is called when the pipe becomes readable.
+        """
+        return fdesc.readFromFD(self.fd, self.dataReceived)
+
+    def dataReceived(self, data):
+        self.proc.childDataReceived(self.name, data)
+
+    def loseConnection(self):
+        if self.connected and not self.disconnecting:
+            self.disconnecting = 1
+            self.stopReading()
+            self.reactor.callLater(
+                0, self.connectionLost, failure.Failure(CONNECTION_DONE)
+            )
+
+    def connectionLost(self, reason):
+        """
+        Close my end of the pipe, signal the Process (which signals the
+        ProcessProtocol).
+        """
+        abstract.FileDescriptor.connectionLost(self, reason)
+        self.proc.childConnectionLost(self.name, reason)
+
+
+class _BaseProcess(BaseProcess):
+    """
+    Base class for Process and PTYProcess.
+    """
+
+    status: Optional[int] = None
+    pid = None
+
+    def reapProcess(self):
+        """
+        Try to reap a process (without blocking) via waitpid.
+
+        This is called when sigchild is caught or a Process object loses its
+        "connection" (stdout is closed) This ought to result in reaping all
+        zombie processes, since it will be called twice as often as it needs
+        to be.
+
+        (Unfortunately, this is a slightly experimental approach, since
+        UNIX has no way to be really sure that your process is going to
+        go away w/o blocking.  I don't want to block.)
+        """
+        try:
+            try:
+                pid, status = os.waitpid(self.pid, os.WNOHANG)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # no child process
+                    pid = None
+                else:
+                    raise
+        except BaseException:
+            log.msg(f"Failed to reap {self.pid}:")
+            log.err()
+            pid = None
+        if pid:
+            unregisterReapProcessHandler(pid, self)
+            self.processEnded(status)
+
+    def _getReason(self, status):
+        exitCode = sig = None
+        if os.WIFEXITED(status):
+            exitCode = os.WEXITSTATUS(status)
+        else:
+            sig = os.WTERMSIG(status)
+        if exitCode or sig:
+            return error.ProcessTerminated(exitCode, sig, status)
+        return error.ProcessDone(status)
+
+    def signalProcess(self, signalID):
+        """
+        Send the given signal C{signalID} to the process. It'll translate a
+        few signals ('HUP', 'STOP', 'INT', 'KILL', 'TERM') from a string
+        representation to its int value, otherwise it'll pass directly the
+        value provided
+
+        @type signalID: C{str} or C{int}
+        """
+        if signalID in ("HUP", "STOP", "INT", "KILL", "TERM"):
+            signalID = getattr(signal, f"SIG{signalID}")
+        if self.pid is None:
+            raise ProcessExitedAlready()
+        try:
+            os.kill(self.pid, signalID)
+        except OSError as e:
+            if e.errno == errno.ESRCH:
+                raise ProcessExitedAlready()
+            else:
+                raise
+
+    def _resetSignalDisposition(self):
+        # The Python interpreter ignores some signals, and our child
+        # process will inherit that behaviour. To have a child process
+        # that responds to signals normally, we need to reset our
+        # child process's signal handling (just) after we fork and
+        # before we execvpe.
+        for signalnum in range(1, signal.NSIG):
+            if signal.getsignal(signalnum) == signal.SIG_IGN:
+                # Reset signal handling to the default
+                signal.signal(signalnum, signal.SIG_DFL)
+
+    def _trySpawnInsteadOfFork(
+        self, path, uid, gid, executable, args, environment, kwargs
+    ):
+        """
+        Try to use posix_spawnp() instead of fork(), if possible.
+
+        This implementation returns False because the non-PTY subclass
+        implements the actual logic; we can't yet use this for pty processes.
+
+        @return: a boolean indicating whether posix_spawnp() was used or not.
+        """
+        return False
+
+    def _fork(self, path, uid, gid, executable, args, environment, **kwargs):
+        """
+        Fork and then exec sub-process.
+
+        @param path: the path where to run the new process.
+        @type path: L{bytes} or L{unicode}
+
+        @param uid: if defined, the uid used to run the new process.
+        @type uid: L{int}
+
+        @param gid: if defined, the gid used to run the new process.
+        @type gid: L{int}
+
+        @param executable: the executable to run in a new process.
+        @type executable: L{str}
+
+        @param args: arguments used to create the new process.
+        @type args: L{list}.
+
+        @param environment: environment used for the new process.
+        @type environment: L{dict}.
+
+        @param kwargs: keyword arguments to L{_setupChild} method.
+        """
+
+        if self._trySpawnInsteadOfFork(
+            path, uid, gid, executable, args, environment, kwargs
+        ):
+            return
+
+        collectorEnabled = gc.isenabled()
+        gc.disable()
+        try:
+            self.pid = os.fork()
+        except BaseException:
+            # Still in the parent process
+            if collectorEnabled:
+                gc.enable()
+            raise
+        else:
+            if self.pid == 0:
+                # A return value of 0 from fork() indicates that we are now
+                # executing in the child process.
+
+                # Do not put *ANY* code outside the try block. The child
+                # process must either exec or _exit. If it gets outside this
+                # block (due to an exception that is not handled here, but
+                # which might be handled higher up), there will be two copies
+                # of the parent running in parallel, doing all kinds of damage.
+
+                # After each change to this code, review it to make sure there
+                # are no exit paths.
+
+                try:
+                    # Stop debugging. If I am, I don't care anymore.
+                    sys.settrace(None)
+                    self._setupChild(**kwargs)
+                    self._execChild(path, uid, gid, executable, args, environment)
+                except BaseException:
+                    # If there are errors, try to write something descriptive
+                    # to stderr before exiting.
+
+                    # The parent's stderr isn't *necessarily* fd 2 anymore, or
+                    # even still available; however, even libc assumes that
+                    # write(2, err) is a useful thing to attempt.
+
+                    try:
+                        # On Python 3, print_exc takes a text stream, but
+                        # on Python 2 it still takes a byte stream.  So on
+                        # Python 3 we will wrap up the byte stream returned
+                        # by os.fdopen using TextIOWrapper.
+
+                        # We hard-code UTF-8 as the encoding here, rather
+                        # than looking at something like
+                        # getfilesystemencoding() or sys.stderr.encoding,
+                        # because we want an encoding that will be able to
+                        # encode the full range of code points.  We are
+                        # (most likely) talking to the parent process on
+                        # the other end of this pipe and not the filesystem
+                        # or the original sys.stderr, so there's no point
+                        # in trying to match the encoding of one of those
+                        # objects.
+
+                        stderr = io.TextIOWrapper(os.fdopen(2, "wb"), encoding="utf-8")
+                        msg = ("Upon execvpe {} {} in environment id {}" "\n:").format(
+                            executable, str(args), id(environment)
+                        )
+                        stderr.write(msg)
+                        traceback.print_exc(file=stderr)
+                        stderr.flush()
+
+                        for fd in range(3):
+                            os.close(fd)
+                    except BaseException:
+                        # Handle all errors during the error-reporting process
+                        # silently to ensure that the child terminates.
+                        pass
+
+                # See comment above about making sure that we reach this line
+                # of code.
+                os._exit(1)
+
+        # we are now in parent process
+        if collectorEnabled:
+            gc.enable()
+        self.status = -1  # this records the exit status of the child
+
+    def _setupChild(self, *args, **kwargs):
+        """
+        Setup the child process. Override in subclasses.
+        """
+        raise NotImplementedError()
+
+    def _execChild(self, path, uid, gid, executable, args, environment):
+        """
+        The exec() which is done in the forked child.
+        """
+        if path:
+            os.chdir(path)
+        if uid is not None or gid is not None:
+            if uid is None:
+                uid = os.geteuid()
+            if gid is None:
+                gid = os.getegid()
+            # set the UID before I actually exec the process
+            os.setuid(0)
+            os.setgid(0)
+            switchUID(uid, gid)
+        os.execvpe(executable, args, environment)
+
+    def __repr__(self) -> str:
+        """
+        String representation of a process.
+        """
+        return "<{} pid={} status={}>".format(
+            self.__class__.__name__,
+            self.pid,
+            self.status,
+        )
+
+
+class _FDDetector:
+    """
+    This class contains the logic necessary to decide which of the available
+    system techniques should be used to detect the open file descriptors for
+    the current process. The chosen technique gets monkey-patched into the
+    _listOpenFDs method of this class so that the detection only needs to occur
+    once.
+
+    @ivar listdir: The implementation of listdir to use. This gets overwritten
+        by the test cases.
+    @ivar getpid: The implementation of getpid to use, returns the PID of the
+        running process.
+    @ivar openfile: The implementation of open() to use, by default the Python
+        builtin.
+    """
+
+    # So that we can unit test this
+    listdir = os.listdir
+    getpid = os.getpid
+    openfile = open
+
+    def __init__(self):
+        self._implementations = [
+            self._procFDImplementation,
+            self._devFDImplementation,
+            self._fallbackFDImplementation,
+        ]
+
+    def _listOpenFDs(self):
+        """
+        Return an iterable of file descriptors which I{may} be open in this
+        process.
+
+        This will try to return the fewest possible descriptors without missing
+        any.
+        """
+        self._listOpenFDs = self._getImplementation()
+        return self._listOpenFDs()
+
+    def _getImplementation(self):
+        """
+        Pick a method which gives correct results for C{_listOpenFDs} in this
+        runtime environment.
+
+        This involves a lot of very platform-specific checks, some of which may
+        be relatively expensive.  Therefore the returned method should be saved
+        and re-used, rather than always calling this method to determine what it
+        is.
+
+        See the implementation for the details of how a method is selected.
+        """
+        for impl in self._implementations:
+            try:
+                before = impl()
+            except BaseException:
+                continue
+            with self.openfile("/dev/null", "r"):
+                after = impl()
+            if before != after:
+                return impl
+        # If no implementation can detect the newly opened file above, then just
+        # return the last one.  The last one should therefore always be one
+        # which makes a simple static guess which includes all possible open
+        # file descriptors, but perhaps also many other values which do not
+        # correspond to file descriptors.  For example, the scheme implemented
+        # by _fallbackFDImplementation is suitable to be the last entry.
+        return impl
+
+    def _devFDImplementation(self):
+        """
+        Simple implementation for systems where /dev/fd actually works.
+        See: http://www.freebsd.org/cgi/man.cgi?fdescfs
+        """
+        dname = "/dev/fd"
+        result = [int(fd) for fd in self.listdir(dname)]
+        return result
+
+    def _procFDImplementation(self):
+        """
+        Simple implementation for systems where /proc/pid/fd exists (we assume
+        it works).
+        """
+        dname = "/proc/%d/fd" % (self.getpid(),)
+        return [int(fd) for fd in self.listdir(dname)]
+
+    def _fallbackFDImplementation(self):
+        """
+        Fallback implementation where either the resource module can inform us
+        about the upper bound of how many FDs to expect, or where we just guess
+        a constant maximum if there is no resource module.
+
+        All possible file descriptors from 0 to that upper bound are returned
+        with no attempt to exclude invalid file descriptor values.
+        """
+        try:
+            import resource
+        except ImportError:
+            maxfds = 1024
+        else:
+            # OS-X reports 9223372036854775808. That's a lot of fds to close.
+            # OS-X should get the /dev/fd implementation instead, so mostly
+            # this check probably isn't necessary.
+            maxfds = min(1024, resource.getrlimit(resource.RLIMIT_NOFILE)[1])
+        return range(maxfds)
+
+
+detector = _FDDetector()
+
+
+def _listOpenFDs():
+    """
+    Use the global detector object to figure out which FD implementation to
+    use.
+    """
+    return detector._listOpenFDs()
+
+
+def _getFileActions(
+    fdState: List[Tuple[int, bool]],
+    childToParentFD: Dict[int, int],
+    doClose: int,
+    doDup2: int,
+) -> List[Tuple[int, ...]]:
+    """
+    Get the C{file_actions} parameter for C{posix_spawn} based on the
+    parameters describing the current process state.
+
+    @param fdState: A list of 2-tuples of (file descriptor, close-on-exec
+        flag).
+
+    @param doClose: the integer to use for the 'close' instruction
+
+    @param doDup2: the integer to use for the 'dup2' instruction
+    """
+    fdStateDict = dict(fdState)
+    parentToChildren: Dict[int, List[int]] = defaultdict(list)
+    for inChild, inParent in childToParentFD.items():
+        parentToChildren[inParent].append(inChild)
+    allocated = set(fdStateDict)
+    allocated |= set(childToParentFD.values())
+    allocated |= set(childToParentFD.keys())
+    nextFD = 0
+
+    def allocateFD() -> int:
+        nonlocal nextFD
+        while nextFD in allocated:
+            nextFD += 1
+        allocated.add(nextFD)
+        return nextFD
+
+    result: List[Tuple[int, ...]] = []
+    relocations = {}
+    for inChild, inParent in sorted(childToParentFD.items()):
+        # The parent FD will later be reused by a child FD.
+        parentToChildren[inParent].remove(inChild)
+        if parentToChildren[inChild]:
+            new = relocations[inChild] = allocateFD()
+            result.append((doDup2, inChild, new))
+        if inParent in relocations:
+            result.append((doDup2, relocations[inParent], inChild))
+            if not parentToChildren[inParent]:
+                result.append((doClose, relocations[inParent]))
+        else:
+            if inParent == inChild:
+                if fdStateDict[inParent]:
+                    # If the child is attempting to inherit the parent as-is,
+                    # and it is not close-on-exec, the job is already done; we
+                    # can bail.  Otherwise...
+
+                    tempFD = allocateFD()
+                    # The child wants to inherit the parent as-is, so the
+                    # handle must be heritable.. dup2 makes the new descriptor
+                    # inheritable by default, *but*, per the man page, “if
+                    # fildes and fildes2 are equal, then dup2() just returns
+                    # fildes2; no other changes are made to the existing
+                    # descriptor”, so we need to dup it somewhere else and dup
+                    # it back before closing the temporary place we put it.
+                    result.extend(
+                        [
+                            (doDup2, inParent, tempFD),
+                            (doDup2, tempFD, inChild),
+                            (doClose, tempFD),
+                        ]
+                    )
+            else:
+                result.append((doDup2, inParent, inChild))
+
+    for eachFD, uninheritable in fdStateDict.items():
+        if eachFD not in childToParentFD and not uninheritable:
+            result.append((doClose, eachFD))
+
+    return result
+
+
+@implementer(IProcessTransport)
+class Process(_BaseProcess):
+    """
+    An operating-system Process.
+
+    This represents an operating-system process with arbitrary input/output
+    pipes connected to it.  Those pipes may represent standard input, standard
+    output, and standard error, or any other file descriptor.
+
+    On UNIX, this is implemented using posix_spawnp() when possible (or fork(),
+    exec(), pipe() and fcntl() when not).  These calls may not exist elsewhere
+    so this code is not cross-platform.  (also, windows can only select on
+    sockets...)
+    """
+
+    debug = False
+    debug_child = False
+
+    status = -1
+    pid = None
+
+    processWriterFactory = ProcessWriter
+    processReaderFactory = ProcessReader
+
+    def __init__(
+        self,
+        reactor,
+        executable,
+        args,
+        environment,
+        path,
+        proto,
+        uid=None,
+        gid=None,
+        childFDs=None,
+    ):
+        """
+        Spawn an operating-system process.
+
+        This is where the hard work of disconnecting all currently open
+        files / forking / executing the new process happens.  (This is
+        executed automatically when a Process is instantiated.)
+
+        This will also run the subprocess as a given user ID and group ID, if
+        specified.  (Implementation Note: this doesn't support all the arcane
+        nuances of setXXuid on UNIX: it will assume that either your effective
+        or real UID is 0.)
+        """
+        self._reactor = reactor
+        if not proto:
+            assert "r" not in childFDs.values()
+            assert "w" not in childFDs.values()
+        _BaseProcess.__init__(self, proto)
+
+        self.pipes = {}
+        # keys are childFDs, we can sense them closing
+        # values are ProcessReader/ProcessWriters
+
+        helpers = {}
+        # keys are childFDs
+        # values are parentFDs
+
+        if childFDs is None:
+            childFDs = {
+                0: "w",  # we write to the child's stdin
+                1: "r",  # we read from their stdout
+                2: "r",  # and we read from their stderr
+            }
+
+        debug = self.debug
+        if debug:
+            print("childFDs", childFDs)
+
+        _openedPipes = []
+
+        def pipe():
+            r, w = os.pipe()
+            _openedPipes.extend([r, w])
+            return r, w
+
+        # fdmap.keys() are filenos of pipes that are used by the child.
+        fdmap = {}  # maps childFD to parentFD
+        try:
+            for childFD, target in childFDs.items():
+                if debug:
+                    print("[%d]" % childFD, target)
+                if target == "r":
+                    # we need a pipe that the parent can read from
+                    readFD, writeFD = pipe()
+                    if debug:
+                        print("readFD=%d, writeFD=%d" % (readFD, writeFD))
+                    fdmap[childFD] = writeFD  # child writes to this
+                    helpers[childFD] = readFD  # parent reads from this
+                elif target == "w":
+                    # we need a pipe that the parent can write to
+                    readFD, writeFD = pipe()
+                    if debug:
+                        print("readFD=%d, writeFD=%d" % (readFD, writeFD))
+                    fdmap[childFD] = readFD  # child reads from this
+                    helpers[childFD] = writeFD  # parent writes to this
+                else:
+                    assert type(target) == int, f"{target!r} should be an int"
+                    fdmap[childFD] = target  # parent ignores this
+            if debug:
+                print("fdmap", fdmap)
+            if debug:
+                print("helpers", helpers)
+            # the child only cares about fdmap.values()
+
+            self._fork(path, uid, gid, executable, args, environment, fdmap=fdmap)
+        except BaseException:
+            for pipe in _openedPipes:
+                os.close(pipe)
+            raise
+
+        # we are the parent process:
+        self.proto = proto
+
+        # arrange for the parent-side pipes to be read and written
+        for childFD, parentFD in helpers.items():
+            os.close(fdmap[childFD])
+            if childFDs[childFD] == "r":
+                reader = self.processReaderFactory(reactor, self, childFD, parentFD)
+                self.pipes[childFD] = reader
+
+            if childFDs[childFD] == "w":
+                writer = self.processWriterFactory(
+                    reactor, self, childFD, parentFD, forceReadHack=True
+                )
+                self.pipes[childFD] = writer
+
+        try:
+            # the 'transport' is used for some compatibility methods
+            if self.proto is not None:
+                self.proto.makeConnection(self)
+        except BaseException:
+            log.err()
+
+        # The reactor might not be running yet.  This might call back into
+        # processEnded synchronously, triggering an application-visible
+        # callback.  That's probably not ideal.  The replacement API for
+        # spawnProcess should improve upon this situation.
+        registerReapProcessHandler(self.pid, self)
+
+    def _trySpawnInsteadOfFork(
+        self, path, uid, gid, executable, args, environment, kwargs
+    ):
+        """
+        Try to use posix_spawnp() instead of fork(), if possible.
+
+        @return: a boolean indicating whether posix_spawnp() was used or not.
+        """
+        if (
+            # no support for setuid/setgid anywhere but in QNX's
+            # posix_spawnattr_setcred
+            (uid is not None)
+            or (gid is not None)
+            or ((path is not None) and (os.path.abspath(path) != os.path.abspath(".")))
+            or getattr(self._reactor, "_neverUseSpawn", False)
+        ):
+            return False
+        fdmap = kwargs.get("fdmap")
+        fdState = []
+        for eachFD in _listOpenFDs():
+            try:
+                isCloseOnExec = fcntl.fcntl(eachFD, fcntl.F_GETFD, fcntl.FD_CLOEXEC)
+            except OSError:
+                pass
+            else:
+                fdState.append((eachFD, isCloseOnExec))
+        if environment is None:
+            environment = {}
+
+        setSigDef = [
+            everySignal
+            for everySignal in range(1, signal.NSIG)
+            if signal.getsignal(everySignal) == signal.SIG_IGN
+        ]
+
+        self.pid = os.posix_spawnp(
+            executable,
+            args,
+            environment,
+            file_actions=_getFileActions(
+                fdState, fdmap, doClose=_PS_CLOSE, doDup2=_PS_DUP2
+            ),
+            setsigdef=setSigDef,
+        )
+        self.status = -1
+        return True
+
+    if getattr(os, "posix_spawnp", None) is None:
+        # If there's no posix_spawn implemented, let the superclass handle it
+        del _trySpawnInsteadOfFork
+
+    def _setupChild(self, fdmap):
+        """
+        fdmap[childFD] = parentFD
+
+        The child wants to end up with 'childFD' attached to what used to be
+        the parent's parentFD. As an example, a bash command run like
+        'command 2>&1' would correspond to an fdmap of {0:0, 1:1, 2:1}.
+        'command >foo.txt' would be {0:0, 1:os.open('foo.txt'), 2:2}.
+
+        This is accomplished in two steps::
+
+            1. close all file descriptors that aren't values of fdmap.  This
+               means 0 .. maxfds (or just the open fds within that range, if
+               the platform supports '/proc/<pid>/fd').
+
+            2. for each childFD::
+
+                 - if fdmap[childFD] == childFD, the descriptor is already in
+                   place.  Make sure the CLOEXEC flag is not set, then delete
+                   the entry from fdmap.
+
+                 - if childFD is in fdmap.values(), then the target descriptor
+                   is busy. Use os.dup() to move it elsewhere, update all
+                   fdmap[childFD] items that point to it, then close the
+                   original. Then fall through to the next case.
+
+                 - now fdmap[childFD] is not in fdmap.values(), and is free.
+                   Use os.dup2() to move it to the right place, then close the
+                   original.
+        """
+        debug = self.debug_child
+        if debug:
+            errfd = sys.stderr
+            errfd.write("starting _setupChild\n")
+
+        destList = fdmap.values()
+        for fd in _listOpenFDs():
+            if fd in destList:
+                continue
+            if debug and fd == errfd.fileno():
+                continue
+            try:
+                os.close(fd)
+            except BaseException:
+                pass
+
+        # at this point, the only fds still open are the ones that need to
+        # be moved to their appropriate positions in the child (the targets
+        # of fdmap, i.e. fdmap.values() )
+
+        if debug:
+            print("fdmap", fdmap, file=errfd)
+        for child in sorted(fdmap.keys()):
+            target = fdmap[child]
+            if target == child:
+                # fd is already in place
+                if debug:
+                    print("%d already in place" % target, file=errfd)
+                fdesc._unsetCloseOnExec(child)
+            else:
+                if child in fdmap.values():
+                    # we can't replace child-fd yet, as some other mapping
+                    # still needs the fd it wants to target. We must preserve
+                    # that old fd by duping it to a new home.
+                    newtarget = os.dup(child)  # give it a safe home
+                    if debug:
+                        print("os.dup(%d) -> %d" % (child, newtarget), file=errfd)
+                    os.close(child)  # close the original
+                    for c, p in list(fdmap.items()):
+                        if p == child:
+                            fdmap[c] = newtarget  # update all pointers
+                # now it should be available
+                if debug:
+                    print("os.dup2(%d,%d)" % (target, child), file=errfd)
+                os.dup2(target, child)
+
+        # At this point, the child has everything it needs. We want to close
+        # everything that isn't going to be used by the child, i.e.
+        # everything not in fdmap.keys(). The only remaining fds open are
+        # those in fdmap.values().
+
+        # Any given fd may appear in fdmap.values() multiple times, so we
+        # need to remove duplicates first.
+
+        old = []
+        for fd in fdmap.values():
+            if fd not in old:
+                if fd not in fdmap.keys():
+                    old.append(fd)
+        if debug:
+            print("old", old, file=errfd)
+        for fd in old:
+            os.close(fd)
+
+        self._resetSignalDisposition()
+
+    def writeToChild(self, childFD, data):
+        self.pipes[childFD].write(data)
+
+    def closeChildFD(self, childFD):
+        # for writer pipes, loseConnection tries to write the remaining data
+        # out to the pipe before closing it
+        # if childFD is not in the list of pipes, assume that it is already
+        # closed
+        if childFD in self.pipes:
+            self.pipes[childFD].loseConnection()
+
+    def pauseProducing(self):
+        for p in self.pipes.values():
+            if isinstance(p, ProcessReader):
+                p.stopReading()
+
+    def resumeProducing(self):
+        for p in self.pipes.values():
+            if isinstance(p, ProcessReader):
+                p.startReading()
+
+    # compatibility
+    def closeStdin(self):
+        """
+        Call this to close standard input on this process.
+        """
+        self.closeChildFD(0)
+
+    def closeStdout(self):
+        self.closeChildFD(1)
+
+    def closeStderr(self):
+        self.closeChildFD(2)
+
+    def loseConnection(self):
+        self.closeStdin()
+        self.closeStderr()
+        self.closeStdout()
+
+    def write(self, data):
+        """
+        Call this to write to standard input on this process.
+
+        NOTE: This will silently lose data if there is no standard input.
+        """
+        if 0 in self.pipes:
+            self.pipes[0].write(data)
+
+    def registerProducer(self, producer, streaming):
+        """
+        Call this to register producer for standard input.
+
+        If there is no standard input producer.stopProducing() will
+        be called immediately.
+        """
+        if 0 in self.pipes:
+            self.pipes[0].registerProducer(producer, streaming)
+        else:
+            producer.stopProducing()
+
+    def unregisterProducer(self):
+        """
+        Call this to unregister producer for standard input."""
+        if 0 in self.pipes:
+            self.pipes[0].unregisterProducer()
+
+    def writeSequence(self, seq):
+        """
+        Call this to write to standard input on this process.
+
+        NOTE: This will silently lose data if there is no standard input.
+        """
+        if 0 in self.pipes:
+            self.pipes[0].writeSequence(seq)
+
+    def childDataReceived(self, name, data):
+        self.proto.childDataReceived(name, data)
+
+    def childConnectionLost(self, childFD, reason):
+        # this is called when one of the helpers (ProcessReader or
+        # ProcessWriter) notices their pipe has been closed
+        os.close(self.pipes[childFD].fileno())
+        del self.pipes[childFD]
+        try:
+            self.proto.childConnectionLost(childFD)
+        except BaseException:
+            log.err()
+        self.maybeCallProcessEnded()
+
+    def maybeCallProcessEnded(self):
+        # we don't call ProcessProtocol.processEnded until:
+        #  the child has terminated, AND
+        #  all writers have indicated an error status, AND
+        #  all readers have indicated EOF
+        # This insures that we've gathered all output from the process.
+        if self.pipes:
+            return
+        if not self.lostProcess:
+            self.reapProcess()
+            return
+        _BaseProcess.maybeCallProcessEnded(self)
+
+    def getHost(self):
+        # ITransport.getHost
+        raise NotImplementedError()
+
+    def getPeer(self):
+        # ITransport.getPeer
+        raise NotImplementedError()
+
+
+@implementer(IProcessTransport)
+class PTYProcess(abstract.FileDescriptor, _BaseProcess):
+    """
+    An operating-system Process that uses PTY support.
+    """
+
+    status = -1
+    pid = None
+
+    def __init__(
+        self,
+        reactor,
+        executable,
+        args,
+        environment,
+        path,
+        proto,
+        uid=None,
+        gid=None,
+        usePTY=None,
+    ):
+        """
+        Spawn an operating-system process.
+
+        This is where the hard work of disconnecting all currently open
+        files / forking / executing the new process happens.  (This is
+        executed automatically when a Process is instantiated.)
+
+        This will also run the subprocess as a given user ID and group ID, if
+        specified.  (Implementation Note: this doesn't support all the arcane
+        nuances of setXXuid on UNIX: it will assume that either your effective
+        or real UID is 0.)
+        """
+        if pty is None and not isinstance(usePTY, (tuple, list)):
+            # no pty module and we didn't get a pty to use
+            raise NotImplementedError(
+                "cannot use PTYProcess on platforms without the pty module."
+            )
+        abstract.FileDescriptor.__init__(self, reactor)
+        _BaseProcess.__init__(self, proto)
+
+        if isinstance(usePTY, (tuple, list)):
+            masterfd, slavefd, _ = usePTY
+        else:
+            masterfd, slavefd = pty.openpty()
+
+        try:
+            self._fork(
+                path,
+                uid,
+                gid,
+                executable,
+                args,
+                environment,
+                masterfd=masterfd,
+                slavefd=slavefd,
+            )
+        except BaseException:
+            if not isinstance(usePTY, (tuple, list)):
+                os.close(masterfd)
+                os.close(slavefd)
+            raise
+
+        # we are now in parent process:
+        os.close(slavefd)
+        fdesc.setNonBlocking(masterfd)
+        self.fd = masterfd
+        self.startReading()
+        self.connected = 1
+        self.status = -1
+        try:
+            self.proto.makeConnection(self)
+        except BaseException:
+            log.err()
+        registerReapProcessHandler(self.pid, self)
+
+    def _setupChild(self, masterfd, slavefd):
+        """
+        Set up child process after C{fork()} but before C{exec()}.
+
+        This involves:
+
+            - closing C{masterfd}, since it is not used in the subprocess
+
+            - creating a new session with C{os.setsid}
+
+            - changing the controlling terminal of the process (and the new
+              session) to point at C{slavefd}
+
+            - duplicating C{slavefd} to standard input, output, and error
+
+            - closing all other open file descriptors (according to
+              L{_listOpenFDs})
+
+            - re-setting all signal handlers to C{SIG_DFL}
+
+        @param masterfd: The master end of a PTY file descriptors opened with
+            C{openpty}.
+        @type masterfd: L{int}
+
+        @param slavefd: The slave end of a PTY opened with C{openpty}.
+        @type slavefd: L{int}
+        """
+        os.close(masterfd)
+        os.setsid()
+        fcntl.ioctl(slavefd, termios.TIOCSCTTY, "")
+
+        for fd in range(3):
+            if fd != slavefd:
+                os.close(fd)
+
+        os.dup2(slavefd, 0)  # stdin
+        os.dup2(slavefd, 1)  # stdout
+        os.dup2(slavefd, 2)  # stderr
+
+        for fd in _listOpenFDs():
+            if fd > 2:
+                try:
+                    os.close(fd)
+                except BaseException:
+                    pass
+
+        self._resetSignalDisposition()
+
+    def closeStdin(self):
+        # PTYs do not have stdin/stdout/stderr. They only have in and out, just
+        # like sockets. You cannot close one without closing off the entire PTY
+        pass
+
+    def closeStdout(self):
+        pass
+
+    def closeStderr(self):
+        pass
+
+    def doRead(self):
+        """
+        Called when my standard output stream is ready for reading.
+        """
+        return fdesc.readFromFD(
+            self.fd, lambda data: self.proto.childDataReceived(1, data)
+        )
+
+    def fileno(self):
+        """
+        This returns the file number of standard output on this process.
+        """
+        return self.fd
+
+    def maybeCallProcessEnded(self):
+        # two things must happen before we call the ProcessProtocol's
+        # processEnded method. 1: the child process must die and be reaped
+        # (which calls our own processEnded method). 2: the child must close
+        # their stdin/stdout/stderr fds, causing the pty to close, causing
+        # our connectionLost method to be called. #2 can also be triggered
+        # by calling .loseConnection().
+        if self.lostProcess == 2:
+            _BaseProcess.maybeCallProcessEnded(self)
+
+    def connectionLost(self, reason):
+        """
+        I call this to clean up when one or all of my connections has died.
+        """
+        abstract.FileDescriptor.connectionLost(self, reason)
+        os.close(self.fd)
+        self.lostProcess += 1
+        self.maybeCallProcessEnded()
+
+    def writeSomeData(self, data):
+        """
+        Write some data to the open process.
+        """
+        return fdesc.writeToFD(self.fd, data)
+
+    def closeChildFD(self, descriptor):
+        # IProcessTransport
+        raise NotImplementedError()
+
+    def writeToChild(self, childFD, data):
+        # IProcessTransport
+        raise NotImplementedError()
diff --git a/contrib/python/Twisted/py3/twisted/internet/protocol.py b/contrib/python/Twisted/py3/twisted/internet/protocol.py
new file mode 100644
index 00000000000..4fcf0e10386
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/protocol.py
@@ -0,0 +1,900 @@
+# -*- test-case-name: twisted.test.test_factories,twisted.internet.test.test_protocol -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Standard implementations of Twisted protocol-related interfaces.
+
+Start here if you are looking to write a new protocol implementation for
+Twisted.  The Protocol class contains some introductory material.
+"""
+
+
+import random
+from typing import Any, Callable, Optional
+
+from zope.interface import implementer
+
+from twisted.internet import defer, error, interfaces
+from twisted.internet.interfaces import IAddress, ITransport
+from twisted.logger import _loggerFor
+from twisted.python import components, failure, log
+
+
+@implementer(interfaces.IProtocolFactory, interfaces.ILoggingContext)
+class Factory:
+    """
+    This is a factory which produces protocols.
+
+    By default, buildProtocol will create a protocol of the class given in
+    self.protocol.
+    """
+
+    protocol: "Optional[Callable[[], Protocol]]" = None
+
+    numPorts = 0
+    noisy = True
+
+    @classmethod
+    def forProtocol(cls, protocol, *args, **kwargs):
+        """
+        Create a factory for the given protocol.
+
+        It sets the C{protocol} attribute and returns the constructed factory
+        instance.
+
+        @param protocol: A L{Protocol} subclass
+
+        @param args: Positional arguments for the factory.
+
+        @param kwargs: Keyword arguments for the factory.
+
+        @return: A L{Factory} instance wired up to C{protocol}.
+        """
+        factory = cls(*args, **kwargs)
+        factory.protocol = protocol
+        return factory
+
+    def logPrefix(self):
+        """
+        Describe this factory for log messages.
+        """
+        return self.__class__.__name__
+
+    def doStart(self):
+        """
+        Make sure startFactory is called.
+
+        Users should not call this function themselves!
+        """
+        if not self.numPorts:
+            if self.noisy:
+                _loggerFor(self).info("Starting factory {factory!r}", factory=self)
+            self.startFactory()
+        self.numPorts = self.numPorts + 1
+
+    def doStop(self):
+        """
+        Make sure stopFactory is called.
+
+        Users should not call this function themselves!
+        """
+        if self.numPorts == 0:
+            # This shouldn't happen, but does sometimes and this is better
+            # than blowing up in assert as we did previously.
+            return
+        self.numPorts = self.numPorts - 1
+        if not self.numPorts:
+            if self.noisy:
+                _loggerFor(self).info("Stopping factory {factory!r}", factory=self)
+            self.stopFactory()
+
+    def startFactory(self):
+        """
+        This will be called before I begin listening on a Port or Connector.
+
+        It will only be called once, even if the factory is connected
+        to multiple ports.
+
+        This can be used to perform 'unserialization' tasks that
+        are best put off until things are actually running, such
+        as connecting to a database, opening files, etcetera.
+        """
+
+    def stopFactory(self):
+        """
+        This will be called before I stop listening on all Ports/Connectors.
+
+        This can be overridden to perform 'shutdown' tasks such as disconnecting
+        database connections, closing files, etc.
+
+        It will be called, for example, before an application shuts down,
+        if it was connected to a port. User code should not call this function
+        directly.
+        """
+
+    def buildProtocol(self, addr: IAddress) -> "Optional[Protocol]":
+        """
+        Create an instance of a subclass of Protocol.
+
+        The returned instance will handle input on an incoming server
+        connection, and an attribute "factory" pointing to the creating
+        factory.
+
+        Alternatively, L{None} may be returned to immediately close the
+        new connection.
+
+        Override this method to alter how Protocol instances get created.
+
+        @param addr: an object implementing L{IAddress}
+        """
+        assert self.protocol is not None
+        p = self.protocol()
+        p.factory = self
+        return p
+
+
+class ClientFactory(Factory):
+    """
+    A Protocol factory for clients.
+
+    This can be used together with the various connectXXX methods in
+    reactors.
+    """
+
+    def startedConnecting(self, connector):
+        """
+        Called when a connection has been started.
+
+        You can call connector.stopConnecting() to stop the connection attempt.
+
+        @param connector: a Connector object.
+        """
+
+    def clientConnectionFailed(self, connector, reason):
+        """
+        Called when a connection has failed to connect.
+
+        It may be useful to call connector.connect() - this will reconnect.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+    def clientConnectionLost(self, connector, reason):
+        """
+        Called when an established connection is lost.
+
+        It may be useful to call connector.connect() - this will reconnect.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+
+class _InstanceFactory(ClientFactory):
+    """
+    Factory used by ClientCreator.
+
+    @ivar deferred: The L{Deferred} which represents this connection attempt and
+        which will be fired when it succeeds or fails.
+
+    @ivar pending: After a connection attempt succeeds or fails, a delayed call
+        which will fire the L{Deferred} representing this connection attempt.
+    """
+
+    noisy = False
+    pending = None
+
+    def __init__(self, reactor, instance, deferred):
+        self.reactor = reactor
+        self.instance = instance
+        self.deferred = deferred
+
+    def __repr__(self) -> str:
+        return f"<ClientCreator factory: {self.instance!r}>"
+
+    def buildProtocol(self, addr):
+        """
+        Return the pre-constructed protocol instance and arrange to fire the
+        waiting L{Deferred} to indicate success establishing the connection.
+        """
+        self.pending = self.reactor.callLater(
+            0, self.fire, self.deferred.callback, self.instance
+        )
+        self.deferred = None
+        return self.instance
+
+    def clientConnectionFailed(self, connector, reason):
+        """
+        Arrange to fire the waiting L{Deferred} with the given failure to
+        indicate the connection could not be established.
+        """
+        self.pending = self.reactor.callLater(
+            0, self.fire, self.deferred.errback, reason
+        )
+        self.deferred = None
+
+    def fire(self, func, value):
+        """
+        Clear C{self.pending} to avoid a reference cycle and then invoke func
+        with the value.
+        """
+        self.pending = None
+        func(value)
+
+
+class ClientCreator:
+    """
+    Client connections that do not require a factory.
+
+    The various connect* methods create a protocol instance using the given
+    protocol class and arguments, and connect it, returning a Deferred of the
+    resulting protocol instance.
+
+    Useful for cases when we don't really need a factory.  Mainly this
+    is when there is no shared state between protocol instances, and no need
+    to reconnect.
+
+    The C{connectTCP}, C{connectUNIX}, and C{connectSSL} methods each return a
+    L{Deferred} which will fire with an instance of the protocol class passed to
+    L{ClientCreator.__init__}.  These Deferred can be cancelled to abort the
+    connection attempt (in a very unlikely case, cancelling the Deferred may not
+    prevent the protocol from being instantiated and connected to a transport;
+    if this happens, it will be disconnected immediately afterwards and the
+    Deferred will still errback with L{CancelledError}).
+    """
+
+    def __init__(self, reactor, protocolClass, *args, **kwargs):
+        self.reactor = reactor
+        self.protocolClass = protocolClass
+        self.args = args
+        self.kwargs = kwargs
+
+    def _connect(self, method, *args, **kwargs):
+        """
+        Initiate a connection attempt.
+
+        @param method: A callable which will actually start the connection
+            attempt.  For example, C{reactor.connectTCP}.
+
+        @param args: Positional arguments to pass to C{method}, excluding the
+            factory.
+
+        @param kwargs: Keyword arguments to pass to C{method}.
+
+        @return: A L{Deferred} which fires with an instance of the protocol
+            class passed to this L{ClientCreator}'s initializer or fails if the
+            connection cannot be set up for some reason.
+        """
+
+        def cancelConnect(deferred):
+            connector.disconnect()
+            if f.pending is not None:
+                f.pending.cancel()
+
+        d = defer.Deferred(cancelConnect)
+        f = _InstanceFactory(
+            self.reactor, self.protocolClass(*self.args, **self.kwargs), d
+        )
+        connector = method(factory=f, *args, **kwargs)
+        return d
+
+    def connectTCP(self, host, port, timeout=30, bindAddress=None):
+        """
+        Connect to a TCP server.
+
+        The parameters are all the same as to L{IReactorTCP.connectTCP} except
+        that the factory parameter is omitted.
+
+        @return: A L{Deferred} which fires with an instance of the protocol
+            class passed to this L{ClientCreator}'s initializer or fails if the
+            connection cannot be set up for some reason.
+        """
+        return self._connect(
+            self.reactor.connectTCP,
+            host,
+            port,
+            timeout=timeout,
+            bindAddress=bindAddress,
+        )
+
+    def connectUNIX(self, address, timeout=30, checkPID=False):
+        """
+        Connect to a Unix socket.
+
+        The parameters are all the same as to L{IReactorUNIX.connectUNIX} except
+        that the factory parameter is omitted.
+
+        @return: A L{Deferred} which fires with an instance of the protocol
+            class passed to this L{ClientCreator}'s initializer or fails if the
+            connection cannot be set up for some reason.
+        """
+        return self._connect(
+            self.reactor.connectUNIX, address, timeout=timeout, checkPID=checkPID
+        )
+
+    def connectSSL(self, host, port, contextFactory, timeout=30, bindAddress=None):
+        """
+        Connect to an SSL server.
+
+        The parameters are all the same as to L{IReactorSSL.connectSSL} except
+        that the factory parameter is omitted.
+
+        @return: A L{Deferred} which fires with an instance of the protocol
+            class passed to this L{ClientCreator}'s initializer or fails if the
+            connection cannot be set up for some reason.
+        """
+        return self._connect(
+            self.reactor.connectSSL,
+            host,
+            port,
+            contextFactory=contextFactory,
+            timeout=timeout,
+            bindAddress=bindAddress,
+        )
+
+
+class ReconnectingClientFactory(ClientFactory):
+    """
+    Factory which auto-reconnects clients with an exponential back-off.
+
+    Note that clients should call my resetDelay method after they have
+    connected successfully.
+
+    @ivar maxDelay: Maximum number of seconds between connection attempts.
+    @ivar initialDelay: Delay for the first reconnection attempt.
+    @ivar factor: A multiplicitive factor by which the delay grows
+    @ivar jitter: Percentage of randomness to introduce into the delay length
+        to prevent stampeding.
+    @ivar clock: The clock used to schedule reconnection. It's mainly useful to
+        be parametrized in tests. If the factory is serialized, this attribute
+        will not be serialized, and the default value (the reactor) will be
+        restored when deserialized.
+    @type clock: L{IReactorTime}
+    @ivar maxRetries: Maximum number of consecutive unsuccessful connection
+        attempts, after which no further connection attempts will be made. If
+        this is not explicitly set, no maximum is applied.
+    """
+
+    maxDelay = 3600
+    initialDelay = 1.0
+    # Note: These highly sensitive factors have been precisely measured by
+    # the National Institute of Science and Technology.  Take extreme care
+    # in altering them, or you may damage your Internet!
+    # (Seriously: <http://physics.nist.gov/cuu/Constants/index.html>)
+    factor = 2.7182818284590451  # (math.e)
+    # Phi = 1.6180339887498948 # (Phi is acceptable for use as a
+    # factor if e is too large for your application.)
+
+    # This is the value of the molar Planck constant times c, joule
+    # meter/mole.  The value is attributable to
+    # https://physics.nist.gov/cgi-bin/cuu/Value?nahc|search_for=molar+planck+constant+times+c
+    jitter = 0.119626565582
+
+    delay = initialDelay
+    retries = 0
+    maxRetries = None
+    _callID = None
+    connector = None
+    clock = None
+
+    continueTrying = 1
+
+    def clientConnectionFailed(self, connector, reason):
+        if self.continueTrying:
+            self.connector = connector
+            self.retry()
+
+    def clientConnectionLost(self, connector, unused_reason):
+        if self.continueTrying:
+            self.connector = connector
+            self.retry()
+
+    def retry(self, connector=None):
+        """
+        Have this connector connect again, after a suitable delay.
+        """
+        if not self.continueTrying:
+            if self.noisy:
+                log.msg(f"Abandoning {connector} on explicit request")
+            return
+
+        if connector is None:
+            if self.connector is None:
+                raise ValueError("no connector to retry")
+            else:
+                connector = self.connector
+
+        self.retries += 1
+        if self.maxRetries is not None and (self.retries > self.maxRetries):
+            if self.noisy:
+                log.msg("Abandoning %s after %d retries." % (connector, self.retries))
+            return
+
+        self.delay = min(self.delay * self.factor, self.maxDelay)
+        if self.jitter:
+            self.delay = random.normalvariate(self.delay, self.delay * self.jitter)
+
+        if self.noisy:
+            log.msg(
+                "%s will retry in %d seconds"
+                % (
+                    connector,
+                    self.delay,
+                )
+            )
+
+        def reconnector():
+            self._callID = None
+            connector.connect()
+
+        if self.clock is None:
+            from twisted.internet import reactor
+
+            self.clock = reactor
+        self._callID = self.clock.callLater(self.delay, reconnector)
+
+    def stopTrying(self):
+        """
+        Put a stop to any attempt to reconnect in progress.
+        """
+        # ??? Is this function really stopFactory?
+        if self._callID:
+            self._callID.cancel()
+            self._callID = None
+        self.continueTrying = 0
+        if self.connector:
+            try:
+                self.connector.stopConnecting()
+            except error.NotConnectingError:
+                pass
+
+    def resetDelay(self):
+        """
+        Call this method after a successful connection: it resets the delay and
+        the retry counter.
+        """
+        self.delay = self.initialDelay
+        self.retries = 0
+        self._callID = None
+        self.continueTrying = 1
+
+    def __getstate__(self):
+        """
+        Remove all of the state which is mutated by connection attempts and
+        failures, returning just the state which describes how reconnections
+        should be attempted.  This will make the unserialized instance
+        behave just as this one did when it was first instantiated.
+        """
+        state = self.__dict__.copy()
+        for key in [
+            "connector",
+            "retries",
+            "delay",
+            "continueTrying",
+            "_callID",
+            "clock",
+        ]:
+            if key in state:
+                del state[key]
+        return state
+
+
+class ServerFactory(Factory):
+    """
+    Subclass this to indicate that your protocol.Factory is only usable for servers.
+    """
+
+
+class BaseProtocol:
+    """
+    This is the abstract superclass of all protocols.
+
+    Some methods have helpful default implementations here so that they can
+    easily be shared, but otherwise the direct subclasses of this class are more
+    interesting, L{Protocol} and L{ProcessProtocol}.
+    """
+
+    connected = 0
+    transport: Optional[ITransport] = None
+
+    def makeConnection(self, transport):
+        """
+        Make a connection to a transport and a server.
+
+        This sets the 'transport' attribute of this Protocol, and calls the
+        connectionMade() callback.
+        """
+        self.connected = 1
+        self.transport = transport
+        self.connectionMade()
+
+    def connectionMade(self):
+        """
+        Called when a connection is made.
+
+        This may be considered the initializer of the protocol, because
+        it is called when the connection is completed.  For clients,
+        this is called once the connection to the server has been
+        established; for servers, this is called after an accept() call
+        stops blocking and a socket has been received.  If you need to
+        send any greeting or initial message, do it here.
+        """
+
+
+connectionDone = failure.Failure(error.ConnectionDone())
+connectionDone.cleanFailure()
+
+
+@implementer(interfaces.IProtocol, interfaces.ILoggingContext)
+class Protocol(BaseProtocol):
+    """
+    This is the base class for streaming connection-oriented protocols.
+
+    If you are going to write a new connection-oriented protocol for Twisted,
+    start here.  Any protocol implementation, either client or server, should
+    be a subclass of this class.
+
+    The API is quite simple.  Implement L{dataReceived} to handle both
+    event-based and synchronous input; output can be sent through the
+    'transport' attribute, which is to be an instance that implements
+    L{twisted.internet.interfaces.ITransport}.  Override C{connectionLost} to be
+    notified when the connection ends.
+
+    Some subclasses exist already to help you write common types of protocols:
+    see the L{twisted.protocols.basic} module for a few of them.
+    """
+
+    factory: Optional[Factory] = None
+
+    def logPrefix(self):
+        """
+        Return a prefix matching the class name, to identify log messages
+        related to this protocol instance.
+        """
+        return self.__class__.__name__
+
+    def dataReceived(self, data: bytes) -> None:
+        """
+        Called whenever data is received.
+
+        Use this method to translate to a higher-level message.  Usually, some
+        callback will be made upon the receipt of each complete protocol
+        message.
+
+        @param data: a string of indeterminate length.  Please keep in mind
+            that you will probably need to buffer some data, as partial
+            (or multiple) protocol messages may be received!  I recommend
+            that unit tests for protocols call through to this method with
+            differing chunk sizes, down to one byte at a time.
+        """
+
+    def connectionLost(self, reason: failure.Failure = connectionDone) -> None:
+        """
+        Called when the connection is shut down.
+
+        Clear any circular references here, and any external references
+        to this Protocol.  The connection has been closed.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+
+@implementer(interfaces.IConsumer)
+class ProtocolToConsumerAdapter(components.Adapter):
+    def write(self, data: bytes) -> None:
+        self.original.dataReceived(data)
+
+    def registerProducer(self, producer, streaming):
+        pass
+
+    def unregisterProducer(self):
+        pass
+
+
+components.registerAdapter(
+    ProtocolToConsumerAdapter, interfaces.IProtocol, interfaces.IConsumer
+)
+
+
+@implementer(interfaces.IProtocol)
+class ConsumerToProtocolAdapter(components.Adapter):
+    def dataReceived(self, data: bytes) -> None:
+        self.original.write(data)
+
+    def connectionLost(self, reason: failure.Failure) -> None:
+        pass
+
+    def makeConnection(self, transport):
+        pass
+
+    def connectionMade(self):
+        pass
+
+
+components.registerAdapter(
+    ConsumerToProtocolAdapter, interfaces.IConsumer, interfaces.IProtocol
+)
+
+
+@implementer(interfaces.IProcessProtocol)
+class ProcessProtocol(BaseProtocol):
+    """
+    Base process protocol implementation which does simple dispatching for
+    stdin, stdout, and stderr file descriptors.
+    """
+
+    transport: Optional[interfaces.IProcessTransport] = None
+
+    def childDataReceived(self, childFD: int, data: bytes) -> None:
+        if childFD == 1:
+            self.outReceived(data)
+        elif childFD == 2:
+            self.errReceived(data)
+
+    def outReceived(self, data: bytes) -> None:
+        """
+        Some data was received from stdout.
+        """
+
+    def errReceived(self, data: bytes) -> None:
+        """
+        Some data was received from stderr.
+        """
+
+    def childConnectionLost(self, childFD: int) -> None:
+        if childFD == 0:
+            self.inConnectionLost()
+        elif childFD == 1:
+            self.outConnectionLost()
+        elif childFD == 2:
+            self.errConnectionLost()
+
+    def inConnectionLost(self):
+        """
+        This will be called when stdin is closed.
+        """
+
+    def outConnectionLost(self):
+        """
+        This will be called when stdout is closed.
+        """
+
+    def errConnectionLost(self):
+        """
+        This will be called when stderr is closed.
+        """
+
+    def processExited(self, reason: failure.Failure) -> None:
+        """
+        This will be called when the subprocess exits.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+    def processEnded(self, reason: failure.Failure) -> None:
+        """
+        Called when the child process exits and all file descriptors
+        associated with it have been closed.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+
+class AbstractDatagramProtocol:
+    """
+    Abstract protocol for datagram-oriented transports, e.g. IP, ICMP, ARP,
+    UDP.
+    """
+
+    transport = None
+    numPorts = 0
+    noisy = True
+
+    def __getstate__(self):
+        d = self.__dict__.copy()
+        d["transport"] = None
+        return d
+
+    def doStart(self):
+        """
+        Make sure startProtocol is called.
+
+        This will be called by makeConnection(), users should not call it.
+        """
+        if not self.numPorts:
+            if self.noisy:
+                log.msg("Starting protocol %s" % self)
+            self.startProtocol()
+        self.numPorts = self.numPorts + 1
+
+    def doStop(self):
+        """
+        Make sure stopProtocol is called.
+
+        This will be called by the port, users should not call it.
+        """
+        assert self.numPorts > 0
+        self.numPorts = self.numPorts - 1
+        self.transport = None
+        if not self.numPorts:
+            if self.noisy:
+                log.msg("Stopping protocol %s" % self)
+            self.stopProtocol()
+
+    def startProtocol(self):
+        """
+        Called when a transport is connected to this protocol.
+
+        Will only be called once, even if multiple ports are connected.
+        """
+
+    def stopProtocol(self):
+        """
+        Called when the transport is disconnected.
+
+        Will only be called once, after all ports are disconnected.
+        """
+
+    def makeConnection(self, transport):
+        """
+        Make a connection to a transport and a server.
+
+        This sets the 'transport' attribute of this DatagramProtocol, and calls the
+        doStart() callback.
+        """
+        assert self.transport == None
+        self.transport = transport
+        self.doStart()
+
+    def datagramReceived(self, datagram: bytes, addr: Any) -> None:
+        """
+        Called when a datagram is received.
+
+        @param datagram: the bytes received from the transport.
+        @param addr: tuple of source of datagram.
+        """
+
+
+@implementer(interfaces.ILoggingContext)
+class DatagramProtocol(AbstractDatagramProtocol):
+    """
+    Protocol for datagram-oriented transport, e.g. UDP.
+
+    @type transport: L{None} or
+        L{IUDPTransport<twisted.internet.interfaces.IUDPTransport>} provider
+    @ivar transport: The transport with which this protocol is associated,
+        if it is associated with one.
+    """
+
+    def logPrefix(self):
+        """
+        Return a prefix matching the class name, to identify log messages
+        related to this protocol instance.
+        """
+        return self.__class__.__name__
+
+    def connectionRefused(self):
+        """
+        Called due to error from write in connected mode.
+
+        Note this is a result of ICMP message generated by *previous*
+        write.
+        """
+
+
+class ConnectedDatagramProtocol(DatagramProtocol):
+    """
+    Protocol for connected datagram-oriented transport.
+
+    No longer necessary for UDP.
+    """
+
+    def datagramReceived(self, datagram):
+        """
+        Called when a datagram is received.
+
+        @param datagram: the string received from the transport.
+        """
+
+    def connectionFailed(self, failure: failure.Failure) -> None:
+        """
+        Called if connecting failed.
+
+        Usually this will be due to a DNS lookup failure.
+        """
+
+
+@implementer(interfaces.ITransport)
+class FileWrapper:
+    """
+    A wrapper around a file-like object to make it behave as a Transport.
+
+    This doesn't actually stream the file to the attached protocol,
+    and is thus useful mainly as a utility for debugging protocols.
+    """
+
+    closed = 0
+    disconnecting = 0
+    producer = None
+    streamingProducer = 0
+
+    def __init__(self, file):
+        self.file = file
+
+    def write(self, data: bytes) -> None:
+        try:
+            self.file.write(data)
+        except BaseException:
+            self.handleException()
+
+    def _checkProducer(self):
+        # Cheating; this is called at "idle" times to allow producers to be
+        # found and dealt with
+        if self.producer:
+            self.producer.resumeProducing()
+
+    def registerProducer(self, producer, streaming):
+        """
+        From abstract.FileDescriptor
+        """
+        self.producer = producer
+        self.streamingProducer = streaming
+        if not streaming:
+            producer.resumeProducing()
+
+    def unregisterProducer(self):
+        self.producer = None
+
+    def stopConsuming(self):
+        self.unregisterProducer()
+        self.loseConnection()
+
+    def writeSequence(self, iovec):
+        self.write(b"".join(iovec))
+
+    def loseConnection(self):
+        self.closed = 1
+        try:
+            self.file.close()
+        except OSError:
+            self.handleException()
+
+    def getPeer(self):
+        # FIXME: https://twistedmatrix.com/trac/ticket/7820
+        # According to ITransport, this should return an IAddress!
+        return "file", "file"
+
+    def getHost(self):
+        # FIXME: https://twistedmatrix.com/trac/ticket/7820
+        # According to ITransport, this should return an IAddress!
+        return "file"
+
+    def handleException(self):
+        pass
+
+    def resumeProducing(self):
+        # Never sends data anyways
+        pass
+
+    def pauseProducing(self):
+        # Never sends data anyways
+        pass
+
+    def stopProducing(self):
+        self.loseConnection()
+
+
+__all__ = [
+    "Factory",
+    "ClientFactory",
+    "ReconnectingClientFactory",
+    "connectionDone",
+    "Protocol",
+    "ProcessProtocol",
+    "FileWrapper",
+    "ServerFactory",
+    "AbstractDatagramProtocol",
+    "DatagramProtocol",
+    "ConnectedDatagramProtocol",
+    "ClientCreator",
+]
diff --git a/contrib/python/Twisted/py3/twisted/internet/pyuisupport.py b/contrib/python/Twisted/py3/twisted/internet/pyuisupport.py
new file mode 100644
index 00000000000..bfbffb9cb40
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/pyuisupport.py
@@ -0,0 +1,39 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+This module integrates PyUI with twisted.internet's mainloop.
+
+Maintainer: Jp Calderone
+
+See doc/examples/pyuidemo.py for example usage.
+"""
+
+# System imports
+import pyui  # type: ignore[import]
+
+
+def _guiUpdate(reactor, delay):
+    pyui.draw()
+    if pyui.update() == 0:
+        pyui.quit()
+        reactor.stop()
+    else:
+        reactor.callLater(delay, _guiUpdate, reactor, delay)
+
+
+def install(ms=10, reactor=None, args=(), kw={}):
+    """
+    Schedule PyUI's display to be updated approximately every C{ms}
+    milliseconds, and initialize PyUI with the specified arguments.
+    """
+    d = pyui.init(*args, **kw)
+
+    if reactor is None:
+        from twisted.internet import reactor
+    _guiUpdate(reactor, ms / 1000.0)
+    return d
+
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/reactor.py b/contrib/python/Twisted/py3/twisted/internet/reactor.py
new file mode 100644
index 00000000000..00f1ef6e012
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/reactor.py
@@ -0,0 +1,40 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+The reactor is the Twisted event loop within Twisted, the loop which drives
+applications using Twisted. The reactor provides APIs for networking,
+threading, dispatching events, and more.
+
+The default reactor depends on the platform and will be installed if this
+module is imported without another reactor being explicitly installed
+beforehand. Regardless of which reactor is installed, importing this module is
+the correct way to get a reference to it.
+
+New application code should prefer to pass and accept the reactor as a
+parameter where it is needed, rather than relying on being able to import this
+module to get a reference.  This simplifies unit testing and may make it easier
+to one day support multiple reactors (as a performance enhancement), though
+this is not currently possible.
+
+@see: L{IReactorCore<twisted.internet.interfaces.IReactorCore>}
+@see: L{IReactorTime<twisted.internet.interfaces.IReactorTime>}
+@see: L{IReactorProcess<twisted.internet.interfaces.IReactorProcess>}
+@see: L{IReactorTCP<twisted.internet.interfaces.IReactorTCP>}
+@see: L{IReactorSSL<twisted.internet.interfaces.IReactorSSL>}
+@see: L{IReactorUDP<twisted.internet.interfaces.IReactorUDP>}
+@see: L{IReactorMulticast<twisted.internet.interfaces.IReactorMulticast>}
+@see: L{IReactorUNIX<twisted.internet.interfaces.IReactorUNIX>}
+@see: L{IReactorUNIXDatagram<twisted.internet.interfaces.IReactorUNIXDatagram>}
+@see: L{IReactorFDSet<twisted.internet.interfaces.IReactorFDSet>}
+@see: L{IReactorThreads<twisted.internet.interfaces.IReactorThreads>}
+@see: L{IReactorPluggableResolver<twisted.internet.interfaces.IReactorPluggableResolver>}
+"""
+
+
+import sys
+
+del sys.modules["twisted.internet.reactor"]
+from twisted.internet import default
+
+default.install()
diff --git a/contrib/python/Twisted/py3/twisted/internet/selectreactor.py b/contrib/python/Twisted/py3/twisted/internet/selectreactor.py
new file mode 100644
index 00000000000..199dc406711
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/selectreactor.py
@@ -0,0 +1,197 @@
+# -*- test-case-name: twisted.test.test_internet -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Select reactor
+"""
+
+
+import select
+import sys
+from errno import EBADF, EINTR
+from time import sleep
+from typing import Type
+
+from zope.interface import implementer
+
+from twisted.internet import posixbase
+from twisted.internet.interfaces import IReactorFDSet
+from twisted.python import log
+from twisted.python.runtime import platformType
+
+
+def win32select(r, w, e, timeout=None):
+    """Win32 select wrapper."""
+    if not (r or w):
+        # windows select() exits immediately when no sockets
+        if timeout is None:
+            timeout = 0.01
+        else:
+            timeout = min(timeout, 0.001)
+        sleep(timeout)
+        return [], [], []
+    # windows doesn't process 'signals' inside select(), so we set a max
+    # time or ctrl-c will never be recognized
+    if timeout is None or timeout > 0.5:
+        timeout = 0.5
+    r, w, e = select.select(r, w, w, timeout)
+    return r, w + e, []
+
+
+if platformType == "win32":
+    _select = win32select
+else:
+    _select = select.select
+
+
+try:
+    from twisted.internet.win32eventreactor import _ThreadedWin32EventsMixin
+except ImportError:
+    _extraBase: Type[object] = object
+else:
+    _extraBase = _ThreadedWin32EventsMixin
+
+
+@implementer(IReactorFDSet)
+class SelectReactor(posixbase.PosixReactorBase, _extraBase):  # type: ignore[misc,valid-type]
+    """
+    A select() based reactor - runs on all POSIX platforms and on Win32.
+
+    @ivar _reads: A set containing L{FileDescriptor} instances which will be
+        checked for read events.
+
+    @ivar _writes: A set containing L{FileDescriptor} instances which will be
+        checked for writability.
+    """
+
+    def __init__(self):
+        """
+        Initialize file descriptor tracking dictionaries and the base class.
+        """
+        self._reads = set()
+        self._writes = set()
+        posixbase.PosixReactorBase.__init__(self)
+
+    def _preenDescriptors(self):
+        log.msg("Malformed file descriptor found.  Preening lists.")
+        readers = list(self._reads)
+        writers = list(self._writes)
+        self._reads.clear()
+        self._writes.clear()
+        for selSet, selList in ((self._reads, readers), (self._writes, writers)):
+            for selectable in selList:
+                try:
+                    select.select([selectable], [selectable], [selectable], 0)
+                except Exception as e:
+                    log.msg("bad descriptor %s" % selectable)
+                    self._disconnectSelectable(selectable, e, False)
+                else:
+                    selSet.add(selectable)
+
+    def doSelect(self, timeout):
+        """
+        Run one iteration of the I/O monitor loop.
+
+        This will run all selectables who had input or output readiness
+        waiting for them.
+        """
+        try:
+            r, w, ignored = _select(self._reads, self._writes, [], timeout)
+        except ValueError:
+            # Possibly a file descriptor has gone negative?
+            self._preenDescriptors()
+            return
+        except TypeError:
+            # Something *totally* invalid (object w/o fileno, non-integral
+            # result) was passed
+            log.err()
+            self._preenDescriptors()
+            return
+        except OSError as se:
+            # select(2) encountered an error, perhaps while calling the fileno()
+            # method of a socket.  (Python 2.6 socket.error is an IOError
+            # subclass, but on Python 2.5 and earlier it is not.)
+            if se.args[0] in (0, 2):
+                # windows does this if it got an empty list
+                if (not self._reads) and (not self._writes):
+                    return
+                else:
+                    raise
+            elif se.args[0] == EINTR:
+                return
+            elif se.args[0] == EBADF:
+                self._preenDescriptors()
+                return
+            else:
+                # OK, I really don't know what's going on.  Blow up.
+                raise
+
+        _drdw = self._doReadOrWrite
+        _logrun = log.callWithLogger
+        for selectables, method, fdset in (
+            (r, "doRead", self._reads),
+            (w, "doWrite", self._writes),
+        ):
+            for selectable in selectables:
+                # if this was disconnected in another thread, kill it.
+                # ^^^^ --- what the !@#*?  serious!  -exarkun
+                if selectable not in fdset:
+                    continue
+                # This for pausing input when we're not ready for more.
+                _logrun(selectable, _drdw, selectable, method)
+
+    doIteration = doSelect
+
+    def _doReadOrWrite(self, selectable, method):
+        try:
+            why = getattr(selectable, method)()
+        except BaseException:
+            why = sys.exc_info()[1]
+            log.err()
+        if why:
+            self._disconnectSelectable(selectable, why, method == "doRead")
+
+    def addReader(self, reader):
+        """
+        Add a FileDescriptor for notification of data available to read.
+        """
+        self._reads.add(reader)
+
+    def addWriter(self, writer):
+        """
+        Add a FileDescriptor for notification of data available to write.
+        """
+        self._writes.add(writer)
+
+    def removeReader(self, reader):
+        """
+        Remove a Selectable for notification of data available to read.
+        """
+        self._reads.discard(reader)
+
+    def removeWriter(self, writer):
+        """
+        Remove a Selectable for notification of data available to write.
+        """
+        self._writes.discard(writer)
+
+    def removeAll(self):
+        return self._removeAll(self._reads, self._writes)
+
+    def getReaders(self):
+        return list(self._reads)
+
+    def getWriters(self):
+        return list(self._writes)
+
+
+def install():
+    """Configure the twisted mainloop to be run using the select() reactor."""
+    reactor = SelectReactor()
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
+
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/serialport.py b/contrib/python/Twisted/py3/twisted/internet/serialport.py
new file mode 100644
index 00000000000..d63d4ce4351
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/serialport.py
@@ -0,0 +1,100 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Serial Port Protocol
+"""
+
+
+# http://twistedmatrix.com/trac/ticket/3725#comment:24
+# Apparently applications use these names even though they should
+# be imported from pyserial
+__all__ = [
+    "serial",
+    "PARITY_ODD",
+    "PARITY_EVEN",
+    "PARITY_NONE",
+    "STOPBITS_TWO",
+    "STOPBITS_ONE",
+    "FIVEBITS",
+    "EIGHTBITS",
+    "SEVENBITS",
+    "SIXBITS",
+    # Name this module is actually trying to export
+    "SerialPort",
+]
+
+# all of them require pyserial at the moment, so check that first
+import serial  # type: ignore[import]
+from serial import (
+    EIGHTBITS,
+    FIVEBITS,
+    PARITY_EVEN,
+    PARITY_NONE,
+    PARITY_ODD,
+    SEVENBITS,
+    SIXBITS,
+    STOPBITS_ONE,
+    STOPBITS_TWO,
+)
+
+from twisted.python.runtime import platform
+
+
+class BaseSerialPort:
+    """
+    Base class for Windows and POSIX serial ports.
+
+    @ivar _serialFactory: a pyserial C{serial.Serial} factory, used to create
+        the instance stored in C{self._serial}. Overrideable to enable easier
+        testing.
+
+    @ivar _serial: a pyserial C{serial.Serial} instance used to manage the
+        options on the serial port.
+    """
+
+    _serialFactory = serial.Serial
+
+    def setBaudRate(self, baudrate):
+        if hasattr(self._serial, "setBaudrate"):
+            self._serial.setBaudrate(baudrate)
+        else:
+            self._serial.setBaudRate(baudrate)
+
+    def inWaiting(self):
+        return self._serial.inWaiting()
+
+    def flushInput(self):
+        self._serial.flushInput()
+
+    def flushOutput(self):
+        self._serial.flushOutput()
+
+    def sendBreak(self):
+        self._serial.sendBreak()
+
+    def getDSR(self):
+        return self._serial.getDSR()
+
+    def getCD(self):
+        return self._serial.getCD()
+
+    def getRI(self):
+        return self._serial.getRI()
+
+    def getCTS(self):
+        return self._serial.getCTS()
+
+    def setDTR(self, on=1):
+        self._serial.setDTR(on)
+
+    def setRTS(self, on=1):
+        self._serial.setRTS(on)
+
+
+# Expert appropriate implementation of SerialPort.
+if platform.isWindows():
+    from twisted.internet._win32serialport import SerialPort
+else:
+    from twisted.internet._posixserialport import SerialPort  # type: ignore[assignment]
diff --git a/contrib/python/Twisted/py3/twisted/internet/ssl.py b/contrib/python/Twisted/py3/twisted/internet/ssl.py
new file mode 100644
index 00000000000..1ad02b3c097
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/ssl.py
@@ -0,0 +1,278 @@
+# -*- test-case-name: twisted.test.test_ssl -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module implements Transport Layer Security (TLS) support for Twisted.  It
+requires U{PyOpenSSL <https://pypi.python.org/pypi/pyOpenSSL>}.
+
+If you wish to establish a TLS connection, please use one of the following
+APIs:
+
+    - SSL endpoints for L{servers
+      <twisted.internet.endpoints.SSL4ServerEndpoint>} and L{clients
+      <twisted.internet.endpoints.SSL4ClientEndpoint>}
+
+    - L{startTLS <twisted.internet.interfaces.ITLSTransport.startTLS>}
+
+    - L{connectSSL <twisted.internet.interfaces.IReactorSSL.connectSSL>}
+
+    - L{listenSSL <twisted.internet.interfaces.IReactorSSL.listenSSL>}
+
+These APIs all require a C{contextFactory} argument that specifies their
+security properties, such as certificate, private key, certificate authorities
+to verify the peer, allowed TLS protocol versions, cipher suites, and so on.
+The recommended value for this argument is a L{CertificateOptions} instance;
+see its documentation for an explanation of the available options.
+
+The C{contextFactory} name is a bit of an anachronism now, as context factories
+have been replaced with "connection creators", but these objects serve the same
+role.
+
+Be warned that implementing your own connection creator (i.e.: value for the
+C{contextFactory}) is both difficult and dangerous; the Twisted team has worked
+hard to make L{CertificateOptions}' API comprehensible and unsurprising, and
+the Twisted team is actively maintaining it to ensure that it becomes more
+secure over time.
+
+If you are really absolutely sure that you want to take on the risk of
+implementing your own connection creator based on the pyOpenSSL API, see the
+L{server connection creator
+<twisted.internet.interfaces.IOpenSSLServerConnectionCreator>} and L{client
+connection creator
+<twisted.internet.interfaces.IOpenSSLServerConnectionCreator>} interfaces.
+
+Developers using Twisted, please ignore the L{Port}, L{Connector}, and
+L{Client} classes defined here, as these are details of certain reactors' TLS
+implementations, exposed by accident (and remaining here only for compatibility
+reasons).  If you wish to establish a TLS connection, please use one of the
+APIs listed above.
+
+@note: "SSL" (Secure Sockets Layer) is an antiquated synonym for "TLS"
+    (Transport Layer Security).  You may see these terms used interchangeably
+    throughout the documentation.
+"""
+
+
+from zope.interface import implementedBy, implementer, implementer_only
+
+# System imports
+from OpenSSL import SSL
+
+# Twisted imports
+from twisted.internet import interfaces, tcp
+
+supported = True
+
+
+@implementer(interfaces.IOpenSSLContextFactory)
+class ContextFactory:
+    """A factory for SSL context objects, for server SSL connections."""
+
+    isClient = 0
+
+    def getContext(self):
+        """Return a SSL.Context object. override in subclasses."""
+        raise NotImplementedError
+
+
+class DefaultOpenSSLContextFactory(ContextFactory):
+    """
+    L{DefaultOpenSSLContextFactory} is a factory for server-side SSL context
+    objects.  These objects define certain parameters related to SSL
+    handshakes and the subsequent connection.
+
+    @ivar _contextFactory: A callable which will be used to create new
+        context objects.  This is typically L{OpenSSL.SSL.Context}.
+    """
+
+    _context = None
+
+    def __init__(
+        self,
+        privateKeyFileName,
+        certificateFileName,
+        sslmethod=SSL.TLS_METHOD,
+        _contextFactory=SSL.Context,
+    ):
+        """
+        @param privateKeyFileName: Name of a file containing a private key
+        @param certificateFileName: Name of a file containing a certificate
+        @param sslmethod: The SSL method to use
+        """
+        self.privateKeyFileName = privateKeyFileName
+        self.certificateFileName = certificateFileName
+        self.sslmethod = sslmethod
+        self._contextFactory = _contextFactory
+
+        # Create a context object right now.  This is to force validation of
+        # the given parameters so that errors are detected earlier rather
+        # than later.
+        self.cacheContext()
+
+    def cacheContext(self):
+        if self._context is None:
+            ctx = self._contextFactory(self.sslmethod)
+            # Disallow SSLv2!  It's insecure!  SSLv3 has been around since
+            # 1996.  It's time to move on.
+            ctx.set_options(SSL.OP_NO_SSLv2)
+            ctx.use_certificate_file(self.certificateFileName)
+            ctx.use_privatekey_file(self.privateKeyFileName)
+            self._context = ctx
+
+    def __getstate__(self):
+        d = self.__dict__.copy()
+        del d["_context"]
+        return d
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+
+    def getContext(self):
+        """
+        Return an SSL context.
+        """
+        return self._context
+
+
+@implementer(interfaces.IOpenSSLContextFactory)
+class ClientContextFactory:
+    """A context factory for SSL clients."""
+
+    isClient = 1
+
+    # TLS_METHOD allows negotiation of multiple TLS versions.
+    method = SSL.TLS_METHOD
+
+    _contextFactory = SSL.Context
+
+    def getContext(self):
+        ctx = self._contextFactory(self.method)
+        ctx.set_options(
+            SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3 | SSL.OP_NO_TLSv1 | SSL.OP_NO_TLSv1_1
+        )
+        return ctx
+
+
+@implementer_only(
+    interfaces.ISSLTransport,
+    *(i for i in implementedBy(tcp.Client) if i != interfaces.ITLSTransport),
+)
+class Client(tcp.Client):
+    """
+    I am an SSL client.
+    """
+
+    def __init__(self, host, port, bindAddress, ctxFactory, connector, reactor=None):
+        # tcp.Client.__init__ depends on self.ctxFactory being set
+        self.ctxFactory = ctxFactory
+        tcp.Client.__init__(self, host, port, bindAddress, connector, reactor)
+
+    def _connectDone(self):
+        self.startTLS(self.ctxFactory)
+        self.startWriting()
+        tcp.Client._connectDone(self)
+
+
+@implementer(interfaces.ISSLTransport)
+class Server(tcp.Server):
+    """
+    I am an SSL server.
+    """
+
+    def __init__(self, *args, **kwargs):
+        tcp.Server.__init__(self, *args, **kwargs)
+        self.startTLS(self.server.ctxFactory)
+
+    def getPeerCertificate(self):
+        # ISSLTransport.getPeerCertificate
+        raise NotImplementedError("Server.getPeerCertificate")
+
+
+class Port(tcp.Port):
+    """
+    I am an SSL port.
+    """
+
+    transport = Server
+
+    _type = "TLS"
+
+    def __init__(
+        self, port, factory, ctxFactory, backlog=50, interface="", reactor=None
+    ):
+        tcp.Port.__init__(self, port, factory, backlog, interface, reactor)
+        self.ctxFactory = ctxFactory
+
+    def _getLogPrefix(self, factory):
+        """
+        Override the normal prefix to include an annotation indicating this is a
+        port for TLS connections.
+        """
+        return tcp.Port._getLogPrefix(self, factory) + " (TLS)"
+
+
+class Connector(tcp.Connector):
+    def __init__(
+        self, host, port, factory, contextFactory, timeout, bindAddress, reactor=None
+    ):
+        self.contextFactory = contextFactory
+        tcp.Connector.__init__(self, host, port, factory, timeout, bindAddress, reactor)
+
+        # Force some parameter checking in pyOpenSSL.  It's better to fail now
+        # than after we've set up the transport.
+        contextFactory.getContext()
+
+    def _makeTransport(self):
+        return Client(
+            self.host,
+            self.port,
+            self.bindAddress,
+            self.contextFactory,
+            self,
+            self.reactor,
+        )
+
+
+from twisted.internet._sslverify import (
+    DN,
+    Certificate,
+    CertificateRequest,
+    DistinguishedName,
+    KeyPair,
+    OpenSSLAcceptableCiphers as AcceptableCiphers,
+    OpenSSLCertificateOptions as CertificateOptions,
+    OpenSSLDefaultPaths,
+    OpenSSLDiffieHellmanParameters as DiffieHellmanParameters,
+    PrivateCertificate,
+    ProtocolNegotiationSupport,
+    TLSVersion,
+    VerificationError,
+    optionsForClientTLS,
+    platformTrust,
+    protocolNegotiationMechanisms,
+    trustRootFromCertificates,
+)
+
+__all__ = [
+    "ContextFactory",
+    "DefaultOpenSSLContextFactory",
+    "ClientContextFactory",
+    "DistinguishedName",
+    "DN",
+    "Certificate",
+    "CertificateRequest",
+    "PrivateCertificate",
+    "KeyPair",
+    "AcceptableCiphers",
+    "CertificateOptions",
+    "DiffieHellmanParameters",
+    "platformTrust",
+    "OpenSSLDefaultPaths",
+    "TLSVersion",
+    "VerificationError",
+    "optionsForClientTLS",
+    "ProtocolNegotiationSupport",
+    "protocolNegotiationMechanisms",
+    "trustRootFromCertificates",
+]
diff --git a/contrib/python/Twisted/py3/twisted/internet/stdio.py b/contrib/python/Twisted/py3/twisted/internet/stdio.py
new file mode 100644
index 00000000000..3196898bf6c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/stdio.py
@@ -0,0 +1,37 @@
+# -*- test-case-name: twisted.test.test_stdio -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Standard input/out/err support.
+
+This module exposes one name, StandardIO, which is a factory that takes an
+IProtocol provider as an argument.  It connects that protocol to standard input
+and output on the current process.
+
+It should work on any UNIX and also on Win32 (with some caveats: due to
+platform limitations, it will perform very poorly on Win32).
+
+Future Plans::
+
+    support for stderr, perhaps
+    Rewrite to use the reactor instead of an ad-hoc mechanism for connecting
+        protocols to transport.
+
+
+Maintainer: James Y Knight
+"""
+
+
+from twisted.python.runtime import platform
+
+if platform.isWindows():
+    from twisted.internet._win32stdio import StandardIO, Win32PipeAddress as PipeAddress
+
+else:
+    from twisted.internet._posixstdio import (  # type: ignore[assignment]
+        PipeAddress,
+        StandardIO,
+    )
+
+__all__ = ["StandardIO", "PipeAddress"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/task.py b/contrib/python/Twisted/py3/twisted/internet/task.py
new file mode 100644
index 00000000000..0319d24a7a4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/task.py
@@ -0,0 +1,976 @@
+# -*- test-case-name: twisted.test.test_task,twisted.test.test_cooperator -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Scheduling utility methods and classes.
+"""
+
+
+import sys
+import time
+import warnings
+from typing import (
+    Callable,
+    Coroutine,
+    Iterable,
+    Iterator,
+    List,
+    NoReturn,
+    Optional,
+    Sequence,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from zope.interface import implementer
+
+from incremental import Version
+
+from twisted.internet.base import DelayedCall
+from twisted.internet.defer import Deferred, ensureDeferred, maybeDeferred
+from twisted.internet.error import ReactorNotRunning
+from twisted.internet.interfaces import IDelayedCall, IReactorCore, IReactorTime
+from twisted.python import log, reflect
+from twisted.python.deprecate import _getDeprecationWarningString
+from twisted.python.failure import Failure
+
+_T = TypeVar("_T")
+
+
+class LoopingCall:
+    """Call a function repeatedly.
+
+    If C{f} returns a deferred, rescheduling will not take place until the
+    deferred has fired. The result value is ignored.
+
+    @ivar f: The function to call.
+    @ivar a: A tuple of arguments to pass the function.
+    @ivar kw: A dictionary of keyword arguments to pass to the function.
+    @ivar clock: A provider of
+        L{twisted.internet.interfaces.IReactorTime}.  The default is
+        L{twisted.internet.reactor}. Feel free to set this to
+        something else, but it probably ought to be set *before*
+        calling L{start}.
+
+    @ivar running: A flag which is C{True} while C{f} is scheduled to be called
+        (or is currently being called). It is set to C{True} when L{start} is
+        called and set to C{False} when L{stop} is called or if C{f} raises an
+        exception. In either case, it will be C{False} by the time the
+        C{Deferred} returned by L{start} fires its callback or errback.
+
+    @ivar _realLastTime: When counting skips, the time at which the skip
+        counter was last invoked.
+
+    @ivar _runAtStart: A flag indicating whether the 'now' argument was passed
+        to L{LoopingCall.start}.
+    """
+
+    call: Optional[IDelayedCall] = None
+    running = False
+    _deferred: Optional[Deferred["LoopingCall"]] = None
+    interval: Optional[float] = None
+    _runAtStart = False
+    starttime: Optional[float] = None
+    _realLastTime: Optional[float] = None
+
+    def __init__(self, f: Callable[..., object], *a: object, **kw: object) -> None:
+        self.f = f
+        self.a = a
+        self.kw = kw
+        from twisted.internet import reactor
+
+        self.clock = cast(IReactorTime, reactor)
+
+    @property
+    def deferred(self) -> Optional[Deferred["LoopingCall"]]:
+        """
+        DEPRECATED. L{Deferred} fired when loop stops or fails.
+
+        Use the L{Deferred} returned by L{LoopingCall.start}.
+        """
+        warningString = _getDeprecationWarningString(
+            "twisted.internet.task.LoopingCall.deferred",
+            Version("Twisted", 16, 0, 0),
+            replacement="the deferred returned by start()",
+        )
+        warnings.warn(warningString, DeprecationWarning, stacklevel=2)
+
+        return self._deferred
+
+    @classmethod
+    def withCount(cls, countCallable: Callable[[int], object]) -> "LoopingCall":
+        """
+        An alternate constructor for L{LoopingCall} that makes available the
+        number of calls which should have occurred since it was last invoked.
+
+        Note that this number is an C{int} value; It represents the discrete
+        number of calls that should have been made.  For example, if you are
+        using a looping call to display an animation with discrete frames, this
+        number would be the number of frames to advance.
+
+        The count is normally 1, but can be higher. For example, if the reactor
+        is blocked and takes too long to invoke the L{LoopingCall}, a Deferred
+        returned from a previous call is not fired before an interval has
+        elapsed, or if the callable itself blocks for longer than an interval,
+        preventing I{itself} from being called.
+
+        When running with an interval of 0, count will be always 1.
+
+        @param countCallable: A callable that will be invoked each time the
+            resulting LoopingCall is run, with an integer specifying the number
+            of calls that should have been invoked.
+
+        @return: An instance of L{LoopingCall} with call counting enabled,
+            which provides the count as the first positional argument.
+
+        @since: 9.0
+        """
+
+        def counter() -> object:
+            now = self.clock.seconds()
+
+            if self.interval == 0:
+                self._realLastTime = now
+                return countCallable(1)
+
+            lastTime = self._realLastTime
+            if lastTime is None:
+                assert (
+                    self.starttime is not None
+                ), "LoopingCall called before it was started"
+                lastTime = self.starttime
+                if self._runAtStart:
+                    assert (
+                        self.interval is not None
+                    ), "Looping call called with None interval"
+                    lastTime -= self.interval
+            lastInterval = self._intervalOf(lastTime)
+            thisInterval = self._intervalOf(now)
+            count = thisInterval - lastInterval
+            if count > 0:
+                self._realLastTime = now
+                return countCallable(count)
+
+            return None
+
+        self = cls(counter)
+
+        return self
+
+    def _intervalOf(self, t: float) -> int:
+        """
+        Determine the number of intervals passed as of the given point in
+        time.
+
+        @param t: The specified time (from the start of the L{LoopingCall}) to
+            be measured in intervals
+
+        @return: The C{int} number of intervals which have passed as of the
+            given point in time.
+        """
+        assert self.starttime is not None
+        assert self.interval is not None
+        elapsedTime = t - self.starttime
+        intervalNum = int(elapsedTime / self.interval)
+        return intervalNum
+
+    def start(self, interval: float, now: bool = True) -> Deferred["LoopingCall"]:
+        """
+        Start running function every interval seconds.
+
+        @param interval: The number of seconds between calls.  May be
+        less than one.  Precision will depend on the underlying
+        platform, the available hardware, and the load on the system.
+
+        @param now: If True, run this call right now.  Otherwise, wait
+        until the interval has elapsed before beginning.
+
+        @return: A Deferred whose callback will be invoked with
+        C{self} when C{self.stop} is called, or whose errback will be
+        invoked when the function raises an exception or returned a
+        deferred that has its errback invoked.
+        """
+        assert not self.running, "Tried to start an already running " "LoopingCall."
+        if interval < 0:
+            raise ValueError("interval must be >= 0")
+        self.running = True
+        # Loop might fail to start and then self._deferred will be cleared.
+        # This why the local C{deferred} variable is used.
+        deferred = self._deferred = Deferred()
+        self.starttime = self.clock.seconds()
+        self.interval = interval
+        self._runAtStart = now
+        if now:
+            self()
+        else:
+            self._scheduleFrom(self.starttime)
+        return deferred
+
+    def stop(self) -> None:
+        """Stop running function."""
+        assert self.running, "Tried to stop a LoopingCall that was " "not running."
+        self.running = False
+        if self.call is not None:
+            self.call.cancel()
+            self.call = None
+            d, self._deferred = self._deferred, None
+            assert d is not None
+            d.callback(self)
+
+    def reset(self) -> None:
+        """
+        Skip the next iteration and reset the timer.
+
+        @since: 11.1
+        """
+        assert self.running, "Tried to reset a LoopingCall that was " "not running."
+        if self.call is not None:
+            self.call.cancel()
+            self.call = None
+            self.starttime = self.clock.seconds()
+            self._scheduleFrom(self.starttime)
+
+    def __call__(self) -> None:
+        def cb(result: object) -> None:
+            if self.running:
+                self._scheduleFrom(self.clock.seconds())
+            else:
+                d, self._deferred = self._deferred, None
+                assert d is not None
+                d.callback(self)
+
+        def eb(failure: Failure) -> None:
+            self.running = False
+            d, self._deferred = self._deferred, None
+            assert d is not None
+            d.errback(failure)
+
+        self.call = None
+        d = maybeDeferred(self.f, *self.a, **self.kw)
+        d.addCallback(cb)
+        d.addErrback(eb)
+
+    def _scheduleFrom(self, when: float) -> None:
+        """
+        Schedule the next iteration of this looping call.
+
+        @param when: The present time from whence the call is scheduled.
+        """
+
+        def howLong() -> float:
+            # How long should it take until the next invocation of our
+            # callable?  Split out into a function because there are multiple
+            # places we want to 'return' out of this.
+            if self.interval == 0:
+                # If the interval is 0, just go as fast as possible, always
+                # return zero, call ourselves ASAP.
+                return 0
+            # Compute the time until the next interval; how long has this call
+            # been running for?
+            assert self.starttime is not None
+            runningFor = when - self.starttime
+            # And based on that start time, when does the current interval end?
+            assert self.interval is not None
+            untilNextInterval = self.interval - (runningFor % self.interval)
+            # Now that we know how long it would be, we have to tell if the
+            # number is effectively zero.  However, we can't just test against
+            # zero.  If a number with a small exponent is added to a number
+            # with a large exponent, it may be so small that the digits just
+            # fall off the end, which means that adding the increment makes no
+            # difference; it's time to tick over into the next interval.
+            if when == when + untilNextInterval:
+                # If it's effectively zero, then we need to add another
+                # interval.
+                return self.interval
+            # Finally, if everything else is normal, we just return the
+            # computed delay.
+            return untilNextInterval
+
+        self.call = self.clock.callLater(howLong(), self)
+
+    def __repr__(self) -> str:
+        # This code should be replaced by a utility function in reflect;
+        # see ticket #6066:
+        func = getattr(self.f, "__qualname__", None)
+        if func is None:
+            func = getattr(self.f, "__name__", None)
+            if func is not None:
+                imClass = getattr(self.f, "im_class", None)
+                if imClass is not None:
+                    func = f"{imClass}.{func}"
+        if func is None:
+            func = reflect.safe_repr(self.f)
+
+        return "LoopingCall<{!r}>({}, *{}, **{})".format(
+            self.interval,
+            func,
+            reflect.safe_repr(self.a),
+            reflect.safe_repr(self.kw),
+        )
+
+
+class SchedulerError(Exception):
+    """
+    The operation could not be completed because the scheduler or one of its
+    tasks was in an invalid state.  This exception should not be raised
+    directly, but is a superclass of various scheduler-state-related
+    exceptions.
+    """
+
+
+class SchedulerStopped(SchedulerError):
+    """
+    The operation could not complete because the scheduler was stopped in
+    progress or was already stopped.
+    """
+
+
+class TaskFinished(SchedulerError):
+    """
+    The operation could not complete because the task was already completed,
+    stopped, encountered an error or otherwise permanently stopped running.
+    """
+
+
+class TaskDone(TaskFinished):
+    """
+    The operation could not complete because the task was already completed.
+    """
+
+
+class TaskStopped(TaskFinished):
+    """
+    The operation could not complete because the task was stopped.
+    """
+
+
+class TaskFailed(TaskFinished):
+    """
+    The operation could not complete because the task died with an unhandled
+    error.
+    """
+
+
+class NotPaused(SchedulerError):
+    """
+    This exception is raised when a task is resumed which was not previously
+    paused.
+    """
+
+
+class _Timer:
+    MAX_SLICE = 0.01
+
+    def __init__(self) -> None:
+        self.end = time.time() + self.MAX_SLICE
+
+    def __call__(self) -> bool:
+        return time.time() >= self.end
+
+
+_EPSILON = 0.00000001
+
+
+def _defaultScheduler(callable: Callable[[], None]) -> IDelayedCall:
+    from twisted.internet import reactor
+
+    return cast(IReactorTime, reactor).callLater(_EPSILON, callable)
+
+
+_TaskResultT = TypeVar("_TaskResultT")
+
+
+class CooperativeTask:
+    """
+    A L{CooperativeTask} is a task object inside a L{Cooperator}, which can be
+    paused, resumed, and stopped.  It can also have its completion (or
+    termination) monitored.
+
+    @see: L{Cooperator.cooperate}
+
+    @ivar _iterator: the iterator to iterate when this L{CooperativeTask} is
+        asked to do work.
+
+    @ivar _cooperator: the L{Cooperator} that this L{CooperativeTask}
+        participates in, which is used to re-insert it upon resume.
+
+    @ivar _deferreds: the list of L{Deferred}s to fire when this task
+        completes, fails, or finishes.
+
+    @ivar _pauseCount: the number of times that this L{CooperativeTask} has
+        been paused; if 0, it is running.
+
+    @ivar _completionState: The completion-state of this L{CooperativeTask}.
+        L{None} if the task is not yet completed, an instance of L{TaskStopped}
+        if C{stop} was called to stop this task early, of L{TaskFailed} if the
+        application code in the iterator raised an exception which caused it to
+        terminate, and of L{TaskDone} if it terminated normally via raising
+        C{StopIteration}.
+    """
+
+    def __init__(
+        self, iterator: Iterator[_TaskResultT], cooperator: "Cooperator"
+    ) -> None:
+        """
+        A private constructor: to create a new L{CooperativeTask}, see
+        L{Cooperator.cooperate}.
+        """
+        self._iterator = iterator
+        self._cooperator = cooperator
+        self._deferreds: List[Deferred[Iterator[_TaskResultT]]] = []
+        self._pauseCount = 0
+        self._completionState: Optional[SchedulerError] = None
+        self._completionResult: Optional[Union[Iterator[_TaskResultT], Failure]] = None
+        cooperator._addTask(self)
+
+    def whenDone(self) -> Deferred[Iterator[_TaskResultT]]:
+        """
+        Get a L{Deferred} notification of when this task is complete.
+
+        @return: a L{Deferred} that fires with the C{iterator} that this
+            L{CooperativeTask} was created with when the iterator has been
+            exhausted (i.e. its C{next} method has raised C{StopIteration}), or
+            fails with the exception raised by C{next} if it raises some other
+            exception.
+
+        @rtype: L{Deferred}
+        """
+        d: Deferred[Iterator[_TaskResultT]] = Deferred()
+        if self._completionState is None:
+            self._deferreds.append(d)
+        else:
+            assert self._completionResult is not None
+            d.callback(self._completionResult)
+        return d
+
+    def pause(self) -> None:
+        """
+        Pause this L{CooperativeTask}.  Stop doing work until
+        L{CooperativeTask.resume} is called.  If C{pause} is called more than
+        once, C{resume} must be called an equal number of times to resume this
+        task.
+
+        @raise TaskFinished: if this task has already finished or completed.
+        """
+        self._checkFinish()
+        self._pauseCount += 1
+        if self._pauseCount == 1:
+            self._cooperator._removeTask(self)
+
+    def resume(self) -> None:
+        """
+        Resume processing of a paused L{CooperativeTask}.
+
+        @raise NotPaused: if this L{CooperativeTask} is not paused.
+        """
+        if self._pauseCount == 0:
+            raise NotPaused()
+        self._pauseCount -= 1
+        if self._pauseCount == 0 and self._completionState is None:
+            self._cooperator._addTask(self)
+
+    def _completeWith(
+        self,
+        completionState: SchedulerError,
+        deferredResult: Union[Iterator[_TaskResultT], Failure],
+    ) -> None:
+        """
+        @param completionState: a L{SchedulerError} exception or a subclass
+            thereof, indicating what exception should be raised when subsequent
+            operations are performed.
+
+        @param deferredResult: the result to fire all the deferreds with.
+        """
+        self._completionState = completionState
+        self._completionResult = deferredResult
+        if not self._pauseCount:
+            self._cooperator._removeTask(self)
+
+        # The Deferreds need to be invoked after all this is completed, because
+        # a Deferred may want to manipulate other tasks in a Cooperator.  For
+        # example, if you call "stop()" on a cooperator in a callback on a
+        # Deferred returned from whenDone(), this CooperativeTask must be gone
+        # from the Cooperator by that point so that _completeWith is not
+        # invoked reentrantly; that would cause these Deferreds to blow up with
+        # an AlreadyCalledError, or the _removeTask to fail with a ValueError.
+        for d in self._deferreds:
+            d.callback(deferredResult)
+
+    def stop(self) -> None:
+        """
+        Stop further processing of this task.
+
+        @raise TaskFinished: if this L{CooperativeTask} has previously
+            completed, via C{stop}, completion, or failure.
+        """
+        self._checkFinish()
+        self._completeWith(TaskStopped(), Failure(TaskStopped()))
+
+    def _checkFinish(self) -> None:
+        """
+        If this task has been stopped, raise the appropriate subclass of
+        L{TaskFinished}.
+        """
+        if self._completionState is not None:
+            raise self._completionState
+
+    def _oneWorkUnit(self) -> None:
+        """
+        Perform one unit of work for this task, retrieving one item from its
+        iterator, stopping if there are no further items in the iterator, and
+        pausing if the result was a L{Deferred}.
+        """
+        try:
+            result = next(self._iterator)
+        except StopIteration:
+            self._completeWith(TaskDone(), self._iterator)
+        except BaseException:
+            self._completeWith(TaskFailed(), Failure())
+        else:
+            if isinstance(result, Deferred):
+                self.pause()
+
+                def failLater(failure: Failure) -> None:
+                    self._completeWith(TaskFailed(), failure)
+
+                result.addCallbacks(lambda result: self.resume(), failLater)
+
+
+class Cooperator:
+    """
+    Cooperative task scheduler.
+
+    A cooperative task is an iterator where each iteration represents an
+    atomic unit of work.  When the iterator yields, it allows the
+    L{Cooperator} to decide which of its tasks to execute next.  If the
+    iterator yields a L{Deferred} then work will pause until the
+    L{Deferred} fires and completes its callback chain.
+
+    When a L{Cooperator} has more than one task, it distributes work between
+    all tasks.
+
+    There are two ways to add tasks to a L{Cooperator}, L{cooperate} and
+    L{coiterate}.  L{cooperate} is the more useful of the two, as it returns a
+    L{CooperativeTask}, which can be L{paused<CooperativeTask.pause>},
+    L{resumed<CooperativeTask.resume>} and L{waited
+    on<CooperativeTask.whenDone>}.  L{coiterate} has the same effect, but
+    returns only a L{Deferred} that fires when the task is done.
+
+    L{Cooperator} can be used for many things, including but not limited to:
+
+      - running one or more computationally intensive tasks without blocking
+      - limiting parallelism by running a subset of the total tasks
+        simultaneously
+      - doing one thing, waiting for a L{Deferred} to fire,
+        doing the next thing, repeat (i.e. serializing a sequence of
+        asynchronous tasks)
+
+    Multiple L{Cooperator}s do not cooperate with each other, so for most
+    cases you should use the L{global cooperator<task.cooperate>}.
+    """
+
+    def __init__(
+        self,
+        terminationPredicateFactory: Callable[[], Callable[[], bool]] = _Timer,
+        scheduler: Callable[[Callable[[], None]], IDelayedCall] = _defaultScheduler,
+        started: bool = True,
+    ):
+        """
+        Create a scheduler-like object to which iterators may be added.
+
+        @param terminationPredicateFactory: A no-argument callable which will
+        be invoked at the beginning of each step and should return a
+        no-argument callable which will return True when the step should be
+        terminated.  The default factory is time-based and allows iterators to
+        run for 1/100th of a second at a time.
+
+        @param scheduler: A one-argument callable which takes a no-argument
+        callable and should invoke it at some future point.  This will be used
+        to schedule each step of this Cooperator.
+
+        @param started: A boolean which indicates whether iterators should be
+        stepped as soon as they are added, or if they will be queued up until
+        L{Cooperator.start} is called.
+        """
+        self._tasks: List[CooperativeTask] = []
+        self._metarator: Iterator[CooperativeTask] = iter(())
+        self._terminationPredicateFactory = terminationPredicateFactory
+        self._scheduler = scheduler
+        self._delayedCall: Optional[IDelayedCall] = None
+        self._stopped = False
+        self._started = started
+
+    def coiterate(
+        self,
+        iterator: Iterator[_TaskResultT],
+        doneDeferred: Optional[Deferred[Iterator[_TaskResultT]]] = None,
+    ) -> Deferred[Iterator[_TaskResultT]]:
+        """
+        Add an iterator to the list of iterators this L{Cooperator} is
+        currently running.
+
+        Equivalent to L{cooperate}, but returns a L{Deferred} that will
+        be fired when the task is done.
+
+        @param doneDeferred: If specified, this will be the Deferred used as
+            the completion deferred.  It is suggested that you use the default,
+            which creates a new Deferred for you.
+
+        @return: a Deferred that will fire when the iterator finishes.
+        """
+        if doneDeferred is None:
+            doneDeferred = Deferred()
+        whenDone: Deferred[Iterator[_TaskResultT]] = CooperativeTask(
+            iterator, self
+        ).whenDone()
+        whenDone.chainDeferred(doneDeferred)
+        return doneDeferred
+
+    def cooperate(self, iterator: Iterator[_TaskResultT]) -> CooperativeTask:
+        """
+        Start running the given iterator as a long-running cooperative task, by
+        calling next() on it as a periodic timed event.
+
+        @param iterator: the iterator to invoke.
+
+        @return: a L{CooperativeTask} object representing this task.
+        """
+        return CooperativeTask(iterator, self)
+
+    def _addTask(self, task: CooperativeTask) -> None:
+        """
+        Add a L{CooperativeTask} object to this L{Cooperator}.
+        """
+        if self._stopped:
+            self._tasks.append(task)  # XXX silly, I know, but _completeWith
+            # does the inverse
+            task._completeWith(SchedulerStopped(), Failure(SchedulerStopped()))
+        else:
+            self._tasks.append(task)
+            self._reschedule()
+
+    def _removeTask(self, task: CooperativeTask) -> None:
+        """
+        Remove a L{CooperativeTask} from this L{Cooperator}.
+        """
+        self._tasks.remove(task)
+        # If no work left to do, cancel the delayed call:
+        if not self._tasks and self._delayedCall:
+            self._delayedCall.cancel()
+            self._delayedCall = None
+
+    def _tasksWhileNotStopped(self) -> Iterable[CooperativeTask]:
+        """
+        Yield all L{CooperativeTask} objects in a loop as long as this
+        L{Cooperator}'s termination condition has not been met.
+        """
+        terminator = self._terminationPredicateFactory()
+        while self._tasks:
+            for t in self._metarator:
+                yield t
+                if terminator():
+                    return
+            self._metarator = iter(self._tasks)
+
+    def _tick(self) -> None:
+        """
+        Run one scheduler tick.
+        """
+        self._delayedCall = None
+        for taskObj in self._tasksWhileNotStopped():
+            taskObj._oneWorkUnit()
+        self._reschedule()
+
+    _mustScheduleOnStart = False
+
+    def _reschedule(self) -> None:
+        if not self._started:
+            self._mustScheduleOnStart = True
+            return
+        if self._delayedCall is None and self._tasks:
+            self._delayedCall = self._scheduler(self._tick)
+
+    def start(self) -> None:
+        """
+        Begin scheduling steps.
+        """
+        self._stopped = False
+        self._started = True
+        if self._mustScheduleOnStart:
+            del self._mustScheduleOnStart
+            self._reschedule()
+
+    def stop(self) -> None:
+        """
+        Stop scheduling steps.  Errback the completion Deferreds of all
+        iterators which have been added and forget about them.
+        """
+        self._stopped = True
+        for taskObj in self._tasks:
+            taskObj._completeWith(SchedulerStopped(), Failure(SchedulerStopped()))
+        self._tasks = []
+        if self._delayedCall is not None:
+            self._delayedCall.cancel()
+            self._delayedCall = None
+
+    @property
+    def running(self) -> bool:
+        """
+        Is this L{Cooperator} is currently running?
+
+        @return: C{True} if the L{Cooperator} is running, C{False} otherwise.
+        @rtype: C{bool}
+        """
+        return self._started and not self._stopped
+
+
+_theCooperator = Cooperator()
+
+
+def coiterate(iterator: Iterator[_T]) -> Deferred[Iterator[_T]]:
+    """
+    Cooperatively iterate over the given iterator, dividing runtime between it
+    and all other iterators which have been passed to this function and not yet
+    exhausted.
+
+    @param iterator: the iterator to invoke.
+
+    @return: a Deferred that will fire when the iterator finishes.
+    """
+    return _theCooperator.coiterate(iterator)
+
+
+def cooperate(iterator: Iterator[_T]) -> CooperativeTask:
+    """
+    Start running the given iterator as a long-running cooperative task, by
+    calling next() on it as a periodic timed event.
+
+    This is very useful if you have computationally expensive tasks that you
+    want to run without blocking the reactor.  Just break each task up so that
+    it yields frequently, pass it in here and the global L{Cooperator} will
+    make sure work is distributed between them without blocking longer than a
+    single iteration of a single task.
+
+    @param iterator: the iterator to invoke.
+
+    @return: a L{CooperativeTask} object representing this task.
+    """
+    return _theCooperator.cooperate(iterator)
+
+
+@implementer(IReactorTime)
+class Clock:
+    """
+    Provide a deterministic, easily-controlled implementation of
+    L{IReactorTime.callLater}.  This is commonly useful for writing
+    deterministic unit tests for code which schedules events using this API.
+    """
+
+    rightNow = 0.0
+
+    def __init__(self) -> None:
+        self.calls: List[DelayedCall] = []
+
+    def seconds(self) -> float:
+        """
+        Pretend to be time.time().  This is used internally when an operation
+        such as L{IDelayedCall.reset} needs to determine a time value
+        relative to the current time.
+
+        @return: The time which should be considered the current time.
+        """
+        return self.rightNow
+
+    def _sortCalls(self) -> None:
+        """
+        Sort the pending calls according to the time they are scheduled.
+        """
+        self.calls.sort(key=lambda a: a.getTime())
+
+    def callLater(
+        self, delay: float, callable: Callable[..., object], *args: object, **kw: object
+    ) -> IDelayedCall:
+        """
+        See L{twisted.internet.interfaces.IReactorTime.callLater}.
+        """
+        dc = DelayedCall(
+            self.seconds() + delay,
+            callable,
+            args,
+            kw,
+            self.calls.remove,
+            lambda c: None,
+            self.seconds,
+        )
+        self.calls.append(dc)
+        self._sortCalls()
+        return dc
+
+    def getDelayedCalls(self) -> Sequence[IDelayedCall]:
+        """
+        See L{twisted.internet.interfaces.IReactorTime.getDelayedCalls}
+        """
+        return self.calls
+
+    def advance(self, amount: float) -> None:
+        """
+        Move time on this clock forward by the given amount and run whatever
+        pending calls should be run.
+
+        @param amount: The number of seconds which to advance this clock's
+        time.
+        """
+        self.rightNow += amount
+        self._sortCalls()
+        while self.calls and self.calls[0].getTime() <= self.seconds():
+            call = self.calls.pop(0)
+            call.called = 1
+            call.func(*call.args, **call.kw)
+            self._sortCalls()
+
+    def pump(self, timings: Iterable[float]) -> None:
+        """
+        Advance incrementally by the given set of times.
+        """
+        for amount in timings:
+            self.advance(amount)
+
+
+def deferLater(
+    clock: IReactorTime,
+    delay: float,
+    callable: Optional[Callable[..., _T]] = None,
+    *args: object,
+    **kw: object,
+) -> Deferred[_T]:
+    """
+    Call the given function after a certain period of time has passed.
+
+    @param clock: The object which will be used to schedule the delayed
+        call.
+
+    @param delay: The number of seconds to wait before calling the function.
+
+    @param callable: The callable to call after the delay, or C{None}.
+
+    @param args: The positional arguments to pass to C{callable}.
+
+    @param kw: The keyword arguments to pass to C{callable}.
+
+    @return: A deferred that fires with the result of the callable when the
+        specified time has elapsed.
+    """
+
+    def deferLaterCancel(deferred: Deferred[object]) -> None:
+        delayedCall.cancel()
+
+    def cb(result: object) -> _T:
+        if callable is None:
+            return None  # type: ignore[return-value]
+        return callable(*args, **kw)
+
+    d: Deferred[_T] = Deferred(deferLaterCancel)
+    d.addCallback(cb)
+    delayedCall = clock.callLater(delay, d.callback, None)
+    return d
+
+
+def react(
+    main: Callable[
+        ...,
+        Union[Deferred[_T], Coroutine["Deferred[_T]", object, _T]],
+    ],
+    argv: Iterable[object] = (),
+    _reactor: Optional[IReactorCore] = None,
+) -> NoReturn:
+    """
+    Call C{main} and run the reactor until the L{Deferred} it returns fires or
+    the coroutine it returns completes.
+
+    This is intended as the way to start up an application with a well-defined
+    completion condition.  Use it to write clients or one-off asynchronous
+    operations.  Prefer this to calling C{reactor.run} directly, as this
+    function will also:
+
+      - Take care to call C{reactor.stop} once and only once, and at the right
+        time.
+      - Log any failures from the C{Deferred} returned by C{main}.
+      - Exit the application when done, with exit code 0 in case of success and
+        1 in case of failure. If C{main} fails with a C{SystemExit} error, the
+        code returned is used.
+
+    The following demonstrates the signature of a C{main} function which can be
+    used with L{react}::
+
+      async def main(reactor, username, password):
+          return "ok"
+
+      task.react(main, ("alice", "secret"))
+
+    @param main: A callable which returns a L{Deferred} or
+        coroutine. It should take the reactor as its first
+        parameter, followed by the elements of C{argv}.
+
+    @param argv: A list of arguments to pass to C{main}. If omitted the
+        callable will be invoked with no additional arguments.
+
+    @param _reactor: An implementation detail to allow easier unit testing.  Do
+        not supply this parameter.
+
+    @since: 12.3
+    """
+    if _reactor is None:
+        from twisted.internet import reactor
+
+        _reactor = cast(IReactorCore, reactor)
+
+    finished = ensureDeferred(main(_reactor, *argv))
+    code = 0
+
+    stopping = False
+
+    def onShutdown() -> None:
+        nonlocal stopping
+        stopping = True
+
+    _reactor.addSystemEventTrigger("before", "shutdown", onShutdown)
+
+    def stop(result: object, stopReactor: bool) -> None:
+        if stopReactor:
+            assert _reactor is not None
+            try:
+                _reactor.stop()
+            except ReactorNotRunning:
+                pass
+
+        if isinstance(result, Failure):
+            nonlocal code
+            if result.check(SystemExit) is not None:
+                code = result.value.code
+            else:
+                log.err(result, "main function encountered error")
+                code = 1
+
+    def cbFinish(result: object) -> None:
+        if stopping:
+            stop(result, False)
+        else:
+            assert _reactor is not None
+            _reactor.callWhenRunning(stop, result, True)
+
+    finished.addBoth(cbFinish)
+    _reactor.run()
+    sys.exit(code)
+
+
+__all__ = [
+    "LoopingCall",
+    "Clock",
+    "SchedulerStopped",
+    "Cooperator",
+    "coiterate",
+    "deferLater",
+    "react",
+]
diff --git a/contrib/python/Twisted/py3/twisted/internet/tcp.py b/contrib/python/Twisted/py3/twisted/internet/tcp.py
new file mode 100644
index 00000000000..c87b5b73339
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/tcp.py
@@ -0,0 +1,1523 @@
+# -*- test-case-name: twisted.test.test_tcp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Various asynchronous TCP/IP classes.
+
+End users shouldn't use this module directly - use the reactor APIs instead.
+"""
+
+import os
+
+# System Imports
+import socket
+import struct
+import sys
+from typing import Callable, ClassVar, List, Optional
+
+from zope.interface import Interface, implementer
+
+import attr
+import typing_extensions
+
+from twisted.internet.interfaces import (
+    IHalfCloseableProtocol,
+    IListeningPort,
+    ISystemHandle,
+    ITCPTransport,
+)
+from twisted.logger import ILogObserver, LogEvent, Logger
+from twisted.python import deprecate, versions
+from twisted.python.compat import lazyByteSlice
+from twisted.python.runtime import platformType
+
+try:
+    # Try to get the memory BIO based startTLS implementation, available since
+    # pyOpenSSL 0.10
+    from twisted.internet._newtls import (
+        ClientMixin as _TLSClientMixin,
+        ConnectionMixin as _TLSConnectionMixin,
+        ServerMixin as _TLSServerMixin,
+    )
+    from twisted.internet.interfaces import ITLSTransport
+except ImportError:
+    # There is no version of startTLS available
+    ITLSTransport = Interface  # type: ignore[misc,assignment]
+
+    class _TLSConnectionMixin:  # type: ignore[no-redef]
+        TLS = False
+
+    class _TLSClientMixin:  # type: ignore[no-redef]
+        pass
+
+    class _TLSServerMixin:  # type: ignore[no-redef]
+        pass
+
+
+if platformType == "win32":
+    # no such thing as WSAEPERM or error code 10001
+    # according to winsock.h or MSDN
+    EPERM = object()
+    from errno import (  # type: ignore[attr-defined]
+        WSAEALREADY as EALREADY,
+        WSAEINPROGRESS as EINPROGRESS,
+        WSAEINVAL as EINVAL,
+        WSAEISCONN as EISCONN,
+        WSAEMFILE as EMFILE,
+        WSAENOBUFS as ENOBUFS,
+        WSAEWOULDBLOCK as EWOULDBLOCK,
+    )
+
+    # No such thing as WSAENFILE, either.
+    ENFILE = object()
+    # Nor ENOMEM
+    ENOMEM = object()
+    EAGAIN = EWOULDBLOCK
+    from errno import WSAECONNRESET as ECONNABORTED  # type: ignore[attr-defined]
+
+    from twisted.python.win32 import formatError as strerror
+else:
+    from errno import EPERM
+    from errno import EINVAL
+    from errno import EWOULDBLOCK
+    from errno import EINPROGRESS
+    from errno import EALREADY
+    from errno import EISCONN
+    from errno import ENOBUFS
+    from errno import EMFILE
+    from errno import ENFILE
+    from errno import ENOMEM
+    from errno import EAGAIN
+    from errno import ECONNABORTED
+
+    from os import strerror
+
+from errno import errorcode
+
+# Twisted Imports
+from twisted.internet import abstract, address, base, error, fdesc, main
+from twisted.internet.error import CannotListenError
+from twisted.internet.protocol import Protocol
+from twisted.internet.task import deferLater
+from twisted.python import failure, log, reflect
+from twisted.python.util import untilConcludes
+
+# Not all platforms have, or support, this flag.
+_AI_NUMERICSERV = getattr(socket, "AI_NUMERICSERV", 0)
+
+
+def _getrealname(addr):
+    """
+    Return a 2-tuple of socket IP and port for IPv4 and a 4-tuple of
+    socket IP, port, flowInfo, and scopeID for IPv6.  For IPv6, it
+    returns the interface portion (the part after the %) as a part of
+    the IPv6 address, which Python 3.7+ does not include.
+
+    @param addr: A 2-tuple for IPv4 information or a 4-tuple for IPv6
+        information.
+    """
+    if len(addr) == 4:
+        # IPv6
+        host = socket.getnameinfo(addr, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV)[
+            0
+        ]
+        return tuple([host] + list(addr[1:]))
+    else:
+        return addr[:2]
+
+
+def _getpeername(skt):
+    """
+    See L{_getrealname}.
+    """
+    return _getrealname(skt.getpeername())
+
+
+def _getsockname(skt):
+    """
+    See L{_getrealname}.
+    """
+    return _getrealname(skt.getsockname())
+
+
+class _SocketCloser:
+    """
+    @ivar _shouldShutdown: Set to C{True} if C{shutdown} should be called
+        before calling C{close} on the underlying socket.
+    @type _shouldShutdown: C{bool}
+    """
+
+    _shouldShutdown = True
+
+    def _closeSocket(self, orderly):
+        # The call to shutdown() before close() isn't really necessary, because
+        # we set FD_CLOEXEC now, which will ensure this is the only process
+        # holding the FD, thus ensuring close() really will shutdown the TCP
+        # socket. However, do it anyways, just to be safe.
+        skt = self.socket
+        try:
+            if orderly:
+                if self._shouldShutdown:
+                    skt.shutdown(2)
+            else:
+                # Set SO_LINGER to 1,0 which, by convention, causes a
+                # connection reset to be sent when close is called,
+                # instead of the standard FIN shutdown sequence.
+                self.socket.setsockopt(
+                    socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)
+                )
+
+        except OSError:
+            pass
+        try:
+            skt.close()
+        except OSError:
+            pass
+
+
+class _AbortingMixin:
+    """
+    Common implementation of C{abortConnection}.
+
+    @ivar _aborting: Set to C{True} when C{abortConnection} is called.
+    @type _aborting: C{bool}
+    """
+
+    _aborting = False
+
+    def abortConnection(self):
+        """
+        Aborts the connection immediately, dropping any buffered data.
+
+        @since: 11.1
+        """
+        if self.disconnected or self._aborting:
+            return
+        self._aborting = True
+        self.stopReading()
+        self.stopWriting()
+        self.doRead = lambda *args, **kwargs: None
+        self.doWrite = lambda *args, **kwargs: None
+        self.reactor.callLater(
+            0, self.connectionLost, failure.Failure(error.ConnectionAborted())
+        )
+
+
+@implementer(ITLSTransport, ITCPTransport, ISystemHandle)
+class Connection(
+    _TLSConnectionMixin, abstract.FileDescriptor, _SocketCloser, _AbortingMixin
+):
+    """
+    Superclass of all socket-based FileDescriptors.
+
+    This is an abstract superclass of all objects which represent a TCP/IP
+    connection based socket.
+
+    @ivar logstr: prefix used when logging events related to this connection.
+    @type logstr: C{str}
+    """
+
+    def __init__(self, skt, protocol, reactor=None):
+        abstract.FileDescriptor.__init__(self, reactor=reactor)
+        self.socket = skt
+        self.socket.setblocking(0)
+        self.fileno = skt.fileno
+        self.protocol = protocol
+
+    def getHandle(self):
+        """Return the socket for this connection."""
+        return self.socket
+
+    def doRead(self):
+        """Calls self.protocol.dataReceived with all available data.
+
+        This reads up to self.bufferSize bytes of data from its socket, then
+        calls self.dataReceived(data) to process it.  If the connection is not
+        lost through an error in the physical recv(), this function will return
+        the result of the dataReceived call.
+        """
+        try:
+            data = self.socket.recv(self.bufferSize)
+        except OSError as se:
+            if se.args[0] == EWOULDBLOCK:
+                return
+            else:
+                return main.CONNECTION_LOST
+
+        return self._dataReceived(data)
+
+    def _dataReceived(self, data):
+        if not data:
+            return main.CONNECTION_DONE
+        rval = self.protocol.dataReceived(data)
+        if rval is not None:
+            offender = self.protocol.dataReceived
+            warningFormat = (
+                "Returning a value other than None from %(fqpn)s is "
+                "deprecated since %(version)s."
+            )
+            warningString = deprecate.getDeprecationWarningString(
+                offender, versions.Version("Twisted", 11, 0, 0), format=warningFormat
+            )
+            deprecate.warnAboutFunction(offender, warningString)
+        return rval
+
+    def writeSomeData(self, data):
+        """
+        Write as much as possible of the given data to this TCP connection.
+
+        This sends up to C{self.SEND_LIMIT} bytes from C{data}.  If the
+        connection is lost, an exception is returned.  Otherwise, the number
+        of bytes successfully written is returned.
+        """
+        # Limit length of buffer to try to send, because some OSes are too
+        # stupid to do so themselves (ahem windows)
+        limitedData = lazyByteSlice(data, 0, self.SEND_LIMIT)
+
+        try:
+            return untilConcludes(self.socket.send, limitedData)
+        except OSError as se:
+            if se.args[0] in (EWOULDBLOCK, ENOBUFS):
+                return 0
+            else:
+                return main.CONNECTION_LOST
+
+    def _closeWriteConnection(self):
+        try:
+            self.socket.shutdown(1)
+        except OSError:
+            pass
+        p = IHalfCloseableProtocol(self.protocol, None)
+        if p:
+            try:
+                p.writeConnectionLost()
+            except BaseException:
+                f = failure.Failure()
+                log.err()
+                self.connectionLost(f)
+
+    def readConnectionLost(self, reason):
+        p = IHalfCloseableProtocol(self.protocol, None)
+        if p:
+            try:
+                p.readConnectionLost()
+            except BaseException:
+                log.err()
+                self.connectionLost(failure.Failure())
+        else:
+            self.connectionLost(reason)
+
+    def connectionLost(self, reason):
+        """See abstract.FileDescriptor.connectionLost()."""
+        # Make sure we're not called twice, which can happen e.g. if
+        # abortConnection() is called from protocol's dataReceived and then
+        # code immediately after throws an exception that reaches the
+        # reactor. We can't rely on "disconnected" attribute for this check
+        # since twisted.internet._oldtls does evil things to it:
+        if not hasattr(self, "socket"):
+            return
+        abstract.FileDescriptor.connectionLost(self, reason)
+        self._closeSocket(not reason.check(error.ConnectionAborted))
+        protocol = self.protocol
+        del self.protocol
+        del self.socket
+        del self.fileno
+        protocol.connectionLost(reason)
+
+    logstr = "Uninitialized"
+
+    def logPrefix(self):
+        """Return the prefix to log with when I own the logging thread."""
+        return self.logstr
+
+    def getTcpNoDelay(self):
+        return bool(self.socket.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY))
+
+    def setTcpNoDelay(self, enabled):
+        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, enabled)
+
+    def getTcpKeepAlive(self):
+        return bool(self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE))
+
+    def setTcpKeepAlive(self, enabled):
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, enabled)
+
+
+class _BaseBaseClient:
+    """
+    Code shared with other (non-POSIX) reactors for management of general
+    outgoing connections.
+
+    Requirements upon subclasses are documented as instance variables rather
+    than abstract methods, in order to avoid MRO confusion, since this base is
+    mixed in to unfortunately weird and distinctive multiple-inheritance
+    hierarchies and many of these attributes are provided by peer classes
+    rather than descendant classes in those hierarchies.
+
+    @ivar addressFamily: The address family constant (C{socket.AF_INET},
+        C{socket.AF_INET6}, C{socket.AF_UNIX}) of the underlying socket of this
+        client connection.
+    @type addressFamily: C{int}
+
+    @ivar socketType: The socket type constant (C{socket.SOCK_STREAM} or
+        C{socket.SOCK_DGRAM}) of the underlying socket.
+    @type socketType: C{int}
+
+    @ivar _requiresResolution: A flag indicating whether the address of this
+        client will require name resolution.  C{True} if the hostname of said
+        address indicates a name that must be resolved by hostname lookup,
+        C{False} if it indicates an IP address literal.
+    @type _requiresResolution: C{bool}
+
+    @cvar _commonConnection: Subclasses must provide this attribute, which
+        indicates the L{Connection}-alike class to invoke C{__init__} and
+        C{connectionLost} on.
+    @type _commonConnection: C{type}
+
+    @ivar _stopReadingAndWriting: Subclasses must implement in order to remove
+        this transport from its reactor's notifications in response to a
+        terminated connection attempt.
+    @type _stopReadingAndWriting: 0-argument callable returning L{None}
+
+    @ivar _closeSocket: Subclasses must implement in order to close the socket
+        in response to a terminated connection attempt.
+    @type _closeSocket: 1-argument callable; see L{_SocketCloser._closeSocket}
+
+    @ivar _collectSocketDetails: Clean up references to the attached socket in
+        its underlying OS resource (such as a file descriptor or file handle),
+        as part of post connection-failure cleanup.
+    @type _collectSocketDetails: 0-argument callable returning L{None}.
+
+    @ivar reactor: The class pointed to by C{_commonConnection} should set this
+        attribute in its constructor.
+    @type reactor: L{twisted.internet.interfaces.IReactorTime},
+        L{twisted.internet.interfaces.IReactorCore},
+        L{twisted.internet.interfaces.IReactorFDSet}
+    """
+
+    addressFamily = socket.AF_INET
+    socketType = socket.SOCK_STREAM
+
+    def _finishInit(self, whenDone, skt, error, reactor):
+        """
+        Called by subclasses to continue to the stage of initialization where
+        the socket connect attempt is made.
+
+        @param whenDone: A 0-argument callable to invoke once the connection is
+            set up.  This is L{None} if the connection could not be prepared
+            due to a previous error.
+
+        @param skt: The socket object to use to perform the connection.
+        @type skt: C{socket._socketobject}
+
+        @param error: The error to fail the connection with.
+
+        @param reactor: The reactor to use for this client.
+        @type reactor: L{twisted.internet.interfaces.IReactorTime}
+        """
+        if whenDone:
+            self._commonConnection.__init__(self, skt, None, reactor)
+            reactor.callLater(0, whenDone)
+        else:
+            reactor.callLater(0, self.failIfNotConnected, error)
+
+    def resolveAddress(self):
+        """
+        Resolve the name that was passed to this L{_BaseBaseClient}, if
+        necessary, and then move on to attempting the connection once an
+        address has been determined.  (The connection will be attempted
+        immediately within this function if either name resolution can be
+        synchronous or the address was an IP address literal.)
+
+        @note: You don't want to call this method from outside, as it won't do
+            anything useful; it's just part of the connection bootstrapping
+            process.  Also, although this method is on L{_BaseBaseClient} for
+            historical reasons, it's not used anywhere except for L{Client}
+            itself.
+
+        @return: L{None}
+        """
+        if self._requiresResolution:
+            d = self.reactor.resolve(self.addr[0])
+            d.addCallback(lambda n: (n,) + self.addr[1:])
+            d.addCallbacks(self._setRealAddress, self.failIfNotConnected)
+        else:
+            self._setRealAddress(self.addr)
+
+    def _setRealAddress(self, address):
+        """
+        Set the resolved address of this L{_BaseBaseClient} and initiate the
+        connection attempt.
+
+        @param address: Depending on whether this is an IPv4 or IPv6 connection
+            attempt, a 2-tuple of C{(host, port)} or a 4-tuple of C{(host,
+            port, flow, scope)}.  At this point it is a fully resolved address,
+            and the 'host' portion will always be an IP address, not a DNS
+            name.
+        """
+        if len(address) == 4:
+            # IPv6, make sure we have the scopeID associated
+            hostname = socket.getnameinfo(
+                address, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV
+            )[0]
+            self.realAddress = tuple([hostname] + list(address[1:]))
+        else:
+            self.realAddress = address
+        self.doConnect()
+
+    def failIfNotConnected(self, err):
+        """
+        Generic method called when the attempts to connect failed. It basically
+        cleans everything it can: call connectionFailed, stop read and write,
+        delete socket related members.
+        """
+        if self.connected or self.disconnected or not hasattr(self, "connector"):
+            return
+
+        self._stopReadingAndWriting()
+        try:
+            self._closeSocket(True)
+        except AttributeError:
+            pass
+        else:
+            self._collectSocketDetails()
+        self.connector.connectionFailed(failure.Failure(err))
+        del self.connector
+
+    def stopConnecting(self):
+        """
+        If a connection attempt is still outstanding (i.e.  no connection is
+        yet established), immediately stop attempting to connect.
+        """
+        self.failIfNotConnected(error.UserError())
+
+    def connectionLost(self, reason):
+        """
+        Invoked by lower-level logic when it's time to clean the socket up.
+        Depending on the state of the connection, either inform the attached
+        L{Connector} that the connection attempt has failed, or inform the
+        connected L{IProtocol} that the established connection has been lost.
+
+        @param reason: the reason that the connection was terminated
+        @type reason: L{Failure}
+        """
+        if not self.connected:
+            self.failIfNotConnected(error.ConnectError(string=reason))
+        else:
+            self._commonConnection.connectionLost(self, reason)
+            self.connector.connectionLost(reason)
+
+
+class BaseClient(_BaseBaseClient, _TLSClientMixin, Connection):
+    """
+    A base class for client TCP (and similar) sockets.
+
+    @ivar realAddress: The address object that will be used for socket.connect;
+        this address is an address tuple (the number of elements dependent upon
+        the address family) which does not contain any names which need to be
+        resolved.
+    @type realAddress: C{tuple}
+
+    @ivar _base: L{Connection}, which is the base class of this class which has
+        all of the useful file descriptor methods.  This is used by
+        L{_TLSServerMixin} to call the right methods to directly manipulate the
+        transport, as is necessary for writing TLS-encrypted bytes (whereas
+        those methods on L{Server} will go through another layer of TLS if it
+        has been enabled).
+    """
+
+    _base = Connection
+    _commonConnection = Connection
+
+    def _stopReadingAndWriting(self):
+        """
+        Implement the POSIX-ish (i.e.
+        L{twisted.internet.interfaces.IReactorFDSet}) method of detaching this
+        socket from the reactor for L{_BaseBaseClient}.
+        """
+        if hasattr(self, "reactor"):
+            # this doesn't happen if we failed in __init__
+            self.stopReading()
+            self.stopWriting()
+
+    def _collectSocketDetails(self):
+        """
+        Clean up references to the socket and its file descriptor.
+
+        @see: L{_BaseBaseClient}
+        """
+        del self.socket, self.fileno
+
+    def createInternetSocket(self):
+        """(internal) Create a non-blocking socket using
+        self.addressFamily, self.socketType.
+        """
+        s = socket.socket(self.addressFamily, self.socketType)
+        s.setblocking(0)
+        fdesc._setCloseOnExec(s.fileno())
+        return s
+
+    def doConnect(self):
+        """
+        Initiate the outgoing connection attempt.
+
+        @note: Applications do not need to call this method; it will be invoked
+            internally as part of L{IReactorTCP.connectTCP}.
+        """
+        self.doWrite = self.doConnect
+        self.doRead = self.doConnect
+        if not hasattr(self, "connector"):
+            # this happens when connection failed but doConnect
+            # was scheduled via a callLater in self._finishInit
+            return
+
+        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+        if err:
+            self.failIfNotConnected(error.getConnectError((err, strerror(err))))
+            return
+
+        # doConnect gets called twice.  The first time we actually need to
+        # start the connection attempt.  The second time we don't really
+        # want to (SO_ERROR above will have taken care of any errors, and if
+        # it reported none, the mere fact that doConnect was called again is
+        # sufficient to indicate that the connection has succeeded), but it
+        # is not /particularly/ detrimental to do so.  This should get
+        # cleaned up some day, though.
+        try:
+            connectResult = self.socket.connect_ex(self.realAddress)
+        except OSError as se:
+            connectResult = se.args[0]
+        if connectResult:
+            if connectResult == EISCONN:
+                pass
+            # on Windows EINVAL means sometimes that we should keep trying:
+            # http://msdn.microsoft.com/library/default.asp?url=/library/en-us/winsock/winsock/connect_2.asp
+            elif (connectResult in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (
+                connectResult == EINVAL and platformType == "win32"
+            ):
+                self.startReading()
+                self.startWriting()
+                return
+            else:
+                self.failIfNotConnected(
+                    error.getConnectError((connectResult, strerror(connectResult)))
+                )
+                return
+
+        # If I have reached this point without raising or returning, that means
+        # that the socket is connected.
+        del self.doWrite
+        del self.doRead
+        # we first stop and then start, to reset any references to the old doRead
+        self.stopReading()
+        self.stopWriting()
+        self._connectDone()
+
+    def _connectDone(self):
+        """
+        This is a hook for when a connection attempt has succeeded.
+
+        Here, we build the protocol from the
+        L{twisted.internet.protocol.ClientFactory} that was passed in, compute
+        a log string, begin reading so as to send traffic to the newly built
+        protocol, and finally hook up the protocol itself.
+
+        This hook is overridden by L{ssl.Client} to initiate the TLS protocol.
+        """
+        self.protocol = self.connector.buildProtocol(self.getPeer())
+        self.connected = 1
+        logPrefix = self._getLogPrefix(self.protocol)
+        self.logstr = "%s,client" % logPrefix
+        if self.protocol is None:
+            # Factory.buildProtocol is allowed to return None.  In that case,
+            # make up a protocol to satisfy the rest of the implementation;
+            # connectionLost is going to be called on something, for example.
+            # This is easier than adding special case support for a None
+            # protocol throughout the rest of the transport implementation.
+            self.protocol = Protocol()
+            # But dispose of the connection quickly.
+            self.loseConnection()
+        else:
+            self.startReading()
+            self.protocol.makeConnection(self)
+
+
+_NUMERIC_ONLY = socket.AI_NUMERICHOST | _AI_NUMERICSERV
+
+
+def _resolveIPv6(ip, port):
+    """
+    Resolve an IPv6 literal into an IPv6 address.
+
+    This is necessary to resolve any embedded scope identifiers to the relevant
+    C{sin6_scope_id} for use with C{socket.connect()}, C{socket.listen()}, or
+    C{socket.bind()}; see U{RFC 3493 <https://tools.ietf.org/html/rfc3493>} for
+    more information.
+
+    @param ip: An IPv6 address literal.
+    @type ip: C{str}
+
+    @param port: A port number.
+    @type port: C{int}
+
+    @return: a 4-tuple of C{(host, port, flow, scope)}, suitable for use as an
+        IPv6 address.
+
+    @raise socket.gaierror: if either the IP or port is not numeric as it
+        should be.
+    """
+    return socket.getaddrinfo(ip, port, 0, 0, 0, _NUMERIC_ONLY)[0][4]
+
+
+class _BaseTCPClient:
+    """
+    Code shared with other (non-POSIX) reactors for management of outgoing TCP
+    connections (both TCPv4 and TCPv6).
+
+    @note: In order to be functional, this class must be mixed into the same
+        hierarchy as L{_BaseBaseClient}.  It would subclass L{_BaseBaseClient}
+        directly, but the class hierarchy here is divided in strange ways out
+        of the need to share code along multiple axes; specifically, with the
+        IOCP reactor and also with UNIX clients in other reactors.
+
+    @ivar _addressType: The Twisted _IPAddress implementation for this client
+    @type _addressType: L{IPv4Address} or L{IPv6Address}
+
+    @ivar connector: The L{Connector} which is driving this L{_BaseTCPClient}'s
+        connection attempt.
+
+    @ivar addr: The address that this socket will be connecting to.
+    @type addr: If IPv4, a 2-C{tuple} of C{(str host, int port)}.  If IPv6, a
+        4-C{tuple} of (C{str host, int port, int ignored, int scope}).
+
+    @ivar createInternetSocket: Subclasses must implement this as a method to
+        create a python socket object of the appropriate address family and
+        socket type.
+    @type createInternetSocket: 0-argument callable returning
+        C{socket._socketobject}.
+    """
+
+    _addressType = address.IPv4Address
+
+    def __init__(self, host, port, bindAddress, connector, reactor=None):
+        # BaseClient.__init__ is invoked later
+        self.connector = connector
+        self.addr = (host, port)
+
+        whenDone = self.resolveAddress
+        err = None
+        skt = None
+
+        if abstract.isIPAddress(host):
+            self._requiresResolution = False
+        elif abstract.isIPv6Address(host):
+            self._requiresResolution = False
+            self.addr = _resolveIPv6(host, port)
+            self.addressFamily = socket.AF_INET6
+            self._addressType = address.IPv6Address
+        else:
+            self._requiresResolution = True
+        try:
+            skt = self.createInternetSocket()
+        except OSError as se:
+            err = error.ConnectBindError(se.args[0], se.args[1])
+            whenDone = None
+        if whenDone and bindAddress is not None:
+            try:
+                if abstract.isIPv6Address(bindAddress[0]):
+                    bindinfo = _resolveIPv6(*bindAddress)
+                else:
+                    bindinfo = bindAddress
+                skt.bind(bindinfo)
+            except OSError as se:
+                err = error.ConnectBindError(se.args[0], se.args[1])
+                whenDone = None
+        self._finishInit(whenDone, skt, err, reactor)
+
+    def getHost(self):
+        """
+        Returns an L{IPv4Address} or L{IPv6Address}.
+
+        This indicates the address from which I am connecting.
+        """
+        return self._addressType("TCP", *_getsockname(self.socket))
+
+    def getPeer(self):
+        """
+        Returns an L{IPv4Address} or L{IPv6Address}.
+
+        This indicates the address that I am connected to.
+        """
+        return self._addressType("TCP", *self.realAddress)
+
+    def __repr__(self) -> str:
+        s = f"<{self.__class__} to {self.addr} at {id(self):x}>"
+        return s
+
+
+class Client(_BaseTCPClient, BaseClient):
+    """
+    A transport for a TCP protocol; either TCPv4 or TCPv6.
+
+    Do not create these directly; use L{IReactorTCP.connectTCP}.
+    """
+
+
+class Server(_TLSServerMixin, Connection):
+    """
+    Serverside socket-stream connection class.
+
+    This is a serverside network connection transport; a socket which came from
+    an accept() on a server.
+
+    @ivar _base: L{Connection}, which is the base class of this class which has
+        all of the useful file descriptor methods.  This is used by
+        L{_TLSServerMixin} to call the right methods to directly manipulate the
+        transport, as is necessary for writing TLS-encrypted bytes (whereas
+        those methods on L{Server} will go through another layer of TLS if it
+        has been enabled).
+    """
+
+    _base = Connection
+
+    _addressType = address.IPv4Address
+
+    def __init__(self, sock, protocol, client, server, sessionno, reactor):
+        """
+        Server(sock, protocol, client, server, sessionno)
+
+        Initialize it with a socket, a protocol, a descriptor for my peer (a
+        tuple of host, port describing the other end of the connection), an
+        instance of Port, and a session number.
+        """
+        Connection.__init__(self, sock, protocol, reactor)
+        if len(client) != 2:
+            self._addressType = address.IPv6Address
+        self.server = server
+        self.client = client
+        self.sessionno = sessionno
+        self.hostname = client[0]
+
+        logPrefix = self._getLogPrefix(self.protocol)
+        self.logstr = f"{logPrefix},{sessionno},{self.hostname}"
+        if self.server is not None:
+            self.repstr: str = "<{} #{} on {}>".format(
+                self.protocol.__class__.__name__,
+                self.sessionno,
+                self.server._realPortNumber,
+            )
+        self.startReading()
+        self.connected = 1
+
+    def __repr__(self) -> str:
+        """
+        A string representation of this connection.
+        """
+        return self.repstr
+
+    @classmethod
+    def _fromConnectedSocket(cls, fileDescriptor, addressFamily, factory, reactor):
+        """
+        Create a new L{Server} based on an existing connected I{SOCK_STREAM}
+        socket.
+
+        Arguments are the same as to L{Server.__init__}, except where noted.
+
+        @param fileDescriptor: An integer file descriptor associated with a
+            connected socket.  The socket must be in non-blocking mode.  Any
+            additional attributes desired, such as I{FD_CLOEXEC}, must also be
+            set already.
+
+        @param addressFamily: The address family (sometimes called I{domain})
+            of the existing socket.  For example, L{socket.AF_INET}.
+
+        @return: A new instance of C{cls} wrapping the socket given by
+            C{fileDescriptor}.
+        """
+        addressType = address.IPv4Address
+        if addressFamily == socket.AF_INET6:
+            addressType = address.IPv6Address
+        skt = socket.fromfd(fileDescriptor, addressFamily, socket.SOCK_STREAM)
+        addr = _getpeername(skt)
+        protocolAddr = addressType("TCP", *addr)
+        localPort = skt.getsockname()[1]
+
+        protocol = factory.buildProtocol(protocolAddr)
+        if protocol is None:
+            skt.close()
+            return
+
+        self = cls(skt, protocol, addr, None, addr[1], reactor)
+        self.repstr = "<{} #{} on {}>".format(
+            self.protocol.__class__.__name__,
+            self.sessionno,
+            localPort,
+        )
+        protocol.makeConnection(self)
+        return self
+
+    def getHost(self):
+        """
+        Returns an L{IPv4Address} or L{IPv6Address}.
+
+        This indicates the server's address.
+        """
+        addr = _getsockname(self.socket)
+        return self._addressType("TCP", *addr)
+
+    def getPeer(self):
+        """
+        Returns an L{IPv4Address} or L{IPv6Address}.
+
+        This indicates the client's address.
+        """
+        return self._addressType("TCP", *self.client)
+
+
+class _IFileDescriptorReservation(Interface):
+    """
+    An open file that represents an emergency reservation in the
+    process' file descriptor table.  If L{Port} encounters C{EMFILE}
+    on C{accept(2)}, it can close this file descriptor, retry the
+    C{accept} so that the incoming connection occupies this file
+    descriptor's space, and then close that connection and reopen this
+    one.
+
+    Calling L{_IFileDescriptorReservation.reserve} attempts to open
+    the reserve file descriptor if it is not already open.
+    L{_IFileDescriptorReservation.available} returns L{True} if the
+    underlying file is open and its descriptor claimed.
+
+    L{_IFileDescriptorReservation} instances are context managers;
+    entering them releases the underlying file descriptor, while
+    exiting them attempts to reacquire it.  The block can take
+    advantage of the free slot in the process' file descriptor table
+    accept and close a client connection.
+
+    Because another thread might open a file descriptor between the
+    time the context manager is entered and the time C{accept} is
+    called, opening the reserve descriptor is best-effort only.
+    """
+
+    def available():
+        """
+        Is the reservation available?
+
+        @return: L{True} if the reserved file descriptor is open and
+            can thus be closed to allow a new file to be opened in its
+            place; L{False} if it is not open.
+        """
+
+    def reserve():
+        """
+        Attempt to open the reserved file descriptor; if this fails
+        because of C{EMFILE}, internal state is reset so that another
+        reservation attempt can be made.
+
+        @raises Exception: Any exception except an L{OSError} whose
+            errno is L{EMFILE}.
+        """
+
+    def __enter__():
+        """
+        Release the underlying file descriptor so that code within the
+        context manager can open a new file.
+        """
+
+    def __exit__(excType, excValue, traceback):
+        """
+        Attempt to re-open the reserved file descriptor.  See
+        L{reserve} for caveats.
+
+        @param excType: See L{object.__exit__}
+        @param excValue: See L{object.__exit__}
+        @param traceback: See L{object.__exit__}
+        """
+
+
+class _HasClose(typing_extensions.Protocol):
+    def close(self) -> object:
+        ...
+
+
+@implementer(_IFileDescriptorReservation)
+@attr.s(auto_attribs=True)
+class _FileDescriptorReservation:
+    """
+    L{_IFileDescriptorReservation} implementation.
+
+    @ivar fileFactory: A factory that will be called to reserve a
+        file descriptor.
+    @type fileFactory: A L{callable} that accepts no arguments and
+        returns an object with a C{close} method.
+    """
+
+    _log: ClassVar[Logger] = Logger()
+
+    _fileFactory: Callable[[], _HasClose]
+    _fileDescriptor: Optional[_HasClose] = attr.ib(init=False, default=None)
+
+    def available(self):
+        """
+        See L{_IFileDescriptorReservation.available}.
+
+        @return: L{True} if the reserved file descriptor is open and
+            can thus be closed to allow a new file to be opened in its
+            place; L{False} if it is not open.
+        """
+        return self._fileDescriptor is not None
+
+    def reserve(self):
+        """
+        See L{_IFileDescriptorReservation.reserve}.
+        """
+        if self._fileDescriptor is None:
+            try:
+                fileDescriptor = self._fileFactory()
+            except OSError as e:
+                if e.errno == EMFILE:
+                    self._log.failure(
+                        "Could not reserve EMFILE recovery file descriptor."
+                    )
+                else:
+                    raise
+            else:
+                self._fileDescriptor = fileDescriptor
+
+    def __enter__(self):
+        """
+        See L{_IFileDescriptorReservation.__enter__}.
+        """
+        if self._fileDescriptor is None:
+            raise RuntimeError("No file reserved.  Have you called my reserve method?")
+        self._fileDescriptor.close()
+        self._fileDescriptor = None
+
+    def __exit__(self, excType, excValue, traceback):
+        """
+        See L{_IFileDescriptorReservation.__exit__}.
+        """
+        try:
+            self.reserve()
+        except Exception:
+            self._log.failure("Could not re-reserve EMFILE recovery file descriptor.")
+
+
+@implementer(_IFileDescriptorReservation)
+class _NullFileDescriptorReservation:
+    """
+    A null implementation of L{_IFileDescriptorReservation}.
+    """
+
+    def available(self):
+        """
+        The reserved file is never available.  See
+        L{_IFileDescriptorReservation.available}.
+
+        @return: L{False}
+        """
+        return False
+
+    def reserve(self):
+        """
+        Do nothing.  See L{_IFileDescriptorReservation.reserve}.
+        """
+
+    def __enter__(self):
+        """
+        Do nothing. See L{_IFileDescriptorReservation.__enter__}
+
+        @return: L{False}
+        """
+
+    def __exit__(self, excType, excValue, traceback):
+        """
+        Do nothing.  See L{_IFileDescriptorReservation.__exit__}.
+
+        @param excType: See L{object.__exit__}
+        @param excValue: See L{object.__exit__}
+        @param traceback: See L{object.__exit__}
+        """
+
+
+# Don't keep a reserve file descriptor for coping with file descriptor
+# exhaustion on Windows.
+
+# WSAEMFILE occurs when a process has run out of memory, not when a
+# specific limit has been reached.  Windows sockets are handles, which
+# differ from UNIX's file descriptors in that they can refer to any
+# "named kernel object", including user interface resources like menu
+# and icons.  The generality of handles results in a much higher limit
+# than UNIX imposes on file descriptors: a single Windows process can
+# allocate up to 16,777,216 handles.  Because they're indexes into a
+# three level table whose upper two layers are allocated from
+# swappable pages, handles compete for heap space with other kernel
+# objects, not with each other.  Closing a given socket handle may not
+# release enough memory to allow the process to make progress.
+#
+# This fundamental difference between file descriptors and handles
+# makes a reserve file descriptor useless on Windows.  Note that other
+# event loops, such as libuv and libevent, also do not special case
+# WSAEMFILE.
+#
+# For an explanation of handles, see the "Object Manager"
+# (pp. 140-175) section of
+#
+# Windows Internals, Part 1: Covering Windows Server 2008 R2 and
+# Windows 7 (6th ed.)
+# Mark E. Russinovich, David A. Solomon, and Alex
+# Ionescu. 2012. Microsoft Press.
+if platformType == "win32":
+    _reservedFD = _NullFileDescriptorReservation()
+else:
+    _reservedFD = _FileDescriptorReservation(lambda: open(os.devnull))  # type: ignore[assignment]
+
+
+# Linux and other UNIX-like operating systems return EMFILE when a
+# process has reached its soft limit of file descriptors. *BSD and
+# Win32 raise (WSA)ENOBUFS when socket limits are reached.  Linux can
+# give ENFILE if the system is out of inodes, or ENOMEM if there is
+# insufficient memory to allocate a new dentry.  ECONNABORTED is
+# documented as possible on all relevant platforms (Linux, Windows,
+# macOS, and the BSDs) but occurs only on the BSDs.  It occurs when a
+# client sends a FIN or RST after the server sends a SYN|ACK but
+# before application code calls accept(2).  On Linux, calling
+# accept(2) on such a listener returns a connection that fails as
+# though the it were terminated after being fully established.  This
+# appears to be an implementation choice (see inet_accept in
+# inet/ipv4/af_inet.c).  On macOS, such a listener is not considered
+# readable, so accept(2) will never be called.  Calling accept(2) on
+# such a listener, however, does not return at all.
+_ACCEPT_ERRORS = (EMFILE, ENOBUFS, ENFILE, ENOMEM, ECONNABORTED)
+
+
+@attr.s(auto_attribs=True)
+class _BuffersLogs:
+    """
+    A context manager that buffers any log events until after its
+    block exits.
+
+    @ivar _namespace: The namespace of the buffered events.
+    @type _namespace: L{str}.
+
+    @ivar _observer: The observer to which buffered log events will be
+        written
+    @type _observer: L{twisted.logger.ILogObserver}.
+    """
+
+    _namespace: str
+    _observer: ILogObserver
+    _logs: List[LogEvent] = attr.ib(default=attr.Factory(list))
+
+    def __enter__(self):
+        """
+        Enter a log buffering context.
+
+        @return: A logger that buffers log events.
+        @rtype: L{Logger}.
+        """
+        return Logger(namespace=self._namespace, observer=self._logs.append)
+
+    def __exit__(self, excValue, excType, traceback):
+        """
+        Exit a log buffering context and log all buffered events to
+        the provided observer.
+
+        @param excType: See L{object.__exit__}
+        @param excValue: See L{object.__exit__}
+        @param traceback: See L{object.__exit__}
+        """
+        for event in self._logs:
+            self._observer(event)
+
+
+def _accept(logger, accepts, listener, reservedFD):
+    """
+    Return a generator that yields client sockets from the provided
+    listening socket until there are none left or an unrecoverable
+    error occurs.
+
+    @param logger: A logger to which C{accept}-related events will be
+        logged.  This should not log to arbitrary observers that might
+        open a file descriptor to avoid claiming the C{EMFILE} file
+        descriptor on UNIX-like systems.
+    @type logger: L{Logger}
+
+    @param accepts: An iterable iterated over to limit the number
+        consecutive C{accept}s.
+    @type accepts: An iterable.
+
+    @param listener: The listening socket.
+    @type listener: L{socket.socket}
+
+    @param reservedFD: A reserved file descriptor that can be used to
+        recover from C{EMFILE} on UNIX-like systems.
+    @type reservedFD: L{_IFileDescriptorReservation}
+
+    @return: A generator that yields C{(socket, addr)} tuples from
+        L{socket.socket.accept}
+    """
+    for _ in accepts:
+        try:
+            client, address = listener.accept()
+        except OSError as e:
+            if e.args[0] in (EWOULDBLOCK, EAGAIN):
+                # No more clients.
+                return
+            elif e.args[0] == EPERM:
+                # Netfilter on Linux may have rejected the
+                # connection, but we get told to try to accept()
+                # anyway.
+                continue
+            elif e.args[0] == EMFILE and reservedFD.available():
+                # Linux and other UNIX-like operating systems return
+                # EMFILE when a process has reached its soft limit of
+                # file descriptors.  The reserved file descriptor is
+                # available, so it can be released to free up a
+                # descriptor for use by listener.accept()'s clients.
+                # Each client socket will be closed until the listener
+                # returns EAGAIN.
+                logger.info(
+                    "EMFILE encountered;" " releasing reserved file descriptor."
+                )
+                # The following block should not run arbitrary code
+                # that might acquire its own file descriptor.
+                with reservedFD:
+                    clientsToClose = _accept(logger, accepts, listener, reservedFD)
+                    for clientToClose, closedAddress in clientsToClose:
+                        clientToClose.close()
+                        logger.info(
+                            "EMFILE recovery:" " Closed socket from {address}",
+                            address=closedAddress,
+                        )
+                    logger.info("Re-reserving EMFILE recovery file descriptor.")
+                return
+            elif e.args[0] in _ACCEPT_ERRORS:
+                logger.info(
+                    "Could not accept new connection ({acceptError})",
+                    acceptError=errorcode[e.args[0]],
+                )
+                return
+            else:
+                raise
+        else:
+            yield client, address
+
+
+@implementer(IListeningPort)
+class Port(base.BasePort, _SocketCloser):
+    """
+    A TCP server port, listening for connections.
+
+    When a connection is accepted, this will call a factory's buildProtocol
+    with the incoming address as an argument, according to the specification
+    described in L{twisted.internet.interfaces.IProtocolFactory}.
+
+    If you wish to change the sort of transport that will be used, the
+    C{transport} attribute will be called with the signature expected for
+    C{Server.__init__}, so it can be replaced.
+
+    @ivar deferred: a deferred created when L{stopListening} is called, and
+        that will fire when connection is lost. This is not to be used it
+        directly: prefer the deferred returned by L{stopListening} instead.
+    @type deferred: L{defer.Deferred}
+
+    @ivar disconnecting: flag indicating that the L{stopListening} method has
+        been called and that no connections should be accepted anymore.
+    @type disconnecting: C{bool}
+
+    @ivar connected: flag set once the listen has successfully been called on
+        the socket.
+    @type connected: C{bool}
+
+    @ivar _type: A string describing the connections which will be created by
+        this port.  Normally this is C{"TCP"}, since this is a TCP port, but
+        when the TLS implementation re-uses this class it overrides the value
+        with C{"TLS"}.  Only used for logging.
+
+    @ivar _preexistingSocket: If not L{None}, a L{socket.socket} instance which
+        was created and initialized outside of the reactor and will be used to
+        listen for connections (instead of a new socket being created by this
+        L{Port}).
+    """
+
+    socketType = socket.SOCK_STREAM
+
+    transport = Server
+    sessionno = 0
+    interface = ""
+    backlog = 50
+
+    _type = "TCP"
+
+    # Actual port number being listened on, only set to a non-None
+    # value when we are actually listening.
+    _realPortNumber: Optional[int] = None
+
+    # An externally initialized socket that we will use, rather than creating
+    # our own.
+    _preexistingSocket = None
+
+    addressFamily = socket.AF_INET
+    _addressType = address.IPv4Address
+    _logger = Logger()
+
+    def __init__(self, port, factory, backlog=50, interface="", reactor=None):
+        """Initialize with a numeric port to listen on."""
+        base.BasePort.__init__(self, reactor=reactor)
+        self.port = port
+        self.factory = factory
+        self.backlog = backlog
+        if abstract.isIPv6Address(interface):
+            self.addressFamily = socket.AF_INET6
+            self._addressType = address.IPv6Address
+        self.interface = interface
+
+    @classmethod
+    def _fromListeningDescriptor(cls, reactor, fd, addressFamily, factory):
+        """
+        Create a new L{Port} based on an existing listening I{SOCK_STREAM}
+        socket.
+
+        Arguments are the same as to L{Port.__init__}, except where noted.
+
+        @param fd: An integer file descriptor associated with a listening
+            socket.  The socket must be in non-blocking mode.  Any additional
+            attributes desired, such as I{FD_CLOEXEC}, must also be set already.
+
+        @param addressFamily: The address family (sometimes called I{domain}) of
+            the existing socket.  For example, L{socket.AF_INET}.
+
+        @return: A new instance of C{cls} wrapping the socket given by C{fd}.
+        """
+        port = socket.fromfd(fd, addressFamily, cls.socketType)
+        interface = _getsockname(port)[0]
+        self = cls(None, factory, None, interface, reactor)
+        self._preexistingSocket = port
+        return self
+
+    def __repr__(self) -> str:
+        if self._realPortNumber is not None:
+            return "<{} of {} on {}>".format(
+                self.__class__,
+                self.factory.__class__,
+                self._realPortNumber,
+            )
+        else:
+            return "<{} of {} (not listening)>".format(
+                self.__class__,
+                self.factory.__class__,
+            )
+
+    def createInternetSocket(self):
+        s = base.BasePort.createInternetSocket(self)
+        if platformType == "posix" and sys.platform != "cygwin":
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        return s
+
+    def startListening(self):
+        """Create and bind my socket, and begin listening on it.
+
+        This is called on unserialization, and must be called after creating a
+        server to begin listening on the specified port.
+        """
+        _reservedFD.reserve()
+        if self._preexistingSocket is None:
+            # Create a new socket and make it listen
+            try:
+                skt = self.createInternetSocket()
+                if self.addressFamily == socket.AF_INET6:
+                    addr = _resolveIPv6(self.interface, self.port)
+                else:
+                    addr = (self.interface, self.port)
+                skt.bind(addr)
+            except OSError as le:
+                raise CannotListenError(self.interface, self.port, le)
+            skt.listen(self.backlog)
+        else:
+            # Re-use the externally specified socket
+            skt = self._preexistingSocket
+            self._preexistingSocket = None
+            # Avoid shutting it down at the end.
+            self._shouldShutdown = False
+
+        # Make sure that if we listened on port 0, we update that to
+        # reflect what the OS actually assigned us.
+        self._realPortNumber = skt.getsockname()[1]
+
+        log.msg(
+            "%s starting on %s"
+            % (self._getLogPrefix(self.factory), self._realPortNumber)
+        )
+
+        # The order of the next 5 lines is kind of bizarre.  If no one
+        # can explain it, perhaps we should re-arrange them.
+        self.factory.doStart()
+        self.connected = True
+        self.socket = skt
+        self.fileno = self.socket.fileno
+        self.numberAccepts = 100
+
+        self.startReading()
+
+    def _buildAddr(self, address):
+        return self._addressType("TCP", *address)
+
+    def doRead(self):
+        """
+        Called when my socket is ready for reading.
+
+        This accepts a connection and calls self.protocol() to handle the
+        wire-level protocol.
+        """
+        try:
+            if platformType == "posix":
+                numAccepts = self.numberAccepts
+            else:
+                # win32 event loop breaks if we do more than one accept()
+                # in an iteration of the event loop.
+                numAccepts = 1
+
+            with _BuffersLogs(
+                self._logger.namespace, self._logger.observer
+            ) as bufferingLogger:
+                accepted = 0
+                clients = _accept(
+                    bufferingLogger, range(numAccepts), self.socket, _reservedFD
+                )
+
+                for accepted, (skt, addr) in enumerate(clients, 1):
+                    fdesc._setCloseOnExec(skt.fileno())
+
+                    if len(addr) == 4:
+                        # IPv6, make sure we get the scopeID if it
+                        # exists
+                        host = socket.getnameinfo(
+                            addr, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV
+                        )
+                        addr = tuple([host[0]] + list(addr[1:]))
+
+                    protocol = self.factory.buildProtocol(self._buildAddr(addr))
+                    if protocol is None:
+                        skt.close()
+                        continue
+                    s = self.sessionno
+                    self.sessionno = s + 1
+                    transport = self.transport(
+                        skt, protocol, addr, self, s, self.reactor
+                    )
+                    protocol.makeConnection(transport)
+
+            # Scale our synchronous accept loop according to traffic
+            # Reaching our limit on consecutive accept calls indicates
+            # there might be still more clients to serve the next time
+            # the reactor calls us.  Prepare to accept some more.
+            if accepted == self.numberAccepts:
+                self.numberAccepts += 20
+            # Otherwise, don't attempt to accept any more clients than
+            # we just accepted or any less than 1.
+            else:
+                self.numberAccepts = max(1, accepted)
+        except BaseException:
+            # Note that in TLS mode, this will possibly catch SSL.Errors
+            # raised by self.socket.accept()
+            #
+            # There is no "except SSL.Error:" above because SSL may be
+            # None if there is no SSL support.  In any case, all the
+            # "except SSL.Error:" suite would probably do is log.deferr()
+            # and return, so handling it here works just as well.
+            log.deferr()
+
+    def loseConnection(self, connDone=failure.Failure(main.CONNECTION_DONE)):
+        """
+        Stop accepting connections on this port.
+
+        This will shut down the socket and call self.connectionLost().  It
+        returns a deferred which will fire successfully when the port is
+        actually closed, or with a failure if an error occurs shutting down.
+        """
+        self.disconnecting = True
+        self.stopReading()
+        if self.connected:
+            self.deferred = deferLater(self.reactor, 0, self.connectionLost, connDone)
+            return self.deferred
+
+    stopListening = loseConnection
+
+    def _logConnectionLostMsg(self):
+        """
+        Log message for closing port
+        """
+        log.msg(f"({self._type} Port {self._realPortNumber} Closed)")
+
+    def connectionLost(self, reason):
+        """
+        Cleans up the socket.
+        """
+        self._logConnectionLostMsg()
+        self._realPortNumber = None
+
+        base.BasePort.connectionLost(self, reason)
+        self.connected = False
+        self._closeSocket(True)
+        del self.socket
+        del self.fileno
+
+        try:
+            self.factory.doStop()
+        finally:
+            self.disconnecting = False
+
+    def logPrefix(self):
+        """Returns the name of my class, to prefix log entries with."""
+        return reflect.qual(self.factory.__class__)
+
+    def getHost(self):
+        """
+        Return an L{IPv4Address} or L{IPv6Address} indicating the listening
+        address of this port.
+        """
+        addr = _getsockname(self.socket)
+        return self._addressType("TCP", *addr)
+
+
+class Connector(base.BaseConnector):
+    """
+    A L{Connector} provides of L{twisted.internet.interfaces.IConnector} for
+    all POSIX-style reactors.
+
+    @ivar _addressType: the type returned by L{Connector.getDestination}.
+        Either L{IPv4Address} or L{IPv6Address}, depending on the type of
+        address.
+    @type _addressType: C{type}
+    """
+
+    _addressType = address.IPv4Address
+
+    def __init__(self, host, port, factory, timeout, bindAddress, reactor=None):
+        if isinstance(port, str):
+            try:
+                port = socket.getservbyname(port, "tcp")
+            except OSError as e:
+                raise error.ServiceNameUnknownError(string=f"{e} ({port!r})")
+        self.host, self.port = host, port
+        if abstract.isIPv6Address(host):
+            self._addressType = address.IPv6Address
+        self.bindAddress = bindAddress
+        base.BaseConnector.__init__(self, factory, timeout, reactor)
+
+    def _makeTransport(self):
+        """
+        Create a L{Client} bound to this L{Connector}.
+
+        @return: a new L{Client}
+        @rtype: L{Client}
+        """
+        return Client(self.host, self.port, self.bindAddress, self, self.reactor)
+
+    def getDestination(self):
+        """
+        @see: L{twisted.internet.interfaces.IConnector.getDestination}.
+        """
+        return self._addressType("TCP", self.host, self.port)
diff --git a/contrib/python/Twisted/py3/twisted/internet/testing.py b/contrib/python/Twisted/py3/twisted/internet/testing.py
new file mode 100644
index 00000000000..2c372495844
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/testing.py
@@ -0,0 +1,969 @@
+# -*- test-case-name: twisted.internet.test.test_testing -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Assorted functionality which is commonly useful when writing unit tests.
+"""
+from __future__ import annotations
+
+from io import BytesIO
+from socket import AF_INET, AF_INET6
+from typing import Callable, Iterator, Sequence, overload
+
+from zope.interface import implementedBy, implementer
+from zope.interface.verify import verifyClass
+
+from typing_extensions import ParamSpec, Self
+
+from twisted.internet import address, error, protocol, task
+from twisted.internet.abstract import _dataMustBeBytes, isIPv6Address
+from twisted.internet.address import IPv4Address, IPv6Address, UNIXAddress
+from twisted.internet.defer import Deferred
+from twisted.internet.error import UnsupportedAddressFamily
+from twisted.internet.interfaces import (
+    IConnector,
+    IConsumer,
+    IListeningPort,
+    IProtocol,
+    IPushProducer,
+    IReactorCore,
+    IReactorFDSet,
+    IReactorSocket,
+    IReactorSSL,
+    IReactorTCP,
+    IReactorUNIX,
+    ITransport,
+)
+from twisted.internet.task import Clock
+from twisted.logger import ILogObserver, LogEvent, LogPublisher
+from twisted.protocols import basic
+from twisted.python import failure
+from twisted.trial.unittest import TestCase
+
+__all__ = [
+    "AccumulatingProtocol",
+    "LineSendingProtocol",
+    "FakeDatagramTransport",
+    "StringTransport",
+    "StringTransportWithDisconnection",
+    "StringIOWithoutClosing",
+    "_FakeConnector",
+    "_FakePort",
+    "MemoryReactor",
+    "MemoryReactorClock",
+    "RaisingMemoryReactor",
+    "NonStreamingProducer",
+    "waitUntilAllDisconnected",
+    "EventLoggingObserver",
+]
+
+_P = ParamSpec("_P")
+
+
+class AccumulatingProtocol(protocol.Protocol):
+    """
+    L{AccumulatingProtocol} is an L{IProtocol} implementation which collects
+    the data delivered to it and can fire a Deferred when it is connected or
+    disconnected.
+
+    @ivar made: A flag indicating whether C{connectionMade} has been called.
+    @ivar data: Bytes giving all the data passed to C{dataReceived}.
+    @ivar closed: A flag indicated whether C{connectionLost} has been called.
+    @ivar closedReason: The value of the I{reason} parameter passed to
+        C{connectionLost}.
+    @ivar closedDeferred: If set to a L{Deferred}, this will be fired when
+        C{connectionLost} is called.
+    """
+
+    made = closed = 0
+    closedReason = None
+
+    closedDeferred = None
+
+    data = b""
+
+    factory = None
+
+    def connectionMade(self):
+        self.made = 1
+        if self.factory is not None and self.factory.protocolConnectionMade is not None:
+            d = self.factory.protocolConnectionMade
+            self.factory.protocolConnectionMade = None
+            d.callback(self)
+
+    def dataReceived(self, data):
+        self.data += data
+
+    def connectionLost(self, reason):
+        self.closed = 1
+        self.closedReason = reason
+        if self.closedDeferred is not None:
+            d, self.closedDeferred = self.closedDeferred, None
+            d.callback(None)
+
+
+class LineSendingProtocol(basic.LineReceiver):
+    lostConn = False
+
+    def __init__(self, lines, start=True):
+        self.lines = lines[:]
+        self.response = []
+        self.start = start
+
+    def connectionMade(self):
+        if self.start:
+            for line in self.lines:
+                self.sendLine(line)
+
+    def lineReceived(self, line):
+        if not self.start:
+            for line in self.lines:
+                self.sendLine(line)
+            self.lines = []
+        self.response.append(line)
+
+    def connectionLost(self, reason):
+        self.lostConn = True
+
+
+class FakeDatagramTransport:
+    noAddr = object()
+
+    def __init__(self):
+        self.written = []
+
+    def write(self, packet, addr=noAddr):
+        self.written.append((packet, addr))
+
+
+@implementer(ITransport, IConsumer, IPushProducer)
+class StringTransport:
+    """
+    A transport implementation which buffers data in memory and keeps track of
+    its other state without providing any behavior.
+
+    L{StringTransport} has a number of attributes which are not part of any of
+    the interfaces it claims to implement.  These attributes are provided for
+    testing purposes.  Implementation code should not use any of these
+    attributes; they are not provided by other transports.
+
+    @ivar disconnecting: A C{bool} which is C{False} until L{loseConnection} is
+        called, then C{True}.
+
+    @ivar disconnected: A C{bool} which is C{False} until L{abortConnection} is
+        called, then C{True}.
+
+    @ivar producer: If a producer is currently registered, C{producer} is a
+        reference to it.  Otherwise, L{None}.
+
+    @ivar streaming: If a producer is currently registered, C{streaming} refers
+        to the value of the second parameter passed to C{registerProducer}.
+
+    @ivar hostAddr: L{None} or an object which will be returned as the host
+        address of this transport.  If L{None}, a nasty tuple will be returned
+        instead.
+
+    @ivar peerAddr: L{None} or an object which will be returned as the peer
+        address of this transport.  If L{None}, a nasty tuple will be returned
+        instead.
+
+    @ivar producerState: The state of this L{StringTransport} in its capacity
+        as an L{IPushProducer}.  One of C{'producing'}, C{'paused'}, or
+        C{'stopped'}.
+
+    @ivar io: A L{io.BytesIO} which holds the data which has been written to
+        this transport since the last call to L{clear}.  Use L{value} instead
+        of accessing this directly.
+
+    @ivar _lenient: By default L{StringTransport} enforces that
+        L{resumeProducing} is not called after the connection is lost. This is
+        to ensure that any code that does call L{resumeProducing} after the
+        connection is lost is not blindly expecting L{resumeProducing} to have
+        any impact.
+
+        However, if your test case is calling L{resumeProducing} after
+        connection close on purpose, and you know it won't block expecting
+        further data to show up, this flag may safely be set to L{True}.
+
+        Defaults to L{False}.
+    @type lenient: L{bool}
+    """
+
+    disconnecting = False
+    disconnected = False
+
+    producer = None
+    streaming = None
+
+    hostAddr = None
+    peerAddr = None
+
+    producerState = "producing"
+
+    def __init__(self, hostAddress=None, peerAddress=None, lenient=False):
+        self.clear()
+        if hostAddress is not None:
+            self.hostAddr = hostAddress
+        if peerAddress is not None:
+            self.peerAddr = peerAddress
+        self.connected = True
+        self._lenient = lenient
+
+    def clear(self):
+        """
+        Discard all data written to this transport so far.
+
+        This is not a transport method.  It is intended for tests.  Do not use
+        it in implementation code.
+        """
+        self.io = BytesIO()
+
+    def value(self):
+        """
+        Retrieve all data which has been buffered by this transport.
+
+        This is not a transport method.  It is intended for tests.  Do not use
+        it in implementation code.
+
+        @return: A C{bytes} giving all data written to this transport since the
+            last call to L{clear}.
+        @rtype: C{bytes}
+        """
+        return self.io.getvalue()
+
+    # ITransport
+    def write(self, data):
+        _dataMustBeBytes(data)
+        self.io.write(data)
+
+    def writeSequence(self, data):
+        self.io.write(b"".join(data))
+
+    def loseConnection(self):
+        """
+        Close the connection. Does nothing besides toggle the C{disconnecting}
+        instance variable to C{True}.
+        """
+        self.disconnecting = True
+
+    def abortConnection(self):
+        """
+        Abort the connection. Same as C{loseConnection}, but also toggles the
+        C{aborted} instance variable to C{True}.
+        """
+        self.disconnected = True
+        self.loseConnection()
+
+    def getPeer(self):
+        if self.peerAddr is None:
+            return address.IPv4Address("TCP", "192.168.1.1", 54321)
+        return self.peerAddr
+
+    def getHost(self):
+        if self.hostAddr is None:
+            return address.IPv4Address("TCP", "10.0.0.1", 12345)
+        return self.hostAddr
+
+    # IConsumer
+    def registerProducer(self, producer, streaming):
+        if self.producer is not None:
+            raise RuntimeError("Cannot register two producers")
+        self.producer = producer
+        self.streaming = streaming
+
+    def unregisterProducer(self):
+        if self.producer is None:
+            raise RuntimeError("Cannot unregister a producer unless one is registered")
+        self.producer = None
+        self.streaming = None
+
+    # IPushProducer
+    def _checkState(self):
+        if self.disconnecting and not self._lenient:
+            raise RuntimeError("Cannot resume producing after loseConnection")
+        if self.producerState == "stopped":
+            raise RuntimeError("Cannot resume a stopped producer")
+
+    def pauseProducing(self):
+        self._checkState()
+        self.producerState = "paused"
+
+    def stopProducing(self):
+        self.producerState = "stopped"
+
+    def resumeProducing(self):
+        self._checkState()
+        self.producerState = "producing"
+
+
+class StringTransportWithDisconnection(StringTransport):
+    """
+    A L{StringTransport} which on disconnection will trigger the connection
+    lost on the attached protocol.
+    """
+
+    protocol: IProtocol
+
+    def loseConnection(self):
+        if self.connected:
+            self.connected = False
+            self.protocol.connectionLost(failure.Failure(error.ConnectionDone("Bye.")))
+
+
+class StringIOWithoutClosing(BytesIO):
+    """
+    A BytesIO that can't be closed.
+    """
+
+    def close(self):
+        """
+        Do nothing.
+        """
+
+
+@implementer(IListeningPort)
+class _FakePort:
+    """
+    A fake L{IListeningPort} to be used in tests.
+
+    @ivar _hostAddress: The L{IAddress} this L{IListeningPort} is pretending
+        to be listening on.
+    """
+
+    def __init__(self, hostAddress):
+        """
+        @param hostAddress: An L{IAddress} this L{IListeningPort} should
+            pretend to be listening on.
+        """
+        self._hostAddress = hostAddress
+
+    def startListening(self):
+        """
+        Fake L{IListeningPort.startListening} that doesn't do anything.
+        """
+
+    def stopListening(self):
+        """
+        Fake L{IListeningPort.stopListening} that doesn't do anything.
+        """
+
+    def getHost(self):
+        """
+        Fake L{IListeningPort.getHost} that returns our L{IAddress}.
+        """
+        return self._hostAddress
+
+
+@implementer(IConnector)
+class _FakeConnector:
+    """
+    A fake L{IConnector} that allows us to inspect if it has been told to stop
+    connecting.
+
+    @ivar stoppedConnecting: has this connector's
+        L{_FakeConnector.stopConnecting} method been invoked yet?
+
+    @ivar _address: An L{IAddress} provider that represents our destination.
+    """
+
+    _disconnected = False
+    stoppedConnecting = False
+
+    def __init__(self, address):
+        """
+        @param address: An L{IAddress} provider that represents this
+            connector's destination.
+        """
+        self._address = address
+
+    def stopConnecting(self):
+        """
+        Implement L{IConnector.stopConnecting} and set
+        L{_FakeConnector.stoppedConnecting} to C{True}
+        """
+        self.stoppedConnecting = True
+
+    def disconnect(self):
+        """
+        Implement L{IConnector.disconnect} as a no-op.
+        """
+        self._disconnected = True
+
+    def connect(self):
+        """
+        Implement L{IConnector.connect} as a no-op.
+        """
+
+    def getDestination(self):
+        """
+        Implement L{IConnector.getDestination} to return the C{address} passed
+        to C{__init__}.
+        """
+        return self._address
+
+
+@implementer(
+    IReactorCore, IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket, IReactorFDSet
+)
+class MemoryReactor:
+    """
+    A fake reactor to be used in tests.  This reactor doesn't actually do
+    much that's useful yet.  It accepts TCP connection setup attempts, but
+    they will never succeed.
+
+    @ivar hasInstalled: Keeps track of whether this reactor has been installed.
+    @type hasInstalled: L{bool}
+
+    @ivar running: Keeps track of whether this reactor is running.
+    @type running: L{bool}
+
+    @ivar hasStopped: Keeps track of whether this reactor has been stopped.
+    @type hasStopped: L{bool}
+
+    @ivar hasCrashed: Keeps track of whether this reactor has crashed.
+    @type hasCrashed: L{bool}
+
+    @ivar whenRunningHooks: Keeps track of hooks registered with
+        C{callWhenRunning}.
+    @type whenRunningHooks: L{list}
+
+    @ivar triggers: Keeps track of hooks registered with
+        C{addSystemEventTrigger}.
+    @type triggers: L{dict}
+
+    @ivar tcpClients: Keeps track of connection attempts (ie, calls to
+        C{connectTCP}).
+    @type tcpClients: L{list}
+
+    @ivar tcpServers: Keeps track of server listen attempts (ie, calls to
+        C{listenTCP}).
+    @type tcpServers: L{list}
+
+    @ivar sslClients: Keeps track of connection attempts (ie, calls to
+        C{connectSSL}).
+    @type sslClients: L{list}
+
+    @ivar sslServers: Keeps track of server listen attempts (ie, calls to
+        C{listenSSL}).
+    @type sslServers: L{list}
+
+    @ivar unixClients: Keeps track of connection attempts (ie, calls to
+        C{connectUNIX}).
+    @type unixClients: L{list}
+
+    @ivar unixServers: Keeps track of server listen attempts (ie, calls to
+        C{listenUNIX}).
+    @type unixServers: L{list}
+
+    @ivar adoptedPorts: Keeps track of server listen attempts (ie, calls to
+        C{adoptStreamPort}).
+
+    @ivar adoptedStreamConnections: Keeps track of stream-oriented
+        connections added using C{adoptStreamConnection}.
+    """
+
+    def __init__(self):
+        """
+        Initialize the tracking lists.
+        """
+        self.hasInstalled = False
+
+        self.running = False
+        self.hasRun = True
+        self.hasStopped = True
+        self.hasCrashed = True
+
+        self.whenRunningHooks = []
+        self.triggers = {}
+
+        self.tcpClients = []
+        self.tcpServers = []
+        self.sslClients = []
+        self.sslServers = []
+        self.unixClients = []
+        self.unixServers = []
+        self.adoptedPorts = []
+        self.adoptedStreamConnections = []
+        self.connectors = []
+
+        self.readers = set()
+        self.writers = set()
+
+    def install(self):
+        """
+        Fake install callable to emulate reactor module installation.
+        """
+        self.hasInstalled = True
+
+    def resolve(self, name, timeout=10):
+        """
+        Not implemented; raises L{NotImplementedError}.
+        """
+        raise NotImplementedError()
+
+    def run(self):
+        """
+        Fake L{IReactorCore.run}.
+        Sets C{self.running} to L{True}, runs all of the hooks passed to
+        C{self.callWhenRunning}, then calls C{self.stop} to simulate a request
+        to stop the reactor.
+        Sets C{self.hasRun} to L{True}.
+        """
+        assert self.running is False
+        self.running = True
+        self.hasRun = True
+
+        for f, args, kwargs in self.whenRunningHooks:
+            f(*args, **kwargs)
+
+        self.stop()
+        # That we stopped means we can return, phew.
+
+    def stop(self):
+        """
+        Fake L{IReactorCore.run}.
+        Sets C{self.running} to L{False}.
+        Sets C{self.hasStopped} to L{True}.
+        """
+        self.running = False
+        self.hasStopped = True
+
+    def crash(self):
+        """
+        Fake L{IReactorCore.crash}.
+        Sets C{self.running} to L{None}, because that feels crashy.
+        Sets C{self.hasCrashed} to L{True}.
+        """
+        self.running = None
+        self.hasCrashed = True
+
+    def iterate(self, delay=0):
+        """
+        Not implemented; raises L{NotImplementedError}.
+        """
+        raise NotImplementedError()
+
+    def fireSystemEvent(self, eventType):
+        """
+        Not implemented; raises L{NotImplementedError}.
+        """
+        raise NotImplementedError()
+
+    def addSystemEventTrigger(
+        self,
+        phase: str,
+        eventType: str,
+        callable: Callable[_P, object],
+        *args: _P.args,
+        **kw: _P.kwargs,
+    ) -> None:
+        """
+        Fake L{IReactorCore.run}.
+        Keep track of trigger by appending it to
+        self.triggers[phase][eventType].
+        """
+        phaseTriggers = self.triggers.setdefault(phase, {})
+        eventTypeTriggers = phaseTriggers.setdefault(eventType, [])
+        eventTypeTriggers.append((callable, args, kw))
+
+    def removeSystemEventTrigger(self, triggerID):
+        """
+        Not implemented; raises L{NotImplementedError}.
+        """
+        raise NotImplementedError()
+
+    def callWhenRunning(
+        self, callable: Callable[_P, object], *args: _P.args, **kw: _P.kwargs
+    ) -> None:
+        """
+        Fake L{IReactorCore.callWhenRunning}.
+        Keeps a list of invocations to make in C{self.whenRunningHooks}.
+        """
+        self.whenRunningHooks.append((callable, args, kw))
+
+    def adoptStreamPort(self, fileno, addressFamily, factory):
+        """
+        Fake L{IReactorSocket.adoptStreamPort}, that logs the call and returns
+        an L{IListeningPort}.
+        """
+        if addressFamily == AF_INET:
+            addr = IPv4Address("TCP", "0.0.0.0", 1234)
+        elif addressFamily == AF_INET6:
+            addr = IPv6Address("TCP", "::", 1234)
+        else:
+            raise UnsupportedAddressFamily()
+
+        self.adoptedPorts.append((fileno, addressFamily, factory))
+        return _FakePort(addr)
+
+    def adoptStreamConnection(self, fileDescriptor, addressFamily, factory):
+        """
+        Record the given stream connection in C{adoptedStreamConnections}.
+
+        @see:
+            L{twisted.internet.interfaces.IReactorSocket.adoptStreamConnection}
+        """
+        self.adoptedStreamConnections.append((fileDescriptor, addressFamily, factory))
+
+    def adoptDatagramPort(self, fileno, addressFamily, protocol, maxPacketSize=8192):
+        """
+        Fake L{IReactorSocket.adoptDatagramPort}, that logs the call and
+        returns a fake L{IListeningPort}.
+
+        @see: L{twisted.internet.interfaces.IReactorSocket.adoptDatagramPort}
+        """
+        if addressFamily == AF_INET:
+            addr = IPv4Address("UDP", "0.0.0.0", 1234)
+        elif addressFamily == AF_INET6:
+            addr = IPv6Address("UDP", "::", 1234)
+        else:
+            raise UnsupportedAddressFamily()
+
+        self.adoptedPorts.append((fileno, addressFamily, protocol, maxPacketSize))
+        return _FakePort(addr)
+
+    def listenTCP(self, port, factory, backlog=50, interface=""):
+        """
+        Fake L{IReactorTCP.listenTCP}, that logs the call and
+        returns an L{IListeningPort}.
+        """
+        self.tcpServers.append((port, factory, backlog, interface))
+        if isIPv6Address(interface):
+            address = IPv6Address("TCP", interface, port)
+        else:
+            address = IPv4Address("TCP", "0.0.0.0", port)
+        return _FakePort(address)
+
+    def connectTCP(self, host, port, factory, timeout=30, bindAddress=None):
+        """
+        Fake L{IReactorTCP.connectTCP}, that logs the call and
+        returns an L{IConnector}.
+        """
+        self.tcpClients.append((host, port, factory, timeout, bindAddress))
+        if isIPv6Address(host):
+            conn = _FakeConnector(IPv6Address("TCP", host, port))
+        else:
+            conn = _FakeConnector(IPv4Address("TCP", host, port))
+        factory.startedConnecting(conn)
+        self.connectors.append(conn)
+        return conn
+
+    def listenSSL(self, port, factory, contextFactory, backlog=50, interface=""):
+        """
+        Fake L{IReactorSSL.listenSSL}, that logs the call and
+        returns an L{IListeningPort}.
+        """
+        self.sslServers.append((port, factory, contextFactory, backlog, interface))
+        return _FakePort(IPv4Address("TCP", "0.0.0.0", port))
+
+    def connectSSL(
+        self, host, port, factory, contextFactory, timeout=30, bindAddress=None
+    ):
+        """
+        Fake L{IReactorSSL.connectSSL}, that logs the call and returns an
+        L{IConnector}.
+        """
+        self.sslClients.append(
+            (host, port, factory, contextFactory, timeout, bindAddress)
+        )
+        conn = _FakeConnector(IPv4Address("TCP", host, port))
+        factory.startedConnecting(conn)
+        self.connectors.append(conn)
+        return conn
+
+    def listenUNIX(self, address, factory, backlog=50, mode=0o666, wantPID=0):
+        """
+        Fake L{IReactorUNIX.listenUNIX}, that logs the call and returns an
+        L{IListeningPort}.
+        """
+        self.unixServers.append((address, factory, backlog, mode, wantPID))
+        return _FakePort(UNIXAddress(address))
+
+    def connectUNIX(self, address, factory, timeout=30, checkPID=0):
+        """
+        Fake L{IReactorUNIX.connectUNIX}, that logs the call and returns an
+        L{IConnector}.
+        """
+        self.unixClients.append((address, factory, timeout, checkPID))
+        conn = _FakeConnector(UNIXAddress(address))
+        factory.startedConnecting(conn)
+        self.connectors.append(conn)
+        return conn
+
+    def addReader(self, reader):
+        """
+        Fake L{IReactorFDSet.addReader} which adds the reader to a local set.
+        """
+        self.readers.add(reader)
+
+    def removeReader(self, reader):
+        """
+        Fake L{IReactorFDSet.removeReader} which removes the reader from a
+        local set.
+        """
+        self.readers.discard(reader)
+
+    def addWriter(self, writer):
+        """
+        Fake L{IReactorFDSet.addWriter} which adds the writer to a local set.
+        """
+        self.writers.add(writer)
+
+    def removeWriter(self, writer):
+        """
+        Fake L{IReactorFDSet.removeWriter} which removes the writer from a
+        local set.
+        """
+        self.writers.discard(writer)
+
+    def getReaders(self):
+        """
+        Fake L{IReactorFDSet.getReaders} which returns a list of readers from
+        the local set.
+        """
+        return list(self.readers)
+
+    def getWriters(self):
+        """
+        Fake L{IReactorFDSet.getWriters} which returns a list of writers from
+        the local set.
+        """
+        return list(self.writers)
+
+    def removeAll(self):
+        """
+        Fake L{IReactorFDSet.removeAll} which removed all readers and writers
+        from the local sets.
+        """
+        self.readers.clear()
+        self.writers.clear()
+
+
+for iface in implementedBy(MemoryReactor):
+    verifyClass(iface, MemoryReactor)
+
+
+class MemoryReactorClock(MemoryReactor, Clock):
+    def __init__(self):
+        MemoryReactor.__init__(self)
+        Clock.__init__(self)
+
+
+@implementer(IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket)
+class RaisingMemoryReactor:
+    """
+    A fake reactor to be used in tests.  It accepts TCP connection setup
+    attempts, but they will fail.
+
+    @ivar _listenException: An instance of an L{Exception}
+    @ivar _connectException: An instance of an L{Exception}
+    """
+
+    def __init__(self, listenException=None, connectException=None):
+        """
+        @param listenException: An instance of an L{Exception} to raise
+            when any C{listen} method is called.
+
+        @param connectException: An instance of an L{Exception} to raise
+            when any C{connect} method is called.
+        """
+        self._listenException = listenException
+        self._connectException = connectException
+
+    def adoptStreamPort(self, fileno, addressFamily, factory):
+        """
+        Fake L{IReactorSocket.adoptStreamPort}, that raises
+        L{_listenException}.
+        """
+        raise self._listenException
+
+    def listenTCP(self, port, factory, backlog=50, interface=""):
+        """
+        Fake L{IReactorTCP.listenTCP}, that raises L{_listenException}.
+        """
+        raise self._listenException
+
+    def connectTCP(self, host, port, factory, timeout=30, bindAddress=None):
+        """
+        Fake L{IReactorTCP.connectTCP}, that raises L{_connectException}.
+        """
+        raise self._connectException
+
+    def listenSSL(self, port, factory, contextFactory, backlog=50, interface=""):
+        """
+        Fake L{IReactorSSL.listenSSL}, that raises L{_listenException}.
+        """
+        raise self._listenException
+
+    def connectSSL(
+        self, host, port, factory, contextFactory, timeout=30, bindAddress=None
+    ):
+        """
+        Fake L{IReactorSSL.connectSSL}, that raises L{_connectException}.
+        """
+        raise self._connectException
+
+    def listenUNIX(self, address, factory, backlog=50, mode=0o666, wantPID=0):
+        """
+        Fake L{IReactorUNIX.listenUNIX}, that raises L{_listenException}.
+        """
+        raise self._listenException
+
+    def connectUNIX(self, address, factory, timeout=30, checkPID=0):
+        """
+        Fake L{IReactorUNIX.connectUNIX}, that raises L{_connectException}.
+        """
+        raise self._connectException
+
+    def adoptDatagramPort(self, fileDescriptor, addressFamily, protocol, maxPacketSize):
+        """
+        Fake L{IReactorSocket.adoptDatagramPort}, that raises
+        L{_connectException}.
+        """
+        raise self._connectException
+
+    def adoptStreamConnection(self, fileDescriptor, addressFamily, factory):
+        """
+        Fake L{IReactorSocket.adoptStreamConnection}, that raises
+        L{_connectException}.
+        """
+        raise self._connectException
+
+
+class NonStreamingProducer:
+    """
+    A pull producer which writes 10 times only.
+    """
+
+    counter = 0
+    stopped = False
+
+    def __init__(self, consumer):
+        self.consumer = consumer
+        self.result = Deferred()
+
+    def resumeProducing(self):
+        """
+        Write the counter value once.
+        """
+        if self.consumer is None or self.counter >= 10:
+            raise RuntimeError("BUG: resume after unregister/stop.")
+        else:
+            self.consumer.write(b"%d" % (self.counter,))
+            self.counter += 1
+            if self.counter == 10:
+                self.consumer.unregisterProducer()
+                self._done()
+
+    def pauseProducing(self):
+        """
+        An implementation of C{IPushProducer.pauseProducing}. This should never
+        be called on a pull producer, so this just raises an error.
+        """
+        raise RuntimeError("BUG: pause should never be called.")
+
+    def _done(self):
+        """
+        Fire a L{Deferred} so that users can wait for this to complete.
+        """
+        self.consumer = None
+        d = self.result
+        del self.result
+        d.callback(None)
+
+    def stopProducing(self):
+        """
+        Stop all production.
+        """
+        self.stopped = True
+        self._done()
+
+
+def waitUntilAllDisconnected(reactor, protocols):
+    """
+    Take a list of disconnecting protocols, callback a L{Deferred} when they're
+    all done.
+
+    This is a hack to make some older tests less flaky, as
+    L{ITransport.loseConnection} is not atomic on all reactors (for example,
+    the CoreFoundation, which sometimes takes a reactor turn for CFSocket to
+    realise). New tests should either not use real sockets in testing, or take
+    the advice in
+    I{https://jml.io/pages/how-to-disconnect-in-twisted-really.html} to heart.
+
+    @param reactor: The reactor to schedule the checks on.
+    @type reactor: L{IReactorTime}
+
+    @param protocols: The protocols to wait for disconnecting.
+    @type protocols: A L{list} of L{IProtocol}s.
+    """
+    lc = None
+
+    def _check():
+        if True not in [x.transport.connected for x in protocols]:
+            lc.stop()
+
+    lc = task.LoopingCall(_check)
+    lc.clock = reactor
+    return lc.start(0.01, now=True)
+
+
+@implementer(ILogObserver)
+class EventLoggingObserver(Sequence[LogEvent]):
+    """
+    L{ILogObserver} That stores its events in a list for later inspection.
+    This class is similar to L{LimitedHistoryLogObserver} save that the
+    internal buffer is public and intended for external inspection.  The
+    observer implements the sequence protocol to ease iteration of the events.
+
+    @ivar _events: The events captured by this observer
+    @type _events: L{list}
+    """
+
+    def __init__(self) -> None:
+        self._events: list[LogEvent] = []
+
+    def __len__(self) -> int:
+        return len(self._events)
+
+    @overload
+    def __getitem__(self, index: int) -> LogEvent:
+        ...
+
+    @overload
+    def __getitem__(self, index: slice) -> Sequence[LogEvent]:
+        ...
+
+    def __getitem__(self, index: int | slice) -> LogEvent | Sequence[LogEvent]:
+        return self._events[index]
+
+    def __iter__(self) -> Iterator[LogEvent]:
+        return iter(self._events)
+
+    def __call__(self, event: LogEvent) -> None:
+        """
+        @see: L{ILogObserver}
+        """
+        self._events.append(event)
+
+    @classmethod
+    def createWithCleanup(cls, testInstance: TestCase, publisher: LogPublisher) -> Self:
+        """
+        Create an L{EventLoggingObserver} instance that observes the provided
+        publisher and will be cleaned up with addCleanup().
+
+        @param testInstance: Test instance in which this logger is used.
+        @type testInstance: L{twisted.trial.unittest.TestCase}
+
+        @param publisher: Log publisher to observe.
+        @type publisher: twisted.logger.LogPublisher
+
+        @return: An EventLoggingObserver configured to observe the provided
+            publisher.
+        @rtype: L{twisted.test.proto_helpers.EventLoggingObserver}
+        """
+        obs = cls()
+        publisher.addObserver(obs)
+        testInstance.addCleanup(lambda: publisher.removeObserver(obs))
+        return obs
diff --git a/contrib/python/Twisted/py3/twisted/internet/threads.py b/contrib/python/Twisted/py3/twisted/internet/threads.py
new file mode 100644
index 00000000000..e9a49cbea81
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/threads.py
@@ -0,0 +1,144 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Extended thread dispatching support.
+
+For basic support see reactor threading API docs.
+"""
+
+from __future__ import annotations
+
+import queue as Queue
+from typing import Callable, TypeVar
+
+from typing_extensions import ParamSpec
+
+from twisted.internet import defer
+from twisted.internet.interfaces import IReactorFromThreads
+from twisted.python import failure
+from twisted.python.threadpool import ThreadPool
+
+_P = ParamSpec("_P")
+_R = TypeVar("_R")
+
+
+def deferToThreadPool(
+    reactor: IReactorFromThreads,
+    threadpool: ThreadPool,
+    f: Callable[_P, _R],
+    *args: _P.args,
+    **kwargs: _P.kwargs,
+) -> defer.Deferred[_R]:
+    """
+    Call the function C{f} using a thread from the given threadpool and return
+    the result as a Deferred.
+
+    This function is only used by client code which is maintaining its own
+    threadpool.  To run a function in the reactor's threadpool, use
+    C{deferToThread}.
+
+    @param reactor: The reactor in whose main thread the Deferred will be
+        invoked.
+
+    @param threadpool: An object which supports the C{callInThreadWithCallback}
+        method of C{twisted.python.threadpool.ThreadPool}.
+
+    @param f: The function to call.
+    @param args: positional arguments to pass to f.
+    @param kwargs: keyword arguments to pass to f.
+
+    @return: A Deferred which fires a callback with the result of f, or an
+        errback with a L{twisted.python.failure.Failure} if f throws an
+        exception.
+    """
+    d: defer.Deferred[_R] = defer.Deferred()
+
+    def onResult(success: bool, result: _R | BaseException) -> None:
+        if success:
+            reactor.callFromThread(d.callback, result)
+        else:
+            reactor.callFromThread(d.errback, result)
+
+    threadpool.callInThreadWithCallback(onResult, f, *args, **kwargs)
+
+    return d
+
+
+def deferToThread(f, *args, **kwargs):
+    """
+    Run a function in a thread and return the result as a Deferred.
+
+    @param f: The function to call.
+    @param args: positional arguments to pass to f.
+    @param kwargs: keyword arguments to pass to f.
+
+    @return: A Deferred which fires a callback with the result of f,
+    or an errback with a L{twisted.python.failure.Failure} if f throws
+    an exception.
+    """
+    from twisted.internet import reactor
+
+    return deferToThreadPool(reactor, reactor.getThreadPool(), f, *args, **kwargs)
+
+
+def _runMultiple(tupleList):
+    """
+    Run a list of functions.
+    """
+    for f, args, kwargs in tupleList:
+        f(*args, **kwargs)
+
+
+def callMultipleInThread(tupleList):
+    """
+    Run a list of functions in the same thread.
+
+    tupleList should be a list of (function, argsList, kwargsDict) tuples.
+    """
+    from twisted.internet import reactor
+
+    reactor.callInThread(_runMultiple, tupleList)
+
+
+def blockingCallFromThread(reactor, f, *a, **kw):
+    """
+    Run a function in the reactor from a thread, and wait for the result
+    synchronously.  If the function returns a L{Deferred}, wait for its
+    result and return that.
+
+    @param reactor: The L{IReactorThreads} provider which will be used to
+        schedule the function call.
+    @param f: the callable to run in the reactor thread
+    @type f: any callable.
+    @param a: the arguments to pass to C{f}.
+    @param kw: the keyword arguments to pass to C{f}.
+
+    @return: the result of the L{Deferred} returned by C{f}, or the result
+        of C{f} if it returns anything other than a L{Deferred}.
+
+    @raise Exception: If C{f} raises a synchronous exception,
+        C{blockingCallFromThread} will raise that exception.  If C{f}
+        returns a L{Deferred} which fires with a L{Failure},
+        C{blockingCallFromThread} will raise that failure's exception (see
+        L{Failure.raiseException}).
+    """
+    queue = Queue.Queue()
+
+    def _callFromThread():
+        result = defer.maybeDeferred(f, *a, **kw)
+        result.addBoth(queue.put)
+
+    reactor.callFromThread(_callFromThread)
+    result = queue.get()
+    if isinstance(result, failure.Failure):
+        result.raiseException()
+    return result
+
+
+__all__ = [
+    "deferToThread",
+    "deferToThreadPool",
+    "callMultipleInThread",
+    "blockingCallFromThread",
+]
diff --git a/contrib/python/Twisted/py3/twisted/internet/tksupport.py b/contrib/python/Twisted/py3/twisted/internet/tksupport.py
new file mode 100644
index 00000000000..35550e0a48a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/tksupport.py
@@ -0,0 +1,78 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+This module integrates Tkinter with twisted.internet's mainloop.
+
+Maintainer: Itamar Shtull-Trauring
+
+To use, do::
+
+    | tksupport.install(rootWidget)
+
+and then run your reactor as usual - do *not* call Tk's mainloop(),
+use Twisted's regular mechanism for running the event loop.
+
+Likewise, to stop your program you will need to stop Twisted's
+event loop. For example, if you want closing your root widget to
+stop Twisted::
+
+    | root.protocol('WM_DELETE_WINDOW', reactor.stop)
+
+When using Aqua Tcl/Tk on macOS the standard Quit menu item in
+your application might become unresponsive without the additional
+fix::
+
+    | root.createcommand("::tk::mac::Quit", reactor.stop)
+
+@see: U{Tcl/TkAqua FAQ for more info<http://wiki.tcl.tk/12987>}
+"""
+
+import tkinter.messagebox as tkMessageBox
+import tkinter.simpledialog as tkSimpleDialog
+
+from twisted.internet import task
+
+_task = None
+
+
+def install(widget, ms=10, reactor=None):
+    """Install a Tkinter.Tk() object into the reactor."""
+    installTkFunctions()
+    global _task
+    _task = task.LoopingCall(widget.update)
+    _task.start(ms / 1000.0, False)
+
+
+def uninstall():
+    """Remove the root Tk widget from the reactor.
+
+    Call this before destroy()ing the root widget.
+    """
+    global _task
+    _task.stop()
+    _task = None
+
+
+def installTkFunctions():
+    import twisted.python.util
+
+    twisted.python.util.getPassword = getPassword
+
+
+def getPassword(prompt="", confirm=0):
+    while 1:
+        try1 = tkSimpleDialog.askstring("Password Dialog", prompt, show="*")
+        if not confirm:
+            return try1
+        try2 = tkSimpleDialog.askstring("Password Dialog", "Confirm Password", show="*")
+        if try1 == try2:
+            return try1
+        else:
+            tkMessageBox.showerror(
+                "Password Mismatch", "Passwords did not match, starting over"
+            )
+
+
+__all__ = ["install", "uninstall"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/udp.py b/contrib/python/Twisted/py3/twisted/internet/udp.py
new file mode 100644
index 00000000000..7601f2dc84c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/udp.py
@@ -0,0 +1,533 @@
+# -*- test-case-name: twisted.test.test_udp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Various asynchronous UDP classes.
+
+Please do not use this module directly.
+
+@var _sockErrReadIgnore: list of symbolic error constants (from the C{errno}
+    module) representing socket errors where the error is temporary and can be
+    ignored.
+
+@var _sockErrReadRefuse: list of symbolic error constants (from the C{errno}
+    module) representing socket errors that indicate connection refused.
+"""
+
+
+# System Imports
+import socket
+import struct
+import warnings
+from typing import Optional
+
+from zope.interface import implementer
+
+from twisted.python.runtime import platformType
+
+if platformType == "win32":
+    from errno import WSAEINPROGRESS  # type: ignore[attr-defined]
+    from errno import WSAEWOULDBLOCK  # type: ignore[attr-defined]
+    from errno import (  # type: ignore[attr-defined]
+        WSAECONNREFUSED,
+        WSAECONNRESET,
+        WSAEINTR,
+        WSAEMSGSIZE,
+        WSAENETRESET,
+        WSAENOPROTOOPT as ENOPROTOOPT,
+        WSAETIMEDOUT,
+    )
+
+    # Classify read and write errors
+    _sockErrReadIgnore = [WSAEINTR, WSAEWOULDBLOCK, WSAEMSGSIZE, WSAEINPROGRESS]
+    _sockErrReadRefuse = [WSAECONNREFUSED, WSAECONNRESET, WSAENETRESET, WSAETIMEDOUT]
+
+    # POSIX-compatible write errors
+    EMSGSIZE = WSAEMSGSIZE
+    ECONNREFUSED = WSAECONNREFUSED
+    EAGAIN = WSAEWOULDBLOCK
+    EINTR = WSAEINTR
+else:
+    from errno import EAGAIN, ECONNREFUSED, EINTR, EMSGSIZE, ENOPROTOOPT, EWOULDBLOCK
+
+    _sockErrReadIgnore = [EAGAIN, EINTR, EWOULDBLOCK]
+    _sockErrReadRefuse = [ECONNREFUSED]
+
+# Twisted Imports
+from twisted.internet import abstract, address, base, defer, error, interfaces
+from twisted.python import failure, log
+
+
+@implementer(
+    interfaces.IListeningPort, interfaces.IUDPTransport, interfaces.ISystemHandle
+)
+class Port(base.BasePort):
+    """
+    UDP port, listening for packets.
+
+    @ivar maxThroughput: Maximum number of bytes read in one event
+        loop iteration.
+
+    @ivar addressFamily: L{socket.AF_INET} or L{socket.AF_INET6}, depending on
+        whether this port is listening on an IPv4 address or an IPv6 address.
+
+    @ivar _realPortNumber: Actual port number being listened on. The
+        value will be L{None} until this L{Port} is listening.
+
+    @ivar _preexistingSocket: If not L{None}, a L{socket.socket} instance which
+        was created and initialized outside of the reactor and will be used to
+        listen for connections (instead of a new socket being created by this
+        L{Port}).
+    """
+
+    addressFamily = socket.AF_INET
+    socketType = socket.SOCK_DGRAM
+    maxThroughput = 256 * 1024
+
+    _realPortNumber: Optional[int] = None
+    _preexistingSocket = None
+
+    def __init__(self, port, proto, interface="", maxPacketSize=8192, reactor=None):
+        """
+        @param port: A port number on which to listen.
+        @type port: L{int}
+
+        @param proto: A C{DatagramProtocol} instance which will be
+            connected to the given C{port}.
+        @type proto: L{twisted.internet.protocol.DatagramProtocol}
+
+        @param interface: The local IPv4 or IPv6 address to which to bind;
+            defaults to '', ie all IPv4 addresses.
+        @type interface: L{str}
+
+        @param maxPacketSize: The maximum packet size to accept.
+        @type maxPacketSize: L{int}
+
+        @param reactor: A reactor which will notify this C{Port} when
+            its socket is ready for reading or writing. Defaults to
+            L{None}, ie the default global reactor.
+        @type reactor: L{interfaces.IReactorFDSet}
+        """
+        base.BasePort.__init__(self, reactor)
+        self.port = port
+        self.protocol = proto
+        self.maxPacketSize = maxPacketSize
+        self.interface = interface
+        self.setLogStr()
+        self._connectedAddr = None
+        self._setAddressFamily()
+
+    @classmethod
+    def _fromListeningDescriptor(
+        cls, reactor, fd, addressFamily, protocol, maxPacketSize
+    ):
+        """
+        Create a new L{Port} based on an existing listening
+        I{SOCK_DGRAM} socket.
+
+        @param reactor: A reactor which will notify this L{Port} when
+            its socket is ready for reading or writing. Defaults to
+            L{None}, ie the default global reactor.
+        @type reactor: L{interfaces.IReactorFDSet}
+
+        @param fd: An integer file descriptor associated with a listening
+            socket.  The socket must be in non-blocking mode.  Any additional
+            attributes desired, such as I{FD_CLOEXEC}, must also be set already.
+        @type fd: L{int}
+
+        @param addressFamily: The address family (sometimes called I{domain}) of
+            the existing socket.  For example, L{socket.AF_INET}.
+        @type addressFamily: L{int}
+
+        @param protocol: A C{DatagramProtocol} instance which will be
+            connected to the C{port}.
+        @type protocol: L{twisted.internet.protocol.DatagramProtocol}
+
+        @param maxPacketSize: The maximum packet size to accept.
+        @type maxPacketSize: L{int}
+
+        @return: A new instance of C{cls} wrapping the socket given by C{fd}.
+        @rtype: L{Port}
+        """
+        port = socket.fromfd(fd, addressFamily, cls.socketType)
+        interface = port.getsockname()[0]
+        self = cls(
+            None,
+            protocol,
+            interface=interface,
+            reactor=reactor,
+            maxPacketSize=maxPacketSize,
+        )
+        self._preexistingSocket = port
+        return self
+
+    def __repr__(self) -> str:
+        if self._realPortNumber is not None:
+            return f"<{self.protocol.__class__} on {self._realPortNumber}>"
+        else:
+            return f"<{self.protocol.__class__} not connected>"
+
+    def getHandle(self):
+        """
+        Return a socket object.
+        """
+        return self.socket
+
+    def startListening(self):
+        """
+        Create and bind my socket, and begin listening on it.
+
+        This is called on unserialization, and must be called after creating a
+        server to begin listening on the specified port.
+        """
+        self._bindSocket()
+        self._connectToProtocol()
+
+    def _bindSocket(self):
+        """
+        Prepare and assign a L{socket.socket} instance to
+        C{self.socket}.
+
+        Either creates a new SOCK_DGRAM L{socket.socket} bound to
+        C{self.interface} and C{self.port} or takes an existing
+        L{socket.socket} provided via the
+        L{interfaces.IReactorSocket.adoptDatagramPort} interface.
+        """
+        if self._preexistingSocket is None:
+            # Create a new socket and make it listen
+            try:
+                skt = self.createInternetSocket()
+                skt.bind((self.interface, self.port))
+            except OSError as le:
+                raise error.CannotListenError(self.interface, self.port, le)
+        else:
+            # Re-use the externally specified socket
+            skt = self._preexistingSocket
+            self._preexistingSocket = None
+
+        # Make sure that if we listened on port 0, we update that to
+        # reflect what the OS actually assigned us.
+        self._realPortNumber = skt.getsockname()[1]
+
+        log.msg(
+            "%s starting on %s"
+            % (self._getLogPrefix(self.protocol), self._realPortNumber)
+        )
+
+        self.connected = 1
+        self.socket = skt
+        self.fileno = self.socket.fileno
+
+    def _connectToProtocol(self):
+        self.protocol.makeConnection(self)
+        self.startReading()
+
+    def doRead(self):
+        """
+        Called when my socket is ready for reading.
+        """
+        read = 0
+        while read < self.maxThroughput:
+            try:
+                data, addr = self.socket.recvfrom(self.maxPacketSize)
+            except OSError as se:
+                no = se.args[0]
+                if no in _sockErrReadIgnore:
+                    return
+                if no in _sockErrReadRefuse:
+                    if self._connectedAddr:
+                        self.protocol.connectionRefused()
+                    return
+                raise
+            else:
+                read += len(data)
+                if self.addressFamily == socket.AF_INET6:
+                    # Remove the flow and scope ID from the address tuple,
+                    # reducing it to a tuple of just (host, port).
+                    #
+                    # TODO: This should be amended to return an object that can
+                    # unpack to (host, port) but also includes the flow info
+                    # and scope ID. See http://tm.tl/6826
+                    addr = addr[:2]
+                try:
+                    self.protocol.datagramReceived(data, addr)
+                except BaseException:
+                    log.err()
+
+    def write(self, datagram, addr=None):
+        """
+        Write a datagram.
+
+        @type datagram: L{bytes}
+        @param datagram: The datagram to be sent.
+
+        @type addr: L{tuple} containing L{str} as first element and L{int} as
+            second element, or L{None}
+        @param addr: A tuple of (I{stringified IPv4 or IPv6 address},
+            I{integer port number}); can be L{None} in connected mode.
+        """
+        if self._connectedAddr:
+            assert addr in (None, self._connectedAddr)
+            try:
+                return self.socket.send(datagram)
+            except OSError as se:
+                no = se.args[0]
+                if no == EINTR:
+                    return self.write(datagram)
+                elif no == EMSGSIZE:
+                    raise error.MessageLengthError("message too long")
+                elif no == ECONNREFUSED:
+                    self.protocol.connectionRefused()
+                else:
+                    raise
+        else:
+            assert addr != None
+            if (
+                not abstract.isIPAddress(addr[0])
+                and not abstract.isIPv6Address(addr[0])
+                and addr[0] != "<broadcast>"
+            ):
+                raise error.InvalidAddressError(
+                    addr[0], "write() only accepts IP addresses, not hostnames"
+                )
+            if (
+                abstract.isIPAddress(addr[0]) or addr[0] == "<broadcast>"
+            ) and self.addressFamily == socket.AF_INET6:
+                raise error.InvalidAddressError(
+                    addr[0], "IPv6 port write() called with IPv4 or broadcast address"
+                )
+            if abstract.isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET:
+                raise error.InvalidAddressError(
+                    addr[0], "IPv4 port write() called with IPv6 address"
+                )
+            try:
+                return self.socket.sendto(datagram, addr)
+            except OSError as se:
+                no = se.args[0]
+                if no == EINTR:
+                    return self.write(datagram, addr)
+                elif no == EMSGSIZE:
+                    raise error.MessageLengthError("message too long")
+                elif no == ECONNREFUSED:
+                    # in non-connected UDP ECONNREFUSED is platform dependent, I
+                    # think and the info is not necessarily useful. Nevertheless
+                    # maybe we should call connectionRefused? XXX
+                    return
+                else:
+                    raise
+
+    def writeSequence(self, seq, addr):
+        """
+        Write a datagram constructed from an iterable of L{bytes}.
+
+        @param seq: The data that will make up the complete datagram to be
+            written.
+        @type seq: an iterable of L{bytes}
+
+        @type addr: L{tuple} containing L{str} as first element and L{int} as
+            second element, or L{None}
+        @param addr: A tuple of (I{stringified IPv4 or IPv6 address},
+            I{integer port number}); can be L{None} in connected mode.
+        """
+        self.write(b"".join(seq), addr)
+
+    def connect(self, host, port):
+        """
+        'Connect' to remote server.
+        """
+        if self._connectedAddr:
+            raise RuntimeError(
+                "already connected, reconnecting is not currently supported"
+            )
+        if not abstract.isIPAddress(host) and not abstract.isIPv6Address(host):
+            raise error.InvalidAddressError(host, "not an IPv4 or IPv6 address.")
+        self._connectedAddr = (host, port)
+        self.socket.connect((host, port))
+
+    def _loseConnection(self):
+        self.stopReading()
+        if self.connected:  # actually means if we are *listening*
+            self.reactor.callLater(0, self.connectionLost)
+
+    def stopListening(self):
+        if self.connected:
+            result = self.d = defer.Deferred()
+        else:
+            result = None
+        self._loseConnection()
+        return result
+
+    def loseConnection(self):
+        warnings.warn(
+            "Please use stopListening() to disconnect port",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.stopListening()
+
+    def connectionLost(self, reason=None):
+        """
+        Cleans up my socket.
+        """
+        log.msg("(UDP Port %s Closed)" % self._realPortNumber)
+        self._realPortNumber = None
+        self.maxThroughput = -1
+        base.BasePort.connectionLost(self, reason)
+        self.protocol.doStop()
+        self.socket.close()
+        del self.socket
+        del self.fileno
+        if hasattr(self, "d"):
+            self.d.callback(None)
+            del self.d
+
+    def setLogStr(self):
+        """
+        Initialize the C{logstr} attribute to be used by C{logPrefix}.
+        """
+        logPrefix = self._getLogPrefix(self.protocol)
+        self.logstr = "%s (UDP)" % logPrefix
+
+    def _setAddressFamily(self):
+        """
+        Resolve address family for the socket.
+        """
+        if abstract.isIPv6Address(self.interface):
+            self.addressFamily = socket.AF_INET6
+        elif abstract.isIPAddress(self.interface):
+            self.addressFamily = socket.AF_INET
+        elif self.interface:
+            raise error.InvalidAddressError(
+                self.interface, "not an IPv4 or IPv6 address."
+            )
+
+    def logPrefix(self):
+        """
+        Return the prefix to log with.
+        """
+        return self.logstr
+
+    def getHost(self):
+        """
+        Return the local address of the UDP connection
+
+        @returns: the local address of the UDP connection
+        @rtype: L{IPv4Address} or L{IPv6Address}
+        """
+        addr = self.socket.getsockname()
+        if self.addressFamily == socket.AF_INET:
+            return address.IPv4Address("UDP", *addr)
+        elif self.addressFamily == socket.AF_INET6:
+            return address.IPv6Address("UDP", *(addr[:2]))
+
+    def setBroadcastAllowed(self, enabled):
+        """
+        Set whether this port may broadcast. This is disabled by default.
+
+        @param enabled: Whether the port may broadcast.
+        @type enabled: L{bool}
+        """
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, enabled)
+
+    def getBroadcastAllowed(self):
+        """
+        Checks if broadcast is currently allowed on this port.
+
+        @return: Whether this port may broadcast.
+        @rtype: L{bool}
+        """
+        return bool(self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST))
+
+
+class MulticastMixin:
+    """
+    Implement multicast functionality.
+    """
+
+    def getOutgoingInterface(self):
+        i = self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF)
+        return socket.inet_ntoa(struct.pack("@i", i))
+
+    def setOutgoingInterface(self, addr):
+        """Returns Deferred of success."""
+        return self.reactor.resolve(addr).addCallback(self._setInterface)
+
+    def _setInterface(self, addr):
+        i = socket.inet_aton(addr)
+        self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, i)
+        return 1
+
+    def getLoopbackMode(self):
+        return self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP)
+
+    def setLoopbackMode(self, mode):
+        mode = struct.pack("b", bool(mode))
+        self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, mode)
+
+    def getTTL(self):
+        return self.socket.getsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL)
+
+    def setTTL(self, ttl):
+        ttl = struct.pack("B", ttl)
+        self.socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl)
+
+    def joinGroup(self, addr, interface=""):
+        """Join a multicast group. Returns Deferred of success."""
+        return self.reactor.resolve(addr).addCallback(self._joinAddr1, interface, 1)
+
+    def _joinAddr1(self, addr, interface, join):
+        return self.reactor.resolve(interface).addCallback(self._joinAddr2, addr, join)
+
+    def _joinAddr2(self, interface, addr, join):
+        addr = socket.inet_aton(addr)
+        interface = socket.inet_aton(interface)
+        if join:
+            cmd = socket.IP_ADD_MEMBERSHIP
+        else:
+            cmd = socket.IP_DROP_MEMBERSHIP
+        try:
+            self.socket.setsockopt(socket.IPPROTO_IP, cmd, addr + interface)
+        except OSError as e:
+            return failure.Failure(error.MulticastJoinError(addr, interface, *e.args))
+
+    def leaveGroup(self, addr, interface=""):
+        """Leave multicast group, return Deferred of success."""
+        return self.reactor.resolve(addr).addCallback(self._joinAddr1, interface, 0)
+
+
+@implementer(interfaces.IMulticastTransport)
+class MulticastPort(MulticastMixin, Port):
+    """
+    UDP Port that supports multicasting.
+    """
+
+    def __init__(
+        self,
+        port,
+        proto,
+        interface="",
+        maxPacketSize=8192,
+        reactor=None,
+        listenMultiple=False,
+    ):
+        """
+        @see: L{twisted.internet.interfaces.IReactorMulticast.listenMulticast}
+        """
+        Port.__init__(self, port, proto, interface, maxPacketSize, reactor)
+        self.listenMultiple = listenMultiple
+
+    def createInternetSocket(self):
+        skt = Port.createInternetSocket(self)
+        if self.listenMultiple:
+            skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            if hasattr(socket, "SO_REUSEPORT"):
+                try:
+                    skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+                except OSError as le:
+                    # RHEL6 defines SO_REUSEPORT but it doesn't work
+                    if le.errno == ENOPROTOOPT:
+                        pass
+                    else:
+                        raise
+        return skt
diff --git a/contrib/python/Twisted/py3/twisted/internet/unix.py b/contrib/python/Twisted/py3/twisted/internet/unix.py
new file mode 100644
index 00000000000..c3fe62b22d1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/unix.py
@@ -0,0 +1,645 @@
+# -*- test-case-name: twisted.test.test_unix,twisted.internet.test.test_unix,twisted.internet.test.test_posixbase -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+UNIX socket support for Twisted.
+
+End users shouldn't use this module directly - use the reactor APIs instead.
+
+Maintainer: Itamar Shtull-Trauring
+"""
+
+
+import os
+import socket
+import stat
+import struct
+from errno import EAGAIN, ECONNREFUSED, EINTR, EMSGSIZE, ENOBUFS, EWOULDBLOCK
+from typing import Optional, Type
+
+from zope.interface import implementedBy, implementer, implementer_only
+
+from twisted.internet import address, base, error, interfaces, main, protocol, tcp, udp
+from twisted.internet.abstract import FileDescriptor
+from twisted.python import failure, lockfile, log, reflect
+from twisted.python.compat import lazyByteSlice
+from twisted.python.filepath import _coerceToFilesystemEncoding
+from twisted.python.util import untilConcludes
+
+try:
+    from twisted.python import sendmsg as _sendmsg
+except ImportError:
+    sendmsg = None
+else:
+    sendmsg = _sendmsg
+
+if not hasattr(socket, "AF_UNIX"):
+    raise ImportError("UNIX sockets not supported on this platform")
+
+
+def _ancillaryDescriptor(fd):
+    """
+    Pack an integer into an ancillary data structure suitable for use with
+    L{sendmsg.sendmsg}.
+    """
+    packed = struct.pack("i", fd)
+    return [(socket.SOL_SOCKET, sendmsg.SCM_RIGHTS, packed)]
+
+
+class _SendmsgMixin:
+    """
+    Mixin for stream-oriented UNIX transports which uses sendmsg and recvmsg to
+    offer additional functionality, such as copying file descriptors into other
+    processes.
+
+    @ivar _writeSomeDataBase: The class which provides the basic implementation
+        of C{writeSomeData}.  Ultimately this should be a subclass of
+        L{twisted.internet.abstract.FileDescriptor}.  Subclasses which mix in
+        L{_SendmsgMixin} must define this.
+
+    @ivar _sendmsgQueue: A C{list} of C{int} holding file descriptors which are
+        currently buffered before being sent.
+
+    @ivar _fileDescriptorBufferSize: An C{int} giving the maximum number of file
+        descriptors to accept and queue for sending before pausing the
+        registered producer, if there is one.
+    """
+
+    _writeSomeDataBase: Optional[Type[FileDescriptor]] = None
+    _fileDescriptorBufferSize = 64
+
+    def __init__(self):
+        self._sendmsgQueue = []
+
+    def _isSendBufferFull(self):
+        """
+        Determine whether the user-space send buffer for this transport is full
+        or not.
+
+        This extends the base determination by adding consideration of how many
+        file descriptors need to be sent using L{sendmsg.sendmsg}.  When there
+        are more than C{self._fileDescriptorBufferSize}, the buffer is
+        considered full.
+
+        @return: C{True} if it is full, C{False} otherwise.
+        """
+        # There must be some bytes in the normal send buffer, checked by
+        # _writeSomeDataBase._isSendBufferFull, in order to send file
+        # descriptors from _sendmsgQueue.  That means that the buffer will
+        # eventually be considered full even without this additional logic.
+        # However, since we send only one byte per file descriptor, having lots
+        # of elements in _sendmsgQueue incurs more overhead and perhaps slows
+        # things down.  Anyway, try this for now, maybe rethink it later.
+        return len(
+            self._sendmsgQueue
+        ) > self._fileDescriptorBufferSize or self._writeSomeDataBase._isSendBufferFull(
+            self
+        )
+
+    def sendFileDescriptor(self, fileno):
+        """
+        Queue the given file descriptor to be sent and start trying to send it.
+        """
+        self._sendmsgQueue.append(fileno)
+        self._maybePauseProducer()
+        self.startWriting()
+
+    def writeSomeData(self, data):
+        """
+        Send as much of C{data} as possible.  Also send any pending file
+        descriptors.
+        """
+        # Make it a programming error to send more file descriptors than you
+        # send regular bytes.  Otherwise, due to the limitation mentioned
+        # below, we could end up with file descriptors left, but no bytes to
+        # send with them, therefore no way to send those file descriptors.
+        if len(self._sendmsgQueue) > len(data):
+            return error.FileDescriptorOverrun()
+
+        # If there are file descriptors to send, try sending them first, using
+        # a little bit of data from the stream-oriented write buffer too.  It
+        # is not possible to send a file descriptor without sending some
+        # regular data.
+        index = 0
+        try:
+            while index < len(self._sendmsgQueue):
+                fd = self._sendmsgQueue[index]
+                try:
+                    untilConcludes(
+                        sendmsg.sendmsg,
+                        self.socket,
+                        data[index : index + 1],
+                        _ancillaryDescriptor(fd),
+                    )
+                except OSError as se:
+                    if se.args[0] in (EWOULDBLOCK, ENOBUFS):
+                        return index
+                    else:
+                        return main.CONNECTION_LOST
+                else:
+                    index += 1
+        finally:
+            del self._sendmsgQueue[:index]
+
+        # Hand the remaining data to the base implementation.  Avoid slicing in
+        # favor of a buffer, in case that happens to be any faster.
+        limitedData = lazyByteSlice(data, index)
+        result = self._writeSomeDataBase.writeSomeData(self, limitedData)
+        try:
+            return index + result
+        except TypeError:
+            return result
+
+    def doRead(self):
+        """
+        Calls {IProtocol.dataReceived} with all available data and
+        L{IFileDescriptorReceiver.fileDescriptorReceived} once for each
+        received file descriptor in ancillary data.
+
+        This reads up to C{self.bufferSize} bytes of data from its socket, then
+        dispatches the data to protocol callbacks to be handled.  If the
+        connection is not lost through an error in the underlying recvmsg(),
+        this function will return the result of the dataReceived call.
+        """
+        try:
+            data, ancillary, flags = untilConcludes(
+                sendmsg.recvmsg, self.socket, self.bufferSize
+            )
+        except OSError as se:
+            if se.args[0] == EWOULDBLOCK:
+                return
+            else:
+                return main.CONNECTION_LOST
+
+        for cmsgLevel, cmsgType, cmsgData in ancillary:
+            if cmsgLevel == socket.SOL_SOCKET and cmsgType == sendmsg.SCM_RIGHTS:
+                self._ancillaryLevelSOLSOCKETTypeSCMRIGHTS(cmsgData)
+            else:
+                log.msg(
+                    format=(
+                        "%(protocolName)s (on %(hostAddress)r) "
+                        "received unsupported ancillary data "
+                        "(level=%(cmsgLevel)r, type=%(cmsgType)r) "
+                        "from %(peerAddress)r."
+                    ),
+                    hostAddress=self.getHost(),
+                    peerAddress=self.getPeer(),
+                    protocolName=self._getLogPrefix(self.protocol),
+                    cmsgLevel=cmsgLevel,
+                    cmsgType=cmsgType,
+                )
+
+        return self._dataReceived(data)
+
+    def _ancillaryLevelSOLSOCKETTypeSCMRIGHTS(self, cmsgData):
+        """
+        Processes ancillary data with level SOL_SOCKET and type SCM_RIGHTS,
+        indicating that the ancillary data payload holds file descriptors.
+
+        Calls L{IFileDescriptorReceiver.fileDescriptorReceived} once for each
+        received file descriptor or logs a message if the protocol does not
+        implement L{IFileDescriptorReceiver}.
+
+        @param cmsgData: Ancillary data payload.
+        @type cmsgData: L{bytes}
+        """
+
+        fdCount = len(cmsgData) // 4
+        fds = struct.unpack("i" * fdCount, cmsgData)
+        if interfaces.IFileDescriptorReceiver.providedBy(self.protocol):
+            for fd in fds:
+                self.protocol.fileDescriptorReceived(fd)
+        else:
+            log.msg(
+                format=(
+                    "%(protocolName)s (on %(hostAddress)r) does not "
+                    "provide IFileDescriptorReceiver; closing file "
+                    "descriptor received (from %(peerAddress)r)."
+                ),
+                hostAddress=self.getHost(),
+                peerAddress=self.getPeer(),
+                protocolName=self._getLogPrefix(self.protocol),
+            )
+            for fd in fds:
+                os.close(fd)
+
+
+class _UnsupportedSendmsgMixin:
+    """
+    Behaviorless placeholder used when C{twisted.python.sendmsg} is not
+    available, preventing L{IUNIXTransport} from being supported.
+    """
+
+
+if sendmsg:
+    _SendmsgMixin = _SendmsgMixin
+else:
+    _SendmsgMixin = _UnsupportedSendmsgMixin  # type: ignore[assignment,misc]
+
+
+@implementer(interfaces.IUNIXTransport)
+class Server(_SendmsgMixin, tcp.Server):
+    _writeSomeDataBase = tcp.Server
+
+    def __init__(self, sock, protocol, client, server, sessionno, reactor):
+        _SendmsgMixin.__init__(self)
+        tcp.Server.__init__(
+            self, sock, protocol, (client, None), server, sessionno, reactor
+        )
+
+    @classmethod
+    def _fromConnectedSocket(cls, fileDescriptor, factory, reactor):
+        """
+        Create a new L{Server} based on an existing connected I{SOCK_STREAM}
+        socket.
+
+        Arguments are the same as to L{Server.__init__}, except where noted.
+
+        @param fileDescriptor: An integer file descriptor associated with a
+            connected socket.  The socket must be in non-blocking mode.  Any
+            additional attributes desired, such as I{FD_CLOEXEC}, must also be
+            set already.
+
+        @return: A new instance of C{cls} wrapping the socket given by
+            C{fileDescriptor}.
+        """
+        skt = socket.fromfd(fileDescriptor, socket.AF_UNIX, socket.SOCK_STREAM)
+        protocolAddr = address.UNIXAddress(skt.getsockname())
+
+        proto = factory.buildProtocol(protocolAddr)
+        if proto is None:
+            skt.close()
+            return
+
+        # FIXME: is this a suitable sessionno?
+        sessionno = 0
+        self = cls(skt, proto, skt.getpeername(), None, sessionno, reactor)
+        self.repstr = "<{} #{} on {}>".format(
+            self.protocol.__class__.__name__,
+            self.sessionno,
+            skt.getsockname(),
+        )
+        self.logstr = "{},{},{}".format(
+            self.protocol.__class__.__name__,
+            self.sessionno,
+            skt.getsockname(),
+        )
+        proto.makeConnection(self)
+        return self
+
+    def getHost(self):
+        return address.UNIXAddress(self.socket.getsockname())
+
+    def getPeer(self):
+        return address.UNIXAddress(self.hostname or None)
+
+
+def _inFilesystemNamespace(path):
+    """
+    Determine whether the given unix socket path is in a filesystem namespace.
+
+    While most PF_UNIX sockets are entries in the filesystem, Linux 2.2 and
+    above support PF_UNIX sockets in an "abstract namespace" that does not
+    correspond to any path. This function returns C{True} if the given socket
+    path is stored in the filesystem and C{False} if the path is in this
+    abstract namespace.
+    """
+    return path[:1] not in (b"\0", "\0")
+
+
+class _UNIXPort:
+    def getHost(self):
+        """
+        Returns a UNIXAddress.
+
+        This indicates the server's address.
+        """
+        return address.UNIXAddress(self.socket.getsockname())
+
+
+class Port(_UNIXPort, tcp.Port):
+    addressFamily = socket.AF_UNIX
+    socketType = socket.SOCK_STREAM
+
+    transport = Server
+    lockFile = None
+
+    def __init__(
+        self, fileName, factory, backlog=50, mode=0o666, reactor=None, wantPID=0
+    ):
+        tcp.Port.__init__(
+            self, self._buildAddr(fileName).name, factory, backlog, reactor=reactor
+        )
+        self.mode = mode
+        self.wantPID = wantPID
+        self._preexistingSocket = None
+
+    @classmethod
+    def _fromListeningDescriptor(cls, reactor, fd, factory):
+        """
+        Create a new L{Port} based on an existing listening I{SOCK_STREAM}
+        socket.
+
+        Arguments are the same as to L{Port.__init__}, except where noted.
+
+        @param fd: An integer file descriptor associated with a listening
+            socket.  The socket must be in non-blocking mode.  Any additional
+            attributes desired, such as I{FD_CLOEXEC}, must also be set already.
+
+        @return: A new instance of C{cls} wrapping the socket given by C{fd}.
+        """
+        port = socket.fromfd(fd, cls.addressFamily, cls.socketType)
+        self = cls(port.getsockname(), factory, reactor=reactor)
+        self._preexistingSocket = port
+        return self
+
+    def __repr__(self) -> str:
+        factoryName = reflect.qual(self.factory.__class__)
+        if hasattr(self, "socket"):
+            return "<{} on {!r}>".format(
+                factoryName,
+                _coerceToFilesystemEncoding("", self.port),
+            )
+        else:
+            return f"<{factoryName} (not listening)>"
+
+    def _buildAddr(self, name):
+        return address.UNIXAddress(name)
+
+    def startListening(self):
+        """
+        Create and bind my socket, and begin listening on it.
+
+        This is called on unserialization, and must be called after creating a
+        server to begin listening on the specified port.
+        """
+        tcp._reservedFD.reserve()
+        log.msg(
+            "%s starting on %r"
+            % (
+                self._getLogPrefix(self.factory),
+                _coerceToFilesystemEncoding("", self.port),
+            )
+        )
+        if self.wantPID:
+            self.lockFile = lockfile.FilesystemLock(self.port + b".lock")
+            if not self.lockFile.lock():
+                raise error.CannotListenError(None, self.port, "Cannot acquire lock")
+            else:
+                if not self.lockFile.clean:
+                    try:
+                        # This is a best-attempt at cleaning up
+                        # left-over unix sockets on the filesystem.
+                        # If it fails, there's not much else we can
+                        # do.  The bind() below will fail with an
+                        # exception that actually propagates.
+                        if stat.S_ISSOCK(os.stat(self.port).st_mode):
+                            os.remove(self.port)
+                    except BaseException:
+                        pass
+
+        self.factory.doStart()
+
+        try:
+            if self._preexistingSocket is not None:
+                skt = self._preexistingSocket
+                self._preexistingSocket = None
+            else:
+                skt = self.createInternetSocket()
+                skt.bind(self.port)
+        except OSError as le:
+            raise error.CannotListenError(None, self.port, le)
+        else:
+            if _inFilesystemNamespace(self.port):
+                # Make the socket readable and writable to the world.
+                os.chmod(self.port, self.mode)
+            skt.listen(self.backlog)
+            self.connected = True
+            self.socket = skt
+            self.fileno = self.socket.fileno
+            self.numberAccepts = 100
+            self.startReading()
+
+    def _logConnectionLostMsg(self):
+        """
+        Log message for closing socket
+        """
+        log.msg(
+            "(UNIX Port %s Closed)"
+            % (
+                _coerceToFilesystemEncoding(
+                    "",
+                    self.port,
+                )
+            )
+        )
+
+    def connectionLost(self, reason):
+        if _inFilesystemNamespace(self.port):
+            os.unlink(self.port)
+        if self.lockFile is not None:
+            self.lockFile.unlock()
+        tcp.Port.connectionLost(self, reason)
+
+
+@implementer(interfaces.IUNIXTransport)
+class Client(_SendmsgMixin, tcp.BaseClient):
+    """A client for Unix sockets."""
+
+    addressFamily = socket.AF_UNIX
+    socketType = socket.SOCK_STREAM
+    _writeSomeDataBase = tcp.BaseClient
+
+    def __init__(self, filename, connector, reactor=None, checkPID=0):
+        _SendmsgMixin.__init__(self)
+        # Normalise the filename using UNIXAddress
+        filename = address.UNIXAddress(filename).name
+        self.connector = connector
+        self.realAddress = self.addr = filename
+        if checkPID and not lockfile.isLocked(filename + b".lock"):
+            self._finishInit(None, None, error.BadFileError(filename), reactor)
+        self._finishInit(self.doConnect, self.createInternetSocket(), None, reactor)
+
+    def getPeer(self):
+        return address.UNIXAddress(self.addr)
+
+    def getHost(self):
+        return address.UNIXAddress(None)
+
+
+class Connector(base.BaseConnector):
+    def __init__(self, address, factory, timeout, reactor, checkPID):
+        base.BaseConnector.__init__(self, factory, timeout, reactor)
+        self.address = address
+        self.checkPID = checkPID
+
+    def _makeTransport(self):
+        return Client(self.address, self, self.reactor, self.checkPID)
+
+    def getDestination(self):
+        return address.UNIXAddress(self.address)
+
+
+@implementer(interfaces.IUNIXDatagramTransport)
+class DatagramPort(_UNIXPort, udp.Port):
+    """
+    Datagram UNIX port, listening for packets.
+    """
+
+    addressFamily = socket.AF_UNIX
+
+    def __init__(self, addr, proto, maxPacketSize=8192, mode=0o666, reactor=None):
+        """Initialize with address to listen on."""
+        udp.Port.__init__(
+            self, addr, proto, maxPacketSize=maxPacketSize, reactor=reactor
+        )
+        self.mode = mode
+
+    def __repr__(self) -> str:
+        protocolName = reflect.qual(
+            self.protocol.__class__,
+        )
+        if hasattr(self, "socket"):
+            return f"<{protocolName} on {self.port!r}>"
+        else:
+            return f"<{protocolName} (not listening)>"
+
+    def _bindSocket(self):
+        log.msg(f"{self.protocol.__class__} starting on {repr(self.port)}")
+        try:
+            skt = self.createInternetSocket()  # XXX: haha misnamed method
+            if self.port:
+                skt.bind(self.port)
+        except OSError as le:
+            raise error.CannotListenError(None, self.port, le)
+        if self.port and _inFilesystemNamespace(self.port):
+            # Make the socket readable and writable to the world.
+            os.chmod(self.port, self.mode)
+        self.connected = 1
+        self.socket = skt
+        self.fileno = self.socket.fileno
+
+    def write(self, datagram, address):
+        """Write a datagram."""
+        try:
+            return self.socket.sendto(datagram, address)
+        except OSError as se:
+            no = se.args[0]
+            if no == EINTR:
+                return self.write(datagram, address)
+            elif no == EMSGSIZE:
+                raise error.MessageLengthError("message too long")
+            elif no == EAGAIN:
+                # oh, well, drop the data. The only difference from UDP
+                # is that UDP won't ever notice.
+                # TODO: add TCP-like buffering
+                pass
+            else:
+                raise
+
+    def connectionLost(self, reason=None):
+        """Cleans up my socket."""
+        log.msg("(Port %s Closed)" % repr(self.port))
+        base.BasePort.connectionLost(self, reason)
+        if hasattr(self, "protocol"):
+            # we won't have attribute in ConnectedPort, in cases
+            # where there was an error in connection process
+            self.protocol.doStop()
+        self.connected = 0
+        self.socket.close()
+        del self.socket
+        del self.fileno
+        if hasattr(self, "d"):
+            self.d.callback(None)
+            del self.d
+
+    def setLogStr(self):
+        self.logstr = reflect.qual(self.protocol.__class__) + " (UDP)"
+
+
+@implementer_only(
+    interfaces.IUNIXDatagramConnectedTransport, *(implementedBy(base.BasePort))
+)
+class ConnectedDatagramPort(DatagramPort):
+    """
+    A connected datagram UNIX socket.
+    """
+
+    def __init__(
+        self,
+        addr,
+        proto,
+        maxPacketSize=8192,
+        mode=0o666,
+        bindAddress=None,
+        reactor=None,
+    ):
+        assert isinstance(proto, protocol.ConnectedDatagramProtocol)
+        DatagramPort.__init__(self, bindAddress, proto, maxPacketSize, mode, reactor)
+        self.remoteaddr = addr
+
+    def startListening(self):
+        try:
+            self._bindSocket()
+            self.socket.connect(self.remoteaddr)
+            self._connectToProtocol()
+        except BaseException:
+            self.connectionFailed(failure.Failure())
+
+    def connectionFailed(self, reason):
+        """
+        Called when a connection fails. Stop listening on the socket.
+
+        @type reason: L{Failure}
+        @param reason: Why the connection failed.
+        """
+        self.stopListening()
+        self.protocol.connectionFailed(reason)
+        del self.protocol
+
+    def doRead(self):
+        """
+        Called when my socket is ready for reading.
+        """
+        read = 0
+        while read < self.maxThroughput:
+            try:
+                data, addr = self.socket.recvfrom(self.maxPacketSize)
+                read += len(data)
+                self.protocol.datagramReceived(data)
+            except OSError as se:
+                no = se.args[0]
+                if no in (EAGAIN, EINTR, EWOULDBLOCK):
+                    return
+                if no == ECONNREFUSED:
+                    self.protocol.connectionRefused()
+                else:
+                    raise
+            except BaseException:
+                log.deferr()
+
+    def write(self, data):
+        """
+        Write a datagram.
+        """
+        try:
+            return self.socket.send(data)
+        except OSError as se:
+            no = se.args[0]
+            if no == EINTR:
+                return self.write(data)
+            elif no == EMSGSIZE:
+                raise error.MessageLengthError("message too long")
+            elif no == ECONNREFUSED:
+                self.protocol.connectionRefused()
+            elif no == EAGAIN:
+                # oh, well, drop the data. The only difference from UDP
+                # is that UDP won't ever notice.
+                # TODO: add TCP-like buffering
+                pass
+            else:
+                raise
+
+    def getPeer(self):
+        return address.UNIXAddress(self.remoteaddr)
diff --git a/contrib/python/Twisted/py3/twisted/internet/utils.py b/contrib/python/Twisted/py3/twisted/internet/utils.py
new file mode 100644
index 00000000000..aaf00e169c5
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/utils.py
@@ -0,0 +1,256 @@
+# -*- test-case-name: twisted.test.test_iutils -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Utility methods.
+"""
+
+
+import sys
+import warnings
+from functools import wraps
+from io import BytesIO
+
+from twisted.internet import defer, protocol
+from twisted.python import failure
+
+
+def _callProtocolWithDeferred(
+    protocol, executable, args, env, path, reactor=None, protoArgs=()
+):
+    if reactor is None:
+        from twisted.internet import reactor
+
+    d = defer.Deferred()
+    p = protocol(d, *protoArgs)
+    reactor.spawnProcess(p, executable, (executable,) + tuple(args), env, path)
+    return d
+
+
+class _UnexpectedErrorOutput(IOError):
+    """
+    Standard error data was received where it was not expected.  This is a
+    subclass of L{IOError} to preserve backward compatibility with the previous
+    error behavior of L{getProcessOutput}.
+
+    @ivar processEnded: A L{Deferred} which will fire when the process which
+        produced the data on stderr has ended (exited and all file descriptors
+        closed).
+    """
+
+    def __init__(self, text, processEnded):
+        IOError.__init__(self, f"got stderr: {text!r}")
+        self.processEnded = processEnded
+
+
+class _BackRelay(protocol.ProcessProtocol):
+    """
+    Trivial protocol for communicating with a process and turning its output
+    into the result of a L{Deferred}.
+
+    @ivar deferred: A L{Deferred} which will be called back with all of stdout
+        and, if C{errortoo} is true, all of stderr as well (mixed together in
+        one string).  If C{errortoo} is false and any bytes are received over
+        stderr, this will fire with an L{_UnexpectedErrorOutput} instance and
+        the attribute will be set to L{None}.
+
+    @ivar onProcessEnded: If C{errortoo} is false and bytes are received over
+        stderr, this attribute will refer to a L{Deferred} which will be called
+        back when the process ends.  This C{Deferred} is also associated with
+        the L{_UnexpectedErrorOutput} which C{deferred} fires with earlier in
+        this case so that users can determine when the process has actually
+        ended, in addition to knowing when bytes have been received via stderr.
+    """
+
+    def __init__(self, deferred, errortoo=0):
+        self.deferred = deferred
+        self.s = BytesIO()
+        if errortoo:
+            self.errReceived = self.errReceivedIsGood
+        else:
+            self.errReceived = self.errReceivedIsBad
+
+    def errReceivedIsBad(self, text):
+        if self.deferred is not None:
+            self.onProcessEnded = defer.Deferred()
+            err = _UnexpectedErrorOutput(text, self.onProcessEnded)
+            self.deferred.errback(failure.Failure(err))
+            self.deferred = None
+            self.transport.loseConnection()
+
+    def errReceivedIsGood(self, text):
+        self.s.write(text)
+
+    def outReceived(self, text):
+        self.s.write(text)
+
+    def processEnded(self, reason):
+        if self.deferred is not None:
+            self.deferred.callback(self.s.getvalue())
+        elif self.onProcessEnded is not None:
+            self.onProcessEnded.errback(reason)
+
+
+def getProcessOutput(executable, args=(), env={}, path=None, reactor=None, errortoo=0):
+    """
+    Spawn a process and return its output as a deferred returning a L{bytes}.
+
+    @param executable: The file name to run and get the output of - the
+                       full path should be used.
+
+    @param args: the command line arguments to pass to the process; a
+                 sequence of strings. The first string should B{NOT} be the
+                 executable's name.
+
+    @param env: the environment variables to pass to the process; a
+                dictionary of strings.
+
+    @param path: the path to run the subprocess in - defaults to the
+                 current directory.
+
+    @param reactor: the reactor to use - defaults to the default reactor
+
+    @param errortoo: If true, include stderr in the result.  If false, if
+        stderr is received the returned L{Deferred} will errback with an
+        L{IOError} instance with a C{processEnded} attribute.  The
+        C{processEnded} attribute refers to a L{Deferred} which fires when the
+        executed process ends.
+    """
+    return _callProtocolWithDeferred(
+        lambda d: _BackRelay(d, errortoo=errortoo), executable, args, env, path, reactor
+    )
+
+
+class _ValueGetter(protocol.ProcessProtocol):
+    def __init__(self, deferred):
+        self.deferred = deferred
+
+    def processEnded(self, reason):
+        self.deferred.callback(reason.value.exitCode)
+
+
+def getProcessValue(executable, args=(), env={}, path=None, reactor=None):
+    """Spawn a process and return its exit code as a Deferred."""
+    return _callProtocolWithDeferred(_ValueGetter, executable, args, env, path, reactor)
+
+
+class _EverythingGetter(protocol.ProcessProtocol):
+    def __init__(self, deferred, stdinBytes=None):
+        self.deferred = deferred
+        self.outBuf = BytesIO()
+        self.errBuf = BytesIO()
+        self.outReceived = self.outBuf.write
+        self.errReceived = self.errBuf.write
+        self.stdinBytes = stdinBytes
+
+    def connectionMade(self):
+        if self.stdinBytes is not None:
+            self.transport.writeToChild(0, self.stdinBytes)
+            # The only compelling reason not to _always_ close stdin here is
+            # backwards compatibility.
+            self.transport.closeStdin()
+
+    def processEnded(self, reason):
+        out = self.outBuf.getvalue()
+        err = self.errBuf.getvalue()
+        e = reason.value
+        code = e.exitCode
+        if e.signal:
+            self.deferred.errback((out, err, e.signal))
+        else:
+            self.deferred.callback((out, err, code))
+
+
+def getProcessOutputAndValue(
+    executable, args=(), env={}, path=None, reactor=None, stdinBytes=None
+):
+    """Spawn a process and returns a Deferred that will be called back with
+    its output (from stdout and stderr) and it's exit code as (out, err, code)
+    If a signal is raised, the Deferred will errback with the stdout and
+    stderr up to that point, along with the signal, as (out, err, signalNum)
+    """
+    return _callProtocolWithDeferred(
+        _EverythingGetter,
+        executable,
+        args,
+        env,
+        path,
+        reactor,
+        protoArgs=(stdinBytes,),
+    )
+
+
+def _resetWarningFilters(passthrough, addedFilters):
+    for f in addedFilters:
+        try:
+            warnings.filters.remove(f)
+        except ValueError:
+            pass
+    return passthrough
+
+
+def runWithWarningsSuppressed(suppressedWarnings, f, *a, **kw):
+    """
+    Run the function I{f}, but with some warnings suppressed.
+
+    This calls L{warnings.filterwarnings} to add warning filters before
+    invoking I{f}. If I{f} returns a L{Deferred} then the added filters are
+    removed once the deferred fires. Otherwise they are removed immediately.
+
+    Note that the list of warning filters is a process-wide resource, so
+    calling this function will affect all threads.
+
+    @param suppressedWarnings:
+        A list of arguments to pass to L{warnings.filterwarnings}, a sequence
+        of (args, kwargs) 2-tuples.
+
+    @param f: A callable, which may return a L{Deferred}.
+
+    @param a: Positional arguments passed to I{f}
+
+    @param kw: Keyword arguments passed to I{f}
+
+    @return: The result of C{f(*a, **kw)}
+
+    @seealso: L{twisted.python.util.runWithWarningsSuppressed}
+        functions similarly, but doesn't handled L{Deferred}s.
+    """
+    for args, kwargs in suppressedWarnings:
+        warnings.filterwarnings(*args, **kwargs)
+    addedFilters = warnings.filters[: len(suppressedWarnings)]
+    try:
+        result = f(*a, **kw)
+    except BaseException:
+        exc_info = sys.exc_info()
+        _resetWarningFilters(None, addedFilters)
+        raise exc_info[1].with_traceback(exc_info[2])
+    else:
+        if isinstance(result, defer.Deferred):
+            result.addBoth(_resetWarningFilters, addedFilters)
+        else:
+            _resetWarningFilters(None, addedFilters)
+        return result
+
+
+def suppressWarnings(f, *suppressedWarnings):
+    """
+    Wrap C{f} in a callable which suppresses the indicated warnings before
+    invoking C{f} and unsuppresses them afterwards.  If f returns a Deferred,
+    warnings will remain suppressed until the Deferred fires.
+    """
+
+    @wraps(f)
+    def warningSuppressingWrapper(*a, **kw):
+        return runWithWarningsSuppressed(suppressedWarnings, f, *a, **kw)
+
+    return warningSuppressingWrapper
+
+
+__all__ = [
+    "runWithWarningsSuppressed",
+    "suppressWarnings",
+    "getProcessOutput",
+    "getProcessValue",
+    "getProcessOutputAndValue",
+]
diff --git a/contrib/python/Twisted/py3/twisted/internet/win32eventreactor.py b/contrib/python/Twisted/py3/twisted/internet/win32eventreactor.py
new file mode 100644
index 00000000000..0e96012ea58
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/win32eventreactor.py
@@ -0,0 +1,425 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+A win32event based implementation of the Twisted main loop.
+
+This requires pywin32 (formerly win32all) or ActivePython to be installed.
+
+To install the event loop (and you should do this before any connections,
+listeners or connectors are added)::
+
+    from twisted.internet import win32eventreactor
+    win32eventreactor.install()
+
+LIMITATIONS:
+ 1. WaitForMultipleObjects and thus the event loop can only handle 64 objects.
+ 2. Process running has some problems (see L{twisted.internet.process} docstring).
+
+
+TODO:
+ 1. Event loop handling of writes is *very* problematic (this is causing failed tests).
+    Switch to doing it the correct way, whatever that means (see below).
+ 2. Replace icky socket loopback waker with event based waker (use dummyEvent object)
+ 3. Switch everyone to using Free Software so we don't have to deal with proprietary APIs.
+
+
+ALTERNATIVE SOLUTIONS:
+ - IIRC, sockets can only be registered once. So we switch to a structure
+   like the poll() reactor, thus allowing us to deal with write events in
+   a decent fashion. This should allow us to pass tests, but we're still
+   limited to 64 events.
+
+Or:
+
+ - Instead of doing a reactor, we make this an addon to the select reactor.
+   The WFMO event loop runs in a separate thread. This means no need to maintain
+   separate code for networking, 64 event limit doesn't apply to sockets,
+   we can run processes and other win32 stuff in default event loop. The
+   only problem is that we're stuck with the icky socket based waker.
+   Another benefit is that this could be extended to support >64 events
+   in a simpler manner than the previous solution.
+
+The 2nd solution is probably what will get implemented.
+"""
+
+import sys
+
+# System imports
+import time
+from threading import Thread
+from weakref import WeakKeyDictionary
+
+from zope.interface import implementer
+
+# Win32 imports
+from win32file import (  # type: ignore[import]
+    FD_ACCEPT,
+    FD_CLOSE,
+    FD_CONNECT,
+    FD_READ,
+    WSAEventSelect,
+)
+
+try:
+    # WSAEnumNetworkEvents was added in pywin32 215
+    from win32file import WSAEnumNetworkEvents
+except ImportError:
+    import warnings
+
+    warnings.warn(
+        "Reliable disconnection notification requires pywin32 215 or later",
+        category=UserWarning,
+    )
+
+    def WSAEnumNetworkEvents(fd, event):
+        return {FD_READ}
+
+
+import win32gui  # type: ignore[import]
+from win32event import (  # type: ignore[import]
+    QS_ALLINPUT,
+    WAIT_OBJECT_0,
+    WAIT_TIMEOUT,
+    CreateEvent,
+    MsgWaitForMultipleObjects,
+)
+
+# Twisted imports
+from twisted.internet import posixbase
+from twisted.internet.interfaces import IReactorFDSet, IReactorWin32Events
+from twisted.internet.threads import blockingCallFromThread
+from twisted.python import failure, log, threadable
+
+
+@implementer(IReactorFDSet, IReactorWin32Events)
+class Win32Reactor(posixbase.PosixReactorBase):
+    """
+    Reactor that uses Win32 event APIs.
+
+    @ivar _reads: A dictionary mapping L{FileDescriptor} instances to a
+        win32 event object used to check for read events for that descriptor.
+
+    @ivar _writes: A dictionary mapping L{FileDescriptor} instances to a
+        arbitrary value.  Keys in this dictionary will be given a chance to
+        write out their data.
+
+    @ivar _events: A dictionary mapping win32 event object to tuples of
+        L{FileDescriptor} instances and event masks.
+
+    @ivar _closedAndReading: Along with C{_closedAndNotReading}, keeps track of
+        descriptors which have had close notification delivered from the OS but
+        which we have not finished reading data from.  MsgWaitForMultipleObjects
+        will only deliver close notification to us once, so we remember it in
+        these two dictionaries until we're ready to act on it.  The OS has
+        delivered close notification for each descriptor in this dictionary, and
+        the descriptors are marked as allowed to handle read events in the
+        reactor, so they can be processed.  When a descriptor is marked as not
+        allowed to handle read events in the reactor (ie, it is passed to
+        L{IReactorFDSet.removeReader}), it is moved out of this dictionary and
+        into C{_closedAndNotReading}.  The descriptors are keys in this
+        dictionary.  The values are arbitrary.
+    @type _closedAndReading: C{dict}
+
+    @ivar _closedAndNotReading: These descriptors have had close notification
+        delivered from the OS, but are not marked as allowed to handle read
+        events in the reactor.  They are saved here to record their closed
+        state, but not processed at all.  When one of these descriptors is
+        passed to L{IReactorFDSet.addReader}, it is moved out of this dictionary
+        and into C{_closedAndReading}.  The descriptors are keys in this
+        dictionary.  The values are arbitrary.  This is a weak key dictionary so
+        that if an application tells the reactor to stop reading from a
+        descriptor and then forgets about that descriptor itself, the reactor
+        will also forget about it.
+    @type _closedAndNotReading: C{WeakKeyDictionary}
+    """
+
+    dummyEvent = CreateEvent(None, 0, 0, None)
+
+    def __init__(self):
+        self._reads = {}
+        self._writes = {}
+        self._events = {}
+        self._closedAndReading = {}
+        self._closedAndNotReading = WeakKeyDictionary()
+        posixbase.PosixReactorBase.__init__(self)
+
+    def _makeSocketEvent(self, fd, action, why):
+        """
+        Make a win32 event object for a socket.
+        """
+        event = CreateEvent(None, 0, 0, None)
+        WSAEventSelect(fd, event, why)
+        self._events[event] = (fd, action)
+        return event
+
+    def addEvent(self, event, fd, action):
+        """
+        Add a new win32 event to the event loop.
+        """
+        self._events[event] = (fd, action)
+
+    def removeEvent(self, event):
+        """
+        Remove an event.
+        """
+        del self._events[event]
+
+    def addReader(self, reader):
+        """
+        Add a socket FileDescriptor for notification of data available to read.
+        """
+        if reader not in self._reads:
+            self._reads[reader] = self._makeSocketEvent(
+                reader, "doRead", FD_READ | FD_ACCEPT | FD_CONNECT | FD_CLOSE
+            )
+            # If the reader is closed, move it over to the dictionary of reading
+            # descriptors.
+            if reader in self._closedAndNotReading:
+                self._closedAndReading[reader] = True
+                del self._closedAndNotReading[reader]
+
+    def addWriter(self, writer):
+        """
+        Add a socket FileDescriptor for notification of data available to write.
+        """
+        if writer not in self._writes:
+            self._writes[writer] = 1
+
+    def removeReader(self, reader):
+        """Remove a Selectable for notification of data available to read."""
+        if reader in self._reads:
+            del self._events[self._reads[reader]]
+            del self._reads[reader]
+
+            # If the descriptor is closed, move it out of the dictionary of
+            # reading descriptors into the dictionary of waiting descriptors.
+            if reader in self._closedAndReading:
+                self._closedAndNotReading[reader] = True
+                del self._closedAndReading[reader]
+
+    def removeWriter(self, writer):
+        """Remove a Selectable for notification of data available to write."""
+        if writer in self._writes:
+            del self._writes[writer]
+
+    def removeAll(self):
+        """
+        Remove all selectables, and return a list of them.
+        """
+        return self._removeAll(self._reads, self._writes)
+
+    def getReaders(self):
+        return list(self._reads.keys())
+
+    def getWriters(self):
+        return list(self._writes.keys())
+
+    def doWaitForMultipleEvents(self, timeout):
+        log.msg(channel="system", event="iteration", reactor=self)
+        if timeout is None:
+            timeout = 100
+
+        # Keep track of whether we run any application code before we get to the
+        # MsgWaitForMultipleObjects.  If so, there's a chance it will schedule a
+        # new timed call or stop the reactor or do something else that means we
+        # shouldn't block in MsgWaitForMultipleObjects for the full timeout.
+        ranUserCode = False
+
+        # If any descriptors are trying to close, try to get them out of the way
+        # first.
+        for reader in list(self._closedAndReading.keys()):
+            ranUserCode = True
+            self._runAction("doRead", reader)
+
+        for fd in list(self._writes.keys()):
+            ranUserCode = True
+            log.callWithLogger(fd, self._runWrite, fd)
+
+        if ranUserCode:
+            # If application code *might* have scheduled an event, assume it
+            # did.  If we're wrong, we'll get back here shortly anyway.  If
+            # we're right, we'll be sure to handle the event (including reactor
+            # shutdown) in a timely manner.
+            timeout = 0
+
+        if not (self._events or self._writes):
+            # sleep so we don't suck up CPU time
+            time.sleep(timeout)
+            return
+
+        handles = list(self._events.keys()) or [self.dummyEvent]
+        timeout = int(timeout * 1000)
+        val = MsgWaitForMultipleObjects(handles, 0, timeout, QS_ALLINPUT)
+        if val == WAIT_TIMEOUT:
+            return
+        elif val == WAIT_OBJECT_0 + len(handles):
+            exit = win32gui.PumpWaitingMessages()
+            if exit:
+                self.callLater(0, self.stop)
+                return
+        elif val >= WAIT_OBJECT_0 and val < WAIT_OBJECT_0 + len(handles):
+            event = handles[val - WAIT_OBJECT_0]
+            fd, action = self._events[event]
+
+            if fd in self._reads:
+                # Before anything, make sure it's still a valid file descriptor.
+                fileno = fd.fileno()
+                if fileno == -1:
+                    self._disconnectSelectable(fd, posixbase._NO_FILEDESC, False)
+                    return
+
+                # Since it's a socket (not another arbitrary event added via
+                # addEvent) and we asked for FD_READ | FD_CLOSE, check to see if
+                # we actually got FD_CLOSE.  This needs a special check because
+                # it only gets delivered once.  If we miss it, it's gone forever
+                # and we'll never know that the connection is closed.
+                events = WSAEnumNetworkEvents(fileno, event)
+                if FD_CLOSE in events:
+                    self._closedAndReading[fd] = True
+            log.callWithLogger(fd, self._runAction, action, fd)
+
+    def _runWrite(self, fd):
+        closed = 0
+        try:
+            closed = fd.doWrite()
+        except BaseException:
+            closed = sys.exc_info()[1]
+            log.deferr()
+
+        if closed:
+            self.removeReader(fd)
+            self.removeWriter(fd)
+            try:
+                fd.connectionLost(failure.Failure(closed))
+            except BaseException:
+                log.deferr()
+        elif closed is None:
+            return 1
+
+    def _runAction(self, action, fd):
+        try:
+            closed = getattr(fd, action)()
+        except BaseException:
+            closed = sys.exc_info()[1]
+            log.deferr()
+        if closed:
+            self._disconnectSelectable(fd, closed, action == "doRead")
+
+    doIteration = doWaitForMultipleEvents
+
+
+class _ThreadFDWrapper:
+    """
+    This wraps an event handler and translates notification in the helper
+    L{Win32Reactor} thread into a notification in the primary reactor thread.
+
+    @ivar _reactor: The primary reactor, the one to which event notification
+        will be sent.
+
+    @ivar _fd: The L{FileDescriptor} to which the event will be dispatched.
+
+    @ivar _action: A C{str} giving the method of C{_fd} which handles the event.
+
+    @ivar _logPrefix: The pre-fetched log prefix string for C{_fd}, so that
+        C{_fd.logPrefix} does not need to be called in a non-main thread.
+    """
+
+    def __init__(self, reactor, fd, action, logPrefix):
+        self._reactor = reactor
+        self._fd = fd
+        self._action = action
+        self._logPrefix = logPrefix
+
+    def logPrefix(self):
+        """
+        Return the original handler's log prefix, as it was given to
+        C{__init__}.
+        """
+        return self._logPrefix
+
+    def _execute(self):
+        """
+        Callback fired when the associated event is set.  Run the C{action}
+        callback on the wrapped descriptor in the main reactor thread and raise
+        or return whatever it raises or returns to cause this event handler to
+        be removed from C{self._reactor} if appropriate.
+        """
+        return blockingCallFromThread(
+            self._reactor, lambda: getattr(self._fd, self._action)()
+        )
+
+    def connectionLost(self, reason):
+        """
+        Pass through to the wrapped descriptor, but in the main reactor thread
+        instead of the helper C{Win32Reactor} thread.
+        """
+        self._reactor.callFromThread(self._fd.connectionLost, reason)
+
+
+@implementer(IReactorWin32Events)
+class _ThreadedWin32EventsMixin:
+    """
+    This mixin implements L{IReactorWin32Events} for another reactor by running
+    a L{Win32Reactor} in a separate thread and dispatching work to it.
+
+    @ivar _reactor: The L{Win32Reactor} running in the other thread.  This is
+        L{None} until it is actually needed.
+
+    @ivar _reactorThread: The L{threading.Thread} which is running the
+        L{Win32Reactor}.  This is L{None} until it is actually needed.
+    """
+
+    _reactor = None
+    _reactorThread = None
+
+    def _unmakeHelperReactor(self):
+        """
+        Stop and discard the reactor started by C{_makeHelperReactor}.
+        """
+        self._reactor.callFromThread(self._reactor.stop)
+        self._reactor = None
+
+    def _makeHelperReactor(self):
+        """
+        Create and (in a new thread) start a L{Win32Reactor} instance to use for
+        the implementation of L{IReactorWin32Events}.
+        """
+        self._reactor = Win32Reactor()
+        # This is a helper reactor, it is not the global reactor and its thread
+        # is not "the" I/O thread.  Prevent it from registering it as such.
+        self._reactor._registerAsIOThread = False
+        self._reactorThread = Thread(target=self._reactor.run, args=(False,))
+        self.addSystemEventTrigger("after", "shutdown", self._unmakeHelperReactor)
+        self._reactorThread.start()
+
+    def addEvent(self, event, fd, action):
+        """
+        @see: L{IReactorWin32Events}
+        """
+        if self._reactor is None:
+            self._makeHelperReactor()
+        self._reactor.callFromThread(
+            self._reactor.addEvent,
+            event,
+            _ThreadFDWrapper(self, fd, action, fd.logPrefix()),
+            "_execute",
+        )
+
+    def removeEvent(self, event):
+        """
+        @see: L{IReactorWin32Events}
+        """
+        self._reactor.callFromThread(self._reactor.removeEvent, event)
+
+
+def install():
+    threadable.init(1)
+    r = Win32Reactor()
+    from . import main
+
+    main.installReactor(r)
+
+
+__all__ = ["Win32Reactor", "install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/wxreactor.py b/contrib/python/Twisted/py3/twisted/internet/wxreactor.py
new file mode 100644
index 00000000000..b988724dfa7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/wxreactor.py
@@ -0,0 +1,188 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module provides wxPython event loop support for Twisted.
+
+In order to use this support, simply do the following::
+
+    |  from twisted.internet import wxreactor
+    |  wxreactor.install()
+
+Then, when your root wxApp has been created::
+
+    | from twisted.internet import reactor
+    | reactor.registerWxApp(yourApp)
+    | reactor.run()
+
+Then use twisted.internet APIs as usual. Stop the event loop using
+reactor.stop(), not yourApp.ExitMainLoop().
+
+IMPORTANT: tests will fail when run under this reactor. This is
+expected and probably does not reflect on the reactor's ability to run
+real applications.
+"""
+
+from queue import Empty, Queue
+
+try:
+    from wx import (  # type: ignore[import]
+        CallAfter as wxCallAfter,
+        PySimpleApp as wxPySimpleApp,
+        Timer as wxTimer,
+    )
+except ImportError:
+    # older version of wxPython:
+    from wxPython.wx import wxPySimpleApp, wxCallAfter, wxTimer  # type: ignore[import]
+
+from twisted.internet import _threadedselect
+from twisted.python import log, runtime
+
+
+class ProcessEventsTimer(wxTimer):
+    """
+    Timer that tells wx to process pending events.
+
+    This is necessary on macOS, probably due to a bug in wx, if we want
+    wxCallAfters to be handled when modal dialogs, menus, etc.  are open.
+    """
+
+    def __init__(self, wxapp):
+        wxTimer.__init__(self)
+        self.wxapp = wxapp
+
+    def Notify(self):
+        """
+        Called repeatedly by wx event loop.
+        """
+        self.wxapp.ProcessPendingEvents()
+
+
+class WxReactor(_threadedselect.ThreadedSelectReactor):
+    """
+    wxPython reactor.
+
+    wxPython drives the event loop, select() runs in a thread.
+    """
+
+    _stopping = False
+
+    def registerWxApp(self, wxapp):
+        """
+        Register wxApp instance with the reactor.
+        """
+        self.wxapp = wxapp
+
+    def _installSignalHandlersAgain(self):
+        """
+        wx sometimes removes our own signal handlers, so re-add them.
+        """
+        try:
+            # make _handleSignals happy:
+            import signal
+
+            signal.signal(signal.SIGINT, signal.default_int_handler)
+        except ImportError:
+            return
+        self._signals.install()
+
+    def stop(self):
+        """
+        Stop the reactor.
+        """
+        if self._stopping:
+            return
+        self._stopping = True
+        _threadedselect.ThreadedSelectReactor.stop(self)
+
+    def _runInMainThread(self, f):
+        """
+        Schedule function to run in main wx/Twisted thread.
+
+        Called by the select() thread.
+        """
+        if hasattr(self, "wxapp"):
+            wxCallAfter(f)
+        else:
+            # wx shutdown but twisted hasn't
+            self._postQueue.put(f)
+
+    def _stopWx(self):
+        """
+        Stop the wx event loop if it hasn't already been stopped.
+
+        Called during Twisted event loop shutdown.
+        """
+        if hasattr(self, "wxapp"):
+            self.wxapp.ExitMainLoop()
+
+    def run(self, installSignalHandlers=True):
+        """
+        Start the reactor.
+        """
+        self._postQueue = Queue()
+        if not hasattr(self, "wxapp"):
+            log.msg(
+                "registerWxApp() was not called on reactor, "
+                "registering my own wxApp instance."
+            )
+            self.registerWxApp(wxPySimpleApp())
+
+        # start select() thread:
+        self.interleave(
+            self._runInMainThread, installSignalHandlers=installSignalHandlers
+        )
+        if installSignalHandlers:
+            self.callLater(0, self._installSignalHandlersAgain)
+
+        # add cleanup events:
+        self.addSystemEventTrigger("after", "shutdown", self._stopWx)
+        self.addSystemEventTrigger(
+            "after", "shutdown", lambda: self._postQueue.put(None)
+        )
+
+        # On macOS, work around wx bug by starting timer to ensure
+        # wxCallAfter calls are always processed. We don't wake up as
+        # often as we could since that uses too much CPU.
+        if runtime.platform.isMacOSX():
+            t = ProcessEventsTimer(self.wxapp)
+            t.Start(2)  # wake up every 2ms
+
+        self.wxapp.MainLoop()
+        wxapp = self.wxapp
+        del self.wxapp
+
+        if not self._stopping:
+            # wx event loop exited without reactor.stop() being
+            # called.  At this point events from select() thread will
+            # be added to _postQueue, but some may still be waiting
+            # unprocessed in wx, thus the ProcessPendingEvents()
+            # below.
+            self.stop()
+            wxapp.ProcessPendingEvents()  # deal with any queued wxCallAfters
+            while 1:
+                try:
+                    f = self._postQueue.get(timeout=0.01)
+                except Empty:
+                    continue
+                else:
+                    if f is None:
+                        break
+                    try:
+                        f()
+                    except BaseException:
+                        log.err()
+
+
+def install():
+    """
+    Configure the twisted mainloop to be run inside the wxPython mainloop.
+    """
+    reactor = WxReactor()
+    from twisted.internet.main import installReactor
+
+    installReactor(reactor)
+    return reactor
+
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/internet/wxsupport.py b/contrib/python/Twisted/py3/twisted/internet/wxsupport.py
new file mode 100644
index 00000000000..a9fab83d372
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/internet/wxsupport.py
@@ -0,0 +1,57 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+"""Old method of wxPython support for Twisted.
+
+twisted.internet.wxreactor is probably a better choice.
+
+To use::
+
+    | # given a wxApp instance called myWxAppInstance:
+    | from twisted.internet import wxsupport
+    | wxsupport.install(myWxAppInstance)
+
+Use Twisted's APIs for running and stopping the event loop, don't use
+wxPython's methods.
+
+On Windows the Twisted event loop might block when dialogs are open
+or menus are selected.
+
+Maintainer: Itamar Shtull-Trauring
+"""
+
+import warnings
+
+warnings.warn("wxsupport is not fully functional on Windows, wxreactor is better.")
+
+from twisted.internet import reactor
+
+
+class wxRunner:
+    """Make sure GUI events are handled."""
+
+    def __init__(self, app):
+        self.app = app
+
+    def run(self):
+        """
+        Execute pending WX events followed by WX idle events and
+        reschedule.
+        """
+        # run wx events
+        while self.app.Pending():
+            self.app.Dispatch()
+
+        # run wx idle events
+        self.app.ProcessIdle()
+        reactor.callLater(0.02, self.run)
+
+
+def install(app):
+    """Install the wxPython support, given a wxApp instance"""
+    runner = wxRunner(app)
+    reactor.callLater(0.02, runner.run)
+
+
+__all__ = ["install"]
diff --git a/contrib/python/Twisted/py3/twisted/logger/__init__.py b/contrib/python/Twisted/py3/twisted/logger/__init__.py
new file mode 100644
index 00000000000..62f2f71f4e3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/__init__.py
@@ -0,0 +1,135 @@
+# -*- test-case-name: twisted.logger.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Logger: Classes and functions to do granular logging.
+
+Example usage in a module C{some.module}::
+
+    from twisted.logger import Logger
+    log = Logger()
+
+    def handleData(data):
+        log.debug("Got data: {data!r}.", data=data)
+
+Or in a class::
+
+    from twisted.logger import Logger
+
+    class Foo:
+        log = Logger()
+
+        def oops(self, data):
+            self.log.error("Oops! Invalid data from server: {data!r}",
+                           data=data)
+
+C{Logger}s have namespaces, for which logging can be configured independently.
+Namespaces may be specified by passing in a C{namespace} argument to L{Logger}
+when instantiating it, but if none is given, the logger will derive its own
+namespace by using the module name of the callable that instantiated it, or, in
+the case of a class, by using the fully qualified name of the class.
+
+In the first example above, the namespace would be C{some.module}, and in the
+second example, it would be C{some.module.Foo}.
+
+@var globalLogPublisher: The L{LogPublisher} that all L{Logger} instances that
+    are not otherwise parameterized will publish events to by default.
+@var globalLogBeginner: The L{LogBeginner} used to activate the main log
+    observer, whether it's a log file, or an observer pointing at stderr.
+"""
+
+__all__ = [
+    # From ._levels
+    "InvalidLogLevelError",
+    "LogLevel",
+    # From ._format
+    "formatEvent",
+    "formatEventAsClassicLogText",
+    "formatTime",
+    "timeFormatRFC3339",
+    "eventAsText",
+    # From ._flatten
+    "extractField",
+    # From ._interfaces
+    "ILogObserver",
+    "LogEvent",
+    # From ._logger
+    "Logger",
+    "_loggerFor",
+    # From ._observer
+    "LogPublisher",
+    # From ._buffer
+    "LimitedHistoryLogObserver",
+    # From ._file
+    "FileLogObserver",
+    "textFileLogObserver",
+    # From ._filter
+    "PredicateResult",
+    "ILogFilterPredicate",
+    "FilteringLogObserver",
+    "LogLevelFilterPredicate",
+    # From ._stdlib
+    "STDLibLogObserver",
+    # From ._io
+    "LoggingFile",
+    # From ._legacy
+    "LegacyLogObserverWrapper",
+    # From ._global
+    "globalLogPublisher",
+    "globalLogBeginner",
+    "LogBeginner",
+    # From ._json
+    "eventAsJSON",
+    "eventFromJSON",
+    "jsonFileLogObserver",
+    "eventsFromJSONLogFile",
+    # From ._capture
+    "capturedLogs",
+]
+
+from ._levels import InvalidLogLevelError, LogLevel
+
+from ._flatten import extractField
+
+from ._format import (
+    formatEvent,
+    formatEventAsClassicLogText,
+    formatTime,
+    timeFormatRFC3339,
+    eventAsText,
+)
+
+from ._interfaces import ILogObserver, LogEvent
+
+from ._logger import Logger, _loggerFor
+
+from ._observer import LogPublisher
+
+from ._buffer import LimitedHistoryLogObserver
+
+from ._file import FileLogObserver, textFileLogObserver
+
+from ._filter import (
+    PredicateResult,
+    ILogFilterPredicate,
+    FilteringLogObserver,
+    LogLevelFilterPredicate,
+)
+
+from ._stdlib import STDLibLogObserver
+
+from ._io import LoggingFile
+
+from ._legacy import LegacyLogObserverWrapper
+
+from ._global import globalLogPublisher, globalLogBeginner, LogBeginner
+
+from ._json import (
+    eventAsJSON,
+    eventFromJSON,
+    jsonFileLogObserver,
+    eventsFromJSONLogFile,
+)
+
+from ._capture import capturedLogs
diff --git a/contrib/python/Twisted/py3/twisted/logger/_buffer.py b/contrib/python/Twisted/py3/twisted/logger/_buffer.py
new file mode 100644
index 00000000000..d5e514f18be
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_buffer.py
@@ -0,0 +1,54 @@
+# -*- test-case-name: twisted.logger.test.test_buffer -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Log observer that maintains a buffer.
+"""
+
+from collections import deque
+from typing import Deque, Optional
+
+from zope.interface import implementer
+
+from ._interfaces import ILogObserver, LogEvent
+
+_DEFAULT_BUFFER_MAXIMUM = 64 * 1024
+
+
+@implementer(ILogObserver)
+class LimitedHistoryLogObserver:
+    """
+    L{ILogObserver} that stores events in a buffer of a fixed size::
+
+        >>> from twisted.logger import LimitedHistoryLogObserver
+        >>> history = LimitedHistoryLogObserver(5)
+        >>> for n in range(10): history({'n': n})
+        ...
+        >>> repeats = []
+        >>> history.replayTo(repeats.append)
+        >>> len(repeats)
+        5
+        >>> repeats
+        [{'n': 5}, {'n': 6}, {'n': 7}, {'n': 8}, {'n': 9}]
+        >>>
+    """
+
+    def __init__(self, size: Optional[int] = _DEFAULT_BUFFER_MAXIMUM) -> None:
+        """
+        @param size: The maximum number of events to buffer.  If L{None}, the
+            buffer is unbounded.
+        """
+        self._buffer: Deque[LogEvent] = deque(maxlen=size)
+
+    def __call__(self, event: LogEvent) -> None:
+        self._buffer.append(event)
+
+    def replayTo(self, otherObserver: ILogObserver) -> None:
+        """
+        Re-play the buffered events to another log observer.
+
+        @param otherObserver: An observer to replay events to.
+        """
+        for event in self._buffer:
+            otherObserver(event)
diff --git a/contrib/python/Twisted/py3/twisted/logger/_capture.py b/contrib/python/Twisted/py3/twisted/logger/_capture.py
new file mode 100644
index 00000000000..9d3ce0e3abf
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_capture.py
@@ -0,0 +1,25 @@
+# -*- test-case-name: twisted.logger.test.test_capture -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Context manager for capturing logs.
+"""
+
+from contextlib import contextmanager
+from typing import Iterator, List, Sequence, cast
+
+from twisted.logger import globalLogPublisher
+from ._interfaces import ILogObserver, LogEvent
+
+
+@contextmanager
+def capturedLogs() -> Iterator[Sequence[LogEvent]]:
+    events: List[LogEvent] = []
+    observer = cast(ILogObserver, events.append)
+
+    globalLogPublisher.addObserver(observer)
+
+    yield events
+
+    globalLogPublisher.removeObserver(observer)
diff --git a/contrib/python/Twisted/py3/twisted/logger/_file.py b/contrib/python/Twisted/py3/twisted/logger/_file.py
new file mode 100644
index 00000000000..43ae32cd29a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_file.py
@@ -0,0 +1,77 @@
+# -*- test-case-name: twisted.logger.test.test_file -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+File log observer.
+"""
+
+from typing import IO, Any, Callable, Optional
+
+from zope.interface import implementer
+
+from twisted.python.compat import ioType
+from ._format import formatEventAsClassicLogText, formatTime, timeFormatRFC3339
+from ._interfaces import ILogObserver, LogEvent
+
+
+@implementer(ILogObserver)
+class FileLogObserver:
+    """
+    Log observer that writes to a file-like object.
+    """
+
+    def __init__(
+        self, outFile: IO[Any], formatEvent: Callable[[LogEvent], Optional[str]]
+    ) -> None:
+        """
+        @param outFile: A file-like object.  Ideally one should be passed which
+            accepts text data.  Otherwise, UTF-8 L{bytes} will be used.
+        @param formatEvent: A callable that formats an event.
+        """
+        if ioType(outFile) is not str:
+            self._encoding: Optional[str] = "utf-8"
+        else:
+            self._encoding = None
+
+        self._outFile = outFile
+        self.formatEvent = formatEvent
+
+    def __call__(self, event: LogEvent) -> None:
+        """
+        Write event to file.
+
+        @param event: An event.
+        """
+        text = self.formatEvent(event)
+
+        if text:
+            if self._encoding is None:
+                self._outFile.write(text)
+            else:
+                self._outFile.write(text.encode(self._encoding))
+            self._outFile.flush()
+
+
+def textFileLogObserver(
+    outFile: IO[Any], timeFormat: Optional[str] = timeFormatRFC3339
+) -> FileLogObserver:
+    """
+    Create a L{FileLogObserver} that emits text to a specified (writable)
+    file-like object.
+
+    @param outFile: A file-like object.  Ideally one should be passed which
+        accepts text data.  Otherwise, UTF-8 L{bytes} will be used.
+    @param timeFormat: The format to use when adding timestamp prefixes to
+        logged events.  If L{None}, or for events with no C{"log_timestamp"}
+        key, the default timestamp prefix of C{"-"} is used.
+
+    @return: A file log observer.
+    """
+
+    def formatEvent(event: LogEvent) -> Optional[str]:
+        return formatEventAsClassicLogText(
+            event, formatTime=lambda e: formatTime(e, timeFormat)
+        )
+
+    return FileLogObserver(outFile, formatEvent)
diff --git a/contrib/python/Twisted/py3/twisted/logger/_filter.py b/contrib/python/Twisted/py3/twisted/logger/_filter.py
new file mode 100644
index 00000000000..fa4220ea3e0
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_filter.py
@@ -0,0 +1,211 @@
+# -*- test-case-name: twisted.logger.test.test_filter -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Filtering log observer.
+"""
+
+from functools import partial
+from typing import Dict, Iterable
+
+from zope.interface import Interface, implementer
+
+from constantly import NamedConstant, Names  # type: ignore[import]
+
+from ._interfaces import ILogObserver, LogEvent
+from ._levels import InvalidLogLevelError, LogLevel
+from ._observer import bitbucketLogObserver
+
+
+class PredicateResult(Names):
+    """
+    Predicate results.
+
+    @see: L{LogLevelFilterPredicate}
+
+    @cvar yes: Log the specified event.  When this value is used,
+        L{FilteringLogObserver} will always log the message, without
+        evaluating other predicates.
+
+    @cvar no: Do not log the specified event.  When this value is used,
+        L{FilteringLogObserver} will I{not} log the message, without
+        evaluating other predicates.
+
+    @cvar maybe: Do not have an opinion on the event.  When this value is used,
+        L{FilteringLogObserver} will consider subsequent predicate results;
+        if returned by the last predicate being considered, then the event will
+        be logged.
+    """
+
+    yes = NamedConstant()
+    no = NamedConstant()
+    maybe = NamedConstant()
+
+
+class ILogFilterPredicate(Interface):
+    """
+    A predicate that determined whether an event should be logged.
+    """
+
+    def __call__(event: LogEvent) -> NamedConstant:
+        """
+        Determine whether an event should be logged.
+
+        @returns: a L{PredicateResult}.
+        """
+
+
+def shouldLogEvent(predicates: Iterable[ILogFilterPredicate], event: LogEvent) -> bool:
+    """
+    Determine whether an event should be logged, based on the result of
+    C{predicates}.
+
+    By default, the result is C{True}; so if there are no predicates,
+    everything will be logged.
+
+    If any predicate returns C{yes}, then we will immediately return C{True}.
+
+    If any predicate returns C{no}, then we will immediately return C{False}.
+
+    As predicates return C{maybe}, we keep calling the next predicate until we
+    run out, at which point we return C{True}.
+
+    @param predicates: The predicates to use.
+    @param event: An event
+
+    @return: True if the message should be forwarded on, C{False} if not.
+    """
+    for predicate in predicates:
+        result = predicate(event)
+        if result == PredicateResult.yes:
+            return True
+        if result == PredicateResult.no:
+            return False
+        if result == PredicateResult.maybe:
+            continue
+        raise TypeError(f"Invalid predicate result: {result!r}")
+    return True
+
+
+@implementer(ILogObserver)
+class FilteringLogObserver:
+    """
+    L{ILogObserver} that wraps another L{ILogObserver}, but filters out events
+    based on applying a series of L{ILogFilterPredicate}s.
+    """
+
+    def __init__(
+        self,
+        observer: ILogObserver,
+        predicates: Iterable[ILogFilterPredicate],
+        negativeObserver: ILogObserver = bitbucketLogObserver,
+    ) -> None:
+        """
+        @param observer: An observer to which this observer will forward
+            events when C{predictates} yield a positive result.
+        @param predicates: Predicates to apply to events before forwarding to
+            the wrapped observer.
+        @param negativeObserver: An observer to which this observer will
+            forward events when C{predictates} yield a negative result.
+        """
+        self._observer = observer
+        self._shouldLogEvent = partial(shouldLogEvent, list(predicates))
+        self._negativeObserver = negativeObserver
+
+    def __call__(self, event: LogEvent) -> None:
+        """
+        Forward to next observer if predicate allows it.
+        """
+        if self._shouldLogEvent(event):
+            if "log_trace" in event:
+                event["log_trace"].append((self, self._observer))
+            self._observer(event)
+        else:
+            self._negativeObserver(event)
+
+
+@implementer(ILogFilterPredicate)
+class LogLevelFilterPredicate:
+    """
+    L{ILogFilterPredicate} that filters out events with a log level lower than
+    the log level for the event's namespace.
+
+    Events that not not have a log level or namespace are also dropped.
+    """
+
+    def __init__(self, defaultLogLevel: NamedConstant = LogLevel.info) -> None:
+        """
+        @param defaultLogLevel: The default minimum log level.
+        """
+        self._logLevelsByNamespace: Dict[str, NamedConstant] = {}
+        self.defaultLogLevel = defaultLogLevel
+        self.clearLogLevels()
+
+    def logLevelForNamespace(self, namespace: str) -> NamedConstant:
+        """
+        Determine an appropriate log level for the given namespace.
+
+        This respects dots in namespaces; for example, if you have previously
+        invoked C{setLogLevelForNamespace("mypackage", LogLevel.debug)}, then
+        C{logLevelForNamespace("mypackage.subpackage")} will return
+        C{LogLevel.debug}.
+
+        @param namespace: A logging namespace.  Use C{""} for the default
+            namespace.
+
+        @return: The log level for the specified namespace.
+        """
+        if not namespace:
+            return self._logLevelsByNamespace[""]
+
+        if namespace in self._logLevelsByNamespace:
+            return self._logLevelsByNamespace[namespace]
+
+        segments = namespace.split(".")
+        index = len(segments) - 1
+
+        while index > 0:
+            namespace = ".".join(segments[:index])
+            if namespace in self._logLevelsByNamespace:
+                return self._logLevelsByNamespace[namespace]
+            index -= 1
+
+        return self._logLevelsByNamespace[""]
+
+    def setLogLevelForNamespace(self, namespace: str, level: NamedConstant) -> None:
+        """
+        Sets the log level for a logging namespace.
+
+        @param namespace: A logging namespace.
+        @param level: The log level for the given namespace.
+        """
+        if level not in LogLevel.iterconstants():
+            raise InvalidLogLevelError(level)
+
+        if namespace:
+            self._logLevelsByNamespace[namespace] = level
+        else:
+            self._logLevelsByNamespace[""] = level
+
+    def clearLogLevels(self) -> None:
+        """
+        Clears all log levels to the default.
+        """
+        self._logLevelsByNamespace.clear()
+        self._logLevelsByNamespace[""] = self.defaultLogLevel
+
+    def __call__(self, event: LogEvent) -> NamedConstant:
+        eventLevel = event.get("log_level", None)
+        if eventLevel is None:
+            return PredicateResult.no
+
+        namespace = event.get("log_namespace", "")
+        if not namespace:
+            return PredicateResult.no
+
+        namespaceLevel = self.logLevelForNamespace(namespace)
+        if eventLevel < namespaceLevel:
+            return PredicateResult.no
+
+        return PredicateResult.maybe
diff --git a/contrib/python/Twisted/py3/twisted/logger/_flatten.py b/contrib/python/Twisted/py3/twisted/logger/_flatten.py
new file mode 100644
index 00000000000..b79476aa248
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_flatten.py
@@ -0,0 +1,175 @@
+# -*- test-case-name: twisted.logger.test.test_flatten -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Code related to "flattening" events; that is, extracting a description of all
+relevant fields from the format string and persisting them for later
+examination.
+"""
+
+from collections import defaultdict
+from string import Formatter
+from typing import Any, Dict, Optional
+
+from ._interfaces import LogEvent
+
+aFormatter = Formatter()
+
+
+class KeyFlattener:
+    """
+    A L{KeyFlattener} computes keys for the things within curly braces in
+    PEP-3101-style format strings as parsed by L{string.Formatter.parse}.
+    """
+
+    def __init__(self) -> None:
+        """
+        Initialize a L{KeyFlattener}.
+        """
+        self.keys: Dict[str, int] = defaultdict(lambda: 0)
+
+    def flatKey(
+        self, fieldName: str, formatSpec: Optional[str], conversion: Optional[str]
+    ) -> str:
+        """
+        Compute a string key for a given field/format/conversion.
+
+        @param fieldName: A format field name.
+        @param formatSpec: A format spec.
+        @param conversion: A format field conversion type.
+
+        @return: A key specific to the given field, format and conversion, as
+            well as the occurrence of that combination within this
+            L{KeyFlattener}'s lifetime.
+        """
+        if formatSpec is None:
+            formatSpec = ""
+
+        if conversion is None:
+            conversion = ""
+
+        result = "{fieldName}!{conversion}:{formatSpec}".format(
+            fieldName=fieldName,
+            formatSpec=formatSpec,
+            conversion=conversion,
+        )
+        self.keys[result] += 1
+        n = self.keys[result]
+        if n != 1:
+            result += "/" + str(self.keys[result])
+        return result
+
+
+def flattenEvent(event: LogEvent) -> None:
+    """
+    Flatten the given event by pre-associating format fields with specific
+    objects and callable results in a L{dict} put into the C{"log_flattened"}
+    key in the event.
+
+    @param event: A logging event.
+    """
+    if event.get("log_format", None) is None:
+        return
+
+    if "log_flattened" in event:
+        fields = event["log_flattened"]
+    else:
+        fields = {}
+
+    keyFlattener = KeyFlattener()
+
+    for literalText, fieldName, formatSpec, conversion in aFormatter.parse(
+        event["log_format"]
+    ):
+        if fieldName is None:
+            continue
+
+        if conversion != "r":
+            conversion = "s"
+
+        flattenedKey = keyFlattener.flatKey(fieldName, formatSpec, conversion)
+        structuredKey = keyFlattener.flatKey(fieldName, formatSpec, "")
+
+        if flattenedKey in fields:
+            # We've already seen and handled this key
+            continue
+
+        if fieldName.endswith("()"):
+            fieldName = fieldName[:-2]
+            callit = True
+        else:
+            callit = False
+
+        field = aFormatter.get_field(fieldName, (), event)
+        fieldValue = field[0]
+
+        if conversion == "r":
+            conversionFunction = repr
+        else:  # Above: if conversion is not "r", it's "s"
+            conversionFunction = str
+
+        if callit:
+            fieldValue = fieldValue()
+
+        flattenedValue = conversionFunction(fieldValue)
+        fields[flattenedKey] = flattenedValue
+        fields[structuredKey] = fieldValue
+
+    if fields:
+        event["log_flattened"] = fields
+
+
+def extractField(field: str, event: LogEvent) -> Any:
+    """
+    Extract a given format field from the given event.
+
+    @param field: A string describing a format field or log key.  This is the
+        text that would normally fall between a pair of curly braces in a
+        format string: for example, C{"key[2].attribute"}.  If a conversion is
+        specified (the thing after the C{"!"} character in a format field) then
+        the result will always be str.
+    @param event: A log event.
+
+    @return: A value extracted from the field.
+
+    @raise KeyError: if the field is not found in the given event.
+    """
+    keyFlattener = KeyFlattener()
+
+    [[literalText, fieldName, formatSpec, conversion]] = aFormatter.parse(
+        "{" + field + "}"
+    )
+
+    assert fieldName is not None
+
+    key = keyFlattener.flatKey(fieldName, formatSpec, conversion)
+
+    if "log_flattened" not in event:
+        flattenEvent(event)
+
+    return event["log_flattened"][key]
+
+
+def flatFormat(event: LogEvent) -> str:
+    """
+    Format an event which has been flattened with L{flattenEvent}.
+
+    @param event: A logging event.
+
+    @return: A formatted string.
+    """
+    fieldValues = event["log_flattened"]
+    keyFlattener = KeyFlattener()
+    s = []
+
+    for literalText, fieldName, formatSpec, conversion in aFormatter.parse(
+        event["log_format"]
+    ):
+        s.append(literalText)
+
+        if fieldName is not None:
+            key = keyFlattener.flatKey(fieldName, formatSpec, conversion or "s")
+            s.append(str(fieldValues[key]))
+
+    return "".join(s)
diff --git a/contrib/python/Twisted/py3/twisted/logger/_format.py b/contrib/python/Twisted/py3/twisted/logger/_format.py
new file mode 100644
index 00000000000..4bc06ec40c4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_format.py
@@ -0,0 +1,373 @@
+# -*- test-case-name: twisted.logger.test.test_format -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Tools for formatting logging events.
+"""
+
+from datetime import datetime as DateTime
+from typing import Any, Callable, Iterator, Mapping, Optional, Union, cast
+
+from constantly import NamedConstant  # type: ignore[import]
+
+from twisted.python._tzhelper import FixedOffsetTimeZone
+from twisted.python.failure import Failure
+from twisted.python.reflect import safe_repr
+from ._flatten import aFormatter, flatFormat
+from ._interfaces import LogEvent
+
+timeFormatRFC3339 = "%Y-%m-%dT%H:%M:%S%z"
+
+
+def formatEvent(event: LogEvent) -> str:
+    """
+    Formats an event as text, using the format in C{event["log_format"]}.
+
+    This implementation should never raise an exception; if the formatting
+    cannot be done, the returned string will describe the event generically so
+    that a useful message is emitted regardless.
+
+    @param event: A logging event.
+
+    @return: A formatted string.
+    """
+    return eventAsText(
+        event,
+        includeTraceback=False,
+        includeTimestamp=False,
+        includeSystem=False,
+    )
+
+
+def formatUnformattableEvent(event: LogEvent, error: BaseException) -> str:
+    """
+    Formats an event as text that describes the event generically and a
+    formatting error.
+
+    @param event: A logging event.
+    @param error: The formatting error.
+
+    @return: A formatted string.
+    """
+    try:
+        return "Unable to format event {event!r}: {error}".format(
+            event=event, error=error
+        )
+    except BaseException:
+        # Yikes, something really nasty happened.
+        #
+        # Try to recover as much formattable data as possible; hopefully at
+        # least the namespace is sane, which will help you find the offending
+        # logger.
+        failure = Failure()
+
+        text = ", ".join(
+            " = ".join((safe_repr(key), safe_repr(value)))
+            for key, value in event.items()
+        )
+
+        return (
+            "MESSAGE LOST: unformattable object logged: {error}\n"
+            "Recoverable data: {text}\n"
+            "Exception during formatting:\n{failure}".format(
+                error=safe_repr(error), failure=failure, text=text
+            )
+        )
+
+
+def formatTime(
+    when: Optional[float],
+    timeFormat: Optional[str] = timeFormatRFC3339,
+    default: str = "-",
+) -> str:
+    """
+    Format a timestamp as text.
+
+    Example::
+
+        >>> from time import time
+        >>> from twisted.logger import formatTime
+        >>>
+        >>> t = time()
+        >>> formatTime(t)
+        u'2013-10-22T14:19:11-0700'
+        >>> formatTime(t, timeFormat="%Y/%W")  # Year and week number
+        u'2013/42'
+        >>>
+
+    @param when: A timestamp.
+    @param timeFormat: A time format.
+    @param default: Text to return if C{when} or C{timeFormat} is L{None}.
+
+    @return: A formatted time.
+    """
+    if timeFormat is None or when is None:
+        return default
+    else:
+        tz = FixedOffsetTimeZone.fromLocalTimeStamp(when)
+        datetime = DateTime.fromtimestamp(when, tz)
+        return str(datetime.strftime(timeFormat))
+
+
+def formatEventAsClassicLogText(
+    event: LogEvent, formatTime: Callable[[Optional[float]], str] = formatTime
+) -> Optional[str]:
+    """
+    Format an event as a line of human-readable text for, e.g. traditional log
+    file output.
+
+    The output format is C{"{timeStamp} [{system}] {event}\\n"}, where:
+
+        - C{timeStamp} is computed by calling the given C{formatTime} callable
+          on the event's C{"log_time"} value
+
+        - C{system} is the event's C{"log_system"} value, if set, otherwise,
+          the C{"log_namespace"} and C{"log_level"}, joined by a C{"#"}.  Each
+          defaults to C{"-"} is not set.
+
+        - C{event} is the event, as formatted by L{formatEvent}.
+
+    Example::
+
+        >>> from time import time
+        >>> from twisted.logger import formatEventAsClassicLogText
+        >>> from twisted.logger import LogLevel
+        >>>
+        >>> formatEventAsClassicLogText(dict())  # No format, returns None
+        >>> formatEventAsClassicLogText(dict(log_format="Hello!"))
+        u'- [-#-] Hello!\\n'
+        >>> formatEventAsClassicLogText(dict(
+        ...     log_format="Hello!",
+        ...     log_time=time(),
+        ...     log_namespace="my_namespace",
+        ...     log_level=LogLevel.info,
+        ... ))
+        u'2013-10-22T17:30:02-0700 [my_namespace#info] Hello!\\n'
+        >>> formatEventAsClassicLogText(dict(
+        ...     log_format="Hello!",
+        ...     log_time=time(),
+        ...     log_system="my_system",
+        ... ))
+        u'2013-11-11T17:22:06-0800 [my_system] Hello!\\n'
+        >>>
+
+    @param event: an event.
+    @param formatTime: A time formatter
+
+    @return: A formatted event, or L{None} if no output is appropriate.
+    """
+    eventText = eventAsText(event, formatTime=formatTime)
+    if not eventText:
+        return None
+    eventText = eventText.replace("\n", "\n\t")
+    return eventText + "\n"
+
+
+class CallMapping(Mapping[str, Any]):
+    """
+    Read-only mapping that turns a C{()}-suffix in key names into an invocation
+    of the key rather than a lookup of the key.
+
+    Implementation support for L{formatWithCall}.
+    """
+
+    def __init__(self, submapping: Mapping[str, Any]) -> None:
+        """
+        @param submapping: Another read-only mapping which will be used to look
+            up items.
+        """
+        self._submapping = submapping
+
+    def __iter__(self) -> Iterator[Any]:
+        return iter(self._submapping)
+
+    def __len__(self) -> int:
+        return len(self._submapping)
+
+    def __getitem__(self, key: str) -> Any:
+        """
+        Look up an item in the submapping for this L{CallMapping}, calling it
+        if C{key} ends with C{"()"}.
+        """
+        callit = key.endswith("()")
+        realKey = key[:-2] if callit else key
+        value = self._submapping[realKey]
+        if callit:
+            value = value()
+        return value
+
+
+def formatWithCall(formatString: str, mapping: Mapping[str, Any]) -> str:
+    """
+    Format a string like L{str.format}, but:
+
+        - taking only a name mapping; no positional arguments
+
+        - with the additional syntax that an empty set of parentheses
+          correspond to a formatting item that should be called, and its result
+          C{str}'d, rather than calling C{str} on the element directly as
+          normal.
+
+    For example::
+
+        >>> formatWithCall("{string}, {function()}.",
+        ...                dict(string="just a string",
+        ...                     function=lambda: "a function"))
+        'just a string, a function.'
+
+    @param formatString: A PEP-3101 format string.
+    @param mapping: A L{dict}-like object to format.
+
+    @return: The string with formatted values interpolated.
+    """
+    return str(aFormatter.vformat(formatString, (), CallMapping(mapping)))
+
+
+def _formatEvent(event: LogEvent) -> str:
+    """
+    Formats an event as a string, using the format in C{event["log_format"]}.
+
+    This implementation should never raise an exception; if the formatting
+    cannot be done, the returned string will describe the event generically so
+    that a useful message is emitted regardless.
+
+    @param event: A logging event.
+
+    @return: A formatted string.
+    """
+    try:
+        if "log_flattened" in event:
+            return flatFormat(event)
+
+        format = cast(Optional[Union[str, bytes]], event.get("log_format", None))
+        if format is None:
+            return ""
+
+        # Make sure format is text.
+        if isinstance(format, str):
+            pass
+        elif isinstance(format, bytes):
+            format = format.decode("utf-8")
+        else:
+            raise TypeError(f"Log format must be str, not {format!r}")
+
+        return formatWithCall(format, event)
+
+    except BaseException as e:
+        return formatUnformattableEvent(event, e)
+
+
+def _formatTraceback(failure: Failure) -> str:
+    """
+    Format a failure traceback, assuming UTF-8 and using a replacement
+    strategy for errors.  Every effort is made to provide a usable
+    traceback, but should not that not be possible, a message and the
+    captured exception are logged.
+
+    @param failure: The failure to retrieve a traceback from.
+
+    @return: The formatted traceback.
+    """
+    try:
+        traceback = failure.getTraceback()
+    except BaseException as e:
+        traceback = "(UNABLE TO OBTAIN TRACEBACK FROM EVENT):" + str(e)
+    return traceback
+
+
+def _formatSystem(event: LogEvent) -> str:
+    """
+    Format the system specified in the event in the "log_system" key if set,
+    otherwise the C{"log_namespace"} and C{"log_level"}, joined by a C{"#"}.
+    Each defaults to C{"-"} is not set.  If formatting fails completely,
+    "UNFORMATTABLE" is returned.
+
+    @param event: The event containing the system specification.
+
+    @return: A formatted string representing the "log_system" key.
+    """
+    system = cast(Optional[str], event.get("log_system", None))
+    if system is None:
+        level = cast(Optional[NamedConstant], event.get("log_level", None))
+        if level is None:
+            levelName = "-"
+        else:
+            levelName = level.name
+
+        system = "{namespace}#{level}".format(
+            namespace=cast(str, event.get("log_namespace", "-")),
+            level=levelName,
+        )
+    else:
+        try:
+            system = str(system)
+        except Exception:
+            system = "UNFORMATTABLE"
+    return system
+
+
+def eventAsText(
+    event: LogEvent,
+    includeTraceback: bool = True,
+    includeTimestamp: bool = True,
+    includeSystem: bool = True,
+    formatTime: Callable[[float], str] = formatTime,
+) -> str:
+    r"""
+    Format an event as text.  Optionally, attach timestamp, traceback, and
+    system information.
+
+    The full output format is:
+    C{"{timeStamp} [{system}] {event}\n{traceback}\n"} where:
+
+        - C{timeStamp} is the event's C{"log_time"} value formatted with
+          the provided C{formatTime} callable.
+
+        - C{system} is the event's C{"log_system"} value, if set, otherwise,
+          the C{"log_namespace"} and C{"log_level"}, joined by a C{"#"}.  Each
+          defaults to C{"-"} is not set.
+
+        - C{event} is the event, as formatted by L{formatEvent}.
+
+        - C{traceback} is the traceback if the event contains a
+          C{"log_failure"} key.  In the event the original traceback cannot
+          be formatted, a message indicating the failure will be substituted.
+
+    If the event cannot be formatted, and no traceback exists, an empty string
+    is returned, even if includeSystem or includeTimestamp are true.
+
+    @param event: A logging event.
+    @param includeTraceback: If true and a C{"log_failure"} key exists, append
+        a traceback.
+    @param includeTimestamp: If true include a formatted timestamp before the
+        event.
+    @param includeSystem:  If true, include the event's C{"log_system"} value.
+    @param formatTime: A time formatter
+
+    @return: A formatted string with specified options.
+
+    @since: Twisted 18.9.0
+    """
+    eventText = _formatEvent(event)
+    if includeTraceback and "log_failure" in event:
+        f = event["log_failure"]
+        traceback = _formatTraceback(f)
+        eventText = "\n".join((eventText, traceback))
+
+    if not eventText:
+        return eventText
+
+    timeStamp = ""
+    if includeTimestamp:
+        timeStamp = "".join([formatTime(cast(float, event.get("log_time", None))), " "])
+
+    system = ""
+    if includeSystem:
+        system = "".join(["[", _formatSystem(event), "]", " "])
+
+    return "{timeStamp}{system}{eventText}".format(
+        timeStamp=timeStamp,
+        system=system,
+        eventText=eventText,
+    )
diff --git a/contrib/python/Twisted/py3/twisted/logger/_global.py b/contrib/python/Twisted/py3/twisted/logger/_global.py
new file mode 100644
index 00000000000..8ae89baf728
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_global.py
@@ -0,0 +1,226 @@
+# -*- test-case-name: twisted.logger.test.test_global -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module includes process-global state associated with the logging system,
+and implementation of logic for managing that global state.
+"""
+
+import sys
+import warnings
+from typing import IO, Any, Iterable, Optional, Type
+
+from twisted.python.compat import currentframe
+from twisted.python.reflect import qual
+from ._buffer import LimitedHistoryLogObserver
+from ._file import FileLogObserver
+from ._filter import FilteringLogObserver, LogLevelFilterPredicate
+from ._format import eventAsText
+from ._interfaces import ILogObserver
+from ._io import LoggingFile
+from ._levels import LogLevel
+from ._logger import Logger
+from ._observer import LogPublisher
+
+MORE_THAN_ONCE_WARNING = (
+    "Warning: primary log target selected twice at <{fileNow}:{lineNow}> - "
+    "previously selected at <{fileThen}:{lineThen}>.  Remove one of the calls "
+    "to beginLoggingTo."
+)
+
+
+class LogBeginner:
+    """
+    A L{LogBeginner} holds state related to logging before logging has begun,
+    and begins logging when told to do so.  Logging "begins" when someone has
+    selected a set of observers, like, for example, a L{FileLogObserver} that
+    writes to a file on disk, or to standard output.
+
+    Applications will not typically need to instantiate this class, except
+    those which intend to initialize the global logging system themselves,
+    which may wish to instantiate this for testing.  The global instance for
+    the current process is exposed as
+    L{twisted.logger.globalLogBeginner}.
+
+    Before logging has begun, a L{LogBeginner} will:
+
+        1. Log any critical messages (e.g.: unhandled exceptions) to the given
+           file-like object.
+
+        2. Save (a limited number of) log events in a
+           L{LimitedHistoryLogObserver}.
+
+    @cvar _DEFAULT_BUFFER_SIZE: The default size for the initial log events
+        buffer.
+
+    @ivar _initialBuffer: A buffer of messages logged before logging began.
+    @ivar _publisher: The log publisher passed in to L{LogBeginner}'s
+        constructor.
+    @ivar _log: The logger used to log messages about the operation of the
+        L{LogBeginner} itself.
+    @ivar _stdio: An object with C{stderr} and C{stdout} attributes (like the
+        L{sys} module) which will be replaced when redirecting standard I/O.
+    @ivar _temporaryObserver: If not L{None}, an L{ILogObserver} that observes
+        events on C{_publisher} for this L{LogBeginner}.
+    """
+
+    _DEFAULT_BUFFER_SIZE = 200
+
+    def __init__(
+        self,
+        publisher: LogPublisher,
+        errorStream: IO[Any],
+        stdio: object,
+        warningsModule: Any,
+        initialBufferSize: Optional[int] = None,
+    ) -> None:
+        """
+        Initialize this L{LogBeginner}.
+
+        @param initialBufferSize: The size of the event buffer into which
+            events are collected until C{beginLoggingTo} is called.  Or
+            C{None} to use the default size.
+        """
+        if initialBufferSize is None:
+            initialBufferSize = self._DEFAULT_BUFFER_SIZE
+        self._initialBuffer = LimitedHistoryLogObserver(size=initialBufferSize)
+        self._publisher = publisher
+        self._log = Logger(observer=publisher)
+        self._stdio = stdio
+        self._warningsModule = warningsModule
+        self._temporaryObserver: Optional[ILogObserver] = LogPublisher(
+            self._initialBuffer,
+            FilteringLogObserver(
+                FileLogObserver(
+                    errorStream,
+                    lambda event: eventAsText(
+                        event,
+                        includeTimestamp=False,
+                        includeSystem=False,
+                    )
+                    + "\n",
+                ),
+                [LogLevelFilterPredicate(defaultLogLevel=LogLevel.critical)],
+            ),
+        )
+        self._previousBegin = ("", 0)
+        publisher.addObserver(self._temporaryObserver)
+        self._oldshowwarning = warningsModule.showwarning
+
+    def beginLoggingTo(
+        self,
+        observers: Iterable[ILogObserver],
+        discardBuffer: bool = False,
+        redirectStandardIO: bool = True,
+    ) -> None:
+        """
+        Begin logging to the given set of observers.  This will:
+
+            1. Add all the observers given in C{observers} to the
+               L{LogPublisher} associated with this L{LogBeginner}.
+
+            2. Optionally re-direct standard output and standard error streams
+               to the logging system.
+
+            3. Re-play any messages that were previously logged to that
+               publisher to the new observers, if C{discardBuffer} is not set.
+
+            4. Stop logging critical errors from the L{LogPublisher} as strings
+               to the C{errorStream} associated with this L{LogBeginner}, and
+               allow them to be logged normally.
+
+            5. Re-direct warnings from the L{warnings} module associated with
+               this L{LogBeginner} to log messages.
+
+        @note: Since a L{LogBeginner} is designed to encapsulate the transition
+            between process-startup and log-system-configuration, this method
+            is intended to be invoked I{once}.
+
+        @param observers: The observers to register.
+        @param discardBuffer: Whether to discard the buffer and not re-play it
+            to the added observers.  (This argument is provided mainly for
+            compatibility with legacy concerns.)
+        @param redirectStandardIO: If true, redirect standard output and
+            standard error to the observers.
+        """
+        caller = currentframe(1)
+        filename = caller.f_code.co_filename
+        lineno = caller.f_lineno
+
+        for observer in observers:
+            self._publisher.addObserver(observer)
+
+        if self._temporaryObserver is not None:
+            self._publisher.removeObserver(self._temporaryObserver)
+            if not discardBuffer:
+                self._initialBuffer.replayTo(self._publisher)
+            self._temporaryObserver = None
+            self._warningsModule.showwarning = self.showwarning
+        else:
+            previousFile, previousLine = self._previousBegin
+            self._log.warn(
+                MORE_THAN_ONCE_WARNING,
+                fileNow=filename,
+                lineNow=lineno,
+                fileThen=previousFile,
+                lineThen=previousLine,
+            )
+
+        self._previousBegin = (filename, lineno)
+        if redirectStandardIO:
+            streams = [("stdout", LogLevel.info), ("stderr", LogLevel.error)]
+        else:
+            streams = []
+
+        for stream, level in streams:
+            oldStream = getattr(self._stdio, stream)
+            loggingFile = LoggingFile(
+                logger=Logger(namespace=stream, observer=self._publisher),
+                level=level,
+                encoding=getattr(oldStream, "encoding", None),
+            )
+            setattr(self._stdio, stream, loggingFile)
+
+    def showwarning(
+        self,
+        message: str,
+        category: Type[Warning],
+        filename: str,
+        lineno: int,
+        file: Optional[IO[Any]] = None,
+        line: Optional[str] = None,
+    ) -> None:
+        """
+        Twisted-enabled wrapper around L{warnings.showwarning}.
+
+        If C{file} is L{None}, the default behaviour is to emit the warning to
+        the log system, otherwise the original L{warnings.showwarning} Python
+        function is called.
+
+        @param message: A warning message to emit.
+        @param category: A warning category to associate with C{message}.
+        @param filename: A file name for the source code file issuing the
+            warning.
+        @param lineno: A line number in the source file where the warning was
+            issued.
+        @param file: A file to write the warning message to.  If L{None},
+            write to L{sys.stderr}.
+        @param line: A line of source code to include with the warning message.
+            If L{None}, attempt to read the line from C{filename} and
+            C{lineno}.
+        """
+        if file is None:
+            self._log.warn(
+                "{filename}:{lineno}: {category}: {warning}",
+                warning=message,
+                category=qual(category),
+                filename=filename,
+                lineno=lineno,
+            )
+        else:
+            self._oldshowwarning(message, category, filename, lineno, file, line)
+
+
+globalLogPublisher = LogPublisher()
+globalLogBeginner = LogBeginner(globalLogPublisher, sys.stderr, sys, warnings)
diff --git a/contrib/python/Twisted/py3/twisted/logger/_interfaces.py b/contrib/python/Twisted/py3/twisted/logger/_interfaces.py
new file mode 100644
index 00000000000..496de1de540
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_interfaces.py
@@ -0,0 +1,63 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Logger interfaces.
+"""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple
+
+from zope.interface import Interface
+
+if TYPE_CHECKING:
+    from ._logger import Logger
+
+
+LogEvent = Dict[str, Any]
+LogTrace = List[Tuple["Logger", "ILogObserver"]]
+
+
+class ILogObserver(Interface):
+    """
+    An observer which can handle log events.
+
+    Unlike most interfaces within Twisted, an L{ILogObserver} I{must be
+    thread-safe}.  Log observers may be called indiscriminately from many
+    different threads, as any thread may wish to log a message at any time.
+    """
+
+    def __call__(event: LogEvent) -> None:
+        """
+        Log an event.
+
+        @param event: A dictionary with arbitrary keys as defined by the
+            application emitting logging events, as well as keys added by the
+            logging system.  The logging system reserves the right to set any
+            key beginning with the prefix C{"log_"}; applications should not
+            use any key so named.  Currently, the following keys are used by
+            the logging system in some way, if they are present (they are all
+            optional):
+
+                - C{"log_format"}: a PEP-3101-style format string which draws
+                  upon the keys in the event as its values, used to format the
+                  event for human consumption.
+
+                - C{"log_flattened"}: a dictionary mapping keys derived from
+                  the names and format values used in the C{"log_format"}
+                  string to their values.  This is used to preserve some
+                  structured information for use with
+                  L{twisted.logger.extractField}.
+
+                - C{"log_trace"}: A L{list} designed to capture information
+                  about which L{LogPublisher}s have observed the event.
+
+                - C{"log_level"}: a L{log level
+                  <twisted.logger.LogLevel>} constant, indicating the
+                  importance of and audience for this event.
+
+                - C{"log_namespace"}: a namespace for the emitter of the event,
+                  given as a L{str}.
+
+                - C{"log_system"}: a string indicating the network event or
+                  method call which resulted in the message being logged.
+        """
diff --git a/contrib/python/Twisted/py3/twisted/logger/_io.py b/contrib/python/Twisted/py3/twisted/logger/_io.py
new file mode 100644
index 00000000000..7a49718db71
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_io.py
@@ -0,0 +1,187 @@
+# -*- test-case-name: twisted.logger.test.test_io -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+File-like object that logs.
+"""
+
+import sys
+from typing import AnyStr, Iterable, Optional
+
+from constantly import NamedConstant  # type: ignore[import]
+from incremental import Version
+
+from twisted.python.deprecate import deprecatedProperty
+from ._levels import LogLevel
+from ._logger import Logger
+
+
+class LoggingFile:
+    """
+    File-like object that turns C{write()} calls into logging events.
+
+    Note that because event formats are L{str}, C{bytes} received via C{write()}
+    are converted to C{str}, which is the opposite of what C{file} does.
+
+    @ivar softspace: Attribute to make this class more file-like under Python 2;
+        value is zero or one.  Do not use.
+    """
+
+    _softspace = 0
+
+    @deprecatedProperty(Version("Twisted", 21, 2, 0))
+    def softspace(self):
+        return self._softspace
+
+    @softspace.setter  # type: ignore[no-redef]
+    def softspace(self, value):
+        self._softspace = value
+
+    def __init__(
+        self,
+        logger: Logger,
+        level: NamedConstant = LogLevel.info,
+        encoding: Optional[str] = None,
+    ) -> None:
+        """
+        @param logger: the logger to log through.
+        @param level: the log level to emit events with.
+        @param encoding: The encoding to expect when receiving bytes via
+            C{write()}.  If L{None}, use C{sys.getdefaultencoding()}.
+        """
+        self.level = level
+        self.log = logger
+
+        if encoding is None:
+            self._encoding = sys.getdefaultencoding()
+        else:
+            self._encoding = encoding
+
+        self._buffer = ""
+        self._closed = False
+
+    @property
+    def closed(self) -> bool:
+        """
+        Read-only property.  Is the file closed?
+
+        @return: true if closed, otherwise false.
+        """
+        return self._closed
+
+    @property
+    def encoding(self) -> str:
+        """
+        Read-only property.   File encoding.
+
+        @return: an encoding.
+        """
+        return self._encoding
+
+    @property
+    def mode(self) -> str:
+        """
+        Read-only property.  File mode.
+
+        @return: "w"
+        """
+        return "w"
+
+    @property
+    def newlines(self) -> None:
+        """
+        Read-only property.  Types of newlines encountered.
+
+        @return: L{None}
+        """
+        return None
+
+    @property
+    def name(self) -> str:
+        """
+        The name of this file; a repr-style string giving information about its
+        namespace.
+
+        @return: A file name.
+        """
+        return "<{} {}#{}>".format(
+            self.__class__.__name__,
+            self.log.namespace,
+            self.level.name,
+        )
+
+    def close(self) -> None:
+        """
+        Close this file so it can no longer be written to.
+        """
+        self._closed = True
+
+    def flush(self) -> None:
+        """
+        No-op; this file does not buffer.
+        """
+        pass
+
+    def fileno(self) -> int:
+        """
+        Returns an invalid file descriptor, since this is not backed by an FD.
+
+        @return: C{-1}
+        """
+        return -1
+
+    def isatty(self) -> bool:
+        """
+        A L{LoggingFile} is not a TTY.
+
+        @return: C{False}
+        """
+        return False
+
+    def write(self, message: AnyStr) -> None:
+        """
+        Log the given message.
+
+        @param message: The message to write.
+        """
+        if self._closed:
+            raise ValueError("I/O operation on closed file")
+
+        if isinstance(message, bytes):
+            text = message.decode(self._encoding)
+        else:
+            text = message
+
+        lines = (self._buffer + text).split("\n")
+        self._buffer = lines[-1]
+        lines = lines[0:-1]
+
+        for line in lines:
+            self.log.emit(self.level, format="{log_io}", log_io=line)
+
+    def writelines(self, lines: Iterable[AnyStr]) -> None:
+        """
+        Log each of the given lines as a separate message.
+
+        @param lines: Data to write.
+        """
+        for line in lines:
+            self.write(line)
+
+    def _unsupported(self, *args: object) -> None:
+        """
+        Template for unsupported operations.
+
+        @param args: Arguments.
+        """
+        raise OSError("unsupported operation")
+
+    read = _unsupported
+    next = _unsupported
+    readline = _unsupported
+    readlines = _unsupported
+    xreadlines = _unsupported
+    seek = _unsupported
+    tell = _unsupported
+    truncate = _unsupported
diff --git a/contrib/python/Twisted/py3/twisted/logger/_json.py b/contrib/python/Twisted/py3/twisted/logger/_json.py
new file mode 100644
index 00000000000..2ecdd43045a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_json.py
@@ -0,0 +1,285 @@
+# -*- test-case-name: twisted.logger.test.test_json -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Tools for saving and loading log events in a structured format.
+"""
+
+from json import dumps, loads
+from typing import IO, Any, AnyStr, Dict, Iterable, Optional, Union, cast
+from uuid import UUID
+
+from constantly import NamedConstant  # type: ignore[import]
+
+from twisted.python.failure import Failure
+from ._file import FileLogObserver
+from ._flatten import flattenEvent
+from ._interfaces import LogEvent
+from ._levels import LogLevel
+from ._logger import Logger
+
+log = Logger()
+
+
+JSONDict = Dict[str, Any]
+
+
+def failureAsJSON(failure: Failure) -> JSONDict:
+    """
+    Convert a failure to a JSON-serializable data structure.
+
+    @param failure: A failure to serialize.
+
+    @return: a mapping of strings to ... stuff, mostly reminiscent of
+        L{Failure.__getstate__}
+    """
+    return dict(
+        failure.__getstate__(),
+        type=dict(
+            __module__=failure.type.__module__,
+            __name__=failure.type.__name__,
+        ),
+    )
+
+
+def failureFromJSON(failureDict: JSONDict) -> Failure:
+    """
+    Load a L{Failure} from a dictionary deserialized from JSON.
+
+    @param failureDict: a JSON-deserialized object like one previously returned
+        by L{failureAsJSON}.
+
+    @return: L{Failure}
+    """
+    f = Failure.__new__(Failure)
+    typeInfo = failureDict["type"]
+    failureDict["type"] = type(typeInfo["__name__"], (), typeInfo)
+    f.__dict__ = failureDict
+    return f
+
+
+classInfo = [
+    (
+        lambda level: (
+            isinstance(level, NamedConstant)
+            and getattr(LogLevel, level.name, None) is level
+        ),
+        UUID("02E59486-F24D-46AD-8224-3ACDF2A5732A"),
+        lambda level: dict(name=level.name),
+        lambda level: getattr(LogLevel, level["name"], None),
+    ),
+    (
+        lambda o: isinstance(o, Failure),
+        UUID("E76887E2-20ED-49BF-A8F8-BA25CC586F2D"),
+        failureAsJSON,
+        failureFromJSON,
+    ),
+]
+
+
+uuidToLoader = {uuid: loader for (predicate, uuid, saver, loader) in classInfo}
+
+
+def objectLoadHook(aDict: JSONDict) -> object:
+    """
+    Dictionary-to-object-translation hook for certain value types used within
+    the logging system.
+
+    @see: the C{object_hook} parameter to L{json.load}
+
+    @param aDict: A dictionary loaded from a JSON object.
+
+    @return: C{aDict} itself, or the object represented by C{aDict}
+    """
+    if "__class_uuid__" in aDict:
+        return uuidToLoader[UUID(aDict["__class_uuid__"])](aDict)
+    return aDict
+
+
+def objectSaveHook(pythonObject: object) -> JSONDict:
+    """
+    Object-to-serializable hook for certain value types used within the logging
+    system.
+
+    @see: the C{default} parameter to L{json.dump}
+
+    @param pythonObject: Any object.
+
+    @return: If the object is one of the special types the logging system
+        supports, a specially-formatted dictionary; otherwise, a marker
+        dictionary indicating that it could not be serialized.
+    """
+    for predicate, uuid, saver, loader in classInfo:
+        if predicate(pythonObject):
+            result = saver(pythonObject)
+            result["__class_uuid__"] = str(uuid)
+            return result
+    return {"unpersistable": True}
+
+
+def eventAsJSON(event: LogEvent) -> str:
+    """
+    Encode an event as JSON, flattening it if necessary to preserve as much
+    structure as possible.
+
+    Not all structure from the log event will be preserved when it is
+    serialized.
+
+    @param event: A log event dictionary.
+
+    @return: A string of the serialized JSON; note that this will contain no
+        newline characters, and may thus safely be stored in a line-delimited
+        file.
+    """
+
+    def default(unencodable: object) -> Union[JSONDict, str]:
+        """
+        Serialize an object not otherwise serializable by L{dumps}.
+
+        @param unencodable: An unencodable object.
+
+        @return: C{unencodable}, serialized
+        """
+        if isinstance(unencodable, bytes):
+            return unencodable.decode("charmap")
+        return objectSaveHook(unencodable)
+
+    flattenEvent(event)
+    return dumps(event, default=default, skipkeys=True)
+
+
+def eventFromJSON(eventText: str) -> JSONDict:
+    """
+    Decode a log event from JSON.
+
+    @param eventText: The output of a previous call to L{eventAsJSON}
+
+    @return: A reconstructed version of the log event.
+    """
+    return cast(JSONDict, loads(eventText, object_hook=objectLoadHook))
+
+
+def jsonFileLogObserver(
+    outFile: IO[Any], recordSeparator: str = "\x1e"
+) -> FileLogObserver:
+    """
+    Create a L{FileLogObserver} that emits JSON-serialized events to a
+    specified (writable) file-like object.
+
+    Events are written in the following form::
+
+        RS + JSON + NL
+
+    C{JSON} is the serialized event, which is JSON text.  C{NL} is a newline
+    (C{"\\n"}).  C{RS} is a record separator.  By default, this is a single
+    RS character (C{"\\x1e"}), which makes the default output conform to the
+    IETF draft document "draft-ietf-json-text-sequence-13".
+
+    @param outFile: A file-like object.  Ideally one should be passed which
+        accepts L{str} data.  Otherwise, UTF-8 L{bytes} will be used.
+    @param recordSeparator: The record separator to use.
+
+    @return: A file log observer.
+    """
+    return FileLogObserver(
+        outFile, lambda event: f"{recordSeparator}{eventAsJSON(event)}\n"
+    )
+
+
+def eventsFromJSONLogFile(
+    inFile: IO[Any],
+    recordSeparator: Optional[str] = None,
+    bufferSize: int = 4096,
+) -> Iterable[LogEvent]:
+    """
+    Load events from a file previously saved with L{jsonFileLogObserver}.
+    Event records that are truncated or otherwise unreadable are ignored.
+
+    @param inFile: A (readable) file-like object.  Data read from C{inFile}
+        should be L{str} or UTF-8 L{bytes}.
+    @param recordSeparator: The expected record separator.
+        If L{None}, attempt to automatically detect the record separator from
+        one of C{"\\x1e"} or C{""}.
+    @param bufferSize: The size of the read buffer used while reading from
+        C{inFile}.
+
+    @return: Log events as read from C{inFile}.
+    """
+
+    def asBytes(s: AnyStr) -> bytes:
+        if isinstance(s, bytes):
+            return s
+        else:
+            return s.encode("utf-8")
+
+    def eventFromBytearray(record: bytearray) -> Optional[LogEvent]:
+        try:
+            text = bytes(record).decode("utf-8")
+        except UnicodeDecodeError:
+            log.error(
+                "Unable to decode UTF-8 for JSON record: {record!r}",
+                record=bytes(record),
+            )
+            return None
+
+        try:
+            return eventFromJSON(text)
+        except ValueError:
+            log.error("Unable to read JSON record: {record!r}", record=bytes(record))
+            return None
+
+    if recordSeparator is None:
+        first = asBytes(inFile.read(1))
+
+        if first == b"\x1e":
+            # This looks json-text-sequence compliant.
+            recordSeparatorBytes = first
+        else:
+            # Default to simpler newline-separated stream, which does not use
+            # a record separator.
+            recordSeparatorBytes = b""
+
+    else:
+        recordSeparatorBytes = asBytes(recordSeparator)
+        first = b""
+
+    if recordSeparatorBytes == b"":
+        recordSeparatorBytes = b"\n"  # Split on newlines below
+
+        eventFromRecord = eventFromBytearray
+
+    else:
+
+        def eventFromRecord(record: bytearray) -> Optional[LogEvent]:
+            if record[-1] == ord("\n"):
+                return eventFromBytearray(record)
+            else:
+                log.error(
+                    "Unable to read truncated JSON record: {record!r}",
+                    record=bytes(record),
+                )
+            return None
+
+    buffer = bytearray(first)
+
+    while True:
+        newData = inFile.read(bufferSize)
+
+        if not newData:
+            if len(buffer) > 0:
+                event = eventFromRecord(buffer)
+                if event is not None:
+                    yield event
+            break
+
+        buffer += asBytes(newData)
+        records = buffer.split(recordSeparatorBytes)
+
+        for record in records[:-1]:
+            if len(record) > 0:
+                event = eventFromRecord(record)
+                if event is not None:
+                    yield event
+
+        buffer = records[-1]
diff --git a/contrib/python/Twisted/py3/twisted/logger/_legacy.py b/contrib/python/Twisted/py3/twisted/logger/_legacy.py
new file mode 100644
index 00000000000..2847bc7a406
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_legacy.py
@@ -0,0 +1,147 @@
+# -*- test-case-name: twisted.logger.test.test_legacy -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Integration with L{twisted.python.log}.
+"""
+
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
+
+from zope.interface import implementer
+
+from ._format import formatEvent
+from ._interfaces import ILogObserver, LogEvent
+from ._levels import LogLevel
+from ._stdlib import StringifiableFromEvent, fromStdlibLogLevelMapping
+
+if TYPE_CHECKING:
+    from twisted.python.log import ILogObserver as ILegacyLogObserver
+
+
+@implementer(ILogObserver)
+class LegacyLogObserverWrapper:
+    """
+    L{ILogObserver} that wraps a L{twisted.python.log.ILogObserver}.
+
+    Received (new-style) events are modified prior to forwarding to
+    the legacy observer to ensure compatibility with observers that
+    expect legacy events.
+    """
+
+    def __init__(self, legacyObserver: "ILegacyLogObserver") -> None:
+        """
+        @param legacyObserver: a legacy observer to which this observer will
+            forward events.
+        """
+        self.legacyObserver = legacyObserver
+
+    def __repr__(self) -> str:
+        return "{self.__class__.__name__}({self.legacyObserver})".format(self=self)
+
+    def __call__(self, event: LogEvent) -> None:
+        """
+        Forward events to the legacy observer after editing them to
+        ensure compatibility.
+
+        @param event: an event
+        """
+
+        # The "message" key is required by textFromEventDict()
+        if "message" not in event:
+            event["message"] = ()
+
+        if "time" not in event:
+            event["time"] = event["log_time"]
+
+        if "system" not in event:
+            event["system"] = event.get("log_system", "-")
+
+        # Format new style -> old style
+        if "format" not in event and event.get("log_format", None) is not None:
+            # Create an object that implements __str__() in order to defer the
+            # work of formatting until it's needed by a legacy log observer.
+            event["format"] = "%(log_legacy)s"
+            event["log_legacy"] = StringifiableFromEvent(event.copy())
+
+            # In the old-style system, the 'message' key always holds a tuple
+            # of messages. If we find the 'message' key here to not be a
+            # tuple, it has been passed as new-style parameter. We drop it
+            # here because we render it using the old-style 'format' key,
+            # which otherwise doesn't get precedence, and the original event
+            # has been copied above.
+            if not isinstance(event["message"], tuple):
+                event["message"] = ()
+
+        # From log.failure() -> isError blah blah
+        if "log_failure" in event:
+            if "failure" not in event:
+                event["failure"] = event["log_failure"]
+            if "isError" not in event:
+                event["isError"] = 1
+            if "why" not in event:
+                event["why"] = formatEvent(event)
+        elif "isError" not in event:
+            if event["log_level"] in (LogLevel.error, LogLevel.critical):
+                event["isError"] = 1
+            else:
+                event["isError"] = 0
+
+        self.legacyObserver(event)
+
+
+def publishToNewObserver(
+    observer: ILogObserver,
+    eventDict: Dict[str, Any],
+    textFromEventDict: Callable[[Dict[str, Any]], Optional[str]],
+) -> None:
+    """
+    Publish an old-style (L{twisted.python.log}) event to a new-style
+    (L{twisted.logger}) observer.
+
+    @note: It's possible that a new-style event was sent to a
+        L{LegacyLogObserverWrapper}, and may now be getting sent back to a
+        new-style observer.  In this case, it's already a new-style event,
+        adapted to also look like an old-style event, and we don't need to
+        tweak it again to be a new-style event, hence this checks for
+        already-defined new-style keys.
+
+    @param observer: A new-style observer to handle this event.
+    @param eventDict: An L{old-style <twisted.python.log>}, log event.
+    @param textFromEventDict: callable that can format an old-style event as a
+        string.  Passed here rather than imported to avoid circular dependency.
+    """
+
+    if "log_time" not in eventDict:
+        eventDict["log_time"] = eventDict["time"]
+
+    if "log_format" not in eventDict:
+        text = textFromEventDict(eventDict)
+        if text is not None:
+            eventDict["log_text"] = text
+            eventDict["log_format"] = "{log_text}"
+
+    if "log_level" not in eventDict:
+        if "logLevel" in eventDict:
+            try:
+                level = fromStdlibLogLevelMapping[eventDict["logLevel"]]
+            except KeyError:
+                level = None
+        elif "isError" in eventDict:
+            if eventDict["isError"]:
+                level = LogLevel.critical
+            else:
+                level = LogLevel.info
+        else:
+            level = LogLevel.info
+
+        if level is not None:
+            eventDict["log_level"] = level
+
+    if "log_namespace" not in eventDict:
+        eventDict["log_namespace"] = "log_legacy"
+
+    if "log_system" not in eventDict and "system" in eventDict:
+        eventDict["log_system"] = eventDict["system"]
+
+    observer(eventDict)
diff --git a/contrib/python/Twisted/py3/twisted/logger/_levels.py b/contrib/python/Twisted/py3/twisted/logger/_levels.py
new file mode 100644
index 00000000000..800a549f88c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_levels.py
@@ -0,0 +1,81 @@
+# -*- test-case-name: twisted.logger.test.test_levels -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Log levels.
+"""
+
+from constantly import NamedConstant, Names  # type: ignore[import]
+
+
+class InvalidLogLevelError(Exception):
+    """
+    Someone tried to use a L{LogLevel} that is unknown to the logging system.
+    """
+
+    def __init__(self, level: NamedConstant) -> None:
+        """
+        @param level: A log level from L{LogLevel}.
+        """
+        super().__init__(str(level))
+        self.level = level
+
+
+class LogLevel(Names):
+    """
+    Constants describing log levels.
+
+    @cvar debug: Debugging events: Information of use to a developer of the
+        software, not generally of interest to someone running the software
+        unless they are attempting to diagnose a software issue.
+
+    @cvar info: Informational events: Routine information about the status of
+        an application, such as incoming connections, startup of a subsystem,
+        etc.
+
+    @cvar warn: Warning events: Events that may require greater attention than
+        informational events but are not a systemic failure condition, such as
+        authorization failures, bad data from a network client, etc.  Such
+        events are of potential interest to system administrators, and should
+        ideally be phrased in such a way, or documented, so as to indicate an
+        action that an administrator might take to mitigate the warning.
+
+    @cvar error: Error conditions: Events indicating a systemic failure, such
+        as programming errors in the form of unhandled exceptions, loss of
+        connectivity to an external system without which no useful work can
+        proceed, such as a database or API endpoint, or resource exhaustion.
+        Similarly to warnings, errors that are related to operational
+        parameters may be actionable to system administrators and should
+        provide references to resources which an administrator might use to
+        resolve them.
+
+    @cvar critical: Critical failures: Errors indicating systemic failure (ie.
+        service outage), data corruption, imminent data loss, etc. which must
+        be handled immediately.  This includes errors unanticipated by the
+        software, such as unhandled exceptions, wherein the cause and
+        consequences are unknown.
+    """
+
+    debug = NamedConstant()
+    info = NamedConstant()
+    warn = NamedConstant()
+    error = NamedConstant()
+    critical = NamedConstant()
+
+    @classmethod
+    def levelWithName(cls, name: str) -> NamedConstant:
+        """
+        Get the log level with the given name.
+
+        @param name: The name of a log level.
+
+        @return: The L{LogLevel} with the specified C{name}.
+
+        @raise InvalidLogLevelError: if the C{name} does not name a valid log
+            level.
+        """
+        try:
+            return cls.lookupByName(name)
+        except ValueError:
+            raise InvalidLogLevelError(name)
diff --git a/contrib/python/Twisted/py3/twisted/logger/_logger.py b/contrib/python/Twisted/py3/twisted/logger/_logger.py
new file mode 100644
index 00000000000..cc428d87af1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_logger.py
@@ -0,0 +1,269 @@
+# -*- test-case-name: twisted.logger.test.test_logger -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Logger class.
+"""
+
+from time import time
+from typing import Any, Optional, cast
+
+from twisted.python.compat import currentframe
+from twisted.python.failure import Failure
+from ._interfaces import ILogObserver, LogTrace
+from ._levels import InvalidLogLevelError, LogLevel
+
+
+class Logger:
+    """
+    A L{Logger} emits log messages to an observer.  You should instantiate it
+    as a class or module attribute, as documented in L{this module's
+    documentation <twisted.logger>}.
+
+    @ivar namespace: the namespace for this logger
+    @ivar source: The object which is emitting events via this logger
+    @ivar observer: The observer that this logger will send events to.
+    """
+
+    @staticmethod
+    def _namespaceFromCallingContext() -> str:
+        """
+        Derive a namespace from the module containing the caller's caller.
+
+        @return: the fully qualified python name of a module.
+        """
+        try:
+            return cast(str, currentframe(2).f_globals["__name__"])
+        except KeyError:
+            return "<unknown>"
+
+    def __init__(
+        self,
+        namespace: Optional[str] = None,
+        source: Optional[object] = None,
+        observer: Optional["ILogObserver"] = None,
+    ) -> None:
+        """
+        @param namespace: The namespace for this logger.  Uses a dotted
+            notation, as used by python modules.  If not L{None}, then the name
+            of the module of the caller is used.
+        @param source: The object which is emitting events via this
+            logger; this is automatically set on instances of a class
+            if this L{Logger} is an attribute of that class.
+        @param observer: The observer that this logger will send events to.
+            If L{None}, use the L{global log publisher <globalLogPublisher>}.
+        """
+        if namespace is None:
+            namespace = self._namespaceFromCallingContext()
+
+        self.namespace = namespace
+        self.source = source
+
+        if observer is None:
+            from ._global import globalLogPublisher
+
+            self.observer: ILogObserver = globalLogPublisher
+        else:
+            self.observer = observer
+
+    def __get__(self, instance: object, owner: Optional[type] = None) -> "Logger":
+        """
+        When used as a descriptor, i.e.::
+
+            # File: athing.py
+            class Something:
+                log = Logger()
+                def hello(self):
+                    self.log.info("Hello")
+
+        a L{Logger}'s namespace will be set to the name of the class it is
+        declared on.  In the above example, the namespace would be
+        C{athing.Something}.
+
+        Additionally, its source will be set to the actual object referring to
+        the L{Logger}.  In the above example, C{Something.log.source} would be
+        C{Something}, and C{Something().log.source} would be an instance of
+        C{Something}.
+        """
+        assert owner is not None
+
+        if instance is None:
+            source: Any = owner
+        else:
+            source = instance
+
+        return self.__class__(
+            ".".join([owner.__module__, owner.__name__]),
+            source,
+            observer=self.observer,
+        )
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.namespace!r}>"
+
+    def emit(
+        self, level: LogLevel, format: Optional[str] = None, **kwargs: object
+    ) -> None:
+        """
+        Emit a log event to all log observers at the given level.
+
+        @param level: a L{LogLevel}
+        @param format: a message format using new-style (PEP 3101)
+            formatting.  The logging event (which is a L{dict}) is
+            used to render this format string.
+        @param kwargs: additional key/value pairs to include in the event.
+            Note that values which are later mutated may result in
+            non-deterministic behavior from observers that schedule work for
+            later execution.
+        """
+        if level not in LogLevel.iterconstants():
+            self.failure(
+                "Got invalid log level {invalidLevel!r} in {logger}.emit().",
+                Failure(InvalidLogLevelError(level)),
+                invalidLevel=level,
+                logger=self,
+            )
+            return
+
+        event = kwargs
+        event.update(
+            log_logger=self,
+            log_level=level,
+            log_namespace=self.namespace,
+            log_source=self.source,
+            log_format=format,
+            log_time=time(),
+        )
+
+        if "log_trace" in event:
+            cast(LogTrace, event["log_trace"]).append((self, self.observer))
+
+        self.observer(event)
+
+    def failure(
+        self,
+        format: str,
+        failure: Optional[Failure] = None,
+        level: LogLevel = LogLevel.critical,
+        **kwargs: object,
+    ) -> None:
+        """
+        Log a failure and emit a traceback.
+
+        For example::
+
+            try:
+                frob(knob)
+            except Exception:
+                log.failure("While frobbing {knob}", knob=knob)
+
+        or::
+
+            d = deferredFrob(knob)
+            d.addErrback(lambda f: log.failure("While frobbing {knob}",
+                                               f, knob=knob))
+
+        This method is generally meant to capture unexpected exceptions in
+        code; an exception that is caught and handled somehow should be logged,
+        if appropriate, via L{Logger.error} instead.  If some unknown exception
+        occurs and your code doesn't know how to handle it, as in the above
+        example, then this method provides a means to describe the failure in
+        nerd-speak.  This is done at L{LogLevel.critical} by default, since no
+        corrective guidance can be offered to an user/administrator, and the
+        impact of the condition is unknown.
+
+        @param format: a message format using new-style (PEP 3101) formatting.
+            The logging event (which is a L{dict}) is used to render this
+            format string.
+        @param failure: a L{Failure} to log.  If L{None}, a L{Failure} is
+            created from the exception in flight.
+        @param level: a L{LogLevel} to use.
+        @param kwargs: additional key/value pairs to include in the event.
+            Note that values which are later mutated may result in
+            non-deterministic behavior from observers that schedule work for
+            later execution.
+        """
+        if failure is None:
+            failure = Failure()
+
+        self.emit(level, format, log_failure=failure, **kwargs)
+
+    def debug(self, format: Optional[str] = None, **kwargs: object) -> None:
+        """
+        Emit a log event at log level L{LogLevel.debug}.
+
+        @param format: a message format using new-style (PEP 3101) formatting.
+            The logging event (which is a L{dict}) is used to render this
+            format string.
+
+        @param kwargs: additional key/value pairs to include in the event.
+            Note that values which are later mutated may result in
+            non-deterministic behavior from observers that schedule work for
+            later execution.
+        """
+        self.emit(LogLevel.debug, format, **kwargs)
+
+    def info(self, format: Optional[str] = None, **kwargs: object) -> None:
+        """
+        Emit a log event at log level L{LogLevel.info}.
+
+        @param format: a message format using new-style (PEP 3101) formatting.
+            The logging event (which is a L{dict}) is used to render this
+            format string.
+
+        @param kwargs: additional key/value pairs to include in the event.
+            Note that values which are later mutated may result in
+            non-deterministic behavior from observers that schedule work for
+            later execution.
+        """
+        self.emit(LogLevel.info, format, **kwargs)
+
+    def warn(self, format: Optional[str] = None, **kwargs: object) -> None:
+        """
+        Emit a log event at log level L{LogLevel.warn}.
+
+        @param format: a message format using new-style (PEP 3101) formatting.
+            The logging event (which is a L{dict}) is used to render this
+            format string.
+
+        @param kwargs: additional key/value pairs to include in the event.
+            Note that values which are later mutated may result in
+            non-deterministic behavior from observers that schedule work for
+            later execution.
+        """
+        self.emit(LogLevel.warn, format, **kwargs)
+
+    def error(self, format: Optional[str] = None, **kwargs: object) -> None:
+        """
+        Emit a log event at log level L{LogLevel.error}.
+
+        @param format: a message format using new-style (PEP 3101) formatting.
+            The logging event (which is a L{dict}) is used to render this
+            format string.
+
+        @param kwargs: additional key/value pairs to include in the event.
+            Note that values which are later mutated may result in
+            non-deterministic behavior from observers that schedule work for
+            later execution.
+        """
+        self.emit(LogLevel.error, format, **kwargs)
+
+    def critical(self, format: Optional[str] = None, **kwargs: object) -> None:
+        """
+        Emit a log event at log level L{LogLevel.critical}.
+
+        @param format: a message format using new-style (PEP 3101) formatting.
+            The logging event (which is a L{dict}) is used to render this
+            format string.
+
+        @param kwargs: additional key/value pairs to include in the event.
+            Note that values which are later mutated may result in
+            non-deterministic behavior from observers that schedule work for
+            later execution.
+        """
+        self.emit(LogLevel.critical, format, **kwargs)
+
+
+_log = Logger()
+_loggerFor = lambda obj: _log.__get__(obj, obj.__class__)
diff --git a/contrib/python/Twisted/py3/twisted/logger/_observer.py b/contrib/python/Twisted/py3/twisted/logger/_observer.py
new file mode 100644
index 00000000000..86f89c37b45
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_observer.py
@@ -0,0 +1,112 @@
+# -*- test-case-name: twisted.logger.test.test_observer -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Basic log observers.
+"""
+
+from typing import Callable, Optional
+
+from zope.interface import implementer
+
+from twisted.python.failure import Failure
+from ._interfaces import ILogObserver, LogEvent
+from ._logger import Logger
+
+OBSERVER_DISABLED = (
+    "Temporarily disabling observer {observer} due to exception: {log_failure}"
+)
+
+
+@implementer(ILogObserver)
+class LogPublisher:
+    """
+    I{ILogObserver} that fans out events to other observers.
+
+    Keeps track of a set of L{ILogObserver} objects and forwards
+    events to each.
+    """
+
+    def __init__(self, *observers: ILogObserver) -> None:
+        self._observers = list(observers)
+        self.log = Logger(observer=self)
+
+    def addObserver(self, observer: ILogObserver) -> None:
+        """
+        Registers an observer with this publisher.
+
+        @param observer: An L{ILogObserver} to add.
+        """
+        if not callable(observer):
+            raise TypeError(f"Observer is not callable: {observer!r}")
+        if observer not in self._observers:
+            self._observers.append(observer)
+
+    def removeObserver(self, observer: ILogObserver) -> None:
+        """
+        Unregisters an observer with this publisher.
+
+        @param observer: An L{ILogObserver} to remove.
+        """
+        try:
+            self._observers.remove(observer)
+        except ValueError:
+            pass
+
+    def __call__(self, event: LogEvent) -> None:
+        """
+        Forward events to contained observers.
+        """
+        if "log_trace" not in event:
+            trace: Optional[Callable[[ILogObserver], None]] = None
+
+        else:
+
+            def trace(observer: ILogObserver) -> None:
+                """
+                Add tracing information for an observer.
+
+                @param observer: an observer being forwarded to
+                """
+                event["log_trace"].append((self, observer))
+
+        brokenObservers = []
+
+        for observer in self._observers:
+            if trace is not None:
+                trace(observer)
+
+            try:
+                observer(event)
+            except Exception:
+                brokenObservers.append((observer, Failure()))
+
+        for brokenObserver, failure in brokenObservers:
+            errorLogger = self._errorLoggerForObserver(brokenObserver)
+            errorLogger.failure(
+                OBSERVER_DISABLED,
+                failure=failure,
+                observer=brokenObserver,
+            )
+
+    def _errorLoggerForObserver(self, observer: ILogObserver) -> Logger:
+        """
+        Create an error-logger based on this logger, which does not contain the
+        given bad observer.
+
+        @param observer: The observer which previously had an error.
+
+        @return: A L{Logger} without the given observer.
+        """
+        errorPublisher = LogPublisher(
+            *(obs for obs in self._observers if obs is not observer)
+        )
+        return Logger(observer=errorPublisher)
+
+
+@implementer(ILogObserver)
+def bitbucketLogObserver(event: LogEvent) -> None:
+    """
+    I{ILogObserver} that does nothing with the events it sees.
+    """
diff --git a/contrib/python/Twisted/py3/twisted/logger/_stdlib.py b/contrib/python/Twisted/py3/twisted/logger/_stdlib.py
new file mode 100644
index 00000000000..030b643883e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_stdlib.py
@@ -0,0 +1,131 @@
+# -*- test-case-name: twisted.logger.test.test_stdlib -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Integration with Python standard library logging.
+"""
+
+import logging as stdlibLogging
+from typing import Mapping, Tuple
+
+from zope.interface import implementer
+
+from constantly import NamedConstant  # type: ignore[import]
+
+from twisted.python.compat import currentframe
+from ._format import formatEvent
+from ._interfaces import ILogObserver, LogEvent
+from ._levels import LogLevel
+
+# Mappings to Python's logging module
+toStdlibLogLevelMapping: Mapping[NamedConstant, int] = {
+    LogLevel.debug: stdlibLogging.DEBUG,
+    LogLevel.info: stdlibLogging.INFO,
+    LogLevel.warn: stdlibLogging.WARNING,
+    LogLevel.error: stdlibLogging.ERROR,
+    LogLevel.critical: stdlibLogging.CRITICAL,
+}
+
+
+def _reverseLogLevelMapping() -> Mapping[int, NamedConstant]:
+    """
+    Reverse the above mapping, adding both the numerical keys used above and
+    the corresponding string keys also used by python logging.
+    @return: the reversed mapping
+    """
+    mapping = {}
+    for logLevel, pyLogLevel in toStdlibLogLevelMapping.items():
+        mapping[pyLogLevel] = logLevel
+        mapping[stdlibLogging.getLevelName(pyLogLevel)] = logLevel
+    return mapping
+
+
+fromStdlibLogLevelMapping = _reverseLogLevelMapping()
+
+
+@implementer(ILogObserver)
+class STDLibLogObserver:
+    """
+    Log observer that writes to the python standard library's C{logging}
+    module.
+
+    @note: Warning: specific logging configurations (example: network) can lead
+        to this observer blocking.  Nothing is done here to prevent that, so be
+        sure to not to configure the standard library logging module to block
+        when used in conjunction with this module: code within Twisted, such as
+        twisted.web, assumes that logging does not block.
+
+    @cvar defaultStackDepth: This is the default number of frames that it takes
+        to get from L{STDLibLogObserver} through the logging module, plus one;
+        in other words, the number of frames if you were to call a
+        L{STDLibLogObserver} directly.  This is useful to use as an offset for
+        the C{stackDepth} parameter to C{__init__}, to add frames for other
+        publishers.
+    """
+
+    defaultStackDepth = 4
+
+    def __init__(
+        self, name: str = "twisted", stackDepth: int = defaultStackDepth
+    ) -> None:
+        """
+        @param name: logger identifier.
+        @param stackDepth: The depth of the stack to investigate for caller
+            metadata.
+        """
+        self.logger = stdlibLogging.getLogger(name)
+        self.logger.findCaller = self._findCaller  # type: ignore[assignment]
+        self.stackDepth = stackDepth
+
+    def _findCaller(
+        self, stackInfo: bool = False, stackLevel: int = 1
+    ) -> Tuple[str, int, str, None]:
+        """
+        Based on the stack depth passed to this L{STDLibLogObserver}, identify
+        the calling function.
+
+        @param stackInfo: Whether or not to construct stack information.
+            (Currently ignored.)
+        @param stackLevel: The number of stack frames to skip when determining
+            the caller (currently ignored; use stackDepth on the instance).
+
+        @return: Depending on Python version, either a 3-tuple of (filename,
+            lineno, name) or a 4-tuple of that plus stack information.
+        """
+        f = currentframe(self.stackDepth)
+        co = f.f_code
+        extra = (None,)
+        return (co.co_filename, f.f_lineno, co.co_name) + extra
+
+    def __call__(self, event: LogEvent) -> None:
+        """
+        Format an event and bridge it to Python logging.
+        """
+        level = event.get("log_level", LogLevel.info)
+        failure = event.get("log_failure")
+        if failure is None:
+            excInfo = None
+        else:
+            excInfo = (failure.type, failure.value, failure.getTracebackObject())
+        stdlibLevel = toStdlibLogLevelMapping.get(level, stdlibLogging.INFO)
+        self.logger.log(stdlibLevel, StringifiableFromEvent(event), exc_info=excInfo)
+
+
+class StringifiableFromEvent:
+    """
+    An object that implements C{__str__()} in order to defer the work of
+    formatting until it's converted into a C{str}.
+    """
+
+    def __init__(self, event: LogEvent) -> None:
+        """
+        @param event: An event.
+        """
+        self.event = event
+
+    def __str__(self) -> str:
+        return formatEvent(self.event)
+
+    def __bytes__(self) -> bytes:
+        return str(self).encode("utf-8")
diff --git a/contrib/python/Twisted/py3/twisted/logger/_util.py b/contrib/python/Twisted/py3/twisted/logger/_util.py
new file mode 100644
index 00000000000..e8f02ddd225
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/logger/_util.py
@@ -0,0 +1,51 @@
+# -*- test-case-name: twisted.logger.test.test_util -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Logging utilities.
+"""
+
+from typing import List
+
+from ._interfaces import LogTrace
+from ._logger import Logger
+
+
+def formatTrace(trace: LogTrace) -> str:
+    """
+    Format a trace (that is, the contents of the C{log_trace} key of a log
+    event) as a visual indication of the message's propagation through various
+    observers.
+
+    @param trace: the contents of the C{log_trace} key from an event.
+
+    @return: A multi-line string with indentation and arrows indicating the
+        flow of the message through various observers.
+    """
+
+    def formatWithName(obj: object) -> str:
+        if hasattr(obj, "name"):
+            return f"{obj} ({obj.name})"
+        else:
+            return f"{obj}"
+
+    result = []
+    lineage: List[Logger] = []
+
+    for parent, child in trace:
+        if not lineage or lineage[-1] is not parent:
+            if parent in lineage:
+                while lineage[-1] is not parent:
+                    lineage.pop()
+
+            else:
+                if not lineage:
+                    result.append(f"{formatWithName(parent)}\n")
+
+                lineage.append(parent)
+
+        result.append("  " * len(lineage))
+        result.append(f"-> {formatWithName(child)}\n")
+
+    return "".join(result)
diff --git a/contrib/python/Twisted/py3/twisted/mail/__init__.py b/contrib/python/Twisted/py3/twisted/mail/__init__.py
new file mode 100644
index 00000000000..0f1604e8a5f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Mail: Servers and clients for POP3, ESMTP, and IMAP.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/mail/_cred.py b/contrib/python/Twisted/py3/twisted/mail/_cred.py
new file mode 100644
index 00000000000..0a9442627d3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/_cred.py
@@ -0,0 +1,105 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Credential managers for L{twisted.mail}.
+"""
+
+
+import hashlib
+import hmac
+
+from zope.interface import implementer
+
+from twisted.cred import credentials
+from twisted.mail._except import IllegalClientResponse
+from twisted.mail.interfaces import IChallengeResponse, IClientAuthentication
+from twisted.python.compat import nativeString
+
+
+@implementer(IClientAuthentication)
+class CramMD5ClientAuthenticator:
+    def __init__(self, user):
+        self.user = user
+
+    def getName(self):
+        return b"CRAM-MD5"
+
+    def challengeResponse(self, secret, chal):
+        response = hmac.HMAC(secret, chal, digestmod=hashlib.md5).hexdigest()
+        return self.user + b" " + response.encode("ascii")
+
+
+@implementer(IClientAuthentication)
+class LOGINAuthenticator:
+    def __init__(self, user):
+        self.user = user
+        self.challengeResponse = self.challengeUsername
+
+    def getName(self):
+        return b"LOGIN"
+
+    def challengeUsername(self, secret, chal):
+        # Respond to something like "Username:"
+        self.challengeResponse = self.challengeSecret
+        return self.user
+
+    def challengeSecret(self, secret, chal):
+        # Respond to something like "Password:"
+        return secret
+
+
+@implementer(IClientAuthentication)
+class PLAINAuthenticator:
+    def __init__(self, user):
+        self.user = user
+
+    def getName(self):
+        return b"PLAIN"
+
+    def challengeResponse(self, secret, chal):
+        return b"\0" + self.user + b"\0" + secret
+
+
+@implementer(IChallengeResponse)
+class LOGINCredentials(credentials.UsernamePassword):
+    def __init__(self):
+        self.challenges = [b"Password\0", b"User Name\0"]
+        self.responses = [b"password", b"username"]
+        credentials.UsernamePassword.__init__(self, None, None)
+
+    def getChallenge(self):
+        return self.challenges.pop()
+
+    def setResponse(self, response):
+        setattr(self, nativeString(self.responses.pop()), response)
+
+    def moreChallenges(self):
+        return bool(self.challenges)
+
+
+@implementer(IChallengeResponse)
+class PLAINCredentials(credentials.UsernamePassword):
+    def __init__(self):
+        credentials.UsernamePassword.__init__(self, None, None)
+
+    def getChallenge(self):
+        return b""
+
+    def setResponse(self, response):
+        parts = response.split(b"\0")
+        if len(parts) != 3:
+            raise IllegalClientResponse("Malformed Response - wrong number of parts")
+        useless, self.username, self.password = parts
+
+    def moreChallenges(self):
+        return False
+
+
+__all__ = [
+    "CramMD5ClientAuthenticator",
+    "LOGINCredentials",
+    "LOGINAuthenticator",
+    "PLAINCredentials",
+    "PLAINAuthenticator",
+]
diff --git a/contrib/python/Twisted/py3/twisted/mail/_except.py b/contrib/python/Twisted/py3/twisted/mail/_except.py
new file mode 100644
index 00000000000..ce01aedeb52
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/_except.py
@@ -0,0 +1,350 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Exceptions in L{twisted.mail}.
+"""
+
+from typing import Optional
+
+
+class IMAP4Exception(Exception):
+    pass
+
+
+class IllegalClientResponse(IMAP4Exception):
+    pass
+
+
+class IllegalOperation(IMAP4Exception):
+    pass
+
+
+class IllegalMailboxEncoding(IMAP4Exception):
+    pass
+
+
+class MailboxException(IMAP4Exception):
+    pass
+
+
+class MailboxCollision(MailboxException):
+    def __str__(self) -> str:
+        return "Mailbox named %s already exists" % self.args
+
+
+class NoSuchMailbox(MailboxException):
+    def __str__(self) -> str:
+        return "No mailbox named %s exists" % self.args
+
+
+class ReadOnlyMailbox(MailboxException):
+    def __str__(self) -> str:
+        return "Mailbox open in read-only state"
+
+
+class UnhandledResponse(IMAP4Exception):
+    pass
+
+
+class NegativeResponse(IMAP4Exception):
+    pass
+
+
+class NoSupportedAuthentication(IMAP4Exception):
+    def __init__(self, serverSupports, clientSupports):
+        IMAP4Exception.__init__(self, "No supported authentication schemes available")
+        self.serverSupports = serverSupports
+        self.clientSupports = clientSupports
+
+    def __str__(self) -> str:
+        return IMAP4Exception.__str__(
+            self
+        ) + ": Server supports {!r}, client supports {!r}".format(
+            self.serverSupports,
+            self.clientSupports,
+        )
+
+
+class IllegalServerResponse(IMAP4Exception):
+    pass
+
+
+class IllegalIdentifierError(IMAP4Exception):
+    pass
+
+
+class IllegalQueryError(IMAP4Exception):
+    pass
+
+
+class MismatchedNesting(IMAP4Exception):
+    pass
+
+
+class MismatchedQuoting(IMAP4Exception):
+    pass
+
+
+class SMTPError(Exception):
+    pass
+
+
+class SMTPClientError(SMTPError):
+    """
+    Base class for SMTP client errors.
+    """
+
+    def __init__(
+        self,
+        code: int,
+        resp: bytes,
+        log: Optional[bytes] = None,
+        addresses: Optional[object] = None,
+        isFatal: bool = False,
+        retry: bool = False,
+    ):
+        """
+        @param code: The SMTP response code associated with this error.
+        @param resp: The string response associated with this error.
+        @param log: A string log of the exchange leading up to and including
+            the error.
+        @param isFatal: A boolean indicating whether this connection can
+            proceed or not. If True, the connection will be dropped.
+        @param retry: A boolean indicating whether the delivery should be
+            retried. If True and the factory indicates further retries are
+            desirable, they will be attempted, otherwise the delivery will be
+            failed.
+        """
+        if isinstance(resp, str):  # type: ignore[unreachable]
+            resp = resp.encode("utf-8")  # type: ignore[unreachable]
+
+        if isinstance(log, str):
+            log = log.encode("utf-8")  # type: ignore[unreachable]
+
+        self.code = code
+        self.resp = resp
+        self.log = log
+        self.addresses = addresses
+        self.isFatal = isFatal
+        self.retry = retry
+
+    def __str__(self) -> str:
+        return self.__bytes__().decode("utf-8")
+
+    def __bytes__(self) -> bytes:
+        if self.code > 0:
+            res = [f"{self.code:03d} ".encode() + self.resp]
+        else:
+            res = [self.resp]
+        if self.log:
+            res.append(self.log)
+            res.append(b"")
+        return b"\n".join(res)
+
+
+class ESMTPClientError(SMTPClientError):
+    """
+    Base class for ESMTP client errors.
+    """
+
+
+class EHLORequiredError(ESMTPClientError):
+    """
+    The server does not support EHLO.
+
+    This is considered a non-fatal error (the connection will not be dropped).
+    """
+
+
+class AUTHRequiredError(ESMTPClientError):
+    """
+    Authentication was required but the server does not support it.
+
+    This is considered a non-fatal error (the connection will not be dropped).
+    """
+
+
+class TLSRequiredError(ESMTPClientError):
+    """
+    Transport security was required but the server does not support it.
+
+    This is considered a non-fatal error (the connection will not be dropped).
+    """
+
+
+class AUTHDeclinedError(ESMTPClientError):
+    """
+    The server rejected our credentials.
+
+    Either the username, password, or challenge response
+    given to the server was rejected.
+
+    This is considered a non-fatal error (the connection will not be
+    dropped).
+    """
+
+
+class AuthenticationError(ESMTPClientError):
+    """
+    An error occurred while authenticating.
+
+    Either the server rejected our request for authentication or the
+    challenge received was malformed.
+
+    This is considered a non-fatal error (the connection will not be
+    dropped).
+    """
+
+
+class SMTPTLSError(ESMTPClientError):
+    """
+    An error occurred while negiotiating for transport security.
+
+    This is considered a non-fatal error (the connection will not be dropped).
+    """
+
+
+class SMTPConnectError(SMTPClientError):
+    """
+    Failed to connect to the mail exchange host.
+
+    This is considered a fatal error.  A retry will be made.
+    """
+
+    def __init__(self, code, resp, log=None, addresses=None, isFatal=True, retry=True):
+        SMTPClientError.__init__(self, code, resp, log, addresses, isFatal, retry)
+
+
+class SMTPTimeoutError(SMTPClientError):
+    """
+    Failed to receive a response from the server in the expected time period.
+
+    This is considered a fatal error.  A retry will be made.
+    """
+
+    def __init__(self, code, resp, log=None, addresses=None, isFatal=True, retry=True):
+        SMTPClientError.__init__(self, code, resp, log, addresses, isFatal, retry)
+
+
+class SMTPProtocolError(SMTPClientError):
+    """
+    The server sent a mangled response.
+
+    This is considered a fatal error.  A retry will not be made.
+    """
+
+    def __init__(self, code, resp, log=None, addresses=None, isFatal=True, retry=False):
+        SMTPClientError.__init__(self, code, resp, log, addresses, isFatal, retry)
+
+
+class SMTPDeliveryError(SMTPClientError):
+    """
+    Indicates that a delivery attempt has had an error.
+    """
+
+
+class SMTPServerError(SMTPError):
+    def __init__(self, code, resp):
+        self.code = code
+        self.resp = resp
+
+    def __str__(self) -> str:
+        return "%.3d %s" % (self.code, self.resp)
+
+
+class SMTPAddressError(SMTPServerError):
+    def __init__(self, addr, code, resp):
+        from twisted.mail.smtp import Address
+
+        SMTPServerError.__init__(self, code, resp)
+        self.addr = Address(addr)
+
+    def __str__(self) -> str:
+        return "%.3d <%s>... %s" % (self.code, self.addr, self.resp)
+
+
+class SMTPBadRcpt(SMTPAddressError):
+    def __init__(self, addr, code=550, resp="Cannot receive for specified address"):
+        SMTPAddressError.__init__(self, addr, code, resp)
+
+
+class SMTPBadSender(SMTPAddressError):
+    def __init__(self, addr, code=550, resp="Sender not acceptable"):
+        SMTPAddressError.__init__(self, addr, code, resp)
+
+
+class AddressError(SMTPError):
+    """
+    Parse error in address
+    """
+
+
+class POP3Error(Exception):
+    """
+    The base class for POP3 errors.
+    """
+
+    pass
+
+
+class _POP3MessageDeleted(Exception):
+    """
+    An internal control-flow error which indicates that a deleted message was
+    requested.
+    """
+
+
+class POP3ClientError(Exception):
+    """
+    The base class for all exceptions raised by POP3Client.
+    """
+
+
+class InsecureAuthenticationDisallowed(POP3ClientError):
+    """
+    An error indicating secure authentication was required but no mechanism
+    could be found.
+    """
+
+
+class TLSError(POP3ClientError):
+    """
+    An error indicating secure authentication was required but either the
+    transport does not support TLS or no TLS context factory was supplied.
+    """
+
+
+class TLSNotSupportedError(POP3ClientError):
+    """
+    An error indicating secure authentication was required but the server does
+    not support TLS.
+    """
+
+
+class ServerErrorResponse(POP3ClientError):
+    """
+    An error indicating that the server returned an error response to a
+    request.
+
+    @ivar consumer: See L{__init__}
+    """
+
+    def __init__(self, reason, consumer=None):
+        """
+        @type reason: L{bytes}
+        @param reason: The server response minus the status indicator.
+
+        @type consumer: callable that takes L{object}
+        @param consumer: The function meant to handle the values for a
+            multi-line response.
+        """
+        POP3ClientError.__init__(self, reason)
+        self.consumer = consumer
+
+
+class LineTooLong(POP3ClientError):
+    """
+    An error indicating that the server sent a line which exceeded the
+    maximum line length (L{LineOnlyReceiver.MAX_LENGTH}).
+    """
diff --git a/contrib/python/Twisted/py3/twisted/mail/_pop3client.py b/contrib/python/Twisted/py3/twisted/mail/_pop3client.py
new file mode 100644
index 00000000000..08efe1ec545
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/_pop3client.py
@@ -0,0 +1,1235 @@
+# -*- test-case-name: twisted.mail.test.test_pop3client -*-
+# Copyright (c) 2001-2004 Divmod Inc.
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A POP3 client protocol implementation.
+
+Don't use this module directly.  Use twisted.mail.pop3 instead.
+
+@author: Jp Calderone
+"""
+
+import re
+from hashlib import md5
+from typing import List
+
+from twisted.internet import defer, error, interfaces
+from twisted.mail._except import (
+    InsecureAuthenticationDisallowed,
+    LineTooLong,
+    ServerErrorResponse,
+    TLSError,
+    TLSNotSupportedError,
+)
+from twisted.protocols import basic, policies
+from twisted.python import log
+
+OK = b"+OK"
+ERR = b"-ERR"
+
+
+class _ListSetter:
+    """
+    A utility class to construct a list from a multi-line response accounting
+    for deleted messages.
+
+    POP3 responses sometimes occur in the form of a list of lines containing
+    two pieces of data, a message index and a value of some sort.  When a
+    message is deleted, it is omitted from these responses.  The L{setitem}
+    method of this class is meant to be called with these two values.  In the
+    cases where indices are skipped, it takes care of padding out the missing
+    values with L{None}.
+
+    @ivar L: See L{__init__}
+    """
+
+    def __init__(self, L):
+        """
+        @type L: L{list} of L{object}
+        @param L: The list being constructed.  An empty list should be
+            passed in.
+        """
+        self.L = L
+
+    def setitem(self, itemAndValue):
+        """
+        Add the value at the specified position, padding out missing entries.
+
+        @type itemAndValue: C{tuple}
+        @param itemAndValue: A tuple of (item, value).  The I{item} is the 0-based
+        index in the list at which the value should be placed.  The value is
+        is an L{object} to put in the list.
+        """
+        (item, value) = itemAndValue
+        diff = item - len(self.L) + 1
+        if diff > 0:
+            self.L.extend([None] * diff)
+        self.L[item] = value
+
+
+def _statXform(line):
+    """
+    Parse the response to a STAT command.
+
+    @type line: L{bytes}
+    @param line: The response from the server to a STAT command minus the
+        status indicator.
+
+    @rtype: 2-L{tuple} of (0) L{int}, (1) L{int}
+    @return: The number of messages in the mailbox and the size of the mailbox.
+    """
+    numMsgs, totalSize = line.split(None, 1)
+    return int(numMsgs), int(totalSize)
+
+
+def _listXform(line):
+    """
+    Parse a line of the response to a LIST command.
+
+    The line from the LIST response consists of a 1-based message number
+    followed by a size.
+
+    @type line: L{bytes}
+    @param line: A non-initial line from the multi-line response to a LIST
+        command.
+
+    @rtype: 2-L{tuple} of (0) L{int}, (1) L{int}
+    @return: The 0-based index of the message and the size of the message.
+    """
+    index, size = line.split(None, 1)
+    return int(index) - 1, int(size)
+
+
+def _uidXform(line):
+    """
+    Parse a line of the response to a UIDL command.
+
+    The line from the UIDL response consists of a 1-based message number
+    followed by a unique id.
+
+    @type line: L{bytes}
+    @param line: A non-initial line from the multi-line response to a UIDL
+        command.
+
+    @rtype: 2-L{tuple} of (0) L{int}, (1) L{bytes}
+    @return: The 0-based index of the message and the unique identifier
+        for the message.
+    """
+    index, uid = line.split(None, 1)
+    return int(index) - 1, uid
+
+
+def _codeStatusSplit(line):
+    """
+    Parse the first line of a multi-line server response.
+
+    @type line: L{bytes}
+    @param line: The first line of a multi-line server response.
+
+    @rtype: 2-tuple of (0) L{bytes}, (1) L{bytes}
+    @return: The status indicator and the rest of the server response.
+    """
+    parts = line.split(b" ", 1)
+    if len(parts) == 1:
+        return parts[0], b""
+    return parts
+
+
+def _dotUnquoter(line):
+    """
+    Remove a byte-stuffed termination character at the beginning of a line if
+    present.
+
+    When the termination character (C{'.'}) appears at the beginning of a line,
+    the server byte-stuffs it by adding another termination character to
+    avoid confusion with the terminating sequence (C{'.\\r\\n'}).
+
+    @type line: L{bytes}
+    @param line: A received line.
+
+    @rtype: L{bytes}
+    @return: The line without the byte-stuffed termination character at the
+        beginning if it was present. Otherwise, the line unchanged.
+    """
+    if line.startswith(b".."):
+        return line[1:]
+    return line
+
+
+class POP3Client(basic.LineOnlyReceiver, policies.TimeoutMixin):
+    """
+    A POP3 client protocol.
+
+    Instances of this class provide a convenient, efficient API for
+    retrieving and deleting messages from a POP3 server.
+
+    This API provides a pipelining interface but POP3 pipelining
+    on the network is not yet supported.
+
+    @type startedTLS: L{bool}
+    @ivar startedTLS: An indication of whether TLS has been negotiated
+        successfully.
+
+    @type allowInsecureLogin: L{bool}
+    @ivar allowInsecureLogin: An indication of whether plaintext login should
+        be allowed when the server offers no authentication challenge and the
+        transport does not offer any protection via encryption.
+
+    @type serverChallenge: L{bytes} or L{None}
+    @ivar serverChallenge: The challenge received in the server greeting.
+
+    @type timeout: L{int}
+    @ivar timeout: The number of seconds to wait on a response from the server
+        before timing out a connection.  If the number is <= 0, no timeout
+        checking will be performed.
+
+    @type _capCache: L{None} or L{dict} mapping L{bytes}
+        to L{list} of L{bytes} and/or L{bytes} to L{None}
+    @ivar _capCache: The cached server capabilities.  Capabilities are not
+        allowed to change during the session (except when TLS is negotiated),
+        so the first response to a capabilities command can be used for
+        later lookups.
+
+    @type _challengeMagicRe: L{Pattern <re.Pattern.search>}
+    @ivar _challengeMagicRe: A regular expression which matches the
+        challenge in the server greeting.
+
+    @type _blockedQueue: L{None} or L{list} of 3-L{tuple}
+        of (0) L{Deferred <defer.Deferred>}, (1) callable which results
+        in a L{Deferred <defer.Deferred>}, (2) L{tuple}
+    @ivar _blockedQueue: A list of blocked commands.  While a command is
+        awaiting a response from the server, other commands are blocked.  When
+        no command is outstanding, C{_blockedQueue} is set to L{None}.
+        Otherwise, it contains a list of information about blocked commands.
+        Each list entry provides the following information about a blocked
+        command: the deferred that should be called when the response to the
+        command is received, the function that sends the command, and the
+        arguments to the function.
+
+    @type _waiting: L{Deferred <defer.Deferred>} or
+        L{None}
+    @ivar _waiting: A deferred which fires when the response to the
+        outstanding command is received from the server.
+
+    @type _timedOut: L{bool}
+    @ivar _timedOut: An indication of whether the connection was dropped
+        because of a timeout.
+
+    @type _greetingError: L{bytes} or L{None}
+    @ivar _greetingError: The server greeting minus the status indicator, when
+        the connection was dropped because of an error in the server greeting.
+        Otherwise, L{None}.
+
+    @type state: L{bytes}
+    @ivar state: The state which indicates what type of response is expected
+        from the server.  Valid states are: 'WELCOME', 'WAITING', 'SHORT',
+        'LONG_INITIAL', 'LONG'.
+
+    @type _xform: L{None} or callable that takes L{bytes}
+        and returns L{object}
+    @ivar _xform: The transform function which is used to convert each
+        line of a multi-line response into usable values for use by the
+        consumer function.  If L{None}, each line of the multi-line response
+        is sent directly to the consumer function.
+
+    @type _consumer: callable that takes L{object}
+    @ivar _consumer: The consumer function which is used to store the
+        values derived by the transform function from each line of a
+        multi-line response into a list.
+    """
+
+    startedTLS = False
+    allowInsecureLogin = False
+    timeout = 0
+    serverChallenge = None
+
+    _capCache = None
+    _challengeMagicRe = re.compile(b"(<[^>]+>)")
+    _blockedQueue = None
+    _waiting = None
+    _timedOut = False
+    _greetingError = None
+
+    def _blocked(self, f, *a):
+        """
+        Block a command, if necessary.
+
+        If commands are being blocked, append information about the function
+        which sends the command to a list and return a deferred that will be
+        chained with the return value of the function when it eventually runs.
+        Otherwise, set up for subsequent commands to be blocked and return
+        L{None}.
+
+        @type f: callable
+        @param f: A function which sends a command.
+
+        @type a: L{tuple}
+        @param a: Arguments to the function.
+
+        @rtype: L{None} or L{Deferred <defer.Deferred>}
+        @return: L{None} if the command can run immediately.  Otherwise,
+            a deferred that will eventually trigger with the return value of
+            the function.
+        """
+        if self._blockedQueue is not None:
+            d = defer.Deferred()
+            self._blockedQueue.append((d, f, a))
+            return d
+        self._blockedQueue = []
+        return None
+
+    def _unblock(self):
+        """
+        Send the next blocked command.
+
+        If there are no more commands in the blocked queue, set up for the next
+        command to be sent immediately.
+        """
+        if self._blockedQueue == []:
+            self._blockedQueue = None
+        elif self._blockedQueue is not None:
+            _blockedQueue = self._blockedQueue
+            self._blockedQueue = None
+
+            d, f, a = _blockedQueue.pop(0)
+            d2 = f(*a)
+            d2.chainDeferred(d)
+            # f is a function which uses _blocked (otherwise it wouldn't
+            # have gotten into the blocked queue), which means it will have
+            # re-set _blockedQueue to an empty list, so we can put the rest
+            # of the blocked queue back into it now.
+            self._blockedQueue.extend(_blockedQueue)
+
+    def sendShort(self, cmd, args):
+        """
+        Send a POP3 command to which a short response is expected.
+
+        Block all further commands from being sent until the response is
+        received.  Transition the state to SHORT.
+
+        @type cmd: L{bytes}
+        @param cmd: A POP3 command.
+
+        @type args: L{bytes}
+        @param args: The command arguments.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the entire response is received.
+            On an OK response, it returns the response from the server minus
+            the status indicator.  On an ERR response, it issues a server
+            error response failure with the response from the server minus the
+            status indicator.
+        """
+        d = self._blocked(self.sendShort, cmd, args)
+        if d is not None:
+            return d
+
+        if args:
+            self.sendLine(cmd + b" " + args)
+        else:
+            self.sendLine(cmd)
+        self.state = "SHORT"
+        self._waiting = defer.Deferred()
+        return self._waiting
+
+    def sendLong(self, cmd, args, consumer, xform):
+        """
+        Send a POP3 command to which a multi-line response is expected.
+
+        Block all further commands from being sent until the entire response is
+        received.  Transition the state to LONG_INITIAL.
+
+        @type cmd: L{bytes}
+        @param cmd: A POP3 command.
+
+        @type args: L{bytes}
+        @param args: The command arguments.
+
+        @type consumer: callable that takes L{object}
+        @param consumer: A consumer function which should be used to put
+            the values derived by a transform function from each line of the
+            multi-line response into a list.
+
+        @type xform: L{None} or callable that takes
+            L{bytes} and returns L{object}
+        @param xform: A transform function which should be used to transform
+            each line of the multi-line response into usable values for use by
+            a consumer function.  If L{None}, each line of the multi-line
+            response should be sent directly to the consumer function.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            callable that takes L{object} and fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the entire response is received.
+            On an OK response, it returns the consumer function.  On an ERR
+            response, it issues a server error response failure with the
+            response from the server minus the status indicator and the
+            consumer function.
+        """
+        d = self._blocked(self.sendLong, cmd, args, consumer, xform)
+        if d is not None:
+            return d
+
+        if args:
+            self.sendLine(cmd + b" " + args)
+        else:
+            self.sendLine(cmd)
+        self.state = "LONG_INITIAL"
+        self._xform = xform
+        self._consumer = consumer
+        self._waiting = defer.Deferred()
+        return self._waiting
+
+    # Twisted protocol callback
+    def connectionMade(self):
+        """
+        Wait for a greeting from the server after the connection has been made.
+
+        Start the connection in the WELCOME state.
+        """
+        if self.timeout > 0:
+            self.setTimeout(self.timeout)
+
+        self.state = "WELCOME"
+        self._blockedQueue = []
+
+    def timeoutConnection(self):
+        """
+        Drop the connection when the server does not respond in time.
+        """
+        self._timedOut = True
+        self.transport.loseConnection()
+
+    def connectionLost(self, reason):
+        """
+        Clean up when the connection has been lost.
+
+        When the loss of connection was initiated by the client due to a
+        timeout, the L{_timedOut} flag will be set.  When it was initiated by
+        the client due to an error in the server greeting, L{_greetingError}
+        will be set to the server response minus the status indicator.
+
+        @type reason: L{Failure <twisted.python.failure.Failure>}
+        @param reason: The reason the connection was terminated.
+        """
+        if self.timeout > 0:
+            self.setTimeout(None)
+
+        if self._timedOut:
+            reason = error.TimeoutError()
+        elif self._greetingError:
+            reason = ServerErrorResponse(self._greetingError)
+
+        d = []
+        if self._waiting is not None:
+            d.append(self._waiting)
+            self._waiting = None
+        if self._blockedQueue is not None:
+            d.extend([deferred for (deferred, f, a) in self._blockedQueue])
+            self._blockedQueue = None
+        for w in d:
+            w.errback(reason)
+
+    def lineReceived(self, line):
+        """
+        Pass a received line to a state machine function and
+        transition to the next state.
+
+        @type line: L{bytes}
+        @param line: A received line.
+        """
+        if self.timeout > 0:
+            self.resetTimeout()
+
+        state = self.state
+        self.state = None
+        state = getattr(self, "state_" + state)(line) or state
+        if self.state is None:
+            self.state = state
+
+    def lineLengthExceeded(self, buffer):
+        """
+        Drop the connection when a server response exceeds the maximum line
+        length (L{LineOnlyReceiver.MAX_LENGTH}).
+
+        @type buffer: L{bytes}
+        @param buffer: A received line which exceeds the maximum line length.
+        """
+        # XXX - We need to be smarter about this
+        if self._waiting is not None:
+            waiting, self._waiting = self._waiting, None
+            waiting.errback(LineTooLong())
+        self.transport.loseConnection()
+
+    # POP3 Client state logic - don't touch this.
+    def state_WELCOME(self, line):
+        """
+        Handle server responses for the WELCOME state in which the server
+        greeting is expected.
+
+        WELCOME is the first state.  The server should send one line of text
+        with a greeting and possibly an APOP challenge.  Transition the state
+        to WAITING.
+
+        @type line: L{bytes}
+        @param line: A line received from the server.
+
+        @rtype: L{bytes}
+        @return: The next state.
+        """
+        code, status = _codeStatusSplit(line)
+        if code != OK:
+            self._greetingError = status
+            self.transport.loseConnection()
+        else:
+            m = self._challengeMagicRe.search(status)
+
+            if m is not None:
+                self.serverChallenge = m.group(1)
+
+            self.serverGreeting(status)
+
+        self._unblock()
+        return "WAITING"
+
+    def state_WAITING(self, line):
+        """
+        Log an error for server responses received in the WAITING state during
+        which the server is not expected to send anything.
+
+        @type line: L{bytes}
+        @param line: A line received from the server.
+        """
+        log.msg("Illegal line from server: " + repr(line))
+
+    def state_SHORT(self, line):
+        """
+        Handle server responses for the SHORT state in which the server is
+        expected to send a single line response.
+
+        Parse the response and fire the deferred which is waiting on receipt of
+        a complete response.  Transition the state back to WAITING.
+
+        @type line: L{bytes}
+        @param line: A line received from the server.
+
+        @rtype: L{bytes}
+        @return: The next state.
+        """
+        deferred, self._waiting = self._waiting, None
+        self._unblock()
+        code, status = _codeStatusSplit(line)
+        if code == OK:
+            deferred.callback(status)
+        else:
+            deferred.errback(ServerErrorResponse(status))
+        return "WAITING"
+
+    def state_LONG_INITIAL(self, line):
+        """
+        Handle server responses for the LONG_INITIAL state in which the server
+        is expected to send the first line of a multi-line response.
+
+        Parse the response.  On an OK response, transition the state to
+        LONG.  On an ERR response, cleanup and transition the state to
+        WAITING.
+
+        @type line: L{bytes}
+        @param line: A line received from the server.
+
+        @rtype: L{bytes}
+        @return: The next state.
+        """
+        code, status = _codeStatusSplit(line)
+        if code == OK:
+            return "LONG"
+        consumer = self._consumer
+        deferred = self._waiting
+        self._consumer = self._waiting = self._xform = None
+        self._unblock()
+        deferred.errback(ServerErrorResponse(status, consumer))
+        return "WAITING"
+
+    def state_LONG(self, line):
+        """
+        Handle server responses for the LONG state in which the server is
+        expected to send a non-initial line of a multi-line response.
+
+        On receipt of the last line of the response, clean up, fire the
+        deferred which is waiting on receipt of a complete response, and
+        transition the state to WAITING. Otherwise, pass the line to the
+        transform function, if provided, and then the consumer function.
+
+        @type line: L{bytes}
+        @param line: A line received from the server.
+
+        @rtype: L{bytes}
+        @return: The next state.
+        """
+        # This is the state for each line of a long response.
+        if line == b".":
+            consumer = self._consumer
+            deferred = self._waiting
+            self._consumer = self._waiting = self._xform = None
+            self._unblock()
+            deferred.callback(consumer)
+            return "WAITING"
+        else:
+            if self._xform is not None:
+                self._consumer(self._xform(line))
+            else:
+                self._consumer(line)
+            return "LONG"
+
+    # Callbacks - override these
+    def serverGreeting(self, greeting):
+        """
+        Handle the server greeting.
+
+        @type greeting: L{bytes}
+        @param greeting: The server greeting minus the status indicator.
+            For servers implementing APOP authentication, this will contain a
+            challenge string.
+        """
+
+    # External API - call these (most of 'em anyway)
+    def startTLS(self, contextFactory=None):
+        """
+        Switch to encrypted communication using TLS.
+
+        The first step of switching to encrypted communication is obtaining
+        the server's capabilities.  When that is complete, the L{_startTLS}
+        callback function continues the switching process.
+
+        @type contextFactory: L{None} or
+            L{ClientContextFactory <twisted.internet.ssl.ClientContextFactory>}
+        @param contextFactory: The context factory with which to negotiate TLS.
+            If not provided, try to create a new one.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully results in
+            L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes} to
+            L{None} or fails with L{TLSError}
+        @return: A deferred which fires when the transport has been
+            secured according to the given context factory with the server
+            capabilities, or which fails with a TLS error if the transport
+            cannot be secured.
+        """
+        tls = interfaces.ITLSTransport(self.transport, None)
+        if tls is None:
+            return defer.fail(
+                TLSError(
+                    "POP3Client transport does not implement "
+                    "interfaces.ITLSTransport"
+                )
+            )
+
+        if contextFactory is None:
+            contextFactory = self._getContextFactory()
+
+        if contextFactory is None:
+            return defer.fail(
+                TLSError(
+                    "POP3Client requires a TLS context to "
+                    "initiate the STLS handshake"
+                )
+            )
+
+        d = self.capabilities()
+        d.addCallback(self._startTLS, contextFactory, tls)
+        return d
+
+    def _startTLS(self, caps, contextFactory, tls):
+        """
+        Continue the process of switching to encrypted communication.
+
+        This callback function runs after the server capabilities are received.
+
+        The next step is sending the server an STLS command to request a
+        switch to encrypted communication.  When an OK response is received,
+        the L{_startedTLS} callback function completes the switch to encrypted
+        communication. Then, the new server capabilities are requested.
+
+        @type caps: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or
+            L{bytes} to L{None}
+        @param caps: The server capabilities.
+
+        @type contextFactory: L{ClientContextFactory
+            <twisted.internet.ssl.ClientContextFactory>}
+        @param contextFactory: A context factory with which to negotiate TLS.
+
+        @type tls: L{ITLSTransport <interfaces.ITLSTransport>}
+        @param tls: A TCP transport that supports switching to TLS midstream.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully triggers with
+            L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes} to
+            L{None} or fails with L{TLSNotSupportedError}
+        @return: A deferred which successfully fires when the response from
+            the server to the request to start TLS has been received and the
+            new server capabilities have been received or fails when the server
+            does not support TLS.
+        """
+        assert (
+            not self.startedTLS
+        ), "Client and Server are currently communicating via TLS"
+
+        if b"STLS" not in caps:
+            return defer.fail(
+                TLSNotSupportedError(
+                    "Server does not support secure communication " "via TLS / SSL"
+                )
+            )
+
+        d = self.sendShort(b"STLS", None)
+        d.addCallback(self._startedTLS, contextFactory, tls)
+        d.addCallback(lambda _: self.capabilities())
+        return d
+
+    def _startedTLS(self, result, context, tls):
+        """
+        Complete the process of switching to encrypted communication.
+
+        This callback function runs after the response to the STLS command has
+        been received.
+
+        The final steps are discarding the cached capabilities and initiating
+        TLS negotiation on the transport.
+
+        @type result: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or
+            L{bytes} to L{None}
+        @param result: The server capabilities.
+
+        @type context: L{ClientContextFactory
+            <twisted.internet.ssl.ClientContextFactory>}
+        @param context: A context factory with which to negotiate TLS.
+
+        @type tls: L{ITLSTransport <interfaces.ITLSTransport>}
+        @param tls: A TCP transport that supports switching to TLS midstream.
+
+        @rtype: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes}
+            to L{None}
+        @return: The server capabilities.
+        """
+        self.transport = tls
+        self.transport.startTLS(context)
+        self._capCache = None
+        self.startedTLS = True
+        return result
+
+    def _getContextFactory(self):
+        """
+        Get a context factory with which to negotiate TLS.
+
+        @rtype: L{None} or
+            L{ClientContextFactory <twisted.internet.ssl.ClientContextFactory>}
+        @return: A context factory or L{None} if TLS is not supported on the
+            client.
+        """
+        try:
+            from twisted.internet import ssl
+        except ImportError:
+            return None
+        else:
+            context = ssl.ClientContextFactory()
+            context.method = ssl.SSL.TLSv1_2_METHOD
+            return context
+
+    def login(self, username, password):
+        """
+        Log in to the server.
+
+        If APOP is available it will be used.  Otherwise, if TLS is
+        available, an encrypted session will be started and plaintext
+        login will proceed.  Otherwise, if L{allowInsecureLogin} is set,
+        insecure plaintext login will proceed.  Otherwise,
+        L{InsecureAuthenticationDisallowed} will be raised.
+
+        The first step of logging into the server is obtaining the server's
+        capabilities.  When that is complete, the L{_login} callback function
+        continues the login process.
+
+        @type username: L{bytes}
+        @param username: The username with which to log in.
+
+        @type password: L{bytes}
+        @param password: The password with which to log in.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes}
+        @return: A deferred which fires when the login process is complete.
+            On a successful login, it returns the server's response minus the
+            status indicator.
+        """
+        d = self.capabilities()
+        d.addCallback(self._login, username, password)
+        return d
+
+    def _login(self, caps, username, password):
+        """
+        Continue the process of logging in to the server.
+
+        This callback function runs after the server capabilities are received.
+
+        If the server provided a challenge in the greeting, proceed with an
+        APOP login.  Otherwise, if the server and the transport support
+        encrypted communication, try to switch to TLS and then complete
+        the login process with the L{_loginTLS} callback function.  Otherwise,
+        if insecure authentication is allowed, do a plaintext login.
+        Otherwise, fail with an L{InsecureAuthenticationDisallowed} error.
+
+        @type caps: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or
+            L{bytes} to L{None}
+        @param caps: The server capabilities.
+
+        @type username: L{bytes}
+        @param username: The username with which to log in.
+
+        @type password: L{bytes}
+        @param password: The password with which to log in.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes}
+        @return: A deferred which fires when the login process is complete.
+            On a successful login, it returns the server's response minus the
+            status indicator.
+        """
+        if self.serverChallenge is not None:
+            return self._apop(username, password, self.serverChallenge)
+
+        tryTLS = b"STLS" in caps
+
+        # If our transport supports switching to TLS, we might want to
+        # try to switch to TLS.
+        tlsableTransport = interfaces.ITLSTransport(self.transport, None) is not None
+
+        # If our transport is not already using TLS, we might want to
+        # try to switch to TLS.
+        nontlsTransport = interfaces.ISSLTransport(self.transport, None) is None
+
+        if not self.startedTLS and tryTLS and tlsableTransport and nontlsTransport:
+            d = self.startTLS()
+
+            d.addCallback(self._loginTLS, username, password)
+            return d
+
+        elif self.startedTLS or not nontlsTransport or self.allowInsecureLogin:
+            return self._plaintext(username, password)
+        else:
+            return defer.fail(InsecureAuthenticationDisallowed())
+
+    def _loginTLS(self, res, username, password):
+        """
+        Do a plaintext login over an encrypted transport.
+
+        This callback function runs after the transport switches to encrypted
+        communication.
+
+        @type res: L{dict} mapping L{bytes} to L{list} of L{bytes} and/or
+            L{bytes} to L{None}
+        @param res: The server capabilities.
+
+        @type username: L{bytes}
+        @param username: The username with which to log in.
+
+        @type password: L{bytes}
+        @param password: The password with which to log in.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server accepts the username
+            and password or fails when the server rejects either.  On a
+            successful login, it returns the server's response minus the
+            status indicator.
+        """
+        return self._plaintext(username, password)
+
+    def _plaintext(self, username, password):
+        """
+        Perform a plaintext login.
+
+        @type username: L{bytes}
+        @param username: The username with which to log in.
+
+        @type password: L{bytes}
+        @param password: The password with which to log in.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server accepts the username
+            and password or fails when the server rejects either.  On a
+            successful login, it returns the server's response minus the
+            status indicator.
+        """
+        return self.user(username).addCallback(lambda r: self.password(password))
+
+    def _apop(self, username, password, challenge):
+        """
+        Perform an APOP login.
+
+        @type username: L{bytes}
+        @param username: The username with which to log in.
+
+        @type password: L{bytes}
+        @param password: The password with which to log in.
+
+        @type challenge: L{bytes}
+        @param challenge: A challenge string.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On a successful login, it returns the server response minus
+            the status indicator.
+        """
+        digest = md5(challenge + password).hexdigest().encode("ascii")
+        return self.apop(username, digest)
+
+    def apop(self, username, digest):
+        """
+        Send an APOP command to perform authenticated login.
+
+        This should be used in special circumstances only, when it is
+        known that the server supports APOP authentication, and APOP
+        authentication is absolutely required.  For the common case,
+        use L{login} instead.
+
+        @type username: L{bytes}
+        @param username: The username with which to log in.
+
+        @type digest: L{bytes}
+        @param digest: The challenge response to authenticate with.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On an OK response, the deferred succeeds with the server
+            response minus the status indicator.  On an ERR response, the
+            deferred fails with a server error response failure.
+        """
+        return self.sendShort(b"APOP", username + b" " + digest)
+
+    def user(self, username):
+        """
+        Send a USER command to perform the first half of plaintext login.
+
+        Unless this is absolutely required, use the L{login} method instead.
+
+        @type username: L{bytes}
+        @param username: The username with which to log in.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On an OK response, the deferred succeeds with the server
+            response minus the status indicator.  On an ERR response, the
+            deferred fails with a server error response failure.
+        """
+        return self.sendShort(b"USER", username)
+
+    def password(self, password):
+        """
+        Send a PASS command to perform the second half of plaintext login.
+
+        Unless this is absolutely required, use the L{login} method instead.
+
+        @type password: L{bytes}
+        @param password: The plaintext password with which to authenticate.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On an OK response, the deferred succeeds with the server
+            response minus the status indicator.  On an ERR response, the
+            deferred fails with a server error response failure.
+        """
+        return self.sendShort(b"PASS", password)
+
+    def delete(self, index):
+        """
+        Send a DELE command to delete a message from the server.
+
+        @type index: L{int}
+        @param index: The 0-based index of the message to delete.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On an OK response, the deferred succeeds with the server
+            response minus the status indicator.  On an ERR response, the
+            deferred fails with a server error response failure.
+        """
+        return self.sendShort(b"DELE", b"%d" % (index + 1,))
+
+    def _consumeOrSetItem(self, cmd, args, consumer, xform):
+        """
+        Send a command to which a long response is expected and process the
+        multi-line response into a list accounting for deleted messages.
+
+        @type cmd: L{bytes}
+        @param cmd: A POP3 command to which a long response is expected.
+
+        @type args: L{bytes}
+        @param args: The command arguments.
+
+        @type consumer: L{None} or callable that takes
+            L{object}
+        @param consumer: L{None} or a function that consumes the output from
+            the transform function.
+
+        @type xform: L{None}, callable that takes
+            L{bytes} and returns 2-L{tuple} of (0) L{int}, (1) L{object},
+            or callable that takes L{bytes} and returns L{object}
+        @param xform: A function that parses a line from a multi-line response
+            and transforms the values into usable form for input to the
+            consumer function.  If no consumer function is specified, the
+            output must be a message index and corresponding value.  If no
+            transform function is specified, the line is used as is.
+
+        @rtype: L{Deferred <defer.Deferred>} which fires with L{list} of
+            L{object} or callable that takes L{list} of L{object}
+        @return: A deferred which fires when the entire response has been
+            received.  When a consumer is not provided, the return value is a
+            list of the value for each message or L{None} for deleted messages.
+            Otherwise, it returns the consumer itself.
+        """
+        if consumer is None:
+            L = []
+            consumer = _ListSetter(L).setitem
+            return self.sendLong(cmd, args, consumer, xform).addCallback(lambda r: L)
+        return self.sendLong(cmd, args, consumer, xform)
+
+    def _consumeOrAppend(self, cmd, args, consumer, xform):
+        """
+        Send a command to which a long response is expected and process the
+        multi-line response into a list.
+
+        @type cmd: L{bytes}
+        @param cmd: A POP3 command which expects a long response.
+
+        @type args: L{bytes}
+        @param args: The command arguments.
+
+        @type consumer: L{None} or callable that takes
+            L{object}
+        @param consumer: L{None} or a function that consumes the output from the
+            transform function.
+
+        @type xform: L{None} or callable that takes
+            L{bytes} and returns L{object}
+        @param xform: A function that transforms a line from a multi-line
+            response into usable form for input to the consumer function.  If
+            no transform function is specified, the line is used as is.
+
+        @rtype: L{Deferred <defer.Deferred>} which fires with L{list} of
+            2-L{tuple} of (0) L{int}, (1) L{object} or callable that
+            takes 2-L{tuple} of (0) L{int}, (1) L{object}
+        @return: A deferred which fires when the entire response has been
+            received.  When a consumer is not provided, the return value is a
+            list of the transformed lines.  Otherwise, it returns the consumer
+            itself.
+        """
+        if consumer is None:
+            L = []
+            consumer = L.append
+            return self.sendLong(cmd, args, consumer, xform).addCallback(lambda r: L)
+        return self.sendLong(cmd, args, consumer, xform)
+
+    def capabilities(self, useCache=True):
+        """
+        Send a CAPA command to retrieve the capabilities supported by
+        the server.
+
+        Not all servers support this command.  If the server does not
+        support this, it is treated as though it returned a successful
+        response listing no capabilities.  At some future time, this may be
+        changed to instead seek out information about a server's
+        capabilities in some other fashion (only if it proves useful to do
+        so, and only if there are servers still in use which do not support
+        CAPA but which do support POP3 extensions that are useful).
+
+        @type useCache: L{bool}
+        @param useCache: A flag that determines whether previously retrieved
+            results should be used if available.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully results in
+            L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes} to
+            L{None}
+        @return: A deferred which fires with a mapping of capability name to
+        parameters.  For example::
+
+            C: CAPA
+            S: +OK Capability list follows
+            S: TOP
+            S: USER
+            S: SASL CRAM-MD5 KERBEROS_V4
+            S: RESP-CODES
+            S: LOGIN-DELAY 900
+            S: PIPELINING
+            S: EXPIRE 60
+            S: UIDL
+            S: IMPLEMENTATION Shlemazle-Plotz-v302
+            S: .
+
+        will be lead to a result of::
+
+            | {'TOP': None,
+            |  'USER': None,
+            |  'SASL': ['CRAM-MD5', 'KERBEROS_V4'],
+            |  'RESP-CODES': None,
+            |  'LOGIN-DELAY': ['900'],
+            |  'PIPELINING': None,
+            |  'EXPIRE': ['60'],
+            |  'UIDL': None,
+            |  'IMPLEMENTATION': ['Shlemazle-Plotz-v302']}
+        """
+        if useCache and self._capCache is not None:
+            return defer.succeed(self._capCache)
+
+        cache = {}
+
+        def consume(line):
+            tmp = line.split()
+            if len(tmp) == 1:
+                cache[tmp[0]] = None
+            elif len(tmp) > 1:
+                cache[tmp[0]] = tmp[1:]
+
+        def capaNotSupported(err):
+            err.trap(ServerErrorResponse)
+            return None
+
+        def gotCapabilities(result):
+            self._capCache = cache
+            return cache
+
+        d = self._consumeOrAppend(b"CAPA", None, consume, None)
+        d.addErrback(capaNotSupported).addCallback(gotCapabilities)
+        return d
+
+    def noop(self):
+        """
+        Send a NOOP command asking the server to do nothing but respond.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On an OK response, the deferred succeeds with the server
+            response minus the status indicator.  On an ERR response, the
+            deferred fails with a server error response failure.
+        """
+        return self.sendShort(b"NOOP", None)
+
+    def reset(self):
+        """
+        Send a RSET command to unmark any messages that have been flagged
+        for deletion on the server.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On an OK response, the deferred succeeds with the server
+            response minus the status indicator.  On an ERR response, the
+            deferred fails with a server error response failure.
+        """
+        return self.sendShort(b"RSET", None)
+
+    def retrieve(self, index, consumer=None, lines=None):
+        """
+        Send a RETR or TOP command to retrieve all or part of a message from
+        the server.
+
+        @type index: L{int}
+        @param index: A 0-based message index.
+
+        @type consumer: L{None} or callable that takes
+            L{bytes}
+        @param consumer: A function which consumes each transformed line from a
+            multi-line response as it is received.
+
+        @type lines: L{None} or L{int}
+        @param lines: If specified, the number of lines of the message to be
+            retrieved.  Otherwise, the entire message is retrieved.
+
+        @rtype: L{Deferred <defer.Deferred>} which fires with L{list} of
+            L{bytes}, or callable that takes 2-L{tuple} of (0) L{int},
+            (1) L{object}
+        @return: A deferred which fires when the entire response has been
+            received.  When a consumer is not provided, the return value is a
+            list of the transformed lines.  Otherwise, it returns the consumer
+            itself.
+        """
+        idx = b"%d" % (index + 1,)
+        if lines is None:
+            return self._consumeOrAppend(b"RETR", idx, consumer, _dotUnquoter)
+
+        return self._consumeOrAppend(
+            b"TOP", b"%b %d" % (idx, lines), consumer, _dotUnquoter
+        )
+
+    def stat(self):
+        """
+        Send a STAT command to get information about the size of the mailbox.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            a 2-tuple of (0) L{int}, (1) L{int} or fails with
+            L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On an OK response, the deferred succeeds with the number of
+            messages in the mailbox and the size of the mailbox in octets.
+            On an ERR response, the deferred fails with a server error
+            response failure.
+        """
+        return self.sendShort(b"STAT", None).addCallback(_statXform)
+
+    def listSize(self, consumer=None):
+        """
+        Send a LIST command to retrieve the sizes of all messages on the
+        server.
+
+        @type consumer: L{None} or callable that takes
+            2-L{tuple} of (0) L{int}, (1) L{int}
+        @param consumer: A function which consumes the 0-based message index
+            and message size derived from the server response.
+
+        @rtype: L{Deferred <defer.Deferred>} which fires L{list} of L{int} or
+            callable that takes 2-L{tuple} of (0) L{int}, (1) L{int}
+        @return: A deferred which fires when the entire response has been
+            received.  When a consumer is not provided, the return value is a
+            list of message sizes.  Otherwise, it returns the consumer itself.
+        """
+        return self._consumeOrSetItem(b"LIST", None, consumer, _listXform)
+
+    def listUID(self, consumer=None):
+        """
+        Send a UIDL command to retrieve the UIDs of all messages on the server.
+
+        @type consumer: L{None} or callable that takes
+            2-L{tuple} of (0) L{int}, (1) L{bytes}
+        @param consumer: A function which consumes the 0-based message index
+            and UID derived from the server response.
+
+        @rtype: L{Deferred <defer.Deferred>} which fires with L{list} of
+            L{object} or callable that takes 2-L{tuple} of (0) L{int},
+            (1) L{bytes}
+        @return: A deferred which fires when the entire response has been
+            received.  When a consumer is not provided, the return value is a
+            list of message sizes.  Otherwise, it returns the consumer itself.
+        """
+        return self._consumeOrSetItem(b"UIDL", None, consumer, _uidXform)
+
+    def quit(self):
+        """
+        Send a QUIT command to disconnect from the server.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            L{bytes} or fails with L{ServerErrorResponse}
+        @return: A deferred which fires when the server response is received.
+            On an OK response, the deferred succeeds with the server
+            response minus the status indicator.  On an ERR response, the
+            deferred fails with a server error response failure.
+        """
+        return self.sendShort(b"QUIT", None)
+
+
+__all__: List[str] = []
diff --git a/contrib/python/Twisted/py3/twisted/mail/alias.py b/contrib/python/Twisted/py3/twisted/mail/alias.py
new file mode 100644
index 00000000000..4713ad38213
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/alias.py
@@ -0,0 +1,765 @@
+# -*- test-case-name: twisted.mail.test.test_mail -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Support for aliases(5) configuration files.
+
+@author: Jp Calderone
+"""
+
+import os
+import tempfile
+
+from zope.interface import implementer
+
+from twisted.internet import defer, protocol, reactor
+from twisted.mail import smtp
+from twisted.mail.interfaces import IAlias
+from twisted.python import failure, log
+
+
+def handle(result, line, filename, lineNo):
+    """
+    Parse a line from an aliases file.
+
+    @type result: L{dict} mapping L{bytes} to L{list} of L{bytes}
+    @param result: A dictionary mapping username to aliases to which
+        the results of parsing the line are added.
+
+    @type line: L{bytes}
+    @param line: A line from an aliases file.
+
+    @type filename: L{bytes}
+    @param filename: The full or relative path to the aliases file.
+
+    @type lineNo: L{int}
+    @param lineNo: The position of the line within the aliases file.
+    """
+    parts = [p.strip() for p in line.split(":", 1)]
+    if len(parts) != 2:
+        fmt = "Invalid format on line %d of alias file %s."
+        arg = (lineNo, filename)
+        log.err(fmt % arg)
+    else:
+        user, alias = parts
+        result.setdefault(user.strip(), []).extend(map(str.strip, alias.split(",")))
+
+
+def loadAliasFile(domains, filename=None, fp=None):
+    """
+    Load a file containing email aliases.
+
+    Lines in the file should be formatted like so::
+
+         username: alias1, alias2, ..., aliasN
+
+    Aliases beginning with a C{|} will be treated as programs, will be run, and
+    the message will be written to their stdin.
+
+    Aliases beginning with a C{:} will be treated as a file containing
+    additional aliases for the username.
+
+    Aliases beginning with a C{/} will be treated as the full pathname to a file
+    to which the message will be appended.
+
+    Aliases without a host part will be assumed to be addresses on localhost.
+
+    If a username is specified multiple times, the aliases for each are joined
+    together as if they had all been on one line.
+
+    Lines beginning with a space or a tab are continuations of the previous
+    line.
+
+    Lines beginning with a C{#} are comments.
+
+    @type domains: L{dict} mapping L{bytes} to L{IDomain} provider
+    @param domains: A mapping of domain name to domain object.
+
+    @type filename: L{bytes} or L{None}
+    @param filename: The full or relative path to a file from which to load
+        aliases. If omitted, the C{fp} parameter must be specified.
+
+    @type fp: file-like object or L{None}
+    @param fp: The file from which to load aliases. If specified,
+        the C{filename} parameter is ignored.
+
+    @rtype: L{dict} mapping L{bytes} to L{AliasGroup}
+    @return: A mapping from username to group of aliases.
+    """
+    result = {}
+    close = False
+    if fp is None:
+        fp = open(filename)
+        close = True
+    else:
+        filename = getattr(fp, "name", "<unknown>")
+    i = 0
+    prev = ""
+    try:
+        for line in fp:
+            i += 1
+            line = line.rstrip()
+            if line.lstrip().startswith("#"):
+                continue
+            elif line.startswith(" ") or line.startswith("\t"):
+                prev = prev + line
+            else:
+                if prev:
+                    handle(result, prev, filename, i)
+                prev = line
+    finally:
+        if close:
+            fp.close()
+    if prev:
+        handle(result, prev, filename, i)
+    for u, a in result.items():
+        result[u] = AliasGroup(a, domains, u)
+    return result
+
+
+class AliasBase:
+    """
+    The default base class for aliases.
+
+    @ivar domains: See L{__init__}.
+
+    @type original: L{Address}
+    @ivar original: The original address being aliased.
+    """
+
+    def __init__(self, domains, original):
+        """
+        @type domains: L{dict} mapping L{bytes} to L{IDomain} provider
+        @param domains: A mapping of domain name to domain object.
+
+        @type original: L{bytes}
+        @param original: The original address being aliased.
+        """
+        self.domains = domains
+        self.original = smtp.Address(original)
+
+    def domain(self):
+        """
+        Return the domain associated with original address.
+
+        @rtype: L{IDomain} provider
+        @return: The domain for the original address.
+        """
+        return self.domains[self.original.domain]
+
+    def resolve(self, aliasmap, memo=None):
+        """
+        Map this alias to its ultimate destination.
+
+        @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase}
+        @param aliasmap: A mapping of username to alias or group of aliases.
+
+        @type memo: L{None} or L{dict} of L{AliasBase}
+        @param memo: A record of the aliases already considered in the
+            resolution process.  If provided, C{memo} is modified to include
+            this alias.
+
+        @rtype: L{IMessage <smtp.IMessage>} or L{None}
+        @return: A message receiver for the ultimate destination or None for
+            an invalid destination.
+        """
+        if memo is None:
+            memo = {}
+        if str(self) in memo:
+            return None
+        memo[str(self)] = None
+        return self.createMessageReceiver()
+
+
+@implementer(IAlias)
+class AddressAlias(AliasBase):
+    """
+    An alias which translates one email address into another.
+
+    @type alias : L{Address}
+    @ivar alias: The destination address.
+    """
+
+    def __init__(self, alias, *args):
+        """
+        @type alias: L{Address}, L{User}, L{bytes} or object which can be
+            converted into L{bytes}
+        @param alias: The destination address.
+
+        @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain}
+            provider, (1) L{bytes}
+        @param args: Arguments for L{AliasBase.__init__}.
+        """
+        AliasBase.__init__(self, *args)
+        self.alias = smtp.Address(alias)
+
+    def __str__(self) -> str:
+        """
+        Build a string representation of this L{AddressAlias} instance.
+
+        @rtype: L{bytes}
+        @return: A string containing the destination address.
+        """
+        return f"<Address {self.alias}>"
+
+    def createMessageReceiver(self):
+        """
+        Create a message receiver which delivers a message to
+        the destination address.
+
+        @rtype: L{IMessage <smtp.IMessage>} provider
+        @return: A message receiver.
+        """
+        return self.domain().exists(str(self.alias))
+
+    def resolve(self, aliasmap, memo=None):
+        """
+        Map this alias to its ultimate destination.
+
+        @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase}
+        @param aliasmap: A mapping of username to alias or group of aliases.
+
+        @type memo: L{None} or L{dict} of L{AliasBase}
+        @param memo: A record of the aliases already considered in the
+            resolution process.  If provided, C{memo} is modified to include
+            this alias.
+
+        @rtype: L{IMessage <smtp.IMessage>} or L{None}
+        @return: A message receiver for the ultimate destination or None for
+            an invalid destination.
+        """
+        if memo is None:
+            memo = {}
+        if str(self) in memo:
+            return None
+        memo[str(self)] = None
+        try:
+            return self.domain().exists(smtp.User(self.alias, None, None, None), memo)()
+        except smtp.SMTPBadRcpt:
+            pass
+        if self.alias.local in aliasmap:
+            return aliasmap[self.alias.local].resolve(aliasmap, memo)
+        return None
+
+
+@implementer(smtp.IMessage)
+class FileWrapper:
+    """
+    A message receiver which delivers a message to a file.
+
+    @type fp: file-like object
+    @ivar fp: A file used for temporary storage of the message.
+
+    @type finalname: L{bytes}
+    @ivar finalname: The name of the file in which the message should be
+        stored.
+    """
+
+    def __init__(self, filename):
+        """
+        @type filename: L{bytes}
+        @param filename: The name of the file in which the message should be
+            stored.
+        """
+        self.fp = tempfile.TemporaryFile()
+        self.finalname = filename
+
+    def lineReceived(self, line):
+        """
+        Write a received line to the temporary file.
+
+        @type line: L{bytes}
+        @param line: A received line of the message.
+        """
+        self.fp.write(line + "\n")
+
+    def eomReceived(self):
+        """
+        Handle end of message by writing the message to the file.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully results in
+            L{bytes}
+        @return: A deferred which succeeds with the name of the file to which
+            the message has been stored or fails if the message cannot be
+            saved to the file.
+        """
+        self.fp.seek(0, 0)
+        try:
+            f = open(self.finalname, "a")
+        except BaseException:
+            return defer.fail(failure.Failure())
+
+        with f:
+            f.write(self.fp.read())
+            self.fp.close()
+
+        return defer.succeed(self.finalname)
+
+    def connectionLost(self):
+        """
+        Close the temporary file when the connection is lost.
+        """
+        self.fp.close()
+        self.fp = None
+
+    def __str__(self) -> str:
+        """
+        Build a string representation of this L{FileWrapper} instance.
+
+        @rtype: L{bytes}
+        @return: A string containing the file name of the message.
+        """
+        return f"<FileWrapper {self.finalname}>"
+
+
+@implementer(IAlias)
+class FileAlias(AliasBase):
+    """
+    An alias which translates an address to a file.
+
+    @ivar filename: See L{__init__}.
+    """
+
+    def __init__(self, filename, *args):
+        """
+        @type filename: L{bytes}
+        @param filename: The name of the file in which to store the message.
+
+        @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain}
+            provider, (1) L{bytes}
+        @param args: Arguments for L{AliasBase.__init__}.
+        """
+        AliasBase.__init__(self, *args)
+        self.filename = filename
+
+    def __str__(self) -> str:
+        """
+        Build a string representation of this L{FileAlias} instance.
+
+        @rtype: L{bytes}
+        @return: A string containing the name of the file.
+        """
+        return f"<File {self.filename}>"
+
+    def createMessageReceiver(self):
+        """
+        Create a message receiver which delivers a message to the file.
+
+        @rtype: L{FileWrapper}
+        @return: A message receiver which writes a message to the file.
+        """
+        return FileWrapper(self.filename)
+
+
+class ProcessAliasTimeout(Exception):
+    """
+    An error indicating that a timeout occurred while waiting for a process
+    to complete.
+    """
+
+
+@implementer(smtp.IMessage)
+class MessageWrapper:
+    """
+    A message receiver which delivers a message to a child process.
+
+    @type completionTimeout: L{int} or L{float}
+    @ivar completionTimeout: The number of seconds to wait for the child
+        process to exit before reporting the delivery as a failure.
+
+    @type _timeoutCallID: L{None} or
+        L{IDelayedCall <twisted.internet.interfaces.IDelayedCall>} provider
+    @ivar _timeoutCallID: The call used to time out delivery, started when the
+        connection to the child process is closed.
+
+    @type done: L{bool}
+    @ivar done: A flag indicating whether the child process has exited
+        (C{True}) or not (C{False}).
+
+    @type reactor: L{IReactorTime <twisted.internet.interfaces.IReactorTime>}
+        provider
+    @ivar reactor: A reactor which will be used to schedule timeouts.
+
+    @ivar protocol: See L{__init__}.
+
+    @type processName: L{bytes} or L{None}
+    @ivar processName: The process name.
+
+    @type completion: L{Deferred <defer.Deferred>}
+    @ivar completion: The deferred which will be triggered by the protocol
+        when the child process exits.
+    """
+
+    done = False
+
+    completionTimeout = 60
+    _timeoutCallID = None
+
+    reactor = reactor
+
+    def __init__(self, protocol, process=None, reactor=None):
+        """
+        @type protocol: L{ProcessAliasProtocol}
+        @param protocol: The protocol associated with the child process.
+
+        @type process: L{bytes} or L{None}
+        @param process: The process name.
+
+        @type reactor: L{None} or L{IReactorTime
+            <twisted.internet.interfaces.IReactorTime>} provider
+        @param reactor: A reactor which will be used to schedule timeouts.
+        """
+        self.processName = process
+        self.protocol = protocol
+        self.completion = defer.Deferred()
+        self.protocol.onEnd = self.completion
+        self.completion.addBoth(self._processEnded)
+
+        if reactor is not None:
+            self.reactor = reactor
+
+    def _processEnded(self, result):
+        """
+        Record process termination and cancel the timeout call if it is active.
+
+        @type result: L{Failure <failure.Failure>}
+        @param result: The reason the child process terminated.
+
+        @rtype: L{None} or L{Failure <failure.Failure>}
+        @return: None, if the process end is expected, or the reason the child
+            process terminated, if the process end is unexpected.
+        """
+        self.done = True
+        if self._timeoutCallID is not None:
+            # eomReceived was called, we're actually waiting for the process to
+            # exit.
+            self._timeoutCallID.cancel()
+            self._timeoutCallID = None
+        else:
+            # eomReceived was not called, this is unexpected, propagate the
+            # error.
+            return result
+
+    def lineReceived(self, line):
+        """
+        Write a received line to the child process.
+
+        @type line: L{bytes}
+        @param line: A received line of the message.
+        """
+        if self.done:
+            return
+        self.protocol.transport.write(line + "\n")
+
+    def eomReceived(self):
+        """
+        Disconnect from the child process and set up a timeout to wait for it
+        to exit.
+
+        @rtype: L{Deferred <defer.Deferred>}
+        @return: A deferred which will be called back when the child process
+            exits.
+        """
+        if not self.done:
+            self.protocol.transport.loseConnection()
+            self._timeoutCallID = self.reactor.callLater(
+                self.completionTimeout, self._completionCancel
+            )
+        return self.completion
+
+    def _completionCancel(self):
+        """
+        Handle the expiration of the timeout for the child process to exit by
+        terminating the child process forcefully and issuing a failure to the
+        L{completion} deferred.
+        """
+        self._timeoutCallID = None
+        self.protocol.transport.signalProcess("KILL")
+        exc = ProcessAliasTimeout(f"No answer after {self.completionTimeout} seconds")
+        self.protocol.onEnd = None
+        self.completion.errback(failure.Failure(exc))
+
+    def connectionLost(self):
+        """
+        Ignore notification of lost connection.
+        """
+
+    def __str__(self) -> str:
+        """
+        Build a string representation of this L{MessageWrapper} instance.
+
+        @rtype: L{bytes}
+        @return: A string containing the name of the process.
+        """
+        return f"<ProcessWrapper {self.processName}>"
+
+
+class ProcessAliasProtocol(protocol.ProcessProtocol):
+    """
+    A process protocol which errbacks a deferred when the associated
+    process ends.
+
+    @type onEnd: L{None} or L{Deferred <defer.Deferred>}
+    @ivar onEnd: If set, a deferred on which to errback when the process ends.
+    """
+
+    onEnd = None
+
+    def processEnded(self, reason):
+        """
+        Call an errback.
+
+        @type reason: L{Failure <failure.Failure>}
+        @param reason: The reason the child process terminated.
+        """
+        if self.onEnd is not None:
+            self.onEnd.errback(reason)
+
+
+@implementer(IAlias)
+class ProcessAlias(AliasBase):
+    """
+    An alias which is handled by the execution of a program.
+
+    @type path: L{list} of L{bytes}
+    @ivar path: The arguments to pass to the process. The first string is
+        the executable's name.
+
+    @type program: L{bytes}
+    @ivar program: The path of the program to be executed.
+
+    @type reactor: L{IReactorTime <twisted.internet.interfaces.IReactorTime>}
+        and L{IReactorProcess <twisted.internet.interfaces.IReactorProcess>}
+        provider
+    @ivar reactor: A reactor which will be used to create and timeout the
+        child process.
+    """
+
+    reactor = reactor
+
+    def __init__(self, path, *args):
+        """
+        @type path: L{bytes}
+        @param path: The command to invoke the program consisting of the path
+            to the executable followed by any arguments.
+
+        @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain}
+            provider, (1) L{bytes}
+        @param args: Arguments for L{AliasBase.__init__}.
+        """
+
+        AliasBase.__init__(self, *args)
+        self.path = path.split()
+        self.program = self.path[0]
+
+    def __str__(self) -> str:
+        """
+        Build a string representation of this L{ProcessAlias} instance.
+
+        @rtype: L{bytes}
+        @return: A string containing the command used to invoke the process.
+        """
+        return f"<Process {self.path}>"
+
+    def spawnProcess(self, proto, program, path):
+        """
+        Spawn a process.
+
+        This wraps the L{spawnProcess
+        <twisted.internet.interfaces.IReactorProcess.spawnProcess>} method on
+        L{reactor} so that it can be customized for test purposes.
+
+        @type proto: L{IProcessProtocol
+            <twisted.internet.interfaces.IProcessProtocol>} provider
+        @param proto: An object which will be notified of all events related to
+            the created process.
+
+        @type program: L{bytes}
+        @param program: The full path name of the file to execute.
+
+        @type path: L{list} of L{bytes}
+        @param path: The arguments to pass to the process. The first string
+            should be the executable's name.
+
+        @rtype: L{IProcessTransport
+            <twisted.internet.interfaces.IProcessTransport>} provider
+        @return: A process transport.
+        """
+        return self.reactor.spawnProcess(proto, program, path)
+
+    def createMessageReceiver(self):
+        """
+        Launch a process and create a message receiver to pass a message
+        to the process.
+
+        @rtype: L{MessageWrapper}
+        @return: A message receiver which delivers a message to the process.
+        """
+        p = ProcessAliasProtocol()
+        m = MessageWrapper(p, self.program, self.reactor)
+        self.spawnProcess(p, self.program, self.path)
+        return m
+
+
+@implementer(smtp.IMessage)
+class MultiWrapper:
+    """
+    A message receiver which delivers a single message to multiple other
+    message receivers.
+
+    @ivar objs: See L{__init__}.
+    """
+
+    def __init__(self, objs):
+        """
+        @type objs: L{list} of L{IMessage <smtp.IMessage>} provider
+        @param objs: Message receivers to which the incoming message should be
+            directed.
+        """
+        self.objs = objs
+
+    def lineReceived(self, line):
+        """
+        Pass a received line to the message receivers.
+
+        @type line: L{bytes}
+        @param line: A line of the message.
+        """
+        for o in self.objs:
+            o.lineReceived(line)
+
+    def eomReceived(self):
+        """
+        Pass the end of message along to the message receivers.
+
+        @rtype: L{DeferredList <defer.DeferredList>} whose successful results
+            are L{bytes} or L{None}
+        @return: A deferred list which triggers when all of the message
+            receivers have finished handling their end of message.
+        """
+        return defer.DeferredList([o.eomReceived() for o in self.objs])
+
+    def connectionLost(self):
+        """
+        Inform the message receivers that the connection has been lost.
+        """
+        for o in self.objs:
+            o.connectionLost()
+
+    def __str__(self) -> str:
+        """
+        Build a string representation of this L{MultiWrapper} instance.
+
+        @rtype: L{bytes}
+        @return: A string containing a list of the message receivers.
+        """
+        return f"<GroupWrapper {map(str, self.objs)!r}>"
+
+
+@implementer(IAlias)
+class AliasGroup(AliasBase):
+    """
+    An alias which points to multiple destination aliases.
+
+    @type processAliasFactory: no-argument callable which returns
+        L{ProcessAlias}
+    @ivar processAliasFactory: A factory for process aliases.
+
+    @type aliases: L{list} of L{AliasBase} which implements L{IAlias}
+    @ivar aliases: The destination aliases.
+    """
+
+    processAliasFactory = ProcessAlias
+
+    def __init__(self, items, *args):
+        """
+        Create a group of aliases.
+
+        Parse a list of alias strings and, for each, create an appropriate
+        alias object.
+
+        @type items: L{list} of L{bytes}
+        @param items: Aliases.
+
+        @type args: n-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain}
+            provider, (1) L{bytes}
+        @param args: Arguments for L{AliasBase.__init__}.
+        """
+
+        AliasBase.__init__(self, *args)
+        self.aliases = []
+        while items:
+            addr = items.pop().strip()
+            if addr.startswith(":"):
+                try:
+                    f = open(addr[1:])
+                except BaseException:
+                    log.err(f"Invalid filename in alias file {addr[1:]!r}")
+                else:
+                    with f:
+                        addr = " ".join([l.strip() for l in f])
+                    items.extend(addr.split(","))
+            elif addr.startswith("|"):
+                self.aliases.append(self.processAliasFactory(addr[1:], *args))
+            elif addr.startswith("/"):
+                if os.path.isdir(addr):
+                    log.err("Directory delivery not supported")
+                else:
+                    self.aliases.append(FileAlias(addr, *args))
+            else:
+                self.aliases.append(AddressAlias(addr, *args))
+
+    def __len__(self):
+        """
+        Return the number of aliases in the group.
+
+        @rtype: L{int}
+        @return: The number of aliases in the group.
+        """
+        return len(self.aliases)
+
+    def __str__(self) -> str:
+        """
+        Build a string representation of this L{AliasGroup} instance.
+
+        @rtype: L{bytes}
+        @return: A string containing the aliases in the group.
+        """
+        return "<AliasGroup [%s]>" % (", ".join(map(str, self.aliases)))
+
+    def createMessageReceiver(self):
+        """
+        Create a message receiver for each alias and return a message receiver
+        which will pass on a message to each of those.
+
+        @rtype: L{MultiWrapper}
+        @return: A message receiver which passes a message on to message
+            receivers for each alias in the group.
+        """
+        return MultiWrapper([a.createMessageReceiver() for a in self.aliases])
+
+    def resolve(self, aliasmap, memo=None):
+        """
+        Map each of the aliases in the group to its ultimate destination.
+
+        @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase}
+        @param aliasmap: A mapping of username to alias or group of aliases.
+
+        @type memo: L{None} or L{dict} of L{AliasBase}
+        @param memo: A record of the aliases already considered in the
+            resolution process.  If provided, C{memo} is modified to include
+            this alias.
+
+        @rtype: L{MultiWrapper}
+        @return: A message receiver which passes the message on to message
+            receivers for the ultimate destination of each alias in the group.
+        """
+        if memo is None:
+            memo = {}
+        r = []
+        for a in self.aliases:
+            r.append(a.resolve(aliasmap, memo))
+        return MultiWrapper(filter(None, r))
diff --git a/contrib/python/Twisted/py3/twisted/mail/bounce.py b/contrib/python/Twisted/py3/twisted/mail/bounce.py
new file mode 100644
index 00000000000..adf9d313af4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/bounce.py
@@ -0,0 +1,107 @@
+# -*- test-case-name: twisted.mail.test.test_bounce -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Support for bounce message generation.
+"""
+import email.utils
+import os
+import time
+from io import SEEK_END, SEEK_SET, StringIO
+
+from twisted.mail import smtp
+
+BOUNCE_FORMAT = """\
+From: postmaster@{failedDomain}
+To: {failedFrom}
+Subject: Returned Mail: see transcript for details
+Message-ID: {messageID}
+Content-Type: multipart/report; report-type=delivery-status;
+    boundary="{boundary}"
+
+--{boundary}
+
+{transcript}
+
+--{boundary}
+Content-Type: message/delivery-status
+Arrival-Date: {ctime}
+Final-Recipient: RFC822; {failedTo}
+"""
+
+
+def generateBounce(message, failedFrom, failedTo, transcript="", encoding="utf-8"):
+    """
+    Generate a bounce message for an undeliverable email message.
+
+    @type message: a file-like object
+    @param message: The undeliverable message.
+
+    @type failedFrom: L{bytes} or L{unicode}
+    @param failedFrom: The originator of the undeliverable message.
+
+    @type failedTo: L{bytes} or L{unicode}
+    @param failedTo: The destination of the undeliverable message.
+
+    @type transcript: L{bytes} or L{unicode}
+    @param transcript: An error message to include in the bounce message.
+
+    @type encoding: L{str} or L{unicode}
+    @param encoding: Encoding to use, default: utf-8
+
+    @rtype: 3-L{tuple} of (E{1}) L{bytes}, (E{2}) L{bytes}, (E{3}) L{bytes}
+    @return: The originator, the destination and the contents of the bounce
+        message.  The destination of the bounce message is the originator of
+        the undeliverable message.
+    """
+
+    if isinstance(failedFrom, bytes):
+        failedFrom = failedFrom.decode(encoding)
+
+    if isinstance(failedTo, bytes):
+        failedTo = failedTo.decode(encoding)
+
+    if not transcript:
+        transcript = """\
+I'm sorry, the following address has permanent errors: {failedTo}.
+I've given up, and I will not retry the message again.
+""".format(
+            failedTo=failedTo
+        )
+
+    failedAddress = email.utils.parseaddr(failedTo)[1]
+    data = {
+        "boundary": "{}_{}_{}".format(time.time(), os.getpid(), "XXXXX"),
+        "ctime": time.ctime(time.time()),
+        "failedAddress": failedAddress,
+        "failedDomain": failedAddress.split("@", 1)[1],
+        "failedFrom": failedFrom,
+        "failedTo": failedTo,
+        "messageID": smtp.messageid(uniq="bounce"),
+        "message": message,
+        "transcript": transcript,
+    }
+
+    fp = StringIO()
+    fp.write(BOUNCE_FORMAT.format(**data))
+    orig = message.tell()
+    message.seek(0, SEEK_END)
+    sz = message.tell()
+    message.seek(orig, SEEK_SET)
+    if sz > 10000:
+        while 1:
+            line = message.readline()
+            if isinstance(line, bytes):
+                line = line.decode(encoding)
+            if len(line) <= 0:
+                break
+            fp.write(line)
+    else:
+        messageContent = message.read()
+        if isinstance(messageContent, bytes):
+            messageContent = messageContent.decode(encoding)
+        fp.write(messageContent)
+    return b"", failedFrom.encode(encoding), fp.getvalue().encode(encoding)
diff --git a/contrib/python/Twisted/py3/twisted/mail/imap4.py b/contrib/python/Twisted/py3/twisted/mail/imap4.py
new file mode 100644
index 00000000000..032624e3dbf
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/imap4.py
@@ -0,0 +1,6233 @@
+# -*- test-case-name: twisted.mail.test.test_imap.IMAP4HelperTests -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An IMAP4 protocol implementation
+
+@author: Jp Calderone
+
+To do::
+  Suspend idle timeout while server is processing
+  Use an async message parser instead of buffering in memory
+  Figure out a way to not queue multi-message client requests (Flow? A simple callback?)
+  Clarify some API docs (Query, etc)
+  Make APPEND recognize (again) non-existent mailboxes before accepting the literal
+"""
+
+import binascii
+import codecs
+import copy
+import email.utils
+import functools
+import re
+import string
+import tempfile
+import time
+import uuid
+from base64 import decodebytes, encodebytes
+from io import BytesIO
+from itertools import chain
+from typing import Any, List, cast
+
+from zope.interface import implementer
+
+from twisted.cred import credentials
+from twisted.cred.error import UnauthorizedLogin, UnhandledCredentials
+from twisted.internet import defer, error, interfaces
+from twisted.internet.defer import maybeDeferred
+from twisted.mail._cred import (
+    CramMD5ClientAuthenticator,
+    LOGINAuthenticator,
+    LOGINCredentials,
+    PLAINAuthenticator,
+    PLAINCredentials,
+)
+from twisted.mail._except import (
+    IllegalClientResponse,
+    IllegalIdentifierError,
+    IllegalMailboxEncoding,
+    IllegalOperation,
+    IllegalQueryError,
+    IllegalServerResponse,
+    IMAP4Exception,
+    MailboxCollision,
+    MailboxException,
+    MismatchedNesting,
+    MismatchedQuoting,
+    NegativeResponse,
+    NoSuchMailbox,
+    NoSupportedAuthentication,
+    ReadOnlyMailbox,
+    UnhandledResponse,
+)
+
+# Re-exported for compatibility reasons
+from twisted.mail.interfaces import (
+    IAccountIMAP as IAccount,
+    IClientAuthentication,
+    ICloseableMailboxIMAP as ICloseableMailbox,
+    IMailboxIMAP as IMailbox,
+    IMailboxIMAPInfo as IMailboxInfo,
+    IMailboxIMAPListener as IMailboxListener,
+    IMessageIMAP as IMessage,
+    IMessageIMAPCopier as IMessageCopier,
+    IMessageIMAPFile as IMessageFile,
+    IMessageIMAPPart as IMessagePart,
+    INamespacePresenter,
+    ISearchableIMAPMailbox as ISearchableMailbox,
+)
+from twisted.protocols import basic, policies
+from twisted.python import log, text
+from twisted.python.compat import (
+    _get_async_param,
+    _matchingString,
+    iterbytes,
+    nativeString,
+    networkString,
+)
+
+# locale-independent month names to use instead of strftime's
+_MONTH_NAMES = dict(
+    zip(range(1, 13), "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split())
+)
+
+
+def _swap(this, that, ifIs):
+    """
+    Swap C{this} with C{that} if C{this} is C{ifIs}.
+
+    @param this: The object that may be replaced.
+
+    @param that: The object that may replace C{this}.
+
+    @param ifIs: An object whose identity will be compared to
+        C{this}.
+    """
+    return that if this is ifIs else this
+
+
+def _swapAllPairs(of, that, ifIs):
+    """
+    Swap each element in each pair in C{of} with C{that} it is
+    C{ifIs}.
+
+    @param of: A list of 2-L{tuple}s, whose members may be the object
+        C{that}
+    @type of: L{list} of 2-L{tuple}s
+
+    @param ifIs: An object whose identity will be compared to members
+        of each pair in C{of}
+
+    @return: A L{list} of 2-L{tuple}s with all occurences of C{ifIs}
+        replaced with C{that}
+    """
+    return [
+        (_swap(first, that, ifIs), _swap(second, that, ifIs)) for first, second in of
+    ]
+
+
+class MessageSet:
+    """
+    A set of message identifiers usable by both L{IMAP4Client} and
+    L{IMAP4Server} via L{IMailboxIMAP.store} and
+    L{IMailboxIMAP.fetch}.
+
+    These identifiers can be either message sequence numbers or unique
+    identifiers.  See Section 2.3.1, "Message Numbers", RFC 3501.
+
+    This represents the C{sequence-set} described in Section 9,
+    "Formal Syntax" of RFC 3501:
+
+        - A L{MessageSet} can describe a single identifier, e.g.
+          C{MessageSet(1)}
+
+        - A L{MessageSet} can describe C{*} via L{None}, e.g.
+          C{MessageSet(None)}
+
+        - A L{MessageSet} can describe a range of identifiers, e.g.
+          C{MessageSet(1, 2)}.  The range is inclusive and unordered
+          (see C{seq-range} in RFC 3501, Section 9), so that
+          C{Message(2, 1)} is equivalent to C{MessageSet(1, 2)}, and
+          both describe messages 1 and 2.  Ranges can include C{*} by
+          specifying L{None}, e.g. C{MessageSet(None, 1)}.  In all
+          cases ranges are normalized so that the smallest identifier
+          comes first, and L{None} always comes last; C{Message(2, 1)}
+          becomes C{MessageSet(1, 2)} and C{MessageSet(None, 1)}
+          becomes C{MessageSet(1, None)}
+
+        - A L{MessageSet} can describe a sequence of single
+          identifiers and ranges, constructed by addition.
+          C{MessageSet(1) + MessageSet(5, 10)} refers the message
+          identified by C{1} and the messages identified by C{5}
+          through C{10}.
+
+    B{NB: The meaning of * varies, but it always represents the
+    largest number in use}.
+
+    B{For servers}: Your L{IMailboxIMAP} provider must set
+    L{MessageSet.last} to the highest-valued identifier (unique or
+    message sequence) before iterating over it.
+
+    B{For clients}: C{*} consumes ranges smaller than it, e.g.
+    C{MessageSet(1, 100) + MessageSet(50, None)} is equivalent to
+    C{1:*}.
+
+    @type getnext: Function taking L{int} returning L{int}
+    @ivar getnext: A function that returns the next message number,
+        used when iterating through the L{MessageSet}.  By default, a
+        function returning the next integer is supplied, but as this
+        can be rather inefficient for sparse UID iterations, it is
+        recommended to supply one when messages are requested by UID.
+        The argument is provided as a hint to the implementation and
+        may be ignored if it makes sense to do so (eg, if an iterator
+        is being used that maintains its own state, it is guaranteed
+        that it will not be called out-of-order).
+    """
+
+    _empty: List[Any] = []
+    _infinity = float("inf")
+
+    def __init__(self, start=_empty, end=_empty):
+        """
+        Create a new MessageSet()
+
+        @type start: Optional L{int}
+        @param start: Start of range, or only message number
+
+        @type end: Optional L{int}
+        @param end: End of range.
+        """
+        self._last = self._empty  # Last message/UID in use
+        self.ranges = []  # List of ranges included
+        self.getnext = lambda x: x + 1  # A function which will return the next
+        # message id. Handy for UID requests.
+
+        if start is self._empty:
+            return
+
+        if isinstance(start, list):
+            self.ranges = start[:]
+            self.clean()
+        else:
+            self.add(start, end)
+
+    @property
+    def last(self):
+        """
+        The largest number in use.
+        This is undefined until it has been set by assigning to this property.
+        """
+        return self._last
+
+    @last.setter
+    def last(self, value):
+        """
+        Replaces all occurrences of "*".  This should be the
+        largest number in use.  Must be set before attempting to
+        use the MessageSet as a container.
+
+        @raises ValueError: if a largest value has already been set.
+        """
+        if self._last is not self._empty:
+            raise ValueError("last already set")
+
+        self._last = value
+        for i, (low, high) in enumerate(self.ranges):
+            if low is None:
+                low = value
+            if high is None:
+                high = value
+            if low > high:
+                low, high = high, low
+            self.ranges[i] = (low, high)
+        self.clean()
+
+    def add(self, start, end=_empty):
+        """
+        Add another range
+
+        @type start: L{int}
+        @param start: Start of range, or only message number
+
+        @type end: Optional L{int}
+        @param end: End of range.
+        """
+        if end is self._empty:
+            end = start
+
+        if self._last is not self._empty:
+            if start is None:
+                start = self.last
+            if end is None:
+                end = self.last
+
+        start, end = sorted(
+            [start, end], key=functools.partial(_swap, that=self._infinity, ifIs=None)
+        )
+        self.ranges.append((start, end))
+        self.clean()
+
+    def __add__(self, other):
+        if isinstance(other, MessageSet):
+            ranges = self.ranges + other.ranges
+            return MessageSet(ranges)
+        else:
+            res = MessageSet(self.ranges)
+            if self.last is not self._empty:
+                res.last = self.last
+            try:
+                res.add(*other)
+            except TypeError:
+                res.add(other)
+            return res
+
+    def extend(self, other):
+        """
+        Extend our messages with another message or set of messages.
+
+        @param other: The messages to include.
+        @type other: L{MessageSet}, L{tuple} of two L{int}s, or a
+            single L{int}
+        """
+        if isinstance(other, MessageSet):
+            self.ranges.extend(other.ranges)
+            self.clean()
+        else:
+            try:
+                self.add(*other)
+            except TypeError:
+                self.add(other)
+
+        return self
+
+    def clean(self):
+        """
+        Clean ranges list, combining adjacent ranges
+        """
+
+        ranges = sorted(_swapAllPairs(self.ranges, that=self._infinity, ifIs=None))
+
+        mergedRanges = [(float("-inf"), float("-inf"))]
+
+        for low, high in ranges:
+            previousLow, previousHigh = mergedRanges[-1]
+
+            if previousHigh < low - 1:
+                mergedRanges.append((low, high))
+                continue
+
+            mergedRanges[-1] = (min(previousLow, low), max(previousHigh, high))
+
+        self.ranges = _swapAllPairs(mergedRanges[1:], that=None, ifIs=self._infinity)
+
+    def _noneInRanges(self):
+        """
+        Is there a L{None} in our ranges?
+
+        L{MessageSet.clean} merges overlapping or consecutive ranges.
+        None is represents a value larger than any number.  There are
+        thus two cases:
+
+            1. C{(x, *) + (y, z)} such that C{x} is smaller than C{y}
+
+            2. C{(z, *) + (x, y)} such that C{z} is larger than C{y}
+
+        (Other cases, such as C{y < x < z}, can be split into these
+        two cases; for example C{(y - 1, y)} + C{(x, x) + (z, z + 1)})
+
+        In case 1, C{* > y} and C{* > z}, so C{(x, *) + (y, z) = (x,
+        *)}
+
+        In case 2, C{z > x and z > y}, so the intervals do not merge,
+        and the ranges are sorted as C{[(x, y), (z, *)]}.  C{*} is
+        represented as C{(*, *)}, so this is the same as 2.  but with
+        a C{z} that is greater than everything.
+
+        The result is that there is a maximum of two L{None}s, and one
+        of them has to be the high element in the last tuple in
+        C{self.ranges}.  That means checking if C{self.ranges[-1][-1]}
+        is L{None} suffices to check if I{any} element is L{None}.
+
+        @return: L{True} if L{None} is in some range in ranges and
+            L{False} if otherwise.
+        """
+        return self.ranges[-1][-1] is None
+
+    def __contains__(self, value):
+        """
+        May raise TypeError if we encounter an open-ended range
+
+        @param value: Is this in our ranges?
+        @type value: L{int}
+        """
+
+        if self._noneInRanges():
+            raise TypeError("Can't determine membership; last value not set")
+
+        for low, high in self.ranges:
+            if low <= value <= high:
+                return True
+
+        return False
+
+    def _iterator(self):
+        for l, h in self.ranges:
+            l = self.getnext(l - 1)
+            while l <= h:
+                yield l
+                l = self.getnext(l)
+
+    def __iter__(self):
+        if self._noneInRanges():
+            raise TypeError("Can't iterate; last value not set")
+
+        return self._iterator()
+
+    def __len__(self):
+        res = 0
+        for l, h in self.ranges:
+            if l is None:
+                res += 1
+            elif h is None:
+                raise TypeError("Can't size object; last value not set")
+            else:
+                res += (h - l) + 1
+
+        return res
+
+    def __str__(self) -> str:
+        p = []
+        for low, high in self.ranges:
+            if low == high:
+                if low is None:
+                    p.append("*")
+                else:
+                    p.append(str(low))
+            elif high is None:
+                p.append("%d:*" % (low,))
+            else:
+                p.append("%d:%d" % (low, high))
+        return ",".join(p)
+
+    def __repr__(self) -> str:
+        return f"<MessageSet {str(self)}>"
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, MessageSet):
+            return cast(bool, self.ranges == other.ranges)
+        return NotImplemented
+
+
+class LiteralString:
+    def __init__(self, size, defered):
+        self.size = size
+        self.data = []
+        self.defer = defered
+
+    def write(self, data):
+        self.size -= len(data)
+        passon = None
+        if self.size > 0:
+            self.data.append(data)
+        else:
+            if self.size:
+                data, passon = data[: self.size], data[self.size :]
+            else:
+                passon = b""
+            if data:
+                self.data.append(data)
+
+        return passon
+
+    def callback(self, line):
+        """
+        Call deferred with data and rest of line
+        """
+        self.defer.callback((b"".join(self.data), line))
+
+
+class LiteralFile:
+    _memoryFileLimit = 1024 * 1024 * 10
+
+    def __init__(self, size, defered):
+        self.size = size
+        self.defer = defered
+        if size > self._memoryFileLimit:
+            self.data = tempfile.TemporaryFile()
+        else:
+            self.data = BytesIO()
+
+    def write(self, data):
+        self.size -= len(data)
+        passon = None
+        if self.size > 0:
+            self.data.write(data)
+        else:
+            if self.size:
+                data, passon = data[: self.size], data[self.size :]
+            else:
+                passon = b""
+            if data:
+                self.data.write(data)
+        return passon
+
+    def callback(self, line):
+        """
+        Call deferred with data and rest of line
+        """
+        self.data.seek(0, 0)
+        self.defer.callback((self.data, line))
+
+
+class WriteBuffer:
+    """
+    Buffer up a bunch of writes before sending them all to a transport at once.
+    """
+
+    def __init__(self, transport, size=8192):
+        self.bufferSize = size
+        self.transport = transport
+        self._length = 0
+        self._writes = []
+
+    def write(self, s):
+        self._length += len(s)
+        self._writes.append(s)
+        if self._length > self.bufferSize:
+            self.flush()
+
+    def flush(self):
+        if self._writes:
+            self.transport.writeSequence(self._writes)
+            self._writes = []
+            self._length = 0
+
+
+class Command:
+    _1_RESPONSES = (
+        b"CAPABILITY",
+        b"FLAGS",
+        b"LIST",
+        b"LSUB",
+        b"STATUS",
+        b"SEARCH",
+        b"NAMESPACE",
+    )
+    _2_RESPONSES = (b"EXISTS", b"EXPUNGE", b"FETCH", b"RECENT")
+    _OK_RESPONSES = (
+        b"UIDVALIDITY",
+        b"UNSEEN",
+        b"READ-WRITE",
+        b"READ-ONLY",
+        b"UIDNEXT",
+        b"PERMANENTFLAGS",
+    )
+    defer = None
+
+    def __init__(
+        self,
+        command,
+        args=None,
+        wantResponse=(),
+        continuation=None,
+        *contArgs,
+        **contKw,
+    ):
+        self.command = command
+        self.args = args
+        self.wantResponse = wantResponse
+        self.continuation = lambda x: continuation(x, *contArgs, **contKw)
+        self.lines = []
+
+    def __repr__(self) -> str:
+        return "<imap4.Command {!r} {!r} {!r} {!r} {!r}>".format(
+            self.command, self.args, self.wantResponse, self.continuation, self.lines
+        )
+
+    def format(self, tag):
+        if self.args is None:
+            return b" ".join((tag, self.command))
+        return b" ".join((tag, self.command, self.args))
+
+    def finish(self, lastLine, unusedCallback):
+        send = []
+        unuse = []
+        for L in self.lines:
+            names = parseNestedParens(L)
+            N = len(names)
+            if (
+                N >= 1
+                and names[0] in self._1_RESPONSES
+                or N >= 2
+                and names[1] in self._2_RESPONSES
+                or N >= 2
+                and names[0] == b"OK"
+                and isinstance(names[1], list)
+                and names[1][0] in self._OK_RESPONSES
+            ):
+                send.append(names)
+            else:
+                unuse.append(names)
+        d, self.defer = self.defer, None
+        d.callback((send, lastLine))
+        if unuse:
+            unusedCallback(unuse)
+
+
+# Some constants to help define what an atom is and is not - see the grammar
+# section of the IMAP4 RFC - <https://tools.ietf.org/html/rfc3501#section-9>.
+# Some definitions (SP, CTL, DQUOTE) are also from the ABNF RFC -
+# <https://tools.ietf.org/html/rfc2234>.
+_SP = b" "
+_CTL = bytes(chain(range(0x21), range(0x80, 0x100)))
+
+# It is easier to define ATOM-CHAR in terms of what it does not match than in
+# terms of what it does match.
+_nonAtomChars = b']\\\\(){%*"' + _SP + _CTL
+
+# _nonAtomRE is only used in Query, so it uses native strings.
+_nativeNonAtomChars = _nonAtomChars.decode("charmap")
+_nonAtomRE = re.compile("[" + _nativeNonAtomChars + "]")
+
+# This is all the bytes that match the ATOM-CHAR from the grammar in the RFC.
+_atomChars = bytes(ch for ch in range(0x100) if ch not in _nonAtomChars)
+
+
+@implementer(IMailboxListener)
+class IMAP4Server(basic.LineReceiver, policies.TimeoutMixin):
+    """
+    Protocol implementation for an IMAP4rev1 server.
+
+    The server can be in any of four states:
+        - Non-authenticated
+        - Authenticated
+        - Selected
+        - Logout
+    """
+
+    # Identifier for this server software
+    IDENT = b"Twisted IMAP4rev1 Ready"
+
+    # Number of seconds before idle timeout
+    # Initially 1 minute.  Raised to 30 minutes after login.
+    timeOut = 60
+
+    POSTAUTH_TIMEOUT = 60 * 30
+
+    # Whether STARTTLS has been issued successfully yet or not.
+    startedTLS = False
+
+    # Whether our transport supports TLS
+    canStartTLS = False
+
+    # Mapping of tags to commands we have received
+    tags = None
+
+    # The object which will handle logins for us
+    portal = None
+
+    # The account object for this connection
+    account = None
+
+    # Logout callback
+    _onLogout = None
+
+    # The currently selected mailbox
+    mbox = None
+
+    # Command data to be processed when literal data is received
+    _pendingLiteral = None
+
+    # Maximum length to accept for a "short" string literal
+    _literalStringLimit = 4096
+
+    # IChallengeResponse factories for AUTHENTICATE command
+    challengers = None
+
+    # Search terms the implementation of which needs to be passed both the last
+    # message identifier (UID) and the last sequence id.
+    _requiresLastMessageInfo = {b"OR", b"NOT", b"UID"}
+
+    state = "unauth"
+
+    parseState = "command"
+
+    def __init__(self, chal=None, contextFactory=None, scheduler=None):
+        if chal is None:
+            chal = {}
+        self.challengers = chal
+        self.ctx = contextFactory
+        if scheduler is None:
+            scheduler = iterateInReactor
+        self._scheduler = scheduler
+        self._queuedAsync = []
+
+    def capabilities(self):
+        cap = {b"AUTH": list(self.challengers.keys())}
+        if self.ctx and self.canStartTLS:
+            if (
+                not self.startedTLS
+                and interfaces.ISSLTransport(self.transport, None) is None
+            ):
+                cap[b"LOGINDISABLED"] = None
+                cap[b"STARTTLS"] = None
+        cap[b"NAMESPACE"] = None
+        cap[b"IDLE"] = None
+        return cap
+
+    def connectionMade(self):
+        self.tags = {}
+        self.canStartTLS = interfaces.ITLSTransport(self.transport, None) is not None
+        self.setTimeout(self.timeOut)
+        self.sendServerGreeting()
+
+    def connectionLost(self, reason):
+        self.setTimeout(None)
+        if self._onLogout:
+            self._onLogout()
+            self._onLogout = None
+
+    def timeoutConnection(self):
+        self.sendLine(b"* BYE Autologout; connection idle too long")
+        self.transport.loseConnection()
+        if self.mbox:
+            self.mbox.removeListener(self)
+            cmbx = ICloseableMailbox(self.mbox, None)
+            if cmbx is not None:
+                maybeDeferred(cmbx.close).addErrback(log.err)
+            self.mbox = None
+        self.state = "timeout"
+
+    def rawDataReceived(self, data):
+        self.resetTimeout()
+        passon = self._pendingLiteral.write(data)
+        if passon is not None:
+            self.setLineMode(passon)
+
+    # Avoid processing commands while buffers are being dumped to
+    # our transport
+    blocked = None
+
+    def _unblock(self):
+        commands = self.blocked
+        self.blocked = None
+        while commands and self.blocked is None:
+            self.lineReceived(commands.pop(0))
+        if self.blocked is not None:
+            self.blocked.extend(commands)
+
+    def lineReceived(self, line):
+        if self.blocked is not None:
+            self.blocked.append(line)
+            return
+
+        self.resetTimeout()
+        f = getattr(self, "parse_" + self.parseState)
+        try:
+            f(line)
+        except Exception as e:
+            self.sendUntaggedResponse(b"BAD Server error: " + networkString(str(e)))
+            log.err()
+
+    def parse_command(self, line):
+        args = line.split(None, 2)
+        rest = None
+        if len(args) == 3:
+            tag, cmd, rest = args
+        elif len(args) == 2:
+            tag, cmd = args
+        elif len(args) == 1:
+            tag = args[0]
+            self.sendBadResponse(tag, b"Missing command")
+            return None
+        else:
+            self.sendBadResponse(None, b"Null command")
+            return None
+
+        cmd = cmd.upper()
+        try:
+            return self.dispatchCommand(tag, cmd, rest)
+        except IllegalClientResponse as e:
+            self.sendBadResponse(tag, b"Illegal syntax: " + networkString(str(e)))
+        except IllegalOperation as e:
+            self.sendNegativeResponse(
+                tag, b"Illegal operation: " + networkString(str(e))
+            )
+        except IllegalMailboxEncoding as e:
+            self.sendNegativeResponse(
+                tag, b"Illegal mailbox name: " + networkString(str(e))
+            )
+
+    def parse_pending(self, line):
+        d = self._pendingLiteral
+        self._pendingLiteral = None
+        self.parseState = "command"
+        d.callback(line)
+
+    def dispatchCommand(self, tag, cmd, rest, uid=None):
+        f = self.lookupCommand(cmd)
+        if f:
+            fn = f[0]
+            parseargs = f[1:]
+            self.__doCommand(tag, fn, [self, tag], parseargs, rest, uid)
+        else:
+            self.sendBadResponse(tag, b"Unsupported command")
+
+    def lookupCommand(self, cmd):
+        return getattr(self, "_".join((self.state, nativeString(cmd.upper()))), None)
+
+    def __doCommand(self, tag, handler, args, parseargs, line, uid):
+        for i, arg in enumerate(parseargs):
+            if callable(arg):
+                parseargs = parseargs[i + 1 :]
+                maybeDeferred(arg, self, line).addCallback(
+                    self.__cbDispatch, tag, handler, args, parseargs, uid
+                ).addErrback(self.__ebDispatch, tag)
+                return
+            else:
+                args.append(arg)
+
+        if line:
+            # Too many arguments
+            raise IllegalClientResponse("Too many arguments for command: " + repr(line))
+
+        if uid is not None:
+            handler(uid=uid, *args)
+        else:
+            handler(*args)
+
+    def __cbDispatch(self, result, tag, fn, args, parseargs, uid):
+        (arg, rest) = result
+        args.append(arg)
+        self.__doCommand(tag, fn, args, parseargs, rest, uid)
+
+    def __ebDispatch(self, failure, tag):
+        if failure.check(IllegalClientResponse):
+            self.sendBadResponse(
+                tag, b"Illegal syntax: " + networkString(str(failure.value))
+            )
+        elif failure.check(IllegalOperation):
+            self.sendNegativeResponse(
+                tag, b"Illegal operation: " + networkString(str(failure.value))
+            )
+        elif failure.check(IllegalMailboxEncoding):
+            self.sendNegativeResponse(
+                tag, b"Illegal mailbox name: " + networkString(str(failure.value))
+            )
+        else:
+            self.sendBadResponse(
+                tag, b"Server error: " + networkString(str(failure.value))
+            )
+            log.err(failure)
+
+    def _stringLiteral(self, size):
+        if size > self._literalStringLimit:
+            raise IllegalClientResponse(
+                "Literal too long! I accept at most %d octets"
+                % (self._literalStringLimit,)
+            )
+        d = defer.Deferred()
+        self.parseState = "pending"
+        self._pendingLiteral = LiteralString(size, d)
+        self.sendContinuationRequest(
+            networkString("Ready for %d octets of text" % size)
+        )
+        self.setRawMode()
+        return d
+
+    def _fileLiteral(self, size):
+        d = defer.Deferred()
+        self.parseState = "pending"
+        self._pendingLiteral = LiteralFile(size, d)
+        self.sendContinuationRequest(
+            networkString("Ready for %d octets of data" % size)
+        )
+        self.setRawMode()
+        return d
+
+    def arg_finalastring(self, line):
+        """
+        Parse an astring from line that represents a command's final
+        argument.  This special case exists to enable parsing empty
+        string literals.
+
+        @param line: A line that contains a string literal.
+        @type line: L{bytes}
+
+        @return: A 2-tuple containing the parsed argument and any
+            trailing data, or a L{Deferred} that fires with that
+            2-tuple
+        @rtype: L{tuple} of (L{bytes}, L{bytes}) or a L{Deferred}
+
+        @see: https://twistedmatrix.com/trac/ticket/9207
+        """
+        return self.arg_astring(line, final=True)
+
+    def arg_astring(self, line, final=False):
+        """
+        Parse an astring from the line, return (arg, rest), possibly
+        via a deferred (to handle literals)
+
+        @param line: A line that contains a string literal.
+        @type line: L{bytes}
+
+        @param final: Is this the final argument?
+        @type final L{bool}
+
+        @return: A 2-tuple containing the parsed argument and any
+            trailing data, or a L{Deferred} that fires with that
+            2-tuple
+        @rtype: L{tuple} of (L{bytes}, L{bytes}) or a L{Deferred}
+
+        """
+        line = line.strip()
+        if not line:
+            raise IllegalClientResponse("Missing argument")
+        d = None
+        arg, rest = None, None
+        if line[0:1] == b'"':
+            try:
+                spam, arg, rest = line.split(b'"', 2)
+                rest = rest[1:]  # Strip space
+            except ValueError:
+                raise IllegalClientResponse("Unmatched quotes")
+        elif line[0:1] == b"{":
+            # literal
+            if line[-1:] != b"}":
+                raise IllegalClientResponse("Malformed literal")
+            try:
+                size = int(line[1:-1])
+            except ValueError:
+                raise IllegalClientResponse("Bad literal size: " + repr(line[1:-1]))
+            if final and not size:
+                return (b"", b"")
+            d = self._stringLiteral(size)
+        else:
+            arg = line.split(b" ", 1)
+            if len(arg) == 1:
+                arg.append(b"")
+            arg, rest = arg
+        return d or (arg, rest)
+
+    # ATOM: Any CHAR except ( ) { % * " \ ] CTL SP (CHAR is 7bit)
+    atomre = re.compile(
+        b"(?P<atom>[" + re.escape(_atomChars) + b"]+)( (?P<rest>.*$)|$)"
+    )
+
+    def arg_atom(self, line):
+        """
+        Parse an atom from the line
+        """
+        if not line:
+            raise IllegalClientResponse("Missing argument")
+        m = self.atomre.match(line)
+        if m:
+            return m.group("atom"), m.group("rest")
+        else:
+            raise IllegalClientResponse("Malformed ATOM")
+
+    def arg_plist(self, line):
+        """
+        Parse a (non-nested) parenthesised list from the line
+        """
+        if not line:
+            raise IllegalClientResponse("Missing argument")
+
+        if line[:1] != b"(":
+            raise IllegalClientResponse("Missing parenthesis")
+
+        i = line.find(b")")
+
+        if i == -1:
+            raise IllegalClientResponse("Mismatched parenthesis")
+
+        return (parseNestedParens(line[1:i], 0), line[i + 2 :])
+
+    def arg_literal(self, line):
+        """
+        Parse a literal from the line
+        """
+        if not line:
+            raise IllegalClientResponse("Missing argument")
+
+        if line[:1] != b"{":
+            raise IllegalClientResponse("Missing literal")
+
+        if line[-1:] != b"}":
+            raise IllegalClientResponse("Malformed literal")
+
+        try:
+            size = int(line[1:-1])
+        except ValueError:
+            raise IllegalClientResponse(f"Bad literal size: {line[1:-1]!r}")
+
+        return self._fileLiteral(size)
+
+    def arg_searchkeys(self, line):
+        """
+        searchkeys
+        """
+        query = parseNestedParens(line)
+        # XXX Should really use list of search terms and parse into
+        # a proper tree
+        return (query, b"")
+
+    def arg_seqset(self, line):
+        """
+        sequence-set
+        """
+        rest = b""
+        arg = line.split(b" ", 1)
+        if len(arg) == 2:
+            rest = arg[1]
+        arg = arg[0]
+
+        try:
+            return (parseIdList(arg), rest)
+        except IllegalIdentifierError as e:
+            raise IllegalClientResponse("Bad message number " + str(e))
+
+    def arg_fetchatt(self, line):
+        """
+        fetch-att
+        """
+        p = _FetchParser()
+        p.parseString(line)
+        return (p.result, b"")
+
+    def arg_flaglist(self, line):
+        """
+        Flag part of store-att-flag
+        """
+        flags = []
+        if line[0:1] == b"(":
+            if line[-1:] != b")":
+                raise IllegalClientResponse("Mismatched parenthesis")
+            line = line[1:-1]
+
+        while line:
+            m = self.atomre.search(line)
+            if not m:
+                raise IllegalClientResponse("Malformed flag")
+            if line[0:1] == b"\\" and m.start() == 1:
+                flags.append(b"\\" + m.group("atom"))
+            elif m.start() == 0:
+                flags.append(m.group("atom"))
+            else:
+                raise IllegalClientResponse("Malformed flag")
+            line = m.group("rest")
+
+        return (flags, b"")
+
+    def arg_line(self, line):
+        """
+        Command line of UID command
+        """
+        return (line, b"")
+
+    def opt_plist(self, line):
+        """
+        Optional parenthesised list
+        """
+        if line.startswith(b"("):
+            return self.arg_plist(line)
+        else:
+            return (None, line)
+
+    def opt_datetime(self, line):
+        """
+        Optional date-time string
+        """
+        if line.startswith(b'"'):
+            try:
+                spam, date, rest = line.split(b'"', 2)
+            except ValueError:
+                raise IllegalClientResponse("Malformed date-time")
+            return (date, rest[1:])
+        else:
+            return (None, line)
+
+    def opt_charset(self, line):
+        """
+        Optional charset of SEARCH command
+        """
+        if line[:7].upper() == b"CHARSET":
+            arg = line.split(b" ", 2)
+            if len(arg) == 1:
+                raise IllegalClientResponse("Missing charset identifier")
+            if len(arg) == 2:
+                arg.append(b"")
+            spam, arg, rest = arg
+            return (arg, rest)
+        else:
+            return (None, line)
+
+    def sendServerGreeting(self):
+        msg = b"[CAPABILITY " + b" ".join(self.listCapabilities()) + b"] " + self.IDENT
+        self.sendPositiveResponse(message=msg)
+
+    def sendBadResponse(self, tag=None, message=b""):
+        self._respond(b"BAD", tag, message)
+
+    def sendPositiveResponse(self, tag=None, message=b""):
+        self._respond(b"OK", tag, message)
+
+    def sendNegativeResponse(self, tag=None, message=b""):
+        self._respond(b"NO", tag, message)
+
+    def sendUntaggedResponse(self, message, isAsync=None, **kwargs):
+        isAsync = _get_async_param(isAsync, **kwargs)
+        if not isAsync or (self.blocked is None):
+            self._respond(message, None, None)
+        else:
+            self._queuedAsync.append(message)
+
+    def sendContinuationRequest(self, msg=b"Ready for additional command text"):
+        if msg:
+            self.sendLine(b"+ " + msg)
+        else:
+            self.sendLine(b"+")
+
+    def _respond(self, state, tag, message):
+        if state in (b"OK", b"NO", b"BAD") and self._queuedAsync:
+            lines = self._queuedAsync
+            self._queuedAsync = []
+            for msg in lines:
+                self._respond(msg, None, None)
+        if not tag:
+            tag = b"*"
+        if message:
+            self.sendLine(b" ".join((tag, state, message)))
+        else:
+            self.sendLine(b" ".join((tag, state)))
+
+    def listCapabilities(self):
+        caps = [b"IMAP4rev1"]
+        for c, v in self.capabilities().items():
+            if v is None:
+                caps.append(c)
+            elif len(v):
+                caps.extend([(c + b"=" + cap) for cap in v])
+        return caps
+
+    def do_CAPABILITY(self, tag):
+        self.sendUntaggedResponse(b"CAPABILITY " + b" ".join(self.listCapabilities()))
+        self.sendPositiveResponse(tag, b"CAPABILITY completed")
+
+    unauth_CAPABILITY = (do_CAPABILITY,)
+    auth_CAPABILITY = unauth_CAPABILITY
+    select_CAPABILITY = unauth_CAPABILITY
+    logout_CAPABILITY = unauth_CAPABILITY
+
+    def do_LOGOUT(self, tag):
+        self.sendUntaggedResponse(b"BYE Nice talking to you")
+        self.sendPositiveResponse(tag, b"LOGOUT successful")
+        self.transport.loseConnection()
+
+    unauth_LOGOUT = (do_LOGOUT,)
+    auth_LOGOUT = unauth_LOGOUT
+    select_LOGOUT = unauth_LOGOUT
+    logout_LOGOUT = unauth_LOGOUT
+
+    def do_NOOP(self, tag):
+        self.sendPositiveResponse(tag, b"NOOP No operation performed")
+
+    unauth_NOOP = (do_NOOP,)
+    auth_NOOP = unauth_NOOP
+    select_NOOP = unauth_NOOP
+    logout_NOOP = unauth_NOOP
+
+    def do_AUTHENTICATE(self, tag, args):
+        args = args.upper().strip()
+        if args not in self.challengers:
+            self.sendNegativeResponse(tag, b"AUTHENTICATE method unsupported")
+        else:
+            self.authenticate(self.challengers[args](), tag)
+
+    unauth_AUTHENTICATE = (do_AUTHENTICATE, arg_atom)
+
+    def authenticate(self, chal, tag):
+        if self.portal is None:
+            self.sendNegativeResponse(tag, b"Temporary authentication failure")
+            return
+
+        self._setupChallenge(chal, tag)
+
+    def _setupChallenge(self, chal, tag):
+        try:
+            challenge = chal.getChallenge()
+        except Exception as e:
+            self.sendBadResponse(tag, b"Server error: " + networkString(str(e)))
+        else:
+            coded = encodebytes(challenge)[:-1]
+            self.parseState = "pending"
+            self._pendingLiteral = defer.Deferred()
+            self.sendContinuationRequest(coded)
+            self._pendingLiteral.addCallback(self.__cbAuthChunk, chal, tag)
+            self._pendingLiteral.addErrback(self.__ebAuthChunk, tag)
+
+    def __cbAuthChunk(self, result, chal, tag):
+        try:
+            uncoded = decodebytes(result)
+        except binascii.Error:
+            raise IllegalClientResponse("Malformed Response - not base64")
+
+        chal.setResponse(uncoded)
+        if chal.moreChallenges():
+            self._setupChallenge(chal, tag)
+        else:
+            self.portal.login(chal, None, IAccount).addCallbacks(
+                self.__cbAuthResp, self.__ebAuthResp, (tag,), None, (tag,), None
+            )
+
+    def __cbAuthResp(self, result, tag):
+        (iface, avatar, logout) = result
+        assert iface is IAccount, "IAccount is the only supported interface"
+        self.account = avatar
+        self.state = "auth"
+        self._onLogout = logout
+        self.sendPositiveResponse(tag, b"Authentication successful")
+        self.setTimeout(self.POSTAUTH_TIMEOUT)
+
+    def __ebAuthResp(self, failure, tag):
+        if failure.check(UnauthorizedLogin):
+            self.sendNegativeResponse(tag, b"Authentication failed: unauthorized")
+        elif failure.check(UnhandledCredentials):
+            self.sendNegativeResponse(
+                tag, b"Authentication failed: server misconfigured"
+            )
+        else:
+            self.sendBadResponse(tag, b"Server error: login failed unexpectedly")
+            log.err(failure)
+
+    def __ebAuthChunk(self, failure, tag):
+        self.sendNegativeResponse(
+            tag, b"Authentication failed: " + networkString(str(failure.value))
+        )
+
+    def do_STARTTLS(self, tag):
+        if self.startedTLS:
+            self.sendNegativeResponse(tag, b"TLS already negotiated")
+        elif self.ctx and self.canStartTLS:
+            self.sendPositiveResponse(tag, b"Begin TLS negotiation now")
+            self.transport.startTLS(self.ctx)
+            self.startedTLS = True
+            self.challengers = self.challengers.copy()
+            if b"LOGIN" not in self.challengers:
+                self.challengers[b"LOGIN"] = LOGINCredentials
+            if b"PLAIN" not in self.challengers:
+                self.challengers[b"PLAIN"] = PLAINCredentials
+        else:
+            self.sendNegativeResponse(tag, b"TLS not available")
+
+    unauth_STARTTLS = (do_STARTTLS,)
+
+    def do_LOGIN(self, tag, user, passwd):
+        if b"LOGINDISABLED" in self.capabilities():
+            self.sendBadResponse(tag, b"LOGIN is disabled before STARTTLS")
+            return
+
+        maybeDeferred(self.authenticateLogin, user, passwd).addCallback(
+            self.__cbLogin, tag
+        ).addErrback(self.__ebLogin, tag)
+
+    unauth_LOGIN = (do_LOGIN, arg_astring, arg_finalastring)
+
+    def authenticateLogin(self, user, passwd):
+        """
+        Lookup the account associated with the given parameters
+
+        Override this method to define the desired authentication behavior.
+
+        The default behavior is to defer authentication to C{self.portal}
+        if it is not None, or to deny the login otherwise.
+
+        @type user: L{str}
+        @param user: The username to lookup
+
+        @type passwd: L{str}
+        @param passwd: The password to login with
+        """
+        if self.portal:
+            return self.portal.login(
+                credentials.UsernamePassword(user, passwd), None, IAccount
+            )
+        raise UnauthorizedLogin()
+
+    def __cbLogin(self, result, tag):
+        (iface, avatar, logout) = result
+        if iface is not IAccount:
+            self.sendBadResponse(tag, b"Server error: login returned unexpected value")
+            log.err(f"__cbLogin called with {iface!r}, IAccount expected")
+        else:
+            self.account = avatar
+            self._onLogout = logout
+            self.sendPositiveResponse(tag, b"LOGIN succeeded")
+            self.state = "auth"
+            self.setTimeout(self.POSTAUTH_TIMEOUT)
+
+    def __ebLogin(self, failure, tag):
+        if failure.check(UnauthorizedLogin):
+            self.sendNegativeResponse(tag, b"LOGIN failed")
+        else:
+            self.sendBadResponse(
+                tag, b"Server error: " + networkString(str(failure.value))
+            )
+            log.err(failure)
+
+    def do_NAMESPACE(self, tag):
+        personal = public = shared = None
+        np = INamespacePresenter(self.account, None)
+        if np is not None:
+            personal = np.getPersonalNamespaces()
+            public = np.getSharedNamespaces()
+            shared = np.getSharedNamespaces()
+        self.sendUntaggedResponse(
+            b"NAMESPACE " + collapseNestedLists([personal, public, shared])
+        )
+        self.sendPositiveResponse(tag, b"NAMESPACE command completed")
+
+    auth_NAMESPACE = (do_NAMESPACE,)
+    select_NAMESPACE = auth_NAMESPACE
+
+    def _selectWork(self, tag, name, rw, cmdName):
+        if self.mbox:
+            self.mbox.removeListener(self)
+            cmbx = ICloseableMailbox(self.mbox, None)
+            if cmbx is not None:
+                maybeDeferred(cmbx.close).addErrback(log.err)
+            self.mbox = None
+            self.state = "auth"
+
+        name = _parseMbox(name)
+        maybeDeferred(self.account.select, _parseMbox(name), rw).addCallback(
+            self._cbSelectWork, cmdName, tag
+        ).addErrback(self._ebSelectWork, cmdName, tag)
+
+    def _ebSelectWork(self, failure, cmdName, tag):
+        self.sendBadResponse(tag, cmdName + b" failed: Server error")
+        log.err(failure)
+
+    def _cbSelectWork(self, mbox, cmdName, tag):
+        if mbox is None:
+            self.sendNegativeResponse(tag, b"No such mailbox")
+            return
+        if "\\noselect" in [s.lower() for s in mbox.getFlags()]:
+            self.sendNegativeResponse(tag, "Mailbox cannot be selected")
+            return
+
+        flags = [networkString(flag) for flag in mbox.getFlags()]
+        self.sendUntaggedResponse(b"%d EXISTS" % (mbox.getMessageCount(),))
+        self.sendUntaggedResponse(b"%d RECENT" % (mbox.getRecentCount(),))
+        self.sendUntaggedResponse(b"FLAGS (" + b" ".join(flags) + b")")
+        self.sendPositiveResponse(None, b"[UIDVALIDITY %d]" % (mbox.getUIDValidity(),))
+
+        s = mbox.isWriteable() and b"READ-WRITE" or b"READ-ONLY"
+        mbox.addListener(self)
+        self.sendPositiveResponse(tag, b"[" + s + b"] " + cmdName + b" successful")
+        self.state = "select"
+        self.mbox = mbox
+
+    auth_SELECT = (_selectWork, arg_astring, 1, b"SELECT")
+    select_SELECT = auth_SELECT
+
+    auth_EXAMINE = (_selectWork, arg_astring, 0, b"EXAMINE")
+    select_EXAMINE = auth_EXAMINE
+
+    def do_IDLE(self, tag):
+        self.sendContinuationRequest(None)
+        self.parseTag = tag
+        self.lastState = self.parseState
+        self.parseState = "idle"
+
+    def parse_idle(self, *args):
+        self.parseState = self.lastState
+        del self.lastState
+        self.sendPositiveResponse(self.parseTag, b"IDLE terminated")
+        del self.parseTag
+
+    select_IDLE = (do_IDLE,)
+    auth_IDLE = select_IDLE
+
+    def do_CREATE(self, tag, name):
+        name = _parseMbox(name)
+        try:
+            result = self.account.create(name)
+        except MailboxException as c:
+            self.sendNegativeResponse(tag, networkString(str(c)))
+        except BaseException:
+            self.sendBadResponse(
+                tag, b"Server error encountered while creating mailbox"
+            )
+            log.err()
+        else:
+            if result:
+                self.sendPositiveResponse(tag, b"Mailbox created")
+            else:
+                self.sendNegativeResponse(tag, b"Mailbox not created")
+
+    auth_CREATE = (do_CREATE, arg_finalastring)
+    select_CREATE = auth_CREATE
+
+    def do_DELETE(self, tag, name):
+        name = _parseMbox(name)
+        if name.lower() == "inbox":
+            self.sendNegativeResponse(tag, b"You cannot delete the inbox")
+            return
+        try:
+            self.account.delete(name)
+        except MailboxException as m:
+            self.sendNegativeResponse(tag, str(m).encode("imap4-utf-7"))
+        except BaseException:
+            self.sendBadResponse(
+                tag, b"Server error encountered while deleting mailbox"
+            )
+            log.err()
+        else:
+            self.sendPositiveResponse(tag, b"Mailbox deleted")
+
+    auth_DELETE = (do_DELETE, arg_finalastring)
+    select_DELETE = auth_DELETE
+
+    def do_RENAME(self, tag, oldname, newname):
+        oldname, newname = (_parseMbox(n) for n in (oldname, newname))
+        if oldname.lower() == "inbox" or newname.lower() == "inbox":
+            self.sendNegativeResponse(
+                tag, b"You cannot rename the inbox, or rename another mailbox to inbox."
+            )
+            return
+        try:
+            self.account.rename(oldname, newname)
+        except TypeError:
+            self.sendBadResponse(tag, b"Invalid command syntax")
+        except MailboxException as m:
+            self.sendNegativeResponse(tag, networkString(str(m)))
+        except BaseException:
+            self.sendBadResponse(
+                tag, b"Server error encountered while renaming mailbox"
+            )
+            log.err()
+        else:
+            self.sendPositiveResponse(tag, b"Mailbox renamed")
+
+    auth_RENAME = (do_RENAME, arg_astring, arg_finalastring)
+    select_RENAME = auth_RENAME
+
+    def do_SUBSCRIBE(self, tag, name):
+        name = _parseMbox(name)
+        try:
+            self.account.subscribe(name)
+        except MailboxException as m:
+            self.sendNegativeResponse(tag, networkString(str(m)))
+        except BaseException:
+            self.sendBadResponse(
+                tag, b"Server error encountered while subscribing to mailbox"
+            )
+            log.err()
+        else:
+            self.sendPositiveResponse(tag, b"Subscribed")
+
+    auth_SUBSCRIBE = (do_SUBSCRIBE, arg_finalastring)
+    select_SUBSCRIBE = auth_SUBSCRIBE
+
+    def do_UNSUBSCRIBE(self, tag, name):
+        name = _parseMbox(name)
+        try:
+            self.account.unsubscribe(name)
+        except MailboxException as m:
+            self.sendNegativeResponse(tag, networkString(str(m)))
+        except BaseException:
+            self.sendBadResponse(
+                tag, b"Server error encountered while unsubscribing from mailbox"
+            )
+            log.err()
+        else:
+            self.sendPositiveResponse(tag, b"Unsubscribed")
+
+    auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_finalastring)
+    select_UNSUBSCRIBE = auth_UNSUBSCRIBE
+
+    def _listWork(self, tag, ref, mbox, sub, cmdName):
+        mbox = _parseMbox(mbox)
+        ref = _parseMbox(ref)
+        maybeDeferred(self.account.listMailboxes, ref, mbox).addCallback(
+            self._cbListWork, tag, sub, cmdName
+        ).addErrback(self._ebListWork, tag)
+
+    def _cbListWork(self, mailboxes, tag, sub, cmdName):
+        for name, box in mailboxes:
+            if not sub or self.account.isSubscribed(name):
+                flags = [networkString(flag) for flag in box.getFlags()]
+                delim = box.getHierarchicalDelimiter().encode("imap4-utf-7")
+                resp = (
+                    DontQuoteMe(cmdName),
+                    map(DontQuoteMe, flags),
+                    delim,
+                    name.encode("imap4-utf-7"),
+                )
+                self.sendUntaggedResponse(collapseNestedLists(resp))
+        self.sendPositiveResponse(tag, cmdName + b" completed")
+
+    def _ebListWork(self, failure, tag):
+        self.sendBadResponse(tag, b"Server error encountered while listing mailboxes.")
+        log.err(failure)
+
+    auth_LIST = (_listWork, arg_astring, arg_astring, 0, b"LIST")
+    select_LIST = auth_LIST
+
+    auth_LSUB = (_listWork, arg_astring, arg_astring, 1, b"LSUB")
+    select_LSUB = auth_LSUB
+
+    def do_STATUS(self, tag, mailbox, names):
+        nativeNames = []
+        for name in names:
+            nativeNames.append(nativeString(name))
+
+        mailbox = _parseMbox(mailbox)
+
+        maybeDeferred(self.account.select, mailbox, 0).addCallback(
+            self._cbStatusGotMailbox, tag, mailbox, nativeNames
+        ).addErrback(self._ebStatusGotMailbox, tag)
+
+    def _cbStatusGotMailbox(self, mbox, tag, mailbox, names):
+        if mbox:
+            maybeDeferred(mbox.requestStatus, names).addCallbacks(
+                self.__cbStatus,
+                self.__ebStatus,
+                (tag, mailbox),
+                None,
+                (tag, mailbox),
+                None,
+            )
+        else:
+            self.sendNegativeResponse(tag, b"Could not open mailbox")
+
+    def _ebStatusGotMailbox(self, failure, tag):
+        self.sendBadResponse(tag, b"Server error encountered while opening mailbox.")
+        log.err(failure)
+
+    auth_STATUS = (do_STATUS, arg_astring, arg_plist)
+    select_STATUS = auth_STATUS
+
+    def __cbStatus(self, status, tag, box):
+        # STATUS names should only be ASCII
+        line = networkString(" ".join(["%s %s" % x for x in status.items()]))
+        self.sendUntaggedResponse(
+            b"STATUS " + box.encode("imap4-utf-7") + b" (" + line + b")"
+        )
+        self.sendPositiveResponse(tag, b"STATUS complete")
+
+    def __ebStatus(self, failure, tag, box):
+        self.sendBadResponse(
+            tag, b"STATUS " + box + b" failed: " + networkString(str(failure.value))
+        )
+
+    def do_APPEND(self, tag, mailbox, flags, date, message):
+        mailbox = _parseMbox(mailbox)
+        maybeDeferred(self.account.select, mailbox).addCallback(
+            self._cbAppendGotMailbox, tag, flags, date, message
+        ).addErrback(self._ebAppendGotMailbox, tag)
+
+    def _cbAppendGotMailbox(self, mbox, tag, flags, date, message):
+        if not mbox:
+            self.sendNegativeResponse(tag, "[TRYCREATE] No such mailbox")
+            return
+
+        decodedFlags = [nativeString(flag) for flag in flags]
+        d = mbox.addMessage(message, decodedFlags, date)
+        d.addCallback(self.__cbAppend, tag, mbox)
+        d.addErrback(self.__ebAppend, tag)
+
+    def _ebAppendGotMailbox(self, failure, tag):
+        self.sendBadResponse(tag, b"Server error encountered while opening mailbox.")
+        log.err(failure)
+
+    auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, arg_literal)
+    select_APPEND = auth_APPEND
+
+    def __cbAppend(self, result, tag, mbox):
+        self.sendUntaggedResponse(b"%d EXISTS" % (mbox.getMessageCount(),))
+        self.sendPositiveResponse(tag, b"APPEND complete")
+
+    def __ebAppend(self, failure, tag):
+        self.sendBadResponse(
+            tag, b"APPEND failed: " + networkString(str(failure.value))
+        )
+
+    def do_CHECK(self, tag):
+        d = self.checkpoint()
+        if d is None:
+            self.__cbCheck(None, tag)
+        else:
+            d.addCallbacks(
+                self.__cbCheck, self.__ebCheck, callbackArgs=(tag,), errbackArgs=(tag,)
+            )
+
+    select_CHECK = (do_CHECK,)
+
+    def __cbCheck(self, result, tag):
+        self.sendPositiveResponse(tag, b"CHECK completed")
+
+    def __ebCheck(self, failure, tag):
+        self.sendBadResponse(tag, b"CHECK failed: " + networkString(str(failure.value)))
+
+    def checkpoint(self):
+        """
+        Called when the client issues a CHECK command.
+
+        This should perform any checkpoint operations required by the server.
+        It may be a long running operation, but may not block.  If it returns
+        a deferred, the client will only be informed of success (or failure)
+        when the deferred's callback (or errback) is invoked.
+        """
+        return None
+
+    def do_CLOSE(self, tag):
+        d = None
+        if self.mbox.isWriteable():
+            d = maybeDeferred(self.mbox.expunge)
+        cmbx = ICloseableMailbox(self.mbox, None)
+        if cmbx is not None:
+            if d is not None:
+                d.addCallback(lambda result: cmbx.close())
+            else:
+                d = maybeDeferred(cmbx.close)
+        if d is not None:
+            d.addCallbacks(self.__cbClose, self.__ebClose, (tag,), None, (tag,), None)
+        else:
+            self.__cbClose(None, tag)
+
+    select_CLOSE = (do_CLOSE,)
+
+    def __cbClose(self, result, tag):
+        self.sendPositiveResponse(tag, b"CLOSE completed")
+        self.mbox.removeListener(self)
+        self.mbox = None
+        self.state = "auth"
+
+    def __ebClose(self, failure, tag):
+        self.sendBadResponse(tag, b"CLOSE failed: " + networkString(str(failure.value)))
+
+    def do_EXPUNGE(self, tag):
+        if self.mbox.isWriteable():
+            maybeDeferred(self.mbox.expunge).addCallbacks(
+                self.__cbExpunge, self.__ebExpunge, (tag,), None, (tag,), None
+            )
+        else:
+            self.sendNegativeResponse(tag, b"EXPUNGE ignored on read-only mailbox")
+
+    select_EXPUNGE = (do_EXPUNGE,)
+
+    def __cbExpunge(self, result, tag):
+        for e in result:
+            self.sendUntaggedResponse(b"%d EXPUNGE" % (e,))
+        self.sendPositiveResponse(tag, b"EXPUNGE completed")
+
+    def __ebExpunge(self, failure, tag):
+        self.sendBadResponse(
+            tag, b"EXPUNGE failed: " + networkString(str(failure.value))
+        )
+        log.err(failure)
+
+    def do_SEARCH(self, tag, charset, query, uid=0):
+        sm = ISearchableMailbox(self.mbox, None)
+        if sm is not None:
+            maybeDeferred(sm.search, query, uid=uid).addCallback(
+                self.__cbSearch, tag, self.mbox, uid
+            ).addErrback(self.__ebSearch, tag)
+        else:
+            # that's not the ideal way to get all messages, there should be a
+            # method on mailboxes that gives you all of them
+            s = parseIdList(b"1:*")
+            maybeDeferred(self.mbox.fetch, s, uid=uid).addCallback(
+                self.__cbManualSearch, tag, self.mbox, query, uid
+            ).addErrback(self.__ebSearch, tag)
+
+    select_SEARCH = (do_SEARCH, opt_charset, arg_searchkeys)
+
+    def __cbSearch(self, result, tag, mbox, uid):
+        if uid:
+            result = map(mbox.getUID, result)
+        ids = networkString(" ".join([str(i) for i in result]))
+        self.sendUntaggedResponse(b"SEARCH " + ids)
+        self.sendPositiveResponse(tag, b"SEARCH completed")
+
+    def __cbManualSearch(self, result, tag, mbox, query, uid, searchResults=None):
+        """
+        Apply the search filter to a set of messages. Send the response to the
+        client.
+
+        @type result: L{list} of L{tuple} of (L{int}, provider of
+            L{imap4.IMessage})
+        @param result: A list two tuples of messages with their sequence ids,
+            sorted by the ids in descending order.
+
+        @type tag: L{str}
+        @param tag: A command tag.
+
+        @type mbox: Provider of L{imap4.IMailbox}
+        @param mbox: The searched mailbox.
+
+        @type query: L{list}
+        @param query: A list representing the parsed form of the search query.
+
+        @param uid: A flag indicating whether the search is over message
+            sequence numbers or UIDs.
+
+        @type searchResults: L{list}
+        @param searchResults: The search results so far or L{None} if no
+            results yet.
+        """
+        if searchResults is None:
+            searchResults = []
+        i = 0
+
+        # result is a list of tuples (sequenceId, Message)
+        lastSequenceId = result and result[-1][0]
+        lastMessageId = result and result[-1][1].getUID()
+        for i, (msgId, msg) in list(zip(range(5), result)):
+            # searchFilter and singleSearchStep will mutate the query.  Dang.
+            # Copy it here or else things will go poorly for subsequent
+            # messages.
+            if self._searchFilter(
+                copy.deepcopy(query), msgId, msg, lastSequenceId, lastMessageId
+            ):
+                searchResults.append(b"%d" % (msg.getUID() if uid else msgId,))
+
+        if i == 4:
+            from twisted.internet import reactor
+
+            reactor.callLater(
+                0,
+                self.__cbManualSearch,
+                list(result[5:]),
+                tag,
+                mbox,
+                query,
+                uid,
+                searchResults,
+            )
+        else:
+            if searchResults:
+                self.sendUntaggedResponse(b"SEARCH " + b" ".join(searchResults))
+            self.sendPositiveResponse(tag, b"SEARCH completed")
+
+    def _searchFilter(self, query, id, msg, lastSequenceId, lastMessageId):
+        """
+        Pop search terms from the beginning of C{query} until there are none
+        left and apply them to the given message.
+
+        @param query: A list representing the parsed form of the search query.
+
+        @param id: The sequence number of the message being checked.
+
+        @param msg: The message being checked.
+
+        @type lastSequenceId: L{int}
+        @param lastSequenceId: The highest sequence number of any message in
+            the mailbox being searched.
+
+        @type lastMessageId: L{int}
+        @param lastMessageId: The highest UID of any message in the mailbox
+            being searched.
+
+        @return: Boolean indicating whether all of the query terms match the
+            message.
+        """
+        while query:
+            if not self._singleSearchStep(
+                query, id, msg, lastSequenceId, lastMessageId
+            ):
+                return False
+        return True
+
+    def _singleSearchStep(self, query, msgId, msg, lastSequenceId, lastMessageId):
+        """
+        Pop one search term from the beginning of C{query} (possibly more than
+        one element) and return whether it matches the given message.
+
+        @param query: A list representing the parsed form of the search query.
+
+        @param msgId: The sequence number of the message being checked.
+
+        @param msg: The message being checked.
+
+        @param lastSequenceId: The highest sequence number of any message in
+            the mailbox being searched.
+
+        @param lastMessageId: The highest UID of any message in the mailbox
+            being searched.
+
+        @return: Boolean indicating whether the query term matched the message.
+        """
+
+        q = query.pop(0)
+        if isinstance(q, list):
+            if not self._searchFilter(q, msgId, msg, lastSequenceId, lastMessageId):
+                return False
+        else:
+            c = q.upper()
+            if not c[:1].isalpha():
+                # A search term may be a word like ALL, ANSWERED, BCC, etc (see
+                # below) or it may be a message sequence set.  Here we
+                # recognize a message sequence set "N:M".
+                messageSet = parseIdList(c, lastSequenceId)
+                return msgId in messageSet
+            else:
+                f = getattr(self, "search_" + nativeString(c), None)
+                if f is None:
+                    raise IllegalQueryError(
+                        "Invalid search command %s" % nativeString(c)
+                    )
+
+                if c in self._requiresLastMessageInfo:
+                    result = f(query, msgId, msg, (lastSequenceId, lastMessageId))
+                else:
+                    result = f(query, msgId, msg)
+
+                if not result:
+                    return False
+        return True
+
+    def search_ALL(self, query, id, msg):
+        """
+        Returns C{True} if the message matches the ALL search key (always).
+
+        @type query: A L{list} of L{str}
+        @param query: A list representing the parsed query string.
+
+        @type id: L{int}
+        @param id: The sequence number of the message being checked.
+
+        @type msg: Provider of L{imap4.IMessage}
+        """
+        return True
+
+    def search_ANSWERED(self, query, id, msg):
+        """
+        Returns C{True} if the message has been answered.
+
+        @type query: A L{list} of L{str}
+        @param query: A list representing the parsed query string.
+
+        @type id: L{int}
+        @param id: The sequence number of the message being checked.
+
+        @type msg: Provider of L{imap4.IMessage}
+        """
+        return "\\Answered" in msg.getFlags()
+
+    def search_BCC(self, query, id, msg):
+        """
+        Returns C{True} if the message has a BCC address matching the query.
+
+        @type query: A L{list} of L{str}
+        @param query: A list whose first element is a BCC L{str}
+
+        @type id: L{int}
+        @param id: The sequence number of the message being checked.
+
+        @type msg: Provider of L{imap4.IMessage}
+        """
+        bcc = msg.getHeaders(False, "bcc").get("bcc", "")
+        return bcc.lower().find(query.pop(0).lower()) != -1
+
+    def search_BEFORE(self, query, id, msg):
+        date = parseTime(query.pop(0))
+        return email.utils.parsedate(nativeString(msg.getInternalDate())) < date
+
+    def search_BODY(self, query, id, msg):
+        body = query.pop(0).lower()
+        return text.strFile(body, msg.getBodyFile(), False)
+
+    def search_CC(self, query, id, msg):
+        cc = msg.getHeaders(False, "cc").get("cc", "")
+        return cc.lower().find(query.pop(0).lower()) != -1
+
+    def search_DELETED(self, query, id, msg):
+        return "\\Deleted" in msg.getFlags()
+
+    def search_DRAFT(self, query, id, msg):
+        return "\\Draft" in msg.getFlags()
+
+    def search_FLAGGED(self, query, id, msg):
+        return "\\Flagged" in msg.getFlags()
+
+    def search_FROM(self, query, id, msg):
+        fm = msg.getHeaders(False, "from").get("from", "")
+        return fm.lower().find(query.pop(0).lower()) != -1
+
+    def search_HEADER(self, query, id, msg):
+        hdr = query.pop(0).lower()
+        hdr = msg.getHeaders(False, hdr).get(hdr, "")
+        return hdr.lower().find(query.pop(0).lower()) != -1
+
+    def search_KEYWORD(self, query, id, msg):
+        query.pop(0)
+        return False
+
+    def search_LARGER(self, query, id, msg):
+        return int(query.pop(0)) < msg.getSize()
+
+    def search_NEW(self, query, id, msg):
+        return "\\Recent" in msg.getFlags() and "\\Seen" not in msg.getFlags()
+
+    def search_NOT(self, query, id, msg, lastIDs):
+        """
+        Returns C{True} if the message does not match the query.
+
+        @type query: A L{list} of L{str}
+        @param query: A list representing the parsed form of the search query.
+
+        @type id: L{int}
+        @param id: The sequence number of the message being checked.
+
+        @type msg: Provider of L{imap4.IMessage}
+        @param msg: The message being checked.
+
+        @type lastIDs: L{tuple}
+        @param lastIDs: A tuple of (last sequence id, last message id).
+        The I{last sequence id} is an L{int} containing the highest sequence
+        number of a message in the mailbox.  The I{last message id} is an
+        L{int} containing the highest UID of a message in the mailbox.
+        """
+        (lastSequenceId, lastMessageId) = lastIDs
+        return not self._singleSearchStep(query, id, msg, lastSequenceId, lastMessageId)
+
+    def search_OLD(self, query, id, msg):
+        return "\\Recent" not in msg.getFlags()
+
+    def search_ON(self, query, id, msg):
+        date = parseTime(query.pop(0))
+        return email.utils.parsedate(msg.getInternalDate()) == date
+
+    def search_OR(self, query, id, msg, lastIDs):
+        """
+        Returns C{True} if the message matches any of the first two query
+        items.
+
+        @type query: A L{list} of L{str}
+        @param query: A list representing the parsed form of the search query.
+
+        @type id: L{int}
+        @param id: The sequence number of the message being checked.
+
+        @type msg: Provider of L{imap4.IMessage}
+        @param msg: The message being checked.
+
+        @type lastIDs: L{tuple}
+        @param lastIDs: A tuple of (last sequence id, last message id).
+        The I{last sequence id} is an L{int} containing the highest sequence
+        number of a message in the mailbox.  The I{last message id} is an
+        L{int} containing the highest UID of a message in the mailbox.
+        """
+        (lastSequenceId, lastMessageId) = lastIDs
+        a = self._singleSearchStep(query, id, msg, lastSequenceId, lastMessageId)
+        b = self._singleSearchStep(query, id, msg, lastSequenceId, lastMessageId)
+        return a or b
+
+    def search_RECENT(self, query, id, msg):
+        return "\\Recent" in msg.getFlags()
+
+    def search_SEEN(self, query, id, msg):
+        return "\\Seen" in msg.getFlags()
+
+    def search_SENTBEFORE(self, query, id, msg):
+        """
+        Returns C{True} if the message date is earlier than the query date.
+
+        @type query: A L{list} of L{str}
+        @param query: A list whose first element starts with a stringified date
+            that is a fragment of an L{imap4.Query()}. The date must be in the
+            format 'DD-Mon-YYYY', for example '03-March-2003' or '03-Mar-2003'.
+
+        @type id: L{int}
+        @param id: The sequence number of the message being checked.
+
+        @type msg: Provider of L{imap4.IMessage}
+        """
+        date = msg.getHeaders(False, "date").get("date", "")
+        date = email.utils.parsedate(date)
+        return date < parseTime(query.pop(0))
+
+    def search_SENTON(self, query, id, msg):
+        """
+        Returns C{True} if the message date is the same as the query date.
+
+        @type query: A L{list} of L{str}
+        @param query: A list whose first element starts with a stringified date
+            that is a fragment of an L{imap4.Query()}. The date must be in the
+            format 'DD-Mon-YYYY', for example '03-March-2003' or '03-Mar-2003'.
+
+        @type msg: Provider of L{imap4.IMessage}
+        """
+        date = msg.getHeaders(False, "date").get("date", "")
+        date = email.utils.parsedate(date)
+        return date[:3] == parseTime(query.pop(0))[:3]
+
+    def search_SENTSINCE(self, query, id, msg):
+        """
+        Returns C{True} if the message date is later than the query date.
+
+        @type query: A L{list} of L{str}
+        @param query: A list whose first element starts with a stringified date
+            that is a fragment of an L{imap4.Query()}. The date must be in the
+            format 'DD-Mon-YYYY', for example '03-March-2003' or '03-Mar-2003'.
+
+        @type msg: Provider of L{imap4.IMessage}
+        """
+        date = msg.getHeaders(False, "date").get("date", "")
+        date = email.utils.parsedate(date)
+        return date > parseTime(query.pop(0))
+
+    def search_SINCE(self, query, id, msg):
+        date = parseTime(query.pop(0))
+        return email.utils.parsedate(msg.getInternalDate()) > date
+
+    def search_SMALLER(self, query, id, msg):
+        return int(query.pop(0)) > msg.getSize()
+
+    def search_SUBJECT(self, query, id, msg):
+        subj = msg.getHeaders(False, "subject").get("subject", "")
+        return subj.lower().find(query.pop(0).lower()) != -1
+
+    def search_TEXT(self, query, id, msg):
+        # XXX - This must search headers too
+        body = query.pop(0).lower()
+        return text.strFile(body, msg.getBodyFile(), False)
+
+    def search_TO(self, query, id, msg):
+        to = msg.getHeaders(False, "to").get("to", "")
+        return to.lower().find(query.pop(0).lower()) != -1
+
+    def search_UID(self, query, id, msg, lastIDs):
+        """
+        Returns C{True} if the message UID is in the range defined by the
+        search query.
+
+        @type query: A L{list} of L{bytes}
+        @param query: A list representing the parsed form of the search
+            query. Its first element should be a L{str} that can be interpreted
+            as a sequence range, for example '2:4,5:*'.
+
+        @type id: L{int}
+        @param id: The sequence number of the message being checked.
+
+        @type msg: Provider of L{imap4.IMessage}
+        @param msg: The message being checked.
+
+        @type lastIDs: L{tuple}
+        @param lastIDs: A tuple of (last sequence id, last message id).
+        The I{last sequence id} is an L{int} containing the highest sequence
+        number of a message in the mailbox.  The I{last message id} is an
+        L{int} containing the highest UID of a message in the mailbox.
+        """
+        (lastSequenceId, lastMessageId) = lastIDs
+        c = query.pop(0)
+        m = parseIdList(c, lastMessageId)
+        return msg.getUID() in m
+
+    def search_UNANSWERED(self, query, id, msg):
+        return "\\Answered" not in msg.getFlags()
+
+    def search_UNDELETED(self, query, id, msg):
+        return "\\Deleted" not in msg.getFlags()
+
+    def search_UNDRAFT(self, query, id, msg):
+        return "\\Draft" not in msg.getFlags()
+
+    def search_UNFLAGGED(self, query, id, msg):
+        return "\\Flagged" not in msg.getFlags()
+
+    def search_UNKEYWORD(self, query, id, msg):
+        query.pop(0)
+        return False
+
+    def search_UNSEEN(self, query, id, msg):
+        return "\\Seen" not in msg.getFlags()
+
+    def __ebSearch(self, failure, tag):
+        self.sendBadResponse(
+            tag, b"SEARCH failed: " + networkString(str(failure.value))
+        )
+        log.err(failure)
+
+    def do_FETCH(self, tag, messages, query, uid=0):
+        if query:
+            self._oldTimeout = self.setTimeout(None)
+            maybeDeferred(self.mbox.fetch, messages, uid=uid).addCallback(
+                iter
+            ).addCallback(self.__cbFetch, tag, query, uid).addErrback(
+                self.__ebFetch, tag
+            )
+        else:
+            self.sendPositiveResponse(tag, b"FETCH complete")
+
+    select_FETCH = (do_FETCH, arg_seqset, arg_fetchatt)
+
+    def __cbFetch(self, results, tag, query, uid):
+        if self.blocked is None:
+            self.blocked = []
+        try:
+            id, msg = next(results)
+        except StopIteration:
+            # The idle timeout was suspended while we delivered results,
+            # restore it now.
+            self.setTimeout(self._oldTimeout)
+            del self._oldTimeout
+
+            # All results have been processed, deliver completion notification.
+
+            # It's important to run this *after* resetting the timeout to "rig
+            # a race" in some test code. writing to the transport will
+            # synchronously call test code, which synchronously loses the
+            # connection, calling our connectionLost method, which cancels the
+            # timeout. We want to make sure that timeout is cancelled *after*
+            # we reset it above, so that the final state is no timed
+            # calls. This avoids reactor uncleanliness errors in the test
+            # suite.
+            # XXX: Perhaps loopback should be fixed to not call the user code
+            # synchronously in transport.write?
+            self.sendPositiveResponse(tag, b"FETCH completed")
+
+            # Instance state is now consistent again (ie, it is as though
+            # the fetch command never ran), so allow any pending blocked
+            # commands to execute.
+            self._unblock()
+        else:
+            self.spewMessage(id, msg, query, uid).addCallback(
+                lambda _: self.__cbFetch(results, tag, query, uid)
+            ).addErrback(self.__ebSpewMessage)
+
+    def __ebSpewMessage(self, failure):
+        # This indicates a programming error.
+        # There's no reliable way to indicate anything to the client, since we
+        # may have already written an arbitrary amount of data in response to
+        # the command.
+        log.err(failure)
+        self.transport.loseConnection()
+
+    def spew_envelope(self, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.write
+        _w(b"ENVELOPE " + collapseNestedLists([getEnvelope(msg)]))
+
+    def spew_flags(self, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.writen
+        encodedFlags = [networkString(flag) for flag in msg.getFlags()]
+        _w(b"FLAGS " + b"(" + b" ".join(encodedFlags) + b")")
+
+    def spew_internaldate(self, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.write
+        idate = msg.getInternalDate()
+        ttup = email.utils.parsedate_tz(nativeString(idate))
+        if ttup is None:
+            log.msg("%d:%r: unpareseable internaldate: %r" % (id, msg, idate))
+            raise IMAP4Exception("Internal failure generating INTERNALDATE")
+
+        # need to specify the month manually, as strftime depends on locale
+        strdate = time.strftime("%d-%%s-%Y %H:%M:%S ", ttup[:9])
+        odate = networkString(strdate % (_MONTH_NAMES[ttup[1]],))
+        if ttup[9] is None:
+            odate = odate + b"+0000"
+        else:
+            if ttup[9] >= 0:
+                sign = b"+"
+            else:
+                sign = b"-"
+            odate = (
+                odate
+                + sign
+                + b"%04d"
+                % ((abs(ttup[9]) // 3600) * 100 + (abs(ttup[9]) % 3600) // 60,)
+            )
+        _w(b"INTERNALDATE " + _quote(odate))
+
+    def spew_rfc822header(self, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.write
+        hdrs = _formatHeaders(msg.getHeaders(True))
+        _w(b"RFC822.HEADER " + _literal(hdrs))
+
+    def spew_rfc822text(self, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.write
+        _w(b"RFC822.TEXT ")
+        _f()
+        return FileProducer(msg.getBodyFile()).beginProducing(self.transport)
+
+    def spew_rfc822size(self, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.write
+        _w(b"RFC822.SIZE %d" % (msg.getSize(),))
+
+    def spew_rfc822(self, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.write
+        _w(b"RFC822 ")
+        _f()
+        mf = IMessageFile(msg, None)
+        if mf is not None:
+            return FileProducer(mf.open()).beginProducing(self.transport)
+        return MessageProducer(msg, None, self._scheduler).beginProducing(
+            self.transport
+        )
+
+    def spew_uid(self, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.write
+        _w(b"UID %d" % (msg.getUID(),))
+
+    def spew_bodystructure(self, id, msg, _w=None, _f=None):
+        _w(b"BODYSTRUCTURE " + collapseNestedLists([getBodyStructure(msg, True)]))
+
+    def spew_body(self, part, id, msg, _w=None, _f=None):
+        if _w is None:
+            _w = self.transport.write
+        for p in part.part:
+            if msg.isMultipart():
+                msg = msg.getSubPart(p)
+            elif p > 0:
+                # Non-multipart messages have an implicit first part but no
+                # other parts - reject any request for any other part.
+                raise TypeError("Requested subpart of non-multipart message")
+
+        if part.header:
+            hdrs = msg.getHeaders(part.header.negate, *part.header.fields)
+            hdrs = _formatHeaders(hdrs)
+            _w(part.__bytes__() + b" " + _literal(hdrs))
+        elif part.text:
+            _w(part.__bytes__() + b" ")
+            _f()
+            return FileProducer(msg.getBodyFile()).beginProducing(self.transport)
+        elif part.mime:
+            hdrs = _formatHeaders(msg.getHeaders(True))
+            _w(part.__bytes__() + b" " + _literal(hdrs))
+        elif part.empty:
+            _w(part.__bytes__() + b" ")
+            _f()
+            if part.part:
+                return FileProducer(msg.getBodyFile()).beginProducing(self.transport)
+            else:
+                mf = IMessageFile(msg, None)
+                if mf is not None:
+                    return FileProducer(mf.open()).beginProducing(self.transport)
+                return MessageProducer(msg, None, self._scheduler).beginProducing(
+                    self.transport
+                )
+
+        else:
+            _w(b"BODY " + collapseNestedLists([getBodyStructure(msg)]))
+
+    def spewMessage(self, id, msg, query, uid):
+        wbuf = WriteBuffer(self.transport)
+        write = wbuf.write
+        flush = wbuf.flush
+
+        def start():
+            write(b"* %d FETCH (" % (id,))
+
+        def finish():
+            write(b")\r\n")
+
+        def space():
+            write(b" ")
+
+        def spew():
+            seenUID = False
+            start()
+            for part in query:
+                if part.type == "uid":
+                    seenUID = True
+                if part.type == "body":
+                    yield self.spew_body(part, id, msg, write, flush)
+                else:
+                    f = getattr(self, "spew_" + part.type)
+                    yield f(id, msg, write, flush)
+                if part is not query[-1]:
+                    space()
+            if uid and not seenUID:
+                space()
+                yield self.spew_uid(id, msg, write, flush)
+            finish()
+            flush()
+
+        return self._scheduler(spew())
+
+    def __ebFetch(self, failure, tag):
+        self.setTimeout(self._oldTimeout)
+        del self._oldTimeout
+        log.err(failure)
+        self.sendBadResponse(tag, b"FETCH failed: " + networkString(str(failure.value)))
+
+    def do_STORE(self, tag, messages, mode, flags, uid=0):
+        mode = mode.upper()
+        silent = mode.endswith(b"SILENT")
+        if mode.startswith(b"+"):
+            mode = 1
+        elif mode.startswith(b"-"):
+            mode = -1
+        else:
+            mode = 0
+
+        flags = [nativeString(flag) for flag in flags]
+        maybeDeferred(self.mbox.store, messages, flags, mode, uid=uid).addCallbacks(
+            self.__cbStore,
+            self.__ebStore,
+            (tag, self.mbox, uid, silent),
+            None,
+            (tag,),
+            None,
+        )
+
+    select_STORE = (do_STORE, arg_seqset, arg_atom, arg_flaglist)
+
+    def __cbStore(self, result, tag, mbox, uid, silent):
+        if result and not silent:
+            for k, v in result.items():
+                if uid:
+                    uidstr = b" UID %d" % (mbox.getUID(k),)
+                else:
+                    uidstr = b""
+
+                flags = [networkString(flag) for flag in v]
+                self.sendUntaggedResponse(
+                    b"%d FETCH (FLAGS (%b)%b)" % (k, b" ".join(flags), uidstr)
+                )
+        self.sendPositiveResponse(tag, b"STORE completed")
+
+    def __ebStore(self, failure, tag):
+        self.sendBadResponse(tag, b"Server error: " + networkString(str(failure.value)))
+
+    def do_COPY(self, tag, messages, mailbox, uid=0):
+        mailbox = _parseMbox(mailbox)
+        maybeDeferred(self.account.select, mailbox).addCallback(
+            self._cbCopySelectedMailbox, tag, messages, mailbox, uid
+        ).addErrback(self._ebCopySelectedMailbox, tag)
+
+    select_COPY = (do_COPY, arg_seqset, arg_finalastring)
+
+    def _cbCopySelectedMailbox(self, mbox, tag, messages, mailbox, uid):
+        if not mbox:
+            self.sendNegativeResponse(tag, "No such mailbox: " + mailbox)
+        else:
+            maybeDeferred(self.mbox.fetch, messages, uid).addCallback(
+                self.__cbCopy, tag, mbox
+            ).addCallback(self.__cbCopied, tag, mbox).addErrback(self.__ebCopy, tag)
+
+    def _ebCopySelectedMailbox(self, failure, tag):
+        self.sendBadResponse(tag, b"Server error: " + networkString(str(failure.value)))
+
+    def __cbCopy(self, messages, tag, mbox):
+        # XXX - This should handle failures with a rollback or something
+        addedDeferreds = []
+
+        fastCopyMbox = IMessageCopier(mbox, None)
+        for id, msg in messages:
+            if fastCopyMbox is not None:
+                d = maybeDeferred(fastCopyMbox.copy, msg)
+                addedDeferreds.append(d)
+                continue
+
+            # XXX - The following should be an implementation of IMessageCopier.copy
+            # on an IMailbox->IMessageCopier adapter.
+
+            flags = msg.getFlags()
+            date = msg.getInternalDate()
+
+            body = IMessageFile(msg, None)
+            if body is not None:
+                bodyFile = body.open()
+                d = maybeDeferred(mbox.addMessage, bodyFile, flags, date)
+            else:
+
+                def rewind(f):
+                    f.seek(0)
+                    return f
+
+                buffer = tempfile.TemporaryFile()
+                d = (
+                    MessageProducer(msg, buffer, self._scheduler)
+                    .beginProducing(None)
+                    .addCallback(
+                        lambda _, b=buffer, f=flags, d=date: mbox.addMessage(
+                            rewind(b), f, d
+                        )
+                    )
+                )
+            addedDeferreds.append(d)
+        return defer.DeferredList(addedDeferreds)
+
+    def __cbCopied(self, deferredIds, tag, mbox):
+        ids = []
+        failures = []
+        for status, result in deferredIds:
+            if status:
+                ids.append(result)
+            else:
+                failures.append(result.value)
+        if failures:
+            self.sendNegativeResponse(tag, "[ALERT] Some messages were not copied")
+        else:
+            self.sendPositiveResponse(tag, b"COPY completed")
+
+    def __ebCopy(self, failure, tag):
+        self.sendBadResponse(tag, b"COPY failed:" + networkString(str(failure.value)))
+        log.err(failure)
+
+    def do_UID(self, tag, command, line):
+        command = command.upper()
+
+        if command not in (b"COPY", b"FETCH", b"STORE", b"SEARCH"):
+            raise IllegalClientResponse(command)
+
+        self.dispatchCommand(tag, command, line, uid=1)
+
+    select_UID = (do_UID, arg_atom, arg_line)
+
+    #
+    # IMailboxListener implementation
+    #
+    def modeChanged(self, writeable):
+        if writeable:
+            self.sendUntaggedResponse(message=b"[READ-WRITE]", isAsync=True)
+        else:
+            self.sendUntaggedResponse(message=b"[READ-ONLY]", isAsync=True)
+
+    def flagsChanged(self, newFlags):
+        for mId, flags in newFlags.items():
+            encodedFlags = [networkString(flag) for flag in flags]
+            msg = b"%d FETCH (FLAGS (%b))" % (mId, b" ".join(encodedFlags))
+            self.sendUntaggedResponse(msg, isAsync=True)
+
+    def newMessages(self, exists, recent):
+        if exists is not None:
+            self.sendUntaggedResponse(b"%d EXISTS" % (exists,), isAsync=True)
+        if recent is not None:
+            self.sendUntaggedResponse(b"%d RECENT" % (recent,), isAsync=True)
+
+
+TIMEOUT_ERROR = error.TimeoutError()
+
+
+@implementer(IMailboxListener)
+class IMAP4Client(basic.LineReceiver, policies.TimeoutMixin):
+    """IMAP4 client protocol implementation
+
+    @ivar state: A string representing the state the connection is currently
+    in.
+    """
+
+    tags = None
+    waiting = None
+    queued = None
+    tagID = 1
+    state = None
+
+    startedTLS = False
+
+    # Number of seconds to wait before timing out a connection.
+    # If the number is <= 0 no timeout checking will be performed.
+    timeout = 0
+
+    # Capabilities are not allowed to change during the session
+    # So cache the first response and use that for all later
+    # lookups
+    _capCache = None
+
+    _memoryFileLimit = 1024 * 1024 * 10
+
+    # Authentication is pluggable.  This maps names to IClientAuthentication
+    # objects.
+    authenticators = None
+
+    STATUS_CODES = ("OK", "NO", "BAD", "PREAUTH", "BYE")
+
+    STATUS_TRANSFORMATIONS = {"MESSAGES": int, "RECENT": int, "UNSEEN": int}
+
+    context = None
+
+    def __init__(self, contextFactory=None):
+        self.tags = {}
+        self.queued = []
+        self.authenticators = {}
+        self.context = contextFactory
+
+        self._tag = None
+        self._parts = None
+        self._lastCmd = None
+
+    def registerAuthenticator(self, auth):
+        """
+        Register a new form of authentication
+
+        When invoking the authenticate() method of IMAP4Client, the first
+        matching authentication scheme found will be used.  The ordering is
+        that in which the server lists support authentication schemes.
+
+        @type auth: Implementor of C{IClientAuthentication}
+        @param auth: The object to use to perform the client
+        side of this authentication scheme.
+        """
+        self.authenticators[auth.getName().upper()] = auth
+
+    def rawDataReceived(self, data):
+        if self.timeout > 0:
+            self.resetTimeout()
+
+        self._pendingSize -= len(data)
+        if self._pendingSize > 0:
+            self._pendingBuffer.write(data)
+        else:
+            passon = b""
+            if self._pendingSize < 0:
+                data, passon = data[: self._pendingSize], data[self._pendingSize :]
+            self._pendingBuffer.write(data)
+            rest = self._pendingBuffer
+            self._pendingBuffer = None
+            self._pendingSize = None
+            rest.seek(0, 0)
+            self._parts.append(rest.read())
+            self.setLineMode(passon.lstrip(b"\r\n"))
+
+    #    def sendLine(self, line):
+    #        print 'S:', repr(line)
+    #        return basic.LineReceiver.sendLine(self, line)
+
+    def _setupForLiteral(self, rest, octets):
+        self._pendingBuffer = self.messageFile(octets)
+        self._pendingSize = octets
+        if self._parts is None:
+            self._parts = [rest, b"\r\n"]
+        else:
+            self._parts.extend([rest, b"\r\n"])
+        self.setRawMode()
+
+    def connectionMade(self):
+        if self.timeout > 0:
+            self.setTimeout(self.timeout)
+
+    def connectionLost(self, reason):
+        """
+        We are no longer connected
+        """
+        if self.timeout > 0:
+            self.setTimeout(None)
+        if self.queued is not None:
+            queued = self.queued
+            self.queued = None
+            for cmd in queued:
+                cmd.defer.errback(reason)
+        if self.tags is not None:
+            tags = self.tags
+            self.tags = None
+            for cmd in tags.values():
+                if cmd is not None and cmd.defer is not None:
+                    cmd.defer.errback(reason)
+
+    def lineReceived(self, line):
+        """
+        Attempt to parse a single line from the server.
+
+        @type line: L{bytes}
+        @param line: The line from the server, without the line delimiter.
+
+        @raise IllegalServerResponse: If the line or some part of the line
+            does not represent an allowed message from the server at this time.
+        """
+        #        print('C: ' + repr(line))
+        if self.timeout > 0:
+            self.resetTimeout()
+
+        lastPart = line.rfind(b"{")
+        if lastPart != -1:
+            lastPart = line[lastPart + 1 :]
+            if lastPart.endswith(b"}"):
+                # It's a literal a-comin' in
+                try:
+                    octets = int(lastPart[:-1])
+                except ValueError:
+                    raise IllegalServerResponse(line)
+                if self._parts is None:
+                    self._tag, parts = line.split(None, 1)
+                else:
+                    parts = line
+                self._setupForLiteral(parts, octets)
+                return
+
+        if self._parts is None:
+            # It isn't a literal at all
+            self._regularDispatch(line)
+        else:
+            # If an expression is in progress, no tag is required here
+            # Since we didn't find a literal indicator, this expression
+            # is done.
+            self._parts.append(line)
+            tag, rest = self._tag, b"".join(self._parts)
+            self._tag = self._parts = None
+            self.dispatchCommand(tag, rest)
+
+    def timeoutConnection(self):
+        if self._lastCmd and self._lastCmd.defer is not None:
+            d, self._lastCmd.defer = self._lastCmd.defer, None
+            d.errback(TIMEOUT_ERROR)
+
+        if self.queued:
+            for cmd in self.queued:
+                if cmd.defer is not None:
+                    d, cmd.defer = cmd.defer, d
+                    d.errback(TIMEOUT_ERROR)
+
+        self.transport.loseConnection()
+
+    def _regularDispatch(self, line):
+        parts = line.split(None, 1)
+        if len(parts) != 2:
+            parts.append(b"")
+        tag, rest = parts
+        self.dispatchCommand(tag, rest)
+
+    def messageFile(self, octets):
+        """
+        Create a file to which an incoming message may be written.
+
+        @type octets: L{int}
+        @param octets: The number of octets which will be written to the file
+
+        @rtype: Any object which implements C{write(string)} and
+        C{seek(int, int)}
+        @return: A file-like object
+        """
+        if octets > self._memoryFileLimit:
+            return tempfile.TemporaryFile()
+        else:
+            return BytesIO()
+
+    def makeTag(self):
+        tag = ("%0.4X" % self.tagID).encode("ascii")
+        self.tagID += 1
+        return tag
+
+    def dispatchCommand(self, tag, rest):
+        if self.state is None:
+            f = self.response_UNAUTH
+        else:
+            f = getattr(self, "response_" + self.state.upper(), None)
+        if f:
+            try:
+                f(tag, rest)
+            except BaseException:
+                log.err()
+                self.transport.loseConnection()
+        else:
+            log.err(f"Cannot dispatch: {self.state}, {tag!r}, {rest!r}")
+            self.transport.loseConnection()
+
+    def response_UNAUTH(self, tag, rest):
+        if self.state is None:
+            # Server greeting, this is
+            status, rest = rest.split(None, 1)
+            if status.upper() == b"OK":
+                self.state = "unauth"
+            elif status.upper() == b"PREAUTH":
+                self.state = "auth"
+            else:
+                # XXX - This is rude.
+                self.transport.loseConnection()
+                raise IllegalServerResponse(tag + b" " + rest)
+
+            b, e = rest.find(b"["), rest.find(b"]")
+            if b != -1 and e != -1:
+                self.serverGreeting(
+                    self.__cbCapabilities(([parseNestedParens(rest[b + 1 : e])], None))
+                )
+            else:
+                self.serverGreeting(None)
+        else:
+            self._defaultHandler(tag, rest)
+
+    def response_AUTH(self, tag, rest):
+        self._defaultHandler(tag, rest)
+
+    def _defaultHandler(self, tag, rest):
+        if tag == b"*" or tag == b"+":
+            if not self.waiting:
+                self._extraInfo([parseNestedParens(rest)])
+            else:
+                cmd = self.tags[self.waiting]
+                if tag == b"+":
+                    cmd.continuation(rest)
+                else:
+                    cmd.lines.append(rest)
+        else:
+            try:
+                cmd = self.tags[tag]
+            except KeyError:
+                # XXX - This is rude.
+                self.transport.loseConnection()
+                raise IllegalServerResponse(tag + b" " + rest)
+            else:
+                status, line = rest.split(None, 1)
+                if status == b"OK":
+                    # Give them this last line, too
+                    cmd.finish(rest, self._extraInfo)
+                else:
+                    cmd.defer.errback(IMAP4Exception(line))
+                del self.tags[tag]
+                self.waiting = None
+                self._flushQueue()
+
+    def _flushQueue(self):
+        if self.queued:
+            cmd = self.queued.pop(0)
+            t = self.makeTag()
+            self.tags[t] = cmd
+            self.sendLine(cmd.format(t))
+            self.waiting = t
+
+    def _extraInfo(self, lines):
+        # XXX - This is terrible.
+        # XXX - Also, this should collapse temporally proximate calls into single
+        #       invocations of IMailboxListener methods, where possible.
+        flags = {}
+        recent = exists = None
+        for response in lines:
+            elements = len(response)
+            if elements == 1 and response[0] == [b"READ-ONLY"]:
+                self.modeChanged(False)
+            elif elements == 1 and response[0] == [b"READ-WRITE"]:
+                self.modeChanged(True)
+            elif elements == 2 and response[1] == b"EXISTS":
+                exists = int(response[0])
+            elif elements == 2 and response[1] == b"RECENT":
+                recent = int(response[0])
+            elif elements == 3 and response[1] == b"FETCH":
+                mId = int(response[0])
+                values, _ = self._parseFetchPairs(response[2])
+                flags.setdefault(mId, []).extend(values.get("FLAGS", ()))
+            else:
+                log.msg(f"Unhandled unsolicited response: {response}")
+
+        if flags:
+            self.flagsChanged(flags)
+        if recent is not None or exists is not None:
+            self.newMessages(exists, recent)
+
+    def sendCommand(self, cmd):
+        cmd.defer = defer.Deferred()
+        if self.waiting:
+            self.queued.append(cmd)
+            return cmd.defer
+        t = self.makeTag()
+        self.tags[t] = cmd
+        self.sendLine(cmd.format(t))
+        self.waiting = t
+        self._lastCmd = cmd
+        return cmd.defer
+
+    def getCapabilities(self, useCache=1):
+        """
+        Request the capabilities available on this server.
+
+        This command is allowed in any state of connection.
+
+        @type useCache: C{bool}
+        @param useCache: Specify whether to use the capability-cache or to
+        re-retrieve the capabilities from the server.  Server capabilities
+        should never change, so for normal use, this flag should never be
+        false.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback will be invoked with a
+        dictionary mapping capability types to lists of supported
+        mechanisms, or to None if a support list is not applicable.
+        """
+        if useCache and self._capCache is not None:
+            return defer.succeed(self._capCache)
+        cmd = b"CAPABILITY"
+        resp = (b"CAPABILITY",)
+        d = self.sendCommand(Command(cmd, wantResponse=resp))
+        d.addCallback(self.__cbCapabilities)
+        return d
+
+    def __cbCapabilities(self, result):
+        (lines, tagline) = result
+        caps = {}
+        for rest in lines:
+            for cap in rest[1:]:
+                parts = cap.split(b"=", 1)
+                if len(parts) == 1:
+                    category, value = parts[0], None
+                else:
+                    category, value = parts
+                caps.setdefault(category, []).append(value)
+
+        # Preserve a non-ideal API for backwards compatibility.  It would
+        # probably be entirely sensible to have an object with a wider API than
+        # dict here so this could be presented less insanely.
+        for category in caps:
+            if caps[category] == [None]:
+                caps[category] = None
+        self._capCache = caps
+        return caps
+
+    def logout(self):
+        """
+        Inform the server that we are done with the connection.
+
+        This command is allowed in any state of connection.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback will be invoked with None
+        when the proper server acknowledgement has been received.
+        """
+        d = self.sendCommand(Command(b"LOGOUT", wantResponse=(b"BYE",)))
+        d.addCallback(self.__cbLogout)
+        return d
+
+    def __cbLogout(self, result):
+        (lines, tagline) = result
+        self.transport.loseConnection()
+        # We don't particularly care what the server said
+        return None
+
+    def noop(self):
+        """
+        Perform no operation.
+
+        This command is allowed in any state of connection.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback will be invoked with a list
+        of untagged status updates the server responds with.
+        """
+        d = self.sendCommand(Command(b"NOOP"))
+        d.addCallback(self.__cbNoop)
+        return d
+
+    def __cbNoop(self, result):
+        # Conceivable, this is elidable.
+        # It is, afterall, a no-op.
+        (lines, tagline) = result
+        return lines
+
+    def startTLS(self, contextFactory=None):
+        """
+        Initiates a 'STARTTLS' request and negotiates the TLS / SSL
+        Handshake.
+
+        @param contextFactory: The TLS / SSL Context Factory to
+        leverage.  If the contextFactory is None the IMAP4Client will
+        either use the current TLS / SSL Context Factory or attempt to
+        create a new one.
+
+        @type contextFactory: C{ssl.ClientContextFactory}
+
+        @return: A Deferred which fires when the transport has been
+        secured according to the given contextFactory, or which fails
+        if the transport cannot be secured.
+        """
+        assert (
+            not self.startedTLS
+        ), "Client and Server are currently communicating via TLS"
+        if contextFactory is None:
+            contextFactory = self._getContextFactory()
+
+        if contextFactory is None:
+            return defer.fail(
+                IMAP4Exception(
+                    "IMAP4Client requires a TLS context to "
+                    "initiate the STARTTLS handshake"
+                )
+            )
+
+        if b"STARTTLS" not in self._capCache:
+            return defer.fail(
+                IMAP4Exception(
+                    "Server does not support secure communication " "via TLS / SSL"
+                )
+            )
+
+        tls = interfaces.ITLSTransport(self.transport, None)
+        if tls is None:
+            return defer.fail(
+                IMAP4Exception(
+                    "IMAP4Client transport does not implement "
+                    "interfaces.ITLSTransport"
+                )
+            )
+
+        d = self.sendCommand(Command(b"STARTTLS"))
+        d.addCallback(self._startedTLS, contextFactory)
+        d.addCallback(lambda _: self.getCapabilities())
+        return d
+
+    def authenticate(self, secret):
+        """
+        Attempt to enter the authenticated state with the server
+
+        This command is allowed in the Non-Authenticated state.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked if the authentication
+        succeeds and whose errback will be invoked otherwise.
+        """
+        if self._capCache is None:
+            d = self.getCapabilities()
+        else:
+            d = defer.succeed(self._capCache)
+        d.addCallback(self.__cbAuthenticate, secret)
+        return d
+
+    def __cbAuthenticate(self, caps, secret):
+        auths = caps.get(b"AUTH", ())
+        for scheme in auths:
+            if scheme.upper() in self.authenticators:
+                cmd = Command(
+                    b"AUTHENTICATE", scheme, (), self.__cbContinueAuth, scheme, secret
+                )
+                return self.sendCommand(cmd)
+
+        if self.startedTLS:
+            return defer.fail(
+                NoSupportedAuthentication(auths, self.authenticators.keys())
+            )
+        else:
+
+            def ebStartTLS(err):
+                err.trap(IMAP4Exception)
+                # We couldn't negotiate TLS for some reason
+                return defer.fail(
+                    NoSupportedAuthentication(auths, self.authenticators.keys())
+                )
+
+            d = self.startTLS()
+            d.addErrback(ebStartTLS)
+            d.addCallback(lambda _: self.getCapabilities())
+            d.addCallback(self.__cbAuthTLS, secret)
+            return d
+
+    def __cbContinueAuth(self, rest, scheme, secret):
+        try:
+            chal = decodebytes(rest + b"\n")
+        except binascii.Error:
+            self.sendLine(b"*")
+            raise IllegalServerResponse(rest)
+        else:
+            auth = self.authenticators[scheme]
+            chal = auth.challengeResponse(secret, chal)
+            self.sendLine(encodebytes(chal).strip())
+
+    def __cbAuthTLS(self, caps, secret):
+        auths = caps.get(b"AUTH", ())
+        for scheme in auths:
+            if scheme.upper() in self.authenticators:
+                cmd = Command(
+                    b"AUTHENTICATE", scheme, (), self.__cbContinueAuth, scheme, secret
+                )
+                return self.sendCommand(cmd)
+        raise NoSupportedAuthentication(auths, self.authenticators.keys())
+
+    def login(self, username, password):
+        """
+        Authenticate with the server using a username and password
+
+        This command is allowed in the Non-Authenticated state.  If the
+        server supports the STARTTLS capability and our transport supports
+        TLS, TLS is negotiated before the login command is issued.
+
+        A more secure way to log in is to use C{startTLS} or
+        C{authenticate} or both.
+
+        @type username: L{str}
+        @param username: The username to log in with
+
+        @type password: L{str}
+        @param password: The password to log in with
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked if login is successful
+        and whose errback is invoked otherwise.
+        """
+        d = maybeDeferred(self.getCapabilities)
+        d.addCallback(self.__cbLoginCaps, username, password)
+        return d
+
+    def serverGreeting(self, caps):
+        """
+        Called when the server has sent us a greeting.
+
+        @type caps: C{dict}
+        @param caps: Capabilities the server advertised in its greeting.
+        """
+
+    def _getContextFactory(self):
+        if self.context is not None:
+            return self.context
+        try:
+            from twisted.internet import ssl
+        except ImportError:
+            return None
+        else:
+            return ssl.ClientContextFactory()
+
+    def __cbLoginCaps(self, capabilities, username, password):
+        # If the server advertises STARTTLS, we might want to try to switch to TLS
+        tryTLS = b"STARTTLS" in capabilities
+
+        # If our transport supports switching to TLS, we might want to try to switch to TLS.
+        tlsableTransport = interfaces.ITLSTransport(self.transport, None) is not None
+
+        # If our transport is not already using TLS, we might want to try to switch to TLS.
+        nontlsTransport = interfaces.ISSLTransport(self.transport, None) is None
+
+        if not self.startedTLS and tryTLS and tlsableTransport and nontlsTransport:
+            d = self.startTLS()
+
+            d.addCallbacks(
+                self.__cbLoginTLS,
+                self.__ebLoginTLS,
+                callbackArgs=(username, password),
+            )
+            return d
+        else:
+            if nontlsTransport:
+                log.msg("Server has no TLS support. logging in over cleartext!")
+            args = b" ".join((_quote(username), _quote(password)))
+            return self.sendCommand(Command(b"LOGIN", args))
+
+    def _startedTLS(self, result, context):
+        self.transport.startTLS(context)
+        self._capCache = None
+        self.startedTLS = True
+        return result
+
+    def __cbLoginTLS(self, result, username, password):
+        args = b" ".join((_quote(username), _quote(password)))
+        return self.sendCommand(Command(b"LOGIN", args))
+
+    def __ebLoginTLS(self, failure):
+        log.err(failure)
+        return failure
+
+    def namespace(self):
+        """
+        Retrieve information about the namespaces available to this account
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with namespace
+        information.  An example of this information is::
+
+            [[['', '/']], [], []]
+
+        which indicates a single personal namespace called '' with '/'
+        as its hierarchical delimiter, and no shared or user namespaces.
+        """
+        cmd = b"NAMESPACE"
+        resp = (b"NAMESPACE",)
+        d = self.sendCommand(Command(cmd, wantResponse=resp))
+        d.addCallback(self.__cbNamespace)
+        return d
+
+    def __cbNamespace(self, result):
+        (lines, last) = result
+
+        # Namespaces and their delimiters qualify and delimit
+        # mailboxes, so they should be native strings
+        #
+        # On Python 2, no decoding is necessary to maintain
+        # the API contract.
+        #
+        # On Python 3, users specify mailboxes with native strings, so
+        # they should receive namespaces and delimiters as native
+        # strings.  Both cases are possible because of the imap4-utf-7
+        # encoding.
+        def _prepareNamespaceOrDelimiter(namespaceList):
+            return [element.decode("imap4-utf-7") for element in namespaceList]
+
+        for parts in lines:
+            if len(parts) == 4 and parts[0] == b"NAMESPACE":
+                return [
+                    []
+                    if pairOrNone is None
+                    else [_prepareNamespaceOrDelimiter(value) for value in pairOrNone]
+                    for pairOrNone in parts[1:]
+                ]
+        log.err("No NAMESPACE response to NAMESPACE command")
+        return [[], [], []]
+
+    def select(self, mailbox):
+        """
+        Select a mailbox
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type mailbox: L{str}
+        @param mailbox: The name of the mailbox to select
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with mailbox
+        information if the select is successful and whose errback is
+        invoked otherwise.  Mailbox information consists of a dictionary
+        with the following L{str} keys and values::
+
+                FLAGS: A list of strings containing the flags settable on
+                        messages in this mailbox.
+
+                EXISTS: An integer indicating the number of messages in this
+                        mailbox.
+
+                RECENT: An integer indicating the number of "recent"
+                        messages in this mailbox.
+
+                UNSEEN: The message sequence number (an integer) of the
+                        first unseen message in the mailbox.
+
+                PERMANENTFLAGS: A list of strings containing the flags that
+                        can be permanently set on messages in this mailbox.
+
+                UIDVALIDITY: An integer uniquely identifying this mailbox.
+        """
+        cmd = b"SELECT"
+        args = _prepareMailboxName(mailbox)
+        # This appears not to be used, so we can use native strings to
+        # indicate that the return type is native strings.
+        resp = ("FLAGS", "EXISTS", "RECENT", "UNSEEN", "PERMANENTFLAGS", "UIDVALIDITY")
+        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
+        d.addCallback(self.__cbSelect, 1)
+        return d
+
+    def examine(self, mailbox):
+        """
+        Select a mailbox in read-only mode
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type mailbox: L{str}
+        @param mailbox: The name of the mailbox to examine
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with mailbox
+        information if the examine is successful and whose errback
+        is invoked otherwise.  Mailbox information consists of a dictionary
+        with the following keys and values::
+
+            'FLAGS': A list of strings containing the flags settable on
+                        messages in this mailbox.
+
+            'EXISTS': An integer indicating the number of messages in this
+                        mailbox.
+
+            'RECENT': An integer indicating the number of \"recent\"
+                        messages in this mailbox.
+
+            'UNSEEN': An integer indicating the number of messages not
+                        flagged \\Seen in this mailbox.
+
+            'PERMANENTFLAGS': A list of strings containing the flags that
+                        can be permanently set on messages in this mailbox.
+
+            'UIDVALIDITY': An integer uniquely identifying this mailbox.
+        """
+        cmd = b"EXAMINE"
+        args = _prepareMailboxName(mailbox)
+        resp = (
+            b"FLAGS",
+            b"EXISTS",
+            b"RECENT",
+            b"UNSEEN",
+            b"PERMANENTFLAGS",
+            b"UIDVALIDITY",
+        )
+        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
+        d.addCallback(self.__cbSelect, 0)
+        return d
+
+    def _intOrRaise(self, value, phrase):
+        """
+        Parse C{value} as an integer and return the result or raise
+        L{IllegalServerResponse} with C{phrase} as an argument if C{value}
+        cannot be parsed as an integer.
+        """
+        try:
+            return int(value)
+        except ValueError:
+            raise IllegalServerResponse(phrase)
+
+    def __cbSelect(self, result, rw):
+        """
+        Handle lines received in response to a SELECT or EXAMINE command.
+
+        See RFC 3501, section 6.3.1.
+        """
+        (lines, tagline) = result
+        # In the absence of specification, we are free to assume:
+        #   READ-WRITE access
+        datum = {"READ-WRITE": rw}
+        lines.append(parseNestedParens(tagline))
+        for split in lines:
+            if len(split) > 0 and split[0].upper() == b"OK":
+                # Handle all the kinds of OK response.
+                content = split[1]
+                if isinstance(content, list):
+                    key = content[0]
+                else:
+                    # not multi-valued, like OK LOGIN
+                    key = content
+                key = key.upper()
+                if key == b"READ-ONLY":
+                    datum["READ-WRITE"] = False
+                elif key == b"READ-WRITE":
+                    datum["READ-WRITE"] = True
+                elif key == b"UIDVALIDITY":
+                    datum["UIDVALIDITY"] = self._intOrRaise(content[1], split)
+                elif key == b"UNSEEN":
+                    datum["UNSEEN"] = self._intOrRaise(content[1], split)
+                elif key == b"UIDNEXT":
+                    datum["UIDNEXT"] = self._intOrRaise(content[1], split)
+                elif key == b"PERMANENTFLAGS":
+                    datum["PERMANENTFLAGS"] = tuple(
+                        nativeString(flag) for flag in content[1]
+                    )
+                else:
+                    log.err(f"Unhandled SELECT response (2): {split}")
+            elif len(split) == 2:
+                # Handle FLAGS, EXISTS, and RECENT
+                if split[0].upper() == b"FLAGS":
+                    datum["FLAGS"] = tuple(nativeString(flag) for flag in split[1])
+                elif isinstance(split[1], bytes):
+                    # Must make sure things are strings before treating them as
+                    # strings since some other forms of response have nesting in
+                    # places which results in lists instead.
+                    if split[1].upper() == b"EXISTS":
+                        datum["EXISTS"] = self._intOrRaise(split[0], split)
+                    elif split[1].upper() == b"RECENT":
+                        datum["RECENT"] = self._intOrRaise(split[0], split)
+                    else:
+                        log.err(f"Unhandled SELECT response (0): {split}")
+                else:
+                    log.err(f"Unhandled SELECT response (1): {split}")
+            else:
+                log.err(f"Unhandled SELECT response (4): {split}")
+        return datum
+
+    def create(self, name):
+        """
+        Create a new mailbox on the server
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type name: L{str}
+        @param name: The name of the mailbox to create.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked if the mailbox creation
+        is successful and whose errback is invoked otherwise.
+        """
+        return self.sendCommand(Command(b"CREATE", _prepareMailboxName(name)))
+
+    def delete(self, name):
+        """
+        Delete a mailbox
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type name: L{str}
+        @param name: The name of the mailbox to delete.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose calblack is invoked if the mailbox is
+        deleted successfully and whose errback is invoked otherwise.
+        """
+        return self.sendCommand(Command(b"DELETE", _prepareMailboxName(name)))
+
+    def rename(self, oldname, newname):
+        """
+        Rename a mailbox
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type oldname: L{str}
+        @param oldname: The current name of the mailbox to rename.
+
+        @type newname: L{str}
+        @param newname: The new name to give the mailbox.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked if the rename is
+        successful and whose errback is invoked otherwise.
+        """
+        oldname = _prepareMailboxName(oldname)
+        newname = _prepareMailboxName(newname)
+        return self.sendCommand(Command(b"RENAME", b" ".join((oldname, newname))))
+
+    def subscribe(self, name):
+        """
+        Add a mailbox to the subscription list
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type name: L{str}
+        @param name: The mailbox to mark as 'active' or 'subscribed'
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked if the subscription
+        is successful and whose errback is invoked otherwise.
+        """
+        return self.sendCommand(Command(b"SUBSCRIBE", _prepareMailboxName(name)))
+
+    def unsubscribe(self, name):
+        """
+        Remove a mailbox from the subscription list
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type name: L{str}
+        @param name: The mailbox to unsubscribe
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked if the unsubscription
+        is successful and whose errback is invoked otherwise.
+        """
+        return self.sendCommand(Command(b"UNSUBSCRIBE", _prepareMailboxName(name)))
+
+    def list(self, reference, wildcard):
+        """
+        List a subset of the available mailboxes
+
+        This command is allowed in the Authenticated and Selected
+        states.
+
+        @type reference: L{str}
+        @param reference: The context in which to interpret
+            C{wildcard}
+
+        @type wildcard: L{str}
+        @param wildcard: The pattern of mailbox names to match,
+            optionally including either or both of the '*' and '%'
+            wildcards.  '*' will match zero or more characters and
+            cross hierarchical boundaries.  '%' will also match zero
+            or more characters, but is limited to a single
+            hierarchical level.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a list of
+            L{tuple}s, the first element of which is a L{tuple} of
+            mailbox flags, the second element of which is the
+            hierarchy delimiter for this mailbox, and the third of
+            which is the mailbox name; if the command is unsuccessful,
+            the deferred's errback is invoked instead.  B{NB}: the
+            delimiter and the mailbox name are L{str}s.
+        """
+        cmd = b"LIST"
+        args = (f'"{reference}" "{wildcard}"').encode("imap4-utf-7")
+        resp = (b"LIST",)
+        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
+        d.addCallback(self.__cbList, b"LIST")
+        return d
+
+    def lsub(self, reference, wildcard):
+        """
+        List a subset of the subscribed available mailboxes
+
+        This command is allowed in the Authenticated and Selected states.
+
+        The parameters and returned object are the same as for the L{list}
+        method, with one slight difference: Only mailboxes which have been
+        subscribed can be included in the resulting list.
+        """
+        cmd = b"LSUB"
+
+        encodedReference = reference.encode("ascii")
+        encodedWildcard = wildcard.encode("imap4-utf-7")
+        args = b"".join(
+            [
+                b'"',
+                encodedReference,
+                b'"' b' "',
+                encodedWildcard,
+                b'"',
+            ]
+        )
+        resp = (b"LSUB",)
+        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
+        d.addCallback(self.__cbList, b"LSUB")
+        return d
+
+    def __cbList(self, result, command):
+        (lines, last) = result
+        results = []
+
+        for parts in lines:
+            if len(parts) == 4 and parts[0] == command:
+                # flags
+                parts[1] = tuple(nativeString(flag) for flag in parts[1])
+
+                # The mailbox should be a native string.
+                # On Python 2, this maintains the API's contract.
+                #
+                # On Python 3, users specify mailboxes with native
+                # strings, so they should receive mailboxes as native
+                # strings.  Both cases are possible because of the
+                # imap4-utf-7 encoding.
+                #
+                # Mailbox names contain the hierarchical delimiter, so
+                # it too should be a native string.
+                # delimiter
+                parts[2] = parts[2].decode("imap4-utf-7")
+                # mailbox
+                parts[3] = parts[3].decode("imap4-utf-7")
+
+                results.append(tuple(parts[1:]))
+        return results
+
+    _statusNames = {
+        name: name.encode("ascii")
+        for name in (
+            "MESSAGES",
+            "RECENT",
+            "UIDNEXT",
+            "UIDVALIDITY",
+            "UNSEEN",
+        )
+    }
+
+    def status(self, mailbox, *names):
+        """
+        Retrieve the status of the given mailbox
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type mailbox: L{str}
+        @param mailbox: The name of the mailbox to query
+
+        @type names: L{bytes}
+        @param names: The status names to query.  These may be any number of:
+            C{'MESSAGES'}, C{'RECENT'}, C{'UIDNEXT'}, C{'UIDVALIDITY'}, and
+            C{'UNSEEN'}.
+
+        @rtype: L{Deferred}
+        @return: A deferred which fires with the status information if the
+            command is successful and whose errback is invoked otherwise.  The
+            status information is in the form of a C{dict}.  Each element of
+            C{names} is a key in the dictionary.  The value for each key is the
+            corresponding response from the server.
+        """
+        cmd = b"STATUS"
+
+        preparedMailbox = _prepareMailboxName(mailbox)
+        try:
+            names = b" ".join(self._statusNames[name] for name in names)
+        except KeyError:
+            raise ValueError(f"Unknown names: {set(names) - set(self._statusNames)!r}")
+
+        args = b"".join([preparedMailbox, b" (", names, b")"])
+        resp = (b"STATUS",)
+        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
+        d.addCallback(self.__cbStatus)
+        return d
+
+    def __cbStatus(self, result):
+        (lines, last) = result
+        status = {}
+        for parts in lines:
+            if parts[0] == b"STATUS":
+                items = parts[2]
+                items = [items[i : i + 2] for i in range(0, len(items), 2)]
+                for k, v in items:
+                    try:
+                        status[nativeString(k)] = v
+                    except UnicodeDecodeError:
+                        raise IllegalServerResponse(repr(items))
+        for k in status.keys():
+            t = self.STATUS_TRANSFORMATIONS.get(k)
+            if t:
+                try:
+                    status[k] = t(status[k])
+                except Exception as e:
+                    raise IllegalServerResponse(
+                        "(" + k + " " + status[k] + "): " + str(e)
+                    )
+        return status
+
+    def append(self, mailbox, message, flags=(), date=None):
+        """
+        Add the given message to the given mailbox.
+
+        This command is allowed in the Authenticated and Selected states.
+
+        @type mailbox: L{str}
+        @param mailbox: The mailbox to which to add this message.
+
+        @type message: Any file-like object opened in B{binary mode}.
+        @param message: The message to add, in RFC822 format.  Newlines
+        in this file should be \\r\\n-style.
+
+        @type flags: Any iterable of L{str}
+        @param flags: The flags to associated with this message.
+
+        @type date: L{str}
+        @param date: The date to associate with this message.  This should
+        be of the format DD-MM-YYYY HH:MM:SS +/-HHMM.  For example, in
+        Eastern Standard Time, on July 1st 2004 at half past 1 PM,
+        \"01-07-2004 13:30:00 -0500\".
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked when this command
+        succeeds or whose errback is invoked if it fails.
+        """
+        message.seek(0, 2)
+        L = message.tell()
+        message.seek(0, 0)
+        if date:
+            date = networkString(' "%s"' % nativeString(date))
+        else:
+            date = b""
+
+        encodedFlags = [networkString(flag) for flag in flags]
+
+        cmd = b"%b (%b)%b {%d}" % (
+            _prepareMailboxName(mailbox),
+            b" ".join(encodedFlags),
+            date,
+            L,
+        )
+
+        d = self.sendCommand(
+            Command(b"APPEND", cmd, (), self.__cbContinueAppend, message)
+        )
+        return d
+
+    def __cbContinueAppend(self, lines, message):
+        s = basic.FileSender()
+        return s.beginFileTransfer(message, self.transport, None).addCallback(
+            self.__cbFinishAppend
+        )
+
+    def __cbFinishAppend(self, foo):
+        self.sendLine(b"")
+
+    def check(self):
+        """
+        Tell the server to perform a checkpoint
+
+        This command is allowed in the Selected state.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked when this command
+        succeeds or whose errback is invoked if it fails.
+        """
+        return self.sendCommand(Command(b"CHECK"))
+
+    def close(self):
+        """
+        Return the connection to the Authenticated state.
+
+        This command is allowed in the Selected state.
+
+        Issuing this command will also remove all messages flagged \\Deleted
+        from the selected mailbox if it is opened in read-write mode,
+        otherwise it indicates success by no messages are removed.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked when the command
+        completes successfully or whose errback is invoked if it fails.
+        """
+        return self.sendCommand(Command(b"CLOSE"))
+
+    def expunge(self):
+        """
+        Return the connection to the Authenticate state.
+
+        This command is allowed in the Selected state.
+
+        Issuing this command will perform the same actions as issuing the
+        close command, but will also generate an 'expunge' response for
+        every message deleted.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a list of the
+        'expunge' responses when this command is successful or whose errback
+        is invoked otherwise.
+        """
+        cmd = b"EXPUNGE"
+        resp = (b"EXPUNGE",)
+        d = self.sendCommand(Command(cmd, wantResponse=resp))
+        d.addCallback(self.__cbExpunge)
+        return d
+
+    def __cbExpunge(self, result):
+        (lines, last) = result
+        ids = []
+        for parts in lines:
+            if len(parts) == 2 and parts[1] == b"EXPUNGE":
+                ids.append(self._intOrRaise(parts[0], parts))
+        return ids
+
+    def search(self, *queries, uid=False):
+        """
+        Search messages in the currently selected mailbox
+
+        This command is allowed in the Selected state.
+
+        Any non-zero number of queries are accepted by this method, as returned
+        by the C{Query}, C{Or}, and C{Not} functions.
+
+        @param uid: if true, the server is asked to return message UIDs instead
+            of message sequence numbers.
+        @type uid: L{bool}
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback will be invoked with a list of all
+            the message sequence numbers return by the search, or whose errback
+            will be invoked if there is an error.
+        """
+        # Queries should be encoded as ASCII unless a charset
+        # identifier is provided.  See #9201.
+        queries = [query.encode("charmap") for query in queries]
+
+        cmd = b"UID SEARCH" if uid else b"SEARCH"
+        args = b" ".join(queries)
+        d = self.sendCommand(Command(cmd, args, wantResponse=(cmd,)))
+        d.addCallback(self.__cbSearch)
+        return d
+
+    def __cbSearch(self, result):
+        (lines, end) = result
+        ids = []
+        for parts in lines:
+            if len(parts) > 0 and parts[0] == b"SEARCH":
+                ids.extend([self._intOrRaise(p, parts) for p in parts[1:]])
+        return ids
+
+    def fetchUID(self, messages, uid=0):
+        """
+        Retrieve the unique identifier for one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message sequence numbers to unique message identifiers, or whose
+        errback is invoked if there is an error.
+        """
+        return self._fetch(messages, useUID=uid, uid=1)
+
+    def fetchFlags(self, messages, uid=0):
+        """
+        Retrieve the flags for one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: The messages for which to retrieve flags.
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to lists of flags, or whose errback is invoked if
+        there is an error.
+        """
+        return self._fetch(messages, useUID=uid, flags=1)
+
+    def fetchInternalDate(self, messages, uid=0):
+        """
+        Retrieve the internal date associated with one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: The messages for which to retrieve the internal date.
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to date strings, or whose errback is invoked
+        if there is an error.  Date strings take the format of
+        \"day-month-year time timezone\".
+        """
+        return self._fetch(messages, useUID=uid, internaldate=1)
+
+    def fetchEnvelope(self, messages, uid=0):
+        """
+        Retrieve the envelope data for one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: The messages for which to retrieve envelope
+            data.
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of
+            message numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict
+            mapping message numbers to envelope data, or whose errback
+            is invoked if there is an error.  Envelope data consists
+            of a sequence of the date, subject, from, sender,
+            reply-to, to, cc, bcc, in-reply-to, and message-id header
+            fields.  The date, subject, in-reply-to, and message-id
+            fields are L{str}, while the from, sender, reply-to, to,
+            cc, and bcc fields contain address data as L{str}s.
+            Address data consists of a sequence of name, source route,
+            mailbox name, and hostname.  Fields which are not present
+            for a particular address may be L{None}.
+        """
+        return self._fetch(messages, useUID=uid, envelope=1)
+
+    def fetchBodyStructure(self, messages, uid=0):
+        """
+        Retrieve the structure of the body of one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: The messages for which to retrieve body structure
+        data.
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to body structure data, or whose errback is invoked
+        if there is an error.  Body structure data describes the MIME-IMB
+        format of a message and consists of a sequence of mime type, mime
+        subtype, parameters, content id, description, encoding, and size.
+        The fields following the size field are variable: if the mime
+        type/subtype is message/rfc822, the contained message's envelope
+        information, body structure data, and number of lines of text; if
+        the mime type is text, the number of lines of text.  Extension fields
+        may also be included; if present, they are: the MD5 hash of the body,
+        body disposition, body language.
+        """
+        return self._fetch(messages, useUID=uid, bodystructure=1)
+
+    def fetchSimplifiedBody(self, messages, uid=0):
+        """
+        Retrieve the simplified body structure of one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: C{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to body data, or whose errback is invoked
+        if there is an error.  The simplified body structure is the same
+        as the body structure, except that extension fields will never be
+        present.
+        """
+        return self._fetch(messages, useUID=uid, body=1)
+
+    def fetchMessage(self, messages, uid=0):
+        """
+        Retrieve one or more entire messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: C{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+
+        @return: A L{Deferred} which will fire with a C{dict} mapping message
+            sequence numbers to C{dict}s giving message data for the
+            corresponding message.  If C{uid} is true, the inner dictionaries
+            have a C{'UID'} key mapped to a L{str} giving the UID for the
+            message.  The text of the message is a L{str} associated with the
+            C{'RFC822'} key in each dictionary.
+        """
+        return self._fetch(messages, useUID=uid, rfc822=1)
+
+    def fetchHeaders(self, messages, uid=0):
+        """
+        Retrieve headers of one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to dicts of message headers, or whose errback is
+        invoked if there is an error.
+        """
+        return self._fetch(messages, useUID=uid, rfc822header=1)
+
+    def fetchBody(self, messages, uid=0):
+        """
+        Retrieve body text of one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to file-like objects containing body text, or whose
+        errback is invoked if there is an error.
+        """
+        return self._fetch(messages, useUID=uid, rfc822text=1)
+
+    def fetchSize(self, messages, uid=0):
+        """
+        Retrieve the size, in octets, of one or more messages
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to sizes, or whose errback is invoked if there is
+        an error.
+        """
+        return self._fetch(messages, useUID=uid, rfc822size=1)
+
+    def fetchFull(self, messages, uid=0):
+        """
+        Retrieve several different fields of one or more messages
+
+        This command is allowed in the Selected state.  This is equivalent
+        to issuing all of the C{fetchFlags}, C{fetchInternalDate},
+        C{fetchSize}, C{fetchEnvelope}, and C{fetchSimplifiedBody}
+        functions.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to dict of the retrieved data values, or whose
+        errback is invoked if there is an error.  They dictionary keys
+        are "flags", "date", "size", "envelope", and "body".
+        """
+        return self._fetch(
+            messages,
+            useUID=uid,
+            flags=1,
+            internaldate=1,
+            rfc822size=1,
+            envelope=1,
+            body=1,
+        )
+
+    def fetchAll(self, messages, uid=0):
+        """
+        Retrieve several different fields of one or more messages
+
+        This command is allowed in the Selected state.  This is equivalent
+        to issuing all of the C{fetchFlags}, C{fetchInternalDate},
+        C{fetchSize}, and C{fetchEnvelope} functions.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to dict of the retrieved data values, or whose
+        errback is invoked if there is an error.  They dictionary keys
+        are "flags", "date", "size", and "envelope".
+        """
+        return self._fetch(
+            messages, useUID=uid, flags=1, internaldate=1, rfc822size=1, envelope=1
+        )
+
+    def fetchFast(self, messages, uid=0):
+        """
+        Retrieve several different fields of one or more messages
+
+        This command is allowed in the Selected state.  This is equivalent
+        to issuing all of the C{fetchFlags}, C{fetchInternalDate}, and
+        C{fetchSize} functions.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a dict mapping
+        message numbers to dict of the retrieved data values, or whose
+        errback is invoked if there is an error.  They dictionary keys are
+        "flags", "date", and "size".
+        """
+        return self._fetch(messages, useUID=uid, flags=1, internaldate=1, rfc822size=1)
+
+    def _parseFetchPairs(self, fetchResponseList):
+        """
+        Given the result of parsing a single I{FETCH} response, construct a
+        L{dict} mapping response keys to response values.
+
+        @param fetchResponseList: The result of parsing a I{FETCH} response
+            with L{parseNestedParens} and extracting just the response data
+            (that is, just the part that comes after C{"FETCH"}).  The form
+            of this input (and therefore the output of this method) is very
+            disagreeable.  A valuable improvement would be to enumerate the
+            possible keys (representing them as structured objects of some
+            sort) rather than using strings and tuples of tuples of strings
+            and so forth.  This would allow the keys to be documented more
+            easily and would allow for a much simpler application-facing API
+            (one not based on looking up somewhat hard to predict keys in a
+            dict).  Since C{fetchResponseList} notionally represents a
+            flattened sequence of pairs (identifying keys followed by their
+            associated values), collapsing such complex elements of this
+            list as C{["BODY", ["HEADER.FIELDS", ["SUBJECT"]]]} into a
+            single object would also greatly simplify the implementation of
+            this method.
+
+        @return: A C{dict} of the response data represented by C{pairs}.  Keys
+            in this dictionary are things like C{"RFC822.TEXT"}, C{"FLAGS"}, or
+            C{("BODY", ("HEADER.FIELDS", ("SUBJECT",)))}.  Values are entirely
+            dependent on the key with which they are associated, but retain the
+            same structured as produced by L{parseNestedParens}.
+        """
+
+        # TODO: RFC 3501 Section 7.4.2, "FETCH Response", says for
+        # BODY responses that "8-bit textual data is permitted if a
+        # charset identifier is part of the body parameter
+        # parenthesized list".  Every other component is 7-bit.  This
+        # should parse out the charset identifier and use it to decode
+        # 8-bit bodies.  Until then, on Python 2 it should continue to
+        # return native (byte) strings, while on Python 3 it should
+        # decode bytes to native strings via charmap, ensuring data
+        # fidelity at the cost of mojibake.
+        def nativeStringResponse(thing):
+            if isinstance(thing, bytes):
+                return thing.decode("charmap")
+            elif isinstance(thing, list):
+                return [nativeStringResponse(subthing) for subthing in thing]
+
+        values = {}
+        unstructured = []
+
+        responseParts = iter(fetchResponseList)
+        while True:
+            try:
+                key = next(responseParts)
+            except StopIteration:
+                break
+
+            try:
+                value = next(responseParts)
+            except StopIteration:
+                raise IllegalServerResponse(b"Not enough arguments", fetchResponseList)
+
+            # The parsed forms of responses like:
+            #
+            # BODY[] VALUE
+            # BODY[TEXT] VALUE
+            # BODY[HEADER.FIELDS (SUBJECT)] VALUE
+            # BODY[HEADER.FIELDS (SUBJECT)]<N.M> VALUE
+            #
+            # are:
+            #
+            # ["BODY", [], VALUE]
+            # ["BODY", ["TEXT"], VALUE]
+            # ["BODY", ["HEADER.FIELDS", ["SUBJECT"]], VALUE]
+            # ["BODY", ["HEADER.FIELDS", ["SUBJECT"]], "<N.M>", VALUE]
+            #
+            # Additionally, BODY responses for multipart messages are
+            # represented as:
+            #
+            #    ["BODY", VALUE]
+            #
+            # with list as the type of VALUE and the type of VALUE[0].
+            #
+            # See #6281 for ideas on how this might be improved.
+
+            if key not in (b"BODY", b"BODY.PEEK"):
+                # Only BODY (and by extension, BODY.PEEK) responses can have
+                # body sections.
+                hasSection = False
+            elif not isinstance(value, list):
+                # A BODY section is always represented as a list.  Any non-list
+                # is not a BODY section.
+                hasSection = False
+            elif len(value) > 2:
+                # The list representing a BODY section has at most two elements.
+                hasSection = False
+            elif value and isinstance(value[0], list):
+                # A list containing a list represents the body structure of a
+                # multipart message, instead.
+                hasSection = False
+            else:
+                # Otherwise it must have a BODY section to examine.
+                hasSection = True
+
+            # If it has a BODY section, grab some extra elements and shuffle
+            # around the shape of the key a little bit.
+
+            key = nativeString(key)
+            unstructured.append(key)
+
+            if hasSection:
+                if len(value) < 2:
+                    value = [nativeString(v) for v in value]
+                    unstructured.append(value)
+
+                    key = (key, tuple(value))
+                else:
+                    valueHead = nativeString(value[0])
+                    valueTail = [nativeString(v) for v in value[1]]
+                    unstructured.append([valueHead, valueTail])
+
+                    key = (key, (valueHead, tuple(valueTail)))
+                try:
+                    value = next(responseParts)
+                except StopIteration:
+                    raise IllegalServerResponse(
+                        b"Not enough arguments", fetchResponseList
+                    )
+
+                # Handle partial ranges
+                if value.startswith(b"<") and value.endswith(b">"):
+                    try:
+                        int(value[1:-1])
+                    except ValueError:
+                        # This isn't really a range, it's some content.
+                        pass
+                    else:
+                        value = nativeString(value)
+                        unstructured.append(value)
+                        key = key + (value,)
+                        try:
+                            value = next(responseParts)
+                        except StopIteration:
+                            raise IllegalServerResponse(
+                                b"Not enough arguments", fetchResponseList
+                            )
+
+            value = nativeStringResponse(value)
+            unstructured.append(value)
+            values[key] = value
+
+        return values, unstructured
+
+    def _cbFetch(self, result, requestedParts, structured):
+        (lines, last) = result
+        info = {}
+        for parts in lines:
+            if len(parts) == 3 and parts[1] == b"FETCH":
+                id = self._intOrRaise(parts[0], parts)
+                if id not in info:
+                    info[id] = [parts[2]]
+                else:
+                    info[id][0].extend(parts[2])
+
+        results = {}
+        decodedInfo = {}
+        for messageId, values in info.items():
+            structuredMap, unstructuredList = self._parseFetchPairs(values[0])
+            decodedInfo.setdefault(messageId, [[]])[0].extend(unstructuredList)
+            results.setdefault(messageId, {}).update(structuredMap)
+        info = decodedInfo
+
+        flagChanges = {}
+        for messageId in list(results.keys()):
+            values = results[messageId]
+            for part in list(values.keys()):
+                if part not in requestedParts and part == "FLAGS":
+                    flagChanges[messageId] = values["FLAGS"]
+                    # Find flags in the result and get rid of them.
+                    for i in range(len(info[messageId][0])):
+                        if info[messageId][0][i] == "FLAGS":
+                            del info[messageId][0][i : i + 2]
+                            break
+                    del values["FLAGS"]
+                    if not values:
+                        del results[messageId]
+
+        if flagChanges:
+            self.flagsChanged(flagChanges)
+
+        if structured:
+            return results
+        else:
+            return info
+
+    def fetchSpecific(
+        self,
+        messages,
+        uid=0,
+        headerType=None,
+        headerNumber=None,
+        headerArgs=None,
+        peek=None,
+        offset=None,
+        length=None,
+    ):
+        """
+        Retrieve a specific section of one or more messages
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+            numbers or of unique message IDs.
+
+        @type headerType: L{str}
+        @param headerType: If specified, must be one of HEADER, HEADER.FIELDS,
+            HEADER.FIELDS.NOT, MIME, or TEXT, and will determine which part of
+            the message is retrieved.  For HEADER.FIELDS and HEADER.FIELDS.NOT,
+            C{headerArgs} must be a sequence of header names.  For MIME,
+            C{headerNumber} must be specified.
+
+        @type headerNumber: L{int} or L{int} sequence
+        @param headerNumber: The nested rfc822 index specifying the entity to
+            retrieve.  For example, C{1} retrieves the first entity of the
+            message, and C{(2, 1, 3}) retrieves the 3rd entity inside the first
+            entity inside the second entity of the message.
+
+        @type headerArgs: A sequence of L{str}
+        @param headerArgs: If C{headerType} is HEADER.FIELDS, these are the
+            headers to retrieve.  If it is HEADER.FIELDS.NOT, these are the
+            headers to exclude from retrieval.
+
+        @type peek: C{bool}
+        @param peek: If true, cause the server to not set the \\Seen flag on
+            this message as a result of this command.
+
+        @type offset: L{int}
+        @param offset: The number of octets at the beginning of the result to
+            skip.
+
+        @type length: L{int}
+        @param length: The number of octets to retrieve.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a mapping of message
+            numbers to retrieved data, or whose errback is invoked if there is
+            an error.
+        """
+        fmt = "%s BODY%s[%s%s%s]%s"
+        if headerNumber is None:
+            number = ""
+        elif isinstance(headerNumber, int):
+            number = str(headerNumber)
+        else:
+            number = ".".join(map(str, headerNumber))
+        if headerType is None:
+            header = ""
+        elif number:
+            header = "." + headerType
+        else:
+            header = headerType
+        if header and headerType in ("HEADER.FIELDS", "HEADER.FIELDS.NOT"):
+            if headerArgs is not None:
+                payload = " (%s)" % " ".join(headerArgs)
+            else:
+                payload = " ()"
+        else:
+            payload = ""
+        if offset is None:
+            extra = ""
+        else:
+            extra = "<%d.%d>" % (offset, length)
+        fetch = uid and b"UID FETCH" or b"FETCH"
+        cmd = fmt % (messages, peek and ".PEEK" or "", number, header, payload, extra)
+
+        # APPEND components should be encoded as ASCII unless a
+        # charset identifier is provided.  See #9201.
+        cmd = cmd.encode("charmap")
+
+        d = self.sendCommand(Command(fetch, cmd, wantResponse=(b"FETCH",)))
+        d.addCallback(self._cbFetch, (), False)
+        return d
+
+    def _fetch(self, messages, useUID=0, **terms):
+        messages = str(messages).encode("ascii")
+        fetch = useUID and b"UID FETCH" or b"FETCH"
+
+        if "rfc822text" in terms:
+            del terms["rfc822text"]
+            terms["rfc822.text"] = True
+        if "rfc822size" in terms:
+            del terms["rfc822size"]
+            terms["rfc822.size"] = True
+        if "rfc822header" in terms:
+            del terms["rfc822header"]
+            terms["rfc822.header"] = True
+
+        # The terms in 6.4.5 are all ASCII congruent, so wing it.
+        # Note that this isn't a public API, so terms in responses
+        # should not be decoded to native strings.
+        encodedTerms = [networkString(s) for s in terms]
+        cmd = messages + b" (" + b" ".join([s.upper() for s in encodedTerms]) + b")"
+
+        d = self.sendCommand(Command(fetch, cmd, wantResponse=(b"FETCH",)))
+        d.addCallback(self._cbFetch, [t.upper() for t in terms.keys()], True)
+        return d
+
+    def setFlags(self, messages, flags, silent=1, uid=0):
+        """
+        Set the flags for one or more messages.
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type flags: Any iterable of L{str}
+        @param flags: The flags to set
+
+        @type silent: L{bool}
+        @param silent: If true, cause the server to suppress its verbose
+        response.
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a list of the
+        server's responses (C{[]} if C{silent} is true) or whose
+        errback is invoked if there is an error.
+        """
+        return self._store(messages, b"FLAGS", silent, flags, uid)
+
+    def addFlags(self, messages, flags, silent=1, uid=0):
+        """
+        Add to the set flags for one or more messages.
+
+        This command is allowed in the Selected state.
+
+        @type messages: C{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type flags: Any iterable of L{str}
+        @param flags: The flags to set
+
+        @type silent: C{bool}
+        @param silent: If true, cause the server to suppress its verbose
+        response.
+
+        @type uid: C{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a list of the
+        server's responses (C{[]} if C{silent} is true) or whose
+        errback is invoked if there is an error.
+        """
+        return self._store(messages, b"+FLAGS", silent, flags, uid)
+
+    def removeFlags(self, messages, flags, silent=1, uid=0):
+        """
+        Remove from the set flags for one or more messages.
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type flags: Any iterable of L{str}
+        @param flags: The flags to set
+
+        @type silent: L{bool}
+        @param silent: If true, cause the server to suppress its verbose
+        response.
+
+        @type uid: L{bool}
+        @param uid: Indicates whether the message sequence set is of message
+        numbers or of unique message IDs.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a list of the
+        server's responses (C{[]} if C{silent} is true) or whose
+        errback is invoked if there is an error.
+        """
+        return self._store(messages, b"-FLAGS", silent, flags, uid)
+
+    def _store(self, messages, cmd, silent, flags, uid):
+        messages = str(messages).encode("ascii")
+        encodedFlags = [networkString(flag) for flag in flags]
+        if silent:
+            cmd = cmd + b".SILENT"
+        store = uid and b"UID STORE" or b"STORE"
+        args = b" ".join((messages, cmd, b"(" + b" ".join(encodedFlags) + b")"))
+        d = self.sendCommand(Command(store, args, wantResponse=(b"FETCH",)))
+        expected = ()
+        if not silent:
+            expected = ("FLAGS",)
+        d.addCallback(self._cbFetch, expected, True)
+        return d
+
+    def copy(self, messages, mailbox, uid):
+        """
+        Copy the specified messages to the specified mailbox.
+
+        This command is allowed in the Selected state.
+
+        @type messages: L{MessageSet} or L{str}
+        @param messages: A message sequence set
+
+        @type mailbox: L{str}
+        @param mailbox: The mailbox to which to copy the messages
+
+        @type uid: C{bool}
+        @param uid: If true, the C{messages} refers to message UIDs, rather
+        than message sequence numbers.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with a true value
+        when the copy is successful, or whose errback is invoked if there
+        is an error.
+        """
+        messages = str(messages).encode("ascii")
+        if uid:
+            cmd = b"UID COPY"
+        else:
+            cmd = b"COPY"
+        args = b" ".join([messages, _prepareMailboxName(mailbox)])
+        return self.sendCommand(Command(cmd, args))
+
+    #
+    # IMailboxListener methods
+    #
+    def modeChanged(self, writeable):
+        """Override me"""
+
+    def flagsChanged(self, newFlags):
+        """Override me"""
+
+    def newMessages(self, exists, recent):
+        """Override me"""
+
+
+def parseIdList(s, lastMessageId=None):
+    """
+    Parse a message set search key into a C{MessageSet}.
+
+    @type s: L{bytes}
+    @param s: A string description of an id list, for example "1:3, 4:*"
+
+    @type lastMessageId: L{int}
+    @param lastMessageId: The last message sequence id or UID, depending on
+        whether we are parsing the list in UID or sequence id context. The
+        caller should pass in the correct value.
+
+    @rtype: C{MessageSet}
+    @return: A C{MessageSet} that contains the ids defined in the list
+    """
+    res = MessageSet()
+    parts = s.split(b",")
+    for p in parts:
+        if b":" in p:
+            low, high = p.split(b":", 1)
+            try:
+                if low == b"*":
+                    low = None
+                else:
+                    low = int(low)
+                if high == b"*":
+                    high = None
+                else:
+                    high = int(high)
+                if low is high is None:
+                    # *:* does not make sense
+                    raise IllegalIdentifierError(p)
+                # non-positive values are illegal according to RFC 3501
+                if (low is not None and low <= 0) or (high is not None and high <= 0):
+                    raise IllegalIdentifierError(p)
+                # star means "highest value of an id in the mailbox"
+                high = high or lastMessageId
+                low = low or lastMessageId
+
+                res.add(low, high)
+            except ValueError:
+                raise IllegalIdentifierError(p)
+        else:
+            try:
+                if p == b"*":
+                    p = None
+                else:
+                    p = int(p)
+                if p is not None and p <= 0:
+                    raise IllegalIdentifierError(p)
+            except ValueError:
+                raise IllegalIdentifierError(p)
+            else:
+                res.extend(p or lastMessageId)
+    return res
+
+
+_SIMPLE_BOOL = (
+    "ALL",
+    "ANSWERED",
+    "DELETED",
+    "DRAFT",
+    "FLAGGED",
+    "NEW",
+    "OLD",
+    "RECENT",
+    "SEEN",
+    "UNANSWERED",
+    "UNDELETED",
+    "UNDRAFT",
+    "UNFLAGGED",
+    "UNSEEN",
+)
+
+_NO_QUOTES = ("LARGER", "SMALLER", "UID")
+
+_sorted = sorted
+
+
+def Query(sorted=0, **kwarg):
+    """
+    Create a query string
+
+    Among the accepted keywords are::
+
+        all         : If set to a true value, search all messages in the
+                      current mailbox
+
+        answered    : If set to a true value, search messages flagged with
+                      \\Answered
+
+        bcc         : A substring to search the BCC header field for
+
+        before      : Search messages with an internal date before this
+                      value.  The given date should be a string in the format
+                      of 'DD-Mon-YYYY'.  For example, '03-Mar-2003'.
+
+        body        : A substring to search the body of the messages for
+
+        cc          : A substring to search the CC header field for
+
+        deleted     : If set to a true value, search messages flagged with
+                      \\Deleted
+
+        draft       : If set to a true value, search messages flagged with
+                      \\Draft
+
+        flagged     : If set to a true value, search messages flagged with
+                      \\Flagged
+
+        from        : A substring to search the From header field for
+
+        header      : A two-tuple of a header name and substring to search
+                      for in that header
+
+        keyword     : Search for messages with the given keyword set
+
+        larger      : Search for messages larger than this number of octets
+
+        messages    : Search only the given message sequence set.
+
+        new         : If set to a true value, search messages flagged with
+                      \\Recent but not \\Seen
+
+        old         : If set to a true value, search messages not flagged with
+                      \\Recent
+
+        on          : Search messages with an internal date which is on this
+                      date.  The given date should be a string in the format
+                      of 'DD-Mon-YYYY'.  For example, '03-Mar-2003'.
+
+        recent      : If set to a true value, search for messages flagged with
+                      \\Recent
+
+        seen        : If set to a true value, search for messages flagged with
+                      \\Seen
+
+        sentbefore  : Search for messages with an RFC822 'Date' header before
+                      this date.  The given date should be a string in the format
+                      of 'DD-Mon-YYYY'.  For example, '03-Mar-2003'.
+
+        senton      : Search for messages with an RFC822 'Date' header which is
+                      on this date  The given date should be a string in the format
+                      of 'DD-Mon-YYYY'.  For example, '03-Mar-2003'.
+
+        sentsince   : Search for messages with an RFC822 'Date' header which is
+                      after this date.  The given date should be a string in the format
+                      of 'DD-Mon-YYYY'.  For example, '03-Mar-2003'.
+
+        since       : Search for messages with an internal date that is after
+                      this date..  The given date should be a string in the format
+                      of 'DD-Mon-YYYY'.  For example, '03-Mar-2003'.
+
+        smaller     : Search for messages smaller than this number of octets
+
+        subject     : A substring to search the 'subject' header for
+
+        text        : A substring to search the entire message for
+
+        to          : A substring to search the 'to' header for
+
+        uid         : Search only the messages in the given message set
+
+        unanswered  : If set to a true value, search for messages not
+                      flagged with \\Answered
+
+        undeleted   : If set to a true value, search for messages not
+                      flagged with \\Deleted
+
+        undraft     : If set to a true value, search for messages not
+                      flagged with \\Draft
+
+        unflagged   : If set to a true value, search for messages not
+                      flagged with \\Flagged
+
+        unkeyword   : Search for messages without the given keyword set
+
+        unseen      : If set to a true value, search for messages not
+                      flagged with \\Seen
+
+    @type sorted: C{bool}
+    @param sorted: If true, the output will be sorted, alphabetically.
+    The standard does not require it, but it makes testing this function
+    easier.  The default is zero, and this should be acceptable for any
+    application.
+
+    @rtype: L{str}
+    @return: The formatted query string
+    """
+    cmd = []
+    keys = kwarg.keys()
+    if sorted:
+        keys = _sorted(keys)
+    for k in keys:
+        v = kwarg[k]
+        k = k.upper()
+        if k in _SIMPLE_BOOL and v:
+            cmd.append(k)
+        elif k == "HEADER":
+            cmd.extend([k, str(v[0]), str(v[1])])
+        elif k == "KEYWORD" or k == "UNKEYWORD":
+            # Discard anything that does not fit into an "atom".  Perhaps turn
+            # the case where this actually removes bytes from the value into a
+            # warning and then an error, eventually.  See #6277.
+            v = _nonAtomRE.sub("", v)
+            cmd.extend([k, v])
+        elif k not in _NO_QUOTES:
+            if isinstance(v, MessageSet):
+                fmt = '"%s"'
+            elif isinstance(v, str):
+                fmt = '"%s"'
+            else:
+                fmt = '"%d"'
+            cmd.extend([k, fmt % (v,)])
+        elif isinstance(v, int):
+            cmd.extend([k, "%d" % (v,)])
+        else:
+            cmd.extend([k, f"{v}"])
+    if len(cmd) > 1:
+        return "(" + " ".join(cmd) + ")"
+    else:
+        return " ".join(cmd)
+
+
+def Or(*args):
+    """
+    The disjunction of two or more queries
+    """
+    if len(args) < 2:
+        raise IllegalQueryError(args)
+    elif len(args) == 2:
+        return "(OR %s %s)" % args
+    else:
+        return f"(OR {args[0]} {Or(*args[1:])})"
+
+
+def Not(query):
+    """The negation of a query"""
+    return f"(NOT {query})"
+
+
+def wildcardToRegexp(wildcard, delim=None):
+    wildcard = wildcard.replace("*", "(?:.*?)")
+    if delim is None:
+        wildcard = wildcard.replace("%", "(?:.*?)")
+    else:
+        wildcard = wildcard.replace("%", "(?:(?:[^%s])*?)" % re.escape(delim))
+    return re.compile(wildcard, re.I)
+
+
+def splitQuoted(s):
+    """
+    Split a string into whitespace delimited tokens
+
+    Tokens that would otherwise be separated but are surrounded by \"
+    remain as a single token.  Any token that is not quoted and is
+    equal to \"NIL\" is tokenized as L{None}.
+
+    @type s: L{bytes}
+    @param s: The string to be split
+
+    @rtype: L{list} of L{bytes}
+    @return: A list of the resulting tokens
+
+    @raise MismatchedQuoting: Raised if an odd number of quotes are present
+    """
+    s = s.strip()
+    result = []
+    word = []
+    inQuote = inWord = False
+    qu = _matchingString('"', s)
+    esc = _matchingString("\x5c", s)
+    empty = _matchingString("", s)
+    nil = _matchingString("NIL", s)
+    for i, c in enumerate(iterbytes(s)):
+        if c == qu:
+            if i and s[i - 1 : i] == esc:
+                word.pop()
+                word.append(qu)
+            elif not inQuote:
+                inQuote = True
+            else:
+                inQuote = False
+                result.append(empty.join(word))
+                word = []
+        elif (
+            not inWord
+            and not inQuote
+            and c not in (qu + (string.whitespace.encode("ascii")))
+        ):
+            inWord = True
+            word.append(c)
+        elif inWord and not inQuote and c in string.whitespace.encode("ascii"):
+            w = empty.join(word)
+            if w == nil:
+                result.append(None)
+            else:
+                result.append(w)
+            word = []
+            inWord = False
+        elif inWord or inQuote:
+            word.append(c)
+
+    if inQuote:
+        raise MismatchedQuoting(s)
+    if inWord:
+        w = empty.join(word)
+        if w == nil:
+            result.append(None)
+        else:
+            result.append(w)
+
+    return result
+
+
+def splitOn(sequence, predicate, transformers):
+    result = []
+    mode = predicate(sequence[0])
+    tmp = [sequence[0]]
+    for e in sequence[1:]:
+        p = predicate(e)
+        if p != mode:
+            result.extend(transformers[mode](tmp))
+            tmp = [e]
+            mode = p
+        else:
+            tmp.append(e)
+    result.extend(transformers[mode](tmp))
+    return result
+
+
+def collapseStrings(results):
+    """
+    Turns a list of length-one strings and lists into a list of longer
+    strings and lists.  For example,
+
+    ['a', 'b', ['c', 'd']] is returned as ['ab', ['cd']]
+
+    @type results: L{list} of L{bytes} and L{list}
+    @param results: The list to be collapsed
+
+    @rtype: L{list} of L{bytes} and L{list}
+    @return: A new list which is the collapsed form of C{results}
+    """
+    copy = []
+    begun = None
+
+    pred = lambda e: isinstance(e, tuple)
+    tran = {
+        0: lambda e: splitQuoted(b"".join(e)),
+        1: lambda e: [b"".join([i[0] for i in e])],
+    }
+    for i, c in enumerate(results):
+        if isinstance(c, list):
+            if begun is not None:
+                copy.extend(splitOn(results[begun:i], pred, tran))
+                begun = None
+            copy.append(collapseStrings(c))
+        elif begun is None:
+            begun = i
+    if begun is not None:
+        copy.extend(splitOn(results[begun:], pred, tran))
+    return copy
+
+
+def parseNestedParens(s, handleLiteral=1):
+    """
+    Parse an s-exp-like string into a more useful data structure.
+
+    @type s: L{bytes}
+    @param s: The s-exp-like string to parse
+
+    @rtype: L{list} of L{bytes} and L{list}
+    @return: A list containing the tokens present in the input.
+
+    @raise MismatchedNesting: Raised if the number or placement
+    of opening or closing parenthesis is invalid.
+    """
+    s = s.strip()
+    inQuote = 0
+    contentStack = [[]]
+    try:
+        i = 0
+        L = len(s)
+        while i < L:
+            c = s[i : i + 1]
+            if inQuote:
+                if c == b"\\":
+                    contentStack[-1].append(s[i : i + 2])
+                    i += 2
+                    continue
+                elif c == b'"':
+                    inQuote = not inQuote
+                contentStack[-1].append(c)
+                i += 1
+            else:
+                if c == b'"':
+                    contentStack[-1].append(c)
+                    inQuote = not inQuote
+                    i += 1
+                elif handleLiteral and c == b"{":
+                    end = s.find(b"}", i)
+                    if end == -1:
+                        raise ValueError("Malformed literal")
+                    literalSize = int(s[i + 1 : end])
+                    contentStack[-1].append((s[end + 3 : end + 3 + literalSize],))
+                    i = end + 3 + literalSize
+                elif c == b"(" or c == b"[":
+                    contentStack.append([])
+                    i += 1
+                elif c == b")" or c == b"]":
+                    contentStack[-2].append(contentStack.pop())
+                    i += 1
+                else:
+                    contentStack[-1].append(c)
+                    i += 1
+    except IndexError:
+        raise MismatchedNesting(s)
+    if len(contentStack) != 1:
+        raise MismatchedNesting(s)
+    return collapseStrings(contentStack[0])
+
+
+def _quote(s):
+    qu = _matchingString('"', s)
+    esc = _matchingString("\x5c", s)
+    return qu + s.replace(esc, esc + esc).replace(qu, esc + qu) + qu
+
+
+def _literal(s: bytes) -> bytes:
+    return b"{%d}\r\n%b" % (len(s), s)
+
+
+class DontQuoteMe:
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self) -> str:
+        return str(self.value)
+
+
+_ATOM_SPECIALS = b'(){ %*"'
+
+
+def _needsQuote(s):
+    if s == b"":
+        return 1
+    for c in iterbytes(s):
+        if c < b"\x20" or c > b"\x7f":
+            return 1
+        if c in _ATOM_SPECIALS:
+            return 1
+    return 0
+
+
+def _parseMbox(name):
+    if isinstance(name, str):
+        return name
+    try:
+        return name.decode("imap4-utf-7")
+    except BaseException:
+        log.err()
+        raise IllegalMailboxEncoding(name)
+
+
+def _prepareMailboxName(name):
+    if not isinstance(name, str):
+        name = name.decode("charmap")
+    name = name.encode("imap4-utf-7")
+    if _needsQuote(name):
+        return _quote(name)
+    return name
+
+
+def _needsLiteral(s):
+    # change this to "return 1" to wig out stupid clients
+    cr = _matchingString("\n", s)
+    lf = _matchingString("\r", s)
+    return cr in s or lf in s or len(s) > 1000
+
+
+def collapseNestedLists(items):
+    """
+    Turn a nested list structure into an s-exp-like string.
+
+    Strings in C{items} will be sent as literals if they contain CR or LF,
+    otherwise they will be quoted.  References to None in C{items} will be
+    translated to the atom NIL.  Objects with a 'read' attribute will have
+    it called on them with no arguments and the returned string will be
+    inserted into the output as a literal.  Integers will be converted to
+    strings and inserted into the output unquoted.  Instances of
+    C{DontQuoteMe} will be converted to strings and inserted into the output
+    unquoted.
+
+    This function used to be much nicer, and only quote things that really
+    needed to be quoted (and C{DontQuoteMe} did not exist), however, many
+    broken IMAP4 clients were unable to deal with this level of sophistication,
+    forcing the current behavior to be adopted for practical reasons.
+
+    @type items: Any iterable
+
+    @rtype: L{str}
+    """
+    pieces = []
+    for i in items:
+        if isinstance(i, str):
+            # anything besides ASCII will have to wait for an RFC 5738
+            # implementation.  See
+            # https://twistedmatrix.com/trac/ticket/9258
+            i = i.encode("ascii")
+        if i is None:
+            pieces.extend([b" ", b"NIL"])
+        elif isinstance(i, int):
+            pieces.extend([b" ", networkString(str(i))])
+        elif isinstance(i, DontQuoteMe):
+            pieces.extend([b" ", i.value])
+        elif isinstance(i, bytes):
+            # XXX warning
+            if _needsLiteral(i):
+                pieces.extend([b" ", b"{%d}" % (len(i),), IMAP4Server.delimiter, i])
+            else:
+                pieces.extend([b" ", _quote(i)])
+        elif hasattr(i, "read"):
+            d = i.read()
+            pieces.extend([b" ", b"{%d}" % (len(d),), IMAP4Server.delimiter, d])
+        else:
+            pieces.extend([b" ", b"(" + collapseNestedLists(i) + b")"])
+    return b"".join(pieces[1:])
+
+
+@implementer(IAccount)
+class MemoryAccountWithoutNamespaces:
+    mailboxes = None
+    subscriptions = None
+    top_id = 0
+
+    def __init__(self, name):
+        self.name = name
+        self.mailboxes = {}
+        self.subscriptions = []
+
+    def allocateID(self):
+        id = self.top_id
+        self.top_id += 1
+        return id
+
+    ##
+    ## IAccount
+    ##
+    def addMailbox(self, name, mbox=None):
+        name = _parseMbox(name.upper())
+        if name in self.mailboxes:
+            raise MailboxCollision(name)
+        if mbox is None:
+            mbox = self._emptyMailbox(name, self.allocateID())
+        self.mailboxes[name] = mbox
+        return 1
+
+    def create(self, pathspec):
+        paths = [path for path in pathspec.split("/") if path]
+        for accum in range(1, len(paths)):
+            try:
+                self.addMailbox("/".join(paths[:accum]))
+            except MailboxCollision:
+                pass
+        try:
+            self.addMailbox("/".join(paths))
+        except MailboxCollision:
+            if not pathspec.endswith("/"):
+                return False
+        return True
+
+    def _emptyMailbox(self, name, id):
+        raise NotImplementedError
+
+    def select(self, name, readwrite=1):
+        return self.mailboxes.get(_parseMbox(name.upper()))
+
+    def delete(self, name):
+        name = _parseMbox(name.upper())
+        # See if this mailbox exists at all
+        mbox = self.mailboxes.get(name)
+        if not mbox:
+            raise MailboxException("No such mailbox")
+        # See if this box is flagged \Noselect
+        if r"\Noselect" in mbox.getFlags():
+            # Check for hierarchically inferior mailboxes with this one
+            # as part of their root.
+            for others in self.mailboxes.keys():
+                if others != name and others.startswith(name):
+                    raise MailboxException(
+                        "Hierarchically inferior mailboxes exist and \\Noselect is set"
+                    )
+        mbox.destroy()
+
+        # iff there are no hierarchically inferior names, we will
+        # delete it from our ken.
+        if len(self._inferiorNames(name)) > 1:
+            raise MailboxException(f'Name "{name}" has inferior hierarchical names')
+        del self.mailboxes[name]
+
+    def rename(self, oldname, newname):
+        oldname = _parseMbox(oldname.upper())
+        newname = _parseMbox(newname.upper())
+        if oldname not in self.mailboxes:
+            raise NoSuchMailbox(oldname)
+
+        inferiors = self._inferiorNames(oldname)
+        inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
+
+        for old, new in inferiors:
+            if new in self.mailboxes:
+                raise MailboxCollision(new)
+
+        for old, new in inferiors:
+            self.mailboxes[new] = self.mailboxes[old]
+            del self.mailboxes[old]
+
+    def _inferiorNames(self, name):
+        inferiors = []
+        for infname in self.mailboxes.keys():
+            if infname.startswith(name):
+                inferiors.append(infname)
+        return inferiors
+
+    def isSubscribed(self, name):
+        return _parseMbox(name.upper()) in self.subscriptions
+
+    def subscribe(self, name):
+        name = _parseMbox(name.upper())
+        if name not in self.subscriptions:
+            self.subscriptions.append(name)
+
+    def unsubscribe(self, name):
+        name = _parseMbox(name.upper())
+        if name not in self.subscriptions:
+            raise MailboxException(f"Not currently subscribed to {name}")
+        self.subscriptions.remove(name)
+
+    def listMailboxes(self, ref, wildcard):
+        ref = self._inferiorNames(_parseMbox(ref.upper()))
+        wildcard = wildcardToRegexp(wildcard, "/")
+        return [(i, self.mailboxes[i]) for i in ref if wildcard.match(i)]
+
+
+@implementer(INamespacePresenter)
+class MemoryAccount(MemoryAccountWithoutNamespaces):
+    ##
+    ## INamespacePresenter
+    ##
+    def getPersonalNamespaces(self):
+        return [[b"", b"/"]]
+
+    def getSharedNamespaces(self):
+        return None
+
+    def getOtherNamespaces(self):
+        return None
+
+    def getUserNamespaces(self):
+        # INamespacePresenter.getUserNamespaces
+        return None
+
+
+_statusRequestDict = {
+    "MESSAGES": "getMessageCount",
+    "RECENT": "getRecentCount",
+    "UIDNEXT": "getUIDNext",
+    "UIDVALIDITY": "getUIDValidity",
+    "UNSEEN": "getUnseenCount",
+}
+
+
+def statusRequestHelper(mbox, names):
+    r = {}
+    for n in names:
+        r[n] = getattr(mbox, _statusRequestDict[n.upper()])()
+    return r
+
+
+def parseAddr(addr):
+    if addr is None:
+        return [
+            (None, None, None),
+        ]
+    addr = email.utils.getaddresses([addr])
+    return [[fn or None, None] + address.split("@") for fn, address in addr]
+
+
+def getEnvelope(msg):
+    headers = msg.getHeaders(True)
+    date = headers.get("date")
+    subject = headers.get("subject")
+    from_ = headers.get("from")
+    sender = headers.get("sender", from_)
+    reply_to = headers.get("reply-to", from_)
+    to = headers.get("to")
+    cc = headers.get("cc")
+    bcc = headers.get("bcc")
+    in_reply_to = headers.get("in-reply-to")
+    mid = headers.get("message-id")
+    return (
+        date,
+        subject,
+        parseAddr(from_),
+        parseAddr(sender),
+        reply_to and parseAddr(reply_to),
+        to and parseAddr(to),
+        cc and parseAddr(cc),
+        bcc and parseAddr(bcc),
+        in_reply_to,
+        mid,
+    )
+
+
+def getLineCount(msg):
+    # XXX - Super expensive, CACHE THIS VALUE FOR LATER RE-USE
+    # XXX - This must be the number of lines in the ENCODED version
+    lines = 0
+    for _ in msg.getBodyFile():
+        lines += 1
+    return lines
+
+
+def unquote(s):
+    if s[0] == s[-1] == '"':
+        return s[1:-1]
+    return s
+
+
+def _getContentType(msg):
+    """
+    Return a two-tuple of the main and subtype of the given message.
+    """
+    attrs = None
+    mm = msg.getHeaders(False, "content-type").get("content-type", "")
+    mm = "".join(mm.splitlines())
+    if mm:
+        mimetype = mm.split(";")
+        type = mimetype[0].split("/", 1)
+        if len(type) == 1:
+            major = type[0]
+            minor = None
+        else:
+            # length must be 2, because of split('/', 1)
+            major, minor = type
+        attrs = dict(x.strip().lower().split("=", 1) for x in mimetype[1:])
+    else:
+        major = minor = None
+    return major, minor, attrs
+
+
+def _getMessageStructure(message):
+    """
+    Construct an appropriate type of message structure object for the given
+    message object.
+
+    @param message: A L{IMessagePart} provider
+
+    @return: A L{_MessageStructure} instance of the most specific type available
+        for the given message, determined by inspecting the MIME type of the
+        message.
+    """
+    main, subtype, attrs = _getContentType(message)
+    if main is not None:
+        main = main.lower()
+    if subtype is not None:
+        subtype = subtype.lower()
+    if main == "multipart":
+        return _MultipartMessageStructure(message, subtype, attrs)
+    elif (main, subtype) == ("message", "rfc822"):
+        return _RFC822MessageStructure(message, main, subtype, attrs)
+    elif main == "text":
+        return _TextMessageStructure(message, main, subtype, attrs)
+    else:
+        return _SinglepartMessageStructure(message, main, subtype, attrs)
+
+
+class _MessageStructure:
+    """
+    L{_MessageStructure} is a helper base class for message structure classes
+    representing the structure of particular kinds of messages, as defined by
+    their MIME type.
+    """
+
+    def __init__(self, message, attrs):
+        """
+        @param message: An L{IMessagePart} provider which this structure object
+            reports on.
+
+        @param attrs: A C{dict} giving the parameters of the I{Content-Type}
+            header of the message.
+        """
+        self.message = message
+        self.attrs = attrs
+
+    def _disposition(self, disp):
+        """
+        Parse a I{Content-Disposition} header into a two-sequence of the
+        disposition and a flattened list of its parameters.
+
+        @return: L{None} if there is no disposition header value, a L{list} with
+            two elements otherwise.
+        """
+        if disp:
+            disp = disp.split("; ")
+            if len(disp) == 1:
+                disp = (disp[0].lower(), None)
+            elif len(disp) > 1:
+                # XXX Poorly tested parser
+                params = [x for param in disp[1:] for x in param.split("=", 1)]
+                disp = [disp[0].lower(), params]
+            return disp
+        else:
+            return None
+
+    def _unquotedAttrs(self):
+        """
+        @return: The I{Content-Type} parameters, unquoted, as a flat list with
+            each Nth element giving a parameter name and N+1th element giving
+            the corresponding parameter value.
+        """
+        if self.attrs:
+            unquoted = [(k, unquote(v)) for (k, v) in self.attrs.items()]
+            return [y for x in sorted(unquoted) for y in x]
+        return None
+
+
+class _SinglepartMessageStructure(_MessageStructure):
+    """
+    L{_SinglepartMessageStructure} represents the message structure of a
+    non-I{multipart/*} message.
+    """
+
+    _HEADERS = ["content-id", "content-description", "content-transfer-encoding"]
+
+    def __init__(self, message, main, subtype, attrs):
+        """
+        @param message: An L{IMessagePart} provider which this structure object
+            reports on.
+
+        @param main: A L{str} giving the main MIME type of the message (for
+            example, C{"text"}).
+
+        @param subtype: A L{str} giving the MIME subtype of the message (for
+            example, C{"plain"}).
+
+        @param attrs: A C{dict} giving the parameters of the I{Content-Type}
+            header of the message.
+        """
+        _MessageStructure.__init__(self, message, attrs)
+        self.main = main
+        self.subtype = subtype
+        self.attrs = attrs
+
+    def _basicFields(self):
+        """
+        Return a list of the basic fields for a single-part message.
+        """
+        headers = self.message.getHeaders(False, *self._HEADERS)
+
+        # Number of octets total
+        size = self.message.getSize()
+
+        major, minor = self.main, self.subtype
+
+        # content-type parameter list
+        unquotedAttrs = self._unquotedAttrs()
+
+        return [
+            major,
+            minor,
+            unquotedAttrs,
+            headers.get("content-id"),
+            headers.get("content-description"),
+            headers.get("content-transfer-encoding"),
+            size,
+        ]
+
+    def encode(self, extended):
+        """
+        Construct and return a list of the basic and extended fields for a
+        single-part message.  The list suitable to be encoded into a BODY or
+        BODYSTRUCTURE response.
+        """
+        result = self._basicFields()
+        if extended:
+            result.extend(self._extended())
+        return result
+
+    def _extended(self):
+        """
+        The extension data of a non-multipart body part are in the
+        following order:
+
+          1. body MD5
+
+             A string giving the body MD5 value as defined in [MD5].
+
+          2. body disposition
+
+             A parenthesized list with the same content and function as
+             the body disposition for a multipart body part.
+
+          3. body language
+
+             A string or parenthesized list giving the body language
+             value as defined in [LANGUAGE-TAGS].
+
+          4. body location
+
+             A string list giving the body content URI as defined in
+             [LOCATION].
+
+        """
+        result = []
+        headers = self.message.getHeaders(
+            False,
+            "content-md5",
+            "content-disposition",
+            "content-language",
+            "content-language",
+        )
+
+        result.append(headers.get("content-md5"))
+        result.append(self._disposition(headers.get("content-disposition")))
+        result.append(headers.get("content-language"))
+        result.append(headers.get("content-location"))
+
+        return result
+
+
+class _TextMessageStructure(_SinglepartMessageStructure):
+    """
+    L{_TextMessageStructure} represents the message structure of a I{text/*}
+    message.
+    """
+
+    def encode(self, extended):
+        """
+        A body type of type TEXT contains, immediately after the basic
+        fields, the size of the body in text lines.  Note that this
+        size is the size in its content transfer encoding and not the
+        resulting size after any decoding.
+        """
+        result = _SinglepartMessageStructure._basicFields(self)
+        result.append(getLineCount(self.message))
+        if extended:
+            result.extend(self._extended())
+        return result
+
+
+class _RFC822MessageStructure(_SinglepartMessageStructure):
+    """
+    L{_RFC822MessageStructure} represents the message structure of a
+    I{message/rfc822} message.
+    """
+
+    def encode(self, extended):
+        """
+        A body type of type MESSAGE and subtype RFC822 contains,
+        immediately after the basic fields, the envelope structure,
+        body structure, and size in text lines of the encapsulated
+        message.
+        """
+        result = _SinglepartMessageStructure.encode(self, extended)
+        contained = self.message.getSubPart(0)
+        result.append(getEnvelope(contained))
+        result.append(getBodyStructure(contained, False))
+        result.append(getLineCount(contained))
+        return result
+
+
+class _MultipartMessageStructure(_MessageStructure):
+    """
+    L{_MultipartMessageStructure} represents the message structure of a
+    I{multipart/*} message.
+    """
+
+    def __init__(self, message, subtype, attrs):
+        """
+        @param message: An L{IMessagePart} provider which this structure object
+            reports on.
+
+        @param subtype: A L{str} giving the MIME subtype of the message (for
+            example, C{"plain"}).
+
+        @param attrs: A C{dict} giving the parameters of the I{Content-Type}
+            header of the message.
+        """
+        _MessageStructure.__init__(self, message, attrs)
+        self.subtype = subtype
+
+    def _getParts(self):
+        """
+        Return an iterator over all of the sub-messages of this message.
+        """
+        i = 0
+        while True:
+            try:
+                part = self.message.getSubPart(i)
+            except IndexError:
+                break
+            else:
+                yield part
+                i += 1
+
+    def encode(self, extended):
+        """
+        Encode each sub-message and added the additional I{multipart} fields.
+        """
+        result = [_getMessageStructure(p).encode(extended) for p in self._getParts()]
+        result.append(self.subtype)
+        if extended:
+            result.extend(self._extended())
+        return result
+
+    def _extended(self):
+        """
+        The extension data of a multipart body part are in the following order:
+
+          1. body parameter parenthesized list
+               A parenthesized list of attribute/value pairs [e.g., ("foo"
+               "bar" "baz" "rag") where "bar" is the value of "foo", and
+               "rag" is the value of "baz"] as defined in [MIME-IMB].
+
+          2. body disposition
+               A parenthesized list, consisting of a disposition type
+               string, followed by a parenthesized list of disposition
+               attribute/value pairs as defined in [DISPOSITION].
+
+          3. body language
+               A string or parenthesized list giving the body language
+               value as defined in [LANGUAGE-TAGS].
+
+          4. body location
+               A string list giving the body content URI as defined in
+               [LOCATION].
+        """
+        result = []
+        headers = self.message.getHeaders(
+            False, "content-language", "content-location", "content-disposition"
+        )
+
+        result.append(self._unquotedAttrs())
+        result.append(self._disposition(headers.get("content-disposition")))
+        result.append(headers.get("content-language", None))
+        result.append(headers.get("content-location", None))
+
+        return result
+
+
+def getBodyStructure(msg, extended=False):
+    """
+    RFC 3501, 7.4.2, BODYSTRUCTURE::
+
+      A parenthesized list that describes the [MIME-IMB] body structure of a
+      message.  This is computed by the server by parsing the [MIME-IMB] header
+      fields, defaulting various fields as necessary.
+
+        For example, a simple text message of 48 lines and 2279 octets can have
+        a body structure of: ("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL
+        "7BIT" 2279 48)
+
+    This is represented as::
+
+        ["TEXT", "PLAIN", ["CHARSET", "US-ASCII"], None, None, "7BIT", 2279, 48]
+
+    These basic fields are documented in the RFC as:
+
+      1. body type
+
+         A string giving the content media type name as defined in
+         [MIME-IMB].
+
+      2. body subtype
+
+         A string giving the content subtype name as defined in
+         [MIME-IMB].
+
+      3. body parameter parenthesized list
+
+         A parenthesized list of attribute/value pairs [e.g., ("foo"
+         "bar" "baz" "rag") where "bar" is the value of "foo" and
+         "rag" is the value of "baz"] as defined in [MIME-IMB].
+
+      4. body id
+
+         A string giving the content id as defined in [MIME-IMB].
+
+      5. body description
+
+         A string giving the content description as defined in
+         [MIME-IMB].
+
+      6. body encoding
+
+         A string giving the content transfer encoding as defined in
+         [MIME-IMB].
+
+      7. body size
+
+         A number giving the size of the body in octets.  Note that this size is
+         the size in its transfer encoding and not the resulting size after any
+         decoding.
+
+    Put another way, the body structure is a list of seven elements.  The
+    semantics of the elements of this list are:
+
+       1. Byte string giving the major MIME type
+       2. Byte string giving the minor MIME type
+       3. A list giving the Content-Type parameters of the message
+       4. A byte string giving the content identifier for the message part, or
+          None if it has no content identifier.
+       5. A byte string giving the content description for the message part, or
+          None if it has no content description.
+       6. A byte string giving the Content-Encoding of the message body
+       7. An integer giving the number of octets in the message body
+
+    The RFC goes on::
+
+        Multiple parts are indicated by parenthesis nesting.  Instead of a body
+        type as the first element of the parenthesized list, there is a sequence
+        of one or more nested body structures.  The second element of the
+        parenthesized list is the multipart subtype (mixed, digest, parallel,
+        alternative, etc.).
+
+        For example, a two part message consisting of a text and a
+        BASE64-encoded text attachment can have a body structure of: (("TEXT"
+        "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 1152 23)("TEXT" "PLAIN"
+        ("CHARSET" "US-ASCII" "NAME" "cc.diff")
+        "<960723163407.20117h@cac.washington.edu>" "Compiler diff" "BASE64" 4554
+        73) "MIXED")
+
+    This is represented as::
+
+        [["TEXT", "PLAIN", ["CHARSET", "US-ASCII"], None, None, "7BIT", 1152,
+          23],
+         ["TEXT", "PLAIN", ["CHARSET", "US-ASCII", "NAME", "cc.diff"],
+          "<960723163407.20117h@cac.washington.edu>", "Compiler diff",
+          "BASE64", 4554, 73],
+         "MIXED"]
+
+    In other words, a list of N + 1 elements, where N is the number of parts in
+    the message.  The first N elements are structures as defined by the previous
+    section.  The last element is the minor MIME subtype of the multipart
+    message.
+
+    Additionally, the RFC describes extension data::
+
+        Extension data follows the multipart subtype.  Extension data is never
+        returned with the BODY fetch, but can be returned with a BODYSTRUCTURE
+        fetch.  Extension data, if present, MUST be in the defined order.
+
+    The C{extended} flag controls whether extension data might be returned with
+    the normal data.
+    """
+    return _getMessageStructure(msg).encode(extended)
+
+
+def _formatHeaders(headers):
+    # TODO: This should use email.header.Header, which handles encoding
+    hdrs = [
+        ": ".join((k.title(), "\r\n".join(v.splitlines())))
+        for (k, v) in headers.items()
+    ]
+    hdrs = "\r\n".join(hdrs) + "\r\n"
+    return networkString(hdrs)
+
+
+def subparts(m):
+    i = 0
+    try:
+        while True:
+            yield m.getSubPart(i)
+            i += 1
+    except IndexError:
+        pass
+
+
+def iterateInReactor(i):
+    """
+    Consume an interator at most a single iteration per reactor iteration.
+
+    If the iterator produces a Deferred, the next iteration will not occur
+    until the Deferred fires, otherwise the next iteration will be taken
+    in the next reactor iteration.
+
+    @rtype: C{Deferred}
+    @return: A deferred which fires (with None) when the iterator is
+    exhausted or whose errback is called if there is an exception.
+    """
+    from twisted.internet import reactor
+
+    d = defer.Deferred()
+
+    def go(last):
+        try:
+            r = next(i)
+        except StopIteration:
+            d.callback(last)
+        except BaseException:
+            d.errback()
+        else:
+            if isinstance(r, defer.Deferred):
+                r.addCallback(go)
+            else:
+                reactor.callLater(0, go, r)
+
+    go(None)
+    return d
+
+
+class MessageProducer:
+    CHUNK_SIZE = 2**2**2**2
+    _uuid4 = staticmethod(uuid.uuid4)
+
+    def __init__(self, msg, buffer=None, scheduler=None):
+        """
+        Produce this message.
+
+        @param msg: The message I am to produce.
+        @type msg: L{IMessage}
+
+        @param buffer: A buffer to hold the message in.  If None, I will
+            use a L{tempfile.TemporaryFile}.
+        @type buffer: file-like
+        """
+        self.msg = msg
+        if buffer is None:
+            buffer = tempfile.TemporaryFile()
+        self.buffer = buffer
+        if scheduler is None:
+            scheduler = iterateInReactor
+        self.scheduler = scheduler
+        self.write = self.buffer.write
+
+    def beginProducing(self, consumer):
+        self.consumer = consumer
+        return self.scheduler(self._produce())
+
+    def _produce(self):
+        headers = self.msg.getHeaders(True)
+        boundary = None
+        if self.msg.isMultipart():
+            content = headers.get("content-type")
+            parts = [x.split("=", 1) for x in content.split(";")[1:]]
+            parts = {k.lower().strip(): v for (k, v) in parts}
+            boundary = parts.get("boundary")
+            if boundary is None:
+                # Bastards
+                boundary = f"----={self._uuid4().hex}"
+                headers["content-type"] += f'; boundary="{boundary}"'
+            else:
+                if boundary.startswith('"') and boundary.endswith('"'):
+                    boundary = boundary[1:-1]
+            boundary = networkString(boundary)
+
+        self.write(_formatHeaders(headers))
+        self.write(b"\r\n")
+        if self.msg.isMultipart():
+            for p in subparts(self.msg):
+                self.write(b"\r\n--" + boundary + b"\r\n")
+                yield MessageProducer(p, self.buffer, self.scheduler).beginProducing(
+                    None
+                )
+            self.write(b"\r\n--" + boundary + b"--\r\n")
+        else:
+            f = self.msg.getBodyFile()
+            while True:
+                b = f.read(self.CHUNK_SIZE)
+                if b:
+                    self.buffer.write(b)
+                    yield None
+                else:
+                    break
+        if self.consumer:
+            self.buffer.seek(0, 0)
+            yield FileProducer(self.buffer).beginProducing(self.consumer).addCallback(
+                lambda _: self
+            )
+
+
+class _FetchParser:
+    class Envelope:
+        # Response should be a list of fields from the message:
+        #   date, subject, from, sender, reply-to, to, cc, bcc, in-reply-to,
+        #   and message-id.
+        #
+        # from, sender, reply-to, to, cc, and bcc are themselves lists of
+        # address information:
+        #   personal name, source route, mailbox name, host name
+        #
+        # reply-to and sender must not be None.  If not present in a message
+        # they should be defaulted to the value of the from field.
+        type = "envelope"
+        __str__ = lambda self: "envelope"
+
+    class Flags:
+        type = "flags"
+        __str__ = lambda self: "flags"
+
+    class InternalDate:
+        type = "internaldate"
+        __str__ = lambda self: "internaldate"
+
+    class RFC822Header:
+        type = "rfc822header"
+        __str__ = lambda self: "rfc822.header"
+
+    class RFC822Text:
+        type = "rfc822text"
+        __str__ = lambda self: "rfc822.text"
+
+    class RFC822Size:
+        type = "rfc822size"
+        __str__ = lambda self: "rfc822.size"
+
+    class RFC822:
+        type = "rfc822"
+        __str__ = lambda self: "rfc822"
+
+    class UID:
+        type = "uid"
+        __str__ = lambda self: "uid"
+
+    class Body:
+        type = "body"
+        peek = False
+        header = None
+        mime = None
+        text = None
+        part = ()
+        empty = False
+        partialBegin = None
+        partialLength = None
+
+        def __str__(self) -> str:
+            return self.__bytes__().decode("ascii")
+
+        def __bytes__(self) -> bytes:
+            base = b"BODY"
+            part = b""
+            separator = b""
+            if self.part:
+                part = b".".join([str(x + 1).encode("ascii") for x in self.part])  # type: ignore[unreachable]
+                separator = b"."
+            #            if self.peek:
+            #                base += '.PEEK'
+            if self.header:
+                base += (  # type: ignore[unreachable]
+                    b"[" + part + separator + str(self.header).encode("ascii") + b"]"
+                )
+            elif self.text:
+                base += b"[" + part + separator + b"TEXT]"  # type: ignore[unreachable]
+            elif self.mime:
+                base += b"[" + part + separator + b"MIME]"  # type: ignore[unreachable]
+            elif self.empty:
+                base += b"[" + part + b"]"
+            if self.partialBegin is not None:
+                base += b"<%d.%d>" % (self.partialBegin, self.partialLength)  # type: ignore[unreachable]
+            return base
+
+    class BodyStructure:
+        type = "bodystructure"
+        __str__ = lambda self: "bodystructure"
+
+    # These three aren't top-level, they don't need type indicators
+    class Header:
+        negate = False
+        fields = None
+        part = None
+
+        def __str__(self) -> str:
+            return self.__bytes__().decode("ascii")
+
+        def __bytes__(self) -> bytes:
+            base = b"HEADER"
+            if self.fields:
+                base += b".FIELDS"  # type: ignore[unreachable]
+                if self.negate:
+                    base += b".NOT"
+                fields = []
+                for f in self.fields:
+                    f = f.title()
+                    if _needsQuote(f):
+                        f = _quote(f)
+                    fields.append(f)
+                base += b" (" + b" ".join(fields) + b")"
+            if self.part:
+                # TODO: _FetchParser never assigns Header.part - dead
+                # code?
+                base = b".".join([(x + 1).__bytes__() for x in self.part]) + b"." + base  # type: ignore[unreachable]
+            return base
+
+    class Text:
+        pass
+
+    class MIME:
+        pass
+
+    parts = None
+
+    _simple_fetch_att = [
+        (b"envelope", Envelope),
+        (b"flags", Flags),
+        (b"internaldate", InternalDate),
+        (b"rfc822.header", RFC822Header),
+        (b"rfc822.text", RFC822Text),
+        (b"rfc822.size", RFC822Size),
+        (b"rfc822", RFC822),
+        (b"uid", UID),
+        (b"bodystructure", BodyStructure),
+    ]
+
+    def __init__(self):
+        self.state = ["initial"]
+        self.result = []
+        self.remaining = b""
+
+    def parseString(self, s):
+        s = self.remaining + s
+        try:
+            while s or self.state:
+                if not self.state:
+                    raise IllegalClientResponse("Invalid Argument")
+                # print 'Entering state_' + self.state[-1] + ' with', repr(s)
+                state = self.state.pop()
+                try:
+                    used = getattr(self, "state_" + state)(s)
+                except BaseException:
+                    self.state.append(state)
+                    raise
+                else:
+                    # print state, 'consumed', repr(s[:used])
+                    s = s[used:]
+        finally:
+            self.remaining = s
+
+    def state_initial(self, s):
+        # In the initial state, the literals "ALL", "FULL", and "FAST"
+        # are accepted, as is a ( indicating the beginning of a fetch_att
+        # token, as is the beginning of a fetch_att token.
+        if s == b"":
+            return 0
+
+        l = s.lower()
+        if l.startswith(b"all"):
+            self.result.extend(
+                (self.Flags(), self.InternalDate(), self.RFC822Size(), self.Envelope())
+            )
+            return 3
+        if l.startswith(b"full"):
+            self.result.extend(
+                (
+                    self.Flags(),
+                    self.InternalDate(),
+                    self.RFC822Size(),
+                    self.Envelope(),
+                    self.Body(),
+                )
+            )
+            return 4
+        if l.startswith(b"fast"):
+            self.result.extend(
+                (
+                    self.Flags(),
+                    self.InternalDate(),
+                    self.RFC822Size(),
+                )
+            )
+            return 4
+
+        if l.startswith(b"("):
+            self.state.extend(("close_paren", "maybe_fetch_att", "fetch_att"))
+            return 1
+
+        self.state.append("fetch_att")
+        return 0
+
+    def state_close_paren(self, s):
+        if s.startswith(b")"):
+            return 1
+        # TODO: does maybe_fetch_att's startswith(b')') make this dead
+        # code?
+        raise Exception("Missing )")
+
+    def state_whitespace(self, s):
+        # Eat up all the leading whitespace
+        if not s or not s[0:1].isspace():
+            raise Exception("Whitespace expected, none found")
+        i = 0
+        for i in range(len(s)):
+            if not s[i : i + 1].isspace():
+                break
+        return i
+
+    def state_maybe_fetch_att(self, s):
+        if not s.startswith(b")"):
+            self.state.extend(("maybe_fetch_att", "fetch_att", "whitespace"))
+        return 0
+
+    def state_fetch_att(self, s):
+        # Allowed fetch_att tokens are "ENVELOPE", "FLAGS", "INTERNALDATE",
+        # "RFC822", "RFC822.HEADER", "RFC822.SIZE", "RFC822.TEXT", "BODY",
+        # "BODYSTRUCTURE", "UID",
+        # "BODY [".PEEK"] [<section>] ["<" <number> "." <nz_number> ">"]
+
+        l = s.lower()
+        for name, cls in self._simple_fetch_att:
+            if l.startswith(name):
+                self.result.append(cls())
+                return len(name)
+
+        b = self.Body()
+        if l.startswith(b"body.peek"):
+            b.peek = True
+            used = 9
+        elif l.startswith(b"body"):
+            used = 4
+        else:
+            raise Exception(f"Nothing recognized in fetch_att: {l}")
+
+        self.pending_body = b
+        self.state.extend(("got_body", "maybe_partial", "maybe_section"))
+        return used
+
+    def state_got_body(self, s):
+        self.result.append(self.pending_body)
+        del self.pending_body
+        return 0
+
+    def state_maybe_section(self, s):
+        if not s.startswith(b"["):
+            return 0
+
+        self.state.extend(("section", "part_number"))
+        return 1
+
+    _partExpr = re.compile(rb"(\d+(?:\.\d+)*)\.?")
+
+    def state_part_number(self, s):
+        m = self._partExpr.match(s)
+        if m is not None:
+            self.parts = [int(p) - 1 for p in m.groups()[0].split(b".")]
+            return m.end()
+        else:
+            self.parts = []
+            return 0
+
+    def state_section(self, s):
+        # Grab "HEADER]" or "HEADER.FIELDS (Header list)]" or
+        # "HEADER.FIELDS.NOT (Header list)]" or "TEXT]" or "MIME]" or
+        # just "]".
+
+        l = s.lower()
+        used = 0
+        if l.startswith(b"]"):
+            self.pending_body.empty = True
+            used += 1
+        elif l.startswith(b"header]"):
+            h = self.pending_body.header = self.Header()
+            h.negate = True
+            h.fields = ()
+            used += 7
+        elif l.startswith(b"text]"):
+            self.pending_body.text = self.Text()
+            used += 5
+        elif l.startswith(b"mime]"):
+            self.pending_body.mime = self.MIME()
+            used += 5
+        else:
+            h = self.Header()
+            if l.startswith(b"header.fields.not"):
+                h.negate = True
+                used += 17
+            elif l.startswith(b"header.fields"):
+                used += 13
+            else:
+                raise Exception(f"Unhandled section contents: {l!r}")
+
+            self.pending_body.header = h
+            self.state.extend(("finish_section", "header_list", "whitespace"))
+        self.pending_body.part = tuple(self.parts)
+        self.parts = None
+        return used
+
+    def state_finish_section(self, s):
+        if not s.startswith(b"]"):
+            raise Exception("section must end with ]")
+        return 1
+
+    def state_header_list(self, s):
+        if not s.startswith(b"("):
+            raise Exception("Header list must begin with (")
+        end = s.find(b")")
+        if end == -1:
+            raise Exception("Header list must end with )")
+
+        headers = s[1:end].split()
+        self.pending_body.header.fields = [h.upper() for h in headers]
+        return end + 1
+
+    def state_maybe_partial(self, s):
+        # Grab <number.number> or nothing at all
+        if not s.startswith(b"<"):
+            return 0
+        end = s.find(b">")
+        if end == -1:
+            raise Exception("Found < but not >")
+
+        partial = s[1:end]
+        parts = partial.split(b".", 1)
+        if len(parts) != 2:
+            raise Exception(
+                "Partial specification did not include two .-delimited integers"
+            )
+        begin, length = map(int, parts)
+        self.pending_body.partialBegin = begin
+        self.pending_body.partialLength = length
+
+        return end + 1
+
+
+class FileProducer:
+    CHUNK_SIZE = 2**2**2**2
+
+    firstWrite = True
+
+    def __init__(self, f):
+        self.f = f
+
+    def beginProducing(self, consumer):
+        self.consumer = consumer
+        self.produce = consumer.write
+        d = self._onDone = defer.Deferred()
+        self.consumer.registerProducer(self, False)
+        return d
+
+    def resumeProducing(self):
+        b = b""
+        if self.firstWrite:
+            b = b"{%d}\r\n" % (self._size(),)
+            self.firstWrite = False
+        if not self.f:
+            return
+        b = b + self.f.read(self.CHUNK_SIZE)
+        if not b:
+            self.consumer.unregisterProducer()
+            self._onDone.callback(self)
+            self._onDone = self.f = self.consumer = None
+        else:
+            self.produce(b)
+
+    def pauseProducing(self):
+        """
+        Pause the producer.  This does nothing.
+        """
+
+    def stopProducing(self):
+        """
+        Stop the producer.  This does nothing.
+        """
+
+    def _size(self):
+        b = self.f.tell()
+        self.f.seek(0, 2)
+        e = self.f.tell()
+        self.f.seek(b, 0)
+        return e - b
+
+
+def parseTime(s):
+    # XXX - This may require localization :(
+    months = [
+        "jan",
+        "feb",
+        "mar",
+        "apr",
+        "may",
+        "jun",
+        "jul",
+        "aug",
+        "sep",
+        "oct",
+        "nov",
+        "dec",
+        "january",
+        "february",
+        "march",
+        "april",
+        "may",
+        "june",
+        "july",
+        "august",
+        "september",
+        "october",
+        "november",
+        "december",
+    ]
+    expr = {
+        "day": r"(?P<day>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
+        "mon": r"(?P<mon>\w+)",
+        "year": r"(?P<year>\d\d\d\d)",
+    }
+    m = re.match("%(day)s-%(mon)s-%(year)s" % expr, s)
+    if not m:
+        raise ValueError(f"Cannot parse time string {s!r}")
+    d = m.groupdict()
+    try:
+        d["mon"] = 1 + (months.index(d["mon"].lower()) % 12)
+        d["year"] = int(d["year"])
+        d["day"] = int(d["day"])
+    except ValueError:
+        raise ValueError(f"Cannot parse time string {s!r}")
+    else:
+        return time.struct_time((d["year"], d["mon"], d["day"], 0, 0, 0, -1, -1, -1))
+
+
+# we need to cast Python >=3.3 memoryview to chars (from unsigned bytes), but
+# cast is absent in previous versions: thus, the lambda returns the
+# memoryview instance while ignoring the format
+memory_cast = getattr(memoryview, "cast", lambda *x: x[0])
+
+
+def modified_base64(s):
+    s_utf7 = s.encode("utf-7")
+    return s_utf7[1:-1].replace(b"/", b",")
+
+
+def modified_unbase64(s):
+    s_utf7 = b"+" + s.replace(b",", b"/") + b"-"
+    return s_utf7.decode("utf-7")
+
+
+def encoder(s, errors=None):
+    """
+    Encode the given C{unicode} string using the IMAP4 specific variation of
+    UTF-7.
+
+    @type s: C{unicode}
+    @param s: The text to encode.
+
+    @param errors: Policy for handling encoding errors.  Currently ignored.
+
+    @return: L{tuple} of a L{str} giving the encoded bytes and an L{int}
+        giving the number of code units consumed from the input.
+    """
+    r = bytearray()
+    _in = []
+    valid_chars = set(map(chr, range(0x20, 0x7F))) - {"&"}
+    for c in s:
+        if c in valid_chars:
+            if _in:
+                r += b"&" + modified_base64("".join(_in)) + b"-"
+                del _in[:]
+            r.append(ord(c))
+        elif c == "&":
+            if _in:
+                r += b"&" + modified_base64("".join(_in)) + b"-"
+                del _in[:]
+            r += b"&-"
+        else:
+            _in.append(c)
+    if _in:
+        r.extend(b"&" + modified_base64("".join(_in)) + b"-")
+    return (bytes(r), len(s))
+
+
+def decoder(s, errors=None):
+    """
+    Decode the given L{str} using the IMAP4 specific variation of UTF-7.
+
+    @type s: L{str}
+    @param s: The bytes to decode.
+
+    @param errors: Policy for handling decoding errors.  Currently ignored.
+
+    @return: a L{tuple} of a C{unicode} string giving the text which was
+        decoded and an L{int} giving the number of bytes consumed from the
+        input.
+    """
+    r = []
+    decode = []
+    s = memory_cast(memoryview(s), "c")
+    for c in s:
+        if c == b"&" and not decode:
+            decode.append(b"&")
+        elif c == b"-" and decode:
+            if len(decode) == 1:
+                r.append("&")
+            else:
+                r.append(modified_unbase64(b"".join(decode[1:])))
+            decode = []
+        elif decode:
+            decode.append(c)
+        else:
+            r.append(c.decode())
+    if decode:
+        r.append(modified_unbase64(b"".join(decode[1:])))
+    return ("".join(r), len(s))
+
+
+class StreamReader(codecs.StreamReader):
+    def decode(self, s, errors="strict"):
+        return decoder(s)
+
+
+class StreamWriter(codecs.StreamWriter):
+    def encode(self, s, errors="strict"):
+        return encoder(s)
+
+
+_codecInfo = codecs.CodecInfo(encoder, decoder, StreamReader, StreamWriter)
+
+
+def imap4_utf_7(name):
+    # In Python 3.9, codecs.lookup() was changed to normalize the codec name
+    # in the same way as encodings.normalize_encoding().  The docstring
+    # for encodings.normalize_encoding() describes how the codec name is
+    # normalized.  We need to replace '-' with '_' to be compatible with
+    # older Python versions.
+    #  See:  https://bugs.python.org/issue37751
+    #        https://github.com/python/cpython/pull/17997
+    if name.replace("-", "_") == "imap4_utf_7":
+        return _codecInfo
+
+
+codecs.register(imap4_utf_7)
+
+__all__ = [
+    # Protocol classes
+    "IMAP4Server",
+    "IMAP4Client",
+    # Interfaces
+    "IMailboxListener",
+    "IClientAuthentication",
+    "IAccount",
+    "IMailbox",
+    "INamespacePresenter",
+    "ICloseableMailbox",
+    "IMailboxInfo",
+    "IMessage",
+    "IMessageCopier",
+    "IMessageFile",
+    "ISearchableMailbox",
+    "IMessagePart",
+    # Exceptions
+    "IMAP4Exception",
+    "IllegalClientResponse",
+    "IllegalOperation",
+    "IllegalMailboxEncoding",
+    "UnhandledResponse",
+    "NegativeResponse",
+    "NoSupportedAuthentication",
+    "IllegalServerResponse",
+    "IllegalIdentifierError",
+    "IllegalQueryError",
+    "MismatchedNesting",
+    "MismatchedQuoting",
+    "MailboxException",
+    "MailboxCollision",
+    "NoSuchMailbox",
+    "ReadOnlyMailbox",
+    # Auth objects
+    "CramMD5ClientAuthenticator",
+    "PLAINAuthenticator",
+    "LOGINAuthenticator",
+    "PLAINCredentials",
+    "LOGINCredentials",
+    # Simple query interface
+    "Query",
+    "Not",
+    "Or",
+    # Miscellaneous
+    "MemoryAccount",
+    "statusRequestHelper",
+]
diff --git a/contrib/python/Twisted/py3/twisted/mail/interfaces.py b/contrib/python/Twisted/py3/twisted/mail/interfaces.py
new file mode 100644
index 00000000000..dd87d35a63c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/interfaces.py
@@ -0,0 +1,1050 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Interfaces for L{twisted.mail}.
+
+@since: 16.5
+"""
+
+
+from zope.interface import Interface
+
+
+class IChallengeResponse(Interface):
+    """
+    An C{IMAPrev4} authorization challenge mechanism.
+    """
+
+    def getChallenge():
+        """
+        Return a client challenge.
+
+        @return: A challenge.
+        @rtype: L{bytes}
+        """
+
+    def setResponse(response):
+        """
+        Extract a username and possibly a password from a response and
+        assign them to C{username} and C{password} instance variables.
+
+        @param response: A decoded response.
+        @type response: L{bytes}
+
+        @see: L{credentials.IUsernamePassword} or
+            L{credentials.IUsernameHashedPassword}
+        """
+
+    def moreChallenges():
+        """
+        Are there more challenges than just the first?  If so, callers
+        should challenge clients with the result of L{getChallenge},
+        and check their response with L{setResponse} in a loop until
+        this returns L{False}
+
+        @return: Are there more challenges?
+        @rtype: L{bool}
+        """
+
+
+class IClientAuthentication(Interface):
+    def getName():
+        """
+        Return an identifier associated with this authentication scheme.
+
+        @rtype: L{bytes}
+        """
+
+    def challengeResponse(secret, challenge):
+        """
+        Generate a challenge response string.
+        """
+
+
+class IServerFactoryPOP3(Interface):
+    """
+    An interface for querying capabilities of a POP3 server.
+
+    Any cap_* method may raise L{NotImplementedError} if the particular
+    capability is not supported. If L{cap_EXPIRE()} does not raise
+    L{NotImplementedError}, L{perUserExpiration()} must be implemented,
+    otherwise they are optional. If L{cap_LOGIN_DELAY()} is implemented,
+    L{perUserLoginDelay()} must be implemented, otherwise they are optional.
+
+    @type challengers: L{dict} of L{bytes} -> L{IUsernameHashedPassword
+        <cred.credentials.IUsernameHashedPassword>}
+    @ivar challengers: A mapping of challenger names to
+        L{IUsernameHashedPassword <cred.credentials.IUsernameHashedPassword>}
+        provider.
+    """
+
+    def cap_IMPLEMENTATION():
+        """
+        Return a string describing the POP3 server implementation.
+
+        @rtype: L{bytes}
+        @return: Server implementation information.
+        """
+
+    def cap_EXPIRE():
+        """
+        Return the minimum number of days messages are retained.
+
+        @rtype: L{int} or L{None}
+        @return: The minimum number of days messages are retained or none, if
+            the server never deletes messages.
+        """
+
+    def perUserExpiration():
+        """
+        Indicate whether the message expiration policy differs per user.
+
+        @rtype: L{bool}
+        @return: C{True} when the message expiration policy differs per user,
+            C{False} otherwise.
+        """
+
+    def cap_LOGIN_DELAY():
+        """
+        Return the minimum number of seconds between client logins.
+
+        @rtype: L{int}
+        @return: The minimum number of seconds between client logins.
+        """
+
+    def perUserLoginDelay():
+        """
+        Indicate whether the login delay period differs per user.
+
+        @rtype: L{bool}
+        @return: C{True} when the login delay differs per user, C{False}
+            otherwise.
+        """
+
+
+class IMailboxPOP3(Interface):
+    """
+    An interface for mailbox access.
+
+    Message indices are 0-based.
+
+    @type loginDelay: L{int}
+    @ivar loginDelay: The number of seconds between allowed logins for the
+        user associated with this mailbox.
+
+    @type messageExpiration: L{int}
+    @ivar messageExpiration: The number of days messages in this mailbox will
+        remain on the server before being deleted.
+    """
+
+    def listMessages(index=None):
+        """
+        Retrieve the size of a message, or, if none is specified, the size of
+        each message in the mailbox.
+
+        @type index: L{int} or L{None}
+        @param index: The 0-based index of the message.
+
+        @rtype: L{int}, sequence of L{int}, or L{Deferred <defer.Deferred>}
+        @return: The number of octets in the specified message, or, if an
+            index is not specified, a sequence of the number of octets for
+            all messages in the mailbox or a deferred which fires with
+            one of those. Any value which corresponds to a deleted message
+            is set to 0.
+
+        @raise ValueError or IndexError: When the index does not correspond to
+            a message in the mailbox.  The use of ValueError is preferred.
+        """
+
+    def getMessage(index):
+        """
+        Retrieve a file containing the contents of a message.
+
+        @type index: L{int}
+        @param index: The 0-based index of a message.
+
+        @rtype: file-like object
+        @return: A file containing the message.
+
+        @raise ValueError or IndexError: When the index does not correspond to
+            a message in the mailbox.  The use of ValueError is preferred.
+        """
+
+    def getUidl(index):
+        """
+        Get a unique identifier for a message.
+
+        @type index: L{int}
+        @param index: The 0-based index of a message.
+
+        @rtype: L{bytes}
+        @return: A string of printable characters uniquely identifying the
+            message for all time.
+
+        @raise ValueError or IndexError: When the index does not correspond to
+            a message in the mailbox.  The use of ValueError is preferred.
+        """
+
+    def deleteMessage(index):
+        """
+        Mark a message for deletion.
+
+        This must not change the number of messages in this mailbox.  Further
+        requests for the size of the deleted message should return 0.  Further
+        requests for the message itself may raise an exception.
+
+        @type index: L{int}
+        @param index: The 0-based index of a message.
+
+        @raise ValueError or IndexError: When the index does not correspond to
+            a message in the mailbox.  The use of ValueError is preferred.
+        """
+
+    def undeleteMessages():
+        """
+        Undelete all messages marked for deletion.
+
+        Any message which can be undeleted should be returned to its original
+        position in the message sequence and retain its original UID.
+        """
+
+    def sync():
+        """
+        Discard the contents of any message marked for deletion.
+        """
+
+
+class IDomain(Interface):
+    """
+    An interface for email domains.
+    """
+
+    def exists(user):
+        """
+        Check whether a user exists in this domain.
+
+        @type user: L{User}
+        @param user: A user.
+
+        @rtype: no-argument callable which returns L{IMessageSMTP} provider
+        @return: A function which takes no arguments and returns a message
+            receiver for the user.
+
+        @raise SMTPBadRcpt: When the given user does not exist in this domain.
+        """
+
+    def addUser(user, password):
+        """
+        Add a user to this domain.
+
+        @type user: L{bytes}
+        @param user: A username.
+
+        @type password: L{bytes}
+        @param password: A password.
+        """
+
+    def getCredentialsCheckers():
+        """
+        Return credentials checkers for this domain.
+
+        @rtype: L{list} of L{ICredentialsChecker
+            <twisted.cred.checkers.ICredentialsChecker>} provider
+        @return: Credentials checkers for this domain.
+        """
+
+
+class IAlias(Interface):
+    """
+    An interface for aliases.
+    """
+
+    def createMessageReceiver():
+        """
+        Create a message receiver.
+
+        @rtype: L{IMessageSMTP} provider
+        @return: A message receiver.
+        """
+
+
+class IAliasableDomain(IDomain):
+    """
+    An interface for email domains which can be aliased to other domains.
+    """
+
+    def setAliasGroup(aliases):
+        """
+        Set the group of defined aliases for this domain.
+
+        @type aliases: L{dict} of L{bytes} -> L{IAlias} provider
+        @param aliases: A mapping of domain name to alias.
+        """
+
+    def exists(user, memo=None):
+        """
+        Check whether a user exists in this domain or an alias of it.
+
+        @type user: L{User}
+        @param user: A user.
+
+        @type memo: L{None} or L{dict} of
+            L{AliasBase <twisted.mail.alias.AliasBase>}
+        @param memo: A record of the addresses already considered while
+            resolving aliases. The default value should be used by all external
+            code.
+
+        @rtype: no-argument callable which returns L{IMessageSMTP} provider
+        @return: A function which takes no arguments and returns a message
+            receiver for the user.
+
+        @raise SMTPBadRcpt: When the given user does not exist in this domain
+            or an alias of it.
+        """
+
+
+class IMessageDelivery(Interface):
+    def receivedHeader(helo, origin, recipients):
+        """
+        Generate the Received header for a message.
+
+        @type helo: 2-L{tuple} of L{bytes} and L{bytes}.
+        @param helo: The argument to the HELO command and the client's IP
+        address.
+
+        @type origin: L{Address}
+        @param origin: The address the message is from
+
+        @type recipients: L{list} of L{User}
+        @param recipients: A list of the addresses for which this message
+        is bound.
+
+        @rtype: L{bytes}
+        @return: The full C{"Received"} header string.
+        """
+
+    def validateTo(user):
+        """
+        Validate the address for which the message is destined.
+
+        @type user: L{User}
+        @param user: The address to validate.
+
+        @rtype: no-argument callable
+        @return: A L{Deferred} which becomes, or a callable which takes no
+            arguments and returns an object implementing L{IMessageSMTP}. This
+            will be called and the returned object used to deliver the message
+            when it arrives.
+
+        @raise SMTPBadRcpt: Raised if messages to the address are not to be
+            accepted.
+        """
+
+    def validateFrom(helo, origin):
+        """
+        Validate the address from which the message originates.
+
+        @type helo: 2-L{tuple} of L{bytes} and L{bytes}.
+        @param helo: The argument to the HELO command and the client's IP
+        address.
+
+        @type origin: L{Address}
+        @param origin: The address the message is from
+
+        @rtype: L{Deferred} or L{Address}
+        @return: C{origin} or a L{Deferred} whose callback will be
+        passed C{origin}.
+
+        @raise SMTPBadSender: Raised of messages from this address are
+        not to be accepted.
+        """
+
+
+class IMessageDeliveryFactory(Interface):
+    """
+    An alternate interface to implement for handling message delivery.
+
+    It is useful to implement this interface instead of L{IMessageDelivery}
+    directly because it allows the implementor to distinguish between different
+    messages delivery over the same connection. This can be used to optimize
+    delivery of a single message to multiple recipients, something which cannot
+    be done by L{IMessageDelivery} implementors due to their lack of
+    information.
+    """
+
+    def getMessageDelivery():
+        """
+        Return an L{IMessageDelivery} object.
+
+        This will be called once per message.
+        """
+
+
+class IMessageSMTP(Interface):
+    """
+    Interface definition for messages that can be sent via SMTP.
+    """
+
+    def lineReceived(line):
+        """
+        Handle another line.
+        """
+
+    def eomReceived():
+        """
+        Handle end of message.
+
+        return a deferred. The deferred should be called with either:
+        callback(string) or errback(error)
+
+        @rtype: L{Deferred}
+        """
+
+    def connectionLost():
+        """
+        Handle message truncated.
+
+        semantics should be to discard the message
+        """
+
+
+class IMessageIMAPPart(Interface):
+    def getHeaders(negate, *names):
+        """
+        Retrieve a group of message headers.
+
+        @type names: L{tuple} of L{str}
+        @param names: The names of the headers to retrieve or omit.
+
+        @type negate: L{bool}
+        @param negate: If True, indicates that the headers listed in C{names}
+            should be omitted from the return value, rather than included.
+
+        @rtype: L{dict}
+        @return: A mapping of header field names to header field values
+        """
+
+    def getBodyFile():
+        """
+        Retrieve a file object containing only the body of this message.
+        """
+
+    def getSize():
+        """
+        Retrieve the total size, in octets, of this message.
+
+        @rtype: L{int}
+        """
+
+    def isMultipart():
+        """
+        Indicate whether this message has subparts.
+
+        @rtype: L{bool}
+        """
+
+    def getSubPart(part):
+        """
+        Retrieve a MIME sub-message
+
+        @type part: L{int}
+        @param part: The number of the part to retrieve, indexed from 0.
+
+        @raise IndexError: Raised if the specified part does not exist.
+        @raise TypeError: Raised if this message is not multipart.
+
+        @rtype: Any object implementing L{IMessageIMAPPart}.
+        @return: The specified sub-part.
+        """
+
+
+class IMessageIMAP(IMessageIMAPPart):
+    def getUID():
+        """
+        Retrieve the unique identifier associated with this message.
+        """
+
+    def getFlags():
+        """
+        Retrieve the flags associated with this message.
+
+        @rtype: C{iterable}
+        @return: The flags, represented as strings.
+        """
+
+    def getInternalDate():
+        """
+        Retrieve the date internally associated with this message.
+
+        @rtype: L{bytes}
+        @return: An RFC822-formatted date string.
+        """
+
+
+class IMessageIMAPFile(Interface):
+    """
+    Optional message interface for representing messages as files.
+
+    If provided by message objects, this interface will be used instead the
+    more complex MIME-based interface.
+    """
+
+    def open():
+        """
+        Return a file-like object opened for reading.
+
+        Reading from the returned file will return all the bytes of which this
+        message consists.
+        """
+
+
+class ISearchableIMAPMailbox(Interface):
+    def search(query, uid):
+        """
+        Search for messages that meet the given query criteria.
+
+        If this interface is not implemented by the mailbox,
+        L{IMailboxIMAP.fetch} and various methods of L{IMessageIMAP} will be
+        used instead.
+
+        Implementations which wish to offer better performance than the default
+        implementation should implement this interface.
+
+        @type query: L{list}
+        @param query: The search criteria
+
+        @type uid: L{bool}
+        @param uid: If true, the IDs specified in the query are UIDs; otherwise
+            they are message sequence IDs.
+
+        @rtype: L{list} or L{Deferred}
+        @return: A list of message sequence numbers or message UIDs which match
+            the search criteria or a L{Deferred} whose callback will be invoked
+            with such a list.
+
+        @raise IllegalQueryError: Raised when query is not valid.
+        """
+
+
+class IMailboxIMAPListener(Interface):
+    """
+    Interface for objects interested in mailbox events
+    """
+
+    def modeChanged(writeable):
+        """
+        Indicates that the write status of a mailbox has changed.
+
+        @type writeable: L{bool}
+        @param writeable: A true value if write is now allowed, false
+            otherwise.
+        """
+
+    def flagsChanged(newFlags):
+        """
+        Indicates that the flags of one or more messages have changed.
+
+        @type newFlags: L{dict}
+        @param newFlags: A mapping of message identifiers to tuples of flags
+            now set on that message.
+        """
+
+    def newMessages(exists, recent):
+        """
+        Indicates that the number of messages in a mailbox has changed.
+
+        @type exists: L{int} or L{None}
+        @param exists: The total number of messages now in this mailbox. If the
+            total number of messages has not changed, this should be L{None}.
+
+        @type recent: L{int}
+        @param recent: The number of messages now flagged C{\\Recent}. If the
+            number of recent messages has not changed, this should be L{None}.
+        """
+
+
+class IMessageIMAPCopier(Interface):
+    def copy(messageObject):
+        """
+        Copy the given message object into this mailbox.
+
+        The message object will be one which was previously returned by
+        L{IMailboxIMAP.fetch}.
+
+        Implementations which wish to offer better performance than the default
+        implementation should implement this interface.
+
+        If this interface is not implemented by the mailbox,
+        L{IMailboxIMAP.addMessage} will be used instead.
+
+        @rtype: L{Deferred} or L{int}
+        @return: Either the UID of the message or a Deferred which fires with
+            the UID when the copy finishes.
+        """
+
+
+class IMailboxIMAPInfo(Interface):
+    """
+    Interface specifying only the methods required for C{listMailboxes}.
+
+    Implementations can return objects implementing only these methods for
+    return to C{listMailboxes} if it can allow them to operate more
+    efficiently.
+    """
+
+    def getFlags():
+        """
+        Return the flags defined in this mailbox
+
+        Flags with the \\ prefix are reserved for use as system flags.
+
+        @rtype: L{list} of L{str}
+        @return: A list of the flags that can be set on messages in this
+            mailbox.
+        """
+
+    def getHierarchicalDelimiter():
+        """
+        Get the character which delimits namespaces for in this mailbox.
+
+        @rtype: L{bytes}
+        """
+
+
+class IMailboxIMAP(IMailboxIMAPInfo):
+    def getUIDValidity():
+        """
+        Return the unique validity identifier for this mailbox.
+
+        @rtype: L{int}
+        """
+
+    def getUIDNext():
+        """
+        Return the likely UID for the next message added to this mailbox.
+
+        @rtype: L{int}
+        """
+
+    def getUID(message):
+        """
+        Return the UID of a message in the mailbox
+
+        @type message: L{int}
+        @param message: The message sequence number
+
+        @rtype: L{int}
+        @return: The UID of the message.
+        """
+
+    def getMessageCount():
+        """
+        Return the number of messages in this mailbox.
+
+        @rtype: L{int}
+        """
+
+    def getRecentCount():
+        """
+        Return the number of messages with the 'Recent' flag.
+
+        @rtype: L{int}
+        """
+
+    def getUnseenCount():
+        """
+        Return the number of messages with the 'Unseen' flag.
+
+        @rtype: L{int}
+        """
+
+    def isWriteable():
+        """
+        Get the read/write status of the mailbox.
+
+        @rtype: L{int}
+        @return: A true value if write permission is allowed, a false value
+            otherwise.
+        """
+
+    def destroy():
+        """
+        Called before this mailbox is deleted, permanently.
+
+        If necessary, all resources held by this mailbox should be cleaned up
+        here. This function _must_ set the \\Noselect flag on this mailbox.
+        """
+
+    def requestStatus(names):
+        """
+        Return status information about this mailbox.
+
+        Mailboxes which do not intend to do any special processing to generate
+        the return value, C{statusRequestHelper} can be used to build the
+        dictionary by calling the other interface methods which return the data
+        for each name.
+
+        @type names: Any iterable
+        @param names: The status names to return information regarding. The
+            possible values for each name are: MESSAGES, RECENT, UIDNEXT,
+            UIDVALIDITY, UNSEEN.
+
+        @rtype: L{dict} or L{Deferred}
+        @return: A dictionary containing status information about the requested
+            names is returned. If the process of looking this information up
+            would be costly, a deferred whose callback will eventually be
+            passed this dictionary is returned instead.
+        """
+
+    def addListener(listener):
+        """
+        Add a mailbox change listener
+
+        @type listener: Any object which implements C{IMailboxIMAPListener}
+        @param listener: An object to add to the set of those which will be
+            notified when the contents of this mailbox change.
+        """
+
+    def removeListener(listener):
+        """
+        Remove a mailbox change listener
+
+        @type listener: Any object previously added to and not removed from
+            this mailbox as a listener.
+        @param listener: The object to remove from the set of listeners.
+
+        @raise ValueError: Raised when the given object is not a listener for
+            this mailbox.
+        """
+
+    def addMessage(message, flags, date):
+        """
+        Add the given message to this mailbox.
+
+        @type message: A file-like object
+        @param message: The RFC822 formatted message
+
+        @type flags: Any iterable of L{bytes}
+        @param flags: The flags to associate with this message
+
+        @type date: L{bytes}
+        @param date: If specified, the date to associate with this message.
+
+        @rtype: L{Deferred}
+        @return: A deferred whose callback is invoked with the message id if
+            the message is added successfully and whose errback is invoked
+            otherwise.
+
+        @raise ReadOnlyMailbox: Raised if this Mailbox is not open for
+            read-write.
+        """
+
+    def expunge():
+        """
+        Remove all messages flagged \\Deleted.
+
+        @rtype: L{list} or L{Deferred}
+        @return: The list of message sequence numbers which were deleted, or a
+            L{Deferred} whose callback will be invoked with such a list.
+
+        @raise ReadOnlyMailbox: Raised if this Mailbox is not open for
+            read-write.
+        """
+
+    def fetch(messages, uid):
+        """
+        Retrieve one or more messages.
+
+        @type messages: C{MessageSet}
+        @param messages: The identifiers of messages to retrieve information
+            about
+
+        @type uid: L{bool}
+        @param uid: If true, the IDs specified in the query are UIDs; otherwise
+            they are message sequence IDs.
+
+        @rtype: Any iterable of two-tuples of message sequence numbers and
+            implementors of C{IMessageIMAP}.
+        """
+
+    def store(messages, flags, mode, uid):
+        """
+        Set the flags of one or more messages.
+
+        @type messages: A MessageSet object with the list of messages requested
+        @param messages: The identifiers of the messages to set the flags of.
+
+        @type flags: sequence of L{str}
+        @param flags: The flags to set, unset, or add.
+
+        @type mode: -1, 0, or 1
+        @param mode: If mode is -1, these flags should be removed from the
+            specified messages. If mode is 1, these flags should be added to
+            the specified messages. If mode is 0, all existing flags should be
+            cleared and these flags should be added.
+
+        @type uid: L{bool}
+        @param uid: If true, the IDs specified in the query are UIDs; otherwise
+            they are message sequence IDs.
+
+        @rtype: L{dict} or L{Deferred}
+        @return: A L{dict} mapping message sequence numbers to sequences of
+            L{str} representing the flags set on the message after this
+            operation has been performed, or a L{Deferred} whose callback will
+            be invoked with such a L{dict}.
+
+        @raise ReadOnlyMailbox: Raised if this mailbox is not open for
+            read-write.
+        """
+
+
+class ICloseableMailboxIMAP(Interface):
+    """
+    A supplementary interface for mailboxes which require cleanup on close.
+
+    Implementing this interface is optional. If it is implemented, the protocol
+    code will call the close method defined whenever a mailbox is closed.
+    """
+
+    def close():
+        """
+        Close this mailbox.
+
+        @return: A L{Deferred} which fires when this mailbox has been closed,
+            or None if the mailbox can be closed immediately.
+        """
+
+
+class IAccountIMAP(Interface):
+    """
+    Interface for Account classes
+
+    Implementors of this interface should consider implementing
+    C{INamespacePresenter}.
+    """
+
+    def addMailbox(name, mbox=None):
+        """
+        Add a new mailbox to this account
+
+        @type name: L{bytes}
+        @param name: The name associated with this mailbox. It may not contain
+            multiple hierarchical parts.
+
+        @type mbox: An object implementing C{IMailboxIMAP}
+        @param mbox: The mailbox to associate with this name. If L{None}, a
+            suitable default is created and used.
+
+        @rtype: L{Deferred} or L{bool}
+        @return: A true value if the creation succeeds, or a deferred whose
+            callback will be invoked when the creation succeeds.
+
+        @raise MailboxException: Raised if this mailbox cannot be added for
+            some reason. This may also be raised asynchronously, if a
+            L{Deferred} is returned.
+        """
+
+    def create(pathspec):
+        """
+        Create a new mailbox from the given hierarchical name.
+
+        @type pathspec: L{bytes}
+        @param pathspec: The full hierarchical name of a new mailbox to create.
+            If any of the inferior hierarchical names to this one do not exist,
+            they are created as well.
+
+        @rtype: L{Deferred} or L{bool}
+        @return: A true value if the creation succeeds, or a deferred whose
+            callback will be invoked when the creation succeeds.
+
+        @raise MailboxException: Raised if this mailbox cannot be added. This
+            may also be raised asynchronously, if a L{Deferred} is returned.
+        """
+
+    def select(name, rw=True):
+        """
+        Acquire a mailbox, given its name.
+
+        @type name: L{bytes}
+        @param name: The mailbox to acquire
+
+        @type rw: L{bool}
+        @param rw: If a true value, request a read-write version of this
+            mailbox. If a false value, request a read-only version.
+
+        @rtype: Any object implementing C{IMailboxIMAP} or L{Deferred}
+        @return: The mailbox object, or a L{Deferred} whose callback will be
+            invoked with the mailbox object. None may be returned if the
+            specified mailbox may not be selected for any reason.
+        """
+
+    def delete(name):
+        """
+        Delete the mailbox with the specified name.
+
+        @type name: L{bytes}
+        @param name: The mailbox to delete.
+
+        @rtype: L{Deferred} or L{bool}
+        @return: A true value if the mailbox is successfully deleted, or a
+            L{Deferred} whose callback will be invoked when the deletion
+            completes.
+
+        @raise MailboxException: Raised if this mailbox cannot be deleted. This
+            may also be raised asynchronously, if a L{Deferred} is returned.
+        """
+
+    def rename(oldname, newname):
+        """
+        Rename a mailbox
+
+        @type oldname: L{bytes}
+        @param oldname: The current name of the mailbox to rename.
+
+        @type newname: L{bytes}
+        @param newname: The new name to associate with the mailbox.
+
+        @rtype: L{Deferred} or L{bool}
+        @return: A true value if the mailbox is successfully renamed, or a
+            L{Deferred} whose callback will be invoked when the rename
+            operation is completed.
+
+        @raise MailboxException: Raised if this mailbox cannot be renamed. This
+            may also be raised asynchronously, if a L{Deferred} is returned.
+        """
+
+    def isSubscribed(name):
+        """
+        Check the subscription status of a mailbox
+
+        @type name: L{bytes}
+        @param name: The name of the mailbox to check
+
+        @rtype: L{Deferred} or L{bool}
+        @return: A true value if the given mailbox is currently subscribed to,
+            a false value otherwise. A L{Deferred} may also be returned whose
+            callback will be invoked with one of these values.
+        """
+
+    def subscribe(name):
+        """
+        Subscribe to a mailbox
+
+        @type name: L{bytes}
+        @param name: The name of the mailbox to subscribe to
+
+        @rtype: L{Deferred} or L{bool}
+        @return: A true value if the mailbox is subscribed to successfully, or
+            a Deferred whose callback will be invoked with this value when the
+            subscription is successful.
+
+        @raise MailboxException: Raised if this mailbox cannot be subscribed
+            to. This may also be raised asynchronously, if a L{Deferred} is
+            returned.
+        """
+
+    def unsubscribe(name):
+        """
+        Unsubscribe from a mailbox
+
+        @type name: L{bytes}
+        @param name: The name of the mailbox to unsubscribe from
+
+        @rtype: L{Deferred} or L{bool}
+        @return: A true value if the mailbox is unsubscribed from successfully,
+            or a Deferred whose callback will be invoked with this value when
+            the unsubscription is successful.
+
+        @raise MailboxException: Raised if this mailbox cannot be unsubscribed
+            from. This may also be raised asynchronously, if a L{Deferred} is
+            returned.
+        """
+
+    def listMailboxes(ref, wildcard):
+        """
+        List all the mailboxes that meet a certain criteria
+
+        @type ref: L{bytes}
+        @param ref: The context in which to apply the wildcard
+
+        @type wildcard: L{bytes}
+        @param wildcard: An expression against which to match mailbox names.
+            '*' matches any number of characters in a mailbox name, and '%'
+            matches similarly, but will not match across hierarchical
+            boundaries.
+
+        @rtype: L{list} of L{tuple}
+        @return: A list of C{(mailboxName, mailboxObject)} which meet the given
+            criteria. C{mailboxObject} should implement either
+            C{IMailboxIMAPInfo} or C{IMailboxIMAP}. A Deferred may also be
+            returned.
+        """
+
+
+class INamespacePresenter(Interface):
+    def getPersonalNamespaces():
+        """
+        Report the available personal namespaces.
+
+        Typically there should be only one personal namespace. A common name
+        for it is C{\"\"}, and its hierarchical delimiter is usually C{\"/\"}.
+
+        @rtype: iterable of two-tuples of strings
+        @return: The personal namespaces and their hierarchical delimiters. If
+            no namespaces of this type exist, None should be returned.
+        """
+
+    def getSharedNamespaces():
+        """
+        Report the available shared namespaces.
+
+        Shared namespaces do not belong to any individual user but are usually
+        to one or more of them. Examples of shared namespaces might be
+        C{\"#news\"} for a usenet gateway.
+
+        @rtype: iterable of two-tuples of strings
+        @return: The shared namespaces and their hierarchical delimiters. If no
+            namespaces of this type exist, None should be returned.
+        """
+
+    def getUserNamespaces():
+        """
+        Report the available user namespaces.
+
+        These are namespaces that contain folders belonging to other users
+        access to which this account has been granted.
+
+        @rtype: iterable of two-tuples of strings
+        @return: The user namespaces and their hierarchical delimiters. If no
+            namespaces of this type exist, None should be returned.
+        """
+
+
+__all__ = [
+    # IMAP
+    "IAccountIMAP",
+    "ICloseableMailboxIMAP",
+    "IMailboxIMAP",
+    "IMailboxIMAPInfo",
+    "IMailboxIMAPListener",
+    "IMessageIMAP",
+    "IMessageIMAPCopier",
+    "IMessageIMAPFile",
+    "IMessageIMAPPart",
+    "ISearchableIMAPMailbox",
+    "INamespacePresenter",
+    # SMTP
+    "IMessageDelivery",
+    "IMessageDeliveryFactory",
+    "IMessageSMTP",
+    # Domains and aliases
+    "IDomain",
+    "IAlias",
+    "IAliasableDomain",
+    # POP3
+    "IMailboxPOP3",
+    "IServerFactoryPOP3",
+    # Authentication
+    "IClientAuthentication",
+]
diff --git a/contrib/python/Twisted/py3/twisted/mail/mail.py b/contrib/python/Twisted/py3/twisted/mail/mail.py
new file mode 100644
index 00000000000..2dc405344b6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/mail.py
@@ -0,0 +1,706 @@
+# -*- test-case-name: twisted.mail.test.test_mail -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Mail service support.
+"""
+
+# System imports
+import os
+import warnings
+
+from zope.interface import implementer
+
+from twisted.application import internet, service
+from twisted.cred.portal import Portal
+
+# Twisted imports
+from twisted.internet import defer
+
+# Sibling imports
+from twisted.mail import protocols, smtp
+from twisted.mail.interfaces import IAliasableDomain, IDomain
+from twisted.python import log, util
+
+
+class DomainWithDefaultDict:
+    """
+    A simulated dictionary for mapping domain names to domain objects with
+    a default value for non-existing keys.
+
+    @ivar domains: See L{__init__}
+    @ivar default: See L{__init__}
+    """
+
+    def __init__(self, domains, default):
+        """
+        @type domains: L{dict} of L{bytes} -> L{IDomain} provider
+        @param domains: A mapping of domain name to domain object.
+
+        @type default: L{IDomain} provider
+        @param default: The default domain.
+        """
+        self.domains = domains
+        self.default = default
+
+    def setDefaultDomain(self, domain):
+        """
+        Set the default domain.
+
+        @type domain: L{IDomain} provider
+        @param domain: The default domain.
+        """
+        self.default = domain
+
+    def has_key(self, name):
+        """
+        Test for the presence of a domain name in this dictionary.
+
+        This always returns C{True} because a default value will be returned
+        if the name doesn't exist in this dictionary.
+
+        @type name: L{bytes}
+        @param name: A domain name.
+
+        @rtype: L{bool}
+        @return: C{True} to indicate that the domain name is in this
+            dictionary.
+        """
+        warnings.warn(
+            "twisted.mail.mail.DomainWithDefaultDict.has_key was deprecated "
+            "in Twisted 16.3.0. "
+            "Use the `in` keyword instead.",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        return 1
+
+    @classmethod
+    def fromkeys(klass, keys, value=None):
+        """
+        Create a new L{DomainWithDefaultDict} with the specified keys.
+
+        @type keys: iterable of L{bytes}
+        @param keys: Domain names to serve as keys in the new dictionary.
+
+        @type value: L{None} or L{IDomain} provider
+        @param value: A domain object to serve as the value for all new keys
+            in the dictionary.
+
+        @rtype: L{DomainWithDefaultDict}
+        @return: A new dictionary.
+        """
+        d = klass()
+        for k in keys:
+            d[k] = value
+        return d
+
+    def __contains__(self, name):
+        """
+        Test for the presence of a domain name in this dictionary.
+
+        This always returns C{True} because a default value will be returned
+        if the name doesn't exist in this dictionary.
+
+        @type name: L{bytes}
+        @param name: A domain name.
+
+        @rtype: L{bool}
+        @return: C{True} to indicate that the domain name is in this
+            dictionary.
+        """
+        return 1
+
+    def __getitem__(self, name):
+        """
+        Look up a domain name and, if it is present, return the domain object
+        associated with it.  Otherwise return the default domain.
+
+        @type name: L{bytes}
+        @param name: A domain name.
+
+        @rtype: L{IDomain} provider or L{None}
+        @return: A domain object.
+        """
+        return self.domains.get(name, self.default)
+
+    def __setitem__(self, name, value):
+        """
+        Associate a domain object with a domain name in this dictionary.
+
+        @type name: L{bytes}
+        @param name: A domain name.
+
+        @type value: L{IDomain} provider
+        @param value: A domain object.
+        """
+        self.domains[name] = value
+
+    def __delitem__(self, name):
+        """
+        Delete the entry for a domain name in this dictionary.
+
+        @type name: L{bytes}
+        @param name: A domain name.
+        """
+        del self.domains[name]
+
+    def __iter__(self):
+        """
+        Return an iterator over the domain names in this dictionary.
+
+        @rtype: iterator over L{bytes}
+        @return: An iterator over the domain names.
+        """
+        return iter(self.domains)
+
+    def __len__(self):
+        """
+        Return the number of domains in this dictionary.
+
+        @rtype: L{int}
+        @return: The number of domains in this dictionary.
+        """
+        return len(self.domains)
+
+    def __str__(self) -> str:
+        """
+        Build an informal string representation of this dictionary.
+
+        @rtype: L{bytes}
+        @return: A string containing the mapping of domain names to domain
+            objects.
+        """
+        return f"<DomainWithDefaultDict {self.domains}>"
+
+    def __repr__(self) -> str:
+        """
+        Build an "official" string representation of this dictionary.
+
+        @rtype: L{bytes}
+        @return: A pseudo-executable string describing the underlying domain
+            mapping of this object.
+        """
+        return f"DomainWithDefaultDict({self.domains})"
+
+    def get(self, key, default=None):
+        """
+        Look up a domain name in this dictionary.
+
+        @type key: L{bytes}
+        @param key: A domain name.
+
+        @type default: L{IDomain} provider or L{None}
+        @param default: A domain object to be returned if the domain name is
+            not in this dictionary.
+
+        @rtype: L{IDomain} provider or L{None}
+        @return: The domain object associated with the domain name if it is in
+            this dictionary.  Otherwise, the default value.
+        """
+        return self.domains.get(key, default)
+
+    def copy(self):
+        """
+        Make a copy of this dictionary.
+
+        @rtype: L{DomainWithDefaultDict}
+        @return: A copy of this dictionary.
+        """
+        return DomainWithDefaultDict(self.domains.copy(), self.default)
+
+    def iteritems(self):
+        """
+        Return an iterator over the domain name/domain object pairs in the
+        dictionary.
+
+        Using the returned iterator while adding or deleting entries from the
+        dictionary may result in a L{RuntimeError} or failing to iterate over
+        all the domain name/domain object pairs.
+
+        @rtype: iterator over 2-L{tuple} of (E{1}) L{bytes},
+            (E{2}) L{IDomain} provider or L{None}
+        @return: An iterator over the domain name/domain object pairs.
+        """
+        return self.domains.iteritems()
+
+    def iterkeys(self):
+        """
+        Return an iterator over the domain names in this dictionary.
+
+        Using the returned iterator while adding or deleting entries from the
+        dictionary may result in a L{RuntimeError} or failing to iterate over
+        all the domain names.
+
+        @rtype: iterator over L{bytes}
+        @return: An iterator over the domain names.
+        """
+        return self.domains.iterkeys()
+
+    def itervalues(self):
+        """
+        Return an iterator over the domain objects in this dictionary.
+
+        Using the returned iterator while adding or deleting entries from the
+        dictionary may result in a L{RuntimeError} or failing to iterate over
+        all the domain objects.
+
+        @rtype: iterator over L{IDomain} provider or
+            L{None}
+        @return: An iterator over the domain objects.
+        """
+        return self.domains.itervalues()
+
+    def keys(self):
+        """
+        Return a list of all domain names in this dictionary.
+
+        @rtype: L{list} of L{bytes}
+        @return: The domain names in this dictionary.
+
+        """
+        return self.domains.keys()
+
+    def values(self):
+        """
+        Return a list of all domain objects in this dictionary.
+
+        @rtype: L{list} of L{IDomain} provider or L{None}
+        @return: The domain objects in this dictionary.
+        """
+        return self.domains.values()
+
+    def items(self):
+        """
+        Return a list of all domain name/domain object pairs in this
+        dictionary.
+
+        @rtype: L{list} of 2-L{tuple} of (E{1}) L{bytes}, (E{2}) L{IDomain}
+            provider or L{None}
+        @return: Domain name/domain object pairs in this dictionary.
+        """
+        return self.domains.items()
+
+    def popitem(self):
+        """
+        Remove a random domain name/domain object pair from this dictionary and
+        return it as a tuple.
+
+        @rtype: 2-L{tuple} of (E{1}) L{bytes}, (E{2}) L{IDomain} provider or
+            L{None}
+        @return: A domain name/domain object pair.
+
+        @raise KeyError: When this dictionary is empty.
+        """
+        return self.domains.popitem()
+
+    def update(self, other):
+        """
+        Update this dictionary with domain name/domain object pairs from
+        another dictionary.
+
+        When this dictionary contains a domain name which is in the other
+        dictionary, its value will be overwritten.
+
+        @type other: L{dict} of L{bytes} -> L{IDomain} provider and/or
+            L{bytes} -> L{None}
+        @param other: Another dictionary of domain name/domain object pairs.
+
+        @rtype: L{None}
+        @return: None.
+        """
+        return self.domains.update(other)
+
+    def clear(self):
+        """
+        Remove all items from this dictionary.
+
+        @rtype: L{None}
+        @return: None.
+        """
+        return self.domains.clear()
+
+    def setdefault(self, key, default):
+        """
+        Return the domain object associated with the domain name if it is
+        present in this dictionary. Otherwise, set the value for the
+        domain name to the default and return that value.
+
+        @type key: L{bytes}
+        @param key: A domain name.
+
+        @type default: L{IDomain} provider
+        @param default: A domain object.
+
+        @rtype: L{IDomain} provider or L{None}
+        @return: The domain object associated with the domain name.
+        """
+        return self.domains.setdefault(key, default)
+
+
+@implementer(IDomain)
+class BounceDomain:
+    """
+    A domain with no users.
+
+    This can be used to block off a domain.
+    """
+
+    def exists(self, user):
+        """
+        Raise an exception to indicate that the user does not exist in this
+        domain.
+
+        @type user: L{User}
+        @param user: A user.
+
+        @raise SMTPBadRcpt: When the given user does not exist in this domain.
+        """
+        raise smtp.SMTPBadRcpt(user)
+
+    def willRelay(self, user, protocol):
+        """
+        Indicate that this domain will not relay.
+
+        @type user: L{Address}
+        @param user: The destination address.
+
+        @type protocol: L{Protocol <twisted.internet.protocol.Protocol>}
+        @param protocol: The protocol over which the message to be relayed is
+            being received.
+
+        @rtype: L{bool}
+        @return: C{False}.
+        """
+        return False
+
+    def addUser(self, user, password):
+        """
+        Ignore attempts to add a user to this domain.
+
+        @type user: L{bytes}
+        @param user: A username.
+
+        @type password: L{bytes}
+        @param password: A password.
+        """
+        pass
+
+    def getCredentialsCheckers(self):
+        """
+        Return no credentials checkers for this domain.
+
+        @rtype: L{list}
+        @return: The empty list.
+        """
+        return []
+
+
+@implementer(smtp.IMessage)
+class FileMessage:
+    """
+    A message receiver which delivers a message to a file.
+
+    @ivar fp: See L{__init__}.
+    @ivar name: See L{__init__}.
+    @ivar finalName: See L{__init__}.
+    """
+
+    def __init__(self, fp, name, finalName):
+        """
+        @type fp: file-like object
+        @param fp: The file in which to store the message while it is being
+            received.
+
+        @type name: L{bytes}
+        @param name: The full path name of the temporary file.
+
+        @type finalName: L{bytes}
+        @param finalName: The full path name that should be given to the file
+            holding the message after it has been fully received.
+        """
+        self.fp = fp
+        self.name = name
+        self.finalName = finalName
+
+    def lineReceived(self, line):
+        """
+        Write a received line to the file.
+
+        @type line: L{bytes}
+        @param line: A received line.
+        """
+        self.fp.write(line + b"\n")
+
+    def eomReceived(self):
+        """
+        At the end of message, rename the file holding the message to its
+        final name.
+
+        @rtype: L{Deferred} which successfully results in L{bytes}
+        @return: A deferred which returns the final name of the file.
+        """
+        self.fp.close()
+        os.rename(self.name, self.finalName)
+        return defer.succeed(self.finalName)
+
+    def connectionLost(self):
+        """
+        Delete the file holding the partially received message.
+        """
+        self.fp.close()
+        os.remove(self.name)
+
+
+class MailService(service.MultiService):
+    """
+    An email service.
+
+    @type queue: L{Queue} or L{None}
+    @ivar queue: A queue for outgoing messages.
+
+    @type domains: L{dict} of L{bytes} -> L{IDomain} provider
+    @ivar domains: A mapping of supported domain name to domain object.
+
+    @type portals: L{dict} of L{bytes} -> L{Portal}
+    @ivar portals: A mapping of domain name to authentication portal.
+
+    @type aliases: L{None} or L{dict} of
+        L{bytes} -> L{IAlias} provider
+    @ivar aliases: A mapping of domain name to alias.
+
+    @type smtpPortal: L{Portal}
+    @ivar smtpPortal: A portal for authentication for the SMTP server.
+
+    @type monitor: L{FileMonitoringService}
+    @ivar monitor: A service to monitor changes to files.
+    """
+
+    queue = None
+    domains = None
+    portals = None
+    aliases = None
+    smtpPortal = None
+
+    def __init__(self):
+        """
+        Initialize the mail service.
+        """
+        service.MultiService.__init__(self)
+        # Domains and portals for "client" protocols - POP3, IMAP4, etc
+        self.domains = DomainWithDefaultDict({}, BounceDomain())
+        self.portals = {}
+
+        self.monitor = FileMonitoringService()
+        self.monitor.setServiceParent(self)
+        self.smtpPortal = Portal(self)
+
+    def getPOP3Factory(self):
+        """
+        Create a POP3 protocol factory.
+
+        @rtype: L{POP3Factory}
+        @return: A POP3 protocol factory.
+        """
+        return protocols.POP3Factory(self)
+
+    def getSMTPFactory(self):
+        """
+        Create an SMTP protocol factory.
+
+        @rtype: L{SMTPFactory <protocols.SMTPFactory>}
+        @return: An SMTP protocol factory.
+        """
+        return protocols.SMTPFactory(self, self.smtpPortal)
+
+    def getESMTPFactory(self):
+        """
+        Create an ESMTP protocol factory.
+
+        @rtype: L{ESMTPFactory <protocols.ESMTPFactory>}
+        @return: An ESMTP protocol factory.
+        """
+        return protocols.ESMTPFactory(self, self.smtpPortal)
+
+    def addDomain(self, name, domain):
+        """
+        Add a domain for which the service will accept email.
+
+        @type name: L{bytes}
+        @param name: A domain name.
+
+        @type domain: L{IDomain} provider
+        @param domain: A domain object.
+        """
+        portal = Portal(domain)
+        map(portal.registerChecker, domain.getCredentialsCheckers())
+        self.domains[name] = domain
+        self.portals[name] = portal
+        if self.aliases and IAliasableDomain.providedBy(domain):
+            domain.setAliasGroup(self.aliases)
+
+    def setQueue(self, queue):
+        """
+        Set the queue for outgoing emails.
+
+        @type queue: L{Queue}
+        @param queue: A queue for outgoing messages.
+        """
+        self.queue = queue
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        """
+        Return a message delivery for an authenticated SMTP user.
+
+        @type avatarId: L{bytes}
+        @param avatarId: A string which identifies an authenticated user.
+
+        @type mind: L{None}
+        @param mind: Unused.
+
+        @type interfaces: n-L{tuple} of C{zope.interface.Interface}
+        @param interfaces: A group of interfaces one of which the avatar must
+            support.
+
+        @rtype: 3-L{tuple} of (E{1}) L{IMessageDelivery},
+            (E{2}) L{ESMTPDomainDelivery}, (E{3}) no-argument callable
+        @return: A tuple of the supported interface, a message delivery, and
+            a logout function.
+
+        @raise NotImplementedError: When the given interfaces do not include
+            L{IMessageDelivery}.
+        """
+        if smtp.IMessageDelivery in interfaces:
+            a = protocols.ESMTPDomainDelivery(self, avatarId)
+            return smtp.IMessageDelivery, a, lambda: None
+        raise NotImplementedError()
+
+    def lookupPortal(self, name):
+        """
+        Find the portal for a domain.
+
+        @type name: L{bytes}
+        @param name: A domain name.
+
+        @rtype: L{Portal}
+        @return: A portal.
+        """
+        return self.portals[name]
+
+    def defaultPortal(self):
+        """
+        Return the portal for the default domain.
+
+        The default domain is named ''.
+
+        @rtype: L{Portal}
+        @return: The portal for the default domain.
+        """
+        return self.portals[""]
+
+
+class FileMonitoringService(internet.TimerService):
+    """
+    A service for monitoring changes to files.
+
+    @type files: L{list} of L{list} of (E{1}) L{float}, (E{2}) L{bytes},
+        (E{3}) callable which takes a L{bytes} argument, (E{4}) L{float}
+    @ivar files: Information about files to be monitored.  Each list entry
+        provides the following information for a file: interval in seconds
+        between checks, filename, callback function, time of last modification
+        to the file.
+
+    @type intervals: L{_IntervalDifferentialIterator
+        <twisted.python.util._IntervalDifferentialIterator>}
+    @ivar intervals: Intervals between successive file checks.
+
+    @type _call: L{IDelayedCall <twisted.internet.interfaces.IDelayedCall>}
+        provider
+    @ivar _call: The next scheduled call to check a file.
+
+    @type index: L{int}
+    @ivar index: The index of the next file to be checked.
+    """
+
+    def __init__(self):
+        """
+        Initialize the file monitoring service.
+        """
+        self.files = []
+        self.intervals = iter(util.IntervalDifferential([], 60))
+
+    def startService(self):
+        """
+        Start the file monitoring service.
+        """
+        service.Service.startService(self)
+        self._setupMonitor()
+
+    def _setupMonitor(self):
+        """
+        Schedule the next monitoring call.
+        """
+        from twisted.internet import reactor
+
+        t, self.index = self.intervals.next()
+        self._call = reactor.callLater(t, self._monitor)
+
+    def stopService(self):
+        """
+        Stop the file monitoring service.
+        """
+        service.Service.stopService(self)
+        if self._call:
+            self._call.cancel()
+            self._call = None
+
+    def monitorFile(self, name, callback, interval=10):
+        """
+        Start monitoring a file for changes.
+
+        @type name: L{bytes}
+        @param name: The name of a file to monitor.
+
+        @type callback: callable which takes a L{bytes} argument
+        @param callback: The function to call when the file has changed.
+
+        @type interval: L{float}
+        @param interval: The interval in seconds between checks.
+        """
+        try:
+            mtime = os.path.getmtime(name)
+        except BaseException:
+            mtime = 0
+        self.files.append([interval, name, callback, mtime])
+        self.intervals.addInterval(interval)
+
+    def unmonitorFile(self, name):
+        """
+        Stop monitoring a file.
+
+        @type name: L{bytes}
+        @param name: A file name.
+        """
+        for i in range(len(self.files)):
+            if name == self.files[i][1]:
+                self.intervals.removeInterval(self.files[i][0])
+                del self.files[i]
+                break
+
+    def _monitor(self):
+        """
+        Monitor a file and make a callback if it has changed.
+        """
+        self._call = None
+        if self.index is not None:
+            name, callback, mtime = self.files[self.index][1:]
+            try:
+                now = os.path.getmtime(name)
+            except BaseException:
+                now = 0
+            if now > mtime:
+                log.msg(f"{name} changed, notifying listener")
+                self.files[self.index][3] = now
+                callback(name)
+        self._setupMonitor()
diff --git a/contrib/python/Twisted/py3/twisted/mail/maildir.py b/contrib/python/Twisted/py3/twisted/mail/maildir.py
new file mode 100644
index 00000000000..c58bf31a941
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/maildir.py
@@ -0,0 +1,910 @@
+# -*- test-case-name: twisted.mail.test.test_mail -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Maildir-style mailbox support.
+"""
+
+import io
+import os
+import socket
+import stat
+from hashlib import md5
+from typing import IO
+
+from zope.interface import implementer
+
+from twisted.cred import checkers, credentials, portal
+from twisted.cred.error import UnauthorizedLogin
+from twisted.internet import defer, interfaces, reactor
+from twisted.mail import mail, pop3, smtp
+from twisted.persisted import dirdbm
+from twisted.protocols import basic
+from twisted.python import failure, log
+
+INTERNAL_ERROR = """\
+From: Twisted.mail Internals
+Subject: An Error Occurred
+
+  An internal server error has occurred.  Please contact the
+  server administrator.
+"""
+
+
+class _MaildirNameGenerator:
+    """
+    A utility class to generate a unique maildir name.
+
+    @type n: L{int}
+    @ivar n: A counter used to generate unique integers.
+
+    @type p: L{int}
+    @ivar p: The ID of the current process.
+
+    @type s: L{bytes}
+    @ivar s: A representation of the hostname.
+
+    @ivar _clock: See C{clock} parameter of L{__init__}.
+    """
+
+    n = 0
+    p = os.getpid()
+    s = socket.gethostname().replace("/", r"\057").replace(":", r"\072")
+
+    def __init__(self, clock):
+        """
+        @type clock: L{IReactorTime <interfaces.IReactorTime>} provider
+        @param clock: A reactor which will be used to learn the current time.
+        """
+        self._clock = clock
+
+    def generate(self):
+        """
+        Generate a string which is intended to be unique across all calls to
+        this function (across all processes, reboots, etc).
+
+        Strings returned by earlier calls to this method will compare less
+        than strings returned by later calls as long as the clock provided
+        doesn't go backwards.
+
+        @rtype: L{bytes}
+        @return: A unique string.
+        """
+        self.n = self.n + 1
+        t = self._clock.seconds()
+        seconds = str(int(t))
+        microseconds = "%07d" % (int((t - int(t)) * 10e6),)
+        return f"{seconds}.M{microseconds}P{self.p}Q{self.n}.{self.s}"
+
+
+_generateMaildirName = _MaildirNameGenerator(reactor).generate
+
+
+def initializeMaildir(dir):
+    """
+    Create a maildir user directory if it doesn't already exist.
+
+    @type dir: L{bytes}
+    @param dir: The path name for a user directory.
+    """
+    dir = os.fsdecode(dir)
+    if not os.path.isdir(dir):
+        os.mkdir(dir, 0o700)
+        for subdir in ["new", "cur", "tmp", ".Trash"]:
+            os.mkdir(os.path.join(dir, subdir), 0o700)
+        for subdir in ["new", "cur", "tmp"]:
+            os.mkdir(os.path.join(dir, ".Trash", subdir), 0o700)
+        # touch
+        open(os.path.join(dir, ".Trash", "maildirfolder"), "w").close()
+
+
+class MaildirMessage(mail.FileMessage):
+    """
+    A message receiver which adds a header and delivers a message to a file
+    whose name includes the size of the message.
+
+    @type size: L{int}
+    @ivar size: The number of octets in the message.
+    """
+
+    size = None
+
+    def __init__(self, address, fp, *a, **kw):
+        """
+        @type address: L{bytes}
+        @param address: The address of the message recipient.
+
+        @type fp: file-like object
+        @param fp: The file in which to store the message while it is being
+            received.
+
+        @type a: 2-L{tuple} of (0) L{bytes}, (1) L{bytes}
+        @param a: Positional arguments for L{FileMessage.__init__}.
+
+        @type kw: L{dict}
+        @param kw: Keyword arguments for L{FileMessage.__init__}.
+        """
+        header = b"Delivered-To: %s\n" % address
+        fp.write(header)
+        self.size = len(header)
+        mail.FileMessage.__init__(self, fp, *a, **kw)
+
+    def lineReceived(self, line):
+        """
+        Write a line to the file.
+
+        @type line: L{bytes}
+        @param line: A received line.
+        """
+        mail.FileMessage.lineReceived(self, line)
+        self.size += len(line) + 1
+
+    def eomReceived(self):
+        """
+        At the end of message, rename the file holding the message to its final
+        name concatenated with the size of the file.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully results in
+            L{bytes}
+        @return: A deferred which returns the name of the file holding the
+            message.
+        """
+        self.finalName = self.finalName + ",S=%d" % self.size
+        return mail.FileMessage.eomReceived(self)
+
+
+@implementer(mail.IAliasableDomain)
+class AbstractMaildirDomain:
+    """
+    An abstract maildir-backed domain.
+
+    @type alias: L{None} or L{dict} mapping
+        L{bytes} to L{AliasBase}
+    @ivar alias: A mapping of username to alias.
+
+    @ivar root: See L{__init__}.
+    """
+
+    alias = None
+    root = None
+
+    def __init__(self, service, root):
+        """
+        @type service: L{MailService}
+        @param service: An email service.
+
+        @type root: L{bytes}
+        @param root: The maildir root directory.
+        """
+        self.root = root
+
+    def userDirectory(self, user):
+        """
+        Return the maildir directory for a user.
+
+        @type user: L{bytes}
+        @param user: A username.
+
+        @rtype: L{bytes} or L{None}
+        @return: The user's mail directory for a valid user. Otherwise,
+            L{None}.
+        """
+        return None
+
+    def setAliasGroup(self, alias):
+        """
+        Set the group of defined aliases for this domain.
+
+        @type alias: L{dict} mapping L{bytes} to L{IAlias} provider.
+        @param alias: A mapping of domain name to alias.
+        """
+        self.alias = alias
+
+    def exists(self, user, memo=None):
+        """
+        Check whether a user exists in this domain or an alias of it.
+
+        @type user: L{User}
+        @param user: A user.
+
+        @type memo: L{None} or L{dict} of L{AliasBase}
+        @param memo: A record of the addresses already considered while
+            resolving aliases. The default value should be used by all
+            external code.
+
+        @rtype: no-argument callable which returns L{IMessage <smtp.IMessage>}
+            provider.
+        @return: A function which takes no arguments and returns a message
+            receiver for the user.
+
+        @raises SMTPBadRcpt: When the given user does not exist in this domain
+            or an alias of it.
+        """
+        if self.userDirectory(user.dest.local) is not None:
+            return lambda: self.startMessage(user)
+        try:
+            a = self.alias[user.dest.local]
+        except BaseException:
+            raise smtp.SMTPBadRcpt(user)
+        else:
+            aliases = a.resolve(self.alias, memo)
+            if aliases:
+                return lambda: aliases
+            log.err("Bad alias configuration: " + str(user))
+            raise smtp.SMTPBadRcpt(user)
+
+    def startMessage(self, user):
+        """
+        Create a maildir message for a user.
+
+        @type user: L{bytes}
+        @param user: A username.
+
+        @rtype: L{MaildirMessage}
+        @return: A message receiver for this user.
+        """
+        if isinstance(user, str):
+            name, domain = user.split("@", 1)
+        else:
+            name, domain = user.dest.local, user.dest.domain
+        dir = self.userDirectory(name)
+        fname = _generateMaildirName()
+        filename = os.path.join(dir, "tmp", fname)
+        fp = open(filename, "w")
+        return MaildirMessage(
+            f"{name}@{domain}", fp, filename, os.path.join(dir, "new", fname)
+        )
+
+    def willRelay(self, user, protocol):
+        """
+        Check whether this domain will relay.
+
+        @type user: L{Address}
+        @param user: The destination address.
+
+        @type protocol: L{SMTP}
+        @param protocol: The protocol over which the message to be relayed is
+            being received.
+
+        @rtype: L{bool}
+        @return: An indication of whether this domain will relay the message to
+            the destination.
+        """
+        return False
+
+    def addUser(self, user, password):
+        """
+        Add a user to this domain.
+
+        Subclasses should override this method.
+
+        @type user: L{bytes}
+        @param user: A username.
+
+        @type password: L{bytes}
+        @param password: A password.
+        """
+        raise NotImplementedError
+
+    def getCredentialsCheckers(self):
+        """
+        Return credentials checkers for this domain.
+
+        Subclasses should override this method.
+
+        @rtype: L{list} of L{ICredentialsChecker
+            <checkers.ICredentialsChecker>} provider
+        @return: Credentials checkers for this domain.
+        """
+        raise NotImplementedError
+
+
+@implementer(interfaces.IConsumer)
+class _MaildirMailboxAppendMessageTask:
+    """
+    A task which adds a message to a maildir mailbox.
+
+    @ivar mbox: See L{__init__}.
+
+    @type defer: L{Deferred <defer.Deferred>} which successfully returns
+        L{None}
+    @ivar defer: A deferred which fires when the task has completed.
+
+    @type opencall: L{IDelayedCall <interfaces.IDelayedCall>} provider or
+        L{None}
+    @ivar opencall: A scheduled call to L{prodProducer}.
+
+    @type msg: file-like object
+    @ivar msg: The message to add.
+
+    @type tmpname: L{bytes}
+    @ivar tmpname: The pathname of the temporary file holding the message while
+        it is being transferred.
+
+    @type fh: file
+    @ivar fh: The new maildir file.
+
+    @type filesender: L{FileSender <basic.FileSender>}
+    @ivar filesender: A file sender which sends the message.
+
+    @type myproducer: L{IProducer <interfaces.IProducer>}
+    @ivar myproducer: The registered producer.
+
+    @type streaming: L{bool}
+    @ivar streaming: Indicates whether the registered producer provides a
+        streaming interface.
+    """
+
+    osopen = staticmethod(os.open)
+    oswrite = staticmethod(os.write)
+    osclose = staticmethod(os.close)
+    osrename = staticmethod(os.rename)
+
+    def __init__(self, mbox, msg):
+        """
+        @type mbox: L{MaildirMailbox}
+        @param mbox: A maildir mailbox.
+
+        @type msg: L{bytes} or file-like object
+        @param msg: The message to add.
+        """
+        self.mbox = mbox
+        self.defer = defer.Deferred()
+        self.openCall = None
+        if not hasattr(msg, "read"):
+            msg = io.BytesIO(msg)
+        self.msg = msg
+
+    def startUp(self):
+        """
+        Start transferring the message to the mailbox.
+        """
+        self.createTempFile()
+        if self.fh != -1:
+            self.filesender = basic.FileSender()
+            self.filesender.beginFileTransfer(self.msg, self)
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register a producer and start asking it for data if it is
+        non-streaming.
+
+        @type producer: L{IProducer <interfaces.IProducer>}
+        @param producer: A producer.
+
+        @type streaming: L{bool}
+        @param streaming: A flag indicating whether the producer provides a
+            streaming interface.
+        """
+        self.myproducer = producer
+        self.streaming = streaming
+        if not streaming:
+            self.prodProducer()
+
+    def prodProducer(self):
+        """
+        Repeatedly prod a non-streaming producer to produce data.
+        """
+        self.openCall = None
+        if self.myproducer is not None:
+            self.openCall = reactor.callLater(0, self.prodProducer)
+            self.myproducer.resumeProducing()
+
+    def unregisterProducer(self):
+        """
+        Finish transferring the message to the mailbox.
+        """
+        self.myproducer = None
+        self.streaming = None
+        self.osclose(self.fh)
+        self.moveFileToNew()
+
+    def write(self, data):
+        """
+        Write data to the maildir file.
+
+        @type data: L{bytes}
+        @param data: Data to be written to the file.
+        """
+        try:
+            self.oswrite(self.fh, data)
+        except BaseException:
+            self.fail()
+
+    def fail(self, err=None):
+        """
+        Fire the deferred to indicate the task completed with a failure.
+
+        @type err: L{Failure <failure.Failure>}
+        @param err: The error that occurred.
+        """
+        if err is None:
+            err = failure.Failure()
+        if self.openCall is not None:
+            self.openCall.cancel()
+        self.defer.errback(err)
+        self.defer = None
+
+    def moveFileToNew(self):
+        """
+        Place the message in the I{new/} directory, add it to the mailbox and
+        fire the deferred to indicate that the task has completed
+        successfully.
+        """
+        while True:
+            newname = os.path.join(self.mbox.path, "new", _generateMaildirName())
+            try:
+                self.osrename(self.tmpname, newname)
+                break
+            except OSError as e:
+                (err, estr) = e.args
+                import errno
+
+                # if the newname exists, retry with a new newname.
+                if err != errno.EEXIST:
+                    self.fail()
+                    newname = None
+                    break
+        if newname is not None:
+            self.mbox.list.append(newname)
+            self.defer.callback(None)
+            self.defer = None
+
+    def createTempFile(self):
+        """
+        Create a temporary file to hold the message as it is being transferred.
+        """
+        attr = (
+            os.O_RDWR
+            | os.O_CREAT
+            | os.O_EXCL
+            | getattr(os, "O_NOINHERIT", 0)
+            | getattr(os, "O_NOFOLLOW", 0)
+        )
+        tries = 0
+        self.fh = -1
+        while True:
+            self.tmpname = os.path.join(self.mbox.path, "tmp", _generateMaildirName())
+            try:
+                self.fh = self.osopen(self.tmpname, attr, 0o600)
+                return None
+            except OSError:
+                tries += 1
+                if tries > 500:
+                    self.defer.errback(
+                        RuntimeError(
+                            "Could not create tmp file for %s" % self.mbox.path
+                        )
+                    )
+                    self.defer = None
+                    return None
+
+
+class MaildirMailbox(pop3.Mailbox):
+    """
+    A maildir-backed mailbox.
+
+    @ivar path: See L{__init__}.
+
+    @type list: L{list} of L{int} or 2-L{tuple} of (0) file-like object,
+        (1) L{bytes}
+    @ivar list: Information about the messages in the mailbox. For undeleted
+        messages, the file containing the message and the
+        full path name of the file are stored.  Deleted messages are indicated
+        by 0.
+
+    @type deleted: L{dict} mapping 2-L{tuple} of (0) file-like object,
+        (1) L{bytes} to L{bytes}
+    @type deleted: A mapping of the information about a file before it was
+        deleted to the full path name of the deleted file in the I{.Trash/}
+        subfolder.
+    """
+
+    AppendFactory = _MaildirMailboxAppendMessageTask
+
+    def __init__(self, path):
+        """
+        @type path: L{bytes}
+        @param path: The directory name for a maildir mailbox.
+        """
+        self.path = path
+        self.list = []
+        self.deleted = {}
+        initializeMaildir(path)
+        for name in ("cur", "new"):
+            for file in os.listdir(os.path.join(path, name)):
+                self.list.append((file, os.path.join(path, name, file)))
+        self.list.sort()
+        self.list = [e[1] for e in self.list]
+
+    def listMessages(self, i=None):
+        """
+        Retrieve the size of a message, or, if none is specified, the size of
+        each message in the mailbox.
+
+        @type i: L{int} or L{None}
+        @param i: The 0-based index of a message.
+
+        @rtype: L{int} or L{list} of L{int}
+        @return: The number of octets in the specified message, or, if an index
+            is not specified, a list of the number of octets for all messages
+            in the mailbox.  Any value which corresponds to a deleted message
+            is set to 0.
+
+        @raise IndexError: When the index does not correspond to a message in
+            the mailbox.
+        """
+        if i is None:
+            ret = []
+            for mess in self.list:
+                if mess:
+                    ret.append(os.stat(mess)[stat.ST_SIZE])
+                else:
+                    ret.append(0)
+            return ret
+        return self.list[i] and os.stat(self.list[i])[stat.ST_SIZE] or 0
+
+    def getMessage(self, i):
+        """
+        Retrieve a file-like object with the contents of a message.
+
+        @type i: L{int}
+        @param i: The 0-based index of a message.
+
+        @rtype: file-like object
+        @return: A file containing the message.
+
+        @raise IndexError: When the index does not correspond to a message in
+            the mailbox.
+        """
+        return open(self.list[i])
+
+    def getUidl(self, i):
+        """
+        Get a unique identifier for a message.
+
+        @type i: L{int}
+        @param i: The 0-based index of a message.
+
+        @rtype: L{bytes}
+        @return: A string of printable characters uniquely identifying the
+            message for all time.
+
+        @raise IndexError: When the index does not correspond to a message in
+            the mailbox.
+        """
+        # Returning the actual filename is a mistake.  Hash it.
+        base = os.path.basename(self.list[i])
+        return md5(base).hexdigest()
+
+    def deleteMessage(self, i):
+        """
+        Mark a message for deletion.
+
+        Move the message to the I{.Trash/} subfolder so it can be undeleted
+        by an administrator.
+
+        @type i: L{int}
+        @param i: The 0-based index of a message.
+
+        @raise IndexError: When the index does not correspond to a message in
+            the mailbox.
+        """
+        trashFile = os.path.join(
+            self.path, ".Trash", "cur", os.path.basename(self.list[i])
+        )
+        os.rename(self.list[i], trashFile)
+        self.deleted[self.list[i]] = trashFile
+        self.list[i] = 0
+
+    def undeleteMessages(self):
+        """
+        Undelete all messages marked for deletion.
+
+        Move each message marked for deletion from the I{.Trash/} subfolder back
+        to its original position.
+        """
+        for real, trash in self.deleted.items():
+            try:
+                os.rename(trash, real)
+            except OSError as e:
+                (err, estr) = e.args
+                import errno
+
+                # If the file has been deleted from disk, oh well!
+                if err != errno.ENOENT:
+                    raise
+                # This is a pass
+            else:
+                try:
+                    self.list[self.list.index(0)] = real
+                except ValueError:
+                    self.list.append(real)
+        self.deleted.clear()
+
+    def appendMessage(self, txt):
+        """
+        Add a message to the mailbox.
+
+        @type txt: L{bytes} or file-like object
+        @param txt: A message to add.
+
+        @rtype: L{Deferred <defer.Deferred>}
+        @return: A deferred which fires when the message has been added to
+            the mailbox.
+        """
+        task = self.AppendFactory(self, txt)
+        result = task.defer
+        task.startUp()
+        return result
+
+
+@implementer(pop3.IMailbox)
+class StringListMailbox:
+    """
+    An in-memory mailbox.
+
+    @ivar  msgs: See L{__init__}.
+
+    @type _delete: L{set} of L{int}
+    @ivar _delete: The indices of messages which have been marked for deletion.
+    """
+
+    def __init__(self, msgs):
+        """
+        @type msgs: L{list} of L{bytes}
+        @param msgs: The contents of each message in the mailbox.
+        """
+        self.msgs = msgs
+        self._delete = set()
+
+    def listMessages(self, i=None):
+        """
+        Retrieve the size of a message, or, if none is specified, the size of
+        each message in the mailbox.
+
+        @type i: L{int} or L{None}
+        @param i: The 0-based index of a message.
+
+        @rtype: L{int} or L{list} of L{int}
+        @return: The number of octets in the specified message, or, if an index
+            is not specified, a list of the number of octets in each message in
+            the mailbox.  Any value which corresponds to a deleted message is
+            set to 0.
+
+        @raise IndexError: When the index does not correspond to a message in
+            the mailbox.
+        """
+        if i is None:
+            return [self.listMessages(msg) for msg in range(len(self.msgs))]
+        if i in self._delete:
+            return 0
+        return len(self.msgs[i])
+
+    def getMessage(self, i: int) -> IO[bytes]:
+        """
+        Return an in-memory file-like object with the contents of a message.
+
+        @param i: The 0-based index of a message.
+
+        @return: An in-memory file-like object containing the message.
+
+        @raise IndexError: When the index does not correspond to a message in
+            the mailbox.
+        """
+        return io.BytesIO(self.msgs[i])
+
+    def getUidl(self, i):
+        """
+        Get a unique identifier for a message.
+
+        @type i: L{int}
+        @param i: The 0-based index of a message.
+
+        @rtype: L{bytes}
+        @return: A hash of the contents of the message at the given index.
+
+        @raise IndexError: When the index does not correspond to a message in
+            the mailbox.
+        """
+        return md5(self.msgs[i]).hexdigest()
+
+    def deleteMessage(self, i):
+        """
+        Mark a message for deletion.
+
+        @type i: L{int}
+        @param i: The 0-based index of a message to delete.
+
+        @raise IndexError: When the index does not correspond to a message in
+            the mailbox.
+        """
+        self._delete.add(i)
+
+    def undeleteMessages(self):
+        """
+        Undelete any messages which have been marked for deletion.
+        """
+        self._delete = set()
+
+    def sync(self):
+        """
+        Discard the contents of any messages marked for deletion.
+        """
+        for index in self._delete:
+            self.msgs[index] = ""
+        self._delete = set()
+
+
+@implementer(portal.IRealm)
+class MaildirDirdbmDomain(AbstractMaildirDomain):
+    """
+    A maildir-backed domain where membership is checked with a
+    L{DirDBM <dirdbm.DirDBM>} database.
+
+    The directory structure of a MaildirDirdbmDomain is:
+
+    /passwd <-- a DirDBM directory
+
+    /USER/{cur, new, del} <-- each user has these three directories
+
+    @ivar postmaster: See L{__init__}.
+
+    @type dbm: L{DirDBM <dirdbm.DirDBM>}
+    @ivar dbm: The authentication database for the domain.
+    """
+
+    portal = None
+    _credcheckers = None
+
+    def __init__(self, service, root, postmaster=0):
+        """
+        @type service: L{MailService}
+        @param service: An email service.
+
+        @type root: L{bytes}
+        @param root: The maildir root directory.
+
+        @type postmaster: L{bool}
+        @param postmaster: A flag indicating whether non-existent addresses
+            should be forwarded to the postmaster (C{True}) or
+            bounced (C{False}).
+        """
+        root = os.fsencode(root)
+        AbstractMaildirDomain.__init__(self, service, root)
+        dbm = os.path.join(root, b"passwd")
+        if not os.path.exists(dbm):
+            os.makedirs(dbm)
+        self.dbm = dirdbm.open(dbm)
+        self.postmaster = postmaster
+
+    def userDirectory(self, name):
+        """
+        Return the path to a user's mail directory.
+
+        @type name: L{bytes}
+        @param name: A username.
+
+        @rtype: L{bytes} or L{None}
+        @return: The path to the user's mail directory for a valid user. For
+            an invalid user, the path to the postmaster's mailbox if bounces
+            are redirected there. Otherwise, L{None}.
+        """
+        if name not in self.dbm:
+            if not self.postmaster:
+                return None
+            name = "postmaster"
+        dir = os.path.join(self.root, name)
+        if not os.path.exists(dir):
+            initializeMaildir(dir)
+        return dir
+
+    def addUser(self, user, password):
+        """
+        Add a user to this domain by adding an entry in the authentication
+        database and initializing the user's mail directory.
+
+        @type user: L{bytes}
+        @param user: A username.
+
+        @type password: L{bytes}
+        @param password: A password.
+        """
+        self.dbm[user] = password
+        # Ensure it is initialized
+        self.userDirectory(user)
+
+    def getCredentialsCheckers(self):
+        """
+        Return credentials checkers for this domain.
+
+        @rtype: L{list} of L{ICredentialsChecker
+            <checkers.ICredentialsChecker>} provider
+        @return: Credentials checkers for this domain.
+        """
+        if self._credcheckers is None:
+            self._credcheckers = [DirdbmDatabase(self.dbm)]
+        return self._credcheckers
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        """
+        Get the mailbox for an authenticated user.
+
+        The mailbox for the authenticated user will be returned only if the
+        given interfaces include L{IMailbox <pop3.IMailbox>}.  Requests for
+        anonymous access will be met with a mailbox containing a message
+        indicating that an internal error has occurred.
+
+        @type avatarId: L{bytes} or C{twisted.cred.checkers.ANONYMOUS}
+        @param avatarId: A string which identifies a user or an object which
+            signals a request for anonymous access.
+
+        @type mind: L{None}
+        @param mind: Unused.
+
+        @type interfaces: n-L{tuple} of C{zope.interface.Interface}
+        @param interfaces: A group of interfaces, one of which the avatar
+            must support.
+
+        @rtype: 3-L{tuple} of (0) L{IMailbox <pop3.IMailbox>},
+            (1) L{IMailbox <pop3.IMailbox>} provider, (2) no-argument
+            callable
+        @return: A tuple of the supported interface, a mailbox, and a
+            logout function.
+
+        @raise NotImplementedError: When the given interfaces do not include
+            L{IMailbox <pop3.IMailbox>}.
+        """
+        if pop3.IMailbox not in interfaces:
+            raise NotImplementedError("No interface")
+        if avatarId == checkers.ANONYMOUS:
+            mbox = StringListMailbox([INTERNAL_ERROR])
+        else:
+            mbox = MaildirMailbox(os.path.join(self.root, avatarId))
+
+        return (pop3.IMailbox, mbox, lambda: None)
+
+
+@implementer(checkers.ICredentialsChecker)
+class DirdbmDatabase:
+    """
+    A credentials checker which authenticates users out of a
+    L{DirDBM <dirdbm.DirDBM>} database.
+
+    @type dirdbm: L{DirDBM <dirdbm.DirDBM>}
+    @ivar dirdbm: An authentication database.
+    """
+
+    # credentialInterfaces is not used by the class
+    credentialInterfaces = (
+        credentials.IUsernamePassword,
+        credentials.IUsernameHashedPassword,
+    )
+
+    def __init__(self, dbm):
+        """
+        @type dbm: L{DirDBM <dirdbm.DirDBM>}
+        @param dbm: An authentication database.
+        """
+        self.dirdbm = dbm
+
+    def requestAvatarId(self, c):
+        """
+        Authenticate a user and, if successful, return their username.
+
+        @type c: L{IUsernamePassword <credentials.IUsernamePassword>} or
+            L{IUsernameHashedPassword <credentials.IUsernameHashedPassword>}
+            provider.
+        @param c: Credentials.
+
+        @rtype: L{bytes}
+        @return: A string which identifies an user.
+
+        @raise UnauthorizedLogin: When the credentials check fails.
+        """
+        if c.username in self.dirdbm:
+            if c.checkPassword(self.dirdbm[c.username]):
+                return c.username
+        raise UnauthorizedLogin()
diff --git a/contrib/python/Twisted/py3/twisted/mail/newsfragments/.gitignore b/contrib/python/Twisted/py3/twisted/mail/newsfragments/.gitignore
new file mode 100644
index 00000000000..f935021a8f8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/newsfragments/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/contrib/python/Twisted/py3/twisted/mail/pb.py b/contrib/python/Twisted/py3/twisted/mail/pb.py
new file mode 100644
index 00000000000..1b57e26c95b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/pb.py
@@ -0,0 +1,117 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+import os
+
+from twisted.spread import pb
+
+
+class Maildir(pb.Referenceable):
+    def __init__(self, directory, rootDirectory):
+        self.virtualDirectory = directory
+        self.rootDirectory = rootDirectory
+        self.directory = os.path.join(rootDirectory, directory)
+
+    def getFolderMessage(self, folder, name):
+        if "/" in name:
+            raise OSError("can only open files in '%s' directory'" % folder)
+        with open(os.path.join(self.directory, "new", name)) as fp:
+            return fp.read()
+
+    def deleteFolderMessage(self, folder, name):
+        if "/" in name:
+            raise OSError("can only delete files in '%s' directory'" % folder)
+        os.rename(
+            os.path.join(self.directory, folder, name),
+            os.path.join(self.rootDirectory, ".Trash", folder, name),
+        )
+
+    def deleteNewMessage(self, name):
+        return self.deleteFolderMessage("new", name)
+
+    remote_deleteNewMessage = deleteNewMessage
+
+    def deleteCurMessage(self, name):
+        return self.deleteFolderMessage("cur", name)
+
+    remote_deleteCurMessage = deleteCurMessage
+
+    def getNewMessages(self):
+        return os.listdir(os.path.join(self.directory, "new"))
+
+    remote_getNewMessages = getNewMessages
+
+    def getCurMessages(self):
+        return os.listdir(os.path.join(self.directory, "cur"))
+
+    remote_getCurMessages = getCurMessages
+
+    def getNewMessage(self, name):
+        return self.getFolderMessage("new", name)
+
+    remote_getNewMessage = getNewMessage
+
+    def getCurMessage(self, name):
+        return self.getFolderMessage("cur", name)
+
+    remote_getCurMessage = getCurMessage
+
+    def getSubFolder(self, name):
+        if name[0] == ".":
+            raise OSError("subfolder name cannot begin with a '.'")
+        name = name.replace("/", ":")
+        if self.virtualDirectoy == ".":
+            name = "." + name
+        else:
+            name = self.virtualDirectory + ":" + name
+        if not self._isSubFolder(name):
+            raise OSError("not a subfolder")
+        return Maildir(name, self.rootDirectory)
+
+    remote_getSubFolder = getSubFolder
+
+    def _isSubFolder(self, name):
+        return not os.path.isdir(
+            os.path.join(self.rootDirectory, name)
+        ) or not os.path.isfile(os.path.join(self.rootDirectory, name, "maildirfolder"))
+
+
+class MaildirCollection(pb.Referenceable):
+    def __init__(self, root):
+        self.root = root
+
+    def getSubFolders(self):
+        return os.listdir(self.getRoot())
+
+    remote_getSubFolders = getSubFolders
+
+    def getSubFolder(self, name):
+        if "/" in name or name[0] == ".":
+            raise OSError("invalid name")
+        return Maildir(".", os.path.join(self.getRoot(), name))
+
+    remote_getSubFolder = getSubFolder
+
+
+class MaildirBroker(pb.Broker):
+    def proto_getCollection(self, requestID, name, domain, password):
+        collection = self._getCollection()
+        if collection is None:
+            self.sendError(requestID, "permission denied")
+        else:
+            self.sendAnswer(requestID, collection)
+
+    def getCollection(self, name, domain, password):
+        if domain not in self.domains:
+            return
+        domain = self.domains[domain]
+        if name in domain.dbm and domain.dbm[name] == password:
+            return MaildirCollection(domain.userDirectory(name))
+
+
+class MaildirClient(pb.Broker):
+    def getCollection(self, name, domain, password, callback, errback):
+        requestID = self.newRequestID()
+        self.waitingForAnswers[requestID] = callback, errback
+        self.sendCall("getCollection", requestID, name, domain, password)
diff --git a/contrib/python/Twisted/py3/twisted/mail/pop3.py b/contrib/python/Twisted/py3/twisted/mail/pop3.py
new file mode 100644
index 00000000000..7b230d20591
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/pop3.py
@@ -0,0 +1,1704 @@
+# -*- test-case-name: twisted.mail.test.test_pop3 -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Post-office Protocol version 3.
+
+@author: Glyph Lefkowitz
+@author: Jp Calderone
+"""
+
+import base64
+import binascii
+import warnings
+from hashlib import md5
+from typing import Optional
+
+from zope.interface import implementer
+
+from twisted import cred
+from twisted.internet import defer, interfaces, task
+from twisted.mail import smtp
+from twisted.mail._except import POP3ClientError, POP3Error, _POP3MessageDeleted
+from twisted.mail.interfaces import (
+    IMailboxPOP3 as IMailbox,
+    IServerFactoryPOP3 as IServerFactory,
+)
+from twisted.protocols import basic, policies
+from twisted.python import log
+
+
+# Authentication
+@implementer(cred.credentials.IUsernameHashedPassword)
+class APOPCredentials:
+    """
+    Credentials for use in APOP authentication.
+
+    @ivar magic: See L{__init__}
+    @ivar username: See L{__init__}
+    @ivar digest: See L{__init__}
+    """
+
+    def __init__(self, magic, username, digest):
+        """
+        @type magic: L{bytes}
+        @param magic: The challenge string used to encrypt the password.
+
+        @type username: L{bytes}
+        @param username: The username associated with these credentials.
+
+        @type digest: L{bytes}
+        @param digest: An encrypted version of the user's password.  Should be
+            generated as an MD5 hash of the challenge string concatenated with
+            the plaintext password.
+        """
+        self.magic = magic
+        self.username = username
+        self.digest = digest
+
+    def checkPassword(self, password):
+        """
+        Validate a plaintext password against the credentials.
+
+        @type password: L{bytes}
+        @param password: A plaintext password.
+
+        @rtype: L{bool}
+        @return: C{True} if the credentials represented by this object match
+        the given password, C{False} if they do not.
+        """
+        seed = self.magic + password
+        myDigest = md5(seed).hexdigest()
+        return myDigest == self.digest
+
+
+class _HeadersPlusNLines:
+    """
+    A utility class to retrieve the header and some lines of the body of a mail
+    message.
+
+    @ivar _file: See L{__init__}
+    @ivar _extraLines: See L{__init__}
+
+    @type linecount: L{int}
+    @ivar linecount: The number of full lines of the message body scanned.
+
+    @type headers: L{bool}
+    @ivar headers: An indication of which part of the message is being scanned.
+        C{True} for the header and C{False} for the body.
+
+    @type done: L{bool}
+    @ivar done: A flag indicating when the desired part of the message has been
+        scanned.
+
+    @type buf: L{bytes}
+    @ivar buf: The portion of the message body that has been scanned, up to
+        C{n} lines.
+    """
+
+    def __init__(self, file, extraLines):
+        """
+        @type file: file-like object
+        @param file: A file containing a mail message.
+
+        @type extraLines: L{int}
+        @param extraLines: The number of lines of the message body to retrieve.
+        """
+        self._file = file
+        self._extraLines = extraLines
+        self.linecount = 0
+        self.headers = 1
+        self.done = 0
+        self.buf = b""
+
+    def read(self, bytes):
+        """
+        Scan bytes from the file.
+
+        @type bytes: L{int}
+        @param bytes: The number of bytes to read from the file.
+
+        @rtype: L{bytes}
+        @return: Each portion of the header as it is scanned.  Then, full lines
+            of the message body as they are scanned.  When more than one line
+            of the header and/or body has been scanned, the result is the
+            concatenation of the lines.  When the scan results in no full
+            lines, the empty string is returned.
+        """
+        if self.done:
+            return b""
+        data = self._file.read(bytes)
+        if not data:
+            return data
+        if self.headers:
+            df, sz = data.find(b"\r\n\r\n"), 4
+            if df == -1:
+                df, sz = data.find(b"\n\n"), 2
+            if df != -1:
+                df += sz
+                val = data[:df]
+                data = data[df:]
+                self.linecount = 1
+                self.headers = 0
+        else:
+            val = b""
+        if self.linecount > 0:
+            dsplit = (self.buf + data).split(b"\n")
+            self.buf = dsplit[-1]
+            for ln in dsplit[:-1]:
+                if self.linecount > self._extraLines:
+                    self.done = 1
+                    return val
+                val += ln + b"\n"
+                self.linecount += 1
+            return val
+        else:
+            return data
+
+
+class _IteratorBuffer:
+    """
+    An iterator which buffers the elements of a container and periodically
+    passes them as input to a writer.
+
+    @ivar write: See L{__init__}.
+    @ivar memoryBufferSize: See L{__init__}.
+
+    @type bufSize: L{int}
+    @ivar bufSize: The number of bytes currently in the buffer.
+
+    @type lines: L{list} of L{bytes}
+    @ivar lines: The buffer, which is a list of strings.
+
+    @type iterator: iterator which yields L{bytes}
+    @ivar iterator: An iterator over a container of strings.
+    """
+
+    bufSize = 0
+
+    def __init__(self, write, iterable, memoryBufferSize=None):
+        """
+        @type write: callable that takes L{list} of L{bytes}
+        @param write: A writer which is a callable that takes a list of
+            strings.
+
+        @type iterable: iterable which yields L{bytes}
+        @param iterable: An iterable container of strings.
+
+        @type memoryBufferSize: L{int} or L{None}
+        @param memoryBufferSize: The number of bytes to buffer before flushing
+            the buffer to the writer.
+        """
+        self.lines = []
+        self.write = write
+        self.iterator = iter(iterable)
+        if memoryBufferSize is None:
+            memoryBufferSize = 2**16
+        self.memoryBufferSize = memoryBufferSize
+
+    def __iter__(self):
+        """
+        Return an iterator.
+
+        @rtype: iterator which yields L{bytes}
+        @return: An iterator over strings.
+        """
+        return self
+
+    def __next__(self):
+        """
+        Get the next string from the container, buffer it, and possibly send
+        the buffer to the writer.
+
+        The contents of the buffer are written when it is full or when no
+        further values are available from the container.
+
+        @raise StopIteration: When no further values are available from the
+        container.
+        """
+        try:
+            v = next(self.iterator)
+        except StopIteration:
+            if self.lines:
+                self.write(self.lines)
+            # Drop some references, in case they're edges in a cycle.
+            del self.iterator, self.lines, self.write
+            raise
+        else:
+            if v is not None:
+                self.lines.append(v)
+                self.bufSize += len(v)
+                if self.bufSize > self.memoryBufferSize:
+                    self.write(self.lines)
+                    self.lines = []
+                    self.bufSize = 0
+
+    next = __next__
+
+
+def iterateLineGenerator(proto, gen):
+    """
+    Direct the output of an iterator to the transport of a protocol and arrange
+    for iteration to take place.
+
+    @type proto: L{POP3}
+    @param proto: A POP3 server protocol.
+
+    @type gen: iterator which yields L{bytes}
+    @param gen: An iterator over strings.
+
+    @rtype: L{Deferred <defer.Deferred>}
+    @return: A deferred which fires when the iterator finishes.
+    """
+    coll = _IteratorBuffer(proto.transport.writeSequence, gen)
+    return proto.schedule(coll)
+
+
+def successResponse(response):
+    """
+    Format an object as a positive response.
+
+    @type response: stringifyable L{object}
+    @param response: An object with a string representation.
+
+    @rtype: L{bytes}
+    @return: A positive POP3 response string.
+    """
+    if not isinstance(response, bytes):
+        response = str(response).encode("utf-8")
+    return b"+OK " + response + b"\r\n"
+
+
+def formatStatResponse(msgs):
+    """
+    Format a list of message sizes into a STAT response.
+
+    This generator function is intended to be used with
+    L{Cooperator <twisted.internet.task.Cooperator>}.
+
+    @type msgs: L{list} of L{int}
+    @param msgs: A list of message sizes.
+
+    @rtype: L{None} or L{bytes}
+    @return: Yields none until a result is available, then a string that is
+        suitable for use in a STAT response. The string consists of the number
+        of messages and the total size of the messages in octets.
+    """
+    i = 0
+    bytes = 0
+    for size in msgs:
+        i += 1
+        bytes += size
+        yield None
+    yield successResponse(b"%d %d" % (i, bytes))
+
+
+def formatListLines(msgs):
+    """
+    Format a list of message sizes for use in a LIST response.
+
+    @type msgs: L{list} of L{int}
+    @param msgs: A list of message sizes.
+
+    @rtype: L{bytes}
+    @return: Yields a series of strings that are suitable for use as scan
+        listings in a LIST response. Each string consists of a message number
+        and its size in octets.
+    """
+    i = 0
+    for size in msgs:
+        i += 1
+        yield b"%d %d\r\n" % (i, size)
+
+
+def formatListResponse(msgs):
+    """
+    Format a list of message sizes into a complete LIST response.
+
+    This generator function is intended to be used with
+    L{Cooperator <twisted.internet.task.Cooperator>}.
+
+    @type msgs: L{list} of L{int}
+    @param msgs: A list of message sizes.
+
+    @rtype: L{bytes}
+    @return: Yields a series of strings which make up a complete LIST response.
+    """
+    yield successResponse(b"%d" % (len(msgs),))
+    yield from formatListLines(msgs)
+    yield b".\r\n"
+
+
+def formatUIDListLines(msgs, getUidl):
+    """
+    Format a list of message sizes for use in a UIDL response.
+
+    @param msgs: See L{formatUIDListResponse}
+    @param getUidl: See L{formatUIDListResponse}
+
+    @rtype: L{bytes}
+    @return: Yields a series of strings that are suitable for use as unique-id
+        listings in a UIDL response. Each string consists of a message number
+        and its unique id.
+    """
+    for i, m in enumerate(msgs):
+        if m is not None:
+            uid = getUidl(i)
+            if not isinstance(uid, bytes):
+                uid = str(uid).encode("utf-8")
+            yield b"%d %b\r\n" % (i + 1, uid)
+
+
+def formatUIDListResponse(msgs, getUidl):
+    """
+    Format a list of message sizes into a complete UIDL response.
+
+    This generator function is intended to be used with
+    L{Cooperator <twisted.internet.task.Cooperator>}.
+
+    @type msgs: L{list} of L{int}
+    @param msgs: A list of message sizes.
+
+    @type getUidl: one-argument callable returning bytes
+    @param getUidl: A callable which takes a message index number and returns
+        the UID of the corresponding message in the mailbox.
+
+    @rtype: L{bytes}
+    @return: Yields a series of strings which make up a complete UIDL response.
+    """
+    yield successResponse("")
+    yield from formatUIDListLines(msgs, getUidl)
+    yield b".\r\n"
+
+
+@implementer(interfaces.IProducer)
+class POP3(basic.LineOnlyReceiver, policies.TimeoutMixin):
+    """
+    A POP3 server protocol.
+
+    @type portal: L{Portal}
+    @ivar portal: A portal for authentication.
+
+    @type factory: L{IServerFactory} provider
+    @ivar factory: A server factory which provides an interface for querying
+        capabilities of the server.
+
+    @type timeOut: L{int}
+    @ivar timeOut: The number of seconds to wait for a command from the client
+        before disconnecting.
+
+    @type schedule: callable that takes interator and returns
+        L{Deferred <defer.Deferred>}
+    @ivar schedule: A callable that arranges for an iterator to be
+        cooperatively iterated over along with all other iterators which have
+        been passed to it such that runtime is divided between all of them.  It
+        returns a deferred which fires when the iterator finishes.
+
+    @type magic: L{bytes} or L{None}
+    @ivar magic: An APOP challenge.  If not set, an APOP challenge string
+        will be generated when a connection is made.
+
+    @type _userIs: L{bytes} or L{None}
+    @ivar _userIs: The username sent with the USER command.
+
+    @type _onLogout: no-argument callable or L{None}
+    @ivar _onLogout: The function to be executed when the connection is
+        lost.
+
+    @type mbox: L{IMailbox} provider
+    @ivar mbox: The mailbox for the authenticated user.
+
+    @type state: L{bytes}
+    @ivar state: The state which indicates what type of messages are expected
+        from the client.  Valid states are 'COMMAND' and 'AUTH'
+
+    @type blocked: L{None} or L{list} of 2-L{tuple} of
+        (E{1}) L{bytes} (E{2}) L{tuple} of L{bytes}
+    @ivar blocked: A list of blocked commands.  While a response to a command
+        is being generated by the server, other commands are blocked.  When
+        no command is outstanding, C{blocked} is set to none.  Otherwise, it
+        contains a list of information about blocked commands.  Each list
+        entry consists of the command and the arguments to the command.
+
+    @type _highest: L{int}
+    @ivar _highest: The 1-based index of the highest message retrieved.
+
+    @type _auth: L{IUsernameHashedPassword
+        <cred.credentials.IUsernameHashedPassword>} provider
+    @ivar _auth: Authorization credentials.
+    """
+
+    magic: Optional[bytes] = None
+    _userIs = None
+    _onLogout = None
+
+    AUTH_CMDS = [b"CAPA", b"USER", b"PASS", b"APOP", b"AUTH", b"RPOP", b"QUIT"]
+
+    portal = None
+    factory = None
+
+    # The mailbox we're serving
+    mbox = None
+
+    # Set this pretty low -- POP3 clients are expected to log in, download
+    # everything, and log out.
+    timeOut = 300
+
+    state = "COMMAND"
+
+    # PIPELINE
+    blocked = None
+
+    # Cooperate and suchlike.
+    schedule = staticmethod(task.coiterate)
+
+    _highest = 0
+
+    def connectionMade(self):
+        """
+        Send a greeting to the client after the connection has been made.
+        """
+        if self.magic is None:
+            self.magic = self.generateMagic()
+        self.successResponse(self.magic)
+        self.setTimeout(self.timeOut)
+        if getattr(self.factory, "noisy", True):
+            log.msg("New connection from " + str(self.transport.getPeer()))
+
+    def connectionLost(self, reason):
+        """
+        Clean up when the connection has been lost.
+
+        @type reason: L{Failure}
+        @param reason: The reason the connection was terminated.
+        """
+        if self._onLogout is not None:
+            self._onLogout()
+            self._onLogout = None
+        self.setTimeout(None)
+
+    def generateMagic(self):
+        """
+        Generate an APOP challenge.
+
+        @rtype: L{bytes}
+        @return: An RFC 822 message id format string.
+        """
+        return smtp.messageid()
+
+    def successResponse(self, message=""):
+        """
+        Send a response indicating success.
+
+        @type message: stringifyable L{object}
+        @param message: An object whose string representation should be
+            included in the response.
+        """
+        self.transport.write(successResponse(message))
+
+    def failResponse(self, message=b""):
+        """
+        Send a response indicating failure.
+
+        @type message: stringifyable L{object}
+        @param message: An object whose string representation should be
+            included in the response.
+        """
+        if not isinstance(message, bytes):
+            message = str(message).encode("utf-8")
+        self.sendLine(b"-ERR " + message)
+
+    def lineReceived(self, line):
+        """
+        Pass a received line to a state machine function.
+
+        @type line: L{bytes}
+        @param line: A received line.
+        """
+        self.resetTimeout()
+        getattr(self, "state_" + self.state)(line)
+
+    def _unblock(self, _):
+        """
+        Process as many blocked commands as possible.
+
+        If there are no more blocked commands, set up for the next command to
+        be sent immediately.
+
+        @type _: L{object}
+        @param _: Ignored.
+        """
+        commands = self.blocked
+        self.blocked = None
+        while commands and self.blocked is None:
+            cmd, args = commands.pop(0)
+            self.processCommand(cmd, *args)
+        if self.blocked is not None:
+            self.blocked.extend(commands)
+
+    def state_COMMAND(self, line):
+        """
+        Handle received lines for the COMMAND state in which commands from the
+        client are expected.
+
+        @type line: L{bytes}
+        @param line: A received command.
+        """
+        try:
+            return self.processCommand(*line.split(b" "))
+        except (ValueError, AttributeError, POP3Error, TypeError) as e:
+            log.err()
+            self.failResponse(
+                b": ".join(
+                    [
+                        b"bad protocol or server",
+                        e.__class__.__name__.encode("utf-8"),
+                        b"".join(e.args),
+                    ]
+                )
+            )
+
+    def processCommand(self, command, *args):
+        """
+        Dispatch a command from the client for handling.
+
+        @type command: L{bytes}
+        @param command: A POP3 command.
+
+        @type args: L{tuple} of L{bytes}
+        @param args: Arguments to the command.
+
+        @raise POP3Error: When the command is invalid or the command requires
+            prior authentication which hasn't been performed.
+        """
+        if self.blocked is not None:
+            self.blocked.append((command, args))
+            return
+
+        command = command.upper()
+        authCmd = command in self.AUTH_CMDS
+        if not self.mbox and not authCmd:
+            raise POP3Error(b"not authenticated yet: cannot do " + command)
+        f = getattr(self, "do_" + command.decode("utf-8"), None)
+        if f:
+            return f(*args)
+        raise POP3Error(b"Unknown protocol command: " + command)
+
+    def listCapabilities(self):
+        """
+        Return a list of server capabilities suitable for use in a CAPA
+        response.
+
+        @rtype: L{list} of L{bytes}
+        @return: A list of server capabilities.
+        """
+        baseCaps = [
+            b"TOP",
+            b"USER",
+            b"UIDL",
+            b"PIPELINE",
+            b"CELERITY",
+            b"AUSPEX",
+            b"POTENCE",
+        ]
+
+        if IServerFactory.providedBy(self.factory):
+            # Oh my god.  We can't just loop over a list of these because
+            # each has spectacularly different return value semantics!
+            try:
+                v = self.factory.cap_IMPLEMENTATION()
+                if v and not isinstance(v, bytes):
+                    v = str(v).encode("utf-8")
+            except NotImplementedError:
+                pass
+            except BaseException:
+                log.err()
+            else:
+                baseCaps.append(b"IMPLEMENTATION " + v)
+
+            try:
+                v = self.factory.cap_EXPIRE()
+                if v and not isinstance(v, bytes):
+                    v = str(v).encode("utf-8")
+            except NotImplementedError:
+                pass
+            except BaseException:
+                log.err()
+            else:
+                if v is None:
+                    v = b"NEVER"
+                if self.factory.perUserExpiration():
+                    if self.mbox:
+                        v = str(self.mbox.messageExpiration).encode("utf-8")
+                    else:
+                        v = v + b" USER"
+                baseCaps.append(b"EXPIRE " + v)
+
+            try:
+                v = self.factory.cap_LOGIN_DELAY()
+                if v and not isinstance(v, bytes):
+                    v = str(v).encode("utf-8")
+            except NotImplementedError:
+                pass
+            except BaseException:
+                log.err()
+            else:
+                if self.factory.perUserLoginDelay():
+                    if self.mbox:
+                        v = str(self.mbox.loginDelay).encode("utf-8")
+                    else:
+                        v = v + b" USER"
+                baseCaps.append(b"LOGIN-DELAY " + v)
+
+            try:
+                v = self.factory.challengers
+            except AttributeError:
+                pass
+            except BaseException:
+                log.err()
+            else:
+                baseCaps.append(b"SASL " + b" ".join(v.keys()))
+        return baseCaps
+
+    def do_CAPA(self):
+        """
+        Handle a CAPA command.
+
+        Respond with the server capabilities.
+        """
+        self.successResponse(b"I can do the following:")
+        for cap in self.listCapabilities():
+            self.sendLine(cap)
+        self.sendLine(b".")
+
+    def do_AUTH(self, args=None):
+        """
+        Handle an AUTH command.
+
+        If the AUTH extension is not supported, send an error response.  If an
+        authentication mechanism was not specified in the command, send a list
+        of all supported authentication methods.  Otherwise, send an
+        authentication challenge to the client and transition to the
+        AUTH state.
+
+        @type args: L{bytes} or L{None}
+        @param args: The name of an authentication mechanism.
+        """
+        if not getattr(self.factory, "challengers", None):
+            self.failResponse(b"AUTH extension unsupported")
+            return
+
+        if args is None:
+            self.successResponse("Supported authentication methods:")
+            for a in self.factory.challengers:
+                self.sendLine(a.upper())
+            self.sendLine(b".")
+            return
+
+        auth = self.factory.challengers.get(args.strip().upper())
+        if not self.portal or not auth:
+            self.failResponse(b"Unsupported SASL selected")
+            return
+
+        self._auth = auth()
+        chal = self._auth.getChallenge()
+
+        self.sendLine(b"+ " + base64.b64encode(chal))
+        self.state = "AUTH"
+
+    def state_AUTH(self, line):
+        """
+        Handle received lines for the AUTH state in which an authentication
+        challenge response from the client is expected.
+
+        Transition back to the COMMAND state.  Check the credentials and
+        complete the authorization process with the L{_cbMailbox}
+        callback function on success or the L{_ebMailbox} and L{_ebUnexpected}
+        errback functions on failure.
+
+        @type line: L{bytes}
+        @param line: The challenge response.
+        """
+        self.state = "COMMAND"
+        try:
+            parts = base64.b64decode(line).split(None, 1)
+        except binascii.Error:
+            self.failResponse(b"Invalid BASE64 encoding")
+        else:
+            if len(parts) != 2:
+                self.failResponse(b"Invalid AUTH response")
+                return
+            self._auth.username = parts[0]
+            self._auth.response = parts[1]
+            d = self.portal.login(self._auth, None, IMailbox)
+            d.addCallback(self._cbMailbox, parts[0])
+            d.addErrback(self._ebMailbox)
+            d.addErrback(self._ebUnexpected)
+
+    def do_APOP(self, user, digest):
+        """
+        Handle an APOP command.
+
+        Perform APOP authentication and complete the authorization process with
+        the L{_cbMailbox} callback function on success or the L{_ebMailbox}
+        and L{_ebUnexpected} errback functions on failure.
+
+        @type user: L{bytes}
+        @param user: A username.
+
+        @type digest: L{bytes}
+        @param digest: An MD5 digest string.
+        """
+        d = defer.maybeDeferred(self.authenticateUserAPOP, user, digest)
+        d.addCallbacks(
+            self._cbMailbox, self._ebMailbox, callbackArgs=(user,)
+        ).addErrback(self._ebUnexpected)
+
+    def _cbMailbox(self, result, user):
+        """
+        Complete successful authentication.
+
+        Save the mailbox and logout function for the authenticated user and
+        send a successful response to the client.
+
+        @type result: C{tuple}
+        @param result: The first item of the tuple is a
+            C{zope.interface.Interface} which is the interface
+            supported by the avatar.  The second item of the tuple is a
+            L{IMailbox} provider which is the mailbox for the
+            authenticated user.  The third item of the tuple is a no-argument
+            callable which is a function to be invoked when the session is
+            terminated.
+
+        @type user: L{bytes}
+        @param user: The user being authenticated.
+        """
+        (interface, avatar, logout) = result
+        if interface is not IMailbox:
+            self.failResponse(b"Authentication failed")
+            log.err("_cbMailbox() called with an interface other than IMailbox")
+            return
+
+        self.mbox = avatar
+        self._onLogout = logout
+        self.successResponse("Authentication succeeded")
+        if getattr(self.factory, "noisy", True):
+            log.msg(b"Authenticated login for " + user)
+
+    def _ebMailbox(self, failure):
+        """
+        Handle an expected authentication failure.
+
+        Send an appropriate error response for a L{LoginDenied} or
+        L{LoginFailed} authentication failure.
+
+        @type failure: L{Failure}
+        @param failure: The authentication error.
+        """
+        failure = failure.trap(cred.error.LoginDenied, cred.error.LoginFailed)
+        if issubclass(failure, cred.error.LoginDenied):
+            self.failResponse("Access denied: " + str(failure))
+        elif issubclass(failure, cred.error.LoginFailed):
+            self.failResponse(b"Authentication failed")
+        if getattr(self.factory, "noisy", True):
+            log.msg("Denied login attempt from " + str(self.transport.getPeer()))
+
+    def _ebUnexpected(self, failure):
+        """
+        Handle an unexpected authentication failure.
+
+        Send an error response for an unexpected authentication failure.
+
+        @type failure: L{Failure}
+        @param failure: The authentication error.
+        """
+        self.failResponse("Server error: " + failure.getErrorMessage())
+        log.err(failure)
+
+    def do_USER(self, user):
+        """
+        Handle a USER command.
+
+        Save the username and send a successful response prompting the client
+        for the password.
+
+        @type user: L{bytes}
+        @param user: A username.
+        """
+        self._userIs = user
+        self.successResponse(b"USER accepted, send PASS")
+
+    def do_PASS(self, password, *words):
+        """
+        Handle a PASS command.
+
+        If a USER command was previously received, authenticate the user and
+        complete the authorization process with the L{_cbMailbox} callback
+        function on success or the L{_ebMailbox} and L{_ebUnexpected} errback
+        functions on failure.  If a USER command was not previously received,
+        send an error response.
+
+        @type password: L{bytes}
+        @param password: A password.
+
+        @type words: L{tuple} of L{bytes}
+        @param words: Other parts of the password split by spaces.
+        """
+        if self._userIs is None:
+            self.failResponse(b"USER required before PASS")
+            return
+        user = self._userIs
+        self._userIs = None
+        password = b" ".join((password,) + words)
+        d = defer.maybeDeferred(self.authenticateUserPASS, user, password)
+        d.addCallbacks(
+            self._cbMailbox, self._ebMailbox, callbackArgs=(user,)
+        ).addErrback(self._ebUnexpected)
+
+    def _longOperation(self, d):
+        """
+        Stop timeouts and block further command processing while a long
+        operation completes.
+
+        @type d: L{Deferred <defer.Deferred>}
+        @param d: A deferred which triggers at the completion of a long
+            operation.
+
+        @rtype: L{Deferred <defer.Deferred>}
+        @return: A deferred which triggers after command processing resumes and
+            timeouts restart after the completion of a long operation.
+        """
+        timeOut = self.timeOut
+        self.setTimeout(None)
+        self.blocked = []
+        d.addCallback(self._unblock)
+        d.addCallback(lambda ign: self.setTimeout(timeOut))
+        return d
+
+    def _coiterate(self, gen):
+        """
+        Direct the output of an iterator to the transport and arrange for
+        iteration to take place.
+
+        @type gen: iterable which yields L{bytes}
+        @param gen: An iterator over strings.
+
+        @rtype: L{Deferred <defer.Deferred>}
+        @return: A deferred which fires when the iterator finishes.
+        """
+        return self.schedule(_IteratorBuffer(self.transport.writeSequence, gen))
+
+    def do_STAT(self):
+        """
+        Handle a STAT command.
+
+        @rtype: L{Deferred <defer.Deferred>}
+        @return: A deferred which triggers after the response to the STAT
+            command has been issued.
+        """
+        d = defer.maybeDeferred(self.mbox.listMessages)
+
+        def cbMessages(msgs):
+            return self._coiterate(formatStatResponse(msgs))
+
+        def ebMessages(err):
+            self.failResponse(err.getErrorMessage())
+            log.msg("Unexpected do_STAT failure:")
+            log.err(err)
+
+        return self._longOperation(d.addCallbacks(cbMessages, ebMessages))
+
+    def do_LIST(self, i=None):
+        """
+        Handle a LIST command.
+
+        @type i: L{bytes} or L{None}
+        @param i: A 1-based message index.
+
+        @rtype: L{Deferred <defer.Deferred>}
+        @return: A deferred which triggers after the response to the LIST
+            command has been issued.
+        """
+        if i is None:
+            d = defer.maybeDeferred(self.mbox.listMessages)
+
+            def cbMessages(msgs):
+                return self._coiterate(formatListResponse(msgs))
+
+            def ebMessages(err):
+                self.failResponse(err.getErrorMessage())
+                log.msg("Unexpected do_LIST failure:")
+                log.err(err)
+
+            return self._longOperation(d.addCallbacks(cbMessages, ebMessages))
+        else:
+            try:
+                i = int(i)
+                if i < 1:
+                    raise ValueError()
+            except ValueError:
+                if not isinstance(i, bytes):
+                    i = str(i).encode("utf-8")
+                self.failResponse(b"Invalid message-number: " + i)
+            else:
+                d = defer.maybeDeferred(self.mbox.listMessages, i - 1)
+
+                def cbMessage(msg):
+                    self.successResponse(b"%d %d" % (i, msg))
+
+                def ebMessage(err):
+                    errcls = err.check(ValueError, IndexError)
+                    if errcls is not None:
+                        if errcls is IndexError:
+                            # IndexError was supported for a while, but really
+                            # shouldn't be.  One error condition, one exception
+                            # type.  See ticket #6669.
+                            warnings.warn(
+                                "twisted.mail.pop3.IMailbox.listMessages may "
+                                "not raise IndexError for out-of-bounds "
+                                "message numbers: raise ValueError instead.",
+                                PendingDeprecationWarning,
+                            )
+                        invalidNum = i
+                        if invalidNum and not isinstance(invalidNum, bytes):
+                            invalidNum = str(invalidNum).encode("utf-8")
+                        self.failResponse(b"Invalid message-number: " + invalidNum)
+                    else:
+                        self.failResponse(err.getErrorMessage())
+                        log.msg("Unexpected do_LIST failure:")
+                        log.err(err)
+
+                d.addCallbacks(cbMessage, ebMessage)
+                return self._longOperation(d)
+
+    def do_UIDL(self, i=None):
+        """
+        Handle a UIDL command.
+
+        @type i: L{bytes} or L{None}
+        @param i: A 1-based message index.
+
+        @rtype: L{Deferred <defer.Deferred>}
+        @return: A deferred which triggers after the response to the UIDL
+            command has been issued.
+        """
+        if i is None:
+            d = defer.maybeDeferred(self.mbox.listMessages)
+
+            def cbMessages(msgs):
+                return self._coiterate(
+                    formatUIDListResponse(msgs, self.mbox.getUidl),
+                )
+
+            def ebMessages(err):
+                self.failResponse(err.getErrorMessage())
+                log.msg("Unexpected do_UIDL failure:")
+                log.err(err)
+
+            return self._longOperation(d.addCallbacks(cbMessages, ebMessages))
+        else:
+            try:
+                i = int(i)
+                if i < 1:
+                    raise ValueError()
+            except ValueError:
+                self.failResponse("Bad message number argument")
+            else:
+                try:
+                    msg = self.mbox.getUidl(i - 1)
+                except IndexError:
+                    # XXX TODO See above comment regarding IndexError.
+                    warnings.warn(
+                        "twisted.mail.pop3.IMailbox.getUidl may not "
+                        "raise IndexError for out-of-bounds message numbers: "
+                        "raise ValueError instead.",
+                        PendingDeprecationWarning,
+                    )
+                    self.failResponse("Bad message number argument")
+                except ValueError:
+                    self.failResponse("Bad message number argument")
+                else:
+                    if not isinstance(msg, bytes):
+                        msg = str(msg).encode("utf-8")
+                    self.successResponse(msg)
+
+    def _getMessageFile(self, i):
+        """
+        Retrieve the size and contents of a message.
+
+        @type i: L{bytes}
+        @param i: A 1-based message index.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            2-L{tuple} of (E{1}) L{int}, (E{2}) file-like object
+        @return: A deferred which successfully fires with the size of the
+            message and a file containing the contents of the message.
+        """
+        try:
+            msg = int(i) - 1
+            if msg < 0:
+                raise ValueError()
+        except ValueError:
+            self.failResponse("Bad message number argument")
+            return defer.succeed(None)
+
+        sizeDeferred = defer.maybeDeferred(self.mbox.listMessages, msg)
+
+        def cbMessageSize(size):
+            if not size:
+                return defer.fail(_POP3MessageDeleted())
+            fileDeferred = defer.maybeDeferred(self.mbox.getMessage, msg)
+            fileDeferred.addCallback(lambda fObj: (size, fObj))
+            return fileDeferred
+
+        def ebMessageSomething(err):
+            errcls = err.check(_POP3MessageDeleted, ValueError, IndexError)
+            if errcls is _POP3MessageDeleted:
+                self.failResponse("message deleted")
+            elif errcls in (ValueError, IndexError):
+                if errcls is IndexError:
+                    # XXX TODO See above comment regarding IndexError.
+                    warnings.warn(
+                        "twisted.mail.pop3.IMailbox.listMessages may not "
+                        "raise IndexError for out-of-bounds message numbers: "
+                        "raise ValueError instead.",
+                        PendingDeprecationWarning,
+                    )
+                self.failResponse("Bad message number argument")
+            else:
+                log.msg("Unexpected _getMessageFile failure:")
+                log.err(err)
+            return None
+
+        sizeDeferred.addCallback(cbMessageSize)
+        sizeDeferred.addErrback(ebMessageSomething)
+        return sizeDeferred
+
+    def _sendMessageContent(self, i, fpWrapper, successResponse):
+        """
+        Send the contents of a message.
+
+        @type i: L{bytes}
+        @param i: A 1-based message index.
+
+        @type fpWrapper: callable that takes a file-like object and returns
+            a file-like object
+        @param fpWrapper:
+
+        @type successResponse: callable that takes L{int} and returns
+            L{bytes}
+        @param successResponse:
+
+        @rtype: L{Deferred}
+        @return: A deferred which triggers after the message has been sent.
+        """
+        d = self._getMessageFile(i)
+
+        def cbMessageFile(info):
+            if info is None:
+                # Some error occurred - a failure response has been sent
+                # already, just give up.
+                return
+
+            self._highest = max(self._highest, int(i))
+            resp, fp = info
+            fp = fpWrapper(fp)
+            self.successResponse(successResponse(resp))
+            s = basic.FileSender()
+            d = s.beginFileTransfer(fp, self.transport, self.transformChunk)
+
+            def cbFileTransfer(lastsent):
+                if lastsent != b"\n":
+                    line = b"\r\n."
+                else:
+                    line = b"."
+                self.sendLine(line)
+
+            def ebFileTransfer(err):
+                self.transport.loseConnection()
+                log.msg("Unexpected error in _sendMessageContent:")
+                log.err(err)
+
+            d.addCallback(cbFileTransfer)
+            d.addErrback(ebFileTransfer)
+            return d
+
+        return self._longOperation(d.addCallback(cbMessageFile))
+
+    def do_TOP(self, i, size):
+        """
+        Handle a TOP command.
+
+        @type i: L{bytes}
+        @param i: A 1-based message index.
+
+        @type size: L{bytes}
+        @param size: The number of lines of the message to retrieve.
+
+        @rtype: L{Deferred}
+        @return: A deferred which triggers after the response to the TOP
+            command has been issued.
+        """
+        try:
+            size = int(size)
+            if size < 0:
+                raise ValueError
+        except ValueError:
+            self.failResponse("Bad line count argument")
+        else:
+            return self._sendMessageContent(
+                i,
+                lambda fp: _HeadersPlusNLines(fp, size),
+                lambda size: "Top of message follows",
+            )
+
+    def do_RETR(self, i):
+        """
+        Handle a RETR command.
+
+        @type i: L{bytes}
+        @param i: A 1-based message index.
+
+        @rtype: L{Deferred}
+        @return: A deferred which triggers after the response to the RETR
+            command has been issued.
+        """
+        return self._sendMessageContent(i, lambda fp: fp, lambda size: "%d" % (size,))
+
+    def transformChunk(self, chunk):
+        """
+        Transform a chunk of a message to POP3 message format.
+
+        Make sure each line ends with C{'\\r\\n'} and byte-stuff the
+        termination character (C{'.'}) by adding an extra one when one appears
+        at the beginning of a line.
+
+        @type chunk: L{bytes}
+        @param chunk: A string to transform.
+
+        @rtype: L{bytes}
+        @return: The transformed string.
+        """
+        return chunk.replace(b"\n", b"\r\n").replace(b"\r\n.", b"\r\n..")
+
+    def finishedFileTransfer(self, lastsent):
+        """
+        Send the termination sequence.
+
+        @type lastsent: L{bytes}
+        @param lastsent: The last character of the file.
+        """
+        if lastsent != b"\n":
+            line = b"\r\n."
+        else:
+            line = b"."
+        self.sendLine(line)
+
+    def do_DELE(self, i):
+        """
+        Handle a DELE command.
+
+        Mark a message for deletion and issue a successful response.
+
+        @type i: L{int}
+        @param i: A 1-based message index.
+        """
+        i = int(i) - 1
+        self.mbox.deleteMessage(i)
+        self.successResponse()
+
+    def do_NOOP(self):
+        """
+        Handle a NOOP command.
+
+        Do nothing but issue a successful response.
+        """
+        self.successResponse()
+
+    def do_RSET(self):
+        """
+        Handle a RSET command.
+
+        Unmark any messages that have been flagged for deletion.
+        """
+        try:
+            self.mbox.undeleteMessages()
+        except BaseException:
+            log.err()
+            self.failResponse()
+        else:
+            self._highest = 0
+            self.successResponse()
+
+    def do_LAST(self):
+        """
+        Handle a LAST command.
+
+        Respond with the 1-based index of the highest retrieved message.
+        """
+        self.successResponse(self._highest)
+
+    def do_RPOP(self, user):
+        """
+        Handle an RPOP command.
+
+        RPOP is not supported.  Send an error response.
+
+        @type user: L{bytes}
+        @param user: A username.
+
+        """
+        self.failResponse("permission denied, sucker")
+
+    def do_QUIT(self):
+        """
+        Handle a QUIT command.
+
+        Remove any messages marked for deletion, issue a successful response,
+        and drop the connection.
+        """
+        if self.mbox:
+            self.mbox.sync()
+        self.successResponse()
+        self.transport.loseConnection()
+
+    def authenticateUserAPOP(self, user, digest):
+        """
+        Perform APOP authentication.
+
+        @type user: L{bytes}
+        @param user: The name of the user attempting to log in.
+
+        @type digest: L{bytes}
+        @param digest: The challenge response.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully results in
+            3-L{tuple} of (E{1}) L{IMailbox <pop3.IMailbox>}, (E{2})
+            L{IMailbox <pop3.IMailbox>} provider, (E{3}) no-argument callable
+        @return: A deferred which fires when authentication is complete.  If
+            successful, it returns an L{IMailbox <pop3.IMailbox>} interface, a
+            mailbox, and a function to be invoked with the session is
+            terminated.  If authentication fails, the deferred fails with an
+            L{UnathorizedLogin <cred.error.UnauthorizedLogin>} error.
+
+        @raise cred.error.UnauthorizedLogin: When authentication fails.
+        """
+        if self.portal is not None:
+            return self.portal.login(
+                APOPCredentials(self.magic, user, digest), None, IMailbox
+            )
+        raise cred.error.UnauthorizedLogin()
+
+    def authenticateUserPASS(self, user, password):
+        """
+        Perform authentication for a username/password login.
+
+        @type user: L{bytes}
+        @param user: The name of the user attempting to log in.
+
+        @type password: L{bytes}
+        @param password: The password to authenticate with.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully results in
+            3-L{tuple} of (E{1}) L{IMailbox <pop3.IMailbox>}, (E{2}) L{IMailbox
+            <pop3.IMailbox>} provider, (E{3}) no-argument callable
+        @return: A deferred which fires when authentication is complete.  If
+            successful, it returns a L{pop3.IMailbox} interface, a mailbox,
+            and a function to be invoked with the session is terminated.
+            If authentication fails, the deferred fails with an
+            L{UnathorizedLogin <cred.error.UnauthorizedLogin>} error.
+
+        @raise cred.error.UnauthorizedLogin: When authentication fails.
+        """
+        if self.portal is not None:
+            return self.portal.login(
+                cred.credentials.UsernamePassword(user, password), None, IMailbox
+            )
+        raise cred.error.UnauthorizedLogin()
+
+    def stopProducing(self):
+        # IProducer.stopProducing
+        raise NotImplementedError()
+
+
+@implementer(IMailbox)
+class Mailbox:
+    """
+    A base class for mailboxes.
+    """
+
+    def listMessages(self, i=None):
+        """
+        Retrieve the size of a message, or, if none is specified, the size of
+        each message in the mailbox.
+
+        @type i: L{int} or L{None}
+        @param i: The 0-based index of the message.
+
+        @rtype: L{int}, sequence of L{int}, or L{Deferred <defer.Deferred>}
+        @return: The number of octets in the specified message, or, if an
+            index is not specified, a sequence of the number of octets for
+            all messages in the mailbox or a deferred which fires with
+            one of those. Any value which corresponds to a deleted message
+            is set to 0.
+
+        @raise ValueError or IndexError: When the index does not correspond to
+            a message in the mailbox.  The use of ValueError is preferred.
+        """
+        return []
+
+    def getMessage(self, i):
+        """
+        Retrieve a file containing the contents of a message.
+
+        @type i: L{int}
+        @param i: The 0-based index of a message.
+
+        @rtype: file-like object
+        @return: A file containing the message.
+
+        @raise ValueError or IndexError: When the index does not correspond to
+            a message in the mailbox.  The use of ValueError is preferred.
+        """
+        raise ValueError
+
+    def getUidl(self, i):
+        """
+        Get a unique identifier for a message.
+
+        @type i: L{int}
+        @param i: The 0-based index of a message.
+
+        @rtype: L{bytes}
+        @return: A string of printable characters uniquely identifying the
+            message for all time.
+
+        @raise ValueError or IndexError: When the index does not correspond to
+            a message in the mailbox.  The use of ValueError is preferred.
+        """
+        raise ValueError
+
+    def deleteMessage(self, i):
+        """
+        Mark a message for deletion.
+
+        This must not change the number of messages in this mailbox.  Further
+        requests for the size of the deleted message should return 0.  Further
+        requests for the message itself may raise an exception.
+
+        @type i: L{int}
+        @param i: The 0-based index of a message.
+
+        @raise ValueError or IndexError: When the index does not correspond to
+            a message in the mailbox.  The use of ValueError is preferred.
+        """
+        raise ValueError
+
+    def undeleteMessages(self):
+        """
+        Undelete all messages marked for deletion.
+
+        Any message which can be undeleted should be returned to its original
+        position in the message sequence and retain its original UID.
+        """
+        pass
+
+    def sync(self):
+        """
+        Discard the contents of any message marked for deletion.
+        """
+        pass
+
+
+NONE, SHORT, FIRST_LONG, LONG = range(4)
+
+NEXT = {}
+NEXT[NONE] = NONE
+NEXT[SHORT] = NONE
+NEXT[FIRST_LONG] = LONG
+NEXT[LONG] = NONE
+
+
+class POP3Client(basic.LineOnlyReceiver):
+    """
+    A POP3 client protocol.
+
+    @type mode: L{int}
+    @ivar mode: The type of response expected from the server.  Choices include
+    none (0), a one line response (1), the first line of a multi-line
+    response (2), and subsequent lines of a multi-line response (3).
+
+    @type command: L{bytes}
+    @ivar command: The command most recently sent to the server.
+
+    @type welcomeRe: L{Pattern <re.Pattern.search>}
+    @ivar welcomeRe: A regular expression which matches the APOP challenge in
+        the server greeting.
+
+    @type welcomeCode: L{bytes}
+    @ivar welcomeCode: The APOP challenge passed in the server greeting.
+    """
+
+    mode = SHORT
+    command = b"WELCOME"
+    import re
+
+    welcomeRe = re.compile(b"<(.*)>")
+
+    def __init__(self):
+        """
+        Issue deprecation warning.
+        """
+        import warnings
+
+        warnings.warn(
+            "twisted.mail.pop3.POP3Client is deprecated, "
+            "please use twisted.mail.pop3.AdvancedPOP3Client "
+            "instead.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+
+    def sendShort(self, command, params=None):
+        """
+        Send a POP3 command to which a short response is expected.
+
+        @type command: L{bytes}
+        @param command: A POP3 command.
+
+        @type params: stringifyable L{object} or L{None}
+        @param params: Command arguments.
+        """
+        if params is not None:
+            if not isinstance(params, bytes):
+                params = str(params).encode("utf-8")
+            self.sendLine(command + b" " + params)
+        else:
+            self.sendLine(command)
+        self.command = command
+        self.mode = SHORT
+
+    def sendLong(self, command, params):
+        """
+        Send a POP3 command to which a long response is expected.
+
+        @type command: L{bytes}
+        @param command: A POP3 command.
+
+        @type params: stringifyable L{object}
+        @param params: Command arguments.
+        """
+        if params:
+            if not isinstance(params, bytes):
+                params = str(params).encode("utf-8")
+            self.sendLine(command + b" " + params)
+        else:
+            self.sendLine(command)
+        self.command = command
+        self.mode = FIRST_LONG
+
+    def handle_default(self, line):
+        """
+        Handle responses from the server for which no other handler exists.
+
+        @type line: L{bytes}
+        @param line: A received line.
+        """
+        if line[:-4] == b"-ERR":
+            self.mode = NONE
+
+    def handle_WELCOME(self, line):
+        """
+        Handle a server response which is expected to be a server greeting.
+
+        @type line: L{bytes}
+        @param line: A received line.
+        """
+        code, data = line.split(b" ", 1)
+        if code != b"+OK":
+            self.transport.loseConnection()
+        else:
+            m = self.welcomeRe.match(line)
+            if m:
+                self.welcomeCode = m.group(1)
+
+    def _dispatch(self, command, default, *args):
+        """
+        Dispatch a response from the server for handling.
+
+        Command X is dispatched to handle_X() if it exists.  If not, it is
+        dispatched to the default handler.
+
+        @type command: L{bytes}
+        @param command: The command.
+
+        @type default: callable that takes L{bytes} or
+            L{None}
+        @param default: The default handler.
+
+        @type args: L{tuple} or L{None}
+        @param args: Arguments to the handler function.
+        """
+        try:
+            method = getattr(self, "handle_" + command.decode("utf-8"), default)
+            if method is not None:
+                method(*args)
+        except BaseException:
+            log.err()
+
+    def lineReceived(self, line):
+        """
+        Dispatch a received line for processing.
+
+        The choice of function to handle the received line is based on the
+        type of response expected to the command sent to the server and how
+        much of that response has been received.
+
+        An expected one line response to command X is handled by handle_X().
+        The first line of a multi-line response to command X is also handled by
+        handle_X().  Subsequent lines of the multi-line response are handled by
+        handle_X_continue() except for the last line which is handled by
+        handle_X_end().
+
+        @type line: L{bytes}
+        @param line: A received line.
+        """
+        if self.mode == SHORT or self.mode == FIRST_LONG:
+            self.mode = NEXT[self.mode]
+            self._dispatch(self.command, self.handle_default, line)
+        elif self.mode == LONG:
+            if line == b".":
+                self.mode = NEXT[self.mode]
+                self._dispatch(self.command + b"_end", None)
+                return
+            if line[:1] == b".":
+                line = line[1:]
+            self._dispatch(self.command + b"_continue", None, line)
+
+    def apopAuthenticate(self, user, password, magic):
+        """
+        Perform an authenticated login.
+
+        @type user: L{bytes}
+        @param user: The username with which to log in.
+
+        @type password: L{bytes}
+        @param password: The password with which to log in.
+
+        @type magic: L{bytes}
+        @param magic: The challenge provided by the server.
+        """
+        digest = md5(magic + password).hexdigest().encode("ascii")
+        self.apop(user, digest)
+
+    def apop(self, user, digest):
+        """
+        Send an APOP command to perform authenticated login.
+
+        @type user: L{bytes}
+        @param user: The username with which to log in.
+
+        @type digest: L{bytes}
+        @param digest: The challenge response with which to authenticate.
+        """
+        self.sendLong(b"APOP", b" ".join((user, digest)))
+
+    def retr(self, i):
+        """
+        Send a RETR command to retrieve a message from the server.
+
+        @type i: L{int} or L{bytes}
+        @param i: A 0-based message index.
+        """
+        self.sendLong(b"RETR", i)
+
+    def dele(self, i):
+        """
+        Send a DELE command to delete a message from the server.
+
+        @type i: L{int} or L{bytes}
+        @param i: A 0-based message index.
+        """
+        self.sendShort(b"DELE", i)
+
+    def list(self, i=""):
+        """
+        Send a LIST command to retrieve the size of a message or, if no message
+        is specified, the sizes of all messages.
+
+        @type i: L{int} or L{bytes}
+        @param i: A 0-based message index or the empty string to specify all
+            messages.
+        """
+        self.sendLong(b"LIST", i)
+
+    def uidl(self, i=""):
+        """
+        Send a UIDL command to retrieve the unique identifier of a message or,
+        if no message is specified, the unique identifiers of all messages.
+
+        @type i: L{int} or L{bytes}
+        @param i: A 0-based message index or the empty string to specify all
+            messages.
+        """
+        self.sendLong(b"UIDL", i)
+
+    def user(self, name):
+        """
+        Send a USER command to perform the first half of a plaintext login.
+
+        @type name: L{bytes}
+        @param name: The username with which to log in.
+        """
+        self.sendShort(b"USER", name)
+
+    def password(self, password):
+        """
+        Perform the second half of a plaintext login.
+
+        @type password: L{bytes}
+        @param password: The plaintext password with which to authenticate.
+        """
+        self.sendShort(b"PASS", password)
+
+    pass_ = password
+
+    def quit(self):
+        """
+        Send a QUIT command to disconnect from the server.
+        """
+        self.sendShort(b"QUIT")
+
+
+from twisted.mail._except import (
+    InsecureAuthenticationDisallowed,
+    LineTooLong,
+    ServerErrorResponse,
+    TLSError,
+    TLSNotSupportedError,
+)
+from twisted.mail._pop3client import POP3Client as AdvancedPOP3Client
+
+__all__ = [
+    # Interfaces
+    "IMailbox",
+    "IServerFactory",
+    # Exceptions
+    "POP3Error",
+    "POP3ClientError",
+    "InsecureAuthenticationDisallowed",
+    "ServerErrorResponse",
+    "LineTooLong",
+    "TLSError",
+    "TLSNotSupportedError",
+    # Protocol classes
+    "POP3",
+    "POP3Client",
+    "AdvancedPOP3Client",
+    # Misc
+    "APOPCredentials",
+    "Mailbox",
+]
diff --git a/contrib/python/Twisted/py3/twisted/mail/pop3client.py b/contrib/python/Twisted/py3/twisted/mail/pop3client.py
new file mode 100644
index 00000000000..72676a69a81
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/pop3client.py
@@ -0,0 +1,22 @@
+"""
+Deprecated POP3 client protocol implementation.
+
+Don't use this module directly.  Use twisted.mail.pop3 instead.
+"""
+import warnings
+from typing import List
+
+from twisted.mail._pop3client import ERR, OK, POP3Client
+
+warnings.warn(
+    "twisted.mail.pop3client was deprecated in Twisted 21.2.0. Use twisted.mail.pop3 instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+# Fake usage to please pyflakes as we don't to add them to __all__.
+OK
+ERR
+POP3Client
+
+__all__: List[str] = []
diff --git a/contrib/python/Twisted/py3/twisted/mail/protocols.py b/contrib/python/Twisted/py3/twisted/mail/protocols.py
new file mode 100644
index 00000000000..7bd1eebbee7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/protocols.py
@@ -0,0 +1,385 @@
+# -*- test-case-name: twisted.mail.test.test_mail -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Mail protocol support.
+"""
+
+
+from zope.interface import implementer
+
+from twisted.copyright import longversion
+from twisted.cred.credentials import CramMD5Credentials, UsernamePassword
+from twisted.cred.error import UnauthorizedLogin
+from twisted.internet import defer, protocol
+from twisted.mail import pop3, relay, smtp
+from twisted.python import log
+
+
+@implementer(smtp.IMessageDelivery)
+class DomainDeliveryBase:
+    """
+    A base class for message delivery using the domains of a mail service.
+
+    @ivar service: See L{__init__}
+    @ivar user: See L{__init__}
+    @ivar host: See L{__init__}
+
+    @type protocolName: L{bytes}
+    @ivar protocolName: The protocol being used to deliver the mail.
+        Sub-classes should set this appropriately.
+    """
+
+    service = None
+    protocolName: bytes = b"not-implemented-protocol"
+
+    def __init__(self, service, user, host=smtp.DNSNAME):
+        """
+        @type service: L{MailService}
+        @param service: A mail service.
+
+        @type user: L{bytes} or L{None}
+        @param user: The authenticated SMTP user.
+
+        @type host: L{bytes}
+        @param host: The hostname.
+        """
+        self.service = service
+        self.user = user
+        self.host = host
+
+    def receivedHeader(self, helo, origin, recipients):
+        """
+        Generate a received header string for a message.
+
+        @type helo: 2-L{tuple} of (L{bytes}, L{bytes})
+        @param helo: The client's identity as sent in the HELO command and its
+            IP address.
+
+        @type origin: L{Address}
+        @param origin: The origination address of the message.
+
+        @type recipients: L{list} of L{User}
+        @param recipients: The destination addresses for the message.
+
+        @rtype: L{bytes}
+        @return: A received header string.
+        """
+        authStr = heloStr = b""
+        if self.user:
+            authStr = b" auth=" + self.user.encode("xtext")
+        if helo[0]:
+            heloStr = b" helo=" + helo[0]
+        fromUser = b"from " + helo[0] + b" ([" + helo[1] + b"]" + heloStr + authStr
+        by = (
+            b"by "
+            + self.host
+            + b" with "
+            + self.protocolName
+            + b" ("
+            + longversion.encode("ascii")
+            + b")"
+        )
+        forUser = (
+            b"for <" + b" ".join(map(bytes, recipients)) + b"> " + smtp.rfc822date()
+        )
+        return b"Received: " + fromUser + b"\n\t" + by + b"\n\t" + forUser
+
+    def validateTo(self, user):
+        """
+        Validate the address for which a message is destined.
+
+        @type user: L{User}
+        @param user: The destination address.
+
+        @rtype: L{Deferred <defer.Deferred>} which successfully fires with
+            no-argument callable which returns L{IMessage <smtp.IMessage>}
+            provider.
+        @return: A deferred which successfully fires with a no-argument
+            callable which returns a message receiver for the destination.
+
+        @raise SMTPBadRcpt: When messages cannot be accepted for the
+            destination address.
+        """
+        # XXX - Yick.  This needs cleaning up.
+        if self.user and self.service.queue:
+            d = self.service.domains.get(user.dest.domain, None)
+            if d is None:
+                d = relay.DomainQueuer(self.service, True)
+        else:
+            d = self.service.domains[user.dest.domain]
+        return defer.maybeDeferred(d.exists, user)
+
+    def validateFrom(self, helo, origin):
+        """
+        Validate the address from which a message originates.
+
+        @type helo: 2-L{tuple} of (L{bytes}, L{bytes})
+        @param helo: The client's identity as sent in the HELO command and its
+            IP address.
+
+        @type origin: L{Address}
+        @param origin: The origination address of the message.
+
+        @rtype: L{Address}
+        @return: The origination address.
+
+        @raise SMTPBadSender: When messages cannot be accepted from the
+            origination address.
+        """
+        if not helo:
+            raise smtp.SMTPBadSender(origin, 503, "Who are you?  Say HELO first.")
+        if origin.local != b"" and origin.domain == b"":
+            raise smtp.SMTPBadSender(origin, 501, "Sender address must contain domain.")
+        return origin
+
+
+class SMTPDomainDelivery(DomainDeliveryBase):
+    """
+    A domain delivery base class for use in an SMTP server.
+    """
+
+    protocolName = b"smtp"
+
+
+class ESMTPDomainDelivery(DomainDeliveryBase):
+    """
+    A domain delivery base class for use in an ESMTP server.
+    """
+
+    protocolName = b"esmtp"
+
+
+class SMTPFactory(smtp.SMTPFactory):
+    """
+    An SMTP server protocol factory.
+
+    @ivar service: See L{__init__}
+    @ivar portal: See L{__init__}
+
+    @type protocol: no-argument callable which returns a L{Protocol
+        <protocol.Protocol>} subclass
+    @ivar protocol: A callable which creates a protocol.  The default value is
+        L{SMTP}.
+    """
+
+    protocol = smtp.SMTP
+    portal = None
+
+    def __init__(self, service, portal=None):
+        """
+        @type service: L{MailService}
+        @param service: An email service.
+
+        @type portal: L{Portal <twisted.cred.portal.Portal>} or
+            L{None}
+        @param portal: A portal to use for authentication.
+        """
+        smtp.SMTPFactory.__init__(self)
+        self.service = service
+        self.portal = portal
+
+    def buildProtocol(self, addr):
+        """
+        Create an instance of an SMTP server protocol.
+
+        @type addr: L{IAddress <twisted.internet.interfaces.IAddress>} provider
+        @param addr: The address of the SMTP client.
+
+        @rtype: L{SMTP}
+        @return: An SMTP protocol.
+        """
+        log.msg(f"Connection from {addr}")
+        p = smtp.SMTPFactory.buildProtocol(self, addr)
+        p.service = self.service
+        p.portal = self.portal
+        return p
+
+
+class ESMTPFactory(SMTPFactory):
+    """
+    An ESMTP server protocol factory.
+
+    @type protocol: no-argument callable which returns a L{Protocol
+        <protocol.Protocol>} subclass
+    @ivar protocol: A callable which creates a protocol.  The default value is
+        L{ESMTP}.
+
+    @type context: L{IOpenSSLContextFactory
+        <twisted.internet.interfaces.IOpenSSLContextFactory>} or L{None}
+    @ivar context: A factory to generate contexts to be used in negotiating
+        encrypted communication.
+
+    @type challengers: L{dict} mapping L{bytes} to no-argument callable which
+        returns L{ICredentials <twisted.cred.credentials.ICredentials>}
+        subclass provider.
+    @ivar challengers: A mapping of acceptable authorization mechanism to
+        callable which creates credentials to use for authentication.
+    """
+
+    protocol = smtp.ESMTP
+    context = None
+
+    def __init__(self, *args):
+        """
+        @param args: Arguments for L{SMTPFactory.__init__}
+
+        @see: L{SMTPFactory.__init__}
+        """
+        SMTPFactory.__init__(self, *args)
+        self.challengers = {b"CRAM-MD5": CramMD5Credentials}
+
+    def buildProtocol(self, addr):
+        """
+        Create an instance of an ESMTP server protocol.
+
+        @type addr: L{IAddress <twisted.internet.interfaces.IAddress>} provider
+        @param addr: The address of the ESMTP client.
+
+        @rtype: L{ESMTP}
+        @return: An ESMTP protocol.
+        """
+        p = SMTPFactory.buildProtocol(self, addr)
+        p.challengers = self.challengers
+        p.ctx = self.context
+        return p
+
+
+class VirtualPOP3(pop3.POP3):
+    """
+    A virtual hosting POP3 server.
+
+    @type service: L{MailService}
+    @ivar service: The email service that created this server.  This must be
+        set by the service.
+
+    @type domainSpecifier: L{bytes}
+    @ivar domainSpecifier: The character to use to split an email address into
+        local-part and domain. The default is '@'.
+    """
+
+    service = None
+
+    domainSpecifier = b"@"  # Gaagh! I hate POP3. No standardized way
+    # to indicate user@host. '@' doesn't work
+    # with NS, e.g.
+
+    def authenticateUserAPOP(self, user, digest):
+        """
+        Perform APOP authentication.
+
+        Override the default lookup scheme to allow virtual domains.
+
+        @type user: L{bytes}
+        @param user: The name of the user attempting to log in.
+
+        @type digest: L{bytes}
+        @param digest: The challenge response.
+
+        @rtype: L{Deferred} which successfully results in 3-L{tuple} of
+            (L{IMailbox <pop3.IMailbox>}, L{IMailbox <pop3.IMailbox>}
+            provider, no-argument callable)
+        @return: A deferred which fires when authentication is complete.
+            If successful, it returns an L{IMailbox <pop3.IMailbox>} interface,
+            a mailbox and a logout function. If authentication fails, the
+            deferred fails with an L{UnauthorizedLogin
+            <twisted.cred.error.UnauthorizedLogin>} error.
+        """
+        user, domain = self.lookupDomain(user)
+        try:
+            portal = self.service.lookupPortal(domain)
+        except KeyError:
+            return defer.fail(UnauthorizedLogin())
+        else:
+            return portal.login(
+                pop3.APOPCredentials(self.magic, user, digest), None, pop3.IMailbox
+            )
+
+    def authenticateUserPASS(self, user, password):
+        """
+        Perform authentication for a username/password login.
+
+        Override the default lookup scheme to allow virtual domains.
+
+        @type user: L{bytes}
+        @param user: The name of the user attempting to log in.
+
+        @type password: L{bytes}
+        @param password: The password to authenticate with.
+
+        @rtype: L{Deferred} which successfully results in 3-L{tuple} of
+            (L{IMailbox <pop3.IMailbox>}, L{IMailbox <pop3.IMailbox>}
+            provider, no-argument callable)
+        @return: A deferred which fires when authentication is complete.
+            If successful, it returns an L{IMailbox <pop3.IMailbox>} interface,
+            a mailbox and a logout function. If authentication fails, the
+            deferred fails with an L{UnauthorizedLogin
+            <twisted.cred.error.UnauthorizedLogin>} error.
+        """
+        user, domain = self.lookupDomain(user)
+        try:
+            portal = self.service.lookupPortal(domain)
+        except KeyError:
+            return defer.fail(UnauthorizedLogin())
+        else:
+            return portal.login(UsernamePassword(user, password), None, pop3.IMailbox)
+
+    def lookupDomain(self, user):
+        """
+        Check whether a domain is among the virtual domains supported by the
+        mail service.
+
+        @type user: L{bytes}
+        @param user: An email address.
+
+        @rtype: 2-L{tuple} of (L{bytes}, L{bytes})
+        @return: The local part and the domain part of the email address if the
+            domain is supported.
+
+        @raise POP3Error: When the domain is not supported by the mail service.
+        """
+        try:
+            user, domain = user.split(self.domainSpecifier, 1)
+        except ValueError:
+            domain = b""
+        if domain not in self.service.domains:
+            raise pop3.POP3Error("no such domain {}".format(domain.decode("utf-8")))
+        return user, domain
+
+
+class POP3Factory(protocol.ServerFactory):
+    """
+    A POP3 server protocol factory.
+
+    @ivar service: See L{__init__}
+
+    @type protocol: no-argument callable which returns a L{Protocol
+        <protocol.Protocol>} subclass
+    @ivar protocol: A callable which creates a protocol.  The default value is
+        L{VirtualPOP3}.
+    """
+
+    protocol = VirtualPOP3
+    service = None
+
+    def __init__(self, service):
+        """
+        @type service: L{MailService}
+        @param service: An email service.
+        """
+        self.service = service
+
+    def buildProtocol(self, addr):
+        """
+        Create an instance of a POP3 server protocol.
+
+        @type addr: L{IAddress <twisted.internet.interfaces.IAddress>} provider
+        @param addr: The address of the POP3 client.
+
+        @rtype: L{POP3}
+        @return: A POP3 protocol.
+        """
+        p = protocol.ServerFactory.buildProtocol(self, addr)
+        p.service = self.service
+        return p
diff --git a/contrib/python/Twisted/py3/twisted/mail/relay.py b/contrib/python/Twisted/py3/twisted/mail/relay.py
new file mode 100644
index 00000000000..4ba50ea3780
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/relay.py
@@ -0,0 +1,164 @@
+# -*- test-case-name: twisted.mail.test.test_mail -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support for relaying mail.
+"""
+
+import os
+import pickle
+
+from twisted.internet.address import UNIXAddress
+from twisted.mail import smtp
+from twisted.python import log
+
+
+class DomainQueuer:
+    """
+    An SMTP domain which add messages to a queue intended for relaying.
+    """
+
+    def __init__(self, service, authenticated=False):
+        self.service = service
+        self.authed = authenticated
+
+    def exists(self, user):
+        """
+        Check whether mail can be relayed to a user.
+
+        @type user: L{User}
+        @param user: A user.
+
+        @rtype: no-argument callable which returns L{IMessage <smtp.IMessage>}
+            provider
+        @return: A function which takes no arguments and returns a message
+            receiver for the user.
+
+        @raise SMTPBadRcpt: When mail cannot be relayed to the user.
+        """
+        if self.willRelay(user.dest, user.protocol):
+            # The most cursor form of verification of the addresses
+            orig = filter(None, str(user.orig).split("@", 1))
+            dest = filter(None, str(user.dest).split("@", 1))
+            if len(orig) == 2 and len(dest) == 2:
+                return lambda: self.startMessage(user)
+        raise smtp.SMTPBadRcpt(user)
+
+    def willRelay(self, address, protocol):
+        """
+        Check whether we agree to relay.
+
+        The default is to relay for all connections over UNIX
+        sockets and all connections from localhost.
+        """
+        peer = protocol.transport.getPeer()
+        return self.authed or isinstance(peer, UNIXAddress) or peer.host == "127.0.0.1"
+
+    def startMessage(self, user):
+        """
+        Create an envelope and a message receiver for the relay queue.
+
+        @type user: L{User}
+        @param user: A user.
+
+        @rtype: L{IMessage <smtp.IMessage>}
+        @return: A message receiver.
+        """
+        queue = self.service.queue
+        envelopeFile, smtpMessage = queue.createNewMessage()
+        with envelopeFile:
+            log.msg(f"Queueing mail {str(user.orig)!r} -> {str(user.dest)!r}")
+            pickle.dump([str(user.orig), str(user.dest)], envelopeFile)
+        return smtpMessage
+
+
+class RelayerMixin:
+    # XXX - This is -totally- bogus
+    # It opens about a -hundred- -billion- files
+    # and -leaves- them open!
+
+    def loadMessages(self, messagePaths):
+        self.messages = []
+        self.names = []
+        for message in messagePaths:
+            with open(message + "-H", "rb") as fp:
+                messageContents = pickle.load(fp)
+            fp = open(message + "-D")
+            messageContents.append(fp)
+            self.messages.append(messageContents)
+            self.names.append(message)
+
+    def getMailFrom(self):
+        if not self.messages:
+            return None
+        return self.messages[0][0]
+
+    def getMailTo(self):
+        if not self.messages:
+            return None
+        return [self.messages[0][1]]
+
+    def getMailData(self):
+        if not self.messages:
+            return None
+        return self.messages[0][2]
+
+    def sentMail(self, code, resp, numOk, addresses, log):
+        """Since we only use one recipient per envelope, this
+        will be called with 0 or 1 addresses. We probably want
+        to do something with the error message if we failed.
+        """
+        if code in smtp.SUCCESS:
+            # At least one, i.e. all, recipients successfully delivered
+            os.remove(self.names[0] + "-D")
+            os.remove(self.names[0] + "-H")
+        del self.messages[0]
+        del self.names[0]
+
+
+class SMTPRelayer(RelayerMixin, smtp.SMTPClient):
+    """
+    A base class for SMTP relayers.
+    """
+
+    def __init__(self, messagePaths, *args, **kw):
+        """
+        @type messagePaths: L{list} of L{bytes}
+        @param messagePaths: The base filename for each message to be relayed.
+
+        @type args: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of
+            (0) L{bytes}, (1) L{int}
+        @param args: Positional arguments for L{SMTPClient.__init__}
+
+        @type kw: L{dict}
+        @param kw: Keyword arguments for L{SMTPClient.__init__}
+        """
+        smtp.SMTPClient.__init__(self, *args, **kw)
+        self.loadMessages(messagePaths)
+
+
+class ESMTPRelayer(RelayerMixin, smtp.ESMTPClient):
+    """
+    A base class for ESMTP relayers.
+    """
+
+    def __init__(self, messagePaths, *args, **kw):
+        """
+        @type messagePaths: L{list} of L{bytes}
+        @param messagePaths: The base filename for each message to be relayed.
+
+        @type args: 3-L{tuple} of (0) L{bytes}, (1) L{None} or
+            L{ClientContextFactory
+            <twisted.internet.ssl.ClientContextFactory>},
+            (2) L{bytes} or 4-L{tuple} of (0) L{bytes}, (1) L{None}
+            or L{ClientContextFactory
+            <twisted.internet.ssl.ClientContextFactory>}, (2) L{bytes},
+            (3) L{int}
+        @param args: Positional arguments for L{ESMTPClient.__init__}
+
+        @type kw: L{dict}
+        @param kw: Keyword arguments for L{ESMTPClient.__init__}
+        """
+        smtp.ESMTPClient.__init__(self, *args, **kw)
+        self.loadMessages(messagePaths)
diff --git a/contrib/python/Twisted/py3/twisted/mail/relaymanager.py b/contrib/python/Twisted/py3/twisted/mail/relaymanager.py
new file mode 100644
index 00000000000..18cc2878331
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/relaymanager.py
@@ -0,0 +1,1135 @@
+# -*- test-case-name: twisted.mail.test.test_mail -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Infrastructure for relaying mail through a smart host.
+
+Traditional peer-to-peer email has been increasingly replaced by smart host
+configurations.  Instead of sending mail directly to the recipient, a sender
+sends mail to a smart host.  The smart host finds the mail exchange server for
+the recipient and sends on the message.
+"""
+
+import email.utils
+import os
+import pickle
+import time
+from typing import Type
+
+from twisted.application import internet
+from twisted.internet import protocol
+from twisted.internet.defer import Deferred, DeferredList
+from twisted.internet.error import DNSLookupError
+from twisted.internet.protocol import connectionDone
+from twisted.mail import bounce, relay, smtp
+from twisted.python import log
+from twisted.python.failure import Failure
+
+
+class ManagedRelayerMixin:
+    """
+    SMTP Relayer which notifies a manager
+
+    Notify the manager about successful mail, failed mail
+    and broken connections
+    """
+
+    def __init__(self, manager):
+        self.manager = manager
+
+    @property
+    def factory(self):
+        return self._factory
+
+    @factory.setter
+    def factory(self, value):
+        self._factory = value
+
+    def sentMail(self, code, resp, numOk, addresses, log):
+        """
+        called when e-mail has been sent
+
+        we will always get 0 or 1 addresses.
+        """
+        message = self.names[0]
+        if code in smtp.SUCCESS:
+            self.manager.notifySuccess(self.factory, message)
+        else:
+            self.manager.notifyFailure(self.factory, message)
+        del self.messages[0]
+        del self.names[0]
+
+    def connectionLost(self, reason: Failure = connectionDone) -> None:
+        """
+        called when connection is broken
+
+        notify manager we will try to send no more e-mail
+        """
+        self.manager.notifyDone(self.factory)
+
+
+class SMTPManagedRelayer(ManagedRelayerMixin, relay.SMTPRelayer):  # type: ignore[misc]
+    """
+    An SMTP managed relayer.
+
+    This managed relayer is an SMTP client which is responsible for sending a
+    set of messages and keeping an attempt manager informed about its progress.
+
+    @type factory: L{SMTPManagedRelayerFactory}
+    @ivar factory: The factory that created this relayer.  This must be set by
+        the factory.
+    """
+
+    def __init__(self, messages, manager, *args, **kw):
+        """
+        @type messages: L{list} of L{bytes}
+        @param messages: The base filenames of messages to be relayed.
+
+        @type manager: L{_AttemptManager}
+        @param manager: An attempt manager.
+
+        @type args: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of
+            (0) L{bytes}, (1) L{int}
+        @param args: Positional arguments for L{SMTPClient.__init__}
+
+        @type kw: L{dict}
+        @param kw: Keyword arguments for L{SMTPClient.__init__}
+        """
+        ManagedRelayerMixin.__init__(self, manager)
+        relay.SMTPRelayer.__init__(self, messages, *args, **kw)
+
+
+class ESMTPManagedRelayer(ManagedRelayerMixin, relay.ESMTPRelayer):  # type: ignore[misc]
+    """
+    An ESMTP managed relayer.
+
+    This managed relayer is an ESMTP client which is responsible for sending a
+    set of messages and keeping an attempt manager informed about its progress.
+    """
+
+    def __init__(self, messages, manager, *args, **kw):
+        """
+        @type messages: L{list} of L{bytes}
+        @param messages: The base filenames of messages to be relayed.
+
+        @type manager: L{_AttemptManager}
+        @param manager: An attempt manager.
+
+        @type args: 3-L{tuple} of (0) L{bytes}, (1) L{None} or
+            L{ClientContextFactory
+            <twisted.internet.ssl.ClientContextFactory>}, (2) L{bytes} or
+            4-L{tuple} of (0) L{bytes}, (1) L{None} or
+            L{ClientContextFactory
+            <twisted.internet.ssl.ClientContextFactory>}, (2) L{bytes},
+            (3) L{int}
+        @param args: Positional arguments for L{ESMTPClient.__init__}
+
+        @type kw: L{dict}
+        @param kw: Keyword arguments for L{ESMTPClient.__init__}
+        """
+        ManagedRelayerMixin.__init__(self, manager)
+        relay.ESMTPRelayer.__init__(self, messages, *args, **kw)
+
+
+class SMTPManagedRelayerFactory(protocol.ClientFactory):
+    """
+    A factory to create an L{SMTPManagedRelayer}.
+
+    This factory creates a managed relayer which relays a set of messages over
+    SMTP and informs an attempt manager of its progress.
+
+    @ivar messages: See L{__init__}
+    @ivar manager: See L{__init__}
+
+    @type protocol: callable which returns L{SMTPManagedRelayer}
+    @ivar protocol: A callable which returns a managed relayer for SMTP.  See
+        L{SMTPManagedRelayer.__init__} for parameters to the callable.
+
+    @type pArgs: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of
+        (0) L{bytes}, (1), L{int}
+    @ivar pArgs: Positional arguments for L{SMTPClient.__init__}
+
+    @type pKwArgs: L{dict}
+    @ivar pKwArgs: Keyword arguments for L{SMTPClient.__init__}
+    """
+
+    protocol: "Type[protocol.Protocol]" = SMTPManagedRelayer
+
+    def __init__(self, messages, manager, *args, **kw):
+        """
+        @type messages: L{list} of L{bytes}
+        @param messages: The base filenames of messages to be relayed.
+
+        @type manager: L{_AttemptManager}
+        @param manager: An attempt manager.
+
+        @type args: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of
+            (0) L{bytes}, (1), L{int}
+        @param args: Positional arguments for L{SMTPClient.__init__}
+
+        @type kw: L{dict}
+        @param kw: Keyword arguments for L{SMTPClient.__init__}
+        """
+        self.messages = messages
+        self.manager = manager
+        self.pArgs = args
+        self.pKwArgs = kw
+
+    def buildProtocol(self, addr):
+        """
+        Create an L{SMTPManagedRelayer}.
+
+        @type addr: L{IAddress <twisted.internet.interfaces.IAddress>} provider
+        @param addr: The address of the SMTP server.
+
+        @rtype: L{SMTPManagedRelayer}
+        @return: A managed relayer for SMTP.
+        """
+        protocol = self.protocol(
+            self.messages, self.manager, *self.pArgs, **self.pKwArgs
+        )
+        protocol.factory = self
+        return protocol
+
+    def clientConnectionFailed(self, connector, reason):
+        """
+        Notify the attempt manager that a connection could not be established.
+
+        @type connector: L{IConnector <twisted.internet.interfaces.IConnector>}
+            provider
+        @param connector: A connector.
+
+        @type reason: L{Failure}
+        @param reason: The reason the connection attempt failed.
+        """
+        self.manager.notifyNoConnection(self)
+        self.manager.notifyDone(self)
+
+
+class ESMTPManagedRelayerFactory(SMTPManagedRelayerFactory):
+    """
+    A factory to create an L{ESMTPManagedRelayer}.
+
+    This factory creates a managed relayer which relays a set of messages over
+    ESMTP and informs an attempt manager of its progress.
+
+    @type protocol: callable which returns L{ESMTPManagedRelayer}
+    @ivar protocol: A callable which returns a managed relayer for ESMTP.  See
+        L{ESMTPManagedRelayer.__init__} for parameters to the callable.
+
+    @ivar secret: See L{__init__}
+    @ivar contextFactory: See L{__init__}
+    """
+
+    protocol = ESMTPManagedRelayer
+
+    def __init__(self, messages, manager, secret, contextFactory, *args, **kw):
+        """
+        @type messages: L{list} of L{bytes}
+        @param messages: The base filenames of messages to be relayed.
+
+        @type manager: L{_AttemptManager}
+        @param manager: An attempt manager.
+
+        @type secret: L{bytes}
+        @param secret: A string for the authentication challenge response.
+
+        @type contextFactory: L{None} or
+            L{ClientContextFactory <twisted.internet.ssl.ClientContextFactory>}
+        @param contextFactory: An SSL context factory.
+
+        @type args: 1-L{tuple} of (0) L{bytes} or 2-L{tuple} of
+            (0) L{bytes}, (1), L{int}
+        @param args: Positional arguments for L{SMTPClient.__init__}
+
+        @param kw: Keyword arguments for L{SMTPClient.__init__}
+        """
+        self.secret = secret
+        self.contextFactory = contextFactory
+        SMTPManagedRelayerFactory.__init__(self, messages, manager, *args, **kw)
+
+    def buildProtocol(self, addr):
+        """
+        Create an L{ESMTPManagedRelayer}.
+
+        @type addr: L{IAddress <twisted.internet.interfaces.IAddress>} provider
+        @param addr: The address of the ESMTP server.
+
+        @rtype: L{ESMTPManagedRelayer}
+        @return: A managed relayer for ESMTP.
+        """
+        s = self.secret and self.secret(addr)
+        protocol = self.protocol(
+            self.messages,
+            self.manager,
+            s,
+            self.contextFactory,
+            *self.pArgs,
+            **self.pKwArgs,
+        )
+        protocol.factory = self
+        return protocol
+
+
+class Queue:
+    """
+    A queue for messages to be relayed.
+
+    @ivar directory: See L{__init__}
+
+    @type n: L{int}
+    @ivar n: A number used to form unique filenames.
+
+    @type waiting: L{dict} of L{bytes}
+    @ivar waiting: The base filenames of messages waiting to be relayed.
+
+    @type relayed: L{dict} of L{bytes}
+    @ivar relayed: The base filenames of messages in the process of being
+        relayed.
+
+    @type noisy: L{bool}
+    @ivar noisy: A flag which determines whether informational log messages
+        will be generated (C{True}) or not (C{False}).
+    """
+
+    noisy = True
+
+    def __init__(self, directory):
+        """
+        Initialize non-volatile state.
+
+        @type directory: L{bytes}
+        @param directory: The pathname of the directory holding messages in the
+            queue.
+        """
+        self.directory = directory
+        self._init()
+
+    def _init(self):
+        """
+        Initialize volatile state.
+        """
+        self.n = 0
+        self.waiting = {}
+        self.relayed = {}
+        self.readDirectory()
+
+    def __getstate__(self):
+        """
+        Create a representation of the non-volatile state of the queue.
+
+        @rtype: L{dict} mapping L{bytes} to L{object}
+        @return: The non-volatile state of the queue.
+        """
+        return {"directory": self.directory}
+
+    def __setstate__(self, state):
+        """
+        Restore the non-volatile state of the queue and recreate the volatile
+        state.
+
+        @type state: L{dict} mapping L{bytes} to L{object}
+        @param state: The non-volatile state of the queue.
+        """
+        self.__dict__.update(state)
+        self._init()
+
+    def readDirectory(self):
+        """
+        Scan the message directory for new messages.
+        """
+        for message in os.listdir(self.directory):
+            # Skip non data files
+            if message[-2:] != "-D":
+                continue
+            self.addMessage(message[:-2])
+
+    def getWaiting(self):
+        """
+        Return the base filenames of messages waiting to be relayed.
+
+        @rtype: L{list} of L{bytes}
+        @return: The base filenames of messages waiting to be relayed.
+        """
+        return self.waiting.keys()
+
+    def hasWaiting(self):
+        """
+        Return an indication of whether the queue has messages waiting to be
+        relayed.
+
+        @rtype: L{bool}
+        @return: C{True} if messages are waiting to be relayed.  C{False}
+            otherwise.
+        """
+        return len(self.waiting) > 0
+
+    def getRelayed(self):
+        """
+        Return the base filenames of messages in the process of being relayed.
+
+        @rtype: L{list} of L{bytes}
+        @return: The base filenames of messages in the process of being
+            relayed.
+        """
+        return self.relayed.keys()
+
+    def setRelaying(self, message):
+        """
+        Mark a message as being relayed.
+
+        @type message: L{bytes}
+        @param message: The base filename of a message.
+        """
+        del self.waiting[message]
+        self.relayed[message] = 1
+
+    def setWaiting(self, message):
+        """
+        Mark a message as waiting to be relayed.
+
+        @type message: L{bytes}
+        @param message: The base filename of a message.
+        """
+        del self.relayed[message]
+        self.waiting[message] = 1
+
+    def addMessage(self, message):
+        """
+        Mark a message as waiting to be relayed unless it is in the process of
+        being relayed.
+
+        @type message: L{bytes}
+        @param message: The base filename of a message.
+        """
+        if message not in self.relayed:
+            self.waiting[message] = 1
+            if self.noisy:
+                log.msg("Set " + message + " waiting")
+
+    def done(self, message):
+        """
+        Remove a message from the queue.
+
+        @type message: L{bytes}
+        @param message: The base filename of a message.
+        """
+        message = os.path.basename(message)
+        os.remove(self.getPath(message) + "-D")
+        os.remove(self.getPath(message) + "-H")
+        del self.relayed[message]
+
+    def getPath(self, message):
+        """
+        Return the full base pathname of a message in the queue.
+
+        @type message: L{bytes}
+        @param message: The base filename of a message.
+
+        @rtype: L{bytes}
+        @return: The full base pathname of the message.
+        """
+        return os.path.join(self.directory, message)
+
+    def getEnvelope(self, message):
+        """
+        Get the envelope for a message.
+
+        @type message: L{bytes}
+        @param message: The base filename of a message.
+
+        @rtype: L{list} of two L{bytes}
+        @return: A list containing the origination and destination addresses
+            for the message.
+        """
+        with self.getEnvelopeFile(message) as f:
+            return pickle.load(f)
+
+    def getEnvelopeFile(self, message):
+        """
+        Return the envelope file for a message in the queue.
+
+        @type message: L{bytes}
+        @param message: The base filename of a message.
+
+        @rtype: file
+        @return: The envelope file for the message.
+        """
+        return open(os.path.join(self.directory, message + "-H"), "rb")
+
+    def createNewMessage(self):
+        """
+        Create a new message in the queue.
+
+        @rtype: 2-L{tuple} of (0) file, (1) L{FileMessage}
+        @return: The envelope file and a message receiver for a new message in
+            the queue.
+        """
+        fname = f"{os.getpid()}_{time.time()}_{self.n}_{id(self)}"
+        self.n = self.n + 1
+        headerFile = open(os.path.join(self.directory, fname + "-H"), "wb")
+        tempFilename = os.path.join(self.directory, fname + "-C")
+        finalFilename = os.path.join(self.directory, fname + "-D")
+        messageFile = open(tempFilename, "wb")
+
+        from twisted.mail.mail import FileMessage
+
+        return headerFile, FileMessage(messageFile, tempFilename, finalFilename)
+
+
+class _AttemptManager:
+    """
+    A manager for an attempt to relay a set of messages to a mail exchange
+    server.
+
+    @ivar manager: See L{__init__}
+
+    @type _completionDeferreds: L{list} of L{Deferred}
+    @ivar _completionDeferreds: Deferreds which are to be notified when the
+        attempt to relay is finished.
+    """
+
+    def __init__(self, manager, noisy=True, reactor=None):
+        """
+        @type manager: L{SmartHostSMTPRelayingManager}
+        @param manager: A smart host.
+
+        @type noisy: L{bool}
+        @param noisy: A flag which determines whether informational log
+            messages will be generated (L{True}) or not (L{False}).
+
+        @type reactor: L{IReactorTime
+            <twisted.internet.interfaces.IReactorTime>} provider
+        @param reactor: A reactor which will be used to schedule delayed calls.
+        """
+        self.manager = manager
+        self._completionDeferreds = []
+        self.noisy = noisy
+
+        if not reactor:
+            from twisted.internet import reactor
+        self.reactor = reactor
+
+    def getCompletionDeferred(self):
+        """
+        Return a deferred which will fire when the attempt to relay is
+        finished.
+
+        @rtype: L{Deferred}
+        @return: A deferred which will fire when the attempt to relay is
+            finished.
+        """
+        self._completionDeferreds.append(Deferred())
+        return self._completionDeferreds[-1]
+
+    def _finish(self, relay, message):
+        """
+        Remove a message from the relay queue and from the smart host's list of
+        messages being relayed.
+
+        @type relay: L{SMTPManagedRelayerFactory}
+        @param relay: The factory for the relayer which sent the message.
+
+        @type message: L{bytes}
+        @param message: The path of the file holding the message.
+        """
+        self.manager.managed[relay].remove(os.path.basename(message))
+        self.manager.queue.done(message)
+
+    def notifySuccess(self, relay, message):
+        """
+        Remove a message from the relay queue after it has been successfully
+        sent.
+
+        @type relay: L{SMTPManagedRelayerFactory}
+        @param relay: The factory for the relayer which sent the message.
+
+        @type message: L{bytes}
+        @param message: The path of the file holding the message.
+        """
+        if self.noisy:
+            log.msg("success sending %s, removing from queue" % message)
+        self._finish(relay, message)
+
+    def notifyFailure(self, relay, message):
+        """
+        Generate a bounce message for a message which cannot be relayed.
+
+        @type relay: L{SMTPManagedRelayerFactory}
+        @param relay: The factory for the relayer responsible for the message.
+
+        @type message: L{bytes}
+        @param message: The path of the file holding the message.
+        """
+        if self.noisy:
+            log.msg("could not relay " + message)
+        # Moshe - Bounce E-mail here
+        # Be careful: if it's a bounced bounce, silently
+        # discard it
+        message = os.path.basename(message)
+        with self.manager.queue.getEnvelopeFile(message) as fp:
+            from_, to = pickle.load(fp)
+        from_, to, bounceMessage = bounce.generateBounce(
+            open(self.manager.queue.getPath(message) + "-D"), from_, to
+        )
+        fp, outgoingMessage = self.manager.queue.createNewMessage()
+        with fp:
+            pickle.dump([from_, to], fp)
+        for line in bounceMessage.splitlines():
+            outgoingMessage.lineReceived(line)
+        outgoingMessage.eomReceived()
+        self._finish(relay, self.manager.queue.getPath(message))
+
+    def notifyDone(self, relay):
+        """
+        When the connection is lost or cannot be established, prepare to
+        resend unsent messages and fire all deferred which are waiting for
+        the completion of the attempt to relay.
+
+        @type relay: L{SMTPManagedRelayerFactory}
+        @param relay: The factory for the relayer for the connection.
+        """
+        for message in self.manager.managed.get(relay, ()):
+            if self.noisy:
+                log.msg("Setting " + message + " waiting")
+            self.manager.queue.setWaiting(message)
+        try:
+            del self.manager.managed[relay]
+        except KeyError:
+            pass
+        notifications = self._completionDeferreds
+        self._completionDeferreds = None
+        for d in notifications:
+            d.callback(None)
+
+    def notifyNoConnection(self, relay):
+        """
+        When a connection to the mail exchange server cannot be established,
+        prepare to resend messages later.
+
+        @type relay: L{SMTPManagedRelayerFactory}
+        @param relay: The factory for the relayer meant to use the connection.
+        """
+        # Back off a bit
+        try:
+            msgs = self.manager.managed[relay]
+        except KeyError:
+            log.msg("notifyNoConnection passed unknown relay!")
+            return
+
+        if self.noisy:
+            log.msg("Backing off on delivery of " + str(msgs))
+
+        def setWaiting(queue, messages):
+            map(queue.setWaiting, messages)
+
+        self.reactor.callLater(30, setWaiting, self.manager.queue, msgs)
+        del self.manager.managed[relay]
+
+
+class SmartHostSMTPRelayingManager:
+    """
+    A smart host which uses SMTP managed relayers to send messages from the
+    relay queue.
+
+    L{checkState} must be called periodically at which time the state of the
+    relay queue is checked and new relayers are created as needed.
+
+    In order to relay a set of messages to a mail exchange server, a smart host
+    creates an attempt manager and a managed relayer factory for that set of
+    messages.  When a connection is made with the mail exchange server, the
+    managed relayer factory creates a managed relayer to send the messages.
+    The managed relayer reports on its progress to the attempt manager which,
+    in turn, updates the smart host's relay queue and information about its
+    managed relayers.
+
+    @ivar queue: See L{__init__}.
+    @ivar maxConnections: See L{__init__}.
+    @ivar maxMessagesPerConnection: See L{__init__}.
+
+    @type fArgs: 3-L{tuple} of (0) L{list} of L{bytes},
+        (1) L{_AttemptManager}, (2) L{bytes} or 4-L{tuple} of (0) L{list}
+        of L{bytes}, (1) L{_AttemptManager}, (2) L{bytes}, (3) L{int}
+    @ivar fArgs: Positional arguments for
+        L{SMTPManagedRelayerFactory.__init__}.
+
+    @type fKwArgs: L{dict}
+    @ivar fKwArgs: Keyword arguments for L{SMTPManagedRelayerFactory.__init__}.
+
+    @type factory: callable which returns L{SMTPManagedRelayerFactory}
+    @ivar factory: A callable which creates a factory for creating a managed
+        relayer. See L{SMTPManagedRelayerFactory.__init__} for parameters to
+        the callable.
+
+    @type PORT: L{int}
+    @ivar PORT: The port over which to connect to the SMTP server.
+
+    @type mxcalc: L{None} or L{MXCalculator}
+    @ivar mxcalc: A resource for mail exchange host lookups.
+
+    @type managed: L{dict} mapping L{SMTPManagedRelayerFactory} to L{list} of
+        L{bytes}
+    @ivar managed: A mapping of factory for a managed relayer to
+        filenames of messages the managed relayer is responsible for.
+    """
+
+    factory: Type[protocol.ClientFactory] = SMTPManagedRelayerFactory
+
+    PORT = 25
+
+    mxcalc = None
+
+    def __init__(self, queue, maxConnections=2, maxMessagesPerConnection=10):
+        """
+        Initialize a smart host.
+
+        The default values specify connection limits appropriate for a
+        low-volume smart host.
+
+        @type queue: L{Queue}
+        @param queue: A relay queue.
+
+        @type maxConnections: L{int}
+        @param maxConnections: The maximum number of concurrent connections to
+            SMTP servers.
+
+        @type maxMessagesPerConnection: L{int}
+        @param maxMessagesPerConnection: The maximum number of messages for
+            which a relayer will be given responsibility.
+        """
+        self.maxConnections = maxConnections
+        self.maxMessagesPerConnection = maxMessagesPerConnection
+        self.managed = {}  # SMTP clients we're managing
+        self.queue = queue
+        self.fArgs = ()
+        self.fKwArgs = {}
+
+    def __getstate__(self):
+        """
+        Create a representation of the non-volatile state of this object.
+
+        @rtype: L{dict} mapping L{bytes} to L{object}
+        @return: The non-volatile state of the queue.
+        """
+        dct = self.__dict__.copy()
+        del dct["managed"]
+        return dct
+
+    def __setstate__(self, state):
+        """
+        Restore the non-volatile state of this object and recreate the volatile
+        state.
+
+        @type state: L{dict} mapping L{bytes} to L{object}
+        @param state: The non-volatile state of the queue.
+        """
+        self.__dict__.update(state)
+        self.managed = {}
+
+    def checkState(self):
+        """
+        Check the state of the relay queue and, if possible, launch relayers to
+        handle waiting messages.
+
+        @rtype: L{None} or L{Deferred}
+        @return: No return value if no further messages can be relayed or a
+            deferred which fires when all of the SMTP connections initiated by
+            this call have disconnected.
+        """
+        self.queue.readDirectory()
+        if len(self.managed) >= self.maxConnections:
+            return
+        if not self.queue.hasWaiting():
+            return
+
+        return self._checkStateMX()
+
+    def _checkStateMX(self):
+        nextMessages = self.queue.getWaiting()
+        nextMessages.reverse()
+
+        exchanges = {}
+        for msg in nextMessages:
+            from_, to = self.queue.getEnvelope(msg)
+            name, addr = email.utils.parseaddr(to)
+            parts = addr.split("@", 1)
+            if len(parts) != 2:
+                log.err("Illegal message destination: " + to)
+                continue
+            domain = parts[1]
+
+            self.queue.setRelaying(msg)
+            exchanges.setdefault(domain, []).append(self.queue.getPath(msg))
+            if len(exchanges) >= (self.maxConnections - len(self.managed)):
+                break
+
+        if self.mxcalc is None:
+            self.mxcalc = MXCalculator()
+
+        relays = []
+        for domain, msgs in exchanges.iteritems():
+            manager = _AttemptManager(self, self.queue.noisy)
+            factory = self.factory(msgs, manager, *self.fArgs, **self.fKwArgs)
+            self.managed[factory] = map(os.path.basename, msgs)
+            relayAttemptDeferred = manager.getCompletionDeferred()
+            connectSetupDeferred = self.mxcalc.getMX(domain)
+            connectSetupDeferred.addCallback(lambda mx: str(mx.name))
+            connectSetupDeferred.addCallback(self._cbExchange, self.PORT, factory)
+            connectSetupDeferred.addErrback(
+                lambda err: (relayAttemptDeferred.errback(err), err)[1]
+            )
+            connectSetupDeferred.addErrback(self._ebExchange, factory, domain)
+            relays.append(relayAttemptDeferred)
+        return DeferredList(relays)
+
+    def _cbExchange(self, address, port, factory):
+        """
+        Initiate a connection with a mail exchange server.
+
+        This callback function runs after mail exchange server for the domain
+        has been looked up.
+
+        @type address: L{bytes}
+        @param address: The hostname of a mail exchange server.
+
+        @type port: L{int}
+        @param port: A port number.
+
+        @type factory: L{SMTPManagedRelayerFactory}
+        @param factory: A factory which can create a relayer for the mail
+            exchange server.
+        """
+        from twisted.internet import reactor
+
+        reactor.connectTCP(address, port, factory)
+
+    def _ebExchange(self, failure, factory, domain):
+        """
+        Prepare to resend messages later.
+
+        This errback function runs when no mail exchange server for the domain
+        can be found.
+
+        @type failure: L{Failure}
+        @param failure: The reason the mail exchange lookup failed.
+
+        @type factory: L{SMTPManagedRelayerFactory}
+        @param factory: A factory which can create a relayer for the mail
+            exchange server.
+
+        @type domain: L{bytes}
+        @param domain: A domain.
+        """
+        log.err("Error setting up managed relay factory for " + domain)
+        log.err(failure)
+
+        def setWaiting(queue, messages):
+            map(queue.setWaiting, messages)
+
+        from twisted.internet import reactor
+
+        reactor.callLater(30, setWaiting, self.queue, self.managed[factory])
+        del self.managed[factory]
+
+
+class SmartHostESMTPRelayingManager(SmartHostSMTPRelayingManager):
+    """
+    A smart host which uses ESMTP managed relayers to send messages from the
+    relay queue.
+
+    @type factory: callable which returns L{ESMTPManagedRelayerFactory}
+    @ivar factory: A callable which creates a factory for creating a managed
+        relayer. See L{ESMTPManagedRelayerFactory.__init__} for parameters to
+        the callable.
+    """
+
+    factory = ESMTPManagedRelayerFactory
+
+
+def _checkState(manager):
+    """
+    Prompt a relaying manager to check state.
+
+    @type manager: L{SmartHostSMTPRelayingManager}
+    @param manager: A relaying manager.
+    """
+    manager.checkState()
+
+
+def RelayStateHelper(manager, delay):
+    """
+    Set up a periodic call to prompt a relaying manager to check state.
+
+    @type manager: L{SmartHostSMTPRelayingManager}
+    @param manager: A relaying manager.
+
+    @type delay: L{float}
+    @param delay: The number of seconds between calls.
+
+    @rtype: L{TimerService <internet.TimerService>}
+    @return: A service which periodically reminds a relaying manager to check
+        state.
+    """
+    return internet.TimerService(delay, _checkState, manager)
+
+
+class CanonicalNameLoop(Exception):
+    """
+    An error indicating that when trying to look up a mail exchange host, a set
+    of canonical name records was found which form a cycle and resolution was
+    abandoned.
+    """
+
+
+class CanonicalNameChainTooLong(Exception):
+    """
+    An error indicating that when trying to look up a mail exchange host, too
+    many canonical name records which point to other canonical name records
+    were encountered and resolution was abandoned.
+    """
+
+
+class MXCalculator:
+    """
+    A utility for looking up mail exchange hosts and tracking whether they are
+    working or not.
+
+    @type clock: L{IReactorTime <twisted.internet.interfaces.IReactorTime>}
+        provider
+    @ivar clock: A reactor which will be used to schedule timeouts.
+
+    @type resolver: L{IResolver <twisted.internet.interfaces.IResolver>}
+    @ivar resolver: A resolver.
+
+    @type badMXs: L{dict} mapping L{bytes} to L{float}
+    @ivar badMXs: A mapping of non-functioning mail exchange hostname to time
+        at which another attempt at contacting it may be made.
+
+    @type timeOutBadMX: L{int}
+    @ivar timeOutBadMX: Period in seconds between attempts to contact a
+        non-functioning mail exchange host.
+
+    @type fallbackToDomain: L{bool}
+    @ivar fallbackToDomain: A flag indicating whether to attempt to use the
+        hostname directly when no mail exchange can be found (C{True}) or
+        not (C{False}).
+    """
+
+    timeOutBadMX = 60 * 60  # One hour
+    fallbackToDomain = True
+
+    def __init__(self, resolver=None, clock=None):
+        """
+        @type resolver: L{IResolver <twisted.internet.interfaces.IResolver>}
+            provider or L{None}
+        @param resolver: A resolver.
+
+        @type clock: L{IReactorTime <twisted.internet.interfaces.IReactorTime>}
+            provider or L{None}
+        @param clock: A reactor which will be used to schedule timeouts.
+        """
+        self.badMXs = {}
+        if resolver is None:
+            from twisted.names.client import createResolver
+
+            resolver = createResolver()
+        self.resolver = resolver
+        if clock is None:
+            from twisted.internet import reactor as clock
+        self.clock = clock
+
+    def markBad(self, mx):
+        """
+        Record that a mail exchange host is not currently functioning.
+
+        @type mx: L{bytes}
+        @param mx: The hostname of a mail exchange host.
+        """
+        self.badMXs[str(mx)] = self.clock.seconds() + self.timeOutBadMX
+
+    def markGood(self, mx):
+        """
+        Record that a mail exchange host is functioning.
+
+        @type mx: L{bytes}
+        @param mx: The hostname of a mail exchange host.
+        """
+        try:
+            del self.badMXs[mx]
+        except KeyError:
+            pass
+
+    def getMX(self, domain, maximumCanonicalChainLength=3):
+        """
+        Find the name of a host that acts as a mail exchange server
+        for a domain.
+
+        @type domain: L{bytes}
+        @param domain: A domain name.
+
+        @type maximumCanonicalChainLength: L{int}
+        @param maximumCanonicalChainLength: The maximum number of unique
+            canonical name records to follow while looking up the mail exchange
+            host.
+
+        @rtype: L{Deferred} which successfully fires with L{Record_MX}
+        @return: A deferred which succeeds with the MX record for the mail
+            exchange server for the domain or fails if none can be found.
+        """
+        mailExchangeDeferred = self.resolver.lookupMailExchange(domain)
+        mailExchangeDeferred.addCallback(self._filterRecords)
+        mailExchangeDeferred.addCallback(
+            self._cbMX, domain, maximumCanonicalChainLength
+        )
+        mailExchangeDeferred.addErrback(self._ebMX, domain)
+        return mailExchangeDeferred
+
+    def _filterRecords(self, records):
+        """
+        Organize the records of a DNS response by record name.
+
+        @type records: 3-L{tuple} of (0) L{list} of L{RRHeader
+            <twisted.names.dns.RRHeader>}, (1) L{list} of L{RRHeader
+            <twisted.names.dns.RRHeader>}, (2) L{list} of L{RRHeader
+            <twisted.names.dns.RRHeader>}
+        @param records: Answer resource records, authority resource records and
+            additional resource records.
+
+        @rtype: L{dict} mapping L{bytes} to L{list} of L{IRecord
+            <twisted.names.dns.IRecord>} provider
+        @return: A mapping of record name to record payload.
+        """
+        recordBag = {}
+        for answer in records[0]:
+            recordBag.setdefault(str(answer.name), []).append(answer.payload)
+        return recordBag
+
+    def _cbMX(self, answers, domain, cnamesLeft):
+        """
+        Try to find the mail exchange host for a domain from the given DNS
+        records.
+
+        This will attempt to resolve canonical name record results.  It can
+        recognize loops and will give up on non-cyclic chains after a specified
+        number of lookups.
+
+        @type answers: L{dict} mapping L{bytes} to L{list} of L{IRecord
+            <twisted.names.dns.IRecord>} provider
+        @param answers: A mapping of record name to record payload.
+
+        @type domain: L{bytes}
+        @param domain: A domain name.
+
+        @type cnamesLeft: L{int}
+        @param cnamesLeft: The number of unique canonical name records
+            left to follow while looking up the mail exchange host.
+
+        @rtype: L{Record_MX <twisted.names.dns.Record_MX>} or L{Failure}
+        @return: An MX record for the mail exchange host or a failure if one
+            cannot be found.
+        """
+        # Do this import here so that relaymanager.py doesn't depend on
+        # twisted.names, only MXCalculator will.
+        from twisted.names import dns, error
+
+        seenAliases = set()
+        exchanges = []
+        # Examine the answers for the domain we asked about
+        pertinentRecords = answers.get(domain, [])
+        while pertinentRecords:
+            record = pertinentRecords.pop()
+
+            # If it's a CNAME, we'll need to do some more processing
+            if record.TYPE == dns.CNAME:
+                # Remember that this name was an alias.
+                seenAliases.add(domain)
+
+                canonicalName = str(record.name)
+                # See if we have some local records which might be relevant.
+                if canonicalName in answers:
+                    # Make sure it isn't a loop contained entirely within the
+                    # results we have here.
+                    if canonicalName in seenAliases:
+                        return Failure(CanonicalNameLoop(record))
+
+                    pertinentRecords = answers[canonicalName]
+                    exchanges = []
+                else:
+                    if cnamesLeft:
+                        # Request more information from the server.
+                        return self.getMX(canonicalName, cnamesLeft - 1)
+                    else:
+                        # Give up.
+                        return Failure(CanonicalNameChainTooLong(record))
+
+            # If it's an MX, collect it.
+            if record.TYPE == dns.MX:
+                exchanges.append((record.preference, record))
+
+        if exchanges:
+            exchanges.sort()
+            for preference, record in exchanges:
+                host = str(record.name)
+                if host not in self.badMXs:
+                    return record
+                t = self.clock.seconds() - self.badMXs[host]
+                if t >= 0:
+                    del self.badMXs[host]
+                    return record
+            return exchanges[0][1]
+        else:
+            # Treat no answers the same as an error - jump to the errback to
+            # try to look up an A record.  This provides behavior described as
+            # a special case in RFC 974 in the section headed I{Interpreting
+            # the List of MX RRs}.
+            return Failure(error.DNSNameError(f"No MX records for {domain!r}"))
+
+    def _ebMX(self, failure, domain):
+        """
+        Attempt to use the name of the domain directly when mail exchange
+        lookup fails.
+
+        @type failure: L{Failure}
+        @param failure: The reason for the lookup failure.
+
+        @type domain: L{bytes}
+        @param domain: The domain name.
+
+        @rtype: L{Record_MX <twisted.names.dns.Record_MX>} or L{Failure}
+        @return: An MX record for the domain or a failure if the fallback to
+            domain option is not in effect and an error, other than not
+            finding an MX record, occurred during lookup.
+
+        @raise IOError: When no MX record could be found and the fallback to
+            domain option is not in effect.
+
+        @raise DNSLookupError: When no MX record could be found and the
+            fallback to domain option is in effect but no address for the
+            domain could be found.
+        """
+        from twisted.names import dns, error
+
+        if self.fallbackToDomain:
+            failure.trap(error.DNSNameError)
+            log.msg(
+                "MX lookup failed; attempting to use hostname ({}) directly".format(
+                    domain
+                )
+            )
+
+            # Alright, I admit, this is a bit icky.
+            d = self.resolver.getHostByName(domain)
+
+            def cbResolved(addr):
+                return dns.Record_MX(name=addr)
+
+            def ebResolved(err):
+                err.trap(error.DNSNameError)
+                raise DNSLookupError()
+
+            d.addCallbacks(cbResolved, ebResolved)
+            return d
+        elif failure.check(error.DNSNameError):
+            raise OSError(f"No MX found for {domain!r}")
+        return failure
diff --git a/contrib/python/Twisted/py3/twisted/mail/scripts/__init__.py b/contrib/python/Twisted/py3/twisted/mail/scripts/__init__.py
new file mode 100644
index 00000000000..f653cc71edb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/scripts/__init__.py
@@ -0,0 +1 @@
+"mail scripts"
diff --git a/contrib/python/Twisted/py3/twisted/mail/scripts/mailmail.py b/contrib/python/Twisted/py3/twisted/mail/scripts/mailmail.py
new file mode 100644
index 00000000000..cf1e58f5b6b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/scripts/mailmail.py
@@ -0,0 +1,386 @@
+# -*- test-case-name: twisted.mail.test.test_mailmail -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation module for the I{mailmail} command.
+"""
+
+
+import email.utils
+import getpass
+import os
+import sys
+from configparser import ConfigParser
+from io import StringIO
+
+from twisted.copyright import version
+from twisted.internet import reactor
+from twisted.logger import Logger, textFileLogObserver
+from twisted.mail import smtp
+
+GLOBAL_CFG = "/etc/mailmail"
+LOCAL_CFG = os.path.expanduser("~/.twisted/mailmail")
+SMARTHOST = "127.0.0.1"
+
+ERROR_FMT = """\
+Subject: Failed Message Delivery
+
+  Message delivery failed.  The following occurred:
+
+  %s
+--
+The Twisted sendmail application.
+"""
+
+_logObserver = textFileLogObserver(sys.stderr)
+_log = Logger(observer=_logObserver)
+
+
+class Options:
+    """
+    Store the values of the parsed command-line options to the I{mailmail}
+    script.
+
+    @type to: L{list} of L{str}
+    @ivar to: The addresses to which to deliver this message.
+
+    @type sender: L{str}
+    @ivar sender: The address from which this message is being sent.
+
+    @type body: C{file}
+    @ivar body: The object from which the message is to be read.
+    """
+
+
+def getlogin():
+    try:
+        return os.getlogin()
+    except BaseException:
+        return getpass.getuser()
+
+
+_unsupportedOption = SystemExit("Unsupported option.")
+
+
+def parseOptions(argv):
+    o = Options()
+    o.to = [e for e in argv if not e.startswith("-")]
+    o.sender = getlogin()
+
+    # Just be very stupid
+
+    # Skip -bm -- it is the default
+
+    # Add a non-standard option for querying the version of this tool.
+    if "--version" in argv:
+        print("mailmail version:", version)
+        raise SystemExit()
+
+    # -bp lists queue information.  Screw that.
+    if "-bp" in argv:
+        raise _unsupportedOption
+
+    # -bs makes sendmail use stdin/stdout as its transport.  Screw that.
+    if "-bs" in argv:
+        raise _unsupportedOption
+
+    # -F sets who the mail is from, but is overridable by the From header
+    if "-F" in argv:
+        o.sender = argv[argv.index("-F") + 1]
+        o.to.remove(o.sender)
+
+    # -i and -oi makes us ignore lone "."
+    if ("-i" in argv) or ("-oi" in argv):
+        raise _unsupportedOption
+
+    # -odb is background delivery
+    if "-odb" in argv:
+        o.background = True
+    else:
+        o.background = False
+
+    # -odf is foreground delivery
+    if "-odf" in argv:
+        o.background = False
+    else:
+        o.background = True
+
+    # -oem and -em cause errors to be mailed back to the sender.
+    # It is also the default.
+
+    # -oep and -ep cause errors to be printed to stderr
+    if ("-oep" in argv) or ("-ep" in argv):
+        o.printErrors = True
+    else:
+        o.printErrors = False
+
+    # -om causes a copy of the message to be sent to the sender if the sender
+    # appears in an alias expansion.  We do not support aliases.
+    if "-om" in argv:
+        raise _unsupportedOption
+
+    # -t causes us to pick the recipients of the message from
+    # the To, Cc, and Bcc headers, and to remove the Bcc header
+    # if present.
+    if "-t" in argv:
+        o.recipientsFromHeaders = True
+        o.excludeAddresses = o.to
+        o.to = []
+    else:
+        o.recipientsFromHeaders = False
+        o.exludeAddresses = []
+
+    requiredHeaders = {
+        "from": [],
+        "to": [],
+        "cc": [],
+        "bcc": [],
+        "date": [],
+    }
+
+    buffer = StringIO()
+    while 1:
+        write = 1
+        line = sys.stdin.readline()
+        if not line.strip():
+            break
+
+        hdrs = line.split(": ", 1)
+
+        hdr = hdrs[0].lower()
+        if o.recipientsFromHeaders and hdr in ("to", "cc", "bcc"):
+            o.to.extend([email.utils.parseaddr(hdrs[1])[1]])
+            if hdr == "bcc":
+                write = 0
+        elif hdr == "from":
+            o.sender = email.utils.parseaddr(hdrs[1])[1]
+
+        if hdr in requiredHeaders:
+            requiredHeaders[hdr].append(hdrs[1])
+
+        if write:
+            buffer.write(line)
+
+    if not requiredHeaders["from"]:
+        buffer.write(f"From: {o.sender}\r\n")
+    if not requiredHeaders["to"]:
+        if not o.to:
+            raise SystemExit("No recipients specified.")
+        buffer.write("To: {}\r\n".format(", ".join(o.to)))
+    if not requiredHeaders["date"]:
+        buffer.write(f"Date: {smtp.rfc822date()}\r\n")
+
+    buffer.write(line)
+
+    if o.recipientsFromHeaders:
+        for a in o.excludeAddresses:
+            try:
+                o.to.remove(a)
+            except BaseException:
+                pass
+
+    buffer.seek(0, 0)
+    o.body = StringIO(buffer.getvalue() + sys.stdin.read())
+    return o
+
+
+class Configuration:
+    """
+
+    @ivar allowUIDs: A list of UIDs which are allowed to send mail.
+    @ivar allowGIDs: A list of GIDs which are allowed to send mail.
+    @ivar denyUIDs: A list of UIDs which are not allowed to send mail.
+    @ivar denyGIDs: A list of GIDs which are not allowed to send mail.
+
+    @type defaultAccess: L{bool}
+    @ivar defaultAccess: L{True} if access will be allowed when no other access
+    control rule matches or L{False} if it will be denied in that case.
+
+    @ivar useraccess: Either C{'allow'} to check C{allowUID} first
+    or C{'deny'} to check C{denyUID} first.
+
+    @ivar groupaccess: Either C{'allow'} to check C{allowGID} first or
+    C{'deny'} to check C{denyGID} first.
+
+    @ivar identities: A L{dict} mapping hostnames to credentials to use when
+    sending mail to that host.
+
+    @ivar smarthost: L{None} or a hostname through which all outgoing mail will
+    be sent.
+
+    @ivar domain: L{None} or the hostname with which to identify ourselves when
+    connecting to an MTA.
+    """
+
+    def __init__(self):
+        self.allowUIDs = []
+        self.denyUIDs = []
+        self.allowGIDs = []
+        self.denyGIDs = []
+        self.useraccess = "deny"
+        self.groupaccess = "deny"
+
+        self.identities = {}
+        self.smarthost = None
+        self.domain = None
+
+        self.defaultAccess = True
+
+
+def loadConfig(path):
+    # [useraccess]
+    # allow=uid1,uid2,...
+    # deny=uid1,uid2,...
+    # order=allow,deny
+    # [groupaccess]
+    # allow=gid1,gid2,...
+    # deny=gid1,gid2,...
+    # order=deny,allow
+    # [identity]
+    # host1=username:password
+    # host2=username:password
+    # [addresses]
+    # smarthost=a.b.c.d
+    # default_domain=x.y.z
+
+    c = Configuration()
+
+    if not os.access(path, os.R_OK):
+        return c
+
+    p = ConfigParser()
+    p.read(path)
+
+    au = c.allowUIDs
+    du = c.denyUIDs
+    ag = c.allowGIDs
+    dg = c.denyGIDs
+    for section, a, d in (("useraccess", au, du), ("groupaccess", ag, dg)):
+        if p.has_section(section):
+            for mode, L in (("allow", a), ("deny", d)):
+                if p.has_option(section, mode) and p.get(section, mode):
+                    for sectionID in p.get(section, mode).split(","):
+                        try:
+                            sectionID = int(sectionID)
+                        except ValueError:
+                            _log.error(
+                                "Illegal {prefix}ID in "
+                                "[{section}] section: {sectionID}",
+                                prefix=section[0].upper(),
+                                section=section,
+                                sectionID=sectionID,
+                            )
+                        else:
+                            L.append(sectionID)
+            order = p.get(section, "order")
+            order = [s.split() for s in [s.lower() for s in order.split(",")]]
+            if order[0] == "allow":
+                setattr(c, section, "allow")
+            else:
+                setattr(c, section, "deny")
+
+    if p.has_section("identity"):
+        for host, up in p.items("identity"):
+            parts = up.split(":", 1)
+            if len(parts) != 2:
+                _log.error("Illegal entry in [identity] section: {section}", section=up)
+                continue
+            c.identities[host] = parts
+
+    if p.has_section("addresses"):
+        if p.has_option("addresses", "smarthost"):
+            c.smarthost = p.get("addresses", "smarthost")
+        if p.has_option("addresses", "default_domain"):
+            c.domain = p.get("addresses", "default_domain")
+
+    return c
+
+
+def success(result):
+    reactor.stop()
+
+
+failed = None
+
+
+def failure(f):
+    global failed
+    reactor.stop()
+    failed = f
+
+
+def sendmail(host, options, ident):
+    d = smtp.sendmail(host, options.sender, options.to, options.body)
+    d.addCallbacks(success, failure)
+    reactor.run()
+
+
+def senderror(failure, options):
+    recipient = [options.sender]
+    sender = '"Internally Generated Message ({})"<postmaster@{}>'.format(
+        sys.argv[0], smtp.DNSNAME.decode("ascii")
+    )
+    error = StringIO()
+    failure.printTraceback(file=error)
+    body = StringIO(ERROR_FMT % error.getvalue())
+    d = smtp.sendmail("localhost", sender, recipient, body)
+    d.addBoth(lambda _: reactor.stop())
+
+
+def deny(conf):
+    uid = os.getuid()
+    gid = os.getgid()
+
+    if conf.useraccess == "deny":
+        if uid in conf.denyUIDs:
+            return True
+        if uid in conf.allowUIDs:
+            return False
+    else:
+        if uid in conf.allowUIDs:
+            return False
+        if uid in conf.denyUIDs:
+            return True
+
+    if conf.groupaccess == "deny":
+        if gid in conf.denyGIDs:
+            return True
+        if gid in conf.allowGIDs:
+            return False
+    else:
+        if gid in conf.allowGIDs:
+            return False
+        if gid in conf.denyGIDs:
+            return True
+
+    return not conf.defaultAccess
+
+
+def run():
+    o = parseOptions(sys.argv[1:])
+    gConf = loadConfig(GLOBAL_CFG)
+    lConf = loadConfig(LOCAL_CFG)
+
+    if deny(gConf) or deny(lConf):
+        _log.error("Permission denied")
+        return
+
+    host = lConf.smarthost or gConf.smarthost or SMARTHOST
+
+    ident = gConf.identities.copy()
+    ident.update(lConf.identities)
+
+    if lConf.domain:
+        smtp.DNSNAME = lConf.domain
+    elif gConf.domain:
+        smtp.DNSNAME = gConf.domain
+
+    sendmail(host, o, ident)
+
+    if failed:
+        if o.printErrors:
+            failed.printTraceback(file=sys.stderr)
+            raise SystemExit(1)
+        else:
+            senderror(failed, o)
diff --git a/contrib/python/Twisted/py3/twisted/mail/smtp.py b/contrib/python/Twisted/py3/twisted/mail/smtp.py
new file mode 100644
index 00000000000..55511647e66
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/smtp.py
@@ -0,0 +1,2270 @@
+# -*- test-case-name: twisted.mail.test.test_smtp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+#
+# pylint: disable=I0011,C0103,C9302
+
+"""
+Simple Mail Transfer Protocol implementation.
+"""
+
+
+import base64
+import binascii
+import os
+import random
+import re
+import socket
+import time
+import warnings
+from email.utils import parseaddr
+from io import BytesIO
+from typing import Type
+
+from zope.interface import implementer
+
+from twisted import cred
+from twisted.copyright import longversion
+from twisted.internet import defer, error, protocol, reactor
+from twisted.internet._idna import _idnaText
+from twisted.internet.interfaces import ISSLTransport, ITLSTransport
+from twisted.mail._cred import (
+    CramMD5ClientAuthenticator,
+    LOGINAuthenticator,
+    LOGINCredentials as _lcredentials,
+)
+from twisted.mail._except import (
+    AddressError,
+    AUTHDeclinedError,
+    AuthenticationError,
+    AUTHRequiredError,
+    EHLORequiredError,
+    ESMTPClientError,
+    SMTPAddressError,
+    SMTPBadRcpt,
+    SMTPBadSender,
+    SMTPClientError,
+    SMTPConnectError,
+    SMTPDeliveryError,
+    SMTPError,
+    SMTPProtocolError,
+    SMTPServerError,
+    SMTPTimeoutError,
+    SMTPTLSError as TLSError,
+    TLSRequiredError,
+)
+from twisted.mail.interfaces import (
+    IClientAuthentication,
+    IMessageDelivery,
+    IMessageDeliveryFactory,
+    IMessageSMTP as IMessage,
+)
+from twisted.protocols import basic, policies
+from twisted.python import log, util
+from twisted.python.compat import iterbytes, nativeString, networkString
+from twisted.python.runtime import platform
+
+__all__ = [
+    "AUTHDeclinedError",
+    "AUTHRequiredError",
+    "AddressError",
+    "AuthenticationError",
+    "EHLORequiredError",
+    "ESMTPClientError",
+    "SMTPAddressError",
+    "SMTPBadRcpt",
+    "SMTPBadSender",
+    "SMTPClientError",
+    "SMTPConnectError",
+    "SMTPDeliveryError",
+    "SMTPError",
+    "SMTPServerError",
+    "SMTPTimeoutError",
+    "TLSError",
+    "TLSRequiredError",
+    "SMTPProtocolError",
+    "IClientAuthentication",
+    "IMessage",
+    "IMessageDelivery",
+    "IMessageDeliveryFactory",
+    "CramMD5ClientAuthenticator",
+    "LOGINAuthenticator",
+    "LOGINCredentials",
+    "PLAINAuthenticator",
+    "Address",
+    "User",
+    "sendmail",
+    "SenderMixin",
+    "ESMTP",
+    "ESMTPClient",
+    "ESMTPSender",
+    "ESMTPSenderFactory",
+    "SMTP",
+    "SMTPClient",
+    "SMTPFactory",
+    "SMTPSender",
+    "SMTPSenderFactory",
+    "idGenerator",
+    "messageid",
+    "quoteaddr",
+    "rfc822date",
+    "xtextStreamReader",
+    "xtextStreamWriter",
+    "xtext_codec",
+    "xtext_decode",
+    "xtext_encode",
+]
+
+
+# Cache the hostname (XXX Yes - this is broken)
+# Encode the DNS name into something we can send over the wire
+if platform.isMacOSX():
+    # On macOS, getfqdn() is ridiculously slow - use the
+    # probably-identical-but-sometimes-not gethostname() there.
+    DNSNAME = socket.gethostname().encode("ascii")
+else:
+    DNSNAME = socket.getfqdn().encode("ascii")
+
+# Used for fast success code lookup
+SUCCESS = dict.fromkeys(range(200, 300))
+
+
+def rfc822date(timeinfo=None, local=1):
+    """
+    Format an RFC-2822 compliant date string.
+
+    @param timeinfo: (optional) A sequence as returned by C{time.localtime()}
+        or C{time.gmtime()}. Default is now.
+    @param local: (optional) Indicates if the supplied time is local or
+        universal time, or if no time is given, whether now should be local or
+        universal time. Default is local, as suggested (SHOULD) by rfc-2822.
+
+    @returns: A L{bytes} representing the time and date in RFC-2822 format.
+    """
+    if not timeinfo:
+        if local:
+            timeinfo = time.localtime()
+        else:
+            timeinfo = time.gmtime()
+    if local:
+        if timeinfo[8]:
+            # DST
+            tz = -time.altzone
+        else:
+            tz = -time.timezone
+
+        (tzhr, tzmin) = divmod(abs(tz), 3600)
+        if tz:
+            tzhr *= int(abs(tz) // tz)
+        (tzmin, tzsec) = divmod(tzmin, 60)
+    else:
+        (tzhr, tzmin) = (0, 0)
+
+    return networkString(
+        "%s, %02d %s %04d %02d:%02d:%02d %+03d%02d"
+        % (
+            ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][timeinfo[6]],
+            timeinfo[2],
+            [
+                "Jan",
+                "Feb",
+                "Mar",
+                "Apr",
+                "May",
+                "Jun",
+                "Jul",
+                "Aug",
+                "Sep",
+                "Oct",
+                "Nov",
+                "Dec",
+            ][timeinfo[1] - 1],
+            timeinfo[0],
+            timeinfo[3],
+            timeinfo[4],
+            timeinfo[5],
+            tzhr,
+            tzmin,
+        )
+    )
+
+
+def idGenerator():
+    i = 0
+    while True:
+        yield i
+        i += 1
+
+
+_gen = idGenerator()
+
+
+def messageid(uniq=None, N=lambda: next(_gen)):
+    """
+    Return a globally unique random string in RFC 2822 Message-ID format
+
+    <datetime.pid.random@host.dom.ain>
+
+    Optional uniq string will be added to strengthen uniqueness if given.
+    """
+    datetime = time.strftime("%Y%m%d%H%M%S", time.gmtime())
+    pid = os.getpid()
+    rand = random.randrange(2**31 - 1)
+    if uniq is None:
+        uniq = ""
+    else:
+        uniq = "." + uniq
+
+    return "<{}.{}.{}{}.{}@{}>".format(
+        datetime, pid, rand, uniq, N(), DNSNAME.decode()
+    ).encode()
+
+
+def quoteaddr(addr):
+    """
+    Turn an email address, possibly with realname part etc, into
+    a form suitable for and SMTP envelope.
+    """
+
+    if isinstance(addr, Address):
+        return b"<" + bytes(addr) + b">"
+
+    if isinstance(addr, bytes):
+        addr = addr.decode("ascii")
+
+    res = parseaddr(addr)
+
+    if res == (None, None):
+        # It didn't parse, use it as-is
+        return b"<" + bytes(addr) + b">"
+    else:
+        return b"<" + res[1].encode("ascii") + b">"
+
+
+COMMAND, DATA, AUTH = "COMMAND", "DATA", "AUTH"
+
+
+# Character classes for parsing addresses
+atom = rb"[-A-Za-z0-9!\#$%&'*+/=?^_`{|}~]"
+
+
+class Address:
+    """Parse and hold an RFC 2821 address.
+
+    Source routes are stipped and ignored, UUCP-style bang-paths
+    and %-style routing are not parsed.
+
+    @type domain: C{bytes}
+    @ivar domain: The domain within which this address resides.
+
+    @type local: C{bytes}
+    @ivar local: The local (\"user\") portion of this address.
+    """
+
+    tstring = re.compile(
+        rb"""( # A string of
+                           (?:"[^"]*" # quoted string
+                           |\\. # backslash-escaped characted
+                           |"""
+        + atom
+        + rb""" # atom character
+                           )+|.) # or any single character""",
+        re.X,
+    )
+    atomre = re.compile(atom)  # match any one atom character
+
+    def __init__(self, addr, defaultDomain=None):
+        if isinstance(addr, User):
+            addr = addr.dest
+        if isinstance(addr, Address):
+            self.__dict__ = addr.__dict__.copy()
+            return
+        elif not isinstance(addr, bytes):
+            addr = str(addr).encode("ascii")
+
+        self.addrstr = addr
+
+        # Tokenize
+        atl = list(filter(None, self.tstring.split(addr)))
+        local = []
+        domain = []
+
+        while atl:
+            if atl[0] == b"<":
+                if atl[-1] != b">":
+                    raise AddressError("Unbalanced <>")
+                atl = atl[1:-1]
+            elif atl[0] == b"@":
+                atl = atl[1:]
+                if not local:
+                    # Source route
+                    while atl and atl[0] != b":":
+                        # remove it
+                        atl = atl[1:]
+                    if not atl:
+                        raise AddressError("Malformed source route")
+                    atl = atl[1:]  # remove :
+                elif domain:
+                    raise AddressError("Too many @")
+                else:
+                    # Now in domain
+                    domain = [b""]
+            elif len(atl[0]) == 1 and not self.atomre.match(atl[0]) and atl[0] != b".":
+                raise AddressError(f"Parse error at {atl[0]!r} of {(addr, atl)!r}")
+            else:
+                if not domain:
+                    local.append(atl[0])
+                else:
+                    domain.append(atl[0])
+                atl = atl[1:]
+
+        self.local = b"".join(local)
+        self.domain = b"".join(domain)
+        if self.local != b"" and self.domain == b"":
+            if defaultDomain is None:
+                defaultDomain = DNSNAME
+            self.domain = defaultDomain
+
+    dequotebs = re.compile(rb"\\(.)")
+
+    def dequote(self, addr):
+        """
+        Remove RFC-2821 quotes from address.
+        """
+        res = []
+
+        if not isinstance(addr, bytes):
+            addr = str(addr).encode("ascii")
+
+        atl = filter(None, self.tstring.split(addr))
+
+        for t in atl:
+            if t[0] == b'"' and t[-1] == b'"':
+                res.append(t[1:-1])
+            elif "\\" in t:
+                res.append(self.dequotebs.sub(rb"\1", t))
+            else:
+                res.append(t)
+
+        return b"".join(res)
+
+    def __str__(self) -> str:
+        return self.__bytes__().decode("ascii")
+
+    def __bytes__(self) -> bytes:
+        if self.local or self.domain:
+            return b"@".join((self.local, self.domain))
+        else:
+            return b""
+
+    def __repr__(self) -> str:
+        return "{}.{}({})".format(
+            self.__module__, self.__class__.__name__, repr(str(self))
+        )
+
+
+class User:
+    """
+    Hold information about and SMTP message recipient,
+    including information on where the message came from
+    """
+
+    def __init__(self, destination, helo, protocol, orig):
+        try:
+            host = protocol.host
+        except AttributeError:
+            host = None
+        self.dest = Address(destination, host)
+        self.helo = helo
+        self.protocol = protocol
+        if isinstance(orig, Address):
+            self.orig = orig
+        else:
+            self.orig = Address(orig, host)
+
+    def __getstate__(self):
+        """
+        Helper for pickle.
+
+        protocol isn't picklabe, but we want User to be, so skip it in
+        the pickle.
+        """
+        return {
+            "dest": self.dest,
+            "helo": self.helo,
+            "protocol": None,
+            "orig": self.orig,
+        }
+
+    def __str__(self) -> str:
+        return self.__bytes__().decode("ascii")
+
+    def __bytes__(self) -> bytes:
+        return bytes(self.dest)
+
+
+class SMTP(basic.LineOnlyReceiver, policies.TimeoutMixin):
+    """
+    SMTP server-side protocol.
+
+    @ivar host: The hostname of this mail server.
+    @type host: L{bytes}
+    """
+
+    timeout = 600
+    portal = None
+
+    # Control whether we log SMTP events
+    noisy = True
+
+    # A factory for IMessageDelivery objects.  If an
+    # avatar implementing IMessageDeliveryFactory can
+    # be acquired from the portal, it will be used to
+    # create a new IMessageDelivery object for each
+    # message which is received.
+    deliveryFactory = None
+
+    # An IMessageDelivery object.  A new instance is
+    # used for each message received if we can get an
+    # IMessageDeliveryFactory from the portal.  Otherwise,
+    # a single instance is used throughout the lifetime
+    # of the connection.
+    delivery = None
+
+    # Cred cleanup function.
+    _onLogout = None
+
+    def __init__(self, delivery=None, deliveryFactory=None):
+        self.mode = COMMAND
+        self._from = None
+        self._helo = None
+        self._to = []
+        self.delivery = delivery
+        self.deliveryFactory = deliveryFactory
+        self.host = DNSNAME
+
+    @property
+    def host(self):
+        return self._host
+
+    @host.setter
+    def host(self, toSet):
+        if not isinstance(toSet, bytes):
+            toSet = str(toSet).encode("ascii")
+        self._host = toSet
+
+    def timeoutConnection(self):
+        msg = self.host + b" Timeout. Try talking faster next time!"
+        self.sendCode(421, msg)
+        self.transport.loseConnection()
+
+    def greeting(self):
+        return self.host + b" NO UCE NO UBE NO RELAY PROBES"
+
+    def connectionMade(self):
+        # Ensure user-code always gets something sane for _helo
+        peer = self.transport.getPeer()
+        try:
+            host = peer.host
+        except AttributeError:  # not an IPv4Address
+            host = str(peer)
+        self._helo = (None, host)
+        self.sendCode(220, self.greeting())
+        self.setTimeout(self.timeout)
+
+    def sendCode(self, code, message=b""):
+        """
+        Send an SMTP code with a message.
+        """
+        lines = message.splitlines()
+        lastline = lines[-1:]
+        for line in lines[:-1]:
+            self.sendLine(networkString("%3.3d-" % (code,)) + line)
+        self.sendLine(
+            networkString("%3.3d " % (code,)) + (lastline and lastline[0] or b"")
+        )
+
+    def lineReceived(self, line):
+        self.resetTimeout()
+        return getattr(self, "state_" + self.mode)(line)
+
+    def state_COMMAND(self, line):
+        # Ignore leading and trailing whitespace, as well as an arbitrary
+        # amount of whitespace between the command and its argument, though
+        # it is not required by the protocol, for it is a nice thing to do.
+        line = line.strip()
+
+        parts = line.split(None, 1)
+        if parts:
+            method = self.lookupMethod(parts[0]) or self.do_UNKNOWN
+            if len(parts) == 2:
+                method(parts[1])
+            else:
+                method(b"")
+        else:
+            self.sendSyntaxError()
+
+    def sendSyntaxError(self):
+        self.sendCode(500, b"Error: bad syntax")
+
+    def lookupMethod(self, command):
+        """
+
+        @param command: The command to get from this class.
+        @type command: L{str}
+        @return: The function which executes this command.
+        """
+        if not isinstance(command, str):
+            command = nativeString(command)
+
+        return getattr(self, "do_" + command.upper(), None)
+
+    def lineLengthExceeded(self, line):
+        if self.mode is DATA:
+            for message in self.__messages:
+                message.connectionLost()
+            self.mode = COMMAND
+            del self.__messages
+        self.sendCode(500, b"Line too long")
+
+    def do_UNKNOWN(self, rest):
+        self.sendCode(500, b"Command not implemented")
+
+    def do_HELO(self, rest):
+        peer = self.transport.getPeer()
+        try:
+            host = peer.host
+        except AttributeError:
+            host = str(peer)
+
+        if not isinstance(host, bytes):
+            host = host.encode("idna")
+
+        self._helo = (rest, host)
+        self._from = None
+        self._to = []
+        self.sendCode(250, self.host + b" Hello " + host + b", nice to meet you")
+
+    def do_QUIT(self, rest):
+        self.sendCode(221, b"See you later")
+        self.transport.loseConnection()
+
+    # A string of quoted strings, backslash-escaped character or
+    # atom characters + '@.,:'
+    qstring = rb'("[^"]*"|\\.|' + atom + rb"|[@.,:])+"
+
+    mail_re = re.compile(
+        rb"""\s*FROM:\s*(?P<path><> # Empty <>
+                          |<"""
+        + qstring
+        + rb"""> # <addr>
+                          |"""
+        + qstring
+        + rb""" # addr
+                          )\s*(\s(?P<opts>.*))? # Optional WS + ESMTP options
+                          $""",
+        re.I | re.X,
+    )
+    rcpt_re = re.compile(
+        rb"\s*TO:\s*(?P<path><"
+        + qstring
+        + rb"""> # <addr>
+                          |"""
+        + qstring
+        + rb""" # addr
+                          )\s*(\s(?P<opts>.*))? # Optional WS + ESMTP options
+                          $""",
+        re.I | re.X,
+    )
+
+    def do_MAIL(self, rest):
+        if self._from:
+            self.sendCode(503, b"Only one sender per message, please")
+            return
+        # Clear old recipient list
+        self._to = []
+        m = self.mail_re.match(rest)
+        if not m:
+            self.sendCode(501, b"Syntax error")
+            return
+
+        try:
+            addr = Address(m.group("path"), self.host)
+        except AddressError as e:
+            self.sendCode(553, networkString(str(e)))
+            return
+
+        validated = defer.maybeDeferred(self.validateFrom, self._helo, addr)
+        validated.addCallbacks(self._cbFromValidate, self._ebFromValidate)
+
+    def _cbFromValidate(self, fromEmail, code=250, msg=b"Sender address accepted"):
+        self._from = fromEmail
+        self.sendCode(code, msg)
+
+    def _ebFromValidate(self, failure):
+        if failure.check(SMTPBadSender):
+            self.sendCode(
+                failure.value.code,
+                (
+                    b"Cannot receive from specified address "
+                    + quoteaddr(failure.value.addr)
+                    + b": "
+                    + networkString(failure.value.resp)
+                ),
+            )
+        elif failure.check(SMTPServerError):
+            self.sendCode(failure.value.code, networkString(failure.value.resp))
+        else:
+            log.err(failure, "SMTP sender validation failure")
+            self.sendCode(451, b"Requested action aborted: local error in processing")
+
+    def do_RCPT(self, rest):
+        if not self._from:
+            self.sendCode(503, b"Must have sender before recipient")
+            return
+        m = self.rcpt_re.match(rest)
+        if not m:
+            self.sendCode(501, b"Syntax error")
+            return
+
+        try:
+            user = User(m.group("path"), self._helo, self, self._from)
+        except AddressError as e:
+            self.sendCode(553, networkString(str(e)))
+            return
+
+        d = defer.maybeDeferred(self.validateTo, user)
+        d.addCallbacks(self._cbToValidate, self._ebToValidate, callbackArgs=(user,))
+
+    def _cbToValidate(self, to, user=None, code=250, msg=b"Recipient address accepted"):
+        if user is None:
+            user = to
+        self._to.append((user, to))
+        self.sendCode(code, msg)
+
+    def _ebToValidate(self, failure):
+        if failure.check(SMTPBadRcpt, SMTPServerError):
+            self.sendCode(failure.value.code, networkString(failure.value.resp))
+        else:
+            log.err(failure)
+            self.sendCode(451, b"Requested action aborted: local error in processing")
+
+    def _disconnect(self, msgs):
+        for msg in msgs:
+            try:
+                msg.connectionLost()
+            except BaseException:
+                log.msg("msg raised exception from connectionLost")
+                log.err()
+
+    def do_DATA(self, rest):
+        if self._from is None or (not self._to):
+            self.sendCode(503, b"Must have valid receiver and originator")
+            return
+        self.mode = DATA
+        helo, origin = self._helo, self._from
+        recipients = self._to
+
+        self._from = None
+        self._to = []
+        self.datafailed = None
+
+        msgs = []
+        for user, msgFunc in recipients:
+            try:
+                msg = msgFunc()
+                rcvdhdr = self.receivedHeader(helo, origin, [user])
+                if rcvdhdr:
+                    msg.lineReceived(rcvdhdr)
+                msgs.append(msg)
+            except SMTPServerError as e:
+                self.sendCode(e.code, e.resp)
+                self.mode = COMMAND
+                self._disconnect(msgs)
+                return
+            except BaseException:
+                log.err()
+                self.sendCode(550, b"Internal server error")
+                self.mode = COMMAND
+                self._disconnect(msgs)
+                return
+        self.__messages = msgs
+
+        self.__inheader = self.__inbody = 0
+        self.sendCode(354, b"Continue")
+
+        if self.noisy:
+            fmt = "Receiving message for delivery: from=%s to=%s"
+            log.msg(fmt % (origin, [str(u) for (u, f) in recipients]))
+
+    def connectionLost(self, reason):
+        # self.sendCode(421, 'Dropping connection.') # This does nothing...
+        # Ideally, if we (rather than the other side) lose the connection,
+        # we should be able to tell the other side that we are going away.
+        # RFC-2821 requires that we try.
+        if self.mode is DATA:
+            try:
+                for message in self.__messages:
+                    try:
+                        message.connectionLost()
+                    except BaseException:
+                        log.err()
+                del self.__messages
+            except AttributeError:
+                pass
+        if self._onLogout:
+            self._onLogout()
+            self._onLogout = None
+        self.setTimeout(None)
+
+    def do_RSET(self, rest):
+        self._from = None
+        self._to = []
+        self.sendCode(250, b"I remember nothing.")
+
+    def dataLineReceived(self, line):
+        if line[:1] == b".":
+            if line == b".":
+                self.mode = COMMAND
+                if self.datafailed:
+                    self.sendCode(self.datafailed.code, self.datafailed.resp)
+                    return
+                if not self.__messages:
+                    self._messageHandled("thrown away")
+                    return
+                defer.DeferredList(
+                    [m.eomReceived() for m in self.__messages], consumeErrors=True
+                ).addCallback(self._messageHandled)
+                del self.__messages
+                return
+            line = line[1:]
+
+        if self.datafailed:
+            return
+
+        try:
+            # Add a blank line between the generated Received:-header
+            # and the message body if the message comes in without any
+            # headers
+            if not self.__inheader and not self.__inbody:
+                if b":" in line:
+                    self.__inheader = 1
+                elif line:
+                    for message in self.__messages:
+                        message.lineReceived(b"")
+                    self.__inbody = 1
+
+            if not line:
+                self.__inbody = 1
+
+            for message in self.__messages:
+                message.lineReceived(line)
+        except SMTPServerError as e:
+            self.datafailed = e
+            for message in self.__messages:
+                message.connectionLost()
+
+    state_DATA = dataLineReceived
+
+    def _messageHandled(self, resultList):
+        failures = 0
+        for success, result in resultList:
+            if not success:
+                failures += 1
+                log.err(result)
+        if failures:
+            msg = "Could not send e-mail"
+            resultLen = len(resultList)
+            if resultLen > 1:
+                msg += f" ({failures} failures out of {resultLen} recipients)"
+            self.sendCode(550, networkString(msg))
+        else:
+            self.sendCode(250, b"Delivery in progress")
+
+    def _cbAnonymousAuthentication(self, result):
+        """
+        Save the state resulting from a successful anonymous cred login.
+        """
+        (iface, avatar, logout) = result
+        if issubclass(iface, IMessageDeliveryFactory):
+            self.deliveryFactory = avatar
+            self.delivery = None
+        elif issubclass(iface, IMessageDelivery):
+            self.deliveryFactory = None
+            self.delivery = avatar
+        else:
+            raise RuntimeError(f"{iface.__name__} is not a supported interface")
+        self._onLogout = logout
+        self.challenger = None
+
+    # overridable methods:
+    def validateFrom(self, helo, origin):
+        """
+        Validate the address from which the message originates.
+
+        @type helo: C{(bytes, bytes)}
+        @param helo: The argument to the HELO command and the client's IP
+        address.
+
+        @type origin: C{Address}
+        @param origin: The address the message is from
+
+        @rtype: C{Deferred} or C{Address}
+        @return: C{origin} or a C{Deferred} whose callback will be
+        passed C{origin}.
+
+        @raise SMTPBadSender: Raised of messages from this address are
+        not to be accepted.
+        """
+        if self.deliveryFactory is not None:
+            self.delivery = self.deliveryFactory.getMessageDelivery()
+
+        if self.delivery is not None:
+            return defer.maybeDeferred(self.delivery.validateFrom, helo, origin)
+
+        # No login has been performed, no default delivery object has been
+        # provided: try to perform an anonymous login and then invoke this
+        # method again.
+        if self.portal:
+            result = self.portal.login(
+                cred.credentials.Anonymous(),
+                None,
+                IMessageDeliveryFactory,
+                IMessageDelivery,
+            )
+
+            def ebAuthentication(err):
+                """
+                Translate cred exceptions into SMTP exceptions so that the
+                protocol code which invokes C{validateFrom} can properly report
+                the failure.
+                """
+                if err.check(cred.error.UnauthorizedLogin):
+                    exc = SMTPBadSender(origin)
+                elif err.check(cred.error.UnhandledCredentials):
+                    exc = SMTPBadSender(
+                        origin, resp="Unauthenticated senders not allowed"
+                    )
+                else:
+                    return err
+                return defer.fail(exc)
+
+            result.addCallbacks(self._cbAnonymousAuthentication, ebAuthentication)
+
+            def continueValidation(ignored):
+                """
+                Re-attempt from address validation.
+                """
+                return self.validateFrom(helo, origin)
+
+            result.addCallback(continueValidation)
+            return result
+
+        raise SMTPBadSender(origin)
+
+    def validateTo(self, user):
+        """
+        Validate the address for which the message is destined.
+
+        @type user: L{User}
+        @param user: The address to validate.
+
+        @rtype: no-argument callable
+        @return: A C{Deferred} which becomes, or a callable which
+        takes no arguments and returns an object implementing C{IMessage}.
+        This will be called and the returned object used to deliver the
+        message when it arrives.
+
+        @raise SMTPBadRcpt: Raised if messages to the address are
+        not to be accepted.
+        """
+        if self.delivery is not None:
+            return self.delivery.validateTo(user)
+        raise SMTPBadRcpt(user)
+
+    def receivedHeader(self, helo, origin, recipients):
+        if self.delivery is not None:
+            return self.delivery.receivedHeader(helo, origin, recipients)
+
+        heloStr = b""
+        if helo[0]:
+            heloStr = b" helo=" + helo[0]
+        domain = networkString(self.transport.getHost().host)
+
+        from_ = b"from " + helo[0] + b" ([" + helo[1] + b"]" + heloStr + b")"
+        by = b"by %s with %s (%s)" % (domain, self.__class__.__name__, longversion)
+        for_ = b"for %s; %s" % (" ".join(map(str, recipients)), rfc822date())
+        return b"Received: " + from_ + b"\n\t" + by + b"\n\t" + for_
+
+
+class SMTPFactory(protocol.ServerFactory):
+    """
+    Factory for SMTP.
+    """
+
+    # override in instances or subclasses
+    domain = DNSNAME
+    timeout = 600
+    protocol = SMTP
+
+    portal = None
+
+    def __init__(self, portal=None):
+        self.portal = portal
+
+    def buildProtocol(self, addr):
+        p = protocol.ServerFactory.buildProtocol(self, addr)
+        p.portal = self.portal
+        p.host = self.domain
+        return p
+
+
+class SMTPClient(basic.LineReceiver, policies.TimeoutMixin):
+    """
+    SMTP client for sending emails.
+
+    After the client has connected to the SMTP server, it repeatedly calls
+    L{SMTPClient.getMailFrom}, L{SMTPClient.getMailTo} and
+    L{SMTPClient.getMailData} and uses this information to send an email.
+    It then calls L{SMTPClient.getMailFrom} again; if it returns L{None}, the
+    client will disconnect, otherwise it will continue as normal i.e. call
+    L{SMTPClient.getMailTo} and L{SMTPClient.getMailData} and send a new email.
+    """
+
+    # If enabled then log SMTP client server communication
+    debug = True
+
+    # Number of seconds to wait before timing out a connection.  If
+    # None, perform no timeout checking.
+    timeout = None
+
+    def __init__(self, identity, logsize=10):
+        if isinstance(identity, str):
+            identity = identity.encode("ascii")
+
+        self.identity = identity or b""
+        self.toAddressesResult = []
+        self.successAddresses = []
+        self._from = None
+        self.resp = []
+        self.code = -1
+        self.log = util.LineLog(logsize)
+
+    def sendLine(self, line):
+        # Log sendLine only if you are in debug mode for performance
+        if self.debug:
+            self.log.append(b">>> " + line)
+
+        basic.LineReceiver.sendLine(self, line)
+
+    def connectionMade(self):
+        self.setTimeout(self.timeout)
+
+        self._expected = [220]
+        self._okresponse = self.smtpState_helo
+        self._failresponse = self.smtpConnectionFailed
+
+    def connectionLost(self, reason=protocol.connectionDone):
+        """
+        We are no longer connected
+        """
+        self.setTimeout(None)
+        self.mailFile = None
+
+    def timeoutConnection(self):
+        self.sendError(
+            SMTPTimeoutError(
+                -1, b"Timeout waiting for SMTP server response", self.log.str()
+            )
+        )
+
+    def lineReceived(self, line):
+        self.resetTimeout()
+
+        # Log lineReceived only if you are in debug mode for performance
+        if self.debug:
+            self.log.append(b"<<< " + line)
+
+        why = None
+
+        try:
+            self.code = int(line[:3])
+        except ValueError:
+            # This is a fatal error and will disconnect the transport
+            # lineReceived will not be called again.
+            self.sendError(
+                SMTPProtocolError(
+                    -1,
+                    f"Invalid response from SMTP server: {line}",
+                    self.log.str(),
+                )
+            )
+            return
+
+        if line[0:1] == b"0":
+            # Verbose informational message, ignore it
+            return
+
+        self.resp.append(line[4:])
+
+        if line[3:4] == b"-":
+            # Continuation
+            return
+
+        if self.code in self._expected:
+            why = self._okresponse(self.code, b"\n".join(self.resp))
+        else:
+            why = self._failresponse(self.code, b"\n".join(self.resp))
+
+        self.code = -1
+        self.resp = []
+        return why
+
+    def smtpConnectionFailed(self, code, resp):
+        self.sendError(SMTPConnectError(code, resp, self.log.str()))
+
+    def smtpTransferFailed(self, code, resp):
+        if code < 0:
+            self.sendError(SMTPProtocolError(code, resp, self.log.str()))
+        else:
+            self.smtpState_msgSent(code, resp)
+
+    def smtpState_helo(self, code, resp):
+        self.sendLine(b"HELO " + self.identity)
+        self._expected = SUCCESS
+        self._okresponse = self.smtpState_from
+
+    def smtpState_from(self, code, resp):
+        self._from = self.getMailFrom()
+        self._failresponse = self.smtpTransferFailed
+        if self._from is not None:
+            self.sendLine(b"MAIL FROM:" + quoteaddr(self._from))
+            self._expected = [250]
+            self._okresponse = self.smtpState_to
+        else:
+            # All messages have been sent, disconnect
+            self._disconnectFromServer()
+
+    def smtpState_disconnect(self, code, resp):
+        self.transport.loseConnection()
+
+    def smtpState_to(self, code, resp):
+        self.toAddresses = iter(self.getMailTo())
+        self.toAddressesResult = []
+        self.successAddresses = []
+        self._okresponse = self.smtpState_toOrData
+        self._expected = range(0, 1000)
+        self.lastAddress = None
+        return self.smtpState_toOrData(0, b"")
+
+    def smtpState_toOrData(self, code, resp):
+        if self.lastAddress is not None:
+            self.toAddressesResult.append((self.lastAddress, code, resp))
+            if code in SUCCESS:
+                self.successAddresses.append(self.lastAddress)
+        try:
+            self.lastAddress = next(self.toAddresses)
+        except StopIteration:
+            if self.successAddresses:
+                self.sendLine(b"DATA")
+                self._expected = [354]
+                self._okresponse = self.smtpState_data
+            else:
+                return self.smtpState_msgSent(code, "No recipients accepted")
+        else:
+            self.sendLine(b"RCPT TO:" + quoteaddr(self.lastAddress))
+
+    def smtpState_data(self, code, resp):
+        s = basic.FileSender()
+        d = s.beginFileTransfer(self.getMailData(), self.transport, self.transformChunk)
+
+        def ebTransfer(err):
+            self.sendError(err.value)
+
+        d.addCallbacks(self.finishedFileTransfer, ebTransfer)
+        self._expected = SUCCESS
+        self._okresponse = self.smtpState_msgSent
+
+    def smtpState_msgSent(self, code, resp):
+        if self._from is not None:
+            self.sentMail(
+                code, resp, len(self.successAddresses), self.toAddressesResult, self.log
+            )
+
+        self.toAddressesResult = []
+        self._from = None
+        self.sendLine(b"RSET")
+        self._expected = SUCCESS
+        self._okresponse = self.smtpState_from
+
+    ##
+    ## Helpers for FileSender
+    ##
+    def transformChunk(self, chunk):
+        """
+        Perform the necessary local to network newline conversion and escape
+        leading periods.
+
+        This method also resets the idle timeout so that as long as process is
+        being made sending the message body, the client will not time out.
+        """
+        self.resetTimeout()
+        return chunk.replace(b"\n", b"\r\n").replace(b"\r\n.", b"\r\n..")
+
+    def finishedFileTransfer(self, lastsent):
+        if lastsent != b"\n":
+            line = b"\r\n."
+        else:
+            line = b"."
+        self.sendLine(line)
+
+    ##
+    # these methods should be overridden in subclasses
+    def getMailFrom(self):
+        """
+        Return the email address the mail is from.
+        """
+        raise NotImplementedError
+
+    def getMailTo(self):
+        """
+        Return a list of emails to send to.
+        """
+        raise NotImplementedError
+
+    def getMailData(self):
+        """
+        Return file-like object containing data of message to be sent.
+
+        Lines in the file should be delimited by '\\n'.
+        """
+        raise NotImplementedError
+
+    def sendError(self, exc):
+        """
+        If an error occurs before a mail message is sent sendError will be
+        called.  This base class method sends a QUIT if the error is
+        non-fatal and disconnects the connection.
+
+        @param exc: The SMTPClientError (or child class) raised
+        @type exc: C{SMTPClientError}
+        """
+        if isinstance(exc, SMTPClientError) and not exc.isFatal:
+            self._disconnectFromServer()
+        else:
+            # If the error was fatal then the communication channel with the
+            # SMTP Server is broken so just close the transport connection
+            self.smtpState_disconnect(-1, None)
+
+    def sentMail(self, code, resp, numOk, addresses, log):
+        """
+        Called when an attempt to send an email is completed.
+
+        If some addresses were accepted, code and resp are the response
+        to the DATA command. If no addresses were accepted, code is -1
+        and resp is an informative message.
+
+        @param code: the code returned by the SMTP Server
+        @param resp: The string response returned from the SMTP Server
+        @param numOk: the number of addresses accepted by the remote host.
+        @param addresses: is a list of tuples (address, code, resp) listing
+                          the response to each RCPT command.
+        @param log: is the SMTP session log
+        """
+        raise NotImplementedError
+
+    def _disconnectFromServer(self):
+        self._expected = range(0, 1000)
+        self._okresponse = self.smtpState_disconnect
+        self.sendLine(b"QUIT")
+
+
+class ESMTPClient(SMTPClient):
+    """
+    A client for sending emails over ESMTP.
+
+    @ivar heloFallback: Whether or not to fall back to plain SMTP if the C{EHLO}
+        command is not recognised by the server. If L{requireAuthentication} is
+        C{True}, or L{requireTransportSecurity} is C{True} and the connection is
+        not over TLS, this fallback flag will not be honored.
+    @type heloFallback: L{bool}
+
+    @ivar requireAuthentication: If C{True}, refuse to proceed if authentication
+        cannot be performed. Overrides L{heloFallback}.
+    @type requireAuthentication: L{bool}
+
+    @ivar requireTransportSecurity: If C{True}, refuse to proceed if the
+        transport cannot be secured. If the transport layer is not already
+        secured via TLS, this will override L{heloFallback}.
+    @type requireAuthentication: L{bool}
+
+    @ivar context: The context factory to use for STARTTLS, if desired.
+    @type context: L{IOpenSSLClientConnectionCreator}
+
+    @ivar _tlsMode: Whether or not the connection is over TLS.
+    @type _tlsMode: L{bool}
+    """
+
+    heloFallback = True
+    requireAuthentication = False
+    requireTransportSecurity = False
+    context = None
+    _tlsMode = False
+
+    def __init__(self, secret, contextFactory=None, *args, **kw):
+        SMTPClient.__init__(self, *args, **kw)
+        self.authenticators = []
+        self.secret = secret
+        self.context = contextFactory
+
+    def __getattr__(self, name):
+        if name == "tlsMode":
+            warnings.warn(
+                "tlsMode attribute of twisted.mail.smtp.ESMTPClient "
+                "is deprecated since Twisted 13.0",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            return self._tlsMode
+        else:
+            raise AttributeError(
+                "%s instance has no attribute %r"
+                % (
+                    self.__class__.__name__,
+                    name,
+                )
+            )
+
+    def __setattr__(self, name, value):
+        if name == "tlsMode":
+            warnings.warn(
+                "tlsMode attribute of twisted.mail.smtp.ESMTPClient "
+                "is deprecated since Twisted 13.0",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            self._tlsMode = value
+        else:
+            self.__dict__[name] = value
+
+    def esmtpEHLORequired(self, code=-1, resp=None):
+        """
+        Fail because authentication is required, but the server does not support
+        ESMTP, which is required for authentication.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+        """
+        self.sendError(
+            EHLORequiredError(
+                502, b"Server does not support ESMTP " b"Authentication", self.log.str()
+            )
+        )
+
+    def esmtpAUTHRequired(self, code=-1, resp=None):
+        """
+        Fail because authentication is required, but the server does not support
+        any schemes we support.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+        """
+        tmp = []
+
+        for a in self.authenticators:
+            tmp.append(a.getName().upper())
+
+        auth = b"[%s]" % b", ".join(tmp)
+
+        self.sendError(
+            AUTHRequiredError(
+                502,
+                b"Server does not support Client " b"Authentication schemes %s" % auth,
+                self.log.str(),
+            )
+        )
+
+    def esmtpTLSRequired(self, code=-1, resp=None):
+        """
+        Fail because TLS is required and the server does not support it.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+        """
+        self.sendError(
+            TLSRequiredError(
+                502,
+                b"Server does not support secure " b"communication via TLS / SSL",
+                self.log.str(),
+            )
+        )
+
+    def esmtpTLSFailed(self, code=-1, resp=None):
+        """
+        Fail because the TLS handshake wasn't able to be completed.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+        """
+        self.sendError(
+            TLSError(
+                code, b"Could not complete the SSL/TLS " b"handshake", self.log.str()
+            )
+        )
+
+    def esmtpAUTHDeclined(self, code=-1, resp=None):
+        """
+        Fail because the authentication was rejected.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+        """
+        self.sendError(AUTHDeclinedError(code, resp, self.log.str()))
+
+    def esmtpAUTHMalformedChallenge(self, code=-1, resp=None):
+        """
+        Fail because the server sent a malformed authentication challenge.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+        """
+        self.sendError(
+            AuthenticationError(
+                501,
+                b"Login failed because the "
+                b"SMTP Server returned a malformed Authentication Challenge",
+                self.log.str(),
+            )
+        )
+
+    def esmtpAUTHServerError(self, code=-1, resp=None):
+        """
+        Fail because of some other authentication error.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+        """
+        self.sendError(AuthenticationError(code, resp, self.log.str()))
+
+    def registerAuthenticator(self, auth):
+        """
+        Registers an Authenticator with the ESMTPClient. The ESMTPClient will
+        attempt to login to the SMTP Server in the order the Authenticators are
+        registered. The most secure Authentication mechanism should be
+        registered first.
+
+        @param auth: The Authentication mechanism to register
+        @type auth: L{IClientAuthentication} implementor
+
+        @return: L{None}
+        """
+        self.authenticators.append(auth)
+
+    def connectionMade(self):
+        """
+        Called when a connection has been made, and triggers sending an C{EHLO}
+        to the server.
+        """
+        self._tlsMode = ISSLTransport.providedBy(self.transport)
+        SMTPClient.connectionMade(self)
+        self._okresponse = self.esmtpState_ehlo
+
+    def esmtpState_ehlo(self, code, resp):
+        """
+        Send an C{EHLO} to the server.
+
+        If L{heloFallback} is C{True}, and there is no requirement for TLS or
+        authentication, the client will fall back to basic SMTP.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+
+        @return: L{None}
+        """
+        self._expected = SUCCESS
+
+        self._okresponse = self.esmtpState_serverConfig
+        self._failresponse = self.esmtpEHLORequired
+
+        if self._tlsMode:
+            needTLS = False
+        else:
+            needTLS = self.requireTransportSecurity
+
+        if self.heloFallback and not self.requireAuthentication and not needTLS:
+            self._failresponse = self.smtpState_helo
+
+        self.sendLine(b"EHLO " + self.identity)
+
+    def esmtpState_serverConfig(self, code, resp):
+        """
+        Handle a positive response to the I{EHLO} command by parsing the
+        capabilities in the server's response and then taking the most
+        appropriate next step towards entering a mail transaction.
+        """
+        items = {}
+        for line in resp.splitlines():
+            e = line.split(None, 1)
+            if len(e) > 1:
+                items[e[0]] = e[1]
+            else:
+                items[e[0]] = None
+
+        self.tryTLS(code, resp, items)
+
+    def tryTLS(self, code, resp, items):
+        """
+        Take a necessary step towards being able to begin a mail transaction.
+
+        The step may be to ask the server to being a TLS session.  If TLS is
+        already in use or not necessary and not available then the step may be
+        to authenticate with the server.  If TLS is necessary and not available,
+        fail the mail transmission attempt.
+
+        This is an internal helper method.
+
+        @param code: The server status code from the most recently received
+            server message.
+        @type code: L{int}
+
+        @param resp: The server status response from the most recently received
+            server message.
+        @type resp: L{bytes}
+
+        @param items: A mapping of ESMTP extensions offered by the server.  Keys
+            are extension identifiers and values are the associated values.
+        @type items: L{dict} mapping L{bytes} to L{bytes}
+
+        @return: L{None}
+        """
+
+        # has tls        can tls         must tls       result
+        #   t               t               t           authenticate
+        #   t               t               f           authenticate
+        #   t               f               t           authenticate
+        #   t               f               f           authenticate
+
+        #   f               t               t           STARTTLS
+        #   f               t               f           STARTTLS
+        #   f               f               t           esmtpTLSRequired
+        #   f               f               f           authenticate
+
+        hasTLS = self._tlsMode
+        canTLS = self.context and b"STARTTLS" in items
+        mustTLS = self.requireTransportSecurity
+
+        if hasTLS or not (canTLS or mustTLS):
+            self.authenticate(code, resp, items)
+        elif canTLS:
+            self._expected = [220]
+            self._okresponse = self.esmtpState_starttls
+            self._failresponse = self.esmtpTLSFailed
+            self.sendLine(b"STARTTLS")
+        else:
+            self.esmtpTLSRequired()
+
+    def esmtpState_starttls(self, code, resp):
+        """
+        Handle a positive response to the I{STARTTLS} command by starting a new
+        TLS session on C{self.transport}.
+
+        Upon success, re-handshake with the server to discover what capabilities
+        it has when TLS is in use.
+        """
+        try:
+            self.transport.startTLS(self.context)
+            self._tlsMode = True
+        except BaseException:
+            log.err()
+            self.esmtpTLSFailed(451)
+
+        # Send another EHLO once TLS has been started to
+        # get the TLS / AUTH schemes. Some servers only allow AUTH in TLS mode.
+        self.esmtpState_ehlo(code, resp)
+
+    def authenticate(self, code, resp, items):
+        if self.secret and items.get(b"AUTH"):
+            schemes = items[b"AUTH"].split()
+            tmpSchemes = {}
+
+            # XXX: May want to come up with a more efficient way to do this
+            for s in schemes:
+                tmpSchemes[s.upper()] = 1
+
+            for a in self.authenticators:
+                auth = a.getName().upper()
+
+                if auth in tmpSchemes:
+                    self._authinfo = a
+
+                    # Special condition handled
+                    if auth == b"PLAIN":
+                        self._okresponse = self.smtpState_from
+                        self._failresponse = self._esmtpState_plainAuth
+                        self._expected = [235]
+                        challenge = base64.b64encode(
+                            self._authinfo.challengeResponse(self.secret, 1)
+                        )
+                        self.sendLine(b"AUTH %s %s" % (auth, challenge))
+                    else:
+                        self._expected = [334]
+                        self._okresponse = self.esmtpState_challenge
+                        # If some error occurs here, the server declined the
+                        # AUTH before the user / password phase. This would be
+                        # a very rare case
+                        self._failresponse = self.esmtpAUTHServerError
+                        self.sendLine(b"AUTH " + auth)
+                    return
+
+        if self.requireAuthentication:
+            self.esmtpAUTHRequired()
+        else:
+            self.smtpState_from(code, resp)
+
+    def _esmtpState_plainAuth(self, code, resp):
+        self._okresponse = self.smtpState_from
+        self._failresponse = self.esmtpAUTHDeclined
+        self._expected = [235]
+        challenge = base64.b64encode(self._authinfo.challengeResponse(self.secret, 2))
+        self.sendLine(b"AUTH PLAIN " + challenge)
+
+    def esmtpState_challenge(self, code, resp):
+        self._authResponse(self._authinfo, resp)
+
+    def _authResponse(self, auth, challenge):
+        self._failresponse = self.esmtpAUTHDeclined
+        try:
+            challenge = base64.b64decode(challenge)
+        except binascii.Error:
+            # Illegal challenge, give up, then quit
+            self.sendLine(b"*")
+            self._okresponse = self.esmtpAUTHMalformedChallenge
+            self._failresponse = self.esmtpAUTHMalformedChallenge
+        else:
+            resp = auth.challengeResponse(self.secret, challenge)
+            self._expected = [235, 334]
+            self._okresponse = self.smtpState_maybeAuthenticated
+            self.sendLine(base64.b64encode(resp))
+
+    def smtpState_maybeAuthenticated(self, code, resp):
+        """
+        Called to handle the next message from the server after sending a
+        response to a SASL challenge.  The server response might be another
+        challenge or it might indicate authentication has succeeded.
+        """
+        if code == 235:
+            # Yes, authenticated!
+            del self._authinfo
+            self.smtpState_from(code, resp)
+        else:
+            # No, not authenticated yet.  Keep trying.
+            self._authResponse(self._authinfo, resp)
+
+
+class ESMTP(SMTP):
+    ctx = None
+    canStartTLS = False
+    startedTLS = False
+
+    authenticated = False
+
+    def __init__(self, chal=None, contextFactory=None):
+        SMTP.__init__(self)
+        if chal is None:
+            chal = {}
+        self.challengers = chal
+        self.authenticated = False
+        self.ctx = contextFactory
+
+    def connectionMade(self):
+        SMTP.connectionMade(self)
+        self.canStartTLS = ITLSTransport.providedBy(self.transport)
+        self.canStartTLS = self.canStartTLS and (self.ctx is not None)
+
+    def greeting(self):
+        return SMTP.greeting(self) + b" ESMTP"
+
+    def extensions(self):
+        """
+        SMTP service extensions
+
+        @return: the SMTP service extensions that are supported.
+        @rtype: L{dict} with L{bytes} keys and a value of either L{None} or a
+            L{list} of L{bytes}.
+        """
+        ext = {b"AUTH": list(self.challengers.keys())}
+        if self.canStartTLS and not self.startedTLS:
+            ext[b"STARTTLS"] = None
+        return ext
+
+    def lookupMethod(self, command):
+        command = nativeString(command)
+
+        m = SMTP.lookupMethod(self, command)
+        if m is None:
+            m = getattr(self, "ext_" + command.upper(), None)
+        return m
+
+    def listExtensions(self):
+        r = []
+        for c, v in self.extensions().items():
+            if v is not None:
+                if v:
+                    # Intentionally omit extensions with empty argument lists
+                    r.append(c + b" " + b" ".join(v))
+            else:
+                r.append(c)
+
+        return b"\n".join(r)
+
+    def do_EHLO(self, rest):
+        peer = self.transport.getPeer().host
+
+        if not isinstance(peer, bytes):
+            peer = peer.encode("idna")
+
+        self._helo = (rest, peer)
+        self._from = None
+        self._to = []
+        self.sendCode(
+            250,
+            (
+                self.host
+                + b" Hello "
+                + peer
+                + b", nice to meet you\n"
+                + self.listExtensions()
+            ),
+        )
+
+    def ext_STARTTLS(self, rest):
+        if self.startedTLS:
+            self.sendCode(503, b"TLS already negotiated")
+        elif self.ctx and self.canStartTLS:
+            self.sendCode(220, b"Begin TLS negotiation now")
+            self.transport.startTLS(self.ctx)
+            self.startedTLS = True
+        else:
+            self.sendCode(454, b"TLS not available")
+
+    def ext_AUTH(self, rest):
+        if self.authenticated:
+            self.sendCode(503, b"Already authenticated")
+            return
+        parts = rest.split(None, 1)
+        chal = self.challengers.get(parts[0].upper(), lambda: None)()
+        if not chal:
+            self.sendCode(504, b"Unrecognized authentication type")
+            return
+
+        self.mode = AUTH
+        self.challenger = chal
+
+        if len(parts) > 1:
+            chal.getChallenge()  # Discard it, apparently the client does not
+            # care about it.
+            rest = parts[1]
+        else:
+            rest = None
+        self.state_AUTH(rest)
+
+    def _cbAuthenticated(self, loginInfo):
+        """
+        Save the state resulting from a successful cred login and mark this
+        connection as authenticated.
+        """
+        result = SMTP._cbAnonymousAuthentication(self, loginInfo)
+        self.authenticated = True
+        return result
+
+    def _ebAuthenticated(self, reason):
+        """
+        Handle cred login errors by translating them to the SMTP authenticate
+        failed.  Translate all other errors into a generic SMTP error code and
+        log the failure for inspection.  Stop all errors from propagating.
+
+        @param reason: Reason for failure.
+        """
+        self.challenge = None
+        if reason.check(cred.error.UnauthorizedLogin):
+            self.sendCode(535, b"Authentication failed")
+        else:
+            log.err(reason, "SMTP authentication failure")
+            self.sendCode(451, b"Requested action aborted: local error in processing")
+
+    def state_AUTH(self, response):
+        """
+        Handle one step of challenge/response authentication.
+
+        @param response: The text of a response. If None, this
+        function has been called as a result of an AUTH command with
+        no initial response. A response of '*' aborts authentication,
+        as per RFC 2554.
+        """
+        if self.portal is None:
+            self.sendCode(454, b"Temporary authentication failure")
+            self.mode = COMMAND
+            return
+
+        if response is None:
+            challenge = self.challenger.getChallenge()
+            encoded = base64.b64encode(challenge)
+            self.sendCode(334, encoded)
+            return
+
+        if response == b"*":
+            self.sendCode(501, b"Authentication aborted")
+            self.challenger = None
+            self.mode = COMMAND
+            return
+
+        try:
+            uncoded = base64.b64decode(response)
+        except (TypeError, binascii.Error):
+            self.sendCode(501, b"Syntax error in parameters or arguments")
+            self.challenger = None
+            self.mode = COMMAND
+            return
+
+        self.challenger.setResponse(uncoded)
+        if self.challenger.moreChallenges():
+            challenge = self.challenger.getChallenge()
+            coded = base64.b64encode(challenge)
+            self.sendCode(334, coded)
+            return
+
+        self.mode = COMMAND
+        result = self.portal.login(
+            self.challenger, None, IMessageDeliveryFactory, IMessageDelivery
+        )
+        result.addCallback(self._cbAuthenticated)
+        result.addCallback(
+            lambda ign: self.sendCode(235, b"Authentication successful.")
+        )
+        result.addErrback(self._ebAuthenticated)
+
+
+class SenderMixin:
+    """
+    Utility class for sending emails easily.
+
+    Use with SMTPSenderFactory or ESMTPSenderFactory.
+    """
+
+    done = 0
+
+    def getMailFrom(self):
+        if not self.done:
+            self.done = 1
+            return str(self.factory.fromEmail)
+        else:
+            return None
+
+    def getMailTo(self):
+        return self.factory.toEmail
+
+    def getMailData(self):
+        return self.factory.file
+
+    def sendError(self, exc):
+        # Call the base class to close the connection with the SMTP server
+        SMTPClient.sendError(self, exc)
+
+        #  Do not retry to connect to SMTP Server if:
+        #   1. No more retries left (This allows the correct error to be returned to the errorback)
+        #   2. retry is false
+        #   3. The error code is not in the 4xx range (Communication Errors)
+
+        if self.factory.retries >= 0 or (
+            not exc.retry and not (exc.code >= 400 and exc.code < 500)
+        ):
+            self.factory.sendFinished = True
+            self.factory.result.errback(exc)
+
+    def sentMail(self, code, resp, numOk, addresses, log):
+        # Do not retry, the SMTP server acknowledged the request
+        self.factory.sendFinished = True
+        if code not in SUCCESS:
+            errlog = []
+            for addr, acode, aresp in addresses:
+                if acode not in SUCCESS:
+                    errlog.append(
+                        addr + b": " + networkString("%03d" % (acode,)) + b" " + aresp
+                    )
+
+            errlog.append(log.str())
+
+            exc = SMTPDeliveryError(code, resp, b"\n".join(errlog), addresses)
+            self.factory.result.errback(exc)
+        else:
+            self.factory.result.callback((numOk, addresses))
+
+
+class SMTPSender(SenderMixin, SMTPClient):
+    """
+    SMTP protocol that sends a single email based on information it
+    gets from its factory, a L{SMTPSenderFactory}.
+    """
+
+
+class SMTPSenderFactory(protocol.ClientFactory):
+    """
+    Utility factory for sending emails easily.
+
+    @type currentProtocol: L{SMTPSender}
+    @ivar currentProtocol: The current running protocol returned by
+        L{buildProtocol}.
+
+    @type sendFinished: C{bool}
+    @ivar sendFinished: When the value is set to True, it means the message has
+        been sent or there has been an unrecoverable error or the sending has
+        been cancelled. The default value is False.
+    """
+
+    domain = DNSNAME
+    protocol: Type[SMTPClient] = SMTPSender
+
+    def __init__(self, fromEmail, toEmail, file, deferred, retries=5, timeout=None):
+        """
+        @param fromEmail: The RFC 2821 address from which to send this
+        message.
+
+        @param toEmail: A sequence of RFC 2821 addresses to which to
+        send this message.
+
+        @param file: A file-like object containing the message to send.
+
+        @param deferred: A Deferred to callback or errback when sending
+        of this message completes.
+        @type deferred: L{defer.Deferred}
+
+        @param retries: The number of times to retry delivery of this
+        message.
+
+        @param timeout: Period, in seconds, for which to wait for
+        server responses, or None to wait forever.
+        """
+        assert isinstance(retries, int)
+
+        if isinstance(toEmail, str):
+            toEmail = [toEmail.encode("ascii")]
+        elif isinstance(toEmail, bytes):
+            toEmail = [toEmail]
+        else:
+            toEmailFinal = []
+            for _email in toEmail:
+                if not isinstance(_email, bytes):
+                    _email = _email.encode("ascii")
+
+                toEmailFinal.append(_email)
+            toEmail = toEmailFinal
+
+        self.fromEmail = Address(fromEmail)
+        self.nEmails = len(toEmail)
+        self.toEmail = toEmail
+        self.file = file
+        self.result = deferred
+        self.result.addBoth(self._removeDeferred)
+        self.sendFinished = False
+        self.currentProtocol = None
+
+        self.retries = -retries
+        self.timeout = timeout
+
+    def _removeDeferred(self, result):
+        del self.result
+        return result
+
+    def clientConnectionFailed(self, connector, err):
+        self._processConnectionError(connector, err)
+
+    def clientConnectionLost(self, connector, err):
+        self._processConnectionError(connector, err)
+
+    def _processConnectionError(self, connector, err):
+        self.currentProtocol = None
+        if (self.retries < 0) and (not self.sendFinished):
+            log.msg("SMTP Client retrying server. Retry: %s" % -self.retries)
+
+            # Rewind the file in case part of it was read while attempting to
+            # send the message.
+            self.file.seek(0, 0)
+            connector.connect()
+            self.retries += 1
+        elif not self.sendFinished:
+            # If we were unable to communicate with the SMTP server a ConnectionDone will be
+            # returned. We want a more clear error message for debugging
+            if err.check(error.ConnectionDone):
+                err.value = SMTPConnectError(-1, "Unable to connect to server.")
+            self.result.errback(err.value)
+
+    def buildProtocol(self, addr):
+        p = self.protocol(self.domain, self.nEmails * 2 + 2)
+        p.factory = self
+        p.timeout = self.timeout
+        self.currentProtocol = p
+        self.result.addBoth(self._removeProtocol)
+        return p
+
+    def _removeProtocol(self, result):
+        """
+        Remove the protocol created in C{buildProtocol}.
+
+        @param result: The result/error passed to the callback/errback of
+            L{defer.Deferred}.
+
+        @return: The C{result} untouched.
+        """
+        if self.currentProtocol:
+            self.currentProtocol = None
+        return result
+
+
+class LOGINCredentials(_lcredentials):
+    """
+    L{LOGINCredentials} generates challenges for I{LOGIN} authentication.
+
+    For interoperability with Outlook, the challenge generated does not exactly
+    match the one defined in the
+    U{draft specification<http://sepp.oetiker.ch/sasl-2.1.19-ds/draft-murchison-sasl-login-00.txt>}.
+    """
+
+    def __init__(self):
+        _lcredentials.__init__(self)
+        self.challenges = [b"Password:", b"Username:"]
+
+
+@implementer(IClientAuthentication)
+class PLAINAuthenticator:
+    def __init__(self, user):
+        self.user = user
+
+    def getName(self):
+        return b"PLAIN"
+
+    def challengeResponse(self, secret, chal=1):
+        if chal == 1:
+            return self.user + b"\0" + self.user + b"\0" + secret
+        else:
+            return b"\0" + self.user + b"\0" + secret
+
+
+class ESMTPSender(SenderMixin, ESMTPClient):
+    requireAuthentication = True
+    requireTransportSecurity = True
+
+    def __init__(self, username, secret, contextFactory=None, *args, **kw):
+        self.heloFallback = 0
+        self.username = username
+
+        self._hostname = kw.pop("hostname", None)
+
+        if contextFactory is None:
+            contextFactory = self._getContextFactory()
+
+        ESMTPClient.__init__(self, secret, contextFactory, *args, **kw)
+
+        self._registerAuthenticators()
+
+    def _registerAuthenticators(self):
+        # Register Authenticator in order from most secure to least secure
+        self.registerAuthenticator(CramMD5ClientAuthenticator(self.username))
+        self.registerAuthenticator(LOGINAuthenticator(self.username))
+        self.registerAuthenticator(PLAINAuthenticator(self.username))
+
+    def _getContextFactory(self):
+        if self.context is not None:
+            return self.context
+        if self._hostname is None:
+            return None
+        try:
+            from twisted.internet.ssl import optionsForClientTLS
+        except ImportError:
+            return None
+        else:
+            context = optionsForClientTLS(self._hostname)
+            return context
+
+
+class ESMTPSenderFactory(SMTPSenderFactory):
+    """
+    Utility factory for sending emails easily.
+
+    @type currentProtocol: L{ESMTPSender}
+    @ivar currentProtocol: The current running protocol as made by
+        L{buildProtocol}.
+    """
+
+    protocol = ESMTPSender
+
+    def __init__(
+        self,
+        username,
+        password,
+        fromEmail,
+        toEmail,
+        file,
+        deferred,
+        retries=5,
+        timeout=None,
+        contextFactory=None,
+        heloFallback=False,
+        requireAuthentication=True,
+        requireTransportSecurity=True,
+        hostname=None,
+    ):
+        SMTPSenderFactory.__init__(
+            self, fromEmail, toEmail, file, deferred, retries, timeout
+        )
+        self.username = username
+        self.password = password
+        self._contextFactory = contextFactory
+        self._heloFallback = heloFallback
+        self._requireAuthentication = requireAuthentication
+        self._requireTransportSecurity = requireTransportSecurity
+        self._hostname = hostname
+
+    def buildProtocol(self, addr):
+        """
+        Build an L{ESMTPSender} protocol configured with C{heloFallback},
+        C{requireAuthentication}, and C{requireTransportSecurity} as specified
+        in L{__init__}.
+
+        This sets L{currentProtocol} on the factory, as well as returning it.
+
+        @rtype: L{ESMTPSender}
+        """
+        p = self.protocol(
+            self.username,
+            self.password,
+            self._contextFactory,
+            self.domain,
+            self.nEmails * 2 + 2,
+            hostname=self._hostname,
+        )
+        p.heloFallback = self._heloFallback
+        p.requireAuthentication = self._requireAuthentication
+        p.requireTransportSecurity = self._requireTransportSecurity
+        p.factory = self
+        p.timeout = self.timeout
+        self.currentProtocol = p
+        self.result.addBoth(self._removeProtocol)
+        return p
+
+
+def sendmail(
+    smtphost,
+    from_addr,
+    to_addrs,
+    msg,
+    senderDomainName=None,
+    port=25,
+    reactor=reactor,
+    username=None,
+    password=None,
+    requireAuthentication=False,
+    requireTransportSecurity=False,
+):
+    """
+    Send an email.
+
+    This interface is intended to be a replacement for L{smtplib.SMTP.sendmail}
+    and related methods. To maintain backwards compatibility, it will fall back
+    to plain SMTP, if ESMTP support is not available. If ESMTP support is
+    available, it will attempt to provide encryption via STARTTLS and
+    authentication if a secret is provided.
+
+    @param smtphost: The host the message should be sent to.
+    @type smtphost: L{bytes}
+
+    @param from_addr: The (envelope) address sending this mail.
+    @type from_addr: L{bytes}
+
+    @param to_addrs: A list of addresses to send this mail to.  A string will
+        be treated as a list of one address.
+    @type to_addrs: L{list} of L{bytes} or L{bytes}
+
+    @param msg: The message, including headers, either as a file or a string.
+        File-like objects need to support read() and close(). Lines must be
+        delimited by '\\n'. If you pass something that doesn't look like a file,
+        we try to convert it to a string (so you should be able to pass an
+        L{email.message} directly, but doing the conversion with
+        L{email.generator} manually will give you more control over the process).
+
+    @param senderDomainName: Name by which to identify. If None, try to pick
+        something sane (but this depends on external configuration and may not
+        succeed).
+    @type senderDomainName: L{bytes}
+
+    @param port: Remote port to which to connect.
+    @type port: L{int}
+
+    @param username: The username to use, if wanting to authenticate.
+    @type username: L{bytes} or L{unicode}
+
+    @param password: The secret to use, if wanting to authenticate. If you do
+        not specify this, SMTP authentication will not occur.
+    @type password: L{bytes} or L{unicode}
+
+    @param requireTransportSecurity: Whether or not STARTTLS is required.
+    @type requireTransportSecurity: L{bool}
+
+    @param requireAuthentication: Whether or not authentication is required.
+    @type requireAuthentication: L{bool}
+
+    @param reactor: The L{reactor} used to make the TCP connection.
+
+    @rtype: L{Deferred}
+    @returns: A cancellable L{Deferred}, its callback will be called if a
+        message is sent to ANY address, the errback if no message is sent. When
+        the C{cancel} method is called, it will stop retrying and disconnect
+        the connection immediately.
+
+        The callback will be called with a tuple (numOk, addresses) where numOk
+        is the number of successful recipient addresses and addresses is a list
+        of tuples (address, code, resp) giving the response to the RCPT command
+        for each address.
+    """
+    if not hasattr(msg, "read"):
+        # It's not a file
+        msg = BytesIO(bytes(msg))
+
+    def cancel(d):
+        """
+        Cancel the L{twisted.mail.smtp.sendmail} call, tell the factory not to
+        retry and disconnect the connection.
+
+        @param d: The L{defer.Deferred} to be cancelled.
+        """
+        factory.sendFinished = True
+        if factory.currentProtocol:
+            factory.currentProtocol.transport.abortConnection()
+        else:
+            # Connection hasn't been made yet
+            connector.disconnect()
+
+    d = defer.Deferred(cancel)
+
+    if isinstance(username, str):
+        username = username.encode("utf-8")
+    if isinstance(password, str):
+        password = password.encode("utf-8")
+
+    tlsHostname = smtphost
+    if not isinstance(tlsHostname, str):
+        tlsHostname = _idnaText(tlsHostname)
+
+    factory = ESMTPSenderFactory(
+        username,
+        password,
+        from_addr,
+        to_addrs,
+        msg,
+        d,
+        heloFallback=True,
+        requireAuthentication=requireAuthentication,
+        requireTransportSecurity=requireTransportSecurity,
+        hostname=tlsHostname,
+    )
+
+    if senderDomainName is not None:
+        factory.domain = networkString(senderDomainName)
+
+    connector = reactor.connectTCP(smtphost, port, factory)
+
+    return d
+
+
+import codecs
+
+
+def xtext_encode(s, errors=None):
+    r = []
+    for ch in iterbytes(s):
+        o = ord(ch)
+        if ch == "+" or ch == "=" or o < 33 or o > 126:
+            r.append(networkString(f"+{o:02X}"))
+        else:
+            r.append(bytes((o,)))
+    return (b"".join(r), len(s))
+
+
+def xtext_decode(s, errors=None):
+    """
+    Decode the xtext-encoded string C{s}.
+
+    @param s: String to decode.
+    @param errors: codec error handling scheme.
+    @return: The decoded string.
+    """
+    r = []
+    i = 0
+    while i < len(s):
+        if s[i : i + 1] == b"+":
+            try:
+                r.append(chr(int(bytes(s[i + 1 : i + 3]), 16)))
+            except ValueError:
+                r.append(ord(s[i : i + 3]))
+            i += 3
+        else:
+            r.append(bytes(s[i : i + 1]).decode("ascii"))
+            i += 1
+    return ("".join(r), len(s))
+
+
+class xtextStreamReader(codecs.StreamReader):
+    def decode(self, s, errors="strict"):
+        return xtext_decode(s)
+
+
+class xtextStreamWriter(codecs.StreamWriter):
+    def decode(self, s, errors="strict"):
+        return xtext_encode(s)
+
+
+def xtext_codec(name):
+    if name == "xtext":
+        return (xtext_encode, xtext_decode, xtextStreamReader, xtextStreamWriter)
+
+
+codecs.register(xtext_codec)
diff --git a/contrib/python/Twisted/py3/twisted/mail/tap.py b/contrib/python/Twisted/py3/twisted/mail/tap.py
new file mode 100644
index 00000000000..1217808e3af
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/mail/tap.py
@@ -0,0 +1,384 @@
+# -*- test-case-name: twisted.mail.test.test_options -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Support for creating mail servers with twistd.
+"""
+
+import os
+
+from twisted.application import internet
+from twisted.cred import checkers, strcred
+from twisted.internet import endpoints
+from twisted.mail import alias, mail, maildir, relay, relaymanager
+from twisted.python import usage
+
+
+class Options(usage.Options, strcred.AuthOptionMixin):
+    """
+    An options list parser for twistd mail.
+
+    @type synopsis: L{bytes}
+    @ivar synopsis: A description of options for use in the usage message.
+
+    @type optParameters: L{list} of L{list} of (0) L{bytes}, (1) L{bytes},
+        (2) L{object}, (3) L{bytes}, (4) L{None} or
+        callable which takes L{bytes} and returns L{object}
+    @ivar optParameters: Information about supported parameters.  See
+        L{Options <twisted.python.usage.Options>} for details.
+
+    @type optFlags: L{list} of L{list} of (0) L{bytes}, (1) L{bytes} or
+        L{None}, (2) L{bytes}
+    @ivar optFlags: Information about supported flags.  See
+        L{Options <twisted.python.usage.Options>} for details.
+
+    @type _protoDefaults: L{dict} mapping L{bytes} to L{int}
+    @ivar _protoDefaults: A mapping of default service to port.
+
+    @type compData: L{Completions <usage.Completions>}
+    @ivar compData: Metadata for the shell tab completion system.
+
+    @type longdesc: L{bytes}
+    @ivar longdesc: A long description of the plugin for use in the usage
+        message.
+
+    @type service: L{MailService}
+    @ivar service: The email service.
+
+    @type last_domain: L{IDomain} provider or L{None}
+    @ivar last_domain: The most recently specified domain.
+    """
+
+    synopsis = "[options]"
+
+    optParameters = [
+        [
+            "relay",
+            "R",
+            None,
+            "Relay messages according to their envelope 'To', using "
+            "the given path as a queue directory.",
+        ],
+        ["hostname", "H", None, "The hostname by which to identify this server."],
+    ]
+
+    optFlags = [
+        ["esmtp", "E", "Use RFC 1425/1869 SMTP extensions"],
+        ["disable-anonymous", None, "Disallow non-authenticated SMTP connections"],
+        ["no-pop3", None, "Disable the default POP3 server."],
+        ["no-smtp", None, "Disable the default SMTP server."],
+    ]
+
+    _protoDefaults = {
+        "pop3": 8110,
+        "smtp": 8025,
+    }
+
+    compData = usage.Completions(optActions={"hostname": usage.CompleteHostnames()})
+
+    longdesc = """
+    An SMTP / POP3 email server plugin for twistd.
+
+    Examples:
+
+    1. SMTP and POP server
+
+    twistd mail --maildirdbmdomain=example.com=/tmp/example.com
+    --user=joe=password
+
+    Starts an SMTP server that only accepts emails to joe@example.com and saves
+    them to /tmp/example.com.
+
+    Also starts a POP mail server which will allow a client to log in using
+    username: joe@example.com and password: password and collect any email that
+    has been saved in /tmp/example.com.
+
+    2. SMTP relay
+
+    twistd mail --relay=/tmp/mail_queue
+
+    Starts an SMTP server that accepts emails to any email address and relays
+    them to an appropriate remote SMTP server. Queued emails will be
+    temporarily stored in /tmp/mail_queue.
+    """
+
+    def __init__(self):
+        """
+        Parse options and create a mail service.
+        """
+        usage.Options.__init__(self)
+        self.service = mail.MailService()
+        self.last_domain = None
+        for service in self._protoDefaults:
+            self[service] = []
+
+    def addEndpoint(self, service, description):
+        """
+        Add an endpoint to a service.
+
+        @type service: L{bytes}
+        @param service: A service, either C{b'smtp'} or C{b'pop3'}.
+
+        @type description: L{bytes}
+        @param description: An endpoint description string or a TCP port
+            number.
+        """
+        from twisted.internet import reactor
+
+        self[service].append(endpoints.serverFromString(reactor, description))
+
+    def opt_pop3(self, description):
+        """
+        Add a POP3 port listener on the specified endpoint.
+
+        You can listen on multiple ports by specifying multiple --pop3 options.
+        """
+        self.addEndpoint("pop3", description)
+
+    opt_p = opt_pop3
+
+    def opt_smtp(self, description):
+        """
+        Add an SMTP port listener on the specified endpoint.
+
+        You can listen on multiple ports by specifying multiple --smtp options.
+        """
+        self.addEndpoint("smtp", description)
+
+    opt_s = opt_smtp
+
+    def opt_default(self):
+        """
+        Make the most recently specified domain the default domain.
+        """
+        if self.last_domain:
+            self.service.addDomain("", self.last_domain)
+        else:
+            raise usage.UsageError("Specify a domain before specifying using --default")
+
+    opt_D = opt_default
+
+    def opt_maildirdbmdomain(self, domain):
+        """
+        Generate an SMTP/POP3 virtual domain.
+
+        This option requires an argument of the form 'NAME=PATH' where NAME is
+        the DNS domain name for which email will be accepted and where PATH is
+        a the filesystem path to a Maildir folder.
+        [Example: 'example.com=/tmp/example.com']
+        """
+        try:
+            name, path = domain.split("=")
+        except ValueError:
+            raise usage.UsageError(
+                "Argument to --maildirdbmdomain must be of the form 'name=path'"
+            )
+
+        self.last_domain = maildir.MaildirDirdbmDomain(
+            self.service, os.path.abspath(path)
+        )
+        self.service.addDomain(name, self.last_domain)
+
+    opt_d = opt_maildirdbmdomain
+
+    def opt_user(self, user_pass):
+        """
+        Add a user and password to the last specified domain.
+        """
+        try:
+            user, password = user_pass.split("=", 1)
+        except ValueError:
+            raise usage.UsageError(
+                "Argument to --user must be of the form 'user=password'"
+            )
+        if self.last_domain:
+            self.last_domain.addUser(user, password)
+        else:
+            raise usage.UsageError("Specify a domain before specifying users")
+
+    opt_u = opt_user
+
+    def opt_bounce_to_postmaster(self):
+        """
+        Send undeliverable messages to the postmaster.
+        """
+        self.last_domain.postmaster = 1
+
+    opt_b = opt_bounce_to_postmaster
+
+    def opt_aliases(self, filename):
+        """
+        Specify an aliases(5) file to use for the last specified domain.
+        """
+        if self.last_domain is not None:
+            if mail.IAliasableDomain.providedBy(self.last_domain):
+                aliases = alias.loadAliasFile(self.service.domains, filename)
+                self.last_domain.setAliasGroup(aliases)
+                self.service.monitor.monitorFile(
+                    filename, AliasUpdater(self.service.domains, self.last_domain)
+                )
+            else:
+                raise usage.UsageError(
+                    "%s does not support alias files"
+                    % (self.last_domain.__class__.__name__,)
+                )
+        else:
+            raise usage.UsageError("Specify a domain before specifying aliases")
+
+    opt_A = opt_aliases
+
+    def _getEndpoints(self, reactor, service):
+        """
+        Return a list of endpoints for the specified service, constructing
+        defaults if necessary.
+
+        If no endpoints were configured for the service and the protocol
+        was not explicitly disabled with a I{--no-*} option, a default
+        endpoint for the service is created.
+
+        @type reactor: L{IReactorTCP <twisted.internet.interfaces.IReactorTCP>}
+            provider
+        @param reactor: If any endpoints are created, the reactor with
+            which they are created.
+
+        @type service: L{bytes}
+        @param service: The type of service for which to retrieve endpoints,
+            either C{b'pop3'} or C{b'smtp'}.
+
+        @rtype: L{list} of L{IStreamServerEndpoint
+            <twisted.internet.interfaces.IStreamServerEndpoint>} provider
+        @return: The endpoints for the specified service as configured by the
+            command line parameters.
+        """
+        if self[service]:
+            # If there are any services set up, just return those.
+            return self[service]
+        elif self["no-" + service]:
+            # If there are no services, but the service was explicitly disabled,
+            # return nothing.
+            return []
+        else:
+            # Otherwise, return the old default service.
+            return [endpoints.TCP4ServerEndpoint(reactor, self._protoDefaults[service])]
+
+    def postOptions(self):
+        """
+        Check the validity of the specified set of options and
+        configure authentication.
+
+        @raise UsageError: When the set of options is invalid.
+        """
+        from twisted.internet import reactor
+
+        if self["esmtp"] and self["hostname"] is None:
+            raise usage.UsageError("--esmtp requires --hostname")
+
+        # If the --auth option was passed, this will be present -- otherwise,
+        # it won't be, which is also a perfectly valid state.
+        if "credCheckers" in self:
+            for ch in self["credCheckers"]:
+                self.service.smtpPortal.registerChecker(ch)
+
+        if not self["disable-anonymous"]:
+            self.service.smtpPortal.registerChecker(checkers.AllowAnonymousAccess())
+
+        anything = False
+        for service in self._protoDefaults:
+            self[service] = self._getEndpoints(reactor, service)
+            if self[service]:
+                anything = True
+
+        if not anything:
+            raise usage.UsageError("You cannot disable all protocols")
+
+
+class AliasUpdater:
+    """
+    A callable object which updates the aliases for a domain from an aliases(5)
+    file.
+
+    @ivar domains: See L{__init__}.
+    @ivar domain: See L{__init__}.
+    """
+
+    def __init__(self, domains, domain):
+        """
+        @type domains: L{dict} mapping L{bytes} to L{IDomain} provider
+        @param domains: A mapping of domain name to domain object
+
+        @type domain: L{IAliasableDomain} provider
+        @param domain: The domain to update.
+        """
+        self.domains = domains
+        self.domain = domain
+
+    def __call__(self, new):
+        """
+        Update the aliases for a domain from an aliases(5) file.
+
+        @type new: L{bytes}
+        @param new: The name of an aliases(5) file.
+        """
+        self.domain.setAliasGroup(alias.loadAliasFile(self.domains, new))
+
+
+def makeService(config):
+    """
+    Configure a service for operating a mail server.
+
+    The returned service may include POP3 servers, SMTP servers, or both,
+    depending on the configuration passed in.  If there are multiple servers,
+    they will share all of their non-network state (i.e. the same user accounts
+    are available on all of them).
+
+    @type config: L{Options <usage.Options>}
+    @param config: Configuration options specifying which servers to include in
+        the returned service and where they should keep mail data.
+
+    @rtype: L{IService <twisted.application.service.IService>} provider
+    @return: A service which contains the requested mail servers.
+    """
+    if config["esmtp"]:
+        rmType = relaymanager.SmartHostESMTPRelayingManager
+        smtpFactory = config.service.getESMTPFactory
+    else:
+        rmType = relaymanager.SmartHostSMTPRelayingManager
+        smtpFactory = config.service.getSMTPFactory
+
+    if config["relay"]:
+        dir = config["relay"]
+        if not os.path.isdir(dir):
+            os.mkdir(dir)
+
+        config.service.setQueue(relaymanager.Queue(dir))
+        default = relay.DomainQueuer(config.service)
+
+        manager = rmType(config.service.queue)
+        if config["esmtp"]:
+            manager.fArgs += (None, None)
+        manager.fArgs += (config["hostname"],)
+
+        helper = relaymanager.RelayStateHelper(manager, 1)
+        helper.setServiceParent(config.service)
+        config.service.domains.setDefaultDomain(default)
+
+    if config["pop3"]:
+        f = config.service.getPOP3Factory()
+        for endpoint in config["pop3"]:
+            svc = internet.StreamServerEndpointService(endpoint, f)
+            svc.setServiceParent(config.service)
+
+    if config["smtp"]:
+        f = smtpFactory()
+        if config["hostname"]:
+            f.domain = config["hostname"]
+            f.fArgs = (f.domain,)
+        if config["esmtp"]:
+            f.fArgs = (None, None) + f.fArgs
+        for endpoint in config["smtp"]:
+            svc = internet.StreamServerEndpointService(endpoint, f)
+            svc.setServiceParent(config.service)
+
+    return config.service
diff --git a/contrib/python/Twisted/py3/twisted/names/__init__.py b/contrib/python/Twisted/py3/twisted/names/__init__.py
new file mode 100644
index 00000000000..ccdf8ba3319
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Names: DNS server and client implementations.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/names/_rfc1982.py b/contrib/python/Twisted/py3/twisted/names/_rfc1982.py
new file mode 100644
index 00000000000..61d43c009b1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/_rfc1982.py
@@ -0,0 +1,261 @@
+# -*- test-case-name: twisted.names.test.test_rfc1982 -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Utilities for handling RFC1982 Serial Number Arithmetic.
+
+@see: U{http://tools.ietf.org/html/rfc1982}
+
+@var RFC4034_TIME_FORMAT: RRSIG Time field presentation format. The Signature
+   Expiration Time and Inception Time field values MUST be represented either
+   as an unsigned decimal integer indicating seconds since 1 January 1970
+   00:00:00 UTC, or in the form YYYYMMDDHHmmSS in UTC. See U{RRSIG Presentation
+   Format<https://tools.ietf.org/html/rfc4034#section-3.2>}
+"""
+
+
+import calendar
+from datetime import datetime, timedelta
+
+from twisted.python.compat import nativeString
+from twisted.python.util import FancyStrMixin
+
+RFC4034_TIME_FORMAT = "%Y%m%d%H%M%S"
+
+
+class SerialNumber(FancyStrMixin):
+    """
+    An RFC1982 Serial Number.
+
+    This class implements RFC1982 DNS Serial Number Arithmetic.
+
+    SNA is used in DNS and specifically in DNSSEC as defined in RFC4034 in the
+    DNSSEC Signature Expiration and Inception Fields.
+
+    @see: U{https://tools.ietf.org/html/rfc1982}
+    @see: U{https://tools.ietf.org/html/rfc4034}
+
+    @ivar _serialBits: See C{serialBits} of L{__init__}.
+    @ivar _number: See C{number} of L{__init__}.
+    @ivar _modulo: The value at which wrapping will occur.
+    @ivar _halfRing: Half C{_modulo}. If another L{SerialNumber} value is larger
+        than this, it would lead to a wrapped value which is larger than the
+        first and comparisons are therefore ambiguous.
+    @ivar _maxAdd: Half C{_modulo} plus 1. If another L{SerialNumber} value is
+        larger than this, it would lead to a wrapped value which is larger than
+        the first. Comparisons with the original value would therefore be
+        ambiguous.
+    """
+
+    showAttributes = (
+        ("_number", "number", "%d"),
+        ("_serialBits", "serialBits", "%d"),
+    )
+
+    def __init__(self, number: int, serialBits: int = 32):
+        """
+        Construct an L{SerialNumber} instance.
+
+        @param number: An L{int} which will be stored as the modulo
+            C{number % 2 ^ serialBits}
+        @type number: L{int}
+
+        @param serialBits: The size of the serial number space. The power of two
+            which results in one larger than the largest integer corresponding
+            to a serial number value.
+        @type serialBits: L{int}
+        """
+        self._serialBits = serialBits
+        self._modulo = 2**serialBits
+        self._halfRing: int = 2 ** (serialBits - 1)
+        self._maxAdd = 2 ** (serialBits - 1) - 1
+        self._number: int = int(number) % self._modulo
+
+    def _convertOther(self, other: object) -> "SerialNumber":
+        """
+        Check that a foreign object is suitable for use in the comparison or
+        arithmetic magic methods of this L{SerialNumber} instance. Raise
+        L{TypeError} if not.
+
+        @param other: The foreign L{object} to be checked.
+        @return: C{other} after compatibility checks and possible coercion.
+        @raise TypeError: If C{other} is not compatible.
+        """
+        if not isinstance(other, SerialNumber):
+            raise TypeError(f"cannot compare or combine {self!r} and {other!r}")
+
+        if self._serialBits != other._serialBits:
+            raise TypeError(
+                "cannot compare or combine SerialNumber instances with "
+                "different serialBits. %r and %r" % (self, other)
+            )
+
+        return other
+
+    def __str__(self) -> str:
+        """
+        Return a string representation of this L{SerialNumber} instance.
+
+        @rtype: L{nativeString}
+        """
+        return nativeString("%d" % (self._number,))
+
+    def __int__(self):
+        """
+        @return: The integer value of this L{SerialNumber} instance.
+        @rtype: L{int}
+        """
+        return self._number
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Allow rich equality comparison with another L{SerialNumber} instance.
+        """
+        try:
+            other = self._convertOther(other)
+        except TypeError:
+            return NotImplemented
+        return other._number == self._number
+
+    def __lt__(self, other: object) -> bool:
+        """
+        Allow I{less than} comparison with another L{SerialNumber} instance.
+        """
+        try:
+            other = self._convertOther(other)
+        except TypeError:
+            return NotImplemented
+        return (
+            self._number < other._number
+            and (other._number - self._number) < self._halfRing
+        ) or (
+            self._number > other._number
+            and (self._number - other._number) > self._halfRing
+        )
+
+    def __gt__(self, other: object) -> bool:
+        """
+        Allow I{greater than} comparison with another L{SerialNumber} instance.
+        """
+        try:
+            other_sn = self._convertOther(other)
+        except TypeError:
+            return NotImplemented
+        return (
+            self._number < other_sn._number
+            and (other_sn._number - self._number) > self._halfRing
+        ) or (
+            self._number > other_sn._number
+            and (self._number - other_sn._number) < self._halfRing
+        )
+
+    def __le__(self, other: object) -> bool:
+        """
+        Allow I{less than or equal} comparison with another L{SerialNumber}
+        instance.
+        """
+        try:
+            other = self._convertOther(other)
+        except TypeError:
+            return NotImplemented
+        return self == other or self < other
+
+    def __ge__(self, other: object) -> bool:
+        """
+        Allow I{greater than or equal} comparison with another L{SerialNumber}
+        instance.
+        """
+        try:
+            other = self._convertOther(other)
+        except TypeError:
+            return NotImplemented
+        return self == other or self > other
+
+    def __add__(self, other: object) -> "SerialNumber":
+        """
+        Allow I{addition} with another L{SerialNumber} instance.
+
+        Serial numbers may be incremented by the addition of a positive
+        integer n, where n is taken from the range of integers
+        [0 .. (2^(SERIAL_BITS - 1) - 1)].  For a sequence number s, the
+        result of such an addition, s', is defined as
+
+        s' = (s + n) modulo (2 ^ SERIAL_BITS)
+
+        where the addition and modulus operations here act upon values that are
+        non-negative values of unbounded size in the usual ways of integer
+        arithmetic.
+
+        Addition of a value outside the range
+        [0 .. (2^(SERIAL_BITS - 1) - 1)] is undefined.
+
+        @see: U{http://tools.ietf.org/html/rfc1982#section-3.1}
+
+        @raise ArithmeticError: If C{other} is more than C{_maxAdd}
+            ie more than half the maximum value of this serial number.
+        """
+        try:
+            other = self._convertOther(other)
+        except TypeError:
+            return NotImplemented
+        if other._number <= self._maxAdd:
+            return SerialNumber(
+                (self._number + other._number) % self._modulo,
+                serialBits=self._serialBits,
+            )
+        else:
+            raise ArithmeticError(
+                "value %r outside the range 0 .. %r"
+                % (
+                    other._number,
+                    self._maxAdd,
+                )
+            )
+
+    def __hash__(self):
+        """
+        Allow L{SerialNumber} instances to be hashed for use as L{dict} keys.
+
+        @rtype: L{int}
+        """
+        return hash(self._number)
+
+    @classmethod
+    def fromRFC4034DateString(cls, utcDateString):
+        """
+        Create an L{SerialNumber} instance from a date string in format
+        'YYYYMMDDHHMMSS' described in U{RFC4034
+        3.2<https://tools.ietf.org/html/rfc4034#section-3.2>}.
+
+        The L{SerialNumber} instance stores the date as a 32bit UNIX timestamp.
+
+        @see: U{https://tools.ietf.org/html/rfc4034#section-3.1.5}
+
+        @param utcDateString: A UTC date/time string of format I{YYMMDDhhmmss}
+            which will be converted to seconds since the UNIX epoch.
+        @type utcDateString: L{unicode}
+
+        @return: An L{SerialNumber} instance containing the supplied date as a
+            32bit UNIX timestamp.
+        """
+        parsedDate = datetime.strptime(utcDateString, RFC4034_TIME_FORMAT)
+        secondsSinceEpoch = calendar.timegm(parsedDate.utctimetuple())
+        return cls(secondsSinceEpoch, serialBits=32)
+
+    def toRFC4034DateString(self):
+        """
+        Calculate a date by treating the current L{SerialNumber} value as a UNIX
+        timestamp and return a date string in the format described in
+        U{RFC4034 3.2<https://tools.ietf.org/html/rfc4034#section-3.2>}.
+
+        @return: The date string.
+        """
+        # Can't use datetime.utcfromtimestamp, because it seems to overflow the
+        # signed 32bit int used in the underlying C library. SNA is unsigned
+        # and capable of handling all timestamps up to 2**32.
+        d = datetime(1970, 1, 1) + timedelta(seconds=self._number)
+        return nativeString(d.strftime(RFC4034_TIME_FORMAT))
+
+
+__all__ = ["SerialNumber"]
diff --git a/contrib/python/Twisted/py3/twisted/names/authority.py b/contrib/python/Twisted/py3/twisted/names/authority.py
new file mode 100644
index 00000000000..33df6c00686
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/authority.py
@@ -0,0 +1,503 @@
+# -*- test-case-name: twisted.names.test.test_names -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Authoritative resolvers.
+"""
+
+
+import os
+import time
+
+from twisted.internet import defer
+from twisted.names import common, dns, error
+from twisted.python import failure
+from twisted.python.compat import execfile, nativeString
+from twisted.python.filepath import FilePath
+
+
+def getSerial(filename="/tmp/twisted-names.serial"):
+    """
+    Return a monotonically increasing (across program runs) integer.
+
+    State is stored in the given file.  If it does not exist, it is
+    created with rw-/---/--- permissions.
+
+    This manipulates process-global state by calling C{os.umask()}, so it isn't
+    thread-safe.
+
+    @param filename: Path to a file that is used to store the state across
+        program runs.
+    @type filename: L{str}
+
+    @return: a monotonically increasing number
+    @rtype: L{str}
+    """
+    serial = time.strftime("%Y%m%d")
+
+    o = os.umask(0o177)
+    try:
+        if not os.path.exists(filename):
+            with open(filename, "w") as f:
+                f.write(serial + " 0")
+    finally:
+        os.umask(o)
+
+    with open(filename) as serialFile:
+        lastSerial, zoneID = serialFile.readline().split()
+
+    zoneID = (lastSerial == serial) and (int(zoneID) + 1) or 0
+
+    with open(filename, "w") as serialFile:
+        serialFile.write("%s %d" % (serial, zoneID))
+
+    serial = serial + ("%02d" % (zoneID,))
+    return serial
+
+
+class FileAuthority(common.ResolverBase):
+    """
+    An Authority that is loaded from a file.
+
+    This is an abstract class that implements record search logic. To create
+    a functional resolver, subclass it and override the L{loadFile} method.
+
+    @ivar _ADDITIONAL_PROCESSING_TYPES: Record types for which additional
+        processing will be done.
+
+    @ivar _ADDRESS_TYPES: Record types which are useful for inclusion in the
+        additional section generated during additional processing.
+
+    @ivar soa: A 2-tuple containing the SOA domain name as a L{bytes} and a
+        L{dns.Record_SOA}.
+
+    @ivar records: A mapping of domains (as lowercased L{bytes}) to records.
+    @type records: L{dict} with L{bytes} keys
+    """
+
+    # See https://twistedmatrix.com/trac/ticket/6650
+    _ADDITIONAL_PROCESSING_TYPES = (dns.CNAME, dns.MX, dns.NS)
+    _ADDRESS_TYPES = (dns.A, dns.AAAA)
+
+    soa = None
+    records = None
+
+    def __init__(self, filename):
+        common.ResolverBase.__init__(self)
+        self.loadFile(filename)
+        self._cache = {}
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+
+    def loadFile(self, filename):
+        """
+        Load DNS records from a file.
+
+        This method populates the I{soa} and I{records} attributes. It must be
+        overridden in a subclass. It is called once from the initializer.
+
+        @param filename: The I{filename} parameter that was passed to the
+        initilizer.
+
+        @returns: L{None} -- the return value is ignored
+        """
+
+    def _additionalRecords(self, answer, authority, ttl):
+        """
+        Find locally known information that could be useful to the consumer of
+        the response and construct appropriate records to include in the
+        I{additional} section of that response.
+
+        Essentially, implement RFC 1034 section 4.3.2 step 6.
+
+        @param answer: A L{list} of the records which will be included in the
+            I{answer} section of the response.
+
+        @param authority: A L{list} of the records which will be included in
+            the I{authority} section of the response.
+
+        @param ttl: The default TTL for records for which this is not otherwise
+            specified.
+
+        @return: A generator of L{dns.RRHeader} instances for inclusion in the
+            I{additional} section.  These instances represent extra information
+            about the records in C{answer} and C{authority}.
+        """
+        for record in answer + authority:
+            if record.type in self._ADDITIONAL_PROCESSING_TYPES:
+                name = record.payload.name.name
+                for rec in self.records.get(name.lower(), ()):
+                    if rec.TYPE in self._ADDRESS_TYPES:
+                        yield dns.RRHeader(
+                            name, rec.TYPE, dns.IN, rec.ttl or ttl, rec, auth=True
+                        )
+
+    def _lookup(self, name, cls, type, timeout=None):
+        """
+        Determine a response to a particular DNS query.
+
+        @param name: The name which is being queried and for which to lookup a
+            response.
+        @type name: L{bytes}
+
+        @param cls: The class which is being queried.  Only I{IN} is
+            implemented here and this value is presently disregarded.
+        @type cls: L{int}
+
+        @param type: The type of records being queried.  See the types defined
+            in L{twisted.names.dns}.
+        @type type: L{int}
+
+        @param timeout: All processing is done locally and a result is
+            available immediately, so the timeout value is ignored.
+
+        @return: A L{Deferred} that fires with a L{tuple} of three sets of
+            response records (to comprise the I{answer}, I{authority}, and
+            I{additional} sections of a DNS response) or with a L{Failure} if
+            there is a problem processing the query.
+        """
+        cnames = []
+        results = []
+        authority = []
+        additional = []
+        default_ttl = max(self.soa[1].minimum, self.soa[1].expire)
+
+        domain_records = self.records.get(name.lower())
+
+        if domain_records:
+            for record in domain_records:
+                if record.ttl is not None:
+                    ttl = record.ttl
+                else:
+                    ttl = default_ttl
+
+                if record.TYPE == dns.NS and name.lower() != self.soa[0].lower():
+                    # NS record belong to a child zone: this is a referral.  As
+                    # NS records are authoritative in the child zone, ours here
+                    # are not.  RFC 2181, section 6.1.
+                    authority.append(
+                        dns.RRHeader(name, record.TYPE, dns.IN, ttl, record, auth=False)
+                    )
+                elif record.TYPE == type or type == dns.ALL_RECORDS:
+                    results.append(
+                        dns.RRHeader(name, record.TYPE, dns.IN, ttl, record, auth=True)
+                    )
+                if record.TYPE == dns.CNAME:
+                    cnames.append(
+                        dns.RRHeader(name, record.TYPE, dns.IN, ttl, record, auth=True)
+                    )
+            if not results:
+                results = cnames
+
+            # Sort of https://tools.ietf.org/html/rfc1034#section-4.3.2 .
+            # See https://twistedmatrix.com/trac/ticket/6732
+            additionalInformation = self._additionalRecords(
+                results, authority, default_ttl
+            )
+            if cnames:
+                results.extend(additionalInformation)
+            else:
+                additional.extend(additionalInformation)
+
+            if not results and not authority:
+                # Empty response. Include SOA record to allow clients to cache
+                # this response. RFC 1034, sections 3.7 and 4.3.4, and RFC 2181
+                # section 7.1.
+                authority.append(
+                    dns.RRHeader(
+                        self.soa[0], dns.SOA, dns.IN, ttl, self.soa[1], auth=True
+                    )
+                )
+            return defer.succeed((results, authority, additional))
+        else:
+            if dns._isSubdomainOf(name, self.soa[0]):
+                # We may be the authority and we didn't find it.
+                # XXX: The QNAME may also be in a delegated child zone. See
+                # #6581 and #6580
+                return defer.fail(failure.Failure(dns.AuthoritativeDomainError(name)))
+            else:
+                # The QNAME is not a descendant of this zone. Fail with
+                # DomainError so that the next chained authority or
+                # resolver will be queried.
+                return defer.fail(failure.Failure(error.DomainError(name)))
+
+    def lookupZone(self, name, timeout=10):
+        name = dns.domainString(name)
+        if self.soa[0].lower() == name.lower():
+            # Wee hee hee hooo yea
+            default_ttl = max(self.soa[1].minimum, self.soa[1].expire)
+            if self.soa[1].ttl is not None:
+                soa_ttl = self.soa[1].ttl
+            else:
+                soa_ttl = default_ttl
+            results = [
+                dns.RRHeader(
+                    self.soa[0], dns.SOA, dns.IN, soa_ttl, self.soa[1], auth=True
+                )
+            ]
+            for k, r in self.records.items():
+                for rec in r:
+                    if rec.ttl is not None:
+                        ttl = rec.ttl
+                    else:
+                        ttl = default_ttl
+                    if rec.TYPE != dns.SOA:
+                        results.append(
+                            dns.RRHeader(k, rec.TYPE, dns.IN, ttl, rec, auth=True)
+                        )
+            results.append(results[0])
+            return defer.succeed((results, (), ()))
+        return defer.fail(failure.Failure(dns.DomainError(name)))
+
+    def _cbAllRecords(self, results):
+        ans, auth, add = [], [], []
+        for res in results:
+            if res[0]:
+                ans.extend(res[1][0])
+                auth.extend(res[1][1])
+                add.extend(res[1][2])
+        return ans, auth, add
+
+
+class PySourceAuthority(FileAuthority):
+    """
+    A FileAuthority that is built up from Python source code.
+    """
+
+    def loadFile(self, filename):
+        g, l = self.setupConfigNamespace(), {}
+        execfile(filename, g, l)
+        if "zone" not in l:
+            raise ValueError("No zone defined in " + filename)
+
+        self.records = {}
+        for rr in l["zone"]:
+            if isinstance(rr[1], dns.Record_SOA):
+                self.soa = rr
+            self.records.setdefault(rr[0].lower(), []).append(rr[1])
+
+    def wrapRecord(self, type):
+        def wrapRecordFunc(name, *arg, **kw):
+            return (dns.domainString(name), type(*arg, **kw))
+
+        return wrapRecordFunc
+
+    def setupConfigNamespace(self):
+        r = {}
+        items = dns.__dict__.keys()
+        for record in [x for x in items if x.startswith("Record_")]:
+            type = getattr(dns, record)
+            f = self.wrapRecord(type)
+            r[record[len("Record_") :]] = f
+        return r
+
+
+class BindAuthority(FileAuthority):
+    """
+    An Authority that loads U{BIND zone files
+    <https://en.wikipedia.org/wiki/Zone_file>}.
+
+    Supports only C{$ORIGIN} and C{$TTL} directives.
+    """
+
+    def loadFile(self, filename):
+        """
+        Load records from C{filename}.
+
+        @param filename: file to read from
+        @type filename: L{bytes}
+        """
+        fp = FilePath(filename)
+        # Not the best way to set an origin. It can be set using $ORIGIN
+        # though.
+        self.origin = nativeString(fp.basename() + b".")
+
+        lines = fp.getContent().splitlines(True)
+        lines = self.stripComments(lines)
+        lines = self.collapseContinuations(lines)
+        self.parseLines(lines)
+
+    def stripComments(self, lines):
+        """
+        Strip comments from C{lines}.
+
+        @param lines: lines to work on
+        @type lines: iterable of L{bytes}
+
+        @return: C{lines} sans comments.
+        """
+        return (
+            a.find(b";") == -1 and a or a[: a.find(b";")]
+            for a in [b.strip() for b in lines]
+        )
+
+    def collapseContinuations(self, lines):
+        """
+        Transform multiline statements into single lines.
+
+        @param lines: lines to work on
+        @type lines: iterable of L{bytes}
+
+        @return: iterable of continuous lines
+        """
+        l = []
+        state = 0
+        for line in lines:
+            if state == 0:
+                if line.find(b"(") == -1:
+                    l.append(line)
+                else:
+                    l.append(line[: line.find(b"(")])
+                    state = 1
+            else:
+                if line.find(b")") != -1:
+                    l[-1] += b" " + line[: line.find(b")")]
+                    state = 0
+                else:
+                    l[-1] += b" " + line
+        return filter(None, (line.split() for line in l))
+
+    def parseLines(self, lines):
+        """
+        Parse C{lines}.
+
+        @param lines: lines to work on
+        @type lines: iterable of L{bytes}
+        """
+        ttl = 60 * 60 * 3
+        origin = self.origin
+
+        self.records = {}
+
+        for line in lines:
+            if line[0] == b"$TTL":
+                ttl = dns.str2time(line[1])
+            elif line[0] == b"$ORIGIN":
+                origin = line[1]
+            elif line[0] == b"$INCLUDE":
+                raise NotImplementedError("$INCLUDE directive not implemented")
+            elif line[0] == b"$GENERATE":
+                raise NotImplementedError("$GENERATE directive not implemented")
+            else:
+                self.parseRecordLine(origin, ttl, line)
+
+        # If the origin changed, reflect that within the instance.
+        self.origin = origin
+
+    def addRecord(self, owner, ttl, type, domain, cls, rdata):
+        """
+        Add a record to our authority.  Expand domain with origin if necessary.
+
+        @param owner: origin?
+        @type owner: L{bytes}
+
+        @param ttl: time to live for the record
+        @type ttl: L{int}
+
+        @param domain: the domain for which the record is to be added
+        @type domain: L{bytes}
+
+        @param type: record type
+        @type type: L{str}
+
+        @param cls: record class
+        @type cls: L{str}
+
+        @param rdata: record data
+        @type rdata: L{list} of L{bytes}
+        """
+        if not domain.endswith(b"."):
+            domain = domain + b"." + owner[:-1]
+        else:
+            domain = domain[:-1]
+        f = getattr(self, f"class_{cls}", None)
+        if f:
+            f(ttl, type, domain, rdata)
+        else:
+            raise NotImplementedError(f"Record class {cls!r} not supported")
+
+    def class_IN(self, ttl, type, domain, rdata):
+        """
+        Simulate a class IN and recurse into the actual class.
+
+        @param ttl: time to live for the record
+        @type ttl: L{int}
+
+        @param type: record type
+        @type type: str
+
+        @param domain: the domain
+        @type domain: bytes
+
+        @param rdata:
+        @type rdata: bytes
+        """
+        record = getattr(dns, f"Record_{nativeString(type)}", None)
+        if record:
+            r = record(*rdata)
+            r.ttl = ttl
+            self.records.setdefault(domain.lower(), []).append(r)
+
+            if type == "SOA":
+                self.soa = (domain, r)
+        else:
+            raise NotImplementedError(
+                f"Record type {nativeString(type)!r} not supported"
+            )
+
+    def parseRecordLine(self, origin, ttl, line):
+        """
+        Parse a C{line} from a zone file respecting C{origin} and C{ttl}.
+
+        Add resulting records to authority.
+
+        @param origin: starting point for the zone
+        @type origin: L{bytes}
+
+        @param ttl: time to live for the record
+        @type ttl: L{int}
+
+        @param line: zone file line to parse; split by word
+        @type line: L{list} of L{bytes}
+        """
+        queryClasses = {qc.encode("ascii") for qc in dns.QUERY_CLASSES.values()}
+        queryTypes = {qt.encode("ascii") for qt in dns.QUERY_TYPES.values()}
+
+        markers = queryClasses | queryTypes
+
+        cls = b"IN"
+        owner = origin
+
+        if line[0] == b"@":
+            line = line[1:]
+            owner = origin
+        elif not line[0].isdigit() and line[0] not in markers:
+            owner = line[0]
+            line = line[1:]
+
+        if line[0].isdigit() or line[0] in markers:
+            domain = owner
+            owner = origin
+        else:
+            domain = line[0]
+            line = line[1:]
+
+        if line[0] in queryClasses:
+            cls = line[0]
+            line = line[1:]
+            if line[0].isdigit():
+                ttl = int(line[0])
+                line = line[1:]
+        elif line[0].isdigit():
+            ttl = int(line[0])
+            line = line[1:]
+            if line[0] in queryClasses:
+                cls = line[0]
+                line = line[1:]
+
+        type = line[0]
+        rdata = line[1:]
+
+        self.addRecord(owner, ttl, nativeString(type), domain, nativeString(cls), rdata)
diff --git a/contrib/python/Twisted/py3/twisted/names/cache.py b/contrib/python/Twisted/py3/twisted/names/cache.py
new file mode 100644
index 00000000000..a3833d7ab1f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/cache.py
@@ -0,0 +1,131 @@
+# -*- test-case-name: twisted.names.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An in-memory caching resolver.
+"""
+
+
+from twisted.internet import defer
+from twisted.names import common, dns
+from twisted.python import failure, log
+
+
+class CacheResolver(common.ResolverBase):
+    """
+    A resolver that serves records from a local, memory cache.
+
+    @ivar _reactor: A provider of L{interfaces.IReactorTime}.
+    """
+
+    cache = None
+
+    def __init__(self, cache=None, verbose=0, reactor=None):
+        common.ResolverBase.__init__(self)
+
+        self.cache = {}
+        self.verbose = verbose
+        self.cancel = {}
+        if reactor is None:
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+        if cache:
+            for query, (seconds, payload) in cache.items():
+                self.cacheResult(query, payload, seconds)
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+
+        now = self._reactor.seconds()
+        for k, (when, (ans, add, ns)) in self.cache.items():
+            diff = now - when
+            for rec in ans + add + ns:
+                if rec.ttl < diff:
+                    del self.cache[k]
+                    break
+
+    def __getstate__(self):
+        for c in self.cancel.values():
+            c.cancel()
+        self.cancel.clear()
+        return self.__dict__
+
+    def _lookup(self, name, cls, type, timeout):
+        now = self._reactor.seconds()
+        q = dns.Query(name, type, cls)
+        try:
+            when, (ans, auth, add) = self.cache[q]
+        except KeyError:
+            if self.verbose > 1:
+                log.msg("Cache miss for " + repr(name))
+            return defer.fail(failure.Failure(dns.DomainError(name)))
+        else:
+            if self.verbose:
+                log.msg("Cache hit for " + repr(name))
+            diff = now - when
+
+            try:
+                result = (
+                    [
+                        dns.RRHeader(
+                            r.name.name, r.type, r.cls, r.ttl - diff, r.payload
+                        )
+                        for r in ans
+                    ],
+                    [
+                        dns.RRHeader(
+                            r.name.name, r.type, r.cls, r.ttl - diff, r.payload
+                        )
+                        for r in auth
+                    ],
+                    [
+                        dns.RRHeader(
+                            r.name.name, r.type, r.cls, r.ttl - diff, r.payload
+                        )
+                        for r in add
+                    ],
+                )
+            except ValueError:
+                return defer.fail(failure.Failure(dns.DomainError(name)))
+            else:
+                return defer.succeed(result)
+
+    def lookupAllRecords(self, name, timeout=None):
+        return defer.fail(failure.Failure(dns.DomainError(name)))
+
+    def cacheResult(self, query, payload, cacheTime=None):
+        """
+        Cache a DNS entry.
+
+        @param query: a L{dns.Query} instance.
+
+        @param payload: a 3-tuple of lists of L{dns.RRHeader} records, the
+            matching result of the query (answers, authority and additional).
+
+        @param cacheTime: The time (seconds since epoch) at which the entry is
+            considered to have been added to the cache. If L{None} is given,
+            the current time is used.
+        """
+        if self.verbose > 1:
+            log.msg("Adding %r to cache" % query)
+
+        self.cache[query] = (cacheTime or self._reactor.seconds(), payload)
+
+        if query in self.cancel:
+            self.cancel[query].cancel()
+
+        s = list(payload[0]) + list(payload[1]) + list(payload[2])
+        if s:
+            m = s[0].ttl
+            for r in s:
+                m = min(m, r.ttl)
+        else:
+            m = 0
+
+        self.cancel[query] = self._reactor.callLater(m, self.clearEntry, query)
+
+    def clearEntry(self, query):
+        del self.cache[query]
+        del self.cancel[query]
diff --git a/contrib/python/Twisted/py3/twisted/names/client.py b/contrib/python/Twisted/py3/twisted/names/client.py
new file mode 100644
index 00000000000..4052936ab9b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/client.py
@@ -0,0 +1,734 @@
+# -*- test-case-name: twisted.names.test.test_names -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Asynchronous client DNS
+
+The functions exposed in this module can be used for asynchronous name
+resolution and dns queries.
+
+If you need to create a resolver with specific requirements, such as needing to
+do queries against a particular host, the L{createResolver} function will
+return an C{IResolver}.
+
+Future plans: Proper nameserver acquisition on Windows/MacOS,
+better caching, respect timeouts
+"""
+
+import errno
+import os
+import warnings
+
+from zope.interface import moduleProvides
+
+from twisted.internet import defer, error, interfaces, protocol
+from twisted.internet.abstract import isIPv6Address
+from twisted.names import cache, common, dns, hosts as hostsModule, resolve, root
+from twisted.python import failure, log
+
+# Twisted imports
+from twisted.python.compat import nativeString
+from twisted.python.filepath import FilePath
+from twisted.python.runtime import platform
+
+moduleProvides(interfaces.IResolver)
+
+
+class Resolver(common.ResolverBase):
+    """
+    @ivar _waiting: A C{dict} mapping tuple keys of query name/type/class to
+        Deferreds which will be called back with the result of those queries.
+        This is used to avoid issuing the same query more than once in
+        parallel.  This is more efficient on the network and helps avoid a
+        "birthday paradox" attack by keeping the number of outstanding requests
+        for a particular query fixed at one instead of allowing the attacker to
+        raise it to an arbitrary number.
+
+    @ivar _reactor: A provider of L{IReactorTCP}, L{IReactorUDP}, and
+        L{IReactorTime} which will be used to set up network resources and
+        track timeouts.
+    """
+
+    index = 0
+    timeout = None
+
+    factory = None
+    servers = None
+    dynServers = ()
+    pending = None
+    connections = None
+
+    resolv = None
+    _lastResolvTime = None
+    _resolvReadInterval = 60
+
+    def __init__(self, resolv=None, servers=None, timeout=(1, 3, 11, 45), reactor=None):
+        """
+        Construct a resolver which will query domain name servers listed in
+        the C{resolv.conf(5)}-format file given by C{resolv} as well as
+        those in the given C{servers} list.  Servers are queried in a
+        round-robin fashion.  If given, C{resolv} is periodically checked
+        for modification and re-parsed if it is noticed to have changed.
+
+        @type servers: C{list} of C{(str, int)} or L{None}
+        @param servers: If not None, interpreted as a list of (host, port)
+            pairs specifying addresses of domain name servers to attempt to use
+            for this lookup.  Host addresses should be in IPv4 dotted-quad
+            form.  If specified, overrides C{resolv}.
+
+        @type resolv: C{str}
+        @param resolv: Filename to read and parse as a resolver(5)
+            configuration file.
+
+        @type timeout: Sequence of C{int}
+        @param timeout: Default number of seconds after which to reissue the
+            query.  When the last timeout expires, the query is considered
+            failed.
+
+        @param reactor: A provider of L{IReactorTime}, L{IReactorUDP}, and
+            L{IReactorTCP} which will be used to establish connections, listen
+            for DNS datagrams, and enforce timeouts.  If not provided, the
+            global reactor will be used.
+
+        @raise ValueError: Raised if no nameserver addresses can be found.
+        """
+        common.ResolverBase.__init__(self)
+
+        if reactor is None:
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+        self.timeout = timeout
+
+        if servers is None:
+            self.servers = []
+        else:
+            self.servers = servers
+
+        self.resolv = resolv
+
+        if not len(self.servers) and not resolv:
+            raise ValueError("No nameservers specified")
+
+        self.factory = DNSClientFactory(self, timeout)
+        self.factory.noisy = 0  # Be quiet by default
+
+        self.connections = []
+        self.pending = []
+
+        self._waiting = {}
+
+        self.maybeParseConfig()
+
+    def __getstate__(self):
+        d = self.__dict__.copy()
+        d["connections"] = []
+        d["_parseCall"] = None
+        return d
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        self.maybeParseConfig()
+
+    def _openFile(self, path):
+        """
+        Wrapper used for opening files in the class, exists primarily for unit
+        testing purposes.
+        """
+        return FilePath(path).open()
+
+    def maybeParseConfig(self):
+        if self.resolv is None:
+            # Don't try to parse it, don't set up a call loop
+            return
+
+        try:
+            resolvConf = self._openFile(self.resolv)
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                # Missing resolv.conf is treated the same as an empty resolv.conf
+                self.parseConfig(())
+            else:
+                raise
+        else:
+            with resolvConf:
+                mtime = os.fstat(resolvConf.fileno()).st_mtime
+                if mtime != self._lastResolvTime:
+                    log.msg(f"{self.resolv} changed, reparsing")
+                    self._lastResolvTime = mtime
+                    self.parseConfig(resolvConf)
+
+        # Check again in a little while
+        self._parseCall = self._reactor.callLater(
+            self._resolvReadInterval, self.maybeParseConfig
+        )
+
+    def parseConfig(self, resolvConf):
+        servers = []
+        for L in resolvConf:
+            L = L.strip()
+            if L.startswith(b"nameserver"):
+                resolver = (nativeString(L.split()[1]), dns.PORT)
+                servers.append(resolver)
+                log.msg(f"Resolver added {resolver!r} to server list")
+            elif L.startswith(b"domain"):
+                try:
+                    self.domain = L.split()[1]
+                except IndexError:
+                    self.domain = b""
+                self.search = None
+            elif L.startswith(b"search"):
+                self.search = L.split()[1:]
+                self.domain = None
+        if not servers:
+            servers.append(("127.0.0.1", dns.PORT))
+        self.dynServers = servers
+
+    def pickServer(self):
+        """
+        Return the address of a nameserver.
+
+        TODO: Weight servers for response time so faster ones can be
+        preferred.
+        """
+        if not self.servers and not self.dynServers:
+            return None
+        serverL = len(self.servers)
+        dynL = len(self.dynServers)
+
+        self.index += 1
+        self.index %= serverL + dynL
+        if self.index < serverL:
+            return self.servers[self.index]
+        else:
+            return self.dynServers[self.index - serverL]
+
+    def _connectedProtocol(self, interface=""):
+        """
+        Return a new L{DNSDatagramProtocol} bound to a randomly selected port
+        number.
+        """
+        failures = 0
+        proto = dns.DNSDatagramProtocol(self, reactor=self._reactor)
+
+        while True:
+            try:
+                self._reactor.listenUDP(dns.randomSource(), proto, interface=interface)
+            except error.CannotListenError as e:
+                failures += 1
+
+                if (
+                    hasattr(e.socketError, "errno")
+                    and e.socketError.errno == errno.EMFILE
+                ):
+                    # We've run out of file descriptors. Stop trying.
+                    raise
+
+                if failures >= 1000:
+                    # We've tried a thousand times and haven't found a port.
+                    # This is almost impossible, and likely means something
+                    # else weird is going on. Raise, as to not infinite loop.
+                    raise
+            else:
+                return proto
+
+    def connectionMade(self, protocol):
+        """
+        Called by associated L{dns.DNSProtocol} instances when they connect.
+        """
+        self.connections.append(protocol)
+        for d, q, t in self.pending:
+            self.queryTCP(q, t).chainDeferred(d)
+        del self.pending[:]
+
+    def connectionLost(self, protocol):
+        """
+        Called by associated L{dns.DNSProtocol} instances when they disconnect.
+        """
+        if protocol in self.connections:
+            self.connections.remove(protocol)
+
+    def messageReceived(self, message, protocol, address=None):
+        log.msg("Unexpected message (%d) received from %r" % (message.id, address))
+
+    def _query(self, *args):
+        """
+        Get a new L{DNSDatagramProtocol} instance from L{_connectedProtocol},
+        issue a query to it using C{*args}, and arrange for it to be
+        disconnected from its transport after the query completes.
+
+        @param args: Positional arguments to be passed to
+            L{DNSDatagramProtocol.query}.
+
+        @return: A L{Deferred} which will be called back with the result of the
+            query.
+        """
+        if isIPv6Address(args[0][0]):
+            protocol = self._connectedProtocol(interface="::")
+        else:
+            protocol = self._connectedProtocol()
+        d = protocol.query(*args)
+
+        def cbQueried(result):
+            protocol.transport.stopListening()
+            return result
+
+        d.addBoth(cbQueried)
+        return d
+
+    def queryUDP(self, queries, timeout=None):
+        """
+        Make a number of DNS queries via UDP.
+
+        @type queries: A C{list} of C{dns.Query} instances
+        @param queries: The queries to make.
+
+        @type timeout: Sequence of C{int}
+        @param timeout: Number of seconds after which to reissue the query.
+        When the last timeout expires, the query is considered failed.
+
+        @rtype: C{Deferred}
+        @raise C{twisted.internet.defer.TimeoutError}: When the query times
+        out.
+        """
+        if timeout is None:
+            timeout = self.timeout
+
+        addresses = self.servers + list(self.dynServers)
+        if not addresses:
+            return defer.fail(IOError("No domain name servers available"))
+
+        # Make sure we go through servers in the list in the order they were
+        # specified.
+        addresses.reverse()
+
+        used = addresses.pop()
+        d = self._query(used, queries, timeout[0])
+        d.addErrback(self._reissue, addresses, [used], queries, timeout)
+        return d
+
+    def _reissue(self, reason, addressesLeft, addressesUsed, query, timeout):
+        reason.trap(dns.DNSQueryTimeoutError)
+
+        # If there are no servers left to be tried, adjust the timeout
+        # to the next longest timeout period and move all the
+        # "used" addresses back to the list of addresses to try.
+        if not addressesLeft:
+            addressesLeft = addressesUsed
+            addressesLeft.reverse()
+            addressesUsed = []
+            timeout = timeout[1:]
+
+        # If all timeout values have been used this query has failed.  Tell the
+        # protocol we're giving up on it and return a terminal timeout failure
+        # to our caller.
+        if not timeout:
+            return failure.Failure(defer.TimeoutError(query))
+
+        # Get an address to try.  Take it out of the list of addresses
+        # to try and put it ino the list of already tried addresses.
+        address = addressesLeft.pop()
+        addressesUsed.append(address)
+
+        # Issue a query to a server.  Use the current timeout.  Add this
+        # function as a timeout errback in case another retry is required.
+        d = self._query(address, query, timeout[0], reason.value.id)
+        d.addErrback(self._reissue, addressesLeft, addressesUsed, query, timeout)
+        return d
+
+    def queryTCP(self, queries, timeout=10):
+        """
+        Make a number of DNS queries via TCP.
+
+        @type queries: Any non-zero number of C{dns.Query} instances
+        @param queries: The queries to make.
+
+        @type timeout: C{int}
+        @param timeout: The number of seconds after which to fail.
+
+        @rtype: C{Deferred}
+        """
+        if not len(self.connections):
+            address = self.pickServer()
+            if address is None:
+                return defer.fail(IOError("No domain name servers available"))
+            host, port = address
+            self._reactor.connectTCP(host, port, self.factory)
+            self.pending.append((defer.Deferred(), queries, timeout))
+            return self.pending[-1][0]
+        else:
+            return self.connections[0].query(queries, timeout)
+
+    def filterAnswers(self, message):
+        """
+        Extract results from the given message.
+
+        If the message was truncated, re-attempt the query over TCP and return
+        a Deferred which will fire with the results of that query.
+
+        If the message's result code is not C{twisted.names.dns.OK}, return a
+        Failure indicating the type of error which occurred.
+
+        Otherwise, return a three-tuple of lists containing the results from
+        the answers section, the authority section, and the additional section.
+        """
+        if message.trunc:
+            return self.queryTCP(message.queries).addCallback(self.filterAnswers)
+        if message.rCode != dns.OK:
+            return failure.Failure(self.exceptionForCode(message.rCode)(message))
+        return (message.answers, message.authority, message.additional)
+
+    def _lookup(self, name, cls, type, timeout):
+        """
+        Build a L{dns.Query} for the given parameters and dispatch it via UDP.
+
+        If this query is already outstanding, it will not be re-issued.
+        Instead, when the outstanding query receives a response, that response
+        will be re-used for this query as well.
+
+        @type name: C{str}
+        @type type: C{int}
+        @type cls: C{int}
+
+        @return: A L{Deferred} which fires with a three-tuple giving the
+            answer, authority, and additional sections of the response or with
+            a L{Failure} if the response code is anything other than C{dns.OK}.
+        """
+        key = (name, type, cls)
+        waiting = self._waiting.get(key)
+        if waiting is None:
+            self._waiting[key] = []
+            d = self.queryUDP([dns.Query(name, type, cls)], timeout)
+
+            def cbResult(result):
+                for d in self._waiting.pop(key):
+                    d.callback(result)
+                return result
+
+            d.addCallback(self.filterAnswers)
+            d.addBoth(cbResult)
+        else:
+            d = defer.Deferred()
+            waiting.append(d)
+        return d
+
+    # This one doesn't ever belong on UDP
+    def lookupZone(self, name, timeout=10):
+        address = self.pickServer()
+        if address is None:
+            return defer.fail(IOError("No domain name servers available"))
+        host, port = address
+        d = defer.Deferred()
+        controller = AXFRController(name, d)
+        factory = DNSClientFactory(controller, timeout)
+        factory.noisy = False  # stfu
+
+        connector = self._reactor.connectTCP(host, port, factory)
+        controller.timeoutCall = self._reactor.callLater(
+            timeout or 10, self._timeoutZone, d, controller, connector, timeout or 10
+        )
+
+        def eliminateTimeout(failure):
+            controller.timeoutCall.cancel()
+            controller.timeoutCall = None
+            return failure
+
+        return d.addCallbacks(
+            self._cbLookupZone, eliminateTimeout, callbackArgs=(connector,)
+        )
+
+    def _timeoutZone(self, d, controller, connector, seconds):
+        connector.disconnect()
+        controller.timeoutCall = None
+        controller.deferred = None
+        d.errback(
+            error.TimeoutError("Zone lookup timed out after %d seconds" % (seconds,))
+        )
+
+    def _cbLookupZone(self, result, connector):
+        connector.disconnect()
+        return (result, [], [])
+
+
+class AXFRController:
+    timeoutCall = None
+
+    def __init__(self, name, deferred):
+        self.name = name
+        self.deferred = deferred
+        self.soa = None
+        self.records = []
+        self.pending = [(deferred,)]
+
+    def connectionMade(self, protocol):
+        # dig saids recursion-desired to 0, so I will too
+        message = dns.Message(protocol.pickID(), recDes=0)
+        message.queries = [dns.Query(self.name, dns.AXFR, dns.IN)]
+        protocol.writeMessage(message)
+
+    def connectionLost(self, protocol):
+        # XXX Do something here - see #3428
+        pass
+
+    def messageReceived(self, message, protocol):
+        # Caveat: We have to handle two cases: All records are in 1
+        # message, or all records are in N messages.
+
+        # According to http://cr.yp.to/djbdns/axfr-notes.html,
+        # 'authority' and 'additional' are always empty, and only
+        # 'answers' is present.
+        self.records.extend(message.answers)
+        if not self.records:
+            return
+        if not self.soa:
+            if self.records[0].type == dns.SOA:
+                # print "first SOA!"
+                self.soa = self.records[0]
+        if len(self.records) > 1 and self.records[-1].type == dns.SOA:
+            # print "It's the second SOA! We're done."
+            if self.timeoutCall is not None:
+                self.timeoutCall.cancel()
+                self.timeoutCall = None
+            if self.deferred is not None:
+                self.deferred.callback(self.records)
+                self.deferred = None
+
+
+from twisted.internet.base import ThreadedResolver as _ThreadedResolverImpl
+
+
+class ThreadedResolver(_ThreadedResolverImpl):
+    def __init__(self, reactor=None):
+        if reactor is None:
+            from twisted.internet import reactor
+        _ThreadedResolverImpl.__init__(self, reactor)
+        warnings.warn(
+            "twisted.names.client.ThreadedResolver is deprecated since "
+            "Twisted 9.0, use twisted.internet.base.ThreadedResolver "
+            "instead.",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+
+
+class DNSClientFactory(protocol.ClientFactory):
+    def __init__(self, controller, timeout=10):
+        self.controller = controller
+        self.timeout = timeout
+
+    def clientConnectionLost(self, connector, reason):
+        pass
+
+    def clientConnectionFailed(self, connector, reason):
+        """
+        Fail all pending TCP DNS queries if the TCP connection attempt
+        fails.
+
+        @see: L{twisted.internet.protocol.ClientFactory}
+
+        @param connector: Not used.
+        @type connector: L{twisted.internet.interfaces.IConnector}
+
+        @param reason: A C{Failure} containing information about the
+            cause of the connection failure. This will be passed as the
+            argument to C{errback} on every pending TCP query
+            C{deferred}.
+        @type reason: L{twisted.python.failure.Failure}
+        """
+        # Copy the current pending deferreds then reset the master
+        # pending list. This prevents triggering new deferreds which
+        # may be added by callback or errback functions on the current
+        # deferreds.
+        pending = self.controller.pending[:]
+        del self.controller.pending[:]
+        for pendingState in pending:
+            d = pendingState[0]
+            d.errback(reason)
+
+    def buildProtocol(self, addr):
+        p = dns.DNSProtocol(self.controller)
+        p.factory = self
+        return p
+
+
+def createResolver(servers=None, resolvconf=None, hosts=None):
+    r"""
+    Create and return a Resolver.
+
+    @type servers: C{list} of C{(str, int)} or L{None}
+
+    @param servers: If not L{None}, interpreted as a list of domain name servers
+    to attempt to use. Each server is a tuple of address in C{str} dotted-quad
+    form and C{int} port number.
+
+    @type resolvconf: C{str} or L{None}
+    @param resolvconf: If not L{None}, on posix systems will be interpreted as
+    an alternate resolv.conf to use. Will do nothing on windows systems. If
+    L{None}, /etc/resolv.conf will be used.
+
+    @type hosts: C{str} or L{None}
+    @param hosts: If not L{None}, an alternate hosts file to use. If L{None}
+    on posix systems, /etc/hosts will be used. On windows, C:\windows\hosts
+    will be used.
+
+    @rtype: C{IResolver}
+    """
+    if platform.getType() == "posix":
+        if resolvconf is None:
+            resolvconf = b"/etc/resolv.conf"
+        if hosts is None:
+            hosts = b"/etc/hosts"
+        theResolver = Resolver(resolvconf, servers)
+        hostResolver = hostsModule.Resolver(hosts)
+    else:
+        if hosts is None:
+            hosts = r"c:\windows\hosts"
+        from twisted.internet import reactor
+
+        bootstrap = _ThreadedResolverImpl(reactor)
+        hostResolver = hostsModule.Resolver(hosts)
+        theResolver = root.bootstrap(bootstrap, resolverFactory=Resolver)
+
+    L = [hostResolver, cache.CacheResolver(), theResolver]
+    return resolve.ResolverChain(L)
+
+
+theResolver = None
+
+
+def getResolver():
+    """
+    Get a Resolver instance.
+
+    Create twisted.names.client.theResolver if it is L{None}, and then return
+    that value.
+
+    @rtype: C{IResolver}
+    """
+    global theResolver
+    if theResolver is None:
+        try:
+            theResolver = createResolver()
+        except ValueError:
+            theResolver = createResolver(servers=[("127.0.0.1", 53)])
+    return theResolver
+
+
+def getHostByName(name, timeout=None, effort=10):
+    """
+    Resolve a name to a valid ipv4 or ipv6 address.
+
+    Will errback with C{DNSQueryTimeoutError} on a timeout, C{DomainError} or
+    C{AuthoritativeDomainError} (or subclasses) on other errors.
+
+    @type name: C{str}
+    @param name: DNS name to resolve.
+
+    @type timeout: Sequence of C{int}
+    @param timeout: Number of seconds after which to reissue the query.
+    When the last timeout expires, the query is considered failed.
+
+    @type effort: C{int}
+    @param effort: How many times CNAME and NS records to follow while
+    resolving this name.
+
+    @rtype: C{Deferred}
+    """
+    return getResolver().getHostByName(name, timeout, effort)
+
+
+def query(query, timeout=None):
+    return getResolver().query(query, timeout)
+
+
+def lookupAddress(name, timeout=None):
+    return getResolver().lookupAddress(name, timeout)
+
+
+def lookupIPV6Address(name, timeout=None):
+    return getResolver().lookupIPV6Address(name, timeout)
+
+
+def lookupAddress6(name, timeout=None):
+    return getResolver().lookupAddress6(name, timeout)
+
+
+def lookupMailExchange(name, timeout=None):
+    return getResolver().lookupMailExchange(name, timeout)
+
+
+def lookupNameservers(name, timeout=None):
+    return getResolver().lookupNameservers(name, timeout)
+
+
+def lookupCanonicalName(name, timeout=None):
+    return getResolver().lookupCanonicalName(name, timeout)
+
+
+def lookupMailBox(name, timeout=None):
+    return getResolver().lookupMailBox(name, timeout)
+
+
+def lookupMailGroup(name, timeout=None):
+    return getResolver().lookupMailGroup(name, timeout)
+
+
+def lookupMailRename(name, timeout=None):
+    return getResolver().lookupMailRename(name, timeout)
+
+
+def lookupPointer(name, timeout=None):
+    return getResolver().lookupPointer(name, timeout)
+
+
+def lookupAuthority(name, timeout=None):
+    return getResolver().lookupAuthority(name, timeout)
+
+
+def lookupNull(name, timeout=None):
+    return getResolver().lookupNull(name, timeout)
+
+
+def lookupWellKnownServices(name, timeout=None):
+    return getResolver().lookupWellKnownServices(name, timeout)
+
+
+def lookupService(name, timeout=None):
+    return getResolver().lookupService(name, timeout)
+
+
+def lookupHostInfo(name, timeout=None):
+    return getResolver().lookupHostInfo(name, timeout)
+
+
+def lookupMailboxInfo(name, timeout=None):
+    return getResolver().lookupMailboxInfo(name, timeout)
+
+
+def lookupText(name, timeout=None):
+    return getResolver().lookupText(name, timeout)
+
+
+def lookupSenderPolicy(name, timeout=None):
+    return getResolver().lookupSenderPolicy(name, timeout)
+
+
+def lookupResponsibility(name, timeout=None):
+    return getResolver().lookupResponsibility(name, timeout)
+
+
+def lookupAFSDatabase(name, timeout=None):
+    return getResolver().lookupAFSDatabase(name, timeout)
+
+
+def lookupZone(name, timeout=None):
+    return getResolver().lookupZone(name, timeout)
+
+
+def lookupAllRecords(name, timeout=None):
+    return getResolver().lookupAllRecords(name, timeout)
+
+
+def lookupNamingAuthorityPointer(name, timeout=None):
+    return getResolver().lookupNamingAuthorityPointer(name, timeout)
diff --git a/contrib/python/Twisted/py3/twisted/names/common.py b/contrib/python/Twisted/py3/twisted/names/common.py
new file mode 100644
index 00000000000..ee64b451f7b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/common.py
@@ -0,0 +1,263 @@
+# -*- test-case-name: twisted.names.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Base functionality useful to various parts of Twisted Names.
+"""
+
+
+import socket
+
+from zope.interface import implementer
+
+from twisted.internet import defer, error, interfaces
+from twisted.logger import Logger
+from twisted.names import dns
+from twisted.names.error import (
+    DNSFormatError,
+    DNSNameError,
+    DNSNotImplementedError,
+    DNSQueryRefusedError,
+    DNSServerError,
+    DNSUnknownError,
+)
+
+# Helpers for indexing the three-tuples that get thrown around by this code a
+# lot.
+_ANS, _AUTH, _ADD = range(3)
+
+EMPTY_RESULT = (), (), ()
+
+
+@implementer(interfaces.IResolver)
+class ResolverBase:
+    """
+    L{ResolverBase} is a base class for implementations of
+    L{interfaces.IResolver} which deals with a lot
+    of the boilerplate of implementing all of the lookup methods.
+
+    @cvar _errormap: A C{dict} mapping DNS protocol failure response codes
+        to exception classes which will be used to represent those failures.
+    """
+
+    _log = Logger()
+    _errormap = {
+        dns.EFORMAT: DNSFormatError,
+        dns.ESERVER: DNSServerError,
+        dns.ENAME: DNSNameError,
+        dns.ENOTIMP: DNSNotImplementedError,
+        dns.EREFUSED: DNSQueryRefusedError,
+    }
+
+    typeToMethod = None
+
+    def __init__(self):
+        self.typeToMethod = {}
+        for k, v in typeToMethod.items():
+            self.typeToMethod[k] = getattr(self, v)
+
+    def exceptionForCode(self, responseCode):
+        """
+        Convert a response code (one of the possible values of
+        L{dns.Message.rCode} to an exception instance representing it.
+
+        @since: 10.0
+        """
+        return self._errormap.get(responseCode, DNSUnknownError)
+
+    def query(self, query, timeout=None):
+        try:
+            method = self.typeToMethod[query.type]
+        except KeyError:
+            self._log.debug(
+                "Query of unknown type {query.type} for {query.name.name!r}",
+                query=query,
+            )
+            return defer.maybeDeferred(
+                self._lookup, query.name.name, dns.IN, query.type, timeout
+            )
+        else:
+            return defer.maybeDeferred(method, query.name.name, timeout)
+
+    def _lookup(self, name, cls, type, timeout):
+        return defer.fail(NotImplementedError("ResolverBase._lookup"))
+
+    def lookupAddress(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.A, timeout)
+
+    def lookupIPV6Address(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.AAAA, timeout)
+
+    def lookupAddress6(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.A6, timeout)
+
+    def lookupMailExchange(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.MX, timeout)
+
+    def lookupNameservers(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.NS, timeout)
+
+    def lookupCanonicalName(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.CNAME, timeout)
+
+    def lookupMailBox(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.MB, timeout)
+
+    def lookupMailGroup(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.MG, timeout)
+
+    def lookupMailRename(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.MR, timeout)
+
+    def lookupPointer(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.PTR, timeout)
+
+    def lookupAuthority(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.SOA, timeout)
+
+    def lookupNull(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.NULL, timeout)
+
+    def lookupWellKnownServices(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.WKS, timeout)
+
+    def lookupService(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.SRV, timeout)
+
+    def lookupHostInfo(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.HINFO, timeout)
+
+    def lookupMailboxInfo(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.MINFO, timeout)
+
+    def lookupText(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.TXT, timeout)
+
+    def lookupSenderPolicy(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.SPF, timeout)
+
+    def lookupResponsibility(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.RP, timeout)
+
+    def lookupAFSDatabase(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.AFSDB, timeout)
+
+    def lookupZone(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.AXFR, timeout)
+
+    def lookupNamingAuthorityPointer(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.NAPTR, timeout)
+
+    def lookupAllRecords(self, name, timeout=None):
+        return self._lookup(dns.domainString(name), dns.IN, dns.ALL_RECORDS, timeout)
+
+    # IResolverSimple
+    def getHostByName(self, name, timeout=None, effort=10):
+        name = dns.domainString(name)
+        # XXX - respect timeout
+        # XXX - this should do A and AAAA lookups, not ANY (see RFC 8482).
+        # https://twistedmatrix.com/trac/ticket/9691
+        d = self.lookupAllRecords(name, timeout)
+        d.addCallback(self._cbRecords, name, effort)
+        return d
+
+    def _cbRecords(self, records, name, effort):
+        (ans, auth, add) = records
+        result = extractRecord(self, dns.Name(name), ans + auth + add, effort)
+        if not result:
+            raise error.DNSLookupError(name)
+        return result
+
+
+def extractRecord(resolver, name, answers, level=10):
+    """
+    Resolve a name to an IP address, following I{CNAME} records and I{NS}
+    referrals recursively.
+
+    This is an implementation detail of L{ResolverBase.getHostByName}.
+
+    @param resolver: The resolver to use for the next query (unless handling
+    an I{NS} referral).
+    @type resolver: L{IResolver}
+
+    @param name: The name being looked up.
+    @type name: L{dns.Name}
+
+    @param answers: All of the records returned by the previous query (answers,
+    authority, and additional concatenated).
+    @type answers: L{list} of L{dns.RRHeader}
+
+    @param level: Remaining recursion budget. This is decremented at each
+    recursion. The query returns L{None} when it reaches 0.
+    @type level: L{int}
+
+    @returns: The first IPv4 or IPv6 address (as a dotted quad or colon
+    quibbles), or L{None} when no result is found.
+    @rtype: native L{str} or L{None}
+    """
+    if not level:
+        return None
+    # FIXME: twisted.python.compat monkeypatches this if missing, so this
+    # condition is always true. https://twistedmatrix.com/trac/ticket/9753
+    if hasattr(socket, "inet_ntop"):
+        for r in answers:
+            if r.name == name and r.type == dns.A6:
+                return socket.inet_ntop(socket.AF_INET6, r.payload.address)
+        for r in answers:
+            if r.name == name and r.type == dns.AAAA:
+                return socket.inet_ntop(socket.AF_INET6, r.payload.address)
+    for r in answers:
+        if r.name == name and r.type == dns.A:
+            return socket.inet_ntop(socket.AF_INET, r.payload.address)
+    for r in answers:
+        if r.name == name and r.type == dns.CNAME:
+            result = extractRecord(resolver, r.payload.name, answers, level - 1)
+            if not result:
+                return resolver.getHostByName(r.payload.name.name, effort=level - 1)
+            return result
+    # No answers, but maybe there's a hint at who we should be asking about
+    # this
+    for r in answers:
+        if r.type != dns.NS:
+            continue
+        from twisted.names import client
+
+        nsResolver = client.Resolver(
+            servers=[
+                (r.payload.name.name.decode("ascii"), dns.PORT),
+            ]
+        )
+
+        def queryAgain(records):
+            (ans, auth, add) = records
+            return extractRecord(nsResolver, name, ans + auth + add, level - 1)
+
+        return nsResolver.lookupAddress(name.name).addCallback(queryAgain)
+
+
+typeToMethod = {
+    dns.A: "lookupAddress",
+    dns.AAAA: "lookupIPV6Address",
+    dns.A6: "lookupAddress6",
+    dns.NS: "lookupNameservers",
+    dns.CNAME: "lookupCanonicalName",
+    dns.SOA: "lookupAuthority",
+    dns.MB: "lookupMailBox",
+    dns.MG: "lookupMailGroup",
+    dns.MR: "lookupMailRename",
+    dns.NULL: "lookupNull",
+    dns.WKS: "lookupWellKnownServices",
+    dns.PTR: "lookupPointer",
+    dns.HINFO: "lookupHostInfo",
+    dns.MINFO: "lookupMailboxInfo",
+    dns.MX: "lookupMailExchange",
+    dns.TXT: "lookupText",
+    dns.SPF: "lookupSenderPolicy",
+    dns.RP: "lookupResponsibility",
+    dns.AFSDB: "lookupAFSDatabase",
+    dns.SRV: "lookupService",
+    dns.NAPTR: "lookupNamingAuthorityPointer",
+    dns.AXFR: "lookupZone",
+    dns.ALL_RECORDS: "lookupAllRecords",
+}
diff --git a/contrib/python/Twisted/py3/twisted/names/dns.py b/contrib/python/Twisted/py3/twisted/names/dns.py
new file mode 100644
index 00000000000..c7644ef50d6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/dns.py
@@ -0,0 +1,3390 @@
+# -*- test-case-name: twisted.names.test.test_dns -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+DNS protocol implementation.
+
+Future Plans:
+    - Get rid of some toplevels, maybe.
+"""
+from __future__ import annotations
+
+# System imports
+import inspect
+import random
+import socket
+import struct
+from io import BytesIO
+from itertools import chain
+from typing import Optional, Sequence, SupportsInt, Union, overload
+
+from zope.interface import Attribute, Interface, implementer
+
+# Twisted imports
+from twisted.internet import defer, protocol
+from twisted.internet.error import CannotListenError
+from twisted.python import failure, log, randbytes, util as tputil
+from twisted.python.compat import cmp, comparable, nativeString
+
+__all__ = [
+    "IEncodable",
+    "IRecord",
+    "IEncodableRecord",
+    "A",
+    "A6",
+    "AAAA",
+    "AFSDB",
+    "CNAME",
+    "DNAME",
+    "HINFO",
+    "MAILA",
+    "MAILB",
+    "MB",
+    "MD",
+    "MF",
+    "MG",
+    "MINFO",
+    "MR",
+    "MX",
+    "NAPTR",
+    "NS",
+    "NULL",
+    "OPT",
+    "PTR",
+    "RP",
+    "SOA",
+    "SPF",
+    "SRV",
+    "TXT",
+    "SSHFP",
+    "TSIG",
+    "WKS",
+    "ANY",
+    "CH",
+    "CS",
+    "HS",
+    "IN",
+    "ALL_RECORDS",
+    "AXFR",
+    "IXFR",
+    "EFORMAT",
+    "ENAME",
+    "ENOTIMP",
+    "EREFUSED",
+    "ESERVER",
+    "EBADVERSION",
+    "EBADSIG",
+    "EBADKEY",
+    "EBADTIME",
+    "Record_A",
+    "Record_A6",
+    "Record_AAAA",
+    "Record_AFSDB",
+    "Record_CNAME",
+    "Record_DNAME",
+    "Record_HINFO",
+    "Record_MB",
+    "Record_MD",
+    "Record_MF",
+    "Record_MG",
+    "Record_MINFO",
+    "Record_MR",
+    "Record_MX",
+    "Record_NAPTR",
+    "Record_NS",
+    "Record_NULL",
+    "Record_PTR",
+    "Record_RP",
+    "Record_SOA",
+    "Record_SPF",
+    "Record_SRV",
+    "Record_SSHFP",
+    "Record_TSIG",
+    "Record_TXT",
+    "Record_WKS",
+    "UnknownRecord",
+    "QUERY_CLASSES",
+    "QUERY_TYPES",
+    "REV_CLASSES",
+    "REV_TYPES",
+    "EXT_QUERIES",
+    "Charstr",
+    "Message",
+    "Name",
+    "Query",
+    "RRHeader",
+    "SimpleRecord",
+    "DNSDatagramProtocol",
+    "DNSMixin",
+    "DNSProtocol",
+    "OK",
+    "OP_INVERSE",
+    "OP_NOTIFY",
+    "OP_QUERY",
+    "OP_STATUS",
+    "OP_UPDATE",
+    "PORT",
+    "AuthoritativeDomainError",
+    "DNSQueryTimeoutError",
+    "DomainError",
+]
+
+
+AF_INET6 = socket.AF_INET6
+
+
+def _ord2bytes(ordinal):
+    """
+    Construct a bytes object representing a single byte with the given
+    ordinal value.
+
+    @type ordinal: L{int}
+    @rtype: L{bytes}
+    """
+    return bytes([ordinal])
+
+
+def _nicebytes(bytes):
+    """
+    Represent a mostly textful bytes object in a way suitable for
+    presentation to an end user.
+
+    @param bytes: The bytes to represent.
+    @rtype: L{str}
+    """
+    return repr(bytes)[1:]
+
+
+def _nicebyteslist(list):
+    """
+    Represent a list of mostly textful bytes objects in a way suitable for
+    presentation to an end user.
+
+    @param list: The list of bytes to represent.
+    @rtype: L{str}
+    """
+    return "[{}]".format(", ".join([_nicebytes(b) for b in list]))
+
+
+def randomSource():
+    """
+    Wrapper around L{twisted.python.randbytes.RandomFactory.secureRandom} to
+    return 2 random bytes.
+
+    @rtype: L{bytes}
+    """
+    return struct.unpack("H", randbytes.secureRandom(2, fallback=True))[0]
+
+
+PORT = 53
+
+(
+    A,
+    NS,
+    MD,
+    MF,
+    CNAME,
+    SOA,
+    MB,
+    MG,
+    MR,
+    NULL,
+    WKS,
+    PTR,
+    HINFO,
+    MINFO,
+    MX,
+    TXT,
+    RP,
+    AFSDB,
+) = range(1, 19)
+AAAA = 28
+SRV = 33
+NAPTR = 35
+A6 = 38
+DNAME = 39
+OPT = 41
+SSHFP = 44
+SPF = 99
+
+# These record types do not exist in zones, but are transferred in
+# messages the same way normal RRs are.
+TKEY = 249
+TSIG = 250
+
+QUERY_TYPES = {
+    A: "A",
+    NS: "NS",
+    MD: "MD",
+    MF: "MF",
+    CNAME: "CNAME",
+    SOA: "SOA",
+    MB: "MB",
+    MG: "MG",
+    MR: "MR",
+    NULL: "NULL",
+    WKS: "WKS",
+    PTR: "PTR",
+    HINFO: "HINFO",
+    MINFO: "MINFO",
+    MX: "MX",
+    TXT: "TXT",
+    RP: "RP",
+    AFSDB: "AFSDB",
+    # 19 through 27?  Eh, I'll get to 'em.
+    AAAA: "AAAA",
+    SRV: "SRV",
+    NAPTR: "NAPTR",
+    A6: "A6",
+    DNAME: "DNAME",
+    OPT: "OPT",
+    SSHFP: "SSHFP",
+    SPF: "SPF",
+    TKEY: "TKEY",
+    TSIG: "TSIG",
+}
+
+IXFR, AXFR, MAILB, MAILA, ALL_RECORDS = range(251, 256)
+
+# "Extended" queries (Hey, half of these are deprecated, good job)
+EXT_QUERIES = {
+    IXFR: "IXFR",
+    AXFR: "AXFR",
+    MAILB: "MAILB",
+    MAILA: "MAILA",
+    ALL_RECORDS: "ALL_RECORDS",
+}
+
+REV_TYPES = {v: k for (k, v) in chain(QUERY_TYPES.items(), EXT_QUERIES.items())}
+
+IN, CS, CH, HS = range(1, 5)
+ANY = 255
+
+QUERY_CLASSES = {IN: "IN", CS: "CS", CH: "CH", HS: "HS", ANY: "ANY"}
+REV_CLASSES = {v: k for (k, v) in QUERY_CLASSES.items()}
+
+
+# Opcodes
+OP_QUERY, OP_INVERSE, OP_STATUS = range(3)
+OP_NOTIFY = 4  # RFC 1996
+OP_UPDATE = 5  # RFC 2136
+
+
+# Response Codes
+OK, EFORMAT, ESERVER, ENAME, ENOTIMP, EREFUSED = range(6)
+# https://tools.ietf.org/html/rfc6891#section-9
+EBADVERSION = 16
+# RFC 2845
+EBADSIG, EBADKEY, EBADTIME = range(16, 19)
+
+
+class IRecord(Interface):
+    """
+    A single entry in a zone of authority.
+    """
+
+    TYPE = Attribute("An indicator of what kind of record this is.")
+
+
+# Backwards compatibility aliases - these should be deprecated or something I
+# suppose. -exarkun
+from twisted.names.error import (
+    AuthoritativeDomainError,
+    DNSQueryTimeoutError,
+    DomainError,
+)
+
+
+def _nameToLabels(name):
+    """
+    Split a domain name into its constituent labels.
+
+    @type name: L{bytes}
+    @param name: A fully qualified domain name (with or without a
+        trailing dot).
+
+    @return: A L{list} of labels ending with an empty label
+        representing the DNS root zone.
+    @rtype: L{list} of L{bytes}
+    """
+    if name in (b"", b"."):
+        return [b""]
+    labels = name.split(b".")
+    if labels[-1] != b"":
+        labels.append(b"")
+    return labels
+
+
+def domainString(domain: str | bytes) -> bytes:
+    """
+    Coerce a domain name string to bytes.
+
+    L{twisted.names} represents domain names as L{bytes}, but many interfaces
+    accept L{bytes} or a text string (L{unicode} on Python 2, L{str} on Python
+    3). This function coerces text strings using IDNA encoding --- see
+    L{encodings.idna}.
+
+    Note that DNS is I{case insensitive} but I{case preserving}. This function
+    doesn't normalize case, so you'll still need to do that whenever comparing
+    the strings it returns.
+
+    @param domain: A domain name.  If passed as a text string it will be
+        C{idna} encoded.
+    @type domain: L{bytes} or L{str}
+
+    @returns: L{bytes} suitable for network transmission.
+    @rtype: L{bytes}
+
+    @since: Twisted 20.3.0
+    """
+    if isinstance(domain, str):
+        domain = domain.encode("idna")
+    if not isinstance(domain, bytes):
+        raise TypeError(
+            "Expected {} or {} but found {!r} of type {}".format(
+                bytes.__name__, str.__name__, domain, type(domain)
+            )
+        )
+    return domain
+
+
+def _isSubdomainOf(descendantName, ancestorName):
+    """
+    Test whether C{descendantName} is equal to or is a I{subdomain} of
+    C{ancestorName}.
+
+    The names are compared case-insensitively.
+
+    The names are treated as byte strings containing one or more
+    DNS labels separated by B{.}.
+
+    C{descendantName} is considered equal if its sequence of labels
+    exactly matches the labels of C{ancestorName}.
+
+    C{descendantName} is considered a I{subdomain} if its sequence of
+    labels ends with the labels of C{ancestorName}.
+
+    @type descendantName: L{bytes}
+    @param descendantName: The DNS subdomain name.
+
+    @type ancestorName: L{bytes}
+    @param ancestorName: The DNS parent or ancestor domain name.
+
+    @return: C{True} if C{descendantName} is equal to or if it is a
+        subdomain of C{ancestorName}. Otherwise returns C{False}.
+    """
+    descendantLabels = _nameToLabels(descendantName.lower())
+    ancestorLabels = _nameToLabels(ancestorName.lower())
+    return descendantLabels[-len(ancestorLabels) :] == ancestorLabels
+
+
+def _str2time(s: str) -> int:
+    """
+    mypy doesn't like type-punning str | bytes | int | None into a str so we have this helper function.
+    """
+    suffixes = (
+        ("S", 1),
+        ("M", 60),
+        ("H", 60 * 60),
+        ("D", 60 * 60 * 24),
+        ("W", 60 * 60 * 24 * 7),
+        ("Y", 60 * 60 * 24 * 365),
+    )
+    s = s.upper().strip()
+    for suff, mult in suffixes:
+        if s.endswith(suff):
+            return int(float(s[:-1]) * mult)
+    try:
+        return int(s)
+    except ValueError:
+        raise ValueError("Invalid time interval specifier: " + s)
+
+
+@overload
+def str2time(s: Union[str, bytes, int]) -> int:
+    ...
+
+
+@overload
+def str2time(s: None) -> None:
+    ...
+
+
+def str2time(s: Union[str, bytes, int, None]) -> Union[int, None]:
+    """
+    Parse a string description of an interval into an integer number of seconds.
+
+    @param s: An interval definition constructed as an interval duration
+        followed by an interval unit.  An interval duration is a base ten
+        representation of an integer.  An interval unit is one of the following
+        letters: S (seconds), M (minutes), H (hours), D (days), W (weeks), or Y
+        (years).  For example: C{"3S"} indicates an interval of three seconds;
+        C{"5D"} indicates an interval of five days.  Alternatively, C{s} may be
+        any non-string and it will be returned unmodified.
+    @type s: text string (L{bytes} or L{str}) for parsing; anything else
+        for passthrough.
+
+    @return: an L{int} giving the interval represented by the string C{s}, or
+        whatever C{s} is if it is not a string.
+    """
+    if isinstance(s, bytes):
+        return _str2time(s.decode("ascii"))
+
+    if isinstance(s, str):
+        return _str2time(s)
+
+    return s
+
+
+def readPrecisely(file, l):
+    buff = file.read(l)
+    if len(buff) < l:
+        raise EOFError
+    return buff
+
+
+class IEncodable(Interface):
+    """
+    Interface for something which can be encoded to and decoded
+    to the DNS wire format.
+
+    A binary-mode file object (such as L{io.BytesIO}) is used as a buffer when
+    encoding or decoding.
+    """
+
+    def encode(strio, compDict=None):
+        """
+        Write a representation of this object to the given
+        file object.
+
+        @type strio: File-like object
+        @param strio: The buffer to write to. It must have a C{tell()} method.
+
+        @type compDict: L{dict} of L{bytes} to L{int} r L{None}
+        @param compDict: A mapping of names to byte offsets that have already
+        been written to the buffer, which may be used for compression (see RFC
+        1035 section 4.1.4). When L{None}, encode without compression.
+        """
+
+    def decode(strio, length=None):
+        """
+        Reconstruct an object from data read from the given
+        file object.
+
+        @type strio: File-like object
+        @param strio: A seekable buffer from which bytes may be read.
+
+        @type length: L{int} or L{None}
+        @param length: The number of bytes in this RDATA field.  Most
+        implementations can ignore this value.  Only in the case of
+        records similar to TXT where the total length is in no way
+        encoded in the data is it necessary.
+        """
+
+
+class IEncodableRecord(IEncodable, IRecord):
+    """
+    Interface for DNS records that can be encoded and decoded.
+
+    @since: Twisted 21.2.0
+    """
+
+
+@implementer(IEncodable)
+class Charstr:
+    def __init__(self, string: bytes = b""):
+        if not isinstance(string, bytes):
+            raise ValueError(f"{string!r} is not a byte string")
+        self.string = string
+
+    def encode(self, strio, compDict=None):
+        """
+        Encode this Character string into the appropriate byte format.
+
+        @type strio: file
+        @param strio: The byte representation of this Charstr will be written
+            to this file.
+        """
+        string = self.string
+        ind = len(string)
+        strio.write(_ord2bytes(ind))
+        strio.write(string)
+
+    def decode(self, strio, length=None):
+        """
+        Decode a byte string into this Charstr.
+
+        @type strio: file
+        @param strio: Bytes will be read from this file until the full string
+            is decoded.
+
+        @raise EOFError: Raised when there are not enough bytes available from
+            C{strio}.
+        """
+        self.string = b""
+        l = ord(readPrecisely(strio, 1))
+        self.string = readPrecisely(strio, l)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Charstr):
+            return self.string == other.string
+        return NotImplemented
+
+    def __hash__(self):
+        return hash(self.string)
+
+    def __str__(self) -> str:
+        """
+        Represent this L{Charstr} instance by its string value.
+        """
+        return nativeString(self.string)
+
+
+@implementer(IEncodable)
+class Name:
+    """
+    A name in the domain name system, made up of multiple labels.  For example,
+    I{twistedmatrix.com}.
+
+    @ivar name: A byte string giving the name.
+    @type name: L{bytes}
+    """
+
+    def __init__(self, name: bytes | str = b""):
+        """
+        @param name: A name.
+        @type name: L{bytes} or L{str}
+        """
+        self.name = domainString(name)
+
+    def encode(self, strio, compDict=None):
+        """
+        Encode this Name into the appropriate byte format.
+
+        @type strio: file
+        @param strio: The byte representation of this Name will be written to
+        this file.
+
+        @type compDict: dict
+        @param compDict: dictionary of Names that have already been encoded
+        and whose addresses may be backreferenced by this Name (for the purpose
+        of reducing the message size).
+        """
+        name = self.name
+        while name:
+            if compDict is not None:
+                if name in compDict:
+                    strio.write(struct.pack("!H", 0xC000 | compDict[name]))
+                    return
+                else:
+                    compDict[name] = strio.tell() + Message.headerSize
+            ind = name.find(b".")
+            if ind > 0:
+                label, name = name[:ind], name[ind + 1 :]
+            else:
+                # This is the last label, end the loop after handling it.
+                label = name
+                name = None
+                ind = len(label)
+            strio.write(_ord2bytes(ind))
+            strio.write(label)
+        strio.write(b"\x00")
+
+    def decode(self, strio, length=None):
+        """
+        Decode a byte string into this Name.
+
+        @type strio: file
+        @param strio: Bytes will be read from this file until the full Name
+        is decoded.
+
+        @raise EOFError: Raised when there are not enough bytes available
+        from C{strio}.
+
+        @raise ValueError: Raised when the name cannot be decoded (for example,
+            because it contains a loop).
+        """
+        visited = set()
+        self.name = b""
+        off = 0
+        while 1:
+            l = ord(readPrecisely(strio, 1))
+            if l == 0:
+                if off > 0:
+                    strio.seek(off)
+                return
+            if (l >> 6) == 3:
+                new_off = (l & 63) << 8 | ord(readPrecisely(strio, 1))
+                if new_off in visited:
+                    raise ValueError("Compression loop in encoded name")
+                visited.add(new_off)
+                if off == 0:
+                    off = strio.tell()
+                strio.seek(new_off)
+                continue
+            label = readPrecisely(strio, l)
+            if self.name == b"":
+                self.name = label
+            else:
+                self.name = self.name + b"." + label
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Name):
+            return self.name.lower() == other.name.lower()
+        return NotImplemented
+
+    def __hash__(self):
+        return hash(self.name)
+
+    def __str__(self) -> str:
+        """
+        Represent this L{Name} instance by its string name.
+        """
+        return nativeString(self.name)
+
+
+@comparable
+@implementer(IEncodable)
+class Query:
+    """
+    Represent a single DNS query.
+
+    @ivar name: The name about which this query is requesting information.
+    @type name: L{Name}
+
+    @ivar type: The query type.
+    @type type: L{int}
+
+    @ivar cls: The query class.
+    @type cls: L{int}
+    """
+
+    def __init__(self, name: Union[bytes, str] = b"", type: int = A, cls: int = IN):
+        """
+        @type name: L{bytes} or L{str}
+        @param name: See L{Query.name}
+
+        @type type: L{int}
+        @param type: The query type.
+
+        @type cls: L{int}
+        @param cls: The query class.
+        """
+        self.name = Name(name)
+        self.type = type
+        self.cls = cls
+
+    def encode(self, strio, compDict=None):
+        self.name.encode(strio, compDict)
+        strio.write(struct.pack("!HH", self.type, self.cls))
+
+    def decode(self, strio, length=None):
+        self.name.decode(strio)
+        buff = readPrecisely(strio, 4)
+        self.type, self.cls = struct.unpack("!HH", buff)
+
+    def __hash__(self):
+        return hash((self.name.name.lower(), self.type, self.cls))
+
+    def __cmp__(self, other):
+        if isinstance(other, Query):
+            return cmp(
+                (self.name.name.lower(), self.type, self.cls),
+                (other.name.name.lower(), other.type, other.cls),
+            )
+        return NotImplemented
+
+    def __str__(self) -> str:
+        t = QUERY_TYPES.get(
+            self.type, EXT_QUERIES.get(self.type, "UNKNOWN (%d)" % self.type)
+        )
+        c = QUERY_CLASSES.get(self.cls, "UNKNOWN (%d)" % self.cls)
+        return f"<Query {self.name} {t} {c}>"
+
+    def __repr__(self) -> str:
+        return f"Query({self.name.name!r}, {self.type!r}, {self.cls!r})"
+
+
+@implementer(IEncodable)
+class _OPTHeader(tputil.FancyStrMixin, tputil.FancyEqMixin):
+    """
+    An OPT record header.
+
+    @ivar name: The DNS name associated with this record. Since this
+        is a pseudo record, the name is always an L{Name} instance
+        with value b'', which represents the DNS root zone. This
+        attribute is a readonly property.
+
+    @ivar type: The DNS record type. This is a fixed value of 41
+        C{dns.OPT} for OPT Record. This attribute is a readonly
+        property.
+
+    @see: L{_OPTHeader.__init__} for documentation of other public
+        instance attributes.
+
+    @see: U{https://tools.ietf.org/html/rfc6891#section-6.1.2}
+
+    @since: 13.2
+    """
+
+    showAttributes = (
+        ("name", lambda n: nativeString(n.name)),
+        "type",
+        "udpPayloadSize",
+        "extendedRCODE",
+        "version",
+        "dnssecOK",
+        "options",
+    )
+
+    compareAttributes = (
+        "name",
+        "type",
+        "udpPayloadSize",
+        "extendedRCODE",
+        "version",
+        "dnssecOK",
+        "options",
+    )
+
+    def __init__(
+        self,
+        udpPayloadSize=4096,
+        extendedRCODE=0,
+        version=0,
+        dnssecOK=False,
+        options=None,
+    ):
+        """
+        @type udpPayloadSize: L{int}
+        @param udpPayloadSize: The number of octets of the largest UDP
+            payload that can be reassembled and delivered in the
+            requestor's network stack.
+
+        @type extendedRCODE: L{int}
+        @param extendedRCODE: Forms the upper 8 bits of extended
+            12-bit RCODE (together with the 4 bits defined in
+            [RFC1035].  Note that EXTENDED-RCODE value 0 indicates
+            that an unextended RCODE is in use (values 0 through 15).
+
+        @type version: L{int}
+        @param version: Indicates the implementation level of the
+            setter.  Full conformance with this specification is
+            indicated by version C{0}.
+
+        @type dnssecOK: L{bool}
+        @param dnssecOK: DNSSEC OK bit as defined by [RFC3225].
+
+        @type options: L{list}
+        @param options: A L{list} of 0 or more L{_OPTVariableOption}
+            instances.
+        """
+        self.udpPayloadSize = udpPayloadSize
+        self.extendedRCODE = extendedRCODE
+        self.version = version
+        self.dnssecOK = dnssecOK
+
+        if options is None:
+            options = []
+        self.options = options
+
+    @property
+    def name(self):
+        """
+        A readonly property for accessing the C{name} attribute of
+        this record.
+
+        @return: The DNS name associated with this record. Since this
+            is a pseudo record, the name is always an L{Name} instance
+            with value b'', which represents the DNS root zone.
+        """
+        return Name(b"")
+
+    @property
+    def type(self):
+        """
+        A readonly property for accessing the C{type} attribute of
+        this record.
+
+        @return: The DNS record type. This is a fixed value of 41
+            (C{dns.OPT} for OPT Record.
+        """
+        return OPT
+
+    def encode(self, strio, compDict=None):
+        """
+        Encode this L{_OPTHeader} instance to bytes.
+
+        @type strio: file
+        @param strio: the byte representation of this L{_OPTHeader}
+            will be written to this file.
+
+        @type compDict: L{dict} or L{None}
+        @param compDict: A dictionary of backreference addresses that
+            have already been written to this stream and that may
+            be used for DNS name compression.
+        """
+        b = BytesIO()
+        for o in self.options:
+            o.encode(b)
+        optionBytes = b.getvalue()
+
+        RRHeader(
+            name=self.name.name,
+            type=self.type,
+            cls=self.udpPayloadSize,
+            ttl=(self.extendedRCODE << 24 | self.version << 16 | self.dnssecOK << 15),
+            payload=UnknownRecord(optionBytes),
+        ).encode(strio, compDict)
+
+    def decode(self, strio, length=None):
+        """
+        Decode bytes into an L{_OPTHeader} instance.
+
+        @type strio: file
+        @param strio: Bytes will be read from this file until the full
+            L{_OPTHeader} is decoded.
+
+        @type length: L{int} or L{None}
+        @param length: Not used.
+        """
+
+        h = RRHeader()
+        h.decode(strio, length)
+        h.payload = UnknownRecord(readPrecisely(strio, h.rdlength))
+
+        newOptHeader = self.fromRRHeader(h)
+
+        for attrName in self.compareAttributes:
+            if attrName not in ("name", "type"):
+                setattr(self, attrName, getattr(newOptHeader, attrName))
+
+    @classmethod
+    def fromRRHeader(cls, rrHeader):
+        """
+        A classmethod for constructing a new L{_OPTHeader} from the
+        attributes and payload of an existing L{RRHeader} instance.
+
+        @type rrHeader: L{RRHeader}
+        @param rrHeader: An L{RRHeader} instance containing an
+            L{UnknownRecord} payload.
+
+        @return: An instance of L{_OPTHeader}.
+        @rtype: L{_OPTHeader}
+        """
+        options = None
+        if rrHeader.payload is not None:
+            options = []
+            optionsBytes = BytesIO(rrHeader.payload.data)
+            optionsBytesLength = len(rrHeader.payload.data)
+            while optionsBytes.tell() < optionsBytesLength:
+                o = _OPTVariableOption()
+                o.decode(optionsBytes)
+                options.append(o)
+
+        # Decode variable options if present
+        return cls(
+            udpPayloadSize=rrHeader.cls,
+            extendedRCODE=rrHeader.ttl >> 24,
+            version=rrHeader.ttl >> 16 & 0xFF,
+            dnssecOK=(rrHeader.ttl & 0xFFFF) >> 15,
+            options=options,
+        )
+
+
+@implementer(IEncodable)
+class _OPTVariableOption(tputil.FancyStrMixin, tputil.FancyEqMixin):
+    """
+    A class to represent OPT record variable options.
+
+    @see: L{_OPTVariableOption.__init__} for documentation of public
+        instance attributes.
+
+    @see: U{https://tools.ietf.org/html/rfc6891#section-6.1.2}
+
+    @since: 13.2
+    """
+
+    showAttributes = ("code", ("data", nativeString))
+    compareAttributes = ("code", "data")
+
+    _fmt = "!HH"
+
+    def __init__(self, code=0, data=b""):
+        """
+        @type code: L{int}
+        @param code: The option code
+
+        @type data: L{bytes}
+        @param data: The option data
+        """
+        self.code = code
+        self.data = data
+
+    def encode(self, strio, compDict=None):
+        """
+        Encode this L{_OPTVariableOption} to bytes.
+
+        @type strio: file
+        @param strio: the byte representation of this
+            L{_OPTVariableOption} will be written to this file.
+
+        @type compDict: L{dict} or L{None}
+        @param compDict: A dictionary of backreference addresses that
+            have already been written to this stream and that may
+            be used for DNS name compression.
+        """
+        strio.write(struct.pack(self._fmt, self.code, len(self.data)) + self.data)
+
+    def decode(self, strio, length=None):
+        """
+        Decode bytes into an L{_OPTVariableOption} instance.
+
+        @type strio: file
+        @param strio: Bytes will be read from this file until the full
+            L{_OPTVariableOption} is decoded.
+
+        @type length: L{int} or L{None}
+        @param length: Not used.
+        """
+        l = struct.calcsize(self._fmt)
+        buff = readPrecisely(strio, l)
+        self.code, length = struct.unpack(self._fmt, buff)
+        self.data = readPrecisely(strio, length)
+
+
+@implementer(IEncodable)
+class RRHeader(tputil.FancyEqMixin):
+    """
+    A resource record header.
+
+    @cvar fmt: L{str} specifying the byte format of an RR.
+
+    @ivar name: The name about which this reply contains information.
+    @type name: L{Name}
+
+    @ivar type: The query type of the original request.
+    @type type: L{int}
+
+    @ivar cls: The query class of the original request.
+
+    @ivar ttl: The time-to-live for this record.
+    @type ttl: L{int}
+
+    @ivar payload: The record described by this header.
+    @type payload: L{IEncodableRecord} or L{None}
+
+    @ivar auth: A L{bool} indicating whether this C{RRHeader} was parsed from
+        an authoritative message.
+    """
+
+    compareAttributes = ("name", "type", "cls", "ttl", "payload", "auth")
+
+    fmt = "!HHIH"
+
+    rdlength = None
+
+    cachedResponse = None
+
+    def __init__(
+        self,
+        name: Union[bytes, str] = b"",
+        type: int = A,
+        cls: int = IN,
+        ttl: SupportsInt = 0,
+        payload: Optional[IEncodableRecord] = None,
+        auth: bool = False,
+    ):
+        """
+        @type name: L{bytes} or L{str}
+        @param name: See L{RRHeader.name}
+
+        @type type: L{int}
+        @param type: The query type.
+
+        @type cls: L{int}
+        @param cls: The query class.
+
+        @type ttl: L{int}
+        @param ttl: Time to live for this record.  This will be
+            converted to an L{int}.
+
+        @type payload: L{IEncodableRecord} or L{None}
+        @param payload: An optional Query Type specific data object.
+
+        @raises TypeError: if the ttl cannot be converted to an L{int}.
+        @raises ValueError: if the ttl is negative.
+        @raises ValueError: if the payload type is not equal to the C{type}
+                            argument.
+        """
+        payloadType = None if payload is None else payload.TYPE
+        if payloadType is not None and payloadType != type:
+            raise ValueError(
+                "Payload type (%s) does not match given type (%s)"
+                % (
+                    QUERY_TYPES.get(payloadType, payloadType),
+                    QUERY_TYPES.get(type, type),
+                )
+            )
+
+        integralTTL = int(ttl)
+
+        if integralTTL < 0:
+            raise ValueError("TTL cannot be negative")
+
+        self.name = Name(name)
+        self.type = type
+        self.cls = cls
+        self.ttl = integralTTL
+        self.payload = payload
+        self.auth = auth
+
+    def encode(self, strio, compDict=None):
+        self.name.encode(strio, compDict)
+        strio.write(struct.pack(self.fmt, self.type, self.cls, self.ttl, 0))
+        if self.payload:
+            prefix = strio.tell()
+            self.payload.encode(strio, compDict)
+            aft = strio.tell()
+            strio.seek(prefix - 2, 0)
+            strio.write(struct.pack("!H", aft - prefix))
+            strio.seek(aft, 0)
+
+    def decode(self, strio, length=None):
+        self.name.decode(strio)
+        l = struct.calcsize(self.fmt)
+        buff = readPrecisely(strio, l)
+        r = struct.unpack(self.fmt, buff)
+        self.type, self.cls, self.ttl, self.rdlength = r
+
+    def isAuthoritative(self):
+        return self.auth
+
+    def __str__(self) -> str:
+        t = QUERY_TYPES.get(
+            self.type, EXT_QUERIES.get(self.type, "UNKNOWN (%d)" % self.type)
+        )
+        c = QUERY_CLASSES.get(self.cls, "UNKNOWN (%d)" % self.cls)
+        return "<RR name=%s type=%s class=%s ttl=%ds auth=%s>" % (
+            self.name,
+            t,
+            c,
+            self.ttl,
+            self.auth and "True" or "False",
+        )
+
+    __repr__ = __str__
+
+
+@implementer(IEncodableRecord)
+class SimpleRecord(tputil.FancyStrMixin, tputil.FancyEqMixin):
+    """
+    A Resource Record which consists of a single RFC 1035 domain-name.
+
+    @type name: L{Name}
+    @ivar name: The name associated with this record.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+    """
+
+    showAttributes = (("name", "name", "%s"), "ttl")
+    compareAttributes = ("name", "ttl")
+
+    TYPE: Optional[int] = None
+    name = None
+
+    def __init__(self, name=b"", ttl=None):
+        """
+        @param name: See L{SimpleRecord.name}
+        @type name: L{bytes} or L{str}
+        """
+        self.name = Name(name)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        self.name.encode(strio, compDict)
+
+    def decode(self, strio, length=None):
+        self.name = Name()
+        self.name.decode(strio)
+
+    def __hash__(self):
+        return hash(self.name)
+
+
+# Kinds of RRs - oh my!
+class Record_NS(SimpleRecord):
+    """
+    An authoritative nameserver.
+    """
+
+    TYPE = NS
+    fancybasename = "NS"
+
+
+class Record_MD(SimpleRecord):
+    """
+    A mail destination.
+
+    This record type is obsolete.
+
+    @see: L{Record_MX}
+    """
+
+    TYPE = MD
+    fancybasename = "MD"
+
+
+class Record_MF(SimpleRecord):
+    """
+    A mail forwarder.
+
+    This record type is obsolete.
+
+    @see: L{Record_MX}
+    """
+
+    TYPE = MF
+    fancybasename = "MF"
+
+
+class Record_CNAME(SimpleRecord):
+    """
+    The canonical name for an alias.
+    """
+
+    TYPE = CNAME
+    fancybasename = "CNAME"
+
+
+class Record_MB(SimpleRecord):
+    """
+    A mailbox domain name.
+
+    This is an experimental record type.
+    """
+
+    TYPE = MB
+    fancybasename = "MB"
+
+
+class Record_MG(SimpleRecord):
+    """
+    A mail group member.
+
+    This is an experimental record type.
+    """
+
+    TYPE = MG
+    fancybasename = "MG"
+
+
+class Record_MR(SimpleRecord):
+    """
+    A mail rename domain name.
+
+    This is an experimental record type.
+    """
+
+    TYPE = MR
+    fancybasename = "MR"
+
+
+class Record_PTR(SimpleRecord):
+    """
+    A domain name pointer.
+    """
+
+    TYPE = PTR
+    fancybasename = "PTR"
+
+
+class Record_DNAME(SimpleRecord):
+    """
+    A non-terminal DNS name redirection.
+
+    This record type provides the capability to map an entire subtree of the
+    DNS name space to another domain.  It differs from the CNAME record which
+    maps a single node of the name space.
+
+    @see: U{http://www.faqs.org/rfcs/rfc2672.html}
+    @see: U{http://www.faqs.org/rfcs/rfc3363.html}
+    """
+
+    TYPE = DNAME
+    fancybasename = "DNAME"
+
+
+@implementer(IEncodableRecord)
+class Record_A(tputil.FancyEqMixin):
+    """
+    An IPv4 host address.
+
+    @type address: L{bytes}
+    @ivar address: The packed network-order representation of the IPv4 address
+        associated with this record.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+    """
+
+    compareAttributes = ("address", "ttl")
+
+    TYPE = A
+    address = None
+
+    def __init__(self, address="0.0.0.0", ttl=None):
+        """
+        @type address: L{bytes} or L{str}
+        @param address: The IPv4 address associated with this record, in
+            quad-dotted notation.
+        """
+        if isinstance(address, bytes):
+            address = address.decode("ascii")
+
+        address = socket.inet_aton(address)
+        self.address = address
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(self.address)
+
+    def decode(self, strio, length=None):
+        self.address = readPrecisely(strio, 4)
+
+    def __hash__(self):
+        return hash(self.address)
+
+    def __str__(self) -> str:
+        return f"<A address={self.dottedQuad()} ttl={self.ttl}>"
+
+    __repr__ = __str__
+
+    def dottedQuad(self):
+        return socket.inet_ntoa(self.address)
+
+
+@implementer(IEncodableRecord)
+class Record_SOA(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    Marks the start of a zone of authority.
+
+    This record describes parameters which are shared by all records within a
+    particular zone.
+
+    @type mname: L{Name}
+    @ivar mname: The domain-name of the name server that was the original or
+        primary source of data for this zone.
+
+    @type rname: L{Name}
+    @ivar rname: A domain-name which specifies the mailbox of the person
+        responsible for this zone.
+
+    @type serial: L{int}
+    @ivar serial: The unsigned 32 bit version number of the original copy of
+        the zone.  Zone transfers preserve this value.  This value wraps and
+        should be compared using sequence space arithmetic.
+
+    @type refresh: L{int}
+    @ivar refresh: A 32 bit time interval before the zone should be refreshed.
+
+    @type minimum: L{int}
+    @ivar minimum: The unsigned 32 bit minimum TTL field that should be
+        exported with any RR from this zone.
+
+    @type expire: L{int}
+    @ivar expire: A 32 bit time value that specifies the upper limit on the
+        time interval that can elapse before the zone is no longer
+        authoritative.
+
+    @type retry: L{int}
+    @ivar retry: A 32 bit time interval that should elapse before a failed
+        refresh should be retried.
+
+    @type ttl: L{int}
+    @ivar ttl: The default TTL to use for records served from this zone.
+    """
+
+    fancybasename = "SOA"
+    compareAttributes = (
+        "serial",
+        "mname",
+        "rname",
+        "refresh",
+        "expire",
+        "retry",
+        "minimum",
+        "ttl",
+    )
+    showAttributes = (
+        ("mname", "mname", "%s"),
+        ("rname", "rname", "%s"),
+        "serial",
+        "refresh",
+        "retry",
+        "expire",
+        "minimum",
+        "ttl",
+    )
+
+    TYPE = SOA
+
+    def __init__(
+        self,
+        mname=b"",
+        rname=b"",
+        serial=0,
+        refresh=0,
+        retry=0,
+        expire=0,
+        minimum=0,
+        ttl=None,
+    ):
+        """
+        @param mname: See L{Record_SOA.mname}
+        @type mname: L{bytes} or L{str}
+
+        @param rname: See L{Record_SOA.rname}
+        @type rname: L{bytes} or L{str}
+        """
+        self.mname, self.rname = Name(mname), Name(rname)
+        self.serial, self.refresh = str2time(serial), str2time(refresh)
+        self.minimum, self.expire = str2time(minimum), str2time(expire)
+        self.retry = str2time(retry)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        self.mname.encode(strio, compDict)
+        self.rname.encode(strio, compDict)
+        strio.write(
+            struct.pack(
+                "!LlllL",
+                self.serial,
+                self.refresh,
+                self.retry,
+                self.expire,
+                self.minimum,
+            )
+        )
+
+    def decode(self, strio, length=None):
+        self.mname, self.rname = Name(), Name()
+        self.mname.decode(strio)
+        self.rname.decode(strio)
+        r = struct.unpack("!LlllL", readPrecisely(strio, 20))
+        self.serial, self.refresh, self.retry, self.expire, self.minimum = r
+
+    def __hash__(self):
+        return hash(
+            (self.serial, self.mname, self.rname, self.refresh, self.expire, self.retry)
+        )
+
+
+@implementer(IEncodableRecord)
+class Record_NULL(tputil.FancyStrMixin, tputil.FancyEqMixin):
+    """
+    A null record.
+
+    This is an experimental record type.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+    """
+
+    fancybasename = "NULL"
+    showAttributes = (("payload", _nicebytes), "ttl")
+    compareAttributes = ("payload", "ttl")
+
+    TYPE = NULL
+
+    def __init__(self, payload=None, ttl=None):
+        self.payload = payload
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(self.payload)
+
+    def decode(self, strio, length=None):
+        self.payload = readPrecisely(strio, length)
+
+    def __hash__(self):
+        return hash(self.payload)
+
+
+@implementer(IEncodableRecord)
+class Record_WKS(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    A well known service description.
+
+    This record type is obsolete.  See L{Record_SRV}.
+
+    @type address: L{bytes}
+    @ivar address: The packed network-order representation of the IPv4 address
+        associated with this record.
+
+    @type protocol: L{int}
+    @ivar protocol: The 8 bit IP protocol number for which this service map is
+        relevant.
+
+    @type map: L{bytes}
+    @ivar map: A bitvector indicating the services available at the specified
+        address.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+    """
+
+    fancybasename = "WKS"
+    compareAttributes = ("address", "protocol", "map", "ttl")
+    showAttributes = [("_address", "address", "%s"), "protocol", "ttl"]
+
+    TYPE = WKS
+
+    @property
+    def _address(self):
+        return socket.inet_ntoa(self.address)
+
+    def __init__(self, address="0.0.0.0", protocol=0, map=b"", ttl=None):
+        """
+        @type address: L{bytes} or L{str}
+        @param address: The IPv4 address associated with this record, in
+            quad-dotted notation.
+        """
+        if isinstance(address, bytes):
+            address = address.decode("idna")
+
+        self.address = socket.inet_aton(address)
+        self.protocol, self.map = protocol, map
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(self.address)
+        strio.write(struct.pack("!B", self.protocol))
+        strio.write(self.map)
+
+    def decode(self, strio, length=None):
+        self.address = readPrecisely(strio, 4)
+        self.protocol = struct.unpack("!B", readPrecisely(strio, 1))[0]
+        self.map = readPrecisely(strio, length - 5)
+
+    def __hash__(self):
+        return hash((self.address, self.protocol, self.map))
+
+
+@implementer(IEncodableRecord)
+class Record_AAAA(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    An IPv6 host address.
+
+    @type address: L{bytes}
+    @ivar address: The packed network-order representation of the IPv6 address
+        associated with this record.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+
+    @see: U{http://www.faqs.org/rfcs/rfc1886.html}
+    """
+
+    TYPE = AAAA
+
+    fancybasename = "AAAA"
+    showAttributes = (("_address", "address", "%s"), "ttl")
+    compareAttributes = ("address", "ttl")
+
+    @property
+    def _address(self):
+        return socket.inet_ntop(AF_INET6, self.address)
+
+    def __init__(self, address="::", ttl=None):
+        """
+        @type address: L{bytes} or L{str}
+        @param address: The IPv6 address for this host, in RFC 2373 format.
+        """
+        if isinstance(address, bytes):
+            address = address.decode("idna")
+
+        self.address = socket.inet_pton(AF_INET6, address)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(self.address)
+
+    def decode(self, strio, length=None):
+        self.address = readPrecisely(strio, 16)
+
+    def __hash__(self):
+        return hash(self.address)
+
+
+@implementer(IEncodableRecord)
+class Record_A6(tputil.FancyStrMixin, tputil.FancyEqMixin):
+    """
+    An IPv6 address.
+
+    This is an experimental record type.
+
+    @type prefixLen: L{int}
+    @ivar prefixLen: The length of the suffix.
+
+    @type suffix: L{bytes}
+    @ivar suffix: An IPv6 address suffix in network order.
+
+    @type prefix: L{Name}
+    @ivar prefix: If specified, a name which will be used as a prefix for other
+        A6 records.
+
+    @type bytes: L{int}
+    @ivar bytes: The length of the prefix.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+
+    @see: U{http://www.faqs.org/rfcs/rfc2874.html}
+    @see: U{http://www.faqs.org/rfcs/rfc3363.html}
+    @see: U{http://www.faqs.org/rfcs/rfc3364.html}
+    """
+
+    TYPE = A6
+
+    fancybasename = "A6"
+    showAttributes = (("_suffix", "suffix", "%s"), ("prefix", "prefix", "%s"), "ttl")
+    compareAttributes = ("prefixLen", "prefix", "suffix", "ttl")
+
+    @property
+    def _suffix(self):
+        return socket.inet_ntop(AF_INET6, self.suffix)
+
+    def __init__(
+        self,
+        prefixLen: int = 0,
+        suffix: bytes | str = "::",
+        prefix: bytes | str = b"",
+        ttl: Union[str, bytes, int, None] = None,
+    ):
+        """
+        @param suffix: An IPv6 address suffix in in RFC 2373 format.
+        @type suffix: L{bytes} or L{str}
+
+        @param prefix: An IPv6 address prefix for other A6 records.
+        @type prefix: L{bytes} or L{str}
+        """
+        if isinstance(suffix, bytes):
+            suffix = suffix.decode("idna")
+
+        self.prefixLen = prefixLen
+        self.suffix = socket.inet_pton(AF_INET6, suffix)
+        self.prefix = Name(prefix)
+        self.bytes = int((128 - self.prefixLen) / 8.0)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(struct.pack("!B", self.prefixLen))
+        if self.bytes:
+            strio.write(self.suffix[-self.bytes :])
+        if self.prefixLen:
+            # This may not be compressed
+            self.prefix.encode(strio, None)
+
+    def decode(self, strio, length=None):
+        self.prefixLen = struct.unpack("!B", readPrecisely(strio, 1))[0]
+        self.bytes = int((128 - self.prefixLen) / 8.0)
+        if self.bytes:
+            self.suffix = b"\x00" * (16 - self.bytes) + readPrecisely(strio, self.bytes)
+        if self.prefixLen:
+            self.prefix.decode(strio)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Record_A6):
+            return (
+                self.prefixLen == other.prefixLen
+                and self.suffix[-self.bytes :] == other.suffix[-self.bytes :]
+                and self.prefix == other.prefix
+                and self.ttl == other.ttl
+            )
+        return NotImplemented
+
+    def __hash__(self):
+        return hash((self.prefixLen, self.suffix[-self.bytes :], self.prefix))
+
+    def __str__(self) -> str:
+        return "<A6 %s %s (%d) ttl=%s>" % (
+            self.prefix,
+            socket.inet_ntop(AF_INET6, self.suffix),
+            self.prefixLen,
+            self.ttl,
+        )
+
+
+@implementer(IEncodableRecord)
+class Record_SRV(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    The location of the server(s) for a specific protocol and domain.
+
+    This is an experimental record type.
+
+    @type priority: L{int}
+    @ivar priority: The priority of this target host.  A client MUST attempt to
+        contact the target host with the lowest-numbered priority it can reach;
+        target hosts with the same priority SHOULD be tried in an order defined
+        by the weight field.
+
+    @type weight: L{int}
+    @ivar weight: Specifies a relative weight for entries with the same
+        priority. Larger weights SHOULD be given a proportionately higher
+        probability of being selected.
+
+    @type port: L{int}
+    @ivar port: The port on this target host of this service.
+
+    @type target: L{Name}
+    @ivar target: The domain name of the target host.  There MUST be one or
+        more address records for this name, the name MUST NOT be an alias (in
+        the sense of RFC 1034 or RFC 2181).  Implementors are urged, but not
+        required, to return the address record(s) in the Additional Data
+        section.  Unless and until permitted by future standards action, name
+        compression is not to be used for this field.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+
+    @see: U{http://www.faqs.org/rfcs/rfc2782.html}
+    """
+
+    TYPE = SRV
+
+    fancybasename = "SRV"
+    compareAttributes = ("priority", "weight", "target", "port", "ttl")
+    showAttributes = ("priority", "weight", ("target", "target", "%s"), "port", "ttl")
+
+    def __init__(self, priority=0, weight=0, port=0, target=b"", ttl=None):
+        """
+        @param target: See L{Record_SRV.target}
+        @type target: L{bytes} or L{str}
+        """
+        self.priority = int(priority)
+        self.weight = int(weight)
+        self.port = int(port)
+        self.target = Name(target)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(struct.pack("!HHH", self.priority, self.weight, self.port))
+        # This can't be compressed
+        self.target.encode(strio, None)
+
+    def decode(self, strio, length=None):
+        r = struct.unpack("!HHH", readPrecisely(strio, struct.calcsize("!HHH")))
+        self.priority, self.weight, self.port = r
+        self.target = Name()
+        self.target.decode(strio)
+
+    def __hash__(self):
+        return hash((self.priority, self.weight, self.port, self.target))
+
+
+@implementer(IEncodableRecord)
+class Record_NAPTR(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    The location of the server(s) for a specific protocol and domain.
+
+    @type order: L{int}
+    @ivar order: An integer specifying the order in which the NAPTR records
+        MUST be processed to ensure the correct ordering of rules.  Low numbers
+        are processed before high numbers.
+
+    @type preference: L{int}
+    @ivar preference: An integer that specifies the order in which NAPTR
+        records with equal "order" values SHOULD be processed, low numbers
+        being processed before high numbers.
+
+    @type flag: L{Charstr}
+    @ivar flag: A <character-string> containing flags to control aspects of the
+        rewriting and interpretation of the fields in the record.  Flags
+        are single characters from the set [A-Z0-9].  The case of the alphabetic
+        characters is not significant.
+
+        At this time only four flags, "S", "A", "U", and "P", are defined.
+
+    @type service: L{Charstr}
+    @ivar service: Specifies the service(s) available down this rewrite path.
+        It may also specify the particular protocol that is used to talk with a
+        service.  A protocol MUST be specified if the flags field states that
+        the NAPTR is terminal.
+
+    @type regexp: L{Charstr}
+    @ivar regexp: A STRING containing a substitution expression that is applied
+        to the original string held by the client in order to construct the
+        next domain name to lookup.
+
+    @type replacement: L{Name}
+    @ivar replacement: The next NAME to query for NAPTR, SRV, or address
+        records depending on the value of the flags field.  This MUST be a
+        fully qualified domain-name.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+
+    @see: U{http://www.faqs.org/rfcs/rfc2915.html}
+    """
+
+    TYPE = NAPTR
+
+    compareAttributes = (
+        "order",
+        "preference",
+        "flags",
+        "service",
+        "regexp",
+        "replacement",
+    )
+    fancybasename = "NAPTR"
+
+    showAttributes = (
+        "order",
+        "preference",
+        ("flags", "flags", "%s"),
+        ("service", "service", "%s"),
+        ("regexp", "regexp", "%s"),
+        ("replacement", "replacement", "%s"),
+        "ttl",
+    )
+
+    def __init__(
+        self,
+        order=0,
+        preference=0,
+        flags=b"",
+        service=b"",
+        regexp=b"",
+        replacement=b"",
+        ttl=None,
+    ):
+        """
+        @param replacement: See L{Record_NAPTR.replacement}
+        @type replacement: L{bytes} or L{str}
+        """
+        self.order = int(order)
+        self.preference = int(preference)
+        self.flags = Charstr(flags)
+        self.service = Charstr(service)
+        self.regexp = Charstr(regexp)
+        self.replacement = Name(replacement)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(struct.pack("!HH", self.order, self.preference))
+        # This can't be compressed
+        self.flags.encode(strio, None)
+        self.service.encode(strio, None)
+        self.regexp.encode(strio, None)
+        self.replacement.encode(strio, None)
+
+    def decode(self, strio, length=None):
+        r = struct.unpack("!HH", readPrecisely(strio, struct.calcsize("!HH")))
+        self.order, self.preference = r
+        self.flags = Charstr()
+        self.service = Charstr()
+        self.regexp = Charstr()
+        self.replacement = Name()
+        self.flags.decode(strio)
+        self.service.decode(strio)
+        self.regexp.decode(strio)
+        self.replacement.decode(strio)
+
+    def __hash__(self):
+        return hash(
+            (
+                self.order,
+                self.preference,
+                self.flags,
+                self.service,
+                self.regexp,
+                self.replacement,
+            )
+        )
+
+
+@implementer(IEncodableRecord)
+class Record_AFSDB(tputil.FancyStrMixin, tputil.FancyEqMixin):
+    """
+    Map from a domain name to the name of an AFS cell database server.
+
+    @type subtype: L{int}
+    @ivar subtype: In the case of subtype 1, the host has an AFS version 3.0
+        Volume Location Server for the named AFS cell.  In the case of subtype
+        2, the host has an authenticated name server holding the cell-root
+        directory node for the named DCE/NCA cell.
+
+    @type hostname: L{Name}
+    @ivar hostname: The domain name of a host that has a server for the cell
+        named by this record.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+
+    @see: U{http://www.faqs.org/rfcs/rfc1183.html}
+    """
+
+    TYPE = AFSDB
+
+    fancybasename = "AFSDB"
+    compareAttributes = ("subtype", "hostname", "ttl")
+    showAttributes = ("subtype", ("hostname", "hostname", "%s"), "ttl")
+
+    def __init__(self, subtype=0, hostname=b"", ttl=None):
+        """
+        @param hostname: See L{Record_AFSDB.hostname}
+        @type hostname: L{bytes} or L{str}
+        """
+        self.subtype = int(subtype)
+        self.hostname = Name(hostname)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(struct.pack("!H", self.subtype))
+        self.hostname.encode(strio, compDict)
+
+    def decode(self, strio, length=None):
+        r = struct.unpack("!H", readPrecisely(strio, struct.calcsize("!H")))
+        (self.subtype,) = r
+        self.hostname.decode(strio)
+
+    def __hash__(self):
+        return hash((self.subtype, self.hostname))
+
+
+@implementer(IEncodableRecord)
+class Record_RP(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    The responsible person for a domain.
+
+    @type mbox: L{Name}
+    @ivar mbox: A domain name that specifies the mailbox for the responsible
+        person.
+
+    @type txt: L{Name}
+    @ivar txt: A domain name for which TXT RR's exist (indirection through
+        which allows information sharing about the contents of this RP record).
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+
+    @see: U{http://www.faqs.org/rfcs/rfc1183.html}
+    """
+
+    TYPE = RP
+
+    fancybasename = "RP"
+    compareAttributes = ("mbox", "txt", "ttl")
+    showAttributes = (("mbox", "mbox", "%s"), ("txt", "txt", "%s"), "ttl")
+
+    def __init__(self, mbox=b"", txt=b"", ttl=None):
+        """
+        @param mbox: See L{Record_RP.mbox}.
+        @type mbox: L{bytes} or L{str}
+
+        @param txt: See L{Record_RP.txt}
+        @type txt: L{bytes} or L{str}
+        """
+        self.mbox = Name(mbox)
+        self.txt = Name(txt)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        self.mbox.encode(strio, compDict)
+        self.txt.encode(strio, compDict)
+
+    def decode(self, strio, length=None):
+        self.mbox = Name()
+        self.txt = Name()
+        self.mbox.decode(strio)
+        self.txt.decode(strio)
+
+    def __hash__(self):
+        return hash((self.mbox, self.txt))
+
+
+@implementer(IEncodableRecord)
+class Record_HINFO(tputil.FancyStrMixin, tputil.FancyEqMixin):
+    """
+    Host information.
+
+    @type cpu: L{bytes}
+    @ivar cpu: Specifies the CPU type.
+
+    @type os: L{bytes}
+    @ivar os: Specifies the OS.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+    """
+
+    TYPE = HINFO
+
+    fancybasename = "HINFO"
+    showAttributes = (("cpu", _nicebytes), ("os", _nicebytes), "ttl")
+    compareAttributes = ("cpu", "os", "ttl")
+
+    def __init__(
+        self,
+        cpu: bytes = b"",
+        os: bytes = b"",
+        ttl: Union[str, bytes, int, None] = None,
+    ):
+        self.cpu, self.os = cpu, os
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(struct.pack("!B", len(self.cpu)) + self.cpu)
+        strio.write(struct.pack("!B", len(self.os)) + self.os)
+
+    def decode(self, strio, length=None):
+        cpu = struct.unpack("!B", readPrecisely(strio, 1))[0]
+        self.cpu = readPrecisely(strio, cpu)
+        os = struct.unpack("!B", readPrecisely(strio, 1))[0]
+        self.os = readPrecisely(strio, os)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Record_HINFO):
+            return (
+                self.os.lower() == other.os.lower()
+                and self.cpu.lower() == other.cpu.lower()
+                and self.ttl == other.ttl
+            )
+        return NotImplemented
+
+    def __hash__(self):
+        return hash((self.os.lower(), self.cpu.lower()))
+
+
+@implementer(IEncodableRecord)
+class Record_MINFO(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    Mailbox or mail list information.
+
+    This is an experimental record type.
+
+    @type rmailbx: L{Name}
+    @ivar rmailbx: A domain-name which specifies a mailbox which is responsible
+        for the mailing list or mailbox.  If this domain name names the root,
+        the owner of the MINFO RR is responsible for itself.
+
+    @type emailbx: L{Name}
+    @ivar emailbx: A domain-name which specifies a mailbox which is to receive
+        error messages related to the mailing list or mailbox specified by the
+        owner of the MINFO record.  If this domain name names the root, errors
+        should be returned to the sender of the message.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+    """
+
+    TYPE = MINFO
+
+    rmailbx = None
+    emailbx = None
+
+    fancybasename = "MINFO"
+    compareAttributes = ("rmailbx", "emailbx", "ttl")
+    showAttributes = (
+        ("rmailbx", "responsibility", "%s"),
+        ("emailbx", "errors", "%s"),
+        "ttl",
+    )
+
+    def __init__(self, rmailbx=b"", emailbx=b"", ttl=None):
+        """
+        @param rmailbx: See L{Record_MINFO.rmailbx}.
+        @type rmailbx: L{bytes} or L{str}
+
+        @param emailbx: See L{Record_MINFO.rmailbx}.
+        @type emailbx: L{bytes} or L{str}
+        """
+        self.rmailbx, self.emailbx = Name(rmailbx), Name(emailbx)
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        self.rmailbx.encode(strio, compDict)
+        self.emailbx.encode(strio, compDict)
+
+    def decode(self, strio, length=None):
+        self.rmailbx, self.emailbx = Name(), Name()
+        self.rmailbx.decode(strio)
+        self.emailbx.decode(strio)
+
+    def __hash__(self):
+        return hash((self.rmailbx, self.emailbx))
+
+
+@implementer(IEncodableRecord)
+class Record_MX(tputil.FancyStrMixin, tputil.FancyEqMixin):
+    """
+    Mail exchange.
+
+    @type preference: L{int}
+    @ivar preference: Specifies the preference given to this RR among others at
+        the same owner.  Lower values are preferred.
+
+    @type name: L{Name}
+    @ivar name: A domain-name which specifies a host willing to act as a mail
+        exchange.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be
+        cached.
+    """
+
+    TYPE = MX
+
+    fancybasename = "MX"
+    compareAttributes = ("preference", "name", "ttl")
+    showAttributes = ("preference", ("name", "name", "%s"), "ttl")
+
+    def __init__(self, preference=0, name=b"", ttl=None, **kwargs):
+        """
+        @param name: See L{Record_MX.name}.
+        @type name: L{bytes} or L{str}
+        """
+        self.preference = int(preference)
+        self.name = Name(kwargs.get("exchange", name))
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        strio.write(struct.pack("!H", self.preference))
+        self.name.encode(strio, compDict)
+
+    def decode(self, strio, length=None):
+        self.preference = struct.unpack("!H", readPrecisely(strio, 2))[0]
+        self.name = Name()
+        self.name.decode(strio)
+
+    def __hash__(self):
+        return hash((self.preference, self.name))
+
+
+@implementer(IEncodableRecord)
+class Record_SSHFP(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    A record containing the fingerprint of an SSH key.
+
+    @type algorithm: L{int}
+    @ivar algorithm: The SSH key's algorithm, such as L{ALGORITHM_RSA}.
+        Note that the numbering used for SSH key algorithms is specific
+        to the SSHFP record, and is not the same as the numbering
+        used for KEY or SIG records.
+
+    @type fingerprintType: L{int}
+    @ivar fingerprintType: The fingerprint type,
+        such as L{FINGERPRINT_TYPE_SHA256}.
+
+    @type fingerprint: L{bytes}
+    @ivar fingerprint: The key's fingerprint, e.g. a 32-byte SHA-256 digest.
+
+    @cvar ALGORITHM_RSA: The algorithm value for C{ssh-rsa} keys.
+    @cvar ALGORITHM_DSS: The algorithm value for C{ssh-dss} keys.
+    @cvar ALGORITHM_ECDSA: The algorithm value for C{ecdsa-sha2-*} keys.
+    @cvar ALGORITHM_Ed25519: The algorithm value for C{ed25519} keys.
+
+    @cvar FINGERPRINT_TYPE_SHA1: The type for SHA-1 fingerprints.
+    @cvar FINGERPRINT_TYPE_SHA256: The type for SHA-256 fingerprints.
+
+    @see: U{RFC 4255 <https://tools.ietf.org/html/rfc4255>}
+          and
+          U{RFC 6594 <https://tools.ietf.org/html/rfc6594>}
+    """
+
+    fancybasename = "SSHFP"
+    compareAttributes = ("algorithm", "fingerprintType", "fingerprint", "ttl")
+    showAttributes = ("algorithm", "fingerprintType", "fingerprint")
+
+    TYPE = SSHFP
+
+    ALGORITHM_RSA = 1
+    ALGORITHM_DSS = 2
+    ALGORITHM_ECDSA = 3
+    ALGORITHM_Ed25519 = 4
+
+    FINGERPRINT_TYPE_SHA1 = 1
+    FINGERPRINT_TYPE_SHA256 = 2
+
+    def __init__(self, algorithm=0, fingerprintType=0, fingerprint=b"", ttl=0):
+        self.algorithm = algorithm
+        self.fingerprintType = fingerprintType
+        self.fingerprint = fingerprint
+        self.ttl = ttl
+
+    def encode(self, strio, compDict=None):
+        strio.write(struct.pack("!BB", self.algorithm, self.fingerprintType))
+        strio.write(self.fingerprint)
+
+    def decode(self, strio, length=None):
+        r = struct.unpack("!BB", readPrecisely(strio, 2))
+        (self.algorithm, self.fingerprintType) = r
+        self.fingerprint = readPrecisely(strio, length - 2)
+
+    def __hash__(self):
+        return hash((self.algorithm, self.fingerprintType, self.fingerprint))
+
+
+@implementer(IEncodableRecord)
+class Record_TXT(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    Freeform text.
+
+    @type data: L{list} of L{bytes}
+    @ivar data: Freeform text which makes up this record.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be cached.
+    """
+
+    TYPE = TXT
+
+    fancybasename = "TXT"
+    showAttributes = (("data", _nicebyteslist), "ttl")
+    compareAttributes = ("data", "ttl")
+
+    def __init__(self, *data, **kw):
+        self.data = list(data)
+        # arg man python sucks so bad
+        self.ttl = str2time(kw.get("ttl", None))
+
+    def encode(self, strio, compDict=None):
+        for d in self.data:
+            strio.write(struct.pack("!B", len(d)) + d)
+
+    def decode(self, strio, length=None):
+        soFar = 0
+        self.data = []
+        while soFar < length:
+            L = struct.unpack("!B", readPrecisely(strio, 1))[0]
+            self.data.append(readPrecisely(strio, L))
+            soFar += L + 1
+        if soFar != length:
+            log.msg(
+                "Decoded %d bytes in %s record, but rdlength is %d"
+                % (soFar, self.fancybasename, length)
+            )
+
+    def __hash__(self):
+        return hash(tuple(self.data))
+
+
+@implementer(IEncodableRecord)
+class UnknownRecord(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    Encapsulate the wire data for unknown record types so that they can
+    pass through the system unchanged.
+
+    @type data: L{bytes}
+    @ivar data: Wire data which makes up this record.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds which this record should be cached.
+
+    @since: 11.1
+    """
+
+    TYPE = None
+
+    fancybasename = "UNKNOWN"
+    compareAttributes = ("data", "ttl")
+    showAttributes = (("data", _nicebytes), "ttl")
+
+    def __init__(self, data=b"", ttl=None):
+        self.data = data
+        self.ttl = str2time(ttl)
+
+    def encode(self, strio, compDict=None):
+        """
+        Write the raw bytes corresponding to this record's payload to the
+        stream.
+        """
+        strio.write(self.data)
+
+    def decode(self, strio, length=None):
+        """
+        Load the bytes which are part of this record from the stream and store
+        them unparsed and unmodified.
+        """
+        if length is None:
+            raise Exception("must know length for unknown record types")
+        self.data = readPrecisely(strio, length)
+
+    def __hash__(self):
+        return hash((self.data, self.ttl))
+
+
+class Record_SPF(Record_TXT):
+    """
+    Structurally, freeform text. Semantically, a policy definition, formatted
+    as defined in U{rfc 4408<http://www.faqs.org/rfcs/rfc4408.html>}.
+
+    @type data: L{list} of L{bytes}
+    @ivar data: Freeform text which makes up this record.
+
+    @type ttl: L{int}
+    @ivar ttl: The maximum number of seconds
+               which this record should be cached.
+    """
+
+    TYPE = SPF
+    fancybasename = "SPF"
+
+
+@implementer(IEncodableRecord)
+class Record_TSIG(tputil.FancyEqMixin, tputil.FancyStrMixin):
+    """
+    A transaction signature, encapsulated in a RR, as described
+    in U{RFC 2845 <https://tools.ietf.org/html/rfc2845>}.
+
+    @type algorithm: L{Name}
+    @ivar algorithm: The name of the signature or MAC algorithm.
+
+    @type timeSigned: L{int}
+    @ivar timeSigned: Signing time, as seconds from the POSIX epoch.
+
+    @type fudge: L{int}
+    @ivar fudge: Allowable time skew, in seconds.
+
+    @type MAC: L{bytes}
+    @ivar MAC: The message digest or signature.
+
+    @type originalID: L{int}
+    @ivar originalID: A message ID.
+
+    @type error: L{int}
+    @ivar error: An error code (extended C{RCODE}) carried
+          in exceptional cases.
+
+    @type otherData: L{bytes}
+    @ivar otherData: Other data carried in exceptional cases.
+
+    """
+
+    fancybasename = "TSIG"
+    compareAttributes = (
+        "algorithm",
+        "timeSigned",
+        "fudge",
+        "MAC",
+        "originalID",
+        "error",
+        "otherData",
+        "ttl",
+    )
+    showAttributes = ["algorithm", "timeSigned", "MAC", "error", "otherData"]
+
+    TYPE = TSIG
+
+    def __init__(
+        self,
+        algorithm=None,
+        timeSigned=None,
+        fudge=5,
+        MAC=None,
+        originalID=0,
+        error=OK,
+        otherData=b"",
+        ttl=0,
+    ):
+        # All of our init arguments have to have defaults, because of
+        # the way IEncodable and Message.parseRecords() work, but for
+        # some of our arguments there is no reasonable default; we use
+        # invalid values here to prevent a user of this class from
+        # relying on what's really an internal implementation detail.
+        self.algorithm = None if algorithm is None else Name(algorithm)
+        self.timeSigned = timeSigned
+        self.fudge = str2time(fudge)
+        self.MAC = MAC
+        self.originalID = originalID
+        self.error = error
+        self.otherData = otherData
+        self.ttl = ttl
+
+    def encode(self, strio, compDict=None):
+        self.algorithm.encode(strio, compDict)
+        strio.write(struct.pack("!Q", self.timeSigned)[2:])  # 48-bit number
+        strio.write(struct.pack("!HH", self.fudge, len(self.MAC)))
+        strio.write(self.MAC)
+        strio.write(
+            struct.pack("!HHH", self.originalID, self.error, len(self.otherData))
+        )
+        strio.write(self.otherData)
+
+    def decode(self, strio, length=None):
+        algorithm = Name()
+        algorithm.decode(strio)
+        self.algorithm = algorithm
+        fields = struct.unpack("!QHH", b"\x00\x00" + readPrecisely(strio, 10))
+        self.timeSigned, self.fudge, macLength = fields
+        self.MAC = readPrecisely(strio, macLength)
+        fields = struct.unpack("!HHH", readPrecisely(strio, 6))
+        self.originalID, self.error, otherLength = fields
+        self.otherData = readPrecisely(strio, otherLength)
+
+    def __hash__(self):
+        return hash((self.algorithm, self.timeSigned, self.MAC, self.originalID))
+
+
+def _responseFromMessage(responseConstructor, message, **kwargs):
+    """
+    Generate a L{Message} like instance suitable for use as the response to
+    C{message}.
+
+    The C{queries}, C{id} attributes will be copied from C{message} and the
+    C{answer} flag will be set to L{True}.
+
+    @param responseConstructor: A response message constructor with an
+         initializer signature matching L{dns.Message.__init__}.
+    @type responseConstructor: C{callable}
+
+    @param message: A request message.
+    @type message: L{Message}
+
+    @param kwargs: Keyword arguments which will be passed to the initialiser
+        of the response message.
+    @type kwargs: L{dict}
+
+    @return: A L{Message} like response instance.
+    @rtype: C{responseConstructor}
+    """
+    response = responseConstructor(id=message.id, answer=True, **kwargs)
+    response.queries = message.queries[:]
+    return response
+
+
+def _getDisplayableArguments(obj, alwaysShow, fieldNames):
+    """
+    Inspect the function signature of C{obj}'s constructor,
+    and get a list of which arguments should be displayed.
+    This is a helper function for C{_compactRepr}.
+
+    @param obj: The instance whose repr is being generated.
+    @param alwaysShow: A L{list} of field names which should always be shown.
+    @param fieldNames: A L{list} of field attribute names which should be shown
+        if they have non-default values.
+    @return: A L{list} of displayable arguments.
+    """
+    displayableArgs = []
+    # Get the argument names and values from the constructor.
+    signature = inspect.signature(obj.__class__.__init__)
+    for name in fieldNames:
+        defaultValue = signature.parameters[name].default
+        fieldValue = getattr(obj, name, defaultValue)
+        if (name in alwaysShow) or (fieldValue != defaultValue):
+            displayableArgs.append(f" {name}={fieldValue!r}")
+
+    return displayableArgs
+
+
+def _compactRepr(
+    obj: object,
+    alwaysShow: Sequence[str] | None = None,
+    flagNames: Sequence[str] | None = None,
+    fieldNames: Sequence[str] | None = None,
+    sectionNames: Sequence[str] | None = None,
+) -> str:
+    """
+    Return a L{str} representation of C{obj} which only shows fields with
+    non-default values, flags which are True and sections which have been
+    explicitly set.
+
+    @param obj: The instance whose repr is being generated.
+    @param alwaysShow: A L{list} of field names which should always be shown.
+    @param flagNames: A L{list} of flag attribute names which should be shown if
+        they are L{True}.
+    @param fieldNames: A L{list} of field attribute names which should be shown
+        if they have non-default values.
+    @param sectionNames: A L{list} of section attribute names which should be
+        shown if they have been assigned a value.
+
+    @return: A L{str} representation of C{obj}.
+    """
+    if alwaysShow is None:
+        alwaysShow = []
+
+    if flagNames is None:
+        flagNames = []
+
+    if fieldNames is None:
+        fieldNames = []
+
+    if sectionNames is None:
+        sectionNames = []
+
+    setFlags = []
+    for name in flagNames:
+        if name in alwaysShow or getattr(obj, name, False) == True:
+            setFlags.append(name)
+
+    displayableArgs = _getDisplayableArguments(obj, alwaysShow, fieldNames)
+    out = ["<", obj.__class__.__name__] + displayableArgs
+
+    if setFlags:
+        out.append(" flags={}".format(",".join(setFlags)))
+
+    for name in sectionNames:
+        section = getattr(obj, name, [])
+        if section:
+            out.append(f" {name}={section!r}")
+
+    out.append(">")
+
+    return "".join(out)
+
+
+class Message(tputil.FancyEqMixin):
+    """
+    L{Message} contains all the information represented by a single
+    DNS request or response.
+
+    @ivar id: See L{__init__}
+    @ivar answer: See L{__init__}
+    @ivar opCode: See L{__init__}
+    @ivar recDes: See L{__init__}
+    @ivar recAv: See L{__init__}
+    @ivar auth: See L{__init__}
+    @ivar rCode: See L{__init__}
+    @ivar trunc: See L{__init__}
+    @ivar maxSize: See L{__init__}
+    @ivar authenticData: See L{__init__}
+    @ivar checkingDisabled: See L{__init__}
+
+    @ivar queries: The queries which are being asked of or answered by
+        DNS server.
+    @type queries: L{list} of L{Query}
+
+    @ivar answers: Records containing the answers to C{queries} if
+        this is a response message.
+    @type answers: L{list} of L{RRHeader}
+
+    @ivar authority: Records containing information about the
+        authoritative DNS servers for the names in C{queries}.
+    @type authority: L{list} of L{RRHeader}
+
+    @ivar additional: Records containing IP addresses of host names
+        in C{answers} and C{authority}.
+    @type additional: L{list} of L{RRHeader}
+
+    @ivar _flagNames: The names of attributes representing the flag header
+        fields.
+    @ivar _fieldNames: The names of attributes representing non-flag fixed
+        header fields.
+    @ivar _sectionNames: The names of attributes representing the record
+        sections of this message.
+    """
+
+    compareAttributes = (
+        "id",
+        "answer",
+        "opCode",
+        "recDes",
+        "recAv",
+        "auth",
+        "rCode",
+        "trunc",
+        "maxSize",
+        "authenticData",
+        "checkingDisabled",
+        "queries",
+        "answers",
+        "authority",
+        "additional",
+    )
+
+    headerFmt = "!H2B4H"
+    headerSize = struct.calcsize(headerFmt)
+
+    # Question, answer, additional, and nameserver lists
+    queries = answers = add = ns = None
+
+    def __init__(
+        self,
+        id=0,
+        answer=0,
+        opCode=0,
+        recDes=0,
+        recAv=0,
+        auth=0,
+        rCode=OK,
+        trunc=0,
+        maxSize=512,
+        authenticData=0,
+        checkingDisabled=0,
+    ):
+        """
+        @param id: A 16 bit identifier assigned by the program that
+            generates any kind of query.  This identifier is copied to
+            the corresponding reply and can be used by the requester
+            to match up replies to outstanding queries.
+        @type id: L{int}
+
+        @param answer: A one bit field that specifies whether this
+            message is a query (0), or a response (1).
+        @type answer: L{int}
+
+        @param opCode: A four bit field that specifies kind of query in
+            this message.  This value is set by the originator of a query
+            and copied into the response.
+        @type opCode: L{int}
+
+        @param recDes: Recursion Desired - this bit may be set in a
+            query and is copied into the response.  If RD is set, it
+            directs the name server to pursue the query recursively.
+            Recursive query support is optional.
+        @type recDes: L{int}
+
+        @param recAv: Recursion Available - this bit is set or cleared
+            in a response and denotes whether recursive query support
+            is available in the name server.
+        @type recAv: L{int}
+
+        @param auth: Authoritative Answer - this bit is valid in
+            responses and specifies that the responding name server
+            is an authority for the domain name in question section.
+        @type auth: L{int}
+
+        @ivar rCode: A response code, used to indicate success or failure in a
+            message which is a response from a server to a client request.
+        @type rCode: C{0 <= int < 16}
+
+        @param trunc: A flag indicating that this message was
+            truncated due to length greater than that permitted on the
+            transmission channel.
+        @type trunc: L{int}
+
+        @param maxSize: The requestor's UDP payload size is the number
+            of octets of the largest UDP payload that can be
+            reassembled and delivered in the requestor's network
+            stack.
+        @type maxSize: L{int}
+
+        @param authenticData: A flag indicating in a response that all
+            the data included in the answer and authority portion of
+            the response has been authenticated by the server
+            according to the policies of that server.
+            See U{RFC2535 section-6.1<https://tools.ietf.org/html/rfc2535#section-6.1>}.
+        @type authenticData: L{int}
+
+        @param checkingDisabled: A flag indicating in a query that
+            pending (non-authenticated) data is acceptable to the
+            resolver sending the query.
+            See U{RFC2535 section-6.1<https://tools.ietf.org/html/rfc2535#section-6.1>}.
+        @type authenticData: L{int}
+        """
+        self.maxSize = maxSize
+        self.id = id
+        self.answer = answer
+        self.opCode = opCode
+        self.auth = auth
+        self.trunc = trunc
+        self.recDes = recDes
+        self.recAv = recAv
+        self.rCode = rCode
+        self.authenticData = authenticData
+        self.checkingDisabled = checkingDisabled
+
+        self.queries = []
+        self.answers = []
+        self.authority = []
+        self.additional = []
+
+    def __repr__(self) -> str:
+        """
+        Generate a repr of this L{Message}.
+
+        Only includes the non-default fields and sections and only includes
+        flags which are set. The C{id} is always shown.
+
+        @return: The native string repr.
+        """
+        return _compactRepr(
+            self,
+            flagNames=(
+                "answer",
+                "auth",
+                "trunc",
+                "recDes",
+                "recAv",
+                "authenticData",
+                "checkingDisabled",
+            ),
+            fieldNames=("id", "opCode", "rCode", "maxSize"),
+            sectionNames=("queries", "answers", "authority", "additional"),
+            alwaysShow=("id",),
+        )
+
+    def addQuery(self, name, type=ALL_RECORDS, cls=IN):
+        """
+        Add another query to this Message.
+
+        @type name: L{bytes}
+        @param name: The name to query.
+
+        @type type: L{int}
+        @param type: Query type
+
+        @type cls: L{int}
+        @param cls: Query class
+        """
+        self.queries.append(Query(name, type, cls))
+
+    def encode(self, strio):
+        compDict = {}
+        body_tmp = BytesIO()
+        for q in self.queries:
+            q.encode(body_tmp, compDict)
+        for q in self.answers:
+            q.encode(body_tmp, compDict)
+        for q in self.authority:
+            q.encode(body_tmp, compDict)
+        for q in self.additional:
+            q.encode(body_tmp, compDict)
+        body = body_tmp.getvalue()
+        size = len(body) + self.headerSize
+        if self.maxSize and size > self.maxSize:
+            self.trunc = 1
+            body = body[: self.maxSize - self.headerSize]
+        byte3 = (
+            ((self.answer & 1) << 7)
+            | ((self.opCode & 0xF) << 3)
+            | ((self.auth & 1) << 2)
+            | ((self.trunc & 1) << 1)
+            | (self.recDes & 1)
+        )
+        byte4 = (
+            ((self.recAv & 1) << 7)
+            | ((self.authenticData & 1) << 5)
+            | ((self.checkingDisabled & 1) << 4)
+            | (self.rCode & 0xF)
+        )
+
+        strio.write(
+            struct.pack(
+                self.headerFmt,
+                self.id,
+                byte3,
+                byte4,
+                len(self.queries),
+                len(self.answers),
+                len(self.authority),
+                len(self.additional),
+            )
+        )
+        strio.write(body)
+
+    def decode(self, strio, length=None):
+        self.maxSize = 0
+        header = readPrecisely(strio, self.headerSize)
+        r = struct.unpack(self.headerFmt, header)
+        self.id, byte3, byte4, nqueries, nans, nns, nadd = r
+        self.answer = (byte3 >> 7) & 1
+        self.opCode = (byte3 >> 3) & 0xF
+        self.auth = (byte3 >> 2) & 1
+        self.trunc = (byte3 >> 1) & 1
+        self.recDes = byte3 & 1
+        self.recAv = (byte4 >> 7) & 1
+        self.authenticData = (byte4 >> 5) & 1
+        self.checkingDisabled = (byte4 >> 4) & 1
+        self.rCode = byte4 & 0xF
+
+        self.queries = []
+        for i in range(nqueries):
+            q = Query()
+            try:
+                q.decode(strio)
+            except EOFError:
+                return
+            self.queries.append(q)
+
+        items = ((self.answers, nans), (self.authority, nns), (self.additional, nadd))
+
+        for l, n in items:
+            self.parseRecords(l, n, strio)
+
+    def parseRecords(self, list, num, strio):
+        for i in range(num):
+            header = RRHeader(auth=self.auth)
+            try:
+                header.decode(strio)
+            except EOFError:
+                return
+            t = self.lookupRecordType(header.type)
+            if not t:
+                continue
+            header.payload = t(ttl=header.ttl)
+            try:
+                header.payload.decode(strio, header.rdlength)
+            except EOFError:
+                return
+            list.append(header)
+
+    # Create a mapping from record types to their corresponding Record_*
+    # classes.  This relies on the global state which has been created so
+    # far in initializing this module (so don't define Record classes after
+    # this).
+    _recordTypes = {}
+    for name in globals():
+        if name.startswith("Record_"):
+            _recordTypes[globals()[name].TYPE] = globals()[name]
+
+    # Clear the iteration variable out of the class namespace so it
+    # doesn't become an attribute.
+    del name
+
+    def lookupRecordType(self, type):
+        """
+        Retrieve the L{IRecord} implementation for the given record type.
+
+        @param type: A record type, such as C{A} or L{NS}.
+        @type type: L{int}
+
+        @return: An object which implements L{IRecord} or L{None} if none
+            can be found for the given type.
+        @rtype: C{Type[IRecord]}
+        """
+        return self._recordTypes.get(type, UnknownRecord)
+
+    def toStr(self):
+        """
+        Encode this L{Message} into a byte string in the format described by RFC
+        1035.
+
+        @rtype: L{bytes}
+        """
+        strio = BytesIO()
+        self.encode(strio)
+        return strio.getvalue()
+
+    def fromStr(self, str):
+        """
+        Decode a byte string in the format described by RFC 1035 into this
+        L{Message}.
+
+        @param str: L{bytes}
+        """
+        strio = BytesIO(str)
+        self.decode(strio)
+
+
+class _EDNSMessage(tputil.FancyEqMixin):
+    """
+    An I{EDNS} message.
+
+    Designed for compatibility with L{Message} but with a narrower public
+    interface.
+
+    Most importantly, L{_EDNSMessage.fromStr} will interpret and remove I{OPT}
+    records that are present in the additional records section.
+
+    The I{OPT} records are used to populate certain I{EDNS} specific attributes.
+
+    L{_EDNSMessage.toStr} will add suitable I{OPT} records to the additional
+    section to represent the extended EDNS information.
+
+    @see: U{https://tools.ietf.org/html/rfc6891}
+
+    @ivar id: See L{__init__}
+    @ivar answer: See L{__init__}
+    @ivar opCode: See L{__init__}
+    @ivar auth: See L{__init__}
+    @ivar trunc: See L{__init__}
+    @ivar recDes: See L{__init__}
+    @ivar recAv: See L{__init__}
+    @ivar rCode: See L{__init__}
+    @ivar ednsVersion: See L{__init__}
+    @ivar dnssecOK: See L{__init__}
+    @ivar authenticData: See L{__init__}
+    @ivar checkingDisabled: See L{__init__}
+    @ivar maxSize: See L{__init__}
+
+    @ivar queries: See L{__init__}
+    @ivar answers: See L{__init__}
+    @ivar authority: See L{__init__}
+    @ivar additional: See L{__init__}
+
+    @ivar _messageFactory: A constructor of L{Message} instances. Called by
+        C{_toMessage} and C{_fromMessage}.
+    """
+
+    compareAttributes = (
+        "id",
+        "answer",
+        "opCode",
+        "auth",
+        "trunc",
+        "recDes",
+        "recAv",
+        "rCode",
+        "ednsVersion",
+        "dnssecOK",
+        "authenticData",
+        "checkingDisabled",
+        "maxSize",
+        "queries",
+        "answers",
+        "authority",
+        "additional",
+    )
+
+    _messageFactory = Message
+
+    def __init__(
+        self,
+        id=0,
+        answer=False,
+        opCode=OP_QUERY,
+        auth=False,
+        trunc=False,
+        recDes=False,
+        recAv=False,
+        rCode=0,
+        ednsVersion=0,
+        dnssecOK=False,
+        authenticData=False,
+        checkingDisabled=False,
+        maxSize=512,
+        queries=None,
+        answers=None,
+        authority=None,
+        additional=None,
+    ):
+        """
+        Construct a new L{_EDNSMessage}
+
+        @see: U{RFC1035 section-4.1.1<https://tools.ietf.org/html/rfc1035#section-4.1.1>}
+        @see: U{RFC2535 section-6.1<https://tools.ietf.org/html/rfc2535#section-6.1>}
+        @see: U{RFC3225 section-3<https://tools.ietf.org/html/rfc3225#section-3>}
+        @see: U{RFC6891 section-6.1.3<https://tools.ietf.org/html/rfc6891#section-6.1.3>}
+
+        @param id: A 16 bit identifier assigned by the program that generates
+            any kind of query.  This identifier is copied the corresponding
+            reply and can be used by the requester to match up replies to
+            outstanding queries.
+        @type id: L{int}
+
+        @param answer: A one bit field that specifies whether this message is a
+            query (0), or a response (1).
+        @type answer: L{bool}
+
+        @param opCode: A four bit field that specifies kind of query in this
+            message.  This value is set by the originator of a query and copied
+            into the response.
+        @type opCode: L{int}
+
+        @param auth: Authoritative Answer - this bit is valid in responses, and
+            specifies that the responding name server is an authority for the
+            domain name in question section.
+        @type auth: L{bool}
+
+        @param trunc: Truncation - specifies that this message was truncated due
+            to length greater than that permitted on the transmission channel.
+        @type trunc: L{bool}
+
+        @param recDes: Recursion Desired - this bit may be set in a query and is
+            copied into the response.  If set, it directs the name server to
+            pursue the query recursively. Recursive query support is optional.
+        @type recDes: L{bool}
+
+        @param recAv: Recursion Available - this bit is set or cleared in a
+            response, and denotes whether recursive query support is available
+            in the name server.
+        @type recAv: L{bool}
+
+        @param rCode: Extended 12-bit RCODE. Derived from the 4 bits defined in
+            U{RFC1035 4.1.1<https://tools.ietf.org/html/rfc1035#section-4.1.1>}
+            and the upper 8bits defined in U{RFC6891
+            6.1.3<https://tools.ietf.org/html/rfc6891#section-6.1.3>}.
+        @type rCode: L{int}
+
+        @param ednsVersion: Indicates the EDNS implementation level. Set to
+            L{None} to prevent any EDNS attributes and options being added to
+            the encoded byte string.
+        @type ednsVersion: L{int} or L{None}
+
+        @param dnssecOK: DNSSEC OK bit as defined by
+            U{RFC3225 3<https://tools.ietf.org/html/rfc3225#section-3>}.
+        @type dnssecOK: L{bool}
+
+        @param authenticData: A flag indicating in a response that all the data
+            included in the answer and authority portion of the response has
+            been authenticated by the server according to the policies of that
+            server.
+            See U{RFC2535 section-6.1<https://tools.ietf.org/html/rfc2535#section-6.1>}.
+        @type authenticData: L{bool}
+
+        @param checkingDisabled: A flag indicating in a query that pending
+            (non-authenticated) data is acceptable to the resolver sending the
+            query.
+            See U{RFC2535 section-6.1<https://tools.ietf.org/html/rfc2535#section-6.1>}.
+        @type authenticData: L{bool}
+
+        @param maxSize: The requestor's UDP payload size is the number of octets
+            of the largest UDP payload that can be reassembled and delivered in
+            the requestor's network stack.
+        @type maxSize: L{int}
+
+        @param queries: The L{list} of L{Query} associated with this message.
+        @type queries: L{list} of L{Query}
+
+        @param answers: The L{list} of answers associated with this message.
+        @type answers: L{list} of L{RRHeader}
+
+        @param authority: The L{list} of authority records associated with this
+            message.
+        @type authority: L{list} of L{RRHeader}
+
+        @param additional: The L{list} of additional records associated with
+            this message.
+        @type additional: L{list} of L{RRHeader}
+        """
+        self.id = id
+        self.answer = answer
+        self.opCode = opCode
+        self.auth = auth
+        self.trunc = trunc
+        self.recDes = recDes
+        self.recAv = recAv
+        self.rCode = rCode
+        self.ednsVersion = ednsVersion
+        self.dnssecOK = dnssecOK
+        self.authenticData = authenticData
+        self.checkingDisabled = checkingDisabled
+        self.maxSize = maxSize
+
+        if queries is None:
+            queries = []
+        self.queries = queries
+
+        if answers is None:
+            answers = []
+        self.answers = answers
+
+        if authority is None:
+            authority = []
+        self.authority = authority
+
+        if additional is None:
+            additional = []
+        self.additional = additional
+
+    def __repr__(self) -> str:
+        return _compactRepr(
+            self,
+            flagNames=(
+                "answer",
+                "auth",
+                "trunc",
+                "recDes",
+                "recAv",
+                "authenticData",
+                "checkingDisabled",
+                "dnssecOK",
+            ),
+            fieldNames=("id", "opCode", "rCode", "maxSize", "ednsVersion"),
+            sectionNames=("queries", "answers", "authority", "additional"),
+            alwaysShow=("id",),
+        )
+
+    def _toMessage(self):
+        """
+        Convert to a standard L{dns.Message}.
+
+        If C{ednsVersion} is not None, an L{_OPTHeader} instance containing all
+        the I{EDNS} specific attributes and options will be appended to the list
+        of C{additional} records.
+
+        @return: A L{dns.Message}
+        @rtype: L{dns.Message}
+        """
+        m = self._messageFactory(
+            id=self.id,
+            answer=self.answer,
+            opCode=self.opCode,
+            auth=self.auth,
+            trunc=self.trunc,
+            recDes=self.recDes,
+            recAv=self.recAv,
+            # Assign the lower 4 bits to the message
+            rCode=self.rCode & 0xF,
+            authenticData=self.authenticData,
+            checkingDisabled=self.checkingDisabled,
+        )
+
+        m.queries = self.queries[:]
+        m.answers = self.answers[:]
+        m.authority = self.authority[:]
+        m.additional = self.additional[:]
+
+        if self.ednsVersion is not None:
+            o = _OPTHeader(
+                version=self.ednsVersion,
+                dnssecOK=self.dnssecOK,
+                udpPayloadSize=self.maxSize,
+                # Assign the upper 8 bits to the OPT record
+                extendedRCODE=self.rCode >> 4,
+            )
+            m.additional.append(o)
+
+        return m
+
+    def toStr(self):
+        """
+        Encode to wire format by first converting to a standard L{dns.Message}.
+
+        @return: A L{bytes} string.
+        """
+        return self._toMessage().toStr()
+
+    @classmethod
+    def _fromMessage(cls, message):
+        """
+        Construct and return a new L{_EDNSMessage} whose attributes and records
+        are derived from the attributes and records of C{message} (a L{Message}
+        instance).
+
+        If present, an C{OPT} record will be extracted from the C{additional}
+        section and its attributes and options will be used to set the EDNS
+        specific attributes C{extendedRCODE}, C{ednsVersion}, C{dnssecOK},
+        C{ednsOptions}.
+
+        The C{extendedRCODE} will be combined with C{message.rCode} and assigned
+        to C{self.rCode}.
+
+        @param message: The source L{Message}.
+        @type message: L{Message}
+
+        @return: A new L{_EDNSMessage}
+        @rtype: L{_EDNSMessage}
+        """
+        additional = []
+        optRecords = []
+        for r in message.additional:
+            if r.type == OPT:
+                optRecords.append(_OPTHeader.fromRRHeader(r))
+            else:
+                additional.append(r)
+
+        newMessage = cls(
+            id=message.id,
+            answer=message.answer,
+            opCode=message.opCode,
+            auth=message.auth,
+            trunc=message.trunc,
+            recDes=message.recDes,
+            recAv=message.recAv,
+            rCode=message.rCode,
+            authenticData=message.authenticData,
+            checkingDisabled=message.checkingDisabled,
+            # Default to None, it will be updated later when the OPT records are
+            # parsed.
+            ednsVersion=None,
+            dnssecOK=False,
+            queries=message.queries[:],
+            answers=message.answers[:],
+            authority=message.authority[:],
+            additional=additional,
+        )
+
+        if len(optRecords) == 1:
+            # XXX: If multiple OPT records are received, an EDNS server should
+            # respond with FORMERR. See ticket:5669#comment:1.
+            opt = optRecords[0]
+            newMessage.ednsVersion = opt.version
+            newMessage.dnssecOK = opt.dnssecOK
+            newMessage.maxSize = opt.udpPayloadSize
+            newMessage.rCode = opt.extendedRCODE << 4 | message.rCode
+
+        return newMessage
+
+    def fromStr(self, bytes):
+        """
+        Decode from wire format, saving flags, values and records to this
+        L{_EDNSMessage} instance in place.
+
+        @param bytes: The full byte string to be decoded.
+        @type bytes: L{bytes}
+        """
+        m = self._messageFactory()
+        m.fromStr(bytes)
+
+        ednsMessage = self._fromMessage(m)
+        for attrName in self.compareAttributes:
+            setattr(self, attrName, getattr(ednsMessage, attrName))
+
+
+class DNSMixin:
+    """
+    DNS protocol mixin shared by UDP and TCP implementations.
+
+    @ivar _reactor: A L{IReactorTime} and L{IReactorUDP} provider which will
+        be used to issue DNS queries and manage request timeouts.
+    """
+
+    id = None
+    liveMessages = None
+
+    def __init__(self, controller, reactor=None):
+        self.controller = controller
+        self.id = random.randrange(2**10, 2**15)
+        if reactor is None:
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+    def pickID(self):
+        """
+        Return a unique ID for queries.
+        """
+        while True:
+            id = randomSource()
+            if id not in self.liveMessages:
+                return id
+
+    def callLater(self, period, func, *args):
+        """
+        Wrapper around reactor.callLater, mainly for test purpose.
+        """
+        return self._reactor.callLater(period, func, *args)
+
+    def _query(self, queries, timeout, id, writeMessage):
+        """
+        Send out a message with the given queries.
+
+        @type queries: L{list} of C{Query} instances
+        @param queries: The queries to transmit
+
+        @type timeout: L{int} or C{float}
+        @param timeout: How long to wait before giving up
+
+        @type id: L{int}
+        @param id: Unique key for this request
+
+        @type writeMessage: C{callable}
+        @param writeMessage: One-parameter callback which writes the message
+
+        @rtype: C{Deferred}
+        @return: a C{Deferred} which will be fired with the result of the
+            query, or errbacked with any errors that could happen (exceptions
+            during writing of the query, timeout errors, ...).
+        """
+        m = Message(id, recDes=1)
+        m.queries = queries
+
+        try:
+            writeMessage(m)
+        except BaseException:
+            return defer.fail()
+
+        resultDeferred = defer.Deferred()
+        cancelCall = self.callLater(timeout, self._clearFailed, resultDeferred, id)
+        self.liveMessages[id] = (resultDeferred, cancelCall)
+
+        return resultDeferred
+
+    def _clearFailed(self, deferred, id):
+        """
+        Clean the Deferred after a timeout.
+        """
+        try:
+            del self.liveMessages[id]
+        except KeyError:
+            pass
+        deferred.errback(failure.Failure(DNSQueryTimeoutError(id)))
+
+
+class DNSDatagramProtocol(DNSMixin, protocol.DatagramProtocol):
+    """
+    DNS protocol over UDP.
+    """
+
+    resends = None
+
+    def stopProtocol(self):
+        """
+        Stop protocol: reset state variables.
+        """
+        self.liveMessages = {}
+        self.resends = {}
+        self.transport = None
+
+    def startProtocol(self):
+        """
+        Upon start, reset internal state.
+        """
+        self.liveMessages = {}
+        self.resends = {}
+
+    def writeMessage(self, message, address):
+        """
+        Send a message holding DNS queries.
+
+        @type message: L{Message}
+        """
+        self.transport.write(message.toStr(), address)
+
+    def startListening(self):
+        self._reactor.listenUDP(0, self, maxPacketSize=512)
+
+    def datagramReceived(self, data, addr):
+        """
+        Read a datagram, extract the message in it and trigger the associated
+        Deferred.
+        """
+        m = Message()
+        try:
+            m.fromStr(data)
+        except EOFError:
+            log.msg("Truncated packet (%d bytes) from %s" % (len(data), addr))
+            return
+        except ValueError as ex:
+            log.msg(f"Invalid packet ({ex}) from {addr}")
+            return
+        except BaseException:
+            # Nothing should trigger this, but since we're potentially
+            # invoking a lot of different decoding methods, we might as well
+            # be extra cautious.  Anything that triggers this is itself
+            # buggy.
+            log.err(failure.Failure(), "Unexpected decoding error")
+            return
+
+        if m.id in self.liveMessages:
+            d, canceller = self.liveMessages[m.id]
+            del self.liveMessages[m.id]
+            canceller.cancel()
+            # XXX we shouldn't need this hack of catching exception on callback()
+            try:
+                d.callback(m)
+            except BaseException:
+                log.err()
+        else:
+            if m.id not in self.resends:
+                self.controller.messageReceived(m, self, addr)
+
+    def removeResend(self, id):
+        """
+        Mark message ID as no longer having duplication suppression.
+        """
+        try:
+            del self.resends[id]
+        except KeyError:
+            pass
+
+    def query(self, address, queries, timeout=10, id=None):
+        """
+        Send out a message with the given queries.
+
+        @type address: L{tuple} of L{str} and L{int}
+        @param address: The address to which to send the query
+
+        @type queries: L{list} of C{Query} instances
+        @param queries: The queries to transmit
+
+        @rtype: C{Deferred}
+        """
+        if not self.transport:
+            # XXX transport might not get created automatically, use callLater?
+            try:
+                self.startListening()
+            except CannotListenError:
+                return defer.fail()
+
+        if id is None:
+            id = self.pickID()
+        else:
+            self.resends[id] = 1
+
+        def writeMessage(m):
+            self.writeMessage(m, address)
+
+        return self._query(queries, timeout, id, writeMessage)
+
+
+class DNSProtocol(DNSMixin, protocol.Protocol):
+    """
+    DNS protocol over TCP.
+    """
+
+    length = None
+    buffer = b""
+
+    def writeMessage(self, message):
+        """
+        Send a message holding DNS queries.
+
+        @type message: L{Message}
+        """
+        s = message.toStr()
+        self.transport.write(struct.pack("!H", len(s)) + s)
+
+    def connectionMade(self):
+        """
+        Connection is made: reset internal state, and notify the controller.
+        """
+        self.liveMessages = {}
+        self.controller.connectionMade(self)
+
+    def connectionLost(self, reason):
+        """
+        Notify the controller that this protocol is no longer
+        connected.
+        """
+        self.controller.connectionLost(self)
+
+    def dataReceived(self, data):
+        self.buffer += data
+
+        while self.buffer:
+            if self.length is None and len(self.buffer) >= 2:
+                self.length = struct.unpack("!H", self.buffer[:2])[0]
+                self.buffer = self.buffer[2:]
+
+            if len(self.buffer) >= self.length:
+                myChunk = self.buffer[: self.length]
+                m = Message()
+                m.fromStr(myChunk)
+
+                try:
+                    d, canceller = self.liveMessages[m.id]
+                except KeyError:
+                    self.controller.messageReceived(m, self)
+                else:
+                    del self.liveMessages[m.id]
+                    canceller.cancel()
+                    # XXX we shouldn't need this hack
+                    try:
+                        d.callback(m)
+                    except BaseException:
+                        log.err()
+
+                self.buffer = self.buffer[self.length :]
+                self.length = None
+            else:
+                break
+
+    def query(self, queries, timeout=60):
+        """
+        Send out a message with the given queries.
+
+        @type queries: L{list} of C{Query} instances
+        @param queries: The queries to transmit
+
+        @rtype: C{Deferred}
+        """
+        id = self.pickID()
+        return self._query(queries, timeout, id, self.writeMessage)
diff --git a/contrib/python/Twisted/py3/twisted/names/error.py b/contrib/python/Twisted/py3/twisted/names/error.py
new file mode 100644
index 00000000000..185c804472e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/error.py
@@ -0,0 +1,94 @@
+# -*- test-case-name: twisted.names.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Exception class definitions for Twisted Names.
+"""
+
+
+from twisted.internet.defer import TimeoutError
+
+
+class DomainError(ValueError):
+    """
+    Indicates a lookup failed because there were no records matching the given
+    C{name, class, type} triple.
+    """
+
+
+class AuthoritativeDomainError(ValueError):
+    """
+    Indicates a lookup failed for a name for which this server is authoritative
+    because there were no records matching the given C{name, class, type}
+    triple.
+    """
+
+
+class DNSQueryTimeoutError(TimeoutError):
+    """
+    Indicates a lookup failed due to a timeout.
+
+    @ivar id: The id of the message which timed out.
+    """
+
+    def __init__(self, id):
+        TimeoutError.__init__(self)
+        self.id = id
+
+
+class DNSFormatError(DomainError):
+    """
+    Indicates a query failed with a result of C{twisted.names.dns.EFORMAT}.
+    """
+
+
+class DNSServerError(DomainError):
+    """
+    Indicates a query failed with a result of C{twisted.names.dns.ESERVER}.
+    """
+
+
+class DNSNameError(DomainError):
+    """
+    Indicates a query failed with a result of C{twisted.names.dns.ENAME}.
+    """
+
+
+class DNSNotImplementedError(DomainError):
+    """
+    Indicates a query failed with a result of C{twisted.names.dns.ENOTIMP}.
+    """
+
+
+class DNSQueryRefusedError(DomainError):
+    """
+    Indicates a query failed with a result of C{twisted.names.dns.EREFUSED}.
+    """
+
+
+class DNSUnknownError(DomainError):
+    """
+    Indicates a query failed with an unknown result.
+    """
+
+
+class ResolverError(Exception):
+    """
+    Indicates a query failed because of a decision made by the local
+    resolver object.
+    """
+
+
+__all__ = [
+    "DomainError",
+    "AuthoritativeDomainError",
+    "DNSQueryTimeoutError",
+    "DNSFormatError",
+    "DNSServerError",
+    "DNSNameError",
+    "DNSNotImplementedError",
+    "DNSQueryRefusedError",
+    "DNSUnknownError",
+    "ResolverError",
+]
diff --git a/contrib/python/Twisted/py3/twisted/names/hosts.py b/contrib/python/Twisted/py3/twisted/names/hosts.py
new file mode 100644
index 00000000000..7d77aa45218
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/hosts.py
@@ -0,0 +1,151 @@
+# -*- test-case-name: twisted.names.test.test_hosts -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+hosts(5) support.
+"""
+
+
+from twisted.internet import defer
+from twisted.internet.abstract import isIPAddress, isIPv6Address
+from twisted.names import common, dns
+from twisted.python import failure
+from twisted.python.compat import nativeString
+from twisted.python.filepath import FilePath
+
+
+def searchFileForAll(hostsFile, name):
+    """
+    Search the given file, which is in hosts(5) standard format, for addresses
+    associated with a given name.
+
+    @param hostsFile: The name of the hosts(5)-format file to search.
+    @type hostsFile: L{FilePath}
+
+    @param name: The name to search for.
+    @type name: C{bytes}
+
+    @return: L{None} if the name is not found in the file, otherwise a
+        C{str} giving the address in the file associated with the name.
+    """
+    results = []
+    try:
+        lines = hostsFile.getContent().splitlines()
+    except BaseException:
+        return results
+
+    name = name.lower()
+    for line in lines:
+        idx = line.find(b"#")
+        if idx != -1:
+            line = line[:idx]
+        if not line:
+            continue
+        parts = line.split()
+
+        if name.lower() in [s.lower() for s in parts[1:]]:
+            try:
+                maybeIP = nativeString(parts[0])
+            except ValueError:  # Not ASCII.
+                continue
+            if isIPAddress(maybeIP) or isIPv6Address(maybeIP):
+                results.append(maybeIP)
+    return results
+
+
+def searchFileFor(file, name):
+    """
+    Grep given file, which is in hosts(5) standard format, for an address
+    entry with a given name.
+
+    @param file: The name of the hosts(5)-format file to search.
+    @type file: C{str} or C{bytes}
+
+    @param name: The name to search for.
+    @type name: C{bytes}
+
+    @return: L{None} if the name is not found in the file, otherwise a
+        C{str} giving the first address in the file associated with
+        the name.
+    """
+    addresses = searchFileForAll(FilePath(file), name)
+    if addresses:
+        return addresses[0]
+    return None
+
+
+class Resolver(common.ResolverBase):
+    """
+    A resolver that services hosts(5) format files.
+    """
+
+    def __init__(self, file=b"/etc/hosts", ttl=60 * 60):
+        common.ResolverBase.__init__(self)
+        self.file = file
+        self.ttl = ttl
+
+    def _aRecords(self, name):
+        """
+        Return a tuple of L{dns.RRHeader} instances for all of the IPv4
+        addresses in the hosts file.
+        """
+        return tuple(
+            dns.RRHeader(name, dns.A, dns.IN, self.ttl, dns.Record_A(addr, self.ttl))
+            for addr in searchFileForAll(FilePath(self.file), name)
+            if isIPAddress(addr)
+        )
+
+    def _aaaaRecords(self, name):
+        """
+        Return a tuple of L{dns.RRHeader} instances for all of the IPv6
+        addresses in the hosts file.
+        """
+        return tuple(
+            dns.RRHeader(
+                name, dns.AAAA, dns.IN, self.ttl, dns.Record_AAAA(addr, self.ttl)
+            )
+            for addr in searchFileForAll(FilePath(self.file), name)
+            if isIPv6Address(addr)
+        )
+
+    def _respond(self, name, records):
+        """
+        Generate a response for the given name containing the given result
+        records, or a failure if there are no result records.
+
+        @param name: The DNS name the response is for.
+        @type name: C{str}
+
+        @param records: A tuple of L{dns.RRHeader} instances giving the results
+            that will go into the response.
+
+        @return: A L{Deferred} which will fire with a three-tuple of result
+            records, authority records, and additional records, or which will
+            fail with L{dns.DomainError} if there are no result records.
+        """
+        if records:
+            return defer.succeed((records, (), ()))
+        return defer.fail(failure.Failure(dns.DomainError(name)))
+
+    def lookupAddress(self, name, timeout=None):
+        """
+        Read any IPv4 addresses from C{self.file} and return them as
+        L{Record_A} instances.
+        """
+        name = dns.domainString(name)
+        return self._respond(name, self._aRecords(name))
+
+    def lookupIPV6Address(self, name, timeout=None):
+        """
+        Read any IPv6 addresses from C{self.file} and return them as
+        L{Record_AAAA} instances.
+        """
+        name = dns.domainString(name)
+        return self._respond(name, self._aaaaRecords(name))
+
+    # Someday this should include IPv6 addresses too, but that will cause
+    # problems if users of the API (mainly via getHostByName) aren't updated to
+    # know about IPv6 first.
+    # FIXME - getHostByName knows about IPv6 now.
+    lookupAllRecords = lookupAddress
diff --git a/contrib/python/Twisted/py3/twisted/names/newsfragments/.gitignore b/contrib/python/Twisted/py3/twisted/names/newsfragments/.gitignore
new file mode 100644
index 00000000000..f935021a8f8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/newsfragments/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/contrib/python/Twisted/py3/twisted/names/resolve.py b/contrib/python/Twisted/py3/twisted/names/resolve.py
new file mode 100644
index 00000000000..af4f40fea9a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/resolve.py
@@ -0,0 +1,91 @@
+# -*- test-case-name: twisted.names.test.test_resolve -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Lookup a name using multiple resolvers.
+
+Future Plans: This needs someway to specify which resolver answered
+the query, or someway to specify (authority|ttl|cache behavior|more?)
+"""
+
+
+from zope.interface import implementer
+
+from twisted.internet import defer, interfaces
+from twisted.names import common, dns, error
+
+
+class FailureHandler:
+    def __init__(self, resolver, query, timeout):
+        self.resolver = resolver
+        self.query = query
+        self.timeout = timeout
+
+    def __call__(self, failure):
+        # AuthoritativeDomainErrors should halt resolution attempts
+        failure.trap(dns.DomainError, defer.TimeoutError, NotImplementedError)
+        return self.resolver(self.query, self.timeout)
+
+
+@implementer(interfaces.IResolver)
+class ResolverChain(common.ResolverBase):
+    """
+    Lookup an address using multiple L{IResolver}s
+    """
+
+    def __init__(self, resolvers):
+        """
+        @type resolvers: L{list}
+        @param resolvers: A L{list} of L{IResolver} providers.
+        """
+        common.ResolverBase.__init__(self)
+        self.resolvers = resolvers
+
+    def _lookup(self, name, cls, type, timeout):
+        """
+        Build a L{dns.Query} for the given parameters and dispatch it
+        to each L{IResolver} in C{self.resolvers} until an answer or
+        L{error.AuthoritativeDomainError} is returned.
+
+        @type name: C{str}
+        @param name: DNS name to resolve.
+
+        @type type: C{int}
+        @param type: DNS record type.
+
+        @type cls: C{int}
+        @param cls: DNS record class.
+
+        @type timeout: Sequence of C{int}
+        @param timeout: Number of seconds after which to reissue the query.
+            When the last timeout expires, the query is considered failed.
+
+        @rtype: L{Deferred}
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} instances.  The first element of the
+            tuple gives answers.  The second element of the tuple gives
+            authorities.  The third element of the tuple gives additional
+            information.  The L{Deferred} may instead fail with one of the
+            exceptions defined in L{twisted.names.error} or with
+            C{NotImplementedError}.
+        """
+        if not self.resolvers:
+            return defer.fail(error.DomainError())
+        q = dns.Query(name, type, cls)
+        d = self.resolvers[0].query(q, timeout)
+        for r in self.resolvers[1:]:
+            d = d.addErrback(FailureHandler(r.query, q, timeout))
+        return d
+
+    def lookupAllRecords(self, name, timeout=None):
+        # XXX: Why is this necessary? dns.ALL_RECORDS queries should
+        # be handled just the same as any other type by _lookup
+        # above. If I remove this method all names tests still
+        # pass. See #6604 -rwall
+        if not self.resolvers:
+            return defer.fail(error.DomainError())
+        d = self.resolvers[0].lookupAllRecords(name, timeout)
+        for r in self.resolvers[1:]:
+            d = d.addErrback(FailureHandler(r.lookupAllRecords, name, timeout))
+        return d
diff --git a/contrib/python/Twisted/py3/twisted/names/root.py b/contrib/python/Twisted/py3/twisted/names/root.py
new file mode 100644
index 00000000000..3531dbfede7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/root.py
@@ -0,0 +1,331 @@
+# -*- test-case-name: twisted.names.test.test_rootresolve -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Resolver implementation for querying successive authoritative servers to
+lookup a record, starting from the root nameservers.
+
+@author: Jp Calderone
+
+todo::
+    robustify it
+    documentation
+"""
+
+from twisted.internet import defer
+from twisted.names import common, dns, error
+from twisted.python.failure import Failure
+
+
+class _DummyController:
+    """
+    A do-nothing DNS controller.  This is useful when all messages received
+    will be responses to previously issued queries.  Anything else received
+    will be ignored.
+    """
+
+    def messageReceived(self, *args):
+        pass
+
+
+class Resolver(common.ResolverBase):
+    """
+    L{Resolver} implements recursive lookup starting from a specified list of
+    root servers.
+
+    @ivar hints: See C{hints} parameter of L{__init__}
+    @ivar _maximumQueries: See C{maximumQueries} parameter of L{__init__}
+    @ivar _reactor: See C{reactor} parameter of L{__init__}
+    @ivar _resolverFactory: See C{resolverFactory} parameter of L{__init__}
+    """
+
+    def __init__(self, hints, maximumQueries=10, reactor=None, resolverFactory=None):
+        """
+        @param hints: A L{list} of L{str} giving the dotted quad
+            representation of IP addresses of root servers at which to
+            begin resolving names.
+        @type hints: L{list} of L{str}
+
+        @param maximumQueries: An optional L{int} giving the maximum
+             number of queries which will be attempted to resolve a
+             single name.
+        @type maximumQueries: L{int}
+
+        @param reactor: An optional L{IReactorTime} and L{IReactorUDP}
+             provider to use to bind UDP ports and manage timeouts.
+        @type reactor: L{IReactorTime} and L{IReactorUDP} provider
+
+        @param resolverFactory: An optional callable which accepts C{reactor}
+             and C{servers} arguments and returns an instance that provides a
+             C{queryUDP} method. Defaults to L{twisted.names.client.Resolver}.
+        @type resolverFactory: callable
+        """
+        common.ResolverBase.__init__(self)
+        self.hints = hints
+        self._maximumQueries = maximumQueries
+        self._reactor = reactor
+        if resolverFactory is None:
+            from twisted.names.client import Resolver as resolverFactory
+        self._resolverFactory = resolverFactory
+
+    def _roots(self):
+        """
+        Return a list of two-tuples representing the addresses of the root
+        servers, as defined by C{self.hints}.
+        """
+        return [(ip, dns.PORT) for ip in self.hints]
+
+    def _query(self, query, servers, timeout, filter):
+        """
+        Issue one query and return a L{Deferred} which fires with its response.
+
+        @param query: The query to issue.
+        @type query: L{dns.Query}
+
+        @param servers: The servers which might have an answer for this
+            query.
+        @type servers: L{list} of L{tuple} of L{str} and L{int}
+
+        @param timeout: A timeout on how long to wait for the response.
+        @type timeout: L{tuple} of L{int}
+
+        @param filter: A flag indicating whether to filter the results.  If
+            C{True}, the returned L{Deferred} will fire with a three-tuple of
+            lists of L{twisted.names.dns.RRHeader} (like the return value of
+            the I{lookup*} methods of L{IResolver}.  IF C{False}, the result
+            will be a L{Message} instance.
+        @type filter: L{bool}
+
+        @return: A L{Deferred} which fires with the response or a timeout
+            error.
+        @rtype: L{Deferred}
+        """
+        r = self._resolverFactory(servers=servers, reactor=self._reactor)
+        d = r.queryUDP([query], timeout)
+        if filter:
+            d.addCallback(r.filterAnswers)
+        return d
+
+    def _lookup(self, name, cls, type, timeout):
+        """
+        Implement name lookup by recursively discovering the authoritative
+        server for the name and then asking it, starting at one of the servers
+        in C{self.hints}.
+        """
+        if timeout is None:
+            # A series of timeouts for semi-exponential backoff, summing to an
+            # arbitrary total of 60 seconds.
+            timeout = (1, 3, 11, 45)
+        return self._discoverAuthority(
+            dns.Query(name, type, cls), self._roots(), timeout, self._maximumQueries
+        )
+
+    def _discoverAuthority(self, query, servers, timeout, queriesLeft):
+        """
+        Issue a query to a server and follow a delegation if necessary.
+
+        @param query: The query to issue.
+        @type query: L{dns.Query}
+
+        @param servers: The servers which might have an answer for this
+            query.
+        @type servers: L{list} of L{tuple} of L{str} and L{int}
+
+        @param timeout: A C{tuple} of C{int} giving the timeout to use for this
+            query.
+
+        @param queriesLeft: A C{int} giving the number of queries which may
+            yet be attempted to answer this query before the attempt will be
+            abandoned.
+
+        @return: A L{Deferred} which fires with a three-tuple of lists of
+            L{twisted.names.dns.RRHeader} giving the response, or with a
+            L{Failure} if there is a timeout or response error.
+        """
+        # Stop now if we've hit the query limit.
+        if queriesLeft <= 0:
+            return Failure(error.ResolverError("Query limit reached without result"))
+
+        d = self._query(query, servers, timeout, False)
+        d.addCallback(self._discoveredAuthority, query, timeout, queriesLeft - 1)
+        return d
+
+    def _discoveredAuthority(self, response, query, timeout, queriesLeft):
+        """
+        Interpret the response to a query, checking for error codes and
+        following delegations if necessary.
+
+        @param response: The L{Message} received in response to issuing C{query}.
+        @type response: L{Message}
+
+        @param query: The L{dns.Query} which was issued.
+        @type query: L{dns.Query}.
+
+        @param timeout: The timeout to use if another query is indicated by
+            this response.
+        @type timeout: L{tuple} of L{int}
+
+        @param queriesLeft: A C{int} giving the number of queries which may
+            yet be attempted to answer this query before the attempt will be
+            abandoned.
+
+        @return: A L{Failure} indicating a response error, a three-tuple of
+            lists of L{twisted.names.dns.RRHeader} giving the response to
+            C{query} or a L{Deferred} which will fire with one of those.
+        """
+        if response.rCode != dns.OK:
+            return Failure(self.exceptionForCode(response.rCode)(response))
+
+        # Turn the answers into a structure that's a little easier to work with.
+        records = {}
+        for answer in response.answers:
+            records.setdefault(answer.name, []).append(answer)
+
+        def findAnswerOrCName(name, type, cls):
+            cname = None
+            for record in records.get(name, []):
+                if record.cls == cls:
+                    if record.type == type:
+                        return record
+                    elif record.type == dns.CNAME:
+                        cname = record
+            # If there were any CNAME records, return the last one.  There's
+            # only supposed to be zero or one, though.
+            return cname
+
+        seen = set()
+        name = query.name
+        record = None
+        while True:
+            seen.add(name)
+            previous = record
+            record = findAnswerOrCName(name, query.type, query.cls)
+            if record is None:
+                if name == query.name:
+                    # If there's no answer for the original name, then this may
+                    # be a delegation.  Code below handles it.
+                    break
+                else:
+                    # Try to resolve the CNAME with another query.
+                    d = self._discoverAuthority(
+                        dns.Query(str(name), query.type, query.cls),
+                        self._roots(),
+                        timeout,
+                        queriesLeft,
+                    )
+                    # We also want to include the CNAME in the ultimate result,
+                    # otherwise this will be pretty confusing.
+
+                    def cbResolved(results):
+                        answers, authority, additional = results
+                        answers.insert(0, previous)
+                        return (answers, authority, additional)
+
+                    d.addCallback(cbResolved)
+                    return d
+            elif record.type == query.type:
+                return (response.answers, response.authority, response.additional)
+            else:
+                # It's a CNAME record.  Try to resolve it from the records
+                # in this response with another iteration around the loop.
+                if record.payload.name in seen:
+                    raise error.ResolverError("Cycle in CNAME processing")
+                name = record.payload.name
+
+        # Build a map to use to convert NS names into IP addresses.
+        addresses = {}
+        for rr in response.additional:
+            if rr.type == dns.A:
+                addresses[rr.name.name] = rr.payload.dottedQuad()
+
+        hints = []
+        traps = []
+        for rr in response.authority:
+            if rr.type == dns.NS:
+                ns = rr.payload.name.name
+                if ns in addresses:
+                    hints.append((addresses[ns], dns.PORT))
+                else:
+                    traps.append(ns)
+        if hints:
+            return self._discoverAuthority(query, hints, timeout, queriesLeft)
+        elif traps:
+            d = self.lookupAddress(traps[0], timeout)
+
+            def getOneAddress(results):
+                answers, authority, additional = results
+                return answers[0].payload.dottedQuad()
+
+            d.addCallback(getOneAddress)
+            d.addCallback(
+                lambda hint: self._discoverAuthority(
+                    query, [(hint, dns.PORT)], timeout, queriesLeft - 1
+                )
+            )
+            return d
+        else:
+            return Failure(
+                error.ResolverError("Stuck at response without answers or delegation")
+            )
+
+
+def makePlaceholder(deferred, name):
+    def placeholder(*args, **kw):
+        deferred.addCallback(lambda r: getattr(r, name)(*args, **kw))
+        return deferred
+
+    return placeholder
+
+
+class DeferredResolver:
+    def __init__(self, resolverDeferred):
+        self.waiting = []
+        resolverDeferred.addCallback(self.gotRealResolver)
+
+    def gotRealResolver(self, resolver):
+        w = self.waiting
+        self.__dict__ = resolver.__dict__
+        self.__class__ = resolver.__class__
+        for d in w:
+            d.callback(resolver)
+
+    def __getattr__(self, name):
+        if name.startswith("lookup") or name in ("getHostByName", "query"):
+            self.waiting.append(defer.Deferred())
+            return makePlaceholder(self.waiting[-1], name)
+        raise AttributeError(name)
+
+
+def bootstrap(resolver, resolverFactory=None):
+    """
+    Lookup the root nameserver addresses using the given resolver
+
+    Return a Resolver which will eventually become a C{root.Resolver}
+    instance that has references to all the root servers that we were able
+    to look up.
+
+    @param resolver: The resolver instance which will be used to
+        lookup the root nameserver addresses.
+    @type resolver: L{twisted.internet.interfaces.IResolverSimple}
+
+    @param resolverFactory: An optional callable which returns a
+        resolver instance. It will passed as the C{resolverFactory}
+        argument to L{Resolver.__init__}.
+    @type resolverFactory: callable
+
+    @return: A L{DeferredResolver} which will be dynamically replaced
+        with L{Resolver} when the root nameservers have been looked up.
+    """
+    domains = [chr(ord("a") + i) for i in range(13)]
+    L = [resolver.getHostByName("%s.root-servers.net" % d) for d in domains]
+    d = defer.DeferredList(L, consumeErrors=True)
+
+    def buildResolver(res):
+        return Resolver(
+            hints=[e[1] for e in res if e[0]], resolverFactory=resolverFactory
+        )
+
+    d.addCallback(buildResolver)
+
+    return DeferredResolver(d)
diff --git a/contrib/python/Twisted/py3/twisted/names/secondary.py b/contrib/python/Twisted/py3/twisted/names/secondary.py
new file mode 100644
index 00000000000..0b9e184b026
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/secondary.py
@@ -0,0 +1,216 @@
+# -*- test-case-name: twisted.names.test.test_names -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+__all__ = ["SecondaryAuthority", "SecondaryAuthorityService"]
+
+from twisted.application import service
+from twisted.internet import defer, task
+from twisted.names import client, common, dns, resolve
+from twisted.names.authority import FileAuthority
+from twisted.python import failure, log
+from twisted.python.compat import nativeString
+
+
+class SecondaryAuthorityService(service.Service):
+    """
+    A service that keeps one or more authorities up to date by doing hourly
+    zone transfers from a master.
+
+    @ivar primary: IP address of the master.
+    @type primary: L{str}
+
+    @ivar domains: An authority for each domain mirrored from the master.
+    @type domains: L{list} of L{SecondaryAuthority}
+    """
+
+    calls = None
+
+    _port = 53
+
+    def __init__(self, primary, domains):
+        """
+        @param primary: The IP address of the server from which to perform
+        zone transfers.
+        @type primary: L{str}
+
+        @param domains: A sequence of domain names for which to perform
+        zone transfers.
+        @type domains: L{list} of L{bytes}
+        """
+        self.primary = nativeString(primary)
+        self.domains = [SecondaryAuthority(primary, d) for d in domains]
+
+    @classmethod
+    def fromServerAddressAndDomains(cls, serverAddress, domains):
+        """
+        Construct a new L{SecondaryAuthorityService} from a tuple giving a
+        server address and a C{str} giving the name of a domain for which this
+        is an authority.
+
+        @param serverAddress: A two-tuple, the first element of which is a
+            C{str} giving an IP address and the second element of which is a
+            C{int} giving a port number.  Together, these define where zone
+            transfers will be attempted from.
+
+        @param domains: Domain names for which to perform zone transfers.
+        @type domains: sequence of L{bytes}
+
+        @return: A new instance of L{SecondaryAuthorityService}.
+        """
+        primary, port = serverAddress
+        service = cls(primary, [])
+        service._port = port
+        service.domains = [
+            SecondaryAuthority.fromServerAddressAndDomain(serverAddress, d)
+            for d in domains
+        ]
+        return service
+
+    def getAuthority(self):
+        """
+        Get a resolver for the transferred domains.
+
+        @rtype: L{ResolverChain}
+        """
+        return resolve.ResolverChain(self.domains)
+
+    def startService(self):
+        service.Service.startService(self)
+        self.calls = [task.LoopingCall(d.transfer) for d in self.domains]
+        i = 0
+        from twisted.internet import reactor
+
+        for c in self.calls:
+            # XXX Add errbacks, respect proper timeouts
+            reactor.callLater(i, c.start, 60 * 60)
+            i += 1
+
+    def stopService(self):
+        service.Service.stopService(self)
+        for c in self.calls:
+            c.stop()
+
+
+class SecondaryAuthority(FileAuthority):
+    """
+    An Authority that keeps itself updated by performing zone transfers.
+
+    @ivar primary: The IP address of the server from which zone transfers will
+    be attempted.
+    @type primary: L{str}
+
+    @ivar _port: The port number of the server from which zone transfers will
+    be attempted.
+    @type _port: L{int}
+
+    @ivar domain: The domain for which this is the secondary authority.
+    @type domain: L{bytes}
+
+    @ivar _reactor: The reactor to use to perform the zone transfers, or
+    L{None} to use the global reactor.
+    """
+
+    transferring = False
+    soa = records = None
+    _port = 53
+    _reactor = None
+
+    def __init__(self, primaryIP, domain):
+        """
+        @param domain: The domain for which this will be the secondary
+            authority.
+        @type domain: L{bytes} or L{str}
+        """
+        # Yep.  Skip over FileAuthority.__init__.  This is a hack until we have
+        # a good composition-based API for the complicated DNS record lookup
+        # logic we want to share.
+        common.ResolverBase.__init__(self)
+        self.primary = nativeString(primaryIP)
+        self.domain = dns.domainString(domain)
+
+    @classmethod
+    def fromServerAddressAndDomain(cls, serverAddress, domain):
+        """
+        Construct a new L{SecondaryAuthority} from a tuple giving a server
+        address and a C{bytes} giving the name of a domain for which this is an
+        authority.
+
+        @param serverAddress: A two-tuple, the first element of which is a
+            C{str} giving an IP address and the second element of which is a
+            C{int} giving a port number.  Together, these define where zone
+            transfers will be attempted from.
+
+        @param domain: A C{bytes} giving the domain to transfer.
+        @type domain: L{bytes}
+
+        @return: A new instance of L{SecondaryAuthority}.
+        """
+        primary, port = serverAddress
+        secondary = cls(primary, domain)
+        secondary._port = port
+        return secondary
+
+    def transfer(self):
+        """
+        Attempt a zone transfer.
+
+        @returns: A L{Deferred} that fires with L{None} when attempted zone
+            transfer has completed.
+        """
+        # FIXME: This logic doesn't avoid duplicate transfers
+        # https://twistedmatrix.com/trac/ticket/9754
+        if self.transferring:  # <-- never true
+            return
+        self.transfering = True  # <-- speling
+
+        reactor = self._reactor
+        if reactor is None:
+            from twisted.internet import reactor
+
+        resolver = client.Resolver(
+            servers=[(self.primary, self._port)], reactor=reactor
+        )
+        return (
+            resolver.lookupZone(self.domain)
+            .addCallback(self._cbZone)
+            .addErrback(self._ebZone)
+        )
+
+    def _lookup(self, name, cls, type, timeout=None):
+        if not self.soa or not self.records:
+            # No transfer has occurred yet. Fail non-authoritatively so that
+            # the caller can try elsewhere.
+            return defer.fail(failure.Failure(dns.DomainError(name)))
+        return FileAuthority._lookup(self, name, cls, type, timeout)
+
+    def _cbZone(self, zone):
+        ans, _, _ = zone
+        self.records = r = {}
+        for rec in ans:
+            if not self.soa and rec.type == dns.SOA:
+                self.soa = (rec.name.name.lower(), rec.payload)
+            else:
+                r.setdefault(rec.name.name.lower(), []).append(rec.payload)
+
+    def _ebZone(self, failure):
+        log.msg(
+            "Updating %s from %s failed during zone transfer"
+            % (self.domain, self.primary)
+        )
+        log.err(failure)
+
+    def update(self):
+        self.transfer().addCallbacks(self._cbTransferred, self._ebTransferred)
+
+    def _cbTransferred(self, result):
+        self.transferring = False
+
+    def _ebTransferred(self, failure):
+        self.transferred = False
+        log.msg(
+            "Transferring %s from %s failed after zone transfer"
+            % (self.domain, self.primary)
+        )
+        log.err(failure)
diff --git a/contrib/python/Twisted/py3/twisted/names/server.py b/contrib/python/Twisted/py3/twisted/names/server.py
new file mode 100644
index 00000000000..63fff7a2778
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/server.py
@@ -0,0 +1,569 @@
+# -*- test-case-name: twisted.names.test.test_names,twisted.names.test.test_server -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Async DNS server
+
+Future plans:
+    - Better config file format maybe
+    - Make sure to differentiate between different classes
+    - notice truncation bit
+
+Important: No additional processing is done on some of the record types.
+This violates the most basic RFC and is just plain annoying
+for resolvers to deal with.  Fix it.
+
+@author: Jp Calderone
+"""
+
+import time
+
+from twisted.internet import protocol
+from twisted.names import dns, resolve
+from twisted.python import log
+
+
+class DNSServerFactory(protocol.ServerFactory):
+    """
+    Server factory and tracker for L{DNSProtocol} connections.  This class also
+    provides records for responses to DNS queries.
+
+    @ivar cache: A L{Cache<twisted.names.cache.CacheResolver>} instance whose
+        C{cacheResult} method is called when a response is received from one of
+        C{clients}. Defaults to L{None} if no caches are specified. See
+        C{caches} of L{__init__} for more details.
+    @type cache: L{Cache<twisted.names.cache.CacheResolver>} or L{None}
+
+    @ivar canRecurse: A flag indicating whether this server is capable of
+        performing recursive DNS resolution.
+    @type canRecurse: L{bool}
+
+    @ivar resolver: A L{resolve.ResolverChain} containing an ordered list of
+        C{authorities}, C{caches} and C{clients} to which queries will be
+        dispatched.
+    @type resolver: L{resolve.ResolverChain}
+
+    @ivar verbose: See L{__init__}
+
+    @ivar connections: A list of all the connected L{DNSProtocol} instances
+        using this object as their controller.
+    @type connections: C{list} of L{DNSProtocol} instances
+
+    @ivar protocol: A callable used for building a DNS stream protocol. Called
+        by L{DNSServerFactory.buildProtocol} and passed the L{DNSServerFactory}
+        instance as the one and only positional argument.  Defaults to
+        L{dns.DNSProtocol}.
+    @type protocol: L{IProtocolFactory} constructor
+
+    @ivar _messageFactory: A response message constructor with an initializer
+         signature matching L{dns.Message.__init__}.
+    @type _messageFactory: C{callable}
+    """
+
+    # Type is wrong.  See: https://twistedmatrix.com/trac/ticket/10004#ticket
+    protocol = dns.DNSProtocol  # type: ignore[assignment]
+    cache = None
+    _messageFactory = dns.Message
+
+    def __init__(self, authorities=None, caches=None, clients=None, verbose=0):
+        """
+        @param authorities: Resolvers which provide authoritative answers.
+        @type authorities: L{list} of L{IResolver} providers
+
+        @param caches: Resolvers which provide cached non-authoritative
+            answers. The first cache instance is assigned to
+            C{DNSServerFactory.cache} and its C{cacheResult} method will be
+            called when a response is received from one of C{clients}.
+        @type caches: L{list} of L{Cache<twisted.names.cache.CacheResolver>} instances
+
+        @param clients: Resolvers which are capable of performing recursive DNS
+            lookups.
+        @type clients: L{list} of L{IResolver} providers
+
+        @param verbose: An integer controlling the verbosity of logging of
+            queries and responses. Default is C{0} which means no logging. Set
+            to C{2} to enable logging of full query and response messages.
+        @type verbose: L{int}
+        """
+        resolvers = []
+        if authorities is not None:
+            resolvers.extend(authorities)
+        if caches is not None:
+            resolvers.extend(caches)
+        if clients is not None:
+            resolvers.extend(clients)
+
+        self.canRecurse = not not clients
+        self.resolver = resolve.ResolverChain(resolvers)
+        self.verbose = verbose
+        if caches:
+            self.cache = caches[-1]
+        self.connections = []
+
+    def _verboseLog(self, *args, **kwargs):
+        """
+        Log a message only if verbose logging is enabled.
+
+        @param args: Positional arguments which will be passed to C{log.msg}
+        @param kwargs: Keyword arguments which will be passed to C{log.msg}
+        """
+        if self.verbose > 0:
+            log.msg(*args, **kwargs)
+
+    def buildProtocol(self, addr):
+        p = self.protocol(self)
+        p.factory = self
+        return p
+
+    def connectionMade(self, protocol):
+        """
+        Track a newly connected L{DNSProtocol}.
+
+        @param protocol: The protocol instance to be tracked.
+        @type protocol: L{dns.DNSProtocol}
+        """
+        self.connections.append(protocol)
+
+    def connectionLost(self, protocol):
+        """
+        Stop tracking a no-longer connected L{DNSProtocol}.
+
+        @param protocol: The tracked protocol instance to be which has been
+            lost.
+        @type protocol: L{dns.DNSProtocol}
+        """
+        self.connections.remove(protocol)
+
+    def sendReply(self, protocol, message, address):
+        """
+        Send a response C{message} to a given C{address} via the supplied
+        C{protocol}.
+
+        Message payload will be logged if C{DNSServerFactory.verbose} is C{>1}.
+
+        @param protocol: The DNS protocol instance to which to send the message.
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param message: The DNS message to be sent.
+        @type message: L{dns.Message}
+
+        @param address: The address to which the message will be sent or L{None}
+            if C{protocol} is a stream protocol.
+        @type address: L{tuple} or L{None}
+        """
+        if self.verbose > 1:
+            s = " ".join([str(a.payload) for a in message.answers])
+            auth = " ".join([str(a.payload) for a in message.authority])
+            add = " ".join([str(a.payload) for a in message.additional])
+            if not s:
+                log.msg("Replying with no answers")
+            else:
+                log.msg("Answers are " + s)
+                log.msg("Authority is " + auth)
+                log.msg("Additional is " + add)
+
+        if address is None:
+            protocol.writeMessage(message)
+        else:
+            protocol.writeMessage(message, address)
+
+        self._verboseLog(
+            "Processed query in %0.3f seconds" % (time.time() - message.timeReceived)
+        )
+
+    def _responseFromMessage(
+        self, message, rCode=dns.OK, answers=None, authority=None, additional=None
+    ):
+        """
+        Generate a L{Message} instance suitable for use as the response to
+        C{message}.
+
+        C{queries} will be copied from the request to the response.
+
+        C{rCode}, C{answers}, C{authority} and C{additional} will be assigned to
+        the response, if supplied.
+
+        The C{recAv} flag will be set on the response if the C{canRecurse} flag
+        on this L{DNSServerFactory} is set to L{True}.
+
+        The C{auth} flag will be set on the response if *any* of the supplied
+        C{answers} have their C{auth} flag set to L{True}.
+
+        The response will have the same C{maxSize} as the request.
+
+        Additionally, the response will have a C{timeReceived} attribute whose
+        value is that of the original request and the
+
+        @see: L{dns._responseFromMessage}
+
+        @param message: The request message
+        @type message: L{Message}
+
+        @param rCode: The response code which will be assigned to the response.
+        @type message: L{int}
+
+        @param answers: An optional list of answer records which will be
+            assigned to the response.
+        @type answers: L{list} of L{dns.RRHeader}
+
+        @param authority: An optional list of authority records which will be
+            assigned to the response.
+        @type authority: L{list} of L{dns.RRHeader}
+
+        @param additional: An optional list of additional records which will be
+            assigned to the response.
+        @type additional: L{list} of L{dns.RRHeader}
+
+        @return: A response L{Message} instance.
+        @rtype: L{Message}
+        """
+        if answers is None:
+            answers = []
+        if authority is None:
+            authority = []
+        if additional is None:
+            additional = []
+        authoritativeAnswer = False
+        for x in answers:
+            if x.isAuthoritative():
+                authoritativeAnswer = True
+                break
+
+        response = dns._responseFromMessage(
+            responseConstructor=self._messageFactory,
+            message=message,
+            recAv=self.canRecurse,
+            rCode=rCode,
+            auth=authoritativeAnswer,
+        )
+
+        # XXX: Timereceived is a hack which probably shouldn't be tacked onto
+        # the message. Use getattr here so that we don't have to set the
+        # timereceived on every message in the tests. See #6957.
+        response.timeReceived = getattr(message, "timeReceived", None)
+
+        # XXX: This is another hack. dns.Message.decode sets maxSize=0 which
+        # means that responses are never truncated. I'll maintain that behaviour
+        # here until #6949 is resolved.
+        response.maxSize = message.maxSize
+
+        response.answers = answers
+        response.authority = authority
+        response.additional = additional
+
+        return response
+
+    def gotResolverResponse(self, response, protocol, message, address):
+        """
+        A callback used by L{DNSServerFactory.handleQuery} for handling the
+        deferred response from C{self.resolver.query}.
+
+        Constructs a response message by combining the original query message
+        with the resolved answer, authority and additional records.
+
+        Marks the response message as authoritative if any of the resolved
+        answers are found to be authoritative.
+
+        The resolved answers count will be logged if C{DNSServerFactory.verbose}
+        is C{>1}.
+
+        @param response: Answer records, authority records and additional records
+        @type response: L{tuple} of L{list} of L{dns.RRHeader} instances
+
+        @param protocol: The DNS protocol instance to which to send a response
+            message.
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param message: The original DNS query message for which a response
+            message will be constructed.
+        @type message: L{dns.Message}
+
+        @param address: The address to which the response message will be sent
+            or L{None} if C{protocol} is a stream protocol.
+        @type address: L{tuple} or L{None}
+        """
+        ans, auth, add = response
+        response = self._responseFromMessage(
+            message=message, rCode=dns.OK, answers=ans, authority=auth, additional=add
+        )
+        self.sendReply(protocol, response, address)
+
+        l = len(ans) + len(auth) + len(add)
+        self._verboseLog("Lookup found %d record%s" % (l, l != 1 and "s" or ""))
+
+        if self.cache and l:
+            self.cache.cacheResult(message.queries[0], (ans, auth, add))
+
+    def gotResolverError(self, failure, protocol, message, address):
+        """
+        A callback used by L{DNSServerFactory.handleQuery} for handling deferred
+        errors from C{self.resolver.query}.
+
+        Constructs a response message from the original query message by
+        assigning a suitable error code to C{rCode}.
+
+        An error message will be logged if C{DNSServerFactory.verbose} is C{>1}.
+
+        @param failure: The reason for the failed resolution (as reported by
+            C{self.resolver.query}).
+        @type failure: L{Failure<twisted.python.failure.Failure>}
+
+        @param protocol: The DNS protocol instance to which to send a response
+            message.
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param message: The original DNS query message for which a response
+            message will be constructed.
+        @type message: L{dns.Message}
+
+        @param address: The address to which the response message will be sent
+            or L{None} if C{protocol} is a stream protocol.
+        @type address: L{tuple} or L{None}
+        """
+        if failure.check(dns.DomainError, dns.AuthoritativeDomainError):
+            rCode = dns.ENAME
+        else:
+            rCode = dns.ESERVER
+            log.err(failure)
+
+        response = self._responseFromMessage(message=message, rCode=rCode)
+
+        self.sendReply(protocol, response, address)
+        self._verboseLog("Lookup failed")
+
+    def handleQuery(self, message, protocol, address):
+        """
+        Called by L{DNSServerFactory.messageReceived} when a query message is
+        received.
+
+        Takes the first query from the received message and dispatches it to
+        C{self.resolver.query}.
+
+        Adds callbacks L{DNSServerFactory.gotResolverResponse} and
+        L{DNSServerFactory.gotResolverError} to the resulting deferred.
+
+        Note: Multiple queries in a single message are not supported because
+        there is no standard way to respond with multiple rCodes, auth,
+        etc. This is consistent with other DNS server implementations. See
+        U{http://tools.ietf.org/html/draft-ietf-dnsext-edns1-03} for a proposed
+        solution.
+
+        @param protocol: The DNS protocol instance to which to send a response
+            message.
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param message: The original DNS query message for which a response
+            message will be constructed.
+        @type message: L{dns.Message}
+
+        @param address: The address to which the response message will be sent
+            or L{None} if C{protocol} is a stream protocol.
+        @type address: L{tuple} or L{None}
+
+        @return: A C{deferred} which fires with the resolved result or error of
+            the first query in C{message}.
+        @rtype: L{Deferred<twisted.internet.defer.Deferred>}
+        """
+        query = message.queries[0]
+
+        return (
+            self.resolver.query(query)
+            .addCallback(self.gotResolverResponse, protocol, message, address)
+            .addErrback(self.gotResolverError, protocol, message, address)
+        )
+
+    def handleInverseQuery(self, message, protocol, address):
+        """
+        Called by L{DNSServerFactory.messageReceived} when an inverse query
+        message is received.
+
+        Replies with a I{Not Implemented} error by default.
+
+        An error message will be logged if C{DNSServerFactory.verbose} is C{>1}.
+
+        Override in a subclass.
+
+        @param protocol: The DNS protocol instance to which to send a response
+            message.
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param message: The original DNS query message for which a response
+            message will be constructed.
+        @type message: L{dns.Message}
+
+        @param address: The address to which the response message will be sent
+            or L{None} if C{protocol} is a stream protocol.
+        @type address: L{tuple} or L{None}
+        """
+        message.rCode = dns.ENOTIMP
+        self.sendReply(protocol, message, address)
+        self._verboseLog(f"Inverse query from {address!r}")
+
+    def handleStatus(self, message, protocol, address):
+        """
+        Called by L{DNSServerFactory.messageReceived} when a status message is
+        received.
+
+        Replies with a I{Not Implemented} error by default.
+
+        An error message will be logged if C{DNSServerFactory.verbose} is C{>1}.
+
+        Override in a subclass.
+
+        @param protocol: The DNS protocol instance to which to send a response
+            message.
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param message: The original DNS query message for which a response
+            message will be constructed.
+        @type message: L{dns.Message}
+
+        @param address: The address to which the response message will be sent
+            or L{None} if C{protocol} is a stream protocol.
+        @type address: L{tuple} or L{None}
+        """
+        message.rCode = dns.ENOTIMP
+        self.sendReply(protocol, message, address)
+        self._verboseLog(f"Status request from {address!r}")
+
+    def handleNotify(self, message, protocol, address):
+        """
+        Called by L{DNSServerFactory.messageReceived} when a notify message is
+        received.
+
+        Replies with a I{Not Implemented} error by default.
+
+        An error message will be logged if C{DNSServerFactory.verbose} is C{>1}.
+
+        Override in a subclass.
+
+        @param protocol: The DNS protocol instance to which to send a response
+            message.
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param message: The original DNS query message for which a response
+            message will be constructed.
+        @type message: L{dns.Message}
+
+        @param address: The address to which the response message will be sent
+            or L{None} if C{protocol} is a stream protocol.
+        @type address: L{tuple} or L{None}
+        """
+        message.rCode = dns.ENOTIMP
+        self.sendReply(protocol, message, address)
+        self._verboseLog(f"Notify message from {address!r}")
+
+    def handleOther(self, message, protocol, address):
+        """
+        Called by L{DNSServerFactory.messageReceived} when a message with
+        unrecognised I{OPCODE} is received.
+
+        Replies with a I{Not Implemented} error by default.
+
+        An error message will be logged if C{DNSServerFactory.verbose} is C{>1}.
+
+        Override in a subclass.
+
+        @param protocol: The DNS protocol instance to which to send a response
+            message.
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param message: The original DNS query message for which a response
+            message will be constructed.
+        @type message: L{dns.Message}
+
+        @param address: The address to which the response message will be sent
+            or L{None} if C{protocol} is a stream protocol.
+        @type address: L{tuple} or L{None}
+        """
+        message.rCode = dns.ENOTIMP
+        self.sendReply(protocol, message, address)
+        self._verboseLog("Unknown op code (%d) from %r" % (message.opCode, address))
+
+    def messageReceived(self, message, proto, address=None):
+        """
+        L{DNSServerFactory.messageReceived} is called by protocols which are
+        under the control of this L{DNSServerFactory} whenever they receive a
+        DNS query message or an unexpected / duplicate / late DNS response
+        message.
+
+        L{DNSServerFactory.allowQuery} is called with the received message,
+        protocol and origin address. If it returns L{False}, a C{dns.EREFUSED}
+        response is sent back to the client.
+
+        Otherwise the received message is dispatched to one of
+        L{DNSServerFactory.handleQuery}, L{DNSServerFactory.handleInverseQuery},
+        L{DNSServerFactory.handleStatus}, L{DNSServerFactory.handleNotify}, or
+        L{DNSServerFactory.handleOther} depending on the I{OPCODE} of the
+        received message.
+
+        If C{DNSServerFactory.verbose} is C{>0} all received messages will be
+        logged in more or less detail depending on the value of C{verbose}.
+
+        @param message: The DNS message that was received.
+        @type message: L{dns.Message}
+
+        @param proto: The DNS protocol instance which received the message
+        @type proto: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param address: The address from which the message was received. Only
+            provided for messages received by datagram protocols. The origin of
+            Messages received from stream protocols can be gleaned from the
+            protocol C{transport} attribute.
+        @type address: L{tuple} or L{None}
+        """
+        message.timeReceived = time.time()
+
+        if self.verbose:
+            if self.verbose > 1:
+                s = " ".join([str(q) for q in message.queries])
+            else:
+                s = " ".join(
+                    [dns.QUERY_TYPES.get(q.type, "UNKNOWN") for q in message.queries]
+                )
+            if not len(s):
+                log.msg(f"Empty query from {address or proto.transport.getPeer()!r}")
+            else:
+                log.msg(f"{s} query from {address or proto.transport.getPeer()!r}")
+
+        if not self.allowQuery(message, proto, address):
+            message.rCode = dns.EREFUSED
+            self.sendReply(proto, message, address)
+        elif message.opCode == dns.OP_QUERY:
+            self.handleQuery(message, proto, address)
+        elif message.opCode == dns.OP_INVERSE:
+            self.handleInverseQuery(message, proto, address)
+        elif message.opCode == dns.OP_STATUS:
+            self.handleStatus(message, proto, address)
+        elif message.opCode == dns.OP_NOTIFY:
+            self.handleNotify(message, proto, address)
+        else:
+            self.handleOther(message, proto, address)
+
+    def allowQuery(self, message, protocol, address):
+        """
+        Called by L{DNSServerFactory.messageReceived} to decide whether to
+        process a received message or to reply with C{dns.EREFUSED}.
+
+        This default implementation permits anything but empty queries.
+
+        Override in a subclass to implement alternative policies.
+
+        @param message: The DNS message that was received.
+        @type message: L{dns.Message}
+
+        @param protocol: The DNS protocol instance which received the message
+        @type protocol: L{dns.DNSDatagramProtocol} or L{dns.DNSProtocol}
+
+        @param address: The address from which the message was received. Only
+            provided for messages received by datagram protocols. The origin of
+            Messages received from stream protocols can be gleaned from the
+            protocol C{transport} attribute.
+        @type address: L{tuple} or L{None}
+
+        @return: L{True} if the received message contained one or more queries,
+            else L{False}.
+        @rtype: L{bool}
+        """
+        return len(message.queries)
diff --git a/contrib/python/Twisted/py3/twisted/names/srvconnect.py b/contrib/python/Twisted/py3/twisted/names/srvconnect.py
new file mode 100644
index 00000000000..6cad7a25391
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/srvconnect.py
@@ -0,0 +1,271 @@
+# -*- test-case-name: twisted.names.test.test_srvconnect -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+import random
+
+from zope.interface import implementer
+
+from twisted.internet import error, interfaces
+from twisted.names import client, dns
+from twisted.names.error import DNSNameError
+from twisted.python.compat import nativeString
+
+
+class _SRVConnector_ClientFactoryWrapper:
+    def __init__(self, connector, wrappedFactory):
+        self.__connector = connector
+        self.__wrappedFactory = wrappedFactory
+
+    def startedConnecting(self, connector):
+        self.__wrappedFactory.startedConnecting(self.__connector)
+
+    def clientConnectionFailed(self, connector, reason):
+        self.__connector.connectionFailed(reason)
+
+    def clientConnectionLost(self, connector, reason):
+        self.__connector.connectionLost(reason)
+
+    def __getattr__(self, key):
+        return getattr(self.__wrappedFactory, key)
+
+
+@implementer(interfaces.IConnector)
+class SRVConnector:
+    """
+    A connector that looks up DNS SRV records.
+
+    RFC 2782 details how SRV records should be interpreted and selected
+    for subsequent connection attempts. The algorithm for using the records'
+    priority and weight is implemented in L{pickServer}.
+
+    @ivar servers: List of candidate server records for future connection
+        attempts.
+    @type servers: L{list} of L{dns.Record_SRV}
+
+    @ivar orderedServers: List of server records that have already been tried
+        in this round of connection attempts.
+    @type orderedServers: L{list} of L{dns.Record_SRV}
+    """
+
+    stopAfterDNS = 0
+
+    def __init__(
+        self,
+        reactor,
+        service,
+        domain,
+        factory,
+        protocol="tcp",
+        connectFuncName="connectTCP",
+        connectFuncArgs=(),
+        connectFuncKwArgs={},
+        defaultPort=None,
+    ):
+        """
+        @param domain: The domain to connect to.  If passed as a text
+            string, it will be encoded using C{idna} encoding.
+        @type domain: L{bytes} or L{str}
+
+        @param defaultPort: Optional default port number to be used when SRV
+            lookup fails and the service name is unknown. This should be the
+            port number associated with the service name as defined by the IANA
+            registry.
+        @type defaultPort: L{int}
+        """
+        self.reactor = reactor
+        self.service = service
+        self.domain = None if domain is None else dns.domainString(domain)
+        self.factory = factory
+
+        self.protocol = protocol
+        self.connectFuncName = connectFuncName
+        self.connectFuncArgs = connectFuncArgs
+        self.connectFuncKwArgs = connectFuncKwArgs
+        self._defaultPort = defaultPort
+
+        self.connector = None
+        self.servers = None
+        # list of servers already used in this round:
+        self.orderedServers = None
+
+    def connect(self):
+        """Start connection to remote server."""
+        self.factory.doStart()
+        self.factory.startedConnecting(self)
+
+        if not self.servers:
+            if self.domain is None:
+                self.connectionFailed(
+                    error.DNSLookupError("Domain is not defined."),
+                )
+                return
+            d = client.lookupService(
+                "_%s._%s.%s"
+                % (
+                    nativeString(self.service),
+                    nativeString(self.protocol),
+                    nativeString(self.domain),
+                ),
+            )
+            d.addCallbacks(self._cbGotServers, self._ebGotServers)
+            d.addCallback(lambda x, self=self: self._reallyConnect())
+            if self._defaultPort:
+                d.addErrback(self._ebServiceUnknown)
+            d.addErrback(self.connectionFailed)
+        elif self.connector is None:
+            self._reallyConnect()
+        else:
+            self.connector.connect()
+
+    def _ebGotServers(self, failure):
+        failure.trap(DNSNameError)
+
+        # Some DNS servers reply with NXDOMAIN when in fact there are
+        # just no SRV records for that domain. Act as if we just got an
+        # empty response and use fallback.
+
+        self.servers = []
+        self.orderedServers = []
+
+    def _cbGotServers(self, result):
+        answers, auth, add = result
+        if (
+            len(answers) == 1
+            and answers[0].type == dns.SRV
+            and answers[0].payload
+            and answers[0].payload.target == dns.Name(b".")
+        ):
+            # decidedly not available
+            raise error.DNSLookupError(
+                "Service %s not available for domain %s."
+                % (repr(self.service), repr(self.domain))
+            )
+
+        self.servers = []
+        self.orderedServers = []
+        for a in answers:
+            if a.type != dns.SRV or not a.payload:
+                continue
+
+            self.orderedServers.append(a.payload)
+
+    def _ebServiceUnknown(self, failure):
+        """
+        Connect to the default port when the service name is unknown.
+
+        If no SRV records were found, the service name will be passed as the
+        port. If resolving the name fails with
+        L{error.ServiceNameUnknownError}, a final attempt is done using the
+        default port.
+        """
+        failure.trap(error.ServiceNameUnknownError)
+        self.servers = [dns.Record_SRV(0, 0, self._defaultPort, self.domain)]
+        self.orderedServers = []
+        self.connect()
+
+    def pickServer(self):
+        """
+        Pick the next server.
+
+        This selects the next server from the list of SRV records according
+        to their priority and weight values, as set out by the default
+        algorithm specified in RFC 2782.
+
+        At the beginning of a round, L{servers} is populated with
+        L{orderedServers}, and the latter is made empty. L{servers}
+        is the list of candidates, and L{orderedServers} is the list of servers
+        that have already been tried.
+
+        First, all records are ordered by priority and weight in ascending
+        order. Then for each priority level, a running sum is calculated
+        over the sorted list of records for that priority. Then a random value
+        between 0 and the final sum is compared to each record in order. The
+        first record that is greater than or equal to that random value is
+        chosen and removed from the list of candidates for this round.
+
+        @return: A tuple of target hostname and port from the chosen DNS SRV
+            record.
+        @rtype: L{tuple} of native L{str} and L{int}
+        """
+        assert self.servers is not None
+        assert self.orderedServers is not None
+
+        if not self.servers and not self.orderedServers:
+            # no SRV record, fall back..
+            return nativeString(self.domain), self.service
+
+        if not self.servers and self.orderedServers:
+            # start new round
+            self.servers = self.orderedServers
+            self.orderedServers = []
+
+        assert self.servers
+
+        self.servers.sort(key=lambda record: (record.priority, record.weight))
+        minPriority = self.servers[0].priority
+
+        index = 0
+        weightSum = 0
+        weightIndex = []
+        for x in self.servers:
+            if x.priority == minPriority:
+                weightSum += x.weight
+                weightIndex.append((index, weightSum))
+                index += 1
+
+        rand = random.randint(0, weightSum)
+        for index, weight in weightIndex:
+            if weight >= rand:
+                chosen = self.servers[index]
+                del self.servers[index]
+                self.orderedServers.append(chosen)
+
+                return str(chosen.target), chosen.port
+
+        raise RuntimeError(f"Impossible {self.__class__.__name__} pickServer result.")
+
+    def _reallyConnect(self):
+        if self.stopAfterDNS:
+            self.stopAfterDNS = 0
+            return
+
+        self.host, self.port = self.pickServer()
+        assert self.host is not None, "Must have a host to connect to."
+        assert self.port is not None, "Must have a port to connect to."
+
+        connectFunc = getattr(self.reactor, self.connectFuncName)
+        self.connector = connectFunc(
+            self.host,
+            self.port,
+            _SRVConnector_ClientFactoryWrapper(self, self.factory),
+            *self.connectFuncArgs,
+            **self.connectFuncKwArgs,
+        )
+
+    def stopConnecting(self):
+        """Stop attempting to connect."""
+        if self.connector:
+            self.connector.stopConnecting()
+        else:
+            self.stopAfterDNS = 1
+
+    def disconnect(self):
+        """Disconnect whatever our are state is."""
+        if self.connector is not None:
+            self.connector.disconnect()
+        else:
+            self.stopConnecting()
+
+    def getDestination(self):
+        assert self.connector
+        return self.connector.getDestination()
+
+    def connectionFailed(self, reason):
+        self.factory.clientConnectionFailed(self, reason)
+        self.factory.doStop()
+
+    def connectionLost(self, reason):
+        self.factory.clientConnectionLost(self, reason)
+        self.factory.doStop()
diff --git a/contrib/python/Twisted/py3/twisted/names/tap.py b/contrib/python/Twisted/py3/twisted/names/tap.py
new file mode 100644
index 00000000000..c971c10386f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/names/tap.py
@@ -0,0 +1,149 @@
+# -*- test-case-name: twisted.names.test.test_tap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Domain Name Server
+"""
+
+import os
+import traceback
+
+from twisted.application import internet, service
+from twisted.names import authority, dns, secondary, server
+from twisted.python import usage
+
+
+class Options(usage.Options):
+    optParameters = [
+        ["interface", "i", "", "The interface to which to bind"],
+        ["port", "p", "53", "The port on which to listen"],
+        [
+            "resolv-conf",
+            None,
+            None,
+            "Override location of resolv.conf (implies --recursive)",
+        ],
+        ["hosts-file", None, None, "Perform lookups with a hosts file"],
+    ]
+
+    optFlags = [
+        ["cache", "c", "Enable record caching"],
+        ["recursive", "r", "Perform recursive lookups"],
+        ["verbose", "v", "Log verbosely"],
+    ]
+
+    compData = usage.Completions(
+        optActions={"interface": usage.CompleteNetInterfaces()}
+    )
+
+    zones = None
+    zonefiles = None
+
+    def __init__(self):
+        usage.Options.__init__(self)
+        self["verbose"] = 0
+        self.bindfiles = []
+        self.zonefiles = []
+        self.secondaries = []
+
+    def opt_pyzone(self, filename):
+        """Specify the filename of a Python syntax zone definition"""
+        if not os.path.exists(filename):
+            raise usage.UsageError(filename + ": No such file")
+        self.zonefiles.append(filename)
+
+    def opt_bindzone(self, filename):
+        """Specify the filename of a BIND9 syntax zone definition"""
+        if not os.path.exists(filename):
+            raise usage.UsageError(filename + ": No such file")
+        self.bindfiles.append(filename)
+
+    def opt_secondary(self, ip_domain):
+        """Act as secondary for the specified domain, performing
+        zone transfers from the specified IP (IP/domain)
+        """
+        args = ip_domain.split("/", 1)
+        if len(args) != 2:
+            raise usage.UsageError("Argument must be of the form IP[:port]/domain")
+        address = args[0].split(":")
+        if len(address) == 1:
+            address = (address[0], dns.PORT)
+        else:
+            try:
+                port = int(address[1])
+            except ValueError:
+                raise usage.UsageError(
+                    f"Specify an integer port number, not {address[1]!r}"
+                )
+            address = (address[0], port)
+        self.secondaries.append((address, [args[1]]))
+
+    def opt_verbose(self):
+        """Increment verbosity level"""
+        self["verbose"] += 1
+
+    def postOptions(self):
+        if self["resolv-conf"]:
+            self["recursive"] = True
+
+        self.svcs = []
+        self.zones = []
+        for f in self.zonefiles:
+            try:
+                self.zones.append(authority.PySourceAuthority(f))
+            except Exception:
+                traceback.print_exc()
+                raise usage.UsageError("Invalid syntax in " + f)
+        for f in self.bindfiles:
+            try:
+                self.zones.append(authority.BindAuthority(f))
+            except Exception:
+                traceback.print_exc()
+                raise usage.UsageError("Invalid syntax in " + f)
+        for f in self.secondaries:
+            svc = secondary.SecondaryAuthorityService.fromServerAddressAndDomains(*f)
+            self.svcs.append(svc)
+            self.zones.append(self.svcs[-1].getAuthority())
+        try:
+            self["port"] = int(self["port"])
+        except ValueError:
+            raise usage.UsageError("Invalid port: {!r}".format(self["port"]))
+
+
+def _buildResolvers(config):
+    """
+    Build DNS resolver instances in an order which leaves recursive
+    resolving as a last resort.
+
+    @type config: L{Options} instance
+    @param config: Parsed command-line configuration
+
+    @return: Two-item tuple of a list of cache resovers and a list of client
+        resolvers
+    """
+    from twisted.names import cache, client, hosts
+
+    ca, cl = [], []
+    if config["cache"]:
+        ca.append(cache.CacheResolver(verbose=config["verbose"]))
+    if config["hosts-file"]:
+        cl.append(hosts.Resolver(file=config["hosts-file"]))
+    if config["recursive"]:
+        cl.append(client.createResolver(resolvconf=config["resolv-conf"]))
+    return ca, cl
+
+
+def makeService(config):
+    ca, cl = _buildResolvers(config)
+
+    f = server.DNSServerFactory(config.zones, ca, cl, config["verbose"])
+    p = dns.DNSDatagramProtocol(f)
+    f.noisy = 0
+    ret = service.MultiService()
+    for klass, arg in [(internet.TCPServer, f), (internet.UDPServer, p)]:
+        s = klass(config["port"], arg, interface=config["interface"])
+        s.setServiceParent(ret)
+    for svc in config.svcs:
+        svc.setServiceParent(ret)
+    return ret
diff --git a/contrib/python/Twisted/py3/twisted/newsfragments/.gitignore b/contrib/python/Twisted/py3/twisted/newsfragments/.gitignore
new file mode 100644
index 00000000000..f935021a8f8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/newsfragments/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/contrib/python/Twisted/py3/twisted/pair/__init__.py b/contrib/python/Twisted/py3/twisted/pair/__init__.py
new file mode 100644
index 00000000000..09fb3dc85db
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/pair/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Pair: The framework of your ethernet.
+
+Low-level networking transports and utilities.
+
+See also twisted.protocols.ethernet, twisted.protocols.ip,
+twisted.protocols.raw and twisted.protocols.rawudp.
+
+Maintainer: Tommi Virtanen
+"""
diff --git a/contrib/python/Twisted/py3/twisted/pair/ethernet.py b/contrib/python/Twisted/py3/twisted/pair/ethernet.py
new file mode 100644
index 00000000000..ff58326235e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/pair/ethernet.py
@@ -0,0 +1,59 @@
+# -*- test-case-name: twisted.pair.test.test_ethernet -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+
+"""Support for working directly with ethernet frames"""
+
+import struct
+
+from zope.interface import Interface, implementer
+
+from twisted.internet import protocol
+from twisted.pair import raw
+
+
+class IEthernetProtocol(Interface):
+    """An interface for protocols that handle Ethernet frames"""
+
+    def addProto(num, proto):
+        """Add an IRawPacketProtocol protocol"""
+
+    def datagramReceived(data, partial):
+        """An Ethernet frame has been received"""
+
+
+class EthernetHeader:
+    def __init__(self, data):
+        (self.dest, self.source, self.proto) = struct.unpack(
+            "!6s6sH", data[: 6 + 6 + 2]
+        )
+
+
+@implementer(IEthernetProtocol)
+class EthernetProtocol(protocol.AbstractDatagramProtocol):
+    def __init__(self):
+        self.etherProtos = {}
+
+    def addProto(self, num, proto):
+        proto = raw.IRawPacketProtocol(proto)
+        if num < 0:
+            raise TypeError("Added protocol must be positive or zero")
+        if num >= 2**16:
+            raise TypeError("Added protocol must fit in 16 bits")
+        if num not in self.etherProtos:
+            self.etherProtos[num] = []
+        self.etherProtos[num].append(proto)
+
+    def datagramReceived(self, data, partial=0):
+        header = EthernetHeader(data[:14])
+        for proto in self.etherProtos.get(header.proto, ()):
+            proto.datagramReceived(
+                data=data[14:],
+                partial=partial,
+                dest=header.dest,
+                source=header.source,
+                protocol=header.proto,
+            )
diff --git a/contrib/python/Twisted/py3/twisted/pair/ip.py b/contrib/python/Twisted/py3/twisted/pair/ip.py
new file mode 100644
index 00000000000..3606abf2721
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/pair/ip.py
@@ -0,0 +1,78 @@
+# -*- test-case-name: twisted.pair.test.test_ip -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+
+"""Support for working directly with IP packets"""
+
+import socket
+import struct
+
+from zope.interface import implementer
+
+from twisted.internet import protocol
+from twisted.pair import raw
+
+
+class IPHeader:
+    def __init__(self, data):
+        (
+            ihlversion,
+            self.tos,
+            self.tot_len,
+            self.fragment_id,
+            frag_off,
+            self.ttl,
+            self.protocol,
+            self.check,
+            saddr,
+            daddr,
+        ) = struct.unpack("!BBHHHBBH4s4s", data[:20])
+        self.saddr = socket.inet_ntoa(saddr)
+        self.daddr = socket.inet_ntoa(daddr)
+        self.version = ihlversion & 0x0F
+        self.ihl = ((ihlversion & 0xF0) >> 4) << 2
+        self.fragment_offset = frag_off & 0x1FFF
+        self.dont_fragment = frag_off & 0x4000 != 0
+        self.more_fragments = frag_off & 0x2000 != 0
+
+
+MAX_SIZE = 2**32
+
+
+@implementer(raw.IRawPacketProtocol)
+class IPProtocol(protocol.AbstractDatagramProtocol):
+    def __init__(self):
+        self.ipProtos = {}
+
+    def addProto(self, num, proto):
+        proto = raw.IRawDatagramProtocol(proto)
+        if num < 0:
+            raise TypeError("Added protocol must be positive or zero")
+        if num >= MAX_SIZE:
+            raise TypeError("Added protocol must fit in 32 bits")
+        if num not in self.ipProtos:
+            self.ipProtos[num] = []
+        self.ipProtos[num].append(proto)
+
+    def datagramReceived(self, data, partial, dest, source, protocol):
+        header = IPHeader(data)
+        for proto in self.ipProtos.get(header.protocol, ()):
+            proto.datagramReceived(
+                data=data[20:],
+                partial=partial,
+                source=header.saddr,
+                dest=header.daddr,
+                protocol=header.protocol,
+                version=header.version,
+                ihl=header.ihl,
+                tos=header.tos,
+                tot_len=header.tot_len,
+                fragment_id=header.fragment_id,
+                fragment_offset=header.fragment_offset,
+                dont_fragment=header.dont_fragment,
+                more_fragments=header.more_fragments,
+                ttl=header.ttl,
+            )
diff --git a/contrib/python/Twisted/py3/twisted/pair/raw.py b/contrib/python/Twisted/py3/twisted/pair/raw.py
new file mode 100644
index 00000000000..cf564fb17d1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/pair/raw.py
@@ -0,0 +1,54 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Interface definitions for working with raw packets
+"""
+
+from zope.interface import Interface
+
+
+class IRawDatagramProtocol(Interface):
+    """
+    An interface for protocols such as UDP, ICMP and TCP.
+    """
+
+    def addProto(num, proto):
+        """
+        Add a protocol on top of this one.
+        """
+
+    def datagramReceived(
+        data,
+        partial,
+        source,
+        dest,
+        protocol,
+        version,
+        ihl,
+        tos,
+        tot_len,
+        fragment_id,
+        fragment_offset,
+        dont_fragment,
+        more_fragments,
+        ttl,
+    ):
+        """
+        An IP datagram has been received. Parse and process it.
+        """
+
+
+class IRawPacketProtocol(Interface):
+    """
+    An interface for low-level protocols such as IP and ARP.
+    """
+
+    def addProto(num, proto):
+        """
+        Add a protocol on top of this one.
+        """
+
+    def datagramReceived(data, partial, dest, source, protocol):
+        """
+        An IP datagram has been received. Parse and process it.
+        """
diff --git a/contrib/python/Twisted/py3/twisted/pair/rawudp.py b/contrib/python/Twisted/py3/twisted/pair/rawudp.py
new file mode 100644
index 00000000000..c4f2d97ef4e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/pair/rawudp.py
@@ -0,0 +1,59 @@
+# -*- test-case-name: twisted.pair.test.test_rawudp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of raw packet interfaces for UDP
+"""
+
+import struct
+
+from zope.interface import implementer
+
+from twisted.internet import protocol
+from twisted.pair import raw
+
+
+class UDPHeader:
+    def __init__(self, data):
+        (self.source, self.dest, self.len, self.check) = struct.unpack(
+            "!HHHH", data[:8]
+        )
+
+
+@implementer(raw.IRawDatagramProtocol)
+class RawUDPProtocol(protocol.AbstractDatagramProtocol):
+    def __init__(self):
+        self.udpProtos = {}
+
+    def addProto(self, num, proto):
+        if not isinstance(proto, protocol.DatagramProtocol):
+            raise TypeError("Added protocol must be an instance of DatagramProtocol")
+        if num < 0:
+            raise TypeError("Added protocol must be positive or zero")
+        if num >= 2**16:
+            raise TypeError("Added protocol must fit in 16 bits")
+        if num not in self.udpProtos:
+            self.udpProtos[num] = []
+        self.udpProtos[num].append(proto)
+
+    def datagramReceived(
+        self,
+        data,
+        partial,
+        source,
+        dest,
+        protocol,
+        version,
+        ihl,
+        tos,
+        tot_len,
+        fragment_id,
+        fragment_offset,
+        dont_fragment,
+        more_fragments,
+        ttl,
+    ):
+        header = UDPHeader(data)
+        for proto in self.udpProtos.get(header.dest, ()):
+            proto.datagramReceived(data[8:], (source, header.source))
diff --git a/contrib/python/Twisted/py3/twisted/pair/testing.py b/contrib/python/Twisted/py3/twisted/pair/testing.py
new file mode 100644
index 00000000000..c5e0476a392
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/pair/testing.py
@@ -0,0 +1,555 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Tools for automated testing of L{twisted.pair}-based applications.
+"""
+
+import socket
+import struct
+from collections import deque
+from errno import EAGAIN, EBADF, EINTR, EINVAL, ENOBUFS, ENOSYS, EPERM, EWOULDBLOCK
+from functools import wraps
+
+from zope.interface import implementer
+
+from twisted.internet.protocol import DatagramProtocol
+from twisted.pair.ethernet import EthernetProtocol
+from twisted.pair.ip import IPProtocol
+from twisted.pair.rawudp import RawUDPProtocol
+from twisted.pair.tuntap import _IFNAMSIZ, _TUNSETIFF, TunnelFlags, _IInputOutputSystem
+from twisted.python.compat import nativeString
+
+# The number of bytes in the "protocol information" header that may be present
+# on datagrams read from a tunnel device.  This is two bytes of flags followed
+# by two bytes of protocol identification.  All this code does with this
+# information is use it to discard the header.
+_PI_SIZE = 4
+
+
+def _H(n):
+    """
+    Pack an integer into a network-order two-byte string.
+
+    @param n: The integer to pack.  Only values that fit into 16 bits are
+        supported.
+
+    @return: The packed representation of the integer.
+    @rtype: L{bytes}
+    """
+    return struct.pack(">H", n)
+
+
+_IPv4 = 0x0800
+
+
+def _ethernet(src, dst, protocol, payload):
+    """
+    Construct an ethernet frame.
+
+    @param src: The source ethernet address, encoded.
+    @type src: L{bytes}
+
+    @param dst: The destination ethernet address, encoded.
+    @type dst: L{bytes}
+
+    @param protocol: The protocol number of the payload of this datagram.
+    @type protocol: L{int}
+
+    @param payload: The content of the ethernet frame (such as an IP datagram).
+    @type payload: L{bytes}
+
+    @return: The full ethernet frame.
+    @rtype: L{bytes}
+    """
+    return dst + src + _H(protocol) + payload
+
+
+def _ip(src, dst, payload):
+    """
+    Construct an IP datagram with the given source, destination, and
+    application payload.
+
+    @param src: The source IPv4 address as a dotted-quad string.
+    @type src: L{bytes}
+
+    @param dst: The destination IPv4 address as a dotted-quad string.
+    @type dst: L{bytes}
+
+    @param payload: The content of the IP datagram (such as a UDP datagram).
+    @type payload: L{bytes}
+
+    @return: An IP datagram header and payload.
+    @rtype: L{bytes}
+    """
+    ipHeader = (
+        # Version and header length, 4 bits each
+        b"\x45"
+        # Differentiated services field
+        b"\x00"
+        # Total length
+        + _H(20 + len(payload))
+        + b"\x00\x01\x00\x00\x40\x11"
+        # Checksum
+        + _H(0)
+        # Source address
+        + socket.inet_pton(socket.AF_INET, nativeString(src))
+        # Destination address
+        + socket.inet_pton(socket.AF_INET, nativeString(dst))
+    )
+
+    # Total all of the 16-bit integers in the header
+    checksumStep1 = sum(struct.unpack("!10H", ipHeader))
+    # Pull off the carry
+    carry = checksumStep1 >> 16
+    # And add it to what was left over
+    checksumStep2 = (checksumStep1 & 0xFFFF) + carry
+    # Compute the one's complement sum
+    checksumStep3 = checksumStep2 ^ 0xFFFF
+
+    # Reconstruct the IP header including the correct checksum so the platform
+    # IP stack, if there is one involved in this test, doesn't drop it on the
+    # floor as garbage.
+    ipHeader = ipHeader[:10] + struct.pack("!H", checksumStep3) + ipHeader[12:]
+
+    return ipHeader + payload
+
+
+def _udp(src, dst, payload):
+    """
+    Construct a UDP datagram with the given source, destination, and
+    application payload.
+
+    @param src: The source port number.
+    @type src: L{int}
+
+    @param dst: The destination port number.
+    @type dst: L{int}
+
+    @param payload: The content of the UDP datagram.
+    @type payload: L{bytes}
+
+    @return: A UDP datagram header and payload.
+    @rtype: L{bytes}
+    """
+    udpHeader = (
+        # Source port
+        _H(src)
+        # Destination port
+        + _H(dst)
+        # Length
+        + _H(len(payload) + 8)
+        # Checksum
+        + _H(0)
+    )
+    return udpHeader + payload
+
+
+class Tunnel:
+    """
+    An in-memory implementation of a tun or tap device.
+
+    @cvar _DEVICE_NAME: A string representing the conventional filesystem entry
+        for the tunnel factory character special device.
+    @type _DEVICE_NAME: C{bytes}
+    """
+
+    _DEVICE_NAME = b"/dev/net/tun"
+
+    # Between POSIX and Python, there are 4 combinations.  Here are two, at
+    # least.
+    EAGAIN_STYLE = IOError(EAGAIN, "Resource temporarily unavailable")
+    EWOULDBLOCK_STYLE = OSError(EWOULDBLOCK, "Operation would block")
+
+    # Oh yea, and then there's the case where maybe we would've read, but
+    # someone sent us a signal instead.
+    EINTR_STYLE = IOError(EINTR, "Interrupted function call")
+
+    nonBlockingExceptionStyle = EAGAIN_STYLE
+
+    SEND_BUFFER_SIZE = 1024
+
+    def __init__(self, system, openFlags, fileMode):
+        """
+        @param system: An L{_IInputOutputSystem} provider to use to perform I/O.
+
+        @param openFlags: Any flags to apply when opening the tunnel device.
+            See C{os.O_*}.
+
+        @type openFlags: L{int}
+
+        @param fileMode: ignored
+        """
+        self.system = system
+
+        # Drop fileMode on the floor - evidence and logic suggest it is
+        # irrelevant with respect to /dev/net/tun
+        self.openFlags = openFlags
+        self.tunnelMode = None
+        self.requestedName = None
+        self.name = None
+        self.readBuffer = deque()
+        self.writeBuffer = deque()
+        self.pendingSignals = deque()
+
+    @property
+    def blocking(self):
+        """
+        If the file descriptor for this tunnel is open in blocking mode,
+        C{True}.  C{False} otherwise.
+        """
+        return not (self.openFlags & self.system.O_NONBLOCK)
+
+    @property
+    def closeOnExec(self):
+        """
+        If the file descriptor for this tunnel is marked as close-on-exec,
+        C{True}.  C{False} otherwise.
+        """
+        return bool(self.openFlags & self.system.O_CLOEXEC)
+
+    def addToReadBuffer(self, datagram):
+        """
+        Deliver a datagram to this tunnel's read buffer.  This makes it
+        available to be read later using the C{read} method.
+
+        @param datagram: The IPv4 datagram to deliver.  If the mode of this
+            tunnel is TAP then ethernet framing will be added automatically.
+        @type datagram: L{bytes}
+        """
+        # TAP devices also include ethernet framing.
+        if self.tunnelMode & TunnelFlags.IFF_TAP.value:
+            datagram = _ethernet(
+                src=b"\x00" * 6, dst=b"\xff" * 6, protocol=_IPv4, payload=datagram
+            )
+
+        self.readBuffer.append(datagram)
+
+    def read(self, limit):
+        """
+        Read a datagram out of this tunnel.
+
+        @param limit: The maximum number of bytes from the datagram to return.
+            If the next datagram is larger than this, extra bytes are dropped
+            and lost forever.
+        @type limit: L{int}
+
+        @raise OSError: Any of the usual I/O problems can result in this
+            exception being raised with some particular error number set.
+
+        @raise IOError: Any of the usual I/O problems can result in this
+            exception being raised with some particular error number set.
+
+        @return: The datagram which was read from the tunnel.  If the tunnel
+            mode does not include L{TunnelFlags.IFF_NO_PI} then the datagram is
+            prefixed with a 4 byte PI header.
+        @rtype: L{bytes}
+        """
+        if self.readBuffer:
+            if self.tunnelMode & TunnelFlags.IFF_NO_PI.value:
+                header = b""
+            else:
+                # Synthesize a PI header to include in the result.  Nothing in
+                # twisted.pair uses the PI information yet so we can synthesize
+                # something incredibly boring (ie 32 bits of 0).
+                header = b"\x00" * _PI_SIZE
+                limit -= 4
+            return header + self.readBuffer.popleft()[:limit]
+        elif self.blocking:
+            raise NotImplementedError()
+        else:
+            raise self.nonBlockingExceptionStyle
+
+    def write(self, datagram):
+        """
+        Write a datagram into this tunnel.
+
+        @param datagram: The datagram to write.
+        @type datagram: L{bytes}
+
+        @raise IOError: Any of the usual I/O problems can result in this
+            exception being raised with some particular error number set.
+
+        @return: The number of bytes of the datagram which were written.
+        @rtype: L{int}
+        """
+        if self.pendingSignals:
+            self.pendingSignals.popleft()
+            raise OSError(EINTR, "Interrupted system call")
+
+        if len(datagram) > self.SEND_BUFFER_SIZE:
+            raise OSError(ENOBUFS, "No buffer space available")
+
+        self.writeBuffer.append(datagram)
+        return len(datagram)
+
+
+def _privileged(original):
+    """
+    Wrap a L{MemoryIOSystem} method with permission-checking logic.  The
+    returned function will check C{self.permissions} and raise L{IOError} with
+    L{errno.EPERM} if the function name is not listed as an available
+    permission.
+
+    @param original: The L{MemoryIOSystem} instance to wrap.
+
+    @return: A wrapper around C{original} that applies permission checks.
+    """
+
+    @wraps(original)
+    def permissionChecker(self, *args, **kwargs):
+        if original.__name__ not in self.permissions:
+            raise OSError(EPERM, "Operation not permitted")
+        return original(self, *args, **kwargs)
+
+    return permissionChecker
+
+
+@implementer(_IInputOutputSystem)
+class MemoryIOSystem:
+    """
+    An in-memory implementation of basic I/O primitives, useful in the context
+    of unit testing as a drop-in replacement for parts of the C{os} module.
+
+    @ivar _devices:
+    @ivar _openFiles:
+    @ivar permissions:
+
+    @ivar _counter:
+    """
+
+    _counter = 8192
+
+    O_RDWR = 1 << 0
+    O_NONBLOCK = 1 << 1
+    O_CLOEXEC = 1 << 2
+
+    def __init__(self):
+        self._devices = {}
+        self._openFiles = {}
+        self.permissions = {"open", "ioctl"}
+
+    def getTunnel(self, port):
+        """
+        Get the L{Tunnel} object associated with the given L{TuntapPort}.
+
+        @param port: A L{TuntapPort} previously initialized using this
+            L{MemoryIOSystem}.
+
+        @return: The tunnel object created by a prior use of C{open} on this
+            object on the tunnel special device file.
+        @rtype: L{Tunnel}
+        """
+        return self._openFiles[port.fileno()]
+
+    def registerSpecialDevice(self, name, cls):
+        """
+        Specify a class which will be used to handle I/O to a device of a
+        particular name.
+
+        @param name: The filesystem path name of the device.
+        @type name: L{bytes}
+
+        @param cls: A class (like L{Tunnel}) to instantiated whenever this
+            device is opened.
+        """
+        self._devices[name] = cls
+
+    @_privileged
+    def open(self, name, flags, mode=None):
+        """
+        A replacement for C{os.open}.  This initializes state in this
+        L{MemoryIOSystem} which will be reflected in the behavior of the other
+        file descriptor-related methods (eg L{MemoryIOSystem.read},
+        L{MemoryIOSystem.write}, etc).
+
+        @param name: A string giving the name of the file to open.
+        @type name: C{bytes}
+
+        @param flags: The flags with which to open the file.
+        @type flags: C{int}
+
+        @param mode: The mode with which to open the file.
+        @type mode: C{int}
+
+        @raise OSError: With C{ENOSYS} if the file is not a recognized special
+            device file.
+
+        @return: A file descriptor associated with the newly opened file
+            description.
+        @rtype: L{int}
+        """
+        if name in self._devices:
+            fd = self._counter
+            self._counter += 1
+            self._openFiles[fd] = self._devices[name](self, flags, mode)
+            return fd
+        raise OSError(ENOSYS, "Function not implemented")
+
+    def read(self, fd, limit):
+        """
+        Try to read some bytes out of one of the in-memory buffers which may
+        previously have been populated by C{write}.
+
+        @see: L{os.read}
+        """
+        try:
+            return self._openFiles[fd].read(limit)
+        except KeyError:
+            raise OSError(EBADF, "Bad file descriptor")
+
+    def write(self, fd, data):
+        """
+        Try to add some bytes to one of the in-memory buffers to be accessed by
+        a later C{read} call.
+
+        @see: L{os.write}
+        """
+        try:
+            return self._openFiles[fd].write(data)
+        except KeyError:
+            raise OSError(EBADF, "Bad file descriptor")
+
+    def close(self, fd):
+        """
+        Discard the in-memory buffer and other in-memory state for the given
+        file descriptor.
+
+        @see: L{os.close}
+        """
+        try:
+            del self._openFiles[fd]
+        except KeyError:
+            raise OSError(EBADF, "Bad file descriptor")
+
+    @_privileged
+    def ioctl(self, fd, request, args):
+        """
+        Perform some configuration change to the in-memory state for the given
+        file descriptor.
+
+        @see: L{fcntl.ioctl}
+        """
+        try:
+            tunnel = self._openFiles[fd]
+        except KeyError:
+            raise OSError(EBADF, "Bad file descriptor")
+
+        if request != _TUNSETIFF:
+            raise OSError(EINVAL, "Request or args is not valid.")
+
+        name, mode = struct.unpack("%dsH" % (_IFNAMSIZ,), args)
+        tunnel.tunnelMode = mode
+        tunnel.requestedName = name
+        tunnel.name = name[: _IFNAMSIZ - 3] + b"123"
+
+        return struct.pack("%dsH" % (_IFNAMSIZ,), tunnel.name, mode)
+
+    def sendUDP(self, datagram, address):
+        """
+        Write an ethernet frame containing an ip datagram containing a udp
+        datagram containing the given payload, addressed to the given address,
+        to a tunnel device previously opened on this I/O system.
+
+        @param datagram: A UDP datagram payload to send.
+        @type datagram: L{bytes}
+
+        @param address: The destination to which to send the datagram.
+        @type address: L{tuple} of (L{bytes}, L{int})
+
+        @return: A two-tuple giving the address from which gives the address
+            from which the datagram was sent.
+        @rtype: L{tuple} of (L{bytes}, L{int})
+        """
+        # Just make up some random thing
+        srcIP = "10.1.2.3"
+        srcPort = 21345
+
+        serialized = _ip(
+            src=srcIP,
+            dst=address[0],
+            payload=_udp(src=srcPort, dst=address[1], payload=datagram),
+        )
+
+        openFiles = list(self._openFiles.values())
+        openFiles[0].addToReadBuffer(serialized)
+
+        return (srcIP, srcPort)
+
+    def receiveUDP(self, fileno, host, port):
+        """
+        Get a socket-like object which can be used to receive a datagram sent
+        from the given address.
+
+        @param fileno: A file descriptor representing a tunnel device which the
+            datagram will be received via.
+        @type fileno: L{int}
+
+        @param host: The IPv4 address to which the datagram was sent.
+        @type host: L{bytes}
+
+        @param port: The UDP port number to which the datagram was sent.
+            received.
+        @type port: L{int}
+
+        @return: A L{socket.socket}-like object which can be used to receive
+            the specified datagram.
+        """
+        return _FakePort(self, fileno)
+
+
+class _FakePort:
+    """
+    A socket-like object which can be used to read UDP datagrams from
+    tunnel-like file descriptors managed by a L{MemoryIOSystem}.
+    """
+
+    def __init__(self, system, fileno):
+        self._system = system
+        self._fileno = fileno
+
+    def recv(self, nbytes):
+        """
+        Receive a datagram sent to this port using the L{MemoryIOSystem} which
+        created this object.
+
+        This behaves like L{socket.socket.recv} but the data being I{sent} and
+        I{received} only passes through various memory buffers managed by this
+        object and L{MemoryIOSystem}.
+
+        @see: L{socket.socket.recv}
+        """
+        data = self._system._openFiles[self._fileno].writeBuffer.popleft()
+
+        datagrams = []
+        receiver = DatagramProtocol()
+
+        def capture(datagram, address):
+            datagrams.append(datagram)
+
+        receiver.datagramReceived = capture
+
+        udp = RawUDPProtocol()
+        udp.addProto(12345, receiver)
+
+        ip = IPProtocol()
+        ip.addProto(17, udp)
+
+        mode = self._system._openFiles[self._fileno].tunnelMode
+        if mode & TunnelFlags.IFF_TAP.value:
+            ether = EthernetProtocol()
+            ether.addProto(0x800, ip)
+            datagramReceived = ether.datagramReceived
+        else:
+            datagramReceived = lambda data: ip.datagramReceived(
+                data, None, None, None, None
+            )
+
+        dataHasPI = not (mode & TunnelFlags.IFF_NO_PI.value)
+
+        if dataHasPI:
+            # datagramReceived can't handle the PI, get rid of it.
+            data = data[_PI_SIZE:]
+
+        datagramReceived(data)
+        return datagrams[0][:nbytes]
diff --git a/contrib/python/Twisted/py3/twisted/pair/tuntap.py b/contrib/python/Twisted/py3/twisted/pair/tuntap.py
new file mode 100644
index 00000000000..a5f6dbef3bf
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/pair/tuntap.py
@@ -0,0 +1,423 @@
+# -*- test-case-name: twisted.pair.test.test_tuntap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support for Linux ethernet and IP tunnel devices.
+
+@see: U{https://en.wikipedia.org/wiki/TUN/TAP}
+"""
+
+import errno
+import fcntl
+import os
+import platform
+import struct
+import warnings
+from collections import namedtuple
+from typing import Tuple
+
+from zope.interface import Attribute, Interface, implementer
+
+from constantly import FlagConstant, Flags  # type: ignore[import]
+from incremental import Version
+
+from twisted.internet import abstract, defer, error, interfaces, task
+from twisted.pair import ethernet, raw
+from twisted.python import log
+from twisted.python.deprecate import deprecated
+from twisted.python.reflect import fullyQualifiedName
+from twisted.python.util import FancyEqMixin, FancyStrMixin
+
+__all__ = [
+    "TunnelFlags",
+    "TunnelAddress",
+    "TuntapPort",
+]
+
+
+_IFNAMSIZ = 16
+if (
+    platform.machine() == "parisc"
+    or platform.machine().startswith("ppc")
+    or platform.machine().startswith("sparc")
+):  # pragma: no coverage
+    # We don't have CI for parisc, hence no coverage is expected.
+    _TUNSETIFF = 0x800454CA
+    _TUNGETIFF = 0x400454D2
+else:
+    _TUNSETIFF = 0x400454CA
+    _TUNGETIFF = 0x800454D2
+_TUN_KO_PATH = b"/dev/net/tun"
+
+
+class TunnelFlags(Flags):
+    """
+    L{TunnelFlags} defines more flags which are used to configure the behavior
+    of a tunnel device.
+
+    @cvar IFF_TUN: This indicates a I{tun}-type device.  This type of tunnel
+        carries IP datagrams.  This flag is mutually exclusive with C{IFF_TAP}.
+
+    @cvar IFF_TAP: This indicates a I{tap}-type device.  This type of tunnel
+        carries ethernet frames.  This flag is mutually exclusive with C{IFF_TUN}.
+
+    @cvar IFF_NO_PI: This indicates the I{protocol information} header will
+        B{not} be included in data read from the tunnel.
+
+    @see: U{https://www.kernel.org/doc/Documentation/networking/tuntap.txt}
+    """
+
+    IFF_TUN = FlagConstant(0x0001)
+    IFF_TAP = FlagConstant(0x0002)
+
+    TUN_FASYNC = FlagConstant(0x0010)
+    TUN_NOCHECKSUM = FlagConstant(0x0020)
+    TUN_NO_PI = FlagConstant(0x0040)
+    TUN_ONE_QUEUE = FlagConstant(0x0080)
+    TUN_PERSIST = FlagConstant(0x0100)
+    TUN_VNET_HDR = FlagConstant(0x0200)
+
+    IFF_NO_PI = FlagConstant(0x1000)
+    IFF_ONE_QUEUE = FlagConstant(0x2000)
+    IFF_VNET_HDR = FlagConstant(0x4000)
+    IFF_TUN_EXCL = FlagConstant(0x8000)
+
+
+@implementer(interfaces.IAddress)
+class TunnelAddress(FancyStrMixin, FancyEqMixin):
+    """
+    A L{TunnelAddress} represents the tunnel to which a L{TuntapPort} is bound.
+    """
+
+    compareAttributes = ("_typeValue", "name")
+    showAttributes = (("type", lambda flag: flag.name), "name")
+
+    @property
+    def _typeValue(self):
+        """
+        Return the integer value of the C{type} attribute.  Used to produce
+        correct results in the equality implementation.
+        """
+        # Work-around for https://twistedmatrix.com/trac/ticket/6878
+        return self.type.value
+
+    def __init__(self, type, name):
+        """
+        @param type: Either L{TunnelFlags.IFF_TUN} or L{TunnelFlags.IFF_TAP},
+            representing the type of this tunnel.
+
+        @param name: The system name of the tunnel.
+        @type name: L{bytes}
+        """
+        self.type = type
+        self.name = name
+
+    def __getitem__(self, index):
+        """
+        Deprecated accessor for the tunnel name.  Use attributes instead.
+        """
+        warnings.warn(
+            "TunnelAddress.__getitem__ is deprecated since Twisted 14.0.0  "
+            "Use attributes instead.",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        return ("TUNTAP", self.name)[index]
+
+
+class _TunnelDescription(namedtuple("_TunnelDescription", "fileno name")):
+    """
+    Describe an existing tunnel.
+
+    @ivar fileno: the file descriptor associated with the tunnel
+    @type fileno: L{int}
+
+    @ivar name: the name of the tunnel
+    @type name: L{bytes}
+    """
+
+
+class _IInputOutputSystem(Interface):
+    """
+    An interface for performing some basic kinds of I/O (particularly that I/O
+    which might be useful for L{twisted.pair.tuntap}-using code).
+    """
+
+    O_RDWR = Attribute("@see: L{os.O_RDWR}")
+    O_NONBLOCK = Attribute("@see: L{os.O_NONBLOCK}")
+    O_CLOEXEC = Attribute("@see: L{os.O_CLOEXEC}")
+
+    def open(filename, flag, mode=0o777):
+        """
+        @see: L{os.open}
+        """
+
+    def ioctl(fd, opt, arg=None, mutate_flag=None):
+        """
+        @see: L{fcntl.ioctl}
+        """
+
+    def read(fd, limit):
+        """
+        @see: L{os.read}
+        """
+
+    def write(fd, data):
+        """
+        @see: L{os.write}
+        """
+
+    def close(fd):
+        """
+        @see: L{os.close}
+        """
+
+    def sendUDP(datagram, address):
+        """
+        Send a datagram to a certain address.
+
+        @param datagram: The payload of a UDP datagram to send.
+        @type datagram: L{bytes}
+
+        @param address: The destination to which to send the datagram.
+        @type address: L{tuple} of (L{bytes}, L{int})
+
+        @return: The local address from which the datagram was sent.
+        @rtype: L{tuple} of (L{bytes}, L{int})
+        """
+
+    def receiveUDP(fileno, host, port):
+        """
+        Return a socket which can be used to receive datagrams sent to the
+        given address.
+
+        @param fileno: A file descriptor representing a tunnel device which the
+            datagram was either sent via or will be received via.
+        @type fileno: L{int}
+
+        @param host: The IPv4 address at which the datagram will be received.
+        @type host: L{bytes}
+
+        @param port: The UDP port number at which the datagram will be
+            received.
+        @type port: L{int}
+
+        @return: A L{socket.socket} which can be used to receive the specified
+            datagram.
+        """
+
+
+class _RealSystem:
+    """
+    An interface to the parts of the operating system which L{TuntapPort}
+    relies on.  This is most of an implementation of L{_IInputOutputSystem}.
+    """
+
+    open = staticmethod(os.open)
+    read = staticmethod(os.read)
+    write = staticmethod(os.write)
+    close = staticmethod(os.close)
+    ioctl = staticmethod(fcntl.ioctl)
+
+    O_RDWR = os.O_RDWR
+    O_NONBLOCK = os.O_NONBLOCK
+    # Introduced in Python 3.x
+    # Ubuntu 12.04, /usr/include/x86_64-linux-gnu/bits/fcntl.h
+    O_CLOEXEC = getattr(os, "O_CLOEXEC", 0o2000000)
+
+
+@implementer(interfaces.IListeningPort)
+class TuntapPort(abstract.FileDescriptor):
+    """
+    A Port that reads and writes packets from/to a TUN/TAP-device.
+    """
+
+    maxThroughput = 256 * 1024  # Max bytes we read in one eventloop iteration
+
+    def __init__(self, interface, proto, maxPacketSize=8192, reactor=None, system=None):
+        if ethernet.IEthernetProtocol.providedBy(proto):
+            self.ethernet = 1
+            self._mode = TunnelFlags.IFF_TAP
+        else:
+            self.ethernet = 0
+            self._mode = TunnelFlags.IFF_TUN
+            assert raw.IRawPacketProtocol.providedBy(proto)
+
+        if system is None:
+            system = _RealSystem()
+        self._system = system
+
+        abstract.FileDescriptor.__init__(self, reactor)
+        self.interface = interface
+        self.protocol = proto
+        self.maxPacketSize = maxPacketSize
+
+        logPrefix = self._getLogPrefix(self.protocol)
+        self.logstr = f"{logPrefix} ({self._mode.name})"
+
+    def __repr__(self) -> str:
+        args: Tuple[str, ...] = (fullyQualifiedName(self.protocol.__class__),)
+        if self.connected:
+            args = args + ("",)
+        else:
+            args = args + ("not ",)
+        args = args + (self._mode.name, self.interface)
+        return "<%s %slistening on %s/%s>" % args
+
+    def startListening(self):
+        """
+        Create and bind my socket, and begin listening on it.
+
+        This must be called after creating a server to begin listening on the
+        specified tunnel.
+        """
+        self._bindSocket()
+        self.protocol.makeConnection(self)
+        self.startReading()
+
+    def _openTunnel(self, name, mode):
+        """
+        Open the named tunnel using the given mode.
+
+        @param name: The name of the tunnel to open.
+        @type name: L{bytes}
+
+        @param mode: Flags from L{TunnelFlags} with exactly one of
+            L{TunnelFlags.IFF_TUN} or L{TunnelFlags.IFF_TAP} set.
+
+        @return: A L{_TunnelDescription} representing the newly opened tunnel.
+        """
+        flags = self._system.O_RDWR | self._system.O_CLOEXEC | self._system.O_NONBLOCK
+        config = struct.pack("%dsH" % (_IFNAMSIZ,), name, mode.value)
+        fileno = self._system.open(_TUN_KO_PATH, flags)
+        result = self._system.ioctl(fileno, _TUNSETIFF, config)
+        return _TunnelDescription(fileno, result[:_IFNAMSIZ].strip(b"\x00"))
+
+    def _bindSocket(self):
+        """
+        Open the tunnel.
+        """
+        log.msg(
+            format="%(protocol)s starting on %(interface)s",
+            protocol=self.protocol.__class__,
+            interface=self.interface,
+        )
+        try:
+            fileno, interface = self._openTunnel(
+                self.interface, self._mode | TunnelFlags.IFF_NO_PI
+            )
+        except OSError as e:
+            raise error.CannotListenError(None, self.interface, e)
+
+        self.interface = interface
+        self._fileno = fileno
+
+        self.connected = 1
+
+    def fileno(self):
+        return self._fileno
+
+    def doRead(self):
+        """
+        Called when my socket is ready for reading.
+        """
+        read = 0
+        while read < self.maxThroughput:
+            try:
+                data = self._system.read(self._fileno, self.maxPacketSize)
+            except OSError as e:
+                if e.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EINTR):
+                    return
+                else:
+                    raise
+            except BaseException:
+                raise
+            read += len(data)
+            # TODO pkt.isPartial()?
+            try:
+                self.protocol.datagramReceived(data, partial=0)
+            except BaseException:
+                cls = fullyQualifiedName(self.protocol.__class__)
+                log.err(None, f"Unhandled exception from {cls}.datagramReceived")
+
+    def write(self, datagram):
+        """
+        Write the given data as a single datagram.
+
+        @param datagram: The data that will make up the complete datagram to be
+            written.
+        @type datagram: L{bytes}
+        """
+        try:
+            return self._system.write(self._fileno, datagram)
+        except OSError as e:
+            if e.errno == errno.EINTR:
+                return self.write(datagram)
+            raise
+
+    def writeSequence(self, seq):
+        """
+        Write a datagram constructed from a L{list} of L{bytes}.
+
+        @param seq: The data that will make up the complete datagram to be
+            written.
+        @type seq: L{list} of L{bytes}
+        """
+        self.write(b"".join(seq))
+
+    def stopListening(self):
+        """
+        Stop accepting connections on this port.
+
+        This will shut down my socket and call self.connectionLost().
+
+        @return: A L{Deferred} that fires when this port has stopped.
+        """
+        self.stopReading()
+        if self.disconnecting:
+            return self._stoppedDeferred
+        elif self.connected:
+            self._stoppedDeferred = task.deferLater(
+                self.reactor, 0, self.connectionLost
+            )
+            self.disconnecting = True
+            return self._stoppedDeferred
+        else:
+            return defer.succeed(None)
+
+    @deprecated(Version("Twisted", 14, 0, 0), stopListening)
+    def loseConnection(self):
+        """
+        Close this tunnel.  Use L{TuntapPort.stopListening} instead.
+        """
+        self.stopListening().addErrback(log.err)
+
+    def connectionLost(self, reason=None):
+        """
+        Cleans up my socket.
+
+        @param reason: Ignored.  Do not use this.
+        """
+        log.msg("(Tuntap %s Closed)" % self.interface)
+        abstract.FileDescriptor.connectionLost(self, reason)
+        self.protocol.doStop()
+        self.connected = 0
+        self._system.close(self._fileno)
+        self._fileno = -1
+
+    def logPrefix(self):
+        """
+        Returns the name of my class, to prefix log entries with.
+        """
+        return self.logstr
+
+    def getHost(self):
+        """
+        Get the local address of this L{TuntapPort}.
+
+        @return: A L{TunnelAddress} which describes the tunnel device to which
+            this object is bound.
+        @rtype: L{TunnelAddress}
+        """
+        return TunnelAddress(self._mode, self.interface)
diff --git a/contrib/python/Twisted/py3/twisted/persisted/__init__.py b/contrib/python/Twisted/py3/twisted/persisted/__init__.py
new file mode 100644
index 00000000000..55893b8cb9c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/persisted/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Persisted: Utilities for managing persistence.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/persisted/_token.py b/contrib/python/Twisted/py3/twisted/persisted/_token.py
new file mode 100644
index 00000000000..0a0453212f1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/persisted/_token.py
@@ -0,0 +1,150 @@
+"""
+FIXME:https://github.com/twisted/twisted/issues/3843
+This can be removed once t.persisted.aot is removed.
+New code should not make use of this.
+
+Token constants.
+vendored from https://github.com/python/cpython/blob/6b825c1b8a14460641ca6f1647d83005c68199aa/Lib/token.py
+Licence: https://docs.python.org/3/license.html
+"""
+# Auto-generated by Tools/scripts/generate_token.py
+
+__all__ = ["tok_name", "ISTERMINAL", "ISNONTERMINAL", "ISEOF"]
+
+ENDMARKER = 0
+NAME = 1
+NUMBER = 2
+STRING = 3
+NEWLINE = 4
+INDENT = 5
+DEDENT = 6
+LPAR = 7
+RPAR = 8
+LSQB = 9
+RSQB = 10
+COLON = 11
+COMMA = 12
+SEMI = 13
+PLUS = 14
+MINUS = 15
+STAR = 16
+SLASH = 17
+VBAR = 18
+AMPER = 19
+LESS = 20
+GREATER = 21
+EQUAL = 22
+DOT = 23
+PERCENT = 24
+LBRACE = 25
+RBRACE = 26
+EQEQUAL = 27
+NOTEQUAL = 28
+LESSEQUAL = 29
+GREATEREQUAL = 30
+TILDE = 31
+CIRCUMFLEX = 32
+LEFTSHIFT = 33
+RIGHTSHIFT = 34
+DOUBLESTAR = 35
+PLUSEQUAL = 36
+MINEQUAL = 37
+STAREQUAL = 38
+SLASHEQUAL = 39
+PERCENTEQUAL = 40
+AMPEREQUAL = 41
+VBAREQUAL = 42
+CIRCUMFLEXEQUAL = 43
+LEFTSHIFTEQUAL = 44
+RIGHTSHIFTEQUAL = 45
+DOUBLESTAREQUAL = 46
+DOUBLESLASH = 47
+DOUBLESLASHEQUAL = 48
+AT = 49
+ATEQUAL = 50
+RARROW = 51
+ELLIPSIS = 52
+COLONEQUAL = 53
+OP = 54
+AWAIT = 55
+ASYNC = 56
+TYPE_IGNORE = 57
+TYPE_COMMENT = 58
+SOFT_KEYWORD = 59
+# These aren't used by the C tokenizer but are needed for tokenize.py
+ERRORTOKEN = 60
+COMMENT = 61
+NL = 62
+ENCODING = 63
+N_TOKENS = 64
+# Special definitions for cooperation with parser
+NT_OFFSET = 256
+
+tok_name = {
+    value: name
+    for name, value in globals().items()
+    if isinstance(value, int) and not name.startswith("_")
+}
+__all__.extend(tok_name.values())
+
+EXACT_TOKEN_TYPES = {
+    "!=": NOTEQUAL,
+    "%": PERCENT,
+    "%=": PERCENTEQUAL,
+    "&": AMPER,
+    "&=": AMPEREQUAL,
+    "(": LPAR,
+    ")": RPAR,
+    "*": STAR,
+    "**": DOUBLESTAR,
+    "**=": DOUBLESTAREQUAL,
+    "*=": STAREQUAL,
+    "+": PLUS,
+    "+=": PLUSEQUAL,
+    ",": COMMA,
+    "-": MINUS,
+    "-=": MINEQUAL,
+    "->": RARROW,
+    ".": DOT,
+    "...": ELLIPSIS,
+    "/": SLASH,
+    "//": DOUBLESLASH,
+    "//=": DOUBLESLASHEQUAL,
+    "/=": SLASHEQUAL,
+    ":": COLON,
+    ":=": COLONEQUAL,
+    ";": SEMI,
+    "<": LESS,
+    "<<": LEFTSHIFT,
+    "<<=": LEFTSHIFTEQUAL,
+    "<=": LESSEQUAL,
+    "=": EQUAL,
+    "==": EQEQUAL,
+    ">": GREATER,
+    ">=": GREATEREQUAL,
+    ">>": RIGHTSHIFT,
+    ">>=": RIGHTSHIFTEQUAL,
+    "@": AT,
+    "@=": ATEQUAL,
+    "[": LSQB,
+    "]": RSQB,
+    "^": CIRCUMFLEX,
+    "^=": CIRCUMFLEXEQUAL,
+    "{": LBRACE,
+    "|": VBAR,
+    "|=": VBAREQUAL,
+    "}": RBRACE,
+    "~": TILDE,
+}
+
+
+def ISTERMINAL(x):
+    return x < NT_OFFSET
+
+
+def ISNONTERMINAL(x):
+    return x >= NT_OFFSET
+
+
+def ISEOF(x):
+    return x == ENDMARKER
diff --git a/contrib/python/Twisted/py3/twisted/persisted/_tokenize.py b/contrib/python/Twisted/py3/twisted/persisted/_tokenize.py
new file mode 100644
index 00000000000..2ae94292a04
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/persisted/_tokenize.py
@@ -0,0 +1,897 @@
+"""
+FIXME:https://github.com/twisted/twisted/issues/3843
+This can be removed once t.persisted.aot is removed.
+New code should not make use of this.
+
+Tokenization help for Python programs.
+vendored from https://github.com/python/cpython/blob/6b825c1b8a14460641ca6f1647d83005c68199aa/Lib/tokenize.py
+Licence: https://docs.python.org/3/license.html
+
+tokenize(readline) is a generator that breaks a stream of bytes into
+Python tokens.  It decodes the bytes according to PEP-0263 for
+determining source file encoding.
+
+It accepts a readline-like method which is called repeatedly to get the
+next line of input (or b"" for EOF).  It generates 5-tuples with these
+members:
+
+    the token type (see token.py)
+    the token (a string)
+    the starting (row, column) indices of the token (a 2-tuple of ints)
+    the ending (row, column) indices of the token (a 2-tuple of ints)
+    the original line (string)
+
+It is designed to match the working of the Python tokenizer exactly, except
+that it produces COMMENT tokens for comments and gives type OP for all
+operators.  Additionally, all token lists start with an ENCODING token
+which tells you which encoding was used to decode the bytes stream.
+"""
+
+__author__ = "Ka-Ping Yee <ping@lfw.org>"
+__credits__ = (
+    "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, "
+    "Skip Montanaro, Raymond Hettinger, Trent Nelson, "
+    "Michael Foord"
+)
+import collections
+import functools
+import itertools as _itertools
+import re
+import sys
+from builtins import open as _builtin_open
+from codecs import BOM_UTF8, lookup
+from io import TextIOWrapper
+
+from ._token import (
+    AMPER,
+    AMPEREQUAL,
+    ASYNC,
+    AT,
+    ATEQUAL,
+    AWAIT,
+    CIRCUMFLEX,
+    CIRCUMFLEXEQUAL,
+    COLON,
+    COLONEQUAL,
+    COMMA,
+    COMMENT,
+    DEDENT,
+    DOT,
+    DOUBLESLASH,
+    DOUBLESLASHEQUAL,
+    DOUBLESTAR,
+    DOUBLESTAREQUAL,
+    ELLIPSIS,
+    ENCODING,
+    ENDMARKER,
+    EQEQUAL,
+    EQUAL,
+    ERRORTOKEN,
+    EXACT_TOKEN_TYPES,
+    GREATER,
+    GREATEREQUAL,
+    INDENT,
+    ISEOF,
+    ISNONTERMINAL,
+    ISTERMINAL,
+    LBRACE,
+    LEFTSHIFT,
+    LEFTSHIFTEQUAL,
+    LESS,
+    LESSEQUAL,
+    LPAR,
+    LSQB,
+    MINEQUAL,
+    MINUS,
+    N_TOKENS,
+    NAME,
+    NEWLINE,
+    NL,
+    NOTEQUAL,
+    NT_OFFSET,
+    NUMBER,
+    OP,
+    PERCENT,
+    PERCENTEQUAL,
+    PLUS,
+    PLUSEQUAL,
+    RARROW,
+    RBRACE,
+    RIGHTSHIFT,
+    RIGHTSHIFTEQUAL,
+    RPAR,
+    RSQB,
+    SEMI,
+    SLASH,
+    SLASHEQUAL,
+    SOFT_KEYWORD,
+    STAR,
+    STAREQUAL,
+    STRING,
+    TILDE,
+    TYPE_COMMENT,
+    TYPE_IGNORE,
+    VBAR,
+    VBAREQUAL,
+    tok_name,
+)
+
+__all__ = [
+    "tok_name",
+    "ISTERMINAL",
+    "ISNONTERMINAL",
+    "ISEOF",
+    "ENDMARKER",
+    "NAME",
+    "NUMBER",
+    "STRING",
+    "NEWLINE",
+    "INDENT",
+    "DEDENT",
+    "LPAR",
+    "RPAR",
+    "LSQB",
+    "RSQB",
+    "COLON",
+    "COMMA",
+    "SEMI",
+    "PLUS",
+    "MINUS",
+    "STAR",
+    "SLASH",
+    "VBAR",
+    "AMPER",
+    "LESS",
+    "GREATER",
+    "EQUAL",
+    "DOT",
+    "PERCENT",
+    "LBRACE",
+    "RBRACE",
+    "EQEQUAL",
+    "NOTEQUAL",
+    "LESSEQUAL",
+    "GREATEREQUAL",
+    "TILDE",
+    "CIRCUMFLEX",
+    "LEFTSHIFT",
+    "RIGHTSHIFT",
+    "DOUBLESTAR",
+    "PLUSEQUAL",
+    "MINEQUAL",
+    "STAREQUAL",
+    "SLASHEQUAL",
+    "PERCENTEQUAL",
+    "AMPEREQUAL",
+    "VBAREQUAL",
+    "CIRCUMFLEXEQUAL",
+    "LEFTSHIFTEQUAL",
+    "RIGHTSHIFTEQUAL",
+    "DOUBLESTAREQUAL",
+    "DOUBLESLASH",
+    "DOUBLESLASHEQUAL",
+    "AT",
+    "ATEQUAL",
+    "RARROW",
+    "ELLIPSIS",
+    "COLONEQUAL",
+    "OP",
+    "AWAIT",
+    "ASYNC",
+    "TYPE_IGNORE",
+    "TYPE_COMMENT",
+    "SOFT_KEYWORD",
+    "ERRORTOKEN",
+    "COMMENT",
+    "NL",
+    "ENCODING",
+    "N_TOKENS",
+    "NT_OFFSET",
+    "tokenize",
+    "generate_tokens",
+    "detect_encoding",
+    "untokenize",
+    "TokenInfo",
+]
+
+cookie_re = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)", re.ASCII)
+blank_re = re.compile(rb"^[ \t\f]*(?:[#\r\n]|$)", re.ASCII)
+
+
+class TokenInfo(collections.namedtuple("TokenInfo", "type string start end line")):
+    def __repr__(self):
+        annotated_type = "%d (%s)" % (self.type, tok_name[self.type])
+        return (
+            "TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)"
+            % self._replace(type=annotated_type)
+        )
+
+    @property
+    def exact_type(self):
+        if self.type == OP and self.string in EXACT_TOKEN_TYPES:
+            return EXACT_TOKEN_TYPES[self.string]
+        else:
+            return self.type
+
+
+def group(*choices):
+    return "(" + "|".join(choices) + ")"
+
+
+def any(*choices):
+    return group(*choices) + "*"
+
+
+def maybe(*choices):
+    return group(*choices) + "?"
+
+
+# Note: we use unicode matching for names ("\w") but ascii matching for
+# number literals.
+Whitespace = r"[ \f\t]*"
+Comment = r"#[^\r\n]*"
+Ignore = Whitespace + any(r"\\\r?\n" + Whitespace) + maybe(Comment)
+Name = r"\w+"
+
+Hexnumber = r"0[xX](?:_?[0-9a-fA-F])+"
+Binnumber = r"0[bB](?:_?[01])+"
+Octnumber = r"0[oO](?:_?[0-7])+"
+Decnumber = r"(?:0(?:_?0)*|[1-9](?:_?[0-9])*)"
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+Exponent = r"[eE][-+]?[0-9](?:_?[0-9])*"
+Pointfloat = group(
+    r"[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?", r"\.[0-9](?:_?[0-9])*"
+) + maybe(Exponent)
+Expfloat = r"[0-9](?:_?[0-9])*" + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group(r"[0-9](?:_?[0-9])*[jJ]", Floatnumber + r"[jJ]")
+Number = group(Imagnumber, Floatnumber, Intnumber)
+
+
+# Return the empty string, plus all of the valid string prefixes.
+def _all_string_prefixes():
+    # The valid string prefixes. Only contain the lower case versions,
+    #  and don't contain any permutations (include 'fr', but not
+    #  'rf'). The various permutations will be generated.
+    _valid_string_prefixes = ["b", "r", "u", "f", "br", "fr"]
+    # if we add binary f-strings, add: ['fb', 'fbr']
+    result = {""}
+    for prefix in _valid_string_prefixes:
+        for t in _itertools.permutations(prefix):
+            # create a list with upper and lower versions of each
+            #  character
+            for u in _itertools.product(*[(c, c.upper()) for c in t]):
+                result.add("".join(u))
+    return result
+
+
+@functools.lru_cache(None)
+def _compile(expr):
+    return re.compile(expr, re.UNICODE)
+
+
+# Note that since _all_string_prefixes includes the empty string,
+#  StringPrefix can be the empty string (making it optional).
+StringPrefix = group(*_all_string_prefixes())
+
+# Tail end of ' string.
+Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
+# Tail end of " string.
+Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
+# Tail end of ''' string.
+Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+# Tail end of """ string.
+Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
+Triple = group(StringPrefix + "'''", StringPrefix + '"""')
+# Single-line ' or " string.
+String = group(
+    StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+    StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"',
+)
+
+# Sorting in reverse order puts the long operators before their prefixes.
+# Otherwise if = came before ==, == would get recognized as two instances
+# of =.
+Special = group(*(re.escape(x) for x in sorted(EXACT_TOKEN_TYPES, reverse=True)))
+Funny = group(r"\r?\n", Special)
+
+PlainToken = group(Number, Funny, String, Name)
+Token = Ignore + PlainToken
+
+# First (or only) line of ' or " string.
+ContStr = group(
+    StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r"\\\r?\n"),
+    StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r"\\\r?\n"),
+)
+PseudoExtras = group(r"\\\r?\n|\Z", Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
+
+# For a given string prefix plus quotes, endpats maps it to a regex
+#  to match the remainder of that string. _prefix can be empty, for
+#  a normal single or triple quoted string (with no prefix).
+endpats = {}
+for _prefix in _all_string_prefixes():
+    endpats[_prefix + "'"] = Single
+    endpats[_prefix + '"'] = Double
+    endpats[_prefix + "'''"] = Single3
+    endpats[_prefix + '"""'] = Double3
+del _prefix
+
+# A set of all of the single and triple quoted string prefixes,
+#  including the opening quotes.
+single_quoted = set()
+triple_quoted = set()
+for t in _all_string_prefixes():
+    for u in (t + '"', t + "'"):
+        single_quoted.add(u)
+    for u in (t + '"""', t + "'''"):
+        triple_quoted.add(u)
+del t, u
+
+tabsize = 8
+
+
+class TokenError(Exception):
+    pass
+
+
+class StopTokenizing(Exception):
+    pass
+
+
+class Untokenizer:
+    def __init__(self):
+        self.tokens = []
+        self.prev_row = 1
+        self.prev_col = 0
+        self.encoding = None
+
+    def add_whitespace(self, start):
+        row, col = start
+        if row < self.prev_row or row == self.prev_row and col < self.prev_col:
+            raise ValueError(
+                "start ({},{}) precedes previous end ({},{})".format(
+                    row, col, self.prev_row, self.prev_col
+                )
+            )
+        row_offset = row - self.prev_row
+        if row_offset:
+            self.tokens.append("\\\n" * row_offset)
+            self.prev_col = 0
+        col_offset = col - self.prev_col
+        if col_offset:
+            self.tokens.append(" " * col_offset)
+
+    def untokenize(self, iterable):
+        it = iter(iterable)
+        indents = []
+        startline = False
+        for t in it:
+            if len(t) == 2:
+                self.compat(t, it)
+                break
+            tok_type, token, start, end, line = t
+            if tok_type == ENCODING:
+                self.encoding = token
+                continue
+            if tok_type == ENDMARKER:
+                break
+            if tok_type == INDENT:
+                indents.append(token)
+                continue
+            elif tok_type == DEDENT:
+                indents.pop()
+                self.prev_row, self.prev_col = end
+                continue
+            elif tok_type in (NEWLINE, NL):
+                startline = True
+            elif startline and indents:
+                indent = indents[-1]
+                if start[1] >= len(indent):
+                    self.tokens.append(indent)
+                    self.prev_col = len(indent)
+                startline = False
+            self.add_whitespace(start)
+            self.tokens.append(token)
+            self.prev_row, self.prev_col = end
+            if tok_type in (NEWLINE, NL):
+                self.prev_row += 1
+                self.prev_col = 0
+        return "".join(self.tokens)
+
+    def compat(self, token, iterable):
+        indents = []
+        toks_append = self.tokens.append
+        startline = token[0] in (NEWLINE, NL)
+        prevstring = False
+
+        for tok in _itertools.chain([token], iterable):
+            toknum, tokval = tok[:2]
+            if toknum == ENCODING:
+                self.encoding = tokval
+                continue
+
+            if toknum in (NAME, NUMBER):
+                tokval += " "
+
+            # Insert a space between two consecutive strings
+            if toknum == STRING:
+                if prevstring:
+                    tokval = " " + tokval
+                prevstring = True
+            else:
+                prevstring = False
+
+            if toknum == INDENT:
+                indents.append(tokval)
+                continue
+            elif toknum == DEDENT:
+                indents.pop()
+                continue
+            elif toknum in (NEWLINE, NL):
+                startline = True
+            elif startline and indents:
+                toks_append(indents[-1])
+                startline = False
+            toks_append(tokval)
+
+
+def untokenize(iterable):
+    """Transform tokens back into Python source code.
+    It returns a bytes object, encoded using the ENCODING
+    token, which is the first token sequence output by tokenize.
+
+    Each element returned by the iterable must be a token sequence
+    with at least two elements, a token number and token value.  If
+    only two tokens are passed, the resulting output is poor.
+
+    Round-trip invariant for full input:
+        Untokenized source will match input source exactly
+
+    Round-trip invariant for limited input:
+        # Output bytes will tokenize back to the input
+        t1 = [tok[:2] for tok in tokenize(f.readline)]
+        newcode = untokenize(t1)
+        readline = BytesIO(newcode).readline
+        t2 = [tok[:2] for tok in tokenize(readline)]
+        assert t1 == t2
+    """
+    ut = Untokenizer()
+    out = ut.untokenize(iterable)
+    if ut.encoding is not None:
+        out = out.encode(ut.encoding)
+    return out
+
+
+def _get_normal_name(orig_enc):
+    """Imitates get_normal_name in tokenizer.c."""
+    # Only care about the first 12 characters.
+    enc = orig_enc[:12].lower().replace("_", "-")
+    if enc == "utf-8" or enc.startswith("utf-8-"):
+        return "utf-8"
+    if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or enc.startswith(
+        ("latin-1-", "iso-8859-1-", "iso-latin-1-")
+    ):
+        return "iso-8859-1"
+    return orig_enc
+
+
+def detect_encoding(readline):
+    """
+    The detect_encoding() function is used to detect the encoding that should
+    be used to decode a Python source file.  It requires one argument, readline,
+    in the same way as the tokenize() generator.
+
+    It will call readline a maximum of twice, and return the encoding used
+    (as a string) and a list of any lines (left as bytes) it has read in.
+
+    It detects the encoding from the presence of a utf-8 bom or an encoding
+    cookie as specified in pep-0263.  If both a bom and a cookie are present,
+    but disagree, a SyntaxError will be raised.  If the encoding cookie is an
+    invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
+    'utf-8-sig' is returned.
+
+    If no encoding is specified, then the default of 'utf-8' will be returned.
+    """
+    try:
+        filename = readline.__self__.name
+    except AttributeError:
+        filename = None
+    bom_found = False
+    encoding = None
+    default = "utf-8"
+
+    def read_or_stop():
+        try:
+            return readline()
+        except StopIteration:
+            return b""
+
+    def find_cookie(line):
+        try:
+            # Decode as UTF-8. Either the line is an encoding declaration,
+            # in which case it should be pure ASCII, or it must be UTF-8
+            # per default encoding.
+            line_string = line.decode("utf-8")
+        except UnicodeDecodeError:
+            msg = "invalid or missing encoding declaration"
+            if filename is not None:
+                msg = "{} for {!r}".format(msg, filename)
+            raise SyntaxError(msg)
+
+        match = cookie_re.match(line_string)
+        if not match:
+            return None
+        encoding = _get_normal_name(match.group(1))
+        try:
+            lookup(encoding)
+        except LookupError:
+            # This behaviour mimics the Python interpreter
+            if filename is None:
+                msg = "unknown encoding: " + encoding
+            else:
+                msg = "unknown encoding for {!r}: {}".format(filename, encoding)
+            raise SyntaxError(msg)
+
+        if bom_found:
+            if encoding != "utf-8":
+                # This behaviour mimics the Python interpreter
+                if filename is None:
+                    msg = "encoding problem: utf-8"
+                else:
+                    msg = "encoding problem for {!r}: utf-8".format(filename)
+                raise SyntaxError(msg)
+            encoding += "-sig"
+        return encoding
+
+    first = read_or_stop()
+    if first.startswith(BOM_UTF8):
+        bom_found = True
+        first = first[3:]
+        default = "utf-8-sig"
+    if not first:
+        return default, []
+
+    encoding = find_cookie(first)
+    if encoding:
+        return encoding, [first]
+    if not blank_re.match(first):
+        return default, [first]
+
+    second = read_or_stop()
+    if not second:
+        return default, [first]
+
+    encoding = find_cookie(second)
+    if encoding:
+        return encoding, [first, second]
+
+    return default, [first, second]
+
+
+def open(filename):
+    """Open a file in read only mode using the encoding detected by
+    detect_encoding().
+    """
+    buffer = _builtin_open(filename, "rb")
+    try:
+        encoding, lines = detect_encoding(buffer.readline)
+        buffer.seek(0)
+        text = TextIOWrapper(buffer, encoding, line_buffering=True)
+        text.mode = "r"
+        return text
+    except BaseException:
+        buffer.close()
+        raise
+
+
+def tokenize(readline):
+    """
+    The tokenize() generator requires one argument, readline, which
+    must be a callable object which provides the same interface as the
+    readline() method of built-in file objects.  Each call to the function
+    should return one line of input as bytes.  Alternatively, readline
+    can be a callable function terminating with StopIteration:
+        readline = open(myfile, 'rb').__next__  # Example of alternate readline
+
+    The generator produces 5-tuples with these members: the token type; the
+    token string; a 2-tuple (srow, scol) of ints specifying the row and
+    column where the token begins in the source; a 2-tuple (erow, ecol) of
+    ints specifying the row and column where the token ends in the source;
+    and the line on which the token was found.  The line passed is the
+    physical line.
+
+    The first token sequence will always be an ENCODING token
+    which tells you which encoding was used to decode the bytes stream.
+    """
+    encoding, consumed = detect_encoding(readline)
+    empty = _itertools.repeat(b"")
+    rl_gen = _itertools.chain(consumed, iter(readline, b""), empty)
+    return _tokenize(rl_gen.__next__, encoding)
+
+
+def _tokenize(readline, encoding):
+    strstart = None
+    endprog = None
+    lnum = parenlev = continued = 0
+    numchars = "0123456789"
+    contstr, needcont = "", 0
+    contline = None
+    indents = [0]
+
+    if encoding is not None:
+        if encoding == "utf-8-sig":
+            # BOM will already have been stripped.
+            encoding = "utf-8"
+        yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), "")
+    last_line = b""
+    line = b""
+    while True:  # loop over lines in stream
+        try:
+            # We capture the value of the line variable here because
+            # readline uses the empty string '' to signal end of input,
+            # hence `line` itself will always be overwritten at the end
+            # of this loop.
+            last_line = line
+            line = readline()
+        except StopIteration:
+            line = b""
+
+        if encoding is not None:
+            line = line.decode(encoding)
+        lnum += 1
+        pos, max = 0, len(line)
+
+        if contstr:  # continued string
+            if not line:
+                raise TokenError("EOF in multi-line string", strstart)
+            endmatch = endprog.match(line)
+            if endmatch:
+                pos = end = endmatch.end(0)
+                yield TokenInfo(
+                    STRING, contstr + line[:end], strstart, (lnum, end), contline + line
+                )
+                contstr, needcont = "", 0
+                contline = None
+            elif needcont and line[-2:] != "\\\n" and line[-3:] != "\\\r\n":
+                yield TokenInfo(
+                    ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), contline
+                )
+                contstr = ""
+                contline = None
+                continue
+            else:
+                contstr = contstr + line
+                contline = contline + line
+                continue
+
+        elif parenlev == 0 and not continued:  # new statement
+            if not line:
+                break
+            column = 0
+            while pos < max:  # measure leading whitespace
+                if line[pos] == " ":
+                    column += 1
+                elif line[pos] == "\t":
+                    column = (column // tabsize + 1) * tabsize
+                elif line[pos] == "\f":
+                    column = 0
+                else:
+                    break
+                pos += 1
+            if pos == max:
+                break
+
+            if line[pos] in "#\r\n":  # skip comments or blank lines
+                if line[pos] == "#":
+                    comment_token = line[pos:].rstrip("\r\n")
+                    yield TokenInfo(
+                        COMMENT,
+                        comment_token,
+                        (lnum, pos),
+                        (lnum, pos + len(comment_token)),
+                        line,
+                    )
+                    pos += len(comment_token)
+
+                yield TokenInfo(NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
+                continue
+
+            if column > indents[-1]:  # count indents or dedents
+                indents.append(column)
+                yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+            while column < indents[-1]:
+                if column not in indents:
+                    raise IndentationError(
+                        "unindent does not match any outer indentation level",
+                        ("<tokenize>", lnum, pos, line),
+                    )
+                indents = indents[:-1]
+
+                yield TokenInfo(DEDENT, "", (lnum, pos), (lnum, pos), line)
+
+        else:  # continued statement
+            if not line:
+                raise TokenError("EOF in multi-line statement", (lnum, 0))
+            continued = 0
+
+        while pos < max:
+            pseudomatch = _compile(PseudoToken).match(line, pos)
+            if pseudomatch:  # scan for tokens
+                start, end = pseudomatch.span(1)
+                spos, epos, pos = (lnum, start), (lnum, end), end
+                if start == end:
+                    continue
+                token, initial = line[start:end], line[start]
+
+                if initial in numchars or (  # ordinary number
+                    initial == "." and token != "." and token != "..."
+                ):
+                    yield TokenInfo(NUMBER, token, spos, epos, line)
+                elif initial in "\r\n":
+                    if parenlev > 0:
+                        yield TokenInfo(NL, token, spos, epos, line)
+                    else:
+                        yield TokenInfo(NEWLINE, token, spos, epos, line)
+
+                elif initial == "#":
+                    assert not token.endswith("\n")
+                    yield TokenInfo(COMMENT, token, spos, epos, line)
+
+                elif token in triple_quoted:
+                    endprog = _compile(endpats[token])
+                    endmatch = endprog.match(line, pos)
+                    if endmatch:  # all on one line
+                        pos = endmatch.end(0)
+                        token = line[start:pos]
+                        yield TokenInfo(STRING, token, spos, (lnum, pos), line)
+                    else:
+                        strstart = (lnum, start)  # multiple lines
+                        contstr = line[start:]
+                        contline = line
+                        break
+
+                # Check up to the first 3 chars of the token to see if
+                #  they're in the single_quoted set. If so, they start
+                #  a string.
+                # We're using the first 3, because we're looking for
+                #  "rb'" (for example) at the start of the token. If
+                #  we switch to longer prefixes, this needs to be
+                #  adjusted.
+                # Note that initial == token[:1].
+                # Also note that single quote checking must come after
+                #  triple quote checking (above).
+                elif (
+                    initial in single_quoted
+                    or token[:2] in single_quoted
+                    or token[:3] in single_quoted
+                ):
+                    if token[-1] == "\n":  # continued string
+                        strstart = (lnum, start)
+                        # Again, using the first 3 chars of the
+                        #  token. This is looking for the matching end
+                        #  regex for the correct type of quote
+                        #  character. So it's really looking for
+                        #  endpats["'"] or endpats['"'], by trying to
+                        #  skip string prefix characters, if any.
+                        endprog = _compile(
+                            endpats.get(initial)
+                            or endpats.get(token[1])
+                            or endpats.get(token[2])
+                        )
+                        contstr, needcont = line[start:], 1
+                        contline = line
+                        break
+                    else:  # ordinary string
+                        yield TokenInfo(STRING, token, spos, epos, line)
+
+                elif initial.isidentifier():  # ordinary name
+                    yield TokenInfo(NAME, token, spos, epos, line)
+                elif initial == "\\":  # continued stmt
+                    continued = 1
+                else:
+                    if initial in "([{":
+                        parenlev += 1
+                    elif initial in ")]}":
+                        parenlev -= 1
+                    yield TokenInfo(OP, token, spos, epos, line)
+            else:
+                yield TokenInfo(
+                    ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line
+                )
+                pos += 1
+
+    # Add an implicit NEWLINE if the input doesn't end in one
+    if (
+        last_line
+        and last_line[-1] not in "\r\n"
+        and not last_line.strip().startswith("#")
+    ):
+        yield TokenInfo(
+            NEWLINE, "", (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), ""
+        )
+    for indent in indents[1:]:  # pop remaining indent levels
+        yield TokenInfo(DEDENT, "", (lnum, 0), (lnum, 0), "")
+    yield TokenInfo(ENDMARKER, "", (lnum, 0), (lnum, 0), "")
+
+
+def generate_tokens(readline):
+    """Tokenize a source reading Python code as unicode strings.
+
+    This has the same API as tokenize(), except that it expects the *readline*
+    callable to return str objects instead of bytes.
+    """
+    return _tokenize(readline, None)
+
+
+def main():
+    import argparse
+
+    # Helper error handling routines
+    def perror(message):
+        sys.stderr.write(message)
+        sys.stderr.write("\n")
+
+    def error(message, filename=None, location=None):
+        if location:
+            args = (filename,) + location + (message,)
+            perror("%s:%d:%d: error: %s" % args)
+        elif filename:
+            perror("%s: error: %s" % (filename, message))
+        else:
+            perror("error: %s" % message)
+        sys.exit(1)
+
+    # Parse the arguments and options
+    parser = argparse.ArgumentParser(prog="python -m tokenize")
+    parser.add_argument(
+        dest="filename",
+        nargs="?",
+        metavar="filename.py",
+        help="the file to tokenize; defaults to stdin",
+    )
+    parser.add_argument(
+        "-e",
+        "--exact",
+        dest="exact",
+        action="store_true",
+        help="display token names using the exact type",
+    )
+    args = parser.parse_args()
+
+    try:
+        # Tokenize the input
+        if args.filename:
+            filename = args.filename
+            with _builtin_open(filename, "rb") as f:
+                tokens = list(tokenize(f.readline))
+        else:
+            filename = "<stdin>"
+            tokens = _tokenize(sys.stdin.readline, None)
+
+        # Output the tokenization
+        for token in tokens:
+            token_type = token.type
+            if args.exact:
+                token_type = token.exact_type
+            token_range = "%d,%d-%d,%d:" % (token.start + token.end)
+            print("%-20s%-15s%-15r" % (token_range, tok_name[token_type], token.string))
+    except IndentationError as err:
+        line, column = err.args[1][1:3]
+        error(err.args[0], filename, (line, column))
+    except TokenError as err:
+        line, column = err.args[1]
+        error(err.args[0], filename, (line, column))
+    except SyntaxError as err:
+        error(err, filename)
+    except OSError as err:
+        error(err)
+    except KeyboardInterrupt:
+        print("interrupted\n")
+    except Exception as err:
+        perror("unexpected error: %s" % err)
+        raise
+
+
+if __name__ == "__main__":
+    main()
diff --git a/contrib/python/Twisted/py3/twisted/persisted/aot.py b/contrib/python/Twisted/py3/twisted/persisted/aot.py
new file mode 100644
index 00000000000..f46866bc025
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/persisted/aot.py
@@ -0,0 +1,631 @@
+# -*- test-case-name: twisted.test.test_persisted -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+AOT: Abstract Object Trees
+The source-code-marshallin'est abstract-object-serializin'est persister
+this side of Marmalade!
+"""
+
+
+import copyreg as copy_reg
+import re
+import types
+
+from twisted.persisted import crefutil
+from twisted.python import log, reflect
+from twisted.python.compat import _constructMethod
+
+# tokenize from py3.11 is vendored to work around https://github.com/python/cpython/issues/105238
+# on 3.12
+from ._tokenize import generate_tokens as tokenize
+
+###########################
+# Abstract Object Classes #
+###########################
+
+# "\0" in a getSource means "insert variable-width indention here".
+# see `indentify'.
+
+
+class Named:
+    def __init__(self, name):
+        self.name = name
+
+
+class Class(Named):
+    def getSource(self):
+        return "Class(%r)" % self.name
+
+
+class Function(Named):
+    def getSource(self):
+        return "Function(%r)" % self.name
+
+
+class Module(Named):
+    def getSource(self):
+        return "Module(%r)" % self.name
+
+
+class InstanceMethod:
+    def __init__(self, name, klass, inst):
+        if not (
+            isinstance(inst, Ref)
+            or isinstance(inst, Instance)
+            or isinstance(inst, Deref)
+        ):
+            raise TypeError("%s isn't an Instance, Ref, or Deref!" % inst)
+        self.name = name
+        self.klass = klass
+        self.instance = inst
+
+    def getSource(self):
+        return "InstanceMethod({!r}, {!r}, \n\0{})".format(
+            self.name,
+            self.klass,
+            prettify(self.instance),
+        )
+
+
+class _NoStateObj:
+    pass
+
+
+NoStateObj = _NoStateObj()
+
+_SIMPLE_BUILTINS = [
+    bool,
+    bytes,
+    str,
+    int,
+    float,
+    complex,
+    type(None),
+    slice,
+    type(Ellipsis),
+]
+
+
+class Instance:
+    def __init__(self, className, __stateObj__=NoStateObj, **state):
+        if not isinstance(className, str):
+            raise TypeError("%s isn't a string!" % className)
+        self.klass = className
+        if __stateObj__ is not NoStateObj:
+            self.state = __stateObj__
+            self.stateIsDict = 0
+        else:
+            self.state = state
+            self.stateIsDict = 1
+
+    def getSource(self):
+        # XXX make state be foo=bar instead of a dict.
+        if self.stateIsDict:
+            stateDict = self.state
+        elif isinstance(self.state, Ref) and isinstance(self.state.obj, dict):
+            stateDict = self.state.obj
+        else:
+            stateDict = None
+        if stateDict is not None:
+            try:
+                return f"Instance({self.klass!r}, {dictToKW(stateDict)})"
+            except NonFormattableDict:
+                return f"Instance({self.klass!r}, {prettify(stateDict)})"
+        return f"Instance({self.klass!r}, {prettify(self.state)})"
+
+
+class Ref:
+    def __init__(self, *args):
+        # blargh, lame.
+        if len(args) == 2:
+            self.refnum = args[0]
+            self.obj = args[1]
+        elif not args:
+            self.refnum = None
+            self.obj = None
+
+    def setRef(self, num):
+        if self.refnum:
+            raise ValueError(f"Error setting id {num}, I already have {self.refnum}")
+        self.refnum = num
+
+    def setObj(self, obj):
+        if self.obj:
+            raise ValueError(f"Error setting obj {obj}, I already have {self.obj}")
+        self.obj = obj
+
+    def getSource(self):
+        if self.obj is None:
+            raise RuntimeError(
+                "Don't try to display me before setting an object on me!"
+            )
+        if self.refnum:
+            return "Ref(%d, \n\0%s)" % (self.refnum, prettify(self.obj))
+        return prettify(self.obj)
+
+
+class Deref:
+    def __init__(self, num):
+        self.refnum = num
+
+    def getSource(self):
+        return "Deref(%d)" % self.refnum
+
+    __repr__ = getSource
+
+
+class Copyreg:
+    def __init__(self, loadfunc, state):
+        self.loadfunc = loadfunc
+        self.state = state
+
+    def getSource(self):
+        return f"Copyreg({self.loadfunc!r}, {prettify(self.state)})"
+
+
+###############
+# Marshalling #
+###############
+
+
+def getSource(ao):
+    """Pass me an AO, I'll return a nicely-formatted source representation."""
+    return indentify("app = " + prettify(ao))
+
+
+class NonFormattableDict(Exception):
+    """A dictionary was not formattable."""
+
+
+r = re.compile("[a-zA-Z_][a-zA-Z0-9_]*$")
+
+
+def dictToKW(d):
+    out = []
+    items = list(d.items())
+    items.sort()
+    for k, v in items:
+        if not isinstance(k, str):
+            raise NonFormattableDict("%r ain't a string" % k)
+        if not r.match(k):
+            raise NonFormattableDict("%r ain't an identifier" % k)
+        out.append(f"\n\0{k}={prettify(v)},")
+    return "".join(out)
+
+
+def prettify(obj):
+    if hasattr(obj, "getSource"):
+        return obj.getSource()
+    else:
+        # basic type
+        t = type(obj)
+
+        if t in _SIMPLE_BUILTINS:
+            return repr(obj)
+
+        elif t is dict:
+            out = ["{"]
+            for k, v in obj.items():
+                out.append(f"\n\0{prettify(k)}: {prettify(v)},")
+            out.append(len(obj) and "\n\0}" or "}")
+            return "".join(out)
+
+        elif t is list:
+            out = ["["]
+            for x in obj:
+                out.append("\n\0%s," % prettify(x))
+            out.append(len(obj) and "\n\0]" or "]")
+            return "".join(out)
+
+        elif t is tuple:
+            out = ["("]
+            for x in obj:
+                out.append("\n\0%s," % prettify(x))
+            out.append(len(obj) and "\n\0)" or ")")
+            return "".join(out)
+        else:
+            raise TypeError(f"Unsupported type {t} when trying to prettify {obj}.")
+
+
+def indentify(s):
+    out = []
+    stack = []
+    l = ["", s]
+    for (
+        tokenType,
+        tokenString,
+        (startRow, startColumn),
+        (endRow, endColumn),
+        logicalLine,
+    ) in tokenize(l.pop):
+        if tokenString in ["[", "(", "{"]:
+            stack.append(tokenString)
+        elif tokenString in ["]", ")", "}"]:
+            stack.pop()
+        if tokenString == "\0":
+            out.append("  " * len(stack))
+        else:
+            out.append(tokenString)
+    return "".join(out)
+
+
+###########
+# Unjelly #
+###########
+
+
+def unjellyFromAOT(aot):
+    """
+    Pass me an Abstract Object Tree, and I'll unjelly it for you.
+    """
+    return AOTUnjellier().unjelly(aot)
+
+
+def unjellyFromSource(stringOrFile):
+    """
+    Pass me a string of code or a filename that defines an 'app' variable (in
+    terms of Abstract Objects!), and I'll execute it and unjelly the resulting
+    AOT for you, returning a newly unpersisted Application object!
+    """
+
+    ns = {
+        "Instance": Instance,
+        "InstanceMethod": InstanceMethod,
+        "Class": Class,
+        "Function": Function,
+        "Module": Module,
+        "Ref": Ref,
+        "Deref": Deref,
+        "Copyreg": Copyreg,
+    }
+
+    if hasattr(stringOrFile, "read"):
+        source = stringOrFile.read()
+    else:
+        source = stringOrFile
+    code = compile(source, "<source>", "exec")
+    eval(code, ns, ns)
+
+    if "app" in ns:
+        return unjellyFromAOT(ns["app"])
+    else:
+        raise ValueError("%s needs to define an 'app', it didn't!" % stringOrFile)
+
+
+class AOTUnjellier:
+    """I handle the unjellying of an Abstract Object Tree.
+    See AOTUnjellier.unjellyAO
+    """
+
+    def __init__(self):
+        self.references = {}
+        self.stack = []
+        self.afterUnjelly = []
+
+    ##
+    # unjelly helpers (copied pretty much directly from (now deleted) marmalade)
+    ##
+    def unjellyLater(self, node):
+        """Unjelly a node, later."""
+        d = crefutil._Defer()
+        self.unjellyInto(d, 0, node)
+        return d
+
+    def unjellyInto(self, obj, loc, ao):
+        """Utility method for unjellying one object into another.
+        This automates the handling of backreferences.
+        """
+        o = self.unjellyAO(ao)
+        obj[loc] = o
+        if isinstance(o, crefutil.NotKnown):
+            o.addDependant(obj, loc)
+        return o
+
+    def callAfter(self, callable, result):
+        if isinstance(result, crefutil.NotKnown):
+            listResult = [None]
+            result.addDependant(listResult, 1)
+        else:
+            listResult = [result]
+        self.afterUnjelly.append((callable, listResult))
+
+    def unjellyAttribute(self, instance, attrName, ao):
+        # XXX this is unused????
+        """Utility method for unjellying into instances of attributes.
+
+        Use this rather than unjellyAO unless you like surprising bugs!
+        Alternatively, you can use unjellyInto on your instance's __dict__.
+        """
+        self.unjellyInto(instance.__dict__, attrName, ao)
+
+    def unjellyAO(self, ao):
+        """Unjelly an Abstract Object and everything it contains.
+        I return the real object.
+        """
+        self.stack.append(ao)
+        t = type(ao)
+        if t in _SIMPLE_BUILTINS:
+            return ao
+
+        elif t is list:
+            l = []
+            for x in ao:
+                l.append(None)
+                self.unjellyInto(l, len(l) - 1, x)
+            return l
+
+        elif t is tuple:
+            l = []
+            tuple_ = tuple
+            for x in ao:
+                l.append(None)
+                if isinstance(self.unjellyInto(l, len(l) - 1, x), crefutil.NotKnown):
+                    tuple_ = crefutil._Tuple
+            return tuple_(l)
+
+        elif t is dict:
+            d = {}
+            for k, v in ao.items():
+                kvd = crefutil._DictKeyAndValue(d)
+                self.unjellyInto(kvd, 0, k)
+                self.unjellyInto(kvd, 1, v)
+            return d
+        else:
+            # Abstract Objects
+            c = ao.__class__
+            if c is Module:
+                return reflect.namedModule(ao.name)
+
+            elif c in [Class, Function] or issubclass(c, type):
+                return reflect.namedObject(ao.name)
+
+            elif c is InstanceMethod:
+                im_name = ao.name
+                im_class = reflect.namedObject(ao.klass)
+                im_self = self.unjellyAO(ao.instance)
+                if im_name in im_class.__dict__:
+                    if im_self is None:
+                        return getattr(im_class, im_name)
+                    elif isinstance(im_self, crefutil.NotKnown):
+                        return crefutil._InstanceMethod(im_name, im_self, im_class)
+                    else:
+                        return _constructMethod(im_class, im_name, im_self)
+                else:
+                    raise TypeError("instance method changed")
+
+            elif c is Instance:
+                klass = reflect.namedObject(ao.klass)
+                state = self.unjellyAO(ao.state)
+                inst = klass.__new__(klass)
+                if hasattr(klass, "__setstate__"):
+                    self.callAfter(inst.__setstate__, state)
+                elif isinstance(state, dict):
+                    inst.__dict__ = state
+                else:
+                    inst.__dict__ = state.__getstate__()
+                return inst
+
+            elif c is Ref:
+                o = self.unjellyAO(ao.obj)  # THIS IS CHANGING THE REF OMG
+                refkey = ao.refnum
+                ref = self.references.get(refkey)
+                if ref is None:
+                    self.references[refkey] = o
+                elif isinstance(ref, crefutil.NotKnown):
+                    ref.resolveDependants(o)
+                    self.references[refkey] = o
+                elif refkey is None:
+                    # This happens when you're unjellying from an AOT not read from source
+                    pass
+                else:
+                    raise ValueError(
+                        "Multiple references with the same ID: %s, %s, %s!"
+                        % (ref, refkey, ao)
+                    )
+                return o
+
+            elif c is Deref:
+                num = ao.refnum
+                ref = self.references.get(num)
+                if ref is None:
+                    der = crefutil._Dereference(num)
+                    self.references[num] = der
+                    return der
+                return ref
+
+            elif c is Copyreg:
+                loadfunc = reflect.namedObject(ao.loadfunc)
+                d = self.unjellyLater(ao.state).addCallback(
+                    lambda result, _l: _l(*result), loadfunc
+                )
+                return d
+            else:
+                raise TypeError("Unsupported AOT type: %s" % t)
+
+    def unjelly(self, ao):
+        try:
+            l = [None]
+            self.unjellyInto(l, 0, ao)
+            for func, v in self.afterUnjelly:
+                func(v[0])
+            return l[0]
+        except BaseException:
+            log.msg("Error jellying object! Stacktrace follows::")
+            log.msg("\n".join(map(repr, self.stack)))
+            raise
+
+
+#########
+# Jelly #
+#########
+
+
+def jellyToAOT(obj):
+    """Convert an object to an Abstract Object Tree."""
+    return AOTJellier().jelly(obj)
+
+
+def jellyToSource(obj, file=None):
+    """
+    Pass me an object and, optionally, a file object.
+    I'll convert the object to an AOT either return it (if no file was
+    specified) or write it to the file.
+    """
+
+    aot = jellyToAOT(obj)
+    if file:
+        file.write(getSource(aot).encode("utf-8"))
+    else:
+        return getSource(aot)
+
+
+def _classOfMethod(methodObject):
+    """
+    Get the associated class of the given method object.
+
+    @param methodObject: a bound method
+    @type methodObject: L{types.MethodType}
+
+    @return: a class
+    @rtype: L{type}
+    """
+    return methodObject.__self__.__class__
+
+
+def _funcOfMethod(methodObject):
+    """
+    Get the associated function of the given method object.
+
+    @param methodObject: a bound method
+    @type methodObject: L{types.MethodType}
+
+    @return: the function implementing C{methodObject}
+    @rtype: L{types.FunctionType}
+    """
+    return methodObject.__func__
+
+
+def _selfOfMethod(methodObject):
+    """
+    Get the object that a bound method is bound to.
+
+    @param methodObject: a bound method
+    @type methodObject: L{types.MethodType}
+
+    @return: the C{self} passed to C{methodObject}
+    @rtype: L{object}
+    """
+    return methodObject.__self__
+
+
+class AOTJellier:
+    def __init__(self):
+        # dict of {id(obj): (obj, node)}
+        self.prepared = {}
+        self._ref_id = 0
+        self.stack = []
+
+    def prepareForRef(self, aoref, object):
+        """I prepare an object for later referencing, by storing its id()
+        and its _AORef in a cache."""
+        self.prepared[id(object)] = aoref
+
+    def jellyToAO(self, obj):
+        """I turn an object into an AOT and return it."""
+        objType = type(obj)
+        self.stack.append(repr(obj))
+
+        # immutable: We don't care if these have multiple refs!
+        if objType in _SIMPLE_BUILTINS:
+            retval = obj
+
+        elif issubclass(objType, types.MethodType):
+            # TODO: make methods 'prefer' not to jelly the object internally,
+            # so that the object will show up where it's referenced first NOT
+            # by a method.
+            retval = InstanceMethod(
+                _funcOfMethod(obj).__name__,
+                reflect.qual(_classOfMethod(obj)),
+                self.jellyToAO(_selfOfMethod(obj)),
+            )
+
+        elif issubclass(objType, types.ModuleType):
+            retval = Module(obj.__name__)
+
+        elif issubclass(objType, type):
+            retval = Class(reflect.qual(obj))
+
+        elif objType is types.FunctionType:
+            retval = Function(reflect.fullFuncName(obj))
+
+        else:  # mutable! gotta watch for refs.
+            # Marmalade had the nicety of being able to just stick a 'reference' attribute
+            # on any Node object that was referenced, but in AOT, the referenced object
+            # is *inside* of a Ref call (Ref(num, obj) instead of
+            # <objtype ... reference="1">). The problem is, especially for built-in types,
+            # I can't just assign some attribute to them to give them a refnum. So, I have
+            # to "wrap" a Ref(..) around them later -- that's why I put *everything* that's
+            # mutable inside one. The Ref() class will only print the "Ref(..)" around an
+            # object if it has a Reference explicitly attached.
+
+            if id(obj) in self.prepared:
+                oldRef = self.prepared[id(obj)]
+                if oldRef.refnum:
+                    # it's been referenced already
+                    key = oldRef.refnum
+                else:
+                    # it hasn't been referenced yet
+                    self._ref_id = self._ref_id + 1
+                    key = self._ref_id
+                    oldRef.setRef(key)
+                return Deref(key)
+
+            retval = Ref()
+
+            def _stateFrom(state):
+                retval.setObj(
+                    Instance(reflect.qual(obj.__class__), self.jellyToAO(state))
+                )
+
+            self.prepareForRef(retval, obj)
+
+            if objType is list:
+                retval.setObj([self.jellyToAO(o) for o in obj])  # hah!
+
+            elif objType is tuple:
+                retval.setObj(tuple(map(self.jellyToAO, obj)))
+
+            elif objType is dict:
+                d = {}
+                for k, v in obj.items():
+                    d[self.jellyToAO(k)] = self.jellyToAO(v)
+                retval.setObj(d)
+
+            elif objType in copy_reg.dispatch_table:
+                unpickleFunc, state = copy_reg.dispatch_table[objType](obj)
+
+                retval.setObj(
+                    Copyreg(reflect.fullFuncName(unpickleFunc), self.jellyToAO(state))
+                )
+
+            elif hasattr(obj, "__getstate__"):
+                _stateFrom(obj.__getstate__())
+            elif hasattr(obj, "__dict__"):
+                _stateFrom(obj.__dict__)
+            else:
+                raise TypeError("Unsupported type: %s" % objType.__name__)
+
+        del self.stack[-1]
+        return retval
+
+    def jelly(self, obj):
+        try:
+            ao = self.jellyToAO(obj)
+            return ao
+        except BaseException:
+            log.msg("Error jellying object! Stacktrace follows::")
+            log.msg("\n".join(self.stack))
+            raise
diff --git a/contrib/python/Twisted/py3/twisted/persisted/crefutil.py b/contrib/python/Twisted/py3/twisted/persisted/crefutil.py
new file mode 100644
index 00000000000..5c14b6194a9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/persisted/crefutil.py
@@ -0,0 +1,160 @@
+# -*- test-case-name: twisted.test.test_persisted -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Utility classes for dealing with circular references.
+"""
+
+
+from twisted.python import log, reflect
+from twisted.python.compat import _constructMethod
+
+
+class NotKnown:
+    def __init__(self):
+        self.dependants = []
+        self.resolved = 0
+
+    def addDependant(self, mutableObject, key):
+        assert not self.resolved
+        self.dependants.append((mutableObject, key))
+
+    resolvedObject = None
+
+    def resolveDependants(self, newObject):
+        self.resolved = 1
+        self.resolvedObject = newObject
+        for mut, key in self.dependants:
+            mut[key] = newObject
+
+    def __hash__(self):
+        assert 0, "I am not to be used as a dictionary key."
+
+
+class _Container(NotKnown):
+    """
+    Helper class to resolve circular references on container objects.
+    """
+
+    def __init__(self, l, containerType):
+        """
+        @param l: The list of object which may contain some not yet referenced
+        objects.
+
+        @param containerType: A type of container objects (e.g., C{tuple} or
+            C{set}).
+        """
+        NotKnown.__init__(self)
+        self.containerType = containerType
+        self.l = l
+        self.locs = list(range(len(l)))
+        for idx in range(len(l)):
+            if not isinstance(l[idx], NotKnown):
+                self.locs.remove(idx)
+            else:
+                l[idx].addDependant(self, idx)
+        if not self.locs:
+            self.resolveDependants(self.containerType(self.l))
+
+    def __setitem__(self, n, obj):
+        """
+        Change the value of one contained objects, and resolve references if
+        all objects have been referenced.
+        """
+        self.l[n] = obj
+        if not isinstance(obj, NotKnown):
+            self.locs.remove(n)
+            if not self.locs:
+                self.resolveDependants(self.containerType(self.l))
+
+
+class _Tuple(_Container):
+    """
+    Manage tuple containing circular references. Deprecated: use C{_Container}
+    instead.
+    """
+
+    def __init__(self, l):
+        """
+        @param l: The list of object which may contain some not yet referenced
+        objects.
+        """
+        _Container.__init__(self, l, tuple)
+
+
+class _InstanceMethod(NotKnown):
+    def __init__(self, im_name, im_self, im_class):
+        NotKnown.__init__(self)
+        self.my_class = im_class
+        self.name = im_name
+        # im_self _must_ be a NotKnown
+        im_self.addDependant(self, 0)
+
+    def __call__(self, *args, **kw):
+        import traceback
+
+        log.msg(f"instance method {reflect.qual(self.my_class)}.{self.name}")
+        log.msg(f"being called with {args!r} {kw!r}")
+        traceback.print_stack(file=log.logfile)
+        assert 0
+
+    def __setitem__(self, n, obj):
+        assert n == 0, "only zero index allowed"
+        if not isinstance(obj, NotKnown):
+            method = _constructMethod(self.my_class, self.name, obj)
+            self.resolveDependants(method)
+
+
+class _DictKeyAndValue:
+    def __init__(self, dict):
+        self.dict = dict
+
+    def __setitem__(self, n, obj):
+        if n not in (1, 0):
+            raise RuntimeError("DictKeyAndValue should only ever be called with 0 or 1")
+        if n:  # value
+            self.value = obj
+        else:
+            self.key = obj
+        if hasattr(self, "key") and hasattr(self, "value"):
+            self.dict[self.key] = self.value
+
+
+class _Dereference(NotKnown):
+    def __init__(self, id):
+        NotKnown.__init__(self)
+        self.id = id
+
+
+from twisted.internet.defer import Deferred
+
+
+class _Defer(Deferred[object], NotKnown):
+    def __init__(self):
+        Deferred.__init__(self)
+        NotKnown.__init__(self)
+        self.pause()
+
+    wasset = 0
+
+    def __setitem__(self, n, obj):
+        if self.wasset:
+            raise RuntimeError(
+                "setitem should only be called once, setting {!r} to {!r}".format(
+                    n, obj
+                )
+            )
+        else:
+            self.wasset = 1
+        self.callback(obj)
+
+    def addDependant(self, dep, key):
+        # by the time I'm adding a dependant, I'm *not* adding any more
+        # callbacks
+        NotKnown.addDependant(self, dep, key)
+        self.unpause()
+        resovd = self.result
+        self.resolveDependants(resovd)
diff --git a/contrib/python/Twisted/py3/twisted/persisted/dirdbm.py b/contrib/python/Twisted/py3/twisted/persisted/dirdbm.py
new file mode 100644
index 00000000000..da8375b77c2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/persisted/dirdbm.py
@@ -0,0 +1,361 @@
+# -*- test-case-name: twisted.test.test_dirdbm -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+DBM-style interface to a directory.
+
+Each key is stored as a single file.  This is not expected to be very fast or
+efficient, but it's good for easy debugging.
+
+DirDBMs are *not* thread-safe, they should only be accessed by one thread at
+a time.
+
+No files should be placed in the working directory of a DirDBM save those
+created by the DirDBM itself!
+
+Maintainer: Itamar Shtull-Trauring
+"""
+
+
+import base64
+import glob
+import os
+import pickle
+
+from twisted.python.filepath import FilePath
+
+try:
+    _open  # type: ignore[has-type, used-before-def]
+except NameError:
+    _open = open
+
+
+class DirDBM:
+    """
+    A directory with a DBM interface.
+
+    This class presents a hash-like interface to a directory of small,
+    flat files. It can only use strings as keys or values.
+    """
+
+    def __init__(self, name):
+        """
+        @type name: str
+        @param name: Base path to use for the directory storage.
+        """
+        self.dname = os.path.abspath(name)
+        self._dnamePath = FilePath(name)
+        if not self._dnamePath.isdir():
+            self._dnamePath.createDirectory()
+        else:
+            # Run recovery, in case we crashed. we delete all files ending
+            # with ".new". Then we find all files who end with ".rpl". If a
+            # corresponding file exists without ".rpl", we assume the write
+            # failed and delete the ".rpl" file. If only a ".rpl" exist we
+            # assume the program crashed right after deleting the old entry
+            # but before renaming the replacement entry.
+            #
+            # NOTE: '.' is NOT in the base64 alphabet!
+            for f in glob.glob(self._dnamePath.child("*.new").path):
+                os.remove(f)
+            replacements = glob.glob(self._dnamePath.child("*.rpl").path)
+            for f in replacements:
+                old = f[:-4]
+                if os.path.exists(old):
+                    os.remove(f)
+                else:
+                    os.rename(f, old)
+
+    def _encode(self, k):
+        """
+        Encode a key so it can be used as a filename.
+        """
+        # NOTE: '_' is NOT in the base64 alphabet!
+        return base64.encodebytes(k).replace(b"\n", b"_").replace(b"/", b"-")
+
+    def _decode(self, k):
+        """
+        Decode a filename to get the key.
+        """
+        return base64.decodebytes(k.replace(b"_", b"\n").replace(b"-", b"/"))
+
+    def _readFile(self, path):
+        """
+        Read in the contents of a file.
+
+        Override in subclasses to e.g. provide transparently encrypted dirdbm.
+        """
+        with _open(path.path, "rb") as f:
+            s = f.read()
+        return s
+
+    def _writeFile(self, path, data):
+        """
+        Write data to a file.
+
+        Override in subclasses to e.g. provide transparently encrypted dirdbm.
+        """
+        with _open(path.path, "wb") as f:
+            f.write(data)
+            f.flush()
+
+    def __len__(self):
+        """
+        @return: The number of key/value pairs in this Shelf
+        """
+        return len(self._dnamePath.listdir())
+
+    def __setitem__(self, k, v):
+        """
+        C{dirdbm[k] = v}
+        Create or modify a textfile in this directory
+
+        @type k: bytes
+        @param k: key to set
+
+        @type v: bytes
+        @param v: value to associate with C{k}
+        """
+        if not type(k) == bytes:
+            raise TypeError("DirDBM key must be bytes")
+        if not type(v) == bytes:
+            raise TypeError("DirDBM value must be bytes")
+        k = self._encode(k)
+
+        # We create a new file with extension .new, write the data to it, and
+        # if the write succeeds delete the old file and rename the new one.
+        old = self._dnamePath.child(k)
+        if old.exists():
+            new = old.siblingExtension(".rpl")  # Replacement entry
+        else:
+            new = old.siblingExtension(".new")  # New entry
+        try:
+            self._writeFile(new, v)
+        except BaseException:
+            new.remove()
+            raise
+        else:
+            if old.exists():
+                old.remove()
+            new.moveTo(old)
+
+    def __getitem__(self, k):
+        """
+        C{dirdbm[k]}
+        Get the contents of a file in this directory as a string.
+
+        @type k: bytes
+        @param k: key to lookup
+
+        @return: The value associated with C{k}
+        @raise KeyError: Raised when there is no such key
+        """
+        if not type(k) == bytes:
+            raise TypeError("DirDBM key must be bytes")
+        path = self._dnamePath.child(self._encode(k))
+        try:
+            return self._readFile(path)
+        except OSError:
+            raise KeyError(k)
+
+    def __delitem__(self, k):
+        """
+        C{del dirdbm[foo]}
+        Delete a file in this directory.
+
+        @type k: bytes
+        @param k: key to delete
+
+        @raise KeyError: Raised when there is no such key
+        """
+        if not type(k) == bytes:
+            raise TypeError("DirDBM key must be bytes")
+        k = self._encode(k)
+        try:
+            self._dnamePath.child(k).remove()
+        except OSError:
+            raise KeyError(self._decode(k))
+
+    def keys(self):
+        """
+        @return: a L{list} of filenames (keys).
+        """
+        return list(map(self._decode, self._dnamePath.asBytesMode().listdir()))
+
+    def values(self):
+        """
+        @return: a L{list} of file-contents (values).
+        """
+        vals = []
+        keys = self.keys()
+        for key in keys:
+            vals.append(self[key])
+        return vals
+
+    def items(self):
+        """
+        @return: a L{list} of 2-tuples containing key/value pairs.
+        """
+        items = []
+        keys = self.keys()
+        for key in keys:
+            items.append((key, self[key]))
+        return items
+
+    def has_key(self, key):
+        """
+        @type key: bytes
+        @param key: The key to test
+
+        @return: A true value if this dirdbm has the specified key, a false
+        value otherwise.
+        """
+        if not type(key) == bytes:
+            raise TypeError("DirDBM key must be bytes")
+        key = self._encode(key)
+        return self._dnamePath.child(key).isfile()
+
+    def setdefault(self, key, value):
+        """
+        @type key: bytes
+        @param key: The key to lookup
+
+        @param value: The value to associate with key if key is not already
+        associated with a value.
+        """
+        if key not in self:
+            self[key] = value
+            return value
+        return self[key]
+
+    def get(self, key, default=None):
+        """
+        @type key: bytes
+        @param key: The key to lookup
+
+        @param default: The value to return if the given key does not exist
+
+        @return: The value associated with C{key} or C{default} if not
+        L{DirDBM.has_key(key)}
+        """
+        if key in self:
+            return self[key]
+        else:
+            return default
+
+    def __contains__(self, key):
+        """
+        @see: L{DirDBM.has_key}
+        """
+        return self.has_key(key)
+
+    def update(self, dict):
+        """
+        Add all the key/value pairs in L{dict} to this dirdbm.  Any conflicting
+        keys will be overwritten with the values from L{dict}.
+
+        @type dict: mapping
+        @param dict: A mapping of key/value pairs to add to this dirdbm.
+        """
+        for key, val in dict.items():
+            self[key] = val
+
+    def copyTo(self, path):
+        """
+        Copy the contents of this dirdbm to the dirdbm at C{path}.
+
+        @type path: L{str}
+        @param path: The path of the dirdbm to copy to.  If a dirdbm
+        exists at the destination path, it is cleared first.
+
+        @rtype: C{DirDBM}
+        @return: The dirdbm this dirdbm was copied to.
+        """
+        path = FilePath(path)
+        assert path != self._dnamePath
+
+        d = self.__class__(path.path)
+        d.clear()
+        for k in self.keys():
+            d[k] = self[k]
+        return d
+
+    def clear(self):
+        """
+        Delete all key/value pairs in this dirdbm.
+        """
+        for k in self.keys():
+            del self[k]
+
+    def close(self):
+        """
+        Close this dbm: no-op, for dbm-style interface compliance.
+        """
+
+    def getModificationTime(self, key):
+        """
+        Returns modification time of an entry.
+
+        @return: Last modification date (seconds since epoch) of entry C{key}
+        @raise KeyError: Raised when there is no such key
+        """
+        if not type(key) == bytes:
+            raise TypeError("DirDBM key must be bytes")
+        path = self._dnamePath.child(self._encode(key))
+        if path.isfile():
+            return path.getModificationTime()
+        else:
+            raise KeyError(key)
+
+
+class Shelf(DirDBM):
+    """
+    A directory with a DBM shelf interface.
+
+    This class presents a hash-like interface to a directory of small,
+    flat files. Keys must be strings, but values can be any given object.
+    """
+
+    def __setitem__(self, k, v):
+        """
+        C{shelf[foo] = bar}
+        Create or modify a textfile in this directory.
+
+        @type k: str
+        @param k: The key to set
+
+        @param v: The value to associate with C{key}
+        """
+        v = pickle.dumps(v)
+        DirDBM.__setitem__(self, k, v)
+
+    def __getitem__(self, k):
+        """
+        C{dirdbm[foo]}
+        Get and unpickle the contents of a file in this directory.
+
+        @type k: bytes
+        @param k: The key to lookup
+
+        @return: The value associated with the given key
+        @raise KeyError: Raised if the given key does not exist
+        """
+        return pickle.loads(DirDBM.__getitem__(self, k))
+
+
+def open(file, flag=None, mode=None):
+    """
+    This is for 'anydbm' compatibility.
+
+    @param file: The parameter to pass to the DirDBM constructor.
+
+    @param flag: ignored
+    @param mode: ignored
+    """
+    return DirDBM(file)
+
+
+__all__ = ["open", "DirDBM", "Shelf"]
diff --git a/contrib/python/Twisted/py3/twisted/persisted/newsfragments/9831.misc b/contrib/python/Twisted/py3/twisted/persisted/newsfragments/9831.misc
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/Twisted/py3/twisted/persisted/sob.py b/contrib/python/Twisted/py3/twisted/persisted/sob.py
new file mode 100644
index 00000000000..b952165172b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/persisted/sob.py
@@ -0,0 +1,200 @@
+# -*- test-case-name: twisted.test.test_sob -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+"""
+Save and load Small OBjects to and from files, using various formats.
+
+Maintainer: Moshe Zadka
+"""
+
+
+import os
+import pickle
+import sys
+
+from zope.interface import Interface, implementer
+
+from twisted.persisted import styles
+from twisted.python import log, runtime
+
+
+class IPersistable(Interface):
+
+    """An object which can be saved in several formats to a file"""
+
+    def setStyle(style):
+        """Set desired format.
+
+        @type style: string (one of 'pickle' or 'source')
+        """
+
+    def save(tag=None, filename=None, passphrase=None):
+        """Save object to file.
+
+        @type tag: string
+        @type filename: string
+        @type passphrase: string
+        """
+
+
+@implementer(IPersistable)
+class Persistent:
+    style = "pickle"
+
+    def __init__(self, original, name):
+        self.original = original
+        self.name = name
+
+    def setStyle(self, style):
+        """Set desired format.
+
+        @type style: string (one of 'pickle' or 'source')
+        """
+        self.style = style
+
+    def _getFilename(self, filename, ext, tag):
+        if filename:
+            finalname = filename
+            filename = finalname + "-2"
+        elif tag:
+            filename = f"{self.name}-{tag}-2.{ext}"
+            finalname = f"{self.name}-{tag}.{ext}"
+        else:
+            filename = f"{self.name}-2.{ext}"
+            finalname = f"{self.name}.{ext}"
+        return finalname, filename
+
+    def _saveTemp(self, filename, dumpFunc):
+        with open(filename, "wb") as f:
+            dumpFunc(self.original, f)
+
+    def _getStyle(self):
+        if self.style == "source":
+            from twisted.persisted.aot import jellyToSource as dumpFunc
+
+            ext = "tas"
+        else:
+
+            def dumpFunc(obj, file=None):
+                pickle.dump(obj, file, 2)
+
+            ext = "tap"
+        return ext, dumpFunc
+
+    def save(self, tag=None, filename=None, passphrase=None):
+        """Save object to file.
+
+        @type tag: string
+        @type filename: string
+        @type passphrase: string
+        """
+        ext, dumpFunc = self._getStyle()
+        if passphrase is not None:
+            raise TypeError("passphrase must be None")
+        finalname, filename = self._getFilename(filename, ext, tag)
+        log.msg("Saving " + self.name + " application to " + finalname + "...")
+        self._saveTemp(filename, dumpFunc)
+        if runtime.platformType == "win32" and os.path.isfile(finalname):
+            os.remove(finalname)
+        os.rename(filename, finalname)
+        log.msg("Saved.")
+
+
+# "Persistant" has been present since 1.0.7, so retain it for compatibility
+Persistant = Persistent
+
+
+class _EverythingEphemeral(styles.Ephemeral):
+    initRun = 0
+
+    def __init__(self, mainMod):
+        """
+        @param mainMod: The '__main__' module that this class will proxy.
+        """
+        self.mainMod = mainMod
+
+    def __getattr__(self, key):
+        try:
+            return getattr(self.mainMod, key)
+        except AttributeError:
+            if self.initRun:
+                raise
+            else:
+                log.msg("Warning!  Loading from __main__: %s" % key)
+                return styles.Ephemeral()
+
+
+def load(filename, style):
+    """Load an object from a file.
+
+    Deserialize an object from a file. The file can be encrypted.
+
+    @param filename: string
+    @param style: string (one of 'pickle' or 'source')
+    """
+    mode = "r"
+    if style == "source":
+        from twisted.persisted.aot import unjellyFromSource as _load
+    else:
+        _load, mode = pickle.load, "rb"
+
+    fp = open(filename, mode)
+    ee = _EverythingEphemeral(sys.modules["__main__"])
+    sys.modules["__main__"] = ee
+    ee.initRun = 1
+    with fp:
+        try:
+            value = _load(fp)
+        finally:
+            # restore __main__ if an exception is raised.
+            sys.modules["__main__"] = ee.mainMod
+
+    styles.doUpgrade()
+    ee.initRun = 0
+    persistable = IPersistable(value, None)
+    if persistable is not None:
+        persistable.setStyle(style)
+    return value
+
+
+def loadValueFromFile(filename, variable):
+    """Load the value of a variable in a Python file.
+
+    Run the contents of the file in a namespace and return the result of the
+    variable named C{variable}.
+
+    @param filename: string
+    @param variable: string
+    """
+    with open(filename) as fileObj:
+        data = fileObj.read()
+    d = {"__file__": filename}
+    codeObj = compile(data, filename, "exec")
+    eval(codeObj, d, d)
+    value = d[variable]
+    return value
+
+
+def guessType(filename):
+    ext = os.path.splitext(filename)[1]
+    return {
+        ".tac": "python",
+        ".etac": "python",
+        ".py": "python",
+        ".tap": "pickle",
+        ".etap": "pickle",
+        ".tas": "source",
+        ".etas": "source",
+    }[ext]
+
+
+__all__ = [
+    "loadValueFromFile",
+    "load",
+    "Persistent",
+    "Persistant",
+    "IPersistable",
+    "guessType",
+]
diff --git a/contrib/python/Twisted/py3/twisted/persisted/styles.py b/contrib/python/Twisted/py3/twisted/persisted/styles.py
new file mode 100644
index 00000000000..2d014dc8477
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/persisted/styles.py
@@ -0,0 +1,391 @@
+# -*- test-case-name: twisted.test.test_persisted -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Different styles of persisted objects.
+"""
+
+import copy
+import copyreg as copy_reg
+import inspect
+import pickle
+import types
+from io import StringIO as _cStringIO
+from typing import Dict
+
+from twisted.python import log, reflect
+from twisted.python.compat import _PYPY
+
+oldModules: Dict[str, types.ModuleType] = {}
+
+
+_UniversalPicklingError = pickle.PicklingError
+
+
+def pickleMethod(method):
+    "support function for copy_reg to pickle method refs"
+    return (
+        unpickleMethod,
+        (method.__name__, method.__self__, method.__self__.__class__),
+    )
+
+
+def _methodFunction(classObject, methodName):
+    """
+    Retrieve the function object implementing a method name given the class
+    it's on and a method name.
+
+    @param classObject: A class to retrieve the method's function from.
+    @type classObject: L{type}
+
+    @param methodName: The name of the method whose function to retrieve.
+    @type methodName: native L{str}
+
+    @return: the function object corresponding to the given method name.
+    @rtype: L{types.FunctionType}
+    """
+    methodObject = getattr(classObject, methodName)
+    return methodObject
+
+
+def unpickleMethod(im_name, im_self, im_class):
+    """
+    Support function for copy_reg to unpickle method refs.
+
+    @param im_name: The name of the method.
+    @type im_name: native L{str}
+
+    @param im_self: The instance that the method was present on.
+    @type im_self: L{object}
+
+    @param im_class: The class where the method was declared.
+    @type im_class: L{type} or L{None}
+    """
+    if im_self is None:
+        return getattr(im_class, im_name)
+    try:
+        methodFunction = _methodFunction(im_class, im_name)
+    except AttributeError:
+        log.msg("Method", im_name, "not on class", im_class)
+        assert im_self is not None, "No recourse: no instance to guess from."
+        # Attempt a last-ditch fix before giving up. If classes have changed
+        # around since we pickled this method, we may still be able to get it
+        # by looking on the instance's current class.
+        if im_self.__class__ is im_class:
+            raise
+        return unpickleMethod(im_name, im_self, im_self.__class__)
+    else:
+        maybeClass = ()
+        bound = types.MethodType(methodFunction, im_self, *maybeClass)
+        return bound
+
+
+copy_reg.pickle(types.MethodType, pickleMethod)
+
+
+def _pickleFunction(f):
+    """
+    Reduce, in the sense of L{pickle}'s C{object.__reduce__} special method, a
+    function object into its constituent parts.
+
+    @param f: The function to reduce.
+    @type f: L{types.FunctionType}
+
+    @return: a 2-tuple of a reference to L{_unpickleFunction} and a tuple of
+        its arguments, a 1-tuple of the function's fully qualified name.
+    @rtype: 2-tuple of C{callable, native string}
+    """
+    if f.__name__ == "<lambda>":
+        raise _UniversalPicklingError(f"Cannot pickle lambda function: {f}")
+    return (_unpickleFunction, tuple([".".join([f.__module__, f.__qualname__])]))
+
+
+def _unpickleFunction(fullyQualifiedName):
+    """
+    Convert a function name into a function by importing it.
+
+    This is a synonym for L{twisted.python.reflect.namedAny}, but imported
+    locally to avoid circular imports, and also to provide a persistent name
+    that can be stored (and deprecated) independently of C{namedAny}.
+
+    @param fullyQualifiedName: The fully qualified name of a function.
+    @type fullyQualifiedName: native C{str}
+
+    @return: A function object imported from the given location.
+    @rtype: L{types.FunctionType}
+    """
+    from twisted.python.reflect import namedAny
+
+    return namedAny(fullyQualifiedName)
+
+
+copy_reg.pickle(types.FunctionType, _pickleFunction)
+
+
+def pickleModule(module):
+    "support function for copy_reg to pickle module refs"
+    return unpickleModule, (module.__name__,)
+
+
+def unpickleModule(name):
+    "support function for copy_reg to unpickle module refs"
+    if name in oldModules:
+        log.msg("Module has moved: %s" % name)
+        name = oldModules[name]
+        log.msg(name)
+    return __import__(name, {}, {}, "x")
+
+
+copy_reg.pickle(types.ModuleType, pickleModule)
+
+
+def pickleStringO(stringo):
+    """
+    Reduce the given cStringO.
+
+    This is only called on Python 2, because the cStringIO module only exists
+    on Python 2.
+
+    @param stringo: The string output to pickle.
+    @type stringo: C{cStringIO.OutputType}
+    """
+    "support function for copy_reg to pickle StringIO.OutputTypes"
+    return unpickleStringO, (stringo.getvalue(), stringo.tell())
+
+
+def unpickleStringO(val, sek):
+    """
+    Convert the output of L{pickleStringO} into an appropriate type for the
+    current python version.  This may be called on Python 3 and will convert a
+    cStringIO into an L{io.StringIO}.
+
+    @param val: The content of the file.
+    @type val: L{bytes}
+
+    @param sek: The seek position of the file.
+    @type sek: L{int}
+
+    @return: a file-like object which you can write bytes to.
+    @rtype: C{cStringIO.OutputType} on Python 2, L{io.StringIO} on Python 3.
+    """
+    x = _cStringIO()
+    x.write(val)
+    x.seek(sek)
+    return x
+
+
+def pickleStringI(stringi):
+    """
+    Reduce the given cStringI.
+
+    This is only called on Python 2, because the cStringIO module only exists
+    on Python 2.
+
+    @param stringi: The string input to pickle.
+    @type stringi: C{cStringIO.InputType}
+
+    @return: a 2-tuple of (C{unpickleStringI}, (bytes, pointer))
+    @rtype: 2-tuple of (function, (bytes, int))
+    """
+    return unpickleStringI, (stringi.getvalue(), stringi.tell())
+
+
+def unpickleStringI(val, sek):
+    """
+    Convert the output of L{pickleStringI} into an appropriate type for the
+    current Python version.
+
+    This may be called on Python 3 and will convert a cStringIO into an
+    L{io.StringIO}.
+
+    @param val: The content of the file.
+    @type val: L{bytes}
+
+    @param sek: The seek position of the file.
+    @type sek: L{int}
+
+    @return: a file-like object which you can read bytes from.
+    @rtype: C{cStringIO.OutputType} on Python 2, L{io.StringIO} on Python 3.
+    """
+    x = _cStringIO(val)
+    x.seek(sek)
+    return x
+
+
+class Ephemeral:
+    """
+    This type of object is never persisted; if possible, even references to it
+    are eliminated.
+    """
+
+    def __reduce__(self):
+        """
+        Serialize any subclass of L{Ephemeral} in a way which replaces it with
+        L{Ephemeral} itself.
+        """
+        return (Ephemeral, ())
+
+    def __getstate__(self):
+        log.msg("WARNING: serializing ephemeral %s" % self)
+        if not _PYPY:
+            import gc
+
+            if getattr(gc, "get_referrers", None):
+                for r in gc.get_referrers(self):
+                    log.msg(f" referred to by {r}")
+        return None
+
+    def __setstate__(self, state):
+        log.msg("WARNING: unserializing ephemeral %s" % self.__class__)
+        self.__class__ = Ephemeral
+
+
+versionedsToUpgrade: Dict[int, "Versioned"] = {}
+upgraded = {}
+
+
+def doUpgrade():
+    global versionedsToUpgrade, upgraded
+    for versioned in list(versionedsToUpgrade.values()):
+        requireUpgrade(versioned)
+    versionedsToUpgrade = {}
+    upgraded = {}
+
+
+def requireUpgrade(obj):
+    """Require that a Versioned instance be upgraded completely first."""
+    objID = id(obj)
+    if objID in versionedsToUpgrade and objID not in upgraded:
+        upgraded[objID] = 1
+        obj.versionUpgrade()
+        return obj
+
+
+def _aybabtu(c):
+    """
+    Get all of the parent classes of C{c}, not including C{c} itself, which are
+    strict subclasses of L{Versioned}.
+
+    @param c: a class
+    @returns: list of classes
+    """
+    # begin with two classes that should *not* be included in the
+    # final result
+    l = [c, Versioned]
+    for b in inspect.getmro(c):
+        if b not in l and issubclass(b, Versioned):
+            l.append(b)
+    # return all except the unwanted classes
+    return l[2:]
+
+
+class Versioned:
+    """
+    This type of object is persisted with versioning information.
+
+    I have a single class attribute, the int persistenceVersion.  After I am
+    unserialized (and styles.doUpgrade() is called), self.upgradeToVersionX()
+    will be called for each version upgrade I must undergo.
+
+    For example, if I serialize an instance of a Foo(Versioned) at version 4
+    and then unserialize it when the code is at version 9, the calls::
+
+      self.upgradeToVersion5()
+      self.upgradeToVersion6()
+      self.upgradeToVersion7()
+      self.upgradeToVersion8()
+      self.upgradeToVersion9()
+
+    will be made.  If any of these methods are undefined, a warning message
+    will be printed.
+    """
+
+    persistenceVersion = 0
+    persistenceForgets = ()
+
+    def __setstate__(self, state):
+        versionedsToUpgrade[id(self)] = self
+        self.__dict__ = state
+
+    def __getstate__(self, dict=None):
+        """Get state, adding a version number to it on its way out."""
+        dct = copy.copy(dict or self.__dict__)
+        bases = _aybabtu(self.__class__)
+        bases.reverse()
+        bases.append(self.__class__)  # don't forget me!!
+        for base in bases:
+            if "persistenceForgets" in base.__dict__:
+                for slot in base.persistenceForgets:
+                    if slot in dct:
+                        del dct[slot]
+            if "persistenceVersion" in base.__dict__:
+                dct[
+                    f"{reflect.qual(base)}.persistenceVersion"
+                ] = base.persistenceVersion
+        return dct
+
+    def versionUpgrade(self):
+        """(internal) Do a version upgrade."""
+        bases = _aybabtu(self.__class__)
+        # put the bases in order so superclasses' persistenceVersion methods
+        # will be called first.
+        bases.reverse()
+        bases.append(self.__class__)  # don't forget me!!
+        # first let's look for old-skool versioned's
+        if "persistenceVersion" in self.__dict__:
+            # Hacky heuristic: if more than one class subclasses Versioned,
+            # we'll assume that the higher version number wins for the older
+            # class, so we'll consider the attribute the version of the older
+            # class.  There are obviously possibly times when this will
+            # eventually be an incorrect assumption, but hopefully old-school
+            # persistenceVersion stuff won't make it that far into multiple
+            # classes inheriting from Versioned.
+
+            pver = self.__dict__["persistenceVersion"]
+            del self.__dict__["persistenceVersion"]
+            highestVersion = 0
+            highestBase = None
+            for base in bases:
+                if "persistenceVersion" not in base.__dict__:
+                    continue
+                if base.persistenceVersion > highestVersion:
+                    highestBase = base
+                    highestVersion = base.persistenceVersion
+            if highestBase:
+                self.__dict__[
+                    "%s.persistenceVersion" % reflect.qual(highestBase)
+                ] = pver
+        for base in bases:
+            # ugly hack, but it's what the user expects, really
+            if (
+                Versioned not in base.__bases__
+                and "persistenceVersion" not in base.__dict__
+            ):
+                continue
+            currentVers = base.persistenceVersion
+            pverName = "%s.persistenceVersion" % reflect.qual(base)
+            persistVers = self.__dict__.get(pverName) or 0
+            if persistVers:
+                del self.__dict__[pverName]
+            assert persistVers <= currentVers, "Sorry, can't go backwards in time."
+            while persistVers < currentVers:
+                persistVers = persistVers + 1
+                method = base.__dict__.get("upgradeToVersion%s" % persistVers, None)
+                if method:
+                    log.msg(
+                        "Upgrading %s (of %s @ %s) to version %s"
+                        % (
+                            reflect.qual(base),
+                            reflect.qual(self.__class__),
+                            id(self),
+                            persistVers,
+                        )
+                    )
+                    method(self)
+                else:
+                    log.msg(
+                        "Warning: cannot upgrade {} to version {}".format(
+                            base, persistVers
+                        )
+                    )
diff --git a/contrib/python/Twisted/py3/twisted/plugin.py b/contrib/python/Twisted/py3/twisted/plugin.py
new file mode 100644
index 00000000000..45180014cdc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugin.py
@@ -0,0 +1,260 @@
+# -*- test-case-name: twisted.test.test_plugin -*-
+# Copyright (c) 2005 Divmod, Inc.
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Plugin system for Twisted.
+
+@author: Jp Calderone
+@author: Glyph Lefkowitz
+"""
+
+
+import os
+import pickle
+import sys
+import types
+from typing import Iterable, Optional, Type, TypeVar
+
+from zope.interface import Interface, providedBy
+
+from twisted.python import log
+from twisted.python.components import getAdapterFactory
+from twisted.python.modules import getModule
+from twisted.python.reflect import namedAny
+
+
+class IPlugin(Interface):
+    """
+    Interface that must be implemented by all plugins.
+
+    Only objects which implement this interface will be considered for return
+    by C{getPlugins}.  To be useful, plugins should also implement some other
+    application-specific interface.
+    """
+
+
+class CachedPlugin:
+    def __init__(self, dropin, name, description, provided):
+        self.dropin = dropin
+        self.name = name
+        self.description = description
+        self.provided = provided
+        self.dropin.plugins.append(self)
+
+    def __repr__(self) -> str:
+        return "<CachedPlugin {!r}/{!r} (provides {!r})>".format(
+            self.name,
+            self.dropin.moduleName,
+            ", ".join([i.__name__ for i in self.provided]),
+        )
+
+    def load(self):
+        return namedAny(self.dropin.moduleName + "." + self.name)
+
+    def __conform__(self, interface, registry=None, default=None):
+        for providedInterface in self.provided:
+            if providedInterface.isOrExtends(interface):
+                return self.load()
+            if getAdapterFactory(providedInterface, interface, None) is not None:
+                return interface(self.load(), default)
+        return default
+
+    # backwards compat HOORJ
+    getComponent = __conform__
+
+
+class CachedDropin:
+    """
+    A collection of L{CachedPlugin} instances from a particular module in a
+    plugin package.
+
+    @type moduleName: C{str}
+    @ivar moduleName: The fully qualified name of the plugin module this
+        represents.
+
+    @type description: C{str} or L{None}
+    @ivar description: A brief explanation of this collection of plugins
+        (probably the plugin module's docstring).
+
+    @type plugins: C{list}
+    @ivar plugins: The L{CachedPlugin} instances which were loaded from this
+        dropin.
+    """
+
+    def __init__(self, moduleName, description):
+        self.moduleName = moduleName
+        self.description = description
+        self.plugins = []
+
+
+def _generateCacheEntry(provider):
+    dropin = CachedDropin(provider.__name__, provider.__doc__)
+    for k, v in provider.__dict__.items():
+        plugin = IPlugin(v, None)
+        if plugin is not None:
+            # Instantiated for its side-effects.
+            CachedPlugin(dropin, k, v.__doc__, list(providedBy(plugin)))
+    return dropin
+
+
+try:
+    fromkeys = dict.fromkeys
+except AttributeError:
+
+    def fromkeys(keys, value=None):
+        d = {}
+        for k in keys:
+            d[k] = value
+        return d
+
+
+def getCache(module):
+    """
+    Compute all the possible loadable plugins, while loading as few as
+    possible and hitting the filesystem as little as possible.
+
+    @param module: a Python module object.  This represents a package to search
+    for plugins.
+
+    @return: a dictionary mapping module names to L{CachedDropin} instances.
+    """
+    allCachesCombined = {}
+    mod = getModule(module.__name__)
+    # don't want to walk deep, only immediate children.
+    buckets = {}
+    # Fill buckets with modules by related entry on the given package's
+    # __path__.  There's an abstraction inversion going on here, because this
+    # information is already represented internally in twisted.python.modules,
+    # but it's simple enough that I'm willing to live with it.  If anyone else
+    # wants to fix up this iteration so that it's one path segment at a time,
+    # be my guest.  --glyph
+    for plugmod in mod.iterModules():
+        fpp = plugmod.filePath.parent()
+        if fpp not in buckets:
+            buckets[fpp] = []
+        bucket = buckets[fpp]
+        bucket.append(plugmod)
+    for pseudoPackagePath, bucket in buckets.items():
+        dropinPath = pseudoPackagePath.child("dropin.cache")
+        try:
+            lastCached = dropinPath.getModificationTime()
+            with dropinPath.open("r") as f:
+                dropinDotCache = pickle.load(f)
+        except BaseException:
+            dropinDotCache = {}
+            lastCached = 0
+
+        needsWrite = False
+        existingKeys = {}
+        for pluginModule in bucket:
+            pluginKey = pluginModule.name.split(".")[-1]
+            existingKeys[pluginKey] = True
+            if (pluginKey not in dropinDotCache) or (
+                pluginModule.filePath.getModificationTime() >= lastCached
+            ):
+                needsWrite = True
+                try:
+                    provider = pluginModule.load()
+                except BaseException:
+                    # dropinDotCache.pop(pluginKey, None)
+                    log.err()
+                else:
+                    entry = _generateCacheEntry(provider)
+                    dropinDotCache[pluginKey] = entry
+        # Make sure that the cache doesn't contain any stale plugins.
+        for pluginKey in list(dropinDotCache.keys()):
+            if pluginKey not in existingKeys:
+                del dropinDotCache[pluginKey]
+                needsWrite = True
+        if needsWrite:
+            try:
+                dropinPath.setContent(pickle.dumps(dropinDotCache))
+            except OSError as e:
+                log.msg(
+                    format=(
+                        "Unable to write to plugin cache %(path)s: error "
+                        "number %(errno)d"
+                    ),
+                    path=dropinPath.path,
+                    errno=e.errno,
+                )
+            except BaseException:
+                log.err(None, "Unexpected error while writing cache file")
+        allCachesCombined.update(dropinDotCache)
+    return allCachesCombined
+
+
+def _pluginsPackage() -> types.ModuleType:
+    import twisted.plugins as package
+
+    return package
+
+
+_TInterface = TypeVar("_TInterface", bound=Interface)
+
+
+def getPlugins(
+    interface: Type[_TInterface], package: Optional[types.ModuleType] = None
+) -> Iterable[_TInterface]:
+    """
+    Retrieve all plugins implementing the given interface beneath the given module.
+
+    @param interface: An interface class.  Only plugins which implement this
+    interface will be returned.
+
+    @param package: A package beneath which plugins are installed.  For
+    most uses, the default value is correct.
+
+    @return: An iterator of plugins.
+    """
+    if package is None:
+        package = _pluginsPackage()
+    allDropins = getCache(package)
+    for key, dropin in allDropins.items():
+        for plugin in dropin.plugins:
+            try:
+                adapted = interface(plugin, None)
+            except BaseException:
+                log.err()
+            else:
+                if adapted is not None:
+                    yield adapted
+
+
+# Old, backwards compatible name.  Don't use this.
+getPlugIns = getPlugins
+
+
+def pluginPackagePaths(name):
+    """
+    Return a list of additional directories which should be searched for
+    modules to be included as part of the named plugin package.
+
+    @type name: C{str}
+    @param name: The fully-qualified Python name of a plugin package, eg
+        C{'twisted.plugins'}.
+
+    @rtype: C{list} of C{str}
+    @return: The absolute paths to other directories which may contain plugin
+        modules for the named plugin package.
+    """
+    package = name.split(".")
+    # Note that this may include directories which do not exist.  It may be
+    # preferable to remove such directories at this point, rather than allow
+    # them to be searched later on.
+    #
+    # Note as well that only '__init__.py' will be considered to make a
+    # directory a package (and thus exclude it from this list).  This means
+    # that if you create a master plugin package which has some other kind of
+    # __init__ (eg, __init__.pyc) it will be incorrectly treated as a
+    # supplementary plugin directory.
+    return [
+        os.path.abspath(os.path.join(x, *package))
+        for x in sys.path
+        if not os.path.exists(os.path.join(x, *package + ["__init__.py"]))
+    ]
+
+
+__all__ = ["getPlugins", "pluginPackagePaths"]
diff --git a/contrib/python/Twisted/py3/twisted/plugins/__init__.py b/contrib/python/Twisted/py3/twisted/plugins/__init__.py
new file mode 100644
index 00000000000..d4b94bbca0d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/__init__.py
@@ -0,0 +1,22 @@
+# -*- test-case-name: twisted.test.test_plugin -*-
+# Copyright (c) 2005 Divmod, Inc.
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Plugins for services implemented in Twisted.
+
+Plugins go in directories on your PYTHONPATH named twisted/plugins:
+this is the only place where an __init__.py is necessary, thanks to
+the __path__ variable.
+
+@author: Jp Calderone
+@author: Glyph Lefkowitz
+"""
+
+from typing import List
+
+from twisted.plugin import pluginPackagePaths
+
+__path__.extend(pluginPackagePaths(__name__))
+__all__: List[str] = []  # nothing to see here, move along, move along
diff --git a/contrib/python/Twisted/py3/twisted/plugins/cred_anonymous.py b/contrib/python/Twisted/py3/twisted/plugins/cred_anonymous.py
new file mode 100644
index 00000000000..63b321d29cc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/cred_anonymous.py
@@ -0,0 +1,38 @@
+# -*- test-case-name: twisted.test.test_strcred -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Cred plugin for anonymous logins.
+"""
+
+
+from zope.interface import implementer
+
+from twisted import plugin
+from twisted.cred.checkers import AllowAnonymousAccess
+from twisted.cred.credentials import IAnonymous
+from twisted.cred.strcred import ICheckerFactory
+
+anonymousCheckerFactoryHelp = """
+This allows anonymous authentication for servers that support it.
+"""
+
+
+@implementer(ICheckerFactory, plugin.IPlugin)
+class AnonymousCheckerFactory:
+    """
+    Generates checkers that will authenticate an anonymous request.
+    """
+
+    authType = "anonymous"
+    authHelp = anonymousCheckerFactoryHelp
+    argStringFormat = "No argstring required."
+    credentialInterfaces = (IAnonymous,)
+
+    def generateChecker(self, argstring=""):
+        return AllowAnonymousAccess()
+
+
+theAnonymousCheckerFactory = AnonymousCheckerFactory()
diff --git a/contrib/python/Twisted/py3/twisted/plugins/cred_file.py b/contrib/python/Twisted/py3/twisted/plugins/cred_file.py
new file mode 100644
index 00000000000..b872a27b4fb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/cred_file.py
@@ -0,0 +1,59 @@
+# -*- test-case-name: twisted.test.test_strcred -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Cred plugin for a file of the format 'username:password'.
+"""
+
+
+import sys
+
+from zope.interface import implementer
+
+from twisted import plugin
+from twisted.cred.checkers import FilePasswordDB
+from twisted.cred.credentials import IUsernameHashedPassword, IUsernamePassword
+from twisted.cred.strcred import ICheckerFactory
+
+fileCheckerFactoryHelp = """
+This checker expects to receive the location of a file that
+conforms to the FilePasswordDB format. Each line in the file
+should be of the format 'username:password', in plain text.
+"""
+
+invalidFileWarning = "Warning: not a valid file"
+
+
+@implementer(ICheckerFactory, plugin.IPlugin)
+class FileCheckerFactory:
+    """
+    A factory for instances of L{FilePasswordDB}.
+    """
+
+    authType = "file"
+    authHelp = fileCheckerFactoryHelp
+    argStringFormat = "Location of a FilePasswordDB-formatted file."
+    # Explicitly defined here because FilePasswordDB doesn't do it for us
+    credentialInterfaces = (IUsernamePassword, IUsernameHashedPassword)
+
+    errorOutput = sys.stderr
+
+    def generateChecker(self, argstring):
+        """
+        This checker factory expects to get the location of a file.
+        The file should conform to the format required by
+        L{FilePasswordDB} (using defaults for all
+        initialization parameters).
+        """
+        from twisted.python.filepath import FilePath
+
+        if not argstring.strip():
+            raise ValueError("%r requires a filename" % self.authType)
+        elif not FilePath(argstring).isfile():
+            self.errorOutput.write(f"{invalidFileWarning}: {argstring}\n")
+        return FilePasswordDB(argstring)
+
+
+theFileCheckerFactory = FileCheckerFactory()
diff --git a/contrib/python/Twisted/py3/twisted/plugins/cred_memory.py b/contrib/python/Twisted/py3/twisted/plugins/cred_memory.py
new file mode 100644
index 00000000000..30ef10a244e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/cred_memory.py
@@ -0,0 +1,65 @@
+# -*- test-case-name: twisted.test.test_strcred -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Cred plugin for an in-memory user database.
+"""
+
+
+from zope.interface import implementer
+
+from twisted import plugin
+from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
+from twisted.cred.credentials import IUsernameHashedPassword, IUsernamePassword
+from twisted.cred.strcred import ICheckerFactory
+
+inMemoryCheckerFactoryHelp = """
+A checker that uses an in-memory user database.
+
+This is only of use in one-off test programs or examples which
+don't want to focus too much on how credentials are verified. You
+really don't want to use this for anything else. It is a toy.
+"""
+
+
+@implementer(ICheckerFactory, plugin.IPlugin)
+class InMemoryCheckerFactory:
+    """
+    A factory for in-memory credentials checkers.
+
+    This is only of use in one-off test programs or examples which don't
+    want to focus too much on how credentials are verified.
+
+    You really don't want to use this for anything else.  It is, at best, a
+    toy.  If you need a simple credentials checker for a real application,
+    see L{cred_file.FileCheckerFactory}.
+    """
+
+    authType = "memory"
+    authHelp = inMemoryCheckerFactoryHelp
+    argStringFormat = "A colon-separated list (name:password:...)"
+    credentialInterfaces = (IUsernamePassword, IUsernameHashedPassword)
+
+    def generateChecker(self, argstring):
+        """
+        This checker factory expects to get a list of
+        username:password pairs, with each pair also separated by a
+        colon. For example, the string 'alice:f:bob:g' would generate
+        two users, one named 'alice' and one named 'bob'.
+        """
+        checker = InMemoryUsernamePasswordDatabaseDontUse()
+        if argstring:
+            pieces = argstring.split(":")
+            if len(pieces) % 2:
+                from twisted.cred.strcred import InvalidAuthArgumentString
+
+                raise InvalidAuthArgumentString("argstring must be in format U:P:...")
+            for i in range(0, len(pieces), 2):
+                username, password = pieces[i], pieces[i + 1]
+                checker.addUser(username, password)
+        return checker
+
+
+theInMemoryCheckerFactory = InMemoryCheckerFactory()
diff --git a/contrib/python/Twisted/py3/twisted/plugins/cred_sshkeys.py b/contrib/python/Twisted/py3/twisted/plugins/cred_sshkeys.py
new file mode 100644
index 00000000000..8b7326b2580
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/cred_sshkeys.py
@@ -0,0 +1,48 @@
+# -*- test-case-name: twisted.test.test_strcred -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Cred plugin for ssh key login.
+"""
+
+
+from zope.interface import implementer
+
+from twisted import plugin
+from twisted.cred.strcred import ICheckerFactory
+
+sshKeyCheckerFactoryHelp = """
+This allows SSH public key authentication, based on public keys listed in
+authorized_keys and authorized_keys2 files in user .ssh/ directories.
+"""
+
+
+try:
+    from twisted.conch.checkers import SSHPublicKeyChecker, UNIXAuthorizedKeysFiles
+
+    @implementer(ICheckerFactory, plugin.IPlugin)
+    class SSHKeyCheckerFactory:
+        """
+        Generates checkers that will authenticate a SSH public key
+        """
+
+        authType = "sshkey"
+        authHelp = sshKeyCheckerFactoryHelp
+        argStringFormat = "No argstring required."
+        credentialInterfaces = SSHPublicKeyChecker.credentialInterfaces
+
+        def generateChecker(self, argstring=""):
+            """
+            This checker factory ignores the argument string. Everything
+            needed to authenticate users is pulled out of the public keys
+            listed in user .ssh/ directories.
+            """
+            return SSHPublicKeyChecker(UNIXAuthorizedKeysFiles())
+
+    theSSHKeyCheckerFactory = SSHKeyCheckerFactory()
+
+except ImportError:
+    # if checkers can't be imported, then there should be no SSH cred plugin
+    pass
diff --git a/contrib/python/Twisted/py3/twisted/plugins/cred_unix.py b/contrib/python/Twisted/py3/twisted/plugins/cred_unix.py
new file mode 100644
index 00000000000..3d929f75cce
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/cred_unix.py
@@ -0,0 +1,184 @@
+# -*- test-case-name: twisted.test.test_strcred -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Cred plugin for UNIX user accounts.
+"""
+
+
+from zope.interface import implementer
+
+from twisted import plugin
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.cred.credentials import IUsernamePassword
+from twisted.cred.error import UnauthorizedLogin
+from twisted.cred.strcred import ICheckerFactory
+from twisted.internet import defer
+
+
+def verifyCryptedPassword(crypted, pw):
+    """
+    Use L{crypt.crypt} to Verify that an unencrypted
+    password matches the encrypted password.
+
+    @param crypted: The encrypted password, obtained from
+                    the Unix password database or Unix shadow
+                    password database.
+    @param pw: The unencrypted password.
+    @return: L{True} if there is successful match, else L{False}.
+    @rtype: L{bool}
+    """
+    try:
+        import crypt
+    except ImportError:
+        crypt = None
+
+    if crypt is None:
+        raise NotImplementedError("cred_unix not supported on this platform")
+    if isinstance(pw, bytes):
+        pw = pw.decode("utf-8")
+    if isinstance(crypted, bytes):
+        crypted = crypted.decode("utf-8")
+    try:
+        crypted_check = crypt.crypt(pw, crypted)
+        if isinstance(crypted_check, bytes):
+            crypted_check = crypted_check.decode("utf-8")
+        return crypted_check == crypted
+    except OSError:
+        return False
+
+
+@implementer(ICredentialsChecker)
+class UNIXChecker:
+    """
+    A credentials checker for a UNIX server. This will check that
+    an authenticating username/password is a valid user on the system.
+
+    Does not work on Windows.
+
+    Right now this supports Python's pwd and spwd modules, if they are
+    installed. It does not support PAM.
+    """
+
+    credentialInterfaces = (IUsernamePassword,)
+
+    def checkPwd(self, pwd, username, password):
+        """
+        Obtain the encrypted password for C{username} from the Unix password
+        database using L{pwd.getpwnam}, and see if it it matches it matches
+        C{password}.
+
+        @param pwd: Module which provides functions which
+                    access to the Unix password database.
+        @type pwd: C{module}
+        @param username: The user to look up in the Unix password database.
+        @type username: L{unicode}/L{str} or L{bytes}
+        @param password: The password to compare.
+        @type username: L{unicode}/L{str} or L{bytes}
+        """
+        try:
+            if isinstance(username, bytes):
+                username = username.decode("utf-8")
+            cryptedPass = pwd.getpwnam(username).pw_passwd
+        except KeyError:
+            return defer.fail(UnauthorizedLogin())
+        else:
+            if cryptedPass in ("*", "x"):
+                # Allow checkSpwd to take over
+                return None
+            elif verifyCryptedPassword(cryptedPass, password):
+                return defer.succeed(username)
+
+    def checkSpwd(self, spwd, username, password):
+        """
+        Obtain the encrypted password for C{username} from the
+        Unix shadow password database using L{spwd.getspnam},
+        and see if it it matches it matches C{password}.
+
+        @param spwd: Module which provides functions which
+                     access to the Unix shadow password database.
+        @type spwd: C{module}
+        @param username: The user to look up in the Unix password database.
+        @type username: L{unicode}/L{str} or L{bytes}
+        @param password: The password to compare.
+        @type username: L{unicode}/L{str} or L{bytes}
+        """
+        try:
+            if isinstance(username, bytes):
+                username = username.decode("utf-8")
+            if getattr(spwd.struct_spwd, "sp_pwdp", None):
+                # Python 3
+                cryptedPass = spwd.getspnam(username).sp_pwdp
+            else:
+                # Python 2
+                cryptedPass = spwd.getspnam(username).sp_pwd
+        except KeyError:
+            return defer.fail(UnauthorizedLogin())
+        else:
+            if verifyCryptedPassword(cryptedPass, password):
+                return defer.succeed(username)
+
+    def requestAvatarId(self, credentials):
+        username, password = credentials.username, credentials.password
+
+        try:
+            import pwd
+        except ImportError:
+            pwd = None
+
+        if pwd is not None:
+            checked = self.checkPwd(pwd, username, password)
+            if checked is not None:
+                return checked
+
+        try:
+            import spwd
+        except ImportError:
+            spwd = None
+
+        if spwd is not None:
+            checked = self.checkSpwd(spwd, username, password)
+            if checked is not None:
+                return checked
+        # TODO: check_pam?
+        # TODO: check_shadow?
+        return defer.fail(UnauthorizedLogin())
+
+
+unixCheckerFactoryHelp = """
+This checker will attempt to use every resource available to
+authenticate against the list of users on the local UNIX system.
+(This does not support Windows servers for very obvious reasons.)
+
+Right now, this includes support for:
+
+  * Python's pwd module (which checks /etc/passwd)
+  * Python's spwd module (which checks /etc/shadow)
+
+Future versions may include support for PAM authentication.
+"""
+
+
+@implementer(ICheckerFactory, plugin.IPlugin)
+class UNIXCheckerFactory:
+    """
+    A factory for L{UNIXChecker}.
+    """
+
+    authType = "unix"
+    authHelp = unixCheckerFactoryHelp
+    argStringFormat = "No argstring required."
+    credentialInterfaces = UNIXChecker.credentialInterfaces
+
+    def generateChecker(self, argstring):
+        """
+        This checker factory ignores the argument string. Everything
+        needed to generate a user database is pulled out of the local
+        UNIX environment.
+        """
+        return UNIXChecker()
+
+
+theUnixCheckerFactory = UNIXCheckerFactory()
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_conch.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_conch.py
new file mode 100644
index 00000000000..b852b325b0e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_conch.py
@@ -0,0 +1,19 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedSSH = ServiceMaker(
+    "Twisted Conch Server", "twisted.conch.tap", "A Conch SSH service.", "conch"
+)
+
+TwistedManhole = ServiceMaker(
+    "Twisted Manhole (new)",
+    "twisted.conch.manhole_tap",
+    (
+        "An interactive remote debugger service accessible via telnet "
+        "and ssh and providing syntax coloring and basic line editing "
+        "functionality."
+    ),
+    "manhole",
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_core.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_core.py
new file mode 100644
index 00000000000..140ac918bdc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_core.py
@@ -0,0 +1,19 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+from twisted.internet.endpoints import (
+    _StandardIOParser,
+    _SystemdParser,
+    _TCP6ServerParser,
+    _TLSClientEndpointParser,
+)
+from twisted.protocols.haproxy._parser import (
+    HAProxyServerParser as _HAProxyServerParser,
+)
+
+systemdEndpointParser = _SystemdParser()
+tcp6ServerEndpointParser = _TCP6ServerParser()
+stdioEndpointParser = _StandardIOParser()
+tlsClientEndpointParser = _TLSClientEndpointParser()
+_haProxyServerEndpointParser = _HAProxyServerParser()
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_ftp.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_ftp.py
new file mode 100644
index 00000000000..b4053f17b54
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_ftp.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedFTP = ServiceMaker("Twisted FTP", "twisted.tap.ftp", "An FTP server.", "ftp")
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_inet.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_inet.py
new file mode 100644
index 00000000000..dd8a3476560
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_inet.py
@@ -0,0 +1,11 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedINETD = ServiceMaker(
+    "Twisted INETD Server",
+    "twisted.runner.inetdtap",
+    "An inetd(8) replacement.",
+    "inetd",
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_mail.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_mail.py
new file mode 100644
index 00000000000..009e9e69021
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_mail.py
@@ -0,0 +1,8 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedMail = ServiceMaker(
+    "Twisted Mail", "twisted.mail.tap", "An email service", "mail"
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_names.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_names.py
new file mode 100644
index 00000000000..bf78a168b21
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_names.py
@@ -0,0 +1,8 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedNames = ServiceMaker(
+    "Twisted DNS Server", "twisted.names.tap", "A domain name server.", "dns"
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_portforward.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_portforward.py
new file mode 100644
index 00000000000..23bebb949c6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_portforward.py
@@ -0,0 +1,11 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedPortForward = ServiceMaker(
+    "Twisted Port-Forwarding",
+    "twisted.tap.portforward",
+    "A simple port-forwarder.",
+    "portforward",
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_reactors.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_reactors.py
new file mode 100644
index 00000000000..bc5e2ffc3f7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_reactors.py
@@ -0,0 +1,59 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+from twisted.application.reactors import Reactor
+
+__all__ = []
+
+default = Reactor(
+    "default",
+    "twisted.internet.default",
+    "A reasonable default: poll(2) if available, otherwise select(2).",
+)
+__all__.append("default")
+
+select = Reactor("select", "twisted.internet.selectreactor", "select(2) based reactor.")
+__all__.append("select")
+
+poll = Reactor("poll", "twisted.internet.pollreactor", "poll(2) based reactor.")
+__all__.append("poll")
+
+epoll = Reactor("epoll", "twisted.internet.epollreactor", "epoll(4) based reactor.")
+__all__.append("epoll")
+
+kqueue = Reactor("kqueue", "twisted.internet.kqreactor", "kqueue(2) based reactor.")
+__all__.append("kqueue")
+
+cf = Reactor("cf", "twisted.internet.cfreactor", "CoreFoundation based reactor.")
+__all__.append("cf")
+
+asyncio = Reactor("asyncio", "twisted.internet.asyncioreactor", "asyncio based reactor")
+__all__.append("asyncio")
+
+wx = Reactor("wx", "twisted.internet.wxreactor", "wxPython based reactor.")
+__all__.append("wx")
+
+gi = Reactor("gi", "twisted.internet.gireactor", "GObject Introspection based reactor.")
+__all__.append("gi")
+
+gtk3 = Reactor("gtk3", "twisted.internet.gtk3reactor", "Gtk3 based reactor.")
+__all__.append("gtk3")
+
+gtk2 = Reactor("gtk2", "twisted.internet.gtk2reactor", "Gtk2 based reactor.")
+__all__.append("gtk2")
+
+glib2 = Reactor("glib2", "twisted.internet.glib2reactor", "GLib2 based reactor.")
+__all__.append("glib2")
+
+win32er = Reactor(
+    "win32",
+    "twisted.internet.win32eventreactor",
+    "Win32 WaitForMultipleObjects based reactor.",
+)
+__all__.append("win32er")
+
+iocp = Reactor(
+    "iocp", "twisted.internet.iocpreactor", "Win32 IO Completion Ports based reactor."
+)
+__all__.append("iocp")
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_runner.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_runner.py
new file mode 100644
index 00000000000..940989ea910
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_runner.py
@@ -0,0 +1,11 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedProcmon = ServiceMaker(
+    "Twisted Process Monitor",
+    "twisted.runner.procmontap",
+    ("A process watchdog / supervisor"),
+    "procmon",
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_socks.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_socks.py
new file mode 100644
index 00000000000..faf47f83fcd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_socks.py
@@ -0,0 +1,8 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedSOCKS = ServiceMaker(
+    "Twisted SOCKS", "twisted.tap.socks", "A SOCKSv4 proxy service.", "socks"
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_trial.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_trial.py
new file mode 100644
index 00000000000..3e64a09e830
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_trial.py
@@ -0,0 +1,172 @@
+from zope.interface import implementer
+
+from twisted.plugin import IPlugin
+from twisted.trial.itrial import IReporter
+
+
+@implementer(IPlugin, IReporter)
+class _Reporter:
+    def __init__(self, name, module, description, longOpt, shortOpt, klass):
+        self.name = name
+        self.module = module
+        self.description = description
+        self.longOpt = longOpt
+        self.shortOpt = shortOpt
+        self.klass = klass
+
+    @property
+    def stream(self):
+        # IReporter.stream
+        pass
+
+    @property
+    def tbformat(self):
+        # IReporter.tbformat
+        pass
+
+    @property
+    def args(self):
+        # IReporter.args
+        pass
+
+    @property
+    def shouldStop(self):
+        # IReporter.shouldStop
+        pass
+
+    @property
+    def separator(self):
+        # IReporter.separator
+        pass
+
+    @property
+    def testsRun(self):
+        # IReporter.testsRun
+        pass
+
+    def addError(self, test, error):
+        # IReporter.addError
+        pass
+
+    def addExpectedFailure(self, test, failure, todo=None):
+        # IReporter.addExpectedFailure
+        pass
+
+    def addFailure(self, test, failure):
+        # IReporter.addFailure
+        pass
+
+    def addSkip(self, test, reason):
+        # IReporter.addSkip
+        pass
+
+    def addSuccess(self, test):
+        # IReporter.addSuccess
+        pass
+
+    def addUnexpectedSuccess(self, test, todo=None):
+        # IReporter.addUnexpectedSuccess
+        pass
+
+    def cleanupErrors(self, errs):
+        # IReporter.cleanupErrors
+        pass
+
+    def done(self):
+        # IReporter.done
+        pass
+
+    def endSuite(self, name):
+        # IReporter.endSuite
+        pass
+
+    def printErrors(self):
+        # IReporter.printErrors
+        pass
+
+    def printSummary(self):
+        # IReporter.printSummary
+        pass
+
+    def startSuite(self, name):
+        # IReporter.startSuite
+        pass
+
+    def startTest(self, method):
+        # IReporter.startTest
+        pass
+
+    def stopTest(self, method):
+        # IReporter.stopTest
+        pass
+
+    def upDownError(self, userMeth, warn=True, printStatus=True):
+        # IReporter.upDownError
+        pass
+
+    def wasSuccessful(self):
+        # IReporter.wasSuccessful
+        pass
+
+    def write(self, string):
+        # IReporter.write
+        pass
+
+    def writeln(self, string):
+        # IReporter.writeln
+        pass
+
+
+Tree = _Reporter(
+    "Tree Reporter",
+    "twisted.trial.reporter",
+    description="verbose color output (default reporter)",
+    longOpt="verbose",
+    shortOpt="v",
+    klass="TreeReporter",
+)
+
+BlackAndWhite = _Reporter(
+    "Black-And-White Reporter",
+    "twisted.trial.reporter",
+    description="Colorless verbose output",
+    longOpt="bwverbose",
+    shortOpt="o",
+    klass="VerboseTextReporter",
+)
+
+Minimal = _Reporter(
+    "Minimal Reporter",
+    "twisted.trial.reporter",
+    description="minimal summary output",
+    longOpt="summary",
+    shortOpt="s",
+    klass="MinimalReporter",
+)
+
+Classic = _Reporter(
+    "Classic Reporter",
+    "twisted.trial.reporter",
+    description="terse text output",
+    longOpt="text",
+    shortOpt="t",
+    klass="TextReporter",
+)
+
+Timing = _Reporter(
+    "Timing Reporter",
+    "twisted.trial.reporter",
+    description="Timing output",
+    longOpt="timing",
+    shortOpt=None,
+    klass="TimingTextReporter",
+)
+
+Subunit = _Reporter(
+    "Subunit Reporter",
+    "twisted.trial.reporter",
+    description="subunit output",
+    longOpt="subunit",
+    shortOpt=None,
+    klass="SubunitReporter",
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_web.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_web.py
new file mode 100644
index 00000000000..a918c2bc108
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_web.py
@@ -0,0 +1,14 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application.service import ServiceMaker
+
+TwistedWeb = ServiceMaker(
+    "Twisted Web",
+    "twisted.web.tap",
+    (
+        "A general-purpose web server which can serve from a "
+        "filesystem or application resource."
+    ),
+    "web",
+)
diff --git a/contrib/python/Twisted/py3/twisted/plugins/twisted_words.py b/contrib/python/Twisted/py3/twisted/plugins/twisted_words.py
new file mode 100644
index 00000000000..f96e0742165
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/plugins/twisted_words.py
@@ -0,0 +1,38 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from zope.interface import provider
+
+from twisted.application.service import ServiceMaker
+from twisted.plugin import IPlugin
+from twisted.words import iwords
+
+NewTwistedWords = ServiceMaker(
+    "New Twisted Words", "twisted.words.tap", "A modern words server", "words"
+)
+
+TwistedXMPPRouter = ServiceMaker(
+    "XMPP Router", "twisted.words.xmpproutertap", "An XMPP Router server", "xmpp-router"
+)
+
+
+@provider(IPlugin, iwords.IProtocolPlugin)
+class RelayChatInterface:
+    name = "irc"
+
+    @classmethod
+    def getFactory(cls, realm, portal):
+        from twisted.words import service
+
+        return service.IRCFactory(realm, portal)
+
+
+@provider(IPlugin, iwords.IProtocolPlugin)
+class PBChatInterface:
+    name = "pb"
+
+    @classmethod
+    def getFactory(cls, realm, portal):
+        from twisted.spread import pb
+
+        return pb.PBServerFactory(portal, True)
diff --git a/contrib/python/Twisted/py3/twisted/positioning/__init__.py b/contrib/python/Twisted/py3/twisted/positioning/__init__.py
new file mode 100644
index 00000000000..a855f41dd8e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/positioning/__init__.py
@@ -0,0 +1,8 @@
+# -*- test-case-name: twisted.positioning.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Twisted Positioning: Framework for applications that make use of positioning.
+
+@since: 14.0
+"""
diff --git a/contrib/python/Twisted/py3/twisted/positioning/_sentence.py b/contrib/python/Twisted/py3/twisted/positioning/_sentence.py
new file mode 100644
index 00000000000..b40dd060ed8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/positioning/_sentence.py
@@ -0,0 +1,118 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Generic sentence handling tools: hopefully reusable.
+"""
+from typing import Set
+
+
+class _BaseSentence:
+    """
+    A base sentence class for a particular protocol.
+
+    Using this base class, specific sentence classes can almost automatically
+    be created for a particular protocol.
+    To do this, fill the ALLOWED_ATTRIBUTES class attribute using
+    the C{getSentenceAttributes} class method of the producer::
+
+        class FooSentence(BaseSentence):
+            \"\"\"
+            A sentence for integalactic transmodulator sentences.
+
+            @ivar transmogrificationConstant: The value used in the
+                transmogrifier while producing this sentence, corrected for
+                gravitational fields.
+            @type transmogrificationConstant: C{Tummy}
+            \"\"\"
+            ALLOWED_ATTRIBUTES = FooProtocol.getSentenceAttributes()
+
+    @ivar presentAttributes: An iterable containing the names of the
+        attributes that are present in this sentence.
+    @type presentAttributes: iterable of C{str}
+
+    @cvar ALLOWED_ATTRIBUTES: A set of attributes that are allowed in this
+        sentence.
+    @type ALLOWED_ATTRIBUTES: C{set} of C{str}
+    """
+
+    ALLOWED_ATTRIBUTES: Set[str] = set()
+
+    def __init__(self, sentenceData):
+        """
+        Initializes a sentence with parsed sentence data.
+
+        @param sentenceData: The parsed sentence data.
+        @type sentenceData: C{dict} (C{str} -> C{str} or L{None})
+        """
+        self._sentenceData = sentenceData
+
+    @property
+    def presentAttributes(self):
+        """
+        An iterable containing the names of the attributes that are present in
+        this sentence.
+
+        @return: The iterable of names of present attributes.
+        @rtype: iterable of C{str}
+        """
+        return iter(self._sentenceData)
+
+    def __getattr__(self, name):
+        """
+        Gets an attribute of this sentence.
+        """
+        if name in self.ALLOWED_ATTRIBUTES:
+            return self._sentenceData.get(name, None)
+        else:
+            className = self.__class__.__name__
+            msg = f"{className} sentences have no {name} attributes"
+            raise AttributeError(msg)
+
+    def __repr__(self) -> str:
+        """
+        Returns a textual representation of this sentence.
+
+        @return: A textual representation of this sentence.
+        @rtype: C{str}
+        """
+        items = self._sentenceData.items()
+        data = [f"{k}: {v}" for k, v in sorted(items) if k != "type"]
+        dataRepr = ", ".join(data)
+
+        typeRepr = self._sentenceData.get("type") or "unknown type"
+        className = self.__class__.__name__
+
+        return f"<{className} ({typeRepr}) {{{dataRepr}}}>"
+
+
+class _PositioningSentenceProducerMixin:
+    """
+    A mixin for certain protocols that produce positioning sentences.
+
+    This mixin helps protocols that store the layout of sentences that they
+    consume in a C{_SENTENCE_CONTENTS} class variable provide all sentence
+    attributes that can ever occur. It does this by providing a class method,
+    C{getSentenceAttributes}, which iterates over all sentence types and
+    collects the possible sentence attributes.
+    """
+
+    @classmethod
+    def getSentenceAttributes(cls):
+        """
+        Returns a set of all attributes that might be found in the sentences
+        produced by this protocol.
+
+        This is basically a set of all the attributes of all the sentences that
+        this protocol can produce.
+
+        @return: The set of all possible sentence attribute names.
+        @rtype: C{set} of C{str}
+        """
+        attributes = {"type"}
+        for attributeList in cls._SENTENCE_CONTENTS.values():
+            for attribute in attributeList:
+                if attribute is None:
+                    continue
+                attributes.add(attribute)
+
+        return attributes
diff --git a/contrib/python/Twisted/py3/twisted/positioning/base.py b/contrib/python/Twisted/py3/twisted/positioning/base.py
new file mode 100644
index 00000000000..7ac5398cd95
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/positioning/base.py
@@ -0,0 +1,926 @@
+# -*- test-case-name: twisted.positioning.test.test_base,twisted.positioning.test.test_sentence -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Generic positioning base classes.
+
+@since: 14.0
+"""
+
+
+from functools import partial
+from operator import attrgetter
+from typing import ClassVar, Sequence
+
+from zope.interface import implementer
+
+from constantly import NamedConstant, Names  # type: ignore[import]
+
+from twisted.positioning import ipositioning
+from twisted.python.util import FancyEqMixin
+
+MPS_PER_KNOT = 0.5144444444444444
+MPS_PER_KPH = 0.27777777777777777
+METERS_PER_FOOT = 0.3048
+
+
+class Angles(Names):
+    """
+    The types of angles.
+
+    @cvar LATITUDE: Angle representing a latitude of an object.
+    @type LATITUDE: L{NamedConstant}
+
+    @cvar LONGITUDE: Angle representing the longitude of an object.
+    @type LONGITUDE: L{NamedConstant}
+
+    @cvar HEADING: Angle representing the heading of an object.
+    @type HEADING: L{NamedConstant}
+
+    @cvar VARIATION: Angle representing a magnetic variation.
+    @type VARIATION: L{NamedConstant}
+
+    """
+
+    LATITUDE = NamedConstant()
+    LONGITUDE = NamedConstant()
+    HEADING = NamedConstant()
+    VARIATION = NamedConstant()
+
+
+class Directions(Names):
+    """
+    The four cardinal directions (north, east, south, west).
+    """
+
+    NORTH = NamedConstant()
+    EAST = NamedConstant()
+    SOUTH = NamedConstant()
+    WEST = NamedConstant()
+
+
+@implementer(ipositioning.IPositioningReceiver)
+class BasePositioningReceiver:
+    """
+    A base positioning receiver.
+
+    This class would be a good base class for building positioning
+    receivers. It implements the interface (so you don't have to) with stub
+    methods.
+
+    People who want to implement positioning receivers should subclass this
+    class and override the specific callbacks they want to handle.
+    """
+
+    def timeReceived(self, time):
+        """
+        Implements L{IPositioningReceiver.timeReceived} stub.
+        """
+
+    def headingReceived(self, heading):
+        """
+        Implements L{IPositioningReceiver.headingReceived} stub.
+        """
+
+    def speedReceived(self, speed):
+        """
+        Implements L{IPositioningReceiver.speedReceived} stub.
+        """
+
+    def climbReceived(self, climb):
+        """
+        Implements L{IPositioningReceiver.climbReceived} stub.
+        """
+
+    def positionReceived(self, latitude, longitude):
+        """
+        Implements L{IPositioningReceiver.positionReceived} stub.
+        """
+
+    def positionErrorReceived(self, positionError):
+        """
+        Implements L{IPositioningReceiver.positionErrorReceived} stub.
+        """
+
+    def altitudeReceived(self, altitude):
+        """
+        Implements L{IPositioningReceiver.altitudeReceived} stub.
+        """
+
+    def beaconInformationReceived(self, beaconInformation):
+        """
+        Implements L{IPositioningReceiver.beaconInformationReceived} stub.
+        """
+
+
+class InvalidSentence(Exception):
+    """
+    An exception raised when a sentence is invalid.
+    """
+
+
+class InvalidChecksum(Exception):
+    """
+    An exception raised when the checksum of a sentence is invalid.
+    """
+
+
+class Angle(FancyEqMixin):
+    """
+    An object representing an angle.
+
+    @cvar _RANGE_EXPRESSIONS: A collection of expressions for the allowable
+        range for the angular value of a particular coordinate value.
+    @type _RANGE_EXPRESSIONS: C{dict} of L{Angles} constants to callables
+    @cvar _ANGLE_TYPE_NAMES: English names for angle types.
+    @type _ANGLE_TYPE_NAMES: C{dict} of L{Angles} constants to C{str}
+    """
+
+    _RANGE_EXPRESSIONS = {
+        Angles.LATITUDE: lambda latitude: -90.0 < latitude < 90.0,
+        Angles.LONGITUDE: lambda longitude: -180.0 < longitude < 180.0,
+        Angles.HEADING: lambda heading: 0 <= heading < 360,
+        Angles.VARIATION: lambda variation: -180 < variation <= 180,
+    }
+
+    _ANGLE_TYPE_NAMES = {
+        Angles.LATITUDE: "Latitude",
+        Angles.LONGITUDE: "Longitude",
+        Angles.VARIATION: "Variation",
+        Angles.HEADING: "Heading",
+    }
+
+    compareAttributes: ClassVar[Sequence[str]] = (
+        "angleType",
+        "inDecimalDegrees",
+    )
+
+    def __init__(self, angle=None, angleType=None):
+        """
+        Initializes an angle.
+
+        @param angle: The value of the angle in decimal degrees. (L{None} if
+            unknown).
+        @type angle: C{float} or L{None}
+
+        @param angleType: A symbolic constant describing the angle type. Should
+            be one of L{Angles} or {None} if unknown.
+
+        @raises ValueError: If the angle type is not the default argument,
+            but it is an unknown type (not in  C{Angle._RANGE_EXPRESSIONS}),
+            or it is a known type but the supplied value was out of the
+            allowable range for said type.
+        """
+        if angleType is not None and angleType not in self._RANGE_EXPRESSIONS:
+            raise ValueError("Unknown angle type")
+
+        if angle is not None and angleType is not None:
+            rangeExpression = self._RANGE_EXPRESSIONS[angleType]
+            if not rangeExpression(angle):
+                template = "Angle {0} not in allowed range for type {1}"
+                raise ValueError(template.format(angle, angleType))
+
+        self.angleType = angleType
+        self._angle = angle
+
+    @property
+    def inDecimalDegrees(self):
+        """
+        The value of this angle in decimal degrees. This value is immutable.
+
+        @return: This angle expressed in decimal degrees, or L{None} if the
+            angle is unknown.
+        @rtype: C{float} (or L{None})
+        """
+        return self._angle
+
+    @property
+    def inDegreesMinutesSeconds(self):
+        """
+        The value of this angle as a degrees, minutes, seconds tuple. This
+        value is immutable.
+
+        @return: This angle expressed in degrees, minutes, seconds. L{None} if
+            the angle is unknown.
+        @rtype: 3-C{tuple} of C{int} (or L{None})
+        """
+        if self._angle is None:
+            return None
+
+        degrees = abs(int(self._angle))
+        fractionalDegrees = abs(self._angle - int(self._angle))
+        decimalMinutes = 60 * fractionalDegrees
+
+        minutes = int(decimalMinutes)
+        fractionalMinutes = decimalMinutes - int(decimalMinutes)
+        decimalSeconds = 60 * fractionalMinutes
+
+        return degrees, minutes, int(decimalSeconds)
+
+    def setSign(self, sign):
+        """
+        Sets the sign of this angle.
+
+        @param sign: The new sign. C{1} for positive and C{-1} for negative
+            signs, respectively.
+        @type sign: C{int}
+
+        @raise ValueError: If the C{sign} parameter is not C{-1} or C{1}.
+        """
+        if sign not in (-1, 1):
+            raise ValueError("bad sign (got %s, expected -1 or 1)" % sign)
+
+        self._angle = sign * abs(self._angle)
+
+    def __float__(self):
+        """
+        Returns this angle as a float.
+
+        @return: The float value of this angle, expressed in degrees.
+        @rtype: C{float}
+        """
+        return self._angle
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of this angle.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        return "<{s._angleTypeNameRepr} ({s._angleValueRepr})>".format(s=self)
+
+    @property
+    def _angleValueRepr(self):
+        """
+        Returns a string representation of the angular value of this angle.
+
+        This is a helper function for the actual C{__repr__}.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        if self.inDecimalDegrees is not None:
+            return "%s degrees" % round(self.inDecimalDegrees, 2)
+        else:
+            return "unknown value"
+
+    @property
+    def _angleTypeNameRepr(self):
+        """
+        Returns a string representation of the type of this angle.
+
+        This is a helper function for the actual C{__repr__}.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        try:
+            return self._ANGLE_TYPE_NAMES[self.angleType]
+        except KeyError:
+            return "Angle of unknown type"
+
+
+class Heading(Angle):
+    """
+    The heading of a mobile object.
+
+    @ivar variation: The (optional) magnetic variation.
+        The sign of the variation is positive for variations towards the east
+        (clockwise from north), and negative for variations towards the west
+        (counterclockwise from north).
+        If the variation is unknown or not applicable, this is L{None}.
+    @type variation: C{Angle} or L{None}.
+    @ivar correctedHeading: The heading, corrected for variation. If the
+        variation is unknown (L{None}), is None. This attribute is read-only
+        (its value is determined by the angle and variation attributes). The
+        value is coerced to being between 0 (inclusive) and 360 (exclusive).
+    """
+
+    def __init__(self, angle=None, variation=None):
+        """
+        Initializes an angle with an optional variation.
+        """
+        Angle.__init__(self, angle, Angles.HEADING)
+        self.variation = variation
+
+    @classmethod
+    def fromFloats(cls, angleValue=None, variationValue=None):
+        """
+        Constructs a Heading from the float values of the angle and variation.
+
+        @param angleValue: The angle value of this heading.
+        @type angleValue: C{float}
+        @param variationValue: The value of the variation of this heading.
+        @type variationValue: C{float}
+        @return: A L{Heading} with the given values.
+        """
+        variation = Angle(variationValue, Angles.VARIATION)
+        return cls(angleValue, variation)
+
+    @property
+    def correctedHeading(self):
+        """
+        Corrects the heading by the given variation. This is sometimes known as
+        the true heading.
+
+        @return: The heading, corrected by the variation. If the variation or
+            the angle are unknown, returns L{None}.
+        @rtype: C{float} or L{None}
+        """
+        if self._angle is None or self.variation is None:
+            return None
+
+        angle = (self.inDecimalDegrees - self.variation.inDecimalDegrees) % 360
+        return Angle(angle, Angles.HEADING)
+
+    def setSign(self, sign):
+        """
+        Sets the sign of the variation of this heading.
+
+        @param sign: The new sign. C{1} for positive and C{-1} for negative
+            signs, respectively.
+        @type sign: C{int}
+
+        @raise ValueError: If the C{sign} parameter is not C{-1} or C{1}.
+        """
+        if self.variation.inDecimalDegrees is None:
+            raise ValueError("can't set the sign of an unknown variation")
+
+        self.variation.setSign(sign)
+
+    compareAttributes = list(Angle.compareAttributes) + ["variation"]
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of this angle.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        if self.variation is None:
+            variationRepr = "unknown variation"
+        else:
+            variationRepr = repr(self.variation)
+
+        return "<{} ({}, {})>".format(
+            self._angleTypeNameRepr,
+            self._angleValueRepr,
+            variationRepr,
+        )
+
+
+class Coordinate(Angle):
+    """
+    A coordinate.
+
+    @ivar angle: The value of the coordinate in decimal degrees, with the usual
+        rules for sign (northern and eastern hemispheres are positive, southern
+        and western hemispheres are negative).
+    @type angle: C{float}
+    """
+
+    def __init__(self, angle, coordinateType=None):
+        """
+        Initializes a coordinate.
+
+        @param angle: The angle of this coordinate in decimal degrees. The
+            hemisphere is determined by the sign (north and east are positive).
+            If this coordinate describes a latitude, this value must be within
+            -90.0 and +90.0 (exclusive). If this value describes a longitude,
+            this value must be within -180.0 and +180.0 (exclusive).
+        @type angle: C{float}
+        @param coordinateType: The coordinate type. One of L{Angles.LATITUDE},
+            L{Angles.LONGITUDE} or L{None} if unknown.
+        """
+        if coordinateType not in [Angles.LATITUDE, Angles.LONGITUDE, None]:
+            raise ValueError(
+                "coordinateType must be one of Angles.LATITUDE, "
+                "Angles.LONGITUDE or None, was {!r}".format(coordinateType)
+            )
+
+        Angle.__init__(self, angle, coordinateType)
+
+    @property
+    def hemisphere(self):
+        """
+        Gets the hemisphere of this coordinate.
+
+        @return: A symbolic constant representing a hemisphere (one of
+            L{Angles})
+        """
+
+        if self.angleType is Angles.LATITUDE:
+            if self.inDecimalDegrees < 0:
+                return Directions.SOUTH
+            else:
+                return Directions.NORTH
+        elif self.angleType is Angles.LONGITUDE:
+            if self.inDecimalDegrees < 0:
+                return Directions.WEST
+            else:
+                return Directions.EAST
+        else:
+            raise ValueError("unknown coordinate type (cant find hemisphere)")
+
+
+class Altitude(FancyEqMixin):
+    """
+    An altitude.
+
+    @ivar inMeters: The altitude represented by this object, in meters. This
+        attribute is read-only.
+    @type inMeters: C{float}
+
+    @ivar inFeet: As above, but expressed in feet.
+    @type inFeet: C{float}
+    """
+
+    compareAttributes = ("inMeters",)
+
+    def __init__(self, altitude):
+        """
+        Initializes an altitude.
+
+        @param altitude: The altitude in meters.
+        @type altitude: C{float}
+        """
+        self._altitude = altitude
+
+    @property
+    def inFeet(self):
+        """
+        Gets the altitude this object represents, in feet.
+
+        @return: The altitude, expressed in feet.
+        @rtype: C{float}
+        """
+        return self._altitude / METERS_PER_FOOT
+
+    @property
+    def inMeters(self):
+        """
+        Returns the altitude this object represents, in meters.
+
+        @return: The altitude, expressed in feet.
+        @rtype: C{float}
+        """
+        return self._altitude
+
+    def __float__(self):
+        """
+        Returns the altitude represented by this object expressed in meters.
+
+        @return: The altitude represented by this object, expressed in meters.
+        @rtype: C{float}
+        """
+        return self._altitude
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of this altitude.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        return f"<Altitude ({self._altitude} m)>"
+
+
+class _BaseSpeed(FancyEqMixin):
+    """
+    An object representing the abstract concept of the speed (rate of
+    movement) of a mobile object.
+
+    This primarily has behavior for converting between units and comparison.
+    """
+
+    compareAttributes = ("inMetersPerSecond",)
+
+    def __init__(self, speed):
+        """
+        Initializes a speed.
+
+        @param speed: The speed that this object represents, expressed in
+            meters per second.
+        @type speed: C{float}
+
+        @raises ValueError: Raised if value was invalid for this particular
+            kind of speed. Only happens in subclasses.
+        """
+        self._speed = speed
+
+    @property
+    def inMetersPerSecond(self):
+        """
+        The speed that this object represents, expressed in meters per second.
+        This attribute is immutable.
+
+        @return: The speed this object represents, in meters per second.
+        @rtype: C{float}
+        """
+        return self._speed
+
+    @property
+    def inKnots(self):
+        """
+        Returns the speed represented by this object, expressed in knots. This
+        attribute is immutable.
+
+        @return: The speed this object represents, in knots.
+        @rtype: C{float}
+        """
+        return self._speed / MPS_PER_KNOT
+
+    def __float__(self):
+        """
+        Returns the speed represented by this object expressed in meters per
+        second.
+
+        @return: The speed represented by this object, expressed in meters per
+            second.
+        @rtype: C{float}
+        """
+        return self._speed
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of this speed object.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        speedValue = round(self.inMetersPerSecond, 2)
+        return f"<{self.__class__.__name__} ({speedValue} m/s)>"
+
+
+class Speed(_BaseSpeed):
+    """
+    The speed (rate of movement) of a mobile object.
+    """
+
+    def __init__(self, speed):
+        """
+        Initializes a L{Speed} object.
+
+        @param speed: The speed that this object represents, expressed in
+            meters per second.
+        @type speed: C{float}
+
+        @raises ValueError: Raised if C{speed} is negative.
+        """
+        if speed < 0:
+            raise ValueError(f"negative speed: {speed!r}")
+
+        _BaseSpeed.__init__(self, speed)
+
+
+class Climb(_BaseSpeed):
+    """
+    The climb ("vertical speed") of an object.
+    """
+
+    def __init__(self, climb):
+        """
+        Initializes a L{Climb} object.
+
+        @param climb: The climb that this object represents, expressed in
+            meters per second.
+        @type climb: C{float}
+        """
+        _BaseSpeed.__init__(self, climb)
+
+
+class PositionError(FancyEqMixin):
+    """
+    Position error information.
+
+    @cvar _ALLOWABLE_THRESHOLD: The maximum allowable difference between PDOP
+        and the geometric mean of VDOP and HDOP. That difference is supposed
+        to be zero, but can be non-zero because of rounding error and limited
+        reporting precision. You should never have to change this value.
+    @type _ALLOWABLE_THRESHOLD: C{float}
+    @cvar _DOP_EXPRESSIONS: A mapping of DOP types (C[hvp]dop) to a list of
+        callables that take self and return that DOP type, or raise
+        C{TypeError}. This allows a DOP value to either be returned directly
+        if it's know, or computed from other DOP types if it isn't.
+    @type _DOP_EXPRESSIONS: C{dict} of C{str} to callables
+    @ivar pdop: The position dilution of precision. L{None} if unknown.
+    @type pdop: C{float} or L{None}
+    @ivar hdop: The horizontal dilution of precision. L{None} if unknown.
+    @type hdop: C{float} or L{None}
+    @ivar vdop: The vertical dilution of precision. L{None} if unknown.
+    @type vdop: C{float} or L{None}
+    """
+
+    compareAttributes = "pdop", "hdop", "vdop"
+
+    def __init__(self, pdop=None, hdop=None, vdop=None, testInvariant=False):
+        """
+        Initializes a positioning error object.
+
+        @param pdop: The position dilution of precision. L{None} if unknown.
+        @type pdop: C{float} or L{None}
+        @param hdop: The horizontal dilution of precision. L{None} if unknown.
+        @type hdop: C{float} or L{None}
+        @param vdop: The vertical dilution of precision. L{None} if unknown.
+        @type vdop: C{float} or L{None}
+        @param testInvariant: Flag to test if the DOP invariant is valid or
+            not. If C{True}, the invariant (PDOP = (HDOP**2 + VDOP**2)*.5) is
+            checked at every mutation. By default, this is false, because the
+            vast majority of DOP-providing devices ignore this invariant.
+        @type testInvariant: c{bool}
+        """
+        self._pdop = pdop
+        self._hdop = hdop
+        self._vdop = vdop
+
+        self._testInvariant = testInvariant
+        self._testDilutionOfPositionInvariant()
+
+    _ALLOWABLE_TRESHOLD = 0.01
+
+    def _testDilutionOfPositionInvariant(self):
+        """
+        Tests if this positioning error object satisfies the dilution of
+        position invariant (PDOP = (HDOP**2 + VDOP**2)*.5), unless the
+        C{self._testInvariant} instance variable is C{False}.
+
+        @return: L{None} if the invariant was not satisfied or not tested.
+        @raises ValueError: Raised if the invariant was tested but not
+            satisfied.
+        """
+        if not self._testInvariant:
+            return
+
+        for x in (self.pdop, self.hdop, self.vdop):
+            if x is None:
+                return
+
+        delta = abs(self.pdop - (self.hdop**2 + self.vdop**2) ** 0.5)
+        if delta > self._ALLOWABLE_TRESHOLD:
+            raise ValueError(
+                "invalid combination of dilutions of precision: "
+                "position: %s, horizontal: %s, vertical: %s"
+                % (self.pdop, self.hdop, self.vdop)
+            )
+
+    _DOP_EXPRESSIONS = {
+        "pdop": [
+            lambda self: float(self._pdop),
+            lambda self: (self._hdop**2 + self._vdop**2) ** 0.5,
+        ],
+        "hdop": [
+            lambda self: float(self._hdop),
+            lambda self: (self._pdop**2 - self._vdop**2) ** 0.5,
+        ],
+        "vdop": [
+            lambda self: float(self._vdop),
+            lambda self: (self._pdop**2 - self._hdop**2) ** 0.5,
+        ],
+    }
+
+    def _getDOP(self, dopType):
+        """
+        Gets a particular dilution of position value.
+
+        @param dopType: The type of dilution of position to get. One of
+            ('pdop', 'hdop', 'vdop').
+        @type dopType: C{str}
+        @return: The DOP if it is known, L{None} otherwise.
+        @rtype: C{float} or L{None}
+        """
+        for dopExpression in self._DOP_EXPRESSIONS[dopType]:
+            try:
+                return dopExpression(self)
+            except TypeError:
+                continue
+
+    def _setDOP(self, dopType, value):
+        """
+        Sets a particular dilution of position value.
+
+        @param dopType: The type of dilution of position to set. One of
+            ('pdop', 'hdop', 'vdop').
+        @type dopType: C{str}
+
+        @param value: The value to set the dilution of position type to.
+        @type value: C{float}
+
+        If this position error tests dilution of precision invariants,
+        it will be checked. If the invariant is not satisfied, the
+        assignment will be undone and C{ValueError} is raised.
+        """
+        attributeName = "_" + dopType
+
+        oldValue = getattr(self, attributeName)
+        setattr(self, attributeName, float(value))
+
+        try:
+            self._testDilutionOfPositionInvariant()
+        except ValueError:
+            setattr(self, attributeName, oldValue)
+            raise
+
+    @property
+    def pdop(self):
+        return self._getDOP("pdop")
+
+    @pdop.setter
+    def pdop(self, value):
+        return self._setDOP("pdop", value)
+
+    @property
+    def hdop(self):
+        return self._getDOP("hdop")
+
+    @hdop.setter
+    def hdop(self, value):
+        return self._setDOP("hdop", value)
+
+    @property
+    def vdop(self):
+        return self._getDOP("vdop")
+
+    @vdop.setter
+    def vdop(self, value):
+        return self._setDOP("vdop", value)
+
+    _REPR_TEMPLATE = "<PositionError (pdop: %s, hdop: %s, vdop: %s)>"
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of positioning information object.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        return self._REPR_TEMPLATE % (self.pdop, self.hdop, self.vdop)
+
+
+class BeaconInformation:
+    """
+    Information about positioning beacons (a generalized term for the reference
+    objects that help you determine your position, such as satellites or cell
+    towers).
+
+    @ivar seenBeacons: A set of visible beacons. Note that visible beacons are not
+        necessarily used in acquiring a positioning fix.
+    @type seenBeacons: C{set} of L{IPositioningBeacon}
+    @ivar usedBeacons: A set of the beacons that were used in obtaining a
+        positioning fix. This only contains beacons that are actually used, not
+        beacons for which it is unknown if they are used or not.
+    @type usedBeacons: C{set} of L{IPositioningBeacon}
+    """
+
+    def __init__(self, seenBeacons=()):
+        """
+        Initializes a beacon information object.
+
+        @param seenBeacons: A collection of beacons that are currently seen.
+        @type seenBeacons: iterable of L{IPositioningBeacon}s
+        """
+        self.seenBeacons = set(seenBeacons)
+        self.usedBeacons = set()
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of this beacon information object.
+
+        The beacons are sorted by their identifier.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        sortedBeacons = partial(sorted, key=attrgetter("identifier"))
+
+        usedBeacons = sortedBeacons(self.usedBeacons)
+        unusedBeacons = sortedBeacons(self.seenBeacons - self.usedBeacons)
+
+        template = (
+            "<BeaconInformation ("
+            "used beacons ({numUsed}): {usedBeacons}, "
+            "unused beacons: {unusedBeacons})>"
+        )
+
+        formatted = template.format(
+            numUsed=len(self.usedBeacons),
+            usedBeacons=usedBeacons,
+            unusedBeacons=unusedBeacons,
+        )
+
+        return formatted
+
+
+@implementer(ipositioning.IPositioningBeacon)
+class PositioningBeacon:
+    """
+    A positioning beacon.
+
+    @ivar identifier: The unique identifier for this beacon. This is usually
+        an integer. For GPS, this is also known as the PRN.
+    @type identifier: Pretty much anything that can be used as a unique
+        identifier. Depends on the implementation.
+    """
+
+    def __init__(self, identifier):
+        """
+        Initializes a positioning beacon.
+
+        @param identifier: The identifier for this beacon.
+        @type identifier: Can be pretty much anything (see ivar documentation).
+        """
+        self.identifier = identifier
+
+    def __hash__(self):
+        """
+        Returns the hash of the identifier for this beacon.
+
+        @return: The hash of the identifier. (C{hash(self.identifier)})
+        @rtype: C{int}
+        """
+        return hash(self.identifier)
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of this beacon.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        return f"<Beacon ({self.identifier})>"
+
+
+class Satellite(PositioningBeacon):
+    """
+    A satellite.
+
+    @ivar azimuth: The azimuth of the satellite. This is the heading (positive
+        angle relative to true north) where the satellite appears to be to the
+        device.
+    @ivar elevation: The (positive) angle above the horizon where this
+        satellite appears to be to the device.
+    @ivar signalToNoiseRatio: The signal to noise ratio of the signal coming
+        from this satellite.
+    """
+
+    def __init__(
+        self, identifier, azimuth=None, elevation=None, signalToNoiseRatio=None
+    ):
+        """
+        Initializes a satellite object.
+
+        @param identifier: The PRN (unique identifier) of this satellite.
+        @type identifier: C{int}
+        @param azimuth: The azimuth of the satellite (see instance variable
+            documentation).
+        @type azimuth: C{float}
+        @param elevation: The elevation of the satellite (see instance variable
+            documentation).
+        @type elevation: C{float}
+        @param signalToNoiseRatio: The signal to noise ratio of the connection
+            to this satellite (see instance variable documentation).
+        @type signalToNoiseRatio: C{float}
+        """
+        PositioningBeacon.__init__(self, int(identifier))
+
+        self.azimuth = azimuth
+        self.elevation = elevation
+        self.signalToNoiseRatio = signalToNoiseRatio
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of this Satellite.
+
+        @return: The string representation.
+        @rtype: C{str}
+        """
+        template = (
+            "<Satellite ({s.identifier}), "
+            "azimuth: {s.azimuth}, "
+            "elevation: {s.elevation}, "
+            "snr: {s.signalToNoiseRatio}>"
+        )
+
+        return template.format(s=self)
+
+
+__all__ = [
+    "Altitude",
+    "Angle",
+    "Angles",
+    "BasePositioningReceiver",
+    "BeaconInformation",
+    "Climb",
+    "Coordinate",
+    "Directions",
+    "Heading",
+    "InvalidChecksum",
+    "InvalidSentence",
+    "METERS_PER_FOOT",
+    "MPS_PER_KNOT",
+    "MPS_PER_KPH",
+    "PositionError",
+    "PositioningBeacon",
+    "Satellite",
+    "Speed",
+]
diff --git a/contrib/python/Twisted/py3/twisted/positioning/ipositioning.py b/contrib/python/Twisted/py3/twisted/positioning/ipositioning.py
new file mode 100644
index 00000000000..9eaee9a519d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/positioning/ipositioning.py
@@ -0,0 +1,113 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Positioning interfaces.
+
+@since: 14.0
+"""
+
+
+from zope.interface import Attribute, Interface
+
+
+class IPositioningReceiver(Interface):
+    """
+    An interface for positioning providers.
+    """
+
+    def positionReceived(latitude, longitude):
+        """
+        Method called when a position is received.
+
+        @param latitude: The latitude of the received position.
+        @type latitude: L{twisted.positioning.base.Coordinate}
+        @param longitude: The longitude of the received position.
+        @type longitude: L{twisted.positioning.base.Coordinate}
+        """
+
+    def positionErrorReceived(positionError):
+        """
+        Method called when position error is received.
+
+        @param positionError: The position error.
+        @type positionError: L{twisted.positioning.base.PositionError}
+        """
+
+    def timeReceived(time):
+        """
+        Method called when time and date information arrives.
+
+        @param time: The date and time (expressed in UTC unless otherwise
+            specified).
+        @type time: L{datetime.datetime}
+        """
+
+    def headingReceived(heading):
+        """
+        Method called when a true heading is received.
+
+        @param heading: The heading.
+        @type heading: L{twisted.positioning.base.Heading}
+        """
+
+    def altitudeReceived(altitude):
+        """
+        Method called when an altitude is received.
+
+        @param altitude: The altitude.
+        @type altitude: L{twisted.positioning.base.Altitude}
+        """
+
+    def speedReceived(speed):
+        """
+        Method called when the speed is received.
+
+        @param speed: The speed of a mobile object.
+        @type speed: L{twisted.positioning.base.Speed}
+        """
+
+    def climbReceived(climb):
+        """
+        Method called when the climb is received.
+
+        @param climb: The climb of the mobile object.
+        @type climb: L{twisted.positioning.base.Climb}
+        """
+
+    def beaconInformationReceived(beaconInformation):
+        """
+        Method called when positioning beacon information is received.
+
+        @param beaconInformation: The beacon information.
+        @type beaconInformation: L{twisted.positioning.base.BeaconInformation}
+        """
+
+
+class IPositioningBeacon(Interface):
+    """
+    A positioning beacon.
+    """
+
+    identifier = Attribute(
+        """
+        A unique identifier for this beacon. The type is dependent on the
+        implementation, but must be immutable.
+        """
+    )
+
+
+class INMEAReceiver(Interface):
+    """
+    An object that can receive NMEA data.
+    """
+
+    def sentenceReceived(sentence):
+        """
+        Method called when a sentence is received.
+
+        @param sentence: The received NMEA sentence.
+        @type L{twisted.positioning.nmea.NMEASentence}
+        """
+
+
+__all__ = ["IPositioningReceiver", "IPositioningBeacon", "INMEAReceiver"]
diff --git a/contrib/python/Twisted/py3/twisted/positioning/nmea.py b/contrib/python/Twisted/py3/twisted/positioning/nmea.py
new file mode 100644
index 00000000000..128c1b7938c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/positioning/nmea.py
@@ -0,0 +1,932 @@
+# -*- test-case-name: twisted.positioning.test.test_nmea -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Classes for working with NMEA 0183 sentence producing devices.
+This standard is generally just called "NMEA", which is actually the
+name of the body that produces the standard, not the standard itself..
+
+For more information, read the blog post on NMEA by ESR (the gpsd
+maintainer) at U{http://esr.ibiblio.org/?p=801}. Unfortunately,
+official specifications on NMEA 0183 are only available at a cost.
+
+More information can be found on the Wikipedia page:
+U{https://en.wikipedia.org/wiki/NMEA_0183}.
+
+The official standard may be obtained through the NMEA's website:
+U{http://www.nmea.org/content/nmea_standards/nmea_0183_v_410.asp}.
+
+@since: 14.0
+"""
+
+
+import datetime
+import operator
+from functools import reduce
+
+from zope.interface import implementer
+
+from constantly import ValueConstant, Values  # type: ignore[import]
+
+from twisted.positioning import _sentence, base, ipositioning
+from twisted.positioning.base import Angles
+from twisted.protocols.basic import LineReceiver
+from twisted.python.compat import iterbytes, nativeString
+
+
+class GPGGAFixQualities(Values):
+    """
+    The possible fix quality indications for GPGGA sentences.
+
+    @cvar INVALID_FIX: The fix is invalid.
+    @cvar GPS_FIX: There is a fix, acquired using GPS.
+    @cvar DGPS_FIX: There is a fix, acquired using differential GPS (DGPS).
+    @cvar PPS_FIX: There is a fix, acquired using the precise positioning
+        service (PPS).
+    @cvar RTK_FIX: There is a fix, acquired using fixed real-time
+        kinematics. This means that there was a sufficient number of shared
+        satellites with the base station, usually yielding a resolution in
+        the centimeter range. This was added in NMEA 0183 version 3.0. This
+        is also called Carrier-Phase Enhancement or CPGPS, particularly when
+        used in combination with GPS.
+    @cvar FLOAT_RTK_FIX: There is a fix, acquired using floating real-time
+        kinematics. The same comments apply as for a fixed real-time
+        kinematics fix, except that there were insufficient shared satellites
+        to acquire it, so instead you got a slightly less good floating fix.
+        Typical resolution in the decimeter range.
+    @cvar DEAD_RECKONING: There is currently no more fix, but this data was
+        computed using a previous fix and some information about motion
+        (either from that fix or from other sources) using simple dead
+        reckoning. Not particularly reliable, but better-than-nonsense data.
+    @cvar MANUAL: There is no real fix from this device, but the location has
+        been manually entered, presumably with data obtained from some other
+        positioning method.
+    @cvar SIMULATED: There is no real fix, but instead it is being simulated.
+    """
+
+    INVALID_FIX = "0"
+    GPS_FIX = "1"
+    DGPS_FIX = "2"
+    PPS_FIX = "3"
+    RTK_FIX = "4"
+    FLOAT_RTK_FIX = "5"
+    DEAD_RECKONING = "6"
+    MANUAL = "7"
+    SIMULATED = "8"
+
+
+class GPGLLGPRMCFixQualities(Values):
+    """
+    The possible fix quality indications in GPGLL and GPRMC sentences.
+
+    Unfortunately, these sentences only indicate whether data is good or void.
+    They provide no other information, such as what went wrong if the data is
+    void, or how good the data is if the data is not void.
+
+    @cvar ACTIVE: The data is okay.
+    @cvar VOID: The data is void, and should not be used.
+    """
+
+    ACTIVE = ValueConstant("A")
+    VOID = ValueConstant("V")
+
+
+class GPGSAFixTypes(Values):
+    """
+    The possible fix types of a GPGSA sentence.
+
+    @cvar GSA_NO_FIX: The sentence reports no fix at all.
+    @cvar GSA_2D_FIX: The sentence reports a 2D fix: position but no altitude.
+    @cvar GSA_3D_FIX: The sentence reports a 3D fix: position with altitude.
+    """
+
+    GSA_NO_FIX = ValueConstant("1")
+    GSA_2D_FIX = ValueConstant("2")
+    GSA_3D_FIX = ValueConstant("3")
+
+
+def _split(sentence):
+    """
+    Returns the split version of an NMEA sentence, minus header
+    and checksum.
+
+    >>> _split(b"$GPGGA,spam,eggs*00")
+    [b'GPGGA', b'spam', b'eggs']
+
+    @param sentence: The NMEA sentence to split.
+    @type sentence: C{bytes}
+    """
+    if sentence[-3:-2] == b"*":  # Sentence with checksum
+        return sentence[1:-3].split(b",")
+    elif sentence[-1:] == b"*":  # Sentence without checksum
+        return sentence[1:-1].split(b",")
+    else:
+        raise base.InvalidSentence(f"malformed sentence {sentence}")
+
+
+def _validateChecksum(sentence):
+    """
+    Validates the checksum of an NMEA sentence.
+
+    @param sentence: The NMEA sentence to check the checksum of.
+    @type sentence: C{bytes}
+
+    @raise ValueError: If the sentence has an invalid checksum.
+
+    Simply returns on sentences that either don't have a checksum,
+    or have a valid checksum.
+    """
+    if sentence[-3:-2] == b"*":  # Sentence has a checksum
+        reference, source = int(sentence[-2:], 16), sentence[1:-3]
+        computed = reduce(operator.xor, [ord(x) for x in iterbytes(source)])
+        if computed != reference:
+            raise base.InvalidChecksum(f"{computed:02x} != {reference:02x}")
+
+
+class NMEAProtocol(LineReceiver, _sentence._PositioningSentenceProducerMixin):
+    """
+    A protocol that parses and verifies the checksum of an NMEA sentence (in
+    string form, not L{NMEASentence}), and delegates to a receiver.
+
+    It receives lines and verifies these lines are NMEA sentences. If
+    they are, verifies their checksum and unpacks them into their
+    components. It then wraps them in L{NMEASentence} objects and
+    calls the appropriate receiver method with them.
+
+    @cvar _SENTENCE_CONTENTS: Has the field names in an NMEA sentence for each
+        sentence type (in order, obviously).
+    @type _SENTENCE_CONTENTS: C{dict} of bytestrings to C{list}s of C{str}
+    @param receiver: A receiver for NMEAProtocol sentence objects.
+    @type receiver: L{INMEAReceiver}
+    @param sentenceCallback: A function that will be called with a new
+        L{NMEASentence} when it is created. Useful for massaging data from
+        particularly misbehaving NMEA receivers.
+    @type sentenceCallback: unary callable
+    """
+
+    def __init__(self, receiver, sentenceCallback=None):
+        """
+        Initializes an NMEAProtocol.
+
+        @param receiver: A receiver for NMEAProtocol sentence objects.
+        @type receiver: L{INMEAReceiver}
+        @param sentenceCallback: A function that will be called with a new
+            L{NMEASentence} when it is created. Useful for massaging data from
+            particularly misbehaving NMEA receivers.
+        @type sentenceCallback: unary callable
+        """
+        self._receiver = receiver
+        self._sentenceCallback = sentenceCallback
+
+    def lineReceived(self, rawSentence):
+        """
+        Parses the data from the sentence and validates the checksum.
+
+        @param rawSentence: The NMEA positioning sentence.
+        @type rawSentence: C{bytes}
+        """
+        sentence = rawSentence.strip()
+
+        _validateChecksum(sentence)
+        splitSentence = _split(sentence)
+
+        sentenceType = nativeString(splitSentence[0])
+        contents = [nativeString(x) for x in splitSentence[1:]]
+
+        try:
+            keys = self._SENTENCE_CONTENTS[sentenceType]
+        except KeyError:
+            raise ValueError("unknown sentence type %s" % sentenceType)
+
+        sentenceData = {"type": sentenceType}
+        for key, value in zip(keys, contents):
+            if key is not None and value != "":
+                sentenceData[key] = value
+
+        sentence = NMEASentence(sentenceData)
+
+        if self._sentenceCallback is not None:
+            self._sentenceCallback(sentence)
+
+        self._receiver.sentenceReceived(sentence)
+
+    _SENTENCE_CONTENTS = {
+        "GPGGA": [
+            "timestamp",
+            "latitudeFloat",
+            "latitudeHemisphere",
+            "longitudeFloat",
+            "longitudeHemisphere",
+            "fixQuality",
+            "numberOfSatellitesSeen",
+            "horizontalDilutionOfPrecision",
+            "altitude",
+            "altitudeUnits",
+            "heightOfGeoidAboveWGS84",
+            "heightOfGeoidAboveWGS84Units",
+            # The next parts are DGPS information, currently unused.
+            None,  # Time since last DGPS update
+            None,  # DGPS reference source id
+        ],
+        "GPRMC": [
+            "timestamp",
+            "dataMode",
+            "latitudeFloat",
+            "latitudeHemisphere",
+            "longitudeFloat",
+            "longitudeHemisphere",
+            "speedInKnots",
+            "trueHeading",
+            "datestamp",
+            "magneticVariation",
+            "magneticVariationDirection",
+        ],
+        "GPGSV": [
+            "numberOfGSVSentences",
+            "GSVSentenceIndex",
+            "numberOfSatellitesSeen",
+            "satellitePRN_0",
+            "elevation_0",
+            "azimuth_0",
+            "signalToNoiseRatio_0",
+            "satellitePRN_1",
+            "elevation_1",
+            "azimuth_1",
+            "signalToNoiseRatio_1",
+            "satellitePRN_2",
+            "elevation_2",
+            "azimuth_2",
+            "signalToNoiseRatio_2",
+            "satellitePRN_3",
+            "elevation_3",
+            "azimuth_3",
+            "signalToNoiseRatio_3",
+        ],
+        "GPGLL": [
+            "latitudeFloat",
+            "latitudeHemisphere",
+            "longitudeFloat",
+            "longitudeHemisphere",
+            "timestamp",
+            "dataMode",
+        ],
+        "GPHDT": [
+            "trueHeading",
+        ],
+        "GPTRF": [
+            "datestamp",
+            "timestamp",
+            "latitudeFloat",
+            "latitudeHemisphere",
+            "longitudeFloat",
+            "longitudeHemisphere",
+            "elevation",
+            "numberOfIterations",  # Unused
+            "numberOfDopplerIntervals",  # Unused
+            "updateDistanceInNauticalMiles",  # Unused
+            "satellitePRN",
+        ],
+        "GPGSA": [
+            "dataMode",
+            "fixType",
+            "usedSatellitePRN_0",
+            "usedSatellitePRN_1",
+            "usedSatellitePRN_2",
+            "usedSatellitePRN_3",
+            "usedSatellitePRN_4",
+            "usedSatellitePRN_5",
+            "usedSatellitePRN_6",
+            "usedSatellitePRN_7",
+            "usedSatellitePRN_8",
+            "usedSatellitePRN_9",
+            "usedSatellitePRN_10",
+            "usedSatellitePRN_11",
+            "positionDilutionOfPrecision",
+            "horizontalDilutionOfPrecision",
+            "verticalDilutionOfPrecision",
+        ],
+    }
+
+
+class NMEASentence(_sentence._BaseSentence):
+    """
+    An object representing an NMEA sentence.
+
+    The attributes of this objects are raw NMEA protocol data, which
+    are all ASCII bytestrings.
+
+    This object contains all the raw NMEA protocol data in a single
+    sentence.  Not all of these necessarily have to be present in the
+    sentence. Missing attributes are L{None} when accessed.
+
+    @ivar type: The sentence type (C{"GPGGA"}, C{"GPGSV"}...).
+    @ivar numberOfGSVSentences: The total number of GSV sentences in a
+        sequence.
+    @ivar GSVSentenceIndex: The index of this GSV sentence in the GSV
+        sequence.
+    @ivar timestamp: A timestamp. (C{"123456"} -> 12:34:56Z)
+    @ivar datestamp: A datestamp. (C{"230394"} -> 23 Mar 1994)
+    @ivar latitudeFloat: Latitude value. (for example: C{"1234.567"} ->
+        12 degrees, 34.567 minutes).
+    @ivar latitudeHemisphere: Latitudinal hemisphere (C{"N"} or C{"S"}).
+    @ivar longitudeFloat: Longitude value. See C{latitudeFloat} for an
+        example.
+    @ivar longitudeHemisphere: Longitudinal hemisphere (C{"E"} or C{"W"}).
+    @ivar altitude: The altitude above mean sea level.
+    @ivar altitudeUnits: Units in which altitude is expressed. (Always
+        C{"M"} for meters.)
+    @ivar heightOfGeoidAboveWGS84: The local height of the geoid above
+        the WGS84 ellipsoid model.
+    @ivar heightOfGeoidAboveWGS84Units: The units in which the height
+        above the geoid is expressed. (Always C{"M"} for meters.)
+    @ivar trueHeading: The true heading.
+    @ivar magneticVariation: The magnetic variation.
+    @ivar magneticVariationDirection: The direction of the magnetic
+        variation. One of C{"E"} or C{"W"}.
+    @ivar speedInKnots: The ground speed, expressed in knots.
+    @ivar fixQuality: The quality of the fix.
+    @type fixQuality: One of L{GPGGAFixQualities}.
+    @ivar dataMode: Signals if the data is usable or not.
+    @type dataMode: One of L{GPGLLGPRMCFixQualities}.
+    @ivar numberOfSatellitesSeen: The number of satellites seen by the
+        receiver.
+    @ivar numberOfSatellitesUsed: The number of satellites used in
+        computing the fix.
+    @ivar horizontalDilutionOfPrecision: The dilution of the precision of the
+        position on a plane tangential to the geoid. (HDOP)
+    @ivar verticalDilutionOfPrecision: As C{horizontalDilutionOfPrecision},
+        but for a position on a plane perpendicular to the geoid. (VDOP)
+    @ivar positionDilutionOfPrecision: Euclidean norm of HDOP and VDOP.
+    @ivar satellitePRN: The unique identifcation number of a particular
+        satellite. Optionally suffixed with C{_N} if multiple satellites are
+        referenced in a sentence, where C{N in range(4)}.
+    @ivar elevation: The elevation of a satellite in decimal degrees.
+        Optionally suffixed with C{_N}, as with C{satellitePRN}.
+    @ivar azimuth: The azimuth of a satellite in decimal degrees.
+        Optionally suffixed with C{_N}, as with C{satellitePRN}.
+    @ivar signalToNoiseRatio: The SNR of a satellite signal, in decibels.
+        Optionally suffixed with C{_N}, as with C{satellitePRN}.
+    @ivar usedSatellitePRN_N: Where C{int(N) in range(12)}. The PRN
+        of a satellite used in computing the fix.
+    """
+
+    ALLOWED_ATTRIBUTES = NMEAProtocol.getSentenceAttributes()
+
+    def _isFirstGSVSentence(self):
+        """
+        Tests if this current GSV sentence is the first one in a sequence.
+
+        @return: C{True} if this is the first GSV sentence.
+        @rtype: C{bool}
+        """
+        return self.GSVSentenceIndex == "1"
+
+    def _isLastGSVSentence(self):
+        """
+        Tests if this current GSV sentence is the final one in a sequence.
+
+        @return: C{True} if this is the last GSV sentence.
+        @rtype: C{bool}
+        """
+        return self.GSVSentenceIndex == self.numberOfGSVSentences
+
+
+@implementer(ipositioning.INMEAReceiver)
+class NMEAAdapter:
+    """
+    An adapter from NMEAProtocol receivers to positioning receivers.
+
+    @cvar _STATEFUL_UPDATE: Information on how to update partial information
+        in the sentence data or internal adapter state. For more information,
+        see C{_statefulUpdate}'s docstring.
+    @type _STATEFUL_UPDATE: See C{_statefulUpdate}'s docstring
+    @cvar _ACCEPTABLE_UNITS: A set of NMEA notations of units that are
+        already acceptable (metric), and therefore don't need to be converted.
+    @type _ACCEPTABLE_UNITS: C{frozenset} of bytestrings
+    @cvar _UNIT_CONVERTERS: Mapping of NMEA notations of units that are not
+        acceptable (not metric) to converters that take a quantity in that
+        unit and produce a metric quantity.
+    @type _UNIT_CONVERTERS: C{dict} of bytestrings to unary callables
+    @cvar  _SPECIFIC_SENTENCE_FIXES: A mapping of sentece types to specific
+        fixes that are required to extract useful information from data from
+        those sentences.
+    @type  _SPECIFIC_SENTENCE_FIXES: C{dict} of sentence types to callables
+        that take self and modify it in-place
+    @cvar _FIXERS: Set of unary callables that take an NMEAAdapter instance
+        and extract useful data from the sentence data, usually modifying the
+        adapter's sentence data in-place.
+    @type _FIXERS: C{dict} of native strings to unary callables
+    @ivar yearThreshold: The earliest possible year that data will be
+        interpreted as. For example, if this value is C{1990}, an NMEA
+        0183 two-digit year of "96" will be interpreted as 1996, and
+        a two-digit year of "13" will be interpreted as 2013.
+    @type yearThreshold: L{int}
+    @ivar _state: The current internal state of the receiver.
+    @type _state: C{dict}
+    @ivar _sentenceData: The data present in the sentence currently being
+        processed. Starts empty, is filled as the sentence is parsed.
+    @type _sentenceData: C{dict}
+    @ivar _receiver: The positioning receiver that will receive parsed data.
+    @type _receiver: L{ipositioning.IPositioningReceiver}
+    """
+
+    def __init__(self, receiver):
+        """
+        Initializes a new NMEA adapter.
+
+        @param receiver: The receiver for positioning sentences.
+        @type receiver: L{ipositioning.IPositioningReceiver}
+        """
+        self._state = {}
+        self._sentenceData = {}
+        self._receiver = receiver
+
+    def _fixTimestamp(self):
+        """
+        Turns the NMEAProtocol timestamp notation into a datetime.time object.
+        The time in this object is expressed as Zulu time.
+        """
+        timestamp = self.currentSentence.timestamp.split(".")[0]
+        timeObject = datetime.datetime.strptime(timestamp, "%H%M%S").time()
+        self._sentenceData["_time"] = timeObject
+
+    yearThreshold = 1980
+
+    def _fixDatestamp(self):
+        """
+        Turns an NMEA datestamp format into a C{datetime.date} object.
+
+        @raise ValueError: When the day or month value was invalid, e.g. 32nd
+            day, or 13th month, or 0th day or month.
+        """
+        date = self.currentSentence.datestamp
+        day, month, year = map(int, [date[0:2], date[2:4], date[4:6]])
+
+        year += self.yearThreshold - (self.yearThreshold % 100)
+        if year < self.yearThreshold:
+            year += 100
+
+        self._sentenceData["_date"] = datetime.date(year, month, day)
+
+    def _fixCoordinateFloat(self, coordinateType):
+        """
+        Turns the NMEAProtocol coordinate format into Python float.
+
+        @param coordinateType: The coordinate type.
+        @type coordinateType: One of L{Angles.LATITUDE} or L{Angles.LONGITUDE}.
+        """
+        if coordinateType is Angles.LATITUDE:
+            coordinateName = "latitude"
+        else:  # coordinateType is Angles.LONGITUDE
+            coordinateName = "longitude"
+        nmeaCoordinate = getattr(self.currentSentence, coordinateName + "Float")
+
+        left, right = nmeaCoordinate.split(".")
+
+        degrees, minutes = int(left[:-2]), float(f"{left[-2:]}.{right}")
+        angle = degrees + minutes / 60
+        coordinate = base.Coordinate(angle, coordinateType)
+        self._sentenceData[coordinateName] = coordinate
+
+    def _fixHemisphereSign(self, coordinateType, sentenceDataKey=None):
+        """
+        Fixes the sign for a hemisphere.
+
+        This method must be called after the magnitude for the thing it
+        determines the sign of has been set. This is done by the following
+        functions:
+
+            - C{self.FIXERS['magneticVariation']}
+            - C{self.FIXERS['latitudeFloat']}
+            - C{self.FIXERS['longitudeFloat']}
+
+        @param coordinateType: Coordinate type. One of L{Angles.LATITUDE},
+            L{Angles.LONGITUDE} or L{Angles.VARIATION}.
+        @param sentenceDataKey: The key name of the hemisphere sign being
+            fixed in the sentence data. If unspecified, C{coordinateType} is
+            used.
+        @type sentenceDataKey: C{str} (unless L{None})
+        """
+        sentenceDataKey = sentenceDataKey or coordinateType
+        sign = self._getHemisphereSign(coordinateType)
+        self._sentenceData[sentenceDataKey].setSign(sign)
+
+    def _getHemisphereSign(self, coordinateType):
+        """
+        Returns the hemisphere sign for a given coordinate type.
+
+        @param coordinateType: The coordinate type to find the hemisphere for.
+        @type coordinateType: L{Angles.LATITUDE}, L{Angles.LONGITUDE} or
+            L{Angles.VARIATION}.
+        @return: The sign of that hemisphere (-1 or 1).
+        @rtype: C{int}
+        """
+        if coordinateType is Angles.LATITUDE:
+            hemisphereKey = "latitudeHemisphere"
+        elif coordinateType is Angles.LONGITUDE:
+            hemisphereKey = "longitudeHemisphere"
+        elif coordinateType is Angles.VARIATION:
+            hemisphereKey = "magneticVariationDirection"
+        else:
+            raise ValueError(f"unknown coordinate type {coordinateType}")
+
+        hemisphere = getattr(self.currentSentence, hemisphereKey).upper()
+
+        if hemisphere in "NE":
+            return 1
+        elif hemisphere in "SW":
+            return -1
+        else:
+            raise ValueError(f"bad hemisphere/direction: {hemisphere}")
+
+    def _convert(self, key, converter):
+        """
+        A simple conversion fix.
+
+        @param key: The attribute name of the value to fix.
+        @type key: native string (Python identifier)
+
+        @param converter: The function that converts the value.
+        @type converter: unary callable
+        """
+        currentValue = getattr(self.currentSentence, key)
+        self._sentenceData[key] = converter(currentValue)
+
+    _STATEFUL_UPDATE = {
+        # sentenceKey: (stateKey, factory, attributeName, converter),
+        "trueHeading": ("heading", base.Heading, "_angle", float),
+        "magneticVariation": (
+            "heading",
+            base.Heading,
+            "variation",
+            lambda angle: base.Angle(float(angle), Angles.VARIATION),
+        ),
+        "horizontalDilutionOfPrecision": (
+            "positionError",
+            base.PositionError,
+            "hdop",
+            float,
+        ),
+        "verticalDilutionOfPrecision": (
+            "positionError",
+            base.PositionError,
+            "vdop",
+            float,
+        ),
+        "positionDilutionOfPrecision": (
+            "positionError",
+            base.PositionError,
+            "pdop",
+            float,
+        ),
+    }
+
+    def _statefulUpdate(self, sentenceKey):
+        """
+        Does a stateful update of a particular positioning attribute.
+        Specifically, this will mutate an object in the current sentence data.
+
+        Using the C{sentenceKey}, this will get a tuple containing, in order,
+        the key name in the current state and sentence data, a factory for
+        new values, the attribute to update, and a converter from sentence
+        data (in NMEA notation) to something useful.
+
+        If the sentence data doesn't have this data yet, it is grabbed from
+        the state. If that doesn't have anything useful yet either, the
+        factory is called to produce a new, empty object. Either way, the
+        object ends up in the sentence data.
+
+        @param sentenceKey: The name of the key in the sentence attributes,
+            C{NMEAAdapter._STATEFUL_UPDATE} dictionary and the adapter state.
+        @type sentenceKey: C{str}
+        """
+        key, factory, attr, converter = self._STATEFUL_UPDATE[sentenceKey]
+
+        if key not in self._sentenceData:
+            try:
+                self._sentenceData[key] = self._state[key]
+            except KeyError:  # state does not have this partial data yet
+                self._sentenceData[key] = factory()
+
+        newValue = converter(getattr(self.currentSentence, sentenceKey))
+        setattr(self._sentenceData[key], attr, newValue)
+
+    _ACCEPTABLE_UNITS = frozenset(["M"])
+    _UNIT_CONVERTERS = {
+        "N": lambda inKnots: base.Speed(float(inKnots) * base.MPS_PER_KNOT),
+        "K": lambda inKPH: base.Speed(float(inKPH) * base.MPS_PER_KPH),
+    }
+
+    def _fixUnits(self, unitKey=None, valueKey=None, sourceKey=None, unit=None):
+        """
+        Fixes the units of a certain value. If the units are already
+        acceptable (metric), does nothing.
+
+        None of the keys are allowed to be the empty string.
+
+        @param unit: The unit that is being converted I{from}. If unspecified
+            or L{None}, asks the current sentence for the C{unitKey}. If that
+            also fails, raises C{AttributeError}.
+        @type unit: C{str}
+        @param unitKey: The name of the key/attribute under which the unit can
+            be found in the current sentence. If the C{unit} parameter is set,
+            this parameter is not used.
+        @type unitKey: C{str}
+        @param sourceKey: The name of the key/attribute that contains the
+            current value to be converted (expressed in units as defined
+            according to the C{unit} parameter). If unset, will use the
+            same key as the value key.
+        @type sourceKey: C{str}
+        @param valueKey: The key name in which the data will be stored in the
+            C{_sentenceData} instance attribute. If unset, attempts to remove
+            "Units" from the end of the C{unitKey} parameter. If that fails,
+            raises C{ValueError}.
+        @type valueKey: C{str}
+        """
+        if unit is None:
+            unit = getattr(self.currentSentence, unitKey)
+        if valueKey is None:
+            if unitKey is not None and unitKey.endswith("Units"):
+                valueKey = unitKey[:-5]
+            else:
+                raise ValueError("valueKey unspecified and couldn't be guessed")
+        if sourceKey is None:
+            sourceKey = valueKey
+
+        if unit not in self._ACCEPTABLE_UNITS:
+            converter = self._UNIT_CONVERTERS[unit]
+            currentValue = getattr(self.currentSentence, sourceKey)
+            self._sentenceData[valueKey] = converter(currentValue)
+
+    def _fixGSV(self):
+        """
+        Parses partial visible satellite information from a GSV sentence.
+        """
+        # To anyone who knows NMEA, this method's name should raise a chuckle's
+        # worth of schadenfreude. 'Fix' GSV? Hah! Ludicrous.
+        beaconInformation = base.BeaconInformation()
+        self._sentenceData["_partialBeaconInformation"] = beaconInformation
+
+        keys = "satellitePRN", "azimuth", "elevation", "signalToNoiseRatio"
+        for index in range(4):
+            prn, azimuth, elevation, snr = (
+                getattr(self.currentSentence, attr)
+                for attr in ("%s_%i" % (key, index) for key in keys)
+            )
+
+            if prn is None or snr is None:
+                # The peephole optimizer optimizes the jump away, meaning that
+                # coverage.py thinks it isn't covered. It is. Replace it with
+                # break, and watch the test case fail.
+                # ML thread about this issue: http://goo.gl/1KNUi
+                # Related CPython bug: http://bugs.python.org/issue2506
+                continue
+
+            satellite = base.Satellite(prn, azimuth, elevation, snr)
+            beaconInformation.seenBeacons.add(satellite)
+
+    def _fixGSA(self):
+        """
+        Extracts the information regarding which satellites were used in
+        obtaining the GPS fix from a GSA sentence.
+
+        Precondition: A GSA sentence was fired. Postcondition: The current
+        sentence data (C{self._sentenceData} will contain a set of the
+        currently used PRNs (under the key C{_usedPRNs}.
+        """
+        self._sentenceData["_usedPRNs"] = set()
+        for key in ("usedSatellitePRN_%d" % (x,) for x in range(12)):
+            prn = getattr(self.currentSentence, key, None)
+            if prn is not None:
+                self._sentenceData["_usedPRNs"].add(int(prn))
+
+    _SPECIFIC_SENTENCE_FIXES = {
+        "GPGSV": _fixGSV,
+        "GPGSA": _fixGSA,
+    }
+
+    def _sentenceSpecificFix(self):
+        """
+        Executes a fix for a specific type of sentence.
+        """
+        fixer = self._SPECIFIC_SENTENCE_FIXES.get(self.currentSentence.type)
+        if fixer is not None:
+            fixer(self)
+
+    _FIXERS = {
+        "type": lambda self: self._sentenceSpecificFix(),
+        "timestamp": lambda self: self._fixTimestamp(),
+        "datestamp": lambda self: self._fixDatestamp(),
+        "latitudeFloat": lambda self: self._fixCoordinateFloat(Angles.LATITUDE),
+        "latitudeHemisphere": lambda self: self._fixHemisphereSign(
+            Angles.LATITUDE, "latitude"
+        ),
+        "longitudeFloat": lambda self: self._fixCoordinateFloat(Angles.LONGITUDE),
+        "longitudeHemisphere": lambda self: self._fixHemisphereSign(
+            Angles.LONGITUDE, "longitude"
+        ),
+        "altitude": lambda self: self._convert(
+            "altitude", converter=lambda strRepr: base.Altitude(float(strRepr))
+        ),
+        "altitudeUnits": lambda self: self._fixUnits(unitKey="altitudeUnits"),
+        "heightOfGeoidAboveWGS84": lambda self: self._convert(
+            "heightOfGeoidAboveWGS84",
+            converter=lambda strRepr: base.Altitude(float(strRepr)),
+        ),
+        "heightOfGeoidAboveWGS84Units": lambda self: self._fixUnits(
+            unitKey="heightOfGeoidAboveWGS84Units"
+        ),
+        "trueHeading": lambda self: self._statefulUpdate("trueHeading"),
+        "magneticVariation": lambda self: self._statefulUpdate("magneticVariation"),
+        "magneticVariationDirection": lambda self: self._fixHemisphereSign(
+            Angles.VARIATION, "heading"
+        ),
+        "speedInKnots": lambda self: self._fixUnits(
+            valueKey="speed", sourceKey="speedInKnots", unit="N"
+        ),
+        "positionDilutionOfPrecision": lambda self: self._statefulUpdate(
+            "positionDilutionOfPrecision"
+        ),
+        "horizontalDilutionOfPrecision": lambda self: self._statefulUpdate(
+            "horizontalDilutionOfPrecision"
+        ),
+        "verticalDilutionOfPrecision": lambda self: self._statefulUpdate(
+            "verticalDilutionOfPrecision"
+        ),
+    }
+
+    def clear(self):
+        """
+        Resets this adapter.
+
+        This will empty the adapter state and the current sentence data.
+        """
+        self._state = {}
+        self._sentenceData = {}
+
+    def sentenceReceived(self, sentence):
+        """
+        Called when a sentence is received.
+
+        Will clean the received NMEAProtocol sentence up, and then update the
+        adapter's state, followed by firing the callbacks.
+
+        If the received sentence was invalid, the state will be cleared.
+
+        @param sentence: The sentence that is received.
+        @type sentence: L{NMEASentence}
+        """
+        self.currentSentence = sentence
+        self._sentenceData = {}
+
+        try:
+            self._validateCurrentSentence()
+            self._cleanCurrentSentence()
+        except base.InvalidSentence:
+            self.clear()
+
+        self._updateState()
+        self._fireSentenceCallbacks()
+
+    def _validateCurrentSentence(self):
+        """
+        Tests if a sentence contains a valid fix.
+        """
+        if (
+            self.currentSentence.fixQuality is GPGGAFixQualities.INVALID_FIX
+            or self.currentSentence.dataMode is GPGLLGPRMCFixQualities.VOID
+            or self.currentSentence.fixType is GPGSAFixTypes.GSA_NO_FIX
+        ):
+            raise base.InvalidSentence("bad sentence")
+
+    def _cleanCurrentSentence(self):
+        """
+        Cleans the current sentence.
+        """
+        for key in sorted(self.currentSentence.presentAttributes):
+            fixer = self._FIXERS.get(key, None)
+
+            if fixer is not None:
+                fixer(self)
+
+    def _updateState(self):
+        """
+        Updates the current state with the new information from the sentence.
+        """
+        self._updateBeaconInformation()
+        self._combineDateAndTime()
+        self._state.update(self._sentenceData)
+
+    def _updateBeaconInformation(self):
+        """
+        Updates existing beacon information state with new data.
+        """
+        new = self._sentenceData.get("_partialBeaconInformation")
+        if new is None:
+            return
+
+        self._updateUsedBeacons(new)
+        self._mergeBeaconInformation(new)
+
+        if self.currentSentence._isLastGSVSentence():
+            if not self.currentSentence._isFirstGSVSentence():
+                # not a 1-sentence sequence, get rid of partial information
+                del self._state["_partialBeaconInformation"]
+            bi = self._sentenceData.pop("_partialBeaconInformation")
+            self._sentenceData["beaconInformation"] = bi
+
+    def _updateUsedBeacons(self, beaconInformation):
+        """
+        Searches the adapter state and sentence data for information about
+        which beacons where used, then adds it to the provided beacon
+        information object.
+
+        If no new beacon usage information is available, does nothing.
+
+        @param beaconInformation: The beacon information object that beacon
+            usage information will be added to (if necessary).
+        @type beaconInformation: L{twisted.positioning.base.BeaconInformation}
+        """
+        for source in [self._state, self._sentenceData]:
+            usedPRNs = source.get("_usedPRNs")
+            if usedPRNs is not None:
+                break
+        else:  # No used PRN info to update
+            return
+
+        for beacon in beaconInformation.seenBeacons:
+            if beacon.identifier in usedPRNs:
+                beaconInformation.usedBeacons.add(beacon)
+
+    def _mergeBeaconInformation(self, newBeaconInformation):
+        """
+        Merges beacon information in the adapter state (if it exists) into
+        the provided beacon information. Specifically, this merges used and
+        seen beacons.
+
+        If the adapter state has no beacon information, does nothing.
+
+        @param newBeaconInformation: The beacon information object that beacon
+            information will be merged into (if necessary).
+        @type newBeaconInformation: L{twisted.positioning.base.BeaconInformation}
+        """
+        old = self._state.get("_partialBeaconInformation")
+        if old is None:
+            return
+
+        for attr in ["seenBeacons", "usedBeacons"]:
+            getattr(newBeaconInformation, attr).update(getattr(old, attr))
+
+    def _combineDateAndTime(self):
+        """
+        Combines a C{datetime.date} object and a C{datetime.time} object,
+        collected from one or more NMEA sentences, into a single
+        C{datetime.datetime} object suitable for sending to the
+        L{IPositioningReceiver}.
+        """
+        if not any(k in self._sentenceData for k in ["_date", "_time"]):
+            # If the sentence has neither date nor time, there's
+            # nothing new to combine here.
+            return
+
+        date, time = (
+            self._sentenceData.get(key) or self._state.get(key)
+            for key in ("_date", "_time")
+        )
+
+        if date is None or time is None:
+            return
+
+        dt = datetime.datetime.combine(date, time)
+        self._sentenceData["time"] = dt
+
+    def _fireSentenceCallbacks(self):
+        """
+        Fires sentence callbacks for the current sentence.
+
+        A callback will only fire if all of the keys it requires are present
+        in the current state and at least one such field was altered in the
+        current sentence.
+
+        The callbacks will only be fired with data from L{_state}.
+        """
+        iface = ipositioning.IPositioningReceiver
+        for name, method in iface.namesAndDescriptions():
+            callback = getattr(self._receiver, name)
+
+            kwargs = {}
+            atLeastOnePresentInSentence = False
+
+            try:
+                for field in method.positional:
+                    if field in self._sentenceData:
+                        atLeastOnePresentInSentence = True
+                    kwargs[field] = self._state[field]
+            except KeyError:
+                continue
+
+            if atLeastOnePresentInSentence:
+                callback(**kwargs)
+
+
+__all__ = ["NMEAProtocol", "NMEASentence", "NMEAAdapter"]
diff --git a/contrib/python/Twisted/py3/twisted/protocols/__init__.py b/contrib/python/Twisted/py3/twisted/protocols/__init__.py
new file mode 100644
index 00000000000..d5286f79592
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Protocols: A collection of internet protocol implementations.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/protocols/amp.py b/contrib/python/Twisted/py3/twisted/protocols/amp.py
new file mode 100644
index 00000000000..a8c35754a99
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/amp.py
@@ -0,0 +1,2833 @@
+# -*- test-case-name: twisted.test.test_amp -*-
+# Copyright (c) 2005 Divmod, Inc.
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module implements AMP, the Asynchronous Messaging Protocol.
+
+AMP is a protocol for sending multiple asynchronous request/response pairs over
+the same connection.  Requests and responses are both collections of key/value
+pairs.
+
+AMP is a very simple protocol which is not an application.  This module is a
+"protocol construction kit" of sorts; it attempts to be the simplest wire-level
+implementation of Deferreds.  AMP provides the following base-level features:
+
+    - Asynchronous request/response handling (hence the name)
+
+    - Requests and responses are both key/value pairs
+
+    - Binary transfer of all data: all data is length-prefixed.  Your
+      application will never need to worry about quoting.
+
+    - Command dispatching (like HTTP Verbs): the protocol is extensible, and
+      multiple AMP sub-protocols can be grouped together easily.
+
+The protocol implementation also provides a few additional features which are
+not part of the core wire protocol, but are nevertheless very useful:
+
+    - Tight TLS integration, with an included StartTLS command.
+
+    - Handshaking to other protocols: because AMP has well-defined message
+      boundaries and maintains all incoming and outgoing requests for you, you
+      can start a connection over AMP and then switch to another protocol.
+      This makes it ideal for firewall-traversal applications where you may
+      have only one forwarded port but multiple applications that want to use
+      it.
+
+Using AMP with Twisted is simple.  Each message is a command, with a response.
+You begin by defining a command type.  Commands specify their input and output
+in terms of the types that they expect to see in the request and response
+key-value pairs.  Here's an example of a command that adds two integers, 'a'
+and 'b'::
+
+    class Sum(amp.Command):
+        arguments = [('a', amp.Integer()),
+                     ('b', amp.Integer())]
+        response = [('total', amp.Integer())]
+
+Once you have specified a command, you need to make it part of a protocol, and
+define a responder for it.  Here's a 'JustSum' protocol that includes a
+responder for our 'Sum' command::
+
+    class JustSum(amp.AMP):
+        def sum(self, a, b):
+            total = a + b
+            print 'Did a sum: %d + %d = %d' % (a, b, total)
+            return {'total': total}
+        Sum.responder(sum)
+
+Later, when you want to actually do a sum, the following expression will return
+a L{Deferred} which will fire with the result::
+
+    ClientCreator(reactor, amp.AMP).connectTCP(...).addCallback(
+        lambda p: p.callRemote(Sum, a=13, b=81)).addCallback(
+            lambda result: result['total'])
+
+Command responders may also return Deferreds, causing the response to be
+sent only once the Deferred fires::
+
+    class DelayedSum(amp.AMP):
+        def slowSum(self, a, b):
+            total = a + b
+            result = defer.Deferred()
+            reactor.callLater(3, result.callback, {'total': total})
+            return result
+        Sum.responder(slowSum)
+
+This is transparent to the caller.
+
+You can also define the propagation of specific errors in AMP.  For example,
+for the slightly more complicated case of division, we might have to deal with
+division by zero::
+
+    class Divide(amp.Command):
+        arguments = [('numerator', amp.Integer()),
+                     ('denominator', amp.Integer())]
+        response = [('result', amp.Float())]
+        errors = {ZeroDivisionError: 'ZERO_DIVISION'}
+
+The 'errors' mapping here tells AMP that if a responder to Divide emits a
+L{ZeroDivisionError}, then the other side should be informed that an error of
+the type 'ZERO_DIVISION' has occurred.  Writing a responder which takes
+advantage of this is very simple - just raise your exception normally::
+
+    class JustDivide(amp.AMP):
+        def divide(self, numerator, denominator):
+            result = numerator / denominator
+            print 'Divided: %d / %d = %d' % (numerator, denominator, total)
+            return {'result': result}
+        Divide.responder(divide)
+
+On the client side, the errors mapping will be used to determine what the
+'ZERO_DIVISION' error means, and translated into an asynchronous exception,
+which can be handled normally as any L{Deferred} would be::
+
+    def trapZero(result):
+        result.trap(ZeroDivisionError)
+        print "Divided by zero: returning INF"
+        return 1e1000
+    ClientCreator(reactor, amp.AMP).connectTCP(...).addCallback(
+        lambda p: p.callRemote(Divide, numerator=1234,
+                               denominator=0)
+        ).addErrback(trapZero)
+
+For a complete, runnable example of both of these commands, see the files in
+the Twisted repository::
+
+    doc/core/examples/ampserver.py
+    doc/core/examples/ampclient.py
+
+On the wire, AMP is a protocol which uses 2-byte lengths to prefix keys and
+values, and empty keys to separate messages::
+
+    <2-byte length><key><2-byte length><value>
+    <2-byte length><key><2-byte length><value>
+    ...
+    <2-byte length><key><2-byte length><value>
+    <NUL><NUL>                  # Empty Key == End of Message
+
+And so on.  Because it's tedious to refer to lengths and NULs constantly, the
+documentation will refer to packets as if they were newline delimited, like
+so::
+
+    C: _command: sum
+    C: _ask: ef639e5c892ccb54
+    C: a: 13
+    C: b: 81
+
+    S: _answer: ef639e5c892ccb54
+    S: total: 94
+
+Notes:
+
+In general, the order of keys is arbitrary.  Specific uses of AMP may impose an
+ordering requirement, but unless this is specified explicitly, any ordering may
+be generated and any ordering must be accepted.  This applies to the
+command-related keys I{_command} and I{_ask} as well as any other keys.
+
+Values are limited to the maximum encodable size in a 16-bit length, 65535
+bytes.
+
+Keys are limited to the maximum encodable size in a 8-bit length, 255 bytes.
+Note that we still use 2-byte lengths to encode keys.  This small redundancy
+has several features:
+
+    - If an implementation becomes confused and starts emitting corrupt data,
+      or gets keys confused with values, many common errors will be signalled
+      immediately instead of delivering obviously corrupt packets.
+
+    - A single NUL will separate every key, and a double NUL separates
+      messages.  This provides some redundancy when debugging traffic dumps.
+
+    - NULs will be present at regular intervals along the protocol, providing
+      some padding for otherwise braindead C implementations of the protocol,
+      so that <stdio.h> string functions will see the NUL and stop.
+
+    - This makes it possible to run an AMP server on a port also used by a
+      plain-text protocol, and easily distinguish between non-AMP clients (like
+      web browsers) which issue non-NUL as the first byte, and AMP clients,
+      which always issue NUL as the first byte.
+
+@var MAX_VALUE_LENGTH: The maximum length of a message.
+@type MAX_VALUE_LENGTH: L{int}
+
+@var ASK: Marker for an Ask packet.
+@type ASK: L{bytes}
+
+@var ANSWER: Marker for an Answer packet.
+@type ANSWER: L{bytes}
+
+@var COMMAND: Marker for a Command packet.
+@type COMMAND: L{bytes}
+
+@var ERROR: Marker for an AMP box of error type.
+@type ERROR: L{bytes}
+
+@var ERROR_CODE: Marker for an AMP box containing the code of an error.
+@type ERROR_CODE: L{bytes}
+
+@var ERROR_DESCRIPTION: Marker for an AMP box containing the description of the
+    error.
+@type ERROR_DESCRIPTION: L{bytes}
+"""
+from __future__ import annotations
+
+import datetime
+import decimal
+import warnings
+from functools import partial
+from io import BytesIO
+from itertools import count
+from struct import pack
+from types import MethodType
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
+
+from zope.interface import Interface, implementer
+
+from twisted.internet.defer import Deferred, fail, maybeDeferred
+from twisted.internet.error import ConnectionClosed, ConnectionLost, PeerVerifyError
+from twisted.internet.interfaces import IFileDescriptorReceiver
+from twisted.internet.main import CONNECTION_LOST
+from twisted.internet.protocol import Protocol
+from twisted.protocols.basic import Int16StringReceiver, StatefulStringProtocol
+from twisted.python import filepath, log
+from twisted.python._tzhelper import (
+    UTC as utc,
+    FixedOffsetTimeZone as _FixedOffsetTZInfo,
+)
+from twisted.python.compat import nativeString
+from twisted.python.failure import Failure
+from twisted.python.reflect import accumulateClassDict
+
+try:
+    from twisted.internet import ssl as _ssl
+
+    if _ssl.supported:
+        from twisted.internet.ssl import DN, Certificate, CertificateOptions, KeyPair
+    else:
+        ssl = None
+except ImportError:
+    ssl = None
+else:
+    ssl = _ssl
+
+
+__all__ = [
+    "AMP",
+    "ANSWER",
+    "ASK",
+    "AmpBox",
+    "AmpError",
+    "AmpList",
+    "Argument",
+    "BadLocalReturn",
+    "BinaryBoxProtocol",
+    "Boolean",
+    "Box",
+    "BoxDispatcher",
+    "COMMAND",
+    "Command",
+    "CommandLocator",
+    "Decimal",
+    "Descriptor",
+    "ERROR",
+    "ERROR_CODE",
+    "ERROR_DESCRIPTION",
+    "Float",
+    "IArgumentType",
+    "IBoxReceiver",
+    "IBoxSender",
+    "IResponderLocator",
+    "IncompatibleVersions",
+    "Integer",
+    "InvalidSignature",
+    "ListOf",
+    "MAX_KEY_LENGTH",
+    "MAX_VALUE_LENGTH",
+    "MalformedAmpBox",
+    "NoEmptyBoxes",
+    "OnlyOneTLS",
+    "PROTOCOL_ERRORS",
+    "PYTHON_KEYWORDS",
+    "Path",
+    "ProtocolSwitchCommand",
+    "ProtocolSwitched",
+    "QuitBox",
+    "RemoteAmpError",
+    "SimpleStringLocator",
+    "StartTLS",
+    "String",
+    "TooLong",
+    "UNHANDLED_ERROR_CODE",
+    "UNKNOWN_ERROR_CODE",
+    "UnhandledCommand",
+    "utc",
+    "Unicode",
+    "UnknownRemoteError",
+    "parse",
+    "parseString",
+]
+
+
+_T_Callable = TypeVar("_T_Callable", bound=Callable[..., object])
+
+
+ASK = b"_ask"
+ANSWER = b"_answer"
+COMMAND = b"_command"
+ERROR = b"_error"
+ERROR_CODE = b"_error_code"
+ERROR_DESCRIPTION = b"_error_description"
+UNKNOWN_ERROR_CODE = b"UNKNOWN"
+UNHANDLED_ERROR_CODE = b"UNHANDLED"
+
+MAX_KEY_LENGTH = 0xFF
+MAX_VALUE_LENGTH = 0xFFFF
+
+
+class IArgumentType(Interface):
+    """
+    An L{IArgumentType} can serialize a Python object into an AMP box and
+    deserialize information from an AMP box back into a Python object.
+
+    @since: 9.0
+    """
+
+    def fromBox(name, strings, objects, proto):
+        """
+        Given an argument name and an AMP box containing serialized values,
+        extract one or more Python objects and add them to the C{objects}
+        dictionary.
+
+        @param name: The name associated with this argument. Most commonly
+            this is the key which can be used to find a serialized value in
+            C{strings}.
+        @type name: C{bytes}
+
+        @param strings: The AMP box from which to extract one or more
+            values.
+        @type strings: C{dict}
+
+        @param objects: The output dictionary to populate with the value for
+            this argument. The key used will be derived from C{name}. It may
+            differ; in Python 3, for example, the key will be a Unicode/native
+            string. See L{_wireNameToPythonIdentifier}.
+        @type objects: C{dict}
+
+        @param proto: The protocol instance which received the AMP box being
+            interpreted.  Most likely this is an instance of L{AMP}, but
+            this is not guaranteed.
+
+        @return: L{None}
+        """
+
+    def toBox(name, strings, objects, proto):
+        """
+        Given an argument name and a dictionary containing structured Python
+        objects, serialize values into one or more strings and add them to
+        the C{strings} dictionary.
+
+        @param name: The name associated with this argument. Most commonly
+            this is the key in C{strings} to associate with a C{bytes} giving
+            the serialized form of that object.
+        @type name: C{bytes}
+
+        @param strings: The AMP box into which to insert one or more strings.
+        @type strings: C{dict}
+
+        @param objects: The input dictionary from which to extract Python
+            objects to serialize. The key used will be derived from C{name}.
+            It may differ; in Python 3, for example, the key will be a
+            Unicode/native string. See L{_wireNameToPythonIdentifier}.
+        @type objects: C{dict}
+
+        @param proto: The protocol instance which will send the AMP box once
+            it is fully populated.  Most likely this is an instance of
+            L{AMP}, but this is not guaranteed.
+
+        @return: L{None}
+        """
+
+
+class IBoxSender(Interface):
+    """
+    A transport which can send L{AmpBox} objects.
+    """
+
+    def sendBox(box):
+        """
+        Send an L{AmpBox}.
+
+        @raise ProtocolSwitched: if the underlying protocol has been
+        switched.
+
+        @raise ConnectionLost: if the underlying connection has already been
+        lost.
+        """
+
+    def unhandledError(failure):
+        """
+        An unhandled error occurred in response to a box.  Log it
+        appropriately.
+
+        @param failure: a L{Failure} describing the error that occurred.
+        """
+
+
+class IBoxReceiver(Interface):
+    """
+    An application object which can receive L{AmpBox} objects and dispatch them
+    appropriately.
+    """
+
+    def startReceivingBoxes(boxSender):
+        """
+        The L{IBoxReceiver.ampBoxReceived} method will start being called;
+        boxes may be responded to by responding to the given L{IBoxSender}.
+
+        @param boxSender: an L{IBoxSender} provider.
+        """
+
+    def ampBoxReceived(box):
+        """
+        A box was received from the transport; dispatch it appropriately.
+        """
+
+    def stopReceivingBoxes(reason):
+        """
+        No further boxes will be received on this connection.
+
+        @type reason: L{Failure}
+        """
+
+
+class IResponderLocator(Interface):
+    """
+    An application object which can look up appropriate responder methods for
+    AMP commands.
+    """
+
+    def locateResponder(name):
+        """
+        Locate a responder method appropriate for the named command.
+
+        @param name: the wire-level name (commandName) of the AMP command to be
+        responded to.
+        @type name: C{bytes}
+
+        @return: a 1-argument callable that takes an L{AmpBox} with argument
+        values for the given command, and returns an L{AmpBox} containing
+        argument values for the named command, or a L{Deferred} that fires the
+        same.
+        """
+
+
+class AmpError(Exception):
+    """
+    Base class of all Amp-related exceptions.
+    """
+
+
+class ProtocolSwitched(Exception):
+    """
+    Connections which have been switched to other protocols can no longer
+    accept traffic at the AMP level.  This is raised when you try to send it.
+    """
+
+
+class OnlyOneTLS(AmpError):
+    """
+    This is an implementation limitation; TLS may only be started once per
+    connection.
+    """
+
+
+class NoEmptyBoxes(AmpError):
+    """
+    You can't have empty boxes on the connection.  This is raised when you
+    receive or attempt to send one.
+    """
+
+
+class InvalidSignature(AmpError):
+    """
+    You didn't pass all the required arguments.
+    """
+
+
+class TooLong(AmpError):
+    """
+    One of the protocol's length limitations was violated.
+
+    @ivar isKey: true if the string being encoded in a key position, false if
+    it was in a value position.
+
+    @ivar isLocal: Was the string encoded locally, or received too long from
+    the network?  (It's only physically possible to encode "too long" values on
+    the network for keys.)
+
+    @ivar value: The string that was too long.
+
+    @ivar keyName: If the string being encoded was in a value position, what
+    key was it being encoded for?
+    """
+
+    def __init__(self, isKey, isLocal, value, keyName=None):
+        AmpError.__init__(self)
+        self.isKey = isKey
+        self.isLocal = isLocal
+        self.value = value
+        self.keyName = keyName
+
+    def __repr__(self) -> str:
+        hdr = self.isKey and "key" or "value"
+        if not self.isKey:
+            hdr += " " + repr(self.keyName)
+        lcl = self.isLocal and "local" or "remote"
+        return "%s %s too long: %d" % (lcl, hdr, len(self.value))
+
+
+class BadLocalReturn(AmpError):
+    """
+    A bad value was returned from a local command; we were unable to coerce it.
+    """
+
+    def __init__(self, message: str, enclosed: Failure) -> None:
+        AmpError.__init__(self)
+        self.message = message
+        self.enclosed = enclosed
+
+    def __repr__(self) -> str:
+        return self.message + " " + self.enclosed.getBriefTraceback()
+
+    __str__ = __repr__
+
+
+class RemoteAmpError(AmpError):
+    """
+    This error indicates that something went wrong on the remote end of the
+    connection, and the error was serialized and transmitted to you.
+    """
+
+    def __init__(self, errorCode, description, fatal=False, local=None):
+        """Create a remote error with an error code and description.
+
+        @param errorCode: the AMP error code of this error.
+        @type errorCode: C{bytes}
+
+        @param description: some text to show to the user.
+        @type description: C{str}
+
+        @param fatal: a boolean, true if this error should terminate the
+        connection.
+
+        @param local: a local Failure, if one exists.
+        """
+        if local:
+            localwhat = " (local)"
+            othertb = local.getBriefTraceback()
+        else:
+            localwhat = ""
+            othertb = ""
+
+        # Backslash-escape errorCode. Python 3.5 can do this natively
+        # ("backslashescape") but Python 2.7 and Python 3.4 can't.
+        errorCodeForMessage = "".join(
+            f"\\x{c:2x}" if c >= 0x80 else chr(c) for c in errorCode
+        )
+
+        if othertb:
+            message = "Code<{}>{}: {}\n{}".format(
+                errorCodeForMessage,
+                localwhat,
+                description,
+                othertb,
+            )
+        else:
+            message = "Code<{}>{}: {}".format(
+                errorCodeForMessage, localwhat, description
+            )
+
+        super().__init__(message)
+        self.local = local
+        self.errorCode = errorCode
+        self.description = description
+        self.fatal = fatal
+
+
+class UnknownRemoteError(RemoteAmpError):
+    """
+    This means that an error whose type we can't identify was raised from the
+    other side.
+    """
+
+    def __init__(self, description):
+        errorCode = UNKNOWN_ERROR_CODE
+        RemoteAmpError.__init__(self, errorCode, description)
+
+
+class MalformedAmpBox(AmpError):
+    """
+    This error indicates that the wire-level protocol was malformed.
+    """
+
+
+class UnhandledCommand(AmpError):
+    """
+    A command received via amp could not be dispatched.
+    """
+
+
+class IncompatibleVersions(AmpError):
+    """
+    It was impossible to negotiate a compatible version of the protocol with
+    the other end of the connection.
+    """
+
+
+PROTOCOL_ERRORS = {UNHANDLED_ERROR_CODE: UnhandledCommand}
+
+
+class AmpBox(Dict[bytes, bytes]):
+    """
+    I am a packet in the AMP protocol, much like a
+    regular bytes:bytes dictionary.
+    """
+
+    # be like a regular dictionary don't magically
+    # acquire a __dict__...
+    __slots__: List[str] = []
+
+    def __init__(self, *args, **kw):
+        """
+        Initialize a new L{AmpBox}.
+
+        In Python 3, keyword arguments MUST be Unicode/native strings whereas
+        in Python 2 they could be either byte strings or Unicode strings.
+
+        However, all keys of an L{AmpBox} MUST be byte strings, or possible to
+        transparently coerce into byte strings (i.e. Python 2).
+
+        In Python 3, therefore, native string keys are coerced to byte strings
+        by encoding as ASCII. This can result in C{UnicodeEncodeError} being
+        raised.
+
+        @param args: See C{dict}, but all keys and values should be C{bytes}.
+            On Python 3, native strings may be used as keys provided they
+            contain only ASCII characters.
+
+        @param kw: See C{dict}, but all keys and values should be C{bytes}.
+            On Python 3, native strings may be used as keys provided they
+            contain only ASCII characters.
+
+        @raise UnicodeEncodeError: When a native string key cannot be coerced
+            to an ASCII byte string (Python 3 only).
+        """
+        super().__init__(*args, **kw)
+        nonByteNames = [n for n in self if not isinstance(n, bytes)]
+        for nonByteName in nonByteNames:
+            byteName = nonByteName.encode("ascii")
+            self[byteName] = self.pop(nonByteName)
+
+    def copy(self):
+        """
+        Return another AmpBox just like me.
+        """
+        newBox = self.__class__()
+        newBox.update(self)
+        return newBox
+
+    def serialize(self):
+        """
+        Convert me into a wire-encoded string.
+
+        @return: a C{bytes} encoded according to the rules described in the
+            module docstring.
+        """
+        i = sorted(self.items())
+        L = []
+        w = L.append
+        for k, v in i:
+            if type(k) == str:
+                raise TypeError("Unicode key not allowed: %r" % k)
+            if type(v) == str:
+                raise TypeError(f"Unicode value for key {k!r} not allowed: {v!r}")
+            if len(k) > MAX_KEY_LENGTH:
+                raise TooLong(True, True, k, None)
+            if len(v) > MAX_VALUE_LENGTH:
+                raise TooLong(False, True, v, k)
+            for kv in k, v:
+                w(pack("!H", len(kv)))
+                w(kv)
+        w(pack("!H", 0))
+        return b"".join(L)
+
+    def _sendTo(self, proto):
+        """
+        Serialize and send this box to an Amp instance.  By the time it is being
+        sent, several keys are required.  I must have exactly ONE of::
+
+            _ask
+            _answer
+            _error
+
+        If the '_ask' key is set, then the '_command' key must also be
+        set.
+
+        @param proto: an AMP instance.
+        """
+        proto.sendBox(self)
+
+    def __repr__(self) -> str:
+        return f"AmpBox({dict.__repr__(self)})"
+
+
+# amp.Box => AmpBox
+
+Box = AmpBox
+
+
+class QuitBox(AmpBox):
+    """
+    I am an AmpBox that, upon being sent, terminates the connection.
+    """
+
+    __slots__: List[str] = []
+
+    def __repr__(self) -> str:
+        return f"QuitBox(**{super().__repr__()})"
+
+    def _sendTo(self, proto):
+        """
+        Immediately call loseConnection after sending.
+        """
+        super()._sendTo(proto)
+        proto.transport.loseConnection()
+
+
+class _SwitchBox(AmpBox):
+    """
+    Implementation detail of ProtocolSwitchCommand: I am an AmpBox which sets
+    up state for the protocol to switch.
+    """
+
+    # DON'T set __slots__ here; we do have an attribute.
+
+    def __init__(self, innerProto, **kw):
+        """
+        Create a _SwitchBox with the protocol to switch to after being sent.
+
+        @param innerProto: the protocol instance to switch to.
+        @type innerProto: an IProtocol provider.
+        """
+        super().__init__(**kw)
+        self.innerProto = innerProto
+
+    def __repr__(self) -> str:
+        return "_SwitchBox({!r}, **{})".format(
+            self.innerProto,
+            dict.__repr__(self),
+        )
+
+    def _sendTo(self, proto):
+        """
+        Send me; I am the last box on the connection.  All further traffic will be
+        over the new protocol.
+        """
+        super()._sendTo(proto)
+        proto._lockForSwitch()
+        proto._switchTo(self.innerProto)
+
+
+@implementer(IBoxReceiver)
+class BoxDispatcher:
+    """
+    A L{BoxDispatcher} dispatches '_ask', '_answer', and '_error' L{AmpBox}es,
+    both incoming and outgoing, to their appropriate destinations.
+
+    Outgoing commands are converted into L{Deferred}s and outgoing boxes, and
+    associated tracking state to fire those L{Deferred} when '_answer' boxes
+    come back.  Incoming '_answer' and '_error' boxes are converted into
+    callbacks and errbacks on those L{Deferred}s, respectively.
+
+    Incoming '_ask' boxes are converted into method calls on a supplied method
+    locator.
+
+    @ivar _outstandingRequests: a dictionary mapping request IDs to
+    L{Deferred}s which were returned for those requests.
+
+    @ivar locator: an object with a L{CommandLocator.locateResponder} method
+        that locates a responder function that takes a Box and returns a result
+        (either a Box or a Deferred which fires one).
+
+    @ivar boxSender: an object which can send boxes, via the L{_sendBoxCommand}
+    method, such as an L{AMP} instance.
+    @type boxSender: L{IBoxSender}
+    """
+
+    _failAllReason = None
+    _outstandingRequests = None
+    _counter = 0
+    boxSender = None
+
+    def __init__(self, locator):
+        self._outstandingRequests = {}
+        self.locator = locator
+
+    def startReceivingBoxes(self, boxSender):
+        """
+        The given boxSender is going to start calling boxReceived on this
+        L{BoxDispatcher}.
+
+        @param boxSender: The L{IBoxSender} to send command responses to.
+        """
+        self.boxSender = boxSender
+
+    def stopReceivingBoxes(self, reason):
+        """
+        No further boxes will be received here.  Terminate all currently
+        outstanding command deferreds with the given reason.
+        """
+        self.failAllOutgoing(reason)
+
+    def failAllOutgoing(self, reason):
+        """
+        Call the errback on all outstanding requests awaiting responses.
+
+        @param reason: the Failure instance to pass to those errbacks.
+        """
+        self._failAllReason = reason
+        OR = self._outstandingRequests.items()
+        self._outstandingRequests = None  # we can never send another request
+        for key, value in OR:
+            value.errback(reason)
+
+    def _nextTag(self):
+        """
+        Generate protocol-local serial numbers for _ask keys.
+
+        @return: a string that has not yet been used on this connection.
+        """
+        self._counter += 1
+        return b"%x" % (self._counter,)
+
+    def _sendBoxCommand(self, command, box, requiresAnswer=True):
+        """
+        Send a command across the wire with the given C{amp.Box}.
+
+        Mutate the given box to give it any additional keys (_command, _ask)
+        required for the command and request/response machinery, then send it.
+
+        If requiresAnswer is True, returns a C{Deferred} which fires when a
+        response is received. The C{Deferred} is fired with an C{amp.Box} on
+        success, or with an C{amp.RemoteAmpError} if an error is received.
+
+        If the Deferred fails and the error is not handled by the caller of
+        this method, the failure will be logged and the connection dropped.
+
+        @param command: a C{bytes}, the name of the command to issue.
+
+        @param box: an AmpBox with the arguments for the command.
+
+        @param requiresAnswer: a boolean.  Defaults to True.  If True, return a
+        Deferred which will fire when the other side responds to this command.
+        If False, return None and do not ask the other side for acknowledgement.
+
+        @return: a Deferred which fires the AmpBox that holds the response to
+        this command, or None, as specified by requiresAnswer.
+
+        @raise ProtocolSwitched: if the protocol has been switched.
+        """
+        if self._failAllReason is not None:
+            if requiresAnswer:
+                return fail(self._failAllReason)
+            else:
+                return None
+        box[COMMAND] = command
+        tag = self._nextTag()
+        if requiresAnswer:
+            box[ASK] = tag
+        box._sendTo(self.boxSender)
+        if requiresAnswer:
+            result = self._outstandingRequests[tag] = Deferred()
+        else:
+            result = None
+        return result
+
+    def callRemoteString(self, command, requiresAnswer=True, **kw):
+        """
+        This is a low-level API, designed only for optimizing simple messages
+        for which the overhead of parsing is too great.
+
+        @param command: a C{bytes} naming the command.
+
+        @param kw: arguments to the amp box.
+
+        @param requiresAnswer: a boolean.  Defaults to True.  If True, return a
+        Deferred which will fire when the other side responds to this command.
+        If False, return None and do not ask the other side for acknowledgement.
+
+        @return: a Deferred which fires the AmpBox that holds the response to
+        this command, or None, as specified by requiresAnswer.
+        """
+        box = Box(kw)
+        return self._sendBoxCommand(command, box, requiresAnswer)
+
+    def callRemote(self, commandType, *a, **kw):
+        """
+        This is the primary high-level API for sending messages via AMP.  Invoke it
+        with a command and appropriate arguments to send a message to this
+        connection's peer.
+
+        @param commandType: a subclass of Command.
+        @type commandType: L{type}
+
+        @param a: Positional (special) parameters taken by the command.
+        Positional parameters will typically not be sent over the wire.  The
+        only command included with AMP which uses positional parameters is
+        L{ProtocolSwitchCommand}, which takes the protocol that will be
+        switched to as its first argument.
+
+        @param kw: Keyword arguments taken by the command.  These are the
+        arguments declared in the command's 'arguments' attribute.  They will
+        be encoded and sent to the peer as arguments for the L{commandType}.
+
+        @return: If L{commandType} has a C{requiresAnswer} attribute set to
+        L{False}, then return L{None}.  Otherwise, return a L{Deferred} which
+        fires with a dictionary of objects representing the result of this
+        call.  Additionally, this L{Deferred} may fail with an exception
+        representing a connection failure, with L{UnknownRemoteError} if the
+        other end of the connection fails for an unknown reason, or with any
+        error specified as a key in L{commandType}'s C{errors} dictionary.
+        """
+
+        # XXX this takes command subclasses and not command objects on purpose.
+        # There's really no reason to have all this back-and-forth between
+        # command objects and the protocol, and the extra object being created
+        # (the Command instance) is pointless.  Command is kind of like
+        # Interface, and should be more like it.
+
+        # In other words, the fact that commandType is instantiated here is an
+        # implementation detail.  Don't rely on it.
+
+        try:
+            co = commandType(*a, **kw)
+        except BaseException:
+            return fail()
+        return co._doCommand(self)
+
+    def unhandledError(self, failure):
+        """
+        This is a terminal callback called after application code has had a
+        chance to quash any errors.
+        """
+        return self.boxSender.unhandledError(failure)
+
+    def _answerReceived(self, box):
+        """
+        An AMP box was received that answered a command previously sent with
+        L{callRemote}.
+
+        @param box: an AmpBox with a value for its L{ANSWER} key.
+        """
+        question = self._outstandingRequests.pop(box[ANSWER])
+        question.addErrback(self.unhandledError)
+        question.callback(box)
+
+    def _errorReceived(self, box):
+        """
+        An AMP box was received that answered a command previously sent with
+        L{callRemote}, with an error.
+
+        @param box: an L{AmpBox} with a value for its L{ERROR}, L{ERROR_CODE},
+        and L{ERROR_DESCRIPTION} keys.
+        """
+        question = self._outstandingRequests.pop(box[ERROR])
+        question.addErrback(self.unhandledError)
+        errorCode = box[ERROR_CODE]
+        description = box[ERROR_DESCRIPTION]
+        if isinstance(description, bytes):
+            description = description.decode("utf-8", "replace")
+        if errorCode in PROTOCOL_ERRORS:
+            exc = PROTOCOL_ERRORS[errorCode](errorCode, description)
+        else:
+            exc = RemoteAmpError(errorCode, description)
+        question.errback(Failure(exc))
+
+    def _commandReceived(self, box):
+        """
+        @param box: an L{AmpBox} with a value for its L{COMMAND} and L{ASK}
+        keys.
+        """
+
+        def formatAnswer(answerBox):
+            answerBox[ANSWER] = box[ASK]
+            return answerBox
+
+        def formatError(error):
+            if error.check(RemoteAmpError):
+                code = error.value.errorCode
+                desc = error.value.description
+                if isinstance(desc, str):
+                    desc = desc.encode("utf-8", "replace")
+                if error.value.fatal:
+                    errorBox = QuitBox()
+                else:
+                    errorBox = AmpBox()
+            else:
+                errorBox = QuitBox()
+                log.err(error)  # here is where server-side logging happens
+                # if the error isn't handled
+                code = UNKNOWN_ERROR_CODE
+                desc = b"Unknown Error"
+            errorBox[ERROR] = box[ASK]
+            errorBox[ERROR_DESCRIPTION] = desc
+            errorBox[ERROR_CODE] = code
+            return errorBox
+
+        deferred = self.dispatchCommand(box)
+        if ASK in box:
+            deferred.addCallbacks(formatAnswer, formatError)
+            deferred.addCallback(self._safeEmit)
+        deferred.addErrback(self.unhandledError)
+
+    def ampBoxReceived(self, box):
+        """
+        An AmpBox was received, representing a command, or an answer to a
+        previously issued command (either successful or erroneous).  Respond to
+        it according to its contents.
+
+        @param box: an AmpBox
+
+        @raise NoEmptyBoxes: when a box is received that does not contain an
+        '_answer', '_command' / '_ask', or '_error' key; i.e. one which does not
+        fit into the command / response protocol defined by AMP.
+        """
+        if ANSWER in box:
+            self._answerReceived(box)
+        elif ERROR in box:
+            self._errorReceived(box)
+        elif COMMAND in box:
+            self._commandReceived(box)
+        else:
+            raise NoEmptyBoxes(box)
+
+    def _safeEmit(self, aBox):
+        """
+        Emit a box, ignoring L{ProtocolSwitched} and L{ConnectionLost} errors
+        which cannot be usefully handled.
+        """
+        try:
+            aBox._sendTo(self.boxSender)
+        except (ProtocolSwitched, ConnectionLost):
+            pass
+
+    def dispatchCommand(self, box):
+        """
+        A box with a _command key was received.
+
+        Dispatch it to a local handler call it.
+
+        @param box: an AmpBox to be dispatched.
+        """
+        cmd = box[COMMAND]
+        responder = self.locator.locateResponder(cmd)
+        if responder is None:
+            description = f"Unhandled Command: {cmd!r}"
+            return fail(
+                RemoteAmpError(
+                    UNHANDLED_ERROR_CODE,
+                    description,
+                    False,
+                    local=Failure(UnhandledCommand()),
+                )
+            )
+        return maybeDeferred(responder, box)
+
+
+class _CommandLocatorMeta(type):
+    """
+    This metaclass keeps track of all of the Command.responder-decorated
+    methods defined since the last CommandLocator subclass was defined.  It
+    assumes (usually correctly, but unfortunately not necessarily so) that
+    those commands responders were all declared as methods of the class
+    being defined.  Note that this list can be incorrect if users use the
+    Command.responder decorator outside the context of a CommandLocator
+    class declaration.
+
+    Command responders defined on subclasses are given precedence over
+    those inherited from a base class.
+
+    The Command.responder decorator explicitly cooperates with this
+    metaclass.
+    """
+
+    _currentClassCommands: "list[tuple[type[Command], Callable[..., Any]]]" = []
+
+    def __new__(cls, name, bases, attrs):
+        commands = cls._currentClassCommands[:]
+        cls._currentClassCommands[:] = []
+        cd = attrs["_commandDispatch"] = {}
+        subcls = type.__new__(cls, name, bases, attrs)
+        ancestors = list(subcls.__mro__[1:])
+        ancestors.reverse()
+        for ancestor in ancestors:
+            cd.update(getattr(ancestor, "_commandDispatch", {}))
+        for commandClass, responderFunc in commands:
+            cd[commandClass.commandName] = (commandClass, responderFunc)
+        if bases and (subcls.lookupFunction != CommandLocator.lookupFunction):
+
+            def locateResponder(self, name):
+                warnings.warn(
+                    "Override locateResponder, not lookupFunction.",
+                    category=PendingDeprecationWarning,
+                    stacklevel=2,
+                )
+                return self.lookupFunction(name)
+
+            subcls.locateResponder = locateResponder
+        return subcls
+
+
+@implementer(IResponderLocator)
+class CommandLocator(metaclass=_CommandLocatorMeta):
+    """
+    A L{CommandLocator} is a collection of responders to AMP L{Command}s, with
+    the help of the L{Command.responder} decorator.
+    """
+
+    def _wrapWithSerialization(self, aCallable, command):
+        """
+        Wrap aCallable with its command's argument de-serialization
+        and result serialization logic.
+
+        @param aCallable: a callable with a 'command' attribute, designed to be
+        called with keyword arguments.
+
+        @param command: the command class whose serialization to use.
+
+        @return: a 1-arg callable which, when invoked with an AmpBox, will
+        deserialize the argument list and invoke appropriate user code for the
+        callable's command, returning a Deferred which fires with the result or
+        fails with an error.
+        """
+
+        def doit(box):
+            kw = command.parseArguments(box, self)
+
+            def checkKnownErrors(error):
+                key = error.trap(*command.allErrors)
+                code = command.allErrors[key]
+                desc = str(error.value)
+                return Failure(
+                    RemoteAmpError(code, desc, key in command.fatalErrors, local=error)
+                )
+
+            def makeResponseFor(objects):
+                try:
+                    return command.makeResponse(objects, self)
+                except BaseException:
+                    # let's helpfully log this.
+                    originalFailure = Failure()
+                    raise BadLocalReturn(
+                        "%r returned %r and %r could not serialize it"
+                        % (aCallable, objects, command),
+                        originalFailure,
+                    )
+
+            return (
+                maybeDeferred(aCallable, **kw)
+                .addCallback(makeResponseFor)
+                .addErrback(checkKnownErrors)
+            )
+
+        return doit
+
+    def lookupFunction(self, name):
+        """
+        Deprecated synonym for L{CommandLocator.locateResponder}
+        """
+        if self.__class__.lookupFunction != CommandLocator.lookupFunction:
+            return CommandLocator.locateResponder(self, name)
+        else:
+            warnings.warn(
+                "Call locateResponder, not lookupFunction.",
+                category=PendingDeprecationWarning,
+                stacklevel=2,
+            )
+        return self.locateResponder(name)
+
+    def locateResponder(self, name):
+        """
+        Locate a callable to invoke when executing the named command.
+
+        @param name: the normalized name (from the wire) of the command.
+        @type name: C{bytes}
+
+        @return: a 1-argument function that takes a Box and returns a box or a
+        Deferred which fires a Box, for handling the command identified by the
+        given name, or None, if no appropriate responder can be found.
+        """
+        # Try to find a high-level method to invoke, and if we can't find one,
+        # fall back to a low-level one.
+        cd = self._commandDispatch
+        if name in cd:
+            commandClass, responderFunc = cd[name]
+            responderMethod = MethodType(responderFunc, self)
+            return self._wrapWithSerialization(responderMethod, commandClass)
+
+
+@implementer(IResponderLocator)
+class SimpleStringLocator:
+    """
+    Implement the L{AMP.locateResponder} method to do simple, string-based
+    dispatch.
+    """
+
+    baseDispatchPrefix = b"amp_"
+
+    def locateResponder(self, name):
+        """
+        Locate a callable to invoke when executing the named command.
+
+        @return: a function with the name C{"amp_" + name} on the same
+            instance, or None if no such function exists.
+            This function will then be called with the L{AmpBox} itself as an
+            argument.
+
+        @param name: the normalized name (from the wire) of the command.
+        @type name: C{bytes}
+        """
+        fName = nativeString(self.baseDispatchPrefix + name.upper())
+        return getattr(self, fName, None)
+
+
+PYTHON_KEYWORDS = [
+    "and",
+    "del",
+    "for",
+    "is",
+    "raise",
+    "assert",
+    "elif",
+    "from",
+    "lambda",
+    "return",
+    "break",
+    "else",
+    "global",
+    "not",
+    "try",
+    "class",
+    "except",
+    "if",
+    "or",
+    "while",
+    "continue",
+    "exec",
+    "import",
+    "pass",
+    "yield",
+    "def",
+    "finally",
+    "in",
+    "print",
+]
+
+
+def _wireNameToPythonIdentifier(key):
+    """
+    (Private) Normalize an argument name from the wire for use with Python
+    code.  If the return value is going to be a python keyword it will be
+    capitalized.  If it contains any dashes they will be replaced with
+    underscores.
+
+    The rationale behind this method is that AMP should be an inherently
+    multi-language protocol, so message keys may contain all manner of bizarre
+    bytes.  This is not a complete solution; there are still forms of arguments
+    that this implementation will be unable to parse.  However, Python
+    identifiers share a huge raft of properties with identifiers from many
+    other languages, so this is a 'good enough' effort for now.  We deal
+    explicitly with dashes because that is the most likely departure: Lisps
+    commonly use dashes to separate method names, so protocols initially
+    implemented in a lisp amp dialect may use dashes in argument or command
+    names.
+
+    @param key: a C{bytes}, looking something like 'foo-bar-baz' or 'from'
+    @type key: C{bytes}
+
+    @return: a native string which is a valid python identifier, looking
+    something like 'foo_bar_baz' or 'From'.
+    """
+    lkey = nativeString(key.replace(b"-", b"_"))
+    if lkey in PYTHON_KEYWORDS:
+        return lkey.title()
+    return lkey
+
+
+@implementer(IArgumentType)
+class Argument:
+    """
+    Base-class of all objects that take values from Amp packets and convert
+    them into objects for Python functions.
+
+    This implementation of L{IArgumentType} provides several higher-level
+    hooks for subclasses to override.  See L{toString} and L{fromString}
+    which will be used to define the behavior of L{IArgumentType.toBox} and
+    L{IArgumentType.fromBox}, respectively.
+    """
+
+    optional = False
+
+    def __init__(self, optional=False):
+        """
+        Create an Argument.
+
+        @param optional: a boolean indicating whether this argument can be
+        omitted in the protocol.
+        """
+        self.optional = optional
+
+    def retrieve(self, d, name, proto):
+        """
+        Retrieve the given key from the given dictionary, removing it if found.
+
+        @param d: a dictionary.
+
+        @param name: a key in I{d}.
+
+        @param proto: an instance of an AMP.
+
+        @raise KeyError: if I am not optional and no value was found.
+
+        @return: d[name].
+        """
+        if self.optional:
+            value = d.get(name)
+            if value is not None:
+                del d[name]
+        else:
+            value = d.pop(name)
+        return value
+
+    def fromBox(self, name, strings, objects, proto):
+        """
+        Populate an 'out' dictionary with mapping names to Python values
+        decoded from an 'in' AmpBox mapping strings to string values.
+
+        @param name: the argument name to retrieve
+        @type name: C{bytes}
+
+        @param strings: The AmpBox to read string(s) from, a mapping of
+        argument names to string values.
+        @type strings: AmpBox
+
+        @param objects: The dictionary to write object(s) to, a mapping of
+        names to Python objects. Keys will be native strings.
+        @type objects: dict
+
+        @param proto: an AMP instance.
+        """
+        st = self.retrieve(strings, name, proto)
+        nk = _wireNameToPythonIdentifier(name)
+        if self.optional and st is None:
+            objects[nk] = None
+        else:
+            objects[nk] = self.fromStringProto(st, proto)
+
+    def toBox(self, name, strings, objects, proto):
+        """
+        Populate an 'out' AmpBox with strings encoded from an 'in' dictionary
+        mapping names to Python values.
+
+        @param name: the argument name to retrieve
+        @type name: C{bytes}
+
+        @param strings: The AmpBox to write string(s) to, a mapping of
+        argument names to string values.
+        @type strings: AmpBox
+
+        @param objects: The dictionary to read object(s) from, a mapping of
+        names to Python objects. Keys should be native strings.
+
+        @type objects: dict
+
+        @param proto: the protocol we are converting for.
+        @type proto: AMP
+        """
+        obj = self.retrieve(objects, _wireNameToPythonIdentifier(name), proto)
+        if self.optional and obj is None:
+            # strings[name] = None
+            pass
+        else:
+            strings[name] = self.toStringProto(obj, proto)
+
+    def fromStringProto(self, inString, proto):
+        """
+        Convert a string to a Python value.
+
+        @param inString: the string to convert.
+        @type inString: C{bytes}
+
+        @param proto: the protocol we are converting for.
+        @type proto: AMP
+
+        @return: a Python object.
+        """
+        return self.fromString(inString)
+
+    def toStringProto(self, inObject, proto):
+        """
+        Convert a Python object to a string.
+
+        @param inObject: the object to convert.
+
+        @param proto: the protocol we are converting for.
+        @type proto: AMP
+        """
+        return self.toString(inObject)
+
+    def fromString(self, inString):
+        """
+        Convert a string to a Python object.  Subclasses must implement this.
+
+        @param inString: the string to convert.
+        @type inString: C{bytes}
+
+        @return: the decoded value from C{inString}
+        """
+
+    def toString(self, inObject):
+        """
+        Convert a Python object into a string for passing over the network.
+
+        @param inObject: an object of the type that this Argument is intended
+        to deal with.
+
+        @return: the wire encoding of inObject
+        @rtype: C{bytes}
+        """
+
+
+class Integer(Argument):
+    """
+    Encode any integer values of any size on the wire as the string
+    representation.
+
+    Example: C{123} becomes C{"123"}
+    """
+
+    fromString = int
+
+    def toString(self, inObject):
+        return b"%d" % (inObject,)
+
+
+class String(Argument):
+    """
+    Don't do any conversion at all; just pass through 'str'.
+    """
+
+    def toString(self, inObject):
+        return inObject
+
+    def fromString(self, inString):
+        return inString
+
+
+class Float(Argument):
+    """
+    Encode floating-point values on the wire as their repr.
+    """
+
+    fromString = float
+
+    def toString(self, inString):
+        if not isinstance(inString, float):
+            raise ValueError(f"Bad float value {inString!r}")
+        return str(inString).encode("ascii")
+
+
+class Boolean(Argument):
+    """
+    Encode True or False as "True" or "False" on the wire.
+    """
+
+    def fromString(self, inString):
+        if inString == b"True":
+            return True
+        elif inString == b"False":
+            return False
+        else:
+            raise TypeError(f"Bad boolean value: {inString!r}")
+
+    def toString(self, inObject):
+        if inObject:
+            return b"True"
+        else:
+            return b"False"
+
+
+class Unicode(String):
+    """
+    Encode a unicode string on the wire as UTF-8.
+    """
+
+    def toString(self, inObject):
+        return String.toString(self, inObject.encode("utf-8"))
+
+    def fromString(self, inString):
+        return String.fromString(self, inString).decode("utf-8")
+
+
+class Path(Unicode):
+    """
+    Encode and decode L{filepath.FilePath} instances as paths on the wire.
+
+    This is really intended for use with subprocess communication tools:
+    exchanging pathnames on different machines over a network is not generally
+    meaningful, but neither is it disallowed; you can use this to communicate
+    about NFS paths, for example.
+    """
+
+    def fromString(self, inString):
+        return filepath.FilePath(Unicode.fromString(self, inString))
+
+    def toString(self, inObject):
+        return Unicode.toString(self, inObject.asTextMode().path)
+
+
+class ListOf(Argument):
+    """
+    Encode and decode lists of instances of a single other argument type.
+
+    For example, if you want to pass::
+
+        [3, 7, 9, 15]
+
+    You can create an argument like this::
+
+        ListOf(Integer())
+
+    The serialized form of the entire list is subject to the limit imposed by
+    L{MAX_VALUE_LENGTH}.  List elements are represented as 16-bit length
+    prefixed strings.  The argument type passed to the L{ListOf} initializer is
+    responsible for producing the serialized form of each element.
+
+    @ivar elementType: The L{Argument} instance used to encode and decode list
+        elements (note, not an arbitrary L{IArgumentType} implementation:
+        arguments must be implemented using only the C{fromString} and
+        C{toString} methods, not the C{fromBox} and C{toBox} methods).
+
+    @param optional: a boolean indicating whether this argument can be
+        omitted in the protocol.
+
+    @since: 10.0
+    """
+
+    def __init__(self, elementType, optional=False):
+        self.elementType = elementType
+        Argument.__init__(self, optional)
+
+    def fromString(self, inString):
+        """
+        Convert the serialized form of a list of instances of some type back
+        into that list.
+        """
+        strings = []
+        parser = Int16StringReceiver()
+        parser.stringReceived = strings.append
+        parser.dataReceived(inString)
+        elementFromString = self.elementType.fromString
+        return [elementFromString(string) for string in strings]
+
+    def toString(self, inObject):
+        """
+        Serialize the given list of objects to a single string.
+        """
+        strings = []
+        for obj in inObject:
+            serialized = self.elementType.toString(obj)
+            strings.append(pack("!H", len(serialized)))
+            strings.append(serialized)
+        return b"".join(strings)
+
+
+class AmpList(Argument):
+    """
+    Convert a list of dictionaries into a list of AMP boxes on the wire.
+
+    For example, if you want to pass::
+
+        [{'a': 7, 'b': u'hello'}, {'a': 9, 'b': u'goodbye'}]
+
+    You might use an AmpList like this in your arguments or response list::
+
+        AmpList([('a', Integer()),
+                 ('b', Unicode())])
+    """
+
+    def __init__(self, subargs, optional=False):
+        """
+        Create an AmpList.
+
+        @param subargs: a list of 2-tuples of ('name', argument) describing the
+        schema of the dictionaries in the sequence of amp boxes.
+        @type subargs: A C{list} of (C{bytes}, L{Argument}) tuples.
+
+        @param optional: a boolean indicating whether this argument can be
+        omitted in the protocol.
+        """
+        assert all(isinstance(name, bytes) for name, _ in subargs), (
+            "AmpList should be defined with a list of (name, argument) "
+            "tuples where `name' is a byte string, got: %r" % (subargs,)
+        )
+        self.subargs = subargs
+        Argument.__init__(self, optional)
+
+    def fromStringProto(self, inString, proto):
+        boxes = parseString(inString)
+        values = [_stringsToObjects(box, self.subargs, proto) for box in boxes]
+        return values
+
+    def toStringProto(self, inObject, proto):
+        return b"".join(
+            [
+                _objectsToStrings(objects, self.subargs, Box(), proto).serialize()
+                for objects in inObject
+            ]
+        )
+
+
+class Descriptor(Integer):
+    """
+    Encode and decode file descriptors for exchange over a UNIX domain socket.
+
+    This argument type requires an AMP connection set up over an
+    L{IUNIXTransport<twisted.internet.interfaces.IUNIXTransport>} provider (for
+    example, the kind of connection created by
+    L{IReactorUNIX.connectUNIX<twisted.internet.interfaces.IReactorUNIX.connectUNIX>}
+    and L{UNIXClientEndpoint<twisted.internet.endpoints.UNIXClientEndpoint>}).
+
+    There is no correspondence between the integer value of the file descriptor
+    on the sending and receiving sides, therefore an alternate approach is taken
+    to matching up received descriptors with particular L{Descriptor}
+    parameters.  The argument is encoded to an ordinal (unique per connection)
+    for inclusion in the AMP command or response box.  The descriptor itself is
+    sent using
+    L{IUNIXTransport.sendFileDescriptor<twisted.internet.interfaces.IUNIXTransport.sendFileDescriptor>}.
+    The receiver uses the order in which file descriptors are received and the
+    ordinal value to come up with the received copy of the descriptor.
+    """
+
+    def fromStringProto(self, inString, proto):
+        """
+        Take a unique identifier associated with a file descriptor which must
+        have been received by now and use it to look up that descriptor in a
+        dictionary where they are kept.
+
+        @param inString: The base representation (as a byte string) of an
+            ordinal indicating which file descriptor corresponds to this usage
+            of this argument.
+        @type inString: C{str}
+
+        @param proto: The protocol used to receive this descriptor.  This
+            protocol must be connected via a transport providing
+            L{IUNIXTransport<twisted.internet.interfaces.IUNIXTransport>}.
+        @type proto: L{BinaryBoxProtocol}
+
+        @return: The file descriptor represented by C{inString}.
+        @rtype: C{int}
+        """
+        return proto._getDescriptor(int(inString))
+
+    def toStringProto(self, inObject, proto):
+        """
+        Send C{inObject}, an integer file descriptor, over C{proto}'s connection
+        and return a unique identifier which will allow the receiver to
+        associate the file descriptor with this argument.
+
+        @param inObject: A file descriptor to duplicate over an AMP connection
+            as the value for this argument.
+        @type inObject: C{int}
+
+        @param proto: The protocol which will be used to send this descriptor.
+            This protocol must be connected via a transport providing
+            L{IUNIXTransport<twisted.internet.interfaces.IUNIXTransport>}.
+
+        @return: A byte string which can be used by the receiver to reconstruct
+            the file descriptor.
+        @rtype: C{bytes}
+        """
+        identifier = proto._sendFileDescriptor(inObject)
+        outString = Integer.toStringProto(self, identifier, proto)
+        return outString
+
+
+class _CommandMeta(type):
+    """
+    Metaclass hack to establish reverse-mappings for 'errors' and
+    'fatalErrors' as class vars.
+    """
+
+    def __new__(cls, name, bases, attrs):
+        reverseErrors = attrs["reverseErrors"] = {}
+        er = attrs["allErrors"] = {}
+        if "commandName" not in attrs:
+            attrs["commandName"] = name.encode("ascii")
+        newtype = type.__new__(cls, name, bases, attrs)
+
+        if not isinstance(newtype.commandName, bytes):
+            raise TypeError(
+                "Command names must be byte strings, got: {!r}".format(
+                    newtype.commandName
+                )
+            )
+        for name, _ in newtype.arguments:
+            if not isinstance(name, bytes):
+                raise TypeError(f"Argument names must be byte strings, got: {name!r}")
+        for name, _ in newtype.response:
+            if not isinstance(name, bytes):
+                raise TypeError(f"Response names must be byte strings, got: {name!r}")
+
+        errors: Dict[Type[Exception], bytes] = {}
+        fatalErrors: Dict[Type[Exception], bytes] = {}
+        accumulateClassDict(newtype, "errors", errors)
+        accumulateClassDict(newtype, "fatalErrors", fatalErrors)
+
+        if not isinstance(newtype.errors, dict):
+            newtype.errors = dict(newtype.errors)
+        if not isinstance(newtype.fatalErrors, dict):
+            newtype.fatalErrors = dict(newtype.fatalErrors)
+
+        for v, k in errors.items():
+            reverseErrors[k] = v
+            er[v] = k
+        for v, k in fatalErrors.items():
+            reverseErrors[k] = v
+            er[v] = k
+
+        for _, name in newtype.errors.items():
+            if not isinstance(name, bytes):
+                raise TypeError(f"Error names must be byte strings, got: {name!r}")
+        for _, name in newtype.fatalErrors.items():
+            if not isinstance(name, bytes):
+                raise TypeError(
+                    f"Fatal error names must be byte strings, got: {name!r}"
+                )
+
+        return newtype
+
+
+class Command(metaclass=_CommandMeta):
+    """
+    Subclass me to specify an AMP Command.
+
+    @cvar arguments: A list of 2-tuples of (name, Argument-subclass-instance),
+    specifying the names and values of the parameters which are required for
+    this command.
+
+    @cvar response: A list like L{arguments}, but instead used for the return
+    value.
+
+    @cvar errors: A mapping of subclasses of L{Exception} to wire-protocol tags
+        for errors represented as L{str}s.  Responders which raise keys from
+        this dictionary will have the error translated to the corresponding tag
+        on the wire.
+        Invokers which receive Deferreds from invoking this command with
+        L{BoxDispatcher.callRemote} will potentially receive Failures with keys
+        from this mapping as their value.
+        This mapping is inherited; if you declare a command which handles
+        C{FooError} as 'FOO_ERROR', then subclass it and specify C{BarError} as
+        'BAR_ERROR', responders to the subclass may raise either C{FooError} or
+        C{BarError}, and invokers must be able to deal with either of those
+        exceptions.
+
+    @cvar fatalErrors: like 'errors', but errors in this list will always
+    terminate the connection, despite being of a recognizable error type.
+
+    @cvar commandType: The type of Box used to issue commands; useful only for
+    protocol-modifying behavior like startTLS or protocol switching.  Defaults
+    to a plain vanilla L{Box}.
+
+    @cvar responseType: The type of Box used to respond to this command; only
+    useful for protocol-modifying behavior like startTLS or protocol switching.
+    Defaults to a plain vanilla L{Box}.
+
+    @ivar requiresAnswer: a boolean; defaults to True.  Set it to False on your
+    subclass if you want callRemote to return None.  Note: this is a hint only
+    to the client side of the protocol.  The return-type of a command responder
+    method must always be a dictionary adhering to the contract specified by
+    L{response}, because clients are always free to request a response if they
+    want one.
+    """
+
+    arguments: List[Tuple[bytes, Argument]] = []
+    response: List[Tuple[bytes, Argument]] = []
+    extra: List[Any] = []
+    errors: Dict[Type[Exception], bytes] = {}
+    fatalErrors: Dict[Type[Exception], bytes] = {}
+
+    commandType: "Union[Type[Command], Type[Box]]" = Box
+    responseType: Type[AmpBox] = Box
+
+    requiresAnswer = True
+
+    def __init__(self, **kw):
+        """
+        Create an instance of this command with specified values for its
+        parameters.
+
+        In Python 3, keyword arguments MUST be Unicode/native strings whereas
+        in Python 2 they could be either byte strings or Unicode strings.
+
+        A L{Command}'s arguments are defined in its schema using C{bytes}
+        names. The values for those arguments are plucked from the keyword
+        arguments using the name returned from L{_wireNameToPythonIdentifier}.
+        In other words, keyword arguments should be named using the
+        Python-side equivalent of the on-wire (C{bytes}) name.
+
+        @param kw: a dict containing an appropriate value for each name
+        specified in the L{arguments} attribute of my class.
+
+        @raise InvalidSignature: if you forgot any required arguments.
+        """
+        self.structured = kw
+        forgotten = []
+        for name, arg in self.arguments:
+            pythonName = _wireNameToPythonIdentifier(name)
+            if pythonName not in self.structured and not arg.optional:
+                forgotten.append(pythonName)
+        if forgotten:
+            raise InvalidSignature(
+                "forgot {} for {}".format(", ".join(forgotten), self.commandName)
+            )
+        forgotten = []
+
+    @classmethod
+    def makeResponse(cls, objects, proto):
+        """
+        Serialize a mapping of arguments using this L{Command}'s
+        response schema.
+
+        @param objects: a dict with keys matching the names specified in
+        self.response, having values of the types that the Argument objects in
+        self.response can format.
+
+        @param proto: an L{AMP}.
+
+        @return: an L{AmpBox}.
+        """
+        try:
+            responseType = cls.responseType()
+        except BaseException:
+            return fail()
+        return _objectsToStrings(objects, cls.response, responseType, proto)
+
+    @classmethod
+    def makeArguments(cls, objects, proto):
+        """
+        Serialize a mapping of arguments using this L{Command}'s
+        argument schema.
+
+        @param objects: a dict with keys similar to the names specified in
+        self.arguments, having values of the types that the Argument objects in
+        self.arguments can parse.
+
+        @param proto: an L{AMP}.
+
+        @return: An instance of this L{Command}'s C{commandType}.
+        """
+        allowedNames = set()
+        for argName, ignored in cls.arguments:
+            allowedNames.add(_wireNameToPythonIdentifier(argName))
+
+        for intendedArg in objects:
+            if intendedArg not in allowedNames:
+                raise InvalidSignature(f"{intendedArg} is not a valid argument")
+        return _objectsToStrings(objects, cls.arguments, cls.commandType(), proto)
+
+    @classmethod
+    def parseResponse(cls, box, protocol):
+        """
+        Parse a mapping of serialized arguments using this
+        L{Command}'s response schema.
+
+        @param box: A mapping of response-argument names to the
+        serialized forms of those arguments.
+        @param protocol: The L{AMP} protocol.
+
+        @return: A mapping of response-argument names to the parsed
+        forms.
+        """
+        return _stringsToObjects(box, cls.response, protocol)
+
+    @classmethod
+    def parseArguments(cls, box, protocol):
+        """
+        Parse a mapping of serialized arguments using this
+        L{Command}'s argument schema.
+
+        @param box: A mapping of argument names to the seralized forms
+        of those arguments.
+        @param protocol: The L{AMP} protocol.
+
+        @return: A mapping of argument names to the parsed forms.
+        """
+        return _stringsToObjects(box, cls.arguments, protocol)
+
+    @classmethod
+    def responder(cls, methodfunc: _T_Callable) -> _T_Callable:
+        """
+        Declare a method to be a responder for a particular command.
+
+        This is a decorator.
+
+        Use like so::
+
+            class MyCommand(Command):
+                arguments = [('a', ...), ('b', ...)]
+
+            class MyProto(AMP):
+                def myFunMethod(self, a, b):
+                    ...
+                MyCommand.responder(myFunMethod)
+
+        Notes: Although decorator syntax is not used within Twisted, this
+        function returns its argument and is therefore safe to use with
+        decorator syntax.
+
+        This is not thread safe.  Don't declare AMP subclasses in other
+        threads.  Don't declare responders outside the scope of AMP subclasses;
+        the behavior is undefined.
+
+        @param methodfunc: A function which will later become a method, which
+        has a keyword signature compatible with this command's L{arguments} list
+        and returns a dictionary with a set of keys compatible with this
+        command's L{response} list.
+
+        @return: the methodfunc parameter.
+        """
+        CommandLocator._currentClassCommands.append((cls, methodfunc))
+        return methodfunc
+
+    # Our only instance method
+    def _doCommand(self, proto):
+        """
+        Encode and send this Command to the given protocol.
+
+        @param proto: an AMP, representing the connection to send to.
+
+        @return: a Deferred which will fire or error appropriately when the
+        other side responds to the command (or error if the connection is lost
+        before it is responded to).
+        """
+
+        def _massageError(error):
+            error.trap(RemoteAmpError)
+            rje = error.value
+            errorType = self.reverseErrors.get(rje.errorCode, UnknownRemoteError)
+            return Failure(errorType(rje.description))
+
+        d = proto._sendBoxCommand(
+            self.commandName,
+            self.makeArguments(self.structured, proto),
+            self.requiresAnswer,
+        )
+
+        if self.requiresAnswer:
+            d.addCallback(self.parseResponse, proto)
+            d.addErrback(_massageError)
+
+        return d
+
+
+class _NoCertificate:
+    """
+    This is for peers which don't want to use a local certificate.  Used by
+    AMP because AMP's internal language is all about certificates and this
+    duck-types in the appropriate place; this API isn't really stable though,
+    so it's not exposed anywhere public.
+
+    For clients, it will use ephemeral DH keys, or whatever the default is for
+    certificate-less clients in OpenSSL.  For servers, it will generate a
+    temporary self-signed certificate with garbage values in the DN and use
+    that.
+    """
+
+    def __init__(self, client):
+        """
+        Create a _NoCertificate which either is or isn't for the client side of
+        the connection.
+
+        @param client: True if we are a client and should truly have no
+        certificate and be anonymous, False if we are a server and actually
+        have to generate a temporary certificate.
+
+        @type client: bool
+        """
+        self.client = client
+
+    def options(self, *authorities):
+        """
+        Behaves like L{twisted.internet.ssl.PrivateCertificate.options}().
+        """
+        if not self.client:
+            # do some crud with sslverify to generate a temporary self-signed
+            # certificate.  This is SLOOOWWWWW so it is only in the absolute
+            # worst, most naive case.
+
+            # We have to do this because OpenSSL will not let both the server
+            # and client be anonymous.
+            sharedDN = DN(CN="TEMPORARY CERTIFICATE")
+            key = KeyPair.generate()
+            cr = key.certificateRequest(sharedDN)
+            sscrd = key.signCertificateRequest(sharedDN, cr, lambda dn: True, 1)
+            cert = key.newCertificate(sscrd)
+            return cert.options(*authorities)
+        options = dict()
+        if authorities:
+            options.update(
+                dict(
+                    verify=True,
+                    requireCertificate=True,
+                    caCerts=[auth.original for auth in authorities],
+                )
+            )
+        occo = CertificateOptions(**options)
+        return occo
+
+
+class _TLSBox(AmpBox):
+    """
+    I am an AmpBox that, upon being sent, initiates a TLS connection.
+    """
+
+    __slots__: List[str] = []
+
+    def __init__(self):
+        if ssl is None:
+            raise RemoteAmpError(b"TLS_ERROR", "TLS not available")
+        AmpBox.__init__(self)
+
+    @property
+    def certificate(self):
+        return self.get(b"tls_localCertificate", _NoCertificate(False))
+
+    @property
+    def verify(self):
+        return self.get(b"tls_verifyAuthorities", None)
+
+    def _sendTo(self, proto):
+        """
+        Send my encoded value to the protocol, then initiate TLS.
+        """
+        ab = AmpBox(self)
+        for k in [b"tls_localCertificate", b"tls_verifyAuthorities"]:
+            ab.pop(k, None)
+        ab._sendTo(proto)
+        proto._startTLS(self.certificate, self.verify)
+
+
+class _LocalArgument(String):
+    """
+    Local arguments are never actually relayed across the wire.  This is just a
+    shim so that StartTLS can pretend to have some arguments: if arguments
+    acquire documentation properties, replace this with something nicer later.
+    """
+
+    def fromBox(self, name, strings, objects, proto):
+        pass
+
+
+class StartTLS(Command):
+    """
+    Use, or subclass, me to implement a command that starts TLS.
+
+    Callers of StartTLS may pass several special arguments, which affect the
+    TLS negotiation:
+
+        - tls_localCertificate: This is a
+        twisted.internet.ssl.PrivateCertificate which will be used to secure
+        the side of the connection it is returned on.
+
+        - tls_verifyAuthorities: This is a list of
+        twisted.internet.ssl.Certificate objects that will be used as the
+        certificate authorities to verify our peer's certificate.
+
+    Each of those special parameters may also be present as a key in the
+    response dictionary.
+    """
+
+    arguments = [
+        (b"tls_localCertificate", _LocalArgument(optional=True)),
+        (b"tls_verifyAuthorities", _LocalArgument(optional=True)),
+    ]
+
+    response = [
+        (b"tls_localCertificate", _LocalArgument(optional=True)),
+        (b"tls_verifyAuthorities", _LocalArgument(optional=True)),
+    ]
+
+    responseType = _TLSBox
+
+    def __init__(self, *, tls_localCertificate=None, tls_verifyAuthorities=None, **kw):
+        """
+        Create a StartTLS command.  (This is private.  Use AMP.callRemote.)
+
+        @param tls_localCertificate: the PrivateCertificate object to use to
+        secure the connection.  If it's L{None}, or unspecified, an ephemeral DH
+        key is used instead.
+
+        @param tls_verifyAuthorities: a list of Certificate objects which
+        represent root certificates to verify our peer with.
+        """
+        if ssl is None:
+            raise RuntimeError("TLS not available.")
+        self.certificate = (
+            _NoCertificate(True)
+            if tls_localCertificate is None
+            else tls_localCertificate
+        )
+        self.authorities = tls_verifyAuthorities
+        Command.__init__(self, **kw)
+
+    def _doCommand(self, proto):
+        """
+        When a StartTLS command is sent, prepare to start TLS, but don't actually
+        do it; wait for the acknowledgement, then initiate the TLS handshake.
+        """
+        d = Command._doCommand(self, proto)
+        proto._prepareTLS(self.certificate, self.authorities)
+        # XXX before we get back to user code we are going to start TLS...
+
+        def actuallystart(response):
+            proto._startTLS(self.certificate, self.authorities)
+            return response
+
+        d.addCallback(actuallystart)
+        return d
+
+
+class ProtocolSwitchCommand(Command):
+    """
+    Use this command to switch from something Amp-derived to a different
+    protocol mid-connection.  This can be useful to use amp as the
+    connection-startup negotiation phase.  Since TLS is a different layer
+    entirely, you can use Amp to negotiate the security parameters of your
+    connection, then switch to a different protocol, and the connection will
+    remain secured.
+    """
+
+    def __init__(self, _protoToSwitchToFactory, **kw):
+        """
+        Create a ProtocolSwitchCommand.
+
+        @param _protoToSwitchToFactory: a ProtocolFactory which will generate
+        the Protocol to switch to.
+
+        @param kw: Keyword arguments, encoded and handled normally as
+        L{Command} would.
+        """
+
+        self.protoToSwitchToFactory = _protoToSwitchToFactory
+        super().__init__(**kw)
+
+    @classmethod
+    def makeResponse(cls, innerProto, proto):
+        return _SwitchBox(innerProto)
+
+    def _doCommand(self, proto):
+        """
+        When we emit a ProtocolSwitchCommand, lock the protocol, but don't actually
+        switch to the new protocol unless an acknowledgement is received.  If
+        an error is received, switch back.
+        """
+        d = super()._doCommand(proto)
+        proto._lockForSwitch()
+
+        def switchNow(ign):
+            innerProto = self.protoToSwitchToFactory.buildProtocol(
+                proto.transport.getPeer()
+            )
+            proto._switchTo(innerProto, self.protoToSwitchToFactory)
+            return ign
+
+        def handle(ign):
+            proto._unlockFromSwitch()
+            self.protoToSwitchToFactory.clientConnectionFailed(
+                None, Failure(CONNECTION_LOST)
+            )
+            return ign
+
+        return d.addCallbacks(switchNow, handle)
+
+
+@implementer(IFileDescriptorReceiver)
+class _DescriptorExchanger:
+    """
+    L{_DescriptorExchanger} is a mixin for L{BinaryBoxProtocol} which adds
+    support for receiving file descriptors, a feature offered by
+    L{IUNIXTransport<twisted.internet.interfaces.IUNIXTransport>}.
+
+    @ivar _descriptors: Temporary storage for all file descriptors received.
+        Values in this dictionary are the file descriptors (as integers).  Keys
+        in this dictionary are ordinals giving the order in which each
+        descriptor was received.  The ordering information is used to allow
+        L{Descriptor} to determine which is the correct descriptor for any
+        particular usage of that argument type.
+    @type _descriptors: C{dict}
+
+    @ivar _sendingDescriptorCounter: A no-argument callable which returns the
+        ordinals, starting from 0.  This is used to construct values for
+        C{_sendFileDescriptor}.
+
+    @ivar _receivingDescriptorCounter: A no-argument callable which returns the
+        ordinals, starting from 0.  This is used to construct values for
+        C{fileDescriptorReceived}.
+    """
+
+    def __init__(self):
+        self._descriptors = {}
+        self._getDescriptor = self._descriptors.pop
+        self._sendingDescriptorCounter = partial(next, count())
+        self._receivingDescriptorCounter = partial(next, count())
+
+    def _sendFileDescriptor(self, descriptor):
+        """
+        Assign and return the next ordinal to the given descriptor after sending
+        the descriptor over this protocol's transport.
+        """
+        self.transport.sendFileDescriptor(descriptor)
+        return self._sendingDescriptorCounter()
+
+    def fileDescriptorReceived(self, descriptor):
+        """
+        Collect received file descriptors to be claimed later by L{Descriptor}.
+
+        @param descriptor: The received file descriptor.
+        @type descriptor: C{int}
+        """
+        self._descriptors[self._receivingDescriptorCounter()] = descriptor
+
+
+@implementer(IBoxSender)
+class BinaryBoxProtocol(
+    StatefulStringProtocol, Int16StringReceiver, _DescriptorExchanger
+):
+    """
+    A protocol for receiving L{AmpBox}es - key/value pairs - via length-prefixed
+    strings.  A box is composed of:
+
+        - any number of key-value pairs, described by:
+            - a 2-byte network-endian packed key length (of which the first
+              byte must be null, and the second must be non-null: i.e. the
+              value of the length must be 1-255)
+            - a key, comprised of that many bytes
+            - a 2-byte network-endian unsigned value length (up to the maximum
+              of 65535)
+            - a value, comprised of that many bytes
+        - 2 null bytes
+
+    In other words, an even number of strings prefixed with packed unsigned
+    16-bit integers, and then a 0-length string to indicate the end of the box.
+
+    This protocol also implements 2 extra private bits of functionality related
+    to the byte boundaries between messages; it can start TLS between two given
+    boxes or switch to an entirely different protocol.  However, due to some
+    tricky elements of the implementation, the public interface to this
+    functionality is L{ProtocolSwitchCommand} and L{StartTLS}.
+
+    @ivar _keyLengthLimitExceeded: A flag which is only true when the
+        connection is being closed because a key length prefix which was longer
+        than allowed by the protocol was received.
+
+    @ivar boxReceiver: an L{IBoxReceiver} provider, whose
+        L{IBoxReceiver.ampBoxReceived} method will be invoked for each
+        L{AmpBox} that is received.
+    """
+
+    _justStartedTLS = False
+    _startingTLSBuffer = None
+    _locked = False
+    _currentKey = None
+    _currentBox = None
+
+    _keyLengthLimitExceeded = False
+
+    hostCertificate = None
+    noPeerCertificate = False  # for tests
+    innerProtocol: Optional[Protocol] = None
+    innerProtocolClientFactory = None
+
+    def __init__(self, boxReceiver):
+        _DescriptorExchanger.__init__(self)
+        self.boxReceiver = boxReceiver
+
+    def _switchTo(self, newProto, clientFactory=None):
+        """
+        Switch this BinaryBoxProtocol's transport to a new protocol.  You need
+        to do this 'simultaneously' on both ends of a connection; the easiest
+        way to do this is to use a subclass of ProtocolSwitchCommand.
+
+        @param newProto: the new protocol instance to switch to.
+
+        @param clientFactory: the ClientFactory to send the
+            L{twisted.internet.protocol.ClientFactory.clientConnectionLost}
+            notification to.
+        """
+        # All the data that Int16Receiver has not yet dealt with belongs to our
+        # new protocol: luckily it's keeping that in a handy (although
+        # ostensibly internal) variable for us:
+        newProtoData = self.recvd
+        # We're quite possibly in the middle of a 'dataReceived' loop in
+        # Int16StringReceiver: let's make sure that the next iteration, the
+        # loop will break and not attempt to look at something that isn't a
+        # length prefix.
+        self.recvd = ""
+        # Finally, do the actual work of setting up the protocol and delivering
+        # its first chunk of data, if one is available.
+        self.innerProtocol = newProto
+        self.innerProtocolClientFactory = clientFactory
+        newProto.makeConnection(self.transport)
+        if newProtoData:
+            newProto.dataReceived(newProtoData)
+
+    def sendBox(self, box):
+        """
+        Send a amp.Box to my peer.
+
+        Note: transport.write is never called outside of this method.
+
+        @param box: an AmpBox.
+
+        @raise ProtocolSwitched: if the protocol has previously been switched.
+
+        @raise ConnectionLost: if the connection has previously been lost.
+        """
+        if self._locked:
+            raise ProtocolSwitched(
+                "This connection has switched: no AMP traffic allowed."
+            )
+        if self.transport is None:
+            raise ConnectionLost()
+        if self._startingTLSBuffer is not None:
+            self._startingTLSBuffer.append(box)
+        else:
+            self.transport.write(box.serialize())
+
+    def makeConnection(self, transport):
+        """
+        Notify L{boxReceiver} that it is about to receive boxes from this
+        protocol by invoking L{IBoxReceiver.startReceivingBoxes}.
+        """
+        self.transport = transport
+        self.boxReceiver.startReceivingBoxes(self)
+        self.connectionMade()
+
+    def dataReceived(self, data):
+        """
+        Either parse incoming data as L{AmpBox}es or relay it to our nested
+        protocol.
+        """
+        if self._justStartedTLS:
+            self._justStartedTLS = False
+        # If we already have an inner protocol, then we don't deliver data to
+        # the protocol parser any more; we just hand it off.
+        if self.innerProtocol is not None:
+            self.innerProtocol.dataReceived(data)
+            return
+        return Int16StringReceiver.dataReceived(self, data)
+
+    def connectionLost(self, reason):
+        """
+        The connection was lost; notify any nested protocol.
+        """
+        if self.innerProtocol is not None:
+            self.innerProtocol.connectionLost(reason)
+            if self.innerProtocolClientFactory is not None:
+                self.innerProtocolClientFactory.clientConnectionLost(None, reason)
+        if self._keyLengthLimitExceeded:
+            failReason = Failure(TooLong(True, False, None, None))
+        elif reason.check(ConnectionClosed) and self._justStartedTLS:
+            # We just started TLS and haven't received any data.  This means
+            # the other connection didn't like our cert (although they may not
+            # have told us why - later Twisted should make 'reason' into a TLS
+            # error.)
+            failReason = PeerVerifyError(
+                "Peer rejected our certificate for an unknown reason."
+            )
+        else:
+            failReason = reason
+        self.boxReceiver.stopReceivingBoxes(failReason)
+
+    # The longest key allowed
+    _MAX_KEY_LENGTH = 255
+
+    # The longest value allowed (this is somewhat redundant, as longer values
+    # cannot be encoded - ah well).
+    _MAX_VALUE_LENGTH = 65535
+
+    # The first thing received is a key.
+    MAX_LENGTH = _MAX_KEY_LENGTH
+
+    def proto_init(self, string):
+        """
+        String received in the 'init' state.
+        """
+        self._currentBox = AmpBox()
+        return self.proto_key(string)
+
+    def proto_key(self, string):
+        """
+        String received in the 'key' state.  If the key is empty, a complete
+        box has been received.
+        """
+        if string:
+            self._currentKey = string
+            self.MAX_LENGTH = self._MAX_VALUE_LENGTH
+            return "value"
+        else:
+            self.boxReceiver.ampBoxReceived(self._currentBox)
+            self._currentBox = None
+            return "init"
+
+    def proto_value(self, string):
+        """
+        String received in the 'value' state.
+        """
+        self._currentBox[self._currentKey] = string
+        self._currentKey = None
+        self.MAX_LENGTH = self._MAX_KEY_LENGTH
+        return "key"
+
+    def lengthLimitExceeded(self, length):
+        """
+        The key length limit was exceeded.  Disconnect the transport and make
+        sure a meaningful exception is reported.
+        """
+        self._keyLengthLimitExceeded = True
+        self.transport.loseConnection()
+
+    def _lockForSwitch(self):
+        """
+        Lock this binary protocol so that no further boxes may be sent.  This
+        is used when sending a request to switch underlying protocols.  You
+        probably want to subclass ProtocolSwitchCommand rather than calling
+        this directly.
+        """
+        self._locked = True
+
+    def _unlockFromSwitch(self):
+        """
+        Unlock this locked binary protocol so that further boxes may be sent
+        again.  This is used after an attempt to switch protocols has failed
+        for some reason.
+        """
+        if self.innerProtocol is not None:
+            raise ProtocolSwitched("Protocol already switched.  Cannot unlock.")
+        self._locked = False
+
+    def _prepareTLS(self, certificate, verifyAuthorities):
+        """
+        Used by StartTLSCommand to put us into the state where we don't
+        actually send things that get sent, instead we buffer them.  see
+        L{_sendBoxCommand}.
+        """
+        self._startingTLSBuffer = []
+        if self.hostCertificate is not None:
+            raise OnlyOneTLS(
+                "Previously authenticated connection between %s and %s "
+                "is trying to re-establish as %s"
+                % (
+                    self.hostCertificate,
+                    self.peerCertificate,
+                    (certificate, verifyAuthorities),
+                )
+            )
+
+    def _startTLS(self, certificate, verifyAuthorities):
+        """
+        Used by TLSBox to initiate the SSL handshake.
+
+        @param certificate: a L{twisted.internet.ssl.PrivateCertificate} for
+        use locally.
+
+        @param verifyAuthorities: L{twisted.internet.ssl.Certificate} instances
+        representing certificate authorities which will verify our peer.
+        """
+        self.hostCertificate = certificate
+        self._justStartedTLS = True
+        if verifyAuthorities is None:
+            verifyAuthorities = ()
+        self.transport.startTLS(certificate.options(*verifyAuthorities))
+        stlsb = self._startingTLSBuffer
+        if stlsb is not None:
+            self._startingTLSBuffer = None
+            for box in stlsb:
+                self.sendBox(box)
+
+    @property
+    def peerCertificate(self):
+        if self.noPeerCertificate:
+            return None
+        return Certificate.peerFromTransport(self.transport)
+
+    def unhandledError(self, failure):
+        """
+        The buck stops here.  This error was completely unhandled, time to
+        terminate the connection.
+        """
+        log.err(
+            failure,
+            "Amp server or network failure unhandled by client application.  "
+            "Dropping connection!  To avoid, add errbacks to ALL remote "
+            "commands!",
+        )
+        if self.transport is not None:
+            self.transport.loseConnection()
+
+    def _defaultStartTLSResponder(self):
+        """
+        The default TLS responder doesn't specify any certificate or anything.
+
+        From a security perspective, it's little better than a plain-text
+        connection - but it is still a *bit* better, so it's included for
+        convenience.
+
+        You probably want to override this by providing your own StartTLS.responder.
+        """
+        return {}
+
+    StartTLS.responder(_defaultStartTLSResponder)
+
+
+class AMP(BinaryBoxProtocol, BoxDispatcher, CommandLocator, SimpleStringLocator):
+    """
+    This protocol is an AMP connection.  See the module docstring for protocol
+    details.
+    """
+
+    _ampInitialized = False
+
+    def __init__(self, boxReceiver=None, locator=None):
+        # For backwards compatibility.  When AMP did not separate parsing logic
+        # (L{BinaryBoxProtocol}), request-response logic (L{BoxDispatcher}) and
+        # command routing (L{CommandLocator}), it did not have a constructor.
+        # Now it does, so old subclasses might have defined their own that did
+        # not upcall.  If this flag isn't set, we'll call the constructor in
+        # makeConnection before anything actually happens.
+        self._ampInitialized = True
+        if boxReceiver is None:
+            boxReceiver = self
+        if locator is None:
+            locator = self
+        BoxDispatcher.__init__(self, locator)
+        BinaryBoxProtocol.__init__(self, boxReceiver)
+
+    def locateResponder(self, name):
+        """
+        Unify the implementations of L{CommandLocator} and
+        L{SimpleStringLocator} to perform both kinds of dispatch, preferring
+        L{CommandLocator}.
+
+        @type name: C{bytes}
+        """
+        firstResponder = CommandLocator.locateResponder(self, name)
+        if firstResponder is not None:
+            return firstResponder
+        secondResponder = SimpleStringLocator.locateResponder(self, name)
+        return secondResponder
+
+    def __repr__(self) -> str:
+        """
+        A verbose string representation which gives us information about this
+        AMP connection.
+        """
+        if self.innerProtocol is not None:
+            innerRepr = f" inner {self.innerProtocol!r}"
+        else:
+            innerRepr = ""
+        return f"<{self.__class__.__name__}{innerRepr} at 0x{id(self):x}>"
+
+    def makeConnection(self, transport):
+        """
+        Emit a helpful log message when the connection is made.
+        """
+        if not self._ampInitialized:
+            # See comment in the constructor re: backward compatibility.  I
+            # should probably emit a deprecation warning here.
+            AMP.__init__(self)
+        # Save these so we can emit a similar log message in L{connectionLost}.
+        self._transportPeer = transport.getPeer()
+        self._transportHost = transport.getHost()
+        log.msg(
+            "%s connection established (HOST:%s PEER:%s)"
+            % (self.__class__.__name__, self._transportHost, self._transportPeer)
+        )
+        BinaryBoxProtocol.makeConnection(self, transport)
+
+    def connectionLost(self, reason):
+        """
+        Emit a helpful log message when the connection is lost.
+        """
+        log.msg(
+            "%s connection lost (HOST:%s PEER:%s)"
+            % (self.__class__.__name__, self._transportHost, self._transportPeer)
+        )
+        BinaryBoxProtocol.connectionLost(self, reason)
+        self.transport = None
+
+
+class _ParserHelper:
+    """
+    A box receiver which records all boxes received.
+    """
+
+    def __init__(self):
+        self.boxes = []
+
+    def getPeer(self):
+        return "string"
+
+    def getHost(self):
+        return "string"
+
+    disconnecting = False
+
+    def startReceivingBoxes(self, sender):
+        """
+        No initialization is required.
+        """
+
+    def ampBoxReceived(self, box):
+        self.boxes.append(box)
+
+    # Synchronous helpers
+    @classmethod
+    def parse(cls, fileObj):
+        """
+        Parse some amp data stored in a file.
+
+        @param fileObj: a file-like object.
+
+        @return: a list of AmpBoxes encoded in the given file.
+        """
+        parserHelper = cls()
+        bbp = BinaryBoxProtocol(boxReceiver=parserHelper)
+        bbp.makeConnection(parserHelper)
+        bbp.dataReceived(fileObj.read())
+        return parserHelper.boxes
+
+    @classmethod
+    def parseString(cls, data):
+        """
+        Parse some amp data stored in a string.
+
+        @param data: a str holding some amp-encoded data.
+
+        @return: a list of AmpBoxes encoded in the given string.
+        """
+        return cls.parse(BytesIO(data))
+
+
+parse = _ParserHelper.parse
+parseString = _ParserHelper.parseString
+
+
+def _stringsToObjects(strings, arglist, proto):
+    """
+    Convert an AmpBox to a dictionary of python objects, converting through a
+    given arglist.
+
+    @param strings: an AmpBox (or dict of strings)
+
+    @param arglist: a list of 2-tuples of strings and Argument objects, as
+    described in L{Command.arguments}.
+
+    @param proto: an L{AMP} instance.
+
+    @return: the converted dictionary mapping names to argument objects.
+    """
+    objects = {}
+    myStrings = strings.copy()
+    for argname, argparser in arglist:
+        argparser.fromBox(argname, myStrings, objects, proto)
+    return objects
+
+
+def _objectsToStrings(objects, arglist, strings, proto):
+    """
+    Convert a dictionary of python objects to an AmpBox, converting through a
+    given arglist.
+
+    @param objects: a dict mapping names to python objects
+
+    @param arglist: a list of 2-tuples of strings and Argument objects, as
+    described in L{Command.arguments}.
+
+    @param strings: [OUT PARAMETER] An object providing the L{dict}
+    interface which will be populated with serialized data.
+
+    @param proto: an L{AMP} instance.
+
+    @return: The converted dictionary mapping names to encoded argument
+    strings (identical to C{strings}).
+    """
+    myObjects = objects.copy()
+    for argname, argparser in arglist:
+        argparser.toBox(argname, strings, myObjects, proto)
+    return strings
+
+
+class Decimal(Argument):
+    """
+    Encodes C{decimal.Decimal} instances.
+
+    There are several ways in which a decimal value might be encoded.
+
+    Special values are encoded as special strings::
+
+      - Positive infinity is encoded as C{"Infinity"}
+      - Negative infinity is encoded as C{"-Infinity"}
+      - Quiet not-a-number is encoded as either C{"NaN"} or C{"-NaN"}
+      - Signalling not-a-number is encoded as either C{"sNaN"} or C{"-sNaN"}
+
+    Normal values are encoded using the base ten string representation, using
+    engineering notation to indicate magnitude without precision, and "normal"
+    digits to indicate precision.  For example::
+
+      - C{"1"} represents the value I{1} with precision to one place.
+      - C{"-1"} represents the value I{-1} with precision to one place.
+      - C{"1.0"} represents the value I{1} with precision to two places.
+      - C{"10"} represents the value I{10} with precision to two places.
+      - C{"1E+2"} represents the value I{10} with precision to one place.
+      - C{"1E-1"} represents the value I{0.1} with precision to one place.
+      - C{"1.5E+2"} represents the value I{15} with precision to two places.
+
+    U{http://speleotrove.com/decimal/} should be considered the authoritative
+    specification for the format.
+    """
+
+    def fromString(self, inString):
+        inString = nativeString(inString)
+        return decimal.Decimal(inString)
+
+    def toString(self, inObject):
+        """
+        Serialize a C{decimal.Decimal} instance to the specified wire format.
+        """
+        if isinstance(inObject, decimal.Decimal):
+            # Hopefully decimal.Decimal.__str__ actually does what we want.
+            return str(inObject).encode("ascii")
+        raise ValueError("amp.Decimal can only encode instances of decimal.Decimal")
+
+
+class DateTime(Argument):
+    """
+    Encodes C{datetime.datetime} instances.
+
+    Wire format: '%04i-%02i-%02iT%02i:%02i:%02i.%06i%s%02i:%02i'. Fields in
+    order are: year, month, day, hour, minute, second, microsecond, timezone
+    direction (+ or -), timezone hour, timezone minute. Encoded string is
+    always exactly 32 characters long. This format is compatible with ISO 8601,
+    but that does not mean all ISO 8601 dates can be accepted.
+
+    Also, note that the datetime module's notion of a "timezone" can be
+    complex, but the wire format includes only a fixed offset, so the
+    conversion is not lossless. A lossless transmission of a C{datetime} instance
+    is not feasible since the receiving end would require a Python interpreter.
+
+    @ivar _positions: A sequence of slices giving the positions of various
+        interesting parts of the wire format.
+    """
+
+    _positions = [
+        slice(0, 4),
+        slice(5, 7),
+        slice(8, 10),  # year, month, day
+        slice(11, 13),
+        slice(14, 16),
+        slice(17, 19),  # hour, minute, second
+        slice(20, 26),  # microsecond
+        # intentionally skip timezone direction, as it is not an integer
+        slice(27, 29),
+        slice(30, 32),  # timezone hour, timezone minute
+    ]
+
+    def fromString(self, s):
+        """
+        Parse a string containing a date and time in the wire format into a
+        C{datetime.datetime} instance.
+        """
+        s = nativeString(s)
+
+        if len(s) != 32:
+            raise ValueError(f"invalid date format {s!r}")
+
+        values = [int(s[p]) for p in self._positions]
+        sign = s[26]
+        timezone = _FixedOffsetTZInfo.fromSignHoursMinutes(sign, *values[7:])
+        values[7:] = [timezone]
+        return datetime.datetime(*values)
+
+    def toString(self, i):
+        """
+        Serialize a C{datetime.datetime} instance to a string in the specified
+        wire format.
+        """
+        offset = i.utcoffset()
+        if offset is None:
+            raise ValueError(
+                "amp.DateTime cannot serialize naive datetime instances.  "
+                "You may find amp.utc useful."
+            )
+
+        minutesOffset = (offset.days * 86400 + offset.seconds) // 60
+
+        if minutesOffset > 0:
+            sign = "+"
+        else:
+            sign = "-"
+
+        # strftime has no way to format the microseconds, or put a ':' in the
+        # timezone. Surprise!
+
+        # Python 3.4 cannot do % interpolation on byte strings so we pack into
+        # an explicitly Unicode string then encode as ASCII.
+        packed = "%04i-%02i-%02iT%02i:%02i:%02i.%06i%s%02i:%02i" % (
+            i.year,
+            i.month,
+            i.day,
+            i.hour,
+            i.minute,
+            i.second,
+            i.microsecond,
+            sign,
+            abs(minutesOffset) // 60,
+            abs(minutesOffset) % 60,
+        )
+
+        return packed.encode("ascii")
diff --git a/contrib/python/Twisted/py3/twisted/protocols/basic.py b/contrib/python/Twisted/py3/twisted/protocols/basic.py
new file mode 100644
index 00000000000..399dea51b3e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/basic.py
@@ -0,0 +1,912 @@
+# -*- test-case-name: twisted.protocols.test.test_basic -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Basic protocols, such as line-oriented, netstring, and int prefixed strings.
+"""
+
+
+import math
+
+# System imports
+import re
+from io import BytesIO
+from struct import calcsize, pack, unpack
+
+from zope.interface import implementer
+
+# Twisted imports
+from twisted.internet import defer, interfaces, protocol
+from twisted.python import log
+
+
+# Unfortunately we cannot use regular string formatting on Python 3; see
+# http://bugs.python.org/issue3982 for details.
+def _formatNetstring(data):
+    return b"".join([str(len(data)).encode("ascii"), b":", data, b","])
+
+
+_formatNetstring.__doc__ = """
+Convert some C{bytes} into netstring format.
+
+@param data: C{bytes} that will be reformatted.
+"""
+
+
+DEBUG = 0
+
+
+class NetstringParseError(ValueError):
+    """
+    The incoming data is not in valid Netstring format.
+    """
+
+
+class IncompleteNetstring(Exception):
+    """
+    Not enough data to complete a netstring.
+    """
+
+
+class NetstringReceiver(protocol.Protocol):
+    """
+    A protocol that sends and receives netstrings.
+
+    See U{http://cr.yp.to/proto/netstrings.txt} for the specification of
+    netstrings. Every netstring starts with digits that specify the length
+    of the data. This length specification is separated from the data by
+    a colon. The data is terminated with a comma.
+
+    Override L{stringReceived} to handle received netstrings. This
+    method is called with the netstring payload as a single argument
+    whenever a complete netstring is received.
+
+    Security features:
+        1. Messages are limited in size, useful if you don't want
+           someone sending you a 500MB netstring (change C{self.MAX_LENGTH}
+           to the maximum length you wish to accept).
+        2. The connection is lost if an illegal message is received.
+
+    @ivar MAX_LENGTH: Defines the maximum length of netstrings that can be
+        received.
+    @type MAX_LENGTH: C{int}
+
+    @ivar _LENGTH: A pattern describing all strings that contain a netstring
+        length specification. Examples for length specifications are C{b'0:'},
+        C{b'12:'}, and C{b'179:'}. C{b'007:'} is not a valid length
+        specification, since leading zeros are not allowed.
+    @type _LENGTH: C{re.Match}
+
+    @ivar _LENGTH_PREFIX: A pattern describing all strings that contain
+        the first part of a netstring length specification (without the
+        trailing comma). Examples are '0', '12', and '179'. '007' does not
+        start a netstring length specification, since leading zeros are
+        not allowed.
+    @type _LENGTH_PREFIX: C{re.Match}
+
+    @ivar _PARSING_LENGTH: Indicates that the C{NetstringReceiver} is in
+        the state of parsing the length portion of a netstring.
+    @type _PARSING_LENGTH: C{int}
+
+    @ivar _PARSING_PAYLOAD: Indicates that the C{NetstringReceiver} is in
+        the state of parsing the payload portion (data and trailing comma)
+        of a netstring.
+    @type _PARSING_PAYLOAD: C{int}
+
+    @ivar brokenPeer: Indicates if the connection is still functional
+    @type brokenPeer: C{int}
+
+    @ivar _state: Indicates if the protocol is consuming the length portion
+        (C{PARSING_LENGTH}) or the payload (C{PARSING_PAYLOAD}) of a netstring
+    @type _state: C{int}
+
+    @ivar _remainingData: Holds the chunk of data that has not yet been consumed
+    @type _remainingData: C{string}
+
+    @ivar _payload: Holds the payload portion of a netstring including the
+        trailing comma
+    @type _payload: C{BytesIO}
+
+    @ivar _expectedPayloadSize: Holds the payload size plus one for the trailing
+        comma.
+    @type _expectedPayloadSize: C{int}
+    """
+
+    MAX_LENGTH = 99999
+    _LENGTH = re.compile(rb"(0|[1-9]\d*)(:)")
+
+    _LENGTH_PREFIX = re.compile(rb"(0|[1-9]\d*)$")
+
+    # Some error information for NetstringParseError instances.
+    _MISSING_LENGTH = (
+        "The received netstring does not start with a " "length specification."
+    )
+    _OVERFLOW = (
+        "The length specification of the received netstring "
+        "cannot be represented in Python - it causes an "
+        "OverflowError!"
+    )
+    _TOO_LONG = (
+        "The received netstring is longer than the maximum %s "
+        "specified by self.MAX_LENGTH"
+    )
+    _MISSING_COMMA = "The received netstring is not terminated by a comma."
+
+    # The following constants are used for determining if the NetstringReceiver
+    # is parsing the length portion of a netstring, or the payload.
+    _PARSING_LENGTH, _PARSING_PAYLOAD = range(2)
+
+    def makeConnection(self, transport):
+        """
+        Initializes the protocol.
+        """
+        protocol.Protocol.makeConnection(self, transport)
+        self._remainingData = b""
+        self._currentPayloadSize = 0
+        self._payload = BytesIO()
+        self._state = self._PARSING_LENGTH
+        self._expectedPayloadSize = 0
+        self.brokenPeer = 0
+
+    def sendString(self, string):
+        """
+        Sends a netstring.
+
+        Wraps up C{string} by adding length information and a
+        trailing comma; writes the result to the transport.
+
+        @param string: The string to send.  The necessary framing (length
+            prefix, etc) will be added.
+        @type string: C{bytes}
+        """
+        self.transport.write(_formatNetstring(string))
+
+    def dataReceived(self, data):
+        """
+        Receives some characters of a netstring.
+
+        Whenever a complete netstring is received, this method extracts
+        its payload and calls L{stringReceived} to process it.
+
+        @param data: A chunk of data representing a (possibly partial)
+            netstring
+        @type data: C{bytes}
+        """
+        self._remainingData += data
+        while self._remainingData:
+            try:
+                self._consumeData()
+            except IncompleteNetstring:
+                break
+            except NetstringParseError:
+                self._handleParseError()
+                break
+
+    def stringReceived(self, string):
+        """
+        Override this for notification when each complete string is received.
+
+        @param string: The complete string which was received with all
+            framing (length prefix, etc) removed.
+        @type string: C{bytes}
+
+        @raise NotImplementedError: because the method has to be implemented
+            by the child class.
+        """
+        raise NotImplementedError()
+
+    def _maxLengthSize(self):
+        """
+        Calculate and return the string size of C{self.MAX_LENGTH}.
+
+        @return: The size of the string representation for C{self.MAX_LENGTH}
+        @rtype: C{float}
+        """
+        return math.ceil(math.log10(self.MAX_LENGTH)) + 1
+
+    def _consumeData(self):
+        """
+        Consumes the content of C{self._remainingData}.
+
+        @raise IncompleteNetstring: if C{self._remainingData} does not
+            contain enough data to complete the current netstring.
+        @raise NetstringParseError: if the received data do not
+            form a valid netstring.
+        """
+        if self._state == self._PARSING_LENGTH:
+            self._consumeLength()
+            self._prepareForPayloadConsumption()
+        if self._state == self._PARSING_PAYLOAD:
+            self._consumePayload()
+
+    def _consumeLength(self):
+        """
+        Consumes the length portion of C{self._remainingData}.
+
+        @raise IncompleteNetstring: if C{self._remainingData} contains
+            a partial length specification (digits without trailing
+            comma).
+        @raise NetstringParseError: if the received data do not form a valid
+            netstring.
+        """
+        lengthMatch = self._LENGTH.match(self._remainingData)
+        if not lengthMatch:
+            self._checkPartialLengthSpecification()
+            raise IncompleteNetstring()
+        self._processLength(lengthMatch)
+
+    def _checkPartialLengthSpecification(self):
+        """
+        Makes sure that the received data represents a valid number.
+
+        Checks if C{self._remainingData} represents a number smaller or
+        equal to C{self.MAX_LENGTH}.
+
+        @raise NetstringParseError: if C{self._remainingData} is no
+            number or is too big (checked by L{_extractLength}).
+        """
+        partialLengthMatch = self._LENGTH_PREFIX.match(self._remainingData)
+        if not partialLengthMatch:
+            raise NetstringParseError(self._MISSING_LENGTH)
+        lengthSpecification = partialLengthMatch.group(1)
+        self._extractLength(lengthSpecification)
+
+    def _processLength(self, lengthMatch):
+        """
+        Processes the length definition of a netstring.
+
+        Extracts and stores in C{self._expectedPayloadSize} the number
+        representing the netstring size.  Removes the prefix
+        representing the length specification from
+        C{self._remainingData}.
+
+        @raise NetstringParseError: if the received netstring does not
+            start with a number or the number is bigger than
+            C{self.MAX_LENGTH}.
+        @param lengthMatch: A regular expression match object matching
+            a netstring length specification
+        @type lengthMatch: C{re.Match}
+        """
+        endOfNumber = lengthMatch.end(1)
+        startOfData = lengthMatch.end(2)
+        lengthString = self._remainingData[:endOfNumber]
+        # Expect payload plus trailing comma:
+        self._expectedPayloadSize = self._extractLength(lengthString) + 1
+        self._remainingData = self._remainingData[startOfData:]
+
+    def _extractLength(self, lengthAsString):
+        """
+        Attempts to extract the length information of a netstring.
+
+        @raise NetstringParseError: if the number is bigger than
+            C{self.MAX_LENGTH}.
+        @param lengthAsString: A chunk of data starting with a length
+            specification
+        @type lengthAsString: C{bytes}
+        @return: The length of the netstring
+        @rtype: C{int}
+        """
+        self._checkStringSize(lengthAsString)
+        length = int(lengthAsString)
+        if length > self.MAX_LENGTH:
+            raise NetstringParseError(self._TOO_LONG % (self.MAX_LENGTH,))
+        return length
+
+    def _checkStringSize(self, lengthAsString):
+        """
+        Checks the sanity of lengthAsString.
+
+        Checks if the size of the length specification exceeds the
+        size of the string representing self.MAX_LENGTH. If this is
+        not the case, the number represented by lengthAsString is
+        certainly bigger than self.MAX_LENGTH, and a
+        NetstringParseError can be raised.
+
+        This method should make sure that netstrings with extremely
+        long length specifications are refused before even attempting
+        to convert them to an integer (which might trigger a
+        MemoryError).
+        """
+        if len(lengthAsString) > self._maxLengthSize():
+            raise NetstringParseError(self._TOO_LONG % (self.MAX_LENGTH,))
+
+    def _prepareForPayloadConsumption(self):
+        """
+        Sets up variables necessary for consuming the payload of a netstring.
+        """
+        self._state = self._PARSING_PAYLOAD
+        self._currentPayloadSize = 0
+        self._payload.seek(0)
+        self._payload.truncate()
+
+    def _consumePayload(self):
+        """
+        Consumes the payload portion of C{self._remainingData}.
+
+        If the payload is complete, checks for the trailing comma and
+        processes the payload. If not, raises an L{IncompleteNetstring}
+        exception.
+
+        @raise IncompleteNetstring: if the payload received so far
+            contains fewer characters than expected.
+        @raise NetstringParseError: if the payload does not end with a
+        comma.
+        """
+        self._extractPayload()
+        if self._currentPayloadSize < self._expectedPayloadSize:
+            raise IncompleteNetstring()
+        self._checkForTrailingComma()
+        self._state = self._PARSING_LENGTH
+        self._processPayload()
+
+    def _extractPayload(self):
+        """
+        Extracts payload information from C{self._remainingData}.
+
+        Splits C{self._remainingData} at the end of the netstring.  The
+        first part becomes C{self._payload}, the second part is stored
+        in C{self._remainingData}.
+
+        If the netstring is not yet complete, the whole content of
+        C{self._remainingData} is moved to C{self._payload}.
+        """
+        if self._payloadComplete():
+            remainingPayloadSize = self._expectedPayloadSize - self._currentPayloadSize
+            self._payload.write(self._remainingData[:remainingPayloadSize])
+            self._remainingData = self._remainingData[remainingPayloadSize:]
+            self._currentPayloadSize = self._expectedPayloadSize
+        else:
+            self._payload.write(self._remainingData)
+            self._currentPayloadSize += len(self._remainingData)
+            self._remainingData = b""
+
+    def _payloadComplete(self):
+        """
+        Checks if enough data have been received to complete the netstring.
+
+        @return: C{True} iff the received data contain at least as many
+            characters as specified in the length section of the
+            netstring
+        @rtype: C{bool}
+        """
+        return (
+            len(self._remainingData) + self._currentPayloadSize
+            >= self._expectedPayloadSize
+        )
+
+    def _processPayload(self):
+        """
+        Processes the actual payload with L{stringReceived}.
+
+        Strips C{self._payload} of the trailing comma and calls
+        L{stringReceived} with the result.
+        """
+        self.stringReceived(self._payload.getvalue()[:-1])
+
+    def _checkForTrailingComma(self):
+        """
+        Checks if the netstring has a trailing comma at the expected position.
+
+        @raise NetstringParseError: if the last payload character is
+            anything but a comma.
+        """
+        if self._payload.getvalue()[-1:] != b",":
+            raise NetstringParseError(self._MISSING_COMMA)
+
+    def _handleParseError(self):
+        """
+        Terminates the connection and sets the flag C{self.brokenPeer}.
+        """
+        self.transport.loseConnection()
+        self.brokenPeer = 1
+
+
+class LineOnlyReceiver(protocol.Protocol):
+    """
+    A protocol that receives only lines.
+
+    This is purely a speed optimisation over LineReceiver, for the
+    cases that raw mode is known to be unnecessary.
+
+    @cvar delimiter: The line-ending delimiter to use. By default this is
+                     C{b'\\r\\n'}.
+    @cvar MAX_LENGTH: The maximum length of a line to allow (If a
+                      sent line is longer than this, the connection is dropped).
+                      Default is 16384.
+    """
+
+    _buffer = b""
+    delimiter = b"\r\n"
+    MAX_LENGTH = 16384
+
+    def dataReceived(self, data):
+        """
+        Translates bytes into lines, and calls lineReceived.
+        """
+        lines = (self._buffer + data).split(self.delimiter)
+        self._buffer = lines.pop(-1)
+        for line in lines:
+            if self.transport.disconnecting:
+                # this is necessary because the transport may be told to lose
+                # the connection by a line within a larger packet, and it is
+                # important to disregard all the lines in that packet following
+                # the one that told it to close.
+                return
+            if len(line) > self.MAX_LENGTH:
+                return self.lineLengthExceeded(line)
+            else:
+                self.lineReceived(line)
+        if len(self._buffer) > self.MAX_LENGTH:
+            return self.lineLengthExceeded(self._buffer)
+
+    def lineReceived(self, line):
+        """
+        Override this for when each line is received.
+
+        @param line: The line which was received with the delimiter removed.
+        @type line: C{bytes}
+        """
+        raise NotImplementedError
+
+    def sendLine(self, line):
+        """
+        Sends a line to the other end of the connection.
+
+        @param line: The line to send, not including the delimiter.
+        @type line: C{bytes}
+        """
+        return self.transport.writeSequence((line, self.delimiter))
+
+    def lineLengthExceeded(self, line):
+        """
+        Called when the maximum line length has been reached.
+        Override if it needs to be dealt with in some special way.
+        """
+        return self.transport.loseConnection()
+
+
+class _PauseableMixin:
+    paused = False
+
+    def pauseProducing(self):
+        self.paused = True
+        self.transport.pauseProducing()
+
+    def resumeProducing(self):
+        self.paused = False
+        self.transport.resumeProducing()
+        self.dataReceived(b"")
+
+    def stopProducing(self):
+        self.paused = True
+        self.transport.stopProducing()
+
+
+class LineReceiver(protocol.Protocol, _PauseableMixin):
+    """
+    A protocol that receives lines and/or raw data, depending on mode.
+
+    In line mode, each line that's received becomes a callback to
+    L{lineReceived}.  In raw data mode, each chunk of raw data becomes a
+    callback to L{LineReceiver.rawDataReceived}.
+    The L{setLineMode} and L{setRawMode} methods switch between the two modes.
+
+    This is useful for line-oriented protocols such as IRC, HTTP, POP, etc.
+
+    @cvar delimiter: The line-ending delimiter to use. By default this is
+                     C{b'\\r\\n'}.
+    @cvar MAX_LENGTH: The maximum length of a line to allow (If a
+                      sent line is longer than this, the connection is dropped).
+                      Default is 16384.
+    """
+
+    line_mode = 1
+    _buffer = b""
+    _busyReceiving = False
+    delimiter = b"\r\n"
+    MAX_LENGTH = 16384
+
+    def clearLineBuffer(self):
+        """
+        Clear buffered data.
+
+        @return: All of the cleared buffered data.
+        @rtype: C{bytes}
+        """
+        b, self._buffer = self._buffer, b""
+        return b
+
+    def dataReceived(self, data):
+        """
+        Protocol.dataReceived.
+        Translates bytes into lines, and calls lineReceived (or
+        rawDataReceived, depending on mode.)
+        """
+        if self._busyReceiving:
+            self._buffer += data
+            return
+
+        try:
+            self._busyReceiving = True
+            self._buffer += data
+            while self._buffer and not self.paused:
+                if self.line_mode:
+                    try:
+                        line, self._buffer = self._buffer.split(self.delimiter, 1)
+                    except ValueError:
+                        if len(self._buffer) >= (self.MAX_LENGTH + len(self.delimiter)):
+                            line, self._buffer = self._buffer, b""
+                            return self.lineLengthExceeded(line)
+                        return
+                    else:
+                        lineLength = len(line)
+                        if lineLength > self.MAX_LENGTH:
+                            exceeded = line + self.delimiter + self._buffer
+                            self._buffer = b""
+                            return self.lineLengthExceeded(exceeded)
+                        why = self.lineReceived(line)
+                        if why or self.transport and self.transport.disconnecting:
+                            return why
+                else:
+                    data = self._buffer
+                    self._buffer = b""
+                    why = self.rawDataReceived(data)
+                    if why:
+                        return why
+        finally:
+            self._busyReceiving = False
+
+    def setLineMode(self, extra=b""):
+        """
+        Sets the line-mode of this receiver.
+
+        If you are calling this from a rawDataReceived callback,
+        you can pass in extra unhandled data, and that data will
+        be parsed for lines.  Further data received will be sent
+        to lineReceived rather than rawDataReceived.
+
+        Do not pass extra data if calling this function from
+        within a lineReceived callback.
+        """
+        self.line_mode = 1
+        if extra:
+            return self.dataReceived(extra)
+
+    def setRawMode(self):
+        """
+        Sets the raw mode of this receiver.
+        Further data received will be sent to rawDataReceived rather
+        than lineReceived.
+        """
+        self.line_mode = 0
+
+    def rawDataReceived(self, data):
+        """
+        Override this for when raw data is received.
+        """
+        raise NotImplementedError
+
+    def lineReceived(self, line):
+        """
+        Override this for when each line is received.
+
+        @param line: The line which was received with the delimiter removed.
+        @type line: C{bytes}
+        """
+        raise NotImplementedError
+
+    def sendLine(self, line):
+        """
+        Sends a line to the other end of the connection.
+
+        @param line: The line to send, not including the delimiter.
+        @type line: C{bytes}
+        """
+        return self.transport.write(line + self.delimiter)
+
+    def lineLengthExceeded(self, line):
+        """
+        Called when the maximum line length has been reached.
+        Override if it needs to be dealt with in some special way.
+
+        The argument 'line' contains the remainder of the buffer, starting
+        with (at least some part) of the line which is too long. This may
+        be more than one line, or may be only the initial portion of the
+        line.
+        """
+        return self.transport.loseConnection()
+
+
+class StringTooLongError(AssertionError):
+    """
+    Raised when trying to send a string too long for a length prefixed
+    protocol.
+    """
+
+
+class _RecvdCompatHack:
+    """
+    Emulates the to-be-deprecated C{IntNStringReceiver.recvd} attribute.
+
+    The C{recvd} attribute was where the working buffer for buffering and
+    parsing netstrings was kept.  It was updated each time new data arrived and
+    each time some of that data was parsed and delivered to application code.
+    The piecemeal updates to its string value were expensive and have been
+    removed from C{IntNStringReceiver} in the normal case.  However, for
+    applications directly reading this attribute, this descriptor restores that
+    behavior.  It only copies the working buffer when necessary (ie, when
+    accessed).  This avoids the cost for applications not using the data.
+
+    This is a custom descriptor rather than a property, because we still need
+    the default __set__ behavior in both new-style and old-style subclasses.
+    """
+
+    def __get__(self, oself, type=None):
+        return oself._unprocessed[oself._compatibilityOffset :]
+
+
+class IntNStringReceiver(protocol.Protocol, _PauseableMixin):
+    """
+    Generic class for length prefixed protocols.
+
+    @ivar _unprocessed: bytes received, but not yet broken up into messages /
+        sent to stringReceived.  _compatibilityOffset must be updated when this
+        value is updated so that the C{recvd} attribute can be generated
+        correctly.
+    @type _unprocessed: C{bytes}
+
+    @ivar structFormat: format used for struct packing/unpacking. Define it in
+        subclass.
+    @type structFormat: C{str}
+
+    @ivar prefixLength: length of the prefix, in bytes. Define it in subclass,
+        using C{struct.calcsize(structFormat)}
+    @type prefixLength: C{int}
+
+    @ivar _compatibilityOffset: the offset within C{_unprocessed} to the next
+        message to be parsed. (used to generate the recvd attribute)
+    @type _compatibilityOffset: C{int}
+    """
+
+    MAX_LENGTH = 99999
+    _unprocessed = b""
+    _compatibilityOffset = 0
+
+    # Backwards compatibility support for applications which directly touch the
+    # "internal" parse buffer.
+    recvd = _RecvdCompatHack()
+
+    def stringReceived(self, string):
+        """
+        Override this for notification when each complete string is received.
+
+        @param string: The complete string which was received with all
+            framing (length prefix, etc) removed.
+        @type string: C{bytes}
+        """
+        raise NotImplementedError
+
+    def lengthLimitExceeded(self, length):
+        """
+        Callback invoked when a length prefix greater than C{MAX_LENGTH} is
+        received.  The default implementation disconnects the transport.
+        Override this.
+
+        @param length: The length prefix which was received.
+        @type length: C{int}
+        """
+        self.transport.loseConnection()
+
+    def dataReceived(self, data):
+        """
+        Convert int prefixed strings into calls to stringReceived.
+        """
+        # Try to minimize string copying (via slices) by keeping one buffer
+        # containing all the data we have so far and a separate offset into that
+        # buffer.
+        alldata = self._unprocessed + data
+        currentOffset = 0
+        prefixLength = self.prefixLength
+        fmt = self.structFormat
+        self._unprocessed = alldata
+
+        while len(alldata) >= (currentOffset + prefixLength) and not self.paused:
+            messageStart = currentOffset + prefixLength
+            (length,) = unpack(fmt, alldata[currentOffset:messageStart])
+            if length > self.MAX_LENGTH:
+                self._unprocessed = alldata
+                self._compatibilityOffset = currentOffset
+                self.lengthLimitExceeded(length)
+                return
+            messageEnd = messageStart + length
+            if len(alldata) < messageEnd:
+                break
+
+            # Here we have to slice the working buffer so we can send just the
+            # netstring into the stringReceived callback.
+            packet = alldata[messageStart:messageEnd]
+            currentOffset = messageEnd
+            self._compatibilityOffset = currentOffset
+            self.stringReceived(packet)
+
+            # Check to see if the backwards compat "recvd" attribute got written
+            # to by application code.  If so, drop the current data buffer and
+            # switch to the new buffer given by that attribute's value.
+            if "recvd" in self.__dict__:
+                alldata = self.__dict__.pop("recvd")
+                self._unprocessed = alldata
+                self._compatibilityOffset = currentOffset = 0
+                if alldata:
+                    continue
+                return
+
+        # Slice off all the data that has been processed, avoiding holding onto
+        # memory to store it, and update the compatibility attributes to reflect
+        # that change.
+        self._unprocessed = alldata[currentOffset:]
+        self._compatibilityOffset = 0
+
+    def sendString(self, string):
+        """
+        Send a prefixed string to the other end of the connection.
+
+        @param string: The string to send.  The necessary framing (length
+            prefix, etc) will be added.
+        @type string: C{bytes}
+        """
+        if len(string) >= 2 ** (8 * self.prefixLength):
+            raise StringTooLongError(
+                "Try to send %s bytes whereas maximum is %s"
+                % (len(string), 2 ** (8 * self.prefixLength))
+            )
+        self.transport.write(pack(self.structFormat, len(string)) + string)
+
+
+class Int32StringReceiver(IntNStringReceiver):
+    """
+    A receiver for int32-prefixed strings.
+
+    An int32 string is a string prefixed by 4 bytes, the 32-bit length of
+    the string encoded in network byte order.
+
+    This class publishes the same interface as NetstringReceiver.
+    """
+
+    structFormat = "!I"
+    prefixLength = calcsize(structFormat)
+
+
+class Int16StringReceiver(IntNStringReceiver):
+    """
+    A receiver for int16-prefixed strings.
+
+    An int16 string is a string prefixed by 2 bytes, the 16-bit length of
+    the string encoded in network byte order.
+
+    This class publishes the same interface as NetstringReceiver.
+    """
+
+    structFormat = "!H"
+    prefixLength = calcsize(structFormat)
+
+
+class Int8StringReceiver(IntNStringReceiver):
+    """
+    A receiver for int8-prefixed strings.
+
+    An int8 string is a string prefixed by 1 byte, the 8-bit length of
+    the string.
+
+    This class publishes the same interface as NetstringReceiver.
+    """
+
+    structFormat = "!B"
+    prefixLength = calcsize(structFormat)
+
+
+class StatefulStringProtocol:
+    """
+    A stateful string protocol.
+
+    This is a mixin for string protocols (L{Int32StringReceiver},
+    L{NetstringReceiver}) which translates L{stringReceived} into a callback
+    (prefixed with C{'proto_'}) depending on state.
+
+    The state C{'done'} is special; if a C{proto_*} method returns it, the
+    connection will be closed immediately.
+
+    @ivar state: Current state of the protocol. Defaults to C{'init'}.
+    @type state: C{str}
+    """
+
+    state = "init"
+
+    def stringReceived(self, string):
+        """
+        Choose a protocol phase function and call it.
+
+        Call back to the appropriate protocol phase; this begins with
+        the function C{proto_init} and moves on to C{proto_*} depending on
+        what each C{proto_*} function returns.  (For example, if
+        C{self.proto_init} returns 'foo', then C{self.proto_foo} will be the
+        next function called when a protocol message is received.
+        """
+        try:
+            pto = "proto_" + self.state
+            statehandler = getattr(self, pto)
+        except AttributeError:
+            log.msg("callback", self.state, "not found")
+        else:
+            self.state = statehandler(string)
+            if self.state == "done":
+                self.transport.loseConnection()
+
+
+@implementer(interfaces.IProducer)
+class FileSender:
+    """
+    A producer that sends the contents of a file to a consumer.
+
+    This is a helper for protocols that, at some point, will take a
+    file-like object, read its contents, and write them out to the network,
+    optionally performing some transformation on the bytes in between.
+    """
+
+    CHUNK_SIZE = 2**14
+
+    lastSent = ""
+    deferred = None
+
+    def beginFileTransfer(self, file, consumer, transform=None):
+        """
+        Begin transferring a file
+
+        @type file: Any file-like object
+        @param file: The file object to read data from
+
+        @type consumer: Any implementor of IConsumer
+        @param consumer: The object to write data to
+
+        @param transform: A callable taking one string argument and returning
+        the same.  All bytes read from the file are passed through this before
+        being written to the consumer.
+
+        @rtype: C{Deferred}
+        @return: A deferred whose callback will be invoked when the file has
+        been completely written to the consumer. The last byte written to the
+        consumer is passed to the callback.
+        """
+        self.file = file
+        self.consumer = consumer
+        self.transform = transform
+
+        self.deferred = deferred = defer.Deferred()
+        self.consumer.registerProducer(self, False)
+        return deferred
+
+    def resumeProducing(self):
+        chunk = ""
+        if self.file:
+            chunk = self.file.read(self.CHUNK_SIZE)
+        if not chunk:
+            self.file = None
+            self.consumer.unregisterProducer()
+            if self.deferred:
+                self.deferred.callback(self.lastSent)
+                self.deferred = None
+            return
+
+        if self.transform:
+            chunk = self.transform(chunk)
+        self.consumer.write(chunk)
+        self.lastSent = chunk[-1:]
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        if self.deferred:
+            self.deferred.errback(Exception("Consumer asked us to stop producing"))
+            self.deferred = None
diff --git a/contrib/python/Twisted/py3/twisted/protocols/finger.py b/contrib/python/Twisted/py3/twisted/protocols/finger.py
new file mode 100644
index 00000000000..428618fda10
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/finger.py
@@ -0,0 +1,42 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""The Finger User Information Protocol (RFC 1288)"""
+
+from twisted.protocols import basic
+
+
+class Finger(basic.LineReceiver):
+    def lineReceived(self, line):
+        parts = line.split()
+        if not parts:
+            parts = [b""]
+        if len(parts) == 1:
+            slash_w = 0
+        else:
+            slash_w = 1
+        user = parts[-1]
+        if b"@" in user:
+            hostPlace = user.rfind(b"@")
+            user = user[:hostPlace]
+            host = user[hostPlace + 1 :]
+            return self.forwardQuery(slash_w, user, host)
+        if user:
+            return self.getUser(slash_w, user)
+        else:
+            return self.getDomain(slash_w)
+
+    def _refuseMessage(self, message):
+        self.transport.write(message + b"\n")
+        self.transport.loseConnection()
+
+    def forwardQuery(self, slash_w, user, host):
+        self._refuseMessage(b"Finger forwarding service denied")
+
+    def getDomain(self, slash_w):
+        self._refuseMessage(b"Finger online list denied")
+
+    def getUser(self, slash_w, user):
+        self.transport.write(b"Login: " + user + b"\n")
+        self._refuseMessage(b"No such user")
diff --git a/contrib/python/Twisted/py3/twisted/protocols/ftp.py b/contrib/python/Twisted/py3/twisted/protocols/ftp.py
new file mode 100644
index 00000000000..ad2f506c987
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/ftp.py
@@ -0,0 +1,3253 @@
+# -*- test-case-name: twisted.test.test_ftp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An FTP protocol implementation
+"""
+
+import errno
+import fnmatch
+
+# System Imports
+import os
+import re
+import stat
+import time
+
+try:
+    import grp
+    import pwd
+except ImportError:
+    pwd = grp = None  # type: ignore[assignment]
+
+from zope.interface import Interface, implementer
+
+# Twisted Imports
+from twisted import copyright
+from twisted.cred import checkers, credentials, error as cred_error, portal
+from twisted.internet import defer, error, interfaces, protocol, reactor
+from twisted.protocols import basic, policies
+from twisted.python import failure, filepath, log
+
+# constants
+# response codes
+
+RESTART_MARKER_REPLY = "100"
+SERVICE_READY_IN_N_MINUTES = "120"
+DATA_CNX_ALREADY_OPEN_START_XFR = "125"
+FILE_STATUS_OK_OPEN_DATA_CNX = "150"
+
+CMD_OK = "200.1"
+TYPE_SET_OK = "200.2"
+ENTERING_PORT_MODE = "200.3"
+CMD_NOT_IMPLMNTD_SUPERFLUOUS = "202"
+SYS_STATUS_OR_HELP_REPLY = "211.1"
+FEAT_OK = "211.2"
+DIR_STATUS = "212"
+FILE_STATUS = "213"
+HELP_MSG = "214"
+NAME_SYS_TYPE = "215"
+SVC_READY_FOR_NEW_USER = "220.1"
+WELCOME_MSG = "220.2"
+SVC_CLOSING_CTRL_CNX = "221.1"
+GOODBYE_MSG = "221.2"
+DATA_CNX_OPEN_NO_XFR_IN_PROGRESS = "225"
+CLOSING_DATA_CNX = "226.1"
+TXFR_COMPLETE_OK = "226.2"
+ENTERING_PASV_MODE = "227"
+ENTERING_EPSV_MODE = "229"
+USR_LOGGED_IN_PROCEED = "230.1"  # v1 of code 230
+GUEST_LOGGED_IN_PROCEED = "230.2"  # v2 of code 230
+REQ_FILE_ACTN_COMPLETED_OK = "250"
+PWD_REPLY = "257.1"
+MKD_REPLY = "257.2"
+
+USR_NAME_OK_NEED_PASS = "331.1"  # v1 of Code 331
+GUEST_NAME_OK_NEED_EMAIL = "331.2"  # v2 of code 331
+NEED_ACCT_FOR_LOGIN = "332"
+REQ_FILE_ACTN_PENDING_FURTHER_INFO = "350"
+
+SVC_NOT_AVAIL_CLOSING_CTRL_CNX = "421.1"
+TOO_MANY_CONNECTIONS = "421.2"
+CANT_OPEN_DATA_CNX = "425"
+CNX_CLOSED_TXFR_ABORTED = "426"
+REQ_ACTN_ABRTD_FILE_UNAVAIL = "450"
+REQ_ACTN_ABRTD_LOCAL_ERR = "451"
+REQ_ACTN_ABRTD_INSUFF_STORAGE = "452"
+
+SYNTAX_ERR = "500"
+SYNTAX_ERR_IN_ARGS = "501"
+CMD_NOT_IMPLMNTD = "502.1"
+OPTS_NOT_IMPLEMENTED = "502.2"
+BAD_CMD_SEQ = "503"
+CMD_NOT_IMPLMNTD_FOR_PARAM = "504"
+NOT_LOGGED_IN = "530.1"  # v1 of code 530 - please log in
+AUTH_FAILURE = "530.2"  # v2 of code 530 - authorization failure
+NEED_ACCT_FOR_STOR = "532"
+FILE_NOT_FOUND = "550.1"  # no such file or directory
+PERMISSION_DENIED = "550.2"  # permission denied
+ANON_USER_DENIED = "550.3"  # anonymous users can't alter filesystem
+IS_NOT_A_DIR = "550.4"  # rmd called on a path that is not a directory
+REQ_ACTN_NOT_TAKEN = "550.5"
+FILE_EXISTS = "550.6"
+IS_A_DIR = "550.7"
+PAGE_TYPE_UNK = "551"
+EXCEEDED_STORAGE_ALLOC = "552"
+FILENAME_NOT_ALLOWED = "553"
+
+
+RESPONSE = {
+    # -- 100's --
+    # TODO: this must be fixed
+    RESTART_MARKER_REPLY: "110 MARK yyyy-mmmm",
+    SERVICE_READY_IN_N_MINUTES: "120 service ready in %s minutes",
+    DATA_CNX_ALREADY_OPEN_START_XFR: "125 Data connection already open, "
+    "starting transfer",
+    FILE_STATUS_OK_OPEN_DATA_CNX: "150 File status okay; about to open "
+    "data connection.",
+    # -- 200's --
+    CMD_OK: "200 Command OK",
+    TYPE_SET_OK: "200 Type set to %s.",
+    ENTERING_PORT_MODE: "200 PORT OK",
+    CMD_NOT_IMPLMNTD_SUPERFLUOUS: "202 Command not implemented, "
+    "superfluous at this site",
+    SYS_STATUS_OR_HELP_REPLY: "211 System status reply",
+    FEAT_OK: ["211-Features:", "211 End"],
+    DIR_STATUS: "212 %s",
+    FILE_STATUS: "213 %s",
+    HELP_MSG: "214 help: %s",
+    NAME_SYS_TYPE: "215 UNIX Type: L8",
+    WELCOME_MSG: "220 %s",
+    SVC_READY_FOR_NEW_USER: "220 Service ready",
+    SVC_CLOSING_CTRL_CNX: "221 Service closing control " "connection",
+    GOODBYE_MSG: "221 Goodbye.",
+    DATA_CNX_OPEN_NO_XFR_IN_PROGRESS: "225 data connection open, no "
+    "transfer in progress",
+    CLOSING_DATA_CNX: "226 Abort successful",
+    TXFR_COMPLETE_OK: "226 Transfer Complete.",
+    ENTERING_PASV_MODE: "227 Entering Passive Mode (%s).",
+    # Where is EPSV defined in the RFCs?
+    ENTERING_EPSV_MODE: "229 Entering Extended Passive Mode " "(|||%s|).",
+    USR_LOGGED_IN_PROCEED: "230 User logged in, proceed",
+    GUEST_LOGGED_IN_PROCEED: "230 Anonymous login ok, access " "restrictions apply.",
+    # i.e. CWD completed OK
+    REQ_FILE_ACTN_COMPLETED_OK: "250 Requested File Action Completed " "OK",
+    PWD_REPLY: '257 "%s"',
+    MKD_REPLY: '257 "%s" created',
+    # -- 300's --
+    USR_NAME_OK_NEED_PASS: "331 Password required for %s.",
+    GUEST_NAME_OK_NEED_EMAIL: "331 Guest login ok, type your email "
+    "address as password.",
+    NEED_ACCT_FOR_LOGIN: "332 Need account for login.",
+    REQ_FILE_ACTN_PENDING_FURTHER_INFO: "350 Requested file action pending "
+    "further information.",
+    # -- 400's --
+    SVC_NOT_AVAIL_CLOSING_CTRL_CNX: "421 Service not available, closing "
+    "control connection.",
+    TOO_MANY_CONNECTIONS: "421 Too many users right now, try "
+    "again in a few minutes.",
+    CANT_OPEN_DATA_CNX: "425 Can't open data connection.",
+    CNX_CLOSED_TXFR_ABORTED: "426 Transfer aborted.  Data " "connection closed.",
+    REQ_ACTN_ABRTD_FILE_UNAVAIL: "450 Requested action aborted. " "File unavailable.",
+    REQ_ACTN_ABRTD_LOCAL_ERR: "451 Requested action aborted. "
+    "Local error in processing.",
+    REQ_ACTN_ABRTD_INSUFF_STORAGE: "452 Requested action aborted. "
+    "Insufficient storage.",
+    # -- 500's --
+    SYNTAX_ERR: "500 Syntax error: %s",
+    SYNTAX_ERR_IN_ARGS: "501 syntax error in argument(s) %s.",
+    CMD_NOT_IMPLMNTD: "502 Command '%s' not implemented",
+    OPTS_NOT_IMPLEMENTED: "502 Option '%s' not implemented.",
+    BAD_CMD_SEQ: "503 Incorrect sequence of commands: " "%s",
+    CMD_NOT_IMPLMNTD_FOR_PARAM: "504 Not implemented for parameter " "'%s'.",
+    NOT_LOGGED_IN: "530 Please login with USER and PASS.",
+    AUTH_FAILURE: "530 Sorry, Authentication failed.",
+    NEED_ACCT_FOR_STOR: "532 Need an account for storing " "files",
+    FILE_NOT_FOUND: "550 %s: No such file or directory.",
+    PERMISSION_DENIED: "550 %s: Permission denied.",
+    ANON_USER_DENIED: "550 Anonymous users are forbidden to " "change the filesystem",
+    IS_NOT_A_DIR: "550 Cannot rmd, %s is not a " "directory",
+    FILE_EXISTS: "550 %s: File exists",
+    IS_A_DIR: "550 %s: is a directory",
+    REQ_ACTN_NOT_TAKEN: "550 Requested action not taken: %s",
+    PAGE_TYPE_UNK: "551 Page type unknown",
+    EXCEEDED_STORAGE_ALLOC: "552 Requested file action aborted, "
+    "exceeded file storage allocation",
+    FILENAME_NOT_ALLOWED: "553 Requested action not taken, file " "name not allowed",
+}
+
+
+class InvalidPath(Exception):
+    """
+    Internal exception used to signify an error during parsing a path.
+    """
+
+
+def toSegments(cwd, path):
+    """
+    Normalize a path, as represented by a list of strings each
+    representing one segment of the path.
+    """
+    if path.startswith("/"):
+        segs = []
+    else:
+        segs = cwd[:]
+
+    for s in path.split("/"):
+        if s == "." or s == "":
+            continue
+        elif s == "..":
+            if segs:
+                segs.pop()
+            else:
+                raise InvalidPath(cwd, path)
+        elif "\0" in s or "/" in s:
+            raise InvalidPath(cwd, path)
+        else:
+            segs.append(s)
+    return segs
+
+
+def errnoToFailure(e, path):
+    """
+    Map C{OSError} and C{IOError} to standard FTP errors.
+    """
+    if e == errno.ENOENT:
+        return defer.fail(FileNotFoundError(path))
+    elif e == errno.EACCES or e == errno.EPERM:
+        return defer.fail(PermissionDeniedError(path))
+    elif e == errno.ENOTDIR:
+        return defer.fail(IsNotADirectoryError(path))
+    elif e == errno.EEXIST:
+        return defer.fail(FileExistsError(path))
+    elif e == errno.EISDIR:
+        return defer.fail(IsADirectoryError(path))
+    else:
+        return defer.fail()
+
+
+_testTranslation = fnmatch.translate("TEST")
+
+
+def _isGlobbingExpression(segments=None):
+    """
+    Helper for checking if a FTPShell `segments` contains a wildcard Unix
+    expression.
+
+    Only filename globbing is supported.
+    This means that wildcards can only be presents in the last element of
+    `segments`.
+
+    @type  segments: C{list}
+    @param segments: List of path elements as used by the FTP server protocol.
+
+    @rtype: Boolean
+    @return: True if `segments` contains a globbing expression.
+    """
+    if not segments:
+        return False
+
+    # To check that something is a glob expression, we convert it to
+    # Regular Expression.
+    # We compare it to the translation of a known non-glob expression.
+    # If the result is the same as the original expression then it contains no
+    # globbing expression.
+    globCandidate = segments[-1]
+    globTranslations = fnmatch.translate(globCandidate)
+    nonGlobTranslations = _testTranslation.replace("TEST", globCandidate, 1)
+
+    if nonGlobTranslations == globTranslations:
+        return False
+    else:
+        return True
+
+
+class FTPCmdError(Exception):
+    """
+    Generic exception for FTP commands.
+    """
+
+    def __init__(self, *msg):
+        Exception.__init__(self, *msg)
+        self.errorMessage = msg
+
+    def response(self):
+        """
+        Generate a FTP response message for this error.
+        """
+        return RESPONSE[self.errorCode] % self.errorMessage
+
+
+class FileNotFoundError(FTPCmdError):
+    """
+    Raised when trying to access a non existent file or directory.
+    """
+
+    errorCode = FILE_NOT_FOUND
+
+
+class AnonUserDeniedError(FTPCmdError):
+    """
+    Raised when an anonymous user issues a command that will alter the
+    filesystem
+    """
+
+    errorCode = ANON_USER_DENIED
+
+
+class PermissionDeniedError(FTPCmdError):
+    """
+    Raised when access is attempted to a resource to which access is
+    not allowed.
+    """
+
+    errorCode = PERMISSION_DENIED
+
+
+class IsNotADirectoryError(FTPCmdError):
+    """
+    Raised when RMD is called on a path that isn't a directory.
+    """
+
+    errorCode = IS_NOT_A_DIR
+
+
+class FileExistsError(FTPCmdError):
+    """
+    Raised when attempted to override an existing resource.
+    """
+
+    errorCode = FILE_EXISTS
+
+
+class IsADirectoryError(FTPCmdError):
+    """
+    Raised when DELE is called on a path that is a directory.
+    """
+
+    errorCode = IS_A_DIR
+
+
+class CmdSyntaxError(FTPCmdError):
+    """
+    Raised when a command syntax is wrong.
+    """
+
+    errorCode = SYNTAX_ERR
+
+
+class CmdArgSyntaxError(FTPCmdError):
+    """
+    Raised when a command is called with wrong value or a wrong number of
+    arguments.
+    """
+
+    errorCode = SYNTAX_ERR_IN_ARGS
+
+
+class CmdNotImplementedError(FTPCmdError):
+    """
+    Raised when an unimplemented command is given to the server.
+    """
+
+    errorCode = CMD_NOT_IMPLMNTD
+
+
+class CmdNotImplementedForArgError(FTPCmdError):
+    """
+    Raised when the handling of a parameter for a command is not implemented by
+    the server.
+    """
+
+    errorCode = CMD_NOT_IMPLMNTD_FOR_PARAM
+
+
+class FTPError(Exception):
+    pass
+
+
+class PortConnectionError(Exception):
+    pass
+
+
+class BadCmdSequenceError(FTPCmdError):
+    """
+    Raised when a client sends a series of commands in an illogical sequence.
+    """
+
+    errorCode = BAD_CMD_SEQ
+
+
+class AuthorizationError(FTPCmdError):
+    """
+    Raised when client authentication fails.
+    """
+
+    errorCode = AUTH_FAILURE
+
+
+def debugDeferred(self, *_):
+    log.msg("debugDeferred(): %s" % str(_), debug=True)
+
+
+# -- DTP Protocol --
+
+
+_months = [
+    None,
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec",
+]
+
+
+@implementer(interfaces.IConsumer)
+class DTP(protocol.Protocol):
+    isConnected = False
+
+    _cons = None
+    _onConnLost = None
+    _buffer = None
+    _encoding = "latin-1"
+
+    def connectionMade(self):
+        self.isConnected = True
+        self.factory.deferred.callback(None)
+        self._buffer = []
+
+    def connectionLost(self, reason):
+        self.isConnected = False
+        if self._onConnLost is not None:
+            self._onConnLost.callback(None)
+
+    def sendLine(self, line):
+        """
+        Send a line to data channel.
+
+        @param line: The line to be sent.
+        @type line: L{bytes}
+        """
+        self.transport.write(line + b"\r\n")
+
+    def _formatOneListResponse(
+        self, name, size, directory, permissions, hardlinks, modified, owner, group
+    ):
+        """
+        Helper method to format one entry's info into a text entry like:
+        'drwxrwxrwx   0 user   group   0 Jan 01  1970 filename.txt'
+
+        @param name: C{bytes} name of the entry (file or directory or link)
+        @param size: C{int} size of the entry
+        @param directory: evals to C{bool} - whether the entry is a directory
+        @param permissions: L{twisted.python.filepath.Permissions} object
+            representing that entry's permissions
+        @param hardlinks: C{int} number of hardlinks
+        @param modified: C{float} - entry's last modified time in seconds
+            since the epoch
+        @param owner: C{str} username of the owner
+        @param group: C{str} group name of the owner
+
+        @return: C{str} in the requisite format
+        """
+
+        def formatDate(mtime):
+            now = time.gmtime()
+            info = {
+                "month": _months[mtime.tm_mon],
+                "day": mtime.tm_mday,
+                "year": mtime.tm_year,
+                "hour": mtime.tm_hour,
+                "minute": mtime.tm_min,
+            }
+            if now.tm_year != mtime.tm_year:
+                return "%(month)s %(day)02d %(year)5d" % info
+            else:
+                return "%(month)s %(day)02d %(hour)02d:%(minute)02d" % info
+
+        format = (
+            "%(directory)s%(permissions)s%(hardlinks)4d "
+            "%(owner)-9s %(group)-9s %(size)15d %(date)12s "
+        )
+
+        msg = (
+            format
+            % {
+                "directory": directory and "d" or "-",
+                "permissions": permissions.shorthand(),
+                "hardlinks": hardlinks,
+                "owner": owner[:8],
+                "group": group[:8],
+                "size": size,
+                "date": formatDate(time.gmtime(modified)),
+            }
+        ).encode(self._encoding)
+        return msg + name
+
+    def sendListResponse(self, name, response):
+        self.sendLine(self._formatOneListResponse(name, *response))
+
+    # Proxy IConsumer to our transport
+    def registerProducer(self, producer, streaming):
+        return self.transport.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        self.transport.unregisterProducer()
+        self.transport.loseConnection()
+
+    def write(self, data):
+        if self.isConnected:
+            return self.transport.write(data)
+        raise Exception("Crap damn crap damn crap damn")
+
+    # Pretend to be a producer, too.
+    def _conswrite(self, bytes):
+        try:
+            self._cons.write(bytes)
+        except BaseException:
+            self._onConnLost.errback()
+
+    def dataReceived(self, bytes):
+        if self._cons is not None:
+            self._conswrite(bytes)
+        else:
+            self._buffer.append(bytes)
+
+    def _unregConsumer(self, ignored):
+        self._cons.unregisterProducer()
+        self._cons = None
+        del self._onConnLost
+        return ignored
+
+    def registerConsumer(self, cons):
+        assert self._cons is None
+        self._cons = cons
+        self._cons.registerProducer(self, True)
+        for chunk in self._buffer:
+            self._conswrite(chunk)
+        self._buffer = None
+        if self.isConnected:
+            self._onConnLost = d = defer.Deferred()
+            d.addBoth(self._unregConsumer)
+            return d
+        else:
+            self._cons.unregisterProducer()
+            self._cons = None
+            return defer.succeed(None)
+
+    def resumeProducing(self):
+        self.transport.resumeProducing()
+
+    def pauseProducing(self):
+        self.transport.pauseProducing()
+
+    def stopProducing(self):
+        self.transport.stopProducing()
+
+
+class DTPFactory(protocol.ClientFactory):
+    """
+    Client factory for I{data transfer process} protocols.
+
+    @ivar peerCheck: perform checks to make sure the ftp-pi's peer is the same
+        as the dtp's
+    @ivar pi: a reference to this factory's protocol interpreter
+
+    @ivar _state: Indicates the current state of the DTPFactory.  Initially,
+        this is L{_IN_PROGRESS}.  If the connection fails or times out, it is
+        L{_FAILED}.  If the connection succeeds before the timeout, it is
+        L{_FINISHED}.
+
+    @cvar _IN_PROGRESS: Token to signal that connection is active.
+    @type _IN_PROGRESS: L{object}.
+
+    @cvar _FAILED: Token to signal that connection has failed.
+    @type _FAILED: L{object}.
+
+    @cvar _FINISHED: Token to signal that connection was successfully closed.
+    @type _FINISHED: L{object}.
+    """
+
+    _IN_PROGRESS = object()
+    _FAILED = object()
+    _FINISHED = object()
+
+    _state = _IN_PROGRESS
+
+    # -- configuration variables --
+    peerCheck = False
+
+    # -- class variables --
+    def __init__(self, pi, peerHost=None, reactor=None):
+        """
+        Constructor
+
+        @param pi: this factory's protocol interpreter
+        @param peerHost: if peerCheck is True, this is the tuple that the
+            generated instance will use to perform security checks
+        """
+        self.pi = pi
+        self.peerHost = peerHost  # from FTP.transport.peerHost()
+        # deferred will fire when instance is connected
+        self.deferred = defer.Deferred()
+        self.delayedCall = None
+        if reactor is None:
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+    def buildProtocol(self, addr):
+        log.msg("DTPFactory.buildProtocol", debug=True)
+
+        if self._state is not self._IN_PROGRESS:
+            return None
+        self._state = self._FINISHED
+
+        self.cancelTimeout()
+        p = DTP()
+        p.factory = self
+        p.pi = self.pi
+        self.pi.dtpInstance = p
+        return p
+
+    def stopFactory(self):
+        log.msg("dtpFactory.stopFactory", debug=True)
+        self.cancelTimeout()
+
+    def timeoutFactory(self):
+        log.msg("timed out waiting for DTP connection")
+        if self._state is not self._IN_PROGRESS:
+            return
+        self._state = self._FAILED
+
+        d = self.deferred
+        self.deferred = None
+        d.errback(PortConnectionError(defer.TimeoutError("DTPFactory timeout")))
+
+    def cancelTimeout(self):
+        if self.delayedCall is not None and self.delayedCall.active():
+            log.msg("cancelling DTP timeout", debug=True)
+            self.delayedCall.cancel()
+
+    def setTimeout(self, seconds):
+        log.msg("DTPFactory.setTimeout set to %s seconds" % seconds)
+        self.delayedCall = self._reactor.callLater(seconds, self.timeoutFactory)
+
+    def clientConnectionFailed(self, connector, reason):
+        if self._state is not self._IN_PROGRESS:
+            return
+        self._state = self._FAILED
+        d = self.deferred
+        self.deferred = None
+        d.errback(PortConnectionError(reason))
+
+
+# -- FTP-PI (Protocol Interpreter) --
+
+
+class ASCIIConsumerWrapper:
+    def __init__(self, cons):
+        self.cons = cons
+        self.registerProducer = cons.registerProducer
+        self.unregisterProducer = cons.unregisterProducer
+
+        assert (
+            os.linesep == "\r\n" or len(os.linesep) == 1
+        ), "Unsupported platform (yea right like this even exists)"
+
+        if os.linesep == "\r\n":
+            self.write = cons.write
+
+    def write(self, bytes):
+        return self.cons.write(bytes.replace(os.linesep, "\r\n"))
+
+
+@implementer(interfaces.IConsumer)
+class FileConsumer:
+    """
+    A consumer for FTP input that writes data to a file.
+
+    @ivar fObj: a file object opened for writing, used to write data received.
+    @type fObj: C{file}
+    """
+
+    def __init__(self, fObj):
+        self.fObj = fObj
+
+    def registerProducer(self, producer, streaming):
+        self.producer = producer
+        assert streaming
+
+    def unregisterProducer(self):
+        self.producer = None
+        self.fObj.close()
+
+    def write(self, bytes):
+        self.fObj.write(bytes)
+
+
+class FTPOverflowProtocol(basic.LineReceiver):
+    """FTP mini-protocol for when there are too many connections."""
+
+    _encoding = "latin-1"
+
+    def connectionMade(self):
+        self.sendLine(RESPONSE[TOO_MANY_CONNECTIONS].encode(self._encoding))
+        self.transport.loseConnection()
+
+
+class FTP(basic.LineReceiver, policies.TimeoutMixin):
+    """
+    Protocol Interpreter for the File Transfer Protocol
+
+    @ivar state: The current server state.  One of L{UNAUTH},
+        L{INAUTH}, L{AUTHED}, L{RENAMING}.
+
+    @ivar shell: The connected avatar
+    @ivar binary: The transfer mode.  If false, ASCII.
+    @ivar dtpFactory: Generates a single DTP for this session
+    @ivar dtpPort: Port returned from listenTCP
+    @ivar listenFactory: A callable with the signature of
+        L{twisted.internet.interfaces.IReactorTCP.listenTCP} which will be used
+        to create Ports for passive connections (mainly for testing).
+
+    @ivar passivePortRange: iterator used as source of passive port numbers.
+    @type passivePortRange: C{iterator}
+
+    @cvar UNAUTH: Command channel is not yet authenticated.
+    @type UNAUTH: L{int}
+
+    @cvar INAUTH: Command channel is in the process of being authenticated.
+    @type INAUTH: L{int}
+
+    @cvar AUTHED: Command channel was successfully authenticated.
+    @type AUTHED: L{int}
+
+    @cvar RENAMING: Command channel is between the renaming command sequence.
+    @type RENAMING: L{int}
+    """
+
+    disconnected = False
+
+    # States an FTP can be in
+    UNAUTH, INAUTH, AUTHED, RENAMING = range(4)
+
+    # how long the DTP waits for a connection
+    dtpTimeout = 10
+
+    portal = None
+    shell = None
+    dtpFactory = None
+    dtpPort = None
+    dtpInstance = None
+    binary = True
+    PUBLIC_COMMANDS = ["FEAT", "QUIT"]
+    FEATURES = ["FEAT", "MDTM", "PASV", "SIZE", "TYPE A;I"]
+
+    passivePortRange = range(0, 1)
+
+    listenFactory = reactor.listenTCP  # type: ignore[attr-defined]
+    _encoding = "latin-1"
+
+    def reply(self, key, *args):
+        msg = RESPONSE[key] % args
+        self.sendLine(msg)
+
+    def sendLine(self, line):
+        """
+        (Private) Encodes and sends a line
+
+        @param line: L{bytes} or L{unicode}
+        """
+        if isinstance(line, str):
+            line = line.encode(self._encoding)
+        super().sendLine(line)
+
+    def connectionMade(self):
+        self.state = self.UNAUTH
+        self.setTimeout(self.timeOut)
+        self.reply(WELCOME_MSG, self.factory.welcomeMessage)
+
+    def connectionLost(self, reason):
+        # if we have a DTP protocol instance running and
+        # we lose connection to the client's PI, kill the
+        # DTP connection and close the port
+        if self.dtpFactory:
+            self.cleanupDTP()
+        self.setTimeout(None)
+        if hasattr(self.shell, "logout") and self.shell.logout is not None:
+            self.shell.logout()
+        self.shell = None
+        self.transport = None
+
+    def timeoutConnection(self):
+        self.transport.loseConnection()
+
+    def lineReceived(self, line):
+        self.resetTimeout()
+        self.pauseProducing()
+        if bytes != str:
+            line = line.decode(self._encoding)
+
+        def processFailed(err):
+            if err.check(FTPCmdError):
+                self.sendLine(err.value.response())
+            elif err.check(TypeError) and any(
+                msg in err.value.args[0]
+                for msg in ("takes exactly", "required positional argument")
+            ):
+                self.reply(SYNTAX_ERR, f"{cmd} requires an argument.")
+            else:
+                log.msg("Unexpected FTP error")
+                log.err(err)
+                self.reply(REQ_ACTN_NOT_TAKEN, "internal server error")
+
+        def processSucceeded(result):
+            if isinstance(result, tuple):
+                self.reply(*result)
+            elif result is not None:
+                self.reply(result)
+
+        def allDone(ignored):
+            if not self.disconnected:
+                self.resumeProducing()
+
+        spaceIndex = line.find(" ")
+        if spaceIndex != -1:
+            cmd = line[:spaceIndex]
+            args = (line[spaceIndex + 1 :],)
+        else:
+            cmd = line
+            args = ()
+        d = defer.maybeDeferred(self.processCommand, cmd, *args)
+        d.addCallbacks(processSucceeded, processFailed)
+        d.addErrback(log.err)
+
+        # XXX It burnsss
+        # LineReceiver doesn't let you resumeProducing inside
+        # lineReceived atm
+        from twisted.internet import reactor
+
+        reactor.callLater(0, d.addBoth, allDone)
+
+    def processCommand(self, cmd, *params):
+        def call_ftp_command(command):
+            method = getattr(self, "ftp_" + command, None)
+            if method is not None:
+                return method(*params)
+            return defer.fail(CmdNotImplementedError(command))
+
+        cmd = cmd.upper()
+
+        if cmd in self.PUBLIC_COMMANDS:
+            return call_ftp_command(cmd)
+
+        elif self.state == self.UNAUTH:
+            if cmd == "USER":
+                return self.ftp_USER(*params)
+            elif cmd == "PASS":
+                return BAD_CMD_SEQ, "USER required before PASS"
+            else:
+                return NOT_LOGGED_IN
+
+        elif self.state == self.INAUTH:
+            if cmd == "PASS":
+                return self.ftp_PASS(*params)
+            else:
+                return BAD_CMD_SEQ, "PASS required after USER"
+
+        elif self.state == self.AUTHED:
+            return call_ftp_command(cmd)
+
+        elif self.state == self.RENAMING:
+            if cmd == "RNTO":
+                return self.ftp_RNTO(*params)
+            else:
+                return BAD_CMD_SEQ, "RNTO required after RNFR"
+
+    def getDTPPort(self, factory):
+        """
+        Return a port for passive access, using C{self.passivePortRange}
+        attribute.
+        """
+        for portn in self.passivePortRange:
+            try:
+                dtpPort = self.listenFactory(portn, factory)
+            except error.CannotListenError:
+                continue
+            else:
+                return dtpPort
+        raise error.CannotListenError(
+            "", portn, f"No port available in range {self.passivePortRange}"
+        )
+
+    def ftp_USER(self, username):
+        """
+        First part of login.  Get the username the peer wants to
+        authenticate as.
+        """
+        if not username:
+            return defer.fail(CmdSyntaxError("USER requires an argument"))
+
+        self._user = username
+        self.state = self.INAUTH
+        if self.factory.allowAnonymous and self._user == self.factory.userAnonymous:
+            return GUEST_NAME_OK_NEED_EMAIL
+        else:
+            return (USR_NAME_OK_NEED_PASS, username)
+
+    # TODO: add max auth try before timeout from ip...
+    # TODO: need to implement minimal ABOR command
+
+    def ftp_PASS(self, password):
+        """
+        Second part of login.  Get the password the peer wants to
+        authenticate with.
+        """
+        if self.factory.allowAnonymous and self._user == self.factory.userAnonymous:
+            # anonymous login
+            creds = credentials.Anonymous()
+            reply = GUEST_LOGGED_IN_PROCEED
+        else:
+            # user login
+            creds = credentials.UsernamePassword(self._user, password)
+            reply = USR_LOGGED_IN_PROCEED
+        del self._user
+
+        def _cbLogin(result):
+            (interface, avatar, logout) = result
+            assert interface is IFTPShell, "The realm is busted, jerk."
+            self.shell = avatar
+            self.logout = logout
+            self.workingDirectory = []
+            self.state = self.AUTHED
+            return reply
+
+        def _ebLogin(failure):
+            failure.trap(cred_error.UnauthorizedLogin, cred_error.UnhandledCredentials)
+            self.state = self.UNAUTH
+            raise AuthorizationError
+
+        d = self.portal.login(creds, None, IFTPShell)
+        d.addCallbacks(_cbLogin, _ebLogin)
+        return d
+
+    def ftp_PASV(self):
+        """
+        Request for a passive connection
+
+        from the rfc::
+
+            This command requests the server-DTP to \"listen\" on a data port
+            (which is not its default data port) and to wait for a connection
+            rather than initiate one upon receipt of a transfer command.  The
+            response to this command includes the host and port address this
+            server is listening on.
+        """
+        # if we have a DTP port set up, lose it.
+        if self.dtpFactory is not None:
+            # cleanupDTP sets dtpFactory to none.  Later we'll do
+            # cleanup here or something.
+            self.cleanupDTP()
+        self.dtpFactory = DTPFactory(pi=self)
+        self.dtpFactory.setTimeout(self.dtpTimeout)
+        self.dtpPort = self.getDTPPort(self.dtpFactory)
+
+        host = self.transport.getHost().host
+        port = self.dtpPort.getHost().port
+        self.reply(ENTERING_PASV_MODE, encodeHostPort(host, port))
+        return self.dtpFactory.deferred.addCallback(lambda ign: None)
+
+    def ftp_PORT(self, address):
+        addr = tuple(map(int, address.split(",")))
+        ip = "%d.%d.%d.%d" % tuple(addr[:4])
+        port = addr[4] << 8 | addr[5]
+
+        # if we have a DTP port set up, lose it.
+        if self.dtpFactory is not None:
+            self.cleanupDTP()
+
+        self.dtpFactory = DTPFactory(pi=self, peerHost=self.transport.getPeer().host)
+        self.dtpFactory.setTimeout(self.dtpTimeout)
+        self.dtpPort = reactor.connectTCP(ip, port, self.dtpFactory)
+
+        def connected(ignored):
+            return ENTERING_PORT_MODE
+
+        def connFailed(err):
+            err.trap(PortConnectionError)
+            return CANT_OPEN_DATA_CNX
+
+        return self.dtpFactory.deferred.addCallbacks(connected, connFailed)
+
+    def _encodeName(self, name):
+        """
+        Encode C{name} to be sent over the wire.
+
+        This encodes L{unicode} objects as UTF-8 and leaves L{bytes} as-is.
+
+        As described by U{RFC 3659 section
+        2.2<https://tools.ietf.org/html/rfc3659#section-2.2>}::
+
+            Various FTP commands take pathnames as arguments, or return
+            pathnames in responses. When the MLST command is supported, as
+            indicated in the response to the FEAT command, pathnames are to be
+            transferred in one of the following two formats.
+
+                pathname = utf-8-name / raw
+                utf-8-name = <a UTF-8 encoded Unicode string>
+                raw = <any string that is not a valid UTF-8 encoding>
+
+            Which format is used is at the option of the user-PI or server-PI
+            sending the pathname.
+
+        @param name: Name to be encoded.
+        @type name: L{bytes} or L{unicode}
+
+        @return: Wire format of C{name}.
+        @rtype: L{bytes}
+        """
+        if isinstance(name, str):
+            return name.encode("utf-8")
+        return name
+
+    def ftp_LIST(self, path=""):
+        """This command causes a list to be sent from the server to the
+        passive DTP.  If the pathname specifies a directory or other
+        group of files, the server should transfer a list of files
+        in the specified directory.  If the pathname specifies a
+        file then the server should send current information on the
+        file.  A null argument implies the user's current working or
+        default directory.
+        """
+        # XXX: why is this check different from ftp_RETR/ftp_STOR? See #4180
+        if self.dtpInstance is None or not self.dtpInstance.isConnected:
+            return defer.fail(BadCmdSequenceError("must send PORT or PASV before RETR"))
+
+        # Various clients send flags like -L or -al etc.  We just ignore them.
+        if path.lower() in ["-a", "-l", "-la", "-al"]:
+            path = ""
+
+        def gotListing(results):
+            self.reply(DATA_CNX_ALREADY_OPEN_START_XFR)
+            for name, attrs in results:
+                name = self._encodeName(name)
+                self.dtpInstance.sendListResponse(name, attrs)
+            self.dtpInstance.transport.loseConnection()
+            return (TXFR_COMPLETE_OK,)
+
+        try:
+            segments = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+
+        d = self.shell.list(
+            segments,
+            (
+                "size",
+                "directory",
+                "permissions",
+                "hardlinks",
+                "modified",
+                "owner",
+                "group",
+            ),
+        )
+        d.addCallback(gotListing)
+        return d
+
+    def ftp_NLST(self, path):
+        """
+        This command causes a directory listing to be sent from the server to
+        the client. The pathname should specify a directory or other
+        system-specific file group descriptor. An empty path implies the
+        current working directory. If the path is non-existent, send nothing.
+        If the path is to a file, send only the file name.
+
+        @type path: C{str}
+        @param path: The path for which a directory listing should be returned.
+
+        @rtype: L{Deferred}
+        @return: a L{Deferred} which will be fired when the listing request
+            is finished.
+        """
+        # XXX: why is this check different from ftp_RETR/ftp_STOR? See #4180
+        if self.dtpInstance is None or not self.dtpInstance.isConnected:
+            return defer.fail(BadCmdSequenceError("must send PORT or PASV before RETR"))
+
+        try:
+            segments = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+
+        def cbList(results, glob):
+            """
+            Send, line by line, each matching file in the directory listing,
+            and then close the connection.
+
+            @type results: A C{list} of C{tuple}. The first element of each
+                C{tuple} is a C{str} and the second element is a C{list}.
+            @param results: The names of the files in the directory.
+
+            @param glob: A shell-style glob through which to filter results
+                (see U{http://docs.python.org/2/library/fnmatch.html}), or
+                L{None} for no filtering.
+            @type glob: L{str} or L{None}
+
+            @return: A C{tuple} containing the status code for a successful
+                transfer.
+            @rtype: C{tuple}
+            """
+            self.reply(DATA_CNX_ALREADY_OPEN_START_XFR)
+            for name, ignored in results:
+                if not glob or (glob and fnmatch.fnmatch(name, glob)):
+                    name = self._encodeName(name)
+                    self.dtpInstance.sendLine(name)
+            self.dtpInstance.transport.loseConnection()
+            return (TXFR_COMPLETE_OK,)
+
+        def listErr(results):
+            """
+            RFC 959 specifies that an NLST request may only return directory
+            listings. Thus, send nothing and just close the connection.
+
+            @type results: L{Failure}
+            @param results: The L{Failure} wrapping a L{FileNotFoundError} that
+                occurred while trying to list the contents of a nonexistent
+                directory.
+
+            @returns: A C{tuple} containing the status code for a successful
+                transfer.
+            @rtype: C{tuple}
+            """
+            self.dtpInstance.transport.loseConnection()
+            return (TXFR_COMPLETE_OK,)
+
+        if _isGlobbingExpression(segments):
+            # Remove globbing expression from path
+            # and keep to be used for filtering.
+            glob = segments.pop()
+        else:
+            glob = None
+
+        d = self.shell.list(segments)
+        d.addCallback(cbList, glob)
+        # self.shell.list will generate an error if the path is invalid
+        d.addErrback(listErr)
+        return d
+
+    def ftp_CWD(self, path):
+        try:
+            segments = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            # XXX Eh, what to fail with here?
+            return defer.fail(FileNotFoundError(path))
+
+        def accessGranted(result):
+            self.workingDirectory = segments
+            return (REQ_FILE_ACTN_COMPLETED_OK,)
+
+        return self.shell.access(segments).addCallback(accessGranted)
+
+    def ftp_CDUP(self):
+        return self.ftp_CWD("..")
+
+    def ftp_PWD(self):
+        return (PWD_REPLY, "/" + "/".join(self.workingDirectory))
+
+    def ftp_RETR(self, path):
+        """
+        This command causes the content of a file to be sent over the data
+        transfer channel. If the path is to a folder, an error will be raised.
+
+        @type path: C{str}
+        @param path: The path to the file which should be transferred over the
+        data transfer channel.
+
+        @rtype: L{Deferred}
+        @return: a L{Deferred} which will be fired when the transfer is done.
+        """
+        if self.dtpInstance is None:
+            raise BadCmdSequenceError("PORT or PASV required before RETR")
+
+        try:
+            newsegs = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+
+        # XXX For now, just disable the timeout.  Later we'll want to
+        # leave it active and have the DTP connection reset it
+        # periodically.
+        self.setTimeout(None)
+
+        # Put it back later
+        def enableTimeout(result):
+            self.setTimeout(self.factory.timeOut)
+            return result
+
+        # And away she goes
+        if not self.binary:
+            cons = ASCIIConsumerWrapper(self.dtpInstance)
+        else:
+            cons = self.dtpInstance
+
+        def cbSent(result):
+            return (TXFR_COMPLETE_OK,)
+
+        def ebSent(err):
+            log.msg("Unexpected error attempting to transmit file to client:")
+            log.err(err)
+            if err.check(FTPCmdError):
+                return err
+            return (CNX_CLOSED_TXFR_ABORTED,)
+
+        def cbOpened(file):
+            # Tell them what to doooo
+            if self.dtpInstance.isConnected:
+                self.reply(DATA_CNX_ALREADY_OPEN_START_XFR)
+            else:
+                self.reply(FILE_STATUS_OK_OPEN_DATA_CNX)
+
+            d = file.send(cons)
+            d.addCallbacks(cbSent, ebSent)
+            return d
+
+        def ebOpened(err):
+            if not err.check(
+                PermissionDeniedError, FileNotFoundError, IsADirectoryError
+            ):
+                log.msg("Unexpected error attempting to open file for " "transmission:")
+                log.err(err)
+            if err.check(FTPCmdError):
+                return (err.value.errorCode, "/".join(newsegs))
+            return (FILE_NOT_FOUND, "/".join(newsegs))
+
+        d = self.shell.openForReading(newsegs)
+        d.addCallbacks(cbOpened, ebOpened)
+        d.addBoth(enableTimeout)
+
+        # Pass back Deferred that fires when the transfer is done
+        return d
+
+    def ftp_STOR(self, path):
+        """
+        STORE (STOR)
+
+        This command causes the server-DTP to accept the data
+        transferred via the data connection and to store the data as
+        a file at the server site.  If the file specified in the
+        pathname exists at the server site, then its contents shall
+        be replaced by the data being transferred.  A new file is
+        created at the server site if the file specified in the
+        pathname does not already exist.
+        """
+        if self.dtpInstance is None:
+            raise BadCmdSequenceError("PORT or PASV required before STOR")
+
+        try:
+            newsegs = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+
+        # XXX For now, just disable the timeout.  Later we'll want to
+        # leave it active and have the DTP connection reset it
+        # periodically.
+        self.setTimeout(None)
+
+        # Put it back later
+        def enableTimeout(result):
+            self.setTimeout(self.factory.timeOut)
+            return result
+
+        def cbOpened(file):
+            """
+            File was open for reading. Launch the data transfer channel via
+            the file consumer.
+            """
+            d = file.receive()
+            d.addCallback(cbConsumer)
+            d.addCallback(lambda ignored: file.close())
+            d.addCallbacks(cbSent, ebSent)
+            return d
+
+        def ebOpened(err):
+            """
+            Called when failed to open the file for reading.
+
+            For known errors, return the FTP error code.
+            For all other, return a file not found error.
+            """
+            if isinstance(err.value, FTPCmdError):
+                return (err.value.errorCode, "/".join(newsegs))
+            log.err(err, "Unexpected error received while opening file:")
+            return (FILE_NOT_FOUND, "/".join(newsegs))
+
+        def cbConsumer(cons):
+            """
+            Called after the file was opended for reading.
+
+            Prepare the data transfer channel and send the response
+            to the command channel.
+            """
+            if not self.binary:
+                cons = ASCIIConsumerWrapper(cons)
+
+            d = self.dtpInstance.registerConsumer(cons)
+
+            # Tell them what to doooo
+            if self.dtpInstance.isConnected:
+                self.reply(DATA_CNX_ALREADY_OPEN_START_XFR)
+            else:
+                self.reply(FILE_STATUS_OK_OPEN_DATA_CNX)
+
+            return d
+
+        def cbSent(result):
+            """
+            Called from data transport when transfer is done.
+            """
+            return (TXFR_COMPLETE_OK,)
+
+        def ebSent(err):
+            """
+            Called from data transport when there are errors during the
+            transfer.
+            """
+            log.err(err, "Unexpected error received during transfer:")
+            if err.check(FTPCmdError):
+                return err
+            return (CNX_CLOSED_TXFR_ABORTED,)
+
+        d = self.shell.openForWriting(newsegs)
+        d.addCallbacks(cbOpened, ebOpened)
+        d.addBoth(enableTimeout)
+
+        # Pass back Deferred that fires when the transfer is done
+        return d
+
+    def ftp_SIZE(self, path):
+        """
+        File SIZE
+
+        The FTP command, SIZE OF FILE (SIZE), is used to obtain the transfer
+        size of a file from the server-FTP process.  This is the exact number
+        of octets (8 bit bytes) that would be transmitted over the data
+        connection should that file be transmitted.  This value will change
+        depending on the current STRUcture, MODE, and TYPE of the data
+        connection or of a data connection that would be created were one
+        created now.  Thus, the result of the SIZE command is dependent on
+        the currently established STRU, MODE, and TYPE parameters.
+
+        The SIZE command returns how many octets would be transferred if the
+        file were to be transferred using the current transfer structure,
+        mode, and type.  This command is normally used in conjunction with
+        the RESTART (REST) command when STORing a file to a remote server in
+        STREAM mode, to determine the restart point.  The server-PI might
+        need to read the partially transferred file, do any appropriate
+        conversion, and count the number of octets that would be generated
+        when sending the file in order to correctly respond to this command.
+        Estimates of the file transfer size MUST NOT be returned; only
+        precise information is acceptable.
+
+        http://tools.ietf.org/html/rfc3659
+        """
+        try:
+            newsegs = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+
+        def cbStat(result):
+            (size,) = result
+            return (FILE_STATUS, str(size))
+
+        return self.shell.stat(newsegs, ("size",)).addCallback(cbStat)
+
+    def ftp_MDTM(self, path):
+        """
+        File Modification Time (MDTM)
+
+        The FTP command, MODIFICATION TIME (MDTM), can be used to determine
+        when a file in the server NVFS was last modified.  This command has
+        existed in many FTP servers for many years, as an adjunct to the REST
+        command for STREAM mode, thus is widely available.  However, where
+        supported, the "modify" fact that can be provided in the result from
+        the new MLST command is recommended as a superior alternative.
+
+        http://tools.ietf.org/html/rfc3659
+        """
+        try:
+            newsegs = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+
+        def cbStat(result):
+            (modified,) = result
+            return (FILE_STATUS, time.strftime("%Y%m%d%H%M%S", time.gmtime(modified)))
+
+        return self.shell.stat(newsegs, ("modified",)).addCallback(cbStat)
+
+    def ftp_TYPE(self, type):
+        """
+        REPRESENTATION TYPE (TYPE)
+
+        The argument specifies the representation type as described
+        in the Section on Data Representation and Storage.  Several
+        types take a second parameter.  The first parameter is
+        denoted by a single Telnet character, as is the second
+        Format parameter for ASCII and EBCDIC; the second parameter
+        for local byte is a decimal integer to indicate Bytesize.
+        The parameters are separated by a <SP> (Space, ASCII code
+        32).
+        """
+        p = type.upper()
+        if p:
+            f = getattr(self, "type_" + p[0], None)
+            if f is not None:
+                return f(p[1:])
+            return self.type_UNKNOWN(p)
+        return (SYNTAX_ERR,)
+
+    def type_A(self, code):
+        if code == "" or code == "N":
+            self.binary = False
+            return (TYPE_SET_OK, "A" + code)
+        else:
+            return defer.fail(CmdArgSyntaxError(code))
+
+    def type_I(self, code):
+        if code == "":
+            self.binary = True
+            return (TYPE_SET_OK, "I")
+        else:
+            return defer.fail(CmdArgSyntaxError(code))
+
+    def type_UNKNOWN(self, code):
+        return defer.fail(CmdNotImplementedForArgError(code))
+
+    def ftp_SYST(self):
+        return NAME_SYS_TYPE
+
+    def ftp_STRU(self, structure):
+        p = structure.upper()
+        if p == "F":
+            return (CMD_OK,)
+        return defer.fail(CmdNotImplementedForArgError(structure))
+
+    def ftp_MODE(self, mode):
+        p = mode.upper()
+        if p == "S":
+            return (CMD_OK,)
+        return defer.fail(CmdNotImplementedForArgError(mode))
+
+    def ftp_MKD(self, path):
+        try:
+            newsegs = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+        return self.shell.makeDirectory(newsegs).addCallback(
+            lambda ign: (MKD_REPLY, path)
+        )
+
+    def ftp_RMD(self, path):
+        try:
+            newsegs = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+        return self.shell.removeDirectory(newsegs).addCallback(
+            lambda ign: (REQ_FILE_ACTN_COMPLETED_OK,)
+        )
+
+    def ftp_DELE(self, path):
+        try:
+            newsegs = toSegments(self.workingDirectory, path)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(path))
+        return self.shell.removeFile(newsegs).addCallback(
+            lambda ign: (REQ_FILE_ACTN_COMPLETED_OK,)
+        )
+
+    def ftp_NOOP(self):
+        return (CMD_OK,)
+
+    def ftp_RNFR(self, fromName):
+        self._fromName = fromName
+        self.state = self.RENAMING
+        return (REQ_FILE_ACTN_PENDING_FURTHER_INFO,)
+
+    def ftp_RNTO(self, toName):
+        fromName = self._fromName
+        del self._fromName
+        self.state = self.AUTHED
+
+        try:
+            fromsegs = toSegments(self.workingDirectory, fromName)
+            tosegs = toSegments(self.workingDirectory, toName)
+        except InvalidPath:
+            return defer.fail(FileNotFoundError(fromName))
+        return self.shell.rename(fromsegs, tosegs).addCallback(
+            lambda ign: (REQ_FILE_ACTN_COMPLETED_OK,)
+        )
+
+    def ftp_FEAT(self):
+        """
+        Advertise the features supported by the server.
+
+        http://tools.ietf.org/html/rfc2389
+        """
+        self.sendLine(RESPONSE[FEAT_OK][0])
+        for feature in self.FEATURES:
+            self.sendLine(" " + feature)
+        self.sendLine(RESPONSE[FEAT_OK][1])
+
+    def ftp_OPTS(self, option):
+        """
+        Handle OPTS command.
+
+        http://tools.ietf.org/html/draft-ietf-ftpext-utf-8-option-00
+        """
+        return self.reply(OPTS_NOT_IMPLEMENTED, option)
+
+    def ftp_QUIT(self):
+        self.reply(GOODBYE_MSG)
+        self.transport.loseConnection()
+        self.disconnected = True
+
+    def cleanupDTP(self):
+        """
+        Call when DTP connection exits
+        """
+        log.msg("cleanupDTP", debug=True)
+
+        log.msg(self.dtpPort)
+        dtpPort, self.dtpPort = self.dtpPort, None
+        if interfaces.IListeningPort.providedBy(dtpPort):
+            dtpPort.stopListening()
+        elif interfaces.IConnector.providedBy(dtpPort):
+            dtpPort.disconnect()
+        else:
+            assert False, (
+                "dtpPort should be an IListeningPort or IConnector, "
+                "instead is %r" % (dtpPort,)
+            )
+
+        self.dtpFactory.stopFactory()
+        self.dtpFactory = None
+
+        if self.dtpInstance is not None:
+            self.dtpInstance = None
+
+
+class FTPFactory(policies.LimitTotalConnectionsFactory):
+    """
+    A factory for producing ftp protocol instances
+
+    @ivar timeOut: the protocol interpreter's idle timeout time in seconds,
+        default is 600 seconds.
+
+    @ivar passivePortRange: value forwarded to C{protocol.passivePortRange}.
+    @type passivePortRange: C{iterator}
+    """
+
+    protocol = FTP
+    overflowProtocol = FTPOverflowProtocol
+    allowAnonymous = True
+    userAnonymous = "anonymous"
+    timeOut = 600
+
+    welcomeMessage = f"Twisted {copyright.version} FTP Server"
+
+    passivePortRange = range(0, 1)
+
+    def __init__(self, portal=None, userAnonymous="anonymous"):
+        self.portal = portal
+        self.userAnonymous = userAnonymous
+        self.instances = []
+
+    def buildProtocol(self, addr):
+        p = policies.LimitTotalConnectionsFactory.buildProtocol(self, addr)
+        if p is not None:
+            p.wrappedProtocol.portal = self.portal
+            p.wrappedProtocol.timeOut = self.timeOut
+            p.wrappedProtocol.passivePortRange = self.passivePortRange
+        return p
+
+    def stopFactory(self):
+        # make sure ftp instance's timeouts are set to None
+        # to avoid reactor complaints
+        [p.setTimeout(None) for p in self.instances if p.timeOut is not None]
+        policies.LimitTotalConnectionsFactory.stopFactory(self)
+
+
+# -- Cred Objects --
+
+
+class IFTPShell(Interface):
+    """
+    An abstraction of the shell commands used by the FTP protocol for
+    a given user account.
+
+    All path names must be absolute.
+    """
+
+    def makeDirectory(path):
+        """
+        Create a directory.
+
+        @param path: The path, as a list of segments, to create
+        @type path: C{list} of C{unicode}
+
+        @return: A Deferred which fires when the directory has been
+        created, or which fails if the directory cannot be created.
+        """
+
+    def removeDirectory(path):
+        """
+        Remove a directory.
+
+        @param path: The path, as a list of segments, to remove
+        @type path: C{list} of C{unicode}
+
+        @return: A Deferred which fires when the directory has been
+        removed, or which fails if the directory cannot be removed.
+        """
+
+    def removeFile(path):
+        """
+        Remove a file.
+
+        @param path: The path, as a list of segments, to remove
+        @type path: C{list} of C{unicode}
+
+        @return: A Deferred which fires when the file has been
+        removed, or which fails if the file cannot be removed.
+        """
+
+    def rename(fromPath, toPath):
+        """
+        Rename a file or directory.
+
+        @param fromPath: The current name of the path.
+        @type fromPath: C{list} of C{unicode}
+
+        @param toPath: The desired new name of the path.
+        @type toPath: C{list} of C{unicode}
+
+        @return: A Deferred which fires when the path has been
+        renamed, or which fails if the path cannot be renamed.
+        """
+
+    def access(path):
+        """
+        Determine whether access to the given path is allowed.
+
+        @param path: The path, as a list of segments
+
+        @return: A Deferred which fires with None if access is allowed
+        or which fails with a specific exception type if access is
+        denied.
+        """
+
+    def stat(path, keys=()):
+        """
+        Retrieve information about the given path.
+
+        This is like list, except it will never return results about
+        child paths.
+        """
+
+    def list(path, keys=()):
+        """
+        Retrieve information about the given path.
+
+        If the path represents a non-directory, the result list should
+        have only one entry with information about that non-directory.
+        Otherwise, the result list should have an element for each
+        child of the directory.
+
+        @param path: The path, as a list of segments, to list
+        @type path: C{list} of C{unicode} or C{bytes}
+
+        @param keys: A tuple of keys desired in the resulting
+        dictionaries.
+
+        @return: A Deferred which fires with a list of (name, list),
+        where the name is the name of the entry as a unicode string or
+        bytes and each list contains values corresponding to the requested
+        keys.  The following are possible elements of keys, and the
+        values which should be returned for them:
+
+            - C{'size'}: size in bytes, as an integer (this is kinda required)
+
+            - C{'directory'}: boolean indicating the type of this entry
+
+            - C{'permissions'}: a bitvector (see os.stat(foo).st_mode)
+
+            - C{'hardlinks'}: Number of hard links to this entry
+
+            - C{'modified'}: number of seconds since the epoch since entry was
+              modified
+
+            - C{'owner'}: string indicating the user owner of this entry
+
+            - C{'group'}: string indicating the group owner of this entry
+        """
+
+    def openForReading(path):
+        """
+        @param path: The path, as a list of segments, to open
+        @type path: C{list} of C{unicode}
+
+        @rtype: C{Deferred} which will fire with L{IReadFile}
+        """
+
+    def openForWriting(path):
+        """
+        @param path: The path, as a list of segments, to open
+        @type path: C{list} of C{unicode}
+
+        @rtype: C{Deferred} which will fire with L{IWriteFile}
+        """
+
+
+class IReadFile(Interface):
+    """
+    A file out of which bytes may be read.
+    """
+
+    def send(consumer):
+        """
+        Produce the contents of the given path to the given consumer.  This
+        method may only be invoked once on each provider.
+
+        @type consumer: C{IConsumer}
+
+        @return: A Deferred which fires when the file has been
+        consumed completely.
+        """
+
+
+class IWriteFile(Interface):
+    """
+    A file into which bytes may be written.
+    """
+
+    def receive():
+        """
+        Create a consumer which will write to this file.  This method may
+        only be invoked once on each provider.
+
+        @rtype: C{Deferred} of C{IConsumer}
+        """
+
+    def close():
+        """
+        Perform any post-write work that needs to be done. This method may
+        only be invoked once on each provider, and will always be invoked
+        after receive().
+
+        @rtype: C{Deferred} of anything: the value is ignored. The FTP client
+        will not see their upload request complete until this Deferred has
+        been fired.
+        """
+
+
+def _getgroups(uid):
+    """
+    Return the primary and supplementary groups for the given UID.
+
+    @type uid: C{int}
+    """
+    result = []
+    pwent = pwd.getpwuid(uid)
+
+    result.append(pwent.pw_gid)
+
+    for grent in grp.getgrall():
+        if pwent.pw_name in grent.gr_mem:
+            result.append(grent.gr_gid)
+
+    return result
+
+
+def _testPermissions(uid, gid, spath, mode="r"):
+    """
+    checks to see if uid has proper permissions to access path with mode
+
+    @type uid: C{int}
+    @param uid: numeric user id
+
+    @type gid: C{int}
+    @param gid: numeric group id
+
+    @type spath: C{str}
+    @param spath: the path on the server to test
+
+    @type mode: C{str}
+    @param mode: 'r' or 'w' (read or write)
+
+    @rtype: C{bool}
+    @return: True if the given credentials have the specified form of
+        access to the given path
+    """
+    if mode == "r":
+        usr = stat.S_IRUSR
+        grp = stat.S_IRGRP
+        oth = stat.S_IROTH
+        amode = os.R_OK
+    elif mode == "w":
+        usr = stat.S_IWUSR
+        grp = stat.S_IWGRP
+        oth = stat.S_IWOTH
+        amode = os.W_OK
+    else:
+        raise ValueError(f"Invalid mode {mode!r}: must specify 'r' or 'w'")
+
+    access = False
+    if os.path.exists(spath):
+        if uid == 0:
+            access = True
+        else:
+            s = os.stat(spath)
+            if usr & s.st_mode and uid == s.st_uid:
+                access = True
+            elif grp & s.st_mode and gid in _getgroups(uid):
+                access = True
+            elif oth & s.st_mode:
+                access = True
+
+    if access:
+        if not os.access(spath, amode):
+            access = False
+            log.msg(
+                "Filesystem grants permission to UID %d but it is "
+                "inaccessible to me running as UID %d" % (uid, os.getuid())
+            )
+    return access
+
+
+@implementer(IFTPShell)
+class FTPAnonymousShell:
+    """
+    An anonymous implementation of IFTPShell
+
+    @type filesystemRoot: L{twisted.python.filepath.FilePath}
+    @ivar filesystemRoot: The path which is considered the root of
+    this shell.
+    """
+
+    def __init__(self, filesystemRoot):
+        self.filesystemRoot = filesystemRoot
+
+    def _path(self, path):
+        return self.filesystemRoot.descendant(path)
+
+    def makeDirectory(self, path):
+        return defer.fail(AnonUserDeniedError())
+
+    def removeDirectory(self, path):
+        return defer.fail(AnonUserDeniedError())
+
+    def removeFile(self, path):
+        return defer.fail(AnonUserDeniedError())
+
+    def rename(self, fromPath, toPath):
+        return defer.fail(AnonUserDeniedError())
+
+    def receive(self, path):
+        path = self._path(path)
+        return defer.fail(AnonUserDeniedError())
+
+    def openForReading(self, path):
+        """
+        Open C{path} for reading.
+
+        @param path: The path, as a list of segments, to open.
+        @type path: C{list} of C{unicode}
+        @return: A L{Deferred} is returned that will fire with an object
+            implementing L{IReadFile} if the file is successfully opened.  If
+            C{path} is a directory, or if an exception is raised while trying
+            to open the file, the L{Deferred} will fire with an error.
+        """
+        p = self._path(path)
+        if p.isdir():
+            # Normally, we would only check for EISDIR in open, but win32
+            # returns EACCES in this case, so we check before
+            return defer.fail(IsADirectoryError(path))
+        try:
+            f = p.open("r")
+        except OSError as e:
+            return errnoToFailure(e.errno, path)
+        except BaseException:
+            return defer.fail()
+        else:
+            return defer.succeed(_FileReader(f))
+
+    def openForWriting(self, path):
+        """
+        Reject write attempts by anonymous users with
+        L{PermissionDeniedError}.
+        """
+        return defer.fail(PermissionDeniedError("STOR not allowed"))
+
+    def access(self, path):
+        p = self._path(path)
+        if not p.exists():
+            # Again, win32 doesn't report a sane error after, so let's fail
+            # early if we can
+            return defer.fail(FileNotFoundError(path))
+        # For now, just see if we can os.listdir() it
+        try:
+            p.listdir()
+        except OSError as e:
+            return errnoToFailure(e.errno, path)
+        except BaseException:
+            return defer.fail()
+        else:
+            return defer.succeed(None)
+
+    def stat(self, path, keys=()):
+        p = self._path(path)
+        if p.isdir():
+            try:
+                statResult = self._statNode(p, keys)
+            except OSError as e:
+                return errnoToFailure(e.errno, path)
+            except BaseException:
+                return defer.fail()
+            else:
+                return defer.succeed(statResult)
+        else:
+            return self.list(path, keys).addCallback(lambda res: res[0][1])
+
+    def list(self, path, keys=()):
+        """
+        Return the list of files at given C{path}, adding C{keys} stat
+        informations if specified.
+
+        @param path: the directory or file to check.
+        @type path: C{str}
+
+        @param keys: the list of desired metadata
+        @type keys: C{list} of C{str}
+        """
+        filePath = self._path(path)
+        if filePath.isdir():
+            entries = filePath.listdir()
+            fileEntries = [filePath.child(p) for p in entries]
+        elif filePath.isfile():
+            entries = [os.path.join(*filePath.segmentsFrom(self.filesystemRoot))]
+            fileEntries = [filePath]
+        else:
+            return defer.fail(FileNotFoundError(path))
+
+        results = []
+        for fileName, filePath in zip(entries, fileEntries):
+            ent = []
+            results.append((fileName, ent))
+            if keys:
+                try:
+                    ent.extend(self._statNode(filePath, keys))
+                except OSError as e:
+                    return errnoToFailure(e.errno, fileName)
+                except BaseException:
+                    return defer.fail()
+
+        return defer.succeed(results)
+
+    def _statNode(self, filePath, keys):
+        """
+        Shortcut method to get stat info on a node.
+
+        @param filePath: the node to stat.
+        @type filePath: C{filepath.FilePath}
+
+        @param keys: the stat keys to get.
+        @type keys: C{iterable}
+        """
+        filePath.restat()
+        return [getattr(self, "_stat_" + k)(filePath) for k in keys]
+
+    def _stat_size(self, fp):
+        """
+        Get the filepath's size as an int
+
+        @param fp: L{twisted.python.filepath.FilePath}
+        @return: C{int} representing the size
+        """
+        return fp.getsize()
+
+    def _stat_permissions(self, fp):
+        """
+        Get the filepath's permissions object
+
+        @param fp: L{twisted.python.filepath.FilePath}
+        @return: L{twisted.python.filepath.Permissions} of C{fp}
+        """
+        return fp.getPermissions()
+
+    def _stat_hardlinks(self, fp):
+        """
+        Get the number of hardlinks for the filepath - if the number of
+        hardlinks is not yet implemented (say in Windows), just return 0 since
+        stat-ing a file in Windows seems to return C{st_nlink=0}.
+
+        (Reference:
+        U{http://stackoverflow.com/questions/5275731/os-stat-on-windows})
+
+        @param fp: L{twisted.python.filepath.FilePath}
+        @return: C{int} representing the number of hardlinks
+        """
+        try:
+            return fp.getNumberOfHardLinks()
+        except NotImplementedError:
+            return 0
+
+    def _stat_modified(self, fp):
+        """
+        Get the filepath's last modified date
+
+        @param fp: L{twisted.python.filepath.FilePath}
+        @return: C{int} as seconds since the epoch
+        """
+        return fp.getModificationTime()
+
+    def _stat_owner(self, fp):
+        """
+        Get the filepath's owner's username.  If this is not implemented
+        (say in Windows) return the string "0" since stat-ing a file in
+        Windows seems to return C{st_uid=0}.
+
+        (Reference:
+        U{http://stackoverflow.com/questions/5275731/os-stat-on-windows})
+
+        @param fp: L{twisted.python.filepath.FilePath}
+        @return: C{str} representing the owner's username
+        """
+        try:
+            userID = fp.getUserID()
+        except NotImplementedError:
+            return "0"
+        else:
+            if pwd is not None:
+                try:
+                    return pwd.getpwuid(userID)[0]
+                except KeyError:
+                    pass
+            return str(userID)
+
+    def _stat_group(self, fp):
+        """
+        Get the filepath's owner's group.  If this is not implemented
+        (say in Windows) return the string "0" since stat-ing a file in
+        Windows seems to return C{st_gid=0}.
+
+        (Reference:
+        U{http://stackoverflow.com/questions/5275731/os-stat-on-windows})
+
+        @param fp: L{twisted.python.filepath.FilePath}
+        @return: C{str} representing the owner's group
+        """
+        try:
+            groupID = fp.getGroupID()
+        except NotImplementedError:
+            return "0"
+        else:
+            if grp is not None:
+                try:
+                    return grp.getgrgid(groupID)[0]
+                except KeyError:
+                    pass
+            return str(groupID)
+
+    def _stat_directory(self, fp):
+        """
+        Get whether the filepath is a directory
+
+        @param fp: L{twisted.python.filepath.FilePath}
+        @return: C{bool}
+        """
+        return fp.isdir()
+
+
+@implementer(IReadFile)
+class _FileReader:
+    def __init__(self, fObj):
+        self.fObj = fObj
+        self._send = False
+
+    def _close(self, passthrough):
+        self._send = True
+        self.fObj.close()
+        return passthrough
+
+    def send(self, consumer):
+        assert not self._send, "Can only call IReadFile.send *once* per instance"
+        self._send = True
+        d = basic.FileSender().beginFileTransfer(self.fObj, consumer)
+        d.addBoth(self._close)
+        return d
+
+
+class FTPShell(FTPAnonymousShell):
+    """
+    An authenticated implementation of L{IFTPShell}.
+    """
+
+    def makeDirectory(self, path):
+        p = self._path(path)
+        try:
+            p.makedirs()
+        except OSError as e:
+            return errnoToFailure(e.errno, path)
+        except BaseException:
+            return defer.fail()
+        else:
+            return defer.succeed(None)
+
+    def removeDirectory(self, path):
+        p = self._path(path)
+        if p.isfile():
+            # Win32 returns the wrong errno when rmdir is called on a file
+            # instead of a directory, so as we have the info here, let's fail
+            # early with a pertinent error
+            return defer.fail(IsNotADirectoryError(path))
+        try:
+            os.rmdir(p.path)
+        except OSError as e:
+            return errnoToFailure(e.errno, path)
+        except BaseException:
+            return defer.fail()
+        else:
+            return defer.succeed(None)
+
+    def removeFile(self, path):
+        p = self._path(path)
+        if p.isdir():
+            # Win32 returns the wrong errno when remove is called on a
+            # directory instead of a file, so as we have the info here,
+            # let's fail early with a pertinent error
+            return defer.fail(IsADirectoryError(path))
+        try:
+            p.remove()
+        except OSError as e:
+            return errnoToFailure(e.errno, path)
+        except BaseException:
+            return defer.fail()
+        else:
+            return defer.succeed(None)
+
+    def rename(self, fromPath, toPath):
+        fp = self._path(fromPath)
+        tp = self._path(toPath)
+        try:
+            os.rename(fp.path, tp.path)
+        except OSError as e:
+            return errnoToFailure(e.errno, fromPath)
+        except BaseException:
+            return defer.fail()
+        else:
+            return defer.succeed(None)
+
+    def openForWriting(self, path):
+        """
+        Open C{path} for writing.
+
+        @param path: The path, as a list of segments, to open.
+        @type path: C{list} of C{unicode}
+        @return: A L{Deferred} is returned that will fire with an object
+            implementing L{IWriteFile} if the file is successfully opened.  If
+            C{path} is a directory, or if an exception is raised while trying
+            to open the file, the L{Deferred} will fire with an error.
+        """
+        p = self._path(path)
+        if p.isdir():
+            # Normally, we would only check for EISDIR in open, but win32
+            # returns EACCES in this case, so we check before
+            return defer.fail(IsADirectoryError(path))
+        try:
+            fObj = p.open("w")
+        except OSError as e:
+            return errnoToFailure(e.errno, path)
+        except BaseException:
+            return defer.fail()
+        return defer.succeed(_FileWriter(fObj))
+
+
+@implementer(IWriteFile)
+class _FileWriter:
+    def __init__(self, fObj):
+        self.fObj = fObj
+        self._receive = False
+
+    def receive(self):
+        assert not self._receive, "Can only call IWriteFile.receive *once* per instance"
+        self._receive = True
+        # FileConsumer will close the file object
+        return defer.succeed(FileConsumer(self.fObj))
+
+    def close(self):
+        return defer.succeed(None)
+
+
+@implementer(portal.IRealm)
+class BaseFTPRealm:
+    """
+    Base class for simple FTP realms which provides an easy hook for specifying
+    the home directory for each user.
+    """
+
+    def __init__(self, anonymousRoot):
+        self.anonymousRoot = filepath.FilePath(anonymousRoot)
+
+    def getHomeDirectory(self, avatarId):
+        """
+        Return a L{FilePath} representing the home directory of the given
+        avatar.  Override this in a subclass.
+
+        @param avatarId: A user identifier returned from a credentials checker.
+        @type avatarId: C{str}
+
+        @rtype: L{FilePath}
+        """
+        raise NotImplementedError(
+            f"{self.__class__!r} did not override getHomeDirectory"
+        )
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        for iface in interfaces:
+            if iface is IFTPShell:
+                if avatarId is checkers.ANONYMOUS:
+                    avatar = FTPAnonymousShell(self.anonymousRoot)
+                else:
+                    avatar = FTPShell(self.getHomeDirectory(avatarId))
+                return (IFTPShell, avatar, getattr(avatar, "logout", lambda: None))
+        raise NotImplementedError("Only IFTPShell interface is supported by this realm")
+
+
+class FTPRealm(BaseFTPRealm):
+    """
+    @type anonymousRoot: L{twisted.python.filepath.FilePath}
+    @ivar anonymousRoot: Root of the filesystem to which anonymous
+        users will be granted access.
+
+    @type userHome: L{filepath.FilePath}
+    @ivar userHome: Root of the filesystem containing user home directories.
+    """
+
+    def __init__(self, anonymousRoot, userHome="/home"):
+        BaseFTPRealm.__init__(self, anonymousRoot)
+        self.userHome = filepath.FilePath(userHome)
+
+    def getHomeDirectory(self, avatarId):
+        """
+        Use C{avatarId} as a single path segment to construct a child of
+        C{self.userHome} and return that child.
+        """
+        return self.userHome.child(avatarId)
+
+
+class SystemFTPRealm(BaseFTPRealm):
+    """
+    L{SystemFTPRealm} uses system user account information to decide what the
+    home directory for a particular avatarId is.
+
+    This works on POSIX but probably is not reliable on Windows.
+    """
+
+    def getHomeDirectory(self, avatarId):
+        """
+        Return the system-defined home directory of the system user account
+        with the name C{avatarId}.
+        """
+        path = os.path.expanduser("~" + avatarId)
+        if path.startswith("~"):
+            raise cred_error.UnauthorizedLogin()
+        return filepath.FilePath(path)
+
+
+# --- FTP CLIENT  -------------------------------------------------------------
+
+####
+# And now for the client...
+
+# Notes:
+#   * Reference: http://cr.yp.to/ftp.html
+#   * FIXME: Does not support pipelining (which is not supported by all
+#     servers anyway).  This isn't a functionality limitation, just a
+#     small performance issue.
+#   * Only has a rudimentary understanding of FTP response codes (although
+#     the full response is passed to the caller if they so choose).
+#   * Assumes that USER and PASS should always be sent
+#   * Always sets TYPE I  (binary mode)
+#   * Doesn't understand any of the weird, obscure TELNET stuff (\377...)
+#   * FIXME: Doesn't share any code with the FTPServer
+
+
+class ConnectionLost(FTPError):
+    pass
+
+
+class CommandFailed(FTPError):
+    pass
+
+
+class BadResponse(FTPError):
+    pass
+
+
+class UnexpectedResponse(FTPError):
+    pass
+
+
+class UnexpectedData(FTPError):
+    pass
+
+
+class FTPCommand:
+    def __init__(self, text=None, public=0):
+        self.text = text
+        self.deferred = defer.Deferred()
+        self.ready = 1
+        self.public = public
+        self.transferDeferred = None
+
+    def fail(self, failure):
+        if self.public:
+            self.deferred.errback(failure)
+
+
+class ProtocolWrapper(protocol.Protocol):
+    def __init__(self, original, deferred):
+        self.original = original
+        self.deferred = deferred
+
+    def makeConnection(self, transport):
+        self.original.makeConnection(transport)
+
+    def dataReceived(self, data):
+        self.original.dataReceived(data)
+
+    def connectionLost(self, reason):
+        self.original.connectionLost(reason)
+        # Signal that transfer has completed
+        self.deferred.callback(None)
+
+
+class IFinishableConsumer(interfaces.IConsumer):
+    """
+    A Consumer for producers that finish.
+
+    @since: 11.0
+    """
+
+    def finish():
+        """
+        The producer has finished producing.
+        """
+
+
+@implementer(IFinishableConsumer)
+class SenderProtocol(protocol.Protocol):
+    def __init__(self):
+        # Fired upon connection
+        self.connectedDeferred = defer.Deferred()
+
+        # Fired upon disconnection
+        self.deferred = defer.Deferred()
+
+    # Protocol stuff
+    def dataReceived(self, data):
+        raise UnexpectedData(
+            "Received data from the server on a " "send-only data-connection"
+        )
+
+    def makeConnection(self, transport):
+        protocol.Protocol.makeConnection(self, transport)
+        self.connectedDeferred.callback(self)
+
+    def connectionLost(self, reason):
+        if reason.check(error.ConnectionDone):
+            self.deferred.callback("connection done")
+        else:
+            self.deferred.errback(reason)
+
+    # IFinishableConsumer stuff
+    def write(self, data):
+        self.transport.write(data)
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register the given producer with our transport.
+        """
+        self.transport.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        """
+        Unregister the previously registered producer.
+        """
+        self.transport.unregisterProducer()
+
+    def finish(self):
+        self.transport.loseConnection()
+
+
+def decodeHostPort(line):
+    """
+    Decode an FTP response specifying a host and port.
+
+    @return: a 2-tuple of (host, port).
+    """
+    abcdef = re.sub("[^0-9, ]", "", line)
+    parsed = [int(p.strip()) for p in abcdef.split(",")]
+    for x in parsed:
+        if x < 0 or x > 255:
+            raise ValueError("Out of range", line, x)
+    a, b, c, d, e, f = parsed
+    host = f"{a}.{b}.{c}.{d}"
+    port = (int(e) << 8) + int(f)
+    return host, port
+
+
+def encodeHostPort(host, port):
+    numbers = host.split(".") + [str(port >> 8), str(port % 256)]
+    return ",".join(numbers)
+
+
+def _unwrapFirstError(failure):
+    failure.trap(defer.FirstError)
+    return failure.value.subFailure
+
+
+class FTPDataPortFactory(protocol.ServerFactory):
+    """
+    Factory for data connections that use the PORT command
+
+    (i.e. "active" transfers)
+    """
+
+    noisy = False
+
+    def buildProtocol(self, addr):
+        # This is a bit hackish -- we already have a Protocol instance,
+        # so just return it instead of making a new one
+        # FIXME: Reject connections from the wrong address/port
+        #        (potential security problem)
+        self.protocol.factory = self
+        self.port.loseConnection()
+        return self.protocol
+
+
+class FTPClientBasic(basic.LineReceiver):
+    """
+    Foundations of an FTP client.
+    """
+
+    debug = False
+    _encoding = "latin-1"
+
+    def __init__(self):
+        self.actionQueue = []
+        self.greeting = None
+        self.nextDeferred = defer.Deferred().addCallback(self._cb_greeting)
+        self.nextDeferred.addErrback(self.fail)
+        self.response = []
+        self._failed = 0
+
+    def fail(self, error):
+        """
+        Give an error to any queued deferreds.
+        """
+        self._fail(error)
+
+    def _fail(self, error):
+        """
+        Errback all queued deferreds.
+        """
+        if self._failed:
+            # We're recursing; bail out here for simplicity
+            return error
+        self._failed = 1
+        if self.nextDeferred:
+            try:
+                self.nextDeferred.errback(
+                    failure.Failure(ConnectionLost("FTP connection lost", error))
+                )
+            except defer.AlreadyCalledError:
+                pass
+        for ftpCommand in self.actionQueue:
+            ftpCommand.fail(
+                failure.Failure(ConnectionLost("FTP connection lost", error))
+            )
+        return error
+
+    def _cb_greeting(self, greeting):
+        self.greeting = greeting
+
+    def sendLine(self, line):
+        """
+        Sends a line, unless line is None.
+
+        @param line: Line to send
+        @type line: L{bytes} or L{unicode}
+        """
+        if line is None:
+            return
+        elif isinstance(line, str):
+            line = line.encode(self._encoding)
+        basic.LineReceiver.sendLine(self, line)
+
+    def sendNextCommand(self):
+        """
+        (Private) Processes the next command in the queue.
+        """
+        ftpCommand = self.popCommandQueue()
+        if ftpCommand is None:
+            self.nextDeferred = None
+            return
+        if not ftpCommand.ready:
+            self.actionQueue.insert(0, ftpCommand)
+            reactor.callLater(1.0, self.sendNextCommand)
+            self.nextDeferred = None
+            return
+
+        # FIXME: this if block doesn't belong in FTPClientBasic, it belongs in
+        #        FTPClient.
+        if ftpCommand.text == "PORT":
+            self.generatePortCommand(ftpCommand)
+
+        if self.debug:
+            log.msg("<-- %s" % ftpCommand.text)
+        self.nextDeferred = ftpCommand.deferred
+        self.sendLine(ftpCommand.text)
+
+    def queueCommand(self, ftpCommand):
+        """
+        Add an FTPCommand object to the queue.
+
+        If it's the only thing in the queue, and we are connected and we aren't
+        waiting for a response of an earlier command, the command will be sent
+        immediately.
+
+        @param ftpCommand: an L{FTPCommand}
+        """
+        self.actionQueue.append(ftpCommand)
+        if (
+            len(self.actionQueue) == 1
+            and self.transport is not None
+            and self.nextDeferred is None
+        ):
+            self.sendNextCommand()
+
+    def queueStringCommand(self, command, public=1):
+        """
+        Queues a string to be issued as an FTP command
+
+        @param command: string of an FTP command to queue
+        @param public: a flag intended for internal use by FTPClient.  Don't
+            change it unless you know what you're doing.
+
+        @return: a L{Deferred} that will be called when the response to the
+            command has been received.
+        """
+        ftpCommand = FTPCommand(command, public)
+        self.queueCommand(ftpCommand)
+        return ftpCommand.deferred
+
+    def popCommandQueue(self):
+        """
+        Return the front element of the command queue, or None if empty.
+        """
+        if self.actionQueue:
+            return self.actionQueue.pop(0)
+        else:
+            return None
+
+    def queueLogin(self, username, password):
+        """
+        Login: send the username, send the password.
+
+        If the password is L{None}, the PASS command won't be sent.  Also, if
+        the response to the USER command has a response code of 230 (User
+        logged in), then PASS won't be sent either.
+        """
+        # Prepare the USER command
+        deferreds = []
+        userDeferred = self.queueStringCommand("USER " + username, public=0)
+        deferreds.append(userDeferred)
+
+        # Prepare the PASS command (if a password is given)
+        if password is not None:
+            passwordCmd = FTPCommand("PASS " + password, public=0)
+            self.queueCommand(passwordCmd)
+            deferreds.append(passwordCmd.deferred)
+
+            # Avoid sending PASS if the response to USER is 230.
+            # (ref: http://cr.yp.to/ftp/user.html#user)
+            def cancelPasswordIfNotNeeded(response):
+                if response[0].startswith("230"):
+                    # No password needed!
+                    self.actionQueue.remove(passwordCmd)
+                return response
+
+            userDeferred.addCallback(cancelPasswordIfNotNeeded)
+
+        # Error handling.
+        for deferred in deferreds:
+            # If something goes wrong, call fail
+            deferred.addErrback(self.fail)
+            # But also swallow the error, so we don't cause spurious errors
+            deferred.addErrback(lambda x: None)
+
+    def lineReceived(self, line):
+        """
+        (Private) Parses the response messages from the FTP server.
+        """
+        # Add this line to the current response
+        if bytes != str:
+            line = line.decode(self._encoding)
+
+        if self.debug:
+            log.msg("--> %s" % line)
+        self.response.append(line)
+
+        # Bail out if this isn't the last line of a response
+        # The last line of response starts with 3 digits followed by a space
+        codeIsValid = re.match(r"\d{3} ", line)
+        if not codeIsValid:
+            return
+
+        code = line[0:3]
+
+        # Ignore marks
+        if code[0] == "1":
+            return
+
+        # Check that we were expecting a response
+        if self.nextDeferred is None:
+            self.fail(UnexpectedResponse(self.response))
+            return
+
+        # Reset the response
+        response = self.response
+        self.response = []
+
+        # Look for a success or error code, and call the appropriate callback
+        if code[0] in ("2", "3"):
+            # Success
+            self.nextDeferred.callback(response)
+        elif code[0] in ("4", "5"):
+            # Failure
+            self.nextDeferred.errback(failure.Failure(CommandFailed(response)))
+        else:
+            # This shouldn't happen unless something screwed up.
+            log.msg(f"Server sent invalid response code {code}")
+            self.nextDeferred.errback(failure.Failure(BadResponse(response)))
+
+        # Run the next command
+        self.sendNextCommand()
+
+    def connectionLost(self, reason):
+        self._fail(reason)
+
+
+class _PassiveConnectionFactory(protocol.ClientFactory):
+    noisy = False
+
+    def __init__(self, protoInstance):
+        self.protoInstance = protoInstance
+
+    def buildProtocol(self, ignored):
+        self.protoInstance.factory = self
+        return self.protoInstance
+
+    def clientConnectionFailed(self, connector, reason):
+        e = FTPError("Connection Failed", reason)
+        self.protoInstance.deferred.errback(e)
+
+
+class FTPClient(FTPClientBasic):
+    """
+    L{FTPClient} is a client implementation of the FTP protocol which
+    exposes FTP commands as methods which return L{Deferred}s.
+
+    Each command method returns a L{Deferred} which is called back when a
+    successful response code (2xx or 3xx) is received from the server or
+    which is error backed if an error response code (4xx or 5xx) is received
+    from the server or if a protocol violation occurs.  If an error response
+    code is received, the L{Deferred} fires with a L{Failure} wrapping a
+    L{CommandFailed} instance.  The L{CommandFailed} instance is created
+    with a list of the response lines received from the server.
+
+    See U{RFC 959<http://www.ietf.org/rfc/rfc959.txt>} for error code
+    definitions.
+
+    Both active and passive transfers are supported.
+
+    @ivar passive: See description in __init__.
+    """
+
+    connectFactory = reactor.connectTCP  # type: ignore[attr-defined]
+
+    def __init__(
+        self, username="anonymous", password="twisted@twistedmatrix.com", passive=1
+    ):
+        """
+        Constructor.
+
+        I will login as soon as I receive the welcome message from the server.
+
+        @param username: FTP username
+        @param password: FTP password
+        @param passive: flag that controls if I use active or passive data
+            connections.  You can also change this after construction by
+            assigning to C{self.passive}.
+        """
+        FTPClientBasic.__init__(self)
+        self.queueLogin(username, password)
+
+        self.passive = passive
+
+    def fail(self, error):
+        """
+        Disconnect, and also give an error to any queued deferreds.
+        """
+        self.transport.loseConnection()
+        self._fail(error)
+
+    def receiveFromConnection(self, commands, protocol):
+        """
+        Retrieves a file or listing generated by the given command,
+        feeding it to the given protocol.
+
+        @param commands: list of strings of FTP commands to execute then
+            receive the results of (e.g. C{LIST}, C{RETR})
+        @param protocol: A L{Protocol} B{instance} e.g. an
+            L{FTPFileListProtocol}, or something that can be adapted to one.
+            Typically this will be an L{IConsumer} implementation.
+
+        @return: L{Deferred}.
+        """
+        protocol = interfaces.IProtocol(protocol)
+        wrapper = ProtocolWrapper(protocol, defer.Deferred())
+        return self._openDataConnection(commands, wrapper)
+
+    def queueLogin(self, username, password):
+        """
+        Login: send the username, send the password, and
+        set retrieval mode to binary
+        """
+        FTPClientBasic.queueLogin(self, username, password)
+        d = self.queueStringCommand("TYPE I", public=0)
+        # If something goes wrong, call fail
+        d.addErrback(self.fail)
+        # But also swallow the error, so we don't cause spurious errors
+        d.addErrback(lambda x: None)
+
+    def sendToConnection(self, commands):
+        """
+        XXX
+
+        @return: A tuple of two L{Deferred}s:
+                  - L{Deferred} L{IFinishableConsumer}. You must call
+                    the C{finish} method on the IFinishableConsumer when the
+                    file is completely transferred.
+                  - L{Deferred} list of control-connection responses.
+        """
+        s = SenderProtocol()
+        r = self._openDataConnection(commands, s)
+        return (s.connectedDeferred, r)
+
+    def _openDataConnection(self, commands, protocol):
+        """
+        This method returns a DeferredList.
+        """
+        cmds = [FTPCommand(command, public=1) for command in commands]
+        cmdsDeferred = defer.DeferredList(
+            [cmd.deferred for cmd in cmds], fireOnOneErrback=True, consumeErrors=True
+        )
+        cmdsDeferred.addErrback(_unwrapFirstError)
+
+        if self.passive:
+            # Hack: use a mutable object to sneak a variable out of the
+            # scope of doPassive
+            _mutable = [None]
+
+            def doPassive(response):
+                """Connect to the port specified in the response to PASV"""
+                host, port = decodeHostPort(response[-1][4:])
+
+                f = _PassiveConnectionFactory(protocol)
+                _mutable[0] = self.connectFactory(host, port, f)
+
+            pasvCmd = FTPCommand("PASV")
+            self.queueCommand(pasvCmd)
+            pasvCmd.deferred.addCallback(doPassive).addErrback(self.fail)
+
+            results = [cmdsDeferred, pasvCmd.deferred, protocol.deferred]
+            d = defer.DeferredList(results, fireOnOneErrback=True, consumeErrors=True)
+            d.addErrback(_unwrapFirstError)
+
+            # Ensure the connection is always closed
+            def close(x, m=_mutable):
+                m[0] and m[0].disconnect()
+                return x
+
+            d.addBoth(close)
+
+        else:
+            # We just place a marker command in the queue, and will fill in
+            # the host and port numbers later (see generatePortCommand)
+            portCmd = FTPCommand("PORT")
+
+            # Ok, now we jump through a few hoops here.
+            # This is the problem: a transfer is not to be trusted as complete
+            # until we get both the "226 Transfer complete" message on the
+            # control connection, and the data socket is closed.  Thus, we use
+            # a DeferredList to make sure we only fire the callback at the
+            # right time.
+
+            portCmd.transferDeferred = protocol.deferred
+            portCmd.protocol = protocol
+            portCmd.deferred.addErrback(portCmd.transferDeferred.errback)
+            self.queueCommand(portCmd)
+
+            # Create dummy functions for the next callback to call.
+            # These will also be replaced with real functions in
+            # generatePortCommand.
+            portCmd.loseConnection = lambda result: result
+            portCmd.fail = lambda error: error
+
+            # Ensure that the connection always gets closed
+            cmdsDeferred.addErrback(lambda e, pc=portCmd: pc.fail(e) or e)
+
+            results = [cmdsDeferred, portCmd.deferred, portCmd.transferDeferred]
+            d = defer.DeferredList(results, fireOnOneErrback=True, consumeErrors=True)
+            d.addErrback(_unwrapFirstError)
+
+        for cmd in cmds:
+            self.queueCommand(cmd)
+        return d
+
+    def generatePortCommand(self, portCmd):
+        """
+        (Private) Generates the text of a given PORT command.
+        """
+
+        # The problem is that we don't create the listening port until we need
+        # it for various reasons, and so we have to muck about to figure out
+        # what interface and port it's listening on, and then finally we can
+        # create the text of the PORT command to send to the FTP server.
+
+        # FIXME: This method is far too ugly.
+
+        # FIXME: The best solution is probably to only create the data port
+        #        once per FTPClient, and just recycle it for each new download.
+        #        This should be ok, because we don't pipeline commands.
+
+        # Start listening on a port
+        factory = FTPDataPortFactory()
+        factory.protocol = portCmd.protocol
+        listener = reactor.listenTCP(0, factory)
+        factory.port = listener
+
+        # Ensure we close the listening port if something goes wrong
+        def listenerFail(error, listener=listener):
+            if listener.connected:
+                listener.loseConnection()
+            return error
+
+        portCmd.fail = listenerFail
+
+        # Construct crufty FTP magic numbers that represent host & port
+        host = self.transport.getHost().host
+        port = listener.getHost().port
+        portCmd.text = "PORT " + encodeHostPort(host, port)
+
+    def escapePath(self, path):
+        """
+        Returns a FTP escaped path (replace newlines with nulls).
+        """
+        # Escape newline characters
+        return path.replace("\n", "\0")
+
+    def retrieveFile(self, path, protocol, offset=0):
+        """
+        Retrieve a file from the given path
+
+        This method issues the 'RETR' FTP command.
+
+        The file is fed into the given Protocol instance.  The data connection
+        will be passive if self.passive is set.
+
+        @param path: path to file that you wish to receive.
+        @param protocol: a L{Protocol} instance.
+        @param offset: offset to start downloading from
+
+        @return: L{Deferred}
+        """
+        cmds = ["RETR " + self.escapePath(path)]
+        if offset:
+            cmds.insert(0, ("REST " + str(offset)))
+        return self.receiveFromConnection(cmds, protocol)
+
+    retr = retrieveFile
+
+    def storeFile(self, path, offset=0):
+        """
+        Store a file at the given path.
+
+        This method issues the 'STOR' FTP command.
+
+        @return: A tuple of two L{Deferred}s:
+                  - L{Deferred} L{IFinishableConsumer}. You must call
+                    the C{finish} method on the IFinishableConsumer when the
+                    file is completely transferred.
+                  - L{Deferred} list of control-connection responses.
+        """
+        cmds = ["STOR " + self.escapePath(path)]
+        if offset:
+            cmds.insert(0, ("REST " + str(offset)))
+        return self.sendToConnection(cmds)
+
+    stor = storeFile
+
+    def rename(self, pathFrom, pathTo):
+        """
+        Rename a file.
+
+        This method issues the I{RNFR}/I{RNTO} command sequence to rename
+        C{pathFrom} to C{pathTo}.
+
+        @param pathFrom: the absolute path to the file to be renamed
+        @type pathFrom: C{str}
+
+        @param pathTo: the absolute path to rename the file to.
+        @type pathTo: C{str}
+
+        @return: A L{Deferred} which fires when the rename operation has
+            succeeded or failed.  If it succeeds, the L{Deferred} is called
+            back with a two-tuple of lists.  The first list contains the
+            responses to the I{RNFR} command.  The second list contains the
+            responses to the I{RNTO} command.  If either I{RNFR} or I{RNTO}
+            fails, the L{Deferred} is errbacked with L{CommandFailed} or
+            L{BadResponse}.
+        @rtype: L{Deferred}
+
+        @since: 8.2
+        """
+        renameFrom = self.queueStringCommand("RNFR " + self.escapePath(pathFrom))
+        renameTo = self.queueStringCommand("RNTO " + self.escapePath(pathTo))
+
+        fromResponse = []
+
+        # Use a separate Deferred for the ultimate result so that Deferred
+        # chaining can't interfere with its result.
+        result = defer.Deferred()
+        # Bundle up all the responses
+        result.addCallback(lambda toResponse: (fromResponse, toResponse))
+
+        def ebFrom(failure):
+            # Make sure the RNTO doesn't run if the RNFR failed.
+            self.popCommandQueue()
+            result.errback(failure)
+
+        # Save the RNFR response to pass to the result Deferred later
+        renameFrom.addCallbacks(fromResponse.extend, ebFrom)
+
+        # Hook up the RNTO to the result Deferred as well
+        renameTo.chainDeferred(result)
+
+        return result
+
+    def list(self, path, protocol):
+        """
+        Retrieve a file listing into the given protocol instance.
+
+        This method issues the 'LIST' FTP command.
+
+        @param path: path to get a file listing for.
+        @param protocol: a L{Protocol} instance, probably a
+            L{FTPFileListProtocol} instance.  It can cope with most common file
+            listing formats.
+
+        @return: L{Deferred}
+        """
+        if path is None:
+            path = ""
+        return self.receiveFromConnection(["LIST " + self.escapePath(path)], protocol)
+
+    def nlst(self, path, protocol):
+        """
+        Retrieve a short file listing into the given protocol instance.
+
+        This method issues the 'NLST' FTP command.
+
+        NLST (should) return a list of filenames, one per line.
+
+        @param path: path to get short file listing for.
+        @param protocol: a L{Protocol} instance.
+        """
+        if path is None:
+            path = ""
+        return self.receiveFromConnection(["NLST " + self.escapePath(path)], protocol)
+
+    def cwd(self, path):
+        """
+        Issues the CWD (Change Working Directory) command.
+
+        @return: a L{Deferred} that will be called when done.
+        """
+        return self.queueStringCommand("CWD " + self.escapePath(path))
+
+    def makeDirectory(self, path):
+        """
+        Make a directory
+
+        This method issues the MKD command.
+
+        @param path: The path to the directory to create.
+        @type path: C{str}
+
+        @return: A L{Deferred} which fires when the server responds.  If the
+            directory is created, the L{Deferred} is called back with the
+            server response.  If the server response indicates the directory
+            was not created, the L{Deferred} is errbacked with a L{Failure}
+            wrapping L{CommandFailed} or L{BadResponse}.
+        @rtype: L{Deferred}
+
+        @since: 8.2
+        """
+        return self.queueStringCommand("MKD " + self.escapePath(path))
+
+    def removeFile(self, path):
+        """
+        Delete a file on the server.
+
+        L{removeFile} issues a I{DELE} command to the server to remove the
+        indicated file.  Note that this command cannot remove a directory.
+
+        @param path: The path to the file to delete. May be relative to the
+            current dir.
+        @type path: C{str}
+
+        @return: A L{Deferred} which fires when the server responds.  On error,
+            it is errbacked with either L{CommandFailed} or L{BadResponse}.  On
+            success, it is called back with a list of response lines.
+        @rtype: L{Deferred}
+
+        @since: 8.2
+        """
+        return self.queueStringCommand("DELE " + self.escapePath(path))
+
+    def removeDirectory(self, path):
+        """
+        Delete a directory on the server.
+
+        L{removeDirectory} issues a I{RMD} command to the server to remove the
+        indicated directory. Described in RFC959.
+
+        @param path: The path to the directory to delete. May be relative to
+            the current working directory.
+        @type path: C{str}
+
+        @return: A L{Deferred} which fires when the server responds. On error,
+            it is errbacked with either L{CommandFailed} or L{BadResponse}. On
+            success, it is called back with a list of response lines.
+        @rtype: L{Deferred}
+
+        @since: 11.1
+        """
+        return self.queueStringCommand("RMD " + self.escapePath(path))
+
+    def cdup(self):
+        """
+        Issues the CDUP (Change Directory UP) command.
+
+        @return: a L{Deferred} that will be called when done.
+        """
+        return self.queueStringCommand("CDUP")
+
+    def pwd(self):
+        """
+        Issues the PWD (Print Working Directory) command.
+
+        The L{getDirectory} does the same job but automatically parses the
+        result.
+
+        @return: a L{Deferred} that will be called when done.  It is up to the
+            caller to interpret the response, but the L{parsePWDResponse}
+            method in this module should work.
+        """
+        return self.queueStringCommand("PWD")
+
+    def getDirectory(self):
+        """
+        Returns the current remote directory.
+
+        @return: a L{Deferred} that will be called back with a C{str} giving
+            the remote directory or which will errback with L{CommandFailed}
+            if an error response is returned.
+        """
+
+        def cbParse(result):
+            try:
+                # The only valid code is 257
+                if int(result[0].split(" ", 1)[0]) != 257:
+                    raise ValueError
+            except (IndexError, ValueError):
+                return failure.Failure(CommandFailed(result))
+            path = parsePWDResponse(result[0])
+            if path is None:
+                return failure.Failure(CommandFailed(result))
+            return path
+
+        return self.pwd().addCallback(cbParse)
+
+    def quit(self):
+        """
+        Issues the I{QUIT} command.
+
+        @return: A L{Deferred} that fires when the server acknowledges the
+            I{QUIT} command.  The transport should not be disconnected until
+            this L{Deferred} fires.
+        """
+        return self.queueStringCommand("QUIT")
+
+
+class FTPFileListProtocol(basic.LineReceiver):
+    """
+    Parser for standard FTP file listings
+
+    This is the evil required to match::
+
+        -rw-r--r--   1 root     other        531 Jan 29 03:26 README
+
+    If you need different evil for a wacky FTP server, you can
+    override either C{fileLinePattern} or C{parseDirectoryLine()}.
+
+    It populates the instance attribute self.files, which is a list containing
+    dicts with the following keys (examples from the above line):
+        - filetype:   e.g. 'd' for directories, or '-' for an ordinary file
+        - perms:      e.g. 'rw-r--r--'
+        - nlinks:     e.g. 1
+        - owner:      e.g. 'root'
+        - group:      e.g. 'other'
+        - size:       e.g. 531
+        - date:       e.g. 'Jan 29 03:26'
+        - filename:   e.g. 'README'
+        - linktarget: e.g. 'some/file'
+
+    Note that the 'date' value will be formatted differently depending on the
+    date.  Check U{http://cr.yp.to/ftp.html} if you really want to try to parse
+    it.
+
+    It also matches the following::
+        -rw-r--r--   1 root     other        531 Jan 29 03:26 I HAVE\\ SPACE
+           - filename:   e.g. 'I HAVE SPACE'
+
+        -rw-r--r--   1 root     other        531 Jan 29 03:26 LINK -> TARGET
+           - filename:   e.g. 'LINK'
+           - linktarget: e.g. 'TARGET'
+
+        -rw-r--r--   1 root     other        531 Jan 29 03:26 N S -> L S
+           - filename:   e.g. 'N S'
+           - linktarget: e.g. 'L S'
+
+    @ivar files: list of dicts describing the files in this listing
+    """
+
+    fileLinePattern = re.compile(
+        r"^(?P<filetype>.)(?P<perms>.{9})\s+(?P<nlinks>\d*)\s*"
+        r"(?P<owner>\S+)\s+(?P<group>\S+)\s+(?P<size>\d+)\s+"
+        r"(?P<date>...\s+\d+\s+[\d:]+)\s+(?P<filename>.{1,}?)"
+        r"( -> (?P<linktarget>[^\r]*))?\r?$"
+    )
+    delimiter = b"\n"
+    _encoding = "latin-1"
+
+    def __init__(self):
+        self.files = []
+
+    def lineReceived(self, line):
+        if bytes != str:
+            line = line.decode(self._encoding)
+        d = self.parseDirectoryLine(line)
+        if d is None:
+            self.unknownLine(line)
+        else:
+            self.addFile(d)
+
+    def parseDirectoryLine(self, line):
+        """
+        Return a dictionary of fields, or None if line cannot be parsed.
+
+        @param line: line of text expected to contain a directory entry
+        @type line: str
+
+        @return: dict
+        """
+        match = self.fileLinePattern.match(line)
+        if match is None:
+            return None
+        else:
+            d = match.groupdict()
+            d["filename"] = d["filename"].replace(r"\ ", " ")
+            d["nlinks"] = int(d["nlinks"])
+            d["size"] = int(d["size"])
+            if d["linktarget"]:
+                d["linktarget"] = d["linktarget"].replace(r"\ ", " ")
+            return d
+
+    def addFile(self, info):
+        """
+        Append file information dictionary to the list of known files.
+
+        Subclasses can override or extend this method to handle file
+        information differently without affecting the parsing of data
+        from the server.
+
+        @param info: dictionary containing the parsed representation
+                     of the file information
+        @type info: dict
+        """
+        self.files.append(info)
+
+    def unknownLine(self, line):
+        """
+        Deal with received lines which could not be parsed as file
+        information.
+
+        Subclasses can override this to perform any special processing
+        needed.
+
+        @param line: unparsable line as received
+        @type line: str
+        """
+        pass
+
+
+def parsePWDResponse(response):
+    """
+    Returns the path from a response to a PWD command.
+
+    Responses typically look like::
+
+        257 "/home/andrew" is current directory.
+
+    For this example, I will return C{'/home/andrew'}.
+
+    If I can't find the path, I return L{None}.
+    """
+    match = re.search('"(.*)"', response)
+    if match:
+        return match.groups()[0]
+    else:
+        return None
diff --git a/contrib/python/Twisted/py3/twisted/protocols/haproxy/__init__.py b/contrib/python/Twisted/py3/twisted/protocols/haproxy/__init__.py
new file mode 100644
index 00000000000..2d13bf5b4c7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/haproxy/__init__.py
@@ -0,0 +1,10 @@
+# -*- test-case-name: twisted.protocols.haproxy.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HAProxy PROXY protocol implementations.
+"""
+__all__ = ["proxyEndpoint"]
+
+from ._wrapper import proxyEndpoint
diff --git a/contrib/python/Twisted/py3/twisted/protocols/haproxy/_exceptions.py b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_exceptions.py
new file mode 100644
index 00000000000..9a521ea2495
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_exceptions.py
@@ -0,0 +1,49 @@
+# -*- test-case-name: twisted.protocols.haproxy.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HAProxy specific exceptions.
+"""
+
+import contextlib
+from typing import Callable, Generator, Type
+
+
+class InvalidProxyHeader(Exception):
+    """
+    The provided PROXY protocol header is invalid.
+    """
+
+
+class InvalidNetworkProtocol(InvalidProxyHeader):
+    """
+    The network protocol was not one of TCP4 TCP6 or UNKNOWN.
+    """
+
+
+class MissingAddressData(InvalidProxyHeader):
+    """
+    The address data is missing or incomplete.
+    """
+
+
+@contextlib.contextmanager
+def convertError(
+    sourceType: Type[BaseException], targetType: Callable[[], BaseException]
+) -> Generator[None, None, None]:
+    """
+    Convert an error into a different error type.
+
+    @param sourceType: The type of exception that should be caught and
+        converted.
+    @type sourceType: L{BaseException}
+
+    @param targetType: The type of exception to which the original should be
+        converted.
+    @type targetType: L{BaseException}
+    """
+    try:
+        yield
+    except sourceType as e:
+        raise targetType().with_traceback(e.__traceback__)
diff --git a/contrib/python/Twisted/py3/twisted/protocols/haproxy/_info.py b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_info.py
new file mode 100644
index 00000000000..9dda6e06efc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_info.py
@@ -0,0 +1,34 @@
+# -*- test-case-name: twisted.protocols.haproxy.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+IProxyInfo implementation.
+"""
+from typing import Optional
+
+from zope.interface import implementer
+
+import attr
+
+from twisted.internet.interfaces import IAddress
+from ._interfaces import IProxyInfo
+
+
+@implementer(IProxyInfo)
+@attr.s(frozen=True, slots=True, auto_attribs=True)
+class ProxyInfo:
+    """
+    A data container for parsed PROXY protocol information.
+
+    @ivar header: The raw header bytes extracted from the connection.
+    @type header: C{bytes}
+    @ivar source: The connection source address.
+    @type source: L{twisted.internet.interfaces.IAddress}
+    @ivar destination: The connection destination address.
+    @type destination: L{twisted.internet.interfaces.IAddress}
+    """
+
+    header: bytes
+    source: Optional[IAddress]
+    destination: Optional[IAddress]
diff --git a/contrib/python/Twisted/py3/twisted/protocols/haproxy/_interfaces.py b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_interfaces.py
new file mode 100644
index 00000000000..8fe90ea37ab
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_interfaces.py
@@ -0,0 +1,63 @@
+# -*- test-case-name: twisted.protocols.haproxy.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Interfaces used by the PROXY protocol modules.
+"""
+from typing import Tuple, Union
+
+import zope.interface
+
+
+class IProxyInfo(zope.interface.Interface):
+    """
+    Data container for PROXY protocol header data.
+    """
+
+    header = zope.interface.Attribute(
+        "The raw byestring that represents the PROXY protocol header.",
+    )
+    source = zope.interface.Attribute(
+        "An L{twisted.internet.interfaces.IAddress} representing the "
+        "connection source."
+    )
+    destination = zope.interface.Attribute(
+        "An L{twisted.internet.interfaces.IAddress} representing the "
+        "connection destination."
+    )
+
+
+class IProxyParser(zope.interface.Interface):
+    """
+    Streaming parser that handles PROXY protocol headers.
+    """
+
+    def feed(data: bytes) -> Union[Tuple[IProxyInfo, bytes], Tuple[None, None]]:
+        """
+        Consume a chunk of data and attempt to parse it.
+
+        @param data: A bytestring.
+        @type data: bytes
+
+        @return: A two-tuple containing, in order, an L{IProxyInfo} and any
+            bytes fed to the parser that followed the end of the header.  Both
+            of these values are None until a complete header is parsed.
+
+        @raises InvalidProxyHeader: If the bytes fed to the parser create an
+            invalid PROXY header.
+        """
+
+    def parse(line: bytes) -> IProxyInfo:
+        """
+        Parse a bytestring as a full PROXY protocol header line.
+
+        @param line: A bytestring that represents a valid HAProxy PROXY
+            protocol header line.
+        @type line: bytes
+
+        @return: An L{IProxyInfo} containing the parsed data.
+
+        @raises InvalidProxyHeader: If the bytestring does not represent a
+            valid PROXY header.
+        """
diff --git a/contrib/python/Twisted/py3/twisted/protocols/haproxy/_parser.py b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_parser.py
new file mode 100644
index 00000000000..834ccb73547
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_parser.py
@@ -0,0 +1,75 @@
+# -*- test-case-name: twisted.protocols.haproxy.test.test_parser -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Parser for 'haproxy:' string endpoint.
+"""
+from typing import Mapping, Tuple
+
+from zope.interface import implementer
+
+from twisted.internet import interfaces
+from twisted.internet.endpoints import (
+    IStreamServerEndpointStringParser,
+    _WrapperServerEndpoint,
+    quoteStringArgument,
+    serverFromString,
+)
+from twisted.plugin import IPlugin
+from . import proxyEndpoint
+
+
+def unparseEndpoint(args: Tuple[object, ...], kwargs: Mapping[str, object]) -> str:
+    """
+    Un-parse the already-parsed args and kwargs back into endpoint syntax.
+
+    @param args: C{:}-separated arguments
+
+    @param kwargs: C{:} and then C{=}-separated keyword arguments
+
+    @return: a string equivalent to the original format which this was parsed
+        as.
+    """
+
+    description = ":".join(
+        [quoteStringArgument(str(arg)) for arg in args]
+        + sorted(
+            "{}={}".format(
+                quoteStringArgument(str(key)), quoteStringArgument(str(value))
+            )
+            for key, value in kwargs.items()
+        )
+    )
+    return description
+
+
+@implementer(IPlugin, IStreamServerEndpointStringParser)
+class HAProxyServerParser:
+    """
+    Stream server endpoint string parser for the HAProxyServerEndpoint type.
+
+    @ivar prefix: See L{IStreamServerEndpointStringParser.prefix}.
+    """
+
+    prefix = "haproxy"
+
+    def parseStreamServer(
+        self, reactor: interfaces.IReactorCore, *args: object, **kwargs: object
+    ) -> _WrapperServerEndpoint:
+        """
+        Parse a stream server endpoint from a reactor and string-only arguments
+        and keyword arguments.
+
+        @param reactor: The reactor.
+
+        @param args: The parsed string arguments.
+
+        @param kwargs: The parsed keyword arguments.
+
+        @return: a stream server endpoint
+        @rtype: L{IStreamServerEndpoint}
+        """
+        subdescription = unparseEndpoint(args, kwargs)
+        wrappedEndpoint = serverFromString(reactor, subdescription)
+        return proxyEndpoint(wrappedEndpoint)
diff --git a/contrib/python/Twisted/py3/twisted/protocols/haproxy/_v1parser.py b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_v1parser.py
new file mode 100644
index 00000000000..fed987c33af
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_v1parser.py
@@ -0,0 +1,142 @@
+# -*- test-case-name: twisted.protocols.haproxy.test.test_v1parser -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+IProxyParser implementation for version one of the PROXY protocol.
+"""
+from typing import Tuple, Union
+
+from zope.interface import implementer
+
+from twisted.internet import address
+from . import _info, _interfaces
+from ._exceptions import (
+    InvalidNetworkProtocol,
+    InvalidProxyHeader,
+    MissingAddressData,
+    convertError,
+)
+
+
+@implementer(_interfaces.IProxyParser)
+class V1Parser:
+    """
+    PROXY protocol version one header parser.
+
+    Version one of the PROXY protocol is a human readable format represented
+    by a single, newline delimited binary string that contains all of the
+    relevant source and destination data.
+    """
+
+    PROXYSTR = b"PROXY"
+    UNKNOWN_PROTO = b"UNKNOWN"
+    TCP4_PROTO = b"TCP4"
+    TCP6_PROTO = b"TCP6"
+    ALLOWED_NET_PROTOS = (
+        TCP4_PROTO,
+        TCP6_PROTO,
+        UNKNOWN_PROTO,
+    )
+    NEWLINE = b"\r\n"
+
+    def __init__(self) -> None:
+        self.buffer = b""
+
+    def feed(
+        self, data: bytes
+    ) -> Union[Tuple[_info.ProxyInfo, bytes], Tuple[None, None]]:
+        """
+        Consume a chunk of data and attempt to parse it.
+
+        @param data: A bytestring.
+        @type data: L{bytes}
+
+        @return: A two-tuple containing, in order, a
+            L{_interfaces.IProxyInfo} and any bytes fed to the
+            parser that followed the end of the header.  Both of these values
+            are None until a complete header is parsed.
+
+        @raises InvalidProxyHeader: If the bytes fed to the parser create an
+            invalid PROXY header.
+        """
+        self.buffer += data
+        if len(self.buffer) > 107 and self.NEWLINE not in self.buffer:
+            raise InvalidProxyHeader()
+        lines = (self.buffer).split(self.NEWLINE, 1)
+        if not len(lines) > 1:
+            return (None, None)
+        self.buffer = b""
+        remaining = lines.pop()
+        header = lines.pop()
+        info = self.parse(header)
+        return (info, remaining)
+
+    @classmethod
+    def parse(cls, line: bytes) -> _info.ProxyInfo:
+        """
+        Parse a bytestring as a full PROXY protocol header line.
+
+        @param line: A bytestring that represents a valid HAProxy PROXY
+            protocol header line.
+        @type line: bytes
+
+        @return: A L{_interfaces.IProxyInfo} containing the parsed data.
+
+        @raises InvalidProxyHeader: If the bytestring does not represent a
+            valid PROXY header.
+
+        @raises InvalidNetworkProtocol: When no protocol can be parsed or is
+            not one of the allowed values.
+
+        @raises MissingAddressData: When the protocol is TCP* but the header
+            does not contain a complete set of addresses and ports.
+        """
+        originalLine = line
+        proxyStr = None
+        networkProtocol = None
+        sourceAddr = None
+        sourcePort = None
+        destAddr = None
+        destPort = None
+
+        with convertError(ValueError, InvalidProxyHeader):
+            proxyStr, line = line.split(b" ", 1)
+
+        if proxyStr != cls.PROXYSTR:
+            raise InvalidProxyHeader()
+
+        with convertError(ValueError, InvalidNetworkProtocol):
+            networkProtocol, line = line.split(b" ", 1)
+
+        if networkProtocol not in cls.ALLOWED_NET_PROTOS:
+            raise InvalidNetworkProtocol()
+
+        if networkProtocol == cls.UNKNOWN_PROTO:
+            return _info.ProxyInfo(originalLine, None, None)
+
+        with convertError(ValueError, MissingAddressData):
+            sourceAddr, line = line.split(b" ", 1)
+
+        with convertError(ValueError, MissingAddressData):
+            destAddr, line = line.split(b" ", 1)
+
+        with convertError(ValueError, MissingAddressData):
+            sourcePort, line = line.split(b" ", 1)
+
+        with convertError(ValueError, MissingAddressData):
+            destPort = line.split(b" ")[0]
+
+        if networkProtocol == cls.TCP4_PROTO:
+            return _info.ProxyInfo(
+                originalLine,
+                address.IPv4Address("TCP", sourceAddr.decode(), int(sourcePort)),
+                address.IPv4Address("TCP", destAddr.decode(), int(destPort)),
+            )
+
+        return _info.ProxyInfo(
+            originalLine,
+            address.IPv6Address("TCP", sourceAddr.decode(), int(sourcePort)),
+            address.IPv6Address("TCP", destAddr.decode(), int(destPort)),
+        )
diff --git a/contrib/python/Twisted/py3/twisted/protocols/haproxy/_v2parser.py b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_v2parser.py
new file mode 100644
index 00000000000..5b8e5874018
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_v2parser.py
@@ -0,0 +1,217 @@
+# -*- test-case-name: twisted.protocols.haproxy.test.test_v2parser -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+IProxyParser implementation for version two of the PROXY protocol.
+"""
+
+import binascii
+import struct
+from typing import Callable, Tuple, Type, Union
+
+from zope.interface import implementer
+
+from constantly import ValueConstant, Values  # type: ignore[import]
+from typing_extensions import Literal
+
+from twisted.internet import address
+from twisted.python import compat
+from . import _info, _interfaces
+from ._exceptions import (
+    InvalidNetworkProtocol,
+    InvalidProxyHeader,
+    MissingAddressData,
+    convertError,
+)
+
+
+class NetFamily(Values):
+    """
+    Values for the 'family' field.
+    """
+
+    UNSPEC = ValueConstant(0x00)
+    INET = ValueConstant(0x10)
+    INET6 = ValueConstant(0x20)
+    UNIX = ValueConstant(0x30)
+
+
+class NetProtocol(Values):
+    """
+    Values for 'protocol' field.
+    """
+
+    UNSPEC = ValueConstant(0)
+    STREAM = ValueConstant(1)
+    DGRAM = ValueConstant(2)
+
+
+_HIGH = 0b11110000
+_LOW = 0b00001111
+_LOCALCOMMAND = "LOCAL"
+_PROXYCOMMAND = "PROXY"
+
+
+@implementer(_interfaces.IProxyParser)
+class V2Parser:
+    """
+    PROXY protocol version two header parser.
+
+    Version two of the PROXY protocol is a binary format.
+    """
+
+    PREFIX = b"\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A"
+    VERSIONS = [32]
+    COMMANDS = {0: _LOCALCOMMAND, 1: _PROXYCOMMAND}
+    ADDRESSFORMATS = {
+        # TCP4
+        17: "!4s4s2H",
+        18: "!4s4s2H",
+        # TCP6
+        33: "!16s16s2H",
+        34: "!16s16s2H",
+        # UNIX
+        49: "!108s108s",
+        50: "!108s108s",
+    }
+
+    def __init__(self) -> None:
+        self.buffer = b""
+
+    def feed(
+        self, data: bytes
+    ) -> Union[Tuple[_info.ProxyInfo, bytes], Tuple[None, None]]:
+        """
+        Consume a chunk of data and attempt to parse it.
+
+        @param data: A bytestring.
+        @type data: bytes
+
+        @return: A two-tuple containing, in order, a L{_interfaces.IProxyInfo}
+            and any bytes fed to the parser that followed the end of the
+            header.  Both of these values are None until a complete header is
+            parsed.
+
+        @raises InvalidProxyHeader: If the bytes fed to the parser create an
+            invalid PROXY header.
+        """
+        self.buffer += data
+        if len(self.buffer) < 16:
+            raise InvalidProxyHeader()
+
+        size = struct.unpack("!H", self.buffer[14:16])[0] + 16
+        if len(self.buffer) < size:
+            return (None, None)
+
+        header, remaining = self.buffer[:size], self.buffer[size:]
+        self.buffer = b""
+        info = self.parse(header)
+        return (info, remaining)
+
+    @staticmethod
+    def _bytesToIPv4(bytestring: bytes) -> bytes:
+        """
+        Convert packed 32-bit IPv4 address bytes into a dotted-quad ASCII bytes
+        representation of that address.
+
+        @param bytestring: 4 octets representing an IPv4 address.
+        @type bytestring: L{bytes}
+
+        @return: a dotted-quad notation IPv4 address.
+        @rtype: L{bytes}
+        """
+        return b".".join(
+            ("%i" % (ord(b),)).encode("ascii") for b in compat.iterbytes(bytestring)
+        )
+
+    @staticmethod
+    def _bytesToIPv6(bytestring: bytes) -> bytes:
+        """
+        Convert packed 128-bit IPv6 address bytes into a colon-separated ASCII
+        bytes representation of that address.
+
+        @param bytestring: 16 octets representing an IPv6 address.
+        @type bytestring: L{bytes}
+
+        @return: a dotted-quad notation IPv6 address.
+        @rtype: L{bytes}
+        """
+        hexString = binascii.b2a_hex(bytestring)
+        return b":".join(
+            (f"{int(hexString[b : b + 4], 16):x}").encode("ascii")
+            for b in range(0, 32, 4)
+        )
+
+    @classmethod
+    def parse(cls, line: bytes) -> _info.ProxyInfo:
+        """
+        Parse a bytestring as a full PROXY protocol header.
+
+        @param line: A bytestring that represents a valid HAProxy PROXY
+            protocol version 2 header.
+        @type line: bytes
+
+        @return: A L{_interfaces.IProxyInfo} containing the
+            parsed data.
+
+        @raises InvalidProxyHeader: If the bytestring does not represent a
+            valid PROXY header.
+        """
+        prefix = line[:12]
+        addrInfo = None
+        with convertError(IndexError, InvalidProxyHeader):
+            # Use single value slices to ensure bytestring values are returned
+            # instead of int in PY3.
+            versionCommand = ord(line[12:13])
+            familyProto = ord(line[13:14])
+
+        if prefix != cls.PREFIX:
+            raise InvalidProxyHeader()
+
+        version, command = versionCommand & _HIGH, versionCommand & _LOW
+        if version not in cls.VERSIONS or command not in cls.COMMANDS:
+            raise InvalidProxyHeader()
+
+        if cls.COMMANDS[command] == _LOCALCOMMAND:
+            return _info.ProxyInfo(line, None, None)
+
+        family, netproto = familyProto & _HIGH, familyProto & _LOW
+        with convertError(ValueError, InvalidNetworkProtocol):
+            family = NetFamily.lookupByValue(family)
+            netproto = NetProtocol.lookupByValue(netproto)
+        if family is NetFamily.UNSPEC or netproto is NetProtocol.UNSPEC:
+            return _info.ProxyInfo(line, None, None)
+
+        addressFormat = cls.ADDRESSFORMATS[familyProto]
+        addrInfo = line[16 : 16 + struct.calcsize(addressFormat)]
+        if family is NetFamily.UNIX:
+            with convertError(struct.error, MissingAddressData):
+                source, dest = struct.unpack(addressFormat, addrInfo)
+            return _info.ProxyInfo(
+                line,
+                address.UNIXAddress(source.rstrip(b"\x00")),
+                address.UNIXAddress(dest.rstrip(b"\x00")),
+            )
+
+        addrType: Union[Literal["TCP"], Literal["UDP"]] = "TCP"
+        if netproto is NetProtocol.DGRAM:
+            addrType = "UDP"
+        addrCls: Union[
+            Type[address.IPv4Address], Type[address.IPv6Address]
+        ] = address.IPv4Address
+        addrParser: Callable[[bytes], bytes] = cls._bytesToIPv4
+        if family is NetFamily.INET6:
+            addrCls = address.IPv6Address
+            addrParser = cls._bytesToIPv6
+
+        with convertError(struct.error, MissingAddressData):
+            info = struct.unpack(addressFormat, addrInfo)
+            source, dest, sPort, dPort = info
+
+        return _info.ProxyInfo(
+            line,
+            addrCls(addrType, addrParser(source).decode(), sPort),
+            addrCls(addrType, addrParser(dest).decode(), dPort),
+        )
diff --git a/contrib/python/Twisted/py3/twisted/protocols/haproxy/_wrapper.py b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_wrapper.py
new file mode 100644
index 00000000000..935dbfa9e20
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/haproxy/_wrapper.py
@@ -0,0 +1,109 @@
+# -*- test-case-name: twisted.protocols.haproxy.test.test_wrapper -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Protocol wrapper that provides HAProxy PROXY protocol support.
+"""
+from typing import Optional, Union
+
+from twisted.internet import interfaces
+from twisted.internet.endpoints import _WrapperServerEndpoint
+from twisted.protocols import policies
+from . import _info
+from ._exceptions import InvalidProxyHeader
+from ._v1parser import V1Parser
+from ._v2parser import V2Parser
+
+
+class HAProxyProtocolWrapper(policies.ProtocolWrapper):
+    """
+    A Protocol wrapper that provides HAProxy support.
+
+    This protocol reads the PROXY stream header, v1 or v2, parses the provided
+    connection data, and modifies the behavior of getPeer and getHost to return
+    the data provided by the PROXY header.
+    """
+
+    def __init__(
+        self, factory: policies.WrappingFactory, wrappedProtocol: interfaces.IProtocol
+    ):
+        super().__init__(factory, wrappedProtocol)
+        self._proxyInfo: Optional[_info.ProxyInfo] = None
+        self._parser: Union[V2Parser, V1Parser, None] = None
+
+    def dataReceived(self, data: bytes) -> None:
+        if self._proxyInfo is not None:
+            return self.wrappedProtocol.dataReceived(data)
+
+        parser = self._parser
+        if parser is None:
+            if (
+                len(data) >= 16
+                and data[:12] == V2Parser.PREFIX
+                and ord(data[12:13]) & 0b11110000 == 0x20
+            ):
+                self._parser = parser = V2Parser()
+            elif len(data) >= 8 and data[:5] == V1Parser.PROXYSTR:
+                self._parser = parser = V1Parser()
+            else:
+                self.loseConnection()
+                return None
+
+        try:
+            self._proxyInfo, remaining = parser.feed(data)
+            if remaining:
+                self.wrappedProtocol.dataReceived(remaining)
+        except InvalidProxyHeader:
+            self.loseConnection()
+
+    def getPeer(self) -> interfaces.IAddress:
+        if self._proxyInfo and self._proxyInfo.source:
+            return self._proxyInfo.source
+        assert self.transport
+        return self.transport.getPeer()
+
+    def getHost(self) -> interfaces.IAddress:
+        if self._proxyInfo and self._proxyInfo.destination:
+            return self._proxyInfo.destination
+        assert self.transport
+        return self.transport.getHost()
+
+
+class HAProxyWrappingFactory(policies.WrappingFactory):
+    """
+    A Factory wrapper that adds PROXY protocol support to connections.
+    """
+
+    protocol = HAProxyProtocolWrapper
+
+    def logPrefix(self) -> str:
+        """
+        Annotate the wrapped factory's log prefix with some text indicating
+        the PROXY protocol is in use.
+
+        @rtype: C{str}
+        """
+        if interfaces.ILoggingContext.providedBy(self.wrappedFactory):
+            logPrefix = self.wrappedFactory.logPrefix()
+        else:
+            logPrefix = self.wrappedFactory.__class__.__name__
+        return f"{logPrefix} (PROXY)"
+
+
+def proxyEndpoint(
+    wrappedEndpoint: interfaces.IStreamServerEndpoint,
+) -> _WrapperServerEndpoint:
+    """
+    Wrap an endpoint with PROXY protocol support, so that the transport's
+    C{getHost} and C{getPeer} methods reflect the attributes of the proxied
+    connection rather than the underlying connection.
+
+    @param wrappedEndpoint: The underlying listening endpoint.
+    @type wrappedEndpoint: L{IStreamServerEndpoint}
+
+    @return: a new listening endpoint that speaks the PROXY protocol.
+    @rtype: L{IStreamServerEndpoint}
+    """
+    return _WrapperServerEndpoint(wrappedEndpoint, HAProxyWrappingFactory)
diff --git a/contrib/python/Twisted/py3/twisted/protocols/htb.py b/contrib/python/Twisted/py3/twisted/protocols/htb.py
new file mode 100644
index 00000000000..66409d36fe6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/htb.py
@@ -0,0 +1,306 @@
+# -*- test-case-name: twisted.test.test_htb -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Hierarchical Token Bucket traffic shaping.
+
+Patterned after U{Martin Devera's Hierarchical Token Bucket traffic
+shaper for the Linux kernel<http://luxik.cdi.cz/~devik/qos/htb/>}.
+
+@seealso: U{HTB Linux queuing discipline manual - user guide
+  <http://luxik.cdi.cz/~devik/qos/htb/manual/userg.htm>}
+@seealso: U{Token Bucket Filter in Linux Advanced Routing & Traffic Control
+    HOWTO<http://lartc.org/howto/lartc.qdisc.classless.html#AEN682>}
+"""
+
+
+# TODO: Investigate whether we should be using os.times()[-1] instead of
+# time.time.  time.time, it has been pointed out, can go backwards.  Is
+# the same true of os.times?
+from time import time
+from typing import Optional
+
+from zope.interface import Interface, implementer
+
+from twisted.protocols import pcp
+
+
+class Bucket:
+    """
+    Implementation of a Token bucket.
+
+    A bucket can hold a certain number of tokens and it drains over time.
+
+    @cvar maxburst: The maximum number of tokens that the bucket can
+        hold at any given time. If this is L{None}, the bucket has
+        an infinite size.
+    @type maxburst: C{int}
+    @cvar rate: The rate at which the bucket drains, in number
+        of tokens per second. If the rate is L{None}, the bucket
+        drains instantaneously.
+    @type rate: C{int}
+    """
+
+    maxburst: Optional[int] = None
+    rate: Optional[int] = None
+
+    _refcount = 0
+
+    def __init__(self, parentBucket=None):
+        """
+        Create a L{Bucket} that may have a parent L{Bucket}.
+
+        @param parentBucket: If a parent Bucket is specified,
+            all L{add} and L{drip} operations on this L{Bucket}
+            will be applied on the parent L{Bucket} as well.
+        @type parentBucket: L{Bucket}
+        """
+        self.content = 0
+        self.parentBucket = parentBucket
+        self.lastDrip = time()
+
+    def add(self, amount):
+        """
+        Adds tokens to the L{Bucket} and its C{parentBucket}.
+
+        This will add as many of the C{amount} tokens as will fit into both
+        this L{Bucket} and its C{parentBucket}.
+
+        @param amount: The number of tokens to try to add.
+        @type amount: C{int}
+
+        @returns: The number of tokens that actually fit.
+        @returntype: C{int}
+        """
+        self.drip()
+        if self.maxburst is None:
+            allowable = amount
+        else:
+            allowable = min(amount, self.maxburst - self.content)
+
+        if self.parentBucket is not None:
+            allowable = self.parentBucket.add(allowable)
+        self.content += allowable
+        return allowable
+
+    def drip(self):
+        """
+        Let some of the bucket drain.
+
+        The L{Bucket} drains at the rate specified by the class
+        variable C{rate}.
+
+        @returns: C{True} if the bucket is empty after this drip.
+        @returntype: C{bool}
+        """
+        if self.parentBucket is not None:
+            self.parentBucket.drip()
+
+        if self.rate is None:
+            self.content = 0
+        else:
+            now = time()
+            deltaTime = now - self.lastDrip
+            deltaTokens = deltaTime * self.rate
+            self.content = max(0, self.content - deltaTokens)
+            self.lastDrip = now
+        return self.content == 0
+
+
+class IBucketFilter(Interface):
+    def getBucketFor(*somethings, **some_kw):
+        """
+        Return a L{Bucket} corresponding to the provided parameters.
+
+        @returntype: L{Bucket}
+        """
+
+
+@implementer(IBucketFilter)
+class HierarchicalBucketFilter:
+    """
+    Filter things into buckets that can be nested.
+
+    @cvar bucketFactory: Class of buckets to make.
+    @type bucketFactory: L{Bucket}
+    @cvar sweepInterval: Seconds between sweeping out the bucket cache.
+    @type sweepInterval: C{int}
+    """
+
+    bucketFactory = Bucket
+    sweepInterval: Optional[int] = None
+
+    def __init__(self, parentFilter=None):
+        self.buckets = {}
+        self.parentFilter = parentFilter
+        self.lastSweep = time()
+
+    def getBucketFor(self, *a, **kw):
+        """
+        Find or create a L{Bucket} corresponding to the provided parameters.
+
+        Any parameters are passed on to L{getBucketKey}, from them it
+        decides which bucket you get.
+
+        @returntype: L{Bucket}
+        """
+        if (self.sweepInterval is not None) and (
+            (time() - self.lastSweep) > self.sweepInterval
+        ):
+            self.sweep()
+
+        if self.parentFilter:
+            parentBucket = self.parentFilter.getBucketFor(self, *a, **kw)
+        else:
+            parentBucket = None
+
+        key = self.getBucketKey(*a, **kw)
+        bucket = self.buckets.get(key)
+        if bucket is None:
+            bucket = self.bucketFactory(parentBucket)
+            self.buckets[key] = bucket
+        return bucket
+
+    def getBucketKey(self, *a, **kw):
+        """
+        Construct a key based on the input parameters to choose a L{Bucket}.
+
+        The default implementation returns the same key for all
+        arguments. Override this method to provide L{Bucket} selection.
+
+        @returns: Something to be used as a key in the bucket cache.
+        """
+        return None
+
+    def sweep(self):
+        """
+        Remove empty buckets.
+        """
+        for key, bucket in self.buckets.items():
+            bucket_is_empty = bucket.drip()
+            if (bucket._refcount == 0) and bucket_is_empty:
+                del self.buckets[key]
+
+        self.lastSweep = time()
+
+
+class FilterByHost(HierarchicalBucketFilter):
+    """
+    A Hierarchical Bucket filter with a L{Bucket} for each host.
+    """
+
+    sweepInterval = 60 * 20
+
+    def getBucketKey(self, transport):
+        return transport.getPeer()[1]
+
+
+class FilterByServer(HierarchicalBucketFilter):
+    """
+    A Hierarchical Bucket filter with a L{Bucket} for each service.
+    """
+
+    sweepInterval = None
+
+    def getBucketKey(self, transport):
+        return transport.getHost()[2]
+
+
+class ShapedConsumer(pcp.ProducerConsumerProxy):
+    """
+    Wraps a C{Consumer} and shapes the rate at which it receives data.
+    """
+
+    # Providing a Pull interface means I don't have to try to schedule
+    # traffic with callLaters.
+    iAmStreaming = False
+
+    def __init__(self, consumer, bucket):
+        pcp.ProducerConsumerProxy.__init__(self, consumer)
+        self.bucket = bucket
+        self.bucket._refcount += 1
+
+    def _writeSomeData(self, data):
+        # In practice, this actually results in obscene amounts of
+        # overhead, as a result of generating lots and lots of packets
+        # with twelve-byte payloads.  We may need to do a version of
+        # this with scheduled writes after all.
+        amount = self.bucket.add(len(data))
+        return pcp.ProducerConsumerProxy._writeSomeData(self, data[:amount])
+
+    def stopProducing(self):
+        pcp.ProducerConsumerProxy.stopProducing(self)
+        self.bucket._refcount -= 1
+
+
+class ShapedTransport(ShapedConsumer):
+    """
+    Wraps a C{Transport} and shapes the rate at which it receives data.
+
+    This is a L{ShapedConsumer} with a little bit of magic to provide for
+    the case where the consumer it wraps is also a C{Transport} and people
+    will be attempting to access attributes this does not proxy as a
+    C{Consumer} (e.g. C{loseConnection}).
+    """
+
+    # Ugh.  We only wanted to filter IConsumer, not ITransport.
+
+    iAmStreaming = False
+
+    def __getattr__(self, name):
+        # Because people will be doing things like .getPeer and
+        # .loseConnection on me.
+        return getattr(self.consumer, name)
+
+
+class ShapedProtocolFactory:
+    """
+    Dispense C{Protocols} with traffic shaping on their transports.
+
+    Usage::
+
+        myserver = SomeFactory()
+        myserver.protocol = ShapedProtocolFactory(myserver.protocol,
+                                                  bucketFilter)
+
+    Where C{SomeServerFactory} is a L{twisted.internet.protocol.Factory}, and
+    C{bucketFilter} is an instance of L{HierarchicalBucketFilter}.
+    """
+
+    def __init__(self, protoClass, bucketFilter):
+        """
+        Tell me what to wrap and where to get buckets.
+
+        @param protoClass: The class of C{Protocol} this will generate
+          wrapped instances of.
+        @type protoClass: L{Protocol<twisted.internet.interfaces.IProtocol>}
+          class
+        @param bucketFilter: The filter which will determine how
+          traffic is shaped.
+        @type bucketFilter: L{HierarchicalBucketFilter}.
+        """
+        # More precisely, protoClass can be any callable that will return
+        # instances of something that implements IProtocol.
+        self.protocol = protoClass
+        self.bucketFilter = bucketFilter
+
+    def __call__(self, *a, **kw):
+        """
+        Make a C{Protocol} instance with a shaped transport.
+
+        Any parameters will be passed on to the protocol's initializer.
+
+        @returns: A C{Protocol} instance with a L{ShapedTransport}.
+        """
+        proto = self.protocol(*a, **kw)
+        origMakeConnection = proto.makeConnection
+
+        def makeConnection(transport):
+            bucket = self.bucketFilter.getBucketFor(transport)
+            shapedTransport = ShapedTransport(transport, bucket)
+            return origMakeConnection(shapedTransport)
+
+        proto.makeConnection = makeConnection
+        return proto
diff --git a/contrib/python/Twisted/py3/twisted/protocols/ident.py b/contrib/python/Twisted/py3/twisted/protocols/ident.py
new file mode 100644
index 00000000000..4401369119a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/ident.py
@@ -0,0 +1,253 @@
+# -*- test-case-name: twisted.test.test_ident -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Ident protocol implementation.
+"""
+
+import struct
+
+from twisted.internet import defer
+from twisted.protocols import basic
+from twisted.python import failure, log
+
+_MIN_PORT = 1
+_MAX_PORT = 2**16 - 1
+
+
+class IdentError(Exception):
+    """
+    Can't determine connection owner; reason unknown.
+    """
+
+    identDescription = "UNKNOWN-ERROR"
+
+    def __str__(self) -> str:
+        return self.identDescription
+
+
+class NoUser(IdentError):
+    """
+    The connection specified by the port pair is not currently in use or
+    currently not owned by an identifiable entity.
+    """
+
+    identDescription = "NO-USER"
+
+
+class InvalidPort(IdentError):
+    """
+    Either the local or foreign port was improperly specified. This should
+    be returned if either or both of the port ids were out of range (TCP
+    port numbers are from 1-65535), negative integers, reals or in any
+    fashion not recognized as a non-negative integer.
+    """
+
+    identDescription = "INVALID-PORT"
+
+
+class HiddenUser(IdentError):
+    """
+    The server was able to identify the user of this port, but the
+    information was not returned at the request of the user.
+    """
+
+    identDescription = "HIDDEN-USER"
+
+
+class IdentServer(basic.LineOnlyReceiver):
+    """
+    The Identification Protocol (a.k.a., "ident", a.k.a., "the Ident
+    Protocol") provides a means to determine the identity of a user of a
+    particular TCP connection. Given a TCP port number pair, it returns a
+    character string which identifies the owner of that connection on the
+    server's system.
+
+    Server authors should subclass this class and override the lookup method.
+    The default implementation returns an UNKNOWN-ERROR response for every
+    query.
+    """
+
+    def lineReceived(self, line):
+        parts = line.split(",")
+        if len(parts) != 2:
+            self.invalidQuery()
+        else:
+            try:
+                portOnServer, portOnClient = map(int, parts)
+            except ValueError:
+                self.invalidQuery()
+            else:
+                if (
+                    _MIN_PORT <= portOnServer <= _MAX_PORT
+                    and _MIN_PORT <= portOnClient <= _MAX_PORT
+                ):
+                    self.validQuery(portOnServer, portOnClient)
+                else:
+                    self._ebLookup(
+                        failure.Failure(InvalidPort()), portOnServer, portOnClient
+                    )
+
+    def invalidQuery(self):
+        self.transport.loseConnection()
+
+    def validQuery(self, portOnServer, portOnClient):
+        """
+        Called when a valid query is received to look up and deliver the
+        response.
+
+        @param portOnServer: The server port from the query.
+        @param portOnClient: The client port from the query.
+        """
+        serverAddr = self.transport.getHost().host, portOnServer
+        clientAddr = self.transport.getPeer().host, portOnClient
+        defer.maybeDeferred(self.lookup, serverAddr, clientAddr).addCallback(
+            self._cbLookup, portOnServer, portOnClient
+        ).addErrback(self._ebLookup, portOnServer, portOnClient)
+
+    def _cbLookup(self, result, sport, cport):
+        (sysName, userId) = result
+        self.sendLine("%d, %d : USERID : %s : %s" % (sport, cport, sysName, userId))
+
+    def _ebLookup(self, failure, sport, cport):
+        if failure.check(IdentError):
+            self.sendLine("%d, %d : ERROR : %s" % (sport, cport, failure.value))
+        else:
+            log.err(failure)
+            self.sendLine(
+                "%d, %d : ERROR : %s" % (sport, cport, IdentError(failure.value))
+            )
+
+    def lookup(self, serverAddress, clientAddress):
+        """
+        Lookup user information about the specified address pair.
+
+        Return value should be a two-tuple of system name and username.
+        Acceptable values for the system name may be found online at::
+
+            U{http://www.iana.org/assignments/operating-system-names}
+
+        This method may also raise any IdentError subclass (or IdentError
+        itself) to indicate user information will not be provided for the
+        given query.
+
+        A Deferred may also be returned.
+
+        @param serverAddress: A two-tuple representing the server endpoint
+        of the address being queried.  The first element is a string holding
+        a dotted-quad IP address.  The second element is an integer
+        representing the port.
+
+        @param clientAddress: Like I{serverAddress}, but represents the
+        client endpoint of the address being queried.
+        """
+        raise IdentError()
+
+
+class ProcServerMixin:
+    """Implements lookup() to grab entries for responses from /proc/net/tcp"""
+
+    SYSTEM_NAME = "LINUX"
+
+    try:
+        from pwd import getpwuid  # type:ignore[misc]
+
+        def getUsername(self, uid, getpwuid=getpwuid):
+            return getpwuid(uid)[0]
+
+        del getpwuid
+    except ImportError:
+
+        def getUsername(self, uid, getpwuid=None):
+            raise IdentError()
+
+    def entries(self):
+        with open("/proc/net/tcp") as f:
+            f.readline()
+            for L in f:
+                yield L.strip()
+
+    def dottedQuadFromHexString(self, hexstr):
+        return ".".join(
+            map(str, struct.unpack("4B", struct.pack("=L", int(hexstr, 16))))
+        )
+
+    def unpackAddress(self, packed):
+        addr, port = packed.split(":")
+        addr = self.dottedQuadFromHexString(addr)
+        port = int(port, 16)
+        return addr, port
+
+    def parseLine(self, line):
+        parts = line.strip().split()
+        localAddr, localPort = self.unpackAddress(parts[1])
+        remoteAddr, remotePort = self.unpackAddress(parts[2])
+        uid = int(parts[7])
+        return (localAddr, localPort), (remoteAddr, remotePort), uid
+
+    def lookup(self, serverAddress, clientAddress):
+        for ent in self.entries():
+            localAddr, remoteAddr, uid = self.parseLine(ent)
+            if remoteAddr == clientAddress and localAddr[1] == serverAddress[1]:
+                return (self.SYSTEM_NAME, self.getUsername(uid))
+
+        raise NoUser()
+
+
+class IdentClient(basic.LineOnlyReceiver):
+    errorTypes = (IdentError, NoUser, InvalidPort, HiddenUser)
+
+    def __init__(self):
+        self.queries = []
+
+    def lookup(self, portOnServer, portOnClient):
+        """
+        Lookup user information about the specified address pair.
+        """
+        self.queries.append((defer.Deferred(), portOnServer, portOnClient))
+        if len(self.queries) > 1:
+            return self.queries[-1][0]
+
+        self.sendLine("%d, %d" % (portOnServer, portOnClient))
+        return self.queries[-1][0]
+
+    def lineReceived(self, line):
+        if not self.queries:
+            log.msg(f"Unexpected server response: {line!r}")
+        else:
+            d, _, _ = self.queries.pop(0)
+            self.parseResponse(d, line)
+            if self.queries:
+                self.sendLine("%d, %d" % (self.queries[0][1], self.queries[0][2]))
+
+    def connectionLost(self, reason):
+        for q in self.queries:
+            q[0].errback(IdentError(reason))
+        self.queries = []
+
+    def parseResponse(self, deferred, line):
+        parts = line.split(":", 2)
+        if len(parts) != 3:
+            deferred.errback(IdentError(line))
+        else:
+            ports, type, addInfo = map(str.strip, parts)
+            if type == "ERROR":
+                for et in self.errorTypes:
+                    if et.identDescription == addInfo:
+                        deferred.errback(et(line))
+                        return
+                deferred.errback(IdentError(line))
+            else:
+                deferred.callback((type, addInfo))
+
+
+__all__ = [
+    "IdentError",
+    "NoUser",
+    "InvalidPort",
+    "HiddenUser",
+    "IdentServer",
+    "IdentClient",
+    "ProcServerMixin",
+]
diff --git a/contrib/python/Twisted/py3/twisted/protocols/loopback.py b/contrib/python/Twisted/py3/twisted/protocols/loopback.py
new file mode 100644
index 00000000000..c3d6226e5b2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/loopback.py
@@ -0,0 +1,387 @@
+# -*- test-case-name: twisted.test.test_loopback -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Testing support for protocols -- loopback between client and server.
+"""
+
+
+# system imports
+import tempfile
+
+from zope.interface import implementer
+
+from twisted.internet import defer, interfaces, main, protocol
+from twisted.internet.interfaces import IAddress
+from twisted.internet.task import deferLater
+
+# Twisted Imports
+from twisted.protocols import policies
+from twisted.python import failure
+
+
+class _LoopbackQueue:
+    """
+    Trivial wrapper around a list to give it an interface like a queue, which
+    the addition of also sending notifications by way of a Deferred whenever
+    the list has something added to it.
+    """
+
+    _notificationDeferred = None
+    disconnect = False
+
+    def __init__(self):
+        self._queue = []
+
+    def put(self, v):
+        self._queue.append(v)
+        if self._notificationDeferred is not None:
+            d, self._notificationDeferred = self._notificationDeferred, None
+            d.callback(None)
+
+    def __nonzero__(self):
+        return bool(self._queue)
+
+    __bool__ = __nonzero__
+
+    def get(self):
+        return self._queue.pop(0)
+
+
+@implementer(IAddress)
+class _LoopbackAddress:
+    pass
+
+
+@implementer(interfaces.ITransport, interfaces.IConsumer)
+class _LoopbackTransport:
+    disconnecting = False
+    producer = None
+
+    # ITransport
+    def __init__(self, q):
+        self.q = q
+
+    def write(self, data):
+        if not isinstance(data, bytes):
+            raise TypeError("Can only write bytes to ITransport")
+        self.q.put(data)
+
+    def writeSequence(self, iovec):
+        self.q.put(b"".join(iovec))
+
+    def loseConnection(self):
+        self.q.disconnect = True
+        self.q.put(None)
+
+    def abortConnection(self):
+        """
+        Abort the connection. Same as L{loseConnection}.
+        """
+        self.loseConnection()
+
+    def getPeer(self):
+        return _LoopbackAddress()
+
+    def getHost(self):
+        return _LoopbackAddress()
+
+    # IConsumer
+    def registerProducer(self, producer, streaming):
+        assert self.producer is None
+        self.producer = producer
+        self.streamingProducer = streaming
+        self._pollProducer()
+
+    def unregisterProducer(self):
+        assert self.producer is not None
+        self.producer = None
+
+    def _pollProducer(self):
+        if self.producer is not None and not self.streamingProducer:
+            self.producer.resumeProducing()
+
+
+def identityPumpPolicy(queue, target):
+    """
+    L{identityPumpPolicy} is a policy which delivers each chunk of data written
+    to the given queue as-is to the target.
+
+    This isn't a particularly realistic policy.
+
+    @see: L{loopbackAsync}
+    """
+    while queue:
+        bytes = queue.get()
+        if bytes is None:
+            break
+        target.dataReceived(bytes)
+
+
+def collapsingPumpPolicy(queue, target):
+    """
+    L{collapsingPumpPolicy} is a policy which collapses all outstanding chunks
+    into a single string and delivers it to the target.
+
+    @see: L{loopbackAsync}
+    """
+    bytes = []
+    while queue:
+        chunk = queue.get()
+        if chunk is None:
+            break
+        bytes.append(chunk)
+    if bytes:
+        target.dataReceived(b"".join(bytes))
+
+
+def loopbackAsync(server, client, pumpPolicy=identityPumpPolicy):
+    """
+    Establish a connection between C{server} and C{client} then transfer data
+    between them until the connection is closed. This is often useful for
+    testing a protocol.
+
+    @param server: The protocol instance representing the server-side of this
+        connection.
+
+    @param client: The protocol instance representing the client-side of this
+        connection.
+
+    @param pumpPolicy: When either C{server} or C{client} writes to its
+        transport, the string passed in is added to a queue of data for the
+        other protocol.  Eventually, C{pumpPolicy} will be called with one such
+        queue and the corresponding protocol object.  The pump policy callable
+        is responsible for emptying the queue and passing the strings it
+        contains to the given protocol's C{dataReceived} method.  The signature
+        of C{pumpPolicy} is C{(queue, protocol)}.  C{queue} is an object with a
+        C{get} method which will return the next string written to the
+        transport, or L{None} if the transport has been disconnected, and which
+        evaluates to C{True} if and only if there are more items to be
+        retrieved via C{get}.
+
+    @return: A L{Deferred} which fires when the connection has been closed and
+        both sides have received notification of this.
+    """
+    serverToClient = _LoopbackQueue()
+    clientToServer = _LoopbackQueue()
+
+    server.makeConnection(_LoopbackTransport(serverToClient))
+    client.makeConnection(_LoopbackTransport(clientToServer))
+
+    return _loopbackAsyncBody(
+        server, serverToClient, client, clientToServer, pumpPolicy
+    )
+
+
+def _loopbackAsyncBody(server, serverToClient, client, clientToServer, pumpPolicy):
+    """
+    Transfer bytes from the output queue of each protocol to the input of the other.
+
+    @param server: The protocol instance representing the server-side of this
+    connection.
+
+    @param serverToClient: The L{_LoopbackQueue} holding the server's output.
+
+    @param client: The protocol instance representing the client-side of this
+    connection.
+
+    @param clientToServer: The L{_LoopbackQueue} holding the client's output.
+
+    @param pumpPolicy: See L{loopbackAsync}.
+
+    @return: A L{Deferred} which fires when the connection has been closed and
+        both sides have received notification of this.
+    """
+
+    def pump(source, q, target):
+        sent = False
+        if q:
+            pumpPolicy(q, target)
+            sent = True
+        if sent and not q:
+            # A write buffer has now been emptied.  Give any producer on that
+            # side an opportunity to produce more data.
+            source.transport._pollProducer()
+
+        return sent
+
+    while 1:
+        disconnect = clientSent = serverSent = False
+
+        # Deliver the data which has been written.
+        serverSent = pump(server, serverToClient, client)
+        clientSent = pump(client, clientToServer, server)
+
+        if not clientSent and not serverSent:
+            # Neither side wrote any data.  Wait for some new data to be added
+            # before trying to do anything further.
+            d = defer.Deferred()
+            clientToServer._notificationDeferred = d
+            serverToClient._notificationDeferred = d
+            d.addCallback(
+                _loopbackAsyncContinue,
+                server,
+                serverToClient,
+                client,
+                clientToServer,
+                pumpPolicy,
+            )
+            return d
+        if serverToClient.disconnect:
+            # The server wants to drop the connection.  Flush any remaining
+            # data it has.
+            disconnect = True
+            pump(server, serverToClient, client)
+        elif clientToServer.disconnect:
+            # The client wants to drop the connection.  Flush any remaining
+            # data it has.
+            disconnect = True
+            pump(client, clientToServer, server)
+        if disconnect:
+            # Someone wanted to disconnect, so okay, the connection is gone.
+            server.connectionLost(failure.Failure(main.CONNECTION_DONE))
+            client.connectionLost(failure.Failure(main.CONNECTION_DONE))
+            return defer.succeed(None)
+
+
+def _loopbackAsyncContinue(
+    ignored, server, serverToClient, client, clientToServer, pumpPolicy
+):
+    # Clear the Deferred from each message queue, since it has already fired
+    # and cannot be used again.
+    clientToServer._notificationDeferred = None
+    serverToClient._notificationDeferred = None
+
+    # Schedule some more byte-pushing to happen.  This isn't done
+    # synchronously because no actual transport can re-enter dataReceived as
+    # a result of calling write, and doing this synchronously could result
+    # in that.
+    from twisted.internet import reactor
+
+    return deferLater(
+        reactor,
+        0,
+        _loopbackAsyncBody,
+        server,
+        serverToClient,
+        client,
+        clientToServer,
+        pumpPolicy,
+    )
+
+
+@implementer(interfaces.ITransport, interfaces.IConsumer)
+class LoopbackRelay:
+    buffer = b""
+    shouldLose = 0
+    disconnecting = 0
+    producer = None
+
+    def __init__(self, target, logFile=None):
+        self.target = target
+        self.logFile = logFile
+
+    def write(self, data):
+        self.buffer = self.buffer + data
+        if self.logFile:
+            self.logFile.write("loopback writing %s\n" % repr(data))
+
+    def writeSequence(self, iovec):
+        self.write(b"".join(iovec))
+
+    def clearBuffer(self):
+        if self.shouldLose == -1:
+            return
+
+        if self.producer:
+            self.producer.resumeProducing()
+        if self.buffer:
+            if self.logFile:
+                self.logFile.write("loopback receiving %s\n" % repr(self.buffer))
+            buffer = self.buffer
+            self.buffer = b""
+            self.target.dataReceived(buffer)
+        if self.shouldLose == 1:
+            self.shouldLose = -1
+            self.target.connectionLost(failure.Failure(main.CONNECTION_DONE))
+
+    def loseConnection(self):
+        if self.shouldLose != -1:
+            self.shouldLose = 1
+
+    def getHost(self):
+        return "loopback"
+
+    def getPeer(self):
+        return "loopback"
+
+    def registerProducer(self, producer, streaming):
+        self.producer = producer
+
+    def unregisterProducer(self):
+        self.producer = None
+
+    def logPrefix(self):
+        return f"Loopback({self.target.__class__.__name__!r})"
+
+
+class LoopbackClientFactory(protocol.ClientFactory):
+    def __init__(self, protocol):
+        self.disconnected = 0
+        self.deferred = defer.Deferred()
+        self.protocol = protocol
+
+    def buildProtocol(self, addr):
+        return self.protocol
+
+    def clientConnectionLost(self, connector, reason):
+        self.disconnected = 1
+        self.deferred.callback(None)
+
+
+class _FireOnClose(policies.ProtocolWrapper):
+    def __init__(self, protocol, factory):
+        policies.ProtocolWrapper.__init__(self, protocol, factory)
+        self.deferred = defer.Deferred()
+
+    def connectionLost(self, reason):
+        policies.ProtocolWrapper.connectionLost(self, reason)
+        self.deferred.callback(None)
+
+
+def loopbackTCP(server, client, port=0, noisy=True):
+    """Run session between server and client protocol instances over TCP."""
+    from twisted.internet import reactor
+
+    f = policies.WrappingFactory(protocol.Factory())
+    serverWrapper = _FireOnClose(f, server)
+    f.noisy = noisy
+    f.buildProtocol = lambda addr: serverWrapper
+    serverPort = reactor.listenTCP(port, f, interface="127.0.0.1")
+    clientF = LoopbackClientFactory(client)
+    clientF.noisy = noisy
+    reactor.connectTCP("127.0.0.1", serverPort.getHost().port, clientF)
+    d = clientF.deferred
+    d.addCallback(lambda x: serverWrapper.deferred)
+    d.addCallback(lambda x: serverPort.stopListening())
+    return d
+
+
+def loopbackUNIX(server, client, noisy=True):
+    """Run session between server and client protocol instances over UNIX socket."""
+    path = tempfile.mktemp()
+    from twisted.internet import reactor
+
+    f = policies.WrappingFactory(protocol.Factory())
+    serverWrapper = _FireOnClose(f, server)
+    f.noisy = noisy
+    f.buildProtocol = lambda addr: serverWrapper
+    serverPort = reactor.listenUNIX(path, f)
+    clientF = LoopbackClientFactory(client)
+    clientF.noisy = noisy
+    reactor.connectUNIX(path, clientF)
+    d = clientF.deferred
+    d.addCallback(lambda x: serverWrapper.deferred)
+    d.addCallback(lambda x: serverPort.stopListening())
+    return d
diff --git a/contrib/python/Twisted/py3/twisted/protocols/memcache.py b/contrib/python/Twisted/py3/twisted/protocols/memcache.py
new file mode 100644
index 00000000000..638d2f8a72c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/memcache.py
@@ -0,0 +1,733 @@
+# -*- test-case-name: twisted.test.test_memcache -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Memcache client protocol. Memcached is a caching server, storing data in the
+form of pairs key/value, and memcache is the protocol to talk with it.
+
+To connect to a server, create a factory for L{MemCacheProtocol}::
+
+    from twisted.internet import reactor, protocol
+    from twisted.protocols.memcache import MemCacheProtocol, DEFAULT_PORT
+    d = protocol.ClientCreator(reactor, MemCacheProtocol
+        ).connectTCP("localhost", DEFAULT_PORT)
+    def doSomething(proto):
+        # Here you call the memcache operations
+        return proto.set("mykey", "a lot of data")
+    d.addCallback(doSomething)
+    reactor.run()
+
+All the operations of the memcache protocol are present, but
+L{MemCacheProtocol.set} and L{MemCacheProtocol.get} are the more important.
+
+See U{http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt} for
+more information about the protocol.
+"""
+
+
+from collections import deque
+
+from twisted.internet.defer import Deferred, TimeoutError, fail
+from twisted.protocols.basic import LineReceiver
+from twisted.protocols.policies import TimeoutMixin
+from twisted.python import log
+from twisted.python.compat import nativeString, networkString
+
+DEFAULT_PORT = 11211
+
+
+class NoSuchCommand(Exception):
+    """
+    Exception raised when a non existent command is called.
+    """
+
+
+class ClientError(Exception):
+    """
+    Error caused by an invalid client call.
+    """
+
+
+class ServerError(Exception):
+    """
+    Problem happening on the server.
+    """
+
+
+class Command:
+    """
+    Wrap a client action into an object, that holds the values used in the
+    protocol.
+
+    @ivar _deferred: the L{Deferred} object that will be fired when the result
+        arrives.
+    @type _deferred: L{Deferred}
+
+    @ivar command: name of the command sent to the server.
+    @type command: L{bytes}
+    """
+
+    def __init__(self, command, **kwargs):
+        """
+        Create a command.
+
+        @param command: the name of the command.
+        @type command: L{bytes}
+
+        @param kwargs: this values will be stored as attributes of the object
+            for future use
+        """
+        self.command = command
+        self._deferred = Deferred()
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+    def success(self, value):
+        """
+        Shortcut method to fire the underlying deferred.
+        """
+        self._deferred.callback(value)
+
+    def fail(self, error):
+        """
+        Make the underlying deferred fails.
+        """
+        self._deferred.errback(error)
+
+
+class MemCacheProtocol(LineReceiver, TimeoutMixin):
+    """
+    MemCache protocol: connect to a memcached server to store/retrieve values.
+
+    @ivar persistentTimeOut: the timeout period used to wait for a response.
+    @type persistentTimeOut: L{int}
+
+    @ivar _current: current list of requests waiting for an answer from the
+        server.
+    @type _current: L{deque} of L{Command}
+
+    @ivar _lenExpected: amount of data expected in raw mode, when reading for
+        a value.
+    @type _lenExpected: L{int}
+
+    @ivar _getBuffer: current buffer of data, used to store temporary data
+        when reading in raw mode.
+    @type _getBuffer: L{list}
+
+    @ivar _bufferLength: the total amount of bytes in C{_getBuffer}.
+    @type _bufferLength: L{int}
+
+    @ivar _disconnected: indicate if the connectionLost has been called or not.
+    @type _disconnected: L{bool}
+    """
+
+    MAX_KEY_LENGTH = 250
+    _disconnected = False
+
+    def __init__(self, timeOut=60):
+        """
+        Create the protocol.
+
+        @param timeOut: the timeout to wait before detecting that the
+            connection is dead and close it. It's expressed in seconds.
+        @type timeOut: L{int}
+        """
+        self._current = deque()
+        self._lenExpected = None
+        self._getBuffer = None
+        self._bufferLength = None
+        self.persistentTimeOut = self.timeOut = timeOut
+
+    def _cancelCommands(self, reason):
+        """
+        Cancel all the outstanding commands, making them fail with C{reason}.
+        """
+        while self._current:
+            cmd = self._current.popleft()
+            cmd.fail(reason)
+
+    def timeoutConnection(self):
+        """
+        Close the connection in case of timeout.
+        """
+        self._cancelCommands(TimeoutError("Connection timeout"))
+        self.transport.loseConnection()
+
+    def connectionLost(self, reason):
+        """
+        Cause any outstanding commands to fail.
+        """
+        self._disconnected = True
+        self._cancelCommands(reason)
+        LineReceiver.connectionLost(self, reason)
+
+    def sendLine(self, line):
+        """
+        Override sendLine to add a timeout to response.
+        """
+        if not self._current:
+            self.setTimeout(self.persistentTimeOut)
+        LineReceiver.sendLine(self, line)
+
+    def rawDataReceived(self, data):
+        """
+        Collect data for a get.
+        """
+        self.resetTimeout()
+        self._getBuffer.append(data)
+        self._bufferLength += len(data)
+        if self._bufferLength >= self._lenExpected + 2:
+            data = b"".join(self._getBuffer)
+            buf = data[: self._lenExpected]
+            rem = data[self._lenExpected + 2 :]
+            val = buf
+            self._lenExpected = None
+            self._getBuffer = None
+            self._bufferLength = None
+            cmd = self._current[0]
+            if cmd.multiple:
+                flags, cas = cmd.values[cmd.currentKey]
+                cmd.values[cmd.currentKey] = (flags, cas, val)
+            else:
+                cmd.value = val
+            self.setLineMode(rem)
+
+    def cmd_STORED(self):
+        """
+        Manage a success response to a set operation.
+        """
+        self._current.popleft().success(True)
+
+    def cmd_NOT_STORED(self):
+        """
+        Manage a specific 'not stored' response to a set operation: this is not
+        an error, but some condition wasn't met.
+        """
+        self._current.popleft().success(False)
+
+    def cmd_END(self):
+        """
+        This the end token to a get or a stat operation.
+        """
+        cmd = self._current.popleft()
+        if cmd.command == b"get":
+            if cmd.multiple:
+                values = {key: val[::2] for key, val in cmd.values.items()}
+                cmd.success(values)
+            else:
+                cmd.success((cmd.flags, cmd.value))
+        elif cmd.command == b"gets":
+            if cmd.multiple:
+                cmd.success(cmd.values)
+            else:
+                cmd.success((cmd.flags, cmd.cas, cmd.value))
+        elif cmd.command == b"stats":
+            cmd.success(cmd.values)
+        else:
+            raise RuntimeError(
+                "Unexpected END response to {} command".format(
+                    nativeString(cmd.command)
+                )
+            )
+
+    def cmd_NOT_FOUND(self):
+        """
+        Manage error response for incr/decr/delete.
+        """
+        self._current.popleft().success(False)
+
+    def cmd_VALUE(self, line):
+        """
+        Prepare the reading a value after a get.
+        """
+        cmd = self._current[0]
+        if cmd.command == b"get":
+            key, flags, length = line.split()
+            cas = b""
+        else:
+            key, flags, length, cas = line.split()
+        self._lenExpected = int(length)
+        self._getBuffer = []
+        self._bufferLength = 0
+        if cmd.multiple:
+            if key not in cmd.keys:
+                raise RuntimeError("Unexpected commands answer.")
+            cmd.currentKey = key
+            cmd.values[key] = [int(flags), cas]
+        else:
+            if cmd.key != key:
+                raise RuntimeError("Unexpected commands answer.")
+            cmd.flags = int(flags)
+            cmd.cas = cas
+        self.setRawMode()
+
+    def cmd_STAT(self, line):
+        """
+        Reception of one stat line.
+        """
+        cmd = self._current[0]
+        key, val = line.split(b" ", 1)
+        cmd.values[key] = val
+
+    def cmd_VERSION(self, versionData):
+        """
+        Read version token.
+        """
+        self._current.popleft().success(versionData)
+
+    def cmd_ERROR(self):
+        """
+        A non-existent command has been sent.
+        """
+        log.err("Non-existent command sent.")
+        cmd = self._current.popleft()
+        cmd.fail(NoSuchCommand())
+
+    def cmd_CLIENT_ERROR(self, errText):
+        """
+        An invalid input as been sent.
+        """
+        errText = repr(errText)
+        log.err("Invalid input: " + errText)
+        cmd = self._current.popleft()
+        cmd.fail(ClientError(errText))
+
+    def cmd_SERVER_ERROR(self, errText):
+        """
+        An error has happened server-side.
+        """
+        errText = repr(errText)
+        log.err("Server error: " + errText)
+        cmd = self._current.popleft()
+        cmd.fail(ServerError(errText))
+
+    def cmd_DELETED(self):
+        """
+        A delete command has completed successfully.
+        """
+        self._current.popleft().success(True)
+
+    def cmd_OK(self):
+        """
+        The last command has been completed.
+        """
+        self._current.popleft().success(True)
+
+    def cmd_EXISTS(self):
+        """
+        A C{checkAndSet} update has failed.
+        """
+        self._current.popleft().success(False)
+
+    def lineReceived(self, line):
+        """
+        Receive line commands from the server.
+        """
+        self.resetTimeout()
+        token = line.split(b" ", 1)[0]
+        # First manage standard commands without space
+        cmd = getattr(self, "cmd_" + nativeString(token), None)
+        if cmd is not None:
+            args = line.split(b" ", 1)[1:]
+            if args:
+                cmd(args[0])
+            else:
+                cmd()
+        else:
+            # Then manage commands with space in it
+            line = line.replace(b" ", b"_")
+            cmd = getattr(self, "cmd_" + nativeString(line), None)
+            if cmd is not None:
+                cmd()
+            else:
+                # Increment/Decrement response
+                cmd = self._current.popleft()
+                val = int(line)
+                cmd.success(val)
+        if not self._current:
+            # No pending request, remove timeout
+            self.setTimeout(None)
+
+    def increment(self, key, val=1):
+        """
+        Increment the value of C{key} by given value (default to 1).
+        C{key} must be consistent with an int. Return the new value.
+
+        @param key: the key to modify.
+        @type key: L{bytes}
+
+        @param val: the value to increment.
+        @type val: L{int}
+
+        @return: a deferred with will be called back with the new value
+            associated with the key (after the increment).
+        @rtype: L{Deferred}
+        """
+        return self._incrdecr(b"incr", key, val)
+
+    def decrement(self, key, val=1):
+        """
+        Decrement the value of C{key} by given value (default to 1).
+        C{key} must be consistent with an int. Return the new value, coerced to
+        0 if negative.
+
+        @param key: the key to modify.
+        @type key: L{bytes}
+
+        @param val: the value to decrement.
+        @type val: L{int}
+
+        @return: a deferred with will be called back with the new value
+            associated with the key (after the decrement).
+        @rtype: L{Deferred}
+        """
+        return self._incrdecr(b"decr", key, val)
+
+    def _incrdecr(self, cmd, key, val):
+        """
+        Internal wrapper for incr/decr.
+        """
+        if self._disconnected:
+            return fail(RuntimeError("not connected"))
+        if not isinstance(key, bytes):
+            return fail(
+                ClientError(f"Invalid type for key: {type(key)}, expecting bytes")
+            )
+        if len(key) > self.MAX_KEY_LENGTH:
+            return fail(ClientError("Key too long"))
+        fullcmd = b" ".join([cmd, key, b"%d" % (int(val),)])
+        self.sendLine(fullcmd)
+        cmdObj = Command(cmd, key=key)
+        self._current.append(cmdObj)
+        return cmdObj._deferred
+
+    def replace(self, key, val, flags=0, expireTime=0):
+        """
+        Replace the given C{key}. It must already exist in the server.
+
+        @param key: the key to replace.
+        @type key: L{bytes}
+
+        @param val: the new value associated with the key.
+        @type val: L{bytes}
+
+        @param flags: the flags to store with the key.
+        @type flags: L{int}
+
+        @param expireTime: if different from 0, the relative time in seconds
+            when the key will be deleted from the store.
+        @type expireTime: L{int}
+
+        @return: a deferred that will fire with C{True} if the operation has
+            succeeded, and C{False} with the key didn't previously exist.
+        @rtype: L{Deferred}
+        """
+        return self._set(b"replace", key, val, flags, expireTime, b"")
+
+    def add(self, key, val, flags=0, expireTime=0):
+        """
+        Add the given C{key}. It must not exist in the server.
+
+        @param key: the key to add.
+        @type key: L{bytes}
+
+        @param val: the value associated with the key.
+        @type val: L{bytes}
+
+        @param flags: the flags to store with the key.
+        @type flags: L{int}
+
+        @param expireTime: if different from 0, the relative time in seconds
+            when the key will be deleted from the store.
+        @type expireTime: L{int}
+
+        @return: a deferred that will fire with C{True} if the operation has
+            succeeded, and C{False} with the key already exists.
+        @rtype: L{Deferred}
+        """
+        return self._set(b"add", key, val, flags, expireTime, b"")
+
+    def set(self, key, val, flags=0, expireTime=0):
+        """
+        Set the given C{key}.
+
+        @param key: the key to set.
+        @type key: L{bytes}
+
+        @param val: the value associated with the key.
+        @type val: L{bytes}
+
+        @param flags: the flags to store with the key.
+        @type flags: L{int}
+
+        @param expireTime: if different from 0, the relative time in seconds
+            when the key will be deleted from the store.
+        @type expireTime: L{int}
+
+        @return: a deferred that will fire with C{True} if the operation has
+            succeeded.
+        @rtype: L{Deferred}
+        """
+        return self._set(b"set", key, val, flags, expireTime, b"")
+
+    def checkAndSet(self, key, val, cas, flags=0, expireTime=0):
+        """
+        Change the content of C{key} only if the C{cas} value matches the
+        current one associated with the key. Use this to store a value which
+        hasn't been modified since last time you fetched it.
+
+        @param key: The key to set.
+        @type key: L{bytes}
+
+        @param val: The value associated with the key.
+        @type val: L{bytes}
+
+        @param cas: Unique 64-bit value returned by previous call of C{get}.
+        @type cas: L{bytes}
+
+        @param flags: The flags to store with the key.
+        @type flags: L{int}
+
+        @param expireTime: If different from 0, the relative time in seconds
+            when the key will be deleted from the store.
+        @type expireTime: L{int}
+
+        @return: A deferred that will fire with C{True} if the operation has
+            succeeded, C{False} otherwise.
+        @rtype: L{Deferred}
+        """
+        return self._set(b"cas", key, val, flags, expireTime, cas)
+
+    def _set(self, cmd, key, val, flags, expireTime, cas):
+        """
+        Internal wrapper for setting values.
+        """
+        if self._disconnected:
+            return fail(RuntimeError("not connected"))
+        if not isinstance(key, bytes):
+            return fail(
+                ClientError(f"Invalid type for key: {type(key)}, expecting bytes")
+            )
+        if len(key) > self.MAX_KEY_LENGTH:
+            return fail(ClientError("Key too long"))
+        if not isinstance(val, bytes):
+            return fail(
+                ClientError(f"Invalid type for value: {type(val)}, expecting bytes")
+            )
+        if cas:
+            cas = b" " + cas
+        length = len(val)
+        fullcmd = (
+            b" ".join(
+                [cmd, key, networkString("%d %d %d" % (flags, expireTime, length))]
+            )
+            + cas
+        )
+        self.sendLine(fullcmd)
+        self.sendLine(val)
+        cmdObj = Command(cmd, key=key, flags=flags, length=length)
+        self._current.append(cmdObj)
+        return cmdObj._deferred
+
+    def append(self, key, val):
+        """
+        Append given data to the value of an existing key.
+
+        @param key: The key to modify.
+        @type key: L{bytes}
+
+        @param val: The value to append to the current value associated with
+            the key.
+        @type val: L{bytes}
+
+        @return: A deferred that will fire with C{True} if the operation has
+            succeeded, C{False} otherwise.
+        @rtype: L{Deferred}
+        """
+        # Even if flags and expTime values are ignored, we have to pass them
+        return self._set(b"append", key, val, 0, 0, b"")
+
+    def prepend(self, key, val):
+        """
+        Prepend given data to the value of an existing key.
+
+        @param key: The key to modify.
+        @type key: L{bytes}
+
+        @param val: The value to prepend to the current value associated with
+            the key.
+        @type val: L{bytes}
+
+        @return: A deferred that will fire with C{True} if the operation has
+            succeeded, C{False} otherwise.
+        @rtype: L{Deferred}
+        """
+        # Even if flags and expTime values are ignored, we have to pass them
+        return self._set(b"prepend", key, val, 0, 0, b"")
+
+    def get(self, key, withIdentifier=False):
+        """
+        Get the given C{key}. It doesn't support multiple keys. If
+        C{withIdentifier} is set to C{True}, the command issued is a C{gets},
+        that will return the current identifier associated with the value. This
+        identifier has to be used when issuing C{checkAndSet} update later,
+        using the corresponding method.
+
+        @param key: The key to retrieve.
+        @type key: L{bytes}
+
+        @param withIdentifier: If set to C{True}, retrieve the current
+            identifier along with the value and the flags.
+        @type withIdentifier: L{bool}
+
+        @return: A deferred that will fire with the tuple (flags, value) if
+            C{withIdentifier} is C{False}, or (flags, cas identifier, value)
+            if C{True}.  If the server indicates there is no value
+            associated with C{key}, the returned value will be L{None} and
+            the returned flags will be C{0}.
+        @rtype: L{Deferred}
+        """
+        return self._get([key], withIdentifier, False)
+
+    def getMultiple(self, keys, withIdentifier=False):
+        """
+        Get the given list of C{keys}.  If C{withIdentifier} is set to C{True},
+        the command issued is a C{gets}, that will return the identifiers
+        associated with each values. This identifier has to be used when
+        issuing C{checkAndSet} update later, using the corresponding method.
+
+        @param keys: The keys to retrieve.
+        @type keys: L{list} of L{bytes}
+
+        @param withIdentifier: If set to C{True}, retrieve the identifiers
+            along with the values and the flags.
+        @type withIdentifier: L{bool}
+
+        @return: A deferred that will fire with a dictionary with the elements
+            of C{keys} as keys and the tuples (flags, value) as values if
+            C{withIdentifier} is C{False}, or (flags, cas identifier, value) if
+            C{True}.  If the server indicates there is no value associated with
+            C{key}, the returned values will be L{None} and the returned flags
+            will be C{0}.
+        @rtype: L{Deferred}
+
+        @since: 9.0
+        """
+        return self._get(keys, withIdentifier, True)
+
+    def _get(self, keys, withIdentifier, multiple):
+        """
+        Helper method for C{get} and C{getMultiple}.
+        """
+        keys = list(keys)
+        if self._disconnected:
+            return fail(RuntimeError("not connected"))
+        for key in keys:
+            if not isinstance(key, bytes):
+                return fail(
+                    ClientError(f"Invalid type for key: {type(key)}, expecting bytes")
+                )
+            if len(key) > self.MAX_KEY_LENGTH:
+                return fail(ClientError("Key too long"))
+        if withIdentifier:
+            cmd = b"gets"
+        else:
+            cmd = b"get"
+        fullcmd = b" ".join([cmd] + keys)
+        self.sendLine(fullcmd)
+        if multiple:
+            values = {key: (0, b"", None) for key in keys}
+            cmdObj = Command(cmd, keys=keys, values=values, multiple=True)
+        else:
+            cmdObj = Command(
+                cmd, key=keys[0], value=None, flags=0, cas=b"", multiple=False
+            )
+        self._current.append(cmdObj)
+        return cmdObj._deferred
+
+    def stats(self, arg=None):
+        """
+        Get some stats from the server. It will be available as a dict.
+
+        @param arg: An optional additional string which will be sent along
+            with the I{stats} command.  The interpretation of this value by
+            the server is left undefined by the memcache protocol
+            specification.
+        @type arg: L{None} or L{bytes}
+
+        @return: a deferred that will fire with a L{dict} of the available
+            statistics.
+        @rtype: L{Deferred}
+        """
+        if arg:
+            cmd = b"stats " + arg
+        else:
+            cmd = b"stats"
+        if self._disconnected:
+            return fail(RuntimeError("not connected"))
+        self.sendLine(cmd)
+        cmdObj = Command(b"stats", values={})
+        self._current.append(cmdObj)
+        return cmdObj._deferred
+
+    def version(self):
+        """
+        Get the version of the server.
+
+        @return: a deferred that will fire with the string value of the
+            version.
+        @rtype: L{Deferred}
+        """
+        if self._disconnected:
+            return fail(RuntimeError("not connected"))
+        self.sendLine(b"version")
+        cmdObj = Command(b"version")
+        self._current.append(cmdObj)
+        return cmdObj._deferred
+
+    def delete(self, key):
+        """
+        Delete an existing C{key}.
+
+        @param key: the key to delete.
+        @type key: L{bytes}
+
+        @return: a deferred that will be called back with C{True} if the key
+            was successfully deleted, or C{False} if not.
+        @rtype: L{Deferred}
+        """
+        if self._disconnected:
+            return fail(RuntimeError("not connected"))
+        if not isinstance(key, bytes):
+            return fail(
+                ClientError(f"Invalid type for key: {type(key)}, expecting bytes")
+            )
+        self.sendLine(b"delete " + key)
+        cmdObj = Command(b"delete", key=key)
+        self._current.append(cmdObj)
+        return cmdObj._deferred
+
+    def flushAll(self):
+        """
+        Flush all cached values.
+
+        @return: a deferred that will be called back with C{True} when the
+            operation has succeeded.
+        @rtype: L{Deferred}
+        """
+        if self._disconnected:
+            return fail(RuntimeError("not connected"))
+        self.sendLine(b"flush_all")
+        cmdObj = Command(b"flush_all")
+        self._current.append(cmdObj)
+        return cmdObj._deferred
+
+
+__all__ = [
+    "MemCacheProtocol",
+    "DEFAULT_PORT",
+    "NoSuchCommand",
+    "ClientError",
+    "ServerError",
+]
diff --git a/contrib/python/Twisted/py3/twisted/protocols/pcp.py b/contrib/python/Twisted/py3/twisted/protocols/pcp.py
new file mode 100644
index 00000000000..978ec64d6dd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/pcp.py
@@ -0,0 +1,211 @@
+# -*- test-case-name: twisted.test.test_pcp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Producer-Consumer Proxy.
+"""
+
+from zope.interface import implementer
+
+from twisted.internet import interfaces
+
+
+@implementer(interfaces.IProducer, interfaces.IConsumer)
+class BasicProducerConsumerProxy:
+    """
+    I can act as a man in the middle between any Producer and Consumer.
+
+    @ivar producer: the Producer I subscribe to.
+    @type producer: L{IProducer<interfaces.IProducer>}
+    @ivar consumer: the Consumer I publish to.
+    @type consumer: L{IConsumer<interfaces.IConsumer>}
+    @ivar paused: As a Producer, am I paused?
+    @type paused: bool
+    """
+
+    consumer = None
+    producer = None
+    producerIsStreaming = None
+    iAmStreaming = True
+    outstandingPull = False
+    paused = False
+    stopped = False
+
+    def __init__(self, consumer):
+        self._buffer = []
+        if consumer is not None:
+            self.consumer = consumer
+            consumer.registerProducer(self, self.iAmStreaming)
+
+    # Producer methods:
+
+    def pauseProducing(self):
+        self.paused = True
+        if self.producer:
+            self.producer.pauseProducing()
+
+    def resumeProducing(self):
+        self.paused = False
+        if self._buffer:
+            # TODO: Check to see if consumer supports writeSeq.
+            self.consumer.write("".join(self._buffer))
+            self._buffer[:] = []
+        else:
+            if not self.iAmStreaming:
+                self.outstandingPull = True
+
+        if self.producer is not None:
+            self.producer.resumeProducing()
+
+    def stopProducing(self):
+        if self.producer is not None:
+            self.producer.stopProducing()
+        if self.consumer is not None:
+            del self.consumer
+
+    # Consumer methods:
+
+    def write(self, data):
+        if self.paused or (not self.iAmStreaming and not self.outstandingPull):
+            # We could use that fifo queue here.
+            self._buffer.append(data)
+
+        elif self.consumer is not None:
+            self.consumer.write(data)
+            self.outstandingPull = False
+
+    def finish(self):
+        if self.consumer is not None:
+            self.consumer.finish()
+        self.unregisterProducer()
+
+    def registerProducer(self, producer, streaming):
+        self.producer = producer
+        self.producerIsStreaming = streaming
+
+    def unregisterProducer(self):
+        if self.producer is not None:
+            del self.producer
+            del self.producerIsStreaming
+        if self.consumer:
+            self.consumer.unregisterProducer()
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__}@{id(self):x} around {self.consumer}>"
+
+
+class ProducerConsumerProxy(BasicProducerConsumerProxy):
+    """ProducerConsumerProxy with a finite buffer.
+
+    When my buffer fills up, I have my parent Producer pause until my buffer
+    has room in it again.
+    """
+
+    # Copies much from abstract.FileDescriptor
+    bufferSize = 2**2**2**2
+
+    producerPaused = False
+    unregistered = False
+
+    def pauseProducing(self):
+        # Does *not* call up to ProducerConsumerProxy to relay the pause
+        # message through to my parent Producer.
+        self.paused = True
+
+    def resumeProducing(self):
+        self.paused = False
+        if self._buffer:
+            data = "".join(self._buffer)
+            bytesSent = self._writeSomeData(data)
+            if bytesSent < len(data):
+                unsent = data[bytesSent:]
+                assert (
+                    not self.iAmStreaming
+                ), "Streaming producer did not write all its data."
+                self._buffer[:] = [unsent]
+            else:
+                self._buffer[:] = []
+        else:
+            bytesSent = 0
+
+        if (
+            self.unregistered
+            and bytesSent
+            and not self._buffer
+            and self.consumer is not None
+        ):
+            self.consumer.unregisterProducer()
+
+        if not self.iAmStreaming:
+            self.outstandingPull = not bytesSent
+
+        if self.producer is not None:
+            bytesBuffered = sum(len(s) for s in self._buffer)
+            # TODO: You can see here the potential for high and low
+            # watermarks, where bufferSize would be the high mark when we
+            # ask the upstream producer to pause, and we wouldn't have
+            # it resume again until it hit the low mark.  Or if producer
+            # is Pull, maybe we'd like to pull from it as much as necessary
+            # to keep our buffer full to the low mark, so we're never caught
+            # without something to send.
+            if self.producerPaused and (bytesBuffered < self.bufferSize):
+                # Now that our buffer is empty,
+                self.producerPaused = False
+                self.producer.resumeProducing()
+            elif self.outstandingPull:
+                # I did not have any data to write in response to a pull,
+                # so I'd better pull some myself.
+                self.producer.resumeProducing()
+
+    def write(self, data):
+        if self.paused or (not self.iAmStreaming and not self.outstandingPull):
+            # We could use that fifo queue here.
+            self._buffer.append(data)
+
+        elif self.consumer is not None:
+            assert (
+                not self._buffer
+            ), "Writing fresh data to consumer before my buffer is empty!"
+            # I'm going to use _writeSomeData here so that there is only one
+            # path to self.consumer.write.  But it doesn't actually make sense,
+            # if I am streaming, for some data to not be all data.  But maybe I
+            # am not streaming, but I am writing here anyway, because there was
+            # an earlier request for data which was not answered.
+            bytesSent = self._writeSomeData(data)
+            self.outstandingPull = False
+            if not bytesSent == len(data):
+                assert (
+                    not self.iAmStreaming
+                ), "Streaming producer did not write all its data."
+                self._buffer.append(data[bytesSent:])
+
+        if (self.producer is not None) and self.producerIsStreaming:
+            bytesBuffered = sum(len(s) for s in self._buffer)
+            if bytesBuffered >= self.bufferSize:
+                self.producer.pauseProducing()
+                self.producerPaused = True
+
+    def registerProducer(self, producer, streaming):
+        self.unregistered = False
+        BasicProducerConsumerProxy.registerProducer(self, producer, streaming)
+        if not streaming:
+            producer.resumeProducing()
+
+    def unregisterProducer(self):
+        if self.producer is not None:
+            del self.producer
+            del self.producerIsStreaming
+        self.unregistered = True
+        if self.consumer and not self._buffer:
+            self.consumer.unregisterProducer()
+
+    def _writeSomeData(self, data):
+        """Write as much of this data as possible.
+
+        @returns: The number of bytes written.
+        """
+        if self.consumer is None:
+            return 0
+        self.consumer.write(data)
+        return len(data)
diff --git a/contrib/python/Twisted/py3/twisted/protocols/policies.py b/contrib/python/Twisted/py3/twisted/protocols/policies.py
new file mode 100644
index 00000000000..a89d4f8a8d1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/policies.py
@@ -0,0 +1,696 @@
+# -*- test-case-name: twisted.test.test_policies -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Resource limiting policies.
+
+@seealso: See also L{twisted.protocols.htb} for rate limiting.
+"""
+
+
+# system imports
+import sys
+from typing import Optional, Type
+
+from zope.interface import directlyProvides, providedBy
+
+from twisted.internet import error, interfaces
+from twisted.internet.interfaces import ILoggingContext
+
+# twisted imports
+from twisted.internet.protocol import ClientFactory, Protocol, ServerFactory
+from twisted.python import log
+
+
+def _wrappedLogPrefix(wrapper, wrapped):
+    """
+    Compute a log prefix for a wrapper and the object it wraps.
+
+    @rtype: C{str}
+    """
+    if ILoggingContext.providedBy(wrapped):
+        logPrefix = wrapped.logPrefix()
+    else:
+        logPrefix = wrapped.__class__.__name__
+    return f"{logPrefix} ({wrapper.__class__.__name__})"
+
+
+class ProtocolWrapper(Protocol):
+    """
+    Wraps protocol instances and acts as their transport as well.
+
+    @ivar wrappedProtocol: An L{IProtocol<twisted.internet.interfaces.IProtocol>}
+        provider to which L{IProtocol<twisted.internet.interfaces.IProtocol>}
+        method calls onto this L{ProtocolWrapper} will be proxied.
+
+    @ivar factory: The L{WrappingFactory} which created this
+        L{ProtocolWrapper}.
+    """
+
+    disconnecting = 0
+
+    def __init__(
+        self, factory: "WrappingFactory", wrappedProtocol: interfaces.IProtocol
+    ):
+        self.wrappedProtocol = wrappedProtocol
+        self.factory = factory
+
+    def logPrefix(self):
+        """
+        Use a customized log prefix mentioning both the wrapped protocol and
+        the current one.
+        """
+        return _wrappedLogPrefix(self, self.wrappedProtocol)
+
+    def makeConnection(self, transport):
+        """
+        When a connection is made, register this wrapper with its factory,
+        save the real transport, and connect the wrapped protocol to this
+        L{ProtocolWrapper} to intercept any transport calls it makes.
+        """
+        directlyProvides(self, providedBy(transport))
+        Protocol.makeConnection(self, transport)
+        self.factory.registerProtocol(self)
+        self.wrappedProtocol.makeConnection(self)
+
+    # Transport relaying
+
+    def write(self, data):
+        self.transport.write(data)
+
+    def writeSequence(self, data):
+        self.transport.writeSequence(data)
+
+    def loseConnection(self):
+        self.disconnecting = 1
+        self.transport.loseConnection()
+
+    def getPeer(self):
+        return self.transport.getPeer()
+
+    def getHost(self):
+        return self.transport.getHost()
+
+    def registerProducer(self, producer, streaming):
+        self.transport.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        self.transport.unregisterProducer()
+
+    def stopConsuming(self):
+        self.transport.stopConsuming()
+
+    def __getattr__(self, name):
+        return getattr(self.transport, name)
+
+    # Protocol relaying
+
+    def dataReceived(self, data):
+        self.wrappedProtocol.dataReceived(data)
+
+    def connectionLost(self, reason):
+        self.factory.unregisterProtocol(self)
+        self.wrappedProtocol.connectionLost(reason)
+
+        # Breaking reference cycle between self and wrappedProtocol.
+        self.wrappedProtocol = None
+
+
+class WrappingFactory(ClientFactory):
+    """
+    Wraps a factory and its protocols, and keeps track of them.
+    """
+
+    protocol: Type[Protocol] = ProtocolWrapper
+
+    def __init__(self, wrappedFactory):
+        self.wrappedFactory = wrappedFactory
+        self.protocols = {}
+
+    def logPrefix(self):
+        """
+        Generate a log prefix mentioning both the wrapped factory and this one.
+        """
+        return _wrappedLogPrefix(self, self.wrappedFactory)
+
+    def doStart(self):
+        self.wrappedFactory.doStart()
+        ClientFactory.doStart(self)
+
+    def doStop(self):
+        self.wrappedFactory.doStop()
+        ClientFactory.doStop(self)
+
+    def startedConnecting(self, connector):
+        self.wrappedFactory.startedConnecting(connector)
+
+    def clientConnectionFailed(self, connector, reason):
+        self.wrappedFactory.clientConnectionFailed(connector, reason)
+
+    def clientConnectionLost(self, connector, reason):
+        self.wrappedFactory.clientConnectionLost(connector, reason)
+
+    def buildProtocol(self, addr):
+        return self.protocol(self, self.wrappedFactory.buildProtocol(addr))
+
+    def registerProtocol(self, p):
+        """
+        Called by protocol to register itself.
+        """
+        self.protocols[p] = 1
+
+    def unregisterProtocol(self, p):
+        """
+        Called by protocols when they go away.
+        """
+        del self.protocols[p]
+
+
+class ThrottlingProtocol(ProtocolWrapper):
+    """
+    Protocol for L{ThrottlingFactory}.
+    """
+
+    # wrap API for tracking bandwidth
+
+    def write(self, data):
+        self.factory.registerWritten(len(data))
+        ProtocolWrapper.write(self, data)
+
+    def writeSequence(self, seq):
+        self.factory.registerWritten(sum(map(len, seq)))
+        ProtocolWrapper.writeSequence(self, seq)
+
+    def dataReceived(self, data):
+        self.factory.registerRead(len(data))
+        ProtocolWrapper.dataReceived(self, data)
+
+    def registerProducer(self, producer, streaming):
+        self.producer = producer
+        ProtocolWrapper.registerProducer(self, producer, streaming)
+
+    def unregisterProducer(self):
+        del self.producer
+        ProtocolWrapper.unregisterProducer(self)
+
+    def throttleReads(self):
+        self.transport.pauseProducing()
+
+    def unthrottleReads(self):
+        self.transport.resumeProducing()
+
+    def throttleWrites(self):
+        if hasattr(self, "producer"):
+            self.producer.pauseProducing()
+
+    def unthrottleWrites(self):
+        if hasattr(self, "producer"):
+            self.producer.resumeProducing()
+
+
+class ThrottlingFactory(WrappingFactory):
+    """
+    Throttles bandwidth and number of connections.
+
+    Write bandwidth will only be throttled if there is a producer
+    registered.
+    """
+
+    protocol = ThrottlingProtocol
+
+    def __init__(
+        self,
+        wrappedFactory,
+        maxConnectionCount=sys.maxsize,
+        readLimit=None,
+        writeLimit=None,
+    ):
+        WrappingFactory.__init__(self, wrappedFactory)
+        self.connectionCount = 0
+        self.maxConnectionCount = maxConnectionCount
+        self.readLimit = readLimit  # max bytes we should read per second
+        self.writeLimit = writeLimit  # max bytes we should write per second
+        self.readThisSecond = 0
+        self.writtenThisSecond = 0
+        self.unthrottleReadsID = None
+        self.checkReadBandwidthID = None
+        self.unthrottleWritesID = None
+        self.checkWriteBandwidthID = None
+
+    def callLater(self, period, func):
+        """
+        Wrapper around
+        L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>}
+        for test purpose.
+        """
+        from twisted.internet import reactor
+
+        return reactor.callLater(period, func)
+
+    def registerWritten(self, length):
+        """
+        Called by protocol to tell us more bytes were written.
+        """
+        self.writtenThisSecond += length
+
+    def registerRead(self, length):
+        """
+        Called by protocol to tell us more bytes were read.
+        """
+        self.readThisSecond += length
+
+    def checkReadBandwidth(self):
+        """
+        Checks if we've passed bandwidth limits.
+        """
+        if self.readThisSecond > self.readLimit:
+            self.throttleReads()
+            throttleTime = (float(self.readThisSecond) / self.readLimit) - 1.0
+            self.unthrottleReadsID = self.callLater(throttleTime, self.unthrottleReads)
+        self.readThisSecond = 0
+        self.checkReadBandwidthID = self.callLater(1, self.checkReadBandwidth)
+
+    def checkWriteBandwidth(self):
+        if self.writtenThisSecond > self.writeLimit:
+            self.throttleWrites()
+            throttleTime = (float(self.writtenThisSecond) / self.writeLimit) - 1.0
+            self.unthrottleWritesID = self.callLater(
+                throttleTime, self.unthrottleWrites
+            )
+        # reset for next round
+        self.writtenThisSecond = 0
+        self.checkWriteBandwidthID = self.callLater(1, self.checkWriteBandwidth)
+
+    def throttleReads(self):
+        """
+        Throttle reads on all protocols.
+        """
+        log.msg("Throttling reads on %s" % self)
+        for p in self.protocols.keys():
+            p.throttleReads()
+
+    def unthrottleReads(self):
+        """
+        Stop throttling reads on all protocols.
+        """
+        self.unthrottleReadsID = None
+        log.msg("Stopped throttling reads on %s" % self)
+        for p in self.protocols.keys():
+            p.unthrottleReads()
+
+    def throttleWrites(self):
+        """
+        Throttle writes on all protocols.
+        """
+        log.msg("Throttling writes on %s" % self)
+        for p in self.protocols.keys():
+            p.throttleWrites()
+
+    def unthrottleWrites(self):
+        """
+        Stop throttling writes on all protocols.
+        """
+        self.unthrottleWritesID = None
+        log.msg("Stopped throttling writes on %s" % self)
+        for p in self.protocols.keys():
+            p.unthrottleWrites()
+
+    def buildProtocol(self, addr):
+        if self.connectionCount == 0:
+            if self.readLimit is not None:
+                self.checkReadBandwidth()
+            if self.writeLimit is not None:
+                self.checkWriteBandwidth()
+
+        if self.connectionCount < self.maxConnectionCount:
+            self.connectionCount += 1
+            return WrappingFactory.buildProtocol(self, addr)
+        else:
+            log.msg("Max connection count reached!")
+            return None
+
+    def unregisterProtocol(self, p):
+        WrappingFactory.unregisterProtocol(self, p)
+        self.connectionCount -= 1
+        if self.connectionCount == 0:
+            if self.unthrottleReadsID is not None:
+                self.unthrottleReadsID.cancel()
+            if self.checkReadBandwidthID is not None:
+                self.checkReadBandwidthID.cancel()
+            if self.unthrottleWritesID is not None:
+                self.unthrottleWritesID.cancel()
+            if self.checkWriteBandwidthID is not None:
+                self.checkWriteBandwidthID.cancel()
+
+
+class SpewingProtocol(ProtocolWrapper):
+    def dataReceived(self, data):
+        log.msg("Received: %r" % data)
+        ProtocolWrapper.dataReceived(self, data)
+
+    def write(self, data):
+        log.msg("Sending: %r" % data)
+        ProtocolWrapper.write(self, data)
+
+
+class SpewingFactory(WrappingFactory):
+    protocol = SpewingProtocol
+
+
+class LimitConnectionsByPeer(WrappingFactory):
+    maxConnectionsPerPeer = 5
+
+    def startFactory(self):
+        self.peerConnections = {}
+
+    def buildProtocol(self, addr):
+        peerHost = addr[0]
+        connectionCount = self.peerConnections.get(peerHost, 0)
+        if connectionCount >= self.maxConnectionsPerPeer:
+            return None
+        self.peerConnections[peerHost] = connectionCount + 1
+        return WrappingFactory.buildProtocol(self, addr)
+
+    def unregisterProtocol(self, p):
+        peerHost = p.getPeer()[1]
+        self.peerConnections[peerHost] -= 1
+        if self.peerConnections[peerHost] == 0:
+            del self.peerConnections[peerHost]
+
+
+class LimitTotalConnectionsFactory(ServerFactory):
+    """
+    Factory that limits the number of simultaneous connections.
+
+    @type connectionCount: C{int}
+    @ivar connectionCount: number of current connections.
+    @type connectionLimit: C{int} or L{None}
+    @cvar connectionLimit: maximum number of connections.
+    @type overflowProtocol: L{Protocol} or L{None}
+    @cvar overflowProtocol: Protocol to use for new connections when
+        connectionLimit is exceeded.  If L{None} (the default value), excess
+        connections will be closed immediately.
+    """
+
+    connectionCount = 0
+    connectionLimit = None
+    overflowProtocol: Optional[Type[Protocol]] = None
+
+    def buildProtocol(self, addr):
+        if self.connectionLimit is None or self.connectionCount < self.connectionLimit:
+            # Build the normal protocol
+            wrappedProtocol = self.protocol()
+        elif self.overflowProtocol is None:
+            # Just drop the connection
+            return None
+        else:
+            # Too many connections, so build the overflow protocol
+            wrappedProtocol = self.overflowProtocol()
+
+        wrappedProtocol.factory = self
+        protocol = ProtocolWrapper(self, wrappedProtocol)
+        self.connectionCount += 1
+        return protocol
+
+    def registerProtocol(self, p):
+        pass
+
+    def unregisterProtocol(self, p):
+        self.connectionCount -= 1
+
+
+class TimeoutProtocol(ProtocolWrapper):
+    """
+    Protocol that automatically disconnects when the connection is idle.
+    """
+
+    def __init__(self, factory, wrappedProtocol, timeoutPeriod):
+        """
+        Constructor.
+
+        @param factory: An L{TimeoutFactory}.
+        @param wrappedProtocol: A L{Protocol} to wrapp.
+        @param timeoutPeriod: Number of seconds to wait for activity before
+            timing out.
+        """
+        ProtocolWrapper.__init__(self, factory, wrappedProtocol)
+        self.timeoutCall = None
+        self.timeoutPeriod = None
+        self.setTimeout(timeoutPeriod)
+
+    def setTimeout(self, timeoutPeriod=None):
+        """
+        Set a timeout.
+
+        This will cancel any existing timeouts.
+
+        @param timeoutPeriod: If not L{None}, change the timeout period.
+            Otherwise, use the existing value.
+        """
+        self.cancelTimeout()
+        self.timeoutPeriod = timeoutPeriod
+        if timeoutPeriod is not None:
+            self.timeoutCall = self.factory.callLater(
+                self.timeoutPeriod, self.timeoutFunc
+            )
+
+    def cancelTimeout(self):
+        """
+        Cancel the timeout.
+
+        If the timeout was already cancelled, this does nothing.
+        """
+        self.timeoutPeriod = None
+        if self.timeoutCall:
+            try:
+                self.timeoutCall.cancel()
+            except (error.AlreadyCalled, error.AlreadyCancelled):
+                pass
+            self.timeoutCall = None
+
+    def resetTimeout(self):
+        """
+        Reset the timeout, usually because some activity just happened.
+        """
+        if self.timeoutCall:
+            self.timeoutCall.reset(self.timeoutPeriod)
+
+    def write(self, data):
+        self.resetTimeout()
+        ProtocolWrapper.write(self, data)
+
+    def writeSequence(self, seq):
+        self.resetTimeout()
+        ProtocolWrapper.writeSequence(self, seq)
+
+    def dataReceived(self, data):
+        self.resetTimeout()
+        ProtocolWrapper.dataReceived(self, data)
+
+    def connectionLost(self, reason):
+        self.cancelTimeout()
+        ProtocolWrapper.connectionLost(self, reason)
+
+    def timeoutFunc(self):
+        """
+        This method is called when the timeout is triggered.
+
+        By default it calls I{loseConnection}.  Override this if you want
+        something else to happen.
+        """
+        self.loseConnection()
+
+
+class TimeoutFactory(WrappingFactory):
+    """
+    Factory for TimeoutWrapper.
+    """
+
+    protocol = TimeoutProtocol
+
+    def __init__(self, wrappedFactory, timeoutPeriod=30 * 60):
+        self.timeoutPeriod = timeoutPeriod
+        WrappingFactory.__init__(self, wrappedFactory)
+
+    def buildProtocol(self, addr):
+        return self.protocol(
+            self,
+            self.wrappedFactory.buildProtocol(addr),
+            timeoutPeriod=self.timeoutPeriod,
+        )
+
+    def callLater(self, period, func):
+        """
+        Wrapper around
+        L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>}
+        for test purpose.
+        """
+        from twisted.internet import reactor
+
+        return reactor.callLater(period, func)
+
+
+class TrafficLoggingProtocol(ProtocolWrapper):
+    def __init__(self, factory, wrappedProtocol, logfile, lengthLimit=None, number=0):
+        """
+        @param factory: factory which created this protocol.
+        @type factory: L{protocol.Factory}.
+        @param wrappedProtocol: the underlying protocol.
+        @type wrappedProtocol: C{protocol.Protocol}.
+        @param logfile: file opened for writing used to write log messages.
+        @type logfile: C{file}
+        @param lengthLimit: maximum size of the datareceived logged.
+        @type lengthLimit: C{int}
+        @param number: identifier of the connection.
+        @type number: C{int}.
+        """
+        ProtocolWrapper.__init__(self, factory, wrappedProtocol)
+        self.logfile = logfile
+        self.lengthLimit = lengthLimit
+        self._number = number
+
+    def _log(self, line):
+        self.logfile.write(line + "\n")
+        self.logfile.flush()
+
+    def _mungeData(self, data):
+        if self.lengthLimit and len(data) > self.lengthLimit:
+            data = data[: self.lengthLimit - 12] + "<... elided>"
+        return data
+
+    # IProtocol
+    def connectionMade(self):
+        self._log("*")
+        return ProtocolWrapper.connectionMade(self)
+
+    def dataReceived(self, data):
+        self._log("C %d: %r" % (self._number, self._mungeData(data)))
+        return ProtocolWrapper.dataReceived(self, data)
+
+    def connectionLost(self, reason):
+        self._log("C %d: %r" % (self._number, reason))
+        return ProtocolWrapper.connectionLost(self, reason)
+
+    # ITransport
+    def write(self, data):
+        self._log("S %d: %r" % (self._number, self._mungeData(data)))
+        return ProtocolWrapper.write(self, data)
+
+    def writeSequence(self, iovec):
+        self._log("SV %d: %r" % (self._number, [self._mungeData(d) for d in iovec]))
+        return ProtocolWrapper.writeSequence(self, iovec)
+
+    def loseConnection(self):
+        self._log("S %d: *" % (self._number,))
+        return ProtocolWrapper.loseConnection(self)
+
+
+class TrafficLoggingFactory(WrappingFactory):
+    protocol = TrafficLoggingProtocol
+
+    _counter = 0
+
+    def __init__(self, wrappedFactory, logfilePrefix, lengthLimit=None):
+        self.logfilePrefix = logfilePrefix
+        self.lengthLimit = lengthLimit
+        WrappingFactory.__init__(self, wrappedFactory)
+
+    def open(self, name):
+        return open(name, "w")
+
+    def buildProtocol(self, addr):
+        self._counter += 1
+        logfile = self.open(self.logfilePrefix + "-" + str(self._counter))
+        return self.protocol(
+            self,
+            self.wrappedFactory.buildProtocol(addr),
+            logfile,
+            self.lengthLimit,
+            self._counter,
+        )
+
+    def resetCounter(self):
+        """
+        Reset the value of the counter used to identify connections.
+        """
+        self._counter = 0
+
+
+class TimeoutMixin:
+    """
+    Mixin for protocols which wish to timeout connections.
+
+    Protocols that mix this in have a single timeout, set using L{setTimeout}.
+    When the timeout is hit, L{timeoutConnection} is called, which, by
+    default, closes the connection.
+
+    @cvar timeOut: The number of seconds after which to timeout the connection.
+    """
+
+    timeOut: Optional[int] = None
+
+    __timeoutCall = None
+
+    def callLater(self, period, func):
+        """
+        Wrapper around
+        L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>}
+        for test purpose.
+        """
+        from twisted.internet import reactor
+
+        return reactor.callLater(period, func)
+
+    def resetTimeout(self):
+        """
+        Reset the timeout count down.
+
+        If the connection has already timed out, then do nothing.  If the
+        timeout has been cancelled (probably using C{setTimeout(None)}), also
+        do nothing.
+
+        It's often a good idea to call this when the protocol has received
+        some meaningful input from the other end of the connection.  "I've got
+        some data, they're still there, reset the timeout".
+        """
+        if self.__timeoutCall is not None and self.timeOut is not None:
+            self.__timeoutCall.reset(self.timeOut)
+
+    def setTimeout(self, period):
+        """
+        Change the timeout period
+
+        @type period: C{int} or L{None}
+        @param period: The period, in seconds, to change the timeout to, or
+        L{None} to disable the timeout.
+        """
+        prev = self.timeOut
+        self.timeOut = period
+
+        if self.__timeoutCall is not None:
+            if period is None:
+                try:
+                    self.__timeoutCall.cancel()
+                except (error.AlreadyCancelled, error.AlreadyCalled):
+                    # Do nothing if the call was already consumed.
+                    pass
+                self.__timeoutCall = None
+            else:
+                self.__timeoutCall.reset(period)
+        elif period is not None:
+            self.__timeoutCall = self.callLater(period, self.__timedOut)
+
+        return prev
+
+    def __timedOut(self):
+        self.__timeoutCall = None
+        self.timeoutConnection()
+
+    def timeoutConnection(self):
+        """
+        Called when the connection times out.
+
+        Override to define behavior other than dropping the connection.
+        """
+        self.transport.loseConnection()
diff --git a/contrib/python/Twisted/py3/twisted/protocols/portforward.py b/contrib/python/Twisted/py3/twisted/protocols/portforward.py
new file mode 100644
index 00000000000..bf3a894dfaa
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/portforward.py
@@ -0,0 +1,90 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A simple port forwarder.
+"""
+
+# Twisted imports
+from twisted.internet import protocol
+from twisted.python import log
+
+
+class Proxy(protocol.Protocol):
+    noisy = True
+
+    peer = None
+
+    def setPeer(self, peer):
+        self.peer = peer
+
+    def connectionLost(self, reason):
+        if self.peer is not None:
+            self.peer.transport.loseConnection()
+            self.peer = None
+        elif self.noisy:
+            log.msg(f"Unable to connect to peer: {reason}")
+
+    def dataReceived(self, data):
+        self.peer.transport.write(data)
+
+
+class ProxyClient(Proxy):
+    def connectionMade(self):
+        self.peer.setPeer(self)
+
+        # Wire this and the peer transport together to enable
+        # flow control (this stops connections from filling
+        # this proxy memory when one side produces data at a
+        # higher rate than the other can consume).
+        self.transport.registerProducer(self.peer.transport, True)
+        self.peer.transport.registerProducer(self.transport, True)
+
+        # We're connected, everybody can read to their hearts content.
+        self.peer.transport.resumeProducing()
+
+
+class ProxyClientFactory(protocol.ClientFactory):
+    protocol = ProxyClient
+
+    def setServer(self, server):
+        self.server = server
+
+    def buildProtocol(self, *args, **kw):
+        prot = protocol.ClientFactory.buildProtocol(self, *args, **kw)
+        prot.setPeer(self.server)
+        return prot
+
+    def clientConnectionFailed(self, connector, reason):
+        self.server.transport.loseConnection()
+
+
+class ProxyServer(Proxy):
+    clientProtocolFactory = ProxyClientFactory
+    reactor = None
+
+    def connectionMade(self):
+        # Don't read anything from the connecting client until we have
+        # somewhere to send it to.
+        self.transport.pauseProducing()
+
+        client = self.clientProtocolFactory()
+        client.setServer(self)
+
+        if self.reactor is None:
+            from twisted.internet import reactor
+
+            self.reactor = reactor
+        self.reactor.connectTCP(self.factory.host, self.factory.port, client)
+
+
+class ProxyFactory(protocol.Factory):
+    """
+    Factory for port forwarder.
+    """
+
+    protocol = ProxyServer
+
+    def __init__(self, host, port):
+        self.host = host
+        self.port = port
diff --git a/contrib/python/Twisted/py3/twisted/protocols/postfix.py b/contrib/python/Twisted/py3/twisted/protocols/postfix.py
new file mode 100644
index 00000000000..5860887cb00
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/postfix.py
@@ -0,0 +1,137 @@
+# -*- test-case-name: twisted.test.test_postfix -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Postfix mail transport agent related protocols.
+"""
+from __future__ import annotations
+
+import sys
+from collections import UserDict
+from typing import TYPE_CHECKING, Union
+from urllib.parse import quote as _quote, unquote as _unquote
+
+from twisted.internet import defer, protocol
+from twisted.protocols import basic, policies
+from twisted.python import log
+
+
+# urllib's quote functions just happen to match
+# the postfix semantics.
+def quote(s):
+    quoted = _quote(s)
+    if isinstance(quoted, str):
+        quoted = quoted.encode("ascii")
+    return quoted
+
+
+def unquote(s):
+    if isinstance(s, bytes):
+        s = s.decode("ascii")
+    quoted = _unquote(s)
+    return quoted.encode("ascii")
+
+
+class PostfixTCPMapServer(basic.LineReceiver, policies.TimeoutMixin):
+    """
+    Postfix mail transport agent TCP map protocol implementation.
+
+    Receive requests for data matching given key via lineReceived,
+    asks it's factory for the data with self.factory.get(key), and
+    returns the data to the requester. None means no entry found.
+
+    You can use postfix's postmap to test the map service::
+
+    /usr/sbin/postmap -q KEY tcp:localhost:4242
+
+    """
+
+    timeout = 600
+    delimiter = b"\n"
+
+    def connectionMade(self):
+        self.setTimeout(self.timeout)
+
+    def sendCode(self, code, message=b""):
+        """
+        Send an SMTP-like code with a message.
+        """
+        self.sendLine(str(code).encode("ascii") + b" " + message)
+
+    def lineReceived(self, line):
+        self.resetTimeout()
+        try:
+            request, params = line.split(None, 1)
+        except ValueError:
+            request = line
+            params = None
+        try:
+            f = getattr(self, "do_" + request.decode("ascii"))
+        except AttributeError:
+            self.sendCode(400, b"unknown command")
+        else:
+            try:
+                f(params)
+            except BaseException:
+                excInfo = str(sys.exc_info()[1]).encode("ascii")
+                self.sendCode(400, b"Command " + request + b" failed: " + excInfo)
+
+    def do_get(self, key):
+        if key is None:
+            self.sendCode(400, b"Command 'get' takes 1 parameters.")
+        else:
+            d = defer.maybeDeferred(self.factory.get, key)
+            d.addCallbacks(self._cbGot, self._cbNot)
+            d.addErrback(log.err)
+
+    def _cbNot(self, fail):
+        msg = fail.getErrorMessage().encode("ascii")
+        self.sendCode(400, msg)
+
+    def _cbGot(self, value):
+        if value is None:
+            self.sendCode(500)
+        else:
+            self.sendCode(200, quote(value))
+
+    def do_put(self, keyAndValue):
+        if keyAndValue is None:
+            self.sendCode(400, b"Command 'put' takes 2 parameters.")
+        else:
+            try:
+                key, value = keyAndValue.split(None, 1)
+            except ValueError:
+                self.sendCode(400, b"Command 'put' takes 2 parameters.")
+            else:
+                self.sendCode(500, b"put is not implemented yet.")
+
+
+if TYPE_CHECKING or sys.version_info >= (3, 9):
+    _PostfixTCPMapDict = UserDict[bytes, Union[str, bytes]]
+else:
+    _PostfixTCPMapDict = UserDict
+
+
+class PostfixTCPMapDictServerFactory(_PostfixTCPMapDict, protocol.ServerFactory):
+    """
+    An in-memory dictionary factory for PostfixTCPMapServer.
+    """
+
+    protocol = PostfixTCPMapServer
+
+
+class PostfixTCPMapDeferringDictServerFactory(protocol.ServerFactory):
+    """
+    An in-memory dictionary factory for PostfixTCPMapServer.
+    """
+
+    protocol = PostfixTCPMapServer
+
+    def __init__(self, data=None):
+        self.data = {}
+        if data is not None:
+            self.data.update(data)
+
+    def get(self, key):
+        return defer.succeed(self.data.get(key))
diff --git a/contrib/python/Twisted/py3/twisted/protocols/shoutcast.py b/contrib/python/Twisted/py3/twisted/protocols/shoutcast.py
new file mode 100644
index 00000000000..5d346abe2ab
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/shoutcast.py
@@ -0,0 +1,111 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Chop up shoutcast stream into MP3s and metadata, if available.
+"""
+
+from twisted import copyright
+from twisted.web import http
+
+
+class ShoutcastClient(http.HTTPClient):
+    """
+    Shoutcast HTTP stream.
+
+    Modes can be 'length', 'meta' and 'mp3'.
+
+    See U{http://www.smackfu.com/stuff/programming/shoutcast.html}
+    for details on the protocol.
+    """
+
+    userAgent = "Twisted Shoutcast client " + copyright.version
+
+    def __init__(self, path="/"):
+        self.path = path
+        self.got_metadata = False
+        self.metaint = None
+        self.metamode = "mp3"
+        self.databuffer = ""
+
+    def connectionMade(self):
+        self.sendCommand("GET", self.path)
+        self.sendHeader("User-Agent", self.userAgent)
+        self.sendHeader("Icy-MetaData", "1")
+        self.endHeaders()
+
+    def lineReceived(self, line):
+        # fix shoutcast crappiness
+        if not self.firstLine and line:
+            if len(line.split(": ", 1)) == 1:
+                line = line.replace(":", ": ", 1)
+        http.HTTPClient.lineReceived(self, line)
+
+    def handleHeader(self, key, value):
+        if key.lower() == "icy-metaint":
+            self.metaint = int(value)
+            self.got_metadata = True
+
+    def handleEndHeaders(self):
+        # Lets check if we got metadata, and set the
+        # appropriate handleResponsePart method.
+        if self.got_metadata:
+            # if we have metadata, then it has to be parsed out of the data stream
+            self.handleResponsePart = self.handleResponsePart_with_metadata
+        else:
+            # otherwise, all the data is MP3 data
+            self.handleResponsePart = self.gotMP3Data
+
+    def handleResponsePart_with_metadata(self, data):
+        self.databuffer += data
+        while self.databuffer:
+            stop = getattr(self, "handle_%s" % self.metamode)()
+            if stop:
+                return
+
+    def handle_length(self):
+        self.remaining = ord(self.databuffer[0]) * 16
+        self.databuffer = self.databuffer[1:]
+        self.metamode = "meta"
+
+    def handle_mp3(self):
+        if len(self.databuffer) > self.metaint:
+            self.gotMP3Data(self.databuffer[: self.metaint])
+            self.databuffer = self.databuffer[self.metaint :]
+            self.metamode = "length"
+        else:
+            return 1
+
+    def handle_meta(self):
+        if len(self.databuffer) >= self.remaining:
+            if self.remaining:
+                data = self.databuffer[: self.remaining]
+                self.gotMetaData(self.parseMetadata(data))
+            self.databuffer = self.databuffer[self.remaining :]
+            self.metamode = "mp3"
+        else:
+            return 1
+
+    def parseMetadata(self, data):
+        meta = []
+        for chunk in data.split(";"):
+            chunk = chunk.strip().replace("\x00", "")
+            if not chunk:
+                continue
+            key, value = chunk.split("=", 1)
+            if value.startswith("'") and value.endswith("'"):
+                value = value[1:-1]
+            meta.append((key, value))
+        return meta
+
+    def gotMetaData(self, metadata):
+        """Called with a list of (key, value) pairs of metadata,
+        if metadata is available on the server.
+
+        Will only be called on non-empty metadata.
+        """
+        raise NotImplementedError("implement in subclass")
+
+    def gotMP3Data(self, data):
+        """Called with chunk of MP3 data."""
+        raise NotImplementedError("implement in subclass")
diff --git a/contrib/python/Twisted/py3/twisted/protocols/sip.py b/contrib/python/Twisted/py3/twisted/protocols/sip.py
new file mode 100644
index 00000000000..51cac143d86
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/sip.py
@@ -0,0 +1,1251 @@
+# -*- test-case-name: twisted.test.test_sip -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Session Initialization Protocol.
+
+Documented in RFC 2543.
+[Superseded by 3261]
+"""
+
+import socket
+import time
+import warnings
+from collections import OrderedDict
+from typing import Dict, List
+
+from zope.interface import Interface, implementer
+
+from twisted import cred
+from twisted.internet import defer, protocol, reactor
+from twisted.protocols import basic
+from twisted.python import log
+
+PORT = 5060
+
+# SIP headers have short forms
+shortHeaders = {
+    "call-id": "i",
+    "contact": "m",
+    "content-encoding": "e",
+    "content-length": "l",
+    "content-type": "c",
+    "from": "f",
+    "subject": "s",
+    "to": "t",
+    "via": "v",
+}
+
+longHeaders = {}
+for k, v in shortHeaders.items():
+    longHeaders[v] = k
+del k, v
+
+statusCodes = {
+    100: "Trying",
+    180: "Ringing",
+    181: "Call Is Being Forwarded",
+    182: "Queued",
+    183: "Session Progress",
+    200: "OK",
+    300: "Multiple Choices",
+    301: "Moved Permanently",
+    302: "Moved Temporarily",
+    303: "See Other",
+    305: "Use Proxy",
+    380: "Alternative Service",
+    400: "Bad Request",
+    401: "Unauthorized",
+    402: "Payment Required",
+    403: "Forbidden",
+    404: "Not Found",
+    405: "Method Not Allowed",
+    406: "Not Acceptable",
+    407: "Proxy Authentication Required",
+    408: "Request Timeout",
+    409: "Conflict",  # Not in RFC3261
+    410: "Gone",
+    411: "Length Required",  # Not in RFC3261
+    413: "Request Entity Too Large",
+    414: "Request-URI Too Large",
+    415: "Unsupported Media Type",
+    416: "Unsupported URI Scheme",
+    420: "Bad Extension",
+    421: "Extension Required",
+    423: "Interval Too Brief",
+    480: "Temporarily Unavailable",
+    481: "Call/Transaction Does Not Exist",
+    482: "Loop Detected",
+    483: "Too Many Hops",
+    484: "Address Incomplete",
+    485: "Ambiguous",
+    486: "Busy Here",
+    487: "Request Terminated",
+    488: "Not Acceptable Here",
+    491: "Request Pending",
+    493: "Undecipherable",
+    500: "Internal Server Error",
+    501: "Not Implemented",
+    502: "Bad Gateway",  # No donut
+    503: "Service Unavailable",
+    504: "Server Time-out",
+    505: "SIP Version not supported",
+    513: "Message Too Large",
+    600: "Busy Everywhere",
+    603: "Decline",
+    604: "Does not exist anywhere",
+    606: "Not Acceptable",
+}
+
+specialCases = {
+    "cseq": "CSeq",
+    "call-id": "Call-ID",
+    "www-authenticate": "WWW-Authenticate",
+}
+
+
+def dashCapitalize(s):
+    """
+    Capitalize a string, making sure to treat '-' as a word separator
+    """
+    return "-".join([x.capitalize() for x in s.split("-")])
+
+
+def unq(s):
+    if s[0] == s[-1] == '"':
+        return s[1:-1]
+    return s
+
+
+_absent = object()
+
+
+class Via:
+    """
+    A L{Via} is a SIP Via header, representing a segment of the path taken by
+    the request.
+
+    See RFC 3261, sections 8.1.1.7, 18.2.2, and 20.42.
+
+    @ivar transport: Network protocol used for this leg. (Probably either "TCP"
+    or "UDP".)
+    @type transport: C{str}
+    @ivar branch: Unique identifier for this request.
+    @type branch: C{str}
+    @ivar host: Hostname or IP for this leg.
+    @type host: C{str}
+    @ivar port: Port used for this leg.
+    @type port C{int}, or None.
+    @ivar rportRequested: Whether to request RFC 3581 client processing or not.
+    @type rportRequested: C{bool}
+    @ivar rportValue: Servers wishing to honor requests for RFC 3581 processing
+    should set this parameter to the source port the request was received
+    from.
+    @type rportValue: C{int}, or None.
+
+    @ivar ttl: Time-to-live for requests on multicast paths.
+    @type ttl: C{int}, or None.
+    @ivar maddr: The destination multicast address, if any.
+    @type maddr: C{str}, or None.
+    @ivar hidden: Obsolete in SIP 2.0.
+    @type hidden: C{bool}
+    @ivar otherParams: Any other parameters in the header.
+    @type otherParams: C{dict}
+    """
+
+    def __init__(
+        self,
+        host,
+        port=PORT,
+        transport="UDP",
+        ttl=None,
+        hidden=False,
+        received=None,
+        rport=_absent,
+        branch=None,
+        maddr=None,
+        **kw,
+    ):
+        """
+        Set parameters of this Via header. All arguments correspond to
+        attributes of the same name.
+
+        To maintain compatibility with old SIP
+        code, the 'rport' argument is used to determine the values of
+        C{rportRequested} and C{rportValue}. If None, C{rportRequested} is set
+        to True. (The deprecated method for doing this is to pass True.) If an
+        integer, C{rportValue} is set to the given value.
+
+        Any arguments not explicitly named here are collected into the
+        C{otherParams} dict.
+        """
+        self.transport = transport
+        self.host = host
+        self.port = port
+        self.ttl = ttl
+        self.hidden = hidden
+        self.received = received
+        if rport is True:
+            warnings.warn(
+                "rport=True is deprecated since Twisted 9.0.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            self.rportValue = None
+            self.rportRequested = True
+        elif rport is None:
+            self.rportValue = None
+            self.rportRequested = True
+        elif rport is _absent:
+            self.rportValue = None
+            self.rportRequested = False
+        else:
+            self.rportValue = rport
+            self.rportRequested = False
+
+        self.branch = branch
+        self.maddr = maddr
+        self.otherParams = kw
+
+    @property
+    def rport(self):
+        """
+        Returns the rport value expected by the old SIP code.
+        """
+        if self.rportRequested == True:
+            return True
+        elif self.rportValue is not None:
+            return self.rportValue
+        else:
+            return None
+
+    @rport.setter
+    def rport(self, newRPort):
+        """
+        L{Base._fixupNAT} sets C{rport} directly, so this method sets
+        C{rportValue} based on that.
+
+        @param newRPort: The new rport value.
+        @type newRPort: C{int}
+        """
+        self.rportValue = newRPort
+        self.rportRequested = False
+
+    def toString(self):
+        """
+        Serialize this header for use in a request or response.
+        """
+        s = f"SIP/2.0/{self.transport} {self.host}:{self.port}"
+        if self.hidden:
+            s += ";hidden"
+        for n in "ttl", "branch", "maddr", "received":
+            value = getattr(self, n)
+            if value is not None:
+                s += f";{n}={value}"
+        if self.rportRequested:
+            s += ";rport"
+        elif self.rportValue is not None:
+            s += f";rport={self.rport}"
+
+        etc = sorted(self.otherParams.items())
+        for k, v in etc:
+            if v is None:
+                s += ";" + k
+            else:
+                s += f";{k}={v}"
+        return s
+
+
+def parseViaHeader(value):
+    """
+    Parse a Via header.
+
+    @return: The parsed version of this header.
+    @rtype: L{Via}
+    """
+    parts = value.split(";")
+    sent, params = parts[0], parts[1:]
+    protocolinfo, by = sent.split(" ", 1)
+    by = by.strip()
+    result = {}
+    pname, pversion, transport = protocolinfo.split("/")
+    if pname != "SIP" or pversion != "2.0":
+        raise ValueError(f"wrong protocol or version: {value!r}")
+    result["transport"] = transport
+    if ":" in by:
+        host, port = by.split(":")
+        result["port"] = int(port)
+        result["host"] = host
+    else:
+        result["host"] = by
+    for p in params:
+        # It's the comment-striping dance!
+        p = p.strip().split(" ", 1)
+        if len(p) == 1:
+            p, comment = p[0], ""
+        else:
+            p, comment = p
+        if p == "hidden":
+            result["hidden"] = True
+            continue
+        parts = p.split("=", 1)
+        if len(parts) == 1:
+            name, value = parts[0], None
+        else:
+            name, value = parts
+            if name in ("rport", "ttl"):
+                value = int(value)
+        result[name] = value
+    return Via(**result)
+
+
+class URL:
+    """
+    A SIP URL.
+    """
+
+    def __init__(
+        self,
+        host,
+        username=None,
+        password=None,
+        port=None,
+        transport=None,
+        usertype=None,
+        method=None,
+        ttl=None,
+        maddr=None,
+        tag=None,
+        other=None,
+        headers=None,
+    ):
+        self.username = username
+        self.host = host
+        self.password = password
+        self.port = port
+        self.transport = transport
+        self.usertype = usertype
+        self.method = method
+        self.tag = tag
+        self.ttl = ttl
+        self.maddr = maddr
+        if other == None:
+            self.other = []
+        else:
+            self.other = other
+        if headers == None:
+            self.headers = {}
+        else:
+            self.headers = headers
+
+    def toString(self) -> str:
+        l: List[str] = []
+        w = l.append
+        w("sip:")
+        if self.username != None:
+            w(self.username)
+            if self.password != None:
+                w(":%s" % self.password)
+            w("@")
+        w(self.host)
+        if self.port != None:
+            w(":%d" % self.port)
+        if self.usertype != None:
+            w(";user=%s" % self.usertype)
+        for n in ("transport", "ttl", "maddr", "method", "tag"):
+            v = getattr(self, n)
+            if v != None:
+                w(f";{n}={v}")
+        for v in self.other:
+            w(";%s" % v)
+        if self.headers:
+            w("?")
+            w(
+                "&".join(
+                    [
+                        (f"{specialCases.get(h) or dashCapitalize(h)}={v}")
+                        for (h, v) in self.headers.items()
+                    ]
+                )
+            )
+        return "".join(l)
+
+    def __str__(self) -> str:
+        return self.toString()
+
+    def __repr__(self) -> str:
+        return "<URL {}:{}@{}:{!r}/{}>".format(
+            self.username,
+            self.password,
+            self.host,
+            self.port,
+            self.transport,
+        )
+
+
+def parseURL(url, host=None, port=None):
+    """
+    Return string into URL object.
+
+    URIs are of form 'sip:user@example.com'.
+    """
+    d = {}
+    if not url.startswith("sip:"):
+        raise ValueError("unsupported scheme: " + url[:4])
+    parts = url[4:].split(";")
+    userdomain, params = parts[0], parts[1:]
+    udparts = userdomain.split("@", 1)
+    if len(udparts) == 2:
+        userpass, hostport = udparts
+        upparts = userpass.split(":", 1)
+        if len(upparts) == 1:
+            d["username"] = upparts[0]
+        else:
+            d["username"] = upparts[0]
+            d["password"] = upparts[1]
+    else:
+        hostport = udparts[0]
+    hpparts = hostport.split(":", 1)
+    if len(hpparts) == 1:
+        d["host"] = hpparts[0]
+    else:
+        d["host"] = hpparts[0]
+        d["port"] = int(hpparts[1])
+    if host != None:
+        d["host"] = host
+    if port != None:
+        d["port"] = port
+    for p in params:
+        if p == params[-1] and "?" in p:
+            d["headers"] = h = {}
+            p, headers = p.split("?", 1)
+            for header in headers.split("&"):
+                k, v = header.split("=")
+                h[k] = v
+        nv = p.split("=", 1)
+        if len(nv) == 1:
+            d.setdefault("other", []).append(p)
+            continue
+        name, value = nv
+        if name == "user":
+            d["usertype"] = value
+        elif name in ("transport", "ttl", "maddr", "method", "tag"):
+            if name == "ttl":
+                value = int(value)
+            d[name] = value
+        else:
+            d.setdefault("other", []).append(p)
+    return URL(**d)
+
+
+def cleanRequestURL(url):
+    """
+    Clean a URL from a Request line.
+    """
+    url.transport = None
+    url.maddr = None
+    url.ttl = None
+    url.headers = {}
+
+
+def parseAddress(address, host=None, port=None, clean=0):
+    """
+    Return (name, uri, params) for From/To/Contact header.
+
+    @param clean: remove unnecessary info, usually for From and To headers.
+    """
+    address = address.strip()
+    # Simple 'sip:foo' case
+    if address.startswith("sip:"):
+        return "", parseURL(address, host=host, port=port), {}
+    params = {}
+    name, url = address.split("<", 1)
+    name = name.strip()
+    if name.startswith('"'):
+        name = name[1:]
+    if name.endswith('"'):
+        name = name[:-1]
+    url, paramstring = url.split(">", 1)
+    url = parseURL(url, host=host, port=port)
+    paramstring = paramstring.strip()
+    if paramstring:
+        for l in paramstring.split(";"):
+            if not l:
+                continue
+            k, v = l.split("=")
+            params[k] = v
+    if clean:
+        # RFC 2543 6.21
+        url.ttl = None
+        url.headers = {}
+        url.transport = None
+        url.maddr = None
+    return name, url, params
+
+
+class SIPError(Exception):
+    def __init__(self, code, phrase=None):
+        if phrase is None:
+            phrase = statusCodes[code]
+        Exception.__init__(self, "SIP error (%d): %s" % (code, phrase))
+        self.code = code
+        self.phrase = phrase
+
+
+class RegistrationError(SIPError):
+    """
+    Registration was not possible.
+    """
+
+
+class Message:
+    """
+    A SIP message.
+    """
+
+    length = None
+
+    def __init__(self):
+        self.headers = OrderedDict()  # Map name to list of values
+        self.body = ""
+        self.finished = 0
+
+    def addHeader(self, name, value):
+        name = name.lower()
+        name = longHeaders.get(name, name)
+        if name == "content-length":
+            self.length = int(value)
+        self.headers.setdefault(name, []).append(value)
+
+    def bodyDataReceived(self, data):
+        self.body += data
+
+    def creationFinished(self):
+        if (self.length != None) and (self.length != len(self.body)):
+            raise ValueError("wrong body length")
+        self.finished = 1
+
+    def toString(self):
+        s = "%s\r\n" % self._getHeaderLine()
+        for n, vs in self.headers.items():
+            for v in vs:
+                s += f"{specialCases.get(n) or dashCapitalize(n)}: {v}\r\n"
+        s += "\r\n"
+        s += self.body
+        return s
+
+    def _getHeaderLine(self):
+        raise NotImplementedError
+
+
+class Request(Message):
+    """
+    A Request for a URI
+    """
+
+    def __init__(self, method, uri, version="SIP/2.0"):
+        Message.__init__(self)
+        self.method = method
+        if isinstance(uri, URL):
+            self.uri = uri
+        else:
+            self.uri = parseURL(uri)
+            cleanRequestURL(self.uri)
+
+    def __repr__(self) -> str:
+        return "<SIP Request %d:%s %s>" % (id(self), self.method, self.uri.toString())
+
+    def _getHeaderLine(self):
+        return f"{self.method} {self.uri.toString()} SIP/2.0"
+
+
+class Response(Message):
+    """
+    A Response to a URI Request
+    """
+
+    def __init__(self, code, phrase=None, version="SIP/2.0"):
+        Message.__init__(self)
+        self.code = code
+        if phrase == None:
+            phrase = statusCodes[code]
+        self.phrase = phrase
+
+    def __repr__(self) -> str:
+        return "<SIP Response %d:%s>" % (id(self), self.code)
+
+    def _getHeaderLine(self):
+        return f"SIP/2.0 {self.code} {self.phrase}"
+
+
+class MessagesParser(basic.LineReceiver):
+    """
+    A SIP messages parser.
+
+    Expects dataReceived, dataDone repeatedly,
+    in that order. Shouldn't be connected to actual transport.
+    """
+
+    version = "SIP/2.0"
+    acceptResponses = 1
+    acceptRequests = 1
+    state = "firstline"  # Or "headers", "body" or "invalid"
+
+    debug = 0
+
+    def __init__(self, messageReceivedCallback):
+        self.messageReceived = messageReceivedCallback
+        self.reset()
+
+    def reset(self, remainingData=""):
+        self.state = "firstline"
+        self.length = None  # Body length
+        self.bodyReceived = 0  # How much of the body we received
+        self.message = None
+        self.header = None
+        self.setLineMode(remainingData)
+
+    def invalidMessage(self):
+        self.state = "invalid"
+        self.setRawMode()
+
+    def dataDone(self):
+        """
+        Clear out any buffered data that may be hanging around.
+        """
+        self.clearLineBuffer()
+        if self.state == "firstline":
+            return
+        if self.state != "body":
+            self.reset()
+            return
+        if self.length == None:
+            # No content-length header, so end of data signals message done
+            self.messageDone()
+        elif self.length < self.bodyReceived:
+            # Aborted in the middle
+            self.reset()
+        else:
+            # We have enough data and message wasn't finished? something is wrong
+            raise RuntimeError("this should never happen")
+
+    def dataReceived(self, data):
+        try:
+            if isinstance(data, str):
+                data = data.encode("utf-8")
+            basic.LineReceiver.dataReceived(self, data)
+        except Exception:
+            log.err()
+            self.invalidMessage()
+
+    def handleFirstLine(self, line):
+        """
+        Expected to create self.message.
+        """
+        raise NotImplementedError
+
+    def lineLengthExceeded(self, line):
+        self.invalidMessage()
+
+    def lineReceived(self, line):
+        if isinstance(line, bytes):
+            line = line.decode("utf-8")
+
+        if self.state == "firstline":
+            while line.startswith("\n") or line.startswith("\r"):
+                line = line[1:]
+            if not line:
+                return
+            try:
+                a, b, c = line.split(" ", 2)
+            except ValueError:
+                self.invalidMessage()
+                return
+            if a == "SIP/2.0" and self.acceptResponses:
+                # Response
+                try:
+                    code = int(b)
+                except ValueError:
+                    self.invalidMessage()
+                    return
+                self.message = Response(code, c)
+            elif c == "SIP/2.0" and self.acceptRequests:
+                self.message = Request(a, b)
+            else:
+                self.invalidMessage()
+                return
+            self.state = "headers"
+            return
+        else:
+            assert self.state == "headers"
+        if line:
+            # Multiline header
+            if line.startswith(" ") or line.startswith("\t"):
+                name, value = self.header
+                self.header = name, (value + line.lstrip())
+            else:
+                # New header
+                if self.header:
+                    self.message.addHeader(*self.header)
+                    self.header = None
+                try:
+                    name, value = line.split(":", 1)
+                except ValueError:
+                    self.invalidMessage()
+                    return
+                self.header = name, value.lstrip()
+                # XXX we assume content-length won't be multiline
+                if name.lower() == "content-length":
+                    try:
+                        self.length = int(value.lstrip())
+                    except ValueError:
+                        self.invalidMessage()
+                        return
+        else:
+            # CRLF, we now have message body until self.length bytes,
+            # or if no length was given, until there is no more data
+            # from the connection sending us data.
+            self.state = "body"
+            if self.header:
+                self.message.addHeader(*self.header)
+                self.header = None
+            if self.length == 0:
+                self.messageDone()
+                return
+            self.setRawMode()
+
+    def messageDone(self, remainingData=""):
+        assert self.state == "body"
+        self.message.creationFinished()
+        self.messageReceived(self.message)
+        self.reset(remainingData)
+
+    def rawDataReceived(self, data):
+        assert self.state in ("body", "invalid")
+        if isinstance(data, bytes):
+            data = data.decode("utf-8")
+        if self.state == "invalid":
+            return
+        if self.length == None:
+            self.message.bodyDataReceived(data)
+        else:
+            dataLen = len(data)
+            expectedLen = self.length - self.bodyReceived
+            if dataLen > expectedLen:
+                self.message.bodyDataReceived(data[:expectedLen])
+                self.messageDone(data[expectedLen:])
+                return
+            else:
+                self.bodyReceived += dataLen
+                self.message.bodyDataReceived(data)
+                if self.bodyReceived == self.length:
+                    self.messageDone()
+
+
+class Base(protocol.DatagramProtocol):
+    """
+    Base class for SIP clients and servers.
+    """
+
+    PORT = PORT
+    debug = False
+
+    def __init__(self):
+        self.messages = []
+        self.parser = MessagesParser(self.addMessage)
+
+    def addMessage(self, msg):
+        self.messages.append(msg)
+
+    def datagramReceived(self, data, addr):
+        self.parser.dataReceived(data)
+        self.parser.dataDone()
+        for m in self.messages:
+            self._fixupNAT(m, addr)
+            if self.debug:
+                log.msg(f"Received {m.toString()!r} from {addr!r}")
+            if isinstance(m, Request):
+                self.handle_request(m, addr)
+            else:
+                self.handle_response(m, addr)
+        self.messages[:] = []
+
+    def _fixupNAT(self, message, sourcePeer):
+        # RFC 2543 6.40.2,
+        (srcHost, srcPort) = sourcePeer
+        senderVia = parseViaHeader(message.headers["via"][0])
+        if senderVia.host != srcHost:
+            senderVia.received = srcHost
+            if senderVia.port != srcPort:
+                senderVia.rport = srcPort
+            message.headers["via"][0] = senderVia.toString()
+        elif senderVia.rport == True:
+            senderVia.received = srcHost
+            senderVia.rport = srcPort
+            message.headers["via"][0] = senderVia.toString()
+
+    def deliverResponse(self, responseMessage):
+        """
+        Deliver response.
+
+        Destination is based on topmost Via header.
+        """
+        destVia = parseViaHeader(responseMessage.headers["via"][0])
+        # XXX we don't do multicast yet
+        host = destVia.received or destVia.host
+        port = destVia.rport or destVia.port or self.PORT
+        destAddr = URL(host=host, port=port)
+        self.sendMessage(destAddr, responseMessage)
+
+    def responseFromRequest(self, code, request):
+        """
+        Create a response to a request message.
+        """
+        response = Response(code)
+        for name in ("via", "to", "from", "call-id", "cseq"):
+            response.headers[name] = request.headers.get(name, [])[:]
+
+        return response
+
+    def sendMessage(self, destURL, message):
+        """
+        Send a message.
+
+        @param destURL: C{URL}. This should be a *physical* URL, not a logical one.
+        @param message: The message to send.
+        """
+        if destURL.transport not in ("udp", None):
+            raise RuntimeError("only UDP currently supported")
+        if self.debug:
+            log.msg(f"Sending {message.toString()!r} to {destURL!r}")
+        data = message.toString()
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        self.transport.write(data, (destURL.host, destURL.port or self.PORT))
+
+    def handle_request(self, message, addr):
+        """
+        Override to define behavior for requests received
+
+        @type message: C{Message}
+        @type addr: C{tuple}
+        """
+        raise NotImplementedError
+
+    def handle_response(self, message, addr):
+        """
+        Override to define behavior for responses received.
+
+        @type message: C{Message}
+        @type addr: C{tuple}
+        """
+        raise NotImplementedError
+
+
+class IContact(Interface):
+    """
+    A user of a registrar or proxy
+    """
+
+
+class Registration:
+    def __init__(self, secondsToExpiry, contactURL):
+        self.secondsToExpiry = secondsToExpiry
+        self.contactURL = contactURL
+
+
+class IRegistry(Interface):
+    """
+    Allows registration of logical->physical URL mapping.
+    """
+
+    def registerAddress(domainURL, logicalURL, physicalURL):
+        """
+        Register the physical address of a logical URL.
+
+        @return: Deferred of C{Registration} or failure with RegistrationError.
+        """
+
+    def unregisterAddress(domainURL, logicalURL, physicalURL):
+        """
+        Unregister the physical address of a logical URL.
+
+        @return: Deferred of C{Registration} or failure with RegistrationError.
+        """
+
+    def getRegistrationInfo(logicalURL):
+        """
+        Get registration info for logical URL.
+
+        @return: Deferred of C{Registration} object or failure of LookupError.
+        """
+
+
+class ILocator(Interface):
+    """
+    Allow looking up physical address for logical URL.
+    """
+
+    def getAddress(logicalURL):
+        """
+        Return physical URL of server for logical URL of user.
+
+        @param logicalURL: a logical C{URL}.
+        @return: Deferred which becomes URL or fails with LookupError.
+        """
+
+
+class Proxy(Base):
+    """
+    SIP proxy.
+    """
+
+    PORT = PORT
+
+    locator = None  # Object implementing ILocator
+
+    def __init__(self, host=None, port=PORT):
+        """
+        Create new instance.
+
+        @param host: our hostname/IP as set in Via headers.
+        @param port: our port as set in Via headers.
+        """
+        self.host = host or socket.getfqdn()
+        self.port = port
+        Base.__init__(self)
+
+    def getVia(self):
+        """
+        Return value of Via header for this proxy.
+        """
+        return Via(host=self.host, port=self.port)
+
+    def handle_request(self, message, addr):
+        # Send immediate 100/trying message before processing
+        # self.deliverResponse(self.responseFromRequest(100, message))
+        f = getattr(self, "handle_%s_request" % message.method, None)
+        if f is None:
+            f = self.handle_request_default
+        try:
+            d = f(message, addr)
+        except SIPError as e:
+            self.deliverResponse(self.responseFromRequest(e.code, message))
+        except BaseException:
+            log.err()
+            self.deliverResponse(self.responseFromRequest(500, message))
+        else:
+            if d is not None:
+                d.addErrback(
+                    lambda e: self.deliverResponse(
+                        self.responseFromRequest(e.code, message)
+                    )
+                )
+
+    def handle_request_default(self, message, sourcePeer):
+        """
+        Default request handler.
+
+        Default behaviour for OPTIONS and unknown methods for proxies
+        is to forward message on to the client.
+
+        Since at the moment we are stateless proxy, that's basically
+        everything.
+        """
+        (srcHost, srcPort) = sourcePeer
+
+        def _mungContactHeader(uri, message):
+            message.headers["contact"][0] = uri.toString()
+            return self.sendMessage(uri, message)
+
+        viaHeader = self.getVia()
+        if viaHeader.toString() in message.headers["via"]:
+            # Must be a loop, so drop message
+            log.msg("Dropping looped message.")
+            return
+
+        message.headers["via"].insert(0, viaHeader.toString())
+        name, uri, tags = parseAddress(message.headers["to"][0], clean=1)
+
+        # This is broken and needs refactoring to use cred
+        d = self.locator.getAddress(uri)
+        d.addCallback(self.sendMessage, message)
+        d.addErrback(self._cantForwardRequest, message)
+
+    def _cantForwardRequest(self, error, message):
+        error.trap(LookupError)
+        del message.headers["via"][0]  # This'll be us
+        self.deliverResponse(self.responseFromRequest(404, message))
+
+    def deliverResponse(self, responseMessage):
+        """
+        Deliver response.
+
+        Destination is based on topmost Via header.
+        """
+        destVia = parseViaHeader(responseMessage.headers["via"][0])
+        # XXX we don't do multicast yet
+        host = destVia.received or destVia.host
+        port = destVia.rport or destVia.port or self.PORT
+
+        destAddr = URL(host=host, port=port)
+        self.sendMessage(destAddr, responseMessage)
+
+    def responseFromRequest(self, code, request):
+        """
+        Create a response to a request message.
+        """
+        response = Response(code)
+        for name in ("via", "to", "from", "call-id", "cseq"):
+            response.headers[name] = request.headers.get(name, [])[:]
+        return response
+
+    def handle_response(self, message, addr):
+        """
+        Default response handler.
+        """
+        v = parseViaHeader(message.headers["via"][0])
+        if (v.host, v.port) != (self.host, self.port):
+            # We got a message not intended for us?
+            # XXX note this check breaks if we have multiple external IPs
+            # yay for suck protocols
+            log.msg("Dropping incorrectly addressed message")
+            return
+        del message.headers["via"][0]
+        if not message.headers["via"]:
+            # This message is addressed to us
+            self.gotResponse(message, addr)
+            return
+        self.deliverResponse(message)
+
+    def gotResponse(self, message, addr):
+        """
+        Called with responses that are addressed at this server.
+        """
+        pass
+
+
+class IAuthorizer(Interface):
+    def getChallenge(peer):
+        """
+        Generate a challenge the client may respond to.
+
+        @type peer: C{tuple}
+        @param peer: The client's address
+
+        @rtype: C{str}
+        @return: The challenge string
+        """
+
+    def decode(response):
+        """
+        Create a credentials object from the given response.
+
+        @type response: C{str}
+        """
+
+
+class RegisterProxy(Proxy):
+    """
+    A proxy that allows registration for a specific domain.
+
+    Unregistered users won't be handled.
+    """
+
+    portal = None
+
+    registry = None  # Should implement IRegistry
+
+    authorizers: Dict[str, IAuthorizer] = {}
+
+    def __init__(self, *args, **kw):
+        Proxy.__init__(self, *args, **kw)
+        self.liveChallenges = {}
+
+    def handle_ACK_request(self, message, host_port):
+        # XXX
+        # ACKs are a client's way of indicating they got the last message
+        # Responding to them is not a good idea.
+        # However, we should keep track of terminal messages and re-transmit
+        # if no ACK is received.
+        (host, port) = host_port
+        pass
+
+    def handle_REGISTER_request(self, message, host_port):
+        """
+        Handle a registration request.
+
+        Currently registration is not proxied.
+        """
+        (host, port) = host_port
+        if self.portal is None:
+            # There is no portal.  Let anyone in.
+            self.register(message, host, port)
+        else:
+            # There is a portal.  Check for credentials.
+            if "authorization" not in message.headers:
+                return self.unauthorized(message, host, port)
+            else:
+                return self.login(message, host, port)
+
+    def unauthorized(self, message, host, port):
+        m = self.responseFromRequest(401, message)
+        for scheme, auth in self.authorizers.items():
+            chal = auth.getChallenge((host, port))
+            if chal is None:
+                value = f'{scheme.title()} realm="{self.host}"'
+            else:
+                value = f'{scheme.title()} {chal},realm="{self.host}"'
+            m.headers.setdefault("www-authenticate", []).append(value)
+        self.deliverResponse(m)
+
+    def login(self, message, host, port):
+        parts = message.headers["authorization"][0].split(None, 1)
+        a = self.authorizers.get(parts[0].lower())
+        if a:
+            try:
+                c = a.decode(parts[1])
+            except SIPError:
+                raise
+            except BaseException:
+                log.err()
+                self.deliverResponse(self.responseFromRequest(500, message))
+            else:
+                c.username += "@" + self.host
+                self.portal.login(c, None, IContact).addCallback(
+                    self._cbLogin, message, host, port
+                ).addErrback(self._ebLogin, message, host, port).addErrback(log.err)
+        else:
+            self.deliverResponse(self.responseFromRequest(501, message))
+
+    def _cbLogin(self, i_a_l, message, host, port):
+        # It's stateless, matey.  What a joke.
+        (i, a, l) = i_a_l
+        self.register(message, host, port)
+
+    def _ebLogin(self, failure, message, host, port):
+        failure.trap(cred.error.UnauthorizedLogin)
+        self.unauthorized(message, host, port)
+
+    def register(self, message, host, port):
+        """
+        Allow all users to register
+        """
+        name, toURL, params = parseAddress(message.headers["to"][0], clean=1)
+        contact = None
+        if "contact" in message.headers:
+            contact = message.headers["contact"][0]
+
+        if message.headers.get("expires", [None])[0] == "0":
+            self.unregister(message, toURL, contact)
+        else:
+            # XXX Check expires on appropriate URL, and pass it to registry
+            # instead of having registry hardcode it.
+            if contact is not None:
+                name, contactURL, params = parseAddress(contact, host=host, port=port)
+                d = self.registry.registerAddress(message.uri, toURL, contactURL)
+            else:
+                d = self.registry.getRegistrationInfo(toURL)
+            d.addCallbacks(
+                self._cbRegister,
+                self._ebRegister,
+                callbackArgs=(message,),
+                errbackArgs=(message,),
+            )
+
+    def _cbRegister(self, registration, message):
+        response = self.responseFromRequest(200, message)
+        if registration.contactURL != None:
+            response.addHeader("contact", registration.contactURL.toString())
+            response.addHeader("expires", "%d" % registration.secondsToExpiry)
+        response.addHeader("content-length", "0")
+        self.deliverResponse(response)
+
+    def _ebRegister(self, error, message):
+        error.trap(RegistrationError, LookupError)
+        # XXX return error message, and alter tests to deal with
+        # this, currently tests assume no message sent on failure
+
+    def unregister(self, message, toURL, contact):
+        try:
+            expires = int(message.headers["expires"][0])
+        except ValueError:
+            self.deliverResponse(self.responseFromRequest(400, message))
+        else:
+            if expires == 0:
+                if contact == "*":
+                    contactURL = "*"
+                else:
+                    name, contactURL, params = parseAddress(contact)
+                d = self.registry.unregisterAddress(message.uri, toURL, contactURL)
+                d.addCallback(self._cbUnregister, message).addErrback(
+                    self._ebUnregister, message
+                )
+
+    def _cbUnregister(self, registration, message):
+        msg = self.responseFromRequest(200, message)
+        msg.headers.setdefault("contact", []).append(registration.contactURL.toString())
+        msg.addHeader("expires", "0")
+        self.deliverResponse(msg)
+
+    def _ebUnregister(self, registration, message):
+        pass
+
+
+@implementer(IRegistry, ILocator)
+class InMemoryRegistry:
+    """
+    A simplistic registry for a specific domain.
+    """
+
+    def __init__(self, domain):
+        self.domain = domain  # The domain we handle registration for
+        self.users = {}  # Map username to (IDelayedCall for expiry, address URI)
+
+    def getAddress(self, userURI):
+        if userURI.host != self.domain:
+            return defer.fail(LookupError("unknown domain"))
+        if userURI.username in self.users:
+            dc, url = self.users[userURI.username]
+            return defer.succeed(url)
+        else:
+            return defer.fail(LookupError("no such user"))
+
+    def getRegistrationInfo(self, userURI):
+        if userURI.host != self.domain:
+            return defer.fail(LookupError("unknown domain"))
+        if userURI.username in self.users:
+            dc, url = self.users[userURI.username]
+            return defer.succeed(Registration(int(dc.getTime() - time.time()), url))
+        else:
+            return defer.fail(LookupError("no such user"))
+
+    def _expireRegistration(self, username):
+        try:
+            dc, url = self.users[username]
+        except KeyError:
+            return defer.fail(LookupError("no such user"))
+        else:
+            dc.cancel()
+            del self.users[username]
+        return defer.succeed(Registration(0, url))
+
+    def registerAddress(self, domainURL, logicalURL, physicalURL):
+        if domainURL.host != self.domain:
+            log.msg("Registration for domain we don't handle.")
+            return defer.fail(RegistrationError(404))
+        if logicalURL.host != self.domain:
+            log.msg("Registration for domain we don't handle.")
+            return defer.fail(RegistrationError(404))
+        if logicalURL.username in self.users:
+            dc, old = self.users[logicalURL.username]
+            dc.reset(3600)
+        else:
+            dc = reactor.callLater(3600, self._expireRegistration, logicalURL.username)
+        log.msg(f"Registered {logicalURL.toString()} at {physicalURL.toString()}")
+        self.users[logicalURL.username] = (dc, physicalURL)
+        return defer.succeed(Registration(int(dc.getTime() - time.time()), physicalURL))
+
+    def unregisterAddress(self, domainURL, logicalURL, physicalURL):
+        return self._expireRegistration(logicalURL.username)
diff --git a/contrib/python/Twisted/py3/twisted/protocols/socks.py b/contrib/python/Twisted/py3/twisted/protocols/socks.py
new file mode 100644
index 00000000000..eec7d481bda
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/socks.py
@@ -0,0 +1,249 @@
+# -*- test-case-name: twisted.test.test_socks -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of the SOCKSv4 protocol.
+"""
+
+import socket
+import string
+
+# python imports
+import struct
+import time
+
+# twisted imports
+from twisted.internet import defer, protocol, reactor
+from twisted.python import log
+
+
+class SOCKSv4Outgoing(protocol.Protocol):
+    def __init__(self, socks):
+        self.socks = socks
+
+    def connectionMade(self):
+        peer = self.transport.getPeer()
+        self.socks.makeReply(90, 0, port=peer.port, ip=peer.host)
+        self.socks.otherConn = self
+
+    def connectionLost(self, reason):
+        self.socks.transport.loseConnection()
+
+    def dataReceived(self, data):
+        self.socks.write(data)
+
+    def write(self, data):
+        self.socks.log(self, data)
+        self.transport.write(data)
+
+
+class SOCKSv4Incoming(protocol.Protocol):
+    def __init__(self, socks):
+        self.socks = socks
+        self.socks.otherConn = self
+
+    def connectionLost(self, reason):
+        self.socks.transport.loseConnection()
+
+    def dataReceived(self, data):
+        self.socks.write(data)
+
+    def write(self, data):
+        self.socks.log(self, data)
+        self.transport.write(data)
+
+
+class SOCKSv4(protocol.Protocol):
+    """
+    An implementation of the SOCKSv4 protocol.
+
+    @type logging: L{str} or L{None}
+    @ivar logging: If not L{None}, the name of the logfile to which connection
+        information will be written.
+
+    @type reactor: object providing L{twisted.internet.interfaces.IReactorTCP}
+    @ivar reactor: The reactor used to create connections.
+
+    @type buf: L{str}
+    @ivar buf: Part of a SOCKSv4 connection request.
+
+    @type otherConn: C{SOCKSv4Incoming}, C{SOCKSv4Outgoing} or L{None}
+    @ivar otherConn: Until the connection has been established, C{otherConn} is
+        L{None}. After that, it is the proxy-to-destination protocol instance
+        along which the client's connection is being forwarded.
+    """
+
+    def __init__(self, logging=None, reactor=reactor):
+        self.logging = logging
+        self.reactor = reactor
+
+    def connectionMade(self):
+        self.buf = b""
+        self.otherConn = None
+
+    def dataReceived(self, data):
+        """
+        Called whenever data is received.
+
+        @type data: L{bytes}
+        @param data: Part or all of a SOCKSv4 packet.
+        """
+        if self.otherConn:
+            self.otherConn.write(data)
+            return
+        self.buf = self.buf + data
+        completeBuffer = self.buf
+        if b"\000" in self.buf[8:]:
+            head, self.buf = self.buf[:8], self.buf[8:]
+            version, code, port = struct.unpack("!BBH", head[:4])
+            user, self.buf = self.buf.split(b"\000", 1)
+            if head[4:7] == b"\000\000\000" and head[7:8] != b"\000":
+                # An IP address of the form 0.0.0.X, where X is non-zero,
+                # signifies that this is a SOCKSv4a packet.
+                # If the complete packet hasn't been received, restore the
+                # buffer and wait for it.
+                if b"\000" not in self.buf:
+                    self.buf = completeBuffer
+                    return
+                server, self.buf = self.buf.split(b"\000", 1)
+                d = self.reactor.resolve(server)
+                d.addCallback(self._dataReceived2, user, version, code, port)
+                d.addErrback(lambda result, self=self: self.makeReply(91))
+                return
+            else:
+                server = socket.inet_ntoa(head[4:8])
+
+            self._dataReceived2(server, user, version, code, port)
+
+    def _dataReceived2(self, server, user, version, code, port):
+        """
+        The second half of the SOCKS connection setup. For a SOCKSv4 packet this
+        is after the server address has been extracted from the header. For a
+        SOCKSv4a packet this is after the host name has been resolved.
+
+        @type server: L{str}
+        @param server: The IP address of the destination, represented as a
+            dotted quad.
+
+        @type user: L{str}
+        @param user: The username associated with the connection.
+
+        @type version: L{int}
+        @param version: The SOCKS protocol version number.
+
+        @type code: L{int}
+        @param code: The command code. 1 means establish a TCP/IP stream
+            connection, and 2 means establish a TCP/IP port binding.
+
+        @type port: L{int}
+        @param port: The port number associated with the connection.
+        """
+        assert version == 4, "Bad version code: %s" % version
+        if not self.authorize(code, server, port, user):
+            self.makeReply(91)
+            return
+        if code == 1:  # CONNECT
+            d = self.connectClass(server, port, SOCKSv4Outgoing, self)
+            d.addErrback(lambda result, self=self: self.makeReply(91))
+        elif code == 2:  # BIND
+            d = self.listenClass(0, SOCKSv4IncomingFactory, self, server)
+            d.addCallback(lambda x, self=self: self.makeReply(90, 0, x[1], x[0]))
+        else:
+            raise RuntimeError(f"Bad Connect Code: {code}")
+        assert self.buf == b"", "hmm, still stuff in buffer... %s" % repr(self.buf)
+
+    def connectionLost(self, reason):
+        if self.otherConn:
+            self.otherConn.transport.loseConnection()
+
+    def authorize(self, code, server, port, user):
+        log.msg(
+            "code %s connection to %s:%s (user %s) authorized"
+            % (code, server, port, user)
+        )
+        return 1
+
+    def connectClass(self, host, port, klass, *args):
+        return protocol.ClientCreator(reactor, klass, *args).connectTCP(host, port)
+
+    def listenClass(self, port, klass, *args):
+        serv = reactor.listenTCP(port, klass(*args))
+        return defer.succeed(serv.getHost()[1:])
+
+    def makeReply(self, reply, version=0, port=0, ip="0.0.0.0"):
+        self.transport.write(
+            struct.pack("!BBH", version, reply, port) + socket.inet_aton(ip)
+        )
+        if reply != 90:
+            self.transport.loseConnection()
+
+    def write(self, data):
+        self.log(self, data)
+        self.transport.write(data)
+
+    def log(self, proto, data):
+        if not self.logging:
+            return
+        peer = self.transport.getPeer()
+        their_peer = self.otherConn.transport.getPeer()
+        f = open(self.logging, "a")
+        f.write(
+            "%s\t%s:%d %s %s:%d\n"
+            % (
+                time.ctime(),
+                peer.host,
+                peer.port,
+                ((proto == self and "<") or ">"),
+                their_peer.host,
+                their_peer.port,
+            )
+        )
+        while data:
+            p, data = data[:16], data[16:]
+            f.write(string.join(map(lambda x: "%02X" % ord(x), p), " ") + " ")
+            f.write((16 - len(p)) * 3 * " ")
+            for c in p:
+                if len(repr(c)) > 3:
+                    f.write(".")
+                else:
+                    f.write(c)
+            f.write("\n")
+        f.write("\n")
+        f.close()
+
+
+class SOCKSv4Factory(protocol.Factory):
+    """
+    A factory for a SOCKSv4 proxy.
+
+    Constructor accepts one argument, a log file name.
+    """
+
+    def __init__(self, log):
+        self.logging = log
+
+    def buildProtocol(self, addr):
+        return SOCKSv4(self.logging, reactor)
+
+
+class SOCKSv4IncomingFactory(protocol.Factory):
+    """
+    A utility class for building protocols for incoming connections.
+    """
+
+    def __init__(self, socks, ip):
+        self.socks = socks
+        self.ip = ip
+
+    def buildProtocol(self, addr):
+        if addr[0] == self.ip:
+            self.ip = ""
+            self.socks.makeReply(90, 0)
+            return SOCKSv4Incoming(self.socks)
+        elif self.ip == "":
+            return None
+        else:
+            self.socks.makeReply(91, 0)
+            self.ip = ""
+            return None
diff --git a/contrib/python/Twisted/py3/twisted/protocols/stateful.py b/contrib/python/Twisted/py3/twisted/protocols/stateful.py
new file mode 100644
index 00000000000..ed2e5c361f9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/stateful.py
@@ -0,0 +1,52 @@
+# -*- test-case-name: twisted.test.test_stateful -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+from io import BytesIO
+
+from twisted.internet import protocol
+
+
+class StatefulProtocol(protocol.Protocol):
+    """A Protocol that stores state for you.
+
+    state is a pair (function, num_bytes). When num_bytes bytes of data arrives
+    from the network, function is called. It is expected to return the next
+    state or None to keep same state. Initial state is returned by
+    getInitialState (override it).
+    """
+
+    _sful_data = None, None, 0
+
+    def makeConnection(self, transport):
+        protocol.Protocol.makeConnection(self, transport)
+        self._sful_data = self.getInitialState(), BytesIO(), 0
+
+    def getInitialState(self):
+        raise NotImplementedError
+
+    def dataReceived(self, data):
+        state, buffer, offset = self._sful_data
+        buffer.seek(0, 2)
+        buffer.write(data)
+        blen = buffer.tell()  # how many bytes total is in the buffer
+        buffer.seek(offset)
+        while blen - offset >= state[1]:
+            d = buffer.read(state[1])
+            offset += state[1]
+            next = state[0](d)
+            if (
+                self.transport.disconnecting
+            ):  # XXX: argh stupid hack borrowed right from LineReceiver
+                return  # dataReceived won't be called again, so who cares about consistent state
+            if next:
+                state = next
+        if offset != 0:
+            b = buffer.read()
+            buffer.seek(0)
+            buffer.truncate()
+            buffer.write(b)
+            offset = 0
+        self._sful_data = state, buffer, offset
diff --git a/contrib/python/Twisted/py3/twisted/protocols/tls.py b/contrib/python/Twisted/py3/twisted/protocols/tls.py
new file mode 100644
index 00000000000..d2ac2d2cf80
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/tls.py
@@ -0,0 +1,936 @@
+# -*- test-case-name: twisted.protocols.test.test_tls,twisted.internet.test.test_tls,twisted.test.test_sslverify -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of a TLS transport (L{ISSLTransport}) as an
+L{IProtocol<twisted.internet.interfaces.IProtocol>} layered on top of any
+L{ITransport<twisted.internet.interfaces.ITransport>} implementation, based on
+U{OpenSSL<http://www.openssl.org>}'s memory BIO features.
+
+L{TLSMemoryBIOFactory} is a L{WrappingFactory} which wraps protocols created by
+the factory it wraps with L{TLSMemoryBIOProtocol}.  L{TLSMemoryBIOProtocol}
+intercedes between the underlying transport and the wrapped protocol to
+implement SSL and TLS.  Typical usage of this module looks like this::
+
+    from twisted.protocols.tls import TLSMemoryBIOFactory
+    from twisted.internet.protocol import ServerFactory
+    from twisted.internet.ssl import PrivateCertificate
+    from twisted.internet import reactor
+
+    from someapplication import ApplicationProtocol
+
+    serverFactory = ServerFactory()
+    serverFactory.protocol = ApplicationProtocol
+    certificate = PrivateCertificate.loadPEM(certPEMData)
+    contextFactory = certificate.options()
+    tlsFactory = TLSMemoryBIOFactory(contextFactory, False, serverFactory)
+    reactor.listenTCP(12345, tlsFactory)
+    reactor.run()
+
+This API offers somewhat more flexibility than
+L{twisted.internet.interfaces.IReactorSSL}; for example, a
+L{TLSMemoryBIOProtocol} instance can use another instance of
+L{TLSMemoryBIOProtocol} as its transport, yielding TLS over TLS - useful to
+implement onion routing.  It can also be used to run TLS over unusual
+transports, such as UNIX sockets and stdio.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Iterable, Optional, cast
+
+from zope.interface import directlyProvides, implementer, providedBy
+
+from OpenSSL.SSL import Connection, Error, SysCallError, WantReadError, ZeroReturnError
+
+from twisted.internet._producer_helpers import _PullToPush
+from twisted.internet._sslverify import _setAcceptableProtocols
+from twisted.internet.interfaces import (
+    IDelayedCall,
+    IHandshakeListener,
+    ILoggingContext,
+    INegotiated,
+    IOpenSSLClientConnectionCreator,
+    IOpenSSLServerConnectionCreator,
+    IProtocol,
+    IProtocolNegotiationFactory,
+    IPushProducer,
+    IReactorTime,
+    ISystemHandle,
+    ITransport,
+)
+from twisted.internet.main import CONNECTION_LOST
+from twisted.internet.protocol import Protocol
+from twisted.protocols.policies import ProtocolWrapper, WrappingFactory
+from twisted.python.failure import Failure
+
+
+@implementer(IPushProducer)
+class _ProducerMembrane:
+    """
+    Stand-in for producer registered with a L{TLSMemoryBIOProtocol} transport.
+
+    Ensures that producer pause/resume events from the undelying transport are
+    coordinated with pause/resume events from the TLS layer.
+
+    @ivar _producer: The application-layer producer.
+    """
+
+    _producerPaused = False
+
+    def __init__(self, producer):
+        self._producer = producer
+
+    def pauseProducing(self):
+        """
+        C{pauseProducing} the underlying producer, if it's not paused.
+        """
+        if self._producerPaused:
+            return
+        self._producerPaused = True
+        self._producer.pauseProducing()
+
+    def resumeProducing(self):
+        """
+        C{resumeProducing} the underlying producer, if it's paused.
+        """
+        if not self._producerPaused:
+            return
+        self._producerPaused = False
+        self._producer.resumeProducing()
+
+    def stopProducing(self):
+        """
+        C{stopProducing} the underlying producer.
+
+        There is only a single source for this event, so it's simply passed
+        on.
+        """
+        self._producer.stopProducing()
+
+
+def _representsEOF(exceptionObject: Error) -> bool:
+    """
+    Does the given OpenSSL.SSL.Error represent an end-of-file?
+    """
+    reasonString: str
+    if isinstance(exceptionObject, SysCallError):
+        _, reasonString = exceptionObject.args
+    else:
+        errorQueue = exceptionObject.args[0]
+        _, _, reasonString = errorQueue[-1]
+    return reasonString.casefold().startswith("unexpected eof")
+
+
+@implementer(ISystemHandle, INegotiated, ITransport)
+class TLSMemoryBIOProtocol(ProtocolWrapper):
+    """
+    L{TLSMemoryBIOProtocol} is a protocol wrapper which uses OpenSSL via a
+    memory BIO to encrypt bytes written to it before sending them on to the
+    underlying transport and decrypts bytes received from the underlying
+    transport before delivering them to the wrapped protocol.
+
+    In addition to producer events from the underlying transport, the need to
+    wait for reads before a write can proceed means the L{TLSMemoryBIOProtocol}
+    may also want to pause a producer.  Pause/resume events are therefore
+    merged using the L{_ProducerMembrane} wrapper.  Non-streaming (pull)
+    producers are supported by wrapping them with L{_PullToPush}.
+
+    Because TLS may need to wait for reads before writing, some writes may be
+    buffered until a read occurs.
+
+    @ivar _tlsConnection: The L{OpenSSL.SSL.Connection} instance which is
+        encrypted and decrypting this connection.
+
+    @ivar _lostTLSConnection: A flag indicating whether connection loss has
+        already been dealt with (C{True}) or not (C{False}).  TLS disconnection
+        is distinct from the underlying connection being lost.
+
+    @ivar _appSendBuffer: application-level (cleartext) data that is waiting to
+        be transferred to the TLS buffer, but can't be because the TLS
+        connection is handshaking.
+    @type _appSendBuffer: L{list} of L{bytes}
+
+    @ivar _connectWrapped: A flag indicating whether or not to call
+        C{makeConnection} on the wrapped protocol.  This is for the reactor's
+        L{twisted.internet.interfaces.ITLSTransport.startTLS} implementation,
+        since it has a protocol which it has already called C{makeConnection}
+        on, and which has no interest in a new transport.  See #3821.
+
+    @ivar _handshakeDone: A flag indicating whether or not the handshake is
+        known to have completed successfully (C{True}) or not (C{False}).  This
+        is used to control error reporting behavior.  If the handshake has not
+        completed, the underlying L{OpenSSL.SSL.Error} will be passed to the
+        application's C{connectionLost} method.  If it has completed, any
+        unexpected L{OpenSSL.SSL.Error} will be turned into a
+        L{ConnectionLost}.  This is weird; however, it is simply an attempt at
+        a faithful re-implementation of the behavior provided by
+        L{twisted.internet.ssl}.
+
+    @ivar _reason: If an unexpected L{OpenSSL.SSL.Error} occurs which causes
+        the connection to be lost, it is saved here.  If appropriate, this may
+        be used as the reason passed to the application protocol's
+        C{connectionLost} method.
+
+    @ivar _producer: The current producer registered via C{registerProducer},
+        or L{None} if no producer has been registered or a previous one was
+        unregistered.
+
+    @ivar _aborted: C{abortConnection} has been called.  No further data will
+        be received to the wrapped protocol's C{dataReceived}.
+    @type _aborted: L{bool}
+    """
+
+    _reason = None
+    _handshakeDone = False
+    _lostTLSConnection = False
+    _producer = None
+    _aborted = False
+
+    def __init__(self, factory, wrappedProtocol, _connectWrapped=True):
+        ProtocolWrapper.__init__(self, factory, wrappedProtocol)
+        self._connectWrapped = _connectWrapped
+
+    def getHandle(self):
+        """
+        Return the L{OpenSSL.SSL.Connection} object being used to encrypt and
+        decrypt this connection.
+
+        This is done for the benefit of L{twisted.internet.ssl.Certificate}'s
+        C{peerFromTransport} and C{hostFromTransport} methods only.  A
+        different system handle may be returned by future versions of this
+        method.
+        """
+        return self._tlsConnection
+
+    def makeConnection(self, transport):
+        """
+        Connect this wrapper to the given transport and initialize the
+        necessary L{OpenSSL.SSL.Connection} with a memory BIO.
+        """
+        self._tlsConnection = self.factory._createConnection(self)
+        self._appSendBuffer = []
+
+        # Add interfaces provided by the transport we are wrapping:
+        for interface in providedBy(transport):
+            directlyProvides(self, interface)
+
+        # Intentionally skip ProtocolWrapper.makeConnection - it might call
+        # wrappedProtocol.makeConnection, which we want to make conditional.
+        Protocol.makeConnection(self, transport)
+        self.factory.registerProtocol(self)
+        if self._connectWrapped:
+            # Now that the TLS layer is initialized, notify the application of
+            # the connection.
+            ProtocolWrapper.makeConnection(self, transport)
+
+        # Now that we ourselves have a transport (initialized by the
+        # ProtocolWrapper.makeConnection call above), kick off the TLS
+        # handshake.
+        self._checkHandshakeStatus()
+
+    def _checkHandshakeStatus(self):
+        """
+        Ask OpenSSL to proceed with a handshake in progress.
+
+        Initially, this just sends the ClientHello; after some bytes have been
+        stuffed in to the C{Connection} object by C{dataReceived}, it will then
+        respond to any C{Certificate} or C{KeyExchange} messages.
+        """
+        # The connection might already be aborted (eg. by a callback during
+        # connection setup), so don't even bother trying to handshake in that
+        # case.
+        if self._aborted:
+            return
+        try:
+            self._tlsConnection.do_handshake()
+        except WantReadError:
+            self._flushSendBIO()
+        except Error:
+            self._tlsShutdownFinished(Failure())
+        else:
+            self._handshakeDone = True
+            if IHandshakeListener.providedBy(self.wrappedProtocol):
+                self.wrappedProtocol.handshakeCompleted()
+
+    def _flushSendBIO(self):
+        """
+        Read any bytes out of the send BIO and write them to the underlying
+        transport.
+        """
+        try:
+            bytes = self._tlsConnection.bio_read(2**15)
+        except WantReadError:
+            # There may be nothing in the send BIO right now.
+            pass
+        else:
+            self.transport.write(bytes)
+
+    def _flushReceiveBIO(self):
+        """
+        Try to receive any application-level bytes which are now available
+        because of a previous write into the receive BIO.  This will take
+        care of delivering any application-level bytes which are received to
+        the protocol, as well as handling of the various exceptions which
+        can come from trying to get such bytes.
+        """
+        # Keep trying this until an error indicates we should stop or we
+        # close the connection.  Looping is necessary to make sure we
+        # process all of the data which was put into the receive BIO, as
+        # there is no guarantee that a single recv call will do it all.
+        while not self._lostTLSConnection:
+            try:
+                bytes = self._tlsConnection.recv(2**15)
+            except WantReadError:
+                # The newly received bytes might not have been enough to produce
+                # any application data.
+                break
+            except ZeroReturnError:
+                # TLS has shut down and no more TLS data will be received over
+                # this connection.
+                self._shutdownTLS()
+                # Passing in None means the user protocol's connnectionLost
+                # will get called with reason from underlying transport:
+                self._tlsShutdownFinished(None)
+            except Error:
+                # Something went pretty wrong.  For example, this might be a
+                # handshake failure during renegotiation (because there were no
+                # shared ciphers, because a certificate failed to verify, etc).
+                # TLS can no longer proceed.
+                failure = Failure()
+                self._tlsShutdownFinished(failure)
+            else:
+                if not self._aborted:
+                    ProtocolWrapper.dataReceived(self, bytes)
+
+        # The received bytes might have generated a response which needs to be
+        # sent now.  For example, the handshake involves several round-trip
+        # exchanges without ever producing application-bytes.
+        self._flushSendBIO()
+
+    def dataReceived(self, bytes):
+        """
+        Deliver any received bytes to the receive BIO and then read and deliver
+        to the application any application-level data which becomes available
+        as a result of this.
+        """
+        # Let OpenSSL know some bytes were just received.
+        self._tlsConnection.bio_write(bytes)
+
+        # If we are still waiting for the handshake to complete, try to
+        # complete the handshake with the bytes we just received.
+        if not self._handshakeDone:
+            self._checkHandshakeStatus()
+
+            # If the handshake still isn't finished, then we've nothing left to
+            # do.
+            if not self._handshakeDone:
+                return
+
+        # If we've any pending writes, this read may have un-blocked them, so
+        # attempt to unbuffer them into the OpenSSL layer.
+        if self._appSendBuffer:
+            self._unbufferPendingWrites()
+
+        # Since the handshake is complete, the wire-level bytes we just
+        # processed might turn into some application-level bytes; try to pull
+        # those out.
+        self._flushReceiveBIO()
+
+    def _shutdownTLS(self):
+        """
+        Initiate, or reply to, the shutdown handshake of the TLS layer.
+        """
+        try:
+            shutdownSuccess = self._tlsConnection.shutdown()
+        except Error:
+            # Mid-handshake, a call to shutdown() can result in a
+            # WantWantReadError, or rather an SSL_ERR_WANT_READ; but pyOpenSSL
+            # doesn't allow us to get at the error.  See:
+            # https://github.com/pyca/pyopenssl/issues/91
+            shutdownSuccess = False
+        self._flushSendBIO()
+        if shutdownSuccess:
+            # Both sides have shutdown, so we can start closing lower-level
+            # transport. This will also happen if we haven't started
+            # negotiation at all yet, in which case shutdown succeeds
+            # immediately.
+            self.transport.loseConnection()
+
+    def _tlsShutdownFinished(self, reason):
+        """
+        Called when TLS connection has gone away; tell underlying transport to
+        disconnect.
+
+        @param reason: a L{Failure} whose value is an L{Exception} if we want to
+            report that failure through to the wrapped protocol's
+            C{connectionLost}, or L{None} if the C{reason} that
+            C{connectionLost} should receive should be coming from the
+            underlying transport.
+        @type reason: L{Failure} or L{None}
+        """
+        if reason is not None:
+            # Squash an EOF in violation of the TLS protocol into
+            # ConnectionLost, so that applications which might run over
+            # multiple protocols can recognize its type.
+            if _representsEOF(reason.value):
+                reason = Failure(CONNECTION_LOST)
+        if self._reason is None:
+            self._reason = reason
+        self._lostTLSConnection = True
+        # We may need to send a TLS alert regarding the nature of the shutdown
+        # here (for example, why a handshake failed), so always flush our send
+        # buffer before telling our lower-level transport to go away.
+        self._flushSendBIO()
+        # Using loseConnection causes the application protocol's
+        # connectionLost method to be invoked non-reentrantly, which is always
+        # a nice feature. However, for error cases (reason != None) we might
+        # want to use abortConnection when it becomes available. The
+        # loseConnection call is basically tested by test_handshakeFailure.
+        # At least one side will need to do it or the test never finishes.
+        self.transport.loseConnection()
+
+    def connectionLost(self, reason):
+        """
+        Handle the possible repetition of calls to this method (due to either
+        the underlying transport going away or due to an error at the TLS
+        layer) and make sure the base implementation only gets invoked once.
+        """
+        if not self._lostTLSConnection:
+            # Tell the TLS connection that it's not going to get any more data
+            # and give it a chance to finish reading.
+            self._tlsConnection.bio_shutdown()
+            self._flushReceiveBIO()
+            self._lostTLSConnection = True
+        reason = self._reason or reason
+        self._reason = None
+        self.connected = False
+        ProtocolWrapper.connectionLost(self, reason)
+
+        # Breaking reference cycle between self._tlsConnection and self.
+        self._tlsConnection = None
+
+    def loseConnection(self):
+        """
+        Send a TLS close alert and close the underlying connection.
+        """
+        if self.disconnecting or not self.connected:
+            return
+        # If connection setup has not finished, OpenSSL 1.0.2f+ will not shut
+        # down the connection until we write some data to the connection which
+        # allows the handshake to complete. However, since no data should be
+        # written after loseConnection, this means we'll be stuck forever
+        # waiting for shutdown to complete. Instead, we simply abort the
+        # connection without trying to shut down cleanly:
+        if not self._handshakeDone and not self._appSendBuffer:
+            self.abortConnection()
+        self.disconnecting = True
+        if not self._appSendBuffer and self._producer is None:
+            self._shutdownTLS()
+
+    def abortConnection(self):
+        """
+        Tear down TLS state so that if the connection is aborted mid-handshake
+        we don't deliver any further data from the application.
+        """
+        self._aborted = True
+        self.disconnecting = True
+        self._shutdownTLS()
+        self.transport.abortConnection()
+
+    def failVerification(self, reason):
+        """
+        Abort the connection during connection setup, giving a reason that
+        certificate verification failed.
+
+        @param reason: The reason that the verification failed; reported to the
+            application protocol's C{connectionLost} method.
+        @type reason: L{Failure}
+        """
+        self._reason = reason
+        self.abortConnection()
+
+    def write(self, bytes):
+        """
+        Process the given application bytes and send any resulting TLS traffic
+        which arrives in the send BIO.
+
+        If C{loseConnection} was called, subsequent calls to C{write} will
+        drop the bytes on the floor.
+        """
+        # Writes after loseConnection are not supported, unless a producer has
+        # been registered, in which case writes can happen until the producer
+        # is unregistered:
+        if self.disconnecting and self._producer is None:
+            return
+        self._write(bytes)
+
+    def _bufferedWrite(self, octets):
+        """
+        Put the given octets into L{TLSMemoryBIOProtocol._appSendBuffer}, and
+        tell any listening producer that it should pause because we are now
+        buffering.
+        """
+        self._appSendBuffer.append(octets)
+        if self._producer is not None:
+            self._producer.pauseProducing()
+
+    def _unbufferPendingWrites(self):
+        """
+        Un-buffer all waiting writes in L{TLSMemoryBIOProtocol._appSendBuffer}.
+        """
+        pendingWrites, self._appSendBuffer = self._appSendBuffer, []
+        for eachWrite in pendingWrites:
+            self._write(eachWrite)
+
+        if self._appSendBuffer:
+            # If OpenSSL ran out of buffer space in the Connection on our way
+            # through the loop earlier and re-buffered any of our outgoing
+            # writes, then we're done; don't consider any future work.
+            return
+
+        if self._producer is not None:
+            # If we have a registered producer, let it know that we have some
+            # more buffer space.
+            self._producer.resumeProducing()
+            return
+
+        if self.disconnecting:
+            # Finally, if we have no further buffered data, no producer wants
+            # to send us more data in the future, and the application told us
+            # to end the stream, initiate a TLS shutdown.
+            self._shutdownTLS()
+
+    def _write(self, bytes):
+        """
+        Process the given application bytes and send any resulting TLS traffic
+        which arrives in the send BIO.
+
+        This may be called by C{dataReceived} with bytes that were buffered
+        before C{loseConnection} was called, which is why this function
+        doesn't check for disconnection but accepts the bytes regardless.
+        """
+        if self._lostTLSConnection:
+            return
+
+        # A TLS payload is 16kB max
+        bufferSize = 2**14
+
+        # How far into the input we've gotten so far
+        alreadySent = 0
+
+        while alreadySent < len(bytes):
+            toSend = bytes[alreadySent : alreadySent + bufferSize]
+            try:
+                sent = self._tlsConnection.send(toSend)
+            except WantReadError:
+                self._bufferedWrite(bytes[alreadySent:])
+                break
+            except Error:
+                # Pretend TLS connection disconnected, which will trigger
+                # disconnect of underlying transport. The error will be passed
+                # to the application protocol's connectionLost method.  The
+                # other SSL implementation doesn't, but losing helpful
+                # debugging information is a bad idea.
+                self._tlsShutdownFinished(Failure())
+                break
+            else:
+                # We've successfully handed off the bytes to the OpenSSL
+                # Connection object.
+                alreadySent += sent
+                # See if OpenSSL wants to hand any bytes off to the underlying
+                # transport as a result.
+                self._flushSendBIO()
+
+    def writeSequence(self, iovec):
+        """
+        Write a sequence of application bytes by joining them into one string
+        and passing them to L{write}.
+        """
+        self.write(b"".join(iovec))
+
+    def getPeerCertificate(self):
+        return self._tlsConnection.get_peer_certificate()
+
+    @property
+    def negotiatedProtocol(self):
+        """
+        @see: L{INegotiated.negotiatedProtocol}
+        """
+        protocolName = None
+
+        try:
+            # If ALPN is not implemented that's ok, NPN might be.
+            protocolName = self._tlsConnection.get_alpn_proto_negotiated()
+        except (NotImplementedError, AttributeError):
+            pass
+
+        if protocolName not in (b"", None):
+            # A protocol was selected using ALPN.
+            return protocolName
+
+        try:
+            protocolName = self._tlsConnection.get_next_proto_negotiated()
+        except (NotImplementedError, AttributeError):
+            pass
+
+        if protocolName != b"":
+            return protocolName
+
+        return None
+
+    def registerProducer(self, producer, streaming):
+        # If we've already disconnected, nothing to do here:
+        if self._lostTLSConnection:
+            producer.stopProducing()
+            return
+
+        # If we received a non-streaming producer, wrap it so it becomes a
+        # streaming producer:
+        if not streaming:
+            producer = streamingProducer = _PullToPush(producer, self)
+        producer = _ProducerMembrane(producer)
+        # This will raise an exception if a producer is already registered:
+        self.transport.registerProducer(producer, True)
+        self._producer = producer
+        # If we received a non-streaming producer, we need to start the
+        # streaming wrapper:
+        if not streaming:
+            streamingProducer.startStreaming()
+
+    def unregisterProducer(self):
+        # If we have no producer, we don't need to do anything here.
+        if self._producer is None:
+            return
+
+        # If we received a non-streaming producer, we need to stop the
+        # streaming wrapper:
+        if isinstance(self._producer._producer, _PullToPush):
+            self._producer._producer.stopStreaming()
+        self._producer = None
+        self._producerPaused = False
+        self.transport.unregisterProducer()
+        if self.disconnecting and not self._appSendBuffer:
+            self._shutdownTLS()
+
+
+@implementer(IOpenSSLClientConnectionCreator, IOpenSSLServerConnectionCreator)
+class _ContextFactoryToConnectionFactory:
+    """
+    Adapter wrapping a L{twisted.internet.interfaces.IOpenSSLContextFactory}
+    into a L{IOpenSSLClientConnectionCreator} or
+    L{IOpenSSLServerConnectionCreator}.
+
+    See U{https://twistedmatrix.com/trac/ticket/7215} for work that should make
+    this unnecessary.
+    """
+
+    def __init__(self, oldStyleContextFactory):
+        """
+        Construct a L{_ContextFactoryToConnectionFactory} with a
+        L{twisted.internet.interfaces.IOpenSSLContextFactory}.
+
+        Immediately call C{getContext} on C{oldStyleContextFactory} in order to
+        force advance parameter checking, since old-style context factories
+        don't actually check that their arguments to L{OpenSSL} are correct.
+
+        @param oldStyleContextFactory: A factory that can produce contexts.
+        @type oldStyleContextFactory:
+            L{twisted.internet.interfaces.IOpenSSLContextFactory}
+        """
+        oldStyleContextFactory.getContext()
+        self._oldStyleContextFactory = oldStyleContextFactory
+
+    def _connectionForTLS(self, protocol):
+        """
+        Create an L{OpenSSL.SSL.Connection} object.
+
+        @param protocol: The protocol initiating a TLS connection.
+        @type protocol: L{TLSMemoryBIOProtocol}
+
+        @return: a connection
+        @rtype: L{OpenSSL.SSL.Connection}
+        """
+        context = self._oldStyleContextFactory.getContext()
+        return Connection(context, None)
+
+    def serverConnectionForTLS(self, protocol):
+        """
+        Construct an OpenSSL server connection from the wrapped old-style
+        context factory.
+
+        @note: Since old-style context factories don't distinguish between
+            clients and servers, this is exactly the same as
+            L{_ContextFactoryToConnectionFactory.clientConnectionForTLS}.
+
+        @param protocol: The protocol initiating a TLS connection.
+        @type protocol: L{TLSMemoryBIOProtocol}
+
+        @return: a connection
+        @rtype: L{OpenSSL.SSL.Connection}
+        """
+        return self._connectionForTLS(protocol)
+
+    def clientConnectionForTLS(self, protocol):
+        """
+        Construct an OpenSSL server connection from the wrapped old-style
+        context factory.
+
+        @note: Since old-style context factories don't distinguish between
+            clients and servers, this is exactly the same as
+            L{_ContextFactoryToConnectionFactory.serverConnectionForTLS}.
+
+        @param protocol: The protocol initiating a TLS connection.
+        @type protocol: L{TLSMemoryBIOProtocol}
+
+        @return: a connection
+        @rtype: L{OpenSSL.SSL.Connection}
+        """
+        return self._connectionForTLS(protocol)
+
+
+class _AggregateSmallWrites:
+    """
+    Aggregate small writes so they get written in large batches.
+
+    If this is used as part of a transport, the transport needs to call
+    ``flush()`` immediately when ``loseConnection()`` is called, otherwise any
+    buffered writes will never get written.
+
+    @cvar MAX_BUFFER_SIZE: The maximum amount of bytes to buffer before writing
+        them out.
+    """
+
+    MAX_BUFFER_SIZE = 64_000
+
+    def __init__(self, write: Callable[[bytes], object], clock: IReactorTime):
+        self._write = write
+        self._clock = clock
+        self._buffer: list[bytes] = []
+        self._bufferLen = 0
+        self._scheduled: Optional[IDelayedCall] = None
+
+    def write(self, data: bytes) -> None:
+        """
+        Buffer the data, or write it immediately if we've accumulated enough to
+        make it worth it.
+
+        Accumulating too much data can result in higher memory usage.
+        """
+        self._buffer.append(data)
+        self._bufferLen += len(data)
+
+        if self._bufferLen > self.MAX_BUFFER_SIZE:
+            # We've accumulated enough we should just write it out. No need to
+            # schedule a flush, since we just flushed everything.
+            self.flush()
+            return
+
+        if self._scheduled:
+            # We already have a scheduled send, so with the data in the buffer,
+            # there is nothing more to do here.
+            return
+
+        # Schedule the write of the accumulated buffer for the next reactor
+        # iteration.
+        self._scheduled = self._clock.callLater(0, self._scheduledFlush)
+
+    def _scheduledFlush(self) -> None:
+        """Called in next reactor iteration."""
+        self._scheduled = None
+        self.flush()
+
+    def flush(self) -> None:
+        """Flush any buffered writes."""
+        if self._buffer:
+            self._bufferLen = 0
+            self._write(b"".join(self._buffer))
+            del self._buffer[:]
+
+
+def _get_default_clock() -> IReactorTime:
+    """
+    Return the default reactor.
+
+    This is a function so it can be monkey-patched in tests, specifically
+    L{twisted.web.test.test_agent}.
+    """
+    from twisted.internet import reactor
+
+    return cast(IReactorTime, reactor)
+
+
+class BufferingTLSTransport(TLSMemoryBIOProtocol):
+    """
+    A TLS transport implemented by wrapping buffering around a
+    ``TLSMemoryBIOProtocol``.
+
+    Doing many small writes directly to a ``OpenSSL.SSL.Connection``, as
+    implemented in ``TLSMemoryBIOProtocol``, can add significant CPU and
+    bandwidth overhead.  Thus, even when writing is possible, small writes will
+    get aggregated and written as a single write at the next reactor iteration.
+    """
+
+    # Note: An implementation based on composition would be nicer, but there's
+    # close integration between ``ProtocolWrapper`` subclasses like
+    # ``TLSMemoryBIOProtocol`` and the corresponding factory. Composition broke
+    # things like ``TLSMemoryBIOFactory.protocols`` having the correct
+    # instances, whereas subclassing makes that work.
+
+    def __init__(
+        self,
+        factory: TLSMemoryBIOFactory,
+        wrappedProtocol: IProtocol,
+        _connectWrapped: bool = True,
+    ):
+        super().__init__(factory, wrappedProtocol, _connectWrapped)
+        actual_write = super().write
+        self._aggregator = _AggregateSmallWrites(actual_write, factory._clock)
+
+    def write(self, data: bytes) -> None:
+        if isinstance(data, str):  # type: ignore[unreachable]
+            raise TypeError("Must write bytes to a TLS transport, not str.")
+        self._aggregator.write(data)
+
+    def writeSequence(self, sequence: Iterable[bytes]) -> None:
+        self._aggregator.write(b"".join(sequence))
+
+    def loseConnection(self) -> None:
+        self._aggregator.flush()
+        super().loseConnection()
+
+
+class TLSMemoryBIOFactory(WrappingFactory):
+    """
+    L{TLSMemoryBIOFactory} adds TLS to connections.
+
+    @ivar _creatorInterface: the interface which L{_connectionCreator} is
+        expected to implement.
+    @type _creatorInterface: L{zope.interface.interfaces.IInterface}
+
+    @ivar _connectionCreator: a callable which creates an OpenSSL Connection
+        object.
+    @type _connectionCreator: 1-argument callable taking
+        L{TLSMemoryBIOProtocol} and returning L{OpenSSL.SSL.Connection}.
+    """
+
+    protocol = BufferingTLSTransport
+
+    noisy = False  # disable unnecessary logging.
+
+    def __init__(
+        self,
+        contextFactory,
+        isClient,
+        wrappedFactory,
+        clock=None,
+    ):
+        """
+        Create a L{TLSMemoryBIOFactory}.
+
+        @param contextFactory: Configuration parameters used to create an
+            OpenSSL connection.  In order of preference, what you should pass
+            here should be:
+
+                1. L{twisted.internet.ssl.CertificateOptions} (if you're
+                   writing a server) or the result of
+                   L{twisted.internet.ssl.optionsForClientTLS} (if you're
+                   writing a client).  If you want security you should really
+                   use one of these.
+
+                2. If you really want to implement something yourself, supply a
+                   provider of L{IOpenSSLClientConnectionCreator} or
+                   L{IOpenSSLServerConnectionCreator}.
+
+                3. If you really have to, supply a
+                   L{twisted.internet.ssl.ContextFactory}.  This will likely be
+                   deprecated at some point so please upgrade to the new
+                   interfaces.
+
+        @type contextFactory: L{IOpenSSLClientConnectionCreator} or
+            L{IOpenSSLServerConnectionCreator}, or, for compatibility with
+            older code, anything implementing
+            L{twisted.internet.interfaces.IOpenSSLContextFactory}.  See
+            U{https://twistedmatrix.com/trac/ticket/7215} for information on
+            the upcoming deprecation of passing a
+            L{twisted.internet.ssl.ContextFactory} here.
+
+        @param isClient: Is this a factory for TLS client connections; in other
+            words, those that will send a C{ClientHello} greeting?  L{True} if
+            so, L{False} otherwise.  This flag determines what interface is
+            expected of C{contextFactory}.  If L{True}, C{contextFactory}
+            should provide L{IOpenSSLClientConnectionCreator}; otherwise it
+            should provide L{IOpenSSLServerConnectionCreator}.
+        @type isClient: L{bool}
+
+        @param wrappedFactory: A factory which will create the
+            application-level protocol.
+        @type wrappedFactory: L{twisted.internet.interfaces.IProtocolFactory}
+        """
+        WrappingFactory.__init__(self, wrappedFactory)
+        if isClient:
+            creatorInterface = IOpenSSLClientConnectionCreator
+        else:
+            creatorInterface = IOpenSSLServerConnectionCreator
+        self._creatorInterface = creatorInterface
+        if not creatorInterface.providedBy(contextFactory):
+            contextFactory = _ContextFactoryToConnectionFactory(contextFactory)
+        self._connectionCreator = contextFactory
+
+        if clock is None:
+            clock = _get_default_clock()
+        self._clock = clock
+
+    def logPrefix(self):
+        """
+        Annotate the wrapped factory's log prefix with some text indicating TLS
+        is in use.
+
+        @rtype: C{str}
+        """
+        if ILoggingContext.providedBy(self.wrappedFactory):
+            logPrefix = self.wrappedFactory.logPrefix()
+        else:
+            logPrefix = self.wrappedFactory.__class__.__name__
+        return f"{logPrefix} (TLS)"
+
+    def _applyProtocolNegotiation(self, connection):
+        """
+        Applies ALPN/NPN protocol neogitation to the connection, if the factory
+        supports it.
+
+        @param connection: The OpenSSL connection object to have ALPN/NPN added
+            to it.
+        @type connection: L{OpenSSL.SSL.Connection}
+
+        @return: Nothing
+        @rtype: L{None}
+        """
+        if IProtocolNegotiationFactory.providedBy(self.wrappedFactory):
+            protocols = self.wrappedFactory.acceptableProtocols()
+            context = connection.get_context()
+            _setAcceptableProtocols(context, protocols)
+
+        return
+
+    def _createConnection(self, tlsProtocol):
+        """
+        Create an OpenSSL connection and set it up good.
+
+        @param tlsProtocol: The protocol which is establishing the connection.
+        @type tlsProtocol: L{TLSMemoryBIOProtocol}
+
+        @return: an OpenSSL connection object for C{tlsProtocol} to use
+        @rtype: L{OpenSSL.SSL.Connection}
+        """
+        connectionCreator = self._connectionCreator
+        if self._creatorInterface is IOpenSSLClientConnectionCreator:
+            connection = connectionCreator.clientConnectionForTLS(tlsProtocol)
+            self._applyProtocolNegotiation(connection)
+            connection.set_connect_state()
+        else:
+            connection = connectionCreator.serverConnectionForTLS(tlsProtocol)
+            self._applyProtocolNegotiation(connection)
+            connection.set_accept_state()
+        return connection
diff --git a/contrib/python/Twisted/py3/twisted/protocols/wire.py b/contrib/python/Twisted/py3/twisted/protocols/wire.py
new file mode 100644
index 00000000000..5ca3703effb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/protocols/wire.py
@@ -0,0 +1,112 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""Implement standard (and unused) TCP protocols.
+
+These protocols are either provided by inetd, or are not provided at all.
+"""
+
+
+import struct
+import time
+
+from zope.interface import implementer
+
+from twisted.internet import interfaces, protocol
+
+
+class Echo(protocol.Protocol):
+    """
+    As soon as any data is received, write it back (RFC 862).
+    """
+
+    def dataReceived(self, data):
+        self.transport.write(data)
+
+
+class Discard(protocol.Protocol):
+    """
+    Discard any received data (RFC 863).
+    """
+
+    def dataReceived(self, data):
+        # I'm ignoring you, nyah-nyah
+        pass
+
+
+@implementer(interfaces.IProducer)
+class Chargen(protocol.Protocol):
+    """
+    Generate repeating noise (RFC 864).
+    """
+
+    noise = rb'@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ !"#$%&?'
+
+    def connectionMade(self):
+        self.transport.registerProducer(self, 0)
+
+    def resumeProducing(self):
+        self.transport.write(self.noise)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass
+
+
+class QOTD(protocol.Protocol):
+    """
+    Return a quote of the day (RFC 865).
+    """
+
+    def connectionMade(self):
+        self.transport.write(self.getQuote())
+        self.transport.loseConnection()
+
+    def getQuote(self):
+        """
+        Return a quote. May be overrriden in subclasses.
+        """
+        return b"An apple a day keeps the doctor away.\r\n"
+
+
+class Who(protocol.Protocol):
+    """
+    Return list of active users (RFC 866)
+    """
+
+    def connectionMade(self):
+        self.transport.write(self.getUsers())
+        self.transport.loseConnection()
+
+    def getUsers(self):
+        """
+        Return active users. Override in subclasses.
+        """
+        return b"root\r\n"
+
+
+class Daytime(protocol.Protocol):
+    """
+    Send back the daytime in ASCII form (RFC 867).
+    """
+
+    def connectionMade(self):
+        self.transport.write(time.asctime(time.gmtime(time.time())) + b"\r\n")
+        self.transport.loseConnection()
+
+
+class Time(protocol.Protocol):
+    """
+    Send back the time in machine readable form (RFC 868).
+    """
+
+    def connectionMade(self):
+        # is this correct only for 32-bit machines?
+        result = struct.pack("!i", int(time.time()))
+        self.transport.write(result)
+        self.transport.loseConnection()
+
+
+__all__ = ["Echo", "Discard", "Chargen", "QOTD", "Who", "Daytime", "Time"]
diff --git a/contrib/python/Twisted/py3/twisted/py.typed b/contrib/python/Twisted/py3/twisted/py.typed
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/Twisted/py3/twisted/python/__init__.py b/contrib/python/Twisted/py3/twisted/python/__init__.py
new file mode 100644
index 00000000000..3dc5524f01f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/__init__.py
@@ -0,0 +1,31 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Python: Utilities and Enhancements for Python.
+"""
+
+
+from .deprecate import deprecatedModuleAttribute
+
+# Deprecating twisted.python.constants.
+from .versions import Version
+
+deprecatedModuleAttribute(
+    Version("Twisted", 16, 5, 0),
+    "Please use constantly from PyPI instead.",
+    "twisted.python",
+    "constants",
+)
+
+
+deprecatedModuleAttribute(
+    Version("Twisted", 17, 5, 0),
+    "Please use hyperlink from PyPI instead.",
+    "twisted.python",
+    "url",
+)
+
+
+del Version
+del deprecatedModuleAttribute
diff --git a/contrib/python/Twisted/py3/twisted/python/_appdirs.py b/contrib/python/Twisted/py3/twisted/python/_appdirs.py
new file mode 100644
index 00000000000..8ceb1116762
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/_appdirs.py
@@ -0,0 +1,32 @@
+# -*- test-case-name: twisted.python.test.test_appdirs -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Application data directory support.
+"""
+
+import inspect
+from typing import cast
+
+import appdirs  # type: ignore[import]
+
+from twisted.python.compat import currentframe
+
+
+def getDataDirectory(moduleName: str = "") -> str:
+    """
+    Get a data directory for the caller function, or C{moduleName} if given.
+
+    @param moduleName: The module name if you don't wish to have the caller's
+        module.
+
+    @returns: A directory for putting data in.
+    """
+    if not moduleName:
+        caller = currentframe(1)
+        module = inspect.getmodule(caller)
+        assert module is not None
+        moduleName = module.__name__
+
+    return cast(str, appdirs.user_data_dir(moduleName))
diff --git a/contrib/python/Twisted/py3/twisted/python/_inotify.py b/contrib/python/Twisted/py3/twisted/python/_inotify.py
new file mode 100644
index 00000000000..1f67a4ae00a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/_inotify.py
@@ -0,0 +1,100 @@
+# -*- test-case-name: twisted.internet.test.test_inotify -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Very low-level ctypes-based interface to Linux inotify(7).
+
+ctypes and a version of libc which supports inotify system calls are
+required.
+"""
+
+import ctypes
+import ctypes.util
+from typing import Any, cast
+
+from twisted.python.filepath import FilePath
+
+
+class INotifyError(Exception):
+    """
+    Unify all the possible exceptions that can be raised by the INotify API.
+    """
+
+
+def init() -> int:
+    """
+    Create an inotify instance and return the associated file descriptor.
+    """
+    fd = cast(int, libc.inotify_init())
+    if fd < 0:
+        raise INotifyError("INotify initialization error.")
+    return fd
+
+
+def add(fd: int, path: FilePath[Any], mask: int) -> int:
+    """
+    Add a watch for the given path to the inotify file descriptor, and return
+    the watch descriptor.
+
+    @param fd: The file descriptor returned by C{libc.inotify_init}.
+    @param path: The path to watch via inotify.
+    @param mask: Bitmask specifying the events that inotify should monitor.
+    """
+    wd = cast(int, libc.inotify_add_watch(fd, path.asBytesMode().path, mask))
+    if wd < 0:
+        raise INotifyError(f"Failed to add watch on '{path!r}' - ({wd!r})")
+    return wd
+
+
+def remove(fd: int, wd: int) -> None:
+    """
+    Remove the given watch descriptor from the inotify file descriptor.
+    """
+    # When inotify_rm_watch returns -1 there's an error:
+    # The errno for this call can be either one of the following:
+    #  EBADF: fd is not a valid file descriptor.
+    #  EINVAL: The watch descriptor wd is not valid; or fd is
+    #          not an inotify file descriptor.
+    #
+    # if we can't access the errno here we cannot even raise
+    # an exception and we need to ignore the problem, one of
+    # the most common cases is when you remove a directory from
+    # the filesystem and that directory is observed. When inotify
+    # tries to call inotify_rm_watch with a non existing directory
+    # either of the 2 errors might come up because the files inside
+    # it might have events generated way before they were handled.
+    # Unfortunately only ctypes in Python 2.6 supports accessing errno:
+    #  http://bugs.python.org/issue1798 and in order to solve
+    # the problem for previous versions we need to introduce
+    # code that is quite complex:
+    #  http://stackoverflow.com/questions/661017/access-to-errno-from-python
+    #
+    # See #4310 for future resolution of this issue.
+    libc.inotify_rm_watch(fd, wd)
+
+
+def initializeModule(libc: ctypes.CDLL) -> None:
+    """
+    Initialize the module, checking if the expected APIs exist and setting the
+    argtypes and restype for C{inotify_init}, C{inotify_add_watch}, and
+    C{inotify_rm_watch}.
+    """
+    for function in ("inotify_add_watch", "inotify_init", "inotify_rm_watch"):
+        if getattr(libc, function, None) is None:
+            raise ImportError("libc6 2.4 or higher needed")
+    libc.inotify_init.argtypes = []
+    libc.inotify_init.restype = ctypes.c_int
+
+    libc.inotify_rm_watch.argtypes = [ctypes.c_int, ctypes.c_int]
+    libc.inotify_rm_watch.restype = ctypes.c_int
+
+    libc.inotify_add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32]
+    libc.inotify_add_watch.restype = ctypes.c_int
+
+
+name = ctypes.util.find_library("c")
+if not name:
+    raise ImportError("Can't find C library.")
+libc = ctypes.cdll.LoadLibrary(name)
+initializeModule(libc)
diff --git a/contrib/python/Twisted/py3/twisted/python/_pydoctortemplates/subheader.html b/contrib/python/Twisted/py3/twisted/python/_pydoctortemplates/subheader.html
new file mode 100644
index 00000000000..7db8ac44af3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/_pydoctortemplates/subheader.html
@@ -0,0 +1,29 @@
+<div style="display: none" id="current-docs-container" class="container">
+  <div class="col-sm-12">
+    <a id="current-docs-link">
+      Go to the latest version of this document.
+    </a>
+  </div>
+
+  <!-- Google analytics, obviously. -->
+  <script src="//www.google-analytics.com/urchin.js" type="text/javascript"></script>
+  <script type="text/javascript">
+    _uacct = "UA-99018-6";
+    urchinTracker();
+  </script>
+
+  <!-- If the documentation isn't current, insert a current link. -->
+  <script type="text/javascript">
+    if (window.location.pathname.indexOf('/current/') == -1) {
+      <!-- Give the user a link to this page, but in the current version of the docs. -->
+      var link = document.getElementById('current-docs-link');
+      link.href = window.location.pathname.replace(/\/\d+\.\d+\.\d+\/api\//, '/current/api/');
+      <!-- And make it visible -->
+      var container = document.getElementById('current-docs-container');
+      container.style.display = "";
+      delete link;
+      delete container;
+    }
+  </script>
+  
+</div>
diff --git a/contrib/python/Twisted/py3/twisted/python/_release.py b/contrib/python/Twisted/py3/twisted/python/_release.py
new file mode 100644
index 00000000000..35220c31953
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/_release.py
@@ -0,0 +1,281 @@
+# -*- test-case-name: twisted.python.test.test_release -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted's automated release system.
+
+This module is only for use within Twisted's release system. If you are anyone
+else, do not use it. The interface and behaviour will change without notice.
+
+Only Linux is supported by this code.  It should not be used by any tools
+which must run on multiple platforms (eg the setup.py script).
+"""
+
+import os
+from subprocess import STDOUT, CalledProcessError, check_output
+from typing import Dict
+
+from zope.interface import Interface, implementer
+
+from twisted.python.compat import execfile
+
+
+def runCommand(args, **kwargs):
+    """Execute a vector of arguments.
+
+    This is a wrapper around L{subprocess.check_output}, so it takes
+    the same arguments as L{subprocess.Popen} with one difference: all
+    arguments after the vector must be keyword arguments.
+
+    @param args: arguments passed to L{subprocess.check_output}
+    @param kwargs: keyword arguments passed to L{subprocess.check_output}
+    @return: command output
+    @rtype: L{bytes}
+    """
+    kwargs["stderr"] = STDOUT
+    return check_output(args, **kwargs)
+
+
+class IVCSCommand(Interface):
+    """
+    An interface for VCS commands.
+    """
+
+    def ensureIsWorkingDirectory(path):
+        """
+        Ensure that C{path} is a working directory of this VCS.
+
+        @type path: L{twisted.python.filepath.FilePath}
+        @param path: The path to check.
+        """
+
+    def isStatusClean(path):
+        """
+        Return the Git status of the files in the specified path.
+
+        @type path: L{twisted.python.filepath.FilePath}
+        @param path: The path to get the status from (can be a directory or a
+            file.)
+        """
+
+    def remove(path):
+        """
+        Remove the specified path from a the VCS.
+
+        @type path: L{twisted.python.filepath.FilePath}
+        @param path: The path to remove from the repository.
+        """
+
+    def exportTo(fromDir, exportDir):
+        """
+        Export the content of the VCSrepository to the specified directory.
+
+        @type fromDir: L{twisted.python.filepath.FilePath}
+        @param fromDir: The path to the VCS repository to export.
+
+        @type exportDir: L{twisted.python.filepath.FilePath}
+        @param exportDir: The directory to export the content of the
+            repository to. This directory doesn't have to exist prior to
+            exporting the repository.
+        """
+
+
+@implementer(IVCSCommand)
+class GitCommand:
+    """
+    Subset of Git commands to release Twisted from a Git repository.
+    """
+
+    @staticmethod
+    def ensureIsWorkingDirectory(path):
+        """
+        Ensure that C{path} is a Git working directory.
+
+        @type path: L{twisted.python.filepath.FilePath}
+        @param path: The path to check.
+        """
+        try:
+            runCommand(["git", "rev-parse"], cwd=path.path)
+        except (CalledProcessError, OSError):
+            raise NotWorkingDirectory(
+                f"{path.path} does not appear to be a Git repository."
+            )
+
+    @staticmethod
+    def isStatusClean(path):
+        """
+        Return the Git status of the files in the specified path.
+
+        @type path: L{twisted.python.filepath.FilePath}
+        @param path: The path to get the status from (can be a directory or a
+            file.)
+        """
+        status = runCommand(["git", "-C", path.path, "status", "--short"]).strip()
+        return status == b""
+
+    @staticmethod
+    def remove(path):
+        """
+        Remove the specified path from a Git repository.
+
+        @type path: L{twisted.python.filepath.FilePath}
+        @param path: The path to remove from the repository.
+        """
+        runCommand(["git", "-C", path.dirname(), "rm", path.path])
+
+    @staticmethod
+    def exportTo(fromDir, exportDir):
+        """
+        Export the content of a Git repository to the specified directory.
+
+        @type fromDir: L{twisted.python.filepath.FilePath}
+        @param fromDir: The path to the Git repository to export.
+
+        @type exportDir: L{twisted.python.filepath.FilePath}
+        @param exportDir: The directory to export the content of the
+            repository to. This directory doesn't have to exist prior to
+            exporting the repository.
+        """
+        runCommand(
+            [
+                "git",
+                "-C",
+                fromDir.path,
+                "checkout-index",
+                "--all",
+                "--force",
+                # prefix has to end up with a "/" so that files get copied
+                # to a directory whose name is the prefix.
+                "--prefix",
+                exportDir.path + "/",
+            ]
+        )
+
+
+def getRepositoryCommand(directory):
+    """
+    Detect the VCS used in the specified directory and return a L{GitCommand}
+    if the directory is a Git repository. If the directory is not git, it
+    raises a L{NotWorkingDirectory} exception.
+
+    @type directory: L{FilePath}
+    @param directory: The directory to detect the VCS used from.
+
+    @rtype: L{GitCommand}
+
+    @raise NotWorkingDirectory: if no supported VCS can be found from the
+        specified directory.
+    """
+    try:
+        GitCommand.ensureIsWorkingDirectory(directory)
+        return GitCommand
+    except (NotWorkingDirectory, OSError):
+        # It's not Git, but that's okay, eat the error
+        pass
+
+    raise NotWorkingDirectory(f"No supported VCS can be found in {directory.path}")
+
+
+class Project:
+    """
+    A representation of a project that has a version.
+
+    @ivar directory: A L{twisted.python.filepath.FilePath} pointing to the base
+        directory of a Twisted-style Python package. The package should contain
+        a C{_version.py} file and a C{newsfragments} directory that contains a
+        C{README} file.
+    """
+
+    def __init__(self, directory):
+        self.directory = directory
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.directory!r})"
+
+    def getVersion(self):
+        """
+        @return: A L{incremental.Version} specifying the version number of the
+            project based on live python modules.
+        """
+        namespace: Dict[str, object] = {}
+        directory = self.directory
+        while not namespace:
+            if directory.path == "/":
+                raise Exception("Not inside a Twisted project.")
+            elif not directory.basename() == "twisted":
+                directory = directory.parent()
+            else:
+                execfile(directory.child("_version.py").path, namespace)
+        return namespace["__version__"]
+
+
+def findTwistedProjects(baseDirectory):
+    """
+    Find all Twisted-style projects beneath a base directory.
+
+    @param baseDirectory: A L{twisted.python.filepath.FilePath} to look inside.
+    @return: A list of L{Project}.
+    """
+    projects = []
+    for filePath in baseDirectory.walk():
+        if filePath.basename() == "newsfragments":
+            projectDirectory = filePath.parent()
+            projects.append(Project(projectDirectory))
+    return projects
+
+
+def replaceInFile(filename, oldToNew):
+    """
+    I replace the text `oldstr' with `newstr' in `filename' using science.
+    """
+    os.rename(filename, filename + ".bak")
+    with open(filename + ".bak") as f:
+        d = f.read()
+    for k, v in oldToNew.items():
+        d = d.replace(k, v)
+    with open(filename + ".new", "w") as f:
+        f.write(d)
+    os.rename(filename + ".new", filename)
+    os.unlink(filename + ".bak")
+
+
+class NoDocumentsFound(Exception):
+    """
+    Raised when no input documents are found.
+    """
+
+
+def filePathDelta(origin, destination):
+    """
+    Return a list of strings that represent C{destination} as a path relative
+    to C{origin}.
+
+    It is assumed that both paths represent directories, not files. That is to
+    say, the delta of L{twisted.python.filepath.FilePath} /foo/bar to
+    L{twisted.python.filepath.FilePath} /foo/baz will be C{../baz},
+    not C{baz}.
+
+    @type origin: L{twisted.python.filepath.FilePath}
+    @param origin: The origin of the relative path.
+
+    @type destination: L{twisted.python.filepath.FilePath}
+    @param destination: The destination of the relative path.
+    """
+    commonItems = 0
+    path1 = origin.path.split(os.sep)
+    path2 = destination.path.split(os.sep)
+    for elem1, elem2 in zip(path1, path2):
+        if elem1 == elem2:
+            commonItems += 1
+        else:
+            break
+    path = [".."] * (len(path1) - commonItems)
+    return path + path2[commonItems:]
+
+
+class NotWorkingDirectory(Exception):
+    """
+    Raised when a directory does not appear to be a repository directory of a
+    supported VCS.
+    """
diff --git a/contrib/python/Twisted/py3/twisted/python/_shellcomp.py b/contrib/python/Twisted/py3/twisted/python/_shellcomp.py
new file mode 100644
index 00000000000..e36620210b2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/_shellcomp.py
@@ -0,0 +1,684 @@
+# -*- test-case-name: twisted.python.test.test_shellcomp -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+No public APIs are provided by this module. Internal use only.
+
+This module implements dynamic tab-completion for any command that uses
+twisted.python.usage. Currently, only zsh is supported. Bash support may
+be added in the future.
+
+Maintainer: Eric P. Mangold - twisted AT teratorn DOT org
+
+In order for zsh completion to take place the shell must be able to find an
+appropriate "stub" file ("completion function") that invokes this code and
+displays the results to the user.
+
+The stub used for Twisted commands is in the file C{twisted-completion.zsh},
+which is also included in the official Zsh distribution at
+C{Completion/Unix/Command/_twisted}. Use this file as a basis for completion
+functions for your own commands. You should only need to change the first line
+to something like C{#compdef mycommand}.
+
+The main public documentation exists in the L{twisted.python.usage.Options}
+docstring, the L{twisted.python.usage.Completions} docstring, and the
+Options howto.
+"""
+
+import getopt
+import inspect
+import itertools
+from types import MethodType
+from typing import Dict, List, Set
+
+from twisted.python import reflect, usage, util
+from twisted.python.compat import ioType
+
+
+def shellComplete(config, cmdName, words, shellCompFile):
+    """
+    Perform shell completion.
+
+    A completion function (shell script) is generated for the requested
+    shell and written to C{shellCompFile}, typically C{stdout}. The result
+    is then eval'd by the shell to produce the desired completions.
+
+    @type config: L{twisted.python.usage.Options}
+    @param config: The L{twisted.python.usage.Options} instance to generate
+        completions for.
+
+    @type cmdName: C{str}
+    @param cmdName: The name of the command we're generating completions for.
+        In the case of zsh, this is used to print an appropriate
+        "#compdef $CMD" line at the top of the output. This is
+        not necessary for the functionality of the system, but it
+        helps in debugging, since the output we produce is properly
+        formed and may be saved in a file and used as a stand-alone
+        completion function.
+
+    @type words: C{list} of C{str}
+    @param words: The raw command-line words passed to use by the shell
+        stub function. argv[0] has already been stripped off.
+
+    @type shellCompFile: C{file}
+    @param shellCompFile: The file to write completion data to.
+    """
+
+    # If given a file with unicode semantics, such as sys.stdout on Python 3,
+    # we must get at the the underlying buffer which has bytes semantics.
+    if shellCompFile and ioType(shellCompFile) == str:
+        shellCompFile = shellCompFile.buffer
+
+    # shellName is provided for forward-compatibility. It is not used,
+    # since we currently only support zsh.
+    shellName, position = words[-1].split(":")
+    position = int(position)
+    # zsh gives the completion position ($CURRENT) as a 1-based index,
+    # and argv[0] has already been stripped off, so we subtract 2 to
+    # get the real 0-based index.
+    position -= 2
+    cWord = words[position]
+
+    # since the user may hit TAB at any time, we may have been called with an
+    # incomplete command-line that would generate getopt errors if parsed
+    # verbatim. However, we must do *some* parsing in order to determine if
+    # there is a specific subcommand that we need to provide completion for.
+    # So, to make the command-line more sane we work backwards from the
+    # current completion position and strip off all words until we find one
+    # that "looks" like a subcommand. It may in fact be the argument to a
+    # normal command-line option, but that won't matter for our purposes.
+    while position >= 1:
+        if words[position - 1].startswith("-"):
+            position -= 1
+        else:
+            break
+    words = words[:position]
+
+    subCommands = getattr(config, "subCommands", None)
+    if subCommands:
+        # OK, this command supports sub-commands, so lets see if we have been
+        # given one.
+
+        # If the command-line arguments are not valid then we won't be able to
+        # sanely detect the sub-command, so just generate completions as if no
+        # sub-command was found.
+        args = None
+        try:
+            opts, args = getopt.getopt(words, config.shortOpt, config.longOpt)
+        except getopt.error:
+            pass
+
+        if args:
+            # yes, we have a subcommand. Try to find it.
+            for cmd, short, parser, doc in config.subCommands:
+                if args[0] == cmd or args[0] == short:
+                    subOptions = parser()
+                    subOptions.parent = config
+
+                    gen: ZshBuilder = ZshSubcommandBuilder(
+                        subOptions, config, cmdName, shellCompFile
+                    )
+                    gen.write()
+                    return
+
+        # sub-command not given, or did not match any knowns sub-command names
+        genSubs = True
+        if cWord.startswith("-"):
+            # optimization: if the current word being completed starts
+            # with a hyphen then it can't be a sub-command, so skip
+            # the expensive generation of the sub-command list
+            genSubs = False
+        gen = ZshBuilder(config, cmdName, shellCompFile)
+        gen.write(genSubs=genSubs)
+    else:
+        gen = ZshBuilder(config, cmdName, shellCompFile)
+        gen.write()
+
+
+class SubcommandAction(usage.Completer):
+    def _shellCode(self, optName, shellType):
+        if shellType == usage._ZSH:
+            return "*::subcmd:->subcmd"
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class ZshBuilder:
+    """
+    Constructs zsh code that will complete options for a given usage.Options
+    instance, possibly including a list of subcommand names.
+
+    Completions for options to subcommands won't be generated because this
+    class will never be used if the user is completing options for a specific
+    subcommand. (See L{ZshSubcommandBuilder} below)
+
+    @type options: L{twisted.python.usage.Options}
+    @ivar options: The L{twisted.python.usage.Options} instance defined for this
+        command.
+
+    @type cmdName: C{str}
+    @ivar cmdName: The name of the command we're generating completions for.
+
+    @type file: C{file}
+    @ivar file: The C{file} to write the completion function to.  The C{file}
+        must have L{bytes} I/O semantics.
+    """
+
+    def __init__(self, options, cmdName, file):
+        self.options = options
+        self.cmdName = cmdName
+        self.file = file
+
+    def write(self, genSubs=True):
+        """
+        Generate the completion function and write it to the output file
+        @return: L{None}
+
+        @type genSubs: C{bool}
+        @param genSubs: Flag indicating whether or not completions for the list
+            of subcommand should be generated. Only has an effect
+            if the C{subCommands} attribute has been defined on the
+            L{twisted.python.usage.Options} instance.
+        """
+        if genSubs and getattr(self.options, "subCommands", None) is not None:
+            gen = ZshArgumentsGenerator(self.options, self.cmdName, self.file)
+            gen.extraActions.insert(0, SubcommandAction())
+            gen.write()
+            self.file.write(b"local _zsh_subcmds_array\n_zsh_subcmds_array=(\n")
+            for cmd, short, parser, desc in self.options.subCommands:
+                self.file.write(
+                    b'"' + cmd.encode("utf-8") + b":" + desc.encode("utf-8") + b'"\n'
+                )
+            self.file.write(b")\n\n")
+            self.file.write(b'_describe "sub-command" _zsh_subcmds_array\n')
+        else:
+            gen = ZshArgumentsGenerator(self.options, self.cmdName, self.file)
+            gen.write()
+
+
+class ZshSubcommandBuilder(ZshBuilder):
+    """
+    Constructs zsh code that will complete options for a given usage.Options
+    instance, and also for a single sub-command. This will only be used in
+    the case where the user is completing options for a specific subcommand.
+
+    @type subOptions: L{twisted.python.usage.Options}
+    @ivar subOptions: The L{twisted.python.usage.Options} instance defined for
+        the sub command.
+    """
+
+    def __init__(self, subOptions, *args):
+        self.subOptions = subOptions
+        ZshBuilder.__init__(self, *args)
+
+    def write(self):
+        """
+        Generate the completion function and write it to the output file
+        @return: L{None}
+        """
+        gen = ZshArgumentsGenerator(self.options, self.cmdName, self.file)
+        gen.extraActions.insert(0, SubcommandAction())
+        gen.write()
+
+        gen = ZshArgumentsGenerator(self.subOptions, self.cmdName, self.file)
+        gen.write()
+
+
+class ZshArgumentsGenerator:
+    """
+    Generate a call to the zsh _arguments completion function
+    based on data in a usage.Options instance
+
+    The first three instance variables are populated based on constructor
+    arguments. The remaining non-constructor variables are populated by this
+    class with data gathered from the C{Options} instance passed in, and its
+    base classes.
+
+    @type options: L{twisted.python.usage.Options}
+    @ivar options: The L{twisted.python.usage.Options} instance to generate for
+
+    @type cmdName: C{str}
+    @ivar cmdName: The name of the command we're generating completions for.
+
+    @type file: C{file}
+    @ivar file: The C{file} to write the completion function to.  The C{file}
+        must have L{bytes} I/O semantics.
+
+    @type descriptions: C{dict}
+    @ivar descriptions: A dict mapping long option names to alternate
+        descriptions. When this variable is defined, the descriptions
+        contained here will override those descriptions provided in the
+        optFlags and optParameters variables.
+
+    @type multiUse: C{list}
+    @ivar multiUse: An iterable containing those long option names which may
+        appear on the command line more than once. By default, options will
+        only be completed one time.
+
+    @type mutuallyExclusive: C{list} of C{tuple}
+    @ivar mutuallyExclusive: A sequence of sequences, with each sub-sequence
+        containing those long option names that are mutually exclusive. That is,
+        those options that cannot appear on the command line together.
+
+    @type optActions: C{dict}
+    @ivar optActions: A dict mapping long option names to shell "actions".
+        These actions define what may be completed as the argument to the
+        given option, and should be given as instances of
+        L{twisted.python.usage.Completer}.
+
+        Callables may instead be given for the values in this dict. The
+        callable should accept no arguments, and return a C{Completer}
+        instance used as the action.
+
+    @type extraActions: C{list} of C{twisted.python.usage.Completer}
+    @ivar extraActions: Extra arguments are those arguments typically
+        appearing at the end of the command-line, which are not associated
+        with any particular named option. That is, the arguments that are
+        given to the parseArgs() method of your usage.Options subclass.
+    """
+
+    def __init__(self, options, cmdName, file):
+        self.options = options
+        self.cmdName = cmdName
+        self.file = file
+
+        self.descriptions = {}
+        self.multiUse = set()
+        self.mutuallyExclusive = []
+        self.optActions = {}
+        self.extraActions = []
+
+        for cls in reversed(inspect.getmro(options.__class__)):
+            data = getattr(cls, "compData", None)
+            if data:
+                self.descriptions.update(data.descriptions)
+                self.optActions.update(data.optActions)
+                self.multiUse.update(data.multiUse)
+
+                self.mutuallyExclusive.extend(data.mutuallyExclusive)
+
+                # I don't see any sane way to aggregate extraActions, so just
+                # take the one at the top of the MRO (nearest the `options'
+                # instance).
+                if data.extraActions:
+                    self.extraActions = data.extraActions
+
+        aCL = reflect.accumulateClassList
+
+        optFlags: List[List[object]] = []
+        optParams: List[List[object]] = []
+
+        aCL(options.__class__, "optFlags", optFlags)
+        aCL(options.__class__, "optParameters", optParams)
+
+        for i, optList in enumerate(optFlags):
+            if len(optList) != 3:
+                optFlags[i] = util.padTo(3, optList)
+
+        for i, optList in enumerate(optParams):
+            if len(optList) != 5:
+                optParams[i] = util.padTo(5, optList)
+
+        self.optFlags = optFlags
+        self.optParams = optParams
+
+        paramNameToDefinition = {}
+        for optList in optParams:
+            paramNameToDefinition[optList[0]] = optList[1:]
+        self.paramNameToDefinition = paramNameToDefinition
+
+        flagNameToDefinition = {}
+        for optList in optFlags:
+            flagNameToDefinition[optList[0]] = optList[1:]
+        self.flagNameToDefinition = flagNameToDefinition
+
+        allOptionsNameToDefinition = {}
+        allOptionsNameToDefinition.update(paramNameToDefinition)
+        allOptionsNameToDefinition.update(flagNameToDefinition)
+        self.allOptionsNameToDefinition = allOptionsNameToDefinition
+
+        self.addAdditionalOptions()
+
+        # makes sure none of the Completions metadata references
+        # option names that don't exist. (great for catching typos)
+        self.verifyZshNames()
+
+        self.excludes = self.makeExcludesDict()
+
+    def write(self):
+        """
+        Write the zsh completion code to the file given to __init__
+        @return: L{None}
+        """
+        self.writeHeader()
+        self.writeExtras()
+        self.writeOptions()
+        self.writeFooter()
+
+    def writeHeader(self):
+        """
+        This is the start of the code that calls _arguments
+        @return: L{None}
+        """
+        self.file.write(
+            b"#compdef " + self.cmdName.encode("utf-8") + b"\n\n"
+            b'_arguments -s -A "-*" \\\n'
+        )
+
+    def writeOptions(self):
+        """
+        Write out zsh code for each option in this command
+        @return: L{None}
+        """
+        optNames = list(self.allOptionsNameToDefinition.keys())
+        optNames.sort()
+        for longname in optNames:
+            self.writeOpt(longname)
+
+    def writeExtras(self):
+        """
+        Write out completion information for extra arguments appearing on the
+        command-line. These are extra positional arguments not associated
+        with a named option. That is, the stuff that gets passed to
+        Options.parseArgs().
+
+        @return: L{None}
+
+        @raise ValueError: If C{Completer} with C{repeat=True} is found and
+            is not the last item in the C{extraActions} list.
+        """
+        for i, action in enumerate(self.extraActions):
+            # a repeatable action must be the last action in the list
+            if action._repeat and i != len(self.extraActions) - 1:
+                raise ValueError(
+                    "Completer with repeat=True must be "
+                    "last item in Options.extraActions"
+                )
+            self.file.write(escape(action._shellCode("", usage._ZSH)).encode("utf-8"))
+            self.file.write(b" \\\n")
+
+    def writeFooter(self):
+        """
+        Write the last bit of code that finishes the call to _arguments
+        @return: L{None}
+        """
+        self.file.write(b"&& return 0\n")
+
+    def verifyZshNames(self):
+        """
+        Ensure that none of the option names given in the metadata are typoed
+        @return: L{None}
+        @raise ValueError: If unknown option names have been found.
+        """
+
+        def err(name):
+            raise ValueError(
+                'Unknown option name "%s" found while\n'
+                "examining Completions instances on %s" % (name, self.options)
+            )
+
+        for name in itertools.chain(self.descriptions, self.optActions, self.multiUse):
+            if name not in self.allOptionsNameToDefinition:
+                err(name)
+
+        for seq in self.mutuallyExclusive:
+            for name in seq:
+                if name not in self.allOptionsNameToDefinition:
+                    err(name)
+
+    def excludeStr(self, longname, buildShort=False):
+        """
+        Generate an "exclusion string" for the given option
+
+        @type longname: C{str}
+        @param longname: The long option name (e.g. "verbose" instead of "v")
+
+        @type buildShort: C{bool}
+        @param buildShort: May be True to indicate we're building an excludes
+            string for the short option that corresponds to the given long opt.
+
+        @return: The generated C{str}
+        """
+        if longname in self.excludes:
+            exclusions = self.excludes[longname].copy()
+        else:
+            exclusions = set()
+
+        # if longname isn't a multiUse option (can't appear on the cmd line more
+        # than once), then we have to exclude the short option if we're
+        # building for the long option, and vice versa.
+        if longname not in self.multiUse:
+            if buildShort is False:
+                short = self.getShortOption(longname)
+                if short is not None:
+                    exclusions.add(short)
+            else:
+                exclusions.add(longname)
+
+        if not exclusions:
+            return ""
+
+        strings = []
+        for optName in exclusions:
+            if len(optName) == 1:
+                # short option
+                strings.append("-" + optName)
+            else:
+                strings.append("--" + optName)
+        strings.sort()  # need deterministic order for reliable unit-tests
+        return "(%s)" % " ".join(strings)
+
+    def makeExcludesDict(self) -> Dict[str, Set[str]]:
+        """
+        @return: A C{dict} that maps each option name appearing in
+            self.mutuallyExclusive to a set of those option names that is it
+            mutually exclusive with (can't appear on the cmd line with).
+        """
+
+        # create a mapping of long option name -> single character name
+        longToShort = {}
+        for optList in itertools.chain(self.optParams, self.optFlags):
+            if optList[1] != None:
+                longToShort[optList[0]] = optList[1]
+
+        excludes: Dict[str, Set[str]] = {}
+        for lst in self.mutuallyExclusive:
+            for i, longname in enumerate(lst):
+                tmp = set(lst[:i] + lst[i + 1 :])
+                for name in tmp.copy():
+                    if name in longToShort:
+                        tmp.add(longToShort[name])
+
+                if longname in excludes:
+                    excludes[longname] = excludes[longname].union(tmp)
+                else:
+                    excludes[longname] = tmp
+        return excludes
+
+    def writeOpt(self, longname):
+        """
+        Write out the zsh code for the given argument. This is just part of the
+        one big call to _arguments
+
+        @type longname: C{str}
+        @param longname: The long option name (e.g. "verbose" instead of "v")
+
+        @return: L{None}
+        """
+        if longname in self.flagNameToDefinition:
+            # It's a flag option. Not one that takes a parameter.
+            longField = "--%s" % longname
+        else:
+            longField = "--%s=" % longname
+
+        short = self.getShortOption(longname)
+        if short != None:
+            shortField = "-" + short
+        else:
+            shortField = ""
+
+        descr = self.getDescription(longname)
+        descriptionField = descr.replace("[", r"\[")
+        descriptionField = descriptionField.replace("]", r"\]")
+        descriptionField = "[%s]" % descriptionField
+
+        actionField = self.getAction(longname)
+        if longname in self.multiUse:
+            multiField = "*"
+        else:
+            multiField = ""
+
+        longExclusionsField = self.excludeStr(longname)
+
+        if short:
+            # we have to write an extra line for the short option if we have one
+            shortExclusionsField = self.excludeStr(longname, buildShort=True)
+            self.file.write(
+                escape(
+                    "%s%s%s%s%s"
+                    % (
+                        shortExclusionsField,
+                        multiField,
+                        shortField,
+                        descriptionField,
+                        actionField,
+                    )
+                ).encode("utf-8")
+            )
+            self.file.write(b" \\\n")
+
+        self.file.write(
+            escape(
+                "%s%s%s%s%s"
+                % (
+                    longExclusionsField,
+                    multiField,
+                    longField,
+                    descriptionField,
+                    actionField,
+                )
+            ).encode("utf-8")
+        )
+        self.file.write(b" \\\n")
+
+    def getAction(self, longname):
+        """
+        Return a zsh "action" string for the given argument
+        @return: C{str}
+        """
+        if longname in self.optActions:
+            if callable(self.optActions[longname]):
+                action = self.optActions[longname]()
+            else:
+                action = self.optActions[longname]
+            return action._shellCode(longname, usage._ZSH)
+
+        if longname in self.paramNameToDefinition:
+            return f":{longname}:_files"
+        return ""
+
+    def getDescription(self, longname):
+        """
+        Return the description to be used for this argument
+        @return: C{str}
+        """
+        # check if we have an alternate descr for this arg, and if so use it
+        if longname in self.descriptions:
+            return self.descriptions[longname]
+
+        # otherwise we have to get it from the optFlags or optParams
+        try:
+            descr = self.flagNameToDefinition[longname][1]
+        except KeyError:
+            try:
+                descr = self.paramNameToDefinition[longname][2]
+            except KeyError:
+                descr = None
+
+        if descr is not None:
+            return descr
+
+        # let's try to get it from the opt_foo method doc string if there is one
+        longMangled = longname.replace("-", "_")  # this is what t.p.usage does
+        obj = getattr(self.options, "opt_%s" % longMangled, None)
+        if obj is not None:
+            descr = descrFromDoc(obj)
+            if descr is not None:
+                return descr
+
+        return longname  # we really ought to have a good description to use
+
+    def getShortOption(self, longname):
+        """
+        Return the short option letter or None
+        @return: C{str} or L{None}
+        """
+        optList = self.allOptionsNameToDefinition[longname]
+        return optList[0] or None
+
+    def addAdditionalOptions(self) -> None:
+        """
+        Add additional options to the optFlags and optParams lists.
+        These will be defined by 'opt_foo' methods of the Options subclass
+        @return: L{None}
+        """
+        methodsDict: Dict[str, MethodType] = {}
+        reflect.accumulateMethods(self.options, methodsDict, "opt_")
+        methodToShort = {}
+        for name in methodsDict.copy():
+            if len(name) == 1:
+                methodToShort[methodsDict[name]] = name
+                del methodsDict[name]
+
+        for methodName, methodObj in methodsDict.items():
+            longname = methodName.replace("_", "-")  # t.p.usage does this
+            # if this option is already defined by the optFlags or
+            # optParameters then we don't want to override that data
+            if longname in self.allOptionsNameToDefinition:
+                continue
+
+            descr = self.getDescription(longname)
+
+            short = None
+            if methodObj in methodToShort:
+                short = methodToShort[methodObj]
+
+            reqArgs = methodObj.__func__.__code__.co_argcount
+            if reqArgs == 2:
+                self.optParams.append([longname, short, None, descr])
+                self.paramNameToDefinition[longname] = [short, None, descr]
+                self.allOptionsNameToDefinition[longname] = [short, None, descr]
+            else:
+                # reqArgs must equal 1. self.options would have failed
+                # to instantiate if it had opt_ methods with bad signatures.
+                self.optFlags.append([longname, short, descr])
+                self.flagNameToDefinition[longname] = [short, descr]
+                self.allOptionsNameToDefinition[longname] = [short, None, descr]
+
+
+def descrFromDoc(obj):
+    """
+    Generate an appropriate description from docstring of the given object
+    """
+    if obj.__doc__ is None or obj.__doc__.isspace():
+        return None
+
+    lines = [x.strip() for x in obj.__doc__.split("\n") if x and not x.isspace()]
+    return " ".join(lines)
+
+
+def escape(x):
+    """
+    Shell escape the given string
+
+    Implementation borrowed from now-deprecated commands.mkarg() in the stdlib
+    """
+    if "'" not in x:
+        return "'" + x + "'"
+    s = '"'
+    for c in x:
+        if c in '\\$"`':
+            s = s + "\\"
+        s = s + c
+    s = s + '"'
+    return s
diff --git a/contrib/python/Twisted/py3/twisted/python/_textattributes.py b/contrib/python/Twisted/py3/twisted/python/_textattributes.py
new file mode 100644
index 00000000000..36403094ed4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/_textattributes.py
@@ -0,0 +1,304 @@
+# -*- test-case-name: twisted.python.test.test_textattributes -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module provides some common functionality for the manipulation of
+formatting states.
+
+Defining the mechanism by which text containing character attributes is
+constructed begins by subclassing L{CharacterAttributesMixin}.
+
+Defining how a single formatting state is to be serialized begins by
+subclassing L{_FormattingStateMixin}.
+
+Serializing a formatting structure is done with L{flatten}.
+
+@see: L{twisted.conch.insults.helper._FormattingState}
+@see: L{twisted.conch.insults.text._CharacterAttributes}
+@see: L{twisted.words.protocols.irc._FormattingState}
+@see: L{twisted.words.protocols.irc._CharacterAttributes}
+"""
+
+
+from typing import ClassVar, List, Sequence
+
+from twisted.python.util import FancyEqMixin
+
+
+class _Attribute(FancyEqMixin):
+    """
+    A text attribute.
+
+    Indexing a text attribute with a C{str} or another text attribute adds that
+    object as a child, indexing with a C{list} or C{tuple} adds the elements as
+    children; in either case C{self} is returned.
+
+    @type children: C{list}
+    @ivar children: Child attributes.
+    """
+
+    compareAttributes: ClassVar[Sequence[str]] = ("children",)
+
+    def __init__(self):
+        self.children = []
+
+    def __repr__(self) -> str:
+        return f"<{type(self).__name__} {vars(self)!r}>"
+
+    def __getitem__(self, item):
+        assert isinstance(item, (list, tuple, _Attribute, str))
+        if isinstance(item, (list, tuple)):
+            self.children.extend(item)
+        else:
+            self.children.append(item)
+        return self
+
+    def serialize(self, write, attrs=None, attributeRenderer="toVT102"):
+        """
+        Serialize the text attribute and its children.
+
+        @param write: C{callable}, taking one C{str} argument, called to output
+            a single text attribute at a time.
+
+        @param attrs: A formatting state instance used to determine how to
+            serialize the attribute children.
+
+        @type attributeRenderer: C{str}
+        @param attributeRenderer: Name of the method on I{attrs} that should be
+            called to render the attributes during serialization. Defaults to
+            C{'toVT102'}.
+        """
+        if attrs is None:
+            attrs = DefaultFormattingState()
+        for ch in self.children:
+            if isinstance(ch, _Attribute):
+                ch.serialize(write, attrs.copy(), attributeRenderer)
+            else:
+                renderMeth = getattr(attrs, attributeRenderer)
+                write(renderMeth())
+                write(ch)
+
+
+class _NormalAttr(_Attribute):
+    """
+    A text attribute for normal text.
+    """
+
+    def serialize(self, write, attrs, attributeRenderer):
+        attrs.__init__()
+        _Attribute.serialize(self, write, attrs, attributeRenderer)
+
+
+class _OtherAttr(_Attribute):
+    """
+    A text attribute for text with formatting attributes.
+
+    The unary minus operator returns the inverse of this attribute, where that
+    makes sense.
+
+    @type attrname: C{str}
+    @ivar attrname: Text attribute name.
+
+    @ivar attrvalue: Text attribute value.
+    """
+
+    compareAttributes = ("attrname", "attrvalue", "children")
+
+    def __init__(self, attrname, attrvalue):
+        _Attribute.__init__(self)
+        self.attrname = attrname
+        self.attrvalue = attrvalue
+
+    def __neg__(self):
+        result = _OtherAttr(self.attrname, not self.attrvalue)
+        result.children.extend(self.children)
+        return result
+
+    def serialize(self, write, attrs, attributeRenderer):
+        attrs = attrs._withAttribute(self.attrname, self.attrvalue)
+        _Attribute.serialize(self, write, attrs, attributeRenderer)
+
+
+class _ColorAttr(_Attribute):
+    """
+    Generic color attribute.
+
+    @param color: Color value.
+
+    @param ground: Foreground or background attribute name.
+    """
+
+    compareAttributes = ("color", "ground", "children")
+
+    def __init__(self, color, ground):
+        _Attribute.__init__(self)
+        self.color = color
+        self.ground = ground
+
+    def serialize(self, write, attrs, attributeRenderer):
+        attrs = attrs._withAttribute(self.ground, self.color)
+        _Attribute.serialize(self, write, attrs, attributeRenderer)
+
+
+class _ForegroundColorAttr(_ColorAttr):
+    """
+    Foreground color attribute.
+    """
+
+    def __init__(self, color):
+        _ColorAttr.__init__(self, color, "foreground")
+
+
+class _BackgroundColorAttr(_ColorAttr):
+    """
+    Background color attribute.
+    """
+
+    def __init__(self, color):
+        _ColorAttr.__init__(self, color, "background")
+
+
+class _ColorAttribute:
+    """
+    A color text attribute.
+
+    Attribute access results in a color value lookup, by name, in
+    I{_ColorAttribute.attrs}.
+
+    @type ground: L{_ColorAttr}
+    @param ground: Foreground or background color attribute to look color names
+        up from.
+
+    @param attrs: Mapping of color names to color values.
+    @type attrs: Dict like object.
+    """
+
+    def __init__(self, ground, attrs):
+        self.ground = ground
+        self.attrs = attrs
+
+    def __getattr__(self, name):
+        try:
+            return self.ground(self.attrs[name])
+        except KeyError:
+            raise AttributeError(name)
+
+
+class CharacterAttributesMixin:
+    """
+    Mixin for character attributes that implements a C{__getattr__} method
+    returning a new C{_NormalAttr} instance when attempting to access
+    a C{'normal'} attribute; otherwise a new C{_OtherAttr} instance is returned
+    for names that appears in the C{'attrs'} attribute.
+    """
+
+    def __getattr__(self, name):
+        if name == "normal":
+            return _NormalAttr()
+        if name in self.attrs:
+            return _OtherAttr(name, True)
+        raise AttributeError(name)
+
+
+class DefaultFormattingState(FancyEqMixin):
+    """
+    A character attribute that does nothing, thus applying no attributes to
+    text.
+    """
+
+    compareAttributes: ClassVar[Sequence[str]] = ("_dummy",)
+
+    _dummy = 0
+
+    def copy(self):
+        """
+        Make a copy of this formatting state.
+
+        @return: A formatting state instance.
+        """
+        return type(self)()
+
+    def _withAttribute(self, name, value):
+        """
+        Add a character attribute to a copy of this formatting state.
+
+        @param name: Attribute name to be added to formatting state.
+
+        @param value: Attribute value.
+
+        @return: A formatting state instance with the new attribute.
+        """
+        return self.copy()
+
+    def toVT102(self):
+        """
+        Emit a VT102 control sequence that will set up all the attributes this
+        formatting state has set.
+
+        @return: A string containing VT102 control sequences that mimic this
+            formatting state.
+        """
+        return ""
+
+
+class _FormattingStateMixin(DefaultFormattingState):
+    """
+    Mixin for the formatting state/attributes of a single character.
+    """
+
+    def copy(self):
+        c = DefaultFormattingState.copy(self)
+        c.__dict__.update(vars(self))
+        return c
+
+    def _withAttribute(self, name, value):
+        if getattr(self, name) != value:
+            attr = self.copy()
+            attr._subtracting = not value
+            setattr(attr, name, value)
+            return attr
+        else:
+            return self.copy()
+
+
+def flatten(output, attrs, attributeRenderer="toVT102"):
+    """
+    Serialize a sequence of characters with attribute information
+
+    The resulting string can be interpreted by compatible software so that the
+    contained characters are displayed and, for those attributes which are
+    supported by the software, the attributes expressed. The exact result of
+    the serialization depends on the behavior of the method specified by
+    I{attributeRenderer}.
+
+    For example, if your terminal is VT102 compatible, you might run
+    this for a colorful variation on the \"hello world\" theme::
+
+        from twisted.conch.insults.text import flatten, attributes as A
+        from twisted.conch.insults.helper import CharacterAttribute
+        print(flatten(
+            A.normal[A.bold[A.fg.red['He'], A.fg.green['ll'], A.fg.magenta['o'], ' ',
+                            A.fg.yellow['Wo'], A.fg.blue['rl'], A.fg.cyan['d!']]],
+            CharacterAttribute()))
+
+    @param output: Object returned by accessing attributes of the
+        module-level attributes object.
+
+    @param attrs: A formatting state instance used to determine how to
+        serialize C{output}.
+
+    @type attributeRenderer: C{str}
+    @param attributeRenderer: Name of the method on I{attrs} that should be
+        called to render the attributes during serialization. Defaults to
+        C{'toVT102'}.
+
+    @return: A string expressing the text and display attributes specified by
+        L{output}.
+    """
+    flattened: List[str] = []
+    output.serialize(flattened.append, attrs, attributeRenderer)
+    return "".join(flattened)
+
+
+__all__ = ["flatten", "DefaultFormattingState", "CharacterAttributesMixin"]
diff --git a/contrib/python/Twisted/py3/twisted/python/_tzhelper.py b/contrib/python/Twisted/py3/twisted/python/_tzhelper.py
new file mode 100644
index 00000000000..2b841889d75
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/_tzhelper.py
@@ -0,0 +1,105 @@
+# -*- test-case-name: twisted.python.test.test_tzhelper -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Time zone utilities.
+"""
+
+from datetime import (
+    datetime as DateTime,
+    timedelta as TimeDelta,
+    timezone,
+    tzinfo as TZInfo,
+)
+from typing import Optional
+
+__all__ = [
+    "FixedOffsetTimeZone",
+    "UTC",
+]
+
+
+class FixedOffsetTimeZone(TZInfo):
+    """
+    Represents a fixed timezone offset (without daylight saving time).
+
+    @ivar name: A L{str} giving the name of this timezone; the name just
+        includes how much time this offset represents.
+
+    @ivar offset: A L{TimeDelta} giving the amount of time this timezone is
+        offset.
+    """
+
+    def __init__(self, offset: TimeDelta, name: Optional[str] = None) -> None:
+        """
+        Construct a L{FixedOffsetTimeZone} with a fixed offset.
+
+        @param offset: a delta representing the offset from UTC.
+        @param name: A name to be given for this timezone.
+        """
+        self.offset = offset
+        self.name = name
+
+    @classmethod
+    def fromSignHoursMinutes(
+        cls, sign: str, hours: int, minutes: int
+    ) -> "FixedOffsetTimeZone":
+        """
+        Construct a L{FixedOffsetTimeZone} from an offset described by sign
+        ('+' or '-'), hours, and minutes.
+
+        @note: For protocol compatibility with AMP, this method never uses 'Z'
+
+        @param sign: A string describing the positive or negative-ness of the
+            offset.
+        @param hours: The number of hours in the offset.
+        @param minutes: The number of minutes in the offset
+
+        @return: A time zone with the given offset, and a name describing the
+            offset.
+        """
+        name = "%s%02i:%02i" % (sign, hours, minutes)
+        if sign == "-":
+            hours = -hours
+            minutes = -minutes
+        elif sign != "+":
+            raise ValueError(f"Invalid sign for timezone {sign!r}")
+        return cls(TimeDelta(hours=hours, minutes=minutes), name)
+
+    @classmethod
+    def fromLocalTimeStamp(cls, timeStamp: float) -> "FixedOffsetTimeZone":
+        """
+        Create a time zone with a fixed offset corresponding to a time stamp in
+        the system's locally configured time zone.
+        """
+        offset = DateTime.fromtimestamp(timeStamp) - DateTime.fromtimestamp(
+            timeStamp, timezone.utc
+        ).replace(tzinfo=None)
+        return cls(offset)
+
+    def utcoffset(self, dt: Optional[DateTime]) -> TimeDelta:
+        """
+        Return the given timezone's offset from UTC.
+        """
+        return self.offset
+
+    def dst(self, dt: Optional[DateTime]) -> TimeDelta:
+        """
+        Return a zero L{TimeDelta} for the daylight saving time
+        offset, since there is never one.
+        """
+        return TimeDelta(0)
+
+    def tzname(self, dt: Optional[DateTime]) -> str:
+        """
+        Return a string describing this timezone.
+        """
+        if self.name is not None:
+            return self.name
+        # XXX this is wrong; the tests are
+        dt = DateTime.fromtimestamp(0, self)
+        return dt.strftime("UTC%z")
+
+
+UTC = FixedOffsetTimeZone.fromSignHoursMinutes("+", 0, 0)
diff --git a/contrib/python/Twisted/py3/twisted/python/_url.py b/contrib/python/Twisted/py3/twisted/python/_url.py
new file mode 100644
index 00000000000..3bf3bc0782a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/_url.py
@@ -0,0 +1,11 @@
+# -*- test-case-name: twisted.python.test.test_url -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+URL parsing, construction and rendering.
+"""
+
+from hyperlink._url import URL
+
+__all__ = ["URL"]
diff --git a/contrib/python/Twisted/py3/twisted/python/compat.py b/contrib/python/Twisted/py3/twisted/python/compat.py
new file mode 100644
index 00000000000..5766ccf6484
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/compat.py
@@ -0,0 +1,650 @@
+# -*- test-case-name: twisted.test.test_compat -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Compatibility module to provide backwards compatibility for useful Python
+features.
+
+This is mainly for use of internal Twisted code. We encourage you to use
+the latest version of Python directly from your code, if possible.
+
+@var unicode: The type of Unicode strings, C{unicode} on Python 2 and C{str}
+    on Python 3.
+
+@var NativeStringIO: An in-memory file-like object that operates on the native
+    string type (bytes in Python 2, unicode in Python 3).
+
+@var urllib_parse: a URL-parsing module (urlparse on Python 2, urllib.parse on
+    Python 3)
+"""
+
+
+import inspect
+import os
+import platform
+import socket
+import urllib.parse as urllib_parse
+import warnings
+from collections.abc import Sequence
+from functools import reduce
+from html import escape
+from http import cookiejar as cookielib
+from io import IOBase, StringIO as NativeStringIO, TextIOBase
+from sys import intern
+from types import FrameType, MethodType as _MethodType
+from typing import Any, AnyStr, cast
+from urllib.parse import quote as urlquote, unquote as urlunquote
+
+from incremental import Version
+
+from twisted.python.deprecate import deprecated, deprecatedModuleAttribute
+
+if platform.python_implementation() == "PyPy":
+    _PYPY = True
+else:
+    _PYPY = False
+
+FileType = IOBase
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for io.IOBase",
+    __name__,
+    "FileType",
+)
+
+frozenset = frozenset
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for frozenset builtin type",
+    __name__,
+    "frozenset",
+)
+
+InstanceType = object
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Old-style classes don't exist in Python 3",
+    __name__,
+    "InstanceType",
+)
+
+izip = zip
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for zip() builtin",
+    __name__,
+    "izip",
+)
+
+long = int
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for int builtin type",
+    __name__,
+    "long",
+)
+
+range = range
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for range() builtin",
+    __name__,
+    "range",
+)
+
+raw_input = input
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for input() builtin",
+    __name__,
+    "raw_input",
+)
+
+set = set
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for set builtin type",
+    __name__,
+    "set",
+)
+
+StringType = str
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for str builtin type",
+    __name__,
+    "StringType",
+)
+
+unichr = chr
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for chr() builtin",
+    __name__,
+    "unichr",
+)
+
+unicode = str
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for str builtin type",
+    __name__,
+    "unicode",
+)
+
+xrange = range
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Obsolete alias for range() builtin",
+    __name__,
+    "xrange",
+)
+
+
+@deprecated(Version("Twisted", 21, 2, 0), replacement="d.items()")
+def iteritems(d):
+    """
+    Return an iterable of the items of C{d}.
+
+    @type d: L{dict}
+    @rtype: iterable
+    """
+    return d.items()
+
+
+@deprecated(Version("Twisted", 21, 2, 0), replacement="d.values()")
+def itervalues(d):
+    """
+    Return an iterable of the values of C{d}.
+
+    @type d: L{dict}
+    @rtype: iterable
+    """
+    return d.values()
+
+
+@deprecated(Version("Twisted", 21, 2, 0), replacement="list(d.items())")
+def items(d):
+    """
+    Return a list of the items of C{d}.
+
+    @type d: L{dict}
+    @rtype: L{list}
+    """
+    return list(d.items())
+
+
+def currentframe(n: int = 0) -> FrameType:
+    """
+    In Python 3, L{inspect.currentframe} does not take a stack-level argument.
+    Restore that functionality from Python 2 so we don't have to re-implement
+    the C{f_back}-walking loop in places where it's called.
+
+    @param n: The number of stack levels above the caller to walk.
+
+    @return: a frame, n levels up the stack from the caller.
+    """
+    f = inspect.currentframe()
+    for x in range(n + 1):
+        assert f is not None
+        f = f.f_back
+    assert f is not None
+    return f
+
+
+def execfile(filename, globals, locals=None):
+    """
+    Execute a Python script in the given namespaces.
+
+    Similar to the execfile builtin, but a namespace is mandatory, partly
+    because that's a sensible thing to require, and because otherwise we'd
+    have to do some frame hacking.
+
+    This is a compatibility implementation for Python 3 porting, to avoid the
+    use of the deprecated builtin C{execfile} function.
+    """
+    if locals is None:
+        locals = globals
+    with open(filename, "rb") as fin:
+        source = fin.read()
+    code = compile(source, filename, "exec")
+    exec(code, globals, locals)
+
+
+# type note: Can't find a Comparable type, despite
+# https://github.com/python/typing/issues/59
+def cmp(a: object, b: object) -> int:
+    """
+    Compare two objects.
+
+    Returns a negative number if C{a < b}, zero if they are equal, and a
+    positive number if C{a > b}.
+    """
+    if a < b:  # type: ignore[operator]
+        return -1
+    elif a == b:
+        return 0
+    else:
+        return 1
+
+
+def comparable(klass):
+    """
+    Class decorator that ensures support for the special C{__cmp__} method.
+
+    C{__eq__}, C{__lt__}, etc. methods are added to the class, relying on
+    C{__cmp__} to implement their comparisons.
+    """
+
+    def __eq__(self: Any, other: object) -> bool:
+        c = cast(bool, self.__cmp__(other))
+        if c is NotImplemented:
+            return c
+        return c == 0
+
+    def __ne__(self: Any, other: object) -> bool:
+        c = cast(bool, self.__cmp__(other))
+        if c is NotImplemented:
+            return c
+        return c != 0
+
+    def __lt__(self: Any, other: object) -> bool:
+        c = cast(bool, self.__cmp__(other))
+        if c is NotImplemented:
+            return c
+        return c < 0
+
+    def __le__(self: Any, other: object) -> bool:
+        c = cast(bool, self.__cmp__(other))
+        if c is NotImplemented:
+            return c
+        return c <= 0
+
+    def __gt__(self: Any, other: object) -> bool:
+        c = cast(bool, self.__cmp__(other))
+        if c is NotImplemented:
+            return c
+        return c > 0
+
+    def __ge__(self: Any, other: object) -> bool:
+        c = cast(bool, self.__cmp__(other))
+        if c is NotImplemented:
+            return c
+        return c >= 0
+
+    klass.__lt__ = __lt__
+    klass.__gt__ = __gt__
+    klass.__le__ = __le__
+    klass.__ge__ = __ge__
+    klass.__eq__ = __eq__
+    klass.__ne__ = __ne__
+    return klass
+
+
+def ioType(fileIshObject, default=str):
+    """
+    Determine the type which will be returned from the given file object's
+    read() and accepted by its write() method as an argument.
+
+    In other words, determine whether the given file is 'opened in text mode'.
+
+    @param fileIshObject: Any object, but ideally one which resembles a file.
+    @type fileIshObject: L{object}
+
+    @param default: A default value to return when the type of C{fileIshObject}
+        cannot be determined.
+    @type default: L{type}
+
+    @return: There are 3 possible return values:
+
+            1. L{str}, if the file is unambiguously opened in text mode.
+
+            2. L{bytes}, if the file is unambiguously opened in binary mode.
+
+            3. The C{default} parameter, if the given type is not understood.
+
+    @rtype: L{type}
+    """
+    if isinstance(fileIshObject, TextIOBase):
+        # If it's for text I/O, then it's for text I/O.
+        return str
+    if isinstance(fileIshObject, IOBase):
+        # If it's for I/O but it's _not_ for text I/O, it's for bytes I/O.
+        return bytes
+    encoding = getattr(fileIshObject, "encoding", None)
+    import codecs
+
+    if isinstance(fileIshObject, (codecs.StreamReader, codecs.StreamWriter)):
+        # On StreamReaderWriter, the 'encoding' attribute has special meaning;
+        # it is unambiguously text.
+        if encoding:
+            return str
+        else:
+            return bytes
+    return default
+
+
+def nativeString(s: AnyStr) -> str:
+    """
+    Convert C{bytes} or C{str} to C{str} type, using ASCII encoding if
+    conversion is necessary.
+
+    @raise UnicodeError: The input string is not ASCII encodable/decodable.
+    @raise TypeError: The input is neither C{bytes} nor C{str}.
+    """
+    if not isinstance(s, (bytes, str)):
+        raise TypeError("%r is neither bytes nor str" % s)
+    if isinstance(s, bytes):
+        return s.decode("ascii")
+    else:
+        # Ensure we're limited to ASCII subset:
+        s.encode("ascii")
+    return s
+
+
+def _matchingString(constantString, inputString):
+    """
+    Some functions, such as C{os.path.join}, operate on string arguments which
+    may be bytes or text, and wish to return a value of the same type.  In
+    those cases you may wish to have a string constant (in the case of
+    C{os.path.join}, that constant would be C{os.path.sep}) involved in the
+    parsing or processing, that must be of a matching type in order to use
+    string operations on it.  L{_matchingString} will take a constant string
+    (either L{bytes} or L{str}) and convert it to the same type as the
+    input string.  C{constantString} should contain only characters from ASCII;
+    to ensure this, it will be encoded or decoded regardless.
+
+    @param constantString: A string literal used in processing.
+    @type constantString: L{str} or L{bytes}
+
+    @param inputString: A byte string or text string provided by the user.
+    @type inputString: L{str} or L{bytes}
+
+    @return: C{constantString} converted into the same type as C{inputString}
+    @rtype: the type of C{inputString}
+    """
+    if isinstance(constantString, bytes):
+        otherType = constantString.decode("ascii")
+    else:
+        otherType = constantString.encode("ascii")
+    if type(constantString) == type(inputString):
+        return constantString
+    else:
+        return otherType
+
+
+@deprecated(
+    Version("Twisted", 21, 2, 0),
+    replacement="raise exception.with_traceback(traceback)",
+)
+def reraise(exception, traceback):
+    """
+    Re-raise an exception, with an optional traceback.
+
+    Re-raised exceptions will be mutated, with their C{__traceback__} attribute
+    being set.
+
+    @param exception: The exception instance.
+    @param traceback: The traceback to use, or L{None} indicating a new
+    traceback.
+    """
+    raise exception.with_traceback(traceback)
+
+
+def iterbytes(originalBytes):
+    """
+    Return an iterable wrapper for a C{bytes} object that provides the behavior
+    of iterating over C{bytes} on Python 2.
+
+    In particular, the results of iteration are the individual bytes (rather
+    than integers as on Python 3).
+
+    @param originalBytes: A C{bytes} object that will be wrapped.
+    """
+    for i in range(len(originalBytes)):
+        yield originalBytes[i : i + 1]
+
+
+@deprecated(Version("Twisted", 21, 2, 0), replacement="b'%d'")
+def intToBytes(i: int) -> bytes:
+    """
+    Convert the given integer into C{bytes}, as ASCII-encoded Arab numeral.
+
+    @param i: The C{int} to convert to C{bytes}.
+    @rtype: C{bytes}
+    """
+    return b"%d" % (i,)
+
+
+def lazyByteSlice(object, offset=0, size=None):
+    """
+    Return a copy of the given bytes-like object.
+
+    If an offset is given, the copy starts at that offset. If a size is
+    given, the copy will only be of that length.
+
+    @param object: C{bytes} to be copied.
+
+    @param offset: C{int}, starting index of copy.
+
+    @param size: Optional, if an C{int} is given limit the length of copy
+        to this size.
+    """
+    view = memoryview(object)
+    if size is None:
+        return view[offset:]
+    else:
+        return view[offset : (offset + size)]
+
+
+def networkString(s: str) -> bytes:
+    """
+    Convert a string to L{bytes} using ASCII encoding.
+
+    This is useful for sending text-like bytes that are constructed using
+    string interpolation.  For example::
+
+        networkString("Hello %d" % (n,))
+
+    @param s: A string to convert to bytes.
+    @type s: L{str}
+
+    @raise UnicodeError: The input string is not ASCII encodable.
+    @raise TypeError: The input is not L{str}.
+
+    @rtype: L{bytes}
+    """
+    if not isinstance(s, str):
+        raise TypeError("Can only convert strings to bytes")
+    return s.encode("ascii")
+
+
+@deprecated(Version("Twisted", 21, 2, 0), replacement="os.environb")
+def bytesEnviron():
+    """
+    Return a L{dict} of L{os.environ} where all text-strings are encoded into
+    L{bytes}.
+
+    This function is POSIX only; environment variables are always text strings
+    on Windows.
+    """
+    encodekey = os.environ.encodekey
+    encodevalue = os.environ.encodevalue
+
+    return {encodekey(x): encodevalue(y) for x, y in os.environ.items()}
+
+
+def _constructMethod(cls, name, self):
+    """
+    Construct a bound method.
+
+    @param cls: The class that the method should be bound to.
+    @type cls: L{type}
+
+    @param name: The name of the method.
+    @type name: native L{str}
+
+    @param self: The object that the method is bound to.
+    @type self: any object
+
+    @return: a bound method
+    @rtype: L{_MethodType}
+    """
+    func = cls.__dict__[name]
+    return _MethodType(func, self)
+
+
+def _get_async_param(isAsync=None, **kwargs):
+    """
+    Provide a backwards-compatible way to get async param value that does not
+    cause a syntax error under Python 3.7.
+
+    @param isAsync: isAsync param value (should default to None)
+    @type isAsync: L{bool}
+
+    @param kwargs: keyword arguments of the caller (only async is allowed)
+    @type kwargs: L{dict}
+
+    @raise TypeError: Both isAsync and async specified.
+
+    @return: Final isAsync param value
+    @rtype: L{bool}
+    """
+    if "async" in kwargs:
+        warnings.warn(
+            "'async' keyword argument is deprecated, please use isAsync",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+    if isAsync is None and "async" in kwargs:
+        isAsync = kwargs.pop("async")
+    if kwargs:
+        raise TypeError
+    return bool(isAsync)
+
+
+def _pypy3BlockingHack():
+    """
+    Work around U{https://foss.heptapod.net/pypy/pypy/-/issues/3051}
+    by replacing C{socket.fromfd} with a more conservative version.
+    """
+    try:
+        from fcntl import F_GETFL, F_SETFL, fcntl
+    except ImportError:
+        return
+    if not _PYPY:
+        return
+
+    def fromFDWithoutModifyingFlags(fd, family, type, proto=None):
+        passproto = [proto] * (proto is not None)
+        flags = fcntl(fd, F_GETFL)
+        try:
+            return realFromFD(fd, family, type, *passproto)
+        finally:
+            fcntl(fd, F_SETFL, flags)
+
+    realFromFD = socket.fromfd
+    if realFromFD.__name__ == fromFDWithoutModifyingFlags.__name__:
+        return
+    socket.fromfd = fromFDWithoutModifyingFlags
+
+
+_pypy3BlockingHack()
+
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Use functools.reduce() directly",
+    __name__,
+    "reduce",
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Use io.StringIO directly",
+    __name__,
+    "NativeStringIO",
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Import urllib.parse directly",
+    __name__,
+    "urllib_parse",
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0), "Use html.escape directly", __name__, "escape"
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Use urllib.parse.quote() directly",
+    __name__,
+    "urlquote",
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Use urllib.parse.unquote() directly",
+    __name__,
+    "urlunquote",
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Use http.cookiejar directly",
+    __name__,
+    "cookielib",
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0), "Use sys.intern() directly", __name__, "intern"
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Use collections.abc.Sequence directly",
+    __name__,
+    "Sequence",
+)
+
+
+__all__ = [
+    "reraise",
+    "execfile",
+    "frozenset",
+    "reduce",
+    "set",
+    "cmp",
+    "comparable",
+    "nativeString",
+    "NativeStringIO",
+    "networkString",
+    "unicode",
+    "iterbytes",
+    "intToBytes",
+    "lazyByteSlice",
+    "StringType",
+    "InstanceType",
+    "FileType",
+    "items",
+    "iteritems",
+    "itervalues",
+    "range",
+    "xrange",
+    "urllib_parse",
+    "bytesEnviron",
+    "escape",
+    "urlquote",
+    "urlunquote",
+    "cookielib",
+    "intern",
+    "unichr",
+    "raw_input",
+    "_get_async_param",
+    "Sequence",
+]
diff --git a/contrib/python/Twisted/py3/twisted/python/components.py b/contrib/python/Twisted/py3/twisted/python/components.py
new file mode 100644
index 00000000000..b6a6015dcc6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/components.py
@@ -0,0 +1,431 @@
+# -*- test-case-name: twisted.python.test.test_components -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Component architecture for Twisted, based on Zope3 components.
+
+Using the Zope3 API directly is strongly recommended. Everything
+you need is in the top-level of the zope.interface package, e.g.::
+
+   from zope.interface import Interface, implementer
+
+   class IFoo(Interface):
+       pass
+
+   @implementer(IFoo)
+   class Foo:
+       pass
+
+   print(IFoo.implementedBy(Foo)) # True
+   print(IFoo.providedBy(Foo())) # True
+
+L{twisted.python.components.registerAdapter} from this module may be used to
+add to Twisted's global adapter registry.
+
+L{twisted.python.components.proxyForInterface} is a factory for classes
+which allow access to only the parts of another class defined by a specified
+interface.
+"""
+
+
+from io import StringIO
+from typing import Dict
+
+# zope3 imports
+from zope.interface import declarations, interface
+from zope.interface.adapter import AdapterRegistry
+
+# twisted imports
+from twisted.python import reflect
+
+# Twisted's global adapter registry
+globalRegistry = AdapterRegistry()
+
+# Attribute that registerAdapter looks at. Is this supposed to be public?
+ALLOW_DUPLICATES = 0
+
+
+def registerAdapter(adapterFactory, origInterface, *interfaceClasses):
+    """Register an adapter class.
+
+    An adapter class is expected to implement the given interface, by
+    adapting instances implementing 'origInterface'. An adapter class's
+    __init__ method should accept one parameter, an instance implementing
+    'origInterface'.
+    """
+    self = globalRegistry
+    assert interfaceClasses, "You need to pass an Interface"
+    global ALLOW_DUPLICATES
+
+    # deal with class->interface adapters:
+    if not isinstance(origInterface, interface.InterfaceClass):
+        origInterface = declarations.implementedBy(origInterface)
+
+    for interfaceClass in interfaceClasses:
+        factory = self.registered([origInterface], interfaceClass)
+        if factory is not None and not ALLOW_DUPLICATES:
+            raise ValueError(f"an adapter ({factory}) was already registered.")
+    for interfaceClass in interfaceClasses:
+        self.register([origInterface], interfaceClass, "", adapterFactory)
+
+
+def getAdapterFactory(fromInterface, toInterface, default):
+    """Return registered adapter for a given class and interface.
+
+    Note that is tied to the *Twisted* global registry, and will
+    thus not find adapters registered elsewhere.
+    """
+    self = globalRegistry
+    if not isinstance(fromInterface, interface.InterfaceClass):
+        fromInterface = declarations.implementedBy(fromInterface)
+    factory = self.lookup1(fromInterface, toInterface)  # type: ignore[attr-defined]
+    if factory is None:
+        factory = default
+    return factory
+
+
+def _addHook(registry):
+    """
+    Add an adapter hook which will attempt to look up adapters in the given
+    registry.
+
+    @type registry: L{zope.interface.adapter.AdapterRegistry}
+
+    @return: The hook which was added, for later use with L{_removeHook}.
+    """
+    lookup = registry.lookup1
+
+    def _hook(iface, ob):
+        factory = lookup(declarations.providedBy(ob), iface)
+        if factory is None:
+            return None
+        else:
+            return factory(ob)
+
+    interface.adapter_hooks.append(_hook)
+    return _hook
+
+
+def _removeHook(hook):
+    """
+    Remove a previously added adapter hook.
+
+    @param hook: An object previously returned by a call to L{_addHook}.  This
+        will be removed from the list of adapter hooks.
+    """
+    interface.adapter_hooks.remove(hook)
+
+
+# add global adapter lookup hook for our newly created registry
+_addHook(globalRegistry)
+
+
+def getRegistry():
+    """Returns the Twisted global
+    C{zope.interface.adapter.AdapterRegistry} instance.
+    """
+    return globalRegistry
+
+
+# FIXME: deprecate attribute somehow?
+CannotAdapt = TypeError
+
+
+class Adapter:
+    """I am the default implementation of an Adapter for some interface.
+
+    This docstring contains a limerick, by popular demand::
+
+        Subclassing made Zope and TR
+        much harder to work with by far.
+            So before you inherit,
+            be sure to declare it
+        Adapter, not PyObject*
+
+    @cvar temporaryAdapter: If this is True, the adapter will not be
+          persisted on the Componentized.
+    @cvar multiComponent: If this adapter is persistent, should it be
+          automatically registered for all appropriate interfaces.
+    """
+
+    # These attributes are used with Componentized.
+
+    temporaryAdapter = 0
+    multiComponent = 1
+
+    def __init__(self, original):
+        """Set my 'original' attribute to be the object I am adapting."""
+        self.original = original
+
+    def __conform__(self, interface):
+        """
+        I forward __conform__ to self.original if it has it, otherwise I
+        simply return None.
+        """
+        if hasattr(self.original, "__conform__"):
+            return self.original.__conform__(interface)
+        return None
+
+    def isuper(self, iface, adapter):
+        """
+        Forward isuper to self.original
+        """
+        return self.original.isuper(iface, adapter)
+
+
+class Componentized:
+    """I am a mixin to allow you to be adapted in various ways persistently.
+
+    I define a list of persistent adapters.  This is to allow adapter classes
+    to store system-specific state, and initialized on demand.  The
+    getComponent method implements this.  You must also register adapters for
+    this class for the interfaces that you wish to pass to getComponent.
+
+    Many other classes and utilities listed here are present in Zope3; this one
+    is specific to Twisted.
+    """
+
+    persistenceVersion = 1
+
+    def __init__(self):
+        self._adapterCache = {}
+
+    def locateAdapterClass(self, klass, interfaceClass, default):
+        return getAdapterFactory(klass, interfaceClass, default)
+
+    def setAdapter(self, interfaceClass, adapterClass):
+        """
+        Cache a provider for the given interface, by adapting C{self} using
+        the given adapter class.
+        """
+        self.setComponent(interfaceClass, adapterClass(self))
+
+    def addAdapter(self, adapterClass, ignoreClass=0):
+        """Utility method that calls addComponent.  I take an adapter class and
+        instantiate it with myself as the first argument.
+
+        @return: The adapter instantiated.
+        """
+        adapt = adapterClass(self)
+        self.addComponent(adapt, ignoreClass)
+        return adapt
+
+    def setComponent(self, interfaceClass, component):
+        """
+        Cache a provider of the given interface.
+        """
+        self._adapterCache[reflect.qual(interfaceClass)] = component
+
+    def addComponent(self, component, ignoreClass=0):
+        """
+        Add a component to me, for all appropriate interfaces.
+
+        In order to determine which interfaces are appropriate, the component's
+        provided interfaces will be scanned.
+
+        If the argument 'ignoreClass' is True, then all interfaces are
+        considered appropriate.
+
+        Otherwise, an 'appropriate' interface is one for which its class has
+        been registered as an adapter for my class according to the rules of
+        getComponent.
+        """
+        for iface in declarations.providedBy(component):
+            if ignoreClass or (
+                self.locateAdapterClass(self.__class__, iface, None)
+                == component.__class__
+            ):
+                self._adapterCache[reflect.qual(iface)] = component
+
+    def unsetComponent(self, interfaceClass):
+        """Remove my component specified by the given interface class."""
+        del self._adapterCache[reflect.qual(interfaceClass)]
+
+    def removeComponent(self, component):
+        """
+        Remove the given component from me entirely, for all interfaces for which
+        it has been registered.
+
+        @return: a list of the interfaces that were removed.
+        """
+        l = []
+        for k, v in list(self._adapterCache.items()):
+            if v is component:
+                del self._adapterCache[k]
+                l.append(reflect.namedObject(k))
+        return l
+
+    def getComponent(self, interface, default=None):
+        """Create or retrieve an adapter for the given interface.
+
+        If such an adapter has already been created, retrieve it from the cache
+        that this instance keeps of all its adapters.  Adapters created through
+        this mechanism may safely store system-specific state.
+
+        If you want to register an adapter that will be created through
+        getComponent, but you don't require (or don't want) your adapter to be
+        cached and kept alive for the lifetime of this Componentized object,
+        set the attribute 'temporaryAdapter' to True on your adapter class.
+
+        If you want to automatically register an adapter for all appropriate
+        interfaces (with addComponent), set the attribute 'multiComponent' to
+        True on your adapter class.
+        """
+        k = reflect.qual(interface)
+        if k in self._adapterCache:
+            return self._adapterCache[k]
+        else:
+            adapter = interface.__adapt__(self)
+            if adapter is not None and not (
+                hasattr(adapter, "temporaryAdapter") and adapter.temporaryAdapter
+            ):
+                self._adapterCache[k] = adapter
+                if hasattr(adapter, "multiComponent") and adapter.multiComponent:
+                    self.addComponent(adapter)
+            if adapter is None:
+                return default
+            return adapter
+
+    def __conform__(self, interface):
+        return self.getComponent(interface)
+
+
+class ReprableComponentized(Componentized):
+    def __init__(self):
+        Componentized.__init__(self)
+
+    def __repr__(self) -> str:
+        from pprint import pprint
+
+        sio = StringIO()
+        pprint(self._adapterCache, sio)
+        return sio.getvalue()
+
+
+def proxyForInterface(iface, originalAttribute="original"):
+    """
+    Create a class which proxies all method calls which adhere to an interface
+    to another provider of that interface.
+
+    This function is intended for creating specialized proxies. The typical way
+    to use it is by subclassing the result::
+
+      class MySpecializedProxy(proxyForInterface(IFoo)):
+          def someInterfaceMethod(self, arg):
+              if arg == 3:
+                  return 3
+              return self.original.someInterfaceMethod(arg)
+
+    @param iface: The Interface to which the resulting object will conform, and
+        which the wrapped object must provide.
+
+    @param originalAttribute: name of the attribute used to save the original
+        object in the resulting class. Default to C{original}.
+    @type originalAttribute: C{str}
+
+    @return: A class whose constructor takes the original object as its only
+        argument. Constructing the class creates the proxy.
+    """
+
+    def __init__(self, original):
+        setattr(self, originalAttribute, original)
+
+    contents: Dict[str, object] = {"__init__": __init__}
+    for name in iface:
+        contents[name] = _ProxyDescriptor(name, originalAttribute)
+    proxy = type(f"(Proxy for {reflect.qual(iface)})", (object,), contents)
+    # mypy-zope declarations.classImplements only works when passing
+    # a concrete class type
+    declarations.classImplements(proxy, iface)  # type: ignore[misc]
+    return proxy
+
+
+class _ProxiedClassMethod:
+    """
+    A proxied class method.
+
+    @ivar methodName: the name of the method which this should invoke when
+        called.
+    @type methodName: L{str}
+
+    @ivar __name__: The name of the method being proxied (the same as
+        C{methodName}).
+    @type __name__: L{str}
+
+    @ivar originalAttribute: name of the attribute of the proxy where the
+        original object is stored.
+    @type originalAttribute: L{str}
+    """
+
+    def __init__(self, methodName, originalAttribute):
+        self.methodName = self.__name__ = methodName
+        self.originalAttribute = originalAttribute
+
+    def __call__(self, oself, *args, **kw):
+        """
+        Invoke the specified L{methodName} method of the C{original} attribute
+        for proxyForInterface.
+
+        @param oself: an instance of a L{proxyForInterface} object.
+
+        @return: the result of the underlying method.
+        """
+        original = getattr(oself, self.originalAttribute)
+        actualMethod = getattr(original, self.methodName)
+        return actualMethod(*args, **kw)
+
+
+class _ProxyDescriptor:
+    """
+    A descriptor which will proxy attribute access, mutation, and
+    deletion to the L{_ProxyDescriptor.originalAttribute} of the
+    object it is being accessed from.
+
+    @ivar attributeName: the name of the attribute which this descriptor will
+        retrieve from instances' C{original} attribute.
+    @type attributeName: C{str}
+
+    @ivar originalAttribute: name of the attribute of the proxy where the
+        original object is stored.
+    @type originalAttribute: C{str}
+    """
+
+    def __init__(self, attributeName, originalAttribute):
+        self.attributeName = attributeName
+        self.originalAttribute = originalAttribute
+
+    def __get__(self, oself, type=None):
+        """
+        Retrieve the C{self.attributeName} property from I{oself}.
+        """
+        if oself is None:
+            return _ProxiedClassMethod(self.attributeName, self.originalAttribute)
+        original = getattr(oself, self.originalAttribute)
+        return getattr(original, self.attributeName)
+
+    def __set__(self, oself, value):
+        """
+        Set the C{self.attributeName} property of I{oself}.
+        """
+        original = getattr(oself, self.originalAttribute)
+        setattr(original, self.attributeName, value)
+
+    def __delete__(self, oself):
+        """
+        Delete the C{self.attributeName} property of I{oself}.
+        """
+        original = getattr(oself, self.originalAttribute)
+        delattr(original, self.attributeName)
+
+
+__all__ = [
+    "registerAdapter",
+    "getAdapterFactory",
+    "Adapter",
+    "Componentized",
+    "ReprableComponentized",
+    "getRegistry",
+    "proxyForInterface",
+]
diff --git a/contrib/python/Twisted/py3/twisted/python/constants.py b/contrib/python/Twisted/py3/twisted/python/constants.py
new file mode 100644
index 00000000000..fbad78e4080
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/constants.py
@@ -0,0 +1,21 @@
+# -*- test-case-name: twisted.python.test.test_constants -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Symbolic constant support, including collections and constants with text,
+numeric, and bit flag values.
+"""
+
+
+# Import and re-export Constantly
+from constantly import (  # type: ignore[import]
+    FlagConstant,
+    Flags,
+    NamedConstant,
+    Names,
+    ValueConstant,
+    Values,
+)
+
+__all__ = ["NamedConstant", "ValueConstant", "FlagConstant", "Names", "Values", "Flags"]
diff --git a/contrib/python/Twisted/py3/twisted/python/context.py b/contrib/python/Twisted/py3/twisted/python/context.py
new file mode 100644
index 00000000000..f2d3bd82e9a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/context.py
@@ -0,0 +1,135 @@
+# -*- test-case-name: twisted.test.test_context -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Dynamic pseudo-scoping for Python.
+
+Call functions with context.call({key: value}, func); func and
+functions that it calls will be able to use 'context.get(key)' to
+retrieve 'value'.
+
+This is thread-safe.
+"""
+
+
+from threading import local
+from typing import Dict, Type
+
+defaultContextDict: Dict[Type[object], Dict[str, str]] = {}
+
+setDefault = defaultContextDict.__setitem__
+
+
+class ContextTracker:
+    """
+    A L{ContextTracker} provides a way to pass arbitrary key/value data up and
+    down a call stack without passing them as parameters to the functions on
+    that call stack.
+
+    This can be useful when functions on the top and bottom of the call stack
+    need to cooperate but the functions in between them do not allow passing the
+    necessary state.  For example::
+
+        from twisted.python.context import call, get
+
+        def handleRequest(request):
+            call({'request-id': request.id}, renderRequest, request.url)
+
+        def renderRequest(url):
+            renderHeader(url)
+            renderBody(url)
+
+        def renderHeader(url):
+            return "the header"
+
+        def renderBody(url):
+            return "the body (request id=%r)" % (get("request-id"),)
+
+    This should be used sparingly, since the lack of a clear connection between
+    the two halves can result in code which is difficult to understand and
+    maintain.
+
+    @ivar contexts: A C{list} of C{dict}s tracking the context state.  Each new
+        L{ContextTracker.callWithContext} pushes a new C{dict} onto this stack
+        for the duration of the call, making the data available to the function
+        called and restoring the previous data once it is complete..
+    """
+
+    def __init__(self):
+        self.contexts = [defaultContextDict]
+
+    def callWithContext(self, newContext, func, *args, **kw):
+        """
+        Call C{func(*args, **kw)} such that the contents of C{newContext} will
+        be available for it to retrieve using L{getContext}.
+
+        @param newContext: A C{dict} of data to push onto the context for the
+            duration of the call to C{func}.
+
+        @param func: A callable which will be called.
+
+        @param args: Any additional positional arguments to pass to C{func}.
+
+        @param kw: Any additional keyword arguments to pass to C{func}.
+
+        @return: Whatever is returned by C{func}
+
+        @raise Exception: Whatever is raised by C{func}.
+        """
+        self.contexts.append(newContext)
+        try:
+            return func(*args, **kw)
+        finally:
+            self.contexts.pop()
+
+    def getContext(self, key, default=None):
+        """
+        Retrieve the value for a key from the context.
+
+        @param key: The key to look up in the context.
+
+        @param default: The value to return if C{key} is not found in the
+            context.
+
+        @return: The value most recently remembered in the context for C{key}.
+        """
+        for ctx in reversed(self.contexts):
+            try:
+                return ctx[key]
+            except KeyError:
+                pass
+        return default
+
+
+class ThreadedContextTracker:
+    def __init__(self):
+        self.storage = local()
+
+    def currentContext(self):
+        try:
+            return self.storage.ct
+        except AttributeError:
+            ct = self.storage.ct = ContextTracker()
+            return ct
+
+    def callWithContext(self, ctx, func, *args, **kw):
+        return self.currentContext().callWithContext(ctx, func, *args, **kw)
+
+    def getContext(self, key, default=None):
+        return self.currentContext().getContext(key, default)
+
+
+theContextTracker = ThreadedContextTracker()
+call = theContextTracker.callWithContext
+get = theContextTracker.getContext
+
+
+def installContextTracker(ctr):
+    global theContextTracker
+    global call
+    global get
+
+    theContextTracker = ctr
+    call = theContextTracker.callWithContext
+    get = theContextTracker.getContext
diff --git a/contrib/python/Twisted/py3/twisted/python/deprecate.py b/contrib/python/Twisted/py3/twisted/python/deprecate.py
new file mode 100644
index 00000000000..c85b98d6272
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/deprecate.py
@@ -0,0 +1,820 @@
+# -*- test-case-name: twisted.python.test.test_deprecate -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Deprecation framework for Twisted.
+
+To mark a method, function, or class as being deprecated do this::
+
+    from incremental import Version
+    from twisted.python.deprecate import deprecated
+
+    @deprecated(Version("Twisted", 22, 10, 0))
+    def badAPI(self, first, second):
+        '''
+        Docstring for badAPI.
+        '''
+        ...
+
+    @deprecated(Version("Twisted", 22, 10, 0))
+    class BadClass:
+        '''
+        Docstring for BadClass.
+        '''
+
+The newly-decorated badAPI will issue a warning when called, and BadClass will
+issue a warning when instantiated. Both will also have  a deprecation notice
+appended to their docstring.
+
+To deprecate properties you can use::
+
+    from incremental import Version
+    from twisted.python.deprecate import deprecatedProperty
+
+    class OtherwiseUndeprecatedClass:
+
+        @deprecatedProperty(Version("Twisted", 22, 10, 0))
+        def badProperty(self):
+            '''
+            Docstring for badProperty.
+            '''
+
+        @badProperty.setter
+        def badProperty(self, value):
+            '''
+            Setter sill also raise the deprecation warning.
+            '''
+
+
+To mark module-level attributes as being deprecated you can use::
+
+    badAttribute = "someValue"
+
+    ...
+
+    deprecatedModuleAttribute(
+        Version("Twisted", 22, 10, 0),
+        "Use goodAttribute instead.",
+        "your.full.module.name",
+        "badAttribute")
+
+The deprecated attributes will issue a warning whenever they are accessed. If
+the attributes being deprecated are in the same module as the
+L{deprecatedModuleAttribute} call is being made from, the C{__name__} global
+can be used as the C{moduleName} parameter.
+
+
+To mark an optional, keyword parameter of a function or method as deprecated
+without deprecating the function itself, you can use::
+
+    @deprecatedKeywordParameter(Version("Twisted", 22, 10, 0), "baz")
+    def someFunction(foo, bar=0, baz=None):
+        ...
+
+See also L{incremental.Version}.
+
+@type DEPRECATION_WARNING_FORMAT: C{str}
+@var DEPRECATION_WARNING_FORMAT: The default deprecation warning string format
+    to use when one is not provided by the user.
+"""
+from __future__ import annotations
+
+__all__ = [
+    "deprecated",
+    "deprecatedProperty",
+    "getDeprecationWarningString",
+    "getWarningMethod",
+    "setWarningMethod",
+    "deprecatedModuleAttribute",
+    "deprecatedKeywordParameter",
+]
+
+
+import inspect
+import sys
+from dis import findlinestarts
+from functools import wraps
+from types import ModuleType
+from typing import Any, Callable, Dict, Optional, TypeVar, cast
+from warnings import warn, warn_explicit
+
+from incremental import Version, getVersionString
+from typing_extensions import ParamSpec
+
+_P = ParamSpec("_P")
+_R = TypeVar("_R")
+
+DEPRECATION_WARNING_FORMAT = "%(fqpn)s was deprecated in %(version)s"
+
+# Notionally, part of twisted.python.reflect, but defining it there causes a
+# cyclic dependency between this module and that module.  Define it here,
+# instead, and let reflect import it to re-expose to the public.
+
+
+def _fullyQualifiedName(obj):
+    """
+    Return the fully qualified name of a module, class, method or function.
+    Classes and functions need to be module level ones to be correctly
+    qualified.
+
+    @rtype: C{str}.
+    """
+    try:
+        name = obj.__qualname__
+    except AttributeError:
+        name = obj.__name__
+
+    if inspect.isclass(obj) or inspect.isfunction(obj):
+        moduleName = obj.__module__
+        return f"{moduleName}.{name}"
+    elif inspect.ismethod(obj):
+        return f"{obj.__module__}.{obj.__qualname__}"
+    return name
+
+
+# Try to keep it looking like something in twisted.python.reflect.
+_fullyQualifiedName.__module__ = "twisted.python.reflect"
+_fullyQualifiedName.__name__ = "fullyQualifiedName"
+_fullyQualifiedName.__qualname__ = "fullyQualifiedName"
+
+
+def _getReplacementString(replacement):
+    """
+    Surround a replacement for a deprecated API with some polite text exhorting
+    the user to consider it as an alternative.
+
+    @type replacement: C{str} or callable
+
+    @return: a string like "please use twisted.python.modules.getModule
+        instead".
+    """
+    if callable(replacement):
+        replacement = _fullyQualifiedName(replacement)
+    return f"please use {replacement} instead"
+
+
+def _getDeprecationDocstring(version, replacement=None):
+    """
+    Generate an addition to a deprecated object's docstring that explains its
+    deprecation.
+
+    @param version: the version it was deprecated.
+    @type version: L{incremental.Version}
+
+    @param replacement: The replacement, if specified.
+    @type replacement: C{str} or callable
+
+    @return: a string like "Deprecated in Twisted 27.2.0; please use
+        twisted.timestream.tachyon.flux instead."
+    """
+    doc = f"Deprecated in {getVersionString(version)}"
+    if replacement:
+        doc = f"{doc}; {_getReplacementString(replacement)}"
+    return doc + "."
+
+
+def _getDeprecationWarningString(fqpn, version, format=None, replacement=None):
+    """
+    Return a string indicating that the Python name was deprecated in the given
+    version.
+
+    @param fqpn: Fully qualified Python name of the thing being deprecated
+    @type fqpn: C{str}
+
+    @param version: Version that C{fqpn} was deprecated in.
+    @type version: L{incremental.Version}
+
+    @param format: A user-provided format to interpolate warning values into, or
+        L{DEPRECATION_WARNING_FORMAT
+        <twisted.python.deprecate.DEPRECATION_WARNING_FORMAT>} if L{None} is
+        given.
+    @type format: C{str}
+
+    @param replacement: what should be used in place of C{fqpn}. Either pass in
+        a string, which will be inserted into the warning message, or a
+        callable, which will be expanded to its full import path.
+    @type replacement: C{str} or callable
+
+    @return: A textual description of the deprecation
+    @rtype: C{str}
+    """
+    if format is None:
+        format = DEPRECATION_WARNING_FORMAT
+    warningString = format % {"fqpn": fqpn, "version": getVersionString(version)}
+    if replacement:
+        warningString = "{}; {}".format(
+            warningString, _getReplacementString(replacement)
+        )
+    return warningString
+
+
+def getDeprecationWarningString(callableThing, version, format=None, replacement=None):
+    """
+    Return a string indicating that the callable was deprecated in the given
+    version.
+
+    @type callableThing: C{callable}
+    @param callableThing: Callable object to be deprecated
+
+    @type version: L{incremental.Version}
+    @param version: Version that C{callableThing} was deprecated in.
+
+    @type format: C{str}
+    @param format: A user-provided format to interpolate warning values into,
+        or L{DEPRECATION_WARNING_FORMAT
+        <twisted.python.deprecate.DEPRECATION_WARNING_FORMAT>} if L{None} is
+        given
+
+    @param replacement: what should be used in place of the callable. Either
+        pass in a string, which will be inserted into the warning message,
+        or a callable, which will be expanded to its full import path.
+    @type replacement: C{str} or callable
+
+    @return: A string describing the deprecation.
+    @rtype: C{str}
+    """
+    return _getDeprecationWarningString(
+        _fullyQualifiedName(callableThing), version, format, replacement
+    )
+
+
+def _appendToDocstring(thingWithDoc, textToAppend):
+    """
+    Append the given text to the docstring of C{thingWithDoc}.
+
+    If C{thingWithDoc} has no docstring, then the text just replaces the
+    docstring. If it has a single-line docstring then it appends a blank line
+    and the message text. If it has a multi-line docstring, then in appends a
+    blank line a the message text, and also does the indentation correctly.
+    """
+    if thingWithDoc.__doc__:
+        docstringLines = thingWithDoc.__doc__.splitlines()
+    else:
+        docstringLines = []
+
+    if len(docstringLines) == 0:
+        docstringLines.append(textToAppend)
+    elif len(docstringLines) == 1:
+        docstringLines.extend(["", textToAppend, ""])
+    else:
+        spaces = docstringLines.pop()
+        docstringLines.extend(["", spaces + textToAppend, spaces])
+    thingWithDoc.__doc__ = "\n".join(docstringLines)
+
+
+def deprecated(
+    version: Version, replacement: str | Callable[..., object] | None = None
+) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]:
+    """
+    Return a decorator that marks callables as deprecated. To deprecate a
+    property, see L{deprecatedProperty}.
+
+    @type version: L{incremental.Version}
+    @param version: The version in which the callable will be marked as
+        having been deprecated.  The decorated function will be annotated
+        with this version, having it set as its C{deprecatedVersion}
+        attribute.
+
+    @param replacement: what should be used in place of the callable. Either
+        pass in a string, which will be inserted into the warning message,
+        or a callable, which will be expanded to its full import path.
+    @type replacement: C{str} or callable
+    """
+
+    def deprecationDecorator(function: Callable[_P, _R]) -> Callable[_P, _R]:
+        """
+        Decorator that marks C{function} as deprecated.
+        """
+        warningString = getDeprecationWarningString(
+            function, version, None, replacement
+        )
+
+        @wraps(function)
+        def deprecatedFunction(*args: _P.args, **kwargs: _P.kwargs) -> _R:
+            warn(warningString, DeprecationWarning, stacklevel=2)
+            return function(*args, **kwargs)
+
+        _appendToDocstring(
+            deprecatedFunction, _getDeprecationDocstring(version, replacement)
+        )
+        deprecatedFunction.deprecatedVersion = version  # type: ignore[attr-defined]
+        return deprecatedFunction
+
+    return deprecationDecorator
+
+
+def deprecatedProperty(version, replacement=None):
+    """
+    Return a decorator that marks a property as deprecated. To deprecate a
+    regular callable or class, see L{deprecated}.
+
+    @type version: L{incremental.Version}
+    @param version: The version in which the callable will be marked as
+        having been deprecated.  The decorated function will be annotated
+        with this version, having it set as its C{deprecatedVersion}
+        attribute.
+
+    @param replacement: what should be used in place of the callable.
+        Either pass in a string, which will be inserted into the warning
+        message, or a callable, which will be expanded to its full import
+        path.
+    @type replacement: C{str} or callable
+
+    @return: A new property with deprecated setter and getter.
+    @rtype: C{property}
+
+    @since: 16.1.0
+    """
+
+    class _DeprecatedProperty(property):
+        """
+        Extension of the build-in property to allow deprecated setters.
+        """
+
+        def _deprecatedWrapper(self, function):
+            @wraps(function)
+            def deprecatedFunction(*args, **kwargs):
+                warn(
+                    self.warningString,  # type: ignore[attr-defined]
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+                return function(*args, **kwargs)
+
+            return deprecatedFunction
+
+        def setter(self, function):
+            return property.setter(self, self._deprecatedWrapper(function))
+
+    def deprecationDecorator(function):
+        warningString = getDeprecationWarningString(
+            function, version, None, replacement
+        )
+
+        @wraps(function)
+        def deprecatedFunction(*args, **kwargs):
+            warn(warningString, DeprecationWarning, stacklevel=2)
+            return function(*args, **kwargs)
+
+        _appendToDocstring(
+            deprecatedFunction, _getDeprecationDocstring(version, replacement)
+        )
+        deprecatedFunction.deprecatedVersion = version  # type: ignore[attr-defined]
+
+        result = _DeprecatedProperty(deprecatedFunction)
+        result.warningString = warningString  # type: ignore[attr-defined]
+        return result
+
+    return deprecationDecorator
+
+
+def getWarningMethod():
+    """
+    Return the warning method currently used to record deprecation warnings.
+    """
+    return warn
+
+
+def setWarningMethod(newMethod):
+    """
+    Set the warning method to use to record deprecation warnings.
+
+    The callable should take message, category and stacklevel. The return
+    value is ignored.
+    """
+    global warn
+    warn = newMethod
+
+
+class _InternalState:
+    """
+    An L{_InternalState} is a helper object for a L{_ModuleProxy}, so that it
+    can easily access its own attributes, bypassing its logic for delegating to
+    another object that it's proxying for.
+
+    @ivar proxy: a L{_ModuleProxy}
+    """
+
+    def __init__(self, proxy):
+        object.__setattr__(self, "proxy", proxy)
+
+    def __getattribute__(self, name):
+        return object.__getattribute__(object.__getattribute__(self, "proxy"), name)
+
+    def __setattr__(self, name, value):
+        return object.__setattr__(object.__getattribute__(self, "proxy"), name, value)
+
+
+class _ModuleProxy:
+    """
+    Python module wrapper to hook module-level attribute access.
+
+    Access to deprecated attributes first checks
+    L{_ModuleProxy._deprecatedAttributes}, if the attribute does not appear
+    there then access falls through to L{_ModuleProxy._module}, the wrapped
+    module object.
+
+    @ivar _module: Module on which to hook attribute access.
+    @type _module: C{module}
+
+    @ivar _deprecatedAttributes: Mapping of attribute names to objects that
+        retrieve the module attribute's original value.
+    @type _deprecatedAttributes: C{dict} mapping C{str} to
+        L{_DeprecatedAttribute}
+
+    @ivar _lastWasPath: Heuristic guess as to whether warnings about this
+        package should be ignored for the next call.  If the last attribute
+        access of this module was a C{getattr} of C{__path__}, we will assume
+        that it was the import system doing it and we won't emit a warning for
+        the next access, even if it is to a deprecated attribute.  The CPython
+        import system always tries to access C{__path__}, then the attribute
+        itself, then the attribute itself again, in both successful and failed
+        cases.
+    @type _lastWasPath: C{bool}
+    """
+
+    def __init__(self, module):
+        state = _InternalState(self)
+        state._module = module
+        state._deprecatedAttributes = {}
+        state._lastWasPath = False
+
+    def __repr__(self) -> str:
+        """
+        Get a string containing the type of the module proxy and a
+        representation of the wrapped module object.
+        """
+        state = _InternalState(self)
+        return f"<{type(self).__name__} module={state._module!r}>"
+
+    def __setattr__(self, name, value):
+        """
+        Set an attribute on the wrapped module object.
+        """
+        state = _InternalState(self)
+        state._lastWasPath = False
+        setattr(state._module, name, value)
+
+    def __getattribute__(self, name):
+        """
+        Get an attribute from the module object, possibly emitting a warning.
+
+        If the specified name has been deprecated, then a warning is issued.
+        (Unless certain obscure conditions are met; see
+        L{_ModuleProxy._lastWasPath} for more information about what might quash
+        such a warning.)
+        """
+        state = _InternalState(self)
+        if state._lastWasPath:
+            deprecatedAttribute = None
+        else:
+            deprecatedAttribute = state._deprecatedAttributes.get(name)
+
+        if deprecatedAttribute is not None:
+            # If we have a _DeprecatedAttribute object from the earlier lookup,
+            # allow it to issue the warning.
+            value = deprecatedAttribute.get()
+        else:
+            # Otherwise, just retrieve the underlying value directly; it's not
+            # deprecated, there's no warning to issue.
+            value = getattr(state._module, name)
+        if name == "__path__":
+            state._lastWasPath = True
+        else:
+            state._lastWasPath = False
+        return value
+
+
+class _DeprecatedAttribute:
+    """
+    Wrapper for deprecated attributes.
+
+    This is intended to be used by L{_ModuleProxy}. Calling
+    L{_DeprecatedAttribute.get} will issue a warning and retrieve the
+    underlying attribute's value.
+
+    @type module: C{module}
+    @ivar module: The original module instance containing this attribute
+
+    @type fqpn: C{str}
+    @ivar fqpn: Fully qualified Python name for the deprecated attribute
+
+    @type version: L{incremental.Version}
+    @ivar version: Version that the attribute was deprecated in
+
+    @type message: C{str}
+    @ivar message: Deprecation message
+    """
+
+    def __init__(self, module, name, version, message):
+        """
+        Initialise a deprecated name wrapper.
+        """
+        self.module = module
+        self.__name__ = name
+        self.fqpn = module.__name__ + "." + name
+        self.version = version
+        self.message = message
+
+    def get(self):
+        """
+        Get the underlying attribute value and issue a deprecation warning.
+        """
+        # This might fail if the deprecated thing is a module inside a package.
+        # In that case, don't emit the warning this time.  The import system
+        # will come back again when it's not an AttributeError and we can emit
+        # the warning then.
+        result = getattr(self.module, self.__name__)
+        message = _getDeprecationWarningString(
+            self.fqpn, self.version, DEPRECATION_WARNING_FORMAT + ": " + self.message
+        )
+        warn(message, DeprecationWarning, stacklevel=3)
+        return result
+
+
+def _deprecateAttribute(proxy, name, version, message):
+    """
+    Mark a module-level attribute as being deprecated.
+
+    @type proxy: L{_ModuleProxy}
+    @param proxy: The module proxy instance proxying the deprecated attributes
+
+    @type name: C{str}
+    @param name: Attribute name
+
+    @type version: L{incremental.Version}
+    @param version: Version that the attribute was deprecated in
+
+    @type message: C{str}
+    @param message: Deprecation message
+    """
+    _module = object.__getattribute__(proxy, "_module")
+    attr = _DeprecatedAttribute(_module, name, version, message)
+    # Add a deprecated attribute marker for this module's attribute. When this
+    # attribute is accessed via _ModuleProxy a warning is emitted.
+    _deprecatedAttributes = object.__getattribute__(proxy, "_deprecatedAttributes")
+    _deprecatedAttributes[name] = attr
+
+
+def deprecatedModuleAttribute(version, message, moduleName, name):
+    """
+    Declare a module-level attribute as being deprecated.
+
+    @type version: L{incremental.Version}
+    @param version: Version that the attribute was deprecated in
+
+    @type message: C{str}
+    @param message: Deprecation message
+
+    @type moduleName: C{str}
+    @param moduleName: Fully-qualified Python name of the module containing
+        the deprecated attribute; if called from the same module as the
+        attributes are being deprecated in, using the C{__name__} global can
+        be helpful
+
+    @type name: C{str}
+    @param name: Attribute name to deprecate
+    """
+    module = sys.modules[moduleName]
+    if not isinstance(module, _ModuleProxy):
+        module = cast(ModuleType, _ModuleProxy(module))
+        sys.modules[moduleName] = module
+
+    _deprecateAttribute(module, name, version, message)
+
+
+def warnAboutFunction(offender, warningString):
+    """
+    Issue a warning string, identifying C{offender} as the responsible code.
+
+    This function is used to deprecate some behavior of a function.  It differs
+    from L{warnings.warn} in that it is not limited to deprecating the behavior
+    of a function currently on the call stack.
+
+    @param offender: The function that is being deprecated.
+
+    @param warningString: The string that should be emitted by this warning.
+    @type warningString: C{str}
+
+    @since: 11.0
+    """
+    # inspect.getmodule() is attractive, but somewhat
+    # broken in Python < 2.6.  See Python bug 4845.
+    offenderModule = sys.modules[offender.__module__]
+    warn_explicit(
+        warningString,
+        category=DeprecationWarning,
+        filename=inspect.getabsfile(offenderModule),
+        lineno=max(lineNumber for _, lineNumber in findlinestarts(offender.__code__)),
+        module=offenderModule.__name__,
+        registry=offender.__globals__.setdefault("__warningregistry__", {}),
+        module_globals=None,
+    )
+
+
+def _passedArgSpec(argspec, positional, keyword):
+    """
+    Take an I{inspect.ArgSpec}, a tuple of positional arguments, and a dict of
+    keyword arguments, and return a mapping of arguments that were actually
+    passed to their passed values.
+
+    @param argspec: The argument specification for the function to inspect.
+    @type argspec: I{inspect.ArgSpec}
+
+    @param positional: The positional arguments that were passed.
+    @type positional: L{tuple}
+
+    @param keyword: The keyword arguments that were passed.
+    @type keyword: L{dict}
+
+    @return: A dictionary mapping argument names (those declared in C{argspec})
+        to values that were passed explicitly by the user.
+    @rtype: L{dict} mapping L{str} to L{object}
+    """
+    result: Dict[str, object] = {}
+    unpassed = len(argspec.args) - len(positional)
+    if argspec.keywords is not None:
+        kwargs = result[argspec.keywords] = {}
+    if unpassed < 0:
+        if argspec.varargs is None:
+            raise TypeError("Too many arguments.")
+        else:
+            result[argspec.varargs] = positional[len(argspec.args) :]
+    for name, value in zip(argspec.args, positional):
+        result[name] = value
+    for name, value in keyword.items():
+        if name in argspec.args:
+            if name in result:
+                raise TypeError("Already passed.")
+            result[name] = value
+        elif argspec.keywords is not None:
+            kwargs[name] = value
+        else:
+            raise TypeError("no such param")
+    return result
+
+
+def _passedSignature(signature, positional, keyword):
+    """
+    Take an L{inspect.Signature}, a tuple of positional arguments, and a dict of
+    keyword arguments, and return a mapping of arguments that were actually
+    passed to their passed values.
+
+    @param signature: The signature of the function to inspect.
+    @type signature: L{inspect.Signature}
+
+    @param positional: The positional arguments that were passed.
+    @type positional: L{tuple}
+
+    @param keyword: The keyword arguments that were passed.
+    @type keyword: L{dict}
+
+    @return: A dictionary mapping argument names (those declared in
+        C{signature}) to values that were passed explicitly by the user.
+    @rtype: L{dict} mapping L{str} to L{object}
+    """
+    result = {}
+    kwargs = None
+    numPositional = 0
+    for n, (name, param) in enumerate(signature.parameters.items()):
+        if param.kind == inspect.Parameter.VAR_POSITIONAL:
+            # Varargs, for example: *args
+            result[name] = positional[n:]
+            numPositional = len(result[name]) + 1
+        elif param.kind == inspect.Parameter.VAR_KEYWORD:
+            # Variable keyword args, for example: **my_kwargs
+            kwargs = result[name] = {}
+        elif param.kind in (
+            inspect.Parameter.POSITIONAL_OR_KEYWORD,
+            inspect.Parameter.POSITIONAL_ONLY,
+        ):
+            if n < len(positional):
+                result[name] = positional[n]
+                numPositional += 1
+        elif param.kind == inspect.Parameter.KEYWORD_ONLY:
+            if name not in keyword:
+                if param.default == inspect.Parameter.empty:
+                    raise TypeError(f"missing keyword arg {name}")
+                else:
+                    result[name] = param.default
+        else:
+            raise TypeError(f"'{name}' parameter is invalid kind: {param.kind}")
+
+    if len(positional) > numPositional:
+        raise TypeError("Too many arguments.")
+    for name, value in keyword.items():
+        if name in signature.parameters.keys():
+            if name in result:
+                raise TypeError("Already passed.")
+            result[name] = value
+        elif kwargs is not None:
+            kwargs[name] = value
+        else:
+            raise TypeError("no such param")
+    return result
+
+
+def _mutuallyExclusiveArguments(argumentPairs):
+    """
+    Decorator which causes its decoratee to raise a L{TypeError} if two of the
+    given arguments are passed at the same time.
+
+    @param argumentPairs: pairs of argument identifiers, each pair indicating
+        an argument that may not be passed in conjunction with another.
+    @type argumentPairs: sequence of 2-sequences of L{str}
+
+    @return: A decorator, used like so::
+
+            @_mutuallyExclusiveArguments([["tweedledum", "tweedledee"]])
+            def function(tweedledum=1, tweedledee=2):
+                "Don't pass tweedledum and tweedledee at the same time."
+
+    @rtype: 1-argument callable taking a callable and returning a callable.
+    """
+
+    def wrapper(wrappee):
+        spec = inspect.signature(wrappee)
+        _passed = _passedSignature
+
+        @wraps(wrappee)
+        def wrapped(*args, **kwargs):
+            arguments = _passed(spec, args, kwargs)
+            for this, that in argumentPairs:
+                if this in arguments and that in arguments:
+                    raise TypeError(
+                        ("The %r and %r arguments to %s " "are mutually exclusive.")
+                        % (this, that, _fullyQualifiedName(wrappee))
+                    )
+            return wrappee(*args, **kwargs)
+
+        return wrapped
+
+    return wrapper
+
+
+_Tc = TypeVar("_Tc", bound=Callable[..., Any])
+
+
+def deprecatedKeywordParameter(
+    version: Version, name: str, replacement: Optional[str] = None
+) -> Callable[[_Tc], _Tc]:
+    """
+    Return a decorator that marks a keyword parameter of a callable
+    as deprecated. A warning will be emitted if a caller supplies
+    a value for the parameter, whether the caller uses a keyword or
+    positional syntax.
+
+    @type version: L{incremental.Version}
+    @param version: The version in which the parameter will be marked as
+        having been deprecated.
+
+    @type name: L{str}
+    @param name: The name of the deprecated parameter.
+
+    @type replacement: L{str}
+    @param replacement: Optional text indicating what should be used in
+        place of the deprecated parameter.
+
+    @since: Twisted 21.2.0
+    """
+
+    def wrapper(wrappee: _Tc) -> _Tc:
+        warningString = _getDeprecationWarningString(
+            f"The {name!r} parameter to {_fullyQualifiedName(wrappee)}",
+            version,
+            replacement=replacement,
+        )
+
+        doc = "The {!r} parameter was deprecated in {}".format(
+            name,
+            getVersionString(version),
+        )
+        if replacement:
+            doc = doc + "; " + _getReplacementString(replacement)
+        doc += "."
+
+        params = inspect.signature(wrappee).parameters
+        if (
+            name in params
+            and params[name].kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
+        ):
+            parameterIndex = list(params).index(name)
+
+            def checkDeprecatedParameter(*args, **kwargs):
+                if len(args) > parameterIndex or name in kwargs:
+                    warn(warningString, DeprecationWarning, stacklevel=2)
+                return wrappee(*args, **kwargs)
+
+        else:
+
+            def checkDeprecatedParameter(*args, **kwargs):
+                if name in kwargs:
+                    warn(warningString, DeprecationWarning, stacklevel=2)
+                return wrappee(*args, **kwargs)
+
+        decorated = cast(_Tc, wraps(wrappee)(checkDeprecatedParameter))
+        _appendToDocstring(decorated, doc)
+        return decorated
+
+    return wrapper
diff --git a/contrib/python/Twisted/py3/twisted/python/failure.py b/contrib/python/Twisted/py3/twisted/python/failure.py
new file mode 100644
index 00000000000..ca893ca4c94
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/failure.py
@@ -0,0 +1,810 @@
+# -*- test-case-name: twisted.test.test_failure -*-
+# See also test suite twisted.test.test_pbfailure
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Asynchronous-friendly error mechanism.
+
+See L{Failure}.
+"""
+
+
+# System Imports
+import builtins
+import copy
+import inspect
+import linecache
+import sys
+from inspect import getmro
+from io import StringIO
+from typing import Callable, NoReturn, TypeVar
+
+import opcode
+
+from twisted.python import reflect
+
+_T_Callable = TypeVar("_T_Callable", bound=Callable[..., object])
+
+count = 0
+traceupLength = 4
+
+
+class DefaultException(Exception):
+    pass
+
+
+def format_frames(frames, write, detail="default"):
+    """
+    Format and write frames.
+
+    @param frames: is a list of frames as used by Failure.frames, with
+        each frame being a list of
+        (funcName, fileName, lineNumber, locals.items(), globals.items())
+    @type frames: list
+    @param write: this will be called with formatted strings.
+    @type write: callable
+    @param detail: Four detail levels are available:
+        default, brief, verbose, and verbose-vars-not-captured.
+        C{Failure.printDetailedTraceback} uses the latter when the caller asks
+        for verbose, but no vars were captured, so that an explicit warning
+        about the missing data is shown.
+    @type detail: string
+    """
+    if detail not in ("default", "brief", "verbose", "verbose-vars-not-captured"):
+        raise ValueError(
+            "Detail must be default, brief, verbose, or "
+            "verbose-vars-not-captured. (not %r)" % (detail,)
+        )
+    w = write
+    if detail == "brief":
+        for method, filename, lineno, localVars, globalVars in frames:
+            w(f"{filename}:{lineno}:{method}\n")
+    elif detail == "default":
+        for method, filename, lineno, localVars, globalVars in frames:
+            w(f'  File "{filename}", line {lineno}, in {method}\n')
+            w("    %s\n" % linecache.getline(filename, lineno).strip())
+    elif detail == "verbose-vars-not-captured":
+        for method, filename, lineno, localVars, globalVars in frames:
+            w("%s:%d: %s(...)\n" % (filename, lineno, method))
+        w(" [Capture of Locals and Globals disabled (use captureVars=True)]\n")
+    elif detail == "verbose":
+        for method, filename, lineno, localVars, globalVars in frames:
+            w("%s:%d: %s(...)\n" % (filename, lineno, method))
+            w(" [ Locals ]\n")
+            # Note: the repr(val) was (self.pickled and val) or repr(val)))
+            for name, val in localVars:
+                w(f"  {name} : {repr(val)}\n")
+            w(" ( Globals )\n")
+            for name, val in globalVars:
+                w(f"  {name} : {repr(val)}\n")
+
+
+# slyphon: i have a need to check for this value in trial
+#          so I made it a module-level constant
+EXCEPTION_CAUGHT_HERE = "--- <exception caught here> ---"
+
+
+class NoCurrentExceptionError(Exception):
+    """
+    Raised when trying to create a Failure from the current interpreter
+    exception state and there is no current exception state.
+    """
+
+
+def _Traceback(stackFrames, tbFrames):
+    """
+    Construct a fake traceback object using a list of frames.
+
+    It should have the same API as stdlib to allow interaction with
+    other tools.
+
+    @param stackFrames: [(methodname, filename, lineno, locals, globals), ...]
+    @param tbFrames: [(methodname, filename, lineno, locals, globals), ...]
+    """
+    assert len(tbFrames) > 0, "Must pass some frames"
+    # We deliberately avoid using recursion here, as the frames list may be
+    # long.
+
+    # 'stackFrames' is a list of frames above (ie, older than) the point the
+    # exception was caught, with oldest at the start. Start by building these
+    # into a linked list of _Frame objects (with the f_back links pointing back
+    # towards the oldest frame).
+    stack = None
+    for sf in stackFrames:
+        stack = _Frame(sf, stack)
+
+    # 'tbFrames' is a list of frames from the point the exception was caught,
+    # down to where it was thrown, with the oldest at the start. Add these to
+    # the linked list of _Frames, but also wrap each one with a _Traceback
+    # frame which is linked in the opposite direction (towards the newest
+    # frame).
+    stack = _Frame(tbFrames[0], stack)
+    firstTb = tb = _TracebackFrame(stack)
+    for sf in tbFrames[1:]:
+        stack = _Frame(sf, stack)
+        tb.tb_next = _TracebackFrame(stack)
+        tb = tb.tb_next
+
+    # Return the first _TracebackFrame.
+    return firstTb
+
+
+# The set of attributes for _TracebackFrame, _Frame and _Code were taken from
+# https://docs.python.org/3.11/library/inspect.html Other Pythons may have a
+# few more attributes that should be added if needed.
+class _TracebackFrame:
+    """
+    Fake traceback object which can be passed to functions in the standard
+    library L{traceback} module.
+    """
+
+    def __init__(self, frame):
+        """
+        @param frame: _Frame object
+        """
+        self.tb_frame = frame
+        self.tb_lineno = frame.f_lineno
+        self.tb_lasti = frame.f_lasti
+        self.tb_next = None
+
+
+class _Frame:
+    """
+    A fake frame object, used by L{_Traceback}.
+
+    @ivar f_code: fake L{code<types.CodeType>} object
+    @ivar f_lineno: line number
+    @ivar f_globals: fake f_globals dictionary (usually empty)
+    @ivar f_locals: fake f_locals dictionary (usually empty)
+    @ivar f_back: previous stack frame (towards the caller)
+    """
+
+    def __init__(self, frameinfo, back):
+        """
+        @param frameinfo: (methodname, filename, lineno, locals, globals)
+        @param back: previous (older) stack frame
+        @type back: C{frame}
+        """
+        name, filename, lineno, localz, globalz = frameinfo
+        self.f_code = _Code(name, filename)
+        self.f_lineno = lineno
+        self.f_globals = dict(globalz or {})
+        self.f_locals = dict(localz or {})
+        self.f_back = back
+        self.f_lasti = 0
+        self.f_builtins = vars(builtins).copy()
+        self.f_trace = None
+
+
+class _Code:
+    """
+    A fake code object, used by L{_Traceback} via L{_Frame}.
+
+    It is intended to have the same API as the stdlib code type to allow
+    interoperation with other tools based on that interface.
+    """
+
+    def __init__(self, name, filename):
+        self.co_name = name
+        self.co_filename = filename
+        self.co_lnotab = b""
+        self.co_firstlineno = 0
+        self.co_argcount = 0
+        self.co_varnames = []
+        self.co_code = b""
+        self.co_cellvars = ()
+        self.co_consts = ()
+        self.co_flags = 0
+        self.co_freevars = ()
+        self.co_posonlyargcount = 0
+        self.co_kwonlyargcount = 0
+        self.co_names = ()
+        self.co_nlocals = 0
+        self.co_stacksize = 0
+
+    def co_positions(self):
+        return ((None, None, None, None),)
+
+
+_inlineCallbacksExtraneous = []
+
+
+def _extraneous(f: _T_Callable) -> _T_Callable:
+    """
+    Mark the given callable as extraneous to inlineCallbacks exception
+    reporting; don't show these functions.
+
+    @param f: a function that you NEVER WANT TO SEE AGAIN in ANY TRACEBACK
+        reported by Failure.
+
+    @type f: function
+
+    @return: f
+    """
+    _inlineCallbacksExtraneous.append(f.__code__)
+    return f
+
+
+class Failure(BaseException):
+    """
+    A basic abstraction for an error that has occurred.
+
+    This is necessary because Python's built-in error mechanisms are
+    inconvenient for asynchronous communication.
+
+    The C{stack} and C{frame} attributes contain frames.  Each frame is a tuple
+    of (funcName, fileName, lineNumber, localsItems, globalsItems), where
+    localsItems and globalsItems are the contents of
+    C{locals().items()}/C{globals().items()} for that frame, or an empty tuple
+    if those details were not captured.
+
+    @ivar value: The exception instance responsible for this failure.
+    @ivar type: The exception's class.
+    @ivar stack: list of frames, innermost last, excluding C{Failure.__init__}.
+    @ivar frames: list of frames, innermost first.
+    """
+
+    pickled = 0
+    stack = None
+
+    # The opcode of "yield" in Python bytecode. We need this in
+    # _findFailure in order to identify whether an exception was
+    # thrown by a throwExceptionIntoGenerator.
+    # on PY3, b'a'[0] == 97 while in py2 b'a'[0] == b'a' opcodes
+    # are stored in bytes so we need to properly account for this
+    # difference.
+    _yieldOpcode = opcode.opmap["YIELD_VALUE"]
+
+    def __init__(self, exc_value=None, exc_type=None, exc_tb=None, captureVars=False):
+        """
+        Initialize me with an explanation of the error.
+
+        By default, this will use the current C{exception}
+        (L{sys.exc_info}()).  However, if you want to specify a
+        particular kind of failure, you can pass an exception as an
+        argument.
+
+        If no C{exc_value} is passed, then an "original" C{Failure} will
+        be searched for. If the current exception handler that this
+        C{Failure} is being constructed in is handling an exception
+        raised by L{raiseException}, then this C{Failure} will act like
+        the original C{Failure}.
+
+        For C{exc_tb} only L{traceback} instances or L{None} are allowed.
+        If L{None} is supplied for C{exc_value}, the value of C{exc_tb} is
+        ignored, otherwise if C{exc_tb} is L{None}, it will be found from
+        execution context (ie, L{sys.exc_info}).
+
+        @param captureVars: if set, capture locals and globals of stack
+            frames.  This is pretty slow, and makes no difference unless you
+            are going to use L{printDetailedTraceback}.
+        """
+        global count
+        count = count + 1
+        self.count = count
+        self.type = self.value = tb = None
+        self.captureVars = captureVars
+
+        if isinstance(exc_value, str) and exc_type is None:
+            raise TypeError("Strings are not supported by Failure")
+
+        stackOffset = 0
+
+        if exc_value is None:
+            exc_value = self._findFailure()
+
+        if exc_value is None:
+            self.type, self.value, tb = sys.exc_info()
+            if self.type is None:
+                raise NoCurrentExceptionError()
+            stackOffset = 1
+        elif exc_type is None:
+            if isinstance(exc_value, Exception):
+                self.type = exc_value.__class__
+            else:
+                # Allow arbitrary objects.
+                self.type = type(exc_value)
+            self.value = exc_value
+        else:
+            self.type = exc_type
+            self.value = exc_value
+
+        if isinstance(self.value, Failure):
+            self._extrapolate(self.value)
+            return
+
+        if hasattr(self.value, "__failure__"):
+            # For exceptions propagated through coroutine-awaiting (see
+            # Deferred.send, AKA Deferred.__next__), which can't be raised as
+            # Failure because that would mess up the ability to except: them:
+            self._extrapolate(self.value.__failure__)
+
+            # Clean up the inherently circular reference established by storing
+            # the failure there.  This should make the common case of a Twisted
+            # / Deferred-returning coroutine somewhat less hard on the garbage
+            # collector.
+            del self.value.__failure__
+            return
+
+        if tb is None:
+            if exc_tb:
+                tb = exc_tb
+            elif getattr(self.value, "__traceback__", None):
+                # Python 3
+                tb = self.value.__traceback__
+
+        frames = self.frames = []
+        stack = self.stack = []
+
+        # Added 2003-06-23 by Chris Armstrong. Yes, I actually have a
+        # use case where I need this traceback object, and I've made
+        # sure that it'll be cleaned up.
+        self.tb = tb
+
+        if tb:
+            f = tb.tb_frame
+        elif not isinstance(self.value, Failure):
+            # We don't do frame introspection since it's expensive,
+            # and if we were passed a plain exception with no
+            # traceback, it's not useful anyway
+            f = stackOffset = None
+
+        while stackOffset and f:
+            # This excludes this Failure.__init__ frame from the
+            # stack, leaving it to start with our caller instead.
+            f = f.f_back
+            stackOffset -= 1
+
+        # Keeps the *full* stack.  Formerly in spread.pb.print_excFullStack:
+        #
+        #   The need for this function arises from the fact that several
+        #   PB classes have the peculiar habit of discarding exceptions
+        #   with bareword "except:"s.  This premature exception
+        #   catching means tracebacks generated here don't tend to show
+        #   what called upon the PB object.
+
+        while f:
+            if captureVars:
+                localz = f.f_locals.copy()
+                if f.f_locals is f.f_globals:
+                    globalz = {}
+                else:
+                    globalz = f.f_globals.copy()
+                for d in globalz, localz:
+                    if "__builtins__" in d:
+                        del d["__builtins__"]
+                localz = localz.items()
+                globalz = globalz.items()
+            else:
+                localz = globalz = ()
+            stack.insert(
+                0,
+                (
+                    f.f_code.co_name,
+                    f.f_code.co_filename,
+                    f.f_lineno,
+                    localz,
+                    globalz,
+                ),
+            )
+            f = f.f_back
+
+        while tb is not None:
+            f = tb.tb_frame
+            if captureVars:
+                localz = f.f_locals.copy()
+                if f.f_locals is f.f_globals:
+                    globalz = {}
+                else:
+                    globalz = f.f_globals.copy()
+                for d in globalz, localz:
+                    if "__builtins__" in d:
+                        del d["__builtins__"]
+                localz = list(localz.items())
+                globalz = list(globalz.items())
+            else:
+                localz = globalz = ()
+            frames.append(
+                (
+                    f.f_code.co_name,
+                    f.f_code.co_filename,
+                    tb.tb_lineno,
+                    localz,
+                    globalz,
+                )
+            )
+            tb = tb.tb_next
+        if inspect.isclass(self.type) and issubclass(self.type, Exception):
+            parentCs = getmro(self.type)
+            self.parents = list(map(reflect.qual, parentCs))
+        else:
+            self.parents = [self.type]
+
+    def _extrapolate(self, otherFailure):
+        """
+        Extrapolate from one failure into another, copying its stack frames.
+
+        @param otherFailure: Another L{Failure}, whose traceback information,
+            if any, should be preserved as part of the stack presented by this
+            one.
+        @type otherFailure: L{Failure}
+        """
+        # Copy all infos from that failure (including self.frames).
+        self.__dict__ = copy.copy(otherFailure.__dict__)
+
+        # If we are re-throwing a Failure, we merge the stack-trace stored in
+        # the failure with the current exception's stack.  This integrated with
+        # throwExceptionIntoGenerator and allows to provide full stack trace,
+        # even if we go through several layers of inlineCallbacks.
+        _, _, tb = sys.exc_info()
+        frames = []
+        while tb is not None:
+            f = tb.tb_frame
+            if f.f_code not in _inlineCallbacksExtraneous:
+                frames.append(
+                    (f.f_code.co_name, f.f_code.co_filename, tb.tb_lineno, (), ())
+                )
+            tb = tb.tb_next
+        # Merging current stack with stack stored in the Failure.
+        frames.extend(self.frames)
+        self.frames = frames
+
+    def trap(self, *errorTypes):
+        """
+        Trap this failure if its type is in a predetermined list.
+
+        This allows you to trap a Failure in an error callback.  It will be
+        automatically re-raised if it is not a type that you expect.
+
+        The reason for having this particular API is because it's very useful
+        in Deferred errback chains::
+
+            def _ebFoo(self, failure):
+                r = failure.trap(Spam, Eggs)
+                print('The Failure is due to either Spam or Eggs!')
+                if r == Spam:
+                    print('Spam did it!')
+                elif r == Eggs:
+                    print('Eggs did it!')
+
+        If the failure is not a Spam or an Eggs, then the Failure will be
+        'passed on' to the next errback. In Python 2 the Failure will be
+        raised; in Python 3 the underlying exception will be re-raised.
+
+        @type errorTypes: L{Exception}
+        """
+        error = self.check(*errorTypes)
+        if not error:
+            self.raiseException()
+        return error
+
+    def check(self, *errorTypes):
+        """
+        Check if this failure's type is in a predetermined list.
+
+        @type errorTypes: list of L{Exception} classes or
+                          fully-qualified class names.
+        @returns: the matching L{Exception} type, or None if no match.
+        """
+        for error in errorTypes:
+            err = error
+            if inspect.isclass(error) and issubclass(error, Exception):
+                err = reflect.qual(error)
+            if err in self.parents:
+                return error
+        return None
+
+    def raiseException(self) -> NoReturn:
+        """
+        raise the original exception, preserving traceback
+        information if available.
+        """
+        raise self.value.with_traceback(self.tb)
+
+    @_extraneous
+    def throwExceptionIntoGenerator(self, g):
+        """
+        Throw the original exception into the given generator,
+        preserving traceback information if available.
+
+        @return: The next value yielded from the generator.
+        @raise StopIteration: If there are no more values in the generator.
+        @raise anything else: Anything that the generator raises.
+        """
+        # Note that the actual magic to find the traceback information
+        # is done in _findFailure.
+        return g.throw(self.type, self.value, self.tb)
+
+    @classmethod
+    def _findFailure(cls):
+        """
+        Find the failure that represents the exception currently in context.
+        """
+        tb = sys.exc_info()[-1]
+        if not tb:
+            return
+
+        secondLastTb = None
+        lastTb = tb
+        while lastTb.tb_next:
+            secondLastTb = lastTb
+            lastTb = lastTb.tb_next
+
+        lastFrame = lastTb.tb_frame
+
+        # NOTE: f_locals.get('self') is used rather than
+        # f_locals['self'] because psyco frames do not contain
+        # anything in their locals() dicts.  psyco makes debugging
+        # difficult anyhow, so losing the Failure objects (and thus
+        # the tracebacks) here when it is used is not that big a deal.
+
+        # Handle raiseException-originated exceptions
+        if lastFrame.f_code is cls.raiseException.__code__:
+            return lastFrame.f_locals.get("self")
+
+        # Handle throwExceptionIntoGenerator-originated exceptions
+        # this is tricky, and differs if the exception was caught
+        # inside the generator, or above it:
+
+        # It is only really originating from
+        # throwExceptionIntoGenerator if the bottom of the traceback
+        # is a yield.
+        # Pyrex and Cython extensions create traceback frames
+        # with no co_code, but they can't yield so we know it's okay to
+        # just return here.
+        if (not lastFrame.f_code.co_code) or lastFrame.f_code.co_code[
+            lastTb.tb_lasti
+        ] != cls._yieldOpcode:
+            return
+
+        # If the exception was caught above the generator.throw
+        # (outside the generator), it will appear in the tb (as the
+        # second last item):
+        if secondLastTb:
+            frame = secondLastTb.tb_frame
+            if frame.f_code is cls.throwExceptionIntoGenerator.__code__:
+                return frame.f_locals.get("self")
+
+        # If the exception was caught below the generator.throw
+        # (inside the generator), it will appear in the frames' linked
+        # list, above the top-level traceback item (which must be the
+        # generator frame itself, thus its caller is
+        # throwExceptionIntoGenerator).
+        frame = tb.tb_frame.f_back
+        if frame and frame.f_code is cls.throwExceptionIntoGenerator.__code__:
+            return frame.f_locals.get("self")
+
+    def __repr__(self) -> str:
+        return "<{} {}: {}>".format(
+            reflect.qual(self.__class__),
+            reflect.qual(self.type),
+            self.getErrorMessage(),
+        )
+
+    def __str__(self) -> str:
+        return "[Failure instance: %s]" % self.getBriefTraceback()
+
+    def __getstate__(self):
+        """Avoid pickling objects in the traceback."""
+        if self.pickled:
+            return self.__dict__
+        c = self.__dict__.copy()
+
+        c["frames"] = [
+            [
+                v[0],
+                v[1],
+                v[2],
+                _safeReprVars(v[3]),
+                _safeReprVars(v[4]),
+            ]
+            for v in self.frames
+        ]
+
+        # Added 2003-06-23. See comment above in __init__
+        c["tb"] = None
+
+        if self.stack is not None:
+            # XXX: This is a band-aid.  I can't figure out where these
+            # (failure.stack is None) instances are coming from.
+            c["stack"] = [
+                [
+                    v[0],
+                    v[1],
+                    v[2],
+                    _safeReprVars(v[3]),
+                    _safeReprVars(v[4]),
+                ]
+                for v in self.stack
+            ]
+
+        c["pickled"] = 1
+        return c
+
+    def cleanFailure(self):
+        """
+        Remove references to other objects, replacing them with strings.
+
+        On Python 3, this will also set the C{__traceback__} attribute of the
+        exception instance to L{None}.
+        """
+        self.__dict__ = self.__getstate__()
+        if getattr(self.value, "__traceback__", None):
+            # Python 3
+            self.value.__traceback__ = None
+
+    def getTracebackObject(self):
+        """
+        Get an object that represents this Failure's stack that can be passed
+        to traceback.extract_tb.
+
+        If the original traceback object is still present, return that. If this
+        traceback object has been lost but we still have the information,
+        return a fake traceback object (see L{_Traceback}). If there is no
+        traceback information at all, return None.
+        """
+        if self.tb is not None:
+            return self.tb
+        elif len(self.frames) > 0:
+            return _Traceback(self.stack, self.frames)
+        else:
+            return None
+
+    def getErrorMessage(self) -> str:
+        """
+        Get a string of the exception which caused this Failure.
+        """
+        if isinstance(self.value, Failure):
+            return self.value.getErrorMessage()
+        return reflect.safe_str(self.value)
+
+    def getBriefTraceback(self) -> str:
+        io = StringIO()
+        self.printBriefTraceback(file=io)
+        return io.getvalue()
+
+    def getTraceback(self, elideFrameworkCode: int = 0, detail: str = "default") -> str:
+        io = StringIO()
+        self.printTraceback(
+            file=io, elideFrameworkCode=elideFrameworkCode, detail=detail
+        )
+        return io.getvalue()
+
+    def printTraceback(self, file=None, elideFrameworkCode=False, detail="default"):
+        """
+        Emulate Python's standard error reporting mechanism.
+
+        @param file: If specified, a file-like object to which to write the
+            traceback.
+
+        @param elideFrameworkCode: A flag indicating whether to attempt to
+            remove uninteresting frames from within Twisted itself from the
+            output.
+
+        @param detail: A string indicating how much information to include
+            in the traceback.  Must be one of C{'brief'}, C{'default'}, or
+            C{'verbose'}.
+        """
+        if file is None:
+            from twisted.python import log
+
+            file = log.logerr
+        w = file.write
+
+        if detail == "verbose" and not self.captureVars:
+            # We don't have any locals or globals, so rather than show them as
+            # empty make the output explicitly say that we don't have them at
+            # all.
+            formatDetail = "verbose-vars-not-captured"
+        else:
+            formatDetail = detail
+
+        # Preamble
+        if detail == "verbose":
+            w(
+                "*--- Failure #%d%s---\n"
+                % (self.count, (self.pickled and " (pickled) ") or " ")
+            )
+        elif detail == "brief":
+            if self.frames:
+                hasFrames = "Traceback"
+            else:
+                hasFrames = "Traceback (failure with no frames)"
+            w(
+                "%s: %s: %s\n"
+                % (hasFrames, reflect.safe_str(self.type), reflect.safe_str(self.value))
+            )
+        else:
+            w("Traceback (most recent call last):\n")
+
+        # Frames, formatted in appropriate style
+        if self.frames:
+            if not elideFrameworkCode:
+                format_frames(self.stack[-traceupLength:], w, formatDetail)
+                w(f"{EXCEPTION_CAUGHT_HERE}\n")
+            format_frames(self.frames, w, formatDetail)
+        elif not detail == "brief":
+            # Yeah, it's not really a traceback, despite looking like one...
+            w("Failure: ")
+
+        # Postamble, if any
+        if not detail == "brief":
+            w(f"{reflect.qual(self.type)}: {reflect.safe_str(self.value)}\n")
+
+        # Chaining
+        if isinstance(self.value, Failure):
+            # TODO: indentation for chained failures?
+            file.write(" (chained Failure)\n")
+            self.value.printTraceback(file, elideFrameworkCode, detail)
+        if detail == "verbose":
+            w("*--- End of Failure #%d ---\n" % self.count)
+
+    def printBriefTraceback(self, file=None, elideFrameworkCode=0):
+        """
+        Print a traceback as densely as possible.
+        """
+        self.printTraceback(file, elideFrameworkCode, detail="brief")
+
+    def printDetailedTraceback(self, file=None, elideFrameworkCode=0):
+        """
+        Print a traceback with detailed locals and globals information.
+        """
+        self.printTraceback(file, elideFrameworkCode, detail="verbose")
+
+
+def _safeReprVars(varsDictItems):
+    """
+    Convert a list of (name, object) pairs into (name, repr) pairs.
+
+    L{twisted.python.reflect.safe_repr} is used to generate the repr, so no
+    exceptions will be raised by faulty C{__repr__} methods.
+
+    @param varsDictItems: a sequence of (name, value) pairs as returned by e.g.
+        C{locals().items()}.
+    @returns: a sequence of (name, repr) pairs.
+    """
+    return [(name, reflect.safe_repr(obj)) for (name, obj) in varsDictItems]
+
+
+# slyphon: make post-morteming exceptions tweakable
+
+DO_POST_MORTEM = True
+
+
+def _debuginit(
+    self,
+    exc_value=None,
+    exc_type=None,
+    exc_tb=None,
+    captureVars=False,
+    Failure__init__=Failure.__init__,
+):
+    """
+    Initialize failure object, possibly spawning pdb.
+    """
+    if (exc_value, exc_type, exc_tb) == (None, None, None):
+        exc = sys.exc_info()
+        if not exc[0] == self.__class__ and DO_POST_MORTEM:
+            try:
+                strrepr = str(exc[1])
+            except BaseException:
+                strrepr = "broken str"
+            print(
+                "Jumping into debugger for post-mortem of exception '{}':".format(
+                    strrepr
+                )
+            )
+            import pdb
+
+            pdb.post_mortem(exc[2])
+    Failure__init__(self, exc_value, exc_type, exc_tb, captureVars)
+
+
+def startDebugMode():
+    """
+    Enable debug hooks for Failures.
+    """
+    Failure.__init__ = _debuginit
diff --git a/contrib/python/Twisted/py3/twisted/python/fakepwd.py b/contrib/python/Twisted/py3/twisted/python/fakepwd.py
new file mode 100644
index 00000000000..39c11008214
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/fakepwd.py
@@ -0,0 +1,263 @@
+# -*- test-case-name: twisted.python.test.test_fakepwd -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+L{twisted.python.fakepwd} provides a fake implementation of the L{pwd} API.
+"""
+
+from typing import List, Optional
+
+__all__ = ["UserDatabase", "ShadowDatabase"]
+
+
+class _UserRecord:
+    """
+    L{_UserRecord} holds the user data for a single user in L{UserDatabase}.
+    It corresponds to the C{passwd} structure from the L{pwd} module.
+    See that module for attribute documentation.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        password: str,
+        uid: int,
+        gid: int,
+        gecos: str,
+        home: str,
+        shell: str,
+    ) -> None:
+        self.pw_name = name
+        self.pw_passwd = password
+        self.pw_uid = uid
+        self.pw_gid = gid
+        self.pw_gecos = gecos
+        self.pw_dir = home
+        self.pw_shell = shell
+
+    def __len__(self) -> int:
+        return 7
+
+    def __getitem__(self, index):
+        return (
+            self.pw_name,
+            self.pw_passwd,
+            self.pw_uid,
+            self.pw_gid,
+            self.pw_gecos,
+            self.pw_dir,
+            self.pw_shell,
+        )[index]
+
+
+class UserDatabase:
+    """
+    L{UserDatabase} holds a traditional POSIX user data in memory and makes it
+    available via the same API as L{pwd}.
+
+    @ivar _users: A C{list} of L{_UserRecord} instances holding all user data
+        added to this database.
+    """
+
+    _users: List[_UserRecord]
+    _lastUID: int = 10101
+    _lastGID: int = 20202
+
+    def __init__(self) -> None:
+        self._users = []
+
+    def addUser(
+        self,
+        username: str,
+        password: str = "password",
+        uid: Optional[int] = None,
+        gid: Optional[int] = None,
+        gecos: str = "",
+        home: str = "",
+        shell: str = "/bin/sh",
+    ) -> None:
+        """
+        Add a new user record to this database.
+
+        @param username: The value for the C{pw_name} field of the user
+            record to add.
+
+        @param password: The value for the C{pw_passwd} field of the user
+            record to add.
+
+        @param uid: The value for the C{pw_uid} field of the user record to
+            add.
+
+        @param gid: The value for the C{pw_gid} field of the user record to
+            add.
+
+        @param gecos: The value for the C{pw_gecos} field of the user record
+            to add.
+
+        @param home: The value for the C{pw_dir} field of the user record to
+            add.
+
+        @param shell: The value for the C{pw_shell} field of the user record to
+            add.
+        """
+        if uid is None:
+            uid = self._lastUID
+            self._lastUID += 1
+        if gid is None:
+            gid = self._lastGID
+            self._lastGID += 1
+        newUser = _UserRecord(username, password, uid, gid, gecos, home, shell)
+        self._users.append(newUser)
+
+    def getpwuid(self, uid: int) -> _UserRecord:
+        """
+        Return the user record corresponding to the given uid.
+        """
+        for entry in self._users:
+            if entry.pw_uid == uid:
+                return entry
+        raise KeyError()
+
+    def getpwnam(self, name: str) -> _UserRecord:
+        """
+        Return the user record corresponding to the given username.
+        """
+        if not isinstance(name, str):
+            raise TypeError(f"getpwuam() argument must be str, not {type(name)}")
+        for entry in self._users:
+            if entry.pw_name == name:
+                return entry
+        raise KeyError()
+
+    def getpwall(self) -> List[_UserRecord]:
+        """
+        Return a list of all user records.
+        """
+        return self._users
+
+
+class _ShadowRecord:
+    """
+    L{_ShadowRecord} holds the shadow user data for a single user in
+    L{ShadowDatabase}.  It corresponds to C{spwd.struct_spwd}.  See that class
+    for attribute documentation.
+    """
+
+    def __init__(
+        self,
+        username: str,
+        password: str,
+        lastChange: int,
+        min: int,
+        max: int,
+        warn: int,
+        inact: int,
+        expire: int,
+        flag: int,
+    ) -> None:
+        self.sp_nam = username
+        self.sp_pwd = password
+        self.sp_lstchg = lastChange
+        self.sp_min = min
+        self.sp_max = max
+        self.sp_warn = warn
+        self.sp_inact = inact
+        self.sp_expire = expire
+        self.sp_flag = flag
+
+    def __len__(self) -> int:
+        return 9
+
+    def __getitem__(self, index):
+        return (
+            self.sp_nam,
+            self.sp_pwd,
+            self.sp_lstchg,
+            self.sp_min,
+            self.sp_max,
+            self.sp_warn,
+            self.sp_inact,
+            self.sp_expire,
+            self.sp_flag,
+        )[index]
+
+
+class ShadowDatabase:
+    """
+    L{ShadowDatabase} holds a shadow user database in memory and makes it
+    available via the same API as C{spwd}.
+
+    @ivar _users: A C{list} of L{_ShadowRecord} instances holding all user data
+        added to this database.
+
+    @since: 12.0
+    """
+
+    _users: List[_ShadowRecord]
+
+    def __init__(self) -> None:
+        self._users = []
+
+    def addUser(
+        self,
+        username: str,
+        password: str,
+        lastChange: int,
+        min: int,
+        max: int,
+        warn: int,
+        inact: int,
+        expire: int,
+        flag: int,
+    ) -> None:
+        """
+        Add a new user record to this database.
+
+        @param username: The value for the C{sp_nam} field of the user record to
+            add.
+
+        @param password: The value for the C{sp_pwd} field of the user record to
+            add.
+
+        @param lastChange: The value for the C{sp_lstchg} field of the user
+            record to add.
+
+        @param min: The value for the C{sp_min} field of the user record to add.
+
+        @param max: The value for the C{sp_max} field of the user record to add.
+
+        @param warn: The value for the C{sp_warn} field of the user record to
+            add.
+
+        @param inact: The value for the C{sp_inact} field of the user record to
+            add.
+
+        @param expire: The value for the C{sp_expire} field of the user record
+            to add.
+
+        @param flag: The value for the C{sp_flag} field of the user record to
+            add.
+        """
+        self._users.append(
+            _ShadowRecord(
+                username, password, lastChange, min, max, warn, inact, expire, flag
+            )
+        )
+
+    def getspnam(self, username: str) -> _ShadowRecord:
+        """
+        Return the shadow user record corresponding to the given username.
+        """
+        if not isinstance(username, str):
+            raise TypeError(f"getspnam() argument must be str, not {type(username)}")
+        for entry in self._users:
+            if entry.sp_nam == username:
+                return entry
+        raise KeyError(username)
+
+    def getspall(self):
+        """
+        Return a list of all shadow user records.
+        """
+        return self._users
diff --git a/contrib/python/Twisted/py3/twisted/python/filepath.py b/contrib/python/Twisted/py3/twisted/python/filepath.py
new file mode 100644
index 00000000000..c5feb2f3f42
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/filepath.py
@@ -0,0 +1,1784 @@
+# -*- test-case-name: twisted.test.test_paths -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Object-oriented filesystem path representation.
+"""
+
+from __future__ import annotations
+
+import base64
+import errno
+import os
+import sys
+from os import listdir, stat, utime
+from os.path import (
+    abspath,
+    basename,
+    dirname,
+    exists,
+    isabs,
+    islink,
+    join as joinpath,
+    normpath,
+    splitext,
+)
+from stat import (
+    S_IMODE,
+    S_IRGRP,
+    S_IROTH,
+    S_IRUSR,
+    S_ISBLK,
+    S_ISDIR,
+    S_ISREG,
+    S_ISSOCK,
+    S_IWGRP,
+    S_IWOTH,
+    S_IWUSR,
+    S_IXGRP,
+    S_IXOTH,
+    S_IXUSR,
+)
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    AnyStr,
+    Callable,
+    Dict,
+    Generic,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+    overload,
+)
+
+from zope.interface import Attribute, Interface, implementer
+
+from typing_extensions import Literal
+
+from twisted.python.compat import cmp, comparable
+from twisted.python.runtime import platform
+from twisted.python.util import FancyEqMixin
+from twisted.python.win32 import (
+    ERROR_DIRECTORY,
+    ERROR_FILE_NOT_FOUND,
+    ERROR_INVALID_NAME,
+    ERROR_PATH_NOT_FOUND,
+    O_BINARY,
+)
+
+# Please keep this as light as possible on other Twisted imports; many, many
+# things import this module, and it would be good if it could easily be
+# modified for inclusion in the standard library.  --glyph
+
+
+_CREATE_FLAGS = os.O_EXCL | os.O_CREAT | os.O_RDWR | O_BINARY
+_Self = TypeVar("_Self", bound="AbstractFilePath[Any]")
+
+
+randomBytes = os.urandom
+armor = base64.urlsafe_b64encode
+
+
+class IFilePath(Interface):
+    """
+    File path object.
+
+    A file path represents a location for a file-like-object and can be
+    organized into a hierarchy; a file path can can children which are
+    themselves file paths.
+
+    A file path has a name which unique identifies it in the context of its
+    parent (if it has one); a file path can not have two children with the same
+    name.  This name is referred to as the file path's "base name".
+
+    A series of such names can be used to locate nested children of a file
+    path; such a series is referred to as the child's "path", relative to the
+    parent.  In this case, each name in the path is referred to as a "path
+    segment"; the child's base name is the segment in the path.
+
+    When representing a file path as a string, a "path separator" is used to
+    delimit the path segments within the string.  For a file system path, that
+    would be C{os.sep}.
+
+    Note that the values of child names may be restricted.  For example, a file
+    system path will not allow the use of the path separator in a name, and
+    certain names (e.g. C{"."} and C{".."}) may be reserved or have special
+    meanings.
+
+    @since: 12.1
+    """
+
+    sep = Attribute("The path separator to use in string representations")
+
+    def child(name: AnyStr) -> IFilePath:
+        """
+        Obtain a direct child of this file path.  The child may or may not
+        exist.
+
+        @param name: the name of a child of this path. C{name} must be a direct
+            child of this path and may not contain a path separator.
+        @return: the child of this path with the given C{name}.
+        @raise InsecurePath: if C{name} describes a file path that is not a
+            direct child of this file path.
+        """
+
+    def open(mode: FileMode = "r") -> IO[bytes]:
+        """
+        Opens this file path with the given mode.
+
+        @return: a file-like object.
+        @raise Exception: if this file path cannot be opened.
+        """
+
+    def changed() -> None:
+        """
+        Clear any cached information about the state of this path on disk.
+        """
+
+    def getsize() -> int:
+        """
+        Retrieve the size of this file in bytes.
+
+        @return: the size of the file at this file path in bytes.
+        @raise Exception: if the size cannot be obtained.
+        """
+
+    def getModificationTime() -> float:
+        """
+        Retrieve the time of last access from this file.
+
+        @return: a number of seconds from the epoch.
+        @rtype: L{float}
+        """
+
+    def getStatusChangeTime() -> float:
+        """
+        Retrieve the time of the last status change for this file.
+
+        @return: a number of seconds from the epoch.
+        @rtype: L{float}
+        """
+
+    def getAccessTime() -> float:
+        """
+        Retrieve the time that this file was last accessed.
+
+        @return: a number of seconds from the epoch.
+        @rtype: L{float}
+        """
+
+    def exists() -> bool:
+        """
+        Check if this file path exists.
+
+        @return: C{True} if the file at this file path exists, C{False}
+            otherwise.
+        @rtype: L{bool}
+        """
+
+    def isdir() -> bool:
+        """
+        Check if this file path refers to a directory.
+
+        @return: C{True} if the file at this file path is a directory, C{False}
+            otherwise.
+        """
+
+    def isfile() -> bool:
+        """
+        Check if this file path refers to a regular file.
+
+        @return: C{True} if the file at this file path is a regular file,
+            C{False} otherwise.
+        """
+
+    def children() -> Iterable[IFilePath]:
+        """
+        List the children of this path object.
+
+        @return: a sequence of the children of the directory at this file path.
+        @raise Exception: if the file at this file path is not a directory.
+        """
+
+    def basename() -> Union[str, bytes]:
+        """
+        Retrieve the final component of the file path's path (everything after
+        the final path separator).
+
+        @note: In implementors, the return type should be generic, i.e.
+            C{AbstractFilePath[str].basename()} is a C{str}.  However,
+            L{Interface} objects cannot be generic as of this writing.
+
+        @return: the base name of this file path.
+        """
+
+    def parent() -> IFilePath:
+        """
+        A file path for the directory containing the file at this file path.
+        """
+
+    def sibling(name: AnyStr) -> IFilePath:
+        """
+        A file path for the directory containing the file at this file path.
+
+        @param name: the name of a sibling of this path.  C{name} must be a
+            direct sibling of this path and may not contain a path separator.
+
+        @return: a sibling file path of this one.
+        """
+
+
+class InsecurePath(Exception):
+    """
+    Error that is raised when the path provided to L{FilePath} is invalid.
+    """
+
+
+class LinkError(Exception):
+    """
+    An error with symlinks - either that there are cyclical symlinks or that
+    symlink are not supported on this platform.
+    """
+
+
+class UnlistableError(OSError):
+    """
+    An exception which is used to distinguish between errors which mean 'this
+    is not a directory you can list' and other, more catastrophic errors.
+
+    This error will try to look as much like the original error as possible,
+    while still being catchable as an independent type.
+
+    @ivar originalException: the actual original exception instance.
+    """
+
+    def __init__(self, originalException: OSError):
+        """
+        Create an UnlistableError exception.
+
+        @param originalException: an instance of OSError.
+        """
+        self.__dict__.update(originalException.__dict__)
+        self.originalException = originalException
+
+
+def _secureEnoughString(path: AnyStr) -> AnyStr:
+    """
+    Compute a string usable as a new, temporary filename.
+
+    @param path: The path that the new temporary filename should be able to be
+        concatenated with.
+
+    @return: A pseudorandom, 16 byte string for use in secure filenames.
+    @rtype: the type of C{path}
+    """
+    secureishString = armor(randomBytes(16))[:16]
+    return _coerceToFilesystemEncoding(path, secureishString)
+
+
+OtherAnyStr = TypeVar("OtherAnyStr", str, bytes)
+FileMode = Literal["r", "w", "a", "r+", "w+", "a+"]
+
+
+class AbstractFilePath(Generic[AnyStr]):
+    """
+    Abstract implementation of an L{IFilePath}; must be completed by a
+    subclass.
+
+    This class primarily exists to provide common implementations of certain
+    methods in L{IFilePath}. It is *not* a required parent class for
+    L{IFilePath} implementations, just a useful starting point.
+
+    @ivar path: Subclasses must set this variable.
+    """
+
+    Selfish = TypeVar("Selfish", bound="AbstractFilePath[AnyStr]")
+
+    path: AnyStr
+
+    def getAccessTime(self) -> float:
+        """
+        Subclasses must implement this.
+
+        @see: L{FilePath.getAccessTime}
+        """
+        raise NotImplementedError()
+
+    def getModificationTime(self) -> float:
+        """
+        Subclasses must implement this.
+
+        @see: L{FilePath.getModificationTime}
+        """
+        raise NotImplementedError()
+
+    def getStatusChangeTime(self) -> float:
+        """
+        Subclasses must implement this.
+
+        @see: L{FilePath.getStatusChangeTime}
+        """
+        raise NotImplementedError()
+
+    def open(self, mode: FileMode = "r") -> IO[bytes]:
+        """
+        Subclasses must implement this.
+        """
+        raise NotImplementedError()
+
+    def isdir(self) -> bool:
+        """
+        Subclasses must implement this.
+        """
+        raise NotImplementedError()
+
+    def basename(self) -> AnyStr:
+        """
+        Subclasses must implement this.
+        """
+        raise NotImplementedError()
+
+    def parent(self) -> AbstractFilePath[AnyStr]:
+        """
+        Subclasses must implement this.
+        """
+        raise NotImplementedError()
+
+    def listdir(self) -> List[AnyStr]:
+        """
+        Subclasses must implement this.
+        """
+        raise NotImplementedError()
+
+    def child(self, path: OtherAnyStr) -> AbstractFilePath[OtherAnyStr]:
+        """
+        Subclasses must implement this.
+        """
+        raise NotImplementedError()
+
+    def getContent(self) -> bytes:
+        """
+        Retrieve the contents of the file at this path.
+
+        @return: the contents of the file
+        @rtype: L{bytes}
+        """
+        with self.open() as fp:
+            return fp.read()
+
+    def parents(self) -> Iterable[AbstractFilePath[AnyStr]]:
+        """
+        Retrieve an iterator of all the ancestors of this path.
+
+        @return: an iterator of all the ancestors of this path, from the most
+        recent (its immediate parent) to the root of its filesystem.
+        """
+        path = self
+        parent = path.parent()
+        # root.parent() == root, so this means "are we the root"
+        while path != parent:
+            yield parent
+            path = parent
+            parent = parent.parent()
+
+    def children(self: _Self) -> Iterable[_Self]:
+        """
+        List the children of this path object.
+
+        @raise OSError: If an error occurs while listing the directory.  If the
+        error is 'serious', meaning that the operation failed due to an access
+        violation, exhaustion of some kind of resource (file descriptors or
+        memory), OSError or a platform-specific variant will be raised.
+
+        @raise UnlistableError: If the inability to list the directory is due
+        to this path not existing or not being a directory, the more specific
+        OSError subclass L{UnlistableError} is raised instead.
+
+        @return: an iterable of all currently-existing children of this object.
+        """
+        try:
+            subnames: List[AnyStr] = self.listdir()
+        except OSError as ose:
+            # Under Python 3.3 and higher on Windows, WindowsError is an
+            # alias for OSError.  OSError has a winerror attribute and an
+            # errno attribute.
+            #
+            # The winerror attribute is bound to the Windows error code while
+            # the errno attribute is bound to a translation of that code to a
+            # perhaps equivalent POSIX error number.
+            #
+            # For further details, refer to:
+            # https://docs.python.org/3/library/exceptions.html#OSError
+            if getattr(ose, "winerror", None) in (
+                ERROR_PATH_NOT_FOUND,
+                ERROR_FILE_NOT_FOUND,
+                ERROR_INVALID_NAME,
+                ERROR_DIRECTORY,
+            ):
+                raise UnlistableError(ose)
+            if ose.errno in (errno.ENOENT, errno.ENOTDIR):
+                raise UnlistableError(ose)
+            # Other possible errors here, according to linux manpages:
+            # EACCES, EMIFLE, ENFILE, ENOMEM.  None of these seem like the
+            # sort of thing which should be handled normally. -glyph
+            raise
+        result = []
+        for name in subnames:
+            # It's not possible to tell mypy that child/clone etc must be
+            # overridden to return respecializable forms of _Self, but they
+            # must, so we will say that they are.
+            child: _Self = self.child(name)  # type:ignore[assignment]
+            result.append(child)
+        return result
+
+    def walk(
+        self: _Self,
+        descend: Optional[Callable[[_Self], bool]] = None,
+    ) -> Iterable[_Self]:
+        """
+        Yield myself, then each of my children, and each of those children's
+        children in turn.
+
+        The optional argument C{descend} is a predicate that takes a FilePath,
+        and determines whether or not that FilePath is traversed/descended
+        into.  It will be called with each path for which C{isdir} returns
+        C{True}.  If C{descend} is not specified, all directories will be
+        traversed (including symbolic links which refer to directories).
+
+        @param descend: A one-argument callable that will return True for
+            FilePaths that should be traversed, False otherwise.
+
+        @return: a generator yielding FilePath-like objects.
+        """
+        yield self
+        if self.isdir():
+            for c in self.children():
+                # we should first see if it's what we want, then we
+                # can walk through the directory
+                if descend is None or descend(c):
+                    for subc in c.walk(descend):
+                        if os.path.realpath(self.path).startswith(
+                            os.path.realpath(subc.path)
+                        ):
+                            raise LinkError("Cycle in file graph.")
+                        yield subc
+                else:
+                    yield c
+
+    def sibling(self: _Self, path: OtherAnyStr) -> AbstractFilePath[OtherAnyStr]:
+        """
+        Return a L{FilePath} with the same directory as this instance but with
+        a basename of C{path}.
+
+        @note: for type-checking, subclasses should override this signature to
+            make it clear that it returns the subclass and not
+            L{AbstractFilePath}.
+
+        @param path: The basename of the L{FilePath} to return.
+        @type path: L{str}
+
+        @return: The sibling path.
+        @rtype: L{FilePath}
+        """
+        return self.parent().child(path)
+
+    def descendant(
+        self, segments: Sequence[OtherAnyStr]
+    ) -> AbstractFilePath[OtherAnyStr]:
+        """
+        Retrieve a child or child's child of this path.
+
+        @note: for type-checking, subclasses should override this signature to
+            make it clear that it returns the subclass and not
+            L{AbstractFilePath}.
+
+        @param segments: A sequence of path segments as L{str} instances.
+
+        @return: A L{FilePath} constructed by looking up the C{segments[0]}
+            child of this path, the C{segments[1]} child of that path, and so
+            on.
+
+        @since: 10.2
+        """
+        path: AbstractFilePath[OtherAnyStr] = self  # type:ignore[assignment]
+        for name in segments:
+            path = path.child(name)
+        return path
+
+    def segmentsFrom(self: _Self, ancestor: _Self) -> List[AnyStr]:
+        """
+        Return a list of segments between a child and its ancestor.
+
+        For example, in the case of a path X representing /a/b/c/d and a path Y
+        representing /a/b, C{Y.segmentsFrom(X)} will return C{['c',
+        'd']}.
+
+        @param ancestor: an instance of the same class as self, ostensibly an
+        ancestor of self.
+
+        @raise ValueError: If the C{ancestor} parameter is not actually an
+        ancestor, i.e. a path for /x/y/z is passed as an ancestor for /a/b/c/d.
+
+        @return: a list of strs
+        """
+        # this might be an unnecessarily inefficient implementation but it will
+        # work on win32 and for zipfiles; later I will deterimine if the
+        # obvious fast implemenation does the right thing too
+        f = self
+        p: _Self = f.parent()  # type:ignore[assignment]
+        segments: List[AnyStr] = []
+        while f != ancestor and p != f:
+            segments[0:0] = [f.basename()]
+            f = p
+            p = p.parent()  # type:ignore[assignment]
+        if f == ancestor and segments:
+            return segments
+        raise ValueError(f"{ancestor!r} not parent of {self!r}")
+
+    # new in 8.0
+    def __hash__(self) -> int:
+        """
+        Hash the same as another L{AbstractFilePath} with the same path as mine.
+        """
+        return hash((self.__class__, self.path))
+
+    # pending deprecation in 8.0
+    def getmtime(self) -> int:
+        """
+        Deprecated.  Use getModificationTime instead.
+        """
+        return int(self.getModificationTime())
+
+    def getatime(self) -> int:
+        """
+        Deprecated.  Use getAccessTime instead.
+        """
+        return int(self.getAccessTime())
+
+    def getctime(self) -> int:
+        """
+        Deprecated.  Use getStatusChangeTime instead.
+        """
+        return int(self.getStatusChangeTime())
+
+
+class RWX(FancyEqMixin):
+    """
+    A class representing read/write/execute permissions for a single user
+    category (i.e. user/owner, group, or other/world).  Instantiate with
+    three boolean values: readable? writable? executable?.
+
+    @type read: C{bool}
+    @ivar read: Whether permission to read is given
+
+    @type write: C{bool}
+    @ivar write: Whether permission to write is given
+
+    @type execute: C{bool}
+    @ivar execute: Whether permission to execute is given
+
+    @since: 11.1
+    """
+
+    compareAttributes = ("read", "write", "execute")
+
+    def __init__(self, readable: bool, writable: bool, executable: bool) -> None:
+        self.read = readable
+        self.write = writable
+        self.execute = executable
+
+    def __repr__(self) -> str:
+        return "RWX(read={}, write={}, execute={})".format(
+            self.read,
+            self.write,
+            self.execute,
+        )
+
+    def shorthand(self) -> str:
+        """
+        Returns a short string representing the permission bits.  Looks like
+        part of what is printed by command line utilities such as 'ls -l'
+        (e.g. 'rwx')
+
+        @return: The shorthand string.
+        @rtype: L{str}
+        """
+        returnval = ["r", "w", "x"]
+        i = 0
+        for val in (self.read, self.write, self.execute):
+            if not val:
+                returnval[i] = "-"
+            i += 1
+        return "".join(returnval)
+
+
+class Permissions(FancyEqMixin):
+    """
+    A class representing read/write/execute permissions.  Instantiate with any
+    portion of the file's mode that includes the permission bits.
+
+    @type user: L{RWX}
+    @ivar user: User/Owner permissions
+
+    @type group: L{RWX}
+    @ivar group: Group permissions
+
+    @type other: L{RWX}
+    @ivar other: Other/World permissions
+
+    @since: 11.1
+    """
+
+    compareAttributes = ("user", "group", "other")
+
+    def __init__(self, statModeInt: int) -> None:
+        self.user, self.group, self.other = (
+            RWX(*(statModeInt & bit > 0 for bit in bitGroup))
+            for bitGroup in [
+                [S_IRUSR, S_IWUSR, S_IXUSR],
+                [S_IRGRP, S_IWGRP, S_IXGRP],
+                [S_IROTH, S_IWOTH, S_IXOTH],
+            ]
+        )
+
+    def __repr__(self) -> str:
+        return f"[{str(self.user)} | {str(self.group)} | {str(self.other)}]"
+
+    def shorthand(self) -> str:
+        """
+        Returns a short string representing the permission bits.  Looks like
+        what is printed by command line utilities such as 'ls -l'
+        (e.g. 'rwx-wx--x')
+
+        @return: The shorthand string.
+        @rtype: L{str}
+        """
+        return "".join([x.shorthand() for x in (self.user, self.group, self.other)])
+
+
+def _asFilesystemBytes(path: Union[bytes, str], encoding: Optional[str] = "") -> bytes:
+    """
+    Return C{path} as a string of L{bytes} suitable for use on this system's
+    filesystem.
+
+    @param path: The path to be made suitable.
+    @type path: L{bytes} or L{unicode}
+    @param encoding: The encoding to use if coercing to L{bytes}. If none is
+        given, L{sys.getfilesystemencoding} is used.
+
+    @return: L{bytes}
+    """
+    if isinstance(path, bytes):
+        return path
+    else:
+        if not encoding:
+            encoding = sys.getfilesystemencoding()
+        return path.encode(encoding, errors="surrogateescape")
+
+
+def _asFilesystemText(path: Union[bytes, str], encoding: Optional[str] = None) -> str:
+    """
+    Return C{path} as a string of L{unicode} suitable for use on this system's
+    filesystem.
+
+    @param path: The path to be made suitable.
+    @type path: L{bytes} or L{unicode}
+
+    @param encoding: The encoding to use if coercing to L{unicode}. If none
+        is given, L{sys.getfilesystemencoding} is used.
+
+    @return: L{unicode}
+    """
+    if isinstance(path, str):
+        return path
+    else:
+        if encoding is None:
+            encoding = sys.getfilesystemencoding()
+        return path.decode(encoding, errors="surrogateescape")
+
+
+def _coerceToFilesystemEncoding(
+    path: AnyStr, newpath: Union[bytes, str], encoding: Optional[str] = None
+) -> AnyStr:
+    """
+    Return a C{newpath} that is suitable for joining to C{path}.
+
+    @param path: The path that it should be suitable for joining to.
+    @param newpath: The new portion of the path to be coerced if needed.
+    @param encoding: If coerced, the encoding that will be used.
+    """
+    if isinstance(path, bytes):
+        return _asFilesystemBytes(newpath, encoding=encoding)
+    else:
+        return _asFilesystemText(newpath, encoding=encoding)
+
+
+@comparable
+@implementer(IFilePath)
+class FilePath(AbstractFilePath[AnyStr]):
+    """
+    I am a path on the filesystem that only permits 'downwards' access.
+
+    Instantiate me with a pathname (for example,
+    FilePath('/home/myuser/public_html')) and I will attempt to only provide
+    access to files which reside inside that path.  I may be a path to a file,
+    a directory, or a file which does not exist.
+
+    The correct way to use me is to instantiate me, and then do ALL filesystem
+    access through me.  In other words, do not import the 'os' module; if you
+    need to open a file, call my 'open' method.  If you need to list a
+    directory, call my 'path' method.
+
+    Even if you pass me a relative path, I will convert that to an absolute
+    path internally.
+
+    The type of C{path} when instantiating decides the mode of the L{FilePath}.
+    That is, C{FilePath(b"/")} will return a L{bytes} mode L{FilePath}, and
+    C{FilePath(u"/")} will return a L{unicode} mode L{FilePath}.
+    C{FilePath("/")} will return a L{bytes} mode L{FilePath} on Python 2, and a
+    L{unicode} mode L{FilePath} on Python 3.
+
+    Methods that return a new L{FilePath} use the type of the given subpath to
+    decide its mode. For example, C{FilePath(b"/").child(u"tmp")} will return a
+    L{unicode} mode L{FilePath}.
+
+    @type alwaysCreate: L{bool}
+    @ivar alwaysCreate: When opening this file, only succeed if the file does
+        not already exist.
+
+    @ivar path: The path from which 'downward' traversal is permitted.
+    """
+
+    _statinfo = None
+    path: AnyStr
+
+    def __init__(self, path: AnyStr, alwaysCreate: bool = False) -> None:
+        """
+        Convert a path string to an absolute path if necessary and initialize
+        the L{FilePath} with the result.
+        """
+        self.path = abspath(path)
+        self.alwaysCreate = alwaysCreate
+
+    if TYPE_CHECKING:
+
+        def sibling(self: _Self, path: OtherAnyStr) -> FilePath[OtherAnyStr]:
+            ...
+
+        def descendant(self, segments: Sequence[OtherAnyStr]) -> FilePath[OtherAnyStr]:
+            ...
+
+        def parents(self) -> Iterable[FilePath[AnyStr]]:
+            ...
+
+        # provided by @comparable
+        def __gt__(self, other: object) -> bool:
+            ...
+
+        def __ge__(self, other: object) -> bool:
+            ...
+
+        def __lt__(self, other: object) -> bool:
+            ...
+
+        def __le__(self, other: object) -> bool:
+            ...
+
+        def __eq__(self, other: object) -> bool:
+            ...
+
+        def __ne__(self, other: object) -> bool:
+            ...
+
+    def clonePath(
+        self, path: OtherAnyStr, alwaysCreate: bool = False
+    ) -> FilePath[OtherAnyStr]:
+        """
+        Make an object of the same type as this FilePath, but with path of
+        C{path}.
+        """
+        return FilePath(path)
+
+    def __getstate__(self) -> Dict[str, object]:
+        """
+        Support serialization by discarding cached L{os.stat} results and
+        returning everything else.
+        """
+        d = self.__dict__.copy()
+        if "_statinfo" in d:
+            del d["_statinfo"]
+        return d
+
+    @property
+    def sep(self) -> AnyStr:
+        """
+        Return a filesystem separator.
+
+        @return: The native filesystem separator.
+        @returntype: The same type as C{self.path}.
+        """
+        return _coerceToFilesystemEncoding(self.path, os.sep)
+
+    def _asBytesPath(self, encoding: Optional[str] = None) -> bytes:
+        """
+        Return the path of this L{FilePath} as bytes.
+
+        @param encoding: The encoding to use if coercing to L{bytes}. If none is
+            given, L{sys.getfilesystemencoding} is used.
+
+        @return: L{bytes}
+        """
+        return _asFilesystemBytes(self.path, encoding=encoding)
+
+    def _asTextPath(self, encoding: Optional[str] = None) -> str:
+        """
+        Return the path of this L{FilePath} as text.
+
+        @param encoding: The encoding to use if coercing to L{unicode}. If none
+            is given, L{sys.getfilesystemencoding} is used.
+
+        @return: L{unicode}
+        """
+        return _asFilesystemText(self.path, encoding=encoding)
+
+    def asBytesMode(self, encoding: Optional[str] = None) -> FilePath[bytes]:
+        """
+        Return this L{FilePath} in L{bytes}-mode.
+
+        @param encoding: The encoding to use if coercing to L{bytes}. If none is
+            given, L{sys.getfilesystemencoding} is used.
+
+        @return: L{bytes} mode L{FilePath}
+        """
+        if isinstance(self.path, str):
+            return self.clonePath(self._asBytesPath(encoding=encoding))
+        return self
+
+    def asTextMode(self, encoding: Optional[str] = None) -> FilePath[str]:
+        """
+        Return this L{FilePath} in L{unicode}-mode.
+
+        @param encoding: The encoding to use if coercing to L{unicode}. If none
+            is given, L{sys.getfilesystemencoding} is used.
+
+        @return: L{unicode} mode L{FilePath}
+        """
+        if isinstance(self.path, bytes):
+            return self.clonePath(self._asTextPath(encoding=encoding))
+        return self
+
+    def _getPathAsSameTypeAs(self, pattern: OtherAnyStr) -> OtherAnyStr:
+        """
+        If C{pattern} is C{bytes}, return L{FilePath.path} as L{bytes}.
+        Otherwise, return L{FilePath.path} as L{unicode}.
+
+        @param pattern: The new element of the path that L{FilePath.path} may
+            need to be coerced to match.
+        """
+        if isinstance(pattern, bytes):
+            return self._asBytesPath()
+        else:
+            return self._asTextPath()
+
+    def child(self, path: OtherAnyStr) -> FilePath[OtherAnyStr]:
+        """
+        Create and return a new L{FilePath} representing a path contained by
+        C{self}.
+
+        @param path: The base name of the new L{FilePath}.  If this contains
+            directory separators or parent references it will be rejected.
+        @type path: L{bytes} or L{unicode}
+
+        @raise InsecurePath: If the result of combining this path with C{path}
+            would result in a path which is not a direct child of this path.
+
+        @return: The child path.
+        @rtype: L{FilePath} with a mode equal to the type of C{path}.
+        """
+        colon = _coerceToFilesystemEncoding(path, ":")
+        sep = _coerceToFilesystemEncoding(path, os.sep)
+        ourPath = self._getPathAsSameTypeAs(path)
+
+        if platform.isWindows() and path.count(colon):
+            # Catch paths like C:blah that don't have a slash
+            raise InsecurePath(f"{path!r} contains a colon.")
+
+        norm = normpath(path)
+        if sep in norm:
+            raise InsecurePath(f"{path!r} contains one or more directory separators")
+
+        newpath = abspath(joinpath(ourPath, norm))
+        if not newpath.startswith(ourPath):
+            raise InsecurePath(f"{newpath!r} is not a child of {ourPath!r}")
+        return self.clonePath(newpath)
+
+    def preauthChild(self, path: OtherAnyStr) -> FilePath[OtherAnyStr]:
+        """
+        Use me if C{path} might have slashes in it, but you know they're safe.
+
+        @param path: A relative path (ie, a path not starting with C{"/"})
+            which will be interpreted as a child or descendant of this path.
+        @type path: L{bytes} or L{unicode}
+
+        @return: The child path.
+        @rtype: L{FilePath} with a mode equal to the type of C{path}.
+        """
+        ourPath = self._getPathAsSameTypeAs(path)
+
+        newpath = abspath(joinpath(ourPath, normpath(path)))
+        if not newpath.startswith(ourPath):
+            raise InsecurePath(f"{newpath!r} is not a child of {ourPath!r}")
+        return self.clonePath(newpath)
+
+    def childSearchPreauth(
+        self, *paths: OtherAnyStr
+    ) -> Optional[FilePath[OtherAnyStr]]:
+        """
+        Return my first existing child with a name in C{paths}.
+
+        C{paths} is expected to be a list of *pre-secured* path fragments;
+        in most cases this will be specified by a system administrator and not
+        an arbitrary user.
+
+        If no appropriately-named children exist, this will return L{None}.
+
+        @return: L{None} or the child path.
+        @rtype: L{None} or L{FilePath}
+        """
+        for child in paths:
+            p = self._getPathAsSameTypeAs(child)
+            jp = joinpath(p, child)
+            if exists(jp):
+                return self.clonePath(jp)
+        return None
+
+    def siblingExtensionSearch(
+        self, *exts: OtherAnyStr
+    ) -> Optional[FilePath[OtherAnyStr]]:
+        """
+        Attempt to return a path with my name, given multiple possible
+        extensions.
+
+        Each extension in C{exts} will be tested and the first path which
+        exists will be returned.  If no path exists, L{None} will be returned.
+        If C{''} is in C{exts}, then if the file referred to by this path
+        exists, C{self} will be returned.
+
+        The extension '*' has a magic meaning, which means "any path that
+        begins with C{self.path + '.'} is acceptable".
+        """
+        for ext in exts:
+            if not ext and self.exists():
+                return self.clonePath(self._getPathAsSameTypeAs(ext))
+
+            p = self._getPathAsSameTypeAs(ext)
+            star = _coerceToFilesystemEncoding(ext, "*")
+            dot = _coerceToFilesystemEncoding(ext, ".")
+
+            if ext == star:
+                basedot = basename(p) + dot
+                for fn in listdir(dirname(p)):
+                    if fn.startswith(basedot):
+                        return self.clonePath(joinpath(dirname(p), fn))
+            p2 = p + ext
+            if exists(p2):
+                return self.clonePath(p2)
+        return None
+
+    def realpath(self) -> FilePath[AnyStr]:
+        """
+        Returns the absolute target as a L{FilePath} if self is a link, self
+        otherwise.
+
+        The absolute link is the ultimate file or directory the
+        link refers to (for instance, if the link refers to another link, and
+        another...).  If the filesystem does not support symlinks, or
+        if the link is cyclical, raises a L{LinkError}.
+
+        Behaves like L{os.path.realpath} in that it does not resolve link
+        names in the middle (ex. /x/y/z, y is a link to w - realpath on z
+        will return /x/y/z, not /x/w/z).
+
+        @return: L{FilePath} of the target path.
+        @rtype: L{FilePath}
+        @raises LinkError: if links are not supported or links are cyclical.
+        """
+        if self.islink():
+            result = os.path.realpath(self.path)
+            if result == self.path:
+                raise LinkError("Cyclical link - will loop forever")
+            return self.clonePath(result)
+        return self
+
+    def siblingExtension(self, ext: OtherAnyStr) -> FilePath[OtherAnyStr]:
+        """
+        Attempt to return a path with my name, given the extension at C{ext}.
+
+        @param ext: File-extension to search for.
+        @type ext: L{bytes} or L{unicode}
+
+        @return: The sibling path.
+        @rtype: L{FilePath} with the same mode as the type of C{ext}.
+        """
+        ourPath = self._getPathAsSameTypeAs(ext)
+        return self.clonePath(ourPath + ext)
+
+    def linkTo(self, linkFilePath: FilePath[AnyStr]) -> None:
+        """
+        Creates a symlink to self to at the path in the L{FilePath}
+        C{linkFilePath}.
+
+        Only works on posix systems due to its dependence on
+        L{os.symlink}.  Propagates L{OSError}s up from L{os.symlink} if
+        C{linkFilePath.parent()} does not exist, or C{linkFilePath} already
+        exists.
+
+        @param linkFilePath: a FilePath representing the link to be created.
+        @type linkFilePath: L{FilePath}
+        """
+        os.symlink(self.path, linkFilePath.path)
+
+    def open(self, mode: FileMode = "r") -> IO[bytes]:
+        """
+        Open this file using C{mode} or for writing if C{alwaysCreate} is
+        C{True}.
+
+        In all cases the file is opened in binary mode, so it is not necessary
+        to include C{"b"} in C{mode}.
+
+        @param mode: The mode to open the file in.  Default is C{"r"}.
+        @raises AssertionError: If C{"a"} is included in the mode and
+            C{alwaysCreate} is C{True}.
+        @return: An open file-like object.
+        """
+        if self.alwaysCreate:
+            assert "a" not in mode, (
+                "Appending not supported when " "alwaysCreate == True"
+            )
+            return self.create()
+        # Make sure we open with exactly one "b" in the mode.
+        return open(self.path, mode.replace("b", "") + "b")
+
+    # stat methods below
+
+    def restat(self, reraise: bool = True) -> None:
+        """
+        Re-calculate cached effects of 'stat'.  To refresh information on this
+        path after you know the filesystem may have changed, call this method.
+
+        @param reraise: a boolean.  If true, re-raise exceptions from
+            L{os.stat}; otherwise, mark this path as not existing, and remove
+            any cached stat information.
+
+        @raise Exception: If C{reraise} is C{True} and an exception occurs
+            while reloading metadata.
+        """
+        try:
+            self._statinfo = stat(self.path)
+        except OSError:
+            self._statinfo = None
+            if reraise:
+                raise
+
+    def changed(self) -> None:
+        """
+        Clear any cached information about the state of this path on disk.
+
+        @since: 10.1.0
+        """
+        self._statinfo = None
+
+    def chmod(self, mode: int) -> None:
+        """
+        Changes the permissions on self, if possible.  Propagates errors from
+        L{os.chmod} up.
+
+        @param mode: integer representing the new permissions desired (same as
+            the command line chmod)
+        @type mode: L{int}
+        """
+        os.chmod(self.path, mode)
+
+    def getsize(self) -> int:
+        """
+        Retrieve the size of this file in bytes.
+
+        @return: The size of the file at this file path in bytes.
+        @raise Exception: if the size cannot be obtained.
+        @rtype: L{int}
+        """
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return st.st_size
+
+    def getModificationTime(self) -> float:
+        """
+        Retrieve the time of last access from this file.
+
+        @return: a number of seconds from the epoch.
+        @rtype: L{float}
+        """
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return float(st.st_mtime)
+
+    def getStatusChangeTime(self) -> float:
+        """
+        Retrieve the time of the last status change for this file.
+
+        @return: a number of seconds from the epoch.
+        @rtype: L{float}
+        """
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return float(st.st_ctime)
+
+    def getAccessTime(self) -> float:
+        """
+        Retrieve the time that this file was last accessed.
+
+        @return: a number of seconds from the epoch.
+        @rtype: L{float}
+        """
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return float(st.st_atime)
+
+    def getInodeNumber(self) -> int:
+        """
+        Retrieve the file serial number, also called inode number, which
+        distinguishes this file from all other files on the same device.
+
+        @raise NotImplementedError: if the platform is Windows, since the
+            inode number would be a dummy value for all files in Windows
+        @return: a number representing the file serial number
+        @rtype: L{int}
+        @since: 11.0
+        """
+        if platform.isWindows():
+            raise NotImplementedError
+
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return st.st_ino
+
+    def getDevice(self) -> int:
+        """
+        Retrieves the device containing the file.  The inode number and device
+        number together uniquely identify the file, but the device number is
+        not necessarily consistent across reboots or system crashes.
+
+        @raise NotImplementedError: if the platform is Windows, since the
+            device number would be 0 for all partitions on a Windows platform
+
+        @return: a number representing the device
+        @rtype: L{int}
+
+        @since: 11.0
+        """
+        if platform.isWindows():
+            raise NotImplementedError
+
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return st.st_dev
+
+    def getNumberOfHardLinks(self) -> int:
+        """
+        Retrieves the number of hard links to the file.
+
+        This count keeps track of how many directories have entries for this
+        file. If the count is ever decremented to zero then the file itself is
+        discarded as soon as no process still holds it open.  Symbolic links
+        are not counted in the total.
+
+        @raise NotImplementedError: if the platform is Windows, since Windows
+            doesn't maintain a link count for directories, and L{os.stat} does
+            not set C{st_nlink} on Windows anyway.
+        @return: the number of hard links to the file
+        @rtype: L{int}
+        @since: 11.0
+        """
+        if platform.isWindows():
+            raise NotImplementedError
+
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return st.st_nlink
+
+    def getUserID(self) -> int:
+        """
+        Returns the user ID of the file's owner.
+
+        @raise NotImplementedError: if the platform is Windows, since the UID
+            is always 0 on Windows
+        @return: the user ID of the file's owner
+        @rtype: L{int}
+        @since: 11.0
+        """
+        if platform.isWindows():
+            raise NotImplementedError
+
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return st.st_uid
+
+    def getGroupID(self) -> int:
+        """
+        Returns the group ID of the file.
+
+        @raise NotImplementedError: if the platform is Windows, since the GID
+            is always 0 on windows
+        @return: the group ID of the file
+        @rtype: L{int}
+        @since: 11.0
+        """
+        if platform.isWindows():
+            raise NotImplementedError
+
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return st.st_gid
+
+    def getPermissions(self) -> Permissions:
+        """
+        Returns the permissions of the file.  Should also work on Windows,
+        however, those permissions may not be what is expected in Windows.
+
+        @return: the permissions for the file
+        @rtype: L{Permissions}
+        @since: 11.1
+        """
+        st = self._statinfo
+        if not st:
+            self.restat()
+            st = self._statinfo
+        assert st is not None
+        return Permissions(S_IMODE(st.st_mode))
+
+    def exists(self) -> bool:
+        """
+        Check if this L{FilePath} exists.
+
+        @return: C{True} if the stats of C{path} can be retrieved successfully,
+            C{False} in the other cases.
+        @rtype: L{bool}
+        """
+        if self._statinfo:
+            return True
+        else:
+            self.restat(False)
+            if self._statinfo:
+                return True
+            else:
+                return False
+
+    def isdir(self) -> bool:
+        """
+        Check if this L{FilePath} refers to a directory.
+
+        @return: C{True} if this L{FilePath} refers to a directory, C{False}
+            otherwise.
+        @rtype: L{bool}
+        """
+        st = self._statinfo
+        if not st:
+            self.restat(False)
+            st = self._statinfo
+            if not st:
+                return False
+        return S_ISDIR(st.st_mode)
+
+    def isfile(self) -> bool:
+        """
+        Check if this file path refers to a regular file.
+
+        @return: C{True} if this L{FilePath} points to a regular file (not a
+            directory, socket, named pipe, etc), C{False} otherwise.
+        @rtype: L{bool}
+        """
+        st = self._statinfo
+        if not st:
+            self.restat(False)
+            st = self._statinfo
+            if not st:
+                return False
+        return S_ISREG(st.st_mode)
+
+    def isBlockDevice(self) -> bool:
+        """
+        Returns whether the underlying path is a block device.
+
+        @return: C{True} if it is a block device, C{False} otherwise
+        @rtype: L{bool}
+        @since: 11.1
+        """
+        st = self._statinfo
+        if not st:
+            self.restat(False)
+            st = self._statinfo
+            if not st:
+                return False
+        return S_ISBLK(st.st_mode)
+
+    def isSocket(self) -> bool:
+        """
+        Returns whether the underlying path is a socket.
+
+        @return: C{True} if it is a socket, C{False} otherwise
+        @rtype: L{bool}
+        @since: 11.1
+        """
+        st = self._statinfo
+        if not st:
+            self.restat(False)
+            st = self._statinfo
+            if not st:
+                return False
+        return S_ISSOCK(st.st_mode)
+
+    def islink(self) -> bool:
+        """
+        Check if this L{FilePath} points to a symbolic link.
+
+        @return: C{True} if this L{FilePath} points to a symbolic link,
+            C{False} otherwise.
+        @rtype: L{bool}
+        """
+        # We can't use cached stat results here, because that is the stat of
+        # the destination - (see #1773) which in *every case* but this one is
+        # the right thing to use.  We could call lstat here and use that, but
+        # it seems unlikely we'd actually save any work that way.  -glyph
+        return islink(self.path)
+
+    def isabs(self) -> bool:
+        """
+        Check if this L{FilePath} refers to an absolute path.
+
+        This always returns C{True}.
+
+        @return: C{True}, always.
+        @rtype: L{bool}
+        """
+        return isabs(self.path)
+
+    def listdir(self) -> List[AnyStr]:
+        """
+        List the base names of the direct children of this L{FilePath}.
+
+        @return: A L{list} of L{bytes}/L{unicode} giving the names of the
+            contents of the directory this L{FilePath} refers to. These names
+            are relative to this L{FilePath}.
+        @rtype: L{list}
+
+        @raise OSError: Any exception the platform L{os.listdir} implementation
+            may raise.
+        """
+        return listdir(self.path)
+
+    def splitext(self) -> Tuple[AnyStr, AnyStr]:
+        """
+        Split the file path into a pair C{(root, ext)} such that
+        C{root + ext == path}.
+
+        @return: Tuple where the first item is the filename and second item is
+            the file extension. See Python docs for L{os.path.splitext}.
+        @rtype: L{tuple}
+        """
+        return splitext(self.path)
+
+    def __repr__(self) -> str:
+        return f"FilePath({self.path!r})"
+
+    def touch(self) -> None:
+        """
+        Updates the access and last modification times of the file at this
+        file path to the current time. Also creates the file if it does not
+        already exist.
+
+        @raise Exception: if unable to create or modify the last modification
+            time of the file.
+        """
+        try:
+            self.open("a").close()
+        except OSError:
+            pass
+        utime(self.path, None)
+
+    def remove(self) -> None:
+        """
+        Removes the file or directory that is represented by self.  If
+        C{self.path} is a directory, recursively remove all its children
+        before removing the directory. If it's a file or link, just delete it.
+        """
+        if self.isdir() and not self.islink():
+            for child in self.children():
+                child.remove()
+            os.rmdir(self.path)
+        else:
+            os.remove(self.path)
+        self.changed()
+
+    def makedirs(self, ignoreExistingDirectory: bool = False) -> None:
+        """
+        Create all directories not yet existing in C{path} segments, using
+        L{os.makedirs}.
+
+        @param ignoreExistingDirectory: Don't raise L{OSError} if directory
+            already exists.
+        @type ignoreExistingDirectory: L{bool}
+
+        @return: L{None}
+        """
+        try:
+            os.makedirs(self.path)
+        except OSError as e:
+            if not (
+                e.errno == errno.EEXIST and ignoreExistingDirectory and self.isdir()
+            ):
+                raise
+
+    def globChildren(self, pattern: OtherAnyStr) -> List[FilePath[OtherAnyStr]]:
+        """
+        Assuming I am representing a directory, return a list of FilePaths
+        representing my children that match the given pattern.
+
+        @param pattern: A glob pattern to use to match child paths.
+        @type pattern: L{unicode} or L{bytes}
+
+        @return: A L{list} of matching children.
+        @rtype: L{list} of L{FilePath}, with the mode of C{pattern}'s type
+        """
+        sep = _coerceToFilesystemEncoding(pattern, os.sep)
+        ourPath = self._getPathAsSameTypeAs(pattern)
+
+        import glob
+
+        path = ourPath[-1] == sep and ourPath + pattern or sep.join([ourPath, pattern])
+        return [self.clonePath(p) for p in glob.glob(path)]
+
+    def basename(self) -> AnyStr:
+        """
+        Retrieve the final component of the file path's path (everything
+        after the final path separator).
+
+        @return: The final component of the L{FilePath}'s path (Everything
+            after the final path separator).
+        @rtype: the same type as this L{FilePath}'s C{path} attribute
+        """
+        return basename(self.path)
+
+    def dirname(self) -> AnyStr:
+        """
+        Retrieve all of the components of the L{FilePath}'s path except the
+        last one (everything up to the final path separator).
+
+        @return: All of the components of the L{FilePath}'s path except the
+            last one (everything up to the final path separator).
+        @rtype: the same type as this L{FilePath}'s C{path} attribute
+        """
+        return dirname(self.path)
+
+    def parent(self) -> FilePath[AnyStr]:
+        """
+        A file path for the directory containing the file at this file path.
+
+        @return: A L{FilePath} representing the path which directly contains
+            this L{FilePath}.
+        @rtype: L{FilePath}
+        """
+        return self.clonePath(self.dirname())
+
+    def setContent(self, content: bytes, ext: Union[str, bytes] = ".new") -> None:
+        """
+        Replace the file at this path with a new file that contains the given
+        bytes, trying to avoid data-loss in the meanwhile.
+
+        On UNIX-like platforms, this method does its best to ensure that by the
+        time this method returns, either the old contents I{or} the new
+        contents of the file will be present at this path for subsequent
+        readers regardless of premature device removal, program crash, or power
+        loss, making the following assumptions:
+
+            - your filesystem is journaled (i.e. your filesystem will not
+              I{itself} lose data due to power loss)
+
+            - your filesystem's C{rename()} is atomic
+
+            - your filesystem will not discard new data while preserving new
+              metadata (see U{http://mjg59.livejournal.com/108257.html} for
+              more detail)
+
+        On most versions of Windows there is no atomic C{rename()} (see
+        U{http://bit.ly/win32-overwrite} for more information), so this method
+        is slightly less helpful.  There is a small window where the file at
+        this path may be deleted before the new file is moved to replace it:
+        however, the new file will be fully written and flushed beforehand so
+        in the unlikely event that there is a crash at that point, it should be
+        possible for the user to manually recover the new version of their
+        data.  In the future, Twisted will support atomic file moves on those
+        versions of Windows which I{do} support them: see U{Twisted ticket
+        3004<http://twistedmatrix.com/trac/ticket/3004>}.
+
+        This method should be safe for use by multiple concurrent processes,
+        but note that it is not easy to predict which process's contents will
+        ultimately end up on disk if they invoke this method at close to the
+        same time.
+
+        @param content: The desired contents of the file at this path.
+        @type content: L{bytes}
+
+        @param ext: An extension to append to the temporary filename used to
+            store the bytes while they are being written.  This can be used to
+            make sure that temporary files can be identified by their suffix,
+            for cleanup in case of crashes.
+        @type ext: L{bytes}
+        """
+        sib = self.temporarySibling(ext)
+        with sib.open("w") as f:
+            f.write(content)
+        if platform.isWindows() and exists(self.path):
+            os.unlink(self.path)
+        os.rename(sib.path, self.asBytesMode().path)
+
+    def __cmp__(self, other: object) -> int:
+        if not isinstance(other, FilePath):
+            return NotImplemented
+        return cmp(self.path, other.path)
+
+    def createDirectory(self) -> None:
+        """
+        Create the directory the L{FilePath} refers to.
+
+        @see: L{makedirs}
+
+        @raise OSError: If the directory cannot be created.
+        """
+        os.mkdir(self.path)
+
+    def requireCreate(self, val: bool = True) -> None:
+        """
+        Sets the C{alwaysCreate} variable.
+
+        @param val: C{True} or C{False}, indicating whether opening this path
+            will be required to create the file or not.
+        @type val: L{bool}
+
+        @return: L{None}
+        """
+        self.alwaysCreate = val
+
+    def create(self) -> IO[bytes]:
+        """
+        Exclusively create a file, only if this file previously did not exist.
+
+        @return: A file-like object opened from this path.
+        """
+        fdint = os.open(self.path, _CREATE_FLAGS)
+
+        # XXX TODO: 'name' attribute of returned files is not mutable or
+        # settable via fdopen, so this file is slightly less functional than the
+        # one returned from 'open' by default.  send a patch to Python...
+
+        return cast(IO[bytes], os.fdopen(fdint, "w+b"))
+
+    @overload
+    def temporarySibling(self) -> FilePath[AnyStr]:
+        ...
+
+    @overload
+    def temporarySibling(
+        self, extension: Optional[OtherAnyStr]
+    ) -> FilePath[OtherAnyStr]:
+        ...
+
+    def temporarySibling(
+        self, extension: Optional[OtherAnyStr] = None
+    ) -> FilePath[OtherAnyStr]:
+        """
+        Construct a path referring to a sibling of this path.
+
+        The resulting path will be unpredictable, so that other subprocesses
+        should neither accidentally attempt to refer to the same path before it
+        is created, nor they should other processes be able to guess its name
+        in advance.
+
+        @param extension: A suffix to append to the created filename.  (Note
+            that if you want an extension with a '.' you must include the '.'
+            yourself.)
+        @type extension: L{bytes} or L{unicode}
+
+        @return: a path object with the given extension suffix, C{alwaysCreate}
+            set to True.
+        @rtype: L{FilePath} with a mode equal to the type of C{extension}
+        """
+        ext: OtherAnyStr
+        if extension is None:
+            # It's not possible to provide a default type argument which is why
+            # the overload is required.
+            ext = self.path[0:0]  # type:ignore[assignment]
+        else:
+            ext = extension
+        ourPath = self._getPathAsSameTypeAs(ext)
+        sib = self.sibling(
+            _secureEnoughString(ourPath) + self.clonePath(ourPath).basename() + ext
+        )
+        sib.requireCreate()
+        return sib
+
+    _chunkSize = 2**2**2**2
+
+    def copyTo(
+        self, destination: FilePath[OtherAnyStr], followLinks: bool = True
+    ) -> None:
+        """
+        Copies self to destination.
+
+        If self doesn't exist, an OSError is raised.
+
+        If self is a directory, this method copies its children (but not
+        itself) recursively to destination - if destination does not exist as a
+        directory, this method creates it.  If destination is a file, an
+        IOError will be raised.
+
+        If self is a file, this method copies it to destination.  If
+        destination is a file, this method overwrites it.  If destination is a
+        directory, an IOError will be raised.
+
+        If self is a link (and followLinks is False), self will be copied
+        over as a new symlink with the same target as returned by os.readlink.
+        That means that if it is absolute, both the old and new symlink will
+        link to the same thing.  If it's relative, then perhaps not (and
+        it's also possible that this relative link will be broken).
+
+        File/directory permissions and ownership will NOT be copied over.
+
+        If followLinks is True, symlinks are followed so that they're treated
+        as their targets.  In other words, if self is a link, the link's target
+        will be copied.  If destination is a link, self will be copied to the
+        destination's target (the actual destination will be destination's
+        target).  Symlinks under self (if self is a directory) will be
+        followed and its target's children be copied recursively.
+
+        If followLinks is False, symlinks will be copied over as symlinks.
+
+        @param destination: the destination (a FilePath) to which self
+            should be copied
+        @param followLinks: whether symlinks in self should be treated as links
+            or as their targets
+        """
+        if self.islink() and not followLinks:
+            os.symlink(os.readlink(self.path), destination.path)
+            return
+        # XXX TODO: *thorough* audit and documentation of the exact desired
+        # semantics of this code.  Right now the behavior of existent
+        # destination symlinks is convenient, and quite possibly correct, but
+        # its security properties need to be explained.
+        if self.isdir():
+            if not destination.exists():
+                destination.createDirectory()
+            for child in self.children():
+                destChild = destination.child(child.basename())
+                child.copyTo(destChild, followLinks)
+        elif self.isfile():
+            with destination.open("w") as writefile, self.open() as readfile:
+                while 1:
+                    # XXX TODO: optionally use os.open, os.read and
+                    # O_DIRECT and use os.fstatvfs to determine chunk sizes
+                    # and make *****sure**** copy is page-atomic; the
+                    # following is good enough for 99.9% of everybody and
+                    # won't take a week to audit though.
+                    chunk = readfile.read(self._chunkSize)
+                    writefile.write(chunk)
+                    if len(chunk) < self._chunkSize:
+                        break
+        elif not self.exists():
+            raise OSError(errno.ENOENT, "No such file or directory")
+        else:
+            # If you see the following message because you want to copy
+            # symlinks, fifos, block devices, character devices, or unix
+            # sockets, please feel free to add support to do sensible things in
+            # reaction to those types!
+            raise NotImplementedError("Only copying of files and directories supported")
+
+    def moveTo(
+        self, destination: FilePath[OtherAnyStr], followLinks: bool = True
+    ) -> None:
+        """
+        Move self to destination - basically renaming self to whatever
+        destination is named.
+
+        If destination is an already-existing directory,
+        moves all children to destination if destination is empty.  If
+        destination is a non-empty directory, or destination is a file, an
+        OSError will be raised.
+
+        If moving between filesystems, self needs to be copied, and everything
+        that applies to copyTo applies to moveTo.
+
+        @param destination: the destination (a FilePath) to which self
+            should be copied
+        @param followLinks: whether symlinks in self should be treated as links
+            or as their targets (only applicable when moving between
+            filesystems)
+        """
+        try:
+            os.rename(self._getPathAsSameTypeAs(destination.path), destination.path)
+        except OSError as ose:
+            if ose.errno == errno.EXDEV:
+                # man 2 rename, ubuntu linux 5.10 "breezy":
+
+                #   oldpath and newpath are not on the same mounted filesystem.
+                #   (Linux permits a filesystem to be mounted at multiple
+                #   points, but rename(2) does not work across different mount
+                #   points, even if the same filesystem is mounted on both.)
+
+                # that means it's time to copy trees of directories!
+                secsib = destination.temporarySibling()
+                self.copyTo(secsib, followLinks)  # slow
+                secsib.moveTo(destination, followLinks)  # visible
+
+                # done creating new stuff.  let's clean me up.
+                mysecsib = self.temporarySibling()
+                self.moveTo(mysecsib, followLinks)  # visible
+                mysecsib.remove()  # slow
+            else:
+                raise
+        else:
+            self.changed()
+            destination.changed()
diff --git a/contrib/python/Twisted/py3/twisted/python/formmethod.py b/contrib/python/Twisted/py3/twisted/python/formmethod.py
new file mode 100644
index 00000000000..aa5fb97a2d3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/formmethod.py
@@ -0,0 +1,446 @@
+# -*- test-case-name: twisted.test.test_formmethod -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Form-based method objects.
+
+This module contains support for descriptive method signatures that can be used
+to format methods.
+"""
+
+import calendar
+from typing import Any, Optional, Tuple
+
+
+class FormException(Exception):
+    """An error occurred calling the form method."""
+
+    def __init__(self, *args, **kwargs):
+        Exception.__init__(self, *args)
+        self.descriptions = kwargs
+
+
+class InputError(FormException):
+    """
+    An error occurred with some input.
+    """
+
+
+class Argument:
+    """Base class for form arguments."""
+
+    # default value for argument, if no other default is given
+    defaultDefault: Any = None
+
+    def __init__(
+        self, name, default=None, shortDesc=None, longDesc=None, hints=None, allowNone=1
+    ):
+        self.name = name
+        self.allowNone = allowNone
+        if default is None:
+            default = self.defaultDefault
+        self.default = default
+        self.shortDesc = shortDesc
+        self.longDesc = longDesc
+        if not hints:
+            hints = {}
+        self.hints = hints
+
+    def addHints(self, **kwargs):
+        self.hints.update(kwargs)
+
+    def getHint(self, name, default=None):
+        return self.hints.get(name, default)
+
+    def getShortDescription(self):
+        return self.shortDesc or self.name.capitalize()
+
+    def getLongDescription(self):
+        return self.longDesc or ""  # self.shortDesc or "The %s." % self.name
+
+    def coerce(self, val):
+        """Convert the value to the correct format."""
+        raise NotImplementedError("implement in subclass")
+
+
+class String(Argument):
+    """A single string."""
+
+    defaultDefault: str = ""
+    min = 0
+    max = None
+
+    def __init__(
+        self,
+        name,
+        default=None,
+        shortDesc=None,
+        longDesc=None,
+        hints=None,
+        allowNone=1,
+        min=0,
+        max=None,
+    ):
+        Argument.__init__(
+            self,
+            name,
+            default=default,
+            shortDesc=shortDesc,
+            longDesc=longDesc,
+            hints=hints,
+            allowNone=allowNone,
+        )
+        self.min = min
+        self.max = max
+
+    def coerce(self, val):
+        s = str(val)
+        if len(s) < self.min:
+            raise InputError("Value must be at least %s characters long" % self.min)
+        if self.max is not None and len(s) > self.max:
+            raise InputError("Value must be at most %s characters long" % self.max)
+        return str(val)
+
+
+class Text(String):
+    """A long string."""
+
+
+class Password(String):
+    """A string which should be obscured when input."""
+
+
+class VerifiedPassword(String):
+    """A string that should be obscured when input and needs verification."""
+
+    def coerce(self, vals):
+        if len(vals) != 2 or vals[0] != vals[1]:
+            raise InputError("Please enter the same password twice.")
+        s = str(vals[0])
+        if len(s) < self.min:
+            raise InputError("Value must be at least %s characters long" % self.min)
+        if self.max is not None and len(s) > self.max:
+            raise InputError("Value must be at most %s characters long" % self.max)
+        return s
+
+
+class Hidden(String):
+    """A string which is not displayed.
+
+    The passed default is used as the value.
+    """
+
+
+class Integer(Argument):
+    """A single integer."""
+
+    defaultDefault: Optional[int] = None
+
+    def __init__(
+        self, name, allowNone=1, default=None, shortDesc=None, longDesc=None, hints=None
+    ):
+        # although Argument now has allowNone, that was recently added, and
+        # putting it at the end kept things which relied on argument order
+        # from breaking.  However, allowNone originally was in here, so
+        # I have to keep the same order, to prevent breaking code that
+        # depends on argument order only
+        Argument.__init__(self, name, default, shortDesc, longDesc, hints, allowNone)
+
+    def coerce(self, val):
+        if not val.strip() and self.allowNone:
+            return None
+        try:
+            return int(val)
+        except ValueError:
+            raise InputError(
+                "{} is not valid, please enter " "a whole number, e.g. 10".format(val)
+            )
+
+
+class IntegerRange(Integer):
+    def __init__(
+        self,
+        name,
+        min,
+        max,
+        allowNone=1,
+        default=None,
+        shortDesc=None,
+        longDesc=None,
+        hints=None,
+    ):
+        self.min = min
+        self.max = max
+        Integer.__init__(
+            self,
+            name,
+            allowNone=allowNone,
+            default=default,
+            shortDesc=shortDesc,
+            longDesc=longDesc,
+            hints=hints,
+        )
+
+    def coerce(self, val):
+        result = Integer.coerce(self, val)
+        if self.allowNone and result == None:
+            return result
+        if result < self.min:
+            raise InputError(
+                "Value {} is too small, it should be at least {}".format(
+                    result, self.min
+                )
+            )
+        if result > self.max:
+            raise InputError(
+                "Value {} is too large, it should be at most {}".format(
+                    result, self.max
+                )
+            )
+        return result
+
+
+class Float(Argument):
+    defaultDefault: Optional[float] = None
+
+    def __init__(
+        self, name, allowNone=1, default=None, shortDesc=None, longDesc=None, hints=None
+    ):
+        # although Argument now has allowNone, that was recently added, and
+        # putting it at the end kept things which relied on argument order
+        # from breaking.  However, allowNone originally was in here, so
+        # I have to keep the same order, to prevent breaking code that
+        # depends on argument order only
+        Argument.__init__(self, name, default, shortDesc, longDesc, hints, allowNone)
+
+    def coerce(self, val):
+        if not val.strip() and self.allowNone:
+            return None
+        try:
+            return float(val)
+        except ValueError:
+            raise InputError("Invalid float: %s" % val)
+
+
+class Choice(Argument):
+    """
+    The result of a choice between enumerated types.  The choices should
+    be a list of tuples of tag, value, and description.  The tag will be
+    the value returned if the user hits "Submit", and the description
+    is the bale for the enumerated type.  default is a list of all the
+    values (seconds element in choices).  If no defaults are specified,
+    initially the first item will be selected.  Only one item can (should)
+    be selected at once.
+    """
+
+    def __init__(
+        self,
+        name,
+        choices=[],
+        default=[],
+        shortDesc=None,
+        longDesc=None,
+        hints=None,
+        allowNone=1,
+    ):
+        self.choices = choices
+        if choices and not default:
+            default.append(choices[0][1])
+        Argument.__init__(
+            self, name, default, shortDesc, longDesc, hints, allowNone=allowNone
+        )
+
+    def coerce(self, inIdent):
+        for ident, val, desc in self.choices:
+            if ident == inIdent:
+                return val
+        else:
+            raise InputError("Invalid Choice: %s" % inIdent)
+
+
+class Flags(Argument):
+    """
+    The result of a checkbox group or multi-menu.  The flags should be a
+    list of tuples of tag, value, and description. The tag will be
+    the value returned if the user hits "Submit", and the description
+    is the bale for the enumerated type.  default is a list of all the
+    values (second elements in flags).  If no defaults are specified,
+    initially nothing will be selected.  Several items may be selected at
+    once.
+    """
+
+    def __init__(
+        self,
+        name,
+        flags=(),
+        default=(),
+        shortDesc=None,
+        longDesc=None,
+        hints=None,
+        allowNone=1,
+    ):
+        self.flags = flags
+        Argument.__init__(
+            self, name, default, shortDesc, longDesc, hints, allowNone=allowNone
+        )
+
+    def coerce(self, inFlagKeys):
+        if not inFlagKeys:
+            return []
+        outFlags = []
+        for inFlagKey in inFlagKeys:
+            for flagKey, flagVal, flagDesc in self.flags:
+                if inFlagKey == flagKey:
+                    outFlags.append(flagVal)
+                    break
+            else:
+                raise InputError("Invalid Flag: %s" % inFlagKey)
+        return outFlags
+
+
+class CheckGroup(Flags):
+    pass
+
+
+class RadioGroup(Choice):
+    pass
+
+
+class Boolean(Argument):
+    def coerce(self, inVal):
+        if not inVal:
+            return 0
+        lInVal = str(inVal).lower()
+        if lInVal in ("no", "n", "f", "false", "0"):
+            return 0
+        return 1
+
+
+class File(Argument):
+    def __init__(self, name, allowNone=1, shortDesc=None, longDesc=None, hints=None):
+        Argument.__init__(
+            self, name, None, shortDesc, longDesc, hints, allowNone=allowNone
+        )
+
+    def coerce(self, file):
+        if not file and self.allowNone:
+            return None
+        elif file:
+            return file
+        else:
+            raise InputError("Invalid File")
+
+
+def positiveInt(x):
+    x = int(x)
+    if x <= 0:
+        raise ValueError
+    return x
+
+
+class Date(Argument):
+    """A date -- (year, month, day) tuple."""
+
+    defaultDefault: Optional[Tuple[int, int, int]] = None
+
+    def __init__(
+        self, name, allowNone=1, default=None, shortDesc=None, longDesc=None, hints=None
+    ):
+        Argument.__init__(self, name, default, shortDesc, longDesc, hints)
+        self.allowNone = allowNone
+        if not allowNone:
+            self.defaultDefault = (1970, 1, 1)
+
+    def coerce(self, args):
+        """Return tuple of ints (year, month, day)."""
+        if tuple(args) == ("", "", "") and self.allowNone:
+            return None
+
+        try:
+            year, month, day = map(positiveInt, args)
+        except ValueError:
+            raise InputError("Invalid date")
+        if (month, day) == (2, 29):
+            if not calendar.isleap(year):
+                raise InputError("%d was not a leap year" % year)
+            else:
+                return year, month, day
+        try:
+            mdays = calendar.mdays[month]
+        except IndexError:
+            raise InputError("Invalid date")
+        if day > mdays:
+            raise InputError("Invalid date")
+        return year, month, day
+
+
+class Submit(Choice):
+    """Submit button or a reasonable facsimile thereof."""
+
+    def __init__(
+        self,
+        name,
+        choices=[("Submit", "submit", "Submit form")],
+        reset=0,
+        shortDesc=None,
+        longDesc=None,
+        allowNone=0,
+        hints=None,
+    ):
+        Choice.__init__(
+            self,
+            name,
+            choices=choices,
+            shortDesc=shortDesc,
+            longDesc=longDesc,
+            hints=hints,
+        )
+        self.allowNone = allowNone
+        self.reset = reset
+
+    def coerce(self, value):
+        if self.allowNone and not value:
+            return None
+        else:
+            return Choice.coerce(self, value)
+
+
+class PresentationHint:
+    """
+    A hint to a particular system.
+    """
+
+
+class MethodSignature:
+    """
+    A signature of a callable.
+    """
+
+    def __init__(self, *sigList):
+        """"""
+        self.methodSignature = sigList
+
+    def getArgument(self, name):
+        for a in self.methodSignature:
+            if a.name == name:
+                return a
+
+    def method(self, callable, takesRequest=False):
+        return FormMethod(self, callable, takesRequest)
+
+
+class FormMethod:
+    """A callable object with a signature."""
+
+    def __init__(self, signature, callable, takesRequest=False):
+        self.signature = signature
+        self.callable = callable
+        self.takesRequest = takesRequest
+
+    def getArgs(self):
+        return tuple(self.signature.methodSignature)
+
+    def call(self, *args, **kw):
+        return self.callable(*args, **kw)
diff --git a/contrib/python/Twisted/py3/twisted/python/htmlizer.py b/contrib/python/Twisted/py3/twisted/python/htmlizer.py
new file mode 100644
index 00000000000..c1d4e43a378
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/htmlizer.py
@@ -0,0 +1,133 @@
+# -*- test-case-name: twisted.python.test.test_htmlizer -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HTML rendering of Python source.
+"""
+
+import keyword
+import tokenize
+from html import escape
+from typing import List
+
+from . import reflect
+
+
+class TokenPrinter:
+    """
+    Format a stream of tokens and intermediate whitespace, for pretty-printing.
+    """
+
+    currentCol, currentLine = 0, 1
+    lastIdentifier = parameters = 0
+    encoding = "utf-8"
+
+    def __init__(self, writer):
+        """
+        @param writer: A file-like object, opened in bytes mode.
+        """
+        self.writer = writer
+
+    def printtoken(self, type, token, sCoordinates, eCoordinates, line):
+        if hasattr(tokenize, "ENCODING") and type == tokenize.ENCODING:
+            self.encoding = token
+            return
+
+        if not isinstance(token, bytes):
+            token = token.encode(self.encoding)
+
+        (srow, scol) = sCoordinates
+        (erow, ecol) = eCoordinates
+        if self.currentLine < srow:
+            self.writer(b"\n" * (srow - self.currentLine))
+            self.currentLine, self.currentCol = srow, 0
+        self.writer(b" " * (scol - self.currentCol))
+        if self.lastIdentifier:
+            type = "identifier"
+            self.parameters = 1
+        elif type == tokenize.NAME:
+            if keyword.iskeyword(token):
+                type = "keyword"
+            else:
+                if self.parameters:
+                    type = "parameter"
+                else:
+                    type = "variable"
+        else:
+            type = tokenize.tok_name.get(type)
+            assert type is not None
+            type = type.lower()
+        self.writer(token, type)
+        self.currentCol = ecol
+        self.currentLine += token.count(b"\n")
+        if self.currentLine != erow:
+            self.currentCol = 0
+        self.lastIdentifier = token in (b"def", b"class")
+        if token == b":":
+            self.parameters = 0
+
+
+class HTMLWriter:
+    """
+    Write the stream of tokens and whitespace from L{TokenPrinter}, formating
+    tokens as HTML spans.
+    """
+
+    noSpan: List[str] = []
+
+    def __init__(self, writer):
+        self.writer = writer
+        noSpan: List[str] = []
+        reflect.accumulateClassList(self.__class__, "noSpan", noSpan)
+        self.noSpan = noSpan
+
+    def write(self, token, type=None):
+        if isinstance(token, bytes):
+            token = token.decode("utf-8")
+        token = escape(token)
+        token = token.encode("utf-8")
+        if (type is None) or (type in self.noSpan):
+            self.writer(token)
+        else:
+            self.writer(
+                b'<span class="py-src-'
+                + type.encode("utf-8")
+                + b'">'
+                + token
+                + b"</span>"
+            )
+
+
+class SmallerHTMLWriter(HTMLWriter):
+    """
+    HTMLWriter that doesn't generate spans for some junk.
+
+    Results in much smaller HTML output.
+    """
+
+    noSpan = ["endmarker", "indent", "dedent", "op", "newline", "nl"]
+
+
+def filter(inp, out, writer=HTMLWriter):
+    out.write(b"<pre>")
+    printer = TokenPrinter(writer(out.write).write).printtoken
+    try:
+        for token in tokenize.tokenize(inp.readline):
+            (tokenType, string, start, end, line) = token
+            printer(tokenType, string, start, end, line)
+    except tokenize.TokenError:
+        pass
+    out.write(b"</pre>\n")
+
+
+def main():
+    import sys
+
+    stdout = getattr(sys.stdout, "buffer", sys.stdout)
+    with open(sys.argv[1], "rb") as f:
+        filter(f, stdout)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/contrib/python/Twisted/py3/twisted/python/lockfile.py b/contrib/python/Twisted/py3/twisted/python/lockfile.py
new file mode 100644
index 00000000000..c285d4ad102
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/lockfile.py
@@ -0,0 +1,241 @@
+# -*- test-case-name: twisted.test.test_lockfile -*-
+# Copyright (c) 2005 Divmod, Inc.
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Filesystem-based interprocess mutex.
+"""
+
+
+import errno
+import os
+from time import time as _uniquefloat
+
+from twisted.python.runtime import platform
+
+
+def unique():
+    return str(int(_uniquefloat() * 1000))
+
+
+from os import rename
+
+if not platform.isWindows():
+    from os import kill, readlink, remove as rmlink, symlink
+
+    _windows = False
+else:
+    _windows = True
+
+    # On UNIX, a symlink can be made to a nonexistent location, and
+    # FilesystemLock uses this by making the target of the symlink an
+    # imaginary, non-existing file named that of the PID of the process with
+    # the lock. This has some benefits on UNIX -- making and removing this
+    # symlink is atomic. However, because Windows doesn't support symlinks (at
+    # least as how we know them), we have to fake this and actually write a
+    # file with the PID of the process holding the lock instead.
+    # These functions below perform that unenviable, probably-fraught-with-
+    # race-conditions duty. - hawkie
+
+    try:
+        import pywintypes  # type: ignore[import]
+        from win32api import OpenProcess  # type: ignore[import]
+    except ImportError:
+        kill = None  # type: ignore[assignment]
+    else:
+        ERROR_ACCESS_DENIED = 5
+        ERROR_INVALID_PARAMETER = 87
+
+        # typing ignored due to:
+        # https://github.com/python/typeshed/issues/4249
+        def kill(pid, signal):  # type: ignore[misc]
+            try:
+                OpenProcess(0, 0, pid)
+            except pywintypes.error as e:
+                if e.args[0] == ERROR_ACCESS_DENIED:
+                    return
+                elif e.args[0] == ERROR_INVALID_PARAMETER:
+                    raise OSError(errno.ESRCH, None)
+                raise
+            else:
+                raise RuntimeError("OpenProcess is required to fail.")
+
+    # For monkeypatching in tests
+    _open = open
+
+    # typing ignored due to:
+    # https://github.com/python/typeshed/issues/4249
+    def symlink(value, filename):  # type: ignore[misc]
+        """
+        Write a file at C{filename} with the contents of C{value}. See the
+        above comment block as to why this is needed.
+        """
+        # XXX Implement an atomic thingamajig for win32
+        newlinkname = filename + "." + unique() + ".newlink"
+        newvalname = os.path.join(newlinkname, "symlink")
+        os.mkdir(newlinkname)
+
+        # Python 3 does not support the 'commit' flag of fopen in the MSVCRT
+        # (http://msdn.microsoft.com/en-us/library/yeby3zcb%28VS.71%29.aspx)
+        mode = "w"
+
+        with _open(newvalname, mode) as f:
+            f.write(value)
+            f.flush()
+
+        try:
+            rename(newlinkname, filename)
+        except BaseException:
+            os.remove(newvalname)
+            os.rmdir(newlinkname)
+            raise
+
+    # typing ignored due to:
+    # https://github.com/python/typeshed/issues/4249
+    def readlink(filename):  # type: ignore[misc]
+        """
+        Read the contents of C{filename}. See the above comment block as to why
+        this is needed.
+        """
+        try:
+            fObj = _open(os.path.join(filename, "symlink"), "r")
+        except OSError as e:
+            if e.errno == errno.ENOENT or e.errno == errno.EIO:
+                raise OSError(e.errno, None)
+            raise
+        else:
+            with fObj:
+                result = fObj.read()
+            return result
+
+    # typing ignored due to:
+    # https://github.com/python/typeshed/issues/4249
+    def rmlink(filename):  # type: ignore[misc]
+        os.remove(os.path.join(filename, "symlink"))
+        os.rmdir(filename)
+
+
+class FilesystemLock:
+    """
+    A mutex.
+
+    This relies on the filesystem property that creating
+    a symlink is an atomic operation and that it will
+    fail if the symlink already exists.  Deleting the
+    symlink will release the lock.
+
+    @ivar name: The name of the file associated with this lock.
+
+    @ivar clean: Indicates whether this lock was released cleanly by its
+        last owner.  Only meaningful after C{lock} has been called and
+        returns True.
+
+    @ivar locked: Indicates whether the lock is currently held by this
+        object.
+    """
+
+    clean = None
+    locked = False
+
+    def __init__(self, name):
+        self.name = name
+
+    def lock(self):
+        """
+        Acquire this lock.
+
+        @rtype: C{bool}
+        @return: True if the lock is acquired, false otherwise.
+
+        @raise OSError: Any exception L{os.symlink()} may raise,
+            other than L{errno.EEXIST}.
+        """
+        clean = True
+        while True:
+            try:
+                symlink(str(os.getpid()), self.name)
+            except OSError as e:
+                if _windows and e.errno in (errno.EACCES, errno.EIO):
+                    # The lock is in the middle of being deleted because we're
+                    # on Windows where lock removal isn't atomic.  Give up, we
+                    # don't know how long this is going to take.
+                    return False
+                if e.errno == errno.EEXIST:
+                    try:
+                        pid = readlink(self.name)
+                    except OSError as e:
+                        if e.errno == errno.ENOENT:
+                            # The lock has vanished, try to claim it in the
+                            # next iteration through the loop.
+                            continue
+                        elif _windows and e.errno == errno.EACCES:
+                            # The lock is in the middle of being
+                            # deleted because we're on Windows where
+                            # lock removal isn't atomic.  Give up, we
+                            # don't know how long this is going to
+                            # take.
+                            return False
+                        raise
+                    try:
+                        if kill is not None:
+                            kill(int(pid), 0)
+                    except OSError as e:
+                        if e.errno == errno.ESRCH:
+                            # The owner has vanished, try to claim it in the
+                            # next iteration through the loop.
+                            try:
+                                rmlink(self.name)
+                            except OSError as e:
+                                if e.errno == errno.ENOENT:
+                                    # Another process cleaned up the lock.
+                                    # Race them to acquire it in the next
+                                    # iteration through the loop.
+                                    continue
+                                raise
+                            clean = False
+                            continue
+                        raise
+                    return False
+                raise
+            self.locked = True
+            self.clean = clean
+            return True
+
+    def unlock(self):
+        """
+        Release this lock.
+
+        This deletes the directory with the given name.
+
+        @raise OSError: Any exception L{os.readlink()} may raise.
+        @raise ValueError: If the lock is not owned by this process.
+        """
+        pid = readlink(self.name)
+        if int(pid) != os.getpid():
+            raise ValueError(f"Lock {self.name!r} not owned by this process")
+        rmlink(self.name)
+        self.locked = False
+
+
+def isLocked(name):
+    """
+    Determine if the lock of the given name is held or not.
+
+    @type name: C{str}
+    @param name: The filesystem path to the lock to test
+
+    @rtype: C{bool}
+    @return: True if the lock is held, False otherwise.
+    """
+    l = FilesystemLock(name)
+    result = None
+    try:
+        result = l.lock()
+    finally:
+        if result:
+            l.unlock()
+    return not result
+
+
+__all__ = ["FilesystemLock", "isLocked"]
diff --git a/contrib/python/Twisted/py3/twisted/python/log.py b/contrib/python/Twisted/py3/twisted/python/log.py
new file mode 100644
index 00000000000..4392486bbee
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/log.py
@@ -0,0 +1,738 @@
+# -*- test-case-name: twisted.test.test_log -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Logging and metrics infrastructure.
+"""
+
+
+import sys
+import time
+import warnings
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from typing import Any, BinaryIO, Dict, Optional, cast
+
+from zope.interface import Interface
+
+from twisted.logger import (
+    LegacyLogObserverWrapper,
+    Logger as NewLogger,
+    LoggingFile,
+    LogLevel as NewLogLevel,
+    LogPublisher as NewPublisher,
+    STDLibLogObserver as NewSTDLibLogObserver,
+    globalLogBeginner as newGlobalLogBeginner,
+    globalLogPublisher as newGlobalLogPublisher,
+)
+from twisted.logger._global import LogBeginner
+from twisted.logger._legacy import publishToNewObserver as _publishNew
+from twisted.python import context, failure, reflect, util
+from twisted.python.threadable import synchronize
+
+EventDict = Dict[str, Any]
+
+
+class ILogContext:
+    """
+    Actually, this interface is just a synonym for the dictionary interface,
+    but it serves as a key for the default information in a log.
+
+    I do not inherit from C{Interface} because the world is a cruel place.
+    """
+
+
+class ILogObserver(Interface):
+    """
+    An observer which can do something with log events.
+
+    Given that most log observers are actually bound methods, it's okay to not
+    explicitly declare provision of this interface.
+    """
+
+    def __call__(eventDict: EventDict) -> None:
+        """
+        Log an event.
+
+        @param eventDict: A dictionary with arbitrary keys.  However, these
+            keys are often available:
+              - C{message}: A C{tuple} of C{str} containing messages to be
+                logged.
+              - C{system}: A C{str} which indicates the "system" which is
+                generating this event.
+              - C{isError}: A C{bool} indicating whether this event represents
+                an error.
+              - C{failure}: A L{failure.Failure} instance
+              - C{why}: Used as header of the traceback in case of errors.
+              - C{format}: A string format used in place of C{message} to
+                customize the event.  The intent is for the observer to format
+                a message by doing something like C{format % eventDict}.
+        """
+
+
+context.setDefault(ILogContext, {"system": "-"})
+
+
+def callWithContext(ctx, func, *args, **kw):
+    newCtx = context.get(ILogContext).copy()
+    newCtx.update(ctx)
+    return context.call({ILogContext: newCtx}, func, *args, **kw)
+
+
+def callWithLogger(logger, func, *args, **kw):
+    """
+    Utility method which wraps a function in a try:/except:, logs a failure if
+    one occurs, and uses the system's logPrefix.
+    """
+    try:
+        lp = logger.logPrefix()
+    except KeyboardInterrupt:
+        raise
+    except BaseException:
+        lp = "(buggy logPrefix method)"
+        err(system=lp)
+    try:
+        return callWithContext({"system": lp}, func, *args, **kw)
+    except KeyboardInterrupt:
+        raise
+    except BaseException:
+        err(system=lp)
+
+
+def err(_stuff=None, _why=None, **kw):
+    """
+    Write a failure to the log.
+
+    The C{_stuff} and C{_why} parameters use an underscore prefix to lessen
+    the chance of colliding with a keyword argument the application wishes
+    to pass.  It is intended that they be supplied with arguments passed
+    positionally, not by keyword.
+
+    @param _stuff: The failure to log.  If C{_stuff} is L{None} a new
+        L{Failure} will be created from the current exception state.  If
+        C{_stuff} is an C{Exception} instance it will be wrapped in a
+        L{Failure}.
+    @type _stuff: L{None}, C{Exception}, or L{Failure}.
+
+    @param _why: The source of this failure.  This will be logged along with
+        C{_stuff} and should describe the context in which the failure
+        occurred.
+    @type _why: C{str}
+    """
+    if _stuff is None:
+        _stuff = failure.Failure()
+    if isinstance(_stuff, failure.Failure):
+        msg(failure=_stuff, why=_why, isError=1, **kw)
+    elif isinstance(_stuff, Exception):
+        msg(failure=failure.Failure(_stuff), why=_why, isError=1, **kw)
+    else:
+        msg(repr(_stuff), why=_why, isError=1, **kw)
+
+
+deferr = err
+
+
+class Logger:
+    """
+    This represents a class which may 'own' a log. Used by subclassing.
+    """
+
+    def logPrefix(self):
+        """
+        Override this method to insert custom logging behavior.  Its
+        return value will be inserted in front of every line.  It may
+        be called more times than the number of output lines.
+        """
+        return "-"
+
+
+class LogPublisher:
+    """
+    Class for singleton log message publishing.
+    """
+
+    synchronized = ["msg"]
+
+    def __init__(
+        self,
+        observerPublisher=None,
+        publishPublisher=None,
+        logBeginner=None,
+        warningsModule=warnings,
+    ):
+        if publishPublisher is None:
+            publishPublisher = NewPublisher()
+            if observerPublisher is None:
+                observerPublisher = publishPublisher
+        if observerPublisher is None:
+            observerPublisher = NewPublisher()
+        self._observerPublisher = observerPublisher
+        self._publishPublisher = publishPublisher
+        self._legacyObservers = []
+        if logBeginner is None:
+            # This default behavior is really only used for testing.
+            beginnerPublisher = NewPublisher()
+            beginnerPublisher.addObserver(observerPublisher)
+            logBeginner = LogBeginner(
+                beginnerPublisher, cast(BinaryIO, NullFile()), sys, warnings
+            )
+        self._logBeginner = logBeginner
+        self._warningsModule = warningsModule
+        self._oldshowwarning = warningsModule.showwarning
+        self.showwarning = self._logBeginner.showwarning
+
+    @property
+    def observers(self):
+        """
+        Property returning all observers registered on this L{LogPublisher}.
+
+        @return: observers previously added with L{LogPublisher.addObserver}
+        @rtype: L{list} of L{callable}
+        """
+        return [x.legacyObserver for x in self._legacyObservers]
+
+    def _startLogging(self, other, setStdout):
+        """
+        Begin logging to the L{LogBeginner} associated with this
+        L{LogPublisher}.
+
+        @param other: the observer to log to.
+        @type other: L{LogBeginner}
+
+        @param setStdout: if true, send standard I/O to the observer as well.
+        @type setStdout: L{bool}
+        """
+        wrapped = LegacyLogObserverWrapper(other)
+        self._legacyObservers.append(wrapped)
+        self._logBeginner.beginLoggingTo([wrapped], True, setStdout)
+
+    def _stopLogging(self):
+        """
+        Clean-up hook for fixing potentially global state.  Only for testing of
+        this module itself.  If you want less global state, use the new
+        warnings system in L{twisted.logger}.
+        """
+        if self._warningsModule.showwarning == self.showwarning:
+            self._warningsModule.showwarning = self._oldshowwarning
+
+    def addObserver(self, other):
+        """
+        Add a new observer.
+
+        @type other: Provider of L{ILogObserver}
+        @param other: A callable object that will be called with each new log
+            message (a dict).
+        """
+        wrapped = LegacyLogObserverWrapper(other)
+        self._legacyObservers.append(wrapped)
+        self._observerPublisher.addObserver(wrapped)
+
+    def removeObserver(self, other):
+        """
+        Remove an observer.
+        """
+        for observer in self._legacyObservers:
+            if observer.legacyObserver == other:
+                self._legacyObservers.remove(observer)
+                self._observerPublisher.removeObserver(observer)
+                break
+
+    def msg(self, *message, **kw):
+        """
+        Log a new message.
+
+        The message should be a native string, i.e. bytes on Python 2 and
+        Unicode on Python 3. For compatibility with both use the native string
+        syntax, for example::
+
+            >>> log.msg('Hello, world.')
+
+        You MUST avoid passing in Unicode on Python 2, and the form::
+
+            >>> log.msg('Hello ', 'world.')
+
+        This form only works (sometimes) by accident.
+
+        Keyword arguments will be converted into items in the event
+        dict that is passed to L{ILogObserver} implementations.
+        Each implementation, in turn, can define keys that are used
+        by it specifically, in addition to common keys listed at
+        L{ILogObserver.__call__}.
+
+        For example, to set the C{system} parameter while logging
+        a message::
+
+        >>> log.msg('Started', system='Foo')
+
+        """
+        actualEventDict = cast(EventDict, (context.get(ILogContext) or {}).copy())
+        actualEventDict.update(kw)
+        actualEventDict["message"] = message
+        actualEventDict["time"] = time.time()
+        if "isError" not in actualEventDict:
+            actualEventDict["isError"] = 0
+
+        _publishNew(self._publishPublisher, actualEventDict, textFromEventDict)
+
+
+synchronize(LogPublisher)
+
+
+if "theLogPublisher" not in globals():
+
+    def _actually(something):
+        """
+        A decorator that returns its argument rather than the thing it is
+        decorating.
+
+        This allows the documentation generator to see an alias for a method or
+        constant as an object with a docstring and thereby document it and
+        allow references to it statically.
+
+        @param something: An object to create an alias for.
+        @type something: L{object}
+
+        @return: a 1-argument callable that returns C{something}
+        @rtype: L{object}
+        """
+
+        def decorate(thingWithADocstring):
+            return something
+
+        return decorate
+
+    theLogPublisher = LogPublisher(
+        observerPublisher=newGlobalLogPublisher,
+        publishPublisher=newGlobalLogPublisher,
+        logBeginner=newGlobalLogBeginner,
+    )
+
+    @_actually(theLogPublisher.addObserver)
+    def addObserver(observer):
+        """
+        Add a log observer to the global publisher.
+
+        @see: L{LogPublisher.addObserver}
+
+        @param observer: a log observer
+        @type observer: L{callable}
+        """
+
+    @_actually(theLogPublisher.removeObserver)
+    def removeObserver(observer):
+        """
+        Remove a log observer from the global publisher.
+
+        @see: L{LogPublisher.removeObserver}
+
+        @param observer: a log observer previously added with L{addObserver}
+        @type observer: L{callable}
+        """
+
+    @_actually(theLogPublisher.msg)
+    def msg(*message, **event):
+        """
+        Publish a message to the global log publisher.
+
+        @see: L{LogPublisher.msg}
+
+        @param message: the log message
+        @type message: C{tuple} of L{str} (native string)
+
+        @param event: fields for the log event
+        @type event: L{dict} mapping L{str} (native string) to L{object}
+        """
+
+    @_actually(theLogPublisher.showwarning)
+    def showwarning():
+        """
+        Publish a Python warning through the global log publisher.
+
+        @see: L{LogPublisher.showwarning}
+        """
+
+
+def _safeFormat(fmtString: str, fmtDict: Dict[str, Any]) -> str:
+    """
+    Try to format a string, swallowing all errors to always return a string.
+
+    @note: For backward-compatibility reasons, this function ensures that it
+        returns a native string, meaning L{bytes} in Python 2 and L{str} in
+        Python 3.
+
+    @param fmtString: a C{%}-format string
+    @param fmtDict: string formatting arguments for C{fmtString}
+
+    @return: A native string, formatted from C{fmtString} and C{fmtDict}.
+    """
+    # There's a way we could make this if not safer at least more
+    # informative: perhaps some sort of str/repr wrapper objects
+    # could be wrapped around the things inside of C{fmtDict}. That way
+    # if the event dict contains an object with a bad __repr__, we
+    # can only cry about that individual object instead of the
+    # entire event dict.
+    try:
+        text = fmtString % fmtDict
+    except KeyboardInterrupt:
+        raise
+    except BaseException:
+        try:
+            text = (
+                "Invalid format string or unformattable object in "
+                "log message: %r, %s" % (fmtString, fmtDict)
+            )
+        except BaseException:
+            try:
+                text = (
+                    "UNFORMATTABLE OBJECT WRITTEN TO LOG with fmt %r, "
+                    "MESSAGE LOST" % (fmtString,)
+                )
+            except BaseException:
+                text = (
+                    "PATHOLOGICAL ERROR IN BOTH FORMAT STRING AND "
+                    "MESSAGE DETAILS, MESSAGE LOST"
+                )
+
+    return text
+
+
+def textFromEventDict(eventDict: EventDict) -> Optional[str]:
+    """
+    Extract text from an event dict passed to a log observer. If it cannot
+    handle the dict, it returns None.
+
+    The possible keys of eventDict are:
+     - C{message}: by default, it holds the final text. It's required, but can
+       be empty if either C{isError} or C{format} is provided (the first
+       having the priority).
+     - C{isError}: boolean indicating the nature of the event.
+     - C{failure}: L{failure.Failure} instance, required if the event is an
+       error.
+     - C{why}: if defined, used as header of the traceback in case of errors.
+     - C{format}: string format used in place of C{message} to customize
+       the event. It uses all keys present in C{eventDict} to format
+       the text.
+    Other keys will be used when applying the C{format}, or ignored.
+    """
+    edm = eventDict["message"]
+    if not edm:
+        if eventDict["isError"] and "failure" in eventDict:
+            why = cast(str, eventDict.get("why"))
+            if why:
+                why = reflect.safe_str(why)
+            else:
+                why = "Unhandled Error"
+            try:
+                traceback = cast(failure.Failure, eventDict["failure"]).getTraceback()
+            except Exception as e:
+                traceback = "(unable to obtain traceback): " + str(e)
+            text = why + "\n" + traceback
+        elif "format" in eventDict:
+            text = _safeFormat(eventDict["format"], eventDict)
+        else:
+            # We don't know how to log this
+            return None
+    else:
+        text = " ".join(map(reflect.safe_str, edm))
+    return text
+
+
+class _GlobalStartStopObserver(ABC):
+    """
+    Mix-in for global log observers that can start and stop.
+    """
+
+    @abstractmethod
+    def emit(self, eventDict: EventDict) -> None:
+        """
+        Emit the given log event.
+
+        @param eventDict: a log event
+        """
+
+    def start(self) -> None:
+        """
+        Start observing log events.
+        """
+        addObserver(self.emit)
+
+    def stop(self) -> None:
+        """
+        Stop observing log events.
+        """
+        removeObserver(self.emit)
+
+
+class FileLogObserver(_GlobalStartStopObserver):
+    """
+    Log observer that writes to a file-like object.
+
+    @type timeFormat: C{str} or L{None}
+    @ivar timeFormat: If not L{None}, the format string passed to strftime().
+    """
+
+    timeFormat: Optional[str] = None
+
+    def __init__(self, f):
+        # Compatibility
+        self.write = f.write
+        self.flush = f.flush
+
+    def getTimezoneOffset(self, when):
+        """
+        Return the current local timezone offset from UTC.
+
+        @type when: C{int}
+        @param when: POSIX (ie, UTC) timestamp for which to find the offset.
+
+        @rtype: C{int}
+        @return: The number of seconds offset from UTC.  West is positive,
+        east is negative.
+        """
+        offset = datetime.fromtimestamp(when, timezone.utc).replace(
+            tzinfo=None
+        ) - datetime.fromtimestamp(when)
+        return offset.days * (60 * 60 * 24) + offset.seconds
+
+    def formatTime(self, when):
+        """
+        Format the given UTC value as a string representing that time in the
+        local timezone.
+
+        By default it's formatted as an ISO8601-like string (ISO8601 date and
+        ISO8601 time separated by a space). It can be customized using the
+        C{timeFormat} attribute, which will be used as input for the underlying
+        L{datetime.datetime.strftime} call.
+
+        @type when: C{int}
+        @param when: POSIX (ie, UTC) timestamp for which to find the offset.
+
+        @rtype: C{str}
+        """
+        if self.timeFormat is not None:
+            return datetime.fromtimestamp(when).strftime(self.timeFormat)
+
+        tzOffset = -self.getTimezoneOffset(when)
+        when = datetime.fromtimestamp(when + tzOffset, timezone.utc).replace(
+            tzinfo=None
+        )
+        tzHour = abs(int(tzOffset / 60 / 60))
+        tzMin = abs(int(tzOffset / 60 % 60))
+        if tzOffset < 0:
+            tzSign = "-"
+        else:
+            tzSign = "+"
+        return "%d-%02d-%02d %02d:%02d:%02d%s%02d%02d" % (
+            when.year,
+            when.month,
+            when.day,
+            when.hour,
+            when.minute,
+            when.second,
+            tzSign,
+            tzHour,
+            tzMin,
+        )
+
+    def emit(self, eventDict: EventDict) -> None:
+        """
+        Format the given log event as text and write it to the output file.
+
+        @param eventDict: a log event
+        """
+        text = textFromEventDict(eventDict)
+        if text is None:
+            return
+
+        timeStr = self.formatTime(eventDict["time"])
+        fmtDict = {"system": eventDict["system"], "text": text.replace("\n", "\n\t")}
+        msgStr = _safeFormat("[%(system)s] %(text)s\n", fmtDict)
+
+        util.untilConcludes(self.write, timeStr + " " + msgStr)
+        util.untilConcludes(self.flush)  # Hoorj!
+
+
+class PythonLoggingObserver(_GlobalStartStopObserver):
+    """
+    Output twisted messages to Python standard library L{logging} module.
+
+    WARNING: specific logging configurations (example: network) can lead to
+    a blocking system. Nothing is done here to prevent that, so be sure to not
+    use this: code within Twisted, such as twisted.web, assumes that logging
+    does not block.
+    """
+
+    def __init__(self, loggerName="twisted"):
+        """
+        @param loggerName: identifier used for getting logger.
+        @type loggerName: C{str}
+        """
+        self._newObserver = NewSTDLibLogObserver(loggerName)
+
+    def emit(self, eventDict: EventDict) -> None:
+        """
+        Receive a twisted log entry, format it and bridge it to python.
+
+        By default the logging level used is info; log.err produces error
+        level, and you can customize the level by using the C{logLevel} key::
+
+            >>> log.msg('debugging', logLevel=logging.DEBUG)
+        """
+        if "log_format" in eventDict:
+            _publishNew(self._newObserver, eventDict, textFromEventDict)
+
+
+class StdioOnnaStick:
+    """
+    Class that pretends to be stdout/err, and turns writes into log messages.
+
+    @ivar isError: boolean indicating whether this is stderr, in which cases
+                   log messages will be logged as errors.
+
+    @ivar encoding: unicode encoding used to encode any unicode strings
+                    written to this object.
+    """
+
+    closed = 0
+    softspace = 0
+    mode = "wb"
+    name = "<stdio (log)>"
+
+    def __init__(self, isError=0, encoding=None):
+        self.isError = isError
+        if encoding is None:
+            encoding = sys.getdefaultencoding()
+        self.encoding = encoding
+        self.buf = ""
+
+    def close(self):
+        pass
+
+    def fileno(self):
+        return -1
+
+    def flush(self):
+        pass
+
+    def read(self):
+        raise OSError("can't read from the log!")
+
+    readline = read
+    readlines = read
+    seek = read
+    tell = read
+
+    def write(self, data):
+        d = (self.buf + data).split("\n")
+        self.buf = d[-1]
+        messages = d[0:-1]
+        for message in messages:
+            msg(message, printed=1, isError=self.isError)
+
+    def writelines(self, lines):
+        for line in lines:
+            msg(line, printed=1, isError=self.isError)
+
+
+def startLogging(file, *a, **kw):
+    """
+    Initialize logging to a specified file.
+
+    @return: A L{FileLogObserver} if a new observer is added, None otherwise.
+    """
+    if isinstance(file, LoggingFile):
+        return
+    flo = FileLogObserver(file)
+    startLoggingWithObserver(flo.emit, *a, **kw)
+    return flo
+
+
+def startLoggingWithObserver(observer, setStdout=1):
+    """
+    Initialize logging to a specified observer. If setStdout is true
+    (defaults to yes), also redirect sys.stdout and sys.stderr
+    to the specified file.
+    """
+    theLogPublisher._startLogging(observer, setStdout)
+    msg("Log opened.")
+
+
+class NullFile:
+    """
+    A file-like object that discards everything.
+    """
+
+    softspace = 0
+
+    def read(self):
+        """
+        Do nothing.
+        """
+
+    def write(self, bytes):
+        """
+        Do nothing.
+
+        @param bytes: data
+        @type bytes: L{bytes}
+        """
+
+    def flush(self):
+        """
+        Do nothing.
+        """
+
+    def close(self):
+        """
+        Do nothing.
+        """
+
+
+def discardLogs():
+    """
+    Discard messages logged via the global C{logfile} object.
+    """
+    global logfile
+    logfile = NullFile()
+
+
+# Prevent logfile from being erased on reload.  This only works in cpython.
+if "logfile" not in globals():
+    logfile = LoggingFile(
+        logger=NewLogger(),
+        level=NewLogLevel.info,
+        encoding=getattr(sys.stdout, "encoding", None),
+    )
+    logerr = LoggingFile(
+        logger=NewLogger(),
+        level=NewLogLevel.error,
+        encoding=getattr(sys.stderr, "encoding", None),
+    )
+
+
+class DefaultObserver(_GlobalStartStopObserver):
+    """
+    Default observer.
+
+    Will ignore all non-error messages and send error messages to sys.stderr.
+    Will be removed when startLogging() is called for the first time.
+    """
+
+    stderr = sys.stderr
+
+    def emit(self, eventDict: EventDict) -> None:
+        """
+        Emit an event dict.
+
+        @param eventDict: an event dict
+        """
+        if eventDict["isError"]:
+            text = textFromEventDict(eventDict)
+            if text is not None:
+                self.stderr.write(text)
+            self.stderr.flush()
+
+
+if "defaultObserver" not in globals():
+    defaultObserver = DefaultObserver()
diff --git a/contrib/python/Twisted/py3/twisted/python/logfile.py b/contrib/python/Twisted/py3/twisted/python/logfile.py
new file mode 100644
index 00000000000..33971bab08e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/logfile.py
@@ -0,0 +1,341 @@
+# -*- test-case-name: twisted.test.test_logfile -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A rotating, browsable log file.
+"""
+
+
+# System Imports
+import glob
+import os
+import stat
+import time
+from typing import BinaryIO, Optional, cast
+
+from twisted.python import threadable
+
+
+class BaseLogFile:
+    """
+    The base class for a log file that can be rotated.
+    """
+
+    synchronized = ["write", "rotate"]
+
+    def __init__(
+        self, name: str, directory: str, defaultMode: Optional[int] = None
+    ) -> None:
+        """
+        Create a log file.
+
+        @param name: name of the file
+        @param directory: directory holding the file
+        @param defaultMode: permissions used to create the file. Default to
+        current permissions of the file if the file exists.
+        """
+        self.directory = directory
+        self.name = name
+        self.path = os.path.join(directory, name)
+        if defaultMode is None and os.path.exists(self.path):
+            self.defaultMode: Optional[int] = stat.S_IMODE(
+                os.stat(self.path)[stat.ST_MODE]
+            )
+        else:
+            self.defaultMode = defaultMode
+        self._openFile()
+
+    @classmethod
+    def fromFullPath(cls, filename, *args, **kwargs):
+        """
+        Construct a log file from a full file path.
+        """
+        logPath = os.path.abspath(filename)
+        return cls(os.path.basename(logPath), os.path.dirname(logPath), *args, **kwargs)
+
+    def shouldRotate(self):
+        """
+        Override with a method to that returns true if the log
+        should be rotated.
+        """
+        raise NotImplementedError
+
+    def _openFile(self):
+        """
+        Open the log file.
+
+        The log file is always opened in binary mode.
+        """
+        self.closed = False
+        if os.path.exists(self.path):
+            self._file = cast(BinaryIO, open(self.path, "rb+", 0))
+            self._file.seek(0, 2)
+        else:
+            if self.defaultMode is not None:
+                # Set the lowest permissions
+                oldUmask = os.umask(0o777)
+                try:
+                    self._file = cast(BinaryIO, open(self.path, "wb+", 0))
+                finally:
+                    os.umask(oldUmask)
+            else:
+                self._file = cast(BinaryIO, open(self.path, "wb+", 0))
+        if self.defaultMode is not None:
+            try:
+                os.chmod(self.path, self.defaultMode)
+            except OSError:
+                # Probably /dev/null or something?
+                pass
+
+    def write(self, data):
+        """
+        Write some data to the file.
+
+        @param data: The data to write.  Text will be encoded as UTF-8.
+        @type data: L{bytes} or L{unicode}
+        """
+        if self.shouldRotate():
+            self.flush()
+            self.rotate()
+        if isinstance(data, str):
+            data = data.encode("utf8")
+        self._file.write(data)
+
+    def flush(self):
+        """
+        Flush the file.
+        """
+        self._file.flush()
+
+    def close(self):
+        """
+        Close the file.
+
+        The file cannot be used once it has been closed.
+        """
+        self.closed = True
+        self._file.close()
+        del self._file
+
+    def reopen(self):
+        """
+        Reopen the log file. This is mainly useful if you use an external log
+        rotation tool, which moves under your feet.
+
+        Note that on Windows you probably need a specific API to rename the
+        file, as it's not supported to simply use os.rename, for example.
+        """
+        self.close()
+        self._openFile()
+
+    def getCurrentLog(self):
+        """
+        Return a LogReader for the current log file.
+        """
+        return LogReader(self.path)
+
+
+class LogFile(BaseLogFile):
+    """
+    A log file that can be rotated.
+
+    A rotateLength of None disables automatic log rotation.
+    """
+
+    def __init__(
+        self,
+        name,
+        directory,
+        rotateLength=1000000,
+        defaultMode=None,
+        maxRotatedFiles=None,
+    ):
+        """
+        Create a log file rotating on length.
+
+        @param name: file name.
+        @type name: C{str}
+        @param directory: path of the log file.
+        @type directory: C{str}
+        @param rotateLength: size of the log file where it rotates. Default to
+            1M.
+        @type rotateLength: C{int}
+        @param defaultMode: mode used to create the file.
+        @type defaultMode: C{int}
+        @param maxRotatedFiles: if not None, max number of log files the class
+            creates. Warning: it removes all log files above this number.
+        @type maxRotatedFiles: C{int}
+        """
+        BaseLogFile.__init__(self, name, directory, defaultMode)
+        self.rotateLength = rotateLength
+        self.maxRotatedFiles = maxRotatedFiles
+
+    def _openFile(self):
+        BaseLogFile._openFile(self)
+        self.size = self._file.tell()
+
+    def shouldRotate(self):
+        """
+        Rotate when the log file size is larger than rotateLength.
+        """
+        return self.rotateLength and self.size >= self.rotateLength
+
+    def getLog(self, identifier):
+        """
+        Given an integer, return a LogReader for an old log file.
+        """
+        filename = "%s.%d" % (self.path, identifier)
+        if not os.path.exists(filename):
+            raise ValueError("no such logfile exists")
+        return LogReader(filename)
+
+    def write(self, data):
+        """
+        Write some data to the file.
+        """
+        BaseLogFile.write(self, data)
+        self.size += len(data)
+
+    def rotate(self):
+        """
+        Rotate the file and create a new one.
+
+        If it's not possible to open new logfile, this will fail silently,
+        and continue logging to old logfile.
+        """
+        if not (os.access(self.directory, os.W_OK) and os.access(self.path, os.W_OK)):
+            return
+        logs = self.listLogs()
+        logs.reverse()
+        for i in logs:
+            if self.maxRotatedFiles is not None and i >= self.maxRotatedFiles:
+                os.remove("%s.%d" % (self.path, i))
+            else:
+                os.rename("%s.%d" % (self.path, i), "%s.%d" % (self.path, i + 1))
+        self._file.close()
+        os.rename(self.path, "%s.1" % self.path)
+        self._openFile()
+
+    def listLogs(self):
+        """
+        Return sorted list of integers - the old logs' identifiers.
+        """
+        result = []
+        for name in glob.glob("%s.*" % self.path):
+            try:
+                counter = int(name.split(".")[-1])
+                if counter:
+                    result.append(counter)
+            except ValueError:
+                pass
+        result.sort()
+        return result
+
+    def __getstate__(self):
+        state = BaseLogFile.__getstate__(self)
+        del state["size"]
+        return state
+
+
+threadable.synchronize(LogFile)
+
+
+class DailyLogFile(BaseLogFile):
+    """A log file that is rotated daily (at or after midnight localtime)"""
+
+    def _openFile(self):
+        BaseLogFile._openFile(self)
+        self.lastDate = self.toDate(os.stat(self.path)[8])
+
+    def shouldRotate(self):
+        """Rotate when the date has changed since last write"""
+        return self.toDate() > self.lastDate
+
+    def toDate(self, *args):
+        """Convert a unixtime to (year, month, day) localtime tuple,
+        or return the current (year, month, day) localtime tuple.
+
+        This function primarily exists so you may overload it with
+        gmtime, or some cruft to make unit testing possible.
+        """
+        # primarily so this can be unit tested easily
+        return time.localtime(*args)[:3]
+
+    def suffix(self, tupledate):
+        """Return the suffix given a (year, month, day) tuple or unixtime"""
+        try:
+            return "_".join(map(str, tupledate))
+        except BaseException:
+            # try taking a float unixtime
+            return "_".join(map(str, self.toDate(tupledate)))
+
+    def getLog(self, identifier):
+        """Given a unix time, return a LogReader for an old log file."""
+        if self.toDate(identifier) == self.lastDate:
+            return self.getCurrentLog()
+        filename = f"{self.path}.{self.suffix(identifier)}"
+        if not os.path.exists(filename):
+            raise ValueError("no such logfile exists")
+        return LogReader(filename)
+
+    def write(self, data):
+        """Write some data to the log file"""
+        BaseLogFile.write(self, data)
+        # Guard against a corner case where time.time()
+        # could potentially run backwards to yesterday.
+        # Primarily due to network time.
+        self.lastDate = max(self.lastDate, self.toDate())
+
+    def rotate(self):
+        """Rotate the file and create a new one.
+
+        If it's not possible to open new logfile, this will fail silently,
+        and continue logging to old logfile.
+        """
+        if not (os.access(self.directory, os.W_OK) and os.access(self.path, os.W_OK)):
+            return
+        newpath = f"{self.path}.{self.suffix(self.lastDate)}"
+        if os.path.exists(newpath):
+            return
+        self._file.close()
+        os.rename(self.path, newpath)
+        self._openFile()
+
+    def __getstate__(self):
+        state = BaseLogFile.__getstate__(self)
+        del state["lastDate"]
+        return state
+
+
+threadable.synchronize(DailyLogFile)
+
+
+class LogReader:
+    """Read from a log file."""
+
+    def __init__(self, name):
+        """
+        Open the log file for reading.
+
+        The comments about binary-mode for L{BaseLogFile._openFile} also apply
+        here.
+        """
+        self._file = open(name)  # Optional[BinaryIO]
+
+    def readLines(self, lines=10):
+        """Read a list of lines from the log file.
+
+        This doesn't returns all of the files lines - call it multiple times.
+        """
+        result = []
+        for i in range(lines):
+            line = self._file.readline()
+            if not line:
+                break
+            result.append(line)
+        return result
+
+    def close(self):
+        self._file.close()
diff --git a/contrib/python/Twisted/py3/twisted/python/modules.py b/contrib/python/Twisted/py3/twisted/python/modules.py
new file mode 100644
index 00000000000..aea6a6ed828
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/modules.py
@@ -0,0 +1,781 @@
+# -*- test-case-name: twisted.test.test_modules -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module aims to provide a unified, object-oriented view of Python's
+runtime hierarchy.
+
+Python is a very dynamic language with wide variety of introspection utilities.
+However, these utilities can be hard to use, because there is no consistent
+API.  The introspection API in python is made up of attributes (__name__,
+__module__, func_name, etc) on instances, modules, classes and functions which
+vary between those four types, utility modules such as 'inspect' which provide
+some functionality, the 'imp' module, the "compiler" module, the semantics of
+PEP 302 support, and setuptools, among other things.
+
+At the top, you have "PythonPath", an abstract representation of sys.path which
+includes methods to locate top-level modules, with or without loading them.
+The top-level exposed functions in this module for accessing the system path
+are "walkModules", "iterModules", and "getModule".
+
+From most to least specific, here are the objects provided::
+
+                  PythonPath  # sys.path
+                      |
+                      v
+                  PathEntry   # one entry on sys.path: an importer
+                      |
+                      v
+                 PythonModule # a module or package that can be loaded
+                      |
+                      v
+                 PythonAttribute # an attribute of a module (function or class)
+                      |
+                      v
+                 PythonAttribute # an attribute of a function or class
+                      |
+                      v
+                     ...
+
+Here's an example of idiomatic usage: this is what you would do to list all of
+the modules outside the standard library's python-files directory::
+
+    import os
+    stdlibdir = os.path.dirname(os.__file__)
+
+    from twisted.python.modules import iterModules
+
+    for modinfo in iterModules():
+        if (modinfo.pathEntry.filePath.path != stdlibdir
+            and not modinfo.isPackage()):
+            print('unpackaged: %s: %s' % (
+                modinfo.name, modinfo.filePath.path))
+
+@var theSystemPath: The very top of the Python object space.
+@type theSystemPath: L{PythonPath}
+"""
+
+
+import inspect
+import sys
+import warnings
+import zipimport
+
+# let's try to keep path imports to a minimum...
+from os.path import dirname, split as splitpath
+
+from zope.interface import Interface, implementer
+
+from twisted.python.compat import nativeString
+from twisted.python.components import registerAdapter
+from twisted.python.filepath import FilePath, UnlistableError
+from twisted.python.reflect import namedAny
+from twisted.python.zippath import ZipArchive
+
+_nothing = object()
+
+PYTHON_EXTENSIONS = [".py"]
+OPTIMIZED_MODE = __doc__ is None
+if OPTIMIZED_MODE:
+    PYTHON_EXTENSIONS.append(".pyo")
+else:
+    PYTHON_EXTENSIONS.append(".pyc")
+
+
+def _isPythonIdentifier(string):
+    """
+    cheezy fake test for proper identifier-ness.
+
+    @param string: a L{str} which might or might not be a valid python
+        identifier.
+    @return: True or False
+    """
+    textString = nativeString(string)
+    return " " not in textString and "." not in textString and "-" not in textString
+
+
+def _isPackagePath(fpath):
+    # Determine if a FilePath-like object is a Python package.  TODO: deal with
+    # __init__module.(so|dll|pyd)?
+    extless = fpath.splitext()[0]
+    basend = splitpath(extless)[1]
+    return basend == "__init__"
+
+
+class _ModuleIteratorHelper:
+    """
+    This mixin provides common behavior between python module and path entries,
+    since the mechanism for searching sys.path and __path__ attributes is
+    remarkably similar.
+    """
+
+    def iterModules(self):
+        """
+        Loop over the modules present below this entry or package on PYTHONPATH.
+
+        For modules which are not packages, this will yield nothing.
+
+        For packages and path entries, this will only yield modules one level
+        down; i.e. if there is a package a.b.c, iterModules on a will only
+        return a.b.  If you want to descend deeply, use walkModules.
+
+        @return: a generator which yields PythonModule instances that describe
+        modules which can be, or have been, imported.
+        """
+        yielded = {}
+        if not self.filePath.exists():
+            return
+
+        for placeToLook in self._packagePaths():
+            try:
+                children = sorted(placeToLook.children())
+            except UnlistableError:
+                continue
+
+            for potentialTopLevel in children:
+                ext = potentialTopLevel.splitext()[1]
+                potentialBasename = potentialTopLevel.basename()[: -len(ext)]
+                if ext in PYTHON_EXTENSIONS:
+                    # TODO: this should be a little choosier about which path entry
+                    # it selects first, and it should do all the .so checking and
+                    # crud
+                    if not _isPythonIdentifier(potentialBasename):
+                        continue
+                    modname = self._subModuleName(potentialBasename)
+                    if modname.split(".")[-1] == "__init__":
+                        # This marks the directory as a package so it can't be
+                        # a module.
+                        continue
+                    if modname not in yielded:
+                        yielded[modname] = True
+                        pm = PythonModule(modname, potentialTopLevel, self._getEntry())
+                        assert pm != self
+                        yield pm
+                else:
+                    if (
+                        ext
+                        or not _isPythonIdentifier(potentialBasename)
+                        or not potentialTopLevel.isdir()
+                    ):
+                        continue
+                    modname = self._subModuleName(potentialTopLevel.basename())
+                    for ext in PYTHON_EXTENSIONS:
+                        initpy = potentialTopLevel.child("__init__" + ext)
+                        if initpy.exists() and modname not in yielded:
+                            yielded[modname] = True
+                            pm = PythonModule(modname, initpy, self._getEntry())
+                            assert pm != self
+                            yield pm
+                            break
+
+    def walkModules(self, importPackages=False):
+        """
+        Similar to L{iterModules}, this yields self, and then every module in my
+        package or entry, and every submodule in each package or entry.
+
+        In other words, this is deep, and L{iterModules} is shallow.
+        """
+        yield self
+        for package in self.iterModules():
+            yield from package.walkModules(importPackages=importPackages)
+
+    def _subModuleName(self, mn):
+        """
+        This is a hook to provide packages with the ability to specify their names
+        as a prefix to submodules here.
+        """
+        return mn
+
+    def _packagePaths(self):
+        """
+        Implement in subclasses to specify where to look for modules.
+
+        @return: iterable of FilePath-like objects.
+        """
+        raise NotImplementedError()
+
+    def _getEntry(self):
+        """
+        Implement in subclasses to specify what path entry submodules will come
+        from.
+
+        @return: a PathEntry instance.
+        """
+        raise NotImplementedError()
+
+    def __getitem__(self, modname):
+        """
+        Retrieve a module from below this path or package.
+
+        @param modname: a str naming a module to be loaded.  For entries, this
+        is a top-level, undotted package name, and for packages it is the name
+        of the module without the package prefix.  For example, if you have a
+        PythonModule representing the 'twisted' package, you could use::
+
+            twistedPackageObj['python']['modules']
+
+        to retrieve this module.
+
+        @raise KeyError: if the module is not found.
+
+        @return: a PythonModule.
+        """
+        for module in self.iterModules():
+            if module.name == self._subModuleName(modname):
+                return module
+        raise KeyError(modname)
+
+    def __iter__(self):
+        """
+        Implemented to raise NotImplementedError for clarity, so that attempting to
+        loop over this object won't call __getitem__.
+
+        Note: in the future there might be some sensible default for iteration,
+        like 'walkEverything', so this is deliberately untested and undefined
+        behavior.
+        """
+        raise NotImplementedError()
+
+
+class PythonAttribute:
+    """
+    I represent a function, class, or other object that is present.
+
+    @ivar name: the fully-qualified python name of this attribute.
+
+    @ivar onObject: a reference to a PythonModule or other PythonAttribute that
+    is this attribute's logical parent.
+
+    @ivar name: the fully qualified python name of the attribute represented by
+    this class.
+    """
+
+    def __init__(self, name, onObject, loaded, pythonValue):
+        """
+        Create a PythonAttribute.  This is a private constructor.  Do not construct
+        me directly, use PythonModule.iterAttributes.
+
+        @param name: the FQPN
+        @param onObject: see ivar
+        @param loaded: always True, for now
+        @param pythonValue: the value of the attribute we're pointing to.
+        """
+        self.name: str = name
+        self.onObject = onObject
+        self._loaded = loaded
+        self.pythonValue = pythonValue
+
+    def __repr__(self) -> str:
+        return f"PythonAttribute<{self.name!r}>"
+
+    def isLoaded(self):
+        """
+        Return a boolean describing whether the attribute this describes has
+        actually been loaded into memory by importing its module.
+
+        Note: this currently always returns true; there is no Python parser
+        support in this module yet.
+        """
+        return self._loaded
+
+    def load(self, default=_nothing):
+        """
+        Load the value associated with this attribute.
+
+        @return: an arbitrary Python object, or 'default' if there is an error
+        loading it.
+        """
+        return self.pythonValue
+
+    def iterAttributes(self):
+        for name, val in inspect.getmembers(self.load()):
+            yield PythonAttribute(self.name + "." + name, self, True, val)
+
+
+class PythonModule(_ModuleIteratorHelper):
+    """
+    Representation of a module which could be imported from sys.path.
+
+    @ivar name: the fully qualified python name of this module.
+
+    @ivar filePath: a FilePath-like object which points to the location of this
+    module.
+
+    @ivar pathEntry: a L{PathEntry} instance which this module was located
+    from.
+    """
+
+    def __init__(self, name, filePath, pathEntry):
+        """
+        Create a PythonModule.  Do not construct this directly, instead inspect a
+        PythonPath or other PythonModule instances.
+
+        @param name: see ivar
+        @param filePath: see ivar
+        @param pathEntry: see ivar
+        """
+        _name = nativeString(name)
+        assert not _name.endswith(".__init__")
+        self.name: str = _name
+        self.filePath = filePath
+        self.parentPath = filePath.parent()
+        self.pathEntry = pathEntry
+
+    def _getEntry(self):
+        return self.pathEntry
+
+    def __repr__(self) -> str:
+        """
+        Return a string representation including the module name.
+        """
+        return f"PythonModule<{self.name!r}>"
+
+    def isLoaded(self):
+        """
+        Determine if the module is loaded into sys.modules.
+
+        @return: a boolean: true if loaded, false if not.
+        """
+        return self.pathEntry.pythonPath.moduleDict.get(self.name) is not None
+
+    def iterAttributes(self):
+        """
+        List all the attributes defined in this module.
+
+        Note: Future work is planned here to make it possible to list python
+        attributes on a module without loading the module by inspecting ASTs or
+        bytecode, but currently any iteration of PythonModule objects insists
+        they must be loaded, and will use inspect.getmodule.
+
+        @raise NotImplementedError: if this module is not loaded.
+
+        @return: a generator yielding PythonAttribute instances describing the
+        attributes of this module.
+        """
+        if not self.isLoaded():
+            raise NotImplementedError(
+                "You can't load attributes from non-loaded modules yet."
+            )
+        for name, val in inspect.getmembers(self.load()):
+            yield PythonAttribute(self.name + "." + name, self, True, val)
+
+    def isPackage(self):
+        """
+        Returns true if this module is also a package, and might yield something
+        from iterModules.
+        """
+        return _isPackagePath(self.filePath)
+
+    def load(self, default=_nothing):
+        """
+        Load this module.
+
+        @param default: if specified, the value to return in case of an error.
+
+        @return: a genuine python module.
+
+        @raise Exception: Importing modules is a risky business;
+        the erorrs of any code run at module scope may be raised from here, as
+        well as ImportError if something bizarre happened to the system path
+        between the discovery of this PythonModule object and the attempt to
+        import it.  If you specify a default, the error will be swallowed
+        entirely, and not logged.
+
+        @rtype: types.ModuleType.
+        """
+        try:
+            return self.pathEntry.pythonPath.moduleLoader(self.name)
+        except BaseException:  # this needs more thought...
+            if default is not _nothing:
+                return default
+            raise
+
+    def __eq__(self, other: object) -> bool:
+        """
+        PythonModules with the same name are equal.
+        """
+        if isinstance(other, PythonModule):
+            return other.name == self.name
+        return NotImplemented
+
+    def walkModules(self, importPackages=False):
+        if importPackages and self.isPackage():
+            self.load()
+        return super().walkModules(importPackages=importPackages)
+
+    def _subModuleName(self, mn):
+        """
+        submodules of this module are prefixed with our name.
+        """
+        return self.name + "." + mn
+
+    def _packagePaths(self):
+        """
+        Yield a sequence of FilePath-like objects which represent path segments.
+        """
+        if not self.isPackage():
+            return
+        if self.isLoaded():
+            load = self.load()
+            if hasattr(load, "__path__"):
+                for fn in load.__path__:
+                    if fn == self.parentPath.path:
+                        # this should _really_ exist.
+                        assert self.parentPath.exists()
+                        yield self.parentPath
+                    else:
+                        smp = self.pathEntry.pythonPath._smartPath(fn)
+                        if smp.exists():
+                            yield smp
+        else:
+            yield self.parentPath
+
+
+class PathEntry(_ModuleIteratorHelper):
+    """
+    I am a proxy for a single entry on sys.path.
+
+    @ivar filePath: a FilePath-like object pointing at the filesystem location
+    or archive file where this path entry is stored.
+
+    @ivar pythonPath: a PythonPath instance.
+    """
+
+    def __init__(self, filePath, pythonPath):
+        """
+        Create a PathEntry.  This is a private constructor.
+        """
+        self.filePath = filePath
+        self.pythonPath = pythonPath
+
+    def _getEntry(self):
+        return self
+
+    def __repr__(self) -> str:
+        return f"PathEntry<{self.filePath!r}>"
+
+    def _packagePaths(self):
+        yield self.filePath
+
+
+class IPathImportMapper(Interface):
+    """
+    This is an internal interface, used to map importers to factories for
+    FilePath-like objects.
+    """
+
+    def mapPath(pathLikeString):
+        """
+        Return a FilePath-like object.
+
+        @param pathLikeString: a path-like string, like one that might be
+        passed to an import hook.
+
+        @return: a L{FilePath}, or something like it (currently only a
+        L{ZipPath}, but more might be added later).
+        """
+
+
+@implementer(IPathImportMapper)
+class _DefaultMapImpl:
+    """Wrapper for the default importer, i.e. None."""
+
+    def mapPath(self, fsPathString):
+        return FilePath(fsPathString)
+
+
+_theDefaultMapper = _DefaultMapImpl()
+
+
+@implementer(IPathImportMapper)
+class _ZipMapImpl:
+    """IPathImportMapper implementation for zipimport.ZipImporter."""
+
+    def __init__(self, importer):
+        self.importer = importer
+
+    def mapPath(self, fsPathString):
+        """
+        Map the given FS path to a ZipPath, by looking at the ZipImporter's
+        "archive" attribute and using it as our ZipArchive root, then walking
+        down into the archive from there.
+
+        @return: a L{zippath.ZipPath} or L{zippath.ZipArchive} instance.
+        """
+        za = ZipArchive(self.importer.archive)
+        myPath = FilePath(self.importer.archive)
+        itsPath = FilePath(fsPathString)
+        if myPath == itsPath:
+            return za
+        # This is NOT a general-purpose rule for sys.path or __file__:
+        # zipimport specifically uses regular OS path syntax in its
+        # pathnames, even though zip files specify that slashes are always
+        # the separator, regardless of platform.
+        segs = itsPath.segmentsFrom(myPath)
+        zp = za
+        for seg in segs:
+            zp = zp.child(seg)
+        return zp
+
+
+registerAdapter(_ZipMapImpl, zipimport.zipimporter, IPathImportMapper)
+
+
+def _defaultSysPathFactory():
+    """
+    Provide the default behavior of PythonPath's sys.path factory, which is to
+    return the current value of sys.path.
+
+    @return: L{sys.path}
+    """
+    return sys.path
+
+
+class PythonPath:
+    """
+    I represent the very top of the Python object-space, the module list in
+    C{sys.path} and the modules list in C{sys.modules}.
+
+    @ivar _sysPath: A sequence of strings like C{sys.path}.  This attribute is
+    read-only.
+
+    @ivar sysPath: The current value of the module search path list.
+    @type sysPath: C{list}
+
+    @ivar moduleDict: A dictionary mapping string module names to module
+    objects, like C{sys.modules}.
+
+    @ivar sysPathHooks: A list of PEP-302 path hooks, like C{sys.path_hooks}.
+
+    @ivar moduleLoader: A function that takes a fully-qualified python name and
+    returns a module, like L{twisted.python.reflect.namedAny}.
+    """
+
+    def __init__(
+        self,
+        sysPath=None,
+        moduleDict=sys.modules,
+        sysPathHooks=sys.path_hooks,
+        importerCache=sys.path_importer_cache,
+        moduleLoader=namedAny,
+        sysPathFactory=None,
+    ):
+        """
+        Create a PythonPath.  You almost certainly want to use
+        modules.theSystemPath, or its aliased methods, rather than creating a
+        new instance yourself, though.
+
+        All parameters are optional, and if unspecified, will use 'system'
+        equivalents that makes this PythonPath like the global L{theSystemPath}
+        instance.
+
+        @param sysPath: a sys.path-like list to use for this PythonPath, to
+        specify where to load modules from.
+
+        @param moduleDict: a sys.modules-like dictionary to use for keeping
+        track of what modules this PythonPath has loaded.
+
+        @param sysPathHooks: sys.path_hooks-like list of PEP-302 path hooks to
+        be used for this PythonPath, to determie which importers should be
+        used.
+
+        @param importerCache: a sys.path_importer_cache-like list of PEP-302
+        importers.  This will be used in conjunction with the given
+        sysPathHooks.
+
+        @param moduleLoader: a module loader function which takes a string and
+        returns a module.  That is to say, it is like L{namedAny} - *not* like
+        L{__import__}.
+
+        @param sysPathFactory: a 0-argument callable which returns the current
+        value of a sys.path-like list of strings.  Specify either this, or
+        sysPath, not both.  This alternative interface is provided because the
+        way the Python import mechanism works, you can re-bind the 'sys.path'
+        name and that is what is used for current imports, so it must be a
+        factory rather than a value to deal with modification by rebinding
+        rather than modification by mutation.  Note: it is not recommended to
+        rebind sys.path.  Although this mechanism can deal with that, it is a
+        subtle point which some tools that it is easy for tools which interact
+        with sys.path to miss.
+        """
+        if sysPath is not None:
+            sysPathFactory = lambda: sysPath
+        elif sysPathFactory is None:
+            sysPathFactory = _defaultSysPathFactory
+        self._sysPathFactory = sysPathFactory
+        self._sysPath = sysPath
+        self.moduleDict = moduleDict
+        self.sysPathHooks = sysPathHooks
+        self.importerCache = importerCache
+        self.moduleLoader = moduleLoader
+
+    @property
+    def sysPath(self):
+        """
+        Retrieve the current value of the module search path list.
+        """
+        return self._sysPathFactory()
+
+    def _findEntryPathString(self, modobj):
+        """
+        Determine where a given Python module object came from by looking at path
+        entries.
+        """
+        topPackageObj = modobj
+        while "." in topPackageObj.__name__:
+            topPackageObj = self.moduleDict[
+                ".".join(topPackageObj.__name__.split(".")[:-1])
+            ]
+        if _isPackagePath(FilePath(topPackageObj.__file__)):
+            # if package 'foo' is on sys.path at /a/b/foo, package 'foo's
+            # __file__ will be /a/b/foo/__init__.py, and we are looking for
+            # /a/b here, the path-entry; so go up two steps.
+            rval = dirname(dirname(topPackageObj.__file__))
+        else:
+            # the module is completely top-level, not within any packages.  The
+            # path entry it's on is just its dirname.
+            rval = dirname(topPackageObj.__file__)
+
+        # There are probably some awful tricks that an importer could pull
+        # which would break this, so let's just make sure... it's a loaded
+        # module after all, which means that its path MUST be in
+        # path_importer_cache according to PEP 302 -glyph
+        if rval not in self.importerCache:
+            warnings.warn(
+                "%s (for module %s) not in path importer cache "
+                "(PEP 302 violation - check your local configuration)."
+                % (rval, modobj.__name__),
+                stacklevel=3,
+            )
+
+        return rval
+
+    def _smartPath(self, pathName):
+        """
+        Given a path entry from sys.path which may refer to an importer,
+        return the appropriate FilePath-like instance.
+
+        @param pathName: a str describing the path.
+
+        @return: a FilePath-like object.
+        """
+        importr = self.importerCache.get(pathName, _nothing)
+        if importr is _nothing:
+            for hook in self.sysPathHooks:
+                try:
+                    importr = hook(pathName)
+                except ImportError:
+                    pass
+            if importr is _nothing:  # still
+                importr = None
+        return IPathImportMapper(importr, _theDefaultMapper).mapPath(pathName)
+
+    def iterEntries(self):
+        """
+        Iterate the entries on my sysPath.
+
+        @return: a generator yielding PathEntry objects
+        """
+        for pathName in self.sysPath:
+            fp = self._smartPath(pathName)
+            yield PathEntry(fp, self)
+
+    def __getitem__(self, modname):
+        """
+        Get a python module by its given fully-qualified name.
+
+        @param modname: The fully-qualified Python module name to load.
+
+        @type modname: C{str}
+
+        @return: an object representing the module identified by C{modname}
+
+        @rtype: L{PythonModule}
+
+        @raise KeyError: if the module name is not a valid module name, or no
+            such module can be identified as loadable.
+        """
+        # See if the module is already somewhere in Python-land.
+        moduleObject = self.moduleDict.get(modname)
+        if moduleObject is not None:
+            # we need 2 paths; one of the path entry and one for the module.
+            pe = PathEntry(
+                self._smartPath(self._findEntryPathString(moduleObject)), self
+            )
+            mp = self._smartPath(moduleObject.__file__)
+            return PythonModule(modname, mp, pe)
+
+        # Recurse if we're trying to get a submodule.
+        if "." in modname:
+            pkg = self
+            for name in modname.split("."):
+                pkg = pkg[name]
+            return pkg
+
+        # Finally do the slowest possible thing and iterate
+        for module in self.iterModules():
+            if module.name == modname:
+                return module
+        raise KeyError(modname)
+
+    def __contains__(self, module):
+        """
+        Check to see whether or not a module exists on my import path.
+
+        @param module: The name of the module to look for on my import path.
+        @type module: C{str}
+        """
+        try:
+            self.__getitem__(module)
+            return True
+        except KeyError:
+            return False
+
+    def __repr__(self) -> str:
+        """
+        Display my sysPath and moduleDict in a string representation.
+        """
+        return f"PythonPath({self.sysPath!r},{self.moduleDict!r})"
+
+    def iterModules(self):
+        """
+        Yield all top-level modules on my sysPath.
+        """
+        for entry in self.iterEntries():
+            yield from entry.iterModules()
+
+    def walkModules(self, importPackages=False):
+        """
+        Similar to L{iterModules}, this yields every module on the path, then every
+        submodule in each package or entry.
+        """
+        for package in self.iterModules():
+            yield from package.walkModules(importPackages=False)
+
+
+theSystemPath = PythonPath()
+
+
+def walkModules(importPackages=False):
+    """
+    Deeply iterate all modules on the global python path.
+
+    @param importPackages: Import packages as they are seen.
+    """
+    return theSystemPath.walkModules(importPackages=importPackages)
+
+
+def iterModules():
+    """
+    Iterate all modules and top-level packages on the global Python path, but
+    do not descend into packages.
+    """
+    return theSystemPath.iterModules()
+
+
+def getModule(moduleName):
+    """
+    Retrieve a module from the system path.
+    """
+    return theSystemPath[moduleName]
diff --git a/contrib/python/Twisted/py3/twisted/python/monkey.py b/contrib/python/Twisted/py3/twisted/python/monkey.py
new file mode 100644
index 00000000000..08ccef2ac1f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/monkey.py
@@ -0,0 +1,73 @@
+# -*- test-case-name: twisted.test.test_monkey -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+class MonkeyPatcher:
+    """
+    Cover up attributes with new objects. Neat for monkey-patching things for
+    unit-testing purposes.
+    """
+
+    def __init__(self, *patches):
+        # List of patches to apply in (obj, name, value).
+        self._patchesToApply = []
+        # List of the original values for things that have been patched.
+        # (obj, name, value) format.
+        self._originals = []
+        for patch in patches:
+            self.addPatch(*patch)
+
+    def addPatch(self, obj, name, value):
+        """
+        Add a patch so that the attribute C{name} on C{obj} will be assigned to
+        C{value} when C{patch} is called or during C{runWithPatches}.
+
+        You can restore the original values with a call to restore().
+        """
+        self._patchesToApply.append((obj, name, value))
+
+    def _alreadyPatched(self, obj, name):
+        """
+        Has the C{name} attribute of C{obj} already been patched by this
+        patcher?
+        """
+        for o, n, v in self._originals:
+            if (o, n) == (obj, name):
+                return True
+        return False
+
+    def patch(self):
+        """
+        Apply all of the patches that have been specified with L{addPatch}.
+        Reverse this operation using L{restore}.
+        """
+        for obj, name, value in self._patchesToApply:
+            if not self._alreadyPatched(obj, name):
+                self._originals.append((obj, name, getattr(obj, name)))
+            setattr(obj, name, value)
+
+    __enter__ = patch
+
+    def restore(self):
+        """
+        Restore all original values to any patched objects.
+        """
+        while self._originals:
+            obj, name, value = self._originals.pop()
+            setattr(obj, name, value)
+
+    def __exit__(self, excType=None, excValue=None, excTraceback=None):
+        self.restore()
+
+    def runWithPatches(self, f, *args, **kw):
+        """
+        Apply each patch already specified. Then run the function f with the
+        given args and kwargs. Restore everything when done.
+        """
+        self.patch()
+        try:
+            return f(*args, **kw)
+        finally:
+            self.restore()
diff --git a/contrib/python/Twisted/py3/twisted/python/procutils.py b/contrib/python/Twisted/py3/twisted/python/procutils.py
new file mode 100644
index 00000000000..c97357a3885
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/procutils.py
@@ -0,0 +1,50 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Utilities for dealing with processes.
+"""
+
+
+import os
+
+
+def which(name, flags=os.X_OK):
+    """
+    Search PATH for executable files with the given name.
+
+    On newer versions of MS-Windows, the PATHEXT environment variable will be
+    set to the list of file extensions for files considered executable. This
+    will normally include things like ".EXE". This function will also find files
+    with the given name ending with any of these extensions.
+
+    On MS-Windows the only flag that has any meaning is os.F_OK. Any other
+    flags will be ignored.
+
+    @type name: C{str}
+    @param name: The name for which to search.
+
+    @type flags: C{int}
+    @param flags: Arguments to L{os.access}.
+
+    @rtype: C{list}
+    @return: A list of the full paths to files found, in the order in which they
+    were found.
+    """
+    result = []
+    exts = list(filter(None, os.environ.get("PATHEXT", "").split(os.pathsep)))
+    path = os.environ.get("PATH", None)
+
+    if path is None:
+        return []
+
+    for p in os.environ.get("PATH", "").split(os.pathsep):
+        p = os.path.join(p, name)
+        if os.access(p, flags):
+            result.append(p)
+        for e in exts:
+            pext = p + e
+            if os.access(pext, flags):
+                result.append(pext)
+
+    return result
diff --git a/contrib/python/Twisted/py3/twisted/python/randbytes.py b/contrib/python/Twisted/py3/twisted/python/randbytes.py
new file mode 100644
index 00000000000..170a18edee4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/randbytes.py
@@ -0,0 +1,128 @@
+# -*- test-case-name: twisted.test.test_randbytes -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Cryptographically secure random implementation, with fallback on normal random.
+"""
+
+
+import os
+import random
+import warnings
+
+getrandbits = getattr(random, "getrandbits", None)
+
+_fromhex = bytes.fromhex
+
+
+class SecureRandomNotAvailable(RuntimeError):
+    """
+    Exception raised when no secure random algorithm is found.
+    """
+
+
+class SourceNotAvailable(RuntimeError):
+    """
+    Internal exception used when a specific random source is not available.
+    """
+
+
+class RandomFactory:
+    """
+    Factory providing L{secureRandom} and L{insecureRandom} methods.
+
+    You shouldn't have to instantiate this class, use the module level
+    functions instead: it is an implementation detail and could be removed or
+    changed arbitrarily.
+    """
+
+    # This variable is no longer used, and will eventually be removed.
+    randomSources = ()
+
+    getrandbits = getrandbits
+
+    def _osUrandom(self, nbytes: int) -> bytes:
+        """
+        Wrapper around C{os.urandom} that cleanly manage its absence.
+        """
+        try:
+            return os.urandom(nbytes)
+        except (AttributeError, NotImplementedError) as e:
+            raise SourceNotAvailable(e)
+
+    def secureRandom(self, nbytes: int, fallback: bool = False) -> bytes:
+        """
+        Return a number of secure random bytes.
+
+        @param nbytes: number of bytes to generate.
+        @type nbytes: C{int}
+        @param fallback: Whether the function should fallback on non-secure
+            random or not.  Default to C{False}.
+        @type fallback: C{bool}
+
+        @return: a string of random bytes.
+        @rtype: C{str}
+        """
+        try:
+            return self._osUrandom(nbytes)
+        except SourceNotAvailable:
+            pass
+
+        if fallback:
+            warnings.warn(
+                "urandom unavailable - "
+                "proceeding with non-cryptographically secure random source",
+                category=RuntimeWarning,
+                stacklevel=2,
+            )
+            return self.insecureRandom(nbytes)
+        else:
+            raise SecureRandomNotAvailable("No secure random source available")
+
+    def _randBits(self, nbytes: int) -> bytes:
+        """
+        Wrapper around C{os.getrandbits}.
+        """
+        if self.getrandbits is not None:
+            n = self.getrandbits(nbytes * 8)
+            hexBytes = ("%%0%dx" % (nbytes * 2)) % n
+            return _fromhex(hexBytes)
+        raise SourceNotAvailable("random.getrandbits is not available")
+
+    _maketrans = bytes.maketrans
+    _BYTES = _maketrans(b"", b"")
+
+    def _randModule(self, nbytes: int) -> bytes:
+        """
+        Wrapper around the C{random} module.
+        """
+        return b"".join([bytes([random.choice(self._BYTES)]) for i in range(nbytes)])
+
+    def insecureRandom(self, nbytes: int) -> bytes:
+        """
+        Return a number of non secure random bytes.
+
+        @param nbytes: number of bytes to generate.
+        @type nbytes: C{int}
+
+        @return: a string of random bytes.
+        @rtype: C{str}
+        """
+        try:
+            return self._randBits(nbytes)
+        except SourceNotAvailable:
+            pass
+        return self._randModule(nbytes)
+
+
+factory = RandomFactory()
+
+secureRandom = factory.secureRandom
+
+insecureRandom = factory.insecureRandom
+
+del factory
+
+
+__all__ = ["secureRandom", "insecureRandom", "SecureRandomNotAvailable"]
diff --git a/contrib/python/Twisted/py3/twisted/python/rebuild.py b/contrib/python/Twisted/py3/twisted/python/rebuild.py
new file mode 100644
index 00000000000..e82beec1b6a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/rebuild.py
@@ -0,0 +1,250 @@
+# -*- test-case-name: twisted.test.test_rebuild -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+*Real* reloading support for Python.
+"""
+
+import linecache
+
+# System Imports
+import sys
+import time
+import types
+from importlib import reload
+from types import ModuleType
+from typing import Dict
+
+# Sibling Imports
+from twisted.python import log, reflect
+
+lastRebuild = time.time()
+
+
+class Sensitive:
+    """
+    A utility mixin that's sensitive to rebuilds.
+
+    This is a mixin for classes (usually those which represent collections of
+    callbacks) to make sure that their code is up-to-date before running.
+    """
+
+    lastRebuild = lastRebuild
+
+    def needRebuildUpdate(self):
+        yn = self.lastRebuild < lastRebuild
+        return yn
+
+    def rebuildUpToDate(self):
+        self.lastRebuild = time.time()
+
+    def latestVersionOf(self, anObject):
+        """
+        Get the latest version of an object.
+
+        This can handle just about anything callable; instances, functions,
+        methods, and classes.
+        """
+        t = type(anObject)
+        if t == types.FunctionType:
+            return latestFunction(anObject)
+        elif t == types.MethodType:
+            if anObject.__self__ is None:
+                return getattr(anObject.im_class, anObject.__name__)
+            else:
+                return getattr(anObject.__self__, anObject.__name__)
+        else:
+            log.msg("warning returning anObject!")
+            return anObject
+
+
+_modDictIDMap: Dict[int, ModuleType] = {}
+
+
+def latestFunction(oldFunc):
+    """
+    Get the latest version of a function.
+    """
+    # This may be CPython specific, since I believe jython instantiates a new
+    # module upon reload.
+    dictID = id(oldFunc.__globals__)
+    module = _modDictIDMap.get(dictID)
+    if module is None:
+        return oldFunc
+    return getattr(module, oldFunc.__name__)
+
+
+def latestClass(oldClass):
+    """
+    Get the latest version of a class.
+    """
+    module = reflect.namedModule(oldClass.__module__)
+    newClass = getattr(module, oldClass.__name__)
+    newBases = [latestClass(base) for base in newClass.__bases__]
+
+    if newClass.__module__ == "builtins":
+        # builtin members can't be reloaded sanely
+        return newClass
+
+    try:
+        # This makes old-style stuff work
+        newClass.__bases__ = tuple(newBases)
+        return newClass
+    except TypeError:
+        ctor = type(newClass)
+        return ctor(newClass.__name__, tuple(newBases), dict(newClass.__dict__))
+
+
+class RebuildError(Exception):
+    """
+    Exception raised when trying to rebuild a class whereas it's not possible.
+    """
+
+
+def updateInstance(self):
+    """
+    Updates an instance to be current.
+    """
+    self.__class__ = latestClass(self.__class__)
+
+
+def __injectedgetattr__(self, name):
+    """
+    A getattr method to cause a class to be refreshed.
+    """
+    if name == "__del__":
+        raise AttributeError("Without this, Python segfaults.")
+    updateInstance(self)
+    log.msg(f"(rebuilding stale {reflect.qual(self.__class__)} instance ({name}))")
+    result = getattr(self, name)
+    return result
+
+
+def rebuild(module, doLog=1):
+    """
+    Reload a module and do as much as possible to replace its references.
+    """
+    global lastRebuild
+    lastRebuild = time.time()
+    if hasattr(module, "ALLOW_TWISTED_REBUILD"):
+        # Is this module allowed to be rebuilt?
+        if not module.ALLOW_TWISTED_REBUILD:
+            raise RuntimeError("I am not allowed to be rebuilt.")
+    if doLog:
+        log.msg(f"Rebuilding {str(module.__name__)}...")
+
+    # Safely handle adapter re-registration
+    from twisted.python import components
+
+    components.ALLOW_DUPLICATES = True
+
+    d = module.__dict__
+    _modDictIDMap[id(d)] = module
+    newclasses = {}
+    classes = {}
+    functions = {}
+    values = {}
+    if doLog:
+        log.msg(f"  (scanning {str(module.__name__)}): ")
+    for k, v in d.items():
+        if issubclass(type(v), types.FunctionType):
+            if v.__globals__ is module.__dict__:
+                functions[v] = 1
+                if doLog:
+                    log.logfile.write("f")
+                    log.logfile.flush()
+        elif isinstance(v, type):
+            if v.__module__ == module.__name__:
+                newclasses[v] = 1
+                if doLog:
+                    log.logfile.write("o")
+                    log.logfile.flush()
+
+    values.update(classes)
+    values.update(functions)
+    fromOldModule = values.__contains__
+    newclasses = newclasses.keys()
+    classes = classes.keys()
+    functions = functions.keys()
+
+    if doLog:
+        log.msg("")
+        log.msg(f"  (reload   {str(module.__name__)})")
+
+    # Boom.
+    reload(module)
+    # Make sure that my traceback printing will at least be recent...
+    linecache.clearcache()
+
+    if doLog:
+        log.msg(f"  (cleaning {str(module.__name__)}): ")
+
+    for clazz in classes:
+        if getattr(module, clazz.__name__) is clazz:
+            log.msg(f"WARNING: class {reflect.qual(clazz)} not replaced by reload!")
+        else:
+            if doLog:
+                log.logfile.write("x")
+                log.logfile.flush()
+            clazz.__bases__ = ()
+            clazz.__dict__.clear()
+            clazz.__getattr__ = __injectedgetattr__
+            clazz.__module__ = module.__name__
+    if newclasses:
+        import gc
+    for nclass in newclasses:
+        ga = getattr(module, nclass.__name__)
+        if ga is nclass:
+            log.msg(
+                "WARNING: new-class {} not replaced by reload!".format(
+                    reflect.qual(nclass)
+                )
+            )
+        else:
+            for r in gc.get_referrers(nclass):
+                if getattr(r, "__class__", None) is nclass:
+                    r.__class__ = ga
+    if doLog:
+        log.msg("")
+        log.msg(f"  (fixing   {str(module.__name__)}): ")
+    modcount = 0
+    for mk, mod in sys.modules.items():
+        modcount = modcount + 1
+        if mod == module or mod is None:
+            continue
+
+        if not hasattr(mod, "__file__"):
+            # It's a builtin module; nothing to replace here.
+            continue
+
+        if hasattr(mod, "__bundle__"):
+            # PyObjC has a few buggy objects which segfault if you hash() them.
+            # It doesn't make sense to try rebuilding extension modules like
+            # this anyway, so don't try.
+            continue
+
+        changed = 0
+
+        for k, v in mod.__dict__.items():
+            try:
+                hash(v)
+            except Exception:
+                continue
+            if fromOldModule(v):
+                if doLog:
+                    log.logfile.write("f")
+                    log.logfile.flush()
+                nv = latestFunction(v)
+                changed = 1
+                setattr(mod, k, nv)
+        if doLog and not changed and ((modcount % 10) == 0):
+            log.logfile.write(".")
+            log.logfile.flush()
+
+    components.ALLOW_DUPLICATES = False
+    if doLog:
+        log.msg("")
+        log.msg(f"   Rebuilt {str(module.__name__)}.")
+    return module
diff --git a/contrib/python/Twisted/py3/twisted/python/reflect.py b/contrib/python/Twisted/py3/twisted/python/reflect.py
new file mode 100644
index 00000000000..9991b4eb63a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/reflect.py
@@ -0,0 +1,686 @@
+# -*- test-case-name: twisted.test.test_reflect -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Standardized versions of various cool and/or strange things that you can do
+with Python's reflection capabilities.
+"""
+
+
+import os
+import pickle
+import re
+import sys
+import traceback
+import types
+import weakref
+from collections import deque
+from io import IOBase, StringIO
+from typing import Type, Union
+
+from twisted.python.compat import nativeString
+from twisted.python.deprecate import _fullyQualifiedName as fullyQualifiedName
+
+RegexType = type(re.compile(""))
+
+
+def prefixedMethodNames(classObj, prefix):
+    """
+    Given a class object C{classObj}, returns a list of method names that match
+    the string C{prefix}.
+
+    @param classObj: A class object from which to collect method names.
+
+    @param prefix: A native string giving a prefix.  Each method with a name
+        which begins with this prefix will be returned.
+    @type prefix: L{str}
+
+    @return: A list of the names of matching methods of C{classObj} (and base
+        classes of C{classObj}).
+    @rtype: L{list} of L{str}
+    """
+    dct = {}
+    addMethodNamesToDict(classObj, dct, prefix)
+    return list(dct.keys())
+
+
+def addMethodNamesToDict(classObj, dict, prefix, baseClass=None):
+    """
+    This goes through C{classObj} (and its bases) and puts method names
+    starting with 'prefix' in 'dict' with a value of 1. if baseClass isn't
+    None, methods will only be added if classObj is-a baseClass
+
+    If the class in question has the methods 'prefix_methodname' and
+    'prefix_methodname2', the resulting dict should look something like:
+    {"methodname": 1, "methodname2": 1}.
+
+    @param classObj: A class object from which to collect method names.
+
+    @param dict: A L{dict} which will be updated with the results of the
+        accumulation.  Items are added to this dictionary, with method names as
+        keys and C{1} as values.
+    @type dict: L{dict}
+
+    @param prefix: A native string giving a prefix.  Each method of C{classObj}
+        (and base classes of C{classObj}) with a name which begins with this
+        prefix will be returned.
+    @type prefix: L{str}
+
+    @param baseClass: A class object at which to stop searching upwards for new
+        methods.  To collect all method names, do not pass a value for this
+        parameter.
+
+    @return: L{None}
+    """
+    for base in classObj.__bases__:
+        addMethodNamesToDict(base, dict, prefix, baseClass)
+
+    if baseClass is None or baseClass in classObj.__bases__:
+        for name, method in classObj.__dict__.items():
+            optName = name[len(prefix) :]
+            if (
+                (type(method) is types.FunctionType)
+                and (name[: len(prefix)] == prefix)
+                and (len(optName))
+            ):
+                dict[optName] = 1
+
+
+def prefixedMethods(obj, prefix=""):
+    """
+    Given an object C{obj}, returns a list of method objects that match the
+    string C{prefix}.
+
+    @param obj: An arbitrary object from which to collect methods.
+
+    @param prefix: A native string giving a prefix.  Each method of C{obj} with
+        a name which begins with this prefix will be returned.
+    @type prefix: L{str}
+
+    @return: A list of the matching method objects.
+    @rtype: L{list}
+    """
+    dct = {}
+    accumulateMethods(obj, dct, prefix)
+    return list(dct.values())
+
+
+def accumulateMethods(obj, dict, prefix="", curClass=None):
+    """
+    Given an object C{obj}, add all methods that begin with C{prefix}.
+
+    @param obj: An arbitrary object to collect methods from.
+
+    @param dict: A L{dict} which will be updated with the results of the
+        accumulation.  Items are added to this dictionary, with method names as
+        keys and corresponding instance method objects as values.
+    @type dict: L{dict}
+
+    @param prefix: A native string giving a prefix.  Each method of C{obj} with
+        a name which begins with this prefix will be returned.
+    @type prefix: L{str}
+
+    @param curClass: The class in the inheritance hierarchy at which to start
+        collecting methods.  Collection proceeds up.  To collect all methods
+        from C{obj}, do not pass a value for this parameter.
+
+    @return: L{None}
+    """
+    if not curClass:
+        curClass = obj.__class__
+    for base in curClass.__bases__:
+        # The implementation of the object class is different on PyPy vs.
+        # CPython.  This has the side effect of making accumulateMethods()
+        # pick up object methods from all new-style classes -
+        # such as __getattribute__, etc.
+        # If we ignore 'object' when accumulating methods, we can get
+        # consistent behavior on Pypy and CPython.
+        if base is not object:
+            accumulateMethods(obj, dict, prefix, base)
+
+    for name, method in curClass.__dict__.items():
+        optName = name[len(prefix) :]
+        if (
+            (type(method) is types.FunctionType)
+            and (name[: len(prefix)] == prefix)
+            and (len(optName))
+        ):
+            dict[optName] = getattr(obj, name)
+
+
+def namedModule(name):
+    """
+    Return a module given its name.
+    """
+    topLevel = __import__(name)
+    packages = name.split(".")[1:]
+    m = topLevel
+    for p in packages:
+        m = getattr(m, p)
+    return m
+
+
+def namedObject(name):
+    """
+    Get a fully named module-global object.
+    """
+    classSplit = name.split(".")
+    module = namedModule(".".join(classSplit[:-1]))
+    return getattr(module, classSplit[-1])
+
+
+namedClass = namedObject  # backwards compat
+
+
+def requireModule(name, default=None):
+    """
+    Try to import a module given its name, returning C{default} value if
+    C{ImportError} is raised during import.
+
+    @param name: Module name as it would have been passed to C{import}.
+    @type name: C{str}.
+
+    @param default: Value returned in case C{ImportError} is raised while
+        importing the module.
+
+    @return: Module or default value.
+    """
+    try:
+        return namedModule(name)
+    except ImportError:
+        return default
+
+
+class _NoModuleFound(Exception):
+    """
+    No module was found because none exists.
+    """
+
+
+class InvalidName(ValueError):
+    """
+    The given name is not a dot-separated list of Python objects.
+    """
+
+
+class ModuleNotFound(InvalidName):
+    """
+    The module associated with the given name doesn't exist and it can't be
+    imported.
+    """
+
+
+class ObjectNotFound(InvalidName):
+    """
+    The object associated with the given name doesn't exist and it can't be
+    imported.
+    """
+
+
+def _importAndCheckStack(importName):
+    """
+    Import the given name as a module, then walk the stack to determine whether
+    the failure was the module not existing, or some code in the module (for
+    example a dependent import) failing.  This can be helpful to determine
+    whether any actual application code was run.  For example, to distiguish
+    administrative error (entering the wrong module name), from programmer
+    error (writing buggy code in a module that fails to import).
+
+    @param importName: The name of the module to import.
+    @type importName: C{str}
+    @raise Exception: if something bad happens.  This can be any type of
+        exception, since nobody knows what loading some arbitrary code might
+        do.
+    @raise _NoModuleFound: if no module was found.
+    """
+    try:
+        return __import__(importName)
+    except ImportError:
+        excType, excValue, excTraceback = sys.exc_info()
+        while excTraceback:
+            execName = excTraceback.tb_frame.f_globals["__name__"]
+            if execName == importName:
+                raise excValue.with_traceback(excTraceback)
+            excTraceback = excTraceback.tb_next
+        raise _NoModuleFound()
+
+
+def namedAny(name):
+    """
+    Retrieve a Python object by its fully qualified name from the global Python
+    module namespace.  The first part of the name, that describes a module,
+    will be discovered and imported.  Each subsequent part of the name is
+    treated as the name of an attribute of the object specified by all of the
+    name which came before it.  For example, the fully-qualified name of this
+    object is 'twisted.python.reflect.namedAny'.
+
+    @type name: L{str}
+    @param name: The name of the object to return.
+
+    @raise InvalidName: If the name is an empty string, starts or ends with
+        a '.', or is otherwise syntactically incorrect.
+
+    @raise ModuleNotFound: If the name is syntactically correct but the
+        module it specifies cannot be imported because it does not appear to
+        exist.
+
+    @raise ObjectNotFound: If the name is syntactically correct, includes at
+        least one '.', but the module it specifies cannot be imported because
+        it does not appear to exist.
+
+    @raise AttributeError: If an attribute of an object along the way cannot be
+        accessed, or a module along the way is not found.
+
+    @return: the Python object identified by 'name'.
+    """
+    if not name:
+        raise InvalidName("Empty module name")
+
+    names = name.split(".")
+
+    # if the name starts or ends with a '.' or contains '..', the __import__
+    # will raise an 'Empty module name' error. This will provide a better error
+    # message.
+    if "" in names:
+        raise InvalidName(
+            "name must be a string giving a '.'-separated list of Python "
+            "identifiers, not %r" % (name,)
+        )
+
+    topLevelPackage = None
+    moduleNames = names[:]
+    while not topLevelPackage:
+        if moduleNames:
+            trialname = ".".join(moduleNames)
+            try:
+                topLevelPackage = _importAndCheckStack(trialname)
+            except _NoModuleFound:
+                moduleNames.pop()
+        else:
+            if len(names) == 1:
+                raise ModuleNotFound(f"No module named {name!r}")
+            else:
+                raise ObjectNotFound(f"{name!r} does not name an object")
+
+    obj = topLevelPackage
+    for n in names[1:]:
+        obj = getattr(obj, n)
+
+    return obj
+
+
+def filenameToModuleName(fn):
+    """
+    Convert a name in the filesystem to the name of the Python module it is.
+
+    This is aggressive about getting a module name back from a file; it will
+    always return a string.  Aggressive means 'sometimes wrong'; it won't look
+    at the Python path or try to do any error checking: don't use this method
+    unless you already know that the filename you're talking about is a Python
+    module.
+
+    @param fn: A filesystem path to a module or package; C{bytes} on Python 2,
+        C{bytes} or C{unicode} on Python 3.
+
+    @return: A hopefully importable module name.
+    @rtype: C{str}
+    """
+    if isinstance(fn, bytes):
+        initPy = b"__init__.py"
+    else:
+        initPy = "__init__.py"
+    fullName = os.path.abspath(fn)
+    base = os.path.basename(fn)
+    if not base:
+        # this happens when fn ends with a path separator, just skit it
+        base = os.path.basename(fn[:-1])
+    modName = nativeString(os.path.splitext(base)[0])
+    while 1:
+        fullName = os.path.dirname(fullName)
+        if os.path.exists(os.path.join(fullName, initPy)):
+            modName = "{}.{}".format(
+                nativeString(os.path.basename(fullName)),
+                nativeString(modName),
+            )
+        else:
+            break
+    return modName
+
+
+def qual(clazz: Type[object]) -> str:
+    """
+    Return full import path of a class.
+    """
+    return clazz.__module__ + "." + clazz.__name__
+
+
+def _determineClass(x):
+    try:
+        return x.__class__
+    except BaseException:
+        return type(x)
+
+
+def _determineClassName(x):
+    c = _determineClass(x)
+    try:
+        return c.__name__
+    except BaseException:
+        try:
+            return str(c)
+        except BaseException:
+            return "<BROKEN CLASS AT 0x%x>" % id(c)
+
+
+def _safeFormat(formatter: Union[types.FunctionType, Type[str]], o: object) -> str:
+    """
+    Helper function for L{safe_repr} and L{safe_str}.
+
+    Called when C{repr} or C{str} fail. Returns a string containing info about
+    C{o} and the latest exception.
+
+    @param formatter: C{str} or C{repr}.
+    @type formatter: C{type}
+    @param o: Any object.
+
+    @rtype: C{str}
+    @return: A string containing information about C{o} and the raised
+        exception.
+    """
+    io = StringIO()
+    traceback.print_exc(file=io)
+    className = _determineClassName(o)
+    tbValue = io.getvalue()
+    return "<{} instance at 0x{:x} with {} error:\n {}>".format(
+        className,
+        id(o),
+        formatter.__name__,
+        tbValue,
+    )
+
+
+def safe_repr(o):
+    """
+    Returns a string representation of an object, or a string containing a
+    traceback, if that object's __repr__ raised an exception.
+
+    @param o: Any object.
+
+    @rtype: C{str}
+    """
+    try:
+        return repr(o)
+    except BaseException:
+        return _safeFormat(repr, o)
+
+
+def safe_str(o: object) -> str:
+    """
+    Returns a string representation of an object, or a string containing a
+    traceback, if that object's __str__ raised an exception.
+
+    @param o: Any object.
+    """
+    if isinstance(o, bytes):
+        # If o is bytes and seems to holds a utf-8 encoded string,
+        # convert it to str.
+        try:
+            return o.decode("utf-8")
+        except BaseException:
+            pass
+    try:
+        return str(o)
+    except BaseException:
+        return _safeFormat(str, o)
+
+
+class QueueMethod:
+    """
+    I represent a method that doesn't exist yet.
+    """
+
+    def __init__(self, name, calls):
+        self.name = name
+        self.calls = calls
+
+    def __call__(self, *args):
+        self.calls.append((self.name, args))
+
+
+def fullFuncName(func):
+    qualName = str(pickle.whichmodule(func, func.__name__)) + "." + func.__name__
+    if namedObject(qualName) is not func:
+        raise Exception(f"Couldn't find {func} as {qualName}.")
+    return qualName
+
+
+def getClass(obj):
+    """
+    Return the class or type of object 'obj'.
+    """
+    return type(obj)
+
+
+def accumulateClassDict(classObj, attr, adict, baseClass=None):
+    """
+    Accumulate all attributes of a given name in a class hierarchy into a single dictionary.
+
+    Assuming all class attributes of this name are dictionaries.
+    If any of the dictionaries being accumulated have the same key, the
+    one highest in the class hierarchy wins.
+    (XXX: If \"highest\" means \"closest to the starting class\".)
+
+    Ex::
+
+      class Soy:
+        properties = {\"taste\": \"bland\"}
+
+      class Plant:
+        properties = {\"colour\": \"green\"}
+
+      class Seaweed(Plant):
+        pass
+
+      class Lunch(Soy, Seaweed):
+        properties = {\"vegan\": 1 }
+
+      dct = {}
+
+      accumulateClassDict(Lunch, \"properties\", dct)
+
+      print(dct)
+
+    {\"taste\": \"bland\", \"colour\": \"green\", \"vegan\": 1}
+    """
+    for base in classObj.__bases__:
+        accumulateClassDict(base, attr, adict)
+    if baseClass is None or baseClass in classObj.__bases__:
+        adict.update(classObj.__dict__.get(attr, {}))
+
+
+def accumulateClassList(classObj, attr, listObj, baseClass=None):
+    """
+    Accumulate all attributes of a given name in a class hierarchy into a single list.
+
+    Assuming all class attributes of this name are lists.
+    """
+    for base in classObj.__bases__:
+        accumulateClassList(base, attr, listObj)
+    if baseClass is None or baseClass in classObj.__bases__:
+        listObj.extend(classObj.__dict__.get(attr, []))
+
+
+def isSame(a, b):
+    return a is b
+
+
+def isLike(a, b):
+    return a == b
+
+
+def modgrep(goal):
+    return objgrep(sys.modules, goal, isLike, "sys.modules")
+
+
+def isOfType(start, goal):
+    return type(start) is goal
+
+
+def findInstances(start, t):
+    return objgrep(start, t, isOfType)
+
+
+def objgrep(
+    start,
+    goal,
+    eq=isLike,
+    path="",
+    paths=None,
+    seen=None,
+    showUnknowns=0,
+    maxDepth=None,
+):
+    """
+    L{objgrep} finds paths between C{start} and C{goal}.
+
+    Starting at the python object C{start}, we will loop over every reachable
+    reference, tring to find the python object C{goal} (i.e. every object
+    C{candidate} for whom C{eq(candidate, goal)} is truthy), and return a
+    L{list} of L{str}, where each L{str} is Python syntax for a path between
+    C{start} and C{goal}.
+
+    Since this can be slightly difficult to visualize, here's an example::
+
+        >>> class Holder:
+        ...     def __init__(self, x):
+        ...         self.x = x
+        ...
+        >>> start = Holder({"irrelevant": "ignore",
+        ...                 "relevant": [7, 1, 3, 5, 7]})
+        >>> for path in objgrep(start, 7):
+        ...     print("start" + path)
+        start.x['relevant'][0]
+        start.x['relevant'][4]
+
+    This can be useful, for example, when debugging stateful graphs of objects
+    attached to a socket, trying to figure out where a particular connection is
+    attached.
+
+    @param start: The object to start looking at.
+
+    @param goal: The object to search for.
+
+    @param eq: A 2-argument predicate which takes an object found by traversing
+        references starting at C{start}, as well as C{goal}, and returns a
+        boolean.
+
+    @param path: The prefix of the path to include in every return value; empty
+        by default.
+
+    @param paths: The result object to append values to; a list of strings.
+
+    @param seen: A dictionary mapping ints (object IDs) to objects already
+        seen.
+
+    @param showUnknowns: if true, print a message to C{stdout} when
+        encountering objects that C{objgrep} does not know how to traverse.
+
+    @param maxDepth: The maximum number of object references to attempt
+        traversing before giving up.  If an integer, limit to that many links,
+        if C{None}, unlimited.
+
+    @return: A list of strings representing python object paths starting at
+        C{start} and terminating at C{goal}.
+    """
+    if paths is None:
+        paths = []
+    if seen is None:
+        seen = {}
+    if eq(start, goal):
+        paths.append(path)
+    if id(start) in seen:
+        if seen[id(start)] is start:
+            return
+    if maxDepth is not None:
+        if maxDepth == 0:
+            return
+        maxDepth -= 1
+    seen[id(start)] = start
+    # Make an alias for those arguments which are passed recursively to
+    # objgrep for container objects.
+    args = (paths, seen, showUnknowns, maxDepth)
+    if isinstance(start, dict):
+        for k, v in start.items():
+            objgrep(k, goal, eq, path + "{" + repr(v) + "}", *args)
+            objgrep(v, goal, eq, path + "[" + repr(k) + "]", *args)
+    elif isinstance(start, (list, tuple, deque)):
+        for idx, _elem in enumerate(start):
+            objgrep(start[idx], goal, eq, path + "[" + str(idx) + "]", *args)
+    elif isinstance(start, types.MethodType):
+        objgrep(start.__self__, goal, eq, path + ".__self__", *args)
+        objgrep(start.__func__, goal, eq, path + ".__func__", *args)
+        objgrep(start.__self__.__class__, goal, eq, path + ".__self__.__class__", *args)
+    elif hasattr(start, "__dict__"):
+        for k, v in start.__dict__.items():
+            objgrep(v, goal, eq, path + "." + k, *args)
+    elif isinstance(start, weakref.ReferenceType):
+        objgrep(start(), goal, eq, path + "()", *args)
+    elif isinstance(
+        start,
+        (
+            str,
+            int,
+            types.FunctionType,
+            types.BuiltinMethodType,
+            RegexType,
+            float,
+            type(None),
+            IOBase,
+        ),
+    ) or type(start).__name__ in (
+        "wrapper_descriptor",
+        "method_descriptor",
+        "member_descriptor",
+        "getset_descriptor",
+    ):
+        pass
+    elif showUnknowns:
+        print("unknown type", type(start), start)
+    return paths
+
+
+__all__ = [
+    "InvalidName",
+    "ModuleNotFound",
+    "ObjectNotFound",
+    "QueueMethod",
+    "namedModule",
+    "namedObject",
+    "namedClass",
+    "namedAny",
+    "requireModule",
+    "safe_repr",
+    "safe_str",
+    "prefixedMethodNames",
+    "addMethodNamesToDict",
+    "prefixedMethods",
+    "accumulateMethods",
+    "fullFuncName",
+    "qual",
+    "getClass",
+    "accumulateClassDict",
+    "accumulateClassList",
+    "isSame",
+    "isLike",
+    "modgrep",
+    "isOfType",
+    "findInstances",
+    "objgrep",
+    "filenameToModuleName",
+    "fullyQualifiedName",
+]
+
+
+# This is to be removed when fixing #6986
+__all__.remove("objgrep")
diff --git a/contrib/python/Twisted/py3/twisted/python/release.py b/contrib/python/Twisted/py3/twisted/python/release.py
new file mode 100644
index 00000000000..943484af733
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/release.py
@@ -0,0 +1,63 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A release-automation toolkit.
+
+Don't use this outside of Twisted.
+
+Maintainer: Christopher Armstrong
+"""
+
+
+import os
+
+# errors
+
+
+class DirectoryExists(OSError):
+    """
+    Some directory exists when it shouldn't.
+    """
+
+    pass
+
+
+class DirectoryDoesntExist(OSError):
+    """
+    Some directory doesn't exist when it should.
+    """
+
+    pass
+
+
+class CommandFailed(OSError):
+    pass
+
+
+# utilities
+
+
+def sh(command, null=True, prompt=False):
+    """
+    I'll try to execute C{command}, and if C{prompt} is true, I'll
+    ask before running it.  If the command returns something other
+    than 0, I'll raise C{CommandFailed(command)}.
+    """
+    print("--$", command)
+
+    if prompt:
+        if input("run ?? ").startswith("n"):
+            return
+    if null:
+        command = "%s > /dev/null" % command
+    if os.system(command) != 0:
+        raise CommandFailed(command)
+
+
+def runChdirSafe(f, *args, **kw):
+    origdir = os.path.abspath(".")
+    try:
+        return f(*args, **kw)
+    finally:
+        os.chdir(origdir)
diff --git a/contrib/python/Twisted/py3/twisted/python/roots.py b/contrib/python/Twisted/py3/twisted/python/roots.py
new file mode 100644
index 00000000000..05e8ef8f618
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/roots.py
@@ -0,0 +1,242 @@
+# -*- test-case-name: twisted.test.test_roots -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Python Roots: an abstract hierarchy representation for Twisted.
+
+Maintainer: Glyph Lefkowitz
+"""
+
+
+from twisted.python import reflect
+
+
+class NotSupportedError(NotImplementedError):
+    """
+    An exception meaning that the tree-manipulation operation
+    you're attempting to perform is not supported.
+    """
+
+
+class Request:
+    """I am an abstract representation of a request for an entity.
+
+    I also function as the response.  The request is responded to by calling
+    self.write(data) until there is no data left and then calling
+    self.finish().
+    """
+
+    # This attribute should be set to the string name of the protocol being
+    # responded to (e.g. HTTP or FTP)
+    wireProtocol = None
+
+    def write(self, data):
+        """Add some data to the response to this request."""
+        raise NotImplementedError("%s.write" % reflect.qual(self.__class__))
+
+    def finish(self):
+        """The response to this request is finished; flush all data to the network stream."""
+        raise NotImplementedError("%s.finish" % reflect.qual(self.__class__))
+
+
+class Entity:
+    """I am a terminal object in a hierarchy, with no children.
+
+    I represent a null interface; certain non-instance objects (strings and
+    integers, notably) are Entities.
+
+    Methods on this class are suggested to be implemented, but are not
+    required, and will be emulated on a per-protocol basis for types which do
+    not handle them.
+    """
+
+    def render(self, request):
+        """
+        I produce a stream of bytes for the request, by calling request.write()
+        and request.finish().
+        """
+        raise NotImplementedError("%s.render" % reflect.qual(self.__class__))
+
+
+class Collection:
+    """I represent a static collection of entities.
+
+    I contain methods designed to represent collections that can be dynamically
+    created.
+    """
+
+    def __init__(self, entities=None):
+        """Initialize me."""
+        if entities is not None:
+            self.entities = entities
+        else:
+            self.entities = {}
+
+    def getStaticEntity(self, name):
+        """Get an entity that was added to me using putEntity.
+
+        This method will return 'None' if it fails.
+        """
+        return self.entities.get(name)
+
+    def getDynamicEntity(self, name, request):
+        """Subclass this to generate an entity on demand.
+
+        This method should return 'None' if it fails.
+        """
+
+    def getEntity(self, name, request):
+        """Retrieve an entity from me.
+
+        I will first attempt to retrieve an entity statically; static entities
+        will obscure dynamic ones.  If that fails, I will retrieve the entity
+        dynamically.
+
+        If I cannot retrieve an entity, I will return 'None'.
+        """
+        ent = self.getStaticEntity(name)
+        if ent is not None:
+            return ent
+        ent = self.getDynamicEntity(name, request)
+        if ent is not None:
+            return ent
+        return None
+
+    def putEntity(self, name, entity):
+        """Store a static reference on 'name' for 'entity'.
+
+        Raises a KeyError if the operation fails.
+        """
+        self.entities[name] = entity
+
+    def delEntity(self, name):
+        """Remove a static reference for 'name'.
+
+        Raises a KeyError if the operation fails.
+        """
+        del self.entities[name]
+
+    def storeEntity(self, name, request):
+        """Store an entity for 'name', based on the content of 'request'."""
+        raise NotSupportedError("%s.storeEntity" % reflect.qual(self.__class__))
+
+    def removeEntity(self, name, request):
+        """Remove an entity for 'name', based on the content of 'request'."""
+        raise NotSupportedError("%s.removeEntity" % reflect.qual(self.__class__))
+
+    def listStaticEntities(self):
+        """Retrieve a list of all name, entity pairs that I store references to.
+
+        See getStaticEntity.
+        """
+        return self.entities.items()
+
+    def listDynamicEntities(self, request):
+        """A list of all name, entity that I can generate on demand.
+
+        See getDynamicEntity.
+        """
+        return []
+
+    def listEntities(self, request):
+        """Retrieve a list of all name, entity pairs I contain.
+
+        See getEntity.
+        """
+        return self.listStaticEntities() + self.listDynamicEntities(request)
+
+    def listStaticNames(self):
+        """Retrieve a list of the names of entities that I store references to.
+
+        See getStaticEntity.
+        """
+        return self.entities.keys()
+
+    def listDynamicNames(self):
+        """Retrieve a list of the names of entities that I store references to.
+
+        See getDynamicEntity.
+        """
+        return []
+
+    def listNames(self, request):
+        """Retrieve a list of all names for entities that I contain.
+
+        See getEntity.
+        """
+        return self.listStaticNames()
+
+
+class ConstraintViolation(Exception):
+    """An exception raised when a constraint is violated."""
+
+
+class Constrained(Collection):
+    """A collection that has constraints on its names and/or entities."""
+
+    def nameConstraint(self, name):
+        """A method that determines whether an entity may be added to me with a given name.
+
+        If the constraint is satisfied, return 1; if the constraint is not
+        satisfied, either return 0 or raise a descriptive ConstraintViolation.
+        """
+        return 1
+
+    def entityConstraint(self, entity):
+        """A method that determines whether an entity may be added to me.
+
+        If the constraint is satisfied, return 1; if the constraint is not
+        satisfied, either return 0 or raise a descriptive ConstraintViolation.
+        """
+        return 1
+
+    def reallyPutEntity(self, name, entity):
+        Collection.putEntity(self, name, entity)
+
+    def putEntity(self, name, entity):
+        """Store an entity if it meets both constraints.
+
+        Otherwise raise a ConstraintViolation.
+        """
+        if self.nameConstraint(name):
+            if self.entityConstraint(entity):
+                self.reallyPutEntity(name, entity)
+            else:
+                raise ConstraintViolation("Entity constraint violated.")
+        else:
+            raise ConstraintViolation("Name constraint violated.")
+
+
+class Locked(Constrained):
+    """A collection that can be locked from adding entities."""
+
+    locked = 0
+
+    def lock(self):
+        self.locked = 1
+
+    def entityConstraint(self, entity):
+        return not self.locked
+
+
+class Homogenous(Constrained):
+    """A homogenous collection of entities.
+
+    I will only contain entities that are an instance of the class or type
+    specified by my 'entityType' attribute.
+    """
+
+    entityType = object
+
+    def entityConstraint(self, entity):
+        if isinstance(entity, self.entityType):
+            return 1
+        else:
+            raise ConstraintViolation(f"{entity} of incorrect type ({self.entityType})")
+
+    def getNameType(self):
+        return "Name"
+
+    def getEntityType(self):
+        return self.entityType.__name__
diff --git a/contrib/python/Twisted/py3/twisted/python/runtime.py b/contrib/python/Twisted/py3/twisted/python/runtime.py
new file mode 100644
index 00000000000..5e9c71357e5
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/runtime.py
@@ -0,0 +1,204 @@
+# -*- test-case-name: twisted.python.test.test_runtime -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+__all__ = [
+    "seconds",
+    "shortPythonVersion",
+    "Platform",
+    "platform",
+    "platformType",
+]
+import os
+import sys
+import warnings
+from time import time as seconds
+from typing import Optional
+
+
+def shortPythonVersion() -> str:
+    """
+    Returns the Python version as a dot-separated string.
+    """
+    return "%s.%s.%s" % sys.version_info[:3]
+
+
+knownPlatforms = {
+    "nt": "win32",
+    "ce": "win32",
+    "posix": "posix",
+    "java": "java",
+    "org.python.modules.os": "java",
+}
+
+
+class Platform:
+    """
+    Gives us information about the platform we're running on.
+    """
+
+    type: Optional[str] = knownPlatforms.get(os.name)
+    seconds = staticmethod(seconds)
+    _platform = sys.platform
+
+    def __init__(
+        self, name: Optional[str] = None, platform: Optional[str] = None
+    ) -> None:
+        if name is not None:
+            self.type = knownPlatforms.get(name)
+        if platform is not None:
+            self._platform = platform
+
+    def isKnown(self) -> bool:
+        """
+        Do we know about this platform?
+
+        @return: Boolean indicating whether this is a known platform or not.
+        """
+        return self.type != None
+
+    def getType(self) -> Optional[str]:
+        """
+        Get platform type.
+
+        @return: Either 'posix', 'win32' or 'java'
+        """
+        return self.type
+
+    def isMacOSX(self) -> bool:
+        """
+        Check if current platform is macOS.
+
+        @return: C{True} if the current platform has been detected as macOS.
+        """
+        return self._platform == "darwin"
+
+    def isWinNT(self) -> bool:
+        """
+        Are we running in Windows NT?
+
+        This is deprecated and always returns C{True} on win32 because
+        Twisted only supports Windows NT-derived platforms at this point.
+
+        @return: C{True} if the current platform has been detected as
+            Windows NT.
+        """
+        warnings.warn(
+            "twisted.python.runtime.Platform.isWinNT was deprecated in "
+            "Twisted 13.0. Use Platform.isWindows instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.isWindows()
+
+    def isWindows(self) -> bool:
+        """
+        Are we running in Windows?
+
+        @return: C{True} if the current platform has been detected as
+            Windows.
+        """
+        return self.getType() == "win32"
+
+    def isVista(self) -> bool:
+        """
+        Check if current platform is Windows Vista or Windows Server 2008.
+
+        @return: C{True} if the current platform has been detected as Vista
+        """
+        return sys.platform == "win32" and sys.getwindowsversion().major == 6
+
+    def isLinux(self) -> bool:
+        """
+        Check if current platform is Linux.
+
+        @return: C{True} if the current platform has been detected as Linux.
+        """
+        return self._platform.startswith("linux")
+
+    def isDocker(self, _initCGroupLocation: str = "/proc/1/cgroup") -> bool:
+        """
+        Check if the current platform is Linux in a Docker container.
+
+        @return: C{True} if the current platform has been detected as Linux
+            inside a Docker container.
+        """
+        if not self.isLinux():
+            return False
+
+        from twisted.python.filepath import FilePath
+
+        # Ask for the cgroups of init (pid 1)
+        initCGroups = FilePath(_initCGroupLocation)
+        if initCGroups.exists():
+            # The cgroups file looks like "2:cpu:/". The third element will
+            # begin with /docker if it is inside a Docker container.
+            controlGroups = [
+                x.split(b":") for x in initCGroups.getContent().split(b"\n")
+            ]
+
+            for group in controlGroups:
+                if len(group) == 3 and group[2].startswith(b"/docker/"):
+                    # If it starts with /docker/, we're in a docker container
+                    return True
+
+        return False
+
+    def _supportsSymlinks(self) -> bool:
+        """
+        Check for symlink support usable for Twisted's purposes.
+
+        @return: C{True} if symlinks are supported on the current platform,
+                 otherwise C{False}.
+        """
+        if self.isWindows():
+            # We do the isWindows() check as newer Pythons support the symlink
+            # support in Vista+, but only if you have some obscure permission
+            # (SeCreateSymbolicLinkPrivilege), which can only be given on
+            # platforms with msc.exe (so, Business/Enterprise editions).
+            # This uncommon requirement makes the Twisted test suite test fail
+            # in 99.99% of cases as general users don't have permission to do
+            # it, even if there is "symlink support".
+            return False
+        else:
+            # If we're not on Windows, check for existence of os.symlink.
+            try:
+                os.symlink
+            except AttributeError:
+                return False
+            else:
+                return True
+
+    def supportsThreads(self) -> bool:
+        """
+        Can threads be created?
+
+        @return: C{True} if the threads are supported on the current platform.
+        """
+        try:
+            import threading
+
+            return threading is not None  # shh pyflakes
+        except ImportError:
+            return False
+
+    def supportsINotify(self) -> bool:
+        """
+        Return C{True} if we can use the inotify API on this platform.
+
+        @since: 10.1
+        """
+        try:
+            from twisted.python._inotify import INotifyError, init
+        except ImportError:
+            return False
+
+        try:
+            os.close(init())
+        except INotifyError:
+            return False
+        return True
+
+
+platform = Platform()
+platformType = platform.getType()
diff --git a/contrib/python/Twisted/py3/twisted/python/sendmsg.py b/contrib/python/Twisted/py3/twisted/python/sendmsg.py
new file mode 100644
index 00000000000..ace9a5231c5
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/sendmsg.py
@@ -0,0 +1,76 @@
+# -*- test-case-name: twisted.test.test_sendmsg -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+sendmsg(2) and recvmsg(2) support for Python.
+"""
+
+
+from collections import namedtuple
+from socket import CMSG_SPACE, SCM_RIGHTS, socket as Socket
+from typing import List, Tuple
+
+__all__ = ["sendmsg", "recvmsg", "getSocketFamily", "SCM_RIGHTS"]
+
+
+ReceivedMessage = namedtuple("ReceivedMessage", ["data", "ancillary", "flags"])
+
+
+def sendmsg(
+    socket: Socket,
+    data: bytes,
+    ancillary: List[Tuple[int, int, bytes]] = [],
+    flags: int = 0,
+) -> int:
+    """
+    Send a message on a socket.
+
+    @param socket: The socket to send the message on.
+    @param data: Bytes to write to the socket.
+    @param ancillary: Extra data to send over the socket outside of the normal
+        datagram or stream mechanism.  By default no ancillary data is sent.
+    @param flags: Flags to affect how the message is sent.  See the C{MSG_}
+        constants in the sendmsg(2) manual page.  By default no flags are set.
+
+    @return: The return value of the underlying syscall, if it succeeds.
+    """
+    return socket.sendmsg([data], ancillary, flags)
+
+
+def recvmsg(
+    socket: Socket, maxSize: int = 8192, cmsgSize: int = 4096, flags: int = 0
+) -> ReceivedMessage:
+    """
+    Receive a message on a socket.
+
+    @param socket: The socket to receive the message on.
+    @param maxSize: The maximum number of bytes to receive from the socket using
+        the datagram or stream mechanism. The default maximum is 8192.
+    @param cmsgSize: The maximum number of bytes to receive from the socket
+        outside of the normal datagram or stream mechanism. The default maximum
+        is 4096.
+    @param flags: Flags to affect how the message is sent.  See the C{MSG_}
+        constants in the sendmsg(2) manual page. By default no flags are set.
+
+    @return: A named 3-tuple of the bytes received using the datagram/stream
+        mechanism, a L{list} of L{tuple}s giving ancillary received data, and
+        flags as an L{int} describing the data received.
+    """
+    # In Twisted's _sendmsg.c, the csmg_space was defined as:
+    #     int cmsg_size = 4096;
+    #     cmsg_space = CMSG_SPACE(cmsg_size);
+    # Since the default in Python 3's socket is 0, we need to define our
+    # own default of 4096. -hawkie
+    data, ancillary, flags = socket.recvmsg(maxSize, CMSG_SPACE(cmsgSize), flags)[0:3]
+
+    return ReceivedMessage(data=data, ancillary=ancillary, flags=flags)
+
+
+def getSocketFamily(socket: Socket) -> int:
+    """
+    Return the family of the given socket.
+
+    @param socket: The socket to get the family of.
+    """
+    return socket.family
diff --git a/contrib/python/Twisted/py3/twisted/python/shortcut.py b/contrib/python/Twisted/py3/twisted/python/shortcut.py
new file mode 100644
index 00000000000..7d08424bdd0
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/shortcut.py
@@ -0,0 +1,88 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Creation of  Windows shortcuts.
+
+Requires win32all.
+"""
+
+import os
+
+import pythoncom  # type: ignore[import]
+from win32com.shell import shell  # type: ignore[import]
+
+
+def open(filename):
+    """
+    Open an existing shortcut for reading.
+
+    @return: The shortcut object
+    @rtype: Shortcut
+    """
+    sc = Shortcut()
+    sc.load(filename)
+    return sc
+
+
+class Shortcut:
+    """
+    A shortcut on Win32.
+    """
+
+    def __init__(
+        self,
+        path=None,
+        arguments=None,
+        description=None,
+        workingdir=None,
+        iconpath=None,
+        iconidx=0,
+    ):
+        """
+        @param path: Location of the target
+        @param arguments: If path points to an executable, optional arguments
+                      to pass
+        @param description: Human-readable description of target
+        @param workingdir: Directory from which target is launched
+        @param iconpath: Filename that contains an icon for the shortcut
+        @param iconidx: If iconpath is set, optional index of the icon desired
+        """
+        self._base = pythoncom.CoCreateInstance(
+            shell.CLSID_ShellLink,
+            None,
+            pythoncom.CLSCTX_INPROC_SERVER,
+            shell.IID_IShellLink,
+        )
+        if path is not None:
+            self.SetPath(os.path.abspath(path))
+        if arguments is not None:
+            self.SetArguments(arguments)
+        if description is not None:
+            self.SetDescription(description)
+        if workingdir is not None:
+            self.SetWorkingDirectory(os.path.abspath(workingdir))
+        if iconpath is not None:
+            self.SetIconLocation(os.path.abspath(iconpath), iconidx)
+
+    def load(self, filename):
+        """
+        Read a shortcut file from disk.
+        """
+        self._base.QueryInterface(pythoncom.IID_IPersistFile).Load(
+            os.path.abspath(filename)
+        )
+
+    def save(self, filename):
+        """
+        Write the shortcut to disk.
+
+        The file should be named something.lnk.
+        """
+        self._base.QueryInterface(pythoncom.IID_IPersistFile).Save(
+            os.path.abspath(filename), 0
+        )
+
+    def __getattr__(self, name):
+        return getattr(self._base, name)
diff --git a/contrib/python/Twisted/py3/twisted/python/syslog.py b/contrib/python/Twisted/py3/twisted/python/syslog.py
new file mode 100644
index 00000000000..781a2a51cbb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/syslog.py
@@ -0,0 +1,106 @@
+# -*- test-case-name: twisted.python.test.test_syslog -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Classes and utility functions for integrating Twisted and syslog.
+
+You probably want to call L{startLogging}.
+"""
+
+syslog = __import__("syslog")
+
+from twisted.python import log
+
+# These defaults come from the Python syslog docs.
+DEFAULT_OPTIONS = 0
+DEFAULT_FACILITY = syslog.LOG_USER
+
+
+class SyslogObserver:
+    """
+    A log observer for logging to syslog.
+
+    See L{twisted.python.log} for context.
+
+    This logObserver will automatically use LOG_ALERT priority for logged
+    failures (such as from C{log.err()}), but you can use any priority and
+    facility by setting the 'C{syslogPriority}' and 'C{syslogFacility}' keys in
+    the event dict.
+    """
+
+    openlog = syslog.openlog
+    syslog = syslog.syslog
+
+    def __init__(self, prefix, options=DEFAULT_OPTIONS, facility=DEFAULT_FACILITY):
+        """
+        @type prefix: C{str}
+        @param prefix: The syslog prefix to use.
+
+        @type options: C{int}
+        @param options: A bitvector represented as an integer of the syslog
+            options to use.
+
+        @type facility: C{int}
+        @param facility: An indication to the syslog daemon of what sort of
+            program this is (essentially, an additional arbitrary metadata
+            classification for messages sent to syslog by this observer).
+        """
+        self.openlog(prefix, options, facility)
+
+    def emit(self, eventDict):
+        """
+        Send a message event to the I{syslog}.
+
+        @param eventDict: The event to send.  If it has no C{'message'} key, it
+            will be ignored.  Otherwise, if it has C{'syslogPriority'} and/or
+            C{'syslogFacility'} keys, these will be used as the syslog priority
+            and facility.  If it has no C{'syslogPriority'} key but a true
+            value for the C{'isError'} key, the B{LOG_ALERT} priority will be
+            used; if it has a false value for C{'isError'}, B{LOG_INFO} will be
+            used.  If the C{'message'} key is multiline, each line will be sent
+            to the syslog separately.
+        """
+        # Figure out what the message-text is.
+        text = log.textFromEventDict(eventDict)
+        if text is None:
+            return
+
+        # Figure out what syslog parameters we might need to use.
+        priority = syslog.LOG_INFO
+        facility = 0
+        if eventDict["isError"]:
+            priority = syslog.LOG_ALERT
+        if "syslogPriority" in eventDict:
+            priority = int(eventDict["syslogPriority"])
+        if "syslogFacility" in eventDict:
+            facility = int(eventDict["syslogFacility"])
+
+        # Break the message up into lines and send them.
+        lines = text.split("\n")
+        while lines[-1:] == [""]:
+            lines.pop()
+
+        firstLine = True
+        for line in lines:
+            if firstLine:
+                firstLine = False
+            else:
+                line = "\t" + line
+            self.syslog(
+                priority | facility, "[{}] {}".format(eventDict["system"], line)
+            )
+
+
+def startLogging(
+    prefix="Twisted", options=DEFAULT_OPTIONS, facility=DEFAULT_FACILITY, setStdout=1
+):
+    """
+    Send all Twisted logging output to syslog from now on.
+
+    The prefix, options and facility arguments are passed to
+    C{syslog.openlog()}, see the Python syslog documentation for details. For
+    other parameters, see L{twisted.python.log.startLoggingWithObserver}.
+    """
+    obs = SyslogObserver(prefix, options, facility)
+    log.startLoggingWithObserver(obs.emit, setStdout=setStdout)
diff --git a/contrib/python/Twisted/py3/twisted/python/systemd.py b/contrib/python/Twisted/py3/twisted/python/systemd.py
new file mode 100644
index 00000000000..51911160eb9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/systemd.py
@@ -0,0 +1,154 @@
+# -*- test-case-name: twisted.python.test.test_systemd -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Integration with systemd.
+
+Currently only the minimum APIs necessary for using systemd's socket activation
+feature are supported.
+"""
+
+
+__all__ = ["ListenFDs"]
+
+from os import getpid
+from typing import Dict, List, Mapping, Optional, Sequence
+
+from attrs import Factory, define
+
+
+@define
+class ListenFDs:
+    """
+    L{ListenFDs} provides access to file descriptors inherited from systemd.
+
+    Typically L{ListenFDs.fromEnvironment} should be used to construct a new
+    instance of L{ListenFDs}.
+
+    @cvar _START: File descriptors inherited from systemd are always
+        consecutively numbered, with a fixed lowest "starting" descriptor.  This
+        gives the default starting descriptor.  Since this must agree with the
+        value systemd is using, it typically should not be overridden.
+
+    @ivar _descriptors: A C{list} of C{int} giving the descriptors which were
+        inherited.
+
+    @ivar _names: A L{Sequence} of C{str} giving the names of the descriptors
+        which were inherited.
+    """
+
+    _descriptors: Sequence[int]
+    _names: Sequence[str] = Factory(tuple)
+
+    _START = 3
+
+    @classmethod
+    def fromEnvironment(
+        cls,
+        environ: Optional[Mapping[str, str]] = None,
+        start: Optional[int] = None,
+    ) -> "ListenFDs":
+        """
+        @param environ: A dictionary-like object to inspect to discover
+            inherited descriptors.  By default, L{None}, indicating that the
+            real process environment should be inspected.  The default is
+            suitable for typical usage.
+
+        @param start: An integer giving the lowest value of an inherited
+            descriptor systemd will give us.  By default, L{None}, indicating
+            the known correct (that is, in agreement with systemd) value will be
+            used.  The default is suitable for typical usage.
+
+        @return: A new instance of C{cls} which can be used to look up the
+            descriptors which have been inherited.
+        """
+        if environ is None:
+            from os import environ as _environ
+
+            environ = _environ
+        if start is None:
+            start = cls._START
+
+        if str(getpid()) == environ.get("LISTEN_PID"):
+            descriptors: List[int] = _parseDescriptors(start, environ)
+            names: Sequence[str] = _parseNames(environ)
+        else:
+            descriptors = []
+            names = ()
+
+        # They may both be missing (consistent with not running under systemd
+        # at all) or they may both be present (consistent with running under
+        # systemd 227 or newer).  It is not allowed for only one to be present
+        # or for the values to disagree with each other (per
+        # systemd.socket(5), systemd will use a default value based on the
+        # socket unit name if the socket unit doesn't explicitly define a name
+        # with FileDescriptorName).
+        if len(names) != len(descriptors):
+            return cls([], ())
+
+        return cls(descriptors, names)
+
+    def inheritedDescriptors(self) -> List[int]:
+        """
+        @return: The configured descriptors.
+        """
+        return list(self._descriptors)
+
+    def inheritedNamedDescriptors(self) -> Dict[str, int]:
+        """
+        @return: A mapping from the names of configured descriptors to
+            their integer values.
+        """
+        return dict(zip(self._names, self._descriptors))
+
+
+def _parseDescriptors(start: int, environ: Mapping[str, str]) -> List[int]:
+    """
+    Parse the I{LISTEN_FDS} environment variable supplied by systemd.
+
+    @param start: systemd provides only a count of the number of descriptors
+        that have been inherited.  This is the integer value of the first
+        inherited descriptor.  Subsequent inherited descriptors are numbered
+        counting up from here.  See L{ListenFDs._START}.
+
+    @param environ: The environment variable mapping in which to look for the
+        value to parse.
+
+    @return: The integer values of the inherited file descriptors, in order.
+    """
+    try:
+        count = int(environ["LISTEN_FDS"])
+    except (KeyError, ValueError):
+        return []
+    else:
+        descriptors = list(range(start, start + count))
+
+        # Remove the information from the environment so that a second
+        # `ListenFDs` cannot find the same information.  This is a precaution
+        # against some application code accidentally trying to handle the same
+        # inherited descriptor more than once - which probably wouldn't work.
+        #
+        # This precaution is perhaps somewhat questionable since it is up to
+        # the application itself to know whether its handling of the file
+        # descriptor will actually be safe.  Also, nothing stops an
+        # application from getting the same descriptor more than once using
+        # multiple calls to `ListenFDs.inheritedDescriptors()` on the same
+        # `ListenFDs` instance.
+        del environ["LISTEN_PID"], environ["LISTEN_FDS"]
+        return descriptors
+
+
+def _parseNames(environ: Mapping[str, str]) -> Sequence[str]:
+    """
+    Parse the I{LISTEN_FDNAMES} environment variable supplied by systemd.
+
+    @param environ: The environment variable mapping in which to look for the
+        value to parse.
+
+    @return: The names of the inherited descriptors, in order.
+    """
+    names = environ.get("LISTEN_FDNAMES", "")
+    if len(names) > 0:
+        return tuple(names.split(":"))
+    return ()
diff --git a/contrib/python/Twisted/py3/twisted/python/text.py b/contrib/python/Twisted/py3/twisted/python/text.py
new file mode 100644
index 00000000000..acd0b0a61e0
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/text.py
@@ -0,0 +1,205 @@
+# -*- test-case-name: twisted.test.test_text -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Miscellany of text-munging functions.
+"""
+
+
+def stringyString(object, indentation=""):
+    """
+    Expansive string formatting for sequence types.
+
+    C{list.__str__} and C{dict.__str__} use C{repr()} to display their
+    elements.  This function also turns these sequence types
+    into strings, but uses C{str()} on their elements instead.
+
+    Sequence elements are also displayed on separate lines, and nested
+    sequences have nested indentation.
+    """
+    braces = ""
+    sl = []
+
+    if type(object) is dict:
+        braces = "{}"
+        for key, value in object.items():
+            value = stringyString(value, indentation + "   ")
+            if isMultiline(value):
+                if endsInNewline(value):
+                    value = value[: -len("\n")]
+                sl.append(f"{indentation} {key}:\n{value}")
+            else:
+                # Oops.  Will have to move that indentation.
+                sl.append(f"{indentation} {key}: {value[len(indentation) + 3 :]}")
+
+    elif type(object) is tuple or type(object) is list:
+        if type(object) is tuple:
+            braces = "()"
+        else:
+            braces = "[]"
+
+        for element in object:
+            element = stringyString(element, indentation + " ")
+            sl.append(element.rstrip() + ",")
+    else:
+        sl[:] = map(lambda s, i=indentation: i + s, str(object).split("\n"))
+
+    if not sl:
+        sl.append(indentation)
+
+    if braces:
+        sl[0] = indentation + braces[0] + sl[0][len(indentation) + 1 :]
+        sl[-1] = sl[-1] + braces[-1]
+
+    s = "\n".join(sl)
+
+    if isMultiline(s) and not endsInNewline(s):
+        s = s + "\n"
+
+    return s
+
+
+def isMultiline(s):
+    """
+    Returns C{True} if this string has a newline in it.
+    """
+    return s.find("\n") != -1
+
+
+def endsInNewline(s):
+    """
+    Returns C{True} if this string ends in a newline.
+    """
+    return s[-len("\n") :] == "\n"
+
+
+def greedyWrap(inString, width=80):
+    """
+    Given a string and a column width, return a list of lines.
+
+    Caveat: I'm use a stupid greedy word-wrapping
+    algorythm.  I won't put two spaces at the end
+    of a sentence.  I don't do full justification.
+    And no, I've never even *heard* of hypenation.
+    """
+
+    outLines = []
+
+    # eww, evil hacks to allow paragraphs delimited by two \ns :(
+    if inString.find("\n\n") >= 0:
+        paragraphs = inString.split("\n\n")
+        for para in paragraphs:
+            outLines.extend(greedyWrap(para, width) + [""])
+        return outLines
+    inWords = inString.split()
+
+    column = 0
+    ptr_line = 0
+    while inWords:
+        column = column + len(inWords[ptr_line])
+        ptr_line = ptr_line + 1
+
+        if column > width:
+            if ptr_line == 1:
+                # This single word is too long, it will be the whole line.
+                pass
+            else:
+                # We've gone too far, stop the line one word back.
+                ptr_line = ptr_line - 1
+            (l, inWords) = (inWords[0:ptr_line], inWords[ptr_line:])
+            outLines.append(" ".join(l))
+
+            ptr_line = 0
+            column = 0
+        elif not (len(inWords) > ptr_line):
+            # Clean up the last bit.
+            outLines.append(" ".join(inWords))
+            del inWords[:]
+        else:
+            # Space
+            column = column + 1
+    # next word
+
+    return outLines
+
+
+wordWrap = greedyWrap
+
+
+def removeLeadingBlanks(lines):
+    ret = []
+    for line in lines:
+        if ret or line.strip():
+            ret.append(line)
+    return ret
+
+
+def removeLeadingTrailingBlanks(s):
+    lines = removeLeadingBlanks(s.split("\n"))
+    lines.reverse()
+    lines = removeLeadingBlanks(lines)
+    lines.reverse()
+    return "\n".join(lines) + "\n"
+
+
+def splitQuoted(s):
+    """
+    Like a string split, but don't break substrings inside quotes.
+
+    >>> splitQuoted('the "hairy monkey" likes pie')
+    ['the', 'hairy monkey', 'likes', 'pie']
+
+    Another one of those "someone must have a better solution for
+    this" things.  This implementation is a VERY DUMB hack done too
+    quickly.
+    """
+    out = []
+    quot = None
+    phrase = None
+    for word in s.split():
+        if phrase is None:
+            if word and (word[0] in ('"', "'")):
+                quot = word[0]
+                word = word[1:]
+                phrase = []
+
+        if phrase is None:
+            out.append(word)
+        else:
+            if word and (word[-1] == quot):
+                word = word[:-1]
+                phrase.append(word)
+                out.append(" ".join(phrase))
+                phrase = None
+            else:
+                phrase.append(word)
+
+    return out
+
+
+def strFile(p, f, caseSensitive=True):
+    """
+    Find whether string C{p} occurs in a read()able object C{f}.
+
+    @rtype: C{bool}
+    """
+    buf = type(p)()
+    buf_len = max(len(p), 2**2**2**2)
+    if not caseSensitive:
+        p = p.lower()
+    while 1:
+        r = f.read(buf_len - len(p))
+        if not caseSensitive:
+            r = r.lower()
+        bytes_read = len(r)
+        if bytes_read == 0:
+            return False
+        l = len(buf) + bytes_read - buf_len
+        if l <= 0:
+            buf = buf + r
+        else:
+            buf = buf[l:] + r
+        if buf.find(p) != -1:
+            return True
diff --git a/contrib/python/Twisted/py3/twisted/python/threadable.py b/contrib/python/Twisted/py3/twisted/python/threadable.py
new file mode 100644
index 00000000000..5109aac49b9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/threadable.py
@@ -0,0 +1,137 @@
+# -*- test-case-name: twisted.python.test_threadable -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A module to provide some very basic threading primitives, such as
+synchronization.
+"""
+
+
+from functools import wraps
+
+
+class DummyLock:
+    """
+    Hack to allow locks to be unpickled on an unthreaded system.
+    """
+
+    def __reduce__(self):
+        return (unpickle_lock, ())
+
+
+def unpickle_lock():
+    if threadingmodule is not None:
+        return XLock()
+    else:
+        return DummyLock()
+
+
+unpickle_lock.__safe_for_unpickling__ = True  # type: ignore[attr-defined]
+
+
+def _synchPre(self):
+    if "_threadable_lock" not in self.__dict__:
+        _synchLockCreator.acquire()
+        if "_threadable_lock" not in self.__dict__:
+            self.__dict__["_threadable_lock"] = XLock()
+        _synchLockCreator.release()
+    self._threadable_lock.acquire()
+
+
+def _synchPost(self):
+    self._threadable_lock.release()
+
+
+def _sync(klass, function):
+    @wraps(function)
+    def sync(self, *args, **kwargs):
+        _synchPre(self)
+        try:
+            return function(self, *args, **kwargs)
+        finally:
+            _synchPost(self)
+
+    return sync
+
+
+def synchronize(*klasses):
+    """
+    Make all methods listed in each class' synchronized attribute synchronized.
+
+    The synchronized attribute should be a list of strings, consisting of the
+    names of methods that must be synchronized. If we are running in threaded
+    mode these methods will be wrapped with a lock.
+    """
+    if threadingmodule is not None:
+        for klass in klasses:
+            for methodName in klass.synchronized:
+                sync = _sync(klass, klass.__dict__[methodName])
+                setattr(klass, methodName, sync)
+
+
+def init(with_threads=1):
+    """Initialize threading.
+
+    Don't bother calling this.  If it needs to happen, it will happen.
+    """
+    global threaded, _synchLockCreator, XLock
+
+    if with_threads:
+        if not threaded:
+            if threadingmodule is not None:
+                threaded = True
+
+                class XLock(threadingmodule._RLock):
+                    def __reduce__(self):
+                        return (unpickle_lock, ())
+
+                _synchLockCreator = XLock()
+            else:
+                raise RuntimeError(
+                    "Cannot initialize threading, platform lacks thread support"
+                )
+    else:
+        if threaded:
+            raise RuntimeError("Cannot uninitialize threads")
+        else:
+            pass
+
+
+_dummyID = object()
+
+
+def getThreadID():
+    if threadingmodule is None:
+        return _dummyID
+    return threadingmodule.current_thread().ident
+
+
+def isInIOThread():
+    """Are we in the thread responsible for I/O requests (the event loop)?"""
+    return ioThread == getThreadID()
+
+
+def registerAsIOThread():
+    """Mark the current thread as responsible for I/O requests."""
+    global ioThread
+    ioThread = getThreadID()
+
+
+ioThread = None
+threaded = False
+# Define these globals which might be overwritten in init().
+_synchLockCreator = None
+XLock = None
+
+
+try:
+    import threading as _threadingmodule
+except ImportError:
+    threadingmodule = None
+else:
+    threadingmodule = _threadingmodule
+    init(True)
+
+
+__all__ = ["isInIOThread", "registerAsIOThread", "getThreadID", "XLock"]
diff --git a/contrib/python/Twisted/py3/twisted/python/threadpool.py b/contrib/python/Twisted/py3/twisted/python/threadpool.py
new file mode 100644
index 00000000000..ab5c0f1e67f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/threadpool.py
@@ -0,0 +1,340 @@
+# -*- test-case-name: twisted.test.test_threadpool -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+twisted.python.threadpool: a pool of threads to which we dispatch tasks.
+
+In most cases you can just use C{reactor.callInThread} and friends
+instead of creating a thread pool directly.
+"""
+
+from __future__ import annotations
+
+from threading import Thread, current_thread
+from typing import Any, Callable, List, Optional, TypeVar
+
+from typing_extensions import ParamSpec, Protocol, TypedDict
+
+from twisted._threads import pool as _pool
+from twisted.python import context, log
+from twisted.python.deprecate import deprecated
+from twisted.python.failure import Failure
+from twisted.python.versions import Version
+
+_P = ParamSpec("_P")
+_R = TypeVar("_R")
+
+
+class _SupportsQsize(Protocol):
+    def qsize(self) -> int:
+        ...
+
+
+class _State(TypedDict):
+    min: int
+    max: int
+
+
+WorkerStop = object()
+
+
+class ThreadPool:
+    """
+    This class (hopefully) generalizes the functionality of a pool of threads
+    to which work can be dispatched.
+
+    L{callInThread} and L{stop} should only be called from a single thread.
+
+    @ivar started: Whether or not the thread pool is currently running.
+    @type started: L{bool}
+
+    @ivar threads: List of workers currently running in this thread pool.
+    @type threads: L{list}
+
+    @ivar _pool: A hook for testing.
+    @type _pool: callable compatible with L{_pool}
+    """
+
+    min = 5
+    max = 20
+    joined = False
+    started = False
+    name = None
+
+    threadFactory = Thread
+    currentThread = staticmethod(
+        deprecated(
+            version=Version("Twisted", 22, 1, 0),
+            replacement="threading.current_thread",
+        )(current_thread)
+    )
+    _pool = staticmethod(_pool)
+
+    def __init__(
+        self, minthreads: int = 5, maxthreads: int = 20, name: Optional[str] = None
+    ):
+        """
+        Create a new threadpool.
+
+        @param minthreads: minimum number of threads in the pool
+        @type minthreads: L{int}
+
+        @param maxthreads: maximum number of threads in the pool
+        @type maxthreads: L{int}
+
+        @param name: The name to give this threadpool; visible in log messages.
+        @type name: native L{str}
+        """
+        assert minthreads >= 0, "minimum is negative"
+        assert minthreads <= maxthreads, "minimum is greater than maximum"
+        self.min = minthreads
+        self.max = maxthreads
+        self.name = name
+        self.threads: List[Thread] = []
+
+        def trackingThreadFactory(*a: Any, **kw: Any) -> Thread:
+            thread = self.threadFactory(  # type: ignore[misc]
+                *a, name=self._generateName(), **kw
+            )
+            self.threads.append(thread)
+            return thread
+
+        def currentLimit() -> int:
+            if not self.started:
+                return 0
+            return self.max
+
+        self._team = self._pool(currentLimit, trackingThreadFactory)
+
+    @property
+    def workers(self) -> int:
+        """
+        For legacy compatibility purposes, return a total number of workers.
+
+        @return: the current number of workers, both idle and busy (but not
+            those that have been quit by L{ThreadPool.adjustPoolsize})
+        @rtype: L{int}
+        """
+        stats = self._team.statistics()
+        return stats.idleWorkerCount + stats.busyWorkerCount
+
+    @property
+    def working(self) -> list[None]:
+        """
+        For legacy compatibility purposes, return the number of busy workers as
+        expressed by a list the length of that number.
+
+        @return: the number of workers currently processing a work item.
+        @rtype: L{list} of L{None}
+        """
+        return [None] * self._team.statistics().busyWorkerCount
+
+    @property
+    def waiters(self) -> list[None]:
+        """
+        For legacy compatibility purposes, return the number of idle workers as
+        expressed by a list the length of that number.
+
+        @return: the number of workers currently alive (with an allocated
+            thread) but waiting for new work.
+        @rtype: L{list} of L{None}
+        """
+        return [None] * self._team.statistics().idleWorkerCount
+
+    @property
+    def _queue(self) -> _SupportsQsize:
+        """
+        For legacy compatibility purposes, return an object with a C{qsize}
+        method that indicates the amount of work not yet allocated to a worker.
+
+        @return: an object with a C{qsize} method.
+        """
+
+        class NotAQueue:
+            def qsize(q) -> int:
+                """
+                Pretend to be a Python threading Queue and return the
+                number of as-yet-unconsumed tasks.
+
+                @return: the amount of backlogged work not yet dispatched to a
+                    worker.
+                @rtype: L{int}
+                """
+                return self._team.statistics().backloggedWorkCount
+
+        return NotAQueue()
+
+    q = _queue  # Yes, twistedchecker, I want a single-letter
+    # attribute name.
+
+    def start(self) -> None:
+        """
+        Start the threadpool.
+        """
+        self.joined = False
+        self.started = True
+        # Start some threads.
+        self.adjustPoolsize()
+        backlog = self._team.statistics().backloggedWorkCount
+        if backlog:
+            self._team.grow(backlog)
+
+    def startAWorker(self) -> None:
+        """
+        Increase the number of available workers for the thread pool by 1, up
+        to the maximum allowed by L{ThreadPool.max}.
+        """
+        self._team.grow(1)
+
+    def _generateName(self) -> str:
+        """
+        Generate a name for a new pool thread.
+
+        @return: A distinctive name for the thread.
+        @rtype: native L{str}
+        """
+        return f"PoolThread-{self.name or id(self)}-{self.workers}"
+
+    def stopAWorker(self) -> None:
+        """
+        Decrease the number of available workers by 1, by quitting one as soon
+        as it's idle.
+        """
+        self._team.shrink(1)
+
+    def __setstate__(self, state: _State) -> None:
+        setattr(self, "__dict__", state)
+        ThreadPool.__init__(self, self.min, self.max)
+
+    def __getstate__(self) -> _State:
+        return _State(min=self.min, max=self.max)
+
+    def callInThread(
+        self, func: Callable[_P, object], *args: _P.args, **kw: _P.kwargs
+    ) -> None:
+        """
+        Call a callable object in a separate thread.
+
+        @param func: callable object to be called in separate thread
+
+        @param args: positional arguments to be passed to C{func}
+
+        @param kw: keyword args to be passed to C{func}
+        """
+        self.callInThreadWithCallback(None, func, *args, **kw)
+
+    def callInThreadWithCallback(
+        self,
+        onResult: Optional[Callable[[bool, _R], object]],
+        func: Callable[_P, _R],
+        *args: _P.args,
+        **kw: _P.kwargs,
+    ) -> None:
+        """
+        Call a callable object in a separate thread and call C{onResult} with
+        the return value, or a L{twisted.python.failure.Failure} if the
+        callable raises an exception.
+
+        The callable is allowed to block, but the C{onResult} function must not
+        block and should perform as little work as possible.
+
+        A typical action for C{onResult} for a threadpool used with a Twisted
+        reactor would be to schedule a L{twisted.internet.defer.Deferred} to
+        fire in the main reactor thread using C{.callFromThread}.  Note that
+        C{onResult} is called inside the separate thread, not inside the
+        reactor thread.
+
+        @param onResult: a callable with the signature C{(success, result)}.
+            If the callable returns normally, C{onResult} is called with
+            C{(True, result)} where C{result} is the return value of the
+            callable.  If the callable throws an exception, C{onResult} is
+            called with C{(False, failure)}.
+
+            Optionally, C{onResult} may be L{None}, in which case it is not
+            called at all.
+
+        @param func: callable object to be called in separate thread
+
+        @param args: positional arguments to be passed to C{func}
+
+        @param kw: keyword arguments to be passed to C{func}
+        """
+        if self.joined:
+            return
+        ctx = context.theContextTracker.currentContext().contexts[-1]
+
+        def inContext() -> None:
+            try:
+                result = inContext.theWork()  # type: ignore[attr-defined]
+                ok = True
+            except BaseException:
+                result = Failure()
+                ok = False
+
+            inContext.theWork = None  # type: ignore[attr-defined]
+            if inContext.onResult is not None:  # type: ignore[attr-defined]
+                inContext.onResult(ok, result)  # type: ignore[attr-defined]
+                inContext.onResult = None  # type: ignore[attr-defined]
+            elif not ok:
+                log.err(result)
+
+        # Avoid closing over func, ctx, args, kw so that we can carefully
+        # manage their lifecycle.  See
+        # test_threadCreationArgumentsCallInThreadWithCallback.
+        inContext.theWork = lambda: context.call(  # type: ignore[attr-defined]
+            ctx, func, *args, **kw
+        )
+        inContext.onResult = onResult  # type: ignore[attr-defined]
+
+        self._team.do(inContext)
+
+    def stop(self) -> None:
+        """
+        Shutdown the threads in the threadpool.
+        """
+        self.joined = True
+        self.started = False
+        self._team.quit()
+        for thread in self.threads:
+            thread.join()
+
+    def adjustPoolsize(
+        self, minthreads: Optional[int] = None, maxthreads: Optional[int] = None
+    ) -> None:
+        """
+        Adjust the number of available threads by setting C{min} and C{max} to
+        new values.
+
+        @param minthreads: The new value for L{ThreadPool.min}.
+
+        @param maxthreads: The new value for L{ThreadPool.max}.
+        """
+        if minthreads is None:
+            minthreads = self.min
+        if maxthreads is None:
+            maxthreads = self.max
+
+        assert minthreads >= 0, "minimum is negative"
+        assert minthreads <= maxthreads, "minimum is greater than maximum"
+
+        self.min = minthreads
+        self.max = maxthreads
+        if not self.started:
+            return
+
+        # Kill of some threads if we have too many.
+        if self.workers > self.max:
+            self._team.shrink(self.workers - self.max)
+        # Start some threads if we have too few.
+        if self.workers < self.min:
+            self._team.grow(self.min - self.workers)
+
+    def dumpStats(self) -> None:
+        """
+        Dump some plain-text informational messages to the log about the state
+        of this L{ThreadPool}.
+        """
+        log.msg(f"waiters: {self.waiters}")
+        log.msg(f"workers: {self.working}")
+        log.msg(f"total: {self.threads}")
diff --git a/contrib/python/Twisted/py3/twisted/python/twisted-completion.zsh b/contrib/python/Twisted/py3/twisted/python/twisted-completion.zsh
new file mode 100644
index 00000000000..102a6780015
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/twisted-completion.zsh
@@ -0,0 +1,33 @@
+#compdef twist twistd trial conch cftp ckeygen pyhtmlizer tkconch
+#
+# This is the ZSH completion file for Twisted commands. It calls the current
+# command-line with the special "--_shell-completion" option which is handled
+# by twisted.python.usage. t.p.usage then generates zsh code on stdout to
+# handle the completions for this particular command-line.
+#
+# 3rd parties that wish to provide zsh completion for commands that
+# use t.p.usage may copy this file and change the first line to reference
+# the name(s) of their command(s).
+#
+# This file is included in the official Zsh distribution as
+# Completion/Unix/Command/_twisted
+
+# redirect stderr to /dev/null otherwise deprecation warnings may get puked all
+# over the user's terminal if completing options for a deprecated command.
+# Redirect stderr to a file to debug errors.
+local cmd output
+cmd=("$words[@]" --_shell-completion zsh:$CURRENT)
+output=$("$cmd[@]" 2>/dev/null)
+
+if [[ $output == "#compdef "* ]]; then
+    # Looks like we got a valid completion function - so eval it to produce
+    # the completion matches.
+    eval $output
+else
+    echo "\nCompletion error running command:" ${(qqq)cmd}
+    echo -n "If output below is unhelpful you may need to edit this file and "
+    echo    "redirect stderr to a file."
+    echo "Expected completion function, but instead got:"
+    echo $output
+    return 1
+fi
diff --git a/contrib/python/Twisted/py3/twisted/python/url.py b/contrib/python/Twisted/py3/twisted/python/url.py
new file mode 100644
index 00000000000..f3591db8722
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/url.py
@@ -0,0 +1,15 @@
+# -*- test-case-name: twisted.python.test.test_url -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+URL parsing, construction and rendering.
+
+@see: L{URL}
+"""
+
+from hyperlink import URL
+
+__all__ = [
+    "URL",
+]
diff --git a/contrib/python/Twisted/py3/twisted/python/urlpath.py b/contrib/python/Twisted/py3/twisted/python/urlpath.py
new file mode 100644
index 00000000000..2e76a2fa1a8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/urlpath.py
@@ -0,0 +1,278 @@
+# -*- test-case-name: twisted.python.test.test_urlpath -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+L{URLPath}, a representation of a URL.
+"""
+
+from typing import cast
+from urllib.parse import quote as urlquote, unquote as urlunquote, urlunsplit
+
+from hyperlink import URL as _URL
+
+_allascii = b"".join([chr(x).encode("ascii") for x in range(1, 128)])
+
+
+def _rereconstituter(name):
+    """
+    Attriute declaration to preserve mutability on L{URLPath}.
+
+    @param name: a public attribute name
+    @type name: native L{str}
+
+    @return: a descriptor which retrieves the private version of the attribute
+        on get and calls rerealize on set.
+    """
+    privateName = "_" + name
+    return property(
+        lambda self: getattr(self, privateName),
+        lambda self, value: (
+            setattr(
+                self,
+                privateName,
+                value if isinstance(value, bytes) else value.encode("charmap"),
+            )
+            or self._reconstitute()
+        ),
+    )
+
+
+class URLPath:
+    """
+    A representation of a URL.
+
+    @ivar scheme: The scheme of the URL (e.g. 'http').
+    @type scheme: L{bytes}
+
+    @ivar netloc: The network location ("host").
+    @type netloc: L{bytes}
+
+    @ivar path: The path on the network location.
+    @type path: L{bytes}
+
+    @ivar query: The query argument (the portion after ?  in the URL).
+    @type query: L{bytes}
+
+    @ivar fragment: The page fragment (the portion after # in the URL).
+    @type fragment: L{bytes}
+    """
+
+    def __init__(
+        self, scheme=b"", netloc=b"localhost", path=b"", query=b"", fragment=b""
+    ):
+        self._scheme = scheme or b"http"
+        self._netloc = netloc
+        self._path = path or b"/"
+        self._query = query
+        self._fragment = fragment
+        self._reconstitute()
+
+    def _reconstitute(self):
+        """
+        Reconstitute this L{URLPath} from all its given attributes.
+        """
+        urltext = urlquote(
+            urlunsplit(
+                (self._scheme, self._netloc, self._path, self._query, self._fragment)
+            ),
+            safe=_allascii,
+        )
+        self._url = _URL.fromText(urltext.encode("ascii").decode("ascii"))
+
+    scheme = _rereconstituter("scheme")
+    netloc = _rereconstituter("netloc")
+    path = _rereconstituter("path")
+    query = _rereconstituter("query")
+    fragment = _rereconstituter("fragment")
+
+    @classmethod
+    def _fromURL(cls, urlInstance):
+        """
+        Reconstruct all the public instance variables of this L{URLPath} from
+        its underlying L{_URL}.
+
+        @param urlInstance: the object to base this L{URLPath} on.
+        @type urlInstance: L{_URL}
+
+        @return: a new L{URLPath}
+        """
+        self = cls.__new__(cls)
+        self._url = urlInstance.replace(path=urlInstance.path or [""])
+        self._scheme = self._url.scheme.encode("ascii")
+        self._netloc = self._url.authority().encode("ascii")
+        self._path = (
+            _URL(path=self._url.path, rooted=True).asURI().asText().encode("ascii")
+        )
+        self._query = (_URL(query=self._url.query).asURI().asText().encode("ascii"))[1:]
+        self._fragment = self._url.fragment.encode("ascii")
+        return self
+
+    def pathList(self, unquote=False, copy=True):
+        """
+        Split this URL's path into its components.
+
+        @param unquote: whether to remove %-encoding from the returned strings.
+
+        @param copy: (ignored, do not use)
+
+        @return: The components of C{self.path}
+        @rtype: L{list} of L{bytes}
+        """
+        segments = self._url.path
+        mapper = lambda x: x.encode("ascii")
+        if unquote:
+            mapper = lambda x, m=mapper: m(urlunquote(x))
+        return [b""] + [mapper(segment) for segment in segments]
+
+    @classmethod
+    def fromString(klass, url):
+        """
+        Make a L{URLPath} from a L{str} or L{unicode}.
+
+        @param url: A L{str} representation of a URL.
+        @type url: L{str} or L{unicode}.
+
+        @return: a new L{URLPath} derived from the given string.
+        @rtype: L{URLPath}
+        """
+        if not isinstance(url, str):
+            raise ValueError("'url' must be a str")
+        return klass._fromURL(_URL.fromText(url))
+
+    @classmethod
+    def fromBytes(klass, url):
+        """
+        Make a L{URLPath} from a L{bytes}.
+
+        @param url: A L{bytes} representation of a URL.
+        @type url: L{bytes}
+
+        @return: a new L{URLPath} derived from the given L{bytes}.
+        @rtype: L{URLPath}
+
+        @since: 15.4
+        """
+        if not isinstance(url, bytes):
+            raise ValueError("'url' must be bytes")
+        quoted = urlquote(url, safe=_allascii)
+        return klass.fromString(quoted)
+
+    @classmethod
+    def fromRequest(klass, request):
+        """
+        Make a L{URLPath} from a L{twisted.web.http.Request}.
+
+        @param request: A L{twisted.web.http.Request} to make the L{URLPath}
+            from.
+
+        @return: a new L{URLPath} derived from the given request.
+        @rtype: L{URLPath}
+        """
+        return klass.fromBytes(request.prePathURL())
+
+    def _mod(self, newURL, keepQuery):
+        """
+        Return a modified copy of C{self} using C{newURL}, keeping the query
+        string if C{keepQuery} is C{True}.
+
+        @param newURL: a L{URL} to derive a new L{URLPath} from
+        @type newURL: L{URL}
+
+        @param keepQuery: if C{True}, preserve the query parameters from
+            C{self} on the new L{URLPath}; if C{False}, give the new L{URLPath}
+            no query parameters.
+        @type keepQuery: L{bool}
+
+        @return: a new L{URLPath}
+        """
+        return self._fromURL(
+            newURL.replace(fragment="", query=self._url.query if keepQuery else ())
+        )
+
+    def sibling(self, path, keepQuery=False):
+        """
+        Get the sibling of the current L{URLPath}.  A sibling is a file which
+        is in the same directory as the current file.
+
+        @param path: The path of the sibling.
+        @type path: L{bytes}
+
+        @param keepQuery: Whether to keep the query parameters on the returned
+            L{URLPath}.
+        @type keepQuery: L{bool}
+
+        @return: a new L{URLPath}
+        """
+        return self._mod(self._url.sibling(path.decode("ascii")), keepQuery)
+
+    def child(self, path, keepQuery=False):
+        """
+        Get the child of this L{URLPath}.
+
+        @param path: The path of the child.
+        @type path: L{bytes}
+
+        @param keepQuery: Whether to keep the query parameters on the returned
+            L{URLPath}.
+        @type keepQuery: L{bool}
+
+        @return: a new L{URLPath}
+        """
+        return self._mod(self._url.child(path.decode("ascii")), keepQuery)
+
+    def parent(self, keepQuery=False):
+        """
+        Get the parent directory of this L{URLPath}.
+
+        @param keepQuery: Whether to keep the query parameters on the returned
+            L{URLPath}.
+        @type keepQuery: L{bool}
+
+        @return: a new L{URLPath}
+        """
+        return self._mod(self._url.click(".."), keepQuery)
+
+    def here(self, keepQuery=False):
+        """
+        Get the current directory of this L{URLPath}.
+
+        @param keepQuery: Whether to keep the query parameters on the returned
+            L{URLPath}.
+        @type keepQuery: L{bool}
+
+        @return: a new L{URLPath}
+        """
+        return self._mod(self._url.click("."), keepQuery)
+
+    def click(self, st):
+        """
+        Return a path which is the URL where a browser would presumably take
+        you if you clicked on a link with an HREF as given.
+
+        @param st: A relative URL, to be interpreted relative to C{self} as the
+            base URL.
+        @type st: L{bytes}
+
+        @return: a new L{URLPath}
+        """
+        return self._fromURL(self._url.click(st.decode("ascii")))
+
+    def __str__(self) -> str:
+        """
+        The L{str} of a L{URLPath} is its URL text.
+        """
+        return cast(str, self._url.asURI().asText())
+
+    def __repr__(self) -> str:
+        """
+        The L{repr} of a L{URLPath} is an eval-able expression which will
+        construct a similar L{URLPath}.
+        """
+        return "URLPath(scheme={!r}, netloc={!r}, path={!r}, query={!r}, fragment={!r})".format(
+            self.scheme,
+            self.netloc,
+            self.path,
+            self.query,
+            self.fragment,
+        )
diff --git a/contrib/python/Twisted/py3/twisted/python/usage.py b/contrib/python/Twisted/py3/twisted/python/usage.py
new file mode 100644
index 00000000000..32f074c7318
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/usage.py
@@ -0,0 +1,1013 @@
+# -*- test-case-name: twisted.test.test_usage -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+twisted.python.usage is a module for parsing/handling the
+command line of your program.
+
+For information on how to use it, see
+U{http://twistedmatrix.com/projects/core/documentation/howto/options.html},
+or doc/core/howto/options.xhtml in your Twisted directory.
+"""
+from __future__ import annotations
+
+import getopt
+
+# System Imports
+import inspect
+import os
+import sys
+import textwrap
+from os import path
+from typing import Any, Dict, Optional, cast
+
+# Sibling Imports
+from twisted.python import reflect, util
+
+
+class UsageError(Exception):
+    pass
+
+
+error = UsageError
+
+
+class CoerceParameter:
+    """
+    Utility class that can corce a parameter before storing it.
+    """
+
+    def __init__(self, options, coerce):
+        """
+        @param options: parent Options object
+        @param coerce: callable used to coerce the value.
+        """
+        self.options = options
+        self.coerce = coerce
+        self.doc = getattr(self.coerce, "coerceDoc", "")
+
+    def dispatch(self, parameterName, value):
+        """
+        When called in dispatch, do the coerce for C{value} and save the
+        returned value.
+        """
+        if value is None:
+            raise UsageError(f"Parameter '{parameterName}' requires an argument.")
+        try:
+            value = self.coerce(value)
+        except ValueError as e:
+            raise UsageError(f"Parameter type enforcement failed: {e}")
+
+        self.options.opts[parameterName] = value
+
+
+class Options(Dict[str, Any]):
+    """
+    An option list parser class
+
+    C{optFlags} and C{optParameters} are lists of available parameters
+    which your program can handle. The difference between the two
+    is the 'flags' have an on(1) or off(0) state (off by default)
+    whereas 'parameters' have an assigned value, with an optional
+    default. (Compare '--verbose' and '--verbosity=2')
+
+    optFlags is assigned a list of lists. Each list represents
+    a flag parameter, as so::
+
+       optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'],
+                   ['quiet', 'q', 'Be vewy vewy quiet.']]
+
+    As you can see, the first item is the long option name
+    (prefixed with '--' on the command line), followed by the
+    short option name (prefixed with '-'), and the description.
+    The description is used for the built-in handling of the
+    --help switch, which prints a usage summary.
+
+    C{optParameters} is much the same, except the list also contains
+    a default value::
+
+       optParameters = [['outfile', 'O', 'outfile.log', 'Description...']]
+
+    A coerce function can also be specified as the last element: it will be
+    called with the argument and should return the value that will be stored
+    for the option. This function can have a C{coerceDoc} attribute which
+    will be appended to the documentation of the option.
+
+    subCommands is a list of 4-tuples of (command name, command shortcut,
+    parser class, documentation).  If the first non-option argument found is
+    one of the given command names, an instance of the given parser class is
+    instantiated and given the remainder of the arguments to parse and
+    self.opts[command] is set to the command name.  For example::
+
+       subCommands = [
+            ['inquisition', 'inquest', InquisitionOptions,
+                 'Perform an inquisition'],
+            ['holyquest', 'quest', HolyQuestOptions,
+                 'Embark upon a holy quest']
+        ]
+
+    In this case, C{"<program> holyquest --horseback --for-grail"} will cause
+    C{HolyQuestOptions} to be instantiated and asked to parse
+    C{['--horseback', '--for-grail']}.  Currently, only the first sub-command
+    is parsed, and all options following it are passed to its parser.  If a
+    subcommand is found, the subCommand attribute is set to its name and the
+    subOptions attribute is set to the Option instance that parses the
+    remaining options. If a subcommand is not given to parseOptions,
+    the subCommand attribute will be None. You can also mark one of
+    the subCommands to be the default::
+
+       defaultSubCommand = 'holyquest'
+
+    In this case, the subCommand attribute will never be None, and
+    the subOptions attribute will always be set.
+
+    If you want to handle your own options, define a method named
+    C{opt_paramname} that takes C{(self, option)} as arguments. C{option}
+    will be whatever immediately follows the parameter on the
+    command line. Options fully supports the mapping interface, so you
+    can do things like C{'self["option"] = val'} in these methods.
+
+    Shell tab-completion is supported by this class, for zsh only at present.
+    Zsh ships with a stub file ("completion function") which, for Twisted
+    commands, performs tab-completion on-the-fly using the support provided
+    by this class. The stub file lives in our tree at
+    C{twisted/python/twisted-completion.zsh}, and in the Zsh tree at
+    C{Completion/Unix/Command/_twisted}.
+
+    Tab-completion is based upon the contents of the optFlags and optParameters
+    lists. And, optionally, additional metadata may be provided by assigning a
+    special attribute, C{compData}, which should be an instance of
+    C{Completions}. See that class for details of what can and should be
+    included - and see the howto for additional help using these features -
+    including how third-parties may take advantage of tab-completion for their
+    own commands.
+
+    Advanced functionality is covered in the howto documentation,
+    available at
+    U{http://twistedmatrix.com/projects/core/documentation/howto/options.html},
+    or doc/core/howto/options.xhtml in your Twisted directory.
+    """
+
+    subCommand: Optional[str] = None
+    defaultSubCommand: Optional[str] = None
+    parent: "Optional[Options]" = None
+    completionData = None
+    _shellCompFile = sys.stdout  # file to use if shell completion is requested
+
+    def __init__(self):
+        super().__init__()
+
+        self.opts = self
+        self.defaults = {}
+
+        # These are strings/lists we will pass to getopt
+        self.longOpt = []
+        self.shortOpt = ""
+        self.docs = {}
+        self.synonyms = {}
+        self._dispatch = {}
+
+        collectors = [
+            self._gather_flags,
+            self._gather_parameters,
+            self._gather_handlers,
+        ]
+
+        for c in collectors:
+            (longOpt, shortOpt, docs, settings, synonyms, dispatch) = c()
+            self.longOpt.extend(longOpt)
+            self.shortOpt = self.shortOpt + shortOpt
+            self.docs.update(docs)
+
+            self.opts.update(settings)
+            self.defaults.update(settings)
+
+            self.synonyms.update(synonyms)
+            self._dispatch.update(dispatch)
+
+    # class Options derives from dict, which defines __hash__ as None,
+    # but we need to set __hash__ to object.__hash__ which is of type
+    # Callable[[object], int].  So we need to ignore mypy error here.
+    __hash__ = object.__hash__  # type: ignore[assignment]
+
+    def opt_help(self):
+        """
+        Display this help and exit.
+        """
+        print(self.__str__())
+        sys.exit(0)
+
+    def opt_version(self):
+        """
+        Display Twisted version and exit.
+        """
+        from twisted import copyright
+
+        print("Twisted version:", copyright.version)
+        sys.exit(0)
+
+    # opt_h = opt_help # this conflicted with existing 'host' options.
+
+    def parseOptions(self, options=None):
+        """
+        The guts of the command-line parser.
+        """
+
+        if options is None:
+            options = sys.argv[1:]
+
+        # we really do need to place the shell completion check here, because
+        # if we used an opt_shell_completion method then it would be possible
+        # for other opt_* methods to be run first, and they could possibly
+        # raise validation errors which would result in error output on the
+        # terminal of the user performing shell completion. Validation errors
+        # would occur quite frequently, in fact, because users often initiate
+        # tab-completion while they are editing an unfinished command-line.
+        if len(options) > 1 and options[-2] == "--_shell-completion":
+            from twisted.python import _shellcomp
+
+            cmdName = path.basename(sys.argv[0])
+            _shellcomp.shellComplete(self, cmdName, options, self._shellCompFile)
+            sys.exit(0)
+
+        try:
+            opts, args = getopt.getopt(options, self.shortOpt, self.longOpt)
+        except getopt.error as e:
+            raise UsageError(str(e))
+
+        for opt, arg in opts:
+            if opt[1] == "-":
+                opt = opt[2:]
+            else:
+                opt = opt[1:]
+
+            optMangled = opt
+            if optMangled not in self.synonyms:
+                optMangled = opt.replace("-", "_")
+                if optMangled not in self.synonyms:
+                    raise UsageError(f"No such option '{opt}'")
+
+            optMangled = self.synonyms[optMangled]
+            if isinstance(self._dispatch[optMangled], CoerceParameter):
+                self._dispatch[optMangled].dispatch(optMangled, arg)
+            else:
+                self._dispatch[optMangled](optMangled, arg)
+
+        if getattr(self, "subCommands", None) and (
+            args or self.defaultSubCommand is not None
+        ):
+            if not args:
+                args = [self.defaultSubCommand]
+            sub, rest = args[0], args[1:]
+            for cmd, short, parser, doc in self.subCommands:
+                if sub == cmd or sub == short:
+                    self.subCommand = cmd
+                    self.subOptions = parser()
+                    self.subOptions.parent = self
+                    self.subOptions.parseOptions(rest)
+                    break
+            else:
+                raise UsageError("Unknown command: %s" % sub)
+        else:
+            try:
+                self.parseArgs(*args)
+            except TypeError:
+                raise UsageError("Wrong number of arguments.")
+
+        self.postOptions()
+
+    def postOptions(self):
+        """
+        I am called after the options are parsed.
+
+        Override this method in your subclass to do something after
+        the options have been parsed and assigned, like validate that
+        all options are sane.
+        """
+
+    def parseArgs(self):
+        """
+        I am called with any leftover arguments which were not options.
+
+        Override me to do something with the remaining arguments on
+        the command line, those which were not flags or options. e.g.
+        interpret them as a list of files to operate on.
+
+        Note that if there more arguments on the command line
+        than this method accepts, parseArgs will blow up with
+        a getopt.error.  This means if you don't override me,
+        parseArgs will blow up if I am passed any arguments at
+        all!
+        """
+
+    def _generic_flag(self, flagName, value=None):
+        if value not in ("", None):
+            raise UsageError(
+                "Flag '%s' takes no argument." ' Not even "%s".' % (flagName, value)
+            )
+
+        self.opts[flagName] = 1
+
+    def _gather_flags(self):
+        """
+        Gather up boolean (flag) options.
+        """
+
+        longOpt, shortOpt = [], ""
+        docs, settings, synonyms, dispatch = {}, {}, {}, {}
+
+        flags = []
+        reflect.accumulateClassList(self.__class__, "optFlags", flags)
+
+        for flag in flags:
+            long, short, doc = util.padTo(3, flag)
+            if not long:
+                raise ValueError("A flag cannot be without a name.")
+
+            docs[long] = doc
+            settings[long] = 0
+            if short:
+                shortOpt = shortOpt + short
+                synonyms[short] = long
+            longOpt.append(long)
+            synonyms[long] = long
+            dispatch[long] = self._generic_flag
+
+        return longOpt, shortOpt, docs, settings, synonyms, dispatch
+
+    def _gather_parameters(self):
+        """
+        Gather options which take a value.
+        """
+        longOpt, shortOpt = [], ""
+        docs, settings, synonyms, dispatch = {}, {}, {}, {}
+
+        parameters = []
+
+        reflect.accumulateClassList(self.__class__, "optParameters", parameters)
+
+        synonyms = {}
+
+        for parameter in parameters:
+            long, short, default, doc, paramType = util.padTo(5, parameter)
+            if not long:
+                raise ValueError("A parameter cannot be without a name.")
+
+            docs[long] = doc
+            settings[long] = default
+            if short:
+                shortOpt = shortOpt + short + ":"
+                synonyms[short] = long
+            longOpt.append(long + "=")
+            synonyms[long] = long
+            if paramType is not None:
+                dispatch[long] = CoerceParameter(self, paramType)
+            else:
+                dispatch[long] = CoerceParameter(self, str)
+
+        return longOpt, shortOpt, docs, settings, synonyms, dispatch
+
+    def _gather_handlers(self):
+        """
+        Gather up options with their own handler methods.
+
+        This returns a tuple of many values.  Amongst those values is a
+        synonyms dictionary, mapping all of the possible aliases (C{str})
+        for an option to the longest spelling of that option's name
+        C({str}).
+
+        Another element is a dispatch dictionary, mapping each user-facing
+        option name (with - substituted for _) to a callable to handle that
+        option.
+        """
+
+        longOpt, shortOpt = [], ""
+        docs, settings, synonyms, dispatch = {}, {}, {}, {}
+
+        dct = {}
+        reflect.addMethodNamesToDict(self.__class__, dct, "opt_")
+
+        for name in dct.keys():
+            method = getattr(self, "opt_" + name)
+
+            takesArg = not flagFunction(method, name)
+
+            prettyName = name.replace("_", "-")
+            doc = getattr(method, "__doc__", None)
+            if doc:
+                ## Only use the first line.
+                # docs[name] = doc.split('\n')[0]
+                docs[prettyName] = doc
+            else:
+                docs[prettyName] = self.docs.get(prettyName)
+
+            synonyms[prettyName] = prettyName
+
+            # A little slight-of-hand here makes dispatching much easier
+            # in parseOptions, as it makes all option-methods have the
+            # same signature.
+            if takesArg:
+                fn = lambda name, value, m=method: m(value)
+            else:
+                # XXX: This won't raise a TypeError if it's called
+                # with a value when it shouldn't be.
+                fn = lambda name, value=None, m=method: m()
+
+            dispatch[prettyName] = fn
+
+            if len(name) == 1:
+                shortOpt = shortOpt + name
+                if takesArg:
+                    shortOpt = shortOpt + ":"
+            else:
+                if takesArg:
+                    prettyName = prettyName + "="
+                longOpt.append(prettyName)
+
+        reverse_dct = {}
+        # Map synonyms
+        for name in dct.keys():
+            method = getattr(self, "opt_" + name)
+            if method not in reverse_dct:
+                reverse_dct[method] = []
+            reverse_dct[method].append(name.replace("_", "-"))
+
+        for method, names in reverse_dct.items():
+            if len(names) < 2:
+                continue
+            longest = max(names, key=len)
+            for name in names:
+                synonyms[name] = longest
+
+        return longOpt, shortOpt, docs, settings, synonyms, dispatch
+
+    def __str__(self) -> str:
+        return self.getSynopsis() + "\n" + self.getUsage(width=None)
+
+    def getSynopsis(self) -> str:
+        """
+        Returns a string containing a description of these options and how to
+        pass them to the executed file.
+        """
+        executableName = reflect.filenameToModuleName(sys.argv[0])
+
+        if executableName.endswith(".__main__"):
+            executableName = "{} -m {}".format(
+                os.path.basename(sys.executable),
+                executableName.replace(".__main__", ""),
+            )
+
+        if self.parent is None:
+            default = "Usage: {}{}".format(
+                executableName,
+                (self.longOpt and " [options]") or "",
+            )
+        else:
+            default = "%s" % ((self.longOpt and "[options]") or "")
+        synopsis = cast(str, getattr(self, "synopsis", default))
+
+        synopsis = synopsis.rstrip()
+
+        if self.parent is not None:
+            assert self.parent.subCommand is not None
+            synopsis = " ".join(
+                (self.parent.getSynopsis(), self.parent.subCommand, synopsis)
+            )
+        return synopsis
+
+    def getUsage(self, width: Optional[int] = None) -> str:
+        # If subOptions exists by now, then there was probably an error while
+        # parsing its options.
+        if hasattr(self, "subOptions"):
+            return cast(Options, self.subOptions).getUsage(width=width)
+
+        if not width:
+            width = int(os.environ.get("COLUMNS", "80"))
+
+        if hasattr(self, "subCommands"):
+            cmdDicts = []
+            for cmd, short, parser, desc in self.subCommands:
+                cmdDicts.append(
+                    {
+                        "long": cmd,
+                        "short": short,
+                        "doc": desc,
+                        "optType": "command",
+                        "default": None,
+                    }
+                )
+            chunks = docMakeChunks(cmdDicts, width)
+            commands = "Commands:\n" + "".join(chunks)
+        else:
+            commands = ""
+
+        longToShort = {}
+        for key, value in self.synonyms.items():
+            longname = value
+            if (key != longname) and (len(key) == 1):
+                longToShort[longname] = key
+            else:
+                if longname not in longToShort:
+                    longToShort[longname] = None
+                else:
+                    pass
+
+        optDicts = []
+        for opt in self.longOpt:
+            if opt[-1] == "=":
+                optType = "parameter"
+                opt = opt[:-1]
+            else:
+                optType = "flag"
+
+            optDicts.append(
+                {
+                    "long": opt,
+                    "short": longToShort[opt],
+                    "doc": self.docs[opt],
+                    "optType": optType,
+                    "default": self.defaults.get(opt, None),
+                    "dispatch": self._dispatch.get(opt, None),
+                }
+            )
+
+        if not (getattr(self, "longdesc", None) is None):
+            longdesc = cast(str, self.longdesc)  # type: ignore[attr-defined]
+        else:
+            import __main__
+
+            if getattr(__main__, "__doc__", None):
+                longdesc = __main__.__doc__
+            else:
+                longdesc = ""
+
+        if longdesc:
+            longdesc = "\n" + "\n".join(textwrap.wrap(longdesc, width)).strip() + "\n"
+
+        if optDicts:
+            chunks = docMakeChunks(optDicts, width)
+            s = "Options:\n%s" % ("".join(chunks))
+        else:
+            s = "Options: None\n"
+
+        return s + longdesc + commands
+
+
+_ZSH = "zsh"
+_BASH = "bash"
+
+
+class Completer:
+    """
+    A completion "action" - provides completion possibilities for a particular
+    command-line option. For example we might provide the user a fixed list of
+    choices, or files/dirs according to a glob.
+
+    This class produces no completion matches itself - see the various
+    subclasses for specific completion functionality.
+    """
+
+    _descr: Optional[str] = None
+
+    def __init__(self, descr=None, repeat=False):
+        """
+        @type descr: C{str}
+        @param descr: An optional descriptive string displayed above matches.
+
+        @type repeat: C{bool}
+        @param repeat: A flag, defaulting to False, indicating whether this
+            C{Completer} should repeat - that is, be used to complete more
+            than one command-line word. This may ONLY be set to True for
+            actions in the C{extraActions} keyword argument to C{Completions}.
+            And ONLY if it is the LAST (or only) action in the C{extraActions}
+            list.
+        """
+        if descr is not None:
+            self._descr = descr
+        self._repeat = repeat
+
+    @property
+    def _repeatFlag(self):
+        if self._repeat:
+            return "*"
+        else:
+            return ""
+
+    def _description(self, optName):
+        if self._descr is not None:
+            return self._descr
+        else:
+            return optName
+
+    def _shellCode(self, optName, shellType):
+        """
+        Fetch a fragment of shell code representing this action which is
+        suitable for use by the completion system in _shellcomp.py
+
+        @type optName: C{str}
+        @param optName: The long name of the option this action is being
+            used for.
+
+        @type shellType: C{str}
+        @param shellType: One of the supported shell constants e.g.
+            C{twisted.python.usage._ZSH}
+        """
+        if shellType == _ZSH:
+            return f"{self._repeatFlag}:{self._description(optName)}:"
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteFiles(Completer):
+    """
+    Completes file names based on a glob pattern
+    """
+
+    def __init__(self, globPattern="*", **kw):
+        Completer.__init__(self, **kw)
+        self._globPattern = globPattern
+
+    def _description(self, optName):
+        if self._descr is not None:
+            return f"{self._descr} ({self._globPattern})"
+        else:
+            return f"{optName} ({self._globPattern})"
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            return '{}:{}:_files -g "{}"'.format(
+                self._repeatFlag,
+                self._description(optName),
+                self._globPattern,
+            )
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteDirs(Completer):
+    """
+    Completes directory names
+    """
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            return "{}:{}:_directories".format(
+                self._repeatFlag, self._description(optName)
+            )
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteList(Completer):
+    """
+    Completes based on a fixed list of words
+    """
+
+    def __init__(self, items, **kw):
+        Completer.__init__(self, **kw)
+        self._items = items
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            return "{}:{}:({})".format(
+                self._repeatFlag,
+                self._description(optName),
+                " ".join(
+                    self._items,
+                ),
+            )
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteMultiList(Completer):
+    """
+    Completes multiple comma-separated items based on a fixed list of words
+    """
+
+    def __init__(self, items, **kw):
+        Completer.__init__(self, **kw)
+        self._items = items
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            return "{}:{}:_values -s , '{}' {}".format(
+                self._repeatFlag,
+                self._description(optName),
+                self._description(optName),
+                " ".join(self._items),
+            )
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteUsernames(Completer):
+    """
+    Complete usernames
+    """
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            return f"{self._repeatFlag}:{self._description(optName)}:_users"
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteGroups(Completer):
+    """
+    Complete system group names
+    """
+
+    _descr = "group"
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            return f"{self._repeatFlag}:{self._description(optName)}:_groups"
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteHostnames(Completer):
+    """
+    Complete hostnames
+    """
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            return f"{self._repeatFlag}:{self._description(optName)}:_hosts"
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteUserAtHost(Completer):
+    """
+    A completion action which produces matches in any of these forms::
+        <username>
+        <hostname>
+        <username>@<hostname>
+    """
+
+    _descr = "host | user@host"
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            # Yes this looks insane but it does work. For bonus points
+            # add code to grep 'Hostname' lines from ~/.ssh/config
+            return (
+                '%s:%s:{_ssh;if compset -P "*@"; '
+                'then _wanted hosts expl "remote host name" _ssh_hosts '
+                '&& ret=0 elif compset -S "@*"; then _wanted users '
+                'expl "login name" _ssh_users -S "" && ret=0 '
+                "else if (( $+opt_args[-l] )); then tmp=() "
+                'else tmp=( "users:login name:_ssh_users -qS@" ) fi; '
+                '_alternative "hosts:remote host name:_ssh_hosts" "$tmp[@]"'
+                " && ret=0 fi}" % (self._repeatFlag, self._description(optName))
+            )
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class CompleteNetInterfaces(Completer):
+    """
+    Complete network interface names
+    """
+
+    def _shellCode(self, optName, shellType):
+        if shellType == _ZSH:
+            return "{}:{}:_net_interfaces".format(
+                self._repeatFlag,
+                self._description(optName),
+            )
+        raise NotImplementedError(f"Unknown shellType {shellType!r}")
+
+
+class Completions:
+    """
+    Extra metadata for the shell tab-completion system.
+
+    @type descriptions: C{dict}
+    @ivar descriptions: ex. C{{"foo" : "use this description for foo instead"}}
+        A dict mapping long option names to alternate descriptions.  When this
+        variable is defined, the descriptions contained here will override
+        those descriptions provided in the optFlags and optParameters
+        variables.
+
+    @type multiUse: C{list}
+    @ivar multiUse: ex. C{ ["foo", "bar"] }
+        An iterable containing those long option names which may appear on the
+        command line more than once. By default, options will only be completed
+        one time.
+
+    @type mutuallyExclusive: C{list} of C{tuple}
+    @ivar mutuallyExclusive: ex. C{ [("foo", "bar"), ("bar", "baz")] }
+        A sequence of sequences, with each sub-sequence containing those long
+        option names that are mutually exclusive. That is, those options that
+        cannot appear on the command line together.
+
+    @type optActions: C{dict}
+    @ivar optActions: A dict mapping long option names to shell "actions".
+        These actions define what may be completed as the argument to the
+        given option. By default, all files/dirs will be completed if no
+        action is given. For example::
+
+            {"foo"    : CompleteFiles("*.py", descr="python files"),
+             "bar"    : CompleteList(["one", "two", "three"]),
+             "colors" : CompleteMultiList(["red", "green", "blue"])}
+
+        Callables may instead be given for the values in this dict. The
+        callable should accept no arguments, and return a C{Completer}
+        instance used as the action in the same way as the literal actions in
+        the example above.
+
+        As you can see in the example above. The "foo" option will have files
+        that end in .py completed when the user presses Tab. The "bar"
+        option will have either of the strings "one", "two", or "three"
+        completed when the user presses Tab.
+
+        "colors" will allow multiple arguments to be completed, separated by
+        commas. The possible arguments are red, green, and blue. Examples::
+
+            my_command --foo some-file.foo --colors=red,green
+            my_command --colors=green
+            my_command --colors=green,blue
+
+        Descriptions for the actions may be given with the optional C{descr}
+        keyword argument. This is separate from the description of the option
+        itself.
+
+        Normally Zsh does not show these descriptions unless you have
+        "verbose" completion turned on. Turn on verbosity with this in your
+        ~/.zshrc::
+
+            zstyle ':completion:*' verbose yes
+            zstyle ':completion:*:descriptions' format '%B%d%b'
+
+    @type extraActions: C{list}
+    @ivar extraActions: Extra arguments are those arguments typically
+        appearing at the end of the command-line, which are not associated
+        with any particular named option. That is, the arguments that are
+        given to the parseArgs() method of your usage.Options subclass. For
+        example::
+            [CompleteFiles(descr="file to read from"),
+             Completer(descr="book title")]
+
+        In the example above, the 1st non-option argument will be described as
+        "file to read from" and all file/dir names will be completed (*). The
+        2nd non-option argument will be described as "book title", but no
+        actual completion matches will be produced.
+
+        See the various C{Completer} subclasses for other types of things which
+        may be tab-completed (users, groups, network interfaces, etc).
+
+        Also note the C{repeat=True} flag which may be passed to any of the
+        C{Completer} classes. This is set to allow the C{Completer} instance
+        to be re-used for subsequent command-line words. See the C{Completer}
+        docstring for details.
+    """
+
+    def __init__(
+        self,
+        descriptions={},
+        multiUse=[],
+        mutuallyExclusive=[],
+        optActions={},
+        extraActions=[],
+    ):
+        self.descriptions = descriptions
+        self.multiUse = multiUse
+        self.mutuallyExclusive = mutuallyExclusive
+        self.optActions = optActions
+        self.extraActions = extraActions
+
+
+def docMakeChunks(optList, width=80):
+    """
+    Makes doc chunks for option declarations.
+
+    Takes a list of dictionaries, each of which may have one or more
+    of the keys 'long', 'short', 'doc', 'default', 'optType'.
+
+    Returns a list of strings.
+    The strings may be multiple lines,
+    all of them end with a newline.
+    """
+
+    # XXX: sanity check to make sure we have a sane combination of keys.
+
+    # Sort the options so they always appear in the same order
+    optList.sort(key=lambda o: o.get("short", None) or o.get("long", None))
+
+    maxOptLen = 0
+    for opt in optList:
+        optLen = len(opt.get("long", ""))
+        if optLen:
+            if opt.get("optType", None) == "parameter":
+                # these take up an extra character
+                optLen = optLen + 1
+            maxOptLen = max(optLen, maxOptLen)
+
+    colWidth1 = maxOptLen + len("  -s, --  ")
+    colWidth2 = width - colWidth1
+    # XXX - impose some sane minimum limit.
+    # Then if we don't have enough room for the option and the doc
+    # to share one line, they can take turns on alternating lines.
+
+    colFiller1 = " " * colWidth1
+
+    optChunks = []
+    seen = {}
+    for opt in optList:
+        if opt.get("short", None) in seen or opt.get("long", None) in seen:
+            continue
+        for x in opt.get("short", None), opt.get("long", None):
+            if x is not None:
+                seen[x] = 1
+
+        optLines = []
+        comma = " "
+        if opt.get("short", None):
+            short = "-%c" % (opt["short"],)
+        else:
+            short = ""
+
+        if opt.get("long", None):
+            long = opt["long"]
+            if opt.get("optType", None) == "parameter":
+                long = long + "="
+
+            long = "%-*s" % (maxOptLen, long)
+            if short:
+                comma = ","
+        else:
+            long = " " * (maxOptLen + len("--"))
+
+        if opt.get("optType", None) == "command":
+            column1 = "    %s      " % long
+        else:
+            column1 = "  %2s%c --%s  " % (short, comma, long)
+
+        if opt.get("doc", ""):
+            doc = opt["doc"].strip()
+        else:
+            doc = ""
+
+        if (opt.get("optType", None) == "parameter") and not (
+            opt.get("default", None) is None
+        ):
+            doc = "{} [default: {}]".format(doc, opt["default"])
+
+        if (opt.get("optType", None) == "parameter") and opt.get(
+            "dispatch", None
+        ) is not None:
+            d = opt["dispatch"]
+            if isinstance(d, CoerceParameter) and d.doc:
+                doc = f"{doc}. {d.doc}"
+
+        if doc:
+            column2_l = textwrap.wrap(doc, colWidth2)
+        else:
+            column2_l = [""]
+
+        optLines.append(f"{column1}{column2_l.pop(0)}\n")
+
+        for line in column2_l:
+            optLines.append(f"{colFiller1}{line}\n")
+
+        optChunks.append("".join(optLines))
+
+    return optChunks
+
+
+def flagFunction(method, name=None):
+    """
+    Determine whether a function is an optional handler for a I{flag} or an
+    I{option}.
+
+    A I{flag} handler takes no additional arguments.  It is used to handle
+    command-line arguments like I{--nodaemon}.
+
+    An I{option} handler takes one argument.  It is used to handle command-line
+    arguments like I{--path=/foo/bar}.
+
+    @param method: The bound method object to inspect.
+
+    @param name: The name of the option for which the function is a handle.
+    @type name: L{str}
+
+    @raise UsageError: If the method takes more than one argument.
+
+    @return: If the method is a flag handler, return C{True}.  Otherwise return
+        C{False}.
+    """
+    reqArgs = len(inspect.signature(method).parameters)
+    if reqArgs > 1:
+        raise UsageError("Invalid Option function for %s" % (name or method.__name__))
+    if reqArgs == 1:
+        return False
+    return True
+
+
+def portCoerce(value):
+    """
+    Coerce a string value to an int port number, and checks the validity.
+    """
+    value = int(value)
+    if value < 0 or value > 65535:
+        raise ValueError(f"Port number not in range: {value}")
+    return value
+
+
+portCoerce.coerceDoc = "Must be an int between 0 and 65535."  # type: ignore[attr-defined]
diff --git a/contrib/python/Twisted/py3/twisted/python/util.py b/contrib/python/Twisted/py3/twisted/python/util.py
new file mode 100644
index 00000000000..058f1044b37
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/util.py
@@ -0,0 +1,987 @@
+# -*- test-case-name: twisted.python.test.test_util -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+from __future__ import annotations
+
+import errno
+import os
+import sys
+import warnings
+from typing import AnyStr
+
+try:
+    import grp as _grp
+    import pwd as _pwd
+except ImportError:
+    pwd = None
+    grp = None
+else:
+    grp = _grp
+    pwd = _pwd
+
+try:
+    from os import getgroups as _getgroups, setgroups as _setgroups
+except ImportError:
+    setgroups = None
+    getgroups = None
+else:
+    setgroups = _setgroups
+    getgroups = _getgroups
+
+# For backwards compatibility, some things import this, so just link it
+from collections import OrderedDict
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Mapping,
+    MutableMapping,
+    Sequence,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from incremental import Version
+
+from twisted.python.deprecate import deprecatedModuleAttribute
+
+deprecatedModuleAttribute(
+    Version("Twisted", 15, 5, 0),
+    "Use collections.OrderedDict instead.",
+    "twisted.python.util",
+    "OrderedDict",
+)
+
+_T = TypeVar("_T")
+
+
+class InsensitiveDict(MutableMapping[str, _T]):
+    """
+    Dictionary, that has case-insensitive keys.
+
+    Normally keys are retained in their original form when queried with
+    .keys() or .items().  If initialized with preserveCase=0, keys are both
+    looked up in lowercase and returned in lowercase by .keys() and .items().
+    """
+
+    """
+    Modified recipe at http://code.activestate.com/recipes/66315/ originally
+    contributed by Sami Hangaslammi.
+    """
+
+    def __init__(self, dict=None, preserve=1):
+        """
+        Create an empty dictionary, or update from 'dict'.
+        """
+        super().__init__()
+        self.data = {}
+        self.preserve = preserve
+        if dict:
+            self.update(dict)
+
+    def __delitem__(self, key):
+        k = self._lowerOrReturn(key)
+        del self.data[k]
+
+    def _lowerOrReturn(self, key):
+        if isinstance(key, bytes) or isinstance(key, str):
+            return key.lower()
+        else:
+            return key
+
+    def __getitem__(self, key):
+        """
+        Retrieve the value associated with 'key' (in any case).
+        """
+        k = self._lowerOrReturn(key)
+        return self.data[k][1]
+
+    def __setitem__(self, key, value):
+        """
+        Associate 'value' with 'key'. If 'key' already exists, but
+        in different case, it will be replaced.
+        """
+        k = self._lowerOrReturn(key)
+        self.data[k] = (key, value)
+
+    def has_key(self, key):
+        """
+        Case insensitive test whether 'key' exists.
+        """
+        k = self._lowerOrReturn(key)
+        return k in self.data
+
+    __contains__ = has_key
+
+    def _doPreserve(self, key):
+        if not self.preserve and (isinstance(key, bytes) or isinstance(key, str)):
+            return key.lower()
+        else:
+            return key
+
+    def keys(self):
+        """
+        List of keys in their original case.
+        """
+        return list(self.iterkeys())
+
+    def values(self):
+        """
+        List of values.
+        """
+        return list(self.itervalues())
+
+    def items(self):
+        """
+        List of (key,value) pairs.
+        """
+        return list(self.iteritems())
+
+    def get(self, key, default=None):
+        """
+        Retrieve value associated with 'key' or return default value
+        if 'key' doesn't exist.
+        """
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def setdefault(self, key, default):
+        """
+        If 'key' doesn't exist, associate it with the 'default' value.
+        Return value associated with 'key'.
+        """
+        if not self.has_key(key):
+            self[key] = default
+        return self[key]
+
+    def update(self, dict):
+        """
+        Copy (key,value) pairs from 'dict'.
+        """
+        for k, v in dict.items():
+            self[k] = v
+
+    def __repr__(self) -> str:
+        """
+        String representation of the dictionary.
+        """
+        items = ", ".join([(f"{k!r}: {v!r}") for k, v in self.items()])
+        return "InsensitiveDict({%s})" % items
+
+    def iterkeys(self):
+        for v in self.data.values():
+            yield self._doPreserve(v[0])
+
+    __iter__ = iterkeys
+
+    def itervalues(self):
+        for v in self.data.values():
+            yield v[1]
+
+    def iteritems(self):
+        for k, v in self.data.values():
+            yield self._doPreserve(k), v
+
+    _notFound = object()
+
+    def pop(self, key, default=_notFound):
+        """
+        @see: L{dict.pop}
+        @since: Twisted 21.2.0
+        """
+        try:
+            return self.data.pop(self._lowerOrReturn(key))[1]
+        except KeyError:
+            if default is self._notFound:
+                raise
+            return default
+
+    def popitem(self):
+        i = self.items()[0]
+        del self[i[0]]
+        return i
+
+    def clear(self):
+        for k in self.keys():
+            del self[k]
+
+    def copy(self):
+        return InsensitiveDict(self, self.preserve)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Mapping):
+            for k, v in self.items():
+                if k not in other or other[k] != v:
+                    return False
+            return len(self) == len(other)
+        else:
+            return NotImplemented
+
+
+def uniquify(lst):
+    """
+    Make the elements of a list unique by inserting them into a dictionary.
+    This must not change the order of the input lst.
+    """
+    seen = set()
+    result = []
+    for k in lst:
+        if k not in seen:
+            result.append(k)
+        seen.add(k)
+    return result
+
+
+def padTo(n, seq, default=None):
+    """
+    Pads a sequence out to n elements,
+
+    filling in with a default value if it is not long enough.
+
+    If the input sequence is longer than n, raises ValueError.
+
+    Details, details:
+    This returns a new list; it does not extend the original sequence.
+    The new list contains the values of the original sequence, not copies.
+    """
+
+    if len(seq) > n:
+        raise ValueError("%d elements is more than %d." % (len(seq), n))
+
+    blank = [default] * n
+
+    blank[: len(seq)] = list(seq)
+
+    return blank
+
+
+def getPluginDirs():
+    warnings.warn(
+        "twisted.python.util.getPluginDirs is deprecated since Twisted 12.2.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    import twisted
+
+    systemPlugins = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(twisted.__file__))), "plugins"
+    )
+    userPlugins = os.path.expanduser("~/TwistedPlugins")
+    confPlugins = os.path.expanduser("~/.twisted")
+    allPlugins = filter(os.path.isdir, [systemPlugins, userPlugins, confPlugins])
+    return allPlugins
+
+
+def addPluginDir():
+    warnings.warn(
+        "twisted.python.util.addPluginDir is deprecated since Twisted 12.2.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    sys.path.extend(getPluginDirs())
+
+
+def sibpath(
+    path: os.PathLike[AnyStr] | AnyStr, sibling: os.PathLike[AnyStr] | AnyStr
+) -> AnyStr:
+    """
+    Return the path to a sibling of a file in the filesystem.
+
+    This is useful in conjunction with the special C{__file__} attribute
+    that Python provides for modules, so modules can load associated
+    resource files.
+    """
+    return os.path.join(os.path.dirname(os.path.abspath(path)), sibling)
+
+
+def _getpass(prompt):
+    """
+    Helper to turn IOErrors into KeyboardInterrupts.
+    """
+    import getpass
+
+    try:
+        return getpass.getpass(prompt)
+    except OSError as e:
+        if e.errno == errno.EINTR:
+            raise KeyboardInterrupt
+        raise
+    except EOFError:
+        raise KeyboardInterrupt
+
+
+def getPassword(
+    prompt="Password: ",
+    confirm=0,
+    forceTTY=0,
+    confirmPrompt="Confirm password: ",
+    mismatchMessage="Passwords don't match.",
+):
+    """
+    Obtain a password by prompting or from stdin.
+
+    If stdin is a terminal, prompt for a new password, and confirm (if
+    C{confirm} is true) by asking again to make sure the user typed the same
+    thing, as keystrokes will not be echoed.
+
+    If stdin is not a terminal, and C{forceTTY} is not true, read in a line
+    and use it as the password, less the trailing newline, if any.  If
+    C{forceTTY} is true, attempt to open a tty and prompt for the password
+    using it.  Raise a RuntimeError if this is not possible.
+
+    @returns: C{str}
+    """
+    isaTTY = hasattr(sys.stdin, "isatty") and sys.stdin.isatty()
+
+    old = None
+    try:
+        if not isaTTY:
+            if forceTTY:
+                try:
+                    old = sys.stdin, sys.stdout
+                    sys.stdin = sys.stdout = open("/dev/tty", "r+")
+                except BaseException:
+                    raise RuntimeError("Cannot obtain a TTY")
+            else:
+                password = sys.stdin.readline()
+                if password[-1] == "\n":
+                    password = password[:-1]
+                return password
+
+        while 1:
+            try1 = _getpass(prompt)
+            if not confirm:
+                return try1
+            try2 = _getpass(confirmPrompt)
+            if try1 == try2:
+                return try1
+            else:
+                sys.stderr.write(mismatchMessage + "\n")
+    finally:
+        if old:
+            sys.stdin.close()
+            sys.stdin, sys.stdout = old
+
+
+def println(*a):
+    sys.stdout.write(" ".join(map(str, a)) + "\n")
+
+
+# XXX
+# This does not belong here
+# But where does it belong?
+
+
+def str_xor(s, b):
+    return "".join([chr(ord(c) ^ b) for c in s])
+
+
+def makeStatBar(width, maxPosition, doneChar="=", undoneChar="-", currentChar=">"):
+    """
+    Creates a function that will return a string representing a progress bar.
+    """
+    aValue = width / float(maxPosition)
+
+    def statBar(position, force=0, last=[""]):
+        assert len(last) == 1, "Don't mess with the last parameter."
+        done = int(aValue * position)
+        toDo = width - done - 2
+        result = f"[{doneChar * done}{currentChar}{undoneChar * toDo}]"
+        if force:
+            last[0] = result
+            return result
+        if result == last[0]:
+            return ""
+        last[0] = result
+        return result
+
+    statBar.__doc__ = """statBar(position, force = 0) -> '[%s%s%s]'-style progress bar
+
+    returned string is %d characters long, and the range goes from 0..%d.
+    The 'position' argument is where the '%s' will be drawn.  If force is false,
+    '' will be returned instead if the resulting progress bar is identical to the
+    previously returned progress bar.
+""" % (
+        doneChar * 3,
+        currentChar,
+        undoneChar * 3,
+        width,
+        maxPosition,
+        currentChar,
+    )
+    return statBar
+
+
+def spewer(frame, s, ignored):
+    """
+    A trace function for sys.settrace that prints every function or method call.
+    """
+    from twisted.python import reflect
+
+    if "self" in frame.f_locals:
+        se = frame.f_locals["self"]
+        if hasattr(se, "__class__"):
+            k = reflect.qual(se.__class__)
+        else:
+            k = reflect.qual(type(se))
+        print(f"method {frame.f_code.co_name} of {k} at {id(se)}")
+    else:
+        print(
+            "function %s in %s, line %s"
+            % (frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno)
+        )
+
+
+def searchupwards(start, files=[], dirs=[]):
+    """
+    Walk upwards from start, looking for a directory containing
+    all files and directories given as arguments::
+    >>> searchupwards('.', ['foo.txt'], ['bar', 'bam'])
+
+    If not found, return None
+    """
+    start = os.path.abspath(start)
+    parents = start.split(os.sep)
+    exists = os.path.exists
+    join = os.sep.join
+    isdir = os.path.isdir
+    while len(parents):
+        candidate = join(parents) + os.sep
+        allpresent = 1
+        for f in files:
+            if not exists(f"{candidate}{f}"):
+                allpresent = 0
+                break
+        if allpresent:
+            for d in dirs:
+                if not isdir(f"{candidate}{d}"):
+                    allpresent = 0
+                    break
+        if allpresent:
+            return candidate
+        parents.pop(-1)
+    return None
+
+
+class LineLog:
+    """
+    A limited-size line-based log, useful for logging line-based
+    protocols such as SMTP.
+
+    When the log fills up, old entries drop off the end.
+    """
+
+    def __init__(self, size=10):
+        """
+        Create a new log, with size lines of storage (default 10).
+        A log size of 0 (or less) means an infinite log.
+        """
+        if size < 0:
+            size = 0
+        self.log = [None] * size
+        self.size = size
+
+    def append(self, line):
+        if self.size:
+            self.log[:-1] = self.log[1:]
+            self.log[-1] = line
+        else:
+            self.log.append(line)
+
+    def str(self):
+        return bytes(self)
+
+    def __bytes__(self):
+        return b"\n".join(filter(None, self.log))
+
+    def __getitem__(self, item):
+        return filter(None, self.log)[item]
+
+    def clear(self):
+        """
+        Empty the log.
+        """
+        self.log = [None] * self.size
+
+
+def raises(exception, f, *args, **kwargs):
+    """
+    Determine whether the given call raises the given exception.
+    """
+    try:
+        f(*args, **kwargs)
+    except exception:
+        return 1
+    return 0
+
+
+class IntervalDifferential:
+    """
+    Given a list of intervals, generate the amount of time to sleep between
+    "instants".
+
+    For example, given 7, 11 and 13, the three (infinite) sequences::
+
+        7 14 21 28 35 ...
+        11 22 33 44 ...
+        13 26 39 52 ...
+
+    will be generated, merged, and used to produce::
+
+        (7, 0) (4, 1) (2, 2) (1, 0) (7, 0) (1, 1) (4, 2) (2, 0) (5, 1) (2, 0)
+
+    New intervals may be added or removed as iteration proceeds using the
+    proper methods.
+    """
+
+    def __init__(self, intervals, default=60):
+        """
+        @type intervals: C{list} of C{int}, C{long}, or C{float} param
+        @param intervals: The intervals between instants.
+
+        @type default: C{int}, C{long}, or C{float}
+        @param default: The duration to generate if the intervals list
+        becomes empty.
+        """
+        self.intervals = intervals[:]
+        self.default = default
+
+    def __iter__(self):
+        return _IntervalDifferentialIterator(self.intervals, self.default)
+
+
+class _IntervalDifferentialIterator:
+    def __init__(self, i, d):
+        self.intervals = [[e, e, n] for (e, n) in zip(i, range(len(i)))]
+        self.default = d
+        self.last = 0
+
+    def __next__(self):
+        if not self.intervals:
+            return (self.default, None)
+        last, index = self.intervals[0][0], self.intervals[0][2]
+        self.intervals[0][0] += self.intervals[0][1]
+        self.intervals.sort()
+        result = last - self.last
+        self.last = last
+        return result, index
+
+    # Iterators on Python 2 use next(), not __next__()
+    next = __next__
+
+    def addInterval(self, i):
+        if self.intervals:
+            delay = self.intervals[0][0] - self.intervals[0][1]
+            self.intervals.append([delay + i, i, len(self.intervals)])
+            self.intervals.sort()
+        else:
+            self.intervals.append([i, i, 0])
+
+    def removeInterval(self, interval):
+        for i in range(len(self.intervals)):
+            if self.intervals[i][1] == interval:
+                index = self.intervals[i][2]
+                del self.intervals[i]
+                for i in self.intervals:
+                    if i[2] > index:
+                        i[2] -= 1
+                return
+        raise ValueError("Specified interval not in IntervalDifferential")
+
+
+class FancyStrMixin:
+    """
+    Mixin providing a flexible implementation of C{__str__}.
+
+    C{__str__} output will begin with the name of the class, or the contents
+    of the attribute C{fancybasename} if it is set.
+
+    The body of C{__str__} can be controlled by overriding C{showAttributes} in
+    a subclass.  Set C{showAttributes} to a sequence of strings naming
+    attributes, or sequences of C{(attributeName, callable)}, or sequences of
+    C{(attributeName, displayName, formatCharacter)}. In the second case, the
+    callable is passed the value of the attribute and its return value used in
+    the output of C{__str__}.  In the final case, the attribute is looked up
+    using C{attributeName}, but the output uses C{displayName} instead, and
+    renders the value of the attribute using C{formatCharacter}, e.g. C{"%.3f"}
+    might be used for a float.
+    """
+
+    # Override in subclasses:
+    showAttributes: Sequence[
+        Union[str, Tuple[str, str, str], Tuple[str, Callable[[Any], str]]]
+    ] = ()
+
+    def __str__(self) -> str:
+        r = ["<", getattr(self, "fancybasename", self.__class__.__name__)]
+        # The casts help mypy understand which type from the Union applies
+        # in each 'if' case.
+        #   https://github.com/python/mypy/issues/9171
+        for attr in self.showAttributes:
+            if isinstance(attr, str):
+                r.append(f" {attr}={getattr(self, attr)!r}")
+            elif len(attr) == 2:
+                attr = cast(Tuple[str, Callable[[Any], str]], attr)
+                r.append((f" {attr[0]}=") + attr[1](getattr(self, attr[0])))
+            else:
+                attr = cast(Tuple[str, str, str], attr)
+                r.append((" %s=" + attr[2]) % (attr[1], getattr(self, attr[0])))
+        r.append(">")
+        return "".join(r)
+
+    __repr__ = __str__
+
+
+class FancyEqMixin:
+    """
+    Mixin that implements C{__eq__} and C{__ne__}.
+
+    Comparison is done using the list of attributes defined in
+    C{compareAttributes}.
+    """
+
+    compareAttributes: ClassVar[Sequence[str]] = ()
+
+    def __eq__(self, other: object) -> bool:
+        if not self.compareAttributes:
+            return self is other
+        if isinstance(self, other.__class__):
+            return all(
+                getattr(self, name) == getattr(other, name)
+                for name in self.compareAttributes
+            )
+        return NotImplemented
+
+    def __ne__(self, other: object) -> bool:
+        result = self.__eq__(other)
+        if result is NotImplemented:
+            return result
+        return not result
+
+
+try:
+    # initgroups is available in Python 2.7+ on UNIX-likes
+    from os import initgroups as __initgroups
+except ImportError:
+    _initgroups = None
+else:
+    _initgroups = __initgroups
+
+
+if _initgroups is None:
+
+    def initgroups(uid, primaryGid):
+        """
+        Do nothing.
+
+        Underlying platform support require to manipulate groups is missing.
+        """
+
+else:
+
+    def initgroups(uid, primaryGid):
+        """
+        Initializes the group access list.
+
+        This uses the stdlib support which calls initgroups(3) under the hood.
+
+        If the given user is a member of more than C{NGROUPS}, arbitrary
+        groups will be silently discarded to bring the number below that
+        limit.
+
+        @type uid: C{int}
+        @param uid: The UID for which to look up group information.
+
+        @type primaryGid: C{int}
+        @param primaryGid: The GID to include when setting the groups.
+        """
+        return _initgroups(pwd.getpwuid(uid).pw_name, primaryGid)
+
+
+def switchUID(uid, gid, euid=False):
+    """
+    Attempts to switch the uid/euid and gid/egid for the current process.
+
+    If C{uid} is the same value as L{os.getuid} (or L{os.geteuid}),
+    this function will issue a L{UserWarning} and not raise an exception.
+
+    @type uid: C{int} or L{None}
+    @param uid: the UID (or EUID) to switch the current process to. This
+                parameter will be ignored if the value is L{None}.
+
+    @type gid: C{int} or L{None}
+    @param gid: the GID (or EGID) to switch the current process to. This
+                parameter will be ignored if the value is L{None}.
+
+    @type euid: C{bool}
+    @param euid: if True, set only effective user-id rather than real user-id.
+                 (This option has no effect unless the process is running
+                 as root, in which case it means not to shed all
+                 privileges, retaining the option to regain privileges
+                 in cases such as spawning processes. Use with caution.)
+    """
+    if euid:
+        setuid = os.seteuid
+        setgid = os.setegid
+        getuid = os.geteuid
+    else:
+        setuid = os.setuid
+        setgid = os.setgid
+        getuid = os.getuid
+    if gid is not None:
+        setgid(gid)
+    if uid is not None:
+        if uid == getuid():
+            uidText = euid and "euid" or "uid"
+            actionText = f"tried to drop privileges and set{uidText} {uid}"
+            problemText = f"{uidText} is already {getuid()}"
+            warnings.warn(
+                "{} but {}; should we be root? Continuing.".format(
+                    actionText, problemText
+                )
+            )
+        else:
+            initgroups(uid, gid)
+            setuid(uid)
+
+
+def untilConcludes(f, *a, **kw):
+    """
+    Call C{f} with the given arguments, handling C{EINTR} by retrying.
+
+    @param f: A function to call.
+
+    @param a: Positional arguments to pass to C{f}.
+
+    @param kw: Keyword arguments to pass to C{f}.
+
+    @return: Whatever C{f} returns.
+
+    @raise Exception: Whatever C{f} raises, except for C{OSError} with
+        C{errno} set to C{EINTR}.
+    """
+    while True:
+        try:
+            return f(*a, **kw)
+        except OSError as e:
+            if e.args[0] == errno.EINTR:
+                continue
+            raise
+
+
+def mergeFunctionMetadata(f, g):
+    """
+    Overwrite C{g}'s name and docstring with values from C{f}.  Update
+    C{g}'s instance dictionary with C{f}'s.
+
+    @return: A function that has C{g}'s behavior and metadata merged from
+        C{f}.
+    """
+    try:
+        g.__name__ = f.__name__
+    except TypeError:
+        pass
+    try:
+        g.__doc__ = f.__doc__
+    except (TypeError, AttributeError):
+        pass
+    try:
+        g.__dict__.update(f.__dict__)
+    except (TypeError, AttributeError):
+        pass
+    try:
+        g.__module__ = f.__module__
+    except TypeError:
+        pass
+    return g
+
+
+def nameToLabel(mname):
+    """
+    Convert a string like a variable name into a slightly more human-friendly
+    string with spaces and capitalized letters.
+
+    @type mname: C{str}
+    @param mname: The name to convert to a label.  This must be a string
+    which could be used as a Python identifier.  Strings which do not take
+    this form will result in unpredictable behavior.
+
+    @rtype: C{str}
+    """
+    labelList = []
+    word = ""
+    lastWasUpper = False
+    for letter in mname:
+        if letter.isupper() == lastWasUpper:
+            # Continuing a word.
+            word += letter
+        else:
+            # breaking a word OR beginning a word
+            if lastWasUpper:
+                # could be either
+                if len(word) == 1:
+                    # keep going
+                    word += letter
+                else:
+                    # acronym
+                    # we're processing the lowercase letter after the acronym-then-capital
+                    lastWord = word[:-1]
+                    firstLetter = word[-1]
+                    labelList.append(lastWord)
+                    word = firstLetter + letter
+            else:
+                # definitely breaking: lower to upper
+                labelList.append(word)
+                word = letter
+        lastWasUpper = letter.isupper()
+    if labelList:
+        labelList[0] = labelList[0].capitalize()
+    else:
+        return mname.capitalize()
+    labelList.append(word)
+    return " ".join(labelList)
+
+
+def uidFromString(uidString):
+    """
+    Convert a user identifier, as a string, into an integer UID.
+
+    @type uidString: C{str}
+    @param uidString: A string giving the base-ten representation of a UID or
+        the name of a user which can be converted to a UID via L{pwd.getpwnam}.
+
+    @rtype: C{int}
+    @return: The integer UID corresponding to the given string.
+
+    @raise ValueError: If the user name is supplied and L{pwd} is not
+        available.
+    """
+    try:
+        return int(uidString)
+    except ValueError:
+        if pwd is None:
+            raise
+        return pwd.getpwnam(uidString)[2]
+
+
+def gidFromString(gidString):
+    """
+    Convert a group identifier, as a string, into an integer GID.
+
+    @type gidString: C{str}
+    @param gidString: A string giving the base-ten representation of a GID or
+        the name of a group which can be converted to a GID via L{grp.getgrnam}.
+
+    @rtype: C{int}
+    @return: The integer GID corresponding to the given string.
+
+    @raise ValueError: If the group name is supplied and L{grp} is not
+        available.
+    """
+    try:
+        return int(gidString)
+    except ValueError:
+        if grp is None:
+            raise
+        return grp.getgrnam(gidString)[2]
+
+
+def runAsEffectiveUser(euid, egid, function, *args, **kwargs):
+    """
+    Run the given function wrapped with seteuid/setegid calls.
+
+    This will try to minimize the number of seteuid/setegid calls, comparing
+    current and wanted permissions
+
+    @param euid: effective UID used to call the function.
+    @type euid: C{int}
+
+    @type egid: effective GID used to call the function.
+    @param egid: C{int}
+
+    @param function: the function run with the specific permission.
+    @type function: any callable
+
+    @param args: arguments passed to C{function}
+    @param kwargs: keyword arguments passed to C{function}
+    """
+    uid, gid = os.geteuid(), os.getegid()
+    if uid == euid and gid == egid:
+        return function(*args, **kwargs)
+    else:
+        if uid != 0 and (uid != euid or gid != egid):
+            os.seteuid(0)
+        if gid != egid:
+            os.setegid(egid)
+        if euid != 0 and (euid != uid or gid != egid):
+            os.seteuid(euid)
+        try:
+            return function(*args, **kwargs)
+        finally:
+            if euid != 0 and (uid != euid or gid != egid):
+                os.seteuid(0)
+            if gid != egid:
+                os.setegid(gid)
+            if uid != 0 and (uid != euid or gid != egid):
+                os.seteuid(uid)
+
+
+def runWithWarningsSuppressed(suppressedWarnings, f, *args, **kwargs):
+    """
+    Run C{f(*args, **kwargs)}, but with some warnings suppressed.
+
+    Unlike L{twisted.internet.utils.runWithWarningsSuppressed}, it has no
+    special support for L{twisted.internet.defer.Deferred}.
+
+    @param suppressedWarnings: A list of arguments to pass to
+        L{warnings.filterwarnings}.  Must be a sequence of 2-tuples (args,
+        kwargs).
+
+    @param f: A callable.
+
+    @param args: Arguments for C{f}.
+
+    @param kwargs: Keyword arguments for C{f}
+
+    @return: The result of C{f(*args, **kwargs)}.
+    """
+    with warnings.catch_warnings():
+        for a, kw in suppressedWarnings:
+            warnings.filterwarnings(*a, **kw)
+        return f(*args, **kwargs)
+
+
+__all__ = [
+    "uniquify",
+    "padTo",
+    "getPluginDirs",
+    "addPluginDir",
+    "sibpath",
+    "getPassword",
+    "println",
+    "makeStatBar",
+    "OrderedDict",
+    "InsensitiveDict",
+    "spewer",
+    "searchupwards",
+    "LineLog",
+    "raises",
+    "IntervalDifferential",
+    "FancyStrMixin",
+    "FancyEqMixin",
+    "switchUID",
+    "mergeFunctionMetadata",
+    "nameToLabel",
+    "uidFromString",
+    "gidFromString",
+    "runAsEffectiveUser",
+    "untilConcludes",
+    "runWithWarningsSuppressed",
+]
diff --git a/contrib/python/Twisted/py3/twisted/python/versions.py b/contrib/python/Twisted/py3/twisted/python/versions.py
new file mode 100644
index 00000000000..c25b475082c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/versions.py
@@ -0,0 +1,13 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Versions for Python packages.
+
+See L{incremental}.
+"""
+
+
+from incremental import IncomparableVersions, Version, getVersionString
+
+__all__ = ["Version", "getVersionString", "IncomparableVersions"]
diff --git a/contrib/python/Twisted/py3/twisted/python/win32.py b/contrib/python/Twisted/py3/twisted/python/win32.py
new file mode 100644
index 00000000000..df1b22fa1ee
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/win32.py
@@ -0,0 +1,163 @@
+# -*- test-case-name: twisted.python.test.test_win32 -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Win32 utilities.
+
+See also twisted.python.shortcut.
+
+@var O_BINARY: the 'binary' mode flag on Windows, or 0 on other platforms, so it
+    may safely be OR'ed into a mask for os.open.
+"""
+
+import os
+import re
+
+from incremental import Version
+
+from twisted.python.deprecate import deprecatedModuleAttribute
+
+# https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes
+ERROR_FILE_NOT_FOUND = 2
+ERROR_PATH_NOT_FOUND = 3
+ERROR_INVALID_NAME = 123
+ERROR_DIRECTORY = 267
+
+O_BINARY = getattr(os, "O_BINARY", 0)
+
+
+class FakeWindowsError(OSError):
+    """
+    Stand-in for sometimes-builtin exception on platforms for which it
+    is missing.
+    """
+
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Catch OSError and check presence of 'winerror' attribute.",
+    "twisted.python.win32",
+    "FakeWindowsError",
+)
+
+
+try:
+    WindowsError: OSError = WindowsError
+except NameError:
+    WindowsError = FakeWindowsError
+
+deprecatedModuleAttribute(
+    Version("Twisted", 21, 2, 0),
+    "Catch OSError and check presence of 'winerror' attribute.",
+    "twisted.python.win32",
+    "WindowsError",
+)
+
+
+_cmdLineQuoteRe = re.compile(r'(\\*)"')
+_cmdLineQuoteRe2 = re.compile(r"(\\+)\Z")
+
+
+def cmdLineQuote(s):
+    """
+    Internal method for quoting a single command-line argument.
+
+    @param s: an unquoted string that you want to quote so that something that
+        does cmd.exe-style unquoting will interpret it as a single argument,
+        even if it contains spaces.
+    @type s: C{str}
+
+    @return: a quoted string.
+    @rtype: C{str}
+    """
+    quote = ((" " in s) or ("\t" in s) or ('"' in s) or s == "") and '"' or ""
+    return (
+        quote
+        + _cmdLineQuoteRe2.sub(r"\1\1", _cmdLineQuoteRe.sub(r'\1\1\\"', s))
+        + quote
+    )
+
+
+def quoteArguments(arguments):
+    """
+    Quote an iterable of command-line arguments for passing to CreateProcess or
+    a similar API.  This allows the list passed to C{reactor.spawnProcess} to
+    match the child process's C{sys.argv} properly.
+
+    @param arguments: an iterable of C{str}, each unquoted.
+
+    @return: a single string, with the given sequence quoted as necessary.
+    """
+    return " ".join([cmdLineQuote(a) for a in arguments])
+
+
+class _ErrorFormatter:
+    """
+    Formatter for Windows error messages.
+
+    @ivar winError: A callable which takes one integer error number argument
+        and returns a L{WindowsError} instance for that error (like
+        L{ctypes.WinError}).
+
+    @ivar formatMessage: A callable which takes one integer error number
+        argument and returns a C{str} giving the message for that error (like
+        U{win32api.FormatMessage<http://
+        timgolden.me.uk/pywin32-docs/win32api__FormatMessage_meth.html>}).
+
+    @ivar errorTab: A mapping from integer error numbers to C{str} messages
+        which correspond to those erorrs (like I{socket.errorTab}).
+    """
+
+    def __init__(self, WinError, FormatMessage, errorTab):
+        self.winError = WinError
+        self.formatMessage = FormatMessage
+        self.errorTab = errorTab
+
+    @classmethod
+    def fromEnvironment(cls):
+        """
+        Get as many of the platform-specific error translation objects as
+        possible and return an instance of C{cls} created with them.
+        """
+        try:
+            from ctypes import WinError
+        except ImportError:
+            WinError = None
+        try:
+            from win32api import FormatMessage  # type: ignore[import]
+        except ImportError:
+            FormatMessage = None
+        try:
+            from socket import errorTab
+        except ImportError:
+            errorTab = None
+        return cls(WinError, FormatMessage, errorTab)
+
+    def formatError(self, errorcode):
+        """
+        Returns the string associated with a Windows error message, such as the
+        ones found in socket.error.
+
+        Attempts direct lookup against the win32 API via ctypes and then
+        pywin32 if available), then in the error table in the socket module,
+        then finally defaulting to C{os.strerror}.
+
+        @param errorcode: the Windows error code
+        @type errorcode: C{int}
+
+        @return: The error message string
+        @rtype: C{str}
+        """
+        if self.winError is not None:
+            return self.winError(errorcode).strerror
+        if self.formatMessage is not None:
+            return self.formatMessage(errorcode)
+        if self.errorTab is not None:
+            result = self.errorTab.get(errorcode)
+            if result is not None:
+                return result
+        return os.strerror(errorcode)
+
+
+formatError = _ErrorFormatter.fromEnvironment().formatError
diff --git a/contrib/python/Twisted/py3/twisted/python/zippath.py b/contrib/python/Twisted/py3/twisted/python/zippath.py
new file mode 100644
index 00000000000..9aa9c7c6b88
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/zippath.py
@@ -0,0 +1,352 @@
+# -*- test-case-name: twisted.python.test.test_zippath -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module contains implementations of L{IFilePath} for zip files.
+
+See the constructor of L{ZipArchive} for use.
+"""
+from __future__ import annotations
+
+import errno
+import os
+import time
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    AnyStr,
+    Dict,
+    Generic,
+    Iterable,
+    List,
+    Tuple,
+    TypeVar,
+    Union,
+)
+from zipfile import ZipFile
+
+from zope.interface import implementer
+
+from typing_extensions import Literal, Self
+
+from twisted.python.compat import cmp, comparable
+from twisted.python.filepath import (
+    AbstractFilePath,
+    FilePath,
+    IFilePath,
+    OtherAnyStr,
+    UnlistableError,
+    _coerceToFilesystemEncoding,
+)
+
+ZIP_PATH_SEP = "/"  # In zipfiles, "/" is universally used as the
+# path separator, regardless of platform.
+
+_ArchiveStr = TypeVar("_ArchiveStr", bytes, str)
+_ZipStr = TypeVar("_ZipStr", bytes, str)
+_ZipSelf = TypeVar("_ZipSelf", bound="ZipPath[Any, Any]")
+
+
+@comparable
+@implementer(IFilePath)
+class ZipPath(Generic[_ZipStr, _ArchiveStr], AbstractFilePath[_ZipStr]):
+    """
+    I represent a file or directory contained within a zip file.
+    """
+
+    path: _ZipStr
+
+    def __init__(
+        self, archive: ZipArchive[_ArchiveStr], pathInArchive: _ZipStr
+    ) -> None:
+        """
+        Don't construct me directly.  Use C{ZipArchive.child()}.
+
+        @param archive: a L{ZipArchive} instance.
+
+        @param pathInArchive: a ZIP_PATH_SEP-separated string.
+        """
+        self.archive: ZipArchive[_ArchiveStr] = archive
+        self.pathInArchive: _ZipStr = pathInArchive
+        self._nativePathInArchive: _ArchiveStr = _coerceToFilesystemEncoding(
+            archive._zipfileFilename, pathInArchive
+        )
+
+        # self.path pretends to be os-specific because that's the way the
+        # 'zipimport' module does it.
+        sep = _coerceToFilesystemEncoding(pathInArchive, ZIP_PATH_SEP)
+        archiveFilename: _ZipStr = _coerceToFilesystemEncoding(
+            pathInArchive, archive._zipfileFilename
+        )
+        segments: List[_ZipStr] = self.pathInArchive.split(sep)
+        fakePath: _ZipStr = os.path.join(archiveFilename, *segments)
+        self.path: _ZipStr = fakePath
+
+    def __cmp__(self, other: object) -> int:
+        if not isinstance(other, ZipPath):
+            return NotImplemented
+        return cmp(
+            (self.archive, self.pathInArchive), (other.archive, other.pathInArchive)
+        )
+
+    def __repr__(self) -> str:
+        parts: List[_ZipStr]
+        parts = [
+            _coerceToFilesystemEncoding(self.sep, os.path.abspath(self.archive.path))
+        ]
+        parts.extend(self.pathInArchive.split(self.sep))
+        ossep = _coerceToFilesystemEncoding(self.sep, os.sep)
+        return f"ZipPath({ossep.join(parts)!r})"
+
+    @property
+    def sep(self) -> _ZipStr:
+        """
+        Return a zip directory separator.
+
+        @return: The zip directory separator.
+        @returntype: The same type as C{self.path}.
+        """
+        return _coerceToFilesystemEncoding(self.path, ZIP_PATH_SEP)
+
+    def _nativeParent(
+        self,
+    ) -> Union[ZipPath[_ZipStr, _ArchiveStr], ZipArchive[_ArchiveStr]]:
+        """
+        Return parent, discarding our own encoding in favor of whatever the
+        archive's is.
+        """
+        splitup = self.pathInArchive.split(self.sep)
+        if len(splitup) == 1:
+            return self.archive
+        return ZipPath(self.archive, self.sep.join(splitup[:-1]))
+
+    def parent(self) -> Union[ZipPath[_ZipStr, _ArchiveStr], ZipArchive[_ZipStr]]:
+        parent = self._nativeParent()
+        if isinstance(parent, ZipArchive):
+            return ZipArchive(
+                _coerceToFilesystemEncoding(self.path, self.archive._zipfileFilename)
+            )
+        return parent
+
+    if TYPE_CHECKING:
+
+        def parents(
+            self,
+        ) -> Iterable[Union[ZipPath[_ZipStr, _ArchiveStr], ZipArchive[_ZipStr]]]:
+            ...
+
+    def child(self, path: OtherAnyStr) -> ZipPath[OtherAnyStr, _ArchiveStr]:
+        """
+        Return a new ZipPath representing a path in C{self.archive} which is
+        a child of this path.
+
+        @note: Requesting the C{".."} (or other special name) child will not
+            cause L{InsecurePath} to be raised since these names do not have
+            any special meaning inside a zip archive.  Be particularly
+            careful with the C{path} attribute (if you absolutely must use
+            it) as this means it may include special names with special
+            meaning outside of the context of a zip archive.
+        """
+        joiner = _coerceToFilesystemEncoding(path, ZIP_PATH_SEP)
+        pathInArchive = _coerceToFilesystemEncoding(path, self.pathInArchive)
+        return ZipPath(self.archive, joiner.join([pathInArchive, path]))
+
+    def sibling(self, path: OtherAnyStr) -> ZipPath[OtherAnyStr, _ArchiveStr]:
+        parent: Union[ZipPath[_ZipStr, _ArchiveStr], ZipArchive[_ZipStr]]
+        rightTypedParent: Union[ZipPath[_ZipStr, _ArchiveStr], ZipArchive[_ArchiveStr]]
+
+        parent = self.parent()
+        rightTypedParent = self.archive if isinstance(parent, ZipArchive) else parent
+        child: ZipPath[OtherAnyStr, _ArchiveStr] = rightTypedParent.child(path)
+        return child
+
+    def exists(self) -> bool:
+        return self.isdir() or self.isfile()
+
+    def isdir(self) -> bool:
+        return self.pathInArchive in self.archive.childmap
+
+    def isfile(self) -> bool:
+        return self.pathInArchive in self.archive.zipfile.NameToInfo
+
+    def islink(self) -> bool:
+        return False
+
+    def listdir(self) -> List[_ZipStr]:
+        if self.exists():
+            if self.isdir():
+                parentArchivePath: _ArchiveStr = _coerceToFilesystemEncoding(
+                    self.archive._zipfileFilename, self.pathInArchive
+                )
+                return [
+                    _coerceToFilesystemEncoding(self.path, each)
+                    for each in self.archive.childmap[parentArchivePath].keys()
+                ]
+            else:
+                raise UnlistableError(OSError(errno.ENOTDIR, "Leaf zip entry listed"))
+        else:
+            raise UnlistableError(
+                OSError(errno.ENOENT, "Non-existent zip entry listed")
+            )
+
+    def splitext(self) -> Tuple[_ZipStr, _ZipStr]:
+        """
+        Return a value similar to that returned by C{os.path.splitext}.
+        """
+        # This happens to work out because of the fact that we use OS-specific
+        # path separators in the constructor to construct our fake 'path'
+        # attribute.
+        return os.path.splitext(self.path)
+
+    def basename(self) -> _ZipStr:
+        return self.pathInArchive.split(self.sep)[-1]
+
+    def dirname(self) -> _ZipStr:
+        # XXX NOTE: This API isn't a very good idea on filepath, but it's even
+        # less meaningful here.
+        return self.parent().path
+
+    def open(self, mode: Literal["r", "w"] = "r") -> IO[bytes]:  # type:ignore[override]
+        # TODO: liskov substitutability is broken here because the stdlib
+        # zipfile does not support appending to files within archives, only to
+        # archives themselves; we could fix this by emulating append mode.
+        pathInArchive = _coerceToFilesystemEncoding("", self.pathInArchive)
+        return self.archive.zipfile.open(pathInArchive, mode=mode)
+
+    def changed(self) -> None:
+        pass
+
+    def getsize(self) -> int:
+        """
+        Retrieve this file's size.
+
+        @return: file size, in bytes
+        """
+        pathInArchive = _coerceToFilesystemEncoding("", self.pathInArchive)
+        return self.archive.zipfile.NameToInfo[pathInArchive].file_size
+
+    def getAccessTime(self) -> float:
+        """
+        Retrieve this file's last access-time.  This is the same as the last access
+        time for the archive.
+
+        @return: a number of seconds since the epoch
+        """
+        return self.archive.getAccessTime()
+
+    def getModificationTime(self) -> float:
+        """
+        Retrieve this file's last modification time.  This is the time of
+        modification recorded in the zipfile.
+
+        @return: a number of seconds since the epoch.
+        """
+        pathInArchive = _coerceToFilesystemEncoding("", self.pathInArchive)
+        return time.mktime(
+            self.archive.zipfile.NameToInfo[pathInArchive].date_time + (0, 0, 0)
+        )
+
+    def getStatusChangeTime(self) -> float:
+        """
+        Retrieve this file's last modification time.  This name is provided for
+        compatibility, and returns the same value as getmtime.
+
+        @return: a number of seconds since the epoch.
+        """
+        return self.getModificationTime()
+
+
+class ZipArchive(ZipPath[AnyStr, AnyStr]):
+    """
+    I am a L{FilePath}-like object which can wrap a zip archive as if it were a
+    directory.
+
+    It works similarly to L{FilePath} in L{bytes} and L{unicode} handling --
+    instantiating with a L{bytes} will return a "bytes mode" L{ZipArchive},
+    and instantiating with a L{unicode} will return a "text mode"
+    L{ZipArchive}. Methods that return new L{ZipArchive} or L{ZipPath}
+    instances will be in the mode of the argument to the creator method,
+    converting if required.
+    """
+
+    _zipfileFilename: AnyStr
+
+    @property
+    def archive(self) -> Self:  # type: ignore[override]
+        return self
+
+    def __init__(self, archivePathname: AnyStr) -> None:
+        """
+        Create a ZipArchive, treating the archive at archivePathname as a zip
+        file.
+
+        @param archivePathname: a L{bytes} or L{unicode}, naming a path in the
+            filesystem.
+        """
+        self.path = archivePathname
+        self.zipfile = ZipFile(_coerceToFilesystemEncoding("", archivePathname))
+        zfname = self.zipfile.filename
+        assert (
+            zfname is not None
+        ), "zipfile must have filename when initialized with a path"
+        self._zipfileFilename = _coerceToFilesystemEncoding(archivePathname, zfname)
+        self.pathInArchive = _coerceToFilesystemEncoding(archivePathname, "")
+        # zipfile is already wasting O(N) memory on cached ZipInfo instances,
+        # so there's no sense in trying to do this lazily or intelligently
+        self.childmap: Dict[AnyStr, Dict[AnyStr, int]] = {}
+
+        for name in self.zipfile.namelist():
+            splitName = _coerceToFilesystemEncoding(self.path, name).split(self.sep)
+            for x in range(len(splitName)):
+                child = splitName[-x]
+                parent = self.sep.join(splitName[:-x])
+                if parent not in self.childmap:
+                    self.childmap[parent] = {}
+                self.childmap[parent][child] = 1
+            parent = _coerceToFilesystemEncoding(archivePathname, "")
+
+    def __cmp__(self, other: object) -> int:
+        if not isinstance(other, ZipArchive):
+            return NotImplemented
+        return cmp(self.path, other.path)
+
+    def child(self, path: OtherAnyStr) -> ZipPath[OtherAnyStr, AnyStr]:
+        """
+        Create a ZipPath pointing at a path within the archive.
+
+        @param path: a L{bytes} or L{unicode} with no path separators in it
+            (either '/' or the system path separator, if it's different).
+        """
+        return ZipPath(self, path)
+
+    def exists(self) -> bool:
+        """
+        Returns C{True} if the underlying archive exists.
+        """
+        return FilePath(self._zipfileFilename).exists()
+
+    def getAccessTime(self) -> float:
+        """
+        Return the archive file's last access time.
+        """
+        return FilePath(self._zipfileFilename).getAccessTime()
+
+    def getModificationTime(self) -> float:
+        """
+        Return the archive file's modification time.
+        """
+        return FilePath(self._zipfileFilename).getModificationTime()
+
+    def getStatusChangeTime(self) -> float:
+        """
+        Return the archive file's status change time.
+        """
+        return FilePath(self._zipfileFilename).getStatusChangeTime()
+
+    def __repr__(self) -> str:
+        return f"ZipArchive({os.path.abspath(self.path)!r})"
+
+
+__all__ = ["ZipArchive", "ZipPath"]
diff --git a/contrib/python/Twisted/py3/twisted/python/zipstream.py b/contrib/python/Twisted/py3/twisted/python/zipstream.py
new file mode 100644
index 00000000000..f9e3c4cfc52
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/python/zipstream.py
@@ -0,0 +1,319 @@
+# -*- test-case-name: twisted.python.test.test_zipstream -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An incremental approach to unzipping files.  This allows you to unzip a little
+bit of a file at a time, which means you can report progress as a file unzips.
+"""
+
+import os.path
+import struct
+import zipfile
+import zlib
+
+
+class ChunkingZipFile(zipfile.ZipFile):
+    """
+    A L{zipfile.ZipFile} object which, with L{readfile}, also gives you access
+    to a file-like object for each entry.
+    """
+
+    def readfile(self, name):
+        """
+        Return file-like object for name.
+        """
+        if self.mode not in ("r", "a"):
+            raise RuntimeError('read() requires mode "r" or "a"')
+        if not self.fp:
+            raise RuntimeError("Attempt to read ZIP archive that was already closed")
+        zinfo = self.getinfo(name)
+
+        self.fp.seek(zinfo.header_offset, 0)
+
+        fheader = self.fp.read(zipfile.sizeFileHeader)
+        if fheader[0:4] != zipfile.stringFileHeader:
+            raise zipfile.BadZipFile("Bad magic number for file header")
+
+        fheader = struct.unpack(zipfile.structFileHeader, fheader)
+        fname = self.fp.read(fheader[zipfile._FH_FILENAME_LENGTH])
+
+        if fheader[zipfile._FH_EXTRA_FIELD_LENGTH]:
+            self.fp.read(fheader[zipfile._FH_EXTRA_FIELD_LENGTH])
+
+        if zinfo.flag_bits & 0x800:
+            # UTF-8 filename
+            fname_str = fname.decode("utf-8")
+        else:
+            fname_str = fname.decode("cp437")
+
+        if fname_str != zinfo.orig_filename:
+            raise zipfile.BadZipFile(
+                'File name in directory "%s" and header "%s" differ.'
+                % (zinfo.orig_filename, fname_str)
+            )
+
+        if zinfo.compress_type == zipfile.ZIP_STORED:
+            return ZipFileEntry(self, zinfo.compress_size)
+        elif zinfo.compress_type == zipfile.ZIP_DEFLATED:
+            return DeflatedZipFileEntry(self, zinfo.compress_size)
+        else:
+            raise zipfile.BadZipFile(
+                "Unsupported compression method %d for file %s"
+                % (zinfo.compress_type, name)
+            )
+
+
+class _FileEntry:
+    """
+    Abstract superclass of both compressed and uncompressed variants of
+    file-like objects within a zip archive.
+
+    @ivar chunkingZipFile: a chunking zip file.
+    @type chunkingZipFile: L{ChunkingZipFile}
+
+    @ivar length: The number of bytes within the zip file that represent this
+    file.  (This is the size on disk, not the number of decompressed bytes
+    which will result from reading it.)
+
+    @ivar fp: the underlying file object (that contains pkzip data).  Do not
+    touch this, please.  It will quite likely move or go away.
+
+    @ivar closed: File-like 'closed' attribute; True before this file has been
+    closed, False after.
+    @type closed: L{bool}
+
+    @ivar finished: An older, broken synonym for 'closed'.  Do not touch this,
+    please.
+    @type finished: L{int}
+    """
+
+    def __init__(self, chunkingZipFile, length):
+        """
+        Create a L{_FileEntry} from a L{ChunkingZipFile}.
+        """
+        self.chunkingZipFile = chunkingZipFile
+        self.fp = self.chunkingZipFile.fp
+        self.length = length
+        self.finished = 0
+        self.closed = False
+
+    def isatty(self):
+        """
+        Returns false because zip files should not be ttys
+        """
+        return False
+
+    def close(self):
+        """
+        Close self (file-like object)
+        """
+        self.closed = True
+        self.finished = 1
+        del self.fp
+
+    def readline(self):
+        """
+        Read a line.
+        """
+        line = b""
+        for byte in iter(lambda: self.read(1), b""):
+            line += byte
+            if byte == b"\n":
+                break
+        return line
+
+    def __next__(self):
+        """
+        Implement next as file does (like readline, except raises StopIteration
+        at EOF)
+        """
+        nextline = self.readline()
+        if nextline:
+            return nextline
+        raise StopIteration()
+
+    # Iterators on Python 2 use next(), not __next__()
+    next = __next__
+
+    def readlines(self):
+        """
+        Returns a list of all the lines
+        """
+        return list(self)
+
+    def xreadlines(self):
+        """
+        Returns an iterator (so self)
+        """
+        return self
+
+    def __iter__(self):
+        """
+        Returns an iterator (so self)
+        """
+        return self
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+
+class ZipFileEntry(_FileEntry):
+    """
+    File-like object used to read an uncompressed entry in a ZipFile
+    """
+
+    def __init__(self, chunkingZipFile, length):
+        _FileEntry.__init__(self, chunkingZipFile, length)
+        self.readBytes = 0
+
+    def tell(self):
+        return self.readBytes
+
+    def read(self, n=None):
+        if n is None:
+            n = self.length - self.readBytes
+        if n == 0 or self.finished:
+            return b""
+        data = self.chunkingZipFile.fp.read(min(n, self.length - self.readBytes))
+        self.readBytes += len(data)
+        if self.readBytes == self.length or len(data) < n:
+            self.finished = 1
+        return data
+
+
+class DeflatedZipFileEntry(_FileEntry):
+    """
+    File-like object used to read a deflated entry in a ZipFile
+    """
+
+    def __init__(self, chunkingZipFile, length):
+        _FileEntry.__init__(self, chunkingZipFile, length)
+        self.returnedBytes = 0
+        self.readBytes = 0
+        self.decomp = zlib.decompressobj(-15)
+        self.buffer = b""
+
+    def tell(self):
+        return self.returnedBytes
+
+    def read(self, n=None):
+        if self.finished:
+            return b""
+        if n is None:
+            result = [
+                self.buffer,
+            ]
+            result.append(
+                self.decomp.decompress(
+                    self.chunkingZipFile.fp.read(self.length - self.readBytes)
+                )
+            )
+            result.append(self.decomp.decompress(b"Z"))
+            result.append(self.decomp.flush())
+            self.buffer = b""
+            self.finished = 1
+            result = b"".join(result)
+            self.returnedBytes += len(result)
+            return result
+        else:
+            while len(self.buffer) < n:
+                data = self.chunkingZipFile.fp.read(
+                    min(n, 1024, self.length - self.readBytes)
+                )
+                self.readBytes += len(data)
+                if not data:
+                    result = (
+                        self.buffer + self.decomp.decompress(b"Z") + self.decomp.flush()
+                    )
+                    self.finished = 1
+                    self.buffer = b""
+                    self.returnedBytes += len(result)
+                    return result
+                else:
+                    self.buffer += self.decomp.decompress(data)
+            result = self.buffer[:n]
+            self.buffer = self.buffer[n:]
+            self.returnedBytes += len(result)
+            return result
+
+
+DIR_BIT = 16
+
+
+def countZipFileChunks(filename, chunksize):
+    """
+    Predict the number of chunks that will be extracted from the entire
+    zipfile, given chunksize blocks.
+    """
+    totalchunks = 0
+    zf = ChunkingZipFile(filename)
+    for info in zf.infolist():
+        totalchunks += countFileChunks(info, chunksize)
+    return totalchunks
+
+
+def countFileChunks(zipinfo, chunksize):
+    """
+    Count the number of chunks that will result from the given C{ZipInfo}.
+
+    @param zipinfo: a C{zipfile.ZipInfo} instance describing an entry in a zip
+    archive to be counted.
+
+    @return: the number of chunks present in the zip file.  (Even an empty file
+    counts as one chunk.)
+    @rtype: L{int}
+    """
+    count, extra = divmod(zipinfo.file_size, chunksize)
+    if extra > 0:
+        count += 1
+    return count or 1
+
+
+def unzipIterChunky(filename, directory=".", overwrite=0, chunksize=4096):
+    """
+    Return a generator for the zipfile.  This implementation will yield after
+    every chunksize uncompressed bytes, or at the end of a file, whichever
+    comes first.
+
+    The value it yields is the number of chunks left to unzip.
+    """
+    czf = ChunkingZipFile(filename, "r")
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    remaining = countZipFileChunks(filename, chunksize)
+    names = czf.namelist()
+    infos = czf.infolist()
+
+    for entry, info in zip(names, infos):
+        isdir = info.external_attr & DIR_BIT
+        f = os.path.join(directory, entry)
+        if isdir:
+            # overwrite flag only applies to files
+            if not os.path.exists(f):
+                os.makedirs(f)
+            remaining -= 1
+            yield remaining
+        else:
+            # create the directory the file will be in first,
+            # since we can't guarantee it exists
+            fdir = os.path.split(f)[0]
+            if not os.path.exists(fdir):
+                os.makedirs(fdir)
+            if overwrite or not os.path.exists(f):
+                fp = czf.readfile(entry)
+                if info.file_size == 0:
+                    remaining -= 1
+                    yield remaining
+                with open(f, "wb") as outfile:
+                    while fp.tell() < info.file_size:
+                        hunk = fp.read(chunksize)
+                        outfile.write(hunk)
+                        remaining -= 1
+                        yield remaining
+            else:
+                remaining -= countFileChunks(info, chunksize)
+                yield remaining
diff --git a/contrib/python/Twisted/py3/twisted/runner/__init__.py b/contrib/python/Twisted/py3/twisted/runner/__init__.py
new file mode 100644
index 00000000000..024a284959f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/runner/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Runner: Run and monitor processes.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/runner/inetd.py b/contrib/python/Twisted/py3/twisted/runner/inetd.py
new file mode 100644
index 00000000000..0d56e04ba8c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/runner/inetd.py
@@ -0,0 +1,80 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+"""
+Twisted inetd.
+
+Maintainer: Andrew Bennetts
+
+Future Plans: Bugfixes.  Specifically for UDP and Sun-RPC, which don't work
+correctly yet.
+"""
+
+import os
+
+from twisted.internet import fdesc, process, reactor
+from twisted.internet.protocol import Protocol, ServerFactory
+from twisted.protocols import wire
+
+# A dict of known 'internal' services (i.e. those that don't involve spawning
+# another process.
+internalProtocols = {
+    "echo": wire.Echo,
+    "chargen": wire.Chargen,
+    "discard": wire.Discard,
+    "daytime": wire.Daytime,
+    "time": wire.Time,
+}
+
+
+class InetdProtocol(Protocol):
+    """Forks a child process on connectionMade, passing the socket as fd 0."""
+
+    def connectionMade(self):
+        sockFD = self.transport.fileno()
+        childFDs = {0: sockFD, 1: sockFD}
+        if self.factory.stderrFile:
+            childFDs[2] = self.factory.stderrFile.fileno()
+
+        # processes run by inetd expect blocking sockets
+        # FIXME: maybe this should be done in process.py?  are other uses of
+        #        Process possibly affected by this?
+        fdesc.setBlocking(sockFD)
+        if 2 in childFDs:
+            fdesc.setBlocking(childFDs[2])
+
+        service = self.factory.service
+        uid = service.user
+        gid = service.group
+
+        # don't tell Process to change our UID/GID if it's what we
+        # already are
+        if uid == os.getuid():
+            uid = None
+        if gid == os.getgid():
+            gid = None
+
+        process.Process(
+            None,
+            service.program,
+            service.programArgs,
+            os.environ,
+            None,
+            None,
+            uid,
+            gid,
+            childFDs,
+        )
+
+        reactor.removeReader(self.transport)
+        reactor.removeWriter(self.transport)
+
+
+class InetdFactory(ServerFactory):
+    protocol = InetdProtocol
+    stderrFile = None
+
+    def __init__(self, service):
+        self.service = service
diff --git a/contrib/python/Twisted/py3/twisted/runner/inetdconf.py b/contrib/python/Twisted/py3/twisted/runner/inetdconf.py
new file mode 100644
index 00000000000..d54c5a6f755
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/runner/inetdconf.py
@@ -0,0 +1,203 @@
+# -*- test-case-name: twisted.runner.test.test_inetdconf -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Parser for inetd.conf files
+"""
+
+from typing import Optional
+
+
+# Various exceptions
+class InvalidConfError(Exception):
+    """
+    Invalid configuration file
+    """
+
+
+class InvalidInetdConfError(InvalidConfError):
+    """
+    Invalid inetd.conf file
+    """
+
+
+class InvalidServicesConfError(InvalidConfError):
+    """
+    Invalid services file
+    """
+
+
+class UnknownService(Exception):
+    """
+    Unknown service name
+    """
+
+
+class SimpleConfFile:
+    """
+    Simple configuration file parser superclass.
+
+    Filters out comments and empty lines (which includes lines that only
+    contain comments).
+
+    To use this class, override parseLine or parseFields.
+    """
+
+    commentChar = "#"
+    defaultFilename: Optional[str] = None
+
+    def parseFile(self, file=None):
+        """
+        Parse a configuration file
+
+        If file is None and self.defaultFilename is set, it will open
+        defaultFilename and use it.
+        """
+        close = False
+        if file is None and self.defaultFilename:
+            file = open(self.defaultFilename)
+            close = True
+
+        try:
+            for line in file.readlines():
+                # Strip out comments
+                comment = line.find(self.commentChar)
+                if comment != -1:
+                    line = line[:comment]
+
+                # Strip whitespace
+                line = line.strip()
+
+                # Skip empty lines (and lines which only contain comments)
+                if not line:
+                    continue
+
+                self.parseLine(line)
+        finally:
+            if close:
+                file.close()
+
+    def parseLine(self, line):
+        """
+        Override this.
+
+        By default, this will split the line on whitespace and call
+        self.parseFields (catching any errors).
+        """
+        try:
+            self.parseFields(*line.split())
+        except ValueError:
+            raise InvalidInetdConfError("Invalid line: " + repr(line))
+
+    def parseFields(self, *fields):
+        """
+        Override this.
+        """
+
+
+class InetdService:
+    """
+    A simple description of an inetd service.
+    """
+
+    name = None
+    port = None
+    socketType = None
+    protocol = None
+    wait = None
+    user = None
+    group = None
+    program = None
+    programArgs = None
+
+    def __init__(
+        self, name, port, socketType, protocol, wait, user, group, program, programArgs
+    ):
+        self.name = name
+        self.port = port
+        self.socketType = socketType
+        self.protocol = protocol
+        self.wait = wait
+        self.user = user
+        self.group = group
+        self.program = program
+        self.programArgs = programArgs
+
+
+class InetdConf(SimpleConfFile):
+    """
+    Configuration parser for a traditional UNIX inetd(8)
+    """
+
+    defaultFilename = "/etc/inetd.conf"
+
+    def __init__(self, knownServices=None):
+        self.services = []
+
+        if knownServices is None:
+            knownServices = ServicesConf()
+            knownServices.parseFile()
+        self.knownServices = knownServices
+
+    def parseFields(
+        self, serviceName, socketType, protocol, wait, user, program, *programArgs
+    ):
+        """
+        Parse an inetd.conf file.
+
+        Implemented from the description in the Debian inetd.conf man page.
+        """
+        # Extract user (and optional group)
+        user, group = (user.split(".") + [None])[:2]
+
+        # Find the port for a service
+        port = self.knownServices.services.get((serviceName, protocol), None)
+        if not port and not protocol.startswith("rpc/"):
+            # FIXME: Should this be discarded/ignored, rather than throwing
+            #        an exception?
+            try:
+                port = int(serviceName)
+                serviceName = "unknown"
+            except BaseException:
+                raise UnknownService(f"Unknown service: {serviceName} ({protocol})")
+
+        self.services.append(
+            InetdService(
+                serviceName,
+                port,
+                socketType,
+                protocol,
+                wait,
+                user,
+                group,
+                program,
+                programArgs,
+            )
+        )
+
+
+class ServicesConf(SimpleConfFile):
+    """
+    /etc/services parser
+
+    @ivar services: dict mapping service names to (port, protocol) tuples.
+    """
+
+    defaultFilename = "/etc/services"
+
+    def __init__(self):
+        self.services = {}
+
+    def parseFields(self, name, portAndProtocol, *aliases):
+        try:
+            port, protocol = portAndProtocol.split("/")
+            port = int(port)
+        except BaseException:
+            raise InvalidServicesConfError(
+                f"Invalid port/protocol: {repr(portAndProtocol)}"
+            )
+
+        self.services[(name, protocol)] = port
+        for alias in aliases:
+            self.services[(alias, protocol)] = port
diff --git a/contrib/python/Twisted/py3/twisted/runner/inetdtap.py b/contrib/python/Twisted/py3/twisted/runner/inetdtap.py
new file mode 100644
index 00000000000..b5a5b40948c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/runner/inetdtap.py
@@ -0,0 +1,109 @@
+# -*- test-case-name: twisted.runner.test.test_inetdtap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted inetd TAP support
+
+The purpose of inetdtap is to provide an inetd-like server, to allow Twisted to
+invoke other programs to handle incoming sockets.
+This is a useful thing as a "networking swiss army knife" tool, like netcat.
+"""
+
+import grp
+import pwd
+import socket
+
+from twisted.application import internet, service as appservice
+from twisted.internet.protocol import ServerFactory
+from twisted.python import log, usage
+from twisted.runner import inetd, inetdconf
+
+# Protocol map
+protocolDict = {"tcp": socket.IPPROTO_TCP, "udp": socket.IPPROTO_UDP}
+
+
+class Options(usage.Options):
+    """
+    To use it, create a file named `sample-inetd.conf` with:
+
+    8123 stream tcp wait some_user /bin/cat -
+
+    You can then run it as in the following example and port 8123 became an
+    echo server.
+
+    twistd -n inetd -f sample-inetd.conf
+    """
+
+    optParameters = [
+        ["rpc", "r", "/etc/rpc", "DEPRECATED. RPC procedure table file"],
+        ["file", "f", "/etc/inetd.conf", "Service configuration file"],
+    ]
+
+    optFlags = [["nointernal", "i", "Don't run internal services"]]
+
+    compData = usage.Completions(optActions={"file": usage.CompleteFiles("*.conf")})
+
+
+def makeService(config):
+    s = appservice.MultiService()
+    conf = inetdconf.InetdConf()
+    with open(config["file"]) as f:
+        conf.parseFile(f)
+
+    for service in conf.services:
+        protocol = service.protocol
+
+        if service.protocol.startswith("rpc/"):
+            log.msg("Skipping rpc service due to lack of rpc support")
+            continue
+
+        if (protocol, service.socketType) not in [("tcp", "stream"), ("udp", "dgram")]:
+            log.msg(
+                "Skipping unsupported type/protocol: %s/%s"
+                % (service.socketType, service.protocol)
+            )
+            continue
+
+        # Convert the username into a uid (if necessary)
+        try:
+            service.user = int(service.user)
+        except ValueError:
+            try:
+                service.user = pwd.getpwnam(service.user)[2]
+            except KeyError:
+                log.msg("Unknown user: " + service.user)
+                continue
+
+        # Convert the group name into a gid (if necessary)
+        if service.group is None:
+            # If no group was specified, use the user's primary group
+            service.group = pwd.getpwuid(service.user)[3]
+        else:
+            try:
+                service.group = int(service.group)
+            except ValueError:
+                try:
+                    service.group = grp.getgrnam(service.group)[2]
+                except KeyError:
+                    log.msg("Unknown group: " + service.group)
+                    continue
+
+        if service.program == "internal":
+            if config["nointernal"]:
+                continue
+
+            # Internal services can use a standard ServerFactory
+            if service.name not in inetd.internalProtocols:
+                log.msg("Unknown internal service: " + service.name)
+                continue
+            factory = ServerFactory()
+            factory.protocol = inetd.internalProtocols[service.name]
+        else:
+            factory = inetd.InetdFactory(service)
+
+        if protocol == "tcp":
+            internet.TCPServer(service.port, factory).setServiceParent(s)
+        elif protocol == "udp":
+            raise RuntimeError("not supporting UDP")
+    return s
diff --git a/contrib/python/Twisted/py3/twisted/runner/newsfragments/11681.misc b/contrib/python/Twisted/py3/twisted/runner/newsfragments/11681.misc
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/Twisted/py3/twisted/runner/newsfragments/9657.doc b/contrib/python/Twisted/py3/twisted/runner/newsfragments/9657.doc
new file mode 100644
index 00000000000..fd2f02b2169
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/runner/newsfragments/9657.doc
@@ -0,0 +1 @@
+twisted.runner.procmon now logs to a twisted.logger.Logger instance defined on instances of ProcessMonitor instead of to the global legacy twisted.python.log instance.
diff --git a/contrib/python/Twisted/py3/twisted/runner/procmon.py b/contrib/python/Twisted/py3/twisted/runner/procmon.py
new file mode 100644
index 00000000000..8b3749a6a43
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/runner/procmon.py
@@ -0,0 +1,407 @@
+# -*- test-case-name: twisted.runner.test.test_procmon -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support for starting, monitoring, and restarting child process.
+"""
+from typing import Dict, List, Optional
+
+import attr
+import incremental
+
+from twisted.application import service
+from twisted.internet import error, protocol, reactor as _reactor
+from twisted.logger import Logger
+from twisted.protocols import basic
+from twisted.python import deprecate
+
+
+@attr.s(frozen=True, auto_attribs=True)
+class _Process:
+    """
+    The parameters of a process to be restarted.
+
+    @ivar args: command-line arguments (including name of command as first one)
+    @type args: C{list}
+
+    @ivar uid: user-id to run process as, or None (which means inherit uid)
+    @type uid: C{int}
+
+    @ivar gid: group-id to run process as, or None (which means inherit gid)
+    @type gid: C{int}
+
+    @ivar env: environment for process
+    @type env: C{dict}
+
+    @ivar cwd: initial working directory for process or None
+               (which means inherit cwd)
+    @type cwd: C{str}
+    """
+
+    args: List[str]
+    uid: Optional[int] = None
+    gid: Optional[int] = None
+    env: Dict[str, str] = attr.ib(default=attr.Factory(dict))
+    cwd: Optional[str] = None
+
+    @deprecate.deprecated(incremental.Version("Twisted", 18, 7, 0))
+    def toTuple(self):
+        """
+        Convert process to tuple.
+
+        Convert process to tuple that looks like the legacy structure
+        of processes, for potential users who inspected processes
+        directly.
+
+        This was only an accidental feature, and will be removed. If
+        you need to remember what processes were added to a process monitor,
+        keep track of that when they are added. The process list
+        inside the process monitor is no longer a public API.
+
+        This allows changing the internal structure of the process list,
+        when warranted by bug fixes or additional features.
+
+        @return: tuple representation of process
+        """
+        return (self.args, self.uid, self.gid, self.env)
+
+
+class DummyTransport:
+    disconnecting = 0
+
+
+transport = DummyTransport()
+
+
+class LineLogger(basic.LineReceiver):
+    tag = None
+    stream = None
+    delimiter = b"\n"
+    service = None
+
+    def lineReceived(self, line):
+        try:
+            line = line.decode("utf-8")
+        except UnicodeDecodeError:
+            line = repr(line)
+
+        self.service.log.info(
+            "[{tag}] {line}", tag=self.tag, line=line, stream=self.stream
+        )
+
+
+class LoggingProtocol(protocol.ProcessProtocol):
+    service = None
+    name = None
+
+    def connectionMade(self):
+        self._output = LineLogger()
+        self._output.tag = self.name
+        self._output.stream = "stdout"
+        self._output.service = self.service
+        self._outputEmpty = True
+
+        self._error = LineLogger()
+        self._error.tag = self.name
+        self._error.stream = "stderr"
+        self._error.service = self.service
+        self._errorEmpty = True
+
+        self._output.makeConnection(transport)
+        self._error.makeConnection(transport)
+
+    def outReceived(self, data):
+        self._output.dataReceived(data)
+        self._outputEmpty = data[-1] == b"\n"
+
+    def errReceived(self, data):
+        self._error.dataReceived(data)
+        self._errorEmpty = data[-1] == b"\n"
+
+    def processEnded(self, reason):
+        if not self._outputEmpty:
+            self._output.dataReceived(b"\n")
+        if not self._errorEmpty:
+            self._error.dataReceived(b"\n")
+        self.service.connectionLost(self.name)
+
+    @property
+    def output(self):
+        return self._output
+
+    @property
+    def empty(self):
+        return self._outputEmpty
+
+
+class ProcessMonitor(service.Service):
+    """
+    ProcessMonitor runs processes, monitors their progress, and restarts
+    them when they die.
+
+    The ProcessMonitor will not attempt to restart a process that appears to
+    die instantly -- with each "instant" death (less than 1 second, by
+    default), it will delay approximately twice as long before restarting
+    it.  A successful run will reset the counter.
+
+    The primary interface is L{addProcess} and L{removeProcess}. When the
+    service is running (that is, when the application it is attached to is
+    running), adding a process automatically starts it.
+
+    Each process has a name. This name string must uniquely identify the
+    process.  In particular, attempting to add two processes with the same
+    name will result in a C{KeyError}.
+
+    @type threshold: C{float}
+    @ivar threshold: How long a process has to live before the death is
+        considered instant, in seconds.  The default value is 1 second.
+
+    @type killTime: C{float}
+    @ivar killTime: How long a process being killed has to get its affairs
+        in order before it gets killed with an unmaskable signal.  The
+        default value is 5 seconds.
+
+    @type minRestartDelay: C{float}
+    @ivar minRestartDelay: The minimum time (in seconds) to wait before
+        attempting to restart a process.  Default 1s.
+
+    @type maxRestartDelay: C{float}
+    @ivar maxRestartDelay: The maximum time (in seconds) to wait before
+        attempting to restart a process.  Default 3600s (1h).
+
+    @type _reactor: L{IReactorProcess} provider
+    @ivar _reactor: A provider of L{IReactorProcess} and L{IReactorTime}
+        which will be used to spawn processes and register delayed calls.
+
+    @type log: L{Logger}
+    @ivar log: The logger used to propagate log messages from spawned
+        processes.
+
+    """
+
+    threshold = 1
+    killTime = 5
+    minRestartDelay = 1
+    maxRestartDelay = 3600
+    log = Logger()
+
+    def __init__(self, reactor=_reactor):
+        self._reactor = reactor
+
+        self._processes = {}
+        self.protocols = {}
+        self.delay = {}
+        self.timeStarted = {}
+        self.murder = {}
+        self.restart = {}
+
+    @deprecate.deprecatedProperty(incremental.Version("Twisted", 18, 7, 0))
+    def processes(self):
+        """
+        Processes as dict of tuples
+
+        @return: Dict of process name to monitored processes as tuples
+        """
+        return {name: process.toTuple() for name, process in self._processes.items()}
+
+    @deprecate.deprecated(incremental.Version("Twisted", 18, 7, 0))
+    def __getstate__(self):
+        dct = service.Service.__getstate__(self)
+        del dct["_reactor"]
+        dct["protocols"] = {}
+        dct["delay"] = {}
+        dct["timeStarted"] = {}
+        dct["murder"] = {}
+        dct["restart"] = {}
+        del dct["_processes"]
+        dct["processes"] = self.processes
+        return dct
+
+    def addProcess(self, name, args, uid=None, gid=None, env={}, cwd=None):
+        """
+        Add a new monitored process and start it immediately if the
+        L{ProcessMonitor} service is running.
+
+        Note that args are passed to the system call, not to the shell. If
+        running the shell is desired, the common idiom is to use
+        C{ProcessMonitor.addProcess("name", ['/bin/sh', '-c', shell_script])}
+
+        @param name: A name for this process.  This value must be
+            unique across all processes added to this monitor.
+        @type name: C{str}
+        @param args: The argv sequence for the process to launch.
+        @param uid: The user ID to use to run the process.  If L{None},
+            the current UID is used.
+        @type uid: C{int}
+        @param gid: The group ID to use to run the process.  If L{None},
+            the current GID is used.
+        @type uid: C{int}
+        @param env: The environment to give to the launched process. See
+            L{IReactorProcess.spawnProcess}'s C{env} parameter.
+        @type env: C{dict}
+        @param cwd: The initial working directory of the launched process.
+            The default of C{None} means inheriting the laucnhing process's
+            working directory.
+        @type env: C{dict}
+        @raise KeyError: If a process with the given name already exists.
+        """
+        if name in self._processes:
+            raise KeyError(f"remove {name} first")
+        self._processes[name] = _Process(args, uid, gid, env, cwd)
+        self.delay[name] = self.minRestartDelay
+        if self.running:
+            self.startProcess(name)
+
+    def removeProcess(self, name):
+        """
+        Stop the named process and remove it from the list of monitored
+        processes.
+
+        @type name: C{str}
+        @param name: A string that uniquely identifies the process.
+        """
+        self.stopProcess(name)
+        del self._processes[name]
+
+    def startService(self):
+        """
+        Start all monitored processes.
+        """
+        service.Service.startService(self)
+        for name in list(self._processes):
+            self.startProcess(name)
+
+    def stopService(self):
+        """
+        Stop all monitored processes and cancel all scheduled process restarts.
+        """
+        service.Service.stopService(self)
+
+        # Cancel any outstanding restarts
+        for name, delayedCall in list(self.restart.items()):
+            if delayedCall.active():
+                delayedCall.cancel()
+
+        for name in list(self._processes):
+            self.stopProcess(name)
+
+    def connectionLost(self, name):
+        """
+        Called when a monitored processes exits. If
+        L{service.IService.running} is L{True} (ie the service is started), the
+        process will be restarted.
+        If the process had been running for more than
+        L{ProcessMonitor.threshold} seconds it will be restarted immediately.
+        If the process had been running for less than
+        L{ProcessMonitor.threshold} seconds, the restart will be delayed and
+        each time the process dies before the configured threshold, the restart
+        delay will be doubled - up to a maximum delay of maxRestartDelay sec.
+
+        @type name: C{str}
+        @param name: A string that uniquely identifies the process
+            which exited.
+        """
+        # Cancel the scheduled _forceStopProcess function if the process
+        # dies naturally
+        if name in self.murder:
+            if self.murder[name].active():
+                self.murder[name].cancel()
+            del self.murder[name]
+
+        del self.protocols[name]
+
+        if self._reactor.seconds() - self.timeStarted[name] < self.threshold:
+            # The process died too fast - backoff
+            nextDelay = self.delay[name]
+            self.delay[name] = min(self.delay[name] * 2, self.maxRestartDelay)
+
+        else:
+            # Process had been running for a significant amount of time
+            # restart immediately
+            nextDelay = 0
+            self.delay[name] = self.minRestartDelay
+
+        # Schedule a process restart if the service is running
+        if self.running and name in self._processes:
+            self.restart[name] = self._reactor.callLater(
+                nextDelay, self.startProcess, name
+            )
+
+    def startProcess(self, name):
+        """
+        @param name: The name of the process to be started
+        """
+        # If a protocol instance already exists, it means the process is
+        # already running
+        if name in self.protocols:
+            return
+
+        process = self._processes[name]
+
+        proto = LoggingProtocol()
+        proto.service = self
+        proto.name = name
+        self.protocols[name] = proto
+        self.timeStarted[name] = self._reactor.seconds()
+        self._reactor.spawnProcess(
+            proto,
+            process.args[0],
+            process.args,
+            uid=process.uid,
+            gid=process.gid,
+            env=process.env,
+            path=process.cwd,
+        )
+
+    def _forceStopProcess(self, proc):
+        """
+        @param proc: An L{IProcessTransport} provider
+        """
+        try:
+            proc.signalProcess("KILL")
+        except error.ProcessExitedAlready:
+            pass
+
+    def stopProcess(self, name):
+        """
+        @param name: The name of the process to be stopped
+        """
+        if name not in self._processes:
+            raise KeyError(f"Unrecognized process name: {name}")
+
+        proto = self.protocols.get(name, None)
+        if proto is not None:
+            proc = proto.transport
+            try:
+                proc.signalProcess("TERM")
+            except error.ProcessExitedAlready:
+                pass
+            else:
+                self.murder[name] = self._reactor.callLater(
+                    self.killTime, self._forceStopProcess, proc
+                )
+
+    def restartAll(self):
+        """
+        Restart all processes. This is useful for third party management
+        services to allow a user to restart servers because of an outside change
+        in circumstances -- for example, a new version of a library is
+        installed.
+        """
+        for name in self._processes:
+            self.stopProcess(name)
+
+    def __repr__(self) -> str:
+        lst = []
+        for name, proc in self._processes.items():
+            uidgid = ""
+            if proc.uid is not None:
+                uidgid = str(proc.uid)
+            if proc.gid is not None:
+                uidgid += ":" + str(proc.gid)
+
+            if uidgid:
+                uidgid = "(" + uidgid + ")"
+            lst.append(f"{name!r}{uidgid}: {proc.args!r}")
+        return "<" + self.__class__.__name__ + " " + " ".join(lst) + ">"
diff --git a/contrib/python/Twisted/py3/twisted/runner/procmontap.py b/contrib/python/Twisted/py3/twisted/runner/procmontap.py
new file mode 100644
index 00000000000..03205783b31
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/runner/procmontap.py
@@ -0,0 +1,96 @@
+# -*- test-case-name: twisted.runner.test.test_procmontap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support for creating a service which runs a process monitor.
+"""
+
+from typing import List, Sequence
+
+from twisted.python import usage
+from twisted.runner.procmon import ProcessMonitor
+
+
+class Options(usage.Options):
+    """
+    Define the options accepted by the I{twistd procmon} plugin.
+    """
+
+    synopsis = "[procmon options] commandline"
+
+    optParameters = [
+        [
+            "threshold",
+            "t",
+            1,
+            "How long a process has to live "
+            "before the death is considered instant, in seconds.",
+            float,
+        ],
+        [
+            "killtime",
+            "k",
+            5,
+            "How long a process being killed "
+            "has to get its affairs in order before it gets killed "
+            "with an unmaskable signal.",
+            float,
+        ],
+        [
+            "minrestartdelay",
+            "m",
+            1,
+            "The minimum time (in "
+            "seconds) to wait before attempting to restart a "
+            "process",
+            float,
+        ],
+        [
+            "maxrestartdelay",
+            "M",
+            3600,
+            "The maximum time (in "
+            "seconds) to wait before attempting to restart a "
+            "process",
+            float,
+        ],
+    ]
+
+    optFlags: List[Sequence[str]] = []
+
+    longdesc = """\
+procmon runs processes, monitors their progress, and restarts them when they
+die.
+
+procmon will not attempt to restart a process that appears to die instantly;
+with each "instant" death (less than 1 second, by default), it will delay
+approximately twice as long before restarting it. A successful run will reset
+the counter.
+
+Eg twistd procmon sleep 10"""
+
+    def parseArgs(self, *args: str) -> None:
+        """
+        Grab the command line that is going to be started and monitored
+        """
+        self["args"] = args
+
+    def postOptions(self) -> None:
+        """
+        Check for dependencies.
+        """
+        if len(self["args"]) < 1:
+            raise usage.UsageError("Please specify a process commandline")
+
+
+def makeService(config: Options) -> ProcessMonitor:
+    s = ProcessMonitor()
+
+    s.threshold = config["threshold"]
+    s.killTime = config["killtime"]
+    s.minRestartDelay = config["minrestartdelay"]
+    s.maxRestartDelay = config["maxrestartdelay"]
+
+    s.addProcess(" ".join(config["args"]), config["args"])
+    return s
diff --git a/contrib/python/Twisted/py3/twisted/scripts/__init__.py b/contrib/python/Twisted/py3/twisted/scripts/__init__.py
new file mode 100644
index 00000000000..73d90a8eaf2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/scripts/__init__.py
@@ -0,0 +1,9 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Subpackage containing the modules that implement the command line tools.
+
+Note that these are imported by top-level scripts which are intended to be
+invoked directly from a shell.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/scripts/_twistd_unix.py b/contrib/python/Twisted/py3/twisted/scripts/_twistd_unix.py
new file mode 100644
index 00000000000..295a32fa01e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/scripts/_twistd_unix.py
@@ -0,0 +1,455 @@
+# -*- test-case-name: twisted.test.test_twistd -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+import errno
+import os
+import pwd
+import sys
+import traceback
+
+from twisted import copyright, logger
+from twisted.application import app, service
+from twisted.internet.interfaces import IReactorDaemonize
+from twisted.python import log, logfile, usage
+from twisted.python.runtime import platformType
+from twisted.python.util import gidFromString, switchUID, uidFromString, untilConcludes
+
+if platformType == "win32":
+    raise ImportError("_twistd_unix doesn't work on Windows.")
+
+
+def _umask(value):
+    return int(value, 8)
+
+
+class ServerOptions(app.ServerOptions):
+    synopsis = "Usage: twistd [options]"
+
+    optFlags = [
+        ["nodaemon", "n", "don't daemonize, don't use default umask of 0077"],
+        ["originalname", None, "Don't try to change the process name"],
+        ["syslog", None, "Log to syslog, not to file"],
+        [
+            "euid",
+            "",
+            "Set only effective user-id rather than real user-id. "
+            "(This option has no effect unless the server is running as "
+            "root, in which case it means not to shed all privileges "
+            "after binding ports, retaining the option to regain "
+            "privileges in cases such as spawning processes. "
+            "Use with caution.)",
+        ],
+    ]
+
+    optParameters = [
+        ["prefix", None, "twisted", "use the given prefix when syslogging"],
+        ["pidfile", "", "twistd.pid", "Name of the pidfile"],
+        ["chroot", None, None, "Chroot to a supplied directory before running"],
+        ["uid", "u", None, "The uid to run as.", uidFromString],
+        [
+            "gid",
+            "g",
+            None,
+            "The gid to run as.  If not specified, the default gid "
+            "associated with the specified --uid is used.",
+            gidFromString,
+        ],
+        ["umask", None, None, "The (octal) file creation mask to apply.", _umask],
+    ]
+
+    compData = usage.Completions(
+        optActions={
+            "pidfile": usage.CompleteFiles("*.pid"),
+            "chroot": usage.CompleteDirs(descr="chroot directory"),
+            "gid": usage.CompleteGroups(descr="gid to run as"),
+            "uid": usage.CompleteUsernames(descr="uid to run as"),
+            "prefix": usage.Completer(descr="syslog prefix"),
+        },
+    )
+
+    def opt_version(self):
+        """
+        Print version information and exit.
+        """
+        print(f"twistd (the Twisted daemon) {copyright.version}", file=self.stdout)
+        print(copyright.copyright, file=self.stdout)
+        sys.exit()
+
+    def postOptions(self):
+        app.ServerOptions.postOptions(self)
+        if self["pidfile"]:
+            self["pidfile"] = os.path.abspath(self["pidfile"])
+
+
+def checkPID(pidfile):
+    if not pidfile:
+        return
+    if os.path.exists(pidfile):
+        try:
+            with open(pidfile) as f:
+                pid = int(f.read())
+        except ValueError:
+            sys.exit(f"Pidfile {pidfile} contains non-numeric value")
+        try:
+            os.kill(pid, 0)
+        except OSError as why:
+            if why.errno == errno.ESRCH:
+                # The pid doesn't exist.
+                log.msg(f"Removing stale pidfile {pidfile}", isError=True)
+                os.remove(pidfile)
+            else:
+                sys.exit(
+                    "Can't check status of PID {} from pidfile {}: {}".format(
+                        pid, pidfile, why
+                    )
+                )
+        else:
+            sys.exit(
+                """\
+Another twistd server is running, PID {}\n
+This could either be a previously started instance of your application or a
+different application entirely. To start a new one, either run it in some other
+directory, or use the --pidfile and --logfile parameters to avoid clashes.
+""".format(
+                    pid
+                )
+            )
+
+
+class UnixAppLogger(app.AppLogger):
+    """
+    A logger able to log to syslog, to files, and to stdout.
+
+    @ivar _syslog: A flag indicating whether to use syslog instead of file
+        logging.
+    @type _syslog: C{bool}
+
+    @ivar _syslogPrefix: If C{sysLog} is C{True}, the string prefix to use for
+        syslog messages.
+    @type _syslogPrefix: C{str}
+
+    @ivar _nodaemon: A flag indicating the process will not be daemonizing.
+    @type _nodaemon: C{bool}
+    """
+
+    def __init__(self, options):
+        app.AppLogger.__init__(self, options)
+        self._syslog = options.get("syslog", False)
+        self._syslogPrefix = options.get("prefix", "")
+        self._nodaemon = options.get("nodaemon", False)
+
+    def _getLogObserver(self):
+        """
+        Create and return a suitable log observer for the given configuration.
+
+        The observer will go to syslog using the prefix C{_syslogPrefix} if
+        C{_syslog} is true.  Otherwise, it will go to the file named
+        C{_logfilename} or, if C{_nodaemon} is true and C{_logfilename} is
+        C{"-"}, to stdout.
+
+        @return: An object suitable to be passed to C{log.addObserver}.
+        """
+        if self._syslog:
+            from twisted.python import syslog
+
+            return syslog.SyslogObserver(self._syslogPrefix).emit
+
+        if self._logfilename == "-":
+            if not self._nodaemon:
+                sys.exit("Daemons cannot log to stdout, exiting!")
+            logFile = sys.stdout
+        elif self._nodaemon and not self._logfilename:
+            logFile = sys.stdout
+        else:
+            if not self._logfilename:
+                self._logfilename = "twistd.log"
+            logFile = logfile.LogFile.fromFullPath(self._logfilename)
+            try:
+                import signal
+            except ImportError:
+                pass
+            else:
+                # Override if signal is set to None or SIG_DFL (0)
+                if not signal.getsignal(signal.SIGUSR1):
+
+                    def rotateLog(signal, frame):
+                        from twisted.internet import reactor
+
+                        reactor.callFromThread(logFile.rotate)
+
+                    signal.signal(signal.SIGUSR1, rotateLog)
+        return logger.textFileLogObserver(logFile)
+
+
+def launchWithName(name):
+    if name and name != sys.argv[0]:
+        exe = os.path.realpath(sys.executable)
+        log.msg("Changing process name to " + name)
+        os.execv(exe, [name, sys.argv[0], "--originalname"] + sys.argv[1:])
+
+
+class UnixApplicationRunner(app.ApplicationRunner):
+    """
+    An ApplicationRunner which does Unix-specific things, like fork,
+    shed privileges, and maintain a PID file.
+    """
+
+    loggerFactory = UnixAppLogger
+
+    def preApplication(self):
+        """
+        Do pre-application-creation setup.
+        """
+        checkPID(self.config["pidfile"])
+        self.config["nodaemon"] = self.config["nodaemon"] or self.config["debug"]
+        self.oldstdout = sys.stdout
+        self.oldstderr = sys.stderr
+
+    def _formatChildException(self, exception):
+        """
+        Format the C{exception} in preparation for writing to the
+        status pipe.  This does the right thing on Python 2 if the
+        exception's message is Unicode, and in all cases limits the
+        length of the message afte* encoding to 100 bytes.
+
+        This means the returned message may be truncated in the middle
+        of a unicode escape.
+
+        @type exception: L{Exception}
+        @param exception: The exception to format.
+
+        @return: The formatted message, suitable for writing to the
+            status pipe.
+        @rtype: L{bytes}
+        """
+        # On Python 2 this will encode Unicode messages with the ascii
+        # codec and the backslashreplace error handler.
+        exceptionLine = traceback.format_exception_only(exception.__class__, exception)[
+            -1
+        ]
+        # remove the trailing newline
+        formattedMessage = f"1 {exceptionLine.strip()}"
+        # On Python 3, encode the message the same way Python 2's
+        # format_exception_only does
+        formattedMessage = formattedMessage.encode("ascii", "backslashreplace")
+        # By this point, the message has been encoded, if appropriate,
+        # with backslashreplace on both Python 2 and Python 3.
+        # Truncating the encoded message won't make it completely
+        # unreadable, and the reader should print out the repr of the
+        # message it receives anyway.  What it will do, however, is
+        # ensure that only 100 bytes are written to the status pipe,
+        # ensuring that the child doesn't block because the pipe's
+        # full.  This assumes PIPE_BUF > 100!
+        return formattedMessage[:100]
+
+    def postApplication(self):
+        """
+        To be called after the application is created: start the application
+        and run the reactor. After the reactor stops, clean up PID files and
+        such.
+        """
+        try:
+            self.startApplication(self.application)
+        except Exception as ex:
+            statusPipe = self.config.get("statusPipe", None)
+            if statusPipe is not None:
+                message = self._formatChildException(ex)
+                untilConcludes(os.write, statusPipe, message)
+                untilConcludes(os.close, statusPipe)
+            self.removePID(self.config["pidfile"])
+            raise
+        else:
+            statusPipe = self.config.get("statusPipe", None)
+            if statusPipe is not None:
+                untilConcludes(os.write, statusPipe, b"0")
+                untilConcludes(os.close, statusPipe)
+        self.startReactor(None, self.oldstdout, self.oldstderr)
+        self.removePID(self.config["pidfile"])
+
+    def removePID(self, pidfile):
+        """
+        Remove the specified PID file, if possible.  Errors are logged, not
+        raised.
+
+        @type pidfile: C{str}
+        @param pidfile: The path to the PID tracking file.
+        """
+        if not pidfile:
+            return
+        try:
+            os.unlink(pidfile)
+        except OSError as e:
+            if e.errno == errno.EACCES or e.errno == errno.EPERM:
+                log.msg("Warning: No permission to delete pid file")
+            else:
+                log.err(e, "Failed to unlink PID file:")
+        except BaseException:
+            log.err(None, "Failed to unlink PID file:")
+
+    def setupEnvironment(self, chroot, rundir, nodaemon, umask, pidfile):
+        """
+        Set the filesystem root, the working directory, and daemonize.
+
+        @type chroot: C{str} or L{None}
+        @param chroot: If not None, a path to use as the filesystem root (using
+            L{os.chroot}).
+
+        @type rundir: C{str}
+        @param rundir: The path to set as the working directory.
+
+        @type nodaemon: C{bool}
+        @param nodaemon: A flag which, if set, indicates that daemonization
+            should not be done.
+
+        @type umask: C{int} or L{None}
+        @param umask: The value to which to change the process umask.
+
+        @type pidfile: C{str} or L{None}
+        @param pidfile: If not L{None}, the path to a file into which to put
+            the PID of this process.
+        """
+        daemon = not nodaemon
+
+        if chroot is not None:
+            os.chroot(chroot)
+            if rundir == ".":
+                rundir = "/"
+        os.chdir(rundir)
+        if daemon and umask is None:
+            umask = 0o077
+        if umask is not None:
+            os.umask(umask)
+        if daemon:
+            from twisted.internet import reactor
+
+            self.config["statusPipe"] = self.daemonize(reactor)
+        if pidfile:
+            with open(pidfile, "wb") as f:
+                f.write(b"%d" % (os.getpid(),))
+
+    def daemonize(self, reactor):
+        """
+        Daemonizes the application on Unix. This is done by the usual double
+        forking approach.
+
+        @see: U{http://code.activestate.com/recipes/278731/}
+        @see: W. Richard Stevens,
+            "Advanced Programming in the Unix Environment",
+            1992, Addison-Wesley, ISBN 0-201-56317-7
+
+        @param reactor: The reactor in use.  If it provides
+            L{IReactorDaemonize}, its daemonization-related callbacks will be
+            invoked.
+
+        @return: A writable pipe to be used to report errors.
+        @rtype: C{int}
+        """
+        # If the reactor requires hooks to be called for daemonization, call
+        # them. Currently the only reactor which provides/needs that is
+        # KQueueReactor.
+        if IReactorDaemonize.providedBy(reactor):
+            reactor.beforeDaemonize()
+        r, w = os.pipe()
+        if os.fork():  # launch child and...
+            code = self._waitForStart(r)
+            os.close(r)
+            os._exit(code)  # kill off parent
+        os.setsid()
+        if os.fork():  # launch child and...
+            os._exit(0)  # kill off parent again.
+        null = os.open("/dev/null", os.O_RDWR)
+        for i in range(3):
+            try:
+                os.dup2(null, i)
+            except OSError as e:
+                if e.errno != errno.EBADF:
+                    raise
+        os.close(null)
+
+        if IReactorDaemonize.providedBy(reactor):
+            reactor.afterDaemonize()
+
+        return w
+
+    def _waitForStart(self, readPipe: int) -> int:
+        """
+        Wait for the daemonization success.
+
+        @param readPipe: file descriptor to read start information from.
+        @type readPipe: C{int}
+
+        @return: code to be passed to C{os._exit}: 0 for success, 1 for error.
+        @rtype: C{int}
+        """
+        data = untilConcludes(os.read, readPipe, 100)
+        dataRepr = repr(data[2:])
+        if data != b"0":
+            msg = (
+                "An error has occurred: {}\nPlease look at log "
+                "file for more information.\n".format(dataRepr)
+            )
+            untilConcludes(sys.__stderr__.write, msg)
+            return 1
+        return 0
+
+    def shedPrivileges(self, euid, uid, gid):
+        """
+        Change the UID and GID or the EUID and EGID of this process.
+
+        @type euid: C{bool}
+        @param euid: A flag which, if set, indicates that only the I{effective}
+            UID and GID should be set.
+
+        @type uid: C{int} or L{None}
+        @param uid: If not L{None}, the UID to which to switch.
+
+        @type gid: C{int} or L{None}
+        @param gid: If not L{None}, the GID to which to switch.
+        """
+        if uid is not None or gid is not None:
+            extra = euid and "e" or ""
+            desc = f"{extra}uid/{extra}gid {uid}/{gid}"
+            try:
+                switchUID(uid, gid, euid)
+            except OSError as e:
+                log.msg(
+                    "failed to set {}: {} (are you root?) -- "
+                    "exiting.".format(desc, e)
+                )
+                sys.exit(1)
+            else:
+                log.msg(f"set {desc}")
+
+    def startApplication(self, application):
+        """
+        Configure global process state based on the given application and run
+        the application.
+
+        @param application: An object which can be adapted to
+            L{service.IProcess} and L{service.IService}.
+        """
+        process = service.IProcess(application)
+        if not self.config["originalname"]:
+            launchWithName(process.processName)
+        self.setupEnvironment(
+            self.config["chroot"],
+            self.config["rundir"],
+            self.config["nodaemon"],
+            self.config["umask"],
+            self.config["pidfile"],
+        )
+
+        service.IService(application).privilegedStartService()
+
+        uid, gid = self.config["uid"], self.config["gid"]
+        if uid is None:
+            uid = process.uid
+        if gid is None:
+            gid = process.gid
+        if uid is not None and gid is None:
+            gid = pwd.getpwuid(uid).pw_gid
+
+        self.shedPrivileges(self.config["euid"], uid, gid)
+        app.startApplication(application, not self.config["no_save"])
diff --git a/contrib/python/Twisted/py3/twisted/scripts/_twistw.py b/contrib/python/Twisted/py3/twisted/scripts/_twistw.py
new file mode 100644
index 00000000000..494fb5c186e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/scripts/_twistw.py
@@ -0,0 +1,55 @@
+# -*- test-case-name: twisted.test.test_twistd -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+import os
+import sys
+
+from twisted import copyright
+from twisted.application import app, internet, service
+from twisted.python import log
+
+
+class ServerOptions(app.ServerOptions):
+    synopsis = "Usage: twistd [options]"
+
+    optFlags = [
+        ["nodaemon", "n", "(for backwards compatibility)."],
+    ]
+
+    def opt_version(self):
+        """
+        Print version information and exit.
+        """
+        print(
+            f"twistd (the Twisted Windows runner) {copyright.version}",
+            file=self.stdout,
+        )
+        print(copyright.copyright, file=self.stdout)
+        sys.exit()
+
+
+class WindowsApplicationRunner(app.ApplicationRunner):
+    """
+    An ApplicationRunner which avoids unix-specific things. No
+    forking, no PID files, no privileges.
+    """
+
+    def preApplication(self):
+        """
+        Do pre-application-creation setup.
+        """
+        self.oldstdout = sys.stdout
+        self.oldstderr = sys.stderr
+        os.chdir(self.config["rundir"])
+
+    def postApplication(self):
+        """
+        Start the application and run the reactor.
+        """
+        service.IService(self.application).privilegedStartService()
+        app.startApplication(self.application, not self.config["no_save"])
+        app.startApplication(internet.TimerService(0.1, lambda: None), 0)
+        self.startReactor(None, self.oldstdout, self.oldstderr)
+        log.msg("Server Shut Down.")
diff --git a/contrib/python/Twisted/py3/twisted/scripts/htmlizer.py b/contrib/python/Twisted/py3/twisted/scripts/htmlizer.py
new file mode 100644
index 00000000000..9c588ae11fc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/scripts/htmlizer.py
@@ -0,0 +1,74 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+"""
+HTML pretty-printing for Python source code.
+"""
+
+
+__version__ = "$Revision: 1.8 $"[11:-2]
+
+import os
+import sys
+
+from twisted import copyright
+from twisted.python import htmlizer, usage
+
+header = """<html><head>
+<title>%(title)s</title>
+<meta name=\"Generator\" content="%(generator)s" />
+%(alternate)s
+%(stylesheet)s
+</head>
+<body>
+"""
+footer = """</body>"""
+
+styleLink = '<link rel="stylesheet" href="%s" type="text/css" />'
+alternateLink = '<link rel="alternate" href="%(source)s" type="text/x-python" />'
+
+
+class Options(usage.Options):
+    synopsis = """{} [options] source.py
+    """.format(
+        os.path.basename(sys.argv[0]),
+    )
+
+    optParameters = [
+        ("stylesheet", "s", None, "URL of stylesheet to link to."),
+    ]
+
+    compData = usage.Completions(
+        extraActions=[usage.CompleteFiles("*.py", descr="source python file")]
+    )
+
+    def parseArgs(self, filename):
+        self["filename"] = filename
+
+
+def run():
+    options = Options()
+    try:
+        options.parseOptions()
+    except usage.UsageError as e:
+        print(str(e))
+        sys.exit(1)
+    filename = options["filename"]
+    if options.get("stylesheet") is not None:
+        stylesheet = styleLink % (options["stylesheet"],)
+    else:
+        stylesheet = ""
+
+    with open(filename + ".html", "wb") as output:
+        outHeader = header % {
+            "title": filename,
+            "generator": f"htmlizer/{copyright.longversion}",
+            "alternate": alternateLink % {"source": filename},
+            "stylesheet": stylesheet,
+        }
+        output.write(outHeader.encode("utf-8"))
+        with open(filename, "rb") as f:
+            htmlizer.filter(f, output, htmlizer.SmallerHTMLWriter)
+        output.write(footer.encode("utf-8"))
diff --git a/contrib/python/Twisted/py3/twisted/scripts/newsfragments/761.bugfix b/contrib/python/Twisted/py3/twisted/scripts/newsfragments/761.bugfix
new file mode 100644
index 00000000000..ba0436bbc38
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/scripts/newsfragments/761.bugfix
@@ -0,0 +1 @@
+If twist or twistd exit with a signal it now delivers that signal to itself instead of exiting normally.  On Unix platforms this results in a nonzero exit code where previously a zero exit code was returned.
diff --git a/contrib/python/Twisted/py3/twisted/scripts/trial.py b/contrib/python/Twisted/py3/twisted/scripts/trial.py
new file mode 100644
index 00000000000..531fe46ce17
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/scripts/trial.py
@@ -0,0 +1,654 @@
+# -*- test-case-name: twisted.trial.test.test_script -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+import gc
+import inspect
+import os
+import pdb
+import random
+import sys
+import time
+import trace
+import warnings
+from typing import NoReturn, Optional, Type
+
+from twisted import plugin
+from twisted.application import app
+from twisted.internet import defer
+from twisted.python import failure, reflect, usage
+from twisted.python.filepath import FilePath
+from twisted.python.reflect import namedModule
+from twisted.trial import itrial, runner
+from twisted.trial._dist.disttrial import DistTrialRunner
+from twisted.trial.unittest import TestSuite
+
+# Yea, this is stupid.  Leave it for command-line compatibility for a
+# while, though.
+TBFORMAT_MAP = {
+    "plain": "default",
+    "default": "default",
+    "emacs": "brief",
+    "brief": "brief",
+    "cgitb": "verbose",
+    "verbose": "verbose",
+}
+
+
+def _parseLocalVariables(line):
+    """
+    Accepts a single line in Emacs local variable declaration format and
+    returns a dict of all the variables {name: value}.
+    Raises ValueError if 'line' is in the wrong format.
+
+    See http://www.gnu.org/software/emacs/manual/html_node/File-Variables.html
+    """
+    paren = "-*-"
+    start = line.find(paren) + len(paren)
+    end = line.rfind(paren)
+    if start == -1 or end == -1:
+        raise ValueError(f"{line!r} not a valid local variable declaration")
+    items = line[start:end].split(";")
+    localVars = {}
+    for item in items:
+        if len(item.strip()) == 0:
+            continue
+        split = item.split(":")
+        if len(split) != 2:
+            raise ValueError(f"{line!r} contains invalid declaration {item!r}")
+        localVars[split[0].strip()] = split[1].strip()
+    return localVars
+
+
+def loadLocalVariables(filename):
+    """
+    Accepts a filename and attempts to load the Emacs variable declarations
+    from that file, simulating what Emacs does.
+
+    See http://www.gnu.org/software/emacs/manual/html_node/File-Variables.html
+    """
+    with open(filename) as f:
+        lines = [f.readline(), f.readline()]
+    for line in lines:
+        try:
+            return _parseLocalVariables(line)
+        except ValueError:
+            pass
+    return {}
+
+
+def getTestModules(filename):
+    testCaseVar = loadLocalVariables(filename).get("test-case-name", None)
+    if testCaseVar is None:
+        return []
+    return testCaseVar.split(",")
+
+
+def isTestFile(filename):
+    """
+    Returns true if 'filename' looks like a file containing unit tests.
+    False otherwise.  Doesn't care whether filename exists.
+    """
+    basename = os.path.basename(filename)
+    return basename.startswith("test_") and os.path.splitext(basename)[1] == (".py")
+
+
+def _reporterAction():
+    return usage.CompleteList([p.longOpt for p in plugin.getPlugins(itrial.IReporter)])
+
+
+def _maybeFindSourceLine(testThing):
+    """
+    Try to find the source line of the given test thing.
+
+    @param testThing: the test item to attempt to inspect
+    @type testThing: an L{TestCase}, test method, or module, though only the
+        former two have a chance to succeed
+    @rtype: int
+    @return: the starting source line, or -1 if one couldn't be found
+    """
+
+    # an instance of L{TestCase} -- locate the test it will run
+    method = getattr(testThing, "_testMethodName", None)
+    if method is not None:
+        testThing = getattr(testThing, method)
+
+    # If it's a function, we can get the line number even if the source file no
+    # longer exists
+    code = getattr(testThing, "__code__", None)
+    if code is not None:
+        return code.co_firstlineno
+
+    try:
+        return inspect.getsourcelines(testThing)[1]
+    except (OSError, TypeError):
+        # either testThing is a module, which raised a TypeError, or the file
+        # couldn't be read
+        return -1
+
+
+# orders which can be passed to trial --order
+_runOrders = {
+    "alphabetical": (
+        "alphabetical order for test methods, arbitrary order for test cases",
+        runner.name,
+    ),
+    "toptobottom": (
+        "attempt to run test cases and methods in the order they were defined",
+        _maybeFindSourceLine,
+    ),
+}
+
+
+def _checkKnownRunOrder(order):
+    """
+    Check that the given order is a known test running order.
+
+    Does nothing else, since looking up the appropriate callable to sort the
+    tests should be done when it actually will be used, as the default argument
+    will not be coerced by this function.
+
+    @param order: one of the known orders in C{_runOrders}
+    @return: the order unmodified
+    """
+    if order not in _runOrders:
+        raise usage.UsageError(
+            "--order must be one of: %s. See --help-orders for details"
+            % (", ".join(repr(order) for order in _runOrders),)
+        )
+    return order
+
+
+class _BasicOptions:
+    """
+    Basic options shared between trial and its local workers.
+    """
+
+    longdesc = (
+        "trial loads and executes a suite of unit tests, obtained "
+        "from modules, packages and files listed on the command line."
+    )
+
+    optFlags = [
+        ["help", "h"],
+        ["no-recurse", "N", "Don't recurse into packages"],
+        ["help-orders", None, "Help on available test running orders"],
+        ["help-reporters", None, "Help on available output plugins (reporters)"],
+        [
+            "rterrors",
+            "e",
+            "realtime errors, print out tracebacks as " "soon as they occur",
+        ],
+        ["unclean-warnings", None, "Turn dirty reactor errors into warnings"],
+        [
+            "force-gc",
+            None,
+            "Have Trial run gc.collect() before and " "after each test case.",
+        ],
+        [
+            "exitfirst",
+            "x",
+            "Exit after the first non-successful result (cannot be "
+            "specified along with --jobs).",
+        ],
+    ]
+
+    optParameters = [
+        [
+            "order",
+            "o",
+            None,
+            "Specify what order to run test cases and methods. "
+            "See --help-orders for more info.",
+            _checkKnownRunOrder,
+        ],
+        ["random", "z", None, "Run tests in random order using the specified seed"],
+        [
+            "temp-directory",
+            None,
+            "_trial_temp",
+            "Path to use as working directory for tests.",
+        ],
+        [
+            "reporter",
+            None,
+            "verbose",
+            "The reporter to use for this test run.  See --help-reporters for "
+            "more info.",
+        ],
+    ]
+
+    compData = usage.Completions(
+        optActions={
+            "order": usage.CompleteList(_runOrders),
+            "reporter": _reporterAction,
+            "logfile": usage.CompleteFiles(descr="log file name"),
+            "random": usage.Completer(descr="random seed"),
+        },
+        extraActions=[
+            usage.CompleteFiles(
+                "*.py",
+                descr="file | module | package | TestCase | testMethod",
+                repeat=True,
+            )
+        ],
+    )
+
+    tracer: Optional[trace.Trace] = None
+
+    def __init__(self):
+        self["tests"] = []
+        usage.Options.__init__(self)
+
+    def getSynopsis(self):
+        executableName = reflect.filenameToModuleName(sys.argv[0])
+
+        if executableName.endswith(".__main__"):
+            executableName = "{} -m {}".format(
+                os.path.basename(sys.executable),
+                executableName.replace(".__main__", ""),
+            )
+
+        return """{} [options] [[file|package|module|TestCase|testmethod]...]
+        """.format(
+            executableName,
+        )
+
+    def coverdir(self):
+        """
+        Return a L{FilePath} representing the directory into which coverage
+        results should be written.
+        """
+        coverdir = "coverage"
+        result = FilePath(self["temp-directory"]).child(coverdir)
+        print(f"Setting coverage directory to {result.path}.")
+        return result
+
+    # TODO: Some of the opt_* methods on this class have docstrings and some do
+    #       not. This is mostly because usage.Options's currently will replace
+    #       any intended output in optFlags and optParameters with the
+    #       docstring. See #6427. When that is fixed, all methods should be
+    #       given docstrings (and it should be verified that those with
+    #       docstrings already have content suitable for printing as usage
+    #       information).
+
+    def opt_coverage(self):
+        """
+        Generate coverage information in the coverage file in the
+        directory specified by the temp-directory option.
+        """
+        self.tracer = trace.Trace(count=1, trace=0)
+        sys.settrace(self.tracer.globaltrace)
+        self["coverage"] = True
+
+    def opt_testmodule(self, filename):
+        """
+        Filename to grep for test cases (-*- test-case-name).
+        """
+        # If the filename passed to this parameter looks like a test module
+        # we just add that to the test suite.
+        #
+        # If not, we inspect it for an Emacs buffer local variable called
+        # 'test-case-name'.  If that variable is declared, we try to add its
+        # value to the test suite as a module.
+        #
+        # This parameter allows automated processes (like Buildbot) to pass
+        # a list of files to Trial with the general expectation of "these files,
+        # whatever they are, will get tested"
+        if not os.path.isfile(filename):
+            sys.stderr.write(f"File {filename!r} doesn't exist\n")
+            return
+        filename = os.path.abspath(filename)
+        if isTestFile(filename):
+            self["tests"].append(filename)
+        else:
+            self["tests"].extend(getTestModules(filename))
+
+    def opt_spew(self):
+        """
+        Print an insanely verbose log of everything that happens.  Useful
+        when debugging freezes or locks in complex code.
+        """
+        from twisted.python.util import spewer
+
+        sys.settrace(spewer)
+
+    def opt_help_orders(self):
+        synopsis = (
+            "Trial can attempt to run test cases and their methods in "
+            "a few different orders. You can select any of the "
+            "following options using --order=<foo>.\n"
+        )
+
+        print(synopsis)
+        for name, (description, _) in sorted(_runOrders.items()):
+            print("   ", name, "\t", description)
+        sys.exit(0)
+
+    def opt_help_reporters(self):
+        synopsis = (
+            "Trial's output can be customized using plugins called "
+            "Reporters. You can\nselect any of the following "
+            "reporters using --reporter=<foo>\n"
+        )
+        print(synopsis)
+        for p in plugin.getPlugins(itrial.IReporter):
+            print("   ", p.longOpt, "\t", p.description)
+        sys.exit(0)
+
+    def opt_disablegc(self):
+        """
+        Disable the garbage collector
+        """
+        self["disablegc"] = True
+        gc.disable()
+
+    def opt_tbformat(self, opt):
+        """
+        Specify the format to display tracebacks with. Valid formats are
+        'plain', 'emacs', and 'cgitb' which uses the nicely verbose stdlib
+        cgitb.text function
+        """
+        try:
+            self["tbformat"] = TBFORMAT_MAP[opt]
+        except KeyError:
+            raise usage.UsageError("tbformat must be 'plain', 'emacs', or 'cgitb'.")
+
+    def opt_recursionlimit(self, arg):
+        """
+        see sys.setrecursionlimit()
+        """
+        try:
+            sys.setrecursionlimit(int(arg))
+        except (TypeError, ValueError):
+            raise usage.UsageError("argument to recursionlimit must be an integer")
+        else:
+            self["recursionlimit"] = int(arg)
+
+    def opt_random(self, option):
+        try:
+            self["random"] = int(option)
+        except ValueError:
+            raise usage.UsageError("Argument to --random must be a positive integer")
+        else:
+            if self["random"] < 0:
+                raise usage.UsageError(
+                    "Argument to --random must be a positive integer"
+                )
+            elif self["random"] == 0:
+                self["random"] = int(time.time() * 100)
+
+    def opt_without_module(self, option):
+        """
+        Fake the lack of the specified modules, separated with commas.
+        """
+        self["without-module"] = option
+        for module in option.split(","):
+            if module in sys.modules:
+                warnings.warn(
+                    "Module '%s' already imported, " "disabling anyway." % (module,),
+                    category=RuntimeWarning,
+                )
+            sys.modules[module] = None
+
+    def parseArgs(self, *args):
+        self["tests"].extend(args)
+
+    def _loadReporterByName(self, name):
+        for p in plugin.getPlugins(itrial.IReporter):
+            qual = f"{p.module}.{p.klass}"
+            if p.longOpt == name:
+                return reflect.namedAny(qual)
+        raise usage.UsageError(
+            "Only pass names of Reporter plugins to "
+            "--reporter. See --help-reporters for "
+            "more info."
+        )
+
+    def postOptions(self):
+        # Only load reporters now, as opposed to any earlier, to avoid letting
+        # application-defined plugins muck up reactor selecting by importing
+        # t.i.reactor and causing the default to be installed.
+        self["reporter"] = self._loadReporterByName(self["reporter"])
+        if "tbformat" not in self:
+            self["tbformat"] = "default"
+        if self["order"] is not None and self["random"] is not None:
+            raise usage.UsageError("You can't specify --random when using --order")
+
+
+class Options(_BasicOptions, usage.Options, app.ReactorSelectionMixin):
+    """
+    Options to the trial command line tool.
+
+    @ivar _workerFlags: List of flags which are accepted by trial distributed
+        workers. This is used by C{_getWorkerArguments} to build the command
+        line arguments.
+    @type _workerFlags: C{list}
+
+    @ivar _workerParameters: List of parameter which are accepted by trial
+        distributed workers. This is used by C{_getWorkerArguments} to build
+        the command line arguments.
+    @type _workerParameters: C{list}
+    """
+
+    optFlags = [
+        [
+            "debug",
+            "b",
+            "Run tests in a debugger. If that debugger is "
+            "pdb, will load '.pdbrc' from current directory if it exists.",
+        ],
+        [
+            "debug-stacktraces",
+            "B",
+            "Report Deferred creation and " "callback stack traces",
+        ],
+        [
+            "nopm",
+            None,
+            "don't automatically jump into debugger for " "postmorteming of exceptions",
+        ],
+        ["dry-run", "n", "do everything but run the tests"],
+        ["profile", None, "Run tests under the Python profiler"],
+        ["until-failure", "u", "Repeat test until it fails"],
+    ]
+
+    optParameters = [
+        [
+            "debugger",
+            None,
+            "pdb",
+            "the fully qualified name of a debugger to " "use if --debug is passed",
+        ],
+        ["logfile", "l", "test.log", "log file name"],
+        ["jobs", "j", None, "Number of local workers to run"],
+    ]
+
+    compData = usage.Completions(
+        optActions={
+            "tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]),
+            "reporter": _reporterAction,
+        },
+    )
+
+    _workerFlags = ["disablegc", "force-gc", "coverage"]
+    _workerParameters = ["recursionlimit", "reactor", "without-module"]
+
+    def opt_jobs(self, number):
+        """
+        Number of local workers to run, a strictly positive integer.
+        """
+        try:
+            number = int(number)
+        except ValueError:
+            raise usage.UsageError(
+                "Expecting integer argument to jobs, got '%s'" % number
+            )
+        if number <= 0:
+            raise usage.UsageError(
+                "Argument to jobs must be a strictly positive integer"
+            )
+        self["jobs"] = number
+
+    def _getWorkerArguments(self):
+        """
+        Return a list of options to pass to distributed workers.
+        """
+        args = []
+        for option in self._workerFlags:
+            if self.get(option) is not None:
+                if self[option]:
+                    args.append(f"--{option}")
+        for option in self._workerParameters:
+            if self.get(option) is not None:
+                args.extend([f"--{option}", str(self[option])])
+        return args
+
+    def postOptions(self):
+        _BasicOptions.postOptions(self)
+        if self["jobs"]:
+            conflicts = ["debug", "profile", "debug-stacktraces"]
+            for option in conflicts:
+                if self[option]:
+                    raise usage.UsageError(
+                        "You can't specify --%s when using --jobs" % option
+                    )
+        if self["nopm"]:
+            if not self["debug"]:
+                raise usage.UsageError("You must specify --debug when using " "--nopm ")
+            failure.DO_POST_MORTEM = False
+
+
+def _initialDebugSetup(config: Options) -> None:
+    # do this part of debug setup first for easy debugging of import failures
+    if config["debug"]:
+        failure.startDebugMode()
+    if config["debug"] or config["debug-stacktraces"]:
+        defer.setDebugging(True)
+
+
+def _getSuite(config: Options) -> TestSuite:
+    loader = _getLoader(config)
+    recurse = not config["no-recurse"]
+    return loader.loadByNames(config["tests"], recurse=recurse)
+
+
+def _getLoader(config: Options) -> runner.TestLoader:
+    loader = runner.TestLoader()
+    if config["random"]:
+        randomer = random.Random()
+        randomer.seed(config["random"])
+        loader.sorter = lambda x: randomer.random()
+        print("Running tests shuffled with seed %d\n" % config["random"])
+    elif config["order"]:
+        _, sorter = _runOrders[config["order"]]
+        loader.sorter = sorter
+    if not config["until-failure"]:
+        loader.suiteFactory = runner.DestructiveTestSuite
+    return loader
+
+
+def _wrappedPdb():
+    """
+    Wrap an instance of C{pdb.Pdb} with readline support and load any .rcs.
+
+    """
+
+    dbg = pdb.Pdb()
+    try:
+        namedModule("readline")
+    except ImportError:
+        print("readline module not available")
+    for path in (".pdbrc", "pdbrc"):
+        if os.path.exists(path):
+            try:
+                rcFile = open(path)
+            except OSError:
+                pass
+            else:
+                with rcFile:
+                    dbg.rcLines.extend(rcFile.readlines())
+    return dbg
+
+
+class _DebuggerNotFound(Exception):
+    """
+    A debugger import failed.
+
+    Used to allow translating these errors into usage error messages.
+
+    """
+
+
+def _makeRunner(config: Options) -> runner._Runner:
+    """
+    Return a trial runner class set up with the parameters extracted from
+    C{config}.
+
+    @return: A trial runner instance.
+    """
+    cls: Type[runner._Runner] = runner.TrialRunner
+    args = {
+        "reporterFactory": config["reporter"],
+        "tracebackFormat": config["tbformat"],
+        "realTimeErrors": config["rterrors"],
+        "uncleanWarnings": config["unclean-warnings"],
+        "logfile": config["logfile"],
+        "workingDirectory": config["temp-directory"],
+        "exitFirst": config["exitfirst"],
+    }
+    if config["dry-run"]:
+        args["mode"] = runner.TrialRunner.DRY_RUN
+    elif config["jobs"]:
+        cls = DistTrialRunner
+        args["maxWorkers"] = config["jobs"]
+        args["workerArguments"] = config._getWorkerArguments()
+    else:
+        if config["debug"]:
+            args["mode"] = runner.TrialRunner.DEBUG
+            debugger = config["debugger"]
+
+            if debugger != "pdb":
+                try:
+                    args["debugger"] = reflect.namedAny(debugger)
+                except reflect.ModuleNotFound:
+                    raise _DebuggerNotFound(
+                        f"{debugger!r} debugger could not be found."
+                    )
+            else:
+                args["debugger"] = _wrappedPdb()
+
+        args["profile"] = config["profile"]
+        args["forceGarbageCollection"] = config["force-gc"]
+
+    return cls(**args)
+
+
+def run() -> NoReturn:
+    if len(sys.argv) == 1:
+        sys.argv.append("--help")
+    config = Options()
+    try:
+        config.parseOptions()
+    except usage.error as ue:
+        raise SystemExit(f"{sys.argv[0]}: {ue}")
+    _initialDebugSetup(config)
+
+    try:
+        trialRunner = _makeRunner(config)
+    except _DebuggerNotFound as e:
+        raise SystemExit(f"{sys.argv[0]}: {str(e)}")
+
+    suite = _getSuite(config)
+    if config["until-failure"]:
+        testResult = trialRunner.runUntilFailure(suite)
+    else:
+        testResult = trialRunner.run(suite)
+    if config.tracer:
+        sys.settrace(None)
+        results = config.tracer.results()
+        results.write_results(
+            show_missing=True, summary=False, coverdir=config.coverdir().path
+        )
+    sys.exit(not testResult.wasSuccessful())
diff --git a/contrib/python/Twisted/py3/twisted/scripts/twistd.py b/contrib/python/Twisted/py3/twisted/scripts/twistd.py
new file mode 100644
index 00000000000..40e5fb3e060
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/scripts/twistd.py
@@ -0,0 +1,38 @@
+# -*- test-case-name: twisted.test.test_twistd -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+The Twisted Daemon: platform-independent interface.
+
+@author: Christopher Armstrong
+"""
+
+
+from twisted.application import app
+from twisted.python.runtime import platformType
+
+if platformType == "win32":
+    from twisted.scripts._twistw import (
+        ServerOptions,
+        WindowsApplicationRunner as _SomeApplicationRunner,
+    )
+else:
+    from twisted.scripts._twistd_unix import (  # type: ignore[assignment]
+        ServerOptions,
+        UnixApplicationRunner as _SomeApplicationRunner,
+    )
+
+
+def runApp(config):
+    runner = _SomeApplicationRunner(config)
+    runner.run()
+    if runner._exitSignal is not None:
+        app._exitWithSignal(runner._exitSignal)
+
+
+def run():
+    app.run(runApp, ServerOptions)
+
+
+__all__ = ["run", "runApp"]
diff --git a/contrib/python/Twisted/py3/twisted/spread/__init__.py b/contrib/python/Twisted/py3/twisted/spread/__init__.py
new file mode 100644
index 00000000000..ab3881055d7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/spread/__init__.py
@@ -0,0 +1,8 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Spread: Spreadable (Distributed) Computing.
+
+@author: Glyph Lefkowitz
+"""
diff --git a/contrib/python/Twisted/py3/twisted/spread/banana.py b/contrib/python/Twisted/py3/twisted/spread/banana.py
new file mode 100644
index 00000000000..ee54c2e2a92
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/spread/banana.py
@@ -0,0 +1,403 @@
+# -*- test-case-name: twisted.spread.test.test_banana -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Banana -- s-exp based protocol.
+
+Future Plans: This module is almost entirely stable.  The same caveat applies
+to it as applies to L{twisted.spread.jelly}, however.  Read its future plans
+for more details.
+
+@author: Glyph Lefkowitz
+"""
+
+
+import copy
+import struct
+from io import BytesIO
+
+from twisted.internet import protocol
+from twisted.persisted import styles
+from twisted.python import log
+from twisted.python.compat import iterbytes
+from twisted.python.reflect import fullyQualifiedName
+
+
+class BananaError(Exception):
+    pass
+
+
+def int2b128(integer, stream):
+    if integer == 0:
+        stream(b"\0")
+        return
+    assert integer > 0, "can only encode positive integers"
+    while integer:
+        stream(bytes((integer & 0x7F,)))
+        integer = integer >> 7
+
+
+def b1282int(st):
+    """
+    Convert an integer represented as a base 128 string into an L{int}.
+
+    @param st: The integer encoded in a byte string.
+    @type st: L{bytes}
+
+    @return: The integer value extracted from the byte string.
+    @rtype: L{int}
+    """
+    e = 1
+    i = 0
+    for char in iterbytes(st):
+        n = ord(char)
+        i += n * e
+        e <<= 7
+    return i
+
+
+# delimiter characters.
+LIST = b"\x80"
+INT = b"\x81"
+STRING = b"\x82"
+NEG = b"\x83"
+FLOAT = b"\x84"
+# "optional" -- these might be refused by a low-level implementation.
+LONGINT = b"\x85"
+LONGNEG = b"\x86"
+# really optional; this is part of the 'pb' vocabulary
+VOCAB = b"\x87"
+
+HIGH_BIT_SET = b"\x80"
+
+
+def setPrefixLimit(limit):
+    """
+    Set the limit on the prefix length for all Banana connections
+    established after this call.
+
+    The prefix length limit determines how many bytes of prefix a banana
+    decoder will allow before rejecting a potential object as too large.
+
+    @type limit: L{int}
+    @param limit: The number of bytes of prefix for banana to allow when
+    decoding.
+    """
+    global _PREFIX_LIMIT
+    _PREFIX_LIMIT = limit
+
+
+_PREFIX_LIMIT = None
+setPrefixLimit(64)
+
+SIZE_LIMIT = 640 * 1024  # 640k is all you'll ever need :-)
+
+
+class Banana(protocol.Protocol, styles.Ephemeral):
+    """
+    L{Banana} implements the I{Banana} s-expression protocol, client and
+    server.
+
+    @ivar knownDialects: These are the profiles supported by this Banana
+        implementation.
+    @type knownDialects: L{list} of L{bytes}
+    """
+
+    # The specification calls these profiles but this implementation calls them
+    # dialects instead.
+    knownDialects = [b"pb", b"none"]
+
+    prefixLimit = None
+    sizeLimit = SIZE_LIMIT
+
+    def setPrefixLimit(self, limit):
+        """
+        Set the prefix limit for decoding done by this protocol instance.
+
+        @see: L{setPrefixLimit}
+        """
+        self.prefixLimit = limit
+        self._smallestLongInt = -(2 ** (limit * 7)) + 1
+        self._smallestInt = -(2**31)
+        self._largestInt = 2**31 - 1
+        self._largestLongInt = 2 ** (limit * 7) - 1
+
+    def connectionReady(self):
+        """Surrogate for connectionMade
+        Called after protocol negotiation.
+        """
+
+    def _selectDialect(self, dialect):
+        self.currentDialect = dialect
+        self.connectionReady()
+
+    def callExpressionReceived(self, obj):
+        if self.currentDialect:
+            self.expressionReceived(obj)
+        else:
+            # this is the first message we've received
+            if self.isClient:
+                # if I'm a client I have to respond
+                for serverVer in obj:
+                    if serverVer in self.knownDialects:
+                        self.sendEncoded(serverVer)
+                        self._selectDialect(serverVer)
+                        break
+                else:
+                    # I can't speak any of those dialects.
+                    log.msg(
+                        "The client doesn't speak any of the protocols "
+                        "offered by the server: disconnecting."
+                    )
+                    self.transport.loseConnection()
+            else:
+                if obj in self.knownDialects:
+                    self._selectDialect(obj)
+                else:
+                    # the client just selected a protocol that I did not suggest.
+                    log.msg(
+                        "The client selected a protocol the server didn't "
+                        "suggest and doesn't know: disconnecting."
+                    )
+                    self.transport.loseConnection()
+
+    def connectionMade(self):
+        self.setPrefixLimit(_PREFIX_LIMIT)
+        self.currentDialect = None
+        if not self.isClient:
+            self.sendEncoded(self.knownDialects)
+
+    def gotItem(self, item):
+        l = self.listStack
+        if l:
+            l[-1][1].append(item)
+        else:
+            self.callExpressionReceived(item)
+
+    buffer = b""
+
+    def dataReceived(self, chunk):
+        buffer = self.buffer + chunk
+        listStack = self.listStack
+        gotItem = self.gotItem
+        while buffer:
+            assert self.buffer != buffer, "This ain't right: {} {}".format(
+                repr(self.buffer),
+                repr(buffer),
+            )
+            self.buffer = buffer
+            pos = 0
+            for ch in iterbytes(buffer):
+                if ch >= HIGH_BIT_SET:
+                    break
+                pos = pos + 1
+            else:
+                if pos > self.prefixLimit:
+                    raise BananaError(
+                        "Security precaution: more than %d bytes of prefix"
+                        % (self.prefixLimit,)
+                    )
+                return
+            num = buffer[:pos]
+            typebyte = buffer[pos : pos + 1]
+            rest = buffer[pos + 1 :]
+            if len(num) > self.prefixLimit:
+                raise BananaError(
+                    "Security precaution: longer than %d bytes worth of prefix"
+                    % (self.prefixLimit,)
+                )
+            if typebyte == LIST:
+                num = b1282int(num)
+                if num > SIZE_LIMIT:
+                    raise BananaError("Security precaution: List too long.")
+                listStack.append((num, []))
+                buffer = rest
+            elif typebyte == STRING:
+                num = b1282int(num)
+                if num > SIZE_LIMIT:
+                    raise BananaError("Security precaution: String too long.")
+                if len(rest) >= num:
+                    buffer = rest[num:]
+                    gotItem(rest[:num])
+                else:
+                    return
+            elif typebyte == INT:
+                buffer = rest
+                num = b1282int(num)
+                gotItem(num)
+            elif typebyte == LONGINT:
+                buffer = rest
+                num = b1282int(num)
+                gotItem(num)
+            elif typebyte == LONGNEG:
+                buffer = rest
+                num = b1282int(num)
+                gotItem(-num)
+            elif typebyte == NEG:
+                buffer = rest
+                num = -b1282int(num)
+                gotItem(num)
+            elif typebyte == VOCAB:
+                buffer = rest
+                num = b1282int(num)
+                item = self.incomingVocabulary[num]
+                if self.currentDialect == b"pb":
+                    # the sender issues VOCAB only for dialect pb
+                    gotItem(item)
+                else:
+                    raise NotImplementedError(f"Invalid item for pb protocol {item!r}")
+            elif typebyte == FLOAT:
+                if len(rest) >= 8:
+                    buffer = rest[8:]
+                    gotItem(struct.unpack("!d", rest[:8])[0])
+                else:
+                    return
+            else:
+                raise NotImplementedError(f"Invalid Type Byte {typebyte!r}")
+            while listStack and (len(listStack[-1][1]) == listStack[-1][0]):
+                item = listStack.pop()[1]
+                gotItem(item)
+        self.buffer = b""
+
+    def expressionReceived(self, lst):
+        """Called when an expression (list, string, or int) is received."""
+        raise NotImplementedError()
+
+    outgoingVocabulary = {
+        # Jelly Data Types
+        b"None": 1,
+        b"class": 2,
+        b"dereference": 3,
+        b"reference": 4,
+        b"dictionary": 5,
+        b"function": 6,
+        b"instance": 7,
+        b"list": 8,
+        b"module": 9,
+        b"persistent": 10,
+        b"tuple": 11,
+        b"unpersistable": 12,
+        # PB Data Types
+        b"copy": 13,
+        b"cache": 14,
+        b"cached": 15,
+        b"remote": 16,
+        b"local": 17,
+        b"lcache": 18,
+        # PB Protocol Messages
+        b"version": 19,
+        b"login": 20,
+        b"password": 21,
+        b"challenge": 22,
+        b"logged_in": 23,
+        b"not_logged_in": 24,
+        b"cachemessage": 25,
+        b"message": 26,
+        b"answer": 27,
+        b"error": 28,
+        b"decref": 29,
+        b"decache": 30,
+        b"uncache": 31,
+    }
+
+    incomingVocabulary = {}
+    for k, v in outgoingVocabulary.items():
+        incomingVocabulary[v] = k
+
+    def __init__(self, isClient=1):
+        self.listStack = []
+        self.outgoingSymbols = copy.copy(self.outgoingVocabulary)
+        self.outgoingSymbolCount = 0
+        self.isClient = isClient
+
+    def sendEncoded(self, obj):
+        """
+        Send the encoded representation of the given object:
+
+        @param obj: An object to encode and send.
+
+        @raise BananaError: If the given object is not an instance of one of
+            the types supported by Banana.
+
+        @return: L{None}
+        """
+        encodeStream = BytesIO()
+        self._encode(obj, encodeStream.write)
+        value = encodeStream.getvalue()
+        self.transport.write(value)
+
+    def _encode(self, obj, write):
+        if isinstance(obj, (list, tuple)):
+            if len(obj) > SIZE_LIMIT:
+                raise BananaError("list/tuple is too long to send (%d)" % (len(obj),))
+            int2b128(len(obj), write)
+            write(LIST)
+            for elem in obj:
+                self._encode(elem, write)
+        elif isinstance(obj, int):
+            if obj < self._smallestLongInt or obj > self._largestLongInt:
+                raise BananaError("int is too large to send (%d)" % (obj,))
+            if obj < self._smallestInt:
+                int2b128(-obj, write)
+                write(LONGNEG)
+            elif obj < 0:
+                int2b128(-obj, write)
+                write(NEG)
+            elif obj <= self._largestInt:
+                int2b128(obj, write)
+                write(INT)
+            else:
+                int2b128(obj, write)
+                write(LONGINT)
+        elif isinstance(obj, float):
+            write(FLOAT)
+            write(struct.pack("!d", obj))
+        elif isinstance(obj, bytes):
+            # TODO: an API for extending banana...
+            if self.currentDialect == b"pb" and obj in self.outgoingSymbols:
+                symbolID = self.outgoingSymbols[obj]
+                int2b128(symbolID, write)
+                write(VOCAB)
+            else:
+                if len(obj) > SIZE_LIMIT:
+                    raise BananaError(
+                        "byte string is too long to send (%d)" % (len(obj),)
+                    )
+                int2b128(len(obj), write)
+                write(STRING)
+                write(obj)
+        else:
+            raise BananaError(
+                "Banana cannot send {} objects: {!r}".format(
+                    fullyQualifiedName(type(obj)), obj
+                )
+            )
+
+
+# For use from the interactive interpreter
+_i = Banana()
+_i.connectionMade()
+_i._selectDialect(b"none")
+
+
+def encode(lst):
+    """Encode a list s-expression."""
+    encodeStream = BytesIO()
+    _i.transport = encodeStream
+    _i.sendEncoded(lst)
+    return encodeStream.getvalue()
+
+
+def decode(st):
+    """
+    Decode a banana-encoded string.
+    """
+    l = []
+    _i.expressionReceived = l.append
+    try:
+        _i.dataReceived(st)
+    finally:
+        _i.buffer = b""
+        del _i.expressionReceived
+    return l[0]
diff --git a/contrib/python/Twisted/py3/twisted/spread/flavors.py b/contrib/python/Twisted/py3/twisted/spread/flavors.py
new file mode 100644
index 00000000000..ef98fee272c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/spread/flavors.py
@@ -0,0 +1,651 @@
+# -*- test-case-name: twisted.spread.test.test_pb -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module represents flavors of remotely accessible objects.
+
+Currently this is only objects accessible through Perspective Broker, but will
+hopefully encompass all forms of remote access which can emulate subsets of PB
+(such as XMLRPC or SOAP).
+
+Future Plans: Optimization.  Exploitation of new-style object model.
+Optimizations to this module should not affect external-use semantics at all,
+but may have a small impact on users who subclass and override methods.
+
+@author: Glyph Lefkowitz
+"""
+
+
+# NOTE: this module should NOT import pb; it is supposed to be a module which
+# abstractly defines remotely accessible types.  Many of these types expect to
+# be serialized by Jelly, but they ought to be accessible through other
+# mechanisms (like XMLRPC)
+
+import sys
+
+from zope.interface import Interface, implementer
+
+from twisted.python import log, reflect
+from twisted.python.compat import cmp, comparable
+from .jelly import (
+    Jellyable,
+    Unjellyable,
+    _createBlank,
+    getInstanceState,
+    setInstanceState,
+    setUnjellyableFactoryForClass,
+    setUnjellyableForClass,
+    setUnjellyableForClassTree,
+    unjellyableRegistry,
+)
+
+# compatibility
+setCopierForClass = setUnjellyableForClass
+setCopierForClassTree = setUnjellyableForClassTree
+setFactoryForClass = setUnjellyableFactoryForClass
+copyTags = unjellyableRegistry
+
+copy_atom = b"copy"
+cache_atom = b"cache"
+cached_atom = b"cached"
+remote_atom = b"remote"
+
+
+class NoSuchMethod(AttributeError):
+    """Raised if there is no such remote method"""
+
+
+class IPBRoot(Interface):
+    """Factory for root Referenceable objects for PB servers."""
+
+    def rootObject(broker):
+        """Return root Referenceable for broker."""
+
+
+class Serializable(Jellyable):
+    """An object that can be passed remotely.
+
+    I am a style of object which can be serialized by Perspective
+    Broker.  Objects which wish to be referenceable or copied remotely
+    have to subclass Serializable.  However, clients of Perspective
+    Broker will probably not want to directly subclass Serializable; the
+    Flavors of transferable objects are listed below.
+
+    What it means to be \"Serializable\" is that an object can be
+    passed to or returned from a remote method.  Certain basic types
+    (dictionaries, lists, tuples, numbers, strings) are serializable by
+    default; however, classes need to choose a specific serialization
+    style: L{Referenceable}, L{Viewable}, L{Copyable} or L{Cacheable}.
+
+    You may also pass C{[lists, dictionaries, tuples]} of L{Serializable}
+    instances to or return them from remote methods, as many levels deep
+    as you like.
+    """
+
+    def processUniqueID(self):
+        """Return an ID which uniquely represents this object for this process.
+
+        By default, this uses the 'id' builtin, but can be overridden to
+        indicate that two values are identity-equivalent (such as proxies
+        for the same object).
+        """
+
+        return id(self)
+
+
+class Referenceable(Serializable):
+    perspective = None
+    """I am an object sent remotely as a direct reference.
+
+    When one of my subclasses is sent as an argument to or returned
+    from a remote method call, I will be serialized by default as a
+    direct reference.
+
+    This means that the peer will be able to call methods on me;
+    a method call xxx() from my peer will be resolved to methods
+    of the name remote_xxx.
+    """
+
+    def remoteMessageReceived(self, broker, message, args, kw):
+        """A remote message has been received.  Dispatch it appropriately.
+
+        The default implementation is to dispatch to a method called
+        'remote_messagename' and call it with the same arguments.
+        """
+        args = broker.unserialize(args)
+        kw = broker.unserialize(kw)
+        # Need this to interoperate with Python 2 clients
+        # which may try to send use keywords where keys are of type
+        # bytes.
+        if [key for key in kw.keys() if isinstance(key, bytes)]:
+            kw = {k.decode("utf8"): v for k, v in kw.items()}
+
+        if not isinstance(message, str):
+            message = message.decode("utf8")
+
+        method = getattr(self, "remote_%s" % message, None)
+        if method is None:
+            raise NoSuchMethod(f"No such method: remote_{message}")
+        try:
+            state = method(*args, **kw)
+        except TypeError:
+            log.msg(f"{method} didn't accept {args} and {kw}")
+            raise
+        return broker.serialize(state, self.perspective)
+
+    def jellyFor(self, jellier):
+        """(internal)
+
+        Return a tuple which will be used as the s-expression to
+        serialize this to a peer.
+        """
+
+        return [b"remote", jellier.invoker.registerReference(self)]
+
+
+@implementer(IPBRoot)
+class Root(Referenceable):
+    """I provide a root object to L{pb.Broker}s for a L{pb.PBClientFactory} or
+    L{pb.PBServerFactory}.
+
+    When a factory produces a L{pb.Broker}, it supplies that
+    L{pb.Broker} with an object named \"root\".  That object is obtained
+    by calling my rootObject method.
+    """
+
+    def rootObject(self, broker):
+        """A factory is requesting to publish me as a root object.
+
+        When a factory is sending me as the root object, this
+        method will be invoked to allow per-broker versions of an
+        object.  By default I return myself.
+        """
+        return self
+
+
+class ViewPoint(Referenceable):
+    """
+    I act as an indirect reference to an object accessed through a
+    L{pb.IPerspective}.
+
+    Simply put, I combine an object with a perspective so that when a
+    peer calls methods on the object I refer to, the method will be
+    invoked with that perspective as a first argument, so that it can
+    know who is calling it.
+
+    While L{Viewable} objects will be converted to ViewPoints by default
+    when they are returned from or sent as arguments to a remote
+    method, any object may be manually proxied as well. (XXX: Now that
+    this class is no longer named C{Proxy}, this is the only occurrence
+    of the term 'proxied' in this docstring, and may be unclear.)
+
+    This can be useful when dealing with L{pb.IPerspective}s, L{Copyable}s,
+    and L{Cacheable}s.  It is legal to implement a method as such on
+    a perspective::
+
+     | def perspective_getViewPointForOther(self, name):
+     |     defr = self.service.getPerspectiveRequest(name)
+     |     defr.addCallbacks(lambda x, self=self: ViewPoint(self, x), log.msg)
+     |     return defr
+
+    This will allow you to have references to Perspective objects in two
+    different ways.  One is through the initial 'attach' call -- each
+    peer will have a L{pb.RemoteReference} to their perspective directly.  The
+    other is through this method; each peer can get a L{pb.RemoteReference} to
+    all other perspectives in the service; but that L{pb.RemoteReference} will
+    be to a L{ViewPoint}, not directly to the object.
+
+    The practical offshoot of this is that you can implement 2 varieties
+    of remotely callable methods on this Perspective; view_xxx and
+    C{perspective_xxx}. C{view_xxx} methods will follow the rules for
+    ViewPoint methods (see ViewPoint.L{remoteMessageReceived}), and
+    C{perspective_xxx} methods will follow the rules for Perspective
+    methods.
+    """
+
+    def __init__(self, perspective, object):
+        """Initialize me with a Perspective and an Object."""
+        self.perspective = perspective
+        self.object = object
+
+    def processUniqueID(self):
+        """Return an ID unique to a proxy for this perspective+object combination."""
+        return (id(self.perspective), id(self.object))
+
+    def remoteMessageReceived(self, broker, message, args, kw):
+        """A remote message has been received.  Dispatch it appropriately.
+
+        The default implementation is to dispatch to a method called
+        'C{view_messagename}' to my Object and call it on my object with
+        the same arguments, modified by inserting my Perspective as
+        the first argument.
+        """
+        args = broker.unserialize(args, self.perspective)
+        kw = broker.unserialize(kw, self.perspective)
+
+        if not isinstance(message, str):
+            message = message.decode("utf8")
+
+        method = getattr(self.object, "view_%s" % message)
+        try:
+            state = method(*(self.perspective,) + args, **kw)
+        except TypeError:
+            log.msg(f"{method} didn't accept {args} and {kw}")
+            raise
+        rv = broker.serialize(state, self.perspective, method, args, kw)
+        return rv
+
+
+class Viewable(Serializable):
+    """I will be converted to a L{ViewPoint} when passed to or returned from a remote method.
+
+    The beginning of a peer's interaction with a PB Service is always
+    through a perspective.  However, if a C{perspective_xxx} method returns
+    a Viewable, it will be serialized to the peer as a response to that
+    method.
+    """
+
+    def jellyFor(self, jellier):
+        """Serialize a L{ViewPoint} for me and the perspective of the given broker."""
+        return ViewPoint(jellier.invoker.serializingPerspective, self).jellyFor(jellier)
+
+
+class Copyable(Serializable):
+    """Subclass me to get copied each time you are returned from or passed to a remote method.
+
+    When I am returned from or passed to a remote method call, I will be
+    converted into data via a set of callbacks (see my methods for more
+    info).  That data will then be serialized using Jelly, and sent to
+    the peer.
+
+    The peer will then look up the type to represent this with; see
+    L{RemoteCopy} for details.
+    """
+
+    def getStateToCopy(self):
+        """Gather state to send when I am serialized for a peer.
+
+        I will default to returning self.__dict__.  Override this to
+        customize this behavior.
+        """
+
+        return self.__dict__
+
+    def getStateToCopyFor(self, perspective):
+        """
+        Gather state to send when I am serialized for a particular
+        perspective.
+
+        I will default to calling L{getStateToCopy}.  Override this to
+        customize this behavior.
+        """
+
+        return self.getStateToCopy()
+
+    def getTypeToCopy(self):
+        """Determine what type tag to send for me.
+
+        By default, send the string representation of my class
+        (package.module.Class); normally this is adequate, but
+        you may override this to change it.
+        """
+
+        return reflect.qual(self.__class__).encode("utf-8")
+
+    def getTypeToCopyFor(self, perspective):
+        """Determine what type tag to send for me.
+
+        By default, defer to self.L{getTypeToCopy}() normally this is
+        adequate, but you may override this to change it.
+        """
+
+        return self.getTypeToCopy()
+
+    def jellyFor(self, jellier):
+        """Assemble type tag and state to copy for this broker.
+
+        This will call L{getTypeToCopyFor} and L{getStateToCopy}, and
+        return an appropriate s-expression to represent me.
+        """
+
+        if jellier.invoker is None:
+            return getInstanceState(self, jellier)
+        p = jellier.invoker.serializingPerspective
+        t = self.getTypeToCopyFor(p)
+        state = self.getStateToCopyFor(p)
+        sxp = jellier.prepare(self)
+        sxp.extend([t, jellier.jelly(state)])
+        return jellier.preserve(self, sxp)
+
+
+class Cacheable(Copyable):
+    """A cached instance.
+
+    This means that it's copied; but there is some logic to make sure
+    that it's only copied once.  Additionally, when state is retrieved,
+    it is passed a "proto-reference" to the state as it will exist on
+    the client.
+
+    XXX: The documentation for this class needs work, but it's the most
+    complex part of PB and it is inherently difficult to explain.
+    """
+
+    def getStateToCacheAndObserveFor(self, perspective, observer):
+        """
+        Get state to cache on the client and client-cache reference
+        to observe locally.
+
+        This is similar to getStateToCopyFor, but it additionally
+        passes in a reference to the client-side RemoteCache instance
+        that will be created when it is unserialized.  This allows
+        Cacheable instances to keep their RemoteCaches up to date when
+        they change, such that no changes can occur between the point
+        at which the state is initially copied and the client receives
+        it that are not propagated.
+        """
+
+        return self.getStateToCopyFor(perspective)
+
+    def jellyFor(self, jellier):
+        """Return an appropriate tuple to serialize me.
+
+        Depending on whether this broker has cached me or not, this may
+        return either a full state or a reference to an existing cache.
+        """
+        if jellier.invoker is None:
+            return getInstanceState(self, jellier)
+        luid = jellier.invoker.cachedRemotelyAs(self, 1)
+        if luid is None:
+            luid = jellier.invoker.cacheRemotely(self)
+            p = jellier.invoker.serializingPerspective
+            type_ = self.getTypeToCopyFor(p)
+            observer = RemoteCacheObserver(jellier.invoker, self, p)
+            state = self.getStateToCacheAndObserveFor(p, observer)
+            l = jellier.prepare(self)
+            jstate = jellier.jelly(state)
+            l.extend([type_, luid, jstate])
+            return jellier.preserve(self, l)
+        else:
+            return cached_atom, luid
+
+    def stoppedObserving(self, perspective, observer):
+        """This method is called when a client has stopped observing me.
+
+        The 'observer' argument is the same as that passed in to
+        getStateToCacheAndObserveFor.
+        """
+
+
+class RemoteCopy(Unjellyable):
+    """I am a remote copy of a Copyable object.
+
+    When the state from a L{Copyable} object is received, an instance will
+    be created based on the copy tags table (see setUnjellyableForClass) and
+    sent the L{setCopyableState} message.  I provide a reasonable default
+    implementation of that message; subclass me if you wish to serve as
+    a copier for remote data.
+
+    NOTE: copiers are invoked with no arguments.  Do not implement a
+    constructor which requires args in a subclass of L{RemoteCopy}!
+    """
+
+    def setCopyableState(self, state):
+        """I will be invoked with the state to copy locally.
+
+        'state' is the data returned from the remote object's
+        'getStateToCopyFor' method, which will often be the remote
+        object's dictionary (or a filtered approximation of it depending
+        on my peer's perspective).
+        """
+        if not state:
+            state = {}
+        state = {
+            x.decode("utf8") if isinstance(x, bytes) else x: y for x, y in state.items()
+        }
+        self.__dict__ = state
+
+    def unjellyFor(self, unjellier, jellyList):
+        if unjellier.invoker is None:
+            return setInstanceState(self, unjellier, jellyList)
+        self.setCopyableState(unjellier.unjelly(jellyList[1]))
+        return self
+
+
+class RemoteCache(RemoteCopy, Serializable):
+    """A cache is a local representation of a remote L{Cacheable} object.
+
+    This represents the last known state of this object.  It may
+    also have methods invoked on it -- in order to update caches,
+    the cached class generates a L{pb.RemoteReference} to this object as
+    it is originally sent.
+
+    Much like copy, I will be invoked with no arguments.  Do not
+    implement a constructor that requires arguments in one of my
+    subclasses.
+    """
+
+    def remoteMessageReceived(self, broker, message, args, kw):
+        """A remote message has been received.  Dispatch it appropriately.
+
+        The default implementation is to dispatch to a method called
+        'C{observe_messagename}' and call it on my  with the same arguments.
+        """
+        if not isinstance(message, str):
+            message = message.decode("utf8")
+
+        args = broker.unserialize(args)
+        kw = broker.unserialize(kw)
+        method = getattr(self, "observe_%s" % message)
+        try:
+            state = method(*args, **kw)
+        except TypeError:
+            log.msg(f"{method} didn't accept {args} and {kw}")
+            raise
+        return broker.serialize(state, None, method, args, kw)
+
+    def jellyFor(self, jellier):
+        """serialize me (only for the broker I'm for) as the original cached reference"""
+        if jellier.invoker is None:
+            return getInstanceState(self, jellier)
+        assert (
+            jellier.invoker is self.broker
+        ), "You cannot exchange cached proxies between brokers."
+        return b"lcache", self.luid
+
+    def unjellyFor(self, unjellier, jellyList):
+        if unjellier.invoker is None:
+            return setInstanceState(self, unjellier, jellyList)
+        self.broker = unjellier.invoker
+        self.luid = jellyList[1]
+        borgCopy = self._borgify()
+        # XXX questionable whether this was a good design idea...
+        init = getattr(borgCopy, "__init__", None)
+        if init:
+            init()
+        unjellier.invoker.cacheLocally(jellyList[1], self)
+        borgCopy.setCopyableState(unjellier.unjelly(jellyList[2]))
+        # Might have changed due to setCopyableState method; we'll assume that
+        # it's bad form to do so afterwards.
+        self.__dict__ = borgCopy.__dict__
+        # chomp, chomp -- some existing code uses "self.__dict__ =", some uses
+        # "__dict__.update".  This is here in order to handle both cases.
+        self.broker = unjellier.invoker
+        self.luid = jellyList[1]
+        return borgCopy
+
+    ##     def __really_del__(self):
+    ##         """Final finalization call, made after all remote references have been lost.
+    ##         """
+
+    def __cmp__(self, other):
+        """Compare me [to another RemoteCache."""
+        if isinstance(other, self.__class__):
+            return cmp(id(self.__dict__), id(other.__dict__))
+        else:
+            return cmp(id(self.__dict__), other)
+
+    def __hash__(self):
+        """Hash me."""
+        return int(id(self.__dict__) % sys.maxsize)
+
+    broker = None
+    luid = None
+
+    def __del__(self):
+        """Do distributed reference counting on finalize."""
+        try:
+            # log.msg( ' --- decache: %s %s' % (self, self.luid) )
+            if self.broker:
+                self.broker.decCacheRef(self.luid)
+        except BaseException:
+            log.deferr()
+
+    def _borgify(self):
+        """
+        Create a new object that shares its state (i.e. its C{__dict__}) and
+        type with this object, but does not share its identity.
+
+        This is an instance of U{the Borg design pattern
+        <https://code.activestate.com/recipes/66531/>} originally described by
+        Alex Martelli, but unlike the example given there, this is not a
+        replacement for a Singleton.  Instead, it is for lifecycle tracking
+        (and distributed garbage collection).  The purpose of these separate
+        objects is to have a separate object tracking each application-level
+        reference to the root L{RemoteCache} object being tracked by the
+        broker, and to have their C{__del__} methods be invoked.
+
+        This may be achievable via a weak value dictionary to track the root
+        L{RemoteCache} instances instead, but this implementation strategy
+        predates the availability of weak references in Python.
+
+        @return: The new instance.
+        @rtype: C{self.__class__}
+        """
+        blank = _createBlank(self.__class__)
+        blank.__dict__ = self.__dict__
+        return blank
+
+
+def unjellyCached(unjellier, unjellyList):
+    luid = unjellyList[1]
+    return unjellier.invoker.cachedLocallyAs(luid)._borgify()
+
+
+setUnjellyableForClass("cached", unjellyCached)
+
+
+def unjellyLCache(unjellier, unjellyList):
+    luid = unjellyList[1]
+    obj = unjellier.invoker.remotelyCachedForLUID(luid)
+    return obj
+
+
+setUnjellyableForClass("lcache", unjellyLCache)
+
+
+def unjellyLocal(unjellier, unjellyList):
+    obj = unjellier.invoker.localObjectForID(unjellyList[1])
+    return obj
+
+
+setUnjellyableForClass("local", unjellyLocal)
+
+
+@comparable
+class RemoteCacheMethod:
+    """A method on a reference to a L{RemoteCache}."""
+
+    def __init__(self, name, broker, cached, perspective):
+        """(internal) initialize."""
+        self.name = name
+        self.broker = broker
+        self.perspective = perspective
+        self.cached = cached
+
+    def __cmp__(self, other):
+        return cmp((self.name, self.broker, self.perspective, self.cached), other)
+
+    def __hash__(self):
+        return hash((self.name, self.broker, self.perspective, self.cached))
+
+    def __call__(self, *args, **kw):
+        """(internal) action method."""
+        cacheID = self.broker.cachedRemotelyAs(self.cached)
+        if cacheID is None:
+            from pb import ProtocolError  # type: ignore[import]
+
+            raise ProtocolError(
+                "You can't call a cached method when the object hasn't been given to the peer yet."
+            )
+        return self.broker._sendMessage(
+            b"cache", self.perspective, cacheID, self.name, args, kw
+        )
+
+
+@comparable
+class RemoteCacheObserver:
+    """I am a reverse-reference to the peer's L{RemoteCache}.
+
+    I am generated automatically when a cache is serialized.  I
+    represent a reference to the client's L{RemoteCache} object that
+    will represent a particular L{Cacheable}; I am the additional
+    object passed to getStateToCacheAndObserveFor.
+    """
+
+    def __init__(self, broker, cached, perspective):
+        """(internal) Initialize me.
+
+        @param broker: a L{pb.Broker} instance.
+
+        @param cached: a L{Cacheable} instance that this L{RemoteCacheObserver}
+            corresponds to.
+
+        @param perspective: a reference to the perspective who is observing this.
+        """
+
+        self.broker = broker
+        self.cached = cached
+        self.perspective = perspective
+
+    def __repr__(self) -> str:
+        return "<RemoteCacheObserver({}, {}, {}) at {}>".format(
+            self.broker,
+            self.cached,
+            self.perspective,
+            id(self),
+        )
+
+    def __hash__(self):
+        """Generate a hash unique to all L{RemoteCacheObserver}s for this broker/perspective/cached triplet"""
+
+        return (
+            (hash(self.broker) % 2**10)
+            + (hash(self.perspective) % 2**10)
+            + (hash(self.cached) % 2**10)
+        )
+
+    def __cmp__(self, other):
+        """Compare me to another L{RemoteCacheObserver}."""
+
+        return cmp((self.broker, self.perspective, self.cached), other)
+
+    def callRemote(self, _name, *args, **kw):
+        """(internal) action method."""
+        cacheID = self.broker.cachedRemotelyAs(self.cached)
+        if isinstance(_name, str):
+            _name = _name.encode("utf-8")
+        if cacheID is None:
+            from pb import ProtocolError
+
+            raise ProtocolError(
+                "You can't call a cached method when the "
+                "object hasn't been given to the peer yet."
+            )
+        return self.broker._sendMessage(
+            b"cache", self.perspective, cacheID, _name, args, kw
+        )
+
+    def remoteMethod(self, key):
+        """Get a L{pb.RemoteMethod} for this key."""
+        return RemoteCacheMethod(key, self.broker, self.cached, self.perspective)
diff --git a/contrib/python/Twisted/py3/twisted/spread/interfaces.py b/contrib/python/Twisted/py3/twisted/spread/interfaces.py
new file mode 100644
index 00000000000..4b8b28935b6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/spread/interfaces.py
@@ -0,0 +1,30 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Spread Interfaces.
+"""
+
+from zope.interface import Interface
+
+
+class IJellyable(Interface):
+    def jellyFor(jellier):
+        """
+        Jelly myself for jellier.
+        """
+
+
+class IUnjellyable(Interface):
+    def unjellyFor(jellier, jellyList):
+        """
+        Unjelly myself for the jellier.
+
+        @param jellier: A stateful object which exists for the lifetime of a
+        single call to L{unjelly}.
+
+        @param jellyList: The C{list} which represents the jellied state of the
+        object to be unjellied.
+
+        @return: The object which results from unjellying.
+        """
diff --git a/contrib/python/Twisted/py3/twisted/spread/jelly.py b/contrib/python/Twisted/py3/twisted/spread/jelly.py
new file mode 100644
index 00000000000..46cda178448
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/spread/jelly.py
@@ -0,0 +1,1092 @@
+# -*- test-case-name: twisted.spread.test.test_jelly -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+S-expression-based persistence of python objects.
+
+It does something very much like L{Pickle<pickle>}; however, pickle's main goal
+seems to be efficiency (both in space and time); jelly's main goals are
+security, human readability, and portability to other environments.
+
+This is how Jelly converts various objects to s-expressions.
+
+Boolean::
+    True --> ['boolean', 'true']
+
+Integer::
+    1 --> 1
+
+List::
+    [1, 2] --> ['list', 1, 2]
+
+String::
+    \"hello\" --> \"hello\"
+
+Float::
+    2.3 --> 2.3
+
+Dictionary::
+    {'a': 1, 'b': 'c'} --> ['dictionary', ['b', 'c'], ['a', 1]]
+
+Module::
+    UserString --> ['module', 'UserString']
+
+Class::
+    UserString.UserString --> ['class', ['module', 'UserString'], 'UserString']
+
+Function::
+    string.join --> ['function', 'join', ['module', 'string']]
+
+Instance: s is an instance of UserString.UserString, with a __dict__
+{'data': 'hello'}::
+    [\"UserString.UserString\", ['dictionary', ['data', 'hello']]]
+
+Class Method: UserString.UserString.center::
+    ['method', 'center', ['None'], ['class', ['module', 'UserString'],
+     'UserString']]
+
+Instance Method: s.center, where s is an instance of UserString.UserString::
+    ['method', 'center', ['instance', ['reference', 1, ['class',
+    ['module', 'UserString'], 'UserString']], ['dictionary', ['data', 'd']]],
+    ['dereference', 1]]
+
+The Python 2.x C{sets.Set} and C{sets.ImmutableSet} classes are
+serialized to the same thing as the builtin C{set} and C{frozenset}
+classes.  (This is only relevant if you are communicating with a
+version of jelly running on an older version of Python.)
+
+@author: Glyph Lefkowitz
+
+"""
+
+import copy
+import datetime
+import decimal
+
+# System Imports
+import types
+import warnings
+from functools import reduce
+
+from zope.interface import implementer
+
+from incremental import Version
+
+from twisted.persisted.crefutil import (
+    NotKnown,
+    _Container,
+    _Dereference,
+    _DictKeyAndValue,
+    _InstanceMethod,
+    _Tuple,
+)
+
+# Twisted Imports
+from twisted.python.compat import nativeString
+from twisted.python.deprecate import deprecatedModuleAttribute
+from twisted.python.reflect import namedAny, namedObject, qual
+from twisted.spread.interfaces import IJellyable, IUnjellyable
+
+DictTypes = (dict,)
+
+None_atom = b"None"  # N
+# code
+class_atom = b"class"  # c
+module_atom = b"module"  # m
+function_atom = b"function"  # f
+
+# references
+dereference_atom = b"dereference"  # D
+persistent_atom = b"persistent"  # p
+reference_atom = b"reference"  # r
+
+# mutable collections
+dictionary_atom = b"dictionary"  # d
+list_atom = b"list"  # l
+set_atom = b"set"
+
+# immutable collections
+#   (assignment to __dict__ and __class__ still might go away!)
+tuple_atom = b"tuple"  # t
+instance_atom = b"instance"  # i
+frozenset_atom = b"frozenset"
+
+
+deprecatedModuleAttribute(
+    Version("Twisted", 15, 0, 0),
+    "instance_atom is unused within Twisted.",
+    "twisted.spread.jelly",
+    "instance_atom",
+)
+
+# errors
+unpersistable_atom = b"unpersistable"  # u
+unjellyableRegistry = {}
+unjellyableFactoryRegistry = {}
+
+
+def _createBlank(cls):
+    """
+    Given an object, if that object is a type, return a new, blank instance
+    of that type which has not had C{__init__} called on it.  If the object
+    is not a type, return L{None}.
+
+    @param cls: The type (or class) to create an instance of.
+    @type cls: L{type} or something else that cannot be
+        instantiated.
+
+    @return: a new blank instance or L{None} if C{cls} is not a class or type.
+    """
+    if isinstance(cls, type):
+        return cls.__new__(cls)
+
+
+def _newInstance(cls, state):
+    """
+    Make a new instance of a class without calling its __init__ method.
+
+    @param state: A C{dict} used to update C{inst.__dict__} either directly or
+        via C{__setstate__}, if available.
+
+    @return: A new instance of C{cls}.
+    """
+    instance = _createBlank(cls)
+
+    def defaultSetter(state):
+        if isinstance(state, dict):
+            instance.__dict__ = state or {}
+
+    setter = getattr(instance, "__setstate__", defaultSetter)
+    setter(state)
+    return instance
+
+
+def _maybeClass(classnamep):
+    isObject = isinstance(classnamep, type)
+
+    if isObject:
+        classnamep = qual(classnamep)
+
+    if not isinstance(classnamep, bytes):
+        classnamep = classnamep.encode("utf-8")
+
+    return classnamep
+
+
+def setUnjellyableForClass(classname, unjellyable):
+    """
+    Set which local class will represent a remote type.
+
+    If you have written a Copyable class that you expect your client to be
+    receiving, write a local "copy" class to represent it, then call::
+
+        jellier.setUnjellyableForClass('module.package.Class', MyCopier).
+
+    Call this at the module level immediately after its class
+    definition. MyCopier should be a subclass of RemoteCopy.
+
+    The classname may be a special tag returned by
+    'Copyable.getTypeToCopyFor' rather than an actual classname.
+
+    This call is also for cached classes, since there will be no
+    overlap.  The rules are the same.
+    """
+
+    global unjellyableRegistry
+    classname = _maybeClass(classname)
+    unjellyableRegistry[classname] = unjellyable
+    globalSecurity.allowTypes(classname)
+
+
+def setUnjellyableFactoryForClass(classname, copyFactory):
+    """
+    Set the factory to construct a remote instance of a type::
+
+      jellier.setUnjellyableFactoryForClass('module.package.Class', MyFactory)
+
+    Call this at the module level immediately after its class definition.
+    C{copyFactory} should return an instance or subclass of
+    L{RemoteCopy<pb.RemoteCopy>}.
+
+    Similar to L{setUnjellyableForClass} except it uses a factory instead
+    of creating an instance.
+    """
+
+    global unjellyableFactoryRegistry
+    classname = _maybeClass(classname)
+    unjellyableFactoryRegistry[classname] = copyFactory
+    globalSecurity.allowTypes(classname)
+
+
+def setUnjellyableForClassTree(module, baseClass, prefix=None):
+    """
+    Set all classes in a module derived from C{baseClass} as copiers for
+    a corresponding remote class.
+
+    When you have a hierarchy of Copyable (or Cacheable) classes on one
+    side, and a mirror structure of Copied (or RemoteCache) classes on the
+    other, use this to setUnjellyableForClass all your Copieds for the
+    Copyables.
+
+    Each copyTag (the \"classname\" argument to getTypeToCopyFor, and
+    what the Copyable's getTypeToCopyFor returns) is formed from
+    adding a prefix to the Copied's class name.  The prefix defaults
+    to module.__name__.  If you wish the copy tag to consist of solely
+    the classname, pass the empty string \'\'.
+
+    @param module: a module object from which to pull the Copied classes.
+        (passing sys.modules[__name__] might be useful)
+
+    @param baseClass: the base class from which all your Copied classes derive.
+
+    @param prefix: the string prefixed to classnames to form the
+        unjellyableRegistry.
+    """
+    if prefix is None:
+        prefix = module.__name__
+
+    if prefix:
+        prefix = "%s." % prefix
+
+    for name in dir(module):
+        loaded = getattr(module, name)
+        try:
+            yes = issubclass(loaded, baseClass)
+        except TypeError:
+            "It's not a class."
+        else:
+            if yes:
+                setUnjellyableForClass(f"{prefix}{name}", loaded)
+
+
+def getInstanceState(inst, jellier):
+    """
+    Utility method to default to 'normal' state rules in serialization.
+    """
+    if hasattr(inst, "__getstate__"):
+        state = inst.__getstate__()
+    else:
+        state = inst.__dict__
+    sxp = jellier.prepare(inst)
+    sxp.extend([qual(inst.__class__).encode("utf-8"), jellier.jelly(state)])
+    return jellier.preserve(inst, sxp)
+
+
+def setInstanceState(inst, unjellier, jellyList):
+    """
+    Utility method to default to 'normal' state rules in unserialization.
+    """
+    state = unjellier.unjelly(jellyList[1])
+    if hasattr(inst, "__setstate__"):
+        inst.__setstate__(state)
+    else:
+        inst.__dict__ = state
+    return inst
+
+
+class Unpersistable:
+    """
+    This is an instance of a class that comes back when something couldn't be
+    unpersisted.
+    """
+
+    def __init__(self, reason):
+        """
+        Initialize an unpersistable object with a descriptive C{reason} string.
+        """
+        self.reason = reason
+
+    def __repr__(self) -> str:
+        return "Unpersistable(%s)" % repr(self.reason)
+
+
+@implementer(IJellyable)
+class Jellyable:
+    """
+    Inherit from me to Jelly yourself directly with the `getStateFor'
+    convenience method.
+    """
+
+    def getStateFor(self, jellier):
+        return self.__dict__
+
+    def jellyFor(self, jellier):
+        """
+        @see: L{twisted.spread.interfaces.IJellyable.jellyFor}
+        """
+        sxp = jellier.prepare(self)
+        sxp.extend(
+            [
+                qual(self.__class__).encode("utf-8"),
+                jellier.jelly(self.getStateFor(jellier)),
+            ]
+        )
+        return jellier.preserve(self, sxp)
+
+
+@implementer(IUnjellyable)
+class Unjellyable:
+    """
+    Inherit from me to Unjelly yourself directly with the
+    C{setStateFor} convenience method.
+    """
+
+    def setStateFor(self, unjellier, state):
+        self.__dict__ = state
+
+    def unjellyFor(self, unjellier, jellyList):
+        """
+        Perform the inverse operation of L{Jellyable.jellyFor}.
+
+        @see: L{twisted.spread.interfaces.IUnjellyable.unjellyFor}
+        """
+        state = unjellier.unjelly(jellyList[1])
+        self.setStateFor(unjellier, state)
+        return self
+
+
+class _Jellier:
+    """
+    (Internal) This class manages state for a call to jelly()
+    """
+
+    def __init__(self, taster, persistentStore, invoker):
+        """
+        Initialize.
+        """
+        self.taster = taster
+        # `preserved' is a dict of previously seen instances.
+        self.preserved = {}
+        # `cooked' is a dict of previously backreferenced instances to their
+        # `ref' lists.
+        self.cooked = {}
+        self.cooker = {}
+        self._ref_id = 1
+        self.persistentStore = persistentStore
+        self.invoker = invoker
+
+    def _cook(self, object):
+        """
+        (internal) Backreference an object.
+
+        Notes on this method for the hapless future maintainer: If I've already
+        gone through the prepare/preserve cycle on the specified object (it is
+        being referenced after the serializer is \"done with\" it, e.g. this
+        reference is NOT circular), the copy-in-place of aList is relevant,
+        since the list being modified is the actual, pre-existing jelly
+        expression that was returned for that object. If not, it's technically
+        superfluous, since the value in self.preserved didn't need to be set,
+        but the invariant that self.preserved[id(object)] is a list is
+        convenient because that means we don't have to test and create it or
+        not create it here, creating fewer code-paths.  that's why
+        self.preserved is always set to a list.
+
+        Sorry that this code is so hard to follow, but Python objects are
+        tricky to persist correctly. -glyph
+        """
+        aList = self.preserved[id(object)]
+        newList = copy.copy(aList)
+        # make a new reference ID
+        refid = self._ref_id
+        self._ref_id = self._ref_id + 1
+        # replace the old list in-place, so that we don't have to track the
+        # previous reference to it.
+        aList[:] = [reference_atom, refid, newList]
+        self.cooked[id(object)] = [dereference_atom, refid]
+        return aList
+
+    def prepare(self, object):
+        """
+        (internal) Create a list for persisting an object to.  This will allow
+        backreferences to be made internal to the object. (circular
+        references).
+
+        The reason this needs to happen is that we don't generate an ID for
+        every object, so we won't necessarily know which ID the object will
+        have in the future.  When it is 'cooked' ( see _cook ), it will be
+        assigned an ID, and the temporary placeholder list created here will be
+        modified in-place to create an expression that gives this object an ID:
+        [reference id# [object-jelly]].
+        """
+
+        # create a placeholder list to be preserved
+        self.preserved[id(object)] = []
+        # keep a reference to this object around, so it doesn't disappear!
+        # (This isn't always necessary, but for cases where the objects are
+        # dynamically generated by __getstate__ or getStateToCopyFor calls, it
+        # is; id() will return the same value for a different object if it gets
+        # garbage collected.  This may be optimized later.)
+        self.cooker[id(object)] = object
+        return []
+
+    def preserve(self, object, sexp):
+        """
+        (internal) Mark an object's persistent list for later referral.
+        """
+        # if I've been cooked in the meanwhile,
+        if id(object) in self.cooked:
+            # replace the placeholder empty list with the real one
+            self.preserved[id(object)][2] = sexp
+            # but give this one back.
+            sexp = self.preserved[id(object)]
+        else:
+            self.preserved[id(object)] = sexp
+        return sexp
+
+    def _checkMutable(self, obj):
+        objId = id(obj)
+        if objId in self.cooked:
+            return self.cooked[objId]
+        if objId in self.preserved:
+            self._cook(obj)
+            return self.cooked[objId]
+
+    def jelly(self, obj):
+        if isinstance(obj, Jellyable):
+            preRef = self._checkMutable(obj)
+            if preRef:
+                return preRef
+            return obj.jellyFor(self)
+        objType = type(obj)
+        if self.taster.isTypeAllowed(qual(objType).encode("utf-8")):
+            # "Immutable" Types
+            if objType in (bytes, int, float):
+                return obj
+            elif isinstance(obj, types.MethodType):
+                aSelf = obj.__self__
+                aFunc = obj.__func__
+                aClass = aSelf.__class__
+                return [
+                    b"method",
+                    aFunc.__name__,
+                    self.jelly(aSelf),
+                    self.jelly(aClass),
+                ]
+            elif objType is str:
+                return [b"unicode", obj.encode("UTF-8")]
+            elif isinstance(obj, type(None)):
+                return [b"None"]
+            elif isinstance(obj, types.FunctionType):
+                return [b"function", obj.__module__ + "." + obj.__qualname__]
+            elif isinstance(obj, types.ModuleType):
+                return [b"module", obj.__name__]
+            elif objType is bool:
+                return [b"boolean", obj and b"true" or b"false"]
+            elif objType is datetime.datetime:
+                if obj.tzinfo:
+                    raise NotImplementedError(
+                        "Currently can't jelly datetime objects with tzinfo"
+                    )
+                return [
+                    b"datetime",
+                    " ".join(
+                        [
+                            str(x)
+                            for x in (
+                                obj.year,
+                                obj.month,
+                                obj.day,
+                                obj.hour,
+                                obj.minute,
+                                obj.second,
+                                obj.microsecond,
+                            )
+                        ]
+                    ).encode("utf-8"),
+                ]
+            elif objType is datetime.time:
+                if obj.tzinfo:
+                    raise NotImplementedError(
+                        "Currently can't jelly datetime objects with tzinfo"
+                    )
+                return [
+                    b"time",
+                    " ".join(
+                        [
+                            str(x)
+                            for x in (obj.hour, obj.minute, obj.second, obj.microsecond)
+                        ]
+                    ).encode("utf-8"),
+                ]
+            elif objType is datetime.date:
+                return [
+                    b"date",
+                    " ".join([str(x) for x in (obj.year, obj.month, obj.day)]).encode(
+                        "utf-8"
+                    ),
+                ]
+            elif objType is datetime.timedelta:
+                return [
+                    b"timedelta",
+                    " ".join(
+                        [str(x) for x in (obj.days, obj.seconds, obj.microseconds)]
+                    ).encode("utf-8"),
+                ]
+            elif issubclass(objType, type):
+                return [b"class", qual(obj).encode("utf-8")]
+            elif objType is decimal.Decimal:
+                return self.jelly_decimal(obj)
+            else:
+                preRef = self._checkMutable(obj)
+                if preRef:
+                    return preRef
+                # "Mutable" Types
+                sxp = self.prepare(obj)
+                if objType is list:
+                    sxp.extend(self._jellyIterable(list_atom, obj))
+                elif objType is tuple:
+                    sxp.extend(self._jellyIterable(tuple_atom, obj))
+                elif objType in DictTypes:
+                    sxp.append(dictionary_atom)
+                    for key, val in obj.items():
+                        sxp.append([self.jelly(key), self.jelly(val)])
+                elif objType is set:
+                    sxp.extend(self._jellyIterable(set_atom, obj))
+                elif objType is frozenset:
+                    sxp.extend(self._jellyIterable(frozenset_atom, obj))
+                else:
+                    className = qual(obj.__class__).encode("utf-8")
+                    persistent = None
+                    if self.persistentStore:
+                        persistent = self.persistentStore(obj, self)
+                    if persistent is not None:
+                        sxp.append(persistent_atom)
+                        sxp.append(persistent)
+                    elif self.taster.isClassAllowed(obj.__class__):
+                        sxp.append(className)
+                        if hasattr(obj, "__getstate__"):
+                            state = obj.__getstate__()
+                        else:
+                            state = obj.__dict__
+                        sxp.append(self.jelly(state))
+                    else:
+                        self.unpersistable(
+                            "instance of class %s deemed insecure"
+                            % qual(obj.__class__),
+                            sxp,
+                        )
+                return self.preserve(obj, sxp)
+        else:
+            raise InsecureJelly(f"Type not allowed for object: {objType} {obj}")
+
+    def _jellyIterable(self, atom, obj):
+        """
+        Jelly an iterable object.
+
+        @param atom: the identifier atom of the object.
+        @type atom: C{str}
+
+        @param obj: any iterable object.
+        @type obj: C{iterable}
+
+        @return: a generator of jellied data.
+        @rtype: C{generator}
+        """
+        yield atom
+        for item in obj:
+            yield self.jelly(item)
+
+    def jelly_decimal(self, d):
+        """
+        Jelly a decimal object.
+
+        @param d: a decimal object to serialize.
+        @type d: C{decimal.Decimal}
+
+        @return: jelly for the decimal object.
+        @rtype: C{list}
+        """
+        sign, guts, exponent = d.as_tuple()
+        value = reduce(lambda left, right: left * 10 + right, guts)
+        if sign:
+            value = -value
+        return [b"decimal", value, exponent]
+
+    def unpersistable(self, reason, sxp=None):
+        """
+        (internal) Returns an sexp: (unpersistable "reason").  Utility method
+        for making note that a particular object could not be serialized.
+        """
+        if sxp is None:
+            sxp = []
+        sxp.append(unpersistable_atom)
+        if isinstance(reason, str):
+            reason = reason.encode("utf-8")
+        sxp.append(reason)
+        return sxp
+
+
+class _Unjellier:
+    def __init__(self, taster, persistentLoad, invoker):
+        self.taster = taster
+        self.persistentLoad = persistentLoad
+        self.references = {}
+        self.postCallbacks = []
+        self.invoker = invoker
+
+    def unjellyFull(self, obj):
+        o = self.unjelly(obj)
+        for m in self.postCallbacks:
+            m()
+        return o
+
+    def _maybePostUnjelly(self, unjellied):
+        """
+        If the given object has support for the C{postUnjelly} hook, set it up
+        to be called at the end of deserialization.
+
+        @param unjellied: an object that has already been unjellied.
+
+        @return: C{unjellied}
+        """
+        if hasattr(unjellied, "postUnjelly"):
+            self.postCallbacks.append(unjellied.postUnjelly)
+        return unjellied
+
+    def unjelly(self, obj):
+        if type(obj) is not list:
+            return obj
+        jelTypeBytes = obj[0]
+        if not self.taster.isTypeAllowed(jelTypeBytes):
+            raise InsecureJelly(jelTypeBytes)
+        regClass = unjellyableRegistry.get(jelTypeBytes)
+        if regClass is not None:
+            method = getattr(_createBlank(regClass), "unjellyFor", regClass)
+            return self._maybePostUnjelly(method(self, obj))
+        regFactory = unjellyableFactoryRegistry.get(jelTypeBytes)
+        if regFactory is not None:
+            return self._maybePostUnjelly(regFactory(self.unjelly(obj[1])))
+
+        jelTypeText = nativeString(jelTypeBytes)
+        thunk = getattr(self, "_unjelly_%s" % jelTypeText, None)
+        if thunk is not None:
+            return thunk(obj[1:])
+        else:
+            nameSplit = jelTypeText.split(".")
+            modName = ".".join(nameSplit[:-1])
+            if not self.taster.isModuleAllowed(modName):
+                raise InsecureJelly(
+                    f"Module {modName} not allowed (in type {jelTypeText})."
+                )
+            clz = namedObject(jelTypeText)
+            if not self.taster.isClassAllowed(clz):
+                raise InsecureJelly("Class %s not allowed." % jelTypeText)
+            return self._genericUnjelly(clz, obj[1])
+
+    def _genericUnjelly(self, cls, state):
+        """
+        Unjelly a type for which no specific unjellier is registered, but which
+        is nonetheless allowed.
+
+        @param cls: the class of the instance we are unjellying.
+        @type cls: L{type}
+
+        @param state: The jellied representation of the object's state; its
+            C{__dict__} unless it has a C{__setstate__} that takes something
+            else.
+        @type state: L{list}
+
+        @return: the new, unjellied instance.
+        """
+        return self._maybePostUnjelly(_newInstance(cls, self.unjelly(state)))
+
+    def _unjelly_None(self, exp):
+        return None
+
+    def _unjelly_unicode(self, exp):
+        return str(exp[0], "UTF-8")
+
+    def _unjelly_decimal(self, exp):
+        """
+        Unjelly decimal objects.
+        """
+        value = exp[0]
+        exponent = exp[1]
+        if value < 0:
+            sign = 1
+        else:
+            sign = 0
+        guts = decimal.Decimal(value).as_tuple()[1]
+        return decimal.Decimal((sign, guts, exponent))
+
+    def _unjelly_boolean(self, exp):
+        assert exp[0] in (b"true", b"false")
+        return exp[0] == b"true"
+
+    def _unjelly_datetime(self, exp):
+        return datetime.datetime(*map(int, exp[0].split()))
+
+    def _unjelly_date(self, exp):
+        return datetime.date(*map(int, exp[0].split()))
+
+    def _unjelly_time(self, exp):
+        return datetime.time(*map(int, exp[0].split()))
+
+    def _unjelly_timedelta(self, exp):
+        days, seconds, microseconds = map(int, exp[0].split())
+        return datetime.timedelta(days=days, seconds=seconds, microseconds=microseconds)
+
+    def unjellyInto(self, obj, loc, jel):
+        o = self.unjelly(jel)
+        if isinstance(o, NotKnown):
+            o.addDependant(obj, loc)
+        obj[loc] = o
+        return o
+
+    def _unjelly_dereference(self, lst):
+        refid = lst[0]
+        x = self.references.get(refid)
+        if x is not None:
+            return x
+        der = _Dereference(refid)
+        self.references[refid] = der
+        return der
+
+    def _unjelly_reference(self, lst):
+        refid = lst[0]
+        exp = lst[1]
+        o = self.unjelly(exp)
+        ref = self.references.get(refid)
+        if ref is None:
+            self.references[refid] = o
+        elif isinstance(ref, NotKnown):
+            ref.resolveDependants(o)
+            self.references[refid] = o
+        else:
+            assert 0, "Multiple references with same ID!"
+        return o
+
+    def _unjelly_tuple(self, lst):
+        l = list(range(len(lst)))
+        finished = 1
+        for elem in l:
+            if isinstance(self.unjellyInto(l, elem, lst[elem]), NotKnown):
+                finished = 0
+        if finished:
+            return tuple(l)
+        else:
+            return _Tuple(l)
+
+    def _unjelly_list(self, lst):
+        l = list(range(len(lst)))
+        for elem in l:
+            self.unjellyInto(l, elem, lst[elem])
+        return l
+
+    def _unjellySetOrFrozenset(self, lst, containerType):
+        """
+        Helper method to unjelly set or frozenset.
+
+        @param lst: the content of the set.
+        @type lst: C{list}
+
+        @param containerType: the type of C{set} to use.
+        """
+        l = list(range(len(lst)))
+        finished = True
+        for elem in l:
+            data = self.unjellyInto(l, elem, lst[elem])
+            if isinstance(data, NotKnown):
+                finished = False
+        if not finished:
+            return _Container(l, containerType)
+        else:
+            return containerType(l)
+
+    def _unjelly_set(self, lst):
+        """
+        Unjelly set using the C{set} builtin.
+        """
+        return self._unjellySetOrFrozenset(lst, set)
+
+    def _unjelly_frozenset(self, lst):
+        """
+        Unjelly frozenset using the C{frozenset} builtin.
+        """
+        return self._unjellySetOrFrozenset(lst, frozenset)
+
+    def _unjelly_dictionary(self, lst):
+        d = {}
+        for k, v in lst:
+            kvd = _DictKeyAndValue(d)
+            self.unjellyInto(kvd, 0, k)
+            self.unjellyInto(kvd, 1, v)
+        return d
+
+    def _unjelly_module(self, rest):
+        moduleName = nativeString(rest[0])
+        if type(moduleName) != str:
+            raise InsecureJelly("Attempted to unjelly a module with a non-string name.")
+        if not self.taster.isModuleAllowed(moduleName):
+            raise InsecureJelly(f"Attempted to unjelly module named {moduleName!r}")
+        mod = __import__(moduleName, {}, {}, "x")
+        return mod
+
+    def _unjelly_class(self, rest):
+        cname = nativeString(rest[0])
+        clist = cname.split(nativeString("."))
+        modName = nativeString(".").join(clist[:-1])
+        if not self.taster.isModuleAllowed(modName):
+            raise InsecureJelly("module %s not allowed" % modName)
+        klaus = namedObject(cname)
+        objType = type(klaus)
+        if objType is not type:
+            raise InsecureJelly(
+                "class %r unjellied to something that isn't a class: %r"
+                % (cname, klaus)
+            )
+        if not self.taster.isClassAllowed(klaus):
+            raise InsecureJelly("class not allowed: %s" % qual(klaus))
+        return klaus
+
+    def _unjelly_function(self, rest):
+        fname = nativeString(rest[0])
+        modSplit = fname.split(nativeString("."))
+        modName = nativeString(".").join(modSplit[:-1])
+        if not self.taster.isModuleAllowed(modName):
+            raise InsecureJelly("Module not allowed: %s" % modName)
+        # XXX do I need an isFunctionAllowed?
+        function = namedAny(fname)
+        return function
+
+    def _unjelly_persistent(self, rest):
+        if self.persistentLoad:
+            pload = self.persistentLoad(rest[0], self)
+            return pload
+        else:
+            return Unpersistable("Persistent callback not found")
+
+    def _unjelly_instance(self, rest):
+        """
+        (internal) Unjelly an instance.
+
+        Called to handle the deprecated I{instance} token.
+
+        @param rest: The s-expression representing the instance.
+
+        @return: The unjellied instance.
+        """
+        warnings.warn_explicit(
+            "Unjelly support for the instance atom is deprecated since "
+            "Twisted 15.0.0.  Upgrade peer for modern instance support.",
+            category=DeprecationWarning,
+            filename="",
+            lineno=0,
+        )
+
+        clz = self.unjelly(rest[0])
+        return self._genericUnjelly(clz, rest[1])
+
+    def _unjelly_unpersistable(self, rest):
+        return Unpersistable(f"Unpersistable data: {rest[0]}")
+
+    def _unjelly_method(self, rest):
+        """
+        (internal) Unjelly a method.
+        """
+        im_name = rest[0]
+        im_self = self.unjelly(rest[1])
+        im_class = self.unjelly(rest[2])
+        if not isinstance(im_class, type):
+            raise InsecureJelly("Method found with non-class class.")
+        if im_name in im_class.__dict__:
+            if im_self is None:
+                im = getattr(im_class, im_name)
+            elif isinstance(im_self, NotKnown):
+                im = _InstanceMethod(im_name, im_self, im_class)
+            else:
+                im = types.MethodType(
+                    im_class.__dict__[im_name], im_self, *([im_class] * (False))
+                )
+        else:
+            raise TypeError("instance method changed")
+        return im
+
+
+#### Published Interface.
+
+
+class InsecureJelly(Exception):
+    """
+    This exception will be raised when a jelly is deemed `insecure'; e.g. it
+    contains a type, class, or module disallowed by the specified `taster'
+    """
+
+
+class DummySecurityOptions:
+    """
+    DummySecurityOptions() -> insecure security options
+    Dummy security options -- this class will allow anything.
+    """
+
+    def isModuleAllowed(self, moduleName):
+        """
+        DummySecurityOptions.isModuleAllowed(moduleName) -> boolean
+        returns 1 if a module by that name is allowed, 0 otherwise
+        """
+        return 1
+
+    def isClassAllowed(self, klass):
+        """
+        DummySecurityOptions.isClassAllowed(class) -> boolean
+        Assumes the module has already been allowed.  Returns 1 if the given
+        class is allowed, 0 otherwise.
+        """
+        return 1
+
+    def isTypeAllowed(self, typeName):
+        """
+        DummySecurityOptions.isTypeAllowed(typeName) -> boolean
+        Returns 1 if the given type is allowed, 0 otherwise.
+        """
+        return 1
+
+
+class SecurityOptions:
+    """
+    This will by default disallow everything, except for 'none'.
+    """
+
+    basicTypes = [
+        "dictionary",
+        "list",
+        "tuple",
+        "reference",
+        "dereference",
+        "unpersistable",
+        "persistent",
+        "long_int",
+        "long",
+        "dict",
+    ]
+
+    def __init__(self):
+        """
+        SecurityOptions() initialize.
+        """
+        # I don't believe any of these types can ever pose a security hazard,
+        # except perhaps "reference"...
+        self.allowedTypes = {
+            b"None": 1,
+            b"bool": 1,
+            b"boolean": 1,
+            b"string": 1,
+            b"str": 1,
+            b"int": 1,
+            b"float": 1,
+            b"datetime": 1,
+            b"time": 1,
+            b"date": 1,
+            b"timedelta": 1,
+            b"NoneType": 1,
+            b"unicode": 1,
+            b"decimal": 1,
+            b"set": 1,
+            b"frozenset": 1,
+        }
+        self.allowedModules = {}
+        self.allowedClasses = {}
+
+    def allowBasicTypes(self):
+        """
+        Allow all `basic' types.  (Dictionary and list.  Int, string, and float
+        are implicitly allowed.)
+        """
+        self.allowTypes(*self.basicTypes)
+
+    def allowTypes(self, *types):
+        """
+        SecurityOptions.allowTypes(typeString): Allow a particular type, by its
+        name.
+        """
+        for typ in types:
+            if isinstance(typ, str):
+                typ = typ.encode("utf-8")
+            if not isinstance(typ, bytes):
+                typ = qual(typ)
+            self.allowedTypes[typ] = 1
+
+    def allowInstancesOf(self, *classes):
+        """
+        SecurityOptions.allowInstances(klass, klass, ...): allow instances
+        of the specified classes
+
+        This will also allow the 'instance', 'class' (renamed 'classobj' in
+        Python 2.3), and 'module' types, as well as basic types.
+        """
+        self.allowBasicTypes()
+        self.allowTypes("instance", "class", "classobj", "module")
+        for klass in classes:
+            self.allowTypes(qual(klass))
+            self.allowModules(klass.__module__)
+            self.allowedClasses[klass] = 1
+
+    def allowModules(self, *modules):
+        """
+        SecurityOptions.allowModules(module, module, ...): allow modules by
+        name. This will also allow the 'module' type.
+        """
+        for module in modules:
+            if type(module) == types.ModuleType:
+                module = module.__name__
+
+            if not isinstance(module, bytes):
+                module = module.encode("utf-8")
+
+            self.allowedModules[module] = 1
+
+    def isModuleAllowed(self, moduleName):
+        """
+        SecurityOptions.isModuleAllowed(moduleName) -> boolean
+        returns 1 if a module by that name is allowed, 0 otherwise
+        """
+        if not isinstance(moduleName, bytes):
+            moduleName = moduleName.encode("utf-8")
+
+        return moduleName in self.allowedModules
+
+    def isClassAllowed(self, klass):
+        """
+        SecurityOptions.isClassAllowed(class) -> boolean
+        Assumes the module has already been allowed.  Returns 1 if the given
+        class is allowed, 0 otherwise.
+        """
+        return klass in self.allowedClasses
+
+    def isTypeAllowed(self, typeName):
+        """
+        SecurityOptions.isTypeAllowed(typeName) -> boolean
+        Returns 1 if the given type is allowed, 0 otherwise.
+        """
+        if not isinstance(typeName, bytes):
+            typeName = typeName.encode("utf-8")
+
+        return typeName in self.allowedTypes or b"." in typeName
+
+
+globalSecurity = SecurityOptions()
+globalSecurity.allowBasicTypes()
+
+
+def jelly(object, taster=DummySecurityOptions(), persistentStore=None, invoker=None):
+    """
+    Serialize to s-expression.
+
+    Returns a list which is the serialized representation of an object.  An
+    optional 'taster' argument takes a SecurityOptions and will mark any
+    insecure objects as unpersistable rather than serializing them.
+    """
+    return _Jellier(taster, persistentStore, invoker).jelly(object)
+
+
+def unjelly(sexp, taster=DummySecurityOptions(), persistentLoad=None, invoker=None):
+    """
+    Unserialize from s-expression.
+
+    Takes a list that was the result from a call to jelly() and unserializes
+    an arbitrary object from it.  The optional 'taster' argument, an instance
+    of SecurityOptions, will cause an InsecureJelly exception to be raised if a
+    disallowed type, module, or class attempted to unserialize.
+    """
+    return _Unjellier(taster, persistentLoad, invoker).unjellyFull(sexp)
diff --git a/contrib/python/Twisted/py3/twisted/spread/pb.py b/contrib/python/Twisted/py3/twisted/spread/pb.py
new file mode 100644
index 00000000000..dcf545015df
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/spread/pb.py
@@ -0,0 +1,1674 @@
+# -*- test-case-name: twisted.spread.test.test_pb -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Perspective Broker
+
+\"This isn\'t a professional opinion, but it's probably got enough
+internet to kill you.\" --glyph
+
+Introduction
+============
+
+This is a broker for proxies for and copies of objects.  It provides a
+translucent interface layer to those proxies.
+
+The protocol is not opaque, because it provides objects which represent the
+remote proxies and require no context (server references, IDs) to operate on.
+
+It is not transparent because it does I{not} attempt to make remote objects
+behave identically, or even similarly, to local objects.  Method calls are
+invoked asynchronously, and specific rules are applied when serializing
+arguments.
+
+To get started, begin with L{PBClientFactory} and L{PBServerFactory}.
+
+@author: Glyph Lefkowitz
+"""
+
+
+import random
+from hashlib import md5
+
+from zope.interface import Interface, implementer
+
+from twisted.cred.credentials import (
+    Anonymous,
+    IAnonymous,
+    ICredentials,
+    IUsernameHashedPassword,
+)
+from twisted.cred.portal import Portal
+from twisted.internet import defer, protocol
+from twisted.persisted import styles
+
+# Twisted Imports
+from twisted.python import failure, log, reflect
+from twisted.python.compat import cmp, comparable
+from twisted.python.components import registerAdapter
+from twisted.spread import banana
+
+# These three are backwards compatibility aliases for the previous three.
+# Ultimately they should be deprecated. -exarkun
+from twisted.spread.flavors import (
+    Cacheable,
+    Copyable,
+    IPBRoot,
+    Jellyable,
+    NoSuchMethod,
+    Referenceable,
+    RemoteCache,
+    RemoteCacheObserver,
+    RemoteCopy,
+    Root,
+    Serializable,
+    Viewable,
+    ViewPoint,
+    copyTags,
+    setCopierForClass,
+    setCopierForClassTree,
+    setFactoryForClass,
+    setUnjellyableFactoryForClass,
+    setUnjellyableForClass,
+    setUnjellyableForClassTree,
+)
+from twisted.spread.interfaces import IJellyable, IUnjellyable
+from twisted.spread.jelly import _newInstance, globalSecurity, jelly, unjelly
+
+MAX_BROKER_REFS = 1024
+
+portno = 8787
+
+
+class ProtocolError(Exception):
+    """
+    This error is raised when an invalid protocol statement is received.
+    """
+
+
+class DeadReferenceError(ProtocolError):
+    """
+    This error is raised when a method is called on a dead reference (one whose
+    broker has been disconnected).
+    """
+
+
+class Error(Exception):
+    """
+    This error can be raised to generate known error conditions.
+
+    When a PB callable method (perspective_, remote_, view_) raises
+    this error, it indicates that a traceback should not be printed,
+    but instead, the string representation of the exception should be
+    sent.
+    """
+
+
+class RemoteError(Exception):
+    """
+    This class is used to wrap a string-ified exception from the remote side to
+    be able to reraise it. (Raising string exceptions is no longer possible in
+    Python 2.6+)
+
+    The value of this exception will be a str() representation of the remote
+    value.
+
+    @ivar remoteType: The full import path of the exception class which was
+        raised on the remote end.
+    @type remoteType: C{str}
+
+    @ivar remoteTraceback: The remote traceback.
+    @type remoteTraceback: C{str}
+
+    @note: It's not possible to include the remoteTraceback if this exception is
+        thrown into a generator. It must be accessed as an attribute.
+    """
+
+    def __init__(self, remoteType, value, remoteTraceback):
+        Exception.__init__(self, value)
+        self.remoteType = remoteType
+        self.remoteTraceback = remoteTraceback
+
+
+@comparable
+class RemoteMethod:
+    """
+    This is a translucent reference to a remote message.
+    """
+
+    def __init__(self, obj, name):
+        """
+        Initialize with a L{RemoteReference} and the name of this message.
+        """
+        self.obj = obj
+        self.name = name
+
+    def __cmp__(self, other):
+        return cmp((self.obj, self.name), other)
+
+    def __hash__(self):
+        return hash((self.obj, self.name))
+
+    def __call__(self, *args, **kw):
+        """
+        Asynchronously invoke a remote method.
+        """
+        return self.obj.broker._sendMessage(
+            b"",
+            self.obj.perspective,
+            self.obj.luid,
+            self.name.encode("utf-8"),
+            args,
+            kw,
+        )
+
+
+class PBConnectionLost(Exception):
+    pass
+
+
+class IPerspective(Interface):
+    """
+    per*spec*tive, n. : The relationship of aspects of a subject to each
+    other and to a whole: 'a perspective of history'; 'a need to view
+    the problem in the proper perspective'.
+
+    This is a Perspective Broker-specific wrapper for an avatar. That
+    is to say, a PB-published view on to the business logic for the
+    system's concept of a 'user'.
+
+    The concept of attached/detached is no longer implemented by the
+    framework. The realm is expected to implement such semantics if
+    needed.
+    """
+
+    def perspectiveMessageReceived(broker, message, args, kwargs):
+        """
+        This method is called when a network message is received.
+
+        @arg broker: The Perspective Broker.
+
+        @type message: str
+        @arg message: The name of the method called by the other end.
+
+        @type args: list in jelly format
+        @arg args: The arguments that were passed by the other end. It
+                   is recommend that you use the `unserialize' method of the
+                   broker to decode this.
+
+        @type kwargs: dict in jelly format
+        @arg kwargs: The keyword arguments that were passed by the
+                     other end.  It is recommended that you use the
+                     `unserialize' method of the broker to decode this.
+
+        @rtype: A jelly list.
+        @return: It is recommended that you use the `serialize' method
+                 of the broker on whatever object you need to return to
+                 generate the return value.
+        """
+
+
+@implementer(IPerspective)
+class Avatar:
+    """
+    A default IPerspective implementor.
+
+    This class is intended to be subclassed, and a realm should return
+    an instance of such a subclass when IPerspective is requested of
+    it.
+
+    A peer requesting a perspective will receive only a
+    L{RemoteReference} to a pb.Avatar.  When a method is called on
+    that L{RemoteReference}, it will translate to a method on the
+    remote perspective named 'perspective_methodname'.  (For more
+    information on invoking methods on other objects, see
+    L{flavors.ViewPoint}.)
+    """
+
+    def perspectiveMessageReceived(self, broker, message, args, kw):
+        """
+        This method is called when a network message is received.
+
+        This will call::
+
+            self.perspective_%(message)s(*broker.unserialize(args),
+                                         **broker.unserialize(kw))
+
+        to handle the method; subclasses of Avatar are expected to
+        implement methods using this naming convention.
+        """
+
+        args = broker.unserialize(args, self)
+        kw = broker.unserialize(kw, self)
+        method = getattr(self, "perspective_%s" % message)
+        try:
+            state = method(*args, **kw)
+        except TypeError:
+            log.msg(f"{method} didn't accept {args} and {kw}")
+            raise
+        return broker.serialize(state, self, method, args, kw)
+
+
+class AsReferenceable(Referenceable):
+    """
+    A reference directed towards another object.
+    """
+
+    def __init__(self, object, messageType="remote"):
+        self.remoteMessageReceived = getattr(object, messageType + "MessageReceived")
+
+
+@implementer(IUnjellyable)
+@comparable
+class RemoteReference(Serializable, styles.Ephemeral):
+    """
+    A translucent reference to a remote object.
+
+    I may be a reference to a L{flavors.ViewPoint}, a
+    L{flavors.Referenceable}, or an L{IPerspective} implementer (e.g.,
+    pb.Avatar).  From the client's perspective, it is not possible to
+    tell which except by convention.
+
+    I am a \"translucent\" reference because although no additional
+    bookkeeping overhead is given to the application programmer for
+    manipulating a reference, return values are asynchronous.
+
+    See also L{twisted.internet.defer}.
+
+    @ivar broker: The broker I am obtained through.
+    @type broker: L{Broker}
+    """
+
+    def __init__(self, perspective, broker, luid, doRefCount):
+        """(internal) Initialize me with a broker and a locally-unique ID.
+
+        The ID is unique only to the particular Perspective Broker
+        instance.
+        """
+        self.luid = luid
+        self.broker = broker
+        self.doRefCount = doRefCount
+        self.perspective = perspective
+        self.disconnectCallbacks = []
+
+    def notifyOnDisconnect(self, callback):
+        """
+        Register a callback to be called if our broker gets disconnected.
+
+        @param callback: a callable which will be called with one
+                         argument, this instance.
+        """
+        assert callable(callback)
+        self.disconnectCallbacks.append(callback)
+        if len(self.disconnectCallbacks) == 1:
+            self.broker.notifyOnDisconnect(self._disconnected)
+
+    def dontNotifyOnDisconnect(self, callback):
+        """
+        Remove a callback that was registered with notifyOnDisconnect.
+
+        @param callback: a callable
+        """
+        self.disconnectCallbacks.remove(callback)
+        if not self.disconnectCallbacks:
+            self.broker.dontNotifyOnDisconnect(self._disconnected)
+
+    def _disconnected(self):
+        """
+        Called if we are disconnected and have callbacks registered.
+        """
+        for callback in self.disconnectCallbacks:
+            callback(self)
+        self.disconnectCallbacks = None
+
+    def jellyFor(self, jellier):
+        """
+        If I am being sent back to where I came from, serialize as a local backreference.
+        """
+        if jellier.invoker:
+            assert (
+                self.broker == jellier.invoker
+            ), "Can't send references to brokers other than their own."
+            return b"local", self.luid
+        else:
+            return b"unpersistable", "References cannot be serialized"
+
+    def unjellyFor(self, unjellier, unjellyList):
+        self.__init__(
+            unjellier.invoker.unserializingPerspective,
+            unjellier.invoker,
+            unjellyList[1],
+            1,
+        )
+        return self
+
+    def callRemote(self, _name, *args, **kw):
+        """
+        Asynchronously invoke a remote method.
+
+        @type _name: L{str}
+        @param _name:  the name of the remote method to invoke
+        @param args: arguments to serialize for the remote function
+        @param kw:  keyword arguments to serialize for the remote function.
+        @rtype:   L{twisted.internet.defer.Deferred}
+        @returns: a Deferred which will be fired when the result of
+                  this remote call is received.
+        """
+        if not isinstance(_name, bytes):
+            _name = _name.encode("utf8")
+
+        # Note that we use '_name' instead of 'name' so the user can call
+        # remote methods with 'name' as a keyword parameter, like this:
+        #  ref.callRemote("getPeopleNamed", count=12, name="Bob")
+        return self.broker._sendMessage(
+            b"", self.perspective, self.luid, _name, args, kw
+        )
+
+    def remoteMethod(self, key):
+        """
+
+        @param key: The key.
+        @return: A L{RemoteMethod} for this key.
+        """
+        return RemoteMethod(self, key)
+
+    def __cmp__(self, other):
+        """
+
+        @param other: another L{RemoteReference} to compare me to.
+        """
+        if isinstance(other, RemoteReference):
+            if other.broker == self.broker:
+                return cmp(self.luid, other.luid)
+        return cmp(self.broker, other)
+
+    def __hash__(self):
+        """
+        Hash me.
+        """
+        return self.luid
+
+    def __del__(self):
+        """
+        Do distributed reference counting on finalization.
+        """
+        if self.doRefCount:
+            self.broker.sendDecRef(self.luid)
+
+
+setUnjellyableForClass("remote", RemoteReference)
+
+
+class Local:
+    """
+    (internal) A reference to a local object.
+    """
+
+    def __init__(self, object, perspective=None):
+        """
+        Initialize.
+        """
+        self.object = object
+        self.perspective = perspective
+        self.refcount = 1
+
+    def __repr__(self) -> str:
+        return f"<pb.Local {self.object!r} ref:{self.refcount}>"
+
+    def incref(self):
+        """
+        Increment the reference count.
+
+        @return: the reference count after incrementing
+        """
+        self.refcount = self.refcount + 1
+        return self.refcount
+
+    def decref(self):
+        """
+        Decrement the reference count.
+
+        @return: the reference count after decrementing
+        """
+        self.refcount = self.refcount - 1
+        return self.refcount
+
+
+# Failure
+class CopyableFailure(failure.Failure, Copyable):
+    """
+    A L{flavors.RemoteCopy} and L{flavors.Copyable} version of
+    L{twisted.python.failure.Failure} for serialization.
+    """
+
+    unsafeTracebacks = 0
+
+    def getStateToCopy(self):
+        """
+        Collect state related to the exception which occurred, discarding
+        state which cannot reasonably be serialized.
+        """
+        state = self.__dict__.copy()
+        state["tb"] = None
+        state["frames"] = []
+        state["stack"] = []
+        state["value"] = str(self.value)  # Exception instance
+        if isinstance(self.type, bytes):
+            state["type"] = self.type
+        else:
+            state["type"] = reflect.qual(self.type).encode("utf-8")  # Exception class
+        if self.unsafeTracebacks:
+            state["traceback"] = self.getTraceback()
+        else:
+            state["traceback"] = "Traceback unavailable\n"
+        return state
+
+
+class CopiedFailure(RemoteCopy, failure.Failure):
+    """
+    A L{CopiedFailure} is a L{pb.RemoteCopy} of a L{failure.Failure}
+    transferred via PB.
+
+    @ivar type: The full import path of the exception class which was raised on
+        the remote end.
+    @type type: C{str}
+
+    @ivar value: A str() representation of the remote value.
+    @type value: L{CopiedFailure} or C{str}
+
+    @ivar traceback: The remote traceback.
+    @type traceback: C{str}
+    """
+
+    def printTraceback(self, file=None, elideFrameworkCode=0, detail="default"):
+        if file is None:
+            file = log.logfile
+        failureType = self.type
+        if not isinstance(failureType, str):
+            failureType = failureType.decode("utf-8")
+        file.write("Traceback from remote host -- ")
+        file.write(failureType + ": " + self.value)
+        file.write("\n")
+
+    def throwExceptionIntoGenerator(self, g):
+        """
+        Throw the original exception into the given generator, preserving
+        traceback information if available. In the case of a L{CopiedFailure}
+        where the exception type is a string, a L{pb.RemoteError} is thrown
+        instead.
+
+        @return: The next value yielded from the generator.
+        @raise StopIteration: If there are no more values in the generator.
+        @raise RemoteError: The wrapped remote exception.
+        """
+        return g.throw(RemoteError(self.type, self.value, self.traceback))
+
+    printBriefTraceback = printTraceback
+    printDetailedTraceback = printTraceback
+
+
+setUnjellyableForClass(CopyableFailure, CopiedFailure)
+
+
+def failure2Copyable(fail, unsafeTracebacks=0):
+    f = _newInstance(CopyableFailure, fail.__dict__)
+    f.unsafeTracebacks = unsafeTracebacks
+    return f
+
+
+class Broker(banana.Banana):
+    """
+    I am a broker for objects.
+    """
+
+    version = 6
+    username = None
+    factory = None
+
+    def __init__(self, isClient=1, security=globalSecurity):
+        banana.Banana.__init__(self, isClient)
+        self.disconnected = 0
+        self.disconnects = []
+        self.failures = []
+        self.connects = []
+        self.localObjects = {}
+        self.security = security
+        self.pageProducers = []
+        self.currentRequestID = 0
+        self.currentLocalID = 0
+        self.unserializingPerspective = None
+        # Some terms:
+        #  PUID: process unique ID; return value of id() function.  type "int".
+        #  LUID: locally unique ID; an ID unique to an object mapped over this
+        #        connection. type "int"
+        #  GUID: (not used yet) globally unique ID; an ID for an object which
+        #        may be on a redirected or meta server.  Type as yet undecided.
+        # Dictionary mapping LUIDs to local objects.
+        # set above to allow root object to be assigned before connection is made
+        # self.localObjects = {}
+        # Dictionary mapping PUIDs to LUIDs.
+        self.luids = {}
+        # Dictionary mapping LUIDs to local (remotely cached) objects. Remotely
+        # cached means that they're objects which originate here, and were
+        # copied remotely.
+        self.remotelyCachedObjects = {}
+        # Dictionary mapping PUIDs to (cached) LUIDs
+        self.remotelyCachedLUIDs = {}
+        # Dictionary mapping (remote) LUIDs to (locally cached) objects.
+        self.locallyCachedObjects = {}
+        self.waitingForAnswers = {}
+
+        # Mapping from LUIDs to weakref objects with callbacks for performing
+        # any local cleanup which may be necessary for the corresponding
+        # object once it no longer exists.
+        self._localCleanup = {}
+
+    def resumeProducing(self):
+        """
+        Called when the consumer attached to me runs out of buffer.
+        """
+        # Go backwards over the list so we can remove indexes from it as we go
+        for pageridx in range(len(self.pageProducers) - 1, -1, -1):
+            pager = self.pageProducers[pageridx]
+            pager.sendNextPage()
+            if not pager.stillPaging():
+                del self.pageProducers[pageridx]
+        if not self.pageProducers:
+            self.transport.unregisterProducer()
+
+    def pauseProducing(self):
+        # Streaming producer method; not necessary to implement.
+        pass
+
+    def stopProducing(self):
+        # Streaming producer method; not necessary to implement.
+        pass
+
+    def registerPageProducer(self, pager):
+        self.pageProducers.append(pager)
+        if len(self.pageProducers) == 1:
+            self.transport.registerProducer(self, 0)
+
+    def expressionReceived(self, sexp):
+        """
+        Evaluate an expression as it's received.
+        """
+        if isinstance(sexp, list):
+            command = sexp[0]
+
+            if not isinstance(command, str):
+                command = command.decode("utf8")
+
+            methodName = "proto_%s" % command
+            method = getattr(self, methodName, None)
+
+            if method:
+                method(*sexp[1:])
+            else:
+                self.sendCall(b"didNotUnderstand", command)
+        else:
+            raise ProtocolError("Non-list expression received.")
+
+    def proto_version(self, vnum):
+        """
+        Protocol message: (version version-number)
+
+        Check to make sure that both ends of the protocol are speaking
+        the same version dialect.
+
+        @param vnum: The version number.
+        """
+
+        if vnum != self.version:
+            raise ProtocolError(f"Version Incompatibility: {self.version} {vnum}")
+
+    def sendCall(self, *exp):
+        """
+        Utility method to send an expression to the other side of the connection.
+
+        @param exp: The expression.
+        """
+        self.sendEncoded(exp)
+
+    def proto_didNotUnderstand(self, command):
+        """
+        Respond to stock 'C{didNotUnderstand}' message.
+
+        Log the command that was not understood and continue. (Note:
+        this will probably be changed to close the connection or raise
+        an exception in the future.)
+
+        @param command: The command to log.
+        """
+        log.msg("Didn't understand command: %r" % command)
+
+    def connectionReady(self):
+        """
+        Initialize. Called after Banana negotiation is done.
+        """
+        self.sendCall(b"version", self.version)
+        for notifier in self.connects:
+            try:
+                notifier()
+            except BaseException:
+                log.deferr()
+        self.connects = None
+        self.factory.clientConnectionMade(self)
+
+    def connectionFailed(self):
+        # XXX should never get called anymore? check!
+        for notifier in self.failures:
+            try:
+                notifier()
+            except BaseException:
+                log.deferr()
+        self.failures = None
+
+    waitingForAnswers = None
+
+    def connectionLost(self, reason):
+        """
+        The connection was lost.
+
+        @param reason: message to put in L{failure.Failure}
+        """
+        self.disconnected = 1
+        # Nuke potential circular references.
+        self.luids = None
+        if self.waitingForAnswers:
+            for d in self.waitingForAnswers.values():
+                try:
+                    d.errback(failure.Failure(PBConnectionLost(reason)))
+                except BaseException:
+                    log.deferr()
+        # Assure all Cacheable.stoppedObserving are called
+        for lobj in self.remotelyCachedObjects.values():
+            cacheable = lobj.object
+            perspective = lobj.perspective
+            try:
+                cacheable.stoppedObserving(
+                    perspective, RemoteCacheObserver(self, cacheable, perspective)
+                )
+            except BaseException:
+                log.deferr()
+        # Loop on a copy to prevent notifiers to mixup
+        # the list by calling dontNotifyOnDisconnect
+        for notifier in self.disconnects[:]:
+            try:
+                notifier()
+            except BaseException:
+                log.deferr()
+        self.disconnects = None
+        self.waitingForAnswers = None
+        self.localSecurity = None
+        self.remoteSecurity = None
+        self.remotelyCachedObjects = None
+        self.remotelyCachedLUIDs = None
+        self.locallyCachedObjects = None
+        self.localObjects = None
+
+    def notifyOnDisconnect(self, notifier):
+        """
+
+        @param notifier: callback to call when the Broker disconnects.
+        """
+        assert callable(notifier)
+        self.disconnects.append(notifier)
+
+    def notifyOnFail(self, notifier):
+        """
+
+        @param notifier: callback to call if the Broker fails to connect.
+        """
+        assert callable(notifier)
+        self.failures.append(notifier)
+
+    def notifyOnConnect(self, notifier):
+        """
+
+        @param notifier: callback to call when the Broker connects.
+        """
+        assert callable(notifier)
+        if self.connects is None:
+            try:
+                notifier()
+            except BaseException:
+                log.err()
+        else:
+            self.connects.append(notifier)
+
+    def dontNotifyOnDisconnect(self, notifier):
+        """
+
+        @param notifier: callback to remove from list of disconnect callbacks.
+        """
+        try:
+            self.disconnects.remove(notifier)
+        except ValueError:
+            pass
+
+    def localObjectForID(self, luid):
+        """
+        Get a local object for a locally unique ID.
+
+        @return: An object previously stored with L{registerReference} or
+            L{None} if there is no object which corresponds to the given
+            identifier.
+        """
+        if isinstance(luid, str):
+            luid = luid.encode("utf8")
+
+        lob = self.localObjects.get(luid)
+        if lob is None:
+            return
+        return lob.object
+
+    maxBrokerRefsViolations = 0
+
+    def registerReference(self, object):
+        """
+        Store a persistent reference to a local object and map its
+        id() to a generated, session-unique ID.
+
+        @param object: a local object
+        @return: the generated ID
+        """
+
+        assert object is not None
+        puid = object.processUniqueID()
+        luid = self.luids.get(puid)
+        if luid is None:
+            if len(self.localObjects) > MAX_BROKER_REFS:
+                self.maxBrokerRefsViolations = self.maxBrokerRefsViolations + 1
+                if self.maxBrokerRefsViolations > 3:
+                    self.transport.loseConnection()
+                    raise Error("Maximum PB reference count exceeded.  " "Goodbye.")
+                raise Error("Maximum PB reference count exceeded.")
+
+            luid = self.newLocalID()
+            self.localObjects[luid] = Local(object)
+            self.luids[puid] = luid
+        else:
+            self.localObjects[luid].incref()
+        return luid
+
+    def setNameForLocal(self, name, object):
+        """
+        Store a special (string) ID for this object.
+
+        This is how you specify a 'base' set of objects that the remote
+        protocol can connect to.
+
+        @param name: An ID.
+        @param object: The object.
+        """
+        if isinstance(name, str):
+            name = name.encode("utf8")
+
+        assert object is not None
+        self.localObjects[name] = Local(object)
+
+    def remoteForName(self, name):
+        """
+        Returns an object from the remote name mapping.
+
+        Note that this does not check the validity of the name, only
+        creates a translucent reference for it.
+
+        @param name: The name to look up.
+        @return: An object which maps to the name.
+        """
+        if isinstance(name, str):
+            name = name.encode("utf8")
+
+        return RemoteReference(None, self, name, 0)
+
+    def cachedRemotelyAs(self, instance, incref=0):
+        """
+
+        @param instance: The instance to look up.
+        @param incref: Flag to specify whether to increment the
+                       reference.
+        @return: An ID that says what this instance is cached as
+                 remotely, or L{None} if it's not.
+        """
+
+        puid = instance.processUniqueID()
+        luid = self.remotelyCachedLUIDs.get(puid)
+        if (luid is not None) and (incref):
+            self.remotelyCachedObjects[luid].incref()
+        return luid
+
+    def remotelyCachedForLUID(self, luid):
+        """
+
+        @param luid: The LUID to look up.
+        @return: An instance which is cached remotely.
+        """
+        return self.remotelyCachedObjects[luid].object
+
+    def cacheRemotely(self, instance):
+        """
+        XXX
+
+        @return: A new LUID.
+        """
+        puid = instance.processUniqueID()
+        luid = self.newLocalID()
+        if len(self.remotelyCachedObjects) > MAX_BROKER_REFS:
+            self.maxBrokerRefsViolations = self.maxBrokerRefsViolations + 1
+            if self.maxBrokerRefsViolations > 3:
+                self.transport.loseConnection()
+                raise Error("Maximum PB cache count exceeded.  " "Goodbye.")
+            raise Error("Maximum PB cache count exceeded.")
+
+        self.remotelyCachedLUIDs[puid] = luid
+        # This table may not be necessary -- for now, it's to make sure that no
+        # monkey business happens with id(instance)
+        self.remotelyCachedObjects[luid] = Local(instance, self.serializingPerspective)
+        return luid
+
+    def cacheLocally(self, cid, instance):
+        """(internal)
+
+        Store a non-filled-out cached instance locally.
+        """
+        self.locallyCachedObjects[cid] = instance
+
+    def cachedLocallyAs(self, cid):
+        instance = self.locallyCachedObjects[cid]
+        return instance
+
+    def serialize(self, object, perspective=None, method=None, args=None, kw=None):
+        """
+        Jelly an object according to the remote security rules for this broker.
+
+        @param object: The object to jelly.
+        @param perspective: The perspective.
+        @param method: The method.
+        @param args: Arguments.
+        @param kw: Keyword arguments.
+        """
+
+        if isinstance(object, defer.Deferred):
+            object.addCallbacks(
+                self.serialize,
+                lambda x: x,
+                callbackKeywords={
+                    "perspective": perspective,
+                    "method": method,
+                    "args": args,
+                    "kw": kw,
+                },
+            )
+            return object
+
+        # XXX This call is NOT REENTRANT and testing for reentrancy is just
+        # crazy, so it likely won't be.  Don't ever write methods that call the
+        # broker's serialize() method recursively (e.g. sending a method call
+        # from within a getState (this causes concurrency problems anyway so
+        # you really, really shouldn't do it))
+
+        self.serializingPerspective = perspective
+        self.jellyMethod = method
+        self.jellyArgs = args
+        self.jellyKw = kw
+        try:
+            return jelly(object, self.security, None, self)
+        finally:
+            self.serializingPerspective = None
+            self.jellyMethod = None
+            self.jellyArgs = None
+            self.jellyKw = None
+
+    def unserialize(self, sexp, perspective=None):
+        """
+        Unjelly an sexp according to the local security rules for this broker.
+
+        @param sexp: The object to unjelly.
+        @param perspective: The perspective.
+        """
+
+        self.unserializingPerspective = perspective
+        try:
+            return unjelly(sexp, self.security, None, self)
+        finally:
+            self.unserializingPerspective = None
+
+    def newLocalID(self):
+        """
+
+        @return: A newly generated LUID.
+        """
+        self.currentLocalID = self.currentLocalID + 1
+        return self.currentLocalID
+
+    def newRequestID(self):
+        """
+
+        @return: A newly generated request ID.
+        """
+        self.currentRequestID = self.currentRequestID + 1
+        return self.currentRequestID
+
+    def _sendMessage(self, prefix, perspective, objectID, message, args, kw):
+        pbc = None
+        pbe = None
+        answerRequired = 1
+        if "pbcallback" in kw:
+            pbc = kw["pbcallback"]
+            del kw["pbcallback"]
+        if "pberrback" in kw:
+            pbe = kw["pberrback"]
+            del kw["pberrback"]
+        if "pbanswer" in kw:
+            assert (not pbe) and (not pbc), "You can't specify a no-answer requirement."
+            answerRequired = kw["pbanswer"]
+            del kw["pbanswer"]
+        if self.disconnected:
+            raise DeadReferenceError("Calling Stale Broker")
+        try:
+            netArgs = self.serialize(args, perspective=perspective, method=message)
+            netKw = self.serialize(kw, perspective=perspective, method=message)
+        except BaseException:
+            return defer.fail(failure.Failure())
+        requestID = self.newRequestID()
+        if answerRequired:
+            rval = defer.Deferred()
+            self.waitingForAnswers[requestID] = rval
+            if pbc or pbe:
+                log.msg('warning! using deprecated "pbcallback"')
+                rval.addCallbacks(pbc, pbe)
+        else:
+            rval = None
+        self.sendCall(
+            prefix + b"message",
+            requestID,
+            objectID,
+            message,
+            answerRequired,
+            netArgs,
+            netKw,
+        )
+        return rval
+
+    def proto_message(
+        self, requestID, objectID, message, answerRequired, netArgs, netKw
+    ):
+        self._recvMessage(
+            self.localObjectForID,
+            requestID,
+            objectID,
+            message,
+            answerRequired,
+            netArgs,
+            netKw,
+        )
+
+    def proto_cachemessage(
+        self, requestID, objectID, message, answerRequired, netArgs, netKw
+    ):
+        self._recvMessage(
+            self.cachedLocallyAs,
+            requestID,
+            objectID,
+            message,
+            answerRequired,
+            netArgs,
+            netKw,
+        )
+
+    def _recvMessage(
+        self,
+        findObjMethod,
+        requestID,
+        objectID,
+        message,
+        answerRequired,
+        netArgs,
+        netKw,
+    ):
+        """
+        Received a message-send.
+
+        Look up message based on object, unserialize the arguments, and
+        invoke it with args, and send an 'answer' or 'error' response.
+
+        @param findObjMethod: A callable which takes C{objectID} as argument.
+        @param requestID: The requiest ID.
+        @param objectID: The object ID.
+        @param message: The message.
+        @param answerRequired:
+        @param netArgs: Arguments.
+        @param netKw: Keyword arguments.
+        """
+        if not isinstance(message, str):
+            message = message.decode("utf8")
+
+        try:
+            object = findObjMethod(objectID)
+            if object is None:
+                raise Error("Invalid Object ID")
+            netResult = object.remoteMessageReceived(self, message, netArgs, netKw)
+        except Error as e:
+            if answerRequired:
+                # If the error is Jellyable or explicitly allowed via our
+                # security options, send it back and let the code on the
+                # other end deal with unjellying.  If it isn't Jellyable,
+                # wrap it in a CopyableFailure, which ensures it can be
+                # unjellied on the other end.  We have to do this because
+                # all errors must be sent back.
+                if isinstance(e, Jellyable) or self.security.isClassAllowed(
+                    e.__class__
+                ):
+                    self._sendError(e, requestID)
+                else:
+                    self._sendError(CopyableFailure(e), requestID)
+        except BaseException:
+            if answerRequired:
+                log.msg("Peer will receive following PB traceback:", isError=True)
+                f = CopyableFailure()
+                self._sendError(f, requestID)
+            log.err()
+        else:
+            if answerRequired:
+                if isinstance(netResult, defer.Deferred):
+                    args = (requestID,)
+                    netResult.addCallbacks(
+                        self._sendAnswer,
+                        self._sendFailureOrError,
+                        callbackArgs=args,
+                        errbackArgs=args,
+                    )
+                    # XXX Should this be done somewhere else?
+                else:
+                    self._sendAnswer(netResult, requestID)
+
+    def _sendAnswer(self, netResult, requestID):
+        """
+        (internal) Send an answer to a previously sent message.
+
+        @param netResult: The answer.
+        @param requestID: The request ID.
+        """
+        self.sendCall(b"answer", requestID, netResult)
+
+    def proto_answer(self, requestID, netResult):
+        """
+        (internal) Got an answer to a previously sent message.
+
+        Look up the appropriate callback and call it.
+
+        @param requestID: The request ID.
+        @param netResult: The answer.
+        """
+        d = self.waitingForAnswers[requestID]
+        del self.waitingForAnswers[requestID]
+        d.callback(self.unserialize(netResult))
+
+    def _sendFailureOrError(self, fail, requestID):
+        """
+        Call L{_sendError} or L{_sendFailure}, depending on whether C{fail}
+        represents an L{Error} subclass or not.
+
+        @param fail: The failure.
+        @param requestID: The request ID.
+        """
+        if fail.check(Error) is None:
+            self._sendFailure(fail, requestID)
+        else:
+            self._sendError(fail, requestID)
+
+    def _sendFailure(self, fail, requestID):
+        """
+        Log error and then send it.
+
+        @param fail: The failure.
+        @param requestID: The request ID.
+        """
+        log.msg("Peer will receive following PB traceback:")
+        log.err(fail)
+        self._sendError(fail, requestID)
+
+    def _sendError(self, fail, requestID):
+        """
+        (internal) Send an error for a previously sent message.
+
+        @param fail: The failure.
+        @param requestID: The request ID.
+        """
+        if isinstance(fail, failure.Failure):
+            # If the failures value is jellyable or allowed through security,
+            # send the value
+            if isinstance(fail.value, Jellyable) or self.security.isClassAllowed(
+                fail.value.__class__
+            ):
+                fail = fail.value
+            elif not isinstance(fail, CopyableFailure):
+                fail = failure2Copyable(fail, self.factory.unsafeTracebacks)
+        if isinstance(fail, CopyableFailure):
+            fail.unsafeTracebacks = self.factory.unsafeTracebacks
+        self.sendCall(b"error", requestID, self.serialize(fail))
+
+    def proto_error(self, requestID, fail):
+        """
+        (internal) Deal with an error.
+
+        @param requestID: The request ID.
+        @param fail: The failure.
+        """
+        d = self.waitingForAnswers[requestID]
+        del self.waitingForAnswers[requestID]
+        d.errback(self.unserialize(fail))
+
+    def sendDecRef(self, objectID):
+        """
+        (internal) Send a DECREF directive.
+
+        @param objectID: The object ID.
+        """
+        self.sendCall(b"decref", objectID)
+
+    def proto_decref(self, objectID):
+        """
+        (internal) Decrement the reference count of an object.
+
+        If the reference count is zero, it will free the reference to this
+        object.
+
+        @param objectID: The object ID.
+        """
+        if isinstance(objectID, str):
+            objectID = objectID.encode("utf8")
+        refs = self.localObjects[objectID].decref()
+        if refs == 0:
+            puid = self.localObjects[objectID].object.processUniqueID()
+            del self.luids[puid]
+            del self.localObjects[objectID]
+            self._localCleanup.pop(puid, lambda: None)()
+
+    def decCacheRef(self, objectID):
+        """
+        (internal) Send a DECACHE directive.
+
+        @param objectID: The object ID.
+        """
+        self.sendCall(b"decache", objectID)
+
+    def proto_decache(self, objectID):
+        """
+        (internal) Decrement the reference count of a cached object.
+
+        If the reference count is zero, free the reference, then send an
+        'uncached' directive.
+
+        @param objectID: The object ID.
+        """
+        refs = self.remotelyCachedObjects[objectID].decref()
+        # log.msg('decaching: %s #refs: %s' % (objectID, refs))
+        if refs == 0:
+            lobj = self.remotelyCachedObjects[objectID]
+            cacheable = lobj.object
+            perspective = lobj.perspective
+            # TODO: force_decache needs to be able to force-invalidate a
+            # cacheable reference.
+            try:
+                cacheable.stoppedObserving(
+                    perspective, RemoteCacheObserver(self, cacheable, perspective)
+                )
+            except BaseException:
+                log.deferr()
+            puid = cacheable.processUniqueID()
+            del self.remotelyCachedLUIDs[puid]
+            del self.remotelyCachedObjects[objectID]
+            self.sendCall(b"uncache", objectID)
+
+    def proto_uncache(self, objectID):
+        """
+        (internal) Tell the client it is now OK to uncache an object.
+
+        @param objectID: The object ID.
+        """
+        # log.msg("uncaching locally %d" % objectID)
+        obj = self.locallyCachedObjects[objectID]
+        obj.broker = None
+        ##         def reallyDel(obj=obj):
+        ##             obj.__really_del__()
+        ##         obj.__del__ = reallyDel
+        del self.locallyCachedObjects[objectID]
+
+
+def respond(challenge, password):
+    """
+    Respond to a challenge.
+
+    This is useful for challenge/response authentication.
+
+    @param challenge: A challenge.
+    @param password: A password.
+    @return: The password hashed twice.
+    """
+    m = md5()
+    m.update(password)
+    hashedPassword = m.digest()
+    m = md5()
+    m.update(hashedPassword)
+    m.update(challenge)
+    doubleHashedPassword = m.digest()
+    return doubleHashedPassword
+
+
+def challenge():
+    """
+
+    @return: Some random data.
+    """
+    crap = bytes(random.randint(65, 90) for x in range(random.randrange(15, 25)))
+    crap = md5(crap).digest()
+    return crap
+
+
+class PBClientFactory(protocol.ClientFactory):
+    """
+    Client factory for PB brokers.
+
+    As with all client factories, use with reactor.connectTCP/SSL/etc..
+    getPerspective and getRootObject can be called either before or
+    after the connect.
+    """
+
+    protocol = Broker
+    unsafeTracebacks = False
+
+    def __init__(self, unsafeTracebacks=False, security=globalSecurity):
+        """
+        @param unsafeTracebacks: if set, tracebacks for exceptions will be sent
+            over the wire.
+        @type unsafeTracebacks: C{bool}
+
+        @param security: security options used by the broker, default to
+            C{globalSecurity}.
+        @type security: L{twisted.spread.jelly.SecurityOptions}
+        """
+        self.unsafeTracebacks = unsafeTracebacks
+        self.security = security
+        self._reset()
+
+    def buildProtocol(self, addr):
+        """
+        Build the broker instance, passing the security options to it.
+        """
+        p = self.protocol(isClient=True, security=self.security)
+        p.factory = self
+        return p
+
+    def _reset(self):
+        self.rootObjectRequests = []  # list of deferred
+        self._broker = None
+        self._root = None
+
+    def _failAll(self, reason):
+        deferreds = self.rootObjectRequests
+        self._reset()
+        for d in deferreds:
+            d.errback(reason)
+
+    def clientConnectionFailed(self, connector, reason):
+        self._failAll(reason)
+
+    def clientConnectionLost(self, connector, reason, reconnecting=0):
+        """
+        Reconnecting subclasses should call with reconnecting=1.
+        """
+        if reconnecting:
+            # Any pending requests will go to next connection attempt
+            # so we don't fail them.
+            self._broker = None
+            self._root = None
+        else:
+            self._failAll(reason)
+
+    def clientConnectionMade(self, broker):
+        self._broker = broker
+        self._root = broker.remoteForName("root")
+        ds = self.rootObjectRequests
+        self.rootObjectRequests = []
+        for d in ds:
+            d.callback(self._root)
+
+    def getRootObject(self):
+        """
+        Get root object of remote PB server.
+
+        @return: Deferred of the root object.
+        """
+        if self._broker and not self._broker.disconnected:
+            return defer.succeed(self._root)
+        d = defer.Deferred()
+        self.rootObjectRequests.append(d)
+        return d
+
+    def disconnect(self):
+        """
+        If the factory is connected, close the connection.
+
+        Note that if you set up the factory to reconnect, you will need to
+        implement extra logic to prevent automatic reconnection after this
+        is called.
+        """
+        if self._broker:
+            self._broker.transport.loseConnection()
+
+    def _cbSendUsername(self, root, username, password, client):
+        return root.callRemote("login", username).addCallback(
+            self._cbResponse, password, client
+        )
+
+    def _cbResponse(self, challenges, password, client):
+        challenge, challenger = challenges
+        return challenger.callRemote("respond", respond(challenge, password), client)
+
+    def _cbLoginAnonymous(self, root, client):
+        """
+        Attempt an anonymous login on the given remote root object.
+
+        @type root: L{RemoteReference}
+        @param root: The object on which to attempt the login, most likely
+            returned by a call to L{PBClientFactory.getRootObject}.
+
+        @param client: A jellyable object which will be used as the I{mind}
+            parameter for the login attempt.
+
+        @rtype: L{Deferred}
+        @return: A L{Deferred} which will be called back with a
+            L{RemoteReference} to an avatar when anonymous login succeeds, or
+            which will errback if anonymous login fails.
+        """
+        return root.callRemote("loginAnonymous", client)
+
+    def login(self, credentials, client=None):
+        """
+        Login and get perspective from remote PB server.
+
+        Currently the following credentials are supported::
+
+            L{twisted.cred.credentials.IUsernamePassword}
+            L{twisted.cred.credentials.IAnonymous}
+
+        @rtype: L{Deferred}
+        @return: A L{Deferred} which will be called back with a
+            L{RemoteReference} for the avatar logged in to, or which will
+            errback if login fails.
+        """
+        d = self.getRootObject()
+
+        if IAnonymous.providedBy(credentials):
+            d.addCallback(self._cbLoginAnonymous, client)
+        else:
+            d.addCallback(
+                self._cbSendUsername, credentials.username, credentials.password, client
+            )
+        return d
+
+
+class PBServerFactory(protocol.ServerFactory):
+    """
+    Server factory for perspective broker.
+
+    Login is done using a Portal object, whose realm is expected to return
+    avatars implementing IPerspective. The credential checkers in the portal
+    should accept IUsernameHashedPassword or IUsernameMD5Password.
+
+    Alternatively, any object providing or adaptable to L{IPBRoot} can be
+    used instead of a portal to provide the root object of the PB server.
+    """
+
+    unsafeTracebacks = False
+
+    # object broker factory
+    protocol = Broker
+
+    def __init__(self, root, unsafeTracebacks=False, security=globalSecurity):
+        """
+        @param root: factory providing the root Referenceable used by the broker.
+        @type root: object providing or adaptable to L{IPBRoot}.
+
+        @param unsafeTracebacks: if set, tracebacks for exceptions will be sent
+            over the wire.
+        @type unsafeTracebacks: C{bool}
+
+        @param security: security options used by the broker, default to
+            C{globalSecurity}.
+        @type security: L{twisted.spread.jelly.SecurityOptions}
+        """
+        self.root = IPBRoot(root)
+        self.unsafeTracebacks = unsafeTracebacks
+        self.security = security
+
+    def buildProtocol(self, addr):
+        """
+        Return a Broker attached to the factory (as the service provider).
+        """
+        proto = self.protocol(isClient=False, security=self.security)
+        proto.factory = self
+        proto.setNameForLocal("root", self.root.rootObject(proto))
+        return proto
+
+    def clientConnectionMade(self, protocol):
+        # XXX does this method make any sense?
+        pass
+
+
+class IUsernameMD5Password(ICredentials):
+    """
+    I encapsulate a username and a hashed password.
+
+    This credential is used for username/password over PB. CredentialCheckers
+    which check this kind of credential must store the passwords in plaintext
+    form or as a MD5 digest.
+
+    @type username: C{str} or C{Deferred}
+    @ivar username: The username associated with these credentials.
+    """
+
+    def checkPassword(password):
+        """
+        Validate these credentials against the correct password.
+
+        @type password: C{str}
+        @param password: The correct, plaintext password against which to
+            check.
+
+        @rtype: C{bool} or L{Deferred}
+        @return: C{True} if the credentials represented by this object match the
+            given password, C{False} if they do not, or a L{Deferred} which will
+            be called back with one of these values.
+        """
+
+    def checkMD5Password(password):
+        """
+        Validate these credentials against the correct MD5 digest of the
+        password.
+
+        @type password: C{str}
+        @param password: The correct MD5 digest of a password against which to
+            check.
+
+        @rtype: C{bool} or L{Deferred}
+        @return: C{True} if the credentials represented by this object match the
+            given digest, C{False} if they do not, or a L{Deferred} which will
+            be called back with one of these values.
+        """
+
+
+@implementer(IPBRoot)
+class _PortalRoot:
+    """
+    Root object, used to login to portal.
+    """
+
+    def __init__(self, portal):
+        self.portal = portal
+
+    def rootObject(self, broker):
+        return _PortalWrapper(self.portal, broker)
+
+
+registerAdapter(_PortalRoot, Portal, IPBRoot)
+
+
+class _JellyableAvatarMixin:
+    """
+    Helper class for code which deals with avatars which PB must be capable of
+    sending to a peer.
+    """
+
+    def _cbLogin(self, result):
+        """
+        Ensure that the avatar to be returned to the client is jellyable and
+        set up disconnection notification to call the realm's logout object.
+        """
+        (interface, avatar, logout) = result
+        if not IJellyable.providedBy(avatar):
+            avatar = AsReferenceable(avatar, "perspective")
+
+        puid = avatar.processUniqueID()
+
+        # only call logout once, whether the connection is dropped (disconnect)
+        # or a logout occurs (cleanup), and be careful to drop the reference to
+        # it in either case
+        logout = [logout]
+
+        def maybeLogout():
+            if not logout:
+                return
+            fn = logout[0]
+            del logout[0]
+            fn()
+
+        self.broker._localCleanup[puid] = maybeLogout
+        self.broker.notifyOnDisconnect(maybeLogout)
+
+        return avatar
+
+
+class _PortalWrapper(Referenceable, _JellyableAvatarMixin):
+    """
+    Root Referenceable object, used to login to portal.
+    """
+
+    def __init__(self, portal, broker):
+        self.portal = portal
+        self.broker = broker
+
+    def remote_login(self, username):
+        """
+        Start of username/password login.
+
+        @param username: The username.
+        """
+        c = challenge()
+        return c, _PortalAuthChallenger(self.portal, self.broker, username, c)
+
+    def remote_loginAnonymous(self, mind):
+        """
+        Attempt an anonymous login.
+
+        @param mind: An object to use as the mind parameter to the portal login
+            call (possibly None).
+
+        @rtype: L{Deferred}
+        @return: A Deferred which will be called back with an avatar when login
+            succeeds or which will be errbacked if login fails somehow.
+        """
+        d = self.portal.login(Anonymous(), mind, IPerspective)
+        d.addCallback(self._cbLogin)
+        return d
+
+
+@implementer(IUsernameHashedPassword, IUsernameMD5Password)
+class _PortalAuthChallenger(Referenceable, _JellyableAvatarMixin):
+    """
+    Called with response to password challenge.
+    """
+
+    def __init__(self, portal, broker, username, challenge):
+        self.portal = portal
+        self.broker = broker
+        self.username = username
+        self.challenge = challenge
+
+    def remote_respond(self, response, mind):
+        self.response = response
+        d = self.portal.login(self, mind, IPerspective)
+        d.addCallback(self._cbLogin)
+        return d
+
+    def checkPassword(self, password):
+        """
+        L{IUsernameHashedPassword}
+
+        @param password: The password.
+        @return: L{_PortalAuthChallenger.checkMD5Password}
+        """
+        return self.checkMD5Password(md5(password).digest())
+
+    def checkMD5Password(self, md5Password):
+        """
+        L{IUsernameMD5Password}
+
+        @param md5Password:
+        @rtype: L{bool}
+        @return: L{True} if password matches.
+        """
+        md = md5()
+        md.update(md5Password)
+        md.update(self.challenge)
+        correct = md.digest()
+        return self.response == correct
+
+
+__all__ = [
+    # Everything from flavors is exposed publicly here.
+    "IPBRoot",
+    "Serializable",
+    "Referenceable",
+    "NoSuchMethod",
+    "Root",
+    "ViewPoint",
+    "Viewable",
+    "Copyable",
+    "Jellyable",
+    "Cacheable",
+    "RemoteCopy",
+    "RemoteCache",
+    "RemoteCacheObserver",
+    "copyTags",
+    "setUnjellyableForClass",
+    "setUnjellyableFactoryForClass",
+    "setUnjellyableForClassTree",
+    "setCopierForClass",
+    "setFactoryForClass",
+    "setCopierForClassTree",
+    "MAX_BROKER_REFS",
+    "portno",
+    "ProtocolError",
+    "DeadReferenceError",
+    "Error",
+    "PBConnectionLost",
+    "RemoteMethod",
+    "IPerspective",
+    "Avatar",
+    "AsReferenceable",
+    "RemoteReference",
+    "CopyableFailure",
+    "CopiedFailure",
+    "failure2Copyable",
+    "Broker",
+    "respond",
+    "challenge",
+    "PBClientFactory",
+    "PBServerFactory",
+    "IUsernameMD5Password",
+]
diff --git a/contrib/python/Twisted/py3/twisted/spread/publish.py b/contrib/python/Twisted/py3/twisted/spread/publish.py
new file mode 100644
index 00000000000..de525083e50
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/spread/publish.py
@@ -0,0 +1,144 @@
+# -*- test-case-name: twisted.spread.test.test_pb -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Persistently cached objects for PB.
+
+Maintainer: Glyph Lefkowitz
+
+Future Plans: None known.
+"""
+
+
+import time
+
+from twisted.internet import defer
+from twisted.spread import banana, flavors, jelly
+
+
+class Publishable(flavors.Cacheable):
+    """An object whose cached state persists across sessions."""
+
+    def __init__(self, publishedID):
+        self.republish()
+        self.publishedID = publishedID
+
+    def republish(self):
+        """Set the timestamp to current and (TODO) update all observers."""
+        self.timestamp = time.time()
+
+    def view_getStateToPublish(self, perspective):
+        "(internal)"
+        return self.getStateToPublishFor(perspective)
+
+    def getStateToPublishFor(self, perspective):
+        """Implement me to special-case your state for a perspective."""
+        return self.getStateToPublish()
+
+    def getStateToPublish(self):
+        """Implement me to return state to copy as part of the publish phase."""
+        raise NotImplementedError("%s.getStateToPublishFor" % self.__class__)
+
+    def getStateToCacheAndObserveFor(self, perspective, observer):
+        """Get all necessary metadata to keep a clientside cache."""
+        if perspective:
+            pname = perspective.perspectiveName
+            sname = perspective.getService().serviceName
+        else:
+            pname = "None"
+            sname = "None"
+
+        return {
+            "remote": flavors.ViewPoint(perspective, self),
+            "publishedID": self.publishedID,
+            "perspective": pname,
+            "service": sname,
+            "timestamp": self.timestamp,
+        }
+
+
+class RemotePublished(flavors.RemoteCache):
+    """The local representation of remote Publishable object."""
+
+    isActivated = 0
+    _wasCleanWhenLoaded = 0
+
+    def getFileName(self, ext="pub"):
+        return "{}-{}-{}.{}".format(
+            self.service,
+            self.perspective,
+            str(self.publishedID),
+            ext,
+        )
+
+    def setCopyableState(self, state):
+        self.__dict__.update(state)
+        self._activationListeners = []
+        try:
+            with open(self.getFileName(), "rb") as dataFile:
+                data = dataFile.read()
+        except OSError:
+            recent = 0
+        else:
+            newself = jelly.unjelly(banana.decode(data))
+            recent = newself.timestamp == self.timestamp
+        if recent:
+            self._cbGotUpdate(newself.__dict__)
+            self._wasCleanWhenLoaded = 1
+        else:
+            self.remote.callRemote("getStateToPublish").addCallbacks(self._cbGotUpdate)
+
+    def __getstate__(self):
+        other = self.__dict__.copy()
+        # Remove PB-specific attributes
+        del other["broker"]
+        del other["remote"]
+        del other["luid"]
+        # remove my own runtime-tracking stuff
+        del other["_activationListeners"]
+        del other["isActivated"]
+        return other
+
+    def _cbGotUpdate(self, newState):
+        self.__dict__.update(newState)
+        self.isActivated = 1
+        # send out notifications
+        for listener in self._activationListeners:
+            listener(self)
+        self._activationListeners = []
+        self.activated()
+        with open(self.getFileName(), "wb") as dataFile:
+            dataFile.write(banana.encode(jelly.jelly(self)))
+
+    def activated(self):
+        """Implement this method if you want to be notified when your
+        publishable subclass is activated.
+        """
+
+    def callWhenActivated(self, callback):
+        """Externally register for notification when this publishable has received all relevant data."""
+        if self.isActivated:
+            callback(self)
+        else:
+            self._activationListeners.append(callback)
+
+
+def whenReady(d):
+    """
+    Wrap a deferred returned from a pb method in another deferred that
+    expects a RemotePublished as a result.  This will allow you to wait until
+    the result is really available.
+
+    Idiomatic usage would look like::
+
+        publish.whenReady(serverObject.getMeAPublishable()).addCallback(lookAtThePublishable)
+    """
+    d2 = defer.Deferred()
+    d.addCallbacks(_pubReady, d2.errback, callbackArgs=(d2,))
+    return d2
+
+
+def _pubReady(result, d2):
+    "(internal)"
+    result.callWhenActivated(d2.callback)
diff --git a/contrib/python/Twisted/py3/twisted/spread/util.py b/contrib/python/Twisted/py3/twisted/spread/util.py
new file mode 100644
index 00000000000..5e9c81624f6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/spread/util.py
@@ -0,0 +1,217 @@
+# -*- test-case-name: twisted.test.test_pb -*-
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Utility classes for spread.
+"""
+
+from zope.interface import implementer
+
+from twisted.internet import defer, interfaces
+from twisted.protocols import basic
+from twisted.python.failure import Failure
+from twisted.spread import pb
+
+
+class LocalMethod:
+    def __init__(self, local, name):
+        self.local = local
+        self.name = name
+
+    def __call__(self, *args, **kw):
+        return self.local.callRemote(self.name, *args, **kw)
+
+
+class LocalAsRemote:
+    """
+    A class useful for emulating the effects of remote behavior locally.
+    """
+
+    reportAllTracebacks = 1
+
+    def callRemote(self, name, *args, **kw):
+        """
+        Call a specially-designated local method.
+
+        self.callRemote('x') will first try to invoke a method named
+        sync_x and return its result (which should probably be a
+        Deferred).  Second, it will look for a method called async_x,
+        which will be called and then have its result (or Failure)
+        automatically wrapped in a Deferred.
+        """
+        if hasattr(self, "sync_" + name):
+            return getattr(self, "sync_" + name)(*args, **kw)
+        try:
+            method = getattr(self, "async_" + name)
+            return defer.succeed(method(*args, **kw))
+        except BaseException:
+            f = Failure()
+            if self.reportAllTracebacks:
+                f.printTraceback()
+            return defer.fail(f)
+
+    def remoteMethod(self, name):
+        return LocalMethod(self, name)
+
+
+class LocalAsyncForwarder:
+    """
+    A class useful for forwarding a locally-defined interface.
+    """
+
+    def __init__(self, forwarded, interfaceClass, failWhenNotImplemented=0):
+        assert interfaceClass.providedBy(forwarded)
+        self.forwarded = forwarded
+        self.interfaceClass = interfaceClass
+        self.failWhenNotImplemented = failWhenNotImplemented
+
+    def _callMethod(self, method, *args, **kw):
+        return getattr(self.forwarded, method)(*args, **kw)
+
+    def callRemote(self, method, *args, **kw):
+        if self.interfaceClass.queryDescriptionFor(method):
+            result = defer.maybeDeferred(self._callMethod, method, *args, **kw)
+            return result
+        elif self.failWhenNotImplemented:
+            return defer.fail(
+                Failure(NotImplementedError, "No Such Method in Interface: %s" % method)
+            )
+        else:
+            return defer.succeed(None)
+
+
+class Pager:
+    """
+    I am an object which pages out information.
+    """
+
+    def __init__(self, collector, callback=None, *args, **kw):
+        """
+        Create a pager with a Reference to a remote collector and
+        an optional callable to invoke upon completion.
+        """
+        if callable(callback):
+            self.callback = callback
+            self.callbackArgs = args
+            self.callbackKeyword = kw
+        else:
+            self.callback = None
+        self._stillPaging = 1
+        self.collector = collector
+        collector.broker.registerPageProducer(self)
+
+    def stillPaging(self):
+        """
+        (internal) Method called by Broker.
+        """
+        if not self._stillPaging:
+            self.collector.callRemote("endedPaging", pbanswer=False)
+            if self.callback is not None:
+                self.callback(*self.callbackArgs, **self.callbackKeyword)
+        return self._stillPaging
+
+    def sendNextPage(self):
+        """
+        (internal) Method called by Broker.
+        """
+        self.collector.callRemote("gotPage", self.nextPage(), pbanswer=False)
+
+    def nextPage(self):
+        """
+        Override this to return an object to be sent to my collector.
+        """
+        raise NotImplementedError()
+
+    def stopPaging(self):
+        """
+        Call this when you're done paging.
+        """
+        self._stillPaging = 0
+
+
+class StringPager(Pager):
+    """
+    A simple pager that splits a string into chunks.
+    """
+
+    def __init__(self, collector, st, chunkSize=8192, callback=None, *args, **kw):
+        self.string = st
+        self.pointer = 0
+        self.chunkSize = chunkSize
+        Pager.__init__(self, collector, callback, *args, **kw)
+
+    def nextPage(self):
+        val = self.string[self.pointer : self.pointer + self.chunkSize]
+        self.pointer += self.chunkSize
+        if self.pointer >= len(self.string):
+            self.stopPaging()
+        return val
+
+
+@implementer(interfaces.IConsumer)
+class FilePager(Pager):
+    """
+    Reads a file in chunks and sends the chunks as they come.
+    """
+
+    def __init__(self, collector, fd, callback=None, *args, **kw):
+        self.chunks = []
+        Pager.__init__(self, collector, callback, *args, **kw)
+        self.startProducing(fd)
+
+    def startProducing(self, fd):
+        self.deferred = basic.FileSender().beginFileTransfer(fd, self)
+        self.deferred.addBoth(lambda x: self.stopPaging())
+
+    def registerProducer(self, producer, streaming):
+        self.producer = producer
+        if not streaming:
+            self.producer.resumeProducing()
+
+    def unregisterProducer(self):
+        self.producer = None
+
+    def write(self, chunk):
+        self.chunks.append(chunk)
+
+    def sendNextPage(self):
+        """
+        Get the first chunk read and send it to collector.
+        """
+        if not self.chunks:
+            return
+        val = self.chunks.pop(0)
+        self.producer.resumeProducing()
+        self.collector.callRemote("gotPage", val, pbanswer=False)
+
+
+# Utility paging stuff.
+class CallbackPageCollector(pb.Referenceable):
+    """
+    I receive pages from the peer. You may instantiate a Pager with a
+    remote reference to me. I will call the callback with a list of pages
+    once they are all received.
+    """
+
+    def __init__(self, callback):
+        self.pages = []
+        self.callback = callback
+
+    def remote_gotPage(self, page):
+        self.pages.append(page)
+
+    def remote_endedPaging(self):
+        self.callback(self.pages)
+
+
+def getAllPages(referenceable, methodName, *args, **kw):
+    """
+    A utility method that will call a remote method which expects a
+    PageCollector as the first argument.
+    """
+    d = defer.Deferred()
+    referenceable.callRemote(methodName, CallbackPageCollector(d.callback), *args, **kw)
+    return d
diff --git a/contrib/python/Twisted/py3/twisted/tap/__init__.py b/contrib/python/Twisted/py3/twisted/tap/__init__.py
new file mode 100644
index 00000000000..cdc430f94e2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/tap/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted TAP: Twisted Application Persistence builders for other Twisted servers.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/tap/ftp.py b/contrib/python/Twisted/py3/twisted/tap/ftp.py
new file mode 100644
index 00000000000..0bee0c56541
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/tap/ftp.py
@@ -0,0 +1,66 @@
+# -*- test-case-name: twisted.test.test_ftp_options -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+I am the support module for making a ftp server with twistd.
+"""
+
+import warnings
+
+from twisted.application import internet
+from twisted.cred import checkers, portal, strcred
+from twisted.protocols import ftp
+from twisted.python import deprecate, usage, versions
+
+
+class Options(usage.Options, strcred.AuthOptionMixin):
+    synopsis = """[options].
+    WARNING: This FTP server is probably INSECURE do not use it.
+    """
+    optParameters = [
+        ["port", "p", "2121", "set the port number"],
+        ["root", "r", "/usr/local/ftp", "define the root of the ftp-site."],
+        ["userAnonymous", "", "anonymous", "Name of the anonymous user."],
+    ]
+
+    compData = usage.Completions(
+        optActions={"root": usage.CompleteDirs(descr="root of the ftp site")}
+    )
+
+    longdesc = ""
+
+    def __init__(self, *a, **kw):
+        usage.Options.__init__(self, *a, **kw)
+        self.addChecker(checkers.AllowAnonymousAccess())
+
+    def opt_password_file(self, filename):
+        """
+        Specify a file containing username:password login info for
+        authenticated connections. (DEPRECATED; see --help-auth instead)
+        """
+        self["password-file"] = filename
+        msg = deprecate.getDeprecationWarningString(
+            self.opt_password_file, versions.Version("Twisted", 11, 1, 0)
+        )
+        warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+        self.addChecker(checkers.FilePasswordDB(filename, cache=True))
+
+
+def makeService(config):
+    f = ftp.FTPFactory()
+
+    r = ftp.FTPRealm(config["root"])
+    p = portal.Portal(r, config.get("credCheckers", []))
+
+    f.tld = config["root"]
+    f.userAnonymous = config["userAnonymous"]
+    f.portal = p
+    f.protocol = ftp.FTP
+
+    try:
+        portno = int(config["port"])
+    except KeyError:
+        portno = 2121
+    return internet.TCPServer(portno, f)
diff --git a/contrib/python/Twisted/py3/twisted/tap/portforward.py b/contrib/python/Twisted/py3/twisted/tap/portforward.py
new file mode 100644
index 00000000000..3fc01460941
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/tap/portforward.py
@@ -0,0 +1,26 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support module for making a port forwarder with twistd.
+"""
+from twisted.application import strports
+from twisted.protocols import portforward
+from twisted.python import usage
+
+
+class Options(usage.Options):
+    synopsis = "[options]"
+    longdesc = "Port Forwarder."
+    optParameters = [
+        ["port", "p", "6666", "Set the port number."],
+        ["host", "h", "localhost", "Set the host."],
+        ["dest_port", "d", 6665, "Set the destination port."],
+    ]
+
+    compData = usage.Completions(optActions={"host": usage.CompleteHostnames()})
+
+
+def makeService(config):
+    f = portforward.ProxyFactory(config["host"], int(config["dest_port"]))
+    return strports.service(config["port"], f)
diff --git a/contrib/python/Twisted/py3/twisted/tap/socks.py b/contrib/python/Twisted/py3/twisted/tap/socks.py
new file mode 100644
index 00000000000..72c7a57ce95
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/tap/socks.py
@@ -0,0 +1,42 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+I am a support module for making SOCKSv4 servers with twistd.
+"""
+
+from twisted.application import internet
+from twisted.protocols import socks
+from twisted.python import usage
+
+
+class Options(usage.Options):
+    synopsis = "[-i <interface>] [-p <port>] [-l <file>]"
+    optParameters = [
+        ["interface", "i", "127.0.0.1", "local interface to which we listen"],
+        ["port", "p", 1080, "Port on which to listen"],
+        ["log", "l", None, "file to log connection data to"],
+    ]
+
+    compData = usage.Completions(
+        optActions={
+            "log": usage.CompleteFiles("*.log"),
+            "interface": usage.CompleteNetInterfaces(),
+        }
+    )
+
+    longdesc = "Makes a SOCKSv4 server."
+
+
+def makeService(config):
+    if config["interface"] != "127.0.0.1":
+        print()
+        print("WARNING:")
+        print("  You have chosen to listen on a non-local interface.")
+        print("  This may allow intruders to access your local network")
+        print("  if you run this on a firewall.")
+        print()
+    t = socks.SOCKSv4Factory(config["log"])
+    portno = int(config["port"])
+    return internet.TCPServer(portno, t, interface=config["interface"])
diff --git a/contrib/python/Twisted/py3/twisted/trial/__init__.py b/contrib/python/Twisted/py3/twisted/trial/__init__.py
new file mode 100644
index 00000000000..5faaa99ddd2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/__init__.py
@@ -0,0 +1,50 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+#
+# Maintainer: Jonathan Lange
+
+"""
+Twisted Trial: Asynchronous unit testing framework.
+
+Trial extends Python's builtin C{unittest} to provide support for asynchronous
+tests.
+
+Trial strives to be compatible with other Python xUnit testing frameworks.
+"Compatibility" is a difficult things to define. In practice, it means that:
+
+ - L{twisted.trial.unittest.TestCase} objects should be able to be used by
+   other test runners without those runners requiring special support for
+   Trial tests.
+
+ - Tests that subclass the standard library C{TestCase} and don't do anything
+   "too weird" should be able to be discoverable and runnable by the Trial
+   test runner without the authors of those tests having to jump through
+   hoops.
+
+ - Tests that implement the interface provided by the standard library
+   C{TestCase} should be runnable by the Trial runner.
+
+ - The Trial test runner and Trial L{unittest.TestCase} objects ought to be
+   able to use standard library C{TestResult} objects, and third party
+   C{TestResult} objects based on the standard library.
+
+This list is not necessarily exhaustive -- compatibility is hard to define.
+Contributors who discover more helpful ways of defining compatibility are
+encouraged to update this document.
+
+
+Examples:
+
+B{Timeouts} for tests should be implemented in the runner. If this is done,
+then timeouts could work for third-party TestCase objects as well as for
+L{twisted.trial.unittest.TestCase} objects. Further, Twisted C{TestCase}
+objects will run in other runners without timing out.
+See U{http://twistedmatrix.com/trac/ticket/2675}.
+
+Running tests in a temporary directory should be a feature of the test case,
+because often tests themselves rely on this behaviour. If the feature is
+implemented in the runner, then tests will change behaviour (possibly
+breaking) when run in a different test runner. Further, many tests don't even
+care about the filesystem.
+See U{http://twistedmatrix.com/trac/ticket/2916}.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/trial/__main__.py b/contrib/python/Twisted/py3/twisted/trial/__main__.py
new file mode 100644
index 00000000000..4c234a162ec
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/__main__.py
@@ -0,0 +1,9 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+if __name__ == "__main__":
+    import sys
+
+    from twisted.scripts.trial import run
+
+    sys.exit(run())
diff --git a/contrib/python/Twisted/py3/twisted/trial/_asyncrunner.py b/contrib/python/Twisted/py3/twisted/trial/_asyncrunner.py
new file mode 100644
index 00000000000..12a8342ffae
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_asyncrunner.py
@@ -0,0 +1,176 @@
+# -*- test-case-name: twisted.trial.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Infrastructure for test running and suites.
+"""
+
+
+import doctest
+import gc
+import unittest as pyunit
+from typing import Iterator, Union
+
+from zope.interface import implementer
+
+from twisted.python import components
+from twisted.trial import itrial, reporter
+from twisted.trial._synctest import _logObserver
+
+
+class TestSuite(pyunit.TestSuite):
+    """
+    Extend the standard library's C{TestSuite} with a consistently overrideable
+    C{run} method.
+    """
+
+    def run(self, result):
+        """
+        Call C{run} on every member of the suite.
+        """
+        for test in self._tests:
+            if result.shouldStop:
+                break
+            test(result)
+        return result
+
+
+@implementer(itrial.ITestCase)
+class TestDecorator(
+    components.proxyForInterface(  # type: ignore[misc]
+        itrial.ITestCase, "_originalTest"
+    )
+):
+    """
+    Decorator for test cases.
+
+    @param _originalTest: The wrapped instance of test.
+    @type _originalTest: A provider of L{itrial.ITestCase}
+    """
+
+    def __call__(self, result):
+        """
+        Run the unit test.
+
+        @param result: A TestResult object.
+        """
+        return self.run(result)
+
+    def run(self, result):
+        """
+        Run the unit test.
+
+        @param result: A TestResult object.
+        """
+        return self._originalTest.run(reporter._AdaptedReporter(result, self.__class__))
+
+
+def _clearSuite(suite):
+    """
+    Clear all tests from C{suite}.
+
+    This messes with the internals of C{suite}. In particular, it assumes that
+    the suite keeps all of its tests in a list in an instance variable called
+    C{_tests}.
+    """
+    suite._tests = []
+
+
+def decorate(test, decorator):
+    """
+    Decorate all test cases in C{test} with C{decorator}.
+
+    C{test} can be a test case or a test suite. If it is a test suite, then the
+    structure of the suite is preserved.
+
+    L{decorate} tries to preserve the class of the test suites it finds, but
+    assumes the presence of the C{_tests} attribute on the suite.
+
+    @param test: The C{TestCase} or C{TestSuite} to decorate.
+
+    @param decorator: A unary callable used to decorate C{TestCase}s.
+
+    @return: A decorated C{TestCase} or a C{TestSuite} containing decorated
+        C{TestCase}s.
+    """
+
+    try:
+        tests = iter(test)
+    except TypeError:
+        return decorator(test)
+
+    # At this point, we know that 'test' is a test suite.
+    _clearSuite(test)
+
+    for case in tests:
+        test.addTest(decorate(case, decorator))
+    return test
+
+
+class _PyUnitTestCaseAdapter(TestDecorator):
+    """
+    Adapt from pyunit.TestCase to ITestCase.
+    """
+
+
+class _BrokenIDTestCaseAdapter(_PyUnitTestCaseAdapter):
+    """
+    Adapter for pyunit-style C{TestCase} subclasses that have undesirable id()
+    methods. That is C{unittest.FunctionTestCase} and C{unittest.DocTestCase}.
+    """
+
+    def id(self):
+        """
+        Return the fully-qualified Python name of the doctest.
+        """
+        testID = self._originalTest.shortDescription()
+        if testID is not None:
+            return testID
+        return self._originalTest.id()
+
+
+class _ForceGarbageCollectionDecorator(TestDecorator):
+    """
+    Forces garbage collection to be run before and after the test. Any errors
+    logged during the post-test collection are added to the test result as
+    errors.
+    """
+
+    def run(self, result):
+        gc.collect()
+        TestDecorator.run(self, result)
+        _logObserver._add()
+        gc.collect()
+        for error in _logObserver.getErrors():
+            result.addError(self, error)
+        _logObserver.flushErrors()
+        _logObserver._remove()
+
+
+components.registerAdapter(_PyUnitTestCaseAdapter, pyunit.TestCase, itrial.ITestCase)
+
+
+components.registerAdapter(
+    _BrokenIDTestCaseAdapter, pyunit.FunctionTestCase, itrial.ITestCase
+)
+
+
+_docTestCase = getattr(doctest, "DocTestCase", None)
+if _docTestCase:
+    components.registerAdapter(_BrokenIDTestCaseAdapter, _docTestCase, itrial.ITestCase)
+
+
+def _iterateTests(
+    testSuiteOrCase: Union[pyunit.TestCase, pyunit.TestSuite]
+) -> Iterator[itrial.ITestCase]:
+    """
+    Iterate through all of the test cases in C{testSuiteOrCase}.
+    """
+    try:
+        suite = iter(testSuiteOrCase)  # type: ignore[arg-type]
+    except TypeError:
+        yield testSuiteOrCase  # type: ignore[misc]
+    else:
+        for test in suite:
+            yield from _iterateTests(test)
diff --git a/contrib/python/Twisted/py3/twisted/trial/_asynctest.py b/contrib/python/Twisted/py3/twisted/trial/_asynctest.py
new file mode 100644
index 00000000000..048e8dc0378
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_asynctest.py
@@ -0,0 +1,413 @@
+# -*- test-case-name: twisted.trial.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Things likely to be used by writers of unit tests.
+
+Maintainer: Jonathan Lange
+"""
+
+
+import inspect
+import warnings
+from typing import Callable, List
+
+from zope.interface import implementer
+
+from typing_extensions import ParamSpec
+
+# We can't import reactor at module-level because this code runs before trial
+# installs a user-specified reactor, installing the default reactor and
+# breaking reactor installation. See also #6047.
+from twisted.internet import defer, utils
+from twisted.python import failure
+from twisted.trial import itrial, util
+from twisted.trial._synctest import FailTest, SkipTest, SynchronousTestCase
+
+_P = ParamSpec("_P")
+
+_wait_is_running: List[None] = []
+
+
+@implementer(itrial.ITestCase)
+class TestCase(SynchronousTestCase):
+    """
+    A unit test. The atom of the unit testing universe.
+
+    This class extends L{SynchronousTestCase} which extends C{unittest.TestCase}
+    from the standard library. The main feature is the ability to return
+    C{Deferred}s from tests and fixture methods and to have the suite wait for
+    those C{Deferred}s to fire.  Also provides new assertions such as
+    L{assertFailure}.
+
+    @ivar timeout: A real number of seconds. If set, the test will
+    raise an error if it takes longer than C{timeout} seconds.
+    If not set, util.DEFAULT_TIMEOUT_DURATION is used.
+    """
+
+    def __init__(self, methodName="runTest"):
+        """
+        Construct an asynchronous test case for C{methodName}.
+
+        @param methodName: The name of a method on C{self}. This method should
+        be a unit test. That is, it should be a short method that calls some of
+        the assert* methods. If C{methodName} is unspecified,
+        L{SynchronousTestCase.runTest} will be used as the test method. This is
+        mostly useful for testing Trial.
+        """
+        super().__init__(methodName)
+
+    def assertFailure(self, deferred, *expectedFailures):
+        """
+        Fail if C{deferred} does not errback with one of C{expectedFailures}.
+        Returns the original Deferred with callbacks added. You will need
+        to return this Deferred from your test case.
+        """
+
+        def _cb(ignore):
+            raise self.failureException(
+                f"did not catch an error, instead got {ignore!r}"
+            )
+
+        def _eb(failure):
+            if failure.check(*expectedFailures):
+                return failure.value
+            else:
+                output = "\nExpected: {!r}\nGot:\n{}".format(
+                    expectedFailures, str(failure)
+                )
+                raise self.failureException(output)
+
+        return deferred.addCallbacks(_cb, _eb)
+
+    failUnlessFailure = assertFailure
+
+    def _run(self, methodName, result):
+        from twisted.internet import reactor
+
+        timeout = self.getTimeout()
+
+        def onTimeout(d):
+            e = defer.TimeoutError(
+                f"{self!r} ({methodName}) still running at {timeout} secs"
+            )
+            f = failure.Failure(e)
+            # try to errback the deferred that the test returns (for no gorram
+            # reason) (see issue1005 and test_errorPropagation in
+            # test_deferred)
+            try:
+                d.errback(f)
+            except defer.AlreadyCalledError:
+                # if the deferred has been called already but the *back chain
+                # is still unfinished, crash the reactor and report timeout
+                # error ourself.
+                reactor.crash()
+                self._timedOut = True  # see self._wait
+                todo = self.getTodo()
+                if todo is not None and todo.expected(f):
+                    result.addExpectedFailure(self, f, todo)
+                else:
+                    result.addError(self, f)
+
+        onTimeout = utils.suppressWarnings(
+            onTimeout, util.suppress(category=DeprecationWarning)
+        )
+        method = getattr(self, methodName)
+        if inspect.isgeneratorfunction(method):
+            exc = TypeError(
+                "{!r} is a generator function and therefore will never run".format(
+                    method
+                )
+            )
+            return defer.fail(exc)
+        d = defer.maybeDeferred(
+            utils.runWithWarningsSuppressed, self._getSuppress(), method
+        )
+        call = reactor.callLater(timeout, onTimeout, d)
+        d.addBoth(lambda x: call.active() and call.cancel() or x)
+        return d
+
+    def __call__(self, *args, **kwargs):
+        return self.run(*args, **kwargs)
+
+    def deferSetUp(self, ignored, result):
+        d = self._run("setUp", result)
+        d.addCallbacks(
+            self.deferTestMethod,
+            self._ebDeferSetUp,
+            callbackArgs=(result,),
+            errbackArgs=(result,),
+        )
+        return d
+
+    def _ebDeferSetUp(self, failure, result):
+        if failure.check(SkipTest):
+            result.addSkip(self, self._getSkipReason(self.setUp, failure.value))
+        else:
+            result.addError(self, failure)
+            if failure.check(KeyboardInterrupt):
+                result.stop()
+        return self.deferRunCleanups(None, result)
+
+    def deferTestMethod(self, ignored, result):
+        d = self._run(self._testMethodName, result)
+        d.addCallbacks(
+            self._cbDeferTestMethod,
+            self._ebDeferTestMethod,
+            callbackArgs=(result,),
+            errbackArgs=(result,),
+        )
+        d.addBoth(self.deferRunCleanups, result)
+        d.addBoth(self.deferTearDown, result)
+        return d
+
+    def _cbDeferTestMethod(self, ignored, result):
+        if self.getTodo() is not None:
+            result.addUnexpectedSuccess(self, self.getTodo())
+        else:
+            self._passed = True
+        return ignored
+
+    def _ebDeferTestMethod(self, f, result):
+        todo = self.getTodo()
+        if todo is not None and todo.expected(f):
+            result.addExpectedFailure(self, f, todo)
+        elif f.check(self.failureException, FailTest):
+            result.addFailure(self, f)
+        elif f.check(KeyboardInterrupt):
+            result.addError(self, f)
+            result.stop()
+        elif f.check(SkipTest):
+            result.addSkip(
+                self, self._getSkipReason(getattr(self, self._testMethodName), f.value)
+            )
+        else:
+            result.addError(self, f)
+
+    def deferTearDown(self, ignored, result):
+        d = self._run("tearDown", result)
+        d.addErrback(self._ebDeferTearDown, result)
+        return d
+
+    def _ebDeferTearDown(self, failure, result):
+        result.addError(self, failure)
+        if failure.check(KeyboardInterrupt):
+            result.stop()
+        self._passed = False
+
+    @defer.inlineCallbacks
+    def deferRunCleanups(self, ignored, result):
+        """
+        Run any scheduled cleanups and report errors (if any) to the result.
+        object.
+        """
+        failures = []
+        while len(self._cleanups) > 0:
+            func, args, kwargs = self._cleanups.pop()
+            try:
+                yield func(*args, **kwargs)
+            except Exception:
+                failures.append(failure.Failure())
+
+        for f in failures:
+            result.addError(self, f)
+            self._passed = False
+
+    def _cleanUp(self, result):
+        try:
+            clean = util._Janitor(self, result).postCaseCleanup()
+            if not clean:
+                self._passed = False
+        except BaseException:
+            result.addError(self, failure.Failure())
+            self._passed = False
+        for error in self._observer.getErrors():
+            result.addError(self, error)
+            self._passed = False
+        self.flushLoggedErrors()
+        self._removeObserver()
+        if self._passed:
+            result.addSuccess(self)
+
+    def _classCleanUp(self, result):
+        try:
+            util._Janitor(self, result).postClassCleanup()
+        except BaseException:
+            result.addError(self, failure.Failure())
+
+    def _makeReactorMethod(self, name):
+        """
+        Create a method which wraps the reactor method C{name}. The new
+        method issues a deprecation warning and calls the original.
+        """
+
+        def _(*a, **kw):
+            warnings.warn(
+                "reactor.%s cannot be used inside unit tests. "
+                "In the future, using %s will fail the test and may "
+                "crash or hang the test run." % (name, name),
+                stacklevel=2,
+                category=DeprecationWarning,
+            )
+            return self._reactorMethods[name](*a, **kw)
+
+        return _
+
+    def _deprecateReactor(self, reactor):
+        """
+        Deprecate C{iterate}, C{crash} and C{stop} on C{reactor}. That is,
+        each method is wrapped in a function that issues a deprecation
+        warning, then calls the original.
+
+        @param reactor: The Twisted reactor.
+        """
+        self._reactorMethods = {}
+        for name in ["crash", "iterate", "stop"]:
+            self._reactorMethods[name] = getattr(reactor, name)
+            setattr(reactor, name, self._makeReactorMethod(name))
+
+    def _undeprecateReactor(self, reactor):
+        """
+        Restore the deprecated reactor methods. Undoes what
+        L{_deprecateReactor} did.
+
+        @param reactor: The Twisted reactor.
+        """
+        for name, method in self._reactorMethods.items():
+            setattr(reactor, name, method)
+        self._reactorMethods = {}
+
+    def _runFixturesAndTest(self, result):
+        """
+        Really run C{setUp}, the test method, and C{tearDown}.  Any of these may
+        return L{defer.Deferred}s. After they complete, do some reactor cleanup.
+
+        @param result: A L{TestResult} object.
+        """
+        from twisted.internet import reactor
+
+        self._deprecateReactor(reactor)
+        self._timedOut = False
+        try:
+            d = self.deferSetUp(None, result)
+            try:
+                self._wait(d)
+            finally:
+                self._cleanUp(result)
+                self._classCleanUp(result)
+        finally:
+            self._undeprecateReactor(reactor)
+
+    # f should be a positional only argument but that is a breaking change
+    # see https://github.com/twisted/twisted/issues/11967
+    def addCleanup(  # type: ignore[override]
+        self, f: Callable[_P, object], *args: _P.args, **kwargs: _P.kwargs
+    ) -> None:
+        """
+        Extend the base cleanup feature with support for cleanup functions which
+        return Deferreds.
+
+        If the function C{f} returns a Deferred, C{TestCase} will wait until the
+        Deferred has fired before proceeding to the next function.
+        """
+        return super().addCleanup(f, *args, **kwargs)
+
+    def getSuppress(self):
+        return self._getSuppress()
+
+    def getTimeout(self):
+        """
+        Returns the timeout value set on this test. Checks on the instance
+        first, then the class, then the module, then packages. As soon as it
+        finds something with a C{timeout} attribute, returns that. Returns
+        L{util.DEFAULT_TIMEOUT_DURATION} if it cannot find anything. See
+        L{TestCase} docstring for more details.
+        """
+        timeout = util.acquireAttribute(
+            self._parents, "timeout", util.DEFAULT_TIMEOUT_DURATION
+        )
+        try:
+            return float(timeout)
+        except (ValueError, TypeError):
+            # XXX -- this is here because sometimes people will have methods
+            # called 'timeout', or set timeout to 'orange', or something
+            # Particularly, test_news.NewsTestCase and ReactorCoreTestCase
+            # both do this.
+            warnings.warn(
+                "'timeout' attribute needs to be a number.", category=DeprecationWarning
+            )
+            return util.DEFAULT_TIMEOUT_DURATION
+
+    def _wait(self, d, running=_wait_is_running):
+        """Take a Deferred that only ever callbacks. Block until it happens."""
+        if running:
+            raise RuntimeError("_wait is not reentrant")
+
+        from twisted.internet import reactor
+
+        results = []
+
+        def append(any):
+            if results is not None:
+                results.append(any)
+
+        def crash(ign):
+            if results is not None:
+                reactor.crash()
+
+        crash = utils.suppressWarnings(
+            crash,
+            util.suppress(
+                message=r"reactor\.crash cannot be used.*", category=DeprecationWarning
+            ),
+        )
+
+        def stop():
+            reactor.crash()
+
+        stop = utils.suppressWarnings(
+            stop,
+            util.suppress(
+                message=r"reactor\.crash cannot be used.*", category=DeprecationWarning
+            ),
+        )
+
+        running.append(None)
+        try:
+            d.addBoth(append)
+            if results:
+                # d might have already been fired, in which case append is
+                # called synchronously. Avoid any reactor stuff.
+                return
+            d.addBoth(crash)
+            reactor.stop = stop
+            try:
+                reactor.run()
+            finally:
+                del reactor.stop
+
+            # If the reactor was crashed elsewhere due to a timeout, hopefully
+            # that crasher also reported an error. Just return.
+            # _timedOut is most likely to be set when d has fired but hasn't
+            # completed its callback chain (see self._run)
+            if results or self._timedOut:  # defined in run() and _run()
+                return
+
+            # If the timeout didn't happen, and we didn't get a result or
+            # a failure, then the user probably aborted the test, so let's
+            # just raise KeyboardInterrupt.
+
+            # FIXME: imagine this:
+            # web/test/test_webclient.py:
+            # exc = self.assertRaises(error.Error, wait, method(url))
+            #
+            # wait() will raise KeyboardInterrupt, and assertRaises will
+            # swallow it. Therefore, wait() raising KeyboardInterrupt is
+            # insufficient to stop trial. A suggested solution is to have
+            # this code set a "stop trial" flag, or otherwise notify trial
+            # that it should really try to stop as soon as possible.
+            raise KeyboardInterrupt()
+        finally:
+            results = None
+            running.pop()
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/__init__.py b/contrib/python/Twisted/py3/twisted/trial/_dist/__init__.py
new file mode 100644
index 00000000000..502e840fef7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/__init__.py
@@ -0,0 +1,47 @@
+# -*- test-case-name: twisted.trial._dist.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This package implements the distributed Trial test runner:
+
+  - The L{twisted.trial._dist.disttrial} module implements a test runner which
+    runs in a manager process and can launch additional worker processes in
+    which to run tests and gather up results from all of them.
+
+  - The L{twisted.trial._dist.options} module defines command line options used
+    to configure the distributed test runner.
+
+  - The L{twisted.trial._dist.managercommands} module defines AMP commands
+    which are sent from worker processes back to the manager process to report
+    the results of tests.
+
+  - The L{twisted.trial._dist.workercommands} module defines AMP commands which
+    are sent from the manager process to the worker processes to control the
+    execution of tests there.
+
+  - The L{twisted.trial._dist.distreporter} module defines a proxy for
+    L{twisted.trial.itrial.IReporter} which enforces the typical requirement
+    that results be passed to a reporter for only one test at a time, allowing
+    any reporter to be used with despite disttrial's simultaneously running
+    tests.
+
+  - The L{twisted.trial._dist.workerreporter} module implements a
+    L{twisted.trial.itrial.IReporter} which is used by worker processes and
+    reports results back to the manager process using AMP commands.
+
+  - The L{twisted.trial._dist.workertrial} module is a runnable script which is
+    the main point for worker processes.
+
+  - The L{twisted.trial._dist.worker} process defines the manager's AMP
+    protocol for accepting results from worker processes and a process protocol
+    for use running workers as local child processes (as opposed to
+    distributing them to another host).
+
+@since: 12.3
+"""
+
+# File descriptors numbers used to set up pipes with the worker.
+_WORKER_AMP_STDIN = 3
+
+_WORKER_AMP_STDOUT = 4
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/distreporter.py b/contrib/python/Twisted/py3/twisted/trial/_dist/distreporter.py
new file mode 100644
index 00000000000..3a45cc48067
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/distreporter.py
@@ -0,0 +1,90 @@
+# -*- test-case-name: twisted.trial._dist.test.test_distreporter -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+The reporter is not made to support concurrent test running, so we will
+hold test results in here and only send them to the reporter once the
+test is over.
+
+@since: 12.3
+"""
+
+from types import TracebackType
+from typing import Optional, Tuple, Union
+
+from zope.interface import implementer
+
+from twisted.python.components import proxyForInterface
+from twisted.python.failure import Failure
+from ..itrial import IReporter, ITestCase
+
+ReporterFailure = Union[Failure, Tuple[type, Exception, TracebackType]]
+
+
+@implementer(IReporter)
+class DistReporter(proxyForInterface(IReporter)):  # type: ignore[misc]
+    """
+    See module docstring.
+    """
+
+    def __init__(self, original):
+        super().__init__(original)
+        self.running = {}
+
+    def startTest(self, test):
+        """
+        Queue test starting.
+        """
+        self.running[test.id()] = []
+        self.running[test.id()].append((self.original.startTest, test))
+
+    def addFailure(self, test: ITestCase, fail: ReporterFailure) -> None:
+        """
+        Queue adding a failure.
+        """
+        self.running[test.id()].append((self.original.addFailure, test, fail))
+
+    def addError(self, test: ITestCase, error: ReporterFailure) -> None:
+        """
+        Queue error adding.
+        """
+        self.running[test.id()].append((self.original.addError, test, error))
+
+    def addSkip(self, test, reason):
+        """
+        Queue adding a skip.
+        """
+        self.running[test.id()].append((self.original.addSkip, test, reason))
+
+    def addUnexpectedSuccess(self, test, todo=None):
+        """
+        Queue adding an unexpected success.
+        """
+        self.running[test.id()].append((self.original.addUnexpectedSuccess, test, todo))
+
+    def addExpectedFailure(
+        self, test: ITestCase, error: ReporterFailure, todo: Optional[str] = None
+    ) -> None:
+        """
+        Queue adding an expected failure.
+        """
+        self.running[test.id()].append(
+            (self.original.addExpectedFailure, test, error, todo)
+        )
+
+    def addSuccess(self, test):
+        """
+        Queue adding a success.
+        """
+        self.running[test.id()].append((self.original.addSuccess, test))
+
+    def stopTest(self, test):
+        """
+        Queue stopping the test, then unroll the queue.
+        """
+        self.running[test.id()].append((self.original.stopTest, test))
+        for step in self.running[test.id()]:
+            step[0](*step[1:])
+        del self.running[test.id()]
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/disttrial.py b/contrib/python/Twisted/py3/twisted/trial/_dist/disttrial.py
new file mode 100644
index 00000000000..bad82a88148
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/disttrial.py
@@ -0,0 +1,512 @@
+# -*- test-case-name: twisted.trial._dist.test.test_disttrial -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module contains the trial distributed runner, the management class
+responsible for coordinating all of trial's behavior at the highest level.
+
+@since: 12.3
+"""
+
+import os
+import sys
+from functools import partial
+from os.path import isabs
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    TextIO,
+    Union,
+    cast,
+)
+from unittest import TestCase, TestSuite
+
+from attrs import define, field, frozen
+from attrs.converters import default_if_none
+
+from twisted.internet.defer import Deferred, DeferredList, gatherResults
+from twisted.internet.interfaces import IReactorCore, IReactorProcess
+from twisted.logger import Logger
+from twisted.python.failure import Failure
+from twisted.python.filepath import FilePath
+from twisted.python.lockfile import FilesystemLock
+from twisted.python.modules import theSystemPath
+from .._asyncrunner import _iterateTests
+from ..itrial import IReporter, ITestCase
+from ..reporter import UncleanWarningsReporterWrapper
+from ..runner import TestHolder
+from ..util import _unusedTestDirectory, openTestLog
+from . import _WORKER_AMP_STDIN, _WORKER_AMP_STDOUT
+from .distreporter import DistReporter
+from .functional import countingCalls, discardResult, iterateWhile, takeWhile
+from .worker import LocalWorker, LocalWorkerAMP, WorkerAction
+
+
+class IDistTrialReactor(IReactorCore, IReactorProcess):
+    """
+    The reactor interfaces required by disttrial.
+    """
+
+
+def _defaultReactor() -> IDistTrialReactor:
+    """
+    Get the default reactor, ensuring it is suitable for use with disttrial.
+    """
+    import twisted.internet.reactor as defaultReactor
+
+    if all(
+        [
+            IReactorCore.providedBy(defaultReactor),
+            IReactorProcess.providedBy(defaultReactor),
+        ]
+    ):
+        # If it provides each of the interfaces then it provides the
+        # intersection interface.  cast it to make it easier to talk about
+        # later on.
+        return cast(IDistTrialReactor, defaultReactor)
+
+    raise TypeError("Reactor does not provide the right interfaces")
+
+
+@frozen
+class WorkerPoolConfig:
+    """
+    Configuration parameters for a pool of test-running workers.
+
+    @ivar numWorkers: The number of workers in the pool.
+
+    @ivar workingDirectory: A directory in which working directories for each
+        of the workers will be created.
+
+    @ivar workerArguments: Extra arguments to pass the worker process in its
+        argv.
+
+    @ivar logFile: The basename of the overall test log file.
+    """
+
+    numWorkers: int
+    workingDirectory: FilePath[Any]
+    workerArguments: Sequence[str]
+    logFile: str
+
+
+@define
+class StartedWorkerPool:
+    """
+    A pool of workers which have already been started.
+
+    @ivar workingDirectory: A directory holding the working directories for
+        each of the workers.
+
+    @ivar testDirLock: An object representing the cooperative lock this pool
+        holds on its working directory.
+
+    @ivar testLog: The open overall test log file.
+
+    @ivar workers: Objects corresponding to the worker child processes and
+        adapting between process-related interfaces and C{IProtocol}.
+
+    @ivar ampWorkers: AMP protocol instances corresponding to the worker child
+        processes.
+    """
+
+    workingDirectory: FilePath[Any]
+    testDirLock: FilesystemLock
+    testLog: TextIO
+    workers: List[LocalWorker]
+    ampWorkers: List[LocalWorkerAMP]
+
+    _logger = Logger()
+
+    async def run(self, workerAction: WorkerAction[Any]) -> None:
+        """
+        Run an action on all of the workers in the pool.
+        """
+        await gatherResults(
+            discardResult(workerAction(worker)) for worker in self.ampWorkers
+        )
+        return None
+
+    async def join(self) -> None:
+        """
+        Shut down all of the workers in the pool.
+
+        The pool is unusable after this method is called.
+        """
+        results = await DeferredList(
+            [Deferred.fromCoroutine(worker.exit()) for worker in self.workers],
+            consumeErrors=True,
+        )
+        for n, (succeeded, failure) in enumerate(results):
+            if not succeeded:
+                self._logger.failure(f"joining disttrial worker #{n} failed", failure)
+
+        del self.workers[:]
+        del self.ampWorkers[:]
+        self.testLog.close()
+        self.testDirLock.unlock()
+
+
+@frozen
+class WorkerPool:
+    """
+    Manage a fixed-size collection of child processes which can run tests.
+
+    @ivar _config: Configuration for the precise way in which the pool is run.
+    """
+
+    _config: WorkerPoolConfig
+
+    def _createLocalWorkers(
+        self,
+        protocols: Iterable[LocalWorkerAMP],
+        workingDirectory: FilePath[Any],
+        logFile: TextIO,
+    ) -> List[LocalWorker]:
+        """
+        Create local worker protocol instances and return them.
+
+        @param protocols: The process/protocol adapters to use for the created
+        workers.
+
+        @param workingDirectory: The base path in which we should run the
+            workers.
+
+        @param logFile: The test log, for workers to write to.
+
+        @return: A list of C{quantity} C{LocalWorker} instances.
+        """
+        return [
+            LocalWorker(protocol, workingDirectory.child(str(x)), logFile)
+            for x, protocol in enumerate(protocols)
+        ]
+
+    def _launchWorkerProcesses(self, spawner, protocols, arguments):
+        """
+        Spawn processes from a list of process protocols.
+
+        @param spawner: A C{IReactorProcess.spawnProcess} implementation.
+
+        @param protocols: An iterable of C{ProcessProtocol} instances.
+
+        @param arguments: Extra arguments passed to the processes.
+        """
+        workertrialPath = theSystemPath["twisted.trial._dist.workertrial"].filePath.path
+        childFDs = {
+            0: "w",
+            1: "r",
+            2: "r",
+            _WORKER_AMP_STDIN: "w",
+            _WORKER_AMP_STDOUT: "r",
+        }
+        environ = os.environ.copy()
+        # Add an environment variable containing the raw sys.path, to be used
+        # by subprocesses to try to make it identical to the parent's.
+        environ["PYTHONPATH"] = os.pathsep.join(sys.path)
+        for worker in protocols:
+            args = [sys.executable, workertrialPath]
+            args.extend(arguments)
+            spawner(worker, sys.executable, args=args, childFDs=childFDs, env=environ)
+
+    async def start(self, reactor: IReactorProcess) -> StartedWorkerPool:
+        """
+        Launch all of the workers for this pool.
+
+        @return: A started pool object that can run jobs using the workers.
+        """
+        testDir, testDirLock = _unusedTestDirectory(
+            self._config.workingDirectory,
+        )
+
+        if isabs(self._config.logFile):
+            # Open a log file wherever the user asked.
+            testLogPath = FilePath(self._config.logFile)
+        else:
+            # Open a log file in the chosen working directory (not necessarily
+            # the same as our configured working directory, if that path was
+            # in use).
+            testLogPath = testDir.preauthChild(self._config.logFile)
+        testLog = openTestLog(testLogPath)
+
+        ampWorkers = [LocalWorkerAMP() for x in range(self._config.numWorkers)]
+        workers = self._createLocalWorkers(
+            ampWorkers,
+            testDir,
+            testLog,
+        )
+        self._launchWorkerProcesses(
+            reactor.spawnProcess,
+            workers,
+            self._config.workerArguments,
+        )
+
+        return StartedWorkerPool(
+            testDir,
+            testDirLock,
+            testLog,
+            workers,
+            ampWorkers,
+        )
+
+
+def shouldContinue(untilFailure: bool, result: IReporter) -> bool:
+    """
+    Determine whether the test suite should be iterated again.
+
+    @param untilFailure: C{True} if the suite is supposed to run until
+        failure.
+
+    @param result: The test result of the test suite iteration which just
+        completed.
+    """
+    return untilFailure and result.wasSuccessful()
+
+
+async def runTests(
+    pool: StartedWorkerPool,
+    testCases: Iterable[ITestCase],
+    result: DistReporter,
+    driveWorker: Callable[
+        [DistReporter, Sequence[ITestCase], LocalWorkerAMP], Awaitable[None]
+    ],
+) -> None:
+    try:
+        # Run the tests using the worker pool.
+        await pool.run(partial(driveWorker, result, testCases))
+    except Exception:
+        # Exceptions from test code are handled somewhere else.  An
+        # exception here is a bug in the runner itself.  The only
+        # convenient place to put it is in the result, though.
+        result.original.addError(TestHolder("<runTests>"), Failure())
+
+
+@define
+class DistTrialRunner:
+    """
+    A specialized runner for distributed trial. The runner launches a number of
+    local worker processes which will run tests.
+
+    @ivar _maxWorkers: the number of workers to be spawned.
+
+    @ivar _exitFirst: ``True`` to stop the run as soon as a test case fails.
+        ``False`` to run through the whole suite and report all of the results
+        at the end.
+
+    @ivar stream: stream which the reporter will use.
+
+    @ivar _reporterFactory: the reporter class to be used.
+    """
+
+    _distReporterFactory = DistReporter
+    _logger = Logger()
+
+    # accepts a `realtime` keyword argument which we can't annotate, so punt
+    # on the argument annotation
+    _reporterFactory: Callable[..., IReporter]
+    _maxWorkers: int
+    _workerArguments: List[str]
+    _exitFirst: bool = False
+    _reactor: IDistTrialReactor = field(
+        # mypy doesn't understand the converter
+        default=None,
+        converter=default_if_none(factory=_defaultReactor),  # type: ignore [misc]
+    )
+    # mypy doesn't understand the converter
+    stream: TextIO = field(default=None, converter=default_if_none(sys.stdout))  # type: ignore [misc]
+
+    _tracebackFormat: str = "default"
+    _realTimeErrors: bool = False
+    _uncleanWarnings: bool = False
+    _logfile: str = "test.log"
+    _workingDirectory: str = "_trial_temp"
+    _workerPoolFactory: Callable[[WorkerPoolConfig], WorkerPool] = WorkerPool
+
+    def _makeResult(self) -> DistReporter:
+        """
+        Make reporter factory, and wrap it with a L{DistReporter}.
+        """
+        reporter = self._reporterFactory(
+            self.stream, self._tracebackFormat, realtime=self._realTimeErrors
+        )
+        if self._uncleanWarnings:
+            reporter = UncleanWarningsReporterWrapper(reporter)
+        return self._distReporterFactory(reporter)
+
+    def writeResults(self, result):
+        """
+        Write test run final outcome to result.
+
+        @param result: A C{TestResult} which will print errors and the summary.
+        """
+        result.done()
+
+    async def _driveWorker(
+        self,
+        result: DistReporter,
+        testCases: Sequence[ITestCase],
+        worker: LocalWorkerAMP,
+    ) -> None:
+        """
+        Drive a L{LocalWorkerAMP} instance, iterating the tests and calling
+        C{run} for every one of them.
+
+        @param worker: The L{LocalWorkerAMP} to drive.
+
+        @param result: The global L{DistReporter} instance.
+
+        @param testCases: The global list of tests to iterate.
+
+        @return: A coroutine that completes after all of the tests have
+            completed.
+        """
+
+        async def task(case):
+            try:
+                await worker.run(case, result)
+            except Exception:
+                result.original.addError(case, Failure())
+
+        for case in testCases:
+            await task(case)
+
+    async def runAsync(
+        self,
+        suite: Union[TestCase, TestSuite],
+        untilFailure: bool = False,
+    ) -> DistReporter:
+        """
+        Spawn local worker processes and load tests. After that, run them.
+
+        @param suite: A test or suite to be run.
+
+        @param untilFailure: If C{True}, continue to run the tests until they
+            fail.
+
+        @return: A coroutine that completes with the test result.
+        """
+
+        # Realize a concrete set of tests to run.
+        testCases = list(_iterateTests(suite))
+
+        # Create a worker pool to use to execute them.
+        poolStarter = self._workerPoolFactory(
+            WorkerPoolConfig(
+                # Don't make it larger than is useful or allowed.
+                min(len(testCases), self._maxWorkers),
+                FilePath(self._workingDirectory),
+                self._workerArguments,
+                self._logfile,
+            ),
+        )
+
+        # Announce that we're beginning.  countTestCases result is preferred
+        # (over len(testCases)) because testCases may contain synthetic cases
+        # for error reporting purposes.
+        self.stream.write(f"Running {suite.countTestCases()} tests.\n")
+
+        # Start the worker pool.
+        startedPool = await poolStarter.start(self._reactor)
+
+        # The condition that will determine whether the test run repeats.
+        condition = partial(shouldContinue, untilFailure)
+
+        # A function that will run the whole suite once.
+        @countingCalls
+        async def runAndReport(n: int) -> DistReporter:
+            if untilFailure:
+                # If and only if we're running the suite more than once,
+                # provide a report about which run this is.
+                self.stream.write(f"Test Pass {n + 1}\n")
+
+            result = self._makeResult()
+
+            if self._exitFirst:
+                # Keep giving out tests as long as the result object has only
+                # seen success.
+                casesCondition = lambda _: result.original.wasSuccessful()
+            else:
+                casesCondition = lambda _: True
+
+            await runTests(
+                startedPool,
+                takeWhile(casesCondition, testCases),
+                result,
+                self._driveWorker,
+            )
+            self.writeResults(result)
+            return result
+
+        try:
+            # Start submitting tests to workers in the pool.  Perhaps repeat
+            # the whole test suite more than once, if appropriate for our
+            # configuration.
+            return await iterateWhile(condition, runAndReport)
+        finally:
+            # Shut down the worker pool.
+            await startedPool.join()
+
+    def _run(self, test: Union[TestCase, TestSuite], untilFailure: bool) -> IReporter:
+        result: Union[Failure, DistReporter, None] = None
+        reactorStopping: bool = False
+        testsInProgress: Deferred[object]
+
+        def capture(r: Union[Failure, DistReporter]) -> None:
+            nonlocal result
+            result = r
+
+        def maybeStopTests() -> Optional[Deferred[object]]:
+            nonlocal reactorStopping
+            reactorStopping = True
+            if result is None:
+                testsInProgress.cancel()
+                return testsInProgress
+            return None
+
+        def maybeStopReactor(result: object) -> object:
+            if not reactorStopping:
+                self._reactor.stop()
+            return result
+
+        self._reactor.addSystemEventTrigger("before", "shutdown", maybeStopTests)
+
+        testsInProgress = (
+            Deferred.fromCoroutine(self.runAsync(test, untilFailure))
+            .addBoth(capture)
+            .addBoth(maybeStopReactor)
+        )
+
+        self._reactor.run()
+
+        if isinstance(result, Failure):
+            result.raiseException()
+
+        # mypy can't see that raiseException raises an exception so we can
+        # only get here if result is not a Failure, so tell mypy result is
+        # certainly a DistReporter at this point.
+        assert isinstance(result, DistReporter), f"{result} is not DistReporter"
+
+        # Unwrap the DistReporter to give the caller some regular IReporter
+        # object.  DistReporter isn't type annotated correctly so fix it here.
+        return cast(IReporter, result.original)
+
+    def run(self, test: Union[TestCase, TestSuite]) -> IReporter:
+        """
+        Run a reactor and a test suite.
+
+        @param test: The test or suite to run.
+        """
+        return self._run(test, untilFailure=False)
+
+    def runUntilFailure(self, test: Union[TestCase, TestSuite]) -> IReporter:
+        """
+        Run the tests with local worker processes until they fail.
+
+        @param test: The test or suite to run.
+        """
+        return self._run(test, untilFailure=True)
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/functional.py b/contrib/python/Twisted/py3/twisted/trial/_dist/functional.py
new file mode 100644
index 00000000000..3db4dca5de5
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/functional.py
@@ -0,0 +1,125 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+General functional-style helpers for disttrial.
+"""
+
+from functools import partial, wraps
+from typing import Awaitable, Callable, Iterable, Optional, TypeVar
+
+from twisted.internet.defer import Deferred, succeed
+
+_A = TypeVar("_A")
+_B = TypeVar("_B")
+_C = TypeVar("_C")
+
+
+def fromOptional(default: _A, optional: Optional[_A]) -> _A:
+    """
+    Get a definite value from an optional value.
+
+    @param default: The value to return if the optional value is missing.
+
+    @param optional: The optional value to return if it exists.
+    """
+    if optional is None:
+        return default
+    return optional
+
+
+def takeWhile(condition: Callable[[_A], bool], xs: Iterable[_A]) -> Iterable[_A]:
+    """
+    :return: An iterable over C{xs} that stops when C{condition} returns
+        ``False`` based on the value of iterated C{xs}.
+    """
+    for x in xs:
+        if condition(x):
+            yield x
+        else:
+            break
+
+
+async def sequence(a: Awaitable[_A], b: Awaitable[_B]) -> _B:
+    """
+    Wait for one action to complete and then another.
+
+    If either action fails, failure is propagated.  If the first action fails,
+    the second action is not waited on.
+    """
+    await a
+    return await b
+
+
+def flip(f: Callable[[_A, _B], _C]) -> Callable[[_B, _A], _C]:
+    """
+    Create a function like another but with the order of the first two
+    arguments flipped.
+    """
+
+    @wraps(f)
+    def g(b, a):
+        return f(a, b)
+
+    return g
+
+
+def compose(fx: Callable[[_B], _C], fy: Callable[[_A], _B]) -> Callable[[_A], _C]:
+    """
+    Create a function that calls one function with an argument and then
+    another function with the result of the first function.
+    """
+
+    @wraps(fx)
+    @wraps(fy)
+    def g(a):
+        return fx(fy(a))
+
+    return g
+
+
+# Discard the result of an awaitable and substitute None in its place.
+#
+# Ignore the `Cannot infer type argument 1 of "compose"`
+# https://github.com/python/mypy/issues/6220
+discardResult: Callable[[Awaitable[_A]], Deferred[None]] = compose(  # type: ignore[misc]
+    Deferred.fromCoroutine,
+    partial(flip(sequence), succeed(None)),
+)
+
+
+async def iterateWhile(
+    predicate: Callable[[_A], bool],
+    action: Callable[[], Awaitable[_A]],
+) -> _A:
+    """
+    Call a function repeatedly until its result fails to satisfy a predicate.
+
+    @param predicate: The check to apply.
+
+    @param action: The function to call.
+
+    @return: The result of C{action} which did not satisfy C{predicate}.
+    """
+    while True:
+        result = await action()
+        if not predicate(result):
+            return result
+
+
+def countingCalls(f: Callable[[int], _A]) -> Callable[[], _A]:
+    """
+    Wrap a function with another that automatically passes an integer counter
+    of the number of calls that have gone through the wrapper.
+    """
+    counter = 0
+
+    def g() -> _A:
+        nonlocal counter
+        try:
+            result = f(counter)
+        finally:
+            counter += 1
+        return result
+
+    return g
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/managercommands.py b/contrib/python/Twisted/py3/twisted/trial/_dist/managercommands.py
new file mode 100644
index 00000000000..4f3080a24f7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/managercommands.py
@@ -0,0 +1,89 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Commands for reporting test success of failure to the manager.
+
+@since: 12.3
+"""
+
+from twisted.protocols.amp import Boolean, Command, Integer, Unicode
+
+NativeString = Unicode
+
+
+class AddSuccess(Command):
+    """
+    Add a success.
+    """
+
+    arguments = [(b"testName", NativeString())]
+    response = [(b"success", Boolean())]
+
+
+class AddError(Command):
+    """
+    Add an error.
+    """
+
+    arguments = [
+        (b"testName", NativeString()),
+        (b"errorClass", NativeString()),
+        (b"errorStreamId", Integer()),
+        (b"framesStreamId", Integer()),
+    ]
+    response = [(b"success", Boolean())]
+
+
+class AddFailure(Command):
+    """
+    Add a failure.
+    """
+
+    arguments = [
+        (b"testName", NativeString()),
+        (b"failStreamId", Integer()),
+        (b"failClass", NativeString()),
+        (b"framesStreamId", Integer()),
+    ]
+    response = [(b"success", Boolean())]
+
+
+class AddSkip(Command):
+    """
+    Add a skip.
+    """
+
+    arguments = [(b"testName", NativeString()), (b"reason", NativeString())]
+    response = [(b"success", Boolean())]
+
+
+class AddExpectedFailure(Command):
+    """
+    Add an expected failure.
+    """
+
+    arguments = [
+        (b"testName", NativeString()),
+        (b"errorStreamId", Integer()),
+        (b"todo", NativeString()),
+    ]
+    response = [(b"success", Boolean())]
+
+
+class AddUnexpectedSuccess(Command):
+    """
+    Add an unexpected success.
+    """
+
+    arguments = [(b"testName", NativeString()), (b"todo", NativeString())]
+    response = [(b"success", Boolean())]
+
+
+class TestWrite(Command):
+    """
+    Write test log.
+    """
+
+    arguments = [(b"out", NativeString())]
+    response = [(b"success", Boolean())]
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/options.py b/contrib/python/Twisted/py3/twisted/trial/_dist/options.py
new file mode 100644
index 00000000000..19f9a5f6cfc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/options.py
@@ -0,0 +1,28 @@
+# -*- test-case-name: twisted.trial._dist.test.test_options -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Options handling specific to trial's workers.
+
+@since: 12.3
+"""
+
+from twisted.application.app import ReactorSelectionMixin
+from twisted.python.filepath import FilePath
+from twisted.python.usage import Options
+from twisted.scripts.trial import _BasicOptions
+
+
+class WorkerOptions(_BasicOptions, Options, ReactorSelectionMixin):
+    """
+    Options forwarded to the trial distributed worker.
+    """
+
+    def coverdir(self):
+        """
+        Return a L{FilePath} representing the directory into which coverage
+        results should be written.
+        """
+        return FilePath("coverage")
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/stream.py b/contrib/python/Twisted/py3/twisted/trial/_dist/stream.py
new file mode 100644
index 00000000000..a53fd4ab214
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/stream.py
@@ -0,0 +1,100 @@
+"""
+Buffer byte streams.
+"""
+
+from itertools import count
+from typing import Dict, Iterator, List, TypeVar
+
+from attrs import Factory, define
+
+from twisted.protocols.amp import AMP, Command, Integer, String as Bytes
+
+T = TypeVar("T")
+
+
+class StreamOpen(Command):
+    """
+    Open a new stream.
+    """
+
+    response = [(b"streamId", Integer())]
+
+
+class StreamWrite(Command):
+    """
+    Write a chunk of data to a stream.
+    """
+
+    arguments = [
+        (b"streamId", Integer()),
+        (b"data", Bytes()),
+    ]
+
+
+@define
+class StreamReceiver:
+    """
+    Buffering de-multiplexing byte stream receiver.
+    """
+
+    _counter: Iterator[int] = count()
+    _streams: Dict[int, List[bytes]] = Factory(dict)
+
+    def open(self) -> int:
+        """
+        Open a new stream and return its unique identifier.
+        """
+        newId = next(self._counter)
+        self._streams[newId] = []
+        return newId
+
+    def write(self, streamId: int, chunk: bytes) -> None:
+        """
+        Write to an open stream using its unique identifier.
+
+        @raise KeyError: If there is no such open stream.
+        """
+        self._streams[streamId].append(chunk)
+
+    def finish(self, streamId: int) -> List[bytes]:
+        """
+        Indicate an open stream may receive no further data and return all of
+        its current contents.
+
+        @raise KeyError: If there is no such open stream.
+        """
+        return self._streams.pop(streamId)
+
+
+def chunk(data: bytes, chunkSize: int) -> Iterator[bytes]:
+    """
+    Break a byte string into pieces of no more than ``chunkSize`` length.
+
+    @param data: The byte string.
+
+    @param chunkSize: The maximum length of the resulting pieces.  All pieces
+        except possibly the last will be this length.
+
+    @return: The pieces.
+    """
+    pos = 0
+    while pos < len(data):
+        yield data[pos : pos + chunkSize]
+        pos += chunkSize
+
+
+async def stream(amp: AMP, chunks: Iterator[bytes]) -> int:
+    """
+    Send the given stream chunks, one by one, over the given connection.
+
+    The chunks are sent using L{StreamWrite} over a stream opened using
+    L{StreamOpen}.
+
+    @return: The identifier of the stream over which the chunks were sent.
+    """
+    streamId = (await amp.callRemote(StreamOpen))["streamId"]
+    assert isinstance(streamId, int)
+
+    for oneChunk in chunks:
+        await amp.callRemote(StreamWrite, streamId=streamId, data=oneChunk)
+    return streamId
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/worker.py b/contrib/python/Twisted/py3/twisted/trial/_dist/worker.py
new file mode 100644
index 00000000000..77e502173ad
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/worker.py
@@ -0,0 +1,465 @@
+# -*- test-case-name: twisted.trial._dist.test.test_worker -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This module implements the worker classes.
+
+@since: 12.3
+"""
+
+import os
+from typing import Any, Awaitable, Callable, Dict, List, Optional, TextIO, TypeVar
+from unittest import TestCase
+
+from zope.interface import implementer
+
+from attrs import frozen
+from typing_extensions import Protocol, TypedDict
+
+from twisted.internet.defer import Deferred, DeferredList
+from twisted.internet.error import ProcessDone
+from twisted.internet.interfaces import IAddress, ITransport
+from twisted.internet.protocol import ProcessProtocol
+from twisted.logger import Logger
+from twisted.protocols.amp import AMP
+from twisted.python.failure import Failure
+from twisted.python.filepath import FilePath
+from twisted.python.reflect import namedObject
+from twisted.trial._dist import (
+    _WORKER_AMP_STDIN,
+    _WORKER_AMP_STDOUT,
+    managercommands,
+    workercommands,
+)
+from twisted.trial._dist.workerreporter import WorkerReporter
+from twisted.trial.reporter import TestResult
+from twisted.trial.runner import TestLoader, TrialSuite
+from twisted.trial.unittest import Todo
+from .stream import StreamOpen, StreamReceiver, StreamWrite
+
+
+@frozen(auto_exc=False)
+class WorkerException(Exception):
+    """
+    An exception was reported by a test running in a worker process.
+
+    @ivar message: An error message describing the exception.
+    """
+
+    message: str
+
+
+class RunResult(TypedDict):
+    """
+    Represent the result of a L{workercommands.Run} command.
+    """
+
+    success: bool
+
+
+class Worker(Protocol):
+    """
+    An object that can run actions.
+    """
+
+    async def run(self, case: TestCase, result: TestResult) -> RunResult:
+        """
+        Run a test case.
+        """
+
+
+_T = TypeVar("_T")
+WorkerAction = Callable[[Worker], Awaitable[_T]]
+
+
+class WorkerProtocol(AMP):
+    """
+    The worker-side trial distributed protocol.
+    """
+
+    logger = Logger()
+
+    def __init__(self, forceGarbageCollection=False):
+        self._loader = TestLoader()
+        self._result = WorkerReporter(self)
+        self._forceGarbageCollection = forceGarbageCollection
+
+    @workercommands.Run.responder
+    async def run(self, testCase: str) -> RunResult:
+        """
+        Run a test case by name.
+        """
+        with self._result.gatherReportingResults() as results:
+            case = self._loader.loadByName(testCase)
+            suite = TrialSuite([case], self._forceGarbageCollection)
+            suite.run(self._result)
+
+        allSucceeded = True
+        for success, result in await DeferredList(results, consumeErrors=True):
+            if success:
+                # Nothing to do here, proceed to the next result.
+                continue
+
+            # There was some error reporting a result to the peer.
+            allSucceeded = False
+
+            # We can try to report the error but since something has already
+            # gone wrong we shouldn't be extremely confident that this will
+            # succeed.  So we will also log it (and any errors reporting *it*)
+            # to our local log.
+            self.logger.failure(
+                "Result reporting for {id} failed",
+                # The DeferredList type annotation assumes all results succeed
+                failure=result,  # type: ignore[arg-type]
+                id=testCase,
+            )
+            try:
+                await self._result.addErrorFallible(
+                    testCase,
+                    # The DeferredList type annotation assumes all results succeed
+                    result,  # type: ignore[arg-type]
+                )
+            except BaseException:
+                # We failed to report the failure to the peer.  It doesn't
+                # seem very likely that reporting this new failure to the peer
+                # will succeed so just log it locally.
+                self.logger.failure(
+                    "Additionally, reporting the reporting failure failed."
+                )
+
+        return {"success": allSucceeded}
+
+    @workercommands.Start.responder
+    def start(self, directory):
+        """
+        Set up the worker, moving into given directory for tests to run in
+        them.
+        """
+        os.chdir(directory)
+        return {"success": True}
+
+
+class LocalWorkerAMP(AMP):
+    """
+    Local implementation of the manager commands.
+    """
+
+    def __init__(self, boxReceiver=None, locator=None):
+        super().__init__(boxReceiver, locator)
+        self._streams = StreamReceiver()
+
+    @StreamOpen.responder
+    def streamOpen(self):
+        return {"streamId": self._streams.open()}
+
+    @StreamWrite.responder
+    def streamWrite(self, streamId, data):
+        self._streams.write(streamId, data)
+        return {}
+
+    @managercommands.AddSuccess.responder
+    def addSuccess(self, testName):
+        """
+        Add a success to the reporter.
+        """
+        self._result.addSuccess(self._testCase)
+        return {"success": True}
+
+    def _buildFailure(
+        self,
+        error: WorkerException,
+        errorClass: str,
+        frames: List[str],
+    ) -> Failure:
+        """
+        Helper to build a C{Failure} with some traceback.
+
+        @param error: An C{Exception} instance.
+
+        @param errorClass: The class name of the C{error} class.
+
+        @param frames: A flat list of strings representing the information need
+            to approximatively rebuild C{Failure} frames.
+
+        @return: A L{Failure} instance with enough information about a test
+           error.
+        """
+        errorType = namedObject(errorClass)
+        failure = Failure(error, errorType)
+        for i in range(0, len(frames), 3):
+            failure.frames.append(
+                (frames[i], frames[i + 1], int(frames[i + 2]), [], [])
+            )
+        return failure
+
+    @managercommands.AddError.responder
+    def addError(
+        self,
+        testName: str,
+        errorClass: str,
+        errorStreamId: int,
+        framesStreamId: int,
+    ) -> Dict[str, bool]:
+        """
+        Add an error to the reporter.
+
+        @param errorStreamId: The identifier of a stream over which the text
+            of this error was previously completely sent to the peer.
+
+        @param framesStreamId: The identifier of a stream over which the lines
+            of the traceback for this error were previously completely sent to
+            the peer.
+
+        @param error: A message describing the error.
+        """
+        error = b"".join(self._streams.finish(errorStreamId)).decode("utf-8")
+        frames = [
+            frame.decode("utf-8") for frame in self._streams.finish(framesStreamId)
+        ]
+        # Wrap the error message in ``WorkerException`` because it is not
+        # possible to transfer arbitrary exception values over the AMP
+        # connection to the main process but we must give *some* Exception
+        # (not a str) to the test result object.
+        failure = self._buildFailure(WorkerException(error), errorClass, frames)
+        self._result.addError(self._testCase, failure)
+        return {"success": True}
+
+    @managercommands.AddFailure.responder
+    def addFailure(
+        self,
+        testName: str,
+        failStreamId: int,
+        failClass: str,
+        framesStreamId: int,
+    ) -> Dict[str, bool]:
+        """
+        Add a failure to the reporter.
+
+        @param failStreamId: The identifier of a stream over which the text of
+            this failure was previously completely sent to the peer.
+
+        @param framesStreamId: The identifier of a stream over which the lines
+            of the traceback for this error were previously completely sent to the
+            peer.
+        """
+        fail = b"".join(self._streams.finish(failStreamId)).decode("utf-8")
+        frames = [
+            frame.decode("utf-8") for frame in self._streams.finish(framesStreamId)
+        ]
+        # See addError for info about use of WorkerException here.
+        failure = self._buildFailure(WorkerException(fail), failClass, frames)
+        self._result.addFailure(self._testCase, failure)
+        return {"success": True}
+
+    @managercommands.AddSkip.responder
+    def addSkip(self, testName, reason):
+        """
+        Add a skip to the reporter.
+        """
+        self._result.addSkip(self._testCase, reason)
+        return {"success": True}
+
+    @managercommands.AddExpectedFailure.responder
+    def addExpectedFailure(
+        self, testName: str, errorStreamId: int, todo: Optional[str]
+    ) -> Dict[str, bool]:
+        """
+        Add an expected failure to the reporter.
+
+        @param errorStreamId: The identifier of a stream over which the text
+            of this error was previously completely sent to the peer.
+        """
+        error = b"".join(self._streams.finish(errorStreamId)).decode("utf-8")
+        _todo = Todo("<unknown>" if todo is None else todo)
+        self._result.addExpectedFailure(self._testCase, error, _todo)
+        return {"success": True}
+
+    @managercommands.AddUnexpectedSuccess.responder
+    def addUnexpectedSuccess(self, testName, todo):
+        """
+        Add an unexpected success to the reporter.
+        """
+        self._result.addUnexpectedSuccess(self._testCase, todo)
+        return {"success": True}
+
+    @managercommands.TestWrite.responder
+    def testWrite(self, out):
+        """
+        Print test output from the worker.
+        """
+        self._testStream.write(out + "\n")
+        self._testStream.flush()
+        return {"success": True}
+
+    async def run(self, testCase: TestCase, result: TestResult) -> RunResult:
+        """
+        Run a test.
+        """
+        self._testCase = testCase
+        self._result = result
+        self._result.startTest(testCase)
+        testCaseId = testCase.id()
+        try:
+            return await self.callRemote(workercommands.Run, testCase=testCaseId)  # type: ignore[no-any-return]
+        finally:
+            self._result.stopTest(testCase)
+
+    def setTestStream(self, stream):
+        """
+        Set the stream used to log output from tests.
+        """
+        self._testStream = stream
+
+
+@implementer(IAddress)
+class LocalWorkerAddress:
+    """
+    A L{IAddress} implementation meant to provide stub addresses for
+    L{ITransport.getPeer} and L{ITransport.getHost}.
+    """
+
+
+@implementer(ITransport)
+class LocalWorkerTransport:
+    """
+    A stub transport implementation used to support L{AMP} over a
+    L{ProcessProtocol} transport.
+    """
+
+    def __init__(self, transport):
+        self._transport = transport
+
+    def write(self, data):
+        """
+        Forward data to transport.
+        """
+        self._transport.writeToChild(_WORKER_AMP_STDIN, data)
+
+    def writeSequence(self, sequence):
+        """
+        Emulate C{writeSequence} by iterating data in the C{sequence}.
+        """
+        for data in sequence:
+            self._transport.writeToChild(_WORKER_AMP_STDIN, data)
+
+    def loseConnection(self):
+        """
+        Closes the transport.
+        """
+        self._transport.loseConnection()
+
+    def getHost(self):
+        """
+        Return a L{LocalWorkerAddress} instance.
+        """
+        return LocalWorkerAddress()
+
+    def getPeer(self):
+        """
+        Return a L{LocalWorkerAddress} instance.
+        """
+        return LocalWorkerAddress()
+
+
+class NotRunning(Exception):
+    """
+    An operation was attempted on a worker process which is not running.
+    """
+
+
+class LocalWorker(ProcessProtocol):
+    """
+    Local process worker protocol. This worker runs as a local process and
+    communicates via stdin/out.
+
+    @ivar _ampProtocol: The L{AMP} protocol instance used to communicate with
+        the worker.
+
+    @ivar _logDirectory: The directory where logs will reside.
+
+    @ivar _logFile: The main log file for tests output.
+    """
+
+    def __init__(
+        self,
+        ampProtocol: LocalWorkerAMP,
+        logDirectory: FilePath[Any],
+        logFile: TextIO,
+    ):
+        self._ampProtocol = ampProtocol
+        self._logDirectory = logDirectory
+        self._logFile = logFile
+        self.endDeferred: Deferred[None] = Deferred()
+
+    async def exit(self) -> None:
+        """
+        Cause the worker process to exit.
+        """
+        if self.transport is None:
+            raise NotRunning()
+
+        endDeferred = self.endDeferred
+        self.transport.closeChildFD(_WORKER_AMP_STDIN)
+        try:
+            await endDeferred
+        except ProcessDone:
+            pass
+
+    def connectionMade(self):
+        """
+        When connection is made, create the AMP protocol instance.
+        """
+        self._ampProtocol.makeConnection(LocalWorkerTransport(self.transport))
+        self._logDirectory.makedirs(ignoreExistingDirectory=True)
+        self._outLog = self._logDirectory.child("out.log").open("w")
+        self._errLog = self._logDirectory.child("err.log").open("w")
+        self._ampProtocol.setTestStream(self._logFile)
+        d = self._ampProtocol.callRemote(
+            workercommands.Start,
+            directory=self._logDirectory.path,
+        )
+        # Ignore the potential errors, the test suite will fail properly and it
+        # would just print garbage.
+        d.addErrback(lambda x: None)
+
+    def connectionLost(self, reason):
+        """
+        On connection lost, close the log files that we're managing for stdin
+        and stdout.
+        """
+        self._outLog.close()
+        self._errLog.close()
+        self.transport = None
+
+    def processEnded(self, reason: Failure) -> None:
+        """
+        When the process closes, call C{connectionLost} for cleanup purposes
+        and forward the information to the C{_ampProtocol}.
+        """
+        self.connectionLost(reason)
+        self._ampProtocol.connectionLost(reason)
+        self.endDeferred.callback(reason)
+
+    def outReceived(self, data):
+        """
+        Send data received from stdout to log.
+        """
+
+        self._outLog.write(data)
+
+    def errReceived(self, data):
+        """
+        Write error data to log.
+        """
+        self._errLog.write(data)
+
+    def childDataReceived(self, childFD, data):
+        """
+        Handle data received on the specific pipe for the C{_ampProtocol}.
+        """
+        if childFD == _WORKER_AMP_STDOUT:
+            self._ampProtocol.dataReceived(data)
+        else:
+            ProcessProtocol.childDataReceived(self, childFD, data)
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/workercommands.py b/contrib/python/Twisted/py3/twisted/trial/_dist/workercommands.py
new file mode 100644
index 00000000000..f7d8d26b2ed
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/workercommands.py
@@ -0,0 +1,30 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Commands for telling a worker to load tests or run tests.
+
+@since: 12.3
+"""
+
+from twisted.protocols.amp import Boolean, Command, Unicode
+
+NativeString = Unicode
+
+
+class Run(Command):
+    """
+    Run a test.
+    """
+
+    arguments = [(b"testCase", NativeString())]
+    response = [(b"success", Boolean())]
+
+
+class Start(Command):
+    """
+    Set up the worker process, giving the running directory.
+    """
+
+    arguments = [(b"directory", NativeString())]
+    response = [(b"success", Boolean())]
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/workerreporter.py b/contrib/python/Twisted/py3/twisted/trial/_dist/workerreporter.py
new file mode 100644
index 00000000000..5f2d7a0cab1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/workerreporter.py
@@ -0,0 +1,354 @@
+# -*- test-case-name: twisted.trial._dist.test.test_workerreporter -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Test reporter forwarding test results over trial distributed AMP commands.
+
+@since: 12.3
+"""
+
+from types import TracebackType
+from typing import Callable, List, Optional, Sequence, Type, TypeVar
+from unittest import TestCase as PyUnitTestCase
+
+from attrs import Factory, define
+from typing_extensions import Literal
+
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.protocols.amp import AMP, MAX_VALUE_LENGTH
+from twisted.python.failure import Failure
+from twisted.python.reflect import qual
+from twisted.trial._dist import managercommands
+from twisted.trial.reporter import TestResult
+from ..reporter import TrialFailure
+from .stream import chunk, stream
+
+T = TypeVar("T")
+
+
+async def addError(
+    amp: AMP, testName: str, errorClass: str, error: str, frames: List[str]
+) -> None:
+    """
+    Send an error to the worker manager over an AMP connection.
+
+    First the pieces which can be large are streamed over the connection.
+    Then, L{managercommands.AddError} is called with the rest of the
+    information and the stream IDs.
+
+    :param amp: The connection to use.
+    :param testName: The name (or ID) of the test the error relates to.
+    :param errorClass: The fully qualified name of the error type.
+    :param error: The string representation of the error.
+    :param frames: The lines of the traceback associated with the error.
+    """
+
+    errorStreamId = await stream(amp, chunk(error.encode("utf-8"), MAX_VALUE_LENGTH))
+    framesStreamId = await stream(amp, (frame.encode("utf-8") for frame in frames))
+
+    await amp.callRemote(
+        managercommands.AddError,
+        testName=testName,
+        errorClass=errorClass,
+        errorStreamId=errorStreamId,
+        framesStreamId=framesStreamId,
+    )
+
+
+async def addFailure(
+    amp: AMP, testName: str, fail: str, failClass: str, frames: List[str]
+) -> None:
+    """
+    Like L{addError} but for failures.
+
+    :param amp: See L{addError}
+    :param testName: See L{addError}
+    :param failClass: The fully qualified name of the exception associated
+        with the failure.
+    :param fail: The string representation of the failure.
+    :param frames: The lines of the traceback associated with the error.
+    """
+    failStreamId = await stream(amp, chunk(fail.encode("utf-8"), MAX_VALUE_LENGTH))
+    framesStreamId = await stream(amp, (frame.encode("utf-8") for frame in frames))
+
+    await amp.callRemote(
+        managercommands.AddFailure,
+        testName=testName,
+        failClass=failClass,
+        failStreamId=failStreamId,
+        framesStreamId=framesStreamId,
+    )
+
+
+async def addExpectedFailure(amp: AMP, testName: str, error: str, todo: str) -> None:
+    """
+    Like L{addError} but for expected failures.
+
+    :param amp: See L{addError}
+    :param testName: See L{addError}
+    :param error: The string representation of the expected failure.
+    :param todo: The string description of the expectation.
+    """
+    errorStreamId = await stream(amp, chunk(error.encode("utf-8"), MAX_VALUE_LENGTH))
+
+    await amp.callRemote(
+        managercommands.AddExpectedFailure,
+        testName=testName,
+        errorStreamId=errorStreamId,
+        todo=todo,
+    )
+
+
+@define
+class ReportingResults:
+    """
+    A mutable container for the result of sending test results back to the
+    parent process.
+
+    Since it is possible for these sends to fail asynchronously but the
+    L{TestResult} protocol is not well suited for asynchronous result
+    reporting, results are collected on an instance of this class and when the
+    runner believes the test is otherwise complete, it can collect the results
+    and do something with any errors.
+
+    :ivar _reporter: The L{WorkerReporter} this object is associated with.
+        This is the object doing the result reporting.
+
+    :ivar _results: A list of L{Deferred} instances representing the results
+        of reporting operations.  This is expected to grow over the course of
+        the test run and then be inspected by the runner once the test is
+        over.  The public interface to this list is via the context manager
+        interface.
+    """
+
+    _reporter: "WorkerReporter"
+    _results: List[Deferred[object]] = Factory(list)
+
+    def __enter__(self) -> Sequence[Deferred[object]]:
+        """
+        Begin a new reportable context in which results can be collected.
+
+        :return: A sequence which will contain the L{Deferred} instances
+            representing the results of all test result reporting that happens
+            while the context manager is active.  The sequence is extended as
+            the test runs so its value should not be consumed until the test
+            is over.
+        """
+        return self._results
+
+    def __exit__(
+        self,
+        excType: Type[BaseException],
+        excValue: BaseException,
+        excTraceback: TracebackType,
+    ) -> Literal[False]:
+        """
+        End the reportable context.
+        """
+        self._reporter._reporting = None
+        return False
+
+    def record(self, result: Deferred[object]) -> None:
+        """
+        Record a L{Deferred} instance representing one test result reporting
+        operation.
+        """
+        self._results.append(result)
+
+
+class WorkerReporter(TestResult):
+    """
+    Reporter for trial's distributed workers. We send things not through a
+    stream, but through an C{AMP} protocol's C{callRemote} method.
+
+    @ivar _DEFAULT_TODO: Default message for expected failures and
+        unexpected successes, used only if a C{Todo} is not provided.
+
+    @ivar _reporting: When a "result reporting" context is active, the
+        corresponding context manager.  Otherwise, L{None}.
+    """
+
+    _DEFAULT_TODO = "Test expected to fail"
+
+    ampProtocol: AMP
+    _reporting: Optional[ReportingResults] = None
+
+    def __init__(self, ampProtocol):
+        """
+        @param ampProtocol: The communication channel with the trial
+            distributed manager which collects all test results.
+        """
+        super().__init__()
+        self.ampProtocol = ampProtocol
+
+    def gatherReportingResults(self) -> ReportingResults:
+        """
+        Get a "result reporting" context manager.
+
+        In a "result reporting" context, asynchronous test result reporting
+        methods may be used safely.  Their results (in particular, failures)
+        are available from the context manager.
+        """
+        self._reporting = ReportingResults(self)
+        return self._reporting
+
+    def _getFailure(self, error: TrialFailure) -> Failure:
+        """
+        Convert a C{sys.exc_info()}-style tuple to a L{Failure}, if necessary.
+        """
+        if isinstance(error, tuple):
+            return Failure(error[1], error[0], error[2])
+        return error
+
+    def _getFrames(self, failure: Failure) -> List[str]:
+        """
+        Extract frames from a C{Failure} instance.
+        """
+        frames: List[str] = []
+        for frame in failure.frames:
+            # The code object's name, the code object's filename, and the line
+            # number.
+            frames.extend([frame[0], frame[1], str(frame[2])])
+        return frames
+
+    def _call(self, f: Callable[[], T]) -> None:
+        """
+        Call L{f} if and only if a "result reporting" context is active.
+
+        @param f: A function to call.  Its result is accumulated into the
+            result reporting context.  It may return a L{Deferred} or a
+            coroutine or synchronously raise an exception or return a result
+            value.
+
+        @raise ValueError: If no result reporting context is active.
+        """
+        if self._reporting is not None:
+            self._reporting.record(maybeDeferred(f))
+        else:
+            raise ValueError(
+                "Cannot call command outside of reporting context manager."
+            )
+
+    def addSuccess(self, test: PyUnitTestCase) -> None:
+        """
+        Send a success to the parent process.
+
+        This must be called in context managed by L{gatherReportingResults}.
+        """
+        super().addSuccess(test)
+        testName = test.id()
+        self._call(
+            lambda: self.ampProtocol.callRemote(
+                managercommands.AddSuccess, testName=testName
+            )
+        )
+
+    async def addErrorFallible(self, testName: str, errorObj: TrialFailure) -> None:
+        """
+        Attempt to report an error to the parent process.
+
+        Unlike L{addError} this can fail asynchronously.  This version is for
+        infrastructure code that can apply its own failure handling.
+
+        @return: A L{Deferred} that fires with the result of the attempt.
+        """
+        failure = self._getFailure(errorObj)
+        errorStr = failure.getErrorMessage()
+        errorClass = qual(failure.type)
+        frames = self._getFrames(failure)
+        await addError(
+            self.ampProtocol,
+            testName,
+            errorClass,
+            errorStr,
+            frames,
+        )
+
+    def addError(self, test: PyUnitTestCase, error: TrialFailure) -> None:
+        """
+        Send an error to the parent process.
+        """
+        super().addError(test, error)
+        testName = test.id()
+        self._call(lambda: self.addErrorFallible(testName, error))
+
+    def addFailure(self, test: PyUnitTestCase, fail: TrialFailure) -> None:
+        """
+        Send a Failure over.
+        """
+        super().addFailure(test, fail)
+        testName = test.id()
+        failure = self._getFailure(fail)
+        failureMessage = failure.getErrorMessage()
+        failClass = qual(failure.type)
+        frames = self._getFrames(failure)
+        self._call(
+            lambda: addFailure(
+                self.ampProtocol,
+                testName,
+                failureMessage,
+                failClass,
+                frames,
+            ),
+        )
+
+    def addSkip(self, test, reason):
+        """
+        Send a skip over.
+        """
+        super().addSkip(test, reason)
+        reason = str(reason)
+        testName = test.id()
+        self._call(
+            lambda: self.ampProtocol.callRemote(
+                managercommands.AddSkip, testName=testName, reason=reason
+            )
+        )
+
+    def _getTodoReason(self, todo):
+        """
+        Get the reason for a C{Todo}.
+
+        If C{todo} is L{None}, return a sensible default.
+        """
+        if todo is None:
+            return self._DEFAULT_TODO
+        else:
+            return todo.reason
+
+    def addExpectedFailure(self, test, error, todo=None):
+        """
+        Send an expected failure over.
+        """
+        super().addExpectedFailure(test, error, todo)
+        errorMessage = error.getErrorMessage()
+        testName = test.id()
+        self._call(
+            lambda: addExpectedFailure(
+                self.ampProtocol,
+                testName=testName,
+                error=errorMessage,
+                todo=self._getTodoReason(todo),
+            )
+        )
+
+    def addUnexpectedSuccess(self, test, todo=None):
+        """
+        Send an unexpected success over.
+        """
+        super().addUnexpectedSuccess(test, todo)
+        testName = test.id()
+        self._call(
+            lambda: self.ampProtocol.callRemote(
+                managercommands.AddUnexpectedSuccess,
+                testName=testName,
+                todo=self._getTodoReason(todo),
+            )
+        )
+
+    def printSummary(self):
+        """
+        I{Don't} print a summary
+        """
diff --git a/contrib/python/Twisted/py3/twisted/trial/_dist/workertrial.py b/contrib/python/Twisted/py3/twisted/trial/_dist/workertrial.py
new file mode 100644
index 00000000000..847dbc51a6f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_dist/workertrial.py
@@ -0,0 +1,93 @@
+# -*- test-case-name: twisted.trial._dist.test.test_workertrial -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of C{AMP} worker commands, and main executable entry point for
+the workers.
+
+@since: 12.3
+"""
+
+import errno
+import os
+import sys
+
+from twisted.internet.protocol import FileWrapper
+from twisted.python.log import startLoggingWithObserver, textFromEventDict
+from twisted.trial._dist import _WORKER_AMP_STDIN, _WORKER_AMP_STDOUT
+from twisted.trial._dist.options import WorkerOptions
+
+
+class WorkerLogObserver:
+    """
+    A log observer that forward its output to a C{AMP} protocol.
+    """
+
+    def __init__(self, protocol):
+        """
+        @param protocol: a connected C{AMP} protocol instance.
+        @type protocol: C{AMP}
+        """
+        self.protocol = protocol
+
+    def emit(self, eventDict):
+        """
+        Produce a log output.
+        """
+        from twisted.trial._dist import managercommands
+
+        text = textFromEventDict(eventDict)
+        if text is None:
+            return
+        self.protocol.callRemote(managercommands.TestWrite, out=text)
+
+
+def main(_fdopen=os.fdopen):
+    """
+    Main function to be run if __name__ == "__main__".
+
+    @param _fdopen: If specified, the function to use in place of C{os.fdopen}.
+    @type _fdopen: C{callable}
+    """
+    config = WorkerOptions()
+    config.parseOptions()
+
+    from twisted.trial._dist.worker import WorkerProtocol
+
+    workerProtocol = WorkerProtocol(config["force-gc"])
+
+    protocolIn = _fdopen(_WORKER_AMP_STDIN, "rb")
+    protocolOut = _fdopen(_WORKER_AMP_STDOUT, "wb")
+    workerProtocol.makeConnection(FileWrapper(protocolOut))
+
+    observer = WorkerLogObserver(workerProtocol)
+    startLoggingWithObserver(observer.emit, False)
+
+    while True:
+        try:
+            r = protocolIn.read(1)
+        except OSError as e:
+            if e.args[0] == errno.EINTR:
+                continue
+            else:
+                raise
+        if r == b"":
+            break
+        else:
+            workerProtocol.dataReceived(r)
+            protocolOut.flush()
+            sys.stdout.flush()
+            sys.stderr.flush()
+
+    if config.tracer:
+        sys.settrace(None)
+        results = config.tracer.results()
+        results.write_results(
+            show_missing=True, summary=False, coverdir=config.coverdir().path
+        )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/contrib/python/Twisted/py3/twisted/trial/_synctest.py b/contrib/python/Twisted/py3/twisted/trial/_synctest.py
new file mode 100644
index 00000000000..2cffc2c79e3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/_synctest.py
@@ -0,0 +1,1464 @@
+# -*- test-case-name: twisted.trial.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Things likely to be used by writers of unit tests.
+
+Maintainer: Jonathan Lange
+"""
+
+
+import inspect
+import os
+import sys
+import tempfile
+import types
+import unittest as pyunit
+import warnings
+from dis import findlinestarts as _findlinestarts
+from typing import (
+    Any,
+    Callable,
+    Coroutine,
+    Generator,
+    Iterable,
+    List,
+    NoReturn,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
+
+# Python 2.7 and higher has skip support built-in
+from unittest import SkipTest
+
+from attrs import frozen
+from typing_extensions import ParamSpec
+
+from twisted.internet.defer import Deferred, ensureDeferred
+from twisted.python import failure, log, monkey
+from twisted.python.deprecate import (
+    DEPRECATION_WARNING_FORMAT,
+    getDeprecationWarningString,
+    getVersionString,
+    warnAboutFunction,
+)
+from twisted.python.reflect import fullyQualifiedName
+from twisted.python.util import runWithWarningsSuppressed
+from twisted.trial import itrial, util
+
+_P = ParamSpec("_P")
+T = TypeVar("T")
+
+
+class FailTest(AssertionError):
+    """
+    Raised to indicate the current test has failed to pass.
+    """
+
+
+@frozen
+class Todo:
+    """
+    Internal object used to mark a L{TestCase} as 'todo'. Tests marked 'todo'
+    are reported differently in Trial L{TestResult}s. If todo'd tests fail,
+    they do not fail the suite and the errors are reported in a separate
+    category. If todo'd tests succeed, Trial L{TestResult}s will report an
+    unexpected success.
+
+    @ivar reason: A string explaining why the test is marked 'todo'
+
+    @ivar errors: An iterable of exception types that the test is expected to
+        raise. If one of these errors is raised by the test, it will be
+        trapped. Raising any other kind of error will fail the test.  If
+        L{None} then all errors will be trapped.
+    """
+
+    reason: str
+    errors: Optional[Iterable[Type[BaseException]]] = None
+
+    def __repr__(self) -> str:
+        return f"<Todo reason={self.reason!r} errors={self.errors!r}>"
+
+    def expected(self, failure):
+        """
+        @param failure: A L{twisted.python.failure.Failure}.
+
+        @return: C{True} if C{failure} is expected, C{False} otherwise.
+        """
+        if self.errors is None:
+            return True
+        for error in self.errors:
+            if failure.check(error):
+                return True
+        return False
+
+
+def makeTodo(
+    value: Union[
+        str, Tuple[Union[Type[BaseException], Iterable[Type[BaseException]]], str]
+    ]
+) -> Todo:
+    """
+    Return a L{Todo} object built from C{value}.
+
+    If C{value} is a string, return a Todo that expects any exception with
+    C{value} as a reason. If C{value} is a tuple, the second element is used
+    as the reason and the first element as the excepted error(s).
+
+    @param value: A string or a tuple of C{(errors, reason)}, where C{errors}
+    is either a single exception class or an iterable of exception classes.
+
+    @return: A L{Todo} object.
+    """
+    if isinstance(value, str):
+        return Todo(reason=value)
+    if isinstance(value, tuple):
+        errors, reason = value
+        if isinstance(errors, type):
+            iterableErrors: Iterable[Type[BaseException]] = [errors]
+        else:
+            iterableErrors = errors
+        return Todo(reason=reason, errors=iterableErrors)
+
+
+class _Warning:
+    """
+    A L{_Warning} instance represents one warning emitted through the Python
+    warning system (L{warnings}).  This is used to insulate callers of
+    L{_collectWarnings} from changes to the Python warnings system which might
+    otherwise require changes to the warning objects that function passes to
+    the observer object it accepts.
+
+    @ivar message: The string which was passed as the message parameter to
+        L{warnings.warn}.
+
+    @ivar category: The L{Warning} subclass which was passed as the category
+        parameter to L{warnings.warn}.
+
+    @ivar filename: The name of the file containing the definition of the code
+        object which was C{stacklevel} frames above the call to
+        L{warnings.warn}, where C{stacklevel} is the value of the C{stacklevel}
+        parameter passed to L{warnings.warn}.
+
+    @ivar lineno: The source line associated with the active instruction of the
+        code object object which was C{stacklevel} frames above the call to
+        L{warnings.warn}, where C{stacklevel} is the value of the C{stacklevel}
+        parameter passed to L{warnings.warn}.
+    """
+
+    def __init__(self, message, category, filename, lineno):
+        self.message = message
+        self.category = category
+        self.filename = filename
+        self.lineno = lineno
+
+
+def _setWarningRegistryToNone(modules):
+    """
+    Disable the per-module cache for every module found in C{modules}, typically
+    C{sys.modules}.
+
+    @param modules: Dictionary of modules, typically sys.module dict
+    """
+    for v in list(modules.values()):
+        if v is not None:
+            try:
+                v.__warningregistry__ = None
+            except BaseException:
+                # Don't specify a particular exception type to handle in case
+                # some wacky object raises some wacky exception in response to
+                # the setattr attempt.
+                pass
+
+
+def _collectWarnings(observeWarning, f, *args, **kwargs):
+    """
+    Call C{f} with C{args} positional arguments and C{kwargs} keyword arguments
+    and collect all warnings which are emitted as a result in a list.
+
+    @param observeWarning: A callable which will be invoked with a L{_Warning}
+        instance each time a warning is emitted.
+
+    @return: The return value of C{f(*args, **kwargs)}.
+    """
+
+    def showWarning(message, category, filename, lineno, file=None, line=None):
+        assert isinstance(message, Warning)
+        observeWarning(_Warning(str(message), category, filename, lineno))
+
+    # Disable the per-module cache for every module otherwise if the warning
+    # which the caller is expecting us to collect was already emitted it won't
+    # be re-emitted by the call to f which happens below.
+    _setWarningRegistryToNone(sys.modules)
+
+    origFilters = warnings.filters[:]
+    origShow = warnings.showwarning
+    warnings.simplefilter("always")
+    try:
+        warnings.showwarning = showWarning
+        result = f(*args, **kwargs)
+    finally:
+        warnings.filters[:] = origFilters
+        warnings.showwarning = origShow
+    return result
+
+
+class UnsupportedTrialFeature(Exception):
+    """A feature of twisted.trial was used that pyunit cannot support."""
+
+
+class PyUnitResultAdapter:
+    """
+    Wrap a C{TestResult} from the standard library's C{unittest} so that it
+    supports the extended result types from Trial, and also supports
+    L{twisted.python.failure.Failure}s being passed to L{addError} and
+    L{addFailure}.
+    """
+
+    def __init__(self, original):
+        """
+        @param original: A C{TestResult} instance from C{unittest}.
+        """
+        self.original = original
+
+    def _exc_info(self, err):
+        return util.excInfoOrFailureToExcInfo(err)
+
+    def startTest(self, method):
+        self.original.startTest(method)
+
+    def stopTest(self, method):
+        self.original.stopTest(method)
+
+    def addFailure(self, test, fail):
+        self.original.addFailure(test, self._exc_info(fail))
+
+    def addError(self, test, error):
+        self.original.addError(test, self._exc_info(error))
+
+    def _unsupported(self, test, feature, info):
+        self.original.addFailure(
+            test,
+            (UnsupportedTrialFeature, UnsupportedTrialFeature(feature, info), None),
+        )
+
+    def addSkip(self, test, reason):
+        """
+        Report the skip as a failure.
+        """
+        self.original.addSkip(test, reason)
+
+    def addUnexpectedSuccess(self, test, todo=None):
+        """
+        Report the unexpected success as a failure.
+        """
+        self._unsupported(test, "unexpected success", todo)
+
+    def addExpectedFailure(self, test, error):
+        """
+        Report the expected failure (i.e. todo) as a failure.
+        """
+        self._unsupported(test, "expected failure", error)
+
+    def addSuccess(self, test):
+        self.original.addSuccess(test)
+
+    def upDownError(self, method, error, warn, printStatus):
+        pass
+
+
+class _AssertRaisesContext:
+    """
+    A helper for implementing C{assertRaises}.  This is a context manager and a
+    helper method to support the non-context manager version of
+    C{assertRaises}.
+
+    @ivar _testCase: See C{testCase} parameter of C{__init__}
+
+    @ivar _expected: See C{expected} parameter of C{__init__}
+
+    @ivar _returnValue: The value returned by the callable being tested (only
+        when not being used as a context manager).
+
+    @ivar _expectedName: A short string describing the expected exception
+        (usually the name of the exception class).
+
+    @ivar exception: The exception which was raised by the function being
+        tested (if it raised one).
+    """
+
+    def __init__(self, testCase, expected):
+        """
+        @param testCase: The L{TestCase} instance which is used to raise a
+            test-failing exception when that is necessary.
+
+        @param expected: The exception type expected to be raised.
+        """
+        self._testCase = testCase
+        self._expected = expected
+        self._returnValue = None
+        try:
+            self._expectedName = self._expected.__name__
+        except AttributeError:
+            self._expectedName = str(self._expected)
+
+    def _handle(self, obj):
+        """
+        Call the given object using this object as a context manager.
+
+        @param obj: The object to call and which is expected to raise some
+            exception.
+        @type obj: L{object}
+
+        @return: Whatever exception is raised by C{obj()}.
+        @rtype: L{BaseException}
+        """
+        with self as context:
+            self._returnValue = obj()
+        return context.exception
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exceptionType, exceptionValue, traceback):
+        """
+        Check exit exception against expected exception.
+        """
+        # No exception raised.
+        if exceptionType is None:
+            self._testCase.fail(
+                "{} not raised ({} returned)".format(
+                    self._expectedName, self._returnValue
+                )
+            )
+
+        if not isinstance(exceptionValue, exceptionType):
+            # Support some Python 2.6 ridiculousness.  Exceptions raised using
+            # the C API appear here as the arguments you might pass to the
+            # exception class to create an exception instance.  So... do that
+            # to turn them into the instances.
+            if isinstance(exceptionValue, tuple):
+                exceptionValue = exceptionType(*exceptionValue)
+            else:
+                exceptionValue = exceptionType(exceptionValue)
+
+        # Store exception so that it can be access from context.
+        self.exception = exceptionValue
+
+        # Wrong exception raised.
+        if not issubclass(exceptionType, self._expected):
+            reason = failure.Failure(exceptionValue, exceptionType, traceback)
+            self._testCase.fail(
+                "{} raised instead of {}:\n {}".format(
+                    fullyQualifiedName(exceptionType),
+                    self._expectedName,
+                    reason.getTraceback(),
+                ),
+            )
+
+        # All good.
+        return True
+
+
+class _Assertions(pyunit.TestCase):
+    """
+    Replaces many of the built-in TestCase assertions. In general, these
+    assertions provide better error messages and are easier to use in
+    callbacks.
+    """
+
+    def fail(self, msg: Optional[object] = None) -> NoReturn:
+        """
+        Absolutely fail the test.  Do not pass go, do not collect $200.
+
+        @param msg: the message that will be displayed as the reason for the
+        failure
+        """
+        raise self.failureException(msg)
+
+    def assertFalse(self, condition, msg=None):
+        """
+        Fail the test if C{condition} evaluates to True.
+
+        @param condition: any object that defines __nonzero__
+        """
+        super().assertFalse(condition, msg)
+        return condition
+
+    assertNot = assertFalse
+    failUnlessFalse = assertFalse
+    failIf = assertFalse
+
+    def assertTrue(self, condition, msg=None):
+        """
+        Fail the test if C{condition} evaluates to False.
+
+        @param condition: any object that defines __nonzero__
+        """
+        super().assertTrue(condition, msg)
+        return condition
+
+    assert_ = assertTrue
+    failUnlessTrue = assertTrue
+    failUnless = assertTrue
+
+    def assertRaises(self, exception, f=None, *args, **kwargs):
+        """
+        Fail the test unless calling the function C{f} with the given
+        C{args} and C{kwargs} raises C{exception}. The failure will report
+        the traceback and call stack of the unexpected exception.
+
+        @param exception: exception type that is to be expected
+        @param f: the function to call
+
+        @return: If C{f} is L{None}, a context manager which will make an
+            assertion about the exception raised from the suite it manages.  If
+            C{f} is not L{None}, the exception raised by C{f}.
+
+        @raise self.failureException: Raised if the function call does
+            not raise an exception or if it raises an exception of a
+            different type.
+        """
+        context = _AssertRaisesContext(self, exception)
+        if f is None:
+            return context
+
+        return context._handle(lambda: f(*args, **kwargs))
+
+    # unittest.TestCase.assertRaises() is defined with 4 arguments
+    # but we define it with 5 arguments.  So we need to tell mypy
+    # to ignore the following assignment to failUnlessRaises
+    failUnlessRaises = assertRaises  # type: ignore[assignment]
+
+    def assertEqual(self, first, second, msg=None):
+        """
+        Fail the test if C{first} and C{second} are not equal.
+
+        @param msg: A string describing the failure that's included in the
+            exception.
+        """
+        super().assertEqual(first, second, msg)
+        return first
+
+    failUnlessEqual = assertEqual
+    failUnlessEquals = assertEqual
+    assertEquals = assertEqual
+
+    def assertIs(self, first, second, msg=None):
+        """
+        Fail the test if C{first} is not C{second}.  This is an
+        obect-identity-equality test, not an object equality
+        (i.e. C{__eq__}) test.
+
+        @param msg: if msg is None, then the failure message will be
+        '%r is not %r' % (first, second)
+        """
+        if first is not second:
+            raise self.failureException(msg or f"{first!r} is not {second!r}")
+        return first
+
+    failUnlessIdentical = assertIs
+    assertIdentical = assertIs
+
+    def assertIsNot(self, first, second, msg=None):
+        """
+        Fail the test if C{first} is C{second}.  This is an
+        obect-identity-equality test, not an object equality
+        (i.e. C{__eq__}) test.
+
+        @param msg: if msg is None, then the failure message will be
+        '%r is %r' % (first, second)
+        """
+        if first is second:
+            raise self.failureException(msg or f"{first!r} is {second!r}")
+        return first
+
+    failIfIdentical = assertIsNot
+    assertNotIdentical = assertIsNot
+
+    def assertNotEqual(self, first, second, msg=None):
+        """
+        Fail the test if C{first} == C{second}.
+
+        @param msg: if msg is None, then the failure message will be
+        '%r == %r' % (first, second)
+        """
+        if not first != second:
+            raise self.failureException(msg or f"{first!r} == {second!r}")
+        return first
+
+    assertNotEquals = assertNotEqual
+    failIfEquals = assertNotEqual
+    failIfEqual = assertNotEqual
+
+    def assertIn(self, containee, container, msg=None):
+        """
+        Fail the test if C{containee} is not found in C{container}.
+
+        @param containee: the value that should be in C{container}
+        @param container: a sequence type, or in the case of a mapping type,
+                          will follow semantics of 'if key in dict.keys()'
+        @param msg: if msg is None, then the failure message will be
+                    '%r not in %r' % (first, second)
+        """
+        if containee not in container:
+            raise self.failureException(msg or f"{containee!r} not in {container!r}")
+        return containee
+
+    failUnlessIn = assertIn
+
+    def assertNotIn(self, containee, container, msg=None):
+        """
+        Fail the test if C{containee} is found in C{container}.
+
+        @param containee: the value that should not be in C{container}
+        @param container: a sequence type, or in the case of a mapping type,
+                          will follow semantics of 'if key in dict.keys()'
+        @param msg: if msg is None, then the failure message will be
+                    '%r in %r' % (first, second)
+        """
+        if containee in container:
+            raise self.failureException(msg or f"{containee!r} in {container!r}")
+        return containee
+
+    failIfIn = assertNotIn
+
+    def assertNotAlmostEqual(self, first, second, places=7, msg=None, delta=None):
+        """
+        Fail if the two objects are equal as determined by their
+        difference rounded to the given number of decimal places
+        (default 7) and comparing to zero.
+
+        @note: decimal places (from zero) is usually not the same
+               as significant digits (measured from the most
+               significant digit).
+
+        @note: included for compatibility with PyUnit test cases
+        """
+        if round(second - first, places) == 0:
+            raise self.failureException(
+                msg or f"{first!r} == {second!r} within {places!r} places"
+            )
+        return first
+
+    assertNotAlmostEquals = assertNotAlmostEqual  # type:ignore[assignment]
+    failIfAlmostEqual = assertNotAlmostEqual  # type:ignore[assignment]
+    failIfAlmostEquals = assertNotAlmostEqual
+
+    def assertAlmostEqual(self, first, second, places=7, msg=None, delta=None):
+        """
+        Fail if the two objects are unequal as determined by their
+        difference rounded to the given number of decimal places
+        (default 7) and comparing to zero.
+
+        @note: decimal places (from zero) is usually not the same
+               as significant digits (measured from the most
+               significant digit).
+
+        @note: included for compatibility with PyUnit test cases
+        """
+        if round(second - first, places) != 0:
+            raise self.failureException(
+                msg or f"{first!r} != {second!r} within {places!r} places"
+            )
+        return first
+
+    assertAlmostEquals = assertAlmostEqual  # type:ignore[assignment]
+    failUnlessAlmostEqual = assertAlmostEqual  # type:ignore[assignment]
+
+    def assertApproximates(self, first, second, tolerance, msg=None):
+        """
+        Fail if C{first} - C{second} > C{tolerance}
+
+        @param msg: if msg is None, then the failure message will be
+                    '%r ~== %r' % (first, second)
+        """
+        if abs(first - second) > tolerance:
+            raise self.failureException(msg or f"{first} ~== {second}")
+        return first
+
+    failUnlessApproximates = assertApproximates
+
+    def assertSubstring(self, substring, astring, msg=None):
+        """
+        Fail if C{substring} does not exist within C{astring}.
+        """
+        return self.failUnlessIn(substring, astring, msg)
+
+    failUnlessSubstring = assertSubstring
+
+    def assertNotSubstring(self, substring, astring, msg=None):
+        """
+        Fail if C{astring} contains C{substring}.
+        """
+        return self.failIfIn(substring, astring, msg)
+
+    failIfSubstring = assertNotSubstring
+
+    def assertWarns(self, category, message, filename, f, *args, **kwargs):
+        """
+        Fail if the given function doesn't generate the specified warning when
+        called. It calls the function, checks the warning, and forwards the
+        result of the function if everything is fine.
+
+        @param category: the category of the warning to check.
+        @param message: the output message of the warning to check.
+        @param filename: the filename where the warning should come from.
+        @param f: the function which is supposed to generate the warning.
+        @type f: any callable.
+        @param args: the arguments to C{f}.
+        @param kwargs: the keywords arguments to C{f}.
+
+        @return: the result of the original function C{f}.
+        """
+        warningsShown = []
+        result = _collectWarnings(warningsShown.append, f, *args, **kwargs)
+
+        if not warningsShown:
+            self.fail("No warnings emitted")
+        first = warningsShown[0]
+        for other in warningsShown[1:]:
+            if (other.message, other.category) != (first.message, first.category):
+                self.fail("Can't handle different warnings")
+        self.assertEqual(first.message, message)
+        self.assertIdentical(first.category, category)
+
+        # Use starts with because of .pyc/.pyo issues.
+        self.assertTrue(
+            filename.startswith(first.filename),
+            f"Warning in {first.filename!r}, expected {filename!r}",
+        )
+
+        # It would be nice to be able to check the line number as well, but
+        # different configurations actually end up reporting different line
+        # numbers (generally the variation is only 1 line, but that's enough
+        # to fail the test erroneously...).
+        # self.assertEqual(lineno, xxx)
+
+        return result
+
+    failUnlessWarns = assertWarns
+
+    def assertIsInstance(self, instance, classOrTuple, message=None):
+        """
+        Fail if C{instance} is not an instance of the given class or of
+        one of the given classes.
+
+        @param instance: the object to test the type (first argument of the
+            C{isinstance} call).
+        @type instance: any.
+        @param classOrTuple: the class or classes to test against (second
+            argument of the C{isinstance} call).
+        @type classOrTuple: class, type, or tuple.
+
+        @param message: Custom text to include in the exception text if the
+            assertion fails.
+        """
+        if not isinstance(instance, classOrTuple):
+            if message is None:
+                suffix = ""
+            else:
+                suffix = ": " + message
+            self.fail(f"{instance!r} is not an instance of {classOrTuple}{suffix}")
+
+    failUnlessIsInstance = assertIsInstance
+
+    def assertNotIsInstance(self, instance, classOrTuple):
+        """
+        Fail if C{instance} is an instance of the given class or of one of the
+        given classes.
+
+        @param instance: the object to test the type (first argument of the
+            C{isinstance} call).
+        @type instance: any.
+        @param classOrTuple: the class or classes to test against (second
+            argument of the C{isinstance} call).
+        @type classOrTuple: class, type, or tuple.
+        """
+        if isinstance(instance, classOrTuple):
+            self.fail(f"{instance!r} is an instance of {classOrTuple}")
+
+    failIfIsInstance = assertNotIsInstance
+
+    def successResultOf(
+        self,
+        deferred: Union[
+            Coroutine[Deferred[T], Any, T],
+            Generator[Deferred[T], Any, T],
+            Deferred[T],
+        ],
+    ) -> T:
+        """
+        Return the current success result of C{deferred} or raise
+        C{self.failureException}.
+
+        @param deferred: A L{Deferred<twisted.internet.defer.Deferred>} or
+            I{coroutine} which has a success result.
+
+            For a L{Deferred<twisted.internet.defer.Deferred>} this means
+            L{Deferred.callback<twisted.internet.defer.Deferred.callback>} or
+            L{Deferred.errback<twisted.internet.defer.Deferred.errback>} has
+            been called on it and it has reached the end of its callback chain
+            and the last callback or errback returned a
+            non-L{failure.Failure}.
+
+            For a I{coroutine} this means all awaited values have a success
+            result.
+
+        @raise SynchronousTestCase.failureException: If the
+            L{Deferred<twisted.internet.defer.Deferred>} has no result or has a
+            failure result.
+
+        @return: The result of C{deferred}.
+        """
+        deferred = ensureDeferred(deferred)
+        results: List[Union[T, failure.Failure]] = []
+        deferred.addBoth(results.append)
+
+        if not results:
+            self.fail(
+                "Success result expected on {!r}, found no result instead".format(
+                    deferred
+                )
+            )
+
+        result = results[0]
+
+        if isinstance(result, failure.Failure):
+            self.fail(
+                "Success result expected on {!r}, "
+                "found failure result instead:\n{}".format(
+                    deferred, result.getTraceback()
+                )
+            )
+        return result
+
+    def failureResultOf(self, deferred, *expectedExceptionTypes):
+        """
+        Return the current failure result of C{deferred} or raise
+        C{self.failureException}.
+
+        @param deferred: A L{Deferred<twisted.internet.defer.Deferred>} which
+            has a failure result.  This means
+            L{Deferred.callback<twisted.internet.defer.Deferred.callback>} or
+            L{Deferred.errback<twisted.internet.defer.Deferred.errback>} has
+            been called on it and it has reached the end of its callback chain
+            and the last callback or errback raised an exception or returned a
+            L{failure.Failure}.
+        @type deferred: L{Deferred<twisted.internet.defer.Deferred>}
+
+        @param expectedExceptionTypes: Exception types to expect - if
+            provided, and the exception wrapped by the failure result is
+            not one of the types provided, then this test will fail.
+
+        @raise SynchronousTestCase.failureException: If the
+            L{Deferred<twisted.internet.defer.Deferred>} has no result, has a
+            success result, or has an unexpected failure result.
+
+        @return: The failure result of C{deferred}.
+        @rtype: L{failure.Failure}
+        """
+        deferred = ensureDeferred(deferred)
+        result = []
+        deferred.addBoth(result.append)
+
+        if not result:
+            self.fail(
+                "Failure result expected on {!r}, found no result instead".format(
+                    deferred
+                )
+            )
+
+        result = result[0]
+
+        if not isinstance(result, failure.Failure):
+            self.fail(
+                "Failure result expected on {!r}, "
+                "found success result ({!r}) instead".format(deferred, result)
+            )
+
+        if expectedExceptionTypes and not result.check(*expectedExceptionTypes):
+            expectedString = " or ".join(
+                [".".join((t.__module__, t.__name__)) for t in expectedExceptionTypes]
+            )
+
+            self.fail(
+                "Failure of type ({}) expected on {!r}, "
+                "found type {!r} instead: {}".format(
+                    expectedString, deferred, result.type, result.getTraceback()
+                )
+            )
+
+        return result
+
+    def assertNoResult(self, deferred):
+        """
+        Assert that C{deferred} does not have a result at this point.
+
+        If the assertion succeeds, then the result of C{deferred} is left
+        unchanged. Otherwise, any L{failure.Failure} result is swallowed.
+
+        @param deferred: A L{Deferred<twisted.internet.defer.Deferred>} without
+            a result.  This means that neither
+            L{Deferred.callback<twisted.internet.defer.Deferred.callback>} nor
+            L{Deferred.errback<twisted.internet.defer.Deferred.errback>} has
+            been called, or that the
+            L{Deferred<twisted.internet.defer.Deferred>} is waiting on another
+            L{Deferred<twisted.internet.defer.Deferred>} for a result.
+        @type deferred: L{Deferred<twisted.internet.defer.Deferred>}
+
+        @raise SynchronousTestCase.failureException: If the
+            L{Deferred<twisted.internet.defer.Deferred>} has a result.
+        """
+        deferred = ensureDeferred(deferred)
+        result = []
+
+        def cb(res):
+            result.append(res)
+            return res
+
+        deferred.addBoth(cb)
+
+        if result:
+            # If there is already a failure, the self.fail below will
+            # report it, so swallow it in the deferred
+            deferred.addErrback(lambda _: None)
+            self.fail(
+                "No result expected on {!r}, found {!r} instead".format(
+                    deferred, result[0]
+                )
+            )
+
+
+class _LogObserver:
+    """
+    Observes the Twisted logs and catches any errors.
+
+    @ivar _errors: A C{list} of L{Failure} instances which were received as
+        error events from the Twisted logging system.
+
+    @ivar _added: A C{int} giving the number of times C{_add} has been called
+        less the number of times C{_remove} has been called; used to only add
+        this observer to the Twisted logging since once, regardless of the
+        number of calls to the add method.
+
+    @ivar _ignored: A C{list} of exception types which will not be recorded.
+    """
+
+    def __init__(self):
+        self._errors = []
+        self._added = 0
+        self._ignored = []
+
+    def _add(self):
+        if self._added == 0:
+            log.addObserver(self.gotEvent)
+        self._added += 1
+
+    def _remove(self):
+        self._added -= 1
+        if self._added == 0:
+            log.removeObserver(self.gotEvent)
+
+    def _ignoreErrors(self, *errorTypes):
+        """
+        Do not store any errors with any of the given types.
+        """
+        self._ignored.extend(errorTypes)
+
+    def _clearIgnores(self):
+        """
+        Stop ignoring any errors we might currently be ignoring.
+        """
+        self._ignored = []
+
+    def flushErrors(self, *errorTypes):
+        """
+        Flush errors from the list of caught errors. If no arguments are
+        specified, remove all errors. If arguments are specified, only remove
+        errors of those types from the stored list.
+        """
+        if errorTypes:
+            flushed = []
+            remainder = []
+            for f in self._errors:
+                if f.check(*errorTypes):
+                    flushed.append(f)
+                else:
+                    remainder.append(f)
+            self._errors = remainder
+        else:
+            flushed = self._errors
+            self._errors = []
+        return flushed
+
+    def getErrors(self):
+        """
+        Return a list of errors caught by this observer.
+        """
+        return self._errors
+
+    def gotEvent(self, event):
+        """
+        The actual observer method. Called whenever a message is logged.
+
+        @param event: A dictionary containing the log message. Actual
+        structure undocumented (see source for L{twisted.python.log}).
+        """
+        if event.get("isError", False) and "failure" in event:
+            f = event["failure"]
+            if len(self._ignored) == 0 or not f.check(*self._ignored):
+                self._errors.append(f)
+
+
+_logObserver = _LogObserver()
+
+
+class SynchronousTestCase(_Assertions):
+    """
+    A unit test. The atom of the unit testing universe.
+
+    This class extends C{unittest.TestCase} from the standard library.  A number
+    of convenient testing helpers are added, including logging and warning
+    integration, monkey-patching support, and more.
+
+    To write a unit test, subclass C{SynchronousTestCase} and define a method
+    (say, 'test_foo') on the subclass. To run the test, instantiate your
+    subclass with the name of the method, and call L{run} on the instance,
+    passing a L{TestResult} object.
+
+    The C{trial} script will automatically find any C{SynchronousTestCase}
+    subclasses defined in modules beginning with 'test_' and construct test
+    cases for all methods beginning with 'test'.
+
+    If an error is logged during the test run, the test will fail with an
+    error. See L{log.err}.
+
+    @ivar failureException: An exception class, defaulting to C{FailTest}. If
+    the test method raises this exception, it will be reported as a failure,
+    rather than an exception. All of the assertion methods raise this if the
+    assertion fails.
+
+    @ivar skip: L{None} or a string explaining why this test is to be
+    skipped. If defined, the test will not be run. Instead, it will be
+    reported to the result object as 'skipped' (if the C{TestResult} supports
+    skipping).
+
+    @ivar todo: L{None}, a string or a tuple of C{(errors, reason)} where
+    C{errors} is either an exception class or an iterable of exception
+    classes, and C{reason} is a string. See L{Todo} or L{makeTodo} for more
+    information.
+
+    @ivar suppress: L{None} or a list of tuples of C{(args, kwargs)} to be
+    passed to C{warnings.filterwarnings}. Use these to suppress warnings
+    raised in a test. Useful for testing deprecated code. See also
+    L{util.suppress}.
+    """
+
+    failureException = FailTest
+
+    def __init__(self, methodName="runTest"):
+        super().__init__(methodName)
+        self._passed = False
+        self._cleanups = []
+        self._testMethodName = methodName
+        testMethod = getattr(self, methodName)
+        self._parents = [testMethod, self, sys.modules.get(self.__class__.__module__)]
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Override the comparison defined by the base TestCase which considers
+        instances of the same class with the same _testMethodName to be
+        equal.  Since trial puts TestCase instances into a set, that
+        definition of comparison makes it impossible to run the same test
+        method twice.  Most likely, trial should stop using a set to hold
+        tests, but until it does, this is necessary on Python 2.6. -exarkun
+        """
+        if isinstance(other, SynchronousTestCase):
+            return self is other
+        else:
+            return NotImplemented
+
+    def __hash__(self):
+        return hash((self.__class__, self._testMethodName))
+
+    def shortDescription(self):
+        desc = super().shortDescription()
+        if desc is None:
+            return self._testMethodName
+        return desc
+
+    def getSkip(self) -> Tuple[bool, Optional[str]]:
+        """
+        Return the skip reason set on this test, if any is set. Checks on the
+        instance first, then the class, then the module, then packages. As
+        soon as it finds something with a C{skip} attribute, returns that in
+        a tuple (L{True}, L{str}).
+        If the C{skip} attribute does not exist, look for C{__unittest_skip__}
+        and C{__unittest_skip_why__} attributes which are set by the standard
+        library L{unittest.skip} function.
+        Returns (L{False}, L{None}) if it cannot find anything.
+        See L{TestCase} docstring for more details.
+        """
+        skipReason = util.acquireAttribute(self._parents, "skip", None)
+        doSkip = skipReason is not None
+        if skipReason is None:
+            doSkip = getattr(self, "__unittest_skip__", False)
+            if doSkip:
+                skipReason = getattr(self, "__unittest_skip_why__", "")
+        return (doSkip, skipReason)
+
+    def getTodo(self):
+        """
+        Return a L{Todo} object if the test is marked todo. Checks on the
+        instance first, then the class, then the module, then packages. As
+        soon as it finds something with a C{todo} attribute, returns that.
+        Returns L{None} if it cannot find anything. See L{TestCase} docstring
+        for more details.
+        """
+        todo = util.acquireAttribute(self._parents, "todo", None)
+        if todo is None:
+            return None
+        return makeTodo(todo)
+
+    def runTest(self):
+        """
+        If no C{methodName} argument is passed to the constructor, L{run} will
+        treat this method as the thing with the actual test inside.
+        """
+
+    def run(self, result):
+        """
+        Run the test case, storing the results in C{result}.
+
+        First runs C{setUp} on self, then runs the test method (defined in the
+        constructor), then runs C{tearDown}.  As with the standard library
+        L{unittest.TestCase}, the return value of these methods is disregarded.
+        In particular, returning a L{Deferred<twisted.internet.defer.Deferred>}
+        has no special additional consequences.
+
+        @param result: A L{TestResult} object.
+        """
+        log.msg("--> %s <--" % (self.id()))
+        new_result = itrial.IReporter(result, None)
+        if new_result is None:
+            result = PyUnitResultAdapter(result)
+        else:
+            result = new_result
+        result.startTest(self)
+        (doSkip, skipReason) = self.getSkip()
+        if doSkip:  # don't run test methods that are marked as .skip
+            result.addSkip(self, skipReason)
+            result.stopTest(self)
+            return
+
+        self._passed = False
+        self._warnings = []
+
+        self._installObserver()
+        # All the code inside _runFixturesAndTest will be run such that warnings
+        # emitted by it will be collected and retrievable by flushWarnings.
+        _collectWarnings(self._warnings.append, self._runFixturesAndTest, result)
+
+        # Any collected warnings which the test method didn't flush get
+        # re-emitted so they'll be logged or show up on stdout or whatever.
+        for w in self.flushWarnings():
+            try:
+                warnings.warn_explicit(**w)
+            except BaseException:
+                result.addError(self, failure.Failure())
+
+        result.stopTest(self)
+
+    # f should be a positional only argument but that is a breaking change
+    # see https://github.com/twisted/twisted/issues/11967
+    def addCleanup(  # type: ignore[override]
+        self, f: Callable[_P, object], *args: _P.args, **kwargs: _P.kwargs
+    ) -> None:
+        """
+        Add the given function to a list of functions to be called after the
+        test has run, but before C{tearDown}.
+
+        Functions will be run in reverse order of being added. This helps
+        ensure that tear down complements set up.
+
+        As with all aspects of L{SynchronousTestCase}, Deferreds are not
+        supported in cleanup functions.
+        """
+        self._cleanups.append((f, args, kwargs))
+
+    def patch(self, obj, attribute, value):
+        """
+        Monkey patch an object for the duration of the test.
+
+        The monkey patch will be reverted at the end of the test using the
+        L{addCleanup} mechanism.
+
+        The L{monkey.MonkeyPatcher} is returned so that users can restore and
+        re-apply the monkey patch within their tests.
+
+        @param obj: The object to monkey patch.
+        @param attribute: The name of the attribute to change.
+        @param value: The value to set the attribute to.
+        @return: A L{monkey.MonkeyPatcher} object.
+        """
+        monkeyPatch = monkey.MonkeyPatcher((obj, attribute, value))
+        monkeyPatch.patch()
+        self.addCleanup(monkeyPatch.restore)
+        return monkeyPatch
+
+    def flushLoggedErrors(self, *errorTypes):
+        """
+        Remove stored errors received from the log.
+
+        C{TestCase} stores each error logged during the run of the test and
+        reports them as errors during the cleanup phase (after C{tearDown}).
+
+        @param errorTypes: If unspecified, flush all errors. Otherwise, only
+        flush errors that match the given types.
+
+        @return: A list of failures that have been removed.
+        """
+        return self._observer.flushErrors(*errorTypes)
+
+    def flushWarnings(self, offendingFunctions=None):
+        """
+        Remove stored warnings from the list of captured warnings and return
+        them.
+
+        @param offendingFunctions: If L{None}, all warnings issued during the
+            currently running test will be flushed.  Otherwise, only warnings
+            which I{point} to a function included in this list will be flushed.
+            All warnings include a filename and source line number; if these
+            parts of a warning point to a source line which is part of a
+            function, then the warning I{points} to that function.
+        @type offendingFunctions: L{None} or L{list} of functions or methods.
+
+        @raise ValueError: If C{offendingFunctions} is not L{None} and includes
+            an object which is not a L{types.FunctionType} or
+            L{types.MethodType} instance.
+
+        @return: A C{list}, each element of which is a C{dict} giving
+            information about one warning which was flushed by this call.  The
+            keys of each C{dict} are:
+
+                - C{'message'}: The string which was passed as the I{message}
+                  parameter to L{warnings.warn}.
+
+                - C{'category'}: The warning subclass which was passed as the
+                  I{category} parameter to L{warnings.warn}.
+
+                - C{'filename'}: The name of the file containing the definition
+                  of the code object which was C{stacklevel} frames above the
+                  call to L{warnings.warn}, where C{stacklevel} is the value of
+                  the C{stacklevel} parameter passed to L{warnings.warn}.
+
+                - C{'lineno'}: The source line associated with the active
+                  instruction of the code object object which was C{stacklevel}
+                  frames above the call to L{warnings.warn}, where
+                  C{stacklevel} is the value of the C{stacklevel} parameter
+                  passed to L{warnings.warn}.
+        """
+        if offendingFunctions is None:
+            toFlush = self._warnings[:]
+            self._warnings[:] = []
+        else:
+            toFlush = []
+            for aWarning in self._warnings:
+                for aFunction in offendingFunctions:
+                    if not isinstance(
+                        aFunction, (types.FunctionType, types.MethodType)
+                    ):
+                        raise ValueError(f"{aFunction!r} is not a function or method")
+
+                    # inspect.getabsfile(aFunction) sometimes returns a
+                    # filename which disagrees with the filename the warning
+                    # system generates.  This seems to be because a
+                    # function's code object doesn't deal with source files
+                    # being renamed.  inspect.getabsfile(module) seems
+                    # better (or at least agrees with the warning system
+                    # more often), and does some normalization for us which
+                    # is desirable.  inspect.getmodule() is attractive, but
+                    # somewhat broken in Python < 2.6.  See Python bug 4845.
+                    aModule = sys.modules[aFunction.__module__]
+                    filename = inspect.getabsfile(aModule)
+
+                    if filename != os.path.normcase(aWarning.filename):
+                        continue
+                    lineNumbers = [
+                        lineNumber
+                        for _, lineNumber in _findlinestarts(aFunction.__code__)
+                    ]
+                    if not (min(lineNumbers) <= aWarning.lineno <= max(lineNumbers)):
+                        continue
+                    # The warning points to this function, flush it and move on
+                    # to the next warning.
+                    toFlush.append(aWarning)
+                    break
+            # Remove everything which is being flushed.
+            list(map(self._warnings.remove, toFlush))
+
+        return [
+            {
+                "message": w.message,
+                "category": w.category,
+                "filename": w.filename,
+                "lineno": w.lineno,
+            }
+            for w in toFlush
+        ]
+
+    def getDeprecatedModuleAttribute(self, moduleName, name, version, message=None):
+        """
+        Retrieve a module attribute which should have been deprecated,
+        and assert that we saw the appropriate deprecation warning.
+
+        @type moduleName: C{str}
+        @param moduleName: Fully-qualified Python name of the module containing
+            the deprecated attribute; if called from the same module as the
+            attributes are being deprecated in, using the C{__name__} global can
+            be helpful
+
+        @type name: C{str}
+        @param name: Attribute name which we expect to be deprecated
+
+        @param version: The first L{version<twisted.python.versions.Version>} that
+            the module attribute was deprecated.
+
+        @type message: C{str}
+        @param message: (optional) The expected deprecation message for the module attribute
+
+        @return: The given attribute from the named module
+
+        @raise FailTest: if no warnings were emitted on getattr, or if the
+            L{DeprecationWarning} emitted did not produce the canonical
+            please-use-something-else message that is standard for Twisted
+            deprecations according to the given version and replacement.
+
+        @since: Twisted 21.2.0
+        """
+        fqpn = moduleName + "." + name
+        module = sys.modules[moduleName]
+        attr = getattr(module, name)
+        warningsShown = self.flushWarnings([self.getDeprecatedModuleAttribute])
+        if len(warningsShown) == 0:
+            self.fail(f"{fqpn} is not deprecated.")
+
+        observedWarning = warningsShown[0]["message"]
+        expectedWarning = DEPRECATION_WARNING_FORMAT % {
+            "fqpn": fqpn,
+            "version": getVersionString(version),
+        }
+        if message is not None:
+            expectedWarning = expectedWarning + ": " + message
+        self.assert_(
+            observedWarning.startswith(expectedWarning),
+            f"Expected {observedWarning!r} to start with {expectedWarning!r}",
+        )
+
+        return attr
+
+    def callDeprecated(self, version, f, *args, **kwargs):
+        """
+        Call a function that should have been deprecated at a specific version
+        and in favor of a specific alternative, and assert that it was thusly
+        deprecated.
+
+        @param version: A 2-sequence of (since, replacement), where C{since} is
+            a the first L{version<incremental.Version>} that C{f}
+            should have been deprecated since, and C{replacement} is a suggested
+            replacement for the deprecated functionality, as described by
+            L{twisted.python.deprecate.deprecated}.  If there is no suggested
+            replacement, this parameter may also be simply a
+            L{version<incremental.Version>} by itself.
+
+        @param f: The deprecated function to call.
+
+        @param args: The arguments to pass to C{f}.
+
+        @param kwargs: The keyword arguments to pass to C{f}.
+
+        @return: Whatever C{f} returns.
+
+        @raise Exception: Whatever C{f} raises.  If any exception is
+            raised by C{f}, though, no assertions will be made about emitted
+            deprecations.
+
+        @raise FailTest: if no warnings were emitted by C{f}, or if the
+            L{DeprecationWarning} emitted did not produce the canonical
+            please-use-something-else message that is standard for Twisted
+            deprecations according to the given version and replacement.
+        """
+        result = f(*args, **kwargs)
+        warningsShown = self.flushWarnings([self.callDeprecated])
+        try:
+            info = list(version)
+        except TypeError:
+            since = version
+            replacement = None
+        else:
+            [since, replacement] = info
+
+        if len(warningsShown) == 0:
+            self.fail(f"{f!r} is not deprecated.")
+
+        observedWarning = warningsShown[0]["message"]
+        expectedWarning = getDeprecationWarningString(f, since, replacement=replacement)
+        self.assertEqual(expectedWarning, observedWarning)
+
+        return result
+
+    def mktemp(self):
+        """
+        Create a new path name which can be used for a new file or directory.
+
+        The result is a relative path that is guaranteed to be unique within the
+        current working directory.  The parent of the path will exist, but the
+        path will not.
+
+        For a temporary directory call os.mkdir on the path.  For a temporary
+        file just create the file (e.g. by opening the path for writing and then
+        closing it).
+
+        @return: The newly created path
+        @rtype: C{str}
+        """
+        MAX_FILENAME = 32  # some platforms limit lengths of filenames
+        base = os.path.join(
+            self.__class__.__module__[:MAX_FILENAME],
+            self.__class__.__name__[:MAX_FILENAME],
+            self._testMethodName[:MAX_FILENAME],
+        )
+        if not os.path.exists(base):
+            os.makedirs(base)
+        # With 3.11 or older mkdtemp returns a relative path.
+        # With newer it is absolute.
+        # Here we make sure we always handle a relative path.
+        # See https://github.com/python/cpython/issues/51574
+        dirname = os.path.relpath(tempfile.mkdtemp("", "", base))
+        return os.path.join(dirname, "temp")
+
+    def _getSuppress(self):
+        """
+        Returns any warning suppressions set for this test. Checks on the
+        instance first, then the class, then the module, then packages. As
+        soon as it finds something with a C{suppress} attribute, returns that.
+        Returns any empty list (i.e. suppress no warnings) if it cannot find
+        anything. See L{TestCase} docstring for more details.
+        """
+        return util.acquireAttribute(self._parents, "suppress", [])
+
+    def _getSkipReason(self, method, skip):
+        """
+        Return the reason to use for skipping a test method.
+
+        @param method: The method which produced the skip.
+        @param skip: A L{unittest.SkipTest} instance raised by C{method}.
+        """
+        if len(skip.args) > 0:
+            return skip.args[0]
+
+        warnAboutFunction(
+            method,
+            "Do not raise unittest.SkipTest with no arguments! Give a reason "
+            "for skipping tests!",
+        )
+        return skip
+
+    def _run(self, suppress, todo, method, result):
+        """
+        Run a single method, either a test method or fixture.
+
+        @param suppress: Any warnings to suppress, as defined by the C{suppress}
+            attribute on this method, test case, or the module it is defined in.
+
+        @param todo: Any expected failure or failures, as defined by the C{todo}
+            attribute on this method, test case, or the module it is defined in.
+
+        @param method: The method to run.
+
+        @param result: The TestResult instance to which to report results.
+
+        @return: C{True} if the method fails and no further method/fixture calls
+            should be made, C{False} otherwise.
+        """
+        if inspect.isgeneratorfunction(method):
+            exc = TypeError(
+                "{!r} is a generator function and therefore will never run".format(
+                    method
+                )
+            )
+            result.addError(self, failure.Failure(exc))
+            return True
+        try:
+            runWithWarningsSuppressed(suppress, method)
+        except SkipTest as e:
+            result.addSkip(self, self._getSkipReason(method, e))
+        except BaseException:
+            reason = failure.Failure()
+            if todo is None or not todo.expected(reason):
+                if reason.check(self.failureException):
+                    addResult = result.addFailure
+                else:
+                    addResult = result.addError
+                addResult(self, reason)
+            else:
+                result.addExpectedFailure(self, reason, todo)
+        else:
+            return False
+        return True
+
+    def _runFixturesAndTest(self, result):
+        """
+        Run C{setUp}, a test method, test cleanups, and C{tearDown}.
+
+        @param result: The TestResult instance to which to report results.
+        """
+        suppress = self._getSuppress()
+        try:
+            if self._run(suppress, None, self.setUp, result):
+                return
+
+            todo = self.getTodo()
+            method = getattr(self, self._testMethodName)
+            failed = self._run(suppress, todo, method, result)
+        finally:
+            self._runCleanups(result)
+
+        if todo and not failed:
+            result.addUnexpectedSuccess(self, todo)
+
+        if self._run(suppress, None, self.tearDown, result):
+            failed = True
+
+        for error in self._observer.getErrors():
+            result.addError(self, error)
+            failed = True
+        self._observer.flushErrors()
+        self._removeObserver()
+
+        if not (failed or todo):
+            result.addSuccess(self)
+
+    def _runCleanups(self, result):
+        """
+        Synchronously run any cleanups which have been added.
+        """
+        while len(self._cleanups) > 0:
+            f, args, kwargs = self._cleanups.pop()
+            try:
+                f(*args, **kwargs)
+            except BaseException:
+                f = failure.Failure()
+                result.addError(self, f)
+
+    def _installObserver(self):
+        self._observer = _logObserver
+        self._observer._add()
+
+    def _removeObserver(self):
+        self._observer._remove()
diff --git a/contrib/python/Twisted/py3/twisted/trial/itrial.py b/contrib/python/Twisted/py3/twisted/trial/itrial.py
new file mode 100644
index 00000000000..840c6e61eeb
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/itrial.py
@@ -0,0 +1,157 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Interfaces for Trial.
+
+Maintainer: Jonathan Lange
+"""
+
+
+import zope.interface as zi
+
+
+class ITestCase(zi.Interface):
+    """
+    The interface that a test case must implement in order to be used in Trial.
+    """
+
+    failureException = zi.Attribute(
+        "The exception class that is raised by failed assertions"
+    )
+
+    def __call__(result):
+        """
+        Run the test. Should always do exactly the same thing as run().
+        """
+
+    def countTestCases():
+        """
+        Return the number of tests in this test case. Usually 1.
+        """
+
+    def id():
+        """
+        Return a unique identifier for the test, usually the fully-qualified
+        Python name.
+        """
+
+    def run(result):
+        """
+        Run the test, storing the results in C{result}.
+
+        @param result: A L{TestResult}.
+        """
+
+    def shortDescription():
+        """
+        Return a short description of the test.
+        """
+
+
+class IReporter(zi.Interface):
+    """
+    I report results from a run of a test suite.
+    """
+
+    shouldStop = zi.Attribute(
+        "A boolean indicating that this reporter would like the " "test run to stop."
+    )
+    testsRun = zi.Attribute(
+        """
+        The number of tests that seem to have been run according to this
+        reporter.
+        """
+    )
+
+    def startTest(method):
+        """
+        Report the beginning of a run of a single test method.
+
+        @param method: an object that is adaptable to ITestMethod
+        """
+
+    def stopTest(method):
+        """
+        Report the status of a single test method
+
+        @param method: an object that is adaptable to ITestMethod
+        """
+
+    def addSuccess(test):
+        """
+        Record that test passed.
+        """
+
+    def addError(test, error):
+        """
+        Record that a test has raised an unexpected exception.
+
+        @param test: The test that has raised an error.
+        @param error: The error that the test raised. It will either be a
+            three-tuple in the style of C{sys.exc_info()} or a
+            L{Failure<twisted.python.failure.Failure>} object.
+        """
+
+    def addFailure(test, failure):
+        """
+        Record that a test has failed with the given failure.
+
+        @param test: The test that has failed.
+        @param failure: The failure that the test failed with. It will
+            either be a three-tuple in the style of C{sys.exc_info()}
+            or a L{Failure<twisted.python.failure.Failure>} object.
+        """
+
+    def addExpectedFailure(test, failure, todo=None):
+        """
+        Record that the given test failed, and was expected to do so.
+
+        In Twisted 15.5 and prior, C{todo} was a mandatory parameter.
+
+        @type test: L{unittest.TestCase}
+        @param test: The test which this is about.
+        @type failure: L{failure.Failure}
+        @param failure: The error which this test failed with.
+        @type todo: L{unittest.Todo}
+        @param todo: The reason for the test's TODO status. If L{None}, a
+            generic reason is used.
+        """
+
+    def addUnexpectedSuccess(test, todo=None):
+        """
+        Record that the given test failed, and was expected to do so.
+
+        In Twisted 15.5 and prior, C{todo} was a mandatory parameter.
+
+        @type test: L{unittest.TestCase}
+        @param test: The test which this is about.
+        @type todo: L{unittest.Todo}
+        @param todo: The reason for the test's TODO status. If L{None}, a
+            generic reason is used.
+        """
+
+    def addSkip(test, reason):
+        """
+        Record that a test has been skipped for the given reason.
+
+        @param test: The test that has been skipped.
+        @param reason: An object that the test case has specified as the reason
+            for skipping the test.
+        """
+
+    def wasSuccessful():
+        """
+        Return a boolean indicating whether all test results that were reported
+        to this reporter were successful or not.
+        """
+
+    def done():
+        """
+        Called when the test run is complete.
+
+        This gives the result object an opportunity to display a summary of
+        information to the user. Once you have called C{done} on an
+        L{IReporter} object, you should assume that the L{IReporter} object is
+        no longer usable.
+        """
diff --git a/contrib/python/Twisted/py3/twisted/trial/newsfragments/.gitignore b/contrib/python/Twisted/py3/twisted/trial/newsfragments/.gitignore
new file mode 100644
index 00000000000..f935021a8f8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/newsfragments/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/contrib/python/Twisted/py3/twisted/trial/reporter.py b/contrib/python/Twisted/py3/twisted/trial/reporter.py
new file mode 100644
index 00000000000..2664b2fe0d5
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/reporter.py
@@ -0,0 +1,1278 @@
+# -*- test-case-name: twisted.trial.test.test_reporter -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+#
+# Maintainer: Jonathan Lange
+
+"""
+Defines classes that handle the results of tests.
+"""
+
+
+import os
+import sys
+import time
+import unittest as pyunit
+import warnings
+from collections import OrderedDict
+from types import TracebackType
+from typing import TYPE_CHECKING, List, Tuple, Type, Union
+
+from zope.interface import implementer
+
+from typing_extensions import TypeAlias
+
+from twisted.python import log, reflect
+from twisted.python.components import proxyForInterface
+from twisted.python.failure import Failure
+from twisted.python.util import untilConcludes
+from twisted.trial import itrial, util
+
+if TYPE_CHECKING:
+    from ._synctest import Todo
+
+try:
+    from subunit import TestProtocolClient  # type: ignore[import]
+except ImportError:
+    TestProtocolClient = None
+
+ExcInfo: TypeAlias = Tuple[Type[BaseException], BaseException, TracebackType]
+XUnitFailure = Union[ExcInfo, Tuple[None, None, None]]
+TrialFailure = Union[XUnitFailure, Failure]
+
+
+def _makeTodo(value: str) -> "Todo":
+    """
+    Return a L{Todo} object built from C{value}.
+
+    This is a synonym for L{twisted.trial.unittest.makeTodo}, but imported
+    locally to avoid circular imports.
+
+    @param value: A string or a tuple of C{(errors, reason)}, where C{errors}
+    is either a single exception class or an iterable of exception classes.
+
+    @return: A L{Todo} object.
+    """
+    from twisted.trial.unittest import makeTodo
+
+    return makeTodo(value)
+
+
+class BrokenTestCaseWarning(Warning):
+    """
+    Emitted as a warning when an exception occurs in one of setUp or tearDown.
+    """
+
+
+class SafeStream:
+    """
+    Wraps a stream object so that all C{write} calls are wrapped in
+    L{untilConcludes<twisted.python.util.untilConcludes>}.
+    """
+
+    def __init__(self, original):
+        self.original = original
+
+    def __getattr__(self, name):
+        return getattr(self.original, name)
+
+    def write(self, *a, **kw):
+        return untilConcludes(self.original.write, *a, **kw)
+
+
+@implementer(itrial.IReporter)
+class TestResult(pyunit.TestResult):
+    """
+    Accumulates the results of several L{twisted.trial.unittest.TestCase}s.
+
+    @ivar successes: count the number of successes achieved by the test run.
+    @type successes: C{int}
+    """
+
+    # Used when no todo provided to addExpectedFailure or addUnexpectedSuccess.
+    _DEFAULT_TODO = "Test expected to fail"
+
+    skips: List[Tuple[itrial.ITestCase, str]]
+    expectedFailures: List[Tuple[itrial.ITestCase, str, "Todo"]]  # type: ignore[assignment]
+    unexpectedSuccesses: List[Tuple[itrial.ITestCase, str]]  # type: ignore[assignment]
+    successes: int
+
+    def __init__(self):
+        super().__init__()
+        self.skips = []
+        self.expectedFailures = []
+        self.unexpectedSuccesses = []
+        self.successes = 0
+        self._timings = []
+
+    def __repr__(self) -> str:
+        return "<%s run=%d errors=%d failures=%d todos=%d dones=%d skips=%d>" % (
+            reflect.qual(self.__class__),
+            self.testsRun,
+            len(self.errors),
+            len(self.failures),
+            len(self.expectedFailures),
+            len(self.skips),
+            len(self.unexpectedSuccesses),
+        )
+
+    def _getTime(self):
+        return time.time()
+
+    def _getFailure(self, error):
+        """
+        Convert a C{sys.exc_info()}-style tuple to a L{Failure}, if necessary.
+        """
+        is_exc_info_tuple = isinstance(error, tuple) and len(error) == 3
+        if is_exc_info_tuple:
+            return Failure(error[1], error[0], error[2])
+        elif isinstance(error, Failure):
+            return error
+        raise TypeError(f"Cannot convert {error} to a Failure")
+
+    def startTest(self, test):
+        """
+        This must be called before the given test is commenced.
+
+        @type test: L{pyunit.TestCase}
+        """
+        super().startTest(test)
+        self._testStarted = self._getTime()
+
+    def stopTest(self, test):
+        """
+        This must be called after the given test is completed.
+
+        @type test: L{pyunit.TestCase}
+        """
+        super().stopTest(test)
+        self._lastTime = self._getTime() - self._testStarted
+
+    def addFailure(self, test, fail):
+        """
+        Report a failed assertion for the given test.
+
+        @type test: L{pyunit.TestCase}
+        @type fail: L{Failure} or L{tuple}
+        """
+        self.failures.append((test, self._getFailure(fail)))
+
+    def addError(self, test, error):
+        """
+        Report an error that occurred while running the given test.
+
+        @type test: L{pyunit.TestCase}
+        @type error: L{Failure} or L{tuple}
+        """
+        self.errors.append((test, self._getFailure(error)))
+
+    def addSkip(self, test, reason):
+        """
+        Report that the given test was skipped.
+
+        In Trial, tests can be 'skipped'. Tests are skipped mostly because
+        there is some platform or configuration issue that prevents them from
+        being run correctly.
+
+        @type test: L{pyunit.TestCase}
+        @type reason: L{str}
+        """
+        self.skips.append((test, reason))
+
+    def addUnexpectedSuccess(self, test, todo=None):
+        """
+        Report that the given test succeeded against expectations.
+
+        In Trial, tests can be marked 'todo'. That is, they are expected to
+        fail.  When a test that is expected to fail instead succeeds, it should
+        call this method to report the unexpected success.
+
+        @type test: L{pyunit.TestCase}
+        @type todo: L{unittest.Todo}, or L{None}, in which case a default todo
+            message is provided.
+        """
+        if todo is None:
+            todo = _makeTodo(self._DEFAULT_TODO)
+        self.unexpectedSuccesses.append((test, todo))
+
+    def addExpectedFailure(self, test, error, todo=None):
+        """
+        Report that the given test failed, and was expected to do so.
+
+        In Trial, tests can be marked 'todo'. That is, they are expected to
+        fail.
+
+        @type test: L{pyunit.TestCase}
+        @type error: L{Failure}
+        @type todo: L{unittest.Todo}, or L{None}, in which case a default todo
+            message is provided.
+        """
+        if todo is None:
+            todo = _makeTodo(self._DEFAULT_TODO)
+        self.expectedFailures.append((test, error, todo))
+
+    def addSuccess(self, test):
+        """
+        Report that the given test succeeded.
+
+        @type test: L{pyunit.TestCase}
+        """
+        self.successes += 1
+
+    def wasSuccessful(self):
+        """
+        Report whether or not this test suite was successful or not.
+
+        The behaviour of this method changed in L{pyunit} in Python 3.4 to
+        fail if there are any errors, failures, or unexpected successes.
+        Previous to 3.4, it was only if there were errors or failures. This
+        method implements the old behaviour for backwards compatibility reasons,
+        checking just for errors and failures.
+
+        @rtype: L{bool}
+        """
+        return len(self.failures) == len(self.errors) == 0
+
+    def done(self):
+        """
+        The test suite has finished running.
+        """
+
+
+@implementer(itrial.IReporter)
+class TestResultDecorator(
+    proxyForInterface(itrial.IReporter, "_originalReporter")  # type: ignore[misc]
+):
+    """
+    Base class for TestResult decorators.
+
+    @ivar _originalReporter: The wrapped instance of reporter.
+    @type _originalReporter: A provider of L{itrial.IReporter}
+    """
+
+
+@implementer(itrial.IReporter)
+class UncleanWarningsReporterWrapper(TestResultDecorator):
+    """
+    A wrapper for a reporter that converts L{util.DirtyReactorAggregateError}s
+    to warnings.
+    """
+
+    def addError(self, test, error):
+        """
+        If the error is a L{util.DirtyReactorAggregateError}, instead of
+        reporting it as a normal error, throw a warning.
+        """
+
+        if isinstance(error, Failure) and error.check(util.DirtyReactorAggregateError):
+            warnings.warn(error.getErrorMessage())
+        else:
+            self._originalReporter.addError(test, error)
+
+
+@implementer(itrial.IReporter)
+class _ExitWrapper(TestResultDecorator):
+    """
+    A wrapper for a reporter that causes the reporter to stop after
+    unsuccessful tests.
+    """
+
+    def addError(self, *args, **kwargs):
+        self.shouldStop = True
+        return self._originalReporter.addError(*args, **kwargs)
+
+    def addFailure(self, *args, **kwargs):
+        self.shouldStop = True
+        return self._originalReporter.addFailure(*args, **kwargs)
+
+
+class _AdaptedReporter(TestResultDecorator):
+    """
+    TestResult decorator that makes sure that addError only gets tests that
+    have been adapted with a particular test adapter.
+    """
+
+    def __init__(self, original, testAdapter):
+        """
+        Construct an L{_AdaptedReporter}.
+
+        @param original: An {itrial.IReporter}.
+        @param testAdapter: A callable that returns an L{itrial.ITestCase}.
+        """
+        TestResultDecorator.__init__(self, original)
+        self.testAdapter = testAdapter
+
+    def addError(self, test, error):
+        """
+        See L{itrial.IReporter}.
+        """
+        test = self.testAdapter(test)
+        return self._originalReporter.addError(test, error)
+
+    def addExpectedFailure(self, test, failure, todo=None):
+        """
+        See L{itrial.IReporter}.
+
+        @type test: A L{pyunit.TestCase}.
+        @type failure: A L{failure.Failure} or L{AssertionError}
+        @type todo: A L{unittest.Todo} or None
+
+        When C{todo} is L{None} a generic C{unittest.Todo} is built.
+
+        L{pyunit.TestCase}'s C{run()} calls this with 3 positional arguments
+        (without C{todo}).
+        """
+        return self._originalReporter.addExpectedFailure(
+            self.testAdapter(test), failure, todo
+        )
+
+    def addFailure(self, test, failure):
+        """
+        See L{itrial.IReporter}.
+        """
+        test = self.testAdapter(test)
+        return self._originalReporter.addFailure(test, failure)
+
+    def addSkip(self, test, skip):
+        """
+        See L{itrial.IReporter}.
+        """
+        test = self.testAdapter(test)
+        return self._originalReporter.addSkip(test, skip)
+
+    def addUnexpectedSuccess(self, test, todo=None):
+        """
+        See L{itrial.IReporter}.
+
+        @type test: A L{pyunit.TestCase}.
+        @type todo: A L{unittest.Todo} or None
+
+        When C{todo} is L{None} a generic C{unittest.Todo} is built.
+
+        L{pyunit.TestCase}'s C{run()} calls this with 2 positional arguments
+        (without C{todo}).
+        """
+        test = self.testAdapter(test)
+        return self._originalReporter.addUnexpectedSuccess(test, todo)
+
+    def startTest(self, test):
+        """
+        See L{itrial.IReporter}.
+        """
+        return self._originalReporter.startTest(self.testAdapter(test))
+
+    def stopTest(self, test):
+        """
+        See L{itrial.IReporter}.
+        """
+        return self._originalReporter.stopTest(self.testAdapter(test))
+
+
+@implementer(itrial.IReporter)
+class Reporter(TestResult):
+    """
+    A basic L{TestResult} with support for writing to a stream.
+
+    @ivar _startTime: The time when the first test was started. It defaults to
+        L{None}, which means that no test was actually launched.
+    @type _startTime: C{float} or L{None}
+
+    @ivar _warningCache: A C{set} of tuples of warning message (file, line,
+        text, category) which have already been written to the output stream
+        during the currently executing test.  This is used to avoid writing
+        duplicates of the same warning to the output stream.
+    @type _warningCache: C{set}
+
+    @ivar _publisher: The log publisher which will be observed for warning
+        events.
+    @type _publisher: L{twisted.python.log.LogPublisher}
+    """
+
+    _separator = "-" * 79
+    _doubleSeparator = "=" * 79
+
+    def __init__(
+        self, stream=sys.stdout, tbformat="default", realtime=False, publisher=None
+    ):
+        super().__init__()
+        self._stream = SafeStream(stream)
+        self.tbformat = tbformat
+        self.realtime = realtime
+        self._startTime = None
+        self._warningCache = set()
+
+        # Start observing log events so as to be able to report warnings.
+        self._publisher = publisher
+        if publisher is not None:
+            publisher.addObserver(self._observeWarnings)
+
+    def _observeWarnings(self, event):
+        """
+        Observe warning events and write them to C{self._stream}.
+
+        This method is a log observer which will be registered with
+        C{self._publisher.addObserver}.
+
+        @param event: A C{dict} from the logging system.  If it has a
+            C{'warning'} key, a logged warning will be extracted from it and
+            possibly written to C{self.stream}.
+        """
+        if "warning" in event:
+            key = (
+                event["filename"],
+                event["lineno"],
+                event["category"].split(".")[-1],
+                str(event["warning"]),
+            )
+            if key not in self._warningCache:
+                self._warningCache.add(key)
+                self._stream.write("%s:%s: %s: %s\n" % key)
+
+    def startTest(self, test):
+        """
+        Called when a test begins to run. Records the time when it was first
+        called and resets the warning cache.
+
+        @param test: L{ITestCase}
+        """
+        super().startTest(test)
+        if self._startTime is None:
+            self._startTime = self._getTime()
+        self._warningCache = set()
+
+    def addFailure(self, test, fail):
+        """
+        Called when a test fails. If C{realtime} is set, then it prints the
+        error to the stream.
+
+        @param test: L{ITestCase} that failed.
+        @param fail: L{failure.Failure} containing the error.
+        """
+        super().addFailure(test, fail)
+        if self.realtime:
+            fail = self.failures[-1][1]  # guarantee it's a Failure
+            self._write(self._formatFailureTraceback(fail))
+
+    def addError(self, test, error):
+        """
+        Called when a test raises an error. If C{realtime} is set, then it
+        prints the error to the stream.
+
+        @param test: L{ITestCase} that raised the error.
+        @param error: L{failure.Failure} containing the error.
+        """
+        error = self._getFailure(error)
+        super().addError(test, error)
+        if self.realtime:
+            error = self.errors[-1][1]  # guarantee it's a Failure
+            self._write(self._formatFailureTraceback(error))
+
+    def _write(self, format, *args):
+        """
+        Safely write to the reporter's stream.
+
+        @param format: A format string to write.
+        @param args: The arguments for the format string.
+        """
+        s = str(format)
+        assert isinstance(s, str)
+        if args:
+            self._stream.write(s % args)
+        else:
+            self._stream.write(s)
+        untilConcludes(self._stream.flush)
+
+    def _writeln(self, format, *args):
+        """
+        Safely write a line to the reporter's stream. Newline is appended to
+        the format string.
+
+        @param format: A format string to write.
+        @param args: The arguments for the format string.
+        """
+        self._write(format, *args)
+        self._write("\n")
+
+    def upDownError(self, method, error, warn=True, printStatus=True):
+        super().upDownError(method, error, warn, printStatus)
+        if warn:
+            tbStr = self._formatFailureTraceback(error)
+            log.msg(tbStr)
+            msg = "caught exception in {}, your TestCase is broken\n\n{}".format(
+                method,
+                tbStr,
+            )
+            warnings.warn(msg, BrokenTestCaseWarning, stacklevel=2)
+
+    def cleanupErrors(self, errs):
+        super().cleanupErrors(errs)
+        warnings.warn(
+            "%s\n%s"
+            % (
+                "REACTOR UNCLEAN! traceback(s) follow: ",
+                self._formatFailureTraceback(errs),
+            ),
+            BrokenTestCaseWarning,
+        )
+
+    def _trimFrames(self, frames):
+        """
+        Trim frames to remove internal paths.
+
+        When a C{SynchronousTestCase} method fails synchronously, the stack
+        looks like this:
+         - [0]: C{SynchronousTestCase._run}
+         - [1]: C{util.runWithWarningsSuppressed}
+         - [2:-2]: code in the test method which failed
+         - [-1]: C{_synctest.fail}
+
+        When a C{TestCase} method fails synchronously, the stack looks like
+        this:
+         - [0]: C{defer.maybeDeferred}
+         - [1]: C{utils.runWithWarningsSuppressed}
+         - [2]: C{utils.runWithWarningsSuppressed}
+         - [3:-2]: code in the test method which failed
+         - [-1]: C{_synctest.fail}
+
+        When a method fails inside a C{Deferred} (i.e., when the test method
+        returns a C{Deferred}, and that C{Deferred}'s errback fires), the stack
+        captured inside the resulting C{Failure} looks like this:
+         - [0]: C{defer.Deferred._runCallbacks}
+         - [1:-2]: code in the testmethod which failed
+         - [-1]: C{_synctest.fail}
+
+        As a result, we want to trim either [maybeDeferred, runWWS, runWWS] or
+        [Deferred._runCallbacks] or [SynchronousTestCase._run, runWWS] from the
+        front, and trim the [unittest.fail] from the end.
+
+        There is also another case, when the test method is badly defined and
+        contains extra arguments.
+
+        If it doesn't recognize one of these cases, it just returns the
+        original frames.
+
+        @param frames: The C{list} of frames from the test failure.
+
+        @return: The C{list} of frames to display.
+        """
+        newFrames = list(frames)
+
+        if len(frames) < 2:
+            return newFrames
+
+        firstMethod = newFrames[0][0]
+        firstFile = os.path.splitext(os.path.basename(newFrames[0][1]))[0]
+
+        secondMethod = newFrames[1][0]
+        secondFile = os.path.splitext(os.path.basename(newFrames[1][1]))[0]
+
+        syncCase = (("_run", "_synctest"), ("runWithWarningsSuppressed", "util"))
+        asyncCase = (("maybeDeferred", "defer"), ("runWithWarningsSuppressed", "utils"))
+
+        twoFrames = ((firstMethod, firstFile), (secondMethod, secondFile))
+
+        # On PY3, we have an extra frame which is reraising the exception
+        for frame in newFrames:
+            frameFile = os.path.splitext(os.path.basename(frame[1]))[0]
+            if frameFile == "compat" and frame[0] == "reraise":
+                # If it's in the compat module and is reraise, BLAM IT
+                newFrames.pop(newFrames.index(frame))
+
+        if twoFrames == syncCase:
+            newFrames = newFrames[2:]
+        elif twoFrames == asyncCase:
+            newFrames = newFrames[3:]
+        elif (firstMethod, firstFile) == ("_runCallbacks", "defer"):
+            newFrames = newFrames[1:]
+
+        if not newFrames:
+            # The method fails before getting called, probably an argument
+            # problem
+            return newFrames
+
+        last = newFrames[-1]
+        if (
+            last[0].startswith("fail")
+            and os.path.splitext(os.path.basename(last[1]))[0] == "_synctest"
+        ):
+            newFrames = newFrames[:-1]
+
+        return newFrames
+
+    def _formatFailureTraceback(self, fail):
+        if isinstance(fail, str):
+            return fail.rstrip() + "\n"
+        fail.frames, frames = self._trimFrames(fail.frames), fail.frames
+        result = fail.getTraceback(detail=self.tbformat, elideFrameworkCode=True)
+        fail.frames = frames
+        return result
+
+    def _groupResults(self, results, formatter):
+        """
+        Group tests together based on their results.
+
+        @param results: An iterable of tuples of two or more elements.  The
+            first element of each tuple is a test case.  The remaining
+            elements describe the outcome of that test case.
+
+        @param formatter: A callable which turns a test case result into a
+            string.  The elements after the first of the tuples in
+            C{results} will be passed as positional arguments to
+            C{formatter}.
+
+        @return: A C{list} of two-tuples.  The first element of each tuple
+            is a unique string describing one result from at least one of
+            the test cases in C{results}.  The second element is a list of
+            the test cases which had that result.
+        """
+        groups = OrderedDict()
+        for content in results:
+            case = content[0]
+            outcome = content[1:]
+            key = formatter(*outcome)
+            groups.setdefault(key, []).append(case)
+        return list(groups.items())
+
+    def _printResults(self, flavor, errors, formatter):
+        """
+        Print a group of errors to the stream.
+
+        @param flavor: A string indicating the kind of error (e.g. 'TODO').
+        @param errors: A list of errors, often L{failure.Failure}s, but
+            sometimes 'todo' errors.
+        @param formatter: A callable that knows how to format the errors.
+        """
+        for reason, cases in self._groupResults(errors, formatter):
+            self._writeln(self._doubleSeparator)
+            self._writeln(flavor)
+            self._write(reason)
+            self._writeln("")
+            for case in cases:
+                self._writeln(case.id())
+
+    def _printExpectedFailure(self, error, todo):
+        return "Reason: {!r}\n{}".format(
+            todo.reason, self._formatFailureTraceback(error)
+        )
+
+    def _printUnexpectedSuccess(self, todo):
+        ret = f"Reason: {todo.reason!r}\n"
+        if todo.errors:
+            ret += "Expected errors: {}\n".format(", ".join(todo.errors))
+        return ret
+
+    def _printErrors(self):
+        """
+        Print all of the non-success results to the stream in full.
+        """
+        self._write("\n")
+        self._printResults("[SKIPPED]", self.skips, lambda x: "%s\n" % x)
+        self._printResults("[TODO]", self.expectedFailures, self._printExpectedFailure)
+        self._printResults("[FAIL]", self.failures, self._formatFailureTraceback)
+        self._printResults("[ERROR]", self.errors, self._formatFailureTraceback)
+        self._printResults(
+            "[SUCCESS!?!]", self.unexpectedSuccesses, self._printUnexpectedSuccess
+        )
+
+    def _getSummary(self):
+        """
+        Return a formatted count of tests status results.
+        """
+        summaries = []
+        for stat in (
+            "skips",
+            "expectedFailures",
+            "failures",
+            "errors",
+            "unexpectedSuccesses",
+        ):
+            num = len(getattr(self, stat))
+            if num:
+                summaries.append("%s=%d" % (stat, num))
+        if self.successes:
+            summaries.append("successes=%d" % (self.successes,))
+        summary = (summaries and " (" + ", ".join(summaries) + ")") or ""
+        return summary
+
+    def _printSummary(self):
+        """
+        Print a line summarising the test results to the stream.
+        """
+        summary = self._getSummary()
+        if self.wasSuccessful():
+            status = "PASSED"
+        else:
+            status = "FAILED"
+        self._write("%s%s\n", status, summary)
+
+    def done(self):
+        """
+        Summarize the result of the test run.
+
+        The summary includes a report of all of the errors, todos, skips and
+        so forth that occurred during the run. It also includes the number of
+        tests that were run and how long it took to run them (not including
+        load time).
+
+        Expects that C{_printErrors}, C{_writeln}, C{_write}, C{_printSummary}
+        and C{_separator} are all implemented.
+        """
+        if self._publisher is not None:
+            self._publisher.removeObserver(self._observeWarnings)
+        self._printErrors()
+        self._writeln(self._separator)
+        if self._startTime is not None:
+            self._writeln(
+                "Ran %d tests in %.3fs", self.testsRun, time.time() - self._startTime
+            )
+        self._write("\n")
+        self._printSummary()
+
+
+class MinimalReporter(Reporter):
+    """
+    A minimalist reporter that prints only a summary of the test result, in
+    the form of (timeTaken, #tests, #tests, #errors, #failures, #skips).
+    """
+
+    def _printErrors(self):
+        """
+        Don't print a detailed summary of errors. We only care about the
+        counts.
+        """
+
+    def _printSummary(self):
+        """
+        Print out a one-line summary of the form:
+        '%(runtime) %(number_of_tests) %(number_of_tests) %(num_errors)
+        %(num_failures) %(num_skips)'
+        """
+        numTests = self.testsRun
+        if self._startTime is not None:
+            timing = self._getTime() - self._startTime
+        else:
+            timing = 0
+        t = (
+            timing,
+            numTests,
+            numTests,
+            len(self.errors),
+            len(self.failures),
+            len(self.skips),
+        )
+        self._writeln(" ".join(map(str, t)))
+
+
+class TextReporter(Reporter):
+    """
+    Simple reporter that prints a single character for each test as it runs,
+    along with the standard Trial summary text.
+    """
+
+    def addSuccess(self, test):
+        super().addSuccess(test)
+        self._write(".")
+
+    def addError(self, *args):
+        super().addError(*args)
+        self._write("E")
+
+    def addFailure(self, *args):
+        super().addFailure(*args)
+        self._write("F")
+
+    def addSkip(self, *args):
+        super().addSkip(*args)
+        self._write("S")
+
+    def addExpectedFailure(self, *args):
+        super().addExpectedFailure(*args)
+        self._write("T")
+
+    def addUnexpectedSuccess(self, *args):
+        super().addUnexpectedSuccess(*args)
+        self._write("!")
+
+
+class VerboseTextReporter(Reporter):
+    """
+    A verbose reporter that prints the name of each test as it is running.
+
+    Each line is printed with the name of the test, followed by the result of
+    that test.
+    """
+
+    # This is actually the bwverbose option
+
+    def startTest(self, tm):
+        self._write("%s ... ", tm.id())
+        super().startTest(tm)
+
+    def addSuccess(self, test):
+        super().addSuccess(test)
+        self._write("[OK]")
+
+    def addError(self, *args):
+        super().addError(*args)
+        self._write("[ERROR]")
+
+    def addFailure(self, *args):
+        super().addFailure(*args)
+        self._write("[FAILURE]")
+
+    def addSkip(self, *args):
+        super().addSkip(*args)
+        self._write("[SKIPPED]")
+
+    def addExpectedFailure(self, *args):
+        super().addExpectedFailure(*args)
+        self._write("[TODO]")
+
+    def addUnexpectedSuccess(self, *args):
+        super().addUnexpectedSuccess(*args)
+        self._write("[SUCCESS!?!]")
+
+    def stopTest(self, test):
+        super().stopTest(test)
+        self._write("\n")
+
+
+class TimingTextReporter(VerboseTextReporter):
+    """
+    Prints out each test as it is running, followed by the time taken for each
+    test to run.
+    """
+
+    def stopTest(self, method):
+        """
+        Mark the test as stopped, and write the time it took to run the test
+        to the stream.
+        """
+        super().stopTest(method)
+        self._write("(%.03f secs)\n" % self._lastTime)
+
+
+class _AnsiColorizer:
+    """
+    A colorizer is an object that loosely wraps around a stream, allowing
+    callers to write text to the stream in a particular color.
+
+    Colorizer classes must implement C{supported()} and C{write(text, color)}.
+    """
+
+    _colors = dict(
+        black=30, red=31, green=32, yellow=33, blue=34, magenta=35, cyan=36, white=37
+    )
+
+    def __init__(self, stream):
+        self.stream = stream
+
+    @classmethod
+    def supported(cls, stream=sys.stdout):
+        """
+        A class method that returns True if the current platform supports
+        coloring terminal output using this method. Returns False otherwise.
+        """
+        if not stream.isatty():
+            return False  # auto color only on TTYs
+        try:
+            import curses
+        except ImportError:
+            return False
+        else:
+            try:
+                try:
+                    return curses.tigetnum("colors") > 2
+                except curses.error:
+                    curses.setupterm()
+                    return curses.tigetnum("colors") > 2
+            except BaseException:
+                # guess false in case of error
+                return False
+
+    def write(self, text, color):
+        """
+        Write the given text to the stream in the given color.
+
+        @param text: Text to be written to the stream.
+
+        @param color: A string label for a color. e.g. 'red', 'white'.
+        """
+        color = self._colors[color]
+        self.stream.write(f"\x1b[{color};1m{text}\x1b[0m")
+
+
+class _Win32Colorizer:
+    """
+    See _AnsiColorizer docstring.
+    """
+
+    def __init__(self, stream):
+        from win32console import (  # type: ignore[import]
+            FOREGROUND_BLUE,
+            FOREGROUND_GREEN,
+            FOREGROUND_INTENSITY,
+            FOREGROUND_RED,
+            STD_OUTPUT_HANDLE,
+            GetStdHandle,
+        )
+
+        red, green, blue, bold = (
+            FOREGROUND_RED,
+            FOREGROUND_GREEN,
+            FOREGROUND_BLUE,
+            FOREGROUND_INTENSITY,
+        )
+        self.stream = stream
+        self.screenBuffer = GetStdHandle(STD_OUTPUT_HANDLE)
+        self._colors = {
+            "normal": red | green | blue,
+            "red": red | bold,
+            "green": green | bold,
+            "blue": blue | bold,
+            "yellow": red | green | bold,
+            "magenta": red | blue | bold,
+            "cyan": green | blue | bold,
+            "white": red | green | blue | bold,
+        }
+
+    @classmethod
+    def supported(cls, stream=sys.stdout):
+        try:
+            import win32console
+
+            screenBuffer = win32console.GetStdHandle(win32console.STD_OUTPUT_HANDLE)
+        except ImportError:
+            return False
+        import pywintypes  # type: ignore[import]
+
+        try:
+            screenBuffer.SetConsoleTextAttribute(
+                win32console.FOREGROUND_RED
+                | win32console.FOREGROUND_GREEN
+                | win32console.FOREGROUND_BLUE
+            )
+        except pywintypes.error:
+            return False
+        else:
+            return True
+
+    def write(self, text, color):
+        color = self._colors[color]
+        self.screenBuffer.SetConsoleTextAttribute(color)
+        self.stream.write(text)
+        self.screenBuffer.SetConsoleTextAttribute(self._colors["normal"])
+
+
+class _NullColorizer:
+    """
+    See _AnsiColorizer docstring.
+    """
+
+    def __init__(self, stream):
+        self.stream = stream
+
+    @classmethod
+    def supported(cls, stream=sys.stdout):
+        return True
+
+    def write(self, text, color):
+        self.stream.write(text)
+
+
+@implementer(itrial.IReporter)
+class SubunitReporter:
+    """
+    Reports test output via Subunit.
+
+    @ivar _subunit: The subunit protocol client that we are wrapping.
+
+    @ivar _successful: An internal variable, used to track whether we have
+        received only successful results.
+
+    @since: 10.0
+    """
+
+    testsRun = None
+
+    def __init__(
+        self, stream=sys.stdout, tbformat="default", realtime=False, publisher=None
+    ):
+        """
+        Construct a L{SubunitReporter}.
+
+        @param stream: A file-like object representing the stream to print
+            output to. Defaults to stdout.
+        @param tbformat: The format for tracebacks. Ignored, since subunit
+            always uses Python's standard format.
+        @param realtime: Whether or not to print exceptions in the middle
+            of the test results. Ignored, since subunit always does this.
+        @param publisher: The log publisher which will be preserved for
+            reporting events. Ignored, as it's not relevant to subunit.
+        """
+        if TestProtocolClient is None:
+            raise Exception("Subunit not available")
+        self._subunit = TestProtocolClient(stream)
+        self._successful = True
+
+    def done(self):
+        """
+        Record that the entire test suite run is finished.
+
+        We do nothing, since a summary clause is irrelevant to the subunit
+        protocol.
+        """
+        pass
+
+    @property
+    def shouldStop(self):
+        """
+        Whether or not the test runner should stop running tests.
+        """
+        return self._subunit.shouldStop
+
+    def stop(self):
+        """
+        Signal that the test runner should stop running tests.
+        """
+        return self._subunit.stop()
+
+    def wasSuccessful(self):
+        """
+        Has the test run been successful so far?
+
+        @return: C{True} if we have received no reports of errors or failures,
+            C{False} otherwise.
+        """
+        # Subunit has a bug in its implementation of wasSuccessful, see
+        # https://bugs.edge.launchpad.net/subunit/+bug/491090, so we can't
+        # simply forward it on.
+        return self._successful
+
+    def startTest(self, test):
+        """
+        Record that C{test} has started.
+        """
+        return self._subunit.startTest(test)
+
+    def stopTest(self, test):
+        """
+        Record that C{test} has completed.
+        """
+        return self._subunit.stopTest(test)
+
+    def addSuccess(self, test):
+        """
+        Record that C{test} was successful.
+        """
+        return self._subunit.addSuccess(test)
+
+    def addSkip(self, test, reason):
+        """
+        Record that C{test} was skipped for C{reason}.
+
+        Some versions of subunit don't have support for addSkip. In those
+        cases, the skip is reported as a success.
+
+        @param test: A unittest-compatible C{TestCase}.
+        @param reason: The reason for it being skipped. The C{str()} of this
+            object will be included in the subunit output stream.
+        """
+        addSkip = getattr(self._subunit, "addSkip", None)
+        if addSkip is None:
+            self.addSuccess(test)
+        else:
+            self._subunit.addSkip(test, reason)
+
+    def addError(self, test, err):
+        """
+        Record that C{test} failed with an unexpected error C{err}.
+
+        Also marks the run as being unsuccessful, causing
+        L{SubunitReporter.wasSuccessful} to return C{False}.
+        """
+        self._successful = False
+        return self._subunit.addError(test, util.excInfoOrFailureToExcInfo(err))
+
+    def addFailure(self, test, err):
+        """
+        Record that C{test} failed an assertion with the error C{err}.
+
+        Also marks the run as being unsuccessful, causing
+        L{SubunitReporter.wasSuccessful} to return C{False}.
+        """
+        self._successful = False
+        return self._subunit.addFailure(test, util.excInfoOrFailureToExcInfo(err))
+
+    def addExpectedFailure(self, test, failure, todo=None):
+        """
+        Record an expected failure from a test.
+
+        Some versions of subunit do not implement this. For those versions, we
+        record a success.
+        """
+        failure = util.excInfoOrFailureToExcInfo(failure)
+        addExpectedFailure = getattr(self._subunit, "addExpectedFailure", None)
+        if addExpectedFailure is None:
+            self.addSuccess(test)
+        else:
+            addExpectedFailure(test, failure)
+
+    def addUnexpectedSuccess(self, test, todo=None):
+        """
+        Record an unexpected success.
+
+        Since subunit has no way of expressing this concept, we record a
+        success on the subunit stream.
+        """
+        # Not represented in pyunit/subunit.
+        self.addSuccess(test)
+
+
+class TreeReporter(Reporter):
+    """
+    Print out the tests in the form a tree.
+
+    Tests are indented according to which class and module they belong.
+    Results are printed in ANSI color.
+    """
+
+    currentLine = ""
+    indent = "  "
+    columns = 79
+
+    FAILURE = "red"
+    ERROR = "red"
+    TODO = "blue"
+    SKIP = "blue"
+    TODONE = "red"
+    SUCCESS = "green"
+
+    def __init__(self, stream=sys.stdout, *args, **kwargs):
+        super().__init__(stream, *args, **kwargs)
+        self._lastTest = []
+        for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]:
+            if colorizer.supported(stream):
+                self._colorizer = colorizer(stream)
+                break
+
+    def getDescription(self, test):
+        """
+        Return the name of the method which 'test' represents.  This is
+        what gets displayed in the leaves of the tree.
+
+        e.g. getDescription(TestCase('test_foo')) ==> test_foo
+        """
+        return test.id().split(".")[-1]
+
+    def addSuccess(self, test):
+        super().addSuccess(test)
+        self.endLine("[OK]", self.SUCCESS)
+
+    def addError(self, *args):
+        super().addError(*args)
+        self.endLine("[ERROR]", self.ERROR)
+
+    def addFailure(self, *args):
+        super().addFailure(*args)
+        self.endLine("[FAIL]", self.FAILURE)
+
+    def addSkip(self, *args):
+        super().addSkip(*args)
+        self.endLine("[SKIPPED]", self.SKIP)
+
+    def addExpectedFailure(self, *args):
+        super().addExpectedFailure(*args)
+        self.endLine("[TODO]", self.TODO)
+
+    def addUnexpectedSuccess(self, *args):
+        super().addUnexpectedSuccess(*args)
+        self.endLine("[SUCCESS!?!]", self.TODONE)
+
+    def _write(self, format, *args):
+        if args:
+            format = format % args
+        self.currentLine = format
+        super()._write(self.currentLine)
+
+    def _getPreludeSegments(self, testID):
+        """
+        Return a list of all non-leaf segments to display in the tree.
+
+        Normally this is the module and class name.
+        """
+        segments = testID.split(".")[:-1]
+        if len(segments) == 0:
+            return segments
+        segments = [
+            seg for seg in (".".join(segments[:-1]), segments[-1]) if len(seg) > 0
+        ]
+        return segments
+
+    def _testPrelude(self, testID):
+        """
+        Write the name of the test to the stream, indenting it appropriately.
+
+        If the test is the first test in a new 'branch' of the tree, also
+        write all of the parents in that branch.
+        """
+        segments = self._getPreludeSegments(testID)
+        indentLevel = 0
+        for seg in segments:
+            if indentLevel < len(self._lastTest):
+                if seg != self._lastTest[indentLevel]:
+                    self._write(f"{self.indent * indentLevel}{seg}\n")
+            else:
+                self._write(f"{self.indent * indentLevel}{seg}\n")
+            indentLevel += 1
+        self._lastTest = segments
+
+    def cleanupErrors(self, errs):
+        self._colorizer.write("    cleanup errors", self.ERROR)
+        self.endLine("[ERROR]", self.ERROR)
+        super().cleanupErrors(errs)
+
+    def upDownError(self, method, error, warn, printStatus):
+        self._colorizer.write("  %s" % method, self.ERROR)
+        if printStatus:
+            self.endLine("[ERROR]", self.ERROR)
+        super().upDownError(method, error, warn, printStatus)
+
+    def startTest(self, test):
+        """
+        Called when C{test} starts. Writes the tests name to the stream using
+        a tree format.
+        """
+        self._testPrelude(test.id())
+        self._write(
+            "%s%s ... "
+            % (self.indent * (len(self._lastTest)), self.getDescription(test))
+        )
+        super().startTest(test)
+
+    def endLine(self, message, color):
+        """
+        Print 'message' in the given color.
+
+        @param message: A string message, usually '[OK]' or something similar.
+        @param color: A string color, 'red', 'green' and so forth.
+        """
+        spaces = " " * (self.columns - len(self.currentLine) - len(message))
+        super()._write(spaces)
+        self._colorizer.write(message, color)
+        super()._write("\n")
+
+    def _printSummary(self):
+        """
+        Print a line summarising the test results to the stream, and color the
+        status result.
+        """
+        summary = self._getSummary()
+        if self.wasSuccessful():
+            status = "PASSED"
+            color = self.SUCCESS
+        else:
+            status = "FAILED"
+            color = self.FAILURE
+        self._colorizer.write(status, color)
+        self._write("%s\n", summary)
diff --git a/contrib/python/Twisted/py3/twisted/trial/runner.py b/contrib/python/Twisted/py3/twisted/trial/runner.py
new file mode 100644
index 00000000000..ffc554e25c8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/runner.py
@@ -0,0 +1,987 @@
+# -*- test-case-name: twisted.trial.test.test_runner -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A miscellany of code used to run Trial tests.
+
+Maintainer: Jonathan Lange
+"""
+
+
+__all__ = [
+    "TestSuite",
+    "DestructiveTestSuite",
+    "ErrorHolder",
+    "LoggedSuite",
+    "TestHolder",
+    "TestLoader",
+    "TrialRunner",
+    "TrialSuite",
+    "filenameToModule",
+    "isPackage",
+    "isPackageDirectory",
+    "isTestCase",
+    "name",
+    "samefile",
+    "NOT_IN_TEST",
+]
+
+import doctest
+import importlib
+import inspect
+import os
+import sys
+import types
+import unittest as pyunit
+import warnings
+from contextlib import contextmanager
+from importlib.machinery import SourceFileLoader
+from typing import Callable, Generator, List, Optional, TextIO, Type, Union
+
+from zope.interface import implementer
+
+from attrs import define
+from typing_extensions import ParamSpec, Protocol, TypeAlias, TypeGuard
+
+from twisted.internet import defer
+from twisted.python import failure, filepath, log, modules, reflect
+from twisted.trial import unittest, util
+from twisted.trial._asyncrunner import _ForceGarbageCollectionDecorator, _iterateTests
+from twisted.trial._synctest import _logObserver
+from twisted.trial.itrial import ITestCase
+from twisted.trial.reporter import UncleanWarningsReporterWrapper, _ExitWrapper
+
+# These are imported so that they remain in the public API for t.trial.runner
+from twisted.trial.unittest import TestSuite
+from . import itrial
+
+_P = ParamSpec("_P")
+
+
+class _Debugger(Protocol):
+    def runcall(
+        self, f: Callable[_P, object], *args: _P.args, **kwargs: _P.kwargs
+    ) -> object:
+        ...
+
+
+def isPackage(module):
+    """Given an object return True if the object looks like a package"""
+    if not isinstance(module, types.ModuleType):
+        return False
+    basename = os.path.splitext(os.path.basename(module.__file__))[0]
+    return basename == "__init__"
+
+
+def isPackageDirectory(dirname):
+    """
+    Is the directory at path 'dirname' a Python package directory?
+    Returns the name of the __init__ file (it may have a weird extension)
+    if dirname is a package directory.  Otherwise, returns False
+    """
+
+    def _getSuffixes():
+        return importlib.machinery.all_suffixes()
+
+    for ext in _getSuffixes():
+        initFile = "__init__" + ext
+        if os.path.exists(os.path.join(dirname, initFile)):
+            return initFile
+    return False
+
+
+def samefile(filename1, filename2):
+    """
+    A hacky implementation of C{os.path.samefile}. Used by L{filenameToModule}
+    when the platform doesn't provide C{os.path.samefile}. Do not use this.
+    """
+    return os.path.abspath(filename1) == os.path.abspath(filename2)
+
+
+def filenameToModule(fn):
+    """
+    Given a filename, do whatever possible to return a module object matching
+    that file.
+
+    If the file in question is a module in Python path, properly import and
+    return that module. Otherwise, load the source manually.
+
+    @param fn: A filename.
+    @return: A module object.
+    @raise ValueError: If C{fn} does not exist.
+    """
+    oldFn = fn
+
+    if (3, 8) <= sys.version_info < (3, 10) and not os.path.isabs(fn):
+        # module.__spec__.__file__ is supposed to be absolute in py3.8+
+        # importlib.util.spec_from_file_location does this automatically from
+        # 3.10+
+        # This was backported to 3.8 and 3.9, but then reverted in 3.8.11 and
+        # 3.9.6
+        # See https://twistedmatrix.com/trac/ticket/10230
+        # and https://bugs.python.org/issue44070
+        fn = os.path.join(os.getcwd(), fn)
+
+    if not os.path.exists(fn):
+        raise ValueError(f"{oldFn!r} doesn't exist")
+
+    moduleName = reflect.filenameToModuleName(fn)
+    try:
+        ret = reflect.namedAny(moduleName)
+    except (ValueError, AttributeError):
+        # Couldn't find module.  The file 'fn' is not in PYTHONPATH
+        return _importFromFile(fn, moduleName=moduleName)
+
+    # >=3.7 has __file__ attribute as None, previously __file__ was not present
+    if getattr(ret, "__file__", None) is None:
+        # This isn't a Python module in a package, so import it from a file
+        return _importFromFile(fn, moduleName=moduleName)
+
+    # ensure that the loaded module matches the file
+    retFile = os.path.splitext(ret.__file__)[0] + ".py"
+    # not all platforms (e.g. win32) have os.path.samefile
+    same = getattr(os.path, "samefile", samefile)
+    if os.path.isfile(fn) and not same(fn, retFile):
+        del sys.modules[ret.__name__]
+        ret = _importFromFile(fn, moduleName=moduleName)
+    return ret
+
+
+def _importFromFile(fn, *, moduleName):
+    fn = _resolveDirectory(fn)
+    if not moduleName:
+        moduleName = os.path.splitext(os.path.split(fn)[-1])[0]
+    if moduleName in sys.modules:
+        return sys.modules[moduleName]
+
+    spec = importlib.util.spec_from_file_location(moduleName, fn)
+    if not spec:
+        raise SyntaxError(fn)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    sys.modules[moduleName] = module
+    return module
+
+
+def _resolveDirectory(fn):
+    if os.path.isdir(fn):
+        initFile = isPackageDirectory(fn)
+        if initFile:
+            fn = os.path.join(fn, initFile)
+        else:
+            raise ValueError(f"{fn!r} is not a package directory")
+    return fn
+
+
+def _getMethodNameInClass(method):
+    """
+    Find the attribute name on the method's class which refers to the method.
+
+    For some methods, notably decorators which have not had __name__ set correctly:
+
+    getattr(method.im_class, method.__name__) != method
+    """
+    if getattr(method.im_class, method.__name__, object()) != method:
+        for alias in dir(method.im_class):
+            if getattr(method.im_class, alias, object()) == method:
+                return alias
+    return method.__name__
+
+
+class DestructiveTestSuite(TestSuite):
+    """
+    A test suite which remove the tests once run, to minimize memory usage.
+    """
+
+    def run(self, result):
+        """
+        Almost the same as L{TestSuite.run}, but with C{self._tests} being
+        empty at the end.
+        """
+        while self._tests:
+            if result.shouldStop:
+                break
+            test = self._tests.pop(0)
+            test(result)
+        return result
+
+
+# When an error occurs outside of any test, the user will see this string
+# in place of a test's name.
+NOT_IN_TEST = "<not in test>"
+
+
+class LoggedSuite(TestSuite):
+    """
+    Any errors logged in this suite will be reported to the L{TestResult}
+    object.
+    """
+
+    def run(self, result):
+        """
+        Run the suite, storing all errors in C{result}. If an error is logged
+        while no tests are running, then it will be added as an error to
+        C{result}.
+
+        @param result: A L{TestResult} object.
+        """
+        observer = _logObserver
+        observer._add()
+        super().run(result)
+        observer._remove()
+        for error in observer.getErrors():
+            result.addError(TestHolder(NOT_IN_TEST), error)
+        observer.flushErrors()
+
+
+class TrialSuite(TestSuite):
+    """
+    Suite to wrap around every single test in a C{trial} run. Used internally
+    by Trial to set up things necessary for Trial tests to work, regardless of
+    what context they are run in.
+    """
+
+    def __init__(self, tests=(), forceGarbageCollection=False):
+        if forceGarbageCollection:
+            newTests = []
+            for test in tests:
+                test = unittest.decorate(test, _ForceGarbageCollectionDecorator)
+                newTests.append(test)
+            tests = newTests
+        suite = LoggedSuite(tests)
+        super().__init__([suite])
+
+    def _bail(self):
+        from twisted.internet import reactor
+
+        d = defer.Deferred()
+        reactor.addSystemEventTrigger("after", "shutdown", lambda: d.callback(None))
+        reactor.fireSystemEvent("shutdown")  # radix's suggestion
+        # As long as TestCase does crap stuff with the reactor we need to
+        # manually shutdown the reactor here, and that requires util.wait
+        # :(
+        # so that the shutdown event completes
+        unittest.TestCase("mktemp")._wait(d)
+
+    def run(self, result):
+        try:
+            TestSuite.run(self, result)
+        finally:
+            self._bail()
+
+
+_Loadable: TypeAlias = Union[
+    modules.PythonAttribute,
+    modules.PythonModule,
+    pyunit.TestCase,
+    Type[pyunit.TestCase],
+]
+
+
+def name(thing: _Loadable) -> str:
+    """
+    @param thing: an object from modules (instance of PythonModule,
+        PythonAttribute), a TestCase subclass, or an instance of a TestCase.
+    """
+    if isinstance(thing, pyunit.TestCase):
+        return thing.id()
+
+    if isinstance(thing, (modules.PythonAttribute, modules.PythonModule)):
+        return thing.name
+
+    if isTestCase(thing):
+        # TestCase subclass
+        return reflect.qual(thing)
+
+    # Based on the type of thing, this is unreachable.  Maybe someone calls
+    # this from un-type-checked code though.  Also, even with the type
+    # information, mypy fails to determine this is unreachable and complains
+    # about a missing return without _something_ here.
+    raise TypeError(f"Cannot name {thing!r}")
+
+
+def isTestCase(obj: type) -> TypeGuard[Type[pyunit.TestCase]]:
+    """
+    @return: C{True} if C{obj} is a class that contains test cases, C{False}
+        otherwise. Used to find all the tests in a module.
+    """
+    try:
+        return issubclass(obj, pyunit.TestCase)
+    except TypeError:
+        return False
+
+
+@implementer(ITestCase)
+class TestHolder:
+    """
+    Placeholder for a L{TestCase} inside a reporter. As far as a L{TestResult}
+    is concerned, this looks exactly like a unit test.
+    """
+
+    failureException = None
+
+    def __init__(self, description):
+        """
+        @param description: A string to be displayed L{TestResult}.
+        """
+        self.description = description
+
+    def __call__(self, result):
+        return self.run(result)
+
+    def id(self):
+        return self.description
+
+    def countTestCases(self):
+        return 0
+
+    def run(self, result):
+        """
+        This test is just a placeholder. Run the test successfully.
+
+        @param result: The C{TestResult} to store the results in.
+        @type result: L{twisted.trial.itrial.IReporter}.
+        """
+        result.startTest(self)
+        result.addSuccess(self)
+        result.stopTest(self)
+
+    def shortDescription(self):
+        return self.description
+
+
+class ErrorHolder(TestHolder):
+    """
+    Used to insert arbitrary errors into a test suite run. Provides enough
+    methods to look like a C{TestCase}, however, when it is run, it simply adds
+    an error to the C{TestResult}. The most common use-case is for when a
+    module fails to import.
+    """
+
+    def __init__(self, description, error):
+        """
+        @param description: A string used by C{TestResult}s to identify this
+        error. Generally, this is the name of a module that failed to import.
+
+        @param error: The error to be added to the result. Can be an `exc_info`
+        tuple or a L{twisted.python.failure.Failure}.
+        """
+        super().__init__(description)
+        self.error = util.excInfoOrFailureToExcInfo(error)
+
+    def __repr__(self) -> str:
+        return "<ErrorHolder description={!r} error={!r}>".format(
+            self.description,
+            self.error[1],
+        )
+
+    def run(self, result):
+        """
+        Run the test, reporting the error.
+
+        @param result: The C{TestResult} to store the results in.
+        @type result: L{twisted.trial.itrial.IReporter}.
+        """
+        result.startTest(self)
+        result.addError(self, self.error)
+        result.stopTest(self)
+
+
+@define
+class TestLoader:
+    """
+    I find tests inside function, modules, files -- whatever -- then return
+    them wrapped inside a Test (either a L{TestSuite} or a L{TestCase}).
+
+    @ivar methodPrefix: A string prefix. C{TestLoader} will assume that all the
+    methods in a class that begin with C{methodPrefix} are test cases.
+
+    @ivar modulePrefix: A string prefix. Every module in a package that begins
+    with C{modulePrefix} is considered a module full of tests.
+
+    @ivar forceGarbageCollection: A flag applied to each C{TestCase} loaded.
+    See L{unittest.TestCase} for more information.
+
+    @ivar sorter: A key function used to sort C{TestCase}s, test classes,
+    modules and packages.
+
+    @ivar suiteFactory: A callable which is passed a list of tests (which
+    themselves may be suites of tests). Must return a test suite.
+    """
+
+    methodPrefix = "test"
+    modulePrefix = "test_"
+
+    suiteFactory: Type[TestSuite] = TestSuite
+    sorter: Callable[[_Loadable], object] = name
+
+    def sort(self, xs):
+        """
+        Sort the given things using L{sorter}.
+
+        @param xs: A list of test cases, class or modules.
+        """
+        return sorted(xs, key=self.sorter)
+
+    def findTestClasses(self, module):
+        """Given a module, return all Trial test classes"""
+        classes = []
+        for name, val in inspect.getmembers(module):
+            if isTestCase(val):
+                classes.append(val)
+        return self.sort(classes)
+
+    def findByName(self, _name, recurse=False):
+        """
+        Find and load tests, given C{name}.
+
+        @param _name: The qualified name of the thing to load.
+        @param recurse: A boolean. If True, inspect modules within packages
+            within the given package (and so on), otherwise, only inspect
+            modules in the package itself.
+
+        @return: If C{name} is a filename, return the module. If C{name} is a
+        fully-qualified Python name, return the object it refers to.
+        """
+        if os.sep in _name:
+            # It's a file, try and get the module name for this file.
+            name = reflect.filenameToModuleName(_name)
+
+            try:
+                # Try and import it, if it's on the path.
+                # CAVEAT: If you have two twisteds, and you try and import the
+                # one NOT on your path, it'll load the one on your path. But
+                # that's silly, nobody should do that, and existing Trial does
+                # that anyway.
+                __import__(name)
+            except ImportError:
+                # If we can't import it, look for one NOT on the path.
+                return self.loadFile(_name, recurse=recurse)
+
+        else:
+            name = _name
+
+        obj = parent = remaining = None
+
+        for searchName, remainingName in _qualNameWalker(name):
+            # Walk down the qualified name, trying to import a module. For
+            # example, `twisted.test.test_paths.FilePathTests` would try
+            # the full qualified name, then just up to test_paths, and then
+            # just up to test, and so forth.
+            # This gets us the highest level thing which is a module.
+            try:
+                obj = reflect.namedModule(searchName)
+                # If we reach here, we have successfully found a module.
+                # obj will be the module, and remaining will be the remaining
+                # part of the qualified name.
+                remaining = remainingName
+                break
+
+            except ImportError:
+                # Check to see where the ImportError happened. If it happened
+                # in this file, ignore it.
+                tb = sys.exc_info()[2]
+
+                # Walk down to the deepest frame, where it actually happened.
+                while tb.tb_next is not None:
+                    tb = tb.tb_next
+
+                # Get the filename that the ImportError originated in.
+                filenameWhereHappened = tb.tb_frame.f_code.co_filename
+
+                # If it originated in the reflect file, then it's because it
+                # doesn't exist. If it originates elsewhere, it's because an
+                # ImportError happened in a module that does exist.
+                if filenameWhereHappened != reflect.__file__:
+                    raise
+
+                if remaining == "":
+                    raise reflect.ModuleNotFound(f"The module {name} does not exist.")
+
+        if obj is None:
+            # If it's none here, we didn't get to import anything.
+            # Try something drastic.
+            obj = reflect.namedAny(name)
+            remaining = name.split(".")[len(".".split(obj.__name__)) + 1 :]
+
+        try:
+            for part in remaining:
+                # Walk down the remaining modules. Hold on to the parent for
+                # methods, as on Python 3, you can no longer get the parent
+                # class from just holding onto the method.
+                parent, obj = obj, getattr(obj, part)
+        except AttributeError:
+            raise AttributeError(f"{name} does not exist.")
+
+        return self.loadAnything(
+            obj, parent=parent, qualName=remaining, recurse=recurse
+        )
+
+    def loadModule(self, module):
+        """
+        Return a test suite with all the tests from a module.
+
+        Included are TestCase subclasses and doctests listed in the module's
+        __doctests__ module. If that's not good for you, put a function named
+        either C{testSuite} or C{test_suite} in your module that returns a
+        TestSuite, and I'll use the results of that instead.
+
+        If C{testSuite} and C{test_suite} are both present, then I'll use
+        C{testSuite}.
+        """
+        ## XXX - should I add an optional parameter to disable the check for
+        ## a custom suite.
+        ## OR, should I add another method
+        if not isinstance(module, types.ModuleType):
+            raise TypeError(f"{module!r} is not a module")
+        if hasattr(module, "testSuite"):
+            return module.testSuite()
+        elif hasattr(module, "test_suite"):
+            return module.test_suite()
+        suite = self.suiteFactory()
+        for testClass in self.findTestClasses(module):
+            suite.addTest(self.loadClass(testClass))
+        if not hasattr(module, "__doctests__"):
+            return suite
+        docSuite = self.suiteFactory()
+        for docTest in module.__doctests__:
+            docSuite.addTest(self.loadDoctests(docTest))
+        return self.suiteFactory([suite, docSuite])
+
+    loadTestsFromModule = loadModule
+
+    def loadClass(self, klass):
+        """
+        Given a class which contains test cases, return a list of L{TestCase}s.
+
+        @param klass: The class to load tests from.
+        """
+        if not isinstance(klass, type):
+            raise TypeError(f"{klass!r} is not a class")
+        if not isTestCase(klass):
+            raise ValueError(f"{klass!r} is not a test case")
+        names = self.getTestCaseNames(klass)
+        tests = self.sort(
+            [self._makeCase(klass, self.methodPrefix + name) for name in names]
+        )
+        return self.suiteFactory(tests)
+
+    loadTestsFromTestCase = loadClass
+
+    def getTestCaseNames(self, klass):
+        """
+        Given a class that contains C{TestCase}s, return a list of names of
+        methods that probably contain tests.
+        """
+        return reflect.prefixedMethodNames(klass, self.methodPrefix)
+
+    def _makeCase(self, klass, methodName):
+        return klass(methodName)
+
+    def loadPackage(self, package, recurse=False):
+        """
+        Load tests from a module object representing a package, and return a
+        TestSuite containing those tests.
+
+        Tests are only loaded from modules whose name begins with 'test_'
+        (or whatever C{modulePrefix} is set to).
+
+        @param package: a types.ModuleType object (or reasonable facsimile
+        obtained by importing) which may contain tests.
+
+        @param recurse: A boolean.  If True, inspect modules within packages
+        within the given package (and so on), otherwise, only inspect modules
+        in the package itself.
+
+        @raise TypeError: If C{package} is not a package.
+
+        @return: a TestSuite created with my suiteFactory, containing all the
+        tests.
+        """
+        if not isPackage(package):
+            raise TypeError(f"{package!r} is not a package")
+        pkgobj = modules.getModule(package.__name__)
+        if recurse:
+            discovery = pkgobj.walkModules()
+        else:
+            discovery = pkgobj.iterModules()
+        discovered = []
+        for disco in discovery:
+            if disco.name.split(".")[-1].startswith(self.modulePrefix):
+                discovered.append(disco)
+        suite = self.suiteFactory()
+        for modinfo in self.sort(discovered):
+            try:
+                module = modinfo.load()
+            except BaseException:
+                thingToAdd = ErrorHolder(modinfo.name, failure.Failure())
+            else:
+                thingToAdd = self.loadModule(module)
+            suite.addTest(thingToAdd)
+        return suite
+
+    def loadDoctests(self, module):
+        """
+        Return a suite of tests for all the doctests defined in C{module}.
+
+        @param module: A module object or a module name.
+        """
+        if isinstance(module, str):
+            try:
+                module = reflect.namedAny(module)
+            except BaseException:
+                return ErrorHolder(module, failure.Failure())
+        if not inspect.ismodule(module):
+            warnings.warn("trial only supports doctesting modules")
+            return
+        extraArgs = {}
+
+        # Work around Python issue2604: DocTestCase.tearDown clobbers globs
+        def saveGlobals(test):
+            """
+            Save C{test.globs} and replace it with a copy so that if
+            necessary, the original will be available for the next test
+            run.
+            """
+            test._savedGlobals = getattr(test, "_savedGlobals", test.globs)
+            test.globs = test._savedGlobals.copy()
+
+        extraArgs["setUp"] = saveGlobals
+        return doctest.DocTestSuite(module, **extraArgs)
+
+    def loadAnything(self, obj, recurse=False, parent=None, qualName=None):
+        """
+        Load absolutely anything (as long as that anything is a module,
+        package, class, or method (with associated parent class and qualname).
+
+        @param obj: The object to load.
+        @param recurse: A boolean. If True, inspect modules within packages
+            within the given package (and so on), otherwise, only inspect
+            modules in the package itself.
+        @param parent: If C{obj} is a method, this is the parent class of the
+            method. C{qualName} is also required.
+        @param qualName: If C{obj} is a method, this a list containing is the
+            qualified name of the method. C{parent} is also required.
+
+        @return: A C{TestCase} or C{TestSuite}.
+        """
+        if isinstance(obj, types.ModuleType):
+            # It looks like a module
+            if isPackage(obj):
+                # It's a package, so recurse down it.
+                return self.loadPackage(obj, recurse=recurse)
+            # Otherwise get all the tests in the module.
+            return self.loadTestsFromModule(obj)
+        elif isinstance(obj, type) and issubclass(obj, pyunit.TestCase):
+            # We've found a raw test case, get the tests from it.
+            return self.loadTestsFromTestCase(obj)
+        elif (
+            isinstance(obj, types.FunctionType)
+            and isinstance(parent, type)
+            and issubclass(parent, pyunit.TestCase)
+        ):
+            # We've found a method, and its parent is a TestCase. Instantiate
+            # it with the name of the method we want.
+            name = qualName[-1]
+            inst = parent(name)
+
+            # Sanity check to make sure that the method we have got from the
+            # test case is the same one as was passed in. This doesn't actually
+            # use the function we passed in, because reasons.
+            assert getattr(inst, inst._testMethodName).__func__ == obj
+
+            return inst
+        elif isinstance(obj, TestSuite):
+            # We've found a test suite.
+            return obj
+        else:
+            raise TypeError(f"don't know how to make test from: {obj}")
+
+    def loadByName(self, name, recurse=False):
+        """
+        Load some tests by name.
+
+        @param name: The qualified name for the test to load.
+        @param recurse: A boolean. If True, inspect modules within packages
+            within the given package (and so on), otherwise, only inspect
+            modules in the package itself.
+        """
+        try:
+            return self.suiteFactory([self.findByName(name, recurse=recurse)])
+        except BaseException:
+            return self.suiteFactory([ErrorHolder(name, failure.Failure())])
+
+    loadTestsFromName = loadByName
+
+    def loadByNames(self, names: List[str], recurse: bool = False) -> TestSuite:
+        """
+        Load some tests by a list of names.
+
+        @param names: A L{list} of qualified names.
+        @param recurse: A boolean. If True, inspect modules within packages
+            within the given package (and so on), otherwise, only inspect
+            modules in the package itself.
+        """
+        things = []
+        errors = []
+        for name in names:
+            try:
+                things.append(self.loadByName(name, recurse=recurse))
+            except BaseException:
+                errors.append(ErrorHolder(name, failure.Failure()))
+        things.extend(errors)
+        return self.suiteFactory(self._uniqueTests(things))
+
+    def _uniqueTests(self, things):
+        """
+        Gather unique suite objects from loaded things. This will guarantee
+        uniqueness of inherited methods on TestCases which would otherwise hash
+        to same value and collapse to one test unexpectedly if using simpler
+        means: e.g. set().
+        """
+        seen = set()
+        for testthing in things:
+            testthings = testthing._tests
+            for thing in testthings:
+                # This is horrible.
+                if str(thing) not in seen:
+                    yield thing
+                    seen.add(str(thing))
+
+    def loadFile(self, fileName, recurse=False):
+        """
+        Load a file, and then the tests in that file.
+
+        @param fileName: The file name to load.
+        @param recurse: A boolean. If True, inspect modules within packages
+            within the given package (and so on), otherwise, only inspect
+            modules in the package itself.
+        """
+        name = reflect.filenameToModuleName(fileName)
+        try:
+            module = SourceFileLoader(name, fileName).load_module()
+            return self.loadAnything(module, recurse=recurse)
+        except OSError:
+            raise ValueError(f"{fileName} is not a Python file.")
+
+
+def _qualNameWalker(qualName):
+    """
+    Given a Python qualified name, this function yields a 2-tuple of the most
+    specific qualified name first, followed by the next-most-specific qualified
+    name, and so on, paired with the remainder of the qualified name.
+
+    @param qualName: A Python qualified name.
+    @type qualName: L{str}
+    """
+    # Yield what we were just given
+    yield (qualName, [])
+
+    # If they want more, split the qualified name up
+    qualParts = qualName.split(".")
+
+    for index in range(1, len(qualParts)):
+        # This code here will produce, from the example walker.texas.ranger:
+        # (walker.texas, ["ranger"])
+        # (walker, ["texas", "ranger"])
+        yield (".".join(qualParts[:-index]), qualParts[-index:])
+
+
+@contextmanager
+def _testDirectory(workingDirectory: str) -> Generator[None, None, None]:
+    """
+    A context manager which obtains a lock on a trial working directory
+    and enters (L{os.chdir}) it and then reverses these things.
+
+    @param workingDirectory: A pattern for the basename of the working
+        directory to acquire.
+    """
+    currentDir = os.getcwd()
+    base = filepath.FilePath(workingDirectory)
+    testdir, testDirLock = util._unusedTestDirectory(base)
+    os.chdir(testdir.path)
+
+    yield
+
+    os.chdir(currentDir)
+    testDirLock.unlock()
+
+
+@contextmanager
+def _logFile(logfile: str) -> Generator[None, None, None]:
+    """
+    A context manager which adds a log observer and then removes it.
+
+    @param logfile: C{"-"} f or stdout logging, otherwise the path to a log
+        file to which to write.
+    """
+    if logfile == "-":
+        logFile = sys.stdout
+    else:
+        logFile = util.openTestLog(filepath.FilePath(logfile))
+
+    logFileObserver = log.FileLogObserver(logFile)
+    observerFunction = logFileObserver.emit
+    log.startLoggingWithObserver(observerFunction, 0)
+
+    yield
+
+    log.removeObserver(observerFunction)
+    logFile.close()
+
+
+class _Runner(Protocol):
+    stream: TextIO
+
+    def run(self, test: Union[pyunit.TestCase, pyunit.TestSuite]) -> itrial.IReporter:
+        ...
+
+    def runUntilFailure(
+        self, test: Union[pyunit.TestCase, pyunit.TestSuite]
+    ) -> itrial.IReporter:
+        ...
+
+
+@define
+class TrialRunner:
+    """
+    A specialised runner that the trial front end uses.
+
+    @ivar reporterFactory: A callable to create a reporter to use.
+
+    @ivar mode: Either C{None} for a normal test run, L{TrialRunner.DEBUG} for
+        a run in the debugger, or L{TrialRunner.DRY_RUN} to collect and report
+        the tests but not call any of them.
+
+    @ivar logfile: The path to the file to write the test run log.
+
+    @ivar stream: The file to report results to.
+
+    @ivar profile: C{True} to run the tests with a profiler enabled.
+
+    @ivar _tracebackFormat: A format name to use with L{Failure} for reporting
+        failures.
+
+    @ivar _realTimeErrors: C{True} if errors should be reported as they
+        happen.  C{False} if they should only be reported at the end of the
+        test run in the summary.
+
+    @ivar uncleanWarnings: C{True} to report dirty reactor errors as warnings,
+        C{False} to report them as test-failing errors.
+
+    @ivar workingDirectory: A path template to a directory which will be the
+        process's working directory while the tests are running.
+
+    @ivar _forceGarbageCollection: C{True} to perform a full garbage
+        collection at least after each test.  C{False} to let garbage
+        collection run only when it normally would.
+
+    @ivar debugger: In debug mode, an object to use to launch the debugger.
+
+    @ivar _exitFirst: C{True} to stop after the first failed test.  C{False}
+        to run the whole suite.
+
+    @ivar log: An object to give to the reporter to use as a log publisher.
+    """
+
+    DEBUG = "debug"
+    DRY_RUN = "dry-run"
+
+    reporterFactory: Callable[[TextIO, str, bool, log.LogPublisher], itrial.IReporter]
+    mode: Optional[str] = None
+    logfile: str = "test.log"
+    stream: TextIO = sys.stdout
+    profile: bool = False
+    _tracebackFormat: str = "default"
+    _realTimeErrors: bool = False
+    uncleanWarnings: bool = False
+    workingDirectory: str = "_trial_temp"
+    _forceGarbageCollection: bool = False
+    debugger: Optional[_Debugger] = None
+    _exitFirst: bool = False
+
+    _log: log.LogPublisher = log  # type: ignore[assignment]
+
+    def _makeResult(self) -> itrial.IReporter:
+        reporter = self.reporterFactory(
+            self.stream, self.tbformat, self.rterrors, self._log
+        )
+        if self._exitFirst:
+            reporter = _ExitWrapper(reporter)
+        if self.uncleanWarnings:
+            reporter = UncleanWarningsReporterWrapper(reporter)
+        return reporter
+
+    @property
+    def tbformat(self) -> str:
+        return self._tracebackFormat
+
+    @property
+    def rterrors(self) -> bool:
+        return self._realTimeErrors
+
+    def run(self, test: Union[pyunit.TestCase, pyunit.TestSuite]) -> itrial.IReporter:
+        """
+        Run the test or suite and return a result object.
+        """
+        test = unittest.decorate(test, ITestCase)
+        if self.profile:
+            run = util.profiled(self._runWithoutDecoration, "profile.data")
+        else:
+            run = self._runWithoutDecoration
+        return run(test, self._forceGarbageCollection)
+
+    def _runWithoutDecoration(
+        self,
+        test: Union[pyunit.TestCase, pyunit.TestSuite],
+        forceGarbageCollection: bool = False,
+    ) -> itrial.IReporter:
+        """
+        Private helper that runs the given test but doesn't decorate it.
+        """
+        result = self._makeResult()
+        # decorate the suite with reactor cleanup and log starting
+        # This should move out of the runner and be presumed to be
+        # present
+        suite = TrialSuite([test], forceGarbageCollection)
+        if self.mode == self.DRY_RUN:
+            for single in _iterateTests(suite):
+                result.startTest(single)
+                result.addSuccess(single)
+                result.stopTest(single)
+        else:
+            if self.mode == self.DEBUG:
+                assert self.debugger is not None
+                run = lambda: self.debugger.runcall(suite.run, result)
+            else:
+                run = lambda: suite.run(result)
+
+            with _testDirectory(self.workingDirectory), _logFile(self.logfile):
+                run()
+
+        result.done()
+        return result
+
+    def runUntilFailure(
+        self, test: Union[pyunit.TestCase, pyunit.TestSuite]
+    ) -> itrial.IReporter:
+        """
+        Repeatedly run C{test} until it fails.
+        """
+        count = 0
+        while True:
+            count += 1
+            self.stream.write("Test Pass %d\n" % (count,))
+            if count == 1:
+                # If test is a TestSuite, run *mutates it*.  So only follow
+                # this code-path once!  Otherwise the decorations accumulate
+                # forever.
+                result = self.run(test)
+            else:
+                result = self._runWithoutDecoration(test)
+            if result.testsRun == 0:
+                break
+            if not result.wasSuccessful():
+                break
+        return result
diff --git a/contrib/python/Twisted/py3/twisted/trial/unittest.py b/contrib/python/Twisted/py3/twisted/trial/unittest.py
new file mode 100644
index 00000000000..6c1dfca9e52
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/unittest.py
@@ -0,0 +1,39 @@
+# -*- test-case-name: twisted.trial.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Things likely to be used by writers of unit tests.
+"""
+
+
+from twisted.trial._asyncrunner import TestDecorator, TestSuite, decorate
+from twisted.trial._asynctest import TestCase
+
+# Define the public API from the two implementation modules
+from twisted.trial._synctest import (
+    FailTest,
+    PyUnitResultAdapter,
+    SkipTest,
+    SynchronousTestCase,
+    Todo,
+    makeTodo,
+)
+
+# Further obscure the origins of these objects, to reduce surprise (and this is
+# what the values were before code got shuffled around between files, but was
+# otherwise unchanged).
+FailTest.__module__ = SkipTest.__module__ = __name__
+
+__all__ = [
+    "decorate",
+    "FailTest",
+    "makeTodo",
+    "PyUnitResultAdapter",
+    "SkipTest",
+    "SynchronousTestCase",
+    "TestCase",
+    "TestDecorator",
+    "TestSuite",
+    "Todo",
+]
diff --git a/contrib/python/Twisted/py3/twisted/trial/util.py b/contrib/python/Twisted/py3/twisted/trial/util.py
new file mode 100644
index 00000000000..a8345ccc6c8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/trial/util.py
@@ -0,0 +1,407 @@
+# -*- test-case-name: twisted.trial.test.test_util -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+#
+
+"""
+A collection of utility functions and classes, used internally by Trial.
+
+This code is for Trial's internal use.  Do NOT use this code if you are writing
+tests.  It is subject to change at the Trial maintainer's whim.  There is
+nothing here in this module for you to use unless you are maintaining Trial.
+
+Any non-Trial Twisted code that uses this module will be shot.
+
+Maintainer: Jonathan Lange
+
+@var DEFAULT_TIMEOUT_DURATION: The default timeout which will be applied to
+    asynchronous (ie, Deferred-returning) test methods, in seconds.
+"""
+from __future__ import annotations
+
+from random import randrange
+from typing import Any, Callable, TextIO, TypeVar
+
+from typing_extensions import ParamSpec
+
+from twisted.internet import interfaces, utils
+from twisted.python.failure import Failure
+from twisted.python.filepath import FilePath
+from twisted.python.lockfile import FilesystemLock
+
+__all__ = [
+    "DEFAULT_TIMEOUT_DURATION",
+    "excInfoOrFailureToExcInfo",
+    "suppress",
+    "acquireAttribute",
+]
+
+DEFAULT_TIMEOUT = object()
+DEFAULT_TIMEOUT_DURATION = 120.0
+
+
+class DirtyReactorAggregateError(Exception):
+    """
+    Passed to L{twisted.trial.itrial.IReporter.addError} when the reactor is
+    left in an unclean state after a test.
+
+    @ivar delayedCalls: The L{DelayedCall<twisted.internet.base.DelayedCall>}
+        objects which weren't cleaned up.
+    @ivar selectables: The selectables which weren't cleaned up.
+    """
+
+    def __init__(self, delayedCalls, selectables=None):
+        self.delayedCalls = delayedCalls
+        self.selectables = selectables
+
+    def __str__(self) -> str:
+        """
+        Return a multi-line message describing all of the unclean state.
+        """
+        msg = "Reactor was unclean."
+        if self.delayedCalls:
+            msg += (
+                "\nDelayedCalls: (set "
+                "twisted.internet.base.DelayedCall.debug = True to "
+                "debug)\n"
+            )
+            msg += "\n".join(map(str, self.delayedCalls))
+        if self.selectables:
+            msg += "\nSelectables:\n"
+            msg += "\n".join(map(str, self.selectables))
+        return msg
+
+
+class _Janitor:
+    """
+    The guy that cleans up after you.
+
+    @ivar test: The L{TestCase} to report errors about.
+    @ivar result: The L{IReporter} to report errors to.
+    @ivar reactor: The reactor to use. If None, the global reactor
+        will be used.
+    """
+
+    def __init__(self, test, result, reactor=None):
+        """
+        @param test: See L{_Janitor.test}.
+        @param result: See L{_Janitor.result}.
+        @param reactor: See L{_Janitor.reactor}.
+        """
+        self.test = test
+        self.result = result
+        self.reactor = reactor
+
+    def postCaseCleanup(self):
+        """
+        Called by L{unittest.TestCase} after a test to catch any logged errors
+        or pending L{DelayedCall<twisted.internet.base.DelayedCall>}s.
+        """
+        calls = self._cleanPending()
+        if calls:
+            aggregate = DirtyReactorAggregateError(calls)
+            self.result.addError(self.test, Failure(aggregate))
+            return False
+        return True
+
+    def postClassCleanup(self):
+        """
+        Called by L{unittest.TestCase} after the last test in a C{TestCase}
+        subclass. Ensures the reactor is clean by murdering the threadpool,
+        catching any pending
+        L{DelayedCall<twisted.internet.base.DelayedCall>}s, open sockets etc.
+        """
+        selectables = self._cleanReactor()
+        calls = self._cleanPending()
+        if selectables or calls:
+            aggregate = DirtyReactorAggregateError(calls, selectables)
+            self.result.addError(self.test, Failure(aggregate))
+        self._cleanThreads()
+
+    def _getReactor(self):
+        """
+        Get either the passed-in reactor or the global reactor.
+        """
+        if self.reactor is not None:
+            reactor = self.reactor
+        else:
+            from twisted.internet import reactor
+        return reactor
+
+    def _cleanPending(self):
+        """
+        Cancel all pending calls and return their string representations.
+        """
+        reactor = self._getReactor()
+
+        # flush short-range timers
+        reactor.iterate(0)
+        reactor.iterate(0)
+
+        delayedCallStrings = []
+        for p in reactor.getDelayedCalls():
+            if p.active():
+                delayedString = str(p)
+                p.cancel()
+            else:
+                print("WEIRDNESS! pending timed call not active!")
+            delayedCallStrings.append(delayedString)
+        return delayedCallStrings
+
+    _cleanPending = utils.suppressWarnings(
+        _cleanPending,
+        (
+            ("ignore",),
+            {
+                "category": DeprecationWarning,
+                "message": r"reactor\.iterate cannot be used.*",
+            },
+        ),
+    )
+
+    def _cleanThreads(self):
+        reactor = self._getReactor()
+        if interfaces.IReactorThreads.providedBy(reactor):
+            if reactor.threadpool is not None:
+                # Stop the threadpool now so that a new one is created.
+                # This improves test isolation somewhat (although this is a
+                # post class cleanup hook, so it's only isolating classes
+                # from each other, not methods from each other).
+                reactor._stopThreadPool()
+
+    def _cleanReactor(self):
+        """
+        Remove all selectables from the reactor, kill any of them that were
+        processes, and return their string representation.
+        """
+        reactor = self._getReactor()
+        selectableStrings = []
+        for sel in reactor.removeAll():
+            if interfaces.IProcessTransport.providedBy(sel):
+                sel.signalProcess("KILL")
+            selectableStrings.append(repr(sel))
+        return selectableStrings
+
+
+_DEFAULT = object()
+
+
+def acquireAttribute(objects, attr, default=_DEFAULT):
+    """
+    Go through the list 'objects' sequentially until we find one which has
+    attribute 'attr', then return the value of that attribute.  If not found,
+    return 'default' if set, otherwise, raise AttributeError.
+    """
+    for obj in objects:
+        if hasattr(obj, attr):
+            return getattr(obj, attr)
+    if default is not _DEFAULT:
+        return default
+    raise AttributeError(f"attribute {attr!r} not found in {objects!r}")
+
+
+def excInfoOrFailureToExcInfo(err):
+    """
+    Coerce a Failure to an _exc_info, if err is a Failure.
+
+    @param err: Either a tuple such as returned by L{sys.exc_info} or a
+        L{Failure} object.
+    @return: A tuple like the one returned by L{sys.exc_info}. e.g.
+        C{exception_type, exception_object, traceback_object}.
+    """
+    if isinstance(err, Failure):
+        # Unwrap the Failure into an exc_info tuple.
+        err = (err.type, err.value, err.getTracebackObject())
+    return err
+
+
+def suppress(action="ignore", **kwarg):
+    """
+    Sets up the .suppress tuple properly, pass options to this method as you
+    would the stdlib warnings.filterwarnings()
+
+    So, to use this with a .suppress magic attribute you would do the
+    following:
+
+      >>> from twisted.trial import unittest, util
+      >>> import warnings
+      >>>
+      >>> class TestFoo(unittest.TestCase):
+      ...     def testFooBar(self):
+      ...         warnings.warn("i am deprecated", DeprecationWarning)
+      ...     testFooBar.suppress = [util.suppress(message='i am deprecated')]
+      ...
+      >>>
+
+    Note that as with the todo and timeout attributes: the module level
+    attribute acts as a default for the class attribute which acts as a default
+    for the method attribute. The suppress attribute can be overridden at any
+    level by specifying C{.suppress = []}
+    """
+    return ((action,), kwarg)
+
+
+# This should be deleted, and replaced with twisted.application's code; see
+# https://github.com/twisted/twisted/issues/6016:
+_P = ParamSpec("_P")
+_T = TypeVar("_T")
+
+
+def profiled(f: Callable[_P, _T], outputFile: str) -> Callable[_P, _T]:
+    def _(*args: _P.args, **kwargs: _P.kwargs) -> _T:
+        import profile
+
+        prof = profile.Profile()
+        try:
+            result = prof.runcall(f, *args, **kwargs)
+            prof.dump_stats(outputFile)
+        except SystemExit:
+            pass
+        prof.print_stats()
+        return result
+
+    return _
+
+
+class _NoTrialMarker(Exception):
+    """
+    No trial marker file could be found.
+
+    Raised when trial attempts to remove a trial temporary working directory
+    that does not contain a marker file.
+    """
+
+
+def _removeSafely(path):
+    """
+    Safely remove a path, recursively.
+
+    If C{path} does not contain a node named C{_trial_marker}, a
+    L{_NoTrialMarker} exception is raised and the path is not removed.
+    """
+    if not path.child(b"_trial_marker").exists():
+        raise _NoTrialMarker(
+            f"{path!r} is not a trial temporary path, refusing to remove it"
+        )
+    try:
+        path.remove()
+    except OSError as e:
+        print(
+            "could not remove %r, caught OSError [Errno %s]: %s"
+            % (path, e.errno, e.strerror)
+        )
+        try:
+            newPath = FilePath(
+                b"_trial_temp_old" + str(randrange(10000000)).encode("utf-8")
+            )
+            path.moveTo(newPath)
+        except OSError as e:
+            print(
+                "could not rename path, caught OSError [Errno %s]: %s"
+                % (e.errno, e.strerror)
+            )
+            raise
+
+
+class _WorkingDirectoryBusy(Exception):
+    """
+    A working directory was specified to the runner, but another test run is
+    currently using that directory.
+    """
+
+
+def _unusedTestDirectory(base):
+    """
+    Find an unused directory named similarly to C{base}.
+
+    Once a directory is found, it will be locked and a marker dropped into it
+    to identify it as a trial temporary directory.
+
+    @param base: A template path for the discovery process.  If this path
+        exactly cannot be used, a path which varies only in a suffix of the
+        basename will be used instead.
+    @type base: L{FilePath}
+
+    @return: A two-tuple.  The first element is a L{FilePath} representing the
+        directory which was found and created.  The second element is a locked
+        L{FilesystemLock<twisted.python.lockfile.FilesystemLock>}.  Another
+        call to C{_unusedTestDirectory} will not be able to reused the
+        same name until the lock is released, either explicitly or by this
+        process exiting.
+    """
+    counter = 0
+    while True:
+        if counter:
+            testdir = base.sibling("%s-%d" % (base.basename(), counter))
+        else:
+            testdir = base
+
+        testdir.parent().makedirs(ignoreExistingDirectory=True)
+        testDirLock = FilesystemLock(testdir.path + ".lock")
+        if testDirLock.lock():
+            # It is not in use
+            if testdir.exists():
+                # It exists though - delete it
+                _removeSafely(testdir)
+
+            # Create it anew and mark it as ours so the next _removeSafely on
+            # it succeeds.
+            testdir.makedirs()
+            testdir.child(b"_trial_marker").setContent(b"")
+            return testdir, testDirLock
+        else:
+            # It is in use
+            if base.basename() == "_trial_temp":
+                counter += 1
+            else:
+                raise _WorkingDirectoryBusy()
+
+
+def _listToPhrase(things, finalDelimiter, delimiter=", "):
+    """
+    Produce a string containing each thing in C{things},
+    separated by a C{delimiter}, with the last couple being separated
+    by C{finalDelimiter}
+
+    @param things: The elements of the resulting phrase
+    @type things: L{list} or L{tuple}
+
+    @param finalDelimiter: What to put between the last two things
+        (typically 'and' or 'or')
+    @type finalDelimiter: L{str}
+
+    @param delimiter: The separator to use between each thing,
+        not including the last two. Should typically include a trailing space.
+    @type delimiter: L{str}
+
+    @return: The resulting phrase
+    @rtype: L{str}
+    """
+    if not isinstance(things, (list, tuple)):
+        raise TypeError("Things must be a list or a tuple")
+    if not things:
+        return ""
+    if len(things) == 1:
+        return str(things[0])
+    if len(things) == 2:
+        return f"{str(things[0])} {finalDelimiter} {str(things[1])}"
+    else:
+        strThings = []
+        for thing in things:
+            strThings.append(str(thing))
+        return "{}{}{} {}".format(
+            delimiter.join(strThings[:-1]),
+            delimiter,
+            finalDelimiter,
+            strThings[-1],
+        )
+
+
+def openTestLog(path: FilePath[Any]) -> TextIO:
+    """
+    Open the given path such that test log messages can be written to it.
+    """
+    path.parent().makedirs(ignoreExistingDirectory=True)
+    # Always use UTF-8 because, considering all platforms, the system default
+    # encoding can not reliably encode all code points.
+    return open(path.path, "a", encoding="utf-8", errors="strict")
diff --git a/contrib/python/Twisted/py3/twisted/web/__init__.py b/contrib/python/Twisted/py3/twisted/web/__init__.py
new file mode 100644
index 00000000000..806dc4a2a42
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/__init__.py
@@ -0,0 +1,12 @@
+# -*- test-case-name: twisted.web.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Web: HTTP clients and servers, plus tools for implementing them.
+
+Contains a L{web server<twisted.web.server>} (including an
+L{HTTP implementation<twisted.web.http>}, a
+L{resource model<twisted.web.resource>}), and
+a L{web client<twisted.web.client>}.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/web/_auth/__init__.py b/contrib/python/Twisted/py3/twisted/web/_auth/__init__.py
new file mode 100644
index 00000000000..6a588700916
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_auth/__init__.py
@@ -0,0 +1,7 @@
+# -*- test-case-name: twisted.web.test.test_httpauth -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HTTP header-based authentication migrated from web2
+"""
diff --git a/contrib/python/Twisted/py3/twisted/web/_auth/basic.py b/contrib/python/Twisted/py3/twisted/web/_auth/basic.py
new file mode 100644
index 00000000000..9eed46928fc
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_auth/basic.py
@@ -0,0 +1,58 @@
+# -*- test-case-name: twisted.web.test.test_httpauth -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HTTP BASIC authentication.
+
+@see: U{http://tools.ietf.org/html/rfc1945}
+@see: U{http://tools.ietf.org/html/rfc2616}
+@see: U{http://tools.ietf.org/html/rfc2617}
+"""
+
+
+import binascii
+
+from zope.interface import implementer
+
+from twisted.cred import credentials, error
+from twisted.web.iweb import ICredentialFactory
+
+
+@implementer(ICredentialFactory)
+class BasicCredentialFactory:
+    """
+    Credential Factory for HTTP Basic Authentication
+
+    @type authenticationRealm: L{bytes}
+    @ivar authenticationRealm: The HTTP authentication realm which will be issued in
+        challenges.
+    """
+
+    scheme = b"basic"
+
+    def __init__(self, authenticationRealm):
+        self.authenticationRealm = authenticationRealm
+
+    def getChallenge(self, request):
+        """
+        Return a challenge including the HTTP authentication realm with which
+        this factory was created.
+        """
+        return {"realm": self.authenticationRealm}
+
+    def decode(self, response, request):
+        """
+        Parse the base64-encoded, colon-separated username and password into a
+        L{credentials.UsernamePassword} instance.
+        """
+        try:
+            creds = binascii.a2b_base64(response + b"===")
+        except binascii.Error:
+            raise error.LoginFailed("Invalid credentials")
+
+        creds = creds.split(b":", 1)
+        if len(creds) == 2:
+            return credentials.UsernamePassword(*creds)
+        else:
+            raise error.LoginFailed("Invalid credentials")
diff --git a/contrib/python/Twisted/py3/twisted/web/_auth/digest.py b/contrib/python/Twisted/py3/twisted/web/_auth/digest.py
new file mode 100644
index 00000000000..e77f337905f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_auth/digest.py
@@ -0,0 +1,56 @@
+# -*- test-case-name: twisted.web.test.test_httpauth -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of RFC2617: HTTP Digest Authentication
+
+@see: U{http://www.faqs.org/rfcs/rfc2617.html}
+"""
+
+
+from zope.interface import implementer
+
+from twisted.cred import credentials
+from twisted.web.iweb import ICredentialFactory
+
+
+@implementer(ICredentialFactory)
+class DigestCredentialFactory:
+    """
+    Wrapper for L{digest.DigestCredentialFactory} that implements the
+    L{ICredentialFactory} interface.
+    """
+
+    scheme = b"digest"
+
+    def __init__(self, algorithm, authenticationRealm):
+        """
+        Create the digest credential factory that this object wraps.
+        """
+        self.digest = credentials.DigestCredentialFactory(
+            algorithm, authenticationRealm
+        )
+
+    def getChallenge(self, request):
+        """
+        Generate the challenge for use in the WWW-Authenticate header
+
+        @param request: The L{IRequest} to with access was denied and for the
+            response to which this challenge is being generated.
+
+        @return: The L{dict} that can be used to generate a WWW-Authenticate
+            header.
+        """
+        return self.digest.getChallenge(request.getClientAddress().host)
+
+    def decode(self, response, request):
+        """
+        Create a L{twisted.cred.credentials.DigestedCredentials} object
+        from the given response and request.
+
+        @see: L{ICredentialFactory.decode}
+        """
+        return self.digest.decode(
+            response, request.method, request.getClientAddress().host
+        )
diff --git a/contrib/python/Twisted/py3/twisted/web/_auth/wrapper.py b/contrib/python/Twisted/py3/twisted/web/_auth/wrapper.py
new file mode 100644
index 00000000000..cffdcff66c9
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_auth/wrapper.py
@@ -0,0 +1,236 @@
+# -*- test-case-name: twisted.web.test.test_httpauth -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A guard implementation which supports HTTP header-based authentication
+schemes.
+
+If no I{Authorization} header is supplied, an anonymous login will be
+attempted by using a L{Anonymous} credentials object.  If such a header is
+supplied and does not contain allowed credentials, or if anonymous login is
+denied, a 401 will be sent in the response along with I{WWW-Authenticate}
+headers for each of the allowed authentication schemes.
+"""
+
+
+from zope.interface import implementer
+
+from twisted.cred import error
+from twisted.cred.credentials import Anonymous
+from twisted.logger import Logger
+from twisted.python.components import proxyForInterface
+from twisted.web import util
+from twisted.web.resource import IResource, _UnsafeErrorPage
+
+
+@implementer(IResource)
+class UnauthorizedResource:
+    """
+    Simple IResource to escape Resource dispatch
+    """
+
+    isLeaf = True
+
+    def __init__(self, factories):
+        self._credentialFactories = factories
+
+    def render(self, request):
+        """
+        Send www-authenticate headers to the client
+        """
+
+        def ensureBytes(s):
+            return s.encode("ascii") if isinstance(s, str) else s
+
+        def generateWWWAuthenticate(scheme, challenge):
+            lst = []
+            for k, v in challenge.items():
+                k = ensureBytes(k)
+                v = ensureBytes(v)
+                lst.append(k + b"=" + quoteString(v))
+            return b" ".join([scheme, b", ".join(lst)])
+
+        def quoteString(s):
+            return b'"' + s.replace(b"\\", rb"\\").replace(b'"', rb"\"") + b'"'
+
+        request.setResponseCode(401)
+        for fact in self._credentialFactories:
+            challenge = fact.getChallenge(request)
+            request.responseHeaders.addRawHeader(
+                b"www-authenticate", generateWWWAuthenticate(fact.scheme, challenge)
+            )
+        if request.method == b"HEAD":
+            return b""
+        return b"Unauthorized"
+
+    def getChildWithDefault(self, path, request):
+        """
+        Disable resource dispatch
+        """
+        return self
+
+    def putChild(self, path, child):
+        # IResource.putChild
+        raise NotImplementedError()
+
+
+@implementer(IResource)
+class HTTPAuthSessionWrapper:
+    """
+    Wrap a portal, enforcing supported header-based authentication schemes.
+
+    @ivar _portal: The L{Portal} which will be used to retrieve L{IResource}
+        avatars.
+
+    @ivar _credentialFactories: A list of L{ICredentialFactory} providers which
+        will be used to decode I{Authorization} headers into L{ICredentials}
+        providers.
+    """
+
+    isLeaf = False
+    _log = Logger()
+
+    def __init__(self, portal, credentialFactories):
+        """
+        Initialize a session wrapper
+
+        @type portal: C{Portal}
+        @param portal: The portal that will authenticate the remote client
+
+        @type credentialFactories: C{Iterable}
+        @param credentialFactories: The portal that will authenticate the
+            remote client based on one submitted C{ICredentialFactory}
+        """
+        self._portal = portal
+        self._credentialFactories = credentialFactories
+
+    def _authorizedResource(self, request):
+        """
+        Get the L{IResource} which the given request is authorized to receive.
+        If the proper authorization headers are present, the resource will be
+        requested from the portal.  If not, an anonymous login attempt will be
+        made.
+        """
+        authheader = request.getHeader(b"authorization")
+        if not authheader:
+            return util.DeferredResource(self._login(Anonymous()))
+
+        factory, respString = self._selectParseHeader(authheader)
+        if factory is None:
+            return UnauthorizedResource(self._credentialFactories)
+        try:
+            credentials = factory.decode(respString, request)
+        except error.LoginFailed:
+            return UnauthorizedResource(self._credentialFactories)
+        except BaseException:
+            self._log.failure("Unexpected failure from credentials factory")
+            return _UnsafeErrorPage(500, "Internal Error", "")
+        else:
+            return util.DeferredResource(self._login(credentials))
+
+    def render(self, request):
+        """
+        Find the L{IResource} avatar suitable for the given request, if
+        possible, and render it.  Otherwise, perhaps render an error page
+        requiring authorization or describing an internal server failure.
+        """
+        return self._authorizedResource(request).render(request)
+
+    def getChildWithDefault(self, path, request):
+        """
+        Inspect the Authorization HTTP header, and return a deferred which,
+        when fired after successful authentication, will return an authorized
+        C{Avatar}. On authentication failure, an C{UnauthorizedResource} will
+        be returned, essentially halting further dispatch on the wrapped
+        resource and all children
+        """
+        # Don't consume any segments of the request - this class should be
+        # transparent!
+        request.postpath.insert(0, request.prepath.pop())
+        return self._authorizedResource(request)
+
+    def _login(self, credentials):
+        """
+        Get the L{IResource} avatar for the given credentials.
+
+        @return: A L{Deferred} which will be called back with an L{IResource}
+            avatar or which will errback if authentication fails.
+        """
+        d = self._portal.login(credentials, None, IResource)
+        d.addCallbacks(self._loginSucceeded, self._loginFailed)
+        return d
+
+    def _loginSucceeded(self, args):
+        """
+        Handle login success by wrapping the resulting L{IResource} avatar
+        so that the C{logout} callback will be invoked when rendering is
+        complete.
+        """
+        interface, avatar, logout = args
+
+        class ResourceWrapper(proxyForInterface(IResource, "resource")):
+            """
+            Wrap an L{IResource} so that whenever it or a child of it
+            completes rendering, the cred logout hook will be invoked.
+
+            An assumption is made here that exactly one L{IResource} from
+            among C{avatar} and all of its children will be rendered.  If
+            more than one is rendered, C{logout} will be invoked multiple
+            times and probably earlier than desired.
+            """
+
+            def getChildWithDefault(self, name, request):
+                """
+                Pass through the lookup to the wrapped resource, wrapping
+                the result in L{ResourceWrapper} to ensure C{logout} is
+                called when rendering of the child is complete.
+                """
+                return ResourceWrapper(self.resource.getChildWithDefault(name, request))
+
+            def render(self, request):
+                """
+                Hook into response generation so that when rendering has
+                finished completely (with or without error), C{logout} is
+                called.
+                """
+                request.notifyFinish().addBoth(lambda ign: logout())
+                return super().render(request)
+
+        return ResourceWrapper(avatar)
+
+    def _loginFailed(self, result):
+        """
+        Handle login failure by presenting either another challenge (for
+        expected authentication/authorization-related failures) or a server
+        error page (for anything else).
+        """
+        if result.check(error.Unauthorized, error.LoginFailed):
+            return UnauthorizedResource(self._credentialFactories)
+        else:
+            self._log.failure(
+                "HTTPAuthSessionWrapper.getChildWithDefault encountered "
+                "unexpected error",
+                failure=result,
+            )
+            return _UnsafeErrorPage(500, "Internal Error", "")
+
+    def _selectParseHeader(self, header):
+        """
+        Choose an C{ICredentialFactory} from C{_credentialFactories}
+        suitable to use to decode the given I{Authenticate} header.
+
+        @return: A two-tuple of a factory and the remaining portion of the
+            header value to be decoded or a two-tuple of L{None} if no
+            factory can decode the header value.
+        """
+        elements = header.split(b" ")
+        scheme = elements[0].lower()
+        for fact in self._credentialFactories:
+            if fact.scheme == scheme:
+                return (fact, b" ".join(elements[1:]))
+        return (None, None)
+
+    def putChild(self, path, child):
+        # IResource.putChild
+        raise NotImplementedError()
diff --git a/contrib/python/Twisted/py3/twisted/web/_element.py b/contrib/python/Twisted/py3/twisted/web/_element.py
new file mode 100644
index 00000000000..81d724071e4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_element.py
@@ -0,0 +1,200 @@
+# -*- test-case-name: twisted.web.test.test_template -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+import itertools
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    List,
+    Optional,
+    TypeVar,
+    Union,
+    overload,
+)
+
+from zope.interface import implementer
+
+from twisted.web.error import (
+    MissingRenderMethod,
+    MissingTemplateLoader,
+    UnexposedMethodError,
+)
+from twisted.web.iweb import IRenderable, IRequest, ITemplateLoader
+
+if TYPE_CHECKING:
+    from twisted.web.template import Flattenable, Tag
+
+
+T = TypeVar("T")
+_Tc = TypeVar("_Tc", bound=Callable[..., object])
+
+
+class Expose:
+    """
+    Helper for exposing methods for various uses using a simple decorator-style
+    callable.
+
+    Instances of this class can be called with one or more functions as
+    positional arguments.  The names of these functions will be added to a list
+    on the class object of which they are methods.
+    """
+
+    def __call__(self, f: _Tc, /, *funcObjs: Callable[..., object]) -> _Tc:
+        """
+        Add one or more functions to the set of exposed functions.
+
+        This is a way to declare something about a class definition, similar to
+        L{zope.interface.implementer}.  Use it like this::
+
+            magic = Expose('perform extra magic')
+            class Foo(Bar):
+                def twiddle(self, x, y):
+                    ...
+                def frob(self, a, b):
+                    ...
+                magic(twiddle, frob)
+
+        Later you can query the object::
+
+            aFoo = Foo()
+            magic.get(aFoo, 'twiddle')(x=1, y=2)
+
+        The call to C{get} will fail if the name it is given has not been
+        exposed using C{magic}.
+
+        @param funcObjs: One or more function objects which will be exposed to
+        the client.
+
+        @return: The first of C{funcObjs}.
+        """
+        for fObj in itertools.chain([f], funcObjs):
+            exposedThrough: List[Expose] = getattr(fObj, "exposedThrough", [])
+            exposedThrough.append(self)
+            setattr(fObj, "exposedThrough", exposedThrough)
+        return f
+
+    _nodefault = object()
+
+    @overload
+    def get(self, instance: object, methodName: str) -> Callable[..., Any]:
+        ...
+
+    @overload
+    def get(
+        self, instance: object, methodName: str, default: T
+    ) -> Union[Callable[..., Any], T]:
+        ...
+
+    def get(
+        self, instance: object, methodName: str, default: object = _nodefault
+    ) -> object:
+        """
+        Retrieve an exposed method with the given name from the given instance.
+
+        @raise UnexposedMethodError: Raised if C{default} is not specified and
+        there is no exposed method with the given name.
+
+        @return: A callable object for the named method assigned to the given
+        instance.
+        """
+        method = getattr(instance, methodName, None)
+        exposedThrough = getattr(method, "exposedThrough", [])
+        if self not in exposedThrough:
+            if default is self._nodefault:
+                raise UnexposedMethodError(self, methodName)
+            return default
+        return method
+
+
+def exposer(thunk: Callable[..., object]) -> Expose:
+    expose = Expose()
+    expose.__doc__ = thunk.__doc__
+    return expose
+
+
+@exposer
+def renderer() -> None:
+    """
+    Decorate with L{renderer} to use methods as template render directives.
+
+    For example::
+
+        class Foo(Element):
+            @renderer
+            def twiddle(self, request, tag):
+                return tag('Hello, world.')
+
+        <div xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
+            <span t:render="twiddle" />
+        </div>
+
+    Will result in this final output::
+
+        <div>
+            <span>Hello, world.</span>
+        </div>
+    """
+
+
+@implementer(IRenderable)
+class Element:
+    """
+    Base for classes which can render part of a page.
+
+    An Element is a renderer that can be embedded in a stan document and can
+    hook its template (from the loader) up to render methods.
+
+    An Element might be used to encapsulate the rendering of a complex piece of
+    data which is to be displayed in multiple different contexts.  The Element
+    allows the rendering logic to be easily re-used in different ways.
+
+    Element returns render methods which are registered using
+    L{twisted.web._element.renderer}.  For example::
+
+        class Menu(Element):
+            @renderer
+            def items(self, request, tag):
+                ....
+
+    Render methods are invoked with two arguments: first, the
+    L{twisted.web.http.Request} being served and second, the tag object which
+    "invoked" the render method.
+
+    @ivar loader: The factory which will be used to load documents to
+        return from C{render}.
+    """
+
+    loader: Optional[ITemplateLoader] = None
+
+    def __init__(self, loader: Optional[ITemplateLoader] = None):
+        if loader is not None:
+            self.loader = loader
+
+    def lookupRenderMethod(
+        self, name: str
+    ) -> Callable[[Optional[IRequest], "Tag"], "Flattenable"]:
+        """
+        Look up and return the named render method.
+        """
+        method = renderer.get(self, name, None)
+        if method is None:
+            raise MissingRenderMethod(self, name)
+        return method
+
+    def render(self, request: Optional[IRequest]) -> "Flattenable":
+        """
+        Implement L{IRenderable} to allow one L{Element} to be embedded in
+        another's template or rendering output.
+
+        (This will simply load the template from the C{loader}; when used in a
+        template, the flattening engine will keep track of this object
+        separately as the object to lookup renderers on and call
+        L{Element.renderer} to look them up.  The resulting object from this
+        method is not directly associated with this L{Element}.)
+        """
+        loader = self.loader
+        if loader is None:
+            raise MissingTemplateLoader(self)
+        return loader.load()
diff --git a/contrib/python/Twisted/py3/twisted/web/_flatten.py b/contrib/python/Twisted/py3/twisted/web/_flatten.py
new file mode 100644
index 00000000000..87a8bf2dfbf
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_flatten.py
@@ -0,0 +1,487 @@
+# -*- test-case-name: twisted.web.test.test_flatten,twisted.web.test.test_template -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Context-free flattener/serializer for rendering Python objects, possibly
+complex or arbitrarily nested, as strings.
+"""
+from __future__ import annotations
+
+from inspect import iscoroutine
+from io import BytesIO
+from sys import exc_info
+from traceback import extract_tb
+from types import GeneratorType
+from typing import (
+    Any,
+    Callable,
+    Coroutine,
+    Generator,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from twisted.internet.defer import Deferred, ensureDeferred
+from twisted.python.compat import nativeString
+from twisted.python.failure import Failure
+from twisted.web._stan import CDATA, CharRef, Comment, Tag, slot, voidElements
+from twisted.web.error import FlattenerError, UnfilledSlot, UnsupportedType
+from twisted.web.iweb import IRenderable, IRequest
+
+T = TypeVar("T")
+
+FlattenableRecursive = Any
+"""
+For documentation purposes, read C{FlattenableRecursive} as L{Flattenable}.
+However, since mypy doesn't support recursive type definitions (yet?),
+we'll put Any in the actual definition.
+"""
+
+Flattenable = Union[
+    bytes,
+    str,
+    slot,
+    CDATA,
+    Comment,
+    Tag,
+    Tuple[FlattenableRecursive, ...],
+    List[FlattenableRecursive],
+    Generator[FlattenableRecursive, None, None],
+    CharRef,
+    Deferred[FlattenableRecursive],
+    Coroutine[Deferred[FlattenableRecursive], object, FlattenableRecursive],
+    IRenderable,
+]
+"""
+Type alias containing all types that can be flattened by L{flatten()}.
+"""
+
+# The maximum number of bytes to synchronously accumulate in the flattener
+# buffer before delivering them onwards.
+BUFFER_SIZE = 2**16
+
+
+def escapeForContent(data: Union[bytes, str]) -> bytes:
+    """
+    Escape some character or UTF-8 byte data for inclusion in an HTML or XML
+    document, by replacing metacharacters (C{&<>}) with their entity
+    equivalents (C{&amp;&lt;&gt;}).
+
+    This is used as an input to L{_flattenElement}'s C{dataEscaper} parameter.
+
+    @param data: The string to escape.
+
+    @return: The quoted form of C{data}.  If C{data} is L{str}, return a utf-8
+        encoded string.
+    """
+    if isinstance(data, str):
+        data = data.encode("utf-8")
+    data = data.replace(b"&", b"&amp;").replace(b"<", b"&lt;").replace(b">", b"&gt;")
+    return data
+
+
+def attributeEscapingDoneOutside(data: Union[bytes, str]) -> bytes:
+    """
+    Escape some character or UTF-8 byte data for inclusion in the top level of
+    an attribute.  L{attributeEscapingDoneOutside} actually passes the data
+    through unchanged, because L{writeWithAttributeEscaping} handles the
+    quoting of the text within attributes outside the generator returned by
+    L{_flattenElement}; this is used as the C{dataEscaper} argument to that
+    L{_flattenElement} call so that that generator does not redundantly escape
+    its text output.
+
+    @param data: The string to escape.
+
+    @return: The string, unchanged, except for encoding.
+    """
+    if isinstance(data, str):
+        return data.encode("utf-8")
+    return data
+
+
+def writeWithAttributeEscaping(
+    write: Callable[[bytes], object]
+) -> Callable[[bytes], None]:
+    """
+    Decorate a C{write} callable so that all output written is properly quoted
+    for inclusion within an XML attribute value.
+
+    If a L{Tag <twisted.web.template.Tag>} C{x} is flattened within the context
+    of the contents of another L{Tag <twisted.web.template.Tag>} C{y}, the
+    metacharacters (C{<>&"}) delimiting C{x} should be passed through
+    unchanged, but the textual content of C{x} should still be quoted, as
+    usual.  For example: C{<y><x>&amp;</x></y>}.  That is the default behavior
+    of L{_flattenElement} when L{escapeForContent} is passed as the
+    C{dataEscaper}.
+
+    However, when a L{Tag <twisted.web.template.Tag>} C{x} is flattened within
+    the context of an I{attribute} of another L{Tag <twisted.web.template.Tag>}
+    C{y}, then the metacharacters delimiting C{x} should be quoted so that it
+    can be parsed from the attribute's value.  In the DOM itself, this is not a
+    valid thing to do, but given that renderers and slots may be freely moved
+    around in a L{twisted.web.template} template, it is a condition which may
+    arise in a document and must be handled in a way which produces valid
+    output.  So, for example, you should be able to get C{<y attr="&lt;x /&gt;"
+    />}.  This should also be true for other XML/HTML meta-constructs such as
+    comments and CDATA, so if you were to serialize a L{comment
+    <twisted.web.template.Comment>} in an attribute you should get C{<y
+    attr="&lt;-- comment --&gt;" />}.  Therefore in order to capture these
+    meta-characters, flattening is done with C{write} callable that is wrapped
+    with L{writeWithAttributeEscaping}.
+
+    The final case, and hopefully the much more common one as compared to
+    serializing L{Tag <twisted.web.template.Tag>} and arbitrary L{IRenderable}
+    objects within an attribute, is to serialize a simple string, and those
+    should be passed through for L{writeWithAttributeEscaping} to quote
+    without applying a second, redundant level of quoting.
+
+    @param write: A callable which will be invoked with the escaped L{bytes}.
+
+    @return: A callable that writes data with escaping.
+    """
+
+    def _write(data: bytes) -> None:
+        write(escapeForContent(data).replace(b'"', b"&quot;"))
+
+    return _write
+
+
+def escapedCDATA(data: Union[bytes, str]) -> bytes:
+    """
+    Escape CDATA for inclusion in a document.
+
+    @param data: The string to escape.
+
+    @return: The quoted form of C{data}. If C{data} is unicode, return a utf-8
+        encoded string.
+    """
+    if isinstance(data, str):
+        data = data.encode("utf-8")
+    return data.replace(b"]]>", b"]]]]><![CDATA[>")
+
+
+def escapedComment(data: Union[bytes, str]) -> bytes:
+    """
+    Within comments the sequence C{-->} can be mistaken as the end of the comment.
+    To ensure consistent parsing and valid output the sequence is replaced with C{--&gt;}.
+    Furthermore, whitespace is added when a comment ends in a dash. This is done to break
+    the connection of the ending C{-} with the closing C{-->}.
+
+    @param data: The string to escape.
+
+    @return: The quoted form of C{data}. If C{data} is unicode, return a utf-8
+        encoded string.
+    """
+    if isinstance(data, str):
+        data = data.encode("utf-8")
+    data = data.replace(b"-->", b"--&gt;")
+    if data and data[-1:] == b"-":
+        data += b" "
+    return data
+
+
+def _getSlotValue(
+    name: str,
+    slotData: Sequence[Optional[Mapping[str, Flattenable]]],
+    default: Optional[Flattenable] = None,
+) -> Flattenable:
+    """
+    Find the value of the named slot in the given stack of slot data.
+    """
+    for slotFrame in reversed(slotData):
+        if slotFrame is not None and name in slotFrame:
+            return slotFrame[name]
+    else:
+        if default is not None:
+            return default
+        raise UnfilledSlot(name)
+
+
+def _fork(d: Deferred[T]) -> Deferred[T]:
+    """
+    Create a new L{Deferred} based on C{d} that will fire and fail with C{d}'s
+    result or error, but will not modify C{d}'s callback type.
+    """
+    d2: Deferred[T] = Deferred(lambda _: d.cancel())
+
+    def callback(result: T) -> T:
+        d2.callback(result)
+        return result
+
+    def errback(failure: Failure) -> Failure:
+        d2.errback(failure)
+        return failure
+
+    d.addCallbacks(callback, errback)
+    return d2
+
+
+def _flattenElement(
+    request: Optional[IRequest],
+    root: Flattenable,
+    write: Callable[[bytes], object],
+    slotData: List[Optional[Mapping[str, Flattenable]]],
+    renderFactory: Optional[IRenderable],
+    dataEscaper: Callable[[Union[bytes, str]], bytes],
+    # This is annotated as Generator[T, None, None] instead of Iterator[T]
+    # because mypy does not consider an Iterator to be an instance of
+    # GeneratorType.
+) -> Generator[Union[Generator[Any, Any, Any], Deferred[Flattenable]], None, None]:
+    """
+    Make C{root} slightly more flat by yielding all its immediate contents as
+    strings, deferreds or generators that are recursive calls to itself.
+
+    @param request: A request object which will be passed to
+        L{IRenderable.render}.
+
+    @param root: An object to be made flatter.  This may be of type C{unicode},
+        L{str}, L{slot}, L{Tag <twisted.web.template.Tag>}, L{tuple}, L{list},
+        L{types.GeneratorType}, L{Deferred}, or an object that implements
+        L{IRenderable}.
+
+    @param write: A callable which will be invoked with each L{bytes} produced
+        by flattening C{root}.
+
+    @param slotData: A L{list} of L{dict} mapping L{str} slot names to data
+        with which those slots will be replaced.
+
+    @param renderFactory: If not L{None}, an object that provides
+        L{IRenderable}.
+
+    @param dataEscaper: A 1-argument callable which takes L{bytes} or
+        L{unicode} and returns L{bytes}, quoted as appropriate for the
+        rendering context.  This is really only one of two values:
+        L{attributeEscapingDoneOutside} or L{escapeForContent}, depending on
+        whether the rendering context is within an attribute or not.  See the
+        explanation in L{writeWithAttributeEscaping}.
+
+    @return: An iterator that eventually writes L{bytes} to C{write}.
+        It can yield other iterators or L{Deferred}s; if it yields another
+        iterator, the caller will iterate it; if it yields a L{Deferred},
+        the result of that L{Deferred} will be another generator, in which
+        case it is iterated.  See L{_flattenTree} for the trampoline that
+        consumes said values.
+    """
+
+    def keepGoing(
+        newRoot: Flattenable,
+        dataEscaper: Callable[[Union[bytes, str]], bytes] = dataEscaper,
+        renderFactory: Optional[IRenderable] = renderFactory,
+        write: Callable[[bytes], object] = write,
+    ) -> Generator[Union[Flattenable, Deferred[Flattenable]], None, None]:
+        return _flattenElement(
+            request, newRoot, write, slotData, renderFactory, dataEscaper
+        )
+
+    def keepGoingAsync(result: Deferred[Flattenable]) -> Deferred[Flattenable]:
+        return result.addCallback(keepGoing)
+
+    if isinstance(root, (bytes, str)):
+        write(dataEscaper(root))
+    elif isinstance(root, slot):
+        slotValue = _getSlotValue(root.name, slotData, root.default)
+        yield keepGoing(slotValue)
+    elif isinstance(root, CDATA):
+        write(b"<![CDATA[")
+        write(escapedCDATA(root.data))
+        write(b"]]>")
+    elif isinstance(root, Comment):
+        write(b"<!--")
+        write(escapedComment(root.data))
+        write(b"-->")
+    elif isinstance(root, Tag):
+        slotData.append(root.slotData)
+        rendererName = root.render
+        if rendererName is not None:
+            if renderFactory is None:
+                raise ValueError(
+                    f'Tag wants to be rendered by method "{rendererName}" '
+                    f"but is not contained in any IRenderable"
+                )
+            rootClone = root.clone(False)
+            rootClone.render = None
+            renderMethod = renderFactory.lookupRenderMethod(rendererName)
+            result = renderMethod(request, rootClone)
+            yield keepGoing(result)
+            slotData.pop()
+            return
+
+        if not root.tagName:
+            yield keepGoing(root.children)
+            return
+
+        write(b"<")
+        if isinstance(root.tagName, str):
+            tagName = root.tagName.encode("ascii")
+        else:
+            tagName = root.tagName
+        write(tagName)
+        for k, v in root.attributes.items():
+            if isinstance(k, str):
+                k = k.encode("ascii")
+            write(b" " + k + b'="')
+            # Serialize the contents of the attribute, wrapping the results of
+            # that serialization so that _everything_ is quoted.
+            yield keepGoing(
+                v, attributeEscapingDoneOutside, write=writeWithAttributeEscaping(write)
+            )
+            write(b'"')
+        if root.children or nativeString(tagName) not in voidElements:
+            write(b">")
+            # Regardless of whether we're in an attribute or not, switch back
+            # to the escapeForContent dataEscaper.  The contents of a tag must
+            # be quoted no matter what; in the top-level document, just so
+            # they're valid, and if they're within an attribute, they have to
+            # be quoted so that after applying the *un*-quoting required to re-
+            # parse the tag within the attribute, all the quoting is still
+            # correct.
+            yield keepGoing(root.children, escapeForContent)
+            write(b"</" + tagName + b">")
+        else:
+            write(b" />")
+
+    elif isinstance(root, (tuple, list, GeneratorType)):
+        for element in root:
+            yield keepGoing(element)
+    elif isinstance(root, CharRef):
+        escaped = "&#%d;" % (root.ordinal,)
+        write(escaped.encode("ascii"))
+    elif isinstance(root, Deferred):
+        yield keepGoingAsync(_fork(root))
+    elif iscoroutine(root):
+        yield keepGoingAsync(
+            Deferred.fromCoroutine(
+                cast(Coroutine[Deferred[Flattenable], object, Flattenable], root)
+            )
+        )
+    elif IRenderable.providedBy(root):
+        result = root.render(request)
+        yield keepGoing(result, renderFactory=root)
+    else:
+        raise UnsupportedType(root)
+
+
+async def _flattenTree(
+    request: Optional[IRequest], root: Flattenable, write: Callable[[bytes], object]
+) -> None:
+    """
+    Make C{root} into an iterable of L{bytes} and L{Deferred} by doing a depth
+    first traversal of the tree.
+
+    @param request: A request object which will be passed to
+        L{IRenderable.render}.
+
+    @param root: An object to be made flatter.  This may be of type C{unicode},
+        L{bytes}, L{slot}, L{Tag <twisted.web.template.Tag>}, L{tuple},
+        L{list}, L{types.GeneratorType}, L{Deferred}, or something providing
+        L{IRenderable}.
+
+    @param write: A callable which will be invoked with each L{bytes} produced
+        by flattening C{root}.
+
+    @return: A C{Deferred}-returning coroutine that resolves to C{None}.
+    """
+    buf = []
+    bufSize = 0
+
+    # Accumulate some bytes up to the buffer size so that we don't annoy the
+    # upstream writer with a million tiny string.
+    def bufferedWrite(bs: bytes) -> None:
+        nonlocal bufSize
+        buf.append(bs)
+        bufSize += len(bs)
+        if bufSize >= BUFFER_SIZE:
+            flushBuffer()
+
+    # Deliver the buffered content to the upstream writer as a single string.
+    # This is how a "big enough" buffer gets delivered, how a buffer of any
+    # size is delivered before execution is suspended to wait for an
+    # asynchronous value, and how anything left in the buffer when we're
+    # finished is delivered.
+    def flushBuffer() -> None:
+        nonlocal bufSize
+        if bufSize > 0:
+            write(b"".join(buf))
+            del buf[:]
+            bufSize = 0
+
+    stack: List[Generator[Any, Any, Any]] = [
+        _flattenElement(request, root, bufferedWrite, [], None, escapeForContent)
+    ]
+
+    while stack:
+        try:
+            frame = stack[-1].gi_frame
+            element = next(stack[-1])
+            if isinstance(element, Deferred):
+                # Before suspending flattening for an unknown amount of time,
+                # flush whatever data we have collected so far.
+                flushBuffer()
+                element = await element
+        except StopIteration:
+            stack.pop()
+        except Exception as e:
+            stack.pop()
+            roots = []
+            for generator in stack:
+                roots.append(generator.gi_frame.f_locals["root"])
+            roots.append(frame.f_locals["root"])
+            raise FlattenerError(e, roots, extract_tb(exc_info()[2]))
+        else:
+            stack.append(element)
+
+    # Flush any data that remains in the buffer before finishing.
+    flushBuffer()
+
+
+def flatten(
+    request: Optional[IRequest], root: Flattenable, write: Callable[[bytes], object]
+) -> Deferred[None]:
+    """
+    Incrementally write out a string representation of C{root} using C{write}.
+
+    In order to create a string representation, C{root} will be decomposed into
+    simpler objects which will themselves be decomposed and so on until strings
+    or objects which can easily be converted to strings are encountered.
+
+    @param request: A request object which will be passed to the C{render}
+        method of any L{IRenderable} provider which is encountered.
+
+    @param root: An object to be made flatter.  This may be of type L{str},
+        L{bytes}, L{slot}, L{Tag <twisted.web.template.Tag>}, L{tuple},
+        L{list}, L{types.GeneratorType}, L{Deferred}, or something that
+        provides L{IRenderable}.
+
+    @param write: A callable which will be invoked with each L{bytes} produced
+        by flattening C{root}.
+
+    @return: A L{Deferred} which will be called back with C{None} when C{root}
+        has been completely flattened into C{write} or which will be errbacked
+        if an unexpected exception occurs.
+    """
+    return ensureDeferred(_flattenTree(request, root, write))
+
+
+def flattenString(request: Optional[IRequest], root: Flattenable) -> Deferred[bytes]:
+    """
+    Collate a string representation of C{root} into a single string.
+
+    This is basically gluing L{flatten} to an L{io.BytesIO} and returning
+    the results. See L{flatten} for the exact meanings of C{request} and
+    C{root}.
+
+    @return: A L{Deferred} which will be called back with a single UTF-8 encoded
+        string as its result when C{root} has been completely flattened or which
+        will be errbacked if an unexpected exception occurs.
+    """
+    io = BytesIO()
+    d = flatten(request, root, io.write)
+    d.addCallback(lambda _: io.getvalue())
+    return cast(Deferred[bytes], d)
diff --git a/contrib/python/Twisted/py3/twisted/web/_http2.py b/contrib/python/Twisted/py3/twisted/web/_http2.py
new file mode 100644
index 00000000000..57762e1805d
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_http2.py
@@ -0,0 +1,1283 @@
+# -*- test-case-name: twisted.web.test.test_http2 -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HTTP2 Implementation
+
+This is the basic server-side protocol implementation used by the Twisted
+Web server for HTTP2.  This functionality is intended to be combined with the
+HTTP/1.1 and HTTP/1.0 functionality in twisted.web.http to provide complete
+protocol support for HTTP-type protocols.
+
+This API is currently considered private because it's in early draft form. When
+it has stabilised, it'll be made public.
+"""
+
+
+import io
+from collections import deque
+from typing import List
+
+from zope.interface import implementer
+
+import h2.config  # type: ignore[import]
+import h2.connection  # type: ignore[import]
+import h2.errors  # type: ignore[import]
+import h2.events  # type: ignore[import]
+import h2.exceptions  # type: ignore[import]
+import priority  # type: ignore[import]
+
+from twisted.internet._producer_helpers import _PullToPush
+from twisted.internet.defer import Deferred
+from twisted.internet.error import ConnectionLost
+from twisted.internet.interfaces import (
+    IConsumer,
+    IProtocol,
+    IPushProducer,
+    ISSLTransport,
+    ITransport,
+)
+from twisted.internet.protocol import Protocol
+from twisted.logger import Logger
+from twisted.protocols.policies import TimeoutMixin
+from twisted.python.failure import Failure
+from twisted.web.error import ExcessiveBufferingError
+
+# This API is currently considered private.
+__all__: List[str] = []
+
+
+_END_STREAM_SENTINEL = object()
+
+
+@implementer(IProtocol, IPushProducer)
+class H2Connection(Protocol, TimeoutMixin):
+    """
+    A class representing a single HTTP/2 connection.
+
+    This implementation of L{IProtocol} works hand in hand with L{H2Stream}.
+    This is because we have the requirement to register multiple producers for
+    a single HTTP/2 connection, one for each stream. The standard Twisted
+    interfaces don't really allow for this, so instead there's a custom
+    interface between the two objects that allows them to work hand-in-hand here.
+
+    @ivar conn: The HTTP/2 connection state machine.
+    @type conn: L{h2.connection.H2Connection}
+
+    @ivar streams: A mapping of stream IDs to L{H2Stream} objects, used to call
+        specific methods on streams when events occur.
+    @type streams: L{dict}, mapping L{int} stream IDs to L{H2Stream} objects.
+
+    @ivar priority: A HTTP/2 priority tree used to ensure that responses are
+        prioritised appropriately.
+    @type priority: L{priority.PriorityTree}
+
+    @ivar _consumerBlocked: A flag tracking whether or not the L{IConsumer}
+        that is consuming this data has asked us to stop producing.
+    @type _consumerBlocked: L{bool}
+
+    @ivar _sendingDeferred: A L{Deferred} used to restart the data-sending loop
+        when more response data has been produced. Will not be present if there
+        is outstanding data still to send.
+    @type _consumerBlocked: A L{twisted.internet.defer.Deferred}, or L{None}
+
+    @ivar _outboundStreamQueues: A map of stream IDs to queues, used to store
+        data blocks that are yet to be sent on the connection. These are used
+        both to handle producers that do not respect L{IConsumer} but also to
+        allow priority to multiplex data appropriately.
+    @type _outboundStreamQueues: A L{dict} mapping L{int} stream IDs to
+        L{collections.deque} queues, which contain either L{bytes} objects or
+        C{_END_STREAM_SENTINEL}.
+
+    @ivar _sender: A handle to the data-sending loop, allowing it to be
+        terminated if needed.
+    @type _sender: L{twisted.internet.task.LoopingCall}
+
+    @ivar abortTimeout: The number of seconds to wait after we attempt to shut
+        the transport down cleanly to give up and forcibly terminate it. This
+        is only used when we time a connection out, to prevent errors causing
+        the FD to get leaked. If this is L{None}, we will wait forever.
+    @type abortTimeout: L{int}
+
+    @ivar _abortingCall: The L{twisted.internet.base.DelayedCall} that will be
+        used to forcibly close the transport if it doesn't close cleanly.
+    @type _abortingCall: L{twisted.internet.base.DelayedCall}
+    """
+
+    factory = None
+    site = None
+    abortTimeout = 15
+
+    _log = Logger()
+    _abortingCall = None
+
+    def __init__(self, reactor=None):
+        config = h2.config.H2Configuration(client_side=False, header_encoding=None)
+        self.conn = h2.connection.H2Connection(config=config)
+        self.streams = {}
+
+        self.priority = priority.PriorityTree()
+        self._consumerBlocked = None
+        self._sendingDeferred = None
+        self._outboundStreamQueues = {}
+        self._streamCleanupCallbacks = {}
+        self._stillProducing = True
+
+        # Limit the number of buffered control frame (e.g. PING and
+        # SETTINGS) bytes.
+        self._maxBufferedControlFrameBytes = 1024 * 17
+        self._bufferedControlFrames = deque()
+        self._bufferedControlFrameBytes = 0
+
+        if reactor is None:
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+        # Start the data sending function.
+        self._reactor.callLater(0, self._sendPrioritisedData)
+
+    # Implementation of IProtocol
+    def connectionMade(self):
+        """
+        Called by the reactor when a connection is received. May also be called
+        by the L{twisted.web.http._GenericHTTPChannelProtocol} during upgrade
+        to HTTP/2.
+        """
+        self.setTimeout(self.timeOut)
+        self.conn.initiate_connection()
+        self.transport.write(self.conn.data_to_send())
+
+    def dataReceived(self, data):
+        """
+        Called whenever a chunk of data is received from the transport.
+
+        @param data: The data received from the transport.
+        @type data: L{bytes}
+        """
+        try:
+            events = self.conn.receive_data(data)
+        except h2.exceptions.ProtocolError:
+            stillActive = self._tryToWriteControlData()
+            if stillActive:
+                self.transport.loseConnection()
+                self.connectionLost(Failure(), _cancelTimeouts=False)
+            return
+
+        # Only reset the timeout if we've received an actual H2
+        # protocol message
+        self.resetTimeout()
+
+        for event in events:
+            if isinstance(event, h2.events.RequestReceived):
+                self._requestReceived(event)
+            elif isinstance(event, h2.events.DataReceived):
+                self._requestDataReceived(event)
+            elif isinstance(event, h2.events.StreamEnded):
+                self._requestEnded(event)
+            elif isinstance(event, h2.events.StreamReset):
+                self._requestAborted(event)
+            elif isinstance(event, h2.events.WindowUpdated):
+                self._handleWindowUpdate(event)
+            elif isinstance(event, h2.events.PriorityUpdated):
+                self._handlePriorityUpdate(event)
+            elif isinstance(event, h2.events.ConnectionTerminated):
+                self.transport.loseConnection()
+                self.connectionLost(
+                    Failure(ConnectionLost("Remote peer sent GOAWAY")),
+                    _cancelTimeouts=False,
+                )
+
+        self._tryToWriteControlData()
+
+    def timeoutConnection(self):
+        """
+        Called when the connection has been inactive for
+        L{self.timeOut<twisted.protocols.policies.TimeoutMixin.timeOut>}
+        seconds. Cleanly tears the connection down, attempting to notify the
+        peer if needed.
+
+        We override this method to add two extra bits of functionality:
+
+         - We want to log the timeout.
+         - We want to send a GOAWAY frame indicating that the connection is
+           being terminated, and whether it was clean or not. We have to do this
+           before the connection is torn down.
+        """
+        self._log.info("Timing out client {client}", client=self.transport.getPeer())
+
+        # Check whether there are open streams. If there are, we're going to
+        # want to use the error code PROTOCOL_ERROR. If there aren't, use
+        # NO_ERROR.
+        if self.conn.open_outbound_streams > 0 or self.conn.open_inbound_streams > 0:
+            error_code = h2.errors.ErrorCodes.PROTOCOL_ERROR
+        else:
+            error_code = h2.errors.ErrorCodes.NO_ERROR
+
+        self.conn.close_connection(error_code=error_code)
+        self.transport.write(self.conn.data_to_send())
+
+        # Don't let the client hold this connection open too long.
+        if self.abortTimeout is not None:
+            # We use self.callLater because that's what TimeoutMixin does, even
+            # though we have a perfectly good reactor sitting around. See
+            # https://twistedmatrix.com/trac/ticket/8488.
+            self._abortingCall = self.callLater(
+                self.abortTimeout, self.forceAbortClient
+            )
+
+        # We're done, throw the connection away.
+        self.transport.loseConnection()
+
+    def forceAbortClient(self):
+        """
+        Called if C{abortTimeout} seconds have passed since the timeout fired,
+        and the connection still hasn't gone away. This can really only happen
+        on extremely bad connections or when clients are maliciously attempting
+        to keep connections open.
+        """
+        self._log.info(
+            "Forcibly timing out client: {client}", client=self.transport.getPeer()
+        )
+        # We want to lose track of the _abortingCall so that no-one tries to
+        # cancel it.
+        self._abortingCall = None
+        self.transport.abortConnection()
+
+    def connectionLost(self, reason, _cancelTimeouts=True):
+        """
+        Called when the transport connection is lost.
+
+        Informs all outstanding response handlers that the connection
+        has been lost, and cleans up all internal state.
+
+        @param reason: See L{IProtocol.connectionLost}
+
+        @param _cancelTimeouts: Propagate the C{reason} to this
+            connection's streams but don't cancel any timers, so that
+            peers who never read the data we've written are eventually
+            timed out.
+        """
+        self._stillProducing = False
+        if _cancelTimeouts:
+            self.setTimeout(None)
+
+        for stream in self.streams.values():
+            stream.connectionLost(reason)
+
+        for streamID in list(self.streams.keys()):
+            self._requestDone(streamID)
+
+        # If we were going to force-close the transport, we don't have to now.
+        if _cancelTimeouts and self._abortingCall is not None:
+            self._abortingCall.cancel()
+            self._abortingCall = None
+
+    # Implementation of IPushProducer
+    #
+    # Here's how we handle IPushProducer. We have multiple outstanding
+    # H2Streams. Each of these exposes an IConsumer interface to the response
+    # handler that allows it to push data into the H2Stream. The H2Stream then
+    # writes the data into the H2Connection object.
+    #
+    # The H2Connection needs to manage these writes to account for:
+    #
+    # - flow control
+    # - priority
+    #
+    # We manage each of these in different ways.
+    #
+    # For flow control, we simply use the equivalent of the IPushProducer
+    # interface. We simply tell the H2Stream: "Hey, you can't send any data
+    # right now, sorry!". When that stream becomes unblocked, we free it up
+    # again. This allows the H2Stream to propagate this backpressure up the
+    # chain.
+    #
+    # For priority, we need to keep a backlog of data frames that we can send,
+    # and interleave them appropriately. This backlog is most sensibly kept in
+    # the H2Connection object itself. We keep one queue per stream, which is
+    # where the writes go, and then we have a loop that manages popping these
+    # streams off in priority order.
+    #
+    # Logically then, we go as follows:
+    #
+    # 1. Stream calls writeDataToStream(). This causes a DataFrame to be placed
+    #    on the queue for that stream. It also informs the priority
+    #    implementation that this stream is unblocked.
+    # 2. The _sendPrioritisedData() function spins in a tight loop. Each
+    #    iteration it asks the priority implementation which stream should send
+    #    next, and pops a data frame off that stream's queue. If, after sending
+    #    that frame, there is no data left on that stream's queue, the function
+    #    informs the priority implementation that the stream is blocked.
+    #
+    # If all streams are blocked, or if there are no outstanding streams, the
+    # _sendPrioritisedData function waits to be awoken when more data is ready
+    # to send.
+    #
+    # Note that all of this only applies to *data*. Headers and other control
+    # frames deliberately skip this processing as they are not subject to flow
+    # control or priority constraints. Instead, they are stored in their own buffer
+    # which is used primarily to detect excessive buffering.
+    def stopProducing(self):
+        """
+        Stop producing data.
+
+        This tells the L{H2Connection} that its consumer has died, so it must
+        stop producing data for good.
+        """
+        self.connectionLost(Failure(ConnectionLost("Producing stopped")))
+
+    def pauseProducing(self):
+        """
+        Pause producing data.
+
+        Tells the L{H2Connection} that it has produced too much data to process
+        for the time being, and to stop until resumeProducing() is called.
+        """
+        self._consumerBlocked = Deferred()
+        # Ensure pending control data (if any) are sent first.
+        self._consumerBlocked.addCallback(self._flushBufferedControlData)
+
+    def resumeProducing(self):
+        """
+        Resume producing data.
+
+        This tells the L{H2Connection} to re-add itself to the main loop and
+        produce more data for the consumer.
+        """
+        if self._consumerBlocked is not None:
+            d = self._consumerBlocked
+            self._consumerBlocked = None
+            d.callback(None)
+
+    def _sendPrioritisedData(self, *args):
+        """
+        The data sending loop. This function repeatedly calls itself, either
+        from L{Deferred}s or from
+        L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>}
+
+        This function sends data on streams according to the rules of HTTP/2
+        priority. It ensures that the data from each stream is interleved
+        according to the priority signalled by the client, making sure that the
+        connection is used with maximal efficiency.
+
+        This function will execute if data is available: if all data is
+        exhausted, the function will place a deferred onto the L{H2Connection}
+        object and wait until it is called to resume executing.
+        """
+        # If producing has stopped, we're done. Don't reschedule ourselves
+        if not self._stillProducing:
+            return
+
+        stream = None
+
+        while stream is None:
+            try:
+                stream = next(self.priority)
+            except priority.DeadlockError:
+                # All streams are currently blocked or not progressing. Wait
+                # until a new one becomes available.
+                assert self._sendingDeferred is None
+                self._sendingDeferred = Deferred()
+                self._sendingDeferred.addCallback(self._sendPrioritisedData)
+                return
+
+        # Wait behind the transport.
+        if self._consumerBlocked is not None:
+            self._consumerBlocked.addCallback(self._sendPrioritisedData)
+            return
+
+        self.resetTimeout()
+
+        remainingWindow = self.conn.local_flow_control_window(stream)
+        frameData = self._outboundStreamQueues[stream].popleft()
+        maxFrameSize = min(self.conn.max_outbound_frame_size, remainingWindow)
+
+        if frameData is _END_STREAM_SENTINEL:
+            # There's no error handling here even though this can throw
+            # ProtocolError because we really shouldn't encounter this problem.
+            # If we do, that's a nasty bug.
+            self.conn.end_stream(stream)
+            self.transport.write(self.conn.data_to_send())
+
+            # Clean up the stream
+            self._requestDone(stream)
+        else:
+            # Respect the max frame size.
+            if len(frameData) > maxFrameSize:
+                excessData = frameData[maxFrameSize:]
+                frameData = frameData[:maxFrameSize]
+                self._outboundStreamQueues[stream].appendleft(excessData)
+
+            # There's deliberately no error handling here, because this just
+            # absolutely should not happen.
+            # If for whatever reason the max frame length is zero and so we
+            # have no frame data to send, don't send any.
+            if frameData:
+                self.conn.send_data(stream, frameData)
+                self.transport.write(self.conn.data_to_send())
+
+            # If there's no data left, this stream is now blocked.
+            if not self._outboundStreamQueues[stream]:
+                self.priority.block(stream)
+
+            # Also, if the stream's flow control window is exhausted, tell it
+            # to stop.
+            if self.remainingOutboundWindow(stream) <= 0:
+                self.streams[stream].flowControlBlocked()
+
+        self._reactor.callLater(0, self._sendPrioritisedData)
+
+    # Internal functions.
+    def _requestReceived(self, event):
+        """
+        Internal handler for when a request has been received.
+
+        @param event: The Hyper-h2 event that encodes information about the
+            received request.
+        @type event: L{h2.events.RequestReceived}
+        """
+        stream = H2Stream(
+            event.stream_id,
+            self,
+            event.headers,
+            self.requestFactory,
+            self.site,
+            self.factory,
+        )
+        self.streams[event.stream_id] = stream
+        self._streamCleanupCallbacks[event.stream_id] = Deferred()
+        self._outboundStreamQueues[event.stream_id] = deque()
+
+        # Add the stream to the priority tree but immediately block it.
+        try:
+            self.priority.insert_stream(event.stream_id)
+        except priority.DuplicateStreamError:
+            # Stream already in the tree. This can happen if we received a
+            # PRIORITY frame before a HEADERS frame. Just move on: we set the
+            # stream up properly in _handlePriorityUpdate.
+            pass
+        else:
+            self.priority.block(event.stream_id)
+
+    def _requestDataReceived(self, event):
+        """
+        Internal handler for when a chunk of data is received for a given
+        request.
+
+        @param event: The Hyper-h2 event that encodes information about the
+            received data.
+        @type event: L{h2.events.DataReceived}
+        """
+        stream = self.streams[event.stream_id]
+        stream.receiveDataChunk(event.data, event.flow_controlled_length)
+
+    def _requestEnded(self, event):
+        """
+        Internal handler for when a request is complete, and we expect no
+        further data for that request.
+
+        @param event: The Hyper-h2 event that encodes information about the
+            completed stream.
+        @type event: L{h2.events.StreamEnded}
+        """
+        stream = self.streams[event.stream_id]
+        stream.requestComplete()
+
+    def _requestAborted(self, event):
+        """
+        Internal handler for when a request is aborted by a remote peer.
+
+        @param event: The Hyper-h2 event that encodes information about the
+            reset stream.
+        @type event: L{h2.events.StreamReset}
+        """
+        stream = self.streams[event.stream_id]
+        stream.connectionLost(
+            Failure(ConnectionLost("Stream reset with code %s" % event.error_code))
+        )
+        self._requestDone(event.stream_id)
+
+    def _handlePriorityUpdate(self, event):
+        """
+        Internal handler for when a stream priority is updated.
+
+        @param event: The Hyper-h2 event that encodes information about the
+            stream reprioritization.
+        @type event: L{h2.events.PriorityUpdated}
+        """
+        try:
+            self.priority.reprioritize(
+                stream_id=event.stream_id,
+                depends_on=event.depends_on or None,
+                weight=event.weight,
+                exclusive=event.exclusive,
+            )
+        except priority.MissingStreamError:
+            # A PRIORITY frame arrived before the HEADERS frame that would
+            # trigger us to insert the stream into the tree. That's fine: we
+            # can create the stream here and mark it as blocked.
+            self.priority.insert_stream(
+                stream_id=event.stream_id,
+                depends_on=event.depends_on or None,
+                weight=event.weight,
+                exclusive=event.exclusive,
+            )
+            self.priority.block(event.stream_id)
+
+    def writeHeaders(self, version, code, reason, headers, streamID):
+        """
+        Called by L{twisted.web.http.Request} objects to write a complete set
+        of HTTP headers to a stream.
+
+        @param version: The HTTP version in use. Unused in HTTP/2.
+        @type version: L{bytes}
+
+        @param code: The HTTP status code to write.
+        @type code: L{bytes}
+
+        @param reason: The HTTP reason phrase to write. Unused in HTTP/2.
+        @type reason: L{bytes}
+
+        @param headers: The headers to write to the stream.
+        @type headers: L{twisted.web.http_headers.Headers}
+
+        @param streamID: The ID of the stream to write the headers to.
+        @type streamID: L{int}
+        """
+        headers.insert(0, (b":status", code))
+
+        try:
+            self.conn.send_headers(streamID, headers)
+        except h2.exceptions.StreamClosedError:
+            # Stream was closed by the client at some point. We need to not
+            # explode here: just swallow the error. That's what write() does
+            # when a connection is lost, so that's what we do too.
+            return
+        else:
+            self._tryToWriteControlData()
+
+    def writeDataToStream(self, streamID, data):
+        """
+        May be called by L{H2Stream} objects to write response data to a given
+        stream. Writes a single data frame.
+
+        @param streamID: The ID of the stream to write the data to.
+        @type streamID: L{int}
+
+        @param data: The data chunk to write to the stream.
+        @type data: L{bytes}
+        """
+        self._outboundStreamQueues[streamID].append(data)
+
+        # There's obviously no point unblocking this stream and the sending
+        # loop if the data can't actually be sent, so confirm that there's
+        # some room to send data.
+        if self.conn.local_flow_control_window(streamID) > 0:
+            self.priority.unblock(streamID)
+            if self._sendingDeferred is not None:
+                d = self._sendingDeferred
+                self._sendingDeferred = None
+                d.callback(streamID)
+
+        if self.remainingOutboundWindow(streamID) <= 0:
+            self.streams[streamID].flowControlBlocked()
+
+    def endRequest(self, streamID):
+        """
+        Called by L{H2Stream} objects to signal completion of a response.
+
+        @param streamID: The ID of the stream to write the data to.
+        @type streamID: L{int}
+        """
+        self._outboundStreamQueues[streamID].append(_END_STREAM_SENTINEL)
+        self.priority.unblock(streamID)
+        if self._sendingDeferred is not None:
+            d = self._sendingDeferred
+            self._sendingDeferred = None
+            d.callback(streamID)
+
+    def abortRequest(self, streamID):
+        """
+        Called by L{H2Stream} objects to request early termination of a stream.
+        This emits a RstStream frame and then removes all stream state.
+
+        @param streamID: The ID of the stream to write the data to.
+        @type streamID: L{int}
+        """
+        self.conn.reset_stream(streamID)
+        stillActive = self._tryToWriteControlData()
+        if stillActive:
+            self._requestDone(streamID)
+
+    def _requestDone(self, streamID):
+        """
+        Called internally by the data sending loop to clean up state that was
+        being used for the stream. Called when the stream is complete.
+
+        @param streamID: The ID of the stream to clean up state for.
+        @type streamID: L{int}
+        """
+        del self._outboundStreamQueues[streamID]
+        self.priority.remove_stream(streamID)
+        del self.streams[streamID]
+        cleanupCallback = self._streamCleanupCallbacks.pop(streamID)
+        cleanupCallback.callback(streamID)
+
+    def remainingOutboundWindow(self, streamID):
+        """
+        Called to determine how much room is left in the send window for a
+        given stream. Allows us to handle blocking and unblocking producers.
+
+        @param streamID: The ID of the stream whose flow control window we'll
+            check.
+        @type streamID: L{int}
+
+        @return: The amount of room remaining in the send window for the given
+            stream, including the data queued to be sent.
+        @rtype: L{int}
+        """
+        # TODO: This involves a fair bit of looping and computation for
+        # something that is called a lot. Consider caching values somewhere.
+        windowSize = self.conn.local_flow_control_window(streamID)
+        sendQueue = self._outboundStreamQueues[streamID]
+        alreadyConsumed = sum(
+            len(chunk) for chunk in sendQueue if chunk is not _END_STREAM_SENTINEL
+        )
+
+        return windowSize - alreadyConsumed
+
+    def _handleWindowUpdate(self, event):
+        """
+        Manage flow control windows.
+
+        Streams that are blocked on flow control will register themselves with
+        the connection. This will fire deferreds that wake those streams up and
+        allow them to continue processing.
+
+        @param event: The Hyper-h2 event that encodes information about the
+            flow control window change.
+        @type event: L{h2.events.WindowUpdated}
+        """
+        streamID = event.stream_id
+
+        if streamID:
+            if not self._streamIsActive(streamID):
+                # We may have already cleaned up our stream state, making this
+                # a late WINDOW_UPDATE frame. That's fine: the update is
+                # unnecessary but benign. We'll ignore it.
+                return
+
+            # If we haven't got any data to send, don't unblock the stream. If
+            # we do, we'll eventually get an exception inside the
+            # _sendPrioritisedData loop some time later.
+            if self._outboundStreamQueues.get(streamID):
+                self.priority.unblock(streamID)
+            self.streams[streamID].windowUpdated()
+        else:
+            # Update strictly applies to all streams.
+            for stream in self.streams.values():
+                stream.windowUpdated()
+
+                # If we still have data to send for this stream, unblock it.
+                if self._outboundStreamQueues.get(stream.streamID):
+                    self.priority.unblock(stream.streamID)
+
+    def getPeer(self):
+        """
+        Get the remote address of this connection.
+
+        Treat this method with caution.  It is the unfortunate result of the
+        CGI and Jabber standards, but should not be considered reliable for
+        the usual host of reasons; port forwarding, proxying, firewalls, IP
+        masquerading, etc.
+
+        @return: An L{IAddress} provider.
+        """
+        return self.transport.getPeer()
+
+    def getHost(self):
+        """
+        Similar to getPeer, but returns an address describing this side of the
+        connection.
+
+        @return: An L{IAddress} provider.
+        """
+        return self.transport.getHost()
+
+    def openStreamWindow(self, streamID, increment):
+        """
+        Open the stream window by a given increment.
+
+        @param streamID: The ID of the stream whose window needs to be opened.
+        @type streamID: L{int}
+
+        @param increment: The amount by which the stream window must be
+        incremented.
+        @type increment: L{int}
+        """
+        self.conn.acknowledge_received_data(increment, streamID)
+        self._tryToWriteControlData()
+
+    def _isSecure(self):
+        """
+        Returns L{True} if this channel is using a secure transport.
+
+        @returns: L{True} if this channel is secure.
+        @rtype: L{bool}
+        """
+        # A channel is secure if its transport is ISSLTransport.
+        return ISSLTransport(self.transport, None) is not None
+
+    def _send100Continue(self, streamID):
+        """
+        Sends a 100 Continue response, used to signal to clients that further
+        processing will be performed.
+
+        @param streamID: The ID of the stream that needs the 100 Continue
+        response
+        @type streamID: L{int}
+        """
+        headers = [(b":status", b"100")]
+        self.conn.send_headers(headers=headers, stream_id=streamID)
+        self._tryToWriteControlData()
+
+    def _respondToBadRequestAndDisconnect(self, streamID):
+        """
+        This is a quick and dirty way of responding to bad requests.
+
+        As described by HTTP standard we should be patient and accept the
+        whole request from the client before sending a polite bad request
+        response, even in the case when clients send tons of data.
+
+        Unlike in the HTTP/1.1 case, this does not actually disconnect the
+        underlying transport: there's no need. This instead just sends a 400
+        response and terminates the stream.
+
+        @param streamID: The ID of the stream that needs the 100 Continue
+        response
+        @type streamID: L{int}
+        """
+        headers = [(b":status", b"400")]
+        self.conn.send_headers(headers=headers, stream_id=streamID, end_stream=True)
+        stillActive = self._tryToWriteControlData()
+        if stillActive:
+            stream = self.streams[streamID]
+            stream.connectionLost(Failure(ConnectionLost("Invalid request")))
+            self._requestDone(streamID)
+
+    def _streamIsActive(self, streamID):
+        """
+        Checks whether Twisted has still got state for a given stream and so
+        can process events for that stream.
+
+        @param streamID: The ID of the stream that needs processing.
+        @type streamID: L{int}
+
+        @return: Whether the stream still has state allocated.
+        @rtype: L{bool}
+        """
+        return streamID in self.streams
+
+    def _tryToWriteControlData(self):
+        """
+        Checks whether the connection is blocked on flow control and,
+        if it isn't, writes any buffered control data.
+
+        @return: L{True} if the connection is still active and
+            L{False} if it was aborted because too many bytes have
+            been written but not consumed by the other end.
+        """
+        bufferedBytes = self.conn.data_to_send()
+        if not bufferedBytes:
+            return True
+
+        if self._consumerBlocked is None and not self._bufferedControlFrames:
+            # The consumer isn't blocked, and we don't have any buffered frames:
+            # write this directly.
+            self.transport.write(bufferedBytes)
+            return True
+        else:
+            # Either the consumer is blocked or we have buffered frames. If the
+            # consumer is blocked, we'll write this when we unblock. If we have
+            # buffered frames, we have presumably been re-entered from
+            # transport.write, and so to avoid reordering issues we'll buffer anyway.
+            self._bufferedControlFrames.append(bufferedBytes)
+            self._bufferedControlFrameBytes += len(bufferedBytes)
+
+            if self._bufferedControlFrameBytes >= self._maxBufferedControlFrameBytes:
+                maxBuffCtrlFrameBytes = self._maxBufferedControlFrameBytes
+                self._log.error(
+                    "Maximum number of control frame bytes buffered: "
+                    "{bufferedControlFrameBytes} > = "
+                    "{maxBufferedControlFrameBytes}. "
+                    "Aborting connection to client: {client} ",
+                    bufferedControlFrameBytes=self._bufferedControlFrameBytes,
+                    maxBufferedControlFrameBytes=maxBuffCtrlFrameBytes,
+                    client=self.transport.getPeer(),
+                )
+                # We've exceeded a reasonable buffer size for max buffered
+                # control frames. This is a denial of service risk, so we're
+                # going to drop this connection.
+                self.transport.abortConnection()
+                self.connectionLost(Failure(ExcessiveBufferingError()))
+                return False
+            return True
+
+    def _flushBufferedControlData(self, *args):
+        """
+        Called when the connection is marked writable again after being marked unwritable.
+        Attempts to flush buffered control data if there is any.
+        """
+        # To respect backpressure here we send each write in order, paying attention to whether
+        # we got blocked
+        while self._consumerBlocked is None and self._bufferedControlFrames:
+            nextWrite = self._bufferedControlFrames.popleft()
+            self._bufferedControlFrameBytes -= len(nextWrite)
+            self.transport.write(nextWrite)
+
+
+@implementer(ITransport, IConsumer, IPushProducer)
+class H2Stream:
+    """
+    A class representing a single HTTP/2 stream.
+
+    This class works hand-in-hand with L{H2Connection}. It acts to provide an
+    implementation of L{ITransport}, L{IConsumer}, and L{IProducer} that work
+    for a single HTTP/2 connection, while tightly cleaving to the interface
+    provided by those interfaces. It does this by having a tight coupling to
+    L{H2Connection}, which allows associating many of the functions of
+    L{ITransport}, L{IConsumer}, and L{IProducer} to objects on a
+    stream-specific level.
+
+    @ivar streamID: The numerical stream ID that this object corresponds to.
+    @type streamID: L{int}
+
+    @ivar producing: Whether this stream is currently allowed to produce data
+        to its consumer.
+    @type producing: L{bool}
+
+    @ivar command: The HTTP verb used on the request.
+    @type command: L{unicode}
+
+    @ivar path: The HTTP path used on the request.
+    @type path: L{unicode}
+
+    @ivar producer: The object producing the response, if any.
+    @type producer: L{IProducer}
+
+    @ivar site: The L{twisted.web.server.Site} object this stream belongs to,
+        if any.
+    @type site: L{twisted.web.server.Site}
+
+    @ivar factory: The L{twisted.web.http.HTTPFactory} object that constructed
+        this stream's parent connection.
+    @type factory: L{twisted.web.http.HTTPFactory}
+
+    @ivar _producerProducing: Whether the producer stored in producer is
+        currently producing data.
+    @type _producerProducing: L{bool}
+
+    @ivar _inboundDataBuffer: Any data that has been received from the network
+        but has not yet been received by the consumer.
+    @type _inboundDataBuffer: A L{collections.deque} containing L{bytes}
+
+    @ivar _conn: A reference to the connection this stream belongs to.
+    @type _conn: L{H2Connection}
+
+    @ivar _request: A request object that this stream corresponds to.
+    @type _request: L{twisted.web.iweb.IRequest}
+
+    @ivar _buffer: A buffer containing data produced by the producer that could
+        not be sent on the network at this time.
+    @type _buffer: L{io.BytesIO}
+    """
+
+    # We need a transport property for t.w.h.Request, but HTTP/2 doesn't want
+    # to expose it. So we just set it to None.
+    transport = None
+
+    def __init__(self, streamID, connection, headers, requestFactory, site, factory):
+        """
+        Initialize this HTTP/2 stream.
+
+        @param streamID: The numerical stream ID that this object corresponds
+            to.
+        @type streamID: L{int}
+
+        @param connection: The HTTP/2 connection this stream belongs to.
+        @type connection: L{H2Connection}
+
+        @param headers: The HTTP/2 request headers.
+        @type headers: A L{list} of L{tuple}s of header name and header value,
+            both as L{bytes}.
+
+        @param requestFactory: A function that builds appropriate request
+            request objects.
+        @type requestFactory: A callable that returns a
+            L{twisted.web.iweb.IRequest}.
+
+        @param site: The L{twisted.web.server.Site} object this stream belongs
+            to, if any.
+        @type site: L{twisted.web.server.Site}
+
+        @param factory: The L{twisted.web.http.HTTPFactory} object that
+            constructed this stream's parent connection.
+        @type factory: L{twisted.web.http.HTTPFactory}
+        """
+
+        self.streamID = streamID
+        self.site = site
+        self.factory = factory
+        self.producing = True
+        self.command = None
+        self.path = None
+        self.producer = None
+        self._producerProducing = False
+        self._hasStreamingProducer = None
+        self._inboundDataBuffer = deque()
+        self._conn = connection
+        self._request = requestFactory(self, queued=False)
+        self._buffer = io.BytesIO()
+
+        self._convertHeaders(headers)
+
+    def _convertHeaders(self, headers):
+        """
+        This method converts the HTTP/2 header set into something that looks
+        like HTTP/1.1. In particular, it strips the 'special' headers and adds
+        a Host: header.
+
+        @param headers: The HTTP/2 header set.
+        @type headers: A L{list} of L{tuple}s of header name and header value,
+            both as L{bytes}.
+        """
+        gotLength = False
+
+        for header in headers:
+            if not header[0].startswith(b":"):
+                gotLength = _addHeaderToRequest(self._request, header) or gotLength
+            elif header[0] == b":method":
+                self.command = header[1]
+            elif header[0] == b":path":
+                self.path = header[1]
+            elif header[0] == b":authority":
+                # This is essentially the Host: header from HTTP/1.1
+                _addHeaderToRequest(self._request, (b"host", header[1]))
+
+        if not gotLength:
+            if self.command in (b"GET", b"HEAD"):
+                self._request.gotLength(0)
+            else:
+                self._request.gotLength(None)
+
+        self._request.parseCookies()
+        expectContinue = self._request.requestHeaders.getRawHeaders(b"expect")
+        if expectContinue and expectContinue[0].lower() == b"100-continue":
+            self._send100Continue()
+
+    # Methods called by the H2Connection
+    def receiveDataChunk(self, data, flowControlledLength):
+        """
+        Called when the connection has received a chunk of data from the
+        underlying transport. If the stream has been registered with a
+        consumer, and is currently able to push data, immediately passes it
+        through. Otherwise, buffers the chunk until we can start producing.
+
+        @param data: The chunk of data that was received.
+        @type data: L{bytes}
+
+        @param flowControlledLength: The total flow controlled length of this
+            chunk, which is used when we want to re-open the window. May be
+            different to C{len(data)}.
+        @type flowControlledLength: L{int}
+        """
+        if not self.producing:
+            # Buffer data.
+            self._inboundDataBuffer.append((data, flowControlledLength))
+        else:
+            self._request.handleContentChunk(data)
+            self._conn.openStreamWindow(self.streamID, flowControlledLength)
+
+    def requestComplete(self):
+        """
+        Called by the L{H2Connection} when the all data for a request has been
+        received. Currently, with the legacy L{twisted.web.http.Request}
+        object, just calls requestReceived unless the producer wants us to be
+        quiet.
+        """
+        if self.producing:
+            self._request.requestReceived(self.command, self.path, b"HTTP/2")
+        else:
+            self._inboundDataBuffer.append((_END_STREAM_SENTINEL, None))
+
+    def connectionLost(self, reason):
+        """
+        Called by the L{H2Connection} when a connection is lost or a stream is
+        reset.
+
+        @param reason: The reason the connection was lost.
+        @type reason: L{str}
+        """
+        self._request.connectionLost(reason)
+
+    def windowUpdated(self):
+        """
+        Called by the L{H2Connection} when this stream's flow control window
+        has been opened.
+        """
+        # If we don't have a producer, we have no-one to tell.
+        if not self.producer:
+            return
+
+        # If we're not blocked on flow control, we don't care.
+        if self._producerProducing:
+            return
+
+        # We check whether the stream's flow control window is actually above
+        # 0, and then, if a producer is registered and we still have space in
+        # the window, we unblock it.
+        remainingWindow = self._conn.remainingOutboundWindow(self.streamID)
+        if not remainingWindow > 0:
+            return
+
+        # We have a producer and space in the window, so that producer can
+        # start producing again!
+        self._producerProducing = True
+        self.producer.resumeProducing()
+
+    def flowControlBlocked(self):
+        """
+        Called by the L{H2Connection} when this stream's flow control window
+        has been exhausted.
+        """
+        if not self.producer:
+            return
+
+        if self._producerProducing:
+            self.producer.pauseProducing()
+            self._producerProducing = False
+
+    # Methods called by the consumer (usually an IRequest).
+    def writeHeaders(self, version, code, reason, headers):
+        """
+        Called by the consumer to write headers to the stream.
+
+        @param version: The HTTP version.
+        @type version: L{bytes}
+
+        @param code: The status code.
+        @type code: L{int}
+
+        @param reason: The reason phrase. Ignored in HTTP/2.
+        @type reason: L{bytes}
+
+        @param headers: The HTTP response headers.
+        @type headers: Any iterable of two-tuples of L{bytes}, representing header
+            names and header values.
+        """
+        self._conn.writeHeaders(version, code, reason, headers, self.streamID)
+
+    def requestDone(self, request):
+        """
+        Called by a consumer to clean up whatever permanent state is in use.
+
+        @param request: The request calling the method.
+        @type request: L{twisted.web.iweb.IRequest}
+        """
+        self._conn.endRequest(self.streamID)
+
+    def _send100Continue(self):
+        """
+        Sends a 100 Continue response, used to signal to clients that further
+        processing will be performed.
+        """
+        self._conn._send100Continue(self.streamID)
+
+    def _respondToBadRequestAndDisconnect(self):
+        """
+        This is a quick and dirty way of responding to bad requests.
+
+        As described by HTTP standard we should be patient and accept the
+        whole request from the client before sending a polite bad request
+        response, even in the case when clients send tons of data.
+
+        Unlike in the HTTP/1.1 case, this does not actually disconnect the
+        underlying transport: there's no need. This instead just sends a 400
+        response and terminates the stream.
+        """
+        self._conn._respondToBadRequestAndDisconnect(self.streamID)
+
+    # Implementation: ITransport
+    def write(self, data):
+        """
+        Write a single chunk of data into a data frame.
+
+        @param data: The data chunk to send.
+        @type data: L{bytes}
+        """
+        self._conn.writeDataToStream(self.streamID, data)
+        return
+
+    def writeSequence(self, iovec):
+        """
+        Write a sequence of chunks of data into data frames.
+
+        @param iovec: A sequence of chunks to send.
+        @type iovec: An iterable of L{bytes} chunks.
+        """
+        for chunk in iovec:
+            self.write(chunk)
+
+    def loseConnection(self):
+        """
+        Close the connection after writing all pending data.
+        """
+        self._conn.endRequest(self.streamID)
+
+    def abortConnection(self):
+        """
+        Forcefully abort the connection by sending a RstStream frame.
+        """
+        self._conn.abortRequest(self.streamID)
+
+    def getPeer(self):
+        """
+        Get information about the peer.
+        """
+        return self._conn.getPeer()
+
+    def getHost(self):
+        """
+        Similar to getPeer, but for this side of the connection.
+        """
+        return self._conn.getHost()
+
+    def isSecure(self):
+        """
+        Returns L{True} if this channel is using a secure transport.
+
+        @returns: L{True} if this channel is secure.
+        @rtype: L{bool}
+        """
+        return self._conn._isSecure()
+
+    # Implementation: IConsumer
+    def registerProducer(self, producer, streaming):
+        """
+        Register to receive data from a producer.
+
+        This sets self to be a consumer for a producer.  When this object runs
+        out of data (as when a send(2) call on a socket succeeds in moving the
+        last data from a userspace buffer into a kernelspace buffer), it will
+        ask the producer to resumeProducing().
+
+        For L{IPullProducer} providers, C{resumeProducing} will be called once
+        each time data is required.
+
+        For L{IPushProducer} providers, C{pauseProducing} will be called
+        whenever the write buffer fills up and C{resumeProducing} will only be
+        called when it empties.
+
+        @param producer: The producer to register.
+        @type producer: L{IProducer} provider
+
+        @param streaming: L{True} if C{producer} provides L{IPushProducer},
+        L{False} if C{producer} provides L{IPullProducer}.
+        @type streaming: L{bool}
+
+        @raise RuntimeError: If a producer is already registered.
+
+        @return: L{None}
+        """
+        if self.producer:
+            raise ValueError(
+                "registering producer %s before previous one (%s) was "
+                "unregistered" % (producer, self.producer)
+            )
+
+        if not streaming:
+            self.hasStreamingProducer = False
+            producer = _PullToPush(producer, self)
+            producer.startStreaming()
+        else:
+            self.hasStreamingProducer = True
+
+        self.producer = producer
+        self._producerProducing = True
+
+    def unregisterProducer(self):
+        """
+        @see: L{IConsumer.unregisterProducer}
+        """
+        # When the producer is unregistered, we're done.
+        if self.producer is not None and not self.hasStreamingProducer:
+            self.producer.stopStreaming()
+
+        self._producerProducing = False
+        self.producer = None
+        self.hasStreamingProducer = None
+
+    # Implementation: IPushProducer
+    def stopProducing(self):
+        """
+        @see: L{IProducer.stopProducing}
+        """
+        self.producing = False
+        self.abortConnection()
+
+    def pauseProducing(self):
+        """
+        @see: L{IPushProducer.pauseProducing}
+        """
+        self.producing = False
+
+    def resumeProducing(self):
+        """
+        @see: L{IPushProducer.resumeProducing}
+        """
+        self.producing = True
+        consumedLength = 0
+
+        while self.producing and self._inboundDataBuffer:
+            # Allow for pauseProducing to be called in response to a call to
+            # resumeProducing.
+            chunk, flowControlledLength = self._inboundDataBuffer.popleft()
+
+            if chunk is _END_STREAM_SENTINEL:
+                self.requestComplete()
+            else:
+                consumedLength += flowControlledLength
+                self._request.handleContentChunk(chunk)
+
+        self._conn.openStreamWindow(self.streamID, consumedLength)
+
+
+def _addHeaderToRequest(request, header):
+    """
+    Add a header tuple to a request header object.
+
+    @param request: The request to add the header tuple to.
+    @type request: L{twisted.web.http.Request}
+
+    @param header: The header tuple to add to the request.
+    @type header: A L{tuple} with two elements, the header name and header
+        value, both as L{bytes}.
+
+    @return: If the header being added was the C{Content-Length} header.
+    @rtype: L{bool}
+    """
+    requestHeaders = request.requestHeaders
+    name, value = header
+    values = requestHeaders.getRawHeaders(name)
+
+    if values is not None:
+        values.append(value)
+    else:
+        requestHeaders.setRawHeaders(name, [value])
+
+    if name == b"content-length":
+        request.gotLength(int(value))
+        return True
+
+    return False
diff --git a/contrib/python/Twisted/py3/twisted/web/_newclient.py b/contrib/python/Twisted/py3/twisted/web/_newclient.py
new file mode 100644
index 00000000000..6fd1ac21bab
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_newclient.py
@@ -0,0 +1,1727 @@
+# -*- test-case-name: twisted.web.test.test_newclient -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An U{HTTP 1.1<http://www.w3.org/Protocols/rfc2616/rfc2616.html>} client.
+
+The way to use the functionality provided by this module is to:
+
+  - Connect a L{HTTP11ClientProtocol} to an HTTP server
+  - Create a L{Request} with the appropriate data
+  - Pass the request to L{HTTP11ClientProtocol.request}
+  - The returned Deferred will fire with a L{Response} object
+  - Create a L{IProtocol} provider which can handle the response body
+  - Connect it to the response with L{Response.deliverBody}
+  - When the protocol's C{connectionLost} method is called, the response is
+    complete.  See L{Response.deliverBody} for details.
+
+Various other classes in this module support this usage:
+
+  - HTTPParser is the basic HTTP parser.  It can handle the parts of HTTP which
+    are symmetric between requests and responses.
+
+  - HTTPClientParser extends HTTPParser to handle response-specific parts of
+    HTTP.  One instance is created for each request to parse the corresponding
+    response.
+"""
+
+import re
+
+from zope.interface import implementer
+
+from twisted.internet.defer import (
+    CancelledError,
+    Deferred,
+    fail,
+    maybeDeferred,
+    succeed,
+)
+from twisted.internet.error import ConnectionDone
+from twisted.internet.interfaces import IConsumer, IPushProducer
+from twisted.internet.protocol import Protocol
+from twisted.logger import Logger
+from twisted.protocols.basic import LineReceiver
+from twisted.python.compat import networkString
+from twisted.python.components import proxyForInterface
+from twisted.python.failure import Failure
+from twisted.python.reflect import fullyQualifiedName
+from twisted.web.http import (
+    NO_CONTENT,
+    NOT_MODIFIED,
+    PotentialDataLoss,
+    _ChunkedTransferDecoder,
+    _DataLoss,
+    _IdentityTransferDecoder,
+)
+from twisted.web.http_headers import Headers
+from twisted.web.iweb import UNKNOWN_LENGTH, IClientRequest, IResponse
+
+# States HTTPParser can be in
+STATUS = "STATUS"
+HEADER = "HEADER"
+BODY = "BODY"
+DONE = "DONE"
+_moduleLog = Logger()
+
+
+class BadHeaders(Exception):
+    """
+    Headers passed to L{Request} were in some way invalid.
+    """
+
+
+class ExcessWrite(Exception):
+    """
+    The body L{IBodyProducer} for a request tried to write data after
+    indicating it had finished writing data.
+    """
+
+
+class ParseError(Exception):
+    """
+    Some received data could not be parsed.
+
+    @ivar data: The string which could not be parsed.
+    """
+
+    def __init__(self, reason, data):
+        Exception.__init__(self, reason, data)
+        self.data = data
+
+
+class BadResponseVersion(ParseError):
+    """
+    The version string in a status line was unparsable.
+    """
+
+
+class _WrapperException(Exception):
+    """
+    L{_WrapperException} is the base exception type for exceptions which
+    include one or more other exceptions as the low-level causes.
+
+    @ivar reasons: A L{list} of one or more L{Failure} instances encountered
+        during an HTTP request.  See subclass documentation for more details.
+    """
+
+    def __init__(self, reasons):
+        Exception.__init__(self, reasons)
+        self.reasons = reasons
+
+
+class RequestGenerationFailed(_WrapperException):
+    """
+    There was an error while creating the bytes which make up a request.
+
+    @ivar reasons: A C{list} of one or more L{Failure} instances giving the
+        reasons the request generation was considered to have failed.
+    """
+
+
+class RequestTransmissionFailed(_WrapperException):
+    """
+    There was an error while sending the bytes which make up a request.
+
+    @ivar reasons: A C{list} of one or more L{Failure} instances giving the
+        reasons the request transmission was considered to have failed.
+    """
+
+
+class ConnectionAborted(Exception):
+    """
+    The connection was explicitly aborted by application code.
+    """
+
+
+class WrongBodyLength(Exception):
+    """
+    An L{IBodyProducer} declared the number of bytes it was going to
+    produce (via its C{length} attribute) and then produced a different number
+    of bytes.
+    """
+
+
+class ResponseDone(Exception):
+    """
+    L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the
+    protocol passed to L{Response.deliverBody} and indicates that the entire
+    response has been delivered.
+    """
+
+
+class ResponseFailed(_WrapperException):
+    """
+    L{ResponseFailed} indicates that all of the response to a request was not
+    received for some reason.
+
+    @ivar reasons: A C{list} of one or more L{Failure} instances giving the
+        reasons the response was considered to have failed.
+
+    @ivar response: If specified, the L{Response} received from the server (and
+        in particular the status code and the headers).
+    """
+
+    def __init__(self, reasons, response=None):
+        _WrapperException.__init__(self, reasons)
+        self.response = response
+
+
+class ResponseNeverReceived(ResponseFailed):
+    """
+    A L{ResponseFailed} that knows no response bytes at all have been received.
+    """
+
+
+class RequestNotSent(Exception):
+    """
+    L{RequestNotSent} indicates that an attempt was made to issue a request but
+    for reasons unrelated to the details of the request itself, the request
+    could not be sent.  For example, this may indicate that an attempt was made
+    to send a request using a protocol which is no longer connected to a
+    server.
+    """
+
+
+def _callAppFunction(function):
+    """
+    Call C{function}.  If it raises an exception, log it with a minimal
+    description of the source.
+
+    @return: L{None}
+    """
+    try:
+        function()
+    except BaseException:
+        _moduleLog.failure(
+            "Unexpected exception from {name}", name=fullyQualifiedName(function)
+        )
+
+
+class HTTPParser(LineReceiver):
+    """
+    L{HTTPParser} handles the parsing side of HTTP processing. With a suitable
+    subclass, it can parse either the client side or the server side of the
+    connection.
+
+    @ivar headers: All of the non-connection control message headers yet
+        received.
+
+    @ivar state: State indicator for the response parsing state machine.  One
+        of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}.
+
+    @ivar _partialHeader: L{None} or a C{list} of the lines of a multiline
+        header while that header is being received.
+    """
+
+    # NOTE: According to HTTP spec, we're supposed to eat the
+    # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that
+    # doesn't sound like a good idea to me, because it makes it impossible to
+    # have a non-authenticating transparent proxy in front of an authenticating
+    # proxy. An authenticating proxy can eat them itself. -jknight
+    #
+    # Further, quoting
+    # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html
+    # regarding the 'Proxy-Connection' header:
+    #
+    #    The Proxy-Connection: header is a mistake in how some web browsers
+    #    use HTTP. Its name is the result of a false analogy. It is not a
+    #    standard part of the protocol. There is a different standard
+    #    protocol mechanism for doing what it does. And its existence
+    #    imposes a requirement upon HTTP servers such that no proxy HTTP
+    #    server can be standards-conforming in practice.
+    #
+    # -exarkun
+
+    # Some servers (like http://news.ycombinator.com/) return status lines and
+    # HTTP headers delimited by \n instead of \r\n.
+    delimiter = b"\n"
+
+    CONNECTION_CONTROL_HEADERS = {
+        b"content-length",
+        b"connection",
+        b"keep-alive",
+        b"te",
+        b"trailers",
+        b"transfer-encoding",
+        b"upgrade",
+        b"proxy-connection",
+    }
+
+    def connectionMade(self):
+        self.headers = Headers()
+        self.connHeaders = Headers()
+        self.state = STATUS
+        self._partialHeader = None
+
+    def switchToBodyMode(self, decoder):
+        """
+        Switch to body parsing mode - interpret any more bytes delivered as
+        part of the message body and deliver them to the given decoder.
+        """
+        if self.state == BODY:
+            raise RuntimeError("already in body mode")
+
+        self.bodyDecoder = decoder
+        self.state = BODY
+        self.setRawMode()
+
+    def lineReceived(self, line):
+        """
+        Handle one line from a response.
+        """
+        # Handle the normal CR LF case.
+        if line[-1:] == b"\r":
+            line = line[:-1]
+
+        if self.state == STATUS:
+            self.statusReceived(line)
+            self.state = HEADER
+        elif self.state == HEADER:
+            if not line or line[0] not in b" \t":
+                if self._partialHeader is not None:
+                    header = b"".join(self._partialHeader)
+                    name, value = header.split(b":", 1)
+                    value = value.strip()
+                    self.headerReceived(name, value)
+                if not line:
+                    # Empty line means the header section is over.
+                    self.allHeadersReceived()
+                else:
+                    # Line not beginning with LWS is another header.
+                    self._partialHeader = [line]
+            else:
+                # A line beginning with LWS is a continuation of a header
+                # begun on a previous line.
+                self._partialHeader.append(line)
+
+    def rawDataReceived(self, data):
+        """
+        Pass data from the message body to the body decoder object.
+        """
+        self.bodyDecoder.dataReceived(data)
+
+    def isConnectionControlHeader(self, name):
+        """
+        Return C{True} if the given lower-cased name is the name of a
+        connection control header (rather than an entity header).
+
+        According to RFC 2616, section 14.10, the tokens in the Connection
+        header are probably relevant here.  However, I am not sure what the
+        practical consequences of either implementing or ignoring that are.
+        So I leave it unimplemented for the time being.
+        """
+        return name in self.CONNECTION_CONTROL_HEADERS
+
+    def statusReceived(self, status):
+        """
+        Callback invoked whenever the first line of a new message is received.
+        Override this.
+
+        @param status: The first line of an HTTP request or response message
+            without trailing I{CR LF}.
+        @type status: C{bytes}
+        """
+
+    def headerReceived(self, name, value):
+        """
+        Store the given header in C{self.headers}.
+        """
+        name = name.lower()
+        if self.isConnectionControlHeader(name):
+            headers = self.connHeaders
+        else:
+            headers = self.headers
+        headers.addRawHeader(name, value)
+
+    def allHeadersReceived(self):
+        """
+        Callback invoked after the last header is passed to C{headerReceived}.
+        Override this to change to the C{BODY} or C{DONE} state.
+        """
+        self.switchToBodyMode(None)
+
+
+class HTTPClientParser(HTTPParser):
+    """
+    An HTTP parser which only handles HTTP responses.
+
+    @ivar request: The request with which the expected response is associated.
+    @type request: L{Request}
+
+    @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a
+        body.
+
+    @ivar finisher: A callable to invoke when this response is fully parsed.
+
+    @ivar _responseDeferred: A L{Deferred} which will be called back with the
+        response when all headers in the response have been received.
+        Thereafter, L{None}.
+
+    @ivar _everReceivedData: C{True} if any bytes have been received.
+    """
+
+    NO_BODY_CODES = {NO_CONTENT, NOT_MODIFIED}
+
+    _transferDecoders = {
+        b"chunked": _ChunkedTransferDecoder,
+    }
+
+    bodyDecoder = None
+    _log = Logger()
+
+    def __init__(self, request, finisher):
+        self.request = request
+        self.finisher = finisher
+        self._responseDeferred = Deferred()
+        self._everReceivedData = False
+
+    def dataReceived(self, data):
+        """
+        Override so that we know if any response has been received.
+        """
+        self._everReceivedData = True
+        HTTPParser.dataReceived(self, data)
+
+    def parseVersion(self, strversion):
+        """
+        Parse version strings of the form Protocol '/' Major '.' Minor. E.g.
+        b'HTTP/1.1'.  Returns (protocol, major, minor).  Will raise ValueError
+        on bad syntax.
+        """
+        try:
+            proto, strnumber = strversion.split(b"/")
+            major, minor = strnumber.split(b".")
+            major, minor = int(major), int(minor)
+        except ValueError as e:
+            raise BadResponseVersion(str(e), strversion)
+        if major < 0 or minor < 0:
+            raise BadResponseVersion("version may not be negative", strversion)
+        return (proto, major, minor)
+
+    def statusReceived(self, status):
+        """
+        Parse the status line into its components and create a response object
+        to keep track of this response's state.
+        """
+        parts = status.split(b" ", 2)
+        if len(parts) == 2:
+            # Some broken servers omit the required `phrase` portion of
+            # `status-line`.  One such server identified as
+            # "cloudflare-nginx".  Others fail to identify themselves
+            # entirely.  Fill in an empty phrase for such cases.
+            version, codeBytes = parts
+            phrase = b""
+        elif len(parts) == 3:
+            version, codeBytes, phrase = parts
+        else:
+            raise ParseError("wrong number of parts", status)
+
+        try:
+            statusCode = int(codeBytes)
+        except ValueError:
+            raise ParseError("non-integer status code", status)
+
+        self.response = Response._construct(
+            self.parseVersion(version),
+            statusCode,
+            phrase,
+            self.headers,
+            self.transport,
+            self.request,
+        )
+
+    def _finished(self, rest):
+        """
+        Called to indicate that an entire response has been received.  No more
+        bytes will be interpreted by this L{HTTPClientParser}.  Extra bytes are
+        passed up and the state of this L{HTTPClientParser} is set to I{DONE}.
+
+        @param rest: A C{bytes} giving any extra bytes delivered to this
+            L{HTTPClientParser} which are not part of the response being
+            parsed.
+        """
+        self.state = DONE
+        self.finisher(rest)
+
+    def isConnectionControlHeader(self, name):
+        """
+        Content-Length in the response to a HEAD request is an entity header,
+        not a connection control header.
+        """
+        if self.request.method == b"HEAD" and name == b"content-length":
+            return False
+        return HTTPParser.isConnectionControlHeader(self, name)
+
+    def allHeadersReceived(self):
+        """
+        Figure out how long the response body is going to be by examining
+        headers and stuff.
+        """
+        if 100 <= self.response.code < 200:
+            # RFC 7231 Section 6.2 says that if we receive a 1XX status code
+            # and aren't expecting it, we MAY ignore it. That's what we're
+            # going to do. We reset the parser here, but we leave
+            # _everReceivedData in its True state because we have, in fact,
+            # received data.
+            self._log.info(
+                "Ignoring unexpected {code} response", code=self.response.code
+            )
+            self.connectionMade()
+            del self.response
+            return
+
+        if self.response.code in self.NO_BODY_CODES or self.request.method == b"HEAD":
+            self.response.length = 0
+            # The order of the next two lines might be of interest when adding
+            # support for pipelining.
+            self._finished(self.clearLineBuffer())
+            self.response._bodyDataFinished()
+        else:
+            transferEncodingHeaders = self.connHeaders.getRawHeaders(
+                b"transfer-encoding"
+            )
+            if transferEncodingHeaders:
+                # This could be a KeyError.  However, that would mean we do not
+                # know how to decode the response body, so failing the request
+                # is as good a behavior as any.  Perhaps someday we will want
+                # to normalize/document/test this specifically, but failing
+                # seems fine to me for now.
+                transferDecoder = self._transferDecoders[
+                    transferEncodingHeaders[0].lower()
+                ]
+
+                # If anyone ever invents a transfer encoding other than
+                # chunked (yea right), and that transfer encoding can predict
+                # the length of the response body, it might be sensible to
+                # allow the transfer decoder to set the response object's
+                # length attribute.
+            else:
+                contentLengthHeaders = self.connHeaders.getRawHeaders(b"content-length")
+                if contentLengthHeaders is None:
+                    contentLength = None
+                elif len(contentLengthHeaders) == 1:
+                    contentLength = int(contentLengthHeaders[0])
+                    self.response.length = contentLength
+                else:
+                    # "HTTP Message Splitting" or "HTTP Response Smuggling"
+                    # potentially happening.  Or it's just a buggy server.
+                    raise ValueError(
+                        "Too many Content-Length headers; " "response is invalid"
+                    )
+
+                if contentLength == 0:
+                    self._finished(self.clearLineBuffer())
+                    transferDecoder = None
+                else:
+                    transferDecoder = lambda x, y: _IdentityTransferDecoder(
+                        contentLength, x, y
+                    )
+
+            if transferDecoder is None:
+                self.response._bodyDataFinished()
+            else:
+                # Make sure as little data as possible from the response body
+                # gets delivered to the response object until the response
+                # object actually indicates it is ready to handle bytes
+                # (probably because an application gave it a way to interpret
+                # them).
+                self.transport.pauseProducing()
+                self.switchToBodyMode(
+                    transferDecoder(self.response._bodyDataReceived, self._finished)
+                )
+
+        # This must be last.  If it were first, then application code might
+        # change some state (for example, registering a protocol to receive the
+        # response body).  Then the pauseProducing above would be wrong since
+        # the response is ready for bytes and nothing else would ever resume
+        # the transport.
+        self._responseDeferred.callback(self.response)
+        del self._responseDeferred
+
+    def connectionLost(self, reason):
+        if self.bodyDecoder is not None:
+            try:
+                try:
+                    self.bodyDecoder.noMoreData()
+                except PotentialDataLoss:
+                    self.response._bodyDataFinished(Failure())
+                except _DataLoss:
+                    self.response._bodyDataFinished(
+                        Failure(ResponseFailed([reason, Failure()], self.response))
+                    )
+                else:
+                    self.response._bodyDataFinished()
+            except BaseException:
+                # Handle exceptions from both the except suites and the else
+                # suite.  Those functions really shouldn't raise exceptions,
+                # but maybe there's some buggy application code somewhere
+                # making things difficult.
+                self._log.failure("")
+        elif self.state != DONE:
+            if self._everReceivedData:
+                exceptionClass = ResponseFailed
+            else:
+                exceptionClass = ResponseNeverReceived
+            self._responseDeferred.errback(Failure(exceptionClass([reason])))
+            del self._responseDeferred
+
+
+_VALID_METHOD = re.compile(
+    rb"\A[%s]+\Z"
+    % (
+        bytes().join(
+            (
+                b"!",
+                b"#",
+                b"$",
+                b"%",
+                b"&",
+                b"'",
+                b"*",
+                b"+",
+                b"-",
+                b".",
+                b"^",
+                b"_",
+                b"`",
+                b"|",
+                b"~",
+                b"\x30-\x39",
+                b"\x41-\x5a",
+                b"\x61-\x7A",
+            ),
+        ),
+    ),
+)
+
+
+def _ensureValidMethod(method):
+    """
+    An HTTP method is an HTTP token, which consists of any visible
+    ASCII character that is not a delimiter (i.e. one of
+    C{"(),/:;<=>?@[\\]{}}.)
+
+    @param method: the method to check
+    @type method: L{bytes}
+
+    @return: the method if it is valid
+    @rtype: L{bytes}
+
+    @raise ValueError: if the method is not valid
+
+    @see: U{https://tools.ietf.org/html/rfc7230#section-3.1.1},
+        U{https://tools.ietf.org/html/rfc7230#section-3.2.6},
+        U{https://tools.ietf.org/html/rfc5234#appendix-B.1}
+    """
+    if _VALID_METHOD.match(method):
+        return method
+    raise ValueError(f"Invalid method {method!r}")
+
+
+_VALID_URI = re.compile(rb"\A[\x21-\x7e]+\Z")
+
+
+def _ensureValidURI(uri):
+    """
+    A valid URI cannot contain control characters (i.e., characters
+    between 0-32, inclusive and 127) or non-ASCII characters (i.e.,
+    characters with values between 128-255, inclusive).
+
+    @param uri: the URI to check
+    @type uri: L{bytes}
+
+    @return: the URI if it is valid
+    @rtype: L{bytes}
+
+    @raise ValueError: if the URI is not valid
+
+    @see: U{https://tools.ietf.org/html/rfc3986#section-3.3},
+        U{https://tools.ietf.org/html/rfc3986#appendix-A},
+        U{https://tools.ietf.org/html/rfc5234#appendix-B.1}
+    """
+    if _VALID_URI.match(uri):
+        return uri
+    raise ValueError(f"Invalid URI {uri!r}")
+
+
+@implementer(IClientRequest)
+class Request:
+    """
+    A L{Request} instance describes an HTTP request to be sent to an HTTP
+    server.
+
+    @ivar method: See L{__init__}.
+    @ivar uri: See L{__init__}.
+    @ivar headers: See L{__init__}.
+    @ivar bodyProducer: See L{__init__}.
+    @ivar persistent: See L{__init__}.
+
+    @ivar _parsedURI: Parsed I{URI} for the request, or L{None}.
+    @type _parsedURI: L{twisted.web.client.URI} or L{None}
+    """
+
+    _log = Logger()
+
+    def __init__(self, method, uri, headers, bodyProducer, persistent=False):
+        """
+        @param method: The HTTP method for this request, ex: b'GET', b'HEAD',
+            b'POST', etc.
+        @type method: L{bytes}
+
+        @param uri: The relative URI of the resource to request.  For example,
+            C{b'/foo/bar?baz=quux'}.
+        @type uri: L{bytes}
+
+        @param headers: Headers to be sent to the server.  It is important to
+            note that this object does not create any implicit headers.  So it
+            is up to the HTTP Client to add required headers such as 'Host'.
+        @type headers: L{twisted.web.http_headers.Headers}
+
+        @param bodyProducer: L{None} or an L{IBodyProducer} provider which
+            produces the content body to send to the remote HTTP server.
+
+        @param persistent: Set to C{True} when you use HTTP persistent
+            connection, defaults to C{False}.
+        @type persistent: L{bool}
+        """
+        self.method = _ensureValidMethod(method)
+        self.uri = _ensureValidURI(uri)
+        self.headers = headers
+        self.bodyProducer = bodyProducer
+        self.persistent = persistent
+        self._parsedURI = None
+
+    @classmethod
+    def _construct(
+        cls, method, uri, headers, bodyProducer, persistent=False, parsedURI=None
+    ):
+        """
+        Private constructor.
+
+        @param method: See L{__init__}.
+        @param uri: See L{__init__}.
+        @param headers: See L{__init__}.
+        @param bodyProducer: See L{__init__}.
+        @param persistent: See L{__init__}.
+        @param parsedURI: See L{Request._parsedURI}.
+
+        @return: L{Request} instance.
+        """
+        request = cls(method, uri, headers, bodyProducer, persistent)
+        request._parsedURI = parsedURI
+        return request
+
+    @property
+    def absoluteURI(self):
+        """
+        The absolute URI of the request as C{bytes}, or L{None} if the
+        absolute URI cannot be determined.
+        """
+        return getattr(self._parsedURI, "toBytes", lambda: None)()
+
+    def _writeHeaders(self, transport, TEorCL):
+        hosts = self.headers.getRawHeaders(b"host", ())
+        if len(hosts) != 1:
+            raise BadHeaders("Exactly one Host header required")
+
+        # In the future, having the protocol version be a parameter to this
+        # method would probably be good.  It would be nice if this method
+        # weren't limited to issuing HTTP/1.1 requests.
+        requestLines = []
+        requestLines.append(
+            b" ".join(
+                [
+                    _ensureValidMethod(self.method),
+                    _ensureValidURI(self.uri),
+                    b"HTTP/1.1\r\n",
+                ]
+            ),
+        )
+        if not self.persistent:
+            requestLines.append(b"Connection: close\r\n")
+        if TEorCL is not None:
+            requestLines.append(TEorCL)
+        for name, values in self.headers.getAllRawHeaders():
+            requestLines.extend([name + b": " + v + b"\r\n" for v in values])
+        requestLines.append(b"\r\n")
+        transport.writeSequence(requestLines)
+
+    def _writeToBodyProducerChunked(self, transport):
+        """
+        Write this request to the given transport using chunked
+        transfer-encoding to frame the body.
+
+        @param transport: See L{writeTo}.
+        @return: See L{writeTo}.
+        """
+        self._writeHeaders(transport, b"Transfer-Encoding: chunked\r\n")
+        encoder = ChunkedEncoder(transport)
+        encoder.registerProducer(self.bodyProducer, True)
+        d = self.bodyProducer.startProducing(encoder)
+
+        def cbProduced(ignored):
+            encoder.unregisterProducer()
+
+        def ebProduced(err):
+            encoder._allowNoMoreWrites()
+            # Don't call the encoder's unregisterProducer because it will write
+            # a zero-length chunk.  This would indicate to the server that the
+            # request body is complete.  There was an error, though, so we
+            # don't want to do that.
+            transport.unregisterProducer()
+            return err
+
+        d.addCallbacks(cbProduced, ebProduced)
+        return d
+
+    def _writeToBodyProducerContentLength(self, transport):
+        """
+        Write this request to the given transport using content-length to frame
+        the body.
+
+        @param transport: See L{writeTo}.
+        @return: See L{writeTo}.
+        """
+        self._writeHeaders(
+            transport,
+            networkString("Content-Length: %d\r\n" % (self.bodyProducer.length,)),
+        )
+
+        # This Deferred is used to signal an error in the data written to the
+        # encoder below.  It can only errback and it will only do so before too
+        # many bytes have been written to the encoder and before the producer
+        # Deferred fires.
+        finishedConsuming = Deferred()
+
+        # This makes sure the producer writes the correct number of bytes for
+        # the request body.
+        encoder = LengthEnforcingConsumer(
+            self.bodyProducer, transport, finishedConsuming
+        )
+
+        transport.registerProducer(self.bodyProducer, True)
+
+        finishedProducing = self.bodyProducer.startProducing(encoder)
+
+        def combine(consuming, producing):
+            # This Deferred is returned and will be fired when the first of
+            # consuming or producing fires. If it's cancelled, forward that
+            # cancellation to the producer.
+            def cancelConsuming(ign):
+                finishedProducing.cancel()
+
+            ultimate = Deferred(cancelConsuming)
+
+            # Keep track of what has happened so far.  This initially
+            # contains None, then an integer uniquely identifying what
+            # sequence of events happened.  See the callbacks and errbacks
+            # defined below for the meaning of each value.
+            state = [None]
+
+            def ebConsuming(err):
+                if state == [None]:
+                    # The consuming Deferred failed first.  This means the
+                    # overall writeTo Deferred is going to errback now.  The
+                    # producing Deferred should not fire later (because the
+                    # consumer should have called stopProducing on the
+                    # producer), but if it does, a callback will be ignored
+                    # and an errback will be logged.
+                    state[0] = 1
+                    ultimate.errback(err)
+                else:
+                    # The consuming Deferred errbacked after the producing
+                    # Deferred fired.  This really shouldn't ever happen.
+                    # If it does, I goofed.  Log the error anyway, just so
+                    # there's a chance someone might notice and complain.
+                    self._log.failure(
+                        "Buggy state machine in {request}/[{state}]: "
+                        "ebConsuming called",
+                        failure=err,
+                        request=repr(self),
+                        state=state[0],
+                    )
+
+            def cbProducing(result):
+                if state == [None]:
+                    # The producing Deferred succeeded first.  Nothing will
+                    # ever happen to the consuming Deferred.  Tell the
+                    # encoder we're done so it can check what the producer
+                    # wrote and make sure it was right.
+                    state[0] = 2
+                    try:
+                        encoder._noMoreWritesExpected()
+                    except BaseException:
+                        # Fail the overall writeTo Deferred - something the
+                        # producer did was wrong.
+                        ultimate.errback()
+                    else:
+                        # Success - succeed the overall writeTo Deferred.
+                        ultimate.callback(None)
+                # Otherwise, the consuming Deferred already errbacked.  The
+                # producing Deferred wasn't supposed to fire, but it did
+                # anyway.  It's buggy, but there's not really anything to be
+                # done about it.  Just ignore this result.
+
+            def ebProducing(err):
+                if state == [None]:
+                    # The producing Deferred failed first.  This means the
+                    # overall writeTo Deferred is going to errback now.
+                    # Tell the encoder that we're done so it knows to reject
+                    # further writes from the producer (which should not
+                    # happen, but the producer may be buggy).
+                    state[0] = 3
+                    encoder._allowNoMoreWrites()
+                    ultimate.errback(err)
+                else:
+                    # The producing Deferred failed after the consuming
+                    # Deferred failed.  It shouldn't have, so it's buggy.
+                    # Log the exception in case anyone who can fix the code
+                    # is watching.
+                    self._log.failure("Producer is buggy", failure=err)
+
+            consuming.addErrback(ebConsuming)
+            producing.addCallbacks(cbProducing, ebProducing)
+
+            return ultimate
+
+        d = combine(finishedConsuming, finishedProducing)
+
+        def f(passthrough):
+            # Regardless of what happens with the overall Deferred, once it
+            # fires, the producer registered way up above the definition of
+            # combine should be unregistered.
+            transport.unregisterProducer()
+            return passthrough
+
+        d.addBoth(f)
+        return d
+
+    def _writeToEmptyBodyContentLength(self, transport):
+        """
+        Write this request to the given transport using content-length to frame
+        the (empty) body.
+
+        @param transport: See L{writeTo}.
+        @return: See L{writeTo}.
+        """
+        self._writeHeaders(transport, b"Content-Length: 0\r\n")
+        return succeed(None)
+
+    def writeTo(self, transport):
+        """
+        Format this L{Request} as an HTTP/1.1 request and write it to the given
+        transport.  If bodyProducer is not None, it will be associated with an
+        L{IConsumer}.
+
+        @param transport: The transport to which to write.
+        @type transport: L{twisted.internet.interfaces.ITransport} provider
+
+        @return: A L{Deferred} which fires with L{None} when the request has
+            been completely written to the transport or with a L{Failure} if
+            there is any problem generating the request bytes.
+        """
+        if self.bodyProducer is None:
+            # If the method semantics anticipate a body, include a
+            # Content-Length even if it is 0.
+            # https://tools.ietf.org/html/rfc7230#section-3.3.2
+            if self.method in (b"PUT", b"POST"):
+                self._writeToEmptyBodyContentLength(transport)
+            else:
+                self._writeHeaders(transport, None)
+        elif self.bodyProducer.length is UNKNOWN_LENGTH:
+            return self._writeToBodyProducerChunked(transport)
+        else:
+            return self._writeToBodyProducerContentLength(transport)
+
+    def stopWriting(self):
+        """
+        Stop writing this request to the transport.  This can only be called
+        after C{writeTo} and before the L{Deferred} returned by C{writeTo}
+        fires.  It should cancel any asynchronous task started by C{writeTo}.
+        The L{Deferred} returned by C{writeTo} need not be fired if this method
+        is called.
+        """
+        # If bodyProducer is None, then the Deferred returned by writeTo has
+        # fired already and this method cannot be called.
+        _callAppFunction(self.bodyProducer.stopProducing)
+
+
+class LengthEnforcingConsumer:
+    """
+    An L{IConsumer} proxy which enforces an exact length requirement on the
+    total data written to it.
+
+    @ivar _length: The number of bytes remaining to be written.
+
+    @ivar _producer: The L{IBodyProducer} which is writing to this
+        consumer.
+
+    @ivar _consumer: The consumer to which at most C{_length} bytes will be
+        forwarded.
+
+    @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too
+        many bytes are written to this consumer.
+    """
+
+    def __init__(self, producer, consumer, finished):
+        self._length = producer.length
+        self._producer = producer
+        self._consumer = consumer
+        self._finished = finished
+
+    def _allowNoMoreWrites(self):
+        """
+        Indicate that no additional writes are allowed.  Attempts to write
+        after calling this method will be met with an exception.
+        """
+        self._finished = None
+
+    def write(self, bytes):
+        """
+        Write C{bytes} to the underlying consumer unless
+        C{_noMoreWritesExpected} has been called or there are/have been too
+        many bytes.
+        """
+        if self._finished is None:
+            # No writes are supposed to happen any more.  Try to convince the
+            # calling code to stop calling this method by calling its
+            # stopProducing method and then throwing an exception at it.  This
+            # exception isn't documented as part of the API because you're
+            # never supposed to expect it: only buggy code will ever receive
+            # it.
+            self._producer.stopProducing()
+            raise ExcessWrite()
+
+        if len(bytes) <= self._length:
+            self._length -= len(bytes)
+            self._consumer.write(bytes)
+        else:
+            # No synchronous exception is raised in *this* error path because
+            # we still have _finished which we can use to report the error to a
+            # better place than the direct caller of this method (some
+            # arbitrary application code).
+            _callAppFunction(self._producer.stopProducing)
+            self._finished.errback(WrongBodyLength("too many bytes written"))
+            self._allowNoMoreWrites()
+
+    def _noMoreWritesExpected(self):
+        """
+        Called to indicate no more bytes will be written to this consumer.
+        Check to see that the correct number have been written.
+
+        @raise WrongBodyLength: If not enough bytes have been written.
+        """
+        if self._finished is not None:
+            self._allowNoMoreWrites()
+            if self._length:
+                raise WrongBodyLength("too few bytes written")
+
+
+def makeStatefulDispatcher(name, template):
+    """
+    Given a I{dispatch} name and a function, return a function which can be
+    used as a method and which, when called, will call another method defined
+    on the instance and return the result.  The other method which is called is
+    determined by the value of the C{_state} attribute of the instance.
+
+    @param name: A string which is used to construct the name of the subsidiary
+        method to invoke.  The subsidiary method is named like C{'_%s_%s' %
+        (name, _state)}.
+
+    @param template: A function object which is used to give the returned
+        function a docstring.
+
+    @return: The dispatcher function.
+    """
+
+    def dispatcher(self, *args, **kwargs):
+        func = getattr(self, "_" + name + "_" + self._state, None)
+        if func is None:
+            raise RuntimeError(f"{self!r} has no {name} method in state {self._state}")
+        return func(*args, **kwargs)
+
+    dispatcher.__doc__ = template.__doc__
+    return dispatcher
+
+
+# This proxy class is used only in the private constructor of the Response
+# class below, in order to prevent users relying on any property of the
+# concrete request object: they can only use what is provided by
+# IClientRequest.
+_ClientRequestProxy = proxyForInterface(IClientRequest)
+
+
+@implementer(IResponse)
+class Response:
+    """
+    A L{Response} instance describes an HTTP response received from an HTTP
+    server.
+
+    L{Response} should not be subclassed or instantiated.
+
+    @ivar _transport: See L{__init__}.
+
+    @ivar _bodyProtocol: The L{IProtocol} provider to which the body is
+        delivered.  L{None} before one has been registered with
+        C{deliverBody}.
+
+    @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived}
+        before C{deliverBody} is called.  L{None} afterwards.
+
+    @ivar _state: Indicates what state this L{Response} instance is in,
+        particularly with respect to delivering bytes from the response body
+        to an application-supplied protocol object.  This may be one of
+        C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'},
+        with the following meanings:
+
+          - INITIAL: This is the state L{Response} objects start in.  No
+            protocol has yet been provided and the underlying transport may
+            still have bytes to deliver to it.
+
+          - DEFERRED_CLOSE: If the underlying transport indicates all bytes
+            have been delivered but no application-provided protocol is yet
+            available, the L{Response} moves to this state.  Data is
+            buffered and waiting for a protocol to be delivered to.
+
+          - CONNECTED: If a protocol is provided when the state is INITIAL,
+            the L{Response} moves to this state.  Any buffered data is
+            delivered and any data which arrives from the transport
+            subsequently is given directly to the protocol.
+
+          - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state,
+            the L{Response} moves to this state after delivering all
+            buffered data to the protocol.  Otherwise, if the L{Response} is
+            in the CONNECTED state, if the transport indicates there is no
+            more data, the L{Response} moves to this state.  Nothing else
+            can happen once the L{Response} is in this state.
+    @type _state: C{str}
+    """
+
+    length = UNKNOWN_LENGTH
+
+    _bodyProtocol = None
+    _bodyFinished = False
+
+    def __init__(self, version, code, phrase, headers, _transport):
+        """
+        @param version: HTTP version components protocol, major, minor. E.g.
+            C{(b'HTTP', 1, 1)} to mean C{b'HTTP/1.1'}.
+
+        @param code: HTTP status code.
+        @type code: L{int}
+
+        @param phrase: HTTP reason phrase, intended to give a short description
+            of the HTTP status code.
+
+        @param headers: HTTP response headers.
+        @type headers: L{twisted.web.http_headers.Headers}
+
+        @param _transport: The transport which is delivering this response.
+        """
+        self.version = version
+        self.code = code
+        self.phrase = phrase
+        self.headers = headers
+        self._transport = _transport
+        self._bodyBuffer = []
+        self._state = "INITIAL"
+        self.request = None
+        self.previousResponse = None
+
+    @classmethod
+    def _construct(cls, version, code, phrase, headers, _transport, request):
+        """
+        Private constructor.
+
+        @param version: See L{__init__}.
+        @param code: See L{__init__}.
+        @param phrase: See L{__init__}.
+        @param headers: See L{__init__}.
+        @param _transport: See L{__init__}.
+        @param request: See L{IResponse.request}.
+
+        @return: L{Response} instance.
+        """
+        response = Response(version, code, phrase, headers, _transport)
+        response.request = _ClientRequestProxy(request)
+        return response
+
+    def setPreviousResponse(self, previousResponse):
+        self.previousResponse = previousResponse
+
+    def deliverBody(self, protocol):
+        """
+        Dispatch the given L{IProtocol} depending of the current state of the
+        response.
+        """
+
+    deliverBody = makeStatefulDispatcher("deliverBody", deliverBody)
+
+    def _deliverBody_INITIAL(self, protocol):
+        """
+        Deliver any buffered data to C{protocol} and prepare to deliver any
+        future data to it.  Move to the C{'CONNECTED'} state.
+        """
+        protocol.makeConnection(self._transport)
+        self._bodyProtocol = protocol
+        for data in self._bodyBuffer:
+            self._bodyProtocol.dataReceived(data)
+        self._bodyBuffer = None
+
+        self._state = "CONNECTED"
+
+        # Now that there's a protocol to consume the body, resume the
+        # transport.  It was previously paused by HTTPClientParser to avoid
+        # reading too much data before it could be handled. We need to do this
+        # after we transition our state as it may recursively lead to more data
+        # being delivered, or even the body completing.
+        self._transport.resumeProducing()
+
+    def _deliverBody_CONNECTED(self, protocol):
+        """
+        It is invalid to attempt to deliver data to a protocol when it is
+        already being delivered to another protocol.
+        """
+        raise RuntimeError(
+            "Response already has protocol %r, cannot deliverBody "
+            "again" % (self._bodyProtocol,)
+        )
+
+    def _deliverBody_DEFERRED_CLOSE(self, protocol):
+        """
+        Deliver any buffered data to C{protocol} and then disconnect the
+        protocol.  Move to the C{'FINISHED'} state.
+        """
+        # Unlike _deliverBody_INITIAL, there is no need to resume the
+        # transport here because all of the response data has been received
+        # already.  Some higher level code may want to resume the transport if
+        # that code expects further data to be received over it.
+
+        protocol.makeConnection(self._transport)
+
+        for data in self._bodyBuffer:
+            protocol.dataReceived(data)
+        self._bodyBuffer = None
+        protocol.connectionLost(self._reason)
+        self._state = "FINISHED"
+
+    def _deliverBody_FINISHED(self, protocol):
+        """
+        It is invalid to attempt to deliver data to a protocol after the
+        response body has been delivered to another protocol.
+        """
+        raise RuntimeError("Response already finished, cannot deliverBody now.")
+
+    def _bodyDataReceived(self, data):
+        """
+        Called by HTTPClientParser with chunks of data from the response body.
+        They will be buffered or delivered to the protocol passed to
+        deliverBody.
+        """
+
+    _bodyDataReceived = makeStatefulDispatcher("bodyDataReceived", _bodyDataReceived)
+
+    def _bodyDataReceived_INITIAL(self, data):
+        """
+        Buffer any data received for later delivery to a protocol passed to
+        C{deliverBody}.
+
+        Little or no data should be buffered by this method, since the
+        transport has been paused and will not be resumed until a protocol
+        is supplied.
+        """
+        self._bodyBuffer.append(data)
+
+    def _bodyDataReceived_CONNECTED(self, data):
+        """
+        Deliver any data received to the protocol to which this L{Response}
+        is connected.
+        """
+        self._bodyProtocol.dataReceived(data)
+
+    def _bodyDataReceived_DEFERRED_CLOSE(self, data):
+        """
+        It is invalid for data to be delivered after it has been indicated
+        that the response body has been completely delivered.
+        """
+        raise RuntimeError("Cannot receive body data after _bodyDataFinished")
+
+    def _bodyDataReceived_FINISHED(self, data):
+        """
+        It is invalid for data to be delivered after the response body has
+        been delivered to a protocol.
+        """
+        raise RuntimeError("Cannot receive body data after " "protocol disconnected")
+
+    def _bodyDataFinished(self, reason=None):
+        """
+        Called by HTTPClientParser when no more body data is available.  If the
+        optional reason is supplied, this indicates a problem or potential
+        problem receiving all of the response body.
+        """
+
+    _bodyDataFinished = makeStatefulDispatcher("bodyDataFinished", _bodyDataFinished)
+
+    def _bodyDataFinished_INITIAL(self, reason=None):
+        """
+        Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to
+        which to deliver the response body.
+        """
+        self._state = "DEFERRED_CLOSE"
+        if reason is None:
+            reason = Failure(ResponseDone("Response body fully received"))
+        self._reason = reason
+
+    def _bodyDataFinished_CONNECTED(self, reason=None):
+        """
+        Disconnect the protocol and move to the C{'FINISHED'} state.
+        """
+        if reason is None:
+            reason = Failure(ResponseDone("Response body fully received"))
+        self._bodyProtocol.connectionLost(reason)
+        self._bodyProtocol = None
+        self._state = "FINISHED"
+
+    def _bodyDataFinished_DEFERRED_CLOSE(self):
+        """
+        It is invalid to attempt to notify the L{Response} of the end of the
+        response body data more than once.
+        """
+        raise RuntimeError("Cannot finish body data more than once")
+
+    def _bodyDataFinished_FINISHED(self):
+        """
+        It is invalid to attempt to notify the L{Response} of the end of the
+        response body data more than once.
+        """
+        raise RuntimeError("Cannot finish body data after " "protocol disconnected")
+
+
+@implementer(IConsumer)
+class ChunkedEncoder:
+    """
+    Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol}
+    for streaming request bodies to the server.
+    """
+
+    def __init__(self, transport):
+        self.transport = transport
+
+    def _allowNoMoreWrites(self):
+        """
+        Indicate that no additional writes are allowed.  Attempts to write
+        after calling this method will be met with an exception.
+        """
+        self.transport = None
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register the given producer with C{self.transport}.
+        """
+        self.transport.registerProducer(producer, streaming)
+
+    def write(self, data):
+        """
+        Write the given request body bytes to the transport using chunked
+        encoding.
+
+        @type data: C{bytes}
+        """
+        if self.transport is None:
+            raise ExcessWrite()
+        self.transport.writeSequence(
+            (networkString("%x\r\n" % len(data)), data, b"\r\n")
+        )
+
+    def unregisterProducer(self):
+        """
+        Indicate that the request body is complete and finish the request.
+        """
+        self.write(b"")
+        self.transport.unregisterProducer()
+        self._allowNoMoreWrites()
+
+
+@implementer(IPushProducer)
+class TransportProxyProducer:
+    """
+    An L{twisted.internet.interfaces.IPushProducer} implementation which
+    wraps another such thing and proxies calls to it until it is told to stop.
+
+    @ivar _producer: The wrapped L{twisted.internet.interfaces.IPushProducer}
+    provider or L{None} after this proxy has been stopped.
+    """
+
+    # LineReceiver uses this undocumented attribute of transports to decide
+    # when to stop calling lineReceived or rawDataReceived (if it finds it to
+    # be true, it doesn't bother to deliver any more data).  Set disconnecting
+    # to False here and never change it to true so that all data is always
+    # delivered to us and so that LineReceiver doesn't fail with an
+    # AttributeError.
+    disconnecting = False
+
+    def __init__(self, producer):
+        self._producer = producer
+
+    def stopProxying(self):
+        """
+        Stop forwarding calls of L{twisted.internet.interfaces.IPushProducer}
+        methods to the underlying L{twisted.internet.interfaces.IPushProducer}
+        provider.
+        """
+        self._producer = None
+
+    def stopProducing(self):
+        """
+        Proxy the stoppage to the underlying producer, unless this proxy has
+        been stopped.
+        """
+        if self._producer is not None:
+            self._producer.stopProducing()
+
+    def resumeProducing(self):
+        """
+        Proxy the resumption to the underlying producer, unless this proxy has
+        been stopped.
+        """
+        if self._producer is not None:
+            self._producer.resumeProducing()
+
+    def pauseProducing(self):
+        """
+        Proxy the pause to the underlying producer, unless this proxy has been
+        stopped.
+        """
+        if self._producer is not None:
+            self._producer.pauseProducing()
+
+    def loseConnection(self):
+        """
+        Proxy the request to lose the connection to the underlying producer,
+        unless this proxy has been stopped.
+        """
+        if self._producer is not None:
+            self._producer.loseConnection()
+
+
+class HTTP11ClientProtocol(Protocol):
+    """
+    L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client
+    protocol.  It supports as few features as possible.
+
+    @ivar _parser: After a request is issued, the L{HTTPClientParser} to
+        which received data making up the response to that request is
+        delivered.
+
+    @ivar _finishedRequest: After a request is issued, the L{Deferred} which
+        will fire when a L{Response} object corresponding to that request is
+        available.  This allows L{HTTP11ClientProtocol} to fail the request
+        if there is a connection or parsing problem.
+
+    @ivar _currentRequest: After a request is issued, the L{Request}
+        instance used to make that request.  This allows
+        L{HTTP11ClientProtocol} to stop request generation if necessary (for
+        example, if the connection is lost).
+
+    @ivar _transportProxy: After a request is issued, the
+        L{TransportProxyProducer} to which C{_parser} is connected.  This
+        allows C{_parser} to pause and resume the transport in a way which
+        L{HTTP11ClientProtocol} can exert some control over.
+
+    @ivar _responseDeferred: After a request is issued, the L{Deferred} from
+        C{_parser} which will fire with a L{Response} when one has been
+        received.  This is eventually chained with C{_finishedRequest}, but
+        only in certain cases to avoid double firing that Deferred.
+
+    @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance
+        is in with respect to transmission of a request and reception of a
+        response.  This may be one of the following strings:
+
+          - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances
+            start in.  Nothing is happening: no request is being sent and no
+            response is being received or expected.
+
+          - TRANSMITTING: When a request is made (via L{request}), the
+            instance moves to this state.  L{Request.writeTo} has been used
+            to start to send a request but it has not yet finished.
+
+          - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a
+            complete response but the request has not yet been fully sent
+            yet.  The instance will remain in this state until the request
+            is fully sent.
+
+          - GENERATION_FAILED: There was an error while the request.  The
+            request was not fully sent to the network.
+
+          - WAITING: The request was fully sent to the network.  The
+            instance is now waiting for the response to be fully received.
+
+          - ABORTING: Application code has requested that the HTTP connection
+            be aborted.
+
+          - CONNECTION_LOST: The connection has been lost.
+    @type _state: C{str}
+
+    @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when
+        the connection is lost.
+    """
+
+    _state = "QUIESCENT"
+    _parser = None
+    _finishedRequest = None
+    _currentRequest = None
+    _transportProxy = None
+    _responseDeferred = None
+    _log = Logger()
+
+    def __init__(self, quiescentCallback=lambda c: None):
+        self._quiescentCallback = quiescentCallback
+        self._abortDeferreds = []
+
+    @property
+    def state(self):
+        return self._state
+
+    def request(self, request):
+        """
+        Issue C{request} over C{self.transport} and return a L{Deferred} which
+        will fire with a L{Response} instance or an error.
+
+        @param request: The object defining the parameters of the request to
+           issue.
+        @type request: L{Request}
+
+        @rtype: L{Deferred}
+        @return: The deferred may errback with L{RequestGenerationFailed} if
+            the request was not fully written to the transport due to a local
+            error.  It may errback with L{RequestTransmissionFailed} if it was
+            not fully written to the transport due to a network error.  It may
+            errback with L{ResponseFailed} if the request was sent (not
+            necessarily received) but some or all of the response was lost.  It
+            may errback with L{RequestNotSent} if it is not possible to send
+            any more requests using this L{HTTP11ClientProtocol}.
+        """
+        if self._state != "QUIESCENT":
+            return fail(RequestNotSent())
+
+        self._state = "TRANSMITTING"
+        _requestDeferred = maybeDeferred(request.writeTo, self.transport)
+
+        def cancelRequest(ign):
+            # Explicitly cancel the request's deferred if it's still trying to
+            # write when this request is cancelled.
+            if self._state in ("TRANSMITTING", "TRANSMITTING_AFTER_RECEIVING_RESPONSE"):
+                _requestDeferred.cancel()
+            else:
+                self.transport.abortConnection()
+                self._disconnectParser(Failure(CancelledError()))
+
+        self._finishedRequest = Deferred(cancelRequest)
+
+        # Keep track of the Request object in case we need to call stopWriting
+        # on it.
+        self._currentRequest = request
+
+        self._transportProxy = TransportProxyProducer(self.transport)
+        self._parser = HTTPClientParser(request, self._finishResponse)
+        self._parser.makeConnection(self._transportProxy)
+        self._responseDeferred = self._parser._responseDeferred
+
+        def cbRequestWritten(ignored):
+            if self._state == "TRANSMITTING":
+                self._state = "WAITING"
+                self._responseDeferred.chainDeferred(self._finishedRequest)
+
+        def ebRequestWriting(err):
+            if self._state == "TRANSMITTING":
+                self._state = "GENERATION_FAILED"
+                self.transport.abortConnection()
+                self._finishedRequest.errback(Failure(RequestGenerationFailed([err])))
+            else:
+                self._log.failure(
+                    "Error writing request, but not in valid state "
+                    "to finalize request: {state}",
+                    failure=err,
+                    state=self._state,
+                )
+
+        _requestDeferred.addCallbacks(cbRequestWritten, ebRequestWriting)
+
+        return self._finishedRequest
+
+    def _finishResponse(self, rest):
+        """
+        Called by an L{HTTPClientParser} to indicate that it has parsed a
+        complete response.
+
+        @param rest: A C{bytes} giving any trailing bytes which were given to
+            the L{HTTPClientParser} which were not part of the response it
+            was parsing.
+        """
+
+    _finishResponse = makeStatefulDispatcher("finishResponse", _finishResponse)
+
+    def _finishResponse_WAITING(self, rest):
+        # Currently the rest parameter is ignored. Don't forget to use it if
+        # we ever add support for pipelining. And maybe check what trailers
+        # mean.
+        if self._state == "WAITING":
+            self._state = "QUIESCENT"
+        else:
+            # The server sent the entire response before we could send the
+            # whole request.  That sucks.  Oh well.  Fire the request()
+            # Deferred with the response.  But first, make sure that if the
+            # request does ever finish being written that it won't try to fire
+            # that Deferred.
+            self._state = "TRANSMITTING_AFTER_RECEIVING_RESPONSE"
+            self._responseDeferred.chainDeferred(self._finishedRequest)
+
+        # This will happen if we're being called due to connection being lost;
+        # if so, no need to disconnect parser again, or to call
+        # _quiescentCallback.
+        if self._parser is None:
+            return
+
+        reason = ConnectionDone("synthetic!")
+        connHeaders = self._parser.connHeaders.getRawHeaders(b"connection", ())
+        if (
+            (b"close" in connHeaders)
+            or self._state != "QUIESCENT"
+            or not self._currentRequest.persistent
+        ):
+            self._giveUp(Failure(reason))
+        else:
+            # Just in case we had paused the transport, resume it before
+            # considering it quiescent again.
+            self.transport.resumeProducing()
+
+            # We call the quiescent callback first, to ensure connection gets
+            # added back to connection pool before we finish the request.
+            try:
+                self._quiescentCallback(self)
+            except BaseException:
+                # If callback throws exception, just log it and disconnect;
+                # keeping persistent connections around is an optimisation:
+                self._log.failure("")
+                self.transport.loseConnection()
+            self._disconnectParser(reason)
+
+    _finishResponse_TRANSMITTING = _finishResponse_WAITING
+
+    def _disconnectParser(self, reason):
+        """
+        If there is still a parser, call its C{connectionLost} method with the
+        given reason.  If there is not, do nothing.
+
+        @type reason: L{Failure}
+        """
+        if self._parser is not None:
+            parser = self._parser
+            self._parser = None
+            self._currentRequest = None
+            self._finishedRequest = None
+            self._responseDeferred = None
+
+            # The parser is no longer allowed to do anything to the real
+            # transport.  Stop proxying from the parser's transport to the real
+            # transport before telling the parser it's done so that it can't do
+            # anything.
+            self._transportProxy.stopProxying()
+            self._transportProxy = None
+            parser.connectionLost(reason)
+
+    def _giveUp(self, reason):
+        """
+        Lose the underlying connection and disconnect the parser with the given
+        L{Failure}.
+
+        Use this method instead of calling the transport's loseConnection
+        method directly otherwise random things will break.
+        """
+        self.transport.loseConnection()
+        self._disconnectParser(reason)
+
+    def dataReceived(self, bytes):
+        """
+        Handle some stuff from some place.
+        """
+        try:
+            self._parser.dataReceived(bytes)
+        except BaseException:
+            self._giveUp(Failure())
+
+    def connectionLost(self, reason):
+        """
+        The underlying transport went away.  If appropriate, notify the parser
+        object.
+        """
+
+    connectionLost = makeStatefulDispatcher("connectionLost", connectionLost)
+
+    def _connectionLost_QUIESCENT(self, reason):
+        """
+        Nothing is currently happening.  Move to the C{'CONNECTION_LOST'}
+        state but otherwise do nothing.
+        """
+        self._state = "CONNECTION_LOST"
+
+    def _connectionLost_GENERATION_FAILED(self, reason):
+        """
+        The connection was in an inconsistent state.  Move to the
+        C{'CONNECTION_LOST'} state but otherwise do nothing.
+        """
+        self._state = "CONNECTION_LOST"
+
+    def _connectionLost_TRANSMITTING(self, reason):
+        """
+        Fail the L{Deferred} for the current request, notify the request
+        object that it does not need to continue transmitting itself, and
+        move to the C{'CONNECTION_LOST'} state.
+        """
+        self._state = "CONNECTION_LOST"
+        self._finishedRequest.errback(Failure(RequestTransmissionFailed([reason])))
+        del self._finishedRequest
+
+        # Tell the request that it should stop bothering now.
+        self._currentRequest.stopWriting()
+
+    def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason):
+        """
+        Move to the C{'CONNECTION_LOST'} state.
+        """
+        self._state = "CONNECTION_LOST"
+
+    def _connectionLost_WAITING(self, reason):
+        """
+        Disconnect the response parser so that it can propagate the event as
+        necessary (for example, to call an application protocol's
+        C{connectionLost} method, or to fail a request L{Deferred}) and move
+        to the C{'CONNECTION_LOST'} state.
+        """
+        self._disconnectParser(reason)
+        self._state = "CONNECTION_LOST"
+
+    def _connectionLost_ABORTING(self, reason):
+        """
+        Disconnect the response parser with a L{ConnectionAborted} failure, and
+        move to the C{'CONNECTION_LOST'} state.
+        """
+        self._disconnectParser(Failure(ConnectionAborted()))
+        self._state = "CONNECTION_LOST"
+        for d in self._abortDeferreds:
+            d.callback(None)
+        self._abortDeferreds = []
+
+    def abort(self):
+        """
+        Close the connection and cause all outstanding L{request} L{Deferred}s
+        to fire with an error.
+        """
+        if self._state == "CONNECTION_LOST":
+            return succeed(None)
+        self.transport.loseConnection()
+        self._state = "ABORTING"
+        d = Deferred()
+        self._abortDeferreds.append(d)
+        return d
diff --git a/contrib/python/Twisted/py3/twisted/web/_responses.py b/contrib/python/Twisted/py3/twisted/web/_responses.py
new file mode 100644
index 00000000000..2b932293503
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_responses.py
@@ -0,0 +1,110 @@
+# -*- test-case-name: twisted.web.test.test_http -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HTTP response code definitions.
+"""
+
+
+_CONTINUE = 100
+SWITCHING = 101
+
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+
+MULTIPLE_CHOICE = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+PERMANENT_REDIRECT = 308
+
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTH_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE_SPACE = 507
+NOT_EXTENDED = 510
+
+RESPONSES = {
+    # 100
+    _CONTINUE: b"Continue",
+    SWITCHING: b"Switching Protocols",
+    # 200
+    OK: b"OK",
+    CREATED: b"Created",
+    ACCEPTED: b"Accepted",
+    NON_AUTHORITATIVE_INFORMATION: b"Non-Authoritative Information",
+    NO_CONTENT: b"No Content",
+    RESET_CONTENT: b"Reset Content.",
+    PARTIAL_CONTENT: b"Partial Content",
+    MULTI_STATUS: b"Multi-Status",
+    # 300
+    MULTIPLE_CHOICE: b"Multiple Choices",
+    MOVED_PERMANENTLY: b"Moved Permanently",
+    FOUND: b"Found",
+    SEE_OTHER: b"See Other",
+    NOT_MODIFIED: b"Not Modified",
+    USE_PROXY: b"Use Proxy",
+    # 306 not defined??
+    TEMPORARY_REDIRECT: b"Temporary Redirect",
+    PERMANENT_REDIRECT: b"Permanent Redirect",
+    # 400
+    BAD_REQUEST: b"Bad Request",
+    UNAUTHORIZED: b"Unauthorized",
+    PAYMENT_REQUIRED: b"Payment Required",
+    FORBIDDEN: b"Forbidden",
+    NOT_FOUND: b"Not Found",
+    NOT_ALLOWED: b"Method Not Allowed",
+    NOT_ACCEPTABLE: b"Not Acceptable",
+    PROXY_AUTH_REQUIRED: b"Proxy Authentication Required",
+    REQUEST_TIMEOUT: b"Request Time-out",
+    CONFLICT: b"Conflict",
+    GONE: b"Gone",
+    LENGTH_REQUIRED: b"Length Required",
+    PRECONDITION_FAILED: b"Precondition Failed",
+    REQUEST_ENTITY_TOO_LARGE: b"Request Entity Too Large",
+    REQUEST_URI_TOO_LONG: b"Request-URI Too Long",
+    UNSUPPORTED_MEDIA_TYPE: b"Unsupported Media Type",
+    REQUESTED_RANGE_NOT_SATISFIABLE: b"Requested Range not satisfiable",
+    EXPECTATION_FAILED: b"Expectation Failed",
+    # 500
+    INTERNAL_SERVER_ERROR: b"Internal Server Error",
+    NOT_IMPLEMENTED: b"Not Implemented",
+    BAD_GATEWAY: b"Bad Gateway",
+    SERVICE_UNAVAILABLE: b"Service Unavailable",
+    GATEWAY_TIMEOUT: b"Gateway Time-out",
+    HTTP_VERSION_NOT_SUPPORTED: b"HTTP Version not supported",
+    INSUFFICIENT_STORAGE_SPACE: b"Insufficient Storage Space",
+    NOT_EXTENDED: b"Not Extended",
+}
diff --git a/contrib/python/Twisted/py3/twisted/web/_stan.py b/contrib/python/Twisted/py3/twisted/web/_stan.py
new file mode 100644
index 00000000000..88e82d2dfe2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_stan.py
@@ -0,0 +1,360 @@
+# -*- test-case-name: twisted.web.test.test_stan -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An s-expression-like syntax for expressing xml in pure python.
+
+Stan tags allow you to build XML documents using Python.
+
+Stan is a DOM, or Document Object Model, implemented using basic Python types
+and functions called "flatteners". A flattener is a function that knows how to
+turn an object of a specific type into something that is closer to an HTML
+string. Stan differs from the W3C DOM by not being as cumbersome and heavy
+weight. Since the object model is built using simple python types such as lists,
+strings, and dictionaries, the API is simpler and constructing a DOM less
+cumbersome.
+
+@var voidElements: the names of HTML 'U{void
+    elements<http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html#void-elements>}';
+    those which can't have contents and can therefore be self-closing in the
+    output.
+"""
+
+
+from inspect import iscoroutine, isgenerator
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from warnings import warn
+
+import attr
+
+if TYPE_CHECKING:
+    from twisted.web.template import Flattenable
+
+
+@attr.s(hash=False, eq=False, auto_attribs=True)
+class slot:
+    """
+    Marker for markup insertion in a template.
+    """
+
+    name: str
+    """
+    The name of this slot.
+
+    The key which must be used in L{Tag.fillSlots} to fill it.
+    """
+
+    children: List["Tag"] = attr.ib(init=False, factory=list)
+    """
+    The L{Tag} objects included in this L{slot}'s template.
+    """
+
+    default: Optional["Flattenable"] = None
+    """
+    The default contents of this slot, if it is left unfilled.
+
+    If this is L{None}, an L{UnfilledSlot} will be raised, rather than
+    L{None} actually being used.
+    """
+
+    filename: Optional[str] = None
+    """
+    The name of the XML file from which this tag was parsed.
+
+    If it was not parsed from an XML file, L{None}.
+    """
+
+    lineNumber: Optional[int] = None
+    """
+    The line number on which this tag was encountered in the XML file
+    from which it was parsed.
+
+    If it was not parsed from an XML file, L{None}.
+    """
+
+    columnNumber: Optional[int] = None
+    """
+    The column number at which this tag was encountered in the XML file
+    from which it was parsed.
+
+    If it was not parsed from an XML file, L{None}.
+    """
+
+
+@attr.s(hash=False, eq=False, repr=False, auto_attribs=True)
+class Tag:
+    """
+    A L{Tag} represents an XML tags with a tag name, attributes, and children.
+    A L{Tag} can be constructed using the special L{twisted.web.template.tags}
+    object, or it may be constructed directly with a tag name. L{Tag}s have a
+    special method, C{__call__}, which makes representing trees of XML natural
+    using pure python syntax.
+    """
+
+    tagName: Union[bytes, str]
+    """
+    The name of the represented element.
+
+    For a tag like C{<div></div>}, this would be C{"div"}.
+    """
+
+    attributes: Dict[Union[bytes, str], "Flattenable"] = attr.ib(factory=dict)
+    """The attributes of the element."""
+
+    children: List["Flattenable"] = attr.ib(factory=list)
+    """The contents of this C{Tag}."""
+
+    render: Optional[str] = None
+    """
+    The name of the render method to use for this L{Tag}.
+
+    This name will be looked up at render time by the
+    L{twisted.web.template.Element} doing the rendering,
+    via L{twisted.web.template.Element.lookupRenderMethod},
+    to determine which method to call.
+    """
+
+    filename: Optional[str] = None
+    """
+    The name of the XML file from which this tag was parsed.
+
+    If it was not parsed from an XML file, L{None}.
+    """
+
+    lineNumber: Optional[int] = None
+    """
+    The line number on which this tag was encountered in the XML file
+    from which it was parsed.
+
+    If it was not parsed from an XML file, L{None}.
+    """
+
+    columnNumber: Optional[int] = None
+    """
+    The column number at which this tag was encountered in the XML file
+    from which it was parsed.
+
+    If it was not parsed from an XML file, L{None}.
+    """
+
+    slotData: Optional[Dict[str, "Flattenable"]] = attr.ib(init=False, default=None)
+    """
+    The data which can fill slots.
+
+    If present, a dictionary mapping slot names to renderable values.
+    The values in this dict might be anything that can be present as
+    the child of a L{Tag}: strings, lists, L{Tag}s, generators, etc.
+    """
+
+    def fillSlots(self, **slots: "Flattenable") -> "Tag":
+        """
+        Remember the slots provided at this position in the DOM.
+
+        During the rendering of children of this node, slots with names in
+        C{slots} will be rendered as their corresponding values.
+
+        @return: C{self}. This enables the idiom C{return tag.fillSlots(...)} in
+            renderers.
+        """
+        if self.slotData is None:
+            self.slotData = {}
+        self.slotData.update(slots)
+        return self
+
+    def __call__(self, *children: "Flattenable", **kw: "Flattenable") -> "Tag":
+        """
+        Add children and change attributes on this tag.
+
+        This is implemented using __call__ because it then allows the natural
+        syntax::
+
+          table(tr1, tr2, width="100%", height="50%", border="1")
+
+        Children may be other tag instances, strings, functions, or any other
+        object which has a registered flatten.
+
+        Attributes may be 'transparent' tag instances (so that
+        C{a(href=transparent(data="foo", render=myhrefrenderer))} works),
+        strings, functions, or any other object which has a registered
+        flattener.
+
+        If the attribute is a python keyword, such as 'class', you can add an
+        underscore to the name, like 'class_'.
+
+        There is one special keyword argument, 'render', which will be used as
+        the name of the renderer and saved as the 'render' attribute of this
+        instance, rather than the DOM 'render' attribute in the attributes
+        dictionary.
+        """
+        self.children.extend(children)
+
+        for k, v in kw.items():
+            if k[-1] == "_":
+                k = k[:-1]
+
+            if k == "render":
+                if not isinstance(v, str):
+                    raise TypeError(
+                        f'Value for "render" attribute must be str, got {v!r}'
+                    )
+                self.render = v
+            else:
+                self.attributes[k] = v
+        return self
+
+    def _clone(self, obj: "Flattenable", deep: bool) -> "Flattenable":
+        """
+        Clone a C{Flattenable} object; used by L{Tag.clone}.
+
+        Note that both lists and tuples are cloned into lists.
+
+        @param obj: an object with a clone method, a list or tuple, or something
+            which should be immutable.
+
+        @param deep: whether to continue cloning child objects; i.e. the
+            contents of lists, the sub-tags within a tag.
+
+        @return: a clone of C{obj}.
+        """
+        if hasattr(obj, "clone"):
+            return obj.clone(deep)
+        elif isinstance(obj, (list, tuple)):
+            return [self._clone(x, deep) for x in obj]
+        elif isgenerator(obj):
+            warn(
+                "Cloning a Tag which contains a generator is unsafe, "
+                "since the generator can be consumed only once; "
+                "this is deprecated since Twisted 21.7.0 and will raise "
+                "an exception in the future",
+                DeprecationWarning,
+            )
+            return obj
+        elif iscoroutine(obj):
+            warn(
+                "Cloning a Tag which contains a coroutine is unsafe, "
+                "since the coroutine can run only once; "
+                "this is deprecated since Twisted 21.7.0 and will raise "
+                "an exception in the future",
+                DeprecationWarning,
+            )
+            return obj
+        else:
+            return obj
+
+    def clone(self, deep: bool = True) -> "Tag":
+        """
+        Return a clone of this tag. If deep is True, clone all of this tag's
+        children. Otherwise, just shallow copy the children list without copying
+        the children themselves.
+        """
+        if deep:
+            newchildren = [self._clone(x, True) for x in self.children]
+        else:
+            newchildren = self.children[:]
+        newattrs = self.attributes.copy()
+        for key in newattrs.keys():
+            newattrs[key] = self._clone(newattrs[key], True)
+
+        newslotdata = None
+        if self.slotData:
+            newslotdata = self.slotData.copy()
+            for key in newslotdata:
+                newslotdata[key] = self._clone(newslotdata[key], True)
+
+        newtag = Tag(
+            self.tagName,
+            attributes=newattrs,
+            children=newchildren,
+            render=self.render,
+            filename=self.filename,
+            lineNumber=self.lineNumber,
+            columnNumber=self.columnNumber,
+        )
+        newtag.slotData = newslotdata
+
+        return newtag
+
+    def clear(self) -> "Tag":
+        """
+        Clear any existing children from this tag.
+        """
+        self.children = []
+        return self
+
+    def __repr__(self) -> str:
+        rstr = ""
+        if self.attributes:
+            rstr += ", attributes=%r" % self.attributes
+        if self.children:
+            rstr += ", children=%r" % self.children
+        return f"Tag({self.tagName!r}{rstr})"
+
+
+voidElements = (
+    "img",
+    "br",
+    "hr",
+    "base",
+    "meta",
+    "link",
+    "param",
+    "area",
+    "input",
+    "col",
+    "basefont",
+    "isindex",
+    "frame",
+    "command",
+    "embed",
+    "keygen",
+    "source",
+    "track",
+    "wbs",
+)
+
+
+@attr.s(hash=False, eq=False, repr=False, auto_attribs=True)
+class CDATA:
+    """
+    A C{<![CDATA[]]>} block from a template.  Given a separate representation in
+    the DOM so that they may be round-tripped through rendering without losing
+    information.
+    """
+
+    data: str
+    """The data between "C{<![CDATA[}" and "C{]]>}"."""
+
+    def __repr__(self) -> str:
+        return f"CDATA({self.data!r})"
+
+
+@attr.s(hash=False, eq=False, repr=False, auto_attribs=True)
+class Comment:
+    """
+    A C{<!-- -->} comment from a template.  Given a separate representation in
+    the DOM so that they may be round-tripped through rendering without losing
+    information.
+    """
+
+    data: str
+    """The data between "C{<!--}" and "C{-->}"."""
+
+    def __repr__(self) -> str:
+        return f"Comment({self.data!r})"
+
+
+@attr.s(hash=False, eq=False, repr=False, auto_attribs=True)
+class CharRef:
+    """
+    A numeric character reference.  Given a separate representation in the DOM
+    so that non-ASCII characters may be output as pure ASCII.
+
+    @since: 12.0
+    """
+
+    ordinal: int
+    """The ordinal value of the unicode character to which this object refers."""
+
+    def __repr__(self) -> str:
+        return "CharRef(%d)" % (self.ordinal,)
diff --git a/contrib/python/Twisted/py3/twisted/web/_template_util.py b/contrib/python/Twisted/py3/twisted/web/_template_util.py
new file mode 100644
index 00000000000..4a9f7f21002
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/_template_util.py
@@ -0,0 +1,1112 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+twisted.web.util and twisted.web.template merged to avoid cyclic deps
+"""
+
+import io
+import linecache
+import warnings
+from collections import OrderedDict
+from html import escape
+from typing import (
+    IO,
+    Any,
+    AnyStr,
+    Callable,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
+from xml.sax import handler, make_parser
+from xml.sax.xmlreader import Locator
+
+from zope.interface import implementer
+
+from twisted.internet.defer import Deferred
+from twisted.logger import Logger
+from twisted.python import urlpath
+from twisted.python.failure import Failure
+from twisted.python.filepath import FilePath
+from twisted.python.reflect import fullyQualifiedName
+from twisted.web import resource
+from twisted.web._element import Element, renderer
+from twisted.web._flatten import Flattenable, flatten, flattenString
+from twisted.web._stan import CDATA, Comment, Tag, slot
+from twisted.web.iweb import IRenderable, IRequest, ITemplateLoader
+
+
+def _PRE(text):
+    """
+    Wraps <pre> tags around some text and HTML-escape it.
+
+    This is here since once twisted.web.html was deprecated it was hard to
+    migrate the html.PRE from current code to twisted.web.template.
+
+    For new code consider using twisted.web.template.
+
+    @return: Escaped text wrapped in <pre> tags.
+    @rtype: C{str}
+    """
+    return f"<pre>{escape(text)}</pre>"
+
+
+def redirectTo(URL: bytes, request: IRequest) -> bytes:
+    """
+    Generate a redirect to the given location.
+
+    @param URL: A L{bytes} giving the location to which to redirect.
+
+    @param request: The request object to use to generate the redirect.
+    @type request: L{IRequest<twisted.web.iweb.IRequest>} provider
+
+    @raise TypeError: If the type of C{URL} a L{str} instead of L{bytes}.
+
+    @return: A L{bytes} containing HTML which tries to convince the client
+        agent
+        to visit the new location even if it doesn't respect the I{FOUND}
+        response code.  This is intended to be returned from a render method,
+        eg::
+
+            def render_GET(self, request):
+                return redirectTo(b"http://example.com/", request)
+    """
+    if not isinstance(URL, bytes):
+        raise TypeError("URL must be bytes")
+    request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+    request.redirect(URL)
+    # FIXME: The URL should be HTML-escaped.
+    # https://twistedmatrix.com/trac/ticket/9839
+    content = b"""
+<html>
+    <head>
+        <meta http-equiv=\"refresh\" content=\"0;URL=%(url)s\">
+    </head>
+    <body bgcolor=\"#FFFFFF\" text=\"#000000\">
+    <a href=\"%(url)s\">click here</a>
+    </body>
+</html>
+""" % {
+        b"url": URL
+    }
+    return content
+
+
+class Redirect(resource.Resource):
+    """
+    Resource that redirects to a specific URL.
+
+    @ivar url: Redirect target URL to put in the I{Location} response header.
+    @type url: L{bytes}
+    """
+
+    isLeaf = True
+
+    def __init__(self, url: bytes):
+        super().__init__()
+        self.url = url
+
+    def render(self, request):
+        return redirectTo(self.url, request)
+
+    def getChild(self, name, request):
+        return self
+
+
+# FIXME: This is totally broken, see https://twistedmatrix.com/trac/ticket/9838
+class ChildRedirector(Redirect):
+    isLeaf = False
+
+    def __init__(self, url):
+        # XXX is this enough?
+        if (
+            (url.find("://") == -1)
+            and (not url.startswith(".."))
+            and (not url.startswith("/"))
+        ):
+            raise ValueError(
+                (
+                    "It seems you've given me a redirect (%s) that is a child of"
+                    " myself! That's not good, it'll cause an infinite redirect."
+                )
+                % url
+            )
+        Redirect.__init__(self, url)
+
+    def getChild(self, name, request):
+        newUrl = self.url
+        if not newUrl.endswith("/"):
+            newUrl += "/"
+        newUrl += name
+        return ChildRedirector(newUrl)
+
+
+class ParentRedirect(resource.Resource):
+    """
+    Redirect to the nearest directory and strip any query string.
+
+    This generates redirects like::
+
+        /              \u2192  /
+        /foo           \u2192  /
+        /foo?bar       \u2192  /
+        /foo/          \u2192  /foo/
+        /foo/bar       \u2192  /foo/
+        /foo/bar?baz   \u2192  /foo/
+
+    However, the generated I{Location} header contains an absolute URL rather
+    than a path.
+
+    The response is the same regardless of HTTP method.
+    """
+
+    isLeaf = 1
+
+    def render(self, request: IRequest) -> bytes:
+        """
+        Respond to all requests by redirecting to nearest directory.
+        """
+        here = str(urlpath.URLPath.fromRequest(request).here()).encode("ascii")
+        return redirectTo(here, request)
+
+
+class DeferredResource(resource.Resource):
+    """
+    I wrap up a Deferred that will eventually result in a Resource
+    object.
+    """
+
+    isLeaf = 1
+
+    def __init__(self, d):
+        resource.Resource.__init__(self)
+        self.d = d
+
+    def getChild(self, name, request):
+        return self
+
+    def render(self, request):
+        self.d.addCallback(self._cbChild, request).addErrback(self._ebChild, request)
+        from twisted.web.server import NOT_DONE_YET
+
+        return NOT_DONE_YET
+
+    def _cbChild(self, child, request):
+        request.render(resource.getChildForRequest(child, request))
+
+    def _ebChild(self, reason, request):
+        request.processingFailed(reason)
+
+
+class _SourceLineElement(Element):
+    """
+    L{_SourceLineElement} is an L{IRenderable} which can render a single line of
+    source code.
+
+    @ivar number: A C{int} giving the line number of the source code to be
+        rendered.
+    @ivar source: A C{str} giving the source code to be rendered.
+    """
+
+    def __init__(self, loader, number, source):
+        Element.__init__(self, loader)
+        self.number = number
+        self.source = source
+
+    @renderer
+    def sourceLine(self, request, tag):
+        """
+        Render the line of source as a child of C{tag}.
+        """
+        return tag(self.source.replace("  ", " \N{NO-BREAK SPACE}"))
+
+    @renderer
+    def lineNumber(self, request, tag):
+        """
+        Render the line number as a child of C{tag}.
+        """
+        return tag(str(self.number))
+
+
+class _SourceFragmentElement(Element):
+    """
+    L{_SourceFragmentElement} is an L{IRenderable} which can render several lines
+    of source code near the line number of a particular frame object.
+
+    @ivar frame: A L{Failure<twisted.python.failure.Failure>}-style frame object
+        for which to load a source line to render.  This is really a tuple
+        holding some information from a frame object.  See
+        L{Failure.frames<twisted.python.failure.Failure>} for specifics.
+    """
+
+    def __init__(self, loader, frame):
+        Element.__init__(self, loader)
+        self.frame = frame
+
+    def _getSourceLines(self):
+        """
+        Find the source line references by C{self.frame} and yield, in source
+        line order, it and the previous and following lines.
+
+        @return: A generator which yields two-tuples.  Each tuple gives a source
+            line number and the contents of that source line.
+        """
+        filename = self.frame[1]
+        lineNumber = self.frame[2]
+        for snipLineNumber in range(lineNumber - 1, lineNumber + 2):
+            yield (snipLineNumber, linecache.getline(filename, snipLineNumber).rstrip())
+
+    @renderer
+    def sourceLines(self, request, tag):
+        """
+        Render the source line indicated by C{self.frame} and several
+        surrounding lines.  The active line will be given a I{class} of
+        C{"snippetHighlightLine"}.  Other lines will be given a I{class} of
+        C{"snippetLine"}.
+        """
+        for lineNumber, sourceLine in self._getSourceLines():
+            newTag = tag.clone()
+            if lineNumber == self.frame[2]:
+                cssClass = "snippetHighlightLine"
+            else:
+                cssClass = "snippetLine"
+            loader = TagLoader(newTag(**{"class": cssClass}))
+            yield _SourceLineElement(loader, lineNumber, sourceLine)
+
+
+class _FrameElement(Element):
+    """
+    L{_FrameElement} is an L{IRenderable} which can render details about one
+    frame from a L{Failure<twisted.python.failure.Failure>}.
+
+    @ivar frame: A L{Failure<twisted.python.failure.Failure>}-style frame object
+        for which to load a source line to render.  This is really a tuple
+        holding some information from a frame object.  See
+        L{Failure.frames<twisted.python.failure.Failure>} for specifics.
+    """
+
+    def __init__(self, loader, frame):
+        Element.__init__(self, loader)
+        self.frame = frame
+
+    @renderer
+    def filename(self, request, tag):
+        """
+        Render the name of the file this frame references as a child of C{tag}.
+        """
+        return tag(self.frame[1])
+
+    @renderer
+    def lineNumber(self, request, tag):
+        """
+        Render the source line number this frame references as a child of
+        C{tag}.
+        """
+        return tag(str(self.frame[2]))
+
+    @renderer
+    def function(self, request, tag):
+        """
+        Render the function name this frame references as a child of C{tag}.
+        """
+        return tag(self.frame[0])
+
+    @renderer
+    def source(self, request, tag):
+        """
+        Render the source code surrounding the line this frame references,
+        replacing C{tag}.
+        """
+        return _SourceFragmentElement(TagLoader(tag), self.frame)
+
+
+class _StackElement(Element):
+    """
+    L{_StackElement} renders an L{IRenderable} which can render a list of frames.
+    """
+
+    def __init__(self, loader, stackFrames):
+        Element.__init__(self, loader)
+        self.stackFrames = stackFrames
+
+    @renderer
+    def frames(self, request, tag):
+        """
+        Render the list of frames in this L{_StackElement}, replacing C{tag}.
+        """
+        return [
+            _FrameElement(TagLoader(tag.clone()), frame) for frame in self.stackFrames
+        ]
+
+
+class _NSContext:
+    """
+    A mapping from XML namespaces onto their prefixes in the document.
+    """
+
+    def __init__(self, parent: Optional["_NSContext"] = None):
+        """
+        Pull out the parent's namespaces, if there's no parent then default to
+        XML.
+        """
+        self.parent = parent
+        if parent is not None:
+            self.nss: Dict[Optional[str], Optional[str]] = OrderedDict(parent.nss)
+        else:
+            self.nss = {"http://www.w3.org/XML/1998/namespace": "xml"}
+
+    def get(self, k: Optional[str], d: Optional[str] = None) -> Optional[str]:
+        """
+        Get a prefix for a namespace.
+
+        @param d: The default prefix value.
+        """
+        return self.nss.get(k, d)
+
+    def __setitem__(self, k: Optional[str], v: Optional[str]) -> None:
+        """
+        Proxy through to setting the prefix for the namespace.
+        """
+        self.nss.__setitem__(k, v)
+
+    def __getitem__(self, k: Optional[str]) -> Optional[str]:
+        """
+        Proxy through to getting the prefix for the namespace.
+        """
+        return self.nss.__getitem__(k)
+
+
+TEMPLATE_NAMESPACE = "http://twistedmatrix.com/ns/twisted.web.template/0.1"
+
+
+class _ToStan(handler.ContentHandler, handler.EntityResolver):
+    """
+    A SAX parser which converts an XML document to the Twisted STAN
+    Document Object Model.
+    """
+
+    def __init__(self, sourceFilename: Optional[str]):
+        """
+        @param sourceFilename: the filename the XML was loaded out of.
+        """
+        self.sourceFilename = sourceFilename
+        self.prefixMap = _NSContext()
+        self.inCDATA = False
+
+    def setDocumentLocator(self, locator: Locator) -> None:
+        """
+        Set the document locator, which knows about line and character numbers.
+        """
+        self.locator = locator
+
+    def startDocument(self) -> None:
+        """
+        Initialise the document.
+        """
+        # Depending on our active context, the element type can be Tag, slot
+        # or str. Since mypy doesn't understand that context, it would be
+        # a pain to not use Any here.
+        self.document: List[Any] = []
+        self.current = self.document
+        self.stack: List[Any] = []
+        self.xmlnsAttrs: List[Tuple[str, str]] = []
+
+    def endDocument(self) -> None:
+        """
+        Document ended.
+        """
+
+    def processingInstruction(self, target: str, data: str) -> None:
+        """
+        Processing instructions are ignored.
+        """
+
+    def startPrefixMapping(self, prefix: Optional[str], uri: str) -> None:
+        """
+        Set up the prefix mapping, which maps fully qualified namespace URIs
+        onto namespace prefixes.
+
+        This gets called before startElementNS whenever an C{xmlns} attribute
+        is seen.
+        """
+
+        self.prefixMap = _NSContext(self.prefixMap)
+        self.prefixMap[uri] = prefix
+
+        # Ignore the template namespace; we'll replace those during parsing.
+        if uri == TEMPLATE_NAMESPACE:
+            return
+
+        # Add to a list that will be applied once we have the element.
+        if prefix is None:
+            self.xmlnsAttrs.append(("xmlns", uri))
+        else:
+            self.xmlnsAttrs.append(("xmlns:%s" % prefix, uri))
+
+    def endPrefixMapping(self, prefix: Optional[str]) -> None:
+        """
+        "Pops the stack" on the prefix mapping.
+
+        Gets called after endElementNS.
+        """
+        parent = self.prefixMap.parent
+        assert parent is not None, "More prefix mapping ends than starts"
+        self.prefixMap = parent
+
+    def startElementNS(
+        self,
+        namespaceAndName: Tuple[str, str],
+        qname: Optional[str],
+        attrs: Mapping[Tuple[Optional[str], str], str],
+    ) -> None:
+        """
+        Gets called when we encounter a new xmlns attribute.
+
+        @param namespaceAndName: a (namespace, name) tuple, where name
+            determines which type of action to take, if the namespace matches
+            L{TEMPLATE_NAMESPACE}.
+        @param qname: ignored.
+        @param attrs: attributes on the element being started.
+        """
+
+        filename = self.sourceFilename
+        lineNumber = self.locator.getLineNumber()
+        columnNumber = self.locator.getColumnNumber()
+
+        ns, name = namespaceAndName
+        if ns == TEMPLATE_NAMESPACE:
+            if name == "transparent":
+                name = ""
+            elif name == "slot":
+                default: Optional[str]
+                try:
+                    # Try to get the default value for the slot
+                    default = attrs[(None, "default")]
+                except KeyError:
+                    # If there wasn't one, then use None to indicate no
+                    # default.
+                    default = None
+                sl = slot(
+                    attrs[(None, "name")],
+                    default=default,
+                    filename=filename,
+                    lineNumber=lineNumber,
+                    columnNumber=columnNumber,
+                )
+                self.stack.append(sl)
+                self.current.append(sl)
+                self.current = sl.children
+                return
+
+        render = None
+
+        attrs = OrderedDict(attrs)
+        for k, v in list(attrs.items()):
+            attrNS, justTheName = k
+            if attrNS != TEMPLATE_NAMESPACE:
+                continue
+            if justTheName == "render":
+                render = v
+                del attrs[k]
+
+        # nonTemplateAttrs is a dictionary mapping attributes that are *not* in
+        # TEMPLATE_NAMESPACE to their values.  Those in TEMPLATE_NAMESPACE were
+        # just removed from 'attrs' in the loop immediately above.  The key in
+        # nonTemplateAttrs is either simply the attribute name (if it was not
+        # specified as having a namespace in the template) or prefix:name,
+        # preserving the xml namespace prefix given in the document.
+
+        nonTemplateAttrs = OrderedDict()
+        for (attrNs, attrName), v in attrs.items():
+            nsPrefix = self.prefixMap.get(attrNs)
+            if nsPrefix is None:
+                attrKey = attrName
+            else:
+                attrKey = f"{nsPrefix}:{attrName}"
+            nonTemplateAttrs[attrKey] = v
+
+        if ns == TEMPLATE_NAMESPACE and name == "attr":
+            if not self.stack:
+                # TODO: define a better exception for this?
+                raise AssertionError(
+                    f"<{{{TEMPLATE_NAMESPACE}}}attr> as top-level element"
+                )
+            if "name" not in nonTemplateAttrs:
+                # TODO: same here
+                raise AssertionError(
+                    f"<{{{TEMPLATE_NAMESPACE}}}attr> requires a name attribute"
+                )
+            el = Tag(
+                "",
+                render=render,
+                filename=filename,
+                lineNumber=lineNumber,
+                columnNumber=columnNumber,
+            )
+            self.stack[-1].attributes[nonTemplateAttrs["name"]] = el
+            self.stack.append(el)
+            self.current = el.children
+            return
+
+        # Apply any xmlns attributes
+        if self.xmlnsAttrs:
+            nonTemplateAttrs.update(OrderedDict(self.xmlnsAttrs))
+            self.xmlnsAttrs = []
+
+        # Add the prefix that was used in the parsed template for non-template
+        # namespaces (which will not be consumed anyway).
+        if ns != TEMPLATE_NAMESPACE and ns is not None:
+            prefix = self.prefixMap[ns]
+            if prefix is not None:
+                name = f"{self.prefixMap[ns]}:{name}"
+        el = Tag(
+            name,
+            attributes=OrderedDict(
+                cast(Mapping[Union[bytes, str], str], nonTemplateAttrs)
+            ),
+            render=render,
+            filename=filename,
+            lineNumber=lineNumber,
+            columnNumber=columnNumber,
+        )
+        self.stack.append(el)
+        self.current.append(el)
+        self.current = el.children
+
+    def characters(self, ch: str) -> None:
+        """
+        Called when we receive some characters.  CDATA characters get passed
+        through as is.
+        """
+        if self.inCDATA:
+            self.stack[-1].append(ch)
+            return
+        self.current.append(ch)
+
+    def endElementNS(self, name: Tuple[str, str], qname: Optional[str]) -> None:
+        """
+        A namespace tag is closed.  Pop the stack, if there's anything left in
+        it, otherwise return to the document's namespace.
+        """
+        self.stack.pop()
+        if self.stack:
+            self.current = self.stack[-1].children
+        else:
+            self.current = self.document
+
+    def startDTD(self, name: str, publicId: str, systemId: str) -> None:
+        """
+        DTDs are ignored.
+        """
+
+    def endDTD(self, *args: object) -> None:
+        """
+        DTDs are ignored.
+        """
+
+    def startCDATA(self) -> None:
+        """
+        We're starting to be in a CDATA element, make a note of this.
+        """
+        self.inCDATA = True
+        self.stack.append([])
+
+    def endCDATA(self) -> None:
+        """
+        We're no longer in a CDATA element.  Collect up the characters we've
+        parsed and put them in a new CDATA object.
+        """
+        self.inCDATA = False
+        comment = "".join(self.stack.pop())
+        self.current.append(CDATA(comment))
+
+    def comment(self, content: str) -> None:
+        """
+        Add an XML comment which we've encountered.
+        """
+        self.current.append(Comment(content))
+
+
+def _flatsaxParse(fl: Union[IO[AnyStr], str]) -> List["Flattenable"]:
+    """
+    Perform a SAX parse of an XML document with the _ToStan class.
+
+    @param fl: The XML document to be parsed.
+
+    @return: a C{list} of Stan objects.
+    """
+    parser = make_parser()
+    parser.setFeature(handler.feature_validation, 0)
+    parser.setFeature(handler.feature_namespaces, 1)
+    parser.setFeature(handler.feature_external_ges, 0)
+    parser.setFeature(handler.feature_external_pes, 0)
+
+    s = _ToStan(getattr(fl, "name", None))
+    parser.setContentHandler(s)
+    parser.setEntityResolver(s)
+    parser.setProperty(handler.property_lexical_handler, s)
+
+    parser.parse(fl)
+
+    return s.document
+
+
+@implementer(ITemplateLoader)
+class XMLString:
+    """
+    An L{ITemplateLoader} that loads and parses XML from a string.
+    """
+
+    def __init__(self, s: Union[str, bytes]):
+        """
+        Run the parser on a L{io.StringIO} copy of the string.
+
+        @param s: The string from which to load the XML.
+        @type s: L{str}, or a UTF-8 encoded L{bytes}.
+        """
+        if not isinstance(s, str):
+            s = s.decode("utf8")
+
+        self._loadedTemplate: List["Flattenable"] = _flatsaxParse(io.StringIO(s))
+        """The loaded document."""
+
+    def load(self) -> List["Flattenable"]:
+        """
+        Return the document.
+
+        @return: the loaded document.
+        """
+        return self._loadedTemplate
+
+
+class FailureElement(Element):
+    """
+    L{FailureElement} is an L{IRenderable} which can render detailed information
+    about a L{Failure<twisted.python.failure.Failure>}.
+
+    @ivar failure: The L{Failure<twisted.python.failure.Failure>} instance which
+        will be rendered.
+
+    @since: 12.1
+    """
+
+    loader = XMLString(
+        """
+<div xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
+  <style type="text/css">
+    div.error {
+      color: red;
+      font-family: Verdana, Arial, helvetica, sans-serif;
+      font-weight: bold;
+    }
+
+    div {
+      font-family: Verdana, Arial, helvetica, sans-serif;
+    }
+
+    div.stackTrace {
+    }
+
+    div.frame {
+      padding: 1em;
+      background: white;
+      border-bottom: thin black dashed;
+    }
+
+    div.frame:first-child {
+      padding: 1em;
+      background: white;
+      border-top: thin black dashed;
+      border-bottom: thin black dashed;
+    }
+
+    div.location {
+    }
+
+    span.function {
+      font-weight: bold;
+      font-family: "Courier New", courier, monospace;
+    }
+
+    div.snippet {
+      margin-bottom: 0.5em;
+      margin-left: 1em;
+      background: #FFFFDD;
+    }
+
+    div.snippetHighlightLine {
+      color: red;
+    }
+
+    span.code {
+      font-family: "Courier New", courier, monospace;
+    }
+  </style>
+
+  <div class="error">
+    <span t:render="type" />: <span t:render="value" />
+  </div>
+  <div class="stackTrace" t:render="traceback">
+    <div class="frame" t:render="frames">
+      <div class="location">
+        <span t:render="filename" />:<span t:render="lineNumber" /> in
+        <span class="function" t:render="function" />
+      </div>
+      <div class="snippet" t:render="source">
+        <div t:render="sourceLines">
+          <span class="lineno" t:render="lineNumber" />
+          <code class="code" t:render="sourceLine" />
+        </div>
+      </div>
+    </div>
+  </div>
+  <div class="error">
+    <span t:render="type" />: <span t:render="value" />
+  </div>
+</div>
+"""
+    )
+
+    def __init__(self, failure, loader=None):
+        Element.__init__(self, loader)
+        self.failure = failure
+
+    @renderer
+    def type(self, request, tag):
+        """
+        Render the exception type as a child of C{tag}.
+        """
+        return tag(fullyQualifiedName(self.failure.type))
+
+    @renderer
+    def value(self, request, tag):
+        """
+        Render the exception value as a child of C{tag}.
+        """
+        return tag(str(self.failure.value).encode("utf8"))
+
+    @renderer
+    def traceback(self, request, tag):
+        """
+        Render all the frames in the wrapped
+        L{Failure<twisted.python.failure.Failure>}'s traceback stack, replacing
+        C{tag}.
+        """
+        return _StackElement(TagLoader(tag), self.failure.frames)
+
+
+def formatFailure(myFailure):
+    """
+    Construct an HTML representation of the given failure.
+
+    Consider using L{FailureElement} instead.
+
+    @type myFailure: L{Failure<twisted.python.failure.Failure>}
+
+    @rtype: L{bytes}
+    @return: A string containing the HTML representation of the given failure.
+    """
+    result = []
+    flattenString(None, FailureElement(myFailure)).addBoth(result.append)
+    if isinstance(result[0], bytes):
+        # Ensure the result string is all ASCII, for compatibility with the
+        # default encoding expected by browsers.
+        return result[0].decode("utf-8").encode("ascii", "xmlcharrefreplace")
+    result[0].raiseException()
+
+
+# Go read the definition of NOT_DONE_YET. For lulz. This is totally
+# equivalent. And this turns out to be necessary, because trying to import
+# NOT_DONE_YET in this module causes a circular import which we cannot escape
+# from. From which we cannot escape. Etc. glyph is okay with this solution for
+# now, and so am I, as long as this comment stays to explain to future
+# maintainers what it means. ~ C.
+#
+# See http://twistedmatrix.com/trac/ticket/5557 for progress on fixing this.
+NOT_DONE_YET = 1
+_moduleLog = Logger()
+
+
+@implementer(ITemplateLoader)
+class TagLoader:
+    """
+    An L{ITemplateLoader} that loads an existing flattenable object.
+    """
+
+    def __init__(self, tag: "Flattenable"):
+        """
+        @param tag: The object which will be loaded.
+        """
+
+        self.tag: "Flattenable" = tag
+        """The object which will be loaded."""
+
+    def load(self) -> List["Flattenable"]:
+        return [self.tag]
+
+
+@implementer(ITemplateLoader)
+class XMLFile:
+    """
+    An L{ITemplateLoader} that loads and parses XML from a file.
+    """
+
+    def __init__(self, path: FilePath[Any]):
+        """
+        Run the parser on a file.
+
+        @param path: The file from which to load the XML.
+        """
+        if not isinstance(path, FilePath):
+            warnings.warn(  # type: ignore[unreachable]
+                "Passing filenames or file objects to XMLFile is deprecated "
+                "since Twisted 12.1.  Pass a FilePath instead.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+
+        self._loadedTemplate: Optional[List["Flattenable"]] = None
+        """The loaded document, or L{None}, if not loaded."""
+
+        self._path: FilePath[Any] = path
+        """The file that is being loaded from."""
+
+    def _loadDoc(self) -> List["Flattenable"]:
+        """
+        Read and parse the XML.
+
+        @return: the loaded document.
+        """
+        if not isinstance(self._path, FilePath):
+            return _flatsaxParse(self._path)  # type: ignore[unreachable]
+        else:
+            with self._path.open("r") as f:
+                return _flatsaxParse(f)
+
+    def __repr__(self) -> str:
+        return f"<XMLFile of {self._path!r}>"
+
+    def load(self) -> List["Flattenable"]:
+        """
+        Return the document, first loading it if necessary.
+
+        @return: the loaded document.
+        """
+        if self._loadedTemplate is None:
+            self._loadedTemplate = self._loadDoc()
+        return self._loadedTemplate
+
+
+# Last updated October 2011, using W3Schools as a reference. Link:
+# http://www.w3schools.com/html5/html5_reference.asp
+# Note that <xmp> is explicitly omitted; its semantics do not work with
+# t.w.template and it is officially deprecated.
+VALID_HTML_TAG_NAMES = {
+    "a",
+    "abbr",
+    "acronym",
+    "address",
+    "applet",
+    "area",
+    "article",
+    "aside",
+    "audio",
+    "b",
+    "base",
+    "basefont",
+    "bdi",
+    "bdo",
+    "big",
+    "blockquote",
+    "body",
+    "br",
+    "button",
+    "canvas",
+    "caption",
+    "center",
+    "cite",
+    "code",
+    "col",
+    "colgroup",
+    "command",
+    "datalist",
+    "dd",
+    "del",
+    "details",
+    "dfn",
+    "dir",
+    "div",
+    "dl",
+    "dt",
+    "em",
+    "embed",
+    "fieldset",
+    "figcaption",
+    "figure",
+    "font",
+    "footer",
+    "form",
+    "frame",
+    "frameset",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "head",
+    "header",
+    "hgroup",
+    "hr",
+    "html",
+    "i",
+    "iframe",
+    "img",
+    "input",
+    "ins",
+    "isindex",
+    "keygen",
+    "kbd",
+    "label",
+    "legend",
+    "li",
+    "link",
+    "map",
+    "mark",
+    "menu",
+    "meta",
+    "meter",
+    "nav",
+    "noframes",
+    "noscript",
+    "object",
+    "ol",
+    "optgroup",
+    "option",
+    "output",
+    "p",
+    "param",
+    "pre",
+    "progress",
+    "q",
+    "rp",
+    "rt",
+    "ruby",
+    "s",
+    "samp",
+    "script",
+    "section",
+    "select",
+    "small",
+    "source",
+    "span",
+    "strike",
+    "strong",
+    "style",
+    "sub",
+    "summary",
+    "sup",
+    "table",
+    "tbody",
+    "td",
+    "textarea",
+    "tfoot",
+    "th",
+    "thead",
+    "time",
+    "title",
+    "tr",
+    "tt",
+    "u",
+    "ul",
+    "var",
+    "video",
+    "wbr",
+}
+
+
+class _TagFactory:
+    """
+    A factory for L{Tag} objects; the implementation of the L{tags} object.
+
+    This allows for the syntactic convenience of C{from twisted.web.template
+    import tags; tags.a(href="linked-page.html")}, where 'a' can be basically
+    any HTML tag.
+
+    The class is not exposed publicly because you only ever need one of these,
+    and we already made it for you.
+
+    @see: L{tags}
+    """
+
+    def __getattr__(self, tagName: str) -> Tag:
+        if tagName == "transparent":
+            return Tag("")
+        # allow for E.del as E.del_
+        tagName = tagName.rstrip("_")
+        if tagName not in VALID_HTML_TAG_NAMES:
+            raise AttributeError(f"unknown tag {tagName!r}")
+        return Tag(tagName)
+
+
+tags = _TagFactory()
+
+
+def renderElement(
+    request: IRequest,
+    element: IRenderable,
+    doctype: Optional[bytes] = b"<!DOCTYPE html>",
+    _failElement: Optional[Callable[[Failure], "Element"]] = None,
+) -> object:
+    """
+    Render an element or other L{IRenderable}.
+
+    @param request: The L{IRequest} being rendered to.
+    @param element: An L{IRenderable} which will be rendered.
+    @param doctype: A L{bytes} which will be written as the first line of
+        the request, or L{None} to disable writing of a doctype.  The argument
+        should not include a trailing newline and will default to the HTML5
+        doctype C{'<!DOCTYPE html>'}.
+
+    @returns: NOT_DONE_YET
+
+    @since: 12.1
+    """
+    if doctype is not None:
+        request.write(doctype)
+        request.write(b"\n")
+
+    if _failElement is None:
+        _failElement = FailureElement
+
+    d = flatten(request, element, request.write)
+
+    def eb(failure: Failure) -> Optional[Deferred[None]]:
+        _moduleLog.failure(
+            "An error occurred while rendering the response.", failure=failure
+        )
+        site = getattr(request, "site", None)
+        if site is not None and site.displayTracebacks:
+            assert _failElement is not None
+            return flatten(request, _failElement(failure), request.write)
+        else:
+            request.write(
+                b'<div style="font-size:800%;'
+                b"background-color:#FFF;"
+                b"color:#F00"
+                b'">An error occurred while rendering the response.</div>'
+            )
+            return None
+
+    def finish(result: object, *, request: IRequest = request) -> object:
+        request.finish()
+        return result
+
+    d.addErrback(eb)
+    d.addBoth(finish)
+    return NOT_DONE_YET
diff --git a/contrib/python/Twisted/py3/twisted/web/client.py b/contrib/python/Twisted/py3/twisted/web/client.py
new file mode 100644
index 00000000000..9a0d5e9e10b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/client.py
@@ -0,0 +1,1789 @@
+# -*- test-case-name: twisted.web.test.test_webclient,twisted.web.test.test_agent -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HTTP client.
+"""
+
+
+import collections
+import os
+import warnings
+import zlib
+from functools import wraps
+from typing import Iterable
+from urllib.parse import urldefrag, urljoin, urlunparse as _urlunparse
+
+from zope.interface import implementer
+
+from incremental import Version
+
+from twisted.internet import defer, protocol, task
+from twisted.internet.abstract import isIPv6Address
+from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
+from twisted.internet.interfaces import IOpenSSLContextFactory, IProtocol
+from twisted.logger import Logger
+from twisted.python.compat import nativeString, networkString
+from twisted.python.components import proxyForInterface
+from twisted.python.deprecate import (
+    deprecatedModuleAttribute,
+    getDeprecationWarningString,
+)
+from twisted.python.failure import Failure
+from twisted.web import error, http
+from twisted.web._newclient import _ensureValidMethod, _ensureValidURI
+from twisted.web.http_headers import Headers
+from twisted.web.iweb import (
+    UNKNOWN_LENGTH,
+    IAgent,
+    IAgentEndpointFactory,
+    IBodyProducer,
+    IPolicyForHTTPS,
+    IResponse,
+)
+
+
+def urlunparse(parts):
+    result = _urlunparse(tuple(p.decode("charmap") for p in parts))
+    return result.encode("charmap")
+
+
+class PartialDownloadError(error.Error):
+    """
+    Page was only partially downloaded, we got disconnected in middle.
+
+    @ivar response: All of the response body which was downloaded.
+    """
+
+
+class URI:
+    """
+    A URI object.
+
+    @see: U{https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21}
+    """
+
+    def __init__(self, scheme, netloc, host, port, path, params, query, fragment):
+        """
+        @type scheme: L{bytes}
+        @param scheme: URI scheme specifier.
+
+        @type netloc: L{bytes}
+        @param netloc: Network location component.
+
+        @type host: L{bytes}
+        @param host: Host name. For IPv6 address literals the brackets are
+            stripped.
+
+        @type port: L{int}
+        @param port: Port number.
+
+        @type path: L{bytes}
+        @param path: Hierarchical path.
+
+        @type params: L{bytes}
+        @param params: Parameters for last path segment.
+
+        @type query: L{bytes}
+        @param query: Query string.
+
+        @type fragment: L{bytes}
+        @param fragment: Fragment identifier.
+        """
+        self.scheme = scheme
+        self.netloc = netloc
+        self.host = host.strip(b"[]")
+        self.port = port
+        self.path = path
+        self.params = params
+        self.query = query
+        self.fragment = fragment
+
+    @classmethod
+    def fromBytes(cls, uri, defaultPort=None):
+        """
+        Parse the given URI into a L{URI}.
+
+        @type uri: C{bytes}
+        @param uri: URI to parse.
+
+        @type defaultPort: C{int} or L{None}
+        @param defaultPort: An alternate value to use as the port if the URI
+            does not include one.
+
+        @rtype: L{URI}
+        @return: Parsed URI instance.
+        """
+        uri = uri.strip()
+        scheme, netloc, path, params, query, fragment = http.urlparse(uri)
+
+        if defaultPort is None:
+            if scheme == b"https":
+                defaultPort = 443
+            else:
+                defaultPort = 80
+
+        if b":" in netloc:
+            host, port = netloc.rsplit(b":", 1)
+            try:
+                port = int(port)
+            except ValueError:
+                host, port = netloc, defaultPort
+        else:
+            host, port = netloc, defaultPort
+        return cls(scheme, netloc, host, port, path, params, query, fragment)
+
+    def toBytes(self):
+        """
+        Assemble the individual parts of the I{URI} into a fully formed I{URI}.
+
+        @rtype: C{bytes}
+        @return: A fully formed I{URI}.
+        """
+        return urlunparse(
+            (
+                self.scheme,
+                self.netloc,
+                self.path,
+                self.params,
+                self.query,
+                self.fragment,
+            )
+        )
+
+    @property
+    def originForm(self):
+        """
+        The absolute I{URI} path including I{URI} parameters, query string and
+        fragment identifier.
+
+        @see: U{https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21#section-5.3}
+
+        @return: The absolute path in original form.
+        @rtype: L{bytes}
+        """
+        # The HTTP bis draft says the origin form should not include the
+        # fragment.
+        path = urlunparse((b"", b"", self.path, self.params, self.query, b""))
+        if path == b"":
+            path = b"/"
+        return path
+
+
+def _urljoin(base, url):
+    """
+    Construct a full ("absolute") URL by combining a "base URL" with another
+    URL. Informally, this uses components of the base URL, in particular the
+    addressing scheme, the network location and (part of) the path, to provide
+    missing components in the relative URL.
+
+    Additionally, the fragment identifier is preserved according to the HTTP
+    1.1 bis draft.
+
+    @type base: C{bytes}
+    @param base: Base URL.
+
+    @type url: C{bytes}
+    @param url: URL to combine with C{base}.
+
+    @return: An absolute URL resulting from the combination of C{base} and
+        C{url}.
+
+    @see: L{urllib.parse.urljoin()}
+
+    @see: U{https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-22#section-7.1.2}
+    """
+    base, baseFrag = urldefrag(base)
+    url, urlFrag = urldefrag(urljoin(base, url))
+    return urljoin(url, b"#" + (urlFrag or baseFrag))
+
+
+def _makeGetterFactory(url, factoryFactory, contextFactory=None, *args, **kwargs):
+    """
+    Create and connect an HTTP page getting factory.
+
+    Any additional positional or keyword arguments are used when calling
+    C{factoryFactory}.
+
+    @param factoryFactory: Factory factory that is called with C{url}, C{args}
+        and C{kwargs} to produce the getter
+
+    @param contextFactory: Context factory to use when creating a secure
+        connection, defaulting to L{None}
+
+    @return: The factory created by C{factoryFactory}
+    """
+    uri = URI.fromBytes(_ensureValidURI(url.strip()))
+    factory = factoryFactory(url, *args, **kwargs)
+    from twisted.internet import reactor
+
+    if uri.scheme == b"https":
+        from twisted.internet import ssl
+
+        if contextFactory is None:
+            contextFactory = ssl.ClientContextFactory()
+        reactor.connectSSL(nativeString(uri.host), uri.port, factory, contextFactory)
+    else:
+        reactor.connectTCP(nativeString(uri.host), uri.port, factory)
+    return factory
+
+
+# The code which follows is based on the new HTTP client implementation.  It
+# should be significantly better than anything above, though it is not yet
+# feature equivalent.
+
+from twisted.web._newclient import (
+    HTTP11ClientProtocol,
+    PotentialDataLoss,
+    Request,
+    RequestGenerationFailed,
+    RequestNotSent,
+    RequestTransmissionFailed,
+    Response,
+    ResponseDone,
+    ResponseFailed,
+    ResponseNeverReceived,
+    _WrapperException,
+)
+from twisted.web.error import SchemeNotSupported
+
+try:
+    from OpenSSL import SSL
+except ImportError:
+    SSL = None  # type: ignore[assignment]
+else:
+    from twisted.internet.ssl import (
+        CertificateOptions,
+        optionsForClientTLS,
+        platformTrust,
+    )
+
+
+def _requireSSL(decoratee):
+    """
+    The decorated method requires pyOpenSSL to be present, or it raises
+    L{NotImplementedError}.
+
+    @param decoratee: A function which requires pyOpenSSL.
+    @type decoratee: L{callable}
+
+    @return: A function which raises L{NotImplementedError} if pyOpenSSL is not
+        installed; otherwise, if it is installed, simply return C{decoratee}.
+    @rtype: L{callable}
+    """
+    if SSL is None:
+
+        @wraps(decoratee)
+        def raiseNotImplemented(*a, **kw):
+            """
+            pyOpenSSL is not available.
+
+            @param a: The positional arguments for C{decoratee}.
+
+            @param kw: The keyword arguments for C{decoratee}.
+
+            @raise NotImplementedError: Always.
+            """
+            raise NotImplementedError("SSL support unavailable")
+
+        return raiseNotImplemented
+    return decoratee
+
+
+class WebClientContextFactory:
+    """
+    This class is deprecated.  Please simply use L{Agent} as-is, or if you want
+    to customize something, use L{BrowserLikePolicyForHTTPS}.
+
+    A L{WebClientContextFactory} is an HTTPS policy which totally ignores the
+    hostname and port.  It performs basic certificate verification, however the
+    lack of validation of service identity (e.g.  hostname validation) means it
+    is still vulnerable to man-in-the-middle attacks.  Don't use it any more.
+    """
+
+    def _getCertificateOptions(self, hostname, port):
+        """
+        Return a L{CertificateOptions}.
+
+        @param hostname: ignored
+
+        @param port: ignored
+
+        @return: A new CertificateOptions instance.
+        @rtype: L{CertificateOptions}
+        """
+        return CertificateOptions(method=SSL.SSLv23_METHOD, trustRoot=platformTrust())
+
+    @_requireSSL
+    def getContext(self, hostname, port):
+        """
+        Return an L{OpenSSL.SSL.Context}.
+
+        @param hostname: ignored
+        @param port: ignored
+
+        @return: A new SSL context.
+        @rtype: L{OpenSSL.SSL.Context}
+        """
+        return self._getCertificateOptions(hostname, port).getContext()
+
+
+@implementer(IPolicyForHTTPS)
+class BrowserLikePolicyForHTTPS:
+    """
+    SSL connection creator for web clients.
+    """
+
+    def __init__(self, trustRoot=None):
+        self._trustRoot = trustRoot
+
+    @_requireSSL
+    def creatorForNetloc(self, hostname, port):
+        """
+        Create a L{client connection creator
+        <twisted.internet.interfaces.IOpenSSLClientConnectionCreator>} for a
+        given network location.
+
+        @param hostname: The hostname part of the URI.
+        @type hostname: L{bytes}
+
+        @param port: The port part of the URI.
+        @type port: L{int}
+
+        @return: a connection creator with appropriate verification
+            restrictions set
+        @rtype: L{client connection creator
+            <twisted.internet.interfaces.IOpenSSLClientConnectionCreator>}
+        """
+        return optionsForClientTLS(hostname.decode("ascii"), trustRoot=self._trustRoot)
+
+
+deprecatedModuleAttribute(
+    Version("Twisted", 14, 0, 0),
+    getDeprecationWarningString(
+        WebClientContextFactory,
+        Version("Twisted", 14, 0, 0),
+        replacement=BrowserLikePolicyForHTTPS,
+    ).split("; ")[1],
+    WebClientContextFactory.__module__,
+    WebClientContextFactory.__name__,
+)
+
+
+@implementer(IPolicyForHTTPS)
+class HostnameCachingHTTPSPolicy:
+    """
+    IPolicyForHTTPS that wraps a L{IPolicyForHTTPS} and caches the created
+    L{IOpenSSLClientConnectionCreator}.
+
+    This policy will cache up to C{cacheSize}
+    L{client connection creators <twisted.internet.interfaces.
+    IOpenSSLClientConnectionCreator>} for reuse in subsequent requests to
+    the same hostname.
+
+    @ivar _policyForHTTPS: See C{policyforHTTPS} parameter of L{__init__}.
+
+    @ivar _cache: A cache associating hostnames to their
+        L{client connection creators <twisted.internet.interfaces.
+        IOpenSSLClientConnectionCreator>}.
+    @type _cache: L{collections.OrderedDict}
+
+    @ivar _cacheSize: See C{cacheSize} parameter of L{__init__}.
+
+    @since: Twisted 19.2.0
+    """
+
+    def __init__(self, policyforHTTPS, cacheSize=20):
+        """
+        @param policyforHTTPS: The IPolicyForHTTPS to wrap.
+        @type policyforHTTPS: L{IPolicyForHTTPS}
+
+        @param cacheSize: The maximum size of the hostname cache.
+        @type cacheSize: L{int}
+        """
+        self._policyForHTTPS = policyforHTTPS
+        self._cache = collections.OrderedDict()
+        self._cacheSize = cacheSize
+
+    def creatorForNetloc(self, hostname, port):
+        """
+        Create a L{client connection creator
+        <twisted.internet.interfaces.IOpenSSLClientConnectionCreator>} for a
+        given network location and cache it for future use.
+
+        @param hostname: The hostname part of the URI.
+        @type hostname: L{bytes}
+
+        @param port: The port part of the URI.
+        @type port: L{int}
+
+        @return: a connection creator with appropriate verification
+            restrictions set
+        @rtype: L{client connection creator
+            <twisted.internet.interfaces.IOpenSSLClientConnectionCreator>}
+        """
+        host = hostname.decode("ascii")
+        try:
+            creator = self._cache.pop(host)
+        except KeyError:
+            creator = self._policyForHTTPS.creatorForNetloc(hostname, port)
+
+        self._cache[host] = creator
+        if len(self._cache) > self._cacheSize:
+            self._cache.popitem(last=False)
+
+        return creator
+
+
+@implementer(IOpenSSLContextFactory)
+class _ContextFactoryWithContext:
+    """
+    A L{_ContextFactoryWithContext} is like a
+    L{twisted.internet.ssl.ContextFactory} with a pre-created context.
+
+    @ivar _context: A Context.
+    @type _context: L{OpenSSL.SSL.Context}
+    """
+
+    def __init__(self, context):
+        """
+        Initialize a L{_ContextFactoryWithContext} with a context.
+
+        @param context: An SSL context.
+        @type context: L{OpenSSL.SSL.Context}
+        """
+        self._context = context
+
+    def getContext(self):
+        """
+        Return the context created by
+        L{_DeprecatedToCurrentPolicyForHTTPS._webContextFactory}.
+
+        @return: A context.
+        @rtype: L{OpenSSL.SSL.Context}
+        """
+        return self._context
+
+
+@implementer(IPolicyForHTTPS)
+class _DeprecatedToCurrentPolicyForHTTPS:
+    """
+    Adapt a web context factory to a normal context factory.
+
+    @ivar _webContextFactory: An object providing a getContext method with
+        C{hostname} and C{port} arguments.
+    @type _webContextFactory: L{WebClientContextFactory} (or object with a
+        similar C{getContext} method).
+    """
+
+    def __init__(self, webContextFactory):
+        """
+        Wrap a web context factory in an L{IPolicyForHTTPS}.
+
+        @param webContextFactory: An object providing a getContext method with
+            C{hostname} and C{port} arguments.
+        @type webContextFactory: L{WebClientContextFactory} (or object with a
+            similar C{getContext} method).
+        """
+        self._webContextFactory = webContextFactory
+
+    def creatorForNetloc(self, hostname, port):
+        """
+        Called the wrapped web context factory's C{getContext} method with a
+        hostname and port number and return the resulting context object.
+
+        @param hostname: The hostname part of the URI.
+        @type hostname: L{bytes}
+
+        @param port: The port part of the URI.
+        @type port: L{int}
+
+        @return: A context factory.
+        @rtype: L{IOpenSSLContextFactory}
+        """
+        context = self._webContextFactory.getContext(hostname, port)
+        return _ContextFactoryWithContext(context)
+
+
+@implementer(IBodyProducer)
+class FileBodyProducer:
+    """
+    L{FileBodyProducer} produces bytes from an input file object incrementally
+    and writes them to a consumer.
+
+    Since file-like objects cannot be read from in an event-driven manner,
+    L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from
+    the file.  This process is also paused and resumed based on notifications
+    from the L{IConsumer} provider being written to.
+
+    The file is closed after it has been read, or if the producer is stopped
+    early.
+
+    @ivar _inputFile: Any file-like object, bytes read from which will be
+        written to a consumer.
+
+    @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to
+        schedule all reads.
+
+    @ivar _readSize: The number of bytes to read from C{_inputFile} at a time.
+    """
+
+    def __init__(self, inputFile, cooperator=task, readSize=2**16):
+        self._inputFile = inputFile
+        self._cooperate = cooperator.cooperate
+        self._readSize = readSize
+        self.length = self._determineLength(inputFile)
+
+    def _determineLength(self, fObj):
+        """
+        Determine how many bytes can be read out of C{fObj} (assuming it is not
+        modified from this point on).  If the determination cannot be made,
+        return C{UNKNOWN_LENGTH}.
+        """
+        try:
+            seek = fObj.seek
+            tell = fObj.tell
+        except AttributeError:
+            return UNKNOWN_LENGTH
+        originalPosition = tell()
+        seek(0, os.SEEK_END)
+        end = tell()
+        seek(originalPosition, os.SEEK_SET)
+        return end - originalPosition
+
+    def stopProducing(self):
+        """
+        Permanently stop writing bytes from the file to the consumer by
+        stopping the underlying L{CooperativeTask}.
+        """
+        self._inputFile.close()
+        try:
+            self._task.stop()
+        except task.TaskFinished:
+            pass
+
+    def startProducing(self, consumer):
+        """
+        Start a cooperative task which will read bytes from the input file and
+        write them to C{consumer}.  Return a L{Deferred} which fires after all
+        bytes have been written.  If this L{Deferred} is cancelled before it is
+        fired, stop reading and writing bytes.
+
+        @param consumer: Any L{IConsumer} provider
+        """
+        self._task = self._cooperate(self._writeloop(consumer))
+        d = self._task.whenDone()
+
+        def maybeStopped(reason):
+            if reason.check(defer.CancelledError):
+                self.stopProducing()
+            elif reason.check(task.TaskStopped):
+                pass
+            else:
+                return reason
+            # IBodyProducer.startProducing's Deferred isn't supposed to fire if
+            # stopProducing is called.
+            return defer.Deferred()
+
+        d.addCallbacks(lambda ignored: None, maybeStopped)
+        return d
+
+    def _writeloop(self, consumer):
+        """
+        Return an iterator which reads one chunk of bytes from the input file
+        and writes them to the consumer for each time it is iterated.
+        """
+        while True:
+            bytes = self._inputFile.read(self._readSize)
+            if not bytes:
+                self._inputFile.close()
+                break
+            consumer.write(bytes)
+            yield None
+
+    def pauseProducing(self):
+        """
+        Temporarily suspend copying bytes from the input file to the consumer
+        by pausing the L{CooperativeTask} which drives that activity.
+        """
+        self._task.pause()
+
+    def resumeProducing(self):
+        """
+        Undo the effects of a previous C{pauseProducing} and resume copying
+        bytes to the consumer by resuming the L{CooperativeTask} which drives
+        the write activity.
+        """
+        self._task.resume()
+
+
+class _HTTP11ClientFactory(protocol.Factory):
+    """
+    A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}.
+
+    @ivar _quiescentCallback: The quiescent callback to be passed to protocol
+        instances, used to return them to the connection pool.
+
+    @ivar _metadata: Metadata about the low-level connection details,
+        used to make the repr more useful.
+
+    @since: 11.1
+    """
+
+    def __init__(self, quiescentCallback, metadata):
+        self._quiescentCallback = quiescentCallback
+        self._metadata = metadata
+
+    def __repr__(self) -> str:
+        return "_HTTP11ClientFactory({}, {})".format(
+            self._quiescentCallback, self._metadata
+        )
+
+    def buildProtocol(self, addr):
+        return HTTP11ClientProtocol(self._quiescentCallback)
+
+
+class _RetryingHTTP11ClientProtocol:
+    """
+    A wrapper for L{HTTP11ClientProtocol} that automatically retries requests.
+
+    @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}.
+
+    @ivar _newConnection: A callable that creates a new connection for a
+        retry.
+    """
+
+    def __init__(self, clientProtocol, newConnection):
+        self._clientProtocol = clientProtocol
+        self._newConnection = newConnection
+
+    def _shouldRetry(self, method, exception, bodyProducer):
+        """
+        Indicate whether request should be retried.
+
+        Only returns C{True} if method is idempotent, no response was
+        received, the reason for the failed request was not due to
+        user-requested cancellation, and no body was sent. The latter
+        requirement may be relaxed in the future, and PUT added to approved
+        method list.
+
+        @param method: The method of the request.
+        @type method: L{bytes}
+        """
+        if method not in (b"GET", b"HEAD", b"OPTIONS", b"DELETE", b"TRACE"):
+            return False
+        if not isinstance(
+            exception,
+            (RequestNotSent, RequestTransmissionFailed, ResponseNeverReceived),
+        ):
+            return False
+        if isinstance(exception, _WrapperException):
+            for aFailure in exception.reasons:
+                if aFailure.check(defer.CancelledError):
+                    return False
+        if bodyProducer is not None:
+            return False
+        return True
+
+    def request(self, request):
+        """
+        Do a request, and retry once (with a new connection) if it fails in
+        a retryable manner.
+
+        @param request: A L{Request} instance that will be requested using the
+            wrapped protocol.
+        """
+        d = self._clientProtocol.request(request)
+
+        def failed(reason):
+            if self._shouldRetry(request.method, reason.value, request.bodyProducer):
+                return self._newConnection().addCallback(
+                    lambda connection: connection.request(request)
+                )
+            else:
+                return reason
+
+        d.addErrback(failed)
+        return d
+
+
+class HTTPConnectionPool:
+    """
+    A pool of persistent HTTP connections.
+
+    Features:
+     - Cached connections will eventually time out.
+     - Limits on maximum number of persistent connections.
+
+    Connections are stored using keys, which should be chosen such that any
+    connections stored under a given key can be used interchangeably.
+
+    Failed requests done using previously cached connections will be retried
+    once if they use an idempotent method (e.g. GET), in case the HTTP server
+    timed them out.
+
+    @ivar persistent: Boolean indicating whether connections should be
+        persistent. Connections are persistent by default.
+
+    @ivar maxPersistentPerHost: The maximum number of cached persistent
+        connections for a C{host:port} destination.
+    @type maxPersistentPerHost: C{int}
+
+    @ivar cachedConnectionTimeout: Number of seconds a cached persistent
+        connection will stay open before disconnecting.
+
+    @ivar retryAutomatically: C{boolean} indicating whether idempotent
+        requests should be retried once if no response was received.
+
+    @ivar _factory: The factory used to connect to the proxy.
+
+    @ivar _connections: Map (scheme, host, port) to lists of
+        L{HTTP11ClientProtocol} instances.
+
+    @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a
+        C{IDelayedCall} instance of their timeout.
+
+    @since: 12.1
+    """
+
+    _factory = _HTTP11ClientFactory
+    maxPersistentPerHost = 2
+    cachedConnectionTimeout = 240
+    retryAutomatically = True
+    _log = Logger()
+
+    def __init__(self, reactor, persistent=True):
+        self._reactor = reactor
+        self.persistent = persistent
+        self._connections = {}
+        self._timeouts = {}
+
+    def getConnection(self, key, endpoint):
+        """
+        Supply a connection, newly created or retrieved from the pool, to be
+        used for one HTTP request.
+
+        The connection will remain out of the pool (not available to be
+        returned from future calls to this method) until one HTTP request has
+        been completed over it.
+
+        Afterwards, if the connection is still open, it will automatically be
+        added to the pool.
+
+        @param key: A unique key identifying connections that can be used
+            interchangeably.
+
+        @param endpoint: An endpoint that can be used to open a new connection
+            if no cached connection is available.
+
+        @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol}
+           (or a wrapper) that can be used to send a single HTTP request.
+        """
+        # Try to get cached version:
+        connections = self._connections.get(key)
+        while connections:
+            connection = connections.pop(0)
+            # Cancel timeout:
+            self._timeouts[connection].cancel()
+            del self._timeouts[connection]
+            if connection.state == "QUIESCENT":
+                if self.retryAutomatically:
+                    newConnection = lambda: self._newConnection(key, endpoint)
+                    connection = _RetryingHTTP11ClientProtocol(
+                        connection, newConnection
+                    )
+                return defer.succeed(connection)
+
+        return self._newConnection(key, endpoint)
+
+    def _newConnection(self, key, endpoint):
+        """
+        Create a new connection.
+
+        This implements the new connection code path for L{getConnection}.
+        """
+
+        def quiescentCallback(protocol):
+            self._putConnection(key, protocol)
+
+        factory = self._factory(quiescentCallback, repr(endpoint))
+        return endpoint.connect(factory)
+
+    def _removeConnection(self, key, connection):
+        """
+        Remove a connection from the cache and disconnect it.
+        """
+        connection.transport.loseConnection()
+        self._connections[key].remove(connection)
+        del self._timeouts[connection]
+
+    def _putConnection(self, key, connection):
+        """
+        Return a persistent connection to the pool. This will be called by
+        L{HTTP11ClientProtocol} when the connection becomes quiescent.
+        """
+        if connection.state != "QUIESCENT":
+            # Log with traceback for debugging purposes:
+            try:
+                raise RuntimeError(
+                    "BUG: Non-quiescent protocol added to connection pool."
+                )
+            except BaseException:
+                self._log.failure(
+                    "BUG: Non-quiescent protocol added to connection pool."
+                )
+            return
+        connections = self._connections.setdefault(key, [])
+        if len(connections) == self.maxPersistentPerHost:
+            dropped = connections.pop(0)
+            dropped.transport.loseConnection()
+            self._timeouts[dropped].cancel()
+            del self._timeouts[dropped]
+        connections.append(connection)
+        cid = self._reactor.callLater(
+            self.cachedConnectionTimeout, self._removeConnection, key, connection
+        )
+        self._timeouts[connection] = cid
+
+    def closeCachedConnections(self):
+        """
+        Close all persistent connections and remove them from the pool.
+
+        @return: L{defer.Deferred} that fires when all connections have been
+            closed.
+        """
+        results = []
+        for protocols in self._connections.values():
+            for p in protocols:
+                results.append(p.abort())
+        self._connections = {}
+        for dc in self._timeouts.values():
+            dc.cancel()
+        self._timeouts = {}
+        return defer.gatherResults(results).addCallback(lambda ign: None)
+
+
+class _AgentBase:
+    """
+    Base class offering common facilities for L{Agent}-type classes.
+
+    @ivar _reactor: The C{IReactorTime} implementation which will be used by
+        the pool, and perhaps by subclasses as well.
+
+    @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections.
+    """
+
+    def __init__(self, reactor, pool):
+        if pool is None:
+            pool = HTTPConnectionPool(reactor, False)
+        self._reactor = reactor
+        self._pool = pool
+
+    def _computeHostValue(self, scheme, host, port):
+        """
+        Compute the string to use for the value of the I{Host} header, based on
+        the given scheme, host name, and port number.
+        """
+        if isIPv6Address(nativeString(host)):
+            host = b"[" + host + b"]"
+        if (scheme, port) in ((b"http", 80), (b"https", 443)):
+            return host
+        return b"%b:%d" % (host, port)
+
+    def _requestWithEndpoint(
+        self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath
+    ):
+        """
+        Issue a new request, given the endpoint and the path sent as part of
+        the request.
+        """
+        if not isinstance(method, bytes):
+            raise TypeError(f"method={method!r} is {type(method)}, but must be bytes")
+
+        method = _ensureValidMethod(method)
+
+        # Create minimal headers, if necessary:
+        if headers is None:
+            headers = Headers()
+        if not headers.hasHeader(b"host"):
+            headers = headers.copy()
+            headers.addRawHeader(
+                b"host",
+                self._computeHostValue(
+                    parsedURI.scheme, parsedURI.host, parsedURI.port
+                ),
+            )
+
+        d = self._pool.getConnection(key, endpoint)
+
+        def cbConnected(proto):
+            return proto.request(
+                Request._construct(
+                    method,
+                    requestPath,
+                    headers,
+                    bodyProducer,
+                    persistent=self._pool.persistent,
+                    parsedURI=parsedURI,
+                )
+            )
+
+        d.addCallback(cbConnected)
+        return d
+
+
+@implementer(IAgentEndpointFactory)
+class _StandardEndpointFactory:
+    """
+    Standard HTTP endpoint destinations - TCP for HTTP, TCP+TLS for HTTPS.
+
+    @ivar _policyForHTTPS: A web context factory which will be used to create
+        SSL context objects for any SSL connections the agent needs to make.
+
+    @ivar _connectTimeout: If not L{None}, the timeout passed to
+        L{HostnameEndpoint} for specifying the connection timeout.
+
+    @ivar _bindAddress: If not L{None}, the address passed to
+        L{HostnameEndpoint} for specifying the local address to bind to.
+    """
+
+    def __init__(self, reactor, contextFactory, connectTimeout, bindAddress):
+        """
+        @param reactor: A provider to use to create endpoints.
+        @type reactor: see L{HostnameEndpoint.__init__} for acceptable reactor
+            types.
+
+        @param contextFactory: A factory for TLS contexts, to control the
+            verification parameters of OpenSSL.
+        @type contextFactory: L{IPolicyForHTTPS}.
+
+        @param connectTimeout: The amount of time that this L{Agent} will wait
+            for the peer to accept a connection.
+        @type connectTimeout: L{float} or L{None}
+
+        @param bindAddress: The local address for client sockets to bind to.
+        @type bindAddress: L{bytes} or L{None}
+        """
+        self._reactor = reactor
+        self._policyForHTTPS = contextFactory
+        self._connectTimeout = connectTimeout
+        self._bindAddress = bindAddress
+
+    def endpointForURI(self, uri):
+        """
+        Connect directly over TCP for C{b'http'} scheme, and TLS for
+        C{b'https'}.
+
+        @param uri: L{URI} to connect to.
+
+        @return: Endpoint to connect to.
+        @rtype: L{IStreamClientEndpoint}
+        """
+        kwargs = {}
+        if self._connectTimeout is not None:
+            kwargs["timeout"] = self._connectTimeout
+        kwargs["bindAddress"] = self._bindAddress
+
+        try:
+            host = nativeString(uri.host)
+        except UnicodeDecodeError:
+            raise ValueError(
+                (
+                    "The host of the provided URI ({uri.host!r}) "
+                    "contains non-ASCII octets, it should be ASCII "
+                    "decodable."
+                ).format(uri=uri)
+            )
+
+        endpoint = HostnameEndpoint(self._reactor, host, uri.port, **kwargs)
+        if uri.scheme == b"http":
+            return endpoint
+        elif uri.scheme == b"https":
+            connectionCreator = self._policyForHTTPS.creatorForNetloc(
+                uri.host, uri.port
+            )
+            return wrapClientTLS(connectionCreator, endpoint)
+        else:
+            raise SchemeNotSupported(f"Unsupported scheme: {uri.scheme!r}")
+
+
+@implementer(IAgent)
+class Agent(_AgentBase):
+    """
+    L{Agent} is a very basic HTTP client.  It supports I{HTTP} and I{HTTPS}
+    scheme URIs.
+
+    @ivar _pool: An L{HTTPConnectionPool} instance.
+
+    @ivar _endpointFactory: The L{IAgentEndpointFactory} which will
+        be used to create endpoints for outgoing connections.
+
+    @since: 9.0
+    """
+
+    def __init__(
+        self,
+        reactor,
+        contextFactory=BrowserLikePolicyForHTTPS(),
+        connectTimeout=None,
+        bindAddress=None,
+        pool=None,
+    ):
+        """
+        Create an L{Agent}.
+
+        @param reactor: A reactor for this L{Agent} to place outgoing
+            connections.
+        @type reactor: see L{HostnameEndpoint.__init__} for acceptable reactor
+            types.
+
+        @param contextFactory: A factory for TLS contexts, to control the
+            verification parameters of OpenSSL.  The default is to use a
+            L{BrowserLikePolicyForHTTPS}, so unless you have special
+            requirements you can leave this as-is.
+        @type contextFactory: L{IPolicyForHTTPS}.
+
+        @param connectTimeout: The amount of time that this L{Agent} will wait
+            for the peer to accept a connection.
+        @type connectTimeout: L{float}
+
+        @param bindAddress: The local address for client sockets to bind to.
+        @type bindAddress: L{bytes}
+
+        @param pool: An L{HTTPConnectionPool} instance, or L{None}, in which
+            case a non-persistent L{HTTPConnectionPool} instance will be
+            created.
+        @type pool: L{HTTPConnectionPool}
+        """
+        if not IPolicyForHTTPS.providedBy(contextFactory):
+            warnings.warn(
+                repr(contextFactory)
+                + " was passed as the HTTPS policy for an Agent, but it does "
+                "not provide IPolicyForHTTPS.  Since Twisted 14.0, you must "
+                "pass a provider of IPolicyForHTTPS.",
+                stacklevel=2,
+                category=DeprecationWarning,
+            )
+            contextFactory = _DeprecatedToCurrentPolicyForHTTPS(contextFactory)
+        endpointFactory = _StandardEndpointFactory(
+            reactor, contextFactory, connectTimeout, bindAddress
+        )
+        self._init(reactor, endpointFactory, pool)
+
+    @classmethod
+    def usingEndpointFactory(cls, reactor, endpointFactory, pool=None):
+        """
+        Create a new L{Agent} that will use the endpoint factory to figure
+        out how to connect to the server.
+
+        @param reactor: A reactor for this L{Agent} to place outgoing
+            connections.
+        @type reactor: see L{HostnameEndpoint.__init__} for acceptable reactor
+            types.
+
+        @param endpointFactory: Used to construct endpoints which the
+            HTTP client will connect with.
+        @type endpointFactory: an L{IAgentEndpointFactory} provider.
+
+        @param pool: An L{HTTPConnectionPool} instance, or L{None}, in which
+            case a non-persistent L{HTTPConnectionPool} instance will be
+            created.
+        @type pool: L{HTTPConnectionPool}
+
+        @return: A new L{Agent}.
+        """
+        agent = cls.__new__(cls)
+        agent._init(reactor, endpointFactory, pool)
+        return agent
+
+    def _init(self, reactor, endpointFactory, pool):
+        """
+        Initialize a new L{Agent}.
+
+        @param reactor: A reactor for this L{Agent} to place outgoing
+            connections.
+        @type reactor: see L{HostnameEndpoint.__init__} for acceptable reactor
+            types.
+
+        @param endpointFactory: Used to construct endpoints which the
+            HTTP client will connect with.
+        @type endpointFactory: an L{IAgentEndpointFactory} provider.
+
+        @param pool: An L{HTTPConnectionPool} instance, or L{None}, in which
+            case a non-persistent L{HTTPConnectionPool} instance will be
+            created.
+        @type pool: L{HTTPConnectionPool}
+
+        @return: A new L{Agent}.
+        """
+        _AgentBase.__init__(self, reactor, pool)
+        self._endpointFactory = endpointFactory
+
+    def _getEndpoint(self, uri):
+        """
+        Get an endpoint for the given URI, using C{self._endpointFactory}.
+
+        @param uri: The URI of the request.
+        @type uri: L{URI}
+
+        @return: An endpoint which can be used to connect to given address.
+        """
+        return self._endpointFactory.endpointForURI(uri)
+
+    def request(self, method, uri, headers=None, bodyProducer=None):
+        """
+        Issue a request to the server indicated by the given C{uri}.
+
+        An existing connection from the connection pool may be used or a new
+        one may be created.
+
+        I{HTTP} and I{HTTPS} schemes are supported in C{uri}.
+
+        @see: L{twisted.web.iweb.IAgent.request}
+        """
+        uri = _ensureValidURI(uri.strip())
+        parsedURI = URI.fromBytes(uri)
+        try:
+            endpoint = self._getEndpoint(parsedURI)
+        except SchemeNotSupported:
+            return defer.fail(Failure())
+        key = (parsedURI.scheme, parsedURI.host, parsedURI.port)
+        return self._requestWithEndpoint(
+            key,
+            endpoint,
+            method,
+            parsedURI,
+            headers,
+            bodyProducer,
+            parsedURI.originForm,
+        )
+
+
+@implementer(IAgent)
+class ProxyAgent(_AgentBase):
+    """
+    An HTTP agent able to cross HTTP proxies.
+
+    @ivar _proxyEndpoint: The endpoint used to connect to the proxy.
+
+    @since: 11.1
+    """
+
+    def __init__(self, endpoint, reactor=None, pool=None):
+        if reactor is None:
+            from twisted.internet import reactor
+        _AgentBase.__init__(self, reactor, pool)
+        self._proxyEndpoint = endpoint
+
+    def request(self, method, uri, headers=None, bodyProducer=None):
+        """
+        Issue a new request via the configured proxy.
+        """
+        uri = _ensureValidURI(uri.strip())
+
+        # Cache *all* connections under the same key, since we are only
+        # connecting to a single destination, the proxy:
+        key = ("http-proxy", self._proxyEndpoint)
+
+        # To support proxying HTTPS via CONNECT, we will use key
+        # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that
+        # wraps _proxyEndpoint with an additional callback to do the CONNECT.
+        return self._requestWithEndpoint(
+            key,
+            self._proxyEndpoint,
+            method,
+            URI.fromBytes(uri),
+            headers,
+            bodyProducer,
+            uri,
+        )
+
+
+class _FakeUrllib2Request:
+    """
+    A fake C{urllib2.Request} object for C{cookielib} to work with.
+
+    @see: U{http://docs.python.org/library/urllib2.html#request-objects}
+
+    @type uri: native L{str}
+    @ivar uri: Request URI.
+
+    @type headers: L{twisted.web.http_headers.Headers}
+    @ivar headers: Request headers.
+
+    @type type: native L{str}
+    @ivar type: The scheme of the URI.
+
+    @type host: native L{str}
+    @ivar host: The host[:port] of the URI.
+
+    @since: 11.1
+    """
+
+    def __init__(self, uri):
+        """
+        Create a fake Urllib2 request.
+
+        @param uri: Request URI.
+        @type uri: L{bytes}
+        """
+        self.uri = nativeString(uri)
+        self.headers = Headers()
+
+        _uri = URI.fromBytes(uri)
+        self.type = nativeString(_uri.scheme)
+        self.host = nativeString(_uri.host)
+
+        if (_uri.scheme, _uri.port) not in ((b"http", 80), (b"https", 443)):
+            # If it's not a schema on the regular port, add the port.
+            self.host += ":" + str(_uri.port)
+
+        self.origin_req_host = nativeString(_uri.host)
+        self.unverifiable = lambda _: False
+
+    def has_header(self, header):
+        return self.headers.hasHeader(networkString(header))
+
+    def add_unredirected_header(self, name, value):
+        self.headers.addRawHeader(networkString(name), networkString(value))
+
+    def get_full_url(self):
+        return self.uri
+
+    def get_header(self, name, default=None):
+        headers = self.headers.getRawHeaders(networkString(name), default)
+        if headers is not None:
+            headers = [nativeString(x) for x in headers]
+            return headers[0]
+        return None
+
+    def get_host(self):
+        return self.host
+
+    def get_type(self):
+        return self.type
+
+    def is_unverifiable(self):
+        # In theory this shouldn't be hardcoded.
+        return False
+
+
+class _FakeUrllib2Response:
+    """
+    A fake C{urllib2.Response} object for C{cookielib} to work with.
+
+    @type response: C{twisted.web.iweb.IResponse}
+    @ivar response: Underlying Twisted Web response.
+
+    @since: 11.1
+    """
+
+    def __init__(self, response):
+        self.response = response
+
+    def info(self):
+        class _Meta:
+            def getheaders(zelf, name):
+                # PY2
+                headers = self.response.headers.getRawHeaders(name, [])
+                return headers
+
+            def get_all(zelf, name, default):
+                # PY3
+                headers = self.response.headers.getRawHeaders(
+                    networkString(name), default
+                )
+                h = [nativeString(x) for x in headers]
+                return h
+
+        return _Meta()
+
+
+@implementer(IAgent)
+class CookieAgent:
+    """
+    L{CookieAgent} extends the basic L{Agent} to add RFC-compliant
+    handling of HTTP cookies.  Cookies are written to and extracted
+    from a C{cookielib.CookieJar} instance.
+
+    The same cookie jar instance will be used for any requests through this
+    agent, mutating it whenever a I{Set-Cookie} header appears in a response.
+
+    @type _agent: L{twisted.web.client.Agent}
+    @ivar _agent: Underlying Twisted Web agent to issue requests through.
+
+    @type cookieJar: C{cookielib.CookieJar}
+    @ivar cookieJar: Initialized cookie jar to read cookies from and store
+        cookies to.
+
+    @since: 11.1
+    """
+
+    def __init__(self, agent, cookieJar):
+        self._agent = agent
+        self.cookieJar = cookieJar
+
+    def request(self, method, uri, headers=None, bodyProducer=None):
+        """
+        Issue a new request to the wrapped L{Agent}.
+
+        Send a I{Cookie} header if a cookie for C{uri} is stored in
+        L{CookieAgent.cookieJar}. Cookies are automatically extracted and
+        stored from requests.
+
+        If a C{'cookie'} header appears in C{headers} it will override the
+        automatic cookie header obtained from the cookie jar.
+
+        @see: L{Agent.request}
+        """
+        if headers is None:
+            headers = Headers()
+        lastRequest = _FakeUrllib2Request(uri)
+        # Setting a cookie header explicitly will disable automatic request
+        # cookies.
+        if not headers.hasHeader(b"cookie"):
+            self.cookieJar.add_cookie_header(lastRequest)
+            cookieHeader = lastRequest.get_header("Cookie", None)
+            if cookieHeader is not None:
+                headers = headers.copy()
+                headers.addRawHeader(b"cookie", networkString(cookieHeader))
+
+        d = self._agent.request(method, uri, headers, bodyProducer)
+        d.addCallback(self._extractCookies, lastRequest)
+        return d
+
+    def _extractCookies(self, response, request):
+        """
+        Extract response cookies and store them in the cookie jar.
+
+        @type response: L{twisted.web.iweb.IResponse}
+        @param response: Twisted Web response.
+
+        @param request: A urllib2 compatible request object.
+        """
+        resp = _FakeUrllib2Response(response)
+        self.cookieJar.extract_cookies(resp, request)
+        return response
+
+
+class GzipDecoder(proxyForInterface(IResponse)):  # type: ignore[misc]
+    """
+    A wrapper for a L{Response} instance which handles gzip'ed body.
+
+    @ivar original: The original L{Response} object.
+
+    @since: 11.1
+    """
+
+    def __init__(self, response):
+        self.original = response
+        self.length = UNKNOWN_LENGTH
+
+    def deliverBody(self, protocol):
+        """
+        Override C{deliverBody} to wrap the given C{protocol} with
+        L{_GzipProtocol}.
+        """
+        self.original.deliverBody(_GzipProtocol(protocol, self.original))
+
+
+class _GzipProtocol(proxyForInterface(IProtocol)):  # type: ignore[misc]
+    """
+    A L{Protocol} implementation which wraps another one, transparently
+    decompressing received data.
+
+    @ivar _zlibDecompress: A zlib decompress object used to decompress the data
+        stream.
+
+    @ivar _response: A reference to the original response, in case of errors.
+
+    @since: 11.1
+    """
+
+    def __init__(self, protocol, response):
+        self.original = protocol
+        self._response = response
+        self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS)
+
+    def dataReceived(self, data):
+        """
+        Decompress C{data} with the zlib decompressor, forwarding the raw data
+        to the original protocol.
+        """
+        try:
+            rawData = self._zlibDecompress.decompress(data)
+        except zlib.error:
+            raise ResponseFailed([Failure()], self._response)
+        if rawData:
+            self.original.dataReceived(rawData)
+
+    def connectionLost(self, reason):
+        """
+        Forward the connection lost event, flushing remaining data from the
+        decompressor if any.
+        """
+        try:
+            rawData = self._zlibDecompress.flush()
+        except zlib.error:
+            raise ResponseFailed([reason, Failure()], self._response)
+        if rawData:
+            self.original.dataReceived(rawData)
+        self.original.connectionLost(reason)
+
+
+@implementer(IAgent)
+class ContentDecoderAgent:
+    """
+    An L{Agent} wrapper to handle encoded content.
+
+    It takes care of declaring the support for content in the
+    I{Accept-Encoding} header and automatically decompresses the received data
+    if the I{Content-Encoding} header indicates a supported encoding.
+
+    For example::
+
+        agent = ContentDecoderAgent(Agent(reactor),
+                                    [(b'gzip', GzipDecoder)])
+
+    @param agent: The agent to wrap
+    @type agent: L{IAgent}
+
+    @param decoders: A sequence of (name, decoder) objects. The name
+        declares which encoding the decoder supports. The decoder must accept
+        an L{IResponse} and return an L{IResponse} when called. The order
+        determines how the decoders are advertised to the server. Names must
+        be unique.not be duplicated.
+    @type decoders: sequence of (L{bytes}, L{callable}) tuples
+
+    @since: 11.1
+
+    @see: L{GzipDecoder}
+    """
+
+    def __init__(self, agent, decoders):
+        self._agent = agent
+        self._decoders = dict(decoders)
+        self._supported = b",".join([decoder[0] for decoder in decoders])
+
+    def request(self, method, uri, headers=None, bodyProducer=None):
+        """
+        Send a client request which declares supporting compressed content.
+
+        @see: L{Agent.request}.
+        """
+        if headers is None:
+            headers = Headers()
+        else:
+            headers = headers.copy()
+        headers.addRawHeader(b"accept-encoding", self._supported)
+        deferred = self._agent.request(method, uri, headers, bodyProducer)
+        return deferred.addCallback(self._handleResponse)
+
+    def _handleResponse(self, response):
+        """
+        Check if the response is encoded, and wrap it to handle decompression.
+        """
+        contentEncodingHeaders = response.headers.getRawHeaders(b"content-encoding", [])
+        contentEncodingHeaders = b",".join(contentEncodingHeaders).split(b",")
+        while contentEncodingHeaders:
+            name = contentEncodingHeaders.pop().strip()
+            decoder = self._decoders.get(name)
+            if decoder is not None:
+                response = decoder(response)
+            else:
+                # Add it back
+                contentEncodingHeaders.append(name)
+                break
+        if contentEncodingHeaders:
+            response.headers.setRawHeaders(
+                b"content-encoding", [b",".join(contentEncodingHeaders)]
+            )
+        else:
+            response.headers.removeHeader(b"content-encoding")
+        return response
+
+
+_canonicalHeaderName = Headers()._canonicalNameCaps
+_defaultSensitiveHeaders = frozenset(
+    [
+        b"Authorization",
+        b"Cookie",
+        b"Cookie2",
+        b"Proxy-Authorization",
+        b"WWW-Authenticate",
+    ]
+)
+
+
+@implementer(IAgent)
+class RedirectAgent:
+    """
+    An L{Agent} wrapper which handles HTTP redirects.
+
+    The implementation is rather strict: 301 and 302 behaves like 307, not
+    redirecting automatically on methods different from I{GET} and I{HEAD}.
+
+    See L{BrowserLikeRedirectAgent} for a redirecting Agent that behaves more
+    like a web browser.
+
+    @param redirectLimit: The maximum number of times the agent is allowed to
+        follow redirects before failing with a L{error.InfiniteRedirection}.
+
+    @param sensitiveHeaderNames: An iterable of C{bytes} enumerating the names
+        of headers that must not be transmitted when redirecting to a different
+        origins.  These will be consulted in addition to the protocol-specified
+        set of headers that contain sensitive information.
+
+    @cvar _redirectResponses: A L{list} of HTTP status codes to be redirected
+        for I{GET} and I{HEAD} methods.
+
+    @cvar _seeOtherResponses: A L{list} of HTTP status codes to be redirected
+        for any method and the method altered to I{GET}.
+
+    @since: 11.1
+    """
+
+    _redirectResponses = [
+        http.MOVED_PERMANENTLY,
+        http.FOUND,
+        http.TEMPORARY_REDIRECT,
+        http.PERMANENT_REDIRECT,
+    ]
+    _seeOtherResponses = [http.SEE_OTHER]
+
+    def __init__(
+        self,
+        agent: IAgent,
+        redirectLimit: int = 20,
+        sensitiveHeaderNames: Iterable[bytes] = (),
+    ):
+        self._agent = agent
+        self._redirectLimit = redirectLimit
+        sensitive = {_canonicalHeaderName(each) for each in sensitiveHeaderNames}
+        sensitive.update(_defaultSensitiveHeaders)
+        self._sensitiveHeaderNames = sensitive
+
+    def request(self, method, uri, headers=None, bodyProducer=None):
+        """
+        Send a client request following HTTP redirects.
+
+        @see: L{Agent.request}.
+        """
+        deferred = self._agent.request(method, uri, headers, bodyProducer)
+        return deferred.addCallback(self._handleResponse, method, uri, headers, 0)
+
+    def _resolveLocation(self, requestURI, location):
+        """
+        Resolve the redirect location against the request I{URI}.
+
+        @type requestURI: C{bytes}
+        @param requestURI: The request I{URI}.
+
+        @type location: C{bytes}
+        @param location: The redirect location.
+
+        @rtype: C{bytes}
+        @return: Final resolved I{URI}.
+        """
+        return _urljoin(requestURI, location)
+
+    def _handleRedirect(self, response, method, uri, headers, redirectCount):
+        """
+        Handle a redirect response, checking the number of redirects already
+        followed, and extracting the location header fields.
+        """
+        if redirectCount >= self._redirectLimit:
+            err = error.InfiniteRedirection(
+                response.code, b"Infinite redirection detected", location=uri
+            )
+            raise ResponseFailed([Failure(err)], response)
+        locationHeaders = response.headers.getRawHeaders(b"location", [])
+        if not locationHeaders:
+            err = error.RedirectWithNoLocation(
+                response.code, b"No location header field", uri
+            )
+            raise ResponseFailed([Failure(err)], response)
+        location = self._resolveLocation(uri, locationHeaders[0])
+        if headers:
+            parsedURI = URI.fromBytes(uri)
+            parsedLocation = URI.fromBytes(location)
+            sameOrigin = (
+                (parsedURI.scheme == parsedLocation.scheme)
+                and (parsedURI.host == parsedLocation.host)
+                and (parsedURI.port == parsedLocation.port)
+            )
+            if not sameOrigin:
+                headers = Headers(
+                    {
+                        rawName: rawValue
+                        for rawName, rawValue in headers.getAllRawHeaders()
+                        if rawName not in self._sensitiveHeaderNames
+                    }
+                )
+        deferred = self._agent.request(method, location, headers)
+
+        def _chainResponse(newResponse):
+            newResponse.setPreviousResponse(response)
+            return newResponse
+
+        deferred.addCallback(_chainResponse)
+        return deferred.addCallback(
+            self._handleResponse, method, uri, headers, redirectCount + 1
+        )
+
+    def _handleResponse(self, response, method, uri, headers, redirectCount):
+        """
+        Handle the response, making another request if it indicates a redirect.
+        """
+        if response.code in self._redirectResponses:
+            if method not in (b"GET", b"HEAD"):
+                err = error.PageRedirect(response.code, location=uri)
+                raise ResponseFailed([Failure(err)], response)
+            return self._handleRedirect(response, method, uri, headers, redirectCount)
+        elif response.code in self._seeOtherResponses:
+            return self._handleRedirect(response, b"GET", uri, headers, redirectCount)
+        return response
+
+
+class BrowserLikeRedirectAgent(RedirectAgent):
+    """
+    An L{Agent} wrapper which handles HTTP redirects in the same fashion as web
+    browsers.
+
+    Unlike L{RedirectAgent}, the implementation is more relaxed: 301 and 302
+    behave like 303, redirecting automatically on any method and altering the
+    redirect request to a I{GET}.
+
+    @see: L{RedirectAgent}
+
+    @since: 13.1
+    """
+
+    _redirectResponses = [http.TEMPORARY_REDIRECT]
+    _seeOtherResponses = [
+        http.MOVED_PERMANENTLY,
+        http.FOUND,
+        http.SEE_OTHER,
+        http.PERMANENT_REDIRECT,
+    ]
+
+
+class _ReadBodyProtocol(protocol.Protocol):
+    """
+    Protocol that collects data sent to it.
+
+    This is a helper for L{IResponse.deliverBody}, which collects the body and
+    fires a deferred with it.
+
+    @ivar deferred: See L{__init__}.
+    @ivar status: See L{__init__}.
+    @ivar message: See L{__init__}.
+
+    @ivar dataBuffer: list of byte-strings received
+    @type dataBuffer: L{list} of L{bytes}
+    """
+
+    def __init__(self, status, message, deferred):
+        """
+        @param status: Status of L{IResponse}
+        @ivar status: L{int}
+
+        @param message: Message of L{IResponse}
+        @type message: L{bytes}
+
+        @param deferred: deferred to fire when response is complete
+        @type deferred: L{Deferred} firing with L{bytes}
+        """
+        self.deferred = deferred
+        self.status = status
+        self.message = message
+        self.dataBuffer = []
+
+    def dataReceived(self, data):
+        """
+        Accumulate some more bytes from the response.
+        """
+        self.dataBuffer.append(data)
+
+    def connectionLost(self, reason):
+        """
+        Deliver the accumulated response bytes to the waiting L{Deferred}, if
+        the response body has been completely received without error.
+        """
+        if reason.check(ResponseDone):
+            self.deferred.callback(b"".join(self.dataBuffer))
+        elif reason.check(PotentialDataLoss):
+            self.deferred.errback(
+                PartialDownloadError(
+                    self.status, self.message, b"".join(self.dataBuffer)
+                )
+            )
+        else:
+            self.deferred.errback(reason)
+
+
+def readBody(response: IResponse) -> defer.Deferred[bytes]:
+    """
+    Get the body of an L{IResponse} and return it as a byte string.
+
+    This is a helper function for clients that don't want to incrementally
+    receive the body of an HTTP response.
+
+    @param response: The HTTP response for which the body will be read.
+    @type response: L{IResponse} provider
+
+    @return: A L{Deferred} which will fire with the body of the response.
+        Cancelling it will close the connection to the server immediately.
+    """
+
+    def cancel(deferred: defer.Deferred[bytes]) -> None:
+        """
+        Cancel a L{readBody} call, close the connection to the HTTP server
+        immediately, if it is still open.
+
+        @param deferred: The cancelled L{defer.Deferred}.
+        """
+        abort = getAbort()
+        if abort is not None:
+            abort()
+
+    d: defer.Deferred[bytes] = defer.Deferred(cancel)
+    protocol = _ReadBodyProtocol(response.code, response.phrase, d)
+
+    def getAbort():
+        return getattr(protocol.transport, "abortConnection", None)
+
+    response.deliverBody(protocol)
+
+    if protocol.transport is not None and getAbort() is None:
+        warnings.warn(
+            "Using readBody with a transport that does not have an "
+            "abortConnection method",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+
+    return d
+
+
+__all__ = [
+    "Agent",
+    "BrowserLikePolicyForHTTPS",
+    "BrowserLikeRedirectAgent",
+    "ContentDecoderAgent",
+    "CookieAgent",
+    "GzipDecoder",
+    "HTTPConnectionPool",
+    "PartialDownloadError",
+    "ProxyAgent",
+    "readBody",
+    "RedirectAgent",
+    "RequestGenerationFailed",
+    "RequestTransmissionFailed",
+    "Response",
+    "ResponseDone",
+    "ResponseFailed",
+    "ResponseNeverReceived",
+    "URI",
+]
diff --git a/contrib/python/Twisted/py3/twisted/web/demo.py b/contrib/python/Twisted/py3/twisted/web/demo.py
new file mode 100644
index 00000000000..2c8a3b69be6
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/demo.py
@@ -0,0 +1,27 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+I am a simple test resource.
+"""
+
+
+from twisted.web import static
+
+
+class Test(static.Data):
+    isLeaf = True
+
+    def __init__(self):
+        static.Data.__init__(
+            self,
+            b"""
+            <html>
+            <head><title>Twisted Web Demo</title><head>
+            <body>
+            Hello! This is a Twisted Web test page.
+            </body>
+            </html>
+            """,
+            "text/html",
+        )
diff --git a/contrib/python/Twisted/py3/twisted/web/distrib.py b/contrib/python/Twisted/py3/twisted/web/distrib.py
new file mode 100644
index 00000000000..4f25c03ee86
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/distrib.py
@@ -0,0 +1,390 @@
+# -*- test-case-name: twisted.web.test.test_distrib -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Distributed web servers.
+
+This is going to have to be refactored so that argument parsing is done
+by each subprocess and not by the main web server (i.e. GET, POST etc.).
+"""
+
+import copy
+import os
+import sys
+
+try:
+    import pwd
+except ImportError:
+    pwd = None  # type: ignore[assignment]
+from io import BytesIO
+from xml.dom.minidom import getDOMImplementation
+
+from twisted.internet import address, reactor
+from twisted.logger import Logger
+from twisted.persisted import styles
+from twisted.spread import pb
+from twisted.spread.banana import SIZE_LIMIT
+from twisted.web import http, resource, server, static, util
+from twisted.web.http_headers import Headers
+
+
+class _ReferenceableProducerWrapper(pb.Referenceable):
+    def __init__(self, producer):
+        self.producer = producer
+
+    def remote_resumeProducing(self):
+        self.producer.resumeProducing()
+
+    def remote_pauseProducing(self):
+        self.producer.pauseProducing()
+
+    def remote_stopProducing(self):
+        self.producer.stopProducing()
+
+
+class Request(pb.RemoteCopy, server.Request):
+    """
+    A request which was received by a L{ResourceSubscription} and sent via
+    PB to a distributed node.
+    """
+
+    def setCopyableState(self, state):
+        """
+        Initialize this L{twisted.web.distrib.Request} based on the copied
+        state so that it closely resembles a L{twisted.web.server.Request}.
+        """
+        for k in "host", "client":
+            tup = state[k]
+            addrdesc = {"INET": "TCP", "UNIX": "UNIX"}[tup[0]]
+            addr = {
+                "TCP": lambda: address.IPv4Address(addrdesc, tup[1], tup[2]),
+                "UNIX": lambda: address.UNIXAddress(tup[1]),
+            }[addrdesc]()
+            state[k] = addr
+        state["requestHeaders"] = Headers(dict(state["requestHeaders"]))
+        pb.RemoteCopy.setCopyableState(self, state)
+        # Emulate the local request interface --
+        self.content = BytesIO(self.content_data)
+        self.finish = self.remote.remoteMethod("finish")
+        self.setHeader = self.remote.remoteMethod("setHeader")
+        self.addCookie = self.remote.remoteMethod("addCookie")
+        self.setETag = self.remote.remoteMethod("setETag")
+        self.setResponseCode = self.remote.remoteMethod("setResponseCode")
+        self.setLastModified = self.remote.remoteMethod("setLastModified")
+
+        # To avoid failing if a resource tries to write a very long string
+        # all at once, this one will be handled slightly differently.
+        self._write = self.remote.remoteMethod("write")
+
+    def write(self, bytes):
+        """
+        Write the given bytes to the response body.
+
+        @param bytes: The bytes to write.  If this is longer than 640k, it
+            will be split up into smaller pieces.
+        """
+        start = 0
+        end = SIZE_LIMIT
+        while True:
+            self._write(bytes[start:end])
+            start += SIZE_LIMIT
+            end += SIZE_LIMIT
+            if start >= len(bytes):
+                break
+
+    def registerProducer(self, producer, streaming):
+        self.remote.callRemote(
+            "registerProducer", _ReferenceableProducerWrapper(producer), streaming
+        ).addErrback(self.fail)
+
+    def unregisterProducer(self):
+        self.remote.callRemote("unregisterProducer").addErrback(self.fail)
+
+    def fail(self, failure):
+        self._log.failure("", failure=failure)
+
+
+pb.setUnjellyableForClass(server.Request, Request)
+
+
+class Issue:
+    _log = Logger()
+
+    def __init__(self, request):
+        self.request = request
+
+    def finished(self, result):
+        if result is not server.NOT_DONE_YET:
+            assert isinstance(result, str), "return value not a string"
+            self.request.write(result)
+            self.request.finish()
+
+    def failed(self, failure):
+        # XXX: Argh. FIXME.
+        failure = str(failure)
+        self.request.write(
+            resource._UnsafeErrorPage(
+                http.INTERNAL_SERVER_ERROR,
+                "Server Connection Lost",
+                # GHSA-vg46-2rrj-3647 note: _PRE does HTML-escape the input.
+                "Connection to distributed server lost:" + util._PRE(failure),
+            ).render(self.request)
+        )
+        self.request.finish()
+        self._log.info(failure)
+
+
+class ResourceSubscription(resource.Resource):
+    isLeaf = 1
+    waiting = 0
+    _log = Logger()
+
+    def __init__(self, host, port):
+        resource.Resource.__init__(self)
+        self.host = host
+        self.port = port
+        self.pending = []
+        self.publisher = None
+
+    def __getstate__(self):
+        """Get persistent state for this ResourceSubscription."""
+        # When I unserialize,
+        state = copy.copy(self.__dict__)
+        # Publisher won't be connected...
+        state["publisher"] = None
+        # I won't be making a connection
+        state["waiting"] = 0
+        # There will be no pending requests.
+        state["pending"] = []
+        return state
+
+    def connected(self, publisher):
+        """I've connected to a publisher; I'll now send all my requests."""
+        self._log.info("connected to publisher")
+        publisher.broker.notifyOnDisconnect(self.booted)
+        self.publisher = publisher
+        self.waiting = 0
+        for request in self.pending:
+            self.render(request)
+        self.pending = []
+
+    def notConnected(self, msg):
+        """I can't connect to a publisher; I'll now reply to all pending
+        requests.
+        """
+        self._log.info("could not connect to distributed web service: {msg}", msg=msg)
+        self.waiting = 0
+        self.publisher = None
+        for request in self.pending:
+            request.write("Unable to connect to distributed server.")
+            request.finish()
+        self.pending = []
+
+    def booted(self):
+        self.notConnected("connection dropped")
+
+    def render(self, request):
+        """Render this request, from my server.
+
+        This will always be asynchronous, and therefore return NOT_DONE_YET.
+        It spins off a request to the pb client, and either adds it to the list
+        of pending issues or requests it immediately, depending on if the
+        client is already connected.
+        """
+        if not self.publisher:
+            self.pending.append(request)
+            if not self.waiting:
+                self.waiting = 1
+                bf = pb.PBClientFactory()
+                timeout = 10
+                if self.host == "unix":
+                    reactor.connectUNIX(self.port, bf, timeout)
+                else:
+                    reactor.connectTCP(self.host, self.port, bf, timeout)
+                d = bf.getRootObject()
+                d.addCallbacks(self.connected, self.notConnected)
+
+        else:
+            i = Issue(request)
+            self.publisher.callRemote("request", request).addCallbacks(
+                i.finished, i.failed
+            )
+        return server.NOT_DONE_YET
+
+
+class ResourcePublisher(pb.Root, styles.Versioned):
+    """
+    L{ResourcePublisher} exposes a remote API which can be used to respond
+    to request.
+
+    @ivar site: The site which will be used for resource lookup.
+    @type site: L{twisted.web.server.Site}
+    """
+
+    _log = Logger()
+
+    def __init__(self, site):
+        self.site = site
+
+    persistenceVersion = 2
+
+    def upgradeToVersion2(self):
+        self.application.authorizer.removeIdentity("web")
+        del self.application.services[self.serviceName]
+        del self.serviceName
+        del self.application
+        del self.perspectiveName
+
+    def getPerspectiveNamed(self, name):
+        return self
+
+    def remote_request(self, request):
+        """
+        Look up the resource for the given request and render it.
+        """
+        res = self.site.getResourceFor(request)
+        self._log.info(request)
+        result = res.render(request)
+        if result is not server.NOT_DONE_YET:
+            request.write(result)
+            request.finish()
+        return server.NOT_DONE_YET
+
+
+class UserDirectory(resource.Resource):
+    """
+    A resource which lists available user resources and serves them as
+    children.
+
+    @ivar _pwd: An object like L{pwd} which is used to enumerate users and
+        their home directories.
+    """
+
+    userDirName = "public_html"
+    userSocketName = ".twistd-web-pb"
+
+    template = """
+<html>
+    <head>
+    <title>twisted.web.distrib.UserDirectory</title>
+    <style>
+
+    a
+    {
+        font-family: Lucida, Verdana, Helvetica, Arial, sans-serif;
+        color: #369;
+        text-decoration: none;
+    }
+
+    th
+    {
+        font-family: Lucida, Verdana, Helvetica, Arial, sans-serif;
+        font-weight: bold;
+        text-decoration: none;
+        text-align: left;
+    }
+
+    pre, code
+    {
+        font-family: "Courier New", Courier, monospace;
+    }
+
+    p, body, td, ol, ul, menu, blockquote, div
+    {
+        font-family: Lucida, Verdana, Helvetica, Arial, sans-serif;
+        color: #000;
+    }
+    </style>
+    </head>
+
+    <body>
+    <h1>twisted.web.distrib.UserDirectory</h1>
+
+    %(users)s
+</body>
+</html>
+"""
+
+    def __init__(self, userDatabase=None):
+        resource.Resource.__init__(self)
+        if userDatabase is None:
+            userDatabase = pwd
+        self._pwd = userDatabase
+
+    def _users(self):
+        """
+        Return a list of two-tuples giving links to user resources and text to
+        associate with those links.
+        """
+        users = []
+        for user in self._pwd.getpwall():
+            name, passwd, uid, gid, gecos, dir, shell = user
+            realname = gecos.split(",")[0]
+            if not realname:
+                realname = name
+            if os.path.exists(os.path.join(dir, self.userDirName)):
+                users.append((name, realname + " (file)"))
+            twistdsock = os.path.join(dir, self.userSocketName)
+            if os.path.exists(twistdsock):
+                linkName = name + ".twistd"
+                users.append((linkName, realname + " (twistd)"))
+        return users
+
+    def render_GET(self, request):
+        """
+        Render as HTML a listing of all known users with links to their
+        personal resources.
+        """
+
+        domImpl = getDOMImplementation()
+        newDoc = domImpl.createDocument(None, "ul", None)
+        listing = newDoc.documentElement
+        for link, text in self._users():
+            linkElement = newDoc.createElement("a")
+            linkElement.setAttribute("href", link + "/")
+            textNode = newDoc.createTextNode(text)
+            linkElement.appendChild(textNode)
+            item = newDoc.createElement("li")
+            item.appendChild(linkElement)
+            listing.appendChild(item)
+
+        htmlDoc = self.template % ({"users": listing.toxml()})
+        return htmlDoc.encode("utf-8")
+
+    def getChild(self, name, request):
+        if name == b"":
+            return self
+
+        td = b".twistd"
+
+        if name.endswith(td):
+            username = name[: -len(td)]
+            sub = 1
+        else:
+            username = name
+            sub = 0
+        try:
+            # Decode using the filesystem encoding to reverse a transformation
+            # done in the pwd module.
+            (
+                pw_name,
+                pw_passwd,
+                pw_uid,
+                pw_gid,
+                pw_gecos,
+                pw_dir,
+                pw_shell,
+            ) = self._pwd.getpwnam(username.decode(sys.getfilesystemencoding()))
+        except KeyError:
+            return resource._UnsafeNoResource()
+        if sub:
+            twistdsock = os.path.join(pw_dir, self.userSocketName)
+            rs = ResourceSubscription("unix", twistdsock)
+            self.putChild(name, rs)
+            return rs
+        else:
+            path = os.path.join(pw_dir, self.userDirName)
+            if not os.path.exists(path):
+                return resource._UnsafeNoResource()
+            return static.File(path)
diff --git a/contrib/python/Twisted/py3/twisted/web/domhelpers.py b/contrib/python/Twisted/py3/twisted/web/domhelpers.py
new file mode 100644
index 00000000000..326c3f8485f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/domhelpers.py
@@ -0,0 +1,313 @@
+# -*- test-case-name: twisted.web.test.test_domhelpers -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A library for performing interesting tasks with DOM objects.
+
+This module is now deprecated.
+"""
+import warnings
+from io import StringIO
+
+from incremental import Version, getVersionString
+
+from twisted.web import microdom
+from twisted.web.microdom import escape, getElementsByTagName, unescape
+
+warningString = "twisted.web.domhelpers was deprecated at {}".format(
+    getVersionString(Version("Twisted", 23, 10, 0))
+)
+warnings.warn(warningString, DeprecationWarning, stacklevel=3)
+
+
+# These modules are imported here as a shortcut.
+escape
+getElementsByTagName
+
+
+class NodeLookupError(Exception):
+    pass
+
+
+def substitute(request, node, subs):
+    """
+    Look through the given node's children for strings, and
+    attempt to do string substitution with the given parameter.
+    """
+    for child in node.childNodes:
+        if hasattr(child, "nodeValue") and child.nodeValue:
+            child.replaceData(0, len(child.nodeValue), child.nodeValue % subs)
+        substitute(request, child, subs)
+
+
+def _get(node, nodeId, nodeAttrs=("id", "class", "model", "pattern")):
+    """
+    (internal) Get a node with the specified C{nodeId} as any of the C{class},
+    C{id} or C{pattern} attributes.
+    """
+
+    if hasattr(node, "hasAttributes") and node.hasAttributes():
+        for nodeAttr in nodeAttrs:
+            if str(node.getAttribute(nodeAttr)) == nodeId:
+                return node
+    if node.hasChildNodes():
+        if hasattr(node.childNodes, "length"):
+            length = node.childNodes.length
+        else:
+            length = len(node.childNodes)
+        for childNum in range(length):
+            result = _get(node.childNodes[childNum], nodeId)
+            if result:
+                return result
+
+
+def get(node, nodeId):
+    """
+    Get a node with the specified C{nodeId} as any of the C{class},
+    C{id} or C{pattern} attributes. If there is no such node, raise
+    L{NodeLookupError}.
+    """
+    result = _get(node, nodeId)
+    if result:
+        return result
+    raise NodeLookupError(nodeId)
+
+
+def getIfExists(node, nodeId):
+    """
+    Get a node with the specified C{nodeId} as any of the C{class},
+    C{id} or C{pattern} attributes.  If there is no such node, return
+    L{None}.
+    """
+    return _get(node, nodeId)
+
+
+def getAndClear(node, nodeId):
+    """Get a node with the specified C{nodeId} as any of the C{class},
+    C{id} or C{pattern} attributes. If there is no such node, raise
+    L{NodeLookupError}. Remove all child nodes before returning.
+    """
+    result = get(node, nodeId)
+    if result:
+        clearNode(result)
+    return result
+
+
+def clearNode(node):
+    """
+    Remove all children from the given node.
+    """
+    node.childNodes[:] = []
+
+
+def locateNodes(nodeList, key, value, noNesting=1):
+    """
+    Find subnodes in the given node where the given attribute
+    has the given value.
+    """
+    returnList = []
+    if not isinstance(nodeList, type([])):
+        return locateNodes(nodeList.childNodes, key, value, noNesting)
+    for childNode in nodeList:
+        if not hasattr(childNode, "getAttribute"):
+            continue
+        if str(childNode.getAttribute(key)) == value:
+            returnList.append(childNode)
+            if noNesting:
+                continue
+        returnList.extend(locateNodes(childNode, key, value, noNesting))
+    return returnList
+
+
+def superSetAttribute(node, key, value):
+    if not hasattr(node, "setAttribute"):
+        return
+    node.setAttribute(key, value)
+    if node.hasChildNodes():
+        for child in node.childNodes:
+            superSetAttribute(child, key, value)
+
+
+def superPrependAttribute(node, key, value):
+    if not hasattr(node, "setAttribute"):
+        return
+    old = node.getAttribute(key)
+    if old:
+        node.setAttribute(key, value + "/" + old)
+    else:
+        node.setAttribute(key, value)
+    if node.hasChildNodes():
+        for child in node.childNodes:
+            superPrependAttribute(child, key, value)
+
+
+def superAppendAttribute(node, key, value):
+    if not hasattr(node, "setAttribute"):
+        return
+    old = node.getAttribute(key)
+    if old:
+        node.setAttribute(key, old + "/" + value)
+    else:
+        node.setAttribute(key, value)
+    if node.hasChildNodes():
+        for child in node.childNodes:
+            superAppendAttribute(child, key, value)
+
+
+def gatherTextNodes(iNode, dounescape=0, joinWith=""):
+    """Visit each child node and collect its text data, if any, into a string.
+    For example::
+        >>> doc=microdom.parseString('<a>1<b>2<c>3</c>4</b></a>')
+        >>> gatherTextNodes(doc.documentElement)
+        '1234'
+    With dounescape=1, also convert entities back into normal characters.
+    @return: the gathered nodes as a single string
+    @rtype: str"""
+    gathered = []
+    gathered_append = gathered.append
+    slice = [iNode]
+    while len(slice) > 0:
+        c = slice.pop(0)
+        if hasattr(c, "nodeValue") and c.nodeValue is not None:
+            if dounescape:
+                val = unescape(c.nodeValue)
+            else:
+                val = c.nodeValue
+            gathered_append(val)
+        slice[:0] = c.childNodes
+    return joinWith.join(gathered)
+
+
+class RawText(microdom.Text):
+    """This is an evil and horrible speed hack. Basically, if you have a big
+    chunk of XML that you want to insert into the DOM, but you don't want to
+    incur the cost of parsing it, you can construct one of these and insert it
+    into the DOM. This will most certainly only work with microdom as the API
+    for converting nodes to xml is different in every DOM implementation.
+
+    This could be improved by making this class a Lazy parser, so if you
+    inserted this into the DOM and then later actually tried to mutate this
+    node, it would be parsed then.
+    """
+
+    def writexml(
+        self,
+        writer,
+        indent="",
+        addindent="",
+        newl="",
+        strip=0,
+        nsprefixes=None,
+        namespace=None,
+    ):
+        writer.write(f"{indent}{self.data}{newl}")
+
+
+def findNodes(parent, matcher, accum=None):
+    if accum is None:
+        accum = []
+    if not parent.hasChildNodes():
+        return accum
+    for child in parent.childNodes:
+        # print child, child.nodeType, child.nodeName
+        if matcher(child):
+            accum.append(child)
+        findNodes(child, matcher, accum)
+    return accum
+
+
+def findNodesShallowOnMatch(parent, matcher, recurseMatcher, accum=None):
+    if accum is None:
+        accum = []
+    if not parent.hasChildNodes():
+        return accum
+    for child in parent.childNodes:
+        # print child, child.nodeType, child.nodeName
+        if matcher(child):
+            accum.append(child)
+        if recurseMatcher(child):
+            findNodesShallowOnMatch(child, matcher, recurseMatcher, accum)
+    return accum
+
+
+def findNodesShallow(parent, matcher, accum=None):
+    if accum is None:
+        accum = []
+    if not parent.hasChildNodes():
+        return accum
+    for child in parent.childNodes:
+        if matcher(child):
+            accum.append(child)
+        else:
+            findNodes(child, matcher, accum)
+    return accum
+
+
+def findElementsWithAttributeShallow(parent, attribute):
+    """
+    Return an iterable of the elements which are direct children of C{parent}
+    and which have the C{attribute} attribute.
+    """
+    return findNodesShallow(
+        parent,
+        lambda n: getattr(n, "tagName", None) is not None and n.hasAttribute(attribute),
+    )
+
+
+def findElements(parent, matcher):
+    """
+    Return an iterable of the elements which are children of C{parent} for
+    which the predicate C{matcher} returns true.
+    """
+    return findNodes(
+        parent,
+        lambda n, matcher=matcher: getattr(n, "tagName", None) is not None
+        and matcher(n),
+    )
+
+
+def findElementsWithAttribute(parent, attribute, value=None):
+    if value:
+        return findElements(
+            parent,
+            lambda n, attribute=attribute, value=value: n.hasAttribute(attribute)
+            and n.getAttribute(attribute) == value,
+        )
+    else:
+        return findElements(
+            parent, lambda n, attribute=attribute: n.hasAttribute(attribute)
+        )
+
+
+def findNodesNamed(parent, name):
+    return findNodes(parent, lambda n, name=name: n.nodeName == name)
+
+
+def writeNodeData(node, oldio):
+    for subnode in node.childNodes:
+        if hasattr(subnode, "data"):
+            oldio.write("" + subnode.data)
+        else:
+            writeNodeData(subnode, oldio)
+
+
+def getNodeText(node):
+    oldio = StringIO()
+    writeNodeData(node, oldio)
+    return oldio.getvalue()
+
+
+def getParents(node):
+    l = []
+    while node:
+        l.append(node)
+        node = node.parentNode
+    return l
+
+
+def namedChildren(parent, nodeName):
+    """namedChildren(parent, nodeName) -> children (not descendants) of parent
+    that have tagName == nodeName
+    """
+    return [n for n in parent.childNodes if getattr(n, "tagName", "") == nodeName]
diff --git a/contrib/python/Twisted/py3/twisted/web/error.py b/contrib/python/Twisted/py3/twisted/web/error.py
new file mode 100644
index 00000000000..cc151d42053
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/error.py
@@ -0,0 +1,442 @@
+# -*- test-case-name: twisted.web.test.test_error -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Exception definitions for L{twisted.web}.
+"""
+
+__all__ = [
+    "Error",
+    "PageRedirect",
+    "InfiniteRedirection",
+    "RenderError",
+    "MissingRenderMethod",
+    "MissingTemplateLoader",
+    "UnexposedMethodError",
+    "UnfilledSlot",
+    "UnsupportedType",
+    "FlattenerError",
+    "RedirectWithNoLocation",
+]
+
+
+from collections.abc import Sequence
+from typing import Optional, Union, cast
+
+from twisted.python.compat import nativeString
+from twisted.web._responses import RESPONSES
+
+
+def _codeToMessage(code: Union[int, bytes]) -> Optional[bytes]:
+    """
+    Returns the response message corresponding to an HTTP code, or None
+    if the code is unknown or unrecognized.
+
+    @param code: HTTP status code, for example C{http.NOT_FOUND}.
+
+    @return: A string message or none
+    """
+    try:
+        return RESPONSES.get(int(code))
+    except (ValueError, AttributeError):
+        return None
+
+
+class Error(Exception):
+    """
+    A basic HTTP error.
+
+    @ivar status: Refers to an HTTP status code, for example C{http.NOT_FOUND}.
+
+    @param message: A short error message, for example "NOT FOUND".
+
+    @ivar response: A complete HTML document for an error page.
+    """
+
+    status: bytes
+    message: Optional[bytes]
+    response: Optional[bytes]
+
+    def __init__(
+        self,
+        code: Union[int, bytes],
+        message: Optional[bytes] = None,
+        response: Optional[bytes] = None,
+    ) -> None:
+        """
+        Initializes a basic exception.
+
+        @type code: L{bytes} or L{int}
+        @param code: Refers to an HTTP status code (for example, 200) either as
+            an integer or a bytestring representing such. If no C{message} is
+            given, C{code} is mapped to a descriptive bytestring that is used
+            instead.
+
+        @type message: L{bytes}
+        @param message: A short error message, for example C{b"NOT FOUND"}.
+
+        @type response: L{bytes}
+        @param response: A complete HTML document for an error page.
+        """
+
+        message = message or _codeToMessage(code)
+
+        Exception.__init__(self, code, message, response)
+
+        if isinstance(code, int):
+            # If we're given an int, convert it to a bytestring
+            # downloadPage gives a bytes, Agent gives an int, and it worked by
+            # accident previously, so just make it keep working.
+            code = b"%d" % (code,)
+        elif len(code) != 3 or not code.isdigit():
+            # Status codes must be 3 digits. See
+            # https://httpwg.org/specs/rfc9110.html#status.code.extensibility
+            raise ValueError(f"Not a valid HTTP status code: {code!r}")
+
+        self.status = code
+        self.message = message
+        self.response = response
+
+    def __str__(self) -> str:
+        s = self.status
+        if self.message:
+            s += b" " + self.message
+        return nativeString(s)
+
+
+class PageRedirect(Error):
+    """
+    A request resulted in an HTTP redirect.
+
+    @ivar location: The location of the redirect which was not followed.
+    """
+
+    location: Optional[bytes]
+
+    def __init__(
+        self,
+        code: Union[int, bytes],
+        message: Optional[bytes] = None,
+        response: Optional[bytes] = None,
+        location: Optional[bytes] = None,
+    ) -> None:
+        """
+        Initializes a page redirect exception.
+
+        @type code: L{bytes}
+        @param code: Refers to an HTTP status code, for example
+            C{http.NOT_FOUND}. If no C{message} is given, C{code} is mapped to a
+            descriptive string that is used instead.
+
+        @type message: L{bytes}
+        @param message: A short error message, for example C{b"NOT FOUND"}.
+
+        @type response: L{bytes}
+        @param response: A complete HTML document for an error page.
+
+        @type location: L{bytes}
+        @param location: The location response-header field value. It is an
+            absolute URI used to redirect the receiver to a location other than
+            the Request-URI so the request can be completed.
+        """
+        Error.__init__(self, code, message, response)
+        if self.message and location:
+            self.message = self.message + b" to " + location
+        self.location = location
+
+
+class InfiniteRedirection(Error):
+    """
+    HTTP redirection is occurring endlessly.
+
+    @ivar location: The first URL in the series of redirections which was
+        not followed.
+    """
+
+    location: Optional[bytes]
+
+    def __init__(
+        self,
+        code: Union[int, bytes],
+        message: Optional[bytes] = None,
+        response: Optional[bytes] = None,
+        location: Optional[bytes] = None,
+    ) -> None:
+        """
+        Initializes an infinite redirection exception.
+
+        @param code: Refers to an HTTP status code, for example
+            C{http.NOT_FOUND}. If no C{message} is given, C{code} is mapped to a
+            descriptive string that is used instead.
+
+        @param message: A short error message, for example C{b"NOT FOUND"}.
+
+        @param response: A complete HTML document for an error page.
+
+        @param location: The location response-header field value. It is an
+            absolute URI used to redirect the receiver to a location other than
+            the Request-URI so the request can be completed.
+        """
+        Error.__init__(self, code, message, response)
+        if self.message and location:
+            self.message = self.message + b" to " + location
+        self.location = location
+
+
+class RedirectWithNoLocation(Error):
+    """
+    Exception passed to L{ResponseFailed} if we got a redirect without a
+    C{Location} header field.
+
+    @type uri: L{bytes}
+    @ivar uri: The URI which failed to give a proper location header
+        field.
+
+    @since: 11.1
+    """
+
+    message: bytes
+    uri: bytes
+
+    def __init__(self, code: Union[bytes, int], message: bytes, uri: bytes) -> None:
+        """
+        Initializes a page redirect exception when no location is given.
+
+        @type code: L{bytes}
+        @param code: Refers to an HTTP status code, for example
+            C{http.NOT_FOUND}. If no C{message} is given, C{code} is mapped to
+            a descriptive string that is used instead.
+
+        @type message: L{bytes}
+        @param message: A short error message.
+
+        @type uri: L{bytes}
+        @param uri: The URI which failed to give a proper location header
+            field.
+        """
+        Error.__init__(self, code, message)
+        self.message = self.message + b" to " + uri
+        self.uri = uri
+
+
+class UnsupportedMethod(Exception):
+    """
+    Raised by a resource when faced with a strange request method.
+
+    RFC 2616 (HTTP 1.1) gives us two choices when faced with this situation:
+    If the type of request is known to us, but not allowed for the requested
+    resource, respond with NOT_ALLOWED.  Otherwise, if the request is something
+    we don't know how to deal with in any case, respond with NOT_IMPLEMENTED.
+
+    When this exception is raised by a Resource's render method, the server
+    will make the appropriate response.
+
+    This exception's first argument MUST be a sequence of the methods the
+    resource *does* support.
+    """
+
+    allowedMethods = ()
+
+    def __init__(self, allowedMethods, *args):
+        Exception.__init__(self, allowedMethods, *args)
+        self.allowedMethods = allowedMethods
+
+        if not isinstance(allowedMethods, Sequence):
+            raise TypeError(
+                "First argument must be a sequence of supported methods, "
+                "but my first argument is not a sequence."
+            )
+
+    def __str__(self) -> str:
+        return f"Expected one of {self.allowedMethods!r}"
+
+
+class SchemeNotSupported(Exception):
+    """
+    The scheme of a URI was not one of the supported values.
+    """
+
+
+class RenderError(Exception):
+    """
+    Base exception class for all errors which can occur during template
+    rendering.
+    """
+
+
+class MissingRenderMethod(RenderError):
+    """
+    Tried to use a render method which does not exist.
+
+    @ivar element: The element which did not have the render method.
+    @ivar renderName: The name of the renderer which could not be found.
+    """
+
+    def __init__(self, element, renderName):
+        RenderError.__init__(self, element, renderName)
+        self.element = element
+        self.renderName = renderName
+
+    def __repr__(self) -> str:
+        return "{!r}: {!r} had no render method named {!r}".format(
+            self.__class__.__name__,
+            self.element,
+            self.renderName,
+        )
+
+
+class MissingTemplateLoader(RenderError):
+    """
+    L{MissingTemplateLoader} is raised when trying to render an Element without
+    a template loader, i.e. a C{loader} attribute.
+
+    @ivar element: The Element which did not have a document factory.
+    """
+
+    def __init__(self, element):
+        RenderError.__init__(self, element)
+        self.element = element
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__!r}: {self.element!r} had no loader"
+
+
+class UnexposedMethodError(Exception):
+    """
+    Raised on any attempt to get a method which has not been exposed.
+    """
+
+
+class UnfilledSlot(Exception):
+    """
+    During flattening, a slot with no associated data was encountered.
+    """
+
+
+class UnsupportedType(Exception):
+    """
+    During flattening, an object of a type which cannot be flattened was
+    encountered.
+    """
+
+
+class ExcessiveBufferingError(Exception):
+    """
+    The HTTP/2 protocol has been forced to buffer an excessive amount of
+    outbound data, and has therefore closed the connection and dropped all
+    outbound data.
+    """
+
+
+class FlattenerError(Exception):
+    """
+    An error occurred while flattening an object.
+
+    @ivar _roots: A list of the objects on the flattener's stack at the time
+        the unflattenable object was encountered.  The first element is least
+        deeply nested object and the last element is the most deeply nested.
+    """
+
+    def __init__(self, exception, roots, traceback):
+        self._exception = exception
+        self._roots = roots
+        self._traceback = traceback
+        Exception.__init__(self, exception, roots, traceback)
+
+    def _formatRoot(self, obj):
+        """
+        Convert an object from C{self._roots} to a string suitable for
+        inclusion in a render-traceback (like a normal Python traceback, but
+        can include "frame" source locations which are not in Python source
+        files).
+
+        @param obj: Any object which can be a render step I{root}.
+            Typically, L{Tag}s, strings, and other simple Python types.
+
+        @return: A string representation of C{obj}.
+        @rtype: L{str}
+        """
+        # There's a circular dependency between this class and 'Tag', although
+        # only for an isinstance() check.
+        from twisted.web.template import Tag
+
+        if isinstance(obj, (bytes, str)):
+            # It's somewhat unlikely that there will ever be a str in the roots
+            # list.  However, something like a MemoryError during a str.replace
+            # call (eg, replacing " with &quot;) could possibly cause this.
+            # Likewise, UTF-8 encoding a unicode string to a byte string might
+            # fail like this.
+            if len(obj) > 40:
+                if isinstance(obj, str):
+                    ellipsis = "<...>"
+                else:
+                    ellipsis = b"<...>"
+                return ascii(obj[:20] + ellipsis + obj[-20:])
+            else:
+                return ascii(obj)
+        elif isinstance(obj, Tag):
+            if obj.filename is None:
+                return "Tag <" + obj.tagName + ">"
+            else:
+                return 'File "%s", line %d, column %d, in "%s"' % (
+                    obj.filename,
+                    obj.lineNumber,
+                    obj.columnNumber,
+                    obj.tagName,
+                )
+        else:
+            return ascii(obj)
+
+    def __repr__(self) -> str:
+        """
+        Present a string representation which includes a template traceback, so
+        we can tell where this error occurred in the template, as well as in
+        Python.
+        """
+        # Avoid importing things unnecessarily until we actually need them;
+        # since this is an 'error' module we should be extra paranoid about
+        # that.
+        from traceback import format_list
+
+        if self._roots:
+            roots = (
+                "  " + "\n  ".join([self._formatRoot(r) for r in self._roots]) + "\n"
+            )
+        else:
+            roots = ""
+        if self._traceback:
+            traceback = (
+                "\n".join(
+                    [
+                        line
+                        for entry in format_list(self._traceback)
+                        for line in entry.splitlines()
+                    ]
+                )
+                + "\n"
+            )
+        else:
+            traceback = ""
+        return cast(
+            str,
+            (
+                "Exception while flattening:\n"
+                + roots
+                + traceback
+                + self._exception.__class__.__name__
+                + ": "
+                + str(self._exception)
+                + "\n"
+            ),
+        )
+
+    def __str__(self) -> str:
+        return repr(self)
+
+
+class UnsupportedSpecialHeader(Exception):
+    """
+    A HTTP/2 request was received that contained a HTTP/2 pseudo-header field
+    that is not recognised by Twisted.
+    """
diff --git a/contrib/python/Twisted/py3/twisted/web/guard.py b/contrib/python/Twisted/py3/twisted/web/guard.py
new file mode 100644
index 00000000000..894823f814c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/guard.py
@@ -0,0 +1,21 @@
+# -*- test-case-name: twisted.web.test.test_httpauth -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Resource traversal integration with L{twisted.cred} to allow for
+authentication and authorization of HTTP requests.
+"""
+
+
+from twisted.web._auth.basic import BasicCredentialFactory
+from twisted.web._auth.digest import DigestCredentialFactory
+
+# Expose HTTP authentication classes here.
+from twisted.web._auth.wrapper import HTTPAuthSessionWrapper
+
+__all__ = [
+    "HTTPAuthSessionWrapper",
+    "BasicCredentialFactory",
+    "DigestCredentialFactory",
+]
diff --git a/contrib/python/Twisted/py3/twisted/web/html.py b/contrib/python/Twisted/py3/twisted/web/html.py
new file mode 100644
index 00000000000..8253b3ef5d3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/html.py
@@ -0,0 +1,56 @@
+# -*- test-case-name: twisted.web.test.test_html -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""I hold HTML generation helpers.
+"""
+
+from html import escape
+from io import StringIO
+
+from incremental import Version
+
+from twisted.python import log
+from twisted.python.deprecate import deprecated
+
+
+@deprecated(Version("Twisted", 15, 3, 0), replacement="twisted.web.template")
+def PRE(text):
+    "Wrap <pre> tags around some text and HTML-escape it."
+    return "<pre>" + escape(text) + "</pre>"
+
+
+@deprecated(Version("Twisted", 15, 3, 0), replacement="twisted.web.template")
+def UL(lst):
+    io = StringIO()
+    io.write("<ul>\n")
+    for el in lst:
+        io.write("<li> %s</li>\n" % el)
+    io.write("</ul>")
+    return io.getvalue()
+
+
+@deprecated(Version("Twisted", 15, 3, 0), replacement="twisted.web.template")
+def linkList(lst):
+    io = StringIO()
+    io.write("<ul>\n")
+    for hr, el in lst:
+        io.write(f'<li> <a href="{hr}">{el}</a></li>\n')
+    io.write("</ul>")
+    return io.getvalue()
+
+
+@deprecated(Version("Twisted", 15, 3, 0), replacement="twisted.web.template")
+def output(func, *args, **kw):
+    """output(func, *args, **kw) -> html string
+    Either return the result of a function (which presumably returns an
+    HTML-legal string) or a sparse HTMLized error message and a message
+    in the server log.
+    """
+    try:
+        return func(*args, **kw)
+    except BaseException:
+        log.msg(f"Error calling {func!r}:")
+        log.err()
+        return PRE("An error occurred.")
diff --git a/contrib/python/Twisted/py3/twisted/web/http.py b/contrib/python/Twisted/py3/twisted/web/http.py
new file mode 100644
index 00000000000..2bad1471dc8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/http.py
@@ -0,0 +1,3305 @@
+# -*- test-case-name: twisted.web.test.test_http -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HyperText Transfer Protocol implementation.
+
+This is the basic server-side protocol implementation used by the Twisted
+Web server.  It can parse HTTP 1.0 requests and supports many HTTP 1.1
+features as well.  Additionally, some functionality implemented here is
+also useful for HTTP clients (such as the chunked encoding parser).
+
+@var CACHED: A marker value to be returned from cache-related request methods
+    to indicate to the caller that a cached response will be usable and no
+    response body should be generated.
+
+@var FOUND: An HTTP response code indicating a temporary redirect.
+
+@var NOT_MODIFIED: An HTTP response code indicating that a requested
+    pre-condition (for example, the condition represented by an
+    I{If-Modified-Since} header is present in the request) has succeeded.  This
+    indicates a response body cached by the client can be used.
+
+@var PRECONDITION_FAILED: An HTTP response code indicating that a requested
+    pre-condition (for example, the condition represented by an I{If-None-Match}
+    header is present in the request) has failed.  This should typically
+    indicate that the server has not taken the requested action.
+
+@var maxChunkSizeLineLength: Maximum allowable length of the CRLF-terminated
+    line that indicates the size of a chunk and the extensions associated with
+    it, as in the HTTP 1.1 chunked I{Transfer-Encoding} (RFC 7230 section 4.1).
+    This limits how much data may be buffered when decoding the line.
+"""
+
+__all__ = [
+    "SWITCHING",
+    "OK",
+    "CREATED",
+    "ACCEPTED",
+    "NON_AUTHORITATIVE_INFORMATION",
+    "NO_CONTENT",
+    "RESET_CONTENT",
+    "PARTIAL_CONTENT",
+    "MULTI_STATUS",
+    "MULTIPLE_CHOICE",
+    "MOVED_PERMANENTLY",
+    "FOUND",
+    "SEE_OTHER",
+    "NOT_MODIFIED",
+    "USE_PROXY",
+    "TEMPORARY_REDIRECT",
+    "PERMANENT_REDIRECT",
+    "BAD_REQUEST",
+    "UNAUTHORIZED",
+    "PAYMENT_REQUIRED",
+    "FORBIDDEN",
+    "NOT_FOUND",
+    "NOT_ALLOWED",
+    "NOT_ACCEPTABLE",
+    "PROXY_AUTH_REQUIRED",
+    "REQUEST_TIMEOUT",
+    "CONFLICT",
+    "GONE",
+    "LENGTH_REQUIRED",
+    "PRECONDITION_FAILED",
+    "REQUEST_ENTITY_TOO_LARGE",
+    "REQUEST_URI_TOO_LONG",
+    "UNSUPPORTED_MEDIA_TYPE",
+    "REQUESTED_RANGE_NOT_SATISFIABLE",
+    "EXPECTATION_FAILED",
+    "INTERNAL_SERVER_ERROR",
+    "NOT_IMPLEMENTED",
+    "BAD_GATEWAY",
+    "SERVICE_UNAVAILABLE",
+    "GATEWAY_TIMEOUT",
+    "HTTP_VERSION_NOT_SUPPORTED",
+    "INSUFFICIENT_STORAGE_SPACE",
+    "NOT_EXTENDED",
+    "RESPONSES",
+    "CACHED",
+    "urlparse",
+    "parse_qs",
+    "datetimeToString",
+    "datetimeToLogString",
+    "timegm",
+    "stringToDatetime",
+    "toChunk",
+    "fromChunk",
+    "parseContentRange",
+    "StringTransport",
+    "HTTPClient",
+    "NO_BODY_CODES",
+    "Request",
+    "PotentialDataLoss",
+    "HTTPChannel",
+    "HTTPFactory",
+]
+
+
+import base64
+import binascii
+import calendar
+import cgi
+import math
+import os
+import re
+import tempfile
+import time
+import warnings
+from io import BytesIO
+from typing import AnyStr, Callable, List, Optional, Tuple
+from urllib.parse import (
+    ParseResultBytes,
+    unquote_to_bytes as unquote,
+    urlparse as _urlparse,
+)
+
+from zope.interface import Attribute, Interface, implementer, provider
+
+from incremental import Version
+
+from twisted.internet import address, interfaces, protocol
+from twisted.internet._producer_helpers import _PullToPush
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IProtocol
+from twisted.logger import Logger
+from twisted.protocols import basic, policies
+from twisted.python import log
+from twisted.python.compat import nativeString, networkString
+from twisted.python.components import proxyForInterface
+from twisted.python.deprecate import deprecated
+from twisted.python.failure import Failure
+
+# twisted imports
+from twisted.web._responses import (
+    ACCEPTED,
+    BAD_GATEWAY,
+    BAD_REQUEST,
+    CONFLICT,
+    CREATED,
+    EXPECTATION_FAILED,
+    FORBIDDEN,
+    FOUND,
+    GATEWAY_TIMEOUT,
+    GONE,
+    HTTP_VERSION_NOT_SUPPORTED,
+    INSUFFICIENT_STORAGE_SPACE,
+    INTERNAL_SERVER_ERROR,
+    LENGTH_REQUIRED,
+    MOVED_PERMANENTLY,
+    MULTI_STATUS,
+    MULTIPLE_CHOICE,
+    NO_CONTENT,
+    NON_AUTHORITATIVE_INFORMATION,
+    NOT_ACCEPTABLE,
+    NOT_ALLOWED,
+    NOT_EXTENDED,
+    NOT_FOUND,
+    NOT_IMPLEMENTED,
+    NOT_MODIFIED,
+    OK,
+    PARTIAL_CONTENT,
+    PAYMENT_REQUIRED,
+    PERMANENT_REDIRECT,
+    PRECONDITION_FAILED,
+    PROXY_AUTH_REQUIRED,
+    REQUEST_ENTITY_TOO_LARGE,
+    REQUEST_TIMEOUT,
+    REQUEST_URI_TOO_LONG,
+    REQUESTED_RANGE_NOT_SATISFIABLE,
+    RESET_CONTENT,
+    RESPONSES,
+    SEE_OTHER,
+    SERVICE_UNAVAILABLE,
+    SWITCHING,
+    TEMPORARY_REDIRECT,
+    UNAUTHORIZED,
+    UNSUPPORTED_MEDIA_TYPE,
+    USE_PROXY,
+)
+from twisted.web.http_headers import Headers, _sanitizeLinearWhitespace
+from twisted.web.iweb import IAccessLogFormatter, INonQueuedRequestFactory, IRequest
+
+try:
+    from twisted.web._http2 import H2Connection
+
+    H2_ENABLED = True
+except ImportError:
+    H2_ENABLED = False
+
+
+# A common request timeout -- 1 minute. This is roughly what nginx uses, and
+# so it seems to be a good choice for us too.
+_REQUEST_TIMEOUT = 1 * 60
+
+protocol_version = "HTTP/1.1"
+
+CACHED = """Magic constant returned by http.Request methods to set cache
+validation headers when the request is conditional and the value fails
+the condition."""
+
+# backwards compatibility
+responses = RESPONSES
+
+
+# datetime parsing and formatting
+weekdayname = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
+monthname = [
+    None,
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec",
+]
+weekdayname_lower = [name.lower() for name in weekdayname]
+monthname_lower = [name and name.lower() for name in monthname]
+
+
+def _parseHeader(line):
+    # cgi.parse_header requires a str
+    key, pdict = cgi.parse_header(line.decode("charmap"))
+
+    # We want the key as bytes, and cgi.parse_multipart (which consumes
+    # pdict) expects a dict of str keys but bytes values
+    key = key.encode("charmap")
+    pdict = {x: y.encode("charmap") for x, y in pdict.items()}
+    return (key, pdict)
+
+
+def urlparse(url):
+    """
+    Parse an URL into six components.
+
+    This is similar to C{urlparse.urlparse}, but rejects C{str} input
+    and always produces C{bytes} output.
+
+    @type url: C{bytes}
+
+    @raise TypeError: The given url was a C{str} string instead of a
+        C{bytes}.
+
+    @return: The scheme, net location, path, params, query string, and fragment
+        of the URL - all as C{bytes}.
+    @rtype: C{ParseResultBytes}
+    """
+    if isinstance(url, str):
+        raise TypeError("url must be bytes, not unicode")
+    scheme, netloc, path, params, query, fragment = _urlparse(url)
+    if isinstance(scheme, str):
+        scheme = scheme.encode("ascii")
+        netloc = netloc.encode("ascii")
+        path = path.encode("ascii")
+        query = query.encode("ascii")
+        fragment = fragment.encode("ascii")
+    return ParseResultBytes(scheme, netloc, path, params, query, fragment)
+
+
+def parse_qs(qs, keep_blank_values=0, strict_parsing=0):
+    """
+    Like C{cgi.parse_qs}, but with support for parsing byte strings on Python 3.
+
+    @type qs: C{bytes}
+    """
+    d = {}
+    items = [s2 for s1 in qs.split(b"&") for s2 in s1.split(b";")]
+    for item in items:
+        try:
+            k, v = item.split(b"=", 1)
+        except ValueError:
+            if strict_parsing:
+                raise
+            continue
+        if v or keep_blank_values:
+            k = unquote(k.replace(b"+", b" "))
+            v = unquote(v.replace(b"+", b" "))
+            if k in d:
+                d[k].append(v)
+            else:
+                d[k] = [v]
+    return d
+
+
+def datetimeToString(msSinceEpoch=None):
+    """
+    Convert seconds since epoch to HTTP datetime string.
+
+    @rtype: C{bytes}
+    """
+    if msSinceEpoch == None:
+        msSinceEpoch = time.time()
+    year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
+    s = networkString(
+        "%s, %02d %3s %4d %02d:%02d:%02d GMT"
+        % (weekdayname[wd], day, monthname[month], year, hh, mm, ss)
+    )
+    return s
+
+
+def datetimeToLogString(msSinceEpoch=None):
+    """
+    Convert seconds since epoch to log datetime string.
+
+    @rtype: C{str}
+    """
+    if msSinceEpoch == None:
+        msSinceEpoch = time.time()
+    year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
+    s = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % (
+        day,
+        monthname[month],
+        year,
+        hh,
+        mm,
+        ss,
+    )
+    return s
+
+
+def timegm(year, month, day, hour, minute, second):
+    """
+    Convert time tuple in GMT to seconds since epoch, GMT
+    """
+    EPOCH = 1970
+    if year < EPOCH:
+        raise ValueError("Years prior to %d not supported" % (EPOCH,))
+    assert 1 <= month <= 12
+    days = 365 * (year - EPOCH) + calendar.leapdays(EPOCH, year)
+    for i in range(1, month):
+        days = days + calendar.mdays[i]
+    if month > 2 and calendar.isleap(year):
+        days = days + 1
+    days = days + day - 1
+    hours = days * 24 + hour
+    minutes = hours * 60 + minute
+    seconds = minutes * 60 + second
+    return seconds
+
+
+def stringToDatetime(dateString):
+    """
+    Convert an HTTP date string (one of three formats) to seconds since epoch.
+
+    @type dateString: C{bytes}
+    """
+    parts = nativeString(dateString).split()
+
+    if not parts[0][0:3].lower() in weekdayname_lower:
+        # Weekday is stupid. Might have been omitted.
+        try:
+            return stringToDatetime(b"Sun, " + dateString)
+        except ValueError:
+            # Guess not.
+            pass
+
+    partlen = len(parts)
+    if (partlen == 5 or partlen == 6) and parts[1].isdigit():
+        # 1st date format: Sun, 06 Nov 1994 08:49:37 GMT
+        # (Note: "GMT" is literal, not a variable timezone)
+        # (also handles without "GMT")
+        # This is the normal format
+        day = parts[1]
+        month = parts[2]
+        year = parts[3]
+        time = parts[4]
+    elif (partlen == 3 or partlen == 4) and parts[1].find("-") != -1:
+        # 2nd date format: Sunday, 06-Nov-94 08:49:37 GMT
+        # (Note: "GMT" is literal, not a variable timezone)
+        # (also handles without without "GMT")
+        # Two digit year, yucko.
+        day, month, year = parts[1].split("-")
+        time = parts[2]
+        year = int(year)
+        if year < 69:
+            year = year + 2000
+        elif year < 100:
+            year = year + 1900
+    elif len(parts) == 5:
+        # 3rd date format: Sun Nov  6 08:49:37 1994
+        # ANSI C asctime() format.
+        day = parts[2]
+        month = parts[1]
+        year = parts[4]
+        time = parts[3]
+    else:
+        raise ValueError("Unknown datetime format %r" % dateString)
+
+    day = int(day)
+    month = int(monthname_lower.index(month.lower()))
+    year = int(year)
+    hour, min, sec = map(int, time.split(":"))
+    return int(timegm(year, month, day, hour, min, sec))
+
+
+def toChunk(data):
+    """
+    Convert string to a chunk.
+
+    @type data: C{bytes}
+
+    @returns: a tuple of C{bytes} representing the chunked encoding of data
+    """
+    return (networkString(f"{len(data):x}"), b"\r\n", data, b"\r\n")
+
+
+def _ishexdigits(b: bytes) -> bool:
+    """
+    Is the string case-insensitively hexidecimal?
+
+    It must be composed of one or more characters in the ranges a-f, A-F
+    and 0-9.
+    """
+    for c in b:
+        if c not in b"0123456789abcdefABCDEF":
+            return False
+    return b != b""
+
+
+def _hexint(b: bytes) -> int:
+    """
+    Decode a hexadecimal integer.
+
+    Unlike L{int(b, 16)}, this raises L{ValueError} when the integer has
+    a prefix like C{b'0x'}, C{b'+'}, or C{b'-'}, which is desirable when
+    parsing network protocols.
+    """
+    if not _ishexdigits(b):
+        raise ValueError(b)
+    return int(b, 16)
+
+
+def fromChunk(data: bytes) -> Tuple[bytes, bytes]:
+    """
+    Convert chunk to string.
+
+    Note that this function is not specification compliant: it doesn't handle
+    chunk extensions.
+
+    @type data: C{bytes}
+
+    @return: tuple of (result, remaining) - both C{bytes}.
+
+    @raise ValueError: If the given data is not a correctly formatted chunked
+        byte string.
+    """
+    prefix, rest = data.split(b"\r\n", 1)
+    length = _hexint(prefix)
+    if length < 0:
+        raise ValueError("Chunk length must be >= 0, not %d" % (length,))
+    if rest[length : length + 2] != b"\r\n":
+        raise ValueError("chunk must end with CRLF")
+    return rest[:length], rest[length + 2 :]
+
+
+def parseContentRange(header):
+    """
+    Parse a content-range header into (start, end, realLength).
+
+    realLength might be None if real length is not known ('*').
+    """
+    kind, other = header.strip().split()
+    if kind.lower() != "bytes":
+        raise ValueError("a range of type %r is not supported")
+    startend, realLength = other.split("/")
+    start, end = map(int, startend.split("-"))
+    if realLength == "*":
+        realLength = None
+    else:
+        realLength = int(realLength)
+    return (start, end, realLength)
+
+
+class _IDeprecatedHTTPChannelToRequestInterface(Interface):
+    """
+    The interface L{HTTPChannel} expects of L{Request}.
+    """
+
+    requestHeaders = Attribute(
+        "A L{http_headers.Headers} instance giving all received HTTP request "
+        "headers."
+    )
+
+    responseHeaders = Attribute(
+        "A L{http_headers.Headers} instance holding all HTTP response "
+        "headers to be sent."
+    )
+
+    def connectionLost(reason):
+        """
+        The underlying connection has been lost.
+
+        @param reason: A failure instance indicating the reason why
+            the connection was lost.
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+    def gotLength(length):
+        """
+        Called when L{HTTPChannel} has determined the length, if any,
+        of the incoming request's body.
+
+        @param length: The length of the request's body.
+        @type length: L{int} if the request declares its body's length
+            and L{None} if it does not.
+        """
+
+    def handleContentChunk(data):
+        """
+        Deliver a received chunk of body data to the request.  Note
+        this does not imply chunked transfer encoding.
+
+        @param data: The received chunk.
+        @type data: L{bytes}
+        """
+
+    def parseCookies():
+        """
+        Parse the request's cookies out of received headers.
+        """
+
+    def requestReceived(command, path, version):
+        """
+        Called when the entire request, including its body, has been
+        received.
+
+        @param command: The request's HTTP command.
+        @type command: L{bytes}
+
+        @param path: The request's path.  Note: this is actually what
+            RFC7320 calls the URI.
+        @type path: L{bytes}
+
+        @param version: The request's HTTP version.
+        @type version: L{bytes}
+        """
+
+    def __eq__(other: object) -> bool:
+        """
+        Determines if two requests are the same object.
+
+        @param other: Another object whose identity will be compared
+            to this instance's.
+
+        @return: L{True} when the two are the same object and L{False}
+            when not.
+        """
+
+    def __ne__(other: object) -> bool:
+        """
+        Determines if two requests are not the same object.
+
+        @param other: Another object whose identity will be compared
+            to this instance's.
+
+        @return: L{True} when the two are not the same object and
+            L{False} when they are.
+        """
+
+    def __hash__():
+        """
+        Generate a hash value for the request.
+
+        @return: The request's hash value.
+        @rtype: L{int}
+        """
+
+
+class StringTransport:
+    """
+    I am a BytesIO wrapper that conforms for the transport API. I support
+    the `writeSequence' method.
+    """
+
+    def __init__(self):
+        self.s = BytesIO()
+
+    def writeSequence(self, seq):
+        self.s.write(b"".join(seq))
+
+    def __getattr__(self, attr):
+        return getattr(self.__dict__["s"], attr)
+
+
+class HTTPClient(basic.LineReceiver):
+    """
+    A client for HTTP 1.0.
+
+    Notes:
+    You probably want to send a 'Host' header with the name of the site you're
+    connecting to, in order to not break name based virtual hosting.
+
+    @ivar length: The length of the request body in bytes.
+    @type length: C{int}
+
+    @ivar firstLine: Are we waiting for the first header line?
+    @type firstLine: C{bool}
+
+    @ivar __buffer: The buffer that stores the response to the HTTP request.
+    @type __buffer: A C{BytesIO} object.
+
+    @ivar _header: Part or all of an HTTP request header.
+    @type _header: C{bytes}
+    """
+
+    length = None
+    firstLine = True
+    __buffer = None
+    _header = b""
+
+    def sendCommand(self, command, path):
+        self.transport.writeSequence([command, b" ", path, b" HTTP/1.0\r\n"])
+
+    def sendHeader(self, name, value):
+        if not isinstance(value, bytes):
+            # XXX Deprecate this case
+            value = networkString(str(value))
+        santizedName = _sanitizeLinearWhitespace(name)
+        santizedValue = _sanitizeLinearWhitespace(value)
+        self.transport.writeSequence([santizedName, b": ", santizedValue, b"\r\n"])
+
+    def endHeaders(self):
+        self.transport.write(b"\r\n")
+
+    def extractHeader(self, header):
+        """
+        Given a complete HTTP header, extract the field name and value and
+        process the header.
+
+        @param header: a complete HTTP request header of the form
+            'field-name: value'.
+        @type header: C{bytes}
+        """
+        key, val = header.split(b":", 1)
+        val = val.lstrip()
+        self.handleHeader(key, val)
+        if key.lower() == b"content-length":
+            self.length = int(val)
+
+    def lineReceived(self, line):
+        """
+        Parse the status line and headers for an HTTP request.
+
+        @param line: Part of an HTTP request header. Request bodies are parsed
+            in L{HTTPClient.rawDataReceived}.
+        @type line: C{bytes}
+        """
+        if self.firstLine:
+            self.firstLine = False
+            l = line.split(None, 2)
+            version = l[0]
+            status = l[1]
+            try:
+                message = l[2]
+            except IndexError:
+                # sometimes there is no message
+                message = b""
+            self.handleStatus(version, status, message)
+            return
+        if not line:
+            if self._header != b"":
+                # Only extract headers if there are any
+                self.extractHeader(self._header)
+            self.__buffer = BytesIO()
+            self.handleEndHeaders()
+            self.setRawMode()
+            return
+
+        if line.startswith(b"\t") or line.startswith(b" "):
+            # This line is part of a multiline header. According to RFC 822, in
+            # "unfolding" multiline headers you do not strip the leading
+            # whitespace on the continuing line.
+            self._header = self._header + line
+        elif self._header:
+            # This line starts a new header, so process the previous one.
+            self.extractHeader(self._header)
+            self._header = line
+        else:  # First header
+            self._header = line
+
+    def connectionLost(self, reason):
+        self.handleResponseEnd()
+
+    def handleResponseEnd(self):
+        """
+        The response has been completely received.
+
+        This callback may be invoked more than once per request.
+        """
+        if self.__buffer is not None:
+            b = self.__buffer.getvalue()
+            self.__buffer = None
+            self.handleResponse(b)
+
+    def handleResponsePart(self, data):
+        self.__buffer.write(data)
+
+    def connectionMade(self):
+        pass
+
+    def handleStatus(self, version, status, message):
+        """
+        Called when the status-line is received.
+
+        @param version: e.g. 'HTTP/1.0'
+        @param status: e.g. '200'
+        @type status: C{bytes}
+        @param message: e.g. 'OK'
+        """
+
+    def handleHeader(self, key, val):
+        """
+        Called every time a header is received.
+        """
+
+    def handleEndHeaders(self):
+        """
+        Called when all headers have been received.
+        """
+
+    def rawDataReceived(self, data):
+        if self.length is not None:
+            data, rest = data[: self.length], data[self.length :]
+            self.length -= len(data)
+        else:
+            rest = b""
+        self.handleResponsePart(data)
+        if self.length == 0:
+            self.handleResponseEnd()
+            self.setLineMode(rest)
+
+
+# response codes that must have empty bodies
+NO_BODY_CODES = (204, 304)
+
+
+# Sentinel object that detects people explicitly passing `queued` to Request.
+_QUEUED_SENTINEL = object()
+
+
+def _getContentFile(length):
+    """
+    Get a writeable file-like object to which request content can be written.
+    """
+    if length is not None and length < 100000:
+        return BytesIO()
+    return tempfile.TemporaryFile()
+
+
+_hostHeaderExpression = re.compile(rb"^\[?(?P<host>.*?)\]?(:\d+)?$")
+
+
+@implementer(interfaces.IConsumer, _IDeprecatedHTTPChannelToRequestInterface)
+class Request:
+    """
+    A HTTP request.
+
+    Subclasses should override the process() method to determine how
+    the request will be processed.
+
+    @ivar method: The HTTP method that was used, e.g. C{b'GET'}.
+    @type method: L{bytes}
+
+    @ivar uri: The full encoded URI which was requested (including query
+        arguments), e.g. C{b'/a/b%20/c?q=v'}.
+    @type uri: L{bytes}
+
+    @ivar path: The encoded path of the request URI (not including query
+        arguments), e.g. C{b'/a/b%20/c'}.
+    @type path: L{bytes}
+
+    @ivar args: A mapping of decoded query argument names as L{bytes} to
+        corresponding query argument values as L{list}s of L{bytes}.
+        For example, for a URI with C{foo=bar&foo=baz&quux=spam}
+        as its query part C{args} will be C{{b'foo': [b'bar', b'baz'],
+        b'quux': [b'spam']}}.
+    @type args: L{dict} of L{bytes} to L{list} of L{bytes}
+
+    @ivar content: A file-like object giving the request body.  This may be
+        a file on disk, an L{io.BytesIO}, or some other type.  The
+        implementation is free to decide on a per-request basis.
+    @type content: L{typing.BinaryIO}
+
+    @ivar cookies: The cookies that will be sent in the response.
+    @type cookies: L{list} of L{bytes}
+
+    @type requestHeaders: L{http_headers.Headers}
+    @ivar requestHeaders: All received HTTP request headers.
+
+    @type responseHeaders: L{http_headers.Headers}
+    @ivar responseHeaders: All HTTP response headers to be sent.
+
+    @ivar notifications: A L{list} of L{Deferred}s which are waiting for
+        notification that the response to this request has been finished
+        (successfully or with an error).  Don't use this attribute directly,
+        instead use the L{Request.notifyFinish} method.
+
+    @ivar _disconnected: A flag which is C{False} until the connection over
+        which this request was received is closed and which is C{True} after
+        that.
+    @type _disconnected: L{bool}
+
+    @ivar _log: A logger instance for request related messages.
+    @type _log: L{twisted.logger.Logger}
+    """
+
+    producer = None
+    finished = 0
+    code = OK
+    code_message = RESPONSES[OK]
+    method = b"(no method yet)"
+    clientproto = b"(no clientproto yet)"
+    uri = b"(no uri yet)"
+    startedWriting = 0
+    chunked = 0
+    sentLength = 0  # content-length of response, or total bytes sent via chunking
+    etag = None
+    lastModified = None
+    args = None
+    path = None
+    content = None
+    _forceSSL = 0
+    _disconnected = False
+    _log = Logger()
+
+    def __init__(self, channel, queued=_QUEUED_SENTINEL):
+        """
+        @param channel: the channel we're connected to.
+        @param queued: (deprecated) are we in the request queue, or can we
+            start writing to the transport?
+        """
+        self.notifications: List[Deferred[None]] = []
+        self.channel = channel
+
+        # Cache the client and server information, we'll need this
+        # later to be serialized and sent with the request so CGIs
+        # will work remotely
+        self.client = self.channel.getPeer()
+        self.host = self.channel.getHost()
+
+        self.requestHeaders: Headers = Headers()
+        self.received_cookies = {}
+        self.responseHeaders: Headers = Headers()
+        self.cookies = []  # outgoing cookies
+        self.transport = self.channel.transport
+
+        if queued is _QUEUED_SENTINEL:
+            queued = False
+
+        self.queued = queued
+
+    def _cleanup(self):
+        """
+        Called when have finished responding and are no longer queued.
+        """
+        if self.producer:
+            self._log.failure(
+                "",
+                Failure(RuntimeError(f"Producer was not unregistered for {self.uri}")),
+            )
+            self.unregisterProducer()
+        self.channel.requestDone(self)
+        del self.channel
+        if self.content is not None:
+            try:
+                self.content.close()
+            except OSError:
+                # win32 suckiness, no idea why it does this
+                pass
+            del self.content
+        for d in self.notifications:
+            d.callback(None)
+        self.notifications = []
+
+    # methods for channel - end users should not use these
+
+    @deprecated(Version("Twisted", 16, 3, 0))
+    def noLongerQueued(self):
+        """
+        Notify the object that it is no longer queued.
+
+        We start writing whatever data we have to the transport, etc.
+
+        This method is not intended for users.
+
+        In 16.3 this method was changed to become a no-op, as L{Request}
+        objects are now never queued.
+        """
+        pass
+
+    def gotLength(self, length):
+        """
+        Called when HTTP channel got length of content in this request.
+
+        This method is not intended for users.
+
+        @param length: The length of the request body, as indicated by the
+            request headers.  L{None} if the request headers do not indicate a
+            length.
+        """
+        self.content = _getContentFile(length)
+
+    def parseCookies(self):
+        """
+        Parse cookie headers.
+
+        This method is not intended for users.
+        """
+        cookieheaders = self.requestHeaders.getRawHeaders(b"cookie")
+
+        if cookieheaders is None:
+            return
+
+        for cookietxt in cookieheaders:
+            if cookietxt:
+                for cook in cookietxt.split(b";"):
+                    cook = cook.lstrip()
+                    try:
+                        k, v = cook.split(b"=", 1)
+                        self.received_cookies[k] = v
+                    except ValueError:
+                        pass
+
+    def handleContentChunk(self, data):
+        """
+        Write a chunk of data.
+
+        This method is not intended for users.
+        """
+        self.content.write(data)
+
+    def requestReceived(self, command, path, version):
+        """
+        Called by channel when all data has been received.
+
+        This method is not intended for users.
+
+        @type command: C{bytes}
+        @param command: The HTTP verb of this request.  This has the case
+            supplied by the client (eg, it maybe "get" rather than "GET").
+
+        @type path: C{bytes}
+        @param path: The URI of this request.
+
+        @type version: C{bytes}
+        @param version: The HTTP version of this request.
+        """
+        clength = self.content.tell()
+        self.content.seek(0, 0)
+        self.args = {}
+
+        self.method, self.uri = command, path
+        self.clientproto = version
+        x = self.uri.split(b"?", 1)
+
+        if len(x) == 1:
+            self.path = self.uri
+        else:
+            self.path, argstring = x
+            self.args = parse_qs(argstring, 1)
+
+        # Argument processing
+        args = self.args
+        ctype = self.requestHeaders.getRawHeaders(b"content-type")
+        if ctype is not None:
+            ctype = ctype[0]
+
+        if self.method == b"POST" and ctype and clength:
+            mfd = b"multipart/form-data"
+            key, pdict = _parseHeader(ctype)
+            # This weird CONTENT-LENGTH param is required by
+            # cgi.parse_multipart() in some versions of Python 3.7+, see
+            # bpo-29979. It looks like this will be relaxed and backported, see
+            # https://github.com/python/cpython/pull/8530.
+            pdict["CONTENT-LENGTH"] = clength
+            if key == b"application/x-www-form-urlencoded":
+                args.update(parse_qs(self.content.read(), 1))
+            elif key == mfd:
+                try:
+                    cgiArgs = cgi.parse_multipart(
+                        self.content,
+                        pdict,
+                        encoding="utf8",
+                        errors="surrogateescape",
+                    )
+
+                    # The parse_multipart function on Python 3.7+
+                    # decodes the header bytes as iso-8859-1 and
+                    # decodes the body bytes as utf8 with
+                    # surrogateescape -- we want bytes
+                    self.args.update(
+                        {
+                            x.encode("iso-8859-1"): [
+                                z.encode("utf8", "surrogateescape")
+                                if isinstance(z, str)
+                                else z
+                                for z in y
+                            ]
+                            for x, y in cgiArgs.items()
+                            if isinstance(x, str)
+                        }
+                    )
+                except Exception as e:
+                    # It was a bad request, or we got a signal.
+                    self.channel._respondToBadRequestAndDisconnect()
+                    if isinstance(e, (TypeError, ValueError, KeyError)):
+                        return
+                    else:
+                        # If it's not a userspace error from CGI, reraise
+                        raise
+
+            self.content.seek(0, 0)
+
+        self.process()
+
+    def __repr__(self) -> str:
+        """
+        Return a string description of the request including such information
+        as the request method and request URI.
+
+        @return: A string loosely describing this L{Request} object.
+        @rtype: L{str}
+        """
+        return "<{} at 0x{:x} method={} uri={} clientproto={}>".format(
+            self.__class__.__name__,
+            id(self),
+            nativeString(self.method),
+            nativeString(self.uri),
+            nativeString(self.clientproto),
+        )
+
+    def process(self):
+        """
+        Override in subclasses.
+
+        This method is not intended for users.
+        """
+        pass
+
+    # consumer interface
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register a producer.
+        """
+        if self.producer:
+            raise ValueError(
+                "registering producer %s before previous one (%s) was "
+                "unregistered" % (producer, self.producer)
+            )
+
+        self.streamingProducer = streaming
+        self.producer = producer
+        self.channel.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        """
+        Unregister the producer.
+        """
+        self.channel.unregisterProducer()
+        self.producer = None
+
+    # The following is the public interface that people should be
+    # writing to.
+    def getHeader(self, key: AnyStr) -> Optional[AnyStr]:
+        """
+        Get an HTTP request header.
+
+        @type key: C{bytes} or C{str}
+        @param key: The name of the header to get the value of.
+
+        @rtype: C{bytes} or C{str} or L{None}
+        @return: The value of the specified header, or L{None} if that header
+            was not present in the request. The string type of the result
+            matches the type of C{key}.
+        """
+        value = self.requestHeaders.getRawHeaders(key)
+        if value is not None:
+            return value[-1]
+        return None
+
+    def getCookie(self, key):
+        """
+        Get a cookie that was sent from the network.
+
+        @type key: C{bytes}
+        @param key: The name of the cookie to get.
+
+        @rtype: C{bytes} or C{None}
+        @returns: The value of the specified cookie, or L{None} if that cookie
+            was not present in the request.
+        """
+        return self.received_cookies.get(key)
+
+    def notifyFinish(self) -> Deferred[None]:
+        """
+        Notify when the response to this request has finished.
+
+        @note: There are some caveats around the reliability of the delivery of
+            this notification.
+
+                1. If this L{Request}'s channel is paused, the notification
+                   will not be delivered.  This can happen in one of two ways;
+                   either you can call C{request.transport.pauseProducing}
+                   yourself, or,
+
+                2. In order to deliver this notification promptly when a client
+                   disconnects, the reactor must continue reading from the
+                   transport, so that it can tell when the underlying network
+                   connection has gone away.  Twisted Web will only keep
+                   reading up until a finite (small) maximum buffer size before
+                   it gives up and pauses the transport itself.  If this
+                   occurs, you will not discover that the connection has gone
+                   away until a timeout fires or until the application attempts
+                   to send some data via L{Request.write}.
+
+                3. It is theoretically impossible to distinguish between
+                   successfully I{sending} a response and the peer successfully
+                   I{receiving} it.  There are several networking edge cases
+                   where the L{Deferred}s returned by C{notifyFinish} will
+                   indicate success, but the data will never be received.
+                   There are also edge cases where the connection will appear
+                   to fail, but in reality the response was delivered.  As a
+                   result, the information provided by the result of the
+                   L{Deferred}s returned by this method should be treated as a
+                   guess; do not make critical decisions in your applications
+                   based upon it.
+
+        @rtype: L{Deferred}
+        @return: A L{Deferred} which will be triggered when the request is
+            finished -- with a L{None} value if the request finishes
+            successfully or with an error if the request is interrupted by an
+            error (for example, the client closing the connection prematurely).
+        """
+        self.notifications.append(Deferred())
+        return self.notifications[-1]
+
+    def finish(self):
+        """
+        Indicate that all response data has been written to this L{Request}.
+        """
+        if self._disconnected:
+            raise RuntimeError(
+                "Request.finish called on a request after its connection was lost; "
+                "use Request.notifyFinish to keep track of this."
+            )
+        if self.finished:
+            warnings.warn("Warning! request.finish called twice.", stacklevel=2)
+            return
+
+        if not self.startedWriting:
+            # write headers
+            self.write(b"")
+
+        if self.chunked:
+            # write last chunk and closing CRLF
+            self.channel.write(b"0\r\n\r\n")
+
+        # log request
+        if hasattr(self.channel, "factory") and self.channel.factory is not None:
+            self.channel.factory.log(self)
+
+        self.finished = 1
+        if not self.queued:
+            self._cleanup()
+
+    def write(self, data):
+        """
+        Write some data as a result of an HTTP request.  The first
+        time this is called, it writes out response data.
+
+        @type data: C{bytes}
+        @param data: Some bytes to be sent as part of the response body.
+        """
+        if self.finished:
+            raise RuntimeError(
+                "Request.write called on a request after " "Request.finish was called."
+            )
+
+        if self._disconnected:
+            # Don't attempt to write any data to a disconnected client.
+            # The RuntimeError exception will be thrown as usual when
+            # request.finish is called
+            return
+
+        if not self.startedWriting:
+            self.startedWriting = 1
+            version = self.clientproto
+            code = b"%d" % (self.code,)
+            reason = self.code_message
+            headers = []
+
+            # if we don't have a content length, we send data in
+            # chunked mode, so that we can support pipelining in
+            # persistent connections.
+            if (
+                (version == b"HTTP/1.1")
+                and (self.responseHeaders.getRawHeaders(b"content-length") is None)
+                and self.method != b"HEAD"
+                and self.code not in NO_BODY_CODES
+            ):
+                headers.append((b"Transfer-Encoding", b"chunked"))
+                self.chunked = 1
+
+            if self.lastModified is not None:
+                if self.responseHeaders.hasHeader(b"last-modified"):
+                    self._log.info(
+                        "Warning: last-modified specified both in"
+                        " header list and lastModified attribute."
+                    )
+                else:
+                    self.responseHeaders.setRawHeaders(
+                        b"last-modified", [datetimeToString(self.lastModified)]
+                    )
+
+            if self.etag is not None:
+                self.responseHeaders.setRawHeaders(b"ETag", [self.etag])
+
+            for name, values in self.responseHeaders.getAllRawHeaders():
+                for value in values:
+                    headers.append((name, value))
+
+            for cookie in self.cookies:
+                headers.append((b"Set-Cookie", cookie))
+
+            self.channel.writeHeaders(version, code, reason, headers)
+
+            # if this is a "HEAD" request, we shouldn't return any data
+            if self.method == b"HEAD":
+                self.write = lambda data: None
+                return
+
+            # for certain result codes, we should never return any data
+            if self.code in NO_BODY_CODES:
+                self.write = lambda data: None
+                return
+
+        self.sentLength = self.sentLength + len(data)
+        if data:
+            if self.chunked:
+                self.channel.writeSequence(toChunk(data))
+            else:
+                self.channel.write(data)
+
+    def addCookie(
+        self,
+        k,
+        v,
+        expires=None,
+        domain=None,
+        path=None,
+        max_age=None,
+        comment=None,
+        secure=None,
+        httpOnly=False,
+        sameSite=None,
+    ):
+        """
+        Set an outgoing HTTP cookie.
+
+        In general, you should consider using sessions instead of cookies, see
+        L{twisted.web.server.Request.getSession} and the
+        L{twisted.web.server.Session} class for details.
+
+        @param k: cookie name
+        @type k: L{bytes} or L{str}
+
+        @param v: cookie value
+        @type v: L{bytes} or L{str}
+
+        @param expires: cookie expire attribute value in
+            "Wdy, DD Mon YYYY HH:MM:SS GMT" format
+        @type expires: L{bytes} or L{str}
+
+        @param domain: cookie domain
+        @type domain: L{bytes} or L{str}
+
+        @param path: cookie path
+        @type path: L{bytes} or L{str}
+
+        @param max_age: cookie expiration in seconds from reception
+        @type max_age: L{bytes} or L{str}
+
+        @param comment: cookie comment
+        @type comment: L{bytes} or L{str}
+
+        @param secure: direct browser to send the cookie on encrypted
+            connections only
+        @type secure: L{bool}
+
+        @param httpOnly: direct browser not to expose cookies through channels
+            other than HTTP (and HTTPS) requests
+        @type httpOnly: L{bool}
+
+        @param sameSite: One of L{None} (default), C{'lax'} or C{'strict'}.
+            Direct browsers not to send this cookie on cross-origin requests.
+            Please see:
+            U{https://tools.ietf.org/html/draft-west-first-party-cookies-07}
+        @type sameSite: L{None}, L{bytes} or L{str}
+
+        @raise ValueError: If the value for C{sameSite} is not supported.
+        """
+
+        def _ensureBytes(val):
+            """
+            Ensure that C{val} is bytes, encoding using UTF-8 if
+            needed.
+
+            @param val: L{bytes} or L{str}
+
+            @return: L{bytes}
+            """
+            if val is None:
+                # It's None, so we don't want to touch it
+                return val
+
+            if isinstance(val, bytes):
+                return val
+            else:
+                return val.encode("utf8")
+
+        def _sanitize(val):
+            r"""
+            Replace linear whitespace (C{\r}, C{\n}, C{\r\n}) and
+            semicolons C{;} in C{val} with a single space.
+
+            @param val: L{bytes}
+            @return: L{bytes}
+            """
+            return _sanitizeLinearWhitespace(val).replace(b";", b" ")
+
+        cookie = _sanitize(_ensureBytes(k)) + b"=" + _sanitize(_ensureBytes(v))
+        if expires is not None:
+            cookie = cookie + b"; Expires=" + _sanitize(_ensureBytes(expires))
+        if domain is not None:
+            cookie = cookie + b"; Domain=" + _sanitize(_ensureBytes(domain))
+        if path is not None:
+            cookie = cookie + b"; Path=" + _sanitize(_ensureBytes(path))
+        if max_age is not None:
+            cookie = cookie + b"; Max-Age=" + _sanitize(_ensureBytes(max_age))
+        if comment is not None:
+            cookie = cookie + b"; Comment=" + _sanitize(_ensureBytes(comment))
+        if secure:
+            cookie = cookie + b"; Secure"
+        if httpOnly:
+            cookie = cookie + b"; HttpOnly"
+        if sameSite:
+            sameSite = _ensureBytes(sameSite).lower()
+            if sameSite not in [b"lax", b"strict"]:
+                raise ValueError("Invalid value for sameSite: " + repr(sameSite))
+            cookie += b"; SameSite=" + sameSite
+        self.cookies.append(cookie)
+
+    def setResponseCode(self, code, message=None):
+        """
+        Set the HTTP response code.
+
+        @type code: L{int}
+        @type message: L{bytes}
+        """
+        if not isinstance(code, int):
+            raise TypeError("HTTP response code must be int or long")
+        self.code = code
+        if message:
+            if not isinstance(message, bytes):
+                raise TypeError("HTTP response status message must be bytes")
+            self.code_message = message
+        else:
+            self.code_message = RESPONSES.get(code, b"Unknown Status")
+
+    def setHeader(self, name, value):
+        """
+        Set an HTTP response header.  Overrides any previously set values for
+        this header.
+
+        @type name: L{bytes} or L{str}
+        @param name: The name of the header for which to set the value.
+
+        @type value: L{bytes} or L{str}
+        @param value: The value to set for the named header. A L{str} will be
+            UTF-8 encoded, which may not interoperable with other
+            implementations. Avoid passing non-ASCII characters if possible.
+        """
+        self.responseHeaders.setRawHeaders(name, [value])
+
+    def redirect(self, url):
+        """
+        Utility function that does a redirect.
+
+        Set the response code to L{FOUND} and the I{Location} header to the
+        given URL.
+
+        The request should have C{finish()} called after this.
+
+        @param url: I{Location} header value.
+        @type url: L{bytes} or L{str}
+        """
+        self.setResponseCode(FOUND)
+        self.setHeader(b"location", url)
+
+    def setLastModified(self, when):
+        """
+        Set the C{Last-Modified} time for the response to this request.
+
+        If I am called more than once, I ignore attempts to set
+        Last-Modified earlier, only replacing the Last-Modified time
+        if it is to a later value.
+
+        If I am a conditional request, I may modify my response code
+        to L{NOT_MODIFIED} if appropriate for the time given.
+
+        @param when: The last time the resource being returned was
+            modified, in seconds since the epoch.
+        @type when: number
+        @return: If I am a I{If-Modified-Since} conditional request and
+            the time given is not newer than the condition, I return
+            L{http.CACHED<CACHED>} to indicate that you should write no
+            body.  Otherwise, I return a false value.
+        """
+        # time.time() may be a float, but the HTTP-date strings are
+        # only good for whole seconds.
+        when = int(math.ceil(when))
+        if (not self.lastModified) or (self.lastModified < when):
+            self.lastModified = when
+
+        modifiedSince = self.getHeader(b"if-modified-since")
+        if modifiedSince:
+            firstPart = modifiedSince.split(b";", 1)[0]
+            try:
+                modifiedSince = stringToDatetime(firstPart)
+            except ValueError:
+                return None
+            if modifiedSince >= self.lastModified:
+                self.setResponseCode(NOT_MODIFIED)
+                return CACHED
+        return None
+
+    def setETag(self, etag):
+        """
+        Set an C{entity tag} for the outgoing response.
+
+        That's \"entity tag\" as in the HTTP/1.1 C{ETag} header, \"used
+        for comparing two or more entities from the same requested
+        resource.\"
+
+        If I am a conditional request, I may modify my response code
+        to L{NOT_MODIFIED} or L{PRECONDITION_FAILED}, if appropriate
+        for the tag given.
+
+        @param etag: The entity tag for the resource being returned.
+        @type etag: string
+        @return: If I am a C{If-None-Match} conditional request and
+            the tag matches one in the request, I return
+            L{http.CACHED<CACHED>} to indicate that you should write
+            no body.  Otherwise, I return a false value.
+        """
+        if etag:
+            self.etag = etag
+
+        tags = self.getHeader(b"if-none-match")
+        if tags:
+            tags = tags.split()
+            if (etag in tags) or (b"*" in tags):
+                self.setResponseCode(
+                    ((self.method in (b"HEAD", b"GET")) and NOT_MODIFIED)
+                    or PRECONDITION_FAILED
+                )
+                return CACHED
+        return None
+
+    def getAllHeaders(self):
+        """
+        Return dictionary mapping the names of all received headers to the last
+        value received for each.
+
+        Since this method does not return all header information,
+        C{self.requestHeaders.getAllRawHeaders()} may be preferred.
+        """
+        headers = {}
+        for k, v in self.requestHeaders.getAllRawHeaders():
+            headers[k.lower()] = v[-1]
+        return headers
+
+    def getRequestHostname(self):
+        """
+        Get the hostname that the HTTP client passed in to the request.
+
+        @see: L{IRequest.getRequestHostname}
+
+        @returns: the requested hostname
+
+        @rtype: C{bytes}
+        """
+        host = self.getHeader(b"host")
+        if host is not None:
+            match = _hostHeaderExpression.match(host)
+            if match is not None:
+                return match.group("host")
+        return networkString(self.getHost().host)
+
+    def getHost(self):
+        """
+        Get my originally requesting transport's host.
+
+        Don't rely on the 'transport' attribute, since Request objects may be
+        copied remotely.  For information on this method's return value, see
+        L{twisted.internet.tcp.Port}.
+        """
+        return self.host
+
+    def setHost(self, host, port, ssl=0):
+        """
+        Change the host and port the request thinks it's using.
+
+        This method is useful for working with reverse HTTP proxies (e.g.
+        both Squid and Apache's mod_proxy can do this), when the address
+        the HTTP client is using is different than the one we're listening on.
+
+        For example, Apache may be listening on https://www.example.com/, and
+        then forwarding requests to http://localhost:8080/, but we don't want
+        HTML produced by Twisted to say b'http://localhost:8080/', they should
+        say b'https://www.example.com/', so we do::
+
+           request.setHost(b'www.example.com', 443, ssl=1)
+
+        @type host: C{bytes}
+        @param host: The value to which to change the host header.
+
+        @type ssl: C{bool}
+        @param ssl: A flag which, if C{True}, indicates that the request is
+            considered secure (if C{True}, L{isSecure} will return C{True}).
+        """
+        self._forceSSL = ssl  # set first so isSecure will work
+        if self.isSecure():
+            default = 443
+        else:
+            default = 80
+        if port == default:
+            hostHeader = host
+        else:
+            hostHeader = b"%b:%d" % (host, port)
+        self.requestHeaders.setRawHeaders(b"host", [hostHeader])
+        self.host = address.IPv4Address("TCP", host, port)
+
+    @deprecated(Version("Twisted", 18, 4, 0), replacement="getClientAddress")
+    def getClientIP(self):
+        """
+        Return the IP address of the client who submitted this request.
+
+        This method is B{deprecated}.  Use L{getClientAddress} instead.
+
+        @returns: the client IP address
+        @rtype: C{str}
+        """
+        if isinstance(self.client, (address.IPv4Address, address.IPv6Address)):
+            return self.client.host
+        else:
+            return None
+
+    def getClientAddress(self):
+        """
+        Return the address of the client who submitted this request.
+
+        This may not be a network address (e.g., a server listening on
+        a UNIX domain socket will cause this to return
+        L{UNIXAddress}).  Callers must check the type of the returned
+        address.
+
+        @since: 18.4
+
+        @return: the client's address.
+        @rtype: L{IAddress}
+        """
+        return self.client
+
+    def isSecure(self):
+        """
+        Return L{True} if this request is using a secure transport.
+
+        Normally this method returns L{True} if this request's L{HTTPChannel}
+        instance is using a transport that implements
+        L{interfaces.ISSLTransport}.
+
+        This will also return L{True} if L{Request.setHost} has been called
+        with C{ssl=True}.
+
+        @returns: L{True} if this request is secure
+        @rtype: C{bool}
+        """
+        if self._forceSSL:
+            return True
+        channel = getattr(self, "channel", None)
+        if channel is None:
+            return False
+        return channel.isSecure()
+
+    def _authorize(self):
+        # Authorization, (mostly) per the RFC
+        try:
+            authh = self.getHeader(b"Authorization")
+            if not authh:
+                self.user = self.password = b""
+                return
+            bas, upw = authh.split()
+            if bas.lower() != b"basic":
+                raise ValueError()
+            upw = base64.b64decode(upw)
+            self.user, self.password = upw.split(b":", 1)
+        except (binascii.Error, ValueError):
+            self.user = self.password = b""
+        except BaseException:
+            self._log.failure("")
+            self.user = self.password = b""
+
+    def getUser(self):
+        """
+        Return the HTTP user sent with this request, if any.
+
+        If no user was supplied, return the empty string.
+
+        @returns: the HTTP user, if any
+        @rtype: C{bytes}
+        """
+        try:
+            return self.user
+        except BaseException:
+            pass
+        self._authorize()
+        return self.user
+
+    def getPassword(self):
+        """
+        Return the HTTP password sent with this request, if any.
+
+        If no password was supplied, return the empty string.
+
+        @returns: the HTTP password, if any
+        @rtype: C{bytes}
+        """
+        try:
+            return self.password
+        except BaseException:
+            pass
+        self._authorize()
+        return self.password
+
+    def connectionLost(self, reason):
+        """
+        There is no longer a connection for this request to respond over.
+        Clean up anything which can't be useful anymore.
+        """
+        self._disconnected = True
+        self.channel = None
+        if self.content is not None:
+            self.content.close()
+        for d in self.notifications:
+            d.errback(reason)
+        self.notifications = []
+
+    def loseConnection(self):
+        """
+        Pass the loseConnection through to the underlying channel.
+        """
+        if self.channel is not None:
+            self.channel.loseConnection()
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Determines if two requests are the same object.
+
+        @param other: Another object whose identity will be compared
+            to this instance's.
+
+        @return: L{True} when the two are the same object and L{False}
+            when not.
+        @rtype: L{bool}
+        """
+        # When other is not an instance of request, return
+        # NotImplemented so that Python uses other.__eq__ to perform
+        # the comparison.  This ensures that a Request proxy generated
+        # by proxyForInterface compares equal to an actual Request
+        # instanceby turning request != proxy into proxy != request.
+        if isinstance(other, Request):
+            return self is other
+        return NotImplemented
+
+    def __hash__(self):
+        """
+        A C{Request} is hashable so that it can be used as a mapping key.
+
+        @return: A C{int} based on the instance's identity.
+        """
+        return id(self)
+
+
+class _DataLoss(Exception):
+    """
+    L{_DataLoss} indicates that not all of a message body was received. This
+    is only one of several possible exceptions which may indicate that data
+    was lost.  Because of this, it should not be checked for by
+    specifically; any unexpected exception should be treated as having
+    caused data loss.
+    """
+
+
+class PotentialDataLoss(Exception):
+    """
+    L{PotentialDataLoss} may be raised by a transfer encoding decoder's
+    C{noMoreData} method to indicate that it cannot be determined if the
+    entire response body has been delivered.  This only occurs when making
+    requests to HTTP servers which do not set I{Content-Length} or a
+    I{Transfer-Encoding} in the response because in this case the end of the
+    response is indicated by the connection being closed, an event which may
+    also be due to a transient network problem or other error.
+    """
+
+
+class _MalformedChunkedDataError(Exception):
+    """
+    C{_ChunkedTransferDecoder} raises L{_MalformedChunkedDataError} from its
+    C{dataReceived} method when it encounters malformed data. This exception
+    indicates a client-side error. If this exception is raised, the connection
+    should be dropped with a 400 error.
+    """
+
+
+class _IdentityTransferDecoder:
+    """
+    Protocol for accumulating bytes up to a specified length.  This handles the
+    case where no I{Transfer-Encoding} is specified.
+
+    @ivar contentLength: Counter keeping track of how many more bytes there are
+        to receive.
+
+    @ivar dataCallback: A one-argument callable which will be invoked each
+        time application data is received.
+
+    @ivar finishCallback: A one-argument callable which will be invoked when
+        the terminal chunk is received.  It will be invoked with all bytes
+        which were delivered to this protocol which came after the terminal
+        chunk.
+    """
+
+    def __init__(self, contentLength, dataCallback, finishCallback):
+        self.contentLength = contentLength
+        self.dataCallback = dataCallback
+        self.finishCallback = finishCallback
+
+    def dataReceived(self, data):
+        """
+        Interpret the next chunk of bytes received.  Either deliver them to the
+        data callback or invoke the finish callback if enough bytes have been
+        received.
+
+        @raise RuntimeError: If the finish callback has already been invoked
+            during a previous call to this methood.
+        """
+        if self.dataCallback is None:
+            raise RuntimeError(
+                "_IdentityTransferDecoder cannot decode data after finishing"
+            )
+
+        if self.contentLength is None:
+            self.dataCallback(data)
+        elif len(data) < self.contentLength:
+            self.contentLength -= len(data)
+            self.dataCallback(data)
+        else:
+            # Make the state consistent before invoking any code belonging to
+            # anyone else in case noMoreData ends up being called beneath this
+            # stack frame.
+            contentLength = self.contentLength
+            dataCallback = self.dataCallback
+            finishCallback = self.finishCallback
+            self.dataCallback = self.finishCallback = None
+            self.contentLength = 0
+
+            dataCallback(data[:contentLength])
+            finishCallback(data[contentLength:])
+
+    def noMoreData(self):
+        """
+        All data which will be delivered to this decoder has been.  Check to
+        make sure as much data as was expected has been received.
+
+        @raise PotentialDataLoss: If the content length is unknown.
+        @raise _DataLoss: If the content length is known and fewer than that
+            many bytes have been delivered.
+
+        @return: L{None}
+        """
+        finishCallback = self.finishCallback
+        self.dataCallback = self.finishCallback = None
+        if self.contentLength is None:
+            finishCallback(b"")
+            raise PotentialDataLoss()
+        elif self.contentLength != 0:
+            raise _DataLoss()
+
+
+maxChunkSizeLineLength = 1024
+
+
+_chunkExtChars = (
+    b"\t !\"#$%&'()*+,-./0123456789:;<=>?@"
+    b"ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_`"
+    b"abcdefghijklmnopqrstuvwxyz{|}~"
+    b"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+    b"\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+    b"\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+    b"\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+    b"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+    b"\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+    b"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+    b"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+)
+"""
+Characters that are valid in a chunk extension.
+
+See RFC 7230 section 4.1.1::
+
+     chunk-ext      = *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
+
+     chunk-ext-name = token
+     chunk-ext-val  = token / quoted-string
+
+And section 3.2.6::
+
+     token          = 1*tchar
+
+     tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+                    / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+                    / DIGIT / ALPHA
+                    ; any VCHAR, except delimiters
+
+     quoted-string  = DQUOTE *( qdtext / quoted-pair ) DQUOTE
+     qdtext         = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text
+     obs-text       = %x80-FF
+
+We don't check if chunk extensions are well-formed beyond validating that they
+don't contain characters outside this range.
+"""
+
+
+class _ChunkedTransferDecoder:
+    """
+    Protocol for decoding I{chunked} Transfer-Encoding, as defined by RFC 7230,
+    section 4.1.  This protocol can interpret the contents of a request or
+    response body which uses the I{chunked} Transfer-Encoding.  It cannot
+    interpret any of the rest of the HTTP protocol.
+
+    It may make sense for _ChunkedTransferDecoder to be an actual IProtocol
+    implementation.  Currently, the only user of this class will only ever
+    call dataReceived on it.  However, it might be an improvement if the
+    user could connect this to a transport and deliver connection lost
+    notification.  This way, `dataCallback` becomes `self.transport.write`
+    and perhaps `finishCallback` becomes `self.transport.loseConnection()`
+    (although I'm not sure where the extra data goes in that case).  This
+    could also allow this object to indicate to the receiver of data that
+    the stream was not completely received, an error case which should be
+    noticed. -exarkun
+
+    @ivar dataCallback: A one-argument callable which will be invoked each
+        time application data is received. This callback is not reentrant.
+
+    @ivar finishCallback: A one-argument callable which will be invoked when
+        the terminal chunk is received.  It will be invoked with all bytes
+        which were delivered to this protocol which came after the terminal
+        chunk.
+
+    @ivar length: Counter keeping track of how many more bytes in a chunk there
+        are to receive.
+
+    @ivar state: One of C{'CHUNK_LENGTH'}, C{'CRLF'}, C{'TRAILER'},
+        C{'BODY'}, or C{'FINISHED'}.  For C{'CHUNK_LENGTH'}, data for the
+        chunk length line is currently being read.  For C{'CRLF'}, the CR LF
+        pair which follows each chunk is being read. For C{'TRAILER'}, the CR
+        LF pair which follows the terminal 0-length chunk is currently being
+        read. For C{'BODY'}, the contents of a chunk are being read. For
+        C{'FINISHED'}, the last chunk has been completely read and no more
+        input is valid.
+
+    @ivar _buffer: Accumulated received data for the current state. At each
+        state transition this is truncated at the front so that index 0 is
+        where the next state shall begin.
+
+    @ivar _start: While in the C{'CHUNK_LENGTH'} state, tracks the index into
+        the buffer at which search for CRLF should resume. Resuming the search
+        at this position avoids doing quadratic work if the chunk length line
+        arrives over many calls to C{dataReceived}.
+
+        Not used in any other state.
+    """
+
+    state = "CHUNK_LENGTH"
+
+    def __init__(
+        self,
+        dataCallback: Callable[[bytes], None],
+        finishCallback: Callable[[bytes], None],
+    ) -> None:
+        self.dataCallback = dataCallback
+        self.finishCallback = finishCallback
+        self._buffer = bytearray()
+        self._start = 0
+
+    def _dataReceived_CHUNK_LENGTH(self) -> bool:
+        """
+        Read the chunk size line, ignoring any extensions.
+
+        @returns: C{True} once the line has been read and removed from
+            C{self._buffer}.  C{False} when more data is required.
+
+        @raises _MalformedChunkedDataError: when the chunk size cannot be
+            decoded or the length of the line exceeds L{maxChunkSizeLineLength}.
+        """
+        eolIndex = self._buffer.find(b"\r\n", self._start)
+
+        if eolIndex >= maxChunkSizeLineLength or (
+            eolIndex == -1 and len(self._buffer) > maxChunkSizeLineLength
+        ):
+            raise _MalformedChunkedDataError(
+                "Chunk size line exceeds maximum of {} bytes.".format(
+                    maxChunkSizeLineLength
+                )
+            )
+
+        if eolIndex == -1:
+            # Restart the search upon receipt of more data at the start of the
+            # new data, minus one in case the last character of the buffer is
+            # CR.
+            self._start = len(self._buffer) - 1
+            return False
+
+        endOfLengthIndex = self._buffer.find(b";", 0, eolIndex)
+        if endOfLengthIndex == -1:
+            endOfLengthIndex = eolIndex
+        rawLength = self._buffer[0:endOfLengthIndex]
+        try:
+            length = _hexint(rawLength)
+        except ValueError:
+            raise _MalformedChunkedDataError("Chunk-size must be an integer.")
+
+        ext = self._buffer[endOfLengthIndex + 1 : eolIndex]
+        if ext and ext.translate(None, _chunkExtChars) != b"":
+            raise _MalformedChunkedDataError(
+                f"Invalid characters in chunk extensions: {ext!r}."
+            )
+
+        if length == 0:
+            self.state = "TRAILER"
+        else:
+            self.state = "BODY"
+
+        self.length = length
+        del self._buffer[0 : eolIndex + 2]
+        self._start = 0
+        return True
+
+    def _dataReceived_CRLF(self) -> bool:
+        """
+        Await the carriage return and line feed characters that are the end of
+        chunk marker that follow the chunk data.
+
+        @returns: C{True} when the CRLF have been read, otherwise C{False}.
+
+        @raises _MalformedChunkedDataError: when anything other than CRLF are
+            received.
+        """
+        if len(self._buffer) < 2:
+            return False
+
+        if not self._buffer.startswith(b"\r\n"):
+            raise _MalformedChunkedDataError("Chunk did not end with CRLF")
+
+        self.state = "CHUNK_LENGTH"
+        del self._buffer[0:2]
+        return True
+
+    def _dataReceived_TRAILER(self) -> bool:
+        """
+        Await the carriage return and line feed characters that follow the
+        terminal zero-length chunk. Then invoke C{finishCallback} and switch to
+        state C{'FINISHED'}.
+
+        @returns: C{False}, as there is either insufficient data to continue,
+            or no data remains.
+
+        @raises _MalformedChunkedDataError: when anything other than CRLF is
+            received.
+        """
+        if len(self._buffer) < 2:
+            return False
+
+        if not self._buffer.startswith(b"\r\n"):
+            raise _MalformedChunkedDataError("Chunk did not end with CRLF")
+
+        data = memoryview(self._buffer)[2:].tobytes()
+        del self._buffer[:]
+        self.state = "FINISHED"
+        self.finishCallback(data)
+        return False
+
+    def _dataReceived_BODY(self) -> bool:
+        """
+        Deliver any available chunk data to the C{dataCallback}. When all the
+        remaining data for the chunk arrives, switch to state C{'CRLF'}.
+
+        @returns: C{True} to continue processing of any buffered data.
+        """
+        if len(self._buffer) >= self.length:
+            chunk = memoryview(self._buffer)[: self.length].tobytes()
+            del self._buffer[: self.length]
+            self.state = "CRLF"
+            self.dataCallback(chunk)
+        else:
+            chunk = bytes(self._buffer)
+            self.length -= len(chunk)
+            del self._buffer[:]
+            self.dataCallback(chunk)
+        return True
+
+    def _dataReceived_FINISHED(self) -> bool:
+        """
+        Once C{finishCallback} has been invoked receipt of additional data
+        raises L{RuntimeError} because it represents a programming error in
+        the caller.
+        """
+        raise RuntimeError(
+            "_ChunkedTransferDecoder.dataReceived called after last "
+            "chunk was processed"
+        )
+
+    def dataReceived(self, data: bytes) -> None:
+        """
+        Interpret data from a request or response body which uses the
+        I{chunked} Transfer-Encoding.
+        """
+        self._buffer += data
+        goOn = True
+        while goOn and self._buffer:
+            goOn = getattr(self, "_dataReceived_" + self.state)()
+
+    def noMoreData(self) -> None:
+        """
+        Verify that all data has been received.  If it has not been, raise
+        L{_DataLoss}.
+        """
+        if self.state != "FINISHED":
+            raise _DataLoss(
+                "Chunked decoder in %r state, still expecting more data to "
+                "get to 'FINISHED' state." % (self.state,)
+            )
+
+
+@implementer(interfaces.IPushProducer)
+class _NoPushProducer:
+    """
+    A no-op version of L{interfaces.IPushProducer}, used to abstract over the
+    possibility that a L{HTTPChannel} transport does not provide
+    L{IPushProducer}.
+    """
+
+    def pauseProducing(self):
+        """
+        Pause producing data.
+
+        Tells a producer that it has produced too much data to process for
+        the time being, and to stop until resumeProducing() is called.
+        """
+
+    def resumeProducing(self):
+        """
+        Resume producing data.
+
+        This tells a producer to re-add itself to the main loop and produce
+        more data for its consumer.
+        """
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register to receive data from a producer.
+
+        @param producer: The producer to register.
+        @param streaming: Whether this is a streaming producer or not.
+        """
+
+    def unregisterProducer(self):
+        """
+        Stop consuming data from a producer, without disconnecting.
+        """
+
+    def stopProducing(self):
+        """
+        IProducer.stopProducing
+        """
+
+
+@implementer(interfaces.ITransport, interfaces.IPushProducer, interfaces.IConsumer)
+class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
+    """
+    A receiver for HTTP requests.
+
+    The L{HTTPChannel} provides L{interfaces.ITransport} and
+    L{interfaces.IConsumer} to the L{Request} objects it creates.  It also
+    implements L{interfaces.IPushProducer} to C{self.transport}, allowing the
+    transport to pause it.
+
+    @ivar MAX_LENGTH: Maximum length for initial request line and each line
+        from the header.
+
+    @ivar _transferDecoder: L{None} or a decoder instance if the request body
+        uses the I{chunked} Transfer-Encoding.
+    @type _transferDecoder: L{_ChunkedTransferDecoder}
+
+    @ivar maxHeaders: Maximum number of headers allowed per request.
+    @type maxHeaders: C{int}
+
+    @ivar totalHeadersSize: Maximum bytes for request line plus all headers
+        from the request.
+    @type totalHeadersSize: C{int}
+
+    @ivar _receivedHeaderSize: Bytes received so far for the header.
+    @type _receivedHeaderSize: C{int}
+
+    @ivar _handlingRequest: Whether a request is currently being processed.
+    @type _handlingRequest: L{bool}
+
+    @ivar _dataBuffer: Any data that has been received from the connection
+        while processing an outstanding request.
+    @type _dataBuffer: L{list} of L{bytes}
+
+    @ivar _networkProducer: Either the transport, if it provides
+        L{interfaces.IPushProducer}, or a null implementation of
+        L{interfaces.IPushProducer}.  Used to attempt to prevent the transport
+        from producing excess data when we're responding to a request.
+    @type _networkProducer: L{interfaces.IPushProducer}
+
+    @ivar _requestProducer: If the L{Request} object or anything it calls
+        registers itself as an L{interfaces.IProducer}, it will be stored here.
+        This is used to create a producing pipeline: pause/resume producing
+        methods will be propagated from the C{transport}, through the
+        L{HTTPChannel} instance, to the c{_requestProducer}.
+
+        The reason we proxy through the producing methods rather than the old
+        behaviour (where we literally just set the L{Request} object as the
+        producer on the transport) is because we want to be able to exert
+        backpressure on the client to prevent it from sending in arbitrarily
+        many requests without ever reading responses.  Essentially, if the
+        client never reads our responses we will eventually stop reading its
+        requests.
+
+    @type _requestProducer: L{interfaces.IPushProducer}
+
+    @ivar _requestProducerStreaming: A boolean that tracks whether the producer
+        on the L{Request} side of this channel has registered itself as a
+        L{interfaces.IPushProducer} or an L{interfaces.IPullProducer}.
+    @type _requestProducerStreaming: L{bool} or L{None}
+
+    @ivar _waitingForTransport: A boolean that tracks whether the transport has
+        asked us to stop producing.  This is used to keep track of what we're
+        waiting for: if the transport has asked us to stop producing then we
+        don't want to unpause the transport until it asks us to produce again.
+    @type _waitingForTransport: L{bool}
+
+    @ivar abortTimeout: The number of seconds to wait after we attempt to shut
+        the transport down cleanly to give up and forcibly terminate it.  This
+        is only used when we time a connection out, to prevent errors causing
+        the FD to get leaked.  If this is L{None}, we will wait forever.
+    @type abortTimeout: L{int}
+
+    @ivar _abortingCall: The L{twisted.internet.base.DelayedCall} that will be
+        used to forcibly close the transport if it doesn't close cleanly.
+    @type _abortingCall: L{twisted.internet.base.DelayedCall}
+
+    @ivar _optimisticEagerReadSize: When a resource takes a long time to answer
+        a request (via L{twisted.web.server.NOT_DONE_YET}, hopefully one day by
+        a L{Deferred}), we would like to be able to let that resource know
+        about the underlying transport disappearing as promptly as possible,
+        via L{Request.notifyFinish}, and therefore via
+        C{self.requests[...].connectionLost()} on this L{HTTPChannel}.
+
+        However, in order to simplify application logic, we implement
+        head-of-line blocking, and do not relay pipelined requests to the
+        application until the previous request has been answered.  This means
+        that said application cannot dispose of any entity-body that comes in
+        from those subsequent requests, which may be arbitrarily large, and it
+        may need to be buffered in memory.
+
+        To implement this tradeoff between prompt notification when possible
+        (in the most frequent case of non-pipelined requests) and correct
+        behavior when not (say, if a client sends a very long-running GET
+        request followed by a PUT request with a very large body) we will
+        continue reading pipelined requests into C{self._dataBuffer} up to a
+        given limit.
+
+        C{_optimisticEagerReadSize} is the number of bytes we will accept from
+        the client and buffer before pausing the transport.
+
+        This behavior has been in place since Twisted 17.9.0 .
+
+    @type _optimisticEagerReadSize: L{int}
+    """
+
+    maxHeaders = 500
+    totalHeadersSize = 16384
+    abortTimeout = 15
+
+    length = 0
+    persistent = 1
+    __header = b""
+    __first_line = 1
+    __content = None
+
+    # set in instances or subclasses
+    requestFactory = Request
+
+    _savedTimeOut = None
+    _receivedHeaderCount = 0
+    _receivedHeaderSize = 0
+    _requestProducer = None
+    _requestProducerStreaming = None
+    _waitingForTransport = False
+    _abortingCall = None
+    _optimisticEagerReadSize = 0x4000
+    _log = Logger()
+
+    def __init__(self):
+        # the request queue
+        self.requests = []
+        self._handlingRequest = False
+        self._dataBuffer = []
+        self._transferDecoder = None
+
+    def connectionMade(self):
+        self.setTimeout(self.timeOut)
+        self._networkProducer = interfaces.IPushProducer(
+            self.transport, _NoPushProducer()
+        )
+        self._networkProducer.registerProducer(self, True)
+
+    def lineReceived(self, line):
+        """
+        Called for each line from request until the end of headers when
+        it enters binary mode.
+        """
+        self.resetTimeout()
+
+        self._receivedHeaderSize += len(line)
+        if self._receivedHeaderSize > self.totalHeadersSize:
+            self._respondToBadRequestAndDisconnect()
+            return
+
+        if self.__first_line:
+            # if this connection is not persistent, drop any data which
+            # the client (illegally) sent after the last request.
+            if not self.persistent:
+                self.dataReceived = self.lineReceived = lambda *args: None
+                return
+
+            # IE sends an extraneous empty line (\r\n) after a POST request;
+            # eat up such a line, but only ONCE
+            if not line and self.__first_line == 1:
+                self.__first_line = 2
+                return
+
+            # create a new Request object
+            if INonQueuedRequestFactory.providedBy(self.requestFactory):
+                request = self.requestFactory(self)
+            else:
+                request = self.requestFactory(self, len(self.requests))
+            self.requests.append(request)
+
+            self.__first_line = 0
+
+            parts = line.split()
+            if len(parts) != 3:
+                self._respondToBadRequestAndDisconnect()
+                return
+            command, request, version = parts
+            try:
+                command.decode("ascii")
+            except UnicodeDecodeError:
+                self._respondToBadRequestAndDisconnect()
+                return
+
+            self._command = command
+            self._path = request
+            self._version = version
+        elif line == b"":
+            # End of headers.
+            if self.__header:
+                ok = self.headerReceived(self.__header)
+                # If the last header we got is invalid, we MUST NOT proceed
+                # with processing. We'll have sent a 400 anyway, so just stop.
+                if not ok:
+                    return
+            self.__header = b""
+            self.allHeadersReceived()
+            if self.length == 0:
+                self.allContentReceived()
+            else:
+                self.setRawMode()
+        elif line[0] in b" \t":
+            # Continuation of a multi line header.
+            self.__header += b" " + line.lstrip(b" \t")
+        # Regular header line.
+        # Processing of header line is delayed to allow accumulating multi
+        # line headers.
+        else:
+            if self.__header:
+                self.headerReceived(self.__header)
+            self.__header = line
+
+    def _finishRequestBody(self, data):
+        self.allContentReceived()
+        self._dataBuffer.append(data)
+
+    def _maybeChooseTransferDecoder(self, header, data):
+        """
+        If the provided header is C{content-length} or
+        C{transfer-encoding}, choose the appropriate decoder if any.
+
+        Returns L{True} if the request can proceed and L{False} if not.
+        """
+
+        def fail():
+            self._respondToBadRequestAndDisconnect()
+            self.length = None
+            return False
+
+        # Can this header determine the length?
+        if header == b"content-length":
+            if not data.isdigit():
+                return fail()
+            try:
+                length = int(data)
+            except ValueError:
+                return fail()
+            newTransferDecoder = _IdentityTransferDecoder(
+                length, self.requests[-1].handleContentChunk, self._finishRequestBody
+            )
+        elif header == b"transfer-encoding":
+            # XXX Rather poorly tested code block, apparently only exercised by
+            # test_chunkedEncoding
+            if data.lower() == b"chunked":
+                length = None
+                newTransferDecoder = _ChunkedTransferDecoder(
+                    self.requests[-1].handleContentChunk, self._finishRequestBody
+                )
+            elif data.lower() == b"identity":
+                return True
+            else:
+                return fail()
+        else:
+            # It's not a length related header, so exit
+            return True
+
+        if self._transferDecoder is not None:
+            return fail()
+        else:
+            self.length = length
+            self._transferDecoder = newTransferDecoder
+            return True
+
+    def headerReceived(self, line):
+        """
+        Do pre-processing (for content-length) and store this header away.
+        Enforce the per-request header limit.
+
+        @type line: C{bytes}
+        @param line: A line from the header section of a request, excluding the
+            line delimiter.
+
+        @return: A flag indicating whether the header was valid.
+        @rtype: L{bool}
+        """
+        try:
+            header, data = line.split(b":", 1)
+        except ValueError:
+            self._respondToBadRequestAndDisconnect()
+            return False
+
+        if not header or header[-1:].isspace():
+            self._respondToBadRequestAndDisconnect()
+            return False
+
+        header = header.lower()
+        data = data.strip(b" \t")
+
+        if not self._maybeChooseTransferDecoder(header, data):
+            return False
+
+        reqHeaders = self.requests[-1].requestHeaders
+        values = reqHeaders.getRawHeaders(header)
+        if values is not None:
+            values.append(data)
+        else:
+            reqHeaders.setRawHeaders(header, [data])
+
+        self._receivedHeaderCount += 1
+        if self._receivedHeaderCount > self.maxHeaders:
+            self._respondToBadRequestAndDisconnect()
+            return False
+
+        return True
+
+    def allContentReceived(self):
+        command = self._command
+        path = self._path
+        version = self._version
+
+        # reset ALL state variables, so we don't interfere with next request
+        self.length = 0
+        self._receivedHeaderCount = 0
+        self._receivedHeaderSize = 0
+        self.__first_line = 1
+        self._transferDecoder = None
+        del self._command, self._path, self._version
+
+        # Disable the idle timeout, in case this request takes a long
+        # time to finish generating output.
+        if self.timeOut:
+            self._savedTimeOut = self.setTimeout(None)
+
+        self._handlingRequest = True
+
+        # We go into raw mode here even though we will be receiving lines next
+        # in the protocol; however, this data will be buffered and then passed
+        # back to line mode in the setLineMode call in requestDone.
+        self.setRawMode()
+
+        req = self.requests[-1]
+        req.requestReceived(command, path, version)
+
+    def rawDataReceived(self, data: bytes) -> None:
+        """
+        This is called when this HTTP/1.1 parser is in raw mode rather than
+        line mode.
+
+        It may be in raw mode for one of two reasons:
+
+            1. All the headers of a request have been received and this
+               L{HTTPChannel} is currently receiving its body.
+
+            2. The full content of a request has been received and is currently
+               being processed asynchronously, and this L{HTTPChannel} is
+               buffering the data of all subsequent requests to be parsed
+               later.
+
+        In the second state, the data will be played back later.
+
+        @note: This isn't really a public API, and should be invoked only by
+            L{LineReceiver}'s line parsing logic.  If you wish to drive an
+            L{HTTPChannel} from a custom data source, call C{dataReceived} on
+            it directly.
+
+        @see: L{LineReceive.rawDataReceived}
+        """
+        if self._handlingRequest:
+            self._dataBuffer.append(data)
+            if (
+                sum(map(len, self._dataBuffer)) > self._optimisticEagerReadSize
+            ) and not self._waitingForTransport:
+                # If we received more data than a small limit while processing
+                # the head-of-line request, apply TCP backpressure to our peer
+                # to get them to stop sending more request data until we're
+                # ready.  See docstring for _optimisticEagerReadSize above.
+                self._networkProducer.pauseProducing()
+            return
+
+        self.resetTimeout()
+
+        try:
+            self._transferDecoder.dataReceived(data)
+        except _MalformedChunkedDataError:
+            self._respondToBadRequestAndDisconnect()
+
+    def allHeadersReceived(self):
+        req = self.requests[-1]
+        req.parseCookies()
+        self.persistent = self.checkPersistence(req, self._version)
+        req.gotLength(self.length)
+        # Handle 'Expect: 100-continue' with automated 100 response code,
+        # a simplistic implementation of RFC 2686 8.2.3:
+        expectContinue = req.requestHeaders.getRawHeaders(b"expect")
+        if (
+            expectContinue
+            and expectContinue[0].lower() == b"100-continue"
+            and self._version == b"HTTP/1.1"
+        ):
+            self._send100Continue()
+
+    def checkPersistence(self, request, version):
+        """
+        Check if the channel should close or not.
+
+        @param request: The request most recently received over this channel
+            against which checks will be made to determine if this connection
+            can remain open after a matching response is returned.
+
+        @type version: C{bytes}
+        @param version: The version of the request.
+
+        @rtype: C{bool}
+        @return: A flag which, if C{True}, indicates that this connection may
+            remain open to receive another request; if C{False}, the connection
+            must be closed in order to indicate the completion of the response
+            to C{request}.
+        """
+        connection = request.requestHeaders.getRawHeaders(b"connection")
+        if connection:
+            tokens = [t.lower() for t in connection[0].split(b" ")]
+        else:
+            tokens = []
+
+        # Once any HTTP 0.9 or HTTP 1.0 request is received, the connection is
+        # no longer allowed to be persistent.  At this point in processing the
+        # request, we don't yet know if it will be possible to set a
+        # Content-Length in the response.  If it is not, then the connection
+        # will have to be closed to end an HTTP 0.9 or HTTP 1.0 response.
+
+        # If the checkPersistence call happened later, after the Content-Length
+        # has been determined (or determined not to be set), it would probably
+        # be possible to have persistent connections with HTTP 0.9 and HTTP 1.0.
+        # This may not be worth the effort, though.  Just use HTTP 1.1, okay?
+
+        if version == b"HTTP/1.1":
+            if b"close" in tokens:
+                request.responseHeaders.setRawHeaders(b"connection", [b"close"])
+                return False
+            else:
+                return True
+        else:
+            return False
+
+    def requestDone(self, request):
+        """
+        Called by first request in queue when it is done.
+        """
+        if request != self.requests[0]:
+            raise TypeError
+        del self.requests[0]
+
+        # We should only resume the producer if we're not waiting for the
+        # transport.
+        if not self._waitingForTransport:
+            self._networkProducer.resumeProducing()
+
+        if self.persistent:
+            self._handlingRequest = False
+
+            if self._savedTimeOut:
+                self.setTimeout(self._savedTimeOut)
+
+            # Receive our buffered data, if any.
+            data = b"".join(self._dataBuffer)
+            self._dataBuffer = []
+            self.setLineMode(data)
+        else:
+            self.loseConnection()
+
+    def timeoutConnection(self):
+        self._log.info("Timing out client: {peer}", peer=str(self.transport.getPeer()))
+        if self.abortTimeout is not None:
+            # We use self.callLater because that's what TimeoutMixin does.
+            self._abortingCall = self.callLater(
+                self.abortTimeout, self.forceAbortClient
+            )
+        self.loseConnection()
+
+    def forceAbortClient(self):
+        """
+        Called if C{abortTimeout} seconds have passed since the timeout fired,
+        and the connection still hasn't gone away. This can really only happen
+        on extremely bad connections or when clients are maliciously attempting
+        to keep connections open.
+        """
+        self._log.info(
+            "Forcibly timing out client: {peer}", peer=str(self.transport.getPeer())
+        )
+        # We want to lose track of the _abortingCall so that no-one tries to
+        # cancel it.
+        self._abortingCall = None
+        self.transport.abortConnection()
+
+    def connectionLost(self, reason):
+        self.setTimeout(None)
+        for request in self.requests:
+            request.connectionLost(reason)
+
+        # If we were going to force-close the transport, we don't have to now.
+        if self._abortingCall is not None:
+            self._abortingCall.cancel()
+            self._abortingCall = None
+
+    def isSecure(self):
+        """
+        Return L{True} if this channel is using a secure transport.
+
+        Normally this method returns L{True} if this instance is using a
+        transport that implements L{interfaces.ISSLTransport}.
+
+        @returns: L{True} if this request is secure
+        @rtype: C{bool}
+        """
+        if interfaces.ISSLTransport(self.transport, None) is not None:
+            return True
+        return False
+
+    def writeHeaders(self, version, code, reason, headers):
+        """
+        Called by L{Request} objects to write a complete set of HTTP headers to
+        a transport.
+
+        @param version: The HTTP version in use.
+        @type version: L{bytes}
+
+        @param code: The HTTP status code to write.
+        @type code: L{bytes}
+
+        @param reason: The HTTP reason phrase to write.
+        @type reason: L{bytes}
+
+        @param headers: The headers to write to the transport.
+        @type headers: L{twisted.web.http_headers.Headers}
+        """
+        sanitizedHeaders = Headers()
+        for name, value in headers:
+            sanitizedHeaders.addRawHeader(name, value)
+
+        responseLine = version + b" " + code + b" " + reason + b"\r\n"
+        headerSequence = [responseLine]
+        headerSequence.extend(
+            name + b": " + value + b"\r\n"
+            for name, values in sanitizedHeaders.getAllRawHeaders()
+            for value in values
+        )
+        headerSequence.append(b"\r\n")
+        self.transport.writeSequence(headerSequence)
+
+    def write(self, data):
+        """
+        Called by L{Request} objects to write response data.
+
+        @param data: The data chunk to write to the stream.
+        @type data: L{bytes}
+
+        @return: L{None}
+        """
+        self.transport.write(data)
+
+    def writeSequence(self, iovec):
+        """
+        Write a list of strings to the HTTP response.
+
+        @param iovec: A list of byte strings to write to the stream.
+        @type iovec: L{list} of L{bytes}
+
+        @return: L{None}
+        """
+        self.transport.writeSequence(iovec)
+
+    def getPeer(self):
+        """
+        Get the remote address of this connection.
+
+        @return: An L{IAddress} provider.
+        """
+        return self.transport.getPeer()
+
+    def getHost(self):
+        """
+        Get the local address of this connection.
+
+        @return: An L{IAddress} provider.
+        """
+        return self.transport.getHost()
+
+    def loseConnection(self):
+        """
+        Closes the connection. Will write any data that is pending to be sent
+        on the network, but if this response has not yet been written to the
+        network will not write anything.
+
+        @return: L{None}
+        """
+        self._networkProducer.unregisterProducer()
+        return self.transport.loseConnection()
+
+    def registerProducer(self, producer, streaming):
+        """
+        Register to receive data from a producer.
+
+        This sets self to be a consumer for a producer.  When this object runs
+        out of data (as when a send(2) call on a socket succeeds in moving the
+        last data from a userspace buffer into a kernelspace buffer), it will
+        ask the producer to resumeProducing().
+
+        For L{IPullProducer} providers, C{resumeProducing} will be called once
+        each time data is required.
+
+        For L{IPushProducer} providers, C{pauseProducing} will be called
+        whenever the write buffer fills up and C{resumeProducing} will only be
+        called when it empties.
+
+        @type producer: L{IProducer} provider
+        @param producer: The L{IProducer} that will be producing data.
+
+        @type streaming: L{bool}
+        @param streaming: C{True} if C{producer} provides L{IPushProducer},
+        C{False} if C{producer} provides L{IPullProducer}.
+
+        @raise RuntimeError: If a producer is already registered.
+
+        @return: L{None}
+        """
+        if self._requestProducer is not None:
+            raise RuntimeError(
+                "Cannot register producer %s, because producer %s was never "
+                "unregistered." % (producer, self._requestProducer)
+            )
+
+        if not streaming:
+            producer = _PullToPush(producer, self)
+
+        self._requestProducer = producer
+        self._requestProducerStreaming = streaming
+
+        if not streaming:
+            producer.startStreaming()
+
+    def unregisterProducer(self):
+        """
+        Stop consuming data from a producer, without disconnecting.
+
+        @return: L{None}
+        """
+        if self._requestProducer is None:
+            return
+
+        if not self._requestProducerStreaming:
+            self._requestProducer.stopStreaming()
+
+        self._requestProducer = None
+        self._requestProducerStreaming = None
+
+    def stopProducing(self):
+        """
+        Stop producing data.
+
+        The HTTPChannel doesn't *actually* implement this, beacuse the
+        assumption is that it will only be called just before C{loseConnection}
+        is called. There's nothing sensible we can do other than call
+        C{loseConnection} anyway.
+        """
+        if self._requestProducer is not None:
+            self._requestProducer.stopProducing()
+
+    def pauseProducing(self):
+        """
+        Pause producing data.
+
+        This will be called by the transport when the send buffers have been
+        filled up. We want to simultaneously pause the producing L{Request}
+        object and also pause our transport.
+
+        The logic behind pausing the transport is specifically to avoid issues
+        like https://twistedmatrix.com/trac/ticket/8868. In this case, our
+        inability to send does not prevent us handling more requests, which
+        means we increasingly queue up more responses in our send buffer
+        without end. The easiest way to handle this is to ensure that if we are
+        unable to send our responses, we will not read further data from the
+        connection until the client pulls some data out. This is a bit of a
+        blunt instrument, but it's ok.
+
+        Note that this potentially interacts with timeout handling in a
+        positive way. Once the transport is paused the client may run into a
+        timeout which will cause us to tear the connection down. That's a good
+        thing!
+        """
+        self._waitingForTransport = True
+
+        # The first step is to tell any producer we might currently have
+        # registered to stop producing. If we can slow our applications down
+        # we should.
+        if self._requestProducer is not None:
+            self._requestProducer.pauseProducing()
+
+        # The next step here is to pause our own transport, as discussed in the
+        # docstring.
+        if not self._handlingRequest:
+            self._networkProducer.pauseProducing()
+
+    def resumeProducing(self):
+        """
+        Resume producing data.
+
+        This will be called by the transport when the send buffer has dropped
+        enough to actually send more data. When this happens we can unpause any
+        outstanding L{Request} producers we have, and also unpause our
+        transport.
+        """
+        self._waitingForTransport = False
+
+        if self._requestProducer is not None:
+            self._requestProducer.resumeProducing()
+
+        # We only want to resume the network producer if we're not currently
+        # waiting for a response to show up.
+        if not self._handlingRequest:
+            self._networkProducer.resumeProducing()
+
+    def _send100Continue(self):
+        """
+        Sends a 100 Continue response, used to signal to clients that further
+        processing will be performed.
+        """
+        self.transport.write(b"HTTP/1.1 100 Continue\r\n\r\n")
+
+    def _respondToBadRequestAndDisconnect(self):
+        """
+        This is a quick and dirty way of responding to bad requests.
+
+        As described by HTTP standard we should be patient and accept the
+        whole request from the client before sending a polite bad request
+        response, even in the case when clients send tons of data.
+        """
+        self.transport.write(b"HTTP/1.1 400 Bad Request\r\n\r\n")
+        self.loseConnection()
+
+
+def _escape(s):
+    """
+    Return a string like python repr, but always escaped as if surrounding
+    quotes were double quotes.
+
+    @param s: The string to escape.
+    @type s: L{bytes} or L{str}
+
+    @return: An escaped string.
+    @rtype: L{str}
+    """
+    if not isinstance(s, bytes):
+        s = s.encode("ascii")
+
+    r = repr(s)
+    if not isinstance(r, str):
+        r = r.decode("ascii")
+    if r.startswith("b"):
+        r = r[1:]
+    if r.startswith("'"):
+        return r[1:-1].replace('"', '\\"').replace("\\'", "'")
+    return r[1:-1]
+
+
+@provider(IAccessLogFormatter)
+def combinedLogFormatter(timestamp, request):
+    """
+    @return: A combined log formatted log line for the given request.
+
+    @see: L{IAccessLogFormatter}
+    """
+    clientAddr = request.getClientAddress()
+    if isinstance(
+        clientAddr, (address.IPv4Address, address.IPv6Address, _XForwardedForAddress)
+    ):
+        ip = clientAddr.host
+    else:
+        ip = b"-"
+    referrer = _escape(request.getHeader(b"referer") or b"-")
+    agent = _escape(request.getHeader(b"user-agent") or b"-")
+    line = (
+        '"%(ip)s" - - %(timestamp)s "%(method)s %(uri)s %(protocol)s" '
+        '%(code)d %(length)s "%(referrer)s" "%(agent)s"'
+        % dict(
+            ip=_escape(ip),
+            timestamp=timestamp,
+            method=_escape(request.method),
+            uri=_escape(request.uri),
+            protocol=_escape(request.clientproto),
+            code=request.code,
+            length=request.sentLength or "-",
+            referrer=referrer,
+            agent=agent,
+        )
+    )
+    return line
+
+
+@implementer(interfaces.IAddress)
+class _XForwardedForAddress:
+    """
+    L{IAddress} which represents the client IP to log for a request, as gleaned
+    from an X-Forwarded-For header.
+
+    @ivar host: An IP address or C{b"-"}.
+    @type host: L{bytes}
+
+    @see: L{proxiedLogFormatter}
+    """
+
+    def __init__(self, host):
+        self.host = host
+
+
+class _XForwardedForRequest(proxyForInterface(IRequest, "_request")):  # type: ignore[misc]
+    """
+    Add a layer on top of another request that only uses the value of an
+    X-Forwarded-For header as the result of C{getClientAddress}.
+    """
+
+    def getClientAddress(self):
+        """
+        The client address (the first address) in the value of the
+        I{X-Forwarded-For header}.  If the header is not present, the IP is
+        considered to be C{b"-"}.
+
+        @return: L{_XForwardedForAddress} which wraps the client address as
+            expected by L{combinedLogFormatter}.
+        """
+        host = (
+            self._request.requestHeaders.getRawHeaders(b"x-forwarded-for", [b"-"])[0]
+            .split(b",")[0]
+            .strip()
+        )
+        return _XForwardedForAddress(host)
+
+    # These are missing from the interface.  Forward them manually.
+    @property
+    def clientproto(self):
+        """
+        @return: The protocol version in the request.
+        @rtype: L{bytes}
+        """
+        return self._request.clientproto
+
+    @property
+    def code(self):
+        """
+        @return: The response code for the request.
+        @rtype: L{int}
+        """
+        return self._request.code
+
+    @property
+    def sentLength(self):
+        """
+        @return: The number of bytes sent in the response body.
+        @rtype: L{int}
+        """
+        return self._request.sentLength
+
+
+@provider(IAccessLogFormatter)
+def proxiedLogFormatter(timestamp, request):
+    """
+    @return: A combined log formatted log line for the given request but use
+        the value of the I{X-Forwarded-For} header as the value for the client
+        IP address.
+
+    @see: L{IAccessLogFormatter}
+    """
+    return combinedLogFormatter(timestamp, _XForwardedForRequest(request))
+
+
+class _GenericHTTPChannelProtocol(proxyForInterface(IProtocol, "_channel")):  # type: ignore[misc]
+    """
+    A proxy object that wraps one of the HTTP protocol objects, and switches
+    between them depending on TLS negotiated protocol.
+
+    @ivar _negotiatedProtocol: The protocol negotiated with ALPN or NPN, if
+        any.
+    @type _negotiatedProtocol: Either a bytestring containing the ALPN token
+        for the negotiated protocol, or L{None} if no protocol has yet been
+        negotiated.
+
+    @ivar _channel: The object capable of behaving like a L{HTTPChannel} that
+        is backing this object. By default this is a L{HTTPChannel}, but if a
+        HTTP protocol upgrade takes place this may be a different channel
+        object. Must implement L{IProtocol}.
+    @type _channel: L{HTTPChannel}
+
+    @ivar _requestFactory: A callable to use to build L{IRequest} objects.
+    @type _requestFactory: L{IRequest}
+
+    @ivar _site: A reference to the creating L{twisted.web.server.Site} object.
+    @type _site: L{twisted.web.server.Site}
+
+    @ivar _factory: A reference to the creating L{HTTPFactory} object.
+    @type _factory: L{HTTPFactory}
+
+    @ivar _timeOut: A timeout value to pass to the backing channel.
+    @type _timeOut: L{int} or L{None}
+
+    @ivar _callLater: A value for the C{callLater} callback.
+    @type _callLater: L{callable}
+    """
+
+    _negotiatedProtocol = None
+    _requestFactory = Request
+    _factory = None
+    _site = None
+    _timeOut = None
+    _callLater = None
+
+    @property
+    def factory(self):
+        """
+        @see: L{_genericHTTPChannelProtocolFactory}
+        """
+        return self._channel.factory
+
+    @factory.setter
+    def factory(self, value):
+        self._factory = value
+        self._channel.factory = value
+
+    @property
+    def requestFactory(self):
+        """
+        A callable to use to build L{IRequest} objects.
+
+        Retries the object from the current backing channel.
+        """
+        return self._channel.requestFactory
+
+    @requestFactory.setter
+    def requestFactory(self, value):
+        """
+        A callable to use to build L{IRequest} objects.
+
+        Sets the object on the backing channel and also stores the value for
+        propagation to any new channel.
+
+        @param value: The new callable to use.
+        @type value: A L{callable} returning L{IRequest}
+        """
+        self._requestFactory = value
+        self._channel.requestFactory = value
+
+    @property
+    def site(self):
+        """
+        A reference to the creating L{twisted.web.server.Site} object.
+
+        Returns the site object from the backing channel.
+        """
+        return self._channel.site
+
+    @site.setter
+    def site(self, value):
+        """
+        A reference to the creating L{twisted.web.server.Site} object.
+
+        Sets the object on the backing channel and also stores the value for
+        propagation to any new channel.
+
+        @param value: The L{twisted.web.server.Site} object to set.
+        @type value: L{twisted.web.server.Site}
+        """
+        self._site = value
+        self._channel.site = value
+
+    @property
+    def timeOut(self):
+        """
+        The idle timeout for the backing channel.
+        """
+        return self._channel.timeOut
+
+    @timeOut.setter
+    def timeOut(self, value):
+        """
+        The idle timeout for the backing channel.
+
+        Sets the idle timeout on both the backing channel and stores it for
+        propagation to any new backing channel.
+
+        @param value: The timeout to set.
+        @type value: L{int} or L{float}
+        """
+        self._timeOut = value
+        self._channel.timeOut = value
+
+    @property
+    def callLater(self):
+        """
+        A value for the C{callLater} callback. This callback is used by the
+        L{twisted.protocols.policies.TimeoutMixin} to handle timeouts.
+        """
+        return self._channel.callLater
+
+    @callLater.setter
+    def callLater(self, value):
+        """
+        Sets the value for the C{callLater} callback. This callback is used by
+        the L{twisted.protocols.policies.TimeoutMixin} to handle timeouts.
+
+        @param value: The new callback to use.
+        @type value: L{callable}
+        """
+        self._callLater = value
+        self._channel.callLater = value
+
+    def dataReceived(self, data):
+        """
+        An override of L{IProtocol.dataReceived} that checks what protocol we're
+        using.
+        """
+        if self._negotiatedProtocol is None:
+            try:
+                negotiatedProtocol = self._channel.transport.negotiatedProtocol
+            except AttributeError:
+                # Plaintext HTTP, always HTTP/1.1
+                negotiatedProtocol = b"http/1.1"
+
+            if negotiatedProtocol is None:
+                negotiatedProtocol = b"http/1.1"
+
+            if negotiatedProtocol == b"h2":
+                if not H2_ENABLED:
+                    raise ValueError("Negotiated HTTP/2 without support.")
+
+                # We need to make sure that the HTTPChannel is unregistered
+                # from the transport so that the H2Connection can register
+                # itself if possible.
+                networkProducer = self._channel._networkProducer
+                networkProducer.unregisterProducer()
+
+                # Cancel the old channel's timeout.
+                self._channel.setTimeout(None)
+
+                transport = self._channel.transport
+                self._channel = H2Connection()
+                self._channel.requestFactory = self._requestFactory
+                self._channel.site = self._site
+                self._channel.factory = self._factory
+                self._channel.timeOut = self._timeOut
+                self._channel.callLater = self._callLater
+                self._channel.makeConnection(transport)
+
+                # Register the H2Connection as the transport's
+                # producer, so that the transport can apply back
+                # pressure.
+                networkProducer.registerProducer(self._channel, True)
+            else:
+                # Only HTTP/2 and HTTP/1.1 are supported right now.
+                assert (
+                    negotiatedProtocol == b"http/1.1"
+                ), "Unsupported protocol negotiated"
+
+            self._negotiatedProtocol = negotiatedProtocol
+
+        return self._channel.dataReceived(data)
+
+
+def _genericHTTPChannelProtocolFactory(self):
+    """
+    Returns an appropriately initialized _GenericHTTPChannelProtocol.
+    """
+    return _GenericHTTPChannelProtocol(HTTPChannel())
+
+
+class HTTPFactory(protocol.ServerFactory):
+    """
+    Factory for HTTP server.
+
+    @ivar _logDateTime: A cached datetime string for log messages, updated by
+        C{_logDateTimeCall}.
+    @type _logDateTime: C{str}
+
+    @ivar _logDateTimeCall: A delayed call for the next update to the cached
+        log datetime string.
+    @type _logDateTimeCall: L{IDelayedCall} provided
+
+    @ivar _logFormatter: See the C{logFormatter} parameter to L{__init__}
+
+    @ivar _nativeize: A flag that indicates whether the log file being written
+        to wants native strings (C{True}) or bytes (C{False}).  This is only to
+        support writing to L{twisted.python.log} which, unfortunately, works
+        with native strings.
+
+    @ivar reactor: An L{IReactorTime} provider used to manage connection
+        timeouts and compute logging timestamps.
+    """
+
+    # We need to ignore the mypy error here, because
+    # _genericHTTPChannelProtocolFactory is a callable which returns a proxy
+    # to a Protocol, instead of a concrete Protocol object, as expected in
+    # the protocol.Factory interface
+    protocol = _genericHTTPChannelProtocolFactory  # type: ignore[assignment]
+
+    logPath = None
+
+    timeOut = _REQUEST_TIMEOUT
+
+    def __init__(
+        self, logPath=None, timeout=_REQUEST_TIMEOUT, logFormatter=None, reactor=None
+    ):
+        """
+        @param logPath: File path to which access log messages will be written
+            or C{None} to disable logging.
+        @type logPath: L{str} or L{bytes}
+
+        @param timeout: The initial value of L{timeOut}, which defines the idle
+            connection timeout in seconds, or C{None} to disable the idle
+            timeout.
+        @type timeout: L{float}
+
+        @param logFormatter: An object to format requests into log lines for
+            the access log.  L{combinedLogFormatter} when C{None} is passed.
+        @type logFormatter: L{IAccessLogFormatter} provider
+
+        @param reactor: An L{IReactorTime} provider used to manage connection
+            timeouts and compute logging timestamps. Defaults to the global
+            reactor.
+        """
+        if not reactor:
+            from twisted.internet import reactor
+        self.reactor = reactor
+
+        if logPath is not None:
+            logPath = os.path.abspath(logPath)
+        self.logPath = logPath
+        self.timeOut = timeout
+        if logFormatter is None:
+            logFormatter = combinedLogFormatter
+        self._logFormatter = logFormatter
+
+        # For storing the cached log datetime and the callback to update it
+        self._logDateTime = None
+        self._logDateTimeCall = None
+
+    def _updateLogDateTime(self):
+        """
+        Update log datetime periodically, so we aren't always recalculating it.
+        """
+        self._logDateTime = datetimeToLogString(self.reactor.seconds())
+        self._logDateTimeCall = self.reactor.callLater(1, self._updateLogDateTime)
+
+    def buildProtocol(self, addr):
+        p = protocol.ServerFactory.buildProtocol(self, addr)
+
+        # This is a bit of a hack to ensure that the HTTPChannel timeouts
+        # occur on the same reactor as the one we're using here. This could
+        # ideally be resolved by passing the reactor more generally to the
+        # HTTPChannel, but that won't work for the TimeoutMixin until we fix
+        # https://twistedmatrix.com/trac/ticket/8488
+        p.callLater = self.reactor.callLater
+
+        # timeOut needs to be on the Protocol instance cause
+        # TimeoutMixin expects it there
+        p.timeOut = self.timeOut
+        return p
+
+    def startFactory(self):
+        """
+        Set up request logging if necessary.
+        """
+        if self._logDateTimeCall is None:
+            self._updateLogDateTime()
+
+        if self.logPath:
+            self.logFile = self._openLogFile(self.logPath)
+        else:
+            self.logFile = log.logfile
+
+    def stopFactory(self):
+        if hasattr(self, "logFile"):
+            if self.logFile != log.logfile:
+                self.logFile.close()
+            del self.logFile
+
+        if self._logDateTimeCall is not None and self._logDateTimeCall.active():
+            self._logDateTimeCall.cancel()
+            self._logDateTimeCall = None
+
+    def _openLogFile(self, path):
+        """
+        Override in subclasses, e.g. to use L{twisted.python.logfile}.
+        """
+        f = open(path, "ab", 1)
+        return f
+
+    def log(self, request):
+        """
+        Write a line representing C{request} to the access log file.
+
+        @param request: The request object about which to log.
+        @type request: L{Request}
+        """
+        try:
+            logFile = self.logFile
+        except AttributeError:
+            pass
+        else:
+            line = self._logFormatter(self._logDateTime, request) + "\n"
+            logFile.write(line.encode("utf8"))
diff --git a/contrib/python/Twisted/py3/twisted/web/http_headers.py b/contrib/python/Twisted/py3/twisted/web/http_headers.py
new file mode 100644
index 00000000000..f810f4bc2c4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/http_headers.py
@@ -0,0 +1,295 @@
+# -*- test-case-name: twisted.web.test.test_http_headers -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An API for storing HTTP header names and values.
+"""
+
+from collections.abc import Sequence as _Sequence
+from typing import (
+    AnyStr,
+    Dict,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    TypeVar,
+    Union,
+    overload,
+)
+
+from twisted.python.compat import cmp, comparable
+
+_T = TypeVar("_T")
+
+
+def _dashCapitalize(name: bytes) -> bytes:
+    """
+    Return a byte string which is capitalized using '-' as a word separator.
+
+    @param name: The name of the header to capitalize.
+
+    @return: The given header capitalized using '-' as a word separator.
+    """
+    return b"-".join([word.capitalize() for word in name.split(b"-")])
+
+
+def _sanitizeLinearWhitespace(headerComponent: bytes) -> bytes:
+    r"""
+    Replace linear whitespace (C{\n}, C{\r\n}, C{\r}) in a header key
+    or value with a single space.
+
+    @param headerComponent: The header key or value to sanitize.
+
+    @return: The sanitized header key or value.
+    """
+    return b" ".join(headerComponent.splitlines())
+
+
+@comparable
+class Headers:
+    """
+    Stores HTTP headers in a key and multiple value format.
+
+    When passed L{str}, header names (e.g. 'Content-Type')
+    are encoded using ISO-8859-1 and header values (e.g.
+    'text/html;charset=utf-8') are encoded using UTF-8. Some methods that return
+    values will return them in the same type as the name given.
+
+    If the header keys or values cannot be encoded or decoded using the rules
+    above, using just L{bytes} arguments to the methods of this class will
+    ensure no decoding or encoding is done, and L{Headers} will treat the keys
+    and values as opaque byte strings.
+
+    @cvar _caseMappings: A L{dict} that maps lowercase header names
+        to their canonicalized representation.
+
+    @ivar _rawHeaders: A L{dict} mapping header names as L{bytes} to L{list}s of
+        header values as L{bytes}.
+    """
+
+    _caseMappings = {
+        b"content-md5": b"Content-MD5",
+        b"dnt": b"DNT",
+        b"etag": b"ETag",
+        b"p3p": b"P3P",
+        b"te": b"TE",
+        b"www-authenticate": b"WWW-Authenticate",
+        b"x-xss-protection": b"X-XSS-Protection",
+    }
+
+    def __init__(
+        self,
+        rawHeaders: Optional[Mapping[AnyStr, Sequence[AnyStr]]] = None,
+    ) -> None:
+        self._rawHeaders: Dict[bytes, List[bytes]] = {}
+        if rawHeaders is not None:
+            for name, values in rawHeaders.items():
+                self.setRawHeaders(name, values)
+
+    def __repr__(self) -> str:
+        """
+        Return a string fully describing the headers set on this object.
+        """
+        return "{}({!r})".format(
+            self.__class__.__name__,
+            self._rawHeaders,
+        )
+
+    def __cmp__(self, other):
+        """
+        Define L{Headers} instances as being equal to each other if they have
+        the same raw headers.
+        """
+        if isinstance(other, Headers):
+            return cmp(
+                sorted(self._rawHeaders.items()), sorted(other._rawHeaders.items())
+            )
+        return NotImplemented
+
+    def _encodeName(self, name: Union[str, bytes]) -> bytes:
+        """
+        Encode the name of a header (eg 'Content-Type') to an ISO-8859-1 encoded
+        bytestring if required.
+
+        @param name: A HTTP header name
+
+        @return: C{name}, encoded if required, lowercased
+        """
+        if isinstance(name, str):
+            return name.lower().encode("iso-8859-1")
+        return name.lower()
+
+    def copy(self):
+        """
+        Return a copy of itself with the same headers set.
+
+        @return: A new L{Headers}
+        """
+        return self.__class__(self._rawHeaders)
+
+    def hasHeader(self, name: AnyStr) -> bool:
+        """
+        Check for the existence of a given header.
+
+        @param name: The name of the HTTP header to check for.
+
+        @return: C{True} if the header exists, otherwise C{False}.
+        """
+        return self._encodeName(name) in self._rawHeaders
+
+    def removeHeader(self, name: AnyStr) -> None:
+        """
+        Remove the named header from this header object.
+
+        @param name: The name of the HTTP header to remove.
+
+        @return: L{None}
+        """
+        self._rawHeaders.pop(self._encodeName(name), None)
+
+    @overload
+    def setRawHeaders(self, name: Union[str, bytes], values: Sequence[bytes]) -> None:
+        ...
+
+    @overload
+    def setRawHeaders(self, name: Union[str, bytes], values: Sequence[str]) -> None:
+        ...
+
+    @overload
+    def setRawHeaders(
+        self, name: Union[str, bytes], values: Sequence[Union[str, bytes]]
+    ) -> None:
+        ...
+
+    def setRawHeaders(self, name: Union[str, bytes], values: object) -> None:
+        """
+        Sets the raw representation of the given header.
+
+        @param name: The name of the HTTP header to set the values for.
+
+        @param values: A list of strings each one being a header value of
+            the given name.
+
+        @raise TypeError: Raised if C{values} is not a sequence of L{bytes}
+            or L{str}, or if C{name} is not L{bytes} or L{str}.
+
+        @return: L{None}
+        """
+        if not isinstance(values, _Sequence):
+            raise TypeError(
+                "Header entry %r should be sequence but found "
+                "instance of %r instead" % (name, type(values))
+            )
+
+        if not isinstance(name, (bytes, str)):
+            raise TypeError(
+                f"Header name is an instance of {type(name)!r}, not bytes or str"
+            )
+
+        for count, value in enumerate(values):
+            if not isinstance(value, (bytes, str)):
+                raise TypeError(
+                    "Header value at position %s is an instance of %r, not "
+                    "bytes or str"
+                    % (
+                        count,
+                        type(value),
+                    )
+                )
+
+        _name = _sanitizeLinearWhitespace(self._encodeName(name))
+        encodedValues: List[bytes] = []
+        for v in values:
+            if isinstance(v, str):
+                _v = v.encode("utf8")
+            else:
+                _v = v
+            encodedValues.append(_sanitizeLinearWhitespace(_v))
+
+        self._rawHeaders[_name] = encodedValues
+
+    def addRawHeader(self, name: Union[str, bytes], value: Union[str, bytes]) -> None:
+        """
+        Add a new raw value for the given header.
+
+        @param name: The name of the header for which to set the value.
+
+        @param value: The value to set for the named header.
+        """
+        if not isinstance(name, (bytes, str)):
+            raise TypeError(
+                f"Header name is an instance of {type(name)!r}, not bytes or str"
+            )
+
+        if not isinstance(value, (bytes, str)):
+            raise TypeError(
+                "Header value is an instance of %r, not "
+                "bytes or str" % (type(value),)
+            )
+
+        self._rawHeaders.setdefault(
+            _sanitizeLinearWhitespace(self._encodeName(name)), []
+        ).append(
+            _sanitizeLinearWhitespace(
+                value.encode("utf8") if isinstance(value, str) else value
+            )
+        )
+
+    @overload
+    def getRawHeaders(self, name: AnyStr) -> Optional[Sequence[AnyStr]]:
+        ...
+
+    @overload
+    def getRawHeaders(self, name: AnyStr, default: _T) -> Union[Sequence[AnyStr], _T]:
+        ...
+
+    def getRawHeaders(
+        self, name: AnyStr, default: Optional[_T] = None
+    ) -> Union[Sequence[AnyStr], Optional[_T]]:
+        """
+        Returns a sequence of headers matching the given name as the raw string
+        given.
+
+        @param name: The name of the HTTP header to get the values of.
+
+        @param default: The value to return if no header with the given C{name}
+            exists.
+
+        @return: If the named header is present, a sequence of its
+            values.  Otherwise, C{default}.
+        """
+        encodedName = self._encodeName(name)
+        values = self._rawHeaders.get(encodedName, [])
+        if not values:
+            return default
+
+        if isinstance(name, str):
+            return [v.decode("utf8") for v in values]
+        return values
+
+    def getAllRawHeaders(self) -> Iterator[Tuple[bytes, Sequence[bytes]]]:
+        """
+        Return an iterator of key, value pairs of all headers contained in this
+        object, as L{bytes}.  The keys are capitalized in canonical
+        capitalization.
+        """
+        for k, v in self._rawHeaders.items():
+            yield self._canonicalNameCaps(k), v
+
+    def _canonicalNameCaps(self, name: bytes) -> bytes:
+        """
+        Return the canonical name for the given header.
+
+        @param name: The all-lowercase header name to capitalize in its
+            canonical form.
+
+        @return: The canonical name of the header.
+        """
+        return self._caseMappings.get(name, _dashCapitalize(name))
+
+
+__all__ = ["Headers"]
diff --git a/contrib/python/Twisted/py3/twisted/web/iweb.py b/contrib/python/Twisted/py3/twisted/web/iweb.py
new file mode 100644
index 00000000000..1aeb152fd9f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/iweb.py
@@ -0,0 +1,830 @@
+# -*- test-case-name: twisted.web.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Interface definitions for L{twisted.web}.
+
+@var UNKNOWN_LENGTH: An opaque object which may be used as the value of
+    L{IBodyProducer.length} to indicate that the length of the entity
+    body is not known in advance.
+"""
+from typing import TYPE_CHECKING, Callable, List, Optional
+
+from zope.interface import Attribute, Interface
+
+from twisted.cred.credentials import IUsernameDigestHash
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IPushProducer
+from twisted.web.http_headers import Headers
+
+if TYPE_CHECKING:
+    from twisted.web.template import Flattenable, Tag
+
+
+class IRequest(Interface):
+    """
+    An HTTP request.
+
+    @since: 9.0
+    """
+
+    method = Attribute("A L{bytes} giving the HTTP method that was used.")
+    uri = Attribute(
+        "A L{bytes} giving the full encoded URI which was requested (including"
+        " query arguments)."
+    )
+    path = Attribute(
+        "A L{bytes} giving the encoded query path of the request URI (not "
+        "including query arguments)."
+    )
+    args = Attribute(
+        "A mapping of decoded query argument names as L{bytes} to "
+        "corresponding query argument values as L{list}s of L{bytes}.  "
+        "For example, for a URI with C{foo=bar&foo=baz&quux=spam} "
+        "for its query part, C{args} will be C{{b'foo': [b'bar', b'baz'], "
+        "b'quux': [b'spam']}}."
+    )
+
+    prepath = Attribute(
+        "The URL path segments which have been processed during resource "
+        "traversal, as a list of L{bytes}."
+    )
+
+    postpath = Attribute(
+        "The URL path segments which have not (yet) been processed "
+        "during resource traversal, as a list of L{bytes}."
+    )
+
+    requestHeaders = Attribute(
+        "A L{http_headers.Headers} instance giving all received HTTP request "
+        "headers."
+    )
+
+    content = Attribute(
+        "A file-like object giving the request body.  This may be a file on "
+        "disk, an L{io.BytesIO}, or some other type.  The implementation is "
+        "free to decide on a per-request basis."
+    )
+
+    responseHeaders = Attribute(
+        "A L{http_headers.Headers} instance holding all HTTP response "
+        "headers to be sent."
+    )
+
+    def getHeader(key):
+        """
+        Get an HTTP request header.
+
+        @type key: L{bytes} or L{str}
+        @param key: The name of the header to get the value of.
+
+        @rtype: L{bytes} or L{str} or L{None}
+        @return: The value of the specified header, or L{None} if that header
+            was not present in the request. The string type of the result
+            matches the type of C{key}.
+        """
+
+    def getCookie(key):
+        """
+        Get a cookie that was sent from the network.
+
+        @type key: L{bytes}
+        @param key: The name of the cookie to get.
+
+        @rtype: L{bytes} or L{None}
+        @returns: The value of the specified cookie, or L{None} if that cookie
+            was not present in the request.
+        """
+
+    def getAllHeaders():
+        """
+        Return dictionary mapping the names of all received headers to the last
+        value received for each.
+
+        Since this method does not return all header information,
+        C{requestHeaders.getAllRawHeaders()} may be preferred.
+        """
+
+    def getRequestHostname():
+        """
+        Get the hostname that the HTTP client passed in to the request.
+
+        This will either use the C{Host:} header (if it is available; which,
+        for a spec-compliant request, it will be) or the IP address of the host
+        we are listening on if the header is unavailable.
+
+        @note: This is the I{host portion} of the requested resource, which
+            means that:
+
+                1. it might be an IPv4 or IPv6 address, not just a DNS host
+                   name,
+
+                2. there's no guarantee it's even a I{valid} host name or IP
+                   address, since the C{Host:} header may be malformed,
+
+                3. it does not include the port number.
+
+        @returns: the requested hostname
+
+        @rtype: L{bytes}
+        """
+
+    def getHost():
+        """
+        Get my originally requesting transport's host.
+
+        @return: An L{IAddress<twisted.internet.interfaces.IAddress>}.
+        """
+
+    def getClientAddress():
+        """
+        Return the address of the client who submitted this request.
+
+        The address may not be a network address.  Callers must check
+        its type before using it.
+
+        @since: 18.4
+
+        @return: the client's address.
+        @rtype: an L{IAddress} provider.
+        """
+
+    def getClientIP():
+        """
+        Return the IP address of the client who submitted this request.
+
+        This method is B{deprecated}.  See L{getClientAddress} instead.
+
+        @returns: the client IP address or L{None} if the request was submitted
+            over a transport where IP addresses do not make sense.
+        @rtype: L{str} or L{None}
+        """
+
+    def getUser():
+        """
+        Return the HTTP user sent with this request, if any.
+
+        If no user was supplied, return the empty string.
+
+        @returns: the HTTP user, if any
+        @rtype: L{str}
+        """
+
+    def getPassword():
+        """
+        Return the HTTP password sent with this request, if any.
+
+        If no password was supplied, return the empty string.
+
+        @returns: the HTTP password, if any
+        @rtype: L{str}
+        """
+
+    def isSecure():
+        """
+        Return True if this request is using a secure transport.
+
+        Normally this method returns True if this request's HTTPChannel
+        instance is using a transport that implements ISSLTransport.
+
+        This will also return True if setHost() has been called
+        with ssl=True.
+
+        @returns: True if this request is secure
+        @rtype: C{bool}
+        """
+
+    def getSession(sessionInterface=None):
+        """
+        Look up the session associated with this request or create a new one if
+        there is not one.
+
+        @return: The L{Session} instance identified by the session cookie in
+            the request, or the C{sessionInterface} component of that session
+            if C{sessionInterface} is specified.
+        """
+
+    def URLPath():
+        """
+        @return: A L{URLPath<twisted.python.urlpath.URLPath>} instance
+            which identifies the URL for which this request is.
+        """
+
+    def prePathURL():
+        """
+        At any time during resource traversal or resource rendering,
+        returns an absolute URL to the most nested resource which has
+        yet been reached.
+
+        @see: {twisted.web.server.Request.prepath}
+
+        @return: An absolute URL.
+        @rtype: L{bytes}
+        """
+
+    def rememberRootURL():
+        """
+        Remember the currently-processed part of the URL for later
+        recalling.
+        """
+
+    def getRootURL():
+        """
+        Get a previously-remembered URL.
+
+        @return: An absolute URL.
+        @rtype: L{bytes}
+        """
+
+    # Methods for outgoing response
+    def finish():
+        """
+        Indicate that the response to this request is complete.
+        """
+
+    def write(data):
+        """
+        Write some data to the body of the response to this request.  Response
+        headers are written the first time this method is called, after which
+        new response headers may not be added.
+
+        @param data: Bytes of the response body.
+        @type data: L{bytes}
+        """
+
+    def addCookie(
+        k,
+        v,
+        expires=None,
+        domain=None,
+        path=None,
+        max_age=None,
+        comment=None,
+        secure=None,
+    ):
+        """
+        Set an outgoing HTTP cookie.
+
+        In general, you should consider using sessions instead of cookies, see
+        L{twisted.web.server.Request.getSession} and the
+        L{twisted.web.server.Session} class for details.
+        """
+
+    def setResponseCode(code, message=None):
+        """
+        Set the HTTP response code.
+
+        @type code: L{int}
+        @type message: L{bytes}
+        """
+
+    def setHeader(k, v):
+        """
+        Set an HTTP response header.  Overrides any previously set values for
+        this header.
+
+        @type k: L{bytes} or L{str}
+        @param k: The name of the header for which to set the value.
+
+        @type v: L{bytes} or L{str}
+        @param v: The value to set for the named header. A L{str} will be
+            UTF-8 encoded, which may not interoperable with other
+            implementations. Avoid passing non-ASCII characters if possible.
+        """
+
+    def redirect(url):
+        """
+        Utility function that does a redirect.
+
+        The request should have finish() called after this.
+        """
+
+    def setLastModified(when):
+        """
+        Set the C{Last-Modified} time for the response to this request.
+
+        If I am called more than once, I ignore attempts to set Last-Modified
+        earlier, only replacing the Last-Modified time if it is to a later
+        value.
+
+        If I am a conditional request, I may modify my response code to
+        L{NOT_MODIFIED<http.NOT_MODIFIED>} if appropriate for the time given.
+
+        @param when: The last time the resource being returned was modified, in
+            seconds since the epoch.
+        @type when: L{int} or L{float}
+
+        @return: If I am a C{If-Modified-Since} conditional request and the time
+            given is not newer than the condition, I return
+            L{CACHED<http.CACHED>} to indicate that you should write no body.
+            Otherwise, I return a false value.
+        """
+
+    def setETag(etag):
+        """
+        Set an C{entity tag} for the outgoing response.
+
+        That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for
+        comparing two or more entities from the same requested resource."
+
+        If I am a conditional request, I may modify my response code to
+        L{NOT_MODIFIED<http.NOT_MODIFIED>} or
+        L{PRECONDITION_FAILED<http.PRECONDITION_FAILED>}, if appropriate for the
+        tag given.
+
+        @param etag: The entity tag for the resource being returned.
+        @type etag: L{str}
+
+        @return: If I am a C{If-None-Match} conditional request and the tag
+            matches one in the request, I return L{CACHED<http.CACHED>} to
+            indicate that you should write no body.  Otherwise, I return a
+            false value.
+        """
+
+    def setHost(host, port, ssl=0):
+        """
+        Change the host and port the request thinks it's using.
+
+        This method is useful for working with reverse HTTP proxies (e.g.  both
+        Squid and Apache's mod_proxy can do this), when the address the HTTP
+        client is using is different than the one we're listening on.
+
+        For example, Apache may be listening on https://www.example.com, and
+        then forwarding requests to http://localhost:8080, but we don't want
+        HTML produced by Twisted to say 'http://localhost:8080', they should
+        say 'https://www.example.com', so we do::
+
+           request.setHost('www.example.com', 443, ssl=1)
+        """
+
+
+class INonQueuedRequestFactory(Interface):
+    """
+    A factory of L{IRequest} objects that does not take a ``queued`` parameter.
+    """
+
+    def __call__(channel):
+        """
+        Create an L{IRequest} that is operating on the given channel. There
+        must only be one L{IRequest} object processing at any given time on a
+        channel.
+
+        @param channel: A L{twisted.web.http.HTTPChannel} object.
+        @type channel: L{twisted.web.http.HTTPChannel}
+
+        @return: A request object.
+        @rtype: L{IRequest}
+        """
+
+
+class IAccessLogFormatter(Interface):
+    """
+    An object which can represent an HTTP request as a line of text for
+    inclusion in an access log file.
+    """
+
+    def __call__(timestamp, request):
+        """
+        Generate a line for the access log.
+
+        @param timestamp: The time at which the request was completed in the
+            standard format for access logs.
+        @type timestamp: L{unicode}
+
+        @param request: The request object about which to log.
+        @type request: L{twisted.web.server.Request}
+
+        @return: One line describing the request without a trailing newline.
+        @rtype: L{unicode}
+        """
+
+
+class ICredentialFactory(Interface):
+    """
+    A credential factory defines a way to generate a particular kind of
+    authentication challenge and a way to interpret the responses to these
+    challenges.  It creates
+    L{ICredentials<twisted.cred.credentials.ICredentials>} providers from
+    responses.  These objects will be used with L{twisted.cred} to authenticate
+    an authorize requests.
+    """
+
+    scheme = Attribute(
+        "A L{str} giving the name of the authentication scheme with which "
+        "this factory is associated.  For example, C{'basic'} or C{'digest'}."
+    )
+
+    def getChallenge(request):
+        """
+        Generate a new challenge to be sent to a client.
+
+        @type request: L{twisted.web.http.Request}
+        @param request: The request the response to which this challenge will
+            be included.
+
+        @rtype: L{dict}
+        @return: A mapping from L{str} challenge fields to associated L{str}
+            values.
+        """
+
+    def decode(response, request):
+        """
+        Create a credentials object from the given response.
+
+        @type response: L{str}
+        @param response: scheme specific response string
+
+        @type request: L{twisted.web.http.Request}
+        @param request: The request being processed (from which the response
+            was taken).
+
+        @raise twisted.cred.error.LoginFailed: If the response is invalid.
+
+        @rtype: L{twisted.cred.credentials.ICredentials} provider
+        @return: The credentials represented by the given response.
+        """
+
+
+class IBodyProducer(IPushProducer):
+    """
+    Objects which provide L{IBodyProducer} write bytes to an object which
+    provides L{IConsumer<twisted.internet.interfaces.IConsumer>} by calling its
+    C{write} method repeatedly.
+
+    L{IBodyProducer} providers may start producing as soon as they have an
+    L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.  That is, they
+    should not wait for a C{resumeProducing} call to begin writing data.
+
+    L{IConsumer.unregisterProducer<twisted.internet.interfaces.IConsumer.unregisterProducer>}
+    must not be called.  Instead, the
+    L{Deferred<twisted.internet.defer.Deferred>} returned from C{startProducing}
+    must be fired when all bytes have been written.
+
+    L{IConsumer.write<twisted.internet.interfaces.IConsumer.write>} may
+    synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or
+    C{stopProducing}.  These methods must be implemented with this in mind.
+
+    @since: 9.0
+    """
+
+    # Despite the restrictions above and the additional requirements of
+    # stopProducing documented below, this interface still needs to be an
+    # IPushProducer subclass.  Providers of it will be passed to IConsumer
+    # providers which only know about IPushProducer and IPullProducer, not
+    # about this interface.  This interface needs to remain close enough to one
+    # of those interfaces for consumers to work with it.
+
+    length = Attribute(
+        """
+        C{length} is a L{int} indicating how many bytes in total this
+        L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH}
+        if this is not known in advance.
+        """
+    )
+
+    def startProducing(consumer):
+        """
+        Start producing to the given
+        L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.
+
+        @return: A L{Deferred<twisted.internet.defer.Deferred>} which stops
+            production of data when L{Deferred.cancel} is called, and which
+            fires with L{None} when all bytes have been produced or with a
+            L{Failure<twisted.python.failure.Failure>} if there is any problem
+            before all bytes have been produced.
+        """
+
+    def stopProducing():
+        """
+        In addition to the standard behavior of
+        L{IProducer.stopProducing<twisted.internet.interfaces.IProducer.stopProducing>}
+        (stop producing data), make sure the
+        L{Deferred<twisted.internet.defer.Deferred>} returned by
+        C{startProducing} is never fired.
+        """
+
+
+class IRenderable(Interface):
+    """
+    An L{IRenderable} is an object that may be rendered by the
+    L{twisted.web.template} templating system.
+    """
+
+    def lookupRenderMethod(
+        name: str,
+    ) -> Callable[[Optional[IRequest], "Tag"], "Flattenable"]:
+        """
+        Look up and return the render method associated with the given name.
+
+        @param name: The value of a render directive encountered in the
+            document returned by a call to L{IRenderable.render}.
+
+        @return: A two-argument callable which will be invoked with the request
+            being responded to and the tag object on which the render directive
+            was encountered.
+        """
+
+    def render(request: Optional[IRequest]) -> "Flattenable":
+        """
+        Get the document for this L{IRenderable}.
+
+        @param request: The request in response to which this method is being
+            invoked.
+
+        @return: An object which can be flattened.
+        """
+
+
+class ITemplateLoader(Interface):
+    """
+    A loader for templates; something usable as a value for
+    L{twisted.web.template.Element}'s C{loader} attribute.
+    """
+
+    def load() -> List["Flattenable"]:
+        """
+        Load a template suitable for rendering.
+
+        @return: a L{list} of flattenable objects, such as byte and unicode
+            strings, L{twisted.web.template.Element}s and L{IRenderable} providers.
+        """
+
+
+class IResponse(Interface):
+    """
+    An object representing an HTTP response received from an HTTP server.
+
+    @since: 11.1
+    """
+
+    version = Attribute(
+        "A three-tuple describing the protocol and protocol version "
+        "of the response.  The first element is of type L{str}, the second "
+        "and third are of type L{int}.  For example, C{(b'HTTP', 1, 1)}."
+    )
+
+    code = Attribute("The HTTP status code of this response, as a L{int}.")
+
+    phrase = Attribute("The HTTP reason phrase of this response, as a L{str}.")
+
+    headers = Attribute("The HTTP response L{Headers} of this response.")
+
+    length = Attribute(
+        "The L{int} number of bytes expected to be in the body of this "
+        "response or L{UNKNOWN_LENGTH} if the server did not indicate how "
+        "many bytes to expect.  For I{HEAD} responses, this will be 0; if "
+        "the response includes a I{Content-Length} header, it will be "
+        "available in C{headers}."
+    )
+
+    request = Attribute("The L{IClientRequest} that resulted in this response.")
+
+    previousResponse = Attribute(
+        "The previous L{IResponse} from a redirect, or L{None} if there was no "
+        "previous response. This can be used to walk the response or request "
+        "history for redirections."
+    )
+
+    def deliverBody(protocol):
+        """
+        Register an L{IProtocol<twisted.internet.interfaces.IProtocol>} provider
+        to receive the response body.
+
+        The protocol will be connected to a transport which provides
+        L{IPushProducer}.  The protocol's C{connectionLost} method will be
+        called with:
+
+            - ResponseDone, which indicates that all bytes from the response
+              have been successfully delivered.
+
+            - PotentialDataLoss, which indicates that it cannot be determined
+              if the entire response body has been delivered.  This only occurs
+              when making requests to HTTP servers which do not set
+              I{Content-Length} or a I{Transfer-Encoding} in the response.
+
+            - ResponseFailed, which indicates that some bytes from the response
+              were lost.  The C{reasons} attribute of the exception may provide
+              more specific indications as to why.
+        """
+
+    def setPreviousResponse(response):
+        """
+        Set the reference to the previous L{IResponse}.
+
+        The value of the previous response can be read via
+        L{IResponse.previousResponse}.
+        """
+
+
+class _IRequestEncoder(Interface):
+    """
+    An object encoding data passed to L{IRequest.write}, for example for
+    compression purpose.
+
+    @since: 12.3
+    """
+
+    def encode(data):
+        """
+        Encode the data given and return the result.
+
+        @param data: The content to encode.
+        @type data: L{str}
+
+        @return: The encoded data.
+        @rtype: L{str}
+        """
+
+    def finish():
+        """
+        Callback called when the request is closing.
+
+        @return: If necessary, the pending data accumulated from previous
+            C{encode} calls.
+        @rtype: L{str}
+        """
+
+
+class _IRequestEncoderFactory(Interface):
+    """
+    A factory for returing L{_IRequestEncoder} instances.
+
+    @since: 12.3
+    """
+
+    def encoderForRequest(request):
+        """
+        If applicable, returns a L{_IRequestEncoder} instance which will encode
+        the request.
+        """
+
+
+class IClientRequest(Interface):
+    """
+    An object representing an HTTP request to make to an HTTP server.
+
+    @since: 13.1
+    """
+
+    method = Attribute(
+        "The HTTP method for this request, as L{bytes}. For example: "
+        "C{b'GET'}, C{b'HEAD'}, C{b'POST'}, etc."
+    )
+
+    absoluteURI = Attribute(
+        "The absolute URI of the requested resource, as L{bytes}; or L{None} "
+        "if the absolute URI cannot be determined."
+    )
+
+    headers = Attribute(
+        "Headers to be sent to the server, as "
+        "a L{twisted.web.http_headers.Headers} instance."
+    )
+
+
+class IAgent(Interface):
+    """
+    An agent makes HTTP requests.
+
+    The way in which requests are issued is left up to each implementation.
+    Some may issue them directly to the server indicated by the net location
+    portion of the request URL.  Others may use a proxy specified by system
+    configuration.
+
+    Processing of responses is also left very widely specified.  An
+    implementation may perform no special handling of responses, or it may
+    implement redirect following or content negotiation, it may implement a
+    cookie store or automatically respond to authentication challenges.  It may
+    implement many other unforeseen behaviors as well.
+
+    It is also intended that L{IAgent} implementations be composable.  An
+    implementation which provides cookie handling features should re-use an
+    implementation that provides connection pooling and this combination could
+    be used by an implementation which adds content negotiation functionality.
+    Some implementations will be completely self-contained, such as those which
+    actually perform the network operations to send and receive requests, but
+    most or all other implementations should implement a small number of new
+    features (perhaps one new feature) and delegate the rest of the
+    request/response machinery to another implementation.
+
+    This allows for great flexibility in the behavior an L{IAgent} will
+    provide.  For example, an L{IAgent} with web browser-like behavior could be
+    obtained by combining a number of (hypothetical) implementations::
+
+        baseAgent = Agent(reactor)
+        decode = ContentDecoderAgent(baseAgent, [(b"gzip", GzipDecoder())])
+        cookie = CookieAgent(decode, diskStore.cookie)
+        authenticate = AuthenticateAgent(
+            cookie, [diskStore.credentials, GtkAuthInterface()])
+        cache = CacheAgent(authenticate, diskStore.cache)
+        redirect = BrowserLikeRedirectAgent(cache, limit=10)
+
+        doSomeRequests(cache)
+    """
+
+    def request(
+        method: bytes,
+        uri: bytes,
+        headers: Optional[Headers] = None,
+        bodyProducer: Optional[IBodyProducer] = None,
+    ) -> Deferred[IResponse]:
+        """
+        Request the resource at the given location.
+
+        @param method: The request method to use, such as C{b"GET"}, C{b"HEAD"},
+            C{b"PUT"}, C{b"POST"}, etc.
+
+        @param uri: The location of the resource to request.  This should be an
+            absolute URI but some implementations may support relative URIs
+            (with absolute or relative paths).  I{HTTP} and I{HTTPS} are the
+            schemes most likely to be supported but others may be as well.
+
+        @param headers: The headers to send with the request (or L{None} to
+            send no extra headers).  An implementation may add its own headers
+            to this (for example for client identification or content
+            negotiation).
+
+        @param bodyProducer: An object which can generate bytes to make up the
+            body of this request (for example, the properly encoded contents of
+            a file for a file upload).  Or, L{None} if the request is to have
+            no body.
+
+        @return: A L{Deferred} that fires with an L{IResponse} provider when
+            the header of the response has been received (regardless of the
+            response status code) or with a L{Failure} if there is any problem
+            which prevents that response from being received (including
+            problems that prevent the request from being sent).
+        """
+
+
+class IPolicyForHTTPS(Interface):
+    """
+    An L{IPolicyForHTTPS} provides a policy for verifying the certificates of
+    HTTPS connections, in the form of a L{client connection creator
+    <twisted.internet.interfaces.IOpenSSLClientConnectionCreator>} per network
+    location.
+
+    @since: 14.0
+    """
+
+    def creatorForNetloc(hostname, port):
+        """
+        Create a L{client connection creator
+        <twisted.internet.interfaces.IOpenSSLClientConnectionCreator>}
+        appropriate for the given URL "netloc"; i.e. hostname and port number
+        pair.
+
+        @param hostname: The name of the requested remote host.
+        @type hostname: L{bytes}
+
+        @param port: The number of the requested remote port.
+        @type port: L{int}
+
+        @return: A client connection creator expressing the security
+            requirements for the given remote host.
+        @rtype: L{client connection creator
+            <twisted.internet.interfaces.IOpenSSLClientConnectionCreator>}
+        """
+
+
+class IAgentEndpointFactory(Interface):
+    """
+    An L{IAgentEndpointFactory} provides a way of constructing an endpoint
+    used for outgoing Agent requests. This is useful in the case of needing to
+    proxy outgoing connections, or to otherwise vary the transport used.
+
+    @since: 15.0
+    """
+
+    def endpointForURI(uri):
+        """
+        Construct and return an L{IStreamClientEndpoint} for the outgoing
+        request's connection.
+
+        @param uri: The URI of the request.
+        @type uri: L{twisted.web.client.URI}
+
+        @return: An endpoint which will have its C{connect} method called to
+            issue the request.
+        @rtype: an L{IStreamClientEndpoint} provider
+
+        @raises twisted.internet.error.SchemeNotSupported: If the given
+            URI's scheme cannot be handled by this factory.
+        """
+
+
+UNKNOWN_LENGTH = "twisted.web.iweb.UNKNOWN_LENGTH"
+
+__all__ = [
+    "IUsernameDigestHash",
+    "ICredentialFactory",
+    "IRequest",
+    "IBodyProducer",
+    "IRenderable",
+    "IResponse",
+    "_IRequestEncoder",
+    "_IRequestEncoderFactory",
+    "IClientRequest",
+    "UNKNOWN_LENGTH",
+]
diff --git a/contrib/python/Twisted/py3/twisted/web/microdom.py b/contrib/python/Twisted/py3/twisted/web/microdom.py
new file mode 100644
index 00000000000..b80db5394ee
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/microdom.py
@@ -0,0 +1,1217 @@
+# -*- test-case-name: twisted.web.test.test_xml -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Micro Document Object Model: a partial DOM implementation with SUX.
+
+This is an implementation of what we consider to be the useful subset of the
+DOM.  The chief advantage of this library is that, not being burdened with
+standards compliance, it can remain very stable between versions.  We can also
+implement utility 'pythonic' ways to access and mutate the XML tree.
+
+Since this has not subjected to a serious trial by fire, it is not recommended
+to use this outside of Twisted applications.  However, it seems to work just
+fine for the documentation generator, which parses a fairly representative
+sample of XML.
+
+Microdom mainly focuses on working with HTML and XHTML.
+
+This module is now deprecated.
+"""
+from __future__ import annotations
+
+# System Imports
+import re
+import warnings
+from io import BytesIO, StringIO
+
+from incremental import Version, getVersionString
+
+# Twisted Imports
+from twisted.python.compat import ioType
+from twisted.python.util import InsensitiveDict
+from twisted.web.sux import ParseError, XMLParser
+
+warningString = "twisted.web.microdom was deprecated at {}".format(
+    getVersionString(Version("Twisted", 23, 10, 0))
+)
+warnings.warn(warningString, DeprecationWarning, stacklevel=3)
+
+
+def getElementsByTagName(iNode, name):
+    """
+    Return a list of all child elements of C{iNode} with a name matching
+    C{name}.
+
+    Note that this implementation does not conform to the DOM Level 1 Core
+    specification because it may return C{iNode}.
+
+    @param iNode: An element at which to begin searching.  If C{iNode} has a
+        name matching C{name}, it will be included in the result.
+
+    @param name: A C{str} giving the name of the elements to return.
+
+    @return: A C{list} of direct or indirect child elements of C{iNode} with
+        the name C{name}.  This may include C{iNode}.
+    """
+    matches = []
+    matches_append = matches.append  # faster lookup. don't do this at home
+    slice = [iNode]
+    while len(slice) > 0:
+        c = slice.pop(0)
+        if c.nodeName == name:
+            matches_append(c)
+        slice[:0] = c.childNodes
+    return matches
+
+
+def getElementsByTagNameNoCase(iNode, name):
+    name = name.lower()
+    matches = []
+    matches_append = matches.append
+    slice = [iNode]
+    while len(slice) > 0:
+        c = slice.pop(0)
+        if c.nodeName.lower() == name:
+            matches_append(c)
+        slice[:0] = c.childNodes
+    return matches
+
+
+def _streamWriteWrapper(stream):
+    if ioType(stream) == bytes:
+
+        def w(s):
+            if isinstance(s, str):
+                s = s.encode("utf-8")
+            stream.write(s)
+
+    else:
+
+        def w(s):
+            if isinstance(s, bytes):
+                s = s.decode("utf-8")
+            stream.write(s)
+
+    return w
+
+
+# order is important
+HTML_ESCAPE_CHARS = (
+    ("&", "&amp;"),  # don't add any entities before this one
+    ("<", "&lt;"),
+    (">", "&gt;"),
+    ('"', "&quot;"),
+)
+REV_HTML_ESCAPE_CHARS = list(HTML_ESCAPE_CHARS)
+REV_HTML_ESCAPE_CHARS.reverse()
+
+XML_ESCAPE_CHARS = HTML_ESCAPE_CHARS + (("'", "&apos;"),)
+REV_XML_ESCAPE_CHARS = list(XML_ESCAPE_CHARS)
+REV_XML_ESCAPE_CHARS.reverse()
+
+
+def unescape(text, chars=REV_HTML_ESCAPE_CHARS):
+    """
+    Perform the exact opposite of 'escape'.
+    """
+    for s, h in chars:
+        text = text.replace(h, s)
+    return text
+
+
+def escape(text, chars=HTML_ESCAPE_CHARS):
+    """
+    Escape a few XML special chars with XML entities.
+    """
+    for s, h in chars:
+        text = text.replace(s, h)
+    return text
+
+
+class MismatchedTags(Exception):
+    def __init__(self, filename, expect, got, endLine, endCol, begLine, begCol):
+        (
+            self.filename,
+            self.expect,
+            self.got,
+            self.begLine,
+            self.begCol,
+            self.endLine,
+            self.endCol,
+        ) = (filename, expect, got, begLine, begCol, endLine, endCol)
+
+    def __str__(self) -> str:
+        return (
+            "expected </%s>, got </%s> line: %s col: %s, "
+            "began line: %s col: %s"
+            % (
+                self.expect,
+                self.got,
+                self.endLine,
+                self.endCol,
+                self.begLine,
+                self.begCol,
+            )
+        )
+
+
+class Node:
+    nodeName = "Node"
+
+    def __init__(self, parentNode=None):
+        self.parentNode = parentNode
+        self.childNodes = []
+
+    def isEqualToNode(self, other):
+        """
+        Compare this node to C{other}.  If the nodes have the same number of
+        children and corresponding children are equal to each other, return
+        C{True}, otherwise return C{False}.
+
+        @type other: L{Node}
+        @rtype: C{bool}
+        """
+        if len(self.childNodes) != len(other.childNodes):
+            return False
+        for a, b in zip(self.childNodes, other.childNodes):
+            if not a.isEqualToNode(b):
+                return False
+        return True
+
+    def writexml(
+        self,
+        stream,
+        indent="",
+        addindent="",
+        newl="",
+        strip=0,
+        nsprefixes={},
+        namespace="",
+    ):
+        raise NotImplementedError()
+
+    def toxml(
+        self, indent="", addindent="", newl="", strip=0, nsprefixes={}, namespace=""
+    ):
+        s = StringIO()
+        self.writexml(s, indent, addindent, newl, strip, nsprefixes, namespace)
+        rv = s.getvalue()
+        return rv
+
+    def writeprettyxml(self, stream, indent="", addindent=" ", newl="\n", strip=0):
+        return self.writexml(stream, indent, addindent, newl, strip)
+
+    def toprettyxml(self, indent="", addindent=" ", newl="\n", strip=0):
+        return self.toxml(indent, addindent, newl, strip)
+
+    def cloneNode(self, deep=0, parent=None):
+        raise NotImplementedError()
+
+    def hasChildNodes(self):
+        if self.childNodes:
+            return 1
+        else:
+            return 0
+
+    def appendChild(self, child):
+        """
+        Make the given L{Node} the last child of this node.
+
+        @param child: The L{Node} which will become a child of this node.
+
+        @raise TypeError: If C{child} is not a C{Node} instance.
+        """
+        if not isinstance(child, Node):
+            raise TypeError("expected Node instance")
+        self.childNodes.append(child)
+        child.parentNode = self
+
+    def insertBefore(self, new, ref):
+        """
+        Make the given L{Node} C{new} a child of this node which comes before
+        the L{Node} C{ref}.
+
+        @param new: A L{Node} which will become a child of this node.
+
+        @param ref: A L{Node} which is already a child of this node which
+            C{new} will be inserted before.
+
+        @raise TypeError: If C{new} or C{ref} is not a C{Node} instance.
+
+        @return: C{new}
+        """
+        if not isinstance(new, Node) or not isinstance(ref, Node):
+            raise TypeError("expected Node instance")
+        i = self.childNodes.index(ref)
+        new.parentNode = self
+        self.childNodes.insert(i, new)
+        return new
+
+    def removeChild(self, child):
+        """
+        Remove the given L{Node} from this node's children.
+
+        @param child: A L{Node} which is a child of this node which will no
+            longer be a child of this node after this method is called.
+
+        @raise TypeError: If C{child} is not a C{Node} instance.
+
+        @return: C{child}
+        """
+        if not isinstance(child, Node):
+            raise TypeError("expected Node instance")
+        if child in self.childNodes:
+            self.childNodes.remove(child)
+            child.parentNode = None
+        return child
+
+    def replaceChild(self, newChild, oldChild):
+        """
+        Replace a L{Node} which is already a child of this node with a
+        different node.
+
+        @param newChild: A L{Node} which will be made a child of this node.
+
+        @param oldChild: A L{Node} which is a child of this node which will
+            give up its position to C{newChild}.
+
+        @raise TypeError: If C{newChild} or C{oldChild} is not a C{Node}
+            instance.
+
+        @raise ValueError: If C{oldChild} is not a child of this C{Node}.
+        """
+        if not isinstance(newChild, Node) or not isinstance(oldChild, Node):
+            raise TypeError("expected Node instance")
+        if oldChild.parentNode is not self:
+            raise ValueError("oldChild is not a child of this node")
+        self.childNodes[self.childNodes.index(oldChild)] = newChild
+        oldChild.parentNode = None
+        newChild.parentNode = self
+
+    def lastChild(self):
+        return self.childNodes[-1]
+
+    def firstChild(self):
+        if len(self.childNodes):
+            return self.childNodes[0]
+        return None
+
+    # def get_ownerDocument(self):
+    #   """This doesn't really get the owner document; microdom nodes
+    #   don't even have one necessarily.  This gets the root node,
+    #   which is usually what you really meant.
+    #   *NOT DOM COMPLIANT.*
+    #   """
+    #   node=self
+    #   while (node.parentNode): node=node.parentNode
+    #   return node
+    # ownerDocument=node.get_ownerDocument()
+    # leaving commented for discussion; see also domhelpers.getParents(node)
+
+
+class Document(Node):
+    def __init__(self, documentElement=None):
+        Node.__init__(self)
+        if documentElement:
+            self.appendChild(documentElement)
+
+    def cloneNode(self, deep=0, parent=None):
+        d = Document()
+        d.doctype = self.doctype
+        if deep:
+            newEl = self.documentElement.cloneNode(1, self)
+        else:
+            newEl = self.documentElement
+        d.appendChild(newEl)
+        return d
+
+    doctype: None | str = None
+
+    def isEqualToDocument(self, n):
+        return (self.doctype == n.doctype) and Node.isEqualToNode(self, n)
+
+    isEqualToNode = isEqualToDocument
+
+    @property
+    def documentElement(self):
+        return self.childNodes[0]
+
+    def appendChild(self, child):
+        """
+        Make the given L{Node} the I{document element} of this L{Document}.
+
+        @param child: The L{Node} to make into this L{Document}'s document
+            element.
+
+        @raise ValueError: If this document already has a document element.
+        """
+        if self.childNodes:
+            raise ValueError("Only one element per document.")
+        Node.appendChild(self, child)
+
+    def writexml(
+        self,
+        stream,
+        indent="",
+        addindent="",
+        newl="",
+        strip=0,
+        nsprefixes={},
+        namespace="",
+    ):
+        w = _streamWriteWrapper(stream)
+
+        w('<?xml version="1.0"?>' + newl)
+        if self.doctype:
+            w(f"<!DOCTYPE {self.doctype}>{newl}")
+        self.documentElement.writexml(
+            stream, indent, addindent, newl, strip, nsprefixes, namespace
+        )
+
+    # of dubious utility (?)
+    def createElement(self, name, **kw):
+        return Element(name, **kw)
+
+    def createTextNode(self, text):
+        return Text(text)
+
+    def createComment(self, text):
+        return Comment(text)
+
+    def getElementsByTagName(self, name):
+        if self.documentElement.caseInsensitive:
+            return getElementsByTagNameNoCase(self, name)
+        return getElementsByTagName(self, name)
+
+    def getElementById(self, id):
+        childNodes = self.childNodes[:]
+        while childNodes:
+            node = childNodes.pop(0)
+            if node.childNodes:
+                childNodes.extend(node.childNodes)
+            if hasattr(node, "getAttribute") and node.getAttribute("id") == id:
+                return node
+
+
+class EntityReference(Node):
+    def __init__(self, eref, parentNode=None):
+        Node.__init__(self, parentNode)
+        self.eref = eref
+        self.nodeValue = self.data = "&" + eref + ";"
+
+    def isEqualToEntityReference(self, n):
+        if not isinstance(n, EntityReference):
+            return 0
+        return (self.eref == n.eref) and (self.nodeValue == n.nodeValue)
+
+    isEqualToNode = isEqualToEntityReference
+
+    def writexml(
+        self,
+        stream,
+        indent="",
+        addindent="",
+        newl="",
+        strip=0,
+        nsprefixes={},
+        namespace="",
+    ):
+        w = _streamWriteWrapper(stream)
+        w("" + self.nodeValue)
+
+    def cloneNode(self, deep=0, parent=None):
+        return EntityReference(self.eref, parent)
+
+
+class CharacterData(Node):
+    def __init__(self, data, parentNode=None):
+        Node.__init__(self, parentNode)
+        self.value = self.data = self.nodeValue = data
+
+    def isEqualToCharacterData(self, n):
+        return self.value == n.value
+
+    isEqualToNode = isEqualToCharacterData
+
+
+class Comment(CharacterData):
+    """
+    A comment node.
+    """
+
+    def writexml(
+        self,
+        stream,
+        indent="",
+        addindent="",
+        newl="",
+        strip=0,
+        nsprefixes={},
+        namespace="",
+    ):
+        w = _streamWriteWrapper(stream)
+        val = self.data
+        w(f"<!--{val}-->")
+
+    def cloneNode(self, deep=0, parent=None):
+        return Comment(self.nodeValue, parent)
+
+
+class Text(CharacterData):
+    def __init__(self, data, parentNode=None, raw=0):
+        CharacterData.__init__(self, data, parentNode)
+        self.raw = raw
+
+    def isEqualToNode(self, other):
+        """
+        Compare this text to C{text}.  If the underlying values and the C{raw}
+        flag are the same, return C{True}, otherwise return C{False}.
+        """
+        return CharacterData.isEqualToNode(self, other) and self.raw == other.raw
+
+    def cloneNode(self, deep=0, parent=None):
+        return Text(self.nodeValue, parent, self.raw)
+
+    def writexml(
+        self,
+        stream,
+        indent="",
+        addindent="",
+        newl="",
+        strip=0,
+        nsprefixes={},
+        namespace="",
+    ):
+        w = _streamWriteWrapper(stream)
+        if self.raw:
+            val = self.nodeValue
+            if not isinstance(val, str):
+                val = str(self.nodeValue)
+        else:
+            v = self.nodeValue
+            if not isinstance(v, str):
+                v = str(v)
+            if strip:
+                v = " ".join(v.split())
+            val = escape(v)
+        w(val)
+
+    def __repr__(self) -> str:
+        return "Text(%s" % repr(self.nodeValue) + ")"
+
+
+class CDATASection(CharacterData):
+    def cloneNode(self, deep=0, parent=None):
+        return CDATASection(self.nodeValue, parent)
+
+    def writexml(
+        self,
+        stream,
+        indent="",
+        addindent="",
+        newl="",
+        strip=0,
+        nsprefixes={},
+        namespace="",
+    ):
+        w = _streamWriteWrapper(stream)
+        w("<![CDATA[")
+        w("" + self.nodeValue)
+        w("]]>")
+
+
+def _genprefix():
+    i = 0
+    while True:
+        yield "p" + str(i)
+        i = i + 1
+
+
+genprefix = _genprefix()
+
+
+class _Attr(CharacterData):
+    "Support class for getAttributeNode."
+
+
+class Element(Node):
+    preserveCase = 0
+    caseInsensitive = 1
+    nsprefixes = None
+
+    def __init__(
+        self,
+        tagName,
+        attributes=None,
+        parentNode=None,
+        filename=None,
+        markpos=None,
+        caseInsensitive=1,
+        preserveCase=0,
+        namespace=None,
+    ):
+        Node.__init__(self, parentNode)
+        self.preserveCase = preserveCase or not caseInsensitive
+        self.caseInsensitive = caseInsensitive
+        if not preserveCase:
+            tagName = tagName.lower()
+        if attributes is None:
+            self.attributes = {}
+        else:
+            self.attributes = attributes
+            for k, v in self.attributes.items():
+                self.attributes[k] = unescape(v)
+
+        if caseInsensitive:
+            self.attributes = InsensitiveDict(self.attributes, preserve=preserveCase)
+
+        self.endTagName = self.nodeName = self.tagName = tagName
+        self._filename = filename
+        self._markpos = markpos
+        self.namespace = namespace
+
+    def addPrefixes(self, pfxs):
+        if self.nsprefixes is None:
+            self.nsprefixes = pfxs
+        else:
+            self.nsprefixes.update(pfxs)
+
+    def endTag(self, endTagName):
+        if not self.preserveCase:
+            endTagName = endTagName.lower()
+        self.endTagName = endTagName
+
+    def isEqualToElement(self, n):
+        if self.caseInsensitive:
+            return (self.attributes == n.attributes) and (
+                self.nodeName.lower() == n.nodeName.lower()
+            )
+        return (self.attributes == n.attributes) and (self.nodeName == n.nodeName)
+
+    def isEqualToNode(self, other):
+        """
+        Compare this element to C{other}.  If the C{nodeName}, C{namespace},
+        C{attributes}, and C{childNodes} are all the same, return C{True},
+        otherwise return C{False}.
+        """
+        return (
+            self.nodeName.lower() == other.nodeName.lower()
+            and self.namespace == other.namespace
+            and self.attributes == other.attributes
+            and Node.isEqualToNode(self, other)
+        )
+
+    def cloneNode(self, deep=0, parent=None):
+        clone = Element(
+            self.tagName,
+            parentNode=parent,
+            namespace=self.namespace,
+            preserveCase=self.preserveCase,
+            caseInsensitive=self.caseInsensitive,
+        )
+        clone.attributes.update(self.attributes)
+        if deep:
+            clone.childNodes = [child.cloneNode(1, clone) for child in self.childNodes]
+        else:
+            clone.childNodes = []
+        return clone
+
+    def getElementsByTagName(self, name):
+        if self.caseInsensitive:
+            return getElementsByTagNameNoCase(self, name)
+        return getElementsByTagName(self, name)
+
+    def hasAttributes(self):
+        return 1
+
+    def getAttribute(self, name, default=None):
+        return self.attributes.get(name, default)
+
+    def getAttributeNS(self, ns, name, default=None):
+        nsk = (ns, name)
+        if nsk in self.attributes:
+            return self.attributes[nsk]
+        if ns == self.namespace:
+            return self.attributes.get(name, default)
+        return default
+
+    def getAttributeNode(self, name):
+        return _Attr(self.getAttribute(name), self)
+
+    def setAttribute(self, name, attr):
+        self.attributes[name] = attr
+
+    def removeAttribute(self, name):
+        if name in self.attributes:
+            del self.attributes[name]
+
+    def hasAttribute(self, name):
+        return name in self.attributes
+
+    def writexml(
+        self,
+        stream,
+        indent="",
+        addindent="",
+        newl="",
+        strip=0,
+        nsprefixes={},
+        namespace="",
+    ):
+        """
+        Serialize this L{Element} to the given stream.
+
+        @param stream: A file-like object to which this L{Element} will be
+            written.
+
+        @param nsprefixes: A C{dict} mapping namespace URIs as C{str} to
+            prefixes as C{str}.  This defines the prefixes which are already in
+            scope in the document at the point at which this L{Element} exists.
+            This is essentially an implementation detail for namespace support.
+            Applications should not try to use it.
+
+        @param namespace: The namespace URI as a C{str} which is the default at
+            the point in the document at which this L{Element} exists.  This is
+            essentially an implementation detail for namespace support.
+            Applications should not try to use it.
+        """
+        # write beginning
+        ALLOWSINGLETON = (
+            "img",
+            "br",
+            "hr",
+            "base",
+            "meta",
+            "link",
+            "param",
+            "area",
+            "input",
+            "col",
+            "basefont",
+            "isindex",
+            "frame",
+        )
+        BLOCKELEMENTS = (
+            "html",
+            "head",
+            "body",
+            "noscript",
+            "ins",
+            "del",
+            "h1",
+            "h2",
+            "h3",
+            "h4",
+            "h5",
+            "h6",
+            "script",
+            "ul",
+            "ol",
+            "dl",
+            "pre",
+            "hr",
+            "blockquote",
+            "address",
+            "p",
+            "div",
+            "fieldset",
+            "table",
+            "tr",
+            "form",
+            "object",
+            "fieldset",
+            "applet",
+            "map",
+        )
+        FORMATNICELY = ("tr", "ul", "ol", "head")
+
+        # this should never be necessary unless people start
+        # changing .tagName on the fly(?)
+        if not self.preserveCase:
+            self.endTagName = self.tagName
+
+        w = _streamWriteWrapper(stream)
+        if self.nsprefixes:
+            newprefixes = self.nsprefixes.copy()
+            for ns in nsprefixes.keys():
+                if ns in newprefixes:
+                    del newprefixes[ns]
+        else:
+            newprefixes = {}
+
+        begin = ["<"]
+        if self.tagName in BLOCKELEMENTS:
+            begin = [newl, indent] + begin
+        bext = begin.extend
+        writeattr = lambda _atr, _val: bext((" ", _atr, '="', escape(_val), '"'))
+
+        # Make a local for tracking what end tag will be used.  If namespace
+        # prefixes are involved, this will be changed to account for that
+        # before it's actually used.
+        endTagName = self.endTagName
+
+        if namespace != self.namespace and self.namespace is not None:
+            # If the current default namespace is not the namespace of this tag
+            # (and this tag has a namespace at all) then we'll write out
+            # something related to namespaces.
+            if self.namespace in nsprefixes:
+                # This tag's namespace already has a prefix bound to it.  Use
+                # that prefix.
+                prefix = nsprefixes[self.namespace]
+                bext(prefix + ":" + self.tagName)
+                # Also make sure we use it for the end tag.
+                endTagName = prefix + ":" + self.endTagName
+            else:
+                # This tag's namespace has no prefix bound to it.  Change the
+                # default namespace to this tag's namespace so we don't need
+                # prefixes.  Alternatively, we could add a new prefix binding.
+                # I'm not sure why the code was written one way rather than the
+                # other. -exarkun
+                bext(self.tagName)
+                writeattr("xmlns", self.namespace)
+                # The default namespace just changed.  Make sure any children
+                # know about this.
+                namespace = self.namespace
+        else:
+            # This tag has no namespace or its namespace is already the default
+            # namespace.  Nothing extra to do here.
+            bext(self.tagName)
+
+        j = "".join
+        for attr, val in sorted(self.attributes.items()):
+            if isinstance(attr, tuple):
+                ns, key = attr
+                if ns in nsprefixes:
+                    prefix = nsprefixes[ns]
+                else:
+                    prefix = next(genprefix)
+                    newprefixes[ns] = prefix
+                assert val is not None
+                writeattr(prefix + ":" + key, val)
+            else:
+                assert val is not None
+                writeattr(attr, val)
+        if newprefixes:
+            for ns, prefix in newprefixes.items():
+                if prefix:
+                    writeattr("xmlns:" + prefix, ns)
+            newprefixes.update(nsprefixes)
+            downprefixes = newprefixes
+        else:
+            downprefixes = nsprefixes
+        w(j(begin))
+        if self.childNodes:
+            w(">")
+            newindent = indent + addindent
+            for child in self.childNodes:
+                if self.tagName in BLOCKELEMENTS and self.tagName in FORMATNICELY:
+                    w(j((newl, newindent)))
+                child.writexml(
+                    stream, newindent, addindent, newl, strip, downprefixes, namespace
+                )
+            if self.tagName in BLOCKELEMENTS:
+                w(j((newl, indent)))
+            w(j(("</", endTagName, ">")))
+        elif self.tagName.lower() not in ALLOWSINGLETON:
+            w(j(("></", endTagName, ">")))
+        else:
+            w(" />")
+
+    def __repr__(self) -> str:
+        rep = "Element(%s" % repr(self.nodeName)
+        if self.attributes:
+            rep += f", attributes={self.attributes!r}"
+        if self._filename:
+            rep += f", filename={self._filename!r}"
+        if self._markpos:
+            rep += f", markpos={self._markpos!r}"
+        return rep + ")"
+
+    def __str__(self) -> str:
+        rep = "<" + self.nodeName
+        if self._filename or self._markpos:
+            rep += " ("
+        if self._filename:
+            rep += repr(self._filename)
+        if self._markpos:
+            rep += " line %s column %s" % self._markpos
+        if self._filename or self._markpos:
+            rep += ")"
+        for item in self.attributes.items():
+            rep += " %s=%r" % item
+        if self.hasChildNodes():
+            rep += " >...</%s>" % self.nodeName
+        else:
+            rep += " />"
+        return rep
+
+
+def _unescapeDict(d):
+    dd = {}
+    for k, v in d.items():
+        dd[k] = unescape(v)
+    return dd
+
+
+def _reverseDict(d):
+    dd = {}
+    for k, v in d.items():
+        dd[v] = k
+    return dd
+
+
+class MicroDOMParser(XMLParser):
+    # <dash> glyph: a quick scan thru the DTD says BODY, AREA, LINK, IMG, HR,
+    # P, DT, DD, LI, INPUT, OPTION, THEAD, TFOOT, TBODY, COLGROUP, COL, TR, TH,
+    # TD, HEAD, BASE, META, HTML all have optional closing tags
+
+    soonClosers = "area link br img hr input base meta".split()
+    laterClosers = {
+        "p": ["p", "dt"],
+        "dt": ["dt", "dd"],
+        "dd": ["dt", "dd"],
+        "li": ["li"],
+        "tbody": ["thead", "tfoot", "tbody"],
+        "thead": ["thead", "tfoot", "tbody"],
+        "tfoot": ["thead", "tfoot", "tbody"],
+        "colgroup": ["colgroup"],
+        "col": ["col"],
+        "tr": ["tr"],
+        "td": ["td"],
+        "th": ["th"],
+        "head": ["body"],
+        "title": ["head", "body"],  # this looks wrong...
+        "option": ["option"],
+    }
+
+    def __init__(
+        self,
+        beExtremelyLenient=0,
+        caseInsensitive=1,
+        preserveCase=0,
+        soonClosers=soonClosers,
+        laterClosers=laterClosers,
+    ):
+        self.elementstack = []
+        d = {"xmlns": "xmlns", "": None}
+        dr = _reverseDict(d)
+        self.nsstack = [(d, None, dr)]
+        self.documents = []
+        self._mddoctype = None
+        self.beExtremelyLenient = beExtremelyLenient
+        self.caseInsensitive = caseInsensitive
+        self.preserveCase = preserveCase or not caseInsensitive
+        self.soonClosers = soonClosers
+        self.laterClosers = laterClosers
+        # self.indentlevel = 0
+
+    def shouldPreserveSpace(self):
+        for edx in range(len(self.elementstack)):
+            el = self.elementstack[-edx]
+            if el.tagName == "pre" or el.getAttribute("xml:space", "") == "preserve":
+                return 1
+        return 0
+
+    def _getparent(self):
+        if self.elementstack:
+            return self.elementstack[-1]
+        else:
+            return None
+
+    COMMENT = re.compile(r"\s*/[/*]\s*")
+
+    def _fixScriptElement(self, el):
+        # this deals with case where there is comment or CDATA inside
+        # <script> tag and we want to do the right thing with it
+        if not self.beExtremelyLenient or not len(el.childNodes) == 1:
+            return
+        c = el.firstChild()
+        if isinstance(c, Text):
+            # deal with nasty people who do stuff like:
+            #   <script> // <!--
+            #      x = 1;
+            #   // --></script>
+            # tidy does this, for example.
+            prefix = ""
+            oldvalue = c.value
+            match = self.COMMENT.match(oldvalue)
+            if match:
+                prefix = match.group()
+                oldvalue = oldvalue[len(prefix) :]
+
+            # now see if contents are actual node and comment or CDATA
+            try:
+                e = parseString("<a>%s</a>" % oldvalue).childNodes[0]
+            except (ParseError, MismatchedTags):
+                return
+            if len(e.childNodes) != 1:
+                return
+            e = e.firstChild()
+            if isinstance(e, (CDATASection, Comment)):
+                el.childNodes = []
+                if prefix:
+                    el.childNodes.append(Text(prefix))
+                el.childNodes.append(e)
+
+    def gotDoctype(self, doctype):
+        self._mddoctype = doctype
+
+    def gotTagStart(self, name, attributes):
+        # print ' '*self.indentlevel, 'start tag',name
+        # self.indentlevel += 1
+        parent = self._getparent()
+        if self.beExtremelyLenient and isinstance(parent, Element):
+            parentName = parent.tagName
+            myName = name
+            if self.caseInsensitive:
+                parentName = parentName.lower()
+                myName = myName.lower()
+            if myName in self.laterClosers.get(parentName, []):
+                self.gotTagEnd(parent.tagName)
+                parent = self._getparent()
+        attributes = _unescapeDict(attributes)
+        namespaces = self.nsstack[-1][0]
+        newspaces = {}
+        keysToDelete = []
+        for k, v in attributes.items():
+            if k.startswith("xmlns"):
+                spacenames = k.split(":", 1)
+                if len(spacenames) == 2:
+                    newspaces[spacenames[1]] = v
+                else:
+                    newspaces[""] = v
+                keysToDelete.append(k)
+        for k in keysToDelete:
+            del attributes[k]
+        if newspaces:
+            namespaces = namespaces.copy()
+            namespaces.update(newspaces)
+        keysToDelete = []
+        for k, v in attributes.items():
+            ksplit = k.split(":", 1)
+            if len(ksplit) == 2:
+                pfx, tv = ksplit
+                if pfx != "xml" and pfx in namespaces:
+                    attributes[namespaces[pfx], tv] = v
+                    keysToDelete.append(k)
+        for k in keysToDelete:
+            del attributes[k]
+        el = Element(
+            name,
+            attributes,
+            parent,
+            self.filename,
+            self.saveMark(),
+            caseInsensitive=self.caseInsensitive,
+            preserveCase=self.preserveCase,
+            namespace=namespaces.get(""),
+        )
+        revspaces = _reverseDict(newspaces)
+        el.addPrefixes(revspaces)
+
+        if newspaces:
+            rscopy = self.nsstack[-1][2].copy()
+            rscopy.update(revspaces)
+            self.nsstack.append((namespaces, el, rscopy))
+        self.elementstack.append(el)
+        if parent:
+            parent.appendChild(el)
+        if self.beExtremelyLenient and el.tagName in self.soonClosers:
+            self.gotTagEnd(name)
+
+    def _gotStandalone(self, factory, data):
+        parent = self._getparent()
+        te = factory(data, parent)
+        if parent:
+            parent.appendChild(te)
+        elif self.beExtremelyLenient:
+            self.documents.append(te)
+
+    def gotText(self, data):
+        if data.strip() or self.shouldPreserveSpace():
+            self._gotStandalone(Text, data)
+
+    def gotComment(self, data):
+        self._gotStandalone(Comment, data)
+
+    def gotEntityReference(self, entityRef):
+        self._gotStandalone(EntityReference, entityRef)
+
+    def gotCData(self, cdata):
+        self._gotStandalone(CDATASection, cdata)
+
+    def gotTagEnd(self, name):
+        # print ' '*self.indentlevel, 'end tag',name
+        # self.indentlevel -= 1
+        if not self.elementstack:
+            if self.beExtremelyLenient:
+                return
+            raise MismatchedTags(
+                *((self.filename, "NOTHING", name) + self.saveMark() + (0, 0))
+            )
+        el = self.elementstack.pop()
+        pfxdix = self.nsstack[-1][2]
+        if self.nsstack[-1][1] is el:
+            nstuple = self.nsstack.pop()
+        else:
+            nstuple = None
+        if self.caseInsensitive:
+            tn = el.tagName.lower()
+            cname = name.lower()
+        else:
+            tn = el.tagName
+            cname = name
+
+        nsplit = name.split(":", 1)
+        if len(nsplit) == 2:
+            pfx, newname = nsplit
+            ns = pfxdix.get(pfx, None)
+            if ns is not None:
+                if el.namespace != ns:
+                    if not self.beExtremelyLenient:
+                        raise MismatchedTags(
+                            *(
+                                (self.filename, el.tagName, name)
+                                + self.saveMark()
+                                + el._markpos
+                            )
+                        )
+        if not (tn == cname):
+            if self.beExtremelyLenient:
+                if self.elementstack:
+                    lastEl = self.elementstack[0]
+                    for idx in range(len(self.elementstack)):
+                        if self.elementstack[-(idx + 1)].tagName == cname:
+                            self.elementstack[-(idx + 1)].endTag(name)
+                            break
+                    else:
+                        # this was a garbage close tag; wait for a real one
+                        self.elementstack.append(el)
+                        if nstuple is not None:
+                            self.nsstack.append(nstuple)
+                        return
+                    del self.elementstack[-(idx + 1) :]
+                    if not self.elementstack:
+                        self.documents.append(lastEl)
+                        return
+            else:
+                raise MismatchedTags(
+                    *((self.filename, el.tagName, name) + self.saveMark() + el._markpos)
+                )
+        el.endTag(name)
+        if not self.elementstack:
+            self.documents.append(el)
+        if self.beExtremelyLenient and el.tagName == "script":
+            self._fixScriptElement(el)
+
+    def connectionLost(self, reason):
+        XMLParser.connectionLost(self, reason)  # This can cause more events!
+        if self.elementstack:
+            if self.beExtremelyLenient:
+                self.documents.append(self.elementstack[0])
+            else:
+                raise MismatchedTags(
+                    *(
+                        (self.filename, self.elementstack[-1], "END_OF_FILE")
+                        + self.saveMark()
+                        + self.elementstack[-1]._markpos
+                    )
+                )
+
+
+def parse(readable, *args, **kwargs):
+    """
+    Parse HTML or XML readable.
+    """
+    if not hasattr(readable, "read"):
+        readable = open(readable, "rb")
+    mdp = MicroDOMParser(*args, **kwargs)
+    mdp.filename = getattr(readable, "name", "<xmlfile />")
+    mdp.makeConnection(None)
+    if hasattr(readable, "getvalue"):
+        mdp.dataReceived(readable.getvalue())
+    else:
+        r = readable.read(1024)
+        while r:
+            mdp.dataReceived(r)
+            r = readable.read(1024)
+    mdp.connectionLost(None)
+
+    if not mdp.documents:
+        raise ParseError(mdp.filename, 0, 0, "No top-level Nodes in document")
+
+    if mdp.beExtremelyLenient:
+        if len(mdp.documents) == 1:
+            d = mdp.documents[0]
+            if not isinstance(d, Element):
+                el = Element("html")
+                el.appendChild(d)
+                d = el
+        else:
+            d = Element("html")
+            for child in mdp.documents:
+                d.appendChild(child)
+    else:
+        d = mdp.documents[0]
+    doc = Document(d)
+    doc.doctype = mdp._mddoctype
+    return doc
+
+
+def parseString(st, *args, **kw):
+    if isinstance(st, str):
+        # this isn't particularly ideal, but it does work.
+        return parse(BytesIO(st.encode("UTF-16")), *args, **kw)
+    return parse(BytesIO(st), *args, **kw)
+
+
+def parseXML(readable):
+    """
+    Parse an XML readable object.
+    """
+    return parse(readable, caseInsensitive=0, preserveCase=1)
+
+
+def parseXMLString(st):
+    """
+    Parse an XML readable object.
+    """
+    return parseString(st, caseInsensitive=0, preserveCase=1)
+
+
+class lmx:
+    """
+    Easy creation of XML.
+    """
+
+    def __init__(self, node="div"):
+        if isinstance(node, str):
+            node = Element(node)
+        self.node = node
+
+    def __getattr__(self, name):
+        if name[0] == "_":
+            raise AttributeError("no private attrs")
+        return lambda **kw: self.add(name, **kw)
+
+    def __setitem__(self, key, val):
+        self.node.setAttribute(key, val)
+
+    def __getitem__(self, key):
+        return self.node.getAttribute(key)
+
+    def text(self, txt, raw=0):
+        nn = Text(txt, raw=raw)
+        self.node.appendChild(nn)
+        return self
+
+    def add(self, tagName, **kw):
+        newNode = Element(tagName, caseInsensitive=0, preserveCase=0)
+        self.node.appendChild(newNode)
+        xf = lmx(newNode)
+        for k, v in kw.items():
+            if k[0] == "_":
+                k = k[1:]
+            xf[k] = v
+        return xf
diff --git a/contrib/python/Twisted/py3/twisted/web/newsfragments/.gitignore b/contrib/python/Twisted/py3/twisted/web/newsfragments/.gitignore
new file mode 100644
index 00000000000..f935021a8f8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/newsfragments/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/contrib/python/Twisted/py3/twisted/web/pages.py b/contrib/python/Twisted/py3/twisted/web/pages.py
new file mode 100644
index 00000000000..54ea1c431b8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/pages.py
@@ -0,0 +1,134 @@
+# -*- test-case-name: twisted.web.test.test_pages -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Utility implementations of L{IResource}.
+"""
+
+__all__ = (
+    "errorPage",
+    "notFound",
+    "forbidden",
+)
+
+from typing import cast
+
+from twisted.web import http
+from twisted.web.iweb import IRenderable, IRequest
+from twisted.web.resource import IResource, Resource
+from twisted.web.template import renderElement, tags
+
+
+class _ErrorPage(Resource):
+    """
+    L{_ErrorPage} is a resource that responds to all requests with a particular
+    (parameterized) HTTP status code and an HTML body containing some
+    descriptive text. This is useful for rendering simple error pages.
+
+    @see: L{twisted.web.pages.errorPage}
+
+    @ivar _code: An integer HTTP status code which will be used for the
+        response.
+
+    @ivar _brief: A short string which will be included in the response body as
+        the page title.
+
+    @ivar _detail: A longer string which will be included in the response body.
+    """
+
+    def __init__(self, code: int, brief: str, detail: str) -> None:
+        super().__init__()
+        self._code: int = code
+        self._brief: str = brief
+        self._detail: str = detail
+
+    def render(self, request: IRequest) -> object:
+        """
+        Respond to all requests with the given HTTP status code and an HTML
+        document containing the explanatory strings.
+        """
+        request.setResponseCode(self._code)
+        request.setHeader(b"content-type", b"text/html; charset=utf-8")
+        return renderElement(
+            request,
+            # cast because the type annotations here seem off; Tag isn't an
+            # IRenderable but also probably should be? See
+            # https://github.com/twisted/twisted/issues/4982
+            cast(
+                IRenderable,
+                tags.html(
+                    tags.head(tags.title(f"{self._code} - {self._brief}")),
+                    tags.body(tags.h1(self._brief), tags.p(self._detail)),
+                ),
+            ),
+        )
+
+    def getChild(self, path: bytes, request: IRequest) -> Resource:
+        """
+        Handle all requests for which L{_ErrorPage} lacks a child by returning
+        this error page.
+
+        @param path: A path segment.
+
+        @param request: HTTP request
+        """
+        return self
+
+
+def errorPage(code: int, brief: str, detail: str) -> IResource:
+    """
+    Build a resource that responds to all requests with a particular HTTP
+    status code and an HTML body containing some descriptive text. This is
+    useful for rendering simple error pages.
+
+    The resource dynamically handles all paths below it. Use
+    L{IResource.putChild()} to override a specific path.
+
+    @param code: An integer HTTP status code which will be used for the
+        response.
+
+    @param brief: A short string which will be included in the response
+        body as the page title.
+
+    @param detail: A longer string which will be included in the
+        response body.
+
+    @returns: An L{IResource}
+    """
+    return _ErrorPage(code, brief, detail)
+
+
+def notFound(
+    brief: str = "No Such Resource",
+    message: str = "Sorry. No luck finding that resource.",
+) -> IResource:
+    """
+    Generate an L{IResource} with a 404 Not Found status code.
+
+    @see: L{twisted.web.pages.errorPage}
+
+    @param brief: A short string displayed as the page title.
+
+    @param brief: A longer string displayed in the page body.
+
+    @returns: An L{IResource}
+    """
+    return _ErrorPage(http.NOT_FOUND, brief, message)
+
+
+def forbidden(
+    brief: str = "Forbidden Resource", message: str = "Sorry, resource is forbidden."
+) -> IResource:
+    """
+    Generate an L{IResource} with a 403 Forbidden status code.
+
+    @see: L{twisted.web.pages.errorPage}
+
+    @param brief: A short string displayed as the page title.
+
+    @param brief: A longer string displayed in the page body.
+
+    @returns: An L{IResource}
+    """
+    return _ErrorPage(http.FORBIDDEN, brief, message)
diff --git a/contrib/python/Twisted/py3/twisted/web/proxy.py b/contrib/python/Twisted/py3/twisted/web/proxy.py
new file mode 100644
index 00000000000..e31ec7a65d3
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/proxy.py
@@ -0,0 +1,296 @@
+# -*- test-case-name: twisted.web.test.test_proxy -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Simplistic HTTP proxy support.
+
+This comes in two main variants - the Proxy and the ReverseProxy.
+
+When a Proxy is in use, a browser trying to connect to a server (say,
+www.yahoo.com) will be intercepted by the Proxy, and the proxy will covertly
+connect to the server, and return the result.
+
+When a ReverseProxy is in use, the client connects directly to the ReverseProxy
+(say, www.yahoo.com) which farms off the request to one of a pool of servers,
+and returns the result.
+
+Normally, a Proxy is used on the client end of an Internet connection, while a
+ReverseProxy is used on the server end.
+"""
+
+from urllib.parse import quote as urlquote, urlparse, urlunparse
+
+from twisted.internet import reactor
+from twisted.internet.protocol import ClientFactory
+from twisted.web.http import _QUEUED_SENTINEL, HTTPChannel, HTTPClient, Request
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
+
+
+class ProxyClient(HTTPClient):
+    """
+    Used by ProxyClientFactory to implement a simple web proxy.
+
+    @ivar _finished: A flag which indicates whether or not the original request
+        has been finished yet.
+    """
+
+    _finished = False
+
+    def __init__(self, command, rest, version, headers, data, father):
+        self.father = father
+        self.command = command
+        self.rest = rest
+        if b"proxy-connection" in headers:
+            del headers[b"proxy-connection"]
+        headers[b"connection"] = b"close"
+        headers.pop(b"keep-alive", None)
+        self.headers = headers
+        self.data = data
+
+    def connectionMade(self):
+        self.sendCommand(self.command, self.rest)
+        for header, value in self.headers.items():
+            self.sendHeader(header, value)
+        self.endHeaders()
+        self.transport.write(self.data)
+
+    def handleStatus(self, version, code, message):
+        self.father.setResponseCode(int(code), message)
+
+    def handleHeader(self, key, value):
+        # t.web.server.Request sets default values for these headers in its
+        # 'process' method. When these headers are received from the remote
+        # server, they ought to override the defaults, rather than append to
+        # them.
+        if key.lower() in [b"server", b"date", b"content-type"]:
+            self.father.responseHeaders.setRawHeaders(key, [value])
+        else:
+            self.father.responseHeaders.addRawHeader(key, value)
+
+    def handleResponsePart(self, buffer):
+        self.father.write(buffer)
+
+    def handleResponseEnd(self):
+        """
+        Finish the original request, indicating that the response has been
+        completely written to it, and disconnect the outgoing transport.
+        """
+        if not self._finished:
+            self._finished = True
+            self.father.finish()
+            self.transport.loseConnection()
+
+
+class ProxyClientFactory(ClientFactory):
+    """
+    Used by ProxyRequest to implement a simple web proxy.
+    """
+
+    # Type is wrong.  See: https://twistedmatrix.com/trac/ticket/10006
+    protocol = ProxyClient  # type: ignore[assignment]
+
+    def __init__(self, command, rest, version, headers, data, father):
+        self.father = father
+        self.command = command
+        self.rest = rest
+        self.headers = headers
+        self.data = data
+        self.version = version
+
+    def buildProtocol(self, addr):
+        return self.protocol(
+            self.command, self.rest, self.version, self.headers, self.data, self.father
+        )
+
+    def clientConnectionFailed(self, connector, reason):
+        """
+        Report a connection failure in a response to the incoming request as
+        an error.
+        """
+        self.father.setResponseCode(501, b"Gateway error")
+        self.father.responseHeaders.addRawHeader(b"Content-Type", b"text/html")
+        self.father.write(b"<H1>Could not connect</H1>")
+        self.father.finish()
+
+
+class ProxyRequest(Request):
+    """
+    Used by Proxy to implement a simple web proxy.
+
+    @ivar reactor: the reactor used to create connections.
+    @type reactor: object providing L{twisted.internet.interfaces.IReactorTCP}
+    """
+
+    protocols = {b"http": ProxyClientFactory}
+    ports = {b"http": 80}
+
+    def __init__(self, channel, queued=_QUEUED_SENTINEL, reactor=reactor):
+        Request.__init__(self, channel, queued)
+        self.reactor = reactor
+
+    def process(self):
+        parsed = urlparse(self.uri)
+        protocol = parsed[0]
+        host = parsed[1].decode("ascii")
+        port = self.ports[protocol]
+        if ":" in host:
+            host, port = host.split(":")
+            port = int(port)
+        rest = urlunparse((b"", b"") + parsed[2:])
+        if not rest:
+            rest = rest + b"/"
+        class_ = self.protocols[protocol]
+        headers = self.getAllHeaders().copy()
+        if b"host" not in headers:
+            headers[b"host"] = host.encode("ascii")
+        self.content.seek(0, 0)
+        s = self.content.read()
+        clientFactory = class_(self.method, rest, self.clientproto, headers, s, self)
+        self.reactor.connectTCP(host, port, clientFactory)
+
+
+class Proxy(HTTPChannel):
+    """
+    This class implements a simple web proxy.
+
+    Since it inherits from L{twisted.web.http.HTTPChannel}, to use it you
+    should do something like this::
+
+        from twisted.web import http
+        f = http.HTTPFactory()
+        f.protocol = Proxy
+
+    Make the HTTPFactory a listener on a port as per usual, and you have
+    a fully-functioning web proxy!
+    """
+
+    requestFactory = ProxyRequest
+
+
+class ReverseProxyRequest(Request):
+    """
+    Used by ReverseProxy to implement a simple reverse proxy.
+
+    @ivar proxyClientFactoryClass: a proxy client factory class, used to create
+        new connections.
+    @type proxyClientFactoryClass: L{ClientFactory}
+
+    @ivar reactor: the reactor used to create connections.
+    @type reactor: object providing L{twisted.internet.interfaces.IReactorTCP}
+    """
+
+    proxyClientFactoryClass = ProxyClientFactory
+
+    def __init__(self, channel, queued=_QUEUED_SENTINEL, reactor=reactor):
+        Request.__init__(self, channel, queued)
+        self.reactor = reactor
+
+    def process(self):
+        """
+        Handle this request by connecting to the proxied server and forwarding
+        it there, then forwarding the response back as the response to this
+        request.
+        """
+        self.requestHeaders.setRawHeaders(b"host", [self.factory.host.encode("ascii")])
+        clientFactory = self.proxyClientFactoryClass(
+            self.method,
+            self.uri,
+            self.clientproto,
+            self.getAllHeaders(),
+            self.content.read(),
+            self,
+        )
+        self.reactor.connectTCP(self.factory.host, self.factory.port, clientFactory)
+
+
+class ReverseProxy(HTTPChannel):
+    """
+    Implements a simple reverse proxy.
+
+    For details of usage, see the file examples/reverse-proxy.py.
+    """
+
+    requestFactory = ReverseProxyRequest
+
+
+class ReverseProxyResource(Resource):
+    """
+    Resource that renders the results gotten from another server
+
+    Put this resource in the tree to cause everything below it to be relayed
+    to a different server.
+
+    @ivar proxyClientFactoryClass: a proxy client factory class, used to create
+        new connections.
+    @type proxyClientFactoryClass: L{ClientFactory}
+
+    @ivar reactor: the reactor used to create connections.
+    @type reactor: object providing L{twisted.internet.interfaces.IReactorTCP}
+    """
+
+    proxyClientFactoryClass = ProxyClientFactory
+
+    def __init__(self, host, port, path, reactor=reactor):
+        """
+        @param host: the host of the web server to proxy.
+        @type host: C{str}
+
+        @param port: the port of the web server to proxy.
+        @type port: C{port}
+
+        @param path: the base path to fetch data from. Note that you shouldn't
+            put any trailing slashes in it, it will be added automatically in
+            request. For example, if you put B{/foo}, a request on B{/bar} will
+            be proxied to B{/foo/bar}.  Any required encoding of special
+            characters (such as " " or "/") should have been done already.
+
+        @type path: C{bytes}
+        """
+        Resource.__init__(self)
+        self.host = host
+        self.port = port
+        self.path = path
+        self.reactor = reactor
+
+    def getChild(self, path, request):
+        """
+        Create and return a proxy resource with the same proxy configuration
+        as this one, except that its path also contains the segment given by
+        C{path} at the end.
+        """
+        return ReverseProxyResource(
+            self.host,
+            self.port,
+            self.path + b"/" + urlquote(path, safe=b"").encode("utf-8"),
+            self.reactor,
+        )
+
+    def render(self, request):
+        """
+        Render a request by forwarding it to the proxied server.
+        """
+        # RFC 2616 tells us that we can omit the port if it's the default port,
+        # but we have to provide it otherwise
+        if self.port == 80:
+            host = self.host
+        else:
+            host = "%s:%d" % (self.host, self.port)
+        request.requestHeaders.setRawHeaders(b"host", [host.encode("ascii")])
+        request.content.seek(0, 0)
+        qs = urlparse(request.uri)[4]
+        if qs:
+            rest = self.path + b"?" + qs
+        else:
+            rest = self.path
+        clientFactory = self.proxyClientFactoryClass(
+            request.method,
+            rest,
+            request.clientproto,
+            request.getAllHeaders(),
+            request.content.read(),
+            request,
+        )
+        self.reactor.connectTCP(self.host, self.port, clientFactory)
+        return NOT_DONE_YET
diff --git a/contrib/python/Twisted/py3/twisted/web/resource.py b/contrib/python/Twisted/py3/twisted/web/resource.py
new file mode 100644
index 00000000000..33b172cdbfd
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/resource.py
@@ -0,0 +1,458 @@
+# -*- test-case-name: twisted.web.test.test_web, twisted.web.test.test_resource -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Implementation of the lowest-level Resource class.
+
+See L{twisted.web.pages} for some utility implementations.
+"""
+
+
+__all__ = [
+    "IResource",
+    "getChildForRequest",
+    "Resource",
+    "ErrorPage",
+    "NoResource",
+    "ForbiddenResource",
+    "EncodingResourceWrapper",
+]
+
+import warnings
+
+from zope.interface import Attribute, Interface, implementer
+
+from incremental import Version
+
+from twisted.python.compat import nativeString
+from twisted.python.components import proxyForInterface
+from twisted.python.deprecate import deprecatedModuleAttribute
+from twisted.python.reflect import prefixedMethodNames
+from twisted.web._responses import FORBIDDEN, NOT_FOUND
+from twisted.web.error import UnsupportedMethod
+
+
+class IResource(Interface):
+    """
+    A web resource.
+    """
+
+    isLeaf = Attribute(
+        """
+        Signal if this IResource implementor is a "leaf node" or not. If True,
+        getChildWithDefault will not be called on this Resource.
+        """
+    )
+
+    def getChildWithDefault(name, request):
+        """
+        Return a child with the given name for the given request.
+        This is the external interface used by the Resource publishing
+        machinery. If implementing IResource without subclassing
+        Resource, it must be provided. However, if subclassing Resource,
+        getChild overridden instead.
+
+        @param name: A single path component from a requested URL.  For example,
+            a request for I{http://example.com/foo/bar} will result in calls to
+            this method with C{b"foo"} and C{b"bar"} as values for this
+            argument.
+        @type name: C{bytes}
+
+        @param request: A representation of all of the information about the
+            request that is being made for this child.
+        @type request: L{twisted.web.server.Request}
+        """
+
+    def putChild(path: bytes, child: "IResource") -> None:
+        """
+        Put a child L{IResource} implementor at the given path.
+
+        @param path: A single path component, to be interpreted relative to the
+            path this resource is found at, at which to put the given child.
+            For example, if resource A can be found at I{http://example.com/foo}
+            then a call like C{A.putChild(b"bar", B)} will make resource B
+            available at I{http://example.com/foo/bar}.
+
+            The path component is I{not} URL-encoded -- pass C{b'foo bar'}
+            rather than C{b'foo%20bar'}.
+        """
+
+    def render(request):
+        """
+        Render a request. This is called on the leaf resource for a request.
+
+        @return: Either C{server.NOT_DONE_YET} to indicate an asynchronous or a
+            C{bytes} instance to write as the response to the request.  If
+            C{NOT_DONE_YET} is returned, at some point later (for example, in a
+            Deferred callback) call C{request.write(b"<html>")} to write data to
+            the request, and C{request.finish()} to send the data to the
+            browser.
+
+        @raise twisted.web.error.UnsupportedMethod: If the HTTP verb
+            requested is not supported by this resource.
+        """
+
+
+def getChildForRequest(resource, request):
+    """
+    Traverse resource tree to find who will handle the request.
+    """
+    while request.postpath and not resource.isLeaf:
+        pathElement = request.postpath.pop(0)
+        request.prepath.append(pathElement)
+        resource = resource.getChildWithDefault(pathElement, request)
+    return resource
+
+
+@implementer(IResource)
+class Resource:
+    """
+    Define a web-accessible resource.
+
+    This serves two main purposes: one is to provide a standard representation
+    for what HTTP specification calls an 'entity', and the other is to provide
+    an abstract directory structure for URL retrieval.
+    """
+
+    entityType = IResource
+
+    server = None
+
+    def __init__(self):
+        """
+        Initialize.
+        """
+        self.children = {}
+
+    isLeaf = 0
+
+    ### Abstract Collection Interface
+
+    def listStaticNames(self):
+        return list(self.children.keys())
+
+    def listStaticEntities(self):
+        return list(self.children.items())
+
+    def listNames(self):
+        return list(self.listStaticNames()) + self.listDynamicNames()
+
+    def listEntities(self):
+        return list(self.listStaticEntities()) + self.listDynamicEntities()
+
+    def listDynamicNames(self):
+        return []
+
+    def listDynamicEntities(self, request=None):
+        return []
+
+    def getStaticEntity(self, name):
+        return self.children.get(name)
+
+    def getDynamicEntity(self, name, request):
+        if name not in self.children:
+            return self.getChild(name, request)
+        else:
+            return None
+
+    def delEntity(self, name):
+        del self.children[name]
+
+    def reallyPutEntity(self, name, entity):
+        self.children[name] = entity
+
+    # Concrete HTTP interface
+
+    def getChild(self, path, request):
+        """
+        Retrieve a 'child' resource from me.
+
+        Implement this to create dynamic resource generation -- resources which
+        are always available may be registered with self.putChild().
+
+        This will not be called if the class-level variable 'isLeaf' is set in
+        your subclass; instead, the 'postpath' attribute of the request will be
+        left as a list of the remaining path elements.
+
+        For example, the URL /foo/bar/baz will normally be::
+
+          | site.resource.getChild('foo').getChild('bar').getChild('baz').
+
+        However, if the resource returned by 'bar' has isLeaf set to true, then
+        the getChild call will never be made on it.
+
+        Parameters and return value have the same meaning and requirements as
+        those defined by L{IResource.getChildWithDefault}.
+        """
+        return _UnsafeNoResource()
+
+    def getChildWithDefault(self, path, request):
+        """
+        Retrieve a static or dynamically generated child resource from me.
+
+        First checks if a resource was added manually by putChild, and then
+        call getChild to check for dynamic resources. Only override if you want
+        to affect behaviour of all child lookups, rather than just dynamic
+        ones.
+
+        This will check to see if I have a pre-registered child resource of the
+        given name, and call getChild if I do not.
+
+        @see: L{IResource.getChildWithDefault}
+        """
+        if path in self.children:
+            return self.children[path]
+        return self.getChild(path, request)
+
+    def getChildForRequest(self, request):
+        """
+        Deprecated in favor of L{getChildForRequest}.
+
+        @see: L{twisted.web.resource.getChildForRequest}.
+        """
+        warnings.warn(
+            "Please use module level getChildForRequest.", DeprecationWarning, 2
+        )
+        return getChildForRequest(self, request)
+
+    def putChild(self, path: bytes, child: IResource) -> None:
+        """
+        Register a static child.
+
+        You almost certainly don't want '/' in your path. If you
+        intended to have the root of a folder, e.g. /foo/, you want
+        path to be ''.
+
+        @param path: A single path component.
+
+        @param child: The child resource to register.
+
+        @see: L{IResource.putChild}
+        """
+        if not isinstance(path, bytes):
+            raise TypeError(f"Path segment must be bytes, but {path!r} is {type(path)}")
+
+        self.children[path] = child
+        # IResource is incomplete and doesn't mention this server attribute, see
+        # https://github.com/twisted/twisted/issues/11717
+        child.server = self.server  # type: ignore[attr-defined]
+
+    def render(self, request):
+        """
+        Render a given resource. See L{IResource}'s render method.
+
+        I delegate to methods of self with the form 'render_METHOD'
+        where METHOD is the HTTP that was used to make the
+        request. Examples: render_GET, render_HEAD, render_POST, and
+        so on. Generally you should implement those methods instead of
+        overriding this one.
+
+        render_METHOD methods are expected to return a byte string which will be
+        the rendered page, unless the return value is C{server.NOT_DONE_YET}, in
+        which case it is this class's responsibility to write the results using
+        C{request.write(data)} and then call C{request.finish()}.
+
+        Old code that overrides render() directly is likewise expected
+        to return a byte string or NOT_DONE_YET.
+
+        @see: L{IResource.render}
+        """
+        m = getattr(self, "render_" + nativeString(request.method), None)
+        if not m:
+            try:
+                allowedMethods = self.allowedMethods
+            except AttributeError:
+                allowedMethods = _computeAllowedMethods(self)
+            raise UnsupportedMethod(allowedMethods)
+        return m(request)
+
+    def render_HEAD(self, request):
+        """
+        Default handling of HEAD method.
+
+        I just return self.render_GET(request). When method is HEAD,
+        the framework will handle this correctly.
+        """
+        return self.render_GET(request)
+
+
+def _computeAllowedMethods(resource):
+    """
+    Compute the allowed methods on a C{Resource} based on defined render_FOO
+    methods. Used when raising C{UnsupportedMethod} but C{Resource} does
+    not define C{allowedMethods} attribute.
+    """
+    allowedMethods = []
+    for name in prefixedMethodNames(resource.__class__, "render_"):
+        # Potentially there should be an API for encode('ascii') in this
+        # situation - an API for taking a Python native string (bytes on Python
+        # 2, text on Python 3) and returning a socket-compatible string type.
+        allowedMethods.append(name.encode("ascii"))
+    return allowedMethods
+
+
+class _UnsafeErrorPage(Resource):
+    """
+    L{_UnsafeErrorPage}, publicly available via the deprecated alias
+    C{ErrorPage}, is a resource which responds with a particular
+    (parameterized) status and a body consisting of HTML containing some
+    descriptive text.  This is useful for rendering simple error pages.
+
+    Deprecated in Twisted 22.10.0 because it permits HTML injection; use
+    L{twisted.web.pages.errorPage} instead.
+
+    @ivar template: A native string which will have a dictionary interpolated
+        into it to generate the response body.  The dictionary has the following
+        keys:
+
+          - C{"code"}: The status code passed to L{_UnsafeErrorPage.__init__}.
+          - C{"brief"}: The brief description passed to
+            L{_UnsafeErrorPage.__init__}.
+          - C{"detail"}: The detailed description passed to
+            L{_UnsafeErrorPage.__init__}.
+
+    @ivar code: An integer status code which will be used for the response.
+    @type code: C{int}
+
+    @ivar brief: A short string which will be included in the response body as
+        the page title.
+    @type brief: C{str}
+
+    @ivar detail: A longer string which will be included in the response body.
+    @type detail: C{str}
+    """
+
+    template = """
+<html>
+  <head><title>%(code)s - %(brief)s</title></head>
+  <body>
+    <h1>%(brief)s</h1>
+    <p>%(detail)s</p>
+  </body>
+</html>
+"""
+
+    def __init__(self, status, brief, detail):
+        Resource.__init__(self)
+        self.code = status
+        self.brief = brief
+        self.detail = detail
+
+    def render(self, request):
+        request.setResponseCode(self.code)
+        request.setHeader(b"content-type", b"text/html; charset=utf-8")
+        interpolated = self.template % dict(
+            code=self.code, brief=self.brief, detail=self.detail
+        )
+        if isinstance(interpolated, str):
+            return interpolated.encode("utf-8")
+        return interpolated
+
+    def getChild(self, chnam, request):
+        return self
+
+
+class _UnsafeNoResource(_UnsafeErrorPage):
+    """
+    L{_UnsafeNoResource}, publicly available via the deprecated alias
+    C{NoResource}, is a specialization of L{_UnsafeErrorPage} which
+    returns the HTTP response code I{NOT FOUND}.
+
+    Deprecated in Twisted 22.10.0 because it permits HTML injection; use
+    L{twisted.web.pages.notFound} instead.
+    """
+
+    def __init__(self, message="Sorry. No luck finding that resource."):
+        _UnsafeErrorPage.__init__(self, NOT_FOUND, "No Such Resource", message)
+
+
+class _UnsafeForbiddenResource(_UnsafeErrorPage):
+    """
+    L{_UnsafeForbiddenResource}, publicly available via the deprecated alias
+    C{ForbiddenResource} is a specialization of L{_UnsafeErrorPage} which
+    returns the I{FORBIDDEN} HTTP response code.
+
+    Deprecated in Twisted 22.10.0 because it permits HTML injection; use
+    L{twisted.web.pages.forbidden} instead.
+    """
+
+    def __init__(self, message="Sorry, resource is forbidden."):
+        _UnsafeErrorPage.__init__(self, FORBIDDEN, "Forbidden Resource", message)
+
+
+# Deliberately undocumented public aliases. See GHSA-vg46-2rrj-3647.
+ErrorPage = _UnsafeErrorPage
+NoResource = _UnsafeNoResource
+ForbiddenResource = _UnsafeForbiddenResource
+
+deprecatedModuleAttribute(
+    Version("Twisted", 22, 10, 0),
+    "Use twisted.web.pages.errorPage instead, which properly escapes HTML.",
+    __name__,
+    "ErrorPage",
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 22, 10, 0),
+    "Use twisted.web.pages.notFound instead, which properly escapes HTML.",
+    __name__,
+    "NoResource",
+)
+
+deprecatedModuleAttribute(
+    Version("Twisted", 22, 10, 0),
+    "Use twisted.web.pages.forbidden instead, which properly escapes HTML.",
+    __name__,
+    "ForbiddenResource",
+)
+
+
+class _IEncodingResource(Interface):
+    """
+    A resource which knows about L{_IRequestEncoderFactory}.
+
+    @since: 12.3
+    """
+
+    def getEncoder(request):
+        """
+        Parse the request and return an encoder if applicable, using
+        L{_IRequestEncoderFactory.encoderForRequest}.
+
+        @return: A L{_IRequestEncoder}, or L{None}.
+        """
+
+
+@implementer(_IEncodingResource)
+class EncodingResourceWrapper(proxyForInterface(IResource)):  # type: ignore[misc]
+    """
+    Wrap a L{IResource}, potentially applying an encoding to the response body
+    generated.
+
+    Note that the returned children resources won't be wrapped, so you have to
+    explicitly wrap them if you want the encoding to be applied.
+
+    @ivar encoders: A list of
+        L{_IRequestEncoderFactory<twisted.web.iweb._IRequestEncoderFactory>}
+        returning L{_IRequestEncoder<twisted.web.iweb._IRequestEncoder>} that
+        may transform the data passed to C{Request.write}. The list must be
+        sorted in order of priority: the first encoder factory handling the
+        request will prevent the others from doing the same.
+    @type encoders: C{list}.
+
+    @since: 12.3
+    """
+
+    def __init__(self, original, encoders):
+        super().__init__(original)
+        self._encoders = encoders
+
+    def getEncoder(self, request):
+        """
+        Browser the list of encoders looking for one applicable encoder.
+        """
+        for encoderFactory in self._encoders:
+            encoder = encoderFactory.encoderForRequest(request)
+            if encoder is not None:
+                return encoder
diff --git a/contrib/python/Twisted/py3/twisted/web/rewrite.py b/contrib/python/Twisted/py3/twisted/web/rewrite.py
new file mode 100644
index 00000000000..73f3c45b68c
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/rewrite.py
@@ -0,0 +1,55 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+from twisted.web import resource
+
+
+class RewriterResource(resource.Resource):
+    def __init__(self, orig, *rewriteRules):
+        resource.Resource.__init__(self)
+        self.resource = orig
+        self.rewriteRules = list(rewriteRules)
+
+    def _rewrite(self, request):
+        for rewriteRule in self.rewriteRules:
+            rewriteRule(request)
+
+    def getChild(self, path, request):
+        request.postpath.insert(0, path)
+        request.prepath.pop()
+        self._rewrite(request)
+        path = request.postpath.pop(0)
+        request.prepath.append(path)
+        return self.resource.getChildWithDefault(path, request)
+
+    def render(self, request):
+        self._rewrite(request)
+        return self.resource.render(request)
+
+
+def tildeToUsers(request):
+    if request.postpath and request.postpath[0][:1] == "~":
+        request.postpath[:1] = ["users", request.postpath[0][1:]]
+        request.path = "/" + "/".join(request.prepath + request.postpath)
+
+
+def alias(aliasPath, sourcePath):
+    """
+    I am not a very good aliaser. But I'm the best I can be. If I'm
+    aliasing to a Resource that generates links, and it uses any parts
+    of request.prepath to do so, the links will not be relative to the
+    aliased path, but rather to the aliased-to path. That I can't
+    alias static.File directory listings that nicely. However, I can
+    still be useful, as many resources will play nice.
+    """
+    sourcePath = sourcePath.split("/")
+    aliasPath = aliasPath.split("/")
+
+    def rewriter(request):
+        if request.postpath[: len(aliasPath)] == aliasPath:
+            after = request.postpath[len(aliasPath) :]
+            request.postpath = sourcePath + after
+            request.path = "/" + "/".join(request.prepath + request.postpath)
+
+    return rewriter
diff --git a/contrib/python/Twisted/py3/twisted/web/script.py b/contrib/python/Twisted/py3/twisted/web/script.py
new file mode 100644
index 00000000000..bc4a90f748a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/script.py
@@ -0,0 +1,193 @@
+# -*- test-case-name: twisted.web.test.test_script -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+I contain PythonScript, which is a very simple python script resource.
+"""
+
+
+import os
+import traceback
+from io import StringIO
+
+from twisted import copyright
+from twisted.python.compat import execfile, networkString
+from twisted.python.filepath import _coerceToFilesystemEncoding
+from twisted.web import http, resource, server, static, util
+
+rpyNoResource = """<p>You forgot to assign to the variable "resource" in your script. For example:</p>
+<pre>
+# MyCoolWebApp.rpy
+
+import mygreatresource
+
+resource = mygreatresource.MyGreatResource()
+</pre>
+"""
+
+
+class AlreadyCached(Exception):
+    """
+    This exception is raised when a path has already been cached.
+    """
+
+
+class CacheScanner:
+    def __init__(self, path, registry):
+        self.path = path
+        self.registry = registry
+        self.doCache = 0
+
+    def cache(self):
+        c = self.registry.getCachedPath(self.path)
+        if c is not None:
+            raise AlreadyCached(c)
+        self.recache()
+
+    def recache(self):
+        self.doCache = 1
+
+
+noRsrc = resource._UnsafeErrorPage(500, "Whoops! Internal Error", rpyNoResource)
+
+
+def ResourceScript(path, registry):
+    """
+    I am a normal py file which must define a 'resource' global, which should
+    be an instance of (a subclass of) web.resource.Resource; it will be
+    renderred.
+    """
+    cs = CacheScanner(path, registry)
+    glob = {
+        "__file__": _coerceToFilesystemEncoding("", path),
+        "resource": noRsrc,
+        "registry": registry,
+        "cache": cs.cache,
+        "recache": cs.recache,
+    }
+    try:
+        execfile(path, glob, glob)
+    except AlreadyCached as ac:
+        return ac.args[0]
+    rsrc = glob["resource"]
+    if cs.doCache and rsrc is not noRsrc:
+        registry.cachePath(path, rsrc)
+    return rsrc
+
+
+def ResourceTemplate(path, registry):
+    from quixote import ptl_compile  # type: ignore[import]
+
+    glob = {
+        "__file__": _coerceToFilesystemEncoding("", path),
+        "resource": resource._UnsafeErrorPage(
+            500, "Whoops! Internal Error", rpyNoResource
+        ),
+        "registry": registry,
+    }
+
+    with open(path) as f:  # Not closed by quixote as of 2.9.1
+        e = ptl_compile.compile_template(f, path)
+    code = compile(e, "<source>", "exec")
+    eval(code, glob, glob)
+    return glob["resource"]
+
+
+class ResourceScriptWrapper(resource.Resource):
+    def __init__(self, path, registry=None):
+        resource.Resource.__init__(self)
+        self.path = path
+        self.registry = registry or static.Registry()
+
+    def render(self, request):
+        res = ResourceScript(self.path, self.registry)
+        return res.render(request)
+
+    def getChildWithDefault(self, path, request):
+        res = ResourceScript(self.path, self.registry)
+        return res.getChildWithDefault(path, request)
+
+
+class ResourceScriptDirectory(resource.Resource):
+    """
+    L{ResourceScriptDirectory} is a resource which serves scripts from a
+    filesystem directory.  File children of a L{ResourceScriptDirectory} will
+    be served using L{ResourceScript}.  Directory children will be served using
+    another L{ResourceScriptDirectory}.
+
+    @ivar path: A C{str} giving the filesystem path in which children will be
+        looked up.
+
+    @ivar registry: A L{static.Registry} instance which will be used to decide
+        how to interpret scripts found as children of this resource.
+    """
+
+    def __init__(self, pathname, registry=None):
+        resource.Resource.__init__(self)
+        self.path = pathname
+        self.registry = registry or static.Registry()
+
+    def getChild(self, path, request):
+        fn = os.path.join(self.path, path)
+
+        if os.path.isdir(fn):
+            return ResourceScriptDirectory(fn, self.registry)
+        if os.path.exists(fn):
+            return ResourceScript(fn, self.registry)
+        return resource._UnsafeNoResource()
+
+    def render(self, request):
+        return resource._UnsafeNoResource().render(request)
+
+
+class PythonScript(resource.Resource):
+    """
+    I am an extremely simple dynamic resource; an embedded python script.
+
+    This will execute a file (usually of the extension '.epy') as Python code,
+    internal to the webserver.
+    """
+
+    isLeaf = True
+
+    def __init__(self, filename, registry):
+        """
+        Initialize me with a script name.
+        """
+        self.filename = filename
+        self.registry = registry
+
+    def render(self, request):
+        """
+        Render me to a web client.
+
+        Load my file, execute it in a special namespace (with 'request' and
+        '__file__' global vars) and finish the request.  Output to the web-page
+        will NOT be handled with print - standard output goes to the log - but
+        with request.write.
+        """
+        request.setHeader(
+            b"x-powered-by", networkString("Twisted/%s" % copyright.version)
+        )
+        namespace = {
+            "request": request,
+            "__file__": _coerceToFilesystemEncoding("", self.filename),
+            "registry": self.registry,
+        }
+        try:
+            execfile(self.filename, namespace, namespace)
+        except OSError as e:
+            if e.errno == 2:  # file not found
+                request.setResponseCode(http.NOT_FOUND)
+                request.write(
+                    resource._UnsafeNoResource("File not found.").render(request)
+                )
+        except BaseException:
+            io = StringIO()
+            traceback.print_exc(file=io)
+            output = util._PRE(io.getvalue())
+            output = output.encode("utf8")
+            request.write(output)
+        request.finish()
+        return server.NOT_DONE_YET
diff --git a/contrib/python/Twisted/py3/twisted/web/server.py b/contrib/python/Twisted/py3/twisted/web/server.py
new file mode 100644
index 00000000000..e8e01ec781b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/server.py
@@ -0,0 +1,906 @@
+# -*- test-case-name: twisted.web.test.test_web -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+This is a web server which integrates with the twisted.internet infrastructure.
+
+@var NOT_DONE_YET: A token value which L{twisted.web.resource.IResource.render}
+    implementations can return to indicate that the application will later call
+    C{.write} and C{.finish} to complete the request, and that the HTTP
+    connection should be left open.
+@type NOT_DONE_YET: Opaque; do not depend on any particular type for this
+    value.
+"""
+
+
+import copy
+import os
+import re
+import zlib
+from binascii import hexlify
+from html import escape
+from typing import List, Optional
+from urllib.parse import quote as _quote
+
+from zope.interface import implementer
+
+from incremental import Version
+
+from twisted import copyright
+from twisted.internet import address, interfaces
+from twisted.internet.error import AlreadyCalled, AlreadyCancelled
+from twisted.logger import Logger
+from twisted.python import components, failure, reflect
+from twisted.python.compat import nativeString, networkString
+from twisted.python.deprecate import deprecatedModuleAttribute
+from twisted.spread.pb import Copyable, ViewPoint
+from twisted.web import http, iweb, resource, util
+from twisted.web.error import UnsupportedMethod
+from twisted.web.http import unquote
+
+NOT_DONE_YET = 1
+
+__all__ = [
+    "supportedMethods",
+    "Request",
+    "Session",
+    "Site",
+    "version",
+    "NOT_DONE_YET",
+    "GzipEncoderFactory",
+]
+
+
+# backwards compatibility
+deprecatedModuleAttribute(
+    Version("Twisted", 12, 1, 0),
+    "Please use twisted.web.http.datetimeToString instead",
+    "twisted.web.server",
+    "date_time_string",
+)
+deprecatedModuleAttribute(
+    Version("Twisted", 12, 1, 0),
+    "Please use twisted.web.http.stringToDatetime instead",
+    "twisted.web.server",
+    "string_date_time",
+)
+date_time_string = http.datetimeToString
+string_date_time = http.stringToDatetime
+
+# Support for other methods may be implemented on a per-resource basis.
+supportedMethods = (b"GET", b"HEAD", b"POST")
+
+
+def quote(string, *args, **kwargs):
+    return _quote(string.decode("charmap"), *args, **kwargs).encode("charmap")
+
+
+def _addressToTuple(addr):
+    if isinstance(addr, address.IPv4Address):
+        return ("INET", addr.host, addr.port)
+    elif isinstance(addr, address.UNIXAddress):
+        return ("UNIX", addr.name)
+    else:
+        return tuple(addr)
+
+
+@implementer(iweb.IRequest)
+class Request(Copyable, http.Request, components.Componentized):
+    """
+    An HTTP request.
+
+    @ivar defaultContentType: A L{bytes} giving the default I{Content-Type}
+        value to send in responses if no other value is set.  L{None} disables
+        the default.
+
+    @ivar _insecureSession: The L{Session} object representing state that will
+        be transmitted over plain-text HTTP.
+
+    @ivar _secureSession: The L{Session} object representing the state that
+        will be transmitted only over HTTPS.
+    """
+
+    defaultContentType = b"text/html"
+
+    site = None
+    appRootURL = None
+    prepath: Optional[List[bytes]] = None
+    postpath: Optional[List[bytes]] = None
+    __pychecker__ = "unusednames=issuer"
+    _inFakeHead = False
+    _encoder = None
+    _log = Logger()
+
+    def __init__(self, *args, **kw):
+        http.Request.__init__(self, *args, **kw)
+        components.Componentized.__init__(self)
+
+    def getStateToCopyFor(self, issuer):
+        x = self.__dict__.copy()
+        del x["transport"]
+        # XXX refactor this attribute out; it's from protocol
+        # del x['server']
+        del x["channel"]
+        del x["content"]
+        del x["site"]
+        self.content.seek(0, 0)
+        x["content_data"] = self.content.read()
+        x["remote"] = ViewPoint(issuer, self)
+
+        # Address objects aren't jellyable
+        x["host"] = _addressToTuple(x["host"])
+        x["client"] = _addressToTuple(x["client"])
+
+        # Header objects also aren't jellyable.
+        x["requestHeaders"] = list(x["requestHeaders"].getAllRawHeaders())
+
+        return x
+
+    # HTML generation helpers
+
+    def sibLink(self, name):
+        """
+        Return the text that links to a sibling of the requested resource.
+
+        @param name: The sibling resource
+        @type name: C{bytes}
+
+        @return: A relative URL.
+        @rtype: C{bytes}
+        """
+        if self.postpath:
+            return (len(self.postpath) * b"../") + name
+        else:
+            return name
+
+    def childLink(self, name):
+        """
+        Return the text that links to a child of the requested resource.
+
+        @param name: The child resource
+        @type name: C{bytes}
+
+        @return: A relative URL.
+        @rtype: C{bytes}
+        """
+        lpp = len(self.postpath)
+        if lpp > 1:
+            return ((lpp - 1) * b"../") + name
+        elif lpp == 1:
+            return name
+        else:  # lpp == 0
+            if len(self.prepath) and self.prepath[-1]:
+                return self.prepath[-1] + b"/" + name
+            else:
+                return name
+
+    def gotLength(self, length):
+        """
+        Called when HTTP channel got length of content in this request.
+
+        This method is not intended for users.
+
+        @param length: The length of the request body, as indicated by the
+            request headers.  L{None} if the request headers do not indicate a
+            length.
+        """
+        try:
+            getContentFile = self.channel.site.getContentFile
+        except AttributeError:
+            http.Request.gotLength(self, length)
+        else:
+            self.content = getContentFile(length)
+
+    def process(self):
+        """
+        Process a request.
+
+        Find the addressed resource in this request's L{Site},
+        and call L{self.render()<Request.render()>} with it.
+
+        @see: L{Site.getResourceFor()}
+        """
+
+        # get site from channel
+        self.site = self.channel.site
+
+        # set various default headers
+        self.setHeader(b"server", version)
+        self.setHeader(b"date", http.datetimeToString())
+
+        # Resource Identification
+        self.prepath = []
+        self.postpath = list(map(unquote, self.path[1:].split(b"/")))
+
+        # Short-circuit for requests whose path is '*'.
+        if self.path == b"*":
+            self._handleStar()
+            return
+
+        try:
+            resrc = self.site.getResourceFor(self)
+            if resource._IEncodingResource.providedBy(resrc):
+                encoder = resrc.getEncoder(self)
+                if encoder is not None:
+                    self._encoder = encoder
+            self.render(resrc)
+        except BaseException:
+            self.processingFailed(failure.Failure())
+
+    def write(self, data):
+        """
+        Write data to the transport (if not responding to a HEAD request).
+
+        @param data: A string to write to the response.
+        @type data: L{bytes}
+        """
+        if not self.startedWriting:
+            # Before doing the first write, check to see if a default
+            # Content-Type header should be supplied. We omit it on
+            # NOT_MODIFIED and NO_CONTENT responses. We also omit it if there
+            # is a Content-Length header set to 0, as empty bodies don't need
+            # a content-type.
+            needsCT = self.code not in (http.NOT_MODIFIED, http.NO_CONTENT)
+            contentType = self.responseHeaders.getRawHeaders(b"content-type")
+            contentLength = self.responseHeaders.getRawHeaders(b"content-length")
+            contentLengthZero = contentLength and (contentLength[0] == b"0")
+
+            if (
+                needsCT
+                and contentType is None
+                and self.defaultContentType is not None
+                and not contentLengthZero
+            ):
+                self.responseHeaders.setRawHeaders(
+                    b"content-type", [self.defaultContentType]
+                )
+
+        # Only let the write happen if we're not generating a HEAD response by
+        # faking out the request method.  Note, if we are doing that,
+        # startedWriting will never be true, and the above logic may run
+        # multiple times.  It will only actually change the responseHeaders
+        # once though, so it's still okay.
+        if not self._inFakeHead:
+            if self._encoder:
+                data = self._encoder.encode(data)
+            http.Request.write(self, data)
+
+    def finish(self):
+        """
+        Override C{http.Request.finish} for possible encoding.
+        """
+        if self._encoder:
+            data = self._encoder.finish()
+            if data:
+                http.Request.write(self, data)
+        return http.Request.finish(self)
+
+    def render(self, resrc):
+        """
+        Ask a resource to render itself.
+
+        If the resource does not support the requested method,
+        generate a C{NOT IMPLEMENTED} or C{NOT ALLOWED} response.
+
+        @param resrc: The resource to render.
+        @type resrc: L{twisted.web.resource.IResource}
+
+        @see: L{IResource.render()<twisted.web.resource.IResource.render()>}
+        """
+        try:
+            body = resrc.render(self)
+        except UnsupportedMethod as e:
+            allowedMethods = e.allowedMethods
+            if (self.method == b"HEAD") and (b"GET" in allowedMethods):
+                # We must support HEAD (RFC 2616, 5.1.1).  If the
+                # resource doesn't, fake it by giving the resource
+                # a 'GET' request and then return only the headers,
+                # not the body.
+                self._log.info(
+                    "Using GET to fake a HEAD request for {resrc}", resrc=resrc
+                )
+                self.method = b"GET"
+                self._inFakeHead = True
+                body = resrc.render(self)
+
+                if body is NOT_DONE_YET:
+                    self._log.info(
+                        "Tried to fake a HEAD request for {resrc}, but "
+                        "it got away from me.",
+                        resrc=resrc,
+                    )
+                    # Oh well, I guess we won't include the content length.
+                else:
+                    self.setHeader(b"content-length", b"%d" % (len(body),))
+
+                self._inFakeHead = False
+                self.method = b"HEAD"
+                self.write(b"")
+                self.finish()
+                return
+
+            if self.method in (supportedMethods):
+                # We MUST include an Allow header
+                # (RFC 2616, 10.4.6 and 14.7)
+                self.setHeader(b"Allow", b", ".join(allowedMethods))
+                s = (
+                    """Your browser approached me (at %(URI)s) with"""
+                    """ the method "%(method)s".  I only allow"""
+                    """ the method%(plural)s %(allowed)s here."""
+                    % {
+                        "URI": escape(nativeString(self.uri)),
+                        "method": nativeString(self.method),
+                        "plural": ((len(allowedMethods) > 1) and "s") or "",
+                        "allowed": ", ".join([nativeString(x) for x in allowedMethods]),
+                    }
+                )
+                epage = resource._UnsafeErrorPage(
+                    http.NOT_ALLOWED, "Method Not Allowed", s
+                )
+                body = epage.render(self)
+            else:
+                epage = resource._UnsafeErrorPage(
+                    http.NOT_IMPLEMENTED,
+                    "Huh?",
+                    "I don't know how to treat a %s request."
+                    % (escape(self.method.decode("charmap")),),
+                )
+                body = epage.render(self)
+        # end except UnsupportedMethod
+
+        if body is NOT_DONE_YET:
+            return
+        if not isinstance(body, bytes):
+            body = resource._UnsafeErrorPage(
+                http.INTERNAL_SERVER_ERROR,
+                "Request did not return bytes",
+                "Request: "
+                # GHSA-vg46-2rrj-3647 note: _PRE does HTML-escape the input.
+                + util._PRE(reflect.safe_repr(self))
+                + "<br />"
+                + "Resource: "
+                + util._PRE(reflect.safe_repr(resrc))
+                + "<br />"
+                + "Value: "
+                + util._PRE(reflect.safe_repr(body)),
+            ).render(self)
+
+        if self.method == b"HEAD":
+            if len(body) > 0:
+                # This is a Bad Thing (RFC 2616, 9.4)
+                self._log.info(
+                    "Warning: HEAD request {slf} for resource {resrc} is"
+                    " returning a message body. I think I'll eat it.",
+                    slf=self,
+                    resrc=resrc,
+                )
+                self.setHeader(b"content-length", b"%d" % (len(body),))
+            self.write(b"")
+        else:
+            self.setHeader(b"content-length", b"%d" % (len(body),))
+            self.write(body)
+        self.finish()
+
+    def processingFailed(self, reason):
+        """
+        Finish this request with an indication that processing failed and
+        possibly display a traceback.
+
+        @param reason: Reason this request has failed.
+        @type reason: L{twisted.python.failure.Failure}
+
+        @return: The reason passed to this method.
+        @rtype: L{twisted.python.failure.Failure}
+        """
+        self._log.failure("", failure=reason)
+        if self.site.displayTracebacks:
+            body = (
+                b"<html><head><title>web.Server Traceback"
+                b" (most recent call last)</title></head>"
+                b"<body><b>web.Server Traceback"
+                b" (most recent call last):</b>\n\n"
+                + util.formatFailure(reason)
+                + b"\n\n</body></html>\n"
+            )
+        else:
+            body = (
+                b"<html><head><title>Processing Failed"
+                b"</title></head><body>"
+                b"<b>Processing Failed</b></body></html>"
+            )
+
+        self.setResponseCode(http.INTERNAL_SERVER_ERROR)
+        self.setHeader(b"content-type", b"text/html")
+        self.setHeader(b"content-length", b"%d" % (len(body),))
+        self.write(body)
+        self.finish()
+        return reason
+
+    def view_write(self, issuer, data):
+        """Remote version of write; same interface."""
+        self.write(data)
+
+    def view_finish(self, issuer):
+        """Remote version of finish; same interface."""
+        self.finish()
+
+    def view_addCookie(self, issuer, k, v, **kwargs):
+        """Remote version of addCookie; same interface."""
+        self.addCookie(k, v, **kwargs)
+
+    def view_setHeader(self, issuer, k, v):
+        """Remote version of setHeader; same interface."""
+        self.setHeader(k, v)
+
+    def view_setLastModified(self, issuer, when):
+        """Remote version of setLastModified; same interface."""
+        self.setLastModified(when)
+
+    def view_setETag(self, issuer, tag):
+        """Remote version of setETag; same interface."""
+        self.setETag(tag)
+
+    def view_setResponseCode(self, issuer, code, message=None):
+        """
+        Remote version of setResponseCode; same interface.
+        """
+        self.setResponseCode(code, message)
+
+    def view_registerProducer(self, issuer, producer, streaming):
+        """Remote version of registerProducer; same interface.
+        (requires a remote producer.)
+        """
+        self.registerProducer(_RemoteProducerWrapper(producer), streaming)
+
+    def view_unregisterProducer(self, issuer):
+        self.unregisterProducer()
+
+    ### these calls remain local
+
+    _secureSession = None
+    _insecureSession = None
+
+    @property
+    def session(self):
+        """
+        If a session has already been created or looked up with
+        L{Request.getSession}, this will return that object.  (This will always
+        be the session that matches the security of the request; so if
+        C{forceNotSecure} is used on a secure request, this will not return
+        that session.)
+
+        @return: the session attribute
+        @rtype: L{Session} or L{None}
+        """
+        if self.isSecure():
+            return self._secureSession
+        else:
+            return self._insecureSession
+
+    def getSession(self, sessionInterface=None, forceNotSecure=False):
+        """
+        Check if there is a session cookie, and if not, create it.
+
+        By default, the cookie with be secure for HTTPS requests and not secure
+        for HTTP requests.  If for some reason you need access to the insecure
+        cookie from a secure request you can set C{forceNotSecure = True}.
+
+        @param forceNotSecure: Should we retrieve a session that will be
+            transmitted over HTTP, even if this L{Request} was delivered over
+            HTTPS?
+        @type forceNotSecure: L{bool}
+        """
+        # Make sure we aren't creating a secure session on a non-secure page
+        secure = self.isSecure() and not forceNotSecure
+
+        if not secure:
+            cookieString = b"TWISTED_SESSION"
+            sessionAttribute = "_insecureSession"
+        else:
+            cookieString = b"TWISTED_SECURE_SESSION"
+            sessionAttribute = "_secureSession"
+
+        session = getattr(self, sessionAttribute)
+
+        if session is not None:
+            # We have a previously created session.
+            try:
+                # Refresh the session, to keep it alive.
+                session.touch()
+            except (AlreadyCalled, AlreadyCancelled):
+                # Session has already expired.
+                session = None
+
+        if session is None:
+            # No session was created yet for this request.
+            cookiename = b"_".join([cookieString] + self.sitepath)
+            sessionCookie = self.getCookie(cookiename)
+            if sessionCookie:
+                try:
+                    session = self.site.getSession(sessionCookie)
+                except KeyError:
+                    pass
+            # if it still hasn't been set, fix it up.
+            if not session:
+                session = self.site.makeSession()
+                self.addCookie(cookiename, session.uid, path=b"/", secure=secure)
+
+        setattr(self, sessionAttribute, session)
+
+        if sessionInterface:
+            return session.getComponent(sessionInterface)
+
+        return session
+
+    def _prePathURL(self, prepath):
+        port = self.getHost().port
+        if self.isSecure():
+            default = 443
+        else:
+            default = 80
+        if port == default:
+            hostport = ""
+        else:
+            hostport = ":%d" % port
+        prefix = networkString(
+            "http%s://%s%s/"
+            % (
+                self.isSecure() and "s" or "",
+                nativeString(self.getRequestHostname()),
+                hostport,
+            )
+        )
+        path = b"/".join([quote(segment, safe=b"") for segment in prepath])
+        return prefix + path
+
+    def prePathURL(self):
+        return self._prePathURL(self.prepath)
+
+    def URLPath(self):
+        from twisted.python import urlpath
+
+        return urlpath.URLPath.fromRequest(self)
+
+    def rememberRootURL(self):
+        """
+        Remember the currently-processed part of the URL for later
+        recalling.
+        """
+        url = self._prePathURL(self.prepath[:-1])
+        self.appRootURL = url
+
+    def getRootURL(self):
+        """
+        Get a previously-remembered URL.
+
+        @return: An absolute URL.
+        @rtype: L{bytes}
+        """
+        return self.appRootURL
+
+    def _handleStar(self):
+        """
+        Handle receiving a request whose path is '*'.
+
+        RFC 7231 defines an OPTIONS * request as being something that a client
+        can send as a low-effort way to probe server capabilities or readiness.
+        Rather than bother the user with this, we simply fast-path it back to
+        an empty 200 OK. Any non-OPTIONS verb gets a 405 Method Not Allowed
+        telling the client they can only use OPTIONS.
+        """
+        if self.method == b"OPTIONS":
+            self.setResponseCode(http.OK)
+        else:
+            self.setResponseCode(http.NOT_ALLOWED)
+            self.setHeader(b"Allow", b"OPTIONS")
+
+        # RFC 7231 says we MUST set content-length 0 when responding to this
+        # with no body.
+        self.setHeader(b"Content-Length", b"0")
+        self.finish()
+
+
+@implementer(iweb._IRequestEncoderFactory)
+class GzipEncoderFactory:
+    """
+    @cvar compressLevel: The compression level used by the compressor, default
+        to 9 (highest).
+
+    @since: 12.3
+    """
+
+    _gzipCheckRegex = re.compile(rb"(:?^|[\s,])gzip(:?$|[\s,])")
+    compressLevel = 9
+
+    def encoderForRequest(self, request):
+        """
+        Check the headers if the client accepts gzip encoding, and encodes the
+        request if so.
+        """
+        acceptHeaders = b",".join(
+            request.requestHeaders.getRawHeaders(b"accept-encoding", [])
+        )
+        if self._gzipCheckRegex.search(acceptHeaders):
+            encoding = request.responseHeaders.getRawHeaders(b"content-encoding")
+            if encoding:
+                encoding = b",".join(encoding + [b"gzip"])
+            else:
+                encoding = b"gzip"
+
+            request.responseHeaders.setRawHeaders(b"content-encoding", [encoding])
+            return _GzipEncoder(self.compressLevel, request)
+
+
+@implementer(iweb._IRequestEncoder)
+class _GzipEncoder:
+    """
+    An encoder which supports gzip.
+
+    @ivar _zlibCompressor: The zlib compressor instance used to compress the
+        stream.
+
+    @ivar _request: A reference to the originating request.
+
+    @since: 12.3
+    """
+
+    _zlibCompressor = None
+
+    def __init__(self, compressLevel, request):
+        self._zlibCompressor = zlib.compressobj(
+            compressLevel, zlib.DEFLATED, 16 + zlib.MAX_WBITS
+        )
+        self._request = request
+
+    def encode(self, data):
+        """
+        Write to the request, automatically compressing data on the fly.
+        """
+        if not self._request.startedWriting:
+            # Remove the content-length header, we can't honor it
+            # because we compress on the fly.
+            self._request.responseHeaders.removeHeader(b"content-length")
+        return self._zlibCompressor.compress(data)
+
+    def finish(self):
+        """
+        Finish handling the request request, flushing any data from the zlib
+        buffer.
+        """
+        remain = self._zlibCompressor.flush()
+        self._zlibCompressor = None
+        return remain
+
+
+class _RemoteProducerWrapper:
+    def __init__(self, remote):
+        self.resumeProducing = remote.remoteMethod("resumeProducing")
+        self.pauseProducing = remote.remoteMethod("pauseProducing")
+        self.stopProducing = remote.remoteMethod("stopProducing")
+
+
+class Session(components.Componentized):
+    """
+    A user's session with a system.
+
+    This utility class contains no functionality, but is used to
+    represent a session.
+
+    @ivar site: The L{Site} that generated the session.
+    @type site: L{Site}
+
+    @ivar uid: A unique identifier for the session.
+    @type uid: L{bytes}
+
+    @ivar _reactor: An object providing L{IReactorTime} to use for scheduling
+        expiration.
+
+    @ivar sessionTimeout: Time after last modification the session will expire,
+        in seconds.
+    @type sessionTimeout: L{float}
+
+    @ivar lastModified: Time the C{touch()} method was last called (or time the
+        session was created). A UNIX timestamp as returned by
+        L{IReactorTime.seconds()}.
+    @type lastModified: L{float}
+    """
+
+    sessionTimeout = 900
+
+    _expireCall = None
+
+    def __init__(self, site, uid, reactor=None):
+        """
+        Initialize a session with a unique ID for that session.
+
+        @param reactor: L{IReactorTime} used to schedule expiration of the
+            session. If C{None}, the reactor associated with I{site} is used.
+        """
+        super().__init__()
+
+        if reactor is None:
+            reactor = site.reactor
+        self._reactor = reactor
+
+        self.site = site
+        self.uid = uid
+        self.expireCallbacks = []
+        self.touch()
+        self.sessionNamespaces = {}
+
+    def startCheckingExpiration(self):
+        """
+        Start expiration tracking.
+
+        @return: L{None}
+        """
+        self._expireCall = self._reactor.callLater(self.sessionTimeout, self.expire)
+
+    def notifyOnExpire(self, callback):
+        """
+        Call this callback when the session expires or logs out.
+        """
+        self.expireCallbacks.append(callback)
+
+    def expire(self):
+        """
+        Expire/logout of the session.
+        """
+        del self.site.sessions[self.uid]
+        for c in self.expireCallbacks:
+            c()
+        self.expireCallbacks = []
+        if self._expireCall and self._expireCall.active():
+            self._expireCall.cancel()
+            # Break reference cycle.
+            self._expireCall = None
+
+    def touch(self):
+        """
+        Mark the session as modified, which resets expiration timer.
+        """
+        self.lastModified = self._reactor.seconds()
+        if self._expireCall is not None:
+            self._expireCall.reset(self.sessionTimeout)
+
+
+version = networkString(f"TwistedWeb/{copyright.version}")
+
+
+@implementer(interfaces.IProtocolNegotiationFactory)
+class Site(http.HTTPFactory):
+    """
+    A web site: manage log, sessions, and resources.
+
+    @ivar requestFactory: A factory which is called with (channel)
+        and creates L{Request} instances. Default to L{Request}.
+
+    @ivar displayTracebacks: If set, unhandled exceptions raised during
+        rendering are returned to the client as HTML. Default to C{False}.
+
+    @ivar sessionFactory: factory for sessions objects. Default to L{Session}.
+
+    @ivar sessions: Mapping of session IDs to objects returned by
+        C{sessionFactory}.
+    @type sessions: L{dict} mapping L{bytes} to L{Session} given the default
+        C{sessionFactory}
+
+    @ivar counter: The number of sessions that have been generated.
+    @type counter: L{int}
+
+    @ivar sessionCheckTime: Deprecated and unused. See
+        L{Session.sessionTimeout} instead.
+    """
+
+    counter = 0
+    requestFactory = Request
+    displayTracebacks = False
+    sessionFactory = Session
+    sessionCheckTime = 1800
+    _entropy = os.urandom
+
+    def __init__(self, resource, requestFactory=None, *args, **kwargs):
+        """
+        @param resource: The root of the resource hierarchy.  All request
+            traversal for requests received by this factory will begin at this
+            resource.
+        @type resource: L{IResource} provider
+        @param requestFactory: Overwrite for default requestFactory.
+        @type requestFactory: C{callable} or C{class}.
+
+        @see: L{twisted.web.http.HTTPFactory.__init__}
+        """
+        super().__init__(*args, **kwargs)
+        self.sessions = {}
+        self.resource = resource
+        if requestFactory is not None:
+            self.requestFactory = requestFactory
+
+    def _openLogFile(self, path):
+        from twisted.python import logfile
+
+        return logfile.LogFile(os.path.basename(path), os.path.dirname(path))
+
+    def __getstate__(self):
+        d = self.__dict__.copy()
+        d["sessions"] = {}
+        return d
+
+    def _mkuid(self):
+        """
+        (internal) Generate an opaque, unique ID for a user's session.
+        """
+        self.counter = self.counter + 1
+        return hexlify(self._entropy(32))
+
+    def makeSession(self):
+        """
+        Generate a new Session instance, and store it for future reference.
+        """
+        uid = self._mkuid()
+        session = self.sessions[uid] = self.sessionFactory(self, uid)
+        session.startCheckingExpiration()
+        return session
+
+    def getSession(self, uid):
+        """
+        Get a previously generated session.
+
+        @param uid: Unique ID of the session.
+        @type uid: L{bytes}.
+
+        @raise KeyError: If the session is not found.
+        """
+        return self.sessions[uid]
+
+    def buildProtocol(self, addr):
+        """
+        Generate a channel attached to this site.
+        """
+        channel = super().buildProtocol(addr)
+        channel.requestFactory = self.requestFactory
+        channel.site = self
+        return channel
+
+    isLeaf = 0
+
+    def render(self, request):
+        """
+        Redirect because a Site is always a directory.
+        """
+        request.redirect(request.prePathURL() + b"/")
+        request.finish()
+
+    def getChildWithDefault(self, pathEl, request):
+        """
+        Emulate a resource's getChild method.
+        """
+        request.site = self
+        return self.resource.getChildWithDefault(pathEl, request)
+
+    def getResourceFor(self, request):
+        """
+        Get a resource for a request.
+
+        This iterates through the resource hierarchy, calling
+        getChildWithDefault on each resource it finds for a path element,
+        stopping when it hits an element where isLeaf is true.
+        """
+        request.site = self
+        # Sitepath is used to determine cookie names between distributed
+        # servers and disconnected sites.
+        request.sitepath = copy.copy(request.prepath)
+        return resource.getChildForRequest(self.resource, request)
+
+    # IProtocolNegotiationFactory
+    def acceptableProtocols(self):
+        """
+        Protocols this server can speak.
+        """
+        baseProtocols = [b"http/1.1"]
+
+        if http.H2_ENABLED:
+            baseProtocols.insert(0, b"h2")
+
+        return baseProtocols
diff --git a/contrib/python/Twisted/py3/twisted/web/soap.py b/contrib/python/Twisted/py3/twisted/web/soap.py
new file mode 100644
index 00000000000..c60bc92b916
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/soap.py
@@ -0,0 +1,166 @@
+# -*- test-case-name: twisted.web.test.test_soap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+SOAP support for twisted.web.
+
+Requires SOAPpy 0.10.1 or later.
+
+Maintainer: Itamar Shtull-Trauring
+
+Future plans:
+SOAPContext support of some kind.
+Pluggable method lookup policies.
+"""
+
+# SOAPpy
+import SOAPpy  # type: ignore[import]
+
+from twisted.internet import defer
+
+# twisted imports
+from twisted.web import client, resource, server
+
+
+class SOAPPublisher(resource.Resource):
+    """Publish SOAP methods.
+
+    By default, publish methods beginning with 'soap_'. If the method
+    has an attribute 'useKeywords', it well get the arguments passed
+    as keyword args.
+    """
+
+    isLeaf = 1
+
+    # override to change the encoding used for responses
+    encoding = "UTF-8"
+
+    def lookupFunction(self, functionName):
+        """Lookup published SOAP function.
+
+        Override in subclasses. Default behaviour - publish methods
+        starting with soap_.
+
+        @return: callable or None if not found.
+        """
+        return getattr(self, "soap_%s" % functionName, None)
+
+    def render(self, request):
+        """Handle a SOAP command."""
+        data = request.content.read()
+
+        p, header, body, attrs = SOAPpy.parseSOAPRPC(data, 1, 1, 1)
+
+        methodName, args, kwargs = p._name, p._aslist, p._asdict
+
+        # deal with changes in SOAPpy 0.11
+        if callable(args):
+            args = args()
+        if callable(kwargs):
+            kwargs = kwargs()
+
+        function = self.lookupFunction(methodName)
+
+        if not function:
+            self._methodNotFound(request, methodName)
+            return server.NOT_DONE_YET
+        else:
+            if hasattr(function, "useKeywords"):
+                keywords = {}
+                for k, v in kwargs.items():
+                    keywords[str(k)] = v
+                d = defer.maybeDeferred(function, **keywords)
+            else:
+                d = defer.maybeDeferred(function, *args)
+
+        d.addCallback(self._gotResult, request, methodName)
+        d.addErrback(self._gotError, request, methodName)
+        return server.NOT_DONE_YET
+
+    def _methodNotFound(self, request, methodName):
+        response = SOAPpy.buildSOAP(
+            SOAPpy.faultType(
+                "%s:Client" % SOAPpy.NS.ENV_T, "Method %s not found" % methodName
+            ),
+            encoding=self.encoding,
+        )
+        self._sendResponse(request, response, status=500)
+
+    def _gotResult(self, result, request, methodName):
+        if not isinstance(result, SOAPpy.voidType):
+            result = {"Result": result}
+        response = SOAPpy.buildSOAP(
+            kw={"%sResponse" % methodName: result}, encoding=self.encoding
+        )
+        self._sendResponse(request, response)
+
+    def _gotError(self, failure, request, methodName):
+        e = failure.value
+        if isinstance(e, SOAPpy.faultType):
+            fault = e
+        else:
+            fault = SOAPpy.faultType(
+                "%s:Server" % SOAPpy.NS.ENV_T, "Method %s failed." % methodName
+            )
+        response = SOAPpy.buildSOAP(fault, encoding=self.encoding)
+        self._sendResponse(request, response, status=500)
+
+    def _sendResponse(self, request, response, status=200):
+        request.setResponseCode(status)
+
+        if self.encoding is not None:
+            mimeType = 'text/xml; charset="%s"' % self.encoding
+        else:
+            mimeType = "text/xml"
+        request.setHeader("Content-type", mimeType)
+        request.setHeader("Content-length", str(len(response)))
+        request.write(response)
+        request.finish()
+
+
+class Proxy:
+    """A Proxy for making remote SOAP calls.
+
+    Pass the URL of the remote SOAP server to the constructor.
+
+    Use proxy.callRemote('foobar', 1, 2) to call remote method
+    'foobar' with args 1 and 2, proxy.callRemote('foobar', x=1)
+    will call foobar with named argument 'x'.
+    """
+
+    # at some point this should have encoding etc. kwargs
+    def __init__(self, url, namespace=None, header=None):
+        self.url = url
+        self.namespace = namespace
+        self.header = header
+
+    def _cbGotResult(self, result):
+        result = SOAPpy.parseSOAPRPC(result)
+        if hasattr(result, "Result"):
+            return result.Result
+        elif len(result) == 1:
+            ## SOAPpy 0.11.6 wraps the return results in a containing structure.
+            ## This check added to make Proxy behaviour emulate SOAPProxy, which
+            ## flattens the structure by default.
+            ## This behaviour is OK because even singleton lists are wrapped in
+            ## another singleton structType, which is almost always useless.
+            return result[0]
+        else:
+            return result
+
+    def callRemote(self, method, *args, **kwargs):
+        payload = SOAPpy.buildSOAP(
+            args=args,
+            kw=kwargs,
+            method=method,
+            header=self.header,
+            namespace=self.namespace,
+        )
+        return client.getPage(
+            self.url,
+            postdata=payload,
+            method="POST",
+            headers={"content-type": "text/xml", "SOAPAction": method},
+        ).addCallback(self._cbGotResult)
diff --git a/contrib/python/Twisted/py3/twisted/web/static.py b/contrib/python/Twisted/py3/twisted/web/static.py
new file mode 100644
index 00000000000..aeffd03fb16
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/static.py
@@ -0,0 +1,1078 @@
+# -*- test-case-name: twisted.web.test.test_static -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Static resources for L{twisted.web}.
+"""
+from __future__ import annotations
+
+import errno
+import itertools
+import mimetypes
+import os
+import time
+import warnings
+from html import escape
+from typing import Any, Callable, Dict, Sequence
+from urllib.parse import quote, unquote
+
+from zope.interface import implementer
+
+from incremental import Version
+from typing_extensions import Literal
+
+from twisted.internet import abstract, interfaces
+from twisted.python import components, filepath, log
+from twisted.python.compat import nativeString, networkString
+from twisted.python.deprecate import deprecated
+from twisted.python.runtime import platformType
+from twisted.python.url import URL
+from twisted.python.util import InsensitiveDict
+from twisted.web import http, resource, server
+from twisted.web.util import redirectTo
+
+dangerousPathError = resource._UnsafeNoResource("Invalid request URL.")
+
+
+def isDangerous(path):
+    return path == b".." or b"/" in path or networkString(os.sep) in path
+
+
+class Data(resource.Resource):
+    """
+    This is a static, in-memory resource.
+    """
+
+    def __init__(self, data, type):
+        """
+        @param data: The bytes that make up this data resource.
+        @type data: L{bytes}
+
+        @param type: A native string giving the Internet media type for this
+            content.
+        @type type: L{str}
+        """
+        resource.Resource.__init__(self)
+        self.data = data
+        self.type = type
+
+    def render_GET(self, request):
+        request.setHeader(b"content-type", networkString(self.type))
+        request.setHeader(b"content-length", b"%d" % (len(self.data),))
+        if request.method == b"HEAD":
+            return b""
+        return self.data
+
+    render_HEAD = render_GET
+
+
+@deprecated(Version("Twisted", 16, 0, 0))
+def addSlash(request):
+    """
+    Add a trailing slash to C{request}'s URI. Deprecated, do not use.
+    """
+    return _addSlash(request)
+
+
+def _addSlash(request):
+    """
+    Add a trailing slash to C{request}'s URI.
+
+    @param request: The incoming request to add the ending slash to.
+    @type request: An object conforming to L{twisted.web.iweb.IRequest}
+
+    @return: A URI with a trailing slash, with query and fragment preserved.
+    @rtype: L{bytes}
+    """
+    url = URL.fromText(request.uri.decode("ascii"))
+    # Add an empty path segment at the end, so that it adds a trailing slash
+    url = url.replace(path=list(url.path) + [""])
+    return url.asText().encode("ascii")
+
+
+class Redirect(resource.Resource):
+    def __init__(self, request):
+        resource.Resource.__init__(self)
+        self.url = _addSlash(request)
+
+    def render(self, request):
+        return redirectTo(self.url, request)
+
+
+class Registry(components.Componentized):
+    """
+    I am a Componentized object that will be made available to internal Twisted
+    file-based dynamic web content such as .rpy and .epy scripts.
+    """
+
+    def __init__(self):
+        components.Componentized.__init__(self)
+        self._pathCache = {}
+
+    def cachePath(self, path, rsrc):
+        self._pathCache[path] = rsrc
+
+    def getCachedPath(self, path):
+        return self._pathCache.get(path)
+
+
+def loadMimeTypes(mimetype_locations=None, init=mimetypes.init):
+    """
+    Produces a mapping of extensions (with leading dot) to MIME types.
+
+    It does this by calling the C{init} function of the L{mimetypes} module.
+    This will have the side effect of modifying the global MIME types cache
+    in that module.
+
+    Multiple file locations containing mime-types can be passed as a list.
+    The files will be sourced in that order, overriding mime-types from the
+    files sourced beforehand, but only if a new entry explicitly overrides
+    the current entry.
+
+    @param mimetype_locations: Optional. List of paths to C{mime.types} style
+        files that should be used.
+    @type mimetype_locations: iterable of paths or L{None}
+    @param init: The init function to call. Defaults to the global C{init}
+        function of the C{mimetypes} module. For internal use (testing) only.
+    @type init: callable
+    """
+    init(mimetype_locations)
+    mimetypes.types_map.update(
+        {
+            ".conf": "text/plain",
+            ".diff": "text/plain",
+            ".flac": "audio/x-flac",
+            ".java": "text/plain",
+            ".oz": "text/x-oz",
+            ".swf": "application/x-shockwave-flash",
+            ".wml": "text/vnd.wap.wml",
+            ".xul": "application/vnd.mozilla.xul+xml",
+            ".patch": "text/plain",
+        }
+    )
+    return mimetypes.types_map
+
+
+def getTypeAndEncoding(filename, types, encodings, defaultType):
+    p, ext = filepath.FilePath(filename).splitext()
+    ext = filepath._coerceToFilesystemEncoding("", ext.lower())
+    if ext in encodings:
+        enc = encodings[ext]
+        ext = os.path.splitext(p)[1].lower()
+    else:
+        enc = None
+    type = types.get(ext, defaultType)
+    return type, enc
+
+
+class File(resource.Resource, filepath.FilePath[str]):
+    """
+    File is a resource that represents a plain non-interpreted file
+    (although it can look for an extension like .rpy or .cgi and hand the
+    file to a processor for interpretation if you wish). Its constructor
+    takes a file path.
+
+    Alternatively, you can give a directory path to the constructor. In this
+    case the resource will represent that directory, and its children will
+    be files underneath that directory. This provides access to an entire
+    filesystem tree with a single Resource.
+
+    If you map the URL 'http://server/FILE' to a resource created as
+    File('/tmp'), then http://server/FILE/ will return an HTML-formatted
+    listing of the /tmp/ directory, and http://server/FILE/foo/bar.html will
+    return the contents of /tmp/foo/bar.html .
+
+    @cvar childNotFound: L{Resource} used to render 404 Not Found error pages.
+    @cvar forbidden: L{Resource} used to render 403 Forbidden error pages.
+
+    @ivar contentTypes: a mapping of extensions to MIME types used to set the
+        default value for the Content-Type header.
+        It is initialized with the values returned by L{loadMimeTypes}.
+    @type contentTypes: C{dict}
+
+    @ivar contentEncodings: a mapping of extensions to encoding types used to
+        set default value for the Content-Encoding header.
+    @type contentEncodings: C{dict}
+    """
+
+    contentTypes = loadMimeTypes()
+
+    contentEncodings = {".gz": "gzip", ".bz2": "bzip2"}
+
+    processors: Dict[str, Callable[[str, Any], Data]] = {}
+
+    indexNames = ["index", "index.html", "index.htm", "index.rpy"]
+
+    type = None
+
+    def __init__(
+        self,
+        path: str,
+        defaultType: str = "text/html",
+        ignoredExts: Sequence[str] = (),
+        registry: Registry | None = None,
+        allowExt: Literal[0] = 0,
+    ) -> None:
+        """
+        Create a file with the given path.
+
+        @param path: The filename of the file from which this L{File} will
+            serve data.
+        @type path: C{str}
+
+        @param defaultType: A I{major/minor}-style MIME type specifier
+            indicating the I{Content-Type} with which this L{File}'s data
+            will be served if a MIME type cannot be determined based on
+            C{path}'s extension.
+        @type defaultType: C{str}
+
+        @param ignoredExts: A sequence giving the extensions of paths in the
+            filesystem which will be ignored for the purposes of child
+            lookup.  For example, if C{ignoredExts} is C{(".bar",)} and
+            C{path} is a directory containing a file named C{"foo.bar"}, a
+            request for the C{"foo"} child of this resource will succeed
+            with a L{File} pointing to C{"foo.bar"}.
+
+        @param registry: The registry object being used to handle this
+            request.  If L{None}, one will be created.
+        @type registry: L{Registry}
+
+        @param allowExt: Ignored parameter, only present for backwards
+            compatibility.  Do not pass a value for this parameter.
+        """
+        resource.Resource.__init__(self)
+        filepath.FilePath.__init__(self, path)
+        self.defaultType = defaultType
+        if ignoredExts in (0, 1) or allowExt:
+            warnings.warn("ignoredExts should receive a list, not a boolean")
+            if ignoredExts or allowExt:
+                self.ignoredExts = ["*"]
+            else:
+                self.ignoredExts = []
+        else:
+            self.ignoredExts = list(ignoredExts)
+        self.registry = registry or Registry()
+
+    def ignoreExt(self, ext):
+        """Ignore the given extension.
+
+        Serve file.ext if file is requested
+        """
+        self.ignoredExts.append(ext)
+
+    childNotFound = resource._UnsafeNoResource("File not found.")
+    forbidden = resource._UnsafeForbiddenResource()
+
+    def directoryListing(self):
+        """
+        Return a resource that generates an HTML listing of the
+        directory this path represents.
+
+        @return: A resource that renders the directory to HTML.
+        @rtype: L{DirectoryLister}
+        """
+        path = self.path
+        names = self.listNames()
+        return DirectoryLister(
+            path, names, self.contentTypes, self.contentEncodings, self.defaultType
+        )
+
+    def getChild(self, path, request):
+        """
+        If this L{File}"s path refers to a directory, return a L{File}
+        referring to the file named C{path} in that directory.
+
+        If C{path} is the empty string, return a L{DirectoryLister}
+        instead.
+
+        @param path: The current path segment.
+        @type path: L{bytes}
+
+        @param request: The incoming request.
+        @type request: An that provides L{twisted.web.iweb.IRequest}.
+
+        @return: A resource representing the requested file or
+            directory, or L{NoResource} if the path cannot be
+            accessed.
+        @rtype: An object that provides L{resource.IResource}.
+        """
+        if isinstance(path, bytes):
+            try:
+                # Request calls urllib.unquote on each path segment,
+                # leaving us with raw bytes.
+                path = path.decode("utf-8")
+            except UnicodeDecodeError:
+                log.err(None, f"Could not decode path segment as utf-8: {path!r}")
+                return self.childNotFound
+
+        self.restat(reraise=False)
+
+        if not self.isdir():
+            return self.childNotFound
+
+        if path:
+            try:
+                fpath = self.child(path)
+            except filepath.InsecurePath:
+                return self.childNotFound
+        else:
+            fpath = self.childSearchPreauth(*self.indexNames)
+            if fpath is None:
+                return self.directoryListing()
+
+        if not fpath.exists():
+            fpath = fpath.siblingExtensionSearch(*self.ignoredExts)
+            if fpath is None:
+                return self.childNotFound
+
+        extension = fpath.splitext()[1]
+        if platformType == "win32":
+            # don't want .RPY to be different than .rpy, since that would allow
+            # source disclosure.
+            processor = InsensitiveDict(self.processors).get(extension)
+        else:
+            processor = self.processors.get(extension)
+        if processor:
+            return resource.IResource(processor(fpath.path, self.registry))
+        return self.createSimilarFile(fpath.path)
+
+    # methods to allow subclasses to e.g. decrypt files on the fly:
+    def openForReading(self):
+        """Open a file and return it."""
+        return self.open()
+
+    def getFileSize(self):
+        """Return file size."""
+        return self.getsize()
+
+    def _parseRangeHeader(self, range):
+        """
+        Parse the value of a Range header into (start, stop) pairs.
+
+        In a given pair, either of start or stop can be None, signifying that
+        no value was provided, but not both.
+
+        @return: A list C{[(start, stop)]} of pairs of length at least one.
+
+        @raise ValueError: if the header is syntactically invalid or if the
+            Bytes-Unit is anything other than "bytes'.
+        """
+        try:
+            kind, value = range.split(b"=", 1)
+        except ValueError:
+            raise ValueError("Missing '=' separator")
+        kind = kind.strip()
+        if kind != b"bytes":
+            raise ValueError(f"Unsupported Bytes-Unit: {kind!r}")
+        unparsedRanges = list(filter(None, map(bytes.strip, value.split(b","))))
+        parsedRanges = []
+        for byteRange in unparsedRanges:
+            try:
+                start, end = byteRange.split(b"-", 1)
+            except ValueError:
+                raise ValueError(f"Invalid Byte-Range: {byteRange!r}")
+            if start:
+                try:
+                    start = int(start)
+                except ValueError:
+                    raise ValueError(f"Invalid Byte-Range: {byteRange!r}")
+            else:
+                start = None
+            if end:
+                try:
+                    end = int(end)
+                except ValueError:
+                    raise ValueError(f"Invalid Byte-Range: {byteRange!r}")
+            else:
+                end = None
+            if start is not None:
+                if end is not None and start > end:
+                    # Start must be less than or equal to end or it is invalid.
+                    raise ValueError(f"Invalid Byte-Range: {byteRange!r}")
+            elif end is None:
+                # One or both of start and end must be specified.  Omitting
+                # both is invalid.
+                raise ValueError(f"Invalid Byte-Range: {byteRange!r}")
+            parsedRanges.append((start, end))
+        return parsedRanges
+
+    def _rangeToOffsetAndSize(self, start, end):
+        """
+        Convert a start and end from a Range header to an offset and size.
+
+        This method checks that the resulting range overlaps with the resource
+        being served (and so has the value of C{getFileSize()} as an indirect
+        input).
+
+        Either but not both of start or end can be L{None}:
+
+         - Omitted start means that the end value is actually a start value
+           relative to the end of the resource.
+
+         - Omitted end means the end of the resource should be the end of
+           the range.
+
+        End is interpreted as inclusive, as per RFC 2616.
+
+        If this range doesn't overlap with any of this resource, C{(0, 0)} is
+        returned, which is not otherwise a value return value.
+
+        @param start: The start value from the header, or L{None} if one was
+            not present.
+        @param end: The end value from the header, or L{None} if one was not
+            present.
+        @return: C{(offset, size)} where offset is how far into this resource
+            this resource the range begins and size is how long the range is,
+            or C{(0, 0)} if the range does not overlap this resource.
+        """
+        size = self.getFileSize()
+        if start is None:
+            start = size - end
+            end = size
+        elif end is None:
+            end = size
+        elif end < size:
+            end += 1
+        elif end > size:
+            end = size
+        if start >= size:
+            start = end = 0
+        return start, (end - start)
+
+    def _contentRange(self, offset, size):
+        """
+        Return a string suitable for the value of a Content-Range header for a
+        range with the given offset and size.
+
+        The offset and size are not sanity checked in any way.
+
+        @param offset: How far into this resource the range begins.
+        @param size: How long the range is.
+        @return: The value as appropriate for the value of a Content-Range
+            header.
+        """
+        return networkString(
+            "bytes %d-%d/%d" % (offset, offset + size - 1, self.getFileSize())
+        )
+
+    def _doSingleRangeRequest(self, request, startAndEnd):
+        """
+        Set up the response for Range headers that specify a single range.
+
+        This method checks if the request is satisfiable and sets the response
+        code and Content-Range header appropriately.  The return value
+        indicates which part of the resource to return.
+
+        @param request: The Request object.
+        @param startAndEnd: A 2-tuple of start of the byte range as specified by
+            the header and the end of the byte range as specified by the header.
+            At most one of the start and end may be L{None}.
+        @return: A 2-tuple of the offset and size of the range to return.
+            offset == size == 0 indicates that the request is not satisfiable.
+        """
+        start, end = startAndEnd
+        offset, size = self._rangeToOffsetAndSize(start, end)
+        if offset == size == 0:
+            # This range doesn't overlap with any of this resource, so the
+            # request is unsatisfiable.
+            request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
+            request.setHeader(
+                b"content-range", networkString("bytes */%d" % (self.getFileSize(),))
+            )
+        else:
+            request.setResponseCode(http.PARTIAL_CONTENT)
+            request.setHeader(b"content-range", self._contentRange(offset, size))
+        return offset, size
+
+    def _doMultipleRangeRequest(self, request, byteRanges):
+        """
+        Set up the response for Range headers that specify a single range.
+
+        This method checks if the request is satisfiable and sets the response
+        code and Content-Type and Content-Length headers appropriately.  The
+        return value, which is a little complicated, indicates which parts of
+        the resource to return and the boundaries that should separate the
+        parts.
+
+        In detail, the return value is a tuple rangeInfo C{rangeInfo} is a
+        list of 3-tuples C{(partSeparator, partOffset, partSize)}.  The
+        response to this request should be, for each element of C{rangeInfo},
+        C{partSeparator} followed by C{partSize} bytes of the resource
+        starting at C{partOffset}.  Each C{partSeparator} includes the
+        MIME-style boundary and the part-specific Content-type and
+        Content-range headers.  It is convenient to return the separator as a
+        concrete string from this method, because this method needs to compute
+        the number of bytes that will make up the response to be able to set
+        the Content-Length header of the response accurately.
+
+        @param request: The Request object.
+        @param byteRanges: A list of C{(start, end)} values as specified by
+            the header.  For each range, at most one of C{start} and C{end}
+            may be L{None}.
+        @return: See above.
+        """
+        matchingRangeFound = False
+        rangeInfo = []
+        contentLength = 0
+        boundary = networkString(f"{int(time.time() * 1000000):x}{os.getpid():x}")
+        if self.type:
+            contentType = self.type
+        else:
+            contentType = b"bytes"  # It's what Apache does...
+        for start, end in byteRanges:
+            partOffset, partSize = self._rangeToOffsetAndSize(start, end)
+            if partOffset == partSize == 0:
+                continue
+            contentLength += partSize
+            matchingRangeFound = True
+            partContentRange = self._contentRange(partOffset, partSize)
+            partSeparator = networkString(
+                (
+                    "\r\n"
+                    "--%s\r\n"
+                    "Content-type: %s\r\n"
+                    "Content-range: %s\r\n"
+                    "\r\n"
+                )
+                % (
+                    nativeString(boundary),
+                    nativeString(contentType),
+                    nativeString(partContentRange),
+                )
+            )
+            contentLength += len(partSeparator)
+            rangeInfo.append((partSeparator, partOffset, partSize))
+        if not matchingRangeFound:
+            request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
+            request.setHeader(b"content-length", b"0")
+            request.setHeader(
+                b"content-range", networkString("bytes */%d" % (self.getFileSize(),))
+            )
+            return [], b""
+        finalBoundary = b"\r\n--" + boundary + b"--\r\n"
+        rangeInfo.append((finalBoundary, 0, 0))
+        request.setResponseCode(http.PARTIAL_CONTENT)
+        request.setHeader(
+            b"content-type",
+            networkString(f'multipart/byteranges; boundary="{nativeString(boundary)}"'),
+        )
+        request.setHeader(
+            b"content-length", b"%d" % (contentLength + len(finalBoundary),)
+        )
+        return rangeInfo
+
+    def _setContentHeaders(self, request, size=None):
+        """
+        Set the Content-length and Content-type headers for this request.
+
+        This method is not appropriate for requests for multiple byte ranges;
+        L{_doMultipleRangeRequest} will set these headers in that case.
+
+        @param request: The L{twisted.web.http.Request} object.
+        @param size: The size of the response.  If not specified, default to
+            C{self.getFileSize()}.
+        """
+        if size is None:
+            size = self.getFileSize()
+        request.setHeader(b"content-length", b"%d" % (size,))
+        if self.type:
+            request.setHeader(b"content-type", networkString(self.type))
+        if self.encoding:
+            request.setHeader(b"content-encoding", networkString(self.encoding))
+
+    def makeProducer(self, request, fileForReading):
+        """
+        Make a L{StaticProducer} that will produce the body of this response.
+
+        This method will also set the response code and Content-* headers.
+
+        @param request: The L{twisted.web.http.Request} object.
+        @param fileForReading: The file object containing the resource.
+        @return: A L{StaticProducer}.  Calling C{.start()} on this will begin
+            producing the response.
+        """
+        byteRange = request.getHeader(b"range")
+        if byteRange is None:
+            self._setContentHeaders(request)
+            request.setResponseCode(http.OK)
+            return NoRangeStaticProducer(request, fileForReading)
+        try:
+            parsedRanges = self._parseRangeHeader(byteRange)
+        except ValueError:
+            log.msg(f"Ignoring malformed Range header {byteRange.decode()!r}")
+            self._setContentHeaders(request)
+            request.setResponseCode(http.OK)
+            return NoRangeStaticProducer(request, fileForReading)
+
+        if len(parsedRanges) == 1:
+            offset, size = self._doSingleRangeRequest(request, parsedRanges[0])
+            self._setContentHeaders(request, size)
+            return SingleRangeStaticProducer(request, fileForReading, offset, size)
+        else:
+            rangeInfo = self._doMultipleRangeRequest(request, parsedRanges)
+            return MultipleRangeStaticProducer(request, fileForReading, rangeInfo)
+
+    def render_GET(self, request):
+        """
+        Begin sending the contents of this L{File} (or a subset of the
+        contents, based on the 'range' header) to the given request.
+        """
+        self.restat(False)
+
+        if self.type is None:
+            self.type, self.encoding = getTypeAndEncoding(
+                self.basename(),
+                self.contentTypes,
+                self.contentEncodings,
+                self.defaultType,
+            )
+
+        if not self.exists():
+            return self.childNotFound.render(request)
+
+        if self.isdir():
+            return self.redirect(request)
+
+        request.setHeader(b"accept-ranges", b"bytes")
+
+        try:
+            fileForReading = self.openForReading()
+        except OSError as e:
+            if e.errno == errno.EACCES:
+                return self.forbidden.render(request)
+            else:
+                raise
+
+        if request.setLastModified(self.getModificationTime()) is http.CACHED:
+            # `setLastModified` also sets the response code for us, so if the
+            # request is cached, we close the file now that we've made sure that
+            # the request would otherwise succeed and return an empty body.
+            fileForReading.close()
+            return b""
+
+        if request.method == b"HEAD":
+            # Set the content headers here, rather than making a producer.
+            self._setContentHeaders(request)
+            # We've opened the file to make sure it's accessible, so close it
+            # now that we don't need it.
+            fileForReading.close()
+            return b""
+
+        producer = self.makeProducer(request, fileForReading)
+        producer.start()
+
+        # and make sure the connection doesn't get closed
+        return server.NOT_DONE_YET
+
+    render_HEAD = render_GET
+
+    def redirect(self, request):
+        return redirectTo(_addSlash(request), request)
+
+    def listNames(self):
+        if not self.isdir():
+            return []
+        directory = self.listdir()
+        directory.sort()
+        return directory
+
+    def listEntities(self):
+        return list(
+            map(
+                lambda fileName, self=self: self.createSimilarFile(
+                    os.path.join(self.path, fileName)
+                ),
+                self.listNames(),
+            )
+        )
+
+    def createSimilarFile(self, path):
+        f = self.__class__(path, self.defaultType, self.ignoredExts, self.registry)
+        # refactoring by steps, here - constructor should almost certainly take these
+        f.processors = self.processors
+        f.indexNames = self.indexNames[:]
+        f.childNotFound = self.childNotFound
+        return f
+
+
+@implementer(interfaces.IPullProducer)
+class StaticProducer:
+    """
+    Superclass for classes that implement the business of producing.
+
+    @ivar request: The L{IRequest} to write the contents of the file to.
+    @ivar fileObject: The file the contents of which to write to the request.
+    """
+
+    bufferSize = abstract.FileDescriptor.bufferSize
+
+    def __init__(self, request, fileObject):
+        """
+        Initialize the instance.
+        """
+        self.request = request
+        self.fileObject = fileObject
+
+    def start(self):
+        raise NotImplementedError(self.start)
+
+    def resumeProducing(self):
+        raise NotImplementedError(self.resumeProducing)
+
+    def stopProducing(self):
+        """
+        Stop producing data.
+
+        L{twisted.internet.interfaces.IProducer.stopProducing}
+        is called when our consumer has died, and subclasses also call this
+        method when they are done producing data.
+        """
+        self.fileObject.close()
+        self.request = None
+
+
+class NoRangeStaticProducer(StaticProducer):
+    """
+    A L{StaticProducer} that writes the entire file to the request.
+    """
+
+    def start(self):
+        self.request.registerProducer(self, False)
+
+    def resumeProducing(self):
+        if not self.request:
+            return
+        data = self.fileObject.read(self.bufferSize)
+        if data:
+            # this .write will spin the reactor, calling .doWrite and then
+            # .resumeProducing again, so be prepared for a re-entrant call
+            self.request.write(data)
+        else:
+            self.request.unregisterProducer()
+            self.request.finish()
+            self.stopProducing()
+
+
+class SingleRangeStaticProducer(StaticProducer):
+    """
+    A L{StaticProducer} that writes a single chunk of a file to the request.
+    """
+
+    def __init__(self, request, fileObject, offset, size):
+        """
+        Initialize the instance.
+
+        @param request: See L{StaticProducer}.
+        @param fileObject: See L{StaticProducer}.
+        @param offset: The offset into the file of the chunk to be written.
+        @param size: The size of the chunk to write.
+        """
+        StaticProducer.__init__(self, request, fileObject)
+        self.offset = offset
+        self.size = size
+
+    def start(self):
+        self.fileObject.seek(self.offset)
+        self.bytesWritten = 0
+        self.request.registerProducer(self, 0)
+
+    def resumeProducing(self):
+        if not self.request:
+            return
+        data = self.fileObject.read(min(self.bufferSize, self.size - self.bytesWritten))
+        if data:
+            self.bytesWritten += len(data)
+            # this .write will spin the reactor, calling .doWrite and then
+            # .resumeProducing again, so be prepared for a re-entrant call
+            self.request.write(data)
+        if self.request and self.bytesWritten == self.size:
+            self.request.unregisterProducer()
+            self.request.finish()
+            self.stopProducing()
+
+
+class MultipleRangeStaticProducer(StaticProducer):
+    """
+    A L{StaticProducer} that writes several chunks of a file to the request.
+    """
+
+    def __init__(self, request, fileObject, rangeInfo):
+        """
+        Initialize the instance.
+
+        @param request: See L{StaticProducer}.
+        @param fileObject: See L{StaticProducer}.
+        @param rangeInfo: A list of tuples C{[(boundary, offset, size)]}
+            where:
+             - C{boundary} will be written to the request first.
+             - C{offset} the offset into the file of chunk to write.
+             - C{size} the size of the chunk to write.
+        """
+        StaticProducer.__init__(self, request, fileObject)
+        self.rangeInfo = rangeInfo
+
+    def start(self):
+        self.rangeIter = iter(self.rangeInfo)
+        self._nextRange()
+        self.request.registerProducer(self, 0)
+
+    def _nextRange(self):
+        self.partBoundary, partOffset, self._partSize = next(self.rangeIter)
+        self._partBytesWritten = 0
+        self.fileObject.seek(partOffset)
+
+    def resumeProducing(self):
+        if not self.request:
+            return
+        data = []
+        dataLength = 0
+        done = False
+        while dataLength < self.bufferSize:
+            if self.partBoundary:
+                dataLength += len(self.partBoundary)
+                data.append(self.partBoundary)
+                self.partBoundary = None
+            p = self.fileObject.read(
+                min(
+                    self.bufferSize - dataLength,
+                    self._partSize - self._partBytesWritten,
+                )
+            )
+            self._partBytesWritten += len(p)
+            dataLength += len(p)
+            data.append(p)
+            if self.request and self._partBytesWritten == self._partSize:
+                try:
+                    self._nextRange()
+                except StopIteration:
+                    done = True
+                    break
+        self.request.write(b"".join(data))
+        if done:
+            self.request.unregisterProducer()
+            self.request.finish()
+            self.stopProducing()
+
+
+class ASISProcessor(resource.Resource):
+    """
+    Serve files exactly as responses without generating a status-line or any
+    headers.  Inspired by Apache's mod_asis.
+    """
+
+    def __init__(self, path, registry=None):
+        resource.Resource.__init__(self)
+        self.path = path
+        self.registry = registry or Registry()
+
+    def render(self, request):
+        request.startedWriting = 1
+        res = File(self.path, registry=self.registry)
+        return res.render(request)
+
+
+def formatFileSize(size):
+    """
+    Format the given file size in bytes to human readable format.
+    """
+    if size < 1024:
+        return "%iB" % size
+    elif size < (1024**2):
+        return "%iK" % (size / 1024)
+    elif size < (1024**3):
+        return "%iM" % (size / (1024**2))
+    else:
+        return "%iG" % (size / (1024**3))
+
+
+class DirectoryLister(resource.Resource):
+    """
+    Print the content of a directory.
+
+    @ivar template: page template used to render the content of the directory.
+        It must contain the format keys B{header} and B{tableContent}.
+    @type template: C{str}
+
+    @ivar linePattern: template used to render one line in the listing table.
+        It must contain the format keys B{class}, B{href}, B{text}, B{size},
+        B{type} and B{encoding}.
+    @type linePattern: C{str}
+
+    @ivar contentTypes: a mapping of extensions to MIME types used to populate
+        the information of a member of this directory.
+        It is initialized with the value L{File.contentTypes}.
+    @type contentTypes: C{dict}
+
+    @ivar contentEncodings: a mapping of extensions to encoding types.
+        It is initialized with the value L{File.contentEncodings}.
+    @type contentEncodings: C{dict}
+
+    @ivar defaultType: default type used when no mimetype is detected.
+    @type defaultType: C{str}
+
+    @ivar dirs: filtered content of C{path}, if the whole content should not be
+        displayed (default to L{None}, which means the actual content of
+        C{path} is printed).
+    @type dirs: L{None} or C{list}
+
+    @ivar path: directory which content should be listed.
+    @type path: C{str}
+    """
+
+    template = """<html>
+<head>
+<title>%(header)s</title>
+<style>
+.even-dir { background-color: #efe0ef }
+.even { background-color: #eee }
+.odd-dir {background-color: #f0d0ef }
+.odd { background-color: #dedede }
+.icon { text-align: center }
+.listing {
+    margin-left: auto;
+    margin-right: auto;
+    width: 50%%;
+    padding: 0.1em;
+    }
+
+body { border: 0; padding: 0; margin: 0; background-color: #efefef; }
+h1 {padding: 0.1em; background-color: #777; color: white; border-bottom: thin white dashed;}
+
+</style>
+</head>
+
+<body>
+<h1>%(header)s</h1>
+
+<table>
+    <thead>
+        <tr>
+            <th>Filename</th>
+            <th>Size</th>
+            <th>Content type</th>
+            <th>Content encoding</th>
+        </tr>
+    </thead>
+    <tbody>
+%(tableContent)s
+    </tbody>
+</table>
+
+</body>
+</html>
+"""
+
+    linePattern = """<tr class="%(class)s">
+    <td><a href="%(href)s">%(text)s</a></td>
+    <td>%(size)s</td>
+    <td>%(type)s</td>
+    <td>%(encoding)s</td>
+</tr>
+"""
+
+    def __init__(
+        self,
+        pathname,
+        dirs=None,
+        contentTypes=File.contentTypes,
+        contentEncodings=File.contentEncodings,
+        defaultType="text/html",
+    ):
+        resource.Resource.__init__(self)
+        self.contentTypes = contentTypes
+        self.contentEncodings = contentEncodings
+        self.defaultType = defaultType
+        # dirs allows usage of the File to specify what gets listed
+        self.dirs = dirs
+        self.path = pathname
+
+    def _getFilesAndDirectories(self, directory):
+        """
+        Helper returning files and directories in given directory listing, with
+        attributes to be used to build a table content with
+        C{self.linePattern}.
+
+        @return: tuple of (directories, files)
+        @rtype: C{tuple} of C{list}
+        """
+        files = []
+        dirs = []
+
+        for path in directory:
+            if isinstance(path, bytes):
+                path = path.decode("utf8")
+
+            url = quote(path, "/")
+            escapedPath = escape(path)
+            childPath = filepath.FilePath(self.path).child(path)
+
+            if childPath.isdir():
+                dirs.append(
+                    {
+                        "text": escapedPath + "/",
+                        "href": url + "/",
+                        "size": "",
+                        "type": "[Directory]",
+                        "encoding": "",
+                    }
+                )
+            else:
+                mimetype, encoding = getTypeAndEncoding(
+                    path, self.contentTypes, self.contentEncodings, self.defaultType
+                )
+                try:
+                    size = childPath.getsize()
+                except OSError:
+                    continue
+                files.append(
+                    {
+                        "text": escapedPath,
+                        "href": url,
+                        "type": "[%s]" % mimetype,
+                        "encoding": (encoding and "[%s]" % encoding or ""),
+                        "size": formatFileSize(size),
+                    }
+                )
+        return dirs, files
+
+    def _buildTableContent(self, elements):
+        """
+        Build a table content using C{self.linePattern} and giving elements odd
+        and even classes.
+        """
+        tableContent = []
+        rowClasses = itertools.cycle(["odd", "even"])
+        for element, rowClass in zip(elements, rowClasses):
+            element["class"] = rowClass
+            tableContent.append(self.linePattern % element)
+        return tableContent
+
+    def render(self, request):
+        """
+        Render a listing of the content of C{self.path}.
+        """
+        request.setHeader(b"content-type", b"text/html; charset=utf-8")
+        if self.dirs is None:
+            directory = os.listdir(self.path)
+            directory.sort()
+        else:
+            directory = self.dirs
+
+        dirs, files = self._getFilesAndDirectories(directory)
+
+        tableContent = "".join(self._buildTableContent(dirs + files))
+
+        header = "Directory listing for {}".format(
+            escape(unquote(nativeString(request.uri))),
+        )
+
+        done = self.template % {"header": header, "tableContent": tableContent}
+        done = done.encode("utf8")
+
+        return done
+
+    def __repr__(self) -> str:
+        return "<DirectoryLister of %r>" % self.path
+
+    __str__ = __repr__
diff --git a/contrib/python/Twisted/py3/twisted/web/sux.py b/contrib/python/Twisted/py3/twisted/web/sux.py
new file mode 100644
index 00000000000..69ad4dff951
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/sux.py
@@ -0,0 +1,644 @@
+# -*- test-case-name: twisted.web.test.test_xml -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+*S*mall, *U*ncomplicated *X*ML.
+
+This is a very simple implementation of XML/HTML as a network
+protocol.  It is not at all clever.  Its main features are that it
+does not:
+
+  - support namespaces
+  - mung mnemonic entity references
+  - validate
+  - perform *any* external actions (such as fetching URLs or writing files)
+    under *any* circumstances
+  - has lots and lots of horrible hacks for supporting broken HTML (as an
+    option, they're not on by default).
+"""
+
+
+from twisted.internet.protocol import Protocol
+from twisted.python.reflect import prefixedMethodNames
+
+# Elements of the three-tuples in the state table.
+BEGIN_HANDLER = 0
+DO_HANDLER = 1
+END_HANDLER = 2
+
+identChars = ".-_:"
+lenientIdentChars = identChars + ";+#/%~"
+
+
+def nop(*args, **kw):
+    "Do nothing."
+
+
+def unionlist(*args):
+    l = []
+    for x in args:
+        l.extend(x)
+    d = {x: 1 for x in l}
+    return d.keys()
+
+
+def zipfndict(*args, **kw):
+    default = kw.get("default", nop)
+    d = {}
+    for key in unionlist(*(fndict.keys() for fndict in args)):
+        d[key] = tuple(x.get(key, default) for x in args)
+    return d
+
+
+def prefixedMethodClassDict(clazz, prefix):
+    return {
+        name: getattr(clazz, prefix + name)
+        for name in prefixedMethodNames(clazz, prefix)
+    }
+
+
+def prefixedMethodObjDict(obj, prefix):
+    return {
+        name: getattr(obj, prefix + name)
+        for name in prefixedMethodNames(obj.__class__, prefix)
+    }
+
+
+class ParseError(Exception):
+    def __init__(self, filename, line, col, message):
+        self.filename = filename
+        self.line = line
+        self.col = col
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"{self.filename}:{self.line}:{self.col}: {self.message}"
+
+
+class XMLParser(Protocol):
+    state = None
+    encodings = None
+    filename = "<xml />"
+    beExtremelyLenient = 0
+    _prepend = None
+
+    # _leadingBodyData will sometimes be set before switching to the
+    # 'bodydata' state, when we "accidentally" read a byte of bodydata
+    # in a different state.
+    _leadingBodyData = None
+
+    def connectionMade(self):
+        self.lineno = 1
+        self.colno = 0
+        self.encodings = []
+
+    def saveMark(self):
+        """Get the line number and column of the last character parsed"""
+        # This gets replaced during dataReceived, restored afterwards
+        return (self.lineno, self.colno)
+
+    def _parseError(self, message):
+        raise ParseError(*((self.filename,) + self.saveMark() + (message,)))
+
+    def _buildStateTable(self):
+        """Return a dictionary of begin, do, end state function tuples"""
+        # _buildStateTable leaves something to be desired but it does what it
+        # does.. probably slowly, so I'm doing some evil caching so it doesn't
+        # get called more than once per class.
+        stateTable = getattr(self.__class__, "__stateTable", None)
+        if stateTable is None:
+            stateTable = self.__class__.__stateTable = zipfndict(
+                *(
+                    prefixedMethodObjDict(self, prefix)
+                    for prefix in ("begin_", "do_", "end_")
+                )
+            )
+        return stateTable
+
+    def _decode(self, data):
+        if "UTF-16" in self.encodings or "UCS-2" in self.encodings:
+            assert not len(data) & 1, "UTF-16 must come in pairs for now"
+        if self._prepend:
+            data = self._prepend + data
+        for encoding in self.encodings:
+            data = str(data, encoding)
+        return data
+
+    def maybeBodyData(self):
+        if self.endtag:
+            return "bodydata"
+
+        # Get ready for fun! We're going to allow
+        # <script>if (foo < bar)</script> to work!
+        # We do this by making everything between <script> and
+        # </script> a Text
+        # BUT <script src="foo"> will be special-cased to do regular,
+        # lenient behavior, because those may not have </script>
+        # -radix
+
+        if self.tagName == "script" and "src" not in self.tagAttributes:
+            # we do this ourselves rather than having begin_waitforendscript
+            # because that can get called multiple times and we don't want
+            # bodydata to get reset other than the first time.
+            self.begin_bodydata(None)
+            return "waitforendscript"
+        return "bodydata"
+
+    def dataReceived(self, data):
+        stateTable = self._buildStateTable()
+        if not self.state:
+            # all UTF-16 starts with this string
+            if data.startswith((b"\xff\xfe", b"\xfe\xff")):
+                self._prepend = data[0:2]
+                self.encodings.append("UTF-16")
+                data = data[2:]
+            self.state = "begin"
+        if self.encodings:
+            data = self._decode(data)
+        else:
+            data = data.decode("utf-8")
+        # bring state, lineno, colno into local scope
+        lineno, colno = self.lineno, self.colno
+        curState = self.state
+        # replace saveMark with a nested scope function
+        _saveMark = self.saveMark
+
+        def saveMark():
+            return (lineno, colno)
+
+        self.saveMark = saveMark
+        # fetch functions from the stateTable
+        beginFn, doFn, endFn = stateTable[curState]
+        try:
+            for byte in data:
+                # do newline stuff
+                if byte == "\n":
+                    lineno += 1
+                    colno = 0
+                else:
+                    colno += 1
+                newState = doFn(byte)
+                if newState is not None and newState != curState:
+                    # this is the endFn from the previous state
+                    endFn()
+                    curState = newState
+                    beginFn, doFn, endFn = stateTable[curState]
+                    beginFn(byte)
+        finally:
+            self.saveMark = _saveMark
+            self.lineno, self.colno = lineno, colno
+        # state doesn't make sense if there's an exception..
+        self.state = curState
+
+    def connectionLost(self, reason):
+        """
+        End the last state we were in.
+        """
+        stateTable = self._buildStateTable()
+        stateTable[self.state][END_HANDLER]()
+
+    # state methods
+
+    def do_begin(self, byte):
+        if byte.isspace():
+            return
+        if byte != "<":
+            if self.beExtremelyLenient:
+                self._leadingBodyData = byte
+                return "bodydata"
+            self._parseError(f"First char of document [{byte!r}] wasn't <")
+        return "tagstart"
+
+    def begin_comment(self, byte):
+        self.commentbuf = ""
+
+    def do_comment(self, byte):
+        self.commentbuf += byte
+        if self.commentbuf.endswith("-->"):
+            self.gotComment(self.commentbuf[:-3])
+            return "bodydata"
+
+    def begin_tagstart(self, byte):
+        self.tagName = ""  # name of the tag
+        self.tagAttributes = {}  # attributes of the tag
+        self.termtag = 0  # is the tag self-terminating
+        self.endtag = 0
+
+    def do_tagstart(self, byte):
+        if byte.isalnum() or byte in identChars:
+            self.tagName += byte
+            if self.tagName == "!--":
+                return "comment"
+        elif byte.isspace():
+            if self.tagName:
+                if self.endtag:
+                    # properly strict thing to do here is probably to only
+                    # accept whitespace
+                    return "waitforgt"
+                return "attrs"
+            else:
+                self._parseError("Whitespace before tag-name")
+        elif byte == ">":
+            if self.endtag:
+                self.gotTagEnd(self.tagName)
+                return "bodydata"
+            else:
+                self.gotTagStart(self.tagName, {})
+                return (
+                    (not self.beExtremelyLenient) and "bodydata" or self.maybeBodyData()
+                )
+        elif byte == "/":
+            if self.tagName:
+                return "afterslash"
+            else:
+                self.endtag = 1
+        elif byte in "!?":
+            if self.tagName:
+                if not self.beExtremelyLenient:
+                    self._parseError("Invalid character in tag-name")
+            else:
+                self.tagName += byte
+                self.termtag = 1
+        elif byte == "[":
+            if self.tagName == "!":
+                return "expectcdata"
+            else:
+                self._parseError("Invalid '[' in tag-name")
+        else:
+            if self.beExtremelyLenient:
+                self.bodydata = "<"
+                return "unentity"
+            self._parseError("Invalid tag character: %r" % byte)
+
+    def begin_unentity(self, byte):
+        self.bodydata += byte
+
+    def do_unentity(self, byte):
+        self.bodydata += byte
+        return "bodydata"
+
+    def end_unentity(self):
+        self.gotText(self.bodydata)
+
+    def begin_expectcdata(self, byte):
+        self.cdatabuf = byte
+
+    def do_expectcdata(self, byte):
+        self.cdatabuf += byte
+        cdb = self.cdatabuf
+        cd = "[CDATA["
+        if len(cd) > len(cdb):
+            if cd.startswith(cdb):
+                return
+            elif self.beExtremelyLenient:
+                ## WHAT THE CRAP!?  MSWord9 generates HTML that includes these
+                ## bizarre <![if !foo]> <![endif]> chunks, so I've gotta ignore
+                ## 'em as best I can.  this should really be a separate parse
+                ## state but I don't even have any idea what these _are_.
+                return "waitforgt"
+            else:
+                self._parseError("Mal-formed CDATA header")
+        if cd == cdb:
+            self.cdatabuf = ""
+            return "cdata"
+        self._parseError("Mal-formed CDATA header")
+
+    def do_cdata(self, byte):
+        self.cdatabuf += byte
+        if self.cdatabuf.endswith("]]>"):
+            self.cdatabuf = self.cdatabuf[:-3]
+            return "bodydata"
+
+    def end_cdata(self):
+        self.gotCData(self.cdatabuf)
+        self.cdatabuf = ""
+
+    def do_attrs(self, byte):
+        if byte.isalnum() or byte in identChars:
+            # XXX FIXME really handle !DOCTYPE at some point
+            if self.tagName == "!DOCTYPE":
+                return "doctype"
+            if self.tagName[0] in "!?":
+                return "waitforgt"
+            return "attrname"
+        elif byte.isspace():
+            return
+        elif byte == ">":
+            self.gotTagStart(self.tagName, self.tagAttributes)
+            return (not self.beExtremelyLenient) and "bodydata" or self.maybeBodyData()
+        elif byte == "/":
+            return "afterslash"
+        elif self.beExtremelyLenient:
+            # discard and move on?  Only case I've seen of this so far was:
+            # <foo bar="baz"">
+            return
+        self._parseError("Unexpected character: %r" % byte)
+
+    def begin_doctype(self, byte):
+        self.doctype = byte
+
+    def do_doctype(self, byte):
+        if byte == ">":
+            return "bodydata"
+        self.doctype += byte
+
+    def end_doctype(self):
+        self.gotDoctype(self.doctype)
+        self.doctype = None
+
+    def do_waitforgt(self, byte):
+        if byte == ">":
+            if self.endtag or not self.beExtremelyLenient:
+                return "bodydata"
+            return self.maybeBodyData()
+
+    def begin_attrname(self, byte):
+        self.attrname = byte
+        self._attrname_termtag = 0
+
+    def do_attrname(self, byte):
+        if byte.isalnum() or byte in identChars:
+            self.attrname += byte
+            return
+        elif byte == "=":
+            return "beforeattrval"
+        elif byte.isspace():
+            return "beforeeq"
+        elif self.beExtremelyLenient:
+            if byte in "\"'":
+                return "attrval"
+            if byte in lenientIdentChars or byte.isalnum():
+                self.attrname += byte
+                return
+            if byte == "/":
+                self._attrname_termtag = 1
+                return
+            if byte == ">":
+                self.attrval = "True"
+                self.tagAttributes[self.attrname] = self.attrval
+                self.gotTagStart(self.tagName, self.tagAttributes)
+                if self._attrname_termtag:
+                    self.gotTagEnd(self.tagName)
+                    return "bodydata"
+                return self.maybeBodyData()
+            # something is really broken. let's leave this attribute where it
+            # is and move on to the next thing
+            return
+        self._parseError(f"Invalid attribute name: {self.attrname!r} {byte!r}")
+
+    def do_beforeattrval(self, byte):
+        if byte in "\"'":
+            return "attrval"
+        elif byte.isspace():
+            return
+        elif self.beExtremelyLenient:
+            if byte in lenientIdentChars or byte.isalnum():
+                return "messyattr"
+            if byte == ">":
+                self.attrval = "True"
+                self.tagAttributes[self.attrname] = self.attrval
+                self.gotTagStart(self.tagName, self.tagAttributes)
+                return self.maybeBodyData()
+            if byte == "\\":
+                # I saw this in actual HTML once:
+                # <font size=\"3\"><sup>SM</sup></font>
+                return
+        self._parseError(
+            "Invalid initial attribute value: %r; Attribute values must be quoted."
+            % byte
+        )
+
+    attrname = ""
+    attrval = ""
+
+    def begin_beforeeq(self, byte):
+        self._beforeeq_termtag = 0
+
+    def do_beforeeq(self, byte):
+        if byte == "=":
+            return "beforeattrval"
+        elif byte.isspace():
+            return
+        elif self.beExtremelyLenient:
+            if byte.isalnum() or byte in identChars:
+                self.attrval = "True"
+                self.tagAttributes[self.attrname] = self.attrval
+                return "attrname"
+            elif byte == ">":
+                self.attrval = "True"
+                self.tagAttributes[self.attrname] = self.attrval
+                self.gotTagStart(self.tagName, self.tagAttributes)
+                if self._beforeeq_termtag:
+                    self.gotTagEnd(self.tagName)
+                    return "bodydata"
+                return self.maybeBodyData()
+            elif byte == "/":
+                self._beforeeq_termtag = 1
+                return
+        self._parseError("Invalid attribute")
+
+    def begin_attrval(self, byte):
+        self.quotetype = byte
+        self.attrval = ""
+
+    def do_attrval(self, byte):
+        if byte == self.quotetype:
+            return "attrs"
+        self.attrval += byte
+
+    def end_attrval(self):
+        self.tagAttributes[self.attrname] = self.attrval
+        self.attrname = self.attrval = ""
+
+    def begin_messyattr(self, byte):
+        self.attrval = byte
+
+    def do_messyattr(self, byte):
+        if byte.isspace():
+            return "attrs"
+        elif byte == ">":
+            endTag = 0
+            if self.attrval.endswith("/"):
+                endTag = 1
+                self.attrval = self.attrval[:-1]
+            self.tagAttributes[self.attrname] = self.attrval
+            self.gotTagStart(self.tagName, self.tagAttributes)
+            if endTag:
+                self.gotTagEnd(self.tagName)
+                return "bodydata"
+            return self.maybeBodyData()
+        else:
+            self.attrval += byte
+
+    def end_messyattr(self):
+        if self.attrval:
+            self.tagAttributes[self.attrname] = self.attrval
+
+    def begin_afterslash(self, byte):
+        self._after_slash_closed = 0
+
+    def do_afterslash(self, byte):
+        # this state is only after a self-terminating slash, e.g. <foo/>
+        if self._after_slash_closed:
+            self._parseError("Mal-formed")  # XXX When does this happen??
+        if byte != ">":
+            if self.beExtremelyLenient:
+                return
+            else:
+                self._parseError("No data allowed after '/'")
+        self._after_slash_closed = 1
+        self.gotTagStart(self.tagName, self.tagAttributes)
+        self.gotTagEnd(self.tagName)
+        # don't need maybeBodyData here because there better not be
+        # any javascript code after a <script/>... we'll see :(
+        return "bodydata"
+
+    def begin_bodydata(self, byte):
+        if self._leadingBodyData:
+            self.bodydata = self._leadingBodyData
+            del self._leadingBodyData
+        else:
+            self.bodydata = ""
+
+    def do_bodydata(self, byte):
+        if byte == "<":
+            return "tagstart"
+        if byte == "&":
+            return "entityref"
+        self.bodydata += byte
+
+    def end_bodydata(self):
+        self.gotText(self.bodydata)
+        self.bodydata = ""
+
+    def do_waitforendscript(self, byte):
+        if byte == "<":
+            return "waitscriptendtag"
+        self.bodydata += byte
+
+    def begin_waitscriptendtag(self, byte):
+        self.temptagdata = ""
+        self.tagName = ""
+        self.endtag = 0
+
+    def do_waitscriptendtag(self, byte):
+        # 1 enforce / as first byte read
+        # 2 enforce following bytes to be subset of "script" until
+        #   tagName == "script"
+        #   2a when that happens, gotText(self.bodydata) and gotTagEnd(self.tagName)
+        # 3 spaces can happen anywhere, they're ignored
+        #   e.g. < / script >
+        # 4 anything else causes all data I've read to be moved to the
+        #   bodydata, and switch back to waitforendscript state
+
+        # If it turns out this _isn't_ a </script>, we need to
+        # remember all the data we've been through so we can append it
+        # to bodydata
+        self.temptagdata += byte
+
+        # 1
+        if byte == "/":
+            self.endtag = True
+        elif not self.endtag:
+            self.bodydata += "<" + self.temptagdata
+            return "waitforendscript"
+        # 2
+        elif byte.isalnum() or byte in identChars:
+            self.tagName += byte
+            if not "script".startswith(self.tagName):
+                self.bodydata += "<" + self.temptagdata
+                return "waitforendscript"
+            elif self.tagName == "script":
+                self.gotText(self.bodydata)
+                self.gotTagEnd(self.tagName)
+                return "waitforgt"
+        # 3
+        elif byte.isspace():
+            return "waitscriptendtag"
+        # 4
+        else:
+            self.bodydata += "<" + self.temptagdata
+            return "waitforendscript"
+
+    def begin_entityref(self, byte):
+        self.erefbuf = ""
+        self.erefextra = ""  # extra bit for lenient mode
+
+    def do_entityref(self, byte):
+        if byte.isspace() or byte == "<":
+            if self.beExtremelyLenient:
+                # '&foo' probably was '&amp;foo'
+                if self.erefbuf and self.erefbuf != "amp":
+                    self.erefextra = self.erefbuf
+                self.erefbuf = "amp"
+                if byte == "<":
+                    return "tagstart"
+                else:
+                    self.erefextra += byte
+                    return "spacebodydata"
+            self._parseError("Bad entity reference")
+        elif byte != ";":
+            self.erefbuf += byte
+        else:
+            return "bodydata"
+
+    def end_entityref(self):
+        self.gotEntityReference(self.erefbuf)
+
+    # hacky support for space after & in entityref in beExtremelyLenient
+    # state should only happen in that case
+    def begin_spacebodydata(self, byte):
+        self.bodydata = self.erefextra
+        self.erefextra = None
+
+    do_spacebodydata = do_bodydata
+    end_spacebodydata = end_bodydata
+
+    # Sorta SAX-ish API
+
+    def gotTagStart(self, name, attributes):
+        """Encountered an opening tag.
+
+        Default behaviour is to print."""
+        print("begin", name, attributes)
+
+    def gotText(self, data):
+        """Encountered text
+
+        Default behaviour is to print."""
+        print("text:", repr(data))
+
+    def gotEntityReference(self, entityRef):
+        """Encountered mnemonic entity reference
+
+        Default behaviour is to print."""
+        print("entityRef: &%s;" % entityRef)
+
+    def gotComment(self, comment):
+        """Encountered comment.
+
+        Default behaviour is to ignore."""
+        pass
+
+    def gotCData(self, cdata):
+        """Encountered CDATA
+
+        Default behaviour is to call the gotText method"""
+        self.gotText(cdata)
+
+    def gotDoctype(self, doctype):
+        """Encountered DOCTYPE
+
+        This is really grotty: it basically just gives you everything between
+        '<!DOCTYPE' and '>' as an argument.
+        """
+        print("!DOCTYPE", repr(doctype))
+
+    def gotTagEnd(self, name):
+        """Encountered closing tag
+
+        Default behaviour is to print."""
+        print("end", name)
diff --git a/contrib/python/Twisted/py3/twisted/web/tap.py b/contrib/python/Twisted/py3/twisted/web/tap.py
new file mode 100644
index 00000000000..2ed783848a4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/tap.py
@@ -0,0 +1,322 @@
+# -*- test-case-name: twisted.web.test.test_tap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Support for creating a service which runs a web server.
+"""
+
+
+import os
+import warnings
+
+import incremental
+
+from twisted.application import service, strports
+from twisted.internet import interfaces, reactor
+from twisted.python import deprecate, reflect, threadpool, usage
+from twisted.spread import pb
+from twisted.web import demo, distrib, resource, script, server, static, twcgi, wsgi
+
+
+class Options(usage.Options):
+    """
+    Define the options accepted by the I{twistd web} plugin.
+    """
+
+    synopsis = "[web options]"
+
+    optParameters = [
+        ["logfile", "l", None, "Path to web CLF (Combined Log Format) log file."],
+        [
+            "certificate",
+            "c",
+            "server.pem",
+            "(DEPRECATED: use --listen) " "SSL certificate to use for HTTPS. ",
+        ],
+        [
+            "privkey",
+            "k",
+            "server.pem",
+            "(DEPRECATED: use --listen) " "SSL certificate to use for HTTPS.",
+        ],
+    ]
+
+    optFlags = [
+        [
+            "notracebacks",
+            "n",
+            (
+                "(DEPRECATED: Tracebacks are disabled by default. "
+                "See --enable-tracebacks to turn them on."
+            ),
+        ],
+        [
+            "display-tracebacks",
+            "",
+            (
+                "Show uncaught exceptions during rendering tracebacks to "
+                "the client. WARNING: This may be a security risk and "
+                "expose private data!"
+            ),
+        ],
+    ]
+
+    optFlags.append(
+        [
+            "personal",
+            "",
+            "Instead of generating a webserver, generate a "
+            "ResourcePublisher which listens on  the port given by "
+            "--listen, or ~/%s " % (distrib.UserDirectory.userSocketName,)
+            + "if --listen is not specified.",
+        ]
+    )
+
+    compData = usage.Completions(
+        optActions={
+            "logfile": usage.CompleteFiles("*.log"),
+            "certificate": usage.CompleteFiles("*.pem"),
+            "privkey": usage.CompleteFiles("*.pem"),
+        }
+    )
+
+    longdesc = """\
+This starts a webserver.  If you specify no arguments, it will be a
+demo webserver that has the Test class from twisted.web.demo in it."""
+
+    def __init__(self):
+        usage.Options.__init__(self)
+        self["indexes"] = []
+        self["root"] = None
+        self["extraHeaders"] = []
+        self["ports"] = []
+        self["port"] = self["https"] = None
+
+    def opt_port(self, port):
+        """
+        (DEPRECATED: use --listen)
+        Strports description of port to start the server on
+        """
+        msg = deprecate.getDeprecationWarningString(
+            self.opt_port, incremental.Version("Twisted", 18, 4, 0)
+        )
+        warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+        self["port"] = port
+
+    opt_p = opt_port
+
+    def opt_https(self, port):
+        """
+        (DEPRECATED: use --listen)
+        Port to listen on for Secure HTTP.
+        """
+        msg = deprecate.getDeprecationWarningString(
+            self.opt_https, incremental.Version("Twisted", 18, 4, 0)
+        )
+        warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+        self["https"] = port
+
+    def opt_listen(self, port):
+        """
+        Add an strports description of port to start the server on.
+        [default: tcp:8080]
+        """
+        self["ports"].append(port)
+
+    def opt_index(self, indexName):
+        """
+        Add the name of a file used to check for directory indexes.
+        [default: index, index.html]
+        """
+        self["indexes"].append(indexName)
+
+    opt_i = opt_index
+
+    def opt_user(self):
+        """
+        Makes a server with ~/public_html and ~/.twistd-web-pb support for
+        users.
+        """
+        self["root"] = distrib.UserDirectory()
+
+    opt_u = opt_user
+
+    def opt_path(self, path):
+        """
+        <path> is either a specific file or a directory to be set as the root
+        of the web server. Use this if you have a directory full of HTML, cgi,
+        epy, or rpy files or any other files that you want to be served up raw.
+        """
+        self["root"] = static.File(os.path.abspath(path))
+        self["root"].processors = {
+            ".epy": script.PythonScript,
+            ".rpy": script.ResourceScript,
+        }
+        self["root"].processors[".cgi"] = twcgi.CGIScript
+
+    def opt_processor(self, proc):
+        """
+        `ext=class' where `class' is added as a Processor for files ending
+        with `ext'.
+        """
+        if not isinstance(self["root"], static.File):
+            raise usage.UsageError("You can only use --processor after --path.")
+        ext, klass = proc.split("=", 1)
+        self["root"].processors[ext] = reflect.namedClass(klass)
+
+    def opt_class(self, className):
+        """
+        Create a Resource subclass with a zero-argument constructor.
+        """
+        classObj = reflect.namedClass(className)
+        self["root"] = classObj()
+
+    def opt_resource_script(self, name):
+        """
+        An .rpy file to be used as the root resource of the webserver.
+        """
+        self["root"] = script.ResourceScriptWrapper(name)
+
+    def opt_wsgi(self, name):
+        """
+        The FQPN of a WSGI application object to serve as the root resource of
+        the webserver.
+        """
+        try:
+            application = reflect.namedAny(name)
+        except (AttributeError, ValueError):
+            raise usage.UsageError(f"No such WSGI application: {name!r}")
+        pool = threadpool.ThreadPool()
+        reactor.callWhenRunning(pool.start)
+        reactor.addSystemEventTrigger("after", "shutdown", pool.stop)
+        self["root"] = wsgi.WSGIResource(reactor, pool, application)
+
+    def opt_mime_type(self, defaultType):
+        """
+        Specify the default mime-type for static files.
+        """
+        if not isinstance(self["root"], static.File):
+            raise usage.UsageError("You can only use --mime_type after --path.")
+        self["root"].defaultType = defaultType
+
+    opt_m = opt_mime_type
+
+    def opt_allow_ignore_ext(self):
+        """
+        Specify whether or not a request for 'foo' should return 'foo.ext'
+        """
+        if not isinstance(self["root"], static.File):
+            raise usage.UsageError(
+                "You can only use --allow_ignore_ext " "after --path."
+            )
+        self["root"].ignoreExt("*")
+
+    def opt_ignore_ext(self, ext):
+        """
+        Specify an extension to ignore.  These will be processed in order.
+        """
+        if not isinstance(self["root"], static.File):
+            raise usage.UsageError("You can only use --ignore_ext " "after --path.")
+        self["root"].ignoreExt(ext)
+
+    def opt_add_header(self, header):
+        """
+        Specify an additional header to be included in all responses. Specified
+        as "HeaderName: HeaderValue".
+        """
+        name, value = header.split(":", 1)
+        self["extraHeaders"].append((name.strip(), value.strip()))
+
+    def postOptions(self):
+        """
+        Set up conditional defaults and check for dependencies.
+
+        If SSL is not available but an HTTPS server was configured, raise a
+        L{UsageError} indicating that this is not possible.
+
+        If no server port was supplied, select a default appropriate for the
+        other options supplied.
+        """
+        if self["port"] is not None:
+            self["ports"].append(self["port"])
+        if self["https"] is not None:
+            try:
+                reflect.namedModule("OpenSSL.SSL")
+            except ImportError:
+                raise usage.UsageError("SSL support not installed")
+            sslStrport = "ssl:port={}:privateKey={}:certKey={}".format(
+                self["https"],
+                self["privkey"],
+                self["certificate"],
+            )
+            self["ports"].append(sslStrport)
+        if len(self["ports"]) == 0:
+            if self["personal"]:
+                path = os.path.expanduser(
+                    os.path.join("~", distrib.UserDirectory.userSocketName)
+                )
+                self["ports"].append("unix:" + path)
+            else:
+                self["ports"].append("tcp:8080")
+
+
+def makePersonalServerFactory(site):
+    """
+    Create and return a factory which will respond to I{distrib} requests
+    against the given site.
+
+    @type site: L{twisted.web.server.Site}
+    @rtype: L{twisted.internet.protocol.Factory}
+    """
+    return pb.PBServerFactory(distrib.ResourcePublisher(site))
+
+
+class _AddHeadersResource(resource.Resource):
+    def __init__(self, originalResource, headers):
+        self._originalResource = originalResource
+        self._headers = headers
+
+    def getChildWithDefault(self, name, request):
+        for k, v in self._headers:
+            request.responseHeaders.addRawHeader(k, v)
+        return self._originalResource.getChildWithDefault(name, request)
+
+
+def makeService(config):
+    s = service.MultiService()
+    if config["root"]:
+        root = config["root"]
+        if config["indexes"]:
+            config["root"].indexNames = config["indexes"]
+    else:
+        # This really ought to be web.Admin or something
+        root = demo.Test()
+
+    if isinstance(root, static.File):
+        root.registry.setComponent(interfaces.IServiceCollection, s)
+
+    if config["extraHeaders"]:
+        root = _AddHeadersResource(root, config["extraHeaders"])
+
+    if config["logfile"]:
+        site = server.Site(root, logPath=config["logfile"])
+    else:
+        site = server.Site(root)
+
+    if config["display-tracebacks"]:
+        site.displayTracebacks = True
+
+    # Deprecate --notracebacks/-n
+    if config["notracebacks"]:
+        msg = deprecate._getDeprecationWarningString(
+            "--notracebacks", incremental.Version("Twisted", 19, 7, 0)
+        )
+        warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+
+    if config["personal"]:
+        site = makePersonalServerFactory(site)
+    for port in config["ports"]:
+        svc = strports.service(port, site)
+        svc.setServiceParent(s)
+    return s
diff --git a/contrib/python/Twisted/py3/twisted/web/template.py b/contrib/python/Twisted/py3/twisted/web/template.py
new file mode 100644
index 00000000000..162dc7d2069
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/template.py
@@ -0,0 +1,60 @@
+# -*- test-case-name: twisted.web.test.test_template -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+HTML rendering for twisted.web.
+
+@var VALID_HTML_TAG_NAMES: A list of recognized HTML tag names, used by the
+    L{tag} object.
+
+@var TEMPLATE_NAMESPACE: The XML namespace used to identify attributes and
+    elements used by the templating system, which should be removed from the
+    final output document.
+
+@var tags: A convenience object which can produce L{Tag} objects on demand via
+    attribute access.  For example: C{tags.div} is equivalent to C{Tag("div")}.
+    Tags not specified in L{VALID_HTML_TAG_NAMES} will result in an
+    L{AttributeError}.
+"""
+
+
+__all__ = [
+    "TEMPLATE_NAMESPACE",
+    "VALID_HTML_TAG_NAMES",
+    "Element",
+    "Flattenable",
+    "TagLoader",
+    "XMLString",
+    "XMLFile",
+    "renderer",
+    "flatten",
+    "flattenString",
+    "tags",
+    "Comment",
+    "CDATA",
+    "Tag",
+    "slot",
+    "CharRef",
+    "renderElement",
+]
+
+from ._stan import CharRef
+from ._template_util import (
+    CDATA,
+    TEMPLATE_NAMESPACE,
+    VALID_HTML_TAG_NAMES,
+    Comment,
+    Element,
+    Flattenable,
+    Tag,
+    TagLoader,
+    XMLFile,
+    XMLString,
+    flatten,
+    flattenString,
+    renderElement,
+    renderer,
+    slot,
+    tags,
+)
diff --git a/contrib/python/Twisted/py3/twisted/web/test/requesthelper.py b/contrib/python/Twisted/py3/twisted/web/test/requesthelper.py
new file mode 100644
index 00000000000..a3b0904427e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/test/requesthelper.py
@@ -0,0 +1,512 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Helpers related to HTTP requests, used by tests.
+"""
+
+from __future__ import annotations
+
+__all__ = ["DummyChannel", "DummyRequest"]
+
+from io import BytesIO
+from typing import Dict, List, Optional
+
+from zope.interface import implementer, verify
+
+from incremental import Version
+
+from twisted.internet.address import IPv4Address, IPv6Address
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IAddress, ISSLTransport
+from twisted.internet.task import Clock
+from twisted.python.deprecate import deprecated
+from twisted.trial import unittest
+from twisted.web._responses import FOUND
+from twisted.web.http_headers import Headers
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET, Session, Site
+
+textLinearWhitespaceComponents = [f"Foo{lw}bar" for lw in ["\r", "\n", "\r\n"]]
+
+sanitizedText = "Foo bar"
+bytesLinearWhitespaceComponents = [
+    component.encode("ascii") for component in textLinearWhitespaceComponents
+]
+sanitizedBytes = sanitizedText.encode("ascii")
+
+
+@implementer(IAddress)
+class NullAddress:
+    """
+    A null implementation of L{IAddress}.
+    """
+
+
+class DummyChannel:
+    class TCP:
+        port = 80
+        disconnected = False
+
+        def __init__(self, peer=None):
+            if peer is None:
+                peer = IPv4Address("TCP", "192.168.1.1", 12344)
+            self._peer = peer
+            self.written = BytesIO()
+            self.producers = []
+
+        def getPeer(self):
+            return self._peer
+
+        def write(self, data):
+            if not isinstance(data, bytes):
+                raise TypeError(f"Can only write bytes to a transport, not {data!r}")
+            self.written.write(data)
+
+        def writeSequence(self, iovec):
+            for data in iovec:
+                self.write(data)
+
+        def getHost(self):
+            return IPv4Address("TCP", "10.0.0.1", self.port)
+
+        def registerProducer(self, producer, streaming):
+            self.producers.append((producer, streaming))
+
+        def unregisterProducer(self):
+            pass
+
+        def loseConnection(self):
+            self.disconnected = True
+
+    @implementer(ISSLTransport)
+    class SSL(TCP):
+        def abortConnection(self):
+            # ITCPTransport.abortConnection
+            pass
+
+        def getTcpKeepAlive(self):
+            # ITCPTransport.getTcpKeepAlive
+            pass
+
+        def getTcpNoDelay(self):
+            # ITCPTransport.getTcpNoDelay
+            pass
+
+        def loseWriteConnection(self):
+            # ITCPTransport.loseWriteConnection
+            pass
+
+        def setTcpKeepAlive(self, enabled):
+            # ITCPTransport.setTcpKeepAlive
+            pass
+
+        def setTcpNoDelay(self, enabled):
+            # ITCPTransport.setTcpNoDelay
+            pass
+
+        def getPeerCertificate(self):
+            # ISSLTransport.getPeerCertificate
+            pass
+
+    site = Site(Resource())
+
+    def __init__(self, peer=None):
+        self.transport = self.TCP(peer)
+
+    def requestDone(self, request):
+        pass
+
+    def writeHeaders(self, version, code, reason, headers):
+        response_line = version + b" " + code + b" " + reason + b"\r\n"
+        headerSequence = [response_line]
+        headerSequence.extend(name + b": " + value + b"\r\n" for name, value in headers)
+        headerSequence.append(b"\r\n")
+        self.transport.writeSequence(headerSequence)
+
+    def getPeer(self):
+        return self.transport.getPeer()
+
+    def getHost(self):
+        return self.transport.getHost()
+
+    def registerProducer(self, producer, streaming):
+        self.transport.registerProducer(producer, streaming)
+
+    def unregisterProducer(self):
+        self.transport.unregisterProducer()
+
+    def write(self, data):
+        self.transport.write(data)
+
+    def writeSequence(self, iovec):
+        self.transport.writeSequence(iovec)
+
+    def loseConnection(self):
+        self.transport.loseConnection()
+
+    def endRequest(self):
+        pass
+
+    def isSecure(self):
+        return isinstance(self.transport, self.SSL)
+
+    def abortConnection(self):
+        # ITCPTransport.abortConnection
+        pass
+
+    def getTcpKeepAlive(self):
+        # ITCPTransport.getTcpKeepAlive
+        pass
+
+    def getTcpNoDelay(self):
+        # ITCPTransport.getTcpNoDelay
+        pass
+
+    def loseWriteConnection(self):
+        # ITCPTransport.loseWriteConnection
+        pass
+
+    def setTcpKeepAlive(self):
+        # ITCPTransport.setTcpKeepAlive
+        pass
+
+    def setTcpNoDelay(self):
+        # ITCPTransport.setTcpNoDelay
+        pass
+
+    def getPeerCertificate(self):
+        # ISSLTransport.getPeerCertificate
+        pass
+
+
+class DummyRequest:
+    """
+    Represents a dummy or fake request. See L{twisted.web.server.Request}.
+
+    @ivar _finishedDeferreds: L{None} or a C{list} of L{Deferreds} which will
+        be called back with L{None} when C{finish} is called or which will be
+        errbacked if C{processingFailed} is called.
+
+    @type requestheaders: C{Headers}
+    @ivar requestheaders: A Headers instance that stores values for all request
+        headers.
+
+    @type responseHeaders: C{Headers}
+    @ivar responseHeaders: A Headers instance that stores values for all
+        response headers.
+
+    @type responseCode: C{int}
+    @ivar responseCode: The response code which was passed to
+        C{setResponseCode}.
+
+    @type written: C{list} of C{bytes}
+    @ivar written: The bytes which have been written to the request.
+    """
+
+    uri = b"http://dummy/"
+    method = b"GET"
+    client: Optional[IAddress] = None
+    sitepath: List[bytes]
+    written: List[bytes]
+    prepath: List[bytes]
+    args: Dict[bytes, List[bytes]]
+    _finishedDeferreds: List[Deferred[None]]
+
+    def registerProducer(self, prod, s):
+        """
+        Call an L{IPullProducer}'s C{resumeProducing} method in a
+        loop until it unregisters itself.
+
+        @param prod: The producer.
+        @type prod: L{IPullProducer}
+
+        @param s: Whether or not the producer is streaming.
+        """
+        # XXX: Handle IPushProducers
+        self.go = 1
+        while self.go:
+            prod.resumeProducing()
+
+    def unregisterProducer(self):
+        self.go = 0
+
+    def __init__(
+        self,
+        postpath: list[bytes],
+        session: Optional[Session] = None,
+        client: Optional[IAddress] = None,
+    ) -> None:
+        self.sitepath = []
+        self.written = []
+        self.finished = 0
+        self.postpath = postpath
+        self.prepath = []
+        self.session = None
+        self.protoSession = session or Session(site=None, uid=b"0", reactor=Clock())
+        self.args = {}
+        self.requestHeaders = Headers()
+        self.responseHeaders = Headers()
+        self.responseCode = None
+        self._finishedDeferreds = []
+        self._serverName = b"dummy"
+        self.clientproto = b"HTTP/1.0"
+
+    def getAllHeaders(self):
+        """
+        Return dictionary mapping the names of all received headers to the last
+        value received for each.
+
+        Since this method does not return all header information,
+        C{self.requestHeaders.getAllRawHeaders()} may be preferred.
+
+        NOTE: This function is a direct copy of
+        C{twisted.web.http.Request.getAllRawHeaders}.
+        """
+        headers = {}
+        for k, v in self.requestHeaders.getAllRawHeaders():
+            headers[k.lower()] = v[-1]
+        return headers
+
+    def getHeader(self, name):
+        """
+        Retrieve the value of a request header.
+
+        @type name: C{bytes}
+        @param name: The name of the request header for which to retrieve the
+            value.  Header names are compared case-insensitively.
+
+        @rtype: C{bytes} or L{None}
+        @return: The value of the specified request header.
+        """
+        return self.requestHeaders.getRawHeaders(name.lower(), [None])[0]
+
+    def setHeader(self, name, value):
+        """TODO: make this assert on write() if the header is content-length"""
+        self.responseHeaders.addRawHeader(name, value)
+
+    def getSession(self, sessionInterface=None):
+        if self.session:
+            return self.session
+        assert (
+            not self.written
+        ), "Session cannot be requested after data has been written."
+        self.session = self.protoSession
+        return self.session
+
+    def render(self, resource):
+        """
+        Render the given resource as a response to this request.
+
+        This implementation only handles a few of the most common behaviors of
+        resources.  It can handle a render method that returns a string or
+        C{NOT_DONE_YET}.  It doesn't know anything about the semantics of
+        request methods (eg HEAD) nor how to set any particular headers.
+        Basically, it's largely broken, but sufficient for some tests at least.
+        It should B{not} be expanded to do all the same stuff L{Request} does.
+        Instead, L{DummyRequest} should be phased out and L{Request} (or some
+        other real code factored in a different way) used.
+        """
+        result = resource.render(self)
+        if result is NOT_DONE_YET:
+            return
+        self.write(result)
+        self.finish()
+
+    def write(self, data):
+        if not isinstance(data, bytes):
+            raise TypeError("write() only accepts bytes")
+        self.written.append(data)
+
+    def notifyFinish(self) -> Deferred[None]:
+        """
+        Return a L{Deferred} which is called back with L{None} when the request
+        is finished.  This will probably only work if you haven't called
+        C{finish} yet.
+        """
+        finished: Deferred[None] = Deferred()
+        self._finishedDeferreds.append(finished)
+        return finished
+
+    def finish(self):
+        """
+        Record that the request is finished and callback and L{Deferred}s
+        waiting for notification of this.
+        """
+        self.finished = self.finished + 1
+        if self._finishedDeferreds is not None:
+            observers = self._finishedDeferreds
+            self._finishedDeferreds = None
+            for obs in observers:
+                obs.callback(None)
+
+    def processingFailed(self, reason):
+        """
+        Errback and L{Deferreds} waiting for finish notification.
+        """
+        if self._finishedDeferreds is not None:
+            observers = self._finishedDeferreds
+            self._finishedDeferreds = None
+            for obs in observers:
+                obs.errback(reason)
+
+    def addArg(self, name, value):
+        self.args[name] = [value]
+
+    def setResponseCode(self, code, message=None):
+        """
+        Set the HTTP status response code, but takes care that this is called
+        before any data is written.
+        """
+        assert (
+            not self.written
+        ), "Response code cannot be set after data has" "been written: {}.".format(
+            "@@@@".join(self.written)
+        )
+        self.responseCode = code
+        self.responseMessage = message
+
+    def setLastModified(self, when):
+        assert (
+            not self.written
+        ), "Last-Modified cannot be set after data has " "been written: {}.".format(
+            "@@@@".join(self.written)
+        )
+
+    def setETag(self, tag):
+        assert (
+            not self.written
+        ), "ETag cannot be set after data has been " "written: {}.".format(
+            "@@@@".join(self.written)
+        )
+
+    @deprecated(Version("Twisted", 18, 4, 0), replacement="getClientAddress")
+    def getClientIP(self):
+        """
+        Return the IPv4 address of the client which made this request, if there
+        is one, otherwise L{None}.
+        """
+        if isinstance(self.client, (IPv4Address, IPv6Address)):
+            return self.client.host
+        return None
+
+    def getClientAddress(self):
+        """
+        Return the L{IAddress} of the client that made this request.
+
+        @return: an address.
+        @rtype: an L{IAddress} provider.
+        """
+        if self.client is None:
+            return NullAddress()
+        return self.client
+
+    def getRequestHostname(self):
+        """
+        Get a dummy hostname associated to the HTTP request.
+
+        @rtype: C{bytes}
+        @returns: a dummy hostname
+        """
+        return self._serverName
+
+    def getHost(self):
+        """
+        Get a dummy transport's host.
+
+        @rtype: C{IPv4Address}
+        @returns: a dummy transport's host
+        """
+        return IPv4Address("TCP", "127.0.0.1", 80)
+
+    def setHost(self, host, port, ssl=0):
+        """
+        Change the host and port the request thinks it's using.
+
+        @type host: C{bytes}
+        @param host: The value to which to change the host header.
+
+        @type ssl: C{bool}
+        @param ssl: A flag which, if C{True}, indicates that the request is
+            considered secure (if C{True}, L{isSecure} will return C{True}).
+        """
+        self._forceSSL = ssl  # set first so isSecure will work
+        if self.isSecure():
+            default = 443
+        else:
+            default = 80
+        if port == default:
+            hostHeader = host
+        else:
+            hostHeader = b"%b:%d" % (host, port)
+        self.requestHeaders.addRawHeader(b"host", hostHeader)
+
+    def redirect(self, url):
+        """
+        Utility function that does a redirect.
+
+        The request should have finish() called after this.
+        """
+        self.setResponseCode(FOUND)
+        self.setHeader(b"location", url)
+
+
+class DummyRequestTests(unittest.SynchronousTestCase):
+    """
+    Tests for L{DummyRequest}.
+    """
+
+    def test_getClientIPDeprecated(self):
+        """
+        L{DummyRequest.getClientIP} is deprecated in favor of
+        L{DummyRequest.getClientAddress}
+        """
+
+        request = DummyRequest([])
+        request.getClientIP()
+
+        warnings = self.flushWarnings(
+            offendingFunctions=[self.test_getClientIPDeprecated]
+        )
+
+        self.assertEqual(1, len(warnings))
+        [warning] = warnings
+        self.assertEqual(warning.get("category"), DeprecationWarning)
+        self.assertEqual(
+            warning.get("message"),
+            (
+                "twisted.web.test.requesthelper.DummyRequest.getClientIP "
+                "was deprecated in Twisted 18.4.0; "
+                "please use getClientAddress instead"
+            ),
+        )
+
+    def test_getClientIPSupportsIPv6(self):
+        """
+        L{DummyRequest.getClientIP} supports IPv6 addresses, just like
+        L{twisted.web.http.Request.getClientIP}.
+        """
+        request = DummyRequest([])
+        client = IPv6Address("TCP", "::1", 12345)
+        request.client = client
+
+        self.assertEqual("::1", request.getClientIP())
+
+    def test_getClientAddressWithoutClient(self):
+        """
+        L{DummyRequest.getClientAddress} returns an L{IAddress}
+        provider no C{client} has been set.
+        """
+        request = DummyRequest([])
+        null = request.getClientAddress()
+        verify.verifyObject(IAddress, null)
+
+    def test_getClientAddress(self):
+        """
+        L{DummyRequest.getClientAddress} returns the C{client}.
+        """
+        request = DummyRequest([])
+        client = IPv4Address("TCP", "127.0.0.1", 12345)
+        request.client = client
+        address = request.getClientAddress()
+        self.assertIs(address, client)
diff --git a/contrib/python/Twisted/py3/twisted/web/twcgi.py b/contrib/python/Twisted/py3/twisted/web/twcgi.py
new file mode 100644
index 00000000000..fcf831108b4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/twcgi.py
@@ -0,0 +1,343 @@
+# -*- test-case-name: twisted.web.test.test_cgi -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+I hold resource classes and helper classes that deal with CGI scripts.
+"""
+
+# System Imports
+import os
+import urllib
+from typing import AnyStr
+
+# Twisted Imports
+from twisted.internet import protocol
+from twisted.logger import Logger
+from twisted.python import filepath
+from twisted.spread import pb
+from twisted.web import http, resource, server, static
+
+
+class CGIDirectory(resource.Resource, filepath.FilePath[AnyStr]):
+    def __init__(self, pathname):
+        resource.Resource.__init__(self)
+        filepath.FilePath.__init__(self, pathname)
+
+    def getChild(self, path, request):
+        fnp = self.child(path)
+        if not fnp.exists():
+            return static.File.childNotFound
+        elif fnp.isdir():
+            return CGIDirectory(fnp.path)
+        else:
+            return CGIScript(fnp.path)
+
+    def render(self, request):
+        notFound = resource.NoResource(
+            "CGI directories do not support directory listing."
+        )
+        return notFound.render(request)
+
+
+class CGIScript(resource.Resource):
+    """
+    L{CGIScript} is a resource which runs child processes according to the CGI
+    specification.
+
+    The implementation is complex due to the fact that it requires asynchronous
+    IPC with an external process with an unpleasant protocol.
+    """
+
+    isLeaf = 1
+
+    def __init__(self, filename, registry=None, reactor=None):
+        """
+        Initialize, with the name of a CGI script file.
+        """
+        self.filename = filename
+        if reactor is None:
+            # This installs a default reactor, if None was installed before.
+            # We do a late import here, so that importing the current module
+            # won't directly trigger installing a default reactor.
+            from twisted.internet import reactor
+        self._reactor = reactor
+
+    def render(self, request):
+        """
+        Do various things to conform to the CGI specification.
+
+        I will set up the usual slew of environment variables, then spin off a
+        process.
+
+        @type request: L{twisted.web.http.Request}
+        @param request: An HTTP request.
+        """
+        scriptName = b"/" + b"/".join(request.prepath)
+        serverName = request.getRequestHostname().split(b":")[0]
+        env = {
+            "SERVER_SOFTWARE": server.version,
+            "SERVER_NAME": serverName,
+            "GATEWAY_INTERFACE": "CGI/1.1",
+            "SERVER_PROTOCOL": request.clientproto,
+            "SERVER_PORT": str(request.getHost().port),
+            "REQUEST_METHOD": request.method,
+            "SCRIPT_NAME": scriptName,
+            "SCRIPT_FILENAME": self.filename,
+            "REQUEST_URI": request.uri,
+        }
+
+        ip = request.getClientAddress().host
+        if ip is not None:
+            env["REMOTE_ADDR"] = ip
+        pp = request.postpath
+        if pp:
+            env["PATH_INFO"] = "/" + "/".join(pp)
+
+        if hasattr(request, "content"):
+            # 'request.content' is either a StringIO or a TemporaryFile, and
+            # the file pointer is sitting at the beginning (seek(0,0))
+            request.content.seek(0, 2)
+            length = request.content.tell()
+            request.content.seek(0, 0)
+            env["CONTENT_LENGTH"] = str(length)
+
+        try:
+            qindex = request.uri.index(b"?")
+        except ValueError:
+            env["QUERY_STRING"] = ""
+            qargs = []
+        else:
+            qs = env["QUERY_STRING"] = request.uri[qindex + 1 :]
+            if b"=" in qs:
+                qargs = []
+            else:
+                qargs = [urllib.parse.unquote(x.decode()) for x in qs.split(b"+")]
+
+        # Propagate HTTP headers
+        for title, header in request.getAllHeaders().items():
+            envname = title.replace(b"-", b"_").upper()
+            if title not in (b"content-type", b"content-length", b"proxy"):
+                envname = b"HTTP_" + envname
+            env[envname] = header
+        # Propagate our environment
+        for key, value in os.environ.items():
+            if key not in env:
+                env[key] = value
+        # And they're off!
+        self.runProcess(env, request, qargs)
+        return server.NOT_DONE_YET
+
+    def runProcess(self, env, request, qargs=[]):
+        """
+        Run the cgi script.
+
+        @type env: A L{dict} of L{str}, or L{None}
+        @param env: The environment variables to pass to the process that will
+            get spawned. See
+            L{twisted.internet.interfaces.IReactorProcess.spawnProcess} for
+            more information about environments and process creation.
+
+        @type request: L{twisted.web.http.Request}
+        @param request: An HTTP request.
+
+        @type qargs: A L{list} of L{str}
+        @param qargs: The command line arguments to pass to the process that
+            will get spawned.
+        """
+        p = CGIProcessProtocol(request)
+        self._reactor.spawnProcess(
+            p,
+            self.filename,
+            [self.filename] + qargs,
+            env,
+            os.path.dirname(self.filename),
+        )
+
+
+class FilteredScript(CGIScript):
+    """
+    I am a special version of a CGI script, that uses a specific executable.
+
+    This is useful for interfacing with other scripting languages that adhere
+    to the CGI standard. My C{filter} attribute specifies what executable to
+    run, and my C{filename} init parameter describes which script to pass to
+    the first argument of that script.
+
+    To customize me for a particular location of a CGI interpreter, override
+    C{filter}.
+
+    @type filter: L{str}
+    @ivar filter: The absolute path to the executable.
+    """
+
+    filter = "/usr/bin/cat"
+
+    def runProcess(self, env, request, qargs=[]):
+        """
+        Run a script through the C{filter} executable.
+
+        @type env: A L{dict} of L{str}, or L{None}
+        @param env: The environment variables to pass to the process that will
+            get spawned. See
+            L{twisted.internet.interfaces.IReactorProcess.spawnProcess}
+            for more information about environments and process creation.
+
+        @type request: L{twisted.web.http.Request}
+        @param request: An HTTP request.
+
+        @type qargs: A L{list} of L{str}
+        @param qargs: The command line arguments to pass to the process that
+            will get spawned.
+        """
+        p = CGIProcessProtocol(request)
+        self._reactor.spawnProcess(
+            p,
+            self.filter,
+            [self.filter, self.filename] + qargs,
+            env,
+            os.path.dirname(self.filename),
+        )
+
+
+class CGIProcessProtocol(protocol.ProcessProtocol, pb.Viewable):
+    handling_headers = 1
+    headers_written = 0
+    headertext = b""
+    errortext = b""
+    _log = Logger()
+    _requestFinished = False
+
+    # Remotely relay producer interface.
+
+    def view_resumeProducing(self, issuer):
+        self.resumeProducing()
+
+    def view_pauseProducing(self, issuer):
+        self.pauseProducing()
+
+    def view_stopProducing(self, issuer):
+        self.stopProducing()
+
+    def resumeProducing(self):
+        self.transport.resumeProducing()
+
+    def pauseProducing(self):
+        self.transport.pauseProducing()
+
+    def stopProducing(self):
+        self.transport.loseConnection()
+
+    def __init__(self, request):
+        self.request = request
+        self.request.notifyFinish().addBoth(self._finished)
+
+    def connectionMade(self):
+        self.request.registerProducer(self, 1)
+        self.request.content.seek(0, 0)
+        content = self.request.content.read()
+        if content:
+            self.transport.write(content)
+        self.transport.closeStdin()
+
+    def errReceived(self, error):
+        self.errortext = self.errortext + error
+
+    def outReceived(self, output):
+        """
+        Handle a chunk of input
+        """
+        # First, make sure that the headers from the script are sorted
+        # out (we'll want to do some parsing on these later.)
+        if self.handling_headers:
+            text = self.headertext + output
+            headerEnds = []
+            for delimiter in b"\n\n", b"\r\n\r\n", b"\r\r", b"\n\r\n":
+                headerend = text.find(delimiter)
+                if headerend != -1:
+                    headerEnds.append((headerend, delimiter))
+            if headerEnds:
+                # The script is entirely in control of response headers;
+                # disable the default Content-Type value normally provided by
+                # twisted.web.server.Request.
+                self.request.defaultContentType = None
+
+                headerEnds.sort()
+                headerend, delimiter = headerEnds[0]
+                self.headertext = text[:headerend]
+                # This is a final version of the header text.
+                linebreak = delimiter[: len(delimiter) // 2]
+                headers = self.headertext.split(linebreak)
+                for header in headers:
+                    br = header.find(b": ")
+                    if br == -1:
+                        self._log.error(
+                            "ignoring malformed CGI header: {header!r}", header=header
+                        )
+                    else:
+                        headerName = header[:br].lower()
+                        headerText = header[br + 2 :]
+                        if headerName == b"location":
+                            self.request.setResponseCode(http.FOUND)
+                        if headerName == b"status":
+                            try:
+                                # "XXX <description>" sometimes happens.
+                                statusNum = int(headerText[:3])
+                            except BaseException:
+                                self._log.error("malformed status header")
+                            else:
+                                self.request.setResponseCode(statusNum)
+                        else:
+                            # Don't allow the application to control
+                            # these required headers.
+                            if headerName.lower() not in (b"server", b"date"):
+                                self.request.responseHeaders.addRawHeader(
+                                    headerName, headerText
+                                )
+                output = text[headerend + len(delimiter) :]
+                self.handling_headers = 0
+            if self.handling_headers:
+                self.headertext = text
+        if not self.handling_headers:
+            self.request.write(output)
+
+    def processEnded(self, reason):
+        if reason.value.exitCode != 0:
+            self._log.error(
+                "CGI {uri} exited with exit code {exitCode}",
+                uri=self.request.uri,
+                exitCode=reason.value.exitCode,
+            )
+        if self.errortext:
+            self._log.error(
+                "Errors from CGI {uri}: {errorText}",
+                uri=self.request.uri,
+                errorText=self.errortext,
+            )
+
+        if self.handling_headers:
+            self._log.error(
+                "Premature end of headers in {uri}: {headerText}",
+                uri=self.request.uri,
+                headerText=self.headertext,
+            )
+            if not self._requestFinished:
+                self.request.write(
+                    resource.ErrorPage(
+                        http.INTERNAL_SERVER_ERROR,
+                        "CGI Script Error",
+                        "Premature end of script headers.",
+                    ).render(self.request)
+                )
+
+        if not self._requestFinished:
+            self.request.unregisterProducer()
+            self.request.finish()
+
+    def _finished(self, ignored):
+        """
+        Record the end of the response generation for the request being
+        serviced.
+        """
+        self._requestFinished = True
diff --git a/contrib/python/Twisted/py3/twisted/web/util.py b/contrib/python/Twisted/py3/twisted/web/util.py
new file mode 100644
index 00000000000..3135f05cd96
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/util.py
@@ -0,0 +1,38 @@
+# -*- test-case-name: twisted.web.test.test_util -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An assortment of web server-related utilities.
+"""
+
+__all__ = [
+    "redirectTo",
+    "Redirect",
+    "ChildRedirector",
+    "ParentRedirect",
+    "DeferredResource",
+    "FailureElement",
+    "formatFailure",
+    # publicized by unit tests:
+    "_FrameElement",
+    "_SourceFragmentElement",
+    "_SourceLineElement",
+    "_StackElement",
+    "_PRE",
+]
+
+from ._template_util import (
+    _PRE,
+    ChildRedirector,
+    DeferredResource,
+    FailureElement,
+    ParentRedirect,
+    Redirect,
+    _FrameElement,
+    _SourceFragmentElement,
+    _SourceLineElement,
+    _StackElement,
+    formatFailure,
+    redirectTo,
+)
diff --git a/contrib/python/Twisted/py3/twisted/web/vhost.py b/contrib/python/Twisted/py3/twisted/web/vhost.py
new file mode 100644
index 00000000000..9576252b0f2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/vhost.py
@@ -0,0 +1,137 @@
+# -*- test-case-name: twisted.web.
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+I am a virtual hosts implementation.
+"""
+
+
+# Twisted Imports
+from twisted.python import roots
+from twisted.web import pages, resource
+
+
+class VirtualHostCollection(roots.Homogenous):
+    """Wrapper for virtual hosts collection.
+
+    This exists for configuration purposes.
+    """
+
+    entityType = resource.Resource
+
+    def __init__(self, nvh):
+        self.nvh = nvh
+
+    def listStaticEntities(self):
+        return self.nvh.hosts.items()
+
+    def getStaticEntity(self, name):
+        return self.nvh.hosts.get(self)
+
+    def reallyPutEntity(self, name, entity):
+        self.nvh.addHost(name, entity)
+
+    def delEntity(self, name):
+        self.nvh.removeHost(name)
+
+
+class NameVirtualHost(resource.Resource):
+    """I am a resource which represents named virtual hosts."""
+
+    default = None
+
+    def __init__(self):
+        """Initialize."""
+        resource.Resource.__init__(self)
+        self.hosts = {}
+
+    def listStaticEntities(self):
+        return resource.Resource.listStaticEntities(self) + [
+            ("Virtual Hosts", VirtualHostCollection(self))
+        ]
+
+    def getStaticEntity(self, name):
+        if name == "Virtual Hosts":
+            return VirtualHostCollection(self)
+        else:
+            return resource.Resource.getStaticEntity(self, name)
+
+    def addHost(self, name, resrc):
+        """Add a host to this virtual host.
+
+        This will take a host named `name', and map it to a resource
+        `resrc'.  For example, a setup for our virtual hosts would be::
+
+            nvh.addHost('divunal.com', divunalDirectory)
+            nvh.addHost('www.divunal.com', divunalDirectory)
+            nvh.addHost('twistedmatrix.com', twistedMatrixDirectory)
+            nvh.addHost('www.twistedmatrix.com', twistedMatrixDirectory)
+        """
+        self.hosts[name] = resrc
+
+    def removeHost(self, name):
+        """Remove a host."""
+        del self.hosts[name]
+
+    def _getResourceForRequest(self, request):
+        """(Internal) Get the appropriate resource for the given host."""
+        hostHeader = request.getHeader(b"host")
+        if hostHeader is None:
+            return self.default or pages.notFound()
+        else:
+            host = hostHeader.lower().split(b":", 1)[0]
+        return self.hosts.get(host, self.default) or pages.notFound(
+            "Not Found",
+            f"host {host.decode('ascii', 'replace')!r} not in vhost map",
+        )
+
+    def render(self, request):
+        """Implementation of resource.Resource's render method."""
+        resrc = self._getResourceForRequest(request)
+        return resrc.render(request)
+
+    def getChild(self, path, request):
+        """Implementation of resource.Resource's getChild method."""
+        resrc = self._getResourceForRequest(request)
+        if resrc.isLeaf:
+            request.postpath.insert(0, request.prepath.pop(-1))
+            return resrc
+        else:
+            return resrc.getChildWithDefault(path, request)
+
+
+class _HostResource(resource.Resource):
+    def getChild(self, path, request):
+        if b":" in path:
+            host, port = path.split(b":", 1)
+            port = int(port)
+        else:
+            host, port = path, 80
+        request.setHost(host, port)
+        prefixLen = 3 + request.isSecure() + 4 + len(path) + len(request.prepath[-3])
+        request.path = b"/" + b"/".join(request.postpath)
+        request.uri = request.uri[prefixLen:]
+        del request.prepath[:3]
+        return request.site.getResourceFor(request)
+
+
+class VHostMonsterResource(resource.Resource):
+
+    """
+    Use this to be able to record the hostname and method (http vs. https)
+    in the URL without disturbing your web site. If you put this resource
+    in a URL http://foo.com/bar then requests to
+    http://foo.com/bar/http/baz.com/something will be equivalent to
+    http://foo.com/something, except that the hostname the request will
+    appear to be accessing will be "baz.com". So if "baz.com" is redirecting
+    all requests for to foo.com, while foo.com is inaccessible from the outside,
+    then redirect and url generation will work correctly
+    """
+
+    def getChild(self, path, request):
+        if path == b"http":
+            request.isSecure = lambda: 0
+        elif path == b"https":
+            request.isSecure = lambda: 1
+        return _HostResource()
diff --git a/contrib/python/Twisted/py3/twisted/web/wsgi.py b/contrib/python/Twisted/py3/twisted/web/wsgi.py
new file mode 100644
index 00000000000..43227f40e32
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/wsgi.py
@@ -0,0 +1,589 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+An implementation of
+U{Python Web Server Gateway Interface v1.0.1<http://www.python.org/dev/peps/pep-3333/>}.
+"""
+
+from collections.abc import Sequence
+from sys import exc_info
+from warnings import warn
+
+from zope.interface import implementer
+
+from twisted.internet.threads import blockingCallFromThread
+from twisted.logger import Logger
+from twisted.python.failure import Failure
+from twisted.web.http import INTERNAL_SERVER_ERROR
+from twisted.web.resource import IResource
+from twisted.web.server import NOT_DONE_YET
+
+# PEP-3333 -- which has superseded PEP-333 -- states that, in both Python 2
+# and Python 3, text strings MUST be represented using the platform's native
+# string type, limited to characters defined in ISO-8859-1. Byte strings are
+# used only for values read from wsgi.input, passed to write() or yielded by
+# the application.
+#
+# Put another way:
+#
+# - In Python 2, all text strings and binary data are of type str/bytes and
+#   NEVER of type unicode. Whether the strings contain binary data or
+#   ISO-8859-1 text depends on context.
+#
+# - In Python 3, all text strings are of type str, and all binary data are of
+#   type bytes. Text MUST always be limited to that which can be encoded as
+#   ISO-8859-1, U+0000 to U+00FF inclusive.
+#
+# The following pair of functions -- _wsgiString() and _wsgiStringToBytes() --
+# are used to make Twisted's WSGI support compliant with the standard.
+if str is bytes:
+
+    def _wsgiString(string):  # Python 2.
+        """
+        Convert C{string} to an ISO-8859-1 byte string, if it is not already.
+
+        @type string: C{str}/C{bytes} or C{unicode}
+        @rtype: C{str}/C{bytes}
+
+        @raise UnicodeEncodeError: If C{string} contains non-ISO-8859-1 chars.
+        """
+        if isinstance(string, str):
+            return string
+        else:
+            return string.encode("iso-8859-1")
+
+    def _wsgiStringToBytes(string):  # Python 2.
+        """
+        Return C{string} as is; a WSGI string is a byte string in Python 2.
+
+        @type string: C{str}/C{bytes}
+        @rtype: C{str}/C{bytes}
+        """
+        return string
+
+else:
+
+    def _wsgiString(string):  # Python 3.
+        """
+        Convert C{string} to a WSGI "bytes-as-unicode" string.
+
+        If it's a byte string, decode as ISO-8859-1. If it's a Unicode string,
+        round-trip it to bytes and back using ISO-8859-1 as the encoding.
+
+        @type string: C{str} or C{bytes}
+        @rtype: C{str}
+
+        @raise UnicodeEncodeError: If C{string} contains non-ISO-8859-1 chars.
+        """
+        if isinstance(string, str):
+            return string.encode("iso-8859-1").decode("iso-8859-1")
+        else:
+            return string.decode("iso-8859-1")
+
+    def _wsgiStringToBytes(string):  # Python 3.
+        """
+        Convert C{string} from a WSGI "bytes-as-unicode" string to an
+        ISO-8859-1 byte string.
+
+        @type string: C{str}
+        @rtype: C{bytes}
+
+        @raise UnicodeEncodeError: If C{string} contains non-ISO-8859-1 chars.
+        """
+        return string.encode("iso-8859-1")
+
+
+class _ErrorStream:
+    """
+    File-like object instances of which are used as the value for the
+    C{'wsgi.errors'} key in the C{environ} dictionary passed to the application
+    object.
+
+    This simply passes writes on to L{logging<twisted.logger>} system as
+    error events from the C{'wsgi'} system.  In the future, it may be desirable
+    to expose more information in the events it logs, such as the application
+    object which generated the message.
+    """
+
+    _log = Logger()
+
+    def write(self, data):
+        """
+        Generate an event for the logging system with the given bytes as the
+        message.
+
+        This is called in a WSGI application thread, not the I/O thread.
+
+        @type data: str
+
+        @raise TypeError: On Python 3, if C{data} is not a native string. On
+            Python 2 a warning will be issued.
+        """
+        if not isinstance(data, str):
+            if str is bytes:
+                warn(
+                    "write() argument should be str, not %r (%s)"
+                    % (data, type(data).__name__),
+                    category=UnicodeWarning,
+                )
+            else:
+                raise TypeError(
+                    "write() argument must be str, not %r (%s)"
+                    % (data, type(data).__name__)
+                )
+
+        # Note that in old style, message was a tuple.  logger._legacy
+        # will overwrite this value if it is not properly formatted here.
+        self._log.error(data, system="wsgi", isError=True, message=(data,))
+
+    def writelines(self, iovec):
+        """
+        Join the given lines and pass them to C{write} to be handled in the
+        usual way.
+
+        This is called in a WSGI application thread, not the I/O thread.
+
+        @param iovec: A C{list} of C{'\\n'}-terminated C{str} which will be
+            logged.
+
+        @raise TypeError: On Python 3, if C{iovec} contains any non-native
+            strings. On Python 2 a warning will be issued.
+        """
+        self.write("".join(iovec))
+
+    def flush(self):
+        """
+        Nothing is buffered, so flushing does nothing.  This method is required
+        to exist by PEP 333, though.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+
+
+class _InputStream:
+    """
+    File-like object instances of which are used as the value for the
+    C{'wsgi.input'} key in the C{environ} dictionary passed to the application
+    object.
+
+    This only exists to make the handling of C{readline(-1)} consistent across
+    different possible underlying file-like object implementations.  The other
+    supported methods pass through directly to the wrapped object.
+    """
+
+    def __init__(self, input):
+        """
+        Initialize the instance.
+
+        This is called in the I/O thread, not a WSGI application thread.
+        """
+        self._wrapped = input
+
+    def read(self, size=None):
+        """
+        Pass through to the underlying C{read}.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+        # Avoid passing None because cStringIO and file don't like it.
+        if size is None:
+            return self._wrapped.read()
+        return self._wrapped.read(size)
+
+    def readline(self, size=None):
+        """
+        Pass through to the underlying C{readline}, with a size of C{-1} replaced
+        with a size of L{None}.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+        # Check for -1 because StringIO doesn't handle it correctly.  Check for
+        # None because files and tempfiles don't accept that.
+        if size == -1 or size is None:
+            return self._wrapped.readline()
+        return self._wrapped.readline(size)
+
+    def readlines(self, size=None):
+        """
+        Pass through to the underlying C{readlines}.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+        # Avoid passing None because cStringIO and file don't like it.
+        if size is None:
+            return self._wrapped.readlines()
+        return self._wrapped.readlines(size)
+
+    def __iter__(self):
+        """
+        Pass through to the underlying C{__iter__}.
+
+        This is called in a WSGI application thread, not the I/O thread.
+        """
+        return iter(self._wrapped)
+
+
+class _WSGIResponse:
+    """
+    Helper for L{WSGIResource} which drives the WSGI application using a
+    threadpool and hooks it up to the L{http.Request}.
+
+    @ivar started: A L{bool} indicating whether or not the response status and
+        headers have been written to the request yet.  This may only be read or
+        written in the WSGI application thread.
+
+    @ivar reactor: An L{IReactorThreads} provider which is used to call methods
+        on the request in the I/O thread.
+
+    @ivar threadpool: A L{ThreadPool} which is used to call the WSGI
+        application object in a non-I/O thread.
+
+    @ivar application: The WSGI application object.
+
+    @ivar request: The L{http.Request} upon which the WSGI environment is
+        based and to which the application's output will be sent.
+
+    @ivar environ: The WSGI environment L{dict}.
+
+    @ivar status: The HTTP response status L{str} supplied to the WSGI
+        I{start_response} callable by the application.
+
+    @ivar headers: A list of HTTP response headers supplied to the WSGI
+        I{start_response} callable by the application.
+
+    @ivar _requestFinished: A flag which indicates whether it is possible to
+        generate more response data or not.  This is L{False} until
+        L{http.Request.notifyFinish} tells us the request is done,
+        then L{True}.
+    """
+
+    _requestFinished = False
+    _log = Logger()
+
+    def __init__(self, reactor, threadpool, application, request):
+        self.started = False
+        self.reactor = reactor
+        self.threadpool = threadpool
+        self.application = application
+        self.request = request
+        self.request.notifyFinish().addBoth(self._finished)
+
+        if request.prepath:
+            scriptName = b"/" + b"/".join(request.prepath)
+        else:
+            scriptName = b""
+
+        if request.postpath:
+            pathInfo = b"/" + b"/".join(request.postpath)
+        else:
+            pathInfo = b""
+
+        parts = request.uri.split(b"?", 1)
+        if len(parts) == 1:
+            queryString = b""
+        else:
+            queryString = parts[1]
+
+        # All keys and values need to be native strings, i.e. of type str in
+        # *both* Python 2 and Python 3, so says PEP-3333.
+        self.environ = {
+            "REQUEST_METHOD": _wsgiString(request.method),
+            "REMOTE_ADDR": _wsgiString(request.getClientAddress().host),
+            "SCRIPT_NAME": _wsgiString(scriptName),
+            "PATH_INFO": _wsgiString(pathInfo),
+            "QUERY_STRING": _wsgiString(queryString),
+            "CONTENT_TYPE": _wsgiString(request.getHeader(b"content-type") or ""),
+            "CONTENT_LENGTH": _wsgiString(request.getHeader(b"content-length") or ""),
+            "SERVER_NAME": _wsgiString(request.getRequestHostname()),
+            "SERVER_PORT": _wsgiString(str(request.getHost().port)),
+            "SERVER_PROTOCOL": _wsgiString(request.clientproto),
+        }
+
+        # The application object is entirely in control of response headers;
+        # disable the default Content-Type value normally provided by
+        # twisted.web.server.Request.
+        self.request.defaultContentType = None
+
+        for name, values in request.requestHeaders.getAllRawHeaders():
+            name = "HTTP_" + _wsgiString(name).upper().replace("-", "_")
+            # It might be preferable for http.HTTPChannel to clear out
+            # newlines.
+            self.environ[name] = ",".join(_wsgiString(v) for v in values).replace(
+                "\n", " "
+            )
+
+        self.environ.update(
+            {
+                "wsgi.version": (1, 0),
+                "wsgi.url_scheme": request.isSecure() and "https" or "http",
+                "wsgi.run_once": False,
+                "wsgi.multithread": True,
+                "wsgi.multiprocess": False,
+                "wsgi.errors": _ErrorStream(),
+                # Attend: request.content was owned by the I/O thread up until
+                # this point.  By wrapping it and putting the result into the
+                # environment dictionary, it is effectively being given to
+                # another thread.  This means that whatever it is, it has to be
+                # safe to access it from two different threads.  The access
+                # *should* all be serialized (first the I/O thread writes to
+                # it, then the WSGI thread reads from it, then the I/O thread
+                # closes it).  However, since the request is made available to
+                # arbitrary application code during resource traversal, it's
+                # possible that some other code might decide to use it in the
+                # I/O thread concurrently with its use in the WSGI thread.
+                # More likely than not, this will break.  This seems like an
+                # unlikely possibility to me, but if it is to be allowed,
+                # something here needs to change. -exarkun
+                "wsgi.input": _InputStream(request.content),
+            }
+        )
+
+    def _finished(self, ignored):
+        """
+        Record the end of the response generation for the request being
+        serviced.
+        """
+        self._requestFinished = True
+
+    def startResponse(self, status, headers, excInfo=None):
+        """
+        The WSGI I{start_response} callable.  The given values are saved until
+        they are needed to generate the response.
+
+        This will be called in a non-I/O thread.
+        """
+        if self.started and excInfo is not None:
+            raise excInfo[1].with_traceback(excInfo[2])
+
+        # PEP-3333 mandates that status should be a native string. In practice
+        # this is mandated by Twisted's HTTP implementation too, so we enforce
+        # on both Python 2 and Python 3.
+        if not isinstance(status, str):
+            raise TypeError(
+                "status must be str, not {!r} ({})".format(
+                    status, type(status).__name__
+                )
+            )
+
+        # PEP-3333 mandates that headers should be a plain list, but in
+        # practice we work with any sequence type and only warn when it's not
+        # a plain list.
+        if isinstance(headers, list):
+            pass  # This is okay.
+        elif isinstance(headers, Sequence):
+            warn(
+                "headers should be a list, not %r (%s)"
+                % (headers, type(headers).__name__),
+                category=RuntimeWarning,
+            )
+        else:
+            raise TypeError(
+                "headers must be a list, not %r (%s)"
+                % (headers, type(headers).__name__)
+            )
+
+        # PEP-3333 mandates that each header should be a (str, str) tuple, but
+        # in practice we work with any sequence type and only warn when it's
+        # not a plain list.
+        for header in headers:
+            if isinstance(header, tuple):
+                pass  # This is okay.
+            elif isinstance(header, Sequence):
+                warn(
+                    "header should be a (str, str) tuple, not %r (%s)"
+                    % (header, type(header).__name__),
+                    category=RuntimeWarning,
+                )
+            else:
+                raise TypeError(
+                    "header must be a (str, str) tuple, not %r (%s)"
+                    % (header, type(header).__name__)
+                )
+
+            # However, the sequence MUST contain only 2 elements.
+            if len(header) != 2:
+                raise TypeError(f"header must be a (str, str) tuple, not {header!r}")
+
+            # Both elements MUST be native strings. Non-native strings will be
+            # rejected by the underlying HTTP machinery in any case, but we
+            # reject them here in order to provide a more informative error.
+            for elem in header:
+                if not isinstance(elem, str):
+                    raise TypeError(f"header must be (str, str) tuple, not {header!r}")
+
+        self.status = status
+        self.headers = headers
+        return self.write
+
+    def write(self, data):
+        """
+        The WSGI I{write} callable returned by the I{start_response} callable.
+        The given bytes will be written to the response body, possibly flushing
+        the status and headers first.
+
+        This will be called in a non-I/O thread.
+        """
+        # PEP-3333 states:
+        #
+        #   The server or gateway must transmit the yielded bytestrings to the
+        #   client in an unbuffered fashion, completing the transmission of
+        #   each bytestring before requesting another one.
+        #
+        # This write() method is used for the imperative and (indirectly) for
+        # the more familiar iterable-of-bytestrings WSGI mechanism. It uses
+        # C{blockingCallFromThread} to schedule writes. This allows exceptions
+        # to propagate up from the underlying HTTP implementation. However,
+        # that underlying implementation does not, as yet, provide any way to
+        # know if the written data has been transmitted, so this method
+        # violates the above part of PEP-3333.
+        #
+        # PEP-3333 also says that a server may:
+        #
+        #   Use a different thread to ensure that the block continues to be
+        #   transmitted while the application produces the next block.
+        #
+        # Which suggests that this is actually compliant with PEP-3333,
+        # because writes are done in the reactor thread.
+        #
+        # However, providing some back-pressure may nevertheless be a Good
+        # Thing at some point in the future.
+
+        def wsgiWrite(started):
+            if not started:
+                self._sendResponseHeaders()
+            self.request.write(data)
+
+        try:
+            return blockingCallFromThread(self.reactor, wsgiWrite, self.started)
+        finally:
+            self.started = True
+
+    def _sendResponseHeaders(self):
+        """
+        Set the response code and response headers on the request object, but
+        do not flush them.  The caller is responsible for doing a write in
+        order for anything to actually be written out in response to the
+        request.
+
+        This must be called in the I/O thread.
+        """
+        code, message = self.status.split(None, 1)
+        code = int(code)
+        self.request.setResponseCode(code, _wsgiStringToBytes(message))
+
+        for name, value in self.headers:
+            # Don't allow the application to control these required headers.
+            if name.lower() not in ("server", "date"):
+                self.request.responseHeaders.addRawHeader(
+                    _wsgiStringToBytes(name), _wsgiStringToBytes(value)
+                )
+
+    def start(self):
+        """
+        Start the WSGI application in the threadpool.
+
+        This must be called in the I/O thread.
+        """
+        self.threadpool.callInThread(self.run)
+
+    def run(self):
+        """
+        Call the WSGI application object, iterate it, and handle its output.
+
+        This must be called in a non-I/O thread (ie, a WSGI application
+        thread).
+        """
+        try:
+            appIterator = self.application(self.environ, self.startResponse)
+            for elem in appIterator:
+                if elem:
+                    self.write(elem)
+                if self._requestFinished:
+                    break
+            close = getattr(appIterator, "close", None)
+            if close is not None:
+                close()
+        except BaseException:
+
+            def wsgiError(started, type, value, traceback):
+                self._log.failure(
+                    "WSGI application error", failure=Failure(value, type, traceback)
+                )
+                if started:
+                    self.request.loseConnection()
+                else:
+                    self.request.setResponseCode(INTERNAL_SERVER_ERROR)
+                    self.request.finish()
+
+            self.reactor.callFromThread(wsgiError, self.started, *exc_info())
+        else:
+
+            def wsgiFinish(started):
+                if not self._requestFinished:
+                    if not started:
+                        self._sendResponseHeaders()
+                    self.request.finish()
+
+            self.reactor.callFromThread(wsgiFinish, self.started)
+        self.started = True
+
+
+@implementer(IResource)
+class WSGIResource:
+    """
+    An L{IResource} implementation which delegates responsibility for all
+    resources hierarchically inferior to it to a WSGI application.
+
+    @ivar _reactor: An L{IReactorThreads} provider which will be passed on to
+        L{_WSGIResponse} to schedule calls in the I/O thread.
+
+    @ivar _threadpool: A L{ThreadPool} which will be passed on to
+        L{_WSGIResponse} to run the WSGI application object.
+
+    @ivar _application: The WSGI application object.
+    """
+
+    # Further resource segments are left up to the WSGI application object to
+    # handle.
+    isLeaf = True
+
+    def __init__(self, reactor, threadpool, application):
+        self._reactor = reactor
+        self._threadpool = threadpool
+        self._application = application
+
+    def render(self, request):
+        """
+        Turn the request into the appropriate C{environ} C{dict} suitable to be
+        passed to the WSGI application object and then pass it on.
+
+        The WSGI application object is given almost complete control of the
+        rendering process.  C{NOT_DONE_YET} will always be returned in order
+        and response completion will be dictated by the application object, as
+        will the status, headers, and the response body.
+        """
+        response = _WSGIResponse(
+            self._reactor, self._threadpool, self._application, request
+        )
+        response.start()
+        return NOT_DONE_YET
+
+    def getChildWithDefault(self, name, request):
+        """
+        Reject attempts to retrieve a child resource.  All path segments beyond
+        the one which refers to this resource are handled by the WSGI
+        application object.
+        """
+        raise RuntimeError("Cannot get IResource children from WSGIResource")
+
+    def putChild(self, path, child):
+        """
+        Reject attempts to add a child resource to this resource.  The WSGI
+        application object handles all path segments beneath this resource, so
+        L{IResource} children can never be found.
+        """
+        raise RuntimeError("Cannot put IResource children under WSGIResource")
+
+
+__all__ = ["WSGIResource"]
diff --git a/contrib/python/Twisted/py3/twisted/web/xmlrpc.py b/contrib/python/Twisted/py3/twisted/web/xmlrpc.py
new file mode 100644
index 00000000000..25797efc2f0
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/web/xmlrpc.py
@@ -0,0 +1,633 @@
+# -*- test-case-name: twisted.web.test.test_xmlrpc -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A generic resource for publishing objects via XML-RPC.
+
+Maintainer: Itamar Shtull-Trauring
+
+@var Fault: See L{xmlrpclib.Fault}
+@type Fault: L{xmlrpclib.Fault}
+"""
+
+
+# System Imports
+import base64
+import xmlrpc.client as xmlrpclib
+from urllib.parse import urlparse
+from xmlrpc.client import Binary, Boolean, DateTime, Fault
+
+from twisted.internet import defer, error, protocol
+from twisted.logger import Logger
+from twisted.python import failure, reflect
+from twisted.python.compat import nativeString
+
+# Sibling Imports
+from twisted.web import http, resource, server
+
+# These are deprecated, use the class level definitions
+NOT_FOUND = 8001
+FAILURE = 8002
+
+
+def withRequest(f):
+    """
+    Decorator to cause the request to be passed as the first argument
+    to the method.
+
+    If an I{xmlrpc_} method is wrapped with C{withRequest}, the
+    request object is passed as the first argument to that method.
+    For example::
+
+        @withRequest
+        def xmlrpc_echo(self, request, s):
+            return s
+
+    @since: 10.2
+    """
+    f.withRequest = True
+    return f
+
+
+class NoSuchFunction(Fault):
+    """
+    There is no function by the given name.
+    """
+
+
+class Handler:
+    """
+    Handle a XML-RPC request and store the state for a request in progress.
+
+    Override the run() method and return result using self.result,
+    a Deferred.
+
+    We require this class since we're not using threads, so we can't
+    encapsulate state in a running function if we're going  to have
+    to wait for results.
+
+    For example, lets say we want to authenticate against twisted.cred,
+    run a LDAP query and then pass its result to a database query, all
+    as a result of a single XML-RPC command. We'd use a Handler instance
+    to store the state of the running command.
+    """
+
+    def __init__(self, resource, *args):
+        self.resource = resource  # the XML-RPC resource we are connected to
+        self.result = defer.Deferred()
+        self.run(*args)
+
+    def run(self, *args):
+        # event driven equivalent of 'raise UnimplementedError'
+        self.result.errback(NotImplementedError("Implement run() in subclasses"))
+
+
+class XMLRPC(resource.Resource):
+    """
+    A resource that implements XML-RPC.
+
+    You probably want to connect this to '/RPC2'.
+
+    Methods published can return XML-RPC serializable results, Faults,
+    Binary, Boolean, DateTime, Deferreds, or Handler instances.
+
+    By default methods beginning with 'xmlrpc_' are published.
+
+    Sub-handlers for prefixed methods (e.g., system.listMethods)
+    can be added with putSubHandler. By default, prefixes are
+    separated with a '.'. Override self.separator to change this.
+
+    @ivar allowNone: Permit XML translating of Python constant None.
+    @type allowNone: C{bool}
+
+    @ivar useDateTime: Present C{datetime} values as C{datetime.datetime}
+        objects?
+    @type useDateTime: C{bool}
+    """
+
+    # Error codes for Twisted, if they conflict with yours then
+    # modify them at runtime.
+    NOT_FOUND = 8001
+    FAILURE = 8002
+
+    isLeaf = 1
+    separator = "."
+    allowedMethods = (b"POST",)
+    _log = Logger()
+
+    def __init__(self, allowNone=False, useDateTime=False):
+        resource.Resource.__init__(self)
+        self.subHandlers = {}
+        self.allowNone = allowNone
+        self.useDateTime = useDateTime
+
+    def __setattr__(self, name, value):
+        self.__dict__[name] = value
+
+    def putSubHandler(self, prefix, handler):
+        self.subHandlers[prefix] = handler
+
+    def getSubHandler(self, prefix):
+        return self.subHandlers.get(prefix, None)
+
+    def getSubHandlerPrefixes(self):
+        return list(self.subHandlers.keys())
+
+    def render_POST(self, request):
+        request.content.seek(0, 0)
+        request.setHeader(b"content-type", b"text/xml; charset=utf-8")
+        try:
+            args, functionPath = xmlrpclib.loads(
+                request.content.read(), use_datetime=self.useDateTime
+            )
+        except Exception as e:
+            f = Fault(self.FAILURE, f"Can't deserialize input: {e}")
+            self._cbRender(f, request)
+        else:
+            try:
+                function = self.lookupProcedure(functionPath)
+            except Fault as f:
+                self._cbRender(f, request)
+            else:
+                # Use this list to track whether the response has failed or not.
+                # This will be used later on to decide if the result of the
+                # Deferred should be written out and Request.finish called.
+                responseFailed = []
+                request.notifyFinish().addErrback(responseFailed.append)
+                if getattr(function, "withRequest", False):
+                    d = defer.maybeDeferred(function, request, *args)
+                else:
+                    d = defer.maybeDeferred(function, *args)
+                d.addErrback(self._ebRender)
+                d.addCallback(self._cbRender, request, responseFailed)
+        return server.NOT_DONE_YET
+
+    def _cbRender(self, result, request, responseFailed=None):
+        if responseFailed:
+            return
+
+        if isinstance(result, Handler):
+            result = result.result
+        if not isinstance(result, Fault):
+            result = (result,)
+        try:
+            try:
+                content = xmlrpclib.dumps(
+                    result, methodresponse=True, allow_none=self.allowNone
+                )
+            except Exception as e:
+                f = Fault(self.FAILURE, f"Can't serialize output: {e}")
+                content = xmlrpclib.dumps(
+                    f, methodresponse=True, allow_none=self.allowNone
+                )
+
+            if isinstance(content, str):
+                content = content.encode("utf8")
+            request.setHeader(b"content-length", b"%d" % (len(content),))
+            request.write(content)
+        except Exception:
+            self._log.failure("")
+        request.finish()
+
+    def _ebRender(self, failure):
+        if isinstance(failure.value, Fault):
+            return failure.value
+        self._log.failure("", failure)
+        return Fault(self.FAILURE, "error")
+
+    def lookupProcedure(self, procedurePath):
+        """
+        Given a string naming a procedure, return a callable object for that
+        procedure or raise NoSuchFunction.
+
+        The returned object will be called, and should return the result of the
+        procedure, a Deferred, or a Fault instance.
+
+        Override in subclasses if you want your own policy.  The base
+        implementation that given C{'foo'}, C{self.xmlrpc_foo} will be returned.
+        If C{procedurePath} contains C{self.separator}, the sub-handler for the
+        initial prefix is used to search for the remaining path.
+
+        If you override C{lookupProcedure}, you may also want to override
+        C{listProcedures} to accurately report the procedures supported by your
+        resource, so that clients using the I{system.listMethods} procedure
+        receive accurate results.
+
+        @since: 11.1
+        """
+        if procedurePath.find(self.separator) != -1:
+            prefix, procedurePath = procedurePath.split(self.separator, 1)
+            handler = self.getSubHandler(prefix)
+            if handler is None:
+                raise NoSuchFunction(self.NOT_FOUND, "no such subHandler %s" % prefix)
+            return handler.lookupProcedure(procedurePath)
+
+        f = getattr(self, "xmlrpc_%s" % procedurePath, None)
+        if not f:
+            raise NoSuchFunction(
+                self.NOT_FOUND, "procedure %s not found" % procedurePath
+            )
+        elif not callable(f):
+            raise NoSuchFunction(
+                self.NOT_FOUND, "procedure %s not callable" % procedurePath
+            )
+        else:
+            return f
+
+    def listProcedures(self):
+        """
+        Return a list of the names of all xmlrpc procedures.
+
+        @since: 11.1
+        """
+        return reflect.prefixedMethodNames(self.__class__, "xmlrpc_")
+
+
+class XMLRPCIntrospection(XMLRPC):
+    """
+    Implement the XML-RPC Introspection API.
+
+    By default, the methodHelp method returns the 'help' method attribute,
+    if it exists, otherwise the __doc__ method attribute, if it exists,
+    otherwise the empty string.
+
+    To enable the methodSignature method, add a 'signature' method attribute
+    containing a list of lists. See methodSignature's documentation for the
+    format. Note the type strings should be XML-RPC types, not Python types.
+    """
+
+    def __init__(self, parent):
+        """
+        Implement Introspection support for an XMLRPC server.
+
+        @param parent: the XMLRPC server to add Introspection support to.
+        @type parent: L{XMLRPC}
+        """
+        XMLRPC.__init__(self)
+        self._xmlrpc_parent = parent
+
+    def xmlrpc_listMethods(self):
+        """
+        Return a list of the method names implemented by this server.
+        """
+        functions = []
+        todo = [(self._xmlrpc_parent, "")]
+        while todo:
+            obj, prefix = todo.pop(0)
+            functions.extend([prefix + name for name in obj.listProcedures()])
+            todo.extend(
+                [
+                    (obj.getSubHandler(name), prefix + name + obj.separator)
+                    for name in obj.getSubHandlerPrefixes()
+                ]
+            )
+        return functions
+
+    xmlrpc_listMethods.signature = [["array"]]  # type: ignore[attr-defined]
+
+    def xmlrpc_methodHelp(self, method):
+        """
+        Return a documentation string describing the use of the given method.
+        """
+        method = self._xmlrpc_parent.lookupProcedure(method)
+        return getattr(method, "help", None) or getattr(method, "__doc__", None) or ""
+
+    xmlrpc_methodHelp.signature = [["string", "string"]]  # type: ignore[attr-defined]
+
+    def xmlrpc_methodSignature(self, method):
+        """
+        Return a list of type signatures.
+
+        Each type signature is a list of the form [rtype, type1, type2, ...]
+        where rtype is the return type and typeN is the type of the Nth
+        argument. If no signature information is available, the empty
+        string is returned.
+        """
+        method = self._xmlrpc_parent.lookupProcedure(method)
+        return getattr(method, "signature", None) or ""
+
+    xmlrpc_methodSignature.signature = [  # type: ignore[attr-defined]
+        ["array", "string"],
+        ["string", "string"],
+    ]
+
+
+def addIntrospection(xmlrpc):
+    """
+    Add Introspection support to an XMLRPC server.
+
+    @param xmlrpc: the XMLRPC server to add Introspection support to.
+    @type xmlrpc: L{XMLRPC}
+    """
+    xmlrpc.putSubHandler("system", XMLRPCIntrospection(xmlrpc))
+
+
+class QueryProtocol(http.HTTPClient):
+    def connectionMade(self):
+        self._response = None
+        self.sendCommand(b"POST", self.factory.path)
+        self.sendHeader(b"User-Agent", b"Twisted/XMLRPClib")
+        self.sendHeader(b"Host", self.factory.host)
+        self.sendHeader(b"Content-type", b"text/xml; charset=utf-8")
+        payload = self.factory.payload
+        self.sendHeader(b"Content-length", b"%d" % (len(payload),))
+
+        if self.factory.user:
+            auth = b":".join([self.factory.user, self.factory.password])
+            authHeader = b"".join([b"Basic ", base64.b64encode(auth)])
+            self.sendHeader(b"Authorization", authHeader)
+        self.endHeaders()
+        self.transport.write(payload)
+
+    def handleStatus(self, version, status, message):
+        if status != b"200":
+            self.factory.badStatus(status, message)
+
+    def handleResponse(self, contents):
+        """
+        Handle the XML-RPC response received from the server.
+
+        Specifically, disconnect from the server and store the XML-RPC
+        response so that it can be properly handled when the disconnect is
+        finished.
+        """
+        self.transport.loseConnection()
+        self._response = contents
+
+    def connectionLost(self, reason):
+        """
+        The connection to the server has been lost.
+
+        If we have a full response from the server, then parse it and fired a
+        Deferred with the return value or C{Fault} that the server gave us.
+        """
+        if not reason.check(error.ConnectionDone, error.ConnectionLost):
+            # for example, ssl.SSL.Error
+            self.factory.clientConnectionLost(None, reason)
+        http.HTTPClient.connectionLost(self, reason)
+        if self._response is not None:
+            response, self._response = self._response, None
+            self.factory.parseResponse(response)
+
+
+payloadTemplate = """<?xml version="1.0"?>
+<methodCall>
+<methodName>%s</methodName>
+%s
+</methodCall>
+"""
+
+
+class QueryFactory(protocol.ClientFactory):
+    """
+    XML-RPC Client Factory
+
+    @ivar path: The path portion of the URL to which to post method calls.
+    @type path: L{bytes}
+
+    @ivar host: The value to use for the Host HTTP header.
+    @type host: L{bytes}
+
+    @ivar user: The username with which to authenticate with the server
+        when making calls.
+    @type user: L{bytes} or L{None}
+
+    @ivar password: The password with which to authenticate with the server
+        when making calls.
+    @type password: L{bytes} or L{None}
+
+    @ivar useDateTime: Accept datetime values as datetime.datetime objects.
+        also passed to the underlying xmlrpclib implementation.  Defaults to
+        C{False}.
+    @type useDateTime: C{bool}
+    """
+
+    deferred = None
+    protocol = QueryProtocol
+
+    def __init__(
+        self,
+        path,
+        host,
+        method,
+        user=None,
+        password=None,
+        allowNone=False,
+        args=(),
+        canceller=None,
+        useDateTime=False,
+    ):
+        """
+        @param method: The name of the method to call.
+        @type method: C{str}
+
+        @param allowNone: allow the use of None values in parameters. It's
+            passed to the underlying xmlrpclib implementation. Defaults to
+            C{False}.
+        @type allowNone: C{bool} or L{None}
+
+        @param args: the arguments to pass to the method.
+        @type args: C{tuple}
+
+        @param canceller: A 1-argument callable passed to the deferred as the
+            canceller callback.
+        @type canceller: callable or L{None}
+        """
+        self.path, self.host = path, host
+        self.user, self.password = user, password
+        self.payload = payloadTemplate % (
+            method,
+            xmlrpclib.dumps(args, allow_none=allowNone),
+        )
+        if isinstance(self.payload, str):
+            self.payload = self.payload.encode("utf8")
+        self.deferred = defer.Deferred(canceller)
+        self.useDateTime = useDateTime
+
+    def parseResponse(self, contents):
+        if not self.deferred:
+            return
+        try:
+            response = xmlrpclib.loads(contents, use_datetime=self.useDateTime)[0][0]
+        except BaseException:
+            deferred, self.deferred = self.deferred, None
+            deferred.errback(failure.Failure())
+        else:
+            deferred, self.deferred = self.deferred, None
+            deferred.callback(response)
+
+    def clientConnectionLost(self, _, reason):
+        if self.deferred is not None:
+            deferred, self.deferred = self.deferred, None
+            deferred.errback(reason)
+
+    clientConnectionFailed = clientConnectionLost
+
+    def badStatus(self, status, message):
+        deferred, self.deferred = self.deferred, None
+        deferred.errback(ValueError(status, message))
+
+
+class Proxy:
+    """
+    A Proxy for making remote XML-RPC calls.
+
+    Pass the URL of the remote XML-RPC server to the constructor.
+
+    Use C{proxy.callRemote('foobar', *args)} to call remote method
+    'foobar' with *args.
+
+    @ivar user: The username with which to authenticate with the server
+        when making calls.  If specified, overrides any username information
+        embedded in C{url}.  If not specified, a value may be taken from
+        C{url} if present.
+    @type user: L{bytes} or L{None}
+
+    @ivar password: The password with which to authenticate with the server
+        when making calls.  If specified, overrides any password information
+        embedded in C{url}.  If not specified, a value may be taken from
+        C{url} if present.
+    @type password: L{bytes} or L{None}
+
+    @ivar allowNone: allow the use of None values in parameters. It's
+        passed to the underlying L{xmlrpclib} implementation. Defaults to
+        C{False}.
+    @type allowNone: C{bool} or L{None}
+
+    @ivar useDateTime: Accept datetime values as datetime.datetime objects.
+        also passed to the underlying L{xmlrpclib} implementation. Defaults to
+        C{False}.
+    @type useDateTime: C{bool}
+
+    @ivar connectTimeout: Number of seconds to wait before assuming the
+        connection has failed.
+    @type connectTimeout: C{float}
+
+    @ivar _reactor: The reactor used to create connections.
+    @type _reactor: Object providing L{twisted.internet.interfaces.IReactorTCP}
+
+    @ivar queryFactory: Object returning a factory for XML-RPC protocol. Use
+        this for testing, or to manipulate the XML-RPC parsing behavior. For
+        example, you may set this to a custom "debugging" factory object that
+        reimplements C{parseResponse} in order to log the raw XML-RPC contents
+        from the server before continuing on with parsing. Another possibility
+        is to implement your own XML-RPC marshaller here to handle non-standard
+        XML-RPC traffic.
+    @type queryFactory: L{twisted.web.xmlrpc.QueryFactory}
+    """
+
+    queryFactory = QueryFactory
+
+    def __init__(
+        self,
+        url,
+        user=None,
+        password=None,
+        allowNone=False,
+        useDateTime=False,
+        connectTimeout=30.0,
+        reactor=None,
+    ):
+        """
+        @param url: The URL to which to post method calls.  Calls will be made
+            over SSL if the scheme is HTTPS.  If netloc contains username or
+            password information, these will be used to authenticate, as long as
+            the C{user} and C{password} arguments are not specified.
+        @type url: L{bytes}
+
+        """
+        if reactor is None:
+            from twisted.internet import reactor
+
+        scheme, netloc, path, params, query, fragment = urlparse(url)
+        netlocParts = netloc.split(b"@")
+        if len(netlocParts) == 2:
+            userpass = netlocParts.pop(0).split(b":")
+            self.user = userpass.pop(0)
+            try:
+                self.password = userpass.pop(0)
+            except BaseException:
+                self.password = None
+        else:
+            self.user = self.password = None
+        hostport = netlocParts[0].split(b":")
+        self.host = hostport.pop(0)
+        try:
+            self.port = int(hostport.pop(0))
+        except BaseException:
+            self.port = None
+        self.path = path
+        if self.path in [b"", None]:
+            self.path = b"/"
+        self.secure = scheme == b"https"
+        if user is not None:
+            self.user = user
+        if password is not None:
+            self.password = password
+        self.allowNone = allowNone
+        self.useDateTime = useDateTime
+        self.connectTimeout = connectTimeout
+        self._reactor = reactor
+
+    def callRemote(self, method, *args):
+        """
+        Call remote XML-RPC C{method} with given arguments.
+
+        @return: a L{defer.Deferred} that will fire with the method response,
+            or a failure if the method failed. Generally, the failure type will
+            be L{Fault}, but you can also have an C{IndexError} on some buggy
+            servers giving empty responses.
+
+            If the deferred is cancelled before the request completes, the
+            connection is closed and the deferred will fire with a
+            L{defer.CancelledError}.
+        """
+
+        def cancel(d):
+            factory.deferred = None
+            connector.disconnect()
+
+        factory = self.queryFactory(
+            self.path,
+            self.host,
+            method,
+            self.user,
+            self.password,
+            self.allowNone,
+            args,
+            cancel,
+            self.useDateTime,
+        )
+
+        if self.secure:
+            from twisted.internet import ssl
+
+            contextFactory = ssl.optionsForClientTLS(hostname=nativeString(self.host))
+            connector = self._reactor.connectSSL(
+                nativeString(self.host),
+                self.port or 443,
+                factory,
+                contextFactory,
+                timeout=self.connectTimeout,
+            )
+        else:
+            connector = self._reactor.connectTCP(
+                nativeString(self.host),
+                self.port or 80,
+                factory,
+                timeout=self.connectTimeout,
+            )
+        return factory.deferred
+
+
+__all__ = [
+    "XMLRPC",
+    "Handler",
+    "NoSuchFunction",
+    "Proxy",
+    "Fault",
+    "Binary",
+    "Boolean",
+    "DateTime",
+]
diff --git a/contrib/python/Twisted/py3/twisted/words/__init__.py b/contrib/python/Twisted/py3/twisted/words/__init__.py
new file mode 100644
index 00000000000..454119778f7
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/__init__.py
@@ -0,0 +1,8 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Twisted Words: Client and server implementations for IRC, XMPP, and other chat
+services.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/words/ewords.py b/contrib/python/Twisted/py3/twisted/words/ewords.py
new file mode 100644
index 00000000000..524d7fd3c94
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/ewords.py
@@ -0,0 +1,41 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""Exception definitions for Words
+"""
+
+
+class WordsError(Exception):
+    def __str__(self) -> str:
+        return self.__class__.__name__ + ": " + Exception.__str__(self)
+
+
+class NoSuchUser(WordsError):
+    pass
+
+
+class DuplicateUser(WordsError):
+    pass
+
+
+class NoSuchGroup(WordsError):
+    pass
+
+
+class DuplicateGroup(WordsError):
+    pass
+
+
+class AlreadyLoggedIn(WordsError):
+    pass
+
+
+__all__ = [
+    "WordsError",
+    "NoSuchUser",
+    "DuplicateUser",
+    "NoSuchGroup",
+    "DuplicateGroup",
+    "AlreadyLoggedIn",
+]
diff --git a/contrib/python/Twisted/py3/twisted/words/im/__init__.py b/contrib/python/Twisted/py3/twisted/words/im/__init__.py
new file mode 100644
index 00000000000..4ea2607ad13
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/__init__.py
@@ -0,0 +1,7 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Instance Messenger, Pan-protocol chat client.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/words/im/baseaccount.py b/contrib/python/Twisted/py3/twisted/words/im/baseaccount.py
new file mode 100644
index 00000000000..be8aa538d04
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/baseaccount.py
@@ -0,0 +1,69 @@
+# -*- Python -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+
+class AccountManager:
+    """I am responsible for managing a user's accounts.
+
+    That is, remembering what accounts are available, their settings,
+    adding and removal of accounts, etc.
+
+    @ivar accounts: A collection of available accounts.
+    @type accounts: mapping of strings to L{Account<interfaces.IAccount>}s.
+    """
+
+    def __init__(self):
+        self.accounts = {}
+
+    def getSnapShot(self):
+        """A snapshot of all the accounts and their status.
+
+        @returns: A list of tuples, each of the form
+            (string:accountName, boolean:isOnline,
+            boolean:autoLogin, string:gatewayType)
+        """
+        data = []
+        for account in self.accounts.values():
+            data.append(
+                (
+                    account.accountName,
+                    account.isOnline(),
+                    account.autoLogin,
+                    account.gatewayType,
+                )
+            )
+        return data
+
+    def isEmpty(self):
+        return len(self.accounts) == 0
+
+    def getConnectionInfo(self):
+        connectioninfo = []
+        for account in self.accounts.values():
+            connectioninfo.append(account.isOnline())
+        return connectioninfo
+
+    def addAccount(self, account):
+        self.accounts[account.accountName] = account
+
+    def delAccount(self, accountName):
+        del self.accounts[accountName]
+
+    def connect(self, accountName, chatui):
+        """
+        @returntype: Deferred L{interfaces.IClient}
+        """
+        return self.accounts[accountName].logOn(chatui)
+
+    def disconnect(self, accountName):
+        pass
+        # self.accounts[accountName].logOff()  - not yet implemented
+
+    def quit(self):
+        pass
+        # for account in self.accounts.values():
+        #    account.logOff()  - not yet implemented
diff --git a/contrib/python/Twisted/py3/twisted/words/im/basechat.py b/contrib/python/Twisted/py3/twisted/words/im/basechat.py
new file mode 100644
index 00000000000..c980c022a36
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/basechat.py
@@ -0,0 +1,486 @@
+# -*- test-case-name: twisted.words.test.test_basechat -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Base classes for Instance Messenger clients.
+"""
+
+from twisted.words.im.locals import AWAY, OFFLINE, ONLINE
+
+
+class ContactsList:
+    """
+    A GUI object that displays a contacts list.
+
+    @ivar chatui: The GUI chat client associated with this contacts list.
+    @type chatui: L{ChatUI}
+
+    @ivar contacts: The contacts.
+    @type contacts: C{dict} mapping C{str} to a L{IPerson<interfaces.IPerson>}
+        provider
+
+    @ivar onlineContacts: The contacts who are currently online (have a status
+        that is not C{OFFLINE}).
+    @type onlineContacts: C{dict} mapping C{str} to a
+        L{IPerson<interfaces.IPerson>} provider
+
+    @ivar clients: The signed-on clients.
+    @type clients: C{list} of L{IClient<interfaces.IClient>} providers
+    """
+
+    def __init__(self, chatui):
+        """
+        @param chatui: The GUI chat client associated with this contacts list.
+        @type chatui: L{ChatUI}
+        """
+        self.chatui = chatui
+        self.contacts = {}
+        self.onlineContacts = {}
+        self.clients = []
+
+    def setContactStatus(self, person):
+        """
+        Inform the user that a person's status has changed.
+
+        @param person: The person whose status has changed.
+        @type person: L{IPerson<interfaces.IPerson>} provider
+        """
+        if person.name not in self.contacts:
+            self.contacts[person.name] = person
+        if person.name not in self.onlineContacts and (
+            person.status == ONLINE or person.status == AWAY
+        ):
+            self.onlineContacts[person.name] = person
+        if person.name in self.onlineContacts and person.status == OFFLINE:
+            del self.onlineContacts[person.name]
+
+    def registerAccountClient(self, client):
+        """
+        Notify the user that an account client has been signed on to.
+
+        @param client: The client being added to your list of account clients.
+        @type client: L{IClient<interfaces.IClient>} provider
+        """
+        if client not in self.clients:
+            self.clients.append(client)
+
+    def unregisterAccountClient(self, client):
+        """
+        Notify the user that an account client has been signed off or
+        disconnected from.
+
+        @param client: The client being removed from the list of account
+            clients.
+        @type client: L{IClient<interfaces.IClient>} provider
+        """
+        if client in self.clients:
+            self.clients.remove(client)
+
+    def contactChangedNick(self, person, newnick):
+        """
+        Update your contact information to reflect a change to a contact's
+        nickname.
+
+        @param person: The person in your contacts list whose nickname is
+            changing.
+        @type person: L{IPerson<interfaces.IPerson>} provider
+
+        @param newnick: The new nickname for this person.
+        @type newnick: C{str}
+        """
+        oldname = person.name
+        if oldname in self.contacts:
+            del self.contacts[oldname]
+            person.name = newnick
+            self.contacts[newnick] = person
+            if oldname in self.onlineContacts:
+                del self.onlineContacts[oldname]
+                self.onlineContacts[newnick] = person
+
+
+class Conversation:
+    """
+    A GUI window of a conversation with a specific person.
+
+    @ivar person: The person who you're having this conversation with.
+    @type person: L{IPerson<interfaces.IPerson>} provider
+
+    @ivar chatui: The GUI chat client associated with this conversation.
+    @type chatui: L{ChatUI}
+    """
+
+    def __init__(self, person, chatui):
+        """
+        @param person: The person who you're having this conversation with.
+        @type person: L{IPerson<interfaces.IPerson>} provider
+
+        @param chatui: The GUI chat client associated with this conversation.
+        @type chatui: L{ChatUI}
+        """
+        self.chatui = chatui
+        self.person = person
+
+    def show(self):
+        """
+        Display the ConversationWindow.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def hide(self):
+        """
+        Hide the ConversationWindow.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def sendText(self, text):
+        """
+        Send text to the person with whom the user is conversing.
+
+        @param text: The text to be sent.
+        @type text: C{str}
+        """
+        self.person.sendMessage(text, None)
+
+    def showMessage(self, text, metadata=None):
+        """
+        Display a message sent from the person with whom the user is conversing.
+
+        @param text: The sent message.
+        @type text: C{str}
+
+        @param metadata: Metadata associated with this message.
+        @type metadata: C{dict}
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def contactChangedNick(self, person, newnick):
+        """
+        Change a person's name.
+
+        @param person: The person whose nickname is changing.
+        @type person: L{IPerson<interfaces.IPerson>} provider
+
+        @param newnick: The new nickname for this person.
+        @type newnick: C{str}
+        """
+        self.person.name = newnick
+
+
+class GroupConversation:
+    """
+    A GUI window of a conversation with a group of people.
+
+    @ivar chatui: The GUI chat client associated with this conversation.
+    @type chatui: L{ChatUI}
+
+    @ivar group: The group of people that are having this conversation.
+    @type group: L{IGroup<interfaces.IGroup>} provider
+
+    @ivar members: The names of the people in this conversation.
+    @type members: C{list} of C{str}
+    """
+
+    def __init__(self, group, chatui):
+        """
+        @param chatui: The GUI chat client associated with this conversation.
+        @type chatui: L{ChatUI}
+
+        @param group: The group of people that are having this conversation.
+        @type group: L{IGroup<interfaces.IGroup>} provider
+        """
+        self.chatui = chatui
+        self.group = group
+        self.members = []
+
+    def show(self):
+        """
+        Display the GroupConversationWindow.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def hide(self):
+        """
+        Hide the GroupConversationWindow.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def sendText(self, text):
+        """
+        Send text to the group.
+
+        @param text: The text to be sent.
+        @type text: C{str}
+        """
+        self.group.sendGroupMessage(text, None)
+
+    def showGroupMessage(self, sender, text, metadata=None):
+        """
+        Display to the user a message sent to this group from the given sender.
+
+        @param sender: The person sending the message.
+        @type sender: C{str}
+
+        @param text: The sent message.
+        @type text: C{str}
+
+        @param metadata: Metadata associated with this message.
+        @type metadata: C{dict}
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def setGroupMembers(self, members):
+        """
+        Set the list of members in the group.
+
+        @param members: The names of the people that will be in this group.
+        @type members: C{list} of C{str}
+        """
+        self.members = members
+
+    def setTopic(self, topic, author):
+        """
+        Change the topic for the group conversation window and display this
+        change to the user.
+
+        @param topic: This group's topic.
+        @type topic: C{str}
+
+        @param author: The person changing the topic.
+        @type author: C{str}
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def memberJoined(self, member):
+        """
+        Add the given member to the list of members in the group conversation
+        and displays this to the user.
+
+        @param member: The person joining the group conversation.
+        @type member: C{str}
+        """
+        if member not in self.members:
+            self.members.append(member)
+
+    def memberChangedNick(self, oldnick, newnick):
+        """
+        Change the nickname for a member of the group conversation and displays
+        this change to the user.
+
+        @param oldnick: The old nickname.
+        @type oldnick: C{str}
+
+        @param newnick: The new nickname.
+        @type newnick: C{str}
+        """
+        if oldnick in self.members:
+            self.members.remove(oldnick)
+            self.members.append(newnick)
+
+    def memberLeft(self, member):
+        """
+        Delete the given member from the list of members in the group
+        conversation and displays the change to the user.
+
+        @param member: The person leaving the group conversation.
+        @type member: C{str}
+        """
+        if member in self.members:
+            self.members.remove(member)
+
+
+class ChatUI:
+    """
+    A GUI chat client.
+
+    @type conversations: C{dict} of L{Conversation}
+    @ivar conversations: A cache of all the direct windows.
+
+    @type groupConversations: C{dict} of L{GroupConversation}
+    @ivar groupConversations: A cache of all the group windows.
+
+    @type persons: C{dict} with keys that are a C{tuple} of (C{str},
+       L{IAccount<interfaces.IAccount>} provider) and values that are
+       L{IPerson<interfaces.IPerson>} provider
+    @ivar persons: A cache of all the users associated with this client.
+
+    @type groups: C{dict} with keys that are a C{tuple} of (C{str},
+        L{IAccount<interfaces.IAccount>} provider) and values that are
+        L{IGroup<interfaces.IGroup>} provider
+    @ivar groups: A cache of all the groups associated with this client.
+
+    @type onlineClients: C{list} of L{IClient<interfaces.IClient>} providers
+    @ivar onlineClients: A list of message sources currently online.
+
+    @type contactsList: L{ContactsList}
+    @ivar contactsList: A contacts list.
+    """
+
+    def __init__(self):
+        self.conversations = {}
+        self.groupConversations = {}
+        self.persons = {}
+        self.groups = {}
+        self.onlineClients = []
+        self.contactsList = ContactsList(self)
+
+    def registerAccountClient(self, client):
+        """
+        Notify the user that an account has been signed on to.
+
+        @type client: L{IClient<interfaces.IClient>} provider
+        @param client: The client account for the person who has just signed on.
+
+        @rtype: L{IClient<interfaces.IClient>} provider
+        @return: The client, so that it may be used in a callback chain.
+        """
+        self.onlineClients.append(client)
+        self.contactsList.registerAccountClient(client)
+        return client
+
+    def unregisterAccountClient(self, client):
+        """
+        Notify the user that an account has been signed off or disconnected.
+
+        @type client: L{IClient<interfaces.IClient>} provider
+        @param client: The client account for the person who has just signed
+            off.
+        """
+        self.onlineClients.remove(client)
+        self.contactsList.unregisterAccountClient(client)
+
+    def getContactsList(self):
+        """
+        Get the contacts list associated with this chat window.
+
+        @rtype: L{ContactsList}
+        @return: The contacts list associated with this chat window.
+        """
+        return self.contactsList
+
+    def getConversation(self, person, Class=Conversation, stayHidden=False):
+        """
+        For the given person object, return the conversation window or create
+        and return a new conversation window if one does not exist.
+
+        @type person: L{IPerson<interfaces.IPerson>} provider
+        @param person: The person whose conversation window we want to get.
+
+        @type Class: L{IConversation<interfaces.IConversation>} implementor
+        @param Class: The kind of conversation window we want. If the conversation
+            window for this person didn't already exist, create one of this type.
+
+        @type stayHidden: C{bool}
+        @param stayHidden: Whether or not the conversation window should stay
+            hidden.
+
+        @rtype: L{IConversation<interfaces.IConversation>} provider
+        @return: The conversation window.
+        """
+        conv = self.conversations.get(person)
+        if not conv:
+            conv = Class(person, self)
+            self.conversations[person] = conv
+        if stayHidden:
+            conv.hide()
+        else:
+            conv.show()
+        return conv
+
+    def getGroupConversation(self, group, Class=GroupConversation, stayHidden=False):
+        """
+        For the given group object, return the group conversation window or
+        create and return a new group conversation window if it doesn't exist.
+
+        @type group: L{IGroup<interfaces.IGroup>} provider
+        @param group: The group whose conversation window we want to get.
+
+        @type Class: L{IConversation<interfaces.IConversation>} implementor
+        @param Class: The kind of conversation window we want. If the conversation
+            window for this person didn't already exist, create one of this type.
+
+        @type stayHidden: C{bool}
+        @param stayHidden: Whether or not the conversation window should stay
+            hidden.
+
+        @rtype: L{IGroupConversation<interfaces.IGroupConversation>} provider
+        @return: The group conversation window.
+        """
+        conv = self.groupConversations.get(group)
+        if not conv:
+            conv = Class(group, self)
+            self.groupConversations[group] = conv
+        if stayHidden:
+            conv.hide()
+        else:
+            conv.show()
+        return conv
+
+    def getPerson(self, name, client):
+        """
+        For the given name and account client, return an instance of a
+        L{IGroup<interfaces.IPerson>} provider or create and return a new
+        instance of a L{IGroup<interfaces.IPerson>} provider.
+
+        @type name: C{str}
+        @param name: The name of the person of interest.
+
+        @type client: L{IClient<interfaces.IClient>} provider
+        @param client: The client account of interest.
+
+        @rtype: L{IPerson<interfaces.IPerson>} provider
+        @return: The person with that C{name}.
+        """
+        account = client.account
+        p = self.persons.get((name, account))
+        if not p:
+            p = account.getPerson(name)
+            self.persons[name, account] = p
+        return p
+
+    def getGroup(self, name, client):
+        """
+        For the given name and account client, return an instance of a
+        L{IGroup<interfaces.IGroup>} provider or create and return a new instance
+        of a L{IGroup<interfaces.IGroup>} provider.
+
+        @type name: C{str}
+        @param name: The name of the group of interest.
+
+        @type client: L{IClient<interfaces.IClient>} provider
+        @param client: The client account of interest.
+
+        @rtype: L{IGroup<interfaces.IGroup>} provider
+        @return: The group with that C{name}.
+        """
+        # I accept 'client' instead of 'account' in my signature for
+        # backwards compatibility.  (Groups changed to be Account-oriented
+        # in CVS revision 1.8.)
+        account = client.account
+        g = self.groups.get((name, account))
+        if not g:
+            g = account.getGroup(name)
+            self.groups[name, account] = g
+        return g
+
+    def contactChangedNick(self, person, newnick):
+        """
+        For the given C{person}, change the C{person}'s C{name} to C{newnick}
+        and tell the contact list and any conversation windows with that
+        C{person} to change as well.
+
+        @type person: L{IPerson<interfaces.IPerson>} provider
+        @param person: The person whose nickname will get changed.
+
+        @type newnick: C{str}
+        @param newnick: The new C{name} C{person} will take.
+        """
+        oldnick = person.name
+        if (oldnick, person.account) in self.persons:
+            conv = self.conversations.get(person)
+            if conv:
+                conv.contactChangedNick(person, newnick)
+            self.contactsList.contactChangedNick(person, newnick)
+            del self.persons[oldnick, person.account]
+            person.name = newnick
+            self.persons[person.name, person.account] = person
diff --git a/contrib/python/Twisted/py3/twisted/words/im/basesupport.py b/contrib/python/Twisted/py3/twisted/words/im/basesupport.py
new file mode 100644
index 00000000000..55519193d4e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/basesupport.py
@@ -0,0 +1,275 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+#
+
+"""Instance Messenger base classes for protocol support.
+
+You will find these useful if you're adding a new protocol to IM.
+"""
+from typing import Type
+
+from twisted.internet import error
+from twisted.internet.protocol import Protocol, connectionDone
+from twisted.persisted import styles
+from twisted.python.failure import Failure
+from twisted.python.reflect import prefixedMethods
+from twisted.words.im.locals import OFFLINE, OfflineError
+
+# Abstract representation of chat "model" classes
+
+
+class AbstractGroup:
+    def __init__(self, name, account):
+        self.name = name
+        self.account = account
+
+    def getGroupCommands(self):
+        """finds group commands
+
+        these commands are methods on me that start with imgroup_; they are
+        called with no arguments
+        """
+        return prefixedMethods(self, "imgroup_")
+
+    def getTargetCommands(self, target):
+        """finds group commands
+
+        these commands are methods on me that start with imgroup_; they are
+        called with a user present within this room as an argument
+
+        you may want to override this in your group in order to filter for
+        appropriate commands on the given user
+        """
+        return prefixedMethods(self, "imtarget_")
+
+    def join(self):
+        if not self.account.client:
+            raise OfflineError
+        self.account.client.joinGroup(self.name)
+
+    def leave(self):
+        if not self.account.client:
+            raise OfflineError
+        self.account.client.leaveGroup(self.name)
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__} {self.name!r}>"
+
+    def __str__(self) -> str:
+        return f"{self.name}@{self.account.accountName}"
+
+
+class AbstractPerson:
+    def __init__(self, name, baseAccount):
+        self.name = name
+        self.account = baseAccount
+        self.status = OFFLINE
+
+    def getPersonCommands(self):
+        """finds person commands
+
+        these commands are methods on me that start with imperson_; they are
+        called with no arguments
+        """
+        return prefixedMethods(self, "imperson_")
+
+    def getIdleTime(self):
+        """
+        Returns a string.
+        """
+        return "--"
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__} {self.name!r}/{self.status}>"
+
+    def __str__(self) -> str:
+        return f"{self.name}@{self.account.accountName}"
+
+
+class AbstractClientMixin:
+    """Designed to be mixed in to a Protocol implementing class.
+
+    Inherit from me first.
+
+    @ivar _logonDeferred: Fired when I am done logging in.
+    """
+
+    _protoBase: Type[Protocol] = None  # type: ignore[assignment]
+
+    def __init__(self, account, chatui, logonDeferred):
+        for base in self.__class__.__bases__:
+            if issubclass(base, Protocol):
+                self.__class__._protoBase = base
+                break
+        else:
+            pass
+        self.account = account
+        self.chat = chatui
+        self._logonDeferred = logonDeferred
+
+    def connectionMade(self):
+        self._protoBase.connectionMade(self)
+
+    def connectionLost(self, reason: Failure = connectionDone) -> None:
+        self.account._clientLost(self, reason)
+        self.unregisterAsAccountClient()
+        return self._protoBase.connectionLost(self, reason)  # type: ignore[arg-type]
+
+    def unregisterAsAccountClient(self):
+        """Tell the chat UI that I have `signed off'."""
+        self.chat.unregisterAccountClient(self)
+
+
+class AbstractAccount(styles.Versioned):
+    """Base class for Accounts.
+
+    I am the start of an implementation of L{IAccount<interfaces.IAccount>}, I
+    implement L{isOnline} and most of L{logOn}, though you'll need to implement
+    L{_startLogOn} in a subclass.
+
+    @cvar _groupFactory: A Callable that will return a L{IGroup} appropriate
+        for this account type.
+    @cvar _personFactory: A Callable that will return a L{IPerson} appropriate
+        for this account type.
+
+    @type _isConnecting: boolean
+    @ivar _isConnecting: Whether I am in the process of establishing a
+    connection to the server.
+    @type _isOnline: boolean
+    @ivar _isOnline: Whether I am currently on-line with the server.
+
+    @ivar accountName:
+    @ivar autoLogin:
+    @ivar username:
+    @ivar password:
+    @ivar host:
+    @ivar port:
+    """
+
+    _isOnline = 0
+    _isConnecting = 0
+    client = None
+
+    _groupFactory = AbstractGroup
+    _personFactory = AbstractPerson
+
+    persistanceVersion = 2
+
+    def __init__(self, accountName, autoLogin, username, password, host, port):
+        self.accountName = accountName
+        self.autoLogin = autoLogin
+        self.username = username
+        self.password = password
+        self.host = host
+        self.port = port
+
+        self._groups = {}
+        self._persons = {}
+
+    def upgrateToVersion2(self):
+        # Added in CVS revision 1.16.
+        for k in ("_groups", "_persons"):
+            if not hasattr(self, k):
+                setattr(self, k, {})
+
+    def __getstate__(self):
+        state = styles.Versioned.__getstate__(self)
+        for k in ("client", "_isOnline", "_isConnecting"):
+            try:
+                del state[k]
+            except KeyError:
+                pass
+        return state
+
+    def isOnline(self):
+        return self._isOnline
+
+    def logOn(self, chatui):
+        """Log on to this account.
+
+        Takes care to not start a connection if a connection is
+        already in progress.  You will need to implement
+        L{_startLogOn} for this to work, and it would be a good idea
+        to override L{_loginFailed} too.
+
+        @returntype: Deferred L{interfaces.IClient}
+        """
+        if (not self._isConnecting) and (not self._isOnline):
+            self._isConnecting = 1
+            d = self._startLogOn(chatui)
+            d.addCallback(self._cb_logOn)
+            # if chatui is not None:
+            # (I don't particularly like having to pass chatUI to this function,
+            # but we haven't factored it out yet.)
+            d.addCallback(chatui.registerAccountClient)
+            d.addErrback(self._loginFailed)
+            return d
+        else:
+            raise error.ConnectError("Connection in progress")
+
+    def getGroup(self, name):
+        """Group factory.
+
+        @param name: Name of the group on this account.
+        @type name: string
+        """
+        group = self._groups.get(name)
+        if group is None:
+            group = self._groupFactory(name, self)
+            self._groups[name] = group
+        return group
+
+    def getPerson(self, name):
+        """Person factory.
+
+        @param name: Name of the person on this account.
+        @type name: string
+        """
+        person = self._persons.get(name)
+        if person is None:
+            person = self._personFactory(name, self)
+            self._persons[name] = person
+        return person
+
+    def _startLogOn(self, chatui):
+        """Start the sign on process.
+
+        Factored out of L{logOn}.
+
+        @returntype: Deferred L{interfaces.IClient}
+        """
+        raise NotImplementedError()
+
+    def _cb_logOn(self, client):
+        self._isConnecting = 0
+        self._isOnline = 1
+        self.client = client
+        return client
+
+    def _loginFailed(self, reason):
+        """Errorback for L{logOn}.
+
+        @type reason: Failure
+
+        @returns: I{reason}, for further processing in the callback chain.
+        @returntype: Failure
+        """
+        self._isConnecting = 0
+        self._isOnline = 0  # just in case
+        return reason
+
+    def _clientLost(self, client, reason):
+        self.client = None
+        self._isConnecting = 0
+        self._isOnline = 0
+        return reason
+
+    def __repr__(self) -> str:
+        return "<{}: {} ({}@{}:{})>".format(
+            self.__class__,
+            self.accountName,
+            self.username,
+            self.host,
+            self.port,
+        )
diff --git a/contrib/python/Twisted/py3/twisted/words/im/instancemessenger.glade b/contrib/python/Twisted/py3/twisted/words/im/instancemessenger.glade
new file mode 100644
index 00000000000..c65359f5f7b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/instancemessenger.glade
@@ -0,0 +1,3164 @@
+<?xml version="1.0"?>
+<GTK-Interface>
+
+<project>
+  <name>InstanceMessenger</name>
+  <program_name>instancemessenger</program_name>
+  <directory></directory>
+  <source_directory>src</source_directory>
+  <pixmaps_directory>pixmaps</pixmaps_directory>
+  <language>C</language>
+  <gnome_support>True</gnome_support>
+  <gettext_support>True</gettext_support>
+  <use_widget_names>True</use_widget_names>
+</project>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>UnseenConversationWindow</name>
+  <visible>False</visible>
+  <title>Unseen Conversation Window</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>False</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>False</auto_shrink>
+
+  <widget>
+    <class>GtkVBox</class>
+    <name>ConversationWidget</name>
+    <homogeneous>False</homogeneous>
+    <spacing>0</spacing>
+
+    <widget>
+      <class>GtkVPaned</class>
+      <name>vpaned1</name>
+      <handle_size>10</handle_size>
+      <gutter_size>6</gutter_size>
+      <position>0</position>
+      <child>
+	<padding>0</padding>
+	<expand>True</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkScrolledWindow</class>
+	<name>scrolledwindow10</name>
+	<hscrollbar_policy>GTK_POLICY_NEVER</hscrollbar_policy>
+	<vscrollbar_policy>GTK_POLICY_ALWAYS</vscrollbar_policy>
+	<hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	<vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	<child>
+	  <shrink>False</shrink>
+	  <resize>True</resize>
+	</child>
+
+	<widget>
+	  <class>GtkText</class>
+	  <name>ConversationOutput</name>
+	  <editable>False</editable>
+	  <text></text>
+	</widget>
+      </widget>
+
+      <widget>
+	<class>GtkScrolledWindow</class>
+	<name>scrolledwindow11</name>
+	<hscrollbar_policy>GTK_POLICY_NEVER</hscrollbar_policy>
+	<vscrollbar_policy>GTK_POLICY_AUTOMATIC</vscrollbar_policy>
+	<hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	<vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	<child>
+	  <shrink>True</shrink>
+	  <resize>False</resize>
+	</child>
+
+	<widget>
+	  <class>GtkText</class>
+	  <name>ConversationMessageEntry</name>
+	  <can_focus>True</can_focus>
+	  <has_focus>True</has_focus>
+	  <signal>
+	    <name>key_press_event</name>
+	    <handler>handle_key_press_event</handler>
+	    <last_modification_time>Tue, 29 Jan 2002 12:42:58 GMT</last_modification_time>
+	  </signal>
+	  <editable>True</editable>
+	  <text></text>
+	</widget>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkHBox</class>
+      <name>hbox9</name>
+      <homogeneous>True</homogeneous>
+      <spacing>0</spacing>
+      <child>
+	<padding>3</padding>
+	<expand>False</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkButton</class>
+	<name>button42</name>
+	<can_focus>True</can_focus>
+	<label> Send Message </label>
+	<relief>GTK_RELIEF_NORMAL</relief>
+	<child>
+	  <padding>3</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkButton</class>
+	<name>AddRemoveContact</name>
+	<can_focus>True</can_focus>
+	<label> Add Contact </label>
+	<relief>GTK_RELIEF_NORMAL</relief>
+	<child>
+	  <padding>3</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkButton</class>
+	<name>CloseContact</name>
+	<can_focus>True</can_focus>
+	<label> Close </label>
+	<relief>GTK_RELIEF_NORMAL</relief>
+	<child>
+	  <padding>3</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+      </widget>
+    </widget>
+  </widget>
+</widget>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>MainIMWindow</name>
+  <signal>
+    <name>destroy</name>
+    <handler>on_MainIMWindow_destroy</handler>
+    <last_modification_time>Sun, 21 Jul 2002 08:16:08 GMT</last_modification_time>
+  </signal>
+  <title>Instance Messenger</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>True</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>False</auto_shrink>
+
+  <widget>
+    <class>GtkNotebook</class>
+    <name>ContactsNotebook</name>
+    <can_focus>True</can_focus>
+    <signal>
+      <name>key_press_event</name>
+      <handler>on_ContactsWidget_key_press_event</handler>
+      <last_modification_time>Tue, 07 May 2002 03:02:33 GMT</last_modification_time>
+    </signal>
+    <show_tabs>True</show_tabs>
+    <show_border>True</show_border>
+    <tab_pos>GTK_POS_TOP</tab_pos>
+    <scrollable>False</scrollable>
+    <tab_hborder>2</tab_hborder>
+    <tab_vborder>2</tab_vborder>
+    <popup_enable>False</popup_enable>
+
+    <widget>
+      <class>GtkVBox</class>
+      <name>vbox11</name>
+      <homogeneous>False</homogeneous>
+      <spacing>0</spacing>
+
+      <widget>
+	<class>GtkLabel</class>
+	<name>OnlineCount</name>
+	<label>Online: %d</label>
+	<justify>GTK_JUSTIFY_CENTER</justify>
+	<wrap>False</wrap>
+	<xalign>0.5</xalign>
+	<yalign>0.5</yalign>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<child>
+	  <padding>0</padding>
+	  <expand>False</expand>
+	  <fill>False</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkScrolledWindow</class>
+	<name>scrolledwindow14</name>
+	<hscrollbar_policy>GTK_POLICY_AUTOMATIC</hscrollbar_policy>
+	<vscrollbar_policy>GTK_POLICY_AUTOMATIC</vscrollbar_policy>
+	<hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	<vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	<child>
+	  <padding>0</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+
+	<widget>
+	  <class>GtkCTree</class>
+	  <name>OnlineContactsTree</name>
+	  <can_focus>True</can_focus>
+	  <signal>
+	    <name>tree_select_row</name>
+	    <handler>on_OnlineContactsTree_tree_select_row</handler>
+	    <last_modification_time>Tue, 07 May 2002 03:06:32 GMT</last_modification_time>
+	  </signal>
+	  <signal>
+	    <name>select_row</name>
+	    <handler>on_OnlineContactsTree_select_row</handler>
+	    <last_modification_time>Tue, 07 May 2002 04:36:10 GMT</last_modification_time>
+	  </signal>
+	  <columns>4</columns>
+	  <column_widths>109,35,23,80</column_widths>
+	  <selection_mode>GTK_SELECTION_SINGLE</selection_mode>
+	  <show_titles>True</show_titles>
+	  <shadow_type>GTK_SHADOW_IN</shadow_type>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CTree:title</child_name>
+	    <name>label77</name>
+	    <label>Alias</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CTree:title</child_name>
+	    <name>label78</name>
+	    <label>Status</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CTree:title</child_name>
+	    <name>label79</name>
+	    <label>Idle</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CTree:title</child_name>
+	    <name>label80</name>
+	    <label>Account</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+	</widget>
+      </widget>
+
+      <widget>
+	<class>GtkVBox</class>
+	<name>vbox30</name>
+	<homogeneous>False</homogeneous>
+	<spacing>2</spacing>
+	<child>
+	  <padding>1</padding>
+	  <expand>False</expand>
+	  <fill>True</fill>
+	</child>
+
+	<widget>
+	  <class>GtkEntry</class>
+	  <name>ContactNameEntry</name>
+	  <can_focus>True</can_focus>
+	  <signal>
+	    <name>activate</name>
+	    <handler>on_ContactNameEntry_activate</handler>
+	    <last_modification_time>Tue, 07 May 2002 04:07:25 GMT</last_modification_time>
+	  </signal>
+	  <editable>True</editable>
+	  <text_visible>True</text_visible>
+	  <text_max_length>0</text_max_length>
+	  <text></text>
+	  <child>
+	    <padding>0</padding>
+	    <expand>False</expand>
+	    <fill>False</fill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkOptionMenu</class>
+	  <name>AccountsListPopup</name>
+	  <can_focus>True</can_focus>
+	  <items>Nothing
+To
+Speak
+Of
+</items>
+	  <initial_choice>1</initial_choice>
+	  <child>
+	    <padding>0</padding>
+	    <expand>False</expand>
+	    <fill>False</fill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkHBox</class>
+	  <name>hbox7</name>
+	  <homogeneous>False</homogeneous>
+	  <spacing>0</spacing>
+	  <child>
+	    <padding>0</padding>
+	    <expand>True</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkButton</class>
+	    <name>PlainSendIM</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>clicked</name>
+	      <handler>on_PlainSendIM_clicked</handler>
+	      <last_modification_time>Tue, 29 Jan 2002 03:17:35 GMT</last_modification_time>
+	    </signal>
+	    <label> Send IM </label>
+	    <relief>GTK_RELIEF_NORMAL</relief>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>False</fill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkButton</class>
+	    <name>PlainGetInfo</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>clicked</name>
+	      <handler>on_PlainGetInfo_clicked</handler>
+	      <last_modification_time>Tue, 07 May 2002 04:06:59 GMT</last_modification_time>
+	    </signal>
+	    <label> Get Info </label>
+	    <relief>GTK_RELIEF_NORMAL</relief>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>False</fill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkButton</class>
+	    <name>PlainJoinChat</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>clicked</name>
+	      <handler>on_PlainJoinChat_clicked</handler>
+	      <last_modification_time>Tue, 29 Jan 2002 13:04:49 GMT</last_modification_time>
+	    </signal>
+	    <label> Join Group </label>
+	    <relief>GTK_RELIEF_NORMAL</relief>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>False</fill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkButton</class>
+	    <name>PlainGoAway</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>clicked</name>
+	      <handler>on_PlainGoAway_clicked</handler>
+	      <last_modification_time>Tue, 07 May 2002 04:06:53 GMT</last_modification_time>
+	    </signal>
+	    <label> Go Away </label>
+	    <relief>GTK_RELIEF_NORMAL</relief>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>False</fill>
+	    </child>
+	  </widget>
+	</widget>
+
+	<widget>
+	  <class>GtkHBox</class>
+	  <name>hbox8</name>
+	  <homogeneous>False</homogeneous>
+	  <spacing>0</spacing>
+	  <child>
+	    <padding>0</padding>
+	    <expand>True</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkButton</class>
+	    <name>AddContactButton</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>clicked</name>
+	      <handler>on_AddContactButton_clicked</handler>
+	      <last_modification_time>Tue, 07 May 2002 04:06:33 GMT</last_modification_time>
+	    </signal>
+	    <label> Add Contact </label>
+	    <relief>GTK_RELIEF_NORMAL</relief>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>False</fill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkButton</class>
+	    <name>RemoveContactButton</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>clicked</name>
+	      <handler>on_RemoveContactButton_clicked</handler>
+	      <last_modification_time>Tue, 07 May 2002 04:06:28 GMT</last_modification_time>
+	    </signal>
+	    <label> Remove Contact </label>
+	    <relief>GTK_RELIEF_NORMAL</relief>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>False</fill>
+	    </child>
+	  </widget>
+	</widget>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <child_name>Notebook:tab</child_name>
+      <name>label35</name>
+      <label> Online Contacts </label>
+      <justify>GTK_JUSTIFY_CENTER</justify>
+      <wrap>False</wrap>
+      <xalign>0.5</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+    </widget>
+
+    <widget>
+      <class>GtkVBox</class>
+      <name>vbox14</name>
+      <homogeneous>False</homogeneous>
+      <spacing>0</spacing>
+
+      <widget>
+	<class>GtkScrolledWindow</class>
+	<name>OfflineContactsScroll</name>
+	<hscrollbar_policy>GTK_POLICY_AUTOMATIC</hscrollbar_policy>
+	<vscrollbar_policy>GTK_POLICY_ALWAYS</vscrollbar_policy>
+	<hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	<vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	<child>
+	  <padding>0</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+
+	<widget>
+	  <class>GtkCList</class>
+	  <name>OfflineContactsList</name>
+	  <can_focus>True</can_focus>
+	  <signal>
+	    <name>select_row</name>
+	    <handler>on_OfflineContactsList_select_row</handler>
+	    <last_modification_time>Tue, 07 May 2002 03:00:07 GMT</last_modification_time>
+	  </signal>
+	  <columns>4</columns>
+	  <column_widths>66,80,80,80</column_widths>
+	  <selection_mode>GTK_SELECTION_SINGLE</selection_mode>
+	  <show_titles>True</show_titles>
+	  <shadow_type>GTK_SHADOW_IN</shadow_type>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CList:title</child_name>
+	    <name>label41</name>
+	    <label>Contact</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CList:title</child_name>
+	    <name>label42</name>
+	    <label>Account</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CList:title</child_name>
+	    <name>label43</name>
+	    <label>Alias</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CList:title</child_name>
+	    <name>label44</name>
+	    <label>Group</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+	</widget>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <child_name>Notebook:tab</child_name>
+      <name>label36</name>
+      <label> All Contacts </label>
+      <justify>GTK_JUSTIFY_CENTER</justify>
+      <wrap>False</wrap>
+      <xalign>0.5</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+    </widget>
+
+    <widget>
+      <class>GtkVBox</class>
+      <name>AccountManWidget</name>
+      <homogeneous>False</homogeneous>
+      <spacing>0</spacing>
+
+      <widget>
+	<class>GtkScrolledWindow</class>
+	<name>scrolledwindow12</name>
+	<hscrollbar_policy>GTK_POLICY_AUTOMATIC</hscrollbar_policy>
+	<vscrollbar_policy>GTK_POLICY_ALWAYS</vscrollbar_policy>
+	<hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	<vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	<child>
+	  <padding>0</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+
+	<widget>
+	  <class>GtkCList</class>
+	  <name>accountsList</name>
+	  <can_focus>True</can_focus>
+	  <columns>4</columns>
+	  <column_widths>80,36,34,80</column_widths>
+	  <selection_mode>GTK_SELECTION_SINGLE</selection_mode>
+	  <show_titles>True</show_titles>
+	  <shadow_type>GTK_SHADOW_IN</shadow_type>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CList:title</child_name>
+	    <name>label45</name>
+	    <label>Service Name</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CList:title</child_name>
+	    <name>label46</name>
+	    <label>Online</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CList:title</child_name>
+	    <name>label47</name>
+	    <label>Auto</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <child_name>CList:title</child_name>
+	    <name>label48</name>
+	    <label>Gateway</label>
+	    <justify>GTK_JUSTIFY_CENTER</justify>
+	    <wrap>False</wrap>
+	    <xalign>0.5</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	  </widget>
+	</widget>
+      </widget>
+
+      <widget>
+	<class>GtkTable</class>
+	<name>table5</name>
+	<rows>2</rows>
+	<columns>3</columns>
+	<homogeneous>False</homogeneous>
+	<row_spacing>0</row_spacing>
+	<column_spacing>0</column_spacing>
+	<child>
+	  <padding>3</padding>
+	  <expand>False</expand>
+	  <fill>True</fill>
+	</child>
+
+	<widget>
+	  <class>GtkButton</class>
+	  <name>NewAccountButton</name>
+	  <can_default>True</can_default>
+	  <can_focus>True</can_focus>
+	  <signal>
+	    <name>clicked</name>
+	    <handler>on_NewAccountButton_clicked</handler>
+	    <last_modification_time>Sun, 27 Jan 2002 10:32:20 GMT</last_modification_time>
+	  </signal>
+	  <label>New Account</label>
+	  <relief>GTK_RELIEF_NORMAL</relief>
+	  <child>
+	    <left_attach>0</left_attach>
+	    <right_attach>1</right_attach>
+	    <top_attach>0</top_attach>
+	    <bottom_attach>1</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>False</xexpand>
+	    <yexpand>False</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>False</yfill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkButton</class>
+	  <name>button46</name>
+	  <sensitive>False</sensitive>
+	  <can_default>True</can_default>
+	  <label>Modify Account</label>
+	  <relief>GTK_RELIEF_NORMAL</relief>
+	  <child>
+	    <left_attach>1</left_attach>
+	    <right_attach>2</right_attach>
+	    <top_attach>0</top_attach>
+	    <bottom_attach>1</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>False</xexpand>
+	    <yexpand>False</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>False</yfill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkButton</class>
+	  <name>LogOnButton</name>
+	  <can_default>True</can_default>
+	  <has_default>True</has_default>
+	  <can_focus>True</can_focus>
+	  <has_focus>True</has_focus>
+	  <signal>
+	    <name>clicked</name>
+	    <handler>on_LogOnButton_clicked</handler>
+	    <last_modification_time>Mon, 28 Jan 2002 04:06:23 GMT</last_modification_time>
+	  </signal>
+	  <label>Logon</label>
+	  <relief>GTK_RELIEF_NORMAL</relief>
+	  <child>
+	    <left_attach>2</left_attach>
+	    <right_attach>3</right_attach>
+	    <top_attach>1</top_attach>
+	    <bottom_attach>2</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>False</xexpand>
+	    <yexpand>False</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>False</yfill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkButton</class>
+	  <name>DeleteAccountButton</name>
+	  <can_default>True</can_default>
+	  <can_focus>True</can_focus>
+	  <signal>
+	    <name>clicked</name>
+	    <handler>on_DeleteAccountButton_clicked</handler>
+	    <last_modification_time>Mon, 28 Jan 2002 00:18:22 GMT</last_modification_time>
+	  </signal>
+	  <label>Delete Account</label>
+	  <relief>GTK_RELIEF_NORMAL</relief>
+	  <child>
+	    <left_attach>2</left_attach>
+	    <right_attach>3</right_attach>
+	    <top_attach>0</top_attach>
+	    <bottom_attach>1</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>False</xexpand>
+	    <yexpand>False</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>False</yfill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkButton</class>
+	  <name>ConsoleButton</name>
+	  <can_default>True</can_default>
+	  <can_focus>True</can_focus>
+	  <signal>
+	    <name>clicked</name>
+	    <handler>on_ConsoleButton_clicked</handler>
+	    <last_modification_time>Mon, 29 Apr 2002 09:13:32 GMT</last_modification_time>
+	  </signal>
+	  <label>Console</label>
+	  <relief>GTK_RELIEF_NORMAL</relief>
+	  <child>
+	    <left_attach>1</left_attach>
+	    <right_attach>2</right_attach>
+	    <top_attach>1</top_attach>
+	    <bottom_attach>2</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>False</xexpand>
+	    <yexpand>False</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>False</yfill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkButton</class>
+	  <name>button75</name>
+	  <can_default>True</can_default>
+	  <can_focus>True</can_focus>
+	  <label>Quit</label>
+	  <relief>GTK_RELIEF_NORMAL</relief>
+	  <child>
+	    <left_attach>0</left_attach>
+	    <right_attach>1</right_attach>
+	    <top_attach>1</top_attach>
+	    <bottom_attach>2</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>True</xexpand>
+	    <yexpand>True</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>True</yfill>
+	  </child>
+	</widget>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <child_name>Notebook:tab</child_name>
+      <name>label107</name>
+      <label>Accounts</label>
+      <justify>GTK_JUSTIFY_CENTER</justify>
+      <wrap>False</wrap>
+      <xalign>0.5</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+    </widget>
+  </widget>
+</widget>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>UnseenGroupWindow</name>
+  <visible>False</visible>
+  <title>Unseen Group Window</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>False</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>False</auto_shrink>
+
+  <widget>
+    <class>GtkVBox</class>
+    <name>GroupChatBox</name>
+    <homogeneous>False</homogeneous>
+    <spacing>0</spacing>
+
+    <widget>
+      <class>GtkHBox</class>
+      <name>hbox5</name>
+      <homogeneous>False</homogeneous>
+      <spacing>0</spacing>
+      <child>
+	<padding>0</padding>
+	<expand>False</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkEntry</class>
+	<name>TopicEntry</name>
+	<can_focus>True</can_focus>
+	<signal>
+	  <name>activate</name>
+	  <handler>on_TopicEntry_activate</handler>
+	  <last_modification_time>Sat, 23 Feb 2002 02:57:41 GMT</last_modification_time>
+	</signal>
+	<signal>
+	  <name>focus_out_event</name>
+	  <handler>on_TopicEntry_focus_out_event</handler>
+	  <last_modification_time>Sun, 21 Jul 2002 09:36:54 GMT</last_modification_time>
+	</signal>
+	<editable>True</editable>
+	<text_visible>True</text_visible>
+	<text_max_length>0</text_max_length>
+	<text>&lt;TOPIC NOT RECEIVED&gt;</text>
+	<child>
+	  <padding>0</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkLabel</class>
+	<name>AuthorLabel</name>
+	<label>&lt;nobody&gt;</label>
+	<justify>GTK_JUSTIFY_CENTER</justify>
+	<wrap>False</wrap>
+	<xalign>0.5</xalign>
+	<yalign>0.5</yalign>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<child>
+	  <padding>0</padding>
+	  <expand>False</expand>
+	  <fill>False</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkButton</class>
+	<name>HideButton</name>
+	<can_focus>True</can_focus>
+	<signal>
+	  <name>clicked</name>
+	  <handler>on_HideButton_clicked</handler>
+	  <last_modification_time>Tue, 29 Jan 2002 14:10:00 GMT</last_modification_time>
+	</signal>
+	<label>&lt;</label>
+	<relief>GTK_RELIEF_NORMAL</relief>
+	<child>
+	  <padding>0</padding>
+	  <expand>False</expand>
+	  <fill>False</fill>
+	</child>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkVPaned</class>
+      <name>vpaned2</name>
+      <handle_size>10</handle_size>
+      <gutter_size>6</gutter_size>
+      <position>0</position>
+      <child>
+	<padding>0</padding>
+	<expand>True</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkHPaned</class>
+	<name>GroupHPaned</name>
+	<handle_size>6</handle_size>
+	<gutter_size>6</gutter_size>
+	<child>
+	  <shrink>False</shrink>
+	  <resize>True</resize>
+	</child>
+
+	<widget>
+	  <class>GtkScrolledWindow</class>
+	  <name>scrolledwindow4</name>
+	  <hscrollbar_policy>GTK_POLICY_NEVER</hscrollbar_policy>
+	  <vscrollbar_policy>GTK_POLICY_ALWAYS</vscrollbar_policy>
+	  <hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	  <vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	  <child>
+	    <shrink>False</shrink>
+	    <resize>True</resize>
+	  </child>
+
+	  <widget>
+	    <class>GtkText</class>
+	    <name>GroupOutput</name>
+	    <can_focus>True</can_focus>
+	    <editable>False</editable>
+	    <text></text>
+	  </widget>
+	</widget>
+
+	<widget>
+	  <class>GtkVBox</class>
+	  <name>actionvbox</name>
+	  <width>110</width>
+	  <homogeneous>False</homogeneous>
+	  <spacing>1</spacing>
+	  <child>
+	    <shrink>True</shrink>
+	    <resize>False</resize>
+	  </child>
+
+	  <widget>
+	    <class>GtkScrolledWindow</class>
+	    <name>scrolledwindow5</name>
+	    <hscrollbar_policy>GTK_POLICY_NEVER</hscrollbar_policy>
+	    <vscrollbar_policy>GTK_POLICY_ALWAYS</vscrollbar_policy>
+	    <hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	    <vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>True</fill>
+	    </child>
+
+	    <widget>
+	      <class>GtkCList</class>
+	      <name>ParticipantList</name>
+	      <can_focus>True</can_focus>
+	      <signal>
+		<name>select_row</name>
+		<handler>on_ParticipantList_select_row</handler>
+		<last_modification_time>Sat, 13 Jul 2002 08:11:12 GMT</last_modification_time>
+	      </signal>
+	      <signal>
+		<name>unselect_row</name>
+		<handler>on_ParticipantList_unselect_row</handler>
+		<last_modification_time>Sat, 13 Jul 2002 08:23:25 GMT</last_modification_time>
+	      </signal>
+	      <columns>1</columns>
+	      <column_widths>80</column_widths>
+	      <selection_mode>GTK_SELECTION_SINGLE</selection_mode>
+	      <show_titles>False</show_titles>
+	      <shadow_type>GTK_SHADOW_IN</shadow_type>
+
+	      <widget>
+		<class>GtkLabel</class>
+		<child_name>CList:title</child_name>
+		<name>label18</name>
+		<label>Users</label>
+		<justify>GTK_JUSTIFY_CENTER</justify>
+		<wrap>False</wrap>
+		<xalign>0.5</xalign>
+		<yalign>0.5</yalign>
+		<xpad>0</xpad>
+		<ypad>0</ypad>
+	      </widget>
+	    </widget>
+	  </widget>
+
+	  <widget>
+	    <class>GtkFrame</class>
+	    <name>frame10</name>
+	    <label>Group</label>
+	    <label_xalign>0</label_xalign>
+	    <shadow_type>GTK_SHADOW_ETCHED_IN</shadow_type>
+	    <child>
+	      <padding>0</padding>
+	      <expand>False</expand>
+	      <fill>False</fill>
+	    </child>
+
+	    <widget>
+	      <class>GtkVBox</class>
+	      <name>GroupActionsBox</name>
+	      <homogeneous>False</homogeneous>
+	      <spacing>0</spacing>
+
+	      <widget>
+		<class>Placeholder</class>
+	      </widget>
+
+	      <widget>
+		<class>Placeholder</class>
+	      </widget>
+
+	      <widget>
+		<class>Placeholder</class>
+	      </widget>
+	    </widget>
+	  </widget>
+
+	  <widget>
+	    <class>GtkFrame</class>
+	    <name>PersonFrame</name>
+	    <label>Person</label>
+	    <label_xalign>0</label_xalign>
+	    <shadow_type>GTK_SHADOW_ETCHED_IN</shadow_type>
+	    <child>
+	      <padding>0</padding>
+	      <expand>False</expand>
+	      <fill>False</fill>
+	    </child>
+
+	    <widget>
+	      <class>GtkVBox</class>
+	      <name>PersonActionsBox</name>
+	      <homogeneous>False</homogeneous>
+	      <spacing>0</spacing>
+
+	      <widget>
+		<class>Placeholder</class>
+	      </widget>
+
+	      <widget>
+		<class>Placeholder</class>
+	      </widget>
+
+	      <widget>
+		<class>Placeholder</class>
+	      </widget>
+	    </widget>
+	  </widget>
+	</widget>
+      </widget>
+
+      <widget>
+	<class>GtkHBox</class>
+	<name>hbox6</name>
+	<homogeneous>False</homogeneous>
+	<spacing>0</spacing>
+	<child>
+	  <shrink>True</shrink>
+	  <resize>False</resize>
+	</child>
+
+	<widget>
+	  <class>GtkLabel</class>
+	  <name>NickLabel</name>
+	  <label>&lt;no nick&gt;</label>
+	  <justify>GTK_JUSTIFY_CENTER</justify>
+	  <wrap>False</wrap>
+	  <xalign>0.5</xalign>
+	  <yalign>0.5</yalign>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <child>
+	    <padding>4</padding>
+	    <expand>False</expand>
+	    <fill>False</fill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkScrolledWindow</class>
+	  <name>scrolledwindow9</name>
+	  <hscrollbar_policy>GTK_POLICY_NEVER</hscrollbar_policy>
+	  <vscrollbar_policy>GTK_POLICY_AUTOMATIC</vscrollbar_policy>
+	  <hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	  <vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	  <child>
+	    <padding>0</padding>
+	    <expand>True</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkText</class>
+	    <name>GroupInput</name>
+	    <can_focus>True</can_focus>
+	    <has_focus>True</has_focus>
+	    <signal>
+	      <name>key_press_event</name>
+	      <handler>handle_key_press_event</handler>
+	      <last_modification_time>Tue, 29 Jan 2002 12:41:03 GMT</last_modification_time>
+	    </signal>
+	    <editable>True</editable>
+	    <text></text>
+	  </widget>
+	</widget>
+      </widget>
+    </widget>
+  </widget>
+</widget>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>NewAccountWindow</name>
+  <border_width>3</border_width>
+  <visible>False</visible>
+  <signal>
+    <name>destroy</name>
+    <handler>on_NewAccountWindow_destroy</handler>
+    <last_modification_time>Sun, 27 Jan 2002 10:35:19 GMT</last_modification_time>
+  </signal>
+  <title>New Account</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>False</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>True</auto_shrink>
+
+  <widget>
+    <class>GtkVBox</class>
+    <name>vbox17</name>
+    <homogeneous>False</homogeneous>
+    <spacing>0</spacing>
+
+    <widget>
+      <class>GtkHBox</class>
+      <name>hbox11</name>
+      <homogeneous>False</homogeneous>
+      <spacing>0</spacing>
+      <child>
+	<padding>3</padding>
+	<expand>False</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkLabel</class>
+	<name>label49</name>
+	<label>Gateway:</label>
+	<justify>GTK_JUSTIFY_CENTER</justify>
+	<wrap>False</wrap>
+	<xalign>0.5</xalign>
+	<yalign>0.5</yalign>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<child>
+	  <padding>0</padding>
+	  <expand>False</expand>
+	  <fill>True</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkOptionMenu</class>
+	<name>GatewayOptionMenu</name>
+	<can_focus>True</can_focus>
+	<items>Twisted (Perspective Broker)
+Internet Relay Chat
+AIM (TOC)
+AIM (OSCAR)
+</items>
+	<initial_choice>0</initial_choice>
+	<child>
+	  <padding>4</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkFrame</class>
+      <name>GatewayFrame</name>
+      <border_width>3</border_width>
+      <label>Gateway Options</label>
+      <label_xalign>0</label_xalign>
+      <shadow_type>GTK_SHADOW_ETCHED_IN</shadow_type>
+      <child>
+	<padding>0</padding>
+	<expand>True</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>Placeholder</class>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkFrame</class>
+      <name>frame2</name>
+      <border_width>3</border_width>
+      <label>Standard Options</label>
+      <label_xalign>0</label_xalign>
+      <shadow_type>GTK_SHADOW_ETCHED_IN</shadow_type>
+      <child>
+	<padding>0</padding>
+	<expand>False</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkTable</class>
+	<name>table1</name>
+	<border_width>3</border_width>
+	<rows>2</rows>
+	<columns>2</columns>
+	<homogeneous>False</homogeneous>
+	<row_spacing>0</row_spacing>
+	<column_spacing>0</column_spacing>
+
+	<widget>
+	  <class>GtkCheckButton</class>
+	  <name>AutoLogin</name>
+	  <can_focus>True</can_focus>
+	  <label>Automatically Log In</label>
+	  <active>False</active>
+	  <draw_indicator>True</draw_indicator>
+	  <child>
+	    <left_attach>1</left_attach>
+	    <right_attach>2</right_attach>
+	    <top_attach>0</top_attach>
+	    <bottom_attach>1</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>True</xexpand>
+	    <yexpand>True</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>False</yfill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkEntry</class>
+	  <name>accountName</name>
+	  <can_focus>True</can_focus>
+	  <editable>True</editable>
+	  <text_visible>True</text_visible>
+	  <text_max_length>0</text_max_length>
+	  <text></text>
+	  <child>
+	    <left_attach>1</left_attach>
+	    <right_attach>2</right_attach>
+	    <top_attach>1</top_attach>
+	    <bottom_attach>2</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>True</xexpand>
+	    <yexpand>True</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>False</yfill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkLabel</class>
+	  <name>label50</name>
+	  <label>   Auto-Login: </label>
+	  <justify>GTK_JUSTIFY_RIGHT</justify>
+	  <wrap>False</wrap>
+	  <xalign>0</xalign>
+	  <yalign>0.5</yalign>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <child>
+	    <left_attach>0</left_attach>
+	    <right_attach>1</right_attach>
+	    <top_attach>0</top_attach>
+	    <bottom_attach>1</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>False</xexpand>
+	    <yexpand>True</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>True</yfill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkLabel</class>
+	  <name>label51</name>
+	  <label>Account Name: </label>
+	  <justify>GTK_JUSTIFY_RIGHT</justify>
+	  <wrap>False</wrap>
+	  <xalign>0</xalign>
+	  <yalign>0.5</yalign>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <child>
+	    <left_attach>0</left_attach>
+	    <right_attach>1</right_attach>
+	    <top_attach>1</top_attach>
+	    <bottom_attach>2</bottom_attach>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <xexpand>False</xexpand>
+	    <yexpand>True</yexpand>
+	    <xshrink>False</xshrink>
+	    <yshrink>False</yshrink>
+	    <xfill>True</xfill>
+	    <yfill>True</yfill>
+	  </child>
+	</widget>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkHButtonBox</class>
+      <name>hbuttonbox2</name>
+      <layout_style>GTK_BUTTONBOX_SPREAD</layout_style>
+      <spacing>30</spacing>
+      <child_min_width>85</child_min_width>
+      <child_min_height>27</child_min_height>
+      <child_ipad_x>7</child_ipad_x>
+      <child_ipad_y>0</child_ipad_y>
+      <child>
+	<padding>0</padding>
+	<expand>False</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkButton</class>
+	<name>button50</name>
+	<can_default>True</can_default>
+	<can_focus>True</can_focus>
+	<signal>
+	  <name>clicked</name>
+	  <handler>createAccount</handler>
+	  <last_modification_time>Sun, 27 Jan 2002 11:25:05 GMT</last_modification_time>
+	</signal>
+	<label>OK</label>
+	<relief>GTK_RELIEF_NORMAL</relief>
+      </widget>
+
+      <widget>
+	<class>GtkButton</class>
+	<name>button51</name>
+	<can_default>True</can_default>
+	<can_focus>True</can_focus>
+	<signal>
+	  <name>clicked</name>
+	  <handler>destroyMe</handler>
+	  <last_modification_time>Sun, 27 Jan 2002 11:27:12 GMT</last_modification_time>
+	</signal>
+	<label>Cancel</label>
+	<relief>GTK_RELIEF_NORMAL</relief>
+      </widget>
+    </widget>
+  </widget>
+</widget>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>PBAccountWindow</name>
+  <visible>False</visible>
+  <title>PB Account Window</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>False</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>False</auto_shrink>
+
+  <widget>
+    <class>GtkVBox</class>
+    <name>PBAccountWidget</name>
+    <border_width>4</border_width>
+    <homogeneous>False</homogeneous>
+    <spacing>0</spacing>
+
+    <widget>
+      <class>GtkTable</class>
+      <name>table3</name>
+      <rows>4</rows>
+      <columns>2</columns>
+      <homogeneous>False</homogeneous>
+      <row_spacing>0</row_spacing>
+      <column_spacing>0</column_spacing>
+      <child>
+	<padding>0</padding>
+	<expand>False</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkEntry</class>
+	<name>hostname</name>
+	<can_focus>True</can_focus>
+	<editable>True</editable>
+	<text_visible>True</text_visible>
+	<text_max_length>0</text_max_length>
+	<text>twistedmatrix.com</text>
+	<child>
+	  <left_attach>1</left_attach>
+	  <right_attach>2</right_attach>
+	  <top_attach>2</top_attach>
+	  <bottom_attach>3</bottom_attach>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <xexpand>True</xexpand>
+	  <yexpand>False</yexpand>
+	  <xshrink>False</xshrink>
+	  <yshrink>False</yshrink>
+	  <xfill>True</xfill>
+	  <yfill>False</yfill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkEntry</class>
+	<name>identity</name>
+	<can_focus>True</can_focus>
+	<has_focus>True</has_focus>
+	<signal>
+	  <name>changed</name>
+	  <handler>on_identity_changed</handler>
+	  <last_modification_time>Sun, 27 Jan 2002 11:52:17 GMT</last_modification_time>
+	</signal>
+	<editable>True</editable>
+	<text_visible>True</text_visible>
+	<text_max_length>0</text_max_length>
+	<text></text>
+	<child>
+	  <left_attach>1</left_attach>
+	  <right_attach>2</right_attach>
+	  <top_attach>0</top_attach>
+	  <bottom_attach>1</bottom_attach>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <xexpand>True</xexpand>
+	  <yexpand>False</yexpand>
+	  <xshrink>False</xshrink>
+	  <yshrink>False</yshrink>
+	  <xfill>True</xfill>
+	  <yfill>False</yfill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkLabel</class>
+	<name>label52</name>
+	<label>     Hostname: </label>
+	<justify>GTK_JUSTIFY_RIGHT</justify>
+	<wrap>False</wrap>
+	<xalign>0</xalign>
+	<yalign>0.5</yalign>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<child>
+	  <left_attach>0</left_attach>
+	  <right_attach>1</right_attach>
+	  <top_attach>2</top_attach>
+	  <bottom_attach>3</bottom_attach>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <xexpand>False</xexpand>
+	  <yexpand>False</yexpand>
+	  <xshrink>False</xshrink>
+	  <yshrink>False</yshrink>
+	  <xfill>True</xfill>
+	  <yfill>False</yfill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkLabel</class>
+	<name>label54</name>
+	<label>Identity Name: </label>
+	<justify>GTK_JUSTIFY_RIGHT</justify>
+	<wrap>False</wrap>
+	<xalign>0</xalign>
+	<yalign>0.5</yalign>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<child>
+	  <left_attach>0</left_attach>
+	  <right_attach>1</right_attach>
+	  <top_attach>0</top_attach>
+	  <bottom_attach>1</bottom_attach>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <xexpand>False</xexpand>
+	  <yexpand>False</yexpand>
+	  <xshrink>False</xshrink>
+	  <yshrink>False</yshrink>
+	  <xfill>True</xfill>
+	  <yfill>False</yfill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkEntry</class>
+	<name>password</name>
+	<can_focus>True</can_focus>
+	<editable>True</editable>
+	<text_visible>False</text_visible>
+	<text_max_length>0</text_max_length>
+	<text></text>
+	<child>
+	  <left_attach>1</left_attach>
+	  <right_attach>2</right_attach>
+	  <top_attach>1</top_attach>
+	  <bottom_attach>2</bottom_attach>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <xexpand>True</xexpand>
+	  <yexpand>False</yexpand>
+	  <xshrink>False</xshrink>
+	  <yshrink>False</yshrink>
+	  <xfill>True</xfill>
+	  <yfill>False</yfill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkEntry</class>
+	<name>portno</name>
+	<can_focus>True</can_focus>
+	<editable>True</editable>
+	<text_visible>True</text_visible>
+	<text_max_length>0</text_max_length>
+	<text>8787</text>
+	<child>
+	  <left_attach>1</left_attach>
+	  <right_attach>2</right_attach>
+	  <top_attach>3</top_attach>
+	  <bottom_attach>4</bottom_attach>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <xexpand>True</xexpand>
+	  <yexpand>False</yexpand>
+	  <xshrink>False</xshrink>
+	  <yshrink>False</yshrink>
+	  <xfill>True</xfill>
+	  <yfill>False</yfill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkLabel</class>
+	<name>label55</name>
+	<label>     Password: </label>
+	<justify>GTK_JUSTIFY_RIGHT</justify>
+	<wrap>False</wrap>
+	<xalign>0</xalign>
+	<yalign>0.5</yalign>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<child>
+	  <left_attach>0</left_attach>
+	  <right_attach>1</right_attach>
+	  <top_attach>1</top_attach>
+	  <bottom_attach>2</bottom_attach>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <xexpand>False</xexpand>
+	  <yexpand>False</yexpand>
+	  <xshrink>False</xshrink>
+	  <yshrink>False</yshrink>
+	  <xfill>True</xfill>
+	  <yfill>False</yfill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkLabel</class>
+	<name>label53</name>
+	<label>  Port Number: </label>
+	<justify>GTK_JUSTIFY_RIGHT</justify>
+	<wrap>False</wrap>
+	<xalign>0</xalign>
+	<yalign>0.5</yalign>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<child>
+	  <left_attach>0</left_attach>
+	  <right_attach>1</right_attach>
+	  <top_attach>3</top_attach>
+	  <bottom_attach>4</bottom_attach>
+	  <xpad>0</xpad>
+	  <ypad>0</ypad>
+	  <xexpand>False</xexpand>
+	  <yexpand>False</yexpand>
+	  <xshrink>False</xshrink>
+	  <yshrink>False</yshrink>
+	  <xfill>True</xfill>
+	  <yfill>False</yfill>
+	</child>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkFrame</class>
+      <name>frame3</name>
+      <label>Perspectives</label>
+      <label_xalign>0</label_xalign>
+      <shadow_type>GTK_SHADOW_ETCHED_IN</shadow_type>
+      <child>
+	<padding>0</padding>
+	<expand>True</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkVBox</class>
+	<name>vbox19</name>
+	<border_width>3</border_width>
+	<homogeneous>False</homogeneous>
+	<spacing>0</spacing>
+
+	<widget>
+	  <class>GtkScrolledWindow</class>
+	  <name>scrolledwindow13</name>
+	  <hscrollbar_policy>GTK_POLICY_AUTOMATIC</hscrollbar_policy>
+	  <vscrollbar_policy>GTK_POLICY_ALWAYS</vscrollbar_policy>
+	  <hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	  <vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	  <child>
+	    <padding>0</padding>
+	    <expand>True</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkCList</class>
+	    <name>serviceList</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>select_row</name>
+	      <handler>on_serviceList_select_row</handler>
+	      <last_modification_time>Sun, 27 Jan 2002 12:04:38 GMT</last_modification_time>
+	    </signal>
+	    <columns>3</columns>
+	    <column_widths>80,80,80</column_widths>
+	    <selection_mode>GTK_SELECTION_SINGLE</selection_mode>
+	    <show_titles>True</show_titles>
+	    <shadow_type>GTK_SHADOW_IN</shadow_type>
+
+	    <widget>
+	      <class>GtkLabel</class>
+	      <child_name>CList:title</child_name>
+	      <name>label60</name>
+	      <label>Service Type</label>
+	      <justify>GTK_JUSTIFY_CENTER</justify>
+	      <wrap>False</wrap>
+	      <xalign>0.5</xalign>
+	      <yalign>0.5</yalign>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	    </widget>
+
+	    <widget>
+	      <class>GtkLabel</class>
+	      <child_name>CList:title</child_name>
+	      <name>label61</name>
+	      <label>Service Name</label>
+	      <justify>GTK_JUSTIFY_CENTER</justify>
+	      <wrap>False</wrap>
+	      <xalign>0.5</xalign>
+	      <yalign>0.5</yalign>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	    </widget>
+
+	    <widget>
+	      <class>GtkLabel</class>
+	      <child_name>CList:title</child_name>
+	      <name>label62</name>
+	      <label>Perspective Name</label>
+	      <justify>GTK_JUSTIFY_CENTER</justify>
+	      <wrap>False</wrap>
+	      <xalign>0.5</xalign>
+	      <yalign>0.5</yalign>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	    </widget>
+	  </widget>
+	</widget>
+
+	<widget>
+	  <class>GtkTable</class>
+	  <name>table4</name>
+	  <rows>3</rows>
+	  <columns>2</columns>
+	  <homogeneous>False</homogeneous>
+	  <row_spacing>0</row_spacing>
+	  <column_spacing>0</column_spacing>
+	  <child>
+	    <padding>0</padding>
+	    <expand>False</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <name>label63</name>
+	    <label>Perspective Name: </label>
+	    <justify>GTK_JUSTIFY_RIGHT</justify>
+	    <wrap>False</wrap>
+	    <xalign>0</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <child>
+	      <left_attach>0</left_attach>
+	      <right_attach>1</right_attach>
+	      <top_attach>2</top_attach>
+	      <bottom_attach>3</bottom_attach>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	      <xexpand>False</xexpand>
+	      <yexpand>False</yexpand>
+	      <xshrink>False</xshrink>
+	      <yshrink>False</yshrink>
+	      <xfill>True</xfill>
+	      <yfill>False</yfill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <name>label59</name>
+	    <label>    Service Type: </label>
+	    <justify>GTK_JUSTIFY_RIGHT</justify>
+	    <wrap>False</wrap>
+	    <xalign>0</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <child>
+	      <left_attach>0</left_attach>
+	      <right_attach>1</right_attach>
+	      <top_attach>0</top_attach>
+	      <bottom_attach>1</bottom_attach>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	      <xexpand>False</xexpand>
+	      <yexpand>False</yexpand>
+	      <xshrink>False</xshrink>
+	      <yshrink>False</yshrink>
+	      <xfill>True</xfill>
+	      <yfill>False</yfill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkCombo</class>
+	    <name>serviceCombo</name>
+	    <value_in_list>False</value_in_list>
+	    <ok_if_empty>True</ok_if_empty>
+	    <case_sensitive>False</case_sensitive>
+	    <use_arrows>True</use_arrows>
+	    <use_arrows_always>False</use_arrows_always>
+	    <items>twisted.words
+twisted.reality
+</items>
+	    <child>
+	      <left_attach>1</left_attach>
+	      <right_attach>2</right_attach>
+	      <top_attach>0</top_attach>
+	      <bottom_attach>1</bottom_attach>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	      <xexpand>True</xexpand>
+	      <yexpand>False</yexpand>
+	      <xshrink>False</xshrink>
+	      <yshrink>False</yshrink>
+	      <xfill>True</xfill>
+	      <yfill>False</yfill>
+	    </child>
+
+	    <widget>
+	      <class>GtkEntry</class>
+	      <child_name>GtkCombo:entry</child_name>
+	      <name>serviceType</name>
+	      <can_focus>True</can_focus>
+	      <signal>
+		<name>changed</name>
+		<handler>on_serviceType_changed</handler>
+		<last_modification_time>Sun, 27 Jan 2002 11:49:07 GMT</last_modification_time>
+	      </signal>
+	      <editable>True</editable>
+	      <text_visible>True</text_visible>
+	      <text_max_length>0</text_max_length>
+	      <text>twisted.words</text>
+	    </widget>
+	  </widget>
+
+	  <widget>
+	    <class>GtkLabel</class>
+	    <name>label64</name>
+	    <label>    Service Name: </label>
+	    <justify>GTK_JUSTIFY_RIGHT</justify>
+	    <wrap>False</wrap>
+	    <xalign>0</xalign>
+	    <yalign>0.5</yalign>
+	    <xpad>0</xpad>
+	    <ypad>0</ypad>
+	    <child>
+	      <left_attach>0</left_attach>
+	      <right_attach>1</right_attach>
+	      <top_attach>1</top_attach>
+	      <bottom_attach>2</bottom_attach>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	      <xexpand>False</xexpand>
+	      <yexpand>False</yexpand>
+	      <xshrink>False</xshrink>
+	      <yshrink>False</yshrink>
+	      <xfill>True</xfill>
+	      <yfill>False</yfill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkEntry</class>
+	    <name>serviceName</name>
+	    <can_focus>True</can_focus>
+	    <editable>True</editable>
+	    <text_visible>True</text_visible>
+	    <text_max_length>0</text_max_length>
+	    <text></text>
+	    <child>
+	      <left_attach>1</left_attach>
+	      <right_attach>2</right_attach>
+	      <top_attach>1</top_attach>
+	      <bottom_attach>2</bottom_attach>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	      <xexpand>True</xexpand>
+	      <yexpand>False</yexpand>
+	      <xshrink>False</xshrink>
+	      <yshrink>False</yshrink>
+	      <xfill>True</xfill>
+	      <yfill>False</yfill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkEntry</class>
+	    <name>perspectiveName</name>
+	    <can_focus>True</can_focus>
+	    <editable>True</editable>
+	    <text_visible>True</text_visible>
+	    <text_max_length>0</text_max_length>
+	    <text></text>
+	    <child>
+	      <left_attach>1</left_attach>
+	      <right_attach>2</right_attach>
+	      <top_attach>2</top_attach>
+	      <bottom_attach>3</bottom_attach>
+	      <xpad>0</xpad>
+	      <ypad>0</ypad>
+	      <xexpand>True</xexpand>
+	      <yexpand>False</yexpand>
+	      <xshrink>False</xshrink>
+	      <yshrink>False</yshrink>
+	      <xfill>True</xfill>
+	      <yfill>False</yfill>
+	    </child>
+	  </widget>
+	</widget>
+
+	<widget>
+	  <class>GtkHBox</class>
+	  <name>hbox13</name>
+	  <homogeneous>False</homogeneous>
+	  <spacing>0</spacing>
+	  <child>
+	    <padding>0</padding>
+	    <expand>False</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkButton</class>
+	    <name>button53</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>clicked</name>
+	      <handler>addPerspective</handler>
+	      <last_modification_time>Mon, 28 Jan 2002 01:07:15 GMT</last_modification_time>
+	    </signal>
+	    <label> Add Perspective </label>
+	    <relief>GTK_RELIEF_NORMAL</relief>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>False</fill>
+	    </child>
+	  </widget>
+
+	  <widget>
+	    <class>GtkButton</class>
+	    <name>button54</name>
+	    <can_focus>True</can_focus>
+	    <signal>
+	      <name>clicked</name>
+	      <handler>removePerspective</handler>
+	      <last_modification_time>Sun, 27 Jan 2002 11:34:36 GMT</last_modification_time>
+	    </signal>
+	    <label>Remove Perspective</label>
+	    <relief>GTK_RELIEF_NORMAL</relief>
+	    <child>
+	      <padding>0</padding>
+	      <expand>True</expand>
+	      <fill>False</fill>
+	    </child>
+	  </widget>
+	</widget>
+      </widget>
+    </widget>
+  </widget>
+</widget>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>IRCAccountWindow</name>
+  <title>IRC Account Window</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>False</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>False</auto_shrink>
+
+  <widget>
+    <class>GtkTable</class>
+    <name>IRCAccountWidget</name>
+    <rows>5</rows>
+    <columns>2</columns>
+    <homogeneous>False</homogeneous>
+    <row_spacing>0</row_spacing>
+    <column_spacing>0</column_spacing>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label65</name>
+      <label> Nickname: </label>
+      <justify>GTK_JUSTIFY_RIGHT</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>0</top_attach>
+	<bottom_attach>1</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label66</name>
+      <label>   Server: </label>
+      <justify>GTK_JUSTIFY_RIGHT</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>1</top_attach>
+	<bottom_attach>2</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label67</name>
+      <label>     Port: </label>
+      <justify>GTK_JUSTIFY_RIGHT</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>2</top_attach>
+	<bottom_attach>3</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label68</name>
+      <label> Channels: </label>
+      <justify>GTK_JUSTIFY_RIGHT</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>3</top_attach>
+	<bottom_attach>4</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label69</name>
+      <label> Password: </label>
+      <justify>GTK_JUSTIFY_RIGHT</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>4</top_attach>
+	<bottom_attach>5</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>ircNick</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>True</text_visible>
+      <text_max_length>0</text_max_length>
+      <text></text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>0</top_attach>
+	<bottom_attach>1</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>ircServer</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>True</text_visible>
+      <text_max_length>0</text_max_length>
+      <text></text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>1</top_attach>
+	<bottom_attach>2</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>ircPort</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>True</text_visible>
+      <text_max_length>0</text_max_length>
+      <text>6667</text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>2</top_attach>
+	<bottom_attach>3</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>ircChannels</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>True</text_visible>
+      <text_max_length>0</text_max_length>
+      <text></text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>3</top_attach>
+	<bottom_attach>4</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>ircPassword</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>True</text_visible>
+      <text_max_length>0</text_max_length>
+      <text></text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>4</top_attach>
+	<bottom_attach>5</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+  </widget>
+</widget>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>TOCAccountWindow</name>
+  <title>TOC Account Window</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>False</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>False</auto_shrink>
+
+  <widget>
+    <class>GtkTable</class>
+    <name>TOCAccountWidget</name>
+    <rows>4</rows>
+    <columns>2</columns>
+    <homogeneous>False</homogeneous>
+    <row_spacing>0</row_spacing>
+    <column_spacing>0</column_spacing>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label70</name>
+      <label> Screen Name: </label>
+      <justify>GTK_JUSTIFY_CENTER</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>0</top_attach>
+	<bottom_attach>1</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label71</name>
+      <label>    Password: </label>
+      <justify>GTK_JUSTIFY_CENTER</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>1</top_attach>
+	<bottom_attach>2</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label72</name>
+      <label>        Host: </label>
+      <justify>GTK_JUSTIFY_CENTER</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>2</top_attach>
+	<bottom_attach>3</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkLabel</class>
+      <name>label73</name>
+      <label>        Port: </label>
+      <justify>GTK_JUSTIFY_CENTER</justify>
+      <wrap>False</wrap>
+      <xalign>0</xalign>
+      <yalign>0.5</yalign>
+      <xpad>0</xpad>
+      <ypad>0</ypad>
+      <child>
+	<left_attach>0</left_attach>
+	<right_attach>1</right_attach>
+	<top_attach>3</top_attach>
+	<bottom_attach>4</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>False</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>TOCName</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>True</text_visible>
+      <text_max_length>0</text_max_length>
+      <text></text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>0</top_attach>
+	<bottom_attach>1</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>TOCPass</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>False</text_visible>
+      <text_max_length>0</text_max_length>
+      <text></text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>1</top_attach>
+	<bottom_attach>2</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>TOCHost</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>True</text_visible>
+      <text_max_length>0</text_max_length>
+      <text>toc.oscar.aol.com</text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>2</top_attach>
+	<bottom_attach>3</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkEntry</class>
+      <name>TOCPort</name>
+      <can_focus>True</can_focus>
+      <editable>True</editable>
+      <text_visible>True</text_visible>
+      <text_max_length>0</text_max_length>
+      <text>9898</text>
+      <child>
+	<left_attach>1</left_attach>
+	<right_attach>2</right_attach>
+	<top_attach>3</top_attach>
+	<bottom_attach>4</bottom_attach>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<xexpand>True</xexpand>
+	<yexpand>False</yexpand>
+	<xshrink>False</xshrink>
+	<yshrink>False</yshrink>
+	<xfill>True</xfill>
+	<yfill>False</yfill>
+      </child>
+    </widget>
+  </widget>
+</widget>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>JoinGroupWindow</name>
+  <border_width>5</border_width>
+  <visible>False</visible>
+  <title>Group to Join</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>False</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>False</auto_shrink>
+
+  <widget>
+    <class>GtkVBox</class>
+    <name>vbox20</name>
+    <homogeneous>False</homogeneous>
+    <spacing>0</spacing>
+
+    <widget>
+      <class>GtkOptionMenu</class>
+      <name>AccountSelector</name>
+      <can_focus>True</can_focus>
+      <items>None
+In
+Particular
+</items>
+      <initial_choice>0</initial_choice>
+      <child>
+	<padding>0</padding>
+	<expand>False</expand>
+	<fill>False</fill>
+      </child>
+    </widget>
+
+    <widget>
+      <class>GtkHBox</class>
+      <name>hbox15</name>
+      <homogeneous>False</homogeneous>
+      <spacing>5</spacing>
+      <child>
+	<padding>0</padding>
+	<expand>True</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkEntry</class>
+	<name>GroupNameEntry</name>
+	<can_focus>True</can_focus>
+	<has_focus>True</has_focus>
+	<signal>
+	  <name>activate</name>
+	  <handler>on_GroupJoinButton_clicked</handler>
+	  <last_modification_time>Tue, 29 Jan 2002 13:27:18 GMT</last_modification_time>
+	</signal>
+	<editable>True</editable>
+	<text_visible>True</text_visible>
+	<text_max_length>0</text_max_length>
+	<text></text>
+	<child>
+	  <padding>0</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkButton</class>
+	<name>GroupJoinButton</name>
+	<can_default>True</can_default>
+	<has_default>True</has_default>
+	<can_focus>True</can_focus>
+	<signal>
+	  <name>clicked</name>
+	  <handler>on_GroupJoinButton_clicked</handler>
+	  <last_modification_time>Tue, 29 Jan 2002 13:16:50 GMT</last_modification_time>
+	</signal>
+	<label>Join</label>
+	<relief>GTK_RELIEF_NORMAL</relief>
+	<child>
+	  <padding>0</padding>
+	  <expand>False</expand>
+	  <fill>False</fill>
+	</child>
+      </widget>
+    </widget>
+  </widget>
+</widget>
+
+<widget>
+  <class>GtkWindow</class>
+  <name>UnifiedWindow</name>
+  <title>Twisted Instance Messenger</title>
+  <type>GTK_WINDOW_TOPLEVEL</type>
+  <position>GTK_WIN_POS_NONE</position>
+  <modal>False</modal>
+  <allow_shrink>False</allow_shrink>
+  <allow_grow>True</allow_grow>
+  <auto_shrink>False</auto_shrink>
+
+  <widget>
+    <class>GtkVBox</class>
+    <name>vbox25</name>
+    <homogeneous>False</homogeneous>
+    <spacing>0</spacing>
+
+    <widget>
+      <class>GtkHBox</class>
+      <name>hbox28</name>
+      <homogeneous>False</homogeneous>
+      <spacing>0</spacing>
+      <child>
+	<padding>0</padding>
+	<expand>False</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkButton</class>
+	<name>button74</name>
+	<can_focus>True</can_focus>
+	<label>&gt;</label>
+	<relief>GTK_RELIEF_NORMAL</relief>
+	<child>
+	  <padding>0</padding>
+	  <expand>False</expand>
+	  <fill>False</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkEntry</class>
+	<name>entry3</name>
+	<can_focus>True</can_focus>
+	<editable>True</editable>
+	<text_visible>True</text_visible>
+	<text_max_length>0</text_max_length>
+	<text></text>
+	<child>
+	  <padding>0</padding>
+	  <expand>True</expand>
+	  <fill>True</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkOptionMenu</class>
+	<name>optionmenu3</name>
+	<items>List
+Of
+Online
+Accounts
+</items>
+	<initial_choice>0</initial_choice>
+	<child>
+	  <padding>0</padding>
+	  <expand>False</expand>
+	  <fill>False</fill>
+	</child>
+      </widget>
+
+      <widget>
+	<class>GtkOptionMenu</class>
+	<name>optionmenu4</name>
+	<can_focus>True</can_focus>
+	<items>Contact
+Person
+Group
+Account
+</items>
+	<initial_choice>0</initial_choice>
+	<child>
+	  <padding>0</padding>
+	  <expand>False</expand>
+	  <fill>False</fill>
+	</child>
+      </widget>
+    </widget>
+
+    <widget>
+      <class>GtkHPaned</class>
+      <name>hpaned1</name>
+      <handle_size>10</handle_size>
+      <gutter_size>6</gutter_size>
+      <position>0</position>
+      <child>
+	<padding>0</padding>
+	<expand>True</expand>
+	<fill>True</fill>
+      </child>
+
+      <widget>
+	<class>GtkVBox</class>
+	<name>vbox26</name>
+	<homogeneous>False</homogeneous>
+	<spacing>0</spacing>
+	<child>
+	  <shrink>True</shrink>
+	  <resize>False</resize>
+	</child>
+
+	<widget>
+	  <class>GtkFrame</class>
+	  <name>frame7</name>
+	  <border_width>2</border_width>
+	  <label>Accounts</label>
+	  <label_xalign>0</label_xalign>
+	  <shadow_type>GTK_SHADOW_ETCHED_IN</shadow_type>
+	  <child>
+	    <padding>0</padding>
+	    <expand>True</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkVBox</class>
+	    <name>vbox27</name>
+	    <homogeneous>False</homogeneous>
+	    <spacing>0</spacing>
+
+	    <widget>
+	      <class>GtkScrolledWindow</class>
+	      <name>scrolledwindow18</name>
+	      <hscrollbar_policy>GTK_POLICY_AUTOMATIC</hscrollbar_policy>
+	      <vscrollbar_policy>GTK_POLICY_AUTOMATIC</vscrollbar_policy>
+	      <hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	      <vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	      <child>
+		<padding>0</padding>
+		<expand>True</expand>
+		<fill>True</fill>
+	      </child>
+
+	      <widget>
+		<class>GtkCList</class>
+		<name>clist4</name>
+		<columns>4</columns>
+		<column_widths>18,25,25,80</column_widths>
+		<selection_mode>GTK_SELECTION_SINGLE</selection_mode>
+		<show_titles>False</show_titles>
+		<shadow_type>GTK_SHADOW_IN</shadow_type>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label95</name>
+		  <label>label87</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label96</name>
+		  <label>label88</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label97</name>
+		  <label>label89</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label98</name>
+		  <label>label90</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+	      </widget>
+	    </widget>
+
+	    <widget>
+	      <class>GtkHBox</class>
+	      <name>hbox23</name>
+	      <homogeneous>True</homogeneous>
+	      <spacing>2</spacing>
+	      <child>
+		<padding>0</padding>
+		<expand>True</expand>
+		<fill>True</fill>
+	      </child>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button65</name>
+		<label>New</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>True</expand>
+		  <fill>True</fill>
+		</child>
+	      </widget>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button66</name>
+		<label>Delete</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>True</expand>
+		  <fill>True</fill>
+		</child>
+	      </widget>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button67</name>
+		<label>Connect</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>True</expand>
+		  <fill>True</fill>
+		</child>
+	      </widget>
+	    </widget>
+	  </widget>
+	</widget>
+
+	<widget>
+	  <class>GtkFrame</class>
+	  <name>frame8</name>
+	  <border_width>2</border_width>
+	  <label>Contacts</label>
+	  <label_xalign>0</label_xalign>
+	  <shadow_type>GTK_SHADOW_ETCHED_IN</shadow_type>
+	  <child>
+	    <padding>0</padding>
+	    <expand>True</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkVBox</class>
+	    <name>vbox28</name>
+	    <homogeneous>False</homogeneous>
+	    <spacing>0</spacing>
+
+	    <widget>
+	      <class>GtkScrolledWindow</class>
+	      <name>scrolledwindow19</name>
+	      <hscrollbar_policy>GTK_POLICY_AUTOMATIC</hscrollbar_policy>
+	      <vscrollbar_policy>GTK_POLICY_AUTOMATIC</vscrollbar_policy>
+	      <hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	      <vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	      <child>
+		<padding>0</padding>
+		<expand>True</expand>
+		<fill>True</fill>
+	      </child>
+
+	      <widget>
+		<class>GtkCList</class>
+		<name>clist5</name>
+		<columns>3</columns>
+		<column_widths>18,17,80</column_widths>
+		<selection_mode>GTK_SELECTION_SINGLE</selection_mode>
+		<show_titles>False</show_titles>
+		<shadow_type>GTK_SHADOW_IN</shadow_type>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label99</name>
+		  <label>label84</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label100</name>
+		  <label>label85</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label101</name>
+		  <label>label86</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+	      </widget>
+	    </widget>
+
+	    <widget>
+	      <class>GtkHBox</class>
+	      <name>hbox24</name>
+	      <homogeneous>True</homogeneous>
+	      <spacing>2</spacing>
+	      <child>
+		<padding>0</padding>
+		<expand>False</expand>
+		<fill>True</fill>
+	      </child>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button68</name>
+		<can_focus>True</can_focus>
+		<label>Talk</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>True</expand>
+		  <fill>True</fill>
+		</child>
+	      </widget>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button69</name>
+		<can_focus>True</can_focus>
+		<label>Info</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>True</expand>
+		  <fill>True</fill>
+		</child>
+	      </widget>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button70</name>
+		<can_focus>True</can_focus>
+		<label>Add</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>True</expand>
+		  <fill>True</fill>
+		</child>
+	      </widget>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button71</name>
+		<can_focus>True</can_focus>
+		<label>Remove</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>False</expand>
+		  <fill>False</fill>
+		</child>
+	      </widget>
+	    </widget>
+	  </widget>
+	</widget>
+
+	<widget>
+	  <class>GtkFrame</class>
+	  <name>frame9</name>
+	  <border_width>2</border_width>
+	  <label>Groups</label>
+	  <label_xalign>0</label_xalign>
+	  <shadow_type>GTK_SHADOW_ETCHED_IN</shadow_type>
+	  <child>
+	    <padding>0</padding>
+	    <expand>True</expand>
+	    <fill>True</fill>
+	  </child>
+
+	  <widget>
+	    <class>GtkVBox</class>
+	    <name>vbox29</name>
+	    <homogeneous>False</homogeneous>
+	    <spacing>0</spacing>
+
+	    <widget>
+	      <class>GtkScrolledWindow</class>
+	      <name>scrolledwindow20</name>
+	      <hscrollbar_policy>GTK_POLICY_AUTOMATIC</hscrollbar_policy>
+	      <vscrollbar_policy>GTK_POLICY_AUTOMATIC</vscrollbar_policy>
+	      <hupdate_policy>GTK_UPDATE_CONTINUOUS</hupdate_policy>
+	      <vupdate_policy>GTK_UPDATE_CONTINUOUS</vupdate_policy>
+	      <child>
+		<padding>0</padding>
+		<expand>True</expand>
+		<fill>True</fill>
+	      </child>
+
+	      <widget>
+		<class>GtkCList</class>
+		<name>clist6</name>
+		<columns>3</columns>
+		<column_widths>21,75,80</column_widths>
+		<selection_mode>GTK_SELECTION_SINGLE</selection_mode>
+		<show_titles>False</show_titles>
+		<shadow_type>GTK_SHADOW_IN</shadow_type>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label102</name>
+		  <label>label91</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label103</name>
+		  <label>label92</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+
+		<widget>
+		  <class>GtkLabel</class>
+		  <child_name>CList:title</child_name>
+		  <name>label104</name>
+		  <label>label93</label>
+		  <justify>GTK_JUSTIFY_CENTER</justify>
+		  <wrap>False</wrap>
+		  <xalign>0.5</xalign>
+		  <yalign>0.5</yalign>
+		  <xpad>0</xpad>
+		  <ypad>0</ypad>
+		</widget>
+	      </widget>
+	    </widget>
+
+	    <widget>
+	      <class>GtkHBox</class>
+	      <name>hbox27</name>
+	      <homogeneous>True</homogeneous>
+	      <spacing>2</spacing>
+	      <child>
+		<padding>0</padding>
+		<expand>False</expand>
+		<fill>True</fill>
+	      </child>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button72</name>
+		<label>Join</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>True</expand>
+		  <fill>True</fill>
+		</child>
+	      </widget>
+
+	      <widget>
+		<class>GtkButton</class>
+		<name>button73</name>
+		<label>Leave</label>
+		<relief>GTK_RELIEF_NORMAL</relief>
+		<child>
+		  <padding>0</padding>
+		  <expand>True</expand>
+		  <fill>True</fill>
+		</child>
+	      </widget>
+	    </widget>
+	  </widget>
+	</widget>
+
+	<widget>
+	  <class>GtkHSeparator</class>
+	  <name>hseparator2</name>
+	  <child>
+	    <padding>0</padding>
+	    <expand>True</expand>
+	    <fill>True</fill>
+	  </child>
+	</widget>
+
+	<widget>
+	  <class>GtkLabel</class>
+	  <name>label105</name>
+	  <label>Twisted IM V. %s</label>
+	  <justify>GTK_JUSTIFY_CENTER</justify>
+	  <wrap>False</wrap>
+	  <xalign>0.5</xalign>
+	  <yalign>0.5</yalign>
+	  <xpad>0</xpad>
+	  <ypad>3</ypad>
+	  <child>
+	    <padding>0</padding>
+	    <expand>False</expand>
+	    <fill>False</fill>
+	  </child>
+	</widget>
+      </widget>
+
+      <widget>
+	<class>GtkLabel</class>
+	<name>label106</name>
+	<label>This
+Space
+Left
+Intentionally
+Blank
+(Here is where the UI for the currently
+selected element
+for interaction
+will go.)</label>
+	<justify>GTK_JUSTIFY_CENTER</justify>
+	<wrap>False</wrap>
+	<xalign>0.5</xalign>
+	<yalign>0.5</yalign>
+	<xpad>0</xpad>
+	<ypad>0</ypad>
+	<child>
+	  <shrink>True</shrink>
+	  <resize>True</resize>
+	</child>
+      </widget>
+    </widget>
+  </widget>
+</widget>
+
+</GTK-Interface>
diff --git a/contrib/python/Twisted/py3/twisted/words/im/interfaces.py b/contrib/python/Twisted/py3/twisted/words/im/interfaces.py
new file mode 100644
index 00000000000..14fc92624ee
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/interfaces.py
@@ -0,0 +1,362 @@
+# -*- Python -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Pan-protocol chat client.
+"""
+
+from zope.interface import Attribute, Interface
+
+# (Random musings, may not reflect on current state of code:)
+#
+# Accounts have Protocol components (clients)
+# Persons have Conversation components
+# Groups have GroupConversation components
+# Persons and Groups are associated with specific Accounts
+# At run-time, Clients/Accounts are slaved to a User Interface
+#   (Note: User may be a bot, so don't assume all UIs are built on gui toolkits)
+
+
+class IAccount(Interface):
+    """
+    I represent a user's account with a chat service.
+    """
+
+    client = Attribute("The L{IClient} currently connecting to this account, if any.")
+    gatewayType = Attribute(
+        "A C{str} that identifies the protocol used by this account."
+    )
+
+    def __init__(accountName, autoLogin, username, password, host, port):
+        """
+        @type accountName: string
+        @param accountName: A name to refer to the account by locally.
+        @type autoLogin: boolean
+        @type username: string
+        @type password: string
+        @type host: string
+        @type port: integer
+        """
+
+    def isOnline():
+        """
+        Am I online?
+
+        @rtype: boolean
+        """
+
+    def logOn(chatui):
+        """
+        Go on-line.
+
+        @type chatui: Implementor of C{IChatUI}
+
+        @rtype: L{Deferred} with an eventual L{IClient} result.
+        """
+
+    def logOff():
+        """
+        Sign off.
+        """
+
+    def getGroup(groupName):
+        """
+        @rtype: L{Group<IGroup>}
+        """
+
+    def getPerson(personName):
+        """
+        @rtype: L{Person<IPerson>}
+        """
+
+
+class IClient(Interface):
+    account = Attribute("The L{IAccount} I am a Client for")
+
+    def __init__(account, chatui, logonDeferred):
+        """
+        @type account: L{IAccount}
+        @type chatui: L{IChatUI}
+        @param logonDeferred: Will be called back once I am logged on.
+        @type logonDeferred: L{Deferred<twisted.internet.defer.Deferred>}
+        """
+
+    def joinGroup(groupName):
+        """
+        @param groupName: The name of the group to join.
+        @type groupName: string
+        """
+
+    def leaveGroup(groupName):
+        """
+        @param groupName: The name of the group to leave.
+        @type groupName: string
+        """
+
+    def getGroupConversation(name, hide=0):
+        pass
+
+    def getPerson(name):
+        pass
+
+
+class IPerson(Interface):
+    def __init__(name, account):
+        """
+        Initialize me.
+
+        @param name: My name, as the server knows me.
+        @type name: string
+        @param account: The account I am accessed through.
+        @type account: I{Account}
+        """
+
+    def isOnline():
+        """
+        Am I online right now?
+
+        @rtype: boolean
+        """
+
+    def getStatus():
+        """
+        What is my on-line status?
+
+        @return: L{locals.StatusEnum}
+        """
+
+    def getIdleTime():
+        """
+        @rtype: string (XXX: How about a scalar?)
+        """
+
+    def sendMessage(text, metadata=None):
+        """
+        Send a message to this person.
+
+        @type text: string
+        @type metadata: dict
+        """
+
+
+class IGroup(Interface):
+    """
+    A group which you may have a conversation with.
+
+    Groups generally have a loosely-defined set of members, who may
+    leave and join at any time.
+    """
+
+    name = Attribute("My C{str} name, as the server knows me.")
+    account = Attribute("The L{Account<IAccount>} I am accessed through.")
+
+    def __init__(name, account):
+        """
+        Initialize me.
+
+        @param name: My name, as the server knows me.
+        @type name: str
+        @param account: The account I am accessed through.
+        @type account: L{Account<IAccount>}
+        """
+
+    def setTopic(text):
+        """
+        Set this Groups topic on the server.
+
+        @type text: string
+        """
+
+    def sendGroupMessage(text, metadata=None):
+        """
+        Send a message to this group.
+
+        @type text: str
+
+        @type metadata: dict
+        @param metadata: Valid keys for this dictionary include:
+
+            - C{'style'}: associated with one of:
+                - C{'emote'}: indicates this is an action
+        """
+
+    def join():
+        """
+        Join this group.
+        """
+
+    def leave():
+        """
+        Depart this group.
+        """
+
+
+class IConversation(Interface):
+    """
+    A conversation with a specific person.
+    """
+
+    def __init__(person, chatui):
+        """
+        @type person: L{IPerson}
+        """
+
+    def show():
+        """
+        doesn't seem like it belongs in this interface.
+        """
+
+    def hide():
+        """
+        nor this neither.
+        """
+
+    def sendText(text, metadata):
+        pass
+
+    def showMessage(text, metadata):
+        pass
+
+    def changedNick(person, newnick):
+        """
+        @param person: XXX Shouldn't this always be Conversation.person?
+        """
+
+
+class IGroupConversation(Interface):
+    def show():
+        """
+        doesn't seem like it belongs in this interface.
+        """
+
+    def hide():
+        """
+        nor this neither.
+        """
+
+    def sendText(text, metadata):
+        pass
+
+    def showGroupMessage(sender, text, metadata):
+        pass
+
+    def setGroupMembers(members):
+        """
+        Sets the list of members in the group and displays it to the user.
+        """
+
+    def setTopic(topic, author):
+        """
+        Displays the topic (from the server) for the group conversation window.
+
+        @type topic: string
+        @type author: string (XXX: Not Person?)
+        """
+
+    def memberJoined(member):
+        """
+        Adds the given member to the list of members in the group conversation
+        and displays this to the user,
+
+        @type member: string (XXX: Not Person?)
+        """
+
+    def memberChangedNick(oldnick, newnick):
+        """
+        Changes the oldnick in the list of members to C{newnick} and displays this
+        change to the user,
+
+        @type oldnick: string (XXX: Not Person?)
+        @type newnick: string
+        """
+
+    def memberLeft(member):
+        """
+        Deletes the given member from the list of members in the group
+        conversation and displays the change to the user.
+
+        @type member: string (XXX: Not Person?)
+        """
+
+
+class IChatUI(Interface):
+    def registerAccountClient(client):
+        """
+        Notifies user that an account has been signed on to.
+
+        @type client: L{Client<IClient>}
+        """
+
+    def unregisterAccountClient(client):
+        """
+        Notifies user that an account has been signed off or disconnected.
+
+        @type client: L{Client<IClient>}
+        """
+
+    def getContactsList():
+        """
+        @rtype: L{ContactsList}
+        """
+
+    # WARNING: You'll want to be polymorphed into something with
+    # intrinsic stoning resistance before continuing.
+
+    def getConversation(person, Class, stayHidden=0):
+        """
+        For the given person object, returns the conversation window
+        or creates and returns a new conversation window if one does not exist.
+
+        @type person: L{Person<IPerson>}
+        @type Class: L{Conversation<IConversation>} class
+        @type stayHidden: boolean
+
+        @rtype: L{Conversation<IConversation>}
+        """
+
+    def getGroupConversation(group, Class, stayHidden=0):
+        """
+        For the given group object, returns the group conversation window or
+        creates and returns a new group conversation window if it doesn't exist.
+
+        @type group: L{Group<interfaces.IGroup>}
+        @type Class: L{Conversation<interfaces.IConversation>} class
+        @type stayHidden: boolean
+
+        @rtype: L{GroupConversation<interfaces.IGroupConversation>}
+        """
+
+    def getPerson(name, client):
+        """
+        Get a Person for a client.
+
+        Duplicates L{IAccount.getPerson}.
+
+        @type name: string
+        @type client: L{Client<IClient>}
+
+        @rtype: L{Person<IPerson>}
+        """
+
+    def getGroup(name, client):
+        """
+        Get a Group for a client.
+
+        Duplicates L{IAccount.getGroup}.
+
+        @type name: string
+        @type client: L{Client<IClient>}
+
+        @rtype: L{Group<IGroup>}
+        """
+
+    def contactChangedNick(oldnick, newnick):
+        """
+        For the given person, changes the person's name to newnick, and
+        tells the contact list and any conversation windows with that person
+        to change as well.
+
+        @type oldnick: string
+        @type newnick: string
+        """
diff --git a/contrib/python/Twisted/py3/twisted/words/im/ircsupport.py b/contrib/python/Twisted/py3/twisted/words/im/ircsupport.py
new file mode 100644
index 00000000000..452587465ab
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/ircsupport.py
@@ -0,0 +1,273 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+IRC support for Instance Messenger.
+"""
+
+from zope.interface import implementer
+
+from twisted.internet import defer, protocol, reactor
+from twisted.internet.defer import succeed
+from twisted.words.im import basesupport, interfaces, locals
+from twisted.words.im.locals import ONLINE
+from twisted.words.protocols import irc
+
+
+class IRCPerson(basesupport.AbstractPerson):
+    def imperson_whois(self):
+        if self.account.client is None:
+            raise locals.OfflineError
+        self.account.client.sendLine("WHOIS %s" % self.name)
+
+    ### interface impl
+    def isOnline(self):
+        return ONLINE
+
+    def getStatus(self):
+        return ONLINE
+
+    def setStatus(self, status):
+        self.status = status
+        self.chat.getContactsList().setContactStatus(self)
+
+    def sendMessage(self, text, meta=None):
+        if self.account.client is None:
+            raise locals.OfflineError
+        for line in text.split("\n"):
+            if meta and meta.get("style", None) == "emote":
+                self.account.client.ctcpMakeQuery(self.name, [("ACTION", line)])
+            else:
+                self.account.client.msg(self.name, line)
+        return succeed(text)
+
+
+@implementer(interfaces.IGroup)
+class IRCGroup(basesupport.AbstractGroup):
+    def imgroup_testAction(self):
+        pass
+
+    def imtarget_kick(self, target):
+        if self.account.client is None:
+            raise locals.OfflineError
+        reason = "for great justice!"
+        self.account.client.sendLine(f"KICK #{self.name} {target.name} :{reason}")
+
+    ### Interface Implementation
+    def setTopic(self, topic):
+        if self.account.client is None:
+            raise locals.OfflineError
+        self.account.client.topic(self.name, topic)
+
+    def sendGroupMessage(self, text, meta={}):
+        if self.account.client is None:
+            raise locals.OfflineError
+        if meta and meta.get("style", None) == "emote":
+            self.account.client.ctcpMakeQuery(self.name, [("ACTION", text)])
+            return succeed(text)
+        # standard shmandard, clients don't support plain escaped newlines!
+        for line in text.split("\n"):
+            self.account.client.say(self.name, line)
+        return succeed(text)
+
+    def leave(self):
+        if self.account.client is None:
+            raise locals.OfflineError
+        self.account.client.leave(self.name)
+        self.account.client.getGroupConversation(self.name, 1)
+
+
+class IRCProto(basesupport.AbstractClientMixin, irc.IRCClient):
+    def __init__(self, account, chatui, logonDeferred=None):
+        basesupport.AbstractClientMixin.__init__(self, account, chatui, logonDeferred)
+        self._namreplies = {}
+        self._ingroups = {}
+        self._groups = {}
+        self._topics = {}
+
+    def getGroupConversation(self, name, hide=0):
+        name = name.lower()
+        return self.chat.getGroupConversation(
+            self.chat.getGroup(name, self), stayHidden=hide
+        )
+
+    def getPerson(self, name):
+        return self.chat.getPerson(name, self)
+
+    def connectionMade(self):
+        # XXX: Why do I duplicate code in IRCClient.register?
+        try:
+            self.performLogin = True
+            self.nickname = self.account.username
+            self.password = self.account.password
+            self.realname = "Twisted-IM user"
+
+            irc.IRCClient.connectionMade(self)
+
+            for channel in self.account.channels:
+                self.joinGroup(channel)
+            self.account._isOnline = 1
+            if self._logonDeferred is not None:
+                self._logonDeferred.callback(self)
+            self.chat.getContactsList()
+        except BaseException:
+            import traceback
+
+            traceback.print_exc()
+
+    def setNick(self, nick):
+        self.name = nick
+        self.accountName = "%s (IRC)" % nick
+        irc.IRCClient.setNick(self, nick)
+
+    def kickedFrom(self, channel, kicker, message):
+        """
+        Called when I am kicked from a channel.
+        """
+        return self.chat.getGroupConversation(self.chat.getGroup(channel[1:], self), 1)
+
+    def userKicked(self, kickee, channel, kicker, message):
+        pass
+
+    def noticed(self, username, channel, message):
+        self.privmsg(username, channel, message, {"dontAutoRespond": 1})
+
+    def privmsg(self, username, channel, message, metadata=None):
+        if metadata is None:
+            metadata = {}
+        username = username.split("!", 1)[0]
+        if username == self.name:
+            return
+        if channel[0] == "#":
+            group = channel[1:]
+            self.getGroupConversation(group).showGroupMessage(
+                username, message, metadata
+            )
+            return
+        self.chat.getConversation(self.getPerson(username)).showMessage(
+            message, metadata
+        )
+
+    def action(self, username, channel, emote):
+        username = username.split("!", 1)[0]
+        if username == self.name:
+            return
+        meta = {"style": "emote"}
+        if channel[0] == "#":
+            group = channel[1:]
+            self.getGroupConversation(group).showGroupMessage(username, emote, meta)
+            return
+        self.chat.getConversation(self.getPerson(username)).showMessage(emote, meta)
+
+    def irc_RPL_NAMREPLY(self, prefix, params):
+        """
+        RPL_NAMREPLY
+        >> NAMES #bnl
+        << :Arlington.VA.US.Undernet.Org 353 z3p = #bnl :pSwede Dan-- SkOyg AG
+        """
+        group = params[2][1:].lower()
+        users = params[3].split()
+        for ui in range(len(users)):
+            while users[ui][0] in ["@", "+"]:  # channel modes
+                users[ui] = users[ui][1:]
+        if group not in self._namreplies:
+            self._namreplies[group] = []
+        self._namreplies[group].extend(users)
+        for nickname in users:
+            try:
+                self._ingroups[nickname].append(group)
+            except BaseException:
+                self._ingroups[nickname] = [group]
+
+    def irc_RPL_ENDOFNAMES(self, prefix, params):
+        group = params[1][1:]
+        self.getGroupConversation(group).setGroupMembers(
+            self._namreplies[group.lower()]
+        )
+        del self._namreplies[group.lower()]
+
+    def irc_RPL_TOPIC(self, prefix, params):
+        self._topics[params[1][1:]] = params[2]
+
+    def irc_333(self, prefix, params):
+        group = params[1][1:]
+        self.getGroupConversation(group).setTopic(self._topics[group], params[2])
+        del self._topics[group]
+
+    def irc_TOPIC(self, prefix, params):
+        nickname = prefix.split("!")[0]
+        group = params[0][1:]
+        topic = params[1]
+        self.getGroupConversation(group).setTopic(topic, nickname)
+
+    def irc_JOIN(self, prefix, params):
+        nickname = prefix.split("!")[0]
+        group = params[0][1:].lower()
+        if nickname != self.nickname:
+            try:
+                self._ingroups[nickname].append(group)
+            except BaseException:
+                self._ingroups[nickname] = [group]
+            self.getGroupConversation(group).memberJoined(nickname)
+
+    def irc_PART(self, prefix, params):
+        nickname = prefix.split("!")[0]
+        group = params[0][1:].lower()
+        if nickname != self.nickname:
+            if group in self._ingroups[nickname]:
+                self._ingroups[nickname].remove(group)
+                self.getGroupConversation(group).memberLeft(nickname)
+
+    def irc_QUIT(self, prefix, params):
+        nickname = prefix.split("!")[0]
+        if nickname in self._ingroups:
+            for group in self._ingroups[nickname]:
+                self.getGroupConversation(group).memberLeft(nickname)
+            self._ingroups[nickname] = []
+
+    def irc_NICK(self, prefix, params):
+        fromNick = prefix.split("!")[0]
+        toNick = params[0]
+        if fromNick not in self._ingroups:
+            return
+        for group in self._ingroups[fromNick]:
+            self.getGroupConversation(group).memberChangedNick(fromNick, toNick)
+        self._ingroups[toNick] = self._ingroups[fromNick]
+        del self._ingroups[fromNick]
+
+    def irc_unknown(self, prefix, command, params):
+        pass
+
+    # GTKIM calls
+    def joinGroup(self, name):
+        self.join(name)
+        self.getGroupConversation(name)
+
+
+@implementer(interfaces.IAccount)
+class IRCAccount(basesupport.AbstractAccount):
+    gatewayType = "IRC"
+
+    _groupFactory = IRCGroup
+    _personFactory = IRCPerson
+
+    def __init__(
+        self, accountName, autoLogin, username, password, host, port, channels=""
+    ):
+        basesupport.AbstractAccount.__init__(
+            self, accountName, autoLogin, username, password, host, port
+        )
+        self.channels = [channel.strip() for channel in channels.split(",")]
+        if self.channels == [""]:
+            self.channels = []
+
+    def _startLogOn(self, chatui):
+        logonDeferred = defer.Deferred()
+        cc = protocol.ClientCreator(reactor, IRCProto, self, chatui, logonDeferred)
+        d = cc.connectTCP(self.host, self.port)
+        d.addErrback(logonDeferred.errback)
+        return logonDeferred
+
+    def logOff(self):
+        # IAccount.logOff
+        pass
diff --git a/contrib/python/Twisted/py3/twisted/words/im/locals.py b/contrib/python/Twisted/py3/twisted/words/im/locals.py
new file mode 100644
index 00000000000..2e63088ac04
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/locals.py
@@ -0,0 +1,30 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from typing import Optional
+
+
+class Enum:
+    group: Optional[str] = None
+
+    def __init__(self, label: str) -> None:
+        self.label = label
+
+    def __repr__(self) -> str:
+        return f"<{self.group}: {self.label}>"
+
+    def __str__(self) -> str:
+        return self.label
+
+
+class StatusEnum(Enum):
+    group = "Status"
+
+
+OFFLINE = Enum("Offline")
+ONLINE = Enum("Online")
+AWAY = Enum("Away")
+
+
+class OfflineError(Exception):
+    """The requested action can't happen while offline."""
diff --git a/contrib/python/Twisted/py3/twisted/words/im/pbsupport.py b/contrib/python/Twisted/py3/twisted/words/im/pbsupport.py
new file mode 100644
index 00000000000..3be8931e6e2
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/im/pbsupport.py
@@ -0,0 +1,278 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+L{twisted.words} support for Instance Messenger.
+"""
+
+
+from zope.interface import implementer
+
+from twisted.internet import defer, error
+from twisted.python import log
+from twisted.python.failure import Failure
+from twisted.spread import pb
+from twisted.words.im import basesupport, interfaces
+from twisted.words.im.locals import AWAY, OFFLINE, ONLINE
+
+
+class TwistedWordsPerson(basesupport.AbstractPerson):
+    """I a facade for a person you can talk to through a twisted.words service."""
+
+    def __init__(self, name, wordsAccount):
+        basesupport.AbstractPerson.__init__(self, name, wordsAccount)
+        self.status = OFFLINE
+
+    def isOnline(self):
+        return (self.status == ONLINE) or (self.status == AWAY)
+
+    def getStatus(self):
+        return self.status
+
+    def sendMessage(self, text, metadata):
+        """Return a deferred..."""
+        if metadata:
+            d = self.account.client.perspective.directMessage(self.name, text, metadata)
+            d.addErrback(self.metadataFailed, "* " + text)
+            return d
+        else:
+            return self.account.client.perspective.callRemote(
+                "directMessage", self.name, text
+            )
+
+    def metadataFailed(self, result, text):
+        print("result:", result, "text:", text)
+        return self.account.client.perspective.directMessage(self.name, text)
+
+    def setStatus(self, status):
+        self.status = status
+        self.chat.getContactsList().setContactStatus(self)
+
+
+@implementer(interfaces.IGroup)
+class TwistedWordsGroup(basesupport.AbstractGroup):
+    def __init__(self, name, wordsClient):
+        basesupport.AbstractGroup.__init__(self, name, wordsClient)
+        self.joined = 0
+
+    def sendGroupMessage(self, text, metadata=None):
+        """Return a deferred."""
+        # for backwards compatibility with older twisted.words servers.
+        if metadata:
+            d = self.account.client.perspective.callRemote(
+                "groupMessage", self.name, text, metadata
+            )
+            d.addErrback(self.metadataFailed, "* " + text)
+            return d
+        else:
+            return self.account.client.perspective.callRemote(
+                "groupMessage", self.name, text
+            )
+
+    def setTopic(self, text):
+        self.account.client.perspective.callRemote(
+            "setGroupMetadata",
+            {"topic": text, "topic_author": self.client.name},
+            self.name,
+        )
+
+    def metadataFailed(self, result, text):
+        print("result:", result, "text:", text)
+        return self.account.client.perspective.callRemote(
+            "groupMessage", self.name, text
+        )
+
+    def joining(self):
+        self.joined = 1
+
+    def leaving(self):
+        self.joined = 0
+
+    def leave(self):
+        return self.account.client.perspective.callRemote("leaveGroup", self.name)
+
+
+class TwistedWordsClient(pb.Referenceable, basesupport.AbstractClientMixin):
+    """In some cases, this acts as an Account, since it a source of text
+    messages (multiple Words instances may be on a single PB connection)
+    """
+
+    def __init__(self, acct, serviceName, perspectiveName, chatui, _logonDeferred=None):
+        self.accountName = "{} ({}:{})".format(
+            acct.accountName,
+            serviceName,
+            perspectiveName,
+        )
+        self.name = perspectiveName
+        print("HELLO I AM A PB SERVICE", serviceName, perspectiveName)
+        self.chat = chatui
+        self.account = acct
+        self._logonDeferred = _logonDeferred
+
+    def getPerson(self, name):
+        return self.chat.getPerson(name, self)
+
+    def getGroup(self, name):
+        return self.chat.getGroup(name, self)
+
+    def getGroupConversation(self, name):
+        return self.chat.getGroupConversation(self.getGroup(name))
+
+    def addContact(self, name):
+        self.perspective.callRemote("addContact", name)
+
+    def remote_receiveGroupMembers(self, names, group):
+        print("received group members:", names, group)
+        self.getGroupConversation(group).setGroupMembers(names)
+
+    def remote_receiveGroupMessage(self, sender, group, message, metadata=None):
+        print("received a group message", sender, group, message, metadata)
+        self.getGroupConversation(group).showGroupMessage(sender, message, metadata)
+
+    def remote_memberJoined(self, member, group):
+        print("member joined", member, group)
+        self.getGroupConversation(group).memberJoined(member)
+
+    def remote_memberLeft(self, member, group):
+        print("member left")
+        self.getGroupConversation(group).memberLeft(member)
+
+    def remote_notifyStatusChanged(self, name, status):
+        self.chat.getPerson(name, self).setStatus(status)
+
+    def remote_receiveDirectMessage(self, name, message, metadata=None):
+        self.chat.getConversation(self.chat.getPerson(name, self)).showMessage(
+            message, metadata
+        )
+
+    def remote_receiveContactList(self, clist):
+        for name, status in clist:
+            self.chat.getPerson(name, self).setStatus(status)
+
+    def remote_setGroupMetadata(self, dict_, groupName):
+        if "topic" in dict_:
+            self.getGroupConversation(groupName).setTopic(
+                dict_["topic"], dict_.get("topic_author", None)
+            )
+
+    def joinGroup(self, name):
+        self.getGroup(name).joining()
+        return self.perspective.callRemote("joinGroup", name).addCallback(
+            self._cbGroupJoined, name
+        )
+
+    def leaveGroup(self, name):
+        self.getGroup(name).leaving()
+        return self.perspective.callRemote("leaveGroup", name).addCallback(
+            self._cbGroupLeft, name
+        )
+
+    def _cbGroupJoined(self, result, name):
+        groupConv = self.chat.getGroupConversation(self.getGroup(name))
+        groupConv.showGroupMessage("sys", "you joined")
+        self.perspective.callRemote("getGroupMembers", name)
+
+    def _cbGroupLeft(self, result, name):
+        print("left", name)
+        groupConv = self.chat.getGroupConversation(self.getGroup(name), 1)
+        groupConv.showGroupMessage("sys", "you left")
+
+    def connected(self, perspective):
+        print("Connected Words Client!", perspective)
+        if self._logonDeferred is not None:
+            self._logonDeferred.callback(self)
+        self.perspective = perspective
+        self.chat.getContactsList()
+
+
+pbFrontEnds = {"twisted.words": TwistedWordsClient, "twisted.reality": None}
+
+
+@implementer(interfaces.IAccount)
+class PBAccount(basesupport.AbstractAccount):
+    gatewayType = "PB"
+    _groupFactory = TwistedWordsGroup
+    _personFactory = TwistedWordsPerson
+
+    def __init__(
+        self, accountName, autoLogin, username, password, host, port, services=None
+    ):
+        """
+        @param username: The name of your PB Identity.
+        @type username: string
+        """
+        basesupport.AbstractAccount.__init__(
+            self, accountName, autoLogin, username, password, host, port
+        )
+        self.services = []
+        if not services:
+            services = [("twisted.words", "twisted.words", username)]
+        for serviceType, serviceName, perspectiveName in services:
+            self.services.append(
+                [pbFrontEnds[serviceType], serviceName, perspectiveName]
+            )
+
+    def logOn(self, chatui):
+        """
+        @returns: this breaks with L{interfaces.IAccount}
+        @returntype: DeferredList of L{interfaces.IClient}s
+        """
+        # Overriding basesupport's implementation on account of the
+        # fact that _startLogOn tends to return a deferredList rather
+        # than a simple Deferred, and we need to do registerAccountClient.
+        if (not self._isConnecting) and (not self._isOnline):
+            self._isConnecting = 1
+            d = self._startLogOn(chatui)
+            d.addErrback(self._loginFailed)
+
+            def registerMany(results):
+                for success, result in results:
+                    if success:
+                        chatui.registerAccountClient(result)
+                        self._cb_logOn(result)
+                    else:
+                        log.err(result)
+
+            d.addCallback(registerMany)
+            return d
+        else:
+            raise error.ConnectionError("Connection in progress")
+
+    def logOff(self):
+        # IAccount.logOff
+        pass
+
+    def _startLogOn(self, chatui):
+        print("Connecting...", end=" ")
+        d = pb.getObjectAt(self.host, self.port)
+        d.addCallbacks(self._cbConnected, self._ebConnected, callbackArgs=(chatui,))
+        return d
+
+    def _cbConnected(self, root, chatui):
+        print("Connected!")
+        print("Identifying...", end=" ")
+        d = pb.authIdentity(root, self.username, self.password)
+        d.addCallbacks(self._cbIdent, self._ebConnected, callbackArgs=(chatui,))
+        return d
+
+    def _cbIdent(self, ident, chatui):
+        if not ident:
+            print("falsely identified.")
+            return self._ebConnected(
+                Failure(Exception("username or password incorrect"))
+            )
+        print("Identified!")
+        dl = []
+        for handlerClass, sname, pname in self.services:
+            d = defer.Deferred()
+            dl.append(d)
+            handler = handlerClass(self, sname, pname, chatui, d)
+            ident.callRemote("attach", sname, pname, handler).addCallback(
+                handler.connected
+            )
+        return defer.DeferredList(dl)
+
+    def _ebConnected(self, error):
+        print("Not connected.")
+        return error
diff --git a/contrib/python/Twisted/py3/twisted/words/iwords.py b/contrib/python/Twisted/py3/twisted/words/iwords.py
new file mode 100644
index 00000000000..f1e71ca56ee
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/iwords.py
@@ -0,0 +1,281 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from zope.interface import Attribute, Interface
+
+
+class IProtocolPlugin(Interface):
+    """Interface for plugins providing an interface to a Words service"""
+
+    name = Attribute(
+        "A single word describing what kind of interface this is (eg, irc or web)"
+    )
+
+    def getFactory(realm, portal):
+        """Retrieve a C{twisted.internet.interfaces.IServerFactory} provider
+
+        @param realm: An object providing C{twisted.cred.portal.IRealm} and
+        L{IChatService}, with which service information should be looked up.
+
+        @param portal: An object providing C{twisted.cred.portal.IPortal},
+        through which logins should be performed.
+        """
+
+
+class IGroup(Interface):
+    name = Attribute("A short string, unique among groups.")
+
+    def add(user):
+        """Include the given user in this group.
+
+        @type user: L{IUser}
+        """
+
+    def remove(user, reason=None):
+        """Remove the given user from this group.
+
+        @type user: L{IUser}
+        @type reason: C{unicode}
+        """
+
+    def size():
+        """Return the number of participants in this group.
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with an C{int} representing the
+        number of participants in this group.
+        """
+
+    def receive(sender, recipient, message):
+        """
+        Broadcast the given message from the given sender to other
+        users in group.
+
+        The message is not re-transmitted to the sender.
+
+        @param sender: L{IUser}
+
+        @type recipient: L{IGroup}
+        @param recipient: This is probably a wart.  Maybe it will be removed
+        in the future.  For now, it should be the group object the message
+        is being delivered to.
+
+        @param message: C{dict}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with None when delivery has been
+        attempted for all users.
+        """
+
+    def setMetadata(meta):
+        """Change the metadata associated with this group.
+
+        @type meta: C{dict}
+        """
+
+    def iterusers():
+        """Return an iterator of all users in this group."""
+
+
+class IChatClient(Interface):
+    """Interface through which IChatService interacts with clients."""
+
+    name = Attribute(
+        "A short string, unique among users.  This will be set by the L{IChatService} at login time."
+    )
+
+    def receive(sender, recipient, message):
+        """
+        Callback notifying this user of the given message sent by the
+        given user.
+
+        This will be invoked whenever another user sends a message to a
+        group this user is participating in, or whenever another user sends
+        a message directly to this user.  In the former case, C{recipient}
+        will be the group to which the message was sent; in the latter, it
+        will be the same object as the user who is receiving the message.
+
+        @type sender: L{IUser}
+        @type recipient: L{IUser} or L{IGroup}
+        @type message: C{dict}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires when the message has been delivered,
+        or which fails in some way.  If the Deferred fails and the message
+        was directed at a group, this user will be removed from that group.
+        """
+
+    def groupMetaUpdate(group, meta):
+        """
+        Callback notifying this user that the metadata for the given
+        group has changed.
+
+        @type group: L{IGroup}
+        @type meta: C{dict}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+    def userJoined(group, user):
+        """
+        Callback notifying this user that the given user has joined
+        the given group.
+
+        @type group: L{IGroup}
+        @type user: L{IUser}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+    def userLeft(group, user, reason=None):
+        """
+        Callback notifying this user that the given user has left the
+        given group for the given reason.
+
+        @type group: L{IGroup}
+        @type user: L{IUser}
+        @type reason: C{unicode}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+
+class IUser(Interface):
+    """Interface through which clients interact with IChatService."""
+
+    realm = Attribute(
+        "A reference to the Realm to which this user belongs.  Set if and only if the user is logged in."
+    )
+    mind = Attribute(
+        "A reference to the mind which logged in to this user.  Set if and only if the user is logged in."
+    )
+    name = Attribute("A short string, unique among users.")
+
+    lastMessage = Attribute(
+        "A POSIX timestamp indicating the time of the last message received from this user."
+    )
+    signOn = Attribute(
+        "A POSIX timestamp indicating this user's most recent sign on time."
+    )
+
+    def loggedIn(realm, mind):
+        """Invoked by the associated L{IChatService} when login occurs.
+
+        @param realm: The L{IChatService} through which login is occurring.
+        @param mind: The mind object used for cred login.
+        """
+
+    def send(recipient, message):
+        """Send the given message to the given user or group.
+
+        @type recipient: Either L{IUser} or L{IGroup}
+        @type message: C{dict}
+        """
+
+    def join(group):
+        """Attempt to join the given group.
+
+        @type group: L{IGroup}
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+    def leave(group):
+        """Discontinue participation in the given group.
+
+        @type group: L{IGroup}
+        @rtype: L{twisted.internet.defer.Deferred}
+        """
+
+    def itergroups():
+        """
+        Return an iterator of all groups of which this user is a
+        member.
+        """
+
+
+class IChatService(Interface):
+    name = Attribute("A short string identifying this chat service (eg, a hostname)")
+
+    createGroupOnRequest = Attribute(
+        "A boolean indicating whether L{getGroup} should implicitly "
+        "create groups which are requested but which do not yet exist."
+    )
+
+    createUserOnRequest = Attribute(
+        "A boolean indicating whether L{getUser} should implicitly "
+        "create users which are requested but which do not yet exist."
+    )
+
+    def itergroups():
+        """Return all groups available on this service.
+
+        @rtype: C{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with a list of C{IGroup} providers.
+        """
+
+    def getGroup(name):
+        """Retrieve the group by the given name.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the group with the given
+        name if one exists (or if one is created due to the setting of
+        L{IChatService.createGroupOnRequest}, or which fails with
+        L{twisted.words.ewords.NoSuchGroup} if no such group exists.
+        """
+
+    def createGroup(name):
+        """Create a new group with the given name.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the created group, or
+        with fails with L{twisted.words.ewords.DuplicateGroup} if a
+        group by that name exists already.
+        """
+
+    def lookupGroup(name):
+        """Retrieve a group by name.
+
+        Unlike C{getGroup}, this will never implicitly create a group.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the group by the given
+        name, or which fails with L{twisted.words.ewords.NoSuchGroup}.
+        """
+
+    def getUser(name):
+        """Retrieve the user by the given name.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the user with the given
+        name if one exists (or if one is created due to the setting of
+        L{IChatService.createUserOnRequest}, or which fails with
+        L{twisted.words.ewords.NoSuchUser} if no such user exists.
+        """
+
+    def createUser(name):
+        """Create a new user with the given name.
+
+        @type name: C{str}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with the created user, or
+        with fails with L{twisted.words.ewords.DuplicateUser} if a
+        user by that name exists already.
+        """
+
+
+__all__ = [
+    "IGroup",
+    "IChatClient",
+    "IUser",
+    "IChatService",
+]
diff --git a/contrib/python/Twisted/py3/twisted/words/newsfragments/.gitignore b/contrib/python/Twisted/py3/twisted/words/newsfragments/.gitignore
new file mode 100644
index 00000000000..f935021a8f8
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/newsfragments/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/__init__.py b/contrib/python/Twisted/py3/twisted/words/protocols/__init__.py
new file mode 100644
index 00000000000..59dc76c0f6a
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Chat protocols.
+"""
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/irc.py b/contrib/python/Twisted/py3/twisted/words/protocols/irc.py
new file mode 100644
index 00000000000..c4ec04579f1
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/irc.py
@@ -0,0 +1,4118 @@
+# -*- test-case-name: twisted.words.test.test_irc -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Internet Relay Chat protocol for client and server.
+
+Future Plans
+============
+
+The way the IRCClient class works here encourages people to implement
+IRC clients by subclassing the ephemeral protocol class, and it tends
+to end up with way more state than it should for an object which will
+be destroyed as soon as the TCP transport drops.  Someone oughta do
+something about that, ya know?
+
+The DCC support needs to have more hooks for the client for it to be
+able to ask the user things like "Do you want to accept this session?"
+and "Transfer #2 is 67% done." and otherwise manage the DCC sessions.
+
+Test coverage needs to be better.
+
+@var MAX_COMMAND_LENGTH: The maximum length of a command, as defined by RFC
+    2812 section 2.3.
+
+@var attributes: Singleton instance of L{_CharacterAttributes}, used for
+    constructing formatted text information.
+
+@author: Kevin Turner
+
+@see: RFC 1459: Internet Relay Chat Protocol
+@see: RFC 2812: Internet Relay Chat: Client Protocol
+@see: U{The Client-To-Client-Protocol
+<http://www.irchelp.org/irchelp/rfc/ctcpspec.html>}
+"""
+
+import errno
+import operator
+import os
+import random
+import re
+import shlex
+import socket
+import stat
+import string
+import struct
+import sys
+import textwrap
+import time
+import traceback
+from functools import reduce
+from os import path
+from typing import Optional
+
+from twisted.internet import protocol, reactor, task
+from twisted.persisted import styles
+from twisted.protocols import basic
+from twisted.python import _textattributes, log, reflect
+
+NUL = chr(0)
+CR = chr(0o15)
+NL = chr(0o12)
+LF = NL
+SPC = chr(0o40)
+
+# This includes the CRLF terminator characters.
+MAX_COMMAND_LENGTH = 512
+
+CHANNEL_PREFIXES = "&#!+"
+
+
+class IRCBadMessage(Exception):
+    pass
+
+
+class IRCPasswordMismatch(Exception):
+    pass
+
+
+class IRCBadModes(ValueError):
+    """
+    A malformed mode was encountered while attempting to parse a mode string.
+    """
+
+
+def parsemsg(s):
+    """
+    Breaks a message from an IRC server into its prefix, command, and
+    arguments.
+
+    @param s: The message to break.
+    @type s: L{bytes}
+
+    @return: A tuple of (prefix, command, args).
+    @rtype: L{tuple}
+    """
+    prefix = ""
+    trailing = []
+    if not s:
+        raise IRCBadMessage("Empty line.")
+    if s[0:1] == ":":
+        prefix, s = s[1:].split(" ", 1)
+    if s.find(" :") != -1:
+        s, trailing = s.split(" :", 1)
+        args = s.split()
+        args.append(trailing)
+    else:
+        args = s.split()
+    command = args.pop(0)
+    return prefix, command, args
+
+
+def split(str, length=80):
+    """
+    Split a string into multiple lines.
+
+    Whitespace near C{str[length]} will be preferred as a breaking point.
+    C{"\\n"} will also be used as a breaking point.
+
+    @param str: The string to split.
+    @type str: C{str}
+
+    @param length: The maximum length which will be allowed for any string in
+        the result.
+    @type length: C{int}
+
+    @return: C{list} of C{str}
+    """
+    return [chunk for line in str.split("\n") for chunk in textwrap.wrap(line, length)]
+
+
+def _intOrDefault(value, default=None):
+    """
+    Convert a value to an integer if possible.
+
+    @rtype: C{int} or type of L{default}
+    @return: An integer when C{value} can be converted to an integer,
+        otherwise return C{default}
+    """
+    if value:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            pass
+    return default
+
+
+class UnhandledCommand(RuntimeError):
+    """
+    A command dispatcher could not locate an appropriate command handler.
+    """
+
+
+class _CommandDispatcherMixin:
+    """
+    Dispatch commands to handlers based on their name.
+
+    Command handler names should be of the form C{prefix_commandName},
+    where C{prefix} is the value specified by L{prefix}, and must
+    accept the parameters as given to L{dispatch}.
+
+    Attempting to mix this in more than once for a single class will cause
+    strange behaviour, due to L{prefix} being overwritten.
+
+    @type prefix: C{str}
+    @ivar prefix: Command handler prefix, used to locate handler attributes
+    """
+
+    prefix: Optional[str] = None
+
+    def dispatch(self, commandName, *args):
+        """
+        Perform actual command dispatch.
+        """
+
+        def _getMethodName(command):
+            return f"{self.prefix}_{command}"
+
+        def _getMethod(name):
+            return getattr(self, _getMethodName(name), None)
+
+        method = _getMethod(commandName)
+        if method is not None:
+            return method(*args)
+
+        method = _getMethod("unknown")
+        if method is None:
+            raise UnhandledCommand(
+                f"No handler for {_getMethodName(commandName)!r} could be found"
+            )
+        return method(commandName, *args)
+
+
+def parseModes(modes, params, paramModes=("", "")):
+    """
+    Parse an IRC mode string.
+
+    The mode string is parsed into two lists of mode changes (added and
+    removed), with each mode change represented as C{(mode, param)} where mode
+    is the mode character, and param is the parameter passed for that mode, or
+    L{None} if no parameter is required.
+
+    @type modes: C{str}
+    @param modes: Modes string to parse.
+
+    @type params: C{list}
+    @param params: Parameters specified along with L{modes}.
+
+    @type paramModes: C{(str, str)}
+    @param paramModes: A pair of strings (C{(add, remove)}) that indicate which modes take
+        parameters when added or removed.
+
+    @returns: Two lists of mode changes, one for modes added and the other for
+        modes removed respectively, mode changes in each list are represented as
+        C{(mode, param)}.
+    """
+    if len(modes) == 0:
+        raise IRCBadModes("Empty mode string")
+
+    if modes[0] not in "+-":
+        raise IRCBadModes(f"Malformed modes string: {modes!r}")
+
+    changes = ([], [])
+
+    direction = None
+    count = -1
+    for ch in modes:
+        if ch in "+-":
+            if count == 0:
+                raise IRCBadModes(f"Empty mode sequence: {modes!r}")
+            direction = "+-".index(ch)
+            count = 0
+        else:
+            param = None
+            if ch in paramModes[direction]:
+                try:
+                    param = params.pop(0)
+                except IndexError:
+                    raise IRCBadModes(f"Not enough parameters: {ch!r}")
+            changes[direction].append((ch, param))
+            count += 1
+
+    if len(params) > 0:
+        raise IRCBadModes(f"Too many parameters: {modes!r} {params!r}")
+
+    if count == 0:
+        raise IRCBadModes(f"Empty mode sequence: {modes!r}")
+
+    return changes
+
+
+class IRC(protocol.Protocol):
+    """
+    Internet Relay Chat server protocol.
+    """
+
+    buffer = ""
+    hostname = None
+
+    encoding: Optional[str] = None
+
+    def connectionMade(self):
+        self.channels = []
+        if self.hostname is None:
+            self.hostname = socket.getfqdn()
+
+    def sendLine(self, line):
+        line = line + CR + LF
+        if isinstance(line, str):
+            useEncoding = self.encoding if self.encoding else "utf-8"
+            line = line.encode(useEncoding)
+        self.transport.write(line)
+
+    def sendMessage(self, command, *parameter_list, **prefix):
+        """
+        Send a line formatted as an IRC message.
+
+        First argument is the command, all subsequent arguments are parameters
+        to that command.  If a prefix is desired, it may be specified with the
+        keyword argument 'prefix'.
+
+        The L{sendCommand} method is generally preferred over this one.
+        Notably, this method does not support sending message tags, while the
+        L{sendCommand} method does.
+        """
+        if not command:
+            raise ValueError("IRC message requires a command.")
+
+        if " " in command or command[0] == ":":
+            # Not the ONLY way to screw up, but provides a little
+            # sanity checking to catch likely dumb mistakes.
+            raise ValueError(
+                "Somebody screwed up, 'cuz this doesn't"
+                " look like a command to me: %s" % command
+            )
+
+        line = " ".join([command] + list(parameter_list))
+        if "prefix" in prefix:
+            line = ":{} {}".format(prefix["prefix"], line)
+        self.sendLine(line)
+
+        if len(parameter_list) > 15:
+            log.msg(
+                "Message has %d parameters (RFC allows 15):\n%s"
+                % (len(parameter_list), line)
+            )
+
+    def sendCommand(self, command, parameters, prefix=None, tags=None):
+        """
+        Send to the remote peer a line formatted as an IRC message.
+
+        @param command: The command or numeric to send.
+        @type command: L{unicode}
+
+        @param parameters: The parameters to send with the command.
+        @type parameters: A L{tuple} or L{list} of L{unicode} parameters
+
+        @param prefix: The prefix to send with the command.  If not
+            given, no prefix is sent.
+        @type prefix: L{unicode}
+
+        @param tags: A dict of message tags.  If not given, no message
+            tags are sent.  The dict key should be the name of the tag
+            to send as a string; the value should be the unescaped value
+            to send with the tag, or either None or "" if no value is to
+            be sent with the tag.
+        @type tags: L{dict} of tags (L{unicode}) => values (L{unicode})
+        @see: U{https://ircv3.net/specs/core/message-tags-3.2.html}
+        """
+        if not command:
+            raise ValueError("IRC message requires a command.")
+
+        if " " in command or command[0] == ":":
+            # Not the ONLY way to screw up, but provides a little
+            # sanity checking to catch likely dumb mistakes.
+            raise ValueError(f'Invalid command: "{command}"')
+
+        if tags is None:
+            tags = {}
+
+        line = " ".join([command] + list(parameters))
+        if prefix:
+            line = f":{prefix} {line}"
+        if tags:
+            tagStr = self._stringTags(tags)
+            line = f"@{tagStr} {line}"
+        self.sendLine(line)
+
+        if len(parameters) > 15:
+            log.msg(
+                "Message has %d parameters (RFC allows 15):\n%s"
+                % (len(parameters), line)
+            )
+
+    def _stringTags(self, tags):
+        """
+        Converts a tag dictionary to a string.
+
+        @param tags: The tag dict passed to sendMsg.
+
+        @rtype: L{unicode}
+        @return: IRCv3-format tag string
+        """
+        self._validateTags(tags)
+        tagStrings = []
+        for tag, value in tags.items():
+            if value:
+                tagStrings.append(f"{tag}={self._escapeTagValue(value)}")
+            else:
+                tagStrings.append(tag)
+        return ";".join(tagStrings)
+
+    def _validateTags(self, tags):
+        """
+        Checks the tag dict for errors and raises L{ValueError} if an
+        error is found.
+
+        @param tags: The tag dict passed to sendMsg.
+        """
+        for tag, value in tags.items():
+            if not tag:
+                raise ValueError("A tag name is required.")
+            for char in tag:
+                if not char.isalnum() and char not in ("-", "/", "."):
+                    raise ValueError("Tag contains invalid characters.")
+
+    def _escapeTagValue(self, value):
+        """
+        Escape the given tag value according to U{escaping rules in IRCv3
+        <https://ircv3.net/specs/core/message-tags-3.2.html>}.
+
+        @param value: The string value to escape.
+        @type value: L{str}
+
+        @return: The escaped string for sending as a message value
+        @rtype: L{str}
+        """
+        return (
+            value.replace("\\", "\\\\")
+            .replace(";", "\\:")
+            .replace(" ", "\\s")
+            .replace("\r", "\\r")
+            .replace("\n", "\\n")
+        )
+
+    def dataReceived(self, data):
+        """
+        This hack is to support mIRC, which sends LF only, even though the RFC
+        says CRLF.  (Also, the flexibility of LineReceiver to turn "line mode"
+        on and off was not required.)
+        """
+        if isinstance(data, bytes):
+            data = data.decode("utf-8")
+        lines = (self.buffer + data).split(LF)
+        # Put the (possibly empty) element after the last LF back in the
+        # buffer
+        self.buffer = lines.pop()
+
+        for line in lines:
+            if len(line) <= 2:
+                # This is a blank line, at best.
+                continue
+            if line[-1] == CR:
+                line = line[:-1]
+            prefix, command, params = parsemsg(line)
+            # mIRC is a big pile of doo-doo
+            command = command.upper()
+            # DEBUG: log.msg( "%s %s %s" % (prefix, command, params))
+
+            self.handleCommand(command, prefix, params)
+
+    def handleCommand(self, command, prefix, params):
+        """
+        Determine the function to call for the given command and call it with
+        the given arguments.
+
+        @param command: The IRC command to determine the function for.
+        @type command: L{bytes}
+
+        @param prefix: The prefix of the IRC message (as returned by
+            L{parsemsg}).
+        @type prefix: L{bytes}
+
+        @param params: A list of parameters to call the function with.
+        @type params: L{list}
+        """
+        method = getattr(self, "irc_%s" % command, None)
+        try:
+            if method is not None:
+                method(prefix, params)
+            else:
+                self.irc_unknown(prefix, command, params)
+        except BaseException:
+            log.deferr()
+
+    def irc_unknown(self, prefix, command, params):
+        """
+        Called by L{handleCommand} on a command that doesn't have a defined
+        handler. Subclasses should override this method.
+        """
+        raise NotImplementedError(command, prefix, params)
+
+    # Helper methods
+    def privmsg(self, sender, recip, message):
+        """
+        Send a message to a channel or user
+
+        @type sender: C{str} or C{unicode}
+        @param sender: Who is sending this message.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type recip: C{str} or C{unicode}
+        @param recip: The recipient of this message.  If a channel, it must
+            start with a channel prefix.
+
+        @type message: C{str} or C{unicode}
+        @param message: The message being sent.
+        """
+        self.sendCommand("PRIVMSG", (recip, f":{lowQuote(message)}"), sender)
+
+    def notice(self, sender, recip, message):
+        """
+        Send a "notice" to a channel or user.
+
+        Notices differ from privmsgs in that the RFC claims they are different.
+        Robots are supposed to send notices and not respond to them.  Clients
+        typically display notices differently from privmsgs.
+
+        @type sender: C{str} or C{unicode}
+        @param sender: Who is sending this message.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type recip: C{str} or C{unicode}
+        @param recip: The recipient of this message.  If a channel, it must
+            start with a channel prefix.
+
+        @type message: C{str} or C{unicode}
+        @param message: The message being sent.
+        """
+        self.sendCommand("NOTICE", (recip, f":{message}"), sender)
+
+    def action(self, sender, recip, message):
+        """
+        Send an action to a channel or user.
+
+        @type sender: C{str} or C{unicode}
+        @param sender: Who is sending this message.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type recip: C{str} or C{unicode}
+        @param recip: The recipient of this message.  If a channel, it must
+            start with a channel prefix.
+
+        @type message: C{str} or C{unicode}
+        @param message: The action being sent.
+        """
+        self.sendLine(f":{sender} ACTION {recip} :{message}")
+
+    def topic(self, user, channel, topic, author=None):
+        """
+        Send the topic to a user.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving the topic.  Only their nickname, not
+            the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this is the topic.
+
+        @type topic: C{str} or C{unicode} or L{None}
+        @param topic: The topic string, unquoted, or None if there is no topic.
+
+        @type author: C{str} or C{unicode}
+        @param author: If the topic is being changed, the full username and
+            hostmask of the person changing it.
+        """
+        if author is None:
+            if topic is None:
+                self.sendLine(
+                    ":%s %s %s %s :%s"
+                    % (self.hostname, RPL_NOTOPIC, user, channel, "No topic is set.")
+                )
+            else:
+                self.sendLine(
+                    ":%s %s %s %s :%s"
+                    % (self.hostname, RPL_TOPIC, user, channel, lowQuote(topic))
+                )
+        else:
+            self.sendLine(f":{author} TOPIC {channel} :{lowQuote(topic)}")
+
+    def topicAuthor(self, user, channel, author, date):
+        """
+        Send the author of and time at which a topic was set for the given
+        channel.
+
+        This sends a 333 reply message, which is not part of the IRC RFC.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving the topic.  Only their nickname, not
+            the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this information is relevant.
+
+        @type author: C{str} or C{unicode}
+        @param author: The nickname (without hostmask) of the user who last set
+            the topic.
+
+        @type date: C{int}
+        @param date: A POSIX timestamp (number of seconds since the epoch) at
+            which the topic was last set.
+        """
+        self.sendLine(
+            ":%s %d %s %s %s %d" % (self.hostname, 333, user, channel, author, date)
+        )
+
+    def names(self, user, channel, names):
+        """
+        Send the names of a channel's participants to a user.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving the name list.  Only their nickname,
+            not the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this is the namelist.
+
+        @type names: C{list} of C{str} or C{unicode}
+        @param names: The names to send.
+        """
+        # XXX If unicode is given, these limits are not quite correct
+        prefixLength = len(channel) + len(user) + 10
+        namesLength = 512 - prefixLength
+
+        L = []
+        count = 0
+        for n in names:
+            if count + len(n) + 1 > namesLength:
+                self.sendLine(
+                    ":%s %s %s = %s :%s"
+                    % (self.hostname, RPL_NAMREPLY, user, channel, " ".join(L))
+                )
+                L = [n]
+                count = len(n)
+            else:
+                L.append(n)
+                count += len(n) + 1
+        if L:
+            self.sendLine(
+                ":%s %s %s = %s :%s"
+                % (self.hostname, RPL_NAMREPLY, user, channel, " ".join(L))
+            )
+        self.sendLine(
+            ":%s %s %s %s :End of /NAMES list"
+            % (self.hostname, RPL_ENDOFNAMES, user, channel)
+        )
+
+    def who(self, user, channel, memberInfo):
+        """
+        Send a list of users participating in a channel.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving this member information.  Only their
+            nickname, not the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this is the member information.
+
+        @type memberInfo: C{list} of C{tuples}
+        @param memberInfo: For each member of the given channel, a 7-tuple
+            containing their username, their hostmask, the server to which they
+            are connected, their nickname, the letter "H" or "G" (standing for
+            "Here" or "Gone"), the hopcount from C{user} to this member, and
+            this member's real name.
+        """
+        for info in memberInfo:
+            (username, hostmask, server, nickname, flag, hops, realName) = info
+            assert flag in ("H", "G")
+            self.sendLine(
+                ":%s %s %s %s %s %s %s %s %s :%d %s"
+                % (
+                    self.hostname,
+                    RPL_WHOREPLY,
+                    user,
+                    channel,
+                    username,
+                    hostmask,
+                    server,
+                    nickname,
+                    flag,
+                    hops,
+                    realName,
+                )
+            )
+
+        self.sendLine(
+            ":%s %s %s %s :End of /WHO list."
+            % (self.hostname, RPL_ENDOFWHO, user, channel)
+        )
+
+    def whois(
+        self,
+        user,
+        nick,
+        username,
+        hostname,
+        realName,
+        server,
+        serverInfo,
+        oper,
+        idle,
+        signOn,
+        channels,
+    ):
+        """
+        Send information about the state of a particular user.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving this information.  Only their nickname,
+            not the full hostmask.
+
+        @type nick: C{str} or C{unicode}
+        @param nick: The nickname of the user this information describes.
+
+        @type username: C{str} or C{unicode}
+        @param username: The user's username (eg, ident response)
+
+        @type hostname: C{str}
+        @param hostname: The user's hostmask
+
+        @type realName: C{str} or C{unicode}
+        @param realName: The user's real name
+
+        @type server: C{str} or C{unicode}
+        @param server: The name of the server to which the user is connected
+
+        @type serverInfo: C{str} or C{unicode}
+        @param serverInfo: A descriptive string about that server
+
+        @type oper: C{bool}
+        @param oper: Indicates whether the user is an IRC operator
+
+        @type idle: C{int}
+        @param idle: The number of seconds since the user last sent a message
+
+        @type signOn: C{int}
+        @param signOn: A POSIX timestamp (number of seconds since the epoch)
+            indicating the time the user signed on
+
+        @type channels: C{list} of C{str} or C{unicode}
+        @param channels: A list of the channels which the user is participating in
+        """
+        self.sendLine(
+            ":%s %s %s %s %s %s * :%s"
+            % (self.hostname, RPL_WHOISUSER, user, nick, username, hostname, realName)
+        )
+        self.sendLine(
+            ":%s %s %s %s %s :%s"
+            % (self.hostname, RPL_WHOISSERVER, user, nick, server, serverInfo)
+        )
+        if oper:
+            self.sendLine(
+                ":%s %s %s %s :is an IRC operator"
+                % (self.hostname, RPL_WHOISOPERATOR, user, nick)
+            )
+        self.sendLine(
+            ":%s %s %s %s %d %d :seconds idle, signon time"
+            % (self.hostname, RPL_WHOISIDLE, user, nick, idle, signOn)
+        )
+        self.sendLine(
+            ":%s %s %s %s :%s"
+            % (self.hostname, RPL_WHOISCHANNELS, user, nick, " ".join(channels))
+        )
+        self.sendLine(
+            ":%s %s %s %s :End of WHOIS list."
+            % (self.hostname, RPL_ENDOFWHOIS, user, nick)
+        )
+
+    def join(self, who, where):
+        """
+        Send a join message.
+
+        @type who: C{str} or C{unicode}
+        @param who: The name of the user joining.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type where: C{str} or C{unicode}
+        @param where: The channel the user is joining.
+        """
+        self.sendLine(f":{who} JOIN {where}")
+
+    def part(self, who, where, reason=None):
+        """
+        Send a part message.
+
+        @type who: C{str} or C{unicode}
+        @param who: The name of the user joining.  Should be of the form
+            username!ident@hostmask (unless you know better!).
+
+        @type where: C{str} or C{unicode}
+        @param where: The channel the user is joining.
+
+        @type reason: C{str} or C{unicode}
+        @param reason: A string describing the misery which caused this poor
+            soul to depart.
+        """
+        if reason:
+            self.sendLine(f":{who} PART {where} :{reason}")
+        else:
+            self.sendLine(f":{who} PART {where}")
+
+    def channelMode(self, user, channel, mode, *args):
+        """
+        Send information about the mode of a channel.
+
+        @type user: C{str} or C{unicode}
+        @param user: The user receiving the name list.  Only their nickname,
+            not the full hostmask.
+
+        @type channel: C{str} or C{unicode}
+        @param channel: The channel for which this is the namelist.
+
+        @type mode: C{str}
+        @param mode: A string describing this channel's modes.
+
+        @param args: Any additional arguments required by the modes.
+        """
+        self.sendLine(
+            ":%s %s %s %s %s %s"
+            % (self.hostname, RPL_CHANNELMODEIS, user, channel, mode, " ".join(args))
+        )
+
+
+class ServerSupportedFeatures(_CommandDispatcherMixin):
+    """
+    Handle ISUPPORT messages.
+
+    Feature names match those in the ISUPPORT RFC draft identically.
+
+    Information regarding the specifics of ISUPPORT was gleaned from
+    <http://www.irc.org/tech_docs/draft-brocklesby-irc-isupport-03.txt>.
+    """
+
+    prefix = "isupport"
+
+    def __init__(self):
+        self._features = {
+            "CHANNELLEN": 200,
+            "CHANTYPES": tuple("#&"),
+            "MODES": 3,
+            "NICKLEN": 9,
+            "PREFIX": self._parsePrefixParam("(ovh)@+%"),
+            # The ISUPPORT draft explicitly says that there is no default for
+            # CHANMODES, but we're defaulting it here to handle the case where
+            # the IRC server doesn't send us any ISUPPORT information, since
+            # IRCClient.getChannelModeParams relies on this value.
+            "CHANMODES": self._parseChanModesParam(["b", "", "lk", ""]),
+        }
+
+    @classmethod
+    def _splitParamArgs(cls, params, valueProcessor=None):
+        """
+        Split ISUPPORT parameter arguments.
+
+        Values can optionally be processed by C{valueProcessor}.
+
+        For example::
+
+            >>> ServerSupportedFeatures._splitParamArgs(['A:1', 'B:2'])
+            (('A', '1'), ('B', '2'))
+
+        @type params: C{iterable} of C{str}
+
+        @type valueProcessor: C{callable} taking {str}
+        @param valueProcessor: Callable to process argument values, or L{None}
+            to perform no processing
+
+        @rtype: C{list} of C{(str, object)}
+        @return: Sequence of C{(name, processedValue)}
+        """
+        if valueProcessor is None:
+            valueProcessor = lambda x: x
+
+        def _parse():
+            for param in params:
+                if ":" not in param:
+                    param += ":"
+                a, b = param.split(":", 1)
+                yield a, valueProcessor(b)
+
+        return list(_parse())
+
+    @classmethod
+    def _unescapeParamValue(cls, value):
+        """
+        Unescape an ISUPPORT parameter.
+
+        The only form of supported escape is C{\\xHH}, where HH must be a valid
+        2-digit hexadecimal number.
+
+        @rtype: C{str}
+        """
+
+        def _unescape():
+            parts = value.split("\\x")
+            # The first part can never be preceded by the escape.
+            yield parts.pop(0)
+            for s in parts:
+                octet, rest = s[:2], s[2:]
+                try:
+                    octet = int(octet, 16)
+                except ValueError:
+                    raise ValueError(f"Invalid hex octet: {octet!r}")
+                yield chr(octet) + rest
+
+        if "\\x" not in value:
+            return value
+        return "".join(_unescape())
+
+    @classmethod
+    def _splitParam(cls, param):
+        """
+        Split an ISUPPORT parameter.
+
+        @type param: C{str}
+
+        @rtype: C{(str, list)}
+        @return: C{(key, arguments)}
+        """
+        if "=" not in param:
+            param += "="
+        key, value = param.split("=", 1)
+        return key, [cls._unescapeParamValue(v) for v in value.split(",")]
+
+    @classmethod
+    def _parsePrefixParam(cls, prefix):
+        """
+        Parse the ISUPPORT "PREFIX" parameter.
+
+        The order in which the parameter arguments appear is significant, the
+        earlier a mode appears the more privileges it gives.
+
+        @rtype: C{dict} mapping C{str} to C{(str, int)}
+        @return: A dictionary mapping a mode character to a two-tuple of
+            C({symbol, priority)}, the lower a priority (the lowest being
+            C{0}) the more privileges it gives
+        """
+        if not prefix:
+            return None
+        if prefix[0] != "(" and ")" not in prefix:
+            raise ValueError("Malformed PREFIX parameter")
+        modes, symbols = prefix.split(")", 1)
+        symbols = zip(symbols, range(len(symbols)))
+        modes = modes[1:]
+        return dict(zip(modes, symbols))
+
+    @classmethod
+    def _parseChanModesParam(self, params):
+        """
+        Parse the ISUPPORT "CHANMODES" parameter.
+
+        See L{isupport_CHANMODES} for a detailed explanation of this parameter.
+        """
+        names = ("addressModes", "param", "setParam", "noParam")
+        if len(params) > len(names):
+            raise ValueError(
+                "Expecting a maximum of %d channel mode parameters, got %d"
+                % (len(names), len(params))
+            )
+        items = map(lambda key, value: (key, value or ""), names, params)
+        return dict(items)
+
+    def getFeature(self, feature, default=None):
+        """
+        Get a server supported feature's value.
+
+        A feature with the value L{None} is equivalent to the feature being
+        unsupported.
+
+        @type feature: C{str}
+        @param feature: Feature name
+
+        @type default: C{object}
+        @param default: The value to default to, assuming that C{feature}
+            is not supported
+
+        @return: Feature value
+        """
+        return self._features.get(feature, default)
+
+    def hasFeature(self, feature):
+        """
+        Determine whether a feature is supported or not.
+
+        @rtype: C{bool}
+        """
+        return self.getFeature(feature) is not None
+
+    def parse(self, params):
+        """
+        Parse ISUPPORT parameters.
+
+        If an unknown parameter is encountered, it is simply added to the
+        dictionary, keyed by its name, as a tuple of the parameters provided.
+
+        @type params: C{iterable} of C{str}
+        @param params: Iterable of ISUPPORT parameters to parse
+        """
+        for param in params:
+            key, value = self._splitParam(param)
+            if key.startswith("-"):
+                self._features.pop(key[1:], None)
+            else:
+                self._features[key] = self.dispatch(key, value)
+
+    def isupport_unknown(self, command, params):
+        """
+        Unknown ISUPPORT parameter.
+        """
+        return tuple(params)
+
+    def isupport_CHANLIMIT(self, params):
+        """
+        The maximum number of each channel type a user may join.
+        """
+        return self._splitParamArgs(params, _intOrDefault)
+
+    def isupport_CHANMODES(self, params):
+        """
+        Available channel modes.
+
+        There are 4 categories of channel mode::
+
+            addressModes - Modes that add or remove an address to or from a
+            list, these modes always take a parameter.
+
+            param - Modes that change a setting on a channel, these modes
+            always take a parameter.
+
+            setParam - Modes that change a setting on a channel, these modes
+            only take a parameter when being set.
+
+            noParam - Modes that change a setting on a channel, these modes
+            never take a parameter.
+        """
+        try:
+            return self._parseChanModesParam(params)
+        except ValueError:
+            return self.getFeature("CHANMODES")
+
+    def isupport_CHANNELLEN(self, params):
+        """
+        Maximum length of a channel name a client may create.
+        """
+        return _intOrDefault(params[0], self.getFeature("CHANNELLEN"))
+
+    def isupport_CHANTYPES(self, params):
+        """
+        Valid channel prefixes.
+        """
+        return tuple(params[0])
+
+    def isupport_EXCEPTS(self, params):
+        """
+        Mode character for "ban exceptions".
+
+        The presence of this parameter indicates that the server supports
+        this functionality.
+        """
+        return params[0] or "e"
+
+    def isupport_IDCHAN(self, params):
+        """
+        Safe channel identifiers.
+
+        The presence of this parameter indicates that the server supports
+        this functionality.
+        """
+        return self._splitParamArgs(params)
+
+    def isupport_INVEX(self, params):
+        """
+        Mode character for "invite exceptions".
+
+        The presence of this parameter indicates that the server supports
+        this functionality.
+        """
+        return params[0] or "I"
+
+    def isupport_KICKLEN(self, params):
+        """
+        Maximum length of a kick message a client may provide.
+        """
+        return _intOrDefault(params[0])
+
+    def isupport_MAXLIST(self, params):
+        """
+        Maximum number of "list modes" a client may set on a channel at once.
+
+        List modes are identified by the "addressModes" key in CHANMODES.
+        """
+        return self._splitParamArgs(params, _intOrDefault)
+
+    def isupport_MODES(self, params):
+        """
+        Maximum number of modes accepting parameters that may be sent, by a
+        client, in a single MODE command.
+        """
+        return _intOrDefault(params[0])
+
+    def isupport_NETWORK(self, params):
+        """
+        IRC network name.
+        """
+        return params[0]
+
+    def isupport_NICKLEN(self, params):
+        """
+        Maximum length of a nickname the client may use.
+        """
+        return _intOrDefault(params[0], self.getFeature("NICKLEN"))
+
+    def isupport_PREFIX(self, params):
+        """
+        Mapping of channel modes that clients may have to status flags.
+        """
+        try:
+            return self._parsePrefixParam(params[0])
+        except ValueError:
+            return self.getFeature("PREFIX")
+
+    def isupport_SAFELIST(self, params):
+        """
+        Flag indicating that a client may request a LIST without being
+        disconnected due to the large amount of data generated.
+        """
+        return True
+
+    def isupport_STATUSMSG(self, params):
+        """
+        The server supports sending messages to only to clients on a channel
+        with a specific status.
+        """
+        return params[0]
+
+    def isupport_TARGMAX(self, params):
+        """
+        Maximum number of targets allowable for commands that accept multiple
+        targets.
+        """
+        return dict(self._splitParamArgs(params, _intOrDefault))
+
+    def isupport_TOPICLEN(self, params):
+        """
+        Maximum length of a topic that may be set.
+        """
+        return _intOrDefault(params[0])
+
+
+class IRCClient(basic.LineReceiver):
+    """
+    Internet Relay Chat client protocol, with sprinkles.
+
+    In addition to providing an interface for an IRC client protocol,
+    this class also contains reasonable implementations of many common
+    CTCP methods.
+
+    TODO
+    ====
+     - Limit the length of messages sent (because the IRC server probably
+       does).
+     - Add flood protection/rate limiting for my CTCP replies.
+     - NickServ cooperation.  (a mix-in?)
+
+    @ivar nickname: Nickname the client will use.
+    @ivar password: Password used to log on to the server.  May be L{None}.
+    @ivar realname: Supplied to the server during login as the "Real name"
+        or "ircname".  May be L{None}.
+    @ivar username: Supplied to the server during login as the "User name".
+        May be L{None}
+
+    @ivar userinfo: Sent in reply to a C{USERINFO} CTCP query.  If L{None}, no
+        USERINFO reply will be sent.
+        "This is used to transmit a string which is settable by
+        the user (and never should be set by the client)."
+    @ivar fingerReply: Sent in reply to a C{FINGER} CTCP query.  If L{None}, no
+        FINGER reply will be sent.
+    @type fingerReply: Callable or String
+
+    @ivar versionName: CTCP VERSION reply, client name.  If L{None}, no VERSION
+        reply will be sent.
+    @type versionName: C{str}, or None.
+    @ivar versionNum: CTCP VERSION reply, client version.
+    @type versionNum: C{str}, or None.
+    @ivar versionEnv: CTCP VERSION reply, environment the client is running in.
+    @type versionEnv: C{str}, or None.
+
+    @ivar sourceURL: CTCP SOURCE reply, a URL where the source code of this
+        client may be found.  If L{None}, no SOURCE reply will be sent.
+
+    @ivar lineRate: Minimum delay between lines sent to the server.  If
+        L{None}, no delay will be imposed.
+    @type lineRate: Number of Seconds.
+
+    @ivar motd: Either L{None} or, between receipt of I{RPL_MOTDSTART} and
+        I{RPL_ENDOFMOTD}, a L{list} of L{str}, each of which is the content
+        of an I{RPL_MOTD} message.
+
+    @ivar erroneousNickFallback: Default nickname assigned when an unregistered
+        client triggers an C{ERR_ERRONEUSNICKNAME} while trying to register
+        with an illegal nickname.
+    @type erroneousNickFallback: C{str}
+
+    @ivar _registered: Whether or not the user is registered. It becomes True
+        once a welcome has been received from the server.
+    @type _registered: C{bool}
+
+    @ivar _attemptedNick: The nickname that will try to get registered. It may
+        change if it is illegal or already taken. L{nickname} becomes the
+        L{_attemptedNick} that is successfully registered.
+    @type _attemptedNick:  C{str}
+
+    @type supported: L{ServerSupportedFeatures}
+    @ivar supported: Available ISUPPORT features on the server
+
+    @type hostname: C{str}
+    @ivar hostname: Host name of the IRC server the client is connected to.
+        Initially the host name is L{None} and later is set to the host name
+        from which the I{RPL_WELCOME} message is received.
+
+    @type _heartbeat: L{task.LoopingCall}
+    @ivar _heartbeat: Looping call to perform the keepalive by calling
+        L{IRCClient._sendHeartbeat} every L{heartbeatInterval} seconds, or
+        L{None} if there is no heartbeat.
+
+    @type heartbeatInterval: C{float}
+    @ivar heartbeatInterval: Interval, in seconds, to send I{PING} messages to
+        the server as a form of keepalive, defaults to 120 seconds. Use L{None}
+        to disable the heartbeat.
+    """
+
+    hostname = None
+    motd = None
+    nickname = "irc"
+    password = None
+    realname = None
+    username = None
+    ### Responses to various CTCP queries.
+
+    userinfo = None
+    # fingerReply is a callable returning a string, or a str()able object.
+    fingerReply = None
+    versionName = None
+    versionNum = None
+    versionEnv = None
+
+    sourceURL = "http://twistedmatrix.com/downloads/"
+
+    dcc_destdir = "."
+    dcc_sessions = None
+
+    # If this is false, no attempt will be made to identify
+    # ourself to the server.
+    performLogin = 1
+
+    lineRate = None
+    _queue = None
+    _queueEmptying = None
+
+    delimiter = b"\n"  # b'\r\n' will also work (see dataReceived)
+
+    __pychecker__ = "unusednames=params,prefix,channel"
+
+    _registered = False
+    _attemptedNick = ""
+    erroneousNickFallback = "defaultnick"
+
+    _heartbeat = None
+    heartbeatInterval = 120
+
+    def _reallySendLine(self, line):
+        quoteLine = lowQuote(line)
+        if isinstance(quoteLine, str):
+            quoteLine = quoteLine.encode("utf-8")
+        quoteLine += b"\r"
+        return basic.LineReceiver.sendLine(self, quoteLine)
+
+    def sendLine(self, line):
+        if self.lineRate is None:
+            self._reallySendLine(line)
+        else:
+            self._queue.append(line)
+            if not self._queueEmptying:
+                self._sendLine()
+
+    def _sendLine(self):
+        if self._queue:
+            self._reallySendLine(self._queue.pop(0))
+            self._queueEmptying = reactor.callLater(self.lineRate, self._sendLine)
+        else:
+            self._queueEmptying = None
+
+    def connectionLost(self, reason):
+        basic.LineReceiver.connectionLost(self, reason)
+        self.stopHeartbeat()
+
+    def _createHeartbeat(self):
+        """
+        Create the heartbeat L{LoopingCall}.
+        """
+        return task.LoopingCall(self._sendHeartbeat)
+
+    def _sendHeartbeat(self):
+        """
+        Send a I{PING} message to the IRC server as a form of keepalive.
+        """
+        self.sendLine("PING " + self.hostname)
+
+    def stopHeartbeat(self):
+        """
+        Stop sending I{PING} messages to keep the connection to the server
+        alive.
+
+        @since: 11.1
+        """
+        if self._heartbeat is not None:
+            self._heartbeat.stop()
+            self._heartbeat = None
+
+    def startHeartbeat(self):
+        """
+        Start sending I{PING} messages every L{IRCClient.heartbeatInterval}
+        seconds to keep the connection to the server alive during periods of no
+        activity.
+
+        @since: 11.1
+        """
+        self.stopHeartbeat()
+        if self.heartbeatInterval is None:
+            return
+        self._heartbeat = self._createHeartbeat()
+        self._heartbeat.start(self.heartbeatInterval, now=False)
+
+    ### Interface level client->user output methods
+    ###
+    ### You'll want to override these.
+
+    ### Methods relating to the server itself
+
+    def created(self, when):
+        """
+        Called with creation date information about the server, usually at logon.
+
+        @type when: C{str}
+        @param when: A string describing when the server was created, probably.
+        """
+
+    def yourHost(self, info):
+        """
+        Called with daemon information about the server, usually at logon.
+
+        @type info: C{str}
+        @param info: A string describing what software the server is running, probably.
+        """
+
+    def myInfo(self, servername, version, umodes, cmodes):
+        """
+        Called with information about the server, usually at logon.
+
+        @type servername: C{str}
+        @param servername: The hostname of this server.
+
+        @type version: C{str}
+        @param version: A description of what software this server runs.
+
+        @type umodes: C{str}
+        @param umodes: All the available user modes.
+
+        @type cmodes: C{str}
+        @param cmodes: All the available channel modes.
+        """
+
+    def luserClient(self, info):
+        """
+        Called with information about the number of connections, usually at logon.
+
+        @type info: C{str}
+        @param info: A description of the number of clients and servers
+        connected to the network, probably.
+        """
+
+    def bounce(self, info):
+        """
+        Called with information about where the client should reconnect.
+
+        @type info: C{str}
+        @param info: A plaintext description of the address that should be
+        connected to.
+        """
+
+    def isupport(self, options):
+        """
+        Called with various information about what the server supports.
+
+        @type options: C{list} of C{str}
+        @param options: Descriptions of features or limits of the server, possibly
+        in the form "NAME=VALUE".
+        """
+
+    def luserChannels(self, channels):
+        """
+        Called with the number of channels existent on the server.
+
+        @type channels: C{int}
+        """
+
+    def luserOp(self, ops):
+        """
+        Called with the number of ops logged on to the server.
+
+        @type ops: C{int}
+        """
+
+    def luserMe(self, info):
+        """
+        Called with information about the server connected to.
+
+        @type info: C{str}
+        @param info: A plaintext string describing the number of users and servers
+        connected to this server.
+        """
+
+    ### Methods involving me directly
+
+    def privmsg(self, user, channel, message):
+        """
+        Called when I have a message from a user to me or a channel.
+        """
+        pass
+
+    def joined(self, channel):
+        """
+        Called when I finish joining a channel.
+
+        channel has the starting character (C{'#'}, C{'&'}, C{'!'}, or C{'+'})
+        intact.
+        """
+
+    def left(self, channel):
+        """
+        Called when I have left a channel.
+
+        channel has the starting character (C{'#'}, C{'&'}, C{'!'}, or C{'+'})
+        intact.
+        """
+
+    def noticed(self, user, channel, message):
+        """
+        Called when I have a notice from a user to me or a channel.
+
+        If the client makes any automated replies, it must not do so in
+        response to a NOTICE message, per the RFC::
+
+            The difference between NOTICE and PRIVMSG is that
+            automatic replies MUST NEVER be sent in response to a
+            NOTICE message. [...] The object of this rule is to avoid
+            loops between clients automatically sending something in
+            response to something it received.
+        """
+
+    def modeChanged(self, user, channel, set, modes, args):
+        """
+        Called when users or channel's modes are changed.
+
+        @type user: C{str}
+        @param user: The user and hostmask which instigated this change.
+
+        @type channel: C{str}
+        @param channel: The channel where the modes are changed. If args is
+        empty the channel for which the modes are changing. If the changes are
+        at server level it could be equal to C{user}.
+
+        @type set: C{bool} or C{int}
+        @param set: True if the mode(s) is being added, False if it is being
+        removed. If some modes are added and others removed at the same time
+        this function will be called twice, the first time with all the added
+        modes, the second with the removed ones. (To change this behaviour
+        override the irc_MODE method)
+
+        @type modes: C{str}
+        @param modes: The mode or modes which are being changed.
+
+        @type args: C{tuple}
+        @param args: Any additional information required for the mode
+        change.
+        """
+
+    def pong(self, user, secs):
+        """
+        Called with the results of a CTCP PING query.
+        """
+        pass
+
+    def signedOn(self):
+        """
+        Called after successfully signing on to the server.
+        """
+        pass
+
+    def kickedFrom(self, channel, kicker, message):
+        """
+        Called when I am kicked from a channel.
+        """
+        pass
+
+    def nickChanged(self, nick):
+        """
+        Called when my nick has been changed.
+        """
+        self.nickname = nick
+
+    ### Things I observe other people doing in a channel.
+
+    def userJoined(self, user, channel):
+        """
+        Called when I see another user joining a channel.
+        """
+        pass
+
+    def userLeft(self, user, channel):
+        """
+        Called when I see another user leaving a channel.
+        """
+        pass
+
+    def userQuit(self, user, quitMessage):
+        """
+        Called when I see another user disconnect from the network.
+        """
+        pass
+
+    def userKicked(self, kickee, channel, kicker, message):
+        """
+        Called when I observe someone else being kicked from a channel.
+        """
+        pass
+
+    def action(self, user, channel, data):
+        """
+        Called when I see a user perform an ACTION on a channel.
+        """
+        pass
+
+    def topicUpdated(self, user, channel, newTopic):
+        """
+        In channel, user changed the topic to newTopic.
+
+        Also called when first joining a channel.
+        """
+        pass
+
+    def userRenamed(self, oldname, newname):
+        """
+        A user changed their name from oldname to newname.
+        """
+        pass
+
+    ### Information from the server.
+
+    def receivedMOTD(self, motd):
+        """
+        I received a message-of-the-day banner from the server.
+
+        motd is a list of strings, where each string was sent as a separate
+        message from the server. To display, you might want to use::
+
+            '\\n'.join(motd)
+
+        to get a nicely formatted string.
+        """
+        pass
+
+    ### user input commands, client->server
+    ### Your client will want to invoke these.
+
+    def join(self, channel, key=None):
+        """
+        Join a channel.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to join. If it has no prefix,
+            C{'#'} will be prepended to it.
+        @type key: C{str}
+        @param key: If specified, the key used to join the channel.
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = "#" + channel
+        if key:
+            self.sendLine(f"JOIN {channel} {key}")
+        else:
+            self.sendLine(f"JOIN {channel}")
+
+    def leave(self, channel, reason=None):
+        """
+        Leave a channel.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to leave. If it has no prefix,
+            C{'#'} will be prepended to it.
+        @type reason: C{str}
+        @param reason: If given, the reason for leaving.
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = "#" + channel
+        if reason:
+            self.sendLine(f"PART {channel} :{reason}")
+        else:
+            self.sendLine(f"PART {channel}")
+
+    def kick(self, channel, user, reason=None):
+        """
+        Attempt to kick a user from a channel.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to kick the user from. If it has
+            no prefix, C{'#'} will be prepended to it.
+        @type user: C{str}
+        @param user: The nick of the user to kick.
+        @type reason: C{str}
+        @param reason: If given, the reason for kicking the user.
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = "#" + channel
+        if reason:
+            self.sendLine(f"KICK {channel} {user} :{reason}")
+        else:
+            self.sendLine(f"KICK {channel} {user}")
+
+    part = leave
+
+    def invite(self, user, channel):
+        """
+        Attempt to invite user to channel
+
+        @type user: C{str}
+        @param user: The user to invite
+        @type channel: C{str}
+        @param channel: The channel to invite the user too
+
+        @since: 11.0
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = "#" + channel
+        self.sendLine(f"INVITE {user} {channel}")
+
+    def topic(self, channel, topic=None):
+        """
+        Attempt to set the topic of the given channel, or ask what it is.
+
+        If topic is None, then I sent a topic query instead of trying to set the
+        topic. The server should respond with a TOPIC message containing the
+        current topic of the given channel.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to change the topic on. If it
+            has no prefix, C{'#'} will be prepended to it.
+        @type topic: C{str}
+        @param topic: If specified, what to set the topic to.
+        """
+        # << TOPIC #xtestx :fff
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = "#" + channel
+        if topic != None:
+            self.sendLine(f"TOPIC {channel} :{topic}")
+        else:
+            self.sendLine(f"TOPIC {channel}")
+
+    def mode(self, chan, set, modes, limit=None, user=None, mask=None):
+        """
+        Change the modes on a user or channel.
+
+        The C{limit}, C{user}, and C{mask} parameters are mutually exclusive.
+
+        @type chan: C{str}
+        @param chan: The name of the channel to operate on.
+        @type set: C{bool}
+        @param set: True to give the user or channel permissions and False to
+            remove them.
+        @type modes: C{str}
+        @param modes: The mode flags to set on the user or channel.
+        @type limit: C{int}
+        @param limit: In conjunction with the C{'l'} mode flag, limits the
+             number of users on the channel.
+        @type user: C{str}
+        @param user: The user to change the mode on.
+        @type mask: C{str}
+        @param mask: In conjunction with the C{'b'} mode flag, sets a mask of
+            users to be banned from the channel.
+        """
+        if set:
+            line = f"MODE {chan} +{modes}"
+        else:
+            line = f"MODE {chan} -{modes}"
+        if limit is not None:
+            line = "%s %d" % (line, limit)
+        elif user is not None:
+            line = f"{line} {user}"
+        elif mask is not None:
+            line = f"{line} {mask}"
+        self.sendLine(line)
+
+    def say(self, channel, message, length=None):
+        """
+        Send a message to a channel
+
+        @type channel: C{str}
+        @param channel: The channel to say the message on. If it has no prefix,
+            C{'#'} will be prepended to it.
+        @type message: C{str}
+        @param message: The message to say.
+        @type length: C{int}
+        @param length: The maximum number of octets to send at a time.  This has
+            the effect of turning a single call to C{msg()} into multiple
+            commands to the server.  This is useful when long messages may be
+            sent that would otherwise cause the server to kick us off or
+            silently truncate the text we are sending.  If None is passed, the
+            entire message is always send in one command.
+        """
+        if channel[0] not in CHANNEL_PREFIXES:
+            channel = "#" + channel
+        self.msg(channel, message, length)
+
+    def _safeMaximumLineLength(self, command):
+        """
+        Estimate a safe maximum line length for the given command.
+
+        This is done by assuming the maximum values for nickname length,
+        realname and hostname combined with the command that needs to be sent
+        and some guessing. A theoretical maximum value is used because it is
+        possible that our nickname, username or hostname changes (on the server
+        side) while the length is still being calculated.
+        """
+        # :nickname!realname@hostname COMMAND ...
+        theoretical = ":{}!{}@{} {}".format(
+            "a" * self.supported.getFeature("NICKLEN"),
+            # This value is based on observation.
+            "b" * 10,
+            # See <http://tools.ietf.org/html/rfc2812#section-2.3.1>.
+            "c" * 63,
+            command,
+        )
+        # Fingers crossed.
+        fudge = 10
+        return MAX_COMMAND_LENGTH - len(theoretical) - fudge
+
+    def _sendMessage(self, msgType, user, message, length=None):
+        """
+        Send a message or notice to a user or channel.
+
+        The message will be split into multiple commands to the server if:
+         - The message contains any newline characters
+         - Any span between newline characters is longer than the given
+           line-length.
+
+        @param msgType: Whether a PRIVMSG or NOTICE should be sent.
+        @type msgType: C{str}
+
+        @param user: Username or channel name to which to direct the
+            message.
+        @type user: C{str}
+
+        @param message: Text to send.
+        @type message: C{str}
+
+        @param length: Maximum number of octets to send in a single
+            command, including the IRC protocol framing. If L{None} is given
+            then L{IRCClient._safeMaximumLineLength} is used to determine a
+            value.
+        @type length: C{int}
+        """
+        fmt = f"{msgType} {user} :"
+
+        if length is None:
+            length = self._safeMaximumLineLength(fmt)
+
+        # Account for the line terminator.
+        minimumLength = len(fmt) + 2
+        if length <= minimumLength:
+            raise ValueError(
+                "Maximum length must exceed %d for message "
+                "to %s" % (minimumLength, user)
+            )
+        for line in split(message, length - minimumLength):
+            self.sendLine(fmt + line)
+
+    def msg(self, user, message, length=None):
+        """
+        Send a message to a user or channel.
+
+        The message will be split into multiple commands to the server if:
+         - The message contains any newline characters
+         - Any span between newline characters is longer than the given
+           line-length.
+
+        @param user: Username or channel name to which to direct the
+            message.
+        @type user: C{str}
+
+        @param message: Text to send.
+        @type message: C{str}
+
+        @param length: Maximum number of octets to send in a single
+            command, including the IRC protocol framing. If L{None} is given
+            then L{IRCClient._safeMaximumLineLength} is used to determine a
+            value.
+        @type length: C{int}
+        """
+        self._sendMessage("PRIVMSG", user, message, length)
+
+    def notice(self, user, message, length=None):
+        """
+        Send a notice to a user.
+
+        Notices are like normal message, but should never get automated
+        replies.
+
+        @type user: C{str}
+        @param user: The user to send a notice to.
+
+        @type message: C{str}
+        @param message: The contents of the notice to send.
+
+        @param length: Maximum number of octets to send in a single
+            command, including the IRC protocol framing. If L{None} is given
+            then L{IRCClient._safeMaximumLineLength} is used to determine a
+            value.
+        @type length: C{int}
+        """
+        self._sendMessage("NOTICE", user, message, length)
+
+    def away(self, message=""):
+        """
+        Mark this client as away.
+
+        @type message: C{str}
+        @param message: If specified, the away message.
+        """
+        self.sendLine("AWAY :%s" % message)
+
+    def back(self):
+        """
+        Clear the away status.
+        """
+        # An empty away marks us as back
+        self.away()
+
+    def whois(self, nickname, server=None):
+        """
+        Retrieve user information about the given nickname.
+
+        @type nickname: C{str}
+        @param nickname: The nickname about which to retrieve information.
+
+        @since: 8.2
+        """
+        if server is None:
+            self.sendLine("WHOIS " + nickname)
+        else:
+            self.sendLine(f"WHOIS {server} {nickname}")
+
+    def register(self, nickname, hostname="foo", servername="bar"):
+        """
+        Login to the server.
+
+        @type nickname: C{str}
+        @param nickname: The nickname to register.
+        @type hostname: C{str}
+        @param hostname: If specified, the hostname to logon as.
+        @type servername: C{str}
+        @param servername: If specified, the servername to logon as.
+        """
+        if self.password is not None:
+            self.sendLine("PASS %s" % self.password)
+        self.setNick(nickname)
+        if self.username is None:
+            self.username = nickname
+        self.sendLine(
+            "USER {} {} {} :{}".format(
+                self.username, hostname, servername, self.realname
+            )
+        )
+
+    def setNick(self, nickname):
+        """
+        Set this client's nickname.
+
+        @type nickname: C{str}
+        @param nickname: The nickname to change to.
+        """
+        self._attemptedNick = nickname
+        self.sendLine("NICK %s" % nickname)
+
+    def quit(self, message=""):
+        """
+        Disconnect from the server
+
+        @type message: C{str}
+
+        @param message: If specified, the message to give when quitting the
+            server.
+        """
+        self.sendLine("QUIT :%s" % message)
+
+    ### user input commands, client->client
+
+    def describe(self, channel, action):
+        """
+        Strike a pose.
+
+        @type channel: C{str}
+        @param channel: The name of the channel to have an action on. If it
+            has no prefix, it is sent to the user of that name.
+        @type action: C{str}
+        @param action: The action to preform.
+        @since: 9.0
+        """
+        self.ctcpMakeQuery(channel, [("ACTION", action)])
+
+    _pings = None
+    _MAX_PINGRING = 12
+
+    def ping(self, user, text=None):
+        """
+        Measure round-trip delay to another IRC client.
+        """
+        if self._pings is None:
+            self._pings = {}
+
+        if text is None:
+            chars = string.ascii_letters + string.digits + string.punctuation
+            key = "".join([random.choice(chars) for i in range(12)])
+        else:
+            key = str(text)
+        self._pings[(user, key)] = time.time()
+        self.ctcpMakeQuery(user, [("PING", key)])
+
+        if len(self._pings) > self._MAX_PINGRING:
+            # Remove some of the oldest entries.
+            byValue = [(v, k) for (k, v) in self._pings.items()]
+            byValue.sort()
+            excess = len(self._pings) - self._MAX_PINGRING
+            for i in range(excess):
+                del self._pings[byValue[i][1]]
+
+    def dccSend(self, user, file):
+        """
+        This is supposed to send a user a file directly.  This generally
+        doesn't work on any client, and this method is included only for
+        backwards compatibility and completeness.
+
+        @param user: C{str} representing the user
+        @param file: an open file (unknown, since this is not implemented)
+        """
+        raise NotImplementedError(
+            "XXX!!! Help!  I need to bind a socket, have it listen, and tell me its address.  "
+            "(and stop accepting once we've made a single connection.)"
+        )
+
+    def dccResume(self, user, fileName, port, resumePos):
+        """
+        Send a DCC RESUME request to another user.
+        """
+        self.ctcpMakeQuery(user, [("DCC", ["RESUME", fileName, port, resumePos])])
+
+    def dccAcceptResume(self, user, fileName, port, resumePos):
+        """
+        Send a DCC ACCEPT response to clients who have requested a resume.
+        """
+        self.ctcpMakeQuery(user, [("DCC", ["ACCEPT", fileName, port, resumePos])])
+
+    ### server->client messages
+    ### You might want to fiddle with these,
+    ### but it is safe to leave them alone.
+
+    def irc_ERR_NICKNAMEINUSE(self, prefix, params):
+        """
+        Called when we try to register or change to a nickname that is already
+        taken.
+        """
+        self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
+        self.setNick(self._attemptedNick)
+
+    def alterCollidedNick(self, nickname):
+        """
+        Generate an altered version of a nickname that caused a collision in an
+        effort to create an unused related name for subsequent registration.
+
+        @param nickname: The nickname a user is attempting to register.
+        @type nickname: C{str}
+
+        @returns: A string that is in some way different from the nickname.
+        @rtype: C{str}
+        """
+        return nickname + "_"
+
+    def irc_ERR_ERRONEUSNICKNAME(self, prefix, params):
+        """
+        Called when we try to register or change to an illegal nickname.
+
+        The server should send this reply when the nickname contains any
+        disallowed characters.  The bot will stall, waiting for RPL_WELCOME, if
+        we don't handle this during sign-on.
+
+        @note: The method uses the spelling I{erroneus}, as it appears in
+            the RFC, section 6.1.
+        """
+        if not self._registered:
+            self.setNick(self.erroneousNickFallback)
+
+    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
+        """
+        Called when the login was incorrect.
+        """
+        raise IRCPasswordMismatch("Password Incorrect.")
+
+    def irc_RPL_WELCOME(self, prefix, params):
+        """
+        Called when we have received the welcome from the server.
+        """
+        self.hostname = prefix
+        self._registered = True
+        self.nickname = self._attemptedNick
+        self.signedOn()
+        self.startHeartbeat()
+
+    def irc_JOIN(self, prefix, params):
+        """
+        Called when a user joins a channel.
+        """
+        nick = prefix.split("!")[0]
+        channel = params[-1]
+        if nick == self.nickname:
+            self.joined(channel)
+        else:
+            self.userJoined(nick, channel)
+
+    def irc_PART(self, prefix, params):
+        """
+        Called when a user leaves a channel.
+        """
+        nick = prefix.split("!")[0]
+        channel = params[0]
+        if nick == self.nickname:
+            self.left(channel)
+        else:
+            self.userLeft(nick, channel)
+
+    def irc_QUIT(self, prefix, params):
+        """
+        Called when a user has quit.
+        """
+        nick = prefix.split("!")[0]
+        self.userQuit(nick, params[0])
+
+    def irc_MODE(self, user, params):
+        """
+        Parse a server mode change message.
+        """
+        channel, modes, args = params[0], params[1], params[2:]
+
+        if modes[0] not in "-+":
+            modes = "+" + modes
+
+        if channel == self.nickname:
+            # This is a mode change to our individual user, not a channel mode
+            # that involves us.
+            paramModes = self.getUserModeParams()
+        else:
+            paramModes = self.getChannelModeParams()
+
+        try:
+            added, removed = parseModes(modes, args, paramModes)
+        except IRCBadModes:
+            log.err(
+                None,
+                "An error occurred while parsing the following "
+                "MODE message: MODE %s" % (" ".join(params),),
+            )
+        else:
+            if added:
+                modes, params = zip(*added)
+                self.modeChanged(user, channel, True, "".join(modes), params)
+
+            if removed:
+                modes, params = zip(*removed)
+                self.modeChanged(user, channel, False, "".join(modes), params)
+
+    def irc_PING(self, prefix, params):
+        """
+        Called when some has pinged us.
+        """
+        self.sendLine("PONG %s" % params[-1])
+
+    def irc_PRIVMSG(self, prefix, params):
+        """
+        Called when we get a message.
+        """
+        user = prefix
+        channel = params[0]
+        message = params[-1]
+
+        if not message:
+            # Don't raise an exception if we get blank message.
+            return
+
+        if message[0] == X_DELIM:
+            m = ctcpExtract(message)
+            if m["extended"]:
+                self.ctcpQuery(user, channel, m["extended"])
+
+            if not m["normal"]:
+                return
+
+            message = " ".join(m["normal"])
+
+        self.privmsg(user, channel, message)
+
+    def irc_NOTICE(self, prefix, params):
+        """
+        Called when a user gets a notice.
+        """
+        user = prefix
+        channel = params[0]
+        message = params[-1]
+
+        if message[0] == X_DELIM:
+            m = ctcpExtract(message)
+            if m["extended"]:
+                self.ctcpReply(user, channel, m["extended"])
+
+            if not m["normal"]:
+                return
+
+            message = " ".join(m["normal"])
+
+        self.noticed(user, channel, message)
+
+    def irc_NICK(self, prefix, params):
+        """
+        Called when a user changes their nickname.
+        """
+        nick = prefix.split("!", 1)[0]
+        if nick == self.nickname:
+            self.nickChanged(params[0])
+        else:
+            self.userRenamed(nick, params[0])
+
+    def irc_KICK(self, prefix, params):
+        """
+        Called when a user is kicked from a channel.
+        """
+        kicker = prefix.split("!")[0]
+        channel = params[0]
+        kicked = params[1]
+        message = params[-1]
+        if kicked.lower() == self.nickname.lower():
+            # Yikes!
+            self.kickedFrom(channel, kicker, message)
+        else:
+            self.userKicked(kicked, channel, kicker, message)
+
+    def irc_TOPIC(self, prefix, params):
+        """
+        Someone in the channel set the topic.
+        """
+        user = prefix.split("!")[0]
+        channel = params[0]
+        newtopic = params[1]
+        self.topicUpdated(user, channel, newtopic)
+
+    def irc_RPL_TOPIC(self, prefix, params):
+        """
+        Called when the topic for a channel is initially reported or when it
+        subsequently changes.
+        """
+        user = prefix.split("!")[0]
+        channel = params[1]
+        newtopic = params[2]
+        self.topicUpdated(user, channel, newtopic)
+
+    def irc_RPL_NOTOPIC(self, prefix, params):
+        user = prefix.split("!")[0]
+        channel = params[1]
+        newtopic = ""
+        self.topicUpdated(user, channel, newtopic)
+
+    def irc_RPL_MOTDSTART(self, prefix, params):
+        if params[-1].startswith("- "):
+            params[-1] = params[-1][2:]
+        self.motd = [params[-1]]
+
+    def irc_RPL_MOTD(self, prefix, params):
+        if params[-1].startswith("- "):
+            params[-1] = params[-1][2:]
+        if self.motd is None:
+            self.motd = []
+        self.motd.append(params[-1])
+
+    def irc_RPL_ENDOFMOTD(self, prefix, params):
+        """
+        I{RPL_ENDOFMOTD} indicates the end of the message of the day
+        messages.  Deliver the accumulated lines to C{receivedMOTD}.
+        """
+        motd = self.motd
+        self.motd = None
+        self.receivedMOTD(motd)
+
+    def irc_RPL_CREATED(self, prefix, params):
+        self.created(params[1])
+
+    def irc_RPL_YOURHOST(self, prefix, params):
+        self.yourHost(params[1])
+
+    def irc_RPL_MYINFO(self, prefix, params):
+        info = params[1].split(None, 3)
+        while len(info) < 4:
+            info.append(None)
+        self.myInfo(*info)
+
+    def irc_RPL_BOUNCE(self, prefix, params):
+        self.bounce(params[1])
+
+    def irc_RPL_ISUPPORT(self, prefix, params):
+        args = params[1:-1]
+        # Several ISUPPORT messages, in no particular order, may be sent
+        # to the client at any given point in time (usually only on connect,
+        # though.) For this reason, ServerSupportedFeatures.parse is intended
+        # to mutate the supported feature list.
+        self.supported.parse(args)
+        self.isupport(args)
+
+    def irc_RPL_LUSERCLIENT(self, prefix, params):
+        self.luserClient(params[1])
+
+    def irc_RPL_LUSEROP(self, prefix, params):
+        try:
+            self.luserOp(int(params[1]))
+        except ValueError:
+            pass
+
+    def irc_RPL_LUSERCHANNELS(self, prefix, params):
+        try:
+            self.luserChannels(int(params[1]))
+        except ValueError:
+            pass
+
+    def irc_RPL_LUSERME(self, prefix, params):
+        self.luserMe(params[1])
+
+    def irc_unknown(self, prefix, command, params):
+        pass
+
+    ### Receiving a CTCP query from another party
+    ### It is safe to leave these alone.
+
+    def ctcpQuery(self, user, channel, messages):
+        """
+        Dispatch method for any CTCP queries received.
+
+        Duplicated CTCP queries are ignored and no dispatch is
+        made. Unrecognized CTCP queries invoke L{IRCClient.ctcpUnknownQuery}.
+        """
+        seen = set()
+        for tag, data in messages:
+            method = getattr(self, "ctcpQuery_%s" % tag, None)
+            if tag not in seen:
+                if method is not None:
+                    method(user, channel, data)
+                else:
+                    self.ctcpUnknownQuery(user, channel, tag, data)
+            seen.add(tag)
+
+    def ctcpUnknownQuery(self, user, channel, tag, data):
+        """
+        Fallback handler for unrecognized CTCP queries.
+
+        No CTCP I{ERRMSG} reply is made to remove a potential denial of service
+        avenue.
+        """
+        log.msg(f"Unknown CTCP query from {user!r}: {tag!r} {data!r}")
+
+    def ctcpQuery_ACTION(self, user, channel, data):
+        self.action(user, channel, data)
+
+    def ctcpQuery_PING(self, user, channel, data):
+        nick = user.split("!")[0]
+        self.ctcpMakeReply(nick, [("PING", data)])
+
+    def ctcpQuery_FINGER(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage(f"Why did {user} send '{data}' with a FINGER query?")
+        if not self.fingerReply:
+            return
+
+        if callable(self.fingerReply):
+            reply = self.fingerReply()
+        else:
+            reply = str(self.fingerReply)
+
+        nick = user.split("!")[0]
+        self.ctcpMakeReply(nick, [("FINGER", reply)])
+
+    def ctcpQuery_VERSION(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage(f"Why did {user} send '{data}' with a VERSION query?")
+
+        if self.versionName:
+            nick = user.split("!")[0]
+            self.ctcpMakeReply(
+                nick,
+                [
+                    (
+                        "VERSION",
+                        "%s:%s:%s"
+                        % (
+                            self.versionName,
+                            self.versionNum or "",
+                            self.versionEnv or "",
+                        ),
+                    )
+                ],
+            )
+
+    def ctcpQuery_SOURCE(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage(f"Why did {user} send '{data}' with a SOURCE query?")
+        if self.sourceURL:
+            nick = user.split("!")[0]
+            # The CTCP document (Zeuge, Rollo, Mesander 1994) says that SOURCE
+            # replies should be responded to with the location of an anonymous
+            # FTP server in host:directory:file format.  I'm taking the liberty
+            # of bringing it into the 21st century by sending a URL instead.
+            self.ctcpMakeReply(nick, [("SOURCE", self.sourceURL), ("SOURCE", None)])
+
+    def ctcpQuery_USERINFO(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage(f"Why did {user} send '{data}' with a USERINFO query?")
+        if self.userinfo:
+            nick = user.split("!")[0]
+            self.ctcpMakeReply(nick, [("USERINFO", self.userinfo)])
+
+    def ctcpQuery_CLIENTINFO(self, user, channel, data):
+        """
+        A master index of what CTCP tags this client knows.
+
+        If no arguments are provided, respond with a list of known tags, sorted
+        in alphabetical order.
+        If an argument is provided, provide human-readable help on
+        the usage of that tag.
+        """
+        nick = user.split("!")[0]
+        if not data:
+            # XXX: prefixedMethodNames gets methods from my *class*,
+            # but it's entirely possible that this *instance* has more
+            # methods.
+            names = sorted(reflect.prefixedMethodNames(self.__class__, "ctcpQuery_"))
+
+            self.ctcpMakeReply(nick, [("CLIENTINFO", " ".join(names))])
+        else:
+            args = data.split()
+            method = getattr(self, f"ctcpQuery_{args[0]}", None)
+            if not method:
+                self.ctcpMakeReply(
+                    nick,
+                    [
+                        (
+                            "ERRMSG",
+                            "CLIENTINFO %s :" "Unknown query '%s'" % (data, args[0]),
+                        )
+                    ],
+                )
+                return
+            doc = getattr(method, "__doc__", "")
+            self.ctcpMakeReply(nick, [("CLIENTINFO", doc)])
+
+    def ctcpQuery_ERRMSG(self, user, channel, data):
+        # Yeah, this seems strange, but that's what the spec says to do
+        # when faced with an ERRMSG query (not a reply).
+        nick = user.split("!")[0]
+        self.ctcpMakeReply(nick, [("ERRMSG", "%s :No error has occurred." % data)])
+
+    def ctcpQuery_TIME(self, user, channel, data):
+        if data is not None:
+            self.quirkyMessage(f"Why did {user} send '{data}' with a TIME query?")
+        nick = user.split("!")[0]
+        self.ctcpMakeReply(
+            nick, [("TIME", ":%s" % time.asctime(time.localtime(time.time())))]
+        )
+
+    def ctcpQuery_DCC(self, user, channel, data):
+        """
+        Initiate a Direct Client Connection
+
+        @param user: The hostmask of the user/client.
+        @type user: L{bytes}
+
+        @param channel: The name of the IRC channel.
+        @type channel: L{bytes}
+
+        @param data: The DCC request message.
+        @type data: L{bytes}
+        """
+
+        if not data:
+            return
+        dcctype = data.split(None, 1)[0].upper()
+        handler = getattr(self, "dcc_" + dcctype, None)
+        if handler:
+            if self.dcc_sessions is None:
+                self.dcc_sessions = []
+            data = data[len(dcctype) + 1 :]
+            handler(user, channel, data)
+        else:
+            nick = user.split("!")[0]
+            self.ctcpMakeReply(
+                nick,
+                [("ERRMSG", f"DCC {data} :Unknown DCC type '{dcctype}'")],
+            )
+            self.quirkyMessage(f"{user} offered unknown DCC type {dcctype}")
+
+    def dcc_SEND(self, user, channel, data):
+        # Use shlex.split for those who send files with spaces in the names.
+        data = shlex.split(data)
+        if len(data) < 3:
+            raise IRCBadMessage(f"malformed DCC SEND request: {data!r}")
+
+        (filename, address, port) = data[:3]
+
+        address = dccParseAddress(address)
+        try:
+            port = int(port)
+        except ValueError:
+            raise IRCBadMessage(f"Indecipherable port {port!r}")
+
+        size = -1
+        if len(data) >= 4:
+            try:
+                size = int(data[3])
+            except ValueError:
+                pass
+
+        # XXX Should we bother passing this data?
+        self.dccDoSend(user, address, port, filename, size, data)
+
+    def dcc_ACCEPT(self, user, channel, data):
+        data = shlex.split(data)
+        if len(data) < 3:
+            raise IRCBadMessage(f"malformed DCC SEND ACCEPT request: {data!r}")
+        (filename, port, resumePos) = data[:3]
+        try:
+            port = int(port)
+            resumePos = int(resumePos)
+        except ValueError:
+            return
+
+        self.dccDoAcceptResume(user, filename, port, resumePos)
+
+    def dcc_RESUME(self, user, channel, data):
+        data = shlex.split(data)
+        if len(data) < 3:
+            raise IRCBadMessage(f"malformed DCC SEND RESUME request: {data!r}")
+        (filename, port, resumePos) = data[:3]
+        try:
+            port = int(port)
+            resumePos = int(resumePos)
+        except ValueError:
+            return
+
+        self.dccDoResume(user, filename, port, resumePos)
+
+    def dcc_CHAT(self, user, channel, data):
+        data = shlex.split(data)
+        if len(data) < 3:
+            raise IRCBadMessage(f"malformed DCC CHAT request: {data!r}")
+
+        (filename, address, port) = data[:3]
+
+        address = dccParseAddress(address)
+        try:
+            port = int(port)
+        except ValueError:
+            raise IRCBadMessage(f"Indecipherable port {port!r}")
+
+        self.dccDoChat(user, channel, address, port, data)
+
+    ### The dccDo methods are the slightly higher-level siblings of
+    ### common dcc_ methods; the arguments have been parsed for them.
+
+    def dccDoSend(self, user, address, port, fileName, size, data):
+        """
+        Called when I receive a DCC SEND offer from a client.
+
+        By default, I do nothing here.
+
+        @param user: The hostmask of the requesting user.
+        @type user: L{bytes}
+
+        @param address: The IP address of the requesting user.
+        @type address: L{bytes}
+
+        @param port: An integer representing the port of the requesting user.
+        @type port: L{int}
+
+        @param fileName: The name of the file to be transferred.
+        @type fileName: L{bytes}
+
+        @param size: The size of the file to be transferred, which may be C{-1}
+            if the size of the file was not specified in the DCC SEND request.
+        @type size: L{int}
+
+        @param data: A 3-list of [fileName, address, port].
+        @type data: L{list}
+        """
+
+    def dccDoResume(self, user, file, port, resumePos):
+        """
+        Called when a client is trying to resume an offered file via DCC send.
+        It should be either replied to with a DCC ACCEPT or ignored (default).
+
+        @param user: The hostmask of the user who wants to resume the transfer
+            of a file previously offered via DCC send.
+        @type user: L{bytes}
+
+        @param file: The name of the file to resume the transfer of.
+        @type file: L{bytes}
+
+        @param port: An integer representing the port of the requesting user.
+        @type port: L{int}
+
+        @param resumePos: The position in the file from where the transfer
+            should resume.
+        @type resumePos: L{int}
+        """
+        pass
+
+    def dccDoAcceptResume(self, user, file, port, resumePos):
+        """
+        Called when a client has verified and accepted a DCC resume request
+        made by us.  By default it will do nothing.
+
+        @param user: The hostmask of the user who has accepted the DCC resume
+            request.
+        @type user: L{bytes}
+
+        @param file: The name of the file to resume the transfer of.
+        @type file: L{bytes}
+
+        @param port: An integer representing the port of the accepting user.
+        @type port: L{int}
+
+        @param resumePos: The position in the file from where the transfer
+            should resume.
+        @type resumePos: L{int}
+        """
+        pass
+
+    def dccDoChat(self, user, channel, address, port, data):
+        pass
+        # factory = DccChatFactory(self, queryData=(user, channel, data))
+        # reactor.connectTCP(address, port, factory)
+        # self.dcc_sessions.append(factory)
+
+    # def ctcpQuery_SED(self, user, data):
+    #    """Simple Encryption Doodoo
+    #
+    #    Feel free to implement this, but no specification is available.
+    #    """
+    #    raise NotImplementedError
+
+    def ctcpMakeReply(self, user, messages):
+        """
+        Send one or more C{extended messages} as a CTCP reply.
+
+        @type messages: a list of extended messages.  An extended
+        message is a (tag, data) tuple, where 'data' may be L{None}.
+        """
+        self.notice(user, ctcpStringify(messages))
+
+    ### client CTCP query commands
+
+    def ctcpMakeQuery(self, user, messages):
+        """
+        Send one or more C{extended messages} as a CTCP query.
+
+        @type messages: a list of extended messages.  An extended
+        message is a (tag, data) tuple, where 'data' may be L{None}.
+        """
+        self.msg(user, ctcpStringify(messages))
+
+    ### Receiving a response to a CTCP query (presumably to one we made)
+    ### You may want to add methods here, or override UnknownReply.
+
+    def ctcpReply(self, user, channel, messages):
+        """
+        Dispatch method for any CTCP replies received.
+        """
+        for m in messages:
+            method = getattr(self, "ctcpReply_%s" % m[0], None)
+            if method:
+                method(user, channel, m[1])
+            else:
+                self.ctcpUnknownReply(user, channel, m[0], m[1])
+
+    def ctcpReply_PING(self, user, channel, data):
+        nick = user.split("!", 1)[0]
+        if (not self._pings) or ((nick, data) not in self._pings):
+            raise IRCBadMessage(f"Bogus PING response from {user}: {data}")
+
+        t0 = self._pings[(nick, data)]
+        self.pong(user, time.time() - t0)
+
+    def ctcpUnknownReply(self, user, channel, tag, data):
+        """
+        Called when a fitting ctcpReply_ method is not found.
+
+        @param user: The hostmask of the user.
+        @type user: L{bytes}
+
+        @param channel: The name of the IRC channel.
+        @type channel: L{bytes}
+
+        @param tag: The CTCP request tag for which no fitting method is found.
+        @type tag: L{bytes}
+
+        @param data: The CTCP message.
+        @type data: L{bytes}
+        """
+        # FIXME:7560:
+        # Add code for handling arbitrary queries and not treat them as
+        # anomalies.
+
+        log.msg(f"Unknown CTCP reply from {user}: {tag} {data}\n")
+
+    ### Error handlers
+    ### You may override these with something more appropriate to your UI.
+
+    def badMessage(self, line, excType, excValue, tb):
+        """
+        When I get a message that's so broken I can't use it.
+
+        @param line: The indecipherable message.
+        @type line: L{bytes}
+
+        @param excType: The exception type of the exception raised by the
+            message.
+        @type excType: L{type}
+
+        @param excValue: The exception parameter of excType or its associated
+            value(the second argument to C{raise}).
+        @type excValue: L{BaseException}
+
+        @param tb: The Traceback as a traceback object.
+        @type tb: L{traceback}
+        """
+        log.msg(line)
+        log.msg("".join(traceback.format_exception(excType, excValue, tb)))
+
+    def quirkyMessage(self, s):
+        """
+        This is called when I receive a message which is peculiar, but not
+        wholly indecipherable.
+
+        @param s: The peculiar message.
+        @type s: L{bytes}
+        """
+        log.msg(s + "\n")
+
+    ### Protocol methods
+
+    def connectionMade(self):
+        self.supported = ServerSupportedFeatures()
+        self._queue = []
+        if self.performLogin:
+            self.register(self.nickname)
+
+    def dataReceived(self, data):
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        data = data.replace(b"\r", b"")
+        basic.LineReceiver.dataReceived(self, data)
+
+    def lineReceived(self, line):
+        if bytes != str and isinstance(line, bytes):
+            # decode bytes from transport to unicode
+            line = line.decode("utf-8")
+
+        line = lowDequote(line)
+        try:
+            prefix, command, params = parsemsg(line)
+            if command in numeric_to_symbolic:
+                command = numeric_to_symbolic[command]
+            self.handleCommand(command, prefix, params)
+        except IRCBadMessage:
+            self.badMessage(line, *sys.exc_info())
+
+    def getUserModeParams(self):
+        """
+        Get user modes that require parameters for correct parsing.
+
+        @rtype: C{[str, str]}
+        @return: C{[add, remove]}
+        """
+        return ["", ""]
+
+    def getChannelModeParams(self):
+        """
+        Get channel modes that require parameters for correct parsing.
+
+        @rtype: C{[str, str]}
+        @return: C{[add, remove]}
+        """
+        # PREFIX modes are treated as "type B" CHANMODES, they always take
+        # parameter.
+        params = ["", ""]
+        prefixes = self.supported.getFeature("PREFIX", {})
+        params[0] = params[1] = "".join(prefixes.keys())
+
+        chanmodes = self.supported.getFeature("CHANMODES")
+        if chanmodes is not None:
+            params[0] += chanmodes.get("addressModes", "")
+            params[0] += chanmodes.get("param", "")
+            params[1] = params[0]
+            params[0] += chanmodes.get("setParam", "")
+        return params
+
+    def handleCommand(self, command, prefix, params):
+        """
+        Determine the function to call for the given command and call it with
+        the given arguments.
+
+        @param command: The IRC command to determine the function for.
+        @type command: L{bytes}
+
+        @param prefix: The prefix of the IRC message (as returned by
+            L{parsemsg}).
+        @type prefix: L{bytes}
+
+        @param params: A list of parameters to call the function with.
+        @type params: L{list}
+        """
+        method = getattr(self, "irc_%s" % command, None)
+        try:
+            if method is not None:
+                method(prefix, params)
+            else:
+                self.irc_unknown(prefix, command, params)
+        except BaseException:
+            log.deferr()
+
+    def __getstate__(self):
+        dct = self.__dict__.copy()
+        dct["dcc_sessions"] = None
+        dct["_pings"] = None
+        return dct
+
+
+def dccParseAddress(address):
+    if "." in address:
+        pass
+    else:
+        try:
+            address = int(address)
+        except ValueError:
+            raise IRCBadMessage(f"Indecipherable address {address!r}")
+        else:
+            address = (
+                (address >> 24) & 0xFF,
+                (address >> 16) & 0xFF,
+                (address >> 8) & 0xFF,
+                address & 0xFF,
+            )
+            address = ".".join(map(str, address))
+    return address
+
+
+class DccFileReceiveBasic(protocol.Protocol, styles.Ephemeral):
+    """
+    Bare protocol to receive a Direct Client Connection SEND stream.
+
+    This does enough to keep the other guy talking, but you'll want to extend
+    my dataReceived method to *do* something with the data I get.
+
+    @ivar bytesReceived: An integer representing the number of bytes of data
+        received.
+    @type bytesReceived: L{int}
+    """
+
+    bytesReceived = 0
+
+    def __init__(self, resumeOffset=0):
+        """
+        @param resumeOffset: An integer representing the amount of bytes from
+            where the transfer of data should be resumed.
+        @type resumeOffset: L{int}
+        """
+        self.bytesReceived = resumeOffset
+        self.resume = resumeOffset != 0
+
+    def dataReceived(self, data):
+        """
+        See: L{protocol.Protocol.dataReceived}
+
+        Warning: This just acknowledges to the remote host that the data has
+        been received; it doesn't I{do} anything with the data, so you'll want
+        to override this.
+        """
+        self.bytesReceived = self.bytesReceived + len(data)
+        self.transport.write(struct.pack("!i", self.bytesReceived))
+
+
+class DccSendProtocol(protocol.Protocol, styles.Ephemeral):
+    """
+    Protocol for an outgoing Direct Client Connection SEND.
+
+    @ivar blocksize: An integer representing the size of an individual block of
+        data.
+    @type blocksize: L{int}
+
+    @ivar file: The file to be sent.  This can be either a file object or
+        simply the name of the file.
+    @type file: L{file} or L{bytes}
+
+    @ivar bytesSent: An integer representing the number of bytes sent.
+    @type bytesSent: L{int}
+
+    @ivar completed: An integer representing whether the transfer has been
+        completed or not.
+    @type completed: L{int}
+
+    @ivar connected: An integer representing whether the connection has been
+        established or not.
+    @type connected: L{int}
+    """
+
+    blocksize = 1024
+    file = None
+    bytesSent = 0
+    completed = 0
+    connected = 0
+
+    def __init__(self, file):
+        if type(file) is str:
+            self.file = open(file)
+
+    def connectionMade(self):
+        self.connected = 1
+        self.sendBlock()
+
+    def dataReceived(self, data):
+        # XXX: Do we need to check to see if len(data) != fmtsize?
+
+        bytesShesGot = struct.unpack("!I", data)
+        if bytesShesGot < self.bytesSent:
+            # Wait for her.
+            # XXX? Add some checks to see if we've stalled out?
+            return
+        elif bytesShesGot > self.bytesSent:
+            # self.transport.log("DCC SEND %s: She says she has %d bytes "
+            #                    "but I've only sent %d.  I'm stopping "
+            #                    "this screwy transfer."
+            #                    % (self.file,
+            #                       bytesShesGot, self.bytesSent))
+            self.transport.loseConnection()
+            return
+
+        self.sendBlock()
+
+    def sendBlock(self):
+        block = self.file.read(self.blocksize)
+        if block:
+            self.transport.write(block)
+            self.bytesSent = self.bytesSent + len(block)
+        else:
+            # Nothing more to send, transfer complete.
+            self.transport.loseConnection()
+            self.completed = 1
+
+    def connectionLost(self, reason):
+        self.connected = 0
+        if hasattr(self.file, "close"):
+            self.file.close()
+
+
+class DccSendFactory(protocol.Factory):
+    protocol = DccSendProtocol  # type: ignore[assignment]
+
+    def __init__(self, file):
+        self.file = file
+
+    def buildProtocol(self, connection):
+        p = self.protocol(self.file)
+        p.factory = self
+        return p
+
+
+def fileSize(file):
+    """
+    I'll try my damndest to determine the size of this file object.
+
+    @param file: The file object to determine the size of.
+    @type file: L{io.IOBase}
+
+    @rtype: L{int} or L{None}
+    @return: The size of the file object as an integer if it can be determined,
+        otherwise return L{None}.
+    """
+    size = None
+    if hasattr(file, "fileno"):
+        fileno = file.fileno()
+        try:
+            stat_ = os.fstat(fileno)
+            size = stat_[stat.ST_SIZE]
+        except BaseException:
+            pass
+        else:
+            return size
+
+    if hasattr(file, "name") and path.exists(file.name):
+        try:
+            size = path.getsize(file.name)
+        except BaseException:
+            pass
+        else:
+            return size
+
+    if hasattr(file, "seek") and hasattr(file, "tell"):
+        try:
+            try:
+                file.seek(0, 2)
+                size = file.tell()
+            finally:
+                file.seek(0, 0)
+        except BaseException:
+            pass
+        else:
+            return size
+
+    return size
+
+
+class DccChat(basic.LineReceiver, styles.Ephemeral):
+    """
+    Direct Client Connection protocol type CHAT.
+
+    DCC CHAT is really just your run o' the mill basic.LineReceiver
+    protocol.  This class only varies from that slightly, accepting
+    either LF or CR LF for a line delimeter for incoming messages
+    while always using CR LF for outgoing.
+
+    The lineReceived method implemented here uses the DCC connection's
+    'client' attribute (provided upon construction) to deliver incoming
+    lines from the DCC chat via IRCClient's normal privmsg interface.
+    That's something of a spoof, which you may well want to override.
+    """
+
+    queryData = None
+    delimiter = CR.encode("ascii") + NL.encode("ascii")
+    client = None
+    remoteParty = None
+    buffer = b""
+
+    def __init__(self, client, queryData=None):
+        """
+        Initialize a new DCC CHAT session.
+
+        queryData is a 3-tuple of
+        (fromUser, targetUserOrChannel, data)
+        as received by the CTCP query.
+
+        (To be honest, fromUser is the only thing that's currently
+        used here. targetUserOrChannel is potentially useful, while
+        the 'data' argument is solely for informational purposes.)
+        """
+        self.client = client
+        if queryData:
+            self.queryData = queryData
+            self.remoteParty = self.queryData[0]
+
+    def dataReceived(self, data):
+        self.buffer = self.buffer + data
+        lines = self.buffer.split(LF)
+        # Put the (possibly empty) element after the last LF back in the
+        # buffer
+        self.buffer = lines.pop()
+
+        for line in lines:
+            if line[-1] == CR:
+                line = line[:-1]
+            self.lineReceived(line)
+
+    def lineReceived(self, line):
+        log.msg(f"DCC CHAT<{self.remoteParty}> {line}")
+        self.client.privmsg(self.remoteParty, self.client.nickname, line)
+
+
+class DccChatFactory(protocol.ClientFactory):
+    protocol = DccChat  # type: ignore[assignment]
+    noisy = False
+
+    def __init__(self, client, queryData):
+        self.client = client
+        self.queryData = queryData
+
+    def buildProtocol(self, addr):
+        p = self.protocol(client=self.client, queryData=self.queryData)
+        p.factory = self
+        return p
+
+    def clientConnectionFailed(self, unused_connector, unused_reason):
+        self.client.dcc_sessions.remove(self)
+
+    def clientConnectionLost(self, unused_connector, unused_reason):
+        self.client.dcc_sessions.remove(self)
+
+
+def dccDescribe(data):
+    """
+    Given the data chunk from a DCC query, return a descriptive string.
+
+    @param data: The data from a DCC query.
+    @type data: L{bytes}
+
+    @rtype: L{bytes}
+    @return: A descriptive string.
+    """
+
+    orig_data = data
+    data = data.split()
+    if len(data) < 4:
+        return orig_data
+
+    (dcctype, arg, address, port) = data[:4]
+
+    if "." in address:
+        pass
+    else:
+        try:
+            address = int(address)
+        except ValueError:
+            pass
+        else:
+            address = (
+                (address >> 24) & 0xFF,
+                (address >> 16) & 0xFF,
+                (address >> 8) & 0xFF,
+                address & 0xFF,
+            )
+            address = ".".join(map(str, address))
+
+    if dcctype == "SEND":
+        filename = arg
+
+        size_txt = ""
+        if len(data) >= 5:
+            try:
+                size = int(data[4])
+                size_txt = " of size %d bytes" % (size,)
+            except ValueError:
+                pass
+
+        dcc_text = "SEND for file '{}'{} at host {}, port {}".format(
+            filename,
+            size_txt,
+            address,
+            port,
+        )
+    elif dcctype == "CHAT":
+        dcc_text = f"CHAT for host {address}, port {port}"
+    else:
+        dcc_text = orig_data
+
+    return dcc_text
+
+
+class DccFileReceive(DccFileReceiveBasic):
+    """
+    Higher-level coverage for getting a file from DCC SEND.
+
+    I allow you to change the file's name and destination directory.  I won't
+    overwrite an existing file unless I've been told it's okay to do so.  If
+    passed the resumeOffset keyword argument I will attempt to resume the file
+    from that amount of bytes.
+
+    XXX: I need to let the client know when I am finished.
+    XXX: I need to decide how to keep a progress indicator updated.
+    XXX: Client needs a way to tell me "Do not finish until I say so."
+    XXX: I need to make sure the client understands if the file cannot be written.
+
+    @ivar filename: The name of the file to get.
+    @type filename: L{bytes}
+
+    @ivar fileSize: The size of the file to get, which has a default value of
+        C{-1} if the size of the file was not specified in the DCC SEND
+        request.
+    @type fileSize: L{int}
+
+    @ivar destDir: The destination directory for the file to be received.
+    @type destDir: L{bytes}
+
+    @ivar overwrite: An integer representing whether an existing file should be
+        overwritten or not.  This initially is an L{int} but can be modified to
+        be a L{bool} using the L{set_overwrite} method.
+    @type overwrite: L{int} or L{bool}
+
+    @ivar queryData: queryData is a 3-tuple of (user, channel, data).
+    @type queryData: L{tuple}
+
+    @ivar fromUser: This is the hostmask of the requesting user and is found at
+        index 0 of L{queryData}.
+    @type fromUser: L{bytes}
+    """
+
+    filename = "dcc"
+    fileSize = -1
+    destDir = "."
+    overwrite = 0
+    fromUser: Optional[bytes] = None
+    queryData = None
+
+    def __init__(
+        self, filename, fileSize=-1, queryData=None, destDir=".", resumeOffset=0
+    ):
+        DccFileReceiveBasic.__init__(self, resumeOffset=resumeOffset)
+        self.filename = filename
+        self.destDir = destDir
+        self.fileSize = fileSize
+        self._resumeOffset = resumeOffset
+
+        if queryData:
+            self.queryData = queryData
+            self.fromUser = self.queryData[0]
+
+    def set_directory(self, directory):
+        """
+        Set the directory where the downloaded file will be placed.
+
+        May raise OSError if the supplied directory path is not suitable.
+
+        @param directory: The directory where the file to be received will be
+            placed.
+        @type directory: L{bytes}
+        """
+        if not path.exists(directory):
+            raise OSError(errno.ENOENT, "You see no directory there.", directory)
+        if not path.isdir(directory):
+            raise OSError(
+                errno.ENOTDIR,
+                "You cannot put a file into " "something which is not a directory.",
+                directory,
+            )
+        if not os.access(directory, os.X_OK | os.W_OK):
+            raise OSError(
+                errno.EACCES, "This directory is too hard to write in to.", directory
+            )
+        self.destDir = directory
+
+    def set_filename(self, filename):
+        """
+        Change the name of the file being transferred.
+
+        This replaces the file name provided by the sender.
+
+        @param filename: The new name for the file.
+        @type filename: L{bytes}
+        """
+        self.filename = filename
+
+    def set_overwrite(self, boolean):
+        """
+        May I overwrite existing files?
+
+        @param boolean: A boolean value representing whether existing files
+            should be overwritten or not.
+        @type boolean: L{bool}
+        """
+        self.overwrite = boolean
+
+    # Protocol-level methods.
+
+    def connectionMade(self):
+        dst = path.abspath(path.join(self.destDir, self.filename))
+        exists = path.exists(dst)
+        if self.resume and exists:
+            # I have been told I want to resume, and a file already
+            # exists - Here we go
+            self.file = open(dst, "rb+")
+            self.file.seek(self._resumeOffset)
+            self.file.truncate()
+            log.msg(
+                "Attempting to resume %s - starting from %d bytes"
+                % (self.file, self.file.tell())
+            )
+        elif self.resume and not exists:
+            raise OSError(
+                errno.ENOENT,
+                "You cannot resume writing to a file " "that does not exist!",
+                dst,
+            )
+        elif self.overwrite or not exists:
+            self.file = open(dst, "wb")
+        else:
+            raise OSError(
+                errno.EEXIST,
+                "There's a file in the way.  " "Perhaps that's why you cannot open it.",
+                dst,
+            )
+
+    def dataReceived(self, data):
+        self.file.write(data)
+        DccFileReceiveBasic.dataReceived(self, data)
+
+        # XXX: update a progress indicator here?
+
+    def connectionLost(self, reason):
+        """
+        When the connection is lost, I close the file.
+
+        @param reason: The reason why the connection was lost.
+        @type reason: L{Failure}
+        """
+        self.connected = 0
+        logmsg = f"{self} closed."
+        if self.fileSize > 0:
+            logmsg = "%s  %d/%d bytes received" % (
+                logmsg,
+                self.bytesReceived,
+                self.fileSize,
+            )
+            if self.bytesReceived == self.fileSize:
+                pass  # Hooray!
+            elif self.bytesReceived < self.fileSize:
+                logmsg = "%s (Warning: %d bytes short)" % (
+                    logmsg,
+                    self.fileSize - self.bytesReceived,
+                )
+            else:
+                logmsg = f"{logmsg} (file larger than expected)"
+        else:
+            logmsg = "%s  %d bytes received" % (logmsg, self.bytesReceived)
+
+        if hasattr(self, "file"):
+            logmsg = f"{logmsg} and written to {self.file.name}.\n"
+            if hasattr(self.file, "close"):
+                self.file.close()
+
+        # self.transport.log(logmsg)
+
+    def __str__(self) -> str:
+        if not self.connected:
+            return f"<Unconnected DccFileReceive object at {id(self):x}>"
+        transport = self.transport
+        assert transport is not None
+        from_ = str(transport.getPeer())
+        if self.fromUser is not None:
+            from_ = f"{self.fromUser!r} ({from_})"
+
+        s = f"DCC transfer of '{self.filename}' from {from_}"
+        return s
+
+    def __repr__(self) -> str:
+        s = f"<{self.__class__} at {id(self):x}: GET {self.filename}>"
+        return s
+
+
+_OFF = "\x0f"
+_BOLD = "\x02"
+_COLOR = "\x03"
+_REVERSE_VIDEO = "\x16"
+_UNDERLINE = "\x1f"
+
+# Mapping of IRC color names to their color values.
+_IRC_COLORS = dict(
+    zip(
+        [
+            "white",
+            "black",
+            "blue",
+            "green",
+            "lightRed",
+            "red",
+            "magenta",
+            "orange",
+            "yellow",
+            "lightGreen",
+            "cyan",
+            "lightCyan",
+            "lightBlue",
+            "lightMagenta",
+            "gray",
+            "lightGray",
+        ],
+        range(16),
+    )
+)
+
+# Mapping of IRC color values to their color names.
+_IRC_COLOR_NAMES = {code: name for name, code in _IRC_COLORS.items()}
+
+
+class _CharacterAttributes(_textattributes.CharacterAttributesMixin):
+    """
+    Factory for character attributes, including foreground and background color
+    and non-color attributes such as bold, reverse video and underline.
+
+    Character attributes are applied to actual text by using object
+    indexing-syntax (C{obj['abc']}) after accessing a factory attribute, for
+    example::
+
+        attributes.bold['Some text']
+
+    These can be nested to mix attributes::
+
+        attributes.bold[attributes.underline['Some text']]
+
+    And multiple values can be passed::
+
+        attributes.normal[attributes.bold['Some'], ' text']
+
+    Non-color attributes can be accessed by attribute name, available
+    attributes are:
+
+        - bold
+        - reverseVideo
+        - underline
+
+    Available colors are:
+
+        0. white
+        1. black
+        2. blue
+        3. green
+        4. light red
+        5. red
+        6. magenta
+        7. orange
+        8. yellow
+        9. light green
+        10. cyan
+        11. light cyan
+        12. light blue
+        13. light magenta
+        14. gray
+        15. light gray
+
+    @ivar fg: Foreground colors accessed by attribute name, see above
+        for possible names.
+
+    @ivar bg: Background colors accessed by attribute name, see above
+        for possible names.
+
+    @since: 13.1
+    """
+
+    fg = _textattributes._ColorAttribute(
+        _textattributes._ForegroundColorAttr, _IRC_COLORS
+    )
+    bg = _textattributes._ColorAttribute(
+        _textattributes._BackgroundColorAttr, _IRC_COLORS
+    )
+
+    attrs = {"bold": _BOLD, "reverseVideo": _REVERSE_VIDEO, "underline": _UNDERLINE}
+
+
+attributes = _CharacterAttributes()
+
+
+class _FormattingState(_textattributes._FormattingStateMixin):
+    """
+    Formatting state/attributes of a single character.
+
+    Attributes include:
+        - Formatting nullifier
+        - Bold
+        - Underline
+        - Reverse video
+        - Foreground color
+        - Background color
+
+    @since: 13.1
+    """
+
+    compareAttributes = (
+        "off",
+        "bold",
+        "underline",
+        "reverseVideo",
+        "foreground",
+        "background",
+    )
+
+    def __init__(
+        self,
+        off=False,
+        bold=False,
+        underline=False,
+        reverseVideo=False,
+        foreground=None,
+        background=None,
+    ):
+        self.off = off
+        self.bold = bold
+        self.underline = underline
+        self.reverseVideo = reverseVideo
+        self.foreground = foreground
+        self.background = background
+
+    def toMIRCControlCodes(self):
+        """
+        Emit a mIRC control sequence that will set up all the attributes this
+        formatting state has set.
+
+        @return: A string containing mIRC control sequences that mimic this
+            formatting state.
+        """
+        attrs = []
+        if self.bold:
+            attrs.append(_BOLD)
+        if self.underline:
+            attrs.append(_UNDERLINE)
+        if self.reverseVideo:
+            attrs.append(_REVERSE_VIDEO)
+        if self.foreground is not None or self.background is not None:
+            c = ""
+            if self.foreground is not None:
+                c += "%02d" % (self.foreground,)
+            if self.background is not None:
+                c += ",%02d" % (self.background,)
+            attrs.append(_COLOR + c)
+        return _OFF + "".join(map(str, attrs))
+
+
+def _foldr(f, z, xs):
+    """
+    Apply a function of two arguments cumulatively to the items of
+    a sequence, from right to left, so as to reduce the sequence to
+    a single value.
+
+    @type f: C{callable} taking 2 arguments
+
+    @param z: Initial value.
+
+    @param xs: Sequence to reduce.
+
+    @return: Single value resulting from reducing C{xs}.
+    """
+    return reduce(lambda x, y: f(y, x), reversed(xs), z)
+
+
+class _FormattingParser(_CommandDispatcherMixin):
+    """
+    A finite-state machine that parses formatted IRC text.
+
+    Currently handled formatting includes: bold, reverse, underline,
+    mIRC color codes and the ability to remove all current formatting.
+
+    @see: U{http://www.mirc.co.uk/help/color.txt}
+
+    @type _formatCodes: C{dict} mapping C{str} to C{str}
+    @cvar _formatCodes: Mapping of format code values to names.
+
+    @type state: C{str}
+    @ivar state: Current state of the finite-state machine.
+
+    @type _buffer: C{str}
+    @ivar _buffer: Buffer, containing the text content, of the formatting
+        sequence currently being parsed, the buffer is used as the content for
+        L{_attrs} before being added to L{_result} and emptied upon calling
+        L{emit}.
+
+    @type _attrs: C{set}
+    @ivar _attrs: Set of the applicable formatting states (bold, underline,
+        etc.) for the current L{_buffer}, these are applied to L{_buffer} when
+        calling L{emit}.
+
+    @type foreground: L{_ForegroundColorAttr}
+    @ivar foreground: Current foreground color attribute, or L{None}.
+
+    @type background: L{_BackgroundColorAttr}
+    @ivar background: Current background color attribute, or L{None}.
+
+    @ivar _result: Current parse result.
+    """
+
+    prefix = "state"
+
+    _formatCodes = {
+        _OFF: "off",
+        _BOLD: "bold",
+        _COLOR: "color",
+        _REVERSE_VIDEO: "reverseVideo",
+        _UNDERLINE: "underline",
+    }
+
+    def __init__(self):
+        self.state = "TEXT"
+        self._buffer = ""
+        self._attrs = set()
+        self._result = None
+        self.foreground = None
+        self.background = None
+
+    def process(self, ch):
+        """
+        Handle input.
+
+        @type ch: C{str}
+        @param ch: A single character of input to process
+        """
+        self.dispatch(self.state, ch)
+
+    def complete(self):
+        """
+        Flush the current buffer and return the final parsed result.
+
+        @return: Structured text and attributes.
+        """
+        self.emit()
+        if self._result is None:
+            self._result = attributes.normal
+        return self._result
+
+    def emit(self):
+        """
+        Add the currently parsed input to the result.
+        """
+        if self._buffer:
+            attrs = [getattr(attributes, name) for name in self._attrs]
+            attrs.extend(filter(None, [self.foreground, self.background]))
+            if not attrs:
+                attrs.append(attributes.normal)
+            attrs.append(self._buffer)
+
+            attr = _foldr(operator.getitem, attrs.pop(), attrs)
+            if self._result is None:
+                self._result = attr
+            else:
+                self._result[attr]
+            self._buffer = ""
+
+    def state_TEXT(self, ch):
+        """
+        Handle the "text" state.
+
+        Along with regular text, single token formatting codes are handled
+        in this state too.
+
+        @param ch: The character being processed.
+        """
+        formatName = self._formatCodes.get(ch)
+        if formatName == "color":
+            self.emit()
+            self.state = "COLOR_FOREGROUND"
+        else:
+            if formatName is None:
+                self._buffer += ch
+            else:
+                self.emit()
+                if formatName == "off":
+                    self._attrs = set()
+                    self.foreground = self.background = None
+                else:
+                    self._attrs.symmetric_difference_update([formatName])
+
+    def state_COLOR_FOREGROUND(self, ch):
+        """
+        Handle the foreground color state.
+
+        Foreground colors can consist of up to two digits and may optionally
+        end in a I{,}. Any non-digit or non-comma characters are treated as
+        invalid input and result in the state being reset to "text".
+
+        @param ch: The character being processed.
+        """
+        # Color codes may only be a maximum of two characters.
+        if ch.isdigit() and len(self._buffer) < 2:
+            self._buffer += ch
+        else:
+            if self._buffer:
+                # Wrap around for color numbers higher than we support, like
+                # most other IRC clients.
+                col = int(self._buffer) % len(_IRC_COLORS)
+                self.foreground = getattr(attributes.fg, _IRC_COLOR_NAMES[col])
+            else:
+                # If there were no digits, then this has been an empty color
+                # code and we can reset the color state.
+                self.foreground = self.background = None
+
+            if ch == "," and self._buffer:
+                # If there's a comma and it's not the first thing, move on to
+                # the background state.
+                self._buffer = ""
+                self.state = "COLOR_BACKGROUND"
+            else:
+                # Otherwise, this is a bogus color code, fall back to text.
+                self._buffer = ""
+                self.state = "TEXT"
+                self.emit()
+                self.process(ch)
+
+    def state_COLOR_BACKGROUND(self, ch):
+        """
+        Handle the background color state.
+
+        Background colors can consist of up to two digits and must occur after
+        a foreground color and must be preceded by a I{,}. Any non-digit
+        character is treated as invalid input and results in the state being
+        set to "text".
+
+        @param ch: The character being processed.
+        """
+        # Color codes may only be a maximum of two characters.
+        if ch.isdigit() and len(self._buffer) < 2:
+            self._buffer += ch
+        else:
+            if self._buffer:
+                # Wrap around for color numbers higher than we support, like
+                # most other IRC clients.
+                col = int(self._buffer) % len(_IRC_COLORS)
+                self.background = getattr(attributes.bg, _IRC_COLOR_NAMES[col])
+                self._buffer = ""
+
+            self.emit()
+            self.state = "TEXT"
+            self.process(ch)
+
+
+def parseFormattedText(text):
+    """
+    Parse text containing IRC formatting codes into structured information.
+
+    Color codes are mapped from 0 to 15 and wrap around if greater than 15.
+
+    @type text: C{str}
+    @param text: Formatted text to parse.
+
+    @return: Structured text and attributes.
+
+    @since: 13.1
+    """
+    state = _FormattingParser()
+    for ch in text:
+        state.process(ch)
+    return state.complete()
+
+
+def assembleFormattedText(formatted):
+    """
+    Assemble formatted text from structured information.
+
+    Currently handled formatting includes: bold, reverse, underline,
+    mIRC color codes and the ability to remove all current formatting.
+
+    It is worth noting that assembled text will always begin with the control
+    code to disable other attributes for the sake of correctness.
+
+    For example::
+
+        from twisted.words.protocols.irc import attributes as A
+        assembleFormattedText(
+            A.normal[A.bold['Time: '], A.fg.lightRed['Now!']])
+
+    Would produce "Time: " in bold formatting, followed by "Now!" with a
+    foreground color of light red and without any additional formatting.
+
+    Available attributes are:
+        - bold
+        - reverseVideo
+        - underline
+
+    Available colors are:
+        0. white
+        1. black
+        2. blue
+        3. green
+        4. light red
+        5. red
+        6. magenta
+        7. orange
+        8. yellow
+        9. light green
+        10. cyan
+        11. light cyan
+        12. light blue
+        13. light magenta
+        14. gray
+        15. light gray
+
+    @see: U{http://www.mirc.co.uk/help/color.txt}
+
+    @param formatted: Structured text and attributes.
+
+    @rtype: C{str}
+    @return: String containing mIRC control sequences that mimic those
+        specified by I{formatted}.
+
+    @since: 13.1
+    """
+    return _textattributes.flatten(formatted, _FormattingState(), "toMIRCControlCodes")
+
+
+def stripFormatting(text):
+    """
+    Remove all formatting codes from C{text}, leaving only the text.
+
+    @type text: C{str}
+    @param text: Formatted text to parse.
+
+    @rtype: C{str}
+    @return: Plain text without any control sequences.
+
+    @since: 13.1
+    """
+    formatted = parseFormattedText(text)
+    return _textattributes.flatten(formatted, _textattributes.DefaultFormattingState())
+
+
+# CTCP constants and helper functions
+
+X_DELIM = chr(0o01)
+
+
+def ctcpExtract(message):
+    """
+    Extract CTCP data from a string.
+
+    @return: A C{dict} containing two keys:
+       - C{'extended'}: A list of CTCP (tag, data) tuples.
+       - C{'normal'}: A list of strings which were not inside a CTCP delimiter.
+    """
+    extended_messages = []
+    normal_messages = []
+    retval = {"extended": extended_messages, "normal": normal_messages}
+
+    messages = message.split(X_DELIM)
+    odd = 0
+
+    # X1 extended data X2 nomal data X3 extended data X4 normal...
+    while messages:
+        if odd:
+            extended_messages.append(messages.pop(0))
+        else:
+            normal_messages.append(messages.pop(0))
+        odd = not odd
+
+    extended_messages[:] = list(filter(None, extended_messages))
+    normal_messages[:] = list(filter(None, normal_messages))
+
+    extended_messages[:] = list(map(ctcpDequote, extended_messages))
+    for i in range(len(extended_messages)):
+        m = extended_messages[i].split(SPC, 1)
+        tag = m[0]
+        if len(m) > 1:
+            data = m[1]
+        else:
+            data = None
+
+        extended_messages[i] = (tag, data)
+
+    return retval
+
+
+# CTCP escaping
+
+M_QUOTE = chr(0o20)
+
+mQuoteTable = {
+    NUL: M_QUOTE + "0",
+    NL: M_QUOTE + "n",
+    CR: M_QUOTE + "r",
+    M_QUOTE: M_QUOTE + M_QUOTE,
+}
+
+mDequoteTable = {}
+for k, v in mQuoteTable.items():
+    mDequoteTable[v[-1]] = k
+del k, v
+
+mEscape_re = re.compile(f"{re.escape(M_QUOTE)}.", re.DOTALL)
+
+
+def lowQuote(s):
+    for c in (M_QUOTE, NUL, NL, CR):
+        s = s.replace(c, mQuoteTable[c])
+    return s
+
+
+def lowDequote(s):
+    def sub(matchobj, mDequoteTable=mDequoteTable):
+        s = matchobj.group()[1]
+        try:
+            s = mDequoteTable[s]
+        except KeyError:
+            s = s
+        return s
+
+    return mEscape_re.sub(sub, s)
+
+
+X_QUOTE = "\\"
+
+xQuoteTable = {X_DELIM: X_QUOTE + "a", X_QUOTE: X_QUOTE + X_QUOTE}
+
+xDequoteTable = {}
+
+for k, v in xQuoteTable.items():
+    xDequoteTable[v[-1]] = k
+
+xEscape_re = re.compile(f"{re.escape(X_QUOTE)}.", re.DOTALL)
+
+
+def ctcpQuote(s):
+    for c in (X_QUOTE, X_DELIM):
+        s = s.replace(c, xQuoteTable[c])
+    return s
+
+
+def ctcpDequote(s):
+    def sub(matchobj, xDequoteTable=xDequoteTable):
+        s = matchobj.group()[1]
+        try:
+            s = xDequoteTable[s]
+        except KeyError:
+            s = s
+        return s
+
+    return xEscape_re.sub(sub, s)
+
+
+def ctcpStringify(messages):
+    """
+    @type messages: a list of extended messages.  An extended
+    message is a (tag, data) tuple, where 'data' may be L{None}, a
+    string, or a list of strings to be joined with whitespace.
+
+    @returns: String
+    """
+    coded_messages = []
+    for tag, data in messages:
+        if data:
+            if not isinstance(data, str):
+                try:
+                    # data as list-of-strings
+                    data = " ".join(map(str, data))
+                except TypeError:
+                    # No?  Then use it's %s representation.
+                    pass
+            m = f"{tag} {data}"
+        else:
+            m = str(tag)
+        m = ctcpQuote(m)
+        m = f"{X_DELIM}{m}{X_DELIM}"
+        coded_messages.append(m)
+
+    line = "".join(coded_messages)
+    return line
+
+
+# Constants (from RFC 2812)
+RPL_WELCOME = "001"
+RPL_YOURHOST = "002"
+RPL_CREATED = "003"
+RPL_MYINFO = "004"
+RPL_ISUPPORT = "005"
+RPL_BOUNCE = "010"
+RPL_USERHOST = "302"
+RPL_ISON = "303"
+RPL_AWAY = "301"
+RPL_UNAWAY = "305"
+RPL_NOWAWAY = "306"
+RPL_WHOISUSER = "311"
+RPL_WHOISSERVER = "312"
+RPL_WHOISOPERATOR = "313"
+RPL_WHOISIDLE = "317"
+RPL_ENDOFWHOIS = "318"
+RPL_WHOISCHANNELS = "319"
+RPL_WHOWASUSER = "314"
+RPL_ENDOFWHOWAS = "369"
+RPL_LISTSTART = "321"
+RPL_LIST = "322"
+RPL_LISTEND = "323"
+RPL_UNIQOPIS = "325"
+RPL_CHANNELMODEIS = "324"
+RPL_NOTOPIC = "331"
+RPL_TOPIC = "332"
+RPL_INVITING = "341"
+RPL_SUMMONING = "342"
+RPL_INVITELIST = "346"
+RPL_ENDOFINVITELIST = "347"
+RPL_EXCEPTLIST = "348"
+RPL_ENDOFEXCEPTLIST = "349"
+RPL_VERSION = "351"
+RPL_WHOREPLY = "352"
+RPL_ENDOFWHO = "315"
+RPL_NAMREPLY = "353"
+RPL_ENDOFNAMES = "366"
+RPL_LINKS = "364"
+RPL_ENDOFLINKS = "365"
+RPL_BANLIST = "367"
+RPL_ENDOFBANLIST = "368"
+RPL_INFO = "371"
+RPL_ENDOFINFO = "374"
+RPL_MOTDSTART = "375"
+RPL_MOTD = "372"
+RPL_ENDOFMOTD = "376"
+RPL_YOUREOPER = "381"
+RPL_REHASHING = "382"
+RPL_YOURESERVICE = "383"
+RPL_TIME = "391"
+RPL_USERSSTART = "392"
+RPL_USERS = "393"
+RPL_ENDOFUSERS = "394"
+RPL_NOUSERS = "395"
+RPL_TRACELINK = "200"
+RPL_TRACECONNECTING = "201"
+RPL_TRACEHANDSHAKE = "202"
+RPL_TRACEUNKNOWN = "203"
+RPL_TRACEOPERATOR = "204"
+RPL_TRACEUSER = "205"
+RPL_TRACESERVER = "206"
+RPL_TRACESERVICE = "207"
+RPL_TRACENEWTYPE = "208"
+RPL_TRACECLASS = "209"
+RPL_TRACERECONNECT = "210"
+RPL_TRACELOG = "261"
+RPL_TRACEEND = "262"
+RPL_STATSLINKINFO = "211"
+RPL_STATSCOMMANDS = "212"
+RPL_ENDOFSTATS = "219"
+RPL_STATSUPTIME = "242"
+RPL_STATSOLINE = "243"
+RPL_UMODEIS = "221"
+RPL_SERVLIST = "234"
+RPL_SERVLISTEND = "235"
+RPL_LUSERCLIENT = "251"
+RPL_LUSEROP = "252"
+RPL_LUSERUNKNOWN = "253"
+RPL_LUSERCHANNELS = "254"
+RPL_LUSERME = "255"
+RPL_ADMINME = "256"
+RPL_ADMINLOC1 = "257"
+RPL_ADMINLOC2 = "258"
+RPL_ADMINEMAIL = "259"
+RPL_TRYAGAIN = "263"
+ERR_NOSUCHNICK = "401"
+ERR_NOSUCHSERVER = "402"
+ERR_NOSUCHCHANNEL = "403"
+ERR_CANNOTSENDTOCHAN = "404"
+ERR_TOOMANYCHANNELS = "405"
+ERR_WASNOSUCHNICK = "406"
+ERR_TOOMANYTARGETS = "407"
+ERR_NOSUCHSERVICE = "408"
+ERR_NOORIGIN = "409"
+ERR_NORECIPIENT = "411"
+ERR_NOTEXTTOSEND = "412"
+ERR_NOTOPLEVEL = "413"
+ERR_WILDTOPLEVEL = "414"
+ERR_BADMASK = "415"
+# Defined in errata.
+# https://www.rfc-editor.org/errata_search.php?rfc=2812&eid=2822
+ERR_TOOMANYMATCHES = "416"
+ERR_UNKNOWNCOMMAND = "421"
+ERR_NOMOTD = "422"
+ERR_NOADMININFO = "423"
+ERR_FILEERROR = "424"
+ERR_NONICKNAMEGIVEN = "431"
+ERR_ERRONEUSNICKNAME = "432"
+ERR_NICKNAMEINUSE = "433"
+ERR_NICKCOLLISION = "436"
+ERR_UNAVAILRESOURCE = "437"
+ERR_USERNOTINCHANNEL = "441"
+ERR_NOTONCHANNEL = "442"
+ERR_USERONCHANNEL = "443"
+ERR_NOLOGIN = "444"
+ERR_SUMMONDISABLED = "445"
+ERR_USERSDISABLED = "446"
+ERR_NOTREGISTERED = "451"
+ERR_NEEDMOREPARAMS = "461"
+ERR_ALREADYREGISTRED = "462"
+ERR_NOPERMFORHOST = "463"
+ERR_PASSWDMISMATCH = "464"
+ERR_YOUREBANNEDCREEP = "465"
+ERR_YOUWILLBEBANNED = "466"
+ERR_KEYSET = "467"
+ERR_CHANNELISFULL = "471"
+ERR_UNKNOWNMODE = "472"
+ERR_INVITEONLYCHAN = "473"
+ERR_BANNEDFROMCHAN = "474"
+ERR_BADCHANNELKEY = "475"
+ERR_BADCHANMASK = "476"
+ERR_NOCHANMODES = "477"
+ERR_BANLISTFULL = "478"
+ERR_NOPRIVILEGES = "481"
+ERR_CHANOPRIVSNEEDED = "482"
+ERR_CANTKILLSERVER = "483"
+ERR_RESTRICTED = "484"
+ERR_UNIQOPPRIVSNEEDED = "485"
+ERR_NOOPERHOST = "491"
+ERR_NOSERVICEHOST = "492"
+ERR_UMODEUNKNOWNFLAG = "501"
+ERR_USERSDONTMATCH = "502"
+
+# And hey, as long as the strings are already intern'd...
+symbolic_to_numeric = {
+    "RPL_WELCOME": "001",
+    "RPL_YOURHOST": "002",
+    "RPL_CREATED": "003",
+    "RPL_MYINFO": "004",
+    "RPL_ISUPPORT": "005",
+    "RPL_BOUNCE": "010",
+    "RPL_USERHOST": "302",
+    "RPL_ISON": "303",
+    "RPL_AWAY": "301",
+    "RPL_UNAWAY": "305",
+    "RPL_NOWAWAY": "306",
+    "RPL_WHOISUSER": "311",
+    "RPL_WHOISSERVER": "312",
+    "RPL_WHOISOPERATOR": "313",
+    "RPL_WHOISIDLE": "317",
+    "RPL_ENDOFWHOIS": "318",
+    "RPL_WHOISCHANNELS": "319",
+    "RPL_WHOWASUSER": "314",
+    "RPL_ENDOFWHOWAS": "369",
+    "RPL_LISTSTART": "321",
+    "RPL_LIST": "322",
+    "RPL_LISTEND": "323",
+    "RPL_UNIQOPIS": "325",
+    "RPL_CHANNELMODEIS": "324",
+    "RPL_NOTOPIC": "331",
+    "RPL_TOPIC": "332",
+    "RPL_INVITING": "341",
+    "RPL_SUMMONING": "342",
+    "RPL_INVITELIST": "346",
+    "RPL_ENDOFINVITELIST": "347",
+    "RPL_EXCEPTLIST": "348",
+    "RPL_ENDOFEXCEPTLIST": "349",
+    "RPL_VERSION": "351",
+    "RPL_WHOREPLY": "352",
+    "RPL_ENDOFWHO": "315",
+    "RPL_NAMREPLY": "353",
+    "RPL_ENDOFNAMES": "366",
+    "RPL_LINKS": "364",
+    "RPL_ENDOFLINKS": "365",
+    "RPL_BANLIST": "367",
+    "RPL_ENDOFBANLIST": "368",
+    "RPL_INFO": "371",
+    "RPL_ENDOFINFO": "374",
+    "RPL_MOTDSTART": "375",
+    "RPL_MOTD": "372",
+    "RPL_ENDOFMOTD": "376",
+    "RPL_YOUREOPER": "381",
+    "RPL_REHASHING": "382",
+    "RPL_YOURESERVICE": "383",
+    "RPL_TIME": "391",
+    "RPL_USERSSTART": "392",
+    "RPL_USERS": "393",
+    "RPL_ENDOFUSERS": "394",
+    "RPL_NOUSERS": "395",
+    "RPL_TRACELINK": "200",
+    "RPL_TRACECONNECTING": "201",
+    "RPL_TRACEHANDSHAKE": "202",
+    "RPL_TRACEUNKNOWN": "203",
+    "RPL_TRACEOPERATOR": "204",
+    "RPL_TRACEUSER": "205",
+    "RPL_TRACESERVER": "206",
+    "RPL_TRACESERVICE": "207",
+    "RPL_TRACENEWTYPE": "208",
+    "RPL_TRACECLASS": "209",
+    "RPL_TRACERECONNECT": "210",
+    "RPL_TRACELOG": "261",
+    "RPL_TRACEEND": "262",
+    "RPL_STATSLINKINFO": "211",
+    "RPL_STATSCOMMANDS": "212",
+    "RPL_ENDOFSTATS": "219",
+    "RPL_STATSUPTIME": "242",
+    "RPL_STATSOLINE": "243",
+    "RPL_UMODEIS": "221",
+    "RPL_SERVLIST": "234",
+    "RPL_SERVLISTEND": "235",
+    "RPL_LUSERCLIENT": "251",
+    "RPL_LUSEROP": "252",
+    "RPL_LUSERUNKNOWN": "253",
+    "RPL_LUSERCHANNELS": "254",
+    "RPL_LUSERME": "255",
+    "RPL_ADMINME": "256",
+    "RPL_ADMINLOC1": "257",
+    "RPL_ADMINLOC2": "258",
+    "RPL_ADMINEMAIL": "259",
+    "RPL_TRYAGAIN": "263",
+    "ERR_NOSUCHNICK": "401",
+    "ERR_NOSUCHSERVER": "402",
+    "ERR_NOSUCHCHANNEL": "403",
+    "ERR_CANNOTSENDTOCHAN": "404",
+    "ERR_TOOMANYCHANNELS": "405",
+    "ERR_WASNOSUCHNICK": "406",
+    "ERR_TOOMANYTARGETS": "407",
+    "ERR_NOSUCHSERVICE": "408",
+    "ERR_NOORIGIN": "409",
+    "ERR_NORECIPIENT": "411",
+    "ERR_NOTEXTTOSEND": "412",
+    "ERR_NOTOPLEVEL": "413",
+    "ERR_WILDTOPLEVEL": "414",
+    "ERR_BADMASK": "415",
+    "ERR_TOOMANYMATCHES": "416",
+    "ERR_UNKNOWNCOMMAND": "421",
+    "ERR_NOMOTD": "422",
+    "ERR_NOADMININFO": "423",
+    "ERR_FILEERROR": "424",
+    "ERR_NONICKNAMEGIVEN": "431",
+    "ERR_ERRONEUSNICKNAME": "432",
+    "ERR_NICKNAMEINUSE": "433",
+    "ERR_NICKCOLLISION": "436",
+    "ERR_UNAVAILRESOURCE": "437",
+    "ERR_USERNOTINCHANNEL": "441",
+    "ERR_NOTONCHANNEL": "442",
+    "ERR_USERONCHANNEL": "443",
+    "ERR_NOLOGIN": "444",
+    "ERR_SUMMONDISABLED": "445",
+    "ERR_USERSDISABLED": "446",
+    "ERR_NOTREGISTERED": "451",
+    "ERR_NEEDMOREPARAMS": "461",
+    "ERR_ALREADYREGISTRED": "462",
+    "ERR_NOPERMFORHOST": "463",
+    "ERR_PASSWDMISMATCH": "464",
+    "ERR_YOUREBANNEDCREEP": "465",
+    "ERR_YOUWILLBEBANNED": "466",
+    "ERR_KEYSET": "467",
+    "ERR_CHANNELISFULL": "471",
+    "ERR_UNKNOWNMODE": "472",
+    "ERR_INVITEONLYCHAN": "473",
+    "ERR_BANNEDFROMCHAN": "474",
+    "ERR_BADCHANNELKEY": "475",
+    "ERR_BADCHANMASK": "476",
+    "ERR_NOCHANMODES": "477",
+    "ERR_BANLISTFULL": "478",
+    "ERR_NOPRIVILEGES": "481",
+    "ERR_CHANOPRIVSNEEDED": "482",
+    "ERR_CANTKILLSERVER": "483",
+    "ERR_RESTRICTED": "484",
+    "ERR_UNIQOPPRIVSNEEDED": "485",
+    "ERR_NOOPERHOST": "491",
+    "ERR_NOSERVICEHOST": "492",
+    "ERR_UMODEUNKNOWNFLAG": "501",
+    "ERR_USERSDONTMATCH": "502",
+}
+
+numeric_to_symbolic = {}
+for k, v in symbolic_to_numeric.items():
+    numeric_to_symbolic[v] = k
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/__init__.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/__init__.py
new file mode 100644
index 00000000000..ad95b6853ec
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/__init__.py
@@ -0,0 +1,8 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Twisted Jabber: Jabber Protocol Helpers
+"""
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/client.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/client.py
new file mode 100644
index 00000000000..21de4774e26
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/client.py
@@ -0,0 +1,394 @@
+# -*- test-case-name: twisted.words.test.test_jabberclient -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+from twisted.words.protocols.jabber import error, sasl, xmlstream
+from twisted.words.protocols.jabber.jid import JID
+from twisted.words.xish import domish, utility, xpath
+
+NS_XMPP_STREAMS = "urn:ietf:params:xml:ns:xmpp-streams"
+NS_XMPP_BIND = "urn:ietf:params:xml:ns:xmpp-bind"
+NS_XMPP_SESSION = "urn:ietf:params:xml:ns:xmpp-session"
+NS_IQ_AUTH_FEATURE = "http://jabber.org/features/iq-auth"
+
+DigestAuthQry = xpath.internQuery("/iq/query/digest")
+PlaintextAuthQry = xpath.internQuery("/iq/query/password")
+
+
+def basicClientFactory(jid, secret):
+    a = BasicAuthenticator(jid, secret)
+    return xmlstream.XmlStreamFactory(a)
+
+
+class IQ(domish.Element):
+    """
+    Wrapper for a Info/Query packet.
+
+    This provides the necessary functionality to send IQs and get notified when
+    a result comes back. It's a subclass from L{domish.Element}, so you can use
+    the standard DOM manipulation calls to add data to the outbound request.
+
+    @type callbacks: L{utility.CallbackList}
+    @cvar callbacks: Callback list to be notified when response comes back
+
+    """
+
+    def __init__(self, xmlstream, type="set"):
+        """
+        @type xmlstream: L{xmlstream.XmlStream}
+        @param xmlstream: XmlStream to use for transmission of this IQ
+
+        @type type: C{str}
+        @param type: IQ type identifier ('get' or 'set')
+        """
+
+        domish.Element.__init__(self, ("jabber:client", "iq"))
+        self.addUniqueId()
+        self["type"] = type
+        self._xmlstream = xmlstream
+        self.callbacks = utility.CallbackList()
+
+    def addCallback(self, fn, *args, **kwargs):
+        """
+        Register a callback for notification when the IQ result is available.
+        """
+
+        self.callbacks.addCallback(True, fn, *args, **kwargs)
+
+    def send(self, to=None):
+        """
+        Call this method to send this IQ request via the associated XmlStream.
+
+        @param to: Jabber ID of the entity to send the request to
+        @type to: C{str}
+
+        @returns: Callback list for this IQ. Any callbacks added to this list
+                  will be fired when the result comes back.
+        """
+        if to != None:
+            self["to"] = to
+        self._xmlstream.addOnetimeObserver(
+            "/iq[@id='%s']" % self["id"], self._resultEvent
+        )
+        self._xmlstream.send(self)
+
+    def _resultEvent(self, iq):
+        self.callbacks.callback(iq)
+        self.callbacks = None
+
+
+class IQAuthInitializer:
+    """
+    Non-SASL Authentication initializer for the initiating entity.
+
+    This protocol is defined in
+    U{JEP-0078<http://www.jabber.org/jeps/jep-0078.html>} and mainly serves for
+    compatibility with pre-XMPP-1.0 server implementations.
+
+    @cvar INVALID_USER_EVENT: Token to signal that authentication failed, due
+        to invalid username.
+    @type INVALID_USER_EVENT: L{str}
+
+    @cvar AUTH_FAILED_EVENT: Token to signal that authentication failed, due to
+        invalid password.
+    @type AUTH_FAILED_EVENT: L{str}
+    """
+
+    INVALID_USER_EVENT = "//event/client/basicauth/invaliduser"
+    AUTH_FAILED_EVENT = "//event/client/basicauth/authfailed"
+
+    def __init__(self, xs):
+        self.xmlstream = xs
+
+    def initialize(self):
+        # Send request for auth fields
+        iq = xmlstream.IQ(self.xmlstream, "get")
+        iq.addElement(("jabber:iq:auth", "query"))
+        jid = self.xmlstream.authenticator.jid
+        iq.query.addElement("username", content=jid.user)
+
+        d = iq.send()
+        d.addCallbacks(self._cbAuthQuery, self._ebAuthQuery)
+        return d
+
+    def _cbAuthQuery(self, iq):
+        jid = self.xmlstream.authenticator.jid
+        password = self.xmlstream.authenticator.password
+
+        # Construct auth request
+        reply = xmlstream.IQ(self.xmlstream, "set")
+        reply.addElement(("jabber:iq:auth", "query"))
+        reply.query.addElement("username", content=jid.user)
+        reply.query.addElement("resource", content=jid.resource)
+
+        # Prefer digest over plaintext
+        if DigestAuthQry.matches(iq):
+            digest = xmlstream.hashPassword(self.xmlstream.sid, password)
+            reply.query.addElement("digest", content=str(digest))
+        else:
+            reply.query.addElement("password", content=password)
+
+        d = reply.send()
+        d.addCallbacks(self._cbAuth, self._ebAuth)
+        return d
+
+    def _ebAuthQuery(self, failure):
+        failure.trap(error.StanzaError)
+        e = failure.value
+        if e.condition == "not-authorized":
+            self.xmlstream.dispatch(e.stanza, self.INVALID_USER_EVENT)
+        else:
+            self.xmlstream.dispatch(e.stanza, self.AUTH_FAILED_EVENT)
+
+        return failure
+
+    def _cbAuth(self, iq):
+        pass
+
+    def _ebAuth(self, failure):
+        failure.trap(error.StanzaError)
+        self.xmlstream.dispatch(failure.value.stanza, self.AUTH_FAILED_EVENT)
+        return failure
+
+
+class BasicAuthenticator(xmlstream.ConnectAuthenticator):
+    """
+    Authenticates an XmlStream against a Jabber server as a Client.
+
+    This only implements non-SASL authentication, per
+    U{JEP-0078<http://www.jabber.org/jeps/jep-0078.html>}. Additionally, this
+    authenticator provides the ability to perform inline registration, per
+    U{JEP-0077<http://www.jabber.org/jeps/jep-0077.html>}.
+
+    Under normal circumstances, the BasicAuthenticator generates the
+    L{xmlstream.STREAM_AUTHD_EVENT} once the stream has authenticated. However,
+    it can also generate other events, such as:
+      - L{INVALID_USER_EVENT} : Authentication failed, due to invalid username
+      - L{AUTH_FAILED_EVENT} : Authentication failed, due to invalid password
+      - L{REGISTER_FAILED_EVENT} : Registration failed
+
+    If authentication fails for any reason, you can attempt to register by
+    calling the L{registerAccount} method. If the registration succeeds, a
+    L{xmlstream.STREAM_AUTHD_EVENT} will be fired. Otherwise, one of the above
+    errors will be generated (again).
+
+
+    @cvar INVALID_USER_EVENT: See L{IQAuthInitializer.INVALID_USER_EVENT}.
+    @type INVALID_USER_EVENT: L{str}
+
+    @cvar AUTH_FAILED_EVENT: See L{IQAuthInitializer.AUTH_FAILED_EVENT}.
+    @type AUTH_FAILED_EVENT: L{str}
+
+    @cvar REGISTER_FAILED_EVENT: Token to signal that registration failed.
+    @type REGISTER_FAILED_EVENT: L{str}
+
+    """
+
+    namespace = "jabber:client"
+
+    INVALID_USER_EVENT = IQAuthInitializer.INVALID_USER_EVENT
+    AUTH_FAILED_EVENT = IQAuthInitializer.AUTH_FAILED_EVENT
+    REGISTER_FAILED_EVENT = "//event/client/basicauth/registerfailed"
+
+    def __init__(self, jid, password):
+        xmlstream.ConnectAuthenticator.__init__(self, jid.host)
+        self.jid = jid
+        self.password = password
+
+    def associateWithStream(self, xs):
+        xs.version = (0, 0)
+        xmlstream.ConnectAuthenticator.associateWithStream(self, xs)
+
+        xs.initializers = [
+            xmlstream.TLSInitiatingInitializer(xs, required=False),
+            IQAuthInitializer(xs),
+        ]
+
+    # TODO: move registration into an Initializer?
+
+    def registerAccount(self, username=None, password=None):
+        if username:
+            self.jid.user = username
+        if password:
+            self.password = password
+
+        iq = IQ(self.xmlstream, "set")
+        iq.addElement(("jabber:iq:register", "query"))
+        iq.query.addElement("username", content=self.jid.user)
+        iq.query.addElement("password", content=self.password)
+
+        iq.addCallback(self._registerResultEvent)
+
+        iq.send()
+
+    def _registerResultEvent(self, iq):
+        if iq["type"] == "result":
+            # Registration succeeded -- go ahead and auth
+            self.streamStarted()
+        else:
+            # Registration failed
+            self.xmlstream.dispatch(iq, self.REGISTER_FAILED_EVENT)
+
+
+class CheckVersionInitializer:
+    """
+    Initializer that checks if the minimum common stream version number is 1.0.
+    """
+
+    def __init__(self, xs):
+        self.xmlstream = xs
+
+    def initialize(self):
+        if self.xmlstream.version < (1, 0):
+            raise error.StreamError("unsupported-version")
+
+
+class BindInitializer(xmlstream.BaseFeatureInitiatingInitializer):
+    """
+    Initializer that implements Resource Binding for the initiating entity.
+
+    This protocol is documented in U{RFC 3920, section
+    7<http://www.xmpp.org/specs/rfc3920.html#bind>}.
+    """
+
+    feature = (NS_XMPP_BIND, "bind")
+
+    def start(self):
+        iq = xmlstream.IQ(self.xmlstream, "set")
+        bind = iq.addElement((NS_XMPP_BIND, "bind"))
+        resource = self.xmlstream.authenticator.jid.resource
+        if resource:
+            bind.addElement("resource", content=resource)
+        d = iq.send()
+        d.addCallback(self.onBind)
+        return d
+
+    def onBind(self, iq):
+        if iq.bind:
+            self.xmlstream.authenticator.jid = JID(str(iq.bind.jid))
+
+
+class SessionInitializer(xmlstream.BaseFeatureInitiatingInitializer):
+    """
+    Initializer that implements session establishment for the initiating
+    entity.
+
+    This protocol is defined in U{RFC 3921, section
+    3<http://www.xmpp.org/specs/rfc3921.html#session>}.
+    """
+
+    feature = (NS_XMPP_SESSION, "session")
+
+    def start(self):
+        iq = xmlstream.IQ(self.xmlstream, "set")
+        iq.addElement((NS_XMPP_SESSION, "session"))
+        return iq.send()
+
+
+def XMPPClientFactory(jid, password, configurationForTLS=None):
+    """
+    Client factory for XMPP 1.0 (only).
+
+    This returns a L{xmlstream.XmlStreamFactory} with an L{XMPPAuthenticator}
+    object to perform the stream initialization steps (such as authentication).
+
+    @see: The notes at L{XMPPAuthenticator} describe how the C{jid} and
+    C{password} parameters are to be used.
+
+    @param jid: Jabber ID to connect with.
+    @type jid: L{jid.JID}
+
+    @param password: password to authenticate with.
+    @type password: L{unicode}
+
+    @param configurationForTLS: An object which creates appropriately
+        configured TLS connections. This is passed to C{startTLS} on the
+        transport and is preferably created using
+        L{twisted.internet.ssl.optionsForClientTLS}. If L{None}, the default is
+        to verify the server certificate against the trust roots as provided by
+        the platform. See L{twisted.internet._sslverify.platformTrust}.
+    @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or L{None}
+
+    @return: XML stream factory.
+    @rtype: L{xmlstream.XmlStreamFactory}
+    """
+    a = XMPPAuthenticator(jid, password, configurationForTLS=configurationForTLS)
+    return xmlstream.XmlStreamFactory(a)
+
+
+class XMPPAuthenticator(xmlstream.ConnectAuthenticator):
+    """
+    Initializes an XmlStream connecting to an XMPP server as a Client.
+
+    This authenticator performs the initialization steps needed to start
+    exchanging XML stanzas with an XMPP server as an XMPP client. It checks if
+    the server advertises XML stream version 1.0, negotiates TLS (when
+    available), performs SASL authentication, binds a resource and establishes
+    a session.
+
+    Upon successful stream initialization, the L{xmlstream.STREAM_AUTHD_EVENT}
+    event will be dispatched through the XML stream object. Otherwise, the
+    L{xmlstream.INIT_FAILED_EVENT} event will be dispatched with a failure
+    object.
+
+    After inspection of the failure, initialization can then be restarted by
+    calling L{ConnectAuthenticator.initializeStream}. For example, in case of
+    authentication failure, a user may be given the opportunity to input the
+    correct password.  By setting the L{password} instance variable and restarting
+    initialization, the stream authentication step is then retried, and subsequent
+    steps are performed if successful.
+
+    @ivar jid: Jabber ID to authenticate with. This may contain a resource
+               part, as a suggestion to the server for resource binding. A
+               server may override this, though. If the resource part is left
+               off, the server will generate a unique resource identifier.
+               The server will always return the full Jabber ID in the
+               resource binding step, and this is stored in this instance
+               variable.
+    @type jid: L{jid.JID}
+
+    @ivar password: password to be used during SASL authentication.
+    @type password: L{unicode}
+    """
+
+    namespace = "jabber:client"
+
+    def __init__(self, jid, password, configurationForTLS=None):
+        """
+        @param configurationForTLS: An object which creates appropriately
+            configured TLS connections. This is passed to C{startTLS} on the
+            transport and is preferably created using
+            L{twisted.internet.ssl.optionsForClientTLS}. If C{None}, the
+            default is to verify the server certificate against the trust roots
+            as provided by the platform. See
+            L{twisted.internet._sslverify.platformTrust}.
+        @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or
+            C{None}
+        """
+        xmlstream.ConnectAuthenticator.__init__(self, jid.host)
+        self.jid = jid
+        self.password = password
+        self._configurationForTLS = configurationForTLS
+
+    def associateWithStream(self, xs):
+        """
+        Register with the XML stream.
+
+        Populates stream's list of initializers, along with their
+        requiredness. This list is used by
+        L{ConnectAuthenticator.initializeStream} to perform the initialization
+        steps.
+        """
+        xmlstream.ConnectAuthenticator.associateWithStream(self, xs)
+
+        xs.initializers = [
+            CheckVersionInitializer(xs),
+            xmlstream.TLSInitiatingInitializer(
+                xs, required=True, configurationForTLS=self._configurationForTLS
+            ),
+            sasl.SASLInitiatingInitializer(xs, required=True),
+            BindInitializer(xs, required=True),
+            SessionInitializer(xs, required=False),
+        ]
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/component.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/component.py
new file mode 100644
index 00000000000..d07c4ee9d7b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/component.py
@@ -0,0 +1,456 @@
+# -*- test-case-name: twisted.words.test.test_jabbercomponent -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+External server-side components.
+
+Most Jabber server implementations allow for add-on components that act as a
+separate entity on the Jabber network, but use the server-to-server
+functionality of a regular Jabber IM server. These so-called 'external
+components' are connected to the Jabber server using the Jabber Component
+Protocol as defined in U{JEP-0114<http://www.jabber.org/jeps/jep-0114.html>}.
+
+This module allows for writing external server-side component by assigning one
+or more services implementing L{ijabber.IService} up to L{ServiceManager}. The
+ServiceManager connects to the Jabber server and is responsible for the
+corresponding XML stream.
+"""
+
+from zope.interface import implementer
+
+from twisted.application import service
+from twisted.internet import defer
+from twisted.python import log
+from twisted.words.protocols.jabber import error, ijabber, jstrports, xmlstream
+from twisted.words.protocols.jabber.jid import internJID as JID
+from twisted.words.xish import domish
+
+NS_COMPONENT_ACCEPT = "jabber:component:accept"
+
+
+def componentFactory(componentid, password):
+    """
+    XML stream factory for external server-side components.
+
+    @param componentid: JID of the component.
+    @type componentid: L{unicode}
+    @param password: password used to authenticate to the server.
+    @type password: C{str}
+    """
+    a = ConnectComponentAuthenticator(componentid, password)
+    return xmlstream.XmlStreamFactory(a)
+
+
+class ComponentInitiatingInitializer:
+    """
+    External server-side component authentication initializer for the
+    initiating entity.
+
+    @ivar xmlstream: XML stream between server and component.
+    @type xmlstream: L{xmlstream.XmlStream}
+    """
+
+    def __init__(self, xs):
+        self.xmlstream = xs
+        self._deferred = None
+
+    def initialize(self):
+        xs = self.xmlstream
+        hs = domish.Element((self.xmlstream.namespace, "handshake"))
+        digest = xmlstream.hashPassword(xs.sid, xs.authenticator.password)
+        hs.addContent(str(digest))
+
+        # Setup observer to watch for handshake result
+        xs.addOnetimeObserver("/handshake", self._cbHandshake)
+        xs.send(hs)
+        self._deferred = defer.Deferred()
+        return self._deferred
+
+    def _cbHandshake(self, _):
+        # we have successfully shaken hands and can now consider this
+        # entity to represent the component JID.
+        self.xmlstream.thisEntity = self.xmlstream.otherEntity
+        self._deferred.callback(None)
+
+
+class ConnectComponentAuthenticator(xmlstream.ConnectAuthenticator):
+    """
+    Authenticator to permit an XmlStream to authenticate against a Jabber
+    server as an external component (where the Authenticator is initiating the
+    stream).
+    """
+
+    namespace = NS_COMPONENT_ACCEPT
+
+    def __init__(self, componentjid, password):
+        """
+        @type componentjid: C{str}
+        @param componentjid: Jabber ID that this component wishes to bind to.
+
+        @type password: C{str}
+        @param password: Password/secret this component uses to authenticate.
+        """
+        # Note that we are sending 'to' our desired component JID.
+        xmlstream.ConnectAuthenticator.__init__(self, componentjid)
+        self.password = password
+
+    def associateWithStream(self, xs):
+        xs.version = (0, 0)
+        xmlstream.ConnectAuthenticator.associateWithStream(self, xs)
+
+        xs.initializers = [ComponentInitiatingInitializer(xs)]
+
+
+class ListenComponentAuthenticator(xmlstream.ListenAuthenticator):
+    """
+    Authenticator for accepting components.
+
+    @since: 8.2
+    @ivar secret: The shared secret used to authorized incoming component
+                  connections.
+    @type secret: C{unicode}.
+    """
+
+    namespace = NS_COMPONENT_ACCEPT
+
+    def __init__(self, secret):
+        self.secret = secret
+        xmlstream.ListenAuthenticator.__init__(self)
+
+    def associateWithStream(self, xs):
+        """
+        Associate the authenticator with a stream.
+
+        This sets the stream's version to 0.0, because the XEP-0114 component
+        protocol was not designed for XMPP 1.0.
+        """
+        xs.version = (0, 0)
+        xmlstream.ListenAuthenticator.associateWithStream(self, xs)
+
+    def streamStarted(self, rootElement):
+        """
+        Called by the stream when it has started.
+
+        This examines the default namespace of the incoming stream and whether
+        there is a requested hostname for the component. Then it generates a
+        stream identifier, sends a response header and adds an observer for
+        the first incoming element, triggering L{onElement}.
+        """
+
+        xmlstream.ListenAuthenticator.streamStarted(self, rootElement)
+
+        if rootElement.defaultUri != self.namespace:
+            exc = error.StreamError("invalid-namespace")
+            self.xmlstream.sendStreamError(exc)
+            return
+
+        # self.xmlstream.thisEntity is set to the address the component
+        # wants to assume.
+        if not self.xmlstream.thisEntity:
+            exc = error.StreamError("improper-addressing")
+            self.xmlstream.sendStreamError(exc)
+            return
+
+        self.xmlstream.sendHeader()
+        self.xmlstream.addOnetimeObserver("/*", self.onElement)
+
+    def onElement(self, element):
+        """
+        Called on incoming XML Stanzas.
+
+        The very first element received should be a request for handshake.
+        Otherwise, the stream is dropped with a 'not-authorized' error. If a
+        handshake request was received, the hash is extracted and passed to
+        L{onHandshake}.
+        """
+        if (element.uri, element.name) == (self.namespace, "handshake"):
+            self.onHandshake(str(element))
+        else:
+            exc = error.StreamError("not-authorized")
+            self.xmlstream.sendStreamError(exc)
+
+    def onHandshake(self, handshake):
+        """
+        Called upon receiving the handshake request.
+
+        This checks that the given hash in C{handshake} is equal to a
+        calculated hash, responding with a handshake reply or a stream error.
+        If the handshake was ok, the stream is authorized, and  XML Stanzas may
+        be exchanged.
+        """
+        calculatedHash = xmlstream.hashPassword(self.xmlstream.sid, str(self.secret))
+        if handshake != calculatedHash:
+            exc = error.StreamError("not-authorized", text="Invalid hash")
+            self.xmlstream.sendStreamError(exc)
+        else:
+            self.xmlstream.send("<handshake/>")
+            self.xmlstream.dispatch(self.xmlstream, xmlstream.STREAM_AUTHD_EVENT)
+
+
+@implementer(ijabber.IService)
+class Service(service.Service):
+    """
+    External server-side component service.
+    """
+
+    def componentConnected(self, xs):
+        pass
+
+    def componentDisconnected(self):
+        pass
+
+    def transportConnected(self, xs):
+        pass
+
+    def send(self, obj):
+        """
+        Send data over service parent's XML stream.
+
+        @note: L{ServiceManager} maintains a queue for data sent using this
+        method when there is no current established XML stream. This data is
+        then sent as soon as a new stream has been established and initialized.
+        Subsequently, L{componentConnected} will be called again. If this
+        queueing is not desired, use C{send} on the XmlStream object (passed to
+        L{componentConnected}) directly.
+
+        @param obj: data to be sent over the XML stream. This is usually an
+        object providing L{domish.IElement}, or serialized XML. See
+        L{xmlstream.XmlStream} for details.
+        """
+
+        self.parent.send(obj)
+
+
+class ServiceManager(service.MultiService):
+    """
+    Business logic for a managed component connection to a Jabber router.
+
+    This service maintains a single connection to a Jabber router and provides
+    facilities for packet routing and transmission. Business logic modules are
+    services implementing L{ijabber.IService} (like subclasses of L{Service}),
+    and added as sub-service.
+    """
+
+    def __init__(self, jid, password):
+        service.MultiService.__init__(self)
+
+        # Setup defaults
+        self.jabberId = jid
+        self.xmlstream = None
+
+        # Internal buffer of packets
+        self._packetQueue = []
+
+        # Setup the xmlstream factory
+        self._xsFactory = componentFactory(self.jabberId, password)
+
+        # Register some lambda functions to keep the self.xmlstream var up to
+        # date
+        self._xsFactory.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT, self._connected)
+        self._xsFactory.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self._authd)
+        self._xsFactory.addBootstrap(xmlstream.STREAM_END_EVENT, self._disconnected)
+
+        # Map addBootstrap and removeBootstrap to the underlying factory -- is
+        # this right? I have no clue...but it'll work for now, until i can
+        # think about it more.
+        self.addBootstrap = self._xsFactory.addBootstrap
+        self.removeBootstrap = self._xsFactory.removeBootstrap
+
+    def getFactory(self):
+        return self._xsFactory
+
+    def _connected(self, xs):
+        self.xmlstream = xs
+        for c in self:
+            if ijabber.IService.providedBy(c):
+                c.transportConnected(xs)
+
+    def _authd(self, xs):
+        # Flush all pending packets
+        for p in self._packetQueue:
+            self.xmlstream.send(p)
+        self._packetQueue = []
+
+        # Notify all child services which implement the IService interface
+        for c in self:
+            if ijabber.IService.providedBy(c):
+                c.componentConnected(xs)
+
+    def _disconnected(self, _):
+        self.xmlstream = None
+
+        # Notify all child services which implement
+        # the IService interface
+        for c in self:
+            if ijabber.IService.providedBy(c):
+                c.componentDisconnected()
+
+    def send(self, obj):
+        """
+        Send data over the XML stream.
+
+        When there is no established XML stream, the data is queued and sent
+        out when a new XML stream has been established and initialized.
+
+        @param obj: data to be sent over the XML stream. This is usually an
+        object providing L{domish.IElement}, or serialized XML. See
+        L{xmlstream.XmlStream} for details.
+        """
+
+        if self.xmlstream != None:
+            self.xmlstream.send(obj)
+        else:
+            self._packetQueue.append(obj)
+
+
+def buildServiceManager(jid, password, strport):
+    """
+    Constructs a pre-built L{ServiceManager}, using the specified strport
+    string.
+    """
+
+    svc = ServiceManager(jid, password)
+    client_svc = jstrports.client(strport, svc.getFactory())
+    client_svc.setServiceParent(svc)
+    return svc
+
+
+class Router:
+    """
+    XMPP Server's Router.
+
+    A router connects the different components of the XMPP service and routes
+    messages between them based on the given routing table.
+
+    Connected components are trusted to have correct addressing in the
+    stanzas they offer for routing.
+
+    A route destination of L{None} adds a default route. Traffic for which no
+    specific route exists, will be routed to this default route.
+
+    @since: 8.2
+    @ivar routes: Routes based on the host part of JIDs. Maps host names to the
+                  L{EventDispatcher<utility.EventDispatcher>}s that should
+                  receive the traffic. A key of L{None} means the default
+                  route.
+    @type routes: C{dict}
+    """
+
+    def __init__(self):
+        self.routes = {}
+
+    def addRoute(self, destination, xs):
+        """
+        Add a new route.
+
+        The passed XML Stream C{xs} will have an observer for all stanzas
+        added to route its outgoing traffic. In turn, traffic for
+        C{destination} will be passed to this stream.
+
+        @param destination: Destination of the route to be added as a host name
+                            or L{None} for the default route.
+        @type destination: C{str} or L{None}.
+        @param xs: XML Stream to register the route for.
+        @type xs: L{EventDispatcher<utility.EventDispatcher>}.
+        """
+        self.routes[destination] = xs
+        xs.addObserver("/*", self.route)
+
+    def removeRoute(self, destination, xs):
+        """
+        Remove a route.
+
+        @param destination: Destination of the route that should be removed.
+        @type destination: C{str}.
+        @param xs: XML Stream to remove the route for.
+        @type xs: L{EventDispatcher<utility.EventDispatcher>}.
+        """
+        xs.removeObserver("/*", self.route)
+        if xs == self.routes[destination]:
+            del self.routes[destination]
+
+    def route(self, stanza):
+        """
+        Route a stanza.
+
+        @param stanza: The stanza to be routed.
+        @type stanza: L{domish.Element}.
+        """
+        destination = JID(stanza["to"])
+
+        log.msg(f"Routing to {destination.full()}: {stanza.toXml()!r}")
+
+        if destination.host in self.routes:
+            self.routes[destination.host].send(stanza)
+        else:
+            self.routes[None].send(stanza)
+
+
+class XMPPComponentServerFactory(xmlstream.XmlStreamServerFactory):
+    """
+    XMPP Component Server factory.
+
+    This factory accepts XMPP external component connections and makes
+    the router service route traffic for a component's bound domain
+    to that component.
+
+    @since: 8.2
+    """
+
+    logTraffic = False
+
+    def __init__(self, router, secret="secret"):
+        self.router = router
+        self.secret = secret
+
+        def authenticatorFactory():
+            return ListenComponentAuthenticator(self.secret)
+
+        xmlstream.XmlStreamServerFactory.__init__(self, authenticatorFactory)
+        self.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT, self.onConnectionMade)
+        self.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self.onAuthenticated)
+
+        self.serial = 0
+
+    def onConnectionMade(self, xs):
+        """
+        Called when a component connection was made.
+
+        This enables traffic debugging on incoming streams.
+        """
+        xs.serial = self.serial
+        self.serial += 1
+
+        def logDataIn(buf):
+            log.msg("RECV (%d): %r" % (xs.serial, buf))
+
+        def logDataOut(buf):
+            log.msg("SEND (%d): %r" % (xs.serial, buf))
+
+        if self.logTraffic:
+            xs.rawDataInFn = logDataIn
+            xs.rawDataOutFn = logDataOut
+
+        xs.addObserver(xmlstream.STREAM_ERROR_EVENT, self.onError)
+
+    def onAuthenticated(self, xs):
+        """
+        Called when a component has successfully authenticated.
+
+        Add the component to the routing table and establish a handler
+        for a closed connection.
+        """
+        destination = xs.thisEntity.host
+
+        self.router.addRoute(destination, xs)
+        xs.addObserver(
+            xmlstream.STREAM_END_EVENT, self.onConnectionLost, 0, destination, xs
+        )
+
+    def onError(self, reason):
+        log.err(reason, "Stream Error")
+
+    def onConnectionLost(self, destination, xs, reason):
+        self.router.removeRoute(destination, xs)
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/error.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/error.py
new file mode 100644
index 00000000000..4d1644767de
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/error.py
@@ -0,0 +1,323 @@
+# -*- test-case-name: twisted.words.test.test_jabbererror -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XMPP Error support.
+"""
+
+
+import copy
+from typing import Optional
+
+from twisted.words.xish import domish
+
+NS_XML = "http://www.w3.org/XML/1998/namespace"
+NS_XMPP_STREAMS = "urn:ietf:params:xml:ns:xmpp-streams"
+NS_XMPP_STANZAS = "urn:ietf:params:xml:ns:xmpp-stanzas"
+
+STANZA_CONDITIONS = {
+    "bad-request": {"code": "400", "type": "modify"},
+    "conflict": {"code": "409", "type": "cancel"},
+    "feature-not-implemented": {"code": "501", "type": "cancel"},
+    "forbidden": {"code": "403", "type": "auth"},
+    "gone": {"code": "302", "type": "modify"},
+    "internal-server-error": {"code": "500", "type": "wait"},
+    "item-not-found": {"code": "404", "type": "cancel"},
+    "jid-malformed": {"code": "400", "type": "modify"},
+    "not-acceptable": {"code": "406", "type": "modify"},
+    "not-allowed": {"code": "405", "type": "cancel"},
+    "not-authorized": {"code": "401", "type": "auth"},
+    "payment-required": {"code": "402", "type": "auth"},
+    "recipient-unavailable": {"code": "404", "type": "wait"},
+    "redirect": {"code": "302", "type": "modify"},
+    "registration-required": {"code": "407", "type": "auth"},
+    "remote-server-not-found": {"code": "404", "type": "cancel"},
+    "remote-server-timeout": {"code": "504", "type": "wait"},
+    "resource-constraint": {"code": "500", "type": "wait"},
+    "service-unavailable": {"code": "503", "type": "cancel"},
+    "subscription-required": {"code": "407", "type": "auth"},
+    "undefined-condition": {"code": "500", "type": None},
+    "unexpected-request": {"code": "400", "type": "wait"},
+}
+
+CODES_TO_CONDITIONS = {
+    "302": ("gone", "modify"),
+    "400": ("bad-request", "modify"),
+    "401": ("not-authorized", "auth"),
+    "402": ("payment-required", "auth"),
+    "403": ("forbidden", "auth"),
+    "404": ("item-not-found", "cancel"),
+    "405": ("not-allowed", "cancel"),
+    "406": ("not-acceptable", "modify"),
+    "407": ("registration-required", "auth"),
+    "408": ("remote-server-timeout", "wait"),
+    "409": ("conflict", "cancel"),
+    "500": ("internal-server-error", "wait"),
+    "501": ("feature-not-implemented", "cancel"),
+    "502": ("service-unavailable", "wait"),
+    "503": ("service-unavailable", "cancel"),
+    "504": ("remote-server-timeout", "wait"),
+    "510": ("service-unavailable", "cancel"),
+}
+
+
+class BaseError(Exception):
+    """
+    Base class for XMPP error exceptions.
+
+    @cvar namespace: The namespace of the C{error} element generated by
+                     C{getElement}.
+    @type namespace: C{str}
+    @ivar condition: The error condition. The valid values are defined by
+                     subclasses of L{BaseError}.
+    @type contition: C{str}
+    @ivar text: Optional text message to supplement the condition or application
+                specific condition.
+    @type text: C{unicode}
+    @ivar textLang: Identifier of the language used for the message in C{text}.
+                    Values are as described in RFC 3066.
+    @type textLang: C{str}
+    @ivar appCondition: Application specific condition element, supplementing
+                        the error condition in C{condition}.
+    @type appCondition: object providing L{domish.IElement}.
+    """
+
+    namespace: Optional[str] = None
+
+    def __init__(self, condition, text=None, textLang=None, appCondition=None):
+        Exception.__init__(self)
+        self.condition = condition
+        self.text = text
+        self.textLang = textLang
+        self.appCondition = appCondition
+
+    def __str__(self) -> str:
+        message = "{} with condition {!r}".format(
+            self.__class__.__name__, self.condition
+        )
+
+        if self.text:
+            message += ": " + self.text
+
+        return message
+
+    def getElement(self):
+        """
+        Get XML representation from self.
+
+        The method creates an L{domish} representation of the
+        error data contained in this exception.
+
+        @rtype: L{domish.Element}
+        """
+        error = domish.Element((None, "error"))
+        error.addElement((self.namespace, self.condition))
+        if self.text:
+            text = error.addElement((self.namespace, "text"), content=self.text)
+            if self.textLang:
+                text[(NS_XML, "lang")] = self.textLang
+        if self.appCondition:
+            error.addChild(self.appCondition)
+        return error
+
+
+class StreamError(BaseError):
+    """
+    Stream Error exception.
+
+    Refer to RFC 3920, section 4.7.3, for the allowed values for C{condition}.
+    """
+
+    namespace = NS_XMPP_STREAMS
+
+    def getElement(self):
+        """
+        Get XML representation from self.
+
+        Overrides the base L{BaseError.getElement} to make sure the returned
+        element is in the XML Stream namespace.
+
+        @rtype: L{domish.Element}
+        """
+        from twisted.words.protocols.jabber.xmlstream import NS_STREAMS
+
+        error = BaseError.getElement(self)
+        error.uri = NS_STREAMS
+        return error
+
+
+class StanzaError(BaseError):
+    """
+    Stanza Error exception.
+
+    Refer to RFC 3920, section 9.3, for the allowed values for C{condition} and
+    C{type}.
+
+    @ivar type: The stanza error type. Gives a suggestion to the recipient
+                of the error on how to proceed.
+    @type type: C{str}
+    @ivar code: A numeric identifier for the error condition for backwards
+                compatibility with pre-XMPP Jabber implementations.
+    """
+
+    namespace = NS_XMPP_STANZAS
+
+    def __init__(
+        self, condition, type=None, text=None, textLang=None, appCondition=None
+    ):
+        BaseError.__init__(self, condition, text, textLang, appCondition)
+
+        if type is None:
+            try:
+                type = STANZA_CONDITIONS[condition]["type"]
+            except KeyError:
+                pass
+        self.type = type
+
+        try:
+            self.code = STANZA_CONDITIONS[condition]["code"]
+        except KeyError:
+            self.code = None
+
+        self.children = []
+        self.iq = None
+
+    def getElement(self):
+        """
+        Get XML representation from self.
+
+        Overrides the base L{BaseError.getElement} to make sure the returned
+        element has a C{type} attribute and optionally a legacy C{code}
+        attribute.
+
+        @rtype: L{domish.Element}
+        """
+        error = BaseError.getElement(self)
+        error["type"] = self.type
+        if self.code:
+            error["code"] = self.code
+        return error
+
+    def toResponse(self, stanza):
+        """
+        Construct error response stanza.
+
+        The C{stanza} is transformed into an error response stanza by
+        swapping the C{to} and C{from} addresses and inserting an error
+        element.
+
+        @note: This creates a shallow copy of the list of child elements of the
+               stanza. The child elements themselves are not copied themselves,
+               and references to their parent element will still point to the
+               original stanza element.
+
+               The serialization of an element does not use the reference to
+               its parent, so the typical use case of immediately sending out
+               the constructed error response is not affected.
+
+        @param stanza: the stanza to respond to
+        @type stanza: L{domish.Element}
+        """
+        from twisted.words.protocols.jabber.xmlstream import toResponse
+
+        response = toResponse(stanza, stanzaType="error")
+        response.children = copy.copy(stanza.children)
+        response.addChild(self.getElement())
+        return response
+
+
+def _parseError(error, errorNamespace):
+    """
+    Parses an error element.
+
+    @param error: The error element to be parsed
+    @type error: L{domish.Element}
+    @param errorNamespace: The namespace of the elements that hold the error
+                           condition and text.
+    @type errorNamespace: C{str}
+    @return: Dictionary with extracted error information. If present, keys
+             C{condition}, C{text}, C{textLang} have a string value,
+             and C{appCondition} has an L{domish.Element} value.
+    @rtype: C{dict}
+    """
+    condition = None
+    text = None
+    textLang = None
+    appCondition = None
+
+    for element in error.elements():
+        if element.uri == errorNamespace:
+            if element.name == "text":
+                text = str(element)
+                textLang = element.getAttribute((NS_XML, "lang"))
+            else:
+                condition = element.name
+        else:
+            appCondition = element
+
+    return {
+        "condition": condition,
+        "text": text,
+        "textLang": textLang,
+        "appCondition": appCondition,
+    }
+
+
+def exceptionFromStreamError(element):
+    """
+    Build an exception object from a stream error.
+
+    @param element: the stream error
+    @type element: L{domish.Element}
+    @return: the generated exception object
+    @rtype: L{StreamError}
+    """
+    error = _parseError(element, NS_XMPP_STREAMS)
+
+    exception = StreamError(
+        error["condition"], error["text"], error["textLang"], error["appCondition"]
+    )
+
+    return exception
+
+
+def exceptionFromStanza(stanza):
+    """
+    Build an exception object from an error stanza.
+
+    @param stanza: the error stanza
+    @type stanza: L{domish.Element}
+    @return: the generated exception object
+    @rtype: L{StanzaError}
+    """
+    children = []
+    condition = text = textLang = appCondition = type = code = None
+
+    for element in stanza.elements():
+        if element.name == "error" and element.uri == stanza.uri:
+            code = element.getAttribute("code")
+            type = element.getAttribute("type")
+            error = _parseError(element, NS_XMPP_STANZAS)
+            condition = error["condition"]
+            text = error["text"]
+            textLang = error["textLang"]
+            appCondition = error["appCondition"]
+
+            if not condition and code:
+                condition, type = CODES_TO_CONDITIONS[code]
+                text = str(stanza.error)
+        else:
+            children.append(element)
+
+    if condition is None:
+        # TODO: raise exception instead?
+        return StanzaError(None)
+
+    exception = StanzaError(condition, type, text, textLang, appCondition)
+
+    exception.children = children
+    exception.stanza = stanza
+
+    return exception
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/ijabber.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/ijabber.py
new file mode 100644
index 00000000000..5408a9ae6c4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/ijabber.py
@@ -0,0 +1,188 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Public Jabber Interfaces.
+"""
+
+from zope.interface import Attribute, Interface
+
+
+class IInitializer(Interface):
+    """
+    Interface for XML stream initializers.
+
+    Initializers perform a step in getting the XML stream ready to be
+    used for the exchange of XML stanzas.
+    """
+
+
+class IInitiatingInitializer(IInitializer):
+    """
+    Interface for XML stream initializers for the initiating entity.
+    """
+
+    xmlstream = Attribute("""The associated XML stream""")
+
+    def initialize():
+        """
+        Initiate the initialization step.
+
+        May return a deferred when the initialization is done asynchronously.
+        """
+
+
+class IIQResponseTracker(Interface):
+    """
+    IQ response tracker interface.
+
+    The XMPP stanza C{iq} has a request-response nature that fits
+    naturally with deferreds. You send out a request and when the response
+    comes back a deferred is fired.
+
+    The L{twisted.words.protocols.jabber.client.IQ} class implements a C{send}
+    method that returns a deferred. This deferred is put in a dictionary that
+    is kept in an L{XmlStream} object, keyed by the request stanzas C{id}
+    attribute.
+
+    An object providing this interface (usually an instance of L{XmlStream}),
+    keeps the said dictionary and sets observers on the iq stanzas of type
+    C{result} and C{error} and lets the callback fire the associated deferred.
+    """
+
+    iqDeferreds = Attribute("Dictionary of deferreds waiting for an iq " "response")
+
+
+class IXMPPHandler(Interface):
+    """
+    Interface for XMPP protocol handlers.
+
+    Objects that provide this interface can be added to a stream manager to
+    handle of (part of) an XMPP extension protocol.
+    """
+
+    parent = Attribute("""XML stream manager for this handler""")
+    xmlstream = Attribute("""The managed XML stream""")
+
+    def setHandlerParent(parent):
+        """
+        Set the parent of the handler.
+
+        @type parent: L{IXMPPHandlerCollection}
+        """
+
+    def disownHandlerParent(parent):
+        """
+        Remove the parent of the handler.
+
+        @type parent: L{IXMPPHandlerCollection}
+        """
+
+    def makeConnection(xs):
+        """
+        A connection over the underlying transport of the XML stream has been
+        established.
+
+        At this point, no traffic has been exchanged over the XML stream
+        given in C{xs}.
+
+        This should setup L{xmlstream} and call L{connectionMade}.
+
+        @type xs:
+               L{twisted.words.protocols.jabber.xmlstream.XmlStream}
+        """
+
+    def connectionMade():
+        """
+        Called after a connection has been established.
+
+        This method can be used to change properties of the XML Stream, its
+        authenticator or the stream manager prior to stream initialization
+        (including authentication).
+        """
+
+    def connectionInitialized():
+        """
+        The XML stream has been initialized.
+
+        At this point, authentication was successful, and XML stanzas can be
+        exchanged over the XML stream L{xmlstream}. This method can be
+        used to setup observers for incoming stanzas.
+        """
+
+    def connectionLost(reason):
+        """
+        The XML stream has been closed.
+
+        Subsequent use of C{parent.send} will result in data being queued
+        until a new connection has been established.
+
+        @type reason: L{twisted.python.failure.Failure}
+        """
+
+
+class IXMPPHandlerCollection(Interface):
+    """
+    Collection of handlers.
+
+    Contain several handlers and manage their connection.
+    """
+
+    def __iter__():
+        """
+        Get an iterator over all child handlers.
+        """
+
+    def addHandler(handler):
+        """
+        Add a child handler.
+
+        @type handler: L{IXMPPHandler}
+        """
+
+    def removeHandler(handler):
+        """
+        Remove a child handler.
+
+        @type handler: L{IXMPPHandler}
+        """
+
+
+class IService(Interface):
+    """
+    External server-side component service interface.
+
+    Services that provide this interface can be added to L{ServiceManager} to
+    implement (part of) the functionality of the server-side component.
+    """
+
+    def componentConnected(xs):
+        """
+        Parent component has established a connection.
+
+        At this point, authentication was successful, and XML stanzas
+        can be exchanged over the XML stream C{xs}. This method can be used
+        to setup observers for incoming stanzas.
+
+        @param xs: XML Stream that represents the established connection.
+        @type xs: L{xmlstream.XmlStream}
+        """
+
+    def componentDisconnected():
+        """
+        Parent component has lost the connection to the Jabber server.
+
+        Subsequent use of C{self.parent.send} will result in data being
+        queued until a new connection has been established.
+        """
+
+    def transportConnected(xs):
+        """
+        Parent component has established a connection over the underlying
+        transport.
+
+        At this point, no traffic has been exchanged over the XML stream. This
+        method can be used to change properties of the XML Stream (in C{xs}),
+        the service manager or it's authenticator prior to stream
+        initialization (including authentication).
+        """
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/jid.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/jid.py
new file mode 100644
index 00000000000..52e154fee4f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/jid.py
@@ -0,0 +1,259 @@
+# -*- test-case-name: twisted.words.test.test_jabberjid -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Jabber Identifier support.
+
+This module provides an object to represent Jabber Identifiers (JIDs) and
+parse string representations into them with proper checking for illegal
+characters, case folding and canonicalisation through
+L{stringprep<twisted.words.protocols.jabber.xmpp_stringprep>}.
+"""
+
+from typing import Dict, Tuple, Union
+
+from twisted.words.protocols.jabber.xmpp_stringprep import (
+    nameprep,
+    nodeprep,
+    resourceprep,
+)
+
+
+class InvalidFormat(Exception):
+    """
+    The given string could not be parsed into a valid Jabber Identifier (JID).
+    """
+
+
+def parse(jidstring: str) -> Tuple[Union[str, None], str, Union[str, None]]:
+    """
+    Parse given JID string into its respective parts and apply stringprep.
+
+    @param jidstring: string representation of a JID.
+    @type jidstring: L{str}
+    @return: tuple of (user, host, resource), each of type L{str} as
+             the parsed and stringprep'd parts of the given JID. If the
+             given string did not have a user or resource part, the respective
+             field in the tuple will hold L{None}.
+    @rtype: L{tuple}
+    """
+    user = None
+    host = None
+    resource = None
+
+    # Search for delimiters
+    user_sep = jidstring.find("@")
+    res_sep = jidstring.find("/")
+
+    if user_sep == -1:
+        if res_sep == -1:
+            # host
+            host = jidstring
+        else:
+            # host/resource
+            host = jidstring[0:res_sep]
+            resource = jidstring[res_sep + 1 :] or None
+    else:
+        if res_sep == -1:
+            # user@host
+            user = jidstring[0:user_sep] or None
+            host = jidstring[user_sep + 1 :]
+        else:
+            if user_sep < res_sep:
+                # user@host/resource
+                user = jidstring[0:user_sep] or None
+                host = jidstring[user_sep + 1 : user_sep + (res_sep - user_sep)]
+                resource = jidstring[res_sep + 1 :] or None
+            else:
+                # host/resource (with an @ in resource)
+                host = jidstring[0:res_sep]
+                resource = jidstring[res_sep + 1 :] or None
+
+    return prep(user, host, resource)
+
+
+def prep(
+    user: Union[str, None], host: str, resource: Union[str, None]
+) -> Tuple[Union[str, None], str, Union[str, None]]:
+    """
+    Perform stringprep on all JID fragments.
+
+    @param user: The user part of the JID.
+    @type user: L{str}
+    @param host: The host part of the JID.
+    @type host: L{str}
+    @param resource: The resource part of the JID.
+    @type resource: L{str}
+    @return: The given parts with stringprep applied.
+    @rtype: L{tuple}
+    """
+
+    if user:
+        try:
+            user = nodeprep.prepare(str(user))
+        except UnicodeError:
+            raise InvalidFormat("Invalid character in username")
+    else:
+        user = None
+
+    if not host:
+        raise InvalidFormat("Server address required.")
+    else:
+        try:
+            host = nameprep.prepare(str(host))
+        except UnicodeError:
+            raise InvalidFormat("Invalid character in hostname")
+
+    if resource:
+        try:
+            resource = resourceprep.prepare(str(resource))
+        except UnicodeError:
+            raise InvalidFormat("Invalid character in resource")
+    else:
+        resource = None
+
+    return (user, host, resource)
+
+
+__internJIDs: Dict[str, "JID"] = {}
+
+
+def internJID(jidstring):
+    """
+    Return interned JID.
+
+    @rtype: L{JID}
+    """
+
+    if jidstring in __internJIDs:
+        return __internJIDs[jidstring]
+    else:
+        j = JID(jidstring)
+        __internJIDs[jidstring] = j
+        return j
+
+
+class JID:
+    """
+    Represents a stringprep'd Jabber ID.
+
+    JID objects are hashable so they can be used in sets and as keys in
+    dictionaries.
+    """
+
+    def __init__(
+        self,
+        str: Union[str, None] = None,
+        tuple: Union[Tuple[Union[str, None], str, Union[str, None]], None] = None,
+    ):
+        if str:
+            user, host, res = parse(str)
+        elif tuple:
+            user, host, res = prep(*tuple)
+        else:
+            raise RuntimeError(
+                "You must provide a value for either 'str' or 'tuple' arguments."
+            )
+
+        self.user = user
+        self.host = host
+        self.resource = res
+
+    def userhost(self):
+        """
+        Extract the bare JID as a unicode string.
+
+        A bare JID does not have a resource part, so this returns either
+        C{user@host} or just C{host}.
+
+        @rtype: L{str}
+        """
+        if self.user:
+            return f"{self.user}@{self.host}"
+        else:
+            return self.host
+
+    def userhostJID(self):
+        """
+        Extract the bare JID.
+
+        A bare JID does not have a resource part, so this returns a
+        L{JID} object representing either C{user@host} or just C{host}.
+
+        If the object this method is called upon doesn't have a resource
+        set, it will return itself. Otherwise, the bare JID object will
+        be created, interned using L{internJID}.
+
+        @rtype: L{JID}
+        """
+        if self.resource:
+            return internJID(self.userhost())
+        else:
+            return self
+
+    def full(self):
+        """
+        Return the string representation of this JID.
+
+        @rtype: L{str}
+        """
+        if self.user:
+            if self.resource:
+                return f"{self.user}@{self.host}/{self.resource}"
+            else:
+                return f"{self.user}@{self.host}"
+        else:
+            if self.resource:
+                return f"{self.host}/{self.resource}"
+            else:
+                return self.host
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Equality comparison.
+
+        L{JID}s compare equal if their user, host and resource parts all
+        compare equal.  When comparing against instances of other types, it
+        uses the default comparison.
+        """
+        if isinstance(other, JID):
+            return (
+                self.user == other.user
+                and self.host == other.host
+                and self.resource == other.resource
+            )
+        else:
+            return NotImplemented
+
+    def __hash__(self):
+        """
+        Calculate hash.
+
+        L{JID}s with identical constituent user, host and resource parts have
+        equal hash values.  In combination with the comparison defined on JIDs,
+        this allows for using L{JID}s in sets and as dictionary keys.
+        """
+        return hash((self.user, self.host, self.resource))
+
+    def __unicode__(self):
+        """
+        Get unicode representation.
+
+        Return the string representation of this JID as a unicode string.
+        @see: L{full}
+        """
+
+        return self.full()
+
+    __str__ = __unicode__
+
+    def __repr__(self) -> str:
+        """
+        Get object representation.
+
+        Returns a string that would create a new JID object that compares equal
+        to this one.
+        """
+        return "JID(%r)" % self.full()
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/jstrports.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/jstrports.py
new file mode 100644
index 00000000000..b564fe3512f
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/jstrports.py
@@ -0,0 +1,34 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+""" A temporary placeholder for client-capable strports, until we
+sufficient use cases get identified """
+
+
+from twisted.internet.endpoints import _parse
+
+
+def _parseTCPSSL(factory, domain, port):
+    """For the moment, parse TCP or SSL connections the same"""
+    return (domain, int(port), factory), {}
+
+
+def _parseUNIX(factory, address):
+    return (address, factory), {}
+
+
+_funcs = {"tcp": _parseTCPSSL, "unix": _parseUNIX, "ssl": _parseTCPSSL}
+
+
+def parse(description, factory):
+    args, kw = _parse(description)
+    return (args[0].upper(),) + _funcs[args[0]](factory, *args[1:], **kw)
+
+
+def client(description, factory):
+    from twisted.application import internet
+
+    name, args, kw = parse(description, factory)
+    return getattr(internet, name + "Client")(*args, **kw)
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/sasl.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/sasl.py
new file mode 100644
index 00000000000..8bcf1a534aa
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/sasl.py
@@ -0,0 +1,229 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XMPP-specific SASL profile.
+"""
+
+
+import re
+from base64 import b64decode, b64encode
+
+from twisted.internet import defer
+from twisted.words.protocols.jabber import sasl_mechanisms, xmlstream
+from twisted.words.xish import domish
+
+NS_XMPP_SASL = "urn:ietf:params:xml:ns:xmpp-sasl"
+
+
+def get_mechanisms(xs):
+    """
+    Parse the SASL feature to extract the available mechanism names.
+    """
+    mechanisms = []
+    for element in xs.features[(NS_XMPP_SASL, "mechanisms")].elements():
+        if element.name == "mechanism":
+            mechanisms.append(str(element))
+
+    return mechanisms
+
+
+class SASLError(Exception):
+    """
+    SASL base exception.
+    """
+
+
+class SASLNoAcceptableMechanism(SASLError):
+    """
+    The server did not present an acceptable SASL mechanism.
+    """
+
+
+class SASLAuthError(SASLError):
+    """
+    SASL Authentication failed.
+    """
+
+    def __init__(self, condition=None):
+        self.condition = condition
+
+    def __str__(self) -> str:
+        return "SASLAuthError with condition %r" % self.condition
+
+
+class SASLIncorrectEncodingError(SASLError):
+    """
+    SASL base64 encoding was incorrect.
+
+    RFC 3920 specifies that any characters not in the base64 alphabet
+    and padding characters present elsewhere than at the end of the string
+    MUST be rejected. See also L{fromBase64}.
+
+    This exception is raised whenever the encoded string does not adhere
+    to these additional restrictions or when the decoding itself fails.
+
+    The recommended behaviour for so-called receiving entities (like servers in
+    client-to-server connections, see RFC 3920 for terminology) is to fail the
+    SASL negotiation with a C{'incorrect-encoding'} condition. For initiating
+    entities, one should assume the receiving entity to be either buggy or
+    malevolent. The stream should be terminated and reconnecting is not
+    advised.
+    """
+
+
+base64Pattern = re.compile("^[0-9A-Za-z+/]*[0-9A-Za-z+/=]{,2}$")
+
+
+def fromBase64(s):
+    """
+    Decode base64 encoded string.
+
+    This helper performs regular decoding of a base64 encoded string, but also
+    rejects any characters that are not in the base64 alphabet and padding
+    occurring elsewhere from the last or last two characters, as specified in
+    section 14.9 of RFC 3920. This safeguards against various attack vectors
+    among which the creation of a covert channel that "leaks" information.
+    """
+
+    if base64Pattern.match(s) is None:
+        raise SASLIncorrectEncodingError()
+
+    try:
+        return b64decode(s)
+    except Exception as e:
+        raise SASLIncorrectEncodingError(str(e))
+
+
+class SASLInitiatingInitializer(xmlstream.BaseFeatureInitiatingInitializer):
+    """
+    Stream initializer that performs SASL authentication.
+
+    The supported mechanisms by this initializer are C{DIGEST-MD5}, C{PLAIN}
+    and C{ANONYMOUS}. The C{ANONYMOUS} SASL mechanism is used when the JID, set
+    on the authenticator, does not have a localpart (username), requesting an
+    anonymous session where the username is generated by the server.
+    Otherwise, C{DIGEST-MD5} and C{PLAIN} are attempted, in that order.
+    """
+
+    feature = (NS_XMPP_SASL, "mechanisms")
+    _deferred = None
+
+    def setMechanism(self):
+        """
+        Select and setup authentication mechanism.
+
+        Uses the authenticator's C{jid} and C{password} attribute for the
+        authentication credentials. If no supported SASL mechanisms are
+        advertized by the receiving party, a failing deferred is returned with
+        a L{SASLNoAcceptableMechanism} exception.
+        """
+
+        jid = self.xmlstream.authenticator.jid
+        password = self.xmlstream.authenticator.password
+
+        mechanisms = get_mechanisms(self.xmlstream)
+        if jid.user is not None:
+            if "DIGEST-MD5" in mechanisms:
+                self.mechanism = sasl_mechanisms.DigestMD5(
+                    "xmpp", jid.host, None, jid.user, password
+                )
+            elif "PLAIN" in mechanisms:
+                self.mechanism = sasl_mechanisms.Plain(None, jid.user, password)
+            else:
+                raise SASLNoAcceptableMechanism()
+        else:
+            if "ANONYMOUS" in mechanisms:
+                self.mechanism = sasl_mechanisms.Anonymous()
+            else:
+                raise SASLNoAcceptableMechanism()
+
+    def start(self):
+        """
+        Start SASL authentication exchange.
+        """
+
+        self.setMechanism()
+        self._deferred = defer.Deferred()
+        self.xmlstream.addObserver("/challenge", self.onChallenge)
+        self.xmlstream.addOnetimeObserver("/success", self.onSuccess)
+        self.xmlstream.addOnetimeObserver("/failure", self.onFailure)
+        self.sendAuth(self.mechanism.getInitialResponse())
+        return self._deferred
+
+    def sendAuth(self, data=None):
+        """
+        Initiate authentication protocol exchange.
+
+        If an initial client response is given in C{data}, it will be
+        sent along.
+
+        @param data: initial client response.
+        @type data: C{str} or L{None}.
+        """
+
+        auth = domish.Element((NS_XMPP_SASL, "auth"))
+        auth["mechanism"] = self.mechanism.name
+        if data is not None:
+            auth.addContent(b64encode(data).decode("ascii") or "=")
+        self.xmlstream.send(auth)
+
+    def sendResponse(self, data=b""):
+        """
+        Send response to a challenge.
+
+        @param data: client response.
+        @type data: L{bytes}.
+        """
+
+        response = domish.Element((NS_XMPP_SASL, "response"))
+        if data:
+            response.addContent(b64encode(data).decode("ascii"))
+        self.xmlstream.send(response)
+
+    def onChallenge(self, element):
+        """
+        Parse challenge and send response from the mechanism.
+
+        @param element: the challenge protocol element.
+        @type element: L{domish.Element}.
+        """
+
+        try:
+            challenge = fromBase64(str(element))
+        except SASLIncorrectEncodingError:
+            self._deferred.errback()
+        else:
+            self.sendResponse(self.mechanism.getResponse(challenge))
+
+    def onSuccess(self, success):
+        """
+        Clean up observers, reset the XML stream and send a new header.
+
+        @param success: the success protocol element. For now unused, but
+                        could hold additional data.
+        @type success: L{domish.Element}
+        """
+
+        self.xmlstream.removeObserver("/challenge", self.onChallenge)
+        self.xmlstream.removeObserver("/failure", self.onFailure)
+        self.xmlstream.reset()
+        self.xmlstream.sendHeader()
+        self._deferred.callback(xmlstream.Reset)
+
+    def onFailure(self, failure):
+        """
+        Clean up observers, parse the failure and errback the deferred.
+
+        @param failure: the failure protocol element. Holds details on
+                        the error condition.
+        @type failure: L{domish.Element}
+        """
+
+        self.xmlstream.removeObserver("/challenge", self.onChallenge)
+        self.xmlstream.removeObserver("/success", self.onSuccess)
+        try:
+            condition = failure.firstChildElement().name
+        except AttributeError:
+            condition = None
+        self._deferred.errback(SASLAuthError(condition))
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/sasl_mechanisms.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/sasl_mechanisms.py
new file mode 100644
index 00000000000..8d9c8fabccf
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/sasl_mechanisms.py
@@ -0,0 +1,307 @@
+# -*- test-case-name: twisted.words.test.test_jabbersaslmechanisms -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Protocol agnostic implementations of SASL authentication mechanisms.
+"""
+
+
+import binascii
+import os
+import random
+import time
+from hashlib import md5
+
+from zope.interface import Attribute, Interface, implementer
+
+from twisted.python.compat import networkString
+
+
+class ISASLMechanism(Interface):
+    name = Attribute("""Common name for the SASL Mechanism.""")
+
+    def getInitialResponse():
+        """
+        Get the initial client response, if defined for this mechanism.
+
+        @return: initial client response string.
+        @rtype: C{str}.
+        """
+
+    def getResponse(challenge):
+        """
+        Get the response to a server challenge.
+
+        @param challenge: server challenge.
+        @type challenge: C{str}.
+        @return: client response.
+        @rtype: C{str}.
+        """
+
+
+@implementer(ISASLMechanism)
+class Anonymous:
+    """
+    Implements the ANONYMOUS SASL authentication mechanism.
+
+    This mechanism is defined in RFC 2245.
+    """
+
+    name = "ANONYMOUS"
+
+    def getInitialResponse(self):
+        return None
+
+    def getResponse(self, challenge):
+        # ISASLMechanism.getResponse
+        pass
+
+
+@implementer(ISASLMechanism)
+class Plain:
+    """
+    Implements the PLAIN SASL authentication mechanism.
+
+    The PLAIN SASL authentication mechanism is defined in RFC 2595.
+    """
+
+    name = "PLAIN"
+
+    def __init__(self, authzid, authcid, password):
+        """
+        @param authzid: The authorization identity.
+        @type authzid: L{unicode}
+
+        @param authcid: The authentication identity.
+        @type authcid: L{unicode}
+
+        @param password: The plain-text password.
+        @type password: L{unicode}
+        """
+
+        self.authzid = authzid or ""
+        self.authcid = authcid or ""
+        self.password = password or ""
+
+    def getInitialResponse(self):
+        return (
+            self.authzid.encode("utf-8")
+            + b"\x00"
+            + self.authcid.encode("utf-8")
+            + b"\x00"
+            + self.password.encode("utf-8")
+        )
+
+    def getResponse(self, challenge):
+        # ISASLMechanism.getResponse
+        pass
+
+
+@implementer(ISASLMechanism)
+class DigestMD5:
+    """
+    Implements the DIGEST-MD5 SASL authentication mechanism.
+
+    The DIGEST-MD5 SASL authentication mechanism is defined in RFC 2831.
+    """
+
+    name = "DIGEST-MD5"
+
+    def __init__(self, serv_type, host, serv_name, username, password):
+        """
+        @param serv_type: An indication of what kind of server authentication
+            is being attempted against.  For example, C{u"xmpp"}.
+        @type serv_type: C{unicode}
+
+        @param host: The authentication hostname.  Also known as the realm.
+            This is used as a scope to help select the right credentials.
+        @type host: C{unicode}
+
+        @param serv_name: An additional identifier for the server.
+        @type serv_name: C{unicode}
+
+        @param username: The authentication username to use to respond to a
+            challenge.
+        @type username: C{unicode}
+
+        @param password: The authentication password to use to respond to a
+            challenge.
+        @type password: C{unicode}
+        """
+        self.username = username
+        self.password = password
+        self.defaultRealm = host
+
+        self.digest_uri = f"{serv_type}/{host}"
+        if serv_name is not None:
+            self.digest_uri += f"/{serv_name}"
+
+    def getInitialResponse(self):
+        return None
+
+    def getResponse(self, challenge):
+        directives = self._parse(challenge)
+
+        # Compat for implementations that do not send this along with
+        # a successful authentication.
+        if b"rspauth" in directives:
+            return b""
+
+        charset = directives[b"charset"].decode("ascii")
+
+        try:
+            realm = directives[b"realm"]
+        except KeyError:
+            realm = self.defaultRealm.encode(charset)
+
+        return self._genResponse(charset, realm, directives[b"nonce"])
+
+    def _parse(self, challenge):
+        """
+        Parses the server challenge.
+
+        Splits the challenge into a dictionary of directives with values.
+
+        @return: challenge directives and their values.
+        @rtype: C{dict} of C{str} to C{str}.
+        """
+        s = challenge
+        paramDict = {}
+        cur = 0
+        remainingParams = True
+        while remainingParams:
+            # Parse a param. We can't just split on commas, because there can
+            # be some commas inside (quoted) param values, e.g.:
+            # qop="auth,auth-int"
+
+            middle = s.index(b"=", cur)
+            name = s[cur:middle].lstrip()
+            middle += 1
+            if s[middle : middle + 1] == b'"':
+                middle += 1
+                end = s.index(b'"', middle)
+                value = s[middle:end]
+                cur = s.find(b",", end) + 1
+                if cur == 0:
+                    remainingParams = False
+            else:
+                end = s.find(b",", middle)
+                if end == -1:
+                    value = s[middle:].rstrip()
+                    remainingParams = False
+                else:
+                    value = s[middle:end].rstrip()
+                cur = end + 1
+            paramDict[name] = value
+
+        for param in (b"qop", b"cipher"):
+            if param in paramDict:
+                paramDict[param] = paramDict[param].split(b",")
+
+        return paramDict
+
+    def _unparse(self, directives):
+        """
+        Create message string from directives.
+
+        @param directives: dictionary of directives (names to their values).
+                           For certain directives, extra quotes are added, as
+                           needed.
+        @type directives: C{dict} of C{str} to C{str}
+        @return: message string.
+        @rtype: C{str}.
+        """
+
+        directive_list = []
+        for name, value in directives.items():
+            if name in (
+                b"username",
+                b"realm",
+                b"cnonce",
+                b"nonce",
+                b"digest-uri",
+                b"authzid",
+                b"cipher",
+            ):
+                directive = name + b"=" + value
+            else:
+                directive = name + b"=" + value
+
+            directive_list.append(directive)
+
+        return b",".join(directive_list)
+
+    def _calculateResponse(self, cnonce, nc, nonce, username, password, realm, uri):
+        """
+        Calculates response with given encoded parameters.
+
+        @return: The I{response} field of a response to a Digest-MD5 challenge
+            of the given parameters.
+        @rtype: L{bytes}
+        """
+
+        def H(s):
+            return md5(s).digest()
+
+        def HEX(n):
+            return binascii.b2a_hex(n)
+
+        def KD(k, s):
+            return H(k + b":" + s)
+
+        a1 = H(username + b":" + realm + b":" + password) + b":" + nonce + b":" + cnonce
+        a2 = b"AUTHENTICATE:" + uri
+
+        response = HEX(
+            KD(
+                HEX(H(a1)),
+                nonce + b":" + nc + b":" + cnonce + b":" + b"auth" + b":" + HEX(H(a2)),
+            )
+        )
+        return response
+
+    def _genResponse(self, charset, realm, nonce):
+        """
+        Generate response-value.
+
+        Creates a response to a challenge according to section 2.1.2.1 of
+        RFC 2831 using the C{charset}, C{realm} and C{nonce} directives
+        from the challenge.
+        """
+        try:
+            username = self.username.encode(charset)
+            password = self.password.encode(charset)
+            digest_uri = self.digest_uri.encode(charset)
+        except UnicodeError:
+            # TODO - add error checking
+            raise
+
+        nc = networkString(f"{1:08x}")  # TODO: support subsequent auth.
+        cnonce = self._gen_nonce()
+        qop = b"auth"
+
+        # TODO - add support for authzid
+        response = self._calculateResponse(
+            cnonce, nc, nonce, username, password, realm, digest_uri
+        )
+
+        directives = {
+            b"username": username,
+            b"realm": realm,
+            b"nonce": nonce,
+            b"cnonce": cnonce,
+            b"nc": nc,
+            b"qop": qop,
+            b"digest-uri": digest_uri,
+            b"response": response,
+            b"charset": charset.encode("ascii"),
+        }
+
+        return self._unparse(directives)
+
+    def _gen_nonce(self):
+        nonceString = "%f:%f:%d" % (random.random(), time.time(), os.getpid())
+        nonceBytes = networkString(nonceString)
+        return md5(nonceBytes).hexdigest().encode("ascii")
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/xmlstream.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/xmlstream.py
new file mode 100644
index 00000000000..601a879aa8b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/xmlstream.py
@@ -0,0 +1,1145 @@
+# -*- test-case-name: twisted.words.test.test_jabberxmlstream -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XMPP XML Streams
+
+Building blocks for setting up XML Streams, including helping classes for
+doing authentication on either client or server side, and working with XML
+Stanzas.
+
+@var STREAM_AUTHD_EVENT: Token dispatched by L{Authenticator} when the
+    stream has been completely initialized
+@type STREAM_AUTHD_EVENT: L{str}.
+
+@var INIT_FAILED_EVENT: Token dispatched by L{Authenticator} when the
+    stream has failed to be initialized
+@type INIT_FAILED_EVENT: L{str}.
+
+@var Reset: Token to signal that the XML stream has been reset.
+@type Reset: Basic object.
+"""
+
+
+from binascii import hexlify
+from hashlib import sha1
+from sys import intern
+from typing import Optional, Tuple
+
+from zope.interface import directlyProvides, implementer
+
+from twisted.internet import defer, protocol
+from twisted.internet.error import ConnectionLost
+from twisted.python import failure, log, randbytes
+from twisted.words.protocols.jabber import error, ijabber, jid
+from twisted.words.xish import domish, xmlstream
+from twisted.words.xish.xmlstream import (
+    STREAM_CONNECTED_EVENT,
+    STREAM_END_EVENT,
+    STREAM_ERROR_EVENT,
+    STREAM_START_EVENT,
+)
+
+try:
+    from twisted.internet import ssl as _ssl
+except ImportError:
+    ssl = None
+else:
+    if not _ssl.supported:
+        ssl = None
+    else:
+        ssl = _ssl
+
+STREAM_AUTHD_EVENT = intern("//event/stream/authd")
+INIT_FAILED_EVENT = intern("//event/xmpp/initfailed")
+
+NS_STREAMS = "http://etherx.jabber.org/streams"
+NS_XMPP_TLS = "urn:ietf:params:xml:ns:xmpp-tls"
+
+Reset = object()
+
+
+def hashPassword(sid, password):
+    """
+    Create a SHA1-digest string of a session identifier and password.
+
+    @param sid: The stream session identifier.
+    @type sid: C{unicode}.
+    @param password: The password to be hashed.
+    @type password: C{unicode}.
+    """
+    if not isinstance(sid, str):
+        raise TypeError("The session identifier must be a unicode object")
+    if not isinstance(password, str):
+        raise TypeError("The password must be a unicode object")
+    input = f"{sid}{password}"
+    return sha1(input.encode("utf-8")).hexdigest()
+
+
+class Authenticator:
+    """
+    Base class for business logic of initializing an XmlStream
+
+    Subclass this object to enable an XmlStream to initialize and authenticate
+    to different types of stream hosts (such as clients, components, etc.).
+
+    Rules:
+      1. The Authenticator MUST dispatch a L{STREAM_AUTHD_EVENT} when the
+         stream has been completely initialized.
+      2. The Authenticator SHOULD reset all state information when
+         L{associateWithStream} is called.
+      3. The Authenticator SHOULD override L{streamStarted}, and start
+         initialization there.
+
+    @type xmlstream: L{XmlStream}
+    @ivar xmlstream: The XmlStream that needs authentication
+
+    @note: the term authenticator is historical. Authenticators perform
+           all steps required to prepare the stream for the exchange
+           of XML stanzas.
+    """
+
+    def __init__(self):
+        self.xmlstream = None
+
+    def connectionMade(self):
+        """
+        Called by the XmlStream when the underlying socket connection is
+        in place.
+
+        This allows the Authenticator to send an initial root element, if it's
+        connecting, or wait for an inbound root from the peer if it's accepting
+        the connection.
+
+        Subclasses can use self.xmlstream.send() to send any initial data to
+        the peer.
+        """
+
+    def streamStarted(self, rootElement):
+        """
+        Called by the XmlStream when the stream has started.
+
+        A stream is considered to have started when the start tag of the root
+        element has been received.
+
+        This examines C{rootElement} to see if there is a version attribute.
+        If absent, C{0.0} is assumed per RFC 3920. Subsequently, the
+        minimum of the version from the received stream header and the
+        value stored in L{xmlstream} is taken and put back in L{xmlstream}.
+
+        Extensions of this method can extract more information from the
+        stream header and perform checks on them, optionally sending
+        stream errors and closing the stream.
+        """
+        if rootElement.hasAttribute("version"):
+            version = rootElement["version"].split(".")
+            try:
+                version = (int(version[0]), int(version[1]))
+            except (IndexError, ValueError):
+                version = (0, 0)
+        else:
+            version = (0, 0)
+
+        self.xmlstream.version = min(self.xmlstream.version, version)
+
+    def associateWithStream(self, xmlstream):
+        """
+        Called by the XmlStreamFactory when a connection has been made
+        to the requested peer, and an XmlStream object has been
+        instantiated.
+
+        The default implementation just saves a handle to the new
+        XmlStream.
+
+        @type xmlstream: L{XmlStream}
+        @param xmlstream: The XmlStream that will be passing events to this
+                          Authenticator.
+
+        """
+        self.xmlstream = xmlstream
+
+
+class ConnectAuthenticator(Authenticator):
+    """
+    Authenticator for initiating entities.
+    """
+
+    namespace: Optional[str] = None
+
+    def __init__(self, otherHost):
+        self.otherHost = otherHost
+
+    def connectionMade(self):
+        self.xmlstream.namespace = self.namespace
+        self.xmlstream.otherEntity = jid.internJID(self.otherHost)
+        self.xmlstream.sendHeader()
+
+    def initializeStream(self):
+        """
+        Perform stream initialization procedures.
+
+        An L{XmlStream} holds a list of initializer objects in its
+        C{initializers} attribute. This method calls these initializers in
+        order and dispatches the L{STREAM_AUTHD_EVENT} event when the list has
+        been successfully processed. Otherwise it dispatches the
+        C{INIT_FAILED_EVENT} event with the failure.
+
+        Initializers may return the special L{Reset} object to halt the
+        initialization processing. It signals that the current initializer was
+        successfully processed, but that the XML Stream has been reset. An
+        example is the TLSInitiatingInitializer.
+        """
+
+        def remove_first(result):
+            self.xmlstream.initializers.pop(0)
+
+            return result
+
+        def do_next(result):
+            """
+            Take the first initializer and process it.
+
+            On success, the initializer is removed from the list and
+            then next initializer will be tried.
+            """
+
+            if result is Reset:
+                return None
+
+            try:
+                init = self.xmlstream.initializers[0]
+            except IndexError:
+                self.xmlstream.dispatch(self.xmlstream, STREAM_AUTHD_EVENT)
+                return None
+            else:
+                d = defer.maybeDeferred(init.initialize)
+                d.addCallback(remove_first)
+                d.addCallback(do_next)
+                return d
+
+        d = defer.succeed(None)
+        d.addCallback(do_next)
+        d.addErrback(self.xmlstream.dispatch, INIT_FAILED_EVENT)
+
+    def streamStarted(self, rootElement):
+        """
+        Called by the XmlStream when the stream has started.
+
+        This extends L{Authenticator.streamStarted} to extract further stream
+        headers from C{rootElement}, optionally wait for stream features being
+        received and then call C{initializeStream}.
+        """
+
+        Authenticator.streamStarted(self, rootElement)
+
+        self.xmlstream.sid = rootElement.getAttribute("id")
+
+        if rootElement.hasAttribute("from"):
+            self.xmlstream.otherEntity = jid.internJID(rootElement["from"])
+
+        # Setup observer for stream features, if applicable
+        if self.xmlstream.version >= (1, 0):
+
+            def onFeatures(element):
+                features = {}
+                for feature in element.elements():
+                    features[(feature.uri, feature.name)] = feature
+
+                self.xmlstream.features = features
+                self.initializeStream()
+
+            self.xmlstream.addOnetimeObserver(
+                '/features[@xmlns="%s"]' % NS_STREAMS, onFeatures
+            )
+        else:
+            self.initializeStream()
+
+
+class ListenAuthenticator(Authenticator):
+    """
+    Authenticator for receiving entities.
+    """
+
+    namespace: Optional[str] = None
+
+    def associateWithStream(self, xmlstream):
+        """
+        Called by the XmlStreamFactory when a connection has been made.
+
+        Extend L{Authenticator.associateWithStream} to set the L{XmlStream}
+        to be non-initiating.
+        """
+        Authenticator.associateWithStream(self, xmlstream)
+        self.xmlstream.initiating = False
+
+    def streamStarted(self, rootElement):
+        """
+        Called by the XmlStream when the stream has started.
+
+        This extends L{Authenticator.streamStarted} to extract further
+        information from the stream headers from C{rootElement}.
+        """
+        Authenticator.streamStarted(self, rootElement)
+
+        self.xmlstream.namespace = rootElement.defaultUri
+
+        if rootElement.hasAttribute("to"):
+            self.xmlstream.thisEntity = jid.internJID(rootElement["to"])
+
+        self.xmlstream.prefixes = {}
+        for prefix, uri in rootElement.localPrefixes.items():
+            self.xmlstream.prefixes[uri] = prefix
+
+        self.xmlstream.sid = hexlify(randbytes.secureRandom(8)).decode("ascii")
+
+
+class FeatureNotAdvertized(Exception):
+    """
+    Exception indicating a stream feature was not advertized, while required by
+    the initiating entity.
+    """
+
+
+@implementer(ijabber.IInitiatingInitializer)
+class BaseFeatureInitiatingInitializer:
+    """
+    Base class for initializers with a stream feature.
+
+    This assumes the associated XmlStream represents the initiating entity
+    of the connection.
+
+    @cvar feature: tuple of (uri, name) of the stream feature root element.
+    @type feature: tuple of (C{str}, C{str})
+
+    @ivar required: whether the stream feature is required to be advertized
+                    by the receiving entity.
+    @type required: C{bool}
+    """
+
+    feature: Optional[Tuple[str, str]] = None
+
+    def __init__(self, xs, required=False):
+        self.xmlstream = xs
+        self.required = required
+
+    def initialize(self):
+        """
+        Initiate the initialization.
+
+        Checks if the receiving entity advertizes the stream feature. If it
+        does, the initialization is started. If it is not advertized, and the
+        C{required} instance variable is C{True}, it raises
+        L{FeatureNotAdvertized}. Otherwise, the initialization silently
+        succeeds.
+        """
+
+        if self.feature in self.xmlstream.features:
+            return self.start()
+        elif self.required:
+            raise FeatureNotAdvertized
+        else:
+            return None
+
+    def start(self):
+        """
+        Start the actual initialization.
+
+        May return a deferred for asynchronous initialization.
+        """
+
+
+class TLSError(Exception):
+    """
+    TLS base exception.
+    """
+
+
+class TLSFailed(TLSError):
+    """
+    Exception indicating failed TLS negotiation
+    """
+
+
+class TLSRequired(TLSError):
+    """
+    Exception indicating required TLS negotiation.
+
+    This exception is raised when the receiving entity requires TLS
+    negotiation and the initiating does not desire to negotiate TLS.
+    """
+
+
+class TLSNotSupported(TLSError):
+    """
+    Exception indicating missing TLS support.
+
+    This exception is raised when the initiating entity wants and requires to
+    negotiate TLS when the OpenSSL library is not available.
+    """
+
+
+class TLSInitiatingInitializer(BaseFeatureInitiatingInitializer):
+    """
+    TLS stream initializer for the initiating entity.
+
+    It is strongly required to include this initializer in the list of
+    initializers for an XMPP stream. By default it will try to negotiate TLS.
+    An XMPP server may indicate that TLS is required. If TLS is not desired,
+    set the C{wanted} attribute to False instead of removing it from the list
+    of initializers, so a proper exception L{TLSRequired} can be raised.
+
+    @ivar wanted: indicates if TLS negotiation is wanted.
+    @type wanted: C{bool}
+    """
+
+    feature = (NS_XMPP_TLS, "starttls")
+    wanted = True
+    _deferred = None
+    _configurationForTLS = None
+
+    def __init__(self, xs, required=True, configurationForTLS=None):
+        """
+        @param configurationForTLS: An object which creates appropriately
+            configured TLS connections. This is passed to C{startTLS} on the
+            transport and is preferably created using
+            L{twisted.internet.ssl.optionsForClientTLS}.  If C{None}, the
+            default is to verify the server certificate against the trust roots
+            as provided by the platform. See
+            L{twisted.internet._sslverify.platformTrust}.
+        @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or
+            C{None}
+        """
+        super().__init__(xs, required=required)
+        self._configurationForTLS = configurationForTLS
+
+    def onProceed(self, obj):
+        """
+        Proceed with TLS negotiation and reset the XML stream.
+        """
+
+        self.xmlstream.removeObserver("/failure", self.onFailure)
+        if self._configurationForTLS:
+            ctx = self._configurationForTLS
+        else:
+            ctx = ssl.optionsForClientTLS(self.xmlstream.otherEntity.host)
+        self.xmlstream.transport.startTLS(ctx)
+        self.xmlstream.reset()
+        self.xmlstream.sendHeader()
+        self._deferred.callback(Reset)
+
+    def onFailure(self, obj):
+        self.xmlstream.removeObserver("/proceed", self.onProceed)
+        self._deferred.errback(TLSFailed())
+
+    def start(self):
+        """
+        Start TLS negotiation.
+
+        This checks if the receiving entity requires TLS, the SSL library is
+        available and uses the C{required} and C{wanted} instance variables to
+        determine what to do in the various different cases.
+
+        For example, if the SSL library is not available, and wanted and
+        required by the user, it raises an exception. However if it is not
+        required by both parties, initialization silently succeeds, moving
+        on to the next step.
+        """
+        if self.wanted:
+            if ssl is None:
+                if self.required:
+                    return defer.fail(TLSNotSupported())
+                else:
+                    return defer.succeed(None)
+            else:
+                pass
+        elif self.xmlstream.features[self.feature].required:
+            return defer.fail(TLSRequired())
+        else:
+            return defer.succeed(None)
+
+        self._deferred = defer.Deferred()
+        self.xmlstream.addOnetimeObserver("/proceed", self.onProceed)
+        self.xmlstream.addOnetimeObserver("/failure", self.onFailure)
+        self.xmlstream.send(domish.Element((NS_XMPP_TLS, "starttls")))
+        return self._deferred
+
+
+class XmlStream(xmlstream.XmlStream):
+    """
+    XMPP XML Stream protocol handler.
+
+    @ivar version: XML stream version as a tuple (major, minor). Initially,
+                   this is set to the minimally supported version. Upon
+                   receiving the stream header of the peer, it is set to the
+                   minimum of that value and the version on the received
+                   header.
+    @type version: (C{int}, C{int})
+    @ivar namespace: default namespace URI for stream
+    @type namespace: C{unicode}
+    @ivar thisEntity: JID of this entity
+    @type thisEntity: L{JID}
+    @ivar otherEntity: JID of the peer entity
+    @type otherEntity: L{JID}
+    @ivar sid: session identifier
+    @type sid: C{unicode}
+    @ivar initiating: True if this is the initiating stream
+    @type initiating: C{bool}
+    @ivar features: map of (uri, name) to stream features element received from
+                    the receiving entity.
+    @type features: C{dict} of (C{unicode}, C{unicode}) to L{domish.Element}.
+    @ivar prefixes: map of URI to prefixes that are to appear on stream
+                    header.
+    @type prefixes: C{dict} of C{unicode} to C{unicode}
+    @ivar initializers: list of stream initializer objects
+    @type initializers: C{list} of objects that provide L{IInitializer}
+    @ivar authenticator: associated authenticator that uses C{initializers} to
+                         initialize the XML stream.
+    """
+
+    version = (1, 0)
+    namespace = "invalid"
+    thisEntity = None
+    otherEntity = None
+    sid = None
+    initiating = True
+
+    _headerSent = False  # True if the stream header has been sent
+
+    def __init__(self, authenticator):
+        xmlstream.XmlStream.__init__(self)
+
+        self.prefixes = {NS_STREAMS: "stream"}
+        self.authenticator = authenticator
+        self.initializers = []
+        self.features = {}
+
+        # Reset the authenticator
+        authenticator.associateWithStream(self)
+
+    def _callLater(self, *args, **kwargs):
+        from twisted.internet import reactor
+
+        return reactor.callLater(*args, **kwargs)
+
+    def reset(self):
+        """
+        Reset XML Stream.
+
+        Resets the XML Parser for incoming data. This is to be used after
+        successfully negotiating a new layer, e.g. TLS and SASL. Note that
+        registered event observers will continue to be in place.
+        """
+        self._headerSent = False
+        self._initializeStream()
+
+    def onStreamError(self, errelem):
+        """
+        Called when a stream:error element has been received.
+
+        Dispatches a L{STREAM_ERROR_EVENT} event with the error element to
+        allow for cleanup actions and drops the connection.
+
+        @param errelem: The received error element.
+        @type errelem: L{domish.Element}
+        """
+        self.dispatch(
+            failure.Failure(error.exceptionFromStreamError(errelem)), STREAM_ERROR_EVENT
+        )
+        self.transport.loseConnection()
+
+    def sendHeader(self):
+        """
+        Send stream header.
+        """
+        # set up optional extra namespaces
+        localPrefixes = {}
+        for uri, prefix in self.prefixes.items():
+            if uri != NS_STREAMS:
+                localPrefixes[prefix] = uri
+
+        rootElement = domish.Element(
+            (NS_STREAMS, "stream"), self.namespace, localPrefixes=localPrefixes
+        )
+
+        if self.otherEntity:
+            rootElement["to"] = self.otherEntity.userhost()
+
+        if self.thisEntity:
+            rootElement["from"] = self.thisEntity.userhost()
+
+        if not self.initiating and self.sid:
+            rootElement["id"] = self.sid
+
+        if self.version >= (1, 0):
+            rootElement["version"] = "%d.%d" % self.version
+
+        self.send(rootElement.toXml(prefixes=self.prefixes, closeElement=0))
+        self._headerSent = True
+
+    def sendFooter(self):
+        """
+        Send stream footer.
+        """
+        self.send("</stream:stream>")
+
+    def sendStreamError(self, streamError):
+        """
+        Send stream level error.
+
+        If we are the receiving entity, and haven't sent the header yet,
+        we sent one first.
+
+        After sending the stream error, the stream is closed and the transport
+        connection dropped.
+
+        @param streamError: stream error instance
+        @type streamError: L{error.StreamError}
+        """
+        if not self._headerSent and not self.initiating:
+            self.sendHeader()
+
+        if self._headerSent:
+            self.send(streamError.getElement())
+            self.sendFooter()
+
+        self.transport.loseConnection()
+
+    def send(self, obj):
+        """
+        Send data over the stream.
+
+        This overrides L{xmlstream.XmlStream.send} to use the default namespace
+        of the stream header when serializing L{domish.IElement}s. It is
+        assumed that if you pass an object that provides L{domish.IElement},
+        it represents a direct child of the stream's root element.
+        """
+        if domish.IElement.providedBy(obj):
+            obj = obj.toXml(
+                prefixes=self.prefixes,
+                defaultUri=self.namespace,
+                prefixesInScope=list(self.prefixes.values()),
+            )
+
+        xmlstream.XmlStream.send(self, obj)
+
+    def connectionMade(self):
+        """
+        Called when a connection is made.
+
+        Notifies the authenticator when a connection has been made.
+        """
+        xmlstream.XmlStream.connectionMade(self)
+        self.authenticator.connectionMade()
+
+    def onDocumentStart(self, rootElement):
+        """
+        Called when the stream header has been received.
+
+        Extracts the header's C{id} and C{version} attributes from the root
+        element. The C{id} attribute is stored in our C{sid} attribute and the
+        C{version} attribute is parsed and the minimum of the version we sent
+        and the parsed C{version} attribute is stored as a tuple (major, minor)
+        in this class' C{version} attribute. If no C{version} attribute was
+        present, we assume version 0.0.
+
+        If appropriate (we are the initiating stream and the minimum of our and
+        the other party's version is at least 1.0), a one-time observer is
+        registered for getting the stream features. The registered function is
+        C{onFeatures}.
+
+        Ultimately, the authenticator's C{streamStarted} method will be called.
+
+        @param rootElement: The root element.
+        @type rootElement: L{domish.Element}
+        """
+        xmlstream.XmlStream.onDocumentStart(self, rootElement)
+
+        # Setup observer for stream errors
+        self.addOnetimeObserver("/error[@xmlns='%s']" % NS_STREAMS, self.onStreamError)
+
+        self.authenticator.streamStarted(rootElement)
+
+
+class XmlStreamFactory(xmlstream.XmlStreamFactory):
+    """
+    Factory for Jabber XmlStream objects as a reconnecting client.
+
+    Note that this differs from L{xmlstream.XmlStreamFactory} in that
+    it generates Jabber specific L{XmlStream} instances that have
+    authenticators.
+    """
+
+    protocol = XmlStream
+
+    def __init__(self, authenticator):
+        xmlstream.XmlStreamFactory.__init__(self, authenticator)
+        self.authenticator = authenticator
+
+
+class XmlStreamServerFactory(xmlstream.BootstrapMixin, protocol.ServerFactory):
+    """
+    Factory for Jabber XmlStream objects as a server.
+
+    @since: 8.2.
+    @ivar authenticatorFactory: Factory callable that takes no arguments, to
+                                create a fresh authenticator to be associated
+                                with the XmlStream.
+    """
+
+    # Type is wrong.  See: https://twistedmatrix.com/trac/ticket/10007#ticket
+    protocol = XmlStream  # type: ignore[assignment]
+
+    def __init__(self, authenticatorFactory):
+        xmlstream.BootstrapMixin.__init__(self)
+        self.authenticatorFactory = authenticatorFactory
+
+    def buildProtocol(self, addr):
+        """
+        Create an instance of XmlStream.
+
+        A new authenticator instance will be created and passed to the new
+        XmlStream. Registered bootstrap event observers are installed as well.
+        """
+        authenticator = self.authenticatorFactory()
+        xs = self.protocol(authenticator)
+        xs.factory = self
+        self.installBootstraps(xs)
+        return xs
+
+
+class TimeoutError(Exception):
+    """
+    Exception raised when no IQ response has been received before the
+    configured timeout.
+    """
+
+
+def upgradeWithIQResponseTracker(xs):
+    """
+    Enhances an XmlStream for iq response tracking.
+
+    This makes an L{XmlStream} object provide L{IIQResponseTracker}. When a
+    response is an error iq stanza, the deferred has its errback invoked with a
+    failure that holds a L{StanzaError<error.StanzaError>} that is
+    easier to examine.
+    """
+
+    def callback(iq):
+        """
+        Handle iq response by firing associated deferred.
+        """
+        if getattr(iq, "handled", False):
+            return
+
+        try:
+            d = xs.iqDeferreds[iq["id"]]
+        except KeyError:
+            pass
+        else:
+            del xs.iqDeferreds[iq["id"]]
+            iq.handled = True
+            if iq["type"] == "error":
+                d.errback(error.exceptionFromStanza(iq))
+            else:
+                d.callback(iq)
+
+    def disconnected(_):
+        """
+        Make sure deferreds do not linger on after disconnect.
+
+        This errbacks all deferreds of iq's for which no response has been
+        received with a L{ConnectionLost} failure. Otherwise, the deferreds
+        will never be fired.
+        """
+        iqDeferreds = xs.iqDeferreds
+        xs.iqDeferreds = {}
+        for d in iqDeferreds.values():
+            d.errback(ConnectionLost())
+
+    xs.iqDeferreds = {}
+    xs.iqDefaultTimeout = getattr(xs, "iqDefaultTimeout", None)
+    xs.addObserver(xmlstream.STREAM_END_EVENT, disconnected)
+    xs.addObserver('/iq[@type="result"]', callback)
+    xs.addObserver('/iq[@type="error"]', callback)
+    directlyProvides(xs, ijabber.IIQResponseTracker)
+
+
+class IQ(domish.Element):
+    """
+    Wrapper for an iq stanza.
+
+    Iq stanzas are used for communications with a request-response behaviour.
+    Each iq request is associated with an XML stream and has its own unique id
+    to be able to track the response.
+
+    @ivar timeout: if set, a timeout period after which the deferred returned
+                   by C{send} will have its errback called with a
+                   L{TimeoutError} failure.
+    @type timeout: C{float}
+    """
+
+    timeout = None
+
+    def __init__(self, xmlstream, stanzaType="set"):
+        """
+        @type xmlstream: L{xmlstream.XmlStream}
+        @param xmlstream: XmlStream to use for transmission of this IQ
+
+        @type stanzaType: C{str}
+        @param stanzaType: IQ type identifier ('get' or 'set')
+        """
+        domish.Element.__init__(self, (None, "iq"))
+        self.addUniqueId()
+        self["type"] = stanzaType
+        self._xmlstream = xmlstream
+
+    def send(self, to=None):
+        """
+        Send out this iq.
+
+        Returns a deferred that is fired when an iq response with the same id
+        is received. Result responses will be passed to the deferred callback.
+        Error responses will be transformed into a
+        L{StanzaError<error.StanzaError>} and result in the errback of the
+        deferred being invoked.
+
+        @rtype: L{defer.Deferred}
+        """
+        if to is not None:
+            self["to"] = to
+
+        if not ijabber.IIQResponseTracker.providedBy(self._xmlstream):
+            upgradeWithIQResponseTracker(self._xmlstream)
+
+        d = defer.Deferred()
+        self._xmlstream.iqDeferreds[self["id"]] = d
+
+        timeout = self.timeout or self._xmlstream.iqDefaultTimeout
+        if timeout is not None:
+
+            def onTimeout():
+                del self._xmlstream.iqDeferreds[self["id"]]
+                d.errback(TimeoutError("IQ timed out"))
+
+            call = self._xmlstream._callLater(timeout, onTimeout)
+
+            def cancelTimeout(result):
+                if call.active():
+                    call.cancel()
+
+                return result
+
+            d.addBoth(cancelTimeout)
+
+        self._xmlstream.send(self)
+        return d
+
+
+def toResponse(stanza, stanzaType=None):
+    """
+    Create a response stanza from another stanza.
+
+    This takes the addressing and id attributes from a stanza to create a (new,
+    empty) response stanza. The addressing attributes are swapped and the id
+    copied. Optionally, the stanza type of the response can be specified.
+
+    @param stanza: the original stanza
+    @type stanza: L{domish.Element}
+    @param stanzaType: optional response stanza type
+    @type stanzaType: C{str}
+    @return: the response stanza.
+    @rtype: L{domish.Element}
+    """
+
+    toAddr = stanza.getAttribute("from")
+    fromAddr = stanza.getAttribute("to")
+    stanzaID = stanza.getAttribute("id")
+
+    response = domish.Element((None, stanza.name))
+    if toAddr:
+        response["to"] = toAddr
+    if fromAddr:
+        response["from"] = fromAddr
+    if stanzaID:
+        response["id"] = stanzaID
+    if stanzaType:
+        response["type"] = stanzaType
+
+    return response
+
+
+@implementer(ijabber.IXMPPHandler)
+class XMPPHandler:
+    """
+    XMPP protocol handler.
+
+    Classes derived from this class implement (part of) one or more XMPP
+    extension protocols, and are referred to as a subprotocol implementation.
+    """
+
+    def __init__(self):
+        self.parent = None
+        self.xmlstream = None
+
+    def setHandlerParent(self, parent):
+        self.parent = parent
+        self.parent.addHandler(self)
+
+    def disownHandlerParent(self, parent):
+        self.parent.removeHandler(self)
+        self.parent = None
+
+    def makeConnection(self, xs):
+        self.xmlstream = xs
+        self.connectionMade()
+
+    def connectionMade(self):
+        """
+        Called after a connection has been established.
+
+        Can be overridden to perform work before stream initialization.
+        """
+
+    def connectionInitialized(self):
+        """
+        The XML stream has been initialized.
+
+        Can be overridden to perform work after stream initialization, e.g. to
+        set up observers and start exchanging XML stanzas.
+        """
+
+    def connectionLost(self, reason):
+        """
+        The XML stream has been closed.
+
+        This method can be extended to inspect the C{reason} argument and
+        act on it.
+        """
+        self.xmlstream = None
+
+    def send(self, obj):
+        """
+        Send data over the managed XML stream.
+
+        @note: The stream manager maintains a queue for data sent using this
+               method when there is no current initialized XML stream. This
+               data is then sent as soon as a new stream has been established
+               and initialized. Subsequently, L{connectionInitialized} will be
+               called again. If this queueing is not desired, use C{send} on
+               C{self.xmlstream}.
+
+        @param obj: data to be sent over the XML stream. This is usually an
+                    object providing L{domish.IElement}, or serialized XML. See
+                    L{xmlstream.XmlStream} for details.
+        """
+        self.parent.send(obj)
+
+
+@implementer(ijabber.IXMPPHandlerCollection)
+class XMPPHandlerCollection:
+    """
+    Collection of XMPP subprotocol handlers.
+
+    This allows for grouping of subprotocol handlers, but is not an
+    L{XMPPHandler} itself, so this is not recursive.
+
+    @ivar handlers: List of protocol handlers.
+    @type handlers: C{list} of objects providing
+                      L{IXMPPHandler}
+    """
+
+    def __init__(self):
+        self.handlers = []
+
+    def __iter__(self):
+        """
+        Act as a container for handlers.
+        """
+        return iter(self.handlers)
+
+    def addHandler(self, handler):
+        """
+        Add protocol handler.
+
+        Protocol handlers are expected to provide L{ijabber.IXMPPHandler}.
+        """
+        self.handlers.append(handler)
+
+    def removeHandler(self, handler):
+        """
+        Remove protocol handler.
+        """
+        self.handlers.remove(handler)
+
+
+class StreamManager(XMPPHandlerCollection):
+    """
+    Business logic representing a managed XMPP connection.
+
+    This maintains a single XMPP connection and provides facilities for packet
+    routing and transmission. Business logic modules are objects providing
+    L{ijabber.IXMPPHandler} (like subclasses of L{XMPPHandler}), and added
+    using L{addHandler}.
+
+    @ivar xmlstream: currently managed XML stream
+    @type xmlstream: L{XmlStream}
+    @ivar logTraffic: if true, log all traffic.
+    @type logTraffic: C{bool}
+    @ivar _initialized: Whether the stream represented by L{xmlstream} has
+                        been initialized. This is used when caching outgoing
+                        stanzas.
+    @type _initialized: C{bool}
+    @ivar _packetQueue: internal buffer of unsent data. See L{send} for details.
+    @type _packetQueue: C{list}
+    """
+
+    logTraffic = False
+
+    def __init__(self, factory):
+        XMPPHandlerCollection.__init__(self)
+        self.xmlstream = None
+        self._packetQueue = []
+        self._initialized = False
+
+        factory.addBootstrap(STREAM_CONNECTED_EVENT, self._connected)
+        factory.addBootstrap(STREAM_AUTHD_EVENT, self._authd)
+        factory.addBootstrap(INIT_FAILED_EVENT, self.initializationFailed)
+        factory.addBootstrap(STREAM_END_EVENT, self._disconnected)
+        self.factory = factory
+
+    def addHandler(self, handler):
+        """
+        Add protocol handler.
+
+        When an XML stream has already been established, the handler's
+        C{connectionInitialized} will be called to get it up to speed.
+        """
+        XMPPHandlerCollection.addHandler(self, handler)
+
+        # get protocol handler up to speed when a connection has already
+        # been established
+        if self.xmlstream and self._initialized:
+            handler.makeConnection(self.xmlstream)
+            handler.connectionInitialized()
+
+    def _connected(self, xs):
+        """
+        Called when the transport connection has been established.
+
+        Here we optionally set up traffic logging (depending on L{logTraffic})
+        and call each handler's C{makeConnection} method with the L{XmlStream}
+        instance.
+        """
+
+        def logDataIn(buf):
+            log.msg("RECV: %r" % buf)
+
+        def logDataOut(buf):
+            log.msg("SEND: %r" % buf)
+
+        if self.logTraffic:
+            xs.rawDataInFn = logDataIn
+            xs.rawDataOutFn = logDataOut
+
+        self.xmlstream = xs
+
+        for e in self:
+            e.makeConnection(xs)
+
+    def _authd(self, xs):
+        """
+        Called when the stream has been initialized.
+
+        Send out cached stanzas and call each handler's
+        C{connectionInitialized} method.
+        """
+        # Flush all pending packets
+        for p in self._packetQueue:
+            xs.send(p)
+        self._packetQueue = []
+        self._initialized = True
+
+        # Notify all child services which implement
+        # the IService interface
+        for e in self:
+            e.connectionInitialized()
+
+    def initializationFailed(self, reason):
+        """
+        Called when stream initialization has failed.
+
+        Stream initialization has halted, with the reason indicated by
+        C{reason}. It may be retried by calling the authenticator's
+        C{initializeStream}. See the respective authenticators for details.
+
+        @param reason: A failure instance indicating why stream initialization
+                       failed.
+        @type reason: L{failure.Failure}
+        """
+
+    def _disconnected(self, reason):
+        """
+        Called when the stream has been closed.
+
+        From this point on, the manager doesn't interact with the
+        L{XmlStream} anymore and notifies each handler that the connection
+        was lost by calling its C{connectionLost} method.
+        """
+        self.xmlstream = None
+        self._initialized = False
+
+        # Notify all child services which implement
+        # the IService interface
+        for e in self:
+            e.connectionLost(reason)
+
+    def send(self, obj):
+        """
+        Send data over the XML stream.
+
+        When there is no established XML stream, the data is queued and sent
+        out when a new XML stream has been established and initialized.
+
+        @param obj: data to be sent over the XML stream. See
+                    L{xmlstream.XmlStream.send} for details.
+        """
+        if self._initialized:
+            self.xmlstream.send(obj)
+        else:
+            self._packetQueue.append(obj)
+
+
+__all__ = [
+    "Authenticator",
+    "BaseFeatureInitiatingInitializer",
+    "ConnectAuthenticator",
+    "FeatureNotAdvertized",
+    "INIT_FAILED_EVENT",
+    "IQ",
+    "ListenAuthenticator",
+    "NS_STREAMS",
+    "NS_XMPP_TLS",
+    "Reset",
+    "STREAM_AUTHD_EVENT",
+    "STREAM_CONNECTED_EVENT",
+    "STREAM_END_EVENT",
+    "STREAM_ERROR_EVENT",
+    "STREAM_START_EVENT",
+    "StreamManager",
+    "TLSError",
+    "TLSFailed",
+    "TLSInitiatingInitializer",
+    "TLSNotSupported",
+    "TLSRequired",
+    "TimeoutError",
+    "XMPPHandler",
+    "XMPPHandlerCollection",
+    "XmlStream",
+    "XmlStreamFactory",
+    "XmlStreamServerFactory",
+    "hashPassword",
+    "toResponse",
+    "upgradeWithIQResponseTracker",
+]
diff --git a/contrib/python/Twisted/py3/twisted/words/protocols/jabber/xmpp_stringprep.py b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/xmpp_stringprep.py
new file mode 100644
index 00000000000..4ffafa70609
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/protocols/jabber/xmpp_stringprep.py
@@ -0,0 +1,257 @@
+# -*- test-case-name: twisted.words.test.test_jabberxmppstringprep -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+import stringprep
+from encodings import idna
+from itertools import chain
+
+# We require Unicode version 3.2.
+from unicodedata import ucd_3_2_0 as unicodedata
+
+from zope.interface import Interface, implementer
+
+from incremental import Version
+
+from twisted.python.deprecate import deprecatedModuleAttribute
+
+crippled = False
+deprecatedModuleAttribute(
+    Version("Twisted", 13, 1, 0), "crippled is always False", __name__, "crippled"
+)
+
+
+class ILookupTable(Interface):
+    """
+    Interface for character lookup classes.
+    """
+
+    def lookup(c):
+        """
+        Return whether character is in this table.
+        """
+
+
+class IMappingTable(Interface):
+    """
+    Interface for character mapping classes.
+    """
+
+    def map(c):
+        """
+        Return mapping for character.
+        """
+
+
+@implementer(ILookupTable)
+class LookupTableFromFunction:
+    def __init__(self, in_table_function):
+        self.lookup = in_table_function
+
+
+@implementer(ILookupTable)
+class LookupTable:
+    def __init__(self, table):
+        self._table = table
+
+    def lookup(self, c):
+        return c in self._table
+
+
+@implementer(IMappingTable)
+class MappingTableFromFunction:
+    def __init__(self, map_table_function):
+        self.map = map_table_function
+
+
+@implementer(IMappingTable)
+class EmptyMappingTable:
+    def __init__(self, in_table_function):
+        self._in_table_function = in_table_function
+
+    def map(self, c):
+        if self._in_table_function(c):
+            return None
+        else:
+            return c
+
+
+class Profile:
+    def __init__(
+        self,
+        mappings=[],
+        normalize=True,
+        prohibiteds=[],
+        check_unassigneds=True,
+        check_bidi=True,
+    ):
+        self.mappings = mappings
+        self.normalize = normalize
+        self.prohibiteds = prohibiteds
+        self.do_check_unassigneds = check_unassigneds
+        self.do_check_bidi = check_bidi
+
+    def prepare(self, string):
+        result = self.map(string)
+        if self.normalize:
+            result = unicodedata.normalize("NFKC", result)
+        self.check_prohibiteds(result)
+        if self.do_check_unassigneds:
+            self.check_unassigneds(result)
+        if self.do_check_bidi:
+            self.check_bidirectionals(result)
+        return result
+
+    def map(self, string):
+        result = []
+
+        for c in string:
+            result_c = c
+
+            for mapping in self.mappings:
+                result_c = mapping.map(c)
+                if result_c != c:
+                    break
+
+            if result_c is not None:
+                result.append(result_c)
+
+        return "".join(result)
+
+    def check_prohibiteds(self, string):
+        for c in string:
+            for table in self.prohibiteds:
+                if table.lookup(c):
+                    raise UnicodeError("Invalid character %s" % repr(c))
+
+    def check_unassigneds(self, string):
+        for c in string:
+            if stringprep.in_table_a1(c):
+                raise UnicodeError("Unassigned code point %s" % repr(c))
+
+    def check_bidirectionals(self, string):
+        found_LCat = False
+        found_RandALCat = False
+
+        for c in string:
+            if stringprep.in_table_d1(c):
+                found_RandALCat = True
+            if stringprep.in_table_d2(c):
+                found_LCat = True
+
+        if found_LCat and found_RandALCat:
+            raise UnicodeError("Violation of BIDI Requirement 2")
+
+        if found_RandALCat and not (
+            stringprep.in_table_d1(string[0]) and stringprep.in_table_d1(string[-1])
+        ):
+            raise UnicodeError("Violation of BIDI Requirement 3")
+
+
+class NamePrep:
+    """Implements preparation of internationalized domain names.
+
+    This class implements preparing internationalized domain names using the
+    rules defined in RFC 3491, section 4 (Conversion operations).
+
+    We do not perform step 4 since we deal with unicode representations of
+    domain names and do not convert from or to ASCII representations using
+    punycode encoding. When such a conversion is needed, the C{idna} standard
+    library provides the C{ToUnicode()} and C{ToASCII()} functions. Note that
+    C{idna} itself assumes UseSTD3ASCIIRules to be false.
+
+    The following steps are performed by C{prepare()}:
+
+      - Split the domain name in labels at the dots (RFC 3490, 3.1)
+      - Apply nameprep proper on each label (RFC 3491)
+      - Enforce the restrictions on ASCII characters in host names by
+        assuming STD3ASCIIRules to be true. (STD 3)
+      - Rejoin the labels using the label separator U+002E (full stop).
+
+    """
+
+    # Prohibited characters.
+    prohibiteds = [
+        chr(n)
+        for n in chain(
+            range(0x00, 0x2C + 1),
+            range(0x2E, 0x2F + 1),
+            range(0x3A, 0x40 + 1),
+            range(0x5B, 0x60 + 1),
+            range(0x7B, 0x7F + 1),
+        )
+    ]
+
+    def prepare(self, string):
+        result = []
+
+        labels = idna.dots.split(string)
+
+        if labels and len(labels[-1]) == 0:
+            trailing_dot = "."
+            del labels[-1]
+        else:
+            trailing_dot = ""
+
+        for label in labels:
+            result.append(self.nameprep(label))
+
+        return ".".join(result) + trailing_dot
+
+    def check_prohibiteds(self, string):
+        for c in string:
+            if c in self.prohibiteds:
+                raise UnicodeError("Invalid character %s" % repr(c))
+
+    def nameprep(self, label):
+        label = idna.nameprep(label)
+        self.check_prohibiteds(label)
+        if label[0] == "-":
+            raise UnicodeError("Invalid leading hyphen-minus")
+        if label[-1] == "-":
+            raise UnicodeError("Invalid trailing hyphen-minus")
+        return label
+
+
+C_11 = LookupTableFromFunction(stringprep.in_table_c11)
+C_12 = LookupTableFromFunction(stringprep.in_table_c12)
+C_21 = LookupTableFromFunction(stringprep.in_table_c21)
+C_22 = LookupTableFromFunction(stringprep.in_table_c22)
+C_3 = LookupTableFromFunction(stringprep.in_table_c3)
+C_4 = LookupTableFromFunction(stringprep.in_table_c4)
+C_5 = LookupTableFromFunction(stringprep.in_table_c5)
+C_6 = LookupTableFromFunction(stringprep.in_table_c6)
+C_7 = LookupTableFromFunction(stringprep.in_table_c7)
+C_8 = LookupTableFromFunction(stringprep.in_table_c8)
+C_9 = LookupTableFromFunction(stringprep.in_table_c9)
+
+B_1 = EmptyMappingTable(stringprep.in_table_b1)
+B_2 = MappingTableFromFunction(stringprep.map_table_b2)
+
+nodeprep = Profile(
+    mappings=[B_1, B_2],
+    prohibiteds=[
+        C_11,
+        C_12,
+        C_21,
+        C_22,
+        C_3,
+        C_4,
+        C_5,
+        C_6,
+        C_7,
+        C_8,
+        C_9,
+        LookupTable(['"', "&", "'", "/", ":", "<", ">", "@"]),
+    ],
+)
+
+resourceprep = Profile(
+    mappings=[
+        B_1,
+    ],
+    prohibiteds=[C_12, C_21, C_22, C_3, C_4, C_5, C_6, C_7, C_8, C_9],
+)
+
+nameprep = NamePrep()
diff --git a/contrib/python/Twisted/py3/twisted/words/service.py b/contrib/python/Twisted/py3/twisted/words/service.py
new file mode 100644
index 00000000000..d65a425f948
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/service.py
@@ -0,0 +1,1278 @@
+# -*- test-case-name: twisted.words.test.test_service -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+A module that needs a better name.
+
+Implements new cred things for words.
+
+How does this thing work?
+
+  - Network connection on some port expecting to speak some protocol
+
+  - Protocol-specific authentication, resulting in some kind of credentials object
+
+  - twisted.cred.portal login using those credentials for the interface
+    IUser and with something implementing IChatClient as the mind
+
+  - successful login results in an IUser avatar the protocol can call
+    methods on, and state added to the realm such that the mind will have
+    methods called on it as is necessary
+
+  - protocol specific actions lead to calls onto the avatar; remote events
+    lead to calls onto the mind
+
+  - protocol specific hangup, realm is notified, user is removed from active
+    play, the end.
+"""
+
+from time import ctime, time
+
+from zope.interface import implementer
+
+from twisted import copyright
+from twisted.cred import credentials, error as ecred, portal
+from twisted.internet import defer, protocol
+from twisted.python import failure, log, reflect
+from twisted.python.components import registerAdapter
+from twisted.spread import pb
+from twisted.words import ewords, iwords
+from twisted.words.protocols import irc
+
+
+@implementer(iwords.IGroup)
+class Group:
+    def __init__(self, name):
+        self.name = name
+        self.users = {}
+        self.meta = {
+            "topic": "",
+            "topic_author": "",
+        }
+
+    def _ebUserCall(self, err, p):
+        return failure.Failure(Exception(p, err))
+
+    def _cbUserCall(self, results):
+        for success, result in results:
+            if not success:
+                user, err = result.value  # XXX
+                self.remove(user, err.getErrorMessage())
+
+    def add(self, user):
+        assert iwords.IChatClient.providedBy(user), "{!r} is not a chat client".format(
+            user
+        )
+        if user.name not in self.users:
+            additions = []
+            self.users[user.name] = user
+            for p in self.users.values():
+                if p is not user:
+                    d = defer.maybeDeferred(p.userJoined, self, user)
+                    d.addErrback(self._ebUserCall, p=p)
+                    additions.append(d)
+            defer.DeferredList(additions).addCallback(self._cbUserCall)
+        return defer.succeed(None)
+
+    def remove(self, user, reason=None):
+        try:
+            del self.users[user.name]
+        except KeyError:
+            pass
+        else:
+            removals = []
+            for p in self.users.values():
+                if p is not user:
+                    d = defer.maybeDeferred(p.userLeft, self, user, reason)
+                    d.addErrback(self._ebUserCall, p=p)
+                    removals.append(d)
+            defer.DeferredList(removals).addCallback(self._cbUserCall)
+        return defer.succeed(None)
+
+    def size(self):
+        return defer.succeed(len(self.users))
+
+    def receive(self, sender, recipient, message):
+        assert recipient is self
+        receives = []
+        for p in self.users.values():
+            if p is not sender:
+                d = defer.maybeDeferred(p.receive, sender, self, message)
+                d.addErrback(self._ebUserCall, p=p)
+                receives.append(d)
+        defer.DeferredList(receives).addCallback(self._cbUserCall)
+        return defer.succeed(None)
+
+    def setMetadata(self, meta):
+        self.meta = meta
+        sets = []
+        for p in self.users.values():
+            d = defer.maybeDeferred(p.groupMetaUpdate, self, meta)
+            d.addErrback(self._ebUserCall, p=p)
+            sets.append(d)
+        defer.DeferredList(sets).addCallback(self._cbUserCall)
+        return defer.succeed(None)
+
+    def iterusers(self):
+        # XXX Deferred?
+        return iter(self.users.values())
+
+
+@implementer(iwords.IUser)
+class User:
+    realm = None
+    mind = None
+
+    def __init__(self, name):
+        self.name = name
+        self.groups = []
+        self.lastMessage = time()
+
+    def loggedIn(self, realm, mind):
+        self.realm = realm
+        self.mind = mind
+        self.signOn = time()
+
+    def join(self, group):
+        def cbJoin(result):
+            self.groups.append(group)
+            return result
+
+        return group.add(self.mind).addCallback(cbJoin)
+
+    def leave(self, group, reason=None):
+        def cbLeave(result):
+            self.groups.remove(group)
+            return result
+
+        return group.remove(self.mind, reason).addCallback(cbLeave)
+
+    def send(self, recipient, message):
+        self.lastMessage = time()
+        return recipient.receive(self.mind, recipient, message)
+
+    def itergroups(self):
+        return iter(self.groups)
+
+    def logout(self):
+        for g in self.groups[:]:
+            self.leave(g)
+
+
+NICKSERV = "NickServ!NickServ@services"
+
+
+@implementer(iwords.IChatClient)
+class IRCUser(irc.IRC):
+    """
+    Protocol instance representing an IRC user connected to the server.
+    """
+
+    # A list of IGroups in which I am participating
+    groups = None
+
+    # A no-argument callable I should invoke when I go away
+    logout = None
+
+    # An IUser we use to interact with the chat service
+    avatar = None
+
+    # To whence I belong
+    realm = None
+
+    # How to handle unicode (TODO: Make this customizable on a per-user basis)
+    encoding = "utf-8"
+
+    # Twisted callbacks
+    def connectionMade(self):
+        self.irc_PRIVMSG = self.irc_NICKSERV_PRIVMSG
+        self.realm = self.factory.realm
+        self.hostname = self.realm.name
+
+    def connectionLost(self, reason):
+        if self.logout is not None:
+            self.logout()
+            self.avatar = None
+
+    # Make sendMessage a bit more useful to us
+    def sendMessage(self, command, *parameter_list, **kw):
+        if "prefix" not in kw:
+            kw["prefix"] = self.hostname
+        if "to" not in kw:
+            kw["to"] = self.name.encode(self.encoding)
+
+        arglist = [self, command, kw["to"]] + list(parameter_list)
+        arglistUnicode = []
+        for arg in arglist:
+            if isinstance(arg, bytes):
+                arg = arg.decode("utf-8")
+            arglistUnicode.append(arg)
+        irc.IRC.sendMessage(*arglistUnicode, **kw)
+
+    # IChatClient implementation
+    def userJoined(self, group, user):
+        self.join(f"{user.name}!{user.name}@{self.hostname}", "#" + group.name)
+
+    def userLeft(self, group, user, reason=None):
+        self.part(
+            f"{user.name}!{user.name}@{self.hostname}",
+            "#" + group.name,
+            (reason or "leaving"),
+        )
+
+    def receive(self, sender, recipient, message):
+        # >> :glyph!glyph@adsl-64-123-27-108.dsl.austtx.swbell.net PRIVMSG glyph_ :hello
+
+        # omg???????????
+        if iwords.IGroup.providedBy(recipient):
+            recipientName = "#" + recipient.name
+        else:
+            recipientName = recipient.name
+
+        text = message.get("text", "<an unrepresentable message>")
+        for L in text.splitlines():
+            self.privmsg(
+                f"{sender.name}!{sender.name}@{self.hostname}",
+                recipientName,
+                L,
+            )
+
+    def groupMetaUpdate(self, group, meta):
+        if "topic" in meta:
+            topic = meta["topic"]
+            author = meta.get("topic_author", "")
+            self.topic(
+                self.name,
+                "#" + group.name,
+                topic,
+                f"{author}!{author}@{self.hostname}",
+            )
+
+    # irc.IRC callbacks - starting with login related stuff.
+    nickname = None
+    password = None
+
+    def irc_PASS(self, prefix, params):
+        """
+        Password message -- Register a password.
+
+        Parameters: <password>
+
+        [REQUIRED]
+
+        Note that IRC requires the client send this *before* NICK
+        and USER.
+        """
+        self.password = params[-1]
+
+    def irc_NICK(self, prefix, params):
+        """
+        Nick message -- Set your nickname.
+
+        Parameters: <nickname>
+
+        [REQUIRED]
+        """
+        nickname = params[0]
+        try:
+            if isinstance(nickname, bytes):
+                nickname = nickname.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.privmsg(
+                NICKSERV,
+                repr(nickname),
+                "Your nickname cannot be decoded. Please use ASCII or UTF-8.",
+            )
+            self.transport.loseConnection()
+            return
+
+        self.nickname = nickname
+        self.name = nickname
+
+        for code, text in self._motdMessages:
+            self.sendMessage(code, text % self.factory._serverInfo)
+
+        if self.password is None:
+            self.privmsg(NICKSERV, nickname, "Password?")
+        else:
+            password = self.password
+            self.password = None
+            self.logInAs(nickname, password)
+
+    def irc_USER(self, prefix, params):
+        """
+        User message -- Set your realname.
+
+        Parameters: <user> <mode> <unused> <realname>
+        """
+        # Note: who gives a crap about this?  The IUser has the real
+        # information we care about.  Save it anyway, I guess, just
+        # for fun.
+        self.realname = params[-1]
+
+    def irc_NICKSERV_PRIVMSG(self, prefix, params):
+        """
+        Send a (private) message.
+
+        Parameters: <msgtarget> <text to be sent>
+        """
+        target = params[0]
+        password = params[-1]
+
+        if self.nickname is None:
+            # XXX Send an error response here
+            self.transport.loseConnection()
+        elif target.lower() != "nickserv":
+            self.privmsg(
+                NICKSERV,
+                self.nickname,
+                "Denied.  Please send me (NickServ) your password.",
+            )
+        else:
+            nickname = self.nickname
+            self.nickname = None
+            self.logInAs(nickname, password)
+
+    def logInAs(self, nickname, password):
+        d = self.factory.portal.login(
+            credentials.UsernamePassword(nickname, password), self, iwords.IUser
+        )
+        d.addCallbacks(self._cbLogin, self._ebLogin, errbackArgs=(nickname,))
+
+    _welcomeMessages = [
+        (irc.RPL_WELCOME, ":connected to Twisted IRC"),
+        (
+            irc.RPL_YOURHOST,
+            ":Your host is %(serviceName)s, running version %(serviceVersion)s",
+        ),
+        (irc.RPL_CREATED, ":This server was created on %(creationDate)s"),
+        # "Bummer.  This server returned a worthless 004 numeric.
+        #  I'll have to guess at all the values"
+        #    -- epic
+        (
+            irc.RPL_MYINFO,
+            # w and n are the currently supported channel and user modes
+            # -- specify this better
+            "%(serviceName)s %(serviceVersion)s w n",
+        ),
+    ]
+
+    _motdMessages = [
+        (irc.RPL_MOTDSTART, ":- %(serviceName)s Message of the Day - "),
+        (irc.RPL_ENDOFMOTD, ":End of /MOTD command."),
+    ]
+
+    def _cbLogin(self, result):
+        (iface, avatar, logout) = result
+        assert iface is iwords.IUser, f"Realm is buggy, got {iface!r}"
+
+        # Let them send messages to the world
+        del self.irc_PRIVMSG
+
+        self.avatar = avatar
+        self.logout = logout
+        for code, text in self._welcomeMessages:
+            self.sendMessage(code, text % self.factory._serverInfo)
+
+    def _ebLogin(self, err, nickname):
+        if err.check(ewords.AlreadyLoggedIn):
+            self.privmsg(
+                NICKSERV, nickname, "Already logged in.  No pod people allowed!"
+            )
+        elif err.check(ecred.UnauthorizedLogin):
+            self.privmsg(NICKSERV, nickname, "Login failed.  Goodbye.")
+        else:
+            log.msg("Unhandled error during login:")
+            log.err(err)
+            self.privmsg(NICKSERV, nickname, "Server error during login.  Sorry.")
+        self.transport.loseConnection()
+
+    # Great, now that's out of the way, here's some of the interesting
+    # bits
+    def irc_PING(self, prefix, params):
+        """
+        Ping message
+
+        Parameters: <server1> [ <server2> ]
+        """
+        if self.realm is not None:
+            self.sendMessage("PONG", self.hostname)
+
+    def irc_QUIT(self, prefix, params):
+        """
+        Quit
+
+        Parameters: [ <Quit Message> ]
+        """
+        self.transport.loseConnection()
+
+    def _channelMode(self, group, modes=None, *args):
+        if modes:
+            self.sendMessage(irc.ERR_UNKNOWNMODE, ":Unknown MODE flag.")
+        else:
+            self.channelMode(self.name, "#" + group.name, "+")
+
+    def _userMode(self, user, modes=None):
+        if modes:
+            self.sendMessage(irc.ERR_UNKNOWNMODE, ":Unknown MODE flag.")
+        elif user is self.avatar:
+            self.sendMessage(irc.RPL_UMODEIS, "+")
+        else:
+            self.sendMessage(
+                irc.ERR_USERSDONTMATCH, ":You can't look at someone else's modes."
+            )
+
+    def irc_MODE(self, prefix, params):
+        """
+        User mode message
+
+        Parameters: <nickname>
+        *( ( "+" / "-" ) *( "i" / "w" / "o" / "O" / "r" ) )
+
+        """
+        try:
+            channelOrUser = params[0]
+            if isinstance(channelOrUser, bytes):
+                channelOrUser = channelOrUser.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHNICK,
+                params[0],
+                ":No such nickname (could not decode your unicode!)",
+            )
+            return
+
+        if channelOrUser.startswith("#"):
+
+            def ebGroup(err):
+                err.trap(ewords.NoSuchGroup)
+                self.sendMessage(
+                    irc.ERR_NOSUCHCHANNEL, params[0], ":That channel doesn't exist."
+                )
+
+            d = self.realm.lookupGroup(channelOrUser[1:])
+            d.addCallbacks(self._channelMode, ebGroup, callbackArgs=tuple(params[1:]))
+        else:
+
+            def ebUser(err):
+                self.sendMessage(irc.ERR_NOSUCHNICK, ":No such nickname.")
+
+            d = self.realm.lookupUser(channelOrUser)
+            d.addCallbacks(self._userMode, ebUser, callbackArgs=tuple(params[1:]))
+
+    def irc_USERHOST(self, prefix, params):
+        """
+        Userhost message
+
+        Parameters: <nickname> *( SPACE <nickname> )
+
+        [Optional]
+        """
+        pass
+
+    def irc_PRIVMSG(self, prefix, params):
+        """
+        Send a (private) message.
+
+        Parameters: <msgtarget> <text to be sent>
+        """
+        try:
+            targetName = params[0]
+            if isinstance(targetName, bytes):
+                targetName = targetName.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHNICK,
+                params[0],
+                ":No such nick/channel (could not decode your unicode!)",
+            )
+            return
+
+        messageText = params[-1]
+        if targetName.startswith("#"):
+            target = self.realm.lookupGroup(targetName[1:])
+        else:
+            target = self.realm.lookupUser(targetName).addCallback(
+                lambda user: user.mind
+            )
+
+        def cbTarget(targ):
+            if targ is not None:
+                return self.avatar.send(targ, {"text": messageText})
+
+        def ebTarget(err):
+            self.sendMessage(irc.ERR_NOSUCHNICK, targetName, ":No such nick/channel.")
+
+        target.addCallbacks(cbTarget, ebTarget)
+
+    def irc_JOIN(self, prefix, params):
+        """
+        Join message
+
+        Parameters: ( <channel> *( "," <channel> ) [ <key> *( "," <key> ) ] )
+        """
+        try:
+            groupName = params[0]
+            if isinstance(groupName, bytes):
+                groupName = groupName.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL,
+                params[0],
+                ":No such channel (could not decode your unicode!)",
+            )
+            return
+
+        if groupName.startswith("#"):
+            groupName = groupName[1:]
+
+        def cbGroup(group):
+            def cbJoin(ign):
+                self.userJoined(group, self)
+                self.names(
+                    self.name,
+                    "#" + group.name,
+                    [user.name for user in group.iterusers()],
+                )
+                self._sendTopic(group)
+
+            return self.avatar.join(group).addCallback(cbJoin)
+
+        def ebGroup(err):
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL, "#" + groupName, ":No such channel."
+            )
+
+        self.realm.getGroup(groupName).addCallbacks(cbGroup, ebGroup)
+
+    def irc_PART(self, prefix, params):
+        """
+        Part message
+
+        Parameters: <channel> *( "," <channel> ) [ <Part Message> ]
+        """
+        try:
+            groupName = params[0]
+            if isinstance(params[0], bytes):
+                groupName = params[0].decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOTONCHANNEL, params[0], ":Could not decode your unicode!"
+            )
+            return
+
+        if groupName.startswith("#"):
+            groupName = groupName[1:]
+
+        if len(params) > 1:
+            reason = params[1]
+            if isinstance(reason, bytes):
+                reason = reason.decode("utf-8")
+        else:
+            reason = None
+
+        def cbGroup(group):
+            def cbLeave(result):
+                self.userLeft(group, self, reason)
+
+            return self.avatar.leave(group, reason).addCallback(cbLeave)
+
+        def ebGroup(err):
+            err.trap(ewords.NoSuchGroup)
+            self.sendMessage(
+                irc.ERR_NOTONCHANNEL, "#" + groupName, ":" + err.getErrorMessage()
+            )
+
+        self.realm.lookupGroup(groupName).addCallbacks(cbGroup, ebGroup)
+
+    def irc_NAMES(self, prefix, params):
+        """
+        Names message
+
+        Parameters: [ <channel> *( "," <channel> ) [ <target> ] ]
+        """
+        # << NAMES #python
+        # >> :benford.openprojects.net 353 glyph = #python :Orban ... @glyph ... Zymurgy skreech
+        # >> :benford.openprojects.net 366 glyph #python :End of /NAMES list.
+        try:
+            channel = params[-1]
+            if isinstance(channel, bytes):
+                channel = channel.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL,
+                params[-1],
+                ":No such channel (could not decode your unicode!)",
+            )
+            return
+
+        if channel.startswith("#"):
+            channel = channel[1:]
+
+        def cbGroup(group):
+            self.names(
+                self.name, "#" + group.name, [user.name for user in group.iterusers()]
+            )
+
+        def ebGroup(err):
+            err.trap(ewords.NoSuchGroup)
+            # No group?  Fine, no names!
+            self.names(self.name, "#" + channel, [])
+
+        self.realm.lookupGroup(channel).addCallbacks(cbGroup, ebGroup)
+
+    def irc_TOPIC(self, prefix, params):
+        """
+        Topic message
+
+        Parameters: <channel> [ <topic> ]
+        """
+        try:
+            channel = params[0]
+            if isinstance(params[0], bytes):
+                channel = channel.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL,
+                ":That channel doesn't exist (could not decode your unicode!)",
+            )
+            return
+
+        if channel.startswith("#"):
+            channel = channel[1:]
+
+        if len(params) > 1:
+            self._setTopic(channel, params[1])
+        else:
+            self._getTopic(channel)
+
+    def _sendTopic(self, group):
+        """
+        Send the topic of the given group to this user, if it has one.
+        """
+        topic = group.meta.get("topic")
+        if topic:
+            author = group.meta.get("topic_author") or "<noone>"
+            date = group.meta.get("topic_date", 0)
+            self.topic(self.name, "#" + group.name, topic)
+            self.topicAuthor(self.name, "#" + group.name, author, date)
+
+    def _getTopic(self, channel):
+        # << TOPIC #python
+        # >> :benford.openprojects.net 332 glyph #python :<churchr> I really did. I sprained all my toes.
+        # >> :benford.openprojects.net 333 glyph #python itamar|nyc 994713482
+        def ebGroup(err):
+            err.trap(ewords.NoSuchGroup)
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL, "=", channel, ":That channel doesn't exist."
+            )
+
+        self.realm.lookupGroup(channel).addCallbacks(self._sendTopic, ebGroup)
+
+    def _setTopic(self, channel, topic):
+        # << TOPIC #divunal :foo
+        # >> :glyph!glyph@adsl-64-123-27-108.dsl.austtx.swbell.net TOPIC #divunal :foo
+
+        def cbGroup(group):
+            newMeta = group.meta.copy()
+            newMeta["topic"] = topic
+            newMeta["topic_author"] = self.name
+            newMeta["topic_date"] = int(time())
+
+            def ebSet(err):
+                self.sendMessage(
+                    irc.ERR_CHANOPRIVSNEEDED,
+                    "#" + group.name,
+                    ":You need to be a channel operator to do that.",
+                )
+
+            return group.setMetadata(newMeta).addErrback(ebSet)
+
+        def ebGroup(err):
+            err.trap(ewords.NoSuchGroup)
+            self.sendMessage(
+                irc.ERR_NOSUCHCHANNEL, "=", channel, ":That channel doesn't exist."
+            )
+
+        self.realm.lookupGroup(channel).addCallbacks(cbGroup, ebGroup)
+
+    def list(self, channels):
+        """
+        Send a group of LIST response lines
+
+        @type channels: C{list} of C{(str, int, str)}
+        @param channels: Information about the channels being sent:
+            their name, the number of participants, and their topic.
+        """
+        for name, size, topic in channels:
+            self.sendMessage(irc.RPL_LIST, name, str(size), ":" + topic)
+        self.sendMessage(irc.RPL_LISTEND, ":End of /LIST")
+
+    def irc_LIST(self, prefix, params):
+        """
+        List query
+
+        Return information about the indicated channels, or about all
+        channels if none are specified.
+
+        Parameters: [ <channel> *( "," <channel> ) [ <target> ] ]
+        """
+        # << list #python
+        # >> :orwell.freenode.net 321 exarkun Channel :Users  Name
+        # >> :orwell.freenode.net 322 exarkun #python 358 :The Python programming language
+        # >> :orwell.freenode.net 323 exarkun :End of /LIST
+        if params:
+            # Return information about indicated channels
+            try:
+                allChannels = params[0]
+                if isinstance(allChannels, bytes):
+                    allChannels = allChannels.decode(self.encoding)
+                channels = allChannels.split(",")
+            except UnicodeDecodeError:
+                self.sendMessage(
+                    irc.ERR_NOSUCHCHANNEL,
+                    params[0],
+                    ":No such channel (could not decode your unicode!)",
+                )
+                return
+
+            groups = []
+            for ch in channels:
+                if ch.startswith("#"):
+                    ch = ch[1:]
+                groups.append(self.realm.lookupGroup(ch))
+
+            groups = defer.DeferredList(groups, consumeErrors=True)
+            groups.addCallback(lambda gs: [r for (s, r) in gs if s])
+        else:
+            # Return information about all channels
+            groups = self.realm.itergroups()
+
+        def cbGroups(groups):
+            def gotSize(size, group):
+                return group.name, size, group.meta.get("topic")
+
+            d = defer.DeferredList(
+                [group.size().addCallback(gotSize, group) for group in groups]
+            )
+            d.addCallback(lambda results: self.list([r for (s, r) in results if s]))
+            return d
+
+        groups.addCallback(cbGroups)
+
+    def _channelWho(self, group):
+        self.who(
+            self.name,
+            "#" + group.name,
+            [
+                (m.name, self.hostname, self.realm.name, m.name, "H", 0, m.name)
+                for m in group.iterusers()
+            ],
+        )
+
+    def _userWho(self, user):
+        self.sendMessage(irc.RPL_ENDOFWHO, ":User /WHO not implemented")
+
+    def irc_WHO(self, prefix, params):
+        """
+        Who query
+
+        Parameters: [ <mask> [ "o" ] ]
+        """
+        # << who #python
+        # >> :x.opn 352 glyph #python aquarius pc-62-31-193-114-du.blueyonder.co.uk y.opn Aquarius H :3 Aquarius
+        # ...
+        # >> :x.opn 352 glyph #python foobar europa.tranquility.net z.opn skreech H :0 skreech
+        # >> :x.opn 315 glyph #python :End of /WHO list.
+        ### also
+        # << who glyph
+        # >> :x.opn 352 glyph #python glyph adsl-64-123-27-108.dsl.austtx.swbell.net x.opn glyph H :0 glyph
+        # >> :x.opn 315 glyph glyph :End of /WHO list.
+        if not params:
+            self.sendMessage(irc.RPL_ENDOFWHO, ":/WHO not supported.")
+            return
+
+        try:
+            channelOrUser = params[0]
+            if isinstance(channelOrUser, bytes):
+                channelOrUser = channelOrUser.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(
+                irc.RPL_ENDOFWHO,
+                params[0],
+                ":End of /WHO list (could not decode your unicode!)",
+            )
+            return
+
+        if channelOrUser.startswith("#"):
+
+            def ebGroup(err):
+                err.trap(ewords.NoSuchGroup)
+                self.sendMessage(irc.RPL_ENDOFWHO, channelOrUser, ":End of /WHO list.")
+
+            d = self.realm.lookupGroup(channelOrUser[1:])
+            d.addCallbacks(self._channelWho, ebGroup)
+        else:
+
+            def ebUser(err):
+                err.trap(ewords.NoSuchUser)
+                self.sendMessage(irc.RPL_ENDOFWHO, channelOrUser, ":End of /WHO list.")
+
+            d = self.realm.lookupUser(channelOrUser)
+            d.addCallbacks(self._userWho, ebUser)
+
+    def irc_WHOIS(self, prefix, params):
+        """
+        Whois query
+
+        Parameters: [ <target> ] <mask> *( "," <mask> )
+        """
+
+        def cbUser(user):
+            self.whois(
+                self.name,
+                user.name,
+                user.name,
+                self.realm.name,
+                user.name,
+                self.realm.name,
+                "Hi mom!",
+                False,
+                int(time() - user.lastMessage),
+                user.signOn,
+                ["#" + group.name for group in user.itergroups()],
+            )
+
+        def ebUser(err):
+            err.trap(ewords.NoSuchUser)
+            self.sendMessage(irc.ERR_NOSUCHNICK, params[0], ":No such nick/channel")
+
+        try:
+            user = params[0]
+            if isinstance(user, bytes):
+                user = user.decode(self.encoding)
+        except UnicodeDecodeError:
+            self.sendMessage(irc.ERR_NOSUCHNICK, params[0], ":No such nick/channel")
+            return
+
+        self.realm.lookupUser(user).addCallbacks(cbUser, ebUser)
+
+    # Unsupported commands, here for legacy compatibility
+    def irc_OPER(self, prefix, params):
+        """
+        Oper message
+
+        Parameters: <name> <password>
+        """
+        self.sendMessage(irc.ERR_NOOPERHOST, ":O-lines not applicable")
+
+
+class IRCFactory(protocol.ServerFactory):
+    """
+    IRC server that creates instances of the L{IRCUser} protocol.
+
+    @ivar _serverInfo: A dictionary mapping:
+        "serviceName" to the name of the server,
+        "serviceVersion" to the copyright version,
+        "creationDate" to the time that the server was started.
+    """
+
+    protocol = IRCUser
+
+    def __init__(self, realm, portal):
+        self.realm = realm
+        self.portal = portal
+        self._serverInfo = {
+            "serviceName": self.realm.name,
+            "serviceVersion": copyright.version,
+            "creationDate": ctime(),
+        }
+
+
+class PBMind(pb.Referenceable):
+    def __init__(self):
+        pass
+
+    def jellyFor(self, jellier):
+        qual = reflect.qual(PBMind)
+        if isinstance(qual, str):
+            qual = qual.encode("utf-8")
+        return qual, jellier.invoker.registerReference(self)
+
+    def remote_userJoined(self, user, group):
+        pass
+
+    def remote_userLeft(self, user, group, reason):
+        pass
+
+    def remote_receive(self, sender, recipient, message):
+        pass
+
+    def remote_groupMetaUpdate(self, group, meta):
+        pass
+
+
+@implementer(iwords.IChatClient)
+class PBMindReference(pb.RemoteReference):
+    name = ""
+
+    def receive(self, sender, recipient, message):
+        if iwords.IGroup.providedBy(recipient):
+            rec = PBGroup(self.realm, self.avatar, recipient)
+        else:
+            rec = PBUser(self.realm, self.avatar, recipient)
+        return self.callRemote(
+            "receive", PBUser(self.realm, self.avatar, sender), rec, message
+        )
+
+    def groupMetaUpdate(self, group, meta):
+        return self.callRemote(
+            "groupMetaUpdate", PBGroup(self.realm, self.avatar, group), meta
+        )
+
+    def userJoined(self, group, user):
+        return self.callRemote(
+            "userJoined",
+            PBGroup(self.realm, self.avatar, group),
+            PBUser(self.realm, self.avatar, user),
+        )
+
+    def userLeft(self, group, user, reason=None):
+        return self.callRemote(
+            "userLeft",
+            PBGroup(self.realm, self.avatar, group),
+            PBUser(self.realm, self.avatar, user),
+            reason,
+        )
+
+
+pb.setUnjellyableForClass(PBMind, PBMindReference)
+
+
+class PBGroup(pb.Referenceable):
+    def __init__(self, realm, avatar, group):
+        self.realm = realm
+        self.avatar = avatar
+        self.group = group
+
+    def processUniqueID(self):
+        return hash((self.realm.name, self.avatar.name, self.group.name))
+
+    def jellyFor(self, jellier):
+        qual = reflect.qual(self.__class__)
+        if isinstance(qual, str):
+            qual = qual.encode("utf-8")
+        group = self.group.name
+        if isinstance(group, str):
+            group = group.encode("utf-8")
+        return qual, group, jellier.invoker.registerReference(self)
+
+    def remote_leave(self, reason=None):
+        return self.avatar.leave(self.group, reason)
+
+    def remote_send(self, message):
+        return self.avatar.send(self.group, message)
+
+
+@implementer(iwords.IGroup)
+class PBGroupReference(pb.RemoteReference):
+    def unjellyFor(self, unjellier, unjellyList):
+        clsName, name, ref = unjellyList
+        self.name = name
+        if bytes != str and isinstance(self.name, bytes):
+            self.name = self.name.decode("utf-8")
+        return pb.RemoteReference.unjellyFor(self, unjellier, [clsName, ref])
+
+    def leave(self, reason=None):
+        return self.callRemote("leave", reason)
+
+    def send(self, message):
+        return self.callRemote("send", message)
+
+    def add(self, user):
+        # IGroup.add
+        pass
+
+    def iterusers(self):
+        # IGroup.iterusers
+        pass
+
+    def receive(self, sender, recipient, message):
+        # IGroup.receive
+        pass
+
+    def remove(self, user, reason=None):
+        # IGroup.remove
+        pass
+
+    def setMetadata(self, meta):
+        # IGroup.setMetadata
+        pass
+
+    def size(self):
+        # IGroup.size
+        pass
+
+
+pb.setUnjellyableForClass(PBGroup, PBGroupReference)
+
+
+class PBUser(pb.Referenceable):
+    def __init__(self, realm, avatar, user):
+        self.realm = realm
+        self.avatar = avatar
+        self.user = user
+
+    def processUniqueID(self):
+        return hash((self.realm.name, self.avatar.name, self.user.name))
+
+
+@implementer(iwords.IChatClient)
+class ChatAvatar(pb.Referenceable):
+    def __init__(self, avatar):
+        self.avatar = avatar
+
+    def jellyFor(self, jellier):
+        qual = reflect.qual(self.__class__)
+        if isinstance(qual, str):
+            qual = qual.encode("utf-8")
+        return qual, jellier.invoker.registerReference(self)
+
+    def remote_join(self, groupName):
+        def cbGroup(group):
+            def cbJoin(ignored):
+                return PBGroup(self.avatar.realm, self.avatar, group)
+
+            d = self.avatar.join(group)
+            d.addCallback(cbJoin)
+            return d
+
+        d = self.avatar.realm.getGroup(groupName)
+        d.addCallback(cbGroup)
+        return d
+
+    @property
+    def name(self):
+        # IChatClient.name
+        pass
+
+    @name.setter
+    def name(self, value):
+        # IChatClient.name
+        pass
+
+    def groupMetaUpdate(self, group, meta):
+        # IChatClient.groupMetaUpdate
+        pass
+
+    def receive(self, sender, recipient, message):
+        # IChatClient.receive
+        pass
+
+    def userJoined(self, group, user):
+        # IChatClient.userJoined
+        pass
+
+    def userLeft(self, group, user, reason=None):
+        # IChatClient.userLeft
+        pass
+
+
+registerAdapter(ChatAvatar, iwords.IUser, pb.IPerspective)
+
+
+class AvatarReference(pb.RemoteReference):
+    def join(self, groupName):
+        return self.callRemote("join", groupName)
+
+    def quit(self):
+        d = defer.Deferred()
+        self.broker.notifyOnDisconnect(lambda: d.callback(None))
+        self.broker.transport.loseConnection()
+        return d
+
+
+pb.setUnjellyableForClass(ChatAvatar, AvatarReference)
+
+
+@implementer(portal.IRealm, iwords.IChatService)
+class WordsRealm:
+    _encoding = "utf-8"
+
+    def __init__(self, name):
+        self.name = name
+
+    def userFactory(self, name):
+        return User(name)
+
+    def groupFactory(self, name):
+        return Group(name)
+
+    def logoutFactory(self, avatar, facet):
+        def logout():
+            # XXX Deferred support here
+            getattr(facet, "logout", lambda: None)()
+            avatar.realm = avatar.mind = None
+
+        return logout
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        if isinstance(avatarId, bytes):
+            avatarId = avatarId.decode(self._encoding)
+
+        def gotAvatar(avatar):
+            if avatar.realm is not None:
+                raise ewords.AlreadyLoggedIn()
+            for iface in interfaces:
+                facet = iface(avatar, None)
+                if facet is not None:
+                    avatar.loggedIn(self, mind)
+                    mind.name = avatarId
+                    mind.realm = self
+                    mind.avatar = avatar
+                    return iface, facet, self.logoutFactory(avatar, facet)
+            raise NotImplementedError(self, interfaces)
+
+        return self.getUser(avatarId).addCallback(gotAvatar)
+
+    def itergroups(self):
+        # IChatServer.itergroups
+        pass
+
+    # IChatService, mostly.
+    createGroupOnRequest = False
+    createUserOnRequest = True
+
+    def lookupUser(self, name):
+        raise NotImplementedError
+
+    def lookupGroup(self, group):
+        raise NotImplementedError
+
+    def addUser(self, user):
+        """
+        Add the given user to this service.
+
+        This is an internal method intended to be overridden by
+        L{WordsRealm} subclasses, not called by external code.
+
+        @type user: L{IUser}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with L{None} when the user is
+        added, or which fails with
+        L{twisted.words.ewords.DuplicateUser} if a user with the
+        same name exists already.
+        """
+        raise NotImplementedError
+
+    def addGroup(self, group):
+        """
+        Add the given group to this service.
+
+        @type group: L{IGroup}
+
+        @rtype: L{twisted.internet.defer.Deferred}
+        @return: A Deferred which fires with L{None} when the group is
+        added, or which fails with
+        L{twisted.words.ewords.DuplicateGroup} if a group with the
+        same name exists already.
+        """
+        raise NotImplementedError
+
+    def getGroup(self, name):
+        if self.createGroupOnRequest:
+
+            def ebGroup(err):
+                err.trap(ewords.DuplicateGroup)
+                return self.lookupGroup(name)
+
+            return self.createGroup(name).addErrback(ebGroup)
+        return self.lookupGroup(name)
+
+    def getUser(self, name):
+        if self.createUserOnRequest:
+
+            def ebUser(err):
+                err.trap(ewords.DuplicateUser)
+                return self.lookupUser(name)
+
+            return self.createUser(name).addErrback(ebUser)
+        return self.lookupUser(name)
+
+    def createUser(self, name):
+        def cbLookup(user):
+            return failure.Failure(ewords.DuplicateUser(name))
+
+        def ebLookup(err):
+            err.trap(ewords.NoSuchUser)
+            return self.userFactory(name)
+
+        name = name.lower()
+        d = self.lookupUser(name)
+        d.addCallbacks(cbLookup, ebLookup)
+        d.addCallback(self.addUser)
+        return d
+
+    def createGroup(self, name):
+        def cbLookup(group):
+            return failure.Failure(ewords.DuplicateGroup(name))
+
+        def ebLookup(err):
+            err.trap(ewords.NoSuchGroup)
+            return self.groupFactory(name)
+
+        name = name.lower()
+        d = self.lookupGroup(name)
+        d.addCallbacks(cbLookup, ebLookup)
+        d.addCallback(self.addGroup)
+        return d
+
+
+class InMemoryWordsRealm(WordsRealm):
+    def __init__(self, *a, **kw):
+        super().__init__(*a, **kw)
+        self.users = {}
+        self.groups = {}
+
+    def itergroups(self):
+        return defer.succeed(self.groups.values())
+
+    def addUser(self, user):
+        if user.name in self.users:
+            return defer.fail(failure.Failure(ewords.DuplicateUser()))
+        self.users[user.name] = user
+        return defer.succeed(user)
+
+    def addGroup(self, group):
+        if group.name in self.groups:
+            return defer.fail(failure.Failure(ewords.DuplicateGroup()))
+        self.groups[group.name] = group
+        return defer.succeed(group)
+
+    def lookupUser(self, name):
+        name = name.lower()
+        try:
+            user = self.users[name]
+        except KeyError:
+            return defer.fail(failure.Failure(ewords.NoSuchUser(name)))
+        else:
+            return defer.succeed(user)
+
+    def lookupGroup(self, name):
+        name = name.lower()
+        try:
+            group = self.groups[name]
+        except KeyError:
+            return defer.fail(failure.Failure(ewords.NoSuchGroup(name)))
+        else:
+            return defer.succeed(group)
+
+
+__all__ = [
+    "Group",
+    "User",
+    "WordsRealm",
+    "InMemoryWordsRealm",
+]
diff --git a/contrib/python/Twisted/py3/twisted/words/tap.py b/contrib/python/Twisted/py3/twisted/words/tap.py
new file mode 100644
index 00000000000..d83db71b76e
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/tap.py
@@ -0,0 +1,89 @@
+# -*- test-case-name: twisted.words.test.test_tap -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+"""
+Shiny new words service maker
+"""
+
+import socket
+import sys
+from typing import List, Optional, Sequence
+
+from twisted import plugin
+from twisted.application import strports
+from twisted.application.service import MultiService
+from twisted.cred import checkers, credentials, portal, strcred
+from twisted.python import usage
+from twisted.words import iwords, service
+
+
+class Options(usage.Options, strcred.AuthOptionMixin):
+    supportedInterfaces = [credentials.IUsernamePassword]
+    optParameters: List[Sequence[Optional[str]]] = [
+        (
+            "hostname",
+            None,
+            socket.gethostname(),
+            "Name of this server; purely an informative",
+        )
+    ]
+
+    compData = usage.Completions(multiUse=["group"])
+
+    interfacePlugins = {}
+    plg = None
+    for plg in plugin.getPlugins(iwords.IProtocolPlugin):
+        assert plg.name not in interfacePlugins
+        interfacePlugins[plg.name] = plg
+        optParameters.append(
+            (
+                plg.name + "-port",
+                None,
+                None,
+                "strports description of the port to bind for the  "
+                + plg.name
+                + " server",
+            )
+        )
+    del plg
+
+    def __init__(self, *a, **kw):
+        usage.Options.__init__(self, *a, **kw)
+        self["groups"] = []
+
+    def opt_group(self, name):
+        """Specify a group which should exist"""
+        self["groups"].append(name.decode(sys.stdin.encoding))
+
+    def opt_passwd(self, filename):
+        """
+        Name of a passwd-style file. (This is for
+        backwards-compatibility only; you should use the --auth
+        command instead.)
+        """
+        self.addChecker(checkers.FilePasswordDB(filename))
+
+
+def makeService(config):
+    credCheckers = config.get("credCheckers", [])
+    wordsRealm = service.InMemoryWordsRealm(config["hostname"])
+    wordsPortal = portal.Portal(wordsRealm, credCheckers)
+
+    msvc = MultiService()
+
+    # XXX Attribute lookup on config is kind of bad - hrm.
+    for plgName in config.interfacePlugins:
+        port = config.get(plgName + "-port")
+        if port is not None:
+            factory = config.interfacePlugins[plgName].getFactory(
+                wordsRealm, wordsPortal
+            )
+            svc = strports.service(port, factory)
+            svc.setServiceParent(msvc)
+
+    # This is bogus.  createGroup is async.  makeService must be
+    # allowed to return a Deferred or some crap.
+    for g in config["groups"]:
+        wordsRealm.createGroup(g)
+
+    return msvc
diff --git a/contrib/python/Twisted/py3/twisted/words/xish/__init__.py b/contrib/python/Twisted/py3/twisted/words/xish/__init__.py
new file mode 100644
index 00000000000..1d2469fe303
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/xish/__init__.py
@@ -0,0 +1,10 @@
+# -*- test-case-name: twisted.words.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+
+Twisted X-ish: XML-ish DOM and XPath-ish engine
+
+"""
diff --git a/contrib/python/Twisted/py3/twisted/words/xish/domish.py b/contrib/python/Twisted/py3/twisted/words/xish/domish.py
new file mode 100644
index 00000000000..7d5009cefa4
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/xish/domish.py
@@ -0,0 +1,901 @@
+# -*- test-case-name: twisted.words.test.test_domish -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+DOM-like XML processing support.
+
+This module provides support for parsing XML into DOM-like object structures
+and serializing such structures to an XML string representation, optimized
+for use in streaming XML applications.
+"""
+
+from typing import cast
+
+from zope.interface import Attribute, Interface, implementer
+
+from twisted.web import sux
+
+
+def _splitPrefix(name):
+    """Internal method for splitting a prefixed Element name into its
+    respective parts"""
+    ntok = name.split(":", 1)
+    if len(ntok) == 2:
+        return ntok
+    else:
+        return (None, ntok[0])
+
+
+# Global map of prefixes that always get injected
+# into the serializers prefix map (note, that doesn't
+# mean they're always _USED_)
+G_PREFIXES = {"http://www.w3.org/XML/1998/namespace": "xml"}
+
+
+class _ListSerializer:
+    """Internal class which serializes an Element tree into a buffer"""
+
+    def __init__(self, prefixes=None, prefixesInScope=None):
+        self.writelist = []
+        self.prefixes = {}
+        if prefixes:
+            self.prefixes.update(prefixes)
+        self.prefixes.update(G_PREFIXES)
+        self.prefixStack = [G_PREFIXES.values()] + (prefixesInScope or [])
+        self.prefixCounter = 0
+
+    def getValue(self):
+        return "".join(self.writelist)
+
+    def getPrefix(self, uri):
+        if uri not in self.prefixes:
+            self.prefixes[uri] = "xn%d" % (self.prefixCounter)
+            self.prefixCounter = self.prefixCounter + 1
+        return self.prefixes[uri]
+
+    def prefixInScope(self, prefix):
+        stack = self.prefixStack
+        for i in range(-1, (len(self.prefixStack) + 1) * -1, -1):
+            if prefix in stack[i]:
+                return True
+        return False
+
+    def serialize(self, elem, closeElement=1, defaultUri=""):
+        # Optimization shortcuts
+        write = self.writelist.append
+
+        # Shortcut, check to see if elem is actually a chunk o' serialized XML
+        if isinstance(elem, SerializedXML):
+            write(elem)
+            return
+
+        # Shortcut, check to see if elem is actually a string (aka Cdata)
+        if isinstance(elem, str):
+            write(escapeToXml(elem))
+            return
+
+        # Further optimizations
+        name = elem.name
+        uri = elem.uri
+        defaultUri, currentDefaultUri = elem.defaultUri, defaultUri
+
+        for p, u in elem.localPrefixes.items():
+            self.prefixes[u] = p
+        self.prefixStack.append(list(elem.localPrefixes.keys()))
+
+        # Inherit the default namespace
+        if defaultUri is None:
+            defaultUri = currentDefaultUri
+
+        if uri is None:
+            uri = defaultUri
+
+        prefix = None
+        if uri != defaultUri or uri in self.prefixes:
+            prefix = self.getPrefix(uri)
+            inScope = self.prefixInScope(prefix)
+
+        # Create the starttag
+
+        if not prefix:
+            write("<%s" % (name))
+        else:
+            write(f"<{prefix}:{name}")
+
+            if not inScope:
+                write(f" xmlns:{prefix}='{uri}'")
+                self.prefixStack[-1].append(prefix)
+                inScope = True
+
+        if defaultUri != currentDefaultUri and (
+            uri != defaultUri or not prefix or not inScope
+        ):
+            write(" xmlns='%s'" % (defaultUri))
+
+        for p, u in elem.localPrefixes.items():
+            write(f" xmlns:{p}='{u}'")
+
+        # Serialize attributes
+        for k, v in elem.attributes.items():
+            # If the attribute name is a tuple, it's a qualified attribute
+            if isinstance(k, tuple):
+                attr_uri, attr_name = k
+                attr_prefix = self.getPrefix(attr_uri)
+
+                if not self.prefixInScope(attr_prefix):
+                    write(f" xmlns:{attr_prefix}='{attr_uri}'")
+                    self.prefixStack[-1].append(attr_prefix)
+
+                write(f" {attr_prefix}:{attr_name}='{escapeToXml(v, 1)}'")
+            else:
+                write(f" {k}='{escapeToXml(v, 1)}'")
+
+        # Shortcut out if this is only going to return
+        # the element (i.e. no children)
+        if closeElement == 0:
+            write(">")
+            return
+
+        # Serialize children
+        if len(elem.children) > 0:
+            write(">")
+            for c in elem.children:
+                self.serialize(c, defaultUri=defaultUri)
+            # Add closing tag
+            if not prefix:
+                write("</%s>" % (name))
+            else:
+                write(f"</{prefix}:{name}>")
+        else:
+            write("/>")
+
+        self.prefixStack.pop()
+
+
+SerializerClass = _ListSerializer
+
+
+def escapeToXml(text, isattrib=0):
+    """Escape text to proper XML form, per section 2.3 in the XML specification.
+
+    @type text: C{str}
+    @param text: Text to escape
+
+    @type isattrib: C{bool}
+    @param isattrib: Triggers escaping of characters necessary for use as
+                     attribute values
+    """
+    text = text.replace("&", "&amp;")
+    text = text.replace("<", "&lt;")
+    text = text.replace(">", "&gt;")
+    if isattrib == 1:
+        text = text.replace("'", "&apos;")
+        text = text.replace('"', "&quot;")
+    return text
+
+
+def unescapeFromXml(text):
+    text = text.replace("&lt;", "<")
+    text = text.replace("&gt;", ">")
+    text = text.replace("&apos;", "'")
+    text = text.replace("&quot;", '"')
+    text = text.replace("&amp;", "&")
+    return text
+
+
+def generateOnlyInterface(list, int):
+    """Filters items in a list by class"""
+    for n in list:
+        if int.providedBy(n):
+            yield n
+
+
+def generateElementsQNamed(list, name, uri):
+    """Filters Element items in a list with matching name and URI."""
+    for n in list:
+        if IElement.providedBy(n) and n.name == name and n.uri == uri:
+            yield n
+
+
+def generateElementsNamed(list, name):
+    """Filters Element items in a list with matching name, regardless of URI."""
+    for n in list:
+        if IElement.providedBy(n) and n.name == name:
+            yield n
+
+
+class SerializedXML(str):
+    """Marker class for pre-serialized XML in the DOM."""
+
+    pass
+
+
+class Namespace:
+    """Convenience object for tracking namespace declarations."""
+
+    def __init__(self, uri):
+        self._uri = uri
+
+    def __getattr__(self, n):
+        return (self._uri, n)
+
+    def __getitem__(self, n):
+        return (self._uri, n)
+
+
+class IElement(Interface):
+    """
+    Interface to XML element nodes.
+
+    See L{Element} for a detailed example of its general use.
+
+    Warning: this Interface is not yet complete!
+    """
+
+    uri = Attribute(""" Element's namespace URI """)
+    name = Attribute(""" Element's local name """)
+    defaultUri = Attribute(""" Default namespace URI of child elements """)
+    attributes = Attribute(""" Dictionary of element attributes """)
+    children = Attribute(""" List of child nodes """)
+    parent = Attribute(""" Reference to element's parent element """)
+    localPrefixes = Attribute(""" Dictionary of local prefixes """)
+
+    def toXml(prefixes=None, closeElement=1, defaultUri="", prefixesInScope=None):
+        """Serializes object to a (partial) XML document
+
+        @param prefixes: dictionary that maps namespace URIs to suggested
+                         prefix names.
+        @type prefixes: L{dict}
+
+        @param closeElement: flag that determines whether to include the
+            closing tag of the element in the serialized string. A value of
+            C{0} only generates the element's start tag. A value of C{1} yields
+            a complete serialization.
+        @type closeElement: L{int}
+
+        @param defaultUri: Initial default namespace URI. This is most useful
+            for partial rendering, where the logical parent element (of which
+            the starttag was already serialized) declares a default namespace
+            that should be inherited.
+        @type defaultUri: L{str}
+
+        @param prefixesInScope: list of prefixes that are assumed to be
+            declared by ancestors.
+        @type prefixesInScope: L{list}
+
+        @return: (partial) serialized XML
+        @rtype: L{str}
+        """
+
+    def addElement(name, defaultUri=None, content=None):
+        """
+        Create an element and add as child.
+
+        The new element is added to this element as a child, and will have
+        this element as its parent.
+
+        @param name: element name. This can be either a L{str} object that
+            contains the local name, or a tuple of (uri, local_name) for a
+            fully qualified name. In the former case, the namespace URI is
+            inherited from this element.
+        @type name: L{str} or L{tuple} of (L{str}, L{str})
+
+        @param defaultUri: default namespace URI for child elements. If
+            L{None}, this is inherited from this element.
+        @type defaultUri: L{str}
+
+        @param content: text contained by the new element.
+        @type content: L{str}
+
+        @return: the created element
+        @rtype: object providing L{IElement}
+        """
+
+    def addChild(node):
+        """
+        Adds a node as child of this element.
+
+        The C{node} will be added to the list of childs of this element, and
+        will have this element set as its parent when C{node} provides
+        L{IElement}. If C{node} is a L{str} and the current last child is
+        character data (L{str}), the text from C{node} is appended to the
+        existing last child.
+
+        @param node: the child node.
+        @type node: L{str} or object implementing L{IElement}
+        """
+
+    def addContent(text):
+        """
+        Adds character data to this element.
+
+        If the current last child of this element is a string, the text will
+        be appended to that string. Otherwise, the text will be added as a new
+        child.
+
+        @param text: The character data to be added to this element.
+        @type text: L{str}
+        """
+
+
+@implementer(IElement)
+class Element:
+    """Represents an XML element node.
+
+    An Element contains a series of attributes (name/value pairs), content
+    (character data), and other child Element objects. When building a document
+    with markup (such as HTML or XML), use this object as the starting point.
+
+    Element objects fully support XML Namespaces. The fully qualified name of
+    the XML Element it represents is stored in the C{uri} and C{name}
+    attributes, where C{uri} holds the namespace URI. There is also a default
+    namespace, for child elements. This is stored in the C{defaultUri}
+    attribute. Note that C{''} means the empty namespace.
+
+    Serialization of Elements through C{toXml()} will use these attributes
+    for generating proper serialized XML. When both C{uri} and C{defaultUri}
+    are not None in the Element and all of its descendents, serialization
+    proceeds as expected:
+
+      >>> from twisted.words.xish import domish
+      >>> root = domish.Element(('myns', 'root'))
+      >>> root.addElement('child', content='test')
+      <twisted.words.xish.domish.Element object at 0x83002ac>
+      >>> root.toXml()
+      u"<root xmlns='myns'><child>test</child></root>"
+
+    For partial serialization, needed for streaming XML, a special value for
+    namespace URIs can be used: L{None}.
+
+    Using L{None} as the value for C{uri} means: this element is in whatever
+    namespace inherited by the closest logical ancestor when the complete XML
+    document has been serialized. The serialized start tag will have a
+    non-prefixed name, and no xmlns declaration will be generated.
+
+    Similarly, L{None} for C{defaultUri} means: the default namespace for my
+    child elements is inherited from the logical ancestors of this element,
+    when the complete XML document has been serialized.
+
+    To illustrate, an example from a Jabber stream. Assume the start tag of the
+    root element of the stream has already been serialized, along with several
+    complete child elements, and sent off, looking like this::
+
+      <stream:stream xmlns:stream='http://etherx.jabber.org/streams'
+                     xmlns='jabber:client' to='example.com'>
+        ...
+
+    Now suppose we want to send a complete element represented by an
+    object C{message} created like:
+
+      >>> message = domish.Element((None, 'message'))
+      >>> message['to'] = 'user@example.com'
+      >>> message.addElement('body', content='Hi!')
+      <twisted.words.xish.domish.Element object at 0x8276e8c>
+      >>> message.toXml()
+      u"<message to='user@example.com'><body>Hi!</body></message>"
+
+    As, you can see, this XML snippet has no xmlns declaration. When sent
+    off, it inherits the C{jabber:client} namespace from the root element.
+    Note that this renders the same as using C{''} instead of L{None}:
+
+      >>> presence = domish.Element(('', 'presence'))
+      >>> presence.toXml()
+      u"<presence/>"
+
+    However, if this object has a parent defined, the difference becomes
+    clear:
+
+      >>> child = message.addElement(('http://example.com/', 'envelope'))
+      >>> child.addChild(presence)
+      <twisted.words.xish.domish.Element object at 0x8276fac>
+      >>> message.toXml()
+      u"<message to='user@example.com'><body>Hi!</body><envelope xmlns='http://example.com/'><presence xmlns=''/></envelope></message>"
+
+    As, you can see, the <presence/> element is now in the empty namespace, not
+    in the default namespace of the parent or the streams'.
+
+    @type uri: L{str} or None
+    @ivar uri: URI of this Element's name
+
+    @type name: L{str}
+    @ivar name: Name of this Element
+
+    @type defaultUri: L{str} or None
+    @ivar defaultUri: URI this Element exists within
+
+    @type children: L{list}
+    @ivar children: List of child Elements and content
+
+    @type parent: L{Element}
+    @ivar parent: Reference to the parent Element, if any.
+
+    @type attributes: L{dict}
+    @ivar attributes: Dictionary of attributes associated with this Element.
+
+    @type localPrefixes: L{dict}
+    @ivar localPrefixes: Dictionary of namespace declarations on this
+                         element. The key is the prefix to bind the
+                         namespace uri to.
+    """
+
+    _idCounter = 0
+
+    def __init__(self, qname, defaultUri=None, attribs=None, localPrefixes=None):
+        """
+        @param qname: Tuple of (uri, name)
+        @param defaultUri: The default URI of the element; defaults to the URI
+                           specified in C{qname}
+        @param attribs: Dictionary of attributes
+        @param localPrefixes: Dictionary of namespace declarations on this
+                              element. The key is the prefix to bind the
+                              namespace uri to.
+        """
+        self.localPrefixes = localPrefixes or {}
+        self.uri, self.name = qname
+        if defaultUri is None and self.uri not in self.localPrefixes.values():
+            self.defaultUri = self.uri
+        else:
+            self.defaultUri = defaultUri
+        self.attributes = attribs or {}
+        self.children = []
+        self.parent = None
+
+    def __getattr__(self, key):
+        # Check child list for first Element with a name matching the key
+        for n in self.children:
+            if IElement.providedBy(n) and n.name == key:
+                return n
+
+        # Tweak the behaviour so that it's more friendly about not
+        # finding elements -- we need to document this somewhere :)
+        if key.startswith("_"):
+            raise AttributeError(key)
+        else:
+            return None
+
+    def __getitem__(self, key):
+        return self.attributes[self._dqa(key)]
+
+    def __delitem__(self, key):
+        del self.attributes[self._dqa(key)]
+
+    def __setitem__(self, key, value):
+        self.attributes[self._dqa(key)] = value
+
+    def __unicode__(self):
+        """
+        Retrieve the first CData (content) node
+        """
+        for n in self.children:
+            if isinstance(n, str):
+                return n
+        return ""
+
+    def __bytes__(self):
+        """
+        Retrieve the first character data node as UTF-8 bytes.
+        """
+        return str(self).encode("utf-8")
+
+    __str__ = __unicode__
+
+    def _dqa(self, attr):
+        """Dequalify an attribute key as needed"""
+        if isinstance(attr, tuple) and not attr[0]:
+            return attr[1]
+        else:
+            return attr
+
+    def getAttribute(self, attribname, default=None):
+        """Retrieve the value of attribname, if it exists"""
+        return self.attributes.get(attribname, default)
+
+    def hasAttribute(self, attrib):
+        """Determine if the specified attribute exists"""
+        return self._dqa(attrib) in self.attributes
+
+    def compareAttribute(self, attrib, value):
+        """Safely compare the value of an attribute against a provided value.
+
+        L{None}-safe.
+        """
+        return self.attributes.get(self._dqa(attrib), None) == value
+
+    def swapAttributeValues(self, left, right):
+        """Swap the values of two attribute."""
+        d = self.attributes
+        l = d[left]
+        d[left] = d[right]
+        d[right] = l
+
+    def addChild(self, node):
+        """Add a child to this Element."""
+        if IElement.providedBy(node):
+            node.parent = self
+        self.children.append(node)
+        return node
+
+    def addContent(self, text: str) -> str:
+        """Add some text data to this Element."""
+        if not isinstance(text, str):
+            raise TypeError(f"Expected str not {text!r} ({type(text).__name__})")
+        c = self.children
+        if len(c) > 0 and isinstance(c[-1], str):
+            c[-1] = c[-1] + text
+        else:
+            c.append(text)
+        return cast(str, c[-1])
+
+    def addElement(self, name, defaultUri=None, content=None):
+        if isinstance(name, tuple):
+            if defaultUri is None:
+                defaultUri = name[0]
+            child = Element(name, defaultUri)
+        else:
+            if defaultUri is None:
+                defaultUri = self.defaultUri
+            child = Element((defaultUri, name), defaultUri)
+
+        self.addChild(child)
+
+        if content:
+            child.addContent(content)
+
+        return child
+
+    def addRawXml(self, rawxmlstring):
+        """Add a pre-serialized chunk o' XML as a child of this Element."""
+        self.children.append(SerializedXML(rawxmlstring))
+
+    def addUniqueId(self):
+        """Add a unique (across a given Python session) id attribute to this
+        Element.
+        """
+        self.attributes["id"] = "H_%d" % Element._idCounter
+        Element._idCounter = Element._idCounter + 1
+
+    def elements(self, uri=None, name=None):
+        """
+        Iterate across all children of this Element that are Elements.
+
+        Returns a generator over the child elements. If both the C{uri} and
+        C{name} parameters are set, the returned generator will only yield
+        on elements matching the qualified name.
+
+        @param uri: Optional element URI.
+        @type uri: L{str}
+        @param name: Optional element name.
+        @type name: L{str}
+        @return: Iterator that yields objects implementing L{IElement}.
+        """
+        if name is None:
+            return generateOnlyInterface(self.children, IElement)
+        else:
+            return generateElementsQNamed(self.children, name, uri)
+
+    def toXml(self, prefixes=None, closeElement=1, defaultUri="", prefixesInScope=None):
+        """Serialize this Element and all children to a string."""
+        s = SerializerClass(prefixes=prefixes, prefixesInScope=prefixesInScope)
+        s.serialize(self, closeElement=closeElement, defaultUri=defaultUri)
+        return s.getValue()
+
+    def firstChildElement(self):
+        for c in self.children:
+            if IElement.providedBy(c):
+                return c
+        return None
+
+
+class ParserError(Exception):
+    """Exception thrown when a parsing error occurs"""
+
+    pass
+
+
+def elementStream():
+    """Preferred method to construct an ElementStream
+
+    Uses Expat-based stream if available, and falls back to Sux if necessary.
+    """
+    try:
+        es = ExpatElementStream()
+        return es
+    except ImportError:
+        if SuxElementStream is None:
+            raise Exception("No parsers available :(")
+        es = SuxElementStream()
+        return es
+
+
+class SuxElementStream(sux.XMLParser):
+    def __init__(self):
+        self.connectionMade()
+        self.DocumentStartEvent = None
+        self.ElementEvent = None
+        self.DocumentEndEvent = None
+        self.currElem = None
+        self.rootElem = None
+        self.documentStarted = False
+        self.defaultNsStack = []
+        self.prefixStack = []
+
+    def parse(self, buffer):
+        try:
+            self.dataReceived(buffer)
+        except sux.ParseError as e:
+            raise ParserError(str(e))
+
+    def findUri(self, prefix):
+        # Walk prefix stack backwards, looking for the uri
+        # matching the specified prefix
+        stack = self.prefixStack
+        for i in range(-1, (len(self.prefixStack) + 1) * -1, -1):
+            if prefix in stack[i]:
+                return stack[i][prefix]
+        return None
+
+    def gotTagStart(self, name, attributes):
+        defaultUri = None
+        localPrefixes = {}
+        attribs = {}
+        uri = None
+
+        # Pass 1 - Identify namespace decls
+        for k, v in list(attributes.items()):
+            if k.startswith("xmlns"):
+                x, p = _splitPrefix(k)
+                if x is None:  # I.e.  default declaration
+                    defaultUri = v
+                else:
+                    localPrefixes[p] = v
+                del attributes[k]
+
+        # Push namespace decls onto prefix stack
+        self.prefixStack.append(localPrefixes)
+
+        # Determine default namespace for this element; if there
+        # is one
+        if defaultUri is None:
+            if len(self.defaultNsStack) > 0:
+                defaultUri = self.defaultNsStack[-1]
+            else:
+                defaultUri = ""
+
+        # Fix up name
+        prefix, name = _splitPrefix(name)
+        if prefix is None:  # This element is in the default namespace
+            uri = defaultUri
+        else:
+            # Find the URI for the prefix
+            uri = self.findUri(prefix)
+
+        # Pass 2 - Fix up and escape attributes
+        for k, v in attributes.items():
+            p, n = _splitPrefix(k)
+            if p is None:
+                attribs[n] = v
+            else:
+                attribs[(self.findUri(p)), n] = unescapeFromXml(v)
+
+        # Construct the actual Element object
+        e = Element((uri, name), defaultUri, attribs, localPrefixes)
+
+        # Save current default namespace
+        self.defaultNsStack.append(defaultUri)
+
+        # Document already started
+        if self.documentStarted:
+            # Starting a new packet
+            if self.currElem is None:
+                self.currElem = e
+            # Adding to existing element
+            else:
+                self.currElem = self.currElem.addChild(e)
+        # New document
+        else:
+            self.rootElem = e
+            self.documentStarted = True
+            self.DocumentStartEvent(e)
+
+    def gotText(self, data):
+        if self.currElem is not None:
+            if isinstance(data, bytes):
+                data = data.decode("ascii")
+            self.currElem.addContent(data)
+
+    def gotCData(self, data):
+        if self.currElem is not None:
+            if isinstance(data, bytes):
+                data = data.decode("ascii")
+            self.currElem.addContent(data)
+
+    def gotComment(self, data):
+        # Ignore comments for the moment
+        pass
+
+    entities = {
+        "amp": "&",
+        "lt": "<",
+        "gt": ">",
+        "apos": "'",
+        "quot": '"',
+    }
+
+    def gotEntityReference(self, entityRef):
+        # If this is an entity we know about, add it as content
+        # to the current element
+        if entityRef in SuxElementStream.entities:
+            data = SuxElementStream.entities[entityRef]
+            if isinstance(data, bytes):
+                data = data.decode("ascii")
+            self.currElem.addContent(data)
+
+    def gotTagEnd(self, name):
+        # Ensure the document hasn't already ended
+        if self.rootElem is None:
+            # XXX: Write more legible explanation
+            raise ParserError("Element closed after end of document.")
+
+        # Fix up name
+        prefix, name = _splitPrefix(name)
+        if prefix is None:
+            uri = self.defaultNsStack[-1]
+        else:
+            uri = self.findUri(prefix)
+
+        # End of document
+        if self.currElem is None:
+            # Ensure element name and uri matches
+            if self.rootElem.name != name or self.rootElem.uri != uri:
+                raise ParserError("Mismatched root elements")
+            self.DocumentEndEvent()
+            self.rootElem = None
+
+        # Other elements
+        else:
+            # Ensure the tag being closed matches the name of the current
+            # element
+            if self.currElem.name != name or self.currElem.uri != uri:
+                # XXX: Write more legible explanation
+                raise ParserError("Malformed element close")
+
+            # Pop prefix and default NS stack
+            self.prefixStack.pop()
+            self.defaultNsStack.pop()
+
+            # Check for parent null parent of current elem;
+            # that's the top of the stack
+            if self.currElem.parent is None:
+                self.currElem.parent = self.rootElem
+                self.ElementEvent(self.currElem)
+                self.currElem = None
+
+            # Anything else is just some element wrapping up
+            else:
+                self.currElem = self.currElem.parent
+
+
+class ExpatElementStream:
+    def __init__(self):
+        import pyexpat
+
+        self.DocumentStartEvent = None
+        self.ElementEvent = None
+        self.DocumentEndEvent = None
+        self.error = pyexpat.error
+        self.parser = pyexpat.ParserCreate("UTF-8", " ")
+        self.parser.StartElementHandler = self._onStartElement
+        self.parser.EndElementHandler = self._onEndElement
+        self.parser.CharacterDataHandler = self._onCdata
+        self.parser.StartNamespaceDeclHandler = self._onStartNamespace
+        self.parser.EndNamespaceDeclHandler = self._onEndNamespace
+        self.currElem = None
+        self.defaultNsStack = [""]
+        self.documentStarted = 0
+        self.localPrefixes = {}
+
+    def parse(self, buffer):
+        try:
+            self.parser.Parse(buffer)
+        except self.error as e:
+            raise ParserError(str(e))
+
+    def _onStartElement(self, name, attrs):
+        # Generate a qname tuple from the provided name.  See
+        # http://docs.python.org/library/pyexpat.html#xml.parsers.expat.ParserCreate
+        # for an explanation of the formatting of name.
+        qname = name.rsplit(" ", 1)
+        if len(qname) == 1:
+            qname = ("", name)
+
+        # Process attributes
+        newAttrs = {}
+        toDelete = []
+        for k, v in attrs.items():
+            if " " in k:
+                aqname = k.rsplit(" ", 1)
+                newAttrs[(aqname[0], aqname[1])] = v
+                toDelete.append(k)
+
+        attrs.update(newAttrs)
+
+        for k in toDelete:
+            del attrs[k]
+
+        # Construct the new element
+        e = Element(qname, self.defaultNsStack[-1], attrs, self.localPrefixes)
+        self.localPrefixes = {}
+
+        # Document already started
+        if self.documentStarted == 1:
+            if self.currElem != None:
+                self.currElem.children.append(e)
+                e.parent = self.currElem
+            self.currElem = e
+
+        # New document
+        else:
+            self.documentStarted = 1
+            self.DocumentStartEvent(e)
+
+    def _onEndElement(self, _):
+        # Check for null current elem; end of doc
+        if self.currElem is None:
+            self.DocumentEndEvent()
+
+        # Check for parent that is None; that's
+        # the top of the stack
+        elif self.currElem.parent is None:
+            self.ElementEvent(self.currElem)
+            self.currElem = None
+
+        # Anything else is just some element in the current
+        # packet wrapping up
+        else:
+            self.currElem = self.currElem.parent
+
+    def _onCdata(self, data):
+        if self.currElem != None:
+            self.currElem.addContent(data)
+
+    def _onStartNamespace(self, prefix, uri):
+        # If this is the default namespace, put
+        # it on the stack
+        if prefix is None:
+            self.defaultNsStack.append(uri)
+        else:
+            self.localPrefixes[prefix] = uri
+
+    def _onEndNamespace(self, prefix):
+        # Remove last element on the stack
+        if prefix is None:
+            self.defaultNsStack.pop()
+
+
+## class FileParser(ElementStream):
+##     def __init__(self):
+##         ElementStream.__init__(self)
+##         self.DocumentStartEvent = self.docStart
+##         self.ElementEvent = self.elem
+##         self.DocumentEndEvent = self.docEnd
+##         self.done = 0
+
+##     def docStart(self, elem):
+##         self.document = elem
+
+##     def elem(self, elem):
+##         self.document.addChild(elem)
+
+##     def docEnd(self):
+##         self.done = 1
+
+##     def parse(self, filename):
+##         with open(filename) as f:
+##             for l in f.readlines():
+##                 self.parser.Parse(l)
+##         assert self.done == 1
+##         return self.document
+
+## def parseFile(filename):
+##     return FileParser().parse(filename)
diff --git a/contrib/python/Twisted/py3/twisted/words/xish/utility.py b/contrib/python/Twisted/py3/twisted/words/xish/utility.py
new file mode 100644
index 00000000000..30a44a0b992
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/xish/utility.py
@@ -0,0 +1,364 @@
+# -*- test-case-name: twisted.words.test.test_xishutil -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Event Dispatching and Callback utilities.
+"""
+
+
+from twisted.python import log
+from twisted.words.xish import xpath
+
+
+class _MethodWrapper:
+    """
+    Internal class for tracking method calls.
+    """
+
+    def __init__(self, method, *args, **kwargs):
+        self.method = method
+        self.args = args
+        self.kwargs = kwargs
+
+    def __call__(self, *args, **kwargs):
+        nargs = self.args + args
+        nkwargs = self.kwargs.copy()
+        nkwargs.update(kwargs)
+        self.method(*nargs, **nkwargs)
+
+
+class CallbackList:
+    """
+    Container for callbacks.
+
+    Event queries are linked to lists of callables. When a matching event
+    occurs, these callables are called in sequence. One-time callbacks
+    are removed from the list after the first time the event was triggered.
+
+    Arguments to callbacks are split spread across two sets. The first set,
+    callback specific, is passed to C{addCallback} and is used for all
+    subsequent event triggers.  The second set is passed to C{callback} and is
+    event specific. Positional arguments in the second set come after the
+    positional arguments of the first set. Keyword arguments in the second set
+    override those in the first set.
+
+    @ivar callbacks: The registered callbacks as mapping from the callable to a
+                     tuple of a wrapper for that callable that keeps the
+                     callback specific arguments and a boolean that signifies
+                     if it is to be called only once.
+    @type callbacks: C{dict}
+    """
+
+    def __init__(self):
+        self.callbacks = {}
+
+    def addCallback(self, onetime, method, *args, **kwargs):
+        """
+        Add callback.
+
+        The arguments passed are used as callback specific arguments.
+
+        @param onetime: If C{True}, this callback is called at most once.
+        @type onetime: C{bool}
+        @param method: The callback callable to be added.
+        @param args: Positional arguments to the callable.
+        @type args: C{list}
+        @param kwargs: Keyword arguments to the callable.
+        @type kwargs: C{dict}
+        """
+
+        if method not in self.callbacks:
+            self.callbacks[method] = (_MethodWrapper(method, *args, **kwargs), onetime)
+
+    def removeCallback(self, method):
+        """
+        Remove callback.
+
+        @param method: The callable to be removed.
+        """
+
+        if method in self.callbacks:
+            del self.callbacks[method]
+
+    def callback(self, *args, **kwargs):
+        """
+        Call all registered callbacks.
+
+        The passed arguments are event specific and augment and override
+        the callback specific arguments as described above.
+
+        @note: Exceptions raised by callbacks are trapped and logged. They will
+               not propagate up to make sure other callbacks will still be
+               called, and the event dispatching always succeeds.
+
+        @param args: Positional arguments to the callable.
+        @type args: C{list}
+        @param kwargs: Keyword arguments to the callable.
+        @type kwargs: C{dict}
+        """
+
+        for key, (methodwrapper, onetime) in list(self.callbacks.items()):
+            try:
+                methodwrapper(*args, **kwargs)
+            except BaseException:
+                log.err()
+
+            if onetime:
+                del self.callbacks[key]
+
+    def isEmpty(self):
+        """
+        Return if list of registered callbacks is empty.
+
+        @rtype: C{bool}
+        """
+
+        return len(self.callbacks) == 0
+
+
+class EventDispatcher:
+    """
+    Event dispatching service.
+
+    The C{EventDispatcher} allows observers to be registered for certain events
+    that are dispatched. There are two types of events: XPath events and Named
+    events.
+
+    Every dispatch is triggered by calling L{dispatch} with a data object and,
+    for named events, the name of the event.
+
+    When an XPath type event is dispatched, the associated object is assumed to
+    be an L{Element<twisted.words.xish.domish.Element>} instance, which is
+    matched against all registered XPath queries. For every match, the
+    respective observer will be called with the data object.
+
+    A named event will simply call each registered observer for that particular
+    event name, with the data object. Unlike XPath type events, the data object
+    is not restricted to L{Element<twisted.words.xish.domish.Element>}, but can
+    be anything.
+
+    When registering observers, the event that is to be observed is specified
+    using an L{xpath.XPathQuery} instance or a string. In the latter case, the
+    string can also contain the string representation of an XPath expression.
+    To distinguish these from named events, each named event should start with
+    a special prefix that is stored in C{self.prefix}. It defaults to
+    C{//event/}.
+
+    Observers registered using L{addObserver} are persistent: after the
+    observer has been triggered by a dispatch, it remains registered for a
+    possible next dispatch. If instead L{addOnetimeObserver} was used to
+    observe an event, the observer is removed from the list of observers after
+    the first observed event.
+
+    Observers can also be prioritized, by providing an optional C{priority}
+    parameter to the L{addObserver} and L{addOnetimeObserver} methods. Higher
+    priority observers are then called before lower priority observers.
+
+    Finally, observers can be unregistered by using L{removeObserver}.
+    """
+
+    def __init__(self, eventprefix="//event/"):
+        self.prefix = eventprefix
+        self._eventObservers = {}
+        self._xpathObservers = {}
+        self._dispatchDepth = 0  # Flag indicating levels of dispatching
+        # in progress
+        self._updateQueue = []  # Queued updates for observer ops
+
+    def _getEventAndObservers(self, event):
+        if isinstance(event, xpath.XPathQuery):
+            # Treat as xpath
+            observers = self._xpathObservers
+        else:
+            if self.prefix == event[: len(self.prefix)]:
+                # Treat as event
+                observers = self._eventObservers
+            else:
+                # Treat as xpath
+                event = xpath.internQuery(event)
+                observers = self._xpathObservers
+
+        return event, observers
+
+    def addOnetimeObserver(self, event, observerfn, priority=0, *args, **kwargs):
+        """
+        Register a one-time observer for an event.
+
+        Like L{addObserver}, but is only triggered at most once. See there
+        for a description of the parameters.
+        """
+        self._addObserver(True, event, observerfn, priority, *args, **kwargs)
+
+    def addObserver(self, event, observerfn, priority=0, *args, **kwargs):
+        """
+        Register an observer for an event.
+
+        Each observer will be registered with a certain priority. Higher
+        priority observers get called before lower priority observers.
+
+        @param event: Name or XPath query for the event to be monitored.
+        @type event: C{str} or L{xpath.XPathQuery}.
+        @param observerfn: Function to be called when the specified event
+                           has been triggered. This callable takes
+                           one parameter: the data object that triggered
+                           the event. When specified, the C{*args} and
+                           C{**kwargs} parameters to addObserver are being used
+                           as additional parameters to the registered observer
+                           callable.
+        @param priority: (Optional) priority of this observer in relation to
+                         other observer that match the same event. Defaults to
+                         C{0}.
+        @type priority: C{int}
+        """
+        self._addObserver(False, event, observerfn, priority, *args, **kwargs)
+
+    def _addObserver(self, onetime, event, observerfn, priority, *args, **kwargs):
+        # If this is happening in the middle of the dispatch, queue
+        # it up for processing after the dispatch completes
+        if self._dispatchDepth > 0:
+            self._updateQueue.append(
+                lambda: self._addObserver(
+                    onetime, event, observerfn, priority, *args, **kwargs
+                )
+            )
+            return
+
+        event, observers = self._getEventAndObservers(event)
+
+        if priority not in observers:
+            cbl = CallbackList()
+            observers[priority] = {event: cbl}
+        else:
+            priorityObservers = observers[priority]
+            if event not in priorityObservers:
+                cbl = CallbackList()
+                observers[priority][event] = cbl
+            else:
+                cbl = priorityObservers[event]
+
+        cbl.addCallback(onetime, observerfn, *args, **kwargs)
+
+    def removeObserver(self, event, observerfn):
+        """
+        Remove callable as observer for an event.
+
+        The observer callable is removed for all priority levels for the
+        specified event.
+
+        @param event: Event for which the observer callable was registered.
+        @type event: C{str} or L{xpath.XPathQuery}
+        @param observerfn: Observer callable to be unregistered.
+        """
+
+        # If this is happening in the middle of the dispatch, queue
+        # it up for processing after the dispatch completes
+        if self._dispatchDepth > 0:
+            self._updateQueue.append(lambda: self.removeObserver(event, observerfn))
+            return
+
+        event, observers = self._getEventAndObservers(event)
+
+        emptyLists = []
+        for priority, priorityObservers in observers.items():
+            for query, callbacklist in priorityObservers.items():
+                if event == query:
+                    callbacklist.removeCallback(observerfn)
+                    if callbacklist.isEmpty():
+                        emptyLists.append((priority, query))
+
+        for priority, query in emptyLists:
+            del observers[priority][query]
+
+    def dispatch(self, obj, event=None):
+        """
+        Dispatch an event.
+
+        When C{event} is L{None}, an XPath type event is triggered, and
+        C{obj} is assumed to be an instance of
+        L{Element<twisted.words.xish.domish.Element>}. Otherwise, C{event}
+        holds the name of the named event being triggered. In the latter case,
+        C{obj} can be anything.
+
+        @param obj: The object to be dispatched.
+        @param event: Optional event name.
+        @type event: C{str}
+        """
+
+        foundTarget = False
+
+        self._dispatchDepth += 1
+
+        if event != None:
+            # Named event
+            observers = self._eventObservers
+            match = lambda query, obj: query == event
+        else:
+            # XPath event
+            observers = self._xpathObservers
+            match = lambda query, obj: query.matches(obj)
+
+        priorities = list(observers.keys())
+        priorities.sort()
+        priorities.reverse()
+
+        emptyLists = []
+        for priority in priorities:
+            for query, callbacklist in observers[priority].items():
+                if match(query, obj):
+                    callbacklist.callback(obj)
+                    foundTarget = True
+                    if callbacklist.isEmpty():
+                        emptyLists.append((priority, query))
+
+        for priority, query in emptyLists:
+            del observers[priority][query]
+
+        self._dispatchDepth -= 1
+
+        # If this is a dispatch within a dispatch, don't
+        # do anything with the updateQueue -- it needs to
+        # wait until we've back all the way out of the stack
+        if self._dispatchDepth == 0:
+            # Deal with pending update operations
+            for f in self._updateQueue:
+                f()
+            self._updateQueue = []
+
+        return foundTarget
+
+
+class XmlPipe:
+    """
+    XML stream pipe.
+
+    Connects two objects that communicate stanzas through an XML stream like
+    interface. Each of the ends of the pipe (sink and source) can be used to
+    send XML stanzas to the other side, or add observers to process XML stanzas
+    that were sent from the other side.
+
+    XML pipes are usually used in place of regular XML streams that are
+    transported over TCP. This is the reason for the use of the names source
+    and sink for both ends of the pipe. The source side corresponds with the
+    entity that initiated the TCP connection, whereas the sink corresponds with
+    the entity that accepts that connection. In this object, though, the source
+    and sink are treated equally.
+
+    Unlike Jabber
+    L{XmlStream<twisted.words.protocols.jabber.xmlstream.XmlStream>}s, the sink
+    and source objects are assumed to represent an eternal connected and
+    initialized XML stream. As such, events corresponding to connection,
+    disconnection, initialization and stream errors are not dispatched or
+    processed.
+
+    @since: 8.2
+    @ivar source: Source XML stream.
+    @ivar sink: Sink XML stream.
+    """
+
+    def __init__(self):
+        self.source = EventDispatcher()
+        self.sink = EventDispatcher()
+        self.source.send = lambda obj: self.sink.dispatch(obj)
+        self.sink.send = lambda obj: self.source.dispatch(obj)
diff --git a/contrib/python/Twisted/py3/twisted/words/xish/xmlstream.py b/contrib/python/Twisted/py3/twisted/words/xish/xmlstream.py
new file mode 100644
index 00000000000..a2b9890939b
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/xish/xmlstream.py
@@ -0,0 +1,274 @@
+# -*- test-case-name: twisted.words.test.test_xmlstream -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XML Stream processing.
+
+An XML Stream is defined as a connection over which two XML documents are
+exchanged during the lifetime of the connection, one for each direction. The
+unit of interaction is a direct child element of the root element (stanza).
+
+The most prominent use of XML Streams is Jabber, but this module is generically
+usable. See Twisted Words for Jabber specific protocol support.
+
+Maintainer: Ralph Meijer
+
+@var STREAM_CONNECTED_EVENT: This event signals that the connection has been
+    established.
+@type STREAM_CONNECTED_EVENT: L{str}.
+
+@var STREAM_END_EVENT: This event signals that the connection has been closed.
+@type STREAM_END_EVENT: L{str}.
+
+@var STREAM_ERROR_EVENT: This event signals that a parse error occurred.
+@type STREAM_ERROR_EVENT: L{str}.
+
+@var STREAM_START_EVENT: This event signals that the root element of the XML
+    Stream has been received.
+    For XMPP, this would be the C{<stream:stream ...>} opening tag.
+@type STREAM_START_EVENT: L{str}.
+"""
+
+
+from sys import intern
+from typing import Type
+
+from twisted.internet import protocol
+from twisted.python import failure
+from twisted.words.xish import domish, utility
+
+STREAM_CONNECTED_EVENT = intern("//event/stream/connected")
+STREAM_START_EVENT = intern("//event/stream/start")
+STREAM_END_EVENT = intern("//event/stream/end")
+STREAM_ERROR_EVENT = intern("//event/stream/error")
+
+
+class XmlStream(protocol.Protocol, utility.EventDispatcher):
+    """Generic Streaming XML protocol handler.
+
+    This protocol handler will parse incoming data as XML and dispatch events
+    accordingly. Incoming stanzas can be handled by registering observers using
+    XPath-like expressions that are matched against each stanza. See
+    L{utility.EventDispatcher} for details.
+    """
+
+    def __init__(self):
+        utility.EventDispatcher.__init__(self)
+        self.stream = None
+        self.rawDataOutFn = None
+        self.rawDataInFn = None
+
+    def _initializeStream(self):
+        """Sets up XML Parser."""
+        self.stream = domish.elementStream()
+        self.stream.DocumentStartEvent = self.onDocumentStart
+        self.stream.ElementEvent = self.onElement
+        self.stream.DocumentEndEvent = self.onDocumentEnd
+
+    ### --------------------------------------------------------------
+    ###
+    ### Protocol events
+    ###
+    ### --------------------------------------------------------------
+
+    def connectionMade(self):
+        """Called when a connection is made.
+
+        Sets up the XML parser and dispatches the L{STREAM_CONNECTED_EVENT}
+        event indicating the connection has been established.
+        """
+        self._initializeStream()
+        self.dispatch(self, STREAM_CONNECTED_EVENT)
+
+    def dataReceived(self, data):
+        """Called whenever data is received.
+
+        Passes the data to the XML parser. This can result in calls to the
+        DOM handlers. If a parse error occurs, the L{STREAM_ERROR_EVENT} event
+        is called to allow for cleanup actions, followed by dropping the
+        connection.
+        """
+        try:
+            if self.rawDataInFn:
+                self.rawDataInFn(data)
+            self.stream.parse(data)
+        except domish.ParserError:
+            self.dispatch(failure.Failure(), STREAM_ERROR_EVENT)
+            self.transport.loseConnection()
+
+    def connectionLost(self, reason):
+        """Called when the connection is shut down.
+
+        Dispatches the L{STREAM_END_EVENT}.
+        """
+        self.dispatch(reason, STREAM_END_EVENT)
+        self.stream = None
+
+    ### --------------------------------------------------------------
+    ###
+    ### DOM events
+    ###
+    ### --------------------------------------------------------------
+
+    def onDocumentStart(self, rootElement):
+        """Called whenever the start tag of a root element has been received.
+
+        Dispatches the L{STREAM_START_EVENT}.
+        """
+        self.dispatch(self, STREAM_START_EVENT)
+
+    def onElement(self, element):
+        """Called whenever a direct child element of the root element has
+        been received.
+
+        Dispatches the received element.
+        """
+        self.dispatch(element)
+
+    def onDocumentEnd(self):
+        """Called whenever the end tag of the root element has been received.
+
+        Closes the connection. This causes C{connectionLost} being called.
+        """
+        self.transport.loseConnection()
+
+    def setDispatchFn(self, fn):
+        """Set another function to handle elements."""
+        self.stream.ElementEvent = fn
+
+    def resetDispatchFn(self):
+        """Set the default function (C{onElement}) to handle elements."""
+        self.stream.ElementEvent = self.onElement
+
+    def send(self, obj):
+        """Send data over the stream.
+
+        Sends the given C{obj} over the connection. C{obj} may be instances of
+        L{domish.Element}, C{unicode} and C{str}. The first two will be
+        properly serialized and/or encoded. C{str} objects must be in UTF-8
+        encoding.
+
+        Note: because it is easy to make mistakes in maintaining a properly
+        encoded C{str} object, it is advised to use C{unicode} objects
+        everywhere when dealing with XML Streams.
+
+        @param obj: Object to be sent over the stream.
+        @type obj: L{domish.Element}, L{domish} or C{str}
+
+        """
+        if domish.IElement.providedBy(obj):
+            obj = obj.toXml()
+
+        if isinstance(obj, str):
+            obj = obj.encode("utf-8")
+
+        if self.rawDataOutFn:
+            self.rawDataOutFn(obj)
+
+        self.transport.write(obj)
+
+
+class BootstrapMixin:
+    """
+    XmlStream factory mixin to install bootstrap event observers.
+
+    This mixin is for factories providing
+    L{IProtocolFactory<twisted.internet.interfaces.IProtocolFactory>} to make
+    sure bootstrap event observers are set up on protocols, before incoming
+    data is processed. Such protocols typically derive from
+    L{utility.EventDispatcher}, like L{XmlStream}.
+
+    You can set up bootstrap event observers using C{addBootstrap}. The
+    C{event} and C{fn} parameters correspond with the C{event} and
+    C{observerfn} arguments to L{utility.EventDispatcher.addObserver}.
+
+    @since: 8.2.
+    @ivar bootstraps: The list of registered bootstrap event observers.
+    @type bootstrap: C{list}
+    """
+
+    def __init__(self):
+        self.bootstraps = []
+
+    def installBootstraps(self, dispatcher):
+        """
+        Install registered bootstrap observers.
+
+        @param dispatcher: Event dispatcher to add the observers to.
+        @type dispatcher: L{utility.EventDispatcher}
+        """
+        for event, fn in self.bootstraps:
+            dispatcher.addObserver(event, fn)
+
+    def addBootstrap(self, event, fn):
+        """
+        Add a bootstrap event handler.
+
+        @param event: The event to register an observer for.
+        @type event: C{str} or L{xpath.XPathQuery}
+        @param fn: The observer callable to be registered.
+        """
+        self.bootstraps.append((event, fn))
+
+    def removeBootstrap(self, event, fn):
+        """
+        Remove a bootstrap event handler.
+
+        @param event: The event the observer is registered for.
+        @type event: C{str} or L{xpath.XPathQuery}
+        @param fn: The registered observer callable.
+        """
+        self.bootstraps.remove((event, fn))
+
+
+class XmlStreamFactoryMixin(BootstrapMixin):
+    """
+    XmlStream factory mixin that takes care of event handlers.
+
+    All positional and keyword arguments passed to create this factory are
+    passed on as-is to the protocol.
+
+    @ivar args: Positional arguments passed to the protocol upon instantiation.
+    @type args: C{tuple}.
+    @ivar kwargs: Keyword arguments passed to the protocol upon instantiation.
+    @type kwargs: C{dict}.
+    """
+
+    def __init__(self, *args, **kwargs):
+        BootstrapMixin.__init__(self)
+        self.args = args
+        self.kwargs = kwargs
+
+    def buildProtocol(self, addr):
+        """
+        Create an instance of XmlStream.
+
+        The returned instance will have bootstrap event observers registered
+        and will proceed to handle input on an incoming connection.
+        """
+        xs = self.protocol(*self.args, **self.kwargs)
+        xs.factory = self
+        self.installBootstraps(xs)
+        return xs
+
+
+class XmlStreamFactory(XmlStreamFactoryMixin, protocol.ReconnectingClientFactory):
+    """
+    Factory for XmlStream protocol objects as a reconnection client.
+    """
+
+    protocol: "Type[protocol.Protocol]" = XmlStream
+
+    def buildProtocol(self, addr):
+        """
+        Create a protocol instance.
+
+        Overrides L{XmlStreamFactoryMixin.buildProtocol} to work with
+        a L{ReconnectingClientFactory}. As this is called upon having an
+        connection established, we are resetting the delay for reconnection
+        attempts when the connection is lost again.
+        """
+        self.resetDelay()
+        return XmlStreamFactoryMixin.buildProtocol(self, addr)
diff --git a/contrib/python/Twisted/py3/twisted/words/xish/xpath.py b/contrib/python/Twisted/py3/twisted/words/xish/xpath.py
new file mode 100644
index 00000000000..418febdd016
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/xish/xpath.py
@@ -0,0 +1,337 @@
+# -*- test-case-name: twisted.words.test.test_xpath -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+XPath query support.
+
+This module provides L{XPathQuery} to match
+L{domish.Element<twisted.words.xish.domish.Element>} instances against
+XPath-like expressions.
+"""
+
+
+from io import StringIO
+
+
+class LiteralValue(str):
+    def value(self, elem):
+        return self
+
+
+class IndexValue:
+    def __init__(self, index):
+        self.index = int(index) - 1
+
+    def value(self, elem):
+        return elem.children[self.index]
+
+
+class AttribValue:
+    def __init__(self, attribname):
+        self.attribname = attribname
+        if self.attribname == "xmlns":
+            self.value = self.value_ns
+
+    def value_ns(self, elem):
+        return elem.uri
+
+    def value(self, elem):
+        if self.attribname in elem.attributes:
+            return elem.attributes[self.attribname]
+        else:
+            return None
+
+
+class CompareValue:
+    def __init__(self, lhs, op, rhs):
+        self.lhs = lhs
+        self.rhs = rhs
+        if op == "=":
+            self.value = self._compareEqual
+        else:
+            self.value = self._compareNotEqual
+
+    def _compareEqual(self, elem):
+        return self.lhs.value(elem) == self.rhs.value(elem)
+
+    def _compareNotEqual(self, elem):
+        return self.lhs.value(elem) != self.rhs.value(elem)
+
+
+class BooleanValue:
+    """
+    Provide boolean XPath expression operators.
+
+    @ivar lhs: Left hand side expression of the operator.
+    @ivar op: The operator. One of C{'and'}, C{'or'}.
+    @ivar rhs: Right hand side expression of the operator.
+    @ivar value: Reference to the method that will calculate the value of
+                 this expression given an element.
+    """
+
+    def __init__(self, lhs, op, rhs):
+        self.lhs = lhs
+        self.rhs = rhs
+        if op == "and":
+            self.value = self._booleanAnd
+        else:
+            self.value = self._booleanOr
+
+    def _booleanAnd(self, elem):
+        """
+        Calculate boolean and of the given expressions given an element.
+
+        @param elem: The element to calculate the value of the expression from.
+        """
+        return self.lhs.value(elem) and self.rhs.value(elem)
+
+    def _booleanOr(self, elem):
+        """
+        Calculate boolean or of the given expressions given an element.
+
+        @param elem: The element to calculate the value of the expression from.
+        """
+        return self.lhs.value(elem) or self.rhs.value(elem)
+
+
+def Function(fname):
+    """
+    Internal method which selects the function object
+    """
+    klassname = "_%s_Function" % fname
+    c = globals()[klassname]()
+    return c
+
+
+class _not_Function:
+    def __init__(self):
+        self.baseValue = None
+
+    def setParams(self, baseValue):
+        self.baseValue = baseValue
+
+    def value(self, elem):
+        return not self.baseValue.value(elem)
+
+
+class _text_Function:
+    def setParams(self):
+        pass
+
+    def value(self, elem):
+        return str(elem)
+
+
+class _Location:
+    def __init__(self):
+        self.predicates = []
+        self.elementName = None
+        self.childLocation = None
+
+    def matchesPredicates(self, elem):
+        if self.elementName != None and self.elementName != elem.name:
+            return 0
+
+        for p in self.predicates:
+            if not p.value(elem):
+                return 0
+
+        return 1
+
+    def matches(self, elem):
+        if not self.matchesPredicates(elem):
+            return 0
+
+        if self.childLocation != None:
+            for c in elem.elements():
+                if self.childLocation.matches(c):
+                    return 1
+        else:
+            return 1
+
+        return 0
+
+    def queryForString(self, elem, resultbuf):
+        if not self.matchesPredicates(elem):
+            return
+
+        if self.childLocation != None:
+            for c in elem.elements():
+                self.childLocation.queryForString(c, resultbuf)
+        else:
+            resultbuf.write(str(elem))
+
+    def queryForNodes(self, elem, resultlist):
+        if not self.matchesPredicates(elem):
+            return
+
+        if self.childLocation != None:
+            for c in elem.elements():
+                self.childLocation.queryForNodes(c, resultlist)
+        else:
+            resultlist.append(elem)
+
+    def queryForStringList(self, elem, resultlist):
+        if not self.matchesPredicates(elem):
+            return
+
+        if self.childLocation != None:
+            for c in elem.elements():
+                self.childLocation.queryForStringList(c, resultlist)
+        else:
+            for c in elem.children:
+                if isinstance(c, str):
+                    resultlist.append(c)
+
+
+class _AnyLocation:
+    def __init__(self):
+        self.predicates = []
+        self.elementName = None
+        self.childLocation = None
+
+    def matchesPredicates(self, elem):
+        for p in self.predicates:
+            if not p.value(elem):
+                return 0
+        return 1
+
+    def listParents(self, elem, parentlist):
+        if elem.parent != None:
+            self.listParents(elem.parent, parentlist)
+        parentlist.append(elem.name)
+
+    def isRootMatch(self, elem):
+        if (
+            self.elementName == None or self.elementName == elem.name
+        ) and self.matchesPredicates(elem):
+            if self.childLocation != None:
+                for c in elem.elements():
+                    if self.childLocation.matches(c):
+                        return True
+            else:
+                return True
+        return False
+
+    def findFirstRootMatch(self, elem):
+        if (
+            self.elementName == None or self.elementName == elem.name
+        ) and self.matchesPredicates(elem):
+            # Thus far, the name matches and the predicates match,
+            # now check into the children and find the first one
+            # that matches the rest of the structure
+            # the rest of the structure
+            if self.childLocation != None:
+                for c in elem.elements():
+                    if self.childLocation.matches(c):
+                        return c
+                return None
+            else:
+                # No children locations; this is a match!
+                return elem
+        else:
+            # Ok, predicates or name didn't match, so we need to start
+            # down each child and treat it as the root and try
+            # again
+            for c in elem.elements():
+                if self.matches(c):
+                    return c
+            # No children matched...
+            return None
+
+    def matches(self, elem):
+        if self.isRootMatch(elem):
+            return True
+        else:
+            # Ok, initial element isn't an exact match, walk
+            # down each child and treat it as the root and try
+            # again
+            for c in elem.elements():
+                if self.matches(c):
+                    return True
+            # No children matched...
+            return False
+
+    def queryForString(self, elem, resultbuf):
+        raise NotImplementedError("queryForString is not implemented for any location")
+
+    def queryForNodes(self, elem, resultlist):
+        # First check to see if _this_ element is a root
+        if self.isRootMatch(elem):
+            resultlist.append(elem)
+
+        # Now check each child
+        for c in elem.elements():
+            self.queryForNodes(c, resultlist)
+
+    def queryForStringList(self, elem, resultlist):
+        if self.isRootMatch(elem):
+            for c in elem.children:
+                if isinstance(c, str):
+                    resultlist.append(c)
+        for c in elem.elements():
+            self.queryForStringList(c, resultlist)
+
+
+class XPathQuery:
+    def __init__(self, queryStr):
+        self.queryStr = queryStr
+        # Prevent a circular import issue, as xpathparser imports this module.
+        from twisted.words.xish.xpathparser import XPathParser, XPathParserScanner
+
+        parser = XPathParser(XPathParserScanner(queryStr))
+        self.baseLocation = getattr(parser, "XPATH")()
+
+    def __hash__(self):
+        return self.queryStr.__hash__()
+
+    def matches(self, elem):
+        return self.baseLocation.matches(elem)
+
+    def queryForString(self, elem):
+        result = StringIO()
+        self.baseLocation.queryForString(elem, result)
+        return result.getvalue()
+
+    def queryForNodes(self, elem):
+        result = []
+        self.baseLocation.queryForNodes(elem, result)
+        if len(result) == 0:
+            return None
+        else:
+            return result
+
+    def queryForStringList(self, elem):
+        result = []
+        self.baseLocation.queryForStringList(elem, result)
+        if len(result) == 0:
+            return None
+        else:
+            return result
+
+
+__internedQueries = {}
+
+
+def internQuery(queryString):
+    if queryString not in __internedQueries:
+        __internedQueries[queryString] = XPathQuery(queryString)
+    return __internedQueries[queryString]
+
+
+def matches(xpathstr, elem):
+    return internQuery(xpathstr).matches(elem)
+
+
+def queryForStringList(xpathstr, elem):
+    return internQuery(xpathstr).queryForStringList(elem)
+
+
+def queryForString(xpathstr, elem):
+    return internQuery(xpathstr).queryForString(elem)
+
+
+def queryForNodes(xpathstr, elem):
+    return internQuery(xpathstr).queryForNodes(elem)
diff --git a/contrib/python/Twisted/py3/twisted/words/xish/xpathparser.g b/contrib/python/Twisted/py3/twisted/words/xish/xpathparser.g
new file mode 100644
index 00000000000..4c783523814
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/xish/xpathparser.g
@@ -0,0 +1,524 @@
+# -*- test-case-name: twisted.words.test.test_xpath -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+# pylint: disable=W9401,W9402
+
+# DO NOT EDIT xpathparser.py!
+#
+# It is generated from xpathparser.g using Yapps. Make needed changes there.
+# This also means that the generated Python may not conform to Twisted's coding
+# standards, so it is wrapped in exec to prevent automated checkers from
+# complaining.
+
+# HOWTO Generate me:
+#
+# 1.) Grab a copy of yapps2:
+#         https://github.com/smurfix/yapps
+#
+#     Note: Do NOT use the package in debian/ubuntu as it has incompatible
+#     modifications. The original at http://theory.stanford.edu/~amitp/yapps/
+#     hasn't been touched since 2003 and has not been updated to work with
+#     Python 3.
+#
+# 2.) Generate the grammar:
+#
+#         yapps2 xpathparser.g xpathparser.py.proto
+#
+# 3.) Edit the output to depend on the embedded runtime, and remove extraneous
+#     imports:
+#
+#         sed -e '/^# Begin/,${/^[^ ].*mport/d}' -e 's/runtime\.//g' \
+#             -e "s/^\(from __future\)/exec(r'''\n\1/" -e"\$a''')"
+#             xpathparser.py.proto > xpathparser.py
+
+"""
+XPath Parser.
+
+Besides the parser code produced by Yapps, this module also defines the
+parse-time exception classes, a scanner class, a base class for parsers
+produced by Yapps, and a context class that keeps track of the parse stack.
+These have been copied from the Yapps runtime module.
+"""
+
+from __future__ import print_function
+import sys, re
+
+MIN_WINDOW=4096
+# File lookup window
+
+class SyntaxError(Exception):
+    """When we run into an unexpected token, this is the exception to use"""
+    def __init__(self, pos=None, msg="Bad Token", context=None):
+        Exception.__init__(self)
+        self.pos = pos
+        self.msg = msg
+        self.context = context
+
+    def __str__(self):
+        if not self.pos: return 'SyntaxError'
+        else: return 'SyntaxError@%s(%s)' % (repr(self.pos), self.msg)
+
+class NoMoreTokens(Exception):
+    """Another exception object, for when we run out of tokens"""
+    pass
+
+class Token:
+    """Yapps token.
+
+    This is a container for a scanned token.
+    """
+
+    def __init__(self, type,value, pos=None):
+        """Initialize a token."""
+        self.type = type
+        self.value = value
+        self.pos = pos
+
+    def __repr__(self):
+        output = '<%s: %s' % (self.type, repr(self.value))
+        if self.pos:
+            output += " @ "
+            if self.pos[0]:
+                output += "%s:" % self.pos[0]
+            if self.pos[1]:
+                output += "%d" % self.pos[1]
+            if self.pos[2] is not None:
+                output += ".%d" % self.pos[2]
+        output += ">"
+        return output
+
+in_name=0
+class Scanner:
+    """Yapps scanner.
+
+    The Yapps scanner can work in context sensitive or context
+    insensitive modes.  The token(i) method is used to retrieve the
+    i-th token.  It takes a restrict set that limits the set of tokens
+    it is allowed to return.  In context sensitive mode, this restrict
+    set guides the scanner.  In context insensitive mode, there is no
+    restriction (the set is always the full set of tokens).
+
+    """
+
+    def __init__(self, patterns, ignore, input="",
+            file=None,filename=None,stacked=False):
+        """Initialize the scanner.
+
+        Parameters:
+          patterns : [(terminal, uncompiled regex), ...] or None
+          ignore : {terminal:None, ...}
+          input : string
+
+        If patterns is None, we assume that the subclass has
+        defined self.patterns : [(terminal, compiled regex), ...].
+        Note that the patterns parameter expects uncompiled regexes,
+        whereas the self.patterns field expects compiled regexes.
+
+        The 'ignore' value is either None or a callable, which is called
+        with the scanner and the to-be-ignored match object; this can
+        be used for include file or comment handling.
+        """
+
+        if not filename:
+            global in_name
+            filename="<f.%d>" % in_name
+            in_name += 1
+
+        self.input = input
+        self.ignore = ignore
+        self.file = file
+        self.filename = filename
+        self.pos = 0
+        self.del_pos = 0 # skipped
+        self.line = 1
+        self.del_line = 0 # skipped
+        self.col = 0
+        self.tokens = []
+        self.stack = None
+        self.stacked = stacked
+
+        self.last_read_token = None
+        self.last_token = None
+        self.last_types = None
+
+        if patterns is not None:
+            # Compile the regex strings into regex objects
+            self.patterns = []
+            for terminal, regex in patterns:
+                self.patterns.append( (terminal, re.compile(regex)) )
+
+    def stack_input(self, input="", file=None, filename=None):
+        """Temporarily parse from a second file."""
+
+        # Already reading from somewhere else: Go on top of that, please.
+        if self.stack:
+            # autogenerate a recursion-level-identifying filename
+            if not filename:
+                filename = 1
+            else:
+                try:
+                    filename += 1
+                except TypeError:
+                    pass
+                # now pass off to the include file
+            self.stack.stack_input(input,file,filename)
+        else:
+
+            try:
+                filename += 0
+            except TypeError:
+                pass
+            else:
+                filename = "<str_%d>" % filename
+
+#			self.stack = object.__new__(self.__class__)
+#			Scanner.__init__(self.stack,self.patterns,self.ignore,input,file,filename, stacked=True)
+
+            # Note that the pattern+ignore are added by the generated
+            # scanner code
+            self.stack = self.__class__(input,file,filename, stacked=True)
+
+    def get_pos(self):
+        """Return a file/line/char tuple."""
+        if self.stack: return self.stack.get_pos()
+
+        return (self.filename, self.line+self.del_line, self.col)
+
+#	def __repr__(self):
+#		"""Print the last few tokens that have been scanned in"""
+#		output = ''
+#		for t in self.tokens:
+#			output += '%s\n' % (repr(t),)
+#		return output
+
+    def print_line_with_pointer(self, pos, length=0, out=sys.stderr):
+        """Print the line of 'text' that includes position 'p',
+        along with a second line with a single caret (^) at position p"""
+
+        file,line,p = pos
+        if file != self.filename:
+            if self.stack: return self.stack.print_line_with_pointer(pos,length=length,out=out)
+            print >>out, "(%s: not in input buffer)" % file
+            return
+
+        text = self.input
+        p += length-1 # starts at pos 1
+
+        origline=line
+        line -= self.del_line
+        spos=0
+        if line > 0:
+            while 1:
+                line = line - 1
+                try:
+                    cr = text.index("\n",spos)
+                except ValueError:
+                    if line:
+                        text = ""
+                    break
+                if line == 0:
+                    text = text[spos:cr]
+                    break
+                spos = cr+1
+        else:
+            print >>out, "(%s:%d not in input buffer)" % (file,origline)
+            return
+
+        # Now try printing part of the line
+        text = text[max(p-80, 0):p+80]
+        p = p - max(p-80, 0)
+
+        # Strip to the left
+        i = text[:p].rfind('\n')
+        j = text[:p].rfind('\r')
+        if i < 0 or (0 <= j < i): i = j
+        if 0 <= i < p:
+            p = p - i - 1
+            text = text[i+1:]
+
+        # Strip to the right
+        i = text.find('\n', p)
+        j = text.find('\r', p)
+        if i < 0 or (0 <= j < i): i = j
+        if i >= 0:
+            text = text[:i]
+
+        # Now shorten the text
+        while len(text) > 70 and p > 60:
+            # Cut off 10 chars
+            text = "..." + text[10:]
+            p = p - 7
+
+        # Now print the string, along with an indicator
+        print >>out, '> ',text
+        print >>out, '> ',' '*p + '^'
+
+    def grab_input(self):
+        """Get more input if possible."""
+        if not self.file: return
+        if len(self.input) - self.pos >= MIN_WINDOW: return
+
+        data = self.file.read(MIN_WINDOW)
+        if data is None or data == "":
+            self.file = None
+
+        # Drop bytes from the start, if necessary.
+        if self.pos > 2*MIN_WINDOW:
+            self.del_pos += MIN_WINDOW
+            self.del_line += self.input[:MIN_WINDOW].count("\n")
+            self.pos -= MIN_WINDOW
+            self.input = self.input[MIN_WINDOW:] + data
+        else:
+            self.input = self.input + data
+
+    def getchar(self):
+        """Return the next character."""
+        self.grab_input()
+
+        c = self.input[self.pos]
+        self.pos += 1
+        return c
+
+    def token(self, restrict, context=None):
+        """Scan for another token."""
+
+        while 1:
+            if self.stack:
+                try:
+                    return self.stack.token(restrict, context)
+                except StopIteration:
+                    self.stack = None
+
+        # Keep looking for a token, ignoring any in self.ignore
+            self.grab_input()
+
+            # special handling for end-of-file
+            if self.stacked and self.pos==len(self.input):
+                raise StopIteration
+
+            # Search the patterns for the longest match, with earlier
+            # tokens in the list having preference
+            best_match = -1
+            best_pat = '(error)'
+            best_m = None
+            for p, regexp in self.patterns:
+                # First check to see if we're ignoring this token
+                if restrict and p not in restrict and p not in self.ignore:
+                    continue
+                m = regexp.match(self.input, self.pos)
+                if m and m.end()-m.start() > best_match:
+                    # We got a match that's better than the previous one
+                    best_pat = p
+                    best_match = m.end()-m.start()
+                    best_m = m
+
+            # If we didn't find anything, raise an error
+            if best_pat == '(error)' and best_match < 0:
+                msg = 'Bad Token'
+                if restrict:
+                    msg = 'Trying to find one of '+', '.join(restrict)
+                raise SyntaxError(self.get_pos(), msg, context=context)
+
+            ignore = best_pat in self.ignore
+            value = self.input[self.pos:self.pos+best_match]
+            if not ignore:
+                tok=Token(type=best_pat, value=value, pos=self.get_pos())
+
+            self.pos += best_match
+
+            npos = value.rfind("\n")
+            if npos > -1:
+                self.col = best_match-npos
+                self.line += value.count("\n")
+            else:
+                self.col += best_match
+
+            # If we found something that isn't to be ignored, return it
+            if not ignore:
+                if len(self.tokens) >= 10:
+                    del self.tokens[0]
+                self.tokens.append(tok)
+                self.last_read_token = tok
+                # print repr(tok)
+                return tok
+            else:
+                ignore = self.ignore[best_pat]
+                if ignore:
+                    ignore(self, best_m)
+
+    def peek(self, *types, **kw):
+        """Returns the token type for lookahead; if there are any args
+        then the list of args is the set of token types to allow"""
+        context = kw.get("context",None)
+        if self.last_token is None:
+            self.last_types = types
+            self.last_token = self.token(types,context)
+        elif self.last_types:
+            for t in types:
+                if t not in self.last_types:
+                    raise NotImplementedError("Unimplemented: restriction set changed")
+        return self.last_token.type
+
+    def scan(self, type, **kw):
+        """Returns the matched text, and moves to the next token"""
+        context = kw.get("context",None)
+
+        if self.last_token is None:
+            tok = self.token([type],context)
+        else:
+            if self.last_types and type not in self.last_types:
+                raise NotImplementedError("Unimplemented: restriction set changed")
+
+            tok = self.last_token
+            self.last_token = None
+        if tok.type != type:
+            if not self.last_types: self.last_types=[]
+            raise SyntaxError(tok.pos, 'Trying to find '+type+': '+ ', '.join(self.last_types)+", got "+tok.type, context=context)
+        return tok.value
+
+class Parser:
+    """Base class for Yapps-generated parsers.
+
+    """
+
+    def __init__(self, scanner):
+        self._scanner = scanner
+
+    def _stack(self, input="",file=None,filename=None):
+        """Temporarily read from someplace else"""
+        self._scanner.stack_input(input,file,filename)
+        self._tok = None
+
+    def _peek(self, *types, **kw):
+        """Returns the token type for lookahead; if there are any args
+        then the list of args is the set of token types to allow"""
+        return self._scanner.peek(*types, **kw)
+
+    def _scan(self, type, **kw):
+        """Returns the matched text, and moves to the next token"""
+        return self._scanner.scan(type, **kw)
+
+class Context:
+    """Class to represent the parser's call stack.
+
+    Every rule creates a Context that links to its parent rule.  The
+    contexts can be used for debugging.
+
+    """
+
+    def __init__(self, parent, scanner, rule, args=()):
+        """Create a new context.
+
+        Args:
+        parent: Context object or None
+        scanner: Scanner object
+        rule: string (name of the rule)
+        args: tuple listing parameters to the rule
+
+        """
+        self.parent = parent
+        self.scanner = scanner
+        self.rule = rule
+        self.args = args
+        while scanner.stack: scanner = scanner.stack
+        self.token = scanner.last_read_token
+
+    def __str__(self):
+        output = ''
+        if self.parent: output = str(self.parent) + ' > '
+        output += self.rule
+        return output
+
+def print_error(err, scanner, max_ctx=None):
+    """Print error messages, the parser stack, and the input text -- for human-readable error messages."""
+    # NOTE: this function assumes 80 columns :-(
+    # Figure out the line number
+    pos = err.pos
+    if not pos:
+        pos = scanner.get_pos()
+
+    file_name, line_number, column_number = pos
+    print('%s:%d:%d: %s' % (file_name, line_number, column_number, err.msg), file=sys.stderr)
+
+    scanner.print_line_with_pointer(pos)
+
+    context = err.context
+    token = None
+    while context:
+        print('while parsing %s%s:' % (context.rule, tuple(context.args)), file=sys.stderr)
+        if context.token:
+            token = context.token
+        if token:
+            scanner.print_line_with_pointer(token.pos, length=len(token.value))
+        context = context.parent
+        if max_ctx:
+            max_ctx = max_ctx-1
+            if not max_ctx:
+                break
+
+def wrap_error_reporter(parser, rule, *args,**kw):
+    try:
+        return getattr(parser, rule)(*args,**kw)
+    except SyntaxError as e:
+        print_error(e, parser._scanner)
+    except NoMoreTokens:
+        print('Could not complete parsing; stopped around here:', file=sys.stderr)
+        print(parser._scanner, file=sys.stderr)
+
+from twisted.words.xish.xpath import AttribValue, BooleanValue, CompareValue
+from twisted.words.xish.xpath import Function, IndexValue, LiteralValue
+from twisted.words.xish.xpath import _AnyLocation, _Location
+
+%%
+parser XPathParser:
+        ignore:             "\\s+"
+        token INDEX:        "[0-9]+"
+        token WILDCARD:     "\*"
+        token IDENTIFIER:   "[a-zA-Z][a-zA-Z0-9_\-]*"
+        token ATTRIBUTE:    "\@[a-zA-Z][a-zA-Z0-9_\-]*"
+        token FUNCNAME:     "[a-zA-Z][a-zA-Z0-9_]*"
+        token CMP_EQ:       "\="
+        token CMP_NE:       "\!\="
+        token STR_DQ:       '"([^"]|(\\"))*?"'
+        token STR_SQ:       "'([^']|(\\'))*?'"
+        token OP_AND:       "and"
+        token OP_OR:        "or"
+        token END:          "$"
+
+        rule XPATH:      PATH {{ result = PATH; current = result }}
+                           ( PATH {{ current.childLocation = PATH; current = current.childLocation }} ) * END
+                           {{ return  result }}
+
+        rule PATH:       ("/" {{ result = _Location() }} | "//" {{ result = _AnyLocation() }} )
+                           ( IDENTIFIER {{ result.elementName = IDENTIFIER }} | WILDCARD {{ result.elementName = None }} )
+                           ( "\[" PREDICATE {{ result.predicates.append(PREDICATE) }} "\]")*
+                           {{ return result }}
+
+        rule PREDICATE:  EXPR  {{ return EXPR }} |
+                         INDEX {{ return IndexValue(INDEX) }}
+
+        rule EXPR:       FACTOR {{ e = FACTOR }}
+                           ( BOOLOP FACTOR {{ e = BooleanValue(e, BOOLOP, FACTOR) }} )*
+                           {{ return e }}
+
+        rule BOOLOP:     ( OP_AND {{ return OP_AND }} | OP_OR {{ return OP_OR }} )
+
+        rule FACTOR:     TERM {{ return TERM }}
+                           | "\(" EXPR "\)" {{ return EXPR }}
+
+        rule TERM:       VALUE            {{ t = VALUE }}
+                           [ CMP VALUE  {{ t = CompareValue(t, CMP, VALUE) }} ]
+                                          {{ return t }}
+
+        rule VALUE:      "@" IDENTIFIER   {{ return AttribValue(IDENTIFIER) }} |
+                         FUNCNAME         {{ f = Function(FUNCNAME); args = [] }}
+                           "\(" [ VALUE      {{ args.append(VALUE) }}
+                             (
+                               "," VALUE     {{ args.append(VALUE) }}
+                             )*
+                           ] "\)"           {{ f.setParams(*args); return f }} |
+                         STR              {{ return LiteralValue(STR[1:len(STR)-1]) }}
+
+        rule CMP: (CMP_EQ  {{ return CMP_EQ }} | CMP_NE {{ return CMP_NE }})
+        rule STR: (STR_DQ  {{ return STR_DQ }} | STR_SQ {{ return STR_SQ }})
diff --git a/contrib/python/Twisted/py3/twisted/words/xish/xpathparser.py b/contrib/python/Twisted/py3/twisted/words/xish/xpathparser.py
new file mode 100644
index 00000000000..8df0d543694
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/xish/xpathparser.py
@@ -0,0 +1,652 @@
+# -*- test-case-name: twisted.words.test.test_xpath -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+# pylint: disable=W9401,W9402
+
+# DO NOT EDIT xpathparser.py!
+#
+# It is generated from xpathparser.g using Yapps. Make needed changes there.
+# This also means that the generated Python may not conform to Twisted's coding
+# standards, so it is wrapped in exec to prevent automated checkers from
+# complaining.
+
+# HOWTO Generate me:
+#
+# 1.) Grab a copy of yapps2:
+#         https://github.com/smurfix/yapps
+#
+#     Note: Do NOT use the package in debian/ubuntu as it has incompatible
+#     modifications. The original at http://theory.stanford.edu/~amitp/yapps/
+#     hasn't been touched since 2003 and has not been updated to work with
+#     Python 3.
+#
+# 2.) Generate the grammar:
+#
+#         yapps2 xpathparser.g xpathparser.py.proto
+#
+# 3.) Edit the output to depend on the embedded runtime, and remove extraneous
+#     imports:
+#
+#         sed -e '/^# Begin/,${/^[^ ].*mport/d}' -e 's/runtime\.//g' \
+#             -e "s/^\(from __future\)/exec(r'''\n\1/" -e"\$a''')"
+#             xpathparser.py.proto > xpathparser.py
+
+"""
+XPath Parser.
+
+Besides the parser code produced by Yapps, this module also defines the
+parse-time exception classes, a scanner class, a base class for parsers
+produced by Yapps, and a context class that keeps track of the parse stack.
+These have been copied from the Yapps runtime module.
+"""
+
+exec(
+    r'''
+from __future__ import print_function
+import sys, re
+
+MIN_WINDOW=4096
+# File lookup window
+
+class SyntaxError(Exception):
+    """When we run into an unexpected token, this is the exception to use"""
+    def __init__(self, pos=None, msg="Bad Token", context=None):
+        Exception.__init__(self)
+        self.pos = pos
+        self.msg = msg
+        self.context = context
+
+    def __str__(self):
+        if not self.pos: return 'SyntaxError'
+        else: return 'SyntaxError@%s(%s)' % (repr(self.pos), self.msg)
+
+class NoMoreTokens(Exception):
+    """Another exception object, for when we run out of tokens"""
+    pass
+
+class Token:
+    """Yapps token.
+
+    This is a container for a scanned token.
+    """
+
+    def __init__(self, type,value, pos=None):
+        """Initialize a token."""
+        self.type = type
+        self.value = value
+        self.pos = pos
+
+    def __repr__(self):
+        output = '<%s: %s' % (self.type, repr(self.value))
+        if self.pos:
+            output += " @ "
+            if self.pos[0]:
+                output += "%s:" % self.pos[0]
+            if self.pos[1]:
+                output += "%d" % self.pos[1]
+            if self.pos[2] is not None:
+                output += ".%d" % self.pos[2]
+        output += ">"
+        return output
+
+in_name=0
+class Scanner:
+    """Yapps scanner.
+
+    The Yapps scanner can work in context sensitive or context
+    insensitive modes.  The token(i) method is used to retrieve the
+    i-th token.  It takes a restrict set that limits the set of tokens
+    it is allowed to return.  In context sensitive mode, this restrict
+    set guides the scanner.  In context insensitive mode, there is no
+    restriction (the set is always the full set of tokens).
+
+    """
+
+    def __init__(self, patterns, ignore, input="",
+            file=None,filename=None,stacked=False):
+        """Initialize the scanner.
+
+        Parameters:
+          patterns : [(terminal, uncompiled regex), ...] or None
+          ignore : {terminal:None, ...}
+          input : string
+
+        If patterns is None, we assume that the subclass has
+        defined self.patterns : [(terminal, compiled regex), ...].
+        Note that the patterns parameter expects uncompiled regexes,
+        whereas the self.patterns field expects compiled regexes.
+
+        The 'ignore' value is either None or a callable, which is called
+        with the scanner and the to-be-ignored match object; this can
+        be used for include file or comment handling.
+        """
+
+        if not filename:
+            global in_name
+            filename="<f.%d>" % in_name
+            in_name += 1
+
+        self.input = input
+        self.ignore = ignore
+        self.file = file
+        self.filename = filename
+        self.pos = 0
+        self.del_pos = 0 # skipped
+        self.line = 1
+        self.del_line = 0 # skipped
+        self.col = 0
+        self.tokens = []
+        self.stack = None
+        self.stacked = stacked
+
+        self.last_read_token = None
+        self.last_token = None
+        self.last_types = None
+
+        if patterns is not None:
+            # Compile the regex strings into regex objects
+            self.patterns = []
+            for terminal, regex in patterns:
+                self.patterns.append( (terminal, re.compile(regex)) )
+
+    def stack_input(self, input="", file=None, filename=None):
+        """Temporarily parse from a second file."""
+
+        # Already reading from somewhere else: Go on top of that, please.
+        if self.stack:
+            # autogenerate a recursion-level-identifying filename
+            if not filename:
+                filename = 1
+            else:
+                try:
+                    filename += 1
+                except TypeError:
+                    pass
+                # now pass off to the include file
+            self.stack.stack_input(input,file,filename)
+        else:
+
+            try:
+                filename += 0
+            except TypeError:
+                pass
+            else:
+                filename = "<str_%d>" % filename
+
+#			self.stack = object.__new__(self.__class__)
+#			Scanner.__init__(self.stack,self.patterns,self.ignore,input,file,filename, stacked=True)
+
+            # Note that the pattern+ignore are added by the generated
+            # scanner code
+            self.stack = self.__class__(input,file,filename, stacked=True)
+
+    def get_pos(self):
+        """Return a file/line/char tuple."""
+        if self.stack: return self.stack.get_pos()
+
+        return (self.filename, self.line+self.del_line, self.col)
+
+#	def __repr__(self):
+#		"""Print the last few tokens that have been scanned in"""
+#		output = ''
+#		for t in self.tokens:
+#			output += '%s\n' % (repr(t),)
+#		return output
+
+    def print_line_with_pointer(self, pos, length=0, out=sys.stderr):
+        """Print the line of 'text' that includes position 'p',
+        along with a second line with a single caret (^) at position p"""
+
+        file,line,p = pos
+        if file != self.filename:
+            if self.stack: return self.stack.print_line_with_pointer(pos,length=length,out=out)
+            print >>out, "(%s: not in input buffer)" % file
+            return
+
+        text = self.input
+        p += length-1 # starts at pos 1
+
+        origline=line
+        line -= self.del_line
+        spos=0
+        if line > 0:
+            while 1:
+                line = line - 1
+                try:
+                    cr = text.index("\n",spos)
+                except ValueError:
+                    if line:
+                        text = ""
+                    break
+                if line == 0:
+                    text = text[spos:cr]
+                    break
+                spos = cr+1
+        else:
+            print >>out, "(%s:%d not in input buffer)" % (file,origline)
+            return
+
+        # Now try printing part of the line
+        text = text[max(p-80, 0):p+80]
+        p = p - max(p-80, 0)
+
+        # Strip to the left
+        i = text[:p].rfind('\n')
+        j = text[:p].rfind('\r')
+        if i < 0 or (0 <= j < i): i = j
+        if 0 <= i < p:
+            p = p - i - 1
+            text = text[i+1:]
+
+        # Strip to the right
+        i = text.find('\n', p)
+        j = text.find('\r', p)
+        if i < 0 or (0 <= j < i): i = j
+        if i >= 0:
+            text = text[:i]
+
+        # Now shorten the text
+        while len(text) > 70 and p > 60:
+            # Cut off 10 chars
+            text = "..." + text[10:]
+            p = p - 7
+
+        # Now print the string, along with an indicator
+        print >>out, '> ',text
+        print >>out, '> ',' '*p + '^'
+
+    def grab_input(self):
+        """Get more input if possible."""
+        if not self.file: return
+        if len(self.input) - self.pos >= MIN_WINDOW: return
+
+        data = self.file.read(MIN_WINDOW)
+        if data is None or data == "":
+            self.file = None
+
+        # Drop bytes from the start, if necessary.
+        if self.pos > 2*MIN_WINDOW:
+            self.del_pos += MIN_WINDOW
+            self.del_line += self.input[:MIN_WINDOW].count("\n")
+            self.pos -= MIN_WINDOW
+            self.input = self.input[MIN_WINDOW:] + data
+        else:
+            self.input = self.input + data
+
+    def getchar(self):
+        """Return the next character."""
+        self.grab_input()
+
+        c = self.input[self.pos]
+        self.pos += 1
+        return c
+
+    def token(self, restrict, context=None):
+        """Scan for another token."""
+
+        while 1:
+            if self.stack:
+                try:
+                    return self.stack.token(restrict, context)
+                except StopIteration:
+                    self.stack = None
+
+        # Keep looking for a token, ignoring any in self.ignore
+            self.grab_input()
+
+            # special handling for end-of-file
+            if self.stacked and self.pos==len(self.input):
+                raise StopIteration
+
+            # Search the patterns for the longest match, with earlier
+            # tokens in the list having preference
+            best_match = -1
+            best_pat = '(error)'
+            best_m = None
+            for p, regexp in self.patterns:
+                # First check to see if we're ignoring this token
+                if restrict and p not in restrict and p not in self.ignore:
+                    continue
+                m = regexp.match(self.input, self.pos)
+                if m and m.end()-m.start() > best_match:
+                    # We got a match that's better than the previous one
+                    best_pat = p
+                    best_match = m.end()-m.start()
+                    best_m = m
+
+            # If we didn't find anything, raise an error
+            if best_pat == '(error)' and best_match < 0:
+                msg = 'Bad Token'
+                if restrict:
+                    msg = 'Trying to find one of '+', '.join(restrict)
+                raise SyntaxError(self.get_pos(), msg, context=context)
+
+            ignore = best_pat in self.ignore
+            value = self.input[self.pos:self.pos+best_match]
+            if not ignore:
+                tok=Token(type=best_pat, value=value, pos=self.get_pos())
+
+            self.pos += best_match
+
+            npos = value.rfind("\n")
+            if npos > -1:
+                self.col = best_match-npos
+                self.line += value.count("\n")
+            else:
+                self.col += best_match
+
+            # If we found something that isn't to be ignored, return it
+            if not ignore:
+                if len(self.tokens) >= 10:
+                    del self.tokens[0]
+                self.tokens.append(tok)
+                self.last_read_token = tok
+                # print repr(tok)
+                return tok
+            else:
+                ignore = self.ignore[best_pat]
+                if ignore:
+                    ignore(self, best_m)
+
+    def peek(self, *types, **kw):
+        """Returns the token type for lookahead; if there are any args
+        then the list of args is the set of token types to allow"""
+        context = kw.get("context",None)
+        if self.last_token is None:
+            self.last_types = types
+            self.last_token = self.token(types,context)
+        elif self.last_types:
+            for t in types:
+                if t not in self.last_types:
+                    raise NotImplementedError("Unimplemented: restriction set changed")
+        return self.last_token.type
+
+    def scan(self, type, **kw):
+        """Returns the matched text, and moves to the next token"""
+        context = kw.get("context",None)
+
+        if self.last_token is None:
+            tok = self.token([type],context)
+        else:
+            if self.last_types and type not in self.last_types:
+                raise NotImplementedError("Unimplemented: restriction set changed")
+
+            tok = self.last_token
+            self.last_token = None
+        if tok.type != type:
+            if not self.last_types: self.last_types=[]
+            raise SyntaxError(tok.pos, 'Trying to find '+type+': '+ ', '.join(self.last_types)+", got "+tok.type, context=context)
+        return tok.value
+
+class Parser:
+    """Base class for Yapps-generated parsers.
+
+    """
+
+    def __init__(self, scanner):
+        self._scanner = scanner
+
+    def _stack(self, input="",file=None,filename=None):
+        """Temporarily read from someplace else"""
+        self._scanner.stack_input(input,file,filename)
+        self._tok = None
+
+    def _peek(self, *types, **kw):
+        """Returns the token type for lookahead; if there are any args
+        then the list of args is the set of token types to allow"""
+        return self._scanner.peek(*types, **kw)
+
+    def _scan(self, type, **kw):
+        """Returns the matched text, and moves to the next token"""
+        return self._scanner.scan(type, **kw)
+
+class Context:
+    """Class to represent the parser's call stack.
+
+    Every rule creates a Context that links to its parent rule.  The
+    contexts can be used for debugging.
+
+    """
+
+    def __init__(self, parent, scanner, rule, args=()):
+        """Create a new context.
+
+        Args:
+        parent: Context object or None
+        scanner: Scanner object
+        rule: string (name of the rule)
+        args: tuple listing parameters to the rule
+
+        """
+        self.parent = parent
+        self.scanner = scanner
+        self.rule = rule
+        self.args = args
+        while scanner.stack: scanner = scanner.stack
+        self.token = scanner.last_read_token
+
+    def __str__(self):
+        output = ''
+        if self.parent: output = str(self.parent) + ' > '
+        output += self.rule
+        return output
+
+def print_error(err, scanner, max_ctx=None):
+    """Print error messages, the parser stack, and the input text -- for human-readable error messages."""
+    # NOTE: this function assumes 80 columns :-(
+    # Figure out the line number
+    pos = err.pos
+    if not pos:
+        pos = scanner.get_pos()
+
+    file_name, line_number, column_number = pos
+    print('%s:%d:%d: %s' % (file_name, line_number, column_number, err.msg), file=sys.stderr)
+
+    scanner.print_line_with_pointer(pos)
+
+    context = err.context
+    token = None
+    while context:
+        print('while parsing %s%s:' % (context.rule, tuple(context.args)), file=sys.stderr)
+        if context.token:
+            token = context.token
+        if token:
+            scanner.print_line_with_pointer(token.pos, length=len(token.value))
+        context = context.parent
+        if max_ctx:
+            max_ctx = max_ctx-1
+            if not max_ctx:
+                break
+
+def wrap_error_reporter(parser, rule, *args,**kw):
+    try:
+        return getattr(parser, rule)(*args,**kw)
+    except SyntaxError as e:
+        print_error(e, parser._scanner)
+    except NoMoreTokens:
+        print('Could not complete parsing; stopped around here:', file=sys.stderr)
+        print(parser._scanner, file=sys.stderr)
+
+from twisted.words.xish.xpath import AttribValue, BooleanValue, CompareValue
+from twisted.words.xish.xpath import Function, IndexValue, LiteralValue
+from twisted.words.xish.xpath import _AnyLocation, _Location
+
+
+# Begin -- grammar generated by Yapps
+
+class XPathParserScanner(Scanner):
+    patterns = [
+        ('","', re.compile(',')),
+        ('"@"', re.compile('@')),
+        ('"\\)"', re.compile('\\)')),
+        ('"\\("', re.compile('\\(')),
+        ('"\\]"', re.compile('\\]')),
+        ('"\\["', re.compile('\\[')),
+        ('"//"', re.compile('//')),
+        ('"/"', re.compile('/')),
+        ('\\s+', re.compile('\\s+')),
+        ('INDEX', re.compile('[0-9]+')),
+        ('WILDCARD', re.compile('\\*')),
+        ('IDENTIFIER', re.compile('[a-zA-Z][a-zA-Z0-9_\\-]*')),
+        ('ATTRIBUTE', re.compile('\\@[a-zA-Z][a-zA-Z0-9_\\-]*')),
+        ('FUNCNAME', re.compile('[a-zA-Z][a-zA-Z0-9_]*')),
+        ('CMP_EQ', re.compile('\\=')),
+        ('CMP_NE', re.compile('\\!\\=')),
+        ('STR_DQ', re.compile('"([^"]|(\\"))*?"')),
+        ('STR_SQ', re.compile("'([^']|(\\'))*?'")),
+        ('OP_AND', re.compile('and')),
+        ('OP_OR', re.compile('or')),
+        ('END', re.compile('$')),
+    ]
+    def __init__(self, str,*args,**kw):
+        Scanner.__init__(self,None,{'\\s+':None,},str,*args,**kw)
+
+class XPathParser(Parser):
+    Context = Context
+    def XPATH(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'XPATH', [])
+        PATH = self.PATH(_context)
+        result = PATH; current = result
+        while self._peek('END', '"/"', '"//"', context=_context) != 'END':
+            PATH = self.PATH(_context)
+            current.childLocation = PATH; current = current.childLocation
+        END = self._scan('END', context=_context)
+        return  result
+
+    def PATH(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'PATH', [])
+        _token = self._peek('"/"', '"//"', context=_context)
+        if _token == '"/"':
+            self._scan('"/"', context=_context)
+            result = _Location()
+        else: # == '"//"'
+            self._scan('"//"', context=_context)
+            result = _AnyLocation()
+        _token = self._peek('IDENTIFIER', 'WILDCARD', context=_context)
+        if _token == 'IDENTIFIER':
+            IDENTIFIER = self._scan('IDENTIFIER', context=_context)
+            result.elementName = IDENTIFIER
+        else: # == 'WILDCARD'
+            WILDCARD = self._scan('WILDCARD', context=_context)
+            result.elementName = None
+        while self._peek('"\\["', 'END', '"/"', '"//"', context=_context) == '"\\["':
+            self._scan('"\\["', context=_context)
+            PREDICATE = self.PREDICATE(_context)
+            result.predicates.append(PREDICATE)
+            self._scan('"\\]"', context=_context)
+        return result
+
+    def PREDICATE(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'PREDICATE', [])
+        _token = self._peek('INDEX', '"\\("', '"@"', 'FUNCNAME', 'STR_DQ', 'STR_SQ', context=_context)
+        if _token != 'INDEX':
+            EXPR = self.EXPR(_context)
+            return EXPR
+        else: # == 'INDEX'
+            INDEX = self._scan('INDEX', context=_context)
+            return IndexValue(INDEX)
+
+    def EXPR(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'EXPR', [])
+        FACTOR = self.FACTOR(_context)
+        e = FACTOR
+        while self._peek('OP_AND', 'OP_OR', '"\\)"', '"\\]"', context=_context) in ['OP_AND', 'OP_OR']:
+            BOOLOP = self.BOOLOP(_context)
+            FACTOR = self.FACTOR(_context)
+            e = BooleanValue(e, BOOLOP, FACTOR)
+        return e
+
+    def BOOLOP(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'BOOLOP', [])
+        _token = self._peek('OP_AND', 'OP_OR', context=_context)
+        if _token == 'OP_AND':
+            OP_AND = self._scan('OP_AND', context=_context)
+            return OP_AND
+        else: # == 'OP_OR'
+            OP_OR = self._scan('OP_OR', context=_context)
+            return OP_OR
+
+    def FACTOR(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'FACTOR', [])
+        _token = self._peek('"\\("', '"@"', 'FUNCNAME', 'STR_DQ', 'STR_SQ', context=_context)
+        if _token != '"\\("':
+            TERM = self.TERM(_context)
+            return TERM
+        else: # == '"\\("'
+            self._scan('"\\("', context=_context)
+            EXPR = self.EXPR(_context)
+            self._scan('"\\)"', context=_context)
+            return EXPR
+
+    def TERM(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'TERM', [])
+        VALUE = self.VALUE(_context)
+        t = VALUE
+        if self._peek('CMP_EQ', 'CMP_NE', 'OP_AND', 'OP_OR', '"\\)"', '"\\]"', context=_context) in ['CMP_EQ', 'CMP_NE']:
+            CMP = self.CMP(_context)
+            VALUE = self.VALUE(_context)
+            t = CompareValue(t, CMP, VALUE)
+        return t
+
+    def VALUE(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'VALUE', [])
+        _token = self._peek('"@"', 'FUNCNAME', 'STR_DQ', 'STR_SQ', context=_context)
+        if _token == '"@"':
+            self._scan('"@"', context=_context)
+            IDENTIFIER = self._scan('IDENTIFIER', context=_context)
+            return AttribValue(IDENTIFIER)
+        elif _token == 'FUNCNAME':
+            FUNCNAME = self._scan('FUNCNAME', context=_context)
+            f = Function(FUNCNAME); args = []
+            self._scan('"\\("', context=_context)
+            if self._peek('"\\)"', '"@"', 'FUNCNAME', '","', 'STR_DQ', 'STR_SQ', context=_context) not in ['"\\)"', '","']:
+                VALUE = self.VALUE(_context)
+                args.append(VALUE)
+                while self._peek('","', '"\\)"', context=_context) == '","':
+                    self._scan('","', context=_context)
+                    VALUE = self.VALUE(_context)
+                    args.append(VALUE)
+            self._scan('"\\)"', context=_context)
+            f.setParams(*args); return f
+        else: # in ['STR_DQ', 'STR_SQ']
+            STR = self.STR(_context)
+            return LiteralValue(STR[1:len(STR)-1])
+
+    def CMP(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'CMP', [])
+        _token = self._peek('CMP_EQ', 'CMP_NE', context=_context)
+        if _token == 'CMP_EQ':
+            CMP_EQ = self._scan('CMP_EQ', context=_context)
+            return CMP_EQ
+        else: # == 'CMP_NE'
+            CMP_NE = self._scan('CMP_NE', context=_context)
+            return CMP_NE
+
+    def STR(self, _parent=None):
+        _context = self.Context(_parent, self._scanner, 'STR', [])
+        _token = self._peek('STR_DQ', 'STR_SQ', context=_context)
+        if _token == 'STR_DQ':
+            STR_DQ = self._scan('STR_DQ', context=_context)
+            return STR_DQ
+        else: # == 'STR_SQ'
+            STR_SQ = self._scan('STR_SQ', context=_context)
+            return STR_SQ
+
+
+def parse(rule, text):
+    P = XPathParser(XPathParserScanner(text))
+    return wrap_error_reporter(P, rule)
+
+if __name__ == '__main__':
+    from sys import argv, stdin
+    if len(argv) >= 2:
+        if len(argv) >= 3:
+            f = open(argv[2],'r')
+        else:
+            f = stdin
+        print(parse(argv[1], f.read()))
+    else: print ('Args:  <rule> [<filename>]', file=sys.stderr)
+# End -- grammar generated by Yapps
+'''
+)
diff --git a/contrib/python/Twisted/py3/twisted/words/xmpproutertap.py b/contrib/python/Twisted/py3/twisted/words/xmpproutertap.py
new file mode 100644
index 00000000000..2a95f844fb0
--- /dev/null
+++ b/contrib/python/Twisted/py3/twisted/words/xmpproutertap.py
@@ -0,0 +1,29 @@
+# -*- test-case-name: twisted.words.test.test_xmpproutertap -*-
+#
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from twisted.application import strports
+from twisted.python import usage
+from twisted.words.protocols.jabber import component
+
+
+class Options(usage.Options):
+    optParameters = [
+        ("port", None, "tcp:5347:interface=127.0.0.1", "Port components connect to"),
+        ("secret", None, "secret", "Router secret"),
+    ]
+
+    optFlags = [
+        ("verbose", "v", "Log traffic"),
+    ]
+
+
+def makeService(config):
+    router = component.Router()
+    factory = component.XMPPComponentServerFactory(router, config["secret"])
+
+    if config["verbose"]:
+        factory.logTraffic = True
+
+    return strports.service(config["port"], factory)
diff --git a/contrib/python/Twisted/py3/ya.make b/contrib/python/Twisted/py3/ya.make
new file mode 100644
index 00000000000..4a28d8ff160
--- /dev/null
+++ b/contrib/python/Twisted/py3/ya.make
@@ -0,0 +1,492 @@
+# Generated by devtools/yamaker (pypi).
+
+PY3_LIBRARY()
+
+VERSION(23.10.0)
+
+LICENSE(MIT)
+
+PEERDIR(
+    contrib/python/Automat
+    contrib/python/attrs
+    contrib/python/constantly
+    contrib/python/hyperlink
+    contrib/python/incremental
+    contrib/python/typing-extensions
+    contrib/python/zope.interface
+)
+
+NO_LINT()
+
+NO_CHECK_IMPORTS(
+    twisted.*
+)
+
+PY_SRCS(
+    TOP_LEVEL
+    twisted/__init__.py
+    twisted/__main__.py
+    twisted/_threads/__init__.py
+    twisted/_threads/_convenience.py
+    twisted/_threads/_ithreads.py
+    twisted/_threads/_memory.py
+    twisted/_threads/_pool.py
+    twisted/_threads/_team.py
+    twisted/_threads/_threadworker.py
+    twisted/_version.py
+    twisted/application/__init__.py
+    twisted/application/app.py
+    twisted/application/internet.py
+    twisted/application/reactors.py
+    twisted/application/runner/__init__.py
+    twisted/application/runner/_exit.py
+    twisted/application/runner/_pidfile.py
+    twisted/application/runner/_runner.py
+    twisted/application/service.py
+    twisted/application/strports.py
+    twisted/application/twist/__init__.py
+    twisted/application/twist/_options.py
+    twisted/application/twist/_twist.py
+    twisted/conch/__init__.py
+    twisted/conch/avatar.py
+    twisted/conch/checkers.py
+    twisted/conch/client/__init__.py
+    twisted/conch/client/agent.py
+    twisted/conch/client/connect.py
+    twisted/conch/client/default.py
+    twisted/conch/client/direct.py
+    twisted/conch/client/knownhosts.py
+    twisted/conch/client/options.py
+    twisted/conch/endpoints.py
+    twisted/conch/error.py
+    twisted/conch/insults/__init__.py
+    twisted/conch/insults/helper.py
+    twisted/conch/insults/insults.py
+    twisted/conch/insults/text.py
+    twisted/conch/insults/window.py
+    twisted/conch/interfaces.py
+    twisted/conch/ls.py
+    twisted/conch/manhole.py
+    twisted/conch/manhole_ssh.py
+    twisted/conch/manhole_tap.py
+    twisted/conch/mixin.py
+    twisted/conch/openssh_compat/__init__.py
+    twisted/conch/openssh_compat/factory.py
+    twisted/conch/openssh_compat/primes.py
+    twisted/conch/recvline.py
+    twisted/conch/scripts/__init__.py
+    twisted/conch/scripts/cftp.py
+    twisted/conch/scripts/ckeygen.py
+    twisted/conch/scripts/conch.py
+    twisted/conch/scripts/tkconch.py
+    twisted/conch/ssh/__init__.py
+    twisted/conch/ssh/_kex.py
+    twisted/conch/ssh/address.py
+    twisted/conch/ssh/agent.py
+    twisted/conch/ssh/channel.py
+    twisted/conch/ssh/common.py
+    twisted/conch/ssh/connection.py
+    twisted/conch/ssh/factory.py
+    twisted/conch/ssh/filetransfer.py
+    twisted/conch/ssh/forwarding.py
+    twisted/conch/ssh/keys.py
+    twisted/conch/ssh/service.py
+    twisted/conch/ssh/session.py
+    twisted/conch/ssh/sexpy.py
+    twisted/conch/ssh/transport.py
+    twisted/conch/ssh/userauth.py
+    twisted/conch/stdio.py
+    twisted/conch/tap.py
+    twisted/conch/telnet.py
+    twisted/conch/ttymodes.py
+    twisted/conch/ui/__init__.py
+    twisted/conch/ui/ansi.py
+    twisted/conch/ui/tkvt100.py
+    twisted/conch/unix.py
+    twisted/copyright.py
+    twisted/cred/__init__.py
+    twisted/cred/_digest.py
+    twisted/cred/checkers.py
+    twisted/cred/credentials.py
+    twisted/cred/error.py
+    twisted/cred/portal.py
+    twisted/cred/strcred.py
+    twisted/enterprise/__init__.py
+    twisted/enterprise/adbapi.py
+    twisted/internet/__init__.py
+    twisted/internet/_baseprocess.py
+    twisted/internet/_deprecate.py
+    twisted/internet/_dumbwin32proc.py
+    twisted/internet/_glibbase.py
+    twisted/internet/_idna.py
+    twisted/internet/_newtls.py
+    twisted/internet/_pollingfile.py
+    twisted/internet/_posixserialport.py
+    twisted/internet/_posixstdio.py
+    twisted/internet/_producer_helpers.py
+    twisted/internet/_resolver.py
+    twisted/internet/_signals.py
+    twisted/internet/_sslverify.py
+    twisted/internet/_threadedselect.py
+    twisted/internet/_win32serialport.py
+    twisted/internet/_win32stdio.py
+    twisted/internet/abstract.py
+    twisted/internet/address.py
+    twisted/internet/asyncioreactor.py
+    twisted/internet/base.py
+    twisted/internet/cfreactor.py
+    twisted/internet/default.py
+    twisted/internet/defer.py
+    twisted/internet/endpoints.py
+    twisted/internet/epollreactor.py
+    twisted/internet/error.py
+    twisted/internet/fdesc.py
+    twisted/internet/gireactor.py
+    twisted/internet/glib2reactor.py
+    twisted/internet/gtk2reactor.py
+    twisted/internet/gtk3reactor.py
+    twisted/internet/inotify.py
+    twisted/internet/interfaces.py
+    twisted/internet/iocpreactor/__init__.py
+    twisted/internet/iocpreactor/abstract.py
+    twisted/internet/iocpreactor/const.py
+    twisted/internet/iocpreactor/interfaces.py
+    twisted/internet/iocpreactor/iocpsupport.py
+    twisted/internet/iocpreactor/reactor.py
+    twisted/internet/iocpreactor/tcp.py
+    twisted/internet/iocpreactor/udp.py
+    twisted/internet/kqreactor.py
+    twisted/internet/main.py
+    twisted/internet/pollreactor.py
+    twisted/internet/posixbase.py
+    twisted/internet/process.py
+    twisted/internet/protocol.py
+    twisted/internet/pyuisupport.py
+    twisted/internet/reactor.py
+    twisted/internet/selectreactor.py
+    twisted/internet/serialport.py
+    twisted/internet/ssl.py
+    twisted/internet/stdio.py
+    twisted/internet/task.py
+    twisted/internet/tcp.py
+    twisted/internet/testing.py
+    twisted/internet/threads.py
+    twisted/internet/tksupport.py
+    twisted/internet/udp.py
+    twisted/internet/unix.py
+    twisted/internet/utils.py
+    twisted/internet/win32eventreactor.py
+    twisted/internet/wxreactor.py
+    twisted/internet/wxsupport.py
+    twisted/logger/__init__.py
+    twisted/logger/_buffer.py
+    twisted/logger/_capture.py
+    twisted/logger/_file.py
+    twisted/logger/_filter.py
+    twisted/logger/_flatten.py
+    twisted/logger/_format.py
+    twisted/logger/_global.py
+    twisted/logger/_interfaces.py
+    twisted/logger/_io.py
+    twisted/logger/_json.py
+    twisted/logger/_legacy.py
+    twisted/logger/_levels.py
+    twisted/logger/_logger.py
+    twisted/logger/_observer.py
+    twisted/logger/_stdlib.py
+    twisted/logger/_util.py
+    twisted/mail/__init__.py
+    twisted/mail/_cred.py
+    twisted/mail/_except.py
+    twisted/mail/_pop3client.py
+    twisted/mail/alias.py
+    twisted/mail/bounce.py
+    twisted/mail/imap4.py
+    twisted/mail/interfaces.py
+    twisted/mail/mail.py
+    twisted/mail/maildir.py
+    twisted/mail/pb.py
+    twisted/mail/pop3.py
+    twisted/mail/pop3client.py
+    twisted/mail/protocols.py
+    twisted/mail/relay.py
+    twisted/mail/relaymanager.py
+    twisted/mail/scripts/__init__.py
+    twisted/mail/scripts/mailmail.py
+    twisted/mail/smtp.py
+    twisted/mail/tap.py
+    twisted/names/__init__.py
+    twisted/names/_rfc1982.py
+    twisted/names/authority.py
+    twisted/names/cache.py
+    twisted/names/client.py
+    twisted/names/common.py
+    twisted/names/dns.py
+    twisted/names/error.py
+    twisted/names/hosts.py
+    twisted/names/resolve.py
+    twisted/names/root.py
+    twisted/names/secondary.py
+    twisted/names/server.py
+    twisted/names/srvconnect.py
+    twisted/names/tap.py
+    twisted/pair/__init__.py
+    twisted/pair/ethernet.py
+    twisted/pair/ip.py
+    twisted/pair/raw.py
+    twisted/pair/rawudp.py
+    twisted/pair/testing.py
+    twisted/pair/tuntap.py
+    twisted/persisted/__init__.py
+    twisted/persisted/_token.py
+    twisted/persisted/_tokenize.py
+    twisted/persisted/aot.py
+    twisted/persisted/crefutil.py
+    twisted/persisted/dirdbm.py
+    twisted/persisted/sob.py
+    twisted/persisted/styles.py
+    twisted/plugin.py
+    twisted/plugins/__init__.py
+    twisted/plugins/cred_anonymous.py
+    twisted/plugins/cred_file.py
+    twisted/plugins/cred_memory.py
+    twisted/plugins/cred_sshkeys.py
+    twisted/plugins/cred_unix.py
+    twisted/plugins/twisted_conch.py
+    twisted/plugins/twisted_core.py
+    twisted/plugins/twisted_ftp.py
+    twisted/plugins/twisted_inet.py
+    twisted/plugins/twisted_mail.py
+    twisted/plugins/twisted_names.py
+    twisted/plugins/twisted_portforward.py
+    twisted/plugins/twisted_reactors.py
+    twisted/plugins/twisted_runner.py
+    twisted/plugins/twisted_socks.py
+    twisted/plugins/twisted_trial.py
+    twisted/plugins/twisted_web.py
+    twisted/plugins/twisted_words.py
+    twisted/positioning/__init__.py
+    twisted/positioning/_sentence.py
+    twisted/positioning/base.py
+    twisted/positioning/ipositioning.py
+    twisted/positioning/nmea.py
+    twisted/protocols/__init__.py
+    twisted/protocols/amp.py
+    twisted/protocols/basic.py
+    twisted/protocols/finger.py
+    twisted/protocols/ftp.py
+    twisted/protocols/haproxy/__init__.py
+    twisted/protocols/haproxy/_exceptions.py
+    twisted/protocols/haproxy/_info.py
+    twisted/protocols/haproxy/_interfaces.py
+    twisted/protocols/haproxy/_parser.py
+    twisted/protocols/haproxy/_v1parser.py
+    twisted/protocols/haproxy/_v2parser.py
+    twisted/protocols/haproxy/_wrapper.py
+    twisted/protocols/htb.py
+    twisted/protocols/ident.py
+    twisted/protocols/loopback.py
+    twisted/protocols/memcache.py
+    twisted/protocols/pcp.py
+    twisted/protocols/policies.py
+    twisted/protocols/portforward.py
+    twisted/protocols/postfix.py
+    twisted/protocols/shoutcast.py
+    twisted/protocols/sip.py
+    twisted/protocols/socks.py
+    twisted/protocols/stateful.py
+    twisted/protocols/tls.py
+    twisted/protocols/wire.py
+    twisted/python/__init__.py
+    twisted/python/_appdirs.py
+    twisted/python/_inotify.py
+    twisted/python/_release.py
+    twisted/python/_shellcomp.py
+    twisted/python/_textattributes.py
+    twisted/python/_tzhelper.py
+    twisted/python/_url.py
+    twisted/python/compat.py
+    twisted/python/components.py
+    twisted/python/constants.py
+    twisted/python/context.py
+    twisted/python/deprecate.py
+    twisted/python/failure.py
+    twisted/python/fakepwd.py
+    twisted/python/filepath.py
+    twisted/python/formmethod.py
+    twisted/python/htmlizer.py
+    twisted/python/lockfile.py
+    twisted/python/log.py
+    twisted/python/logfile.py
+    twisted/python/modules.py
+    twisted/python/monkey.py
+    twisted/python/procutils.py
+    twisted/python/randbytes.py
+    twisted/python/rebuild.py
+    twisted/python/reflect.py
+    twisted/python/release.py
+    twisted/python/roots.py
+    twisted/python/runtime.py
+    twisted/python/sendmsg.py
+    twisted/python/shortcut.py
+    twisted/python/syslog.py
+    twisted/python/systemd.py
+    twisted/python/text.py
+    twisted/python/threadable.py
+    twisted/python/threadpool.py
+    twisted/python/url.py
+    twisted/python/urlpath.py
+    twisted/python/usage.py
+    twisted/python/util.py
+    twisted/python/versions.py
+    twisted/python/win32.py
+    twisted/python/zippath.py
+    twisted/python/zipstream.py
+    twisted/runner/__init__.py
+    twisted/runner/inetd.py
+    twisted/runner/inetdconf.py
+    twisted/runner/inetdtap.py
+    twisted/runner/procmon.py
+    twisted/runner/procmontap.py
+    twisted/scripts/__init__.py
+    twisted/scripts/_twistd_unix.py
+    twisted/scripts/_twistw.py
+    twisted/scripts/htmlizer.py
+    twisted/scripts/trial.py
+    twisted/scripts/twistd.py
+    twisted/spread/__init__.py
+    twisted/spread/banana.py
+    twisted/spread/flavors.py
+    twisted/spread/interfaces.py
+    twisted/spread/jelly.py
+    twisted/spread/pb.py
+    twisted/spread/publish.py
+    twisted/spread/util.py
+    twisted/tap/__init__.py
+    twisted/tap/ftp.py
+    twisted/tap/portforward.py
+    twisted/tap/socks.py
+    twisted/trial/__init__.py
+    twisted/trial/__main__.py
+    twisted/trial/_asyncrunner.py
+    twisted/trial/_asynctest.py
+    twisted/trial/_dist/__init__.py
+    twisted/trial/_dist/distreporter.py
+    twisted/trial/_dist/disttrial.py
+    twisted/trial/_dist/functional.py
+    twisted/trial/_dist/managercommands.py
+    twisted/trial/_dist/options.py
+    twisted/trial/_dist/stream.py
+    twisted/trial/_dist/worker.py
+    twisted/trial/_dist/workercommands.py
+    twisted/trial/_dist/workerreporter.py
+    twisted/trial/_dist/workertrial.py
+    twisted/trial/_synctest.py
+    twisted/trial/itrial.py
+    twisted/trial/reporter.py
+    twisted/trial/runner.py
+    twisted/trial/unittest.py
+    twisted/trial/util.py
+    twisted/web/__init__.py
+    twisted/web/_auth/__init__.py
+    twisted/web/_auth/basic.py
+    twisted/web/_auth/digest.py
+    twisted/web/_auth/wrapper.py
+    twisted/web/_element.py
+    twisted/web/_flatten.py
+    twisted/web/_http2.py
+    twisted/web/_newclient.py
+    twisted/web/_responses.py
+    twisted/web/_stan.py
+    twisted/web/_template_util.py
+    twisted/web/client.py
+    twisted/web/demo.py
+    twisted/web/distrib.py
+    twisted/web/domhelpers.py
+    twisted/web/error.py
+    twisted/web/guard.py
+    twisted/web/html.py
+    twisted/web/http.py
+    twisted/web/http_headers.py
+    twisted/web/iweb.py
+    twisted/web/microdom.py
+    twisted/web/pages.py
+    twisted/web/proxy.py
+    twisted/web/resource.py
+    twisted/web/rewrite.py
+    twisted/web/script.py
+    twisted/web/server.py
+    twisted/web/soap.py
+    twisted/web/static.py
+    twisted/web/sux.py
+    twisted/web/tap.py
+    twisted/web/template.py
+    twisted/web/test/requesthelper.py
+    twisted/web/twcgi.py
+    twisted/web/util.py
+    twisted/web/vhost.py
+    twisted/web/wsgi.py
+    twisted/web/xmlrpc.py
+    twisted/words/__init__.py
+    twisted/words/ewords.py
+    twisted/words/im/__init__.py
+    twisted/words/im/baseaccount.py
+    twisted/words/im/basechat.py
+    twisted/words/im/basesupport.py
+    twisted/words/im/interfaces.py
+    twisted/words/im/ircsupport.py
+    twisted/words/im/locals.py
+    twisted/words/im/pbsupport.py
+    twisted/words/iwords.py
+    twisted/words/protocols/__init__.py
+    twisted/words/protocols/irc.py
+    twisted/words/protocols/jabber/__init__.py
+    twisted/words/protocols/jabber/client.py
+    twisted/words/protocols/jabber/component.py
+    twisted/words/protocols/jabber/error.py
+    twisted/words/protocols/jabber/ijabber.py
+    twisted/words/protocols/jabber/jid.py
+    twisted/words/protocols/jabber/jstrports.py
+    twisted/words/protocols/jabber/sasl.py
+    twisted/words/protocols/jabber/sasl_mechanisms.py
+    twisted/words/protocols/jabber/xmlstream.py
+    twisted/words/protocols/jabber/xmpp_stringprep.py
+    twisted/words/service.py
+    twisted/words/tap.py
+    twisted/words/xish/__init__.py
+    twisted/words/xish/domish.py
+    twisted/words/xish/utility.py
+    twisted/words/xish/xmlstream.py
+    twisted/words/xish/xpath.py
+    twisted/words/xish/xpathparser.py
+    twisted/words/xmpproutertap.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/Twisted/py3/
+    .dist-info/METADATA
+    .dist-info/entry_points.txt
+    .dist-info/top_level.txt
+    twisted/11715.misc
+    twisted/application/newsfragments/10146.misc
+    twisted/application/newsfragments/9746.misc
+    twisted/conch/newsfragments/.gitignore
+    twisted/internet/iocpreactor/notes.txt
+    twisted/mail/newsfragments/.gitignore
+    twisted/names/newsfragments/.gitignore
+    twisted/newsfragments/.gitignore
+    twisted/persisted/newsfragments/9831.misc
+    twisted/py.typed
+    twisted/python/_pydoctortemplates/subheader.html
+    twisted/python/twisted-completion.zsh
+    twisted/runner/newsfragments/11681.misc
+    twisted/runner/newsfragments/9657.doc
+    twisted/scripts/newsfragments/761.bugfix
+    twisted/trial/newsfragments/.gitignore
+    twisted/web/newsfragments/.gitignore
+    twisted/words/im/instancemessenger.glade
+    twisted/words/newsfragments/.gitignore
+    twisted/words/xish/xpathparser.g
+)
+
+END()
diff --git a/contrib/python/Twisted/ya.make b/contrib/python/Twisted/ya.make
new file mode 100644
index 00000000000..a8fec1f62b8
--- /dev/null
+++ b/contrib/python/Twisted/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+    PEERDIR(contrib/python/Twisted/py2)
+ELSE()
+    PEERDIR(contrib/python/Twisted/py3)
+ENDIF()
+
+NO_LINT()
+
+END()
+
+RECURSE(
+    py2
+    py3
+)
diff --git a/contrib/python/chardet/py2/test.py b/contrib/python/chardet/py2/test.py
new file mode 100644
index 00000000000..4235e1f49de
--- /dev/null
+++ b/contrib/python/chardet/py2/test.py
@@ -0,0 +1,151 @@
+"""
+Run chardet on a bunch of documents and see that we get the correct encodings.
+
+:author: Dan Blanchard
+:author: Ian Cordasco
+"""
+
+from __future__ import with_statement
+
+import textwrap
+from difflib import ndiff
+from io import open
+from os import listdir
+from os.path import dirname, isdir, join, splitext, basename
+
+try:
+    import hypothesis.strategies as st
+    from hypothesis import given, assume, settings, Verbosity
+    HAVE_HYPOTHESIS = True
+except ImportError:
+    HAVE_HYPOTHESIS = False
+import pytest
+
+import chardet
+import yatest.common
+
+
+# TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250) after we
+#       retrain model.
+MISSING_ENCODINGS = {'iso-8859-2', 'iso-8859-6', 'windows-1250',
+                     'windows-1254', 'windows-1256'}
+EXPECTED_FAILURES = {'iso-8859-7-greek/disabled.gr.xml',
+                     'iso-8859-9-turkish/divxplanet.com.xml',
+                     'iso-8859-9-turkish/subtitle.srt',
+                     'iso-8859-9-turkish/wikitop_tr_ISO-8859-9.txt'}
+
+def gen_test_params():
+    """Yields tuples of paths and encodings to use for test_encoding_detection"""
+    base_path = yatest.common.work_path('test_data')
+    for encoding in listdir(base_path):
+        path = join(base_path, encoding)
+        # Skip files in tests directory
+        if not isdir(path):
+            continue
+        # Remove language suffixes from encoding if pressent
+        encoding = encoding.lower()
+        for postfix in ['-arabic', '-bulgarian', '-cyrillic', '-greek',
+                        '-hebrew', '-hungarian', '-turkish']:
+            if encoding.endswith(postfix):
+                encoding = encoding.rpartition(postfix)[0]
+                break
+        # Skip directories for encodings we don't handle yet.
+        if encoding in MISSING_ENCODINGS:
+            continue
+        # Test encoding detection for each file we have of encoding for
+        for file_name in listdir(path):
+            ext = splitext(file_name)[1].lower()
+            if ext not in ['.html', '.txt', '.xml', '.srt']:
+                continue
+            full_path = join(path, file_name)
+            test_case = full_path, encoding
+            if join(basename(path), file_name) in EXPECTED_FAILURES:
+                test_case = pytest.param(*test_case, marks=pytest.mark.xfail)
+            yield test_case
+
+
+def get_test_name(args):
+    return join(basename(dirname(args)), basename(args))
+
+
+@pytest.mark.parametrize ('file_name, encoding', gen_test_params(), ids=get_test_name)
+def test_encoding_detection(file_name, encoding):
+    with open(file_name, 'rb') as f:
+        input_bytes = f.read()
+        result = chardet.detect(input_bytes)
+        try:
+            expected_unicode = input_bytes.decode(encoding)
+        except LookupError:
+            expected_unicode = ''
+        try:
+            detected_unicode = input_bytes.decode(result['encoding'])
+        except (LookupError, UnicodeDecodeError, TypeError):
+            detected_unicode = ''
+    if result:
+        encoding_match = (result['encoding'] or '').lower() == encoding
+    else:
+        encoding_match = False
+    # Only care about mismatches that would actually result in different
+    # behavior when decoding
+    if not encoding_match and expected_unicode != detected_unicode:
+        wrapped_expected = '\n'.join(textwrap.wrap(expected_unicode, 100)) + '\n'
+        wrapped_detected = '\n'.join(textwrap.wrap(detected_unicode, 100)) + '\n'
+        diff = ''.join(ndiff(wrapped_expected.splitlines(True),
+                             wrapped_detected.splitlines(True)))
+    else:
+        diff = ''
+        encoding_match = True
+    assert encoding_match, ("Expected %s, but got %s for %s.  Character "
+                            "differences: \n%s" % (encoding,
+                                                   result,
+                                                   file_name,
+                                                   diff))
+
+
+if HAVE_HYPOTHESIS:
+    class JustALengthIssue(Exception):
+        pass
+
+
+    @pytest.mark.xfail
+    @given(st.text(min_size=1), st.sampled_from(['ascii', 'utf-8', 'utf-16',
+                                                 'utf-32', 'iso-8859-7',
+                                                 'iso-8859-8', 'windows-1255']),
+           st.randoms())
+    @settings(max_examples=200)
+    def test_never_fails_to_detect_if_there_is_a_valid_encoding(txt, enc, rnd):
+        try:
+            data = txt.encode(enc)
+        except UnicodeEncodeError:
+            assume(False)
+        detected = chardet.detect(data)['encoding']
+        if detected is None:
+            with pytest.raises(JustALengthIssue):
+                @given(st.text(), random=rnd)
+                @settings(verbosity=Verbosity.quiet, max_shrinks=0, max_examples=50)
+                def string_poisons_following_text(suffix):
+                    try:
+                        extended = (txt + suffix).encode(enc)
+                    except UnicodeEncodeError:
+                        assume(False)
+                    result = chardet.detect(extended)
+                    if result and result['encoding'] is not None:
+                        raise JustALengthIssue()
+
+
+    @given(st.text(min_size=1), st.sampled_from(['ascii', 'utf-8', 'utf-16',
+                                                 'utf-32', 'iso-8859-7',
+                                                 'iso-8859-8', 'windows-1255']),
+           st.randoms())
+    @settings(max_examples=200)
+    def test_detect_all_and_detect_one_should_agree(txt, enc, rnd):
+        try:
+            data = txt.encode(enc)
+        except UnicodeEncodeError:
+            assume(False)
+        try:
+            result = chardet.detect(data)
+            results = chardet.detect_all(data)
+            assert result['encoding'] == results[0]['encoding']
+        except Exception:
+            raise Exception('%s != %s' % (result, results))
diff --git a/contrib/python/chardet/py2/tests/ya.make b/contrib/python/chardet/py2/tests/ya.make
new file mode 100644
index 00000000000..3795bfa7fbd
--- /dev/null
+++ b/contrib/python/chardet/py2/tests/ya.make
@@ -0,0 +1,21 @@
+PY2TEST()
+
+SRCDIR(contrib/python/chardet/py2)
+
+TEST_SRCS(
+    test.py
+)
+
+PEERDIR(
+    contrib/python/chardet
+)
+
+DATA(
+    sbr://405525759
+)
+
+SIZE(MEDIUM)
+
+NO_LINT()
+
+END()
diff --git a/contrib/python/chardet/py3/.dist-info/METADATA b/contrib/python/chardet/py3/.dist-info/METADATA
new file mode 100644
index 00000000000..3ec4d797a2d
--- /dev/null
+++ b/contrib/python/chardet/py3/.dist-info/METADATA
@@ -0,0 +1,97 @@
+Metadata-Version: 2.1
+Name: chardet
+Version: 5.2.0
+Summary: Universal encoding detector for Python 3
+Home-page: https://github.com/chardet/chardet
+Author: Mark Pilgrim
+Author-email: mark@diveintomark.org
+Maintainer: Daniel Blanchard
+Maintainer-email: dan.blanchard@gmail.com
+License: LGPL
+Project-URL: Documentation, https://chardet.readthedocs.io/
+Project-URL: GitHub Project, https://github.com/chardet/chardet
+Project-URL: Issue Tracker, https://github.com/chardet/chardet/issues
+Keywords: encoding,i18n,xml
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.7
+License-File: LICENSE
+
+Chardet: The Universal Character Encoding Detector
+--------------------------------------------------
+
+.. image:: https://img.shields.io/travis/chardet/chardet/stable.svg
+   :alt: Build status
+   :target: https://travis-ci.org/chardet/chardet
+
+.. image:: https://img.shields.io/coveralls/chardet/chardet/stable.svg
+   :target: https://coveralls.io/r/chardet/chardet
+
+.. image:: https://img.shields.io/pypi/v/chardet.svg
+   :target: https://warehouse.python.org/project/chardet/
+   :alt: Latest version on PyPI
+
+.. image:: https://img.shields.io/pypi/l/chardet.svg
+   :alt: License
+
+
+Detects
+ - ASCII, UTF-8, UTF-16 (2 variants), UTF-32 (4 variants)
+ - Big5, GB2312, EUC-TW, HZ-GB-2312, ISO-2022-CN (Traditional and Simplified Chinese)
+ - EUC-JP, SHIFT_JIS, CP932, ISO-2022-JP (Japanese)
+ - EUC-KR, ISO-2022-KR, Johab (Korean)
+ - KOI8-R, MacCyrillic, IBM855, IBM866, ISO-8859-5, windows-1251 (Cyrillic)
+ - ISO-8859-5, windows-1251 (Bulgarian)
+ - ISO-8859-1, windows-1252, MacRoman (Western European languages)
+ - ISO-8859-7, windows-1253 (Greek)
+ - ISO-8859-8, windows-1255 (Visual and Logical Hebrew)
+ - TIS-620 (Thai)
+
+.. note::
+   Our ISO-8859-2 and windows-1250 (Hungarian) probers have been temporarily
+   disabled until we can retrain the models.
+
+Requires Python 3.7+.
+
+Installation
+------------
+
+Install from `PyPI <https://pypi.org/project/chardet/>`_::
+
+    pip install chardet
+
+Documentation
+-------------
+
+For users, docs are now available at https://chardet.readthedocs.io/.
+
+Command-line Tool
+-----------------
+
+chardet comes with a command-line script which reports on the encodings of one
+or more files::
+
+    % chardetect somefile someotherfile
+    somefile: windows-1252 with confidence 0.5
+    someotherfile: ascii with confidence 1.0
+
+About
+-----
+
+This is a continuation of Mark Pilgrim's excellent original chardet port from C, and `Ian Cordasco <https://github.com/sigmavirus24>`_'s
+`charade <https://github.com/sigmavirus24/charade>`_ Python 3-compatible fork.
+
+:maintainer: Dan Blanchard
diff --git a/contrib/python/chardet/py3/.dist-info/entry_points.txt b/contrib/python/chardet/py3/.dist-info/entry_points.txt
new file mode 100644
index 00000000000..c36a5e3ee47
--- /dev/null
+++ b/contrib/python/chardet/py3/.dist-info/entry_points.txt
@@ -0,0 +1,2 @@
+[console_scripts]
+chardetect = chardet.cli.chardetect:main
diff --git a/contrib/python/chardet/py3/.dist-info/top_level.txt b/contrib/python/chardet/py3/.dist-info/top_level.txt
new file mode 100644
index 00000000000..79236f25cda
--- /dev/null
+++ b/contrib/python/chardet/py3/.dist-info/top_level.txt
@@ -0,0 +1 @@
+chardet
diff --git a/contrib/python/chardet/py3/chardet/__init__.py b/contrib/python/chardet/py3/chardet/__init__.py
new file mode 100644
index 00000000000..fe581623d89
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/__init__.py
@@ -0,0 +1,115 @@
+######################## BEGIN LICENSE BLOCK ########################
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import List, Union
+
+from .charsetgroupprober import CharSetGroupProber
+from .charsetprober import CharSetProber
+from .enums import InputState
+from .resultdict import ResultDict
+from .universaldetector import UniversalDetector
+from .version import VERSION, __version__
+
+__all__ = ["UniversalDetector", "detect", "detect_all", "__version__", "VERSION"]
+
+
+def detect(
+    byte_str: Union[bytes, bytearray], should_rename_legacy: bool = False
+) -> ResultDict:
+    """
+    Detect the encoding of the given byte string.
+
+    :param byte_str:     The byte sequence to examine.
+    :type byte_str:      ``bytes`` or ``bytearray``
+    :param should_rename_legacy:  Should we rename legacy encodings
+                                  to their more modern equivalents?
+    :type should_rename_legacy:   ``bool``
+    """
+    if not isinstance(byte_str, bytearray):
+        if not isinstance(byte_str, bytes):
+            raise TypeError(
+                f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
+            )
+        byte_str = bytearray(byte_str)
+    detector = UniversalDetector(should_rename_legacy=should_rename_legacy)
+    detector.feed(byte_str)
+    return detector.close()
+
+
+def detect_all(
+    byte_str: Union[bytes, bytearray],
+    ignore_threshold: bool = False,
+    should_rename_legacy: bool = False,
+) -> List[ResultDict]:
+    """
+    Detect all the possible encodings of the given byte string.
+
+    :param byte_str:          The byte sequence to examine.
+    :type byte_str:           ``bytes`` or ``bytearray``
+    :param ignore_threshold:  Include encodings that are below
+                              ``UniversalDetector.MINIMUM_THRESHOLD``
+                              in results.
+    :type ignore_threshold:   ``bool``
+    :param should_rename_legacy:  Should we rename legacy encodings
+                                  to their more modern equivalents?
+    :type should_rename_legacy:   ``bool``
+    """
+    if not isinstance(byte_str, bytearray):
+        if not isinstance(byte_str, bytes):
+            raise TypeError(
+                f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
+            )
+        byte_str = bytearray(byte_str)
+
+    detector = UniversalDetector(should_rename_legacy=should_rename_legacy)
+    detector.feed(byte_str)
+    detector.close()
+
+    if detector.input_state == InputState.HIGH_BYTE:
+        results: List[ResultDict] = []
+        probers: List[CharSetProber] = []
+        for prober in detector.charset_probers:
+            if isinstance(prober, CharSetGroupProber):
+                probers.extend(p for p in prober.probers)
+            else:
+                probers.append(prober)
+        for prober in probers:
+            if ignore_threshold or prober.get_confidence() > detector.MINIMUM_THRESHOLD:
+                charset_name = prober.charset_name or ""
+                lower_charset_name = charset_name.lower()
+                # Use Windows encoding name instead of ISO-8859 if we saw any
+                # extra Windows-specific bytes
+                if lower_charset_name.startswith("iso-8859") and detector.has_win_bytes:
+                    charset_name = detector.ISO_WIN_MAP.get(
+                        lower_charset_name, charset_name
+                    )
+                # Rename legacy encodings with superset encodings if asked
+                if should_rename_legacy:
+                    charset_name = detector.LEGACY_MAP.get(
+                        charset_name.lower(), charset_name
+                    )
+                results.append(
+                    {
+                        "encoding": charset_name,
+                        "confidence": prober.get_confidence(),
+                        "language": prober.language,
+                    }
+                )
+        if len(results) > 0:
+            return sorted(results, key=lambda result: -result["confidence"])
+
+    return [detector.result]
diff --git a/contrib/python/chardet/py3/chardet/__main__.py b/contrib/python/chardet/py3/chardet/__main__.py
new file mode 100644
index 00000000000..c19b0d2d7a3
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/__main__.py
@@ -0,0 +1,6 @@
+"""Wrapper so people can run python -m chardet"""
+
+from .cli.chardetect import main
+
+if __name__ == "__main__":
+    main()
diff --git a/contrib/python/chardet/py3/chardet/big5freq.py b/contrib/python/chardet/py3/chardet/big5freq.py
new file mode 100644
index 00000000000..87d9f972edd
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/big5freq.py
@@ -0,0 +1,386 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+# Big5 frequency table
+# by Taiwan's Mandarin Promotion Council
+# <http://www.edu.tw:81/mandr/>
+#
+# 128  --> 0.42261
+# 256  --> 0.57851
+# 512  --> 0.74851
+# 1024 --> 0.89384
+# 2048 --> 0.97583
+#
+# Ideal Distribution Ratio = 0.74851/(1-0.74851) =2.98
+# Random Distribution Ration = 512/(5401-512)=0.105
+#
+# Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR
+
+BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75
+
+# Char to FreqOrder table
+BIG5_TABLE_SIZE = 5376
+# fmt: off
+BIG5_CHAR_TO_FREQ_ORDER = (
+   1,1801,1506, 255,1431, 198,   9,  82,   6,5008, 177, 202,3681,1256,2821, 110, #   16
+3814,  33,3274, 261,  76,  44,2114,  16,2946,2187,1176, 659,3971,  26,3451,2653, #   32
+1198,3972,3350,4202, 410,2215, 302, 590, 361,1964,   8, 204,  58,4510,5009,1932, #   48
+  63,5010,5011, 317,1614,  75, 222, 159,4203,2417,1480,5012,3555,3091, 224,2822, #   64
+3682,   3,  10,3973,1471,  29,2787,1135,2866,1940, 873, 130,3275,1123, 312,5013, #   80
+4511,2052, 507, 252, 682,5014, 142,1915, 124, 206,2947,  34,3556,3204,  64, 604, #   96
+5015,2501,1977,1978, 155,1991, 645, 641,1606,5016,3452, 337,  72, 406,5017,  80, #  112
+ 630, 238,3205,1509, 263, 939,1092,2654, 756,1440,1094,3453, 449,  69,2987, 591, #  128
+ 179,2096, 471, 115,2035,1844,  60,  50,2988, 134, 806,1869, 734,2036,3454, 180, #  144
+ 995,1607, 156, 537,2907, 688,5018, 319,1305, 779,2145, 514,2379, 298,4512, 359, #  160
+2502,  90,2716,1338, 663,  11, 906,1099,2553,  20,2441, 182, 532,1716,5019, 732, #  176
+1376,4204,1311,1420,3206,  25,2317,1056, 113, 399, 382,1950, 242,3455,2474, 529, #  192
+3276, 475,1447,3683,5020, 117,  21, 656, 810,1297,2300,2334,3557,5021, 126,4205, #  208
+ 706, 456, 150, 613,4513,  71,1118,2037,4206, 145,3092,  85, 835, 486,2115,1246, #  224
+1426, 428, 727,1285,1015, 800, 106, 623, 303,1281,5022,2128,2359, 347,3815, 221, #  240
+3558,3135,5023,1956,1153,4207,  83, 296,1199,3093, 192, 624,  93,5024, 822,1898, #  256
+2823,3136, 795,2065, 991,1554,1542,1592,  27,  43,2867, 859, 139,1456, 860,4514, #  272
+ 437, 712,3974, 164,2397,3137, 695, 211,3037,2097, 195,3975,1608,3559,3560,3684, #  288
+3976, 234, 811,2989,2098,3977,2233,1441,3561,1615,2380, 668,2077,1638, 305, 228, #  304
+1664,4515, 467, 415,5025, 262,2099,1593, 239, 108, 300, 200,1033, 512,1247,2078, #  320
+5026,5027,2176,3207,3685,2682, 593, 845,1062,3277,  88,1723,2038,3978,1951, 212, #  336
+ 266, 152, 149, 468,1899,4208,4516,  77, 187,5028,3038,  37,   5,2990,5029,3979, #  352
+5030,5031,  39,2524,4517,2908,3208,2079,  55, 148,  74,4518, 545, 483,1474,1029, #  368
+1665, 217,1870,1531,3138,1104,2655,4209,  24, 172,3562, 900,3980,3563,3564,4519, #  384
+  32,1408,2824,1312, 329, 487,2360,2251,2717, 784,2683,   4,3039,3351,1427,1789, #  400
+ 188, 109, 499,5032,3686,1717,1790, 888,1217,3040,4520,5033,3565,5034,3352,1520, #  416
+3687,3981, 196,1034, 775,5035,5036, 929,1816, 249, 439,  38,5037,1063,5038, 794, #  432
+3982,1435,2301,  46, 178,3278,2066,5039,2381,5040, 214,1709,4521, 804,  35, 707, #  448
+ 324,3688,1601,2554, 140, 459,4210,5041,5042,1365, 839, 272, 978,2262,2580,3456, #  464
+2129,1363,3689,1423, 697, 100,3094,  48,  70,1231, 495,3139,2196,5043,1294,5044, #  480
+2080, 462, 586,1042,3279, 853, 256, 988, 185,2382,3457,1698, 434,1084,5045,3458, #  496
+ 314,2625,2788,4522,2335,2336, 569,2285, 637,1817,2525, 757,1162,1879,1616,3459, #  512
+ 287,1577,2116, 768,4523,1671,2868,3566,2526,1321,3816, 909,2418,5046,4211, 933, #  528
+3817,4212,2053,2361,1222,4524, 765,2419,1322, 786,4525,5047,1920,1462,1677,2909, #  544
+1699,5048,4526,1424,2442,3140,3690,2600,3353,1775,1941,3460,3983,4213, 309,1369, #  560
+1130,2825, 364,2234,1653,1299,3984,3567,3985,3986,2656, 525,1085,3041, 902,2001, #  576
+1475, 964,4527, 421,1845,1415,1057,2286, 940,1364,3141, 376,4528,4529,1381,   7, #  592
+2527, 983,2383, 336,1710,2684,1846, 321,3461, 559,1131,3042,2752,1809,1132,1313, #  608
+ 265,1481,1858,5049, 352,1203,2826,3280, 167,1089, 420,2827, 776, 792,1724,3568, #  624
+4214,2443,3281,5050,4215,5051, 446, 229, 333,2753, 901,3818,1200,1557,4530,2657, #  640
+1921, 395,2754,2685,3819,4216,1836, 125, 916,3209,2626,4531,5052,5053,3820,5054, #  656
+5055,5056,4532,3142,3691,1133,2555,1757,3462,1510,2318,1409,3569,5057,2146, 438, #  672
+2601,2910,2384,3354,1068, 958,3043, 461, 311,2869,2686,4217,1916,3210,4218,1979, #  688
+ 383, 750,2755,2627,4219, 274, 539, 385,1278,1442,5058,1154,1965, 384, 561, 210, #  704
+  98,1295,2556,3570,5059,1711,2420,1482,3463,3987,2911,1257, 129,5060,3821, 642, #  720
+ 523,2789,2790,2658,5061, 141,2235,1333,  68, 176, 441, 876, 907,4220, 603,2602, #  736
+ 710, 171,3464, 404, 549,  18,3143,2398,1410,3692,1666,5062,3571,4533,2912,4534, #  752
+5063,2991, 368,5064, 146, 366,  99, 871,3693,1543, 748, 807,1586,1185,  22,2263, #  768
+ 379,3822,3211,5065,3212, 505,1942,2628,1992,1382,2319,5066, 380,2362, 218, 702, #  784
+1818,1248,3465,3044,3572,3355,3282,5067,2992,3694, 930,3283,3823,5068,  59,5069, #  800
+ 585, 601,4221, 497,3466,1112,1314,4535,1802,5070,1223,1472,2177,5071, 749,1837, #  816
+ 690,1900,3824,1773,3988,1476, 429,1043,1791,2236,2117, 917,4222, 447,1086,1629, #  832
+5072, 556,5073,5074,2021,1654, 844,1090, 105, 550, 966,1758,2828,1008,1783, 686, #  848
+1095,5075,2287, 793,1602,5076,3573,2603,4536,4223,2948,2302,4537,3825, 980,2503, #  864
+ 544, 353, 527,4538, 908,2687,2913,5077, 381,2629,1943,1348,5078,1341,1252, 560, #  880
+3095,5079,3467,2870,5080,2054, 973, 886,2081, 143,4539,5081,5082, 157,3989, 496, #  896
+4224,  57, 840, 540,2039,4540,4541,3468,2118,1445, 970,2264,1748,1966,2082,4225, #  912
+3144,1234,1776,3284,2829,3695, 773,1206,2130,1066,2040,1326,3990,1738,1725,4226, #  928
+ 279,3145,  51,1544,2604, 423,1578,2131,2067, 173,4542,1880,5083,5084,1583, 264, #  944
+ 610,3696,4543,2444, 280, 154,5085,5086,5087,1739, 338,1282,3096, 693,2871,1411, #  960
+1074,3826,2445,5088,4544,5089,5090,1240, 952,2399,5091,2914,1538,2688, 685,1483, #  976
+4227,2475,1436, 953,4228,2055,4545, 671,2400,  79,4229,2446,3285, 608, 567,2689, #  992
+3469,4230,4231,1691, 393,1261,1792,2401,5092,4546,5093,5094,5095,5096,1383,1672, # 1008
+3827,3213,1464, 522,1119, 661,1150, 216, 675,4547,3991,1432,3574, 609,4548,2690, # 1024
+2402,5097,5098,5099,4232,3045,   0,5100,2476, 315, 231,2447, 301,3356,4549,2385, # 1040
+5101, 233,4233,3697,1819,4550,4551,5102,  96,1777,1315,2083,5103, 257,5104,1810, # 1056
+3698,2718,1139,1820,4234,2022,1124,2164,2791,1778,2659,5105,3097, 363,1655,3214, # 1072
+5106,2993,5107,5108,5109,3992,1567,3993, 718, 103,3215, 849,1443, 341,3357,2949, # 1088
+1484,5110,1712, 127,  67, 339,4235,2403, 679,1412, 821,5111,5112, 834, 738, 351, # 1104
+2994,2147, 846, 235,1497,1881, 418,1993,3828,2719, 186,1100,2148,2756,3575,1545, # 1120
+1355,2950,2872,1377, 583,3994,4236,2581,2995,5113,1298,3699,1078,2557,3700,2363, # 1136
+  78,3829,3830, 267,1289,2100,2002,1594,4237, 348, 369,1274,2197,2178,1838,4552, # 1152
+1821,2830,3701,2757,2288,2003,4553,2951,2758, 144,3358, 882,4554,3995,2759,3470, # 1168
+4555,2915,5114,4238,1726, 320,5115,3996,3046, 788,2996,5116,2831,1774,1327,2873, # 1184
+3997,2832,5117,1306,4556,2004,1700,3831,3576,2364,2660, 787,2023, 506, 824,3702, # 1200
+ 534, 323,4557,1044,3359,2024,1901, 946,3471,5118,1779,1500,1678,5119,1882,4558, # 1216
+ 165, 243,4559,3703,2528, 123, 683,4239, 764,4560,  36,3998,1793, 589,2916, 816, # 1232
+ 626,1667,3047,2237,1639,1555,1622,3832,3999,5120,4000,2874,1370,1228,1933, 891, # 1248
+2084,2917, 304,4240,5121, 292,2997,2720,3577, 691,2101,4241,1115,4561, 118, 662, # 1264
+5122, 611,1156, 854,2386,1316,2875,   2, 386, 515,2918,5123,5124,3286, 868,2238, # 1280
+1486, 855,2661, 785,2216,3048,5125,1040,3216,3578,5126,3146, 448,5127,1525,5128, # 1296
+2165,4562,5129,3833,5130,4242,2833,3579,3147, 503, 818,4001,3148,1568, 814, 676, # 1312
+1444, 306,1749,5131,3834,1416,1030, 197,1428, 805,2834,1501,4563,5132,5133,5134, # 1328
+1994,5135,4564,5136,5137,2198,  13,2792,3704,2998,3149,1229,1917,5138,3835,2132, # 1344
+5139,4243,4565,2404,3580,5140,2217,1511,1727,1120,5141,5142, 646,3836,2448, 307, # 1360
+5143,5144,1595,3217,5145,5146,5147,3705,1113,1356,4002,1465,2529,2530,5148, 519, # 1376
+5149, 128,2133,  92,2289,1980,5150,4003,1512, 342,3150,2199,5151,2793,2218,1981, # 1392
+3360,4244, 290,1656,1317, 789, 827,2365,5152,3837,4566, 562, 581,4004,5153, 401, # 1408
+4567,2252,  94,4568,5154,1399,2794,5155,1463,2025,4569,3218,1944,5156, 828,1105, # 1424
+4245,1262,1394,5157,4246, 605,4570,5158,1784,2876,5159,2835, 819,2102, 578,2200, # 1440
+2952,5160,1502, 436,3287,4247,3288,2836,4005,2919,3472,3473,5161,2721,2320,5162, # 1456
+5163,2337,2068,  23,4571, 193, 826,3838,2103, 699,1630,4248,3098, 390,1794,1064, # 1472
+3581,5164,1579,3099,3100,1400,5165,4249,1839,1640,2877,5166,4572,4573, 137,4250, # 1488
+ 598,3101,1967, 780, 104, 974,2953,5167, 278, 899, 253, 402, 572, 504, 493,1339, # 1504
+5168,4006,1275,4574,2582,2558,5169,3706,3049,3102,2253, 565,1334,2722, 863,  41, # 1520
+5170,5171,4575,5172,1657,2338,  19, 463,2760,4251, 606,5173,2999,3289,1087,2085, # 1536
+1323,2662,3000,5174,1631,1623,1750,4252,2691,5175,2878, 791,2723,2663,2339, 232, # 1552
+2421,5176,3001,1498,5177,2664,2630, 755,1366,3707,3290,3151,2026,1609, 119,1918, # 1568
+3474, 862,1026,4253,5178,4007,3839,4576,4008,4577,2265,1952,2477,5179,1125, 817, # 1584
+4254,4255,4009,1513,1766,2041,1487,4256,3050,3291,2837,3840,3152,5180,5181,1507, # 1600
+5182,2692, 733,  40,1632,1106,2879, 345,4257, 841,2531, 230,4578,3002,1847,3292, # 1616
+3475,5183,1263, 986,3476,5184, 735, 879, 254,1137, 857, 622,1300,1180,1388,1562, # 1632
+4010,4011,2954, 967,2761,2665,1349, 592,2134,1692,3361,3003,1995,4258,1679,4012, # 1648
+1902,2188,5185, 739,3708,2724,1296,1290,5186,4259,2201,2202,1922,1563,2605,2559, # 1664
+1871,2762,3004,5187, 435,5188, 343,1108, 596,  17,1751,4579,2239,3477,3709,5189, # 1680
+4580, 294,3582,2955,1693, 477, 979, 281,2042,3583, 643,2043,3710,2631,2795,2266, # 1696
+1031,2340,2135,2303,3584,4581, 367,1249,2560,5190,3585,5191,4582,1283,3362,2005, # 1712
+ 240,1762,3363,4583,4584, 836,1069,3153, 474,5192,2149,2532, 268,3586,5193,3219, # 1728
+1521,1284,5194,1658,1546,4260,5195,3587,3588,5196,4261,3364,2693,1685,4262, 961, # 1744
+1673,2632, 190,2006,2203,3841,4585,4586,5197, 570,2504,3711,1490,5198,4587,2633, # 1760
+3293,1957,4588, 584,1514, 396,1045,1945,5199,4589,1968,2449,5200,5201,4590,4013, # 1776
+ 619,5202,3154,3294, 215,2007,2796,2561,3220,4591,3221,4592, 763,4263,3842,4593, # 1792
+5203,5204,1958,1767,2956,3365,3712,1174, 452,1477,4594,3366,3155,5205,2838,1253, # 1808
+2387,2189,1091,2290,4264, 492,5206, 638,1169,1825,2136,1752,4014, 648, 926,1021, # 1824
+1324,4595, 520,4596, 997, 847,1007, 892,4597,3843,2267,1872,3713,2405,1785,4598, # 1840
+1953,2957,3103,3222,1728,4265,2044,3714,4599,2008,1701,3156,1551,  30,2268,4266, # 1856
+5207,2027,4600,3589,5208, 501,5209,4267, 594,3478,2166,1822,3590,3479,3591,3223, # 1872
+ 829,2839,4268,5210,1680,3157,1225,4269,5211,3295,4601,4270,3158,2341,5212,4602, # 1888
+4271,5213,4015,4016,5214,1848,2388,2606,3367,5215,4603, 374,4017, 652,4272,4273, # 1904
+ 375,1140, 798,5216,5217,5218,2366,4604,2269, 546,1659, 138,3051,2450,4605,5219, # 1920
+2254, 612,1849, 910, 796,3844,1740,1371, 825,3845,3846,5220,2920,2562,5221, 692, # 1936
+ 444,3052,2634, 801,4606,4274,5222,1491, 244,1053,3053,4275,4276, 340,5223,4018, # 1952
+1041,3005, 293,1168,  87,1357,5224,1539, 959,5225,2240, 721, 694,4277,3847, 219, # 1968
+1478, 644,1417,3368,2666,1413,1401,1335,1389,4019,5226,5227,3006,2367,3159,1826, # 1984
+ 730,1515, 184,2840,  66,4607,5228,1660,2958, 246,3369, 378,1457, 226,3480, 975, # 2000
+4020,2959,1264,3592, 674, 696,5229, 163,5230,1141,2422,2167, 713,3593,3370,4608, # 2016
+4021,5231,5232,1186,  15,5233,1079,1070,5234,1522,3224,3594, 276,1050,2725, 758, # 2032
+1126, 653,2960,3296,5235,2342, 889,3595,4022,3104,3007, 903,1250,4609,4023,3481, # 2048
+3596,1342,1681,1718, 766,3297, 286,  89,2961,3715,5236,1713,5237,2607,3371,3008, # 2064
+5238,2962,2219,3225,2880,5239,4610,2505,2533, 181, 387,1075,4024, 731,2190,3372, # 2080
+5240,3298, 310, 313,3482,2304, 770,4278,  54,3054, 189,4611,3105,3848,4025,5241, # 2096
+1230,1617,1850, 355,3597,4279,4612,3373, 111,4280,3716,1350,3160,3483,3055,4281, # 2112
+2150,3299,3598,5242,2797,4026,4027,3009, 722,2009,5243,1071, 247,1207,2343,2478, # 2128
+1378,4613,2010, 864,1437,1214,4614, 373,3849,1142,2220, 667,4615, 442,2763,2563, # 2144
+3850,4028,1969,4282,3300,1840, 837, 170,1107, 934,1336,1883,5244,5245,2119,4283, # 2160
+2841, 743,1569,5246,4616,4284, 582,2389,1418,3484,5247,1803,5248, 357,1395,1729, # 2176
+3717,3301,2423,1564,2241,5249,3106,3851,1633,4617,1114,2086,4285,1532,5250, 482, # 2192
+2451,4618,5251,5252,1492, 833,1466,5253,2726,3599,1641,2842,5254,1526,1272,3718, # 2208
+4286,1686,1795, 416,2564,1903,1954,1804,5255,3852,2798,3853,1159,2321,5256,2881, # 2224
+4619,1610,1584,3056,2424,2764, 443,3302,1163,3161,5257,5258,4029,5259,4287,2506, # 2240
+3057,4620,4030,3162,2104,1647,3600,2011,1873,4288,5260,4289, 431,3485,5261, 250, # 2256
+  97,  81,4290,5262,1648,1851,1558, 160, 848,5263, 866, 740,1694,5264,2204,2843, # 2272
+3226,4291,4621,3719,1687, 950,2479, 426, 469,3227,3720,3721,4031,5265,5266,1188, # 2288
+ 424,1996, 861,3601,4292,3854,2205,2694, 168,1235,3602,4293,5267,2087,1674,4622, # 2304
+3374,3303, 220,2565,1009,5268,3855, 670,3010, 332,1208, 717,5269,5270,3603,2452, # 2320
+4032,3375,5271, 513,5272,1209,2882,3376,3163,4623,1080,5273,5274,5275,5276,2534, # 2336
+3722,3604, 815,1587,4033,4034,5277,3605,3486,3856,1254,4624,1328,3058,1390,4035, # 2352
+1741,4036,3857,4037,5278, 236,3858,2453,3304,5279,5280,3723,3859,1273,3860,4625, # 2368
+5281, 308,5282,4626, 245,4627,1852,2480,1307,2583, 430, 715,2137,2454,5283, 270, # 2384
+ 199,2883,4038,5284,3606,2727,1753, 761,1754, 725,1661,1841,4628,3487,3724,5285, # 2400
+5286, 587,  14,3305, 227,2608, 326, 480,2270, 943,2765,3607, 291, 650,1884,5287, # 2416
+1702,1226, 102,1547,  62,3488, 904,4629,3489,1164,4294,5288,5289,1224,1548,2766, # 2432
+ 391, 498,1493,5290,1386,1419,5291,2056,1177,4630, 813, 880,1081,2368, 566,1145, # 2448
+4631,2291,1001,1035,2566,2609,2242, 394,1286,5292,5293,2069,5294,  86,1494,1730, # 2464
+4039, 491,1588, 745, 897,2963, 843,3377,4040,2767,2884,3306,1768, 998,2221,2070, # 2480
+ 397,1827,1195,1970,3725,3011,3378, 284,5295,3861,2507,2138,2120,1904,5296,4041, # 2496
+2151,4042,4295,1036,3490,1905, 114,2567,4296, 209,1527,5297,5298,2964,2844,2635, # 2512
+2390,2728,3164, 812,2568,5299,3307,5300,1559, 737,1885,3726,1210, 885,  28,2695, # 2528
+3608,3862,5301,4297,1004,1780,4632,5302, 346,1982,2222,2696,4633,3863,1742, 797, # 2544
+1642,4043,1934,1072,1384,2152, 896,4044,3308,3727,3228,2885,3609,5303,2569,1959, # 2560
+4634,2455,1786,5304,5305,5306,4045,4298,1005,1308,3728,4299,2729,4635,4636,1528, # 2576
+2610, 161,1178,4300,1983, 987,4637,1101,4301, 631,4046,1157,3229,2425,1343,1241, # 2592
+1016,2243,2570, 372, 877,2344,2508,1160, 555,1935, 911,4047,5307, 466,1170, 169, # 2608
+1051,2921,2697,3729,2481,3012,1182,2012,2571,1251,2636,5308, 992,2345,3491,1540, # 2624
+2730,1201,2071,2406,1997,2482,5309,4638, 528,1923,2191,1503,1874,1570,2369,3379, # 2640
+3309,5310, 557,1073,5311,1828,3492,2088,2271,3165,3059,3107, 767,3108,2799,4639, # 2656
+1006,4302,4640,2346,1267,2179,3730,3230, 778,4048,3231,2731,1597,2667,5312,4641, # 2672
+5313,3493,5314,5315,5316,3310,2698,1433,3311, 131,  95,1504,4049, 723,4303,3166, # 2688
+1842,3610,2768,2192,4050,2028,2105,3731,5317,3013,4051,1218,5318,3380,3232,4052, # 2704
+4304,2584, 248,1634,3864, 912,5319,2845,3732,3060,3865, 654,  53,5320,3014,5321, # 2720
+1688,4642, 777,3494,1032,4053,1425,5322, 191, 820,2121,2846, 971,4643, 931,3233, # 2736
+ 135, 664, 783,3866,1998, 772,2922,1936,4054,3867,4644,2923,3234, 282,2732, 640, # 2752
+1372,3495,1127, 922, 325,3381,5323,5324, 711,2045,5325,5326,4055,2223,2800,1937, # 2768
+4056,3382,2224,2255,3868,2305,5327,4645,3869,1258,3312,4057,3235,2139,2965,4058, # 2784
+4059,5328,2225, 258,3236,4646, 101,1227,5329,3313,1755,5330,1391,3314,5331,2924, # 2800
+2057, 893,5332,5333,5334,1402,4305,2347,5335,5336,3237,3611,5337,5338, 878,1325, # 2816
+1781,2801,4647, 259,1385,2585, 744,1183,2272,4648,5339,4060,2509,5340, 684,1024, # 2832
+4306,5341, 472,3612,3496,1165,3315,4061,4062, 322,2153, 881, 455,1695,1152,1340, # 2848
+ 660, 554,2154,4649,1058,4650,4307, 830,1065,3383,4063,4651,1924,5342,1703,1919, # 2864
+5343, 932,2273, 122,5344,4652, 947, 677,5345,3870,2637, 297,1906,1925,2274,4653, # 2880
+2322,3316,5346,5347,4308,5348,4309,  84,4310, 112, 989,5349, 547,1059,4064, 701, # 2896
+3613,1019,5350,4311,5351,3497, 942, 639, 457,2306,2456, 993,2966, 407, 851, 494, # 2912
+4654,3384, 927,5352,1237,5353,2426,3385, 573,4312, 680, 921,2925,1279,1875, 285, # 2928
+ 790,1448,1984, 719,2168,5354,5355,4655,4065,4066,1649,5356,1541, 563,5357,1077, # 2944
+5358,3386,3061,3498, 511,3015,4067,4068,3733,4069,1268,2572,3387,3238,4656,4657, # 2960
+5359, 535,1048,1276,1189,2926,2029,3167,1438,1373,2847,2967,1134,2013,5360,4313, # 2976
+1238,2586,3109,1259,5361, 700,5362,2968,3168,3734,4314,5363,4315,1146,1876,1907, # 2992
+4658,2611,4070, 781,2427, 132,1589, 203, 147, 273,2802,2407, 898,1787,2155,4071, # 3008
+4072,5364,3871,2803,5365,5366,4659,4660,5367,3239,5368,1635,3872, 965,5369,1805, # 3024
+2699,1516,3614,1121,1082,1329,3317,4073,1449,3873,  65,1128,2848,2927,2769,1590, # 3040
+3874,5370,5371,  12,2668,  45, 976,2587,3169,4661, 517,2535,1013,1037,3240,5372, # 3056
+3875,2849,5373,3876,5374,3499,5375,2612, 614,1999,2323,3877,3110,2733,2638,5376, # 3072
+2588,4316, 599,1269,5377,1811,3735,5378,2700,3111, 759,1060, 489,1806,3388,3318, # 3088
+1358,5379,5380,2391,1387,1215,2639,2256, 490,5381,5382,4317,1759,2392,2348,5383, # 3104
+4662,3878,1908,4074,2640,1807,3241,4663,3500,3319,2770,2349, 874,5384,5385,3501, # 3120
+3736,1859,  91,2928,3737,3062,3879,4664,5386,3170,4075,2669,5387,3502,1202,1403, # 3136
+3880,2969,2536,1517,2510,4665,3503,2511,5388,4666,5389,2701,1886,1495,1731,4076, # 3152
+2370,4667,5390,2030,5391,5392,4077,2702,1216, 237,2589,4318,2324,4078,3881,4668, # 3168
+4669,2703,3615,3504, 445,4670,5393,5394,5395,5396,2771,  61,4079,3738,1823,4080, # 3184
+5397, 687,2046, 935, 925, 405,2670, 703,1096,1860,2734,4671,4081,1877,1367,2704, # 3200
+3389, 918,2106,1782,2483, 334,3320,1611,1093,4672, 564,3171,3505,3739,3390, 945, # 3216
+2641,2058,4673,5398,1926, 872,4319,5399,3506,2705,3112, 349,4320,3740,4082,4674, # 3232
+3882,4321,3741,2156,4083,4675,4676,4322,4677,2408,2047, 782,4084, 400, 251,4323, # 3248
+1624,5400,5401, 277,3742, 299,1265, 476,1191,3883,2122,4324,4325,1109, 205,5402, # 3264
+2590,1000,2157,3616,1861,5403,5404,5405,4678,5406,4679,2573, 107,2484,2158,4085, # 3280
+3507,3172,5407,1533, 541,1301, 158, 753,4326,2886,3617,5408,1696, 370,1088,4327, # 3296
+4680,3618, 579, 327, 440, 162,2244, 269,1938,1374,3508, 968,3063,  56,1396,3113, # 3312
+2107,3321,3391,5409,1927,2159,4681,3016,5410,3619,5411,5412,3743,4682,2485,5413, # 3328
+2804,5414,1650,4683,5415,2613,5416,5417,4086,2671,3392,1149,3393,4087,3884,4088, # 3344
+5418,1076,  49,5419, 951,3242,3322,3323, 450,2850, 920,5420,1812,2805,2371,4328, # 3360
+1909,1138,2372,3885,3509,5421,3243,4684,1910,1147,1518,2428,4685,3886,5422,4686, # 3376
+2393,2614, 260,1796,3244,5423,5424,3887,3324, 708,5425,3620,1704,5426,3621,1351, # 3392
+1618,3394,3017,1887, 944,4329,3395,4330,3064,3396,4331,5427,3744, 422, 413,1714, # 3408
+3325, 500,2059,2350,4332,2486,5428,1344,1911, 954,5429,1668,5430,5431,4089,2409, # 3424
+4333,3622,3888,4334,5432,2307,1318,2512,3114, 133,3115,2887,4687, 629,  31,2851, # 3440
+2706,3889,4688, 850, 949,4689,4090,2970,1732,2089,4335,1496,1853,5433,4091, 620, # 3456
+3245, 981,1242,3745,3397,1619,3746,1643,3326,2140,2457,1971,1719,3510,2169,5434, # 3472
+3246,5435,5436,3398,1829,5437,1277,4690,1565,2048,5438,1636,3623,3116,5439, 869, # 3488
+2852, 655,3890,3891,3117,4092,3018,3892,1310,3624,4691,5440,5441,5442,1733, 558, # 3504
+4692,3747, 335,1549,3065,1756,4336,3748,1946,3511,1830,1291,1192, 470,2735,2108, # 3520
+2806, 913,1054,4093,5443,1027,5444,3066,4094,4693, 982,2672,3399,3173,3512,3247, # 3536
+3248,1947,2807,5445, 571,4694,5446,1831,5447,3625,2591,1523,2429,5448,2090, 984, # 3552
+4695,3749,1960,5449,3750, 852, 923,2808,3513,3751, 969,1519, 999,2049,2325,1705, # 3568
+5450,3118, 615,1662, 151, 597,4095,2410,2326,1049, 275,4696,3752,4337, 568,3753, # 3584
+3626,2487,4338,3754,5451,2430,2275, 409,3249,5452,1566,2888,3514,1002, 769,2853, # 3600
+ 194,2091,3174,3755,2226,3327,4339, 628,1505,5453,5454,1763,2180,3019,4096, 521, # 3616
+1161,2592,1788,2206,2411,4697,4097,1625,4340,4341, 412,  42,3119, 464,5455,2642, # 3632
+4698,3400,1760,1571,2889,3515,2537,1219,2207,3893,2643,2141,2373,4699,4700,3328, # 3648
+1651,3401,3627,5456,5457,3628,2488,3516,5458,3756,5459,5460,2276,2092, 460,5461, # 3664
+4701,5462,3020, 962, 588,3629, 289,3250,2644,1116,  52,5463,3067,1797,5464,5465, # 3680
+5466,1467,5467,1598,1143,3757,4342,1985,1734,1067,4702,1280,3402, 465,4703,1572, # 3696
+ 510,5468,1928,2245,1813,1644,3630,5469,4704,3758,5470,5471,2673,1573,1534,5472, # 3712
+5473, 536,1808,1761,3517,3894,3175,2645,5474,5475,5476,4705,3518,2929,1912,2809, # 3728
+5477,3329,1122, 377,3251,5478, 360,5479,5480,4343,1529, 551,5481,2060,3759,1769, # 3744
+2431,5482,2930,4344,3330,3120,2327,2109,2031,4706,1404, 136,1468,1479, 672,1171, # 3760
+3252,2308, 271,3176,5483,2772,5484,2050, 678,2736, 865,1948,4707,5485,2014,4098, # 3776
+2971,5486,2737,2227,1397,3068,3760,4708,4709,1735,2931,3403,3631,5487,3895, 509, # 3792
+2854,2458,2890,3896,5488,5489,3177,3178,4710,4345,2538,4711,2309,1166,1010, 552, # 3808
+ 681,1888,5490,5491,2972,2973,4099,1287,1596,1862,3179, 358, 453, 736, 175, 478, # 3824
+1117, 905,1167,1097,5492,1854,1530,5493,1706,5494,2181,3519,2292,3761,3520,3632, # 3840
+4346,2093,4347,5495,3404,1193,2489,4348,1458,2193,2208,1863,1889,1421,3331,2932, # 3856
+3069,2182,3521, 595,2123,5496,4100,5497,5498,4349,1707,2646, 223,3762,1359, 751, # 3872
+3121, 183,3522,5499,2810,3021, 419,2374, 633, 704,3897,2394, 241,5500,5501,5502, # 3888
+ 838,3022,3763,2277,2773,2459,3898,1939,2051,4101,1309,3122,2246,1181,5503,1136, # 3904
+2209,3899,2375,1446,4350,2310,4712,5504,5505,4351,1055,2615, 484,3764,5506,4102, # 3920
+ 625,4352,2278,3405,1499,4353,4103,5507,4104,4354,3253,2279,2280,3523,5508,5509, # 3936
+2774, 808,2616,3765,3406,4105,4355,3123,2539, 526,3407,3900,4356, 955,5510,1620, # 3952
+4357,2647,2432,5511,1429,3766,1669,1832, 994, 928,5512,3633,1260,5513,5514,5515, # 3968
+1949,2293, 741,2933,1626,4358,2738,2460, 867,1184, 362,3408,1392,5516,5517,4106, # 3984
+4359,1770,1736,3254,2934,4713,4714,1929,2707,1459,1158,5518,3070,3409,2891,1292, # 4000
+1930,2513,2855,3767,1986,1187,2072,2015,2617,4360,5519,2574,2514,2170,3768,2490, # 4016
+3332,5520,3769,4715,5521,5522, 666,1003,3023,1022,3634,4361,5523,4716,1814,2257, # 4032
+ 574,3901,1603, 295,1535, 705,3902,4362, 283, 858, 417,5524,5525,3255,4717,4718, # 4048
+3071,1220,1890,1046,2281,2461,4107,1393,1599, 689,2575, 388,4363,5526,2491, 802, # 4064
+5527,2811,3903,2061,1405,2258,5528,4719,3904,2110,1052,1345,3256,1585,5529, 809, # 4080
+5530,5531,5532, 575,2739,3524, 956,1552,1469,1144,2328,5533,2329,1560,2462,3635, # 4096
+3257,4108, 616,2210,4364,3180,2183,2294,5534,1833,5535,3525,4720,5536,1319,3770, # 4112
+3771,1211,3636,1023,3258,1293,2812,5537,5538,5539,3905, 607,2311,3906, 762,2892, # 4128
+1439,4365,1360,4721,1485,3072,5540,4722,1038,4366,1450,2062,2648,4367,1379,4723, # 4144
+2593,5541,5542,4368,1352,1414,2330,2935,1172,5543,5544,3907,3908,4724,1798,1451, # 4160
+5545,5546,5547,5548,2936,4109,4110,2492,2351, 411,4111,4112,3637,3333,3124,4725, # 4176
+1561,2674,1452,4113,1375,5549,5550,  47,2974, 316,5551,1406,1591,2937,3181,5552, # 4192
+1025,2142,3125,3182, 354,2740, 884,2228,4369,2412, 508,3772, 726,3638, 996,2433, # 4208
+3639, 729,5553, 392,2194,1453,4114,4726,3773,5554,5555,2463,3640,2618,1675,2813, # 4224
+ 919,2352,2975,2353,1270,4727,4115,  73,5556,5557, 647,5558,3259,2856,2259,1550, # 4240
+1346,3024,5559,1332, 883,3526,5560,5561,5562,5563,3334,2775,5564,1212, 831,1347, # 4256
+4370,4728,2331,3909,1864,3073, 720,3910,4729,4730,3911,5565,4371,5566,5567,4731, # 4272
+5568,5569,1799,4732,3774,2619,4733,3641,1645,2376,4734,5570,2938, 669,2211,2675, # 4288
+2434,5571,2893,5572,5573,1028,3260,5574,4372,2413,5575,2260,1353,5576,5577,4735, # 4304
+3183, 518,5578,4116,5579,4373,1961,5580,2143,4374,5581,5582,3025,2354,2355,3912, # 4320
+ 516,1834,1454,4117,2708,4375,4736,2229,2620,1972,1129,3642,5583,2776,5584,2976, # 4336
+1422, 577,1470,3026,1524,3410,5585,5586, 432,4376,3074,3527,5587,2594,1455,2515, # 4352
+2230,1973,1175,5588,1020,2741,4118,3528,4737,5589,2742,5590,1743,1361,3075,3529, # 4368
+2649,4119,4377,4738,2295, 895, 924,4378,2171, 331,2247,3076, 166,1627,3077,1098, # 4384
+5591,1232,2894,2231,3411,4739, 657, 403,1196,2377, 542,3775,3412,1600,4379,3530, # 4400
+5592,4740,2777,3261, 576, 530,1362,4741,4742,2540,2676,3776,4120,5593, 842,3913, # 4416
+5594,2814,2032,1014,4121, 213,2709,3413, 665, 621,4380,5595,3777,2939,2435,5596, # 4432
+2436,3335,3643,3414,4743,4381,2541,4382,4744,3644,1682,4383,3531,1380,5597, 724, # 4448
+2282, 600,1670,5598,1337,1233,4745,3126,2248,5599,1621,4746,5600, 651,4384,5601, # 4464
+1612,4385,2621,5602,2857,5603,2743,2312,3078,5604, 716,2464,3079, 174,1255,2710, # 4480
+4122,3645, 548,1320,1398, 728,4123,1574,5605,1891,1197,3080,4124,5606,3081,3082, # 4496
+3778,3646,3779, 747,5607, 635,4386,4747,5608,5609,5610,4387,5611,5612,4748,5613, # 4512
+3415,4749,2437, 451,5614,3780,2542,2073,4388,2744,4389,4125,5615,1764,4750,5616, # 4528
+4390, 350,4751,2283,2395,2493,5617,4391,4126,2249,1434,4127, 488,4752, 458,4392, # 4544
+4128,3781, 771,1330,2396,3914,2576,3184,2160,2414,1553,2677,3185,4393,5618,2494, # 4560
+2895,2622,1720,2711,4394,3416,4753,5619,2543,4395,5620,3262,4396,2778,5621,2016, # 4576
+2745,5622,1155,1017,3782,3915,5623,3336,2313, 201,1865,4397,1430,5624,4129,5625, # 4592
+5626,5627,5628,5629,4398,1604,5630, 414,1866, 371,2595,4754,4755,3532,2017,3127, # 4608
+4756,1708, 960,4399, 887, 389,2172,1536,1663,1721,5631,2232,4130,2356,2940,1580, # 4624
+5632,5633,1744,4757,2544,4758,4759,5634,4760,5635,2074,5636,4761,3647,3417,2896, # 4640
+4400,5637,4401,2650,3418,2815, 673,2712,2465, 709,3533,4131,3648,4402,5638,1148, # 4656
+ 502, 634,5639,5640,1204,4762,3649,1575,4763,2623,3783,5641,3784,3128, 948,3263, # 4672
+ 121,1745,3916,1110,5642,4403,3083,2516,3027,4132,3785,1151,1771,3917,1488,4133, # 4688
+1987,5643,2438,3534,5644,5645,2094,5646,4404,3918,1213,1407,2816, 531,2746,2545, # 4704
+3264,1011,1537,4764,2779,4405,3129,1061,5647,3786,3787,1867,2897,5648,2018, 120, # 4720
+4406,4407,2063,3650,3265,2314,3919,2678,3419,1955,4765,4134,5649,3535,1047,2713, # 4736
+1266,5650,1368,4766,2858, 649,3420,3920,2546,2747,1102,2859,2679,5651,5652,2000, # 4752
+5653,1111,3651,2977,5654,2495,3921,3652,2817,1855,3421,3788,5655,5656,3422,2415, # 4768
+2898,3337,3266,3653,5657,2577,5658,3654,2818,4135,1460, 856,5659,3655,5660,2899, # 4784
+2978,5661,2900,3922,5662,4408, 632,2517, 875,3923,1697,3924,2296,5663,5664,4767, # 4800
+3028,1239, 580,4768,4409,5665, 914, 936,2075,1190,4136,1039,2124,5666,5667,5668, # 4816
+5669,3423,1473,5670,1354,4410,3925,4769,2173,3084,4137, 915,3338,4411,4412,3339, # 4832
+1605,1835,5671,2748, 398,3656,4413,3926,4138, 328,1913,2860,4139,3927,1331,4414, # 4848
+3029, 937,4415,5672,3657,4140,4141,3424,2161,4770,3425, 524, 742, 538,3085,1012, # 4864
+5673,5674,3928,2466,5675, 658,1103, 225,3929,5676,5677,4771,5678,4772,5679,3267, # 4880
+1243,5680,4142, 963,2250,4773,5681,2714,3658,3186,5682,5683,2596,2332,5684,4774, # 4896
+5685,5686,5687,3536, 957,3426,2547,2033,1931,2941,2467, 870,2019,3659,1746,2780, # 4912
+2781,2439,2468,5688,3930,5689,3789,3130,3790,3537,3427,3791,5690,1179,3086,5691, # 4928
+3187,2378,4416,3792,2548,3188,3131,2749,4143,5692,3428,1556,2549,2297, 977,2901, # 4944
+2034,4144,1205,3429,5693,1765,3430,3189,2125,1271, 714,1689,4775,3538,5694,2333, # 4960
+3931, 533,4417,3660,2184, 617,5695,2469,3340,3539,2315,5696,5697,3190,5698,5699, # 4976
+3932,1988, 618, 427,2651,3540,3431,5700,5701,1244,1690,5702,2819,4418,4776,5703, # 4992
+3541,4777,5704,2284,1576, 473,3661,4419,3432, 972,5705,3662,5706,3087,5707,5708, # 5008
+4778,4779,5709,3793,4145,4146,5710, 153,4780, 356,5711,1892,2902,4420,2144, 408, # 5024
+ 803,2357,5712,3933,5713,4421,1646,2578,2518,4781,4782,3934,5714,3935,4422,5715, # 5040
+2416,3433, 752,5716,5717,1962,3341,2979,5718, 746,3030,2470,4783,4423,3794, 698, # 5056
+4784,1893,4424,3663,2550,4785,3664,3936,5719,3191,3434,5720,1824,1302,4147,2715, # 5072
+3937,1974,4425,5721,4426,3192, 823,1303,1288,1236,2861,3542,4148,3435, 774,3938, # 5088
+5722,1581,4786,1304,2862,3939,4787,5723,2440,2162,1083,3268,4427,4149,4428, 344, # 5104
+1173, 288,2316, 454,1683,5724,5725,1461,4788,4150,2597,5726,5727,4789, 985, 894, # 5120
+5728,3436,3193,5729,1914,2942,3795,1989,5730,2111,1975,5731,4151,5732,2579,1194, # 5136
+ 425,5733,4790,3194,1245,3796,4429,5734,5735,2863,5736, 636,4791,1856,3940, 760, # 5152
+1800,5737,4430,2212,1508,4792,4152,1894,1684,2298,5738,5739,4793,4431,4432,2213, # 5168
+ 479,5740,5741, 832,5742,4153,2496,5743,2980,2497,3797, 990,3132, 627,1815,2652, # 5184
+4433,1582,4434,2126,2112,3543,4794,5744, 799,4435,3195,5745,4795,2113,1737,3031, # 5200
+1018, 543, 754,4436,3342,1676,4796,4797,4154,4798,1489,5746,3544,5747,2624,2903, # 5216
+4155,5748,5749,2981,5750,5751,5752,5753,3196,4799,4800,2185,1722,5754,3269,3270, # 5232
+1843,3665,1715, 481, 365,1976,1857,5755,5756,1963,2498,4801,5757,2127,3666,3271, # 5248
+ 433,1895,2064,2076,5758, 602,2750,5759,5760,5761,5762,5763,3032,1628,3437,5764, # 5264
+3197,4802,4156,2904,4803,2519,5765,2551,2782,5766,5767,5768,3343,4804,2905,5769, # 5280
+4805,5770,2864,4806,4807,1221,2982,4157,2520,5771,5772,5773,1868,1990,5774,5775, # 5296
+5776,1896,5777,5778,4808,1897,4158, 318,5779,2095,4159,4437,5780,5781, 485,5782, # 5312
+ 938,3941, 553,2680, 116,5783,3942,3667,5784,3545,2681,2783,3438,3344,2820,5785, # 5328
+3668,2943,4160,1747,2944,2983,5786,5787, 207,5788,4809,5789,4810,2521,5790,3033, # 5344
+ 890,3669,3943,5791,1878,3798,3439,5792,2186,2358,3440,1652,5793,5794,5795, 941, # 5360
+2299, 208,3546,4161,2020, 330,4438,3944,2906,2499,3799,4439,4811,5796,5797,5798, # 5376
+)
+# fmt: on
diff --git a/contrib/python/chardet/py3/chardet/big5prober.py b/contrib/python/chardet/py3/chardet/big5prober.py
new file mode 100644
index 00000000000..ef09c60e327
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/big5prober.py
@@ -0,0 +1,47 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .chardistribution import Big5DistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import BIG5_SM_MODEL
+
+
+class Big5Prober(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(BIG5_SM_MODEL)
+        self.distribution_analyzer = Big5DistributionAnalysis()
+        self.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "Big5"
+
+    @property
+    def language(self) -> str:
+        return "Chinese"
diff --git a/contrib/python/chardet/py3/chardet/chardistribution.py b/contrib/python/chardet/py3/chardet/chardistribution.py
new file mode 100644
index 00000000000..176cb996408
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/chardistribution.py
@@ -0,0 +1,261 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import Tuple, Union
+
+from .big5freq import (
+    BIG5_CHAR_TO_FREQ_ORDER,
+    BIG5_TABLE_SIZE,
+    BIG5_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .euckrfreq import (
+    EUCKR_CHAR_TO_FREQ_ORDER,
+    EUCKR_TABLE_SIZE,
+    EUCKR_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .euctwfreq import (
+    EUCTW_CHAR_TO_FREQ_ORDER,
+    EUCTW_TABLE_SIZE,
+    EUCTW_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .gb2312freq import (
+    GB2312_CHAR_TO_FREQ_ORDER,
+    GB2312_TABLE_SIZE,
+    GB2312_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .jisfreq import (
+    JIS_CHAR_TO_FREQ_ORDER,
+    JIS_TABLE_SIZE,
+    JIS_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .johabfreq import JOHAB_TO_EUCKR_ORDER_TABLE
+
+
+class CharDistributionAnalysis:
+    ENOUGH_DATA_THRESHOLD = 1024
+    SURE_YES = 0.99
+    SURE_NO = 0.01
+    MINIMUM_DATA_THRESHOLD = 3
+
+    def __init__(self) -> None:
+        # Mapping table to get frequency order from char order (get from
+        # GetOrder())
+        self._char_to_freq_order: Tuple[int, ...] = tuple()
+        self._table_size = 0  # Size of above table
+        # This is a constant value which varies from language to language,
+        # used in calculating confidence.  See
+        # http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html
+        # for further detail.
+        self.typical_distribution_ratio = 0.0
+        self._done = False
+        self._total_chars = 0
+        self._freq_chars = 0
+        self.reset()
+
+    def reset(self) -> None:
+        """reset analyser, clear any state"""
+        # If this flag is set to True, detection is done and conclusion has
+        # been made
+        self._done = False
+        self._total_chars = 0  # Total characters encountered
+        # The number of characters whose frequency order is less than 512
+        self._freq_chars = 0
+
+    def feed(self, char: Union[bytes, bytearray], char_len: int) -> None:
+        """feed a character with known length"""
+        if char_len == 2:
+            # we only care about 2-bytes character in our distribution analysis
+            order = self.get_order(char)
+        else:
+            order = -1
+        if order >= 0:
+            self._total_chars += 1
+            # order is valid
+            if order < self._table_size:
+                if 512 > self._char_to_freq_order[order]:
+                    self._freq_chars += 1
+
+    def get_confidence(self) -> float:
+        """return confidence based on existing data"""
+        # if we didn't receive any character in our consideration range,
+        # return negative answer
+        if self._total_chars <= 0 or self._freq_chars <= self.MINIMUM_DATA_THRESHOLD:
+            return self.SURE_NO
+
+        if self._total_chars != self._freq_chars:
+            r = self._freq_chars / (
+                (self._total_chars - self._freq_chars) * self.typical_distribution_ratio
+            )
+            if r < self.SURE_YES:
+                return r
+
+        # normalize confidence (we don't want to be 100% sure)
+        return self.SURE_YES
+
+    def got_enough_data(self) -> bool:
+        # It is not necessary to receive all data to draw conclusion.
+        # For charset detection, certain amount of data is enough
+        return self._total_chars > self.ENOUGH_DATA_THRESHOLD
+
+    def get_order(self, _: Union[bytes, bytearray]) -> int:
+        # We do not handle characters based on the original encoding string,
+        # but convert this encoding string to a number, here called order.
+        # This allows multiple encodings of a language to share one frequency
+        # table.
+        return -1
+
+
+class EUCTWDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._char_to_freq_order = EUCTW_CHAR_TO_FREQ_ORDER
+        self._table_size = EUCTW_TABLE_SIZE
+        self.typical_distribution_ratio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
+        # for euc-TW encoding, we are interested
+        #   first  byte range: 0xc4 -- 0xfe
+        #   second byte range: 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        first_char = byte_str[0]
+        if first_char >= 0xC4:
+            return 94 * (first_char - 0xC4) + byte_str[1] - 0xA1
+        return -1
+
+
+class EUCKRDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._char_to_freq_order = EUCKR_CHAR_TO_FREQ_ORDER
+        self._table_size = EUCKR_TABLE_SIZE
+        self.typical_distribution_ratio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
+        # for euc-KR encoding, we are interested
+        #   first  byte range: 0xb0 -- 0xfe
+        #   second byte range: 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        first_char = byte_str[0]
+        if first_char >= 0xB0:
+            return 94 * (first_char - 0xB0) + byte_str[1] - 0xA1
+        return -1
+
+
+class JOHABDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._char_to_freq_order = EUCKR_CHAR_TO_FREQ_ORDER
+        self._table_size = EUCKR_TABLE_SIZE
+        self.typical_distribution_ratio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
+        first_char = byte_str[0]
+        if 0x88 <= first_char < 0xD4:
+            code = first_char * 256 + byte_str[1]
+            return JOHAB_TO_EUCKR_ORDER_TABLE.get(code, -1)
+        return -1
+
+
+class GB2312DistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._char_to_freq_order = GB2312_CHAR_TO_FREQ_ORDER
+        self._table_size = GB2312_TABLE_SIZE
+        self.typical_distribution_ratio = GB2312_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
+        # for GB2312 encoding, we are interested
+        #  first  byte range: 0xb0 -- 0xfe
+        #  second byte range: 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        first_char, second_char = byte_str[0], byte_str[1]
+        if (first_char >= 0xB0) and (second_char >= 0xA1):
+            return 94 * (first_char - 0xB0) + second_char - 0xA1
+        return -1
+
+
+class Big5DistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._char_to_freq_order = BIG5_CHAR_TO_FREQ_ORDER
+        self._table_size = BIG5_TABLE_SIZE
+        self.typical_distribution_ratio = BIG5_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
+        # for big5 encoding, we are interested
+        #   first  byte range: 0xa4 -- 0xfe
+        #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        first_char, second_char = byte_str[0], byte_str[1]
+        if first_char >= 0xA4:
+            if second_char >= 0xA1:
+                return 157 * (first_char - 0xA4) + second_char - 0xA1 + 63
+            return 157 * (first_char - 0xA4) + second_char - 0x40
+        return -1
+
+
+class SJISDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._char_to_freq_order = JIS_CHAR_TO_FREQ_ORDER
+        self._table_size = JIS_TABLE_SIZE
+        self.typical_distribution_ratio = JIS_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
+        # for sjis encoding, we are interested
+        #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
+        #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
+        # no validation needed here. State machine has done that
+        first_char, second_char = byte_str[0], byte_str[1]
+        if 0x81 <= first_char <= 0x9F:
+            order = 188 * (first_char - 0x81)
+        elif 0xE0 <= first_char <= 0xEF:
+            order = 188 * (first_char - 0xE0 + 31)
+        else:
+            return -1
+        order = order + second_char - 0x40
+        if second_char > 0x7F:
+            order = -1
+        return order
+
+
+class EUCJPDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._char_to_freq_order = JIS_CHAR_TO_FREQ_ORDER
+        self._table_size = JIS_TABLE_SIZE
+        self.typical_distribution_ratio = JIS_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
+        # for euc-JP encoding, we are interested
+        #   first  byte range: 0xa0 -- 0xfe
+        #   second byte range: 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        char = byte_str[0]
+        if char >= 0xA0:
+            return 94 * (char - 0xA1) + byte_str[1] - 0xA1
+        return -1
diff --git a/contrib/python/chardet/py3/chardet/charsetgroupprober.py b/contrib/python/chardet/py3/chardet/charsetgroupprober.py
new file mode 100644
index 00000000000..6def56b4a75
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/charsetgroupprober.py
@@ -0,0 +1,106 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import List, Optional, Union
+
+from .charsetprober import CharSetProber
+from .enums import LanguageFilter, ProbingState
+
+
+class CharSetGroupProber(CharSetProber):
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        super().__init__(lang_filter=lang_filter)
+        self._active_num = 0
+        self.probers: List[CharSetProber] = []
+        self._best_guess_prober: Optional[CharSetProber] = None
+
+    def reset(self) -> None:
+        super().reset()
+        self._active_num = 0
+        for prober in self.probers:
+            prober.reset()
+            prober.active = True
+            self._active_num += 1
+        self._best_guess_prober = None
+
+    @property
+    def charset_name(self) -> Optional[str]:
+        if not self._best_guess_prober:
+            self.get_confidence()
+            if not self._best_guess_prober:
+                return None
+        return self._best_guess_prober.charset_name
+
+    @property
+    def language(self) -> Optional[str]:
+        if not self._best_guess_prober:
+            self.get_confidence()
+            if not self._best_guess_prober:
+                return None
+        return self._best_guess_prober.language
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        for prober in self.probers:
+            if not prober.active:
+                continue
+            state = prober.feed(byte_str)
+            if not state:
+                continue
+            if state == ProbingState.FOUND_IT:
+                self._best_guess_prober = prober
+                self._state = ProbingState.FOUND_IT
+                return self.state
+            if state == ProbingState.NOT_ME:
+                prober.active = False
+                self._active_num -= 1
+                if self._active_num <= 0:
+                    self._state = ProbingState.NOT_ME
+                    return self.state
+        return self.state
+
+    def get_confidence(self) -> float:
+        state = self.state
+        if state == ProbingState.FOUND_IT:
+            return 0.99
+        if state == ProbingState.NOT_ME:
+            return 0.01
+        best_conf = 0.0
+        self._best_guess_prober = None
+        for prober in self.probers:
+            if not prober.active:
+                self.logger.debug("%s not active", prober.charset_name)
+                continue
+            conf = prober.get_confidence()
+            self.logger.debug(
+                "%s %s confidence = %s", prober.charset_name, prober.language, conf
+            )
+            if best_conf < conf:
+                best_conf = conf
+                self._best_guess_prober = prober
+        if not self._best_guess_prober:
+            return 0.0
+        return best_conf
diff --git a/contrib/python/chardet/py3/chardet/charsetprober.py b/contrib/python/chardet/py3/chardet/charsetprober.py
new file mode 100644
index 00000000000..a103ca11356
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/charsetprober.py
@@ -0,0 +1,147 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+import logging
+import re
+from typing import Optional, Union
+
+from .enums import LanguageFilter, ProbingState
+
+INTERNATIONAL_WORDS_PATTERN = re.compile(
+    b"[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?"
+)
+
+
+class CharSetProber:
+
+    SHORTCUT_THRESHOLD = 0.95
+
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        self._state = ProbingState.DETECTING
+        self.active = True
+        self.lang_filter = lang_filter
+        self.logger = logging.getLogger(__name__)
+
+    def reset(self) -> None:
+        self._state = ProbingState.DETECTING
+
+    @property
+    def charset_name(self) -> Optional[str]:
+        return None
+
+    @property
+    def language(self) -> Optional[str]:
+        raise NotImplementedError
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        raise NotImplementedError
+
+    @property
+    def state(self) -> ProbingState:
+        return self._state
+
+    def get_confidence(self) -> float:
+        return 0.0
+
+    @staticmethod
+    def filter_high_byte_only(buf: Union[bytes, bytearray]) -> bytes:
+        buf = re.sub(b"([\x00-\x7F])+", b" ", buf)
+        return buf
+
+    @staticmethod
+    def filter_international_words(buf: Union[bytes, bytearray]) -> bytearray:
+        """
+        We define three types of bytes:
+        alphabet: english alphabets [a-zA-Z]
+        international: international characters [\x80-\xFF]
+        marker: everything else [^a-zA-Z\x80-\xFF]
+        The input buffer can be thought to contain a series of words delimited
+        by markers. This function works to filter all words that contain at
+        least one international character. All contiguous sequences of markers
+        are replaced by a single space ascii character.
+        This filter applies to all scripts which do not use English characters.
+        """
+        filtered = bytearray()
+
+        # This regex expression filters out only words that have at-least one
+        # international character. The word may include one marker character at
+        # the end.
+        words = INTERNATIONAL_WORDS_PATTERN.findall(buf)
+
+        for word in words:
+            filtered.extend(word[:-1])
+
+            # If the last character in the word is a marker, replace it with a
+            # space as markers shouldn't affect our analysis (they are used
+            # similarly across all languages and may thus have similar
+            # frequencies).
+            last_char = word[-1:]
+            if not last_char.isalpha() and last_char < b"\x80":
+                last_char = b" "
+            filtered.extend(last_char)
+
+        return filtered
+
+    @staticmethod
+    def remove_xml_tags(buf: Union[bytes, bytearray]) -> bytes:
+        """
+        Returns a copy of ``buf`` that retains only the sequences of English
+        alphabet and high byte characters that are not between <> characters.
+        This filter can be applied to all scripts which contain both English
+        characters and extended ASCII characters, but is currently only used by
+        ``Latin1Prober``.
+        """
+        filtered = bytearray()
+        in_tag = False
+        prev = 0
+        buf = memoryview(buf).cast("c")
+
+        for curr, buf_char in enumerate(buf):
+            # Check if we're coming out of or entering an XML tag
+
+            # https://github.com/python/typeshed/issues/8182
+            if buf_char == b">":  # type: ignore[comparison-overlap]
+                prev = curr + 1
+                in_tag = False
+            # https://github.com/python/typeshed/issues/8182
+            elif buf_char == b"<":  # type: ignore[comparison-overlap]
+                if curr > prev and not in_tag:
+                    # Keep everything after last non-extended-ASCII,
+                    # non-alphabetic character
+                    filtered.extend(buf[prev:curr])
+                    # Output a space to delimit stretch we kept
+                    filtered.extend(b" ")
+                in_tag = True
+
+        # If we're not in a tag...
+        if not in_tag:
+            # Keep everything after last non-extended-ASCII, non-alphabetic
+            # character
+            filtered.extend(buf[prev:])
+
+        return filtered
diff --git a/contrib/python/chardet/py3/chardet/cli/__init__.py b/contrib/python/chardet/py3/chardet/cli/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/chardet/py3/chardet/cli/chardetect.py b/contrib/python/chardet/py3/chardet/cli/chardetect.py
new file mode 100644
index 00000000000..43f6e144f67
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/cli/chardetect.py
@@ -0,0 +1,112 @@
+"""
+Script which takes one or more file paths and reports on their detected
+encodings
+
+Example::
+
+    % chardetect somefile someotherfile
+    somefile: windows-1252 with confidence 0.5
+    someotherfile: ascii with confidence 1.0
+
+If no paths are provided, it takes its input from stdin.
+
+"""
+
+
+import argparse
+import sys
+from typing import Iterable, List, Optional
+
+from .. import __version__
+from ..universaldetector import UniversalDetector
+
+
+def description_of(
+    lines: Iterable[bytes],
+    name: str = "stdin",
+    minimal: bool = False,
+    should_rename_legacy: bool = False,
+) -> Optional[str]:
+    """
+    Return a string describing the probable encoding of a file or
+    list of strings.
+
+    :param lines: The lines to get the encoding of.
+    :type lines: Iterable of bytes
+    :param name: Name of file or collection of lines
+    :type name: str
+    :param should_rename_legacy:  Should we rename legacy encodings to
+                                  their more modern equivalents?
+    :type should_rename_legacy:   ``bool``
+    """
+    u = UniversalDetector(should_rename_legacy=should_rename_legacy)
+    for line in lines:
+        line = bytearray(line)
+        u.feed(line)
+        # shortcut out of the loop to save reading further - particularly useful if we read a BOM.
+        if u.done:
+            break
+    u.close()
+    result = u.result
+    if minimal:
+        return result["encoding"]
+    if result["encoding"]:
+        return f'{name}: {result["encoding"]} with confidence {result["confidence"]}'
+    return f"{name}: no result"
+
+
+def main(argv: Optional[List[str]] = None) -> None:
+    """
+    Handles command line arguments and gets things started.
+
+    :param argv: List of arguments, as if specified on the command-line.
+                 If None, ``sys.argv[1:]`` is used instead.
+    :type argv: list of str
+    """
+    # Get command line arguments
+    parser = argparse.ArgumentParser(
+        description=(
+            "Takes one or more file paths and reports their detected encodings"
+        )
+    )
+    parser.add_argument(
+        "input",
+        help="File whose encoding we would like to determine. (default: stdin)",
+        type=argparse.FileType("rb"),
+        nargs="*",
+        default=[sys.stdin.buffer],
+    )
+    parser.add_argument(
+        "--minimal",
+        help="Print only the encoding to standard output",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-l",
+        "--legacy",
+        help="Rename legacy encodings to more modern ones.",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
+    args = parser.parse_args(argv)
+
+    for f in args.input:
+        if f.isatty():
+            print(
+                "You are running chardetect interactively. Press "
+                "CTRL-D twice at the start of a blank line to signal the "
+                "end of your input. If you want help, run chardetect "
+                "--help\n",
+                file=sys.stderr,
+            )
+        print(
+            description_of(
+                f, f.name, minimal=args.minimal, should_rename_legacy=args.legacy
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/contrib/python/chardet/py3/chardet/codingstatemachine.py b/contrib/python/chardet/py3/chardet/codingstatemachine.py
new file mode 100644
index 00000000000..8ed4a8773b8
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/codingstatemachine.py
@@ -0,0 +1,90 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+import logging
+
+from .codingstatemachinedict import CodingStateMachineDict
+from .enums import MachineState
+
+
+class CodingStateMachine:
+    """
+    A state machine to verify a byte sequence for a particular encoding. For
+    each byte the detector receives, it will feed that byte to every active
+    state machine available, one byte at a time. The state machine changes its
+    state based on its previous state and the byte it receives. There are 3
+    states in a state machine that are of interest to an auto-detector:
+
+    START state: This is the state to start with, or a legal byte sequence
+                 (i.e. a valid code point) for character has been identified.
+
+    ME state:  This indicates that the state machine identified a byte sequence
+               that is specific to the charset it is designed for and that
+               there is no other possible encoding which can contain this byte
+               sequence. This will to lead to an immediate positive answer for
+               the detector.
+
+    ERROR state: This indicates the state machine identified an illegal byte
+                 sequence for that encoding. This will lead to an immediate
+                 negative answer for this encoding. Detector will exclude this
+                 encoding from consideration from here on.
+    """
+
+    def __init__(self, sm: CodingStateMachineDict) -> None:
+        self._model = sm
+        self._curr_byte_pos = 0
+        self._curr_char_len = 0
+        self._curr_state = MachineState.START
+        self.active = True
+        self.logger = logging.getLogger(__name__)
+        self.reset()
+
+    def reset(self) -> None:
+        self._curr_state = MachineState.START
+
+    def next_state(self, c: int) -> int:
+        # for each byte we get its class
+        # if it is first byte, we also get byte length
+        byte_class = self._model["class_table"][c]
+        if self._curr_state == MachineState.START:
+            self._curr_byte_pos = 0
+            self._curr_char_len = self._model["char_len_table"][byte_class]
+        # from byte's class and state_table, we get its next state
+        curr_state = self._curr_state * self._model["class_factor"] + byte_class
+        self._curr_state = self._model["state_table"][curr_state]
+        self._curr_byte_pos += 1
+        return self._curr_state
+
+    def get_current_charlen(self) -> int:
+        return self._curr_char_len
+
+    def get_coding_state_machine(self) -> str:
+        return self._model["name"]
+
+    @property
+    def language(self) -> str:
+        return self._model["language"]
diff --git a/contrib/python/chardet/py3/chardet/codingstatemachinedict.py b/contrib/python/chardet/py3/chardet/codingstatemachinedict.py
new file mode 100644
index 00000000000..7a3c4c7e3fe
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/codingstatemachinedict.py
@@ -0,0 +1,19 @@
+from typing import TYPE_CHECKING, Tuple
+
+if TYPE_CHECKING:
+    # TypedDict was introduced in Python 3.8.
+    #
+    # TODO: Remove the else block and TYPE_CHECKING check when dropping support
+    # for Python 3.7.
+    from typing import TypedDict
+
+    class CodingStateMachineDict(TypedDict, total=False):
+        class_table: Tuple[int, ...]
+        class_factor: int
+        state_table: Tuple[int, ...]
+        char_len_table: Tuple[int, ...]
+        name: str
+        language: str  # Optional key
+
+else:
+    CodingStateMachineDict = dict
diff --git a/contrib/python/chardet/py3/chardet/cp949prober.py b/contrib/python/chardet/py3/chardet/cp949prober.py
new file mode 100644
index 00000000000..fa7307ed898
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/cp949prober.py
@@ -0,0 +1,49 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .chardistribution import EUCKRDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import CP949_SM_MODEL
+
+
+class CP949Prober(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(CP949_SM_MODEL)
+        # NOTE: CP949 is a superset of EUC-KR, so the distribution should be
+        #       not different.
+        self.distribution_analyzer = EUCKRDistributionAnalysis()
+        self.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "CP949"
+
+    @property
+    def language(self) -> str:
+        return "Korean"
diff --git a/contrib/python/chardet/py3/chardet/enums.py b/contrib/python/chardet/py3/chardet/enums.py
new file mode 100644
index 00000000000..5e3e1982336
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/enums.py
@@ -0,0 +1,85 @@
+"""
+All of the Enums that are used throughout the chardet package.
+
+:author: Dan Blanchard (dan.blanchard@gmail.com)
+"""
+
+from enum import Enum, Flag
+
+
+class InputState:
+    """
+    This enum represents the different states a universal detector can be in.
+    """
+
+    PURE_ASCII = 0
+    ESC_ASCII = 1
+    HIGH_BYTE = 2
+
+
+class LanguageFilter(Flag):
+    """
+    This enum represents the different language filters we can apply to a
+    ``UniversalDetector``.
+    """
+
+    NONE = 0x00
+    CHINESE_SIMPLIFIED = 0x01
+    CHINESE_TRADITIONAL = 0x02
+    JAPANESE = 0x04
+    KOREAN = 0x08
+    NON_CJK = 0x10
+    ALL = 0x1F
+    CHINESE = CHINESE_SIMPLIFIED | CHINESE_TRADITIONAL
+    CJK = CHINESE | JAPANESE | KOREAN
+
+
+class ProbingState(Enum):
+    """
+    This enum represents the different states a prober can be in.
+    """
+
+    DETECTING = 0
+    FOUND_IT = 1
+    NOT_ME = 2
+
+
+class MachineState:
+    """
+    This enum represents the different states a state machine can be in.
+    """
+
+    START = 0
+    ERROR = 1
+    ITS_ME = 2
+
+
+class SequenceLikelihood:
+    """
+    This enum represents the likelihood of a character following the previous one.
+    """
+
+    NEGATIVE = 0
+    UNLIKELY = 1
+    LIKELY = 2
+    POSITIVE = 3
+
+    @classmethod
+    def get_num_categories(cls) -> int:
+        """:returns: The number of likelihood categories in the enum."""
+        return 4
+
+
+class CharacterCategory:
+    """
+    This enum represents the different categories language models for
+    ``SingleByteCharsetProber`` put characters into.
+
+    Anything less than CONTROL is considered a letter.
+    """
+
+    UNDEFINED = 255
+    LINE_BREAK = 254
+    SYMBOL = 253
+    DIGIT = 252
+    CONTROL = 251
diff --git a/contrib/python/chardet/py3/chardet/escprober.py b/contrib/python/chardet/py3/chardet/escprober.py
new file mode 100644
index 00000000000..fd713830d36
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/escprober.py
@@ -0,0 +1,102 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import Optional, Union
+
+from .charsetprober import CharSetProber
+from .codingstatemachine import CodingStateMachine
+from .enums import LanguageFilter, MachineState, ProbingState
+from .escsm import (
+    HZ_SM_MODEL,
+    ISO2022CN_SM_MODEL,
+    ISO2022JP_SM_MODEL,
+    ISO2022KR_SM_MODEL,
+)
+
+
+class EscCharSetProber(CharSetProber):
+    """
+    This CharSetProber uses a "code scheme" approach for detecting encodings,
+    whereby easily recognizable escape or shift sequences are relied on to
+    identify these encodings.
+    """
+
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        super().__init__(lang_filter=lang_filter)
+        self.coding_sm = []
+        if self.lang_filter & LanguageFilter.CHINESE_SIMPLIFIED:
+            self.coding_sm.append(CodingStateMachine(HZ_SM_MODEL))
+            self.coding_sm.append(CodingStateMachine(ISO2022CN_SM_MODEL))
+        if self.lang_filter & LanguageFilter.JAPANESE:
+            self.coding_sm.append(CodingStateMachine(ISO2022JP_SM_MODEL))
+        if self.lang_filter & LanguageFilter.KOREAN:
+            self.coding_sm.append(CodingStateMachine(ISO2022KR_SM_MODEL))
+        self.active_sm_count = 0
+        self._detected_charset: Optional[str] = None
+        self._detected_language: Optional[str] = None
+        self._state = ProbingState.DETECTING
+        self.reset()
+
+    def reset(self) -> None:
+        super().reset()
+        for coding_sm in self.coding_sm:
+            coding_sm.active = True
+            coding_sm.reset()
+        self.active_sm_count = len(self.coding_sm)
+        self._detected_charset = None
+        self._detected_language = None
+
+    @property
+    def charset_name(self) -> Optional[str]:
+        return self._detected_charset
+
+    @property
+    def language(self) -> Optional[str]:
+        return self._detected_language
+
+    def get_confidence(self) -> float:
+        return 0.99 if self._detected_charset else 0.00
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        for c in byte_str:
+            for coding_sm in self.coding_sm:
+                if not coding_sm.active:
+                    continue
+                coding_state = coding_sm.next_state(c)
+                if coding_state == MachineState.ERROR:
+                    coding_sm.active = False
+                    self.active_sm_count -= 1
+                    if self.active_sm_count <= 0:
+                        self._state = ProbingState.NOT_ME
+                        return self.state
+                elif coding_state == MachineState.ITS_ME:
+                    self._state = ProbingState.FOUND_IT
+                    self._detected_charset = coding_sm.get_coding_state_machine()
+                    self._detected_language = coding_sm.language
+                    return self.state
+
+        return self.state
diff --git a/contrib/python/chardet/py3/chardet/escsm.py b/contrib/python/chardet/py3/chardet/escsm.py
new file mode 100644
index 00000000000..11d4adf771f
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/escsm.py
@@ -0,0 +1,261 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License,  or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not,  write to the Free Software
+# Foundation,  Inc.,  51 Franklin St,  Fifth Floor,  Boston,  MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .codingstatemachinedict import CodingStateMachineDict
+from .enums import MachineState
+
+# fmt: off
+HZ_CLS = (
+    1, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 0, 0, 0, 0, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 1, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 20 - 27
+    0, 0, 0, 0, 0, 0, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 4, 0, 5, 2, 0,  # 78 - 7f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 80 - 87
+    1, 1, 1, 1, 1, 1, 1, 1,  # 88 - 8f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 90 - 97
+    1, 1, 1, 1, 1, 1, 1, 1,  # 98 - 9f
+    1, 1, 1, 1, 1, 1, 1, 1,  # a0 - a7
+    1, 1, 1, 1, 1, 1, 1, 1,  # a8 - af
+    1, 1, 1, 1, 1, 1, 1, 1,  # b0 - b7
+    1, 1, 1, 1, 1, 1, 1, 1,  # b8 - bf
+    1, 1, 1, 1, 1, 1, 1, 1,  # c0 - c7
+    1, 1, 1, 1, 1, 1, 1, 1,  # c8 - cf
+    1, 1, 1, 1, 1, 1, 1, 1,  # d0 - d7
+    1, 1, 1, 1, 1, 1, 1, 1,  # d8 - df
+    1, 1, 1, 1, 1, 1, 1, 1,  # e0 - e7
+    1, 1, 1, 1, 1, 1, 1, 1,  # e8 - ef
+    1, 1, 1, 1, 1, 1, 1, 1,  # f0 - f7
+    1, 1, 1, 1, 1, 1, 1, 1,  # f8 - ff
+)
+
+HZ_ST = (
+MachineState.START, MachineState.ERROR,      3, MachineState.START, MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, # 00-07
+MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 08-0f
+MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.START, MachineState.START,      4, MachineState.ERROR, # 10-17
+     5, MachineState.ERROR,      6, MachineState.ERROR,      5,      5,      4, MachineState.ERROR, # 18-1f
+     4, MachineState.ERROR,      4,      4,      4, MachineState.ERROR,      4, MachineState.ERROR, # 20-27
+     4, MachineState.ITS_ME, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 28-2f
+)
+# fmt: on
+
+HZ_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0)
+
+HZ_SM_MODEL: CodingStateMachineDict = {
+    "class_table": HZ_CLS,
+    "class_factor": 6,
+    "state_table": HZ_ST,
+    "char_len_table": HZ_CHAR_LEN_TABLE,
+    "name": "HZ-GB-2312",
+    "language": "Chinese",
+}
+
+# fmt: off
+ISO2022CN_CLS = (
+    2, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 0, 0, 0, 0, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 1, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 20 - 27
+    0, 3, 0, 0, 0, 0, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 4, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 80 - 87
+    2, 2, 2, 2, 2, 2, 2, 2,  # 88 - 8f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 90 - 97
+    2, 2, 2, 2, 2, 2, 2, 2,  # 98 - 9f
+    2, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    2, 2, 2, 2, 2, 2, 2, 2,  # e0 - e7
+    2, 2, 2, 2, 2, 2, 2, 2,  # e8 - ef
+    2, 2, 2, 2, 2, 2, 2, 2,  # f0 - f7
+    2, 2, 2, 2, 2, 2, 2, 2,  # f8 - ff
+)
+
+ISO2022CN_ST = (
+    MachineState.START,      3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 00-07
+    MachineState.START, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 08-0f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 10-17
+    MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      4, MachineState.ERROR, # 18-1f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 20-27
+        5,      6, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 28-2f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 30-37
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.START, # 38-3f
+)
+# fmt: on
+
+ISO2022CN_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0, 0, 0, 0)
+
+ISO2022CN_SM_MODEL: CodingStateMachineDict = {
+    "class_table": ISO2022CN_CLS,
+    "class_factor": 9,
+    "state_table": ISO2022CN_ST,
+    "char_len_table": ISO2022CN_CHAR_LEN_TABLE,
+    "name": "ISO-2022-CN",
+    "language": "Chinese",
+}
+
+# fmt: off
+ISO2022JP_CLS = (
+    2, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 0, 0, 0, 0, 2, 2,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 1, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 7, 0, 0, 0,  # 20 - 27
+    3, 0, 0, 0, 0, 0, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    6, 0, 4, 0, 8, 0, 0, 0,  # 40 - 47
+    0, 9, 5, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 80 - 87
+    2, 2, 2, 2, 2, 2, 2, 2,  # 88 - 8f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 90 - 97
+    2, 2, 2, 2, 2, 2, 2, 2,  # 98 - 9f
+    2, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    2, 2, 2, 2, 2, 2, 2, 2,  # e0 - e7
+    2, 2, 2, 2, 2, 2, 2, 2,  # e8 - ef
+    2, 2, 2, 2, 2, 2, 2, 2,  # f0 - f7
+    2, 2, 2, 2, 2, 2, 2, 2,  # f8 - ff
+)
+
+ISO2022JP_ST = (
+    MachineState.START,      3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 00-07
+    MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 08-0f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 10-17
+    MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, # 18-1f
+    MachineState.ERROR,      5, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      4, MachineState.ERROR, MachineState.ERROR, # 20-27
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      6, MachineState.ITS_ME, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, # 28-2f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, # 30-37
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 38-3f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.START, MachineState.START, # 40-47
+)
+# fmt: on
+
+ISO2022JP_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+
+ISO2022JP_SM_MODEL: CodingStateMachineDict = {
+    "class_table": ISO2022JP_CLS,
+    "class_factor": 10,
+    "state_table": ISO2022JP_ST,
+    "char_len_table": ISO2022JP_CHAR_LEN_TABLE,
+    "name": "ISO-2022-JP",
+    "language": "Japanese",
+}
+
+# fmt: off
+ISO2022KR_CLS = (
+    2, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 0, 0, 0, 0, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 1, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 3, 0, 0, 0,  # 20 - 27
+    0, 4, 0, 0, 0, 0, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 5, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 80 - 87
+    2, 2, 2, 2, 2, 2, 2, 2,  # 88 - 8f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 90 - 97
+    2, 2, 2, 2, 2, 2, 2, 2,  # 98 - 9f
+    2, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    2, 2, 2, 2, 2, 2, 2, 2,  # e0 - e7
+    2, 2, 2, 2, 2, 2, 2, 2,  # e8 - ef
+    2, 2, 2, 2, 2, 2, 2, 2,  # f0 - f7
+    2, 2, 2, 2, 2, 2, 2, 2,  # f8 - ff
+)
+
+ISO2022KR_ST = (
+    MachineState.START,      3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, # 00-07
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 08-0f
+    MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      4, MachineState.ERROR, MachineState.ERROR, # 10-17
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      5, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 18-1f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 20-27
+)
+# fmt: on
+
+ISO2022KR_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0)
+
+ISO2022KR_SM_MODEL: CodingStateMachineDict = {
+    "class_table": ISO2022KR_CLS,
+    "class_factor": 6,
+    "state_table": ISO2022KR_ST,
+    "char_len_table": ISO2022KR_CHAR_LEN_TABLE,
+    "name": "ISO-2022-KR",
+    "language": "Korean",
+}
diff --git a/contrib/python/chardet/py3/chardet/eucjpprober.py b/contrib/python/chardet/py3/chardet/eucjpprober.py
new file mode 100644
index 00000000000..39487f4098d
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/eucjpprober.py
@@ -0,0 +1,102 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import Union
+
+from .chardistribution import EUCJPDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .enums import MachineState, ProbingState
+from .jpcntx import EUCJPContextAnalysis
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import EUCJP_SM_MODEL
+
+
+class EUCJPProber(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(EUCJP_SM_MODEL)
+        self.distribution_analyzer = EUCJPDistributionAnalysis()
+        self.context_analyzer = EUCJPContextAnalysis()
+        self.reset()
+
+    def reset(self) -> None:
+        super().reset()
+        self.context_analyzer.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "EUC-JP"
+
+    @property
+    def language(self) -> str:
+        return "Japanese"
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        assert self.coding_sm is not None
+        assert self.distribution_analyzer is not None
+
+        for i, byte in enumerate(byte_str):
+            # PY3K: byte_str is a byte array, so byte is an int, not a byte
+            coding_state = self.coding_sm.next_state(byte)
+            if coding_state == MachineState.ERROR:
+                self.logger.debug(
+                    "%s %s prober hit error at byte %s",
+                    self.charset_name,
+                    self.language,
+                    i,
+                )
+                self._state = ProbingState.NOT_ME
+                break
+            if coding_state == MachineState.ITS_ME:
+                self._state = ProbingState.FOUND_IT
+                break
+            if coding_state == MachineState.START:
+                char_len = self.coding_sm.get_current_charlen()
+                if i == 0:
+                    self._last_char[1] = byte
+                    self.context_analyzer.feed(self._last_char, char_len)
+                    self.distribution_analyzer.feed(self._last_char, char_len)
+                else:
+                    self.context_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
+                    self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
+
+        self._last_char[0] = byte_str[-1]
+
+        if self.state == ProbingState.DETECTING:
+            if self.context_analyzer.got_enough_data() and (
+                self.get_confidence() > self.SHORTCUT_THRESHOLD
+            ):
+                self._state = ProbingState.FOUND_IT
+
+        return self.state
+
+    def get_confidence(self) -> float:
+        assert self.distribution_analyzer is not None
+
+        context_conf = self.context_analyzer.get_confidence()
+        distrib_conf = self.distribution_analyzer.get_confidence()
+        return max(context_conf, distrib_conf)
diff --git a/contrib/python/chardet/py3/chardet/euckrfreq.py b/contrib/python/chardet/py3/chardet/euckrfreq.py
new file mode 100644
index 00000000000..7dc3b10387d
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/euckrfreq.py
@@ -0,0 +1,196 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+# Sampling from about 20M text materials include literature and computer technology
+
+# 128  --> 0.79
+# 256  --> 0.92
+# 512  --> 0.986
+# 1024 --> 0.99944
+# 2048 --> 0.99999
+#
+# Idea Distribution Ratio = 0.98653 / (1-0.98653) = 73.24
+# Random Distribution Ration = 512 / (2350-512) = 0.279.
+#
+# Typical Distribution Ratio
+
+EUCKR_TYPICAL_DISTRIBUTION_RATIO = 6.0
+
+EUCKR_TABLE_SIZE = 2352
+
+# Char to FreqOrder table ,
+# fmt: off
+EUCKR_CHAR_TO_FREQ_ORDER = (
+  13, 130, 120,1396, 481,1719,1720, 328, 609, 212,1721, 707, 400, 299,1722,  87,
+1397,1723, 104, 536,1117,1203,1724,1267, 685,1268, 508,1725,1726,1727,1728,1398,
+1399,1729,1730,1731, 141, 621, 326,1057, 368,1732, 267, 488,  20,1733,1269,1734,
+ 945,1400,1735,  47, 904,1270,1736,1737, 773, 248,1738, 409, 313, 786, 429,1739,
+ 116, 987, 813,1401, 683,  75,1204, 145,1740,1741,1742,1743,  16, 847, 667, 622,
+ 708,1744,1745,1746, 966, 787, 304, 129,1747,  60, 820, 123, 676,1748,1749,1750,
+1751, 617,1752, 626,1753,1754,1755,1756, 653,1757,1758,1759,1760,1761,1762, 856,
+ 344,1763,1764,1765,1766,  89, 401, 418, 806, 905, 848,1767,1768,1769, 946,1205,
+ 709,1770,1118,1771, 241,1772,1773,1774,1271,1775, 569,1776, 999,1777,1778,1779,
+1780, 337, 751,1058,  28, 628, 254,1781, 177, 906, 270, 349, 891,1079,1782,  19,
+1783, 379,1784, 315,1785, 629, 754,1402, 559,1786, 636, 203,1206,1787, 710, 567,
+1788, 935, 814,1789,1790,1207, 766, 528,1791,1792,1208,1793,1794,1795,1796,1797,
+1403,1798,1799, 533,1059,1404,1405,1156,1406, 936, 884,1080,1800, 351,1801,1802,
+1803,1804,1805, 801,1806,1807,1808,1119,1809,1157, 714, 474,1407,1810, 298, 899,
+ 885,1811,1120, 802,1158,1812, 892,1813,1814,1408, 659,1815,1816,1121,1817,1818,
+1819,1820,1821,1822, 319,1823, 594, 545,1824, 815, 937,1209,1825,1826, 573,1409,
+1022,1827,1210,1828,1829,1830,1831,1832,1833, 556, 722, 807,1122,1060,1834, 697,
+1835, 900, 557, 715,1836,1410, 540,1411, 752,1159, 294, 597,1211, 976, 803, 770,
+1412,1837,1838,  39, 794,1413, 358,1839, 371, 925,1840, 453, 661, 788, 531, 723,
+ 544,1023,1081, 869,  91,1841, 392, 430, 790, 602,1414, 677,1082, 457,1415,1416,
+1842,1843, 475, 327,1024,1417, 795, 121,1844, 733, 403,1418,1845,1846,1847, 300,
+ 119, 711,1212, 627,1848,1272, 207,1849,1850, 796,1213, 382,1851, 519,1852,1083,
+ 893,1853,1854,1855, 367, 809, 487, 671,1856, 663,1857,1858, 956, 471, 306, 857,
+1859,1860,1160,1084,1861,1862,1863,1864,1865,1061,1866,1867,1868,1869,1870,1871,
+ 282,  96, 574,1872, 502,1085,1873,1214,1874, 907,1875,1876, 827, 977,1419,1420,
+1421, 268,1877,1422,1878,1879,1880, 308,1881,   2, 537,1882,1883,1215,1884,1885,
+ 127, 791,1886,1273,1423,1887,  34, 336, 404, 643,1888, 571, 654, 894, 840,1889,
+   0, 886,1274, 122, 575, 260, 908, 938,1890,1275, 410, 316,1891,1892, 100,1893,
+1894,1123,  48,1161,1124,1025,1895, 633, 901,1276,1896,1897, 115, 816,1898, 317,
+1899, 694,1900, 909, 734,1424, 572, 866,1425, 691,  85, 524,1010, 543, 394, 841,
+1901,1902,1903,1026,1904,1905,1906,1907,1908,1909,  30, 451, 651, 988, 310,1910,
+1911,1426, 810,1216,  93,1912,1913,1277,1217,1914, 858, 759,  45,  58, 181, 610,
+ 269,1915,1916, 131,1062, 551, 443,1000, 821,1427, 957, 895,1086,1917,1918, 375,
+1919, 359,1920, 687,1921, 822,1922, 293,1923,1924,  40, 662, 118, 692,  29, 939,
+ 887, 640, 482, 174,1925,  69,1162, 728,1428, 910,1926,1278,1218,1279, 386, 870,
+ 217, 854,1163, 823,1927,1928,1929,1930, 834,1931,  78,1932, 859,1933,1063,1934,
+1935,1936,1937, 438,1164, 208, 595,1938,1939,1940,1941,1219,1125,1942, 280, 888,
+1429,1430,1220,1431,1943,1944,1945,1946,1947,1280, 150, 510,1432,1948,1949,1950,
+1951,1952,1953,1954,1011,1087,1955,1433,1043,1956, 881,1957, 614, 958,1064,1065,
+1221,1958, 638,1001, 860, 967, 896,1434, 989, 492, 553,1281,1165,1959,1282,1002,
+1283,1222,1960,1961,1962,1963,  36, 383, 228, 753, 247, 454,1964, 876, 678,1965,
+1966,1284, 126, 464, 490, 835, 136, 672, 529, 940,1088,1435, 473,1967,1968, 467,
+  50, 390, 227, 587, 279, 378, 598, 792, 968, 240, 151, 160, 849, 882,1126,1285,
+ 639,1044, 133, 140, 288, 360, 811, 563,1027, 561, 142, 523,1969,1970,1971,   7,
+ 103, 296, 439, 407, 506, 634, 990,1972,1973,1974,1975, 645,1976,1977,1978,1979,
+1980,1981, 236,1982,1436,1983,1984,1089, 192, 828, 618, 518,1166, 333,1127,1985,
+ 818,1223,1986,1987,1988,1989,1990,1991,1992,1993, 342,1128,1286, 746, 842,1994,
+1995, 560, 223,1287,  98,   8, 189, 650, 978,1288,1996,1437,1997,  17, 345, 250,
+ 423, 277, 234, 512, 226,  97, 289,  42, 167,1998, 201,1999,2000, 843, 836, 824,
+ 532, 338, 783,1090, 182, 576, 436,1438,1439, 527, 500,2001, 947, 889,2002,2003,
+2004,2005, 262, 600, 314, 447,2006, 547,2007, 693, 738,1129,2008,  71,1440, 745,
+ 619, 688,2009, 829,2010,2011, 147,2012,  33, 948,2013,2014,  74, 224,2015,  61,
+ 191, 918, 399, 637,2016,1028,1130, 257, 902,2017,2018,2019,2020,2021,2022,2023,
+2024,2025,2026, 837,2027,2028,2029,2030, 179, 874, 591,  52, 724, 246,2031,2032,
+2033,2034,1167, 969,2035,1289, 630, 605, 911,1091,1168,2036,2037,2038,1441, 912,
+2039, 623,2040,2041, 253,1169,1290,2042,1442, 146, 620, 611, 577, 433,2043,1224,
+ 719,1170, 959, 440, 437, 534,  84, 388, 480,1131, 159, 220, 198, 679,2044,1012,
+ 819,1066,1443, 113,1225, 194, 318,1003,1029,2045,2046,2047,2048,1067,2049,2050,
+2051,2052,2053,  59, 913, 112,2054, 632,2055, 455, 144, 739,1291,2056, 273, 681,
+ 499,2057, 448,2058,2059, 760,2060,2061, 970, 384, 169, 245,1132,2062,2063, 414,
+1444,2064,2065,  41, 235,2066, 157, 252, 877, 568, 919, 789, 580,2067, 725,2068,
+2069,1292,2070,2071,1445,2072,1446,2073,2074,  55, 588,  66,1447, 271,1092,2075,
+1226,2076, 960,1013, 372,2077,2078,2079,2080,2081,1293,2082,2083,2084,2085, 850,
+2086,2087,2088,2089,2090, 186,2091,1068, 180,2092,2093,2094, 109,1227, 522, 606,
+2095, 867,1448,1093, 991,1171, 926, 353,1133,2096, 581,2097,2098,2099,1294,1449,
+1450,2100, 596,1172,1014,1228,2101,1451,1295,1173,1229,2102,2103,1296,1134,1452,
+ 949,1135,2104,2105,1094,1453,1454,1455,2106,1095,2107,2108,2109,2110,2111,2112,
+2113,2114,2115,2116,2117, 804,2118,2119,1230,1231, 805,1456, 405,1136,2120,2121,
+2122,2123,2124, 720, 701,1297, 992,1457, 927,1004,2125,2126,2127,2128,2129,2130,
+  22, 417,2131, 303,2132, 385,2133, 971, 520, 513,2134,1174,  73,1096, 231, 274,
+ 962,1458, 673,2135,1459,2136, 152,1137,2137,2138,2139,2140,1005,1138,1460,1139,
+2141,2142,2143,2144,  11, 374, 844,2145, 154,1232,  46,1461,2146, 838, 830, 721,
+1233, 106,2147,  90, 428, 462, 578, 566,1175, 352,2148,2149, 538,1234, 124,1298,
+2150,1462, 761, 565,2151, 686,2152, 649,2153,  72, 173,2154, 460, 415,2155,1463,
+2156,1235, 305,2157,2158,2159,2160,2161,2162, 579,2163,2164,2165,2166,2167, 747,
+2168,2169,2170,2171,1464, 669,2172,2173,2174,2175,2176,1465,2177,  23, 530, 285,
+2178, 335, 729,2179, 397,2180,2181,2182,1030,2183,2184, 698,2185,2186, 325,2187,
+2188, 369,2189, 799,1097,1015, 348,2190,1069, 680,2191, 851,1466,2192,2193,  10,
+2194, 613, 424,2195, 979, 108, 449, 589,  27, 172,  81,1031,  80, 774, 281, 350,
+1032, 525, 301, 582,1176,2196, 674,1045,2197,2198,1467, 730, 762,2199,2200,2201,
+2202,1468,2203, 993,2204,2205, 266,1070, 963,1140,2206,2207,2208, 664,1098, 972,
+2209,2210,2211,1177,1469,1470, 871,2212,2213,2214,2215,2216,1471,2217,2218,2219,
+2220,2221,2222,2223,2224,2225,2226,2227,1472,1236,2228,2229,2230,2231,2232,2233,
+2234,2235,1299,2236,2237, 200,2238, 477, 373,2239,2240, 731, 825, 777,2241,2242,
+2243, 521, 486, 548,2244,2245,2246,1473,1300,  53, 549, 137, 875,  76, 158,2247,
+1301,1474, 469, 396,1016, 278, 712,2248, 321, 442, 503, 767, 744, 941,1237,1178,
+1475,2249,  82, 178,1141,1179, 973,2250,1302,2251, 297,2252,2253, 570,2254,2255,
+2256,  18, 450, 206,2257, 290, 292,1142,2258, 511, 162,  99, 346, 164, 735,2259,
+1476,1477,   4, 554, 343, 798,1099,2260,1100,2261,  43, 171,1303, 139, 215,2262,
+2263, 717, 775,2264,1033, 322, 216,2265, 831,2266, 149,2267,1304,2268,2269, 702,
+1238, 135, 845, 347, 309,2270, 484,2271, 878, 655, 238,1006,1478,2272,  67,2273,
+ 295,2274,2275, 461,2276, 478, 942, 412,2277,1034,2278,2279,2280, 265,2281, 541,
+2282,2283,2284,2285,2286,  70, 852,1071,2287,2288,2289,2290,  21,  56, 509, 117,
+ 432,2291,2292, 331, 980, 552,1101, 148, 284, 105, 393,1180,1239, 755,2293, 187,
+2294,1046,1479,2295, 340,2296,  63,1047, 230,2297,2298,1305, 763,1306, 101, 800,
+ 808, 494,2299,2300,2301, 903,2302,  37,1072,  14,   5,2303,  79, 675,2304, 312,
+2305,2306,2307,2308,2309,1480,   6,1307,2310,2311,2312,   1, 470,  35,  24, 229,
+2313, 695, 210,  86, 778,  15, 784, 592, 779,  32,  77, 855, 964,2314, 259,2315,
+ 501, 380,2316,2317,  83, 981, 153, 689,1308,1481,1482,1483,2318,2319, 716,1484,
+2320,2321,2322,2323,2324,2325,1485,2326,2327, 128,  57,  68, 261,1048, 211, 170,
+1240,  31,2328,  51, 435, 742,2329,2330,2331, 635,2332, 264, 456,2333,2334,2335,
+ 425,2336,1486, 143, 507, 263, 943,2337, 363, 920,1487, 256,1488,1102, 243, 601,
+1489,2338,2339,2340,2341,2342,2343,2344, 861,2345,2346,2347,2348,2349,2350, 395,
+2351,1490,1491,  62, 535, 166, 225,2352,2353, 668, 419,1241, 138, 604, 928,2354,
+1181,2355,1492,1493,2356,2357,2358,1143,2359, 696,2360, 387, 307,1309, 682, 476,
+2361,2362, 332,  12, 222, 156,2363, 232,2364, 641, 276, 656, 517,1494,1495,1035,
+ 416, 736,1496,2365,1017, 586,2366,2367,2368,1497,2369, 242,2370,2371,2372,1498,
+2373, 965, 713,2374,2375,2376,2377, 740, 982,1499, 944,1500,1007,2378,2379,1310,
+1501,2380,2381,2382, 785, 329,2383,2384,1502,2385,2386,2387, 932,2388,1503,2389,
+2390,2391,2392,1242,2393,2394,2395,2396,2397, 994, 950,2398,2399,2400,2401,1504,
+1311,2402,2403,2404,2405,1049, 749,2406,2407, 853, 718,1144,1312,2408,1182,1505,
+2409,2410, 255, 516, 479, 564, 550, 214,1506,1507,1313, 413, 239, 444, 339,1145,
+1036,1508,1509,1314,1037,1510,1315,2411,1511,2412,2413,2414, 176, 703, 497, 624,
+ 593, 921, 302,2415, 341, 165,1103,1512,2416,1513,2417,2418,2419, 376,2420, 700,
+2421,2422,2423, 258, 768,1316,2424,1183,2425, 995, 608,2426,2427,2428,2429, 221,
+2430,2431,2432,2433,2434,2435,2436,2437, 195, 323, 726, 188, 897, 983,1317, 377,
+ 644,1050, 879,2438, 452,2439,2440,2441,2442,2443,2444, 914,2445,2446,2447,2448,
+ 915, 489,2449,1514,1184,2450,2451, 515,  64, 427, 495,2452, 583,2453, 483, 485,
+1038, 562, 213,1515, 748, 666,2454,2455,2456,2457, 334,2458, 780, 996,1008, 705,
+1243,2459,2460,2461,2462,2463, 114,2464, 493,1146, 366, 163,1516, 961,1104,2465,
+ 291,2466,1318,1105,2467,1517, 365,2468, 355, 951,1244,2469,1319,2470, 631,2471,
+2472, 218,1320, 364, 320, 756,1518,1519,1321,1520,1322,2473,2474,2475,2476, 997,
+2477,2478,2479,2480, 665,1185,2481, 916,1521,2482,2483,2484, 584, 684,2485,2486,
+ 797,2487,1051,1186,2488,2489,2490,1522,2491,2492, 370,2493,1039,1187,  65,2494,
+ 434, 205, 463,1188,2495, 125, 812, 391, 402, 826, 699, 286, 398, 155, 781, 771,
+ 585,2496, 590, 505,1073,2497, 599, 244, 219, 917,1018, 952, 646,1523,2498,1323,
+2499,2500,  49, 984, 354, 741,2501, 625,2502,1324,2503,1019, 190, 357, 757, 491,
+  95, 782, 868,2504,2505,2506,2507,2508,2509, 134,1524,1074, 422,1525, 898,2510,
+ 161,2511,2512,2513,2514, 769,2515,1526,2516,2517, 411,1325,2518, 472,1527,2519,
+2520,2521,2522,2523,2524, 985,2525,2526,2527,2528,2529,2530, 764,2531,1245,2532,
+2533,  25, 204, 311,2534, 496,2535,1052,2536,2537,2538,2539,2540,2541,2542, 199,
+ 704, 504, 468, 758, 657,1528, 196,  44, 839,1246, 272, 750,2543, 765, 862,2544,
+2545,1326,2546, 132, 615, 933,2547, 732,2548,2549,2550,1189,1529,2551, 283,1247,
+1053, 607, 929,2552,2553,2554, 930, 183, 872, 616,1040,1147,2555,1148,1020, 441,
+ 249,1075,2556,2557,2558, 466, 743,2559,2560,2561,  92, 514, 426, 420, 526,2562,
+2563,2564,2565,2566,2567,2568, 185,2569,2570,2571,2572, 776,1530, 658,2573, 362,
+2574, 361, 922,1076, 793,2575,2576,2577,2578,2579,2580,1531, 251,2581,2582,2583,
+2584,1532,  54, 612, 237,1327,2585,2586, 275, 408, 647, 111,2587,1533,1106, 465,
+   3, 458,   9,  38,2588, 107, 110, 890, 209,  26, 737, 498,2589,1534,2590, 431,
+ 202,  88,1535, 356, 287,1107, 660,1149,2591, 381,1536, 986,1150, 445,1248,1151,
+ 974,2592,2593, 846,2594, 446, 953, 184,1249,1250, 727,2595, 923, 193, 883,2596,
+2597,2598, 102, 324, 539, 817,2599, 421,1041,2600, 832,2601,  94, 175, 197, 406,
+2602, 459,2603,2604,2605,2606,2607, 330, 555,2608,2609,2610, 706,1108, 389,2611,
+2612,2613,2614, 233,2615, 833, 558, 931, 954,1251,2616,2617,1537, 546,2618,2619,
+1009,2620,2621,2622,1538, 690,1328,2623, 955,2624,1539,2625,2626, 772,2627,2628,
+2629,2630,2631, 924, 648, 863, 603,2632,2633, 934,1540, 864, 865,2634, 642,1042,
+ 670,1190,2635,2636,2637,2638, 168,2639, 652, 873, 542,1054,1541,2640,2641,2642,  # 512, 256
+)
+# fmt: on
diff --git a/contrib/python/chardet/py3/chardet/euckrprober.py b/contrib/python/chardet/py3/chardet/euckrprober.py
new file mode 100644
index 00000000000..1fc5de0462c
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/euckrprober.py
@@ -0,0 +1,47 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .chardistribution import EUCKRDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import EUCKR_SM_MODEL
+
+
+class EUCKRProber(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(EUCKR_SM_MODEL)
+        self.distribution_analyzer = EUCKRDistributionAnalysis()
+        self.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "EUC-KR"
+
+    @property
+    def language(self) -> str:
+        return "Korean"
diff --git a/contrib/python/chardet/py3/chardet/euctwfreq.py b/contrib/python/chardet/py3/chardet/euctwfreq.py
new file mode 100644
index 00000000000..4900ccc160a
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/euctwfreq.py
@@ -0,0 +1,388 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+# EUCTW frequency table
+# Converted from big5 work
+# by Taiwan's Mandarin Promotion Council
+# <http:#www.edu.tw:81/mandr/>
+
+# 128  --> 0.42261
+# 256  --> 0.57851
+# 512  --> 0.74851
+# 1024 --> 0.89384
+# 2048 --> 0.97583
+#
+# Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98
+# Random Distribution Ration = 512/(5401-512)=0.105
+#
+# Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR
+
+EUCTW_TYPICAL_DISTRIBUTION_RATIO = 0.75
+
+# Char to FreqOrder table
+EUCTW_TABLE_SIZE = 5376
+
+# fmt: off
+EUCTW_CHAR_TO_FREQ_ORDER = (
+    1, 1800, 1506, 255, 1431, 198, 9, 82, 6, 7310, 177, 202, 3615, 1256, 2808, 110,  # 2742
+    3735, 33, 3241, 261, 76, 44, 2113, 16, 2931, 2184, 1176, 659, 3868, 26, 3404, 2643,  # 2758
+    1198, 3869, 3313, 4060, 410, 2211, 302, 590, 361, 1963, 8, 204, 58, 4296, 7311, 1931,  # 2774
+    63, 7312, 7313, 317, 1614, 75, 222, 159, 4061, 2412, 1480, 7314, 3500, 3068, 224, 2809,  # 2790
+    3616, 3, 10, 3870, 1471, 29, 2774, 1135, 2852, 1939, 873, 130, 3242, 1123, 312, 7315,  # 2806
+    4297, 2051, 507, 252, 682, 7316, 142, 1914, 124, 206, 2932, 34, 3501, 3173, 64, 604,  # 2822
+    7317, 2494, 1976, 1977, 155, 1990, 645, 641, 1606, 7318, 3405, 337, 72, 406, 7319, 80,  # 2838
+    630, 238, 3174, 1509, 263, 939, 1092, 2644, 756, 1440, 1094, 3406, 449, 69, 2969, 591,  # 2854
+    179, 2095, 471, 115, 2034, 1843, 60, 50, 2970, 134, 806, 1868, 734, 2035, 3407, 180,  # 2870
+    995, 1607, 156, 537, 2893, 688, 7320, 319, 1305, 779, 2144, 514, 2374, 298, 4298, 359,  # 2886
+    2495, 90, 2707, 1338, 663, 11, 906, 1099, 2545, 20, 2436, 182, 532, 1716, 7321, 732,  # 2902
+    1376, 4062, 1311, 1420, 3175, 25, 2312, 1056, 113, 399, 382, 1949, 242, 3408, 2467, 529,  # 2918
+    3243, 475, 1447, 3617, 7322, 117, 21, 656, 810, 1297, 2295, 2329, 3502, 7323, 126, 4063,  # 2934
+    706, 456, 150, 613, 4299, 71, 1118, 2036, 4064, 145, 3069, 85, 835, 486, 2114, 1246,  # 2950
+    1426, 428, 727, 1285, 1015, 800, 106, 623, 303, 1281, 7324, 2127, 2354, 347, 3736, 221,  # 2966
+    3503, 3110, 7325, 1955, 1153, 4065, 83, 296, 1199, 3070, 192, 624, 93, 7326, 822, 1897,  # 2982
+    2810, 3111, 795, 2064, 991, 1554, 1542, 1592, 27, 43, 2853, 859, 139, 1456, 860, 4300,  # 2998
+    437, 712, 3871, 164, 2392, 3112, 695, 211, 3017, 2096, 195, 3872, 1608, 3504, 3505, 3618,  # 3014
+    3873, 234, 811, 2971, 2097, 3874, 2229, 1441, 3506, 1615, 2375, 668, 2076, 1638, 305, 228,  # 3030
+    1664, 4301, 467, 415, 7327, 262, 2098, 1593, 239, 108, 300, 200, 1033, 512, 1247, 2077,  # 3046
+    7328, 7329, 2173, 3176, 3619, 2673, 593, 845, 1062, 3244, 88, 1723, 2037, 3875, 1950, 212,  # 3062
+    266, 152, 149, 468, 1898, 4066, 4302, 77, 187, 7330, 3018, 37, 5, 2972, 7331, 3876,  # 3078
+    7332, 7333, 39, 2517, 4303, 2894, 3177, 2078, 55, 148, 74, 4304, 545, 483, 1474, 1029,  # 3094
+    1665, 217, 1869, 1531, 3113, 1104, 2645, 4067, 24, 172, 3507, 900, 3877, 3508, 3509, 4305,  # 3110
+    32, 1408, 2811, 1312, 329, 487, 2355, 2247, 2708, 784, 2674, 4, 3019, 3314, 1427, 1788,  # 3126
+    188, 109, 499, 7334, 3620, 1717, 1789, 888, 1217, 3020, 4306, 7335, 3510, 7336, 3315, 1520,  # 3142
+    3621, 3878, 196, 1034, 775, 7337, 7338, 929, 1815, 249, 439, 38, 7339, 1063, 7340, 794,  # 3158
+    3879, 1435, 2296, 46, 178, 3245, 2065, 7341, 2376, 7342, 214, 1709, 4307, 804, 35, 707,  # 3174
+    324, 3622, 1601, 2546, 140, 459, 4068, 7343, 7344, 1365, 839, 272, 978, 2257, 2572, 3409,  # 3190
+    2128, 1363, 3623, 1423, 697, 100, 3071, 48, 70, 1231, 495, 3114, 2193, 7345, 1294, 7346,  # 3206
+    2079, 462, 586, 1042, 3246, 853, 256, 988, 185, 2377, 3410, 1698, 434, 1084, 7347, 3411,  # 3222
+    314, 2615, 2775, 4308, 2330, 2331, 569, 2280, 637, 1816, 2518, 757, 1162, 1878, 1616, 3412,  # 3238
+    287, 1577, 2115, 768, 4309, 1671, 2854, 3511, 2519, 1321, 3737, 909, 2413, 7348, 4069, 933,  # 3254
+    3738, 7349, 2052, 2356, 1222, 4310, 765, 2414, 1322, 786, 4311, 7350, 1919, 1462, 1677, 2895,  # 3270
+    1699, 7351, 4312, 1424, 2437, 3115, 3624, 2590, 3316, 1774, 1940, 3413, 3880, 4070, 309, 1369,  # 3286
+    1130, 2812, 364, 2230, 1653, 1299, 3881, 3512, 3882, 3883, 2646, 525, 1085, 3021, 902, 2000,  # 3302
+    1475, 964, 4313, 421, 1844, 1415, 1057, 2281, 940, 1364, 3116, 376, 4314, 4315, 1381, 7,  # 3318
+    2520, 983, 2378, 336, 1710, 2675, 1845, 321, 3414, 559, 1131, 3022, 2742, 1808, 1132, 1313,  # 3334
+    265, 1481, 1857, 7352, 352, 1203, 2813, 3247, 167, 1089, 420, 2814, 776, 792, 1724, 3513,  # 3350
+    4071, 2438, 3248, 7353, 4072, 7354, 446, 229, 333, 2743, 901, 3739, 1200, 1557, 4316, 2647,  # 3366
+    1920, 395, 2744, 2676, 3740, 4073, 1835, 125, 916, 3178, 2616, 4317, 7355, 7356, 3741, 7357,  # 3382
+    7358, 7359, 4318, 3117, 3625, 1133, 2547, 1757, 3415, 1510, 2313, 1409, 3514, 7360, 2145, 438,  # 3398
+    2591, 2896, 2379, 3317, 1068, 958, 3023, 461, 311, 2855, 2677, 4074, 1915, 3179, 4075, 1978,  # 3414
+    383, 750, 2745, 2617, 4076, 274, 539, 385, 1278, 1442, 7361, 1154, 1964, 384, 561, 210,  # 3430
+    98, 1295, 2548, 3515, 7362, 1711, 2415, 1482, 3416, 3884, 2897, 1257, 129, 7363, 3742, 642,  # 3446
+    523, 2776, 2777, 2648, 7364, 141, 2231, 1333, 68, 176, 441, 876, 907, 4077, 603, 2592,  # 3462
+    710, 171, 3417, 404, 549, 18, 3118, 2393, 1410, 3626, 1666, 7365, 3516, 4319, 2898, 4320,  # 3478
+    7366, 2973, 368, 7367, 146, 366, 99, 871, 3627, 1543, 748, 807, 1586, 1185, 22, 2258,  # 3494
+    379, 3743, 3180, 7368, 3181, 505, 1941, 2618, 1991, 1382, 2314, 7369, 380, 2357, 218, 702,  # 3510
+    1817, 1248, 3418, 3024, 3517, 3318, 3249, 7370, 2974, 3628, 930, 3250, 3744, 7371, 59, 7372,  # 3526
+    585, 601, 4078, 497, 3419, 1112, 1314, 4321, 1801, 7373, 1223, 1472, 2174, 7374, 749, 1836,  # 3542
+    690, 1899, 3745, 1772, 3885, 1476, 429, 1043, 1790, 2232, 2116, 917, 4079, 447, 1086, 1629,  # 3558
+    7375, 556, 7376, 7377, 2020, 1654, 844, 1090, 105, 550, 966, 1758, 2815, 1008, 1782, 686,  # 3574
+    1095, 7378, 2282, 793, 1602, 7379, 3518, 2593, 4322, 4080, 2933, 2297, 4323, 3746, 980, 2496,  # 3590
+    544, 353, 527, 4324, 908, 2678, 2899, 7380, 381, 2619, 1942, 1348, 7381, 1341, 1252, 560,  # 3606
+    3072, 7382, 3420, 2856, 7383, 2053, 973, 886, 2080, 143, 4325, 7384, 7385, 157, 3886, 496,  # 3622
+    4081, 57, 840, 540, 2038, 4326, 4327, 3421, 2117, 1445, 970, 2259, 1748, 1965, 2081, 4082,  # 3638
+    3119, 1234, 1775, 3251, 2816, 3629, 773, 1206, 2129, 1066, 2039, 1326, 3887, 1738, 1725, 4083,  # 3654
+    279, 3120, 51, 1544, 2594, 423, 1578, 2130, 2066, 173, 4328, 1879, 7386, 7387, 1583, 264,  # 3670
+    610, 3630, 4329, 2439, 280, 154, 7388, 7389, 7390, 1739, 338, 1282, 3073, 693, 2857, 1411,  # 3686
+    1074, 3747, 2440, 7391, 4330, 7392, 7393, 1240, 952, 2394, 7394, 2900, 1538, 2679, 685, 1483,  # 3702
+    4084, 2468, 1436, 953, 4085, 2054, 4331, 671, 2395, 79, 4086, 2441, 3252, 608, 567, 2680,  # 3718
+    3422, 4087, 4088, 1691, 393, 1261, 1791, 2396, 7395, 4332, 7396, 7397, 7398, 7399, 1383, 1672,  # 3734
+    3748, 3182, 1464, 522, 1119, 661, 1150, 216, 675, 4333, 3888, 1432, 3519, 609, 4334, 2681,  # 3750
+    2397, 7400, 7401, 7402, 4089, 3025, 0, 7403, 2469, 315, 231, 2442, 301, 3319, 4335, 2380,  # 3766
+    7404, 233, 4090, 3631, 1818, 4336, 4337, 7405, 96, 1776, 1315, 2082, 7406, 257, 7407, 1809,  # 3782
+    3632, 2709, 1139, 1819, 4091, 2021, 1124, 2163, 2778, 1777, 2649, 7408, 3074, 363, 1655, 3183,  # 3798
+    7409, 2975, 7410, 7411, 7412, 3889, 1567, 3890, 718, 103, 3184, 849, 1443, 341, 3320, 2934,  # 3814
+    1484, 7413, 1712, 127, 67, 339, 4092, 2398, 679, 1412, 821, 7414, 7415, 834, 738, 351,  # 3830
+    2976, 2146, 846, 235, 1497, 1880, 418, 1992, 3749, 2710, 186, 1100, 2147, 2746, 3520, 1545,  # 3846
+    1355, 2935, 2858, 1377, 583, 3891, 4093, 2573, 2977, 7416, 1298, 3633, 1078, 2549, 3634, 2358,  # 3862
+    78, 3750, 3751, 267, 1289, 2099, 2001, 1594, 4094, 348, 369, 1274, 2194, 2175, 1837, 4338,  # 3878
+    1820, 2817, 3635, 2747, 2283, 2002, 4339, 2936, 2748, 144, 3321, 882, 4340, 3892, 2749, 3423,  # 3894
+    4341, 2901, 7417, 4095, 1726, 320, 7418, 3893, 3026, 788, 2978, 7419, 2818, 1773, 1327, 2859,  # 3910
+    3894, 2819, 7420, 1306, 4342, 2003, 1700, 3752, 3521, 2359, 2650, 787, 2022, 506, 824, 3636,  # 3926
+    534, 323, 4343, 1044, 3322, 2023, 1900, 946, 3424, 7421, 1778, 1500, 1678, 7422, 1881, 4344,  # 3942
+    165, 243, 4345, 3637, 2521, 123, 683, 4096, 764, 4346, 36, 3895, 1792, 589, 2902, 816,  # 3958
+    626, 1667, 3027, 2233, 1639, 1555, 1622, 3753, 3896, 7423, 3897, 2860, 1370, 1228, 1932, 891,  # 3974
+    2083, 2903, 304, 4097, 7424, 292, 2979, 2711, 3522, 691, 2100, 4098, 1115, 4347, 118, 662,  # 3990
+    7425, 611, 1156, 854, 2381, 1316, 2861, 2, 386, 515, 2904, 7426, 7427, 3253, 868, 2234,  # 4006
+    1486, 855, 2651, 785, 2212, 3028, 7428, 1040, 3185, 3523, 7429, 3121, 448, 7430, 1525, 7431,  # 4022
+    2164, 4348, 7432, 3754, 7433, 4099, 2820, 3524, 3122, 503, 818, 3898, 3123, 1568, 814, 676,  # 4038
+    1444, 306, 1749, 7434, 3755, 1416, 1030, 197, 1428, 805, 2821, 1501, 4349, 7435, 7436, 7437,  # 4054
+    1993, 7438, 4350, 7439, 7440, 2195, 13, 2779, 3638, 2980, 3124, 1229, 1916, 7441, 3756, 2131,  # 4070
+    7442, 4100, 4351, 2399, 3525, 7443, 2213, 1511, 1727, 1120, 7444, 7445, 646, 3757, 2443, 307,  # 4086
+    7446, 7447, 1595, 3186, 7448, 7449, 7450, 3639, 1113, 1356, 3899, 1465, 2522, 2523, 7451, 519,  # 4102
+    7452, 128, 2132, 92, 2284, 1979, 7453, 3900, 1512, 342, 3125, 2196, 7454, 2780, 2214, 1980,  # 4118
+    3323, 7455, 290, 1656, 1317, 789, 827, 2360, 7456, 3758, 4352, 562, 581, 3901, 7457, 401,  # 4134
+    4353, 2248, 94, 4354, 1399, 2781, 7458, 1463, 2024, 4355, 3187, 1943, 7459, 828, 1105, 4101,  # 4150
+    1262, 1394, 7460, 4102, 605, 4356, 7461, 1783, 2862, 7462, 2822, 819, 2101, 578, 2197, 2937,  # 4166
+    7463, 1502, 436, 3254, 4103, 3255, 2823, 3902, 2905, 3425, 3426, 7464, 2712, 2315, 7465, 7466,  # 4182
+    2332, 2067, 23, 4357, 193, 826, 3759, 2102, 699, 1630, 4104, 3075, 390, 1793, 1064, 3526,  # 4198
+    7467, 1579, 3076, 3077, 1400, 7468, 4105, 1838, 1640, 2863, 7469, 4358, 4359, 137, 4106, 598,  # 4214
+    3078, 1966, 780, 104, 974, 2938, 7470, 278, 899, 253, 402, 572, 504, 493, 1339, 7471,  # 4230
+    3903, 1275, 4360, 2574, 2550, 7472, 3640, 3029, 3079, 2249, 565, 1334, 2713, 863, 41, 7473,  # 4246
+    7474, 4361, 7475, 1657, 2333, 19, 463, 2750, 4107, 606, 7476, 2981, 3256, 1087, 2084, 1323,  # 4262
+    2652, 2982, 7477, 1631, 1623, 1750, 4108, 2682, 7478, 2864, 791, 2714, 2653, 2334, 232, 2416,  # 4278
+    7479, 2983, 1498, 7480, 2654, 2620, 755, 1366, 3641, 3257, 3126, 2025, 1609, 119, 1917, 3427,  # 4294
+    862, 1026, 4109, 7481, 3904, 3760, 4362, 3905, 4363, 2260, 1951, 2470, 7482, 1125, 817, 4110,  # 4310
+    4111, 3906, 1513, 1766, 2040, 1487, 4112, 3030, 3258, 2824, 3761, 3127, 7483, 7484, 1507, 7485,  # 4326
+    2683, 733, 40, 1632, 1106, 2865, 345, 4113, 841, 2524, 230, 4364, 2984, 1846, 3259, 3428,  # 4342
+    7486, 1263, 986, 3429, 7487, 735, 879, 254, 1137, 857, 622, 1300, 1180, 1388, 1562, 3907,  # 4358
+    3908, 2939, 967, 2751, 2655, 1349, 592, 2133, 1692, 3324, 2985, 1994, 4114, 1679, 3909, 1901,  # 4374
+    2185, 7488, 739, 3642, 2715, 1296, 1290, 7489, 4115, 2198, 2199, 1921, 1563, 2595, 2551, 1870,  # 4390
+    2752, 2986, 7490, 435, 7491, 343, 1108, 596, 17, 1751, 4365, 2235, 3430, 3643, 7492, 4366,  # 4406
+    294, 3527, 2940, 1693, 477, 979, 281, 2041, 3528, 643, 2042, 3644, 2621, 2782, 2261, 1031,  # 4422
+    2335, 2134, 2298, 3529, 4367, 367, 1249, 2552, 7493, 3530, 7494, 4368, 1283, 3325, 2004, 240,  # 4438
+    1762, 3326, 4369, 4370, 836, 1069, 3128, 474, 7495, 2148, 2525, 268, 3531, 7496, 3188, 1521,  # 4454
+    1284, 7497, 1658, 1546, 4116, 7498, 3532, 3533, 7499, 4117, 3327, 2684, 1685, 4118, 961, 1673,  # 4470
+    2622, 190, 2005, 2200, 3762, 4371, 4372, 7500, 570, 2497, 3645, 1490, 7501, 4373, 2623, 3260,  # 4486
+    1956, 4374, 584, 1514, 396, 1045, 1944, 7502, 4375, 1967, 2444, 7503, 7504, 4376, 3910, 619,  # 4502
+    7505, 3129, 3261, 215, 2006, 2783, 2553, 3189, 4377, 3190, 4378, 763, 4119, 3763, 4379, 7506,  # 4518
+    7507, 1957, 1767, 2941, 3328, 3646, 1174, 452, 1477, 4380, 3329, 3130, 7508, 2825, 1253, 2382,  # 4534
+    2186, 1091, 2285, 4120, 492, 7509, 638, 1169, 1824, 2135, 1752, 3911, 648, 926, 1021, 1324,  # 4550
+    4381, 520, 4382, 997, 847, 1007, 892, 4383, 3764, 2262, 1871, 3647, 7510, 2400, 1784, 4384,  # 4566
+    1952, 2942, 3080, 3191, 1728, 4121, 2043, 3648, 4385, 2007, 1701, 3131, 1551, 30, 2263, 4122,  # 4582
+    7511, 2026, 4386, 3534, 7512, 501, 7513, 4123, 594, 3431, 2165, 1821, 3535, 3432, 3536, 3192,  # 4598
+    829, 2826, 4124, 7514, 1680, 3132, 1225, 4125, 7515, 3262, 4387, 4126, 3133, 2336, 7516, 4388,  # 4614
+    4127, 7517, 3912, 3913, 7518, 1847, 2383, 2596, 3330, 7519, 4389, 374, 3914, 652, 4128, 4129,  # 4630
+    375, 1140, 798, 7520, 7521, 7522, 2361, 4390, 2264, 546, 1659, 138, 3031, 2445, 4391, 7523,  # 4646
+    2250, 612, 1848, 910, 796, 3765, 1740, 1371, 825, 3766, 3767, 7524, 2906, 2554, 7525, 692,  # 4662
+    444, 3032, 2624, 801, 4392, 4130, 7526, 1491, 244, 1053, 3033, 4131, 4132, 340, 7527, 3915,  # 4678
+    1041, 2987, 293, 1168, 87, 1357, 7528, 1539, 959, 7529, 2236, 721, 694, 4133, 3768, 219,  # 4694
+    1478, 644, 1417, 3331, 2656, 1413, 1401, 1335, 1389, 3916, 7530, 7531, 2988, 2362, 3134, 1825,  # 4710
+    730, 1515, 184, 2827, 66, 4393, 7532, 1660, 2943, 246, 3332, 378, 1457, 226, 3433, 975,  # 4726
+    3917, 2944, 1264, 3537, 674, 696, 7533, 163, 7534, 1141, 2417, 2166, 713, 3538, 3333, 4394,  # 4742
+    3918, 7535, 7536, 1186, 15, 7537, 1079, 1070, 7538, 1522, 3193, 3539, 276, 1050, 2716, 758,  # 4758
+    1126, 653, 2945, 3263, 7539, 2337, 889, 3540, 3919, 3081, 2989, 903, 1250, 4395, 3920, 3434,  # 4774
+    3541, 1342, 1681, 1718, 766, 3264, 286, 89, 2946, 3649, 7540, 1713, 7541, 2597, 3334, 2990,  # 4790
+    7542, 2947, 2215, 3194, 2866, 7543, 4396, 2498, 2526, 181, 387, 1075, 3921, 731, 2187, 3335,  # 4806
+    7544, 3265, 310, 313, 3435, 2299, 770, 4134, 54, 3034, 189, 4397, 3082, 3769, 3922, 7545,  # 4822
+    1230, 1617, 1849, 355, 3542, 4135, 4398, 3336, 111, 4136, 3650, 1350, 3135, 3436, 3035, 4137,  # 4838
+    2149, 3266, 3543, 7546, 2784, 3923, 3924, 2991, 722, 2008, 7547, 1071, 247, 1207, 2338, 2471,  # 4854
+    1378, 4399, 2009, 864, 1437, 1214, 4400, 373, 3770, 1142, 2216, 667, 4401, 442, 2753, 2555,  # 4870
+    3771, 3925, 1968, 4138, 3267, 1839, 837, 170, 1107, 934, 1336, 1882, 7548, 7549, 2118, 4139,  # 4886
+    2828, 743, 1569, 7550, 4402, 4140, 582, 2384, 1418, 3437, 7551, 1802, 7552, 357, 1395, 1729,  # 4902
+    3651, 3268, 2418, 1564, 2237, 7553, 3083, 3772, 1633, 4403, 1114, 2085, 4141, 1532, 7554, 482,  # 4918
+    2446, 4404, 7555, 7556, 1492, 833, 1466, 7557, 2717, 3544, 1641, 2829, 7558, 1526, 1272, 3652,  # 4934
+    4142, 1686, 1794, 416, 2556, 1902, 1953, 1803, 7559, 3773, 2785, 3774, 1159, 2316, 7560, 2867,  # 4950
+    4405, 1610, 1584, 3036, 2419, 2754, 443, 3269, 1163, 3136, 7561, 7562, 3926, 7563, 4143, 2499,  # 4966
+    3037, 4406, 3927, 3137, 2103, 1647, 3545, 2010, 1872, 4144, 7564, 4145, 431, 3438, 7565, 250,  # 4982
+    97, 81, 4146, 7566, 1648, 1850, 1558, 160, 848, 7567, 866, 740, 1694, 7568, 2201, 2830,  # 4998
+    3195, 4147, 4407, 3653, 1687, 950, 2472, 426, 469, 3196, 3654, 3655, 3928, 7569, 7570, 1188,  # 5014
+    424, 1995, 861, 3546, 4148, 3775, 2202, 2685, 168, 1235, 3547, 4149, 7571, 2086, 1674, 4408,  # 5030
+    3337, 3270, 220, 2557, 1009, 7572, 3776, 670, 2992, 332, 1208, 717, 7573, 7574, 3548, 2447,  # 5046
+    3929, 3338, 7575, 513, 7576, 1209, 2868, 3339, 3138, 4409, 1080, 7577, 7578, 7579, 7580, 2527,  # 5062
+    3656, 3549, 815, 1587, 3930, 3931, 7581, 3550, 3439, 3777, 1254, 4410, 1328, 3038, 1390, 3932,  # 5078
+    1741, 3933, 3778, 3934, 7582, 236, 3779, 2448, 3271, 7583, 7584, 3657, 3780, 1273, 3781, 4411,  # 5094
+    7585, 308, 7586, 4412, 245, 4413, 1851, 2473, 1307, 2575, 430, 715, 2136, 2449, 7587, 270,  # 5110
+    199, 2869, 3935, 7588, 3551, 2718, 1753, 761, 1754, 725, 1661, 1840, 4414, 3440, 3658, 7589,  # 5126
+    7590, 587, 14, 3272, 227, 2598, 326, 480, 2265, 943, 2755, 3552, 291, 650, 1883, 7591,  # 5142
+    1702, 1226, 102, 1547, 62, 3441, 904, 4415, 3442, 1164, 4150, 7592, 7593, 1224, 1548, 2756,  # 5158
+    391, 498, 1493, 7594, 1386, 1419, 7595, 2055, 1177, 4416, 813, 880, 1081, 2363, 566, 1145,  # 5174
+    4417, 2286, 1001, 1035, 2558, 2599, 2238, 394, 1286, 7596, 7597, 2068, 7598, 86, 1494, 1730,  # 5190
+    3936, 491, 1588, 745, 897, 2948, 843, 3340, 3937, 2757, 2870, 3273, 1768, 998, 2217, 2069,  # 5206
+    397, 1826, 1195, 1969, 3659, 2993, 3341, 284, 7599, 3782, 2500, 2137, 2119, 1903, 7600, 3938,  # 5222
+    2150, 3939, 4151, 1036, 3443, 1904, 114, 2559, 4152, 209, 1527, 7601, 7602, 2949, 2831, 2625,  # 5238
+    2385, 2719, 3139, 812, 2560, 7603, 3274, 7604, 1559, 737, 1884, 3660, 1210, 885, 28, 2686,  # 5254
+    3553, 3783, 7605, 4153, 1004, 1779, 4418, 7606, 346, 1981, 2218, 2687, 4419, 3784, 1742, 797,  # 5270
+    1642, 3940, 1933, 1072, 1384, 2151, 896, 3941, 3275, 3661, 3197, 2871, 3554, 7607, 2561, 1958,  # 5286
+    4420, 2450, 1785, 7608, 7609, 7610, 3942, 4154, 1005, 1308, 3662, 4155, 2720, 4421, 4422, 1528,  # 5302
+    2600, 161, 1178, 4156, 1982, 987, 4423, 1101, 4157, 631, 3943, 1157, 3198, 2420, 1343, 1241,  # 5318
+    1016, 2239, 2562, 372, 877, 2339, 2501, 1160, 555, 1934, 911, 3944, 7611, 466, 1170, 169,  # 5334
+    1051, 2907, 2688, 3663, 2474, 2994, 1182, 2011, 2563, 1251, 2626, 7612, 992, 2340, 3444, 1540,  # 5350
+    2721, 1201, 2070, 2401, 1996, 2475, 7613, 4424, 528, 1922, 2188, 1503, 1873, 1570, 2364, 3342,  # 5366
+    3276, 7614, 557, 1073, 7615, 1827, 3445, 2087, 2266, 3140, 3039, 3084, 767, 3085, 2786, 4425,  # 5382
+    1006, 4158, 4426, 2341, 1267, 2176, 3664, 3199, 778, 3945, 3200, 2722, 1597, 2657, 7616, 4427,  # 5398
+    7617, 3446, 7618, 7619, 7620, 3277, 2689, 1433, 3278, 131, 95, 1504, 3946, 723, 4159, 3141,  # 5414
+    1841, 3555, 2758, 2189, 3947, 2027, 2104, 3665, 7621, 2995, 3948, 1218, 7622, 3343, 3201, 3949,  # 5430
+    4160, 2576, 248, 1634, 3785, 912, 7623, 2832, 3666, 3040, 3786, 654, 53, 7624, 2996, 7625,  # 5446
+    1688, 4428, 777, 3447, 1032, 3950, 1425, 7626, 191, 820, 2120, 2833, 971, 4429, 931, 3202,  # 5462
+    135, 664, 783, 3787, 1997, 772, 2908, 1935, 3951, 3788, 4430, 2909, 3203, 282, 2723, 640,  # 5478
+    1372, 3448, 1127, 922, 325, 3344, 7627, 7628, 711, 2044, 7629, 7630, 3952, 2219, 2787, 1936,  # 5494
+    3953, 3345, 2220, 2251, 3789, 2300, 7631, 4431, 3790, 1258, 3279, 3954, 3204, 2138, 2950, 3955,  # 5510
+    3956, 7632, 2221, 258, 3205, 4432, 101, 1227, 7633, 3280, 1755, 7634, 1391, 3281, 7635, 2910,  # 5526
+    2056, 893, 7636, 7637, 7638, 1402, 4161, 2342, 7639, 7640, 3206, 3556, 7641, 7642, 878, 1325,  # 5542
+    1780, 2788, 4433, 259, 1385, 2577, 744, 1183, 2267, 4434, 7643, 3957, 2502, 7644, 684, 1024,  # 5558
+    4162, 7645, 472, 3557, 3449, 1165, 3282, 3958, 3959, 322, 2152, 881, 455, 1695, 1152, 1340,  # 5574
+    660, 554, 2153, 4435, 1058, 4436, 4163, 830, 1065, 3346, 3960, 4437, 1923, 7646, 1703, 1918,  # 5590
+    7647, 932, 2268, 122, 7648, 4438, 947, 677, 7649, 3791, 2627, 297, 1905, 1924, 2269, 4439,  # 5606
+    2317, 3283, 7650, 7651, 4164, 7652, 4165, 84, 4166, 112, 989, 7653, 547, 1059, 3961, 701,  # 5622
+    3558, 1019, 7654, 4167, 7655, 3450, 942, 639, 457, 2301, 2451, 993, 2951, 407, 851, 494,  # 5638
+    4440, 3347, 927, 7656, 1237, 7657, 2421, 3348, 573, 4168, 680, 921, 2911, 1279, 1874, 285,  # 5654
+    790, 1448, 1983, 719, 2167, 7658, 7659, 4441, 3962, 3963, 1649, 7660, 1541, 563, 7661, 1077,  # 5670
+    7662, 3349, 3041, 3451, 511, 2997, 3964, 3965, 3667, 3966, 1268, 2564, 3350, 3207, 4442, 4443,  # 5686
+    7663, 535, 1048, 1276, 1189, 2912, 2028, 3142, 1438, 1373, 2834, 2952, 1134, 2012, 7664, 4169,  # 5702
+    1238, 2578, 3086, 1259, 7665, 700, 7666, 2953, 3143, 3668, 4170, 7667, 4171, 1146, 1875, 1906,  # 5718
+    4444, 2601, 3967, 781, 2422, 132, 1589, 203, 147, 273, 2789, 2402, 898, 1786, 2154, 3968,  # 5734
+    3969, 7668, 3792, 2790, 7669, 7670, 4445, 4446, 7671, 3208, 7672, 1635, 3793, 965, 7673, 1804,  # 5750
+    2690, 1516, 3559, 1121, 1082, 1329, 3284, 3970, 1449, 3794, 65, 1128, 2835, 2913, 2759, 1590,  # 5766
+    3795, 7674, 7675, 12, 2658, 45, 976, 2579, 3144, 4447, 517, 2528, 1013, 1037, 3209, 7676,  # 5782
+    3796, 2836, 7677, 3797, 7678, 3452, 7679, 2602, 614, 1998, 2318, 3798, 3087, 2724, 2628, 7680,  # 5798
+    2580, 4172, 599, 1269, 7681, 1810, 3669, 7682, 2691, 3088, 759, 1060, 489, 1805, 3351, 3285,  # 5814
+    1358, 7683, 7684, 2386, 1387, 1215, 2629, 2252, 490, 7685, 7686, 4173, 1759, 2387, 2343, 7687,  # 5830
+    4448, 3799, 1907, 3971, 2630, 1806, 3210, 4449, 3453, 3286, 2760, 2344, 874, 7688, 7689, 3454,  # 5846
+    3670, 1858, 91, 2914, 3671, 3042, 3800, 4450, 7690, 3145, 3972, 2659, 7691, 3455, 1202, 1403,  # 5862
+    3801, 2954, 2529, 1517, 2503, 4451, 3456, 2504, 7692, 4452, 7693, 2692, 1885, 1495, 1731, 3973,  # 5878
+    2365, 4453, 7694, 2029, 7695, 7696, 3974, 2693, 1216, 237, 2581, 4174, 2319, 3975, 3802, 4454,  # 5894
+    4455, 2694, 3560, 3457, 445, 4456, 7697, 7698, 7699, 7700, 2761, 61, 3976, 3672, 1822, 3977,  # 5910
+    7701, 687, 2045, 935, 925, 405, 2660, 703, 1096, 1859, 2725, 4457, 3978, 1876, 1367, 2695,  # 5926
+    3352, 918, 2105, 1781, 2476, 334, 3287, 1611, 1093, 4458, 564, 3146, 3458, 3673, 3353, 945,  # 5942
+    2631, 2057, 4459, 7702, 1925, 872, 4175, 7703, 3459, 2696, 3089, 349, 4176, 3674, 3979, 4460,  # 5958
+    3803, 4177, 3675, 2155, 3980, 4461, 4462, 4178, 4463, 2403, 2046, 782, 3981, 400, 251, 4179,  # 5974
+    1624, 7704, 7705, 277, 3676, 299, 1265, 476, 1191, 3804, 2121, 4180, 4181, 1109, 205, 7706,  # 5990
+    2582, 1000, 2156, 3561, 1860, 7707, 7708, 7709, 4464, 7710, 4465, 2565, 107, 2477, 2157, 3982,  # 6006
+    3460, 3147, 7711, 1533, 541, 1301, 158, 753, 4182, 2872, 3562, 7712, 1696, 370, 1088, 4183,  # 6022
+    4466, 3563, 579, 327, 440, 162, 2240, 269, 1937, 1374, 3461, 968, 3043, 56, 1396, 3090,  # 6038
+    2106, 3288, 3354, 7713, 1926, 2158, 4467, 2998, 7714, 3564, 7715, 7716, 3677, 4468, 2478, 7717,  # 6054
+    2791, 7718, 1650, 4469, 7719, 2603, 7720, 7721, 3983, 2661, 3355, 1149, 3356, 3984, 3805, 3985,  # 6070
+    7722, 1076, 49, 7723, 951, 3211, 3289, 3290, 450, 2837, 920, 7724, 1811, 2792, 2366, 4184,  # 6086
+    1908, 1138, 2367, 3806, 3462, 7725, 3212, 4470, 1909, 1147, 1518, 2423, 4471, 3807, 7726, 4472,  # 6102
+    2388, 2604, 260, 1795, 3213, 7727, 7728, 3808, 3291, 708, 7729, 3565, 1704, 7730, 3566, 1351,  # 6118
+    1618, 3357, 2999, 1886, 944, 4185, 3358, 4186, 3044, 3359, 4187, 7731, 3678, 422, 413, 1714,  # 6134
+    3292, 500, 2058, 2345, 4188, 2479, 7732, 1344, 1910, 954, 7733, 1668, 7734, 7735, 3986, 2404,  # 6150
+    4189, 3567, 3809, 4190, 7736, 2302, 1318, 2505, 3091, 133, 3092, 2873, 4473, 629, 31, 2838,  # 6166
+    2697, 3810, 4474, 850, 949, 4475, 3987, 2955, 1732, 2088, 4191, 1496, 1852, 7737, 3988, 620,  # 6182
+    3214, 981, 1242, 3679, 3360, 1619, 3680, 1643, 3293, 2139, 2452, 1970, 1719, 3463, 2168, 7738,  # 6198
+    3215, 7739, 7740, 3361, 1828, 7741, 1277, 4476, 1565, 2047, 7742, 1636, 3568, 3093, 7743, 869,  # 6214
+    2839, 655, 3811, 3812, 3094, 3989, 3000, 3813, 1310, 3569, 4477, 7744, 7745, 7746, 1733, 558,  # 6230
+    4478, 3681, 335, 1549, 3045, 1756, 4192, 3682, 1945, 3464, 1829, 1291, 1192, 470, 2726, 2107,  # 6246
+    2793, 913, 1054, 3990, 7747, 1027, 7748, 3046, 3991, 4479, 982, 2662, 3362, 3148, 3465, 3216,  # 6262
+    3217, 1946, 2794, 7749, 571, 4480, 7750, 1830, 7751, 3570, 2583, 1523, 2424, 7752, 2089, 984,  # 6278
+    4481, 3683, 1959, 7753, 3684, 852, 923, 2795, 3466, 3685, 969, 1519, 999, 2048, 2320, 1705,  # 6294
+    7754, 3095, 615, 1662, 151, 597, 3992, 2405, 2321, 1049, 275, 4482, 3686, 4193, 568, 3687,  # 6310
+    3571, 2480, 4194, 3688, 7755, 2425, 2270, 409, 3218, 7756, 1566, 2874, 3467, 1002, 769, 2840,  # 6326
+    194, 2090, 3149, 3689, 2222, 3294, 4195, 628, 1505, 7757, 7758, 1763, 2177, 3001, 3993, 521,  # 6342
+    1161, 2584, 1787, 2203, 2406, 4483, 3994, 1625, 4196, 4197, 412, 42, 3096, 464, 7759, 2632,  # 6358
+    4484, 3363, 1760, 1571, 2875, 3468, 2530, 1219, 2204, 3814, 2633, 2140, 2368, 4485, 4486, 3295,  # 6374
+    1651, 3364, 3572, 7760, 7761, 3573, 2481, 3469, 7762, 3690, 7763, 7764, 2271, 2091, 460, 7765,  # 6390
+    4487, 7766, 3002, 962, 588, 3574, 289, 3219, 2634, 1116, 52, 7767, 3047, 1796, 7768, 7769,  # 6406
+    7770, 1467, 7771, 1598, 1143, 3691, 4198, 1984, 1734, 1067, 4488, 1280, 3365, 465, 4489, 1572,  # 6422
+    510, 7772, 1927, 2241, 1812, 1644, 3575, 7773, 4490, 3692, 7774, 7775, 2663, 1573, 1534, 7776,  # 6438
+    7777, 4199, 536, 1807, 1761, 3470, 3815, 3150, 2635, 7778, 7779, 7780, 4491, 3471, 2915, 1911,  # 6454
+    2796, 7781, 3296, 1122, 377, 3220, 7782, 360, 7783, 7784, 4200, 1529, 551, 7785, 2059, 3693,  # 6470
+    1769, 2426, 7786, 2916, 4201, 3297, 3097, 2322, 2108, 2030, 4492, 1404, 136, 1468, 1479, 672,  # 6486
+    1171, 3221, 2303, 271, 3151, 7787, 2762, 7788, 2049, 678, 2727, 865, 1947, 4493, 7789, 2013,  # 6502
+    3995, 2956, 7790, 2728, 2223, 1397, 3048, 3694, 4494, 4495, 1735, 2917, 3366, 3576, 7791, 3816,  # 6518
+    509, 2841, 2453, 2876, 3817, 7792, 7793, 3152, 3153, 4496, 4202, 2531, 4497, 2304, 1166, 1010,  # 6534
+    552, 681, 1887, 7794, 7795, 2957, 2958, 3996, 1287, 1596, 1861, 3154, 358, 453, 736, 175,  # 6550
+    478, 1117, 905, 1167, 1097, 7796, 1853, 1530, 7797, 1706, 7798, 2178, 3472, 2287, 3695, 3473,  # 6566
+    3577, 4203, 2092, 4204, 7799, 3367, 1193, 2482, 4205, 1458, 2190, 2205, 1862, 1888, 1421, 3298,  # 6582
+    2918, 3049, 2179, 3474, 595, 2122, 7800, 3997, 7801, 7802, 4206, 1707, 2636, 223, 3696, 1359,  # 6598
+    751, 3098, 183, 3475, 7803, 2797, 3003, 419, 2369, 633, 704, 3818, 2389, 241, 7804, 7805,  # 6614
+    7806, 838, 3004, 3697, 2272, 2763, 2454, 3819, 1938, 2050, 3998, 1309, 3099, 2242, 1181, 7807,  # 6630
+    1136, 2206, 3820, 2370, 1446, 4207, 2305, 4498, 7808, 7809, 4208, 1055, 2605, 484, 3698, 7810,  # 6646
+    3999, 625, 4209, 2273, 3368, 1499, 4210, 4000, 7811, 4001, 4211, 3222, 2274, 2275, 3476, 7812,  # 6662
+    7813, 2764, 808, 2606, 3699, 3369, 4002, 4212, 3100, 2532, 526, 3370, 3821, 4213, 955, 7814,  # 6678
+    1620, 4214, 2637, 2427, 7815, 1429, 3700, 1669, 1831, 994, 928, 7816, 3578, 1260, 7817, 7818,  # 6694
+    7819, 1948, 2288, 741, 2919, 1626, 4215, 2729, 2455, 867, 1184, 362, 3371, 1392, 7820, 7821,  # 6710
+    4003, 4216, 1770, 1736, 3223, 2920, 4499, 4500, 1928, 2698, 1459, 1158, 7822, 3050, 3372, 2877,  # 6726
+    1292, 1929, 2506, 2842, 3701, 1985, 1187, 2071, 2014, 2607, 4217, 7823, 2566, 2507, 2169, 3702,  # 6742
+    2483, 3299, 7824, 3703, 4501, 7825, 7826, 666, 1003, 3005, 1022, 3579, 4218, 7827, 4502, 1813,  # 6758
+    2253, 574, 3822, 1603, 295, 1535, 705, 3823, 4219, 283, 858, 417, 7828, 7829, 3224, 4503,  # 6774
+    4504, 3051, 1220, 1889, 1046, 2276, 2456, 4004, 1393, 1599, 689, 2567, 388, 4220, 7830, 2484,  # 6790
+    802, 7831, 2798, 3824, 2060, 1405, 2254, 7832, 4505, 3825, 2109, 1052, 1345, 3225, 1585, 7833,  # 6806
+    809, 7834, 7835, 7836, 575, 2730, 3477, 956, 1552, 1469, 1144, 2323, 7837, 2324, 1560, 2457,  # 6822
+    3580, 3226, 4005, 616, 2207, 3155, 2180, 2289, 7838, 1832, 7839, 3478, 4506, 7840, 1319, 3704,  # 6838
+    3705, 1211, 3581, 1023, 3227, 1293, 2799, 7841, 7842, 7843, 3826, 607, 2306, 3827, 762, 2878,  # 6854
+    1439, 4221, 1360, 7844, 1485, 3052, 7845, 4507, 1038, 4222, 1450, 2061, 2638, 4223, 1379, 4508,  # 6870
+    2585, 7846, 7847, 4224, 1352, 1414, 2325, 2921, 1172, 7848, 7849, 3828, 3829, 7850, 1797, 1451,  # 6886
+    7851, 7852, 7853, 7854, 2922, 4006, 4007, 2485, 2346, 411, 4008, 4009, 3582, 3300, 3101, 4509,  # 6902
+    1561, 2664, 1452, 4010, 1375, 7855, 7856, 47, 2959, 316, 7857, 1406, 1591, 2923, 3156, 7858,  # 6918
+    1025, 2141, 3102, 3157, 354, 2731, 884, 2224, 4225, 2407, 508, 3706, 726, 3583, 996, 2428,  # 6934
+    3584, 729, 7859, 392, 2191, 1453, 4011, 4510, 3707, 7860, 7861, 2458, 3585, 2608, 1675, 2800,  # 6950
+    919, 2347, 2960, 2348, 1270, 4511, 4012, 73, 7862, 7863, 647, 7864, 3228, 2843, 2255, 1550,  # 6966
+    1346, 3006, 7865, 1332, 883, 3479, 7866, 7867, 7868, 7869, 3301, 2765, 7870, 1212, 831, 1347,  # 6982
+    4226, 4512, 2326, 3830, 1863, 3053, 720, 3831, 4513, 4514, 3832, 7871, 4227, 7872, 7873, 4515,  # 6998
+    7874, 7875, 1798, 4516, 3708, 2609, 4517, 3586, 1645, 2371, 7876, 7877, 2924, 669, 2208, 2665,  # 7014
+    2429, 7878, 2879, 7879, 7880, 1028, 3229, 7881, 4228, 2408, 7882, 2256, 1353, 7883, 7884, 4518,  # 7030
+    3158, 518, 7885, 4013, 7886, 4229, 1960, 7887, 2142, 4230, 7888, 7889, 3007, 2349, 2350, 3833,  # 7046
+    516, 1833, 1454, 4014, 2699, 4231, 4519, 2225, 2610, 1971, 1129, 3587, 7890, 2766, 7891, 2961,  # 7062
+    1422, 577, 1470, 3008, 1524, 3373, 7892, 7893, 432, 4232, 3054, 3480, 7894, 2586, 1455, 2508,  # 7078
+    2226, 1972, 1175, 7895, 1020, 2732, 4015, 3481, 4520, 7896, 2733, 7897, 1743, 1361, 3055, 3482,  # 7094
+    2639, 4016, 4233, 4521, 2290, 895, 924, 4234, 2170, 331, 2243, 3056, 166, 1627, 3057, 1098,  # 7110
+    7898, 1232, 2880, 2227, 3374, 4522, 657, 403, 1196, 2372, 542, 3709, 3375, 1600, 4235, 3483,  # 7126
+    7899, 4523, 2767, 3230, 576, 530, 1362, 7900, 4524, 2533, 2666, 3710, 4017, 7901, 842, 3834,  # 7142
+    7902, 2801, 2031, 1014, 4018, 213, 2700, 3376, 665, 621, 4236, 7903, 3711, 2925, 2430, 7904,  # 7158
+    2431, 3302, 3588, 3377, 7905, 4237, 2534, 4238, 4525, 3589, 1682, 4239, 3484, 1380, 7906, 724,  # 7174
+    2277, 600, 1670, 7907, 1337, 1233, 4526, 3103, 2244, 7908, 1621, 4527, 7909, 651, 4240, 7910,  # 7190
+    1612, 4241, 2611, 7911, 2844, 7912, 2734, 2307, 3058, 7913, 716, 2459, 3059, 174, 1255, 2701,  # 7206
+    4019, 3590, 548, 1320, 1398, 728, 4020, 1574, 7914, 1890, 1197, 3060, 4021, 7915, 3061, 3062,  # 7222
+    3712, 3591, 3713, 747, 7916, 635, 4242, 4528, 7917, 7918, 7919, 4243, 7920, 7921, 4529, 7922,  # 7238
+    3378, 4530, 2432, 451, 7923, 3714, 2535, 2072, 4244, 2735, 4245, 4022, 7924, 1764, 4531, 7925,  # 7254
+    4246, 350, 7926, 2278, 2390, 2486, 7927, 4247, 4023, 2245, 1434, 4024, 488, 4532, 458, 4248,  # 7270
+    4025, 3715, 771, 1330, 2391, 3835, 2568, 3159, 2159, 2409, 1553, 2667, 3160, 4249, 7928, 2487,  # 7286
+    2881, 2612, 1720, 2702, 4250, 3379, 4533, 7929, 2536, 4251, 7930, 3231, 4252, 2768, 7931, 2015,  # 7302
+    2736, 7932, 1155, 1017, 3716, 3836, 7933, 3303, 2308, 201, 1864, 4253, 1430, 7934, 4026, 7935,  # 7318
+    7936, 7937, 7938, 7939, 4254, 1604, 7940, 414, 1865, 371, 2587, 4534, 4535, 3485, 2016, 3104,  # 7334
+    4536, 1708, 960, 4255, 887, 389, 2171, 1536, 1663, 1721, 7941, 2228, 4027, 2351, 2926, 1580,  # 7350
+    7942, 7943, 7944, 1744, 7945, 2537, 4537, 4538, 7946, 4539, 7947, 2073, 7948, 7949, 3592, 3380,  # 7366
+    2882, 4256, 7950, 4257, 2640, 3381, 2802, 673, 2703, 2460, 709, 3486, 4028, 3593, 4258, 7951,  # 7382
+    1148, 502, 634, 7952, 7953, 1204, 4540, 3594, 1575, 4541, 2613, 3717, 7954, 3718, 3105, 948,  # 7398
+    3232, 121, 1745, 3837, 1110, 7955, 4259, 3063, 2509, 3009, 4029, 3719, 1151, 1771, 3838, 1488,  # 7414
+    4030, 1986, 7956, 2433, 3487, 7957, 7958, 2093, 7959, 4260, 3839, 1213, 1407, 2803, 531, 2737,  # 7430
+    2538, 3233, 1011, 1537, 7960, 2769, 4261, 3106, 1061, 7961, 3720, 3721, 1866, 2883, 7962, 2017,  # 7446
+    120, 4262, 4263, 2062, 3595, 3234, 2309, 3840, 2668, 3382, 1954, 4542, 7963, 7964, 3488, 1047,  # 7462
+    2704, 1266, 7965, 1368, 4543, 2845, 649, 3383, 3841, 2539, 2738, 1102, 2846, 2669, 7966, 7967,  # 7478
+    1999, 7968, 1111, 3596, 2962, 7969, 2488, 3842, 3597, 2804, 1854, 3384, 3722, 7970, 7971, 3385,  # 7494
+    2410, 2884, 3304, 3235, 3598, 7972, 2569, 7973, 3599, 2805, 4031, 1460, 856, 7974, 3600, 7975,  # 7510
+    2885, 2963, 7976, 2886, 3843, 7977, 4264, 632, 2510, 875, 3844, 1697, 3845, 2291, 7978, 7979,  # 7526
+    4544, 3010, 1239, 580, 4545, 4265, 7980, 914, 936, 2074, 1190, 4032, 1039, 2123, 7981, 7982,  # 7542
+    7983, 3386, 1473, 7984, 1354, 4266, 3846, 7985, 2172, 3064, 4033, 915, 3305, 4267, 4268, 3306,  # 7558
+    1605, 1834, 7986, 2739, 398, 3601, 4269, 3847, 4034, 328, 1912, 2847, 4035, 3848, 1331, 4270,  # 7574
+    3011, 937, 4271, 7987, 3602, 4036, 4037, 3387, 2160, 4546, 3388, 524, 742, 538, 3065, 1012,  # 7590
+    7988, 7989, 3849, 2461, 7990, 658, 1103, 225, 3850, 7991, 7992, 4547, 7993, 4548, 7994, 3236,  # 7606
+    1243, 7995, 4038, 963, 2246, 4549, 7996, 2705, 3603, 3161, 7997, 7998, 2588, 2327, 7999, 4550,  # 7622
+    8000, 8001, 8002, 3489, 3307, 957, 3389, 2540, 2032, 1930, 2927, 2462, 870, 2018, 3604, 1746,  # 7638
+    2770, 2771, 2434, 2463, 8003, 3851, 8004, 3723, 3107, 3724, 3490, 3390, 3725, 8005, 1179, 3066,  # 7654
+    8006, 3162, 2373, 4272, 3726, 2541, 3163, 3108, 2740, 4039, 8007, 3391, 1556, 2542, 2292, 977,  # 7670
+    2887, 2033, 4040, 1205, 3392, 8008, 1765, 3393, 3164, 2124, 1271, 1689, 714, 4551, 3491, 8009,  # 7686
+    2328, 3852, 533, 4273, 3605, 2181, 617, 8010, 2464, 3308, 3492, 2310, 8011, 8012, 3165, 8013,  # 7702
+    8014, 3853, 1987, 618, 427, 2641, 3493, 3394, 8015, 8016, 1244, 1690, 8017, 2806, 4274, 4552,  # 7718
+    8018, 3494, 8019, 8020, 2279, 1576, 473, 3606, 4275, 3395, 972, 8021, 3607, 8022, 3067, 8023,  # 7734
+    8024, 4553, 4554, 8025, 3727, 4041, 4042, 8026, 153, 4555, 356, 8027, 1891, 2888, 4276, 2143,  # 7750
+    408, 803, 2352, 8028, 3854, 8029, 4277, 1646, 2570, 2511, 4556, 4557, 3855, 8030, 3856, 4278,  # 7766
+    8031, 2411, 3396, 752, 8032, 8033, 1961, 2964, 8034, 746, 3012, 2465, 8035, 4279, 3728, 698,  # 7782
+    4558, 1892, 4280, 3608, 2543, 4559, 3609, 3857, 8036, 3166, 3397, 8037, 1823, 1302, 4043, 2706,  # 7798
+    3858, 1973, 4281, 8038, 4282, 3167, 823, 1303, 1288, 1236, 2848, 3495, 4044, 3398, 774, 3859,  # 7814
+    8039, 1581, 4560, 1304, 2849, 3860, 4561, 8040, 2435, 2161, 1083, 3237, 4283, 4045, 4284, 344,  # 7830
+    1173, 288, 2311, 454, 1683, 8041, 8042, 1461, 4562, 4046, 2589, 8043, 8044, 4563, 985, 894,  # 7846
+    8045, 3399, 3168, 8046, 1913, 2928, 3729, 1988, 8047, 2110, 1974, 8048, 4047, 8049, 2571, 1194,  # 7862
+    425, 8050, 4564, 3169, 1245, 3730, 4285, 8051, 8052, 2850, 8053, 636, 4565, 1855, 3861, 760,  # 7878
+    1799, 8054, 4286, 2209, 1508, 4566, 4048, 1893, 1684, 2293, 8055, 8056, 8057, 4287, 4288, 2210,  # 7894
+    479, 8058, 8059, 832, 8060, 4049, 2489, 8061, 2965, 2490, 3731, 990, 3109, 627, 1814, 2642,  # 7910
+    4289, 1582, 4290, 2125, 2111, 3496, 4567, 8062, 799, 4291, 3170, 8063, 4568, 2112, 1737, 3013,  # 7926
+    1018, 543, 754, 4292, 3309, 1676, 4569, 4570, 4050, 8064, 1489, 8065, 3497, 8066, 2614, 2889,  # 7942
+    4051, 8067, 8068, 2966, 8069, 8070, 8071, 8072, 3171, 4571, 4572, 2182, 1722, 8073, 3238, 3239,  # 7958
+    1842, 3610, 1715, 481, 365, 1975, 1856, 8074, 8075, 1962, 2491, 4573, 8076, 2126, 3611, 3240,  # 7974
+    433, 1894, 2063, 2075, 8077, 602, 2741, 8078, 8079, 8080, 8081, 8082, 3014, 1628, 3400, 8083,  # 7990
+    3172, 4574, 4052, 2890, 4575, 2512, 8084, 2544, 2772, 8085, 8086, 8087, 3310, 4576, 2891, 8088,  # 8006
+    4577, 8089, 2851, 4578, 4579, 1221, 2967, 4053, 2513, 8090, 8091, 8092, 1867, 1989, 8093, 8094,  # 8022
+    8095, 1895, 8096, 8097, 4580, 1896, 4054, 318, 8098, 2094, 4055, 4293, 8099, 8100, 485, 8101,  # 8038
+    938, 3862, 553, 2670, 116, 8102, 3863, 3612, 8103, 3498, 2671, 2773, 3401, 3311, 2807, 8104,  # 8054
+    3613, 2929, 4056, 1747, 2930, 2968, 8105, 8106, 207, 8107, 8108, 2672, 4581, 2514, 8109, 3015,  # 8070
+    890, 3614, 3864, 8110, 1877, 3732, 3402, 8111, 2183, 2353, 3403, 1652, 8112, 8113, 8114, 941,  # 8086
+    2294, 208, 3499, 4057, 2019, 330, 4294, 3865, 2892, 2492, 3733, 4295, 8115, 8116, 8117, 8118,  # 8102
+)
+# fmt: on
diff --git a/contrib/python/chardet/py3/chardet/euctwprober.py b/contrib/python/chardet/py3/chardet/euctwprober.py
new file mode 100644
index 00000000000..a37ab189958
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/euctwprober.py
@@ -0,0 +1,47 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .chardistribution import EUCTWDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import EUCTW_SM_MODEL
+
+
+class EUCTWProber(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(EUCTW_SM_MODEL)
+        self.distribution_analyzer = EUCTWDistributionAnalysis()
+        self.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "EUC-TW"
+
+    @property
+    def language(self) -> str:
+        return "Taiwan"
diff --git a/contrib/python/chardet/py3/chardet/gb2312freq.py b/contrib/python/chardet/py3/chardet/gb2312freq.py
new file mode 100644
index 00000000000..b32bfc74213
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/gb2312freq.py
@@ -0,0 +1,284 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+# GB2312 most frequently used character table
+#
+# Char to FreqOrder table , from hz6763
+
+# 512  --> 0.79  -- 0.79
+# 1024 --> 0.92  -- 0.13
+# 2048 --> 0.98  -- 0.06
+# 6768 --> 1.00  -- 0.02
+#
+# Ideal Distribution Ratio = 0.79135/(1-0.79135) = 3.79
+# Random Distribution Ration = 512 / (3755 - 512) = 0.157
+#
+# Typical Distribution Ratio about 25% of Ideal one, still much higher that RDR
+
+GB2312_TYPICAL_DISTRIBUTION_RATIO = 0.9
+
+GB2312_TABLE_SIZE = 3760
+
+# fmt: off
+GB2312_CHAR_TO_FREQ_ORDER = (
+1671, 749,1443,2364,3924,3807,2330,3921,1704,3463,2691,1511,1515, 572,3191,2205,
+2361, 224,2558, 479,1711, 963,3162, 440,4060,1905,2966,2947,3580,2647,3961,3842,
+2204, 869,4207, 970,2678,5626,2944,2956,1479,4048, 514,3595, 588,1346,2820,3409,
+ 249,4088,1746,1873,2047,1774, 581,1813, 358,1174,3590,1014,1561,4844,2245, 670,
+1636,3112, 889,1286, 953, 556,2327,3060,1290,3141, 613, 185,3477,1367, 850,3820,
+1715,2428,2642,2303,2732,3041,2562,2648,3566,3946,1349, 388,3098,2091,1360,3585,
+ 152,1687,1539, 738,1559,  59,1232,2925,2267,1388,1249,1741,1679,2960, 151,1566,
+1125,1352,4271, 924,4296, 385,3166,4459, 310,1245,2850,  70,3285,2729,3534,3575,
+2398,3298,3466,1960,2265, 217,3647, 864,1909,2084,4401,2773,1010,3269,5152, 853,
+3051,3121,1244,4251,1895, 364,1499,1540,2313,1180,3655,2268, 562, 715,2417,3061,
+ 544, 336,3768,2380,1752,4075, 950, 280,2425,4382, 183,2759,3272, 333,4297,2155,
+1688,2356,1444,1039,4540, 736,1177,3349,2443,2368,2144,2225, 565, 196,1482,3406,
+ 927,1335,4147, 692, 878,1311,1653,3911,3622,1378,4200,1840,2969,3149,2126,1816,
+2534,1546,2393,2760, 737,2494,  13, 447, 245,2747,  38,2765,2129,2589,1079, 606,
+ 360, 471,3755,2890, 404, 848, 699,1785,1236, 370,2221,1023,3746,2074,2026,2023,
+2388,1581,2119, 812,1141,3091,2536,1519, 804,2053, 406,1596,1090, 784, 548,4414,
+1806,2264,2936,1100, 343,4114,5096, 622,3358, 743,3668,1510,1626,5020,3567,2513,
+3195,4115,5627,2489,2991,  24,2065,2697,1087,2719,  48,1634, 315,  68, 985,2052,
+ 198,2239,1347,1107,1439, 597,2366,2172, 871,3307, 919,2487,2790,1867, 236,2570,
+1413,3794, 906,3365,3381,1701,1982,1818,1524,2924,1205, 616,2586,2072,2004, 575,
+ 253,3099,  32,1365,1182, 197,1714,2454,1201, 554,3388,3224,2748, 756,2587, 250,
+2567,1507,1517,3529,1922,2761,2337,3416,1961,1677,2452,2238,3153, 615, 911,1506,
+1474,2495,1265,1906,2749,3756,3280,2161, 898,2714,1759,3450,2243,2444, 563,  26,
+3286,2266,3769,3344,2707,3677, 611,1402, 531,1028,2871,4548,1375, 261,2948, 835,
+1190,4134, 353, 840,2684,1900,3082,1435,2109,1207,1674, 329,1872,2781,4055,2686,
+2104, 608,3318,2423,2957,2768,1108,3739,3512,3271,3985,2203,1771,3520,1418,2054,
+1681,1153, 225,1627,2929, 162,2050,2511,3687,1954, 124,1859,2431,1684,3032,2894,
+ 585,4805,3969,2869,2704,2088,2032,2095,3656,2635,4362,2209, 256, 518,2042,2105,
+3777,3657, 643,2298,1148,1779, 190, 989,3544, 414,  11,2135,2063,2979,1471, 403,
+3678, 126, 770,1563, 671,2499,3216,2877, 600,1179, 307,2805,4937,1268,1297,2694,
+ 252,4032,1448,1494,1331,1394, 127,2256, 222,1647,1035,1481,3056,1915,1048, 873,
+3651, 210,  33,1608,2516, 200,1520, 415, 102,   0,3389,1287, 817,  91,3299,2940,
+ 836,1814, 549,2197,1396,1669,2987,3582,2297,2848,4528,1070, 687,  20,1819, 121,
+1552,1364,1461,1968,2617,3540,2824,2083, 177, 948,4938,2291, 110,4549,2066, 648,
+3359,1755,2110,2114,4642,4845,1693,3937,3308,1257,1869,2123, 208,1804,3159,2992,
+2531,2549,3361,2418,1350,2347,2800,2568,1291,2036,2680,  72, 842,1990, 212,1233,
+1154,1586,  75,2027,3410,4900,1823,1337,2710,2676, 728,2810,1522,3026,4995, 157,
+ 755,1050,4022, 710, 785,1936,2194,2085,1406,2777,2400, 150,1250,4049,1206, 807,
+1910, 534, 529,3309,1721,1660, 274,  39,2827, 661,2670,1578, 925,3248,3815,1094,
+4278,4901,4252,  41,1150,3747,2572,2227,4501,3658,4902,3813,3357,3617,2884,2258,
+ 887, 538,4187,3199,1294,2439,3042,2329,2343,2497,1255, 107, 543,1527, 521,3478,
+3568, 194,5062,  15, 961,3870,1241,1192,2664,  66,5215,3260,2111,1295,1127,2152,
+3805,4135, 901,1164,1976, 398,1278, 530,1460, 748, 904,1054,1966,1426,  53,2909,
+ 509, 523,2279,1534, 536,1019, 239,1685, 460,2353, 673,1065,2401,3600,4298,2272,
+1272,2363, 284,1753,3679,4064,1695,  81, 815,2677,2757,2731,1386, 859, 500,4221,
+2190,2566, 757,1006,2519,2068,1166,1455, 337,2654,3203,1863,1682,1914,3025,1252,
+1409,1366, 847, 714,2834,2038,3209, 964,2970,1901, 885,2553,1078,1756,3049, 301,
+1572,3326, 688,2130,1996,2429,1805,1648,2930,3421,2750,3652,3088, 262,1158,1254,
+ 389,1641,1812, 526,1719, 923,2073,1073,1902, 468, 489,4625,1140, 857,2375,3070,
+3319,2863, 380, 116,1328,2693,1161,2244, 273,1212,1884,2769,3011,1775,1142, 461,
+3066,1200,2147,2212, 790, 702,2695,4222,1601,1058, 434,2338,5153,3640,  67,2360,
+4099,2502, 618,3472,1329, 416,1132, 830,2782,1807,2653,3211,3510,1662, 192,2124,
+ 296,3979,1739,1611,3684,  23, 118, 324, 446,1239,1225, 293,2520,3814,3795,2535,
+3116,  17,1074, 467,2692,2201, 387,2922,  45,1326,3055,1645,3659,2817, 958, 243,
+1903,2320,1339,2825,1784,3289, 356, 576, 865,2315,2381,3377,3916,1088,3122,1713,
+1655, 935, 628,4689,1034,1327, 441, 800, 720, 894,1979,2183,1528,5289,2702,1071,
+4046,3572,2399,1571,3281,  79, 761,1103, 327, 134, 758,1899,1371,1615, 879, 442,
+ 215,2605,2579, 173,2048,2485,1057,2975,3317,1097,2253,3801,4263,1403,1650,2946,
+ 814,4968,3487,1548,2644,1567,1285,   2, 295,2636,  97, 946,3576, 832, 141,4257,
+3273, 760,3821,3521,3156,2607, 949,1024,1733,1516,1803,1920,2125,2283,2665,3180,
+1501,2064,3560,2171,1592, 803,3518,1416, 732,3897,4258,1363,1362,2458, 119,1427,
+ 602,1525,2608,1605,1639,3175, 694,3064,  10, 465,  76,2000,4846,4208, 444,3781,
+1619,3353,2206,1273,3796, 740,2483, 320,1723,2377,3660,2619,1359,1137,1762,1724,
+2345,2842,1850,1862, 912, 821,1866, 612,2625,1735,2573,3369,1093, 844,  89, 937,
+ 930,1424,3564,2413,2972,1004,3046,3019,2011, 711,3171,1452,4178, 428, 801,1943,
+ 432, 445,2811, 206,4136,1472, 730, 349,  73, 397,2802,2547, 998,1637,1167, 789,
+ 396,3217, 154,1218, 716,1120,1780,2819,4826,1931,3334,3762,2139,1215,2627, 552,
+3664,3628,3232,1405,2383,3111,1356,2652,3577,3320,3101,1703, 640,1045,1370,1246,
+4996, 371,1575,2436,1621,2210, 984,4033,1734,2638,  16,4529, 663,2755,3255,1451,
+3917,2257,1253,1955,2234,1263,2951, 214,1229, 617, 485, 359,1831,1969, 473,2310,
+ 750,2058, 165,  80,2864,2419, 361,4344,2416,2479,1134, 796,3726,1266,2943, 860,
+2715, 938, 390,2734,1313,1384, 248, 202, 877,1064,2854, 522,3907, 279,1602, 297,
+2357, 395,3740, 137,2075, 944,4089,2584,1267,3802,  62,1533,2285, 178, 176, 780,
+2440, 201,3707, 590, 478,1560,4354,2117,1075,  30,  74,4643,4004,1635,1441,2745,
+ 776,2596, 238,1077,1692,1912,2844, 605, 499,1742,3947, 241,3053, 980,1749, 936,
+2640,4511,2582, 515,1543,2162,5322,2892,2993, 890,2148,1924, 665,1827,3581,1032,
+ 968,3163, 339,1044,1896, 270, 583,1791,1720,4367,1194,3488,3669,  43,2523,1657,
+ 163,2167, 290,1209,1622,3378, 550, 634,2508,2510, 695,2634,2384,2512,1476,1414,
+ 220,1469,2341,2138,2852,3183,2900,4939,2865,3502,1211,3680, 854,3227,1299,2976,
+3172, 186,2998,1459, 443,1067,3251,1495, 321,1932,3054, 909, 753,1410,1828, 436,
+2441,1119,1587,3164,2186,1258, 227, 231,1425,1890,3200,3942, 247, 959, 725,5254,
+2741, 577,2158,2079, 929, 120, 174, 838,2813, 591,1115, 417,2024,  40,3240,1536,
+1037, 291,4151,2354, 632,1298,2406,2500,3535,1825,1846,3451, 205,1171, 345,4238,
+  18,1163, 811, 685,2208,1217, 425,1312,1508,1175,4308,2552,1033, 587,1381,3059,
+2984,3482, 340,1316,4023,3972, 792,3176, 519, 777,4690, 918, 933,4130,2981,3741,
+  90,3360,2911,2200,5184,4550, 609,3079,2030, 272,3379,2736, 363,3881,1130,1447,
+ 286, 779, 357,1169,3350,3137,1630,1220,2687,2391, 747,1277,3688,2618,2682,2601,
+1156,3196,5290,4034,3102,1689,3596,3128, 874, 219,2783, 798, 508,1843,2461, 269,
+1658,1776,1392,1913,2983,3287,2866,2159,2372, 829,4076,  46,4253,2873,1889,1894,
+ 915,1834,1631,2181,2318, 298, 664,2818,3555,2735, 954,3228,3117, 527,3511,2173,
+ 681,2712,3033,2247,2346,3467,1652, 155,2164,3382, 113,1994, 450, 899, 494, 994,
+1237,2958,1875,2336,1926,3727, 545,1577,1550, 633,3473, 204,1305,3072,2410,1956,
+2471, 707,2134, 841,2195,2196,2663,3843,1026,4940, 990,3252,4997, 368,1092, 437,
+3212,3258,1933,1829, 675,2977,2893, 412, 943,3723,4644,3294,3283,2230,2373,5154,
+2389,2241,2661,2323,1404,2524, 593, 787, 677,3008,1275,2059, 438,2709,2609,2240,
+2269,2246,1446,  36,1568,1373,3892,1574,2301,1456,3962, 693,2276,5216,2035,1143,
+2720,1919,1797,1811,2763,4137,2597,1830,1699,1488,1198,2090, 424,1694, 312,3634,
+3390,4179,3335,2252,1214, 561,1059,3243,2295,2561, 975,5155,2321,2751,3772, 472,
+1537,3282,3398,1047,2077,2348,2878,1323,3340,3076, 690,2906,  51, 369, 170,3541,
+1060,2187,2688,3670,2541,1083,1683, 928,3918, 459, 109,4427, 599,3744,4286, 143,
+2101,2730,2490,  82,1588,3036,2121, 281,1860, 477,4035,1238,2812,3020,2716,3312,
+1530,2188,2055,1317, 843, 636,1808,1173,3495, 649, 181,1002, 147,3641,1159,2414,
+3750,2289,2795, 813,3123,2610,1136,4368,   5,3391,4541,2174, 420, 429,1728, 754,
+1228,2115,2219, 347,2223,2733, 735,1518,3003,2355,3134,1764,3948,3329,1888,2424,
+1001,1234,1972,3321,3363,1672,1021,1450,1584, 226, 765, 655,2526,3404,3244,2302,
+3665, 731, 594,2184, 319,1576, 621, 658,2656,4299,2099,3864,1279,2071,2598,2739,
+ 795,3086,3699,3908,1707,2352,2402,1382,3136,2475,1465,4847,3496,3865,1085,3004,
+2591,1084, 213,2287,1963,3565,2250, 822, 793,4574,3187,1772,1789,3050, 595,1484,
+1959,2770,1080,2650, 456, 422,2996, 940,3322,4328,4345,3092,2742, 965,2784, 739,
+4124, 952,1358,2498,2949,2565, 332,2698,2378, 660,2260,2473,4194,3856,2919, 535,
+1260,2651,1208,1428,1300,1949,1303,2942, 433,2455,2450,1251,1946, 614,1269, 641,
+1306,1810,2737,3078,2912, 564,2365,1419,1415,1497,4460,2367,2185,1379,3005,1307,
+3218,2175,1897,3063, 682,1157,4040,4005,1712,1160,1941,1399, 394, 402,2952,1573,
+1151,2986,2404, 862, 299,2033,1489,3006, 346, 171,2886,3401,1726,2932, 168,2533,
+  47,2507,1030,3735,1145,3370,1395,1318,1579,3609,4560,2857,4116,1457,2529,1965,
+ 504,1036,2690,2988,2405, 745,5871, 849,2397,2056,3081, 863,2359,3857,2096,  99,
+1397,1769,2300,4428,1643,3455,1978,1757,3718,1440,  35,4879,3742,1296,4228,2280,
+ 160,5063,1599,2013, 166, 520,3479,1646,3345,3012, 490,1937,1545,1264,2182,2505,
+1096,1188,1369,1436,2421,1667,2792,2460,1270,2122, 727,3167,2143, 806,1706,1012,
+1800,3037, 960,2218,1882, 805, 139,2456,1139,1521, 851,1052,3093,3089, 342,2039,
+ 744,5097,1468,1502,1585,2087, 223, 939, 326,2140,2577, 892,2481,1623,4077, 982,
+3708, 135,2131,  87,2503,3114,2326,1106, 876,1616, 547,2997,2831,2093,3441,4530,
+4314,   9,3256,4229,4148, 659,1462,1986,1710,2046,2913,2231,4090,4880,5255,3392,
+3274,1368,3689,4645,1477, 705,3384,3635,1068,1529,2941,1458,3782,1509, 100,1656,
+2548, 718,2339, 408,1590,2780,3548,1838,4117,3719,1345,3530, 717,3442,2778,3220,
+2898,1892,4590,3614,3371,2043,1998,1224,3483, 891, 635, 584,2559,3355, 733,1766,
+1729,1172,3789,1891,2307, 781,2982,2271,1957,1580,5773,2633,2005,4195,3097,1535,
+3213,1189,1934,5693,3262, 586,3118,1324,1598, 517,1564,2217,1868,1893,4445,3728,
+2703,3139,1526,1787,1992,3882,2875,1549,1199,1056,2224,1904,2711,5098,4287, 338,
+1993,3129,3489,2689,1809,2815,1997, 957,1855,3898,2550,3275,3057,1105,1319, 627,
+1505,1911,1883,3526, 698,3629,3456,1833,1431, 746,  77,1261,2017,2296,1977,1885,
+ 125,1334,1600, 525,1798,1109,2222,1470,1945, 559,2236,1186,3443,2476,1929,1411,
+2411,3135,1777,3372,2621,1841,1613,3229, 668,1430,1839,2643,2916, 195,1989,2671,
+2358,1387, 629,3205,2293,5256,4439, 123,1310, 888,1879,4300,3021,3605,1003,1162,
+3192,2910,2010, 140,2395,2859,  55,1082,2012,2901, 662, 419,2081,1438, 680,2774,
+4654,3912,1620,1731,1625,5035,4065,2328, 512,1344, 802,5443,2163,2311,2537, 524,
+3399,  98,1155,2103,1918,2606,3925,2816,1393,2465,1504,3773,2177,3963,1478,4346,
+ 180,1113,4655,3461,2028,1698, 833,2696,1235,1322,1594,4408,3623,3013,3225,2040,
+3022, 541,2881, 607,3632,2029,1665,1219, 639,1385,1686,1099,2803,3231,1938,3188,
+2858, 427, 676,2772,1168,2025, 454,3253,2486,3556, 230,1950, 580, 791,1991,1280,
+1086,1974,2034, 630, 257,3338,2788,4903,1017,  86,4790, 966,2789,1995,1696,1131,
+ 259,3095,4188,1308, 179,1463,5257, 289,4107,1248,  42,3413,1725,2288, 896,1947,
+ 774,4474,4254, 604,3430,4264, 392,2514,2588, 452, 237,1408,3018, 988,4531,1970,
+3034,3310, 540,2370,1562,1288,2990, 502,4765,1147,   4,1853,2708, 207, 294,2814,
+4078,2902,2509, 684,  34,3105,3532,2551, 644, 709,2801,2344, 573,1727,3573,3557,
+2021,1081,3100,4315,2100,3681, 199,2263,1837,2385, 146,3484,1195,2776,3949, 997,
+1939,3973,1008,1091,1202,1962,1847,1149,4209,5444,1076, 493, 117,5400,2521, 972,
+1490,2934,1796,4542,2374,1512,2933,2657, 413,2888,1135,2762,2314,2156,1355,2369,
+ 766,2007,2527,2170,3124,2491,2593,2632,4757,2437, 234,3125,3591,1898,1750,1376,
+1942,3468,3138, 570,2127,2145,3276,4131, 962, 132,1445,4196,  19, 941,3624,3480,
+3366,1973,1374,4461,3431,2629, 283,2415,2275, 808,2887,3620,2112,2563,1353,3610,
+ 955,1089,3103,1053,  96,  88,4097, 823,3808,1583, 399, 292,4091,3313, 421,1128,
+ 642,4006, 903,2539,1877,2082, 596,  29,4066,1790, 722,2157, 130, 995,1569, 769,
+1485, 464, 513,2213, 288,1923,1101,2453,4316, 133, 486,2445,  50, 625, 487,2207,
+  57, 423, 481,2962, 159,3729,1558, 491, 303, 482, 501, 240,2837, 112,3648,2392,
+1783, 362,   8,3433,3422, 610,2793,3277,1390,1284,1654,  21,3823, 734, 367, 623,
+ 193, 287, 374,1009,1483, 816, 476, 313,2255,2340,1262,2150,2899,1146,2581, 782,
+2116,1659,2018,1880, 255,3586,3314,1110,2867,2137,2564, 986,2767,5185,2006, 650,
+ 158, 926, 762, 881,3157,2717,2362,3587, 306,3690,3245,1542,3077,2427,1691,2478,
+2118,2985,3490,2438, 539,2305, 983, 129,1754, 355,4201,2386, 827,2923, 104,1773,
+2838,2771, 411,2905,3919, 376, 767, 122,1114, 828,2422,1817,3506, 266,3460,1007,
+1609,4998, 945,2612,4429,2274, 726,1247,1964,2914,2199,2070,4002,4108, 657,3323,
+1422, 579, 455,2764,4737,1222,2895,1670, 824,1223,1487,2525, 558, 861,3080, 598,
+2659,2515,1967, 752,2583,2376,2214,4180, 977, 704,2464,4999,2622,4109,1210,2961,
+ 819,1541, 142,2284,  44, 418, 457,1126,3730,4347,4626,1644,1876,3671,1864, 302,
+1063,5694, 624, 723,1984,3745,1314,1676,2488,1610,1449,3558,3569,2166,2098, 409,
+1011,2325,3704,2306, 818,1732,1383,1824,1844,3757, 999,2705,3497,1216,1423,2683,
+2426,2954,2501,2726,2229,1475,2554,5064,1971,1794,1666,2014,1343, 783, 724, 191,
+2434,1354,2220,5065,1763,2752,2472,4152, 131, 175,2885,3434,  92,1466,4920,2616,
+3871,3872,3866, 128,1551,1632, 669,1854,3682,4691,4125,1230, 188,2973,3290,1302,
+1213, 560,3266, 917, 763,3909,3249,1760, 868,1958, 764,1782,2097, 145,2277,3774,
+4462,  64,1491,3062, 971,2132,3606,2442, 221,1226,1617, 218, 323,1185,3207,3147,
+ 571, 619,1473,1005,1744,2281, 449,1887,2396,3685, 275, 375,3816,1743,3844,3731,
+ 845,1983,2350,4210,1377, 773, 967,3499,3052,3743,2725,4007,1697,1022,3943,1464,
+3264,2855,2722,1952,1029,2839,2467,  84,4383,2215, 820,1391,2015,2448,3672, 377,
+1948,2168, 797,2545,3536,2578,2645,  94,2874,1678, 405,1259,3071, 771, 546,1315,
+ 470,1243,3083, 895,2468, 981, 969,2037, 846,4181, 653,1276,2928,  14,2594, 557,
+3007,2474, 156, 902,1338,1740,2574, 537,2518, 973,2282,2216,2433,1928, 138,2903,
+1293,2631,1612, 646,3457, 839,2935, 111, 496,2191,2847, 589,3186, 149,3994,2060,
+4031,2641,4067,3145,1870,  37,3597,2136,1025,2051,3009,3383,3549,1121,1016,3261,
+1301, 251,2446,2599,2153, 872,3246, 637, 334,3705, 831, 884, 921,3065,3140,4092,
+2198,1944, 246,2964, 108,2045,1152,1921,2308,1031, 203,3173,4170,1907,3890, 810,
+1401,2003,1690, 506, 647,1242,2828,1761,1649,3208,2249,1589,3709,2931,5156,1708,
+ 498, 666,2613, 834,3817,1231, 184,2851,1124, 883,3197,2261,3710,1765,1553,2658,
+1178,2639,2351,  93,1193, 942,2538,2141,4402, 235,1821, 870,1591,2192,1709,1871,
+3341,1618,4126,2595,2334, 603, 651,  69, 701, 268,2662,3411,2555,1380,1606, 503,
+ 448, 254,2371,2646, 574,1187,2309,1770, 322,2235,1292,1801, 305, 566,1133, 229,
+2067,2057, 706, 167, 483,2002,2672,3295,1820,3561,3067, 316, 378,2746,3452,1112,
+ 136,1981, 507,1651,2917,1117, 285,4591, 182,2580,3522,1304, 335,3303,1835,2504,
+1795,1792,2248, 674,1018,2106,2449,1857,2292,2845, 976,3047,1781,2600,2727,1389,
+1281,  52,3152, 153, 265,3950, 672,3485,3951,4463, 430,1183, 365, 278,2169,  27,
+1407,1336,2304, 209,1340,1730,2202,1852,2403,2883, 979,1737,1062, 631,2829,2542,
+3876,2592, 825,2086,2226,3048,3625, 352,1417,3724, 542, 991, 431,1351,3938,1861,
+2294, 826,1361,2927,3142,3503,1738, 463,2462,2723, 582,1916,1595,2808, 400,3845,
+3891,2868,3621,2254,  58,2492,1123, 910,2160,2614,1372,1603,1196,1072,3385,1700,
+3267,1980, 696, 480,2430, 920, 799,1570,2920,1951,2041,4047,2540,1321,4223,2469,
+3562,2228,1271,2602, 401,2833,3351,2575,5157, 907,2312,1256, 410, 263,3507,1582,
+ 996, 678,1849,2316,1480, 908,3545,2237, 703,2322, 667,1826,2849,1531,2604,2999,
+2407,3146,2151,2630,1786,3711, 469,3542, 497,3899,2409, 858, 837,4446,3393,1274,
+ 786, 620,1845,2001,3311, 484, 308,3367,1204,1815,3691,2332,1532,2557,1842,2020,
+2724,1927,2333,4440, 567,  22,1673,2728,4475,1987,1858,1144,1597, 101,1832,3601,
+  12, 974,3783,4391, 951,1412,   1,3720, 453,4608,4041, 528,1041,1027,3230,2628,
+1129, 875,1051,3291,1203,2262,1069,2860,2799,2149,2615,3278, 144,1758,3040,  31,
+ 475,1680, 366,2685,3184, 311,1642,4008,2466,5036,1593,1493,2809, 216,1420,1668,
+ 233, 304,2128,3284, 232,1429,1768,1040,2008,3407,2740,2967,2543, 242,2133, 778,
+1565,2022,2620, 505,2189,2756,1098,2273, 372,1614, 708, 553,2846,2094,2278, 169,
+3626,2835,4161, 228,2674,3165, 809,1454,1309, 466,1705,1095, 900,3423, 880,2667,
+3751,5258,2317,3109,2571,4317,2766,1503,1342, 866,4447,1118,  63,2076, 314,1881,
+1348,1061, 172, 978,3515,1747, 532, 511,3970,   6, 601, 905,2699,3300,1751, 276,
+1467,3725,2668,  65,4239,2544,2779,2556,1604, 578,2451,1802, 992,2331,2624,1320,
+3446, 713,1513,1013, 103,2786,2447,1661, 886,1702, 916, 654,3574,2031,1556, 751,
+2178,2821,2179,1498,1538,2176, 271, 914,2251,2080,1325, 638,1953,2937,3877,2432,
+2754,  95,3265,1716, 260,1227,4083, 775, 106,1357,3254, 426,1607, 555,2480, 772,
+1985, 244,2546, 474, 495,1046,2611,1851,2061,  71,2089,1675,2590, 742,3758,2843,
+3222,1433, 267,2180,2576,2826,2233,2092,3913,2435, 956,1745,3075, 856,2113,1116,
+ 451,   3,1988,2896,1398, 993,2463,1878,2049,1341,2718,2721,2870,2108, 712,2904,
+4363,2753,2324, 277,2872,2349,2649, 384, 987, 435, 691,3000, 922, 164,3939, 652,
+1500,1184,4153,2482,3373,2165,4848,2335,3775,3508,3154,2806,2830,1554,2102,1664,
+2530,1434,2408, 893,1547,2623,3447,2832,2242,2532,3169,2856,3223,2078,  49,3770,
+3469, 462, 318, 656,2259,3250,3069, 679,1629,2758, 344,1138,1104,3120,1836,1283,
+3115,2154,1437,4448, 934, 759,1999, 794,2862,1038, 533,2560,1722,2342, 855,2626,
+1197,1663,4476,3127,  85,4240,2528,  25,1111,1181,3673, 407,3470,4561,2679,2713,
+ 768,1925,2841,3986,1544,1165, 932, 373,1240,2146,1930,2673, 721,4766, 354,4333,
+ 391,2963, 187,  61,3364,1442,1102, 330,1940,1767, 341,3809,4118, 393,2496,2062,
+2211, 105, 331, 300, 439, 913,1332, 626, 379,3304,1557, 328, 689,3952, 309,1555,
+ 931, 317,2517,3027, 325, 569, 686,2107,3084,  60,1042,1333,2794, 264,3177,4014,
+1628, 258,3712,   7,4464,1176,1043,1778, 683, 114,1975,  78,1492, 383,1886, 510,
+ 386, 645,5291,2891,2069,3305,4138,3867,2939,2603,2493,1935,1066,1848,3588,1015,
+1282,1289,4609, 697,1453,3044,2666,3611,1856,2412,  54, 719,1330, 568,3778,2459,
+1748, 788, 492, 551,1191,1000, 488,3394,3763, 282,1799, 348,2016,1523,3155,2390,
+1049, 382,2019,1788,1170, 729,2968,3523, 897,3926,2785,2938,3292, 350,2319,3238,
+1718,1717,2655,3453,3143,4465, 161,2889,2980,2009,1421,  56,1908,1640,2387,2232,
+1917,1874,2477,4921, 148,  83,3438, 592,4245,2882,1822,1055, 741, 115,1496,1624,
+ 381,1638,4592,1020, 516,3214, 458, 947,4575,1432, 211,1514,2926,1865,2142, 189,
+ 852,1221,1400,1486, 882,2299,4036, 351,  28,1122, 700,6479,6480,6481,6482,6483,  #last 512
+)
+# fmt: on
diff --git a/contrib/python/chardet/py3/chardet/gb2312prober.py b/contrib/python/chardet/py3/chardet/gb2312prober.py
new file mode 100644
index 00000000000..d423e7311e2
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/gb2312prober.py
@@ -0,0 +1,47 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .chardistribution import GB2312DistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import GB2312_SM_MODEL
+
+
+class GB2312Prober(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(GB2312_SM_MODEL)
+        self.distribution_analyzer = GB2312DistributionAnalysis()
+        self.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "GB2312"
+
+    @property
+    def language(self) -> str:
+        return "Chinese"
diff --git a/contrib/python/chardet/py3/chardet/hebrewprober.py b/contrib/python/chardet/py3/chardet/hebrewprober.py
new file mode 100644
index 00000000000..785d0057bcc
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/hebrewprober.py
@@ -0,0 +1,316 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+#          Shy Shalom
+# Portions created by the Initial Developer are Copyright (C) 2005
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import Optional, Union
+
+from .charsetprober import CharSetProber
+from .enums import ProbingState
+from .sbcharsetprober import SingleByteCharSetProber
+
+# This prober doesn't actually recognize a language or a charset.
+# It is a helper prober for the use of the Hebrew model probers
+
+### General ideas of the Hebrew charset recognition ###
+#
+# Four main charsets exist in Hebrew:
+# "ISO-8859-8" - Visual Hebrew
+# "windows-1255" - Logical Hebrew
+# "ISO-8859-8-I" - Logical Hebrew
+# "x-mac-hebrew" - ?? Logical Hebrew ??
+#
+# Both "ISO" charsets use a completely identical set of code points, whereas
+# "windows-1255" and "x-mac-hebrew" are two different proper supersets of
+# these code points. windows-1255 defines additional characters in the range
+# 0x80-0x9F as some misc punctuation marks as well as some Hebrew-specific
+# diacritics and additional 'Yiddish' ligature letters in the range 0xc0-0xd6.
+# x-mac-hebrew defines similar additional code points but with a different
+# mapping.
+#
+# As far as an average Hebrew text with no diacritics is concerned, all four
+# charsets are identical with respect to code points. Meaning that for the
+# main Hebrew alphabet, all four map the same values to all 27 Hebrew letters
+# (including final letters).
+#
+# The dominant difference between these charsets is their directionality.
+# "Visual" directionality means that the text is ordered as if the renderer is
+# not aware of a BIDI rendering algorithm. The renderer sees the text and
+# draws it from left to right. The text itself when ordered naturally is read
+# backwards. A buffer of Visual Hebrew generally looks like so:
+# "[last word of first line spelled backwards] [whole line ordered backwards
+# and spelled backwards] [first word of first line spelled backwards]
+# [end of line] [last word of second line] ... etc' "
+# adding punctuation marks, numbers and English text to visual text is
+# naturally also "visual" and from left to right.
+#
+# "Logical" directionality means the text is ordered "naturally" according to
+# the order it is read. It is the responsibility of the renderer to display
+# the text from right to left. A BIDI algorithm is used to place general
+# punctuation marks, numbers and English text in the text.
+#
+# Texts in x-mac-hebrew are almost impossible to find on the Internet. From
+# what little evidence I could find, it seems that its general directionality
+# is Logical.
+#
+# To sum up all of the above, the Hebrew probing mechanism knows about two
+# charsets:
+# Visual Hebrew - "ISO-8859-8" - backwards text - Words and sentences are
+#    backwards while line order is natural. For charset recognition purposes
+#    the line order is unimportant (In fact, for this implementation, even
+#    word order is unimportant).
+# Logical Hebrew - "windows-1255" - normal, naturally ordered text.
+#
+# "ISO-8859-8-I" is a subset of windows-1255 and doesn't need to be
+#    specifically identified.
+# "x-mac-hebrew" is also identified as windows-1255. A text in x-mac-hebrew
+#    that contain special punctuation marks or diacritics is displayed with
+#    some unconverted characters showing as question marks. This problem might
+#    be corrected using another model prober for x-mac-hebrew. Due to the fact
+#    that x-mac-hebrew texts are so rare, writing another model prober isn't
+#    worth the effort and performance hit.
+#
+#### The Prober ####
+#
+# The prober is divided between two SBCharSetProbers and a HebrewProber,
+# all of which are managed, created, fed data, inquired and deleted by the
+# SBCSGroupProber. The two SBCharSetProbers identify that the text is in
+# fact some kind of Hebrew, Logical or Visual. The final decision about which
+# one is it is made by the HebrewProber by combining final-letter scores
+# with the scores of the two SBCharSetProbers to produce a final answer.
+#
+# The SBCSGroupProber is responsible for stripping the original text of HTML
+# tags, English characters, numbers, low-ASCII punctuation characters, spaces
+# and new lines. It reduces any sequence of such characters to a single space.
+# The buffer fed to each prober in the SBCS group prober is pure text in
+# high-ASCII.
+# The two SBCharSetProbers (model probers) share the same language model:
+# Win1255Model.
+# The first SBCharSetProber uses the model normally as any other
+# SBCharSetProber does, to recognize windows-1255, upon which this model was
+# built. The second SBCharSetProber is told to make the pair-of-letter
+# lookup in the language model backwards. This in practice exactly simulates
+# a visual Hebrew model using the windows-1255 logical Hebrew model.
+#
+# The HebrewProber is not using any language model. All it does is look for
+# final-letter evidence suggesting the text is either logical Hebrew or visual
+# Hebrew. Disjointed from the model probers, the results of the HebrewProber
+# alone are meaningless. HebrewProber always returns 0.00 as confidence
+# since it never identifies a charset by itself. Instead, the pointer to the
+# HebrewProber is passed to the model probers as a helper "Name Prober".
+# When the Group prober receives a positive identification from any prober,
+# it asks for the name of the charset identified. If the prober queried is a
+# Hebrew model prober, the model prober forwards the call to the
+# HebrewProber to make the final decision. In the HebrewProber, the
+# decision is made according to the final-letters scores maintained and Both
+# model probers scores. The answer is returned in the form of the name of the
+# charset identified, either "windows-1255" or "ISO-8859-8".
+
+
+class HebrewProber(CharSetProber):
+    SPACE = 0x20
+    # windows-1255 / ISO-8859-8 code points of interest
+    FINAL_KAF = 0xEA
+    NORMAL_KAF = 0xEB
+    FINAL_MEM = 0xED
+    NORMAL_MEM = 0xEE
+    FINAL_NUN = 0xEF
+    NORMAL_NUN = 0xF0
+    FINAL_PE = 0xF3
+    NORMAL_PE = 0xF4
+    FINAL_TSADI = 0xF5
+    NORMAL_TSADI = 0xF6
+
+    # Minimum Visual vs Logical final letter score difference.
+    # If the difference is below this, don't rely solely on the final letter score
+    # distance.
+    MIN_FINAL_CHAR_DISTANCE = 5
+
+    # Minimum Visual vs Logical model score difference.
+    # If the difference is below this, don't rely at all on the model score
+    # distance.
+    MIN_MODEL_DISTANCE = 0.01
+
+    VISUAL_HEBREW_NAME = "ISO-8859-8"
+    LOGICAL_HEBREW_NAME = "windows-1255"
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._final_char_logical_score = 0
+        self._final_char_visual_score = 0
+        self._prev = self.SPACE
+        self._before_prev = self.SPACE
+        self._logical_prober: Optional[SingleByteCharSetProber] = None
+        self._visual_prober: Optional[SingleByteCharSetProber] = None
+        self.reset()
+
+    def reset(self) -> None:
+        self._final_char_logical_score = 0
+        self._final_char_visual_score = 0
+        # The two last characters seen in the previous buffer,
+        # mPrev and mBeforePrev are initialized to space in order to simulate
+        # a word delimiter at the beginning of the data
+        self._prev = self.SPACE
+        self._before_prev = self.SPACE
+        # These probers are owned by the group prober.
+
+    def set_model_probers(
+        self,
+        logical_prober: SingleByteCharSetProber,
+        visual_prober: SingleByteCharSetProber,
+    ) -> None:
+        self._logical_prober = logical_prober
+        self._visual_prober = visual_prober
+
+    def is_final(self, c: int) -> bool:
+        return c in [
+            self.FINAL_KAF,
+            self.FINAL_MEM,
+            self.FINAL_NUN,
+            self.FINAL_PE,
+            self.FINAL_TSADI,
+        ]
+
+    def is_non_final(self, c: int) -> bool:
+        # The normal Tsadi is not a good Non-Final letter due to words like
+        # 'lechotet' (to chat) containing an apostrophe after the tsadi. This
+        # apostrophe is converted to a space in FilterWithoutEnglishLetters
+        # causing the Non-Final tsadi to appear at an end of a word even
+        # though this is not the case in the original text.
+        # The letters Pe and Kaf rarely display a related behavior of not being
+        # a good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak'
+        # for example legally end with a Non-Final Pe or Kaf. However, the
+        # benefit of these letters as Non-Final letters outweighs the damage
+        # since these words are quite rare.
+        return c in [self.NORMAL_KAF, self.NORMAL_MEM, self.NORMAL_NUN, self.NORMAL_PE]
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        # Final letter analysis for logical-visual decision.
+        # Look for evidence that the received buffer is either logical Hebrew
+        # or visual Hebrew.
+        # The following cases are checked:
+        # 1) A word longer than 1 letter, ending with a final letter. This is
+        #    an indication that the text is laid out "naturally" since the
+        #    final letter really appears at the end. +1 for logical score.
+        # 2) A word longer than 1 letter, ending with a Non-Final letter. In
+        #    normal Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi,
+        #    should not end with the Non-Final form of that letter. Exceptions
+        #    to this rule are mentioned above in isNonFinal(). This is an
+        #    indication that the text is laid out backwards. +1 for visual
+        #    score
+        # 3) A word longer than 1 letter, starting with a final letter. Final
+        #    letters should not appear at the beginning of a word. This is an
+        #    indication that the text is laid out backwards. +1 for visual
+        #    score.
+        #
+        # The visual score and logical score are accumulated throughout the
+        # text and are finally checked against each other in GetCharSetName().
+        # No checking for final letters in the middle of words is done since
+        # that case is not an indication for either Logical or Visual text.
+        #
+        # We automatically filter out all 7-bit characters (replace them with
+        # spaces) so the word boundary detection works properly. [MAP]
+
+        if self.state == ProbingState.NOT_ME:
+            # Both model probers say it's not them. No reason to continue.
+            return ProbingState.NOT_ME
+
+        byte_str = self.filter_high_byte_only(byte_str)
+
+        for cur in byte_str:
+            if cur == self.SPACE:
+                # We stand on a space - a word just ended
+                if self._before_prev != self.SPACE:
+                    # next-to-last char was not a space so self._prev is not a
+                    # 1 letter word
+                    if self.is_final(self._prev):
+                        # case (1) [-2:not space][-1:final letter][cur:space]
+                        self._final_char_logical_score += 1
+                    elif self.is_non_final(self._prev):
+                        # case (2) [-2:not space][-1:Non-Final letter][
+                        #  cur:space]
+                        self._final_char_visual_score += 1
+            else:
+                # Not standing on a space
+                if (
+                    (self._before_prev == self.SPACE)
+                    and (self.is_final(self._prev))
+                    and (cur != self.SPACE)
+                ):
+                    # case (3) [-2:space][-1:final letter][cur:not space]
+                    self._final_char_visual_score += 1
+            self._before_prev = self._prev
+            self._prev = cur
+
+        # Forever detecting, till the end or until both model probers return
+        # ProbingState.NOT_ME (handled above)
+        return ProbingState.DETECTING
+
+    @property
+    def charset_name(self) -> str:
+        assert self._logical_prober is not None
+        assert self._visual_prober is not None
+
+        # Make the decision: is it Logical or Visual?
+        # If the final letter score distance is dominant enough, rely on it.
+        finalsub = self._final_char_logical_score - self._final_char_visual_score
+        if finalsub >= self.MIN_FINAL_CHAR_DISTANCE:
+            return self.LOGICAL_HEBREW_NAME
+        if finalsub <= -self.MIN_FINAL_CHAR_DISTANCE:
+            return self.VISUAL_HEBREW_NAME
+
+        # It's not dominant enough, try to rely on the model scores instead.
+        modelsub = (
+            self._logical_prober.get_confidence() - self._visual_prober.get_confidence()
+        )
+        if modelsub > self.MIN_MODEL_DISTANCE:
+            return self.LOGICAL_HEBREW_NAME
+        if modelsub < -self.MIN_MODEL_DISTANCE:
+            return self.VISUAL_HEBREW_NAME
+
+        # Still no good, back to final letter distance, maybe it'll save the
+        # day.
+        if finalsub < 0.0:
+            return self.VISUAL_HEBREW_NAME
+
+        # (finalsub > 0 - Logical) or (don't know what to do) default to
+        # Logical.
+        return self.LOGICAL_HEBREW_NAME
+
+    @property
+    def language(self) -> str:
+        return "Hebrew"
+
+    @property
+    def state(self) -> ProbingState:
+        assert self._logical_prober is not None
+        assert self._visual_prober is not None
+
+        # Remain active as long as any of the model probers are active.
+        if (self._logical_prober.state == ProbingState.NOT_ME) and (
+            self._visual_prober.state == ProbingState.NOT_ME
+        ):
+            return ProbingState.NOT_ME
+        return ProbingState.DETECTING
diff --git a/contrib/python/chardet/py3/chardet/jisfreq.py b/contrib/python/chardet/py3/chardet/jisfreq.py
new file mode 100644
index 00000000000..3293576e012
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/jisfreq.py
@@ -0,0 +1,325 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+# Sampling from about 20M text materials include literature and computer technology
+#
+# Japanese frequency table, applied to both S-JIS and EUC-JP
+# They are sorted in order.
+
+# 128  --> 0.77094
+# 256  --> 0.85710
+# 512  --> 0.92635
+# 1024 --> 0.97130
+# 2048 --> 0.99431
+#
+# Ideal Distribution Ratio = 0.92635 / (1-0.92635) = 12.58
+# Random Distribution Ration = 512 / (2965+62+83+86-512) = 0.191
+#
+# Typical Distribution Ratio, 25% of IDR
+
+JIS_TYPICAL_DISTRIBUTION_RATIO = 3.0
+
+# Char to FreqOrder table ,
+JIS_TABLE_SIZE = 4368
+
+# fmt: off
+JIS_CHAR_TO_FREQ_ORDER = (
+  40,   1,   6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, #   16
+3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247,  18, 179,5071, 856,1661, #   32
+1262,5072, 619, 127,3431,3512,3230,1899,1700, 232, 228,1294,1298, 284, 283,2041, #   48
+2042,1061,1062,  48,  49,  44,  45, 433, 434,1040,1041, 996, 787,2997,1255,4305, #   64
+2108,4609,1684,1648,5073,5074,5075,5076,5077,5078,3687,5079,4610,5080,3927,3928, #   80
+5081,3296,3432, 290,2285,1471,2187,5082,2580,2825,1303,2140,1739,1445,2691,3375, #   96
+1691,3297,4306,4307,4611, 452,3376,1182,2713,3688,3069,4308,5083,5084,5085,5086, #  112
+5087,5088,5089,5090,5091,5092,5093,5094,5095,5096,5097,5098,5099,5100,5101,5102, #  128
+5103,5104,5105,5106,5107,5108,5109,5110,5111,5112,4097,5113,5114,5115,5116,5117, #  144
+5118,5119,5120,5121,5122,5123,5124,5125,5126,5127,5128,5129,5130,5131,5132,5133, #  160
+5134,5135,5136,5137,5138,5139,5140,5141,5142,5143,5144,5145,5146,5147,5148,5149, #  176
+5150,5151,5152,4612,5153,5154,5155,5156,5157,5158,5159,5160,5161,5162,5163,5164, #  192
+5165,5166,5167,5168,5169,5170,5171,5172,5173,5174,5175,1472, 598, 618, 820,1205, #  208
+1309,1412,1858,1307,1692,5176,5177,5178,5179,5180,5181,5182,1142,1452,1234,1172, #  224
+1875,2043,2149,1793,1382,2973, 925,2404,1067,1241, 960,1377,2935,1491, 919,1217, #  240
+1865,2030,1406,1499,2749,4098,5183,5184,5185,5186,5187,5188,2561,4099,3117,1804, #  256
+2049,3689,4309,3513,1663,5189,3166,3118,3298,1587,1561,3433,5190,3119,1625,2998, #  272
+3299,4613,1766,3690,2786,4614,5191,5192,5193,5194,2161,  26,3377,   2,3929,  20, #  288
+3691,  47,4100,  50,  17,  16,  35, 268,  27, 243,  42, 155,  24, 154,  29, 184, #  304
+   4,  91,  14,  92,  53, 396,  33, 289,   9,  37,  64, 620,  21,  39, 321,   5, #  320
+  12,  11,  52,  13,   3, 208, 138,   0,   7,  60, 526, 141, 151,1069, 181, 275, #  336
+1591,  83, 132,1475, 126, 331, 829,  15,  69, 160,  59,  22, 157,  55,1079, 312, #  352
+ 109,  38,  23,  25,  10,  19,  79,5195,  61, 382,1124,   8,  30,5196,5197,5198, #  368
+5199,5200,5201,5202,5203,5204,5205,5206,  89,  62,  74,  34,2416, 112, 139, 196, #  384
+ 271, 149,  84, 607, 131, 765,  46,  88, 153, 683,  76, 874, 101, 258,  57,  80, #  400
+  32, 364, 121,1508, 169,1547,  68, 235, 145,2999,  41, 360,3027,  70,  63,  31, #  416
+  43, 259, 262,1383,  99, 533, 194,  66,  93, 846, 217, 192,  56, 106,  58, 565, #  432
+ 280, 272, 311, 256, 146,  82, 308,  71, 100, 128, 214, 655, 110, 261, 104,1140, #  448
+  54,  51,  36,  87,  67,3070, 185,2618,2936,2020,  28,1066,2390,2059,5207,5208, #  464
+5209,5210,5211,5212,5213,5214,5215,5216,4615,5217,5218,5219,5220,5221,5222,5223, #  480
+5224,5225,5226,5227,5228,5229,5230,5231,5232,5233,5234,5235,5236,3514,5237,5238, #  496
+5239,5240,5241,5242,5243,5244,2297,2031,4616,4310,3692,5245,3071,5246,3598,5247, #  512
+4617,3231,3515,5248,4101,4311,4618,3808,4312,4102,5249,4103,4104,3599,5250,5251, #  528
+5252,5253,5254,5255,5256,5257,5258,5259,5260,5261,5262,5263,5264,5265,5266,5267, #  544
+5268,5269,5270,5271,5272,5273,5274,5275,5276,5277,5278,5279,5280,5281,5282,5283, #  560
+5284,5285,5286,5287,5288,5289,5290,5291,5292,5293,5294,5295,5296,5297,5298,5299, #  576
+5300,5301,5302,5303,5304,5305,5306,5307,5308,5309,5310,5311,5312,5313,5314,5315, #  592
+5316,5317,5318,5319,5320,5321,5322,5323,5324,5325,5326,5327,5328,5329,5330,5331, #  608
+5332,5333,5334,5335,5336,5337,5338,5339,5340,5341,5342,5343,5344,5345,5346,5347, #  624
+5348,5349,5350,5351,5352,5353,5354,5355,5356,5357,5358,5359,5360,5361,5362,5363, #  640
+5364,5365,5366,5367,5368,5369,5370,5371,5372,5373,5374,5375,5376,5377,5378,5379, #  656
+5380,5381, 363, 642,2787,2878,2788,2789,2316,3232,2317,3434,2011, 165,1942,3930, #  672
+3931,3932,3933,5382,4619,5383,4620,5384,5385,5386,5387,5388,5389,5390,5391,5392, #  688
+5393,5394,5395,5396,5397,5398,5399,5400,5401,5402,5403,5404,5405,5406,5407,5408, #  704
+5409,5410,5411,5412,5413,5414,5415,5416,5417,5418,5419,5420,5421,5422,5423,5424, #  720
+5425,5426,5427,5428,5429,5430,5431,5432,5433,5434,5435,5436,5437,5438,5439,5440, #  736
+5441,5442,5443,5444,5445,5446,5447,5448,5449,5450,5451,5452,5453,5454,5455,5456, #  752
+5457,5458,5459,5460,5461,5462,5463,5464,5465,5466,5467,5468,5469,5470,5471,5472, #  768
+5473,5474,5475,5476,5477,5478,5479,5480,5481,5482,5483,5484,5485,5486,5487,5488, #  784
+5489,5490,5491,5492,5493,5494,5495,5496,5497,5498,5499,5500,5501,5502,5503,5504, #  800
+5505,5506,5507,5508,5509,5510,5511,5512,5513,5514,5515,5516,5517,5518,5519,5520, #  816
+5521,5522,5523,5524,5525,5526,5527,5528,5529,5530,5531,5532,5533,5534,5535,5536, #  832
+5537,5538,5539,5540,5541,5542,5543,5544,5545,5546,5547,5548,5549,5550,5551,5552, #  848
+5553,5554,5555,5556,5557,5558,5559,5560,5561,5562,5563,5564,5565,5566,5567,5568, #  864
+5569,5570,5571,5572,5573,5574,5575,5576,5577,5578,5579,5580,5581,5582,5583,5584, #  880
+5585,5586,5587,5588,5589,5590,5591,5592,5593,5594,5595,5596,5597,5598,5599,5600, #  896
+5601,5602,5603,5604,5605,5606,5607,5608,5609,5610,5611,5612,5613,5614,5615,5616, #  912
+5617,5618,5619,5620,5621,5622,5623,5624,5625,5626,5627,5628,5629,5630,5631,5632, #  928
+5633,5634,5635,5636,5637,5638,5639,5640,5641,5642,5643,5644,5645,5646,5647,5648, #  944
+5649,5650,5651,5652,5653,5654,5655,5656,5657,5658,5659,5660,5661,5662,5663,5664, #  960
+5665,5666,5667,5668,5669,5670,5671,5672,5673,5674,5675,5676,5677,5678,5679,5680, #  976
+5681,5682,5683,5684,5685,5686,5687,5688,5689,5690,5691,5692,5693,5694,5695,5696, #  992
+5697,5698,5699,5700,5701,5702,5703,5704,5705,5706,5707,5708,5709,5710,5711,5712, # 1008
+5713,5714,5715,5716,5717,5718,5719,5720,5721,5722,5723,5724,5725,5726,5727,5728, # 1024
+5729,5730,5731,5732,5733,5734,5735,5736,5737,5738,5739,5740,5741,5742,5743,5744, # 1040
+5745,5746,5747,5748,5749,5750,5751,5752,5753,5754,5755,5756,5757,5758,5759,5760, # 1056
+5761,5762,5763,5764,5765,5766,5767,5768,5769,5770,5771,5772,5773,5774,5775,5776, # 1072
+5777,5778,5779,5780,5781,5782,5783,5784,5785,5786,5787,5788,5789,5790,5791,5792, # 1088
+5793,5794,5795,5796,5797,5798,5799,5800,5801,5802,5803,5804,5805,5806,5807,5808, # 1104
+5809,5810,5811,5812,5813,5814,5815,5816,5817,5818,5819,5820,5821,5822,5823,5824, # 1120
+5825,5826,5827,5828,5829,5830,5831,5832,5833,5834,5835,5836,5837,5838,5839,5840, # 1136
+5841,5842,5843,5844,5845,5846,5847,5848,5849,5850,5851,5852,5853,5854,5855,5856, # 1152
+5857,5858,5859,5860,5861,5862,5863,5864,5865,5866,5867,5868,5869,5870,5871,5872, # 1168
+5873,5874,5875,5876,5877,5878,5879,5880,5881,5882,5883,5884,5885,5886,5887,5888, # 1184
+5889,5890,5891,5892,5893,5894,5895,5896,5897,5898,5899,5900,5901,5902,5903,5904, # 1200
+5905,5906,5907,5908,5909,5910,5911,5912,5913,5914,5915,5916,5917,5918,5919,5920, # 1216
+5921,5922,5923,5924,5925,5926,5927,5928,5929,5930,5931,5932,5933,5934,5935,5936, # 1232
+5937,5938,5939,5940,5941,5942,5943,5944,5945,5946,5947,5948,5949,5950,5951,5952, # 1248
+5953,5954,5955,5956,5957,5958,5959,5960,5961,5962,5963,5964,5965,5966,5967,5968, # 1264
+5969,5970,5971,5972,5973,5974,5975,5976,5977,5978,5979,5980,5981,5982,5983,5984, # 1280
+5985,5986,5987,5988,5989,5990,5991,5992,5993,5994,5995,5996,5997,5998,5999,6000, # 1296
+6001,6002,6003,6004,6005,6006,6007,6008,6009,6010,6011,6012,6013,6014,6015,6016, # 1312
+6017,6018,6019,6020,6021,6022,6023,6024,6025,6026,6027,6028,6029,6030,6031,6032, # 1328
+6033,6034,6035,6036,6037,6038,6039,6040,6041,6042,6043,6044,6045,6046,6047,6048, # 1344
+6049,6050,6051,6052,6053,6054,6055,6056,6057,6058,6059,6060,6061,6062,6063,6064, # 1360
+6065,6066,6067,6068,6069,6070,6071,6072,6073,6074,6075,6076,6077,6078,6079,6080, # 1376
+6081,6082,6083,6084,6085,6086,6087,6088,6089,6090,6091,6092,6093,6094,6095,6096, # 1392
+6097,6098,6099,6100,6101,6102,6103,6104,6105,6106,6107,6108,6109,6110,6111,6112, # 1408
+6113,6114,2044,2060,4621, 997,1235, 473,1186,4622, 920,3378,6115,6116, 379,1108, # 1424
+4313,2657,2735,3934,6117,3809, 636,3233, 573,1026,3693,3435,2974,3300,2298,4105, # 1440
+ 854,2937,2463, 393,2581,2417, 539, 752,1280,2750,2480, 140,1161, 440, 708,1569, # 1456
+ 665,2497,1746,1291,1523,3000, 164,1603, 847,1331, 537,1997, 486, 508,1693,2418, # 1472
+1970,2227, 878,1220, 299,1030, 969, 652,2751, 624,1137,3301,2619,  65,3302,2045, # 1488
+1761,1859,3120,1930,3694,3516, 663,1767, 852, 835,3695, 269, 767,2826,2339,1305, # 1504
+ 896,1150, 770,1616,6118, 506,1502,2075,1012,2519, 775,2520,2975,2340,2938,4314, # 1520
+3028,2086,1224,1943,2286,6119,3072,4315,2240,1273,1987,3935,1557, 175, 597, 985, # 1536
+3517,2419,2521,1416,3029, 585, 938,1931,1007,1052,1932,1685,6120,3379,4316,4623, # 1552
+ 804, 599,3121,1333,2128,2539,1159,1554,2032,3810, 687,2033,2904, 952, 675,1467, # 1568
+3436,6121,2241,1096,1786,2440,1543,1924, 980,1813,2228, 781,2692,1879, 728,1918, # 1584
+3696,4624, 548,1950,4625,1809,1088,1356,3303,2522,1944, 502, 972, 373, 513,2827, # 1600
+ 586,2377,2391,1003,1976,1631,6122,2464,1084, 648,1776,4626,2141, 324, 962,2012, # 1616
+2177,2076,1384, 742,2178,1448,1173,1810, 222, 102, 301, 445, 125,2420, 662,2498, # 1632
+ 277, 200,1476,1165,1068, 224,2562,1378,1446, 450,1880, 659, 791, 582,4627,2939, # 1648
+3936,1516,1274, 555,2099,3697,1020,1389,1526,3380,1762,1723,1787,2229, 412,2114, # 1664
+1900,2392,3518, 512,2597, 427,1925,2341,3122,1653,1686,2465,2499, 697, 330, 273, # 1680
+ 380,2162, 951, 832, 780, 991,1301,3073, 965,2270,3519, 668,2523,2636,1286, 535, # 1696
+1407, 518, 671, 957,2658,2378, 267, 611,2197,3030,6123, 248,2299, 967,1799,2356, # 1712
+ 850,1418,3437,1876,1256,1480,2828,1718,6124,6125,1755,1664,2405,6126,4628,2879, # 1728
+2829, 499,2179, 676,4629, 557,2329,2214,2090, 325,3234, 464, 811,3001, 992,2342, # 1744
+2481,1232,1469, 303,2242, 466,1070,2163, 603,1777,2091,4630,2752,4631,2714, 322, # 1760
+2659,1964,1768, 481,2188,1463,2330,2857,3600,2092,3031,2421,4632,2318,2070,1849, # 1776
+2598,4633,1302,2254,1668,1701,2422,3811,2905,3032,3123,2046,4106,1763,1694,4634, # 1792
+1604, 943,1724,1454, 917, 868,2215,1169,2940, 552,1145,1800,1228,1823,1955, 316, # 1808
+1080,2510, 361,1807,2830,4107,2660,3381,1346,1423,1134,4108,6127, 541,1263,1229, # 1824
+1148,2540, 545, 465,1833,2880,3438,1901,3074,2482, 816,3937, 713,1788,2500, 122, # 1840
+1575, 195,1451,2501,1111,6128, 859, 374,1225,2243,2483,4317, 390,1033,3439,3075, # 1856
+2524,1687, 266, 793,1440,2599, 946, 779, 802, 507, 897,1081, 528,2189,1292, 711, # 1872
+1866,1725,1167,1640, 753, 398,2661,1053, 246, 348,4318, 137,1024,3440,1600,2077, # 1888
+2129, 825,4319, 698, 238, 521, 187,2300,1157,2423,1641,1605,1464,1610,1097,2541, # 1904
+1260,1436, 759,2255,1814,2150, 705,3235, 409,2563,3304, 561,3033,2005,2564, 726, # 1920
+1956,2343,3698,4109, 949,3812,3813,3520,1669, 653,1379,2525, 881,2198, 632,2256, # 1936
+1027, 778,1074, 733,1957, 514,1481,2466, 554,2180, 702,3938,1606,1017,1398,6129, # 1952
+1380,3521, 921, 993,1313, 594, 449,1489,1617,1166, 768,1426,1360, 495,1794,3601, # 1968
+1177,3602,1170,4320,2344, 476, 425,3167,4635,3168,1424, 401,2662,1171,3382,1998, # 1984
+1089,4110, 477,3169, 474,6130,1909, 596,2831,1842, 494, 693,1051,1028,1207,3076, # 2000
+ 606,2115, 727,2790,1473,1115, 743,3522, 630, 805,1532,4321,2021, 366,1057, 838, # 2016
+ 684,1114,2142,4322,2050,1492,1892,1808,2271,3814,2424,1971,1447,1373,3305,1090, # 2032
+1536,3939,3523,3306,1455,2199, 336, 369,2331,1035, 584,2393, 902, 718,2600,6131, # 2048
+2753, 463,2151,1149,1611,2467, 715,1308,3124,1268, 343,1413,3236,1517,1347,2663, # 2064
+2093,3940,2022,1131,1553,2100,2941,1427,3441,2942,1323,2484,6132,1980, 872,2368, # 2080
+2441,2943, 320,2369,2116,1082, 679,1933,3941,2791,3815, 625,1143,2023, 422,2200, # 2096
+3816,6133, 730,1695, 356,2257,1626,2301,2858,2637,1627,1778, 937, 883,2906,2693, # 2112
+3002,1769,1086, 400,1063,1325,3307,2792,4111,3077, 456,2345,1046, 747,6134,1524, # 2128
+ 884,1094,3383,1474,2164,1059, 974,1688,2181,2258,1047, 345,1665,1187, 358, 875, # 2144
+3170, 305, 660,3524,2190,1334,1135,3171,1540,1649,2542,1527, 927, 968,2793, 885, # 2160
+1972,1850, 482, 500,2638,1218,1109,1085,2543,1654,2034, 876,  78,2287,1482,1277, # 2176
+ 861,1675,1083,1779, 724,2754, 454, 397,1132,1612,2332, 893, 672,1237, 257,2259, # 2192
+2370, 135,3384, 337,2244, 547, 352, 340, 709,2485,1400, 788,1138,2511, 540, 772, # 2208
+1682,2260,2272,2544,2013,1843,1902,4636,1999,1562,2288,4637,2201,1403,1533, 407, # 2224
+ 576,3308,1254,2071, 978,3385, 170, 136,1201,3125,2664,3172,2394, 213, 912, 873, # 2240
+3603,1713,2202, 699,3604,3699, 813,3442, 493, 531,1054, 468,2907,1483, 304, 281, # 2256
+4112,1726,1252,2094, 339,2319,2130,2639, 756,1563,2944, 748, 571,2976,1588,2425, # 2272
+2715,1851,1460,2426,1528,1392,1973,3237, 288,3309, 685,3386, 296, 892,2716,2216, # 2288
+1570,2245, 722,1747,2217, 905,3238,1103,6135,1893,1441,1965, 251,1805,2371,3700, # 2304
+2601,1919,1078,  75,2182,1509,1592,1270,2640,4638,2152,6136,3310,3817, 524, 706, # 2320
+1075, 292,3818,1756,2602, 317,  98,3173,3605,3525,1844,2218,3819,2502, 814, 567, # 2336
+ 385,2908,1534,6137, 534,1642,3239, 797,6138,1670,1529, 953,4323, 188,1071, 538, # 2352
+ 178, 729,3240,2109,1226,1374,2000,2357,2977, 731,2468,1116,2014,2051,6139,1261, # 2368
+1593, 803,2859,2736,3443, 556, 682, 823,1541,6140,1369,2289,1706,2794, 845, 462, # 2384
+2603,2665,1361, 387, 162,2358,1740, 739,1770,1720,1304,1401,3241,1049, 627,1571, # 2400
+2427,3526,1877,3942,1852,1500, 431,1910,1503, 677, 297,2795, 286,1433,1038,1198, # 2416
+2290,1133,1596,4113,4639,2469,1510,1484,3943,6141,2442, 108, 712,4640,2372, 866, # 2432
+3701,2755,3242,1348, 834,1945,1408,3527,2395,3243,1811, 824, 994,1179,2110,1548, # 2448
+1453, 790,3003, 690,4324,4325,2832,2909,3820,1860,3821, 225,1748, 310, 346,1780, # 2464
+2470, 821,1993,2717,2796, 828, 877,3528,2860,2471,1702,2165,2910,2486,1789, 453, # 2480
+ 359,2291,1676,  73,1164,1461,1127,3311, 421, 604, 314,1037, 589, 116,2487, 737, # 2496
+ 837,1180, 111, 244, 735,6142,2261,1861,1362, 986, 523, 418, 581,2666,3822, 103, # 2512
+ 855, 503,1414,1867,2488,1091, 657,1597, 979, 605,1316,4641,1021,2443,2078,2001, # 2528
+1209,  96, 587,2166,1032, 260,1072,2153, 173,  94, 226,3244, 819,2006,4642,4114, # 2544
+2203, 231,1744, 782,  97,2667, 786,3387, 887, 391, 442,2219,4326,1425,6143,2694, # 2560
+ 633,1544,1202, 483,2015, 592,2052,1958,2472,1655, 419, 129,4327,3444,3312,1714, # 2576
+1257,3078,4328,1518,1098, 865,1310,1019,1885,1512,1734, 469,2444, 148, 773, 436, # 2592
+1815,1868,1128,1055,4329,1245,2756,3445,2154,1934,1039,4643, 579,1238, 932,2320, # 2608
+ 353, 205, 801, 115,2428, 944,2321,1881, 399,2565,1211, 678, 766,3944, 335,2101, # 2624
+1459,1781,1402,3945,2737,2131,1010, 844, 981,1326,1013, 550,1816,1545,2620,1335, # 2640
+1008, 371,2881, 936,1419,1613,3529,1456,1395,2273,1834,2604,1317,2738,2503, 416, # 2656
+1643,4330, 806,1126, 229, 591,3946,1314,1981,1576,1837,1666, 347,1790, 977,3313, # 2672
+ 764,2861,1853, 688,2429,1920,1462,  77, 595, 415,2002,3034, 798,1192,4115,6144, # 2688
+2978,4331,3035,2695,2582,2072,2566, 430,2430,1727, 842,1396,3947,3702, 613, 377, # 2704
+ 278, 236,1417,3388,3314,3174, 757,1869, 107,3530,6145,1194, 623,2262, 207,1253, # 2720
+2167,3446,3948, 492,1117,1935, 536,1838,2757,1246,4332, 696,2095,2406,1393,1572, # 2736
+3175,1782, 583, 190, 253,1390,2230, 830,3126,3389, 934,3245,1703,1749,2979,1870, # 2752
+2545,1656,2204, 869,2346,4116,3176,1817, 496,1764,4644, 942,1504, 404,1903,1122, # 2768
+1580,3606,2945,1022, 515, 372,1735, 955,2431,3036,6146,2797,1110,2302,2798, 617, # 2784
+6147, 441, 762,1771,3447,3607,3608,1904, 840,3037,  86, 939,1385, 572,1370,2445, # 2800
+1336, 114,3703, 898, 294, 203,3315, 703,1583,2274, 429, 961,4333,1854,1951,3390, # 2816
+2373,3704,4334,1318,1381, 966,1911,2322,1006,1155, 309, 989, 458,2718,1795,1372, # 2832
+1203, 252,1689,1363,3177, 517,1936, 168,1490, 562, 193,3823,1042,4117,1835, 551, # 2848
+ 470,4645, 395, 489,3448,1871,1465,2583,2641, 417,1493, 279,1295, 511,1236,1119, # 2864
+  72,1231,1982,1812,3004, 871,1564, 984,3449,1667,2696,2096,4646,2347,2833,1673, # 2880
+3609, 695,3246,2668, 807,1183,4647, 890, 388,2333,1801,1457,2911,1765,1477,1031, # 2896
+3316,3317,1278,3391,2799,2292,2526, 163,3450,4335,2669,1404,1802,6148,2323,2407, # 2912
+1584,1728,1494,1824,1269, 298, 909,3318,1034,1632, 375, 776,1683,2061, 291, 210, # 2928
+1123, 809,1249,1002,2642,3038, 206,1011,2132, 144, 975, 882,1565, 342, 667, 754, # 2944
+1442,2143,1299,2303,2062, 447, 626,2205,1221,2739,2912,1144,1214,2206,2584, 760, # 2960
+1715, 614, 950,1281,2670,2621, 810, 577,1287,2546,4648, 242,2168, 250,2643, 691, # 2976
+ 123,2644, 647, 313,1029, 689,1357,2946,1650, 216, 771,1339,1306, 808,2063, 549, # 2992
+ 913,1371,2913,2914,6149,1466,1092,1174,1196,1311,2605,2396,1783,1796,3079, 406, # 3008
+2671,2117,3949,4649, 487,1825,2220,6150,2915, 448,2348,1073,6151,2397,1707, 130, # 3024
+ 900,1598, 329, 176,1959,2527,1620,6152,2275,4336,3319,1983,2191,3705,3610,2155, # 3040
+3706,1912,1513,1614,6153,1988, 646, 392,2304,1589,3320,3039,1826,1239,1352,1340, # 3056
+2916, 505,2567,1709,1437,2408,2547, 906,6154,2672, 384,1458,1594,1100,1329, 710, # 3072
+ 423,3531,2064,2231,2622,1989,2673,1087,1882, 333, 841,3005,1296,2882,2379, 580, # 3088
+1937,1827,1293,2585, 601, 574, 249,1772,4118,2079,1120, 645, 901,1176,1690, 795, # 3104
+2207, 478,1434, 516,1190,1530, 761,2080, 930,1264, 355, 435,1552, 644,1791, 987, # 3120
+ 220,1364,1163,1121,1538, 306,2169,1327,1222, 546,2645, 218, 241, 610,1704,3321, # 3136
+1984,1839,1966,2528, 451,6155,2586,3707,2568, 907,3178, 254,2947, 186,1845,4650, # 3152
+ 745, 432,1757, 428,1633, 888,2246,2221,2489,3611,2118,1258,1265, 956,3127,1784, # 3168
+4337,2490, 319, 510, 119, 457,3612, 274,2035,2007,4651,1409,3128, 970,2758, 590, # 3184
+2800, 661,2247,4652,2008,3950,1420,1549,3080,3322,3951,1651,1375,2111, 485,2491, # 3200
+1429,1156,6156,2548,2183,1495, 831,1840,2529,2446, 501,1657, 307,1894,3247,1341, # 3216
+ 666, 899,2156,1539,2549,1559, 886, 349,2208,3081,2305,1736,3824,2170,2759,1014, # 3232
+1913,1386, 542,1397,2948, 490, 368, 716, 362, 159, 282,2569,1129,1658,1288,1750, # 3248
+2674, 276, 649,2016, 751,1496, 658,1818,1284,1862,2209,2087,2512,3451, 622,2834, # 3264
+ 376, 117,1060,2053,1208,1721,1101,1443, 247,1250,3179,1792,3952,2760,2398,3953, # 3280
+6157,2144,3708, 446,2432,1151,2570,3452,2447,2761,2835,1210,2448,3082, 424,2222, # 3296
+1251,2449,2119,2836, 504,1581,4338, 602, 817, 857,3825,2349,2306, 357,3826,1470, # 3312
+1883,2883, 255, 958, 929,2917,3248, 302,4653,1050,1271,1751,2307,1952,1430,2697, # 3328
+2719,2359, 354,3180, 777, 158,2036,4339,1659,4340,4654,2308,2949,2248,1146,2232, # 3344
+3532,2720,1696,2623,3827,6158,3129,1550,2698,1485,1297,1428, 637, 931,2721,2145, # 3360
+ 914,2550,2587,  81,2450, 612, 827,2646,1242,4655,1118,2884, 472,1855,3181,3533, # 3376
+3534, 569,1353,2699,1244,1758,2588,4119,2009,2762,2171,3709,1312,1531,6159,1152, # 3392
+1938, 134,1830, 471,3710,2276,1112,1535,3323,3453,3535, 982,1337,2950, 488, 826, # 3408
+ 674,1058,1628,4120,2017, 522,2399, 211, 568,1367,3454, 350, 293,1872,1139,3249, # 3424
+1399,1946,3006,1300,2360,3324, 588, 736,6160,2606, 744, 669,3536,3828,6161,1358, # 3440
+ 199, 723, 848, 933, 851,1939,1505,1514,1338,1618,1831,4656,1634,3613, 443,2740, # 3456
+3829, 717,1947, 491,1914,6162,2551,1542,4121,1025,6163,1099,1223, 198,3040,2722, # 3472
+ 370, 410,1905,2589, 998,1248,3182,2380, 519,1449,4122,1710, 947, 928,1153,4341, # 3488
+2277, 344,2624,1511, 615, 105, 161,1212,1076,1960,3130,2054,1926,1175,1906,2473, # 3504
+ 414,1873,2801,6164,2309, 315,1319,3325, 318,2018,2146,2157, 963, 631, 223,4342, # 3520
+4343,2675, 479,3711,1197,2625,3712,2676,2361,6165,4344,4123,6166,2451,3183,1886, # 3536
+2184,1674,1330,1711,1635,1506, 799, 219,3250,3083,3954,1677,3713,3326,2081,3614, # 3552
+1652,2073,4657,1147,3041,1752, 643,1961, 147,1974,3955,6167,1716,2037, 918,3007, # 3568
+1994, 120,1537, 118, 609,3184,4345, 740,3455,1219, 332,1615,3830,6168,1621,2980, # 3584
+1582, 783, 212, 553,2350,3714,1349,2433,2082,4124, 889,6169,2310,1275,1410, 973, # 3600
+ 166,1320,3456,1797,1215,3185,2885,1846,2590,2763,4658, 629, 822,3008, 763, 940, # 3616
+1990,2862, 439,2409,1566,1240,1622, 926,1282,1907,2764, 654,2210,1607, 327,1130, # 3632
+3956,1678,1623,6170,2434,2192, 686, 608,3831,3715, 903,3957,3042,6171,2741,1522, # 3648
+1915,1105,1555,2552,1359, 323,3251,4346,3457, 738,1354,2553,2311,2334,1828,2003, # 3664
+3832,1753,2351,1227,6172,1887,4125,1478,6173,2410,1874,1712,1847, 520,1204,2607, # 3680
+ 264,4659, 836,2677,2102, 600,4660,3833,2278,3084,6174,4347,3615,1342, 640, 532, # 3696
+ 543,2608,1888,2400,2591,1009,4348,1497, 341,1737,3616,2723,1394, 529,3252,1321, # 3712
+ 983,4661,1515,2120, 971,2592, 924, 287,1662,3186,4349,2700,4350,1519, 908,1948, # 3728
+2452, 156, 796,1629,1486,2223,2055, 694,4126,1259,1036,3392,1213,2249,2742,1889, # 3744
+1230,3958,1015, 910, 408, 559,3617,4662, 746, 725, 935,4663,3959,3009,1289, 563, # 3760
+ 867,4664,3960,1567,2981,2038,2626, 988,2263,2381,4351, 143,2374, 704,1895,6175, # 3776
+1188,3716,2088, 673,3085,2362,4352, 484,1608,1921,2765,2918, 215, 904,3618,3537, # 3792
+ 894, 509, 976,3043,2701,3961,4353,2837,2982, 498,6176,6177,1102,3538,1332,3393, # 3808
+1487,1636,1637, 233, 245,3962, 383, 650, 995,3044, 460,1520,1206,2352, 749,3327, # 3824
+ 530, 700, 389,1438,1560,1773,3963,2264, 719,2951,2724,3834, 870,1832,1644,1000, # 3840
+ 839,2474,3717, 197,1630,3394, 365,2886,3964,1285,2133, 734, 922, 818,1106, 732, # 3856
+ 480,2083,1774,3458, 923,2279,1350, 221,3086,  85,2233,2234,3835,1585,3010,2147, # 3872
+1387,1705,2382,1619,2475, 133, 239,2802,1991,1016,2084,2383, 411,2838,1113, 651, # 3888
+1985,1160,3328, 990,1863,3087,1048,1276,2647, 265,2627,1599,3253,2056, 150, 638, # 3904
+2019, 656, 853, 326,1479, 680,1439,4354,1001,1759, 413,3459,3395,2492,1431, 459, # 3920
+4355,1125,3329,2265,1953,1450,2065,2863, 849, 351,2678,3131,3254,3255,1104,1577, # 3936
+ 227,1351,1645,2453,2193,1421,2887, 812,2121, 634,  95,2435, 201,2312,4665,1646, # 3952
+1671,2743,1601,2554,2702,2648,2280,1315,1366,2089,3132,1573,3718,3965,1729,1189, # 3968
+ 328,2679,1077,1940,1136, 558,1283, 964,1195, 621,2074,1199,1743,3460,3619,1896, # 3984
+1916,1890,3836,2952,1154,2112,1064, 862, 378,3011,2066,2113,2803,1568,2839,6178, # 4000
+3088,2919,1941,1660,2004,1992,2194, 142, 707,1590,1708,1624,1922,1023,1836,1233, # 4016
+1004,2313, 789, 741,3620,6179,1609,2411,1200,4127,3719,3720,4666,2057,3721, 593, # 4032
+2840, 367,2920,1878,6180,3461,1521, 628,1168, 692,2211,2649, 300, 720,2067,2571, # 4048
+2953,3396, 959,2504,3966,3539,3462,1977, 701,6181, 954,1043, 800, 681, 183,3722, # 4064
+1803,1730,3540,4128,2103, 815,2314, 174, 467, 230,2454,1093,2134, 755,3541,3397, # 4080
+1141,1162,6182,1738,2039, 270,3256,2513,1005,1647,2185,3837, 858,1679,1897,1719, # 4096
+2954,2324,1806, 402, 670, 167,4129,1498,2158,2104, 750,6183, 915, 189,1680,1551, # 4112
+ 455,4356,1501,2455, 405,1095,2955, 338,1586,1266,1819, 570, 641,1324, 237,1556, # 4128
+2650,1388,3723,6184,1368,2384,1343,1978,3089,2436, 879,3724, 792,1191, 758,3012, # 4144
+1411,2135,1322,4357, 240,4667,1848,3725,1574,6185, 420,3045,1546,1391, 714,4358, # 4160
+1967, 941,1864, 863, 664, 426, 560,1731,2680,1785,2864,1949,2363, 403,3330,1415, # 4176
+1279,2136,1697,2335, 204, 721,2097,3838,  90,6186,2085,2505, 191,3967, 124,2148, # 4192
+1376,1798,1178,1107,1898,1405, 860,4359,1243,1272,2375,2983,1558,2456,1638, 113, # 4208
+3621, 578,1923,2609, 880, 386,4130, 784,2186,2266,1422,2956,2172,1722, 497, 263, # 4224
+2514,1267,2412,2610, 177,2703,3542, 774,1927,1344, 616,1432,1595,1018, 172,4360, # 4240
+2325, 911,4361, 438,1468,3622, 794,3968,2024,2173,1681,1829,2957, 945, 895,3090, # 4256
+ 575,2212,2476, 475,2401,2681, 785,2744,1745,2293,2555,1975,3133,2865, 394,4668, # 4272
+3839, 635,4131, 639, 202,1507,2195,2766,1345,1435,2572,3726,1908,1184,1181,2457, # 4288
+3727,3134,4362, 843,2611, 437, 916,4669, 234, 769,1884,3046,3047,3623, 833,6187, # 4304
+1639,2250,2402,1355,1185,2010,2047, 999, 525,1732,1290,1488,2612, 948,1578,3728, # 4320
+2413,2477,1216,2725,2159, 334,3840,1328,3624,2921,1525,4132, 564,1056, 891,4363, # 4336
+1444,1698,2385,2251,3729,1365,2281,2235,1717,6188, 864,3841,2515, 444, 527,2767, # 4352
+2922,3625, 544, 461,6189, 566, 209,2437,3398,2098,1065,2068,3331,3626,3257,2137, # 4368  #last 512
+)
+# fmt: on
diff --git a/contrib/python/chardet/py3/chardet/johabfreq.py b/contrib/python/chardet/py3/chardet/johabfreq.py
new file mode 100644
index 00000000000..c12969990d7
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/johabfreq.py
@@ -0,0 +1,2382 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+# The frequency data itself is the same as euc-kr.
+# This is just a mapping table to euc-kr.
+
+JOHAB_TO_EUCKR_ORDER_TABLE = {
+    0x8861: 0,
+    0x8862: 1,
+    0x8865: 2,
+    0x8868: 3,
+    0x8869: 4,
+    0x886A: 5,
+    0x886B: 6,
+    0x8871: 7,
+    0x8873: 8,
+    0x8874: 9,
+    0x8875: 10,
+    0x8876: 11,
+    0x8877: 12,
+    0x8878: 13,
+    0x8879: 14,
+    0x887B: 15,
+    0x887C: 16,
+    0x887D: 17,
+    0x8881: 18,
+    0x8882: 19,
+    0x8885: 20,
+    0x8889: 21,
+    0x8891: 22,
+    0x8893: 23,
+    0x8895: 24,
+    0x8896: 25,
+    0x8897: 26,
+    0x88A1: 27,
+    0x88A2: 28,
+    0x88A5: 29,
+    0x88A9: 30,
+    0x88B5: 31,
+    0x88B7: 32,
+    0x88C1: 33,
+    0x88C5: 34,
+    0x88C9: 35,
+    0x88E1: 36,
+    0x88E2: 37,
+    0x88E5: 38,
+    0x88E8: 39,
+    0x88E9: 40,
+    0x88EB: 41,
+    0x88F1: 42,
+    0x88F3: 43,
+    0x88F5: 44,
+    0x88F6: 45,
+    0x88F7: 46,
+    0x88F8: 47,
+    0x88FB: 48,
+    0x88FC: 49,
+    0x88FD: 50,
+    0x8941: 51,
+    0x8945: 52,
+    0x8949: 53,
+    0x8951: 54,
+    0x8953: 55,
+    0x8955: 56,
+    0x8956: 57,
+    0x8957: 58,
+    0x8961: 59,
+    0x8962: 60,
+    0x8963: 61,
+    0x8965: 62,
+    0x8968: 63,
+    0x8969: 64,
+    0x8971: 65,
+    0x8973: 66,
+    0x8975: 67,
+    0x8976: 68,
+    0x8977: 69,
+    0x897B: 70,
+    0x8981: 71,
+    0x8985: 72,
+    0x8989: 73,
+    0x8993: 74,
+    0x8995: 75,
+    0x89A1: 76,
+    0x89A2: 77,
+    0x89A5: 78,
+    0x89A8: 79,
+    0x89A9: 80,
+    0x89AB: 81,
+    0x89AD: 82,
+    0x89B0: 83,
+    0x89B1: 84,
+    0x89B3: 85,
+    0x89B5: 86,
+    0x89B7: 87,
+    0x89B8: 88,
+    0x89C1: 89,
+    0x89C2: 90,
+    0x89C5: 91,
+    0x89C9: 92,
+    0x89CB: 93,
+    0x89D1: 94,
+    0x89D3: 95,
+    0x89D5: 96,
+    0x89D7: 97,
+    0x89E1: 98,
+    0x89E5: 99,
+    0x89E9: 100,
+    0x89F3: 101,
+    0x89F6: 102,
+    0x89F7: 103,
+    0x8A41: 104,
+    0x8A42: 105,
+    0x8A45: 106,
+    0x8A49: 107,
+    0x8A51: 108,
+    0x8A53: 109,
+    0x8A55: 110,
+    0x8A57: 111,
+    0x8A61: 112,
+    0x8A65: 113,
+    0x8A69: 114,
+    0x8A73: 115,
+    0x8A75: 116,
+    0x8A81: 117,
+    0x8A82: 118,
+    0x8A85: 119,
+    0x8A88: 120,
+    0x8A89: 121,
+    0x8A8A: 122,
+    0x8A8B: 123,
+    0x8A90: 124,
+    0x8A91: 125,
+    0x8A93: 126,
+    0x8A95: 127,
+    0x8A97: 128,
+    0x8A98: 129,
+    0x8AA1: 130,
+    0x8AA2: 131,
+    0x8AA5: 132,
+    0x8AA9: 133,
+    0x8AB6: 134,
+    0x8AB7: 135,
+    0x8AC1: 136,
+    0x8AD5: 137,
+    0x8AE1: 138,
+    0x8AE2: 139,
+    0x8AE5: 140,
+    0x8AE9: 141,
+    0x8AF1: 142,
+    0x8AF3: 143,
+    0x8AF5: 144,
+    0x8B41: 145,
+    0x8B45: 146,
+    0x8B49: 147,
+    0x8B61: 148,
+    0x8B62: 149,
+    0x8B65: 150,
+    0x8B68: 151,
+    0x8B69: 152,
+    0x8B6A: 153,
+    0x8B71: 154,
+    0x8B73: 155,
+    0x8B75: 156,
+    0x8B77: 157,
+    0x8B81: 158,
+    0x8BA1: 159,
+    0x8BA2: 160,
+    0x8BA5: 161,
+    0x8BA8: 162,
+    0x8BA9: 163,
+    0x8BAB: 164,
+    0x8BB1: 165,
+    0x8BB3: 166,
+    0x8BB5: 167,
+    0x8BB7: 168,
+    0x8BB8: 169,
+    0x8BBC: 170,
+    0x8C61: 171,
+    0x8C62: 172,
+    0x8C63: 173,
+    0x8C65: 174,
+    0x8C69: 175,
+    0x8C6B: 176,
+    0x8C71: 177,
+    0x8C73: 178,
+    0x8C75: 179,
+    0x8C76: 180,
+    0x8C77: 181,
+    0x8C7B: 182,
+    0x8C81: 183,
+    0x8C82: 184,
+    0x8C85: 185,
+    0x8C89: 186,
+    0x8C91: 187,
+    0x8C93: 188,
+    0x8C95: 189,
+    0x8C96: 190,
+    0x8C97: 191,
+    0x8CA1: 192,
+    0x8CA2: 193,
+    0x8CA9: 194,
+    0x8CE1: 195,
+    0x8CE2: 196,
+    0x8CE3: 197,
+    0x8CE5: 198,
+    0x8CE9: 199,
+    0x8CF1: 200,
+    0x8CF3: 201,
+    0x8CF5: 202,
+    0x8CF6: 203,
+    0x8CF7: 204,
+    0x8D41: 205,
+    0x8D42: 206,
+    0x8D45: 207,
+    0x8D51: 208,
+    0x8D55: 209,
+    0x8D57: 210,
+    0x8D61: 211,
+    0x8D65: 212,
+    0x8D69: 213,
+    0x8D75: 214,
+    0x8D76: 215,
+    0x8D7B: 216,
+    0x8D81: 217,
+    0x8DA1: 218,
+    0x8DA2: 219,
+    0x8DA5: 220,
+    0x8DA7: 221,
+    0x8DA9: 222,
+    0x8DB1: 223,
+    0x8DB3: 224,
+    0x8DB5: 225,
+    0x8DB7: 226,
+    0x8DB8: 227,
+    0x8DB9: 228,
+    0x8DC1: 229,
+    0x8DC2: 230,
+    0x8DC9: 231,
+    0x8DD6: 232,
+    0x8DD7: 233,
+    0x8DE1: 234,
+    0x8DE2: 235,
+    0x8DF7: 236,
+    0x8E41: 237,
+    0x8E45: 238,
+    0x8E49: 239,
+    0x8E51: 240,
+    0x8E53: 241,
+    0x8E57: 242,
+    0x8E61: 243,
+    0x8E81: 244,
+    0x8E82: 245,
+    0x8E85: 246,
+    0x8E89: 247,
+    0x8E90: 248,
+    0x8E91: 249,
+    0x8E93: 250,
+    0x8E95: 251,
+    0x8E97: 252,
+    0x8E98: 253,
+    0x8EA1: 254,
+    0x8EA9: 255,
+    0x8EB6: 256,
+    0x8EB7: 257,
+    0x8EC1: 258,
+    0x8EC2: 259,
+    0x8EC5: 260,
+    0x8EC9: 261,
+    0x8ED1: 262,
+    0x8ED3: 263,
+    0x8ED6: 264,
+    0x8EE1: 265,
+    0x8EE5: 266,
+    0x8EE9: 267,
+    0x8EF1: 268,
+    0x8EF3: 269,
+    0x8F41: 270,
+    0x8F61: 271,
+    0x8F62: 272,
+    0x8F65: 273,
+    0x8F67: 274,
+    0x8F69: 275,
+    0x8F6B: 276,
+    0x8F70: 277,
+    0x8F71: 278,
+    0x8F73: 279,
+    0x8F75: 280,
+    0x8F77: 281,
+    0x8F7B: 282,
+    0x8FA1: 283,
+    0x8FA2: 284,
+    0x8FA5: 285,
+    0x8FA9: 286,
+    0x8FB1: 287,
+    0x8FB3: 288,
+    0x8FB5: 289,
+    0x8FB7: 290,
+    0x9061: 291,
+    0x9062: 292,
+    0x9063: 293,
+    0x9065: 294,
+    0x9068: 295,
+    0x9069: 296,
+    0x906A: 297,
+    0x906B: 298,
+    0x9071: 299,
+    0x9073: 300,
+    0x9075: 301,
+    0x9076: 302,
+    0x9077: 303,
+    0x9078: 304,
+    0x9079: 305,
+    0x907B: 306,
+    0x907D: 307,
+    0x9081: 308,
+    0x9082: 309,
+    0x9085: 310,
+    0x9089: 311,
+    0x9091: 312,
+    0x9093: 313,
+    0x9095: 314,
+    0x9096: 315,
+    0x9097: 316,
+    0x90A1: 317,
+    0x90A2: 318,
+    0x90A5: 319,
+    0x90A9: 320,
+    0x90B1: 321,
+    0x90B7: 322,
+    0x90E1: 323,
+    0x90E2: 324,
+    0x90E4: 325,
+    0x90E5: 326,
+    0x90E9: 327,
+    0x90EB: 328,
+    0x90EC: 329,
+    0x90F1: 330,
+    0x90F3: 331,
+    0x90F5: 332,
+    0x90F6: 333,
+    0x90F7: 334,
+    0x90FD: 335,
+    0x9141: 336,
+    0x9142: 337,
+    0x9145: 338,
+    0x9149: 339,
+    0x9151: 340,
+    0x9153: 341,
+    0x9155: 342,
+    0x9156: 343,
+    0x9157: 344,
+    0x9161: 345,
+    0x9162: 346,
+    0x9165: 347,
+    0x9169: 348,
+    0x9171: 349,
+    0x9173: 350,
+    0x9176: 351,
+    0x9177: 352,
+    0x917A: 353,
+    0x9181: 354,
+    0x9185: 355,
+    0x91A1: 356,
+    0x91A2: 357,
+    0x91A5: 358,
+    0x91A9: 359,
+    0x91AB: 360,
+    0x91B1: 361,
+    0x91B3: 362,
+    0x91B5: 363,
+    0x91B7: 364,
+    0x91BC: 365,
+    0x91BD: 366,
+    0x91C1: 367,
+    0x91C5: 368,
+    0x91C9: 369,
+    0x91D6: 370,
+    0x9241: 371,
+    0x9245: 372,
+    0x9249: 373,
+    0x9251: 374,
+    0x9253: 375,
+    0x9255: 376,
+    0x9261: 377,
+    0x9262: 378,
+    0x9265: 379,
+    0x9269: 380,
+    0x9273: 381,
+    0x9275: 382,
+    0x9277: 383,
+    0x9281: 384,
+    0x9282: 385,
+    0x9285: 386,
+    0x9288: 387,
+    0x9289: 388,
+    0x9291: 389,
+    0x9293: 390,
+    0x9295: 391,
+    0x9297: 392,
+    0x92A1: 393,
+    0x92B6: 394,
+    0x92C1: 395,
+    0x92E1: 396,
+    0x92E5: 397,
+    0x92E9: 398,
+    0x92F1: 399,
+    0x92F3: 400,
+    0x9341: 401,
+    0x9342: 402,
+    0x9349: 403,
+    0x9351: 404,
+    0x9353: 405,
+    0x9357: 406,
+    0x9361: 407,
+    0x9362: 408,
+    0x9365: 409,
+    0x9369: 410,
+    0x936A: 411,
+    0x936B: 412,
+    0x9371: 413,
+    0x9373: 414,
+    0x9375: 415,
+    0x9377: 416,
+    0x9378: 417,
+    0x937C: 418,
+    0x9381: 419,
+    0x9385: 420,
+    0x9389: 421,
+    0x93A1: 422,
+    0x93A2: 423,
+    0x93A5: 424,
+    0x93A9: 425,
+    0x93AB: 426,
+    0x93B1: 427,
+    0x93B3: 428,
+    0x93B5: 429,
+    0x93B7: 430,
+    0x93BC: 431,
+    0x9461: 432,
+    0x9462: 433,
+    0x9463: 434,
+    0x9465: 435,
+    0x9468: 436,
+    0x9469: 437,
+    0x946A: 438,
+    0x946B: 439,
+    0x946C: 440,
+    0x9470: 441,
+    0x9471: 442,
+    0x9473: 443,
+    0x9475: 444,
+    0x9476: 445,
+    0x9477: 446,
+    0x9478: 447,
+    0x9479: 448,
+    0x947D: 449,
+    0x9481: 450,
+    0x9482: 451,
+    0x9485: 452,
+    0x9489: 453,
+    0x9491: 454,
+    0x9493: 455,
+    0x9495: 456,
+    0x9496: 457,
+    0x9497: 458,
+    0x94A1: 459,
+    0x94E1: 460,
+    0x94E2: 461,
+    0x94E3: 462,
+    0x94E5: 463,
+    0x94E8: 464,
+    0x94E9: 465,
+    0x94EB: 466,
+    0x94EC: 467,
+    0x94F1: 468,
+    0x94F3: 469,
+    0x94F5: 470,
+    0x94F7: 471,
+    0x94F9: 472,
+    0x94FC: 473,
+    0x9541: 474,
+    0x9542: 475,
+    0x9545: 476,
+    0x9549: 477,
+    0x9551: 478,
+    0x9553: 479,
+    0x9555: 480,
+    0x9556: 481,
+    0x9557: 482,
+    0x9561: 483,
+    0x9565: 484,
+    0x9569: 485,
+    0x9576: 486,
+    0x9577: 487,
+    0x9581: 488,
+    0x9585: 489,
+    0x95A1: 490,
+    0x95A2: 491,
+    0x95A5: 492,
+    0x95A8: 493,
+    0x95A9: 494,
+    0x95AB: 495,
+    0x95AD: 496,
+    0x95B1: 497,
+    0x95B3: 498,
+    0x95B5: 499,
+    0x95B7: 500,
+    0x95B9: 501,
+    0x95BB: 502,
+    0x95C1: 503,
+    0x95C5: 504,
+    0x95C9: 505,
+    0x95E1: 506,
+    0x95F6: 507,
+    0x9641: 508,
+    0x9645: 509,
+    0x9649: 510,
+    0x9651: 511,
+    0x9653: 512,
+    0x9655: 513,
+    0x9661: 514,
+    0x9681: 515,
+    0x9682: 516,
+    0x9685: 517,
+    0x9689: 518,
+    0x9691: 519,
+    0x9693: 520,
+    0x9695: 521,
+    0x9697: 522,
+    0x96A1: 523,
+    0x96B6: 524,
+    0x96C1: 525,
+    0x96D7: 526,
+    0x96E1: 527,
+    0x96E5: 528,
+    0x96E9: 529,
+    0x96F3: 530,
+    0x96F5: 531,
+    0x96F7: 532,
+    0x9741: 533,
+    0x9745: 534,
+    0x9749: 535,
+    0x9751: 536,
+    0x9757: 537,
+    0x9761: 538,
+    0x9762: 539,
+    0x9765: 540,
+    0x9768: 541,
+    0x9769: 542,
+    0x976B: 543,
+    0x9771: 544,
+    0x9773: 545,
+    0x9775: 546,
+    0x9777: 547,
+    0x9781: 548,
+    0x97A1: 549,
+    0x97A2: 550,
+    0x97A5: 551,
+    0x97A8: 552,
+    0x97A9: 553,
+    0x97B1: 554,
+    0x97B3: 555,
+    0x97B5: 556,
+    0x97B6: 557,
+    0x97B7: 558,
+    0x97B8: 559,
+    0x9861: 560,
+    0x9862: 561,
+    0x9865: 562,
+    0x9869: 563,
+    0x9871: 564,
+    0x9873: 565,
+    0x9875: 566,
+    0x9876: 567,
+    0x9877: 568,
+    0x987D: 569,
+    0x9881: 570,
+    0x9882: 571,
+    0x9885: 572,
+    0x9889: 573,
+    0x9891: 574,
+    0x9893: 575,
+    0x9895: 576,
+    0x9896: 577,
+    0x9897: 578,
+    0x98E1: 579,
+    0x98E2: 580,
+    0x98E5: 581,
+    0x98E9: 582,
+    0x98EB: 583,
+    0x98EC: 584,
+    0x98F1: 585,
+    0x98F3: 586,
+    0x98F5: 587,
+    0x98F6: 588,
+    0x98F7: 589,
+    0x98FD: 590,
+    0x9941: 591,
+    0x9942: 592,
+    0x9945: 593,
+    0x9949: 594,
+    0x9951: 595,
+    0x9953: 596,
+    0x9955: 597,
+    0x9956: 598,
+    0x9957: 599,
+    0x9961: 600,
+    0x9976: 601,
+    0x99A1: 602,
+    0x99A2: 603,
+    0x99A5: 604,
+    0x99A9: 605,
+    0x99B7: 606,
+    0x99C1: 607,
+    0x99C9: 608,
+    0x99E1: 609,
+    0x9A41: 610,
+    0x9A45: 611,
+    0x9A81: 612,
+    0x9A82: 613,
+    0x9A85: 614,
+    0x9A89: 615,
+    0x9A90: 616,
+    0x9A91: 617,
+    0x9A97: 618,
+    0x9AC1: 619,
+    0x9AE1: 620,
+    0x9AE5: 621,
+    0x9AE9: 622,
+    0x9AF1: 623,
+    0x9AF3: 624,
+    0x9AF7: 625,
+    0x9B61: 626,
+    0x9B62: 627,
+    0x9B65: 628,
+    0x9B68: 629,
+    0x9B69: 630,
+    0x9B71: 631,
+    0x9B73: 632,
+    0x9B75: 633,
+    0x9B81: 634,
+    0x9B85: 635,
+    0x9B89: 636,
+    0x9B91: 637,
+    0x9B93: 638,
+    0x9BA1: 639,
+    0x9BA5: 640,
+    0x9BA9: 641,
+    0x9BB1: 642,
+    0x9BB3: 643,
+    0x9BB5: 644,
+    0x9BB7: 645,
+    0x9C61: 646,
+    0x9C62: 647,
+    0x9C65: 648,
+    0x9C69: 649,
+    0x9C71: 650,
+    0x9C73: 651,
+    0x9C75: 652,
+    0x9C76: 653,
+    0x9C77: 654,
+    0x9C78: 655,
+    0x9C7C: 656,
+    0x9C7D: 657,
+    0x9C81: 658,
+    0x9C82: 659,
+    0x9C85: 660,
+    0x9C89: 661,
+    0x9C91: 662,
+    0x9C93: 663,
+    0x9C95: 664,
+    0x9C96: 665,
+    0x9C97: 666,
+    0x9CA1: 667,
+    0x9CA2: 668,
+    0x9CA5: 669,
+    0x9CB5: 670,
+    0x9CB7: 671,
+    0x9CE1: 672,
+    0x9CE2: 673,
+    0x9CE5: 674,
+    0x9CE9: 675,
+    0x9CF1: 676,
+    0x9CF3: 677,
+    0x9CF5: 678,
+    0x9CF6: 679,
+    0x9CF7: 680,
+    0x9CFD: 681,
+    0x9D41: 682,
+    0x9D42: 683,
+    0x9D45: 684,
+    0x9D49: 685,
+    0x9D51: 686,
+    0x9D53: 687,
+    0x9D55: 688,
+    0x9D57: 689,
+    0x9D61: 690,
+    0x9D62: 691,
+    0x9D65: 692,
+    0x9D69: 693,
+    0x9D71: 694,
+    0x9D73: 695,
+    0x9D75: 696,
+    0x9D76: 697,
+    0x9D77: 698,
+    0x9D81: 699,
+    0x9D85: 700,
+    0x9D93: 701,
+    0x9D95: 702,
+    0x9DA1: 703,
+    0x9DA2: 704,
+    0x9DA5: 705,
+    0x9DA9: 706,
+    0x9DB1: 707,
+    0x9DB3: 708,
+    0x9DB5: 709,
+    0x9DB7: 710,
+    0x9DC1: 711,
+    0x9DC5: 712,
+    0x9DD7: 713,
+    0x9DF6: 714,
+    0x9E41: 715,
+    0x9E45: 716,
+    0x9E49: 717,
+    0x9E51: 718,
+    0x9E53: 719,
+    0x9E55: 720,
+    0x9E57: 721,
+    0x9E61: 722,
+    0x9E65: 723,
+    0x9E69: 724,
+    0x9E73: 725,
+    0x9E75: 726,
+    0x9E77: 727,
+    0x9E81: 728,
+    0x9E82: 729,
+    0x9E85: 730,
+    0x9E89: 731,
+    0x9E91: 732,
+    0x9E93: 733,
+    0x9E95: 734,
+    0x9E97: 735,
+    0x9EA1: 736,
+    0x9EB6: 737,
+    0x9EC1: 738,
+    0x9EE1: 739,
+    0x9EE2: 740,
+    0x9EE5: 741,
+    0x9EE9: 742,
+    0x9EF1: 743,
+    0x9EF5: 744,
+    0x9EF7: 745,
+    0x9F41: 746,
+    0x9F42: 747,
+    0x9F45: 748,
+    0x9F49: 749,
+    0x9F51: 750,
+    0x9F53: 751,
+    0x9F55: 752,
+    0x9F57: 753,
+    0x9F61: 754,
+    0x9F62: 755,
+    0x9F65: 756,
+    0x9F69: 757,
+    0x9F71: 758,
+    0x9F73: 759,
+    0x9F75: 760,
+    0x9F77: 761,
+    0x9F78: 762,
+    0x9F7B: 763,
+    0x9F7C: 764,
+    0x9FA1: 765,
+    0x9FA2: 766,
+    0x9FA5: 767,
+    0x9FA9: 768,
+    0x9FB1: 769,
+    0x9FB3: 770,
+    0x9FB5: 771,
+    0x9FB7: 772,
+    0xA061: 773,
+    0xA062: 774,
+    0xA065: 775,
+    0xA067: 776,
+    0xA068: 777,
+    0xA069: 778,
+    0xA06A: 779,
+    0xA06B: 780,
+    0xA071: 781,
+    0xA073: 782,
+    0xA075: 783,
+    0xA077: 784,
+    0xA078: 785,
+    0xA07B: 786,
+    0xA07D: 787,
+    0xA081: 788,
+    0xA082: 789,
+    0xA085: 790,
+    0xA089: 791,
+    0xA091: 792,
+    0xA093: 793,
+    0xA095: 794,
+    0xA096: 795,
+    0xA097: 796,
+    0xA098: 797,
+    0xA0A1: 798,
+    0xA0A2: 799,
+    0xA0A9: 800,
+    0xA0B7: 801,
+    0xA0E1: 802,
+    0xA0E2: 803,
+    0xA0E5: 804,
+    0xA0E9: 805,
+    0xA0EB: 806,
+    0xA0F1: 807,
+    0xA0F3: 808,
+    0xA0F5: 809,
+    0xA0F7: 810,
+    0xA0F8: 811,
+    0xA0FD: 812,
+    0xA141: 813,
+    0xA142: 814,
+    0xA145: 815,
+    0xA149: 816,
+    0xA151: 817,
+    0xA153: 818,
+    0xA155: 819,
+    0xA156: 820,
+    0xA157: 821,
+    0xA161: 822,
+    0xA162: 823,
+    0xA165: 824,
+    0xA169: 825,
+    0xA175: 826,
+    0xA176: 827,
+    0xA177: 828,
+    0xA179: 829,
+    0xA181: 830,
+    0xA1A1: 831,
+    0xA1A2: 832,
+    0xA1A4: 833,
+    0xA1A5: 834,
+    0xA1A9: 835,
+    0xA1AB: 836,
+    0xA1B1: 837,
+    0xA1B3: 838,
+    0xA1B5: 839,
+    0xA1B7: 840,
+    0xA1C1: 841,
+    0xA1C5: 842,
+    0xA1D6: 843,
+    0xA1D7: 844,
+    0xA241: 845,
+    0xA245: 846,
+    0xA249: 847,
+    0xA253: 848,
+    0xA255: 849,
+    0xA257: 850,
+    0xA261: 851,
+    0xA265: 852,
+    0xA269: 853,
+    0xA273: 854,
+    0xA275: 855,
+    0xA281: 856,
+    0xA282: 857,
+    0xA283: 858,
+    0xA285: 859,
+    0xA288: 860,
+    0xA289: 861,
+    0xA28A: 862,
+    0xA28B: 863,
+    0xA291: 864,
+    0xA293: 865,
+    0xA295: 866,
+    0xA297: 867,
+    0xA29B: 868,
+    0xA29D: 869,
+    0xA2A1: 870,
+    0xA2A5: 871,
+    0xA2A9: 872,
+    0xA2B3: 873,
+    0xA2B5: 874,
+    0xA2C1: 875,
+    0xA2E1: 876,
+    0xA2E5: 877,
+    0xA2E9: 878,
+    0xA341: 879,
+    0xA345: 880,
+    0xA349: 881,
+    0xA351: 882,
+    0xA355: 883,
+    0xA361: 884,
+    0xA365: 885,
+    0xA369: 886,
+    0xA371: 887,
+    0xA375: 888,
+    0xA3A1: 889,
+    0xA3A2: 890,
+    0xA3A5: 891,
+    0xA3A8: 892,
+    0xA3A9: 893,
+    0xA3AB: 894,
+    0xA3B1: 895,
+    0xA3B3: 896,
+    0xA3B5: 897,
+    0xA3B6: 898,
+    0xA3B7: 899,
+    0xA3B9: 900,
+    0xA3BB: 901,
+    0xA461: 902,
+    0xA462: 903,
+    0xA463: 904,
+    0xA464: 905,
+    0xA465: 906,
+    0xA468: 907,
+    0xA469: 908,
+    0xA46A: 909,
+    0xA46B: 910,
+    0xA46C: 911,
+    0xA471: 912,
+    0xA473: 913,
+    0xA475: 914,
+    0xA477: 915,
+    0xA47B: 916,
+    0xA481: 917,
+    0xA482: 918,
+    0xA485: 919,
+    0xA489: 920,
+    0xA491: 921,
+    0xA493: 922,
+    0xA495: 923,
+    0xA496: 924,
+    0xA497: 925,
+    0xA49B: 926,
+    0xA4A1: 927,
+    0xA4A2: 928,
+    0xA4A5: 929,
+    0xA4B3: 930,
+    0xA4E1: 931,
+    0xA4E2: 932,
+    0xA4E5: 933,
+    0xA4E8: 934,
+    0xA4E9: 935,
+    0xA4EB: 936,
+    0xA4F1: 937,
+    0xA4F3: 938,
+    0xA4F5: 939,
+    0xA4F7: 940,
+    0xA4F8: 941,
+    0xA541: 942,
+    0xA542: 943,
+    0xA545: 944,
+    0xA548: 945,
+    0xA549: 946,
+    0xA551: 947,
+    0xA553: 948,
+    0xA555: 949,
+    0xA556: 950,
+    0xA557: 951,
+    0xA561: 952,
+    0xA562: 953,
+    0xA565: 954,
+    0xA569: 955,
+    0xA573: 956,
+    0xA575: 957,
+    0xA576: 958,
+    0xA577: 959,
+    0xA57B: 960,
+    0xA581: 961,
+    0xA585: 962,
+    0xA5A1: 963,
+    0xA5A2: 964,
+    0xA5A3: 965,
+    0xA5A5: 966,
+    0xA5A9: 967,
+    0xA5B1: 968,
+    0xA5B3: 969,
+    0xA5B5: 970,
+    0xA5B7: 971,
+    0xA5C1: 972,
+    0xA5C5: 973,
+    0xA5D6: 974,
+    0xA5E1: 975,
+    0xA5F6: 976,
+    0xA641: 977,
+    0xA642: 978,
+    0xA645: 979,
+    0xA649: 980,
+    0xA651: 981,
+    0xA653: 982,
+    0xA661: 983,
+    0xA665: 984,
+    0xA681: 985,
+    0xA682: 986,
+    0xA685: 987,
+    0xA688: 988,
+    0xA689: 989,
+    0xA68A: 990,
+    0xA68B: 991,
+    0xA691: 992,
+    0xA693: 993,
+    0xA695: 994,
+    0xA697: 995,
+    0xA69B: 996,
+    0xA69C: 997,
+    0xA6A1: 998,
+    0xA6A9: 999,
+    0xA6B6: 1000,
+    0xA6C1: 1001,
+    0xA6E1: 1002,
+    0xA6E2: 1003,
+    0xA6E5: 1004,
+    0xA6E9: 1005,
+    0xA6F7: 1006,
+    0xA741: 1007,
+    0xA745: 1008,
+    0xA749: 1009,
+    0xA751: 1010,
+    0xA755: 1011,
+    0xA757: 1012,
+    0xA761: 1013,
+    0xA762: 1014,
+    0xA765: 1015,
+    0xA769: 1016,
+    0xA771: 1017,
+    0xA773: 1018,
+    0xA775: 1019,
+    0xA7A1: 1020,
+    0xA7A2: 1021,
+    0xA7A5: 1022,
+    0xA7A9: 1023,
+    0xA7AB: 1024,
+    0xA7B1: 1025,
+    0xA7B3: 1026,
+    0xA7B5: 1027,
+    0xA7B7: 1028,
+    0xA7B8: 1029,
+    0xA7B9: 1030,
+    0xA861: 1031,
+    0xA862: 1032,
+    0xA865: 1033,
+    0xA869: 1034,
+    0xA86B: 1035,
+    0xA871: 1036,
+    0xA873: 1037,
+    0xA875: 1038,
+    0xA876: 1039,
+    0xA877: 1040,
+    0xA87D: 1041,
+    0xA881: 1042,
+    0xA882: 1043,
+    0xA885: 1044,
+    0xA889: 1045,
+    0xA891: 1046,
+    0xA893: 1047,
+    0xA895: 1048,
+    0xA896: 1049,
+    0xA897: 1050,
+    0xA8A1: 1051,
+    0xA8A2: 1052,
+    0xA8B1: 1053,
+    0xA8E1: 1054,
+    0xA8E2: 1055,
+    0xA8E5: 1056,
+    0xA8E8: 1057,
+    0xA8E9: 1058,
+    0xA8F1: 1059,
+    0xA8F5: 1060,
+    0xA8F6: 1061,
+    0xA8F7: 1062,
+    0xA941: 1063,
+    0xA957: 1064,
+    0xA961: 1065,
+    0xA962: 1066,
+    0xA971: 1067,
+    0xA973: 1068,
+    0xA975: 1069,
+    0xA976: 1070,
+    0xA977: 1071,
+    0xA9A1: 1072,
+    0xA9A2: 1073,
+    0xA9A5: 1074,
+    0xA9A9: 1075,
+    0xA9B1: 1076,
+    0xA9B3: 1077,
+    0xA9B7: 1078,
+    0xAA41: 1079,
+    0xAA61: 1080,
+    0xAA77: 1081,
+    0xAA81: 1082,
+    0xAA82: 1083,
+    0xAA85: 1084,
+    0xAA89: 1085,
+    0xAA91: 1086,
+    0xAA95: 1087,
+    0xAA97: 1088,
+    0xAB41: 1089,
+    0xAB57: 1090,
+    0xAB61: 1091,
+    0xAB65: 1092,
+    0xAB69: 1093,
+    0xAB71: 1094,
+    0xAB73: 1095,
+    0xABA1: 1096,
+    0xABA2: 1097,
+    0xABA5: 1098,
+    0xABA9: 1099,
+    0xABB1: 1100,
+    0xABB3: 1101,
+    0xABB5: 1102,
+    0xABB7: 1103,
+    0xAC61: 1104,
+    0xAC62: 1105,
+    0xAC64: 1106,
+    0xAC65: 1107,
+    0xAC68: 1108,
+    0xAC69: 1109,
+    0xAC6A: 1110,
+    0xAC6B: 1111,
+    0xAC71: 1112,
+    0xAC73: 1113,
+    0xAC75: 1114,
+    0xAC76: 1115,
+    0xAC77: 1116,
+    0xAC7B: 1117,
+    0xAC81: 1118,
+    0xAC82: 1119,
+    0xAC85: 1120,
+    0xAC89: 1121,
+    0xAC91: 1122,
+    0xAC93: 1123,
+    0xAC95: 1124,
+    0xAC96: 1125,
+    0xAC97: 1126,
+    0xACA1: 1127,
+    0xACA2: 1128,
+    0xACA5: 1129,
+    0xACA9: 1130,
+    0xACB1: 1131,
+    0xACB3: 1132,
+    0xACB5: 1133,
+    0xACB7: 1134,
+    0xACC1: 1135,
+    0xACC5: 1136,
+    0xACC9: 1137,
+    0xACD1: 1138,
+    0xACD7: 1139,
+    0xACE1: 1140,
+    0xACE2: 1141,
+    0xACE3: 1142,
+    0xACE4: 1143,
+    0xACE5: 1144,
+    0xACE8: 1145,
+    0xACE9: 1146,
+    0xACEB: 1147,
+    0xACEC: 1148,
+    0xACF1: 1149,
+    0xACF3: 1150,
+    0xACF5: 1151,
+    0xACF6: 1152,
+    0xACF7: 1153,
+    0xACFC: 1154,
+    0xAD41: 1155,
+    0xAD42: 1156,
+    0xAD45: 1157,
+    0xAD49: 1158,
+    0xAD51: 1159,
+    0xAD53: 1160,
+    0xAD55: 1161,
+    0xAD56: 1162,
+    0xAD57: 1163,
+    0xAD61: 1164,
+    0xAD62: 1165,
+    0xAD65: 1166,
+    0xAD69: 1167,
+    0xAD71: 1168,
+    0xAD73: 1169,
+    0xAD75: 1170,
+    0xAD76: 1171,
+    0xAD77: 1172,
+    0xAD81: 1173,
+    0xAD85: 1174,
+    0xAD89: 1175,
+    0xAD97: 1176,
+    0xADA1: 1177,
+    0xADA2: 1178,
+    0xADA3: 1179,
+    0xADA5: 1180,
+    0xADA9: 1181,
+    0xADAB: 1182,
+    0xADB1: 1183,
+    0xADB3: 1184,
+    0xADB5: 1185,
+    0xADB7: 1186,
+    0xADBB: 1187,
+    0xADC1: 1188,
+    0xADC2: 1189,
+    0xADC5: 1190,
+    0xADC9: 1191,
+    0xADD7: 1192,
+    0xADE1: 1193,
+    0xADE5: 1194,
+    0xADE9: 1195,
+    0xADF1: 1196,
+    0xADF5: 1197,
+    0xADF6: 1198,
+    0xAE41: 1199,
+    0xAE45: 1200,
+    0xAE49: 1201,
+    0xAE51: 1202,
+    0xAE53: 1203,
+    0xAE55: 1204,
+    0xAE61: 1205,
+    0xAE62: 1206,
+    0xAE65: 1207,
+    0xAE69: 1208,
+    0xAE71: 1209,
+    0xAE73: 1210,
+    0xAE75: 1211,
+    0xAE77: 1212,
+    0xAE81: 1213,
+    0xAE82: 1214,
+    0xAE85: 1215,
+    0xAE88: 1216,
+    0xAE89: 1217,
+    0xAE91: 1218,
+    0xAE93: 1219,
+    0xAE95: 1220,
+    0xAE97: 1221,
+    0xAE99: 1222,
+    0xAE9B: 1223,
+    0xAE9C: 1224,
+    0xAEA1: 1225,
+    0xAEB6: 1226,
+    0xAEC1: 1227,
+    0xAEC2: 1228,
+    0xAEC5: 1229,
+    0xAEC9: 1230,
+    0xAED1: 1231,
+    0xAED7: 1232,
+    0xAEE1: 1233,
+    0xAEE2: 1234,
+    0xAEE5: 1235,
+    0xAEE9: 1236,
+    0xAEF1: 1237,
+    0xAEF3: 1238,
+    0xAEF5: 1239,
+    0xAEF7: 1240,
+    0xAF41: 1241,
+    0xAF42: 1242,
+    0xAF49: 1243,
+    0xAF51: 1244,
+    0xAF55: 1245,
+    0xAF57: 1246,
+    0xAF61: 1247,
+    0xAF62: 1248,
+    0xAF65: 1249,
+    0xAF69: 1250,
+    0xAF6A: 1251,
+    0xAF71: 1252,
+    0xAF73: 1253,
+    0xAF75: 1254,
+    0xAF77: 1255,
+    0xAFA1: 1256,
+    0xAFA2: 1257,
+    0xAFA5: 1258,
+    0xAFA8: 1259,
+    0xAFA9: 1260,
+    0xAFB0: 1261,
+    0xAFB1: 1262,
+    0xAFB3: 1263,
+    0xAFB5: 1264,
+    0xAFB7: 1265,
+    0xAFBC: 1266,
+    0xB061: 1267,
+    0xB062: 1268,
+    0xB064: 1269,
+    0xB065: 1270,
+    0xB069: 1271,
+    0xB071: 1272,
+    0xB073: 1273,
+    0xB076: 1274,
+    0xB077: 1275,
+    0xB07D: 1276,
+    0xB081: 1277,
+    0xB082: 1278,
+    0xB085: 1279,
+    0xB089: 1280,
+    0xB091: 1281,
+    0xB093: 1282,
+    0xB096: 1283,
+    0xB097: 1284,
+    0xB0B7: 1285,
+    0xB0E1: 1286,
+    0xB0E2: 1287,
+    0xB0E5: 1288,
+    0xB0E9: 1289,
+    0xB0EB: 1290,
+    0xB0F1: 1291,
+    0xB0F3: 1292,
+    0xB0F6: 1293,
+    0xB0F7: 1294,
+    0xB141: 1295,
+    0xB145: 1296,
+    0xB149: 1297,
+    0xB185: 1298,
+    0xB1A1: 1299,
+    0xB1A2: 1300,
+    0xB1A5: 1301,
+    0xB1A8: 1302,
+    0xB1A9: 1303,
+    0xB1AB: 1304,
+    0xB1B1: 1305,
+    0xB1B3: 1306,
+    0xB1B7: 1307,
+    0xB1C1: 1308,
+    0xB1C2: 1309,
+    0xB1C5: 1310,
+    0xB1D6: 1311,
+    0xB1E1: 1312,
+    0xB1F6: 1313,
+    0xB241: 1314,
+    0xB245: 1315,
+    0xB249: 1316,
+    0xB251: 1317,
+    0xB253: 1318,
+    0xB261: 1319,
+    0xB281: 1320,
+    0xB282: 1321,
+    0xB285: 1322,
+    0xB289: 1323,
+    0xB291: 1324,
+    0xB293: 1325,
+    0xB297: 1326,
+    0xB2A1: 1327,
+    0xB2B6: 1328,
+    0xB2C1: 1329,
+    0xB2E1: 1330,
+    0xB2E5: 1331,
+    0xB357: 1332,
+    0xB361: 1333,
+    0xB362: 1334,
+    0xB365: 1335,
+    0xB369: 1336,
+    0xB36B: 1337,
+    0xB370: 1338,
+    0xB371: 1339,
+    0xB373: 1340,
+    0xB381: 1341,
+    0xB385: 1342,
+    0xB389: 1343,
+    0xB391: 1344,
+    0xB3A1: 1345,
+    0xB3A2: 1346,
+    0xB3A5: 1347,
+    0xB3A9: 1348,
+    0xB3B1: 1349,
+    0xB3B3: 1350,
+    0xB3B5: 1351,
+    0xB3B7: 1352,
+    0xB461: 1353,
+    0xB462: 1354,
+    0xB465: 1355,
+    0xB466: 1356,
+    0xB467: 1357,
+    0xB469: 1358,
+    0xB46A: 1359,
+    0xB46B: 1360,
+    0xB470: 1361,
+    0xB471: 1362,
+    0xB473: 1363,
+    0xB475: 1364,
+    0xB476: 1365,
+    0xB477: 1366,
+    0xB47B: 1367,
+    0xB47C: 1368,
+    0xB481: 1369,
+    0xB482: 1370,
+    0xB485: 1371,
+    0xB489: 1372,
+    0xB491: 1373,
+    0xB493: 1374,
+    0xB495: 1375,
+    0xB496: 1376,
+    0xB497: 1377,
+    0xB4A1: 1378,
+    0xB4A2: 1379,
+    0xB4A5: 1380,
+    0xB4A9: 1381,
+    0xB4AC: 1382,
+    0xB4B1: 1383,
+    0xB4B3: 1384,
+    0xB4B5: 1385,
+    0xB4B7: 1386,
+    0xB4BB: 1387,
+    0xB4BD: 1388,
+    0xB4C1: 1389,
+    0xB4C5: 1390,
+    0xB4C9: 1391,
+    0xB4D3: 1392,
+    0xB4E1: 1393,
+    0xB4E2: 1394,
+    0xB4E5: 1395,
+    0xB4E6: 1396,
+    0xB4E8: 1397,
+    0xB4E9: 1398,
+    0xB4EA: 1399,
+    0xB4EB: 1400,
+    0xB4F1: 1401,
+    0xB4F3: 1402,
+    0xB4F4: 1403,
+    0xB4F5: 1404,
+    0xB4F6: 1405,
+    0xB4F7: 1406,
+    0xB4F8: 1407,
+    0xB4FA: 1408,
+    0xB4FC: 1409,
+    0xB541: 1410,
+    0xB542: 1411,
+    0xB545: 1412,
+    0xB549: 1413,
+    0xB551: 1414,
+    0xB553: 1415,
+    0xB555: 1416,
+    0xB557: 1417,
+    0xB561: 1418,
+    0xB562: 1419,
+    0xB563: 1420,
+    0xB565: 1421,
+    0xB569: 1422,
+    0xB56B: 1423,
+    0xB56C: 1424,
+    0xB571: 1425,
+    0xB573: 1426,
+    0xB574: 1427,
+    0xB575: 1428,
+    0xB576: 1429,
+    0xB577: 1430,
+    0xB57B: 1431,
+    0xB57C: 1432,
+    0xB57D: 1433,
+    0xB581: 1434,
+    0xB585: 1435,
+    0xB589: 1436,
+    0xB591: 1437,
+    0xB593: 1438,
+    0xB595: 1439,
+    0xB596: 1440,
+    0xB5A1: 1441,
+    0xB5A2: 1442,
+    0xB5A5: 1443,
+    0xB5A9: 1444,
+    0xB5AA: 1445,
+    0xB5AB: 1446,
+    0xB5AD: 1447,
+    0xB5B0: 1448,
+    0xB5B1: 1449,
+    0xB5B3: 1450,
+    0xB5B5: 1451,
+    0xB5B7: 1452,
+    0xB5B9: 1453,
+    0xB5C1: 1454,
+    0xB5C2: 1455,
+    0xB5C5: 1456,
+    0xB5C9: 1457,
+    0xB5D1: 1458,
+    0xB5D3: 1459,
+    0xB5D5: 1460,
+    0xB5D6: 1461,
+    0xB5D7: 1462,
+    0xB5E1: 1463,
+    0xB5E2: 1464,
+    0xB5E5: 1465,
+    0xB5F1: 1466,
+    0xB5F5: 1467,
+    0xB5F7: 1468,
+    0xB641: 1469,
+    0xB642: 1470,
+    0xB645: 1471,
+    0xB649: 1472,
+    0xB651: 1473,
+    0xB653: 1474,
+    0xB655: 1475,
+    0xB657: 1476,
+    0xB661: 1477,
+    0xB662: 1478,
+    0xB665: 1479,
+    0xB669: 1480,
+    0xB671: 1481,
+    0xB673: 1482,
+    0xB675: 1483,
+    0xB677: 1484,
+    0xB681: 1485,
+    0xB682: 1486,
+    0xB685: 1487,
+    0xB689: 1488,
+    0xB68A: 1489,
+    0xB68B: 1490,
+    0xB691: 1491,
+    0xB693: 1492,
+    0xB695: 1493,
+    0xB697: 1494,
+    0xB6A1: 1495,
+    0xB6A2: 1496,
+    0xB6A5: 1497,
+    0xB6A9: 1498,
+    0xB6B1: 1499,
+    0xB6B3: 1500,
+    0xB6B6: 1501,
+    0xB6B7: 1502,
+    0xB6C1: 1503,
+    0xB6C2: 1504,
+    0xB6C5: 1505,
+    0xB6C9: 1506,
+    0xB6D1: 1507,
+    0xB6D3: 1508,
+    0xB6D7: 1509,
+    0xB6E1: 1510,
+    0xB6E2: 1511,
+    0xB6E5: 1512,
+    0xB6E9: 1513,
+    0xB6F1: 1514,
+    0xB6F3: 1515,
+    0xB6F5: 1516,
+    0xB6F7: 1517,
+    0xB741: 1518,
+    0xB742: 1519,
+    0xB745: 1520,
+    0xB749: 1521,
+    0xB751: 1522,
+    0xB753: 1523,
+    0xB755: 1524,
+    0xB757: 1525,
+    0xB759: 1526,
+    0xB761: 1527,
+    0xB762: 1528,
+    0xB765: 1529,
+    0xB769: 1530,
+    0xB76F: 1531,
+    0xB771: 1532,
+    0xB773: 1533,
+    0xB775: 1534,
+    0xB777: 1535,
+    0xB778: 1536,
+    0xB779: 1537,
+    0xB77A: 1538,
+    0xB77B: 1539,
+    0xB77C: 1540,
+    0xB77D: 1541,
+    0xB781: 1542,
+    0xB785: 1543,
+    0xB789: 1544,
+    0xB791: 1545,
+    0xB795: 1546,
+    0xB7A1: 1547,
+    0xB7A2: 1548,
+    0xB7A5: 1549,
+    0xB7A9: 1550,
+    0xB7AA: 1551,
+    0xB7AB: 1552,
+    0xB7B0: 1553,
+    0xB7B1: 1554,
+    0xB7B3: 1555,
+    0xB7B5: 1556,
+    0xB7B6: 1557,
+    0xB7B7: 1558,
+    0xB7B8: 1559,
+    0xB7BC: 1560,
+    0xB861: 1561,
+    0xB862: 1562,
+    0xB865: 1563,
+    0xB867: 1564,
+    0xB868: 1565,
+    0xB869: 1566,
+    0xB86B: 1567,
+    0xB871: 1568,
+    0xB873: 1569,
+    0xB875: 1570,
+    0xB876: 1571,
+    0xB877: 1572,
+    0xB878: 1573,
+    0xB881: 1574,
+    0xB882: 1575,
+    0xB885: 1576,
+    0xB889: 1577,
+    0xB891: 1578,
+    0xB893: 1579,
+    0xB895: 1580,
+    0xB896: 1581,
+    0xB897: 1582,
+    0xB8A1: 1583,
+    0xB8A2: 1584,
+    0xB8A5: 1585,
+    0xB8A7: 1586,
+    0xB8A9: 1587,
+    0xB8B1: 1588,
+    0xB8B7: 1589,
+    0xB8C1: 1590,
+    0xB8C5: 1591,
+    0xB8C9: 1592,
+    0xB8E1: 1593,
+    0xB8E2: 1594,
+    0xB8E5: 1595,
+    0xB8E9: 1596,
+    0xB8EB: 1597,
+    0xB8F1: 1598,
+    0xB8F3: 1599,
+    0xB8F5: 1600,
+    0xB8F7: 1601,
+    0xB8F8: 1602,
+    0xB941: 1603,
+    0xB942: 1604,
+    0xB945: 1605,
+    0xB949: 1606,
+    0xB951: 1607,
+    0xB953: 1608,
+    0xB955: 1609,
+    0xB957: 1610,
+    0xB961: 1611,
+    0xB965: 1612,
+    0xB969: 1613,
+    0xB971: 1614,
+    0xB973: 1615,
+    0xB976: 1616,
+    0xB977: 1617,
+    0xB981: 1618,
+    0xB9A1: 1619,
+    0xB9A2: 1620,
+    0xB9A5: 1621,
+    0xB9A9: 1622,
+    0xB9AB: 1623,
+    0xB9B1: 1624,
+    0xB9B3: 1625,
+    0xB9B5: 1626,
+    0xB9B7: 1627,
+    0xB9B8: 1628,
+    0xB9B9: 1629,
+    0xB9BD: 1630,
+    0xB9C1: 1631,
+    0xB9C2: 1632,
+    0xB9C9: 1633,
+    0xB9D3: 1634,
+    0xB9D5: 1635,
+    0xB9D7: 1636,
+    0xB9E1: 1637,
+    0xB9F6: 1638,
+    0xB9F7: 1639,
+    0xBA41: 1640,
+    0xBA45: 1641,
+    0xBA49: 1642,
+    0xBA51: 1643,
+    0xBA53: 1644,
+    0xBA55: 1645,
+    0xBA57: 1646,
+    0xBA61: 1647,
+    0xBA62: 1648,
+    0xBA65: 1649,
+    0xBA77: 1650,
+    0xBA81: 1651,
+    0xBA82: 1652,
+    0xBA85: 1653,
+    0xBA89: 1654,
+    0xBA8A: 1655,
+    0xBA8B: 1656,
+    0xBA91: 1657,
+    0xBA93: 1658,
+    0xBA95: 1659,
+    0xBA97: 1660,
+    0xBAA1: 1661,
+    0xBAB6: 1662,
+    0xBAC1: 1663,
+    0xBAE1: 1664,
+    0xBAE2: 1665,
+    0xBAE5: 1666,
+    0xBAE9: 1667,
+    0xBAF1: 1668,
+    0xBAF3: 1669,
+    0xBAF5: 1670,
+    0xBB41: 1671,
+    0xBB45: 1672,
+    0xBB49: 1673,
+    0xBB51: 1674,
+    0xBB61: 1675,
+    0xBB62: 1676,
+    0xBB65: 1677,
+    0xBB69: 1678,
+    0xBB71: 1679,
+    0xBB73: 1680,
+    0xBB75: 1681,
+    0xBB77: 1682,
+    0xBBA1: 1683,
+    0xBBA2: 1684,
+    0xBBA5: 1685,
+    0xBBA8: 1686,
+    0xBBA9: 1687,
+    0xBBAB: 1688,
+    0xBBB1: 1689,
+    0xBBB3: 1690,
+    0xBBB5: 1691,
+    0xBBB7: 1692,
+    0xBBB8: 1693,
+    0xBBBB: 1694,
+    0xBBBC: 1695,
+    0xBC61: 1696,
+    0xBC62: 1697,
+    0xBC65: 1698,
+    0xBC67: 1699,
+    0xBC69: 1700,
+    0xBC6C: 1701,
+    0xBC71: 1702,
+    0xBC73: 1703,
+    0xBC75: 1704,
+    0xBC76: 1705,
+    0xBC77: 1706,
+    0xBC81: 1707,
+    0xBC82: 1708,
+    0xBC85: 1709,
+    0xBC89: 1710,
+    0xBC91: 1711,
+    0xBC93: 1712,
+    0xBC95: 1713,
+    0xBC96: 1714,
+    0xBC97: 1715,
+    0xBCA1: 1716,
+    0xBCA5: 1717,
+    0xBCB7: 1718,
+    0xBCE1: 1719,
+    0xBCE2: 1720,
+    0xBCE5: 1721,
+    0xBCE9: 1722,
+    0xBCF1: 1723,
+    0xBCF3: 1724,
+    0xBCF5: 1725,
+    0xBCF6: 1726,
+    0xBCF7: 1727,
+    0xBD41: 1728,
+    0xBD57: 1729,
+    0xBD61: 1730,
+    0xBD76: 1731,
+    0xBDA1: 1732,
+    0xBDA2: 1733,
+    0xBDA5: 1734,
+    0xBDA9: 1735,
+    0xBDB1: 1736,
+    0xBDB3: 1737,
+    0xBDB5: 1738,
+    0xBDB7: 1739,
+    0xBDB9: 1740,
+    0xBDC1: 1741,
+    0xBDC2: 1742,
+    0xBDC9: 1743,
+    0xBDD6: 1744,
+    0xBDE1: 1745,
+    0xBDF6: 1746,
+    0xBE41: 1747,
+    0xBE45: 1748,
+    0xBE49: 1749,
+    0xBE51: 1750,
+    0xBE53: 1751,
+    0xBE77: 1752,
+    0xBE81: 1753,
+    0xBE82: 1754,
+    0xBE85: 1755,
+    0xBE89: 1756,
+    0xBE91: 1757,
+    0xBE93: 1758,
+    0xBE97: 1759,
+    0xBEA1: 1760,
+    0xBEB6: 1761,
+    0xBEB7: 1762,
+    0xBEE1: 1763,
+    0xBF41: 1764,
+    0xBF61: 1765,
+    0xBF71: 1766,
+    0xBF75: 1767,
+    0xBF77: 1768,
+    0xBFA1: 1769,
+    0xBFA2: 1770,
+    0xBFA5: 1771,
+    0xBFA9: 1772,
+    0xBFB1: 1773,
+    0xBFB3: 1774,
+    0xBFB7: 1775,
+    0xBFB8: 1776,
+    0xBFBD: 1777,
+    0xC061: 1778,
+    0xC062: 1779,
+    0xC065: 1780,
+    0xC067: 1781,
+    0xC069: 1782,
+    0xC071: 1783,
+    0xC073: 1784,
+    0xC075: 1785,
+    0xC076: 1786,
+    0xC077: 1787,
+    0xC078: 1788,
+    0xC081: 1789,
+    0xC082: 1790,
+    0xC085: 1791,
+    0xC089: 1792,
+    0xC091: 1793,
+    0xC093: 1794,
+    0xC095: 1795,
+    0xC096: 1796,
+    0xC097: 1797,
+    0xC0A1: 1798,
+    0xC0A5: 1799,
+    0xC0A7: 1800,
+    0xC0A9: 1801,
+    0xC0B1: 1802,
+    0xC0B7: 1803,
+    0xC0E1: 1804,
+    0xC0E2: 1805,
+    0xC0E5: 1806,
+    0xC0E9: 1807,
+    0xC0F1: 1808,
+    0xC0F3: 1809,
+    0xC0F5: 1810,
+    0xC0F6: 1811,
+    0xC0F7: 1812,
+    0xC141: 1813,
+    0xC142: 1814,
+    0xC145: 1815,
+    0xC149: 1816,
+    0xC151: 1817,
+    0xC153: 1818,
+    0xC155: 1819,
+    0xC157: 1820,
+    0xC161: 1821,
+    0xC165: 1822,
+    0xC176: 1823,
+    0xC181: 1824,
+    0xC185: 1825,
+    0xC197: 1826,
+    0xC1A1: 1827,
+    0xC1A2: 1828,
+    0xC1A5: 1829,
+    0xC1A9: 1830,
+    0xC1B1: 1831,
+    0xC1B3: 1832,
+    0xC1B5: 1833,
+    0xC1B7: 1834,
+    0xC1C1: 1835,
+    0xC1C5: 1836,
+    0xC1C9: 1837,
+    0xC1D7: 1838,
+    0xC241: 1839,
+    0xC245: 1840,
+    0xC249: 1841,
+    0xC251: 1842,
+    0xC253: 1843,
+    0xC255: 1844,
+    0xC257: 1845,
+    0xC261: 1846,
+    0xC271: 1847,
+    0xC281: 1848,
+    0xC282: 1849,
+    0xC285: 1850,
+    0xC289: 1851,
+    0xC291: 1852,
+    0xC293: 1853,
+    0xC295: 1854,
+    0xC297: 1855,
+    0xC2A1: 1856,
+    0xC2B6: 1857,
+    0xC2C1: 1858,
+    0xC2C5: 1859,
+    0xC2E1: 1860,
+    0xC2E5: 1861,
+    0xC2E9: 1862,
+    0xC2F1: 1863,
+    0xC2F3: 1864,
+    0xC2F5: 1865,
+    0xC2F7: 1866,
+    0xC341: 1867,
+    0xC345: 1868,
+    0xC349: 1869,
+    0xC351: 1870,
+    0xC357: 1871,
+    0xC361: 1872,
+    0xC362: 1873,
+    0xC365: 1874,
+    0xC369: 1875,
+    0xC371: 1876,
+    0xC373: 1877,
+    0xC375: 1878,
+    0xC377: 1879,
+    0xC3A1: 1880,
+    0xC3A2: 1881,
+    0xC3A5: 1882,
+    0xC3A8: 1883,
+    0xC3A9: 1884,
+    0xC3AA: 1885,
+    0xC3B1: 1886,
+    0xC3B3: 1887,
+    0xC3B5: 1888,
+    0xC3B7: 1889,
+    0xC461: 1890,
+    0xC462: 1891,
+    0xC465: 1892,
+    0xC469: 1893,
+    0xC471: 1894,
+    0xC473: 1895,
+    0xC475: 1896,
+    0xC477: 1897,
+    0xC481: 1898,
+    0xC482: 1899,
+    0xC485: 1900,
+    0xC489: 1901,
+    0xC491: 1902,
+    0xC493: 1903,
+    0xC495: 1904,
+    0xC496: 1905,
+    0xC497: 1906,
+    0xC4A1: 1907,
+    0xC4A2: 1908,
+    0xC4B7: 1909,
+    0xC4E1: 1910,
+    0xC4E2: 1911,
+    0xC4E5: 1912,
+    0xC4E8: 1913,
+    0xC4E9: 1914,
+    0xC4F1: 1915,
+    0xC4F3: 1916,
+    0xC4F5: 1917,
+    0xC4F6: 1918,
+    0xC4F7: 1919,
+    0xC541: 1920,
+    0xC542: 1921,
+    0xC545: 1922,
+    0xC549: 1923,
+    0xC551: 1924,
+    0xC553: 1925,
+    0xC555: 1926,
+    0xC557: 1927,
+    0xC561: 1928,
+    0xC565: 1929,
+    0xC569: 1930,
+    0xC571: 1931,
+    0xC573: 1932,
+    0xC575: 1933,
+    0xC576: 1934,
+    0xC577: 1935,
+    0xC581: 1936,
+    0xC5A1: 1937,
+    0xC5A2: 1938,
+    0xC5A5: 1939,
+    0xC5A9: 1940,
+    0xC5B1: 1941,
+    0xC5B3: 1942,
+    0xC5B5: 1943,
+    0xC5B7: 1944,
+    0xC5C1: 1945,
+    0xC5C2: 1946,
+    0xC5C5: 1947,
+    0xC5C9: 1948,
+    0xC5D1: 1949,
+    0xC5D7: 1950,
+    0xC5E1: 1951,
+    0xC5F7: 1952,
+    0xC641: 1953,
+    0xC649: 1954,
+    0xC661: 1955,
+    0xC681: 1956,
+    0xC682: 1957,
+    0xC685: 1958,
+    0xC689: 1959,
+    0xC691: 1960,
+    0xC693: 1961,
+    0xC695: 1962,
+    0xC697: 1963,
+    0xC6A1: 1964,
+    0xC6A5: 1965,
+    0xC6A9: 1966,
+    0xC6B7: 1967,
+    0xC6C1: 1968,
+    0xC6D7: 1969,
+    0xC6E1: 1970,
+    0xC6E2: 1971,
+    0xC6E5: 1972,
+    0xC6E9: 1973,
+    0xC6F1: 1974,
+    0xC6F3: 1975,
+    0xC6F5: 1976,
+    0xC6F7: 1977,
+    0xC741: 1978,
+    0xC745: 1979,
+    0xC749: 1980,
+    0xC751: 1981,
+    0xC761: 1982,
+    0xC762: 1983,
+    0xC765: 1984,
+    0xC769: 1985,
+    0xC771: 1986,
+    0xC773: 1987,
+    0xC777: 1988,
+    0xC7A1: 1989,
+    0xC7A2: 1990,
+    0xC7A5: 1991,
+    0xC7A9: 1992,
+    0xC7B1: 1993,
+    0xC7B3: 1994,
+    0xC7B5: 1995,
+    0xC7B7: 1996,
+    0xC861: 1997,
+    0xC862: 1998,
+    0xC865: 1999,
+    0xC869: 2000,
+    0xC86A: 2001,
+    0xC871: 2002,
+    0xC873: 2003,
+    0xC875: 2004,
+    0xC876: 2005,
+    0xC877: 2006,
+    0xC881: 2007,
+    0xC882: 2008,
+    0xC885: 2009,
+    0xC889: 2010,
+    0xC891: 2011,
+    0xC893: 2012,
+    0xC895: 2013,
+    0xC896: 2014,
+    0xC897: 2015,
+    0xC8A1: 2016,
+    0xC8B7: 2017,
+    0xC8E1: 2018,
+    0xC8E2: 2019,
+    0xC8E5: 2020,
+    0xC8E9: 2021,
+    0xC8EB: 2022,
+    0xC8F1: 2023,
+    0xC8F3: 2024,
+    0xC8F5: 2025,
+    0xC8F6: 2026,
+    0xC8F7: 2027,
+    0xC941: 2028,
+    0xC942: 2029,
+    0xC945: 2030,
+    0xC949: 2031,
+    0xC951: 2032,
+    0xC953: 2033,
+    0xC955: 2034,
+    0xC957: 2035,
+    0xC961: 2036,
+    0xC965: 2037,
+    0xC976: 2038,
+    0xC981: 2039,
+    0xC985: 2040,
+    0xC9A1: 2041,
+    0xC9A2: 2042,
+    0xC9A5: 2043,
+    0xC9A9: 2044,
+    0xC9B1: 2045,
+    0xC9B3: 2046,
+    0xC9B5: 2047,
+    0xC9B7: 2048,
+    0xC9BC: 2049,
+    0xC9C1: 2050,
+    0xC9C5: 2051,
+    0xC9E1: 2052,
+    0xCA41: 2053,
+    0xCA45: 2054,
+    0xCA55: 2055,
+    0xCA57: 2056,
+    0xCA61: 2057,
+    0xCA81: 2058,
+    0xCA82: 2059,
+    0xCA85: 2060,
+    0xCA89: 2061,
+    0xCA91: 2062,
+    0xCA93: 2063,
+    0xCA95: 2064,
+    0xCA97: 2065,
+    0xCAA1: 2066,
+    0xCAB6: 2067,
+    0xCAC1: 2068,
+    0xCAE1: 2069,
+    0xCAE2: 2070,
+    0xCAE5: 2071,
+    0xCAE9: 2072,
+    0xCAF1: 2073,
+    0xCAF3: 2074,
+    0xCAF7: 2075,
+    0xCB41: 2076,
+    0xCB45: 2077,
+    0xCB49: 2078,
+    0xCB51: 2079,
+    0xCB57: 2080,
+    0xCB61: 2081,
+    0xCB62: 2082,
+    0xCB65: 2083,
+    0xCB68: 2084,
+    0xCB69: 2085,
+    0xCB6B: 2086,
+    0xCB71: 2087,
+    0xCB73: 2088,
+    0xCB75: 2089,
+    0xCB81: 2090,
+    0xCB85: 2091,
+    0xCB89: 2092,
+    0xCB91: 2093,
+    0xCB93: 2094,
+    0xCBA1: 2095,
+    0xCBA2: 2096,
+    0xCBA5: 2097,
+    0xCBA9: 2098,
+    0xCBB1: 2099,
+    0xCBB3: 2100,
+    0xCBB5: 2101,
+    0xCBB7: 2102,
+    0xCC61: 2103,
+    0xCC62: 2104,
+    0xCC63: 2105,
+    0xCC65: 2106,
+    0xCC69: 2107,
+    0xCC6B: 2108,
+    0xCC71: 2109,
+    0xCC73: 2110,
+    0xCC75: 2111,
+    0xCC76: 2112,
+    0xCC77: 2113,
+    0xCC7B: 2114,
+    0xCC81: 2115,
+    0xCC82: 2116,
+    0xCC85: 2117,
+    0xCC89: 2118,
+    0xCC91: 2119,
+    0xCC93: 2120,
+    0xCC95: 2121,
+    0xCC96: 2122,
+    0xCC97: 2123,
+    0xCCA1: 2124,
+    0xCCA2: 2125,
+    0xCCE1: 2126,
+    0xCCE2: 2127,
+    0xCCE5: 2128,
+    0xCCE9: 2129,
+    0xCCF1: 2130,
+    0xCCF3: 2131,
+    0xCCF5: 2132,
+    0xCCF6: 2133,
+    0xCCF7: 2134,
+    0xCD41: 2135,
+    0xCD42: 2136,
+    0xCD45: 2137,
+    0xCD49: 2138,
+    0xCD51: 2139,
+    0xCD53: 2140,
+    0xCD55: 2141,
+    0xCD57: 2142,
+    0xCD61: 2143,
+    0xCD65: 2144,
+    0xCD69: 2145,
+    0xCD71: 2146,
+    0xCD73: 2147,
+    0xCD76: 2148,
+    0xCD77: 2149,
+    0xCD81: 2150,
+    0xCD89: 2151,
+    0xCD93: 2152,
+    0xCD95: 2153,
+    0xCDA1: 2154,
+    0xCDA2: 2155,
+    0xCDA5: 2156,
+    0xCDA9: 2157,
+    0xCDB1: 2158,
+    0xCDB3: 2159,
+    0xCDB5: 2160,
+    0xCDB7: 2161,
+    0xCDC1: 2162,
+    0xCDD7: 2163,
+    0xCE41: 2164,
+    0xCE45: 2165,
+    0xCE61: 2166,
+    0xCE65: 2167,
+    0xCE69: 2168,
+    0xCE73: 2169,
+    0xCE75: 2170,
+    0xCE81: 2171,
+    0xCE82: 2172,
+    0xCE85: 2173,
+    0xCE88: 2174,
+    0xCE89: 2175,
+    0xCE8B: 2176,
+    0xCE91: 2177,
+    0xCE93: 2178,
+    0xCE95: 2179,
+    0xCE97: 2180,
+    0xCEA1: 2181,
+    0xCEB7: 2182,
+    0xCEE1: 2183,
+    0xCEE5: 2184,
+    0xCEE9: 2185,
+    0xCEF1: 2186,
+    0xCEF5: 2187,
+    0xCF41: 2188,
+    0xCF45: 2189,
+    0xCF49: 2190,
+    0xCF51: 2191,
+    0xCF55: 2192,
+    0xCF57: 2193,
+    0xCF61: 2194,
+    0xCF65: 2195,
+    0xCF69: 2196,
+    0xCF71: 2197,
+    0xCF73: 2198,
+    0xCF75: 2199,
+    0xCFA1: 2200,
+    0xCFA2: 2201,
+    0xCFA5: 2202,
+    0xCFA9: 2203,
+    0xCFB1: 2204,
+    0xCFB3: 2205,
+    0xCFB5: 2206,
+    0xCFB7: 2207,
+    0xD061: 2208,
+    0xD062: 2209,
+    0xD065: 2210,
+    0xD069: 2211,
+    0xD06E: 2212,
+    0xD071: 2213,
+    0xD073: 2214,
+    0xD075: 2215,
+    0xD077: 2216,
+    0xD081: 2217,
+    0xD082: 2218,
+    0xD085: 2219,
+    0xD089: 2220,
+    0xD091: 2221,
+    0xD093: 2222,
+    0xD095: 2223,
+    0xD096: 2224,
+    0xD097: 2225,
+    0xD0A1: 2226,
+    0xD0B7: 2227,
+    0xD0E1: 2228,
+    0xD0E2: 2229,
+    0xD0E5: 2230,
+    0xD0E9: 2231,
+    0xD0EB: 2232,
+    0xD0F1: 2233,
+    0xD0F3: 2234,
+    0xD0F5: 2235,
+    0xD0F7: 2236,
+    0xD141: 2237,
+    0xD142: 2238,
+    0xD145: 2239,
+    0xD149: 2240,
+    0xD151: 2241,
+    0xD153: 2242,
+    0xD155: 2243,
+    0xD157: 2244,
+    0xD161: 2245,
+    0xD162: 2246,
+    0xD165: 2247,
+    0xD169: 2248,
+    0xD171: 2249,
+    0xD173: 2250,
+    0xD175: 2251,
+    0xD176: 2252,
+    0xD177: 2253,
+    0xD181: 2254,
+    0xD185: 2255,
+    0xD189: 2256,
+    0xD193: 2257,
+    0xD1A1: 2258,
+    0xD1A2: 2259,
+    0xD1A5: 2260,
+    0xD1A9: 2261,
+    0xD1AE: 2262,
+    0xD1B1: 2263,
+    0xD1B3: 2264,
+    0xD1B5: 2265,
+    0xD1B7: 2266,
+    0xD1BB: 2267,
+    0xD1C1: 2268,
+    0xD1C2: 2269,
+    0xD1C5: 2270,
+    0xD1C9: 2271,
+    0xD1D5: 2272,
+    0xD1D7: 2273,
+    0xD1E1: 2274,
+    0xD1E2: 2275,
+    0xD1E5: 2276,
+    0xD1F5: 2277,
+    0xD1F7: 2278,
+    0xD241: 2279,
+    0xD242: 2280,
+    0xD245: 2281,
+    0xD249: 2282,
+    0xD253: 2283,
+    0xD255: 2284,
+    0xD257: 2285,
+    0xD261: 2286,
+    0xD265: 2287,
+    0xD269: 2288,
+    0xD273: 2289,
+    0xD275: 2290,
+    0xD281: 2291,
+    0xD282: 2292,
+    0xD285: 2293,
+    0xD289: 2294,
+    0xD28E: 2295,
+    0xD291: 2296,
+    0xD295: 2297,
+    0xD297: 2298,
+    0xD2A1: 2299,
+    0xD2A5: 2300,
+    0xD2A9: 2301,
+    0xD2B1: 2302,
+    0xD2B7: 2303,
+    0xD2C1: 2304,
+    0xD2C2: 2305,
+    0xD2C5: 2306,
+    0xD2C9: 2307,
+    0xD2D7: 2308,
+    0xD2E1: 2309,
+    0xD2E2: 2310,
+    0xD2E5: 2311,
+    0xD2E9: 2312,
+    0xD2F1: 2313,
+    0xD2F3: 2314,
+    0xD2F5: 2315,
+    0xD2F7: 2316,
+    0xD341: 2317,
+    0xD342: 2318,
+    0xD345: 2319,
+    0xD349: 2320,
+    0xD351: 2321,
+    0xD355: 2322,
+    0xD357: 2323,
+    0xD361: 2324,
+    0xD362: 2325,
+    0xD365: 2326,
+    0xD367: 2327,
+    0xD368: 2328,
+    0xD369: 2329,
+    0xD36A: 2330,
+    0xD371: 2331,
+    0xD373: 2332,
+    0xD375: 2333,
+    0xD377: 2334,
+    0xD37B: 2335,
+    0xD381: 2336,
+    0xD385: 2337,
+    0xD389: 2338,
+    0xD391: 2339,
+    0xD393: 2340,
+    0xD397: 2341,
+    0xD3A1: 2342,
+    0xD3A2: 2343,
+    0xD3A5: 2344,
+    0xD3A9: 2345,
+    0xD3B1: 2346,
+    0xD3B3: 2347,
+    0xD3B5: 2348,
+    0xD3B7: 2349,
+}
diff --git a/contrib/python/chardet/py3/chardet/johabprober.py b/contrib/python/chardet/py3/chardet/johabprober.py
new file mode 100644
index 00000000000..d7364ba61ec
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/johabprober.py
@@ -0,0 +1,47 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .chardistribution import JOHABDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import JOHAB_SM_MODEL
+
+
+class JOHABProber(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(JOHAB_SM_MODEL)
+        self.distribution_analyzer = JOHABDistributionAnalysis()
+        self.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "Johab"
+
+    @property
+    def language(self) -> str:
+        return "Korean"
diff --git a/contrib/python/chardet/py3/chardet/jpcntx.py b/contrib/python/chardet/py3/chardet/jpcntx.py
new file mode 100644
index 00000000000..2f53bdda09e
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/jpcntx.py
@@ -0,0 +1,238 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import List, Tuple, Union
+
+# This is hiragana 2-char sequence table, the number in each cell represents its frequency category
+# fmt: off
+jp2_char_context = (
+    (0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1),
+    (2, 4, 0, 4, 0, 3, 0, 4, 0, 3, 4, 4, 4, 2, 4, 3, 3, 4, 3, 2, 3, 3, 4, 2, 3, 3, 3, 2, 4, 1, 4, 3, 3, 1, 5, 4, 3, 4, 3, 4, 3, 5, 3, 0, 3, 5, 4, 2, 0, 3, 1, 0, 3, 3, 0, 3, 3, 0, 1, 1, 0, 4, 3, 0, 3, 3, 0, 4, 0, 2, 0, 3, 5, 5, 5, 5, 4, 0, 4, 1, 0, 3, 4),
+    (0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2),
+    (0, 4, 0, 5, 0, 5, 0, 4, 0, 4, 5, 4, 4, 3, 5, 3, 5, 1, 5, 3, 4, 3, 4, 4, 3, 4, 3, 3, 4, 3, 5, 4, 4, 3, 5, 5, 3, 5, 5, 5, 3, 5, 5, 3, 4, 5, 5, 3, 1, 3, 2, 0, 3, 4, 0, 4, 2, 0, 4, 2, 1, 5, 3, 2, 3, 5, 0, 4, 0, 2, 0, 5, 4, 4, 5, 4, 5, 0, 4, 0, 0, 4, 4),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+    (0, 3, 0, 4, 0, 3, 0, 3, 0, 4, 5, 4, 3, 3, 3, 3, 4, 3, 5, 4, 4, 3, 5, 4, 4, 3, 4, 3, 4, 4, 4, 4, 5, 3, 4, 4, 3, 4, 5, 5, 4, 5, 5, 1, 4, 5, 4, 3, 0, 3, 3, 1, 3, 3, 0, 4, 4, 0, 3, 3, 1, 5, 3, 3, 3, 5, 0, 4, 0, 3, 0, 4, 4, 3, 4, 3, 3, 0, 4, 1, 1, 3, 4),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+    (0, 4, 0, 3, 0, 3, 0, 4, 0, 3, 4, 4, 3, 2, 2, 1, 2, 1, 3, 1, 3, 3, 3, 3, 3, 4, 3, 1, 3, 3, 5, 3, 3, 0, 4, 3, 0, 5, 4, 3, 3, 5, 4, 4, 3, 4, 4, 5, 0, 1, 2, 0, 1, 2, 0, 2, 2, 0, 1, 0, 0, 5, 2, 2, 1, 4, 0, 3, 0, 1, 0, 4, 4, 3, 5, 4, 3, 0, 2, 1, 0, 4, 3),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+    (0, 3, 0, 5, 0, 4, 0, 2, 1, 4, 4, 2, 4, 1, 4, 2, 4, 2, 4, 3, 3, 3, 4, 3, 3, 3, 3, 1, 4, 2, 3, 3, 3, 1, 4, 4, 1, 1, 1, 4, 3, 3, 2, 0, 2, 4, 3, 2, 0, 3, 3, 0, 3, 1, 1, 0, 0, 0, 3, 3, 0, 4, 2, 2, 3, 4, 0, 4, 0, 3, 0, 4, 4, 5, 3, 4, 4, 0, 3, 0, 0, 1, 4),
+    (1, 4, 0, 4, 0, 4, 0, 4, 0, 3, 5, 4, 4, 3, 4, 3, 5, 4, 3, 3, 4, 3, 5, 4, 4, 4, 4, 3, 4, 2, 4, 3, 3, 1, 5, 4, 3, 2, 4, 5, 4, 5, 5, 4, 4, 5, 4, 4, 0, 3, 2, 2, 3, 3, 0, 4, 3, 1, 3, 2, 1, 4, 3, 3, 4, 5, 0, 3, 0, 2, 0, 4, 5, 5, 4, 5, 4, 0, 4, 0, 0, 5, 4),
+    (0, 5, 0, 5, 0, 4, 0, 3, 0, 4, 4, 3, 4, 3, 3, 3, 4, 0, 4, 4, 4, 3, 4, 3, 4, 3, 3, 1, 4, 2, 4, 3, 4, 0, 5, 4, 1, 4, 5, 4, 4, 5, 3, 2, 4, 3, 4, 3, 2, 4, 1, 3, 3, 3, 2, 3, 2, 0, 4, 3, 3, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 5, 4, 4, 4, 3, 0, 4, 1, 0, 1, 3),
+    (0, 3, 1, 4, 0, 3, 0, 2, 0, 3, 4, 4, 3, 1, 4, 2, 3, 3, 4, 3, 4, 3, 4, 3, 4, 4, 3, 2, 3, 1, 5, 4, 4, 1, 4, 4, 3, 5, 4, 4, 3, 5, 5, 4, 3, 4, 4, 3, 1, 2, 3, 1, 2, 2, 0, 3, 2, 0, 3, 1, 0, 5, 3, 3, 3, 4, 3, 3, 3, 3, 4, 4, 4, 4, 5, 4, 2, 0, 3, 3, 2, 4, 3),
+    (0, 2, 0, 3, 0, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 1, 0, 2, 1, 3, 3, 3, 1, 2, 3, 1, 0, 1, 0, 4, 2, 1, 1, 3, 3, 0, 4, 3, 3, 1, 4, 3, 3, 0, 3, 3, 2, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 4, 1, 0, 2, 3, 2, 2, 2, 1, 3, 3, 3, 4, 4, 3, 2, 0, 3, 1, 0, 3, 3),
+    (0, 4, 0, 4, 0, 3, 0, 3, 0, 4, 4, 4, 3, 3, 3, 3, 3, 3, 4, 3, 4, 2, 4, 3, 4, 3, 3, 2, 4, 3, 4, 5, 4, 1, 4, 5, 3, 5, 4, 5, 3, 5, 4, 0, 3, 5, 5, 3, 1, 3, 3, 2, 2, 3, 0, 3, 4, 1, 3, 3, 2, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 5, 4, 4, 5, 3, 0, 4, 1, 0, 3, 4),
+    (0, 2, 0, 3, 0, 3, 0, 0, 0, 2, 2, 2, 1, 0, 1, 0, 0, 0, 3, 0, 3, 0, 3, 0, 1, 3, 1, 0, 3, 1, 3, 3, 3, 1, 3, 3, 3, 0, 1, 3, 1, 3, 4, 0, 0, 3, 1, 1, 0, 3, 2, 0, 0, 0, 0, 1, 3, 0, 1, 0, 0, 3, 3, 2, 0, 3, 0, 0, 0, 0, 0, 3, 4, 3, 4, 3, 3, 0, 3, 0, 0, 2, 3),
+    (2, 3, 0, 3, 0, 2, 0, 1, 0, 3, 3, 4, 3, 1, 3, 1, 1, 1, 3, 1, 4, 3, 4, 3, 3, 3, 0, 0, 3, 1, 5, 4, 3, 1, 4, 3, 2, 5, 5, 4, 4, 4, 4, 3, 3, 4, 4, 4, 0, 2, 1, 1, 3, 2, 0, 1, 2, 0, 0, 1, 0, 4, 1, 3, 3, 3, 0, 3, 0, 1, 0, 4, 4, 4, 5, 5, 3, 0, 2, 0, 0, 4, 4),
+    (0, 2, 0, 1, 0, 3, 1, 3, 0, 2, 3, 3, 3, 0, 3, 1, 0, 0, 3, 0, 3, 2, 3, 1, 3, 2, 1, 1, 0, 0, 4, 2, 1, 0, 2, 3, 1, 4, 3, 2, 0, 4, 4, 3, 1, 3, 1, 3, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4, 1, 1, 1, 2, 0, 3, 0, 0, 0, 3, 4, 2, 4, 3, 2, 0, 1, 0, 0, 3, 3),
+    (0, 1, 0, 4, 0, 5, 0, 4, 0, 2, 4, 4, 2, 3, 3, 2, 3, 3, 5, 3, 3, 3, 4, 3, 4, 2, 3, 0, 4, 3, 3, 3, 4, 1, 4, 3, 2, 1, 5, 5, 3, 4, 5, 1, 3, 5, 4, 2, 0, 3, 3, 0, 1, 3, 0, 4, 2, 0, 1, 3, 1, 4, 3, 3, 3, 3, 0, 3, 0, 1, 0, 3, 4, 4, 4, 5, 5, 0, 3, 0, 1, 4, 5),
+    (0, 2, 0, 3, 0, 3, 0, 0, 0, 2, 3, 1, 3, 0, 4, 0, 1, 1, 3, 0, 3, 4, 3, 2, 3, 1, 0, 3, 3, 2, 3, 1, 3, 0, 2, 3, 0, 2, 1, 4, 1, 2, 2, 0, 0, 3, 3, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 0, 3, 2, 1, 3, 3, 0, 2, 0, 2, 0, 0, 3, 3, 1, 2, 4, 0, 3, 0, 2, 2, 3),
+    (2, 4, 0, 5, 0, 4, 0, 4, 0, 2, 4, 4, 4, 3, 4, 3, 3, 3, 1, 2, 4, 3, 4, 3, 4, 4, 5, 0, 3, 3, 3, 3, 2, 0, 4, 3, 1, 4, 3, 4, 1, 4, 4, 3, 3, 4, 4, 3, 1, 2, 3, 0, 4, 2, 0, 4, 1, 0, 3, 3, 0, 4, 3, 3, 3, 4, 0, 4, 0, 2, 0, 3, 5, 3, 4, 5, 2, 0, 3, 0, 0, 4, 5),
+    (0, 3, 0, 4, 0, 1, 0, 1, 0, 1, 3, 2, 2, 1, 3, 0, 3, 0, 2, 0, 2, 0, 3, 0, 2, 0, 0, 0, 1, 0, 1, 1, 0, 0, 3, 1, 0, 0, 0, 4, 0, 3, 1, 0, 2, 1, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 2, 2, 3, 1, 0, 3, 0, 0, 0, 1, 4, 4, 4, 3, 0, 0, 4, 0, 0, 1, 4),
+    (1, 4, 1, 5, 0, 3, 0, 3, 0, 4, 5, 4, 4, 3, 5, 3, 3, 4, 4, 3, 4, 1, 3, 3, 3, 3, 2, 1, 4, 1, 5, 4, 3, 1, 4, 4, 3, 5, 4, 4, 3, 5, 4, 3, 3, 4, 4, 4, 0, 3, 3, 1, 2, 3, 0, 3, 1, 0, 3, 3, 0, 5, 4, 4, 4, 4, 4, 4, 3, 3, 5, 4, 4, 3, 3, 5, 4, 0, 3, 2, 0, 4, 4),
+    (0, 2, 0, 3, 0, 1, 0, 0, 0, 1, 3, 3, 3, 2, 4, 1, 3, 0, 3, 1, 3, 0, 2, 2, 1, 1, 0, 0, 2, 0, 4, 3, 1, 0, 4, 3, 0, 4, 4, 4, 1, 4, 3, 1, 1, 3, 3, 1, 0, 2, 0, 0, 1, 3, 0, 0, 0, 0, 2, 0, 0, 4, 3, 2, 4, 3, 5, 4, 3, 3, 3, 4, 3, 3, 4, 3, 3, 0, 2, 1, 0, 3, 3),
+    (0, 2, 0, 4, 0, 3, 0, 2, 0, 2, 5, 5, 3, 4, 4, 4, 4, 1, 4, 3, 3, 0, 4, 3, 4, 3, 1, 3, 3, 2, 4, 3, 0, 3, 4, 3, 0, 3, 4, 4, 2, 4, 4, 0, 4, 5, 3, 3, 2, 2, 1, 1, 1, 2, 0, 1, 5, 0, 3, 3, 2, 4, 3, 3, 3, 4, 0, 3, 0, 2, 0, 4, 4, 3, 5, 5, 0, 0, 3, 0, 2, 3, 3),
+    (0, 3, 0, 4, 0, 3, 0, 1, 0, 3, 4, 3, 3, 1, 3, 3, 3, 0, 3, 1, 3, 0, 4, 3, 3, 1, 1, 0, 3, 0, 3, 3, 0, 0, 4, 4, 0, 1, 5, 4, 3, 3, 5, 0, 3, 3, 4, 3, 0, 2, 0, 1, 1, 1, 0, 1, 3, 0, 1, 2, 1, 3, 3, 2, 3, 3, 0, 3, 0, 1, 0, 1, 3, 3, 4, 4, 1, 0, 1, 2, 2, 1, 3),
+    (0, 1, 0, 4, 0, 4, 0, 3, 0, 1, 3, 3, 3, 2, 3, 1, 1, 0, 3, 0, 3, 3, 4, 3, 2, 4, 2, 0, 1, 0, 4, 3, 2, 0, 4, 3, 0, 5, 3, 3, 2, 4, 4, 4, 3, 3, 3, 4, 0, 1, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4, 2, 3, 3, 3, 0, 3, 0, 0, 0, 4, 4, 4, 5, 3, 2, 0, 3, 3, 0, 3, 5),
+    (0, 2, 0, 3, 0, 0, 0, 3, 0, 1, 3, 0, 2, 0, 0, 0, 1, 0, 3, 1, 1, 3, 3, 0, 0, 3, 0, 0, 3, 0, 2, 3, 1, 0, 3, 1, 0, 3, 3, 2, 0, 4, 2, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 1, 0, 1, 0, 0, 0, 1, 3, 1, 2, 0, 0, 0, 1, 0, 0, 1, 4),
+    (0, 3, 0, 3, 0, 5, 0, 1, 0, 2, 4, 3, 1, 3, 3, 2, 1, 1, 5, 2, 1, 0, 5, 1, 2, 0, 0, 0, 3, 3, 2, 2, 3, 2, 4, 3, 0, 0, 3, 3, 1, 3, 3, 0, 2, 5, 3, 4, 0, 3, 3, 0, 1, 2, 0, 2, 2, 0, 3, 2, 0, 2, 2, 3, 3, 3, 0, 2, 0, 1, 0, 3, 4, 4, 2, 5, 4, 0, 3, 0, 0, 3, 5),
+    (0, 3, 0, 3, 0, 3, 0, 1, 0, 3, 3, 3, 3, 0, 3, 0, 2, 0, 2, 1, 1, 0, 2, 0, 1, 0, 0, 0, 2, 1, 0, 0, 1, 0, 3, 2, 0, 0, 3, 3, 1, 2, 3, 1, 0, 3, 3, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 3, 1, 2, 3, 0, 3, 0, 1, 0, 3, 2, 1, 0, 4, 3, 0, 1, 1, 0, 3, 3),
+    (0, 4, 0, 5, 0, 3, 0, 3, 0, 4, 5, 5, 4, 3, 5, 3, 4, 3, 5, 3, 3, 2, 5, 3, 4, 4, 4, 3, 4, 3, 4, 5, 5, 3, 4, 4, 3, 4, 4, 5, 4, 4, 4, 3, 4, 5, 5, 4, 2, 3, 4, 2, 3, 4, 0, 3, 3, 1, 4, 3, 2, 4, 3, 3, 5, 5, 0, 3, 0, 3, 0, 5, 5, 5, 5, 4, 4, 0, 4, 0, 1, 4, 4),
+    (0, 4, 0, 4, 0, 3, 0, 3, 0, 3, 5, 4, 4, 2, 3, 2, 5, 1, 3, 2, 5, 1, 4, 2, 3, 2, 3, 3, 4, 3, 3, 3, 3, 2, 5, 4, 1, 3, 3, 5, 3, 4, 4, 0, 4, 4, 3, 1, 1, 3, 1, 0, 2, 3, 0, 2, 3, 0, 3, 0, 0, 4, 3, 1, 3, 4, 0, 3, 0, 2, 0, 4, 4, 4, 3, 4, 5, 0, 4, 0, 0, 3, 4),
+    (0, 3, 0, 3, 0, 3, 1, 2, 0, 3, 4, 4, 3, 3, 3, 0, 2, 2, 4, 3, 3, 1, 3, 3, 3, 1, 1, 0, 3, 1, 4, 3, 2, 3, 4, 4, 2, 4, 4, 4, 3, 4, 4, 3, 2, 4, 4, 3, 1, 3, 3, 1, 3, 3, 0, 4, 1, 0, 2, 2, 1, 4, 3, 2, 3, 3, 5, 4, 3, 3, 5, 4, 4, 3, 3, 0, 4, 0, 3, 2, 2, 4, 4),
+    (0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 2, 1, 3, 0, 0, 0, 0, 0, 2, 0, 1, 2, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 1, 0, 1, 1, 3, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 0, 3, 4, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1),
+    (0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 4, 1, 4, 0, 3, 0, 4, 0, 3, 0, 4, 0, 3, 0, 3, 0, 4, 1, 5, 1, 4, 0, 0, 3, 0, 5, 0, 5, 2, 0, 1, 0, 0, 0, 2, 1, 4, 0, 1, 3, 0, 0, 3, 0, 0, 3, 1, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0),
+    (1, 4, 0, 5, 0, 3, 0, 2, 0, 3, 5, 4, 4, 3, 4, 3, 5, 3, 4, 3, 3, 0, 4, 3, 3, 3, 3, 3, 3, 2, 4, 4, 3, 1, 3, 4, 4, 5, 4, 4, 3, 4, 4, 1, 3, 5, 4, 3, 3, 3, 1, 2, 2, 3, 3, 1, 3, 1, 3, 3, 3, 5, 3, 3, 4, 5, 0, 3, 0, 3, 0, 3, 4, 3, 4, 4, 3, 0, 3, 0, 2, 4, 3),
+    (0, 1, 0, 4, 0, 0, 0, 0, 0, 1, 4, 0, 4, 1, 4, 2, 4, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 0, 3, 1, 1, 1, 0, 3, 0, 0, 0, 1, 2, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 3, 2, 0, 2, 2, 0, 1, 0, 0, 0, 2, 3, 2, 3, 3, 0, 0, 0, 0, 2, 1, 0),
+    (0, 5, 1, 5, 0, 3, 0, 3, 0, 5, 4, 4, 5, 1, 5, 3, 3, 0, 4, 3, 4, 3, 5, 3, 4, 3, 3, 2, 4, 3, 4, 3, 3, 0, 3, 3, 1, 4, 4, 3, 4, 4, 4, 3, 4, 5, 5, 3, 2, 3, 1, 1, 3, 3, 1, 3, 1, 1, 3, 3, 2, 4, 5, 3, 3, 5, 0, 4, 0, 3, 0, 4, 4, 3, 5, 3, 3, 0, 3, 4, 0, 4, 3),
+    (0, 5, 0, 5, 0, 3, 0, 2, 0, 4, 4, 3, 5, 2, 4, 3, 3, 3, 4, 4, 4, 3, 5, 3, 5, 3, 3, 1, 4, 0, 4, 3, 3, 0, 3, 3, 0, 4, 4, 4, 4, 5, 4, 3, 3, 5, 5, 3, 2, 3, 1, 2, 3, 2, 0, 1, 0, 0, 3, 2, 2, 4, 4, 3, 1, 5, 0, 4, 0, 3, 0, 4, 3, 1, 3, 2, 1, 0, 3, 3, 0, 3, 3),
+    (0, 4, 0, 5, 0, 5, 0, 4, 0, 4, 5, 5, 5, 3, 4, 3, 3, 2, 5, 4, 4, 3, 5, 3, 5, 3, 4, 0, 4, 3, 4, 4, 3, 2, 4, 4, 3, 4, 5, 4, 4, 5, 5, 0, 3, 5, 5, 4, 1, 3, 3, 2, 3, 3, 1, 3, 1, 0, 4, 3, 1, 4, 4, 3, 4, 5, 0, 4, 0, 2, 0, 4, 3, 4, 4, 3, 3, 0, 4, 0, 0, 5, 5),
+    (0, 4, 0, 4, 0, 5, 0, 1, 1, 3, 3, 4, 4, 3, 4, 1, 3, 0, 5, 1, 3, 0, 3, 1, 3, 1, 1, 0, 3, 0, 3, 3, 4, 0, 4, 3, 0, 4, 4, 4, 3, 4, 4, 0, 3, 5, 4, 1, 0, 3, 0, 0, 2, 3, 0, 3, 1, 0, 3, 1, 0, 3, 2, 1, 3, 5, 0, 3, 0, 1, 0, 3, 2, 3, 3, 4, 4, 0, 2, 2, 0, 4, 4),
+    (2, 4, 0, 5, 0, 4, 0, 3, 0, 4, 5, 5, 4, 3, 5, 3, 5, 3, 5, 3, 5, 2, 5, 3, 4, 3, 3, 4, 3, 4, 5, 3, 2, 1, 5, 4, 3, 2, 3, 4, 5, 3, 4, 1, 2, 5, 4, 3, 0, 3, 3, 0, 3, 2, 0, 2, 3, 0, 4, 1, 0, 3, 4, 3, 3, 5, 0, 3, 0, 1, 0, 4, 5, 5, 5, 4, 3, 0, 4, 2, 0, 3, 5),
+    (0, 5, 0, 4, 0, 4, 0, 2, 0, 5, 4, 3, 4, 3, 4, 3, 3, 3, 4, 3, 4, 2, 5, 3, 5, 3, 4, 1, 4, 3, 4, 4, 4, 0, 3, 5, 0, 4, 4, 4, 4, 5, 3, 1, 3, 4, 5, 3, 3, 3, 3, 3, 3, 3, 0, 2, 2, 0, 3, 3, 2, 4, 3, 3, 3, 5, 3, 4, 1, 3, 3, 5, 3, 2, 0, 0, 0, 0, 4, 3, 1, 3, 3),
+    (0, 1, 0, 3, 0, 3, 0, 1, 0, 1, 3, 3, 3, 2, 3, 3, 3, 0, 3, 0, 0, 0, 3, 1, 3, 0, 0, 0, 2, 2, 2, 3, 0, 0, 3, 2, 0, 1, 2, 4, 1, 3, 3, 0, 0, 3, 3, 3, 0, 1, 0, 0, 2, 1, 0, 0, 3, 0, 3, 1, 0, 3, 0, 0, 1, 3, 0, 2, 0, 1, 0, 3, 3, 1, 3, 3, 0, 0, 1, 1, 0, 3, 3),
+    (0, 2, 0, 3, 0, 2, 1, 4, 0, 2, 2, 3, 1, 1, 3, 1, 1, 0, 2, 0, 3, 1, 2, 3, 1, 3, 0, 0, 1, 0, 4, 3, 2, 3, 3, 3, 1, 4, 2, 3, 3, 3, 3, 1, 0, 3, 1, 4, 0, 1, 1, 0, 1, 2, 0, 1, 1, 0, 1, 1, 0, 3, 1, 3, 2, 2, 0, 1, 0, 0, 0, 2, 3, 3, 3, 1, 0, 0, 0, 0, 0, 2, 3),
+    (0, 5, 0, 4, 0, 5, 0, 2, 0, 4, 5, 5, 3, 3, 4, 3, 3, 1, 5, 4, 4, 2, 4, 4, 4, 3, 4, 2, 4, 3, 5, 5, 4, 3, 3, 4, 3, 3, 5, 5, 4, 5, 5, 1, 3, 4, 5, 3, 1, 4, 3, 1, 3, 3, 0, 3, 3, 1, 4, 3, 1, 4, 5, 3, 3, 5, 0, 4, 0, 3, 0, 5, 3, 3, 1, 4, 3, 0, 4, 0, 1, 5, 3),
+    (0, 5, 0, 5, 0, 4, 0, 2, 0, 4, 4, 3, 4, 3, 3, 3, 3, 3, 5, 4, 4, 4, 4, 4, 4, 5, 3, 3, 5, 2, 4, 4, 4, 3, 4, 4, 3, 3, 4, 4, 5, 5, 3, 3, 4, 3, 4, 3, 3, 4, 3, 3, 3, 3, 1, 2, 2, 1, 4, 3, 3, 5, 4, 4, 3, 4, 0, 4, 0, 3, 0, 4, 4, 4, 4, 4, 1, 0, 4, 2, 0, 2, 4),
+    (0, 4, 0, 4, 0, 3, 0, 1, 0, 3, 5, 2, 3, 0, 3, 0, 2, 1, 4, 2, 3, 3, 4, 1, 4, 3, 3, 2, 4, 1, 3, 3, 3, 0, 3, 3, 0, 0, 3, 3, 3, 5, 3, 3, 3, 3, 3, 2, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 1, 0, 0, 3, 1, 2, 2, 3, 0, 3, 0, 2, 0, 4, 4, 3, 3, 4, 1, 0, 3, 0, 0, 2, 4),
+    (0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 3, 1, 3, 0, 3, 2, 0, 0, 0, 1, 0, 3, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 2, 0, 0, 0, 0, 0, 0, 2),
+    (0, 2, 1, 3, 0, 2, 0, 2, 0, 3, 3, 3, 3, 1, 3, 1, 3, 3, 3, 3, 3, 3, 4, 2, 2, 1, 2, 1, 4, 0, 4, 3, 1, 3, 3, 3, 2, 4, 3, 5, 4, 3, 3, 3, 3, 3, 3, 3, 0, 1, 3, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 4, 2, 0, 2, 3, 0, 3, 3, 0, 3, 3, 4, 2, 3, 1, 4, 0, 1, 2, 0, 2, 3),
+    (0, 3, 0, 3, 0, 1, 0, 3, 0, 2, 3, 3, 3, 0, 3, 1, 2, 0, 3, 3, 2, 3, 3, 2, 3, 2, 3, 1, 3, 0, 4, 3, 2, 0, 3, 3, 1, 4, 3, 3, 2, 3, 4, 3, 1, 3, 3, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 4, 1, 1, 0, 3, 0, 3, 1, 0, 2, 3, 3, 3, 3, 3, 1, 0, 0, 2, 0, 3, 3),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 3, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3),
+    (0, 2, 0, 3, 1, 3, 0, 3, 0, 2, 3, 3, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 1, 3, 0, 2, 3, 1, 1, 4, 3, 3, 2, 3, 3, 1, 2, 2, 4, 1, 3, 3, 0, 1, 4, 2, 3, 0, 1, 3, 0, 3, 0, 0, 1, 3, 0, 2, 0, 0, 3, 3, 2, 1, 3, 0, 3, 0, 2, 0, 3, 4, 4, 4, 3, 1, 0, 3, 0, 0, 3, 3),
+    (0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 3, 2, 2, 1, 3, 0, 1, 1, 3, 0, 3, 2, 3, 1, 2, 0, 2, 0, 1, 1, 3, 3, 3, 0, 3, 3, 1, 1, 2, 3, 2, 3, 3, 1, 2, 3, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 2, 1, 2, 1, 3, 0, 3, 0, 0, 0, 3, 4, 4, 4, 3, 2, 0, 2, 0, 0, 2, 4),
+    (0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 3),
+    (0, 3, 0, 3, 0, 2, 0, 3, 0, 3, 3, 3, 2, 3, 2, 2, 2, 0, 3, 1, 3, 3, 3, 2, 3, 3, 0, 0, 3, 0, 3, 2, 2, 0, 2, 3, 1, 4, 3, 4, 3, 3, 2, 3, 1, 5, 4, 4, 0, 3, 1, 2, 1, 3, 0, 3, 1, 1, 2, 0, 2, 3, 1, 3, 1, 3, 0, 3, 0, 1, 0, 3, 3, 4, 4, 2, 1, 0, 2, 1, 0, 2, 4),
+    (0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 4, 2, 5, 1, 4, 0, 2, 0, 2, 1, 3, 1, 4, 0, 2, 1, 0, 0, 2, 1, 4, 1, 1, 0, 3, 3, 0, 5, 1, 3, 2, 3, 3, 1, 0, 3, 2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 1, 0, 3, 0, 2, 0, 1, 0, 3, 3, 3, 4, 3, 3, 0, 0, 0, 0, 2, 3),
+    (0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3),
+    (0, 1, 0, 3, 0, 4, 0, 3, 0, 2, 4, 3, 1, 0, 3, 2, 2, 1, 3, 1, 2, 2, 3, 1, 1, 1, 2, 1, 3, 0, 1, 2, 0, 1, 3, 2, 1, 3, 0, 5, 5, 1, 0, 0, 1, 3, 2, 1, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4, 0, 1, 1, 1, 3, 2, 0, 2, 0, 1, 0, 2, 3, 3, 1, 2, 3, 0, 1, 0, 1, 0, 4),
+    (0, 0, 0, 1, 0, 3, 0, 3, 0, 2, 2, 1, 0, 0, 4, 0, 3, 0, 3, 1, 3, 0, 3, 0, 3, 0, 1, 0, 3, 0, 3, 1, 3, 0, 3, 3, 0, 0, 1, 2, 1, 1, 1, 0, 1, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 2, 0, 0, 2, 0, 0, 0, 0, 2, 3, 3, 3, 3, 0, 0, 0, 0, 1, 4),
+    (0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 3, 1, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 2, 0, 2, 3, 0, 0, 2, 2, 3, 1, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 2, 3),
+    (2, 4, 0, 5, 0, 5, 0, 4, 0, 3, 4, 3, 3, 3, 4, 3, 3, 3, 4, 3, 4, 4, 5, 4, 5, 5, 5, 2, 3, 0, 5, 5, 4, 1, 5, 4, 3, 1, 5, 4, 3, 4, 4, 3, 3, 4, 3, 3, 0, 3, 2, 0, 2, 3, 0, 3, 0, 0, 3, 3, 0, 5, 3, 2, 3, 3, 0, 3, 0, 3, 0, 3, 4, 5, 4, 5, 3, 0, 4, 3, 0, 3, 4),
+    (0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 3, 4, 3, 2, 3, 2, 3, 0, 4, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 2, 4, 3, 3, 1, 3, 4, 3, 4, 4, 4, 3, 4, 4, 3, 2, 4, 4, 1, 0, 2, 0, 0, 1, 1, 0, 2, 0, 0, 3, 1, 0, 5, 3, 2, 1, 3, 0, 3, 0, 1, 2, 4, 3, 2, 4, 3, 3, 0, 3, 2, 0, 4, 4),
+    (0, 3, 0, 3, 0, 1, 0, 0, 0, 1, 4, 3, 3, 2, 3, 1, 3, 1, 4, 2, 3, 2, 4, 2, 3, 4, 3, 0, 2, 2, 3, 3, 3, 0, 3, 3, 3, 0, 3, 4, 1, 3, 3, 0, 3, 4, 3, 3, 0, 1, 1, 0, 1, 0, 0, 0, 4, 0, 3, 0, 0, 3, 1, 2, 1, 3, 0, 4, 0, 1, 0, 4, 3, 3, 4, 3, 3, 0, 2, 0, 0, 3, 3),
+    (0, 3, 0, 4, 0, 1, 0, 3, 0, 3, 4, 3, 3, 0, 3, 3, 3, 1, 3, 1, 3, 3, 4, 3, 3, 3, 0, 0, 3, 1, 5, 3, 3, 1, 3, 3, 2, 5, 4, 3, 3, 4, 5, 3, 2, 5, 3, 4, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 4, 2, 2, 1, 3, 0, 3, 0, 2, 0, 4, 4, 3, 5, 3, 2, 0, 1, 1, 0, 3, 4),
+    (0, 5, 0, 4, 0, 5, 0, 2, 0, 4, 4, 3, 3, 2, 3, 3, 3, 1, 4, 3, 4, 1, 5, 3, 4, 3, 4, 0, 4, 2, 4, 3, 4, 1, 5, 4, 0, 4, 4, 4, 4, 5, 4, 1, 3, 5, 4, 2, 1, 4, 1, 1, 3, 2, 0, 3, 1, 0, 3, 2, 1, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 4, 4, 3, 3, 3, 0, 4, 2, 0, 3, 4),
+    (1, 4, 0, 4, 0, 3, 0, 1, 0, 3, 3, 3, 1, 1, 3, 3, 2, 2, 3, 3, 1, 0, 3, 2, 2, 1, 2, 0, 3, 1, 2, 1, 2, 0, 3, 2, 0, 2, 2, 3, 3, 4, 3, 0, 3, 3, 1, 2, 0, 1, 1, 3, 1, 2, 0, 0, 3, 0, 1, 1, 0, 3, 2, 2, 3, 3, 0, 3, 0, 0, 0, 2, 3, 3, 4, 3, 3, 0, 1, 0, 0, 1, 4),
+    (0, 4, 0, 4, 0, 4, 0, 0, 0, 3, 4, 4, 3, 1, 4, 2, 3, 2, 3, 3, 3, 1, 4, 3, 4, 0, 3, 0, 4, 2, 3, 3, 2, 2, 5, 4, 2, 1, 3, 4, 3, 4, 3, 1, 3, 3, 4, 2, 0, 2, 1, 0, 3, 3, 0, 0, 2, 0, 3, 1, 0, 4, 4, 3, 4, 3, 0, 4, 0, 1, 0, 2, 4, 4, 4, 4, 4, 0, 3, 2, 0, 3, 3),
+    (0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2),
+    (0, 2, 0, 3, 0, 4, 0, 4, 0, 1, 3, 3, 3, 0, 4, 0, 2, 1, 2, 1, 1, 1, 2, 0, 3, 1, 1, 0, 1, 0, 3, 1, 0, 0, 3, 3, 2, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 2, 2, 0, 3, 1, 0, 0, 1, 0, 1, 1, 0, 1, 2, 0, 3, 0, 0, 0, 0, 1, 0, 0, 3, 3, 4, 3, 1, 0, 1, 0, 3, 0, 2),
+    (0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 1, 0, 2, 0, 3, 1, 0, 1, 3, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 0, 2, 3, 0, 1, 4, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 3),
+    (0, 2, 0, 5, 0, 5, 0, 1, 0, 2, 4, 3, 3, 2, 5, 1, 3, 2, 3, 3, 3, 0, 4, 1, 2, 0, 3, 0, 4, 0, 2, 2, 1, 1, 5, 3, 0, 0, 1, 4, 2, 3, 2, 0, 3, 3, 3, 2, 0, 2, 4, 1, 1, 2, 0, 1, 1, 0, 3, 1, 0, 1, 3, 1, 2, 3, 0, 2, 0, 0, 0, 1, 3, 5, 4, 4, 4, 0, 3, 0, 0, 1, 3),
+    (0, 4, 0, 5, 0, 4, 0, 4, 0, 4, 5, 4, 3, 3, 4, 3, 3, 3, 4, 3, 4, 4, 5, 3, 4, 5, 4, 2, 4, 2, 3, 4, 3, 1, 4, 4, 1, 3, 5, 4, 4, 5, 5, 4, 4, 5, 5, 5, 2, 3, 3, 1, 4, 3, 1, 3, 3, 0, 3, 3, 1, 4, 3, 4, 4, 4, 0, 3, 0, 4, 0, 3, 3, 4, 4, 5, 0, 0, 4, 3, 0, 4, 5),
+    (0, 4, 0, 4, 0, 3, 0, 3, 0, 3, 4, 4, 4, 3, 3, 2, 4, 3, 4, 3, 4, 3, 5, 3, 4, 3, 2, 1, 4, 2, 4, 4, 3, 1, 3, 4, 2, 4, 5, 5, 3, 4, 5, 4, 1, 5, 4, 3, 0, 3, 2, 2, 3, 2, 1, 3, 1, 0, 3, 3, 3, 5, 3, 3, 3, 5, 4, 4, 2, 3, 3, 4, 3, 3, 3, 2, 1, 0, 3, 2, 1, 4, 3),
+    (0, 4, 0, 5, 0, 4, 0, 3, 0, 3, 5, 5, 3, 2, 4, 3, 4, 0, 5, 4, 4, 1, 4, 4, 4, 3, 3, 3, 4, 3, 5, 5, 2, 3, 3, 4, 1, 2, 5, 5, 3, 5, 5, 2, 3, 5, 5, 4, 0, 3, 2, 0, 3, 3, 1, 1, 5, 1, 4, 1, 0, 4, 3, 2, 3, 5, 0, 4, 0, 3, 0, 5, 4, 3, 4, 3, 0, 0, 4, 1, 0, 4, 4),
+    (1, 3, 0, 4, 0, 2, 0, 2, 0, 2, 5, 5, 3, 3, 3, 3, 3, 0, 4, 2, 3, 4, 4, 4, 3, 4, 0, 0, 3, 4, 5, 4, 3, 3, 3, 3, 2, 5, 5, 4, 5, 5, 5, 4, 3, 5, 5, 5, 1, 3, 1, 0, 1, 0, 0, 3, 2, 0, 4, 2, 0, 5, 2, 3, 2, 4, 1, 3, 0, 3, 0, 4, 5, 4, 5, 4, 3, 0, 4, 2, 0, 5, 4),
+    (0, 3, 0, 4, 0, 5, 0, 3, 0, 3, 4, 4, 3, 2, 3, 2, 3, 3, 3, 3, 3, 2, 4, 3, 3, 2, 2, 0, 3, 3, 3, 3, 3, 1, 3, 3, 3, 0, 4, 4, 3, 4, 4, 1, 1, 4, 4, 2, 0, 3, 1, 0, 1, 1, 0, 4, 1, 0, 2, 3, 1, 3, 3, 1, 3, 4, 0, 3, 0, 1, 0, 3, 1, 3, 0, 0, 1, 0, 2, 0, 0, 4, 4),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+    (0, 3, 0, 3, 0, 2, 0, 3, 0, 1, 5, 4, 3, 3, 3, 1, 4, 2, 1, 2, 3, 4, 4, 2, 4, 4, 5, 0, 3, 1, 4, 3, 4, 0, 4, 3, 3, 3, 2, 3, 2, 5, 3, 4, 3, 2, 2, 3, 0, 0, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 0, 2, 1, 1, 3, 1, 0, 2, 0, 4, 0, 3, 4, 4, 4, 5, 2, 0, 2, 0, 0, 1, 3),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 4, 2, 1, 1, 0, 1, 0, 3, 2, 0, 0, 3, 1, 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 4, 0, 4, 2, 1, 0, 0, 0, 0, 0, 1),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 2, 1, 0, 0, 1, 2, 1, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2),
+    (0, 4, 0, 4, 0, 4, 0, 3, 0, 4, 4, 3, 4, 2, 4, 3, 2, 0, 4, 4, 4, 3, 5, 3, 5, 3, 3, 2, 4, 2, 4, 3, 4, 3, 1, 4, 0, 2, 3, 4, 4, 4, 3, 3, 3, 4, 4, 4, 3, 4, 1, 3, 4, 3, 2, 1, 2, 1, 3, 3, 3, 4, 4, 3, 3, 5, 0, 4, 0, 3, 0, 4, 3, 3, 3, 2, 1, 0, 3, 0, 0, 3, 3),
+    (0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 5, 5, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 4, 4, 4, 3, 3, 3, 3, 4, 3, 5, 3, 3, 1, 3, 2, 4, 5, 5, 5, 5, 4, 3, 4, 5, 5, 3, 2, 2, 3, 3, 3, 3, 2, 3, 3, 1, 2, 3, 2, 4, 3, 3, 3, 4, 0, 4, 0, 2, 0, 4, 3, 2, 2, 1, 2, 0, 3, 0, 0, 4, 1),
+)
+# fmt: on
+
+
+class JapaneseContextAnalysis:
+    NUM_OF_CATEGORY = 6
+    DONT_KNOW = -1
+    ENOUGH_REL_THRESHOLD = 100
+    MAX_REL_THRESHOLD = 1000
+    MINIMUM_DATA_THRESHOLD = 4
+
+    def __init__(self) -> None:
+        self._total_rel = 0
+        self._rel_sample: List[int] = []
+        self._need_to_skip_char_num = 0
+        self._last_char_order = -1
+        self._done = False
+        self.reset()
+
+    def reset(self) -> None:
+        self._total_rel = 0  # total sequence received
+        # category counters, each integer counts sequence in its category
+        self._rel_sample = [0] * self.NUM_OF_CATEGORY
+        # if last byte in current buffer is not the last byte of a character,
+        # we need to know how many bytes to skip in next buffer
+        self._need_to_skip_char_num = 0
+        self._last_char_order = -1  # The order of previous char
+        # If this flag is set to True, detection is done and conclusion has
+        # been made
+        self._done = False
+
+    def feed(self, byte_str: Union[bytes, bytearray], num_bytes: int) -> None:
+        if self._done:
+            return
+
+        # The buffer we got is byte oriented, and a character may span in more than one
+        # buffers. In case the last one or two byte in last buffer is not
+        # complete, we record how many byte needed to complete that character
+        # and skip these bytes here.  We can choose to record those bytes as
+        # well and analyse the character once it is complete, but since a
+        # character will not make much difference, by simply skipping
+        # this character will simply our logic and improve performance.
+        i = self._need_to_skip_char_num
+        while i < num_bytes:
+            order, char_len = self.get_order(byte_str[i : i + 2])
+            i += char_len
+            if i > num_bytes:
+                self._need_to_skip_char_num = i - num_bytes
+                self._last_char_order = -1
+            else:
+                if (order != -1) and (self._last_char_order != -1):
+                    self._total_rel += 1
+                    if self._total_rel > self.MAX_REL_THRESHOLD:
+                        self._done = True
+                        break
+                    self._rel_sample[
+                        jp2_char_context[self._last_char_order][order]
+                    ] += 1
+                self._last_char_order = order
+
+    def got_enough_data(self) -> bool:
+        return self._total_rel > self.ENOUGH_REL_THRESHOLD
+
+    def get_confidence(self) -> float:
+        # This is just one way to calculate confidence. It works well for me.
+        if self._total_rel > self.MINIMUM_DATA_THRESHOLD:
+            return (self._total_rel - self._rel_sample[0]) / self._total_rel
+        return self.DONT_KNOW
+
+    def get_order(self, _: Union[bytes, bytearray]) -> Tuple[int, int]:
+        return -1, 1
+
+
+class SJISContextAnalysis(JapaneseContextAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._charset_name = "SHIFT_JIS"
+
+    @property
+    def charset_name(self) -> str:
+        return self._charset_name
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> Tuple[int, int]:
+        if not byte_str:
+            return -1, 1
+        # find out current char's byte length
+        first_char = byte_str[0]
+        if (0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC):
+            char_len = 2
+            if (first_char == 0x87) or (0xFA <= first_char <= 0xFC):
+                self._charset_name = "CP932"
+        else:
+            char_len = 1
+
+        # return its order if it is hiragana
+        if len(byte_str) > 1:
+            second_char = byte_str[1]
+            if (first_char == 202) and (0x9F <= second_char <= 0xF1):
+                return second_char - 0x9F, char_len
+
+        return -1, char_len
+
+
+class EUCJPContextAnalysis(JapaneseContextAnalysis):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> Tuple[int, int]:
+        if not byte_str:
+            return -1, 1
+        # find out current char's byte length
+        first_char = byte_str[0]
+        if (first_char == 0x8E) or (0xA1 <= first_char <= 0xFE):
+            char_len = 2
+        elif first_char == 0x8F:
+            char_len = 3
+        else:
+            char_len = 1
+
+        # return its order if it is hiragana
+        if len(byte_str) > 1:
+            second_char = byte_str[1]
+            if (first_char == 0xA4) and (0xA1 <= second_char <= 0xF3):
+                return second_char - 0xA1, char_len
+
+        return -1, char_len
diff --git a/contrib/python/chardet/py3/chardet/langbulgarianmodel.py b/contrib/python/chardet/py3/chardet/langbulgarianmodel.py
new file mode 100644
index 00000000000..2f771bb8170
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/langbulgarianmodel.py
@@ -0,0 +1,4649 @@
+from chardet.sbcharsetprober import SingleByteCharSetModel
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+BULGARIAN_LANG_MODEL = {
+    63: {  # 'e'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 1,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 0,  # 'и'
+        26: 1,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 1,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 0,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    45: {  # '\xad'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 1,  # 'М'
+        36: 0,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 0,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    31: {  # 'А'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 2,  # 'З'
+        40: 1,  # 'И'
+        59: 1,  # 'Й'
+        33: 1,  # 'К'
+        46: 2,  # 'Л'
+        38: 1,  # 'М'
+        36: 2,  # 'Н'
+        41: 1,  # 'О'
+        30: 2,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 2,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 2,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 1,  # 'а'
+        18: 2,  # 'б'
+        9: 2,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 1,  # 'е'
+        23: 1,  # 'ж'
+        15: 2,  # 'з'
+        2: 0,  # 'и'
+        26: 2,  # 'й'
+        12: 2,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 0,  # 'о'
+        13: 2,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 1,  # 'у'
+        29: 2,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    32: {  # 'Б'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 2,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 1,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 2,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 2,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 2,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 2,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 1,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    35: {  # 'В'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 2,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 2,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 2,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    43: {  # 'Г'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 1,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    37: {  # 'Д'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 2,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 2,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    44: {  # 'Е'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 1,  # 'Й'
+        33: 2,  # 'К'
+        46: 2,  # 'Л'
+        38: 1,  # 'М'
+        36: 2,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 2,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 2,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 0,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 0,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 0,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 2,  # 'н'
+        4: 0,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 1,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    55: {  # 'Ж'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    47: {  # 'З'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 2,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 1,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 1,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    40: {  # 'И'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 2,  # 'З'
+        40: 1,  # 'И'
+        59: 1,  # 'Й'
+        33: 2,  # 'К'
+        46: 2,  # 'Л'
+        38: 2,  # 'М'
+        36: 2,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 0,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 2,  # 'Я'
+        1: 1,  # 'а'
+        18: 1,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 1,  # 'д'
+        3: 1,  # 'е'
+        23: 0,  # 'ж'
+        15: 3,  # 'з'
+        2: 0,  # 'и'
+        26: 1,  # 'й'
+        12: 1,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 2,  # 'н'
+        4: 0,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 0,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    59: {  # 'Й'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 1,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    33: {  # 'К'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 2,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 1,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 3,  # 'р'
+        8: 1,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    46: {  # 'Л'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 2,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 0,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    38: {  # 'М'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 0,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    36: {  # 'Н'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 2,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 1,  # 'Й'
+        33: 2,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 1,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 1,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 2,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    41: {  # 'О'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 1,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 1,  # 'Й'
+        33: 2,  # 'К'
+        46: 2,  # 'Л'
+        38: 2,  # 'М'
+        36: 2,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 1,  # 'а'
+        18: 2,  # 'б'
+        9: 2,  # 'в'
+        20: 2,  # 'г'
+        11: 1,  # 'д'
+        3: 1,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 0,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 0,  # 'о'
+        13: 2,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 2,  # 'ч'
+        27: 0,  # 'ш'
+        24: 2,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    30: {  # 'П'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 2,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 3,  # 'л'
+        14: 0,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 3,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 2,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    39: {  # 'Р'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 2,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 1,  # 'с'
+        5: 0,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    28: {  # 'С'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 3,  # 'А'
+        32: 2,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 2,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 2,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    34: {  # 'Т'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 2,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 2,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 3,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    51: {  # 'У'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 0,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 2,  # 'Т'
+        51: 0,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 2,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 2,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 2,  # 'с'
+        5: 1,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    48: {  # 'Ф'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    49: {  # 'Х'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    53: {  # 'Ц'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 2,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 2,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 1,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    50: {  # 'Ч'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 1,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    54: {  # 'Ш'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 2,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    57: {  # 'Щ'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 1,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 1,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    61: {  # 'Ъ'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 0,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 2,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 0,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 1,  # 'л'
+        14: 0,  # 'м'
+        6: 1,  # 'н'
+        4: 0,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    60: {  # 'Ю'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 0,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 0,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 2,  # 'г'
+        11: 1,  # 'д'
+        3: 0,  # 'е'
+        23: 2,  # 'ж'
+        15: 1,  # 'з'
+        2: 1,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 0,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    56: {  # 'Я'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 1,  # 'С'
+        34: 2,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 1,  # 'и'
+        26: 1,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 2,  # 'м'
+        6: 2,  # 'н'
+        4: 0,  # 'о'
+        13: 2,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    1: {  # 'а'
+        63: 1,  # 'e'
+        45: 1,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 3,  # 'и'
+        26: 3,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 3,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 3,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    18: {  # 'б'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 3,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 0,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 2,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 3,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    9: {  # 'в'
+        63: 1,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 1,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 0,  # 'в'
+        20: 2,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 3,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 2,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 3,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    20: {  # 'г'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 3,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    11: {  # 'д'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 2,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 1,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    3: {  # 'е'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 2,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 2,  # 'и'
+        26: 3,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 3,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 3,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    23: {  # 'ж'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    15: {  # 'з'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    2: {  # 'и'
+        63: 1,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 1,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 1,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 3,  # 'и'
+        26: 3,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 3,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 3,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    26: {  # 'й'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 2,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 2,  # 'з'
+        2: 1,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 2,  # 'ф'
+        25: 1,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    12: {  # 'к'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 3,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    10: {  # 'л'
+        63: 1,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 1,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 1,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 2,  # 'п'
+        7: 2,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 2,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 3,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    14: {  # 'м'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 3,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 1,  # 'т'
+        19: 3,  # 'у'
+        29: 2,  # 'ф'
+        25: 1,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    6: {  # 'н'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 1,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 2,  # 'б'
+        9: 2,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 2,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 3,  # 'ф'
+        25: 2,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    4: {  # 'о'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 3,  # 'и'
+        26: 3,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 3,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 3,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    13: {  # 'п'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 3,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    7: {  # 'р'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 2,  # 'п'
+        7: 1,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 2,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 2,  # 'ч'
+        27: 3,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    8: {  # 'с'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 2,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 1,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 2,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 2,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    5: {  # 'т'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 2,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    19: {  # 'у'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 2,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 2,  # 'и'
+        26: 2,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 2,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 2,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    29: {  # 'ф'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 1,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    25: {  # 'х'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 3,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 1,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    22: {  # 'ц'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 2,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 0,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    21: {  # 'ч'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 3,  # 'в'
+        20: 1,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    27: {  # 'ш'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 2,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 2,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 1,  # 'т'
+        19: 2,  # 'у'
+        29: 1,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    24: {  # 'щ'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 1,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    17: {  # 'ъ'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 2,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 1,  # 'и'
+        26: 2,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 3,  # 'ч'
+        27: 2,  # 'ш'
+        24: 3,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 2,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    52: {  # 'ь'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 1,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 1,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 1,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    42: {  # 'ю'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 2,  # 'б'
+        9: 1,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 1,  # 'е'
+        23: 2,  # 'ж'
+        15: 2,  # 'з'
+        2: 1,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 2,  # 'н'
+        4: 1,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 2,  # 'ц'
+        21: 3,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    16: {  # 'я'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 3,  # 'д'
+        3: 2,  # 'е'
+        23: 1,  # 'ж'
+        15: 2,  # 'з'
+        2: 1,  # 'и'
+        26: 2,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 1,  # 'о'
+        13: 2,  # 'п'
+        7: 2,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 3,  # 'х'
+        22: 2,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 2,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    58: {  # 'є'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 0,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    62: {  # '№'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 0,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
+
+# Character Mapping Table(s):
+ISO_8859_5_BULGARIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 77,  # 'A'
+    66: 90,  # 'B'
+    67: 99,  # 'C'
+    68: 100,  # 'D'
+    69: 72,  # 'E'
+    70: 109,  # 'F'
+    71: 107,  # 'G'
+    72: 101,  # 'H'
+    73: 79,  # 'I'
+    74: 185,  # 'J'
+    75: 81,  # 'K'
+    76: 102,  # 'L'
+    77: 76,  # 'M'
+    78: 94,  # 'N'
+    79: 82,  # 'O'
+    80: 110,  # 'P'
+    81: 186,  # 'Q'
+    82: 108,  # 'R'
+    83: 91,  # 'S'
+    84: 74,  # 'T'
+    85: 119,  # 'U'
+    86: 84,  # 'V'
+    87: 96,  # 'W'
+    88: 111,  # 'X'
+    89: 187,  # 'Y'
+    90: 115,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 65,  # 'a'
+    98: 69,  # 'b'
+    99: 70,  # 'c'
+    100: 66,  # 'd'
+    101: 63,  # 'e'
+    102: 68,  # 'f'
+    103: 112,  # 'g'
+    104: 103,  # 'h'
+    105: 92,  # 'i'
+    106: 194,  # 'j'
+    107: 104,  # 'k'
+    108: 95,  # 'l'
+    109: 86,  # 'm'
+    110: 87,  # 'n'
+    111: 71,  # 'o'
+    112: 116,  # 'p'
+    113: 195,  # 'q'
+    114: 85,  # 'r'
+    115: 93,  # 's'
+    116: 97,  # 't'
+    117: 113,  # 'u'
+    118: 196,  # 'v'
+    119: 197,  # 'w'
+    120: 198,  # 'x'
+    121: 199,  # 'y'
+    122: 200,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 194,  # '\x80'
+    129: 195,  # '\x81'
+    130: 196,  # '\x82'
+    131: 197,  # '\x83'
+    132: 198,  # '\x84'
+    133: 199,  # '\x85'
+    134: 200,  # '\x86'
+    135: 201,  # '\x87'
+    136: 202,  # '\x88'
+    137: 203,  # '\x89'
+    138: 204,  # '\x8a'
+    139: 205,  # '\x8b'
+    140: 206,  # '\x8c'
+    141: 207,  # '\x8d'
+    142: 208,  # '\x8e'
+    143: 209,  # '\x8f'
+    144: 210,  # '\x90'
+    145: 211,  # '\x91'
+    146: 212,  # '\x92'
+    147: 213,  # '\x93'
+    148: 214,  # '\x94'
+    149: 215,  # '\x95'
+    150: 216,  # '\x96'
+    151: 217,  # '\x97'
+    152: 218,  # '\x98'
+    153: 219,  # '\x99'
+    154: 220,  # '\x9a'
+    155: 221,  # '\x9b'
+    156: 222,  # '\x9c'
+    157: 223,  # '\x9d'
+    158: 224,  # '\x9e'
+    159: 225,  # '\x9f'
+    160: 81,  # '\xa0'
+    161: 226,  # 'Ё'
+    162: 227,  # 'Ђ'
+    163: 228,  # 'Ѓ'
+    164: 229,  # 'Є'
+    165: 230,  # 'Ѕ'
+    166: 105,  # 'І'
+    167: 231,  # 'Ї'
+    168: 232,  # 'Ј'
+    169: 233,  # 'Љ'
+    170: 234,  # 'Њ'
+    171: 235,  # 'Ћ'
+    172: 236,  # 'Ќ'
+    173: 45,  # '\xad'
+    174: 237,  # 'Ў'
+    175: 238,  # 'Џ'
+    176: 31,  # 'А'
+    177: 32,  # 'Б'
+    178: 35,  # 'В'
+    179: 43,  # 'Г'
+    180: 37,  # 'Д'
+    181: 44,  # 'Е'
+    182: 55,  # 'Ж'
+    183: 47,  # 'З'
+    184: 40,  # 'И'
+    185: 59,  # 'Й'
+    186: 33,  # 'К'
+    187: 46,  # 'Л'
+    188: 38,  # 'М'
+    189: 36,  # 'Н'
+    190: 41,  # 'О'
+    191: 30,  # 'П'
+    192: 39,  # 'Р'
+    193: 28,  # 'С'
+    194: 34,  # 'Т'
+    195: 51,  # 'У'
+    196: 48,  # 'Ф'
+    197: 49,  # 'Х'
+    198: 53,  # 'Ц'
+    199: 50,  # 'Ч'
+    200: 54,  # 'Ш'
+    201: 57,  # 'Щ'
+    202: 61,  # 'Ъ'
+    203: 239,  # 'Ы'
+    204: 67,  # 'Ь'
+    205: 240,  # 'Э'
+    206: 60,  # 'Ю'
+    207: 56,  # 'Я'
+    208: 1,  # 'а'
+    209: 18,  # 'б'
+    210: 9,  # 'в'
+    211: 20,  # 'г'
+    212: 11,  # 'д'
+    213: 3,  # 'е'
+    214: 23,  # 'ж'
+    215: 15,  # 'з'
+    216: 2,  # 'и'
+    217: 26,  # 'й'
+    218: 12,  # 'к'
+    219: 10,  # 'л'
+    220: 14,  # 'м'
+    221: 6,  # 'н'
+    222: 4,  # 'о'
+    223: 13,  # 'п'
+    224: 7,  # 'р'
+    225: 8,  # 'с'
+    226: 5,  # 'т'
+    227: 19,  # 'у'
+    228: 29,  # 'ф'
+    229: 25,  # 'х'
+    230: 22,  # 'ц'
+    231: 21,  # 'ч'
+    232: 27,  # 'ш'
+    233: 24,  # 'щ'
+    234: 17,  # 'ъ'
+    235: 75,  # 'ы'
+    236: 52,  # 'ь'
+    237: 241,  # 'э'
+    238: 42,  # 'ю'
+    239: 16,  # 'я'
+    240: 62,  # '№'
+    241: 242,  # 'ё'
+    242: 243,  # 'ђ'
+    243: 244,  # 'ѓ'
+    244: 58,  # 'є'
+    245: 245,  # 'ѕ'
+    246: 98,  # 'і'
+    247: 246,  # 'ї'
+    248: 247,  # 'ј'
+    249: 248,  # 'љ'
+    250: 249,  # 'њ'
+    251: 250,  # 'ћ'
+    252: 251,  # 'ќ'
+    253: 91,  # '§'
+    254: 252,  # 'ў'
+    255: 253,  # 'џ'
+}
+
+ISO_8859_5_BULGARIAN_MODEL = SingleByteCharSetModel(
+    charset_name="ISO-8859-5",
+    language="Bulgarian",
+    char_to_order_map=ISO_8859_5_BULGARIAN_CHAR_TO_ORDER,
+    language_model=BULGARIAN_LANG_MODEL,
+    typical_positive_ratio=0.969392,
+    keep_ascii_letters=False,
+    alphabet="АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя",
+)
+
+WINDOWS_1251_BULGARIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 77,  # 'A'
+    66: 90,  # 'B'
+    67: 99,  # 'C'
+    68: 100,  # 'D'
+    69: 72,  # 'E'
+    70: 109,  # 'F'
+    71: 107,  # 'G'
+    72: 101,  # 'H'
+    73: 79,  # 'I'
+    74: 185,  # 'J'
+    75: 81,  # 'K'
+    76: 102,  # 'L'
+    77: 76,  # 'M'
+    78: 94,  # 'N'
+    79: 82,  # 'O'
+    80: 110,  # 'P'
+    81: 186,  # 'Q'
+    82: 108,  # 'R'
+    83: 91,  # 'S'
+    84: 74,  # 'T'
+    85: 119,  # 'U'
+    86: 84,  # 'V'
+    87: 96,  # 'W'
+    88: 111,  # 'X'
+    89: 187,  # 'Y'
+    90: 115,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 65,  # 'a'
+    98: 69,  # 'b'
+    99: 70,  # 'c'
+    100: 66,  # 'd'
+    101: 63,  # 'e'
+    102: 68,  # 'f'
+    103: 112,  # 'g'
+    104: 103,  # 'h'
+    105: 92,  # 'i'
+    106: 194,  # 'j'
+    107: 104,  # 'k'
+    108: 95,  # 'l'
+    109: 86,  # 'm'
+    110: 87,  # 'n'
+    111: 71,  # 'o'
+    112: 116,  # 'p'
+    113: 195,  # 'q'
+    114: 85,  # 'r'
+    115: 93,  # 's'
+    116: 97,  # 't'
+    117: 113,  # 'u'
+    118: 196,  # 'v'
+    119: 197,  # 'w'
+    120: 198,  # 'x'
+    121: 199,  # 'y'
+    122: 200,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 206,  # 'Ђ'
+    129: 207,  # 'Ѓ'
+    130: 208,  # '‚'
+    131: 209,  # 'ѓ'
+    132: 210,  # '„'
+    133: 211,  # '…'
+    134: 212,  # '†'
+    135: 213,  # '‡'
+    136: 120,  # '€'
+    137: 214,  # '‰'
+    138: 215,  # 'Љ'
+    139: 216,  # '‹'
+    140: 217,  # 'Њ'
+    141: 218,  # 'Ќ'
+    142: 219,  # 'Ћ'
+    143: 220,  # 'Џ'
+    144: 221,  # 'ђ'
+    145: 78,  # '‘'
+    146: 64,  # '’'
+    147: 83,  # '“'
+    148: 121,  # '”'
+    149: 98,  # '•'
+    150: 117,  # '–'
+    151: 105,  # '—'
+    152: 222,  # None
+    153: 223,  # '™'
+    154: 224,  # 'љ'
+    155: 225,  # '›'
+    156: 226,  # 'њ'
+    157: 227,  # 'ќ'
+    158: 228,  # 'ћ'
+    159: 229,  # 'џ'
+    160: 88,  # '\xa0'
+    161: 230,  # 'Ў'
+    162: 231,  # 'ў'
+    163: 232,  # 'Ј'
+    164: 233,  # '¤'
+    165: 122,  # 'Ґ'
+    166: 89,  # '¦'
+    167: 106,  # '§'
+    168: 234,  # 'Ё'
+    169: 235,  # '©'
+    170: 236,  # 'Є'
+    171: 237,  # '«'
+    172: 238,  # '¬'
+    173: 45,  # '\xad'
+    174: 239,  # '®'
+    175: 240,  # 'Ї'
+    176: 73,  # '°'
+    177: 80,  # '±'
+    178: 118,  # 'І'
+    179: 114,  # 'і'
+    180: 241,  # 'ґ'
+    181: 242,  # 'µ'
+    182: 243,  # '¶'
+    183: 244,  # '·'
+    184: 245,  # 'ё'
+    185: 62,  # '№'
+    186: 58,  # 'є'
+    187: 246,  # '»'
+    188: 247,  # 'ј'
+    189: 248,  # 'Ѕ'
+    190: 249,  # 'ѕ'
+    191: 250,  # 'ї'
+    192: 31,  # 'А'
+    193: 32,  # 'Б'
+    194: 35,  # 'В'
+    195: 43,  # 'Г'
+    196: 37,  # 'Д'
+    197: 44,  # 'Е'
+    198: 55,  # 'Ж'
+    199: 47,  # 'З'
+    200: 40,  # 'И'
+    201: 59,  # 'Й'
+    202: 33,  # 'К'
+    203: 46,  # 'Л'
+    204: 38,  # 'М'
+    205: 36,  # 'Н'
+    206: 41,  # 'О'
+    207: 30,  # 'П'
+    208: 39,  # 'Р'
+    209: 28,  # 'С'
+    210: 34,  # 'Т'
+    211: 51,  # 'У'
+    212: 48,  # 'Ф'
+    213: 49,  # 'Х'
+    214: 53,  # 'Ц'
+    215: 50,  # 'Ч'
+    216: 54,  # 'Ш'
+    217: 57,  # 'Щ'
+    218: 61,  # 'Ъ'
+    219: 251,  # 'Ы'
+    220: 67,  # 'Ь'
+    221: 252,  # 'Э'
+    222: 60,  # 'Ю'
+    223: 56,  # 'Я'
+    224: 1,  # 'а'
+    225: 18,  # 'б'
+    226: 9,  # 'в'
+    227: 20,  # 'г'
+    228: 11,  # 'д'
+    229: 3,  # 'е'
+    230: 23,  # 'ж'
+    231: 15,  # 'з'
+    232: 2,  # 'и'
+    233: 26,  # 'й'
+    234: 12,  # 'к'
+    235: 10,  # 'л'
+    236: 14,  # 'м'
+    237: 6,  # 'н'
+    238: 4,  # 'о'
+    239: 13,  # 'п'
+    240: 7,  # 'р'
+    241: 8,  # 'с'
+    242: 5,  # 'т'
+    243: 19,  # 'у'
+    244: 29,  # 'ф'
+    245: 25,  # 'х'
+    246: 22,  # 'ц'
+    247: 21,  # 'ч'
+    248: 27,  # 'ш'
+    249: 24,  # 'щ'
+    250: 17,  # 'ъ'
+    251: 75,  # 'ы'
+    252: 52,  # 'ь'
+    253: 253,  # 'э'
+    254: 42,  # 'ю'
+    255: 16,  # 'я'
+}
+
+WINDOWS_1251_BULGARIAN_MODEL = SingleByteCharSetModel(
+    charset_name="windows-1251",
+    language="Bulgarian",
+    char_to_order_map=WINDOWS_1251_BULGARIAN_CHAR_TO_ORDER,
+    language_model=BULGARIAN_LANG_MODEL,
+    typical_positive_ratio=0.969392,
+    keep_ascii_letters=False,
+    alphabet="АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя",
+)
diff --git a/contrib/python/chardet/py3/chardet/langgreekmodel.py b/contrib/python/chardet/py3/chardet/langgreekmodel.py
new file mode 100644
index 00000000000..0471d8bb189
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/langgreekmodel.py
@@ -0,0 +1,4397 @@
+from chardet.sbcharsetprober import SingleByteCharSetModel
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+GREEK_LANG_MODEL = {
+    60: {  # 'e'
+        60: 2,  # 'e'
+        55: 1,  # 'o'
+        58: 2,  # 't'
+        36: 1,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    55: {  # 'o'
+        60: 0,  # 'e'
+        55: 2,  # 'o'
+        58: 2,  # 't'
+        36: 1,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 1,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 1,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    58: {  # 't'
+        60: 2,  # 'e'
+        55: 1,  # 'o'
+        58: 1,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 1,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    36: {  # '·'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    61: {  # 'Ά'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 1,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 1,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    46: {  # 'Έ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 2,  # 'β'
+        20: 2,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 2,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 1,  # 'σ'
+        2: 2,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    54: {  # 'Ό'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 2,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    31: {  # 'Α'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 2,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 1,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 2,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 1,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 2,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 2,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 1,  # 'θ'
+        5: 0,  # 'ι'
+        11: 2,  # 'κ'
+        16: 3,  # 'λ'
+        10: 2,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 2,  # 'ς'
+        7: 2,  # 'σ'
+        2: 0,  # 'τ'
+        12: 3,  # 'υ'
+        28: 2,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    51: {  # 'Β'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 1,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 1,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 2,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    43: {  # 'Γ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 1,  # 'Α'
+        51: 0,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 1,  # 'Κ'
+        53: 1,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 1,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    41: {  # 'Δ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 2,  # 'ή'
+        15: 2,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 1,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    34: {  # 'Ε'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 2,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 1,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 2,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 3,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 1,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 1,  # 'θ'
+        5: 2,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 2,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 2,  # 'τ'
+        12: 2,  # 'υ'
+        28: 2,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 1,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    40: {  # 'Η'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 1,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 2,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 1,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 1,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 1,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    52: {  # 'Θ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 1,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 1,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    47: {  # 'Ι'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 1,  # 'Β'
+        43: 1,  # 'Γ'
+        41: 2,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 2,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 1,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 1,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 1,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    44: {  # 'Κ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 1,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 1,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 1,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 1,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    53: {  # 'Λ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 1,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    38: {  # 'Μ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 2,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 2,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    49: {  # 'Ν'
+        60: 2,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 1,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 1,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    59: {  # 'Ξ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 1,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    39: {  # 'Ο'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 1,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 2,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 2,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 2,  # 'Φ'
+        50: 2,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 2,  # 'κ'
+        16: 2,  # 'λ'
+        10: 2,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 2,  # 'τ'
+        12: 2,  # 'υ'
+        28: 1,  # 'φ'
+        23: 1,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    35: {  # 'Π'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 1,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 1,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 1,  # 'έ'
+        22: 1,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    48: {  # 'Ρ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 1,  # 'Γ'
+        41: 1,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 1,  # 'Τ'
+        45: 1,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 1,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 1,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    37: {  # 'Σ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 1,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 2,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 2,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 2,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 2,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 2,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    33: {  # 'Τ'
+        60: 0,  # 'e'
+        55: 1,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 1,  # 'Τ'
+        45: 1,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 2,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    45: {  # 'Υ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 2,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 1,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 1,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    56: {  # 'Φ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 1,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 2,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 1,  # 'ύ'
+        27: 1,  # 'ώ'
+    },
+    50: {  # 'Χ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 1,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 1,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 1,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 1,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 2,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    57: {  # 'Ω'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 1,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 1,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 2,  # 'ς'
+        7: 2,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 1,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    17: {  # 'ά'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 3,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 2,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 3,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    18: {  # 'έ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 3,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 3,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    22: {  # 'ή'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 1,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    15: {  # 'ί'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 3,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 1,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    1: {  # 'α'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 3,  # 'ί'
+        1: 0,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 2,  # 'ε'
+        32: 3,  # 'ζ'
+        13: 1,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    29: {  # 'β'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 2,  # 'γ'
+        21: 2,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 3,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    20: {  # 'γ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 3,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 3,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    21: {  # 'δ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    3: {  # 'ε'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 3,  # 'ί'
+        1: 2,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 2,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 2,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    32: {  # 'ζ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 2,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 1,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    13: {  # 'η'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    25: {  # 'θ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 1,  # 'λ'
+        10: 3,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    5: {  # 'ι'
+        60: 0,  # 'e'
+        55: 1,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    11: {  # 'κ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 2,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 2,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 2,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    16: {  # 'λ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 1,  # 'β'
+        20: 2,  # 'γ'
+        21: 1,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 2,  # 'θ'
+        5: 3,  # 'ι'
+        11: 2,  # 'κ'
+        16: 3,  # 'λ'
+        10: 2,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 2,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    10: {  # 'μ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 3,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 3,  # 'φ'
+        23: 0,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    6: {  # 'ν'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 1,  # 'λ'
+        10: 0,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    30: {  # 'ξ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 2,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 1,  # 'ώ'
+    },
+    4: {  # 'ο'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 2,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 1,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    9: {  # 'π'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 3,  # 'λ'
+        10: 0,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 2,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    8: {  # 'ρ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 1,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 2,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    14: {  # 'ς'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    7: {  # 'σ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 3,  # 'β'
+        20: 0,  # 'γ'
+        21: 2,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    2: {  # 'τ'
+        60: 0,  # 'e'
+        55: 2,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 2,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 2,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    12: {  # 'υ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 2,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 2,  # 'η'
+        25: 3,  # 'θ'
+        5: 2,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    28: {  # 'φ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 2,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 1,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 1,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    23: {  # 'χ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 2,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 2,  # 'μ'
+        6: 3,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    42: {  # 'ψ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 1,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 2,  # 'τ'
+        12: 1,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    24: {  # 'ω'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 1,  # 'ά'
+        18: 0,  # 'έ'
+        22: 2,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    19: {  # 'ό'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 1,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 2,  # 'η'
+        25: 2,  # 'θ'
+        5: 2,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 1,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    26: {  # 'ύ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 2,  # 'β'
+        20: 2,  # 'γ'
+        21: 1,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 2,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    27: {  # 'ώ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 1,  # 'β'
+        20: 0,  # 'γ'
+        21: 3,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 1,  # 'η'
+        25: 2,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 1,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 1,  # 'φ'
+        23: 1,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
+
+# Character Mapping Table(s):
+WINDOWS_1253_GREEK_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 82,  # 'A'
+    66: 100,  # 'B'
+    67: 104,  # 'C'
+    68: 94,  # 'D'
+    69: 98,  # 'E'
+    70: 101,  # 'F'
+    71: 116,  # 'G'
+    72: 102,  # 'H'
+    73: 111,  # 'I'
+    74: 187,  # 'J'
+    75: 117,  # 'K'
+    76: 92,  # 'L'
+    77: 88,  # 'M'
+    78: 113,  # 'N'
+    79: 85,  # 'O'
+    80: 79,  # 'P'
+    81: 118,  # 'Q'
+    82: 105,  # 'R'
+    83: 83,  # 'S'
+    84: 67,  # 'T'
+    85: 114,  # 'U'
+    86: 119,  # 'V'
+    87: 95,  # 'W'
+    88: 99,  # 'X'
+    89: 109,  # 'Y'
+    90: 188,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 72,  # 'a'
+    98: 70,  # 'b'
+    99: 80,  # 'c'
+    100: 81,  # 'd'
+    101: 60,  # 'e'
+    102: 96,  # 'f'
+    103: 93,  # 'g'
+    104: 89,  # 'h'
+    105: 68,  # 'i'
+    106: 120,  # 'j'
+    107: 97,  # 'k'
+    108: 77,  # 'l'
+    109: 86,  # 'm'
+    110: 69,  # 'n'
+    111: 55,  # 'o'
+    112: 78,  # 'p'
+    113: 115,  # 'q'
+    114: 65,  # 'r'
+    115: 66,  # 's'
+    116: 58,  # 't'
+    117: 76,  # 'u'
+    118: 106,  # 'v'
+    119: 103,  # 'w'
+    120: 87,  # 'x'
+    121: 107,  # 'y'
+    122: 112,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 255,  # '€'
+    129: 255,  # None
+    130: 255,  # '‚'
+    131: 255,  # 'ƒ'
+    132: 255,  # '„'
+    133: 255,  # '…'
+    134: 255,  # '†'
+    135: 255,  # '‡'
+    136: 255,  # None
+    137: 255,  # '‰'
+    138: 255,  # None
+    139: 255,  # '‹'
+    140: 255,  # None
+    141: 255,  # None
+    142: 255,  # None
+    143: 255,  # None
+    144: 255,  # None
+    145: 255,  # '‘'
+    146: 255,  # '’'
+    147: 255,  # '“'
+    148: 255,  # '”'
+    149: 255,  # '•'
+    150: 255,  # '–'
+    151: 255,  # '—'
+    152: 255,  # None
+    153: 255,  # '™'
+    154: 255,  # None
+    155: 255,  # '›'
+    156: 255,  # None
+    157: 255,  # None
+    158: 255,  # None
+    159: 255,  # None
+    160: 253,  # '\xa0'
+    161: 233,  # '΅'
+    162: 61,  # 'Ά'
+    163: 253,  # '£'
+    164: 253,  # '¤'
+    165: 253,  # '¥'
+    166: 253,  # '¦'
+    167: 253,  # '§'
+    168: 253,  # '¨'
+    169: 253,  # '©'
+    170: 253,  # None
+    171: 253,  # '«'
+    172: 253,  # '¬'
+    173: 74,  # '\xad'
+    174: 253,  # '®'
+    175: 253,  # '―'
+    176: 253,  # '°'
+    177: 253,  # '±'
+    178: 253,  # '²'
+    179: 253,  # '³'
+    180: 247,  # '΄'
+    181: 253,  # 'µ'
+    182: 253,  # '¶'
+    183: 36,  # '·'
+    184: 46,  # 'Έ'
+    185: 71,  # 'Ή'
+    186: 73,  # 'Ί'
+    187: 253,  # '»'
+    188: 54,  # 'Ό'
+    189: 253,  # '½'
+    190: 108,  # 'Ύ'
+    191: 123,  # 'Ώ'
+    192: 110,  # 'ΐ'
+    193: 31,  # 'Α'
+    194: 51,  # 'Β'
+    195: 43,  # 'Γ'
+    196: 41,  # 'Δ'
+    197: 34,  # 'Ε'
+    198: 91,  # 'Ζ'
+    199: 40,  # 'Η'
+    200: 52,  # 'Θ'
+    201: 47,  # 'Ι'
+    202: 44,  # 'Κ'
+    203: 53,  # 'Λ'
+    204: 38,  # 'Μ'
+    205: 49,  # 'Ν'
+    206: 59,  # 'Ξ'
+    207: 39,  # 'Ο'
+    208: 35,  # 'Π'
+    209: 48,  # 'Ρ'
+    210: 250,  # None
+    211: 37,  # 'Σ'
+    212: 33,  # 'Τ'
+    213: 45,  # 'Υ'
+    214: 56,  # 'Φ'
+    215: 50,  # 'Χ'
+    216: 84,  # 'Ψ'
+    217: 57,  # 'Ω'
+    218: 120,  # 'Ϊ'
+    219: 121,  # 'Ϋ'
+    220: 17,  # 'ά'
+    221: 18,  # 'έ'
+    222: 22,  # 'ή'
+    223: 15,  # 'ί'
+    224: 124,  # 'ΰ'
+    225: 1,  # 'α'
+    226: 29,  # 'β'
+    227: 20,  # 'γ'
+    228: 21,  # 'δ'
+    229: 3,  # 'ε'
+    230: 32,  # 'ζ'
+    231: 13,  # 'η'
+    232: 25,  # 'θ'
+    233: 5,  # 'ι'
+    234: 11,  # 'κ'
+    235: 16,  # 'λ'
+    236: 10,  # 'μ'
+    237: 6,  # 'ν'
+    238: 30,  # 'ξ'
+    239: 4,  # 'ο'
+    240: 9,  # 'π'
+    241: 8,  # 'ρ'
+    242: 14,  # 'ς'
+    243: 7,  # 'σ'
+    244: 2,  # 'τ'
+    245: 12,  # 'υ'
+    246: 28,  # 'φ'
+    247: 23,  # 'χ'
+    248: 42,  # 'ψ'
+    249: 24,  # 'ω'
+    250: 64,  # 'ϊ'
+    251: 75,  # 'ϋ'
+    252: 19,  # 'ό'
+    253: 26,  # 'ύ'
+    254: 27,  # 'ώ'
+    255: 253,  # None
+}
+
+WINDOWS_1253_GREEK_MODEL = SingleByteCharSetModel(
+    charset_name="windows-1253",
+    language="Greek",
+    char_to_order_map=WINDOWS_1253_GREEK_CHAR_TO_ORDER,
+    language_model=GREEK_LANG_MODEL,
+    typical_positive_ratio=0.982851,
+    keep_ascii_letters=False,
+    alphabet="ΆΈΉΊΌΎΏΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩάέήίαβγδεζηθικλμνξοπρςστυφχψωόύώ",
+)
+
+ISO_8859_7_GREEK_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 82,  # 'A'
+    66: 100,  # 'B'
+    67: 104,  # 'C'
+    68: 94,  # 'D'
+    69: 98,  # 'E'
+    70: 101,  # 'F'
+    71: 116,  # 'G'
+    72: 102,  # 'H'
+    73: 111,  # 'I'
+    74: 187,  # 'J'
+    75: 117,  # 'K'
+    76: 92,  # 'L'
+    77: 88,  # 'M'
+    78: 113,  # 'N'
+    79: 85,  # 'O'
+    80: 79,  # 'P'
+    81: 118,  # 'Q'
+    82: 105,  # 'R'
+    83: 83,  # 'S'
+    84: 67,  # 'T'
+    85: 114,  # 'U'
+    86: 119,  # 'V'
+    87: 95,  # 'W'
+    88: 99,  # 'X'
+    89: 109,  # 'Y'
+    90: 188,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 72,  # 'a'
+    98: 70,  # 'b'
+    99: 80,  # 'c'
+    100: 81,  # 'd'
+    101: 60,  # 'e'
+    102: 96,  # 'f'
+    103: 93,  # 'g'
+    104: 89,  # 'h'
+    105: 68,  # 'i'
+    106: 120,  # 'j'
+    107: 97,  # 'k'
+    108: 77,  # 'l'
+    109: 86,  # 'm'
+    110: 69,  # 'n'
+    111: 55,  # 'o'
+    112: 78,  # 'p'
+    113: 115,  # 'q'
+    114: 65,  # 'r'
+    115: 66,  # 's'
+    116: 58,  # 't'
+    117: 76,  # 'u'
+    118: 106,  # 'v'
+    119: 103,  # 'w'
+    120: 87,  # 'x'
+    121: 107,  # 'y'
+    122: 112,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 255,  # '\x80'
+    129: 255,  # '\x81'
+    130: 255,  # '\x82'
+    131: 255,  # '\x83'
+    132: 255,  # '\x84'
+    133: 255,  # '\x85'
+    134: 255,  # '\x86'
+    135: 255,  # '\x87'
+    136: 255,  # '\x88'
+    137: 255,  # '\x89'
+    138: 255,  # '\x8a'
+    139: 255,  # '\x8b'
+    140: 255,  # '\x8c'
+    141: 255,  # '\x8d'
+    142: 255,  # '\x8e'
+    143: 255,  # '\x8f'
+    144: 255,  # '\x90'
+    145: 255,  # '\x91'
+    146: 255,  # '\x92'
+    147: 255,  # '\x93'
+    148: 255,  # '\x94'
+    149: 255,  # '\x95'
+    150: 255,  # '\x96'
+    151: 255,  # '\x97'
+    152: 255,  # '\x98'
+    153: 255,  # '\x99'
+    154: 255,  # '\x9a'
+    155: 255,  # '\x9b'
+    156: 255,  # '\x9c'
+    157: 255,  # '\x9d'
+    158: 255,  # '\x9e'
+    159: 255,  # '\x9f'
+    160: 253,  # '\xa0'
+    161: 233,  # '‘'
+    162: 90,  # '’'
+    163: 253,  # '£'
+    164: 253,  # '€'
+    165: 253,  # '₯'
+    166: 253,  # '¦'
+    167: 253,  # '§'
+    168: 253,  # '¨'
+    169: 253,  # '©'
+    170: 253,  # 'ͺ'
+    171: 253,  # '«'
+    172: 253,  # '¬'
+    173: 74,  # '\xad'
+    174: 253,  # None
+    175: 253,  # '―'
+    176: 253,  # '°'
+    177: 253,  # '±'
+    178: 253,  # '²'
+    179: 253,  # '³'
+    180: 247,  # '΄'
+    181: 248,  # '΅'
+    182: 61,  # 'Ά'
+    183: 36,  # '·'
+    184: 46,  # 'Έ'
+    185: 71,  # 'Ή'
+    186: 73,  # 'Ί'
+    187: 253,  # '»'
+    188: 54,  # 'Ό'
+    189: 253,  # '½'
+    190: 108,  # 'Ύ'
+    191: 123,  # 'Ώ'
+    192: 110,  # 'ΐ'
+    193: 31,  # 'Α'
+    194: 51,  # 'Β'
+    195: 43,  # 'Γ'
+    196: 41,  # 'Δ'
+    197: 34,  # 'Ε'
+    198: 91,  # 'Ζ'
+    199: 40,  # 'Η'
+    200: 52,  # 'Θ'
+    201: 47,  # 'Ι'
+    202: 44,  # 'Κ'
+    203: 53,  # 'Λ'
+    204: 38,  # 'Μ'
+    205: 49,  # 'Ν'
+    206: 59,  # 'Ξ'
+    207: 39,  # 'Ο'
+    208: 35,  # 'Π'
+    209: 48,  # 'Ρ'
+    210: 250,  # None
+    211: 37,  # 'Σ'
+    212: 33,  # 'Τ'
+    213: 45,  # 'Υ'
+    214: 56,  # 'Φ'
+    215: 50,  # 'Χ'
+    216: 84,  # 'Ψ'
+    217: 57,  # 'Ω'
+    218: 120,  # 'Ϊ'
+    219: 121,  # 'Ϋ'
+    220: 17,  # 'ά'
+    221: 18,  # 'έ'
+    222: 22,  # 'ή'
+    223: 15,  # 'ί'
+    224: 124,  # 'ΰ'
+    225: 1,  # 'α'
+    226: 29,  # 'β'
+    227: 20,  # 'γ'
+    228: 21,  # 'δ'
+    229: 3,  # 'ε'
+    230: 32,  # 'ζ'
+    231: 13,  # 'η'
+    232: 25,  # 'θ'
+    233: 5,  # 'ι'
+    234: 11,  # 'κ'
+    235: 16,  # 'λ'
+    236: 10,  # 'μ'
+    237: 6,  # 'ν'
+    238: 30,  # 'ξ'
+    239: 4,  # 'ο'
+    240: 9,  # 'π'
+    241: 8,  # 'ρ'
+    242: 14,  # 'ς'
+    243: 7,  # 'σ'
+    244: 2,  # 'τ'
+    245: 12,  # 'υ'
+    246: 28,  # 'φ'
+    247: 23,  # 'χ'
+    248: 42,  # 'ψ'
+    249: 24,  # 'ω'
+    250: 64,  # 'ϊ'
+    251: 75,  # 'ϋ'
+    252: 19,  # 'ό'
+    253: 26,  # 'ύ'
+    254: 27,  # 'ώ'
+    255: 253,  # None
+}
+
+ISO_8859_7_GREEK_MODEL = SingleByteCharSetModel(
+    charset_name="ISO-8859-7",
+    language="Greek",
+    char_to_order_map=ISO_8859_7_GREEK_CHAR_TO_ORDER,
+    language_model=GREEK_LANG_MODEL,
+    typical_positive_ratio=0.982851,
+    keep_ascii_letters=False,
+    alphabet="ΆΈΉΊΌΎΏΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩάέήίαβγδεζηθικλμνξοπρςστυφχψωόύώ",
+)
diff --git a/contrib/python/chardet/py3/chardet/langhebrewmodel.py b/contrib/python/chardet/py3/chardet/langhebrewmodel.py
new file mode 100644
index 00000000000..86b3c5e6455
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/langhebrewmodel.py
@@ -0,0 +1,4380 @@
+from chardet.sbcharsetprober import SingleByteCharSetModel
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+HEBREW_LANG_MODEL = {
+    50: {  # 'a'
+        50: 0,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 2,  # 'l'
+        54: 2,  # 'n'
+        49: 0,  # 'o'
+        51: 2,  # 'r'
+        43: 1,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 1,  # 'ק'
+        7: 0,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    60: {  # 'c'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 0,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 0,  # 'n'
+        49: 1,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    61: {  # 'd'
+        50: 1,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 2,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 0,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 1,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    42: {  # 'e'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 2,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 2,  # 'l'
+        54: 2,  # 'n'
+        49: 1,  # 'o'
+        51: 2,  # 'r'
+        43: 2,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 1,  # '–'
+        52: 2,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    53: {  # 'i'
+        50: 1,  # 'a'
+        60: 2,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 0,  # 'i'
+        56: 1,  # 'l'
+        54: 2,  # 'n'
+        49: 2,  # 'o'
+        51: 1,  # 'r'
+        43: 2,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    56: {  # 'l'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 2,  # 'e'
+        53: 2,  # 'i'
+        56: 2,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 0,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    54: {  # 'n'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 0,  # 'r'
+        43: 1,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 2,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    49: {  # 'o'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 2,  # 'n'
+        49: 1,  # 'o'
+        51: 2,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    51: {  # 'r'
+        50: 2,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 2,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 2,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 2,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    43: {  # 's'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 0,  # 'd'
+        42: 2,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 2,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    44: {  # 't'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 0,  # 'd'
+        42: 2,  # 'e'
+        53: 2,  # 'i'
+        56: 1,  # 'l'
+        54: 0,  # 'n'
+        49: 1,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 2,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    63: {  # 'u'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 0,  # 'o'
+        51: 1,  # 'r'
+        43: 2,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    34: {  # '\xa0'
+        50: 1,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 0,  # 'e'
+        53: 1,  # 'i'
+        56: 0,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 0,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 2,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 2,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 2,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    55: {  # '´'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 1,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 2,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 1,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    48: {  # '¼'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    39: {  # '½'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    57: {  # '¾'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    30: {  # 'ְ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 2,  # 'ו'
+        24: 2,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 2,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 1,  # 'ם'
+        6: 2,  # 'מ'
+        23: 0,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 2,  # 'ע'
+        26: 0,  # 'ף'
+        18: 2,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    59: {  # 'ֱ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 1,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 0,  # 'ם'
+        6: 2,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    41: {  # 'ֲ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 1,  # 'י'
+        25: 1,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 0,  # 'ם'
+        6: 2,  # 'מ'
+        23: 0,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 1,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    33: {  # 'ִ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 1,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 1,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 2,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    37: {  # 'ֵ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 2,  # 'ח'
+        22: 1,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 1,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    36: {  # 'ֶ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 2,  # 'ח'
+        22: 1,  # 'ט'
+        1: 2,  # 'י'
+        25: 2,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 1,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    31: {  # 'ַ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 1,  # 'ו'
+        24: 2,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 2,  # 'ע'
+        26: 2,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    29: {  # 'ָ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 1,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 3,  # 'ה'
+        2: 2,  # 'ו'
+        24: 2,  # 'ז'
+        14: 2,  # 'ח'
+        22: 1,  # 'ט'
+        1: 2,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    35: {  # 'ֹ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 1,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    62: {  # 'ֻ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 1,  # 'ם'
+        6: 1,  # 'מ'
+        23: 1,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    28: {  # 'ּ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 3,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 1,  # 'ֲ'
+        33: 3,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 3,  # 'ַ'
+        29: 3,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 2,  # 'ׁ'
+        45: 1,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 1,  # 'ה'
+        2: 2,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 2,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 1,  # 'ם'
+        6: 2,  # 'מ'
+        23: 1,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    38: {  # 'ׁ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 2,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    45: {  # 'ׂ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 1,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 2,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 1,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 0,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    9: {  # 'א'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 2,  # 'ֱ'
+        41: 2,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 2,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    8: {  # 'ב'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 3,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 1,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    20: {  # 'ג'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 2,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 1,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 1,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    16: {  # 'ד'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 1,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    3: {  # 'ה'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 1,  # 'ֱ'
+        41: 2,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 3,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 0,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    2: {  # 'ו'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 3,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 3,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 3,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    24: {  # 'ז'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 1,  # 'ֲ'
+        33: 1,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 2,  # 'ח'
+        22: 1,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 1,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    14: {  # 'ח'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 1,  # 'ֱ'
+        41: 2,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 1,  # 'ע'
+        26: 2,  # 'ף'
+        18: 2,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    22: {  # 'ט'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 1,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 1,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 3,  # 'ר'
+        10: 2,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    1: {  # 'י'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 3,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    25: {  # 'ך'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 1,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    15: {  # 'כ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 3,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 2,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    4: {  # 'ל'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 3,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    11: {  # 'ם'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 1,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    6: {  # 'מ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 0,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    23: {  # 'ן'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 1,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    12: {  # 'נ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    19: {  # 'ס'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 2,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 1,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 1,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    13: {  # 'ע'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 1,  # 'ֱ'
+        41: 2,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 1,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 2,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    26: {  # 'ף'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 1,  # 'ס'
+        13: 0,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    18: {  # 'פ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 2,  # 'ב'
+        20: 3,  # 'ג'
+        16: 2,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 2,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    27: {  # 'ץ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 1,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 0,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    21: {  # 'צ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 1,  # 'ז'
+        14: 3,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 1,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 1,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 0,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    17: {  # 'ק'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 1,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 2,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    7: {  # 'ר'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 2,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 1,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 3,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    10: {  # 'ש'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 1,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 3,  # 'ׁ'
+        45: 2,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    5: {  # 'ת'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 1,  # '¼'
+        39: 1,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 2,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    32: {  # '–'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 1,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    52: {  # '’'
+        50: 1,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 1,  # 'r'
+        43: 2,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    47: {  # '“'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    46: {  # '”'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    58: {  # '†'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 2,  # '†'
+        40: 0,  # '…'
+    },
+    40: {  # '…'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 0,  # 'l'
+        54: 1,  # 'n'
+        49: 0,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
+
+# Character Mapping Table(s):
+WINDOWS_1255_HEBREW_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 69,  # 'A'
+    66: 91,  # 'B'
+    67: 79,  # 'C'
+    68: 80,  # 'D'
+    69: 92,  # 'E'
+    70: 89,  # 'F'
+    71: 97,  # 'G'
+    72: 90,  # 'H'
+    73: 68,  # 'I'
+    74: 111,  # 'J'
+    75: 112,  # 'K'
+    76: 82,  # 'L'
+    77: 73,  # 'M'
+    78: 95,  # 'N'
+    79: 85,  # 'O'
+    80: 78,  # 'P'
+    81: 121,  # 'Q'
+    82: 86,  # 'R'
+    83: 71,  # 'S'
+    84: 67,  # 'T'
+    85: 102,  # 'U'
+    86: 107,  # 'V'
+    87: 84,  # 'W'
+    88: 114,  # 'X'
+    89: 103,  # 'Y'
+    90: 115,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 50,  # 'a'
+    98: 74,  # 'b'
+    99: 60,  # 'c'
+    100: 61,  # 'd'
+    101: 42,  # 'e'
+    102: 76,  # 'f'
+    103: 70,  # 'g'
+    104: 64,  # 'h'
+    105: 53,  # 'i'
+    106: 105,  # 'j'
+    107: 93,  # 'k'
+    108: 56,  # 'l'
+    109: 65,  # 'm'
+    110: 54,  # 'n'
+    111: 49,  # 'o'
+    112: 66,  # 'p'
+    113: 110,  # 'q'
+    114: 51,  # 'r'
+    115: 43,  # 's'
+    116: 44,  # 't'
+    117: 63,  # 'u'
+    118: 81,  # 'v'
+    119: 77,  # 'w'
+    120: 98,  # 'x'
+    121: 75,  # 'y'
+    122: 108,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 124,  # '€'
+    129: 202,  # None
+    130: 203,  # '‚'
+    131: 204,  # 'ƒ'
+    132: 205,  # '„'
+    133: 40,  # '…'
+    134: 58,  # '†'
+    135: 206,  # '‡'
+    136: 207,  # 'ˆ'
+    137: 208,  # '‰'
+    138: 209,  # None
+    139: 210,  # '‹'
+    140: 211,  # None
+    141: 212,  # None
+    142: 213,  # None
+    143: 214,  # None
+    144: 215,  # None
+    145: 83,  # '‘'
+    146: 52,  # '’'
+    147: 47,  # '“'
+    148: 46,  # '”'
+    149: 72,  # '•'
+    150: 32,  # '–'
+    151: 94,  # '—'
+    152: 216,  # '˜'
+    153: 113,  # '™'
+    154: 217,  # None
+    155: 109,  # '›'
+    156: 218,  # None
+    157: 219,  # None
+    158: 220,  # None
+    159: 221,  # None
+    160: 34,  # '\xa0'
+    161: 116,  # '¡'
+    162: 222,  # '¢'
+    163: 118,  # '£'
+    164: 100,  # '₪'
+    165: 223,  # '¥'
+    166: 224,  # '¦'
+    167: 117,  # '§'
+    168: 119,  # '¨'
+    169: 104,  # '©'
+    170: 125,  # '×'
+    171: 225,  # '«'
+    172: 226,  # '¬'
+    173: 87,  # '\xad'
+    174: 99,  # '®'
+    175: 227,  # '¯'
+    176: 106,  # '°'
+    177: 122,  # '±'
+    178: 123,  # '²'
+    179: 228,  # '³'
+    180: 55,  # '´'
+    181: 229,  # 'µ'
+    182: 230,  # '¶'
+    183: 101,  # '·'
+    184: 231,  # '¸'
+    185: 232,  # '¹'
+    186: 120,  # '÷'
+    187: 233,  # '»'
+    188: 48,  # '¼'
+    189: 39,  # '½'
+    190: 57,  # '¾'
+    191: 234,  # '¿'
+    192: 30,  # 'ְ'
+    193: 59,  # 'ֱ'
+    194: 41,  # 'ֲ'
+    195: 88,  # 'ֳ'
+    196: 33,  # 'ִ'
+    197: 37,  # 'ֵ'
+    198: 36,  # 'ֶ'
+    199: 31,  # 'ַ'
+    200: 29,  # 'ָ'
+    201: 35,  # 'ֹ'
+    202: 235,  # None
+    203: 62,  # 'ֻ'
+    204: 28,  # 'ּ'
+    205: 236,  # 'ֽ'
+    206: 126,  # '־'
+    207: 237,  # 'ֿ'
+    208: 238,  # '׀'
+    209: 38,  # 'ׁ'
+    210: 45,  # 'ׂ'
+    211: 239,  # '׃'
+    212: 240,  # 'װ'
+    213: 241,  # 'ױ'
+    214: 242,  # 'ײ'
+    215: 243,  # '׳'
+    216: 127,  # '״'
+    217: 244,  # None
+    218: 245,  # None
+    219: 246,  # None
+    220: 247,  # None
+    221: 248,  # None
+    222: 249,  # None
+    223: 250,  # None
+    224: 9,  # 'א'
+    225: 8,  # 'ב'
+    226: 20,  # 'ג'
+    227: 16,  # 'ד'
+    228: 3,  # 'ה'
+    229: 2,  # 'ו'
+    230: 24,  # 'ז'
+    231: 14,  # 'ח'
+    232: 22,  # 'ט'
+    233: 1,  # 'י'
+    234: 25,  # 'ך'
+    235: 15,  # 'כ'
+    236: 4,  # 'ל'
+    237: 11,  # 'ם'
+    238: 6,  # 'מ'
+    239: 23,  # 'ן'
+    240: 12,  # 'נ'
+    241: 19,  # 'ס'
+    242: 13,  # 'ע'
+    243: 26,  # 'ף'
+    244: 18,  # 'פ'
+    245: 27,  # 'ץ'
+    246: 21,  # 'צ'
+    247: 17,  # 'ק'
+    248: 7,  # 'ר'
+    249: 10,  # 'ש'
+    250: 5,  # 'ת'
+    251: 251,  # None
+    252: 252,  # None
+    253: 128,  # '\u200e'
+    254: 96,  # '\u200f'
+    255: 253,  # None
+}
+
+WINDOWS_1255_HEBREW_MODEL = SingleByteCharSetModel(
+    charset_name="windows-1255",
+    language="Hebrew",
+    char_to_order_map=WINDOWS_1255_HEBREW_CHAR_TO_ORDER,
+    language_model=HEBREW_LANG_MODEL,
+    typical_positive_ratio=0.984004,
+    keep_ascii_letters=False,
+    alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ",
+)
diff --git a/contrib/python/chardet/py3/chardet/langhungarianmodel.py b/contrib/python/chardet/py3/chardet/langhungarianmodel.py
new file mode 100644
index 00000000000..bd6630a0513
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/langhungarianmodel.py
@@ -0,0 +1,4649 @@
+from chardet.sbcharsetprober import SingleByteCharSetModel
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+HUNGARIAN_LANG_MODEL = {
+    28: {  # 'A'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 2,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 2,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 2,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 2,  # 'N'
+        47: 1,  # 'O'
+        46: 2,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 2,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 2,  # 'p'
+        10: 2,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 1,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 1,  # 'Á'
+        44: 0,  # 'É'
+        61: 1,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    40: {  # 'B'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 0,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 1,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 3,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    54: {  # 'C'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 0,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 0,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 1,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 3,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    45: {  # 'D'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 0,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 0,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 1,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    32: {  # 'E'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 2,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 2,  # 'K'
+        41: 2,  # 'L'
+        34: 2,  # 'M'
+        35: 2,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 1,  # 't'
+        21: 2,  # 'u'
+        19: 1,  # 'v'
+        62: 1,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    50: {  # 'F'
+        28: 1,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 0,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 0,  # 'V'
+        55: 1,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 1,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 1,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    49: {  # 'G'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 2,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    38: {  # 'H'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 0,  # 'D'
+        32: 1,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 1,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 1,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 1,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 0,  # 'V'
+        55: 1,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 1,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 0,  # 'n'
+        8: 3,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 2,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 2,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    39: {  # 'I'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 2,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 2,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 2,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 0,  # 'e'
+        27: 1,  # 'f'
+        12: 2,  # 'g'
+        20: 1,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    53: {  # 'J'
+        28: 2,  # 'A'
+        40: 0,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 1,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 0,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    36: {  # 'K'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 1,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 3,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 2,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    41: {  # 'L'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 1,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    34: {  # 'M'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 3,  # 'a'
+        18: 0,  # 'b'
+        26: 1,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 3,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    35: {  # 'N'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 2,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 2,  # 'Y'
+        52: 1,  # 'Z'
+        2: 3,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    47: {  # 'O'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 2,  # 'K'
+        41: 2,  # 'L'
+        34: 2,  # 'M'
+        35: 2,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 1,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 1,  # 's'
+        3: 2,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 1,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    46: {  # 'P'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 0,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 1,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 1,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 3,  # 'á'
+        15: 2,  # 'é'
+        30: 0,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    43: {  # 'R'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 2,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 2,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    33: {  # 'S'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 3,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 1,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 1,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 1,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    37: {  # 'T'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 1,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 2,  # 'Á'
+        44: 2,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    57: {  # 'U'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 0,  # 'f'
+        12: 2,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 1,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    48: {  # 'V'
+        28: 2,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 0,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 2,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    55: {  # 'Y'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 2,  # 'Z'
+        2: 1,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 1,  # 'o'
+        23: 1,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    52: {  # 'Z'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 0,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 1,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 1,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 1,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 2,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    2: {  # 'a'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 2,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 2,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 2,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    18: {  # 'b'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 2,  # 's'
+        3: 1,  # 't'
+        21: 3,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 3,  # 'ó'
+        24: 2,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    26: {  # 'c'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 1,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 1,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 1,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 2,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 2,  # 't'
+        21: 2,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 2,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    17: {  # 'd'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 2,  # 'k'
+        6: 1,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    1: {  # 'e'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 2,  # 'e'
+        27: 3,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 2,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 2,  # 'u'
+        19: 3,  # 'v'
+        62: 2,  # 'x'
+        16: 2,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    27: {  # 'f'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 0,  # 'p'
+        10: 3,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 2,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 3,  # 'ö'
+        31: 1,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    12: {  # 'g'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 2,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 2,  # 'k'
+        6: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 3,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 3,  # 'ó'
+        24: 2,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    20: {  # 'h'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 2,  # 's'
+        3: 1,  # 't'
+        21: 3,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 2,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    9: {  # 'i'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 3,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 2,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 3,  # 'ó'
+        24: 1,  # 'ö'
+        31: 2,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    22: {  # 'j'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 1,  # 'i'
+        22: 2,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 1,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 3,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    7: {  # 'k'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 2,  # 'ó'
+        24: 3,  # 'ö'
+        31: 1,  # 'ú'
+        29: 3,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    6: {  # 'l'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 1,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 3,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 3,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 3,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    13: {  # 'm'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 1,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        8: 3,  # 'o'
+        23: 3,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 3,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 2,  # 'ű'
+    },
+    4: {  # 'n'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 1,  # 'x'
+        16: 3,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 3,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    8: {  # 'o'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 1,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 2,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 2,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    23: {  # 'p'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 1,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 2,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 3,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    10: {  # 'r'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 2,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 3,  # 'ú'
+        29: 3,  # 'ü'
+        42: 2,  # 'ő'
+        56: 2,  # 'ű'
+    },
+    5: {  # 's'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 2,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 2,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 3,  # 'k'
+        6: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 3,  # 'ú'
+        29: 3,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    3: {  # 't'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 1,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 3,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 3,  # 'ú'
+        29: 3,  # 'ü'
+        42: 3,  # 'ő'
+        56: 2,  # 'ű'
+    },
+    21: {  # 'u'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 2,  # 'b'
+        26: 2,  # 'c'
+        17: 3,  # 'd'
+        1: 2,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 2,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 1,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 1,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 0,  # 'ö'
+        31: 1,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    19: {  # 'v'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 2,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    62: {  # 'x'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 0,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 1,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    16: {  # 'y'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 2,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 2,  # 'ű'
+    },
+    11: {  # 'z'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 2,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 3,  # 'k'
+        6: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 3,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    51: {  # 'Á'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 1,  # 'F'
+        49: 2,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 2,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 1,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    44: {  # 'É'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 0,  # 'F'
+        49: 2,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 2,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    61: {  # 'Í'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 1,  # 'J'
+        36: 0,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 2,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 0,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 0,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    58: {  # 'Ó'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 2,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 0,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    59: {  # 'Ö'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 0,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 0,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    60: {  # 'Ú'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 2,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 2,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    63: {  # 'Ü'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 0,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 0,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 0,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 1,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    14: {  # 'á'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 1,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 2,  # 'h'
+        9: 2,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 2,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    15: {  # 'é'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 3,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 0,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    30: {  # 'í'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 2,  # 's'
+        3: 3,  # 't'
+        21: 0,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    25: {  # 'ó'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 3,  # 'd'
+        1: 1,  # 'e'
+        27: 2,  # 'f'
+        12: 2,  # 'g'
+        20: 2,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 1,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    24: {  # 'ö'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 0,  # 'a'
+        18: 3,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 0,  # 'e'
+        27: 1,  # 'f'
+        12: 2,  # 'g'
+        20: 1,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 0,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 0,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    31: {  # 'ú'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 3,  # 'j'
+        7: 1,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 2,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    29: {  # 'ü'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 2,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 0,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    42: {  # 'ő'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 2,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 1,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    56: {  # 'ű'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 0,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
+
+# Character Mapping Table(s):
+WINDOWS_1250_HUNGARIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 28,  # 'A'
+    66: 40,  # 'B'
+    67: 54,  # 'C'
+    68: 45,  # 'D'
+    69: 32,  # 'E'
+    70: 50,  # 'F'
+    71: 49,  # 'G'
+    72: 38,  # 'H'
+    73: 39,  # 'I'
+    74: 53,  # 'J'
+    75: 36,  # 'K'
+    76: 41,  # 'L'
+    77: 34,  # 'M'
+    78: 35,  # 'N'
+    79: 47,  # 'O'
+    80: 46,  # 'P'
+    81: 72,  # 'Q'
+    82: 43,  # 'R'
+    83: 33,  # 'S'
+    84: 37,  # 'T'
+    85: 57,  # 'U'
+    86: 48,  # 'V'
+    87: 64,  # 'W'
+    88: 68,  # 'X'
+    89: 55,  # 'Y'
+    90: 52,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 2,  # 'a'
+    98: 18,  # 'b'
+    99: 26,  # 'c'
+    100: 17,  # 'd'
+    101: 1,  # 'e'
+    102: 27,  # 'f'
+    103: 12,  # 'g'
+    104: 20,  # 'h'
+    105: 9,  # 'i'
+    106: 22,  # 'j'
+    107: 7,  # 'k'
+    108: 6,  # 'l'
+    109: 13,  # 'm'
+    110: 4,  # 'n'
+    111: 8,  # 'o'
+    112: 23,  # 'p'
+    113: 67,  # 'q'
+    114: 10,  # 'r'
+    115: 5,  # 's'
+    116: 3,  # 't'
+    117: 21,  # 'u'
+    118: 19,  # 'v'
+    119: 65,  # 'w'
+    120: 62,  # 'x'
+    121: 16,  # 'y'
+    122: 11,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 161,  # '€'
+    129: 162,  # None
+    130: 163,  # '‚'
+    131: 164,  # None
+    132: 165,  # '„'
+    133: 166,  # '…'
+    134: 167,  # '†'
+    135: 168,  # '‡'
+    136: 169,  # None
+    137: 170,  # '‰'
+    138: 171,  # 'Š'
+    139: 172,  # '‹'
+    140: 173,  # 'Ś'
+    141: 174,  # 'Ť'
+    142: 175,  # 'Ž'
+    143: 176,  # 'Ź'
+    144: 177,  # None
+    145: 178,  # '‘'
+    146: 179,  # '’'
+    147: 180,  # '“'
+    148: 78,  # '”'
+    149: 181,  # '•'
+    150: 69,  # '–'
+    151: 182,  # '—'
+    152: 183,  # None
+    153: 184,  # '™'
+    154: 185,  # 'š'
+    155: 186,  # '›'
+    156: 187,  # 'ś'
+    157: 188,  # 'ť'
+    158: 189,  # 'ž'
+    159: 190,  # 'ź'
+    160: 191,  # '\xa0'
+    161: 192,  # 'ˇ'
+    162: 193,  # '˘'
+    163: 194,  # 'Ł'
+    164: 195,  # '¤'
+    165: 196,  # 'Ą'
+    166: 197,  # '¦'
+    167: 76,  # '§'
+    168: 198,  # '¨'
+    169: 199,  # '©'
+    170: 200,  # 'Ş'
+    171: 201,  # '«'
+    172: 202,  # '¬'
+    173: 203,  # '\xad'
+    174: 204,  # '®'
+    175: 205,  # 'Ż'
+    176: 81,  # '°'
+    177: 206,  # '±'
+    178: 207,  # '˛'
+    179: 208,  # 'ł'
+    180: 209,  # '´'
+    181: 210,  # 'µ'
+    182: 211,  # '¶'
+    183: 212,  # '·'
+    184: 213,  # '¸'
+    185: 214,  # 'ą'
+    186: 215,  # 'ş'
+    187: 216,  # '»'
+    188: 217,  # 'Ľ'
+    189: 218,  # '˝'
+    190: 219,  # 'ľ'
+    191: 220,  # 'ż'
+    192: 221,  # 'Ŕ'
+    193: 51,  # 'Á'
+    194: 83,  # 'Â'
+    195: 222,  # 'Ă'
+    196: 80,  # 'Ä'
+    197: 223,  # 'Ĺ'
+    198: 224,  # 'Ć'
+    199: 225,  # 'Ç'
+    200: 226,  # 'Č'
+    201: 44,  # 'É'
+    202: 227,  # 'Ę'
+    203: 228,  # 'Ë'
+    204: 229,  # 'Ě'
+    205: 61,  # 'Í'
+    206: 230,  # 'Î'
+    207: 231,  # 'Ď'
+    208: 232,  # 'Đ'
+    209: 233,  # 'Ń'
+    210: 234,  # 'Ň'
+    211: 58,  # 'Ó'
+    212: 235,  # 'Ô'
+    213: 66,  # 'Ő'
+    214: 59,  # 'Ö'
+    215: 236,  # '×'
+    216: 237,  # 'Ř'
+    217: 238,  # 'Ů'
+    218: 60,  # 'Ú'
+    219: 70,  # 'Ű'
+    220: 63,  # 'Ü'
+    221: 239,  # 'Ý'
+    222: 240,  # 'Ţ'
+    223: 241,  # 'ß'
+    224: 84,  # 'ŕ'
+    225: 14,  # 'á'
+    226: 75,  # 'â'
+    227: 242,  # 'ă'
+    228: 71,  # 'ä'
+    229: 82,  # 'ĺ'
+    230: 243,  # 'ć'
+    231: 73,  # 'ç'
+    232: 244,  # 'č'
+    233: 15,  # 'é'
+    234: 85,  # 'ę'
+    235: 79,  # 'ë'
+    236: 86,  # 'ě'
+    237: 30,  # 'í'
+    238: 77,  # 'î'
+    239: 87,  # 'ď'
+    240: 245,  # 'đ'
+    241: 246,  # 'ń'
+    242: 247,  # 'ň'
+    243: 25,  # 'ó'
+    244: 74,  # 'ô'
+    245: 42,  # 'ő'
+    246: 24,  # 'ö'
+    247: 248,  # '÷'
+    248: 249,  # 'ř'
+    249: 250,  # 'ů'
+    250: 31,  # 'ú'
+    251: 56,  # 'ű'
+    252: 29,  # 'ü'
+    253: 251,  # 'ý'
+    254: 252,  # 'ţ'
+    255: 253,  # '˙'
+}
+
+WINDOWS_1250_HUNGARIAN_MODEL = SingleByteCharSetModel(
+    charset_name="windows-1250",
+    language="Hungarian",
+    char_to_order_map=WINDOWS_1250_HUNGARIAN_CHAR_TO_ORDER,
+    language_model=HUNGARIAN_LANG_MODEL,
+    typical_positive_ratio=0.947368,
+    keep_ascii_letters=True,
+    alphabet="ABCDEFGHIJKLMNOPRSTUVZabcdefghijklmnoprstuvzÁÉÍÓÖÚÜáéíóöúüŐőŰű",
+)
+
+ISO_8859_2_HUNGARIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 28,  # 'A'
+    66: 40,  # 'B'
+    67: 54,  # 'C'
+    68: 45,  # 'D'
+    69: 32,  # 'E'
+    70: 50,  # 'F'
+    71: 49,  # 'G'
+    72: 38,  # 'H'
+    73: 39,  # 'I'
+    74: 53,  # 'J'
+    75: 36,  # 'K'
+    76: 41,  # 'L'
+    77: 34,  # 'M'
+    78: 35,  # 'N'
+    79: 47,  # 'O'
+    80: 46,  # 'P'
+    81: 71,  # 'Q'
+    82: 43,  # 'R'
+    83: 33,  # 'S'
+    84: 37,  # 'T'
+    85: 57,  # 'U'
+    86: 48,  # 'V'
+    87: 64,  # 'W'
+    88: 68,  # 'X'
+    89: 55,  # 'Y'
+    90: 52,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 2,  # 'a'
+    98: 18,  # 'b'
+    99: 26,  # 'c'
+    100: 17,  # 'd'
+    101: 1,  # 'e'
+    102: 27,  # 'f'
+    103: 12,  # 'g'
+    104: 20,  # 'h'
+    105: 9,  # 'i'
+    106: 22,  # 'j'
+    107: 7,  # 'k'
+    108: 6,  # 'l'
+    109: 13,  # 'm'
+    110: 4,  # 'n'
+    111: 8,  # 'o'
+    112: 23,  # 'p'
+    113: 67,  # 'q'
+    114: 10,  # 'r'
+    115: 5,  # 's'
+    116: 3,  # 't'
+    117: 21,  # 'u'
+    118: 19,  # 'v'
+    119: 65,  # 'w'
+    120: 62,  # 'x'
+    121: 16,  # 'y'
+    122: 11,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 159,  # '\x80'
+    129: 160,  # '\x81'
+    130: 161,  # '\x82'
+    131: 162,  # '\x83'
+    132: 163,  # '\x84'
+    133: 164,  # '\x85'
+    134: 165,  # '\x86'
+    135: 166,  # '\x87'
+    136: 167,  # '\x88'
+    137: 168,  # '\x89'
+    138: 169,  # '\x8a'
+    139: 170,  # '\x8b'
+    140: 171,  # '\x8c'
+    141: 172,  # '\x8d'
+    142: 173,  # '\x8e'
+    143: 174,  # '\x8f'
+    144: 175,  # '\x90'
+    145: 176,  # '\x91'
+    146: 177,  # '\x92'
+    147: 178,  # '\x93'
+    148: 179,  # '\x94'
+    149: 180,  # '\x95'
+    150: 181,  # '\x96'
+    151: 182,  # '\x97'
+    152: 183,  # '\x98'
+    153: 184,  # '\x99'
+    154: 185,  # '\x9a'
+    155: 186,  # '\x9b'
+    156: 187,  # '\x9c'
+    157: 188,  # '\x9d'
+    158: 189,  # '\x9e'
+    159: 190,  # '\x9f'
+    160: 191,  # '\xa0'
+    161: 192,  # 'Ą'
+    162: 193,  # '˘'
+    163: 194,  # 'Ł'
+    164: 195,  # '¤'
+    165: 196,  # 'Ľ'
+    166: 197,  # 'Ś'
+    167: 75,  # '§'
+    168: 198,  # '¨'
+    169: 199,  # 'Š'
+    170: 200,  # 'Ş'
+    171: 201,  # 'Ť'
+    172: 202,  # 'Ź'
+    173: 203,  # '\xad'
+    174: 204,  # 'Ž'
+    175: 205,  # 'Ż'
+    176: 79,  # '°'
+    177: 206,  # 'ą'
+    178: 207,  # '˛'
+    179: 208,  # 'ł'
+    180: 209,  # '´'
+    181: 210,  # 'ľ'
+    182: 211,  # 'ś'
+    183: 212,  # 'ˇ'
+    184: 213,  # '¸'
+    185: 214,  # 'š'
+    186: 215,  # 'ş'
+    187: 216,  # 'ť'
+    188: 217,  # 'ź'
+    189: 218,  # '˝'
+    190: 219,  # 'ž'
+    191: 220,  # 'ż'
+    192: 221,  # 'Ŕ'
+    193: 51,  # 'Á'
+    194: 81,  # 'Â'
+    195: 222,  # 'Ă'
+    196: 78,  # 'Ä'
+    197: 223,  # 'Ĺ'
+    198: 224,  # 'Ć'
+    199: 225,  # 'Ç'
+    200: 226,  # 'Č'
+    201: 44,  # 'É'
+    202: 227,  # 'Ę'
+    203: 228,  # 'Ë'
+    204: 229,  # 'Ě'
+    205: 61,  # 'Í'
+    206: 230,  # 'Î'
+    207: 231,  # 'Ď'
+    208: 232,  # 'Đ'
+    209: 233,  # 'Ń'
+    210: 234,  # 'Ň'
+    211: 58,  # 'Ó'
+    212: 235,  # 'Ô'
+    213: 66,  # 'Ő'
+    214: 59,  # 'Ö'
+    215: 236,  # '×'
+    216: 237,  # 'Ř'
+    217: 238,  # 'Ů'
+    218: 60,  # 'Ú'
+    219: 69,  # 'Ű'
+    220: 63,  # 'Ü'
+    221: 239,  # 'Ý'
+    222: 240,  # 'Ţ'
+    223: 241,  # 'ß'
+    224: 82,  # 'ŕ'
+    225: 14,  # 'á'
+    226: 74,  # 'â'
+    227: 242,  # 'ă'
+    228: 70,  # 'ä'
+    229: 80,  # 'ĺ'
+    230: 243,  # 'ć'
+    231: 72,  # 'ç'
+    232: 244,  # 'č'
+    233: 15,  # 'é'
+    234: 83,  # 'ę'
+    235: 77,  # 'ë'
+    236: 84,  # 'ě'
+    237: 30,  # 'í'
+    238: 76,  # 'î'
+    239: 85,  # 'ď'
+    240: 245,  # 'đ'
+    241: 246,  # 'ń'
+    242: 247,  # 'ň'
+    243: 25,  # 'ó'
+    244: 73,  # 'ô'
+    245: 42,  # 'ő'
+    246: 24,  # 'ö'
+    247: 248,  # '÷'
+    248: 249,  # 'ř'
+    249: 250,  # 'ů'
+    250: 31,  # 'ú'
+    251: 56,  # 'ű'
+    252: 29,  # 'ü'
+    253: 251,  # 'ý'
+    254: 252,  # 'ţ'
+    255: 253,  # '˙'
+}
+
+ISO_8859_2_HUNGARIAN_MODEL = SingleByteCharSetModel(
+    charset_name="ISO-8859-2",
+    language="Hungarian",
+    char_to_order_map=ISO_8859_2_HUNGARIAN_CHAR_TO_ORDER,
+    language_model=HUNGARIAN_LANG_MODEL,
+    typical_positive_ratio=0.947368,
+    keep_ascii_letters=True,
+    alphabet="ABCDEFGHIJKLMNOPRSTUVZabcdefghijklmnoprstuvzÁÉÍÓÖÚÜáéíóöúüŐőŰű",
+)
diff --git a/contrib/python/chardet/py3/chardet/langrussianmodel.py b/contrib/python/chardet/py3/chardet/langrussianmodel.py
new file mode 100644
index 00000000000..0d5b178446d
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/langrussianmodel.py
@@ -0,0 +1,5725 @@
+from chardet.sbcharsetprober import SingleByteCharSetModel
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+RUSSIAN_LANG_MODEL = {
+    37: {  # 'А'
+        37: 0,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 2,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 1,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 0,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 1,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 0,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    44: {  # 'Б'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 1,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 2,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    33: {  # 'В'
+        37: 2,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 2,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 1,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 0,  # 'ю'
+        16: 1,  # 'я'
+    },
+    46: {  # 'Г'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 2,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    41: {  # 'Д'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 2,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 2,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 3,  # 'ж'
+        20: 1,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    48: {  # 'Е'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 2,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 2,  # 'Р'
+        32: 2,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 2,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 2,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 1,  # 'н'
+        1: 0,  # 'о'
+        15: 1,  # 'п'
+        9: 1,  # 'р'
+        7: 3,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 2,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    56: {  # 'Ж'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 1,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 2,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 1,  # 'м'
+        5: 0,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 2,  # 'ю'
+        16: 0,  # 'я'
+    },
+    51: {  # 'З'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 0,  # 'г'
+        13: 2,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 1,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 1,  # 'я'
+    },
+    42: {  # 'И'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 2,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 2,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 1,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 2,  # 'з'
+        4: 1,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 1,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 1,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    60: {  # 'Й'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 1,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    36: {  # 'К'
+        37: 2,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 2,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 0,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    49: {  # 'Л'
+        37: 2,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 1,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 0,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 0,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 1,  # 'л'
+        12: 0,  # 'м'
+        5: 1,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 2,  # 'ю'
+        16: 1,  # 'я'
+    },
+    38: {  # 'М'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 1,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 1,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    31: {  # 'Н'
+        37: 2,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 1,  # 'З'
+        42: 2,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 3,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    34: {  # 'О'
+        37: 0,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 2,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 2,  # 'Л'
+        38: 1,  # 'М'
+        31: 2,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 2,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 1,  # 'а'
+        21: 2,  # 'б'
+        10: 1,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 0,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 1,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 0,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 1,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    35: {  # 'П'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 2,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 0,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 3,  # 'р'
+        7: 1,  # 'с'
+        6: 1,  # 'т'
+        14: 2,  # 'у'
+        39: 1,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 0,  # 'ю'
+        16: 2,  # 'я'
+    },
+    45: {  # 'Р'
+        37: 2,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 2,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 0,  # 'З'
+        42: 2,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 2,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 2,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 2,  # 'я'
+    },
+    32: {  # 'С'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 2,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 2,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 1,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 1,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    40: {  # 'Т'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 2,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 1,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    52: {  # 'У'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 1,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 2,  # 'и'
+        23: 1,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 1,  # 'н'
+        1: 2,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 0,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    53: {  # 'Ф'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 1,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    55: {  # 'Х'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 2,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 0,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    58: {  # 'Ц'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 0,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    50: {  # 'Ч'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 1,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 1,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 1,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    57: {  # 'Ш'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 1,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 2,  # 'о'
+        15: 2,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    63: {  # 'Щ'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 1,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 1,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 1,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    62: {  # 'Ы'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 0,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 0,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    61: {  # 'Ь'
+        37: 0,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 1,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 1,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 0,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 0,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    47: {  # 'Э'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 0,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 2,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    59: {  # 'Ю'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 1,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 0,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 1,  # 'п'
+        9: 1,  # 'р'
+        7: 1,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    43: {  # 'Я'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 0,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 0,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 1,  # 'й'
+        11: 1,  # 'к'
+        8: 1,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 1,  # 'п'
+        9: 1,  # 'р'
+        7: 1,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    3: {  # 'а'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 3,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 3,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    21: {  # 'б'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 1,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 0,  # 'ф'
+        26: 2,  # 'х'
+        28: 1,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 3,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    10: {  # 'в'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 3,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 3,  # 'ш'
+        29: 2,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 3,  # 'я'
+    },
+    19: {  # 'г'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 3,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    13: {  # 'д'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 3,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    2: {  # 'е'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 2,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 3,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    24: {  # 'ж'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 1,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 1,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 0,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    20: {  # 'з'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 3,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 3,  # 'я'
+    },
+    4: {  # 'и'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 3,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 3,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    23: {  # 'й'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 2,  # 'з'
+        4: 1,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 2,  # 'ф'
+        26: 1,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 2,  # 'я'
+    },
+    11: {  # 'к'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 3,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    8: {  # 'л'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 3,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 1,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 1,  # 'ц'
+        22: 3,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    12: {  # 'м'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 1,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 2,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 3,  # 'я'
+    },
+    5: {  # 'н'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 3,  # 'ц'
+        22: 3,  # 'ч'
+        25: 2,  # 'ш'
+        29: 2,  # 'щ'
+        54: 1,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    1: {  # 'о'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 3,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    15: {  # 'п'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 3,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 0,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 3,  # 'я'
+    },
+    9: {  # 'р'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 2,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 3,  # 'ш'
+        29: 2,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 2,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    7: {  # 'с'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 1,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    6: {  # 'т'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 2,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 2,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    14: {  # 'у'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 2,  # 'и'
+        23: 2,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 2,  # 'я'
+    },
+    39: {  # 'ф'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 1,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 2,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    26: {  # 'х'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 3,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 1,  # 'п'
+        9: 3,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 0,  # 'щ'
+        54: 1,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    28: {  # 'ц'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 1,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 1,  # 'т'
+        14: 3,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 1,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 1,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    22: {  # 'ч'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 3,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    25: {  # 'ш'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 1,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    29: {  # 'щ'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 1,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 2,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    54: {  # 'ъ'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 0,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 2,  # 'я'
+    },
+    18: {  # 'ы'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 2,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 1,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 2,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 2,  # 'я'
+    },
+    17: {  # 'ь'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 3,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 0,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 2,  # 'п'
+        9: 1,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 0,  # 'у'
+        39: 2,  # 'ф'
+        26: 1,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 3,  # 'ш'
+        29: 2,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    30: {  # 'э'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 1,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 1,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 2,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 2,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    27: {  # 'ю'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 1,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 1,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 1,  # 'и'
+        23: 1,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 1,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 0,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 2,  # 'ю'
+        16: 1,  # 'я'
+    },
+    16: {  # 'я'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 2,  # 'и'
+        23: 2,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 0,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 1,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 2,  # 'ю'
+        16: 2,  # 'я'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
+
+# Character Mapping Table(s):
+IBM866_RUSSIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 142,  # 'A'
+    66: 143,  # 'B'
+    67: 144,  # 'C'
+    68: 145,  # 'D'
+    69: 146,  # 'E'
+    70: 147,  # 'F'
+    71: 148,  # 'G'
+    72: 149,  # 'H'
+    73: 150,  # 'I'
+    74: 151,  # 'J'
+    75: 152,  # 'K'
+    76: 74,  # 'L'
+    77: 153,  # 'M'
+    78: 75,  # 'N'
+    79: 154,  # 'O'
+    80: 155,  # 'P'
+    81: 156,  # 'Q'
+    82: 157,  # 'R'
+    83: 158,  # 'S'
+    84: 159,  # 'T'
+    85: 160,  # 'U'
+    86: 161,  # 'V'
+    87: 162,  # 'W'
+    88: 163,  # 'X'
+    89: 164,  # 'Y'
+    90: 165,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 71,  # 'a'
+    98: 172,  # 'b'
+    99: 66,  # 'c'
+    100: 173,  # 'd'
+    101: 65,  # 'e'
+    102: 174,  # 'f'
+    103: 76,  # 'g'
+    104: 175,  # 'h'
+    105: 64,  # 'i'
+    106: 176,  # 'j'
+    107: 177,  # 'k'
+    108: 77,  # 'l'
+    109: 72,  # 'm'
+    110: 178,  # 'n'
+    111: 69,  # 'o'
+    112: 67,  # 'p'
+    113: 179,  # 'q'
+    114: 78,  # 'r'
+    115: 73,  # 's'
+    116: 180,  # 't'
+    117: 181,  # 'u'
+    118: 79,  # 'v'
+    119: 182,  # 'w'
+    120: 183,  # 'x'
+    121: 184,  # 'y'
+    122: 185,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 37,  # 'А'
+    129: 44,  # 'Б'
+    130: 33,  # 'В'
+    131: 46,  # 'Г'
+    132: 41,  # 'Д'
+    133: 48,  # 'Е'
+    134: 56,  # 'Ж'
+    135: 51,  # 'З'
+    136: 42,  # 'И'
+    137: 60,  # 'Й'
+    138: 36,  # 'К'
+    139: 49,  # 'Л'
+    140: 38,  # 'М'
+    141: 31,  # 'Н'
+    142: 34,  # 'О'
+    143: 35,  # 'П'
+    144: 45,  # 'Р'
+    145: 32,  # 'С'
+    146: 40,  # 'Т'
+    147: 52,  # 'У'
+    148: 53,  # 'Ф'
+    149: 55,  # 'Х'
+    150: 58,  # 'Ц'
+    151: 50,  # 'Ч'
+    152: 57,  # 'Ш'
+    153: 63,  # 'Щ'
+    154: 70,  # 'Ъ'
+    155: 62,  # 'Ы'
+    156: 61,  # 'Ь'
+    157: 47,  # 'Э'
+    158: 59,  # 'Ю'
+    159: 43,  # 'Я'
+    160: 3,  # 'а'
+    161: 21,  # 'б'
+    162: 10,  # 'в'
+    163: 19,  # 'г'
+    164: 13,  # 'д'
+    165: 2,  # 'е'
+    166: 24,  # 'ж'
+    167: 20,  # 'з'
+    168: 4,  # 'и'
+    169: 23,  # 'й'
+    170: 11,  # 'к'
+    171: 8,  # 'л'
+    172: 12,  # 'м'
+    173: 5,  # 'н'
+    174: 1,  # 'о'
+    175: 15,  # 'п'
+    176: 191,  # '░'
+    177: 192,  # '▒'
+    178: 193,  # '▓'
+    179: 194,  # '│'
+    180: 195,  # '┤'
+    181: 196,  # '╡'
+    182: 197,  # '╢'
+    183: 198,  # '╖'
+    184: 199,  # '╕'
+    185: 200,  # '╣'
+    186: 201,  # '║'
+    187: 202,  # '╗'
+    188: 203,  # '╝'
+    189: 204,  # '╜'
+    190: 205,  # '╛'
+    191: 206,  # '┐'
+    192: 207,  # '└'
+    193: 208,  # '┴'
+    194: 209,  # '┬'
+    195: 210,  # '├'
+    196: 211,  # '─'
+    197: 212,  # '┼'
+    198: 213,  # '╞'
+    199: 214,  # '╟'
+    200: 215,  # '╚'
+    201: 216,  # '╔'
+    202: 217,  # '╩'
+    203: 218,  # '╦'
+    204: 219,  # '╠'
+    205: 220,  # '═'
+    206: 221,  # '╬'
+    207: 222,  # '╧'
+    208: 223,  # '╨'
+    209: 224,  # '╤'
+    210: 225,  # '╥'
+    211: 226,  # '╙'
+    212: 227,  # '╘'
+    213: 228,  # '╒'
+    214: 229,  # '╓'
+    215: 230,  # '╫'
+    216: 231,  # '╪'
+    217: 232,  # '┘'
+    218: 233,  # '┌'
+    219: 234,  # '█'
+    220: 235,  # '▄'
+    221: 236,  # '▌'
+    222: 237,  # '▐'
+    223: 238,  # '▀'
+    224: 9,  # 'р'
+    225: 7,  # 'с'
+    226: 6,  # 'т'
+    227: 14,  # 'у'
+    228: 39,  # 'ф'
+    229: 26,  # 'х'
+    230: 28,  # 'ц'
+    231: 22,  # 'ч'
+    232: 25,  # 'ш'
+    233: 29,  # 'щ'
+    234: 54,  # 'ъ'
+    235: 18,  # 'ы'
+    236: 17,  # 'ь'
+    237: 30,  # 'э'
+    238: 27,  # 'ю'
+    239: 16,  # 'я'
+    240: 239,  # 'Ё'
+    241: 68,  # 'ё'
+    242: 240,  # 'Є'
+    243: 241,  # 'є'
+    244: 242,  # 'Ї'
+    245: 243,  # 'ї'
+    246: 244,  # 'Ў'
+    247: 245,  # 'ў'
+    248: 246,  # '°'
+    249: 247,  # '∙'
+    250: 248,  # '·'
+    251: 249,  # '√'
+    252: 250,  # '№'
+    253: 251,  # '¤'
+    254: 252,  # '■'
+    255: 255,  # '\xa0'
+}
+
+IBM866_RUSSIAN_MODEL = SingleByteCharSetModel(
+    charset_name="IBM866",
+    language="Russian",
+    char_to_order_map=IBM866_RUSSIAN_CHAR_TO_ORDER,
+    language_model=RUSSIAN_LANG_MODEL,
+    typical_positive_ratio=0.976601,
+    keep_ascii_letters=False,
+    alphabet="ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё",
+)
+
+WINDOWS_1251_RUSSIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 142,  # 'A'
+    66: 143,  # 'B'
+    67: 144,  # 'C'
+    68: 145,  # 'D'
+    69: 146,  # 'E'
+    70: 147,  # 'F'
+    71: 148,  # 'G'
+    72: 149,  # 'H'
+    73: 150,  # 'I'
+    74: 151,  # 'J'
+    75: 152,  # 'K'
+    76: 74,  # 'L'
+    77: 153,  # 'M'
+    78: 75,  # 'N'
+    79: 154,  # 'O'
+    80: 155,  # 'P'
+    81: 156,  # 'Q'
+    82: 157,  # 'R'
+    83: 158,  # 'S'
+    84: 159,  # 'T'
+    85: 160,  # 'U'
+    86: 161,  # 'V'
+    87: 162,  # 'W'
+    88: 163,  # 'X'
+    89: 164,  # 'Y'
+    90: 165,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 71,  # 'a'
+    98: 172,  # 'b'
+    99: 66,  # 'c'
+    100: 173,  # 'd'
+    101: 65,  # 'e'
+    102: 174,  # 'f'
+    103: 76,  # 'g'
+    104: 175,  # 'h'
+    105: 64,  # 'i'
+    106: 176,  # 'j'
+    107: 177,  # 'k'
+    108: 77,  # 'l'
+    109: 72,  # 'm'
+    110: 178,  # 'n'
+    111: 69,  # 'o'
+    112: 67,  # 'p'
+    113: 179,  # 'q'
+    114: 78,  # 'r'
+    115: 73,  # 's'
+    116: 180,  # 't'
+    117: 181,  # 'u'
+    118: 79,  # 'v'
+    119: 182,  # 'w'
+    120: 183,  # 'x'
+    121: 184,  # 'y'
+    122: 185,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 191,  # 'Ђ'
+    129: 192,  # 'Ѓ'
+    130: 193,  # '‚'
+    131: 194,  # 'ѓ'
+    132: 195,  # '„'
+    133: 196,  # '…'
+    134: 197,  # '†'
+    135: 198,  # '‡'
+    136: 199,  # '€'
+    137: 200,  # '‰'
+    138: 201,  # 'Љ'
+    139: 202,  # '‹'
+    140: 203,  # 'Њ'
+    141: 204,  # 'Ќ'
+    142: 205,  # 'Ћ'
+    143: 206,  # 'Џ'
+    144: 207,  # 'ђ'
+    145: 208,  # '‘'
+    146: 209,  # '’'
+    147: 210,  # '“'
+    148: 211,  # '”'
+    149: 212,  # '•'
+    150: 213,  # '–'
+    151: 214,  # '—'
+    152: 215,  # None
+    153: 216,  # '™'
+    154: 217,  # 'љ'
+    155: 218,  # '›'
+    156: 219,  # 'њ'
+    157: 220,  # 'ќ'
+    158: 221,  # 'ћ'
+    159: 222,  # 'џ'
+    160: 223,  # '\xa0'
+    161: 224,  # 'Ў'
+    162: 225,  # 'ў'
+    163: 226,  # 'Ј'
+    164: 227,  # '¤'
+    165: 228,  # 'Ґ'
+    166: 229,  # '¦'
+    167: 230,  # '§'
+    168: 231,  # 'Ё'
+    169: 232,  # '©'
+    170: 233,  # 'Є'
+    171: 234,  # '«'
+    172: 235,  # '¬'
+    173: 236,  # '\xad'
+    174: 237,  # '®'
+    175: 238,  # 'Ї'
+    176: 239,  # '°'
+    177: 240,  # '±'
+    178: 241,  # 'І'
+    179: 242,  # 'і'
+    180: 243,  # 'ґ'
+    181: 244,  # 'µ'
+    182: 245,  # '¶'
+    183: 246,  # '·'
+    184: 68,  # 'ё'
+    185: 247,  # '№'
+    186: 248,  # 'є'
+    187: 249,  # '»'
+    188: 250,  # 'ј'
+    189: 251,  # 'Ѕ'
+    190: 252,  # 'ѕ'
+    191: 253,  # 'ї'
+    192: 37,  # 'А'
+    193: 44,  # 'Б'
+    194: 33,  # 'В'
+    195: 46,  # 'Г'
+    196: 41,  # 'Д'
+    197: 48,  # 'Е'
+    198: 56,  # 'Ж'
+    199: 51,  # 'З'
+    200: 42,  # 'И'
+    201: 60,  # 'Й'
+    202: 36,  # 'К'
+    203: 49,  # 'Л'
+    204: 38,  # 'М'
+    205: 31,  # 'Н'
+    206: 34,  # 'О'
+    207: 35,  # 'П'
+    208: 45,  # 'Р'
+    209: 32,  # 'С'
+    210: 40,  # 'Т'
+    211: 52,  # 'У'
+    212: 53,  # 'Ф'
+    213: 55,  # 'Х'
+    214: 58,  # 'Ц'
+    215: 50,  # 'Ч'
+    216: 57,  # 'Ш'
+    217: 63,  # 'Щ'
+    218: 70,  # 'Ъ'
+    219: 62,  # 'Ы'
+    220: 61,  # 'Ь'
+    221: 47,  # 'Э'
+    222: 59,  # 'Ю'
+    223: 43,  # 'Я'
+    224: 3,  # 'а'
+    225: 21,  # 'б'
+    226: 10,  # 'в'
+    227: 19,  # 'г'
+    228: 13,  # 'д'
+    229: 2,  # 'е'
+    230: 24,  # 'ж'
+    231: 20,  # 'з'
+    232: 4,  # 'и'
+    233: 23,  # 'й'
+    234: 11,  # 'к'
+    235: 8,  # 'л'
+    236: 12,  # 'м'
+    237: 5,  # 'н'
+    238: 1,  # 'о'
+    239: 15,  # 'п'
+    240: 9,  # 'р'
+    241: 7,  # 'с'
+    242: 6,  # 'т'
+    243: 14,  # 'у'
+    244: 39,  # 'ф'
+    245: 26,  # 'х'
+    246: 28,  # 'ц'
+    247: 22,  # 'ч'
+    248: 25,  # 'ш'
+    249: 29,  # 'щ'
+    250: 54,  # 'ъ'
+    251: 18,  # 'ы'
+    252: 17,  # 'ь'
+    253: 30,  # 'э'
+    254: 27,  # 'ю'
+    255: 16,  # 'я'
+}
+
+WINDOWS_1251_RUSSIAN_MODEL = SingleByteCharSetModel(
+    charset_name="windows-1251",
+    language="Russian",
+    char_to_order_map=WINDOWS_1251_RUSSIAN_CHAR_TO_ORDER,
+    language_model=RUSSIAN_LANG_MODEL,
+    typical_positive_ratio=0.976601,
+    keep_ascii_letters=False,
+    alphabet="ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё",
+)
+
+IBM855_RUSSIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 142,  # 'A'
+    66: 143,  # 'B'
+    67: 144,  # 'C'
+    68: 145,  # 'D'
+    69: 146,  # 'E'
+    70: 147,  # 'F'
+    71: 148,  # 'G'
+    72: 149,  # 'H'
+    73: 150,  # 'I'
+    74: 151,  # 'J'
+    75: 152,  # 'K'
+    76: 74,  # 'L'
+    77: 153,  # 'M'
+    78: 75,  # 'N'
+    79: 154,  # 'O'
+    80: 155,  # 'P'
+    81: 156,  # 'Q'
+    82: 157,  # 'R'
+    83: 158,  # 'S'
+    84: 159,  # 'T'
+    85: 160,  # 'U'
+    86: 161,  # 'V'
+    87: 162,  # 'W'
+    88: 163,  # 'X'
+    89: 164,  # 'Y'
+    90: 165,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 71,  # 'a'
+    98: 172,  # 'b'
+    99: 66,  # 'c'
+    100: 173,  # 'd'
+    101: 65,  # 'e'
+    102: 174,  # 'f'
+    103: 76,  # 'g'
+    104: 175,  # 'h'
+    105: 64,  # 'i'
+    106: 176,  # 'j'
+    107: 177,  # 'k'
+    108: 77,  # 'l'
+    109: 72,  # 'm'
+    110: 178,  # 'n'
+    111: 69,  # 'o'
+    112: 67,  # 'p'
+    113: 179,  # 'q'
+    114: 78,  # 'r'
+    115: 73,  # 's'
+    116: 180,  # 't'
+    117: 181,  # 'u'
+    118: 79,  # 'v'
+    119: 182,  # 'w'
+    120: 183,  # 'x'
+    121: 184,  # 'y'
+    122: 185,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 191,  # 'ђ'
+    129: 192,  # 'Ђ'
+    130: 193,  # 'ѓ'
+    131: 194,  # 'Ѓ'
+    132: 68,  # 'ё'
+    133: 195,  # 'Ё'
+    134: 196,  # 'є'
+    135: 197,  # 'Є'
+    136: 198,  # 'ѕ'
+    137: 199,  # 'Ѕ'
+    138: 200,  # 'і'
+    139: 201,  # 'І'
+    140: 202,  # 'ї'
+    141: 203,  # 'Ї'
+    142: 204,  # 'ј'
+    143: 205,  # 'Ј'
+    144: 206,  # 'љ'
+    145: 207,  # 'Љ'
+    146: 208,  # 'њ'
+    147: 209,  # 'Њ'
+    148: 210,  # 'ћ'
+    149: 211,  # 'Ћ'
+    150: 212,  # 'ќ'
+    151: 213,  # 'Ќ'
+    152: 214,  # 'ў'
+    153: 215,  # 'Ў'
+    154: 216,  # 'џ'
+    155: 217,  # 'Џ'
+    156: 27,  # 'ю'
+    157: 59,  # 'Ю'
+    158: 54,  # 'ъ'
+    159: 70,  # 'Ъ'
+    160: 3,  # 'а'
+    161: 37,  # 'А'
+    162: 21,  # 'б'
+    163: 44,  # 'Б'
+    164: 28,  # 'ц'
+    165: 58,  # 'Ц'
+    166: 13,  # 'д'
+    167: 41,  # 'Д'
+    168: 2,  # 'е'
+    169: 48,  # 'Е'
+    170: 39,  # 'ф'
+    171: 53,  # 'Ф'
+    172: 19,  # 'г'
+    173: 46,  # 'Г'
+    174: 218,  # '«'
+    175: 219,  # '»'
+    176: 220,  # '░'
+    177: 221,  # '▒'
+    178: 222,  # '▓'
+    179: 223,  # '│'
+    180: 224,  # '┤'
+    181: 26,  # 'х'
+    182: 55,  # 'Х'
+    183: 4,  # 'и'
+    184: 42,  # 'И'
+    185: 225,  # '╣'
+    186: 226,  # '║'
+    187: 227,  # '╗'
+    188: 228,  # '╝'
+    189: 23,  # 'й'
+    190: 60,  # 'Й'
+    191: 229,  # '┐'
+    192: 230,  # '└'
+    193: 231,  # '┴'
+    194: 232,  # '┬'
+    195: 233,  # '├'
+    196: 234,  # '─'
+    197: 235,  # '┼'
+    198: 11,  # 'к'
+    199: 36,  # 'К'
+    200: 236,  # '╚'
+    201: 237,  # '╔'
+    202: 238,  # '╩'
+    203: 239,  # '╦'
+    204: 240,  # '╠'
+    205: 241,  # '═'
+    206: 242,  # '╬'
+    207: 243,  # '¤'
+    208: 8,  # 'л'
+    209: 49,  # 'Л'
+    210: 12,  # 'м'
+    211: 38,  # 'М'
+    212: 5,  # 'н'
+    213: 31,  # 'Н'
+    214: 1,  # 'о'
+    215: 34,  # 'О'
+    216: 15,  # 'п'
+    217: 244,  # '┘'
+    218: 245,  # '┌'
+    219: 246,  # '█'
+    220: 247,  # '▄'
+    221: 35,  # 'П'
+    222: 16,  # 'я'
+    223: 248,  # '▀'
+    224: 43,  # 'Я'
+    225: 9,  # 'р'
+    226: 45,  # 'Р'
+    227: 7,  # 'с'
+    228: 32,  # 'С'
+    229: 6,  # 'т'
+    230: 40,  # 'Т'
+    231: 14,  # 'у'
+    232: 52,  # 'У'
+    233: 24,  # 'ж'
+    234: 56,  # 'Ж'
+    235: 10,  # 'в'
+    236: 33,  # 'В'
+    237: 17,  # 'ь'
+    238: 61,  # 'Ь'
+    239: 249,  # '№'
+    240: 250,  # '\xad'
+    241: 18,  # 'ы'
+    242: 62,  # 'Ы'
+    243: 20,  # 'з'
+    244: 51,  # 'З'
+    245: 25,  # 'ш'
+    246: 57,  # 'Ш'
+    247: 30,  # 'э'
+    248: 47,  # 'Э'
+    249: 29,  # 'щ'
+    250: 63,  # 'Щ'
+    251: 22,  # 'ч'
+    252: 50,  # 'Ч'
+    253: 251,  # '§'
+    254: 252,  # '■'
+    255: 255,  # '\xa0'
+}
+
+IBM855_RUSSIAN_MODEL = SingleByteCharSetModel(
+    charset_name="IBM855",
+    language="Russian",
+    char_to_order_map=IBM855_RUSSIAN_CHAR_TO_ORDER,
+    language_model=RUSSIAN_LANG_MODEL,
+    typical_positive_ratio=0.976601,
+    keep_ascii_letters=False,
+    alphabet="ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё",
+)
+
+KOI8_R_RUSSIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 142,  # 'A'
+    66: 143,  # 'B'
+    67: 144,  # 'C'
+    68: 145,  # 'D'
+    69: 146,  # 'E'
+    70: 147,  # 'F'
+    71: 148,  # 'G'
+    72: 149,  # 'H'
+    73: 150,  # 'I'
+    74: 151,  # 'J'
+    75: 152,  # 'K'
+    76: 74,  # 'L'
+    77: 153,  # 'M'
+    78: 75,  # 'N'
+    79: 154,  # 'O'
+    80: 155,  # 'P'
+    81: 156,  # 'Q'
+    82: 157,  # 'R'
+    83: 158,  # 'S'
+    84: 159,  # 'T'
+    85: 160,  # 'U'
+    86: 161,  # 'V'
+    87: 162,  # 'W'
+    88: 163,  # 'X'
+    89: 164,  # 'Y'
+    90: 165,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 71,  # 'a'
+    98: 172,  # 'b'
+    99: 66,  # 'c'
+    100: 173,  # 'd'
+    101: 65,  # 'e'
+    102: 174,  # 'f'
+    103: 76,  # 'g'
+    104: 175,  # 'h'
+    105: 64,  # 'i'
+    106: 176,  # 'j'
+    107: 177,  # 'k'
+    108: 77,  # 'l'
+    109: 72,  # 'm'
+    110: 178,  # 'n'
+    111: 69,  # 'o'
+    112: 67,  # 'p'
+    113: 179,  # 'q'
+    114: 78,  # 'r'
+    115: 73,  # 's'
+    116: 180,  # 't'
+    117: 181,  # 'u'
+    118: 79,  # 'v'
+    119: 182,  # 'w'
+    120: 183,  # 'x'
+    121: 184,  # 'y'
+    122: 185,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 191,  # '─'
+    129: 192,  # '│'
+    130: 193,  # '┌'
+    131: 194,  # '┐'
+    132: 195,  # '└'
+    133: 196,  # '┘'
+    134: 197,  # '├'
+    135: 198,  # '┤'
+    136: 199,  # '┬'
+    137: 200,  # '┴'
+    138: 201,  # '┼'
+    139: 202,  # '▀'
+    140: 203,  # '▄'
+    141: 204,  # '█'
+    142: 205,  # '▌'
+    143: 206,  # '▐'
+    144: 207,  # '░'
+    145: 208,  # '▒'
+    146: 209,  # '▓'
+    147: 210,  # '⌠'
+    148: 211,  # '■'
+    149: 212,  # '∙'
+    150: 213,  # '√'
+    151: 214,  # '≈'
+    152: 215,  # '≤'
+    153: 216,  # '≥'
+    154: 217,  # '\xa0'
+    155: 218,  # '⌡'
+    156: 219,  # '°'
+    157: 220,  # '²'
+    158: 221,  # '·'
+    159: 222,  # '÷'
+    160: 223,  # '═'
+    161: 224,  # '║'
+    162: 225,  # '╒'
+    163: 68,  # 'ё'
+    164: 226,  # '╓'
+    165: 227,  # '╔'
+    166: 228,  # '╕'
+    167: 229,  # '╖'
+    168: 230,  # '╗'
+    169: 231,  # '╘'
+    170: 232,  # '╙'
+    171: 233,  # '╚'
+    172: 234,  # '╛'
+    173: 235,  # '╜'
+    174: 236,  # '╝'
+    175: 237,  # '╞'
+    176: 238,  # '╟'
+    177: 239,  # '╠'
+    178: 240,  # '╡'
+    179: 241,  # 'Ё'
+    180: 242,  # '╢'
+    181: 243,  # '╣'
+    182: 244,  # '╤'
+    183: 245,  # '╥'
+    184: 246,  # '╦'
+    185: 247,  # '╧'
+    186: 248,  # '╨'
+    187: 249,  # '╩'
+    188: 250,  # '╪'
+    189: 251,  # '╫'
+    190: 252,  # '╬'
+    191: 253,  # '©'
+    192: 27,  # 'ю'
+    193: 3,  # 'а'
+    194: 21,  # 'б'
+    195: 28,  # 'ц'
+    196: 13,  # 'д'
+    197: 2,  # 'е'
+    198: 39,  # 'ф'
+    199: 19,  # 'г'
+    200: 26,  # 'х'
+    201: 4,  # 'и'
+    202: 23,  # 'й'
+    203: 11,  # 'к'
+    204: 8,  # 'л'
+    205: 12,  # 'м'
+    206: 5,  # 'н'
+    207: 1,  # 'о'
+    208: 15,  # 'п'
+    209: 16,  # 'я'
+    210: 9,  # 'р'
+    211: 7,  # 'с'
+    212: 6,  # 'т'
+    213: 14,  # 'у'
+    214: 24,  # 'ж'
+    215: 10,  # 'в'
+    216: 17,  # 'ь'
+    217: 18,  # 'ы'
+    218: 20,  # 'з'
+    219: 25,  # 'ш'
+    220: 30,  # 'э'
+    221: 29,  # 'щ'
+    222: 22,  # 'ч'
+    223: 54,  # 'ъ'
+    224: 59,  # 'Ю'
+    225: 37,  # 'А'
+    226: 44,  # 'Б'
+    227: 58,  # 'Ц'
+    228: 41,  # 'Д'
+    229: 48,  # 'Е'
+    230: 53,  # 'Ф'
+    231: 46,  # 'Г'
+    232: 55,  # 'Х'
+    233: 42,  # 'И'
+    234: 60,  # 'Й'
+    235: 36,  # 'К'
+    236: 49,  # 'Л'
+    237: 38,  # 'М'
+    238: 31,  # 'Н'
+    239: 34,  # 'О'
+    240: 35,  # 'П'
+    241: 43,  # 'Я'
+    242: 45,  # 'Р'
+    243: 32,  # 'С'
+    244: 40,  # 'Т'
+    245: 52,  # 'У'
+    246: 56,  # 'Ж'
+    247: 33,  # 'В'
+    248: 61,  # 'Ь'
+    249: 62,  # 'Ы'
+    250: 51,  # 'З'
+    251: 57,  # 'Ш'
+    252: 47,  # 'Э'
+    253: 63,  # 'Щ'
+    254: 50,  # 'Ч'
+    255: 70,  # 'Ъ'
+}
+
+KOI8_R_RUSSIAN_MODEL = SingleByteCharSetModel(
+    charset_name="KOI8-R",
+    language="Russian",
+    char_to_order_map=KOI8_R_RUSSIAN_CHAR_TO_ORDER,
+    language_model=RUSSIAN_LANG_MODEL,
+    typical_positive_ratio=0.976601,
+    keep_ascii_letters=False,
+    alphabet="ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё",
+)
+
+MACCYRILLIC_RUSSIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 142,  # 'A'
+    66: 143,  # 'B'
+    67: 144,  # 'C'
+    68: 145,  # 'D'
+    69: 146,  # 'E'
+    70: 147,  # 'F'
+    71: 148,  # 'G'
+    72: 149,  # 'H'
+    73: 150,  # 'I'
+    74: 151,  # 'J'
+    75: 152,  # 'K'
+    76: 74,  # 'L'
+    77: 153,  # 'M'
+    78: 75,  # 'N'
+    79: 154,  # 'O'
+    80: 155,  # 'P'
+    81: 156,  # 'Q'
+    82: 157,  # 'R'
+    83: 158,  # 'S'
+    84: 159,  # 'T'
+    85: 160,  # 'U'
+    86: 161,  # 'V'
+    87: 162,  # 'W'
+    88: 163,  # 'X'
+    89: 164,  # 'Y'
+    90: 165,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 71,  # 'a'
+    98: 172,  # 'b'
+    99: 66,  # 'c'
+    100: 173,  # 'd'
+    101: 65,  # 'e'
+    102: 174,  # 'f'
+    103: 76,  # 'g'
+    104: 175,  # 'h'
+    105: 64,  # 'i'
+    106: 176,  # 'j'
+    107: 177,  # 'k'
+    108: 77,  # 'l'
+    109: 72,  # 'm'
+    110: 178,  # 'n'
+    111: 69,  # 'o'
+    112: 67,  # 'p'
+    113: 179,  # 'q'
+    114: 78,  # 'r'
+    115: 73,  # 's'
+    116: 180,  # 't'
+    117: 181,  # 'u'
+    118: 79,  # 'v'
+    119: 182,  # 'w'
+    120: 183,  # 'x'
+    121: 184,  # 'y'
+    122: 185,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 37,  # 'А'
+    129: 44,  # 'Б'
+    130: 33,  # 'В'
+    131: 46,  # 'Г'
+    132: 41,  # 'Д'
+    133: 48,  # 'Е'
+    134: 56,  # 'Ж'
+    135: 51,  # 'З'
+    136: 42,  # 'И'
+    137: 60,  # 'Й'
+    138: 36,  # 'К'
+    139: 49,  # 'Л'
+    140: 38,  # 'М'
+    141: 31,  # 'Н'
+    142: 34,  # 'О'
+    143: 35,  # 'П'
+    144: 45,  # 'Р'
+    145: 32,  # 'С'
+    146: 40,  # 'Т'
+    147: 52,  # 'У'
+    148: 53,  # 'Ф'
+    149: 55,  # 'Х'
+    150: 58,  # 'Ц'
+    151: 50,  # 'Ч'
+    152: 57,  # 'Ш'
+    153: 63,  # 'Щ'
+    154: 70,  # 'Ъ'
+    155: 62,  # 'Ы'
+    156: 61,  # 'Ь'
+    157: 47,  # 'Э'
+    158: 59,  # 'Ю'
+    159: 43,  # 'Я'
+    160: 191,  # '†'
+    161: 192,  # '°'
+    162: 193,  # 'Ґ'
+    163: 194,  # '£'
+    164: 195,  # '§'
+    165: 196,  # '•'
+    166: 197,  # '¶'
+    167: 198,  # 'І'
+    168: 199,  # '®'
+    169: 200,  # '©'
+    170: 201,  # '™'
+    171: 202,  # 'Ђ'
+    172: 203,  # 'ђ'
+    173: 204,  # '≠'
+    174: 205,  # 'Ѓ'
+    175: 206,  # 'ѓ'
+    176: 207,  # '∞'
+    177: 208,  # '±'
+    178: 209,  # '≤'
+    179: 210,  # '≥'
+    180: 211,  # 'і'
+    181: 212,  # 'µ'
+    182: 213,  # 'ґ'
+    183: 214,  # 'Ј'
+    184: 215,  # 'Є'
+    185: 216,  # 'є'
+    186: 217,  # 'Ї'
+    187: 218,  # 'ї'
+    188: 219,  # 'Љ'
+    189: 220,  # 'љ'
+    190: 221,  # 'Њ'
+    191: 222,  # 'њ'
+    192: 223,  # 'ј'
+    193: 224,  # 'Ѕ'
+    194: 225,  # '¬'
+    195: 226,  # '√'
+    196: 227,  # 'ƒ'
+    197: 228,  # '≈'
+    198: 229,  # '∆'
+    199: 230,  # '«'
+    200: 231,  # '»'
+    201: 232,  # '…'
+    202: 233,  # '\xa0'
+    203: 234,  # 'Ћ'
+    204: 235,  # 'ћ'
+    205: 236,  # 'Ќ'
+    206: 237,  # 'ќ'
+    207: 238,  # 'ѕ'
+    208: 239,  # '–'
+    209: 240,  # '—'
+    210: 241,  # '“'
+    211: 242,  # '”'
+    212: 243,  # '‘'
+    213: 244,  # '’'
+    214: 245,  # '÷'
+    215: 246,  # '„'
+    216: 247,  # 'Ў'
+    217: 248,  # 'ў'
+    218: 249,  # 'Џ'
+    219: 250,  # 'џ'
+    220: 251,  # '№'
+    221: 252,  # 'Ё'
+    222: 68,  # 'ё'
+    223: 16,  # 'я'
+    224: 3,  # 'а'
+    225: 21,  # 'б'
+    226: 10,  # 'в'
+    227: 19,  # 'г'
+    228: 13,  # 'д'
+    229: 2,  # 'е'
+    230: 24,  # 'ж'
+    231: 20,  # 'з'
+    232: 4,  # 'и'
+    233: 23,  # 'й'
+    234: 11,  # 'к'
+    235: 8,  # 'л'
+    236: 12,  # 'м'
+    237: 5,  # 'н'
+    238: 1,  # 'о'
+    239: 15,  # 'п'
+    240: 9,  # 'р'
+    241: 7,  # 'с'
+    242: 6,  # 'т'
+    243: 14,  # 'у'
+    244: 39,  # 'ф'
+    245: 26,  # 'х'
+    246: 28,  # 'ц'
+    247: 22,  # 'ч'
+    248: 25,  # 'ш'
+    249: 29,  # 'щ'
+    250: 54,  # 'ъ'
+    251: 18,  # 'ы'
+    252: 17,  # 'ь'
+    253: 30,  # 'э'
+    254: 27,  # 'ю'
+    255: 255,  # '€'
+}
+
+MACCYRILLIC_RUSSIAN_MODEL = SingleByteCharSetModel(
+    charset_name="MacCyrillic",
+    language="Russian",
+    char_to_order_map=MACCYRILLIC_RUSSIAN_CHAR_TO_ORDER,
+    language_model=RUSSIAN_LANG_MODEL,
+    typical_positive_ratio=0.976601,
+    keep_ascii_letters=False,
+    alphabet="ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё",
+)
+
+ISO_8859_5_RUSSIAN_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 142,  # 'A'
+    66: 143,  # 'B'
+    67: 144,  # 'C'
+    68: 145,  # 'D'
+    69: 146,  # 'E'
+    70: 147,  # 'F'
+    71: 148,  # 'G'
+    72: 149,  # 'H'
+    73: 150,  # 'I'
+    74: 151,  # 'J'
+    75: 152,  # 'K'
+    76: 74,  # 'L'
+    77: 153,  # 'M'
+    78: 75,  # 'N'
+    79: 154,  # 'O'
+    80: 155,  # 'P'
+    81: 156,  # 'Q'
+    82: 157,  # 'R'
+    83: 158,  # 'S'
+    84: 159,  # 'T'
+    85: 160,  # 'U'
+    86: 161,  # 'V'
+    87: 162,  # 'W'
+    88: 163,  # 'X'
+    89: 164,  # 'Y'
+    90: 165,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 71,  # 'a'
+    98: 172,  # 'b'
+    99: 66,  # 'c'
+    100: 173,  # 'd'
+    101: 65,  # 'e'
+    102: 174,  # 'f'
+    103: 76,  # 'g'
+    104: 175,  # 'h'
+    105: 64,  # 'i'
+    106: 176,  # 'j'
+    107: 177,  # 'k'
+    108: 77,  # 'l'
+    109: 72,  # 'm'
+    110: 178,  # 'n'
+    111: 69,  # 'o'
+    112: 67,  # 'p'
+    113: 179,  # 'q'
+    114: 78,  # 'r'
+    115: 73,  # 's'
+    116: 180,  # 't'
+    117: 181,  # 'u'
+    118: 79,  # 'v'
+    119: 182,  # 'w'
+    120: 183,  # 'x'
+    121: 184,  # 'y'
+    122: 185,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 191,  # '\x80'
+    129: 192,  # '\x81'
+    130: 193,  # '\x82'
+    131: 194,  # '\x83'
+    132: 195,  # '\x84'
+    133: 196,  # '\x85'
+    134: 197,  # '\x86'
+    135: 198,  # '\x87'
+    136: 199,  # '\x88'
+    137: 200,  # '\x89'
+    138: 201,  # '\x8a'
+    139: 202,  # '\x8b'
+    140: 203,  # '\x8c'
+    141: 204,  # '\x8d'
+    142: 205,  # '\x8e'
+    143: 206,  # '\x8f'
+    144: 207,  # '\x90'
+    145: 208,  # '\x91'
+    146: 209,  # '\x92'
+    147: 210,  # '\x93'
+    148: 211,  # '\x94'
+    149: 212,  # '\x95'
+    150: 213,  # '\x96'
+    151: 214,  # '\x97'
+    152: 215,  # '\x98'
+    153: 216,  # '\x99'
+    154: 217,  # '\x9a'
+    155: 218,  # '\x9b'
+    156: 219,  # '\x9c'
+    157: 220,  # '\x9d'
+    158: 221,  # '\x9e'
+    159: 222,  # '\x9f'
+    160: 223,  # '\xa0'
+    161: 224,  # 'Ё'
+    162: 225,  # 'Ђ'
+    163: 226,  # 'Ѓ'
+    164: 227,  # 'Є'
+    165: 228,  # 'Ѕ'
+    166: 229,  # 'І'
+    167: 230,  # 'Ї'
+    168: 231,  # 'Ј'
+    169: 232,  # 'Љ'
+    170: 233,  # 'Њ'
+    171: 234,  # 'Ћ'
+    172: 235,  # 'Ќ'
+    173: 236,  # '\xad'
+    174: 237,  # 'Ў'
+    175: 238,  # 'Џ'
+    176: 37,  # 'А'
+    177: 44,  # 'Б'
+    178: 33,  # 'В'
+    179: 46,  # 'Г'
+    180: 41,  # 'Д'
+    181: 48,  # 'Е'
+    182: 56,  # 'Ж'
+    183: 51,  # 'З'
+    184: 42,  # 'И'
+    185: 60,  # 'Й'
+    186: 36,  # 'К'
+    187: 49,  # 'Л'
+    188: 38,  # 'М'
+    189: 31,  # 'Н'
+    190: 34,  # 'О'
+    191: 35,  # 'П'
+    192: 45,  # 'Р'
+    193: 32,  # 'С'
+    194: 40,  # 'Т'
+    195: 52,  # 'У'
+    196: 53,  # 'Ф'
+    197: 55,  # 'Х'
+    198: 58,  # 'Ц'
+    199: 50,  # 'Ч'
+    200: 57,  # 'Ш'
+    201: 63,  # 'Щ'
+    202: 70,  # 'Ъ'
+    203: 62,  # 'Ы'
+    204: 61,  # 'Ь'
+    205: 47,  # 'Э'
+    206: 59,  # 'Ю'
+    207: 43,  # 'Я'
+    208: 3,  # 'а'
+    209: 21,  # 'б'
+    210: 10,  # 'в'
+    211: 19,  # 'г'
+    212: 13,  # 'д'
+    213: 2,  # 'е'
+    214: 24,  # 'ж'
+    215: 20,  # 'з'
+    216: 4,  # 'и'
+    217: 23,  # 'й'
+    218: 11,  # 'к'
+    219: 8,  # 'л'
+    220: 12,  # 'м'
+    221: 5,  # 'н'
+    222: 1,  # 'о'
+    223: 15,  # 'п'
+    224: 9,  # 'р'
+    225: 7,  # 'с'
+    226: 6,  # 'т'
+    227: 14,  # 'у'
+    228: 39,  # 'ф'
+    229: 26,  # 'х'
+    230: 28,  # 'ц'
+    231: 22,  # 'ч'
+    232: 25,  # 'ш'
+    233: 29,  # 'щ'
+    234: 54,  # 'ъ'
+    235: 18,  # 'ы'
+    236: 17,  # 'ь'
+    237: 30,  # 'э'
+    238: 27,  # 'ю'
+    239: 16,  # 'я'
+    240: 239,  # '№'
+    241: 68,  # 'ё'
+    242: 240,  # 'ђ'
+    243: 241,  # 'ѓ'
+    244: 242,  # 'є'
+    245: 243,  # 'ѕ'
+    246: 244,  # 'і'
+    247: 245,  # 'ї'
+    248: 246,  # 'ј'
+    249: 247,  # 'љ'
+    250: 248,  # 'њ'
+    251: 249,  # 'ћ'
+    252: 250,  # 'ќ'
+    253: 251,  # '§'
+    254: 252,  # 'ў'
+    255: 255,  # 'џ'
+}
+
+ISO_8859_5_RUSSIAN_MODEL = SingleByteCharSetModel(
+    charset_name="ISO-8859-5",
+    language="Russian",
+    char_to_order_map=ISO_8859_5_RUSSIAN_CHAR_TO_ORDER,
+    language_model=RUSSIAN_LANG_MODEL,
+    typical_positive_ratio=0.976601,
+    keep_ascii_letters=False,
+    alphabet="ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё",
+)
diff --git a/contrib/python/chardet/py3/chardet/langthaimodel.py b/contrib/python/chardet/py3/chardet/langthaimodel.py
new file mode 100644
index 00000000000..883fdb1eafe
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/langthaimodel.py
@@ -0,0 +1,4380 @@
+from chardet.sbcharsetprober import SingleByteCharSetModel
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+THAI_LANG_MODEL = {
+    5: {  # 'ก'
+        5: 2,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 2,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 3,  # 'ฎ'
+        57: 2,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 2,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 3,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 1,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 3,  # 'ร'
+        61: 2,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 3,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 3,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    30: {  # 'ข'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 1,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 2,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 2,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 2,  # '่'
+        7: 3,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    24: {  # 'ค'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 2,  # 'ค'
+        8: 2,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 2,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 0,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 3,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 2,  # 'า'
+        36: 3,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 3,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    8: {  # 'ง'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 3,  # 'ค'
+        8: 2,  # 'ง'
+        26: 2,  # 'จ'
+        52: 1,  # 'ฉ'
+        34: 2,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 1,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 3,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    26: {  # 'จ'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 0,  # 'ค'
+        8: 2,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 1,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 1,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 2,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    52: {  # 'ฉ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 3,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 1,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 1,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    34: {  # 'ช'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 1,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 1,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 1,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    51: {  # 'ซ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 1,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 1,  # '่'
+        7: 2,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    47: {  # 'ญ'
+        5: 1,  # 'ก'
+        30: 1,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 3,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 2,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    58: {  # 'ฎ'
+        5: 2,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 1,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    57: {  # 'ฏ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    49: {  # 'ฐ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    53: {  # 'ฑ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    55: {  # 'ฒ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    43: {  # 'ณ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 3,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 3,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    20: {  # 'ด'
+        5: 2,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 3,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 2,  # 'า'
+        36: 2,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 1,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 2,  # 'ๆ'
+        37: 2,  # '็'
+        6: 1,  # '่'
+        7: 3,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    19: {  # 'ต'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 1,  # 'ต'
+        44: 2,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 2,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 1,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 0,  # 'ห'
+        4: 3,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 1,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 2,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    44: {  # 'ถ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    14: {  # 'ท'
+        5: 1,  # 'ก'
+        30: 1,  # 'ข'
+        24: 3,  # 'ค'
+        8: 1,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 3,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 3,  # 'ย'
+        2: 3,  # 'ร'
+        61: 1,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 2,  # 'ว'
+        42: 3,  # 'ศ'
+        46: 1,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 0,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    48: {  # 'ธ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 1,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 2,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    3: {  # 'น'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 3,  # 'ค'
+        8: 1,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 1,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 2,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 3,  # 'ธ'
+        3: 2,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 1,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 3,  # 'โ'
+        29: 3,  # 'ใ'
+        33: 3,  # 'ไ'
+        50: 2,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    17: {  # 'บ'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 1,  # 'ง'
+        26: 1,  # 'จ'
+        52: 1,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 2,  # '่'
+        7: 2,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    25: {  # 'ป'
+        5: 2,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 1,  # 'ฎ'
+        57: 3,  # 'ฏ'
+        49: 1,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 0,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 1,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 1,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 2,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 3,  # '็'
+        6: 1,  # '่'
+        7: 2,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    39: {  # 'ผ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 1,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 2,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 1,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    62: {  # 'ฝ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 1,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    31: {  # 'พ'
+        5: 1,  # 'ก'
+        30: 1,  # 'ข'
+        24: 1,  # 'ค'
+        8: 1,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 2,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 1,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 0,  # '่'
+        7: 1,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    54: {  # 'ฟ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 2,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 2,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    45: {  # 'ภ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    9: {  # 'ม'
+        5: 2,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 2,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 2,  # 'ร'
+        61: 2,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 1,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 2,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 2,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    16: {  # 'ย'
+        5: 3,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 2,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 3,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 1,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 2,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 2,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 2,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    2: {  # 'ร'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 2,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 3,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 3,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 2,  # 'น'
+        17: 2,  # 'บ'
+        25: 3,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 3,  # 'เ'
+        28: 3,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 3,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    61: {  # 'ฤ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    15: {  # 'ล'
+        5: 2,  # 'ก'
+        30: 3,  # 'ข'
+        24: 1,  # 'ค'
+        8: 3,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 3,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 3,  # 'อ'
+        63: 2,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 2,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 2,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    12: {  # 'ว'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 1,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    42: {  # 'ศ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 1,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 2,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 0,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    46: {  # 'ษ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 2,  # 'ฎ'
+        57: 1,  # 'ฏ'
+        49: 2,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    18: {  # 'ส'
+        5: 2,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 2,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 3,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 2,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 1,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 2,  # 'เ'
+        28: 0,  # 'แ'
+        41: 1,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 1,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    21: {  # 'ห'
+        5: 3,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 1,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 3,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 0,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 3,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    4: {  # 'อ'
+        5: 3,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 1,  # '็'
+        6: 2,  # '่'
+        7: 2,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    63: {  # 'ฯ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    22: {  # 'ะ'
+        5: 3,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 1,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 2,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    10: {  # 'ั'
+        5: 3,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 3,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 3,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 2,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 3,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    1: {  # 'า'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 3,  # 'ค'
+        8: 3,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 2,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 3,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    36: {  # 'ำ'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 3,  # 'ค'
+        8: 2,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 1,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 3,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    23: {  # 'ิ'
+        5: 3,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 3,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 2,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 3,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 3,  # 'ห'
+        4: 1,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 2,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    13: {  # 'ี'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 3,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 2,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    40: {  # 'ึ'
+        5: 3,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 3,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    27: {  # 'ื'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 3,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    32: {  # 'ุ'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 3,  # 'ค'
+        8: 3,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 1,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 2,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 1,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 1,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 1,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 2,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    35: {  # 'ู'
+        5: 3,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 2,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 0,  # 'บ'
+        25: 3,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    11: {  # 'เ'
+        5: 3,  # 'ก'
+        30: 3,  # 'ข'
+        24: 3,  # 'ค'
+        8: 2,  # 'ง'
+        26: 3,  # 'จ'
+        52: 3,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 2,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 3,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 3,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    28: {  # 'แ'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 1,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 3,  # 'ต'
+        44: 2,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 3,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 2,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    41: {  # 'โ'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 1,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 1,  # 'บ'
+        25: 3,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 0,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    29: {  # 'ใ'
+        5: 2,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 1,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    33: {  # 'ไ'
+        5: 1,  # 'ก'
+        30: 2,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 1,  # 'บ'
+        25: 3,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 2,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 0,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 3,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 2,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    50: {  # 'ๆ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    37: {  # '็'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 2,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 1,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    6: {  # '่'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 1,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 1,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    7: {  # '้'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 3,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    38: {  # '์'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 1,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    56: {  # '๑'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 2,  # '๑'
+        59: 1,  # '๒'
+        60: 1,  # '๕'
+    },
+    59: {  # '๒'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 1,  # '๑'
+        59: 1,  # '๒'
+        60: 3,  # '๕'
+    },
+    60: {  # '๕'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 2,  # '๑'
+        59: 1,  # '๒'
+        60: 0,  # '๕'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
+
+# Character Mapping Table(s):
+TIS_620_THAI_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 182,  # 'A'
+    66: 106,  # 'B'
+    67: 107,  # 'C'
+    68: 100,  # 'D'
+    69: 183,  # 'E'
+    70: 184,  # 'F'
+    71: 185,  # 'G'
+    72: 101,  # 'H'
+    73: 94,  # 'I'
+    74: 186,  # 'J'
+    75: 187,  # 'K'
+    76: 108,  # 'L'
+    77: 109,  # 'M'
+    78: 110,  # 'N'
+    79: 111,  # 'O'
+    80: 188,  # 'P'
+    81: 189,  # 'Q'
+    82: 190,  # 'R'
+    83: 89,  # 'S'
+    84: 95,  # 'T'
+    85: 112,  # 'U'
+    86: 113,  # 'V'
+    87: 191,  # 'W'
+    88: 192,  # 'X'
+    89: 193,  # 'Y'
+    90: 194,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 64,  # 'a'
+    98: 72,  # 'b'
+    99: 73,  # 'c'
+    100: 114,  # 'd'
+    101: 74,  # 'e'
+    102: 115,  # 'f'
+    103: 116,  # 'g'
+    104: 102,  # 'h'
+    105: 81,  # 'i'
+    106: 201,  # 'j'
+    107: 117,  # 'k'
+    108: 90,  # 'l'
+    109: 103,  # 'm'
+    110: 78,  # 'n'
+    111: 82,  # 'o'
+    112: 96,  # 'p'
+    113: 202,  # 'q'
+    114: 91,  # 'r'
+    115: 79,  # 's'
+    116: 84,  # 't'
+    117: 104,  # 'u'
+    118: 105,  # 'v'
+    119: 97,  # 'w'
+    120: 98,  # 'x'
+    121: 92,  # 'y'
+    122: 203,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 209,  # '\x80'
+    129: 210,  # '\x81'
+    130: 211,  # '\x82'
+    131: 212,  # '\x83'
+    132: 213,  # '\x84'
+    133: 88,  # '\x85'
+    134: 214,  # '\x86'
+    135: 215,  # '\x87'
+    136: 216,  # '\x88'
+    137: 217,  # '\x89'
+    138: 218,  # '\x8a'
+    139: 219,  # '\x8b'
+    140: 220,  # '\x8c'
+    141: 118,  # '\x8d'
+    142: 221,  # '\x8e'
+    143: 222,  # '\x8f'
+    144: 223,  # '\x90'
+    145: 224,  # '\x91'
+    146: 99,  # '\x92'
+    147: 85,  # '\x93'
+    148: 83,  # '\x94'
+    149: 225,  # '\x95'
+    150: 226,  # '\x96'
+    151: 227,  # '\x97'
+    152: 228,  # '\x98'
+    153: 229,  # '\x99'
+    154: 230,  # '\x9a'
+    155: 231,  # '\x9b'
+    156: 232,  # '\x9c'
+    157: 233,  # '\x9d'
+    158: 234,  # '\x9e'
+    159: 235,  # '\x9f'
+    160: 236,  # None
+    161: 5,  # 'ก'
+    162: 30,  # 'ข'
+    163: 237,  # 'ฃ'
+    164: 24,  # 'ค'
+    165: 238,  # 'ฅ'
+    166: 75,  # 'ฆ'
+    167: 8,  # 'ง'
+    168: 26,  # 'จ'
+    169: 52,  # 'ฉ'
+    170: 34,  # 'ช'
+    171: 51,  # 'ซ'
+    172: 119,  # 'ฌ'
+    173: 47,  # 'ญ'
+    174: 58,  # 'ฎ'
+    175: 57,  # 'ฏ'
+    176: 49,  # 'ฐ'
+    177: 53,  # 'ฑ'
+    178: 55,  # 'ฒ'
+    179: 43,  # 'ณ'
+    180: 20,  # 'ด'
+    181: 19,  # 'ต'
+    182: 44,  # 'ถ'
+    183: 14,  # 'ท'
+    184: 48,  # 'ธ'
+    185: 3,  # 'น'
+    186: 17,  # 'บ'
+    187: 25,  # 'ป'
+    188: 39,  # 'ผ'
+    189: 62,  # 'ฝ'
+    190: 31,  # 'พ'
+    191: 54,  # 'ฟ'
+    192: 45,  # 'ภ'
+    193: 9,  # 'ม'
+    194: 16,  # 'ย'
+    195: 2,  # 'ร'
+    196: 61,  # 'ฤ'
+    197: 15,  # 'ล'
+    198: 239,  # 'ฦ'
+    199: 12,  # 'ว'
+    200: 42,  # 'ศ'
+    201: 46,  # 'ษ'
+    202: 18,  # 'ส'
+    203: 21,  # 'ห'
+    204: 76,  # 'ฬ'
+    205: 4,  # 'อ'
+    206: 66,  # 'ฮ'
+    207: 63,  # 'ฯ'
+    208: 22,  # 'ะ'
+    209: 10,  # 'ั'
+    210: 1,  # 'า'
+    211: 36,  # 'ำ'
+    212: 23,  # 'ิ'
+    213: 13,  # 'ี'
+    214: 40,  # 'ึ'
+    215: 27,  # 'ื'
+    216: 32,  # 'ุ'
+    217: 35,  # 'ู'
+    218: 86,  # 'ฺ'
+    219: 240,  # None
+    220: 241,  # None
+    221: 242,  # None
+    222: 243,  # None
+    223: 244,  # '฿'
+    224: 11,  # 'เ'
+    225: 28,  # 'แ'
+    226: 41,  # 'โ'
+    227: 29,  # 'ใ'
+    228: 33,  # 'ไ'
+    229: 245,  # 'ๅ'
+    230: 50,  # 'ๆ'
+    231: 37,  # '็'
+    232: 6,  # '่'
+    233: 7,  # '้'
+    234: 67,  # '๊'
+    235: 77,  # '๋'
+    236: 38,  # '์'
+    237: 93,  # 'ํ'
+    238: 246,  # '๎'
+    239: 247,  # '๏'
+    240: 68,  # '๐'
+    241: 56,  # '๑'
+    242: 59,  # '๒'
+    243: 65,  # '๓'
+    244: 69,  # '๔'
+    245: 60,  # '๕'
+    246: 70,  # '๖'
+    247: 80,  # '๗'
+    248: 71,  # '๘'
+    249: 87,  # '๙'
+    250: 248,  # '๚'
+    251: 249,  # '๛'
+    252: 250,  # None
+    253: 251,  # None
+    254: 252,  # None
+    255: 253,  # None
+}
+
+TIS_620_THAI_MODEL = SingleByteCharSetModel(
+    charset_name="TIS-620",
+    language="Thai",
+    char_to_order_map=TIS_620_THAI_CHAR_TO_ORDER,
+    language_model=THAI_LANG_MODEL,
+    typical_positive_ratio=0.926386,
+    keep_ascii_letters=False,
+    alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛",
+)
diff --git a/contrib/python/chardet/py3/chardet/langturkishmodel.py b/contrib/python/chardet/py3/chardet/langturkishmodel.py
new file mode 100644
index 00000000000..64c94336cb2
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/langturkishmodel.py
@@ -0,0 +1,4380 @@
+from chardet.sbcharsetprober import SingleByteCharSetModel
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+TURKISH_LANG_MODEL = {
+    23: {  # 'A'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 1,  # 'i'
+        24: 0,  # 'j'
+        10: 2,  # 'k'
+        5: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    37: {  # 'B'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 2,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    47: {  # 'C'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 1,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 2,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    39: {  # 'D'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 1,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    29: {  # 'E'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 1,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 1,  # 'j'
+        10: 0,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 1,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    52: {  # 'F'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 1,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 2,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 1,  # 'b'
+        28: 1,  # 'c'
+        12: 1,  # 'd'
+        2: 0,  # 'e'
+        18: 1,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 1,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 2,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 2,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    36: {  # 'G'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 2,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 2,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 1,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 1,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 0,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 1,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    45: {  # 'H'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 2,  # 'G'
+        45: 1,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 1,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 2,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 1,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 0,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    53: {  # 'I'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    60: {  # 'J'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 0,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 1,  # 's'
+        9: 0,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    16: {  # 'K'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 1,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 0,  # 'u'
+        32: 3,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    49: {  # 'L'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 2,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 2,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 0,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 1,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 2,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 1,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    20: {  # 'M'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 2,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 0,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    46: {  # 'N'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 1,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 1,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 1,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    42: {  # 'O'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 1,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 2,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 2,  # 'İ'
+        6: 1,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    48: {  # 'P'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 2,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 0,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    44: {  # 'R'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 1,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 1,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    35: {  # 'S'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 1,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 1,  # 'l'
+        13: 2,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 1,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 2,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    31: {  # 'T'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 2,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 1,  # 'j'
+        10: 2,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 2,  # 'r'
+        8: 0,  # 's'
+        9: 2,  # 't'
+        14: 2,  # 'u'
+        32: 1,  # 'v'
+        57: 1,  # 'w'
+        58: 1,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    51: {  # 'U'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 1,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 1,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    38: {  # 'V'
+        23: 1,  # 'A'
+        37: 1,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 1,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 1,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    62: {  # 'W'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 0,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    43: {  # 'Y'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 1,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 2,  # 'N'
+        42: 0,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 1,  # 'j'
+        10: 1,  # 'k'
+        5: 1,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 1,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 0,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    56: {  # 'Z'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 2,  # 'Z'
+        1: 2,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 1,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 1,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    1: {  # 'a'
+        23: 3,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 2,  # 'Z'
+        1: 2,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 2,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 3,  # 'v'
+        57: 2,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 1,  # 'î'
+        34: 1,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    21: {  # 'b'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 3,  # 'g'
+        25: 1,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 1,  # 'r'
+        8: 2,  # 's'
+        9: 2,  # 't'
+        14: 2,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    28: {  # 'c'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 2,  # 'E'
+        52: 0,  # 'F'
+        36: 2,  # 'G'
+        45: 2,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 2,  # 'T'
+        51: 2,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 3,  # 'Y'
+        56: 0,  # 'Z'
+        1: 1,  # 'a'
+        21: 1,  # 'b'
+        28: 2,  # 'c'
+        12: 2,  # 'd'
+        2: 1,  # 'e'
+        18: 1,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 1,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 2,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 1,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 1,  # 'î'
+        34: 2,  # 'ö'
+        17: 2,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    12: {  # 'd'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 2,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 1,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 2,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 1,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    2: {  # 'e'
+        23: 2,  # 'A'
+        37: 0,  # 'B'
+        47: 2,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 2,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 3,  # 'v'
+        57: 2,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 1,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    18: {  # 'f'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 1,  # 'i'
+        24: 1,  # 'j'
+        10: 1,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 1,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 1,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    27: {  # 'g'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 2,  # 'r'
+        8: 2,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    25: {  # 'h'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    3: {  # 'i'
+        23: 2,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 1,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 3,  # 'g'
+        25: 1,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 1,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 1,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    24: {  # 'j'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 1,  # 'j'
+        10: 2,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 2,  # 'r'
+        8: 3,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    10: {  # 'k'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 1,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 2,  # 'r'
+        8: 2,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 3,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    5: {  # 'l'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 1,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 2,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    13: {  # 'm'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 2,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 2,  # 'u'
+        32: 2,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    4: {  # 'n'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 2,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 1,  # 'f'
+        27: 2,  # 'g'
+        25: 3,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 3,  # 'p'
+        7: 2,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    15: {  # 'o'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 2,  # 'L'
+        20: 0,  # 'M'
+        46: 2,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 2,  # 'İ'
+        6: 3,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    26: {  # 'p'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 1,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 2,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    7: {  # 'r'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 2,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 1,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 3,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    8: {  # 's'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 2,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    9: {  # 't'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 3,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    14: {  # 'u'
+        23: 3,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 2,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 2,  # 'Z'
+        1: 2,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 2,  # 'e'
+        18: 2,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 2,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    32: {  # 'v'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 1,  # 'j'
+        10: 1,  # 'k'
+        5: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 1,  # 'r'
+        8: 2,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    57: {  # 'w'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 1,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 1,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 1,  # 's'
+        9: 0,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 2,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    58: {  # 'x'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 1,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 1,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 2,  # 's'
+        9: 1,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    11: {  # 'y'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 1,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 2,  # 'r'
+        8: 1,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    22: {  # 'z'
+        23: 2,  # 'A'
+        37: 2,  # 'B'
+        47: 1,  # 'C'
+        39: 2,  # 'D'
+        29: 3,  # 'E'
+        52: 1,  # 'F'
+        36: 2,  # 'G'
+        45: 2,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 2,  # 'N'
+        42: 2,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 3,  # 'T'
+        51: 2,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 1,  # 'Z'
+        1: 1,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 2,  # 'd'
+        2: 2,  # 'e'
+        18: 3,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 2,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 0,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 2,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 2,  # 'Ü'
+        59: 1,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 2,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    63: {  # '·'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 1,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    54: {  # 'Ç'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 0,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 0,  # 'h'
+        3: 3,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 2,  # 'r'
+        8: 0,  # 's'
+        9: 1,  # 't'
+        14: 0,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 2,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    50: {  # 'Ö'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 2,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 2,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 1,  # 'N'
+        42: 2,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 2,  # 'd'
+        2: 0,  # 'e'
+        18: 1,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 0,  # 'j'
+        10: 2,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 3,  # 'n'
+        15: 2,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 1,  # 's'
+        9: 2,  # 't'
+        14: 0,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 2,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    55: {  # 'Ü'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 1,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    59: {  # 'â'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    33: {  # 'ç'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 0,  # 'e'
+        18: 2,  # 'f'
+        27: 1,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 0,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 2,  # 's'
+        9: 3,  # 't'
+        14: 0,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    61: {  # 'î'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 1,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 1,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    34: {  # 'ö'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 1,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 2,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 1,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 3,  # 's'
+        9: 1,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 1,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    17: {  # 'ü'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 1,  # 'f'
+        27: 2,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 1,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 2,  # 'r'
+        8: 3,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 1,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    30: {  # 'ğ'
+        23: 0,  # 'A'
+        37: 2,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 2,  # 'N'
+        42: 2,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 3,  # 'j'
+        10: 1,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 2,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 2,  # 'İ'
+        6: 2,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    41: {  # 'İ'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 1,  # 'E'
+        52: 0,  # 'F'
+        36: 2,  # 'G'
+        45: 2,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 1,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 2,  # 'd'
+        2: 1,  # 'e'
+        18: 0,  # 'f'
+        27: 3,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 2,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 2,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 1,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 1,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    6: {  # 'ı'
+        23: 2,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 2,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 3,  # 'v'
+        57: 1,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    40: {  # 'Ş'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 1,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 2,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 2,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 1,  # 'Z'
+        1: 0,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 0,  # 'e'
+        18: 3,  # 'f'
+        27: 0,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 3,  # 'r'
+        8: 2,  # 's'
+        9: 2,  # 't'
+        14: 1,  # 'u'
+        32: 3,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 1,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    19: {  # 'ş'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 2,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 1,  # 'h'
+        3: 1,  # 'i'
+        24: 0,  # 'j'
+        10: 2,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 1,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 1,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
+
+# Character Mapping Table(s):
+ISO_8859_9_TURKISH_CHAR_TO_ORDER = {
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 255,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 255,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 255,  # ' '
+    33: 255,  # '!'
+    34: 255,  # '"'
+    35: 255,  # '#'
+    36: 255,  # '$'
+    37: 255,  # '%'
+    38: 255,  # '&'
+    39: 255,  # "'"
+    40: 255,  # '('
+    41: 255,  # ')'
+    42: 255,  # '*'
+    43: 255,  # '+'
+    44: 255,  # ','
+    45: 255,  # '-'
+    46: 255,  # '.'
+    47: 255,  # '/'
+    48: 255,  # '0'
+    49: 255,  # '1'
+    50: 255,  # '2'
+    51: 255,  # '3'
+    52: 255,  # '4'
+    53: 255,  # '5'
+    54: 255,  # '6'
+    55: 255,  # '7'
+    56: 255,  # '8'
+    57: 255,  # '9'
+    58: 255,  # ':'
+    59: 255,  # ';'
+    60: 255,  # '<'
+    61: 255,  # '='
+    62: 255,  # '>'
+    63: 255,  # '?'
+    64: 255,  # '@'
+    65: 23,  # 'A'
+    66: 37,  # 'B'
+    67: 47,  # 'C'
+    68: 39,  # 'D'
+    69: 29,  # 'E'
+    70: 52,  # 'F'
+    71: 36,  # 'G'
+    72: 45,  # 'H'
+    73: 53,  # 'I'
+    74: 60,  # 'J'
+    75: 16,  # 'K'
+    76: 49,  # 'L'
+    77: 20,  # 'M'
+    78: 46,  # 'N'
+    79: 42,  # 'O'
+    80: 48,  # 'P'
+    81: 69,  # 'Q'
+    82: 44,  # 'R'
+    83: 35,  # 'S'
+    84: 31,  # 'T'
+    85: 51,  # 'U'
+    86: 38,  # 'V'
+    87: 62,  # 'W'
+    88: 65,  # 'X'
+    89: 43,  # 'Y'
+    90: 56,  # 'Z'
+    91: 255,  # '['
+    92: 255,  # '\\'
+    93: 255,  # ']'
+    94: 255,  # '^'
+    95: 255,  # '_'
+    96: 255,  # '`'
+    97: 1,  # 'a'
+    98: 21,  # 'b'
+    99: 28,  # 'c'
+    100: 12,  # 'd'
+    101: 2,  # 'e'
+    102: 18,  # 'f'
+    103: 27,  # 'g'
+    104: 25,  # 'h'
+    105: 3,  # 'i'
+    106: 24,  # 'j'
+    107: 10,  # 'k'
+    108: 5,  # 'l'
+    109: 13,  # 'm'
+    110: 4,  # 'n'
+    111: 15,  # 'o'
+    112: 26,  # 'p'
+    113: 64,  # 'q'
+    114: 7,  # 'r'
+    115: 8,  # 's'
+    116: 9,  # 't'
+    117: 14,  # 'u'
+    118: 32,  # 'v'
+    119: 57,  # 'w'
+    120: 58,  # 'x'
+    121: 11,  # 'y'
+    122: 22,  # 'z'
+    123: 255,  # '{'
+    124: 255,  # '|'
+    125: 255,  # '}'
+    126: 255,  # '~'
+    127: 255,  # '\x7f'
+    128: 180,  # '\x80'
+    129: 179,  # '\x81'
+    130: 178,  # '\x82'
+    131: 177,  # '\x83'
+    132: 176,  # '\x84'
+    133: 175,  # '\x85'
+    134: 174,  # '\x86'
+    135: 173,  # '\x87'
+    136: 172,  # '\x88'
+    137: 171,  # '\x89'
+    138: 170,  # '\x8a'
+    139: 169,  # '\x8b'
+    140: 168,  # '\x8c'
+    141: 167,  # '\x8d'
+    142: 166,  # '\x8e'
+    143: 165,  # '\x8f'
+    144: 164,  # '\x90'
+    145: 163,  # '\x91'
+    146: 162,  # '\x92'
+    147: 161,  # '\x93'
+    148: 160,  # '\x94'
+    149: 159,  # '\x95'
+    150: 101,  # '\x96'
+    151: 158,  # '\x97'
+    152: 157,  # '\x98'
+    153: 156,  # '\x99'
+    154: 155,  # '\x9a'
+    155: 154,  # '\x9b'
+    156: 153,  # '\x9c'
+    157: 152,  # '\x9d'
+    158: 151,  # '\x9e'
+    159: 106,  # '\x9f'
+    160: 150,  # '\xa0'
+    161: 149,  # '¡'
+    162: 148,  # '¢'
+    163: 147,  # '£'
+    164: 146,  # '¤'
+    165: 145,  # '¥'
+    166: 144,  # '¦'
+    167: 100,  # '§'
+    168: 143,  # '¨'
+    169: 142,  # '©'
+    170: 141,  # 'ª'
+    171: 140,  # '«'
+    172: 139,  # '¬'
+    173: 138,  # '\xad'
+    174: 137,  # '®'
+    175: 136,  # '¯'
+    176: 94,  # '°'
+    177: 80,  # '±'
+    178: 93,  # '²'
+    179: 135,  # '³'
+    180: 105,  # '´'
+    181: 134,  # 'µ'
+    182: 133,  # '¶'
+    183: 63,  # '·'
+    184: 132,  # '¸'
+    185: 131,  # '¹'
+    186: 130,  # 'º'
+    187: 129,  # '»'
+    188: 128,  # '¼'
+    189: 127,  # '½'
+    190: 126,  # '¾'
+    191: 125,  # '¿'
+    192: 124,  # 'À'
+    193: 104,  # 'Á'
+    194: 73,  # 'Â'
+    195: 99,  # 'Ã'
+    196: 79,  # 'Ä'
+    197: 85,  # 'Å'
+    198: 123,  # 'Æ'
+    199: 54,  # 'Ç'
+    200: 122,  # 'È'
+    201: 98,  # 'É'
+    202: 92,  # 'Ê'
+    203: 121,  # 'Ë'
+    204: 120,  # 'Ì'
+    205: 91,  # 'Í'
+    206: 103,  # 'Î'
+    207: 119,  # 'Ï'
+    208: 68,  # 'Ğ'
+    209: 118,  # 'Ñ'
+    210: 117,  # 'Ò'
+    211: 97,  # 'Ó'
+    212: 116,  # 'Ô'
+    213: 115,  # 'Õ'
+    214: 50,  # 'Ö'
+    215: 90,  # '×'
+    216: 114,  # 'Ø'
+    217: 113,  # 'Ù'
+    218: 112,  # 'Ú'
+    219: 111,  # 'Û'
+    220: 55,  # 'Ü'
+    221: 41,  # 'İ'
+    222: 40,  # 'Ş'
+    223: 86,  # 'ß'
+    224: 89,  # 'à'
+    225: 70,  # 'á'
+    226: 59,  # 'â'
+    227: 78,  # 'ã'
+    228: 71,  # 'ä'
+    229: 82,  # 'å'
+    230: 88,  # 'æ'
+    231: 33,  # 'ç'
+    232: 77,  # 'è'
+    233: 66,  # 'é'
+    234: 84,  # 'ê'
+    235: 83,  # 'ë'
+    236: 110,  # 'ì'
+    237: 75,  # 'í'
+    238: 61,  # 'î'
+    239: 96,  # 'ï'
+    240: 30,  # 'ğ'
+    241: 67,  # 'ñ'
+    242: 109,  # 'ò'
+    243: 74,  # 'ó'
+    244: 87,  # 'ô'
+    245: 102,  # 'õ'
+    246: 34,  # 'ö'
+    247: 95,  # '÷'
+    248: 81,  # 'ø'
+    249: 108,  # 'ù'
+    250: 76,  # 'ú'
+    251: 72,  # 'û'
+    252: 17,  # 'ü'
+    253: 6,  # 'ı'
+    254: 19,  # 'ş'
+    255: 107,  # 'ÿ'
+}
+
+ISO_8859_9_TURKISH_MODEL = SingleByteCharSetModel(
+    charset_name="ISO-8859-9",
+    language="Turkish",
+    char_to_order_map=ISO_8859_9_TURKISH_CHAR_TO_ORDER,
+    language_model=TURKISH_LANG_MODEL,
+    typical_positive_ratio=0.97029,
+    keep_ascii_letters=True,
+    alphabet="ABCDEFGHIJKLMNOPRSTUVYZabcdefghijklmnoprstuvyzÂÇÎÖÛÜâçîöûüĞğİıŞş",
+)
diff --git a/contrib/python/chardet/py3/chardet/latin1prober.py b/contrib/python/chardet/py3/chardet/latin1prober.py
new file mode 100644
index 00000000000..59a01d91b87
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/latin1prober.py
@@ -0,0 +1,147 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import List, Union
+
+from .charsetprober import CharSetProber
+from .enums import ProbingState
+
+FREQ_CAT_NUM = 4
+
+UDF = 0  # undefined
+OTH = 1  # other
+ASC = 2  # ascii capital letter
+ASS = 3  # ascii small letter
+ACV = 4  # accent capital vowel
+ACO = 5  # accent capital other
+ASV = 6  # accent small vowel
+ASO = 7  # accent small other
+CLASS_NUM = 8  # total classes
+
+# fmt: off
+Latin1_CharToClass = (
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 00 - 07
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 08 - 0F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 10 - 17
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 18 - 1F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 20 - 27
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 28 - 2F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 30 - 37
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 38 - 3F
+    OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 40 - 47
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 48 - 4F
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 50 - 57
+    ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,   # 58 - 5F
+    OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 60 - 67
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 68 - 6F
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 70 - 77
+    ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,   # 78 - 7F
+    OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH,   # 80 - 87
+    OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF,   # 88 - 8F
+    UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 90 - 97
+    OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO,   # 98 - 9F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A0 - A7
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A8 - AF
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B0 - B7
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B8 - BF
+    ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO,   # C0 - C7
+    ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,   # C8 - CF
+    ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH,   # D0 - D7
+    ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO,   # D8 - DF
+    ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO,   # E0 - E7
+    ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,   # E8 - EF
+    ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH,   # F0 - F7
+    ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO,   # F8 - FF
+)
+
+# 0 : illegal
+# 1 : very unlikely
+# 2 : normal
+# 3 : very likely
+Latin1ClassModel = (
+# UDF OTH ASC ASS ACV ACO ASV ASO
+    0,  0,  0,  0,  0,  0,  0,  0,  # UDF
+    0,  3,  3,  3,  3,  3,  3,  3,  # OTH
+    0,  3,  3,  3,  3,  3,  3,  3,  # ASC
+    0,  3,  3,  3,  1,  1,  3,  3,  # ASS
+    0,  3,  3,  3,  1,  2,  1,  2,  # ACV
+    0,  3,  3,  3,  3,  3,  3,  3,  # ACO
+    0,  3,  1,  3,  1,  1,  1,  3,  # ASV
+    0,  3,  1,  3,  1,  1,  3,  3,  # ASO
+)
+# fmt: on
+
+
+class Latin1Prober(CharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self._last_char_class = OTH
+        self._freq_counter: List[int] = []
+        self.reset()
+
+    def reset(self) -> None:
+        self._last_char_class = OTH
+        self._freq_counter = [0] * FREQ_CAT_NUM
+        super().reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "ISO-8859-1"
+
+    @property
+    def language(self) -> str:
+        return ""
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        byte_str = self.remove_xml_tags(byte_str)
+        for c in byte_str:
+            char_class = Latin1_CharToClass[c]
+            freq = Latin1ClassModel[(self._last_char_class * CLASS_NUM) + char_class]
+            if freq == 0:
+                self._state = ProbingState.NOT_ME
+                break
+            self._freq_counter[freq] += 1
+            self._last_char_class = char_class
+
+        return self.state
+
+    def get_confidence(self) -> float:
+        if self.state == ProbingState.NOT_ME:
+            return 0.01
+
+        total = sum(self._freq_counter)
+        confidence = (
+            0.0
+            if total < 0.01
+            else (self._freq_counter[3] - self._freq_counter[1] * 20.0) / total
+        )
+        confidence = max(confidence, 0.0)
+        # lower the confidence of latin1 so that other more accurate
+        # detector can take priority.
+        confidence *= 0.73
+        return confidence
diff --git a/contrib/python/chardet/py3/chardet/macromanprober.py b/contrib/python/chardet/py3/chardet/macromanprober.py
new file mode 100644
index 00000000000..1425d10ecaa
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/macromanprober.py
@@ -0,0 +1,162 @@
+######################## BEGIN LICENSE BLOCK ########################
+# This code was modified from latin1prober.py by Rob Speer <rob@lumino.so>.
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Rob Speer - adapt to MacRoman encoding
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import List, Union
+
+from .charsetprober import CharSetProber
+from .enums import ProbingState
+
+FREQ_CAT_NUM = 4
+
+UDF = 0  # undefined
+OTH = 1  # other
+ASC = 2  # ascii capital letter
+ASS = 3  # ascii small letter
+ACV = 4  # accent capital vowel
+ACO = 5  # accent capital other
+ASV = 6  # accent small vowel
+ASO = 7  # accent small other
+ODD = 8  # character that is unlikely to appear
+CLASS_NUM = 9  # total classes
+
+# The change from Latin1 is that we explicitly look for extended characters
+# that are infrequently-occurring symbols, and consider them to always be
+# improbable. This should let MacRoman get out of the way of more likely
+# encodings in most situations.
+
+# fmt: off
+MacRoman_CharToClass = (
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 00 - 07
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 08 - 0F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 10 - 17
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 18 - 1F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 20 - 27
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 28 - 2F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 30 - 37
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 38 - 3F
+    OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,  # 40 - 47
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,  # 48 - 4F
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,  # 50 - 57
+    ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,  # 58 - 5F
+    OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,  # 60 - 67
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,  # 68 - 6F
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,  # 70 - 77
+    ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,  # 78 - 7F
+    ACV, ACV, ACO, ACV, ACO, ACV, ACV, ASV,  # 80 - 87
+    ASV, ASV, ASV, ASV, ASV, ASO, ASV, ASV,  # 88 - 8F
+    ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASV,  # 90 - 97
+    ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,  # 98 - 9F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, ASO,  # A0 - A7
+    OTH, OTH, ODD, ODD, OTH, OTH, ACV, ACV,  # A8 - AF
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # B0 - B7
+    OTH, OTH, OTH, OTH, OTH, OTH, ASV, ASV,  # B8 - BF
+    OTH, OTH, ODD, OTH, ODD, OTH, OTH, OTH,  # C0 - C7
+    OTH, OTH, OTH, ACV, ACV, ACV, ACV, ASV,  # C8 - CF
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, ODD,  # D0 - D7
+    ASV, ACV, ODD, OTH, OTH, OTH, OTH, OTH,  # D8 - DF
+    OTH, OTH, OTH, OTH, OTH, ACV, ACV, ACV,  # E0 - E7
+    ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,  # E8 - EF
+    ODD, ACV, ACV, ACV, ACV, ASV, ODD, ODD,  # F0 - F7
+    ODD, ODD, ODD, ODD, ODD, ODD, ODD, ODD,  # F8 - FF
+)
+
+# 0 : illegal
+# 1 : very unlikely
+# 2 : normal
+# 3 : very likely
+MacRomanClassModel = (
+# UDF OTH ASC ASS ACV ACO ASV ASO ODD
+    0,  0,  0,  0,  0,  0,  0,  0,  0,  # UDF
+    0,  3,  3,  3,  3,  3,  3,  3,  1,  # OTH
+    0,  3,  3,  3,  3,  3,  3,  3,  1,  # ASC
+    0,  3,  3,  3,  1,  1,  3,  3,  1,  # ASS
+    0,  3,  3,  3,  1,  2,  1,  2,  1,  # ACV
+    0,  3,  3,  3,  3,  3,  3,  3,  1,  # ACO
+    0,  3,  1,  3,  1,  1,  1,  3,  1,  # ASV
+    0,  3,  1,  3,  1,  1,  3,  3,  1,  # ASO
+    0,  1,  1,  1,  1,  1,  1,  1,  1,  # ODD
+)
+# fmt: on
+
+
+class MacRomanProber(CharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self._last_char_class = OTH
+        self._freq_counter: List[int] = []
+        self.reset()
+
+    def reset(self) -> None:
+        self._last_char_class = OTH
+        self._freq_counter = [0] * FREQ_CAT_NUM
+
+        # express the prior that MacRoman is a somewhat rare encoding;
+        # this can be done by starting out in a slightly improbable state
+        # that must be overcome
+        self._freq_counter[2] = 10
+
+        super().reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "MacRoman"
+
+    @property
+    def language(self) -> str:
+        return ""
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        byte_str = self.remove_xml_tags(byte_str)
+        for c in byte_str:
+            char_class = MacRoman_CharToClass[c]
+            freq = MacRomanClassModel[(self._last_char_class * CLASS_NUM) + char_class]
+            if freq == 0:
+                self._state = ProbingState.NOT_ME
+                break
+            self._freq_counter[freq] += 1
+            self._last_char_class = char_class
+
+        return self.state
+
+    def get_confidence(self) -> float:
+        if self.state == ProbingState.NOT_ME:
+            return 0.01
+
+        total = sum(self._freq_counter)
+        confidence = (
+            0.0
+            if total < 0.01
+            else (self._freq_counter[3] - self._freq_counter[1] * 20.0) / total
+        )
+        confidence = max(confidence, 0.0)
+        # lower the confidence of MacRoman so that other more accurate
+        # detector can take priority.
+        confidence *= 0.73
+        return confidence
diff --git a/contrib/python/chardet/py3/chardet/mbcharsetprober.py b/contrib/python/chardet/py3/chardet/mbcharsetprober.py
new file mode 100644
index 00000000000..666307e8fe0
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/mbcharsetprober.py
@@ -0,0 +1,95 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#   Proofpoint, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import Optional, Union
+
+from .chardistribution import CharDistributionAnalysis
+from .charsetprober import CharSetProber
+from .codingstatemachine import CodingStateMachine
+from .enums import LanguageFilter, MachineState, ProbingState
+
+
+class MultiByteCharSetProber(CharSetProber):
+    """
+    MultiByteCharSetProber
+    """
+
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        super().__init__(lang_filter=lang_filter)
+        self.distribution_analyzer: Optional[CharDistributionAnalysis] = None
+        self.coding_sm: Optional[CodingStateMachine] = None
+        self._last_char = bytearray(b"\0\0")
+
+    def reset(self) -> None:
+        super().reset()
+        if self.coding_sm:
+            self.coding_sm.reset()
+        if self.distribution_analyzer:
+            self.distribution_analyzer.reset()
+        self._last_char = bytearray(b"\0\0")
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        assert self.coding_sm is not None
+        assert self.distribution_analyzer is not None
+
+        for i, byte in enumerate(byte_str):
+            coding_state = self.coding_sm.next_state(byte)
+            if coding_state == MachineState.ERROR:
+                self.logger.debug(
+                    "%s %s prober hit error at byte %s",
+                    self.charset_name,
+                    self.language,
+                    i,
+                )
+                self._state = ProbingState.NOT_ME
+                break
+            if coding_state == MachineState.ITS_ME:
+                self._state = ProbingState.FOUND_IT
+                break
+            if coding_state == MachineState.START:
+                char_len = self.coding_sm.get_current_charlen()
+                if i == 0:
+                    self._last_char[1] = byte
+                    self.distribution_analyzer.feed(self._last_char, char_len)
+                else:
+                    self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
+
+        self._last_char[0] = byte_str[-1]
+
+        if self.state == ProbingState.DETECTING:
+            if self.distribution_analyzer.got_enough_data() and (
+                self.get_confidence() > self.SHORTCUT_THRESHOLD
+            ):
+                self._state = ProbingState.FOUND_IT
+
+        return self.state
+
+    def get_confidence(self) -> float:
+        assert self.distribution_analyzer is not None
+        return self.distribution_analyzer.get_confidence()
diff --git a/contrib/python/chardet/py3/chardet/mbcsgroupprober.py b/contrib/python/chardet/py3/chardet/mbcsgroupprober.py
new file mode 100644
index 00000000000..6cb9cc7b3bc
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/mbcsgroupprober.py
@@ -0,0 +1,57 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#   Proofpoint, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .big5prober import Big5Prober
+from .charsetgroupprober import CharSetGroupProber
+from .cp949prober import CP949Prober
+from .enums import LanguageFilter
+from .eucjpprober import EUCJPProber
+from .euckrprober import EUCKRProber
+from .euctwprober import EUCTWProber
+from .gb2312prober import GB2312Prober
+from .johabprober import JOHABProber
+from .sjisprober import SJISProber
+from .utf8prober import UTF8Prober
+
+
+class MBCSGroupProber(CharSetGroupProber):
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        super().__init__(lang_filter=lang_filter)
+        self.probers = [
+            UTF8Prober(),
+            SJISProber(),
+            EUCJPProber(),
+            GB2312Prober(),
+            EUCKRProber(),
+            CP949Prober(),
+            Big5Prober(),
+            EUCTWProber(),
+            JOHABProber(),
+        ]
+        self.reset()
diff --git a/contrib/python/chardet/py3/chardet/mbcssm.py b/contrib/python/chardet/py3/chardet/mbcssm.py
new file mode 100644
index 00000000000..7bbe97e6665
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/mbcssm.py
@@ -0,0 +1,661 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .codingstatemachinedict import CodingStateMachineDict
+from .enums import MachineState
+
+# BIG5
+
+# fmt: off
+BIG5_CLS = (
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07    #allow 0x00 as legal value
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 37
+    1, 1, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 40 - 47
+    2, 2, 2, 2, 2, 2, 2, 2,  # 48 - 4f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 50 - 57
+    2, 2, 2, 2, 2, 2, 2, 2,  # 58 - 5f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 60 - 67
+    2, 2, 2, 2, 2, 2, 2, 2,  # 68 - 6f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 70 - 77
+    2, 2, 2, 2, 2, 2, 2, 1,  # 78 - 7f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 80 - 87
+    4, 4, 4, 4, 4, 4, 4, 4,  # 88 - 8f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 90 - 97
+    4, 4, 4, 4, 4, 4, 4, 4,  # 98 - 9f
+    4, 3, 3, 3, 3, 3, 3, 3,  # a0 - a7
+    3, 3, 3, 3, 3, 3, 3, 3,  # a8 - af
+    3, 3, 3, 3, 3, 3, 3, 3,  # b0 - b7
+    3, 3, 3, 3, 3, 3, 3, 3,  # b8 - bf
+    3, 3, 3, 3, 3, 3, 3, 3,  # c0 - c7
+    3, 3, 3, 3, 3, 3, 3, 3,  # c8 - cf
+    3, 3, 3, 3, 3, 3, 3, 3,  # d0 - d7
+    3, 3, 3, 3, 3, 3, 3, 3,  # d8 - df
+    3, 3, 3, 3, 3, 3, 3, 3,  # e0 - e7
+    3, 3, 3, 3, 3, 3, 3, 3,  # e8 - ef
+    3, 3, 3, 3, 3, 3, 3, 3,  # f0 - f7
+    3, 3, 3, 3, 3, 3, 3, 0  # f8 - ff
+)
+
+BIG5_ST = (
+    MachineState.ERROR,MachineState.START,MachineState.START,     3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07
+    MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,#08-0f
+    MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START#10-17
+)
+# fmt: on
+
+BIG5_CHAR_LEN_TABLE = (0, 1, 1, 2, 0)
+
+BIG5_SM_MODEL: CodingStateMachineDict = {
+    "class_table": BIG5_CLS,
+    "class_factor": 5,
+    "state_table": BIG5_ST,
+    "char_len_table": BIG5_CHAR_LEN_TABLE,
+    "name": "Big5",
+}
+
+# CP949
+# fmt: off
+CP949_CLS  = (
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,  # 00 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1,  # 10 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 3f
+    1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,  # 40 - 4f
+    4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1,  # 50 - 5f
+    1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,  # 60 - 6f
+    5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1,  # 70 - 7f
+    0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,  # 80 - 8f
+    6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,  # 90 - 9f
+    6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8,  # a0 - af
+    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,  # b0 - bf
+    7, 7, 7, 7, 7, 7, 9, 2, 2, 3, 2, 2, 2, 2, 2, 2,  # c0 - cf
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  # d0 - df
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  # e0 - ef
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0,  # f0 - ff
+)
+
+CP949_ST = (
+#cls=    0      1      2      3      4      5      6      7      8      9  # previous state =
+    MachineState.ERROR,MachineState.START,     3,MachineState.ERROR,MachineState.START,MachineState.START,     4,     5,MachineState.ERROR,     6, # MachineState.START
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, # MachineState.ERROR
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME, # MachineState.ITS_ME
+    MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START, # 3
+    MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START, # 4
+    MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START, # 5
+    MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START, # 6
+)
+# fmt: on
+
+CP949_CHAR_LEN_TABLE = (0, 1, 2, 0, 1, 1, 2, 2, 0, 2)
+
+CP949_SM_MODEL: CodingStateMachineDict = {
+    "class_table": CP949_CLS,
+    "class_factor": 10,
+    "state_table": CP949_ST,
+    "char_len_table": CP949_CHAR_LEN_TABLE,
+    "name": "CP949",
+}
+
+# EUC-JP
+# fmt: off
+EUCJP_CLS = (
+    4, 4, 4, 4, 4, 4, 4, 4,  # 00 - 07
+    4, 4, 4, 4, 4, 4, 5, 5,  # 08 - 0f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 10 - 17
+    4, 4, 4, 5, 4, 4, 4, 4,  # 18 - 1f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 20 - 27
+    4, 4, 4, 4, 4, 4, 4, 4,  # 28 - 2f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 30 - 37
+    4, 4, 4, 4, 4, 4, 4, 4,  # 38 - 3f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 40 - 47
+    4, 4, 4, 4, 4, 4, 4, 4,  # 48 - 4f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 50 - 57
+    4, 4, 4, 4, 4, 4, 4, 4,  # 58 - 5f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 60 - 67
+    4, 4, 4, 4, 4, 4, 4, 4,  # 68 - 6f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 70 - 77
+    4, 4, 4, 4, 4, 4, 4, 4,  # 78 - 7f
+    5, 5, 5, 5, 5, 5, 5, 5,  # 80 - 87
+    5, 5, 5, 5, 5, 5, 1, 3,  # 88 - 8f
+    5, 5, 5, 5, 5, 5, 5, 5,  # 90 - 97
+    5, 5, 5, 5, 5, 5, 5, 5,  # 98 - 9f
+    5, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    0, 0, 0, 0, 0, 0, 0, 0,  # e0 - e7
+    0, 0, 0, 0, 0, 0, 0, 0,  # e8 - ef
+    0, 0, 0, 0, 0, 0, 0, 0,  # f0 - f7
+    0, 0, 0, 0, 0, 0, 0, 5  # f8 - ff
+)
+
+EUCJP_ST = (
+          3,     4,     3,     5,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07
+     MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f
+     MachineState.ITS_ME,MachineState.ITS_ME,MachineState.START,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#10-17
+     MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     3,MachineState.ERROR,#18-1f
+          3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START#20-27
+)
+# fmt: on
+
+EUCJP_CHAR_LEN_TABLE = (2, 2, 2, 3, 1, 0)
+
+EUCJP_SM_MODEL: CodingStateMachineDict = {
+    "class_table": EUCJP_CLS,
+    "class_factor": 6,
+    "state_table": EUCJP_ST,
+    "char_len_table": EUCJP_CHAR_LEN_TABLE,
+    "name": "EUC-JP",
+}
+
+# EUC-KR
+# fmt: off
+EUCKR_CLS  = (
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 37
+    1, 1, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 40 - 47
+    1, 1, 1, 1, 1, 1, 1, 1,  # 48 - 4f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 50 - 57
+    1, 1, 1, 1, 1, 1, 1, 1,  # 58 - 5f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 60 - 67
+    1, 1, 1, 1, 1, 1, 1, 1,  # 68 - 6f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 70 - 77
+    1, 1, 1, 1, 1, 1, 1, 1,  # 78 - 7f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 80 - 87
+    0, 0, 0, 0, 0, 0, 0, 0,  # 88 - 8f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 90 - 97
+    0, 0, 0, 0, 0, 0, 0, 0,  # 98 - 9f
+    0, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 3, 3, 3,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 3, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    2, 2, 2, 2, 2, 2, 2, 2,  # e0 - e7
+    2, 2, 2, 2, 2, 2, 2, 2,  # e8 - ef
+    2, 2, 2, 2, 2, 2, 2, 2,  # f0 - f7
+    2, 2, 2, 2, 2, 2, 2, 0   # f8 - ff
+)
+
+EUCKR_ST = (
+    MachineState.ERROR,MachineState.START,     3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START #08-0f
+)
+# fmt: on
+
+EUCKR_CHAR_LEN_TABLE = (0, 1, 2, 0)
+
+EUCKR_SM_MODEL: CodingStateMachineDict = {
+    "class_table": EUCKR_CLS,
+    "class_factor": 4,
+    "state_table": EUCKR_ST,
+    "char_len_table": EUCKR_CHAR_LEN_TABLE,
+    "name": "EUC-KR",
+}
+
+# JOHAB
+# fmt: off
+JOHAB_CLS = (
+    4,4,4,4,4,4,4,4,  # 00 - 07
+    4,4,4,4,4,4,0,0,  # 08 - 0f
+    4,4,4,4,4,4,4,4,  # 10 - 17
+    4,4,4,0,4,4,4,4,  # 18 - 1f
+    4,4,4,4,4,4,4,4,  # 20 - 27
+    4,4,4,4,4,4,4,4,  # 28 - 2f
+    4,3,3,3,3,3,3,3,  # 30 - 37
+    3,3,3,3,3,3,3,3,  # 38 - 3f
+    3,1,1,1,1,1,1,1,  # 40 - 47
+    1,1,1,1,1,1,1,1,  # 48 - 4f
+    1,1,1,1,1,1,1,1,  # 50 - 57
+    1,1,1,1,1,1,1,1,  # 58 - 5f
+    1,1,1,1,1,1,1,1,  # 60 - 67
+    1,1,1,1,1,1,1,1,  # 68 - 6f
+    1,1,1,1,1,1,1,1,  # 70 - 77
+    1,1,1,1,1,1,1,2,  # 78 - 7f
+    6,6,6,6,8,8,8,8,  # 80 - 87
+    8,8,8,8,8,8,8,8,  # 88 - 8f
+    8,7,7,7,7,7,7,7,  # 90 - 97
+    7,7,7,7,7,7,7,7,  # 98 - 9f
+    7,7,7,7,7,7,7,7,  # a0 - a7
+    7,7,7,7,7,7,7,7,  # a8 - af
+    7,7,7,7,7,7,7,7,  # b0 - b7
+    7,7,7,7,7,7,7,7,  # b8 - bf
+    7,7,7,7,7,7,7,7,  # c0 - c7
+    7,7,7,7,7,7,7,7,  # c8 - cf
+    7,7,7,7,5,5,5,5,  # d0 - d7
+    5,9,9,9,9,9,9,5,  # d8 - df
+    9,9,9,9,9,9,9,9,  # e0 - e7
+    9,9,9,9,9,9,9,9,  # e8 - ef
+    9,9,9,9,9,9,9,9,  # f0 - f7
+    9,9,5,5,5,5,5,0   # f8 - ff
+)
+
+JOHAB_ST = (
+# cls = 0                   1                   2                   3                   4                   5                   6                   7                   8                   9
+    MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.ERROR ,MachineState.ERROR ,3                  ,3                  ,4                  ,  # MachineState.START
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,  # MachineState.ITS_ME
+    MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,  # MachineState.ERROR
+    MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.ERROR ,MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START ,  # 3
+    MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,  # 4
+)
+# fmt: on
+
+JOHAB_CHAR_LEN_TABLE = (0, 1, 1, 1, 1, 0, 0, 2, 2, 2)
+
+JOHAB_SM_MODEL: CodingStateMachineDict = {
+    "class_table": JOHAB_CLS,
+    "class_factor": 10,
+    "state_table": JOHAB_ST,
+    "char_len_table": JOHAB_CHAR_LEN_TABLE,
+    "name": "Johab",
+}
+
+# EUC-TW
+# fmt: off
+EUCTW_CLS = (
+    2, 2, 2, 2, 2, 2, 2, 2,  # 00 - 07
+    2, 2, 2, 2, 2, 2, 0, 0,  # 08 - 0f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 10 - 17
+    2, 2, 2, 0, 2, 2, 2, 2,  # 18 - 1f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 20 - 27
+    2, 2, 2, 2, 2, 2, 2, 2,  # 28 - 2f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 30 - 37
+    2, 2, 2, 2, 2, 2, 2, 2,  # 38 - 3f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 40 - 47
+    2, 2, 2, 2, 2, 2, 2, 2,  # 48 - 4f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 50 - 57
+    2, 2, 2, 2, 2, 2, 2, 2,  # 58 - 5f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 60 - 67
+    2, 2, 2, 2, 2, 2, 2, 2,  # 68 - 6f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 70 - 77
+    2, 2, 2, 2, 2, 2, 2, 2,  # 78 - 7f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 80 - 87
+    0, 0, 0, 0, 0, 0, 6, 0,  # 88 - 8f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 90 - 97
+    0, 0, 0, 0, 0, 0, 0, 0,  # 98 - 9f
+    0, 3, 4, 4, 4, 4, 4, 4,  # a0 - a7
+    5, 5, 1, 1, 1, 1, 1, 1,  # a8 - af
+    1, 1, 1, 1, 1, 1, 1, 1,  # b0 - b7
+    1, 1, 1, 1, 1, 1, 1, 1,  # b8 - bf
+    1, 1, 3, 1, 3, 3, 3, 3,  # c0 - c7
+    3, 3, 3, 3, 3, 3, 3, 3,  # c8 - cf
+    3, 3, 3, 3, 3, 3, 3, 3,  # d0 - d7
+    3, 3, 3, 3, 3, 3, 3, 3,  # d8 - df
+    3, 3, 3, 3, 3, 3, 3, 3,  # e0 - e7
+    3, 3, 3, 3, 3, 3, 3, 3,  # e8 - ef
+    3, 3, 3, 3, 3, 3, 3, 3,  # f0 - f7
+    3, 3, 3, 3, 3, 3, 3, 0   # f8 - ff
+)
+
+EUCTW_ST = (
+    MachineState.ERROR,MachineState.ERROR,MachineState.START,     3,     3,     3,     4,MachineState.ERROR,#00-07
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.START,MachineState.ERROR,#10-17
+    MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#18-1f
+         5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.START,MachineState.START,#20-27
+    MachineState.START,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START #28-2f
+)
+# fmt: on
+
+EUCTW_CHAR_LEN_TABLE = (0, 0, 1, 2, 2, 2, 3)
+
+EUCTW_SM_MODEL: CodingStateMachineDict = {
+    "class_table": EUCTW_CLS,
+    "class_factor": 7,
+    "state_table": EUCTW_ST,
+    "char_len_table": EUCTW_CHAR_LEN_TABLE,
+    "name": "x-euc-tw",
+}
+
+# GB2312
+# fmt: off
+GB2312_CLS = (
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    3, 3, 3, 3, 3, 3, 3, 3,  # 30 - 37
+    3, 3, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 40 - 47
+    2, 2, 2, 2, 2, 2, 2, 2,  # 48 - 4f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 50 - 57
+    2, 2, 2, 2, 2, 2, 2, 2,  # 58 - 5f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 60 - 67
+    2, 2, 2, 2, 2, 2, 2, 2,  # 68 - 6f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 70 - 77
+    2, 2, 2, 2, 2, 2, 2, 4,  # 78 - 7f
+    5, 6, 6, 6, 6, 6, 6, 6,  # 80 - 87
+    6, 6, 6, 6, 6, 6, 6, 6,  # 88 - 8f
+    6, 6, 6, 6, 6, 6, 6, 6,  # 90 - 97
+    6, 6, 6, 6, 6, 6, 6, 6,  # 98 - 9f
+    6, 6, 6, 6, 6, 6, 6, 6,  # a0 - a7
+    6, 6, 6, 6, 6, 6, 6, 6,  # a8 - af
+    6, 6, 6, 6, 6, 6, 6, 6,  # b0 - b7
+    6, 6, 6, 6, 6, 6, 6, 6,  # b8 - bf
+    6, 6, 6, 6, 6, 6, 6, 6,  # c0 - c7
+    6, 6, 6, 6, 6, 6, 6, 6,  # c8 - cf
+    6, 6, 6, 6, 6, 6, 6, 6,  # d0 - d7
+    6, 6, 6, 6, 6, 6, 6, 6,  # d8 - df
+    6, 6, 6, 6, 6, 6, 6, 6,  # e0 - e7
+    6, 6, 6, 6, 6, 6, 6, 6,  # e8 - ef
+    6, 6, 6, 6, 6, 6, 6, 6,  # f0 - f7
+    6, 6, 6, 6, 6, 6, 6, 0   # f8 - ff
+)
+
+GB2312_ST = (
+    MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,     3,MachineState.ERROR,#00-07
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,#10-17
+         4,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#18-1f
+    MachineState.ERROR,MachineState.ERROR,     5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,#20-27
+    MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START #28-2f
+)
+# fmt: on
+
+# To be accurate, the length of class 6 can be either 2 or 4.
+# But it is not necessary to discriminate between the two since
+# it is used for frequency analysis only, and we are validating
+# each code range there as well. So it is safe to set it to be
+# 2 here.
+GB2312_CHAR_LEN_TABLE = (0, 1, 1, 1, 1, 1, 2)
+
+GB2312_SM_MODEL: CodingStateMachineDict = {
+    "class_table": GB2312_CLS,
+    "class_factor": 7,
+    "state_table": GB2312_ST,
+    "char_len_table": GB2312_CHAR_LEN_TABLE,
+    "name": "GB2312",
+}
+
+# Shift_JIS
+# fmt: off
+SJIS_CLS = (
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 37
+    1, 1, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 40 - 47
+    2, 2, 2, 2, 2, 2, 2, 2,  # 48 - 4f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 50 - 57
+    2, 2, 2, 2, 2, 2, 2, 2,  # 58 - 5f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 60 - 67
+    2, 2, 2, 2, 2, 2, 2, 2,  # 68 - 6f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 70 - 77
+    2, 2, 2, 2, 2, 2, 2, 1,  # 78 - 7f
+    3, 3, 3, 3, 3, 2, 2, 3,  # 80 - 87
+    3, 3, 3, 3, 3, 3, 3, 3,  # 88 - 8f
+    3, 3, 3, 3, 3, 3, 3, 3,  # 90 - 97
+    3, 3, 3, 3, 3, 3, 3, 3,  # 98 - 9f
+    #0xa0 is illegal in sjis encoding, but some pages does
+    #contain such byte. We need to be more error forgiven.
+    2, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    3, 3, 3, 3, 3, 3, 3, 3,  # e0 - e7
+    3, 3, 3, 3, 3, 4, 4, 4,  # e8 - ef
+    3, 3, 3, 3, 3, 3, 3, 3,  # f0 - f7
+    3, 3, 3, 3, 3, 0, 0, 0,  # f8 - ff
+)
+
+SJIS_ST = (
+    MachineState.ERROR,MachineState.START,MachineState.START,     3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START #10-17
+)
+# fmt: on
+
+SJIS_CHAR_LEN_TABLE = (0, 1, 1, 2, 0, 0)
+
+SJIS_SM_MODEL: CodingStateMachineDict = {
+    "class_table": SJIS_CLS,
+    "class_factor": 6,
+    "state_table": SJIS_ST,
+    "char_len_table": SJIS_CHAR_LEN_TABLE,
+    "name": "Shift_JIS",
+}
+
+# UCS2-BE
+# fmt: off
+UCS2BE_CLS = (
+    0, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 1, 0, 0, 2, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 3, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 20 - 27
+    0, 3, 3, 3, 3, 3, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 80 - 87
+    0, 0, 0, 0, 0, 0, 0, 0,  # 88 - 8f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 90 - 97
+    0, 0, 0, 0, 0, 0, 0, 0,  # 98 - 9f
+    0, 0, 0, 0, 0, 0, 0, 0,  # a0 - a7
+    0, 0, 0, 0, 0, 0, 0, 0,  # a8 - af
+    0, 0, 0, 0, 0, 0, 0, 0,  # b0 - b7
+    0, 0, 0, 0, 0, 0, 0, 0,  # b8 - bf
+    0, 0, 0, 0, 0, 0, 0, 0,  # c0 - c7
+    0, 0, 0, 0, 0, 0, 0, 0,  # c8 - cf
+    0, 0, 0, 0, 0, 0, 0, 0,  # d0 - d7
+    0, 0, 0, 0, 0, 0, 0, 0,  # d8 - df
+    0, 0, 0, 0, 0, 0, 0, 0,  # e0 - e7
+    0, 0, 0, 0, 0, 0, 0, 0,  # e8 - ef
+    0, 0, 0, 0, 0, 0, 0, 0,  # f0 - f7
+    0, 0, 0, 0, 0, 0, 4, 5   # f8 - ff
+)
+
+UCS2BE_ST  = (
+          5,     7,     7,MachineState.ERROR,     4,     3,MachineState.ERROR,MachineState.ERROR,#00-07
+     MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f
+     MachineState.ITS_ME,MachineState.ITS_ME,     6,     6,     6,     6,MachineState.ERROR,MachineState.ERROR,#10-17
+          6,     6,     6,     6,     6,MachineState.ITS_ME,     6,     6,#18-1f
+          6,     6,     6,     6,     5,     7,     7,MachineState.ERROR,#20-27
+          5,     8,     6,     6,MachineState.ERROR,     6,     6,     6,#28-2f
+          6,     6,     6,     6,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START #30-37
+)
+# fmt: on
+
+UCS2BE_CHAR_LEN_TABLE = (2, 2, 2, 0, 2, 2)
+
+UCS2BE_SM_MODEL: CodingStateMachineDict = {
+    "class_table": UCS2BE_CLS,
+    "class_factor": 6,
+    "state_table": UCS2BE_ST,
+    "char_len_table": UCS2BE_CHAR_LEN_TABLE,
+    "name": "UTF-16BE",
+}
+
+# UCS2-LE
+# fmt: off
+UCS2LE_CLS = (
+    0, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 1, 0, 0, 2, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 3, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 20 - 27
+    0, 3, 3, 3, 3, 3, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 80 - 87
+    0, 0, 0, 0, 0, 0, 0, 0,  # 88 - 8f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 90 - 97
+    0, 0, 0, 0, 0, 0, 0, 0,  # 98 - 9f
+    0, 0, 0, 0, 0, 0, 0, 0,  # a0 - a7
+    0, 0, 0, 0, 0, 0, 0, 0,  # a8 - af
+    0, 0, 0, 0, 0, 0, 0, 0,  # b0 - b7
+    0, 0, 0, 0, 0, 0, 0, 0,  # b8 - bf
+    0, 0, 0, 0, 0, 0, 0, 0,  # c0 - c7
+    0, 0, 0, 0, 0, 0, 0, 0,  # c8 - cf
+    0, 0, 0, 0, 0, 0, 0, 0,  # d0 - d7
+    0, 0, 0, 0, 0, 0, 0, 0,  # d8 - df
+    0, 0, 0, 0, 0, 0, 0, 0,  # e0 - e7
+    0, 0, 0, 0, 0, 0, 0, 0,  # e8 - ef
+    0, 0, 0, 0, 0, 0, 0, 0,  # f0 - f7
+    0, 0, 0, 0, 0, 0, 4, 5   # f8 - ff
+)
+
+UCS2LE_ST = (
+          6,     6,     7,     6,     4,     3,MachineState.ERROR,MachineState.ERROR,#00-07
+     MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f
+     MachineState.ITS_ME,MachineState.ITS_ME,     5,     5,     5,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,#10-17
+          5,     5,     5,MachineState.ERROR,     5,MachineState.ERROR,     6,     6,#18-1f
+          7,     6,     8,     8,     5,     5,     5,MachineState.ERROR,#20-27
+          5,     5,     5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     5,     5,#28-2f
+          5,     5,     5,MachineState.ERROR,     5,MachineState.ERROR,MachineState.START,MachineState.START #30-37
+)
+# fmt: on
+
+UCS2LE_CHAR_LEN_TABLE = (2, 2, 2, 2, 2, 2)
+
+UCS2LE_SM_MODEL: CodingStateMachineDict = {
+    "class_table": UCS2LE_CLS,
+    "class_factor": 6,
+    "state_table": UCS2LE_ST,
+    "char_len_table": UCS2LE_CHAR_LEN_TABLE,
+    "name": "UTF-16LE",
+}
+
+# UTF-8
+# fmt: off
+UTF8_CLS = (
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07  #allow 0x00 as a legal value
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 37
+    1, 1, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 40 - 47
+    1, 1, 1, 1, 1, 1, 1, 1,  # 48 - 4f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 50 - 57
+    1, 1, 1, 1, 1, 1, 1, 1,  # 58 - 5f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 60 - 67
+    1, 1, 1, 1, 1, 1, 1, 1,  # 68 - 6f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 70 - 77
+    1, 1, 1, 1, 1, 1, 1, 1,  # 78 - 7f
+    2, 2, 2, 2, 3, 3, 3, 3,  # 80 - 87
+    4, 4, 4, 4, 4, 4, 4, 4,  # 88 - 8f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 90 - 97
+    4, 4, 4, 4, 4, 4, 4, 4,  # 98 - 9f
+    5, 5, 5, 5, 5, 5, 5, 5,  # a0 - a7
+    5, 5, 5, 5, 5, 5, 5, 5,  # a8 - af
+    5, 5, 5, 5, 5, 5, 5, 5,  # b0 - b7
+    5, 5, 5, 5, 5, 5, 5, 5,  # b8 - bf
+    0, 0, 6, 6, 6, 6, 6, 6,  # c0 - c7
+    6, 6, 6, 6, 6, 6, 6, 6,  # c8 - cf
+    6, 6, 6, 6, 6, 6, 6, 6,  # d0 - d7
+    6, 6, 6, 6, 6, 6, 6, 6,  # d8 - df
+    7, 8, 8, 8, 8, 8, 8, 8,  # e0 - e7
+    8, 8, 8, 8, 8, 9, 8, 8,  # e8 - ef
+    10, 11, 11, 11, 11, 11, 11, 11,  # f0 - f7
+    12, 13, 13, 13, 14, 15, 0, 0    # f8 - ff
+)
+
+UTF8_ST = (
+    MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     12,   10,#00-07
+         9,     11,     8,     7,     6,     5,     4,    3,#08-0f
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#10-17
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#18-1f
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#20-27
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#28-2f
+    MachineState.ERROR,MachineState.ERROR,     5,     5,     5,     5,MachineState.ERROR,MachineState.ERROR,#30-37
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#38-3f
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     5,     5,     5,MachineState.ERROR,MachineState.ERROR,#40-47
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#48-4f
+    MachineState.ERROR,MachineState.ERROR,     7,     7,     7,     7,MachineState.ERROR,MachineState.ERROR,#50-57
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#58-5f
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     7,     7,MachineState.ERROR,MachineState.ERROR,#60-67
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#68-6f
+    MachineState.ERROR,MachineState.ERROR,     9,     9,     9,     9,MachineState.ERROR,MachineState.ERROR,#70-77
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#78-7f
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     9,MachineState.ERROR,MachineState.ERROR,#80-87
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#88-8f
+    MachineState.ERROR,MachineState.ERROR,    12,    12,    12,    12,MachineState.ERROR,MachineState.ERROR,#90-97
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#98-9f
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,    12,MachineState.ERROR,MachineState.ERROR,#a0-a7
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#a8-af
+    MachineState.ERROR,MachineState.ERROR,    12,    12,    12,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#b0-b7
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#b8-bf
+    MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,#c0-c7
+    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR #c8-cf
+)
+# fmt: on
+
+UTF8_CHAR_LEN_TABLE = (0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6)
+
+UTF8_SM_MODEL: CodingStateMachineDict = {
+    "class_table": UTF8_CLS,
+    "class_factor": 16,
+    "state_table": UTF8_ST,
+    "char_len_table": UTF8_CHAR_LEN_TABLE,
+    "name": "UTF-8",
+}
diff --git a/contrib/python/chardet/py3/chardet/metadata/__init__.py b/contrib/python/chardet/py3/chardet/metadata/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/chardet/py3/chardet/metadata/languages.py b/contrib/python/chardet/py3/chardet/metadata/languages.py
new file mode 100644
index 00000000000..eb40c5f0c85
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/metadata/languages.py
@@ -0,0 +1,352 @@
+"""
+Metadata about languages used by our model training code for our
+SingleByteCharSetProbers.  Could be used for other things in the future.
+
+This code is based on the language metadata from the uchardet project.
+"""
+
+from string import ascii_letters
+from typing import List, Optional
+
+# TODO: Add Ukrainian (KOI8-U)
+
+
+class Language:
+    """Metadata about a language useful for training models
+
+    :ivar name: The human name for the language, in English.
+    :type name: str
+    :ivar iso_code: 2-letter ISO 639-1 if possible, 3-letter ISO code otherwise,
+                    or use another catalog as a last resort.
+    :type iso_code: str
+    :ivar use_ascii: Whether or not ASCII letters should be included in trained
+                     models.
+    :type use_ascii: bool
+    :ivar charsets: The charsets we want to support and create data for.
+    :type charsets: list of str
+    :ivar alphabet: The characters in the language's alphabet. If `use_ascii` is
+                    `True`, you only need to add those not in the ASCII set.
+    :type alphabet: str
+    :ivar wiki_start_pages: The Wikipedia pages to start from if we're crawling
+                            Wikipedia for training data.
+    :type wiki_start_pages: list of str
+    """
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        iso_code: Optional[str] = None,
+        use_ascii: bool = True,
+        charsets: Optional[List[str]] = None,
+        alphabet: Optional[str] = None,
+        wiki_start_pages: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__()
+        self.name = name
+        self.iso_code = iso_code
+        self.use_ascii = use_ascii
+        self.charsets = charsets
+        if self.use_ascii:
+            if alphabet:
+                alphabet += ascii_letters
+            else:
+                alphabet = ascii_letters
+        elif not alphabet:
+            raise ValueError("Must supply alphabet if use_ascii is False")
+        self.alphabet = "".join(sorted(set(alphabet))) if alphabet else None
+        self.wiki_start_pages = wiki_start_pages
+
+    def __repr__(self) -> str:
+        param_str = ", ".join(
+            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
+        )
+        return f"{self.__class__.__name__}({param_str})"
+
+
+LANGUAGES = {
+    "Arabic": Language(
+        name="Arabic",
+        iso_code="ar",
+        use_ascii=False,
+        # We only support encodings that use isolated
+        # forms, because the current recommendation is
+        # that the rendering system handles presentation
+        # forms. This means we purposefully skip IBM864.
+        charsets=["ISO-8859-6", "WINDOWS-1256", "CP720", "CP864"],
+        alphabet="ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ",
+        wiki_start_pages=["الصفحة_الرئيسية"],
+    ),
+    "Belarusian": Language(
+        name="Belarusian",
+        iso_code="be",
+        use_ascii=False,
+        charsets=["ISO-8859-5", "WINDOWS-1251", "IBM866", "MacCyrillic"],
+        alphabet="АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯабвгдеёжзійклмнопрстуўфхцчшыьэюяʼ",
+        wiki_start_pages=["Галоўная_старонка"],
+    ),
+    "Bulgarian": Language(
+        name="Bulgarian",
+        iso_code="bg",
+        use_ascii=False,
+        charsets=["ISO-8859-5", "WINDOWS-1251", "IBM855"],
+        alphabet="АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя",
+        wiki_start_pages=["Начална_страница"],
+    ),
+    "Czech": Language(
+        name="Czech",
+        iso_code="cz",
+        use_ascii=True,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ",
+        wiki_start_pages=["Hlavní_strana"],
+    ),
+    "Danish": Language(
+        name="Danish",
+        iso_code="da",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="æøåÆØÅ",
+        wiki_start_pages=["Forside"],
+    ),
+    "German": Language(
+        name="German",
+        iso_code="de",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="äöüßẞÄÖÜ",
+        wiki_start_pages=["Wikipedia:Hauptseite"],
+    ),
+    "Greek": Language(
+        name="Greek",
+        iso_code="el",
+        use_ascii=False,
+        charsets=["ISO-8859-7", "WINDOWS-1253"],
+        alphabet="αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ",
+        wiki_start_pages=["Πύλη:Κύρια"],
+    ),
+    "English": Language(
+        name="English",
+        iso_code="en",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "WINDOWS-1252", "MacRoman"],
+        wiki_start_pages=["Main_Page"],
+    ),
+    "Esperanto": Language(
+        name="Esperanto",
+        iso_code="eo",
+        # Q, W, X, and Y not used at all
+        use_ascii=False,
+        charsets=["ISO-8859-3"],
+        alphabet="abcĉdefgĝhĥijĵklmnoprsŝtuŭvzABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ",
+        wiki_start_pages=["Vikipedio:Ĉefpaĝo"],
+    ),
+    "Spanish": Language(
+        name="Spanish",
+        iso_code="es",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="ñáéíóúüÑÁÉÍÓÚÜ",
+        wiki_start_pages=["Wikipedia:Portada"],
+    ),
+    "Estonian": Language(
+        name="Estonian",
+        iso_code="et",
+        use_ascii=False,
+        charsets=["ISO-8859-4", "ISO-8859-13", "WINDOWS-1257"],
+        # C, F, Š, Q, W, X, Y, Z, Ž are only for
+        # loanwords
+        alphabet="ABDEGHIJKLMNOPRSTUVÕÄÖÜabdeghijklmnoprstuvõäöü",
+        wiki_start_pages=["Esileht"],
+    ),
+    "Finnish": Language(
+        name="Finnish",
+        iso_code="fi",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="ÅÄÖŠŽåäöšž",
+        wiki_start_pages=["Wikipedia:Etusivu"],
+    ),
+    "French": Language(
+        name="French",
+        iso_code="fr",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ",
+        wiki_start_pages=["Wikipédia:Accueil_principal", "Bœuf (animal)"],
+    ),
+    "Hebrew": Language(
+        name="Hebrew",
+        iso_code="he",
+        use_ascii=False,
+        charsets=["ISO-8859-8", "WINDOWS-1255"],
+        alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ",
+        wiki_start_pages=["עמוד_ראשי"],
+    ),
+    "Croatian": Language(
+        name="Croatian",
+        iso_code="hr",
+        # Q, W, X, Y are only used for foreign words.
+        use_ascii=False,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="abcčćdđefghijklmnoprsštuvzžABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ",
+        wiki_start_pages=["Glavna_stranica"],
+    ),
+    "Hungarian": Language(
+        name="Hungarian",
+        iso_code="hu",
+        # Q, W, X, Y are only used for foreign words.
+        use_ascii=False,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="abcdefghijklmnoprstuvzáéíóöőúüűABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ",
+        wiki_start_pages=["Kezdőlap"],
+    ),
+    "Italian": Language(
+        name="Italian",
+        iso_code="it",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="ÀÈÉÌÒÓÙàèéìòóù",
+        wiki_start_pages=["Pagina_principale"],
+    ),
+    "Lithuanian": Language(
+        name="Lithuanian",
+        iso_code="lt",
+        use_ascii=False,
+        charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"],
+        # Q, W, and X not used at all
+        alphabet="AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽaąbcčdeęėfghiįyjklmnoprsštuųūvzž",
+        wiki_start_pages=["Pagrindinis_puslapis"],
+    ),
+    "Latvian": Language(
+        name="Latvian",
+        iso_code="lv",
+        use_ascii=False,
+        charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"],
+        # Q, W, X, Y are only for loanwords
+        alphabet="AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽaābcčdeēfgģhiījkķlļmnņoprsštuūvzž",
+        wiki_start_pages=["Sākumlapa"],
+    ),
+    "Macedonian": Language(
+        name="Macedonian",
+        iso_code="mk",
+        use_ascii=False,
+        charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"],
+        alphabet="АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШабвгдѓежзѕијклљмнњопрстќуфхцчџш",
+        wiki_start_pages=["Главна_страница"],
+    ),
+    "Dutch": Language(
+        name="Dutch",
+        iso_code="nl",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "WINDOWS-1252", "MacRoman"],
+        wiki_start_pages=["Hoofdpagina"],
+    ),
+    "Polish": Language(
+        name="Polish",
+        iso_code="pl",
+        # Q and X are only used for foreign words.
+        use_ascii=False,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻaąbcćdeęfghijklłmnńoóprsśtuwyzźż",
+        wiki_start_pages=["Wikipedia:Strona_główna"],
+    ),
+    "Portuguese": Language(
+        name="Portuguese",
+        iso_code="pt",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú",
+        wiki_start_pages=["Wikipédia:Página_principal"],
+    ),
+    "Romanian": Language(
+        name="Romanian",
+        iso_code="ro",
+        use_ascii=True,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="ăâîșțĂÂÎȘȚ",
+        wiki_start_pages=["Pagina_principală"],
+    ),
+    "Russian": Language(
+        name="Russian",
+        iso_code="ru",
+        use_ascii=False,
+        charsets=[
+            "ISO-8859-5",
+            "WINDOWS-1251",
+            "KOI8-R",
+            "MacCyrillic",
+            "IBM866",
+            "IBM855",
+        ],
+        alphabet="абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ",
+        wiki_start_pages=["Заглавная_страница"],
+    ),
+    "Slovak": Language(
+        name="Slovak",
+        iso_code="sk",
+        use_ascii=True,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ",
+        wiki_start_pages=["Hlavná_stránka"],
+    ),
+    "Slovene": Language(
+        name="Slovene",
+        iso_code="sl",
+        # Q, W, X, Y are only used for foreign words.
+        use_ascii=False,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="abcčdefghijklmnoprsštuvzžABCČDEFGHIJKLMNOPRSŠTUVZŽ",
+        wiki_start_pages=["Glavna_stran"],
+    ),
+    # Serbian can be written in both Latin and Cyrillic, but there's no
+    # simple way to get the Latin alphabet pages from Wikipedia through
+    # the API, so for now we just support Cyrillic.
+    "Serbian": Language(
+        name="Serbian",
+        iso_code="sr",
+        alphabet="АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШабвгдђежзијклљмнњопрстћуфхцчџш",
+        charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"],
+        wiki_start_pages=["Главна_страна"],
+    ),
+    "Thai": Language(
+        name="Thai",
+        iso_code="th",
+        use_ascii=False,
+        charsets=["ISO-8859-11", "TIS-620", "CP874"],
+        alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛",
+        wiki_start_pages=["หน้าหลัก"],
+    ),
+    "Turkish": Language(
+        name="Turkish",
+        iso_code="tr",
+        # Q, W, and X are not used by Turkish
+        use_ascii=False,
+        charsets=["ISO-8859-3", "ISO-8859-9", "WINDOWS-1254"],
+        alphabet="abcçdefgğhıijklmnoöprsştuüvyzâîûABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ",
+        wiki_start_pages=["Ana_Sayfa"],
+    ),
+    "Vietnamese": Language(
+        name="Vietnamese",
+        iso_code="vi",
+        use_ascii=False,
+        # Windows-1258 is the only common 8-bit
+        # Vietnamese encoding supported by Python.
+        # From Wikipedia:
+        # For systems that lack support for Unicode,
+        # dozens of 8-bit Vietnamese code pages are
+        # available.[1] The most common are VISCII
+        # (TCVN 5712:1993), VPS, and Windows-1258.[3]
+        # Where ASCII is required, such as when
+        # ensuring readability in plain text e-mail,
+        # Vietnamese letters are often encoded
+        # according to Vietnamese Quoted-Readable
+        # (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
+        # though usage of either variable-width
+        # scheme has declined dramatically following
+        # the adoption of Unicode on the World Wide
+        # Web.
+        charsets=["WINDOWS-1258"],
+        alphabet="aăâbcdđeêghiklmnoôơpqrstuưvxyAĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY",
+        wiki_start_pages=["Chữ_Quốc_ngữ"],
+    ),
+}
diff --git a/contrib/python/chardet/py3/chardet/py.typed b/contrib/python/chardet/py3/chardet/py.typed
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/chardet/py3/chardet/resultdict.py b/contrib/python/chardet/py3/chardet/resultdict.py
new file mode 100644
index 00000000000..7d36e64c467
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/resultdict.py
@@ -0,0 +1,16 @@
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    # TypedDict was introduced in Python 3.8.
+    #
+    # TODO: Remove the else block and TYPE_CHECKING check when dropping support
+    # for Python 3.7.
+    from typing import TypedDict
+
+    class ResultDict(TypedDict):
+        encoding: Optional[str]
+        confidence: float
+        language: Optional[str]
+
+else:
+    ResultDict = dict
diff --git a/contrib/python/chardet/py3/chardet/sbcharsetprober.py b/contrib/python/chardet/py3/chardet/sbcharsetprober.py
new file mode 100644
index 00000000000..0ffbcdd2c3e
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/sbcharsetprober.py
@@ -0,0 +1,162 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import Dict, List, NamedTuple, Optional, Union
+
+from .charsetprober import CharSetProber
+from .enums import CharacterCategory, ProbingState, SequenceLikelihood
+
+
+class SingleByteCharSetModel(NamedTuple):
+    charset_name: str
+    language: str
+    char_to_order_map: Dict[int, int]
+    language_model: Dict[int, Dict[int, int]]
+    typical_positive_ratio: float
+    keep_ascii_letters: bool
+    alphabet: str
+
+
+class SingleByteCharSetProber(CharSetProber):
+    SAMPLE_SIZE = 64
+    SB_ENOUGH_REL_THRESHOLD = 1024  # 0.25 * SAMPLE_SIZE^2
+    POSITIVE_SHORTCUT_THRESHOLD = 0.95
+    NEGATIVE_SHORTCUT_THRESHOLD = 0.05
+
+    def __init__(
+        self,
+        model: SingleByteCharSetModel,
+        is_reversed: bool = False,
+        name_prober: Optional[CharSetProber] = None,
+    ) -> None:
+        super().__init__()
+        self._model = model
+        # TRUE if we need to reverse every pair in the model lookup
+        self._reversed = is_reversed
+        # Optional auxiliary prober for name decision
+        self._name_prober = name_prober
+        self._last_order = 255
+        self._seq_counters: List[int] = []
+        self._total_seqs = 0
+        self._total_char = 0
+        self._control_char = 0
+        self._freq_char = 0
+        self.reset()
+
+    def reset(self) -> None:
+        super().reset()
+        # char order of last character
+        self._last_order = 255
+        self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
+        self._total_seqs = 0
+        self._total_char = 0
+        self._control_char = 0
+        # characters that fall in our sampling range
+        self._freq_char = 0
+
+    @property
+    def charset_name(self) -> Optional[str]:
+        if self._name_prober:
+            return self._name_prober.charset_name
+        return self._model.charset_name
+
+    @property
+    def language(self) -> Optional[str]:
+        if self._name_prober:
+            return self._name_prober.language
+        return self._model.language
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        # TODO: Make filter_international_words keep things in self.alphabet
+        if not self._model.keep_ascii_letters:
+            byte_str = self.filter_international_words(byte_str)
+        else:
+            byte_str = self.remove_xml_tags(byte_str)
+        if not byte_str:
+            return self.state
+        char_to_order_map = self._model.char_to_order_map
+        language_model = self._model.language_model
+        for char in byte_str:
+            order = char_to_order_map.get(char, CharacterCategory.UNDEFINED)
+            # XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
+            #      CharacterCategory.SYMBOL is actually 253, so we use CONTROL
+            #      to make it closer to the original intent. The only difference
+            #      is whether or not we count digits and control characters for
+            #      _total_char purposes.
+            if order < CharacterCategory.CONTROL:
+                self._total_char += 1
+            if order < self.SAMPLE_SIZE:
+                self._freq_char += 1
+                if self._last_order < self.SAMPLE_SIZE:
+                    self._total_seqs += 1
+                    if not self._reversed:
+                        lm_cat = language_model[self._last_order][order]
+                    else:
+                        lm_cat = language_model[order][self._last_order]
+                    self._seq_counters[lm_cat] += 1
+            self._last_order = order
+
+        charset_name = self._model.charset_name
+        if self.state == ProbingState.DETECTING:
+            if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
+                confidence = self.get_confidence()
+                if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
+                    self.logger.debug(
+                        "%s confidence = %s, we have a winner", charset_name, confidence
+                    )
+                    self._state = ProbingState.FOUND_IT
+                elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
+                    self.logger.debug(
+                        "%s confidence = %s, below negative shortcut threshold %s",
+                        charset_name,
+                        confidence,
+                        self.NEGATIVE_SHORTCUT_THRESHOLD,
+                    )
+                    self._state = ProbingState.NOT_ME
+
+        return self.state
+
+    def get_confidence(self) -> float:
+        r = 0.01
+        if self._total_seqs > 0:
+            r = (
+                (
+                    self._seq_counters[SequenceLikelihood.POSITIVE]
+                    + 0.25 * self._seq_counters[SequenceLikelihood.LIKELY]
+                )
+                / self._total_seqs
+                / self._model.typical_positive_ratio
+            )
+            # The more control characters (proportionnaly to the size
+            # of the text), the less confident we become in the current
+            # charset.
+            r = r * (self._total_char - self._control_char) / self._total_char
+            r = r * self._freq_char / self._total_char
+            if r >= 1.0:
+                r = 0.99
+        return r
diff --git a/contrib/python/chardet/py3/chardet/sbcsgroupprober.py b/contrib/python/chardet/py3/chardet/sbcsgroupprober.py
new file mode 100644
index 00000000000..890ae8465c5
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/sbcsgroupprober.py
@@ -0,0 +1,88 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .charsetgroupprober import CharSetGroupProber
+from .hebrewprober import HebrewProber
+from .langbulgarianmodel import ISO_8859_5_BULGARIAN_MODEL, WINDOWS_1251_BULGARIAN_MODEL
+from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
+from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
+
+# from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
+#                                  WINDOWS_1250_HUNGARIAN_MODEL)
+from .langrussianmodel import (
+    IBM855_RUSSIAN_MODEL,
+    IBM866_RUSSIAN_MODEL,
+    ISO_8859_5_RUSSIAN_MODEL,
+    KOI8_R_RUSSIAN_MODEL,
+    MACCYRILLIC_RUSSIAN_MODEL,
+    WINDOWS_1251_RUSSIAN_MODEL,
+)
+from .langthaimodel import TIS_620_THAI_MODEL
+from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
+from .sbcharsetprober import SingleByteCharSetProber
+
+
+class SBCSGroupProber(CharSetGroupProber):
+    def __init__(self) -> None:
+        super().__init__()
+        hebrew_prober = HebrewProber()
+        logical_hebrew_prober = SingleByteCharSetProber(
+            WINDOWS_1255_HEBREW_MODEL, is_reversed=False, name_prober=hebrew_prober
+        )
+        # TODO: See if using ISO-8859-8 Hebrew model works better here, since
+        #       it's actually the visual one
+        visual_hebrew_prober = SingleByteCharSetProber(
+            WINDOWS_1255_HEBREW_MODEL, is_reversed=True, name_prober=hebrew_prober
+        )
+        hebrew_prober.set_model_probers(logical_hebrew_prober, visual_hebrew_prober)
+        # TODO: ORDER MATTERS HERE. I changed the order vs what was in master
+        #       and several tests failed that did not before. Some thought
+        #       should be put into the ordering, and we should consider making
+        #       order not matter here, because that is very counter-intuitive.
+        self.probers = [
+            SingleByteCharSetProber(WINDOWS_1251_RUSSIAN_MODEL),
+            SingleByteCharSetProber(KOI8_R_RUSSIAN_MODEL),
+            SingleByteCharSetProber(ISO_8859_5_RUSSIAN_MODEL),
+            SingleByteCharSetProber(MACCYRILLIC_RUSSIAN_MODEL),
+            SingleByteCharSetProber(IBM866_RUSSIAN_MODEL),
+            SingleByteCharSetProber(IBM855_RUSSIAN_MODEL),
+            SingleByteCharSetProber(ISO_8859_7_GREEK_MODEL),
+            SingleByteCharSetProber(WINDOWS_1253_GREEK_MODEL),
+            SingleByteCharSetProber(ISO_8859_5_BULGARIAN_MODEL),
+            SingleByteCharSetProber(WINDOWS_1251_BULGARIAN_MODEL),
+            # TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250)
+            #       after we retrain model.
+            # SingleByteCharSetProber(ISO_8859_2_HUNGARIAN_MODEL),
+            # SingleByteCharSetProber(WINDOWS_1250_HUNGARIAN_MODEL),
+            SingleByteCharSetProber(TIS_620_THAI_MODEL),
+            SingleByteCharSetProber(ISO_8859_9_TURKISH_MODEL),
+            hebrew_prober,
+            logical_hebrew_prober,
+            visual_hebrew_prober,
+        ]
+        self.reset()
diff --git a/contrib/python/chardet/py3/chardet/sjisprober.py b/contrib/python/chardet/py3/chardet/sjisprober.py
new file mode 100644
index 00000000000..91df077961b
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/sjisprober.py
@@ -0,0 +1,105 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import Union
+
+from .chardistribution import SJISDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .enums import MachineState, ProbingState
+from .jpcntx import SJISContextAnalysis
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import SJIS_SM_MODEL
+
+
+class SJISProber(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(SJIS_SM_MODEL)
+        self.distribution_analyzer = SJISDistributionAnalysis()
+        self.context_analyzer = SJISContextAnalysis()
+        self.reset()
+
+    def reset(self) -> None:
+        super().reset()
+        self.context_analyzer.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return self.context_analyzer.charset_name
+
+    @property
+    def language(self) -> str:
+        return "Japanese"
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        assert self.coding_sm is not None
+        assert self.distribution_analyzer is not None
+
+        for i, byte in enumerate(byte_str):
+            coding_state = self.coding_sm.next_state(byte)
+            if coding_state == MachineState.ERROR:
+                self.logger.debug(
+                    "%s %s prober hit error at byte %s",
+                    self.charset_name,
+                    self.language,
+                    i,
+                )
+                self._state = ProbingState.NOT_ME
+                break
+            if coding_state == MachineState.ITS_ME:
+                self._state = ProbingState.FOUND_IT
+                break
+            if coding_state == MachineState.START:
+                char_len = self.coding_sm.get_current_charlen()
+                if i == 0:
+                    self._last_char[1] = byte
+                    self.context_analyzer.feed(
+                        self._last_char[2 - char_len :], char_len
+                    )
+                    self.distribution_analyzer.feed(self._last_char, char_len)
+                else:
+                    self.context_analyzer.feed(
+                        byte_str[i + 1 - char_len : i + 3 - char_len], char_len
+                    )
+                    self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
+
+        self._last_char[0] = byte_str[-1]
+
+        if self.state == ProbingState.DETECTING:
+            if self.context_analyzer.got_enough_data() and (
+                self.get_confidence() > self.SHORTCUT_THRESHOLD
+            ):
+                self._state = ProbingState.FOUND_IT
+
+        return self.state
+
+    def get_confidence(self) -> float:
+        assert self.distribution_analyzer is not None
+
+        context_conf = self.context_analyzer.get_confidence()
+        distrib_conf = self.distribution_analyzer.get_confidence()
+        return max(context_conf, distrib_conf)
diff --git a/contrib/python/chardet/py3/chardet/universaldetector.py b/contrib/python/chardet/py3/chardet/universaldetector.py
new file mode 100644
index 00000000000..30c441dc28e
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/universaldetector.py
@@ -0,0 +1,362 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+"""
+Module containing the UniversalDetector detector class, which is the primary
+class a user of ``chardet`` should use.
+
+:author: Mark Pilgrim (initial port to Python)
+:author: Shy Shalom (original C code)
+:author: Dan Blanchard (major refactoring for 3.0)
+:author: Ian Cordasco
+"""
+
+
+import codecs
+import logging
+import re
+from typing import List, Optional, Union
+
+from .charsetgroupprober import CharSetGroupProber
+from .charsetprober import CharSetProber
+from .enums import InputState, LanguageFilter, ProbingState
+from .escprober import EscCharSetProber
+from .latin1prober import Latin1Prober
+from .macromanprober import MacRomanProber
+from .mbcsgroupprober import MBCSGroupProber
+from .resultdict import ResultDict
+from .sbcsgroupprober import SBCSGroupProber
+from .utf1632prober import UTF1632Prober
+
+
+class UniversalDetector:
+    """
+    The ``UniversalDetector`` class underlies the ``chardet.detect`` function
+    and coordinates all of the different charset probers.
+
+    To get a ``dict`` containing an encoding and its confidence, you can simply
+    run:
+
+    .. code::
+
+            u = UniversalDetector()
+            u.feed(some_bytes)
+            u.close()
+            detected = u.result
+
+    """
+
+    MINIMUM_THRESHOLD = 0.20
+    HIGH_BYTE_DETECTOR = re.compile(b"[\x80-\xFF]")
+    ESC_DETECTOR = re.compile(b"(\033|~{)")
+    WIN_BYTE_DETECTOR = re.compile(b"[\x80-\x9F]")
+    ISO_WIN_MAP = {
+        "iso-8859-1": "Windows-1252",
+        "iso-8859-2": "Windows-1250",
+        "iso-8859-5": "Windows-1251",
+        "iso-8859-6": "Windows-1256",
+        "iso-8859-7": "Windows-1253",
+        "iso-8859-8": "Windows-1255",
+        "iso-8859-9": "Windows-1254",
+        "iso-8859-13": "Windows-1257",
+    }
+    # Based on https://encoding.spec.whatwg.org/#names-and-labels
+    # but altered to match Python names for encodings and remove mappings
+    # that break tests.
+    LEGACY_MAP = {
+        "ascii": "Windows-1252",
+        "iso-8859-1": "Windows-1252",
+        "tis-620": "ISO-8859-11",
+        "iso-8859-9": "Windows-1254",
+        "gb2312": "GB18030",
+        "euc-kr": "CP949",
+        "utf-16le": "UTF-16",
+    }
+
+    def __init__(
+        self,
+        lang_filter: LanguageFilter = LanguageFilter.ALL,
+        should_rename_legacy: bool = False,
+    ) -> None:
+        self._esc_charset_prober: Optional[EscCharSetProber] = None
+        self._utf1632_prober: Optional[UTF1632Prober] = None
+        self._charset_probers: List[CharSetProber] = []
+        self.result: ResultDict = {
+            "encoding": None,
+            "confidence": 0.0,
+            "language": None,
+        }
+        self.done = False
+        self._got_data = False
+        self._input_state = InputState.PURE_ASCII
+        self._last_char = b""
+        self.lang_filter = lang_filter
+        self.logger = logging.getLogger(__name__)
+        self._has_win_bytes = False
+        self.should_rename_legacy = should_rename_legacy
+        self.reset()
+
+    @property
+    def input_state(self) -> int:
+        return self._input_state
+
+    @property
+    def has_win_bytes(self) -> bool:
+        return self._has_win_bytes
+
+    @property
+    def charset_probers(self) -> List[CharSetProber]:
+        return self._charset_probers
+
+    def reset(self) -> None:
+        """
+        Reset the UniversalDetector and all of its probers back to their
+        initial states.  This is called by ``__init__``, so you only need to
+        call this directly in between analyses of different documents.
+        """
+        self.result = {"encoding": None, "confidence": 0.0, "language": None}
+        self.done = False
+        self._got_data = False
+        self._has_win_bytes = False
+        self._input_state = InputState.PURE_ASCII
+        self._last_char = b""
+        if self._esc_charset_prober:
+            self._esc_charset_prober.reset()
+        if self._utf1632_prober:
+            self._utf1632_prober.reset()
+        for prober in self._charset_probers:
+            prober.reset()
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> None:
+        """
+        Takes a chunk of a document and feeds it through all of the relevant
+        charset probers.
+
+        After calling ``feed``, you can check the value of the ``done``
+        attribute to see if you need to continue feeding the
+        ``UniversalDetector`` more data, or if it has made a prediction
+        (in the ``result`` attribute).
+
+        .. note::
+           You should always call ``close`` when you're done feeding in your
+           document if ``done`` is not already ``True``.
+        """
+        if self.done:
+            return
+
+        if not byte_str:
+            return
+
+        if not isinstance(byte_str, bytearray):
+            byte_str = bytearray(byte_str)
+
+        # First check for known BOMs, since these are guaranteed to be correct
+        if not self._got_data:
+            # If the data starts with BOM, we know it is UTF
+            if byte_str.startswith(codecs.BOM_UTF8):
+                # EF BB BF  UTF-8 with BOM
+                self.result = {
+                    "encoding": "UTF-8-SIG",
+                    "confidence": 1.0,
+                    "language": "",
+                }
+            elif byte_str.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+                # FF FE 00 00  UTF-32, little-endian BOM
+                # 00 00 FE FF  UTF-32, big-endian BOM
+                self.result = {"encoding": "UTF-32", "confidence": 1.0, "language": ""}
+            elif byte_str.startswith(b"\xFE\xFF\x00\x00"):
+                # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
+                self.result = {
+                    # TODO: This encoding is not supported by Python. Should remove?
+                    "encoding": "X-ISO-10646-UCS-4-3412",
+                    "confidence": 1.0,
+                    "language": "",
+                }
+            elif byte_str.startswith(b"\x00\x00\xFF\xFE"):
+                # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
+                self.result = {
+                    # TODO: This encoding is not supported by Python. Should remove?
+                    "encoding": "X-ISO-10646-UCS-4-2143",
+                    "confidence": 1.0,
+                    "language": "",
+                }
+            elif byte_str.startswith((codecs.BOM_LE, codecs.BOM_BE)):
+                # FF FE  UTF-16, little endian BOM
+                # FE FF  UTF-16, big endian BOM
+                self.result = {"encoding": "UTF-16", "confidence": 1.0, "language": ""}
+
+            self._got_data = True
+            if self.result["encoding"] is not None:
+                self.done = True
+                return
+
+        # If none of those matched and we've only see ASCII so far, check
+        # for high bytes and escape sequences
+        if self._input_state == InputState.PURE_ASCII:
+            if self.HIGH_BYTE_DETECTOR.search(byte_str):
+                self._input_state = InputState.HIGH_BYTE
+            elif (
+                self._input_state == InputState.PURE_ASCII
+                and self.ESC_DETECTOR.search(self._last_char + byte_str)
+            ):
+                self._input_state = InputState.ESC_ASCII
+
+        self._last_char = byte_str[-1:]
+
+        # next we will look to see if it is appears to be either a UTF-16 or
+        # UTF-32 encoding
+        if not self._utf1632_prober:
+            self._utf1632_prober = UTF1632Prober()
+
+        if self._utf1632_prober.state == ProbingState.DETECTING:
+            if self._utf1632_prober.feed(byte_str) == ProbingState.FOUND_IT:
+                self.result = {
+                    "encoding": self._utf1632_prober.charset_name,
+                    "confidence": self._utf1632_prober.get_confidence(),
+                    "language": "",
+                }
+                self.done = True
+                return
+
+        # If we've seen escape sequences, use the EscCharSetProber, which
+        # uses a simple state machine to check for known escape sequences in
+        # HZ and ISO-2022 encodings, since those are the only encodings that
+        # use such sequences.
+        if self._input_state == InputState.ESC_ASCII:
+            if not self._esc_charset_prober:
+                self._esc_charset_prober = EscCharSetProber(self.lang_filter)
+            if self._esc_charset_prober.feed(byte_str) == ProbingState.FOUND_IT:
+                self.result = {
+                    "encoding": self._esc_charset_prober.charset_name,
+                    "confidence": self._esc_charset_prober.get_confidence(),
+                    "language": self._esc_charset_prober.language,
+                }
+                self.done = True
+        # If we've seen high bytes (i.e., those with values greater than 127),
+        # we need to do more complicated checks using all our multi-byte and
+        # single-byte probers that are left.  The single-byte probers
+        # use character bigram distributions to determine the encoding, whereas
+        # the multi-byte probers use a combination of character unigram and
+        # bigram distributions.
+        elif self._input_state == InputState.HIGH_BYTE:
+            if not self._charset_probers:
+                self._charset_probers = [MBCSGroupProber(self.lang_filter)]
+                # If we're checking non-CJK encodings, use single-byte prober
+                if self.lang_filter & LanguageFilter.NON_CJK:
+                    self._charset_probers.append(SBCSGroupProber())
+                self._charset_probers.append(Latin1Prober())
+                self._charset_probers.append(MacRomanProber())
+            for prober in self._charset_probers:
+                if prober.feed(byte_str) == ProbingState.FOUND_IT:
+                    self.result = {
+                        "encoding": prober.charset_name,
+                        "confidence": prober.get_confidence(),
+                        "language": prober.language,
+                    }
+                    self.done = True
+                    break
+            if self.WIN_BYTE_DETECTOR.search(byte_str):
+                self._has_win_bytes = True
+
+    def close(self) -> ResultDict:
+        """
+        Stop analyzing the current document and come up with a final
+        prediction.
+
+        :returns:  The ``result`` attribute, a ``dict`` with the keys
+                   `encoding`, `confidence`, and `language`.
+        """
+        # Don't bother with checks if we're already done
+        if self.done:
+            return self.result
+        self.done = True
+
+        if not self._got_data:
+            self.logger.debug("no data received!")
+
+        # Default to ASCII if it is all we've seen so far
+        elif self._input_state == InputState.PURE_ASCII:
+            self.result = {"encoding": "ascii", "confidence": 1.0, "language": ""}
+
+        # If we have seen non-ASCII, return the best that met MINIMUM_THRESHOLD
+        elif self._input_state == InputState.HIGH_BYTE:
+            prober_confidence = None
+            max_prober_confidence = 0.0
+            max_prober = None
+            for prober in self._charset_probers:
+                if not prober:
+                    continue
+                prober_confidence = prober.get_confidence()
+                if prober_confidence > max_prober_confidence:
+                    max_prober_confidence = prober_confidence
+                    max_prober = prober
+            if max_prober and (max_prober_confidence > self.MINIMUM_THRESHOLD):
+                charset_name = max_prober.charset_name
+                assert charset_name is not None
+                lower_charset_name = charset_name.lower()
+                confidence = max_prober.get_confidence()
+                # Use Windows encoding name instead of ISO-8859 if we saw any
+                # extra Windows-specific bytes
+                if lower_charset_name.startswith("iso-8859"):
+                    if self._has_win_bytes:
+                        charset_name = self.ISO_WIN_MAP.get(
+                            lower_charset_name, charset_name
+                        )
+                # Rename legacy encodings with superset encodings if asked
+                if self.should_rename_legacy:
+                    charset_name = self.LEGACY_MAP.get(
+                        (charset_name or "").lower(), charset_name
+                    )
+                self.result = {
+                    "encoding": charset_name,
+                    "confidence": confidence,
+                    "language": max_prober.language,
+                }
+
+        # Log all prober confidences if none met MINIMUM_THRESHOLD
+        if self.logger.getEffectiveLevel() <= logging.DEBUG:
+            if self.result["encoding"] is None:
+                self.logger.debug("no probers hit minimum threshold")
+                for group_prober in self._charset_probers:
+                    if not group_prober:
+                        continue
+                    if isinstance(group_prober, CharSetGroupProber):
+                        for prober in group_prober.probers:
+                            self.logger.debug(
+                                "%s %s confidence = %s",
+                                prober.charset_name,
+                                prober.language,
+                                prober.get_confidence(),
+                            )
+                    else:
+                        self.logger.debug(
+                            "%s %s confidence = %s",
+                            group_prober.charset_name,
+                            group_prober.language,
+                            group_prober.get_confidence(),
+                        )
+        return self.result
diff --git a/contrib/python/chardet/py3/chardet/utf1632prober.py b/contrib/python/chardet/py3/chardet/utf1632prober.py
new file mode 100644
index 00000000000..6bdec63d686
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/utf1632prober.py
@@ -0,0 +1,225 @@
+######################## BEGIN LICENSE BLOCK ########################
+#
+# Contributor(s):
+#   Jason Zavaglia
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+from typing import List, Union
+
+from .charsetprober import CharSetProber
+from .enums import ProbingState
+
+
+class UTF1632Prober(CharSetProber):
+    """
+    This class simply looks for occurrences of zero bytes, and infers
+    whether the file is UTF16 or UTF32 (low-endian or big-endian)
+    For instance, files looking like ( \0 \0 \0 [nonzero] )+
+    have a good probability to be UTF32BE.  Files looking like ( \0 [nonzero] )+
+    may be guessed to be UTF16BE, and inversely for little-endian varieties.
+    """
+
+    # how many logical characters to scan before feeling confident of prediction
+    MIN_CHARS_FOR_DETECTION = 20
+    # a fixed constant ratio of expected zeros or non-zeros in modulo-position.
+    EXPECTED_RATIO = 0.94
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.position = 0
+        self.zeros_at_mod = [0] * 4
+        self.nonzeros_at_mod = [0] * 4
+        self._state = ProbingState.DETECTING
+        self.quad = [0, 0, 0, 0]
+        self.invalid_utf16be = False
+        self.invalid_utf16le = False
+        self.invalid_utf32be = False
+        self.invalid_utf32le = False
+        self.first_half_surrogate_pair_detected_16be = False
+        self.first_half_surrogate_pair_detected_16le = False
+        self.reset()
+
+    def reset(self) -> None:
+        super().reset()
+        self.position = 0
+        self.zeros_at_mod = [0] * 4
+        self.nonzeros_at_mod = [0] * 4
+        self._state = ProbingState.DETECTING
+        self.invalid_utf16be = False
+        self.invalid_utf16le = False
+        self.invalid_utf32be = False
+        self.invalid_utf32le = False
+        self.first_half_surrogate_pair_detected_16be = False
+        self.first_half_surrogate_pair_detected_16le = False
+        self.quad = [0, 0, 0, 0]
+
+    @property
+    def charset_name(self) -> str:
+        if self.is_likely_utf32be():
+            return "utf-32be"
+        if self.is_likely_utf32le():
+            return "utf-32le"
+        if self.is_likely_utf16be():
+            return "utf-16be"
+        if self.is_likely_utf16le():
+            return "utf-16le"
+        # default to something valid
+        return "utf-16"
+
+    @property
+    def language(self) -> str:
+        return ""
+
+    def approx_32bit_chars(self) -> float:
+        return max(1.0, self.position / 4.0)
+
+    def approx_16bit_chars(self) -> float:
+        return max(1.0, self.position / 2.0)
+
+    def is_likely_utf32be(self) -> bool:
+        approx_chars = self.approx_32bit_chars()
+        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
+            self.zeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
+            and self.nonzeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
+            and not self.invalid_utf32be
+        )
+
+    def is_likely_utf32le(self) -> bool:
+        approx_chars = self.approx_32bit_chars()
+        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
+            self.nonzeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
+            and not self.invalid_utf32le
+        )
+
+    def is_likely_utf16be(self) -> bool:
+        approx_chars = self.approx_16bit_chars()
+        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
+            (self.nonzeros_at_mod[1] + self.nonzeros_at_mod[3]) / approx_chars
+            > self.EXPECTED_RATIO
+            and (self.zeros_at_mod[0] + self.zeros_at_mod[2]) / approx_chars
+            > self.EXPECTED_RATIO
+            and not self.invalid_utf16be
+        )
+
+    def is_likely_utf16le(self) -> bool:
+        approx_chars = self.approx_16bit_chars()
+        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
+            (self.nonzeros_at_mod[0] + self.nonzeros_at_mod[2]) / approx_chars
+            > self.EXPECTED_RATIO
+            and (self.zeros_at_mod[1] + self.zeros_at_mod[3]) / approx_chars
+            > self.EXPECTED_RATIO
+            and not self.invalid_utf16le
+        )
+
+    def validate_utf32_characters(self, quad: List[int]) -> None:
+        """
+        Validate if the quad of bytes is valid UTF-32.
+
+        UTF-32 is valid in the range 0x00000000 - 0x0010FFFF
+        excluding 0x0000D800 - 0x0000DFFF
+
+        https://en.wikipedia.org/wiki/UTF-32
+        """
+        if (
+            quad[0] != 0
+            or quad[1] > 0x10
+            or (quad[0] == 0 and quad[1] == 0 and 0xD8 <= quad[2] <= 0xDF)
+        ):
+            self.invalid_utf32be = True
+        if (
+            quad[3] != 0
+            or quad[2] > 0x10
+            or (quad[3] == 0 and quad[2] == 0 and 0xD8 <= quad[1] <= 0xDF)
+        ):
+            self.invalid_utf32le = True
+
+    def validate_utf16_characters(self, pair: List[int]) -> None:
+        """
+        Validate if the pair of bytes is  valid UTF-16.
+
+        UTF-16 is valid in the range 0x0000 - 0xFFFF excluding 0xD800 - 0xFFFF
+        with an exception for surrogate pairs, which must be in the range
+        0xD800-0xDBFF followed by 0xDC00-0xDFFF
+
+        https://en.wikipedia.org/wiki/UTF-16
+        """
+        if not self.first_half_surrogate_pair_detected_16be:
+            if 0xD8 <= pair[0] <= 0xDB:
+                self.first_half_surrogate_pair_detected_16be = True
+            elif 0xDC <= pair[0] <= 0xDF:
+                self.invalid_utf16be = True
+        else:
+            if 0xDC <= pair[0] <= 0xDF:
+                self.first_half_surrogate_pair_detected_16be = False
+            else:
+                self.invalid_utf16be = True
+
+        if not self.first_half_surrogate_pair_detected_16le:
+            if 0xD8 <= pair[1] <= 0xDB:
+                self.first_half_surrogate_pair_detected_16le = True
+            elif 0xDC <= pair[1] <= 0xDF:
+                self.invalid_utf16le = True
+        else:
+            if 0xDC <= pair[1] <= 0xDF:
+                self.first_half_surrogate_pair_detected_16le = False
+            else:
+                self.invalid_utf16le = True
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        for c in byte_str:
+            mod4 = self.position % 4
+            self.quad[mod4] = c
+            if mod4 == 3:
+                self.validate_utf32_characters(self.quad)
+                self.validate_utf16_characters(self.quad[0:2])
+                self.validate_utf16_characters(self.quad[2:4])
+            if c == 0:
+                self.zeros_at_mod[mod4] += 1
+            else:
+                self.nonzeros_at_mod[mod4] += 1
+            self.position += 1
+        return self.state
+
+    @property
+    def state(self) -> ProbingState:
+        if self._state in {ProbingState.NOT_ME, ProbingState.FOUND_IT}:
+            # terminal, decided states
+            return self._state
+        if self.get_confidence() > 0.80:
+            self._state = ProbingState.FOUND_IT
+        elif self.position > 4 * 1024:
+            # if we get to 4kb into the file, and we can't conclude it's UTF,
+            # let's give up
+            self._state = ProbingState.NOT_ME
+        return self._state
+
+    def get_confidence(self) -> float:
+        return (
+            0.85
+            if (
+                self.is_likely_utf16le()
+                or self.is_likely_utf16be()
+                or self.is_likely_utf32le()
+                or self.is_likely_utf32be()
+            )
+            else 0.00
+        )
diff --git a/contrib/python/chardet/py3/chardet/utf8prober.py b/contrib/python/chardet/py3/chardet/utf8prober.py
new file mode 100644
index 00000000000..d96354d97c2
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/utf8prober.py
@@ -0,0 +1,82 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import Union
+
+from .charsetprober import CharSetProber
+from .codingstatemachine import CodingStateMachine
+from .enums import MachineState, ProbingState
+from .mbcssm import UTF8_SM_MODEL
+
+
+class UTF8Prober(CharSetProber):
+    ONE_CHAR_PROB = 0.5
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(UTF8_SM_MODEL)
+        self._num_mb_chars = 0
+        self.reset()
+
+    def reset(self) -> None:
+        super().reset()
+        self.coding_sm.reset()
+        self._num_mb_chars = 0
+
+    @property
+    def charset_name(self) -> str:
+        return "utf-8"
+
+    @property
+    def language(self) -> str:
+        return ""
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        for c in byte_str:
+            coding_state = self.coding_sm.next_state(c)
+            if coding_state == MachineState.ERROR:
+                self._state = ProbingState.NOT_ME
+                break
+            if coding_state == MachineState.ITS_ME:
+                self._state = ProbingState.FOUND_IT
+                break
+            if coding_state == MachineState.START:
+                if self.coding_sm.get_current_charlen() >= 2:
+                    self._num_mb_chars += 1
+
+        if self.state == ProbingState.DETECTING:
+            if self.get_confidence() > self.SHORTCUT_THRESHOLD:
+                self._state = ProbingState.FOUND_IT
+
+        return self.state
+
+    def get_confidence(self) -> float:
+        unlike = 0.99
+        if self._num_mb_chars < 6:
+            unlike *= self.ONE_CHAR_PROB**self._num_mb_chars
+            return 1.0 - unlike
+        return unlike
diff --git a/contrib/python/chardet/py3/chardet/version.py b/contrib/python/chardet/py3/chardet/version.py
new file mode 100644
index 00000000000..19dd01e0301
--- /dev/null
+++ b/contrib/python/chardet/py3/chardet/version.py
@@ -0,0 +1,9 @@
+"""
+This module exists only to simplify retrieving the version number of chardet
+from within setuptools and from chardet subpackages.
+
+:author: Dan Blanchard (dan.blanchard@gmail.com)
+"""
+
+__version__ = "5.2.0"
+VERSION = __version__.split(".")
diff --git a/contrib/python/chardet/py3/test.py b/contrib/python/chardet/py3/test.py
new file mode 100644
index 00000000000..5be3ab3fa0f
--- /dev/null
+++ b/contrib/python/chardet/py3/test.py
@@ -0,0 +1,240 @@
+"""
+Run chardet on a bunch of documents and see that we get the correct encodings.
+
+:author: Dan Blanchard
+:author: Ian Cordasco
+"""
+
+
+import textwrap
+from difflib import ndiff
+from os import listdir
+from os.path import dirname, isdir, join, realpath, relpath, splitext
+from pprint import pformat
+from unicodedata import normalize
+
+try:
+    import hypothesis.strategies as st
+    from hypothesis import Verbosity, assume, given, settings
+
+    HAVE_HYPOTHESIS = True
+except ImportError:
+    HAVE_HYPOTHESIS = False
+import pytest  # pylint: disable=import-error
+
+import chardet
+from chardet.metadata.languages import LANGUAGES
+
+# TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250) after we
+#       retrain model.
+MISSING_ENCODINGS = {
+    "iso-8859-2",
+    "iso-8859-6",
+    "windows-1250",
+    "windows-1254",
+    "windows-1256",
+}
+EXPECTED_FAILURES = {
+    "tests/iso-8859-9-turkish/_ude_1.txt",
+    "tests/iso-8859-9-turkish/_ude_2.txt",
+    "tests/iso-8859-9-turkish/divxplanet.com.xml",
+    "tests/iso-8859-9-turkish/subtitle.srt",
+    "tests/iso-8859-9-turkish/wikitop_tr_ISO-8859-9.txt",
+}
+
+
+def gen_test_params():
+    """Yields tuples of paths and encodings to use for test_encoding_detection"""
+    import yatest.common
+    base_path = yatest.common.work_path('test_data/tests')
+    for encoding in listdir(base_path):
+        path = join(base_path, encoding)
+        # Skip files in tests directory
+        if not isdir(path):
+            continue
+        # Remove language suffixes from encoding if present
+        encoding = encoding.lower()
+        for language in sorted(LANGUAGES.keys()):
+            postfix = "-" + language.lower()
+            if encoding.endswith(postfix):
+                encoding = encoding.rpartition(postfix)[0]
+                break
+        # Skip directories for encodings we don't handle yet.
+        if encoding in MISSING_ENCODINGS:
+            continue
+        # Test encoding detection for each file we have of encoding for
+        for file_name in listdir(path):
+            ext = splitext(file_name)[1].lower()
+            if ext not in [".html", ".txt", ".xml", ".srt"]:
+                continue
+            full_path = join(path, file_name)
+            test_case = full_path, encoding
+            name_test = full_path.split("/test_data/")[-1]
+            if name_test in EXPECTED_FAILURES:
+                test_case = pytest.param(*test_case, marks=pytest.mark.xfail, id=name_test)
+            else:
+                test_case = pytest.param(*test_case, id=name_test)
+            yield test_case
+
+
+@pytest.mark.parametrize("file_name, encoding", gen_test_params())
+def test_encoding_detection(file_name, encoding):
+    with open(file_name, "rb") as f:
+        input_bytes = f.read()
+        result = chardet.detect(input_bytes)
+        try:
+            expected_unicode = input_bytes.decode(encoding)
+        except LookupError:
+            expected_unicode = ""
+        try:
+            detected_unicode = input_bytes.decode(result["encoding"])
+        except (LookupError, UnicodeDecodeError, TypeError):
+            detected_unicode = ""
+    if result:
+        encoding_match = (result["encoding"] or "").lower() == encoding
+    else:
+        encoding_match = False
+    # Only care about mismatches that would actually result in different
+    # behavior when decoding
+    expected_unicode = normalize("NFKC", expected_unicode)
+    detected_unicode = normalize("NFKC", detected_unicode)
+    if not encoding_match and expected_unicode != detected_unicode:
+        wrapped_expected = "\n".join(textwrap.wrap(expected_unicode, 100)) + "\n"
+        wrapped_detected = "\n".join(textwrap.wrap(detected_unicode, 100)) + "\n"
+        diff = "".join(
+            [
+                line
+                for line in ndiff(
+                    wrapped_expected.splitlines(True), wrapped_detected.splitlines(True)
+                )
+                if not line.startswith(" ")
+            ][:20]
+        )
+        all_encodings = chardet.detect_all(input_bytes, ignore_threshold=True)
+    else:
+        diff = ""
+        encoding_match = True
+        all_encodings = [result]
+    assert encoding_match, (
+        f"Expected {encoding}, but got {result} for {file_name}.  First 20 "
+        f"lines with character differences: \n{diff}\n"
+        f"All encodings: {pformat(all_encodings)}"
+    )
+
+
+@pytest.mark.parametrize("file_name, encoding", gen_test_params())
+def test_encoding_detection_rename_legacy(file_name, encoding):
+    with open(file_name, "rb") as f:
+        input_bytes = f.read()
+        result = chardet.detect(input_bytes, should_rename_legacy=True)
+        try:
+            expected_unicode = input_bytes.decode(encoding)
+        except LookupError:
+            expected_unicode = ""
+        try:
+            detected_unicode = input_bytes.decode(result["encoding"])
+        except (LookupError, UnicodeDecodeError, TypeError):
+            detected_unicode = ""
+    if result:
+        encoding_match = (result["encoding"] or "").lower() == encoding
+    else:
+        encoding_match = False
+    # Only care about mismatches that would actually result in different
+    # behavior when decoding
+    expected_unicode = normalize("NFKD", expected_unicode)
+    detected_unicode = normalize("NFKD", detected_unicode)
+    if not encoding_match and expected_unicode != detected_unicode:
+        wrapped_expected = "\n".join(textwrap.wrap(expected_unicode, 100)) + "\n"
+        wrapped_detected = "\n".join(textwrap.wrap(detected_unicode, 100)) + "\n"
+        diff = "".join(
+            [
+                line
+                for line in ndiff(
+                    wrapped_expected.splitlines(True), wrapped_detected.splitlines(True)
+                )
+                if not line.startswith(" ")
+            ][:20]
+        )
+        all_encodings = chardet.detect_all(
+            input_bytes, ignore_threshold=True, should_rename_legacy=True
+        )
+    else:
+        diff = ""
+        encoding_match = True
+        all_encodings = [result]
+    assert encoding_match, (
+        f"Expected {encoding}, but got {result} for {file_name}.  First 20 "
+        f"lines of character differences: \n{diff}\n"
+        f"All encodings: {pformat(all_encodings)}"
+    )
+
+
+if HAVE_HYPOTHESIS:
+
+    class JustALengthIssue(Exception):
+        pass
+
+    @pytest.mark.xfail
+    @given(
+        st.text(min_size=1),
+        st.sampled_from(
+            [
+                "ascii",
+                "utf-8",
+                "utf-16",
+                "utf-32",
+                "iso-8859-7",
+                "iso-8859-8",
+                "windows-1255",
+            ]
+        ),
+        st.randoms(),
+    )
+    @settings(max_examples=200)
+    def test_never_fails_to_detect_if_there_is_a_valid_encoding(txt, enc, rnd):
+        try:
+            data = txt.encode(enc)
+        except UnicodeEncodeError:
+            assume(False)
+        detected = chardet.detect(data)["encoding"]
+        if detected is None:
+            with pytest.raises(JustALengthIssue):
+
+                @given(st.text(), random=rnd)
+                @settings(verbosity=Verbosity.quiet, max_examples=50)
+                def string_poisons_following_text(suffix):
+                    try:
+                        extended = (txt + suffix).encode(enc)
+                    except UnicodeEncodeError:
+                        assume(False)
+                    result = chardet.detect(extended)
+                    if result and result["encoding"] is not None:
+                        raise JustALengthIssue()
+
+    @given(
+        st.text(min_size=1),
+        st.sampled_from(
+            [
+                "ascii",
+                "utf-8",
+                "utf-16",
+                "utf-32",
+                "iso-8859-7",
+                "iso-8859-8",
+                "windows-1255",
+            ]
+        ),
+        st.randoms(),
+    )
+    @settings(max_examples=200)
+    def test_detect_all_and_detect_one_should_agree(txt, enc, _):
+        try:
+            data = txt.encode(enc)
+        except UnicodeEncodeError:
+            assume(False)
+        try:
+            result = chardet.detect(data)
+            results = chardet.detect_all(data)
+            assert result["encoding"] == results[0]["encoding"]
+        except Exception as exc:
+            raise RuntimeError(f"{result} != {results}") from exc
diff --git a/contrib/python/chardet/py3/tests/ya.make b/contrib/python/chardet/py3/tests/ya.make
new file mode 100644
index 00000000000..c1f16c7df7f
--- /dev/null
+++ b/contrib/python/chardet/py3/tests/ya.make
@@ -0,0 +1,21 @@
+PY3TEST()
+
+SRCDIR(contrib/python/chardet/py3)
+
+TEST_SRCS(
+    test.py
+)
+
+PEERDIR(
+    contrib/python/chardet
+)
+
+DATA(
+    sbr://3544728585
+)
+
+SIZE(MEDIUM)
+
+NO_LINT()
+
+END()
diff --git a/contrib/python/chardet/py3/ya.make b/contrib/python/chardet/py3/ya.make
new file mode 100644
index 00000000000..b14449360dd
--- /dev/null
+++ b/contrib/python/chardet/py3/ya.make
@@ -0,0 +1,76 @@
+# Generated by devtools/yamaker (pypi).
+
+PY3_LIBRARY()
+
+VERSION(5.2.0)
+
+LICENSE(LGPL-2.1-or-later)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    chardet/__init__.py
+    chardet/__main__.py
+    chardet/big5freq.py
+    chardet/big5prober.py
+    chardet/chardistribution.py
+    chardet/charsetgroupprober.py
+    chardet/charsetprober.py
+    chardet/cli/__init__.py
+    chardet/cli/chardetect.py
+    chardet/codingstatemachine.py
+    chardet/codingstatemachinedict.py
+    chardet/cp949prober.py
+    chardet/enums.py
+    chardet/escprober.py
+    chardet/escsm.py
+    chardet/eucjpprober.py
+    chardet/euckrfreq.py
+    chardet/euckrprober.py
+    chardet/euctwfreq.py
+    chardet/euctwprober.py
+    chardet/gb2312freq.py
+    chardet/gb2312prober.py
+    chardet/hebrewprober.py
+    chardet/jisfreq.py
+    chardet/johabfreq.py
+    chardet/johabprober.py
+    chardet/jpcntx.py
+    chardet/langbulgarianmodel.py
+    chardet/langgreekmodel.py
+    chardet/langhebrewmodel.py
+    chardet/langhungarianmodel.py
+    chardet/langrussianmodel.py
+    chardet/langthaimodel.py
+    chardet/langturkishmodel.py
+    chardet/latin1prober.py
+    chardet/macromanprober.py
+    chardet/mbcharsetprober.py
+    chardet/mbcsgroupprober.py
+    chardet/mbcssm.py
+    chardet/metadata/__init__.py
+    chardet/metadata/languages.py
+    chardet/resultdict.py
+    chardet/sbcharsetprober.py
+    chardet/sbcsgroupprober.py
+    chardet/sjisprober.py
+    chardet/universaldetector.py
+    chardet/utf1632prober.py
+    chardet/utf8prober.py
+    chardet/version.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/chardet/py3/
+    .dist-info/METADATA
+    .dist-info/entry_points.txt
+    .dist-info/top_level.txt
+    chardet/py.typed
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/constantly/py2/.dist-info/METADATA b/contrib/python/constantly/py2/.dist-info/METADATA
new file mode 100644
index 00000000000..dd2e58d4220
--- /dev/null
+++ b/contrib/python/constantly/py2/.dist-info/METADATA
@@ -0,0 +1,39 @@
+Metadata-Version: 2.0
+Name: constantly
+Version: 15.1.0
+Summary: Symbolic constants in Python
+Home-page: https://github.com/twisted/constantly
+Author: Twisted Matrix Labs Developers
+Author-email: UNKNOWN
+License: MIT
+Keywords: constants,enum,twisted
+Platform: UNKNOWN
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+
+Constantly
+==========
+
+A library that provides symbolic constant support.
+It includes collections and constants with text, numeric, and bit flag values.
+Originally ``twisted.python.constants`` from the `Twisted <https://twistedmatrix.com/>`_ project.
+
+
+Tests
+-----
+
+To run tests::
+
+    $ tox
+
+This will run tests on Python 2.7, 3.3, 3.4, and PyPy, as well as doing coverage and pyflakes checks.
+
+
diff --git a/contrib/python/constantly/py2/.dist-info/top_level.txt b/contrib/python/constantly/py2/.dist-info/top_level.txt
new file mode 100644
index 00000000000..605718946ba
--- /dev/null
+++ b/contrib/python/constantly/py2/.dist-info/top_level.txt
@@ -0,0 +1 @@
+constantly
diff --git a/contrib/python/constantly/py2/LICENSE b/contrib/python/constantly/py2/LICENSE
new file mode 100644
index 00000000000..2684131b571
--- /dev/null
+++ b/contrib/python/constantly/py2/LICENSE
@@ -0,0 +1,21 @@
+Copyright (c) 2011-2015 Twisted Matrix Laboratories &
+Individual Contributors (see CREDITS)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/python/constantly/py2/README.rst b/contrib/python/constantly/py2/README.rst
new file mode 100644
index 00000000000..123348ad85f
--- /dev/null
+++ b/contrib/python/constantly/py2/README.rst
@@ -0,0 +1,16 @@
+Constantly
+==========
+
+A library that provides symbolic constant support.
+It includes collections and constants with text, numeric, and bit flag values.
+Originally ``twisted.python.constants`` from the `Twisted <https://twistedmatrix.com/>`_ project.
+
+
+Tests
+-----
+
+To run tests::
+
+    $ tox
+
+This will run tests on Python 2.7, 3.3, 3.4, and PyPy, as well as doing coverage and pyflakes checks.
diff --git a/contrib/python/constantly/py2/constantly/__init__.py b/contrib/python/constantly/py2/constantly/__init__.py
new file mode 100644
index 00000000000..d9e6ec7ad53
--- /dev/null
+++ b/contrib/python/constantly/py2/constantly/__init__.py
@@ -0,0 +1,24 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from constantly._constants import (
+    NamedConstant, Names, ValueConstant, Values, FlagConstant, Flags
+)
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
+
+__author__  = "Twisted Matrix Laboratories"
+__license__ = "MIT"
+__copyright__ = "Copyright 2011-2015 {0}".format(__author__)
+
+
+__all__ = [
+    'NamedConstant',
+    'ValueConstant',
+    'FlagConstant',
+    'Names',
+    'Values',
+    'Flags',
+]
diff --git a/contrib/python/constantly/py2/constantly/_constants.py b/contrib/python/constantly/py2/constantly/_constants.py
new file mode 100644
index 00000000000..44087b6497d
--- /dev/null
+++ b/contrib/python/constantly/py2/constantly/_constants.py
@@ -0,0 +1,500 @@
+# -*- test-case-name: constantly.test.test_constants -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Symbolic constant support, including collections and constants with text,
+numeric, and bit flag values.
+"""
+
+from __future__ import division, absolute_import
+
+__all__ = [
+    'NamedConstant', 'ValueConstant', 'FlagConstant',
+    'Names', 'Values', 'Flags']
+
+from functools import partial
+from itertools import count
+from operator import and_, or_, xor
+
+_unspecified = object()
+_constantOrder = partial(next, count())
+
+
+class _Constant(object):
+    """
+    @ivar _index: A C{int} allocated from a shared counter in order to keep
+        track of the order in which L{_Constant}s are instantiated.
+
+    @ivar name: A C{str} giving the name of this constant; only set once the
+        constant is initialized by L{_ConstantsContainer}.
+
+    @ivar _container: The L{_ConstantsContainer} subclass this constant belongs
+        to; C{None} until the constant is initialized by that subclass.
+    """
+    def __init__(self):
+        self._container = None
+        self._index = _constantOrder()
+
+
+    def __repr__(self):
+        """
+        Return text identifying both which constant this is and which
+        collection it belongs to.
+        """
+        return "<%s=%s>" % (self._container.__name__, self.name)
+
+
+    def __lt__(self, other):
+        """
+        Implements C{<}.  Order is defined by instantiation order.
+
+        @param other: An object.
+
+        @return: C{NotImplemented} if C{other} is not a constant belonging to
+            the same container as this constant, C{True} if this constant is
+            defined before C{other}, otherwise C{False}.
+        """
+        if (
+            not isinstance(other, self.__class__) or
+            not self._container == other._container
+        ):
+            return NotImplemented
+        return self._index < other._index
+
+
+    def __le__(self, other):
+        """
+        Implements C{<=}.  Order is defined by instantiation order.
+
+        @param other: An object.
+
+        @return: C{NotImplemented} if C{other} is not a constant belonging to
+            the same container as this constant, C{True} if this constant is
+            defined before or equal to C{other}, otherwise C{False}.
+        """
+        if (
+            not isinstance(other, self.__class__) or
+            not self._container == other._container
+        ):
+            return NotImplemented
+        return self is other or self._index < other._index
+
+
+    def __gt__(self, other):
+        """
+        Implements C{>}.  Order is defined by instantiation order.
+
+        @param other: An object.
+
+        @return: C{NotImplemented} if C{other} is not a constant belonging to
+            the same container as this constant, C{True} if this constant is
+            defined after C{other}, otherwise C{False}.
+        """
+        if (
+            not isinstance(other, self.__class__) or
+            not self._container == other._container
+        ):
+            return NotImplemented
+        return self._index > other._index
+
+
+    def __ge__(self, other):
+        """
+        Implements C{>=}.  Order is defined by instantiation order.
+
+        @param other: An object.
+
+        @return: C{NotImplemented} if C{other} is not a constant belonging to
+            the same container as this constant, C{True} if this constant is
+            defined after or equal to C{other}, otherwise C{False}.
+        """
+        if (
+            not isinstance(other, self.__class__) or
+            not self._container == other._container
+        ):
+            return NotImplemented
+        return self is other or self._index > other._index
+
+
+    def _realize(self, container, name, value):
+        """
+        Complete the initialization of this L{_Constant}.
+
+        @param container: The L{_ConstantsContainer} subclass this constant is
+            part of.
+
+        @param name: The name of this constant in its container.
+
+        @param value: The value of this constant; not used, as named constants
+            have no value apart from their identity.
+        """
+        self._container = container
+        self.name = name
+
+
+
+class _ConstantsContainerType(type):
+    """
+    L{_ConstantsContainerType} is a metaclass for creating constants container
+    classes.
+    """
+    def __new__(self, name, bases, attributes):
+        """
+        Create a new constants container class.
+
+        If C{attributes} includes a value of C{None} for the C{"_constantType"}
+        key, the new class will not be initialized as a constants container and
+        it will behave as a normal class.
+
+        @param name: The name of the container class.
+        @type name: L{str}
+
+        @param bases: A tuple of the base classes for the new container class.
+        @type bases: L{tuple} of L{_ConstantsContainerType} instances
+
+        @param attributes: The attributes of the new container class, including
+            any constants it is to contain.
+        @type attributes: L{dict}
+        """
+        cls = super(_ConstantsContainerType, self).__new__(
+            self, name, bases, attributes)
+
+        # Only realize constants in concrete _ConstantsContainer subclasses.
+        # Ignore intermediate base classes.
+        constantType = getattr(cls, '_constantType', None)
+        if constantType is None:
+            return cls
+
+        constants = []
+        for (name, descriptor) in attributes.items():
+            if isinstance(descriptor, cls._constantType):
+                if descriptor._container is not None:
+                    raise ValueError(
+                        "Cannot use %s as the value of an attribute on %s" % (
+                            descriptor, cls.__name__))
+                constants.append((descriptor._index, name, descriptor))
+
+        enumerants = {}
+        for (index, enumerant, descriptor) in sorted(constants):
+            value = cls._constantFactory(enumerant, descriptor)
+            descriptor._realize(cls, enumerant, value)
+            enumerants[enumerant] = descriptor
+
+        # Save the dictionary which contains *only* constants (distinct from
+        # any other attributes the application may have given the container)
+        # where the class can use it later (eg for lookupByName).
+        cls._enumerants = enumerants
+
+        return cls
+
+
+
+# In Python3 metaclasses are defined using a C{metaclass} keyword argument in
+# the class definition. This would cause a syntax error in Python2.
+# So we use L{type} to introduce an intermediate base class with the desired
+# metaclass.
+# See:
+# * http://docs.python.org/2/library/functions.html#type
+# * http://docs.python.org/3/reference/datamodel.html#customizing-class-creation
+class _ConstantsContainer(_ConstantsContainerType('', (object,), {})):
+    """
+    L{_ConstantsContainer} is a class with attributes used as symbolic
+    constants.  It is up to subclasses to specify what kind of constants are
+    allowed.
+
+    @cvar _constantType: Specified by a L{_ConstantsContainer} subclass to
+        specify the type of constants allowed by that subclass.
+
+    @cvar _enumerants: A C{dict} mapping the names of constants (eg
+        L{NamedConstant} instances) found in the class definition to those
+        instances.
+    """
+
+    _constantType = None
+
+    def __new__(cls):
+        """
+        Classes representing constants containers are not intended to be
+        instantiated.
+
+        The class object itself is used directly.
+        """
+        raise TypeError("%s may not be instantiated." % (cls.__name__,))
+
+
+    @classmethod
+    def _constantFactory(cls, name, descriptor):
+        """
+        Construct the value for a new constant to add to this container.
+
+        @param name: The name of the constant to create.
+
+        @param descriptor: An instance of a L{_Constant} subclass (eg
+            L{NamedConstant}) which is assigned to C{name}.
+
+        @return: L{NamedConstant} instances have no value apart from identity,
+            so return a meaningless dummy value.
+        """
+        return _unspecified
+
+
+    @classmethod
+    def lookupByName(cls, name):
+        """
+        Retrieve a constant by its name or raise a C{ValueError} if there is no
+        constant associated with that name.
+
+        @param name: A C{str} giving the name of one of the constants defined
+            by C{cls}.
+
+        @raise ValueError: If C{name} is not the name of one of the constants
+            defined by C{cls}.
+
+        @return: The L{NamedConstant} associated with C{name}.
+        """
+        if name in cls._enumerants:
+            return getattr(cls, name)
+        raise ValueError(name)
+
+
+    @classmethod
+    def iterconstants(cls):
+        """
+        Iteration over a L{Names} subclass results in all of the constants it
+        contains.
+
+        @return: an iterator the elements of which are the L{NamedConstant}
+            instances defined in the body of this L{Names} subclass.
+        """
+        constants = cls._enumerants.values()
+
+        return iter(
+            sorted(constants, key=lambda descriptor: descriptor._index))
+
+
+
+class NamedConstant(_Constant):
+    """
+    L{NamedConstant} defines an attribute to be a named constant within a
+    collection defined by a L{Names} subclass.
+
+    L{NamedConstant} is only for use in the definition of L{Names}
+    subclasses.  Do not instantiate L{NamedConstant} elsewhere and do not
+    subclass it.
+    """
+
+
+
+class Names(_ConstantsContainer):
+    """
+    A L{Names} subclass contains constants which differ only in their names and
+    identities.
+    """
+    _constantType = NamedConstant
+
+
+
+class ValueConstant(_Constant):
+    """
+    L{ValueConstant} defines an attribute to be a named constant within a
+    collection defined by a L{Values} subclass.
+
+    L{ValueConstant} is only for use in the definition of L{Values} subclasses.
+    Do not instantiate L{ValueConstant} elsewhere and do not subclass it.
+    """
+    def __init__(self, value):
+        _Constant.__init__(self)
+        self.value = value
+
+
+
+class Values(_ConstantsContainer):
+    """
+    A L{Values} subclass contains constants which are associated with arbitrary
+    values.
+    """
+    _constantType = ValueConstant
+
+    @classmethod
+    def lookupByValue(cls, value):
+        """
+        Retrieve a constant by its value or raise a C{ValueError} if there is
+        no constant associated with that value.
+
+        @param value: The value of one of the constants defined by C{cls}.
+
+        @raise ValueError: If C{value} is not the value of one of the constants
+            defined by C{cls}.
+
+        @return: The L{ValueConstant} associated with C{value}.
+        """
+        for constant in cls.iterconstants():
+            if constant.value == value:
+                return constant
+        raise ValueError(value)
+
+
+
+def _flagOp(op, left, right):
+    """
+    Implement a binary operator for a L{FlagConstant} instance.
+
+    @param op: A two-argument callable implementing the binary operation.  For
+        example, C{operator.or_}.
+
+    @param left: The left-hand L{FlagConstant} instance.
+    @param right: The right-hand L{FlagConstant} instance.
+
+    @return: A new L{FlagConstant} instance representing the result of the
+        operation.
+    """
+    value = op(left.value, right.value)
+    names = op(left.names, right.names)
+    result = FlagConstant()
+    result._realize(left._container, names, value)
+    return result
+
+
+
+class FlagConstant(_Constant):
+    """
+    L{FlagConstant} defines an attribute to be a flag constant within a
+    collection defined by a L{Flags} subclass.
+
+    L{FlagConstant} is only for use in the definition of L{Flags} subclasses.
+    Do not instantiate L{FlagConstant} elsewhere and do not subclass it.
+    """
+    def __init__(self, value=_unspecified):
+        _Constant.__init__(self)
+        self.value = value
+
+
+    def _realize(self, container, names, value):
+        """
+        Complete the initialization of this L{FlagConstant}.
+
+        This implementation differs from other C{_realize} implementations in
+        that a L{FlagConstant} may have several names which apply to it, due to
+        flags being combined with various operators.
+
+        @param container: The L{Flags} subclass this constant is part of.
+
+        @param names: When a single-flag value is being initialized, a C{str}
+            giving the name of that flag.  This is the case which happens when
+            a L{Flags} subclass is being initialized and L{FlagConstant}
+            instances from its body are being realized.  Otherwise, a C{set} of
+            C{str} giving names of all the flags set on this L{FlagConstant}
+            instance.  This is the case when two flags are combined using C{|},
+            for example.
+        """
+        if isinstance(names, str):
+            name = names
+            names = set([names])
+        elif len(names) == 1:
+            (name,) = names
+        else:
+            name = "{" + ",".join(sorted(names)) + "}"
+        _Constant._realize(self, container, name, value)
+        self.value = value
+        self.names = names
+
+
+    def __or__(self, other):
+        """
+        Define C{|} on two L{FlagConstant} instances to create a new
+        L{FlagConstant} instance with all flags set in either instance set.
+        """
+        return _flagOp(or_, self, other)
+
+
+    def __and__(self, other):
+        """
+        Define C{&} on two L{FlagConstant} instances to create a new
+        L{FlagConstant} instance with only flags set in both instances set.
+        """
+        return _flagOp(and_, self, other)
+
+
+    def __xor__(self, other):
+        """
+        Define C{^} on two L{FlagConstant} instances to create a new
+        L{FlagConstant} instance with only flags set on exactly one instance
+        set.
+        """
+        return _flagOp(xor, self, other)
+
+
+    def __invert__(self):
+        """
+        Define C{~} on a L{FlagConstant} instance to create a new
+        L{FlagConstant} instance with all flags not set on this instance set.
+        """
+        result = FlagConstant()
+        result._realize(self._container, set(), 0)
+        for flag in self._container.iterconstants():
+            if flag.value & self.value == 0:
+                result |= flag
+        return result
+
+
+    def __iter__(self):
+        """
+        @return: An iterator of flags set on this instance set.
+        """
+        return (self._container.lookupByName(name) for name in self.names)
+
+
+    def __contains__(self, flag):
+        """
+        @param flag: The flag to test for membership in this instance
+            set.
+
+        @return: C{True} if C{flag} is in this instance set, else
+            C{False}.
+        """
+        # Optimization for testing membership without iteration.
+        return bool(flag & self)
+
+
+    def __nonzero__(self):
+        """
+        @return: C{False} if this flag's value is 0, else C{True}.
+        """
+        return bool(self.value)
+    __bool__ = __nonzero__
+
+
+
+class Flags(Values):
+    """
+    A L{Flags} subclass contains constants which can be combined using the
+    common bitwise operators (C{|}, C{&}, etc) similar to a I{bitvector} from a
+    language like C.
+    """
+    _constantType = FlagConstant
+
+    _value = 1
+
+    @classmethod
+    def _constantFactory(cls, name, descriptor):
+        """
+        For L{FlagConstant} instances with no explicitly defined value, assign
+        the next power of two as its value.
+
+        @param name: The name of the constant to create.
+
+        @param descriptor: An instance of a L{FlagConstant} which is assigned
+            to C{name}.
+
+        @return: Either the value passed to the C{descriptor} constructor, or
+            the next power of 2 value which will be assigned to C{descriptor},
+            relative to the value of the last defined L{FlagConstant}.
+        """
+        if descriptor.value is _unspecified:
+            value = cls._value
+            cls._value <<= 1
+        else:
+            value = descriptor.value
+            cls._value = value << 1
+        return value
diff --git a/contrib/python/constantly/py2/constantly/_version.py b/contrib/python/constantly/py2/constantly/_version.py
new file mode 100644
index 00000000000..2b1beb58516
--- /dev/null
+++ b/contrib/python/constantly/py2/constantly/_version.py
@@ -0,0 +1,21 @@
+
+# This file was generated by 'versioneer.py' (0.15) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+import json
+import sys
+
+version_json = '''
+{
+ "dirty": false,
+ "error": null,
+ "full-revisionid": "c8375a7e3431792ea1b1b44678f3f6878d5e8c9a",
+ "version": "15.1.0"
+}
+'''  # END VERSION_JSON
+
+
+def get_versions():
+    return json.loads(version_json)
diff --git a/contrib/python/constantly/py2/ya.make b/contrib/python/constantly/py2/ya.make
new file mode 100644
index 00000000000..54b3fe0d625
--- /dev/null
+++ b/contrib/python/constantly/py2/ya.make
@@ -0,0 +1,24 @@
+# Generated by devtools/yamaker (pypi).
+
+PY2_LIBRARY()
+
+VERSION(15.1.0)
+
+LICENSE(MIT)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    constantly/__init__.py
+    constantly/_constants.py
+    constantly/_version.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/constantly/py2/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+)
+
+END()
diff --git a/contrib/python/constantly/py3/.dist-info/METADATA b/contrib/python/constantly/py3/.dist-info/METADATA
new file mode 100644
index 00000000000..3c64036c091
--- /dev/null
+++ b/contrib/python/constantly/py3/.dist-info/METADATA
@@ -0,0 +1,59 @@
+Metadata-Version: 2.1
+Name: constantly
+Version: 23.10.4
+Summary: Symbolic constants in Python
+Maintainer: Twisted Matrix Labs Developers
+License: MIT
+Project-URL: Homepage, https://github.com/twisted/constantly
+Keywords: constants,enum,twisted
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+
+Constantly
+==========
+
+A library that provides symbolic constant support.  It includes collections and
+constants with text, numeric, and bit flag values.  Originally
+``twisted.python.constants`` from the `Twisted <https://twistedmatrix.com/>`_
+project.
+
+
+Installing
+----------
+
+constantly is available in `PyPI <https://pypi.org/project/constantly/>`_, and
+can be installed via pip::
+
+  $ pip install constantly
+
+
+Documentation
+-------------------------
+
+Documentation is available at `<https://constantly.readthedocs.io/en/latest/>`_.
+
+
+Tests
+-----
+
+To run tests::
+
+    $ tox
+
+This will run tests on Python 2.7, 3.3, 3.4, and PyPy, as well as doing
+coverage and pyflakes checks.
diff --git a/contrib/python/constantly/py3/.dist-info/top_level.txt b/contrib/python/constantly/py3/.dist-info/top_level.txt
new file mode 100644
index 00000000000..605718946ba
--- /dev/null
+++ b/contrib/python/constantly/py3/.dist-info/top_level.txt
@@ -0,0 +1 @@
+constantly
diff --git a/contrib/python/constantly/py3/LICENSE b/contrib/python/constantly/py3/LICENSE
new file mode 100644
index 00000000000..2684131b571
--- /dev/null
+++ b/contrib/python/constantly/py3/LICENSE
@@ -0,0 +1,21 @@
+Copyright (c) 2011-2015 Twisted Matrix Laboratories &
+Individual Contributors (see CREDITS)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/python/constantly/py3/README.rst b/contrib/python/constantly/py3/README.rst
new file mode 100644
index 00000000000..fdd945a4b99
--- /dev/null
+++ b/contrib/python/constantly/py3/README.rst
@@ -0,0 +1,33 @@
+Constantly
+==========
+
+A library that provides symbolic constant support.  It includes collections and
+constants with text, numeric, and bit flag values.  Originally
+``twisted.python.constants`` from the `Twisted <https://twistedmatrix.com/>`_
+project.
+
+
+Installing
+----------
+
+constantly is available in `PyPI <https://pypi.org/project/constantly/>`_, and
+can be installed via pip::
+
+  $ pip install constantly
+
+
+Documentation
+-------------------------
+
+Documentation is available at `<https://constantly.readthedocs.io/en/latest/>`_.
+
+
+Tests
+-----
+
+To run tests::
+
+    $ tox
+
+This will run tests on Python 2.7, 3.3, 3.4, and PyPy, as well as doing
+coverage and pyflakes checks.
diff --git a/contrib/python/constantly/py3/constantly/__init__.py b/contrib/python/constantly/py3/constantly/__init__.py
new file mode 100644
index 00000000000..ed8adf3d3d0
--- /dev/null
+++ b/contrib/python/constantly/py3/constantly/__init__.py
@@ -0,0 +1,23 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from constantly._constants import (
+    NamedConstant, Names, ValueConstant, Values, FlagConstant, Flags
+)
+
+from . import _version
+__version__ = _version.get_versions()['version']
+
+__author__  = "Twisted Matrix Laboratories"
+__license__ = "MIT"
+__copyright__ = "Copyright 2011-2015 {0}".format(__author__)
+
+
+__all__ = [
+    'NamedConstant',
+    'ValueConstant',
+    'FlagConstant',
+    'Names',
+    'Values',
+    'Flags',
+]
diff --git a/contrib/python/constantly/py3/constantly/_constants.py b/contrib/python/constantly/py3/constantly/_constants.py
new file mode 100644
index 00000000000..f911f2eee1d
--- /dev/null
+++ b/contrib/python/constantly/py3/constantly/_constants.py
@@ -0,0 +1,498 @@
+# -*- test-case-name: constantly.test.test_constants -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Symbolic constant support, including collections and constants with text,
+numeric, and bit flag values.
+"""
+
+from __future__ import division, absolute_import
+
+__all__ = []
+
+from functools import partial
+from itertools import count
+from operator import and_, or_, xor
+
+_unspecified = object()
+_constantOrder = partial(next, count())
+
+
+class _Constant(object):
+    """
+    @ivar _index: A C{int} allocated from a shared counter in order to keep
+        track of the order in which L{_Constant}s are instantiated.
+
+    @ivar name: A C{str} giving the name of this constant; only set once the
+        constant is initialized by L{_ConstantsContainer}.
+
+    @ivar _container: The L{_ConstantsContainer} subclass this constant belongs
+        to; C{None} until the constant is initialized by that subclass.
+    """
+    def __init__(self):
+        self._container = None
+        self._index = _constantOrder()
+
+
+    def __repr__(self):
+        """
+        Return text identifying both which constant this is and which
+        collection it belongs to.
+        """
+        return "<%s=%s>" % (self._container.__name__, self.name)
+
+
+    def __lt__(self, other):
+        """
+        Implements C{<}.  Order is defined by instantiation order.
+
+        @param other: An object.
+
+        @return: C{NotImplemented} if C{other} is not a constant belonging to
+            the same container as this constant, C{True} if this constant is
+            defined before C{other}, otherwise C{False}.
+        """
+        if (
+            not isinstance(other, self.__class__) or
+            not self._container == other._container
+        ):
+            return NotImplemented
+        return self._index < other._index
+
+
+    def __le__(self, other):
+        """
+        Implements C{<=}.  Order is defined by instantiation order.
+
+        @param other: An object.
+
+        @return: C{NotImplemented} if C{other} is not a constant belonging to
+            the same container as this constant, C{True} if this constant is
+            defined before or equal to C{other}, otherwise C{False}.
+        """
+        if (
+            not isinstance(other, self.__class__) or
+            not self._container == other._container
+        ):
+            return NotImplemented
+        return self is other or self._index < other._index
+
+
+    def __gt__(self, other):
+        """
+        Implements C{>}.  Order is defined by instantiation order.
+
+        @param other: An object.
+
+        @return: C{NotImplemented} if C{other} is not a constant belonging to
+            the same container as this constant, C{True} if this constant is
+            defined after C{other}, otherwise C{False}.
+        """
+        if (
+            not isinstance(other, self.__class__) or
+            not self._container == other._container
+        ):
+            return NotImplemented
+        return self._index > other._index
+
+
+    def __ge__(self, other):
+        """
+        Implements C{>=}.  Order is defined by instantiation order.
+
+        @param other: An object.
+
+        @return: C{NotImplemented} if C{other} is not a constant belonging to
+            the same container as this constant, C{True} if this constant is
+            defined after or equal to C{other}, otherwise C{False}.
+        """
+        if (
+            not isinstance(other, self.__class__) or
+            not self._container == other._container
+        ):
+            return NotImplemented
+        return self is other or self._index > other._index
+
+
+    def _realize(self, container, name, value):
+        """
+        Complete the initialization of this L{_Constant}.
+
+        @param container: The L{_ConstantsContainer} subclass this constant is
+            part of.
+
+        @param name: The name of this constant in its container.
+
+        @param value: The value of this constant; not used, as named constants
+            have no value apart from their identity.
+        """
+        self._container = container
+        self.name = name
+
+
+
+class _ConstantsContainerType(type):
+    """
+    L{_ConstantsContainerType} is a metaclass for creating constants container
+    classes.
+    """
+    def __new__(self, name, bases, attributes):
+        """
+        Create a new constants container class.
+
+        If C{attributes} includes a value of C{None} for the C{"_constantType"}
+        key, the new class will not be initialized as a constants container and
+        it will behave as a normal class.
+
+        @param name: The name of the container class.
+        @type name: L{str}
+
+        @param bases: A tuple of the base classes for the new container class.
+        @type bases: L{tuple} of L{_ConstantsContainerType} instances
+
+        @param attributes: The attributes of the new container class, including
+            any constants it is to contain.
+        @type attributes: L{dict}
+        """
+        cls = super(_ConstantsContainerType, self).__new__(
+            self, name, bases, attributes)
+
+        # Only realize constants in concrete _ConstantsContainer subclasses.
+        # Ignore intermediate base classes.
+        constantType = getattr(cls, '_constantType', None)
+        if constantType is None:
+            return cls
+
+        constants = []
+        for (name, descriptor) in attributes.items():
+            if isinstance(descriptor, cls._constantType):
+                if descriptor._container is not None:
+                    raise ValueError(
+                        "Cannot use %s as the value of an attribute on %s" % (
+                            descriptor, cls.__name__))
+                constants.append((descriptor._index, name, descriptor))
+
+        enumerants = {}
+        for (index, enumerant, descriptor) in sorted(constants):
+            value = cls._constantFactory(enumerant, descriptor)
+            descriptor._realize(cls, enumerant, value)
+            enumerants[enumerant] = descriptor
+
+        # Save the dictionary which contains *only* constants (distinct from
+        # any other attributes the application may have given the container)
+        # where the class can use it later (eg for lookupByName).
+        cls._enumerants = enumerants
+
+        return cls
+
+
+
+# In Python3 metaclasses are defined using a C{metaclass} keyword argument in
+# the class definition. This would cause a syntax error in Python2.
+# So we use L{type} to introduce an intermediate base class with the desired
+# metaclass.
+# See:
+# * http://docs.python.org/2/library/functions.html#type
+# * http://docs.python.org/3/reference/datamodel.html#customizing-class-creation
+class _ConstantsContainer(_ConstantsContainerType('', (object,), {})):
+    """
+    L{_ConstantsContainer} is a class with attributes used as symbolic
+    constants.  It is up to subclasses to specify what kind of constants are
+    allowed.
+
+    @cvar _constantType: Specified by a L{_ConstantsContainer} subclass to
+        specify the type of constants allowed by that subclass.
+
+    @cvar _enumerants: A C{dict} mapping the names of constants (eg
+        L{NamedConstant} instances) found in the class definition to those
+        instances.
+    """
+
+    _constantType = None
+
+    def __new__(cls):
+        """
+        Classes representing constants containers are not intended to be
+        instantiated.
+
+        The class object itself is used directly.
+        """
+        raise TypeError("%s may not be instantiated." % (cls.__name__,))
+
+
+    @classmethod
+    def _constantFactory(cls, name, descriptor):
+        """
+        Construct the value for a new constant to add to this container.
+
+        @param name: The name of the constant to create.
+
+        @param descriptor: An instance of a L{_Constant} subclass (eg
+            L{NamedConstant}) which is assigned to C{name}.
+
+        @return: L{NamedConstant} instances have no value apart from identity,
+            so return a meaningless dummy value.
+        """
+        return _unspecified
+
+
+    @classmethod
+    def lookupByName(cls, name):
+        """
+        Retrieve a constant by its name or raise a C{ValueError} if there is no
+        constant associated with that name.
+
+        @param name: A C{str} giving the name of one of the constants defined
+            by C{cls}.
+
+        @raise ValueError: If C{name} is not the name of one of the constants
+            defined by C{cls}.
+
+        @return: The L{NamedConstant} associated with C{name}.
+        """
+        if name in cls._enumerants:
+            return getattr(cls, name)
+        raise ValueError(name)
+
+
+    @classmethod
+    def iterconstants(cls):
+        """
+        Iteration over a L{Names} subclass results in all of the constants it
+        contains.
+
+        @return: an iterator the elements of which are the L{NamedConstant}
+            instances defined in the body of this L{Names} subclass.
+        """
+        constants = cls._enumerants.values()
+
+        return iter(
+            sorted(constants, key=lambda descriptor: descriptor._index))
+
+
+
+class NamedConstant(_Constant):
+    """
+    L{NamedConstant} defines an attribute to be a named constant within a
+    collection defined by a L{Names} subclass.
+
+    L{NamedConstant} is only for use in the definition of L{Names}
+    subclasses.  Do not instantiate L{NamedConstant} elsewhere and do not
+    subclass it.
+    """
+
+
+
+class Names(_ConstantsContainer):
+    """
+    A L{Names} subclass contains constants which differ only in their names and
+    identities.
+    """
+    _constantType = NamedConstant
+
+
+
+class ValueConstant(_Constant):
+    """
+    L{ValueConstant} defines an attribute to be a named constant within a
+    collection defined by a L{Values} subclass.
+
+    L{ValueConstant} is only for use in the definition of L{Values} subclasses.
+    Do not instantiate L{ValueConstant} elsewhere and do not subclass it.
+    """
+    def __init__(self, value):
+        _Constant.__init__(self)
+        self.value = value
+
+
+
+class Values(_ConstantsContainer):
+    """
+    A L{Values} subclass contains constants which are associated with arbitrary
+    values.
+    """
+    _constantType = ValueConstant
+
+    @classmethod
+    def lookupByValue(cls, value):
+        """
+        Retrieve a constant by its value or raise a C{ValueError} if there is
+        no constant associated with that value.
+
+        @param value: The value of one of the constants defined by C{cls}.
+
+        @raise ValueError: If C{value} is not the value of one of the constants
+            defined by C{cls}.
+
+        @return: The L{ValueConstant} associated with C{value}.
+        """
+        for constant in cls.iterconstants():
+            if constant.value == value:
+                return constant
+        raise ValueError(value)
+
+
+
+def _flagOp(op, left, right):
+    """
+    Implement a binary operator for a L{FlagConstant} instance.
+
+    @param op: A two-argument callable implementing the binary operation.  For
+        example, C{operator.or_}.
+
+    @param left: The left-hand L{FlagConstant} instance.
+    @param right: The right-hand L{FlagConstant} instance.
+
+    @return: A new L{FlagConstant} instance representing the result of the
+        operation.
+    """
+    value = op(left.value, right.value)
+    names = op(left.names, right.names)
+    result = FlagConstant()
+    result._realize(left._container, names, value)
+    return result
+
+
+
+class FlagConstant(_Constant):
+    """
+    L{FlagConstant} defines an attribute to be a flag constant within a
+    collection defined by a L{Flags} subclass.
+
+    L{FlagConstant} is only for use in the definition of L{Flags} subclasses.
+    Do not instantiate L{FlagConstant} elsewhere and do not subclass it.
+    """
+    def __init__(self, value=_unspecified):
+        _Constant.__init__(self)
+        self.value = value
+
+
+    def _realize(self, container, names, value):
+        """
+        Complete the initialization of this L{FlagConstant}.
+
+        This implementation differs from other C{_realize} implementations in
+        that a L{FlagConstant} may have several names which apply to it, due to
+        flags being combined with various operators.
+
+        @param container: The L{Flags} subclass this constant is part of.
+
+        @param names: When a single-flag value is being initialized, a C{str}
+            giving the name of that flag.  This is the case which happens when
+            a L{Flags} subclass is being initialized and L{FlagConstant}
+            instances from its body are being realized.  Otherwise, a C{set} of
+            C{str} giving names of all the flags set on this L{FlagConstant}
+            instance.  This is the case when two flags are combined using C{|},
+            for example.
+        """
+        if isinstance(names, str):
+            name = names
+            names = set([names])
+        elif len(names) == 1:
+            (name,) = names
+        else:
+            name = "{" + ",".join(sorted(names)) + "}"
+        _Constant._realize(self, container, name, value)
+        self.value = value
+        self.names = names
+
+
+    def __or__(self, other):
+        """
+        Define C{|} on two L{FlagConstant} instances to create a new
+        L{FlagConstant} instance with all flags set in either instance set.
+        """
+        return _flagOp(or_, self, other)
+
+
+    def __and__(self, other):
+        """
+        Define C{&} on two L{FlagConstant} instances to create a new
+        L{FlagConstant} instance with only flags set in both instances set.
+        """
+        return _flagOp(and_, self, other)
+
+
+    def __xor__(self, other):
+        """
+        Define C{^} on two L{FlagConstant} instances to create a new
+        L{FlagConstant} instance with only flags set on exactly one instance
+        set.
+        """
+        return _flagOp(xor, self, other)
+
+
+    def __invert__(self):
+        """
+        Define C{~} on a L{FlagConstant} instance to create a new
+        L{FlagConstant} instance with all flags not set on this instance set.
+        """
+        result = FlagConstant()
+        result._realize(self._container, set(), 0)
+        for flag in self._container.iterconstants():
+            if flag.value & self.value == 0:
+                result |= flag
+        return result
+
+
+    def __iter__(self):
+        """
+        @return: An iterator of flags set on this instance set.
+        """
+        return (self._container.lookupByName(name) for name in self.names)
+
+
+    def __contains__(self, flag):
+        """
+        @param flag: The flag to test for membership in this instance
+            set.
+
+        @return: C{True} if C{flag} is in this instance set, else
+            C{False}.
+        """
+        # Optimization for testing membership without iteration.
+        return bool(flag & self)
+
+
+    def __nonzero__(self):
+        """
+        @return: C{False} if this flag's value is 0, else C{True}.
+        """
+        return bool(self.value)
+    __bool__ = __nonzero__
+
+
+
+class Flags(Values):
+    """
+    A L{Flags} subclass contains constants which can be combined using the
+    common bitwise operators (C{|}, C{&}, etc) similar to a I{bitvector} from a
+    language like C.
+    """
+    _constantType = FlagConstant
+
+    _value = 1
+
+    @classmethod
+    def _constantFactory(cls, name, descriptor):
+        """
+        For L{FlagConstant} instances with no explicitly defined value, assign
+        the next power of two as its value.
+
+        @param name: The name of the constant to create.
+
+        @param descriptor: An instance of a L{FlagConstant} which is assigned
+            to C{name}.
+
+        @return: Either the value passed to the C{descriptor} constructor, or
+            the next power of 2 value which will be assigned to C{descriptor},
+            relative to the value of the last defined L{FlagConstant}.
+        """
+        if descriptor.value is _unspecified:
+            value = cls._value
+            cls._value <<= 1
+        else:
+            value = descriptor.value
+            cls._value = value << 1
+        return value
diff --git a/contrib/python/constantly/py3/constantly/_version.py b/contrib/python/constantly/py3/constantly/_version.py
new file mode 100644
index 00000000000..58bf9262731
--- /dev/null
+++ b/contrib/python/constantly/py3/constantly/_version.py
@@ -0,0 +1,21 @@
+
+# This file was generated by 'versioneer.py' (0.29) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+import json
+
+version_json = '''
+{
+ "date": "2023-10-28T16:14:45-0700",
+ "dirty": false,
+ "error": null,
+ "full-revisionid": "c63aa51794c314778b5699dd1cec9b3547fe6911",
+ "version": "23.10.4"
+}
+'''  # END VERSION_JSON
+
+
+def get_versions():
+    return json.loads(version_json)
diff --git a/contrib/python/constantly/py3/ya.make b/contrib/python/constantly/py3/ya.make
new file mode 100644
index 00000000000..2d40cadc335
--- /dev/null
+++ b/contrib/python/constantly/py3/ya.make
@@ -0,0 +1,24 @@
+# Generated by devtools/yamaker (pypi).
+
+PY3_LIBRARY()
+
+VERSION(23.10.4)
+
+LICENSE(MIT)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    constantly/__init__.py
+    constantly/_constants.py
+    constantly/_version.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/constantly/py3/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+)
+
+END()
diff --git a/contrib/python/constantly/ya.make b/contrib/python/constantly/ya.make
new file mode 100644
index 00000000000..9302f732d6e
--- /dev/null
+++ b/contrib/python/constantly/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+    PEERDIR(contrib/python/constantly/py2)
+ELSE()
+    PEERDIR(contrib/python/constantly/py3)
+ENDIF()
+
+NO_LINT()
+
+END()
+
+RECURSE(
+    py2
+    py3
+)
diff --git a/contrib/python/hyperlink/py2/.dist-info/METADATA b/contrib/python/hyperlink/py2/.dist-info/METADATA
new file mode 100644
index 00000000000..fc5922ba875
--- /dev/null
+++ b/contrib/python/hyperlink/py2/.dist-info/METADATA
@@ -0,0 +1,38 @@
+Metadata-Version: 2.1
+Name: hyperlink
+Version: 21.0.0
+Summary: A featureful, immutable, and correct URL for Python.
+Home-page: https://github.com/python-hyper/hyperlink
+Author: Mahmoud Hashemi and Glyph Lefkowitz
+Author-email: mahmoud@hatnote.com
+License: MIT
+Platform: any
+Classifier: Topic :: Utilities
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.6
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Requires-Dist: idna (>=2.5)
+Requires-Dist: typing ; python_version < "3.5"
+
+The humble, but powerful, URL runs everything around us. Chances
+are you've used several just to read this text.
+
+Hyperlink is a featureful, pure-Python implementation of the URL, with
+an emphasis on correctness. MIT licensed.
+
+See the docs at http://hyperlink.readthedocs.io.
+
+
diff --git a/contrib/python/hyperlink/py2/.dist-info/top_level.txt b/contrib/python/hyperlink/py2/.dist-info/top_level.txt
new file mode 100644
index 00000000000..81722ce1d88
--- /dev/null
+++ b/contrib/python/hyperlink/py2/.dist-info/top_level.txt
@@ -0,0 +1 @@
+hyperlink
diff --git a/contrib/python/hyperlink/py2/LICENSE b/contrib/python/hyperlink/py2/LICENSE
new file mode 100644
index 00000000000..a73f882ffb0
--- /dev/null
+++ b/contrib/python/hyperlink/py2/LICENSE
@@ -0,0 +1,29 @@
+Copyright (c) 2017
+Glyph Lefkowitz
+Itamar Turner-Trauring
+Jean Paul Calderone
+Adi Roiban
+Amber Hawkie Brown
+Mahmoud Hashemi
+Wilfredo Sanchez Vega
+
+and others that have contributed code to the public domain.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/python/hyperlink/py2/README.md b/contrib/python/hyperlink/py2/README.md
new file mode 100644
index 00000000000..017f9eb88c1
--- /dev/null
+++ b/contrib/python/hyperlink/py2/README.md
@@ -0,0 +1,67 @@
+# Hyperlink
+
+*Cool URLs that don't change.*
+
+<a href="https://hyperlink.readthedocs.io/en/latest/">
+    <img src="https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat" alt="Documentation">
+</a>
+<a href="https://pypi.org/project/hyperlink/">
+    <img src="https://img.shields.io/pypi/v/hyperlink.svg" alt="PyPI">
+</a>
+<a href="http://calver.org">
+    <img src="https://img.shields.io/badge/calver-YY.MINOR.MICRO-22bfda.svg" alt="Calendar Versioning">
+</a>
+<a href="https://pypi.org/project/hyperlink/">
+    <img src="https://img.shields.io/pypi/pyversions/hyperlink.svg" alt="Python Version Compatibility">
+</a>
+<a href="https://https://codecov.io/github/python-hyper/hyperlink?branch=master">
+    <img src="https://codecov.io/github/python-hyper/hyperlink/coverage.svg?branch=master" alt="Code Coverage">
+</a>
+<a href="https://requires.io/github/python-hyper/hyperlink/requirements/?branch=master">
+    <img src="https://requires.io/github/python-hyper/hyperlink/requirements.svg?branch=master" alt="Requirements Status">
+</a>
+
+Hyperlink provides a pure-Python implementation of immutable
+URLs. Based on [RFC 3986][rfc3986] and [3987][rfc3987], the Hyperlink URL
+makes working with both URIs and IRIs easy.
+
+Hyperlink is tested against Python 2.7, 3.4, 3.5, 3.6, 3.7, 3.8, and PyPy.
+
+Full documentation is available on [Read the Docs][docs].
+
+[rfc3986]: https://tools.ietf.org/html/rfc3986
+[rfc3987]: https://tools.ietf.org/html/rfc3987
+[docs]: http://hyperlink.readthedocs.io/en/latest/
+
+## Installation
+
+Hyperlink is a pure-Python package and requires nothing but
+Python. The easiest way to install is with pip:
+
+```
+pip install hyperlink
+```
+
+Then, hyperlink away!
+
+```python
+from hyperlink import URL
+
+url = URL.from_text(u'http://github.com/python-hyper/hyperlink?utm_source=README')
+utm_source = url.get(u'utm_source')
+better_url = url.replace(scheme=u'https', port=443)
+org_url = better_url.click(u'.')
+```
+
+See the full API docs on [Read the Docs][docs].
+
+## More information
+
+Hyperlink would not have been possible without the help of
+[Glyph Lefkowitz](https://glyph.twistedmatrix.com/) and many other
+community members, especially considering that it started as an
+extract from the Twisted networking library. Thanks to them,
+Hyperlink's URL has been production-grade for well over a decade.
+
+Still, should you encounter any issues, do file an issue, or submit a
+pull request.
diff --git a/contrib/python/hyperlink/py2/hyperlink/__init__.py b/contrib/python/hyperlink/py2/hyperlink/__init__.py
new file mode 100644
index 00000000000..f680b01a905
--- /dev/null
+++ b/contrib/python/hyperlink/py2/hyperlink/__init__.py
@@ -0,0 +1,17 @@
+from ._url import (
+    parse,
+    register_scheme,
+    URL,
+    EncodedURL,
+    DecodedURL,
+    URLParseError,
+)
+
+__all__ = (
+    "parse",
+    "register_scheme",
+    "URL",
+    "EncodedURL",
+    "DecodedURL",
+    "URLParseError",
+)
diff --git a/contrib/python/hyperlink/py2/hyperlink/_socket.py b/contrib/python/hyperlink/py2/hyperlink/_socket.py
new file mode 100644
index 00000000000..3bcf89706df
--- /dev/null
+++ b/contrib/python/hyperlink/py2/hyperlink/_socket.py
@@ -0,0 +1,53 @@
+try:
+    from socket import inet_pton
+except ImportError:
+    from typing import TYPE_CHECKING
+
+    if TYPE_CHECKING:  # pragma: no cover
+        pass
+    else:
+        # based on https://gist.github.com/nnemkin/4966028
+        # this code only applies on Windows Python 2.7
+        import ctypes
+        import socket
+
+        class SockAddr(ctypes.Structure):
+            _fields_ = [
+                ("sa_family", ctypes.c_short),
+                ("__pad1", ctypes.c_ushort),
+                ("ipv4_addr", ctypes.c_byte * 4),
+                ("ipv6_addr", ctypes.c_byte * 16),
+                ("__pad2", ctypes.c_ulong),
+            ]
+
+        WSAStringToAddressA = ctypes.windll.ws2_32.WSAStringToAddressA
+        WSAAddressToStringA = ctypes.windll.ws2_32.WSAAddressToStringA
+
+        def inet_pton(address_family, ip_string):
+            # type: (int, str) -> bytes
+            addr = SockAddr()
+            ip_string_bytes = ip_string.encode("ascii")
+            addr.sa_family = address_family
+            addr_size = ctypes.c_int(ctypes.sizeof(addr))
+
+            try:
+                attribute, size = {
+                    socket.AF_INET: ("ipv4_addr", 4),
+                    socket.AF_INET6: ("ipv6_addr", 16),
+                }[address_family]
+            except KeyError:
+                raise socket.error("unknown address family")
+
+            if (
+                WSAStringToAddressA(
+                    ip_string_bytes,
+                    address_family,
+                    None,
+                    ctypes.byref(addr),
+                    ctypes.byref(addr_size),
+                )
+                != 0
+            ):
+                raise socket.error(ctypes.FormatError())
+
+            return ctypes.string_at(getattr(addr, attribute), size)
diff --git a/contrib/python/hyperlink/py2/hyperlink/_url.py b/contrib/python/hyperlink/py2/hyperlink/_url.py
new file mode 100644
index 00000000000..be69baf696a
--- /dev/null
+++ b/contrib/python/hyperlink/py2/hyperlink/_url.py
@@ -0,0 +1,2448 @@
+# -*- coding: utf-8 -*-
+u"""Hyperlink provides Pythonic URL parsing, construction, and rendering.
+
+Usage is straightforward::
+
+   >>> import hyperlink
+   >>> url = hyperlink.parse(u'http://github.com/mahmoud/hyperlink?utm_source=docs')
+   >>> url.host
+   u'github.com'
+   >>> secure_url = url.replace(scheme=u'https')
+   >>> secure_url.get('utm_source')[0]
+   u'docs'
+
+Hyperlink's API centers on the :class:`DecodedURL` type, which wraps
+the lower-level :class:`URL`, both of which can be returned by the
+:func:`parse()` convenience function.
+
+"""  # noqa: E501
+
+import re
+import sys
+import string
+import socket
+from socket import AF_INET, AF_INET6
+
+try:
+    from socket import AddressFamily
+except ImportError:
+    AddressFamily = int  # type: ignore[assignment,misc]
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Text,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+)
+from unicodedata import normalize
+from ._socket import inet_pton
+
+try:
+    from collections.abc import Mapping as MappingABC
+except ImportError:  # Python 2
+    from collections import Mapping as MappingABC
+
+from idna import encode as idna_encode, decode as idna_decode
+
+
+PY2 = sys.version_info[0] == 2
+try:
+    unichr
+except NameError:  # Py3
+    unichr = chr  # type: Callable[[int], Text]
+NoneType = type(None)  # type: Type[None]
+QueryPairs = Tuple[Tuple[Text, Optional[Text]], ...]  # internal representation
+QueryParameters = Union[
+    Mapping[Text, Optional[Text]],
+    QueryPairs,
+    Sequence[Tuple[Text, Optional[Text]]],
+]
+T = TypeVar("T")
+
+
+# from boltons.typeutils
+def make_sentinel(name="_MISSING", var_name=""):
+    # type: (str, str) -> object
+    """Creates and returns a new **instance** of a new class, suitable for
+    usage as a "sentinel", a kind of singleton often used to indicate
+    a value is missing when ``None`` is a valid input.
+
+    Args:
+        name: Name of the Sentinel
+        var_name: Set this name to the name of the variable in its respective
+            module enable pickle-ability.
+
+    >>> make_sentinel(var_name='_MISSING')
+    _MISSING
+
+    The most common use cases here in boltons are as default values
+    for optional function arguments, partly because of its
+    less-confusing appearance in automatically generated
+    documentation. Sentinels also function well as placeholders in queues
+    and linked lists.
+
+    .. note::
+
+        By design, additional calls to ``make_sentinel`` with the same
+        values will not produce equivalent objects.
+
+        >>> make_sentinel('TEST') == make_sentinel('TEST')
+        False
+        >>> type(make_sentinel('TEST')) == type(make_sentinel('TEST'))
+        False
+    """
+
+    class Sentinel(object):
+        def __init__(self):
+            # type: () -> None
+            self.name = name
+            self.var_name = var_name
+
+        def __repr__(self):
+            # type: () -> str
+            if self.var_name:
+                return self.var_name
+            return "%s(%r)" % (self.__class__.__name__, self.name)
+
+        if var_name:
+            # superclass type hints don't allow str return type, but it is
+            # allowed in the docs, hence the ignore[override] below
+            def __reduce__(self):
+                # type: () -> str
+                return self.var_name
+
+        def __nonzero__(self):
+            # type: () -> bool
+            return False
+
+        __bool__ = __nonzero__
+
+    return Sentinel()
+
+
+_unspecified = _UNSET = make_sentinel("_UNSET")  # type: Any
+
+
+# RFC 3986 Section 2.3, Unreserved URI Characters
+#   https://tools.ietf.org/html/rfc3986#section-2.3
+_UNRESERVED_CHARS = frozenset(
+    "~-._0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
+)
+
+
+# URL parsing regex (based on RFC 3986 Appendix B, with modifications)
+_URL_RE = re.compile(
+    r"^((?P<scheme>[^:/?#]+):)?"
+    r"((?P<_netloc_sep>//)"
+    r"(?P<authority>[^/?#]*))?"
+    r"(?P<path>[^?#]*)"
+    r"(\?(?P<query>[^#]*))?"
+    r"(#(?P<fragment>.*))?$"
+)
+_SCHEME_RE = re.compile(r"^[a-zA-Z0-9+-.]*$")
+_AUTHORITY_RE = re.compile(
+    r"^(?:(?P<userinfo>[^@/?#]*)@)?"
+    r"(?P<host>"
+    r"(?:\[(?P<ipv6_host>[^[\]/?#]*)\])"
+    r"|(?P<plain_host>[^:/?#[\]]*)"
+    r"|(?P<bad_host>.*?))?"
+    r"(?::(?P<port>.*))?$"
+)
+
+
+_HEX_CHAR_MAP = dict(
+    [
+        ((a + b).encode("ascii"), unichr(int(a + b, 16)).encode("charmap"))
+        for a in string.hexdigits
+        for b in string.hexdigits
+    ]
+)
+_ASCII_RE = re.compile("([\x00-\x7f]+)")
+
+# RFC 3986 section 2.2, Reserved Characters
+#   https://tools.ietf.org/html/rfc3986#section-2.2
+_GEN_DELIMS = frozenset(u":/?#[]@")
+_SUB_DELIMS = frozenset(u"!$&'()*+,;=")
+_ALL_DELIMS = _GEN_DELIMS | _SUB_DELIMS
+
+_USERINFO_SAFE = _UNRESERVED_CHARS | _SUB_DELIMS | set(u"%")
+_USERINFO_DELIMS = _ALL_DELIMS - _USERINFO_SAFE
+_PATH_SAFE = _USERINFO_SAFE | set(u":@")
+_PATH_DELIMS = _ALL_DELIMS - _PATH_SAFE
+_SCHEMELESS_PATH_SAFE = _PATH_SAFE - set(":")
+_SCHEMELESS_PATH_DELIMS = _ALL_DELIMS - _SCHEMELESS_PATH_SAFE
+_FRAGMENT_SAFE = _UNRESERVED_CHARS | _PATH_SAFE | set(u"/?")
+_FRAGMENT_DELIMS = _ALL_DELIMS - _FRAGMENT_SAFE
+_QUERY_VALUE_SAFE = _UNRESERVED_CHARS | _FRAGMENT_SAFE - set(u"&")
+_QUERY_VALUE_DELIMS = _ALL_DELIMS - _QUERY_VALUE_SAFE
+_QUERY_KEY_SAFE = _UNRESERVED_CHARS | _QUERY_VALUE_SAFE - set(u"=")
+_QUERY_KEY_DELIMS = _ALL_DELIMS - _QUERY_KEY_SAFE
+
+
+def _make_decode_map(delims, allow_percent=False):
+    # type: (Iterable[Text], bool) -> Mapping[bytes, bytes]
+    ret = dict(_HEX_CHAR_MAP)
+    if not allow_percent:
+        delims = set(delims) | set([u"%"])
+    for delim in delims:
+        _hexord = "{0:02X}".format(ord(delim)).encode("ascii")
+        _hexord_lower = _hexord.lower()
+        ret.pop(_hexord)
+        if _hexord != _hexord_lower:
+            ret.pop(_hexord_lower)
+    return ret
+
+
+def _make_quote_map(safe_chars):
+    # type: (Iterable[Text]) -> Mapping[Union[int, Text], Text]
+    ret = {}  # type: Dict[Union[int, Text], Text]
+    # v is included in the dict for py3 mostly, because bytestrings
+    # are iterables of ints, of course!
+    for i, v in zip(range(256), range(256)):
+        c = chr(v)
+        if c in safe_chars:
+            ret[c] = ret[v] = c
+        else:
+            ret[c] = ret[v] = "%{0:02X}".format(i)
+    return ret
+
+
+_USERINFO_PART_QUOTE_MAP = _make_quote_map(_USERINFO_SAFE)
+_USERINFO_DECODE_MAP = _make_decode_map(_USERINFO_DELIMS)
+_PATH_PART_QUOTE_MAP = _make_quote_map(_PATH_SAFE)
+_SCHEMELESS_PATH_PART_QUOTE_MAP = _make_quote_map(_SCHEMELESS_PATH_SAFE)
+_PATH_DECODE_MAP = _make_decode_map(_PATH_DELIMS)
+_QUERY_KEY_QUOTE_MAP = _make_quote_map(_QUERY_KEY_SAFE)
+_QUERY_KEY_DECODE_MAP = _make_decode_map(_QUERY_KEY_DELIMS)
+_QUERY_VALUE_QUOTE_MAP = _make_quote_map(_QUERY_VALUE_SAFE)
+_QUERY_VALUE_DECODE_MAP = _make_decode_map(_QUERY_VALUE_DELIMS)
+_FRAGMENT_QUOTE_MAP = _make_quote_map(_FRAGMENT_SAFE)
+_FRAGMENT_DECODE_MAP = _make_decode_map(_FRAGMENT_DELIMS)
+_UNRESERVED_QUOTE_MAP = _make_quote_map(_UNRESERVED_CHARS)
+_UNRESERVED_DECODE_MAP = dict(
+    [
+        (k, v)
+        for k, v in _HEX_CHAR_MAP.items()
+        if v.decode("ascii", "replace") in _UNRESERVED_CHARS
+    ]
+)
+
+_ROOT_PATHS = frozenset(((), (u"",)))
+
+
+def _encode_reserved(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """A very comprehensive percent encoding for encoding all
+    delimiters. Used for arguments to DecodedURL, where a % means a
+    percent sign, and not the character used by URLs for escaping
+    bytes.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_UNRESERVED_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [
+            _UNRESERVED_QUOTE_MAP[t] if t in _UNRESERVED_CHARS else t
+            for t in text
+        ]
+    )
+
+
+def _encode_path_part(text, maximal=True):
+    # type: (Text, bool) -> Text
+    "Percent-encode a single segment of a URL path."
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_PATH_PART_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [_PATH_PART_QUOTE_MAP[t] if t in _PATH_DELIMS else t for t in text]
+    )
+
+
+def _encode_schemeless_path_part(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """Percent-encode the first segment of a URL path for a URL without a
+    scheme specified.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_SCHEMELESS_PATH_PART_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [
+            _SCHEMELESS_PATH_PART_QUOTE_MAP[t]
+            if t in _SCHEMELESS_PATH_DELIMS
+            else t
+            for t in text
+        ]
+    )
+
+
+def _encode_path_parts(
+    text_parts,  # type: Sequence[Text]
+    rooted=False,  # type: bool
+    has_scheme=True,  # type: bool
+    has_authority=True,  # type: bool
+    maximal=True,  # type: bool
+):
+    # type: (...) -> Sequence[Text]
+    """
+    Percent-encode a tuple of path parts into a complete path.
+
+    Setting *maximal* to False percent-encodes only the reserved
+    characters that are syntactically necessary for serialization,
+    preserving any IRI-style textual data.
+
+    Leaving *maximal* set to its default True percent-encodes
+    everything required to convert a portion of an IRI to a portion of
+    a URI.
+
+    RFC 3986 3.3:
+
+       If a URI contains an authority component, then the path component
+       must either be empty or begin with a slash ("/") character.  If a URI
+       does not contain an authority component, then the path cannot begin
+       with two slash characters ("//").  In addition, a URI reference
+       (Section 4.1) may be a relative-path reference, in which case the
+       first path segment cannot contain a colon (":") character.
+    """
+    if not text_parts:
+        return ()
+    if rooted:
+        text_parts = (u"",) + tuple(text_parts)
+    # elif has_authority and text_parts:
+    #     raise Exception('see rfc above')  # TODO: too late to fail like this?
+    encoded_parts = []  # type: List[Text]
+    if has_scheme:
+        encoded_parts = [
+            _encode_path_part(part, maximal=maximal) if part else part
+            for part in text_parts
+        ]
+    else:
+        encoded_parts = [_encode_schemeless_path_part(text_parts[0])]
+        encoded_parts.extend(
+            [
+                _encode_path_part(part, maximal=maximal) if part else part
+                for part in text_parts[1:]
+            ]
+        )
+    return tuple(encoded_parts)
+
+
+def _encode_query_key(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """
+    Percent-encode a single query string key or value.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_QUERY_KEY_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [_QUERY_KEY_QUOTE_MAP[t] if t in _QUERY_KEY_DELIMS else t for t in text]
+    )
+
+
+def _encode_query_value(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """
+    Percent-encode a single query string key or value.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_QUERY_VALUE_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [
+            _QUERY_VALUE_QUOTE_MAP[t] if t in _QUERY_VALUE_DELIMS else t
+            for t in text
+        ]
+    )
+
+
+def _encode_fragment_part(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """Quote the fragment part of the URL. Fragments don't have
+    subdelimiters, so the whole URL fragment can be passed.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_FRAGMENT_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [_FRAGMENT_QUOTE_MAP[t] if t in _FRAGMENT_DELIMS else t for t in text]
+    )
+
+
+def _encode_userinfo_part(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """Quote special characters in either the username or password
+    section of the URL.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_USERINFO_PART_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [
+            _USERINFO_PART_QUOTE_MAP[t] if t in _USERINFO_DELIMS else t
+            for t in text
+        ]
+    )
+
+
+# This port list painstakingly curated by hand searching through
+# https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml
+# and
+# https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml
+SCHEME_PORT_MAP = {
+    "acap": 674,
+    "afp": 548,
+    "dict": 2628,
+    "dns": 53,
+    "file": None,
+    "ftp": 21,
+    "git": 9418,
+    "gopher": 70,
+    "http": 80,
+    "https": 443,
+    "imap": 143,
+    "ipp": 631,
+    "ipps": 631,
+    "irc": 194,
+    "ircs": 6697,
+    "ldap": 389,
+    "ldaps": 636,
+    "mms": 1755,
+    "msrp": 2855,
+    "msrps": None,
+    "mtqp": 1038,
+    "nfs": 111,
+    "nntp": 119,
+    "nntps": 563,
+    "pop": 110,
+    "prospero": 1525,
+    "redis": 6379,
+    "rsync": 873,
+    "rtsp": 554,
+    "rtsps": 322,
+    "rtspu": 5005,
+    "sftp": 22,
+    "smb": 445,
+    "snmp": 161,
+    "ssh": 22,
+    "steam": None,
+    "svn": 3690,
+    "telnet": 23,
+    "ventrilo": 3784,
+    "vnc": 5900,
+    "wais": 210,
+    "ws": 80,
+    "wss": 443,
+    "xmpp": None,
+}
+
+# This list of schemes that don't use authorities is also from the link above.
+NO_NETLOC_SCHEMES = set(
+    [
+        "urn",
+        "about",
+        "bitcoin",
+        "blob",
+        "data",
+        "geo",
+        "magnet",
+        "mailto",
+        "news",
+        "pkcs11",
+        "sip",
+        "sips",
+        "tel",
+    ]
+)
+# As of Mar 11, 2017, there were 44 netloc schemes, and 13 non-netloc
+
+NO_QUERY_PLUS_SCHEMES = set()
+
+
+def register_scheme(
+    text, uses_netloc=True, default_port=None, query_plus_is_space=True
+):
+    # type: (Text, bool, Optional[int], bool) -> None
+    """Registers new scheme information, resulting in correct port and
+    slash behavior from the URL object. There are dozens of standard
+    schemes preregistered, so this function is mostly meant for
+    proprietary internal customizations or stopgaps on missing
+    standards information. If a scheme seems to be missing, please
+    `file an issue`_!
+
+    Args:
+        text: A string representation of the scheme.
+            (the 'http' in 'http://hatnote.com')
+        uses_netloc: Does the scheme support specifying a
+            network host? For instance, "http" does, "mailto" does
+            not. Defaults to True.
+        default_port: The default port, if any, for
+            netloc-using schemes.
+        query_plus_is_space: If true, a "+" in the query string should be
+            decoded as a space by DecodedURL.
+
+    .. _file an issue: https://github.com/mahmoud/hyperlink/issues
+    """
+    text = text.lower()
+    if default_port is not None:
+        try:
+            default_port = int(default_port)
+        except (ValueError, TypeError):
+            raise ValueError(
+                "default_port expected integer or None, not %r"
+                % (default_port,)
+            )
+
+    if uses_netloc is True:
+        SCHEME_PORT_MAP[text] = default_port
+    elif uses_netloc is False:
+        if default_port is not None:
+            raise ValueError(
+                "unexpected default port while specifying"
+                " non-netloc scheme: %r" % default_port
+            )
+        NO_NETLOC_SCHEMES.add(text)
+    else:
+        raise ValueError("uses_netloc expected bool, not: %r" % uses_netloc)
+
+    if not query_plus_is_space:
+        NO_QUERY_PLUS_SCHEMES.add(text)
+
+    return
+
+
+def scheme_uses_netloc(scheme, default=None):
+    # type: (Text, Optional[bool]) -> Optional[bool]
+    """Whether or not a URL uses :code:`:` or :code:`://` to separate the
+    scheme from the rest of the URL depends on the scheme's own
+    standard definition. There is no way to infer this behavior
+    from other parts of the URL. A scheme either supports network
+    locations or it does not.
+
+    The URL type's approach to this is to check for explicitly
+    registered schemes, with common schemes like HTTP
+    preregistered. This is the same approach taken by
+    :mod:`urlparse`.
+
+    URL adds two additional heuristics if the scheme as a whole is
+    not registered. First, it attempts to check the subpart of the
+    scheme after the last ``+`` character. This adds intuitive
+    behavior for schemes like ``git+ssh``. Second, if a URL with
+    an unrecognized scheme is loaded, it will maintain the
+    separator it sees.
+    """
+    if not scheme:
+        return False
+    scheme = scheme.lower()
+    if scheme in SCHEME_PORT_MAP:
+        return True
+    if scheme in NO_NETLOC_SCHEMES:
+        return False
+    if scheme.split("+")[-1] in SCHEME_PORT_MAP:
+        return True
+    return default
+
+
+class URLParseError(ValueError):
+    """Exception inheriting from :exc:`ValueError`, raised when failing to
+    parse a URL. Mostly raised on invalid ports and IPv6 addresses.
+    """
+
+    pass
+
+
+def _optional(argument, default):
+    # type: (Any, Any) -> Any
+    if argument is _UNSET:
+        return default
+    else:
+        return argument
+
+
+def _typecheck(name, value, *types):
+    # type: (Text, T, Type[Any]) -> T
+    """
+    Check that the given *value* is one of the given *types*, or raise an
+    exception describing the problem using *name*.
+    """
+    if not types:
+        raise ValueError("expected one or more types, maybe use _textcheck?")
+    if not isinstance(value, types):
+        raise TypeError(
+            "expected %s for %s, got %r"
+            % (" or ".join([t.__name__ for t in types]), name, value)
+        )
+    return value
+
+
+def _textcheck(name, value, delims=frozenset(), nullable=False):
+    # type: (Text, T, Iterable[Text], bool) -> T
+    if not isinstance(value, Text):
+        if nullable and value is None:
+            # used by query string values
+            return value  # type: ignore[unreachable]
+        else:
+            str_name = "unicode" if PY2 else "str"
+            exp = str_name + " or NoneType" if nullable else str_name
+            raise TypeError("expected %s for %s, got %r" % (exp, name, value))
+    if delims and set(value) & set(delims):  # TODO: test caching into regexes
+        raise ValueError(
+            "one or more reserved delimiters %s present in %s: %r"
+            % ("".join(delims), name, value)
+        )
+    return value  # type: ignore[return-value] # T vs. Text
+
+
+def iter_pairs(iterable):
+    # type: (Iterable[Any]) -> Iterator[Any]
+    """
+    Iterate over the (key, value) pairs in ``iterable``.
+
+    This handles dictionaries sensibly, and falls back to assuming the
+    iterable yields (key, value) pairs. This behaviour is similar to
+    what Python's ``dict()`` constructor does.
+    """
+    if isinstance(iterable, MappingABC):
+        iterable = iterable.items()
+    return iter(iterable)
+
+
+def _decode_unreserved(text, normalize_case=False, encode_stray_percents=False):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_UNRESERVED_DECODE_MAP,
+    )
+
+
+def _decode_userinfo_part(
+    text, normalize_case=False, encode_stray_percents=False
+):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_USERINFO_DECODE_MAP,
+    )
+
+
+def _decode_path_part(text, normalize_case=False, encode_stray_percents=False):
+    # type: (Text, bool, bool) -> Text
+    """
+    >>> _decode_path_part(u'%61%77%2f%7a')
+    u'aw%2fz'
+    >>> _decode_path_part(u'%61%77%2f%7a', normalize_case=True)
+    u'aw%2Fz'
+    """
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_PATH_DECODE_MAP,
+    )
+
+
+def _decode_query_key(text, normalize_case=False, encode_stray_percents=False):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_QUERY_KEY_DECODE_MAP,
+    )
+
+
+def _decode_query_value(
+    text, normalize_case=False, encode_stray_percents=False
+):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_QUERY_VALUE_DECODE_MAP,
+    )
+
+
+def _decode_fragment_part(
+    text, normalize_case=False, encode_stray_percents=False
+):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_FRAGMENT_DECODE_MAP,
+    )
+
+
+def _percent_decode(
+    text,  # type: Text
+    normalize_case=False,  # type: bool
+    subencoding="utf-8",  # type: Text
+    raise_subencoding_exc=False,  # type: bool
+    encode_stray_percents=False,  # type: bool
+    _decode_map=_HEX_CHAR_MAP,  # type: Mapping[bytes, bytes]
+):
+    # type: (...) -> Text
+    """Convert percent-encoded text characters to their normal,
+    human-readable equivalents.
+
+    All characters in the input text must be encodable by
+    *subencoding*. All special characters underlying the values in the
+    percent-encoding must be decodable as *subencoding*. If a
+    non-*subencoding*-valid string is passed, the original text is
+    returned with no changes applied.
+
+    Only called by field-tailored variants, e.g.,
+    :func:`_decode_path_part`, as every percent-encodable part of the
+    URL has characters which should not be percent decoded.
+
+    >>> _percent_decode(u'abc%20def')
+    u'abc def'
+
+    Args:
+        text: Text with percent-encoding present.
+        normalize_case: Whether undecoded percent segments, such as encoded
+            delimiters, should be uppercased, per RFC 3986 Section 2.1.
+            See :func:`_decode_path_part` for an example.
+        subencoding: The name of the encoding underlying the percent-encoding.
+        raise_subencoding_exc: Whether an error in decoding the bytes
+            underlying the percent-decoding should be raised.
+
+    Returns:
+        Text: The percent-decoded version of *text*, decoded by *subencoding*.
+    """
+    try:
+        quoted_bytes = text.encode(subencoding)
+    except UnicodeEncodeError:
+        return text
+
+    bits = quoted_bytes.split(b"%")
+    if len(bits) == 1:
+        return text
+
+    res = [bits[0]]
+    append = res.append
+
+    for item in bits[1:]:
+        hexpair, rest = item[:2], item[2:]
+        try:
+            append(_decode_map[hexpair])
+            append(rest)
+        except KeyError:
+            pair_is_hex = hexpair in _HEX_CHAR_MAP
+            if pair_is_hex or not encode_stray_percents:
+                append(b"%")
+            else:
+                # if it's undecodable, treat as a real percent sign,
+                # which is reserved (because it wasn't in the
+                # context-aware _decode_map passed in), and should
+                # stay in an encoded state.
+                append(b"%25")
+            if normalize_case and pair_is_hex:
+                append(hexpair.upper())
+                append(rest)
+            else:
+                append(item)
+
+    unquoted_bytes = b"".join(res)
+
+    try:
+        return unquoted_bytes.decode(subencoding)
+    except UnicodeDecodeError:
+        if raise_subencoding_exc:
+            raise
+        return text
+
+
+def _decode_host(host):
+    # type: (Text) -> Text
+    """Decode a host from ASCII-encodable text to IDNA-decoded text. If
+    the host text is not ASCII, it is returned unchanged, as it is
+    presumed that it is already IDNA-decoded.
+
+    Some technical details: _decode_host is built on top of the "idna"
+    package, which has some quirks:
+
+    Capital letters are not valid IDNA2008. The idna package will
+    raise an exception like this on capital letters:
+
+    > idna.core.InvalidCodepoint: Codepoint U+004B at position 1 ... not allowed
+
+    However, if a segment of a host (i.e., something in
+    url.host.split('.')) is already ASCII, idna doesn't perform its
+    usual checks. In fact, for capital letters it automatically
+    lowercases them.
+
+    This check and some other functionality can be bypassed by passing
+    uts46=True to idna.encode/decode. This allows a more permissive and
+    convenient interface. So far it seems like the balanced approach.
+
+    Example output (from idna==2.6):
+
+    >> idna.encode(u'mahmöud.io')
+    'xn--mahmud-zxa.io'
+    >> idna.encode(u'Mahmöud.io')
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 355, in encode
+        result.append(alabel(label))
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 276, in alabel
+        check_label(label)
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 253, in check_label
+        raise InvalidCodepoint('Codepoint {0} at position {1} of {2} not allowed'.format(_unot(cp_value), pos+1, repr(label)))
+    idna.core.InvalidCodepoint: Codepoint U+004D at position 1 of u'Mahm\xf6ud' not allowed
+    >> idna.encode(u'Mahmoud.io')
+    'Mahmoud.io'
+
+    # Similar behavior for decodes below
+    >> idna.decode(u'Mahmoud.io')
+    u'mahmoud.io
+    >> idna.decode(u'Méhmoud.io', uts46=True)
+    u'm\xe9hmoud.io'
+    """  # noqa: E501
+    if not host:
+        return u""
+    try:
+        host_bytes = host.encode("ascii")
+    except UnicodeEncodeError:
+        host_text = host
+    else:
+        try:
+            host_text = idna_decode(host_bytes, uts46=True)
+        except ValueError:
+            # only reached on "narrow" (UCS-2) Python builds <3.4, see #7
+            # NOTE: not going to raise here, because there's no
+            # ambiguity in the IDNA, and the host is still
+            # technically usable
+            host_text = host
+    return host_text
+
+
+def _resolve_dot_segments(path):
+    # type: (Sequence[Text]) -> Sequence[Text]
+    """Normalize the URL path by resolving segments of '.' and '..'. For
+    more details, see `RFC 3986 section 5.2.4, Remove Dot Segments`_.
+
+    Args:
+       path: sequence of path segments in text form
+
+    Returns:
+       A new sequence of path segments with the '.' and '..' elements removed
+           and resolved.
+
+    .. _RFC 3986 section 5.2.4, Remove Dot Segments: https://tools.ietf.org/html/rfc3986#section-5.2.4
+    """  # noqa: E501
+    segs = []  # type: List[Text]
+
+    for seg in path:
+        if seg == u".":
+            pass
+        elif seg == u"..":
+            if segs:
+                segs.pop()
+        else:
+            segs.append(seg)
+
+    if list(path[-1:]) in ([u"."], [u".."]):
+        segs.append(u"")
+
+    return segs
+
+
+def parse_host(host):
+    # type: (Text) -> Tuple[Optional[AddressFamily], Text]
+    """Parse the host into a tuple of ``(family, host)``, where family
+    is the appropriate :mod:`socket` module constant when the host is
+    an IP address. Family is ``None`` when the host is not an IP.
+
+    Will raise :class:`URLParseError` on invalid IPv6 constants.
+
+    Returns:
+        family (socket constant or None), host (string)
+
+    >>> import socket
+    >>> parse_host('googlewebsite.com') == (None, 'googlewebsite.com')
+    True
+    >>> parse_host('::1') == (socket.AF_INET6, '::1')
+    True
+    >>> parse_host('192.168.1.1') == (socket.AF_INET, '192.168.1.1')
+    True
+    """
+    if not host:
+        return None, u""
+
+    if u":" in host:
+        try:
+            inet_pton(AF_INET6, host)
+        except socket.error as se:
+            raise URLParseError("invalid IPv6 host: %r (%r)" % (host, se))
+        except UnicodeEncodeError:
+            pass  # TODO: this can't be a real host right?
+        else:
+            family = AF_INET6  # type: Optional[AddressFamily]
+    else:
+        try:
+            inet_pton(AF_INET, host)
+        except (socket.error, UnicodeEncodeError):
+            family = None  # not an IP
+        else:
+            family = AF_INET
+
+    return family, host
+
+
+class URL(object):
+    r"""From blogs to billboards, URLs are so common, that it's easy to
+    overlook their complexity and power. With hyperlink's
+    :class:`URL` type, working with URLs doesn't have to be hard.
+
+    URLs are made of many parts. Most of these parts are officially
+    named in `RFC 3986`_ and this diagram may prove handy in identifying
+    them::
+
+       foo://user:pass@example.com:8042/over/there?name=ferret#nose
+       \_/   \_______/ \_________/ \__/\_________/ \_________/ \__/
+        |        |          |        |      |           |        |
+      scheme  userinfo     host     port   path       query   fragment
+
+    While :meth:`~URL.from_text` is used for parsing whole URLs, the
+    :class:`URL` constructor builds a URL from the individual
+    components, like so::
+
+        >>> from hyperlink import URL
+        >>> url = URL(scheme=u'https', host=u'example.com', path=[u'hello', u'world'])
+        >>> print(url.to_text())
+        https://example.com/hello/world
+
+    The constructor runs basic type checks. All strings are expected
+    to be text (:class:`str` in Python 3, :class:`unicode` in Python 2). All
+    arguments are optional, defaulting to appropriately empty values. A full
+    list of constructor arguments is below.
+
+    Args:
+        scheme: The text name of the scheme.
+        host: The host portion of the network location
+        port: The port part of the network location. If ``None`` or no port is
+            passed, the port will default to the default port of the scheme, if
+            it is known. See the ``SCHEME_PORT_MAP`` and
+            :func:`register_default_port` for more info.
+        path: A tuple of strings representing the slash-separated parts of the
+            path, each percent-encoded.
+        query: The query parameters, as a dictionary or as an sequence of
+            percent-encoded key-value pairs.
+        fragment: The fragment part of the URL.
+        rooted: A rooted URL is one which indicates an absolute path.
+            This is True on any URL that includes a host, or any relative URL
+            that starts with a slash.
+        userinfo: The username or colon-separated username:password pair.
+        uses_netloc: Indicates whether ``://`` (the "netloc separator") will
+            appear to separate the scheme from the *path* in cases where no
+            host is present.
+            Setting this to ``True`` is a non-spec-compliant affordance for the
+            common practice of having URIs that are *not* URLs (cannot have a
+            'host' part) but nevertheless use the common ``://`` idiom that
+            most people associate with URLs; e.g. ``message:`` URIs like
+            ``message://message-id`` being equivalent to ``message:message-id``.
+            This may be inferred based on the scheme depending on whether
+            :func:`register_scheme` has been used to register the scheme and
+            should not be passed directly unless you know the scheme works like
+            this and you know it has not been registered.
+
+    All of these parts are also exposed as read-only attributes of :class:`URL`
+    instances, along with several useful methods.
+
+    .. _RFC 3986: https://tools.ietf.org/html/rfc3986
+    .. _RFC 3987: https://tools.ietf.org/html/rfc3987
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        scheme=None,  # type: Optional[Text]
+        host=None,  # type: Optional[Text]
+        path=(),  # type: Iterable[Text]
+        query=(),  # type: QueryParameters
+        fragment=u"",  # type: Text
+        port=None,  # type: Optional[int]
+        rooted=None,  # type: Optional[bool]
+        userinfo=u"",  # type: Text
+        uses_netloc=None,  # type: Optional[bool]
+    ):
+        # type: (...) -> None
+        if host is not None and scheme is None:
+            scheme = u"http"  # TODO: why
+        if port is None and scheme is not None:
+            port = SCHEME_PORT_MAP.get(scheme)
+        if host and query and not path:
+            # per RFC 3986 6.2.3, "a URI that uses the generic syntax
+            # for authority with an empty path should be normalized to
+            # a path of '/'."
+            path = (u"",)
+
+        # Now that we're done detecting whether they were passed, we can set
+        # them to their defaults:
+        if scheme is None:
+            scheme = u""
+        if host is None:
+            host = u""
+        if rooted is None:
+            rooted = bool(host)
+
+        # Set attributes.
+        self._scheme = _textcheck("scheme", scheme)
+        if self._scheme:
+            if not _SCHEME_RE.match(self._scheme):
+                raise ValueError(
+                    'invalid scheme: %r. Only alphanumeric, "+",'
+                    ' "-", and "." allowed. Did you meant to call'
+                    " %s.from_text()?" % (self._scheme, self.__class__.__name__)
+                )
+
+        _, self._host = parse_host(_textcheck("host", host, "/?#@"))
+        if isinstance(path, Text):
+            raise TypeError(
+                "expected iterable of text for path, not: %r" % (path,)
+            )
+        self._path = tuple(
+            (_textcheck("path segment", segment, "/?#") for segment in path)
+        )
+        self._query = tuple(
+            (
+                _textcheck("query parameter name", k, "&=#"),
+                _textcheck("query parameter value", v, "&#", nullable=True),
+            )
+            for k, v in iter_pairs(query)
+        )
+        self._fragment = _textcheck("fragment", fragment)
+        self._port = _typecheck("port", port, int, NoneType)
+        self._rooted = _typecheck("rooted", rooted, bool)
+        self._userinfo = _textcheck("userinfo", userinfo, "/?#@")
+
+        if uses_netloc is None:
+            uses_netloc = scheme_uses_netloc(self._scheme, uses_netloc)
+        self._uses_netloc = _typecheck(
+            "uses_netloc", uses_netloc, bool, NoneType
+        )
+        will_have_authority = self._host or (
+            self._port and self._port != SCHEME_PORT_MAP.get(scheme)
+        )
+        if will_have_authority:
+            # fixup for rooted consistency; if there's any 'authority'
+            # represented in the textual URL, then the path must be rooted, and
+            # we're definitely using a netloc (there must be a ://).
+            self._rooted = True
+            self._uses_netloc = True
+        if (not self._rooted) and self.path[:1] == (u"",):
+            self._rooted = True
+            self._path = self._path[1:]
+        if not will_have_authority and self._path and not self._rooted:
+            # If, after fixing up the path, there *is* a path and it *isn't*
+            # rooted, then we are definitely not using a netloc; if we did, it
+            # would make the path (erroneously) look like a hostname.
+            self._uses_netloc = False
+
+    def get_decoded_url(self, lazy=False):
+        # type: (bool) -> DecodedURL
+        try:
+            return self._decoded_url
+        except AttributeError:
+            self._decoded_url = DecodedURL(self, lazy=lazy)  # type: DecodedURL
+        return self._decoded_url
+
+    @property
+    def scheme(self):
+        # type: () -> Text
+        """The scheme is a string, and the first part of an absolute URL, the
+        part before the first colon, and the part which defines the
+        semantics of the rest of the URL. Examples include "http",
+        "https", "ssh", "file", "mailto", and many others. See
+        :func:`~hyperlink.register_scheme()` for more info.
+        """
+        return self._scheme
+
+    @property
+    def host(self):
+        # type: () -> Text
+        """The host is a string, and the second standard part of an absolute
+        URL. When present, a valid host must be a domain name, or an
+        IP (v4 or v6). It occurs before the first slash, or the second
+        colon, if a :attr:`~hyperlink.URL.port` is provided.
+        """
+        return self._host
+
+    @property
+    def port(self):
+        # type: () -> Optional[int]
+        """The port is an integer that is commonly used in connecting to the
+        :attr:`host`, and almost never appears without it.
+
+        When not present in the original URL, this attribute defaults
+        to the scheme's default port. If the scheme's default port is
+        not known, and the port is not provided, this attribute will
+        be set to None.
+
+        >>> URL.from_text(u'http://example.com/pa/th').port
+        80
+        >>> URL.from_text(u'foo://example.com/pa/th').port
+        >>> URL.from_text(u'foo://example.com:8042/pa/th').port
+        8042
+
+        .. note::
+
+           Per the standard, when the port is the same as the schemes
+           default port, it will be omitted in the text URL.
+        """
+        return self._port
+
+    @property
+    def path(self):
+        # type: () -> Sequence[Text]
+        """A tuple of strings, created by splitting the slash-separated
+        hierarchical path. Started by the first slash after the host,
+        terminated by a "?", which indicates the start of the
+        :attr:`~hyperlink.URL.query` string.
+        """
+        return self._path
+
+    @property
+    def query(self):
+        # type: () -> QueryPairs
+        """Tuple of pairs, created by splitting the ampersand-separated
+        mapping of keys and optional values representing
+        non-hierarchical data used to identify the resource. Keys are
+        always strings. Values are strings when present, or None when
+        missing.
+
+        For more operations on the mapping, see
+        :meth:`~hyperlink.URL.get()`, :meth:`~hyperlink.URL.add()`,
+        :meth:`~hyperlink.URL.set()`, and
+        :meth:`~hyperlink.URL.delete()`.
+        """
+        return self._query
+
+    @property
+    def fragment(self):
+        # type: () -> Text
+        """A string, the last part of the URL, indicated by the first "#"
+        after the :attr:`~hyperlink.URL.path` or
+        :attr:`~hyperlink.URL.query`. Enables indirect identification
+        of a secondary resource, like an anchor within an HTML page.
+        """
+        return self._fragment
+
+    @property
+    def rooted(self):
+        # type: () -> bool
+        """Whether or not the path starts with a forward slash (``/``).
+
+        This is taken from the terminology in the BNF grammar,
+        specifically the "path-rootless", rule, since "absolute path"
+        and "absolute URI" are somewhat ambiguous. :attr:`path` does
+        not contain the implicit prefixed ``"/"`` since that is
+        somewhat awkward to work with.
+        """
+        return self._rooted
+
+    @property
+    def userinfo(self):
+        # type: () -> Text
+        """The colon-separated string forming the username-password
+        combination.
+        """
+        return self._userinfo
+
+    @property
+    def uses_netloc(self):
+        # type: () -> Optional[bool]
+        """
+        Indicates whether ``://`` (the "netloc separator") will appear to
+        separate the scheme from the *path* in cases where no host is present.
+        """
+        return self._uses_netloc
+
+    @property
+    def user(self):
+        # type: () -> Text
+        """
+        The user portion of :attr:`~hyperlink.URL.userinfo`.
+        """
+        return self.userinfo.split(u":")[0]
+
+    def authority(self, with_password=False, **kw):
+        # type: (bool, Any) -> Text
+        """Compute and return the appropriate host/port/userinfo combination.
+
+        >>> url = URL.from_text(u'http://user:pass@localhost:8080/a/b?x=y')
+        >>> url.authority()
+        u'user:@localhost:8080'
+        >>> url.authority(with_password=True)
+        u'user:pass@localhost:8080'
+
+        Args:
+            with_password: Whether the return value of this method include the
+            password in the URL, if it is set.
+            Defaults to False.
+
+        Returns:
+            Text: The authority (network location and user information) portion
+                of the URL.
+        """
+        # first, a bit of twisted compat
+        with_password = kw.pop("includeSecrets", with_password)
+        if kw:
+            raise TypeError("got unexpected keyword arguments: %r" % kw.keys())
+        host = self.host
+        if ":" in host:
+            hostport = ["[" + host + "]"]
+        else:
+            hostport = [self.host]
+        if self.port != SCHEME_PORT_MAP.get(self.scheme):
+            hostport.append(Text(self.port))
+        authority = []
+        if self.userinfo:
+            userinfo = self.userinfo
+            if not with_password and u":" in userinfo:
+                userinfo = userinfo[: userinfo.index(u":") + 1]
+            authority.append(userinfo)
+        authority.append(u":".join(hostport))
+        return u"@".join(authority)
+
+    def __eq__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        for attr in [
+            "scheme",
+            "userinfo",
+            "host",
+            "query",
+            "fragment",
+            "port",
+            "uses_netloc",
+            "rooted",
+        ]:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        if self.path == other.path or (
+            self.path in _ROOT_PATHS and other.path in _ROOT_PATHS
+        ):
+            return True
+        return False
+
+    def __ne__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        # type: () -> int
+        return hash(
+            (
+                self.__class__,
+                self.scheme,
+                self.userinfo,
+                self.host,
+                self.path,
+                self.query,
+                self.fragment,
+                self.port,
+                self.rooted,
+                self.uses_netloc,
+            )
+        )
+
+    @property
+    def absolute(self):
+        # type: () -> bool
+        """Whether or not the URL is "absolute". Absolute URLs are complete
+        enough to resolve to a network resource without being relative
+        to a base URI.
+
+        >>> URL.from_text(u'http://wikipedia.org/').absolute
+        True
+        >>> URL.from_text(u'?a=b&c=d').absolute
+        False
+
+        Absolute URLs must have both a scheme and a host set.
+        """
+        return bool(self.scheme and self.host)
+
+    def replace(
+        self,
+        scheme=_UNSET,  # type: Optional[Text]
+        host=_UNSET,  # type: Optional[Text]
+        path=_UNSET,  # type: Iterable[Text]
+        query=_UNSET,  # type: QueryParameters
+        fragment=_UNSET,  # type: Text
+        port=_UNSET,  # type: Optional[int]
+        rooted=_UNSET,  # type: Optional[bool]
+        userinfo=_UNSET,  # type: Text
+        uses_netloc=_UNSET,  # type: Optional[bool]
+    ):
+        # type: (...) -> URL
+        """:class:`URL` objects are immutable, which means that attributes
+        are designed to be set only once, at construction. Instead of
+        modifying an existing URL, one simply creates a copy with the
+        desired changes.
+
+        If any of the following arguments is omitted, it defaults to
+        the value on the current URL.
+
+        Args:
+            scheme: The text name of the scheme.
+            host: The host portion of the network location.
+            path: A tuple of strings representing the slash-separated parts of
+                the path.
+            query: The query parameters, as a dictionary or as an sequence of
+                key-value pairs.
+            fragment: The fragment part of the URL.
+            port: The port part of the network location.
+            rooted: Whether or not the path begins with a slash.
+            userinfo: The username or colon-separated username:password pair.
+            uses_netloc: Indicates whether ``://`` (the "netloc separator")
+                will appear to separate the scheme from the *path* in cases
+                where no host is present.
+                Setting this to ``True`` is a non-spec-compliant affordance for
+                the common practice of having URIs that are *not* URLs (cannot
+                have a 'host' part) but nevertheless use the common ``://``
+                idiom that most people associate with URLs; e.g. ``message:``
+                URIs like ``message://message-id`` being equivalent to
+                ``message:message-id``.
+                This may be inferred based on the scheme depending on whether
+                :func:`register_scheme` has been used to register the scheme
+                and should not be passed directly unless you know the scheme
+                works like this and you know it has not been registered.
+
+        Returns:
+            URL: A copy of the current :class:`URL`, with new values for
+                parameters passed.
+        """
+        if scheme is not _UNSET and scheme != self.scheme:
+            # when changing schemes, reset the explicit uses_netloc preference
+            # to honor the new scheme.
+            uses_netloc = None
+        return self.__class__(
+            scheme=_optional(scheme, self.scheme),
+            host=_optional(host, self.host),
+            path=_optional(path, self.path),
+            query=_optional(query, self.query),
+            fragment=_optional(fragment, self.fragment),
+            port=_optional(port, self.port),
+            rooted=_optional(rooted, self.rooted),
+            userinfo=_optional(userinfo, self.userinfo),
+            uses_netloc=_optional(uses_netloc, self.uses_netloc),
+        )
+
+    @classmethod
+    def from_text(cls, text):
+        # type: (Text) -> URL
+        """Whereas the :class:`URL` constructor is useful for constructing
+        URLs from parts, :meth:`~URL.from_text` supports parsing whole
+        URLs from their string form::
+
+            >>> URL.from_text(u'http://example.com')
+            URL.from_text(u'http://example.com')
+            >>> URL.from_text(u'?a=b&x=y')
+            URL.from_text(u'?a=b&x=y')
+
+        As you can see above, it's also used as the :func:`repr` of
+        :class:`URL` objects. The natural counterpart to
+        :func:`~URL.to_text()`. This method only accepts *text*, so be
+        sure to decode those bytestrings.
+
+        Args:
+           text: A valid URL string.
+
+        Returns:
+           URL: The structured object version of the parsed string.
+
+        .. note::
+
+            Somewhat unexpectedly, URLs are a far more permissive
+            format than most would assume. Many strings which don't
+            look like URLs are still valid URLs. As a result, this
+            method only raises :class:`URLParseError` on invalid port
+            and IPv6 values in the host portion of the URL.
+        """
+        um = _URL_RE.match(_textcheck("text", text))
+        if um is None:
+            raise URLParseError("could not parse url: %r" % text)
+        gs = um.groupdict()
+
+        au_text = gs["authority"] or u""
+        au_m = _AUTHORITY_RE.match(au_text)
+        if au_m is None:
+            raise URLParseError(
+                "invalid authority %r in url: %r" % (au_text, text)
+            )
+        au_gs = au_m.groupdict()
+        if au_gs["bad_host"]:
+            raise URLParseError(
+                "invalid host %r in url: %r" % (au_gs["bad_host"], text)
+            )
+
+        userinfo = au_gs["userinfo"] or u""
+
+        host = au_gs["ipv6_host"] or au_gs["plain_host"]
+        port = au_gs["port"]
+        if port is not None:
+            try:
+                port = int(port)  # type: ignore[assignment] # FIXME, see below
+            except ValueError:
+                if not port:  # TODO: excessive?
+                    raise URLParseError("port must not be empty: %r" % au_text)
+                raise URLParseError("expected integer for port, not %r" % port)
+
+        scheme = gs["scheme"] or u""
+        fragment = gs["fragment"] or u""
+        uses_netloc = bool(gs["_netloc_sep"])
+
+        if gs["path"]:
+            path = tuple(gs["path"].split(u"/"))
+            if not path[0]:
+                path = path[1:]
+                rooted = True
+            else:
+                rooted = False
+        else:
+            path = ()
+            rooted = bool(au_text)
+        if gs["query"]:
+            query = tuple(
+                (
+                    qe.split(u"=", 1)  # type: ignore[misc]
+                    if u"=" in qe
+                    else (qe, None)
+                )
+                for qe in gs["query"].split(u"&")
+            )  # type: QueryPairs
+        else:
+            query = ()
+        return cls(
+            scheme,
+            host,
+            path,
+            query,
+            fragment,
+            port,  # type: ignore[arg-type] # FIXME, see above
+            rooted,
+            userinfo,
+            uses_netloc,
+        )
+
+    def normalize(
+        self,
+        scheme=True,
+        host=True,
+        path=True,
+        query=True,
+        fragment=True,
+        userinfo=True,
+        percents=True,
+    ):
+        # type: (bool, bool, bool, bool, bool, bool, bool) -> URL
+        """Return a new URL object with several standard normalizations
+        applied:
+
+        * Decode unreserved characters (`RFC 3986 2.3`_)
+        * Uppercase remaining percent-encoded octets (`RFC 3986 2.1`_)
+        * Convert scheme and host casing to lowercase (`RFC 3986 3.2.2`_)
+        * Resolve any "." and ".." references in the path (`RFC 3986 6.2.2.3`_)
+        * Ensure an ending slash on URLs with an empty path (`RFC 3986 6.2.3`_)
+        * Encode any stray percent signs (`%`) in percent-encoded
+          fields (path, query, fragment, userinfo) (`RFC 3986 2.4`_)
+
+        All are applied by default, but normalizations can be disabled
+        per-part by passing `False` for that part's corresponding
+        name.
+
+        Args:
+            scheme: Convert the scheme to lowercase
+            host: Convert the host to lowercase
+            path: Normalize the path (see above for details)
+            query: Normalize the query string
+            fragment: Normalize the fragment
+            userinfo: Normalize the userinfo
+            percents: Encode isolated percent signs for any percent-encoded
+                fields which are being normalized (defaults to `True`).
+
+        >>> url = URL.from_text(u'Http://example.COM/a/../b/./c%2f?%61%')
+        >>> print(url.normalize().to_text())
+        http://example.com/b/c%2F?a%25
+
+        .. _RFC 3986 3.2.2: https://tools.ietf.org/html/rfc3986#section-3.2.2
+        .. _RFC 3986 2.3: https://tools.ietf.org/html/rfc3986#section-2.3
+        .. _RFC 3986 2.1: https://tools.ietf.org/html/rfc3986#section-2.1
+        .. _RFC 3986 6.2.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.2.3
+        .. _RFC 3986 6.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.3
+        .. _RFC 3986 2.4: https://tools.ietf.org/html/rfc3986#section-2.4
+        """  # noqa: E501
+        kw = {}  # type: Dict[str, Any]
+        if scheme:
+            kw["scheme"] = self.scheme.lower()
+        if host:
+            kw["host"] = self.host.lower()
+
+        def _dec_unres(target):
+            # type: (Text) -> Text
+            return _decode_unreserved(
+                target, normalize_case=True, encode_stray_percents=percents
+            )
+
+        if path:
+            if self.path:
+                kw["path"] = [
+                    _dec_unres(p) for p in _resolve_dot_segments(self.path)
+                ]
+            else:
+                kw["path"] = (u"",)
+        if query:
+            kw["query"] = [
+                (_dec_unres(k), _dec_unres(v) if v else v)
+                for k, v in self.query
+            ]
+        if fragment:
+            kw["fragment"] = _dec_unres(self.fragment)
+        if userinfo:
+            kw["userinfo"] = u":".join(
+                [_dec_unres(p) for p in self.userinfo.split(":", 1)]
+            )
+
+        return self.replace(**kw)
+
+    def child(self, *segments):
+        # type: (Text) -> URL
+        """Make a new :class:`URL` where the given path segments are a child
+        of this URL, preserving other parts of the URL, including the
+        query string and fragment.
+
+        For example::
+
+            >>> url = URL.from_text(u'http://localhost/a/b?x=y')
+            >>> child_url = url.child(u"c", u"d")
+            >>> child_url.to_text()
+            u'http://localhost/a/b/c/d?x=y'
+
+        Args:
+            segments: Additional parts to be joined and added to the path, like
+            :func:`os.path.join`. Special characters in segments will be
+            percent encoded.
+
+        Returns:
+           URL: A copy of the current URL with the extra path segments.
+        """
+        if not segments:
+            return self
+
+        segments = [  # type: ignore[assignment] # variable is tuple
+            _textcheck("path segment", s) for s in segments
+        ]
+        new_path = tuple(self.path)
+        if self.path and self.path[-1] == u"":
+            new_path = new_path[:-1]
+        new_path += tuple(_encode_path_parts(segments, maximal=False))
+        return self.replace(path=new_path)
+
+    def sibling(self, segment):
+        # type: (Text) -> URL
+        """Make a new :class:`URL` with a single path segment that is a
+        sibling of this URL path.
+
+        Args:
+            segment: A single path segment.
+
+        Returns:
+            URL: A copy of the current URL with the last path segment
+                replaced by *segment*. Special characters such as
+                ``/?#`` will be percent encoded.
+        """
+        _textcheck("path segment", segment)
+        new_path = tuple(self.path)[:-1] + (_encode_path_part(segment),)
+        return self.replace(path=new_path)
+
+    def click(self, href=u""):
+        # type: (Union[Text, URL]) -> URL
+        """Resolve the given URL relative to this URL.
+
+        The resulting URI should match what a web browser would
+        generate if you visited the current URL and clicked on *href*.
+
+            >>> url = URL.from_text(u'http://blog.hatnote.com/')
+            >>> url.click(u'/post/155074058790').to_text()
+            u'http://blog.hatnote.com/post/155074058790'
+            >>> url = URL.from_text(u'http://localhost/a/b/c/')
+            >>> url.click(u'../d/./e').to_text()
+            u'http://localhost/a/b/d/e'
+
+        Args (Text):
+            href: A string representing a clicked URL.
+
+        Return:
+            A copy of the current URL with navigation logic applied.
+
+        For more information, see `RFC 3986 section 5`_.
+
+        .. _RFC 3986 section 5: https://tools.ietf.org/html/rfc3986#section-5
+        """
+        if href:
+            if isinstance(href, URL):
+                clicked = href
+            else:
+                # TODO: This error message is not completely accurate,
+                # as URL objects are now also valid, but Twisted's
+                # test suite (wrongly) relies on this exact message.
+                _textcheck("relative URL", href)
+                clicked = URL.from_text(href)
+            if clicked.absolute:
+                return clicked
+        else:
+            clicked = self
+
+        query = clicked.query
+        if clicked.scheme and not clicked.rooted:
+            # Schemes with relative paths are not well-defined.  RFC 3986 calls
+            # them a "loophole in prior specifications" that should be avoided,
+            # or supported only for backwards compatibility.
+            raise NotImplementedError(
+                "absolute URI with rootless path: %r" % (href,)
+            )
+        else:
+            if clicked.rooted:
+                path = clicked.path
+            elif clicked.path:
+                path = tuple(self.path)[:-1] + tuple(clicked.path)
+            else:
+                path = self.path
+                if not query:
+                    query = self.query
+        return self.replace(
+            scheme=clicked.scheme or self.scheme,
+            host=clicked.host or self.host,
+            port=clicked.port or self.port,
+            path=_resolve_dot_segments(path),
+            query=query,
+            fragment=clicked.fragment,
+        )
+
+    def to_uri(self):
+        # type: () -> URL
+        u"""Make a new :class:`URL` instance with all non-ASCII characters
+        appropriately percent-encoded. This is useful to do in preparation
+        for sending a :class:`URL` over a network protocol.
+
+        For example::
+
+            >>> URL.from_text(u'https://ايران.com/foo⇧bar/').to_uri()
+            URL.from_text(u'https://xn--mgba3a4fra.com/foo%E2%87%A7bar/')
+
+        Returns:
+            URL: A new instance with its path segments, query parameters, and
+                hostname encoded, so that they are all in the standard
+                US-ASCII range.
+        """
+        new_userinfo = u":".join(
+            [_encode_userinfo_part(p) for p in self.userinfo.split(":", 1)]
+        )
+        new_path = _encode_path_parts(
+            self.path, has_scheme=bool(self.scheme), rooted=False, maximal=True
+        )
+        new_host = (
+            self.host
+            if not self.host
+            else idna_encode(self.host, uts46=True).decode("ascii")
+        )
+        return self.replace(
+            userinfo=new_userinfo,
+            host=new_host,
+            path=new_path,
+            query=tuple(
+                [
+                    (
+                        _encode_query_key(k, maximal=True),
+                        _encode_query_value(v, maximal=True)
+                        if v is not None
+                        else None,
+                    )
+                    for k, v in self.query
+                ]
+            ),
+            fragment=_encode_fragment_part(self.fragment, maximal=True),
+        )
+
+    def to_iri(self):
+        # type: () -> URL
+        u"""Make a new :class:`URL` instance with all but a few reserved
+        characters decoded into human-readable format.
+
+        Percent-encoded Unicode and IDNA-encoded hostnames are
+        decoded, like so::
+
+            >>> url = URL.from_text(u'https://xn--mgba3a4fra.example.com/foo%E2%87%A7bar/')
+            >>> print(url.to_iri().to_text())
+            https://ايران.example.com/foo⇧bar/
+
+        .. note::
+
+            As a general Python issue, "narrow" (UCS-2) builds of
+            Python may not be able to fully decode certain URLs, and
+            the in those cases, this method will return a best-effort,
+            partially-decoded, URL which is still valid. This issue
+            does not affect any Python builds 3.4+.
+
+        Returns:
+            URL: A new instance with its path segments, query parameters, and
+                hostname decoded for display purposes.
+        """  # noqa: E501
+        new_userinfo = u":".join(
+            [_decode_userinfo_part(p) for p in self.userinfo.split(":", 1)]
+        )
+        host_text = _decode_host(self.host)
+
+        return self.replace(
+            userinfo=new_userinfo,
+            host=host_text,
+            path=[_decode_path_part(segment) for segment in self.path],
+            query=tuple(
+                (
+                    _decode_query_key(k),
+                    _decode_query_value(v) if v is not None else None,
+                )
+                for k, v in self.query
+            ),
+            fragment=_decode_fragment_part(self.fragment),
+        )
+
+    def to_text(self, with_password=False):
+        # type: (bool) -> Text
+        """Render this URL to its textual representation.
+
+        By default, the URL text will *not* include a password, if one
+        is set. RFC 3986 considers using URLs to represent such
+        sensitive information as deprecated. Quoting from RFC 3986,
+        `section 3.2.1`:
+
+            "Applications should not render as clear text any data after the
+            first colon (":") character found within a userinfo subcomponent
+            unless the data after the colon is the empty string (indicating no
+            password)."
+
+        Args (bool):
+            with_password: Whether or not to include the password in the URL
+                text. Defaults to False.
+
+        Returns:
+            Text: The serialized textual representation of this URL, such as
+                ``u"http://example.com/some/path?some=query"``.
+
+        The natural counterpart to :class:`URL.from_text()`.
+
+        .. _section 3.2.1: https://tools.ietf.org/html/rfc3986#section-3.2.1
+        """
+        scheme = self.scheme
+        authority = self.authority(with_password)
+        path = "/".join(
+            _encode_path_parts(
+                self.path,
+                rooted=self.rooted,
+                has_scheme=bool(scheme),
+                has_authority=bool(authority),
+                maximal=False,
+            )
+        )
+        query_parts = []
+        for k, v in self.query:
+            if v is None:
+                query_parts.append(_encode_query_key(k, maximal=False))
+            else:
+                query_parts.append(
+                    u"=".join(
+                        (
+                            _encode_query_key(k, maximal=False),
+                            _encode_query_value(v, maximal=False),
+                        )
+                    )
+                )
+        query_string = u"&".join(query_parts)
+
+        fragment = self.fragment
+
+        parts = []  # type: List[Text]
+        _add = parts.append
+        if scheme:
+            _add(scheme)
+            _add(":")
+        if authority:
+            _add("//")
+            _add(authority)
+        elif scheme and path[:2] != "//" and self.uses_netloc:
+            _add("//")
+        if path:
+            if scheme and authority and path[:1] != "/":
+                _add("/")  # relpaths with abs authorities auto get '/'
+            _add(path)
+        if query_string:
+            _add("?")
+            _add(query_string)
+        if fragment:
+            _add("#")
+            _add(fragment)
+        return u"".join(parts)
+
+    def __repr__(self):
+        # type: () -> str
+        """Convert this URL to an representation that shows all of its
+        constituent parts, as well as being a valid argument to
+        :func:`eval`.
+        """
+        return "%s.from_text(%r)" % (self.__class__.__name__, self.to_text())
+
+    def _to_bytes(self):
+        # type: () -> bytes
+        """
+        Allows for direct usage of URL objects with libraries like
+        requests, which automatically stringify URL parameters. See
+        issue #49.
+        """
+        return self.to_uri().to_text().encode("ascii")
+
+    if PY2:
+        __str__ = _to_bytes
+        __unicode__ = to_text
+    else:
+        __bytes__ = _to_bytes
+        __str__ = to_text
+
+    # # Begin Twisted Compat Code
+    asURI = to_uri
+    asIRI = to_iri
+
+    @classmethod
+    def fromText(cls, s):
+        # type: (Text) -> URL
+        return cls.from_text(s)
+
+    def asText(self, includeSecrets=False):
+        # type: (bool) -> Text
+        return self.to_text(with_password=includeSecrets)
+
+    def __dir__(self):
+        # type: () -> Sequence[Text]
+        try:
+            ret = object.__dir__(self)
+        except AttributeError:
+            # object.__dir__ == AttributeError  # pdw for py2
+            ret = dir(self.__class__) + list(self.__dict__.keys())
+        ret = sorted(set(ret) - set(["fromText", "asURI", "asIRI", "asText"]))
+        return ret
+
+    # # End Twisted Compat Code
+
+    def add(self, name, value=None):
+        # type: (Text, Optional[Text]) -> URL
+        """Make a new :class:`URL` instance with a given query argument,
+        *name*, added to it with the value *value*, like so::
+
+            >>> URL.from_text(u'https://example.com/?x=y').add(u'x')
+            URL.from_text(u'https://example.com/?x=y&x')
+            >>> URL.from_text(u'https://example.com/?x=y').add(u'x', u'z')
+            URL.from_text(u'https://example.com/?x=y&x=z')
+
+        Args:
+            name: The name of the query parameter to add.
+                The part before the ``=``.
+            value: The value of the query parameter to add.
+                The part after the ``=``.
+                Defaults to ``None``, meaning no value.
+
+        Returns:
+            URL: A new :class:`URL` instance with the parameter added.
+        """
+        return self.replace(query=self.query + ((name, value),))
+
+    def set(self, name, value=None):
+        # type: (Text, Optional[Text]) -> URL
+        """Make a new :class:`URL` instance with the query parameter *name*
+        set to *value*. All existing occurences, if any are replaced
+        by the single name-value pair.
+
+            >>> URL.from_text(u'https://example.com/?x=y').set(u'x')
+            URL.from_text(u'https://example.com/?x')
+            >>> URL.from_text(u'https://example.com/?x=y').set(u'x', u'z')
+            URL.from_text(u'https://example.com/?x=z')
+
+        Args:
+            name: The name of the query parameter to set.
+                The part before the ``=``.
+            value: The value of the query parameter to set.
+                The part after the ``=``.
+                Defaults to ``None``, meaning no value.
+
+        Returns:
+            URL: A new :class:`URL` instance with the parameter set.
+        """
+        # Preserve the original position of the query key in the list
+        q = [(k, v) for (k, v) in self.query if k != name]
+        idx = next(
+            (i for (i, (k, v)) in enumerate(self.query) if k == name), -1
+        )
+        q[idx:idx] = [(name, value)]
+        return self.replace(query=q)
+
+    def get(self, name):
+        # type: (Text) -> List[Optional[Text]]
+        """Get a list of values for the given query parameter, *name*::
+
+            >>> url = URL.from_text(u'?x=1&x=2')
+            >>> url.get('x')
+            [u'1', u'2']
+            >>> url.get('y')
+            []
+
+        If the given *name* is not set, an empty list is returned. A
+        list is always returned, and this method raises no exceptions.
+
+        Args:
+            name: The name of the query parameter to get.
+
+        Returns:
+            List[Optional[Text]]: A list of all the values associated with the
+                key, in string form.
+        """
+        return [value for (key, value) in self.query if name == key]
+
+    def remove(
+        self,
+        name,  # type: Text
+        value=_UNSET,  # type: Text
+        limit=None,  # type: Optional[int]
+    ):
+        # type: (...) -> URL
+        """Make a new :class:`URL` instance with occurrences of the query
+        parameter *name* removed, or, if *value* is set, parameters
+        matching *name* and *value*. No exception is raised if the
+        parameter is not already set.
+
+        Args:
+            name: The name of the query parameter to remove.
+            value: Optional value to additionally filter on.
+                Setting this removes query parameters which match both name
+                and value.
+            limit: Optional maximum number of parameters to remove.
+
+        Returns:
+            URL: A new :class:`URL` instance with the parameter removed.
+        """
+        if limit is None:
+            if value is _UNSET:
+                nq = [(k, v) for (k, v) in self.query if k != name]
+            else:
+                nq = [
+                    (k, v)
+                    for (k, v) in self.query
+                    if not (k == name and v == value)
+                ]
+        else:
+            nq, removed_count = [], 0
+
+            for k, v in self.query:
+                if (
+                    k == name
+                    and (value is _UNSET or v == value)
+                    and removed_count < limit
+                ):
+                    removed_count += 1  # drop it
+                else:
+                    nq.append((k, v))  # keep it
+
+        return self.replace(query=nq)
+
+
+EncodedURL = URL  # An alias better describing what the URL really is
+
+_EMPTY_URL = URL()
+
+
+def _replace_plus(text):
+    # type: (Text) -> Text
+    return text.replace("+", "%20")
+
+
+def _no_op(text):
+    # type: (Text) -> Text
+    return text
+
+
+class DecodedURL(object):
+    """
+    :class:`DecodedURL` is a type designed to act as a higher-level
+    interface to :class:`URL` and the recommended type for most
+    operations. By analogy, :class:`DecodedURL` is the
+    :class:`unicode` to URL's :class:`bytes`.
+
+    :class:`DecodedURL` automatically handles encoding and decoding
+    all its components, such that all inputs and outputs are in a
+    maximally-decoded state.  Note that this means, for some special
+    cases, a URL may not "roundtrip" character-for-character, but this
+    is considered a good tradeoff for the safety of automatic
+    encoding.
+
+    Otherwise, :class:`DecodedURL` has almost exactly the same API as
+    :class:`URL`.
+
+    Where applicable, a UTF-8 encoding is presumed. Be advised that
+    some interactions can raise :exc:`UnicodeEncodeErrors` and
+    :exc:`UnicodeDecodeErrors`, just like when working with
+    bytestrings. Examples of such interactions include handling query
+    strings encoding binary data, and paths containing segments with
+    special characters encoded with codecs other than UTF-8.
+
+    Args:
+        url: A :class:`URL` object to wrap.
+        lazy: Set to True to avoid pre-decode all parts of the URL to check for
+            validity.
+            Defaults to False.
+        query_plus_is_space: + characters in the query string should be treated
+            as spaces when decoding.  If unspecified, the default is taken from
+            the scheme.
+
+    .. note::
+
+      The :class:`DecodedURL` initializer takes a :class:`URL` object,
+      not URL components, like :class:`URL`. To programmatically
+      construct a :class:`DecodedURL`, you can use this pattern:
+
+        >>> print(DecodedURL().replace(scheme=u'https',
+        ... host=u'pypi.org', path=(u'projects', u'hyperlink')).to_text())
+        https://pypi.org/projects/hyperlink
+
+    .. versionadded:: 18.0.0
+    """
+
+    def __init__(self, url=_EMPTY_URL, lazy=False, query_plus_is_space=None):
+        # type: (URL, bool, Optional[bool]) -> None
+        self._url = url
+        if query_plus_is_space is None:
+            query_plus_is_space = url.scheme not in NO_QUERY_PLUS_SCHEMES
+        self._query_plus_is_space = query_plus_is_space
+        if not lazy:
+            # cache the following, while triggering any decoding
+            # issues with decodable fields
+            self.host, self.userinfo, self.path, self.query, self.fragment
+        return
+
+    @classmethod
+    def from_text(cls, text, lazy=False, query_plus_is_space=None):
+        # type: (Text, bool, Optional[bool]) -> DecodedURL
+        """\
+        Make a `DecodedURL` instance from any text string containing a URL.
+
+        Args:
+          text: Text containing the URL
+          lazy: Whether to pre-decode all parts of the URL to check for
+              validity.
+              Defaults to True.
+        """
+        _url = URL.from_text(text)
+        return cls(_url, lazy=lazy, query_plus_is_space=query_plus_is_space)
+
+    @property
+    def encoded_url(self):
+        # type: () -> URL
+        """Access the underlying :class:`URL` object, which has any special
+        characters encoded.
+        """
+        return self._url
+
+    def to_text(self, with_password=False):
+        # type: (bool) -> Text
+        "Passthrough to :meth:`~hyperlink.URL.to_text()`"
+        return self._url.to_text(with_password)
+
+    def to_uri(self):
+        # type: () -> URL
+        "Passthrough to :meth:`~hyperlink.URL.to_uri()`"
+        return self._url.to_uri()
+
+    def to_iri(self):
+        # type: () -> URL
+        "Passthrough to :meth:`~hyperlink.URL.to_iri()`"
+        return self._url.to_iri()
+
+    def _clone(self, url):
+        # type: (URL) -> DecodedURL
+        return self.__class__(
+            url,
+            # TODO: propagate laziness?
+            query_plus_is_space=self._query_plus_is_space,
+        )
+
+    def click(self, href=u""):
+        # type: (Union[Text, URL, DecodedURL]) -> DecodedURL
+        """Return a new DecodedURL wrapping the result of
+        :meth:`~hyperlink.URL.click()`
+        """
+        if isinstance(href, DecodedURL):
+            href = href._url
+        return self._clone(
+            self._url.click(href=href),
+        )
+
+    def sibling(self, segment):
+        # type: (Text) -> DecodedURL
+        """Automatically encode any reserved characters in *segment* and
+        return a new `DecodedURL` wrapping the result of
+        :meth:`~hyperlink.URL.sibling()`
+        """
+        return self._clone(
+            self._url.sibling(_encode_reserved(segment)),
+        )
+
+    def child(self, *segments):
+        # type: (Text) -> DecodedURL
+        """Automatically encode any reserved characters in *segments* and
+        return a new `DecodedURL` wrapping the result of
+        :meth:`~hyperlink.URL.child()`.
+        """
+        if not segments:
+            return self
+        new_segs = [_encode_reserved(s) for s in segments]
+        return self._clone(self._url.child(*new_segs))
+
+    def normalize(
+        self,
+        scheme=True,
+        host=True,
+        path=True,
+        query=True,
+        fragment=True,
+        userinfo=True,
+        percents=True,
+    ):
+        # type: (bool, bool, bool, bool, bool, bool, bool) -> DecodedURL
+        """Return a new `DecodedURL` wrapping the result of
+        :meth:`~hyperlink.URL.normalize()`
+        """
+        return self._clone(
+            self._url.normalize(
+                scheme, host, path, query, fragment, userinfo, percents
+            )
+        )
+
+    @property
+    def absolute(self):
+        # type: () -> bool
+        return self._url.absolute
+
+    @property
+    def scheme(self):
+        # type: () -> Text
+        return self._url.scheme
+
+    @property
+    def host(self):
+        # type: () -> Text
+        return _decode_host(self._url.host)
+
+    @property
+    def port(self):
+        # type: () -> Optional[int]
+        return self._url.port
+
+    @property
+    def rooted(self):
+        # type: () -> bool
+        return self._url.rooted
+
+    @property
+    def path(self):
+        # type: () -> Sequence[Text]
+        if not hasattr(self, "_path"):
+            self._path = tuple(
+                [
+                    _percent_decode(p, raise_subencoding_exc=True)
+                    for p in self._url.path
+                ]
+            )
+        return self._path
+
+    @property
+    def query(self):
+        # type: () -> QueryPairs
+        if not hasattr(self, "_query"):
+            if self._query_plus_is_space:
+                predecode = _replace_plus
+            else:
+                predecode = _no_op
+
+            self._query = cast(
+                QueryPairs,
+                tuple(
+                    tuple(
+                        _percent_decode(
+                            predecode(x), raise_subencoding_exc=True
+                        )
+                        if x is not None
+                        else None
+                        for x in (k, v)
+                    )
+                    for k, v in self._url.query
+                ),
+            )
+        return self._query
+
+    @property
+    def fragment(self):
+        # type: () -> Text
+        if not hasattr(self, "_fragment"):
+            frag = self._url.fragment
+            self._fragment = _percent_decode(frag, raise_subencoding_exc=True)
+        return self._fragment
+
+    @property
+    def userinfo(self):
+        # type: () -> Union[Tuple[str], Tuple[str, str]]
+        if not hasattr(self, "_userinfo"):
+            self._userinfo = cast(
+                Union[Tuple[str], Tuple[str, str]],
+                tuple(
+                    tuple(
+                        _percent_decode(p, raise_subencoding_exc=True)
+                        for p in self._url.userinfo.split(":", 1)
+                    )
+                ),
+            )
+        return self._userinfo
+
+    @property
+    def user(self):
+        # type: () -> Text
+        return self.userinfo[0]
+
+    @property
+    def uses_netloc(self):
+        # type: () -> Optional[bool]
+        return self._url.uses_netloc
+
+    def replace(
+        self,
+        scheme=_UNSET,  # type: Optional[Text]
+        host=_UNSET,  # type: Optional[Text]
+        path=_UNSET,  # type: Iterable[Text]
+        query=_UNSET,  # type: QueryParameters
+        fragment=_UNSET,  # type: Text
+        port=_UNSET,  # type: Optional[int]
+        rooted=_UNSET,  # type: Optional[bool]
+        userinfo=_UNSET,  # type: Union[Tuple[str], Tuple[str, str]]
+        uses_netloc=_UNSET,  # type: Optional[bool]
+    ):
+        # type: (...) -> DecodedURL
+        """While the signature is the same, this `replace()` differs a little
+        from URL.replace. For instance, it accepts userinfo as a
+        tuple, not as a string, handling the case of having a username
+        containing a `:`. As with the rest of the methods on
+        DecodedURL, if you pass a reserved character, it will be
+        automatically encoded instead of an error being raised.
+        """
+        if path is not _UNSET:
+            path = tuple(_encode_reserved(p) for p in path)
+        if query is not _UNSET:
+            query = cast(
+                QueryPairs,
+                tuple(
+                    tuple(
+                        _encode_reserved(x) if x is not None else None
+                        for x in (k, v)
+                    )
+                    for k, v in iter_pairs(query)
+                ),
+            )
+        if userinfo is not _UNSET:
+            if len(userinfo) > 2:
+                raise ValueError(
+                    'userinfo expected sequence of ["user"] or'
+                    ' ["user", "password"], got %r' % (userinfo,)
+                )
+            userinfo_text = u":".join([_encode_reserved(p) for p in userinfo])
+        else:
+            userinfo_text = _UNSET
+        new_url = self._url.replace(
+            scheme=scheme,
+            host=host,
+            path=path,
+            query=query,
+            fragment=fragment,
+            port=port,
+            rooted=rooted,
+            userinfo=userinfo_text,
+            uses_netloc=uses_netloc,
+        )
+        return self._clone(url=new_url)
+
+    def get(self, name):
+        # type: (Text) -> List[Optional[Text]]
+        "Get the value of all query parameters whose name matches *name*"
+        return [v for (k, v) in self.query if name == k]
+
+    def add(self, name, value=None):
+        # type: (Text, Optional[Text]) -> DecodedURL
+        """Return a new DecodedURL with the query parameter *name* and *value*
+        added."""
+        return self.replace(query=self.query + ((name, value),))
+
+    def set(self, name, value=None):
+        # type: (Text, Optional[Text]) -> DecodedURL
+        "Return a new DecodedURL with query parameter *name* set to *value*"
+        query = self.query
+        q = [(k, v) for (k, v) in query if k != name]
+        idx = next((i for (i, (k, v)) in enumerate(query) if k == name), -1)
+        q[idx:idx] = [(name, value)]
+        return self.replace(query=q)
+
+    def remove(
+        self,
+        name,  # type: Text
+        value=_UNSET,  # type: Text
+        limit=None,  # type: Optional[int]
+    ):
+        # type: (...) -> DecodedURL
+        """Return a new DecodedURL with query parameter *name* removed.
+
+        Optionally also filter for *value*, as well as cap the number
+        of parameters removed with *limit*.
+        """
+        if limit is None:
+            if value is _UNSET:
+                nq = [(k, v) for (k, v) in self.query if k != name]
+            else:
+                nq = [
+                    (k, v)
+                    for (k, v) in self.query
+                    if not (k == name and v == value)
+                ]
+        else:
+            nq, removed_count = [], 0
+            for k, v in self.query:
+                if (
+                    k == name
+                    and (value is _UNSET or v == value)
+                    and removed_count < limit
+                ):
+                    removed_count += 1  # drop it
+                else:
+                    nq.append((k, v))  # keep it
+
+        return self.replace(query=nq)
+
+    def __repr__(self):
+        # type: () -> str
+        cn = self.__class__.__name__
+        return "%s(url=%r)" % (cn, self._url)
+
+    def __str__(self):
+        # type: () -> str
+        # TODO: the underlying URL's __str__ needs to change to make
+        # this work as the URL, see #55
+        return str(self._url)
+
+    def __eq__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return self.normalize().to_uri() == other.normalize().to_uri()
+
+    def __ne__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        # type: () -> int
+        return hash(
+            (
+                self.__class__,
+                self.scheme,
+                self.userinfo,
+                self.host,
+                self.path,
+                self.query,
+                self.fragment,
+                self.port,
+                self.rooted,
+                self.uses_netloc,
+            )
+        )
+
+    # # Begin Twisted Compat Code
+    asURI = to_uri
+    asIRI = to_iri
+
+    @classmethod
+    def fromText(cls, s, lazy=False):
+        # type: (Text, bool) -> DecodedURL
+        return cls.from_text(s, lazy=lazy)
+
+    def asText(self, includeSecrets=False):
+        # type: (bool) -> Text
+        return self.to_text(with_password=includeSecrets)
+
+    def __dir__(self):
+        # type: () -> Sequence[Text]
+        try:
+            ret = object.__dir__(self)
+        except AttributeError:
+            # object.__dir__ == AttributeError  # pdw for py2
+            ret = dir(self.__class__) + list(self.__dict__.keys())
+        ret = sorted(set(ret) - set(["fromText", "asURI", "asIRI", "asText"]))
+        return ret
+
+    # # End Twisted Compat Code
+
+
+def parse(url, decoded=True, lazy=False):
+    # type: (Text, bool, bool) -> Union[URL, DecodedURL]
+    """
+    Automatically turn text into a structured URL object.
+
+    >>> url = parse(u"https://github.com/python-hyper/hyperlink")
+    >>> print(url.to_text())
+    https://github.com/python-hyper/hyperlink
+
+    Args:
+        url: A text string representation of a URL.
+
+        decoded: Whether or not to return a :class:`DecodedURL`,
+            which automatically handles all
+            encoding/decoding/quoting/unquoting for all the various
+            accessors of parts of the URL, or a :class:`URL`,
+            which has the same API, but requires handling of special
+            characters for different parts of the URL.
+
+        lazy: In the case of `decoded=True`, this controls
+            whether the URL is decoded immediately or as accessed. The
+            default, `lazy=False`, checks all encoded parts of the URL
+            for decodability.
+
+    .. versionadded:: 18.0.0
+    """
+    enc_url = EncodedURL.from_text(url)
+    if not decoded:
+        return enc_url
+    dec_url = DecodedURL(enc_url, lazy=lazy)
+    return dec_url
diff --git a/contrib/python/hyperlink/py2/hyperlink/hypothesis.py b/contrib/python/hyperlink/py2/hyperlink/hypothesis.py
new file mode 100644
index 00000000000..45fd9a99569
--- /dev/null
+++ b/contrib/python/hyperlink/py2/hyperlink/hypothesis.py
@@ -0,0 +1,324 @@
+# -*- coding: utf-8 -*-
+"""
+Hypothesis strategies.
+"""
+from __future__ import absolute_import
+
+try:
+    import hypothesis
+
+    del hypothesis
+except ImportError:
+    from typing import Tuple
+
+    __all__ = ()  # type: Tuple[str, ...]
+else:
+    import io
+    import pkgutil
+    from csv import reader as csv_reader
+    from os.path import dirname, join
+    from string import ascii_letters, digits
+    from sys import maxunicode
+    from typing import (
+        Callable,
+        Iterable,
+        List,
+        Optional,
+        Sequence,
+        Text,
+        TypeVar,
+        cast,
+    )
+    from gzip import open as open_gzip
+
+    from . import DecodedURL, EncodedURL
+
+    from hypothesis import assume
+    from hypothesis.strategies import (
+        composite,
+        integers,
+        lists,
+        sampled_from,
+        text,
+    )
+
+    from idna import IDNAError, check_label, encode as idna_encode
+
+    __all__ = (
+        "decoded_urls",
+        "encoded_urls",
+        "hostname_labels",
+        "hostnames",
+        "idna_text",
+        "paths",
+        "port_numbers",
+    )
+
+    T = TypeVar("T")
+    DrawCallable = Callable[[Callable[..., T]], T]
+
+    try:
+        unichr
+    except NameError:  # Py3
+        unichr = chr  # type: Callable[[int], Text]
+
+    def idna_characters():
+        # type: () -> Text
+        """
+        Returns a string containing IDNA characters.
+        """
+        global _idnaCharacters
+
+        if not _idnaCharacters:
+            result = []
+
+            # Data source "IDNA Derived Properties":
+            # https://www.iana.org/assignments/idna-tables-6.3.0/
+            #   idna-tables-6.3.0.xhtml#idna-tables-properties
+            dataFileName = join(
+                dirname(__file__), "idna-tables-properties.csv.gz"
+            )
+            data = io.BytesIO(pkgutil.get_data(__name__, "idna-tables-properties.csv.gz"))
+            with open_gzip(data) as dataFile:
+                reader = csv_reader(
+                    (line.decode("utf-8") for line in dataFile),
+                    delimiter=",",
+                )
+                next(reader)  # Skip header row
+                for row in reader:
+                    codes, prop, description = row
+
+                    if prop != "PVALID":
+                        # CONTEXTO or CONTEXTJ are also allowed, but they come
+                        # with rules, so we're punting on those here.
+                        # See: https://tools.ietf.org/html/rfc5892
+                        continue
+
+                    startEnd = row[0].split("-", 1)
+                    if len(startEnd) == 1:
+                        # No end of range given; use start
+                        startEnd.append(startEnd[0])
+                    start, end = (int(i, 16) for i in startEnd)
+
+                    for i in range(start, end + 1):
+                        if i > maxunicode:  # Happens using Py2 on Windows
+                            break
+                        result.append(unichr(i))
+
+            _idnaCharacters = u"".join(result)
+
+        return _idnaCharacters
+
+    _idnaCharacters = ""  # type: Text
+
+    @composite
+    def idna_text(draw, min_size=1, max_size=None):
+        # type: (DrawCallable, int, Optional[int]) -> Text
+        """
+        A strategy which generates IDNA-encodable text.
+
+        @param min_size: The minimum number of characters in the text.
+            C{None} is treated as C{0}.
+
+        @param max_size: The maximum number of characters in the text.
+            Use C{None} for an unbounded size.
+        """
+        alphabet = idna_characters()
+
+        assert min_size >= 1
+
+        if max_size is not None:
+            assert max_size >= 1
+
+        result = cast(
+            Text,
+            draw(text(min_size=min_size, max_size=max_size, alphabet=alphabet)),
+        )
+
+        # FIXME: There should be a more efficient way to ensure we produce
+        # valid IDNA text.
+        try:
+            idna_encode(result)
+        except IDNAError:
+            assume(False)
+
+        return result
+
+    @composite
+    def port_numbers(draw, allow_zero=False):
+        # type: (DrawCallable, bool) -> int
+        """
+        A strategy which generates port numbers.
+
+        @param allow_zero: Whether to allow port C{0} as a possible value.
+        """
+        if allow_zero:
+            min_value = 0
+        else:
+            min_value = 1
+
+        return cast(int, draw(integers(min_value=min_value, max_value=65535)))
+
+    @composite
+    def hostname_labels(draw, allow_idn=True):
+        # type: (DrawCallable, bool) -> Text
+        """
+        A strategy which generates host name labels.
+
+        @param allow_idn: Whether to allow non-ASCII characters as allowed by
+            internationalized domain names (IDNs).
+        """
+        if allow_idn:
+            label = cast(Text, draw(idna_text(min_size=1, max_size=63)))
+
+            try:
+                label.encode("ascii")
+            except UnicodeEncodeError:
+                # If the label doesn't encode to ASCII, then we need to check
+                # the length of the label after encoding to punycode and adding
+                # the xn-- prefix.
+                while len(label.encode("punycode")) > 63 - len("xn--"):
+                    # Rather than bombing out, just trim from the end until it
+                    # is short enough, so hypothesis doesn't have to generate
+                    # new data.
+                    label = label[:-1]
+
+        else:
+            label = cast(
+                Text,
+                draw(
+                    text(
+                        min_size=1,
+                        max_size=63,
+                        alphabet=Text(ascii_letters + digits + u"-"),
+                    )
+                ),
+            )
+
+        # Filter invalid labels.
+        # It would be better to reliably avoid generation of bogus labels in
+        # the first place, but it's hard...
+        try:
+            check_label(label)
+        except UnicodeError:  # pragma: no cover (not always drawn)
+            assume(False)
+
+        return label
+
+    @composite
+    def hostnames(draw, allow_leading_digit=True, allow_idn=True):
+        # type: (DrawCallable, bool, bool) -> Text
+        """
+        A strategy which generates host names.
+
+        @param allow_leading_digit: Whether to allow a leading digit in host
+            names; they were not allowed prior to RFC 1123.
+
+        @param allow_idn: Whether to allow non-ASCII characters as allowed by
+            internationalized domain names (IDNs).
+        """
+        # Draw first label, filtering out labels with leading digits if needed
+        labels = [
+            cast(
+                Text,
+                draw(
+                    hostname_labels(allow_idn=allow_idn).filter(
+                        lambda l: (
+                            True if allow_leading_digit else l[0] not in digits
+                        )
+                    )
+                ),
+            )
+        ]
+        # Draw remaining labels
+        labels += cast(
+            List[Text],
+            draw(
+                lists(
+                    hostname_labels(allow_idn=allow_idn),
+                    min_size=1,
+                    max_size=4,
+                )
+            ),
+        )
+
+        # Trim off labels until the total host name length fits in 252
+        # characters.  This avoids having to filter the data.
+        while sum(len(label) for label in labels) + len(labels) - 1 > 252:
+            labels = labels[:-1]
+
+        return u".".join(labels)
+
+    def path_characters():
+        # type: () -> str
+        """
+        Returns a string containing valid URL path characters.
+        """
+        global _path_characters
+
+        if _path_characters is None:
+
+            def chars():
+                # type: () -> Iterable[Text]
+                for i in range(maxunicode):
+                    c = unichr(i)
+
+                    # Exclude reserved characters
+                    if c in "#/?":
+                        continue
+
+                    # Exclude anything not UTF-8 compatible
+                    try:
+                        c.encode("utf-8")
+                    except UnicodeEncodeError:
+                        continue
+
+                    yield c
+
+            _path_characters = "".join(chars())
+
+        return _path_characters
+
+    _path_characters = None  # type: Optional[str]
+
+    @composite
+    def paths(draw):
+        # type: (DrawCallable) -> Sequence[Text]
+        return cast(
+            List[Text],
+            draw(
+                lists(text(min_size=1, alphabet=path_characters()), max_size=10)
+            ),
+        )
+
+    @composite
+    def encoded_urls(draw):
+        # type: (DrawCallable) -> EncodedURL
+        """
+        A strategy which generates L{EncodedURL}s.
+        Call the L{EncodedURL.to_uri} method on each URL to get an HTTP
+        protocol-friendly URI.
+        """
+        port = cast(Optional[int], draw(port_numbers(allow_zero=True)))
+        host = cast(Text, draw(hostnames()))
+        path = cast(Sequence[Text], draw(paths()))
+
+        if port == 0:
+            port = None
+
+        return EncodedURL(
+            scheme=cast(Text, draw(sampled_from((u"http", u"https")))),
+            host=host,
+            port=port,
+            path=path,
+        )
+
+    @composite
+    def decoded_urls(draw):
+        # type: (DrawCallable) -> DecodedURL
+        """
+        A strategy which generates L{DecodedURL}s.
+        Call the L{EncodedURL.to_uri} method on each URL to get an HTTP
+        protocol-friendly URI.
+        """
+        return DecodedURL(draw(encoded_urls()))
diff --git a/contrib/python/hyperlink/py2/hyperlink/idna-tables-properties.csv.gz b/contrib/python/hyperlink/py2/hyperlink/idna-tables-properties.csv.gz
new file mode 100644
index 00000000000..48e9f06742f
Binary files /dev/null and b/contrib/python/hyperlink/py2/hyperlink/idna-tables-properties.csv.gz differ
diff --git a/contrib/python/hyperlink/py2/hyperlink/py.typed b/contrib/python/hyperlink/py2/hyperlink/py.typed
new file mode 100644
index 00000000000..d2dfd5e4915
--- /dev/null
+++ b/contrib/python/hyperlink/py2/hyperlink/py.typed
@@ -0,0 +1 @@
+# See: https://www.python.org/dev/peps/pep-0561/
diff --git a/contrib/python/hyperlink/py2/ya.make b/contrib/python/hyperlink/py2/ya.make
new file mode 100644
index 00000000000..5611a958d80
--- /dev/null
+++ b/contrib/python/hyperlink/py2/ya.make
@@ -0,0 +1,36 @@
+# Generated by devtools/yamaker (pypi).
+
+PY2_LIBRARY()
+
+VERSION(21.0.0)
+
+LICENSE(MIT)
+
+PEERDIR(
+    contrib/deprecated/python/typing
+    contrib/python/idna
+)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    hyperlink/__init__.py
+    hyperlink/_socket.py
+    hyperlink/_url.py
+    hyperlink/hypothesis.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/hyperlink/py2/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+    hyperlink/idna-tables-properties.csv.gz
+    hyperlink/py.typed
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/hyperlink/py3/.dist-info/METADATA b/contrib/python/hyperlink/py3/.dist-info/METADATA
new file mode 100644
index 00000000000..fc5922ba875
--- /dev/null
+++ b/contrib/python/hyperlink/py3/.dist-info/METADATA
@@ -0,0 +1,38 @@
+Metadata-Version: 2.1
+Name: hyperlink
+Version: 21.0.0
+Summary: A featureful, immutable, and correct URL for Python.
+Home-page: https://github.com/python-hyper/hyperlink
+Author: Mahmoud Hashemi and Glyph Lefkowitz
+Author-email: mahmoud@hatnote.com
+License: MIT
+Platform: any
+Classifier: Topic :: Utilities
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.6
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Requires-Dist: idna (>=2.5)
+Requires-Dist: typing ; python_version < "3.5"
+
+The humble, but powerful, URL runs everything around us. Chances
+are you've used several just to read this text.
+
+Hyperlink is a featureful, pure-Python implementation of the URL, with
+an emphasis on correctness. MIT licensed.
+
+See the docs at http://hyperlink.readthedocs.io.
+
+
diff --git a/contrib/python/hyperlink/py3/.dist-info/top_level.txt b/contrib/python/hyperlink/py3/.dist-info/top_level.txt
new file mode 100644
index 00000000000..81722ce1d88
--- /dev/null
+++ b/contrib/python/hyperlink/py3/.dist-info/top_level.txt
@@ -0,0 +1 @@
+hyperlink
diff --git a/contrib/python/hyperlink/py3/LICENSE b/contrib/python/hyperlink/py3/LICENSE
new file mode 100644
index 00000000000..a73f882ffb0
--- /dev/null
+++ b/contrib/python/hyperlink/py3/LICENSE
@@ -0,0 +1,29 @@
+Copyright (c) 2017
+Glyph Lefkowitz
+Itamar Turner-Trauring
+Jean Paul Calderone
+Adi Roiban
+Amber Hawkie Brown
+Mahmoud Hashemi
+Wilfredo Sanchez Vega
+
+and others that have contributed code to the public domain.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/python/hyperlink/py3/README.md b/contrib/python/hyperlink/py3/README.md
new file mode 100644
index 00000000000..017f9eb88c1
--- /dev/null
+++ b/contrib/python/hyperlink/py3/README.md
@@ -0,0 +1,67 @@
+# Hyperlink
+
+*Cool URLs that don't change.*
+
+<a href="https://hyperlink.readthedocs.io/en/latest/">
+    <img src="https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat" alt="Documentation">
+</a>
+<a href="https://pypi.org/project/hyperlink/">
+    <img src="https://img.shields.io/pypi/v/hyperlink.svg" alt="PyPI">
+</a>
+<a href="http://calver.org">
+    <img src="https://img.shields.io/badge/calver-YY.MINOR.MICRO-22bfda.svg" alt="Calendar Versioning">
+</a>
+<a href="https://pypi.org/project/hyperlink/">
+    <img src="https://img.shields.io/pypi/pyversions/hyperlink.svg" alt="Python Version Compatibility">
+</a>
+<a href="https://https://codecov.io/github/python-hyper/hyperlink?branch=master">
+    <img src="https://codecov.io/github/python-hyper/hyperlink/coverage.svg?branch=master" alt="Code Coverage">
+</a>
+<a href="https://requires.io/github/python-hyper/hyperlink/requirements/?branch=master">
+    <img src="https://requires.io/github/python-hyper/hyperlink/requirements.svg?branch=master" alt="Requirements Status">
+</a>
+
+Hyperlink provides a pure-Python implementation of immutable
+URLs. Based on [RFC 3986][rfc3986] and [3987][rfc3987], the Hyperlink URL
+makes working with both URIs and IRIs easy.
+
+Hyperlink is tested against Python 2.7, 3.4, 3.5, 3.6, 3.7, 3.8, and PyPy.
+
+Full documentation is available on [Read the Docs][docs].
+
+[rfc3986]: https://tools.ietf.org/html/rfc3986
+[rfc3987]: https://tools.ietf.org/html/rfc3987
+[docs]: http://hyperlink.readthedocs.io/en/latest/
+
+## Installation
+
+Hyperlink is a pure-Python package and requires nothing but
+Python. The easiest way to install is with pip:
+
+```
+pip install hyperlink
+```
+
+Then, hyperlink away!
+
+```python
+from hyperlink import URL
+
+url = URL.from_text(u'http://github.com/python-hyper/hyperlink?utm_source=README')
+utm_source = url.get(u'utm_source')
+better_url = url.replace(scheme=u'https', port=443)
+org_url = better_url.click(u'.')
+```
+
+See the full API docs on [Read the Docs][docs].
+
+## More information
+
+Hyperlink would not have been possible without the help of
+[Glyph Lefkowitz](https://glyph.twistedmatrix.com/) and many other
+community members, especially considering that it started as an
+extract from the Twisted networking library. Thanks to them,
+Hyperlink's URL has been production-grade for well over a decade.
+
+Still, should you encounter any issues, do file an issue, or submit a
+pull request.
diff --git a/contrib/python/hyperlink/py3/hyperlink/__init__.py b/contrib/python/hyperlink/py3/hyperlink/__init__.py
new file mode 100644
index 00000000000..f680b01a905
--- /dev/null
+++ b/contrib/python/hyperlink/py3/hyperlink/__init__.py
@@ -0,0 +1,17 @@
+from ._url import (
+    parse,
+    register_scheme,
+    URL,
+    EncodedURL,
+    DecodedURL,
+    URLParseError,
+)
+
+__all__ = (
+    "parse",
+    "register_scheme",
+    "URL",
+    "EncodedURL",
+    "DecodedURL",
+    "URLParseError",
+)
diff --git a/contrib/python/hyperlink/py3/hyperlink/_socket.py b/contrib/python/hyperlink/py3/hyperlink/_socket.py
new file mode 100644
index 00000000000..3bcf89706df
--- /dev/null
+++ b/contrib/python/hyperlink/py3/hyperlink/_socket.py
@@ -0,0 +1,53 @@
+try:
+    from socket import inet_pton
+except ImportError:
+    from typing import TYPE_CHECKING
+
+    if TYPE_CHECKING:  # pragma: no cover
+        pass
+    else:
+        # based on https://gist.github.com/nnemkin/4966028
+        # this code only applies on Windows Python 2.7
+        import ctypes
+        import socket
+
+        class SockAddr(ctypes.Structure):
+            _fields_ = [
+                ("sa_family", ctypes.c_short),
+                ("__pad1", ctypes.c_ushort),
+                ("ipv4_addr", ctypes.c_byte * 4),
+                ("ipv6_addr", ctypes.c_byte * 16),
+                ("__pad2", ctypes.c_ulong),
+            ]
+
+        WSAStringToAddressA = ctypes.windll.ws2_32.WSAStringToAddressA
+        WSAAddressToStringA = ctypes.windll.ws2_32.WSAAddressToStringA
+
+        def inet_pton(address_family, ip_string):
+            # type: (int, str) -> bytes
+            addr = SockAddr()
+            ip_string_bytes = ip_string.encode("ascii")
+            addr.sa_family = address_family
+            addr_size = ctypes.c_int(ctypes.sizeof(addr))
+
+            try:
+                attribute, size = {
+                    socket.AF_INET: ("ipv4_addr", 4),
+                    socket.AF_INET6: ("ipv6_addr", 16),
+                }[address_family]
+            except KeyError:
+                raise socket.error("unknown address family")
+
+            if (
+                WSAStringToAddressA(
+                    ip_string_bytes,
+                    address_family,
+                    None,
+                    ctypes.byref(addr),
+                    ctypes.byref(addr_size),
+                )
+                != 0
+            ):
+                raise socket.error(ctypes.FormatError())
+
+            return ctypes.string_at(getattr(addr, attribute), size)
diff --git a/contrib/python/hyperlink/py3/hyperlink/_url.py b/contrib/python/hyperlink/py3/hyperlink/_url.py
new file mode 100644
index 00000000000..be69baf696a
--- /dev/null
+++ b/contrib/python/hyperlink/py3/hyperlink/_url.py
@@ -0,0 +1,2448 @@
+# -*- coding: utf-8 -*-
+u"""Hyperlink provides Pythonic URL parsing, construction, and rendering.
+
+Usage is straightforward::
+
+   >>> import hyperlink
+   >>> url = hyperlink.parse(u'http://github.com/mahmoud/hyperlink?utm_source=docs')
+   >>> url.host
+   u'github.com'
+   >>> secure_url = url.replace(scheme=u'https')
+   >>> secure_url.get('utm_source')[0]
+   u'docs'
+
+Hyperlink's API centers on the :class:`DecodedURL` type, which wraps
+the lower-level :class:`URL`, both of which can be returned by the
+:func:`parse()` convenience function.
+
+"""  # noqa: E501
+
+import re
+import sys
+import string
+import socket
+from socket import AF_INET, AF_INET6
+
+try:
+    from socket import AddressFamily
+except ImportError:
+    AddressFamily = int  # type: ignore[assignment,misc]
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Text,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+)
+from unicodedata import normalize
+from ._socket import inet_pton
+
+try:
+    from collections.abc import Mapping as MappingABC
+except ImportError:  # Python 2
+    from collections import Mapping as MappingABC
+
+from idna import encode as idna_encode, decode as idna_decode
+
+
+PY2 = sys.version_info[0] == 2
+try:
+    unichr
+except NameError:  # Py3
+    unichr = chr  # type: Callable[[int], Text]
+NoneType = type(None)  # type: Type[None]
+QueryPairs = Tuple[Tuple[Text, Optional[Text]], ...]  # internal representation
+QueryParameters = Union[
+    Mapping[Text, Optional[Text]],
+    QueryPairs,
+    Sequence[Tuple[Text, Optional[Text]]],
+]
+T = TypeVar("T")
+
+
+# from boltons.typeutils
+def make_sentinel(name="_MISSING", var_name=""):
+    # type: (str, str) -> object
+    """Creates and returns a new **instance** of a new class, suitable for
+    usage as a "sentinel", a kind of singleton often used to indicate
+    a value is missing when ``None`` is a valid input.
+
+    Args:
+        name: Name of the Sentinel
+        var_name: Set this name to the name of the variable in its respective
+            module enable pickle-ability.
+
+    >>> make_sentinel(var_name='_MISSING')
+    _MISSING
+
+    The most common use cases here in boltons are as default values
+    for optional function arguments, partly because of its
+    less-confusing appearance in automatically generated
+    documentation. Sentinels also function well as placeholders in queues
+    and linked lists.
+
+    .. note::
+
+        By design, additional calls to ``make_sentinel`` with the same
+        values will not produce equivalent objects.
+
+        >>> make_sentinel('TEST') == make_sentinel('TEST')
+        False
+        >>> type(make_sentinel('TEST')) == type(make_sentinel('TEST'))
+        False
+    """
+
+    class Sentinel(object):
+        def __init__(self):
+            # type: () -> None
+            self.name = name
+            self.var_name = var_name
+
+        def __repr__(self):
+            # type: () -> str
+            if self.var_name:
+                return self.var_name
+            return "%s(%r)" % (self.__class__.__name__, self.name)
+
+        if var_name:
+            # superclass type hints don't allow str return type, but it is
+            # allowed in the docs, hence the ignore[override] below
+            def __reduce__(self):
+                # type: () -> str
+                return self.var_name
+
+        def __nonzero__(self):
+            # type: () -> bool
+            return False
+
+        __bool__ = __nonzero__
+
+    return Sentinel()
+
+
+_unspecified = _UNSET = make_sentinel("_UNSET")  # type: Any
+
+
+# RFC 3986 Section 2.3, Unreserved URI Characters
+#   https://tools.ietf.org/html/rfc3986#section-2.3
+_UNRESERVED_CHARS = frozenset(
+    "~-._0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
+)
+
+
+# URL parsing regex (based on RFC 3986 Appendix B, with modifications)
+_URL_RE = re.compile(
+    r"^((?P<scheme>[^:/?#]+):)?"
+    r"((?P<_netloc_sep>//)"
+    r"(?P<authority>[^/?#]*))?"
+    r"(?P<path>[^?#]*)"
+    r"(\?(?P<query>[^#]*))?"
+    r"(#(?P<fragment>.*))?$"
+)
+_SCHEME_RE = re.compile(r"^[a-zA-Z0-9+-.]*$")
+_AUTHORITY_RE = re.compile(
+    r"^(?:(?P<userinfo>[^@/?#]*)@)?"
+    r"(?P<host>"
+    r"(?:\[(?P<ipv6_host>[^[\]/?#]*)\])"
+    r"|(?P<plain_host>[^:/?#[\]]*)"
+    r"|(?P<bad_host>.*?))?"
+    r"(?::(?P<port>.*))?$"
+)
+
+
+_HEX_CHAR_MAP = dict(
+    [
+        ((a + b).encode("ascii"), unichr(int(a + b, 16)).encode("charmap"))
+        for a in string.hexdigits
+        for b in string.hexdigits
+    ]
+)
+_ASCII_RE = re.compile("([\x00-\x7f]+)")
+
+# RFC 3986 section 2.2, Reserved Characters
+#   https://tools.ietf.org/html/rfc3986#section-2.2
+_GEN_DELIMS = frozenset(u":/?#[]@")
+_SUB_DELIMS = frozenset(u"!$&'()*+,;=")
+_ALL_DELIMS = _GEN_DELIMS | _SUB_DELIMS
+
+_USERINFO_SAFE = _UNRESERVED_CHARS | _SUB_DELIMS | set(u"%")
+_USERINFO_DELIMS = _ALL_DELIMS - _USERINFO_SAFE
+_PATH_SAFE = _USERINFO_SAFE | set(u":@")
+_PATH_DELIMS = _ALL_DELIMS - _PATH_SAFE
+_SCHEMELESS_PATH_SAFE = _PATH_SAFE - set(":")
+_SCHEMELESS_PATH_DELIMS = _ALL_DELIMS - _SCHEMELESS_PATH_SAFE
+_FRAGMENT_SAFE = _UNRESERVED_CHARS | _PATH_SAFE | set(u"/?")
+_FRAGMENT_DELIMS = _ALL_DELIMS - _FRAGMENT_SAFE
+_QUERY_VALUE_SAFE = _UNRESERVED_CHARS | _FRAGMENT_SAFE - set(u"&")
+_QUERY_VALUE_DELIMS = _ALL_DELIMS - _QUERY_VALUE_SAFE
+_QUERY_KEY_SAFE = _UNRESERVED_CHARS | _QUERY_VALUE_SAFE - set(u"=")
+_QUERY_KEY_DELIMS = _ALL_DELIMS - _QUERY_KEY_SAFE
+
+
+def _make_decode_map(delims, allow_percent=False):
+    # type: (Iterable[Text], bool) -> Mapping[bytes, bytes]
+    ret = dict(_HEX_CHAR_MAP)
+    if not allow_percent:
+        delims = set(delims) | set([u"%"])
+    for delim in delims:
+        _hexord = "{0:02X}".format(ord(delim)).encode("ascii")
+        _hexord_lower = _hexord.lower()
+        ret.pop(_hexord)
+        if _hexord != _hexord_lower:
+            ret.pop(_hexord_lower)
+    return ret
+
+
+def _make_quote_map(safe_chars):
+    # type: (Iterable[Text]) -> Mapping[Union[int, Text], Text]
+    ret = {}  # type: Dict[Union[int, Text], Text]
+    # v is included in the dict for py3 mostly, because bytestrings
+    # are iterables of ints, of course!
+    for i, v in zip(range(256), range(256)):
+        c = chr(v)
+        if c in safe_chars:
+            ret[c] = ret[v] = c
+        else:
+            ret[c] = ret[v] = "%{0:02X}".format(i)
+    return ret
+
+
+_USERINFO_PART_QUOTE_MAP = _make_quote_map(_USERINFO_SAFE)
+_USERINFO_DECODE_MAP = _make_decode_map(_USERINFO_DELIMS)
+_PATH_PART_QUOTE_MAP = _make_quote_map(_PATH_SAFE)
+_SCHEMELESS_PATH_PART_QUOTE_MAP = _make_quote_map(_SCHEMELESS_PATH_SAFE)
+_PATH_DECODE_MAP = _make_decode_map(_PATH_DELIMS)
+_QUERY_KEY_QUOTE_MAP = _make_quote_map(_QUERY_KEY_SAFE)
+_QUERY_KEY_DECODE_MAP = _make_decode_map(_QUERY_KEY_DELIMS)
+_QUERY_VALUE_QUOTE_MAP = _make_quote_map(_QUERY_VALUE_SAFE)
+_QUERY_VALUE_DECODE_MAP = _make_decode_map(_QUERY_VALUE_DELIMS)
+_FRAGMENT_QUOTE_MAP = _make_quote_map(_FRAGMENT_SAFE)
+_FRAGMENT_DECODE_MAP = _make_decode_map(_FRAGMENT_DELIMS)
+_UNRESERVED_QUOTE_MAP = _make_quote_map(_UNRESERVED_CHARS)
+_UNRESERVED_DECODE_MAP = dict(
+    [
+        (k, v)
+        for k, v in _HEX_CHAR_MAP.items()
+        if v.decode("ascii", "replace") in _UNRESERVED_CHARS
+    ]
+)
+
+_ROOT_PATHS = frozenset(((), (u"",)))
+
+
+def _encode_reserved(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """A very comprehensive percent encoding for encoding all
+    delimiters. Used for arguments to DecodedURL, where a % means a
+    percent sign, and not the character used by URLs for escaping
+    bytes.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_UNRESERVED_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [
+            _UNRESERVED_QUOTE_MAP[t] if t in _UNRESERVED_CHARS else t
+            for t in text
+        ]
+    )
+
+
+def _encode_path_part(text, maximal=True):
+    # type: (Text, bool) -> Text
+    "Percent-encode a single segment of a URL path."
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_PATH_PART_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [_PATH_PART_QUOTE_MAP[t] if t in _PATH_DELIMS else t for t in text]
+    )
+
+
+def _encode_schemeless_path_part(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """Percent-encode the first segment of a URL path for a URL without a
+    scheme specified.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_SCHEMELESS_PATH_PART_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [
+            _SCHEMELESS_PATH_PART_QUOTE_MAP[t]
+            if t in _SCHEMELESS_PATH_DELIMS
+            else t
+            for t in text
+        ]
+    )
+
+
+def _encode_path_parts(
+    text_parts,  # type: Sequence[Text]
+    rooted=False,  # type: bool
+    has_scheme=True,  # type: bool
+    has_authority=True,  # type: bool
+    maximal=True,  # type: bool
+):
+    # type: (...) -> Sequence[Text]
+    """
+    Percent-encode a tuple of path parts into a complete path.
+
+    Setting *maximal* to False percent-encodes only the reserved
+    characters that are syntactically necessary for serialization,
+    preserving any IRI-style textual data.
+
+    Leaving *maximal* set to its default True percent-encodes
+    everything required to convert a portion of an IRI to a portion of
+    a URI.
+
+    RFC 3986 3.3:
+
+       If a URI contains an authority component, then the path component
+       must either be empty or begin with a slash ("/") character.  If a URI
+       does not contain an authority component, then the path cannot begin
+       with two slash characters ("//").  In addition, a URI reference
+       (Section 4.1) may be a relative-path reference, in which case the
+       first path segment cannot contain a colon (":") character.
+    """
+    if not text_parts:
+        return ()
+    if rooted:
+        text_parts = (u"",) + tuple(text_parts)
+    # elif has_authority and text_parts:
+    #     raise Exception('see rfc above')  # TODO: too late to fail like this?
+    encoded_parts = []  # type: List[Text]
+    if has_scheme:
+        encoded_parts = [
+            _encode_path_part(part, maximal=maximal) if part else part
+            for part in text_parts
+        ]
+    else:
+        encoded_parts = [_encode_schemeless_path_part(text_parts[0])]
+        encoded_parts.extend(
+            [
+                _encode_path_part(part, maximal=maximal) if part else part
+                for part in text_parts[1:]
+            ]
+        )
+    return tuple(encoded_parts)
+
+
+def _encode_query_key(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """
+    Percent-encode a single query string key or value.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_QUERY_KEY_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [_QUERY_KEY_QUOTE_MAP[t] if t in _QUERY_KEY_DELIMS else t for t in text]
+    )
+
+
+def _encode_query_value(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """
+    Percent-encode a single query string key or value.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_QUERY_VALUE_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [
+            _QUERY_VALUE_QUOTE_MAP[t] if t in _QUERY_VALUE_DELIMS else t
+            for t in text
+        ]
+    )
+
+
+def _encode_fragment_part(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """Quote the fragment part of the URL. Fragments don't have
+    subdelimiters, so the whole URL fragment can be passed.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_FRAGMENT_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [_FRAGMENT_QUOTE_MAP[t] if t in _FRAGMENT_DELIMS else t for t in text]
+    )
+
+
+def _encode_userinfo_part(text, maximal=True):
+    # type: (Text, bool) -> Text
+    """Quote special characters in either the username or password
+    section of the URL.
+    """
+    if maximal:
+        bytestr = normalize("NFC", text).encode("utf8")
+        return u"".join([_USERINFO_PART_QUOTE_MAP[b] for b in bytestr])
+    return u"".join(
+        [
+            _USERINFO_PART_QUOTE_MAP[t] if t in _USERINFO_DELIMS else t
+            for t in text
+        ]
+    )
+
+
+# This port list painstakingly curated by hand searching through
+# https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml
+# and
+# https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml
+SCHEME_PORT_MAP = {
+    "acap": 674,
+    "afp": 548,
+    "dict": 2628,
+    "dns": 53,
+    "file": None,
+    "ftp": 21,
+    "git": 9418,
+    "gopher": 70,
+    "http": 80,
+    "https": 443,
+    "imap": 143,
+    "ipp": 631,
+    "ipps": 631,
+    "irc": 194,
+    "ircs": 6697,
+    "ldap": 389,
+    "ldaps": 636,
+    "mms": 1755,
+    "msrp": 2855,
+    "msrps": None,
+    "mtqp": 1038,
+    "nfs": 111,
+    "nntp": 119,
+    "nntps": 563,
+    "pop": 110,
+    "prospero": 1525,
+    "redis": 6379,
+    "rsync": 873,
+    "rtsp": 554,
+    "rtsps": 322,
+    "rtspu": 5005,
+    "sftp": 22,
+    "smb": 445,
+    "snmp": 161,
+    "ssh": 22,
+    "steam": None,
+    "svn": 3690,
+    "telnet": 23,
+    "ventrilo": 3784,
+    "vnc": 5900,
+    "wais": 210,
+    "ws": 80,
+    "wss": 443,
+    "xmpp": None,
+}
+
+# This list of schemes that don't use authorities is also from the link above.
+NO_NETLOC_SCHEMES = set(
+    [
+        "urn",
+        "about",
+        "bitcoin",
+        "blob",
+        "data",
+        "geo",
+        "magnet",
+        "mailto",
+        "news",
+        "pkcs11",
+        "sip",
+        "sips",
+        "tel",
+    ]
+)
+# As of Mar 11, 2017, there were 44 netloc schemes, and 13 non-netloc
+
+NO_QUERY_PLUS_SCHEMES = set()
+
+
+def register_scheme(
+    text, uses_netloc=True, default_port=None, query_plus_is_space=True
+):
+    # type: (Text, bool, Optional[int], bool) -> None
+    """Registers new scheme information, resulting in correct port and
+    slash behavior from the URL object. There are dozens of standard
+    schemes preregistered, so this function is mostly meant for
+    proprietary internal customizations or stopgaps on missing
+    standards information. If a scheme seems to be missing, please
+    `file an issue`_!
+
+    Args:
+        text: A string representation of the scheme.
+            (the 'http' in 'http://hatnote.com')
+        uses_netloc: Does the scheme support specifying a
+            network host? For instance, "http" does, "mailto" does
+            not. Defaults to True.
+        default_port: The default port, if any, for
+            netloc-using schemes.
+        query_plus_is_space: If true, a "+" in the query string should be
+            decoded as a space by DecodedURL.
+
+    .. _file an issue: https://github.com/mahmoud/hyperlink/issues
+    """
+    text = text.lower()
+    if default_port is not None:
+        try:
+            default_port = int(default_port)
+        except (ValueError, TypeError):
+            raise ValueError(
+                "default_port expected integer or None, not %r"
+                % (default_port,)
+            )
+
+    if uses_netloc is True:
+        SCHEME_PORT_MAP[text] = default_port
+    elif uses_netloc is False:
+        if default_port is not None:
+            raise ValueError(
+                "unexpected default port while specifying"
+                " non-netloc scheme: %r" % default_port
+            )
+        NO_NETLOC_SCHEMES.add(text)
+    else:
+        raise ValueError("uses_netloc expected bool, not: %r" % uses_netloc)
+
+    if not query_plus_is_space:
+        NO_QUERY_PLUS_SCHEMES.add(text)
+
+    return
+
+
+def scheme_uses_netloc(scheme, default=None):
+    # type: (Text, Optional[bool]) -> Optional[bool]
+    """Whether or not a URL uses :code:`:` or :code:`://` to separate the
+    scheme from the rest of the URL depends on the scheme's own
+    standard definition. There is no way to infer this behavior
+    from other parts of the URL. A scheme either supports network
+    locations or it does not.
+
+    The URL type's approach to this is to check for explicitly
+    registered schemes, with common schemes like HTTP
+    preregistered. This is the same approach taken by
+    :mod:`urlparse`.
+
+    URL adds two additional heuristics if the scheme as a whole is
+    not registered. First, it attempts to check the subpart of the
+    scheme after the last ``+`` character. This adds intuitive
+    behavior for schemes like ``git+ssh``. Second, if a URL with
+    an unrecognized scheme is loaded, it will maintain the
+    separator it sees.
+    """
+    if not scheme:
+        return False
+    scheme = scheme.lower()
+    if scheme in SCHEME_PORT_MAP:
+        return True
+    if scheme in NO_NETLOC_SCHEMES:
+        return False
+    if scheme.split("+")[-1] in SCHEME_PORT_MAP:
+        return True
+    return default
+
+
+class URLParseError(ValueError):
+    """Exception inheriting from :exc:`ValueError`, raised when failing to
+    parse a URL. Mostly raised on invalid ports and IPv6 addresses.
+    """
+
+    pass
+
+
+def _optional(argument, default):
+    # type: (Any, Any) -> Any
+    if argument is _UNSET:
+        return default
+    else:
+        return argument
+
+
+def _typecheck(name, value, *types):
+    # type: (Text, T, Type[Any]) -> T
+    """
+    Check that the given *value* is one of the given *types*, or raise an
+    exception describing the problem using *name*.
+    """
+    if not types:
+        raise ValueError("expected one or more types, maybe use _textcheck?")
+    if not isinstance(value, types):
+        raise TypeError(
+            "expected %s for %s, got %r"
+            % (" or ".join([t.__name__ for t in types]), name, value)
+        )
+    return value
+
+
+def _textcheck(name, value, delims=frozenset(), nullable=False):
+    # type: (Text, T, Iterable[Text], bool) -> T
+    if not isinstance(value, Text):
+        if nullable and value is None:
+            # used by query string values
+            return value  # type: ignore[unreachable]
+        else:
+            str_name = "unicode" if PY2 else "str"
+            exp = str_name + " or NoneType" if nullable else str_name
+            raise TypeError("expected %s for %s, got %r" % (exp, name, value))
+    if delims and set(value) & set(delims):  # TODO: test caching into regexes
+        raise ValueError(
+            "one or more reserved delimiters %s present in %s: %r"
+            % ("".join(delims), name, value)
+        )
+    return value  # type: ignore[return-value] # T vs. Text
+
+
+def iter_pairs(iterable):
+    # type: (Iterable[Any]) -> Iterator[Any]
+    """
+    Iterate over the (key, value) pairs in ``iterable``.
+
+    This handles dictionaries sensibly, and falls back to assuming the
+    iterable yields (key, value) pairs. This behaviour is similar to
+    what Python's ``dict()`` constructor does.
+    """
+    if isinstance(iterable, MappingABC):
+        iterable = iterable.items()
+    return iter(iterable)
+
+
+def _decode_unreserved(text, normalize_case=False, encode_stray_percents=False):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_UNRESERVED_DECODE_MAP,
+    )
+
+
+def _decode_userinfo_part(
+    text, normalize_case=False, encode_stray_percents=False
+):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_USERINFO_DECODE_MAP,
+    )
+
+
+def _decode_path_part(text, normalize_case=False, encode_stray_percents=False):
+    # type: (Text, bool, bool) -> Text
+    """
+    >>> _decode_path_part(u'%61%77%2f%7a')
+    u'aw%2fz'
+    >>> _decode_path_part(u'%61%77%2f%7a', normalize_case=True)
+    u'aw%2Fz'
+    """
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_PATH_DECODE_MAP,
+    )
+
+
+def _decode_query_key(text, normalize_case=False, encode_stray_percents=False):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_QUERY_KEY_DECODE_MAP,
+    )
+
+
+def _decode_query_value(
+    text, normalize_case=False, encode_stray_percents=False
+):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_QUERY_VALUE_DECODE_MAP,
+    )
+
+
+def _decode_fragment_part(
+    text, normalize_case=False, encode_stray_percents=False
+):
+    # type: (Text, bool, bool) -> Text
+    return _percent_decode(
+        text,
+        normalize_case=normalize_case,
+        encode_stray_percents=encode_stray_percents,
+        _decode_map=_FRAGMENT_DECODE_MAP,
+    )
+
+
+def _percent_decode(
+    text,  # type: Text
+    normalize_case=False,  # type: bool
+    subencoding="utf-8",  # type: Text
+    raise_subencoding_exc=False,  # type: bool
+    encode_stray_percents=False,  # type: bool
+    _decode_map=_HEX_CHAR_MAP,  # type: Mapping[bytes, bytes]
+):
+    # type: (...) -> Text
+    """Convert percent-encoded text characters to their normal,
+    human-readable equivalents.
+
+    All characters in the input text must be encodable by
+    *subencoding*. All special characters underlying the values in the
+    percent-encoding must be decodable as *subencoding*. If a
+    non-*subencoding*-valid string is passed, the original text is
+    returned with no changes applied.
+
+    Only called by field-tailored variants, e.g.,
+    :func:`_decode_path_part`, as every percent-encodable part of the
+    URL has characters which should not be percent decoded.
+
+    >>> _percent_decode(u'abc%20def')
+    u'abc def'
+
+    Args:
+        text: Text with percent-encoding present.
+        normalize_case: Whether undecoded percent segments, such as encoded
+            delimiters, should be uppercased, per RFC 3986 Section 2.1.
+            See :func:`_decode_path_part` for an example.
+        subencoding: The name of the encoding underlying the percent-encoding.
+        raise_subencoding_exc: Whether an error in decoding the bytes
+            underlying the percent-decoding should be raised.
+
+    Returns:
+        Text: The percent-decoded version of *text*, decoded by *subencoding*.
+    """
+    try:
+        quoted_bytes = text.encode(subencoding)
+    except UnicodeEncodeError:
+        return text
+
+    bits = quoted_bytes.split(b"%")
+    if len(bits) == 1:
+        return text
+
+    res = [bits[0]]
+    append = res.append
+
+    for item in bits[1:]:
+        hexpair, rest = item[:2], item[2:]
+        try:
+            append(_decode_map[hexpair])
+            append(rest)
+        except KeyError:
+            pair_is_hex = hexpair in _HEX_CHAR_MAP
+            if pair_is_hex or not encode_stray_percents:
+                append(b"%")
+            else:
+                # if it's undecodable, treat as a real percent sign,
+                # which is reserved (because it wasn't in the
+                # context-aware _decode_map passed in), and should
+                # stay in an encoded state.
+                append(b"%25")
+            if normalize_case and pair_is_hex:
+                append(hexpair.upper())
+                append(rest)
+            else:
+                append(item)
+
+    unquoted_bytes = b"".join(res)
+
+    try:
+        return unquoted_bytes.decode(subencoding)
+    except UnicodeDecodeError:
+        if raise_subencoding_exc:
+            raise
+        return text
+
+
+def _decode_host(host):
+    # type: (Text) -> Text
+    """Decode a host from ASCII-encodable text to IDNA-decoded text. If
+    the host text is not ASCII, it is returned unchanged, as it is
+    presumed that it is already IDNA-decoded.
+
+    Some technical details: _decode_host is built on top of the "idna"
+    package, which has some quirks:
+
+    Capital letters are not valid IDNA2008. The idna package will
+    raise an exception like this on capital letters:
+
+    > idna.core.InvalidCodepoint: Codepoint U+004B at position 1 ... not allowed
+
+    However, if a segment of a host (i.e., something in
+    url.host.split('.')) is already ASCII, idna doesn't perform its
+    usual checks. In fact, for capital letters it automatically
+    lowercases them.
+
+    This check and some other functionality can be bypassed by passing
+    uts46=True to idna.encode/decode. This allows a more permissive and
+    convenient interface. So far it seems like the balanced approach.
+
+    Example output (from idna==2.6):
+
+    >> idna.encode(u'mahmöud.io')
+    'xn--mahmud-zxa.io'
+    >> idna.encode(u'Mahmöud.io')
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 355, in encode
+        result.append(alabel(label))
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 276, in alabel
+        check_label(label)
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 253, in check_label
+        raise InvalidCodepoint('Codepoint {0} at position {1} of {2} not allowed'.format(_unot(cp_value), pos+1, repr(label)))
+    idna.core.InvalidCodepoint: Codepoint U+004D at position 1 of u'Mahm\xf6ud' not allowed
+    >> idna.encode(u'Mahmoud.io')
+    'Mahmoud.io'
+
+    # Similar behavior for decodes below
+    >> idna.decode(u'Mahmoud.io')
+    u'mahmoud.io
+    >> idna.decode(u'Méhmoud.io', uts46=True)
+    u'm\xe9hmoud.io'
+    """  # noqa: E501
+    if not host:
+        return u""
+    try:
+        host_bytes = host.encode("ascii")
+    except UnicodeEncodeError:
+        host_text = host
+    else:
+        try:
+            host_text = idna_decode(host_bytes, uts46=True)
+        except ValueError:
+            # only reached on "narrow" (UCS-2) Python builds <3.4, see #7
+            # NOTE: not going to raise here, because there's no
+            # ambiguity in the IDNA, and the host is still
+            # technically usable
+            host_text = host
+    return host_text
+
+
+def _resolve_dot_segments(path):
+    # type: (Sequence[Text]) -> Sequence[Text]
+    """Normalize the URL path by resolving segments of '.' and '..'. For
+    more details, see `RFC 3986 section 5.2.4, Remove Dot Segments`_.
+
+    Args:
+       path: sequence of path segments in text form
+
+    Returns:
+       A new sequence of path segments with the '.' and '..' elements removed
+           and resolved.
+
+    .. _RFC 3986 section 5.2.4, Remove Dot Segments: https://tools.ietf.org/html/rfc3986#section-5.2.4
+    """  # noqa: E501
+    segs = []  # type: List[Text]
+
+    for seg in path:
+        if seg == u".":
+            pass
+        elif seg == u"..":
+            if segs:
+                segs.pop()
+        else:
+            segs.append(seg)
+
+    if list(path[-1:]) in ([u"."], [u".."]):
+        segs.append(u"")
+
+    return segs
+
+
+def parse_host(host):
+    # type: (Text) -> Tuple[Optional[AddressFamily], Text]
+    """Parse the host into a tuple of ``(family, host)``, where family
+    is the appropriate :mod:`socket` module constant when the host is
+    an IP address. Family is ``None`` when the host is not an IP.
+
+    Will raise :class:`URLParseError` on invalid IPv6 constants.
+
+    Returns:
+        family (socket constant or None), host (string)
+
+    >>> import socket
+    >>> parse_host('googlewebsite.com') == (None, 'googlewebsite.com')
+    True
+    >>> parse_host('::1') == (socket.AF_INET6, '::1')
+    True
+    >>> parse_host('192.168.1.1') == (socket.AF_INET, '192.168.1.1')
+    True
+    """
+    if not host:
+        return None, u""
+
+    if u":" in host:
+        try:
+            inet_pton(AF_INET6, host)
+        except socket.error as se:
+            raise URLParseError("invalid IPv6 host: %r (%r)" % (host, se))
+        except UnicodeEncodeError:
+            pass  # TODO: this can't be a real host right?
+        else:
+            family = AF_INET6  # type: Optional[AddressFamily]
+    else:
+        try:
+            inet_pton(AF_INET, host)
+        except (socket.error, UnicodeEncodeError):
+            family = None  # not an IP
+        else:
+            family = AF_INET
+
+    return family, host
+
+
+class URL(object):
+    r"""From blogs to billboards, URLs are so common, that it's easy to
+    overlook their complexity and power. With hyperlink's
+    :class:`URL` type, working with URLs doesn't have to be hard.
+
+    URLs are made of many parts. Most of these parts are officially
+    named in `RFC 3986`_ and this diagram may prove handy in identifying
+    them::
+
+       foo://user:pass@example.com:8042/over/there?name=ferret#nose
+       \_/   \_______/ \_________/ \__/\_________/ \_________/ \__/
+        |        |          |        |      |           |        |
+      scheme  userinfo     host     port   path       query   fragment
+
+    While :meth:`~URL.from_text` is used for parsing whole URLs, the
+    :class:`URL` constructor builds a URL from the individual
+    components, like so::
+
+        >>> from hyperlink import URL
+        >>> url = URL(scheme=u'https', host=u'example.com', path=[u'hello', u'world'])
+        >>> print(url.to_text())
+        https://example.com/hello/world
+
+    The constructor runs basic type checks. All strings are expected
+    to be text (:class:`str` in Python 3, :class:`unicode` in Python 2). All
+    arguments are optional, defaulting to appropriately empty values. A full
+    list of constructor arguments is below.
+
+    Args:
+        scheme: The text name of the scheme.
+        host: The host portion of the network location
+        port: The port part of the network location. If ``None`` or no port is
+            passed, the port will default to the default port of the scheme, if
+            it is known. See the ``SCHEME_PORT_MAP`` and
+            :func:`register_default_port` for more info.
+        path: A tuple of strings representing the slash-separated parts of the
+            path, each percent-encoded.
+        query: The query parameters, as a dictionary or as an sequence of
+            percent-encoded key-value pairs.
+        fragment: The fragment part of the URL.
+        rooted: A rooted URL is one which indicates an absolute path.
+            This is True on any URL that includes a host, or any relative URL
+            that starts with a slash.
+        userinfo: The username or colon-separated username:password pair.
+        uses_netloc: Indicates whether ``://`` (the "netloc separator") will
+            appear to separate the scheme from the *path* in cases where no
+            host is present.
+            Setting this to ``True`` is a non-spec-compliant affordance for the
+            common practice of having URIs that are *not* URLs (cannot have a
+            'host' part) but nevertheless use the common ``://`` idiom that
+            most people associate with URLs; e.g. ``message:`` URIs like
+            ``message://message-id`` being equivalent to ``message:message-id``.
+            This may be inferred based on the scheme depending on whether
+            :func:`register_scheme` has been used to register the scheme and
+            should not be passed directly unless you know the scheme works like
+            this and you know it has not been registered.
+
+    All of these parts are also exposed as read-only attributes of :class:`URL`
+    instances, along with several useful methods.
+
+    .. _RFC 3986: https://tools.ietf.org/html/rfc3986
+    .. _RFC 3987: https://tools.ietf.org/html/rfc3987
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        scheme=None,  # type: Optional[Text]
+        host=None,  # type: Optional[Text]
+        path=(),  # type: Iterable[Text]
+        query=(),  # type: QueryParameters
+        fragment=u"",  # type: Text
+        port=None,  # type: Optional[int]
+        rooted=None,  # type: Optional[bool]
+        userinfo=u"",  # type: Text
+        uses_netloc=None,  # type: Optional[bool]
+    ):
+        # type: (...) -> None
+        if host is not None and scheme is None:
+            scheme = u"http"  # TODO: why
+        if port is None and scheme is not None:
+            port = SCHEME_PORT_MAP.get(scheme)
+        if host and query and not path:
+            # per RFC 3986 6.2.3, "a URI that uses the generic syntax
+            # for authority with an empty path should be normalized to
+            # a path of '/'."
+            path = (u"",)
+
+        # Now that we're done detecting whether they were passed, we can set
+        # them to their defaults:
+        if scheme is None:
+            scheme = u""
+        if host is None:
+            host = u""
+        if rooted is None:
+            rooted = bool(host)
+
+        # Set attributes.
+        self._scheme = _textcheck("scheme", scheme)
+        if self._scheme:
+            if not _SCHEME_RE.match(self._scheme):
+                raise ValueError(
+                    'invalid scheme: %r. Only alphanumeric, "+",'
+                    ' "-", and "." allowed. Did you meant to call'
+                    " %s.from_text()?" % (self._scheme, self.__class__.__name__)
+                )
+
+        _, self._host = parse_host(_textcheck("host", host, "/?#@"))
+        if isinstance(path, Text):
+            raise TypeError(
+                "expected iterable of text for path, not: %r" % (path,)
+            )
+        self._path = tuple(
+            (_textcheck("path segment", segment, "/?#") for segment in path)
+        )
+        self._query = tuple(
+            (
+                _textcheck("query parameter name", k, "&=#"),
+                _textcheck("query parameter value", v, "&#", nullable=True),
+            )
+            for k, v in iter_pairs(query)
+        )
+        self._fragment = _textcheck("fragment", fragment)
+        self._port = _typecheck("port", port, int, NoneType)
+        self._rooted = _typecheck("rooted", rooted, bool)
+        self._userinfo = _textcheck("userinfo", userinfo, "/?#@")
+
+        if uses_netloc is None:
+            uses_netloc = scheme_uses_netloc(self._scheme, uses_netloc)
+        self._uses_netloc = _typecheck(
+            "uses_netloc", uses_netloc, bool, NoneType
+        )
+        will_have_authority = self._host or (
+            self._port and self._port != SCHEME_PORT_MAP.get(scheme)
+        )
+        if will_have_authority:
+            # fixup for rooted consistency; if there's any 'authority'
+            # represented in the textual URL, then the path must be rooted, and
+            # we're definitely using a netloc (there must be a ://).
+            self._rooted = True
+            self._uses_netloc = True
+        if (not self._rooted) and self.path[:1] == (u"",):
+            self._rooted = True
+            self._path = self._path[1:]
+        if not will_have_authority and self._path and not self._rooted:
+            # If, after fixing up the path, there *is* a path and it *isn't*
+            # rooted, then we are definitely not using a netloc; if we did, it
+            # would make the path (erroneously) look like a hostname.
+            self._uses_netloc = False
+
+    def get_decoded_url(self, lazy=False):
+        # type: (bool) -> DecodedURL
+        try:
+            return self._decoded_url
+        except AttributeError:
+            self._decoded_url = DecodedURL(self, lazy=lazy)  # type: DecodedURL
+        return self._decoded_url
+
+    @property
+    def scheme(self):
+        # type: () -> Text
+        """The scheme is a string, and the first part of an absolute URL, the
+        part before the first colon, and the part which defines the
+        semantics of the rest of the URL. Examples include "http",
+        "https", "ssh", "file", "mailto", and many others. See
+        :func:`~hyperlink.register_scheme()` for more info.
+        """
+        return self._scheme
+
+    @property
+    def host(self):
+        # type: () -> Text
+        """The host is a string, and the second standard part of an absolute
+        URL. When present, a valid host must be a domain name, or an
+        IP (v4 or v6). It occurs before the first slash, or the second
+        colon, if a :attr:`~hyperlink.URL.port` is provided.
+        """
+        return self._host
+
+    @property
+    def port(self):
+        # type: () -> Optional[int]
+        """The port is an integer that is commonly used in connecting to the
+        :attr:`host`, and almost never appears without it.
+
+        When not present in the original URL, this attribute defaults
+        to the scheme's default port. If the scheme's default port is
+        not known, and the port is not provided, this attribute will
+        be set to None.
+
+        >>> URL.from_text(u'http://example.com/pa/th').port
+        80
+        >>> URL.from_text(u'foo://example.com/pa/th').port
+        >>> URL.from_text(u'foo://example.com:8042/pa/th').port
+        8042
+
+        .. note::
+
+           Per the standard, when the port is the same as the schemes
+           default port, it will be omitted in the text URL.
+        """
+        return self._port
+
+    @property
+    def path(self):
+        # type: () -> Sequence[Text]
+        """A tuple of strings, created by splitting the slash-separated
+        hierarchical path. Started by the first slash after the host,
+        terminated by a "?", which indicates the start of the
+        :attr:`~hyperlink.URL.query` string.
+        """
+        return self._path
+
+    @property
+    def query(self):
+        # type: () -> QueryPairs
+        """Tuple of pairs, created by splitting the ampersand-separated
+        mapping of keys and optional values representing
+        non-hierarchical data used to identify the resource. Keys are
+        always strings. Values are strings when present, or None when
+        missing.
+
+        For more operations on the mapping, see
+        :meth:`~hyperlink.URL.get()`, :meth:`~hyperlink.URL.add()`,
+        :meth:`~hyperlink.URL.set()`, and
+        :meth:`~hyperlink.URL.delete()`.
+        """
+        return self._query
+
+    @property
+    def fragment(self):
+        # type: () -> Text
+        """A string, the last part of the URL, indicated by the first "#"
+        after the :attr:`~hyperlink.URL.path` or
+        :attr:`~hyperlink.URL.query`. Enables indirect identification
+        of a secondary resource, like an anchor within an HTML page.
+        """
+        return self._fragment
+
+    @property
+    def rooted(self):
+        # type: () -> bool
+        """Whether or not the path starts with a forward slash (``/``).
+
+        This is taken from the terminology in the BNF grammar,
+        specifically the "path-rootless", rule, since "absolute path"
+        and "absolute URI" are somewhat ambiguous. :attr:`path` does
+        not contain the implicit prefixed ``"/"`` since that is
+        somewhat awkward to work with.
+        """
+        return self._rooted
+
+    @property
+    def userinfo(self):
+        # type: () -> Text
+        """The colon-separated string forming the username-password
+        combination.
+        """
+        return self._userinfo
+
+    @property
+    def uses_netloc(self):
+        # type: () -> Optional[bool]
+        """
+        Indicates whether ``://`` (the "netloc separator") will appear to
+        separate the scheme from the *path* in cases where no host is present.
+        """
+        return self._uses_netloc
+
+    @property
+    def user(self):
+        # type: () -> Text
+        """
+        The user portion of :attr:`~hyperlink.URL.userinfo`.
+        """
+        return self.userinfo.split(u":")[0]
+
+    def authority(self, with_password=False, **kw):
+        # type: (bool, Any) -> Text
+        """Compute and return the appropriate host/port/userinfo combination.
+
+        >>> url = URL.from_text(u'http://user:pass@localhost:8080/a/b?x=y')
+        >>> url.authority()
+        u'user:@localhost:8080'
+        >>> url.authority(with_password=True)
+        u'user:pass@localhost:8080'
+
+        Args:
+            with_password: Whether the return value of this method include the
+            password in the URL, if it is set.
+            Defaults to False.
+
+        Returns:
+            Text: The authority (network location and user information) portion
+                of the URL.
+        """
+        # first, a bit of twisted compat
+        with_password = kw.pop("includeSecrets", with_password)
+        if kw:
+            raise TypeError("got unexpected keyword arguments: %r" % kw.keys())
+        host = self.host
+        if ":" in host:
+            hostport = ["[" + host + "]"]
+        else:
+            hostport = [self.host]
+        if self.port != SCHEME_PORT_MAP.get(self.scheme):
+            hostport.append(Text(self.port))
+        authority = []
+        if self.userinfo:
+            userinfo = self.userinfo
+            if not with_password and u":" in userinfo:
+                userinfo = userinfo[: userinfo.index(u":") + 1]
+            authority.append(userinfo)
+        authority.append(u":".join(hostport))
+        return u"@".join(authority)
+
+    def __eq__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        for attr in [
+            "scheme",
+            "userinfo",
+            "host",
+            "query",
+            "fragment",
+            "port",
+            "uses_netloc",
+            "rooted",
+        ]:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        if self.path == other.path or (
+            self.path in _ROOT_PATHS and other.path in _ROOT_PATHS
+        ):
+            return True
+        return False
+
+    def __ne__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        # type: () -> int
+        return hash(
+            (
+                self.__class__,
+                self.scheme,
+                self.userinfo,
+                self.host,
+                self.path,
+                self.query,
+                self.fragment,
+                self.port,
+                self.rooted,
+                self.uses_netloc,
+            )
+        )
+
+    @property
+    def absolute(self):
+        # type: () -> bool
+        """Whether or not the URL is "absolute". Absolute URLs are complete
+        enough to resolve to a network resource without being relative
+        to a base URI.
+
+        >>> URL.from_text(u'http://wikipedia.org/').absolute
+        True
+        >>> URL.from_text(u'?a=b&c=d').absolute
+        False
+
+        Absolute URLs must have both a scheme and a host set.
+        """
+        return bool(self.scheme and self.host)
+
+    def replace(
+        self,
+        scheme=_UNSET,  # type: Optional[Text]
+        host=_UNSET,  # type: Optional[Text]
+        path=_UNSET,  # type: Iterable[Text]
+        query=_UNSET,  # type: QueryParameters
+        fragment=_UNSET,  # type: Text
+        port=_UNSET,  # type: Optional[int]
+        rooted=_UNSET,  # type: Optional[bool]
+        userinfo=_UNSET,  # type: Text
+        uses_netloc=_UNSET,  # type: Optional[bool]
+    ):
+        # type: (...) -> URL
+        """:class:`URL` objects are immutable, which means that attributes
+        are designed to be set only once, at construction. Instead of
+        modifying an existing URL, one simply creates a copy with the
+        desired changes.
+
+        If any of the following arguments is omitted, it defaults to
+        the value on the current URL.
+
+        Args:
+            scheme: The text name of the scheme.
+            host: The host portion of the network location.
+            path: A tuple of strings representing the slash-separated parts of
+                the path.
+            query: The query parameters, as a dictionary or as an sequence of
+                key-value pairs.
+            fragment: The fragment part of the URL.
+            port: The port part of the network location.
+            rooted: Whether or not the path begins with a slash.
+            userinfo: The username or colon-separated username:password pair.
+            uses_netloc: Indicates whether ``://`` (the "netloc separator")
+                will appear to separate the scheme from the *path* in cases
+                where no host is present.
+                Setting this to ``True`` is a non-spec-compliant affordance for
+                the common practice of having URIs that are *not* URLs (cannot
+                have a 'host' part) but nevertheless use the common ``://``
+                idiom that most people associate with URLs; e.g. ``message:``
+                URIs like ``message://message-id`` being equivalent to
+                ``message:message-id``.
+                This may be inferred based on the scheme depending on whether
+                :func:`register_scheme` has been used to register the scheme
+                and should not be passed directly unless you know the scheme
+                works like this and you know it has not been registered.
+
+        Returns:
+            URL: A copy of the current :class:`URL`, with new values for
+                parameters passed.
+        """
+        if scheme is not _UNSET and scheme != self.scheme:
+            # when changing schemes, reset the explicit uses_netloc preference
+            # to honor the new scheme.
+            uses_netloc = None
+        return self.__class__(
+            scheme=_optional(scheme, self.scheme),
+            host=_optional(host, self.host),
+            path=_optional(path, self.path),
+            query=_optional(query, self.query),
+            fragment=_optional(fragment, self.fragment),
+            port=_optional(port, self.port),
+            rooted=_optional(rooted, self.rooted),
+            userinfo=_optional(userinfo, self.userinfo),
+            uses_netloc=_optional(uses_netloc, self.uses_netloc),
+        )
+
+    @classmethod
+    def from_text(cls, text):
+        # type: (Text) -> URL
+        """Whereas the :class:`URL` constructor is useful for constructing
+        URLs from parts, :meth:`~URL.from_text` supports parsing whole
+        URLs from their string form::
+
+            >>> URL.from_text(u'http://example.com')
+            URL.from_text(u'http://example.com')
+            >>> URL.from_text(u'?a=b&x=y')
+            URL.from_text(u'?a=b&x=y')
+
+        As you can see above, it's also used as the :func:`repr` of
+        :class:`URL` objects. The natural counterpart to
+        :func:`~URL.to_text()`. This method only accepts *text*, so be
+        sure to decode those bytestrings.
+
+        Args:
+           text: A valid URL string.
+
+        Returns:
+           URL: The structured object version of the parsed string.
+
+        .. note::
+
+            Somewhat unexpectedly, URLs are a far more permissive
+            format than most would assume. Many strings which don't
+            look like URLs are still valid URLs. As a result, this
+            method only raises :class:`URLParseError` on invalid port
+            and IPv6 values in the host portion of the URL.
+        """
+        um = _URL_RE.match(_textcheck("text", text))
+        if um is None:
+            raise URLParseError("could not parse url: %r" % text)
+        gs = um.groupdict()
+
+        au_text = gs["authority"] or u""
+        au_m = _AUTHORITY_RE.match(au_text)
+        if au_m is None:
+            raise URLParseError(
+                "invalid authority %r in url: %r" % (au_text, text)
+            )
+        au_gs = au_m.groupdict()
+        if au_gs["bad_host"]:
+            raise URLParseError(
+                "invalid host %r in url: %r" % (au_gs["bad_host"], text)
+            )
+
+        userinfo = au_gs["userinfo"] or u""
+
+        host = au_gs["ipv6_host"] or au_gs["plain_host"]
+        port = au_gs["port"]
+        if port is not None:
+            try:
+                port = int(port)  # type: ignore[assignment] # FIXME, see below
+            except ValueError:
+                if not port:  # TODO: excessive?
+                    raise URLParseError("port must not be empty: %r" % au_text)
+                raise URLParseError("expected integer for port, not %r" % port)
+
+        scheme = gs["scheme"] or u""
+        fragment = gs["fragment"] or u""
+        uses_netloc = bool(gs["_netloc_sep"])
+
+        if gs["path"]:
+            path = tuple(gs["path"].split(u"/"))
+            if not path[0]:
+                path = path[1:]
+                rooted = True
+            else:
+                rooted = False
+        else:
+            path = ()
+            rooted = bool(au_text)
+        if gs["query"]:
+            query = tuple(
+                (
+                    qe.split(u"=", 1)  # type: ignore[misc]
+                    if u"=" in qe
+                    else (qe, None)
+                )
+                for qe in gs["query"].split(u"&")
+            )  # type: QueryPairs
+        else:
+            query = ()
+        return cls(
+            scheme,
+            host,
+            path,
+            query,
+            fragment,
+            port,  # type: ignore[arg-type] # FIXME, see above
+            rooted,
+            userinfo,
+            uses_netloc,
+        )
+
+    def normalize(
+        self,
+        scheme=True,
+        host=True,
+        path=True,
+        query=True,
+        fragment=True,
+        userinfo=True,
+        percents=True,
+    ):
+        # type: (bool, bool, bool, bool, bool, bool, bool) -> URL
+        """Return a new URL object with several standard normalizations
+        applied:
+
+        * Decode unreserved characters (`RFC 3986 2.3`_)
+        * Uppercase remaining percent-encoded octets (`RFC 3986 2.1`_)
+        * Convert scheme and host casing to lowercase (`RFC 3986 3.2.2`_)
+        * Resolve any "." and ".." references in the path (`RFC 3986 6.2.2.3`_)
+        * Ensure an ending slash on URLs with an empty path (`RFC 3986 6.2.3`_)
+        * Encode any stray percent signs (`%`) in percent-encoded
+          fields (path, query, fragment, userinfo) (`RFC 3986 2.4`_)
+
+        All are applied by default, but normalizations can be disabled
+        per-part by passing `False` for that part's corresponding
+        name.
+
+        Args:
+            scheme: Convert the scheme to lowercase
+            host: Convert the host to lowercase
+            path: Normalize the path (see above for details)
+            query: Normalize the query string
+            fragment: Normalize the fragment
+            userinfo: Normalize the userinfo
+            percents: Encode isolated percent signs for any percent-encoded
+                fields which are being normalized (defaults to `True`).
+
+        >>> url = URL.from_text(u'Http://example.COM/a/../b/./c%2f?%61%')
+        >>> print(url.normalize().to_text())
+        http://example.com/b/c%2F?a%25
+
+        .. _RFC 3986 3.2.2: https://tools.ietf.org/html/rfc3986#section-3.2.2
+        .. _RFC 3986 2.3: https://tools.ietf.org/html/rfc3986#section-2.3
+        .. _RFC 3986 2.1: https://tools.ietf.org/html/rfc3986#section-2.1
+        .. _RFC 3986 6.2.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.2.3
+        .. _RFC 3986 6.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.3
+        .. _RFC 3986 2.4: https://tools.ietf.org/html/rfc3986#section-2.4
+        """  # noqa: E501
+        kw = {}  # type: Dict[str, Any]
+        if scheme:
+            kw["scheme"] = self.scheme.lower()
+        if host:
+            kw["host"] = self.host.lower()
+
+        def _dec_unres(target):
+            # type: (Text) -> Text
+            return _decode_unreserved(
+                target, normalize_case=True, encode_stray_percents=percents
+            )
+
+        if path:
+            if self.path:
+                kw["path"] = [
+                    _dec_unres(p) for p in _resolve_dot_segments(self.path)
+                ]
+            else:
+                kw["path"] = (u"",)
+        if query:
+            kw["query"] = [
+                (_dec_unres(k), _dec_unres(v) if v else v)
+                for k, v in self.query
+            ]
+        if fragment:
+            kw["fragment"] = _dec_unres(self.fragment)
+        if userinfo:
+            kw["userinfo"] = u":".join(
+                [_dec_unres(p) for p in self.userinfo.split(":", 1)]
+            )
+
+        return self.replace(**kw)
+
+    def child(self, *segments):
+        # type: (Text) -> URL
+        """Make a new :class:`URL` where the given path segments are a child
+        of this URL, preserving other parts of the URL, including the
+        query string and fragment.
+
+        For example::
+
+            >>> url = URL.from_text(u'http://localhost/a/b?x=y')
+            >>> child_url = url.child(u"c", u"d")
+            >>> child_url.to_text()
+            u'http://localhost/a/b/c/d?x=y'
+
+        Args:
+            segments: Additional parts to be joined and added to the path, like
+            :func:`os.path.join`. Special characters in segments will be
+            percent encoded.
+
+        Returns:
+           URL: A copy of the current URL with the extra path segments.
+        """
+        if not segments:
+            return self
+
+        segments = [  # type: ignore[assignment] # variable is tuple
+            _textcheck("path segment", s) for s in segments
+        ]
+        new_path = tuple(self.path)
+        if self.path and self.path[-1] == u"":
+            new_path = new_path[:-1]
+        new_path += tuple(_encode_path_parts(segments, maximal=False))
+        return self.replace(path=new_path)
+
+    def sibling(self, segment):
+        # type: (Text) -> URL
+        """Make a new :class:`URL` with a single path segment that is a
+        sibling of this URL path.
+
+        Args:
+            segment: A single path segment.
+
+        Returns:
+            URL: A copy of the current URL with the last path segment
+                replaced by *segment*. Special characters such as
+                ``/?#`` will be percent encoded.
+        """
+        _textcheck("path segment", segment)
+        new_path = tuple(self.path)[:-1] + (_encode_path_part(segment),)
+        return self.replace(path=new_path)
+
+    def click(self, href=u""):
+        # type: (Union[Text, URL]) -> URL
+        """Resolve the given URL relative to this URL.
+
+        The resulting URI should match what a web browser would
+        generate if you visited the current URL and clicked on *href*.
+
+            >>> url = URL.from_text(u'http://blog.hatnote.com/')
+            >>> url.click(u'/post/155074058790').to_text()
+            u'http://blog.hatnote.com/post/155074058790'
+            >>> url = URL.from_text(u'http://localhost/a/b/c/')
+            >>> url.click(u'../d/./e').to_text()
+            u'http://localhost/a/b/d/e'
+
+        Args (Text):
+            href: A string representing a clicked URL.
+
+        Return:
+            A copy of the current URL with navigation logic applied.
+
+        For more information, see `RFC 3986 section 5`_.
+
+        .. _RFC 3986 section 5: https://tools.ietf.org/html/rfc3986#section-5
+        """
+        if href:
+            if isinstance(href, URL):
+                clicked = href
+            else:
+                # TODO: This error message is not completely accurate,
+                # as URL objects are now also valid, but Twisted's
+                # test suite (wrongly) relies on this exact message.
+                _textcheck("relative URL", href)
+                clicked = URL.from_text(href)
+            if clicked.absolute:
+                return clicked
+        else:
+            clicked = self
+
+        query = clicked.query
+        if clicked.scheme and not clicked.rooted:
+            # Schemes with relative paths are not well-defined.  RFC 3986 calls
+            # them a "loophole in prior specifications" that should be avoided,
+            # or supported only for backwards compatibility.
+            raise NotImplementedError(
+                "absolute URI with rootless path: %r" % (href,)
+            )
+        else:
+            if clicked.rooted:
+                path = clicked.path
+            elif clicked.path:
+                path = tuple(self.path)[:-1] + tuple(clicked.path)
+            else:
+                path = self.path
+                if not query:
+                    query = self.query
+        return self.replace(
+            scheme=clicked.scheme or self.scheme,
+            host=clicked.host or self.host,
+            port=clicked.port or self.port,
+            path=_resolve_dot_segments(path),
+            query=query,
+            fragment=clicked.fragment,
+        )
+
+    def to_uri(self):
+        # type: () -> URL
+        u"""Make a new :class:`URL` instance with all non-ASCII characters
+        appropriately percent-encoded. This is useful to do in preparation
+        for sending a :class:`URL` over a network protocol.
+
+        For example::
+
+            >>> URL.from_text(u'https://ايران.com/foo⇧bar/').to_uri()
+            URL.from_text(u'https://xn--mgba3a4fra.com/foo%E2%87%A7bar/')
+
+        Returns:
+            URL: A new instance with its path segments, query parameters, and
+                hostname encoded, so that they are all in the standard
+                US-ASCII range.
+        """
+        new_userinfo = u":".join(
+            [_encode_userinfo_part(p) for p in self.userinfo.split(":", 1)]
+        )
+        new_path = _encode_path_parts(
+            self.path, has_scheme=bool(self.scheme), rooted=False, maximal=True
+        )
+        new_host = (
+            self.host
+            if not self.host
+            else idna_encode(self.host, uts46=True).decode("ascii")
+        )
+        return self.replace(
+            userinfo=new_userinfo,
+            host=new_host,
+            path=new_path,
+            query=tuple(
+                [
+                    (
+                        _encode_query_key(k, maximal=True),
+                        _encode_query_value(v, maximal=True)
+                        if v is not None
+                        else None,
+                    )
+                    for k, v in self.query
+                ]
+            ),
+            fragment=_encode_fragment_part(self.fragment, maximal=True),
+        )
+
+    def to_iri(self):
+        # type: () -> URL
+        u"""Make a new :class:`URL` instance with all but a few reserved
+        characters decoded into human-readable format.
+
+        Percent-encoded Unicode and IDNA-encoded hostnames are
+        decoded, like so::
+
+            >>> url = URL.from_text(u'https://xn--mgba3a4fra.example.com/foo%E2%87%A7bar/')
+            >>> print(url.to_iri().to_text())
+            https://ايران.example.com/foo⇧bar/
+
+        .. note::
+
+            As a general Python issue, "narrow" (UCS-2) builds of
+            Python may not be able to fully decode certain URLs, and
+            the in those cases, this method will return a best-effort,
+            partially-decoded, URL which is still valid. This issue
+            does not affect any Python builds 3.4+.
+
+        Returns:
+            URL: A new instance with its path segments, query parameters, and
+                hostname decoded for display purposes.
+        """  # noqa: E501
+        new_userinfo = u":".join(
+            [_decode_userinfo_part(p) for p in self.userinfo.split(":", 1)]
+        )
+        host_text = _decode_host(self.host)
+
+        return self.replace(
+            userinfo=new_userinfo,
+            host=host_text,
+            path=[_decode_path_part(segment) for segment in self.path],
+            query=tuple(
+                (
+                    _decode_query_key(k),
+                    _decode_query_value(v) if v is not None else None,
+                )
+                for k, v in self.query
+            ),
+            fragment=_decode_fragment_part(self.fragment),
+        )
+
+    def to_text(self, with_password=False):
+        # type: (bool) -> Text
+        """Render this URL to its textual representation.
+
+        By default, the URL text will *not* include a password, if one
+        is set. RFC 3986 considers using URLs to represent such
+        sensitive information as deprecated. Quoting from RFC 3986,
+        `section 3.2.1`:
+
+            "Applications should not render as clear text any data after the
+            first colon (":") character found within a userinfo subcomponent
+            unless the data after the colon is the empty string (indicating no
+            password)."
+
+        Args (bool):
+            with_password: Whether or not to include the password in the URL
+                text. Defaults to False.
+
+        Returns:
+            Text: The serialized textual representation of this URL, such as
+                ``u"http://example.com/some/path?some=query"``.
+
+        The natural counterpart to :class:`URL.from_text()`.
+
+        .. _section 3.2.1: https://tools.ietf.org/html/rfc3986#section-3.2.1
+        """
+        scheme = self.scheme
+        authority = self.authority(with_password)
+        path = "/".join(
+            _encode_path_parts(
+                self.path,
+                rooted=self.rooted,
+                has_scheme=bool(scheme),
+                has_authority=bool(authority),
+                maximal=False,
+            )
+        )
+        query_parts = []
+        for k, v in self.query:
+            if v is None:
+                query_parts.append(_encode_query_key(k, maximal=False))
+            else:
+                query_parts.append(
+                    u"=".join(
+                        (
+                            _encode_query_key(k, maximal=False),
+                            _encode_query_value(v, maximal=False),
+                        )
+                    )
+                )
+        query_string = u"&".join(query_parts)
+
+        fragment = self.fragment
+
+        parts = []  # type: List[Text]
+        _add = parts.append
+        if scheme:
+            _add(scheme)
+            _add(":")
+        if authority:
+            _add("//")
+            _add(authority)
+        elif scheme and path[:2] != "//" and self.uses_netloc:
+            _add("//")
+        if path:
+            if scheme and authority and path[:1] != "/":
+                _add("/")  # relpaths with abs authorities auto get '/'
+            _add(path)
+        if query_string:
+            _add("?")
+            _add(query_string)
+        if fragment:
+            _add("#")
+            _add(fragment)
+        return u"".join(parts)
+
+    def __repr__(self):
+        # type: () -> str
+        """Convert this URL to an representation that shows all of its
+        constituent parts, as well as being a valid argument to
+        :func:`eval`.
+        """
+        return "%s.from_text(%r)" % (self.__class__.__name__, self.to_text())
+
+    def _to_bytes(self):
+        # type: () -> bytes
+        """
+        Allows for direct usage of URL objects with libraries like
+        requests, which automatically stringify URL parameters. See
+        issue #49.
+        """
+        return self.to_uri().to_text().encode("ascii")
+
+    if PY2:
+        __str__ = _to_bytes
+        __unicode__ = to_text
+    else:
+        __bytes__ = _to_bytes
+        __str__ = to_text
+
+    # # Begin Twisted Compat Code
+    asURI = to_uri
+    asIRI = to_iri
+
+    @classmethod
+    def fromText(cls, s):
+        # type: (Text) -> URL
+        return cls.from_text(s)
+
+    def asText(self, includeSecrets=False):
+        # type: (bool) -> Text
+        return self.to_text(with_password=includeSecrets)
+
+    def __dir__(self):
+        # type: () -> Sequence[Text]
+        try:
+            ret = object.__dir__(self)
+        except AttributeError:
+            # object.__dir__ == AttributeError  # pdw for py2
+            ret = dir(self.__class__) + list(self.__dict__.keys())
+        ret = sorted(set(ret) - set(["fromText", "asURI", "asIRI", "asText"]))
+        return ret
+
+    # # End Twisted Compat Code
+
+    def add(self, name, value=None):
+        # type: (Text, Optional[Text]) -> URL
+        """Make a new :class:`URL` instance with a given query argument,
+        *name*, added to it with the value *value*, like so::
+
+            >>> URL.from_text(u'https://example.com/?x=y').add(u'x')
+            URL.from_text(u'https://example.com/?x=y&x')
+            >>> URL.from_text(u'https://example.com/?x=y').add(u'x', u'z')
+            URL.from_text(u'https://example.com/?x=y&x=z')
+
+        Args:
+            name: The name of the query parameter to add.
+                The part before the ``=``.
+            value: The value of the query parameter to add.
+                The part after the ``=``.
+                Defaults to ``None``, meaning no value.
+
+        Returns:
+            URL: A new :class:`URL` instance with the parameter added.
+        """
+        return self.replace(query=self.query + ((name, value),))
+
+    def set(self, name, value=None):
+        # type: (Text, Optional[Text]) -> URL
+        """Make a new :class:`URL` instance with the query parameter *name*
+        set to *value*. All existing occurences, if any are replaced
+        by the single name-value pair.
+
+            >>> URL.from_text(u'https://example.com/?x=y').set(u'x')
+            URL.from_text(u'https://example.com/?x')
+            >>> URL.from_text(u'https://example.com/?x=y').set(u'x', u'z')
+            URL.from_text(u'https://example.com/?x=z')
+
+        Args:
+            name: The name of the query parameter to set.
+                The part before the ``=``.
+            value: The value of the query parameter to set.
+                The part after the ``=``.
+                Defaults to ``None``, meaning no value.
+
+        Returns:
+            URL: A new :class:`URL` instance with the parameter set.
+        """
+        # Preserve the original position of the query key in the list
+        q = [(k, v) for (k, v) in self.query if k != name]
+        idx = next(
+            (i for (i, (k, v)) in enumerate(self.query) if k == name), -1
+        )
+        q[idx:idx] = [(name, value)]
+        return self.replace(query=q)
+
+    def get(self, name):
+        # type: (Text) -> List[Optional[Text]]
+        """Get a list of values for the given query parameter, *name*::
+
+            >>> url = URL.from_text(u'?x=1&x=2')
+            >>> url.get('x')
+            [u'1', u'2']
+            >>> url.get('y')
+            []
+
+        If the given *name* is not set, an empty list is returned. A
+        list is always returned, and this method raises no exceptions.
+
+        Args:
+            name: The name of the query parameter to get.
+
+        Returns:
+            List[Optional[Text]]: A list of all the values associated with the
+                key, in string form.
+        """
+        return [value for (key, value) in self.query if name == key]
+
+    def remove(
+        self,
+        name,  # type: Text
+        value=_UNSET,  # type: Text
+        limit=None,  # type: Optional[int]
+    ):
+        # type: (...) -> URL
+        """Make a new :class:`URL` instance with occurrences of the query
+        parameter *name* removed, or, if *value* is set, parameters
+        matching *name* and *value*. No exception is raised if the
+        parameter is not already set.
+
+        Args:
+            name: The name of the query parameter to remove.
+            value: Optional value to additionally filter on.
+                Setting this removes query parameters which match both name
+                and value.
+            limit: Optional maximum number of parameters to remove.
+
+        Returns:
+            URL: A new :class:`URL` instance with the parameter removed.
+        """
+        if limit is None:
+            if value is _UNSET:
+                nq = [(k, v) for (k, v) in self.query if k != name]
+            else:
+                nq = [
+                    (k, v)
+                    for (k, v) in self.query
+                    if not (k == name and v == value)
+                ]
+        else:
+            nq, removed_count = [], 0
+
+            for k, v in self.query:
+                if (
+                    k == name
+                    and (value is _UNSET or v == value)
+                    and removed_count < limit
+                ):
+                    removed_count += 1  # drop it
+                else:
+                    nq.append((k, v))  # keep it
+
+        return self.replace(query=nq)
+
+
+EncodedURL = URL  # An alias better describing what the URL really is
+
+_EMPTY_URL = URL()
+
+
+def _replace_plus(text):
+    # type: (Text) -> Text
+    return text.replace("+", "%20")
+
+
+def _no_op(text):
+    # type: (Text) -> Text
+    return text
+
+
+class DecodedURL(object):
+    """
+    :class:`DecodedURL` is a type designed to act as a higher-level
+    interface to :class:`URL` and the recommended type for most
+    operations. By analogy, :class:`DecodedURL` is the
+    :class:`unicode` to URL's :class:`bytes`.
+
+    :class:`DecodedURL` automatically handles encoding and decoding
+    all its components, such that all inputs and outputs are in a
+    maximally-decoded state.  Note that this means, for some special
+    cases, a URL may not "roundtrip" character-for-character, but this
+    is considered a good tradeoff for the safety of automatic
+    encoding.
+
+    Otherwise, :class:`DecodedURL` has almost exactly the same API as
+    :class:`URL`.
+
+    Where applicable, a UTF-8 encoding is presumed. Be advised that
+    some interactions can raise :exc:`UnicodeEncodeErrors` and
+    :exc:`UnicodeDecodeErrors`, just like when working with
+    bytestrings. Examples of such interactions include handling query
+    strings encoding binary data, and paths containing segments with
+    special characters encoded with codecs other than UTF-8.
+
+    Args:
+        url: A :class:`URL` object to wrap.
+        lazy: Set to True to avoid pre-decode all parts of the URL to check for
+            validity.
+            Defaults to False.
+        query_plus_is_space: + characters in the query string should be treated
+            as spaces when decoding.  If unspecified, the default is taken from
+            the scheme.
+
+    .. note::
+
+      The :class:`DecodedURL` initializer takes a :class:`URL` object,
+      not URL components, like :class:`URL`. To programmatically
+      construct a :class:`DecodedURL`, you can use this pattern:
+
+        >>> print(DecodedURL().replace(scheme=u'https',
+        ... host=u'pypi.org', path=(u'projects', u'hyperlink')).to_text())
+        https://pypi.org/projects/hyperlink
+
+    .. versionadded:: 18.0.0
+    """
+
+    def __init__(self, url=_EMPTY_URL, lazy=False, query_plus_is_space=None):
+        # type: (URL, bool, Optional[bool]) -> None
+        self._url = url
+        if query_plus_is_space is None:
+            query_plus_is_space = url.scheme not in NO_QUERY_PLUS_SCHEMES
+        self._query_plus_is_space = query_plus_is_space
+        if not lazy:
+            # cache the following, while triggering any decoding
+            # issues with decodable fields
+            self.host, self.userinfo, self.path, self.query, self.fragment
+        return
+
+    @classmethod
+    def from_text(cls, text, lazy=False, query_plus_is_space=None):
+        # type: (Text, bool, Optional[bool]) -> DecodedURL
+        """\
+        Make a `DecodedURL` instance from any text string containing a URL.
+
+        Args:
+          text: Text containing the URL
+          lazy: Whether to pre-decode all parts of the URL to check for
+              validity.
+              Defaults to True.
+        """
+        _url = URL.from_text(text)
+        return cls(_url, lazy=lazy, query_plus_is_space=query_plus_is_space)
+
+    @property
+    def encoded_url(self):
+        # type: () -> URL
+        """Access the underlying :class:`URL` object, which has any special
+        characters encoded.
+        """
+        return self._url
+
+    def to_text(self, with_password=False):
+        # type: (bool) -> Text
+        "Passthrough to :meth:`~hyperlink.URL.to_text()`"
+        return self._url.to_text(with_password)
+
+    def to_uri(self):
+        # type: () -> URL
+        "Passthrough to :meth:`~hyperlink.URL.to_uri()`"
+        return self._url.to_uri()
+
+    def to_iri(self):
+        # type: () -> URL
+        "Passthrough to :meth:`~hyperlink.URL.to_iri()`"
+        return self._url.to_iri()
+
+    def _clone(self, url):
+        # type: (URL) -> DecodedURL
+        return self.__class__(
+            url,
+            # TODO: propagate laziness?
+            query_plus_is_space=self._query_plus_is_space,
+        )
+
+    def click(self, href=u""):
+        # type: (Union[Text, URL, DecodedURL]) -> DecodedURL
+        """Return a new DecodedURL wrapping the result of
+        :meth:`~hyperlink.URL.click()`
+        """
+        if isinstance(href, DecodedURL):
+            href = href._url
+        return self._clone(
+            self._url.click(href=href),
+        )
+
+    def sibling(self, segment):
+        # type: (Text) -> DecodedURL
+        """Automatically encode any reserved characters in *segment* and
+        return a new `DecodedURL` wrapping the result of
+        :meth:`~hyperlink.URL.sibling()`
+        """
+        return self._clone(
+            self._url.sibling(_encode_reserved(segment)),
+        )
+
+    def child(self, *segments):
+        # type: (Text) -> DecodedURL
+        """Automatically encode any reserved characters in *segments* and
+        return a new `DecodedURL` wrapping the result of
+        :meth:`~hyperlink.URL.child()`.
+        """
+        if not segments:
+            return self
+        new_segs = [_encode_reserved(s) for s in segments]
+        return self._clone(self._url.child(*new_segs))
+
+    def normalize(
+        self,
+        scheme=True,
+        host=True,
+        path=True,
+        query=True,
+        fragment=True,
+        userinfo=True,
+        percents=True,
+    ):
+        # type: (bool, bool, bool, bool, bool, bool, bool) -> DecodedURL
+        """Return a new `DecodedURL` wrapping the result of
+        :meth:`~hyperlink.URL.normalize()`
+        """
+        return self._clone(
+            self._url.normalize(
+                scheme, host, path, query, fragment, userinfo, percents
+            )
+        )
+
+    @property
+    def absolute(self):
+        # type: () -> bool
+        return self._url.absolute
+
+    @property
+    def scheme(self):
+        # type: () -> Text
+        return self._url.scheme
+
+    @property
+    def host(self):
+        # type: () -> Text
+        return _decode_host(self._url.host)
+
+    @property
+    def port(self):
+        # type: () -> Optional[int]
+        return self._url.port
+
+    @property
+    def rooted(self):
+        # type: () -> bool
+        return self._url.rooted
+
+    @property
+    def path(self):
+        # type: () -> Sequence[Text]
+        if not hasattr(self, "_path"):
+            self._path = tuple(
+                [
+                    _percent_decode(p, raise_subencoding_exc=True)
+                    for p in self._url.path
+                ]
+            )
+        return self._path
+
+    @property
+    def query(self):
+        # type: () -> QueryPairs
+        if not hasattr(self, "_query"):
+            if self._query_plus_is_space:
+                predecode = _replace_plus
+            else:
+                predecode = _no_op
+
+            self._query = cast(
+                QueryPairs,
+                tuple(
+                    tuple(
+                        _percent_decode(
+                            predecode(x), raise_subencoding_exc=True
+                        )
+                        if x is not None
+                        else None
+                        for x in (k, v)
+                    )
+                    for k, v in self._url.query
+                ),
+            )
+        return self._query
+
+    @property
+    def fragment(self):
+        # type: () -> Text
+        if not hasattr(self, "_fragment"):
+            frag = self._url.fragment
+            self._fragment = _percent_decode(frag, raise_subencoding_exc=True)
+        return self._fragment
+
+    @property
+    def userinfo(self):
+        # type: () -> Union[Tuple[str], Tuple[str, str]]
+        if not hasattr(self, "_userinfo"):
+            self._userinfo = cast(
+                Union[Tuple[str], Tuple[str, str]],
+                tuple(
+                    tuple(
+                        _percent_decode(p, raise_subencoding_exc=True)
+                        for p in self._url.userinfo.split(":", 1)
+                    )
+                ),
+            )
+        return self._userinfo
+
+    @property
+    def user(self):
+        # type: () -> Text
+        return self.userinfo[0]
+
+    @property
+    def uses_netloc(self):
+        # type: () -> Optional[bool]
+        return self._url.uses_netloc
+
+    def replace(
+        self,
+        scheme=_UNSET,  # type: Optional[Text]
+        host=_UNSET,  # type: Optional[Text]
+        path=_UNSET,  # type: Iterable[Text]
+        query=_UNSET,  # type: QueryParameters
+        fragment=_UNSET,  # type: Text
+        port=_UNSET,  # type: Optional[int]
+        rooted=_UNSET,  # type: Optional[bool]
+        userinfo=_UNSET,  # type: Union[Tuple[str], Tuple[str, str]]
+        uses_netloc=_UNSET,  # type: Optional[bool]
+    ):
+        # type: (...) -> DecodedURL
+        """While the signature is the same, this `replace()` differs a little
+        from URL.replace. For instance, it accepts userinfo as a
+        tuple, not as a string, handling the case of having a username
+        containing a `:`. As with the rest of the methods on
+        DecodedURL, if you pass a reserved character, it will be
+        automatically encoded instead of an error being raised.
+        """
+        if path is not _UNSET:
+            path = tuple(_encode_reserved(p) for p in path)
+        if query is not _UNSET:
+            query = cast(
+                QueryPairs,
+                tuple(
+                    tuple(
+                        _encode_reserved(x) if x is not None else None
+                        for x in (k, v)
+                    )
+                    for k, v in iter_pairs(query)
+                ),
+            )
+        if userinfo is not _UNSET:
+            if len(userinfo) > 2:
+                raise ValueError(
+                    'userinfo expected sequence of ["user"] or'
+                    ' ["user", "password"], got %r' % (userinfo,)
+                )
+            userinfo_text = u":".join([_encode_reserved(p) for p in userinfo])
+        else:
+            userinfo_text = _UNSET
+        new_url = self._url.replace(
+            scheme=scheme,
+            host=host,
+            path=path,
+            query=query,
+            fragment=fragment,
+            port=port,
+            rooted=rooted,
+            userinfo=userinfo_text,
+            uses_netloc=uses_netloc,
+        )
+        return self._clone(url=new_url)
+
+    def get(self, name):
+        # type: (Text) -> List[Optional[Text]]
+        "Get the value of all query parameters whose name matches *name*"
+        return [v for (k, v) in self.query if name == k]
+
+    def add(self, name, value=None):
+        # type: (Text, Optional[Text]) -> DecodedURL
+        """Return a new DecodedURL with the query parameter *name* and *value*
+        added."""
+        return self.replace(query=self.query + ((name, value),))
+
+    def set(self, name, value=None):
+        # type: (Text, Optional[Text]) -> DecodedURL
+        "Return a new DecodedURL with query parameter *name* set to *value*"
+        query = self.query
+        q = [(k, v) for (k, v) in query if k != name]
+        idx = next((i for (i, (k, v)) in enumerate(query) if k == name), -1)
+        q[idx:idx] = [(name, value)]
+        return self.replace(query=q)
+
+    def remove(
+        self,
+        name,  # type: Text
+        value=_UNSET,  # type: Text
+        limit=None,  # type: Optional[int]
+    ):
+        # type: (...) -> DecodedURL
+        """Return a new DecodedURL with query parameter *name* removed.
+
+        Optionally also filter for *value*, as well as cap the number
+        of parameters removed with *limit*.
+        """
+        if limit is None:
+            if value is _UNSET:
+                nq = [(k, v) for (k, v) in self.query if k != name]
+            else:
+                nq = [
+                    (k, v)
+                    for (k, v) in self.query
+                    if not (k == name and v == value)
+                ]
+        else:
+            nq, removed_count = [], 0
+            for k, v in self.query:
+                if (
+                    k == name
+                    and (value is _UNSET or v == value)
+                    and removed_count < limit
+                ):
+                    removed_count += 1  # drop it
+                else:
+                    nq.append((k, v))  # keep it
+
+        return self.replace(query=nq)
+
+    def __repr__(self):
+        # type: () -> str
+        cn = self.__class__.__name__
+        return "%s(url=%r)" % (cn, self._url)
+
+    def __str__(self):
+        # type: () -> str
+        # TODO: the underlying URL's __str__ needs to change to make
+        # this work as the URL, see #55
+        return str(self._url)
+
+    def __eq__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return self.normalize().to_uri() == other.normalize().to_uri()
+
+    def __ne__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        # type: () -> int
+        return hash(
+            (
+                self.__class__,
+                self.scheme,
+                self.userinfo,
+                self.host,
+                self.path,
+                self.query,
+                self.fragment,
+                self.port,
+                self.rooted,
+                self.uses_netloc,
+            )
+        )
+
+    # # Begin Twisted Compat Code
+    asURI = to_uri
+    asIRI = to_iri
+
+    @classmethod
+    def fromText(cls, s, lazy=False):
+        # type: (Text, bool) -> DecodedURL
+        return cls.from_text(s, lazy=lazy)
+
+    def asText(self, includeSecrets=False):
+        # type: (bool) -> Text
+        return self.to_text(with_password=includeSecrets)
+
+    def __dir__(self):
+        # type: () -> Sequence[Text]
+        try:
+            ret = object.__dir__(self)
+        except AttributeError:
+            # object.__dir__ == AttributeError  # pdw for py2
+            ret = dir(self.__class__) + list(self.__dict__.keys())
+        ret = sorted(set(ret) - set(["fromText", "asURI", "asIRI", "asText"]))
+        return ret
+
+    # # End Twisted Compat Code
+
+
+def parse(url, decoded=True, lazy=False):
+    # type: (Text, bool, bool) -> Union[URL, DecodedURL]
+    """
+    Automatically turn text into a structured URL object.
+
+    >>> url = parse(u"https://github.com/python-hyper/hyperlink")
+    >>> print(url.to_text())
+    https://github.com/python-hyper/hyperlink
+
+    Args:
+        url: A text string representation of a URL.
+
+        decoded: Whether or not to return a :class:`DecodedURL`,
+            which automatically handles all
+            encoding/decoding/quoting/unquoting for all the various
+            accessors of parts of the URL, or a :class:`URL`,
+            which has the same API, but requires handling of special
+            characters for different parts of the URL.
+
+        lazy: In the case of `decoded=True`, this controls
+            whether the URL is decoded immediately or as accessed. The
+            default, `lazy=False`, checks all encoded parts of the URL
+            for decodability.
+
+    .. versionadded:: 18.0.0
+    """
+    enc_url = EncodedURL.from_text(url)
+    if not decoded:
+        return enc_url
+    dec_url = DecodedURL(enc_url, lazy=lazy)
+    return dec_url
diff --git a/contrib/python/hyperlink/py3/hyperlink/hypothesis.py b/contrib/python/hyperlink/py3/hyperlink/hypothesis.py
new file mode 100644
index 00000000000..45fd9a99569
--- /dev/null
+++ b/contrib/python/hyperlink/py3/hyperlink/hypothesis.py
@@ -0,0 +1,324 @@
+# -*- coding: utf-8 -*-
+"""
+Hypothesis strategies.
+"""
+from __future__ import absolute_import
+
+try:
+    import hypothesis
+
+    del hypothesis
+except ImportError:
+    from typing import Tuple
+
+    __all__ = ()  # type: Tuple[str, ...]
+else:
+    import io
+    import pkgutil
+    from csv import reader as csv_reader
+    from os.path import dirname, join
+    from string import ascii_letters, digits
+    from sys import maxunicode
+    from typing import (
+        Callable,
+        Iterable,
+        List,
+        Optional,
+        Sequence,
+        Text,
+        TypeVar,
+        cast,
+    )
+    from gzip import open as open_gzip
+
+    from . import DecodedURL, EncodedURL
+
+    from hypothesis import assume
+    from hypothesis.strategies import (
+        composite,
+        integers,
+        lists,
+        sampled_from,
+        text,
+    )
+
+    from idna import IDNAError, check_label, encode as idna_encode
+
+    __all__ = (
+        "decoded_urls",
+        "encoded_urls",
+        "hostname_labels",
+        "hostnames",
+        "idna_text",
+        "paths",
+        "port_numbers",
+    )
+
+    T = TypeVar("T")
+    DrawCallable = Callable[[Callable[..., T]], T]
+
+    try:
+        unichr
+    except NameError:  # Py3
+        unichr = chr  # type: Callable[[int], Text]
+
+    def idna_characters():
+        # type: () -> Text
+        """
+        Returns a string containing IDNA characters.
+        """
+        global _idnaCharacters
+
+        if not _idnaCharacters:
+            result = []
+
+            # Data source "IDNA Derived Properties":
+            # https://www.iana.org/assignments/idna-tables-6.3.0/
+            #   idna-tables-6.3.0.xhtml#idna-tables-properties
+            dataFileName = join(
+                dirname(__file__), "idna-tables-properties.csv.gz"
+            )
+            data = io.BytesIO(pkgutil.get_data(__name__, "idna-tables-properties.csv.gz"))
+            with open_gzip(data) as dataFile:
+                reader = csv_reader(
+                    (line.decode("utf-8") for line in dataFile),
+                    delimiter=",",
+                )
+                next(reader)  # Skip header row
+                for row in reader:
+                    codes, prop, description = row
+
+                    if prop != "PVALID":
+                        # CONTEXTO or CONTEXTJ are also allowed, but they come
+                        # with rules, so we're punting on those here.
+                        # See: https://tools.ietf.org/html/rfc5892
+                        continue
+
+                    startEnd = row[0].split("-", 1)
+                    if len(startEnd) == 1:
+                        # No end of range given; use start
+                        startEnd.append(startEnd[0])
+                    start, end = (int(i, 16) for i in startEnd)
+
+                    for i in range(start, end + 1):
+                        if i > maxunicode:  # Happens using Py2 on Windows
+                            break
+                        result.append(unichr(i))
+
+            _idnaCharacters = u"".join(result)
+
+        return _idnaCharacters
+
+    _idnaCharacters = ""  # type: Text
+
+    @composite
+    def idna_text(draw, min_size=1, max_size=None):
+        # type: (DrawCallable, int, Optional[int]) -> Text
+        """
+        A strategy which generates IDNA-encodable text.
+
+        @param min_size: The minimum number of characters in the text.
+            C{None} is treated as C{0}.
+
+        @param max_size: The maximum number of characters in the text.
+            Use C{None} for an unbounded size.
+        """
+        alphabet = idna_characters()
+
+        assert min_size >= 1
+
+        if max_size is not None:
+            assert max_size >= 1
+
+        result = cast(
+            Text,
+            draw(text(min_size=min_size, max_size=max_size, alphabet=alphabet)),
+        )
+
+        # FIXME: There should be a more efficient way to ensure we produce
+        # valid IDNA text.
+        try:
+            idna_encode(result)
+        except IDNAError:
+            assume(False)
+
+        return result
+
+    @composite
+    def port_numbers(draw, allow_zero=False):
+        # type: (DrawCallable, bool) -> int
+        """
+        A strategy which generates port numbers.
+
+        @param allow_zero: Whether to allow port C{0} as a possible value.
+        """
+        if allow_zero:
+            min_value = 0
+        else:
+            min_value = 1
+
+        return cast(int, draw(integers(min_value=min_value, max_value=65535)))
+
+    @composite
+    def hostname_labels(draw, allow_idn=True):
+        # type: (DrawCallable, bool) -> Text
+        """
+        A strategy which generates host name labels.
+
+        @param allow_idn: Whether to allow non-ASCII characters as allowed by
+            internationalized domain names (IDNs).
+        """
+        if allow_idn:
+            label = cast(Text, draw(idna_text(min_size=1, max_size=63)))
+
+            try:
+                label.encode("ascii")
+            except UnicodeEncodeError:
+                # If the label doesn't encode to ASCII, then we need to check
+                # the length of the label after encoding to punycode and adding
+                # the xn-- prefix.
+                while len(label.encode("punycode")) > 63 - len("xn--"):
+                    # Rather than bombing out, just trim from the end until it
+                    # is short enough, so hypothesis doesn't have to generate
+                    # new data.
+                    label = label[:-1]
+
+        else:
+            label = cast(
+                Text,
+                draw(
+                    text(
+                        min_size=1,
+                        max_size=63,
+                        alphabet=Text(ascii_letters + digits + u"-"),
+                    )
+                ),
+            )
+
+        # Filter invalid labels.
+        # It would be better to reliably avoid generation of bogus labels in
+        # the first place, but it's hard...
+        try:
+            check_label(label)
+        except UnicodeError:  # pragma: no cover (not always drawn)
+            assume(False)
+
+        return label
+
+    @composite
+    def hostnames(draw, allow_leading_digit=True, allow_idn=True):
+        # type: (DrawCallable, bool, bool) -> Text
+        """
+        A strategy which generates host names.
+
+        @param allow_leading_digit: Whether to allow a leading digit in host
+            names; they were not allowed prior to RFC 1123.
+
+        @param allow_idn: Whether to allow non-ASCII characters as allowed by
+            internationalized domain names (IDNs).
+        """
+        # Draw first label, filtering out labels with leading digits if needed
+        labels = [
+            cast(
+                Text,
+                draw(
+                    hostname_labels(allow_idn=allow_idn).filter(
+                        lambda l: (
+                            True if allow_leading_digit else l[0] not in digits
+                        )
+                    )
+                ),
+            )
+        ]
+        # Draw remaining labels
+        labels += cast(
+            List[Text],
+            draw(
+                lists(
+                    hostname_labels(allow_idn=allow_idn),
+                    min_size=1,
+                    max_size=4,
+                )
+            ),
+        )
+
+        # Trim off labels until the total host name length fits in 252
+        # characters.  This avoids having to filter the data.
+        while sum(len(label) for label in labels) + len(labels) - 1 > 252:
+            labels = labels[:-1]
+
+        return u".".join(labels)
+
+    def path_characters():
+        # type: () -> str
+        """
+        Returns a string containing valid URL path characters.
+        """
+        global _path_characters
+
+        if _path_characters is None:
+
+            def chars():
+                # type: () -> Iterable[Text]
+                for i in range(maxunicode):
+                    c = unichr(i)
+
+                    # Exclude reserved characters
+                    if c in "#/?":
+                        continue
+
+                    # Exclude anything not UTF-8 compatible
+                    try:
+                        c.encode("utf-8")
+                    except UnicodeEncodeError:
+                        continue
+
+                    yield c
+
+            _path_characters = "".join(chars())
+
+        return _path_characters
+
+    _path_characters = None  # type: Optional[str]
+
+    @composite
+    def paths(draw):
+        # type: (DrawCallable) -> Sequence[Text]
+        return cast(
+            List[Text],
+            draw(
+                lists(text(min_size=1, alphabet=path_characters()), max_size=10)
+            ),
+        )
+
+    @composite
+    def encoded_urls(draw):
+        # type: (DrawCallable) -> EncodedURL
+        """
+        A strategy which generates L{EncodedURL}s.
+        Call the L{EncodedURL.to_uri} method on each URL to get an HTTP
+        protocol-friendly URI.
+        """
+        port = cast(Optional[int], draw(port_numbers(allow_zero=True)))
+        host = cast(Text, draw(hostnames()))
+        path = cast(Sequence[Text], draw(paths()))
+
+        if port == 0:
+            port = None
+
+        return EncodedURL(
+            scheme=cast(Text, draw(sampled_from((u"http", u"https")))),
+            host=host,
+            port=port,
+            path=path,
+        )
+
+    @composite
+    def decoded_urls(draw):
+        # type: (DrawCallable) -> DecodedURL
+        """
+        A strategy which generates L{DecodedURL}s.
+        Call the L{EncodedURL.to_uri} method on each URL to get an HTTP
+        protocol-friendly URI.
+        """
+        return DecodedURL(draw(encoded_urls()))
diff --git a/contrib/python/hyperlink/py3/hyperlink/idna-tables-properties.csv.gz b/contrib/python/hyperlink/py3/hyperlink/idna-tables-properties.csv.gz
new file mode 100644
index 00000000000..48e9f06742f
Binary files /dev/null and b/contrib/python/hyperlink/py3/hyperlink/idna-tables-properties.csv.gz differ
diff --git a/contrib/python/hyperlink/py3/hyperlink/py.typed b/contrib/python/hyperlink/py3/hyperlink/py.typed
new file mode 100644
index 00000000000..d2dfd5e4915
--- /dev/null
+++ b/contrib/python/hyperlink/py3/hyperlink/py.typed
@@ -0,0 +1 @@
+# See: https://www.python.org/dev/peps/pep-0561/
diff --git a/contrib/python/hyperlink/py3/ya.make b/contrib/python/hyperlink/py3/ya.make
new file mode 100644
index 00000000000..2ada924679c
--- /dev/null
+++ b/contrib/python/hyperlink/py3/ya.make
@@ -0,0 +1,35 @@
+# Generated by devtools/yamaker (pypi).
+
+PY3_LIBRARY()
+
+VERSION(21.0.0)
+
+LICENSE(MIT)
+
+PEERDIR(
+    contrib/python/idna
+)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    hyperlink/__init__.py
+    hyperlink/_socket.py
+    hyperlink/_url.py
+    hyperlink/hypothesis.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/hyperlink/py3/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+    hyperlink/idna-tables-properties.csv.gz
+    hyperlink/py.typed
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/hyperlink/ya.make b/contrib/python/hyperlink/ya.make
new file mode 100644
index 00000000000..64a73ff34ed
--- /dev/null
+++ b/contrib/python/hyperlink/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+    PEERDIR(contrib/python/hyperlink/py2)
+ELSE()
+    PEERDIR(contrib/python/hyperlink/py3)
+ENDIF()
+
+NO_LINT()
+
+END()
+
+RECURSE(
+    py2
+    py3
+)
diff --git a/contrib/python/incremental/py2/.dist-info/METADATA b/contrib/python/incremental/py2/.dist-info/METADATA
new file mode 100644
index 00000000000..c4a94090826
--- /dev/null
+++ b/contrib/python/incremental/py2/.dist-info/METADATA
@@ -0,0 +1,136 @@
+Metadata-Version: 2.1
+Name: incremental
+Version: 22.10.0
+Summary: "A small library that versions your Python projects."
+Home-page: https://github.com/twisted/incremental
+Maintainer: Amber Brown
+Maintainer-email: hawkowl@twistedmatrix.com
+License: MIT
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+License-File: LICENSE
+Provides-Extra: mypy
+Requires-Dist: click (>=6.0) ; extra == 'mypy'
+Requires-Dist: twisted (>=16.4.0) ; extra == 'mypy'
+Requires-Dist: mypy (==0.812) ; extra == 'mypy'
+Provides-Extra: scripts
+Requires-Dist: click (>=6.0) ; extra == 'scripts'
+Requires-Dist: twisted (>=16.4.0) ; extra == 'scripts'
+
+Incremental
+===========
+
+|gha|
+|pypi|
+|coverage|
+
+Incremental is a small library that versions your Python projects.
+
+API documentation can be found `here <https://twisted.github.io/incremental/docs/>`_.
+
+
+Quick Start
+-----------
+
+Add this to your ``setup.py``\ 's ``setup()`` call, removing any other versioning arguments:
+
+.. code::
+
+   setup(
+       use_incremental=True,
+       setup_requires=['incremental'],
+       install_requires=['incremental'], # along with any other install dependencies
+       ...
+   }
+
+
+Install Incremental to your local environment with ``pip install incremental[scripts]``.
+Then run ``python -m incremental.update <projectname> --create``.
+It will create a file in your package named ``_version.py`` and look like this:
+
+.. code::
+
+   from incremental import Version
+
+   __version__ = Version("widgetbox", 17, 1, 0)
+   __all__ = ["__version__"]
+
+
+Then, so users of your project can find your version, in your root package's ``__init__.py`` add:
+
+.. code::
+
+   from ._version import __version__
+
+
+Subsequent installations of your project will then use Incremental for versioning.
+
+
+Incremental Versions
+--------------------
+
+``incremental.Version`` is a class that represents a version of a given project.
+It is made up of the following elements (which are given during instantiation):
+
+- ``package`` (required), the name of the package this ``Version`` represents.
+- ``major``, ``minor``, ``micro`` (all required), the X.Y.Z of your project's ``Version``.
+- ``release_candidate`` (optional), set to 0 or higher to mark this ``Version`` being of a release candidate (also sometimes called a "prerelease").
+- ``post`` (optional), set to 0 or higher to mark this ``Version`` as a postrelease.
+- ``dev`` (optional), set to 0 or higher to mark this ``Version`` as a development release.
+
+You can extract a PEP-440 compatible version string by using the ``.public()`` method, which returns a ``str`` containing the full version. This is the version you should provide to users, or publicly use. An example output would be ``"13.2.0"``, ``"17.1.2dev1"``, or ``"18.8.0rc2"``.
+
+Calling ``repr()`` with a ``Version`` will give a Python-source-code representation of it, and calling ``str()`` with a ``Version`` will provide a string similar to ``'[Incremental, version 16.10.1]'``.
+
+
+Updating
+--------
+
+Incremental includes a tool to automate updating your Incremental-using project's version called ``incremental.update``.
+It updates the ``_version.py`` file and automatically updates some uses of Incremental versions from an indeterminate version to the current one.
+It requires ``click`` from PyPI.
+
+``python -m incremental.update <projectname>`` will perform updates on that package.
+The commands that can be given after that will determine what the next version is.
+
+- ``--newversion=<version>``, to set the project version to a fully-specified version (like 1.2.3, or 17.1.0dev1).
+- ``--rc``, to set the project version to ``<year-2000>.<month>.0rc1`` if the current version is not a release candidate, or bump the release candidate number by 1 if it is.
+- ``--dev``, to set the project development release number to 0 if it is not a development release, or bump the development release number by 1 if it is.
+- ``--patch``, to increment the patch number of the release. This will also reset the release candidate number, pass ``--rc`` at the same time to increment the patch number and make it a release candidate.
+- ``--post``, to set the project postrelease number to 0 if it is not a postrelease, or bump the postrelease number by 1 if it is. This will also reset the release candidate and development release numbers.
+
+If you give no arguments, it will strip the release candidate number, making it a "full release".
+
+Incremental supports "indeterminate" versions, as a stand-in for the next "full" version. This can be used when the version which will be displayed to the end-user is unknown (for example "introduced in" or "deprecated in"). Incremental supports the following indeterminate versions:
+
+- ``Version("<projectname>", "NEXT", 0, 0)``
+- ``<projectname> NEXT``
+
+When you run ``python -m incremental.update <projectname> --rc``, these will be updated to real versions (assuming the target final version is 17.1.0):
+
+- ``Version("<projectname>", 17, 1, 0, release_candidate=1)``
+- ``<projectname> 17.1.0rc1``
+
+Once the final version is made, it will become:
+
+- ``Version("<projectname>", 17, 1, 0)``
+- ``<projectname> 17.1.0``
+
+
+.. |coverage| image:: https://codecov.io/gh/twisted/incremental/branch/master/graph/badge.svg?token=K2ieeL887X
+.. _coverage: https://codecov.io/gh/twisted/incremental
+
+.. |gha| image:: https://github.com/twisted/incremental/actions/workflows/tests.yaml/badge.svg
+.. _gha: https://github.com/twisted/incremental/actions/workflows/tests.yaml
+
+.. |pypi| image:: http://img.shields.io/pypi/v/incremental.svg
+.. _pypi: https://pypi.python.org/pypi/incremental
diff --git a/contrib/python/incremental/py2/.dist-info/entry_points.txt b/contrib/python/incremental/py2/.dist-info/entry_points.txt
new file mode 100644
index 00000000000..1c7b4877c1a
--- /dev/null
+++ b/contrib/python/incremental/py2/.dist-info/entry_points.txt
@@ -0,0 +1,2 @@
+[distutils.setup_keywords]
+use_incremental = incremental:_get_version
diff --git a/contrib/python/incremental/py2/.dist-info/top_level.txt b/contrib/python/incremental/py2/.dist-info/top_level.txt
new file mode 100644
index 00000000000..cb4023922ac
--- /dev/null
+++ b/contrib/python/incremental/py2/.dist-info/top_level.txt
@@ -0,0 +1 @@
+incremental
diff --git a/contrib/python/incremental/py2/LICENSE b/contrib/python/incremental/py2/LICENSE
new file mode 100644
index 00000000000..bc9d47f3322
--- /dev/null
+++ b/contrib/python/incremental/py2/LICENSE
@@ -0,0 +1,74 @@
+Incremental
+-----------
+
+This project includes code from the Twisted Project, which is licensed as below.
+
+Copyright (c) 2001-2015
+Allen Short
+Amber Hawkie Brown
+Andrew Bennetts
+Andy Gayton
+Antoine Pitrou
+Apple Computer, Inc.
+Ashwini Oruganti
+Benjamin Bruheim
+Bob Ippolito
+Canonical Limited
+Christopher Armstrong
+David Reid
+Divmod Inc.
+Donovan Preston
+Eric Mangold
+Eyal Lotem
+Google Inc.
+Hybrid Logic Ltd.
+Hynek Schlawack
+Itamar Turner-Trauring
+James Knight
+Jason A. Mobarak
+Jean-Paul Calderone
+Jessica McKellar
+Jonathan D. Simms
+Jonathan Jacobs
+Jonathan Lange
+Julian Berman
+Jürgen Hermann
+Kevin Horn
+Kevin Turner
+Laurens Van Houtven
+Mary Gardiner
+Massachusetts Institute of Technology
+Matthew Lefkowitz
+Moshe Zadka
+Paul Swartz
+Pavel Pergamenshchik
+Rackspace, US Inc.
+Ralph Meijer
+Richard Wall
+Sean Riley
+Software Freedom Conservancy
+Tavendo GmbH
+Thijs Triemstra
+Thomas Herve
+Timothy Allen
+Tom Prince
+Travis B. Hartwell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/python/incremental/py2/README.rst b/contrib/python/incremental/py2/README.rst
new file mode 100644
index 00000000000..7a20d077af2
--- /dev/null
+++ b/contrib/python/incremental/py2/README.rst
@@ -0,0 +1,108 @@
+Incremental
+===========
+
+|gha|
+|pypi|
+|coverage|
+
+Incremental is a small library that versions your Python projects.
+
+API documentation can be found `here <https://twisted.github.io/incremental/docs/>`_.
+
+
+Quick Start
+-----------
+
+Add this to your ``setup.py``\ 's ``setup()`` call, removing any other versioning arguments:
+
+.. code::
+
+   setup(
+       use_incremental=True,
+       setup_requires=['incremental'],
+       install_requires=['incremental'], # along with any other install dependencies
+       ...
+   }
+
+
+Install Incremental to your local environment with ``pip install incremental[scripts]``.
+Then run ``python -m incremental.update <projectname> --create``.
+It will create a file in your package named ``_version.py`` and look like this:
+
+.. code::
+
+   from incremental import Version
+
+   __version__ = Version("widgetbox", 17, 1, 0)
+   __all__ = ["__version__"]
+
+
+Then, so users of your project can find your version, in your root package's ``__init__.py`` add:
+
+.. code::
+
+   from ._version import __version__
+
+
+Subsequent installations of your project will then use Incremental for versioning.
+
+
+Incremental Versions
+--------------------
+
+``incremental.Version`` is a class that represents a version of a given project.
+It is made up of the following elements (which are given during instantiation):
+
+- ``package`` (required), the name of the package this ``Version`` represents.
+- ``major``, ``minor``, ``micro`` (all required), the X.Y.Z of your project's ``Version``.
+- ``release_candidate`` (optional), set to 0 or higher to mark this ``Version`` being of a release candidate (also sometimes called a "prerelease").
+- ``post`` (optional), set to 0 or higher to mark this ``Version`` as a postrelease.
+- ``dev`` (optional), set to 0 or higher to mark this ``Version`` as a development release.
+
+You can extract a PEP-440 compatible version string by using the ``.public()`` method, which returns a ``str`` containing the full version. This is the version you should provide to users, or publicly use. An example output would be ``"13.2.0"``, ``"17.1.2dev1"``, or ``"18.8.0rc2"``.
+
+Calling ``repr()`` with a ``Version`` will give a Python-source-code representation of it, and calling ``str()`` with a ``Version`` will provide a string similar to ``'[Incremental, version 16.10.1]'``.
+
+
+Updating
+--------
+
+Incremental includes a tool to automate updating your Incremental-using project's version called ``incremental.update``.
+It updates the ``_version.py`` file and automatically updates some uses of Incremental versions from an indeterminate version to the current one.
+It requires ``click`` from PyPI.
+
+``python -m incremental.update <projectname>`` will perform updates on that package.
+The commands that can be given after that will determine what the next version is.
+
+- ``--newversion=<version>``, to set the project version to a fully-specified version (like 1.2.3, or 17.1.0dev1).
+- ``--rc``, to set the project version to ``<year-2000>.<month>.0rc1`` if the current version is not a release candidate, or bump the release candidate number by 1 if it is.
+- ``--dev``, to set the project development release number to 0 if it is not a development release, or bump the development release number by 1 if it is.
+- ``--patch``, to increment the patch number of the release. This will also reset the release candidate number, pass ``--rc`` at the same time to increment the patch number and make it a release candidate.
+- ``--post``, to set the project postrelease number to 0 if it is not a postrelease, or bump the postrelease number by 1 if it is. This will also reset the release candidate and development release numbers.
+
+If you give no arguments, it will strip the release candidate number, making it a "full release".
+
+Incremental supports "indeterminate" versions, as a stand-in for the next "full" version. This can be used when the version which will be displayed to the end-user is unknown (for example "introduced in" or "deprecated in"). Incremental supports the following indeterminate versions:
+
+- ``Version("<projectname>", "NEXT", 0, 0)``
+- ``<projectname> NEXT``
+
+When you run ``python -m incremental.update <projectname> --rc``, these will be updated to real versions (assuming the target final version is 17.1.0):
+
+- ``Version("<projectname>", 17, 1, 0, release_candidate=1)``
+- ``<projectname> 17.1.0rc1``
+
+Once the final version is made, it will become:
+
+- ``Version("<projectname>", 17, 1, 0)``
+- ``<projectname> 17.1.0``
+
+
+.. |coverage| image:: https://codecov.io/gh/twisted/incremental/branch/master/graph/badge.svg?token=K2ieeL887X
+.. _coverage: https://codecov.io/gh/twisted/incremental
+
+.. |gha| image:: https://github.com/twisted/incremental/actions/workflows/tests.yaml/badge.svg
+.. _gha: https://github.com/twisted/incremental/actions/workflows/tests.yaml
+
+.. |pypi| image:: http://img.shields.io/pypi/v/incremental.svg
+.. _pypi: https://pypi.python.org/pypi/incremental
diff --git a/contrib/python/incremental/py2/incremental/__init__.py b/contrib/python/incremental/py2/incremental/__init__.py
new file mode 100644
index 00000000000..c0c06c0ae1c
--- /dev/null
+++ b/contrib/python/incremental/py2/incremental/__init__.py
@@ -0,0 +1,395 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Versions for Python packages.
+
+See L{Version}.
+"""
+
+from __future__ import division, absolute_import
+
+import sys
+import warnings
+from typing import TYPE_CHECKING, Any, TypeVar, Union, Optional, Dict
+
+#
+# Compat functions
+#
+
+_T = TypeVar("_T", contravariant=True)
+
+
+if TYPE_CHECKING:
+    from typing_extensions import Literal
+    from distutils.dist import Distribution as _Distribution
+
+
+else:
+    _Distribution = object
+
+if sys.version_info > (3,):
+
+    def _cmp(a, b):  # type: (Any, Any) -> int
+        """
+        Compare two objects.
+
+        Returns a negative number if C{a < b}, zero if they are equal, and a
+        positive number if C{a > b}.
+        """
+        if a < b:
+            return -1
+        elif a == b:
+            return 0
+        else:
+            return 1
+
+
+else:
+    _cmp = cmp  # noqa: F821
+
+
+#
+# Versioning
+#
+
+
+class _Inf(object):
+    """
+    An object that is bigger than all other objects.
+    """
+
+    def __cmp__(self, other):  # type: (object) -> int
+        """
+        @param other: Another object.
+        @type other: any
+
+        @return: 0 if other is inf, 1 otherwise.
+        @rtype: C{int}
+        """
+        if other is _inf:
+            return 0
+        return 1
+
+    if sys.version_info >= (3,):
+
+        def __lt__(self, other):  # type: (object) -> bool
+            return self.__cmp__(other) < 0
+
+        def __le__(self, other):  # type: (object) -> bool
+            return self.__cmp__(other) <= 0
+
+        def __gt__(self, other):  # type: (object) -> bool
+            return self.__cmp__(other) > 0
+
+        def __ge__(self, other):  # type: (object) -> bool
+            return self.__cmp__(other) >= 0
+
+
+_inf = _Inf()
+
+
+class IncomparableVersions(TypeError):
+    """
+    Two versions could not be compared.
+    """
+
+
+class Version(object):
+    """
+    An encapsulation of a version for a project, with support for outputting
+    PEP-440 compatible version strings.
+
+    This class supports the standard major.minor.micro[rcN] scheme of
+    versioning.
+    """
+
+    def __init__(
+        self,
+        package,  # type: str
+        major,  # type: Union[Literal["NEXT"], int]
+        minor,  # type: int
+        micro,  # type: int
+        release_candidate=None,  # type: Optional[int]
+        prerelease=None,  # type: Optional[int]
+        post=None,  # type: Optional[int]
+        dev=None,  # type: Optional[int]
+    ):
+        """
+        @param package: Name of the package that this is a version of.
+        @type package: C{str}
+        @param major: The major version number.
+        @type major: C{int} or C{str} (for the "NEXT" symbol)
+        @param minor: The minor version number.
+        @type minor: C{int}
+        @param micro: The micro version number.
+        @type micro: C{int}
+        @param release_candidate: The release candidate number.
+        @type release_candidate: C{int}
+        @param prerelease: The prerelease number. (Deprecated)
+        @type prerelease: C{int}
+        @param post: The postrelease number.
+        @type post: C{int}
+        @param dev: The development release number.
+        @type dev: C{int}
+        """
+        if release_candidate and prerelease:
+            raise ValueError("Please only return one of these.")
+        elif prerelease and not release_candidate:
+            release_candidate = prerelease
+            warnings.warn(
+                "Passing prerelease to incremental.Version was "
+                "deprecated in Incremental 16.9.0. Please pass "
+                "release_candidate instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+        if major == "NEXT":
+            if minor or micro or release_candidate or post or dev:
+                raise ValueError(
+                    "When using NEXT, all other values except Package must be 0."
+                )
+
+        self.package = package
+        self.major = major
+        self.minor = minor
+        self.micro = micro
+        self.release_candidate = release_candidate
+        self.post = post
+        self.dev = dev
+
+    @property
+    def prerelease(self):  # type: () -> Optional[int]
+        warnings.warn(
+            "Accessing incremental.Version.prerelease was "
+            "deprecated in Incremental 16.9.0. Use "
+            "Version.release_candidate instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        ),
+        return self.release_candidate
+
+    def public(self):  # type: () -> str
+        """
+        Return a PEP440-compatible "public" representation of this L{Version}.
+
+        Examples:
+
+          - 14.4.0
+          - 1.2.3rc1
+          - 14.2.1rc1dev9
+          - 16.04.0dev0
+        """
+        if self.major == "NEXT":
+            return self.major
+
+        if self.release_candidate is None:
+            rc = ""
+        else:
+            rc = ".rc%s" % (self.release_candidate,)
+
+        if self.post is None:
+            post = ""
+        else:
+            post = ".post%s" % (self.post,)
+
+        if self.dev is None:
+            dev = ""
+        else:
+            dev = ".dev%s" % (self.dev,)
+
+        return "%r.%d.%d%s%s%s" % (self.major, self.minor, self.micro, rc, post, dev)
+
+    base = public
+    short = public
+    local = public
+
+    def __repr__(self):  # type: () -> str
+
+        if self.release_candidate is None:
+            release_candidate = ""
+        else:
+            release_candidate = ", release_candidate=%r" % (self.release_candidate,)
+
+        if self.post is None:
+            post = ""
+        else:
+            post = ", post=%r" % (self.post,)
+
+        if self.dev is None:
+            dev = ""
+        else:
+            dev = ", dev=%r" % (self.dev,)
+
+        return "%s(%r, %r, %d, %d%s%s%s)" % (
+            self.__class__.__name__,
+            self.package,
+            self.major,
+            self.minor,
+            self.micro,
+            release_candidate,
+            post,
+            dev,
+        )
+
+    def __str__(self):  # type: () -> str
+        return "[%s, version %s]" % (self.package, self.short())
+
+    def __cmp__(self, other):  # type: (Version) -> int
+        """
+        Compare two versions, considering major versions, minor versions, micro
+        versions, then release candidates, then postreleases, then dev
+        releases. Package names are case insensitive.
+
+        A version with a release candidate is always less than a version
+        without a release candidate. If both versions have release candidates,
+        they will be included in the comparison.
+
+        Likewise, a version with a dev release is always less than a version
+        without a dev release. If both versions have dev releases, they will
+        be included in the comparison.
+
+        @param other: Another version.
+        @type other: L{Version}
+
+        @return: NotImplemented when the other object is not a Version, or one
+            of -1, 0, or 1.
+
+        @raise IncomparableVersions: when the package names of the versions
+            differ.
+        """
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        if self.package.lower() != other.package.lower():
+            raise IncomparableVersions("%r != %r" % (self.package, other.package))
+
+        if self.major == "NEXT":
+            major = _inf  # type: Union[int, _Inf]
+        else:
+            major = self.major
+
+        if self.release_candidate is None:
+            release_candidate = _inf  # type: Union[int, _Inf]
+        else:
+            release_candidate = self.release_candidate
+
+        if self.post is None:
+            post = -1
+        else:
+            post = self.post
+
+        if self.dev is None:
+            dev = _inf  # type: Union[int, _Inf]
+        else:
+            dev = self.dev
+
+        if other.major == "NEXT":
+            othermajor = _inf  # type: Union[int, _Inf]
+        else:
+            othermajor = other.major
+
+        if other.release_candidate is None:
+            otherrc = _inf  # type: Union[int, _Inf]
+        else:
+            otherrc = other.release_candidate
+
+        if other.post is None:
+            otherpost = -1
+        else:
+            otherpost = other.post
+
+        if other.dev is None:
+            otherdev = _inf  # type: Union[int, _Inf]
+        else:
+            otherdev = other.dev
+
+        x = _cmp(
+            (major, self.minor, self.micro, release_candidate, post, dev),
+            (othermajor, other.minor, other.micro, otherrc, otherpost, otherdev),
+        )
+        return x
+
+    if sys.version_info >= (3,):
+
+        def __eq__(self, other):  # type: (Any) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c == 0
+
+        def __ne__(self, other):  # type: (Any) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c != 0
+
+        def __lt__(self, other):  # type: (Version) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c < 0
+
+        def __le__(self, other):  # type: (Version) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c <= 0
+
+        def __gt__(self, other):  # type: (Version) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c > 0
+
+        def __ge__(self, other):  # type: (Version) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c >= 0
+
+
+def getVersionString(version):  # type: (Version) -> str
+    """
+    Get a friendly string for the given version object.
+
+    @param version: A L{Version} object.
+    @return: A string containing the package and short version number.
+    """
+    result = "%s %s" % (version.package, version.short())
+    return result
+
+
+def _get_version(dist, keyword, value):  # type: (_Distribution, object, object) -> None
+    """
+    Get the version from the package listed in the Distribution.
+    """
+    if not value:
+        return
+
+    from distutils.command import build_py
+
+    sp_command = build_py.build_py(dist)
+    sp_command.finalize_options()
+
+    for item in sp_command.find_all_modules():  # type: ignore[attr-defined]
+        if item[1] == "_version":
+            version_file = {}  # type: Dict[str, Version]
+
+            with open(item[2]) as f:
+                exec(f.read(), version_file)
+
+            dist.metadata.version = version_file["__version__"].public()
+            return None
+
+    raise Exception("No _version.py found.")
+
+
+from ._version import __version__  # noqa: E402
+
+
+def _setuptools_version():  # type: () -> str
+    return __version__.public()
+
+
+__all__ = ["__version__", "Version", "getVersionString"]
diff --git a/contrib/python/incremental/py2/incremental/_version.py b/contrib/python/incremental/py2/incremental/_version.py
new file mode 100644
index 00000000000..12cb1b81510
--- /dev/null
+++ b/contrib/python/incremental/py2/incremental/_version.py
@@ -0,0 +1,11 @@
+"""
+Provides Incremental version information.
+"""
+
+# This file is auto-generated! Do not edit!
+# Use `python -m incremental.update Incremental` to change this file.
+
+from incremental import Version
+
+__version__ = Version("Incremental", 22, 10, 0)
+__all__ = ["__version__"]
diff --git a/contrib/python/incremental/py2/incremental/py.typed b/contrib/python/incremental/py2/incremental/py.typed
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/incremental/py2/incremental/update.py b/contrib/python/incremental/py2/incremental/update.py
new file mode 100644
index 00000000000..64a5cc84e79
--- /dev/null
+++ b/contrib/python/incremental/py2/incremental/update.py
@@ -0,0 +1,331 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from __future__ import absolute_import, division, print_function
+
+import click
+import os
+import datetime
+from typing import TYPE_CHECKING, Dict, Optional, Callable, Iterable
+
+from incremental import Version
+
+if TYPE_CHECKING:
+    from typing_extensions import Protocol
+
+    class _ReadableWritable(Protocol):
+        def read(self):  # type: () -> bytes
+            pass
+
+        def write(self, v):  # type: (bytes) -> object
+            pass
+
+        def __enter__(self):  # type: () -> _ReadableWritable
+            pass
+
+        def __exit__(self, *args, **kwargs):  # type: (object, object) -> Optional[bool]
+            pass
+
+    # FilePath is missing type annotations
+    # https://twistedmatrix.com/trac/ticket/10148
+    class FilePath(object):
+        def __init__(self, path):  # type: (str) -> None
+            self.path = path
+
+        def child(self, v):  # type: (str) -> FilePath
+            pass
+
+        def isdir(self):  # type: () -> bool
+            pass
+
+        def isfile(self):  # type: () -> bool
+            pass
+
+        def getContent(self):  # type: () -> bytes
+            pass
+
+        def open(self, mode):  # type: (str) -> _ReadableWritable
+            pass
+
+        def walk(self):  # type: () -> Iterable[FilePath]
+            pass
+
+
+else:
+    from twisted.python.filepath import FilePath
+
+_VERSIONPY_TEMPLATE = '''"""
+Provides {package} version information.
+"""
+
+# This file is auto-generated! Do not edit!
+# Use `python -m incremental.update {package}` to change this file.
+
+from incremental import Version
+
+__version__ = {version_repr}
+__all__ = ["__version__"]
+'''
+
+_YEAR_START = 2000
+
+
+def _findPath(path, package):  # type: (str, str) -> FilePath
+
+    cwd = FilePath(path)
+
+    src_dir = cwd.child("src").child(package.lower())
+    current_dir = cwd.child(package.lower())
+
+    if src_dir.isdir():
+        return src_dir
+    elif current_dir.isdir():
+        return current_dir
+    else:
+        raise ValueError(
+            "Can't find under `./src` or `./`. Check the "
+            "package name is right (note that we expect your "
+            "package name to be lower cased), or pass it using "
+            "'--path'."
+        )
+
+
+def _existing_version(path):  # type: (FilePath) -> Version
+    version_info = {}  # type: Dict[str, Version]
+
+    with path.child("_version.py").open("r") as f:
+        exec(f.read(), version_info)
+
+    return version_info["__version__"]
+
+
+def _run(
+    package,  # type: str
+    path,  # type: Optional[str]
+    newversion,  # type: Optional[str]
+    patch,  # type: bool
+    rc,  # type: bool
+    post,  # type: bool
+    dev,  # type: bool
+    create,  # type: bool
+    _date=None,  # type: Optional[datetime.date]
+    _getcwd=None,  # type: Optional[Callable[[], str]]
+    _print=print,  # type: Callable[[object], object]
+):  # type: (...) -> None
+
+    if not _getcwd:
+        _getcwd = os.getcwd
+
+    if not _date:
+        _date = datetime.date.today()
+
+    if type(package) != str:
+        package = package.encode("utf8")  # type: ignore[assignment]
+
+    _path = FilePath(path) if path else _findPath(_getcwd(), package)
+
+    if (
+        newversion
+        and patch
+        or newversion
+        and dev
+        or newversion
+        and rc
+        or newversion
+        and post
+    ):
+        raise ValueError("Only give --newversion")
+
+    if dev and patch or dev and rc or dev and post:
+        raise ValueError("Only give --dev")
+
+    if (
+        create
+        and dev
+        or create
+        and patch
+        or create
+        and rc
+        or create
+        and post
+        or create
+        and newversion
+    ):
+        raise ValueError("Only give --create")
+
+    if newversion:
+        from pkg_resources import parse_version
+
+        existing = _existing_version(_path)
+        st_version = parse_version(newversion)._version  # type: ignore[attr-defined]
+
+        release = list(st_version.release)
+
+        minor = 0
+        micro = 0
+        if len(release) == 1:
+            (major,) = release
+        elif len(release) == 2:
+            major, minor = release
+        else:
+            major, minor, micro = release
+
+        v = Version(
+            package,
+            major,
+            minor,
+            micro,
+            release_candidate=st_version.pre[1] if st_version.pre else None,
+            post=st_version.post[1] if st_version.post else None,
+            dev=st_version.dev[1] if st_version.dev else None,
+        )
+
+    elif create:
+        v = Version(package, _date.year - _YEAR_START, _date.month, 0)
+        existing = v
+
+    elif rc and not patch:
+        existing = _existing_version(_path)
+
+        if existing.release_candidate:
+            v = Version(
+                package,
+                existing.major,
+                existing.minor,
+                existing.micro,
+                existing.release_candidate + 1,
+            )
+        else:
+            v = Version(package, _date.year - _YEAR_START, _date.month, 0, 1)
+
+    elif patch:
+        existing = _existing_version(_path)
+        v = Version(
+            package,
+            existing.major,
+            existing.minor,
+            existing.micro + 1,
+            1 if rc else None,
+        )
+
+    elif post:
+        existing = _existing_version(_path)
+
+        if existing.post is None:
+            _post = 0
+        else:
+            _post = existing.post + 1
+
+        v = Version(package, existing.major, existing.minor, existing.micro, post=_post)
+
+    elif dev:
+        existing = _existing_version(_path)
+
+        if existing.dev is None:
+            _dev = 0
+        else:
+            _dev = existing.dev + 1
+
+        v = Version(
+            package,
+            existing.major,
+            existing.minor,
+            existing.micro,
+            existing.release_candidate,
+            dev=_dev,
+        )
+
+    else:
+        existing = _existing_version(_path)
+
+        if existing.release_candidate:
+            v = Version(package, existing.major, existing.minor, existing.micro)
+        else:
+            raise ValueError("You need to issue a rc before updating the major/minor")
+
+    NEXT_repr = repr(Version(package, "NEXT", 0, 0)).split("#")[0].replace("'", '"')
+    NEXT_repr_bytes = NEXT_repr.encode("utf8")
+
+    version_repr = repr(v).split("#")[0].replace("'", '"')
+    version_repr_bytes = version_repr.encode("utf8")
+
+    existing_version_repr = repr(existing).split("#")[0].replace("'", '"')
+    existing_version_repr_bytes = existing_version_repr.encode("utf8")
+
+    _print("Updating codebase to %s" % (v.public()))
+
+    for x in _path.walk():
+
+        if not x.isfile():
+            continue
+
+        original_content = x.getContent()
+        content = original_content
+
+        # Replace previous release_candidate calls to the new one
+        if existing.release_candidate:
+            content = content.replace(existing_version_repr_bytes, version_repr_bytes)
+            content = content.replace(
+                (package.encode("utf8") + b" " + existing.public().encode("utf8")),
+                (package.encode("utf8") + b" " + v.public().encode("utf8")),
+            )
+
+        # Replace NEXT Version calls with the new one
+        content = content.replace(NEXT_repr_bytes, version_repr_bytes)
+        content = content.replace(
+            NEXT_repr_bytes.replace(b"'", b'"'), version_repr_bytes
+        )
+
+        # Replace <package> NEXT with <package> <public>
+        content = content.replace(
+            package.encode("utf8") + b" NEXT",
+            (package.encode("utf8") + b" " + v.public().encode("utf8")),
+        )
+
+        if content != original_content:
+            _print("Updating %s" % (x.path,))
+            with x.open("w") as f:
+                f.write(content)
+
+    _print("Updating %s/_version.py" % (_path.path))
+    with _path.child("_version.py").open("w") as f:
+        f.write(
+            (
+                _VERSIONPY_TEMPLATE.format(package=package, version_repr=version_repr)
+            ).encode("utf8")
+        )
+
+
+@click.command()
+@click.argument("package")
+@click.option("--path", default=None)
+@click.option("--newversion", default=None)
+@click.option("--patch", is_flag=True)
+@click.option("--rc", is_flag=True)
+@click.option("--post", is_flag=True)
+@click.option("--dev", is_flag=True)
+@click.option("--create", is_flag=True)
+def run(
+    package,  # type: str
+    path,  # type: Optional[str]
+    newversion,  # type: Optional[str]
+    patch,  # type: bool
+    rc,  # type: bool
+    post,  # type: bool
+    dev,  # type: bool
+    create,  # type: bool
+):  # type: (...) -> None
+    return _run(
+        package=package,
+        path=path,
+        newversion=newversion,
+        patch=patch,
+        rc=rc,
+        post=post,
+        dev=dev,
+        create=create,
+    )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    run()
diff --git a/contrib/python/incremental/py2/ya.make b/contrib/python/incremental/py2/ya.make
new file mode 100644
index 00000000000..bf41780b7e1
--- /dev/null
+++ b/contrib/python/incremental/py2/ya.make
@@ -0,0 +1,30 @@
+# Generated by devtools/yamaker (pypi).
+
+PY2_LIBRARY()
+
+VERSION(22.10.0)
+
+LICENSE(MIT)
+
+NO_LINT()
+
+NO_CHECK_IMPORTS(
+    incremental.update
+)
+
+PY_SRCS(
+    TOP_LEVEL
+    incremental/__init__.py
+    incremental/_version.py
+    incremental/update.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/incremental/py2/
+    .dist-info/METADATA
+    .dist-info/entry_points.txt
+    .dist-info/top_level.txt
+    incremental/py.typed
+)
+
+END()
diff --git a/contrib/python/incremental/py3/.dist-info/METADATA b/contrib/python/incremental/py3/.dist-info/METADATA
new file mode 100644
index 00000000000..c4a94090826
--- /dev/null
+++ b/contrib/python/incremental/py3/.dist-info/METADATA
@@ -0,0 +1,136 @@
+Metadata-Version: 2.1
+Name: incremental
+Version: 22.10.0
+Summary: "A small library that versions your Python projects."
+Home-page: https://github.com/twisted/incremental
+Maintainer: Amber Brown
+Maintainer-email: hawkowl@twistedmatrix.com
+License: MIT
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+License-File: LICENSE
+Provides-Extra: mypy
+Requires-Dist: click (>=6.0) ; extra == 'mypy'
+Requires-Dist: twisted (>=16.4.0) ; extra == 'mypy'
+Requires-Dist: mypy (==0.812) ; extra == 'mypy'
+Provides-Extra: scripts
+Requires-Dist: click (>=6.0) ; extra == 'scripts'
+Requires-Dist: twisted (>=16.4.0) ; extra == 'scripts'
+
+Incremental
+===========
+
+|gha|
+|pypi|
+|coverage|
+
+Incremental is a small library that versions your Python projects.
+
+API documentation can be found `here <https://twisted.github.io/incremental/docs/>`_.
+
+
+Quick Start
+-----------
+
+Add this to your ``setup.py``\ 's ``setup()`` call, removing any other versioning arguments:
+
+.. code::
+
+   setup(
+       use_incremental=True,
+       setup_requires=['incremental'],
+       install_requires=['incremental'], # along with any other install dependencies
+       ...
+   }
+
+
+Install Incremental to your local environment with ``pip install incremental[scripts]``.
+Then run ``python -m incremental.update <projectname> --create``.
+It will create a file in your package named ``_version.py`` and look like this:
+
+.. code::
+
+   from incremental import Version
+
+   __version__ = Version("widgetbox", 17, 1, 0)
+   __all__ = ["__version__"]
+
+
+Then, so users of your project can find your version, in your root package's ``__init__.py`` add:
+
+.. code::
+
+   from ._version import __version__
+
+
+Subsequent installations of your project will then use Incremental for versioning.
+
+
+Incremental Versions
+--------------------
+
+``incremental.Version`` is a class that represents a version of a given project.
+It is made up of the following elements (which are given during instantiation):
+
+- ``package`` (required), the name of the package this ``Version`` represents.
+- ``major``, ``minor``, ``micro`` (all required), the X.Y.Z of your project's ``Version``.
+- ``release_candidate`` (optional), set to 0 or higher to mark this ``Version`` being of a release candidate (also sometimes called a "prerelease").
+- ``post`` (optional), set to 0 or higher to mark this ``Version`` as a postrelease.
+- ``dev`` (optional), set to 0 or higher to mark this ``Version`` as a development release.
+
+You can extract a PEP-440 compatible version string by using the ``.public()`` method, which returns a ``str`` containing the full version. This is the version you should provide to users, or publicly use. An example output would be ``"13.2.0"``, ``"17.1.2dev1"``, or ``"18.8.0rc2"``.
+
+Calling ``repr()`` with a ``Version`` will give a Python-source-code representation of it, and calling ``str()`` with a ``Version`` will provide a string similar to ``'[Incremental, version 16.10.1]'``.
+
+
+Updating
+--------
+
+Incremental includes a tool to automate updating your Incremental-using project's version called ``incremental.update``.
+It updates the ``_version.py`` file and automatically updates some uses of Incremental versions from an indeterminate version to the current one.
+It requires ``click`` from PyPI.
+
+``python -m incremental.update <projectname>`` will perform updates on that package.
+The commands that can be given after that will determine what the next version is.
+
+- ``--newversion=<version>``, to set the project version to a fully-specified version (like 1.2.3, or 17.1.0dev1).
+- ``--rc``, to set the project version to ``<year-2000>.<month>.0rc1`` if the current version is not a release candidate, or bump the release candidate number by 1 if it is.
+- ``--dev``, to set the project development release number to 0 if it is not a development release, or bump the development release number by 1 if it is.
+- ``--patch``, to increment the patch number of the release. This will also reset the release candidate number, pass ``--rc`` at the same time to increment the patch number and make it a release candidate.
+- ``--post``, to set the project postrelease number to 0 if it is not a postrelease, or bump the postrelease number by 1 if it is. This will also reset the release candidate and development release numbers.
+
+If you give no arguments, it will strip the release candidate number, making it a "full release".
+
+Incremental supports "indeterminate" versions, as a stand-in for the next "full" version. This can be used when the version which will be displayed to the end-user is unknown (for example "introduced in" or "deprecated in"). Incremental supports the following indeterminate versions:
+
+- ``Version("<projectname>", "NEXT", 0, 0)``
+- ``<projectname> NEXT``
+
+When you run ``python -m incremental.update <projectname> --rc``, these will be updated to real versions (assuming the target final version is 17.1.0):
+
+- ``Version("<projectname>", 17, 1, 0, release_candidate=1)``
+- ``<projectname> 17.1.0rc1``
+
+Once the final version is made, it will become:
+
+- ``Version("<projectname>", 17, 1, 0)``
+- ``<projectname> 17.1.0``
+
+
+.. |coverage| image:: https://codecov.io/gh/twisted/incremental/branch/master/graph/badge.svg?token=K2ieeL887X
+.. _coverage: https://codecov.io/gh/twisted/incremental
+
+.. |gha| image:: https://github.com/twisted/incremental/actions/workflows/tests.yaml/badge.svg
+.. _gha: https://github.com/twisted/incremental/actions/workflows/tests.yaml
+
+.. |pypi| image:: http://img.shields.io/pypi/v/incremental.svg
+.. _pypi: https://pypi.python.org/pypi/incremental
diff --git a/contrib/python/incremental/py3/.dist-info/entry_points.txt b/contrib/python/incremental/py3/.dist-info/entry_points.txt
new file mode 100644
index 00000000000..1c7b4877c1a
--- /dev/null
+++ b/contrib/python/incremental/py3/.dist-info/entry_points.txt
@@ -0,0 +1,2 @@
+[distutils.setup_keywords]
+use_incremental = incremental:_get_version
diff --git a/contrib/python/incremental/py3/.dist-info/top_level.txt b/contrib/python/incremental/py3/.dist-info/top_level.txt
new file mode 100644
index 00000000000..cb4023922ac
--- /dev/null
+++ b/contrib/python/incremental/py3/.dist-info/top_level.txt
@@ -0,0 +1 @@
+incremental
diff --git a/contrib/python/incremental/py3/LICENSE b/contrib/python/incremental/py3/LICENSE
new file mode 100644
index 00000000000..bc9d47f3322
--- /dev/null
+++ b/contrib/python/incremental/py3/LICENSE
@@ -0,0 +1,74 @@
+Incremental
+-----------
+
+This project includes code from the Twisted Project, which is licensed as below.
+
+Copyright (c) 2001-2015
+Allen Short
+Amber Hawkie Brown
+Andrew Bennetts
+Andy Gayton
+Antoine Pitrou
+Apple Computer, Inc.
+Ashwini Oruganti
+Benjamin Bruheim
+Bob Ippolito
+Canonical Limited
+Christopher Armstrong
+David Reid
+Divmod Inc.
+Donovan Preston
+Eric Mangold
+Eyal Lotem
+Google Inc.
+Hybrid Logic Ltd.
+Hynek Schlawack
+Itamar Turner-Trauring
+James Knight
+Jason A. Mobarak
+Jean-Paul Calderone
+Jessica McKellar
+Jonathan D. Simms
+Jonathan Jacobs
+Jonathan Lange
+Julian Berman
+Jürgen Hermann
+Kevin Horn
+Kevin Turner
+Laurens Van Houtven
+Mary Gardiner
+Massachusetts Institute of Technology
+Matthew Lefkowitz
+Moshe Zadka
+Paul Swartz
+Pavel Pergamenshchik
+Rackspace, US Inc.
+Ralph Meijer
+Richard Wall
+Sean Riley
+Software Freedom Conservancy
+Tavendo GmbH
+Thijs Triemstra
+Thomas Herve
+Timothy Allen
+Tom Prince
+Travis B. Hartwell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/python/incremental/py3/README.rst b/contrib/python/incremental/py3/README.rst
new file mode 100644
index 00000000000..7a20d077af2
--- /dev/null
+++ b/contrib/python/incremental/py3/README.rst
@@ -0,0 +1,108 @@
+Incremental
+===========
+
+|gha|
+|pypi|
+|coverage|
+
+Incremental is a small library that versions your Python projects.
+
+API documentation can be found `here <https://twisted.github.io/incremental/docs/>`_.
+
+
+Quick Start
+-----------
+
+Add this to your ``setup.py``\ 's ``setup()`` call, removing any other versioning arguments:
+
+.. code::
+
+   setup(
+       use_incremental=True,
+       setup_requires=['incremental'],
+       install_requires=['incremental'], # along with any other install dependencies
+       ...
+   }
+
+
+Install Incremental to your local environment with ``pip install incremental[scripts]``.
+Then run ``python -m incremental.update <projectname> --create``.
+It will create a file in your package named ``_version.py`` and look like this:
+
+.. code::
+
+   from incremental import Version
+
+   __version__ = Version("widgetbox", 17, 1, 0)
+   __all__ = ["__version__"]
+
+
+Then, so users of your project can find your version, in your root package's ``__init__.py`` add:
+
+.. code::
+
+   from ._version import __version__
+
+
+Subsequent installations of your project will then use Incremental for versioning.
+
+
+Incremental Versions
+--------------------
+
+``incremental.Version`` is a class that represents a version of a given project.
+It is made up of the following elements (which are given during instantiation):
+
+- ``package`` (required), the name of the package this ``Version`` represents.
+- ``major``, ``minor``, ``micro`` (all required), the X.Y.Z of your project's ``Version``.
+- ``release_candidate`` (optional), set to 0 or higher to mark this ``Version`` being of a release candidate (also sometimes called a "prerelease").
+- ``post`` (optional), set to 0 or higher to mark this ``Version`` as a postrelease.
+- ``dev`` (optional), set to 0 or higher to mark this ``Version`` as a development release.
+
+You can extract a PEP-440 compatible version string by using the ``.public()`` method, which returns a ``str`` containing the full version. This is the version you should provide to users, or publicly use. An example output would be ``"13.2.0"``, ``"17.1.2dev1"``, or ``"18.8.0rc2"``.
+
+Calling ``repr()`` with a ``Version`` will give a Python-source-code representation of it, and calling ``str()`` with a ``Version`` will provide a string similar to ``'[Incremental, version 16.10.1]'``.
+
+
+Updating
+--------
+
+Incremental includes a tool to automate updating your Incremental-using project's version called ``incremental.update``.
+It updates the ``_version.py`` file and automatically updates some uses of Incremental versions from an indeterminate version to the current one.
+It requires ``click`` from PyPI.
+
+``python -m incremental.update <projectname>`` will perform updates on that package.
+The commands that can be given after that will determine what the next version is.
+
+- ``--newversion=<version>``, to set the project version to a fully-specified version (like 1.2.3, or 17.1.0dev1).
+- ``--rc``, to set the project version to ``<year-2000>.<month>.0rc1`` if the current version is not a release candidate, or bump the release candidate number by 1 if it is.
+- ``--dev``, to set the project development release number to 0 if it is not a development release, or bump the development release number by 1 if it is.
+- ``--patch``, to increment the patch number of the release. This will also reset the release candidate number, pass ``--rc`` at the same time to increment the patch number and make it a release candidate.
+- ``--post``, to set the project postrelease number to 0 if it is not a postrelease, or bump the postrelease number by 1 if it is. This will also reset the release candidate and development release numbers.
+
+If you give no arguments, it will strip the release candidate number, making it a "full release".
+
+Incremental supports "indeterminate" versions, as a stand-in for the next "full" version. This can be used when the version which will be displayed to the end-user is unknown (for example "introduced in" or "deprecated in"). Incremental supports the following indeterminate versions:
+
+- ``Version("<projectname>", "NEXT", 0, 0)``
+- ``<projectname> NEXT``
+
+When you run ``python -m incremental.update <projectname> --rc``, these will be updated to real versions (assuming the target final version is 17.1.0):
+
+- ``Version("<projectname>", 17, 1, 0, release_candidate=1)``
+- ``<projectname> 17.1.0rc1``
+
+Once the final version is made, it will become:
+
+- ``Version("<projectname>", 17, 1, 0)``
+- ``<projectname> 17.1.0``
+
+
+.. |coverage| image:: https://codecov.io/gh/twisted/incremental/branch/master/graph/badge.svg?token=K2ieeL887X
+.. _coverage: https://codecov.io/gh/twisted/incremental
+
+.. |gha| image:: https://github.com/twisted/incremental/actions/workflows/tests.yaml/badge.svg
+.. _gha: https://github.com/twisted/incremental/actions/workflows/tests.yaml
+
+.. |pypi| image:: http://img.shields.io/pypi/v/incremental.svg
+.. _pypi: https://pypi.python.org/pypi/incremental
diff --git a/contrib/python/incremental/py3/incremental/__init__.py b/contrib/python/incremental/py3/incremental/__init__.py
new file mode 100644
index 00000000000..c0c06c0ae1c
--- /dev/null
+++ b/contrib/python/incremental/py3/incremental/__init__.py
@@ -0,0 +1,395 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Versions for Python packages.
+
+See L{Version}.
+"""
+
+from __future__ import division, absolute_import
+
+import sys
+import warnings
+from typing import TYPE_CHECKING, Any, TypeVar, Union, Optional, Dict
+
+#
+# Compat functions
+#
+
+_T = TypeVar("_T", contravariant=True)
+
+
+if TYPE_CHECKING:
+    from typing_extensions import Literal
+    from distutils.dist import Distribution as _Distribution
+
+
+else:
+    _Distribution = object
+
+if sys.version_info > (3,):
+
+    def _cmp(a, b):  # type: (Any, Any) -> int
+        """
+        Compare two objects.
+
+        Returns a negative number if C{a < b}, zero if they are equal, and a
+        positive number if C{a > b}.
+        """
+        if a < b:
+            return -1
+        elif a == b:
+            return 0
+        else:
+            return 1
+
+
+else:
+    _cmp = cmp  # noqa: F821
+
+
+#
+# Versioning
+#
+
+
+class _Inf(object):
+    """
+    An object that is bigger than all other objects.
+    """
+
+    def __cmp__(self, other):  # type: (object) -> int
+        """
+        @param other: Another object.
+        @type other: any
+
+        @return: 0 if other is inf, 1 otherwise.
+        @rtype: C{int}
+        """
+        if other is _inf:
+            return 0
+        return 1
+
+    if sys.version_info >= (3,):
+
+        def __lt__(self, other):  # type: (object) -> bool
+            return self.__cmp__(other) < 0
+
+        def __le__(self, other):  # type: (object) -> bool
+            return self.__cmp__(other) <= 0
+
+        def __gt__(self, other):  # type: (object) -> bool
+            return self.__cmp__(other) > 0
+
+        def __ge__(self, other):  # type: (object) -> bool
+            return self.__cmp__(other) >= 0
+
+
+_inf = _Inf()
+
+
+class IncomparableVersions(TypeError):
+    """
+    Two versions could not be compared.
+    """
+
+
+class Version(object):
+    """
+    An encapsulation of a version for a project, with support for outputting
+    PEP-440 compatible version strings.
+
+    This class supports the standard major.minor.micro[rcN] scheme of
+    versioning.
+    """
+
+    def __init__(
+        self,
+        package,  # type: str
+        major,  # type: Union[Literal["NEXT"], int]
+        minor,  # type: int
+        micro,  # type: int
+        release_candidate=None,  # type: Optional[int]
+        prerelease=None,  # type: Optional[int]
+        post=None,  # type: Optional[int]
+        dev=None,  # type: Optional[int]
+    ):
+        """
+        @param package: Name of the package that this is a version of.
+        @type package: C{str}
+        @param major: The major version number.
+        @type major: C{int} or C{str} (for the "NEXT" symbol)
+        @param minor: The minor version number.
+        @type minor: C{int}
+        @param micro: The micro version number.
+        @type micro: C{int}
+        @param release_candidate: The release candidate number.
+        @type release_candidate: C{int}
+        @param prerelease: The prerelease number. (Deprecated)
+        @type prerelease: C{int}
+        @param post: The postrelease number.
+        @type post: C{int}
+        @param dev: The development release number.
+        @type dev: C{int}
+        """
+        if release_candidate and prerelease:
+            raise ValueError("Please only return one of these.")
+        elif prerelease and not release_candidate:
+            release_candidate = prerelease
+            warnings.warn(
+                "Passing prerelease to incremental.Version was "
+                "deprecated in Incremental 16.9.0. Please pass "
+                "release_candidate instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+        if major == "NEXT":
+            if minor or micro or release_candidate or post or dev:
+                raise ValueError(
+                    "When using NEXT, all other values except Package must be 0."
+                )
+
+        self.package = package
+        self.major = major
+        self.minor = minor
+        self.micro = micro
+        self.release_candidate = release_candidate
+        self.post = post
+        self.dev = dev
+
+    @property
+    def prerelease(self):  # type: () -> Optional[int]
+        warnings.warn(
+            "Accessing incremental.Version.prerelease was "
+            "deprecated in Incremental 16.9.0. Use "
+            "Version.release_candidate instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        ),
+        return self.release_candidate
+
+    def public(self):  # type: () -> str
+        """
+        Return a PEP440-compatible "public" representation of this L{Version}.
+
+        Examples:
+
+          - 14.4.0
+          - 1.2.3rc1
+          - 14.2.1rc1dev9
+          - 16.04.0dev0
+        """
+        if self.major == "NEXT":
+            return self.major
+
+        if self.release_candidate is None:
+            rc = ""
+        else:
+            rc = ".rc%s" % (self.release_candidate,)
+
+        if self.post is None:
+            post = ""
+        else:
+            post = ".post%s" % (self.post,)
+
+        if self.dev is None:
+            dev = ""
+        else:
+            dev = ".dev%s" % (self.dev,)
+
+        return "%r.%d.%d%s%s%s" % (self.major, self.minor, self.micro, rc, post, dev)
+
+    base = public
+    short = public
+    local = public
+
+    def __repr__(self):  # type: () -> str
+
+        if self.release_candidate is None:
+            release_candidate = ""
+        else:
+            release_candidate = ", release_candidate=%r" % (self.release_candidate,)
+
+        if self.post is None:
+            post = ""
+        else:
+            post = ", post=%r" % (self.post,)
+
+        if self.dev is None:
+            dev = ""
+        else:
+            dev = ", dev=%r" % (self.dev,)
+
+        return "%s(%r, %r, %d, %d%s%s%s)" % (
+            self.__class__.__name__,
+            self.package,
+            self.major,
+            self.minor,
+            self.micro,
+            release_candidate,
+            post,
+            dev,
+        )
+
+    def __str__(self):  # type: () -> str
+        return "[%s, version %s]" % (self.package, self.short())
+
+    def __cmp__(self, other):  # type: (Version) -> int
+        """
+        Compare two versions, considering major versions, minor versions, micro
+        versions, then release candidates, then postreleases, then dev
+        releases. Package names are case insensitive.
+
+        A version with a release candidate is always less than a version
+        without a release candidate. If both versions have release candidates,
+        they will be included in the comparison.
+
+        Likewise, a version with a dev release is always less than a version
+        without a dev release. If both versions have dev releases, they will
+        be included in the comparison.
+
+        @param other: Another version.
+        @type other: L{Version}
+
+        @return: NotImplemented when the other object is not a Version, or one
+            of -1, 0, or 1.
+
+        @raise IncomparableVersions: when the package names of the versions
+            differ.
+        """
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        if self.package.lower() != other.package.lower():
+            raise IncomparableVersions("%r != %r" % (self.package, other.package))
+
+        if self.major == "NEXT":
+            major = _inf  # type: Union[int, _Inf]
+        else:
+            major = self.major
+
+        if self.release_candidate is None:
+            release_candidate = _inf  # type: Union[int, _Inf]
+        else:
+            release_candidate = self.release_candidate
+
+        if self.post is None:
+            post = -1
+        else:
+            post = self.post
+
+        if self.dev is None:
+            dev = _inf  # type: Union[int, _Inf]
+        else:
+            dev = self.dev
+
+        if other.major == "NEXT":
+            othermajor = _inf  # type: Union[int, _Inf]
+        else:
+            othermajor = other.major
+
+        if other.release_candidate is None:
+            otherrc = _inf  # type: Union[int, _Inf]
+        else:
+            otherrc = other.release_candidate
+
+        if other.post is None:
+            otherpost = -1
+        else:
+            otherpost = other.post
+
+        if other.dev is None:
+            otherdev = _inf  # type: Union[int, _Inf]
+        else:
+            otherdev = other.dev
+
+        x = _cmp(
+            (major, self.minor, self.micro, release_candidate, post, dev),
+            (othermajor, other.minor, other.micro, otherrc, otherpost, otherdev),
+        )
+        return x
+
+    if sys.version_info >= (3,):
+
+        def __eq__(self, other):  # type: (Any) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c == 0
+
+        def __ne__(self, other):  # type: (Any) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c != 0
+
+        def __lt__(self, other):  # type: (Version) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c < 0
+
+        def __le__(self, other):  # type: (Version) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c <= 0
+
+        def __gt__(self, other):  # type: (Version) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c > 0
+
+        def __ge__(self, other):  # type: (Version) -> bool
+            c = self.__cmp__(other)
+            if c is NotImplemented:
+                return c  # type: ignore[return-value]
+            return c >= 0
+
+
+def getVersionString(version):  # type: (Version) -> str
+    """
+    Get a friendly string for the given version object.
+
+    @param version: A L{Version} object.
+    @return: A string containing the package and short version number.
+    """
+    result = "%s %s" % (version.package, version.short())
+    return result
+
+
+def _get_version(dist, keyword, value):  # type: (_Distribution, object, object) -> None
+    """
+    Get the version from the package listed in the Distribution.
+    """
+    if not value:
+        return
+
+    from distutils.command import build_py
+
+    sp_command = build_py.build_py(dist)
+    sp_command.finalize_options()
+
+    for item in sp_command.find_all_modules():  # type: ignore[attr-defined]
+        if item[1] == "_version":
+            version_file = {}  # type: Dict[str, Version]
+
+            with open(item[2]) as f:
+                exec(f.read(), version_file)
+
+            dist.metadata.version = version_file["__version__"].public()
+            return None
+
+    raise Exception("No _version.py found.")
+
+
+from ._version import __version__  # noqa: E402
+
+
+def _setuptools_version():  # type: () -> str
+    return __version__.public()
+
+
+__all__ = ["__version__", "Version", "getVersionString"]
diff --git a/contrib/python/incremental/py3/incremental/_version.py b/contrib/python/incremental/py3/incremental/_version.py
new file mode 100644
index 00000000000..12cb1b81510
--- /dev/null
+++ b/contrib/python/incremental/py3/incremental/_version.py
@@ -0,0 +1,11 @@
+"""
+Provides Incremental version information.
+"""
+
+# This file is auto-generated! Do not edit!
+# Use `python -m incremental.update Incremental` to change this file.
+
+from incremental import Version
+
+__version__ = Version("Incremental", 22, 10, 0)
+__all__ = ["__version__"]
diff --git a/contrib/python/incremental/py3/incremental/py.typed b/contrib/python/incremental/py3/incremental/py.typed
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/incremental/py3/incremental/update.py b/contrib/python/incremental/py3/incremental/update.py
new file mode 100644
index 00000000000..64a5cc84e79
--- /dev/null
+++ b/contrib/python/incremental/py3/incremental/update.py
@@ -0,0 +1,331 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+from __future__ import absolute_import, division, print_function
+
+import click
+import os
+import datetime
+from typing import TYPE_CHECKING, Dict, Optional, Callable, Iterable
+
+from incremental import Version
+
+if TYPE_CHECKING:
+    from typing_extensions import Protocol
+
+    class _ReadableWritable(Protocol):
+        def read(self):  # type: () -> bytes
+            pass
+
+        def write(self, v):  # type: (bytes) -> object
+            pass
+
+        def __enter__(self):  # type: () -> _ReadableWritable
+            pass
+
+        def __exit__(self, *args, **kwargs):  # type: (object, object) -> Optional[bool]
+            pass
+
+    # FilePath is missing type annotations
+    # https://twistedmatrix.com/trac/ticket/10148
+    class FilePath(object):
+        def __init__(self, path):  # type: (str) -> None
+            self.path = path
+
+        def child(self, v):  # type: (str) -> FilePath
+            pass
+
+        def isdir(self):  # type: () -> bool
+            pass
+
+        def isfile(self):  # type: () -> bool
+            pass
+
+        def getContent(self):  # type: () -> bytes
+            pass
+
+        def open(self, mode):  # type: (str) -> _ReadableWritable
+            pass
+
+        def walk(self):  # type: () -> Iterable[FilePath]
+            pass
+
+
+else:
+    from twisted.python.filepath import FilePath
+
+_VERSIONPY_TEMPLATE = '''"""
+Provides {package} version information.
+"""
+
+# This file is auto-generated! Do not edit!
+# Use `python -m incremental.update {package}` to change this file.
+
+from incremental import Version
+
+__version__ = {version_repr}
+__all__ = ["__version__"]
+'''
+
+_YEAR_START = 2000
+
+
+def _findPath(path, package):  # type: (str, str) -> FilePath
+
+    cwd = FilePath(path)
+
+    src_dir = cwd.child("src").child(package.lower())
+    current_dir = cwd.child(package.lower())
+
+    if src_dir.isdir():
+        return src_dir
+    elif current_dir.isdir():
+        return current_dir
+    else:
+        raise ValueError(
+            "Can't find under `./src` or `./`. Check the "
+            "package name is right (note that we expect your "
+            "package name to be lower cased), or pass it using "
+            "'--path'."
+        )
+
+
+def _existing_version(path):  # type: (FilePath) -> Version
+    version_info = {}  # type: Dict[str, Version]
+
+    with path.child("_version.py").open("r") as f:
+        exec(f.read(), version_info)
+
+    return version_info["__version__"]
+
+
+def _run(
+    package,  # type: str
+    path,  # type: Optional[str]
+    newversion,  # type: Optional[str]
+    patch,  # type: bool
+    rc,  # type: bool
+    post,  # type: bool
+    dev,  # type: bool
+    create,  # type: bool
+    _date=None,  # type: Optional[datetime.date]
+    _getcwd=None,  # type: Optional[Callable[[], str]]
+    _print=print,  # type: Callable[[object], object]
+):  # type: (...) -> None
+
+    if not _getcwd:
+        _getcwd = os.getcwd
+
+    if not _date:
+        _date = datetime.date.today()
+
+    if type(package) != str:
+        package = package.encode("utf8")  # type: ignore[assignment]
+
+    _path = FilePath(path) if path else _findPath(_getcwd(), package)
+
+    if (
+        newversion
+        and patch
+        or newversion
+        and dev
+        or newversion
+        and rc
+        or newversion
+        and post
+    ):
+        raise ValueError("Only give --newversion")
+
+    if dev and patch or dev and rc or dev and post:
+        raise ValueError("Only give --dev")
+
+    if (
+        create
+        and dev
+        or create
+        and patch
+        or create
+        and rc
+        or create
+        and post
+        or create
+        and newversion
+    ):
+        raise ValueError("Only give --create")
+
+    if newversion:
+        from pkg_resources import parse_version
+
+        existing = _existing_version(_path)
+        st_version = parse_version(newversion)._version  # type: ignore[attr-defined]
+
+        release = list(st_version.release)
+
+        minor = 0
+        micro = 0
+        if len(release) == 1:
+            (major,) = release
+        elif len(release) == 2:
+            major, minor = release
+        else:
+            major, minor, micro = release
+
+        v = Version(
+            package,
+            major,
+            minor,
+            micro,
+            release_candidate=st_version.pre[1] if st_version.pre else None,
+            post=st_version.post[1] if st_version.post else None,
+            dev=st_version.dev[1] if st_version.dev else None,
+        )
+
+    elif create:
+        v = Version(package, _date.year - _YEAR_START, _date.month, 0)
+        existing = v
+
+    elif rc and not patch:
+        existing = _existing_version(_path)
+
+        if existing.release_candidate:
+            v = Version(
+                package,
+                existing.major,
+                existing.minor,
+                existing.micro,
+                existing.release_candidate + 1,
+            )
+        else:
+            v = Version(package, _date.year - _YEAR_START, _date.month, 0, 1)
+
+    elif patch:
+        existing = _existing_version(_path)
+        v = Version(
+            package,
+            existing.major,
+            existing.minor,
+            existing.micro + 1,
+            1 if rc else None,
+        )
+
+    elif post:
+        existing = _existing_version(_path)
+
+        if existing.post is None:
+            _post = 0
+        else:
+            _post = existing.post + 1
+
+        v = Version(package, existing.major, existing.minor, existing.micro, post=_post)
+
+    elif dev:
+        existing = _existing_version(_path)
+
+        if existing.dev is None:
+            _dev = 0
+        else:
+            _dev = existing.dev + 1
+
+        v = Version(
+            package,
+            existing.major,
+            existing.minor,
+            existing.micro,
+            existing.release_candidate,
+            dev=_dev,
+        )
+
+    else:
+        existing = _existing_version(_path)
+
+        if existing.release_candidate:
+            v = Version(package, existing.major, existing.minor, existing.micro)
+        else:
+            raise ValueError("You need to issue a rc before updating the major/minor")
+
+    NEXT_repr = repr(Version(package, "NEXT", 0, 0)).split("#")[0].replace("'", '"')
+    NEXT_repr_bytes = NEXT_repr.encode("utf8")
+
+    version_repr = repr(v).split("#")[0].replace("'", '"')
+    version_repr_bytes = version_repr.encode("utf8")
+
+    existing_version_repr = repr(existing).split("#")[0].replace("'", '"')
+    existing_version_repr_bytes = existing_version_repr.encode("utf8")
+
+    _print("Updating codebase to %s" % (v.public()))
+
+    for x in _path.walk():
+
+        if not x.isfile():
+            continue
+
+        original_content = x.getContent()
+        content = original_content
+
+        # Replace previous release_candidate calls to the new one
+        if existing.release_candidate:
+            content = content.replace(existing_version_repr_bytes, version_repr_bytes)
+            content = content.replace(
+                (package.encode("utf8") + b" " + existing.public().encode("utf8")),
+                (package.encode("utf8") + b" " + v.public().encode("utf8")),
+            )
+
+        # Replace NEXT Version calls with the new one
+        content = content.replace(NEXT_repr_bytes, version_repr_bytes)
+        content = content.replace(
+            NEXT_repr_bytes.replace(b"'", b'"'), version_repr_bytes
+        )
+
+        # Replace <package> NEXT with <package> <public>
+        content = content.replace(
+            package.encode("utf8") + b" NEXT",
+            (package.encode("utf8") + b" " + v.public().encode("utf8")),
+        )
+
+        if content != original_content:
+            _print("Updating %s" % (x.path,))
+            with x.open("w") as f:
+                f.write(content)
+
+    _print("Updating %s/_version.py" % (_path.path))
+    with _path.child("_version.py").open("w") as f:
+        f.write(
+            (
+                _VERSIONPY_TEMPLATE.format(package=package, version_repr=version_repr)
+            ).encode("utf8")
+        )
+
+
+@click.command()
+@click.argument("package")
+@click.option("--path", default=None)
+@click.option("--newversion", default=None)
+@click.option("--patch", is_flag=True)
+@click.option("--rc", is_flag=True)
+@click.option("--post", is_flag=True)
+@click.option("--dev", is_flag=True)
+@click.option("--create", is_flag=True)
+def run(
+    package,  # type: str
+    path,  # type: Optional[str]
+    newversion,  # type: Optional[str]
+    patch,  # type: bool
+    rc,  # type: bool
+    post,  # type: bool
+    dev,  # type: bool
+    create,  # type: bool
+):  # type: (...) -> None
+    return _run(
+        package=package,
+        path=path,
+        newversion=newversion,
+        patch=patch,
+        rc=rc,
+        post=post,
+        dev=dev,
+        create=create,
+    )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    run()
diff --git a/contrib/python/incremental/py3/ya.make b/contrib/python/incremental/py3/ya.make
new file mode 100644
index 00000000000..57846dd33a5
--- /dev/null
+++ b/contrib/python/incremental/py3/ya.make
@@ -0,0 +1,30 @@
+# Generated by devtools/yamaker (pypi).
+
+PY3_LIBRARY()
+
+VERSION(22.10.0)
+
+LICENSE(MIT)
+
+NO_LINT()
+
+NO_CHECK_IMPORTS(
+    incremental.update
+)
+
+PY_SRCS(
+    TOP_LEVEL
+    incremental/__init__.py
+    incremental/_version.py
+    incremental/update.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/incremental/py3/
+    .dist-info/METADATA
+    .dist-info/entry_points.txt
+    .dist-info/top_level.txt
+    incremental/py.typed
+)
+
+END()
diff --git a/contrib/python/incremental/ya.make b/contrib/python/incremental/ya.make
new file mode 100644
index 00000000000..ad23ac6b27b
--- /dev/null
+++ b/contrib/python/incremental/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+    PEERDIR(contrib/python/incremental/py2)
+ELSE()
+    PEERDIR(contrib/python/incremental/py3)
+ENDIF()
+
+NO_LINT()
+
+END()
+
+RECURSE(
+    py2
+    py3
+)
diff --git a/contrib/python/jsonschema/py2/.dist-info/METADATA b/contrib/python/jsonschema/py2/.dist-info/METADATA
new file mode 100644
index 00000000000..aef9b18d586
--- /dev/null
+++ b/contrib/python/jsonschema/py2/.dist-info/METADATA
@@ -0,0 +1,224 @@
+Metadata-Version: 2.1
+Name: jsonschema
+Version: 3.2.0
+Summary: An implementation of JSON Schema validation for Python
+Home-page: https://github.com/Julian/jsonschema
+Author: Julian Berman
+Author-email: Julian@GrayVines.com
+License: UNKNOWN
+Project-URL: Docs, https://python-jsonschema.readthedocs.io/en/latest/
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Dist: attrs (>=17.4.0)
+Requires-Dist: pyrsistent (>=0.14.0)
+Requires-Dist: setuptools
+Requires-Dist: six (>=1.11.0)
+Requires-Dist: functools32 ; python_version < "3"
+Requires-Dist: importlib-metadata ; python_version < "3.8"
+Provides-Extra: format
+Requires-Dist: idna ; extra == 'format'
+Requires-Dist: jsonpointer (>1.13) ; extra == 'format'
+Requires-Dist: rfc3987 ; extra == 'format'
+Requires-Dist: strict-rfc3339 ; extra == 'format'
+Requires-Dist: webcolors ; extra == 'format'
+Provides-Extra: format_nongpl
+Requires-Dist: idna ; extra == 'format_nongpl'
+Requires-Dist: jsonpointer (>1.13) ; extra == 'format_nongpl'
+Requires-Dist: webcolors ; extra == 'format_nongpl'
+Requires-Dist: rfc3986-validator (>0.1.0) ; extra == 'format_nongpl'
+Requires-Dist: rfc3339-validator ; extra == 'format_nongpl'
+
+==========
+jsonschema
+==========
+
+|PyPI| |Pythons| |Travis| |AppVeyor| |Codecov| |ReadTheDocs|
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/jsonschema.svg
+   :alt: PyPI version
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Pythons| image:: https://img.shields.io/pypi/pyversions/jsonschema.svg
+   :alt: Supported Python versions
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Travis| image:: https://travis-ci.com/Julian/jsonschema.svg?branch=master
+   :alt: Travis build status
+   :target: https://travis-ci.com/Julian/jsonschema
+
+.. |AppVeyor| image:: https://ci.appveyor.com/api/projects/status/adtt0aiaihy6muyn/branch/master?svg=true
+   :alt: AppVeyor build status
+   :target: https://ci.appveyor.com/project/Julian/jsonschema
+
+.. |Codecov| image:: https://codecov.io/gh/Julian/jsonschema/branch/master/graph/badge.svg
+   :alt: Codecov Code coverage
+   :target: https://codecov.io/gh/Julian/jsonschema
+
+.. |ReadTheDocs| image:: https://readthedocs.org/projects/python-jsonschema/badge/?version=stable&style=flat
+   :alt: ReadTheDocs status
+   :target: https://python-jsonschema.readthedocs.io/en/stable/
+
+
+``jsonschema`` is an implementation of `JSON Schema <https://json-schema.org>`_
+for Python (supporting 2.7+ including Python 3).
+
+.. code-block:: python
+
+    >>> from jsonschema import validate
+
+    >>> # A sample schema, like what we'd get from json.load()
+    >>> schema = {
+    ...     "type" : "object",
+    ...     "properties" : {
+    ...         "price" : {"type" : "number"},
+    ...         "name" : {"type" : "string"},
+    ...     },
+    ... }
+
+    >>> # If no exception is raised by validate(), the instance is valid.
+    >>> validate(instance={"name" : "Eggs", "price" : 34.99}, schema=schema)
+
+    >>> validate(
+    ...     instance={"name" : "Eggs", "price" : "Invalid"}, schema=schema,
+    ... )                                   # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValidationError: 'Invalid' is not of type 'number'
+
+It can also be used from console:
+
+.. code-block:: bash
+
+    $ jsonschema -i sample.json sample.schema
+
+Features
+--------
+
+* Full support for
+  `Draft 7 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft7Validator>`_,
+  `Draft 6 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft6Validator>`_,
+  `Draft 4 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft4Validator>`_
+  and
+  `Draft 3 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft3Validator>`_
+
+* `Lazy validation <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.IValidator.iter_errors>`_
+  that can iteratively report *all* validation errors.
+
+* `Programmatic querying <https://python-jsonschema.readthedocs.io/en/latest/errors/>`_
+  of which properties or items failed validation.
+
+
+Installation
+------------
+
+``jsonschema`` is available on `PyPI <https://pypi.org/project/jsonschema/>`_. You can install using `pip <https://pip.pypa.io/en/stable/>`_:
+
+.. code-block:: bash
+
+    $ pip install jsonschema
+
+
+Demo
+----
+
+Try ``jsonschema`` interactively in this online demo:
+
+.. image:: https://user-images.githubusercontent.com/1155573/56745335-8b158a00-6750-11e9-8776-83fa675939c4.png
+    :target: https://notebooks.ai/demo/gh/Julian/jsonschema
+    :alt: Open Live Demo
+
+
+Online demo Notebook will look similar to this:
+
+
+.. image:: https://user-images.githubusercontent.com/1155573/56820861-5c1c1880-6823-11e9-802a-ce01c5ec574f.gif
+    :alt: Open Live Demo
+    :width: 480 px
+
+
+Release Notes
+-------------
+
+v3.1 brings support for ECMA 262 dialect regular expressions
+throughout schemas, as recommended by the specification. Big
+thanks to @Zac-HD for authoring support in a new `js-regex
+<https://pypi.org/project/js-regex/>`_ library.
+
+
+Running the Test Suite
+----------------------
+
+If you have ``tox`` installed (perhaps via ``pip install tox`` or your
+package manager), running ``tox`` in the directory of your source
+checkout will run ``jsonschema``'s test suite on all of the versions
+of Python ``jsonschema`` supports. If you don't have all of the
+versions that ``jsonschema`` is tested under, you'll likely want to run
+using ``tox``'s ``--skip-missing-interpreters`` option.
+
+Of course you're also free to just run the tests on a single version with your
+favorite test runner. The tests live in the ``jsonschema.tests`` package.
+
+
+Benchmarks
+----------
+
+``jsonschema``'s benchmarks make use of `pyperf
+<https://pyperf.readthedocs.io>`_.
+
+Running them can be done via ``tox -e perf``, or by invoking the ``pyperf``
+commands externally (after ensuring that both it and ``jsonschema`` itself are
+installed)::
+
+    $ python -m pyperf jsonschema/benchmarks/test_suite.py --hist --output results.json
+
+To compare to a previous run, use::
+
+    $ python -m pyperf compare_to --table reference.json results.json
+
+See the ``pyperf`` documentation for more details.
+
+
+Community
+---------
+
+There's a `mailing list <https://groups.google.com/forum/#!forum/jsonschema>`_
+for this implementation on Google Groups.
+
+Please join, and feel free to send questions there.
+
+
+Contributing
+------------
+
+I'm Julian Berman.
+
+``jsonschema`` is on `GitHub <https://github.com/Julian/jsonschema>`_.
+
+Get in touch, via GitHub or otherwise, if you've got something to contribute,
+it'd be most welcome!
+
+You can also generally find me on Freenode (nick: ``tos9``) in various
+channels, including ``#python``.
+
+If you feel overwhelmingly grateful, you can also woo me with beer money
+via Google Pay with the email in my GitHub profile.
+
+And for companies who appreciate ``jsonschema`` and its continued support
+and growth, ``jsonschema`` is also now supportable via `TideLift
+<https://tidelift.com/subscription/pkg/pypi-jsonschema?utm_source=pypi-j
+sonschema&utm_medium=referral&utm_campaign=readme>`_.
+
+
diff --git a/contrib/python/jsonschema/py2/.dist-info/entry_points.txt b/contrib/python/jsonschema/py2/.dist-info/entry_points.txt
new file mode 100644
index 00000000000..c627b310cd0
--- /dev/null
+++ b/contrib/python/jsonschema/py2/.dist-info/entry_points.txt
@@ -0,0 +1,3 @@
+[console_scripts]
+jsonschema = jsonschema.cli:main
+
diff --git a/contrib/python/jsonschema/py2/.dist-info/top_level.txt b/contrib/python/jsonschema/py2/.dist-info/top_level.txt
new file mode 100644
index 00000000000..d89304b1a89
--- /dev/null
+++ b/contrib/python/jsonschema/py2/.dist-info/top_level.txt
@@ -0,0 +1 @@
+jsonschema
diff --git a/contrib/python/jsonschema/py2/COPYING b/contrib/python/jsonschema/py2/COPYING
new file mode 100644
index 00000000000..af9cfbdb134
--- /dev/null
+++ b/contrib/python/jsonschema/py2/COPYING
@@ -0,0 +1,19 @@
+Copyright (c) 2013 Julian Berman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/contrib/python/jsonschema/py2/README.rst b/contrib/python/jsonschema/py2/README.rst
new file mode 100644
index 00000000000..ccfb55d02d4
--- /dev/null
+++ b/contrib/python/jsonschema/py2/README.rst
@@ -0,0 +1,179 @@
+==========
+jsonschema
+==========
+
+|PyPI| |Pythons| |Travis| |AppVeyor| |Codecov| |ReadTheDocs|
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/jsonschema.svg
+   :alt: PyPI version
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Pythons| image:: https://img.shields.io/pypi/pyversions/jsonschema.svg
+   :alt: Supported Python versions
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Travis| image:: https://travis-ci.com/Julian/jsonschema.svg?branch=master
+   :alt: Travis build status
+   :target: https://travis-ci.com/Julian/jsonschema
+
+.. |AppVeyor| image:: https://ci.appveyor.com/api/projects/status/adtt0aiaihy6muyn/branch/master?svg=true
+   :alt: AppVeyor build status
+   :target: https://ci.appveyor.com/project/Julian/jsonschema
+
+.. |Codecov| image:: https://codecov.io/gh/Julian/jsonschema/branch/master/graph/badge.svg
+   :alt: Codecov Code coverage
+   :target: https://codecov.io/gh/Julian/jsonschema
+
+.. |ReadTheDocs| image:: https://readthedocs.org/projects/python-jsonschema/badge/?version=stable&style=flat
+   :alt: ReadTheDocs status
+   :target: https://python-jsonschema.readthedocs.io/en/stable/
+
+
+``jsonschema`` is an implementation of `JSON Schema <https://json-schema.org>`_
+for Python (supporting 2.7+ including Python 3).
+
+.. code-block:: python
+
+    >>> from jsonschema import validate
+
+    >>> # A sample schema, like what we'd get from json.load()
+    >>> schema = {
+    ...     "type" : "object",
+    ...     "properties" : {
+    ...         "price" : {"type" : "number"},
+    ...         "name" : {"type" : "string"},
+    ...     },
+    ... }
+
+    >>> # If no exception is raised by validate(), the instance is valid.
+    >>> validate(instance={"name" : "Eggs", "price" : 34.99}, schema=schema)
+
+    >>> validate(
+    ...     instance={"name" : "Eggs", "price" : "Invalid"}, schema=schema,
+    ... )                                   # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValidationError: 'Invalid' is not of type 'number'
+
+It can also be used from console:
+
+.. code-block:: bash
+
+    $ jsonschema -i sample.json sample.schema
+
+Features
+--------
+
+* Full support for
+  `Draft 7 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft7Validator>`_,
+  `Draft 6 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft6Validator>`_,
+  `Draft 4 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft4Validator>`_
+  and
+  `Draft 3 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft3Validator>`_
+
+* `Lazy validation <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.IValidator.iter_errors>`_
+  that can iteratively report *all* validation errors.
+
+* `Programmatic querying <https://python-jsonschema.readthedocs.io/en/latest/errors/>`_
+  of which properties or items failed validation.
+
+
+Installation
+------------
+
+``jsonschema`` is available on `PyPI <https://pypi.org/project/jsonschema/>`_. You can install using `pip <https://pip.pypa.io/en/stable/>`_:
+
+.. code-block:: bash
+
+    $ pip install jsonschema
+
+
+Demo
+----
+
+Try ``jsonschema`` interactively in this online demo:
+
+.. image:: https://user-images.githubusercontent.com/1155573/56745335-8b158a00-6750-11e9-8776-83fa675939c4.png
+    :target: https://notebooks.ai/demo/gh/Julian/jsonschema
+    :alt: Open Live Demo
+
+
+Online demo Notebook will look similar to this:
+
+
+.. image:: https://user-images.githubusercontent.com/1155573/56820861-5c1c1880-6823-11e9-802a-ce01c5ec574f.gif
+    :alt: Open Live Demo
+    :width: 480 px
+
+
+Release Notes
+-------------
+
+v3.1 brings support for ECMA 262 dialect regular expressions
+throughout schemas, as recommended by the specification. Big
+thanks to @Zac-HD for authoring support in a new `js-regex
+<https://pypi.org/project/js-regex/>`_ library.
+
+
+Running the Test Suite
+----------------------
+
+If you have ``tox`` installed (perhaps via ``pip install tox`` or your
+package manager), running ``tox`` in the directory of your source
+checkout will run ``jsonschema``'s test suite on all of the versions
+of Python ``jsonschema`` supports. If you don't have all of the
+versions that ``jsonschema`` is tested under, you'll likely want to run
+using ``tox``'s ``--skip-missing-interpreters`` option.
+
+Of course you're also free to just run the tests on a single version with your
+favorite test runner. The tests live in the ``jsonschema.tests`` package.
+
+
+Benchmarks
+----------
+
+``jsonschema``'s benchmarks make use of `pyperf
+<https://pyperf.readthedocs.io>`_.
+
+Running them can be done via ``tox -e perf``, or by invoking the ``pyperf``
+commands externally (after ensuring that both it and ``jsonschema`` itself are
+installed)::
+
+    $ python -m pyperf jsonschema/benchmarks/test_suite.py --hist --output results.json
+
+To compare to a previous run, use::
+
+    $ python -m pyperf compare_to --table reference.json results.json
+
+See the ``pyperf`` documentation for more details.
+
+
+Community
+---------
+
+There's a `mailing list <https://groups.google.com/forum/#!forum/jsonschema>`_
+for this implementation on Google Groups.
+
+Please join, and feel free to send questions there.
+
+
+Contributing
+------------
+
+I'm Julian Berman.
+
+``jsonschema`` is on `GitHub <https://github.com/Julian/jsonschema>`_.
+
+Get in touch, via GitHub or otherwise, if you've got something to contribute,
+it'd be most welcome!
+
+You can also generally find me on Freenode (nick: ``tos9``) in various
+channels, including ``#python``.
+
+If you feel overwhelmingly grateful, you can also woo me with beer money
+via Google Pay with the email in my GitHub profile.
+
+And for companies who appreciate ``jsonschema`` and its continued support
+and growth, ``jsonschema`` is also now supportable via `TideLift
+<https://tidelift.com/subscription/pkg/pypi-jsonschema?utm_source=pypi-j
+sonschema&utm_medium=referral&utm_campaign=readme>`_.
diff --git a/contrib/python/jsonschema/py2/jsonschema/__init__.py b/contrib/python/jsonschema/py2/jsonschema/__init__.py
new file mode 100644
index 00000000000..6b630cdfbbe
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/__init__.py
@@ -0,0 +1,34 @@
+"""
+An implementation of JSON Schema for Python
+
+The main functionality is provided by the validator classes for each of the
+supported JSON Schema versions.
+
+Most commonly, `validate` is the quickest way to simply validate a given
+instance under a schema, and will create a validator for you.
+"""
+
+from jsonschema.exceptions import (
+    ErrorTree, FormatError, RefResolutionError, SchemaError, ValidationError
+)
+from jsonschema._format import (
+    FormatChecker,
+    draft3_format_checker,
+    draft4_format_checker,
+    draft6_format_checker,
+    draft7_format_checker,
+)
+from jsonschema._types import TypeChecker
+from jsonschema.validators import (
+    Draft3Validator,
+    Draft4Validator,
+    Draft6Validator,
+    Draft7Validator,
+    RefResolver,
+    validate,
+)
+try:
+    from importlib import metadata
+except ImportError: # for Python<3.8
+    import importlib_metadata as metadata
+__version__ = metadata.version("jsonschema")
diff --git a/contrib/python/jsonschema/py2/jsonschema/__main__.py b/contrib/python/jsonschema/py2/jsonschema/__main__.py
new file mode 100644
index 00000000000..82c29fd39e7
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/__main__.py
@@ -0,0 +1,2 @@
+from jsonschema.cli import main
+main()
diff --git a/contrib/python/jsonschema/py2/jsonschema/_format.py b/contrib/python/jsonschema/py2/jsonschema/_format.py
new file mode 100644
index 00000000000..281a7cfcffe
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/_format.py
@@ -0,0 +1,425 @@
+import datetime
+import re
+import socket
+import struct
+
+from jsonschema.compat import str_types
+from jsonschema.exceptions import FormatError
+
+
+class FormatChecker(object):
+    """
+    A ``format`` property checker.
+
+    JSON Schema does not mandate that the ``format`` property actually do any
+    validation. If validation is desired however, instances of this class can
+    be hooked into validators to enable format validation.
+
+    `FormatChecker` objects always return ``True`` when asked about
+    formats that they do not know how to validate.
+
+    To check a custom format using a function that takes an instance and
+    returns a ``bool``, use the `FormatChecker.checks` or
+    `FormatChecker.cls_checks` decorators.
+
+    Arguments:
+
+        formats (~collections.Iterable):
+
+            The known formats to validate. This argument can be used to
+            limit which formats will be used during validation.
+    """
+
+    checkers = {}
+
+    def __init__(self, formats=None):
+        if formats is None:
+            self.checkers = self.checkers.copy()
+        else:
+            self.checkers = dict((k, self.checkers[k]) for k in formats)
+
+    def __repr__(self):
+        return "<FormatChecker checkers={}>".format(sorted(self.checkers))
+
+    def checks(self, format, raises=()):
+        """
+        Register a decorated function as validating a new format.
+
+        Arguments:
+
+            format (str):
+
+                The format that the decorated function will check.
+
+            raises (Exception):
+
+                The exception(s) raised by the decorated function when an
+                invalid instance is found.
+
+                The exception object will be accessible as the
+                `jsonschema.exceptions.ValidationError.cause` attribute of the
+                resulting validation error.
+        """
+
+        def _checks(func):
+            self.checkers[format] = (func, raises)
+            return func
+        return _checks
+
+    cls_checks = classmethod(checks)
+
+    def check(self, instance, format):
+        """
+        Check whether the instance conforms to the given format.
+
+        Arguments:
+
+            instance (*any primitive type*, i.e. str, number, bool):
+
+                The instance to check
+
+            format (str):
+
+                The format that instance should conform to
+
+
+        Raises:
+
+            FormatError: if the instance does not conform to ``format``
+        """
+
+        if format not in self.checkers:
+            return
+
+        func, raises = self.checkers[format]
+        result, cause = None, None
+        try:
+            result = func(instance)
+        except raises as e:
+            cause = e
+        if not result:
+            raise FormatError(
+                "%r is not a %r" % (instance, format), cause=cause,
+            )
+
+    def conforms(self, instance, format):
+        """
+        Check whether the instance conforms to the given format.
+
+        Arguments:
+
+            instance (*any primitive type*, i.e. str, number, bool):
+
+                The instance to check
+
+            format (str):
+
+                The format that instance should conform to
+
+        Returns:
+
+            bool: whether it conformed
+        """
+
+        try:
+            self.check(instance, format)
+        except FormatError:
+            return False
+        else:
+            return True
+
+
+draft3_format_checker = FormatChecker()
+draft4_format_checker = FormatChecker()
+draft6_format_checker = FormatChecker()
+draft7_format_checker = FormatChecker()
+
+
+_draft_checkers = dict(
+    draft3=draft3_format_checker,
+    draft4=draft4_format_checker,
+    draft6=draft6_format_checker,
+    draft7=draft7_format_checker,
+)
+
+
+def _checks_drafts(
+    name=None,
+    draft3=None,
+    draft4=None,
+    draft6=None,
+    draft7=None,
+    raises=(),
+):
+    draft3 = draft3 or name
+    draft4 = draft4 or name
+    draft6 = draft6 or name
+    draft7 = draft7 or name
+
+    def wrap(func):
+        if draft3:
+            func = _draft_checkers["draft3"].checks(draft3, raises)(func)
+        if draft4:
+            func = _draft_checkers["draft4"].checks(draft4, raises)(func)
+        if draft6:
+            func = _draft_checkers["draft6"].checks(draft6, raises)(func)
+        if draft7:
+            func = _draft_checkers["draft7"].checks(draft7, raises)(func)
+
+        # Oy. This is bad global state, but relied upon for now, until
+        # deprecation. See https://github.com/Julian/jsonschema/issues/519
+        # and test_format_checkers_come_with_defaults
+        FormatChecker.cls_checks(draft7 or draft6 or draft4 or draft3, raises)(
+            func,
+        )
+        return func
+    return wrap
+
+
+@_checks_drafts(name="idn-email")
+@_checks_drafts(name="email")
+def is_email(instance):
+    if not isinstance(instance, str_types):
+        return True
+    return "@" in instance
+
+
+_ipv4_re = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
+
+
+@_checks_drafts(
+    draft3="ip-address", draft4="ipv4", draft6="ipv4", draft7="ipv4",
+)
+def is_ipv4(instance):
+    if not isinstance(instance, str_types):
+        return True
+    if not _ipv4_re.match(instance):
+        return False
+    return all(0 <= int(component) <= 255 for component in instance.split("."))
+
+
+if hasattr(socket, "inet_pton"):
+    # FIXME: Really this only should raise struct.error, but see the sadness
+    #        that is https://twistedmatrix.com/trac/ticket/9409
+    @_checks_drafts(
+        name="ipv6", raises=(socket.error, struct.error, ValueError),
+    )
+    def is_ipv6(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return socket.inet_pton(socket.AF_INET6, instance)
+
+
+_host_name_re = re.compile(r"^[A-Za-z0-9][A-Za-z0-9\.\-]{1,255}$")
+
+
+@_checks_drafts(
+    draft3="host-name",
+    draft4="hostname",
+    draft6="hostname",
+    draft7="hostname",
+)
+def is_host_name(instance):
+    if not isinstance(instance, str_types):
+        return True
+    if not _host_name_re.match(instance):
+        return False
+    components = instance.split(".")
+    for component in components:
+        if len(component) > 63:
+            return False
+    return True
+
+
+try:
+    # The built-in `idna` codec only implements RFC 3890, so we go elsewhere.
+    import idna
+except ImportError:
+    pass
+else:
+    @_checks_drafts(draft7="idn-hostname", raises=idna.IDNAError)
+    def is_idn_host_name(instance):
+        if not isinstance(instance, str_types):
+            return True
+        idna.encode(instance)
+        return True
+
+
+try:
+    import rfc3987
+except ImportError:
+    try:
+        from rfc3986_validator import validate_rfc3986
+    except ImportError:
+        pass
+    else:
+        @_checks_drafts(name="uri")
+        def is_uri(instance):
+            if not isinstance(instance, str_types):
+                return True
+            return validate_rfc3986(instance, rule="URI")
+
+        @_checks_drafts(
+            draft6="uri-reference",
+            draft7="uri-reference",
+            raises=ValueError,
+        )
+        def is_uri_reference(instance):
+            if not isinstance(instance, str_types):
+                return True
+            return validate_rfc3986(instance, rule="URI_reference")
+
+else:
+    @_checks_drafts(draft7="iri", raises=ValueError)
+    def is_iri(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return rfc3987.parse(instance, rule="IRI")
+
+    @_checks_drafts(draft7="iri-reference", raises=ValueError)
+    def is_iri_reference(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return rfc3987.parse(instance, rule="IRI_reference")
+
+    @_checks_drafts(name="uri", raises=ValueError)
+    def is_uri(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return rfc3987.parse(instance, rule="URI")
+
+    @_checks_drafts(
+        draft6="uri-reference",
+        draft7="uri-reference",
+        raises=ValueError,
+    )
+    def is_uri_reference(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return rfc3987.parse(instance, rule="URI_reference")
+
+
+try:
+    from strict_rfc3339 import validate_rfc3339
+except ImportError:
+    try:
+        from rfc3339_validator import validate_rfc3339
+    except ImportError:
+        validate_rfc3339 = None
+
+if validate_rfc3339:
+    @_checks_drafts(name="date-time")
+    def is_datetime(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return validate_rfc3339(instance)
+
+    @_checks_drafts(draft7="time")
+    def is_time(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return is_datetime("1970-01-01T" + instance)
+
+
+@_checks_drafts(name="regex", raises=re.error)
+def is_regex(instance):
+    if not isinstance(instance, str_types):
+        return True
+    return re.compile(instance)
+
+
+@_checks_drafts(draft3="date", draft7="date", raises=ValueError)
+def is_date(instance):
+    if not isinstance(instance, str_types):
+        return True
+    return datetime.datetime.strptime(instance, "%Y-%m-%d")
+
+
+@_checks_drafts(draft3="time", raises=ValueError)
+def is_draft3_time(instance):
+    if not isinstance(instance, str_types):
+        return True
+    return datetime.datetime.strptime(instance, "%H:%M:%S")
+
+
+try:
+    import webcolors
+except ImportError:
+    pass
+else:
+    def is_css_color_code(instance):
+        return webcolors.normalize_hex(instance)
+
+    @_checks_drafts(draft3="color", raises=(ValueError, TypeError))
+    def is_css21_color(instance):
+        if (
+            not isinstance(instance, str_types) or
+            instance.lower() in webcolors.css21_names_to_hex
+        ):
+            return True
+        return is_css_color_code(instance)
+
+    def is_css3_color(instance):
+        if instance.lower() in webcolors.css3_names_to_hex:
+            return True
+        return is_css_color_code(instance)
+
+
+try:
+    import jsonpointer
+except ImportError:
+    pass
+else:
+    @_checks_drafts(
+        draft6="json-pointer",
+        draft7="json-pointer",
+        raises=jsonpointer.JsonPointerException,
+    )
+    def is_json_pointer(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return jsonpointer.JsonPointer(instance)
+
+    # TODO: I don't want to maintain this, so it
+    #       needs to go either into jsonpointer (pending
+    #       https://github.com/stefankoegl/python-json-pointer/issues/34) or
+    #       into a new external library.
+    @_checks_drafts(
+        draft7="relative-json-pointer",
+        raises=jsonpointer.JsonPointerException,
+    )
+    def is_relative_json_pointer(instance):
+        # Definition taken from:
+        # https://tools.ietf.org/html/draft-handrews-relative-json-pointer-01#section-3
+        if not isinstance(instance, str_types):
+            return True
+        non_negative_integer, rest = [], ""
+        for i, character in enumerate(instance):
+            if character.isdigit():
+                non_negative_integer.append(character)
+                continue
+
+            if not non_negative_integer:
+                return False
+
+            rest = instance[i:]
+            break
+        return (rest == "#") or jsonpointer.JsonPointer(rest)
+
+
+try:
+    import uritemplate.exceptions
+except ImportError:
+    pass
+else:
+    @_checks_drafts(
+        draft6="uri-template",
+        draft7="uri-template",
+        raises=uritemplate.exceptions.InvalidTemplate,
+    )
+    def is_uri_template(
+        instance,
+        template_validator=uritemplate.Validator().force_balanced_braces(),
+    ):
+        template = uritemplate.URITemplate(instance)
+        return template_validator.validate(template)
diff --git a/contrib/python/jsonschema/py2/jsonschema/_legacy_validators.py b/contrib/python/jsonschema/py2/jsonschema/_legacy_validators.py
new file mode 100644
index 00000000000..264ff7d7135
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/_legacy_validators.py
@@ -0,0 +1,141 @@
+from jsonschema import _utils
+from jsonschema.compat import iteritems
+from jsonschema.exceptions import ValidationError
+
+
+def dependencies_draft3(validator, dependencies, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property, dependency in iteritems(dependencies):
+        if property not in instance:
+            continue
+
+        if validator.is_type(dependency, "object"):
+            for error in validator.descend(
+                instance, dependency, schema_path=property,
+            ):
+                yield error
+        elif validator.is_type(dependency, "string"):
+            if dependency not in instance:
+                yield ValidationError(
+                    "%r is a dependency of %r" % (dependency, property)
+                )
+        else:
+            for each in dependency:
+                if each not in instance:
+                    message = "%r is a dependency of %r"
+                    yield ValidationError(message % (each, property))
+
+
+def disallow_draft3(validator, disallow, instance, schema):
+    for disallowed in _utils.ensure_list(disallow):
+        if validator.is_valid(instance, {"type": [disallowed]}):
+            yield ValidationError(
+                "%r is disallowed for %r" % (disallowed, instance)
+            )
+
+
+def extends_draft3(validator, extends, instance, schema):
+    if validator.is_type(extends, "object"):
+        for error in validator.descend(instance, extends):
+            yield error
+        return
+    for index, subschema in enumerate(extends):
+        for error in validator.descend(instance, subschema, schema_path=index):
+            yield error
+
+
+def items_draft3_draft4(validator, items, instance, schema):
+    if not validator.is_type(instance, "array"):
+        return
+
+    if validator.is_type(items, "object"):
+        for index, item in enumerate(instance):
+            for error in validator.descend(item, items, path=index):
+                yield error
+    else:
+        for (index, item), subschema in zip(enumerate(instance), items):
+            for error in validator.descend(
+                item, subschema, path=index, schema_path=index,
+            ):
+                yield error
+
+
+def minimum_draft3_draft4(validator, minimum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if schema.get("exclusiveMinimum", False):
+        failed = instance <= minimum
+        cmp = "less than or equal to"
+    else:
+        failed = instance < minimum
+        cmp = "less than"
+
+    if failed:
+        yield ValidationError(
+            "%r is %s the minimum of %r" % (instance, cmp, minimum)
+        )
+
+
+def maximum_draft3_draft4(validator, maximum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if schema.get("exclusiveMaximum", False):
+        failed = instance >= maximum
+        cmp = "greater than or equal to"
+    else:
+        failed = instance > maximum
+        cmp = "greater than"
+
+    if failed:
+        yield ValidationError(
+            "%r is %s the maximum of %r" % (instance, cmp, maximum)
+        )
+
+
+def properties_draft3(validator, properties, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property, subschema in iteritems(properties):
+        if property in instance:
+            for error in validator.descend(
+                instance[property],
+                subschema,
+                path=property,
+                schema_path=property,
+            ):
+                yield error
+        elif subschema.get("required", False):
+            error = ValidationError("%r is a required property" % property)
+            error._set(
+                validator="required",
+                validator_value=subschema["required"],
+                instance=instance,
+                schema=schema,
+            )
+            error.path.appendleft(property)
+            error.schema_path.extend([property, "required"])
+            yield error
+
+
+def type_draft3(validator, types, instance, schema):
+    types = _utils.ensure_list(types)
+
+    all_errors = []
+    for index, type in enumerate(types):
+        if validator.is_type(type, "object"):
+            errors = list(validator.descend(instance, type, schema_path=index))
+            if not errors:
+                return
+            all_errors.extend(errors)
+        else:
+            if validator.is_type(instance, type):
+                return
+    else:
+        yield ValidationError(
+            _utils.types_msg(instance, types), context=all_errors,
+        )
diff --git a/contrib/python/jsonschema/py2/jsonschema/_reflect.py b/contrib/python/jsonschema/py2/jsonschema/_reflect.py
new file mode 100644
index 00000000000..d09e38fbdcf
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/_reflect.py
@@ -0,0 +1,155 @@
+# -*- test-case-name: twisted.test.test_reflect -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Standardized versions of various cool and/or strange things that you can do
+with Python's reflection capabilities.
+"""
+
+import sys
+
+from jsonschema.compat import PY3
+
+
+class _NoModuleFound(Exception):
+    """
+    No module was found because none exists.
+    """
+
+
+
+class InvalidName(ValueError):
+    """
+    The given name is not a dot-separated list of Python objects.
+    """
+
+
+
+class ModuleNotFound(InvalidName):
+    """
+    The module associated with the given name doesn't exist and it can't be
+    imported.
+    """
+
+
+
+class ObjectNotFound(InvalidName):
+    """
+    The object associated with the given name doesn't exist and it can't be
+    imported.
+    """
+
+
+
+if PY3:
+    def reraise(exception, traceback):
+        raise exception.with_traceback(traceback)
+else:
+    exec("""def reraise(exception, traceback):
+        raise exception.__class__, exception, traceback""")
+
+reraise.__doc__ = """
+Re-raise an exception, with an optional traceback, in a way that is compatible
+with both Python 2 and Python 3.
+
+Note that on Python 3, re-raised exceptions will be mutated, with their
+C{__traceback__} attribute being set.
+
+@param exception: The exception instance.
+@param traceback: The traceback to use, or C{None} indicating a new traceback.
+"""
+
+
+def _importAndCheckStack(importName):
+    """
+    Import the given name as a module, then walk the stack to determine whether
+    the failure was the module not existing, or some code in the module (for
+    example a dependent import) failing.  This can be helpful to determine
+    whether any actual application code was run.  For example, to distiguish
+    administrative error (entering the wrong module name), from programmer
+    error (writing buggy code in a module that fails to import).
+
+    @param importName: The name of the module to import.
+    @type importName: C{str}
+    @raise Exception: if something bad happens.  This can be any type of
+        exception, since nobody knows what loading some arbitrary code might
+        do.
+    @raise _NoModuleFound: if no module was found.
+    """
+    try:
+        return __import__(importName)
+    except ImportError:
+        excType, excValue, excTraceback = sys.exc_info()
+        while excTraceback:
+            execName = excTraceback.tb_frame.f_globals["__name__"]
+            # in Python 2 execName is None when an ImportError is encountered,
+            # where in Python 3 execName is equal to the importName.
+            if execName is None or execName == importName:
+                reraise(excValue, excTraceback)
+            excTraceback = excTraceback.tb_next
+        raise _NoModuleFound()
+
+
+
+def namedAny(name):
+    """
+    Retrieve a Python object by its fully qualified name from the global Python
+    module namespace.  The first part of the name, that describes a module,
+    will be discovered and imported.  Each subsequent part of the name is
+    treated as the name of an attribute of the object specified by all of the
+    name which came before it.  For example, the fully-qualified name of this
+    object is 'twisted.python.reflect.namedAny'.
+
+    @type name: L{str}
+    @param name: The name of the object to return.
+
+    @raise InvalidName: If the name is an empty string, starts or ends with
+        a '.', or is otherwise syntactically incorrect.
+
+    @raise ModuleNotFound: If the name is syntactically correct but the
+        module it specifies cannot be imported because it does not appear to
+        exist.
+
+    @raise ObjectNotFound: If the name is syntactically correct, includes at
+        least one '.', but the module it specifies cannot be imported because
+        it does not appear to exist.
+
+    @raise AttributeError: If an attribute of an object along the way cannot be
+        accessed, or a module along the way is not found.
+
+    @return: the Python object identified by 'name'.
+    """
+    if not name:
+        raise InvalidName('Empty module name')
+
+    names = name.split('.')
+
+    # if the name starts or ends with a '.' or contains '..', the __import__
+    # will raise an 'Empty module name' error. This will provide a better error
+    # message.
+    if '' in names:
+        raise InvalidName(
+            "name must be a string giving a '.'-separated list of Python "
+            "identifiers, not %r" % (name,))
+
+    topLevelPackage = None
+    moduleNames = names[:]
+    while not topLevelPackage:
+        if moduleNames:
+            trialname = '.'.join(moduleNames)
+            try:
+                topLevelPackage = _importAndCheckStack(trialname)
+            except _NoModuleFound:
+                moduleNames.pop()
+        else:
+            if len(names) == 1:
+                raise ModuleNotFound("No module named %r" % (name,))
+            else:
+                raise ObjectNotFound('%r does not name an object' % (name,))
+
+    obj = topLevelPackage
+    for n in names[1:]:
+        obj = getattr(obj, n)
+
+    return obj
diff --git a/contrib/python/jsonschema/py2/jsonschema/_types.py b/contrib/python/jsonschema/py2/jsonschema/_types.py
new file mode 100644
index 00000000000..a71a4e34bdc
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/_types.py
@@ -0,0 +1,188 @@
+import numbers
+
+from pyrsistent import pmap
+import attr
+
+from jsonschema.compat import int_types, str_types
+from jsonschema.exceptions import UndefinedTypeCheck
+
+
+def is_array(checker, instance):
+    return isinstance(instance, list)
+
+
+def is_bool(checker, instance):
+    return isinstance(instance, bool)
+
+
+def is_integer(checker, instance):
+    # bool inherits from int, so ensure bools aren't reported as ints
+    if isinstance(instance, bool):
+        return False
+    return isinstance(instance, int_types)
+
+
+def is_null(checker, instance):
+    return instance is None
+
+
+def is_number(checker, instance):
+    # bool inherits from int, so ensure bools aren't reported as ints
+    if isinstance(instance, bool):
+        return False
+    return isinstance(instance, numbers.Number)
+
+
+def is_object(checker, instance):
+    return isinstance(instance, dict)
+
+
+def is_string(checker, instance):
+    return isinstance(instance, str_types)
+
+
+def is_any(checker, instance):
+    return True
+
+
+@attr.s(frozen=True)
+class TypeChecker(object):
+    """
+    A ``type`` property checker.
+
+    A `TypeChecker` performs type checking for an `IValidator`. Type
+    checks to perform are updated using `TypeChecker.redefine` or
+    `TypeChecker.redefine_many` and removed via `TypeChecker.remove`.
+    Each of these return a new `TypeChecker` object.
+
+    Arguments:
+
+        type_checkers (dict):
+
+            The initial mapping of types to their checking functions.
+    """
+    _type_checkers = attr.ib(default=pmap(), converter=pmap)
+
+    def is_type(self, instance, type):
+        """
+        Check if the instance is of the appropriate type.
+
+        Arguments:
+
+            instance (object):
+
+                The instance to check
+
+            type (str):
+
+                The name of the type that is expected.
+
+        Returns:
+
+            bool: Whether it conformed.
+
+
+        Raises:
+
+            `jsonschema.exceptions.UndefinedTypeCheck`:
+                if type is unknown to this object.
+        """
+        try:
+            fn = self._type_checkers[type]
+        except KeyError:
+            raise UndefinedTypeCheck(type)
+
+        return fn(self, instance)
+
+    def redefine(self, type, fn):
+        """
+        Produce a new checker with the given type redefined.
+
+        Arguments:
+
+            type (str):
+
+                The name of the type to check.
+
+            fn (collections.Callable):
+
+                A function taking exactly two parameters - the type
+                checker calling the function and the instance to check.
+                The function should return true if instance is of this
+                type and false otherwise.
+
+        Returns:
+
+            A new `TypeChecker` instance.
+        """
+        return self.redefine_many({type: fn})
+
+    def redefine_many(self, definitions=()):
+        """
+        Produce a new checker with the given types redefined.
+
+        Arguments:
+
+            definitions (dict):
+
+                A dictionary mapping types to their checking functions.
+
+        Returns:
+
+            A new `TypeChecker` instance.
+        """
+        return attr.evolve(
+            self, type_checkers=self._type_checkers.update(definitions),
+        )
+
+    def remove(self, *types):
+        """
+        Produce a new checker with the given types forgotten.
+
+        Arguments:
+
+            types (~collections.Iterable):
+
+                the names of the types to remove.
+
+        Returns:
+
+            A new `TypeChecker` instance
+
+        Raises:
+
+            `jsonschema.exceptions.UndefinedTypeCheck`:
+
+                if any given type is unknown to this object
+        """
+
+        checkers = self._type_checkers
+        for each in types:
+            try:
+                checkers = checkers.remove(each)
+            except KeyError:
+                raise UndefinedTypeCheck(each)
+        return attr.evolve(self, type_checkers=checkers)
+
+
+draft3_type_checker = TypeChecker(
+    {
+        u"any": is_any,
+        u"array": is_array,
+        u"boolean": is_bool,
+        u"integer": is_integer,
+        u"object": is_object,
+        u"null": is_null,
+        u"number": is_number,
+        u"string": is_string,
+    },
+)
+draft4_type_checker = draft3_type_checker.remove(u"any")
+draft6_type_checker = draft4_type_checker.redefine(
+    u"integer",
+    lambda checker, instance: (
+        is_integer(checker, instance) or
+        isinstance(instance, float) and instance.is_integer()
+    ),
+)
+draft7_type_checker = draft6_type_checker
diff --git a/contrib/python/jsonschema/py2/jsonschema/_utils.py b/contrib/python/jsonschema/py2/jsonschema/_utils.py
new file mode 100644
index 00000000000..ceb880198d1
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/_utils.py
@@ -0,0 +1,212 @@
+import itertools
+import json
+import pkgutil
+import re
+
+from jsonschema.compat import MutableMapping, str_types, urlsplit
+
+
+class URIDict(MutableMapping):
+    """
+    Dictionary which uses normalized URIs as keys.
+    """
+
+    def normalize(self, uri):
+        return urlsplit(uri).geturl()
+
+    def __init__(self, *args, **kwargs):
+        self.store = dict()
+        self.store.update(*args, **kwargs)
+
+    def __getitem__(self, uri):
+        return self.store[self.normalize(uri)]
+
+    def __setitem__(self, uri, value):
+        self.store[self.normalize(uri)] = value
+
+    def __delitem__(self, uri):
+        del self.store[self.normalize(uri)]
+
+    def __iter__(self):
+        return iter(self.store)
+
+    def __len__(self):
+        return len(self.store)
+
+    def __repr__(self):
+        return repr(self.store)
+
+
+class Unset(object):
+    """
+    An as-of-yet unset attribute or unprovided default parameter.
+    """
+
+    def __repr__(self):
+        return "<unset>"
+
+
+def load_schema(name):
+    """
+    Load a schema from ./schemas/``name``.json and return it.
+    """
+
+    data = pkgutil.get_data("jsonschema", "schemas/{0}.json".format(name))
+    return json.loads(data.decode("utf-8"))
+
+
+def indent(string, times=1):
+    """
+    A dumb version of `textwrap.indent` from Python 3.3.
+    """
+
+    return "\n".join(" " * (4 * times) + line for line in string.splitlines())
+
+
+def format_as_index(indices):
+    """
+    Construct a single string containing indexing operations for the indices.
+
+    For example, [1, 2, "foo"] -> [1][2]["foo"]
+
+    Arguments:
+
+        indices (sequence):
+
+            The indices to format.
+    """
+
+    if not indices:
+        return ""
+    return "[%s]" % "][".join(repr(index) for index in indices)
+
+
+def find_additional_properties(instance, schema):
+    """
+    Return the set of additional properties for the given ``instance``.
+
+    Weeds out properties that should have been validated by ``properties`` and
+    / or ``patternProperties``.
+
+    Assumes ``instance`` is dict-like already.
+    """
+
+    properties = schema.get("properties", {})
+    patterns = "|".join(schema.get("patternProperties", {}))
+    for property in instance:
+        if property not in properties:
+            if patterns and re.search(patterns, property):
+                continue
+            yield property
+
+
+def extras_msg(extras):
+    """
+    Create an error message for extra items or properties.
+    """
+
+    if len(extras) == 1:
+        verb = "was"
+    else:
+        verb = "were"
+    return ", ".join(repr(extra) for extra in extras), verb
+
+
+def types_msg(instance, types):
+    """
+    Create an error message for a failure to match the given types.
+
+    If the ``instance`` is an object and contains a ``name`` property, it will
+    be considered to be a description of that object and used as its type.
+
+    Otherwise the message is simply the reprs of the given ``types``.
+    """
+
+    reprs = []
+    for type in types:
+        try:
+            reprs.append(repr(type["name"]))
+        except Exception:
+            reprs.append(repr(type))
+    return "%r is not of type %s" % (instance, ", ".join(reprs))
+
+
+def flatten(suitable_for_isinstance):
+    """
+    isinstance() can accept a bunch of really annoying different types:
+        * a single type
+        * a tuple of types
+        * an arbitrary nested tree of tuples
+
+    Return a flattened tuple of the given argument.
+    """
+
+    types = set()
+
+    if not isinstance(suitable_for_isinstance, tuple):
+        suitable_for_isinstance = (suitable_for_isinstance,)
+    for thing in suitable_for_isinstance:
+        if isinstance(thing, tuple):
+            types.update(flatten(thing))
+        else:
+            types.add(thing)
+    return tuple(types)
+
+
+def ensure_list(thing):
+    """
+    Wrap ``thing`` in a list if it's a single str.
+
+    Otherwise, return it unchanged.
+    """
+
+    if isinstance(thing, str_types):
+        return [thing]
+    return thing
+
+
+def equal(one, two):
+    """
+    Check if two things are equal, but evade booleans and ints being equal.
+    """
+    return unbool(one) == unbool(two)
+
+
+def unbool(element, true=object(), false=object()):
+    """
+    A hack to make True and 1 and False and 0 unique for ``uniq``.
+    """
+
+    if element is True:
+        return true
+    elif element is False:
+        return false
+    return element
+
+
+def uniq(container):
+    """
+    Check if all of a container's elements are unique.
+
+    Successively tries first to rely that the elements are hashable, then
+    falls back on them being sortable, and finally falls back on brute
+    force.
+    """
+
+    try:
+        return len(set(unbool(i) for i in container)) == len(container)
+    except TypeError:
+        try:
+            sort = sorted(unbool(i) for i in container)
+            sliced = itertools.islice(sort, 1, None)
+            for i, j in zip(sort, sliced):
+                if i == j:
+                    return False
+        except (NotImplementedError, TypeError):
+            seen = []
+            for e in container:
+                e = unbool(e)
+                if e in seen:
+                    return False
+                seen.append(e)
+    return True
diff --git a/contrib/python/jsonschema/py2/jsonschema/_validators.py b/contrib/python/jsonschema/py2/jsonschema/_validators.py
new file mode 100644
index 00000000000..179fec09a94
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/_validators.py
@@ -0,0 +1,373 @@
+import re
+
+from jsonschema._utils import (
+    ensure_list,
+    equal,
+    extras_msg,
+    find_additional_properties,
+    types_msg,
+    unbool,
+    uniq,
+)
+from jsonschema.exceptions import FormatError, ValidationError
+from jsonschema.compat import iteritems
+
+
+def patternProperties(validator, patternProperties, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for pattern, subschema in iteritems(patternProperties):
+        for k, v in iteritems(instance):
+            if re.search(pattern, k):
+                for error in validator.descend(
+                    v, subschema, path=k, schema_path=pattern,
+                ):
+                    yield error
+
+
+def propertyNames(validator, propertyNames, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property in instance:
+        for error in validator.descend(
+            instance=property,
+            schema=propertyNames,
+        ):
+            yield error
+
+
+def additionalProperties(validator, aP, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    extras = set(find_additional_properties(instance, schema))
+
+    if validator.is_type(aP, "object"):
+        for extra in extras:
+            for error in validator.descend(instance[extra], aP, path=extra):
+                yield error
+    elif not aP and extras:
+        if "patternProperties" in schema:
+            patterns = sorted(schema["patternProperties"])
+            if len(extras) == 1:
+                verb = "does"
+            else:
+                verb = "do"
+            error = "%s %s not match any of the regexes: %s" % (
+                ", ".join(map(repr, sorted(extras))),
+                verb,
+                ", ".join(map(repr, patterns)),
+            )
+            yield ValidationError(error)
+        else:
+            error = "Additional properties are not allowed (%s %s unexpected)"
+            yield ValidationError(error % extras_msg(extras))
+
+
+def items(validator, items, instance, schema):
+    if not validator.is_type(instance, "array"):
+        return
+
+    if validator.is_type(items, "array"):
+        for (index, item), subschema in zip(enumerate(instance), items):
+            for error in validator.descend(
+                item, subschema, path=index, schema_path=index,
+            ):
+                yield error
+    else:
+        for index, item in enumerate(instance):
+            for error in validator.descend(item, items, path=index):
+                yield error
+
+
+def additionalItems(validator, aI, instance, schema):
+    if (
+        not validator.is_type(instance, "array") or
+        validator.is_type(schema.get("items", {}), "object")
+    ):
+        return
+
+    len_items = len(schema.get("items", []))
+    if validator.is_type(aI, "object"):
+        for index, item in enumerate(instance[len_items:], start=len_items):
+            for error in validator.descend(item, aI, path=index):
+                yield error
+    elif not aI and len(instance) > len(schema.get("items", [])):
+        error = "Additional items are not allowed (%s %s unexpected)"
+        yield ValidationError(
+            error %
+            extras_msg(instance[len(schema.get("items", [])):])
+        )
+
+
+def const(validator, const, instance, schema):
+    if not equal(instance, const):
+        yield ValidationError("%r was expected" % (const,))
+
+
+def contains(validator, contains, instance, schema):
+    if not validator.is_type(instance, "array"):
+        return
+
+    if not any(validator.is_valid(element, contains) for element in instance):
+        yield ValidationError(
+            "None of %r are valid under the given schema" % (instance,)
+        )
+
+
+def exclusiveMinimum(validator, minimum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if instance <= minimum:
+        yield ValidationError(
+            "%r is less than or equal to the minimum of %r" % (
+                instance, minimum,
+            ),
+        )
+
+
+def exclusiveMaximum(validator, maximum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if instance >= maximum:
+        yield ValidationError(
+            "%r is greater than or equal to the maximum of %r" % (
+                instance, maximum,
+            ),
+        )
+
+
+def minimum(validator, minimum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if instance < minimum:
+        yield ValidationError(
+            "%r is less than the minimum of %r" % (instance, minimum)
+        )
+
+
+def maximum(validator, maximum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if instance > maximum:
+        yield ValidationError(
+            "%r is greater than the maximum of %r" % (instance, maximum)
+        )
+
+
+def multipleOf(validator, dB, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if isinstance(dB, float):
+        quotient = instance / dB
+        failed = int(quotient) != quotient
+    else:
+        failed = instance % dB
+
+    if failed:
+        yield ValidationError("%r is not a multiple of %r" % (instance, dB))
+
+
+def minItems(validator, mI, instance, schema):
+    if validator.is_type(instance, "array") and len(instance) < mI:
+        yield ValidationError("%r is too short" % (instance,))
+
+
+def maxItems(validator, mI, instance, schema):
+    if validator.is_type(instance, "array") and len(instance) > mI:
+        yield ValidationError("%r is too long" % (instance,))
+
+
+def uniqueItems(validator, uI, instance, schema):
+    if (
+        uI and
+        validator.is_type(instance, "array") and
+        not uniq(instance)
+    ):
+        yield ValidationError("%r has non-unique elements" % (instance,))
+
+
+def pattern(validator, patrn, instance, schema):
+    if (
+        validator.is_type(instance, "string") and
+        not re.search(patrn, instance)
+    ):
+        yield ValidationError("%r does not match %r" % (instance, patrn))
+
+
+def format(validator, format, instance, schema):
+    if validator.format_checker is not None:
+        try:
+            validator.format_checker.check(instance, format)
+        except FormatError as error:
+            yield ValidationError(error.message, cause=error.cause)
+
+
+def minLength(validator, mL, instance, schema):
+    if validator.is_type(instance, "string") and len(instance) < mL:
+        yield ValidationError("%r is too short" % (instance,))
+
+
+def maxLength(validator, mL, instance, schema):
+    if validator.is_type(instance, "string") and len(instance) > mL:
+        yield ValidationError("%r is too long" % (instance,))
+
+
+def dependencies(validator, dependencies, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property, dependency in iteritems(dependencies):
+        if property not in instance:
+            continue
+
+        if validator.is_type(dependency, "array"):
+            for each in dependency:
+                if each not in instance:
+                    message = "%r is a dependency of %r"
+                    yield ValidationError(message % (each, property))
+        else:
+            for error in validator.descend(
+                instance, dependency, schema_path=property,
+            ):
+                yield error
+
+
+def enum(validator, enums, instance, schema):
+    if instance == 0 or instance == 1:
+        unbooled = unbool(instance)
+        if all(unbooled != unbool(each) for each in enums):
+            yield ValidationError("%r is not one of %r" % (instance, enums))
+    elif instance not in enums:
+        yield ValidationError("%r is not one of %r" % (instance, enums))
+
+
+def ref(validator, ref, instance, schema):
+    resolve = getattr(validator.resolver, "resolve", None)
+    if resolve is None:
+        with validator.resolver.resolving(ref) as resolved:
+            for error in validator.descend(instance, resolved):
+                yield error
+    else:
+        scope, resolved = validator.resolver.resolve(ref)
+        validator.resolver.push_scope(scope)
+
+        try:
+            for error in validator.descend(instance, resolved):
+                yield error
+        finally:
+            validator.resolver.pop_scope()
+
+
+def type(validator, types, instance, schema):
+    types = ensure_list(types)
+
+    if not any(validator.is_type(instance, type) for type in types):
+        yield ValidationError(types_msg(instance, types))
+
+
+def properties(validator, properties, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property, subschema in iteritems(properties):
+        if property in instance:
+            for error in validator.descend(
+                instance[property],
+                subschema,
+                path=property,
+                schema_path=property,
+            ):
+                yield error
+
+
+def required(validator, required, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+    for property in required:
+        if property not in instance:
+            yield ValidationError("%r is a required property" % property)
+
+
+def minProperties(validator, mP, instance, schema):
+    if validator.is_type(instance, "object") and len(instance) < mP:
+        yield ValidationError(
+            "%r does not have enough properties" % (instance,)
+        )
+
+
+def maxProperties(validator, mP, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+    if validator.is_type(instance, "object") and len(instance) > mP:
+        yield ValidationError("%r has too many properties" % (instance,))
+
+
+def allOf(validator, allOf, instance, schema):
+    for index, subschema in enumerate(allOf):
+        for error in validator.descend(instance, subschema, schema_path=index):
+            yield error
+
+
+def anyOf(validator, anyOf, instance, schema):
+    all_errors = []
+    for index, subschema in enumerate(anyOf):
+        errs = list(validator.descend(instance, subschema, schema_path=index))
+        if not errs:
+            break
+        all_errors.extend(errs)
+    else:
+        yield ValidationError(
+            "%r is not valid under any of the given schemas" % (instance,),
+            context=all_errors,
+        )
+
+
+def oneOf(validator, oneOf, instance, schema):
+    subschemas = enumerate(oneOf)
+    all_errors = []
+    for index, subschema in subschemas:
+        errs = list(validator.descend(instance, subschema, schema_path=index))
+        if not errs:
+            first_valid = subschema
+            break
+        all_errors.extend(errs)
+    else:
+        yield ValidationError(
+            "%r is not valid under any of the given schemas" % (instance,),
+            context=all_errors,
+        )
+
+    more_valid = [s for i, s in subschemas if validator.is_valid(instance, s)]
+    if more_valid:
+        more_valid.append(first_valid)
+        reprs = ", ".join(repr(schema) for schema in more_valid)
+        yield ValidationError(
+            "%r is valid under each of %s" % (instance, reprs)
+        )
+
+
+def not_(validator, not_schema, instance, schema):
+    if validator.is_valid(instance, not_schema):
+        yield ValidationError(
+            "%r is not allowed for %r" % (not_schema, instance)
+        )
+
+
+def if_(validator, if_schema, instance, schema):
+    if validator.is_valid(instance, if_schema):
+        if u"then" in schema:
+            then = schema[u"then"]
+            for error in validator.descend(instance, then, schema_path="then"):
+                yield error
+    elif u"else" in schema:
+        else_ = schema[u"else"]
+        for error in validator.descend(instance, else_, schema_path="else"):
+            yield error
diff --git a/contrib/python/jsonschema/py2/jsonschema/cli.py b/contrib/python/jsonschema/py2/jsonschema/cli.py
new file mode 100644
index 00000000000..ab3335b27c5
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/cli.py
@@ -0,0 +1,90 @@
+"""
+The ``jsonschema`` command line.
+"""
+from __future__ import absolute_import
+import argparse
+import json
+import sys
+
+from jsonschema import __version__
+from jsonschema._reflect import namedAny
+from jsonschema.validators import validator_for
+
+
+def _namedAnyWithDefault(name):
+    if "." not in name:
+        name = "jsonschema." + name
+    return namedAny(name)
+
+
+def _json_file(path):
+    with open(path) as file:
+        return json.load(file)
+
+
+parser = argparse.ArgumentParser(
+    description="JSON Schema Validation CLI",
+)
+parser.add_argument(
+    "-i", "--instance",
+    action="append",
+    dest="instances",
+    type=_json_file,
+    help=(
+        "a path to a JSON instance (i.e. filename.json) "
+        "to validate (may be specified multiple times)"
+    ),
+)
+parser.add_argument(
+    "-F", "--error-format",
+    default="{error.instance}: {error.message}\n",
+    help=(
+        "the format to use for each error output message, specified in "
+        "a form suitable for passing to str.format, which will be called "
+        "with 'error' for each error"
+    ),
+)
+parser.add_argument(
+    "-V", "--validator",
+    type=_namedAnyWithDefault,
+    help=(
+        "the fully qualified object name of a validator to use, or, for "
+        "validators that are registered with jsonschema, simply the name "
+        "of the class."
+    ),
+)
+parser.add_argument(
+    "--version",
+    action="version",
+    version=__version__,
+)
+parser.add_argument(
+    "schema",
+    help="the JSON Schema to validate with (i.e. schema.json)",
+    type=_json_file,
+)
+
+
+def parse_args(args):
+    arguments = vars(parser.parse_args(args=args or ["--help"]))
+    if arguments["validator"] is None:
+        arguments["validator"] = validator_for(arguments["schema"])
+    return arguments
+
+
+def main(args=sys.argv[1:]):
+    sys.exit(run(arguments=parse_args(args=args)))
+
+
+def run(arguments, stdout=sys.stdout, stderr=sys.stderr):
+    error_format = arguments["error_format"]
+    validator = arguments["validator"](schema=arguments["schema"])
+
+    validator.check_schema(arguments["schema"])
+
+    errored = False
+    for instance in arguments["instances"] or ():
+        for error in validator.iter_errors(instance):
+            stderr.write(error_format.format(error=error))
+            errored = True
+    return errored
diff --git a/contrib/python/jsonschema/py2/jsonschema/compat.py b/contrib/python/jsonschema/py2/jsonschema/compat.py
new file mode 100644
index 00000000000..47e09804551
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/compat.py
@@ -0,0 +1,55 @@
+"""
+Python 2/3 compatibility helpers.
+
+Note: This module is *not* public API.
+"""
+import contextlib
+import operator
+import sys
+
+
+try:
+    from collections.abc import MutableMapping, Sequence  # noqa
+except ImportError:
+    from collections import MutableMapping, Sequence  # noqa
+
+PY3 = sys.version_info[0] >= 3
+
+if PY3:
+    zip = zip
+    from functools import lru_cache
+    from io import StringIO as NativeIO
+    from urllib.parse import (
+        unquote, urljoin, urlunsplit, SplitResult, urlsplit
+    )
+    from urllib.request import pathname2url, urlopen
+    str_types = str,
+    int_types = int,
+    iteritems = operator.methodcaller("items")
+else:
+    from itertools import izip as zip  # noqa
+    from io import BytesIO as NativeIO
+    from urlparse import urljoin, urlunsplit, SplitResult, urlsplit
+    from urllib import pathname2url, unquote  # noqa
+    import urllib2  # noqa
+    def urlopen(*args, **kwargs):
+        return contextlib.closing(urllib2.urlopen(*args, **kwargs))
+
+    str_types = basestring
+    int_types = int, long
+    iteritems = operator.methodcaller("iteritems")
+
+    from functools32 import lru_cache
+
+
+def urldefrag(url):
+    if "#" in url:
+        s, n, p, q, frag = urlsplit(url)
+        defrag = urlunsplit((s, n, p, q, ""))
+    else:
+        defrag = url
+        frag = ""
+    return defrag, frag
+
+
+# flake8: noqa
diff --git a/contrib/python/jsonschema/py2/jsonschema/exceptions.py b/contrib/python/jsonschema/py2/jsonschema/exceptions.py
new file mode 100644
index 00000000000..691dcffe6c7
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/exceptions.py
@@ -0,0 +1,374 @@
+"""
+Validation errors, and some surrounding helpers.
+"""
+from collections import defaultdict, deque
+import itertools
+import pprint
+import textwrap
+
+import attr
+
+from jsonschema import _utils
+from jsonschema.compat import PY3, iteritems
+
+
+WEAK_MATCHES = frozenset(["anyOf", "oneOf"])
+STRONG_MATCHES = frozenset()
+
+_unset = _utils.Unset()
+
+
+class _Error(Exception):
+    def __init__(
+        self,
+        message,
+        validator=_unset,
+        path=(),
+        cause=None,
+        context=(),
+        validator_value=_unset,
+        instance=_unset,
+        schema=_unset,
+        schema_path=(),
+        parent=None,
+    ):
+        super(_Error, self).__init__(
+            message,
+            validator,
+            path,
+            cause,
+            context,
+            validator_value,
+            instance,
+            schema,
+            schema_path,
+            parent,
+        )
+        self.message = message
+        self.path = self.relative_path = deque(path)
+        self.schema_path = self.relative_schema_path = deque(schema_path)
+        self.context = list(context)
+        self.cause = self.__cause__ = cause
+        self.validator = validator
+        self.validator_value = validator_value
+        self.instance = instance
+        self.schema = schema
+        self.parent = parent
+
+        for error in context:
+            error.parent = self
+
+    def __repr__(self):
+        return "<%s: %r>" % (self.__class__.__name__, self.message)
+
+    def __unicode__(self):
+        essential_for_verbose = (
+            self.validator, self.validator_value, self.instance, self.schema,
+        )
+        if any(m is _unset for m in essential_for_verbose):
+            return self.message
+
+        pschema = pprint.pformat(self.schema, width=72)
+        pinstance = pprint.pformat(self.instance, width=72)
+        return self.message + textwrap.dedent("""
+
+            Failed validating %r in %s%s:
+            %s
+
+            On %s%s:
+            %s
+            """.rstrip()
+        ) % (
+            self.validator,
+            self._word_for_schema_in_error_message,
+            _utils.format_as_index(list(self.relative_schema_path)[:-1]),
+            _utils.indent(pschema),
+            self._word_for_instance_in_error_message,
+            _utils.format_as_index(self.relative_path),
+            _utils.indent(pinstance),
+        )
+
+    if PY3:
+        __str__ = __unicode__
+    else:
+        def __str__(self):
+            return unicode(self).encode("utf-8")
+
+    @classmethod
+    def create_from(cls, other):
+        return cls(**other._contents())
+
+    @property
+    def absolute_path(self):
+        parent = self.parent
+        if parent is None:
+            return self.relative_path
+
+        path = deque(self.relative_path)
+        path.extendleft(reversed(parent.absolute_path))
+        return path
+
+    @property
+    def absolute_schema_path(self):
+        parent = self.parent
+        if parent is None:
+            return self.relative_schema_path
+
+        path = deque(self.relative_schema_path)
+        path.extendleft(reversed(parent.absolute_schema_path))
+        return path
+
+    def _set(self, **kwargs):
+        for k, v in iteritems(kwargs):
+            if getattr(self, k) is _unset:
+                setattr(self, k, v)
+
+    def _contents(self):
+        attrs = (
+            "message", "cause", "context", "validator", "validator_value",
+            "path", "schema_path", "instance", "schema", "parent",
+        )
+        return dict((attr, getattr(self, attr)) for attr in attrs)
+
+
+class ValidationError(_Error):
+    """
+    An instance was invalid under a provided schema.
+    """
+
+    _word_for_schema_in_error_message = "schema"
+    _word_for_instance_in_error_message = "instance"
+
+
+class SchemaError(_Error):
+    """
+    A schema was invalid under its corresponding metaschema.
+    """
+
+    _word_for_schema_in_error_message = "metaschema"
+    _word_for_instance_in_error_message = "schema"
+
+
+@attr.s(hash=True)
+class RefResolutionError(Exception):
+    """
+    A ref could not be resolved.
+    """
+
+    _cause = attr.ib()
+
+    def __str__(self):
+        return str(self._cause)
+
+
+class UndefinedTypeCheck(Exception):
+    """
+    A type checker was asked to check a type it did not have registered.
+    """
+
+    def __init__(self, type):
+        self.type = type
+
+    def __unicode__(self):
+        return "Type %r is unknown to this type checker" % self.type
+
+    if PY3:
+        __str__ = __unicode__
+    else:
+        def __str__(self):
+            return unicode(self).encode("utf-8")
+
+
+class UnknownType(Exception):
+    """
+    A validator was asked to validate an instance against an unknown type.
+    """
+
+    def __init__(self, type, instance, schema):
+        self.type = type
+        self.instance = instance
+        self.schema = schema
+
+    def __unicode__(self):
+        pschema = pprint.pformat(self.schema, width=72)
+        pinstance = pprint.pformat(self.instance, width=72)
+        return textwrap.dedent("""
+            Unknown type %r for validator with schema:
+            %s
+
+            While checking instance:
+            %s
+            """.rstrip()
+        ) % (self.type, _utils.indent(pschema), _utils.indent(pinstance))
+
+    if PY3:
+        __str__ = __unicode__
+    else:
+        def __str__(self):
+            return unicode(self).encode("utf-8")
+
+
+class FormatError(Exception):
+    """
+    Validating a format failed.
+    """
+
+    def __init__(self, message, cause=None):
+        super(FormatError, self).__init__(message, cause)
+        self.message = message
+        self.cause = self.__cause__ = cause
+
+    def __unicode__(self):
+        return self.message
+
+    if PY3:
+        __str__ = __unicode__
+    else:
+        def __str__(self):
+            return self.message.encode("utf-8")
+
+
+class ErrorTree(object):
+    """
+    ErrorTrees make it easier to check which validations failed.
+    """
+
+    _instance = _unset
+
+    def __init__(self, errors=()):
+        self.errors = {}
+        self._contents = defaultdict(self.__class__)
+
+        for error in errors:
+            container = self
+            for element in error.path:
+                container = container[element]
+            container.errors[error.validator] = error
+
+            container._instance = error.instance
+
+    def __contains__(self, index):
+        """
+        Check whether ``instance[index]`` has any errors.
+        """
+
+        return index in self._contents
+
+    def __getitem__(self, index):
+        """
+        Retrieve the child tree one level down at the given ``index``.
+
+        If the index is not in the instance that this tree corresponds to and
+        is not known by this tree, whatever error would be raised by
+        ``instance.__getitem__`` will be propagated (usually this is some
+        subclass of `exceptions.LookupError`.
+        """
+
+        if self._instance is not _unset and index not in self:
+            self._instance[index]
+        return self._contents[index]
+
+    def __setitem__(self, index, value):
+        """
+        Add an error to the tree at the given ``index``.
+        """
+        self._contents[index] = value
+
+    def __iter__(self):
+        """
+        Iterate (non-recursively) over the indices in the instance with errors.
+        """
+
+        return iter(self._contents)
+
+    def __len__(self):
+        """
+        Return the `total_errors`.
+        """
+        return self.total_errors
+
+    def __repr__(self):
+        return "<%s (%s total errors)>" % (self.__class__.__name__, len(self))
+
+    @property
+    def total_errors(self):
+        """
+        The total number of errors in the entire tree, including children.
+        """
+
+        child_errors = sum(len(tree) for _, tree in iteritems(self._contents))
+        return len(self.errors) + child_errors
+
+
+def by_relevance(weak=WEAK_MATCHES, strong=STRONG_MATCHES):
+    """
+    Create a key function that can be used to sort errors by relevance.
+
+    Arguments:
+        weak (set):
+            a collection of validator names to consider to be "weak".
+            If there are two errors at the same level of the instance
+            and one is in the set of weak validator names, the other
+            error will take priority. By default, :validator:`anyOf` and
+            :validator:`oneOf` are considered weak validators and will
+            be superseded by other same-level validation errors.
+
+        strong (set):
+            a collection of validator names to consider to be "strong"
+    """
+    def relevance(error):
+        validator = error.validator
+        return -len(error.path), validator not in weak, validator in strong
+    return relevance
+
+
+relevance = by_relevance()
+
+
+def best_match(errors, key=relevance):
+    """
+    Try to find an error that appears to be the best match among given errors.
+
+    In general, errors that are higher up in the instance (i.e. for which
+    `ValidationError.path` is shorter) are considered better matches,
+    since they indicate "more" is wrong with the instance.
+
+    If the resulting match is either :validator:`oneOf` or :validator:`anyOf`,
+    the *opposite* assumption is made -- i.e. the deepest error is picked,
+    since these validators only need to match once, and any other errors may
+    not be relevant.
+
+    Arguments:
+        errors (collections.Iterable):
+
+            the errors to select from. Do not provide a mixture of
+            errors from different validation attempts (i.e. from
+            different instances or schemas), since it won't produce
+            sensical output.
+
+        key (collections.Callable):
+
+            the key to use when sorting errors. See `relevance` and
+            transitively `by_relevance` for more details (the default is
+            to sort with the defaults of that function). Changing the
+            default is only useful if you want to change the function
+            that rates errors but still want the error context descent
+            done by this function.
+
+    Returns:
+        the best matching error, or ``None`` if the iterable was empty
+
+    .. note::
+
+        This function is a heuristic. Its return value may change for a given
+        set of inputs from version to version if better heuristics are added.
+    """
+    errors = iter(errors)
+    best = next(errors, None)
+    if best is None:
+        return
+    best = max(itertools.chain([best], errors), key=key)
+
+    while best.context:
+        best = min(best.context, key=key)
+    return best
diff --git a/contrib/python/jsonschema/py2/jsonschema/schemas/draft3.json b/contrib/python/jsonschema/py2/jsonschema/schemas/draft3.json
new file mode 100644
index 00000000000..f8a09c563b4
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/schemas/draft3.json
@@ -0,0 +1,199 @@
+{
+    "$schema": "http://json-schema.org/draft-03/schema#",
+    "dependencies": {
+        "exclusiveMaximum": "maximum",
+        "exclusiveMinimum": "minimum"
+    },
+    "id": "http://json-schema.org/draft-03/schema#",
+    "properties": {
+        "$ref": {
+            "format": "uri",
+            "type": "string"
+        },
+        "$schema": {
+            "format": "uri",
+            "type": "string"
+        },
+        "additionalItems": {
+            "default": {},
+            "type": [
+                {
+                    "$ref": "#"
+                },
+                "boolean"
+            ]
+        },
+        "additionalProperties": {
+            "default": {},
+            "type": [
+                {
+                    "$ref": "#"
+                },
+                "boolean"
+            ]
+        },
+        "default": {
+            "type": "any"
+        },
+        "dependencies": {
+            "additionalProperties": {
+                "items": {
+                    "type": "string"
+                },
+                "type": [
+                    "string",
+                    "array",
+                    {
+                        "$ref": "#"
+                    }
+                ]
+            },
+            "default": {},
+            "type": [
+                "string",
+                "array",
+                "object"
+            ]
+        },
+        "description": {
+            "type": "string"
+        },
+        "disallow": {
+            "items": {
+                "type": [
+                    "string",
+                    {
+                        "$ref": "#"
+                    }
+                ]
+            },
+            "type": [
+                "string",
+                "array"
+            ],
+            "uniqueItems": true
+        },
+        "divisibleBy": {
+            "default": 1,
+            "exclusiveMinimum": true,
+            "minimum": 0,
+            "type": "number"
+        },
+        "enum": {
+            "type": "array"
+        },
+        "exclusiveMaximum": {
+            "default": false,
+            "type": "boolean"
+        },
+        "exclusiveMinimum": {
+            "default": false,
+            "type": "boolean"
+        },
+        "extends": {
+            "default": {},
+            "items": {
+                "$ref": "#"
+            },
+            "type": [
+                {
+                    "$ref": "#"
+                },
+                "array"
+            ]
+        },
+        "format": {
+            "type": "string"
+        },
+        "id": {
+            "format": "uri",
+            "type": "string"
+        },
+        "items": {
+            "default": {},
+            "items": {
+                "$ref": "#"
+            },
+            "type": [
+                {
+                    "$ref": "#"
+                },
+                "array"
+            ]
+        },
+        "maxDecimal": {
+            "minimum": 0,
+            "type": "number"
+        },
+        "maxItems": {
+            "minimum": 0,
+            "type": "integer"
+        },
+        "maxLength": {
+            "type": "integer"
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "minItems": {
+            "default": 0,
+            "minimum": 0,
+            "type": "integer"
+        },
+        "minLength": {
+            "default": 0,
+            "minimum": 0,
+            "type": "integer"
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "pattern": {
+            "format": "regex",
+            "type": "string"
+        },
+        "patternProperties": {
+            "additionalProperties": {
+                "$ref": "#"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "properties": {
+            "additionalProperties": {
+                "$ref": "#",
+                "type": "object"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "required": {
+            "default": false,
+            "type": "boolean"
+        },
+        "title": {
+            "type": "string"
+        },
+        "type": {
+            "default": "any",
+            "items": {
+                "type": [
+                    "string",
+                    {
+                        "$ref": "#"
+                    }
+                ]
+            },
+            "type": [
+                "string",
+                "array"
+            ],
+            "uniqueItems": true
+        },
+        "uniqueItems": {
+            "default": false,
+            "type": "boolean"
+        }
+    },
+    "type": "object"
+}
diff --git a/contrib/python/jsonschema/py2/jsonschema/schemas/draft4.json b/contrib/python/jsonschema/py2/jsonschema/schemas/draft4.json
new file mode 100644
index 00000000000..9b666cff88a
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/schemas/draft4.json
@@ -0,0 +1,222 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "default": {},
+    "definitions": {
+        "positiveInteger": {
+            "minimum": 0,
+            "type": "integer"
+        },
+        "positiveIntegerDefault0": {
+            "allOf": [
+                {
+                    "$ref": "#/definitions/positiveInteger"
+                },
+                {
+                    "default": 0
+                }
+            ]
+        },
+        "schemaArray": {
+            "items": {
+                "$ref": "#"
+            },
+            "minItems": 1,
+            "type": "array"
+        },
+        "simpleTypes": {
+            "enum": [
+                "array",
+                "boolean",
+                "integer",
+                "null",
+                "number",
+                "object",
+                "string"
+            ]
+        },
+        "stringArray": {
+            "items": {
+                "type": "string"
+            },
+            "minItems": 1,
+            "type": "array",
+            "uniqueItems": true
+        }
+    },
+    "dependencies": {
+        "exclusiveMaximum": [
+            "maximum"
+        ],
+        "exclusiveMinimum": [
+            "minimum"
+        ]
+    },
+    "description": "Core schema meta-schema",
+    "id": "http://json-schema.org/draft-04/schema#",
+    "properties": {
+        "$schema": {
+            "format": "uri",
+            "type": "string"
+        },
+        "additionalItems": {
+            "anyOf": [
+                {
+                    "type": "boolean"
+                },
+                {
+                    "$ref": "#"
+                }
+            ],
+            "default": {}
+        },
+        "additionalProperties": {
+            "anyOf": [
+                {
+                    "type": "boolean"
+                },
+                {
+                    "$ref": "#"
+                }
+            ],
+            "default": {}
+        },
+        "allOf": {
+            "$ref": "#/definitions/schemaArray"
+        },
+        "anyOf": {
+            "$ref": "#/definitions/schemaArray"
+        },
+        "default": {},
+        "definitions": {
+            "additionalProperties": {
+                "$ref": "#"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "dependencies": {
+            "additionalProperties": {
+                "anyOf": [
+                    {
+                        "$ref": "#"
+                    },
+                    {
+                        "$ref": "#/definitions/stringArray"
+                    }
+                ]
+            },
+            "type": "object"
+        },
+        "description": {
+            "type": "string"
+        },
+        "enum": {
+            "type": "array"
+        },
+        "exclusiveMaximum": {
+            "default": false,
+            "type": "boolean"
+        },
+        "exclusiveMinimum": {
+            "default": false,
+            "type": "boolean"
+        },
+        "format": {
+            "type": "string"
+        },
+        "id": {
+            "format": "uri",
+            "type": "string"
+        },
+        "items": {
+            "anyOf": [
+                {
+                    "$ref": "#"
+                },
+                {
+                    "$ref": "#/definitions/schemaArray"
+                }
+            ],
+            "default": {}
+        },
+        "maxItems": {
+            "$ref": "#/definitions/positiveInteger"
+        },
+        "maxLength": {
+            "$ref": "#/definitions/positiveInteger"
+        },
+        "maxProperties": {
+            "$ref": "#/definitions/positiveInteger"
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "minItems": {
+            "$ref": "#/definitions/positiveIntegerDefault0"
+        },
+        "minLength": {
+            "$ref": "#/definitions/positiveIntegerDefault0"
+        },
+        "minProperties": {
+            "$ref": "#/definitions/positiveIntegerDefault0"
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "multipleOf": {
+            "exclusiveMinimum": true,
+            "minimum": 0,
+            "type": "number"
+        },
+        "not": {
+            "$ref": "#"
+        },
+        "oneOf": {
+            "$ref": "#/definitions/schemaArray"
+        },
+        "pattern": {
+            "format": "regex",
+            "type": "string"
+        },
+        "patternProperties": {
+            "additionalProperties": {
+                "$ref": "#"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "properties": {
+            "additionalProperties": {
+                "$ref": "#"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "required": {
+            "$ref": "#/definitions/stringArray"
+        },
+        "title": {
+            "type": "string"
+        },
+        "type": {
+            "anyOf": [
+                {
+                    "$ref": "#/definitions/simpleTypes"
+                },
+                {
+                    "items": {
+                        "$ref": "#/definitions/simpleTypes"
+                    },
+                    "minItems": 1,
+                    "type": "array",
+                    "uniqueItems": true
+                }
+            ]
+        },
+        "uniqueItems": {
+            "default": false,
+            "type": "boolean"
+        }
+    },
+    "type": "object"
+}
diff --git a/contrib/python/jsonschema/py2/jsonschema/schemas/draft6.json b/contrib/python/jsonschema/py2/jsonschema/schemas/draft6.json
new file mode 100644
index 00000000000..a0d2bf7896c
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/schemas/draft6.json
@@ -0,0 +1,153 @@
+{
+    "$schema": "http://json-schema.org/draft-06/schema#",
+    "$id": "http://json-schema.org/draft-06/schema#",
+    "title": "Core schema meta-schema",
+    "definitions": {
+        "schemaArray": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "$ref": "#" }
+        },
+        "nonNegativeInteger": {
+            "type": "integer",
+            "minimum": 0
+        },
+        "nonNegativeIntegerDefault0": {
+            "allOf": [
+                { "$ref": "#/definitions/nonNegativeInteger" },
+                { "default": 0 }
+            ]
+        },
+        "simpleTypes": {
+            "enum": [
+                "array",
+                "boolean",
+                "integer",
+                "null",
+                "number",
+                "object",
+                "string"
+            ]
+        },
+        "stringArray": {
+            "type": "array",
+            "items": { "type": "string" },
+            "uniqueItems": true,
+            "default": []
+        }
+    },
+    "type": ["object", "boolean"],
+    "properties": {
+        "$id": {
+            "type": "string",
+            "format": "uri-reference"
+        },
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
+        "$ref": {
+            "type": "string",
+            "format": "uri-reference"
+        },
+        "title": {
+            "type": "string"
+        },
+        "description": {
+            "type": "string"
+        },
+        "default": {},
+        "examples": {
+            "type": "array",
+            "items": {}
+        },
+        "multipleOf": {
+            "type": "number",
+            "exclusiveMinimum": 0
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "exclusiveMaximum": {
+            "type": "number"
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "exclusiveMinimum": {
+            "type": "number"
+        },
+        "maxLength": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "pattern": {
+            "type": "string",
+            "format": "regex"
+        },
+        "additionalItems": { "$ref": "#" },
+        "items": {
+            "anyOf": [
+                { "$ref": "#" },
+                { "$ref": "#/definitions/schemaArray" }
+            ],
+            "default": {}
+        },
+        "maxItems": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "uniqueItems": {
+            "type": "boolean",
+            "default": false
+        },
+        "contains": { "$ref": "#" },
+        "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "required": { "$ref": "#/definitions/stringArray" },
+        "additionalProperties": { "$ref": "#" },
+        "definitions": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "properties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "patternProperties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "propertyNames": { "format": "regex" },
+            "default": {}
+        },
+        "dependencies": {
+            "type": "object",
+            "additionalProperties": {
+                "anyOf": [
+                    { "$ref": "#" },
+                    { "$ref": "#/definitions/stringArray" }
+                ]
+            }
+        },
+        "propertyNames": { "$ref": "#" },
+        "const": {},
+        "enum": {
+            "type": "array"
+        },
+        "type": {
+            "anyOf": [
+                { "$ref": "#/definitions/simpleTypes" },
+                {
+                    "type": "array",
+                    "items": { "$ref": "#/definitions/simpleTypes" },
+                    "minItems": 1,
+                    "uniqueItems": true
+                }
+            ]
+        },
+        "format": { "type": "string" },
+        "allOf": { "$ref": "#/definitions/schemaArray" },
+        "anyOf": { "$ref": "#/definitions/schemaArray" },
+        "oneOf": { "$ref": "#/definitions/schemaArray" },
+        "not": { "$ref": "#" }
+    },
+    "default": {}
+}
diff --git a/contrib/python/jsonschema/py2/jsonschema/schemas/draft7.json b/contrib/python/jsonschema/py2/jsonschema/schemas/draft7.json
new file mode 100644
index 00000000000..746cde96901
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/schemas/draft7.json
@@ -0,0 +1,166 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "http://json-schema.org/draft-07/schema#",
+    "title": "Core schema meta-schema",
+    "definitions": {
+        "schemaArray": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "$ref": "#" }
+        },
+        "nonNegativeInteger": {
+            "type": "integer",
+            "minimum": 0
+        },
+        "nonNegativeIntegerDefault0": {
+            "allOf": [
+                { "$ref": "#/definitions/nonNegativeInteger" },
+                { "default": 0 }
+            ]
+        },
+        "simpleTypes": {
+            "enum": [
+                "array",
+                "boolean",
+                "integer",
+                "null",
+                "number",
+                "object",
+                "string"
+            ]
+        },
+        "stringArray": {
+            "type": "array",
+            "items": { "type": "string" },
+            "uniqueItems": true,
+            "default": []
+        }
+    },
+    "type": ["object", "boolean"],
+    "properties": {
+        "$id": {
+            "type": "string",
+            "format": "uri-reference"
+        },
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
+        "$ref": {
+            "type": "string",
+            "format": "uri-reference"
+        },
+        "$comment": {
+            "type": "string"
+        },
+        "title": {
+            "type": "string"
+        },
+        "description": {
+            "type": "string"
+        },
+        "default": true,
+        "readOnly": {
+            "type": "boolean",
+            "default": false
+        },
+        "examples": {
+            "type": "array",
+            "items": true
+        },
+        "multipleOf": {
+            "type": "number",
+            "exclusiveMinimum": 0
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "exclusiveMaximum": {
+            "type": "number"
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "exclusiveMinimum": {
+            "type": "number"
+        },
+        "maxLength": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "pattern": {
+            "type": "string",
+            "format": "regex"
+        },
+        "additionalItems": { "$ref": "#" },
+        "items": {
+            "anyOf": [
+                { "$ref": "#" },
+                { "$ref": "#/definitions/schemaArray" }
+            ],
+            "default": true
+        },
+        "maxItems": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "uniqueItems": {
+            "type": "boolean",
+            "default": false
+        },
+        "contains": { "$ref": "#" },
+        "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "required": { "$ref": "#/definitions/stringArray" },
+        "additionalProperties": { "$ref": "#" },
+        "definitions": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "properties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "patternProperties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "propertyNames": { "format": "regex" },
+            "default": {}
+        },
+        "dependencies": {
+            "type": "object",
+            "additionalProperties": {
+                "anyOf": [
+                    { "$ref": "#" },
+                    { "$ref": "#/definitions/stringArray" }
+                ]
+            }
+        },
+        "propertyNames": { "$ref": "#" },
+        "const": true,
+        "enum": {
+            "type": "array",
+            "items": true
+        },
+        "type": {
+            "anyOf": [
+                { "$ref": "#/definitions/simpleTypes" },
+                {
+                    "type": "array",
+                    "items": { "$ref": "#/definitions/simpleTypes" },
+                    "minItems": 1,
+                    "uniqueItems": true
+                }
+            ]
+        },
+        "format": { "type": "string" },
+        "contentMediaType": { "type": "string" },
+        "contentEncoding": { "type": "string" },
+        "if": {"$ref": "#"},
+        "then": {"$ref": "#"},
+        "else": {"$ref": "#"},
+        "allOf": { "$ref": "#/definitions/schemaArray" },
+        "anyOf": { "$ref": "#/definitions/schemaArray" },
+        "oneOf": { "$ref": "#/definitions/schemaArray" },
+        "not": { "$ref": "#" }
+    },
+    "default": true
+}
diff --git a/contrib/python/jsonschema/py2/jsonschema/tests/__init__.py b/contrib/python/jsonschema/py2/jsonschema/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/jsonschema/py2/jsonschema/tests/_helpers.py b/contrib/python/jsonschema/py2/jsonschema/tests/_helpers.py
new file mode 100644
index 00000000000..70f291fe2ab
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/tests/_helpers.py
@@ -0,0 +1,5 @@
+def bug(issue=None):
+    message = "A known bug."
+    if issue is not None:
+        message += " See issue #{issue}.".format(issue=issue)
+    return message
diff --git a/contrib/python/jsonschema/py2/jsonschema/tests/test_cli.py b/contrib/python/jsonschema/py2/jsonschema/tests/test_cli.py
new file mode 100644
index 00000000000..328c85106f1
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/tests/test_cli.py
@@ -0,0 +1,143 @@
+from unittest import TestCase
+import json
+import subprocess
+import sys
+
+from jsonschema import Draft4Validator, ValidationError, cli, __version__
+from jsonschema.compat import NativeIO
+from jsonschema.exceptions import SchemaError
+
+
+def fake_validator(*errors):
+    errors = list(reversed(errors))
+
+    class FakeValidator(object):
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def iter_errors(self, instance):
+            if errors:
+                return errors.pop()
+            return []
+
+        def check_schema(self, schema):
+            pass
+
+    return FakeValidator
+
+
+class TestParser(TestCase):
+
+    FakeValidator = fake_validator()
+    instance_file = "foo.json"
+    schema_file = "schema.json"
+
+    def setUp(self):
+        cli.open = self.fake_open
+        self.addCleanup(delattr, cli, "open")
+
+    def fake_open(self, path):
+        if path == self.instance_file:
+            contents = ""
+        elif path == self.schema_file:
+            contents = {}
+        else:  # pragma: no cover
+            self.fail("What is {!r}".format(path))
+        return NativeIO(json.dumps(contents))
+
+    def test_find_validator_by_fully_qualified_object_name(self):
+        arguments = cli.parse_args(
+            [
+                "--validator",
+                "__tests__.test_cli.TestParser.FakeValidator",  # XXX Arcadia
+                "--instance", self.instance_file,
+                self.schema_file,
+            ]
+        )
+        self.assertIs(arguments["validator"], self.FakeValidator)
+
+    def test_find_validator_in_jsonschema(self):
+        arguments = cli.parse_args(
+            [
+                "--validator", "Draft4Validator",
+                "--instance", self.instance_file,
+                self.schema_file,
+            ]
+        )
+        self.assertIs(arguments["validator"], Draft4Validator)
+
+
+class TestCLI(TestCase):
+    def test_draft3_schema_draft4_validator(self):
+        stdout, stderr = NativeIO(), NativeIO()
+        with self.assertRaises(SchemaError):
+            cli.run(
+                {
+                    "validator": Draft4Validator,
+                    "schema": {
+                        "anyOf": [
+                            {"minimum": 20},
+                            {"type": "string"},
+                            {"required": True},
+                        ],
+                    },
+                    "instances": [1],
+                    "error_format": "{error.message}",
+                },
+                stdout=stdout,
+                stderr=stderr,
+            )
+
+    def test_successful_validation(self):
+        stdout, stderr = NativeIO(), NativeIO()
+        exit_code = cli.run(
+            {
+                "validator": fake_validator(),
+                "schema": {},
+                "instances": [1],
+                "error_format": "{error.message}",
+            },
+            stdout=stdout,
+            stderr=stderr,
+        )
+        self.assertFalse(stdout.getvalue())
+        self.assertFalse(stderr.getvalue())
+        self.assertEqual(exit_code, 0)
+
+    def test_unsuccessful_validation(self):
+        error = ValidationError("I am an error!", instance=1)
+        stdout, stderr = NativeIO(), NativeIO()
+        exit_code = cli.run(
+            {
+                "validator": fake_validator([error]),
+                "schema": {},
+                "instances": [1],
+                "error_format": "{error.instance} - {error.message}",
+            },
+            stdout=stdout,
+            stderr=stderr,
+        )
+        self.assertFalse(stdout.getvalue())
+        self.assertEqual(stderr.getvalue(), "1 - I am an error!")
+        self.assertEqual(exit_code, 1)
+
+    def test_unsuccessful_validation_multiple_instances(self):
+        first_errors = [
+            ValidationError("9", instance=1),
+            ValidationError("8", instance=1),
+        ]
+        second_errors = [ValidationError("7", instance=2)]
+        stdout, stderr = NativeIO(), NativeIO()
+        exit_code = cli.run(
+            {
+                "validator": fake_validator(first_errors, second_errors),
+                "schema": {},
+                "instances": [1, 2],
+                "error_format": "{error.instance} - {error.message}\t",
+            },
+            stdout=stdout,
+            stderr=stderr,
+        )
+        self.assertFalse(stdout.getvalue())
+        self.assertEqual(stderr.getvalue(), "1 - 9\t1 - 8\t2 - 7\t")
+        self.assertEqual(exit_code, 1)
diff --git a/contrib/python/jsonschema/py2/jsonschema/tests/test_exceptions.py b/contrib/python/jsonschema/py2/jsonschema/tests/test_exceptions.py
new file mode 100644
index 00000000000..eae00d76d77
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/tests/test_exceptions.py
@@ -0,0 +1,462 @@
+from unittest import TestCase
+import textwrap
+
+from jsonschema import Draft4Validator, exceptions
+from jsonschema.compat import PY3
+
+
+class TestBestMatch(TestCase):
+    def best_match(self, errors):
+        errors = list(errors)
+        best = exceptions.best_match(errors)
+        reversed_best = exceptions.best_match(reversed(errors))
+        msg = "Didn't return a consistent best match!\nGot: {0}\n\nThen: {1}"
+        self.assertEqual(
+            best._contents(), reversed_best._contents(),
+            msg=msg.format(best, reversed_best),
+        )
+        return best
+
+    def test_shallower_errors_are_better_matches(self):
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "minProperties": 2,
+                        "properties": {"bar": {"type": "object"}},
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": []}}))
+        self.assertEqual(best.validator, "minProperties")
+
+    def test_oneOf_and_anyOf_are_weak_matches(self):
+        """
+        A property you *must* match is probably better than one you have to
+        match a part of.
+        """
+
+        validator = Draft4Validator(
+            {
+                "minProperties": 2,
+                "anyOf": [{"type": "string"}, {"type": "number"}],
+                "oneOf": [{"type": "string"}, {"type": "number"}],
+            }
+        )
+        best = self.best_match(validator.iter_errors({}))
+        self.assertEqual(best.validator, "minProperties")
+
+    def test_if_the_most_relevant_error_is_anyOf_it_is_traversed(self):
+        """
+        If the most relevant error is an anyOf, then we traverse its context
+        and select the otherwise *least* relevant error, since in this case
+        that means the most specific, deep, error inside the instance.
+
+        I.e. since only one of the schemas must match, we look for the most
+        relevant one.
+        """
+
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "anyOf": [
+                            {"type": "string"},
+                            {"properties": {"bar": {"type": "array"}}},
+                        ],
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": 12}}))
+        self.assertEqual(best.validator_value, "array")
+
+    def test_if_the_most_relevant_error_is_oneOf_it_is_traversed(self):
+        """
+        If the most relevant error is an oneOf, then we traverse its context
+        and select the otherwise *least* relevant error, since in this case
+        that means the most specific, deep, error inside the instance.
+
+        I.e. since only one of the schemas must match, we look for the most
+        relevant one.
+        """
+
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "oneOf": [
+                            {"type": "string"},
+                            {"properties": {"bar": {"type": "array"}}},
+                        ],
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": 12}}))
+        self.assertEqual(best.validator_value, "array")
+
+    def test_if_the_most_relevant_error_is_allOf_it_is_traversed(self):
+        """
+        Now, if the error is allOf, we traverse but select the *most* relevant
+        error from the context, because all schemas here must match anyways.
+        """
+
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "allOf": [
+                            {"type": "string"},
+                            {"properties": {"bar": {"type": "array"}}},
+                        ],
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": 12}}))
+        self.assertEqual(best.validator_value, "string")
+
+    def test_nested_context_for_oneOf(self):
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "oneOf": [
+                            {"type": "string"},
+                            {
+                                "oneOf": [
+                                    {"type": "string"},
+                                    {
+                                        "properties": {
+                                            "bar": {"type": "array"},
+                                        },
+                                    },
+                                ],
+                            },
+                        ],
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": 12}}))
+        self.assertEqual(best.validator_value, "array")
+
+    def test_one_error(self):
+        validator = Draft4Validator({"minProperties": 2})
+        error, = validator.iter_errors({})
+        self.assertEqual(
+            exceptions.best_match(validator.iter_errors({})).validator,
+            "minProperties",
+        )
+
+    def test_no_errors(self):
+        validator = Draft4Validator({})
+        self.assertIsNone(exceptions.best_match(validator.iter_errors({})))
+
+
+class TestByRelevance(TestCase):
+    def test_short_paths_are_better_matches(self):
+        shallow = exceptions.ValidationError("Oh no!", path=["baz"])
+        deep = exceptions.ValidationError("Oh yes!", path=["foo", "bar"])
+        match = max([shallow, deep], key=exceptions.relevance)
+        self.assertIs(match, shallow)
+
+        match = max([deep, shallow], key=exceptions.relevance)
+        self.assertIs(match, shallow)
+
+    def test_global_errors_are_even_better_matches(self):
+        shallow = exceptions.ValidationError("Oh no!", path=[])
+        deep = exceptions.ValidationError("Oh yes!", path=["foo"])
+
+        errors = sorted([shallow, deep], key=exceptions.relevance)
+        self.assertEqual(
+            [list(error.path) for error in errors],
+            [["foo"], []],
+        )
+
+        errors = sorted([deep, shallow], key=exceptions.relevance)
+        self.assertEqual(
+            [list(error.path) for error in errors],
+            [["foo"], []],
+        )
+
+    def test_weak_validators_are_lower_priority(self):
+        weak = exceptions.ValidationError("Oh no!", path=[], validator="a")
+        normal = exceptions.ValidationError("Oh yes!", path=[], validator="b")
+
+        best_match = exceptions.by_relevance(weak="a")
+
+        match = max([weak, normal], key=best_match)
+        self.assertIs(match, normal)
+
+        match = max([normal, weak], key=best_match)
+        self.assertIs(match, normal)
+
+    def test_strong_validators_are_higher_priority(self):
+        weak = exceptions.ValidationError("Oh no!", path=[], validator="a")
+        normal = exceptions.ValidationError("Oh yes!", path=[], validator="b")
+        strong = exceptions.ValidationError("Oh fine!", path=[], validator="c")
+
+        best_match = exceptions.by_relevance(weak="a", strong="c")
+
+        match = max([weak, normal, strong], key=best_match)
+        self.assertIs(match, strong)
+
+        match = max([strong, normal, weak], key=best_match)
+        self.assertIs(match, strong)
+
+
+class TestErrorTree(TestCase):
+    def test_it_knows_how_many_total_errors_it_contains(self):
+        # FIXME: https://github.com/Julian/jsonschema/issues/442
+        errors = [
+            exceptions.ValidationError("Something", validator=i)
+            for i in range(8)
+        ]
+        tree = exceptions.ErrorTree(errors)
+        self.assertEqual(tree.total_errors, 8)
+
+    def test_it_contains_an_item_if_the_item_had_an_error(self):
+        errors = [exceptions.ValidationError("a message", path=["bar"])]
+        tree = exceptions.ErrorTree(errors)
+        self.assertIn("bar", tree)
+
+    def test_it_does_not_contain_an_item_if_the_item_had_no_error(self):
+        errors = [exceptions.ValidationError("a message", path=["bar"])]
+        tree = exceptions.ErrorTree(errors)
+        self.assertNotIn("foo", tree)
+
+    def test_validators_that_failed_appear_in_errors_dict(self):
+        error = exceptions.ValidationError("a message", validator="foo")
+        tree = exceptions.ErrorTree([error])
+        self.assertEqual(tree.errors, {"foo": error})
+
+    def test_it_creates_a_child_tree_for_each_nested_path(self):
+        errors = [
+            exceptions.ValidationError("a bar message", path=["bar"]),
+            exceptions.ValidationError("a bar -> 0 message", path=["bar", 0]),
+        ]
+        tree = exceptions.ErrorTree(errors)
+        self.assertIn(0, tree["bar"])
+        self.assertNotIn(1, tree["bar"])
+
+    def test_children_have_their_errors_dicts_built(self):
+        e1, e2 = (
+            exceptions.ValidationError("1", validator="foo", path=["bar", 0]),
+            exceptions.ValidationError("2", validator="quux", path=["bar", 0]),
+        )
+        tree = exceptions.ErrorTree([e1, e2])
+        self.assertEqual(tree["bar"][0].errors, {"foo": e1, "quux": e2})
+
+    def test_multiple_errors_with_instance(self):
+        e1, e2 = (
+            exceptions.ValidationError(
+                "1",
+                validator="foo",
+                path=["bar", "bar2"],
+                instance="i1"),
+            exceptions.ValidationError(
+                "2",
+                validator="quux",
+                path=["foobar", 2],
+                instance="i2"),
+        )
+        exceptions.ErrorTree([e1, e2])
+
+    def test_it_does_not_contain_subtrees_that_are_not_in_the_instance(self):
+        error = exceptions.ValidationError("123", validator="foo", instance=[])
+        tree = exceptions.ErrorTree([error])
+
+        with self.assertRaises(IndexError):
+            tree[0]
+
+    def test_if_its_in_the_tree_anyhow_it_does_not_raise_an_error(self):
+        """
+        If a validator is dumb (like :validator:`required` in draft 3) and
+        refers to a path that isn't in the instance, the tree still properly
+        returns a subtree for that path.
+        """
+
+        error = exceptions.ValidationError(
+            "a message", validator="foo", instance={}, path=["foo"],
+        )
+        tree = exceptions.ErrorTree([error])
+        self.assertIsInstance(tree["foo"], exceptions.ErrorTree)
+
+
+class TestErrorInitReprStr(TestCase):
+    def make_error(self, **kwargs):
+        defaults = dict(
+            message=u"hello",
+            validator=u"type",
+            validator_value=u"string",
+            instance=5,
+            schema={u"type": u"string"},
+        )
+        defaults.update(kwargs)
+        return exceptions.ValidationError(**defaults)
+
+    def assertShows(self, expected, **kwargs):
+        if PY3:  # pragma: no cover
+            expected = expected.replace("u'", "'")
+        expected = textwrap.dedent(expected).rstrip("\n")
+
+        error = self.make_error(**kwargs)
+        message_line, _, rest = str(error).partition("\n")
+        self.assertEqual(message_line, error.message)
+        self.assertEqual(rest, expected)
+
+    def test_it_calls_super_and_sets_args(self):
+        error = self.make_error()
+        self.assertGreater(len(error.args), 1)
+
+    def test_repr(self):
+        self.assertEqual(
+            repr(exceptions.ValidationError(message="Hello!")),
+            "<ValidationError: %r>" % "Hello!",
+        )
+
+    def test_unset_error(self):
+        error = exceptions.ValidationError("message")
+        self.assertEqual(str(error), "message")
+
+        kwargs = {
+            "validator": "type",
+            "validator_value": "string",
+            "instance": 5,
+            "schema": {"type": "string"},
+        }
+        # Just the message should show if any of the attributes are unset
+        for attr in kwargs:
+            k = dict(kwargs)
+            del k[attr]
+            error = exceptions.ValidationError("message", **k)
+            self.assertEqual(str(error), "message")
+
+    def test_empty_paths(self):
+        self.assertShows(
+            """
+            Failed validating u'type' in schema:
+                {u'type': u'string'}
+
+            On instance:
+                5
+            """,
+            path=[],
+            schema_path=[],
+        )
+
+    def test_one_item_paths(self):
+        self.assertShows(
+            """
+            Failed validating u'type' in schema:
+                {u'type': u'string'}
+
+            On instance[0]:
+                5
+            """,
+            path=[0],
+            schema_path=["items"],
+        )
+
+    def test_multiple_item_paths(self):
+        self.assertShows(
+            """
+            Failed validating u'type' in schema[u'items'][0]:
+                {u'type': u'string'}
+
+            On instance[0][u'a']:
+                5
+            """,
+            path=[0, u"a"],
+            schema_path=[u"items", 0, 1],
+        )
+
+    def test_uses_pprint(self):
+        self.assertShows(
+            """
+            Failed validating u'maxLength' in schema:
+                {0: 0,
+                 1: 1,
+                 2: 2,
+                 3: 3,
+                 4: 4,
+                 5: 5,
+                 6: 6,
+                 7: 7,
+                 8: 8,
+                 9: 9,
+                 10: 10,
+                 11: 11,
+                 12: 12,
+                 13: 13,
+                 14: 14,
+                 15: 15,
+                 16: 16,
+                 17: 17,
+                 18: 18,
+                 19: 19}
+
+            On instance:
+                [0,
+                 1,
+                 2,
+                 3,
+                 4,
+                 5,
+                 6,
+                 7,
+                 8,
+                 9,
+                 10,
+                 11,
+                 12,
+                 13,
+                 14,
+                 15,
+                 16,
+                 17,
+                 18,
+                 19,
+                 20,
+                 21,
+                 22,
+                 23,
+                 24]
+            """,
+            instance=list(range(25)),
+            schema=dict(zip(range(20), range(20))),
+            validator=u"maxLength",
+        )
+
+    def test_str_works_with_instances_having_overriden_eq_operator(self):
+        """
+        Check for https://github.com/Julian/jsonschema/issues/164 which
+        rendered exceptions unusable when a `ValidationError` involved
+        instances with an `__eq__` method that returned truthy values.
+        """
+
+        class DontEQMeBro(object):
+            def __eq__(this, other):  # pragma: no cover
+                self.fail("Don't!")
+
+            def __ne__(this, other):  # pragma: no cover
+                self.fail("Don't!")
+
+        instance = DontEQMeBro()
+        error = exceptions.ValidationError(
+            "a message",
+            validator="foo",
+            instance=instance,
+            validator_value="some",
+            schema="schema",
+        )
+        self.assertIn(repr(instance), str(error))
+
+
+class TestHashable(TestCase):
+    def test_hashable(self):
+        set([exceptions.ValidationError("")])
+        set([exceptions.SchemaError("")])
diff --git a/contrib/python/jsonschema/py2/jsonschema/tests/test_format.py b/contrib/python/jsonschema/py2/jsonschema/tests/test_format.py
new file mode 100644
index 00000000000..254985f6156
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/tests/test_format.py
@@ -0,0 +1,89 @@
+"""
+Tests for the parts of jsonschema related to the :validator:`format` property.
+"""
+
+from unittest import TestCase
+
+from jsonschema import FormatError, ValidationError, FormatChecker
+from jsonschema.validators import Draft4Validator
+
+
+BOOM = ValueError("Boom!")
+BANG = ZeroDivisionError("Bang!")
+
+
+def boom(thing):
+    if thing == "bang":
+        raise BANG
+    raise BOOM
+
+
+class TestFormatChecker(TestCase):
+    def test_it_can_validate_no_formats(self):
+        checker = FormatChecker(formats=())
+        self.assertFalse(checker.checkers)
+
+    def test_it_raises_a_key_error_for_unknown_formats(self):
+        with self.assertRaises(KeyError):
+            FormatChecker(formats=["o noes"])
+
+    def test_it_can_register_cls_checkers(self):
+        original = dict(FormatChecker.checkers)
+        self.addCleanup(FormatChecker.checkers.pop, "boom")
+        FormatChecker.cls_checks("boom")(boom)
+        self.assertEqual(
+            FormatChecker.checkers,
+            dict(original, boom=(boom, ())),
+        )
+
+    def test_it_can_register_checkers(self):
+        checker = FormatChecker()
+        checker.checks("boom")(boom)
+        self.assertEqual(
+            checker.checkers,
+            dict(FormatChecker.checkers, boom=(boom, ()))
+        )
+
+    def test_it_catches_registered_errors(self):
+        checker = FormatChecker()
+        checker.checks("boom", raises=type(BOOM))(boom)
+
+        with self.assertRaises(FormatError) as cm:
+            checker.check(instance=12, format="boom")
+
+        self.assertIs(cm.exception.cause, BOOM)
+        self.assertIs(cm.exception.__cause__, BOOM)
+
+        # Unregistered errors should not be caught
+        with self.assertRaises(type(BANG)):
+            checker.check(instance="bang", format="boom")
+
+    def test_format_error_causes_become_validation_error_causes(self):
+        checker = FormatChecker()
+        checker.checks("boom", raises=ValueError)(boom)
+        validator = Draft4Validator({"format": "boom"}, format_checker=checker)
+
+        with self.assertRaises(ValidationError) as cm:
+            validator.validate("BOOM")
+
+        self.assertIs(cm.exception.cause, BOOM)
+        self.assertIs(cm.exception.__cause__, BOOM)
+
+    def test_format_checkers_come_with_defaults(self):
+        # This is bad :/ but relied upon.
+        # The docs for quite awhile recommended people do things like
+        # validate(..., format_checker=FormatChecker())
+        # We should change that, but we can't without deprecation...
+        checker = FormatChecker()
+        with self.assertRaises(FormatError):
+            checker.check(instance="not-an-ipv4", format="ipv4")
+
+    def test_repr(self):
+        checker = FormatChecker(formats=())
+        checker.checks("foo")(lambda thing: True)
+        checker.checks("bar")(lambda thing: True)
+        checker.checks("baz")(lambda thing: True)
+        self.assertEqual(
+            repr(checker),
+            "<FormatChecker checkers=['bar', 'baz', 'foo']>",
+        )
diff --git a/contrib/python/jsonschema/py2/jsonschema/tests/test_types.py b/contrib/python/jsonschema/py2/jsonschema/tests/test_types.py
new file mode 100644
index 00000000000..2280cc395b2
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/tests/test_types.py
@@ -0,0 +1,190 @@
+"""
+Tests on the new type interface. The actual correctness of the type checking
+is handled in test_jsonschema_test_suite; these tests check that TypeChecker
+functions correctly and can facilitate extensions to type checking
+"""
+from collections import namedtuple
+from unittest import TestCase
+
+from jsonschema import ValidationError, _validators
+from jsonschema._types import TypeChecker
+from jsonschema.exceptions import UndefinedTypeCheck
+from jsonschema.validators import Draft4Validator, extend
+
+
+def equals_2(checker, instance):
+    return instance == 2
+
+
+def is_namedtuple(instance):
+    return isinstance(instance, tuple) and getattr(instance, "_fields", None)
+
+
+def is_object_or_named_tuple(checker, instance):
+    if Draft4Validator.TYPE_CHECKER.is_type(instance, "object"):
+        return True
+    return is_namedtuple(instance)
+
+
+def coerce_named_tuple(fn):
+    def coerced(validator, value, instance, schema):
+        if is_namedtuple(instance):
+            instance = instance._asdict()
+        return fn(validator, value, instance, schema)
+    return coerced
+
+
+required = coerce_named_tuple(_validators.required)
+properties = coerce_named_tuple(_validators.properties)
+
+
+class TestTypeChecker(TestCase):
+    def test_is_type(self):
+        checker = TypeChecker({"two": equals_2})
+        self.assertEqual(
+            (
+                checker.is_type(instance=2, type="two"),
+                checker.is_type(instance="bar", type="two"),
+            ),
+            (True, False),
+        )
+
+    def test_is_unknown_type(self):
+        with self.assertRaises(UndefinedTypeCheck) as context:
+            TypeChecker().is_type(4, "foobar")
+        self.assertIn("foobar", str(context.exception))
+
+    def test_checks_can_be_added_at_init(self):
+        checker = TypeChecker({"two": equals_2})
+        self.assertEqual(checker, TypeChecker().redefine("two", equals_2))
+
+    def test_redefine_existing_type(self):
+        self.assertEqual(
+            TypeChecker().redefine("two", object()).redefine("two", equals_2),
+            TypeChecker().redefine("two", equals_2),
+        )
+
+    def test_remove(self):
+        self.assertEqual(
+            TypeChecker({"two": equals_2}).remove("two"),
+            TypeChecker(),
+        )
+
+    def test_remove_unknown_type(self):
+        with self.assertRaises(UndefinedTypeCheck) as context:
+            TypeChecker().remove("foobar")
+        self.assertIn("foobar", str(context.exception))
+
+    def test_redefine_many(self):
+        self.assertEqual(
+            TypeChecker().redefine_many({"foo": int, "bar": str}),
+            TypeChecker().redefine("foo", int).redefine("bar", str),
+        )
+
+    def test_remove_multiple(self):
+        self.assertEqual(
+            TypeChecker({"foo": int, "bar": str}).remove("foo", "bar"),
+            TypeChecker(),
+        )
+
+    def test_type_check_can_raise_key_error(self):
+        """
+        Make sure no one writes:
+
+            try:
+                self._type_checkers[type](...)
+            except KeyError:
+
+        ignoring the fact that the function itself can raise that.
+        """
+
+        error = KeyError("Stuff")
+
+        def raises_keyerror(checker, instance):
+            raise error
+
+        with self.assertRaises(KeyError) as context:
+            TypeChecker({"foo": raises_keyerror}).is_type(4, "foo")
+
+        self.assertIs(context.exception, error)
+
+
+class TestCustomTypes(TestCase):
+    def test_simple_type_can_be_extended(self):
+        def int_or_str_int(checker, instance):
+            if not isinstance(instance, (int, str)):
+                return False
+            try:
+                int(instance)
+            except ValueError:
+                return False
+            return True
+
+        CustomValidator = extend(
+            Draft4Validator,
+            type_checker=Draft4Validator.TYPE_CHECKER.redefine(
+                "integer", int_or_str_int,
+            ),
+        )
+        validator = CustomValidator({"type": "integer"})
+
+        validator.validate(4)
+        validator.validate("4")
+
+        with self.assertRaises(ValidationError):
+            validator.validate(4.4)
+
+    def test_object_can_be_extended(self):
+        schema = {"type": "object"}
+
+        Point = namedtuple("Point", ["x", "y"])
+
+        type_checker = Draft4Validator.TYPE_CHECKER.redefine(
+            u"object", is_object_or_named_tuple,
+        )
+
+        CustomValidator = extend(Draft4Validator, type_checker=type_checker)
+        validator = CustomValidator(schema)
+
+        validator.validate(Point(x=4, y=5))
+
+    def test_object_extensions_require_custom_validators(self):
+        schema = {"type": "object", "required": ["x"]}
+
+        type_checker = Draft4Validator.TYPE_CHECKER.redefine(
+            u"object", is_object_or_named_tuple,
+        )
+
+        CustomValidator = extend(Draft4Validator, type_checker=type_checker)
+        validator = CustomValidator(schema)
+
+        Point = namedtuple("Point", ["x", "y"])
+        # Cannot handle required
+        with self.assertRaises(ValidationError):
+            validator.validate(Point(x=4, y=5))
+
+    def test_object_extensions_can_handle_custom_validators(self):
+        schema = {
+            "type": "object",
+            "required": ["x"],
+            "properties": {"x": {"type": "integer"}},
+        }
+
+        type_checker = Draft4Validator.TYPE_CHECKER.redefine(
+            u"object", is_object_or_named_tuple,
+        )
+
+        CustomValidator = extend(
+            Draft4Validator,
+            type_checker=type_checker,
+            validators={"required": required, "properties": properties},
+        )
+
+        validator = CustomValidator(schema)
+
+        Point = namedtuple("Point", ["x", "y"])
+        # Can now process required and properties
+        validator.validate(Point(x=4, y=5))
+
+        with self.assertRaises(ValidationError):
+            validator.validate(Point(x="not an integer", y=5))
diff --git a/contrib/python/jsonschema/py2/jsonschema/tests/test_validators.py b/contrib/python/jsonschema/py2/jsonschema/tests/test_validators.py
new file mode 100644
index 00000000000..07be4f08bc2
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/tests/test_validators.py
@@ -0,0 +1,1762 @@
+from collections import deque
+from contextlib import contextmanager
+from decimal import Decimal
+from io import BytesIO
+from unittest import TestCase
+import json
+import os
+import sys
+import tempfile
+import unittest
+
+from twisted.trial.unittest import SynchronousTestCase
+import attr
+
+from jsonschema import FormatChecker, TypeChecker, exceptions, validators
+from jsonschema.compat import PY3, pathname2url
+from jsonschema.tests._helpers import bug
+
+
+def startswith(validator, startswith, instance, schema):
+    if not instance.startswith(startswith):
+        yield exceptions.ValidationError(u"Whoops!")
+
+
+class TestCreateAndExtend(SynchronousTestCase):
+    def setUp(self):
+        self.addCleanup(
+            self.assertEqual,
+            validators.meta_schemas,
+            dict(validators.meta_schemas),
+        )
+
+        self.meta_schema = {u"$id": "some://meta/schema"}
+        self.validators = {u"startswith": startswith}
+        self.type_checker = TypeChecker()
+        self.Validator = validators.create(
+            meta_schema=self.meta_schema,
+            validators=self.validators,
+            type_checker=self.type_checker,
+        )
+
+    def test_attrs(self):
+        self.assertEqual(
+            (
+                self.Validator.VALIDATORS,
+                self.Validator.META_SCHEMA,
+                self.Validator.TYPE_CHECKER,
+            ), (
+                self.validators,
+                self.meta_schema,
+                self.type_checker,
+            ),
+        )
+
+    def test_init(self):
+        schema = {u"startswith": u"foo"}
+        self.assertEqual(self.Validator(schema).schema, schema)
+
+    def test_iter_errors(self):
+        schema = {u"startswith": u"hel"}
+        iter_errors = self.Validator(schema).iter_errors
+
+        errors = list(iter_errors(u"hello"))
+        self.assertEqual(errors, [])
+
+        expected_error = exceptions.ValidationError(
+            u"Whoops!",
+            instance=u"goodbye",
+            schema=schema,
+            validator=u"startswith",
+            validator_value=u"hel",
+            schema_path=deque([u"startswith"]),
+        )
+
+        errors = list(iter_errors(u"goodbye"))
+        self.assertEqual(len(errors), 1)
+        self.assertEqual(errors[0]._contents(), expected_error._contents())
+
+    def test_if_a_version_is_provided_it_is_registered(self):
+        Validator = validators.create(
+            meta_schema={u"$id": "something"},
+            version="my version",
+        )
+        self.addCleanup(validators.meta_schemas.pop, "something")
+        self.assertEqual(Validator.__name__, "MyVersionValidator")
+
+    def test_if_a_version_is_not_provided_it_is_not_registered(self):
+        original = dict(validators.meta_schemas)
+        validators.create(meta_schema={u"id": "id"})
+        self.assertEqual(validators.meta_schemas, original)
+
+    def test_validates_registers_meta_schema_id(self):
+        meta_schema_key = "meta schema id"
+        my_meta_schema = {u"id": meta_schema_key}
+
+        validators.create(
+            meta_schema=my_meta_schema,
+            version="my version",
+            id_of=lambda s: s.get("id", ""),
+        )
+        self.addCleanup(validators.meta_schemas.pop, meta_schema_key)
+
+        self.assertIn(meta_schema_key, validators.meta_schemas)
+
+    def test_validates_registers_meta_schema_draft6_id(self):
+        meta_schema_key = "meta schema $id"
+        my_meta_schema = {u"$id": meta_schema_key}
+
+        validators.create(
+            meta_schema=my_meta_schema,
+            version="my version",
+        )
+        self.addCleanup(validators.meta_schemas.pop, meta_schema_key)
+
+        self.assertIn(meta_schema_key, validators.meta_schemas)
+
+    def test_create_default_types(self):
+        Validator = validators.create(meta_schema={}, validators=())
+        self.assertTrue(
+            all(
+                Validator({}).is_type(instance=instance, type=type)
+                for type, instance in [
+                    (u"array", []),
+                    (u"boolean", True),
+                    (u"integer", 12),
+                    (u"null", None),
+                    (u"number", 12.0),
+                    (u"object", {}),
+                    (u"string", u"foo"),
+                ]
+            ),
+        )
+
+    def test_extend(self):
+        original = dict(self.Validator.VALIDATORS)
+        new = object()
+
+        Extended = validators.extend(
+            self.Validator,
+            validators={u"new": new},
+        )
+        self.assertEqual(
+            (
+                Extended.VALIDATORS,
+                Extended.META_SCHEMA,
+                Extended.TYPE_CHECKER,
+                self.Validator.VALIDATORS,
+            ), (
+                dict(original, new=new),
+                self.Validator.META_SCHEMA,
+                self.Validator.TYPE_CHECKER,
+                original,
+            ),
+        )
+
+    def test_extend_idof(self):
+        """
+        Extending a validator preserves its notion of schema IDs.
+        """
+        def id_of(schema):
+            return schema.get(u"__test__", self.Validator.ID_OF(schema))
+        correct_id = "the://correct/id/"
+        meta_schema = {
+            u"$id": "the://wrong/id/",
+            u"__test__": correct_id,
+        }
+        Original = validators.create(
+            meta_schema=meta_schema,
+            validators=self.validators,
+            type_checker=self.type_checker,
+            id_of=id_of,
+        )
+        self.assertEqual(Original.ID_OF(Original.META_SCHEMA), correct_id)
+
+        Derived = validators.extend(Original)
+        self.assertEqual(Derived.ID_OF(Derived.META_SCHEMA), correct_id)
+
+
+class TestLegacyTypeChecking(SynchronousTestCase):
+    def test_create_default_types(self):
+        Validator = validators.create(meta_schema={}, validators=())
+        self.assertEqual(
+            set(Validator.DEFAULT_TYPES), {
+                u"array",
+                u"boolean",
+                u"integer",
+                u"null",
+                u"number",
+                u"object", u"string",
+            },
+        )
+        self.flushWarnings()
+
+    def test_extend(self):
+        Validator = validators.create(meta_schema={}, validators=())
+        original = dict(Validator.VALIDATORS)
+        new = object()
+
+        Extended = validators.extend(
+            Validator,
+            validators={u"new": new},
+        )
+        self.assertEqual(
+            (
+                Extended.VALIDATORS,
+                Extended.META_SCHEMA,
+                Extended.TYPE_CHECKER,
+                Validator.VALIDATORS,
+
+                Extended.DEFAULT_TYPES,
+                Extended({}).DEFAULT_TYPES,
+                self.flushWarnings()[0]["message"],
+            ), (
+                dict(original, new=new),
+                Validator.META_SCHEMA,
+                Validator.TYPE_CHECKER,
+                original,
+
+                Validator.DEFAULT_TYPES,
+                Validator.DEFAULT_TYPES,
+                self.flushWarnings()[0]["message"],
+            ),
+        )
+
+    def test_types_redefines_the_validators_type_checker(self):
+        schema = {"type": "string"}
+        self.assertFalse(validators.Draft7Validator(schema).is_valid(12))
+
+        validator = validators.Draft7Validator(
+            schema,
+            types={"string": (str, int)},
+        )
+        self.assertTrue(validator.is_valid(12))
+        self.flushWarnings()
+
+    def test_providing_default_types_warns(self):
+        self.assertWarns(
+            category=DeprecationWarning,
+            message=(
+                "The default_types argument is deprecated. "
+                "Use the type_checker argument instead."
+            ),
+            # https://tm.tl/9363 :'(
+            filename=sys.modules[self.assertWarns.__module__].__file__,
+
+            f=validators.create,
+            meta_schema={},
+            validators={},
+            default_types={"foo": object},
+        )
+
+    def test_cannot_ask_for_default_types_with_non_default_type_checker(self):
+        """
+        We raise an error when you ask a validator with non-default
+        type checker for its DEFAULT_TYPES.
+
+        The type checker argument is new, so no one but this library
+        itself should be trying to use it, and doing so while then
+        asking for DEFAULT_TYPES makes no sense (not to mention is
+        deprecated), since type checkers are not strictly about Python
+        type.
+        """
+        Validator = validators.create(
+            meta_schema={},
+            validators={},
+            type_checker=TypeChecker(),
+        )
+        with self.assertRaises(validators._DontDoThat) as e:
+            Validator.DEFAULT_TYPES
+
+        self.assertIn(
+            "DEFAULT_TYPES cannot be used on Validators using TypeCheckers",
+            str(e.exception),
+        )
+        with self.assertRaises(validators._DontDoThat):
+            Validator({}).DEFAULT_TYPES
+
+        self.assertFalse(self.flushWarnings())
+
+    def test_providing_explicit_type_checker_does_not_warn(self):
+        Validator = validators.create(
+            meta_schema={},
+            validators={},
+            type_checker=TypeChecker(),
+        )
+        self.assertFalse(self.flushWarnings())
+
+        Validator({})
+        self.assertFalse(self.flushWarnings())
+
+    def test_providing_neither_does_not_warn(self):
+        Validator = validators.create(meta_schema={}, validators={})
+        self.assertFalse(self.flushWarnings())
+
+        Validator({})
+        self.assertFalse(self.flushWarnings())
+
+    def test_providing_default_types_with_type_checker_errors(self):
+        with self.assertRaises(TypeError) as e:
+            validators.create(
+                meta_schema={},
+                validators={},
+                default_types={"foo": object},
+                type_checker=TypeChecker(),
+            )
+
+        self.assertIn(
+            "Do not specify default_types when providing a type checker",
+            str(e.exception),
+        )
+        self.assertFalse(self.flushWarnings())
+
+    def test_extending_a_legacy_validator_with_a_type_checker_errors(self):
+        Validator = validators.create(
+            meta_schema={},
+            validators={},
+            default_types={u"array": list}
+        )
+        with self.assertRaises(TypeError) as e:
+            validators.extend(
+                Validator,
+                validators={},
+                type_checker=TypeChecker(),
+            )
+
+        self.assertIn(
+            (
+                "Cannot extend a validator created with default_types "
+                "with a type_checker. Update the validator to use a "
+                "type_checker when created."
+            ),
+            str(e.exception),
+        )
+        self.flushWarnings()
+
+    def test_extending_a_legacy_validator_does_not_rewarn(self):
+        Validator = validators.create(meta_schema={}, default_types={})
+        self.assertTrue(self.flushWarnings())
+
+        validators.extend(Validator)
+        self.assertFalse(self.flushWarnings())
+
+    def test_accessing_default_types_warns(self):
+        Validator = validators.create(meta_schema={}, validators={})
+        self.assertFalse(self.flushWarnings())
+
+        self.assertWarns(
+            DeprecationWarning,
+            (
+                "The DEFAULT_TYPES attribute is deprecated. "
+                "See the type checker attached to this validator instead."
+            ),
+            # https://tm.tl/9363 :'(
+            sys.modules[self.assertWarns.__module__].__file__,
+
+            getattr,
+            Validator,
+            "DEFAULT_TYPES",
+        )
+
+    def test_accessing_default_types_on_the_instance_warns(self):
+        Validator = validators.create(meta_schema={}, validators={})
+        self.assertFalse(self.flushWarnings())
+
+        self.assertWarns(
+            DeprecationWarning,
+            (
+                "The DEFAULT_TYPES attribute is deprecated. "
+                "See the type checker attached to this validator instead."
+            ),
+            # https://tm.tl/9363 :'(
+            sys.modules[self.assertWarns.__module__].__file__,
+
+            getattr,
+            Validator({}),
+            "DEFAULT_TYPES",
+        )
+
+    def test_providing_types_to_init_warns(self):
+        Validator = validators.create(meta_schema={}, validators={})
+        self.assertFalse(self.flushWarnings())
+
+        self.assertWarns(
+            category=DeprecationWarning,
+            message=(
+                "The types argument is deprecated. "
+                "Provide a type_checker to jsonschema.validators.extend "
+                "instead."
+            ),
+            # https://tm.tl/9363 :'(
+            filename=sys.modules[self.assertWarns.__module__].__file__,
+
+            f=Validator,
+            schema={},
+            types={"bar": object},
+        )
+
+
+class TestIterErrors(TestCase):
+    def setUp(self):
+        self.validator = validators.Draft3Validator({})
+
+    def test_iter_errors(self):
+        instance = [1, 2]
+        schema = {
+            u"disallow": u"array",
+            u"enum": [["a", "b", "c"], ["d", "e", "f"]],
+            u"minItems": 3,
+        }
+
+        got = (e.message for e in self.validator.iter_errors(instance, schema))
+        expected = [
+            "%r is disallowed for [1, 2]" % (schema["disallow"],),
+            "[1, 2] is too short",
+            "[1, 2] is not one of %r" % (schema["enum"],),
+        ]
+        self.assertEqual(sorted(got), sorted(expected))
+
+    def test_iter_errors_multiple_failures_one_validator(self):
+        instance = {"foo": 2, "bar": [1], "baz": 15, "quux": "spam"}
+        schema = {
+            u"properties": {
+                "foo": {u"type": "string"},
+                "bar": {u"minItems": 2},
+                "baz": {u"maximum": 10, u"enum": [2, 4, 6, 8]},
+            },
+        }
+
+        errors = list(self.validator.iter_errors(instance, schema))
+        self.assertEqual(len(errors), 4)
+
+
+class TestValidationErrorMessages(TestCase):
+    def message_for(self, instance, schema, *args, **kwargs):
+        kwargs.setdefault("cls", validators.Draft3Validator)
+        with self.assertRaises(exceptions.ValidationError) as e:
+            validators.validate(instance, schema, *args, **kwargs)
+        return e.exception.message
+
+    def test_single_type_failure(self):
+        message = self.message_for(instance=1, schema={u"type": u"string"})
+        self.assertEqual(message, "1 is not of type %r" % u"string")
+
+    def test_single_type_list_failure(self):
+        message = self.message_for(instance=1, schema={u"type": [u"string"]})
+        self.assertEqual(message, "1 is not of type %r" % u"string")
+
+    def test_multiple_type_failure(self):
+        types = u"string", u"object"
+        message = self.message_for(instance=1, schema={u"type": list(types)})
+        self.assertEqual(message, "1 is not of type %r, %r" % types)
+
+    def test_object_without_title_type_failure(self):
+        type = {u"type": [{u"minimum": 3}]}
+        message = self.message_for(instance=1, schema={u"type": [type]})
+        self.assertEqual(message, "1 is less than the minimum of 3")
+
+    def test_object_with_named_type_failure(self):
+        schema = {u"type": [{u"name": "Foo", u"minimum": 3}]}
+        message = self.message_for(instance=1, schema=schema)
+        self.assertEqual(message, "1 is less than the minimum of 3")
+
+    def test_minimum(self):
+        message = self.message_for(instance=1, schema={"minimum": 2})
+        self.assertEqual(message, "1 is less than the minimum of 2")
+
+    def test_maximum(self):
+        message = self.message_for(instance=1, schema={"maximum": 0})
+        self.assertEqual(message, "1 is greater than the maximum of 0")
+
+    def test_dependencies_single_element(self):
+        depend, on = "bar", "foo"
+        schema = {u"dependencies": {depend: on}}
+        message = self.message_for(
+            instance={"bar": 2},
+            schema=schema,
+            cls=validators.Draft3Validator,
+        )
+        self.assertEqual(message, "%r is a dependency of %r" % (on, depend))
+
+    def test_dependencies_list_draft3(self):
+        depend, on = "bar", "foo"
+        schema = {u"dependencies": {depend: [on]}}
+        message = self.message_for(
+            instance={"bar": 2},
+            schema=schema,
+            cls=validators.Draft3Validator,
+        )
+        self.assertEqual(message, "%r is a dependency of %r" % (on, depend))
+
+    def test_dependencies_list_draft7(self):
+        depend, on = "bar", "foo"
+        schema = {u"dependencies": {depend: [on]}}
+        message = self.message_for(
+            instance={"bar": 2},
+            schema=schema,
+            cls=validators.Draft7Validator,
+        )
+        self.assertEqual(message, "%r is a dependency of %r" % (on, depend))
+
+    def test_additionalItems_single_failure(self):
+        message = self.message_for(
+            instance=[2],
+            schema={u"items": [], u"additionalItems": False},
+        )
+        self.assertIn("(2 was unexpected)", message)
+
+    def test_additionalItems_multiple_failures(self):
+        message = self.message_for(
+            instance=[1, 2, 3],
+            schema={u"items": [], u"additionalItems": False}
+        )
+        self.assertIn("(1, 2, 3 were unexpected)", message)
+
+    def test_additionalProperties_single_failure(self):
+        additional = "foo"
+        schema = {u"additionalProperties": False}
+        message = self.message_for(instance={additional: 2}, schema=schema)
+        self.assertIn("(%r was unexpected)" % (additional,), message)
+
+    def test_additionalProperties_multiple_failures(self):
+        schema = {u"additionalProperties": False}
+        message = self.message_for(
+            instance=dict.fromkeys(["foo", "bar"]),
+            schema=schema,
+        )
+
+        self.assertIn(repr("foo"), message)
+        self.assertIn(repr("bar"), message)
+        self.assertIn("were unexpected)", message)
+
+    def test_const(self):
+        schema = {u"const": 12}
+        message = self.message_for(
+            instance={"foo": "bar"},
+            schema=schema,
+            cls=validators.Draft6Validator,
+        )
+        self.assertIn("12 was expected", message)
+
+    def test_contains(self):
+        schema = {u"contains": {u"const": 12}}
+        message = self.message_for(
+            instance=[2, {}, []],
+            schema=schema,
+            cls=validators.Draft6Validator,
+        )
+        self.assertIn(
+            "None of [2, {}, []] are valid under the given schema",
+            message,
+        )
+
+    def test_invalid_format_default_message(self):
+        checker = FormatChecker(formats=())
+        checker.checks(u"thing")(lambda value: False)
+
+        schema = {u"format": u"thing"}
+        message = self.message_for(
+            instance="bla",
+            schema=schema,
+            format_checker=checker,
+        )
+
+        self.assertIn(repr("bla"), message)
+        self.assertIn(repr("thing"), message)
+        self.assertIn("is not a", message)
+
+    def test_additionalProperties_false_patternProperties(self):
+        schema = {u"type": u"object",
+                  u"additionalProperties": False,
+                  u"patternProperties": {
+                      u"^abc$": {u"type": u"string"},
+                      u"^def$": {u"type": u"string"},
+                  }}
+        message = self.message_for(
+            instance={u"zebra": 123},
+            schema=schema,
+            cls=validators.Draft4Validator,
+        )
+        self.assertEqual(
+            message,
+            "{} does not match any of the regexes: {}, {}".format(
+                repr(u"zebra"), repr(u"^abc$"), repr(u"^def$"),
+            ),
+        )
+        message = self.message_for(
+            instance={u"zebra": 123, u"fish": 456},
+            schema=schema,
+            cls=validators.Draft4Validator,
+        )
+        self.assertEqual(
+            message,
+            "{}, {} do not match any of the regexes: {}, {}".format(
+                repr(u"fish"), repr(u"zebra"), repr(u"^abc$"), repr(u"^def$")
+            ),
+        )
+
+    def test_False_schema(self):
+        message = self.message_for(
+            instance="something",
+            schema=False,
+            cls=validators.Draft7Validator,
+        )
+        self.assertIn("False schema does not allow 'something'", message)
+
+
+class TestValidationErrorDetails(TestCase):
+    # TODO: These really need unit tests for each individual validator, rather
+    #       than just these higher level tests.
+    def test_anyOf(self):
+        instance = 5
+        schema = {
+            "anyOf": [
+                {"minimum": 20},
+                {"type": "string"},
+            ],
+        }
+
+        validator = validators.Draft4Validator(schema)
+        errors = list(validator.iter_errors(instance))
+        self.assertEqual(len(errors), 1)
+        e = errors[0]
+
+        self.assertEqual(e.validator, "anyOf")
+        self.assertEqual(e.validator_value, schema["anyOf"])
+        self.assertEqual(e.instance, instance)
+        self.assertEqual(e.schema, schema)
+        self.assertIsNone(e.parent)
+
+        self.assertEqual(e.path, deque([]))
+        self.assertEqual(e.relative_path, deque([]))
+        self.assertEqual(e.absolute_path, deque([]))
+
+        self.assertEqual(e.schema_path, deque(["anyOf"]))
+        self.assertEqual(e.relative_schema_path, deque(["anyOf"]))
+        self.assertEqual(e.absolute_schema_path, deque(["anyOf"]))
+
+        self.assertEqual(len(e.context), 2)
+
+        e1, e2 = sorted_errors(e.context)
+
+        self.assertEqual(e1.validator, "minimum")
+        self.assertEqual(e1.validator_value, schema["anyOf"][0]["minimum"])
+        self.assertEqual(e1.instance, instance)
+        self.assertEqual(e1.schema, schema["anyOf"][0])
+        self.assertIs(e1.parent, e)
+
+        self.assertEqual(e1.path, deque([]))
+        self.assertEqual(e1.absolute_path, deque([]))
+        self.assertEqual(e1.relative_path, deque([]))
+
+        self.assertEqual(e1.schema_path, deque([0, "minimum"]))
+        self.assertEqual(e1.relative_schema_path, deque([0, "minimum"]))
+        self.assertEqual(
+            e1.absolute_schema_path, deque(["anyOf", 0, "minimum"]),
+        )
+
+        self.assertFalse(e1.context)
+
+        self.assertEqual(e2.validator, "type")
+        self.assertEqual(e2.validator_value, schema["anyOf"][1]["type"])
+        self.assertEqual(e2.instance, instance)
+        self.assertEqual(e2.schema, schema["anyOf"][1])
+        self.assertIs(e2.parent, e)
+
+        self.assertEqual(e2.path, deque([]))
+        self.assertEqual(e2.relative_path, deque([]))
+        self.assertEqual(e2.absolute_path, deque([]))
+
+        self.assertEqual(e2.schema_path, deque([1, "type"]))
+        self.assertEqual(e2.relative_schema_path, deque([1, "type"]))
+        self.assertEqual(e2.absolute_schema_path, deque(["anyOf", 1, "type"]))
+
+        self.assertEqual(len(e2.context), 0)
+
+    def test_type(self):
+        instance = {"foo": 1}
+        schema = {
+            "type": [
+                {"type": "integer"},
+                {
+                    "type": "object",
+                    "properties": {"foo": {"enum": [2]}},
+                },
+            ],
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = list(validator.iter_errors(instance))
+        self.assertEqual(len(errors), 1)
+        e = errors[0]
+
+        self.assertEqual(e.validator, "type")
+        self.assertEqual(e.validator_value, schema["type"])
+        self.assertEqual(e.instance, instance)
+        self.assertEqual(e.schema, schema)
+        self.assertIsNone(e.parent)
+
+        self.assertEqual(e.path, deque([]))
+        self.assertEqual(e.relative_path, deque([]))
+        self.assertEqual(e.absolute_path, deque([]))
+
+        self.assertEqual(e.schema_path, deque(["type"]))
+        self.assertEqual(e.relative_schema_path, deque(["type"]))
+        self.assertEqual(e.absolute_schema_path, deque(["type"]))
+
+        self.assertEqual(len(e.context), 2)
+
+        e1, e2 = sorted_errors(e.context)
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e1.validator_value, schema["type"][0]["type"])
+        self.assertEqual(e1.instance, instance)
+        self.assertEqual(e1.schema, schema["type"][0])
+        self.assertIs(e1.parent, e)
+
+        self.assertEqual(e1.path, deque([]))
+        self.assertEqual(e1.relative_path, deque([]))
+        self.assertEqual(e1.absolute_path, deque([]))
+
+        self.assertEqual(e1.schema_path, deque([0, "type"]))
+        self.assertEqual(e1.relative_schema_path, deque([0, "type"]))
+        self.assertEqual(e1.absolute_schema_path, deque(["type", 0, "type"]))
+
+        self.assertFalse(e1.context)
+
+        self.assertEqual(e2.validator, "enum")
+        self.assertEqual(e2.validator_value, [2])
+        self.assertEqual(e2.instance, 1)
+        self.assertEqual(e2.schema, {u"enum": [2]})
+        self.assertIs(e2.parent, e)
+
+        self.assertEqual(e2.path, deque(["foo"]))
+        self.assertEqual(e2.relative_path, deque(["foo"]))
+        self.assertEqual(e2.absolute_path, deque(["foo"]))
+
+        self.assertEqual(
+            e2.schema_path, deque([1, "properties", "foo", "enum"]),
+        )
+        self.assertEqual(
+            e2.relative_schema_path, deque([1, "properties", "foo", "enum"]),
+        )
+        self.assertEqual(
+            e2.absolute_schema_path,
+            deque(["type", 1, "properties", "foo", "enum"]),
+        )
+
+        self.assertFalse(e2.context)
+
+    def test_single_nesting(self):
+        instance = {"foo": 2, "bar": [1], "baz": 15, "quux": "spam"}
+        schema = {
+            "properties": {
+                "foo": {"type": "string"},
+                "bar": {"minItems": 2},
+                "baz": {"maximum": 10, "enum": [2, 4, 6, 8]},
+            },
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2, e3, e4 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque(["bar"]))
+        self.assertEqual(e2.path, deque(["baz"]))
+        self.assertEqual(e3.path, deque(["baz"]))
+        self.assertEqual(e4.path, deque(["foo"]))
+
+        self.assertEqual(e1.relative_path, deque(["bar"]))
+        self.assertEqual(e2.relative_path, deque(["baz"]))
+        self.assertEqual(e3.relative_path, deque(["baz"]))
+        self.assertEqual(e4.relative_path, deque(["foo"]))
+
+        self.assertEqual(e1.absolute_path, deque(["bar"]))
+        self.assertEqual(e2.absolute_path, deque(["baz"]))
+        self.assertEqual(e3.absolute_path, deque(["baz"]))
+        self.assertEqual(e4.absolute_path, deque(["foo"]))
+
+        self.assertEqual(e1.validator, "minItems")
+        self.assertEqual(e2.validator, "enum")
+        self.assertEqual(e3.validator, "maximum")
+        self.assertEqual(e4.validator, "type")
+
+    def test_multiple_nesting(self):
+        instance = [1, {"foo": 2, "bar": {"baz": [1]}}, "quux"]
+        schema = {
+            "type": "string",
+            "items": {
+                "type": ["string", "object"],
+                "properties": {
+                    "foo": {"enum": [1, 3]},
+                    "bar": {
+                        "type": "array",
+                        "properties": {
+                            "bar": {"required": True},
+                            "baz": {"minItems": 2},
+                        },
+                    },
+                },
+            },
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2, e3, e4, e5, e6 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque([]))
+        self.assertEqual(e2.path, deque([0]))
+        self.assertEqual(e3.path, deque([1, "bar"]))
+        self.assertEqual(e4.path, deque([1, "bar", "bar"]))
+        self.assertEqual(e5.path, deque([1, "bar", "baz"]))
+        self.assertEqual(e6.path, deque([1, "foo"]))
+
+        self.assertEqual(e1.schema_path, deque(["type"]))
+        self.assertEqual(e2.schema_path, deque(["items", "type"]))
+        self.assertEqual(
+            list(e3.schema_path), ["items", "properties", "bar", "type"],
+        )
+        self.assertEqual(
+            list(e4.schema_path),
+            ["items", "properties", "bar", "properties", "bar", "required"],
+        )
+        self.assertEqual(
+            list(e5.schema_path),
+            ["items", "properties", "bar", "properties", "baz", "minItems"]
+        )
+        self.assertEqual(
+            list(e6.schema_path), ["items", "properties", "foo", "enum"],
+        )
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "type")
+        self.assertEqual(e3.validator, "type")
+        self.assertEqual(e4.validator, "required")
+        self.assertEqual(e5.validator, "minItems")
+        self.assertEqual(e6.validator, "enum")
+
+    def test_recursive(self):
+        schema = {
+            "definitions": {
+                "node": {
+                    "anyOf": [{
+                        "type": "object",
+                        "required": ["name", "children"],
+                        "properties": {
+                            "name": {
+                                "type": "string",
+                            },
+                            "children": {
+                                "type": "object",
+                                "patternProperties": {
+                                    "^.*$": {
+                                        "$ref": "#/definitions/node",
+                                    },
+                                },
+                            },
+                        },
+                    }],
+                },
+            },
+            "type": "object",
+            "required": ["root"],
+            "properties": {"root": {"$ref": "#/definitions/node"}},
+        }
+
+        instance = {
+            "root": {
+                "name": "root",
+                "children": {
+                    "a": {
+                        "name": "a",
+                        "children": {
+                            "ab": {
+                                "name": "ab",
+                                # missing "children"
+                            },
+                        },
+                    },
+                },
+            },
+        }
+        validator = validators.Draft4Validator(schema)
+
+        e, = validator.iter_errors(instance)
+        self.assertEqual(e.absolute_path, deque(["root"]))
+        self.assertEqual(
+            e.absolute_schema_path, deque(["properties", "root", "anyOf"]),
+        )
+
+        e1, = e.context
+        self.assertEqual(e1.absolute_path, deque(["root", "children", "a"]))
+        self.assertEqual(
+            e1.absolute_schema_path, deque(
+                [
+                    "properties",
+                    "root",
+                    "anyOf",
+                    0,
+                    "properties",
+                    "children",
+                    "patternProperties",
+                    "^.*$",
+                    "anyOf",
+                ],
+            ),
+        )
+
+        e2, = e1.context
+        self.assertEqual(
+            e2.absolute_path, deque(
+                ["root", "children", "a", "children", "ab"],
+            ),
+        )
+        self.assertEqual(
+            e2.absolute_schema_path, deque(
+                [
+                    "properties",
+                    "root",
+                    "anyOf",
+                    0,
+                    "properties",
+                    "children",
+                    "patternProperties",
+                    "^.*$",
+                    "anyOf",
+                    0,
+                    "properties",
+                    "children",
+                    "patternProperties",
+                    "^.*$",
+                    "anyOf",
+                ],
+            ),
+        )
+
+    def test_additionalProperties(self):
+        instance = {"bar": "bar", "foo": 2}
+        schema = {"additionalProperties": {"type": "integer", "minimum": 5}}
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque(["bar"]))
+        self.assertEqual(e2.path, deque(["foo"]))
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "minimum")
+
+    def test_patternProperties(self):
+        instance = {"bar": 1, "foo": 2}
+        schema = {
+            "patternProperties": {
+                "bar": {"type": "string"},
+                "foo": {"minimum": 5},
+            },
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque(["bar"]))
+        self.assertEqual(e2.path, deque(["foo"]))
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "minimum")
+
+    def test_additionalItems(self):
+        instance = ["foo", 1]
+        schema = {
+            "items": [],
+            "additionalItems": {"type": "integer", "minimum": 5},
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque([0]))
+        self.assertEqual(e2.path, deque([1]))
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "minimum")
+
+    def test_additionalItems_with_items(self):
+        instance = ["foo", "bar", 1]
+        schema = {
+            "items": [{}],
+            "additionalItems": {"type": "integer", "minimum": 5},
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque([1]))
+        self.assertEqual(e2.path, deque([2]))
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "minimum")
+
+    def test_propertyNames(self):
+        instance = {"foo": 12}
+        schema = {"propertyNames": {"not": {"const": "foo"}}}
+
+        validator = validators.Draft7Validator(schema)
+        error, = validator.iter_errors(instance)
+
+        self.assertEqual(error.validator, "not")
+        self.assertEqual(
+            error.message,
+            "%r is not allowed for %r" % ({"const": "foo"}, "foo"),
+        )
+        self.assertEqual(error.path, deque([]))
+        self.assertEqual(error.schema_path, deque(["propertyNames", "not"]))
+
+    def test_if_then(self):
+        schema = {
+            "if": {"const": 12},
+            "then": {"const": 13},
+        }
+
+        validator = validators.Draft7Validator(schema)
+        error, = validator.iter_errors(12)
+
+        self.assertEqual(error.validator, "const")
+        self.assertEqual(error.message, "13 was expected")
+        self.assertEqual(error.path, deque([]))
+        self.assertEqual(error.schema_path, deque(["if", "then", "const"]))
+
+    def test_if_else(self):
+        schema = {
+            "if": {"const": 12},
+            "else": {"const": 13},
+        }
+
+        validator = validators.Draft7Validator(schema)
+        error, = validator.iter_errors(15)
+
+        self.assertEqual(error.validator, "const")
+        self.assertEqual(error.message, "13 was expected")
+        self.assertEqual(error.path, deque([]))
+        self.assertEqual(error.schema_path, deque(["if", "else", "const"]))
+
+    def test_boolean_schema_False(self):
+        validator = validators.Draft7Validator(False)
+        error, = validator.iter_errors(12)
+
+        self.assertEqual(
+            (
+                error.message,
+                error.validator,
+                error.validator_value,
+                error.instance,
+                error.schema,
+                error.schema_path,
+            ),
+            (
+                "False schema does not allow 12",
+                None,
+                None,
+                12,
+                False,
+                deque([]),
+            ),
+        )
+
+    def test_ref(self):
+        ref, schema = "someRef", {"additionalProperties": {"type": "integer"}}
+        validator = validators.Draft7Validator(
+            {"$ref": ref},
+            resolver=validators.RefResolver("", {}, store={ref: schema}),
+        )
+        error, = validator.iter_errors({"foo": "notAnInteger"})
+
+        self.assertEqual(
+            (
+                error.message,
+                error.validator,
+                error.validator_value,
+                error.instance,
+                error.absolute_path,
+                error.schema,
+                error.schema_path,
+            ),
+            (
+                "'notAnInteger' is not of type 'integer'",
+                "type",
+                "integer",
+                "notAnInteger",
+                deque(["foo"]),
+                {"type": "integer"},
+                deque(["additionalProperties", "type"]),
+            ),
+        )
+
+
+class MetaSchemaTestsMixin(object):
+    # TODO: These all belong upstream
+    def test_invalid_properties(self):
+        with self.assertRaises(exceptions.SchemaError):
+            self.Validator.check_schema({"properties": {"test": object()}})
+
+    def test_minItems_invalid_string(self):
+        with self.assertRaises(exceptions.SchemaError):
+            # needs to be an integer
+            self.Validator.check_schema({"minItems": "1"})
+
+    def test_enum_allows_empty_arrays(self):
+        """
+        Technically, all the spec says is they SHOULD have elements, not MUST.
+
+        See https://github.com/Julian/jsonschema/issues/529.
+        """
+        self.Validator.check_schema({"enum": []})
+
+    def test_enum_allows_non_unique_items(self):
+        """
+        Technically, all the spec says is they SHOULD be unique, not MUST.
+
+        See https://github.com/Julian/jsonschema/issues/529.
+        """
+        self.Validator.check_schema({"enum": [12, 12]})
+
+
+class ValidatorTestMixin(MetaSchemaTestsMixin, object):
+    def test_valid_instances_are_valid(self):
+        schema, instance = self.valid
+        self.assertTrue(self.Validator(schema).is_valid(instance))
+
+    def test_invalid_instances_are_not_valid(self):
+        schema, instance = self.invalid
+        self.assertFalse(self.Validator(schema).is_valid(instance))
+
+    def test_non_existent_properties_are_ignored(self):
+        self.Validator({object(): object()}).validate(instance=object())
+
+    def test_it_creates_a_ref_resolver_if_not_provided(self):
+        self.assertIsInstance(
+            self.Validator({}).resolver,
+            validators.RefResolver,
+        )
+
+    def test_it_delegates_to_a_ref_resolver(self):
+        ref, schema = "someCoolRef", {"type": "integer"}
+        resolver = validators.RefResolver("", {}, store={ref: schema})
+        validator = self.Validator({"$ref": ref}, resolver=resolver)
+
+        with self.assertRaises(exceptions.ValidationError):
+            validator.validate(None)
+
+    def test_it_delegates_to_a_legacy_ref_resolver(self):
+        """
+        Legacy RefResolvers support only the context manager form of
+        resolution.
+        """
+
+        class LegacyRefResolver(object):
+            @contextmanager
+            def resolving(this, ref):
+                self.assertEqual(ref, "the ref")
+                yield {"type": "integer"}
+
+        resolver = LegacyRefResolver()
+        schema = {"$ref": "the ref"}
+
+        with self.assertRaises(exceptions.ValidationError):
+            self.Validator(schema, resolver=resolver).validate(None)
+
+    def test_is_type_is_true_for_valid_type(self):
+        self.assertTrue(self.Validator({}).is_type("foo", "string"))
+
+    def test_is_type_is_false_for_invalid_type(self):
+        self.assertFalse(self.Validator({}).is_type("foo", "array"))
+
+    def test_is_type_evades_bool_inheriting_from_int(self):
+        self.assertFalse(self.Validator({}).is_type(True, "integer"))
+        self.assertFalse(self.Validator({}).is_type(True, "number"))
+
+    @unittest.skipIf(PY3, "In Python 3 json.load always produces unicode")
+    def test_string_a_bytestring_is_a_string(self):
+        self.Validator({"type": "string"}).validate(b"foo")
+
+    def test_patterns_can_be_native_strings(self):
+        """
+        See https://github.com/Julian/jsonschema/issues/611.
+        """
+        self.Validator({"pattern": "foo"}).validate("foo")
+
+    def test_it_can_validate_with_decimals(self):
+        schema = {"items": {"type": "number"}}
+        Validator = validators.extend(
+            self.Validator,
+            type_checker=self.Validator.TYPE_CHECKER.redefine(
+                "number",
+                lambda checker, thing: isinstance(
+                    thing, (int, float, Decimal),
+                ) and not isinstance(thing, bool),
+            )
+        )
+
+        validator = Validator(schema)
+        validator.validate([1, 1.1, Decimal(1) / Decimal(8)])
+
+        invalid = ["foo", {}, [], True, None]
+        self.assertEqual(
+            [error.instance for error in validator.iter_errors(invalid)],
+            invalid,
+        )
+
+    def test_it_returns_true_for_formats_it_does_not_know_about(self):
+        validator = self.Validator(
+            {"format": "carrot"}, format_checker=FormatChecker(),
+        )
+        validator.validate("bugs")
+
+    def test_it_does_not_validate_formats_by_default(self):
+        validator = self.Validator({})
+        self.assertIsNone(validator.format_checker)
+
+    def test_it_validates_formats_if_a_checker_is_provided(self):
+        checker = FormatChecker()
+        bad = ValueError("Bad!")
+
+        @checker.checks("foo", raises=ValueError)
+        def check(value):
+            if value == "good":
+                return True
+            elif value == "bad":
+                raise bad
+            else:  # pragma: no cover
+                self.fail("What is {}? [Baby Don't Hurt Me]".format(value))
+
+        validator = self.Validator(
+            {"format": "foo"}, format_checker=checker,
+        )
+
+        validator.validate("good")
+        with self.assertRaises(exceptions.ValidationError) as cm:
+            validator.validate("bad")
+
+        # Make sure original cause is attached
+        self.assertIs(cm.exception.cause, bad)
+
+    def test_non_string_custom_type(self):
+        non_string_type = object()
+        schema = {"type": [non_string_type]}
+        Crazy = validators.extend(
+            self.Validator,
+            type_checker=self.Validator.TYPE_CHECKER.redefine(
+                non_string_type,
+                lambda checker, thing: isinstance(thing, int),
+            )
+        )
+        Crazy(schema).validate(15)
+
+    def test_it_properly_formats_tuples_in_errors(self):
+        """
+        A tuple instance properly formats validation errors for uniqueItems.
+
+        See https://github.com/Julian/jsonschema/pull/224
+        """
+        TupleValidator = validators.extend(
+            self.Validator,
+            type_checker=self.Validator.TYPE_CHECKER.redefine(
+                "array",
+                lambda checker, thing: isinstance(thing, tuple),
+            )
+        )
+        with self.assertRaises(exceptions.ValidationError) as e:
+            TupleValidator({"uniqueItems": True}).validate((1, 1))
+        self.assertIn("(1, 1) has non-unique elements", str(e.exception))
+
+
+class AntiDraft6LeakMixin(object):
+    """
+    Make sure functionality from draft 6 doesn't leak backwards in time.
+    """
+
+    def test_True_is_not_a_schema(self):
+        with self.assertRaises(exceptions.SchemaError) as e:
+            self.Validator.check_schema(True)
+        self.assertIn("True is not of type", str(e.exception))
+
+    def test_False_is_not_a_schema(self):
+        with self.assertRaises(exceptions.SchemaError) as e:
+            self.Validator.check_schema(False)
+        self.assertIn("False is not of type", str(e.exception))
+
+    @unittest.skip(bug(523))
+    def test_True_is_not_a_schema_even_if_you_forget_to_check(self):
+        resolver = validators.RefResolver("", {})
+        with self.assertRaises(Exception) as e:
+            self.Validator(True, resolver=resolver).validate(12)
+        self.assertNotIsInstance(e.exception, exceptions.ValidationError)
+
+    @unittest.skip(bug(523))
+    def test_False_is_not_a_schema_even_if_you_forget_to_check(self):
+        resolver = validators.RefResolver("", {})
+        with self.assertRaises(Exception) as e:
+            self.Validator(False, resolver=resolver).validate(12)
+        self.assertNotIsInstance(e.exception, exceptions.ValidationError)
+
+
+class TestDraft3Validator(AntiDraft6LeakMixin, ValidatorTestMixin, TestCase):
+    Validator = validators.Draft3Validator
+    valid = {}, {}
+    invalid = {"type": "integer"}, "foo"
+
+    def test_any_type_is_valid_for_type_any(self):
+        validator = self.Validator({"type": "any"})
+        validator.validate(object())
+
+    def test_any_type_is_redefinable(self):
+        """
+        Sigh, because why not.
+        """
+        Crazy = validators.extend(
+            self.Validator,
+            type_checker=self.Validator.TYPE_CHECKER.redefine(
+                "any", lambda checker, thing: isinstance(thing, int),
+            )
+        )
+        validator = Crazy({"type": "any"})
+        validator.validate(12)
+        with self.assertRaises(exceptions.ValidationError):
+            validator.validate("foo")
+
+    def test_is_type_is_true_for_any_type(self):
+        self.assertTrue(self.Validator({}).is_valid(object(), {"type": "any"}))
+
+    def test_is_type_does_not_evade_bool_if_it_is_being_tested(self):
+        self.assertTrue(self.Validator({}).is_type(True, "boolean"))
+        self.assertTrue(self.Validator({}).is_valid(True, {"type": "any"}))
+
+
+class TestDraft4Validator(AntiDraft6LeakMixin, ValidatorTestMixin, TestCase):
+    Validator = validators.Draft4Validator
+    valid = {}, {}
+    invalid = {"type": "integer"}, "foo"
+
+
+class TestDraft6Validator(ValidatorTestMixin, TestCase):
+    Validator = validators.Draft6Validator
+    valid = {}, {}
+    invalid = {"type": "integer"}, "foo"
+
+
+class TestDraft7Validator(ValidatorTestMixin, TestCase):
+    Validator = validators.Draft7Validator
+    valid = {}, {}
+    invalid = {"type": "integer"}, "foo"
+
+
+class TestValidatorFor(SynchronousTestCase):
+    def test_draft_3(self):
+        schema = {"$schema": "http://json-schema.org/draft-03/schema"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft3Validator,
+        )
+
+        schema = {"$schema": "http://json-schema.org/draft-03/schema#"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft3Validator,
+        )
+
+    def test_draft_4(self):
+        schema = {"$schema": "http://json-schema.org/draft-04/schema"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft4Validator,
+        )
+
+        schema = {"$schema": "http://json-schema.org/draft-04/schema#"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft4Validator,
+        )
+
+    def test_draft_6(self):
+        schema = {"$schema": "http://json-schema.org/draft-06/schema"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft6Validator,
+        )
+
+        schema = {"$schema": "http://json-schema.org/draft-06/schema#"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft6Validator,
+        )
+
+    def test_draft_7(self):
+        schema = {"$schema": "http://json-schema.org/draft-07/schema"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft7Validator,
+        )
+
+        schema = {"$schema": "http://json-schema.org/draft-07/schema#"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft7Validator,
+        )
+
+    def test_True(self):
+        self.assertIs(
+            validators.validator_for(True),
+            validators._LATEST_VERSION,
+        )
+
+    def test_False(self):
+        self.assertIs(
+            validators.validator_for(False),
+            validators._LATEST_VERSION,
+        )
+
+    def test_custom_validator(self):
+        Validator = validators.create(
+            meta_schema={"id": "meta schema id"},
+            version="12",
+            id_of=lambda s: s.get("id", ""),
+        )
+        schema = {"$schema": "meta schema id"}
+        self.assertIs(
+            validators.validator_for(schema),
+            Validator,
+        )
+
+    def test_custom_validator_draft6(self):
+        Validator = validators.create(
+            meta_schema={"$id": "meta schema $id"},
+            version="13",
+        )
+        schema = {"$schema": "meta schema $id"}
+        self.assertIs(
+            validators.validator_for(schema),
+            Validator,
+        )
+
+    def test_validator_for_jsonschema_default(self):
+        self.assertIs(validators.validator_for({}), validators._LATEST_VERSION)
+
+    def test_validator_for_custom_default(self):
+        self.assertIs(validators.validator_for({}, default=None), None)
+
+    def test_warns_if_meta_schema_specified_was_not_found(self):
+        self.assertWarns(
+            category=DeprecationWarning,
+            message=(
+                "The metaschema specified by $schema was not found. "
+                "Using the latest draft to validate, but this will raise "
+                "an error in the future."
+            ),
+            # https://tm.tl/9363 :'(
+            filename=sys.modules[self.assertWarns.__module__].__file__,
+
+            f=validators.validator_for,
+            schema={u"$schema": "unknownSchema"},
+            default={},
+        )
+
+    def test_does_not_warn_if_meta_schema_is_unspecified(self):
+        validators.validator_for(schema={}, default={}),
+        self.assertFalse(self.flushWarnings())
+
+
+class TestValidate(SynchronousTestCase):
+    def assertUses(self, schema, Validator):
+        result = []
+        self.patch(Validator, "check_schema", result.append)
+        validators.validate({}, schema)
+        self.assertEqual(result, [schema])
+
+    def test_draft3_validator_is_chosen(self):
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-03/schema#"},
+            Validator=validators.Draft3Validator,
+        )
+        # Make sure it works without the empty fragment
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-03/schema"},
+            Validator=validators.Draft3Validator,
+        )
+
+    def test_draft4_validator_is_chosen(self):
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-04/schema#"},
+            Validator=validators.Draft4Validator,
+        )
+        # Make sure it works without the empty fragment
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-04/schema"},
+            Validator=validators.Draft4Validator,
+        )
+
+    def test_draft6_validator_is_chosen(self):
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-06/schema#"},
+            Validator=validators.Draft6Validator,
+        )
+        # Make sure it works without the empty fragment
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-06/schema"},
+            Validator=validators.Draft6Validator,
+        )
+
+    def test_draft7_validator_is_chosen(self):
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-07/schema#"},
+            Validator=validators.Draft7Validator,
+        )
+        # Make sure it works without the empty fragment
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-07/schema"},
+            Validator=validators.Draft7Validator,
+        )
+
+    def test_draft7_validator_is_the_default(self):
+        self.assertUses(schema={}, Validator=validators.Draft7Validator)
+
+    def test_validation_error_message(self):
+        with self.assertRaises(exceptions.ValidationError) as e:
+            validators.validate(12, {"type": "string"})
+        self.assertRegexpMatches(
+            str(e.exception),
+            "(?s)Failed validating u?'.*' in schema.*On instance",
+        )
+
+    def test_schema_error_message(self):
+        with self.assertRaises(exceptions.SchemaError) as e:
+            validators.validate(12, {"type": 12})
+        self.assertRegexpMatches(
+            str(e.exception),
+            "(?s)Failed validating u?'.*' in metaschema.*On schema",
+        )
+
+    def test_it_uses_best_match(self):
+        # This is a schema that best_match will recurse into
+        schema = {"oneOf": [{"type": "string"}, {"type": "array"}]}
+        with self.assertRaises(exceptions.ValidationError) as e:
+            validators.validate(12, schema)
+        self.assertIn("12 is not of type", str(e.exception))
+
+
+class TestRefResolver(SynchronousTestCase):
+
+    base_uri = ""
+    stored_uri = "foo://stored"
+    stored_schema = {"stored": "schema"}
+
+    def setUp(self):
+        self.referrer = {}
+        self.store = {self.stored_uri: self.stored_schema}
+        self.resolver = validators.RefResolver(
+            self.base_uri, self.referrer, self.store,
+        )
+
+    def test_it_does_not_retrieve_schema_urls_from_the_network(self):
+        ref = validators.Draft3Validator.META_SCHEMA["id"]
+        self.patch(
+            self.resolver,
+            "resolve_remote",
+            lambda *args, **kwargs: self.fail("Should not have been called!"),
+        )
+        with self.resolver.resolving(ref) as resolved:
+            pass
+        self.assertEqual(resolved, validators.Draft3Validator.META_SCHEMA)
+
+    def test_it_resolves_local_refs(self):
+        ref = "#/properties/foo"
+        self.referrer["properties"] = {"foo": object()}
+        with self.resolver.resolving(ref) as resolved:
+            self.assertEqual(resolved, self.referrer["properties"]["foo"])
+
+    def test_it_resolves_local_refs_with_id(self):
+        schema = {"id": "http://bar/schema#", "a": {"foo": "bar"}}
+        resolver = validators.RefResolver.from_schema(
+            schema,
+            id_of=lambda schema: schema.get(u"id", u""),
+        )
+        with resolver.resolving("#/a") as resolved:
+            self.assertEqual(resolved, schema["a"])
+        with resolver.resolving("http://bar/schema#/a") as resolved:
+            self.assertEqual(resolved, schema["a"])
+
+    def test_it_retrieves_stored_refs(self):
+        with self.resolver.resolving(self.stored_uri) as resolved:
+            self.assertIs(resolved, self.stored_schema)
+
+        self.resolver.store["cached_ref"] = {"foo": 12}
+        with self.resolver.resolving("cached_ref#/foo") as resolved:
+            self.assertEqual(resolved, 12)
+
+    def test_it_retrieves_unstored_refs_via_requests(self):
+        ref = "http://bar#baz"
+        schema = {"baz": 12}
+
+        if "requests" in sys.modules:
+            self.addCleanup(
+                sys.modules.__setitem__, "requests", sys.modules["requests"],
+            )
+        sys.modules["requests"] = ReallyFakeRequests({"http://bar": schema})
+
+        with self.resolver.resolving(ref) as resolved:
+            self.assertEqual(resolved, 12)
+
+    def test_it_retrieves_unstored_refs_via_urlopen(self):
+        ref = "http://bar#baz"
+        schema = {"baz": 12}
+
+        if "requests" in sys.modules:
+            self.addCleanup(
+                sys.modules.__setitem__, "requests", sys.modules["requests"],
+            )
+        sys.modules["requests"] = None
+
+        @contextmanager
+        def fake_urlopen(url):
+            self.assertEqual(url, "http://bar")
+            yield BytesIO(json.dumps(schema).encode("utf8"))
+
+        self.addCleanup(setattr, validators, "urlopen", validators.urlopen)
+        validators.urlopen = fake_urlopen
+
+        with self.resolver.resolving(ref) as resolved:
+            pass
+        self.assertEqual(resolved, 12)
+
+    def test_it_retrieves_local_refs_via_urlopen(self):
+        with tempfile.NamedTemporaryFile(delete=False, mode="wt") as tempf:
+            self.addCleanup(os.remove, tempf.name)
+            json.dump({"foo": "bar"}, tempf)
+
+        ref = "file://{}#foo".format(pathname2url(tempf.name))
+        with self.resolver.resolving(ref) as resolved:
+            self.assertEqual(resolved, "bar")
+
+    def test_it_can_construct_a_base_uri_from_a_schema(self):
+        schema = {"id": "foo"}
+        resolver = validators.RefResolver.from_schema(
+            schema,
+            id_of=lambda schema: schema.get(u"id", u""),
+        )
+        self.assertEqual(resolver.base_uri, "foo")
+        self.assertEqual(resolver.resolution_scope, "foo")
+        with resolver.resolving("") as resolved:
+            self.assertEqual(resolved, schema)
+        with resolver.resolving("#") as resolved:
+            self.assertEqual(resolved, schema)
+        with resolver.resolving("foo") as resolved:
+            self.assertEqual(resolved, schema)
+        with resolver.resolving("foo#") as resolved:
+            self.assertEqual(resolved, schema)
+
+    def test_it_can_construct_a_base_uri_from_a_schema_without_id(self):
+        schema = {}
+        resolver = validators.RefResolver.from_schema(schema)
+        self.assertEqual(resolver.base_uri, "")
+        self.assertEqual(resolver.resolution_scope, "")
+        with resolver.resolving("") as resolved:
+            self.assertEqual(resolved, schema)
+        with resolver.resolving("#") as resolved:
+            self.assertEqual(resolved, schema)
+
+    def test_custom_uri_scheme_handlers(self):
+        def handler(url):
+            self.assertEqual(url, ref)
+            return schema
+
+        schema = {"foo": "bar"}
+        ref = "foo://bar"
+        resolver = validators.RefResolver("", {}, handlers={"foo": handler})
+        with resolver.resolving(ref) as resolved:
+            self.assertEqual(resolved, schema)
+
+    def test_cache_remote_on(self):
+        response = [object()]
+
+        def handler(url):
+            try:
+                return response.pop()
+            except IndexError:  # pragma: no cover
+                self.fail("Response must not have been cached!")
+
+        ref = "foo://bar"
+        resolver = validators.RefResolver(
+            "", {}, cache_remote=True, handlers={"foo": handler},
+        )
+        with resolver.resolving(ref):
+            pass
+        with resolver.resolving(ref):
+            pass
+
+    def test_cache_remote_off(self):
+        response = [object()]
+
+        def handler(url):
+            try:
+                return response.pop()
+            except IndexError:  # pragma: no cover
+                self.fail("Handler called twice!")
+
+        ref = "foo://bar"
+        resolver = validators.RefResolver(
+            "", {}, cache_remote=False, handlers={"foo": handler},
+        )
+        with resolver.resolving(ref):
+            pass
+
+    def test_if_you_give_it_junk_you_get_a_resolution_error(self):
+        error = ValueError("Oh no! What's this?")
+
+        def handler(url):
+            raise error
+
+        ref = "foo://bar"
+        resolver = validators.RefResolver("", {}, handlers={"foo": handler})
+        with self.assertRaises(exceptions.RefResolutionError) as err:
+            with resolver.resolving(ref):
+                self.fail("Shouldn't get this far!")  # pragma: no cover
+        self.assertEqual(err.exception, exceptions.RefResolutionError(error))
+
+    def test_helpful_error_message_on_failed_pop_scope(self):
+        resolver = validators.RefResolver("", {})
+        resolver.pop_scope()
+        with self.assertRaises(exceptions.RefResolutionError) as exc:
+            resolver.pop_scope()
+        self.assertIn("Failed to pop the scope", str(exc.exception))
+
+
+def sorted_errors(errors):
+    def key(error):
+        return (
+            [str(e) for e in error.path],
+            [str(e) for e in error.schema_path],
+        )
+    return sorted(errors, key=key)
+
+
+@attr.s
+class ReallyFakeRequests(object):
+
+    _responses = attr.ib()
+
+    def get(self, url):
+        response = self._responses.get(url)
+        if url is None:  # pragma: no cover
+            raise ValueError("Unknown URL: " + repr(url))
+        return _ReallyFakeJSONResponse(json.dumps(response))
+
+
+@attr.s
+class _ReallyFakeJSONResponse(object):
+
+    _response = attr.ib()
+
+    def json(self):
+        return json.loads(self._response)
diff --git a/contrib/python/jsonschema/py2/jsonschema/validators.py b/contrib/python/jsonschema/py2/jsonschema/validators.py
new file mode 100644
index 00000000000..1dc420c70d2
--- /dev/null
+++ b/contrib/python/jsonschema/py2/jsonschema/validators.py
@@ -0,0 +1,970 @@
+"""
+Creation and extension of validators, with implementations for existing drafts.
+"""
+from __future__ import division
+
+from warnings import warn
+import contextlib
+import json
+import numbers
+
+from six import add_metaclass
+
+from jsonschema import (
+    _legacy_validators,
+    _types,
+    _utils,
+    _validators,
+    exceptions,
+)
+from jsonschema.compat import (
+    Sequence,
+    int_types,
+    iteritems,
+    lru_cache,
+    str_types,
+    unquote,
+    urldefrag,
+    urljoin,
+    urlopen,
+    urlsplit,
+)
+
+# Sigh. https://gitlab.com/pycqa/flake8/issues/280
+#       https://github.com/pyga/ebb-lint/issues/7
+# Imported for backwards compatibility.
+from jsonschema.exceptions import ErrorTree
+ErrorTree
+
+
+class _DontDoThat(Exception):
+    """
+    Raised when a Validators with non-default type checker is misused.
+
+    Asking one for DEFAULT_TYPES doesn't make sense, since type checkers
+    exist for the unrepresentable cases where DEFAULT_TYPES can't
+    represent the type relationship.
+    """
+
+    def __str__(self):
+        return "DEFAULT_TYPES cannot be used on Validators using TypeCheckers"
+
+
+validators = {}
+meta_schemas = _utils.URIDict()
+
+
+def _generate_legacy_type_checks(types=()):
+    """
+    Generate newer-style type checks out of JSON-type-name-to-type mappings.
+
+    Arguments:
+
+        types (dict):
+
+            A mapping of type names to their Python types
+
+    Returns:
+
+        A dictionary of definitions to pass to `TypeChecker`
+    """
+    types = dict(types)
+
+    def gen_type_check(pytypes):
+        pytypes = _utils.flatten(pytypes)
+
+        def type_check(checker, instance):
+            if isinstance(instance, bool):
+                if bool not in pytypes:
+                    return False
+            return isinstance(instance, pytypes)
+
+        return type_check
+
+    definitions = {}
+    for typename, pytypes in iteritems(types):
+        definitions[typename] = gen_type_check(pytypes)
+
+    return definitions
+
+
+_DEPRECATED_DEFAULT_TYPES = {
+    u"array": list,
+    u"boolean": bool,
+    u"integer": int_types,
+    u"null": type(None),
+    u"number": numbers.Number,
+    u"object": dict,
+    u"string": str_types,
+}
+_TYPE_CHECKER_FOR_DEPRECATED_DEFAULT_TYPES = _types.TypeChecker(
+    type_checkers=_generate_legacy_type_checks(_DEPRECATED_DEFAULT_TYPES),
+)
+
+
+def validates(version):
+    """
+    Register the decorated validator for a ``version`` of the specification.
+
+    Registered validators and their meta schemas will be considered when
+    parsing ``$schema`` properties' URIs.
+
+    Arguments:
+
+        version (str):
+
+            An identifier to use as the version's name
+
+    Returns:
+
+        collections.Callable:
+
+            a class decorator to decorate the validator with the version
+    """
+
+    def _validates(cls):
+        validators[version] = cls
+        meta_schema_id = cls.ID_OF(cls.META_SCHEMA)
+        if meta_schema_id:
+            meta_schemas[meta_schema_id] = cls
+        return cls
+    return _validates
+
+
+def _DEFAULT_TYPES(self):
+    if self._CREATED_WITH_DEFAULT_TYPES is None:
+        raise _DontDoThat()
+
+    warn(
+        (
+            "The DEFAULT_TYPES attribute is deprecated. "
+            "See the type checker attached to this validator instead."
+        ),
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return self._DEFAULT_TYPES
+
+
+class _DefaultTypesDeprecatingMetaClass(type):
+    DEFAULT_TYPES = property(_DEFAULT_TYPES)
+
+
+def _id_of(schema):
+    if schema is True or schema is False:
+        return u""
+    return schema.get(u"$id", u"")
+
+
+def create(
+    meta_schema,
+    validators=(),
+    version=None,
+    default_types=None,
+    type_checker=None,
+    id_of=_id_of,
+):
+    """
+    Create a new validator class.
+
+    Arguments:
+
+        meta_schema (collections.Mapping):
+
+            the meta schema for the new validator class
+
+        validators (collections.Mapping):
+
+            a mapping from names to callables, where each callable will
+            validate the schema property with the given name.
+
+            Each callable should take 4 arguments:
+
+                1. a validator instance,
+                2. the value of the property being validated within the
+                   instance
+                3. the instance
+                4. the schema
+
+        version (str):
+
+            an identifier for the version that this validator class will
+            validate. If provided, the returned validator class will
+            have its ``__name__`` set to include the version, and also
+            will have `jsonschema.validators.validates` automatically
+            called for the given version.
+
+        type_checker (jsonschema.TypeChecker):
+
+            a type checker, used when applying the :validator:`type` validator.
+
+            If unprovided, a `jsonschema.TypeChecker` will be created
+            with a set of default types typical of JSON Schema drafts.
+
+        default_types (collections.Mapping):
+
+            .. deprecated:: 3.0.0
+
+                Please use the type_checker argument instead.
+
+            If set, it provides mappings of JSON types to Python types
+            that will be converted to functions and redefined in this
+            object's `jsonschema.TypeChecker`.
+
+        id_of (collections.Callable):
+
+            A function that given a schema, returns its ID.
+
+    Returns:
+
+        a new `jsonschema.IValidator` class
+    """
+
+    if default_types is not None:
+        if type_checker is not None:
+            raise TypeError(
+                "Do not specify default_types when providing a type checker.",
+            )
+        _created_with_default_types = True
+        warn(
+            (
+                "The default_types argument is deprecated. "
+                "Use the type_checker argument instead."
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        type_checker = _types.TypeChecker(
+            type_checkers=_generate_legacy_type_checks(default_types),
+        )
+    else:
+        default_types = _DEPRECATED_DEFAULT_TYPES
+        if type_checker is None:
+            _created_with_default_types = False
+            type_checker = _TYPE_CHECKER_FOR_DEPRECATED_DEFAULT_TYPES
+        elif type_checker is _TYPE_CHECKER_FOR_DEPRECATED_DEFAULT_TYPES:
+            _created_with_default_types = False
+        else:
+            _created_with_default_types = None
+
+    @add_metaclass(_DefaultTypesDeprecatingMetaClass)
+    class Validator(object):
+
+        VALIDATORS = dict(validators)
+        META_SCHEMA = dict(meta_schema)
+        TYPE_CHECKER = type_checker
+        ID_OF = staticmethod(id_of)
+
+        DEFAULT_TYPES = property(_DEFAULT_TYPES)
+        _DEFAULT_TYPES = dict(default_types)
+        _CREATED_WITH_DEFAULT_TYPES = _created_with_default_types
+
+        def __init__(
+            self,
+            schema,
+            types=(),
+            resolver=None,
+            format_checker=None,
+        ):
+            if types:
+                warn(
+                    (
+                        "The types argument is deprecated. Provide "
+                        "a type_checker to jsonschema.validators.extend "
+                        "instead."
+                    ),
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+
+                self.TYPE_CHECKER = self.TYPE_CHECKER.redefine_many(
+                    _generate_legacy_type_checks(types),
+                )
+
+            if resolver is None:
+                resolver = RefResolver.from_schema(schema, id_of=id_of)
+
+            self.resolver = resolver
+            self.format_checker = format_checker
+            self.schema = schema
+
+        @classmethod
+        def check_schema(cls, schema):
+            for error in cls(cls.META_SCHEMA).iter_errors(schema):
+                raise exceptions.SchemaError.create_from(error)
+
+        def iter_errors(self, instance, _schema=None):
+            if _schema is None:
+                _schema = self.schema
+
+            if _schema is True:
+                return
+            elif _schema is False:
+                yield exceptions.ValidationError(
+                    "False schema does not allow %r" % (instance,),
+                    validator=None,
+                    validator_value=None,
+                    instance=instance,
+                    schema=_schema,
+                )
+                return
+
+            scope = id_of(_schema)
+            if scope:
+                self.resolver.push_scope(scope)
+            try:
+                ref = _schema.get(u"$ref")
+                if ref is not None:
+                    validators = [(u"$ref", ref)]
+                else:
+                    validators = iteritems(_schema)
+
+                for k, v in validators:
+                    validator = self.VALIDATORS.get(k)
+                    if validator is None:
+                        continue
+
+                    errors = validator(self, v, instance, _schema) or ()
+                    for error in errors:
+                        # set details if not already set by the called fn
+                        error._set(
+                            validator=k,
+                            validator_value=v,
+                            instance=instance,
+                            schema=_schema,
+                        )
+                        if k != u"$ref":
+                            error.schema_path.appendleft(k)
+                        yield error
+            finally:
+                if scope:
+                    self.resolver.pop_scope()
+
+        def descend(self, instance, schema, path=None, schema_path=None):
+            for error in self.iter_errors(instance, schema):
+                if path is not None:
+                    error.path.appendleft(path)
+                if schema_path is not None:
+                    error.schema_path.appendleft(schema_path)
+                yield error
+
+        def validate(self, *args, **kwargs):
+            for error in self.iter_errors(*args, **kwargs):
+                raise error
+
+        def is_type(self, instance, type):
+            try:
+                return self.TYPE_CHECKER.is_type(instance, type)
+            except exceptions.UndefinedTypeCheck:
+                raise exceptions.UnknownType(type, instance, self.schema)
+
+        def is_valid(self, instance, _schema=None):
+            error = next(self.iter_errors(instance, _schema), None)
+            return error is None
+
+    if version is not None:
+        Validator = validates(version)(Validator)
+        Validator.__name__ = version.title().replace(" ", "") + "Validator"
+
+    return Validator
+
+
+def extend(validator, validators=(), version=None, type_checker=None):
+    """
+    Create a new validator class by extending an existing one.
+
+    Arguments:
+
+        validator (jsonschema.IValidator):
+
+            an existing validator class
+
+        validators (collections.Mapping):
+
+            a mapping of new validator callables to extend with, whose
+            structure is as in `create`.
+
+            .. note::
+
+                Any validator callables with the same name as an
+                existing one will (silently) replace the old validator
+                callable entirely, effectively overriding any validation
+                done in the "parent" validator class.
+
+                If you wish to instead extend the behavior of a parent's
+                validator callable, delegate and call it directly in
+                the new validator function by retrieving it using
+                ``OldValidator.VALIDATORS["validator_name"]``.
+
+        version (str):
+
+            a version for the new validator class
+
+        type_checker (jsonschema.TypeChecker):
+
+            a type checker, used when applying the :validator:`type` validator.
+
+            If unprovided, the type checker of the extended
+            `jsonschema.IValidator` will be carried along.`
+
+    Returns:
+
+        a new `jsonschema.IValidator` class extending the one provided
+
+    .. note:: Meta Schemas
+
+        The new validator class will have its parent's meta schema.
+
+        If you wish to change or extend the meta schema in the new
+        validator class, modify ``META_SCHEMA`` directly on the returned
+        class. Note that no implicit copying is done, so a copy should
+        likely be made before modifying it, in order to not affect the
+        old validator.
+    """
+
+    all_validators = dict(validator.VALIDATORS)
+    all_validators.update(validators)
+
+    if type_checker is None:
+        type_checker = validator.TYPE_CHECKER
+    elif validator._CREATED_WITH_DEFAULT_TYPES:
+        raise TypeError(
+            "Cannot extend a validator created with default_types "
+            "with a type_checker. Update the validator to use a "
+            "type_checker when created."
+        )
+    return create(
+        meta_schema=validator.META_SCHEMA,
+        validators=all_validators,
+        version=version,
+        type_checker=type_checker,
+        id_of=validator.ID_OF,
+    )
+
+
+Draft3Validator = create(
+    meta_schema=_utils.load_schema("draft3"),
+    validators={
+        u"$ref": _validators.ref,
+        u"additionalItems": _validators.additionalItems,
+        u"additionalProperties": _validators.additionalProperties,
+        u"dependencies": _legacy_validators.dependencies_draft3,
+        u"disallow": _legacy_validators.disallow_draft3,
+        u"divisibleBy": _validators.multipleOf,
+        u"enum": _validators.enum,
+        u"extends": _legacy_validators.extends_draft3,
+        u"format": _validators.format,
+        u"items": _legacy_validators.items_draft3_draft4,
+        u"maxItems": _validators.maxItems,
+        u"maxLength": _validators.maxLength,
+        u"maximum": _legacy_validators.maximum_draft3_draft4,
+        u"minItems": _validators.minItems,
+        u"minLength": _validators.minLength,
+        u"minimum": _legacy_validators.minimum_draft3_draft4,
+        u"pattern": _validators.pattern,
+        u"patternProperties": _validators.patternProperties,
+        u"properties": _legacy_validators.properties_draft3,
+        u"type": _legacy_validators.type_draft3,
+        u"uniqueItems": _validators.uniqueItems,
+    },
+    type_checker=_types.draft3_type_checker,
+    version="draft3",
+    id_of=lambda schema: schema.get(u"id", ""),
+)
+
+Draft4Validator = create(
+    meta_schema=_utils.load_schema("draft4"),
+    validators={
+        u"$ref": _validators.ref,
+        u"additionalItems": _validators.additionalItems,
+        u"additionalProperties": _validators.additionalProperties,
+        u"allOf": _validators.allOf,
+        u"anyOf": _validators.anyOf,
+        u"dependencies": _validators.dependencies,
+        u"enum": _validators.enum,
+        u"format": _validators.format,
+        u"items": _legacy_validators.items_draft3_draft4,
+        u"maxItems": _validators.maxItems,
+        u"maxLength": _validators.maxLength,
+        u"maxProperties": _validators.maxProperties,
+        u"maximum": _legacy_validators.maximum_draft3_draft4,
+        u"minItems": _validators.minItems,
+        u"minLength": _validators.minLength,
+        u"minProperties": _validators.minProperties,
+        u"minimum": _legacy_validators.minimum_draft3_draft4,
+        u"multipleOf": _validators.multipleOf,
+        u"not": _validators.not_,
+        u"oneOf": _validators.oneOf,
+        u"pattern": _validators.pattern,
+        u"patternProperties": _validators.patternProperties,
+        u"properties": _validators.properties,
+        u"required": _validators.required,
+        u"type": _validators.type,
+        u"uniqueItems": _validators.uniqueItems,
+    },
+    type_checker=_types.draft4_type_checker,
+    version="draft4",
+    id_of=lambda schema: schema.get(u"id", ""),
+)
+
+Draft6Validator = create(
+    meta_schema=_utils.load_schema("draft6"),
+    validators={
+        u"$ref": _validators.ref,
+        u"additionalItems": _validators.additionalItems,
+        u"additionalProperties": _validators.additionalProperties,
+        u"allOf": _validators.allOf,
+        u"anyOf": _validators.anyOf,
+        u"const": _validators.const,
+        u"contains": _validators.contains,
+        u"dependencies": _validators.dependencies,
+        u"enum": _validators.enum,
+        u"exclusiveMaximum": _validators.exclusiveMaximum,
+        u"exclusiveMinimum": _validators.exclusiveMinimum,
+        u"format": _validators.format,
+        u"items": _validators.items,
+        u"maxItems": _validators.maxItems,
+        u"maxLength": _validators.maxLength,
+        u"maxProperties": _validators.maxProperties,
+        u"maximum": _validators.maximum,
+        u"minItems": _validators.minItems,
+        u"minLength": _validators.minLength,
+        u"minProperties": _validators.minProperties,
+        u"minimum": _validators.minimum,
+        u"multipleOf": _validators.multipleOf,
+        u"not": _validators.not_,
+        u"oneOf": _validators.oneOf,
+        u"pattern": _validators.pattern,
+        u"patternProperties": _validators.patternProperties,
+        u"properties": _validators.properties,
+        u"propertyNames": _validators.propertyNames,
+        u"required": _validators.required,
+        u"type": _validators.type,
+        u"uniqueItems": _validators.uniqueItems,
+    },
+    type_checker=_types.draft6_type_checker,
+    version="draft6",
+)
+
+Draft7Validator = create(
+    meta_schema=_utils.load_schema("draft7"),
+    validators={
+        u"$ref": _validators.ref,
+        u"additionalItems": _validators.additionalItems,
+        u"additionalProperties": _validators.additionalProperties,
+        u"allOf": _validators.allOf,
+        u"anyOf": _validators.anyOf,
+        u"const": _validators.const,
+        u"contains": _validators.contains,
+        u"dependencies": _validators.dependencies,
+        u"enum": _validators.enum,
+        u"exclusiveMaximum": _validators.exclusiveMaximum,
+        u"exclusiveMinimum": _validators.exclusiveMinimum,
+        u"format": _validators.format,
+        u"if": _validators.if_,
+        u"items": _validators.items,
+        u"maxItems": _validators.maxItems,
+        u"maxLength": _validators.maxLength,
+        u"maxProperties": _validators.maxProperties,
+        u"maximum": _validators.maximum,
+        u"minItems": _validators.minItems,
+        u"minLength": _validators.minLength,
+        u"minProperties": _validators.minProperties,
+        u"minimum": _validators.minimum,
+        u"multipleOf": _validators.multipleOf,
+        u"oneOf": _validators.oneOf,
+        u"not": _validators.not_,
+        u"pattern": _validators.pattern,
+        u"patternProperties": _validators.patternProperties,
+        u"properties": _validators.properties,
+        u"propertyNames": _validators.propertyNames,
+        u"required": _validators.required,
+        u"type": _validators.type,
+        u"uniqueItems": _validators.uniqueItems,
+    },
+    type_checker=_types.draft7_type_checker,
+    version="draft7",
+)
+
+_LATEST_VERSION = Draft7Validator
+
+
+class RefResolver(object):
+    """
+    Resolve JSON References.
+
+    Arguments:
+
+        base_uri (str):
+
+            The URI of the referring document
+
+        referrer:
+
+            The actual referring document
+
+        store (dict):
+
+            A mapping from URIs to documents to cache
+
+        cache_remote (bool):
+
+            Whether remote refs should be cached after first resolution
+
+        handlers (dict):
+
+            A mapping from URI schemes to functions that should be used
+            to retrieve them
+
+        urljoin_cache (:func:`functools.lru_cache`):
+
+            A cache that will be used for caching the results of joining
+            the resolution scope to subscopes.
+
+        remote_cache (:func:`functools.lru_cache`):
+
+            A cache that will be used for caching the results of
+            resolved remote URLs.
+
+    Attributes:
+
+        cache_remote (bool):
+
+            Whether remote refs should be cached after first resolution
+    """
+
+    def __init__(
+        self,
+        base_uri,
+        referrer,
+        store=(),
+        cache_remote=True,
+        handlers=(),
+        urljoin_cache=None,
+        remote_cache=None,
+    ):
+        if urljoin_cache is None:
+            urljoin_cache = lru_cache(1024)(urljoin)
+        if remote_cache is None:
+            remote_cache = lru_cache(1024)(self.resolve_from_url)
+
+        self.referrer = referrer
+        self.cache_remote = cache_remote
+        self.handlers = dict(handlers)
+
+        self._scopes_stack = [base_uri]
+        self.store = _utils.URIDict(
+            (id, validator.META_SCHEMA)
+            for id, validator in iteritems(meta_schemas)
+        )
+        self.store.update(store)
+        self.store[base_uri] = referrer
+
+        self._urljoin_cache = urljoin_cache
+        self._remote_cache = remote_cache
+
+    @classmethod
+    def from_schema(cls, schema, id_of=_id_of, *args, **kwargs):
+        """
+        Construct a resolver from a JSON schema object.
+
+        Arguments:
+
+            schema:
+
+                the referring schema
+
+        Returns:
+
+            `RefResolver`
+        """
+
+        return cls(base_uri=id_of(schema), referrer=schema, *args, **kwargs)
+
+    def push_scope(self, scope):
+        """
+        Enter a given sub-scope.
+
+        Treats further dereferences as being performed underneath the
+        given scope.
+        """
+        self._scopes_stack.append(
+            self._urljoin_cache(self.resolution_scope, scope),
+        )
+
+    def pop_scope(self):
+        """
+        Exit the most recent entered scope.
+
+        Treats further dereferences as being performed underneath the
+        original scope.
+
+        Don't call this method more times than `push_scope` has been
+        called.
+        """
+        try:
+            self._scopes_stack.pop()
+        except IndexError:
+            raise exceptions.RefResolutionError(
+                "Failed to pop the scope from an empty stack. "
+                "`pop_scope()` should only be called once for every "
+                "`push_scope()`"
+            )
+
+    @property
+    def resolution_scope(self):
+        """
+        Retrieve the current resolution scope.
+        """
+        return self._scopes_stack[-1]
+
+    @property
+    def base_uri(self):
+        """
+        Retrieve the current base URI, not including any fragment.
+        """
+        uri, _ = urldefrag(self.resolution_scope)
+        return uri
+
+    @contextlib.contextmanager
+    def in_scope(self, scope):
+        """
+        Temporarily enter the given scope for the duration of the context.
+        """
+        self.push_scope(scope)
+        try:
+            yield
+        finally:
+            self.pop_scope()
+
+    @contextlib.contextmanager
+    def resolving(self, ref):
+        """
+        Resolve the given ``ref`` and enter its resolution scope.
+
+        Exits the scope on exit of this context manager.
+
+        Arguments:
+
+            ref (str):
+
+                The reference to resolve
+        """
+
+        url, resolved = self.resolve(ref)
+        self.push_scope(url)
+        try:
+            yield resolved
+        finally:
+            self.pop_scope()
+
+    def resolve(self, ref):
+        """
+        Resolve the given reference.
+        """
+        url = self._urljoin_cache(self.resolution_scope, ref)
+        return url, self._remote_cache(url)
+
+    def resolve_from_url(self, url):
+        """
+        Resolve the given remote URL.
+        """
+        url, fragment = urldefrag(url)
+        try:
+            document = self.store[url]
+        except KeyError:
+            try:
+                document = self.resolve_remote(url)
+            except Exception as exc:
+                raise exceptions.RefResolutionError(exc)
+
+        return self.resolve_fragment(document, fragment)
+
+    def resolve_fragment(self, document, fragment):
+        """
+        Resolve a ``fragment`` within the referenced ``document``.
+
+        Arguments:
+
+            document:
+
+                The referent document
+
+            fragment (str):
+
+                a URI fragment to resolve within it
+        """
+
+        fragment = fragment.lstrip(u"/")
+        parts = unquote(fragment).split(u"/") if fragment else []
+
+        for part in parts:
+            part = part.replace(u"~1", u"/").replace(u"~0", u"~")
+
+            if isinstance(document, Sequence):
+                # Array indexes should be turned into integers
+                try:
+                    part = int(part)
+                except ValueError:
+                    pass
+            try:
+                document = document[part]
+            except (TypeError, LookupError):
+                raise exceptions.RefResolutionError(
+                    "Unresolvable JSON pointer: %r" % fragment
+                )
+
+        return document
+
+    def resolve_remote(self, uri):
+        """
+        Resolve a remote ``uri``.
+
+        If called directly, does not check the store first, but after
+        retrieving the document at the specified URI it will be saved in
+        the store if :attr:`cache_remote` is True.
+
+        .. note::
+
+            If the requests_ library is present, ``jsonschema`` will use it to
+            request the remote ``uri``, so that the correct encoding is
+            detected and used.
+
+            If it isn't, or if the scheme of the ``uri`` is not ``http`` or
+            ``https``, UTF-8 is assumed.
+
+        Arguments:
+
+            uri (str):
+
+                The URI to resolve
+
+        Returns:
+
+            The retrieved document
+
+        .. _requests: https://pypi.org/project/requests/
+        """
+        try:
+            import requests
+        except ImportError:
+            requests = None
+
+        scheme = urlsplit(uri).scheme
+
+        if scheme in self.handlers:
+            result = self.handlers[scheme](uri)
+        elif scheme in [u"http", u"https"] and requests:
+            # Requests has support for detecting the correct encoding of
+            # json over http
+            result = requests.get(uri).json()
+        else:
+            # Otherwise, pass off to urllib and assume utf-8
+            with urlopen(uri) as url:
+                result = json.loads(url.read().decode("utf-8"))
+
+        if self.cache_remote:
+            self.store[uri] = result
+        return result
+
+
+def validate(instance, schema, cls=None, *args, **kwargs):
+    """
+    Validate an instance under the given schema.
+
+        >>> validate([2, 3, 4], {"maxItems": 2})
+        Traceback (most recent call last):
+            ...
+        ValidationError: [2, 3, 4] is too long
+
+    :func:`validate` will first verify that the provided schema is
+    itself valid, since not doing so can lead to less obvious error
+    messages and fail in less obvious or consistent ways.
+
+    If you know you have a valid schema already, especially if you
+    intend to validate multiple instances with the same schema, you
+    likely would prefer using the `IValidator.validate` method directly
+    on a specific validator (e.g. ``Draft7Validator.validate``).
+
+
+    Arguments:
+
+        instance:
+
+            The instance to validate
+
+        schema:
+
+            The schema to validate with
+
+        cls (IValidator):
+
+            The class that will be used to validate the instance.
+
+    If the ``cls`` argument is not provided, two things will happen
+    in accordance with the specification. First, if the schema has a
+    :validator:`$schema` property containing a known meta-schema [#]_
+    then the proper validator will be used. The specification recommends
+    that all schemas contain :validator:`$schema` properties for this
+    reason. If no :validator:`$schema` property is found, the default
+    validator class is the latest released draft.
+
+    Any other provided positional and keyword arguments will be passed
+    on when instantiating the ``cls``.
+
+    Raises:
+
+        `jsonschema.exceptions.ValidationError` if the instance
+            is invalid
+
+        `jsonschema.exceptions.SchemaError` if the schema itself
+            is invalid
+
+    .. rubric:: Footnotes
+    .. [#] known by a validator registered with
+        `jsonschema.validators.validates`
+    """
+    if cls is None:
+        cls = validator_for(schema)
+
+    cls.check_schema(schema)
+    validator = cls(schema, *args, **kwargs)
+    error = exceptions.best_match(validator.iter_errors(instance))
+    if error is not None:
+        raise error
+
+
+def validator_for(schema, default=_LATEST_VERSION):
+    """
+    Retrieve the validator class appropriate for validating the given schema.
+
+    Uses the :validator:`$schema` property that should be present in the
+    given schema to look up the appropriate validator class.
+
+    Arguments:
+
+        schema (collections.Mapping or bool):
+
+            the schema to look at
+
+        default:
+
+            the default to return if the appropriate validator class
+            cannot be determined.
+
+            If unprovided, the default is to return the latest supported
+            draft.
+    """
+    if schema is True or schema is False or u"$schema" not in schema:
+        return default
+    if schema[u"$schema"] not in meta_schemas:
+        warn(
+            (
+                "The metaschema specified by $schema was not found. "
+                "Using the latest draft to validate, but this will raise "
+                "an error in the future."
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+    return meta_schemas.get(schema[u"$schema"], _LATEST_VERSION)
diff --git a/contrib/python/jsonschema/py2/tests/ya.make b/contrib/python/jsonschema/py2/tests/ya.make
new file mode 100644
index 00000000000..198984175b0
--- /dev/null
+++ b/contrib/python/jsonschema/py2/tests/ya.make
@@ -0,0 +1,32 @@
+PY2TEST()
+
+PEERDIR(
+    contrib/python/jsonschema
+    contrib/python/Twisted
+)
+
+IF (PYTHON2)
+    PEERDIR(
+        contrib/python/mock
+    )
+ENDIF()
+
+SRCDIR(contrib/python/jsonschema/py2/jsonschema/tests)
+
+PY_SRCS(
+    NAMESPACE jsonschema.tests
+    _helpers.py
+)
+
+TEST_SRCS(
+    __init__.py
+    test_cli.py
+    test_exceptions.py
+    test_format.py
+    test_types.py
+    test_validators.py
+)
+
+NO_LINT()
+
+END()
diff --git a/contrib/python/jsonschema/py2/ya.make b/contrib/python/jsonschema/py2/ya.make
new file mode 100644
index 00000000000..6860467224d
--- /dev/null
+++ b/contrib/python/jsonschema/py2/ya.make
@@ -0,0 +1,51 @@
+# Generated by devtools/yamaker (pypi).
+
+PY2_LIBRARY()
+
+VERSION(3.2.0)
+
+LICENSE(MIT)
+
+PEERDIR(
+    contrib/deprecated/python/functools32
+    contrib/python/attrs
+    contrib/python/importlib-metadata
+    contrib/python/pyrsistent
+    contrib/python/setuptools
+    contrib/python/six
+)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    jsonschema/__init__.py
+    jsonschema/__main__.py
+    jsonschema/_format.py
+    jsonschema/_legacy_validators.py
+    jsonschema/_reflect.py
+    jsonschema/_types.py
+    jsonschema/_utils.py
+    jsonschema/_validators.py
+    jsonschema/cli.py
+    jsonschema/compat.py
+    jsonschema/exceptions.py
+    jsonschema/validators.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/jsonschema/py2/
+    .dist-info/METADATA
+    .dist-info/entry_points.txt
+    .dist-info/top_level.txt
+    jsonschema/schemas/draft3.json
+    jsonschema/schemas/draft4.json
+    jsonschema/schemas/draft6.json
+    jsonschema/schemas/draft7.json
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/jsonschema/py3/.dist-info/METADATA b/contrib/python/jsonschema/py3/.dist-info/METADATA
new file mode 100644
index 00000000000..aef9b18d586
--- /dev/null
+++ b/contrib/python/jsonschema/py3/.dist-info/METADATA
@@ -0,0 +1,224 @@
+Metadata-Version: 2.1
+Name: jsonschema
+Version: 3.2.0
+Summary: An implementation of JSON Schema validation for Python
+Home-page: https://github.com/Julian/jsonschema
+Author: Julian Berman
+Author-email: Julian@GrayVines.com
+License: UNKNOWN
+Project-URL: Docs, https://python-jsonschema.readthedocs.io/en/latest/
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Dist: attrs (>=17.4.0)
+Requires-Dist: pyrsistent (>=0.14.0)
+Requires-Dist: setuptools
+Requires-Dist: six (>=1.11.0)
+Requires-Dist: functools32 ; python_version < "3"
+Requires-Dist: importlib-metadata ; python_version < "3.8"
+Provides-Extra: format
+Requires-Dist: idna ; extra == 'format'
+Requires-Dist: jsonpointer (>1.13) ; extra == 'format'
+Requires-Dist: rfc3987 ; extra == 'format'
+Requires-Dist: strict-rfc3339 ; extra == 'format'
+Requires-Dist: webcolors ; extra == 'format'
+Provides-Extra: format_nongpl
+Requires-Dist: idna ; extra == 'format_nongpl'
+Requires-Dist: jsonpointer (>1.13) ; extra == 'format_nongpl'
+Requires-Dist: webcolors ; extra == 'format_nongpl'
+Requires-Dist: rfc3986-validator (>0.1.0) ; extra == 'format_nongpl'
+Requires-Dist: rfc3339-validator ; extra == 'format_nongpl'
+
+==========
+jsonschema
+==========
+
+|PyPI| |Pythons| |Travis| |AppVeyor| |Codecov| |ReadTheDocs|
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/jsonschema.svg
+   :alt: PyPI version
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Pythons| image:: https://img.shields.io/pypi/pyversions/jsonschema.svg
+   :alt: Supported Python versions
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Travis| image:: https://travis-ci.com/Julian/jsonschema.svg?branch=master
+   :alt: Travis build status
+   :target: https://travis-ci.com/Julian/jsonschema
+
+.. |AppVeyor| image:: https://ci.appveyor.com/api/projects/status/adtt0aiaihy6muyn/branch/master?svg=true
+   :alt: AppVeyor build status
+   :target: https://ci.appveyor.com/project/Julian/jsonschema
+
+.. |Codecov| image:: https://codecov.io/gh/Julian/jsonschema/branch/master/graph/badge.svg
+   :alt: Codecov Code coverage
+   :target: https://codecov.io/gh/Julian/jsonschema
+
+.. |ReadTheDocs| image:: https://readthedocs.org/projects/python-jsonschema/badge/?version=stable&style=flat
+   :alt: ReadTheDocs status
+   :target: https://python-jsonschema.readthedocs.io/en/stable/
+
+
+``jsonschema`` is an implementation of `JSON Schema <https://json-schema.org>`_
+for Python (supporting 2.7+ including Python 3).
+
+.. code-block:: python
+
+    >>> from jsonschema import validate
+
+    >>> # A sample schema, like what we'd get from json.load()
+    >>> schema = {
+    ...     "type" : "object",
+    ...     "properties" : {
+    ...         "price" : {"type" : "number"},
+    ...         "name" : {"type" : "string"},
+    ...     },
+    ... }
+
+    >>> # If no exception is raised by validate(), the instance is valid.
+    >>> validate(instance={"name" : "Eggs", "price" : 34.99}, schema=schema)
+
+    >>> validate(
+    ...     instance={"name" : "Eggs", "price" : "Invalid"}, schema=schema,
+    ... )                                   # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValidationError: 'Invalid' is not of type 'number'
+
+It can also be used from console:
+
+.. code-block:: bash
+
+    $ jsonschema -i sample.json sample.schema
+
+Features
+--------
+
+* Full support for
+  `Draft 7 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft7Validator>`_,
+  `Draft 6 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft6Validator>`_,
+  `Draft 4 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft4Validator>`_
+  and
+  `Draft 3 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft3Validator>`_
+
+* `Lazy validation <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.IValidator.iter_errors>`_
+  that can iteratively report *all* validation errors.
+
+* `Programmatic querying <https://python-jsonschema.readthedocs.io/en/latest/errors/>`_
+  of which properties or items failed validation.
+
+
+Installation
+------------
+
+``jsonschema`` is available on `PyPI <https://pypi.org/project/jsonschema/>`_. You can install using `pip <https://pip.pypa.io/en/stable/>`_:
+
+.. code-block:: bash
+
+    $ pip install jsonschema
+
+
+Demo
+----
+
+Try ``jsonschema`` interactively in this online demo:
+
+.. image:: https://user-images.githubusercontent.com/1155573/56745335-8b158a00-6750-11e9-8776-83fa675939c4.png
+    :target: https://notebooks.ai/demo/gh/Julian/jsonschema
+    :alt: Open Live Demo
+
+
+Online demo Notebook will look similar to this:
+
+
+.. image:: https://user-images.githubusercontent.com/1155573/56820861-5c1c1880-6823-11e9-802a-ce01c5ec574f.gif
+    :alt: Open Live Demo
+    :width: 480 px
+
+
+Release Notes
+-------------
+
+v3.1 brings support for ECMA 262 dialect regular expressions
+throughout schemas, as recommended by the specification. Big
+thanks to @Zac-HD for authoring support in a new `js-regex
+<https://pypi.org/project/js-regex/>`_ library.
+
+
+Running the Test Suite
+----------------------
+
+If you have ``tox`` installed (perhaps via ``pip install tox`` or your
+package manager), running ``tox`` in the directory of your source
+checkout will run ``jsonschema``'s test suite on all of the versions
+of Python ``jsonschema`` supports. If you don't have all of the
+versions that ``jsonschema`` is tested under, you'll likely want to run
+using ``tox``'s ``--skip-missing-interpreters`` option.
+
+Of course you're also free to just run the tests on a single version with your
+favorite test runner. The tests live in the ``jsonschema.tests`` package.
+
+
+Benchmarks
+----------
+
+``jsonschema``'s benchmarks make use of `pyperf
+<https://pyperf.readthedocs.io>`_.
+
+Running them can be done via ``tox -e perf``, or by invoking the ``pyperf``
+commands externally (after ensuring that both it and ``jsonschema`` itself are
+installed)::
+
+    $ python -m pyperf jsonschema/benchmarks/test_suite.py --hist --output results.json
+
+To compare to a previous run, use::
+
+    $ python -m pyperf compare_to --table reference.json results.json
+
+See the ``pyperf`` documentation for more details.
+
+
+Community
+---------
+
+There's a `mailing list <https://groups.google.com/forum/#!forum/jsonschema>`_
+for this implementation on Google Groups.
+
+Please join, and feel free to send questions there.
+
+
+Contributing
+------------
+
+I'm Julian Berman.
+
+``jsonschema`` is on `GitHub <https://github.com/Julian/jsonschema>`_.
+
+Get in touch, via GitHub or otherwise, if you've got something to contribute,
+it'd be most welcome!
+
+You can also generally find me on Freenode (nick: ``tos9``) in various
+channels, including ``#python``.
+
+If you feel overwhelmingly grateful, you can also woo me with beer money
+via Google Pay with the email in my GitHub profile.
+
+And for companies who appreciate ``jsonschema`` and its continued support
+and growth, ``jsonschema`` is also now supportable via `TideLift
+<https://tidelift.com/subscription/pkg/pypi-jsonschema?utm_source=pypi-j
+sonschema&utm_medium=referral&utm_campaign=readme>`_.
+
+
diff --git a/contrib/python/jsonschema/py3/.dist-info/entry_points.txt b/contrib/python/jsonschema/py3/.dist-info/entry_points.txt
new file mode 100644
index 00000000000..c627b310cd0
--- /dev/null
+++ b/contrib/python/jsonschema/py3/.dist-info/entry_points.txt
@@ -0,0 +1,3 @@
+[console_scripts]
+jsonschema = jsonschema.cli:main
+
diff --git a/contrib/python/jsonschema/py3/.dist-info/top_level.txt b/contrib/python/jsonschema/py3/.dist-info/top_level.txt
new file mode 100644
index 00000000000..d89304b1a89
--- /dev/null
+++ b/contrib/python/jsonschema/py3/.dist-info/top_level.txt
@@ -0,0 +1 @@
+jsonschema
diff --git a/contrib/python/jsonschema/py3/COPYING b/contrib/python/jsonschema/py3/COPYING
new file mode 100644
index 00000000000..af9cfbdb134
--- /dev/null
+++ b/contrib/python/jsonschema/py3/COPYING
@@ -0,0 +1,19 @@
+Copyright (c) 2013 Julian Berman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/contrib/python/jsonschema/py3/README.rst b/contrib/python/jsonschema/py3/README.rst
new file mode 100644
index 00000000000..ccfb55d02d4
--- /dev/null
+++ b/contrib/python/jsonschema/py3/README.rst
@@ -0,0 +1,179 @@
+==========
+jsonschema
+==========
+
+|PyPI| |Pythons| |Travis| |AppVeyor| |Codecov| |ReadTheDocs|
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/jsonschema.svg
+   :alt: PyPI version
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Pythons| image:: https://img.shields.io/pypi/pyversions/jsonschema.svg
+   :alt: Supported Python versions
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Travis| image:: https://travis-ci.com/Julian/jsonschema.svg?branch=master
+   :alt: Travis build status
+   :target: https://travis-ci.com/Julian/jsonschema
+
+.. |AppVeyor| image:: https://ci.appveyor.com/api/projects/status/adtt0aiaihy6muyn/branch/master?svg=true
+   :alt: AppVeyor build status
+   :target: https://ci.appveyor.com/project/Julian/jsonschema
+
+.. |Codecov| image:: https://codecov.io/gh/Julian/jsonschema/branch/master/graph/badge.svg
+   :alt: Codecov Code coverage
+   :target: https://codecov.io/gh/Julian/jsonschema
+
+.. |ReadTheDocs| image:: https://readthedocs.org/projects/python-jsonschema/badge/?version=stable&style=flat
+   :alt: ReadTheDocs status
+   :target: https://python-jsonschema.readthedocs.io/en/stable/
+
+
+``jsonschema`` is an implementation of `JSON Schema <https://json-schema.org>`_
+for Python (supporting 2.7+ including Python 3).
+
+.. code-block:: python
+
+    >>> from jsonschema import validate
+
+    >>> # A sample schema, like what we'd get from json.load()
+    >>> schema = {
+    ...     "type" : "object",
+    ...     "properties" : {
+    ...         "price" : {"type" : "number"},
+    ...         "name" : {"type" : "string"},
+    ...     },
+    ... }
+
+    >>> # If no exception is raised by validate(), the instance is valid.
+    >>> validate(instance={"name" : "Eggs", "price" : 34.99}, schema=schema)
+
+    >>> validate(
+    ...     instance={"name" : "Eggs", "price" : "Invalid"}, schema=schema,
+    ... )                                   # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValidationError: 'Invalid' is not of type 'number'
+
+It can also be used from console:
+
+.. code-block:: bash
+
+    $ jsonschema -i sample.json sample.schema
+
+Features
+--------
+
+* Full support for
+  `Draft 7 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft7Validator>`_,
+  `Draft 6 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft6Validator>`_,
+  `Draft 4 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft4Validator>`_
+  and
+  `Draft 3 <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.Draft3Validator>`_
+
+* `Lazy validation <https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.IValidator.iter_errors>`_
+  that can iteratively report *all* validation errors.
+
+* `Programmatic querying <https://python-jsonschema.readthedocs.io/en/latest/errors/>`_
+  of which properties or items failed validation.
+
+
+Installation
+------------
+
+``jsonschema`` is available on `PyPI <https://pypi.org/project/jsonschema/>`_. You can install using `pip <https://pip.pypa.io/en/stable/>`_:
+
+.. code-block:: bash
+
+    $ pip install jsonschema
+
+
+Demo
+----
+
+Try ``jsonschema`` interactively in this online demo:
+
+.. image:: https://user-images.githubusercontent.com/1155573/56745335-8b158a00-6750-11e9-8776-83fa675939c4.png
+    :target: https://notebooks.ai/demo/gh/Julian/jsonschema
+    :alt: Open Live Demo
+
+
+Online demo Notebook will look similar to this:
+
+
+.. image:: https://user-images.githubusercontent.com/1155573/56820861-5c1c1880-6823-11e9-802a-ce01c5ec574f.gif
+    :alt: Open Live Demo
+    :width: 480 px
+
+
+Release Notes
+-------------
+
+v3.1 brings support for ECMA 262 dialect regular expressions
+throughout schemas, as recommended by the specification. Big
+thanks to @Zac-HD for authoring support in a new `js-regex
+<https://pypi.org/project/js-regex/>`_ library.
+
+
+Running the Test Suite
+----------------------
+
+If you have ``tox`` installed (perhaps via ``pip install tox`` or your
+package manager), running ``tox`` in the directory of your source
+checkout will run ``jsonschema``'s test suite on all of the versions
+of Python ``jsonschema`` supports. If you don't have all of the
+versions that ``jsonschema`` is tested under, you'll likely want to run
+using ``tox``'s ``--skip-missing-interpreters`` option.
+
+Of course you're also free to just run the tests on a single version with your
+favorite test runner. The tests live in the ``jsonschema.tests`` package.
+
+
+Benchmarks
+----------
+
+``jsonschema``'s benchmarks make use of `pyperf
+<https://pyperf.readthedocs.io>`_.
+
+Running them can be done via ``tox -e perf``, or by invoking the ``pyperf``
+commands externally (after ensuring that both it and ``jsonschema`` itself are
+installed)::
+
+    $ python -m pyperf jsonschema/benchmarks/test_suite.py --hist --output results.json
+
+To compare to a previous run, use::
+
+    $ python -m pyperf compare_to --table reference.json results.json
+
+See the ``pyperf`` documentation for more details.
+
+
+Community
+---------
+
+There's a `mailing list <https://groups.google.com/forum/#!forum/jsonschema>`_
+for this implementation on Google Groups.
+
+Please join, and feel free to send questions there.
+
+
+Contributing
+------------
+
+I'm Julian Berman.
+
+``jsonschema`` is on `GitHub <https://github.com/Julian/jsonschema>`_.
+
+Get in touch, via GitHub or otherwise, if you've got something to contribute,
+it'd be most welcome!
+
+You can also generally find me on Freenode (nick: ``tos9``) in various
+channels, including ``#python``.
+
+If you feel overwhelmingly grateful, you can also woo me with beer money
+via Google Pay with the email in my GitHub profile.
+
+And for companies who appreciate ``jsonschema`` and its continued support
+and growth, ``jsonschema`` is also now supportable via `TideLift
+<https://tidelift.com/subscription/pkg/pypi-jsonschema?utm_source=pypi-j
+sonschema&utm_medium=referral&utm_campaign=readme>`_.
diff --git a/contrib/python/jsonschema/py3/jsonschema/__init__.py b/contrib/python/jsonschema/py3/jsonschema/__init__.py
new file mode 100644
index 00000000000..6b630cdfbbe
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/__init__.py
@@ -0,0 +1,34 @@
+"""
+An implementation of JSON Schema for Python
+
+The main functionality is provided by the validator classes for each of the
+supported JSON Schema versions.
+
+Most commonly, `validate` is the quickest way to simply validate a given
+instance under a schema, and will create a validator for you.
+"""
+
+from jsonschema.exceptions import (
+    ErrorTree, FormatError, RefResolutionError, SchemaError, ValidationError
+)
+from jsonschema._format import (
+    FormatChecker,
+    draft3_format_checker,
+    draft4_format_checker,
+    draft6_format_checker,
+    draft7_format_checker,
+)
+from jsonschema._types import TypeChecker
+from jsonschema.validators import (
+    Draft3Validator,
+    Draft4Validator,
+    Draft6Validator,
+    Draft7Validator,
+    RefResolver,
+    validate,
+)
+try:
+    from importlib import metadata
+except ImportError: # for Python<3.8
+    import importlib_metadata as metadata
+__version__ = metadata.version("jsonschema")
diff --git a/contrib/python/jsonschema/py3/jsonschema/__main__.py b/contrib/python/jsonschema/py3/jsonschema/__main__.py
new file mode 100644
index 00000000000..82c29fd39e7
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/__main__.py
@@ -0,0 +1,2 @@
+from jsonschema.cli import main
+main()
diff --git a/contrib/python/jsonschema/py3/jsonschema/_format.py b/contrib/python/jsonschema/py3/jsonschema/_format.py
new file mode 100644
index 00000000000..281a7cfcffe
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/_format.py
@@ -0,0 +1,425 @@
+import datetime
+import re
+import socket
+import struct
+
+from jsonschema.compat import str_types
+from jsonschema.exceptions import FormatError
+
+
+class FormatChecker(object):
+    """
+    A ``format`` property checker.
+
+    JSON Schema does not mandate that the ``format`` property actually do any
+    validation. If validation is desired however, instances of this class can
+    be hooked into validators to enable format validation.
+
+    `FormatChecker` objects always return ``True`` when asked about
+    formats that they do not know how to validate.
+
+    To check a custom format using a function that takes an instance and
+    returns a ``bool``, use the `FormatChecker.checks` or
+    `FormatChecker.cls_checks` decorators.
+
+    Arguments:
+
+        formats (~collections.Iterable):
+
+            The known formats to validate. This argument can be used to
+            limit which formats will be used during validation.
+    """
+
+    checkers = {}
+
+    def __init__(self, formats=None):
+        if formats is None:
+            self.checkers = self.checkers.copy()
+        else:
+            self.checkers = dict((k, self.checkers[k]) for k in formats)
+
+    def __repr__(self):
+        return "<FormatChecker checkers={}>".format(sorted(self.checkers))
+
+    def checks(self, format, raises=()):
+        """
+        Register a decorated function as validating a new format.
+
+        Arguments:
+
+            format (str):
+
+                The format that the decorated function will check.
+
+            raises (Exception):
+
+                The exception(s) raised by the decorated function when an
+                invalid instance is found.
+
+                The exception object will be accessible as the
+                `jsonschema.exceptions.ValidationError.cause` attribute of the
+                resulting validation error.
+        """
+
+        def _checks(func):
+            self.checkers[format] = (func, raises)
+            return func
+        return _checks
+
+    cls_checks = classmethod(checks)
+
+    def check(self, instance, format):
+        """
+        Check whether the instance conforms to the given format.
+
+        Arguments:
+
+            instance (*any primitive type*, i.e. str, number, bool):
+
+                The instance to check
+
+            format (str):
+
+                The format that instance should conform to
+
+
+        Raises:
+
+            FormatError: if the instance does not conform to ``format``
+        """
+
+        if format not in self.checkers:
+            return
+
+        func, raises = self.checkers[format]
+        result, cause = None, None
+        try:
+            result = func(instance)
+        except raises as e:
+            cause = e
+        if not result:
+            raise FormatError(
+                "%r is not a %r" % (instance, format), cause=cause,
+            )
+
+    def conforms(self, instance, format):
+        """
+        Check whether the instance conforms to the given format.
+
+        Arguments:
+
+            instance (*any primitive type*, i.e. str, number, bool):
+
+                The instance to check
+
+            format (str):
+
+                The format that instance should conform to
+
+        Returns:
+
+            bool: whether it conformed
+        """
+
+        try:
+            self.check(instance, format)
+        except FormatError:
+            return False
+        else:
+            return True
+
+
+draft3_format_checker = FormatChecker()
+draft4_format_checker = FormatChecker()
+draft6_format_checker = FormatChecker()
+draft7_format_checker = FormatChecker()
+
+
+_draft_checkers = dict(
+    draft3=draft3_format_checker,
+    draft4=draft4_format_checker,
+    draft6=draft6_format_checker,
+    draft7=draft7_format_checker,
+)
+
+
+def _checks_drafts(
+    name=None,
+    draft3=None,
+    draft4=None,
+    draft6=None,
+    draft7=None,
+    raises=(),
+):
+    draft3 = draft3 or name
+    draft4 = draft4 or name
+    draft6 = draft6 or name
+    draft7 = draft7 or name
+
+    def wrap(func):
+        if draft3:
+            func = _draft_checkers["draft3"].checks(draft3, raises)(func)
+        if draft4:
+            func = _draft_checkers["draft4"].checks(draft4, raises)(func)
+        if draft6:
+            func = _draft_checkers["draft6"].checks(draft6, raises)(func)
+        if draft7:
+            func = _draft_checkers["draft7"].checks(draft7, raises)(func)
+
+        # Oy. This is bad global state, but relied upon for now, until
+        # deprecation. See https://github.com/Julian/jsonschema/issues/519
+        # and test_format_checkers_come_with_defaults
+        FormatChecker.cls_checks(draft7 or draft6 or draft4 or draft3, raises)(
+            func,
+        )
+        return func
+    return wrap
+
+
+@_checks_drafts(name="idn-email")
+@_checks_drafts(name="email")
+def is_email(instance):
+    if not isinstance(instance, str_types):
+        return True
+    return "@" in instance
+
+
+_ipv4_re = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
+
+
+@_checks_drafts(
+    draft3="ip-address", draft4="ipv4", draft6="ipv4", draft7="ipv4",
+)
+def is_ipv4(instance):
+    if not isinstance(instance, str_types):
+        return True
+    if not _ipv4_re.match(instance):
+        return False
+    return all(0 <= int(component) <= 255 for component in instance.split("."))
+
+
+if hasattr(socket, "inet_pton"):
+    # FIXME: Really this only should raise struct.error, but see the sadness
+    #        that is https://twistedmatrix.com/trac/ticket/9409
+    @_checks_drafts(
+        name="ipv6", raises=(socket.error, struct.error, ValueError),
+    )
+    def is_ipv6(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return socket.inet_pton(socket.AF_INET6, instance)
+
+
+_host_name_re = re.compile(r"^[A-Za-z0-9][A-Za-z0-9\.\-]{1,255}$")
+
+
+@_checks_drafts(
+    draft3="host-name",
+    draft4="hostname",
+    draft6="hostname",
+    draft7="hostname",
+)
+def is_host_name(instance):
+    if not isinstance(instance, str_types):
+        return True
+    if not _host_name_re.match(instance):
+        return False
+    components = instance.split(".")
+    for component in components:
+        if len(component) > 63:
+            return False
+    return True
+
+
+try:
+    # The built-in `idna` codec only implements RFC 3890, so we go elsewhere.
+    import idna
+except ImportError:
+    pass
+else:
+    @_checks_drafts(draft7="idn-hostname", raises=idna.IDNAError)
+    def is_idn_host_name(instance):
+        if not isinstance(instance, str_types):
+            return True
+        idna.encode(instance)
+        return True
+
+
+try:
+    import rfc3987
+except ImportError:
+    try:
+        from rfc3986_validator import validate_rfc3986
+    except ImportError:
+        pass
+    else:
+        @_checks_drafts(name="uri")
+        def is_uri(instance):
+            if not isinstance(instance, str_types):
+                return True
+            return validate_rfc3986(instance, rule="URI")
+
+        @_checks_drafts(
+            draft6="uri-reference",
+            draft7="uri-reference",
+            raises=ValueError,
+        )
+        def is_uri_reference(instance):
+            if not isinstance(instance, str_types):
+                return True
+            return validate_rfc3986(instance, rule="URI_reference")
+
+else:
+    @_checks_drafts(draft7="iri", raises=ValueError)
+    def is_iri(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return rfc3987.parse(instance, rule="IRI")
+
+    @_checks_drafts(draft7="iri-reference", raises=ValueError)
+    def is_iri_reference(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return rfc3987.parse(instance, rule="IRI_reference")
+
+    @_checks_drafts(name="uri", raises=ValueError)
+    def is_uri(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return rfc3987.parse(instance, rule="URI")
+
+    @_checks_drafts(
+        draft6="uri-reference",
+        draft7="uri-reference",
+        raises=ValueError,
+    )
+    def is_uri_reference(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return rfc3987.parse(instance, rule="URI_reference")
+
+
+try:
+    from strict_rfc3339 import validate_rfc3339
+except ImportError:
+    try:
+        from rfc3339_validator import validate_rfc3339
+    except ImportError:
+        validate_rfc3339 = None
+
+if validate_rfc3339:
+    @_checks_drafts(name="date-time")
+    def is_datetime(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return validate_rfc3339(instance)
+
+    @_checks_drafts(draft7="time")
+    def is_time(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return is_datetime("1970-01-01T" + instance)
+
+
+@_checks_drafts(name="regex", raises=re.error)
+def is_regex(instance):
+    if not isinstance(instance, str_types):
+        return True
+    return re.compile(instance)
+
+
+@_checks_drafts(draft3="date", draft7="date", raises=ValueError)
+def is_date(instance):
+    if not isinstance(instance, str_types):
+        return True
+    return datetime.datetime.strptime(instance, "%Y-%m-%d")
+
+
+@_checks_drafts(draft3="time", raises=ValueError)
+def is_draft3_time(instance):
+    if not isinstance(instance, str_types):
+        return True
+    return datetime.datetime.strptime(instance, "%H:%M:%S")
+
+
+try:
+    import webcolors
+except ImportError:
+    pass
+else:
+    def is_css_color_code(instance):
+        return webcolors.normalize_hex(instance)
+
+    @_checks_drafts(draft3="color", raises=(ValueError, TypeError))
+    def is_css21_color(instance):
+        if (
+            not isinstance(instance, str_types) or
+            instance.lower() in webcolors.css21_names_to_hex
+        ):
+            return True
+        return is_css_color_code(instance)
+
+    def is_css3_color(instance):
+        if instance.lower() in webcolors.css3_names_to_hex:
+            return True
+        return is_css_color_code(instance)
+
+
+try:
+    import jsonpointer
+except ImportError:
+    pass
+else:
+    @_checks_drafts(
+        draft6="json-pointer",
+        draft7="json-pointer",
+        raises=jsonpointer.JsonPointerException,
+    )
+    def is_json_pointer(instance):
+        if not isinstance(instance, str_types):
+            return True
+        return jsonpointer.JsonPointer(instance)
+
+    # TODO: I don't want to maintain this, so it
+    #       needs to go either into jsonpointer (pending
+    #       https://github.com/stefankoegl/python-json-pointer/issues/34) or
+    #       into a new external library.
+    @_checks_drafts(
+        draft7="relative-json-pointer",
+        raises=jsonpointer.JsonPointerException,
+    )
+    def is_relative_json_pointer(instance):
+        # Definition taken from:
+        # https://tools.ietf.org/html/draft-handrews-relative-json-pointer-01#section-3
+        if not isinstance(instance, str_types):
+            return True
+        non_negative_integer, rest = [], ""
+        for i, character in enumerate(instance):
+            if character.isdigit():
+                non_negative_integer.append(character)
+                continue
+
+            if not non_negative_integer:
+                return False
+
+            rest = instance[i:]
+            break
+        return (rest == "#") or jsonpointer.JsonPointer(rest)
+
+
+try:
+    import uritemplate.exceptions
+except ImportError:
+    pass
+else:
+    @_checks_drafts(
+        draft6="uri-template",
+        draft7="uri-template",
+        raises=uritemplate.exceptions.InvalidTemplate,
+    )
+    def is_uri_template(
+        instance,
+        template_validator=uritemplate.Validator().force_balanced_braces(),
+    ):
+        template = uritemplate.URITemplate(instance)
+        return template_validator.validate(template)
diff --git a/contrib/python/jsonschema/py3/jsonschema/_legacy_validators.py b/contrib/python/jsonschema/py3/jsonschema/_legacy_validators.py
new file mode 100644
index 00000000000..264ff7d7135
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/_legacy_validators.py
@@ -0,0 +1,141 @@
+from jsonschema import _utils
+from jsonschema.compat import iteritems
+from jsonschema.exceptions import ValidationError
+
+
+def dependencies_draft3(validator, dependencies, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property, dependency in iteritems(dependencies):
+        if property not in instance:
+            continue
+
+        if validator.is_type(dependency, "object"):
+            for error in validator.descend(
+                instance, dependency, schema_path=property,
+            ):
+                yield error
+        elif validator.is_type(dependency, "string"):
+            if dependency not in instance:
+                yield ValidationError(
+                    "%r is a dependency of %r" % (dependency, property)
+                )
+        else:
+            for each in dependency:
+                if each not in instance:
+                    message = "%r is a dependency of %r"
+                    yield ValidationError(message % (each, property))
+
+
+def disallow_draft3(validator, disallow, instance, schema):
+    for disallowed in _utils.ensure_list(disallow):
+        if validator.is_valid(instance, {"type": [disallowed]}):
+            yield ValidationError(
+                "%r is disallowed for %r" % (disallowed, instance)
+            )
+
+
+def extends_draft3(validator, extends, instance, schema):
+    if validator.is_type(extends, "object"):
+        for error in validator.descend(instance, extends):
+            yield error
+        return
+    for index, subschema in enumerate(extends):
+        for error in validator.descend(instance, subschema, schema_path=index):
+            yield error
+
+
+def items_draft3_draft4(validator, items, instance, schema):
+    if not validator.is_type(instance, "array"):
+        return
+
+    if validator.is_type(items, "object"):
+        for index, item in enumerate(instance):
+            for error in validator.descend(item, items, path=index):
+                yield error
+    else:
+        for (index, item), subschema in zip(enumerate(instance), items):
+            for error in validator.descend(
+                item, subschema, path=index, schema_path=index,
+            ):
+                yield error
+
+
+def minimum_draft3_draft4(validator, minimum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if schema.get("exclusiveMinimum", False):
+        failed = instance <= minimum
+        cmp = "less than or equal to"
+    else:
+        failed = instance < minimum
+        cmp = "less than"
+
+    if failed:
+        yield ValidationError(
+            "%r is %s the minimum of %r" % (instance, cmp, minimum)
+        )
+
+
+def maximum_draft3_draft4(validator, maximum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if schema.get("exclusiveMaximum", False):
+        failed = instance >= maximum
+        cmp = "greater than or equal to"
+    else:
+        failed = instance > maximum
+        cmp = "greater than"
+
+    if failed:
+        yield ValidationError(
+            "%r is %s the maximum of %r" % (instance, cmp, maximum)
+        )
+
+
+def properties_draft3(validator, properties, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property, subschema in iteritems(properties):
+        if property in instance:
+            for error in validator.descend(
+                instance[property],
+                subschema,
+                path=property,
+                schema_path=property,
+            ):
+                yield error
+        elif subschema.get("required", False):
+            error = ValidationError("%r is a required property" % property)
+            error._set(
+                validator="required",
+                validator_value=subschema["required"],
+                instance=instance,
+                schema=schema,
+            )
+            error.path.appendleft(property)
+            error.schema_path.extend([property, "required"])
+            yield error
+
+
+def type_draft3(validator, types, instance, schema):
+    types = _utils.ensure_list(types)
+
+    all_errors = []
+    for index, type in enumerate(types):
+        if validator.is_type(type, "object"):
+            errors = list(validator.descend(instance, type, schema_path=index))
+            if not errors:
+                return
+            all_errors.extend(errors)
+        else:
+            if validator.is_type(instance, type):
+                return
+    else:
+        yield ValidationError(
+            _utils.types_msg(instance, types), context=all_errors,
+        )
diff --git a/contrib/python/jsonschema/py3/jsonschema/_reflect.py b/contrib/python/jsonschema/py3/jsonschema/_reflect.py
new file mode 100644
index 00000000000..d09e38fbdcf
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/_reflect.py
@@ -0,0 +1,155 @@
+# -*- test-case-name: twisted.test.test_reflect -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Standardized versions of various cool and/or strange things that you can do
+with Python's reflection capabilities.
+"""
+
+import sys
+
+from jsonschema.compat import PY3
+
+
+class _NoModuleFound(Exception):
+    """
+    No module was found because none exists.
+    """
+
+
+
+class InvalidName(ValueError):
+    """
+    The given name is not a dot-separated list of Python objects.
+    """
+
+
+
+class ModuleNotFound(InvalidName):
+    """
+    The module associated with the given name doesn't exist and it can't be
+    imported.
+    """
+
+
+
+class ObjectNotFound(InvalidName):
+    """
+    The object associated with the given name doesn't exist and it can't be
+    imported.
+    """
+
+
+
+if PY3:
+    def reraise(exception, traceback):
+        raise exception.with_traceback(traceback)
+else:
+    exec("""def reraise(exception, traceback):
+        raise exception.__class__, exception, traceback""")
+
+reraise.__doc__ = """
+Re-raise an exception, with an optional traceback, in a way that is compatible
+with both Python 2 and Python 3.
+
+Note that on Python 3, re-raised exceptions will be mutated, with their
+C{__traceback__} attribute being set.
+
+@param exception: The exception instance.
+@param traceback: The traceback to use, or C{None} indicating a new traceback.
+"""
+
+
+def _importAndCheckStack(importName):
+    """
+    Import the given name as a module, then walk the stack to determine whether
+    the failure was the module not existing, or some code in the module (for
+    example a dependent import) failing.  This can be helpful to determine
+    whether any actual application code was run.  For example, to distiguish
+    administrative error (entering the wrong module name), from programmer
+    error (writing buggy code in a module that fails to import).
+
+    @param importName: The name of the module to import.
+    @type importName: C{str}
+    @raise Exception: if something bad happens.  This can be any type of
+        exception, since nobody knows what loading some arbitrary code might
+        do.
+    @raise _NoModuleFound: if no module was found.
+    """
+    try:
+        return __import__(importName)
+    except ImportError:
+        excType, excValue, excTraceback = sys.exc_info()
+        while excTraceback:
+            execName = excTraceback.tb_frame.f_globals["__name__"]
+            # in Python 2 execName is None when an ImportError is encountered,
+            # where in Python 3 execName is equal to the importName.
+            if execName is None or execName == importName:
+                reraise(excValue, excTraceback)
+            excTraceback = excTraceback.tb_next
+        raise _NoModuleFound()
+
+
+
+def namedAny(name):
+    """
+    Retrieve a Python object by its fully qualified name from the global Python
+    module namespace.  The first part of the name, that describes a module,
+    will be discovered and imported.  Each subsequent part of the name is
+    treated as the name of an attribute of the object specified by all of the
+    name which came before it.  For example, the fully-qualified name of this
+    object is 'twisted.python.reflect.namedAny'.
+
+    @type name: L{str}
+    @param name: The name of the object to return.
+
+    @raise InvalidName: If the name is an empty string, starts or ends with
+        a '.', or is otherwise syntactically incorrect.
+
+    @raise ModuleNotFound: If the name is syntactically correct but the
+        module it specifies cannot be imported because it does not appear to
+        exist.
+
+    @raise ObjectNotFound: If the name is syntactically correct, includes at
+        least one '.', but the module it specifies cannot be imported because
+        it does not appear to exist.
+
+    @raise AttributeError: If an attribute of an object along the way cannot be
+        accessed, or a module along the way is not found.
+
+    @return: the Python object identified by 'name'.
+    """
+    if not name:
+        raise InvalidName('Empty module name')
+
+    names = name.split('.')
+
+    # if the name starts or ends with a '.' or contains '..', the __import__
+    # will raise an 'Empty module name' error. This will provide a better error
+    # message.
+    if '' in names:
+        raise InvalidName(
+            "name must be a string giving a '.'-separated list of Python "
+            "identifiers, not %r" % (name,))
+
+    topLevelPackage = None
+    moduleNames = names[:]
+    while not topLevelPackage:
+        if moduleNames:
+            trialname = '.'.join(moduleNames)
+            try:
+                topLevelPackage = _importAndCheckStack(trialname)
+            except _NoModuleFound:
+                moduleNames.pop()
+        else:
+            if len(names) == 1:
+                raise ModuleNotFound("No module named %r" % (name,))
+            else:
+                raise ObjectNotFound('%r does not name an object' % (name,))
+
+    obj = topLevelPackage
+    for n in names[1:]:
+        obj = getattr(obj, n)
+
+    return obj
diff --git a/contrib/python/jsonschema/py3/jsonschema/_types.py b/contrib/python/jsonschema/py3/jsonschema/_types.py
new file mode 100644
index 00000000000..a71a4e34bdc
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/_types.py
@@ -0,0 +1,188 @@
+import numbers
+
+from pyrsistent import pmap
+import attr
+
+from jsonschema.compat import int_types, str_types
+from jsonschema.exceptions import UndefinedTypeCheck
+
+
+def is_array(checker, instance):
+    return isinstance(instance, list)
+
+
+def is_bool(checker, instance):
+    return isinstance(instance, bool)
+
+
+def is_integer(checker, instance):
+    # bool inherits from int, so ensure bools aren't reported as ints
+    if isinstance(instance, bool):
+        return False
+    return isinstance(instance, int_types)
+
+
+def is_null(checker, instance):
+    return instance is None
+
+
+def is_number(checker, instance):
+    # bool inherits from int, so ensure bools aren't reported as ints
+    if isinstance(instance, bool):
+        return False
+    return isinstance(instance, numbers.Number)
+
+
+def is_object(checker, instance):
+    return isinstance(instance, dict)
+
+
+def is_string(checker, instance):
+    return isinstance(instance, str_types)
+
+
+def is_any(checker, instance):
+    return True
+
+
+@attr.s(frozen=True)
+class TypeChecker(object):
+    """
+    A ``type`` property checker.
+
+    A `TypeChecker` performs type checking for an `IValidator`. Type
+    checks to perform are updated using `TypeChecker.redefine` or
+    `TypeChecker.redefine_many` and removed via `TypeChecker.remove`.
+    Each of these return a new `TypeChecker` object.
+
+    Arguments:
+
+        type_checkers (dict):
+
+            The initial mapping of types to their checking functions.
+    """
+    _type_checkers = attr.ib(default=pmap(), converter=pmap)
+
+    def is_type(self, instance, type):
+        """
+        Check if the instance is of the appropriate type.
+
+        Arguments:
+
+            instance (object):
+
+                The instance to check
+
+            type (str):
+
+                The name of the type that is expected.
+
+        Returns:
+
+            bool: Whether it conformed.
+
+
+        Raises:
+
+            `jsonschema.exceptions.UndefinedTypeCheck`:
+                if type is unknown to this object.
+        """
+        try:
+            fn = self._type_checkers[type]
+        except KeyError:
+            raise UndefinedTypeCheck(type)
+
+        return fn(self, instance)
+
+    def redefine(self, type, fn):
+        """
+        Produce a new checker with the given type redefined.
+
+        Arguments:
+
+            type (str):
+
+                The name of the type to check.
+
+            fn (collections.Callable):
+
+                A function taking exactly two parameters - the type
+                checker calling the function and the instance to check.
+                The function should return true if instance is of this
+                type and false otherwise.
+
+        Returns:
+
+            A new `TypeChecker` instance.
+        """
+        return self.redefine_many({type: fn})
+
+    def redefine_many(self, definitions=()):
+        """
+        Produce a new checker with the given types redefined.
+
+        Arguments:
+
+            definitions (dict):
+
+                A dictionary mapping types to their checking functions.
+
+        Returns:
+
+            A new `TypeChecker` instance.
+        """
+        return attr.evolve(
+            self, type_checkers=self._type_checkers.update(definitions),
+        )
+
+    def remove(self, *types):
+        """
+        Produce a new checker with the given types forgotten.
+
+        Arguments:
+
+            types (~collections.Iterable):
+
+                the names of the types to remove.
+
+        Returns:
+
+            A new `TypeChecker` instance
+
+        Raises:
+
+            `jsonschema.exceptions.UndefinedTypeCheck`:
+
+                if any given type is unknown to this object
+        """
+
+        checkers = self._type_checkers
+        for each in types:
+            try:
+                checkers = checkers.remove(each)
+            except KeyError:
+                raise UndefinedTypeCheck(each)
+        return attr.evolve(self, type_checkers=checkers)
+
+
+draft3_type_checker = TypeChecker(
+    {
+        u"any": is_any,
+        u"array": is_array,
+        u"boolean": is_bool,
+        u"integer": is_integer,
+        u"object": is_object,
+        u"null": is_null,
+        u"number": is_number,
+        u"string": is_string,
+    },
+)
+draft4_type_checker = draft3_type_checker.remove(u"any")
+draft6_type_checker = draft4_type_checker.redefine(
+    u"integer",
+    lambda checker, instance: (
+        is_integer(checker, instance) or
+        isinstance(instance, float) and instance.is_integer()
+    ),
+)
+draft7_type_checker = draft6_type_checker
diff --git a/contrib/python/jsonschema/py3/jsonschema/_utils.py b/contrib/python/jsonschema/py3/jsonschema/_utils.py
new file mode 100644
index 00000000000..ceb880198d1
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/_utils.py
@@ -0,0 +1,212 @@
+import itertools
+import json
+import pkgutil
+import re
+
+from jsonschema.compat import MutableMapping, str_types, urlsplit
+
+
+class URIDict(MutableMapping):
+    """
+    Dictionary which uses normalized URIs as keys.
+    """
+
+    def normalize(self, uri):
+        return urlsplit(uri).geturl()
+
+    def __init__(self, *args, **kwargs):
+        self.store = dict()
+        self.store.update(*args, **kwargs)
+
+    def __getitem__(self, uri):
+        return self.store[self.normalize(uri)]
+
+    def __setitem__(self, uri, value):
+        self.store[self.normalize(uri)] = value
+
+    def __delitem__(self, uri):
+        del self.store[self.normalize(uri)]
+
+    def __iter__(self):
+        return iter(self.store)
+
+    def __len__(self):
+        return len(self.store)
+
+    def __repr__(self):
+        return repr(self.store)
+
+
+class Unset(object):
+    """
+    An as-of-yet unset attribute or unprovided default parameter.
+    """
+
+    def __repr__(self):
+        return "<unset>"
+
+
+def load_schema(name):
+    """
+    Load a schema from ./schemas/``name``.json and return it.
+    """
+
+    data = pkgutil.get_data("jsonschema", "schemas/{0}.json".format(name))
+    return json.loads(data.decode("utf-8"))
+
+
+def indent(string, times=1):
+    """
+    A dumb version of `textwrap.indent` from Python 3.3.
+    """
+
+    return "\n".join(" " * (4 * times) + line for line in string.splitlines())
+
+
+def format_as_index(indices):
+    """
+    Construct a single string containing indexing operations for the indices.
+
+    For example, [1, 2, "foo"] -> [1][2]["foo"]
+
+    Arguments:
+
+        indices (sequence):
+
+            The indices to format.
+    """
+
+    if not indices:
+        return ""
+    return "[%s]" % "][".join(repr(index) for index in indices)
+
+
+def find_additional_properties(instance, schema):
+    """
+    Return the set of additional properties for the given ``instance``.
+
+    Weeds out properties that should have been validated by ``properties`` and
+    / or ``patternProperties``.
+
+    Assumes ``instance`` is dict-like already.
+    """
+
+    properties = schema.get("properties", {})
+    patterns = "|".join(schema.get("patternProperties", {}))
+    for property in instance:
+        if property not in properties:
+            if patterns and re.search(patterns, property):
+                continue
+            yield property
+
+
+def extras_msg(extras):
+    """
+    Create an error message for extra items or properties.
+    """
+
+    if len(extras) == 1:
+        verb = "was"
+    else:
+        verb = "were"
+    return ", ".join(repr(extra) for extra in extras), verb
+
+
+def types_msg(instance, types):
+    """
+    Create an error message for a failure to match the given types.
+
+    If the ``instance`` is an object and contains a ``name`` property, it will
+    be considered to be a description of that object and used as its type.
+
+    Otherwise the message is simply the reprs of the given ``types``.
+    """
+
+    reprs = []
+    for type in types:
+        try:
+            reprs.append(repr(type["name"]))
+        except Exception:
+            reprs.append(repr(type))
+    return "%r is not of type %s" % (instance, ", ".join(reprs))
+
+
+def flatten(suitable_for_isinstance):
+    """
+    isinstance() can accept a bunch of really annoying different types:
+        * a single type
+        * a tuple of types
+        * an arbitrary nested tree of tuples
+
+    Return a flattened tuple of the given argument.
+    """
+
+    types = set()
+
+    if not isinstance(suitable_for_isinstance, tuple):
+        suitable_for_isinstance = (suitable_for_isinstance,)
+    for thing in suitable_for_isinstance:
+        if isinstance(thing, tuple):
+            types.update(flatten(thing))
+        else:
+            types.add(thing)
+    return tuple(types)
+
+
+def ensure_list(thing):
+    """
+    Wrap ``thing`` in a list if it's a single str.
+
+    Otherwise, return it unchanged.
+    """
+
+    if isinstance(thing, str_types):
+        return [thing]
+    return thing
+
+
+def equal(one, two):
+    """
+    Check if two things are equal, but evade booleans and ints being equal.
+    """
+    return unbool(one) == unbool(two)
+
+
+def unbool(element, true=object(), false=object()):
+    """
+    A hack to make True and 1 and False and 0 unique for ``uniq``.
+    """
+
+    if element is True:
+        return true
+    elif element is False:
+        return false
+    return element
+
+
+def uniq(container):
+    """
+    Check if all of a container's elements are unique.
+
+    Successively tries first to rely that the elements are hashable, then
+    falls back on them being sortable, and finally falls back on brute
+    force.
+    """
+
+    try:
+        return len(set(unbool(i) for i in container)) == len(container)
+    except TypeError:
+        try:
+            sort = sorted(unbool(i) for i in container)
+            sliced = itertools.islice(sort, 1, None)
+            for i, j in zip(sort, sliced):
+                if i == j:
+                    return False
+        except (NotImplementedError, TypeError):
+            seen = []
+            for e in container:
+                e = unbool(e)
+                if e in seen:
+                    return False
+                seen.append(e)
+    return True
diff --git a/contrib/python/jsonschema/py3/jsonschema/_validators.py b/contrib/python/jsonschema/py3/jsonschema/_validators.py
new file mode 100644
index 00000000000..179fec09a94
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/_validators.py
@@ -0,0 +1,373 @@
+import re
+
+from jsonschema._utils import (
+    ensure_list,
+    equal,
+    extras_msg,
+    find_additional_properties,
+    types_msg,
+    unbool,
+    uniq,
+)
+from jsonschema.exceptions import FormatError, ValidationError
+from jsonschema.compat import iteritems
+
+
+def patternProperties(validator, patternProperties, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for pattern, subschema in iteritems(patternProperties):
+        for k, v in iteritems(instance):
+            if re.search(pattern, k):
+                for error in validator.descend(
+                    v, subschema, path=k, schema_path=pattern,
+                ):
+                    yield error
+
+
+def propertyNames(validator, propertyNames, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property in instance:
+        for error in validator.descend(
+            instance=property,
+            schema=propertyNames,
+        ):
+            yield error
+
+
+def additionalProperties(validator, aP, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    extras = set(find_additional_properties(instance, schema))
+
+    if validator.is_type(aP, "object"):
+        for extra in extras:
+            for error in validator.descend(instance[extra], aP, path=extra):
+                yield error
+    elif not aP and extras:
+        if "patternProperties" in schema:
+            patterns = sorted(schema["patternProperties"])
+            if len(extras) == 1:
+                verb = "does"
+            else:
+                verb = "do"
+            error = "%s %s not match any of the regexes: %s" % (
+                ", ".join(map(repr, sorted(extras))),
+                verb,
+                ", ".join(map(repr, patterns)),
+            )
+            yield ValidationError(error)
+        else:
+            error = "Additional properties are not allowed (%s %s unexpected)"
+            yield ValidationError(error % extras_msg(extras))
+
+
+def items(validator, items, instance, schema):
+    if not validator.is_type(instance, "array"):
+        return
+
+    if validator.is_type(items, "array"):
+        for (index, item), subschema in zip(enumerate(instance), items):
+            for error in validator.descend(
+                item, subschema, path=index, schema_path=index,
+            ):
+                yield error
+    else:
+        for index, item in enumerate(instance):
+            for error in validator.descend(item, items, path=index):
+                yield error
+
+
+def additionalItems(validator, aI, instance, schema):
+    if (
+        not validator.is_type(instance, "array") or
+        validator.is_type(schema.get("items", {}), "object")
+    ):
+        return
+
+    len_items = len(schema.get("items", []))
+    if validator.is_type(aI, "object"):
+        for index, item in enumerate(instance[len_items:], start=len_items):
+            for error in validator.descend(item, aI, path=index):
+                yield error
+    elif not aI and len(instance) > len(schema.get("items", [])):
+        error = "Additional items are not allowed (%s %s unexpected)"
+        yield ValidationError(
+            error %
+            extras_msg(instance[len(schema.get("items", [])):])
+        )
+
+
+def const(validator, const, instance, schema):
+    if not equal(instance, const):
+        yield ValidationError("%r was expected" % (const,))
+
+
+def contains(validator, contains, instance, schema):
+    if not validator.is_type(instance, "array"):
+        return
+
+    if not any(validator.is_valid(element, contains) for element in instance):
+        yield ValidationError(
+            "None of %r are valid under the given schema" % (instance,)
+        )
+
+
+def exclusiveMinimum(validator, minimum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if instance <= minimum:
+        yield ValidationError(
+            "%r is less than or equal to the minimum of %r" % (
+                instance, minimum,
+            ),
+        )
+
+
+def exclusiveMaximum(validator, maximum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if instance >= maximum:
+        yield ValidationError(
+            "%r is greater than or equal to the maximum of %r" % (
+                instance, maximum,
+            ),
+        )
+
+
+def minimum(validator, minimum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if instance < minimum:
+        yield ValidationError(
+            "%r is less than the minimum of %r" % (instance, minimum)
+        )
+
+
+def maximum(validator, maximum, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if instance > maximum:
+        yield ValidationError(
+            "%r is greater than the maximum of %r" % (instance, maximum)
+        )
+
+
+def multipleOf(validator, dB, instance, schema):
+    if not validator.is_type(instance, "number"):
+        return
+
+    if isinstance(dB, float):
+        quotient = instance / dB
+        failed = int(quotient) != quotient
+    else:
+        failed = instance % dB
+
+    if failed:
+        yield ValidationError("%r is not a multiple of %r" % (instance, dB))
+
+
+def minItems(validator, mI, instance, schema):
+    if validator.is_type(instance, "array") and len(instance) < mI:
+        yield ValidationError("%r is too short" % (instance,))
+
+
+def maxItems(validator, mI, instance, schema):
+    if validator.is_type(instance, "array") and len(instance) > mI:
+        yield ValidationError("%r is too long" % (instance,))
+
+
+def uniqueItems(validator, uI, instance, schema):
+    if (
+        uI and
+        validator.is_type(instance, "array") and
+        not uniq(instance)
+    ):
+        yield ValidationError("%r has non-unique elements" % (instance,))
+
+
+def pattern(validator, patrn, instance, schema):
+    if (
+        validator.is_type(instance, "string") and
+        not re.search(patrn, instance)
+    ):
+        yield ValidationError("%r does not match %r" % (instance, patrn))
+
+
+def format(validator, format, instance, schema):
+    if validator.format_checker is not None:
+        try:
+            validator.format_checker.check(instance, format)
+        except FormatError as error:
+            yield ValidationError(error.message, cause=error.cause)
+
+
+def minLength(validator, mL, instance, schema):
+    if validator.is_type(instance, "string") and len(instance) < mL:
+        yield ValidationError("%r is too short" % (instance,))
+
+
+def maxLength(validator, mL, instance, schema):
+    if validator.is_type(instance, "string") and len(instance) > mL:
+        yield ValidationError("%r is too long" % (instance,))
+
+
+def dependencies(validator, dependencies, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property, dependency in iteritems(dependencies):
+        if property not in instance:
+            continue
+
+        if validator.is_type(dependency, "array"):
+            for each in dependency:
+                if each not in instance:
+                    message = "%r is a dependency of %r"
+                    yield ValidationError(message % (each, property))
+        else:
+            for error in validator.descend(
+                instance, dependency, schema_path=property,
+            ):
+                yield error
+
+
+def enum(validator, enums, instance, schema):
+    if instance == 0 or instance == 1:
+        unbooled = unbool(instance)
+        if all(unbooled != unbool(each) for each in enums):
+            yield ValidationError("%r is not one of %r" % (instance, enums))
+    elif instance not in enums:
+        yield ValidationError("%r is not one of %r" % (instance, enums))
+
+
+def ref(validator, ref, instance, schema):
+    resolve = getattr(validator.resolver, "resolve", None)
+    if resolve is None:
+        with validator.resolver.resolving(ref) as resolved:
+            for error in validator.descend(instance, resolved):
+                yield error
+    else:
+        scope, resolved = validator.resolver.resolve(ref)
+        validator.resolver.push_scope(scope)
+
+        try:
+            for error in validator.descend(instance, resolved):
+                yield error
+        finally:
+            validator.resolver.pop_scope()
+
+
+def type(validator, types, instance, schema):
+    types = ensure_list(types)
+
+    if not any(validator.is_type(instance, type) for type in types):
+        yield ValidationError(types_msg(instance, types))
+
+
+def properties(validator, properties, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+
+    for property, subschema in iteritems(properties):
+        if property in instance:
+            for error in validator.descend(
+                instance[property],
+                subschema,
+                path=property,
+                schema_path=property,
+            ):
+                yield error
+
+
+def required(validator, required, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+    for property in required:
+        if property not in instance:
+            yield ValidationError("%r is a required property" % property)
+
+
+def minProperties(validator, mP, instance, schema):
+    if validator.is_type(instance, "object") and len(instance) < mP:
+        yield ValidationError(
+            "%r does not have enough properties" % (instance,)
+        )
+
+
+def maxProperties(validator, mP, instance, schema):
+    if not validator.is_type(instance, "object"):
+        return
+    if validator.is_type(instance, "object") and len(instance) > mP:
+        yield ValidationError("%r has too many properties" % (instance,))
+
+
+def allOf(validator, allOf, instance, schema):
+    for index, subschema in enumerate(allOf):
+        for error in validator.descend(instance, subschema, schema_path=index):
+            yield error
+
+
+def anyOf(validator, anyOf, instance, schema):
+    all_errors = []
+    for index, subschema in enumerate(anyOf):
+        errs = list(validator.descend(instance, subschema, schema_path=index))
+        if not errs:
+            break
+        all_errors.extend(errs)
+    else:
+        yield ValidationError(
+            "%r is not valid under any of the given schemas" % (instance,),
+            context=all_errors,
+        )
+
+
+def oneOf(validator, oneOf, instance, schema):
+    subschemas = enumerate(oneOf)
+    all_errors = []
+    for index, subschema in subschemas:
+        errs = list(validator.descend(instance, subschema, schema_path=index))
+        if not errs:
+            first_valid = subschema
+            break
+        all_errors.extend(errs)
+    else:
+        yield ValidationError(
+            "%r is not valid under any of the given schemas" % (instance,),
+            context=all_errors,
+        )
+
+    more_valid = [s for i, s in subschemas if validator.is_valid(instance, s)]
+    if more_valid:
+        more_valid.append(first_valid)
+        reprs = ", ".join(repr(schema) for schema in more_valid)
+        yield ValidationError(
+            "%r is valid under each of %s" % (instance, reprs)
+        )
+
+
+def not_(validator, not_schema, instance, schema):
+    if validator.is_valid(instance, not_schema):
+        yield ValidationError(
+            "%r is not allowed for %r" % (not_schema, instance)
+        )
+
+
+def if_(validator, if_schema, instance, schema):
+    if validator.is_valid(instance, if_schema):
+        if u"then" in schema:
+            then = schema[u"then"]
+            for error in validator.descend(instance, then, schema_path="then"):
+                yield error
+    elif u"else" in schema:
+        else_ = schema[u"else"]
+        for error in validator.descend(instance, else_, schema_path="else"):
+            yield error
diff --git a/contrib/python/jsonschema/py3/jsonschema/cli.py b/contrib/python/jsonschema/py3/jsonschema/cli.py
new file mode 100644
index 00000000000..ab3335b27c5
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/cli.py
@@ -0,0 +1,90 @@
+"""
+The ``jsonschema`` command line.
+"""
+from __future__ import absolute_import
+import argparse
+import json
+import sys
+
+from jsonschema import __version__
+from jsonschema._reflect import namedAny
+from jsonschema.validators import validator_for
+
+
+def _namedAnyWithDefault(name):
+    if "." not in name:
+        name = "jsonschema." + name
+    return namedAny(name)
+
+
+def _json_file(path):
+    with open(path) as file:
+        return json.load(file)
+
+
+parser = argparse.ArgumentParser(
+    description="JSON Schema Validation CLI",
+)
+parser.add_argument(
+    "-i", "--instance",
+    action="append",
+    dest="instances",
+    type=_json_file,
+    help=(
+        "a path to a JSON instance (i.e. filename.json) "
+        "to validate (may be specified multiple times)"
+    ),
+)
+parser.add_argument(
+    "-F", "--error-format",
+    default="{error.instance}: {error.message}\n",
+    help=(
+        "the format to use for each error output message, specified in "
+        "a form suitable for passing to str.format, which will be called "
+        "with 'error' for each error"
+    ),
+)
+parser.add_argument(
+    "-V", "--validator",
+    type=_namedAnyWithDefault,
+    help=(
+        "the fully qualified object name of a validator to use, or, for "
+        "validators that are registered with jsonschema, simply the name "
+        "of the class."
+    ),
+)
+parser.add_argument(
+    "--version",
+    action="version",
+    version=__version__,
+)
+parser.add_argument(
+    "schema",
+    help="the JSON Schema to validate with (i.e. schema.json)",
+    type=_json_file,
+)
+
+
+def parse_args(args):
+    arguments = vars(parser.parse_args(args=args or ["--help"]))
+    if arguments["validator"] is None:
+        arguments["validator"] = validator_for(arguments["schema"])
+    return arguments
+
+
+def main(args=sys.argv[1:]):
+    sys.exit(run(arguments=parse_args(args=args)))
+
+
+def run(arguments, stdout=sys.stdout, stderr=sys.stderr):
+    error_format = arguments["error_format"]
+    validator = arguments["validator"](schema=arguments["schema"])
+
+    validator.check_schema(arguments["schema"])
+
+    errored = False
+    for instance in arguments["instances"] or ():
+        for error in validator.iter_errors(instance):
+            stderr.write(error_format.format(error=error))
+            errored = True
+    return errored
diff --git a/contrib/python/jsonschema/py3/jsonschema/compat.py b/contrib/python/jsonschema/py3/jsonschema/compat.py
new file mode 100644
index 00000000000..47e09804551
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/compat.py
@@ -0,0 +1,55 @@
+"""
+Python 2/3 compatibility helpers.
+
+Note: This module is *not* public API.
+"""
+import contextlib
+import operator
+import sys
+
+
+try:
+    from collections.abc import MutableMapping, Sequence  # noqa
+except ImportError:
+    from collections import MutableMapping, Sequence  # noqa
+
+PY3 = sys.version_info[0] >= 3
+
+if PY3:
+    zip = zip
+    from functools import lru_cache
+    from io import StringIO as NativeIO
+    from urllib.parse import (
+        unquote, urljoin, urlunsplit, SplitResult, urlsplit
+    )
+    from urllib.request import pathname2url, urlopen
+    str_types = str,
+    int_types = int,
+    iteritems = operator.methodcaller("items")
+else:
+    from itertools import izip as zip  # noqa
+    from io import BytesIO as NativeIO
+    from urlparse import urljoin, urlunsplit, SplitResult, urlsplit
+    from urllib import pathname2url, unquote  # noqa
+    import urllib2  # noqa
+    def urlopen(*args, **kwargs):
+        return contextlib.closing(urllib2.urlopen(*args, **kwargs))
+
+    str_types = basestring
+    int_types = int, long
+    iteritems = operator.methodcaller("iteritems")
+
+    from functools32 import lru_cache
+
+
+def urldefrag(url):
+    if "#" in url:
+        s, n, p, q, frag = urlsplit(url)
+        defrag = urlunsplit((s, n, p, q, ""))
+    else:
+        defrag = url
+        frag = ""
+    return defrag, frag
+
+
+# flake8: noqa
diff --git a/contrib/python/jsonschema/py3/jsonschema/exceptions.py b/contrib/python/jsonschema/py3/jsonschema/exceptions.py
new file mode 100644
index 00000000000..691dcffe6c7
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/exceptions.py
@@ -0,0 +1,374 @@
+"""
+Validation errors, and some surrounding helpers.
+"""
+from collections import defaultdict, deque
+import itertools
+import pprint
+import textwrap
+
+import attr
+
+from jsonschema import _utils
+from jsonschema.compat import PY3, iteritems
+
+
+WEAK_MATCHES = frozenset(["anyOf", "oneOf"])
+STRONG_MATCHES = frozenset()
+
+_unset = _utils.Unset()
+
+
+class _Error(Exception):
+    def __init__(
+        self,
+        message,
+        validator=_unset,
+        path=(),
+        cause=None,
+        context=(),
+        validator_value=_unset,
+        instance=_unset,
+        schema=_unset,
+        schema_path=(),
+        parent=None,
+    ):
+        super(_Error, self).__init__(
+            message,
+            validator,
+            path,
+            cause,
+            context,
+            validator_value,
+            instance,
+            schema,
+            schema_path,
+            parent,
+        )
+        self.message = message
+        self.path = self.relative_path = deque(path)
+        self.schema_path = self.relative_schema_path = deque(schema_path)
+        self.context = list(context)
+        self.cause = self.__cause__ = cause
+        self.validator = validator
+        self.validator_value = validator_value
+        self.instance = instance
+        self.schema = schema
+        self.parent = parent
+
+        for error in context:
+            error.parent = self
+
+    def __repr__(self):
+        return "<%s: %r>" % (self.__class__.__name__, self.message)
+
+    def __unicode__(self):
+        essential_for_verbose = (
+            self.validator, self.validator_value, self.instance, self.schema,
+        )
+        if any(m is _unset for m in essential_for_verbose):
+            return self.message
+
+        pschema = pprint.pformat(self.schema, width=72)
+        pinstance = pprint.pformat(self.instance, width=72)
+        return self.message + textwrap.dedent("""
+
+            Failed validating %r in %s%s:
+            %s
+
+            On %s%s:
+            %s
+            """.rstrip()
+        ) % (
+            self.validator,
+            self._word_for_schema_in_error_message,
+            _utils.format_as_index(list(self.relative_schema_path)[:-1]),
+            _utils.indent(pschema),
+            self._word_for_instance_in_error_message,
+            _utils.format_as_index(self.relative_path),
+            _utils.indent(pinstance),
+        )
+
+    if PY3:
+        __str__ = __unicode__
+    else:
+        def __str__(self):
+            return unicode(self).encode("utf-8")
+
+    @classmethod
+    def create_from(cls, other):
+        return cls(**other._contents())
+
+    @property
+    def absolute_path(self):
+        parent = self.parent
+        if parent is None:
+            return self.relative_path
+
+        path = deque(self.relative_path)
+        path.extendleft(reversed(parent.absolute_path))
+        return path
+
+    @property
+    def absolute_schema_path(self):
+        parent = self.parent
+        if parent is None:
+            return self.relative_schema_path
+
+        path = deque(self.relative_schema_path)
+        path.extendleft(reversed(parent.absolute_schema_path))
+        return path
+
+    def _set(self, **kwargs):
+        for k, v in iteritems(kwargs):
+            if getattr(self, k) is _unset:
+                setattr(self, k, v)
+
+    def _contents(self):
+        attrs = (
+            "message", "cause", "context", "validator", "validator_value",
+            "path", "schema_path", "instance", "schema", "parent",
+        )
+        return dict((attr, getattr(self, attr)) for attr in attrs)
+
+
+class ValidationError(_Error):
+    """
+    An instance was invalid under a provided schema.
+    """
+
+    _word_for_schema_in_error_message = "schema"
+    _word_for_instance_in_error_message = "instance"
+
+
+class SchemaError(_Error):
+    """
+    A schema was invalid under its corresponding metaschema.
+    """
+
+    _word_for_schema_in_error_message = "metaschema"
+    _word_for_instance_in_error_message = "schema"
+
+
+@attr.s(hash=True)
+class RefResolutionError(Exception):
+    """
+    A ref could not be resolved.
+    """
+
+    _cause = attr.ib()
+
+    def __str__(self):
+        return str(self._cause)
+
+
+class UndefinedTypeCheck(Exception):
+    """
+    A type checker was asked to check a type it did not have registered.
+    """
+
+    def __init__(self, type):
+        self.type = type
+
+    def __unicode__(self):
+        return "Type %r is unknown to this type checker" % self.type
+
+    if PY3:
+        __str__ = __unicode__
+    else:
+        def __str__(self):
+            return unicode(self).encode("utf-8")
+
+
+class UnknownType(Exception):
+    """
+    A validator was asked to validate an instance against an unknown type.
+    """
+
+    def __init__(self, type, instance, schema):
+        self.type = type
+        self.instance = instance
+        self.schema = schema
+
+    def __unicode__(self):
+        pschema = pprint.pformat(self.schema, width=72)
+        pinstance = pprint.pformat(self.instance, width=72)
+        return textwrap.dedent("""
+            Unknown type %r for validator with schema:
+            %s
+
+            While checking instance:
+            %s
+            """.rstrip()
+        ) % (self.type, _utils.indent(pschema), _utils.indent(pinstance))
+
+    if PY3:
+        __str__ = __unicode__
+    else:
+        def __str__(self):
+            return unicode(self).encode("utf-8")
+
+
+class FormatError(Exception):
+    """
+    Validating a format failed.
+    """
+
+    def __init__(self, message, cause=None):
+        super(FormatError, self).__init__(message, cause)
+        self.message = message
+        self.cause = self.__cause__ = cause
+
+    def __unicode__(self):
+        return self.message
+
+    if PY3:
+        __str__ = __unicode__
+    else:
+        def __str__(self):
+            return self.message.encode("utf-8")
+
+
+class ErrorTree(object):
+    """
+    ErrorTrees make it easier to check which validations failed.
+    """
+
+    _instance = _unset
+
+    def __init__(self, errors=()):
+        self.errors = {}
+        self._contents = defaultdict(self.__class__)
+
+        for error in errors:
+            container = self
+            for element in error.path:
+                container = container[element]
+            container.errors[error.validator] = error
+
+            container._instance = error.instance
+
+    def __contains__(self, index):
+        """
+        Check whether ``instance[index]`` has any errors.
+        """
+
+        return index in self._contents
+
+    def __getitem__(self, index):
+        """
+        Retrieve the child tree one level down at the given ``index``.
+
+        If the index is not in the instance that this tree corresponds to and
+        is not known by this tree, whatever error would be raised by
+        ``instance.__getitem__`` will be propagated (usually this is some
+        subclass of `exceptions.LookupError`.
+        """
+
+        if self._instance is not _unset and index not in self:
+            self._instance[index]
+        return self._contents[index]
+
+    def __setitem__(self, index, value):
+        """
+        Add an error to the tree at the given ``index``.
+        """
+        self._contents[index] = value
+
+    def __iter__(self):
+        """
+        Iterate (non-recursively) over the indices in the instance with errors.
+        """
+
+        return iter(self._contents)
+
+    def __len__(self):
+        """
+        Return the `total_errors`.
+        """
+        return self.total_errors
+
+    def __repr__(self):
+        return "<%s (%s total errors)>" % (self.__class__.__name__, len(self))
+
+    @property
+    def total_errors(self):
+        """
+        The total number of errors in the entire tree, including children.
+        """
+
+        child_errors = sum(len(tree) for _, tree in iteritems(self._contents))
+        return len(self.errors) + child_errors
+
+
+def by_relevance(weak=WEAK_MATCHES, strong=STRONG_MATCHES):
+    """
+    Create a key function that can be used to sort errors by relevance.
+
+    Arguments:
+        weak (set):
+            a collection of validator names to consider to be "weak".
+            If there are two errors at the same level of the instance
+            and one is in the set of weak validator names, the other
+            error will take priority. By default, :validator:`anyOf` and
+            :validator:`oneOf` are considered weak validators and will
+            be superseded by other same-level validation errors.
+
+        strong (set):
+            a collection of validator names to consider to be "strong"
+    """
+    def relevance(error):
+        validator = error.validator
+        return -len(error.path), validator not in weak, validator in strong
+    return relevance
+
+
+relevance = by_relevance()
+
+
+def best_match(errors, key=relevance):
+    """
+    Try to find an error that appears to be the best match among given errors.
+
+    In general, errors that are higher up in the instance (i.e. for which
+    `ValidationError.path` is shorter) are considered better matches,
+    since they indicate "more" is wrong with the instance.
+
+    If the resulting match is either :validator:`oneOf` or :validator:`anyOf`,
+    the *opposite* assumption is made -- i.e. the deepest error is picked,
+    since these validators only need to match once, and any other errors may
+    not be relevant.
+
+    Arguments:
+        errors (collections.Iterable):
+
+            the errors to select from. Do not provide a mixture of
+            errors from different validation attempts (i.e. from
+            different instances or schemas), since it won't produce
+            sensical output.
+
+        key (collections.Callable):
+
+            the key to use when sorting errors. See `relevance` and
+            transitively `by_relevance` for more details (the default is
+            to sort with the defaults of that function). Changing the
+            default is only useful if you want to change the function
+            that rates errors but still want the error context descent
+            done by this function.
+
+    Returns:
+        the best matching error, or ``None`` if the iterable was empty
+
+    .. note::
+
+        This function is a heuristic. Its return value may change for a given
+        set of inputs from version to version if better heuristics are added.
+    """
+    errors = iter(errors)
+    best = next(errors, None)
+    if best is None:
+        return
+    best = max(itertools.chain([best], errors), key=key)
+
+    while best.context:
+        best = min(best.context, key=key)
+    return best
diff --git a/contrib/python/jsonschema/py3/jsonschema/schemas/draft3.json b/contrib/python/jsonschema/py3/jsonschema/schemas/draft3.json
new file mode 100644
index 00000000000..f8a09c563b4
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/schemas/draft3.json
@@ -0,0 +1,199 @@
+{
+    "$schema": "http://json-schema.org/draft-03/schema#",
+    "dependencies": {
+        "exclusiveMaximum": "maximum",
+        "exclusiveMinimum": "minimum"
+    },
+    "id": "http://json-schema.org/draft-03/schema#",
+    "properties": {
+        "$ref": {
+            "format": "uri",
+            "type": "string"
+        },
+        "$schema": {
+            "format": "uri",
+            "type": "string"
+        },
+        "additionalItems": {
+            "default": {},
+            "type": [
+                {
+                    "$ref": "#"
+                },
+                "boolean"
+            ]
+        },
+        "additionalProperties": {
+            "default": {},
+            "type": [
+                {
+                    "$ref": "#"
+                },
+                "boolean"
+            ]
+        },
+        "default": {
+            "type": "any"
+        },
+        "dependencies": {
+            "additionalProperties": {
+                "items": {
+                    "type": "string"
+                },
+                "type": [
+                    "string",
+                    "array",
+                    {
+                        "$ref": "#"
+                    }
+                ]
+            },
+            "default": {},
+            "type": [
+                "string",
+                "array",
+                "object"
+            ]
+        },
+        "description": {
+            "type": "string"
+        },
+        "disallow": {
+            "items": {
+                "type": [
+                    "string",
+                    {
+                        "$ref": "#"
+                    }
+                ]
+            },
+            "type": [
+                "string",
+                "array"
+            ],
+            "uniqueItems": true
+        },
+        "divisibleBy": {
+            "default": 1,
+            "exclusiveMinimum": true,
+            "minimum": 0,
+            "type": "number"
+        },
+        "enum": {
+            "type": "array"
+        },
+        "exclusiveMaximum": {
+            "default": false,
+            "type": "boolean"
+        },
+        "exclusiveMinimum": {
+            "default": false,
+            "type": "boolean"
+        },
+        "extends": {
+            "default": {},
+            "items": {
+                "$ref": "#"
+            },
+            "type": [
+                {
+                    "$ref": "#"
+                },
+                "array"
+            ]
+        },
+        "format": {
+            "type": "string"
+        },
+        "id": {
+            "format": "uri",
+            "type": "string"
+        },
+        "items": {
+            "default": {},
+            "items": {
+                "$ref": "#"
+            },
+            "type": [
+                {
+                    "$ref": "#"
+                },
+                "array"
+            ]
+        },
+        "maxDecimal": {
+            "minimum": 0,
+            "type": "number"
+        },
+        "maxItems": {
+            "minimum": 0,
+            "type": "integer"
+        },
+        "maxLength": {
+            "type": "integer"
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "minItems": {
+            "default": 0,
+            "minimum": 0,
+            "type": "integer"
+        },
+        "minLength": {
+            "default": 0,
+            "minimum": 0,
+            "type": "integer"
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "pattern": {
+            "format": "regex",
+            "type": "string"
+        },
+        "patternProperties": {
+            "additionalProperties": {
+                "$ref": "#"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "properties": {
+            "additionalProperties": {
+                "$ref": "#",
+                "type": "object"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "required": {
+            "default": false,
+            "type": "boolean"
+        },
+        "title": {
+            "type": "string"
+        },
+        "type": {
+            "default": "any",
+            "items": {
+                "type": [
+                    "string",
+                    {
+                        "$ref": "#"
+                    }
+                ]
+            },
+            "type": [
+                "string",
+                "array"
+            ],
+            "uniqueItems": true
+        },
+        "uniqueItems": {
+            "default": false,
+            "type": "boolean"
+        }
+    },
+    "type": "object"
+}
diff --git a/contrib/python/jsonschema/py3/jsonschema/schemas/draft4.json b/contrib/python/jsonschema/py3/jsonschema/schemas/draft4.json
new file mode 100644
index 00000000000..9b666cff88a
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/schemas/draft4.json
@@ -0,0 +1,222 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "default": {},
+    "definitions": {
+        "positiveInteger": {
+            "minimum": 0,
+            "type": "integer"
+        },
+        "positiveIntegerDefault0": {
+            "allOf": [
+                {
+                    "$ref": "#/definitions/positiveInteger"
+                },
+                {
+                    "default": 0
+                }
+            ]
+        },
+        "schemaArray": {
+            "items": {
+                "$ref": "#"
+            },
+            "minItems": 1,
+            "type": "array"
+        },
+        "simpleTypes": {
+            "enum": [
+                "array",
+                "boolean",
+                "integer",
+                "null",
+                "number",
+                "object",
+                "string"
+            ]
+        },
+        "stringArray": {
+            "items": {
+                "type": "string"
+            },
+            "minItems": 1,
+            "type": "array",
+            "uniqueItems": true
+        }
+    },
+    "dependencies": {
+        "exclusiveMaximum": [
+            "maximum"
+        ],
+        "exclusiveMinimum": [
+            "minimum"
+        ]
+    },
+    "description": "Core schema meta-schema",
+    "id": "http://json-schema.org/draft-04/schema#",
+    "properties": {
+        "$schema": {
+            "format": "uri",
+            "type": "string"
+        },
+        "additionalItems": {
+            "anyOf": [
+                {
+                    "type": "boolean"
+                },
+                {
+                    "$ref": "#"
+                }
+            ],
+            "default": {}
+        },
+        "additionalProperties": {
+            "anyOf": [
+                {
+                    "type": "boolean"
+                },
+                {
+                    "$ref": "#"
+                }
+            ],
+            "default": {}
+        },
+        "allOf": {
+            "$ref": "#/definitions/schemaArray"
+        },
+        "anyOf": {
+            "$ref": "#/definitions/schemaArray"
+        },
+        "default": {},
+        "definitions": {
+            "additionalProperties": {
+                "$ref": "#"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "dependencies": {
+            "additionalProperties": {
+                "anyOf": [
+                    {
+                        "$ref": "#"
+                    },
+                    {
+                        "$ref": "#/definitions/stringArray"
+                    }
+                ]
+            },
+            "type": "object"
+        },
+        "description": {
+            "type": "string"
+        },
+        "enum": {
+            "type": "array"
+        },
+        "exclusiveMaximum": {
+            "default": false,
+            "type": "boolean"
+        },
+        "exclusiveMinimum": {
+            "default": false,
+            "type": "boolean"
+        },
+        "format": {
+            "type": "string"
+        },
+        "id": {
+            "format": "uri",
+            "type": "string"
+        },
+        "items": {
+            "anyOf": [
+                {
+                    "$ref": "#"
+                },
+                {
+                    "$ref": "#/definitions/schemaArray"
+                }
+            ],
+            "default": {}
+        },
+        "maxItems": {
+            "$ref": "#/definitions/positiveInteger"
+        },
+        "maxLength": {
+            "$ref": "#/definitions/positiveInteger"
+        },
+        "maxProperties": {
+            "$ref": "#/definitions/positiveInteger"
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "minItems": {
+            "$ref": "#/definitions/positiveIntegerDefault0"
+        },
+        "minLength": {
+            "$ref": "#/definitions/positiveIntegerDefault0"
+        },
+        "minProperties": {
+            "$ref": "#/definitions/positiveIntegerDefault0"
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "multipleOf": {
+            "exclusiveMinimum": true,
+            "minimum": 0,
+            "type": "number"
+        },
+        "not": {
+            "$ref": "#"
+        },
+        "oneOf": {
+            "$ref": "#/definitions/schemaArray"
+        },
+        "pattern": {
+            "format": "regex",
+            "type": "string"
+        },
+        "patternProperties": {
+            "additionalProperties": {
+                "$ref": "#"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "properties": {
+            "additionalProperties": {
+                "$ref": "#"
+            },
+            "default": {},
+            "type": "object"
+        },
+        "required": {
+            "$ref": "#/definitions/stringArray"
+        },
+        "title": {
+            "type": "string"
+        },
+        "type": {
+            "anyOf": [
+                {
+                    "$ref": "#/definitions/simpleTypes"
+                },
+                {
+                    "items": {
+                        "$ref": "#/definitions/simpleTypes"
+                    },
+                    "minItems": 1,
+                    "type": "array",
+                    "uniqueItems": true
+                }
+            ]
+        },
+        "uniqueItems": {
+            "default": false,
+            "type": "boolean"
+        }
+    },
+    "type": "object"
+}
diff --git a/contrib/python/jsonschema/py3/jsonschema/schemas/draft6.json b/contrib/python/jsonschema/py3/jsonschema/schemas/draft6.json
new file mode 100644
index 00000000000..a0d2bf7896c
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/schemas/draft6.json
@@ -0,0 +1,153 @@
+{
+    "$schema": "http://json-schema.org/draft-06/schema#",
+    "$id": "http://json-schema.org/draft-06/schema#",
+    "title": "Core schema meta-schema",
+    "definitions": {
+        "schemaArray": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "$ref": "#" }
+        },
+        "nonNegativeInteger": {
+            "type": "integer",
+            "minimum": 0
+        },
+        "nonNegativeIntegerDefault0": {
+            "allOf": [
+                { "$ref": "#/definitions/nonNegativeInteger" },
+                { "default": 0 }
+            ]
+        },
+        "simpleTypes": {
+            "enum": [
+                "array",
+                "boolean",
+                "integer",
+                "null",
+                "number",
+                "object",
+                "string"
+            ]
+        },
+        "stringArray": {
+            "type": "array",
+            "items": { "type": "string" },
+            "uniqueItems": true,
+            "default": []
+        }
+    },
+    "type": ["object", "boolean"],
+    "properties": {
+        "$id": {
+            "type": "string",
+            "format": "uri-reference"
+        },
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
+        "$ref": {
+            "type": "string",
+            "format": "uri-reference"
+        },
+        "title": {
+            "type": "string"
+        },
+        "description": {
+            "type": "string"
+        },
+        "default": {},
+        "examples": {
+            "type": "array",
+            "items": {}
+        },
+        "multipleOf": {
+            "type": "number",
+            "exclusiveMinimum": 0
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "exclusiveMaximum": {
+            "type": "number"
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "exclusiveMinimum": {
+            "type": "number"
+        },
+        "maxLength": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "pattern": {
+            "type": "string",
+            "format": "regex"
+        },
+        "additionalItems": { "$ref": "#" },
+        "items": {
+            "anyOf": [
+                { "$ref": "#" },
+                { "$ref": "#/definitions/schemaArray" }
+            ],
+            "default": {}
+        },
+        "maxItems": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "uniqueItems": {
+            "type": "boolean",
+            "default": false
+        },
+        "contains": { "$ref": "#" },
+        "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "required": { "$ref": "#/definitions/stringArray" },
+        "additionalProperties": { "$ref": "#" },
+        "definitions": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "properties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "patternProperties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "propertyNames": { "format": "regex" },
+            "default": {}
+        },
+        "dependencies": {
+            "type": "object",
+            "additionalProperties": {
+                "anyOf": [
+                    { "$ref": "#" },
+                    { "$ref": "#/definitions/stringArray" }
+                ]
+            }
+        },
+        "propertyNames": { "$ref": "#" },
+        "const": {},
+        "enum": {
+            "type": "array"
+        },
+        "type": {
+            "anyOf": [
+                { "$ref": "#/definitions/simpleTypes" },
+                {
+                    "type": "array",
+                    "items": { "$ref": "#/definitions/simpleTypes" },
+                    "minItems": 1,
+                    "uniqueItems": true
+                }
+            ]
+        },
+        "format": { "type": "string" },
+        "allOf": { "$ref": "#/definitions/schemaArray" },
+        "anyOf": { "$ref": "#/definitions/schemaArray" },
+        "oneOf": { "$ref": "#/definitions/schemaArray" },
+        "not": { "$ref": "#" }
+    },
+    "default": {}
+}
diff --git a/contrib/python/jsonschema/py3/jsonschema/schemas/draft7.json b/contrib/python/jsonschema/py3/jsonschema/schemas/draft7.json
new file mode 100644
index 00000000000..746cde96901
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/schemas/draft7.json
@@ -0,0 +1,166 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "http://json-schema.org/draft-07/schema#",
+    "title": "Core schema meta-schema",
+    "definitions": {
+        "schemaArray": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "$ref": "#" }
+        },
+        "nonNegativeInteger": {
+            "type": "integer",
+            "minimum": 0
+        },
+        "nonNegativeIntegerDefault0": {
+            "allOf": [
+                { "$ref": "#/definitions/nonNegativeInteger" },
+                { "default": 0 }
+            ]
+        },
+        "simpleTypes": {
+            "enum": [
+                "array",
+                "boolean",
+                "integer",
+                "null",
+                "number",
+                "object",
+                "string"
+            ]
+        },
+        "stringArray": {
+            "type": "array",
+            "items": { "type": "string" },
+            "uniqueItems": true,
+            "default": []
+        }
+    },
+    "type": ["object", "boolean"],
+    "properties": {
+        "$id": {
+            "type": "string",
+            "format": "uri-reference"
+        },
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
+        "$ref": {
+            "type": "string",
+            "format": "uri-reference"
+        },
+        "$comment": {
+            "type": "string"
+        },
+        "title": {
+            "type": "string"
+        },
+        "description": {
+            "type": "string"
+        },
+        "default": true,
+        "readOnly": {
+            "type": "boolean",
+            "default": false
+        },
+        "examples": {
+            "type": "array",
+            "items": true
+        },
+        "multipleOf": {
+            "type": "number",
+            "exclusiveMinimum": 0
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "exclusiveMaximum": {
+            "type": "number"
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "exclusiveMinimum": {
+            "type": "number"
+        },
+        "maxLength": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "pattern": {
+            "type": "string",
+            "format": "regex"
+        },
+        "additionalItems": { "$ref": "#" },
+        "items": {
+            "anyOf": [
+                { "$ref": "#" },
+                { "$ref": "#/definitions/schemaArray" }
+            ],
+            "default": true
+        },
+        "maxItems": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "uniqueItems": {
+            "type": "boolean",
+            "default": false
+        },
+        "contains": { "$ref": "#" },
+        "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" },
+        "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
+        "required": { "$ref": "#/definitions/stringArray" },
+        "additionalProperties": { "$ref": "#" },
+        "definitions": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "properties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "patternProperties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "propertyNames": { "format": "regex" },
+            "default": {}
+        },
+        "dependencies": {
+            "type": "object",
+            "additionalProperties": {
+                "anyOf": [
+                    { "$ref": "#" },
+                    { "$ref": "#/definitions/stringArray" }
+                ]
+            }
+        },
+        "propertyNames": { "$ref": "#" },
+        "const": true,
+        "enum": {
+            "type": "array",
+            "items": true
+        },
+        "type": {
+            "anyOf": [
+                { "$ref": "#/definitions/simpleTypes" },
+                {
+                    "type": "array",
+                    "items": { "$ref": "#/definitions/simpleTypes" },
+                    "minItems": 1,
+                    "uniqueItems": true
+                }
+            ]
+        },
+        "format": { "type": "string" },
+        "contentMediaType": { "type": "string" },
+        "contentEncoding": { "type": "string" },
+        "if": {"$ref": "#"},
+        "then": {"$ref": "#"},
+        "else": {"$ref": "#"},
+        "allOf": { "$ref": "#/definitions/schemaArray" },
+        "anyOf": { "$ref": "#/definitions/schemaArray" },
+        "oneOf": { "$ref": "#/definitions/schemaArray" },
+        "not": { "$ref": "#" }
+    },
+    "default": true
+}
diff --git a/contrib/python/jsonschema/py3/jsonschema/tests/__init__.py b/contrib/python/jsonschema/py3/jsonschema/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/jsonschema/py3/jsonschema/tests/_helpers.py b/contrib/python/jsonschema/py3/jsonschema/tests/_helpers.py
new file mode 100644
index 00000000000..70f291fe2ab
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/tests/_helpers.py
@@ -0,0 +1,5 @@
+def bug(issue=None):
+    message = "A known bug."
+    if issue is not None:
+        message += " See issue #{issue}.".format(issue=issue)
+    return message
diff --git a/contrib/python/jsonschema/py3/jsonschema/tests/test_cli.py b/contrib/python/jsonschema/py3/jsonschema/tests/test_cli.py
new file mode 100644
index 00000000000..328c85106f1
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/tests/test_cli.py
@@ -0,0 +1,143 @@
+from unittest import TestCase
+import json
+import subprocess
+import sys
+
+from jsonschema import Draft4Validator, ValidationError, cli, __version__
+from jsonschema.compat import NativeIO
+from jsonschema.exceptions import SchemaError
+
+
+def fake_validator(*errors):
+    errors = list(reversed(errors))
+
+    class FakeValidator(object):
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def iter_errors(self, instance):
+            if errors:
+                return errors.pop()
+            return []
+
+        def check_schema(self, schema):
+            pass
+
+    return FakeValidator
+
+
+class TestParser(TestCase):
+
+    FakeValidator = fake_validator()
+    instance_file = "foo.json"
+    schema_file = "schema.json"
+
+    def setUp(self):
+        cli.open = self.fake_open
+        self.addCleanup(delattr, cli, "open")
+
+    def fake_open(self, path):
+        if path == self.instance_file:
+            contents = ""
+        elif path == self.schema_file:
+            contents = {}
+        else:  # pragma: no cover
+            self.fail("What is {!r}".format(path))
+        return NativeIO(json.dumps(contents))
+
+    def test_find_validator_by_fully_qualified_object_name(self):
+        arguments = cli.parse_args(
+            [
+                "--validator",
+                "__tests__.test_cli.TestParser.FakeValidator",  # XXX Arcadia
+                "--instance", self.instance_file,
+                self.schema_file,
+            ]
+        )
+        self.assertIs(arguments["validator"], self.FakeValidator)
+
+    def test_find_validator_in_jsonschema(self):
+        arguments = cli.parse_args(
+            [
+                "--validator", "Draft4Validator",
+                "--instance", self.instance_file,
+                self.schema_file,
+            ]
+        )
+        self.assertIs(arguments["validator"], Draft4Validator)
+
+
+class TestCLI(TestCase):
+    def test_draft3_schema_draft4_validator(self):
+        stdout, stderr = NativeIO(), NativeIO()
+        with self.assertRaises(SchemaError):
+            cli.run(
+                {
+                    "validator": Draft4Validator,
+                    "schema": {
+                        "anyOf": [
+                            {"minimum": 20},
+                            {"type": "string"},
+                            {"required": True},
+                        ],
+                    },
+                    "instances": [1],
+                    "error_format": "{error.message}",
+                },
+                stdout=stdout,
+                stderr=stderr,
+            )
+
+    def test_successful_validation(self):
+        stdout, stderr = NativeIO(), NativeIO()
+        exit_code = cli.run(
+            {
+                "validator": fake_validator(),
+                "schema": {},
+                "instances": [1],
+                "error_format": "{error.message}",
+            },
+            stdout=stdout,
+            stderr=stderr,
+        )
+        self.assertFalse(stdout.getvalue())
+        self.assertFalse(stderr.getvalue())
+        self.assertEqual(exit_code, 0)
+
+    def test_unsuccessful_validation(self):
+        error = ValidationError("I am an error!", instance=1)
+        stdout, stderr = NativeIO(), NativeIO()
+        exit_code = cli.run(
+            {
+                "validator": fake_validator([error]),
+                "schema": {},
+                "instances": [1],
+                "error_format": "{error.instance} - {error.message}",
+            },
+            stdout=stdout,
+            stderr=stderr,
+        )
+        self.assertFalse(stdout.getvalue())
+        self.assertEqual(stderr.getvalue(), "1 - I am an error!")
+        self.assertEqual(exit_code, 1)
+
+    def test_unsuccessful_validation_multiple_instances(self):
+        first_errors = [
+            ValidationError("9", instance=1),
+            ValidationError("8", instance=1),
+        ]
+        second_errors = [ValidationError("7", instance=2)]
+        stdout, stderr = NativeIO(), NativeIO()
+        exit_code = cli.run(
+            {
+                "validator": fake_validator(first_errors, second_errors),
+                "schema": {},
+                "instances": [1, 2],
+                "error_format": "{error.instance} - {error.message}\t",
+            },
+            stdout=stdout,
+            stderr=stderr,
+        )
+        self.assertFalse(stdout.getvalue())
+        self.assertEqual(stderr.getvalue(), "1 - 9\t1 - 8\t2 - 7\t")
+        self.assertEqual(exit_code, 1)
diff --git a/contrib/python/jsonschema/py3/jsonschema/tests/test_exceptions.py b/contrib/python/jsonschema/py3/jsonschema/tests/test_exceptions.py
new file mode 100644
index 00000000000..eae00d76d77
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/tests/test_exceptions.py
@@ -0,0 +1,462 @@
+from unittest import TestCase
+import textwrap
+
+from jsonschema import Draft4Validator, exceptions
+from jsonschema.compat import PY3
+
+
+class TestBestMatch(TestCase):
+    def best_match(self, errors):
+        errors = list(errors)
+        best = exceptions.best_match(errors)
+        reversed_best = exceptions.best_match(reversed(errors))
+        msg = "Didn't return a consistent best match!\nGot: {0}\n\nThen: {1}"
+        self.assertEqual(
+            best._contents(), reversed_best._contents(),
+            msg=msg.format(best, reversed_best),
+        )
+        return best
+
+    def test_shallower_errors_are_better_matches(self):
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "minProperties": 2,
+                        "properties": {"bar": {"type": "object"}},
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": []}}))
+        self.assertEqual(best.validator, "minProperties")
+
+    def test_oneOf_and_anyOf_are_weak_matches(self):
+        """
+        A property you *must* match is probably better than one you have to
+        match a part of.
+        """
+
+        validator = Draft4Validator(
+            {
+                "minProperties": 2,
+                "anyOf": [{"type": "string"}, {"type": "number"}],
+                "oneOf": [{"type": "string"}, {"type": "number"}],
+            }
+        )
+        best = self.best_match(validator.iter_errors({}))
+        self.assertEqual(best.validator, "minProperties")
+
+    def test_if_the_most_relevant_error_is_anyOf_it_is_traversed(self):
+        """
+        If the most relevant error is an anyOf, then we traverse its context
+        and select the otherwise *least* relevant error, since in this case
+        that means the most specific, deep, error inside the instance.
+
+        I.e. since only one of the schemas must match, we look for the most
+        relevant one.
+        """
+
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "anyOf": [
+                            {"type": "string"},
+                            {"properties": {"bar": {"type": "array"}}},
+                        ],
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": 12}}))
+        self.assertEqual(best.validator_value, "array")
+
+    def test_if_the_most_relevant_error_is_oneOf_it_is_traversed(self):
+        """
+        If the most relevant error is an oneOf, then we traverse its context
+        and select the otherwise *least* relevant error, since in this case
+        that means the most specific, deep, error inside the instance.
+
+        I.e. since only one of the schemas must match, we look for the most
+        relevant one.
+        """
+
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "oneOf": [
+                            {"type": "string"},
+                            {"properties": {"bar": {"type": "array"}}},
+                        ],
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": 12}}))
+        self.assertEqual(best.validator_value, "array")
+
+    def test_if_the_most_relevant_error_is_allOf_it_is_traversed(self):
+        """
+        Now, if the error is allOf, we traverse but select the *most* relevant
+        error from the context, because all schemas here must match anyways.
+        """
+
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "allOf": [
+                            {"type": "string"},
+                            {"properties": {"bar": {"type": "array"}}},
+                        ],
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": 12}}))
+        self.assertEqual(best.validator_value, "string")
+
+    def test_nested_context_for_oneOf(self):
+        validator = Draft4Validator(
+            {
+                "properties": {
+                    "foo": {
+                        "oneOf": [
+                            {"type": "string"},
+                            {
+                                "oneOf": [
+                                    {"type": "string"},
+                                    {
+                                        "properties": {
+                                            "bar": {"type": "array"},
+                                        },
+                                    },
+                                ],
+                            },
+                        ],
+                    },
+                },
+            },
+        )
+        best = self.best_match(validator.iter_errors({"foo": {"bar": 12}}))
+        self.assertEqual(best.validator_value, "array")
+
+    def test_one_error(self):
+        validator = Draft4Validator({"minProperties": 2})
+        error, = validator.iter_errors({})
+        self.assertEqual(
+            exceptions.best_match(validator.iter_errors({})).validator,
+            "minProperties",
+        )
+
+    def test_no_errors(self):
+        validator = Draft4Validator({})
+        self.assertIsNone(exceptions.best_match(validator.iter_errors({})))
+
+
+class TestByRelevance(TestCase):
+    def test_short_paths_are_better_matches(self):
+        shallow = exceptions.ValidationError("Oh no!", path=["baz"])
+        deep = exceptions.ValidationError("Oh yes!", path=["foo", "bar"])
+        match = max([shallow, deep], key=exceptions.relevance)
+        self.assertIs(match, shallow)
+
+        match = max([deep, shallow], key=exceptions.relevance)
+        self.assertIs(match, shallow)
+
+    def test_global_errors_are_even_better_matches(self):
+        shallow = exceptions.ValidationError("Oh no!", path=[])
+        deep = exceptions.ValidationError("Oh yes!", path=["foo"])
+
+        errors = sorted([shallow, deep], key=exceptions.relevance)
+        self.assertEqual(
+            [list(error.path) for error in errors],
+            [["foo"], []],
+        )
+
+        errors = sorted([deep, shallow], key=exceptions.relevance)
+        self.assertEqual(
+            [list(error.path) for error in errors],
+            [["foo"], []],
+        )
+
+    def test_weak_validators_are_lower_priority(self):
+        weak = exceptions.ValidationError("Oh no!", path=[], validator="a")
+        normal = exceptions.ValidationError("Oh yes!", path=[], validator="b")
+
+        best_match = exceptions.by_relevance(weak="a")
+
+        match = max([weak, normal], key=best_match)
+        self.assertIs(match, normal)
+
+        match = max([normal, weak], key=best_match)
+        self.assertIs(match, normal)
+
+    def test_strong_validators_are_higher_priority(self):
+        weak = exceptions.ValidationError("Oh no!", path=[], validator="a")
+        normal = exceptions.ValidationError("Oh yes!", path=[], validator="b")
+        strong = exceptions.ValidationError("Oh fine!", path=[], validator="c")
+
+        best_match = exceptions.by_relevance(weak="a", strong="c")
+
+        match = max([weak, normal, strong], key=best_match)
+        self.assertIs(match, strong)
+
+        match = max([strong, normal, weak], key=best_match)
+        self.assertIs(match, strong)
+
+
+class TestErrorTree(TestCase):
+    def test_it_knows_how_many_total_errors_it_contains(self):
+        # FIXME: https://github.com/Julian/jsonschema/issues/442
+        errors = [
+            exceptions.ValidationError("Something", validator=i)
+            for i in range(8)
+        ]
+        tree = exceptions.ErrorTree(errors)
+        self.assertEqual(tree.total_errors, 8)
+
+    def test_it_contains_an_item_if_the_item_had_an_error(self):
+        errors = [exceptions.ValidationError("a message", path=["bar"])]
+        tree = exceptions.ErrorTree(errors)
+        self.assertIn("bar", tree)
+
+    def test_it_does_not_contain_an_item_if_the_item_had_no_error(self):
+        errors = [exceptions.ValidationError("a message", path=["bar"])]
+        tree = exceptions.ErrorTree(errors)
+        self.assertNotIn("foo", tree)
+
+    def test_validators_that_failed_appear_in_errors_dict(self):
+        error = exceptions.ValidationError("a message", validator="foo")
+        tree = exceptions.ErrorTree([error])
+        self.assertEqual(tree.errors, {"foo": error})
+
+    def test_it_creates_a_child_tree_for_each_nested_path(self):
+        errors = [
+            exceptions.ValidationError("a bar message", path=["bar"]),
+            exceptions.ValidationError("a bar -> 0 message", path=["bar", 0]),
+        ]
+        tree = exceptions.ErrorTree(errors)
+        self.assertIn(0, tree["bar"])
+        self.assertNotIn(1, tree["bar"])
+
+    def test_children_have_their_errors_dicts_built(self):
+        e1, e2 = (
+            exceptions.ValidationError("1", validator="foo", path=["bar", 0]),
+            exceptions.ValidationError("2", validator="quux", path=["bar", 0]),
+        )
+        tree = exceptions.ErrorTree([e1, e2])
+        self.assertEqual(tree["bar"][0].errors, {"foo": e1, "quux": e2})
+
+    def test_multiple_errors_with_instance(self):
+        e1, e2 = (
+            exceptions.ValidationError(
+                "1",
+                validator="foo",
+                path=["bar", "bar2"],
+                instance="i1"),
+            exceptions.ValidationError(
+                "2",
+                validator="quux",
+                path=["foobar", 2],
+                instance="i2"),
+        )
+        exceptions.ErrorTree([e1, e2])
+
+    def test_it_does_not_contain_subtrees_that_are_not_in_the_instance(self):
+        error = exceptions.ValidationError("123", validator="foo", instance=[])
+        tree = exceptions.ErrorTree([error])
+
+        with self.assertRaises(IndexError):
+            tree[0]
+
+    def test_if_its_in_the_tree_anyhow_it_does_not_raise_an_error(self):
+        """
+        If a validator is dumb (like :validator:`required` in draft 3) and
+        refers to a path that isn't in the instance, the tree still properly
+        returns a subtree for that path.
+        """
+
+        error = exceptions.ValidationError(
+            "a message", validator="foo", instance={}, path=["foo"],
+        )
+        tree = exceptions.ErrorTree([error])
+        self.assertIsInstance(tree["foo"], exceptions.ErrorTree)
+
+
+class TestErrorInitReprStr(TestCase):
+    def make_error(self, **kwargs):
+        defaults = dict(
+            message=u"hello",
+            validator=u"type",
+            validator_value=u"string",
+            instance=5,
+            schema={u"type": u"string"},
+        )
+        defaults.update(kwargs)
+        return exceptions.ValidationError(**defaults)
+
+    def assertShows(self, expected, **kwargs):
+        if PY3:  # pragma: no cover
+            expected = expected.replace("u'", "'")
+        expected = textwrap.dedent(expected).rstrip("\n")
+
+        error = self.make_error(**kwargs)
+        message_line, _, rest = str(error).partition("\n")
+        self.assertEqual(message_line, error.message)
+        self.assertEqual(rest, expected)
+
+    def test_it_calls_super_and_sets_args(self):
+        error = self.make_error()
+        self.assertGreater(len(error.args), 1)
+
+    def test_repr(self):
+        self.assertEqual(
+            repr(exceptions.ValidationError(message="Hello!")),
+            "<ValidationError: %r>" % "Hello!",
+        )
+
+    def test_unset_error(self):
+        error = exceptions.ValidationError("message")
+        self.assertEqual(str(error), "message")
+
+        kwargs = {
+            "validator": "type",
+            "validator_value": "string",
+            "instance": 5,
+            "schema": {"type": "string"},
+        }
+        # Just the message should show if any of the attributes are unset
+        for attr in kwargs:
+            k = dict(kwargs)
+            del k[attr]
+            error = exceptions.ValidationError("message", **k)
+            self.assertEqual(str(error), "message")
+
+    def test_empty_paths(self):
+        self.assertShows(
+            """
+            Failed validating u'type' in schema:
+                {u'type': u'string'}
+
+            On instance:
+                5
+            """,
+            path=[],
+            schema_path=[],
+        )
+
+    def test_one_item_paths(self):
+        self.assertShows(
+            """
+            Failed validating u'type' in schema:
+                {u'type': u'string'}
+
+            On instance[0]:
+                5
+            """,
+            path=[0],
+            schema_path=["items"],
+        )
+
+    def test_multiple_item_paths(self):
+        self.assertShows(
+            """
+            Failed validating u'type' in schema[u'items'][0]:
+                {u'type': u'string'}
+
+            On instance[0][u'a']:
+                5
+            """,
+            path=[0, u"a"],
+            schema_path=[u"items", 0, 1],
+        )
+
+    def test_uses_pprint(self):
+        self.assertShows(
+            """
+            Failed validating u'maxLength' in schema:
+                {0: 0,
+                 1: 1,
+                 2: 2,
+                 3: 3,
+                 4: 4,
+                 5: 5,
+                 6: 6,
+                 7: 7,
+                 8: 8,
+                 9: 9,
+                 10: 10,
+                 11: 11,
+                 12: 12,
+                 13: 13,
+                 14: 14,
+                 15: 15,
+                 16: 16,
+                 17: 17,
+                 18: 18,
+                 19: 19}
+
+            On instance:
+                [0,
+                 1,
+                 2,
+                 3,
+                 4,
+                 5,
+                 6,
+                 7,
+                 8,
+                 9,
+                 10,
+                 11,
+                 12,
+                 13,
+                 14,
+                 15,
+                 16,
+                 17,
+                 18,
+                 19,
+                 20,
+                 21,
+                 22,
+                 23,
+                 24]
+            """,
+            instance=list(range(25)),
+            schema=dict(zip(range(20), range(20))),
+            validator=u"maxLength",
+        )
+
+    def test_str_works_with_instances_having_overriden_eq_operator(self):
+        """
+        Check for https://github.com/Julian/jsonschema/issues/164 which
+        rendered exceptions unusable when a `ValidationError` involved
+        instances with an `__eq__` method that returned truthy values.
+        """
+
+        class DontEQMeBro(object):
+            def __eq__(this, other):  # pragma: no cover
+                self.fail("Don't!")
+
+            def __ne__(this, other):  # pragma: no cover
+                self.fail("Don't!")
+
+        instance = DontEQMeBro()
+        error = exceptions.ValidationError(
+            "a message",
+            validator="foo",
+            instance=instance,
+            validator_value="some",
+            schema="schema",
+        )
+        self.assertIn(repr(instance), str(error))
+
+
+class TestHashable(TestCase):
+    def test_hashable(self):
+        set([exceptions.ValidationError("")])
+        set([exceptions.SchemaError("")])
diff --git a/contrib/python/jsonschema/py3/jsonschema/tests/test_format.py b/contrib/python/jsonschema/py3/jsonschema/tests/test_format.py
new file mode 100644
index 00000000000..254985f6156
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/tests/test_format.py
@@ -0,0 +1,89 @@
+"""
+Tests for the parts of jsonschema related to the :validator:`format` property.
+"""
+
+from unittest import TestCase
+
+from jsonschema import FormatError, ValidationError, FormatChecker
+from jsonschema.validators import Draft4Validator
+
+
+BOOM = ValueError("Boom!")
+BANG = ZeroDivisionError("Bang!")
+
+
+def boom(thing):
+    if thing == "bang":
+        raise BANG
+    raise BOOM
+
+
+class TestFormatChecker(TestCase):
+    def test_it_can_validate_no_formats(self):
+        checker = FormatChecker(formats=())
+        self.assertFalse(checker.checkers)
+
+    def test_it_raises_a_key_error_for_unknown_formats(self):
+        with self.assertRaises(KeyError):
+            FormatChecker(formats=["o noes"])
+
+    def test_it_can_register_cls_checkers(self):
+        original = dict(FormatChecker.checkers)
+        self.addCleanup(FormatChecker.checkers.pop, "boom")
+        FormatChecker.cls_checks("boom")(boom)
+        self.assertEqual(
+            FormatChecker.checkers,
+            dict(original, boom=(boom, ())),
+        )
+
+    def test_it_can_register_checkers(self):
+        checker = FormatChecker()
+        checker.checks("boom")(boom)
+        self.assertEqual(
+            checker.checkers,
+            dict(FormatChecker.checkers, boom=(boom, ()))
+        )
+
+    def test_it_catches_registered_errors(self):
+        checker = FormatChecker()
+        checker.checks("boom", raises=type(BOOM))(boom)
+
+        with self.assertRaises(FormatError) as cm:
+            checker.check(instance=12, format="boom")
+
+        self.assertIs(cm.exception.cause, BOOM)
+        self.assertIs(cm.exception.__cause__, BOOM)
+
+        # Unregistered errors should not be caught
+        with self.assertRaises(type(BANG)):
+            checker.check(instance="bang", format="boom")
+
+    def test_format_error_causes_become_validation_error_causes(self):
+        checker = FormatChecker()
+        checker.checks("boom", raises=ValueError)(boom)
+        validator = Draft4Validator({"format": "boom"}, format_checker=checker)
+
+        with self.assertRaises(ValidationError) as cm:
+            validator.validate("BOOM")
+
+        self.assertIs(cm.exception.cause, BOOM)
+        self.assertIs(cm.exception.__cause__, BOOM)
+
+    def test_format_checkers_come_with_defaults(self):
+        # This is bad :/ but relied upon.
+        # The docs for quite awhile recommended people do things like
+        # validate(..., format_checker=FormatChecker())
+        # We should change that, but we can't without deprecation...
+        checker = FormatChecker()
+        with self.assertRaises(FormatError):
+            checker.check(instance="not-an-ipv4", format="ipv4")
+
+    def test_repr(self):
+        checker = FormatChecker(formats=())
+        checker.checks("foo")(lambda thing: True)
+        checker.checks("bar")(lambda thing: True)
+        checker.checks("baz")(lambda thing: True)
+        self.assertEqual(
+            repr(checker),
+            "<FormatChecker checkers=['bar', 'baz', 'foo']>",
+        )
diff --git a/contrib/python/jsonschema/py3/jsonschema/tests/test_types.py b/contrib/python/jsonschema/py3/jsonschema/tests/test_types.py
new file mode 100644
index 00000000000..2280cc395b2
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/tests/test_types.py
@@ -0,0 +1,190 @@
+"""
+Tests on the new type interface. The actual correctness of the type checking
+is handled in test_jsonschema_test_suite; these tests check that TypeChecker
+functions correctly and can facilitate extensions to type checking
+"""
+from collections import namedtuple
+from unittest import TestCase
+
+from jsonschema import ValidationError, _validators
+from jsonschema._types import TypeChecker
+from jsonschema.exceptions import UndefinedTypeCheck
+from jsonschema.validators import Draft4Validator, extend
+
+
+def equals_2(checker, instance):
+    return instance == 2
+
+
+def is_namedtuple(instance):
+    return isinstance(instance, tuple) and getattr(instance, "_fields", None)
+
+
+def is_object_or_named_tuple(checker, instance):
+    if Draft4Validator.TYPE_CHECKER.is_type(instance, "object"):
+        return True
+    return is_namedtuple(instance)
+
+
+def coerce_named_tuple(fn):
+    def coerced(validator, value, instance, schema):
+        if is_namedtuple(instance):
+            instance = instance._asdict()
+        return fn(validator, value, instance, schema)
+    return coerced
+
+
+required = coerce_named_tuple(_validators.required)
+properties = coerce_named_tuple(_validators.properties)
+
+
+class TestTypeChecker(TestCase):
+    def test_is_type(self):
+        checker = TypeChecker({"two": equals_2})
+        self.assertEqual(
+            (
+                checker.is_type(instance=2, type="two"),
+                checker.is_type(instance="bar", type="two"),
+            ),
+            (True, False),
+        )
+
+    def test_is_unknown_type(self):
+        with self.assertRaises(UndefinedTypeCheck) as context:
+            TypeChecker().is_type(4, "foobar")
+        self.assertIn("foobar", str(context.exception))
+
+    def test_checks_can_be_added_at_init(self):
+        checker = TypeChecker({"two": equals_2})
+        self.assertEqual(checker, TypeChecker().redefine("two", equals_2))
+
+    def test_redefine_existing_type(self):
+        self.assertEqual(
+            TypeChecker().redefine("two", object()).redefine("two", equals_2),
+            TypeChecker().redefine("two", equals_2),
+        )
+
+    def test_remove(self):
+        self.assertEqual(
+            TypeChecker({"two": equals_2}).remove("two"),
+            TypeChecker(),
+        )
+
+    def test_remove_unknown_type(self):
+        with self.assertRaises(UndefinedTypeCheck) as context:
+            TypeChecker().remove("foobar")
+        self.assertIn("foobar", str(context.exception))
+
+    def test_redefine_many(self):
+        self.assertEqual(
+            TypeChecker().redefine_many({"foo": int, "bar": str}),
+            TypeChecker().redefine("foo", int).redefine("bar", str),
+        )
+
+    def test_remove_multiple(self):
+        self.assertEqual(
+            TypeChecker({"foo": int, "bar": str}).remove("foo", "bar"),
+            TypeChecker(),
+        )
+
+    def test_type_check_can_raise_key_error(self):
+        """
+        Make sure no one writes:
+
+            try:
+                self._type_checkers[type](...)
+            except KeyError:
+
+        ignoring the fact that the function itself can raise that.
+        """
+
+        error = KeyError("Stuff")
+
+        def raises_keyerror(checker, instance):
+            raise error
+
+        with self.assertRaises(KeyError) as context:
+            TypeChecker({"foo": raises_keyerror}).is_type(4, "foo")
+
+        self.assertIs(context.exception, error)
+
+
+class TestCustomTypes(TestCase):
+    def test_simple_type_can_be_extended(self):
+        def int_or_str_int(checker, instance):
+            if not isinstance(instance, (int, str)):
+                return False
+            try:
+                int(instance)
+            except ValueError:
+                return False
+            return True
+
+        CustomValidator = extend(
+            Draft4Validator,
+            type_checker=Draft4Validator.TYPE_CHECKER.redefine(
+                "integer", int_or_str_int,
+            ),
+        )
+        validator = CustomValidator({"type": "integer"})
+
+        validator.validate(4)
+        validator.validate("4")
+
+        with self.assertRaises(ValidationError):
+            validator.validate(4.4)
+
+    def test_object_can_be_extended(self):
+        schema = {"type": "object"}
+
+        Point = namedtuple("Point", ["x", "y"])
+
+        type_checker = Draft4Validator.TYPE_CHECKER.redefine(
+            u"object", is_object_or_named_tuple,
+        )
+
+        CustomValidator = extend(Draft4Validator, type_checker=type_checker)
+        validator = CustomValidator(schema)
+
+        validator.validate(Point(x=4, y=5))
+
+    def test_object_extensions_require_custom_validators(self):
+        schema = {"type": "object", "required": ["x"]}
+
+        type_checker = Draft4Validator.TYPE_CHECKER.redefine(
+            u"object", is_object_or_named_tuple,
+        )
+
+        CustomValidator = extend(Draft4Validator, type_checker=type_checker)
+        validator = CustomValidator(schema)
+
+        Point = namedtuple("Point", ["x", "y"])
+        # Cannot handle required
+        with self.assertRaises(ValidationError):
+            validator.validate(Point(x=4, y=5))
+
+    def test_object_extensions_can_handle_custom_validators(self):
+        schema = {
+            "type": "object",
+            "required": ["x"],
+            "properties": {"x": {"type": "integer"}},
+        }
+
+        type_checker = Draft4Validator.TYPE_CHECKER.redefine(
+            u"object", is_object_or_named_tuple,
+        )
+
+        CustomValidator = extend(
+            Draft4Validator,
+            type_checker=type_checker,
+            validators={"required": required, "properties": properties},
+        )
+
+        validator = CustomValidator(schema)
+
+        Point = namedtuple("Point", ["x", "y"])
+        # Can now process required and properties
+        validator.validate(Point(x=4, y=5))
+
+        with self.assertRaises(ValidationError):
+            validator.validate(Point(x="not an integer", y=5))
diff --git a/contrib/python/jsonschema/py3/jsonschema/tests/test_validators.py b/contrib/python/jsonschema/py3/jsonschema/tests/test_validators.py
new file mode 100644
index 00000000000..07be4f08bc2
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/tests/test_validators.py
@@ -0,0 +1,1762 @@
+from collections import deque
+from contextlib import contextmanager
+from decimal import Decimal
+from io import BytesIO
+from unittest import TestCase
+import json
+import os
+import sys
+import tempfile
+import unittest
+
+from twisted.trial.unittest import SynchronousTestCase
+import attr
+
+from jsonschema import FormatChecker, TypeChecker, exceptions, validators
+from jsonschema.compat import PY3, pathname2url
+from jsonschema.tests._helpers import bug
+
+
+def startswith(validator, startswith, instance, schema):
+    if not instance.startswith(startswith):
+        yield exceptions.ValidationError(u"Whoops!")
+
+
+class TestCreateAndExtend(SynchronousTestCase):
+    def setUp(self):
+        self.addCleanup(
+            self.assertEqual,
+            validators.meta_schemas,
+            dict(validators.meta_schemas),
+        )
+
+        self.meta_schema = {u"$id": "some://meta/schema"}
+        self.validators = {u"startswith": startswith}
+        self.type_checker = TypeChecker()
+        self.Validator = validators.create(
+            meta_schema=self.meta_schema,
+            validators=self.validators,
+            type_checker=self.type_checker,
+        )
+
+    def test_attrs(self):
+        self.assertEqual(
+            (
+                self.Validator.VALIDATORS,
+                self.Validator.META_SCHEMA,
+                self.Validator.TYPE_CHECKER,
+            ), (
+                self.validators,
+                self.meta_schema,
+                self.type_checker,
+            ),
+        )
+
+    def test_init(self):
+        schema = {u"startswith": u"foo"}
+        self.assertEqual(self.Validator(schema).schema, schema)
+
+    def test_iter_errors(self):
+        schema = {u"startswith": u"hel"}
+        iter_errors = self.Validator(schema).iter_errors
+
+        errors = list(iter_errors(u"hello"))
+        self.assertEqual(errors, [])
+
+        expected_error = exceptions.ValidationError(
+            u"Whoops!",
+            instance=u"goodbye",
+            schema=schema,
+            validator=u"startswith",
+            validator_value=u"hel",
+            schema_path=deque([u"startswith"]),
+        )
+
+        errors = list(iter_errors(u"goodbye"))
+        self.assertEqual(len(errors), 1)
+        self.assertEqual(errors[0]._contents(), expected_error._contents())
+
+    def test_if_a_version_is_provided_it_is_registered(self):
+        Validator = validators.create(
+            meta_schema={u"$id": "something"},
+            version="my version",
+        )
+        self.addCleanup(validators.meta_schemas.pop, "something")
+        self.assertEqual(Validator.__name__, "MyVersionValidator")
+
+    def test_if_a_version_is_not_provided_it_is_not_registered(self):
+        original = dict(validators.meta_schemas)
+        validators.create(meta_schema={u"id": "id"})
+        self.assertEqual(validators.meta_schemas, original)
+
+    def test_validates_registers_meta_schema_id(self):
+        meta_schema_key = "meta schema id"
+        my_meta_schema = {u"id": meta_schema_key}
+
+        validators.create(
+            meta_schema=my_meta_schema,
+            version="my version",
+            id_of=lambda s: s.get("id", ""),
+        )
+        self.addCleanup(validators.meta_schemas.pop, meta_schema_key)
+
+        self.assertIn(meta_schema_key, validators.meta_schemas)
+
+    def test_validates_registers_meta_schema_draft6_id(self):
+        meta_schema_key = "meta schema $id"
+        my_meta_schema = {u"$id": meta_schema_key}
+
+        validators.create(
+            meta_schema=my_meta_schema,
+            version="my version",
+        )
+        self.addCleanup(validators.meta_schemas.pop, meta_schema_key)
+
+        self.assertIn(meta_schema_key, validators.meta_schemas)
+
+    def test_create_default_types(self):
+        Validator = validators.create(meta_schema={}, validators=())
+        self.assertTrue(
+            all(
+                Validator({}).is_type(instance=instance, type=type)
+                for type, instance in [
+                    (u"array", []),
+                    (u"boolean", True),
+                    (u"integer", 12),
+                    (u"null", None),
+                    (u"number", 12.0),
+                    (u"object", {}),
+                    (u"string", u"foo"),
+                ]
+            ),
+        )
+
+    def test_extend(self):
+        original = dict(self.Validator.VALIDATORS)
+        new = object()
+
+        Extended = validators.extend(
+            self.Validator,
+            validators={u"new": new},
+        )
+        self.assertEqual(
+            (
+                Extended.VALIDATORS,
+                Extended.META_SCHEMA,
+                Extended.TYPE_CHECKER,
+                self.Validator.VALIDATORS,
+            ), (
+                dict(original, new=new),
+                self.Validator.META_SCHEMA,
+                self.Validator.TYPE_CHECKER,
+                original,
+            ),
+        )
+
+    def test_extend_idof(self):
+        """
+        Extending a validator preserves its notion of schema IDs.
+        """
+        def id_of(schema):
+            return schema.get(u"__test__", self.Validator.ID_OF(schema))
+        correct_id = "the://correct/id/"
+        meta_schema = {
+            u"$id": "the://wrong/id/",
+            u"__test__": correct_id,
+        }
+        Original = validators.create(
+            meta_schema=meta_schema,
+            validators=self.validators,
+            type_checker=self.type_checker,
+            id_of=id_of,
+        )
+        self.assertEqual(Original.ID_OF(Original.META_SCHEMA), correct_id)
+
+        Derived = validators.extend(Original)
+        self.assertEqual(Derived.ID_OF(Derived.META_SCHEMA), correct_id)
+
+
+class TestLegacyTypeChecking(SynchronousTestCase):
+    def test_create_default_types(self):
+        Validator = validators.create(meta_schema={}, validators=())
+        self.assertEqual(
+            set(Validator.DEFAULT_TYPES), {
+                u"array",
+                u"boolean",
+                u"integer",
+                u"null",
+                u"number",
+                u"object", u"string",
+            },
+        )
+        self.flushWarnings()
+
+    def test_extend(self):
+        Validator = validators.create(meta_schema={}, validators=())
+        original = dict(Validator.VALIDATORS)
+        new = object()
+
+        Extended = validators.extend(
+            Validator,
+            validators={u"new": new},
+        )
+        self.assertEqual(
+            (
+                Extended.VALIDATORS,
+                Extended.META_SCHEMA,
+                Extended.TYPE_CHECKER,
+                Validator.VALIDATORS,
+
+                Extended.DEFAULT_TYPES,
+                Extended({}).DEFAULT_TYPES,
+                self.flushWarnings()[0]["message"],
+            ), (
+                dict(original, new=new),
+                Validator.META_SCHEMA,
+                Validator.TYPE_CHECKER,
+                original,
+
+                Validator.DEFAULT_TYPES,
+                Validator.DEFAULT_TYPES,
+                self.flushWarnings()[0]["message"],
+            ),
+        )
+
+    def test_types_redefines_the_validators_type_checker(self):
+        schema = {"type": "string"}
+        self.assertFalse(validators.Draft7Validator(schema).is_valid(12))
+
+        validator = validators.Draft7Validator(
+            schema,
+            types={"string": (str, int)},
+        )
+        self.assertTrue(validator.is_valid(12))
+        self.flushWarnings()
+
+    def test_providing_default_types_warns(self):
+        self.assertWarns(
+            category=DeprecationWarning,
+            message=(
+                "The default_types argument is deprecated. "
+                "Use the type_checker argument instead."
+            ),
+            # https://tm.tl/9363 :'(
+            filename=sys.modules[self.assertWarns.__module__].__file__,
+
+            f=validators.create,
+            meta_schema={},
+            validators={},
+            default_types={"foo": object},
+        )
+
+    def test_cannot_ask_for_default_types_with_non_default_type_checker(self):
+        """
+        We raise an error when you ask a validator with non-default
+        type checker for its DEFAULT_TYPES.
+
+        The type checker argument is new, so no one but this library
+        itself should be trying to use it, and doing so while then
+        asking for DEFAULT_TYPES makes no sense (not to mention is
+        deprecated), since type checkers are not strictly about Python
+        type.
+        """
+        Validator = validators.create(
+            meta_schema={},
+            validators={},
+            type_checker=TypeChecker(),
+        )
+        with self.assertRaises(validators._DontDoThat) as e:
+            Validator.DEFAULT_TYPES
+
+        self.assertIn(
+            "DEFAULT_TYPES cannot be used on Validators using TypeCheckers",
+            str(e.exception),
+        )
+        with self.assertRaises(validators._DontDoThat):
+            Validator({}).DEFAULT_TYPES
+
+        self.assertFalse(self.flushWarnings())
+
+    def test_providing_explicit_type_checker_does_not_warn(self):
+        Validator = validators.create(
+            meta_schema={},
+            validators={},
+            type_checker=TypeChecker(),
+        )
+        self.assertFalse(self.flushWarnings())
+
+        Validator({})
+        self.assertFalse(self.flushWarnings())
+
+    def test_providing_neither_does_not_warn(self):
+        Validator = validators.create(meta_schema={}, validators={})
+        self.assertFalse(self.flushWarnings())
+
+        Validator({})
+        self.assertFalse(self.flushWarnings())
+
+    def test_providing_default_types_with_type_checker_errors(self):
+        with self.assertRaises(TypeError) as e:
+            validators.create(
+                meta_schema={},
+                validators={},
+                default_types={"foo": object},
+                type_checker=TypeChecker(),
+            )
+
+        self.assertIn(
+            "Do not specify default_types when providing a type checker",
+            str(e.exception),
+        )
+        self.assertFalse(self.flushWarnings())
+
+    def test_extending_a_legacy_validator_with_a_type_checker_errors(self):
+        Validator = validators.create(
+            meta_schema={},
+            validators={},
+            default_types={u"array": list}
+        )
+        with self.assertRaises(TypeError) as e:
+            validators.extend(
+                Validator,
+                validators={},
+                type_checker=TypeChecker(),
+            )
+
+        self.assertIn(
+            (
+                "Cannot extend a validator created with default_types "
+                "with a type_checker. Update the validator to use a "
+                "type_checker when created."
+            ),
+            str(e.exception),
+        )
+        self.flushWarnings()
+
+    def test_extending_a_legacy_validator_does_not_rewarn(self):
+        Validator = validators.create(meta_schema={}, default_types={})
+        self.assertTrue(self.flushWarnings())
+
+        validators.extend(Validator)
+        self.assertFalse(self.flushWarnings())
+
+    def test_accessing_default_types_warns(self):
+        Validator = validators.create(meta_schema={}, validators={})
+        self.assertFalse(self.flushWarnings())
+
+        self.assertWarns(
+            DeprecationWarning,
+            (
+                "The DEFAULT_TYPES attribute is deprecated. "
+                "See the type checker attached to this validator instead."
+            ),
+            # https://tm.tl/9363 :'(
+            sys.modules[self.assertWarns.__module__].__file__,
+
+            getattr,
+            Validator,
+            "DEFAULT_TYPES",
+        )
+
+    def test_accessing_default_types_on_the_instance_warns(self):
+        Validator = validators.create(meta_schema={}, validators={})
+        self.assertFalse(self.flushWarnings())
+
+        self.assertWarns(
+            DeprecationWarning,
+            (
+                "The DEFAULT_TYPES attribute is deprecated. "
+                "See the type checker attached to this validator instead."
+            ),
+            # https://tm.tl/9363 :'(
+            sys.modules[self.assertWarns.__module__].__file__,
+
+            getattr,
+            Validator({}),
+            "DEFAULT_TYPES",
+        )
+
+    def test_providing_types_to_init_warns(self):
+        Validator = validators.create(meta_schema={}, validators={})
+        self.assertFalse(self.flushWarnings())
+
+        self.assertWarns(
+            category=DeprecationWarning,
+            message=(
+                "The types argument is deprecated. "
+                "Provide a type_checker to jsonschema.validators.extend "
+                "instead."
+            ),
+            # https://tm.tl/9363 :'(
+            filename=sys.modules[self.assertWarns.__module__].__file__,
+
+            f=Validator,
+            schema={},
+            types={"bar": object},
+        )
+
+
+class TestIterErrors(TestCase):
+    def setUp(self):
+        self.validator = validators.Draft3Validator({})
+
+    def test_iter_errors(self):
+        instance = [1, 2]
+        schema = {
+            u"disallow": u"array",
+            u"enum": [["a", "b", "c"], ["d", "e", "f"]],
+            u"minItems": 3,
+        }
+
+        got = (e.message for e in self.validator.iter_errors(instance, schema))
+        expected = [
+            "%r is disallowed for [1, 2]" % (schema["disallow"],),
+            "[1, 2] is too short",
+            "[1, 2] is not one of %r" % (schema["enum"],),
+        ]
+        self.assertEqual(sorted(got), sorted(expected))
+
+    def test_iter_errors_multiple_failures_one_validator(self):
+        instance = {"foo": 2, "bar": [1], "baz": 15, "quux": "spam"}
+        schema = {
+            u"properties": {
+                "foo": {u"type": "string"},
+                "bar": {u"minItems": 2},
+                "baz": {u"maximum": 10, u"enum": [2, 4, 6, 8]},
+            },
+        }
+
+        errors = list(self.validator.iter_errors(instance, schema))
+        self.assertEqual(len(errors), 4)
+
+
+class TestValidationErrorMessages(TestCase):
+    def message_for(self, instance, schema, *args, **kwargs):
+        kwargs.setdefault("cls", validators.Draft3Validator)
+        with self.assertRaises(exceptions.ValidationError) as e:
+            validators.validate(instance, schema, *args, **kwargs)
+        return e.exception.message
+
+    def test_single_type_failure(self):
+        message = self.message_for(instance=1, schema={u"type": u"string"})
+        self.assertEqual(message, "1 is not of type %r" % u"string")
+
+    def test_single_type_list_failure(self):
+        message = self.message_for(instance=1, schema={u"type": [u"string"]})
+        self.assertEqual(message, "1 is not of type %r" % u"string")
+
+    def test_multiple_type_failure(self):
+        types = u"string", u"object"
+        message = self.message_for(instance=1, schema={u"type": list(types)})
+        self.assertEqual(message, "1 is not of type %r, %r" % types)
+
+    def test_object_without_title_type_failure(self):
+        type = {u"type": [{u"minimum": 3}]}
+        message = self.message_for(instance=1, schema={u"type": [type]})
+        self.assertEqual(message, "1 is less than the minimum of 3")
+
+    def test_object_with_named_type_failure(self):
+        schema = {u"type": [{u"name": "Foo", u"minimum": 3}]}
+        message = self.message_for(instance=1, schema=schema)
+        self.assertEqual(message, "1 is less than the minimum of 3")
+
+    def test_minimum(self):
+        message = self.message_for(instance=1, schema={"minimum": 2})
+        self.assertEqual(message, "1 is less than the minimum of 2")
+
+    def test_maximum(self):
+        message = self.message_for(instance=1, schema={"maximum": 0})
+        self.assertEqual(message, "1 is greater than the maximum of 0")
+
+    def test_dependencies_single_element(self):
+        depend, on = "bar", "foo"
+        schema = {u"dependencies": {depend: on}}
+        message = self.message_for(
+            instance={"bar": 2},
+            schema=schema,
+            cls=validators.Draft3Validator,
+        )
+        self.assertEqual(message, "%r is a dependency of %r" % (on, depend))
+
+    def test_dependencies_list_draft3(self):
+        depend, on = "bar", "foo"
+        schema = {u"dependencies": {depend: [on]}}
+        message = self.message_for(
+            instance={"bar": 2},
+            schema=schema,
+            cls=validators.Draft3Validator,
+        )
+        self.assertEqual(message, "%r is a dependency of %r" % (on, depend))
+
+    def test_dependencies_list_draft7(self):
+        depend, on = "bar", "foo"
+        schema = {u"dependencies": {depend: [on]}}
+        message = self.message_for(
+            instance={"bar": 2},
+            schema=schema,
+            cls=validators.Draft7Validator,
+        )
+        self.assertEqual(message, "%r is a dependency of %r" % (on, depend))
+
+    def test_additionalItems_single_failure(self):
+        message = self.message_for(
+            instance=[2],
+            schema={u"items": [], u"additionalItems": False},
+        )
+        self.assertIn("(2 was unexpected)", message)
+
+    def test_additionalItems_multiple_failures(self):
+        message = self.message_for(
+            instance=[1, 2, 3],
+            schema={u"items": [], u"additionalItems": False}
+        )
+        self.assertIn("(1, 2, 3 were unexpected)", message)
+
+    def test_additionalProperties_single_failure(self):
+        additional = "foo"
+        schema = {u"additionalProperties": False}
+        message = self.message_for(instance={additional: 2}, schema=schema)
+        self.assertIn("(%r was unexpected)" % (additional,), message)
+
+    def test_additionalProperties_multiple_failures(self):
+        schema = {u"additionalProperties": False}
+        message = self.message_for(
+            instance=dict.fromkeys(["foo", "bar"]),
+            schema=schema,
+        )
+
+        self.assertIn(repr("foo"), message)
+        self.assertIn(repr("bar"), message)
+        self.assertIn("were unexpected)", message)
+
+    def test_const(self):
+        schema = {u"const": 12}
+        message = self.message_for(
+            instance={"foo": "bar"},
+            schema=schema,
+            cls=validators.Draft6Validator,
+        )
+        self.assertIn("12 was expected", message)
+
+    def test_contains(self):
+        schema = {u"contains": {u"const": 12}}
+        message = self.message_for(
+            instance=[2, {}, []],
+            schema=schema,
+            cls=validators.Draft6Validator,
+        )
+        self.assertIn(
+            "None of [2, {}, []] are valid under the given schema",
+            message,
+        )
+
+    def test_invalid_format_default_message(self):
+        checker = FormatChecker(formats=())
+        checker.checks(u"thing")(lambda value: False)
+
+        schema = {u"format": u"thing"}
+        message = self.message_for(
+            instance="bla",
+            schema=schema,
+            format_checker=checker,
+        )
+
+        self.assertIn(repr("bla"), message)
+        self.assertIn(repr("thing"), message)
+        self.assertIn("is not a", message)
+
+    def test_additionalProperties_false_patternProperties(self):
+        schema = {u"type": u"object",
+                  u"additionalProperties": False,
+                  u"patternProperties": {
+                      u"^abc$": {u"type": u"string"},
+                      u"^def$": {u"type": u"string"},
+                  }}
+        message = self.message_for(
+            instance={u"zebra": 123},
+            schema=schema,
+            cls=validators.Draft4Validator,
+        )
+        self.assertEqual(
+            message,
+            "{} does not match any of the regexes: {}, {}".format(
+                repr(u"zebra"), repr(u"^abc$"), repr(u"^def$"),
+            ),
+        )
+        message = self.message_for(
+            instance={u"zebra": 123, u"fish": 456},
+            schema=schema,
+            cls=validators.Draft4Validator,
+        )
+        self.assertEqual(
+            message,
+            "{}, {} do not match any of the regexes: {}, {}".format(
+                repr(u"fish"), repr(u"zebra"), repr(u"^abc$"), repr(u"^def$")
+            ),
+        )
+
+    def test_False_schema(self):
+        message = self.message_for(
+            instance="something",
+            schema=False,
+            cls=validators.Draft7Validator,
+        )
+        self.assertIn("False schema does not allow 'something'", message)
+
+
+class TestValidationErrorDetails(TestCase):
+    # TODO: These really need unit tests for each individual validator, rather
+    #       than just these higher level tests.
+    def test_anyOf(self):
+        instance = 5
+        schema = {
+            "anyOf": [
+                {"minimum": 20},
+                {"type": "string"},
+            ],
+        }
+
+        validator = validators.Draft4Validator(schema)
+        errors = list(validator.iter_errors(instance))
+        self.assertEqual(len(errors), 1)
+        e = errors[0]
+
+        self.assertEqual(e.validator, "anyOf")
+        self.assertEqual(e.validator_value, schema["anyOf"])
+        self.assertEqual(e.instance, instance)
+        self.assertEqual(e.schema, schema)
+        self.assertIsNone(e.parent)
+
+        self.assertEqual(e.path, deque([]))
+        self.assertEqual(e.relative_path, deque([]))
+        self.assertEqual(e.absolute_path, deque([]))
+
+        self.assertEqual(e.schema_path, deque(["anyOf"]))
+        self.assertEqual(e.relative_schema_path, deque(["anyOf"]))
+        self.assertEqual(e.absolute_schema_path, deque(["anyOf"]))
+
+        self.assertEqual(len(e.context), 2)
+
+        e1, e2 = sorted_errors(e.context)
+
+        self.assertEqual(e1.validator, "minimum")
+        self.assertEqual(e1.validator_value, schema["anyOf"][0]["minimum"])
+        self.assertEqual(e1.instance, instance)
+        self.assertEqual(e1.schema, schema["anyOf"][0])
+        self.assertIs(e1.parent, e)
+
+        self.assertEqual(e1.path, deque([]))
+        self.assertEqual(e1.absolute_path, deque([]))
+        self.assertEqual(e1.relative_path, deque([]))
+
+        self.assertEqual(e1.schema_path, deque([0, "minimum"]))
+        self.assertEqual(e1.relative_schema_path, deque([0, "minimum"]))
+        self.assertEqual(
+            e1.absolute_schema_path, deque(["anyOf", 0, "minimum"]),
+        )
+
+        self.assertFalse(e1.context)
+
+        self.assertEqual(e2.validator, "type")
+        self.assertEqual(e2.validator_value, schema["anyOf"][1]["type"])
+        self.assertEqual(e2.instance, instance)
+        self.assertEqual(e2.schema, schema["anyOf"][1])
+        self.assertIs(e2.parent, e)
+
+        self.assertEqual(e2.path, deque([]))
+        self.assertEqual(e2.relative_path, deque([]))
+        self.assertEqual(e2.absolute_path, deque([]))
+
+        self.assertEqual(e2.schema_path, deque([1, "type"]))
+        self.assertEqual(e2.relative_schema_path, deque([1, "type"]))
+        self.assertEqual(e2.absolute_schema_path, deque(["anyOf", 1, "type"]))
+
+        self.assertEqual(len(e2.context), 0)
+
+    def test_type(self):
+        instance = {"foo": 1}
+        schema = {
+            "type": [
+                {"type": "integer"},
+                {
+                    "type": "object",
+                    "properties": {"foo": {"enum": [2]}},
+                },
+            ],
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = list(validator.iter_errors(instance))
+        self.assertEqual(len(errors), 1)
+        e = errors[0]
+
+        self.assertEqual(e.validator, "type")
+        self.assertEqual(e.validator_value, schema["type"])
+        self.assertEqual(e.instance, instance)
+        self.assertEqual(e.schema, schema)
+        self.assertIsNone(e.parent)
+
+        self.assertEqual(e.path, deque([]))
+        self.assertEqual(e.relative_path, deque([]))
+        self.assertEqual(e.absolute_path, deque([]))
+
+        self.assertEqual(e.schema_path, deque(["type"]))
+        self.assertEqual(e.relative_schema_path, deque(["type"]))
+        self.assertEqual(e.absolute_schema_path, deque(["type"]))
+
+        self.assertEqual(len(e.context), 2)
+
+        e1, e2 = sorted_errors(e.context)
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e1.validator_value, schema["type"][0]["type"])
+        self.assertEqual(e1.instance, instance)
+        self.assertEqual(e1.schema, schema["type"][0])
+        self.assertIs(e1.parent, e)
+
+        self.assertEqual(e1.path, deque([]))
+        self.assertEqual(e1.relative_path, deque([]))
+        self.assertEqual(e1.absolute_path, deque([]))
+
+        self.assertEqual(e1.schema_path, deque([0, "type"]))
+        self.assertEqual(e1.relative_schema_path, deque([0, "type"]))
+        self.assertEqual(e1.absolute_schema_path, deque(["type", 0, "type"]))
+
+        self.assertFalse(e1.context)
+
+        self.assertEqual(e2.validator, "enum")
+        self.assertEqual(e2.validator_value, [2])
+        self.assertEqual(e2.instance, 1)
+        self.assertEqual(e2.schema, {u"enum": [2]})
+        self.assertIs(e2.parent, e)
+
+        self.assertEqual(e2.path, deque(["foo"]))
+        self.assertEqual(e2.relative_path, deque(["foo"]))
+        self.assertEqual(e2.absolute_path, deque(["foo"]))
+
+        self.assertEqual(
+            e2.schema_path, deque([1, "properties", "foo", "enum"]),
+        )
+        self.assertEqual(
+            e2.relative_schema_path, deque([1, "properties", "foo", "enum"]),
+        )
+        self.assertEqual(
+            e2.absolute_schema_path,
+            deque(["type", 1, "properties", "foo", "enum"]),
+        )
+
+        self.assertFalse(e2.context)
+
+    def test_single_nesting(self):
+        instance = {"foo": 2, "bar": [1], "baz": 15, "quux": "spam"}
+        schema = {
+            "properties": {
+                "foo": {"type": "string"},
+                "bar": {"minItems": 2},
+                "baz": {"maximum": 10, "enum": [2, 4, 6, 8]},
+            },
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2, e3, e4 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque(["bar"]))
+        self.assertEqual(e2.path, deque(["baz"]))
+        self.assertEqual(e3.path, deque(["baz"]))
+        self.assertEqual(e4.path, deque(["foo"]))
+
+        self.assertEqual(e1.relative_path, deque(["bar"]))
+        self.assertEqual(e2.relative_path, deque(["baz"]))
+        self.assertEqual(e3.relative_path, deque(["baz"]))
+        self.assertEqual(e4.relative_path, deque(["foo"]))
+
+        self.assertEqual(e1.absolute_path, deque(["bar"]))
+        self.assertEqual(e2.absolute_path, deque(["baz"]))
+        self.assertEqual(e3.absolute_path, deque(["baz"]))
+        self.assertEqual(e4.absolute_path, deque(["foo"]))
+
+        self.assertEqual(e1.validator, "minItems")
+        self.assertEqual(e2.validator, "enum")
+        self.assertEqual(e3.validator, "maximum")
+        self.assertEqual(e4.validator, "type")
+
+    def test_multiple_nesting(self):
+        instance = [1, {"foo": 2, "bar": {"baz": [1]}}, "quux"]
+        schema = {
+            "type": "string",
+            "items": {
+                "type": ["string", "object"],
+                "properties": {
+                    "foo": {"enum": [1, 3]},
+                    "bar": {
+                        "type": "array",
+                        "properties": {
+                            "bar": {"required": True},
+                            "baz": {"minItems": 2},
+                        },
+                    },
+                },
+            },
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2, e3, e4, e5, e6 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque([]))
+        self.assertEqual(e2.path, deque([0]))
+        self.assertEqual(e3.path, deque([1, "bar"]))
+        self.assertEqual(e4.path, deque([1, "bar", "bar"]))
+        self.assertEqual(e5.path, deque([1, "bar", "baz"]))
+        self.assertEqual(e6.path, deque([1, "foo"]))
+
+        self.assertEqual(e1.schema_path, deque(["type"]))
+        self.assertEqual(e2.schema_path, deque(["items", "type"]))
+        self.assertEqual(
+            list(e3.schema_path), ["items", "properties", "bar", "type"],
+        )
+        self.assertEqual(
+            list(e4.schema_path),
+            ["items", "properties", "bar", "properties", "bar", "required"],
+        )
+        self.assertEqual(
+            list(e5.schema_path),
+            ["items", "properties", "bar", "properties", "baz", "minItems"]
+        )
+        self.assertEqual(
+            list(e6.schema_path), ["items", "properties", "foo", "enum"],
+        )
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "type")
+        self.assertEqual(e3.validator, "type")
+        self.assertEqual(e4.validator, "required")
+        self.assertEqual(e5.validator, "minItems")
+        self.assertEqual(e6.validator, "enum")
+
+    def test_recursive(self):
+        schema = {
+            "definitions": {
+                "node": {
+                    "anyOf": [{
+                        "type": "object",
+                        "required": ["name", "children"],
+                        "properties": {
+                            "name": {
+                                "type": "string",
+                            },
+                            "children": {
+                                "type": "object",
+                                "patternProperties": {
+                                    "^.*$": {
+                                        "$ref": "#/definitions/node",
+                                    },
+                                },
+                            },
+                        },
+                    }],
+                },
+            },
+            "type": "object",
+            "required": ["root"],
+            "properties": {"root": {"$ref": "#/definitions/node"}},
+        }
+
+        instance = {
+            "root": {
+                "name": "root",
+                "children": {
+                    "a": {
+                        "name": "a",
+                        "children": {
+                            "ab": {
+                                "name": "ab",
+                                # missing "children"
+                            },
+                        },
+                    },
+                },
+            },
+        }
+        validator = validators.Draft4Validator(schema)
+
+        e, = validator.iter_errors(instance)
+        self.assertEqual(e.absolute_path, deque(["root"]))
+        self.assertEqual(
+            e.absolute_schema_path, deque(["properties", "root", "anyOf"]),
+        )
+
+        e1, = e.context
+        self.assertEqual(e1.absolute_path, deque(["root", "children", "a"]))
+        self.assertEqual(
+            e1.absolute_schema_path, deque(
+                [
+                    "properties",
+                    "root",
+                    "anyOf",
+                    0,
+                    "properties",
+                    "children",
+                    "patternProperties",
+                    "^.*$",
+                    "anyOf",
+                ],
+            ),
+        )
+
+        e2, = e1.context
+        self.assertEqual(
+            e2.absolute_path, deque(
+                ["root", "children", "a", "children", "ab"],
+            ),
+        )
+        self.assertEqual(
+            e2.absolute_schema_path, deque(
+                [
+                    "properties",
+                    "root",
+                    "anyOf",
+                    0,
+                    "properties",
+                    "children",
+                    "patternProperties",
+                    "^.*$",
+                    "anyOf",
+                    0,
+                    "properties",
+                    "children",
+                    "patternProperties",
+                    "^.*$",
+                    "anyOf",
+                ],
+            ),
+        )
+
+    def test_additionalProperties(self):
+        instance = {"bar": "bar", "foo": 2}
+        schema = {"additionalProperties": {"type": "integer", "minimum": 5}}
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque(["bar"]))
+        self.assertEqual(e2.path, deque(["foo"]))
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "minimum")
+
+    def test_patternProperties(self):
+        instance = {"bar": 1, "foo": 2}
+        schema = {
+            "patternProperties": {
+                "bar": {"type": "string"},
+                "foo": {"minimum": 5},
+            },
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque(["bar"]))
+        self.assertEqual(e2.path, deque(["foo"]))
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "minimum")
+
+    def test_additionalItems(self):
+        instance = ["foo", 1]
+        schema = {
+            "items": [],
+            "additionalItems": {"type": "integer", "minimum": 5},
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque([0]))
+        self.assertEqual(e2.path, deque([1]))
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "minimum")
+
+    def test_additionalItems_with_items(self):
+        instance = ["foo", "bar", 1]
+        schema = {
+            "items": [{}],
+            "additionalItems": {"type": "integer", "minimum": 5},
+        }
+
+        validator = validators.Draft3Validator(schema)
+        errors = validator.iter_errors(instance)
+        e1, e2 = sorted_errors(errors)
+
+        self.assertEqual(e1.path, deque([1]))
+        self.assertEqual(e2.path, deque([2]))
+
+        self.assertEqual(e1.validator, "type")
+        self.assertEqual(e2.validator, "minimum")
+
+    def test_propertyNames(self):
+        instance = {"foo": 12}
+        schema = {"propertyNames": {"not": {"const": "foo"}}}
+
+        validator = validators.Draft7Validator(schema)
+        error, = validator.iter_errors(instance)
+
+        self.assertEqual(error.validator, "not")
+        self.assertEqual(
+            error.message,
+            "%r is not allowed for %r" % ({"const": "foo"}, "foo"),
+        )
+        self.assertEqual(error.path, deque([]))
+        self.assertEqual(error.schema_path, deque(["propertyNames", "not"]))
+
+    def test_if_then(self):
+        schema = {
+            "if": {"const": 12},
+            "then": {"const": 13},
+        }
+
+        validator = validators.Draft7Validator(schema)
+        error, = validator.iter_errors(12)
+
+        self.assertEqual(error.validator, "const")
+        self.assertEqual(error.message, "13 was expected")
+        self.assertEqual(error.path, deque([]))
+        self.assertEqual(error.schema_path, deque(["if", "then", "const"]))
+
+    def test_if_else(self):
+        schema = {
+            "if": {"const": 12},
+            "else": {"const": 13},
+        }
+
+        validator = validators.Draft7Validator(schema)
+        error, = validator.iter_errors(15)
+
+        self.assertEqual(error.validator, "const")
+        self.assertEqual(error.message, "13 was expected")
+        self.assertEqual(error.path, deque([]))
+        self.assertEqual(error.schema_path, deque(["if", "else", "const"]))
+
+    def test_boolean_schema_False(self):
+        validator = validators.Draft7Validator(False)
+        error, = validator.iter_errors(12)
+
+        self.assertEqual(
+            (
+                error.message,
+                error.validator,
+                error.validator_value,
+                error.instance,
+                error.schema,
+                error.schema_path,
+            ),
+            (
+                "False schema does not allow 12",
+                None,
+                None,
+                12,
+                False,
+                deque([]),
+            ),
+        )
+
+    def test_ref(self):
+        ref, schema = "someRef", {"additionalProperties": {"type": "integer"}}
+        validator = validators.Draft7Validator(
+            {"$ref": ref},
+            resolver=validators.RefResolver("", {}, store={ref: schema}),
+        )
+        error, = validator.iter_errors({"foo": "notAnInteger"})
+
+        self.assertEqual(
+            (
+                error.message,
+                error.validator,
+                error.validator_value,
+                error.instance,
+                error.absolute_path,
+                error.schema,
+                error.schema_path,
+            ),
+            (
+                "'notAnInteger' is not of type 'integer'",
+                "type",
+                "integer",
+                "notAnInteger",
+                deque(["foo"]),
+                {"type": "integer"},
+                deque(["additionalProperties", "type"]),
+            ),
+        )
+
+
+class MetaSchemaTestsMixin(object):
+    # TODO: These all belong upstream
+    def test_invalid_properties(self):
+        with self.assertRaises(exceptions.SchemaError):
+            self.Validator.check_schema({"properties": {"test": object()}})
+
+    def test_minItems_invalid_string(self):
+        with self.assertRaises(exceptions.SchemaError):
+            # needs to be an integer
+            self.Validator.check_schema({"minItems": "1"})
+
+    def test_enum_allows_empty_arrays(self):
+        """
+        Technically, all the spec says is they SHOULD have elements, not MUST.
+
+        See https://github.com/Julian/jsonschema/issues/529.
+        """
+        self.Validator.check_schema({"enum": []})
+
+    def test_enum_allows_non_unique_items(self):
+        """
+        Technically, all the spec says is they SHOULD be unique, not MUST.
+
+        See https://github.com/Julian/jsonschema/issues/529.
+        """
+        self.Validator.check_schema({"enum": [12, 12]})
+
+
+class ValidatorTestMixin(MetaSchemaTestsMixin, object):
+    def test_valid_instances_are_valid(self):
+        schema, instance = self.valid
+        self.assertTrue(self.Validator(schema).is_valid(instance))
+
+    def test_invalid_instances_are_not_valid(self):
+        schema, instance = self.invalid
+        self.assertFalse(self.Validator(schema).is_valid(instance))
+
+    def test_non_existent_properties_are_ignored(self):
+        self.Validator({object(): object()}).validate(instance=object())
+
+    def test_it_creates_a_ref_resolver_if_not_provided(self):
+        self.assertIsInstance(
+            self.Validator({}).resolver,
+            validators.RefResolver,
+        )
+
+    def test_it_delegates_to_a_ref_resolver(self):
+        ref, schema = "someCoolRef", {"type": "integer"}
+        resolver = validators.RefResolver("", {}, store={ref: schema})
+        validator = self.Validator({"$ref": ref}, resolver=resolver)
+
+        with self.assertRaises(exceptions.ValidationError):
+            validator.validate(None)
+
+    def test_it_delegates_to_a_legacy_ref_resolver(self):
+        """
+        Legacy RefResolvers support only the context manager form of
+        resolution.
+        """
+
+        class LegacyRefResolver(object):
+            @contextmanager
+            def resolving(this, ref):
+                self.assertEqual(ref, "the ref")
+                yield {"type": "integer"}
+
+        resolver = LegacyRefResolver()
+        schema = {"$ref": "the ref"}
+
+        with self.assertRaises(exceptions.ValidationError):
+            self.Validator(schema, resolver=resolver).validate(None)
+
+    def test_is_type_is_true_for_valid_type(self):
+        self.assertTrue(self.Validator({}).is_type("foo", "string"))
+
+    def test_is_type_is_false_for_invalid_type(self):
+        self.assertFalse(self.Validator({}).is_type("foo", "array"))
+
+    def test_is_type_evades_bool_inheriting_from_int(self):
+        self.assertFalse(self.Validator({}).is_type(True, "integer"))
+        self.assertFalse(self.Validator({}).is_type(True, "number"))
+
+    @unittest.skipIf(PY3, "In Python 3 json.load always produces unicode")
+    def test_string_a_bytestring_is_a_string(self):
+        self.Validator({"type": "string"}).validate(b"foo")
+
+    def test_patterns_can_be_native_strings(self):
+        """
+        See https://github.com/Julian/jsonschema/issues/611.
+        """
+        self.Validator({"pattern": "foo"}).validate("foo")
+
+    def test_it_can_validate_with_decimals(self):
+        schema = {"items": {"type": "number"}}
+        Validator = validators.extend(
+            self.Validator,
+            type_checker=self.Validator.TYPE_CHECKER.redefine(
+                "number",
+                lambda checker, thing: isinstance(
+                    thing, (int, float, Decimal),
+                ) and not isinstance(thing, bool),
+            )
+        )
+
+        validator = Validator(schema)
+        validator.validate([1, 1.1, Decimal(1) / Decimal(8)])
+
+        invalid = ["foo", {}, [], True, None]
+        self.assertEqual(
+            [error.instance for error in validator.iter_errors(invalid)],
+            invalid,
+        )
+
+    def test_it_returns_true_for_formats_it_does_not_know_about(self):
+        validator = self.Validator(
+            {"format": "carrot"}, format_checker=FormatChecker(),
+        )
+        validator.validate("bugs")
+
+    def test_it_does_not_validate_formats_by_default(self):
+        validator = self.Validator({})
+        self.assertIsNone(validator.format_checker)
+
+    def test_it_validates_formats_if_a_checker_is_provided(self):
+        checker = FormatChecker()
+        bad = ValueError("Bad!")
+
+        @checker.checks("foo", raises=ValueError)
+        def check(value):
+            if value == "good":
+                return True
+            elif value == "bad":
+                raise bad
+            else:  # pragma: no cover
+                self.fail("What is {}? [Baby Don't Hurt Me]".format(value))
+
+        validator = self.Validator(
+            {"format": "foo"}, format_checker=checker,
+        )
+
+        validator.validate("good")
+        with self.assertRaises(exceptions.ValidationError) as cm:
+            validator.validate("bad")
+
+        # Make sure original cause is attached
+        self.assertIs(cm.exception.cause, bad)
+
+    def test_non_string_custom_type(self):
+        non_string_type = object()
+        schema = {"type": [non_string_type]}
+        Crazy = validators.extend(
+            self.Validator,
+            type_checker=self.Validator.TYPE_CHECKER.redefine(
+                non_string_type,
+                lambda checker, thing: isinstance(thing, int),
+            )
+        )
+        Crazy(schema).validate(15)
+
+    def test_it_properly_formats_tuples_in_errors(self):
+        """
+        A tuple instance properly formats validation errors for uniqueItems.
+
+        See https://github.com/Julian/jsonschema/pull/224
+        """
+        TupleValidator = validators.extend(
+            self.Validator,
+            type_checker=self.Validator.TYPE_CHECKER.redefine(
+                "array",
+                lambda checker, thing: isinstance(thing, tuple),
+            )
+        )
+        with self.assertRaises(exceptions.ValidationError) as e:
+            TupleValidator({"uniqueItems": True}).validate((1, 1))
+        self.assertIn("(1, 1) has non-unique elements", str(e.exception))
+
+
+class AntiDraft6LeakMixin(object):
+    """
+    Make sure functionality from draft 6 doesn't leak backwards in time.
+    """
+
+    def test_True_is_not_a_schema(self):
+        with self.assertRaises(exceptions.SchemaError) as e:
+            self.Validator.check_schema(True)
+        self.assertIn("True is not of type", str(e.exception))
+
+    def test_False_is_not_a_schema(self):
+        with self.assertRaises(exceptions.SchemaError) as e:
+            self.Validator.check_schema(False)
+        self.assertIn("False is not of type", str(e.exception))
+
+    @unittest.skip(bug(523))
+    def test_True_is_not_a_schema_even_if_you_forget_to_check(self):
+        resolver = validators.RefResolver("", {})
+        with self.assertRaises(Exception) as e:
+            self.Validator(True, resolver=resolver).validate(12)
+        self.assertNotIsInstance(e.exception, exceptions.ValidationError)
+
+    @unittest.skip(bug(523))
+    def test_False_is_not_a_schema_even_if_you_forget_to_check(self):
+        resolver = validators.RefResolver("", {})
+        with self.assertRaises(Exception) as e:
+            self.Validator(False, resolver=resolver).validate(12)
+        self.assertNotIsInstance(e.exception, exceptions.ValidationError)
+
+
+class TestDraft3Validator(AntiDraft6LeakMixin, ValidatorTestMixin, TestCase):
+    Validator = validators.Draft3Validator
+    valid = {}, {}
+    invalid = {"type": "integer"}, "foo"
+
+    def test_any_type_is_valid_for_type_any(self):
+        validator = self.Validator({"type": "any"})
+        validator.validate(object())
+
+    def test_any_type_is_redefinable(self):
+        """
+        Sigh, because why not.
+        """
+        Crazy = validators.extend(
+            self.Validator,
+            type_checker=self.Validator.TYPE_CHECKER.redefine(
+                "any", lambda checker, thing: isinstance(thing, int),
+            )
+        )
+        validator = Crazy({"type": "any"})
+        validator.validate(12)
+        with self.assertRaises(exceptions.ValidationError):
+            validator.validate("foo")
+
+    def test_is_type_is_true_for_any_type(self):
+        self.assertTrue(self.Validator({}).is_valid(object(), {"type": "any"}))
+
+    def test_is_type_does_not_evade_bool_if_it_is_being_tested(self):
+        self.assertTrue(self.Validator({}).is_type(True, "boolean"))
+        self.assertTrue(self.Validator({}).is_valid(True, {"type": "any"}))
+
+
+class TestDraft4Validator(AntiDraft6LeakMixin, ValidatorTestMixin, TestCase):
+    Validator = validators.Draft4Validator
+    valid = {}, {}
+    invalid = {"type": "integer"}, "foo"
+
+
+class TestDraft6Validator(ValidatorTestMixin, TestCase):
+    Validator = validators.Draft6Validator
+    valid = {}, {}
+    invalid = {"type": "integer"}, "foo"
+
+
+class TestDraft7Validator(ValidatorTestMixin, TestCase):
+    Validator = validators.Draft7Validator
+    valid = {}, {}
+    invalid = {"type": "integer"}, "foo"
+
+
+class TestValidatorFor(SynchronousTestCase):
+    def test_draft_3(self):
+        schema = {"$schema": "http://json-schema.org/draft-03/schema"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft3Validator,
+        )
+
+        schema = {"$schema": "http://json-schema.org/draft-03/schema#"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft3Validator,
+        )
+
+    def test_draft_4(self):
+        schema = {"$schema": "http://json-schema.org/draft-04/schema"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft4Validator,
+        )
+
+        schema = {"$schema": "http://json-schema.org/draft-04/schema#"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft4Validator,
+        )
+
+    def test_draft_6(self):
+        schema = {"$schema": "http://json-schema.org/draft-06/schema"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft6Validator,
+        )
+
+        schema = {"$schema": "http://json-schema.org/draft-06/schema#"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft6Validator,
+        )
+
+    def test_draft_7(self):
+        schema = {"$schema": "http://json-schema.org/draft-07/schema"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft7Validator,
+        )
+
+        schema = {"$schema": "http://json-schema.org/draft-07/schema#"}
+        self.assertIs(
+            validators.validator_for(schema),
+            validators.Draft7Validator,
+        )
+
+    def test_True(self):
+        self.assertIs(
+            validators.validator_for(True),
+            validators._LATEST_VERSION,
+        )
+
+    def test_False(self):
+        self.assertIs(
+            validators.validator_for(False),
+            validators._LATEST_VERSION,
+        )
+
+    def test_custom_validator(self):
+        Validator = validators.create(
+            meta_schema={"id": "meta schema id"},
+            version="12",
+            id_of=lambda s: s.get("id", ""),
+        )
+        schema = {"$schema": "meta schema id"}
+        self.assertIs(
+            validators.validator_for(schema),
+            Validator,
+        )
+
+    def test_custom_validator_draft6(self):
+        Validator = validators.create(
+            meta_schema={"$id": "meta schema $id"},
+            version="13",
+        )
+        schema = {"$schema": "meta schema $id"}
+        self.assertIs(
+            validators.validator_for(schema),
+            Validator,
+        )
+
+    def test_validator_for_jsonschema_default(self):
+        self.assertIs(validators.validator_for({}), validators._LATEST_VERSION)
+
+    def test_validator_for_custom_default(self):
+        self.assertIs(validators.validator_for({}, default=None), None)
+
+    def test_warns_if_meta_schema_specified_was_not_found(self):
+        self.assertWarns(
+            category=DeprecationWarning,
+            message=(
+                "The metaschema specified by $schema was not found. "
+                "Using the latest draft to validate, but this will raise "
+                "an error in the future."
+            ),
+            # https://tm.tl/9363 :'(
+            filename=sys.modules[self.assertWarns.__module__].__file__,
+
+            f=validators.validator_for,
+            schema={u"$schema": "unknownSchema"},
+            default={},
+        )
+
+    def test_does_not_warn_if_meta_schema_is_unspecified(self):
+        validators.validator_for(schema={}, default={}),
+        self.assertFalse(self.flushWarnings())
+
+
+class TestValidate(SynchronousTestCase):
+    def assertUses(self, schema, Validator):
+        result = []
+        self.patch(Validator, "check_schema", result.append)
+        validators.validate({}, schema)
+        self.assertEqual(result, [schema])
+
+    def test_draft3_validator_is_chosen(self):
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-03/schema#"},
+            Validator=validators.Draft3Validator,
+        )
+        # Make sure it works without the empty fragment
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-03/schema"},
+            Validator=validators.Draft3Validator,
+        )
+
+    def test_draft4_validator_is_chosen(self):
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-04/schema#"},
+            Validator=validators.Draft4Validator,
+        )
+        # Make sure it works without the empty fragment
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-04/schema"},
+            Validator=validators.Draft4Validator,
+        )
+
+    def test_draft6_validator_is_chosen(self):
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-06/schema#"},
+            Validator=validators.Draft6Validator,
+        )
+        # Make sure it works without the empty fragment
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-06/schema"},
+            Validator=validators.Draft6Validator,
+        )
+
+    def test_draft7_validator_is_chosen(self):
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-07/schema#"},
+            Validator=validators.Draft7Validator,
+        )
+        # Make sure it works without the empty fragment
+        self.assertUses(
+            schema={"$schema": "http://json-schema.org/draft-07/schema"},
+            Validator=validators.Draft7Validator,
+        )
+
+    def test_draft7_validator_is_the_default(self):
+        self.assertUses(schema={}, Validator=validators.Draft7Validator)
+
+    def test_validation_error_message(self):
+        with self.assertRaises(exceptions.ValidationError) as e:
+            validators.validate(12, {"type": "string"})
+        self.assertRegexpMatches(
+            str(e.exception),
+            "(?s)Failed validating u?'.*' in schema.*On instance",
+        )
+
+    def test_schema_error_message(self):
+        with self.assertRaises(exceptions.SchemaError) as e:
+            validators.validate(12, {"type": 12})
+        self.assertRegexpMatches(
+            str(e.exception),
+            "(?s)Failed validating u?'.*' in metaschema.*On schema",
+        )
+
+    def test_it_uses_best_match(self):
+        # This is a schema that best_match will recurse into
+        schema = {"oneOf": [{"type": "string"}, {"type": "array"}]}
+        with self.assertRaises(exceptions.ValidationError) as e:
+            validators.validate(12, schema)
+        self.assertIn("12 is not of type", str(e.exception))
+
+
+class TestRefResolver(SynchronousTestCase):
+
+    base_uri = ""
+    stored_uri = "foo://stored"
+    stored_schema = {"stored": "schema"}
+
+    def setUp(self):
+        self.referrer = {}
+        self.store = {self.stored_uri: self.stored_schema}
+        self.resolver = validators.RefResolver(
+            self.base_uri, self.referrer, self.store,
+        )
+
+    def test_it_does_not_retrieve_schema_urls_from_the_network(self):
+        ref = validators.Draft3Validator.META_SCHEMA["id"]
+        self.patch(
+            self.resolver,
+            "resolve_remote",
+            lambda *args, **kwargs: self.fail("Should not have been called!"),
+        )
+        with self.resolver.resolving(ref) as resolved:
+            pass
+        self.assertEqual(resolved, validators.Draft3Validator.META_SCHEMA)
+
+    def test_it_resolves_local_refs(self):
+        ref = "#/properties/foo"
+        self.referrer["properties"] = {"foo": object()}
+        with self.resolver.resolving(ref) as resolved:
+            self.assertEqual(resolved, self.referrer["properties"]["foo"])
+
+    def test_it_resolves_local_refs_with_id(self):
+        schema = {"id": "http://bar/schema#", "a": {"foo": "bar"}}
+        resolver = validators.RefResolver.from_schema(
+            schema,
+            id_of=lambda schema: schema.get(u"id", u""),
+        )
+        with resolver.resolving("#/a") as resolved:
+            self.assertEqual(resolved, schema["a"])
+        with resolver.resolving("http://bar/schema#/a") as resolved:
+            self.assertEqual(resolved, schema["a"])
+
+    def test_it_retrieves_stored_refs(self):
+        with self.resolver.resolving(self.stored_uri) as resolved:
+            self.assertIs(resolved, self.stored_schema)
+
+        self.resolver.store["cached_ref"] = {"foo": 12}
+        with self.resolver.resolving("cached_ref#/foo") as resolved:
+            self.assertEqual(resolved, 12)
+
+    def test_it_retrieves_unstored_refs_via_requests(self):
+        ref = "http://bar#baz"
+        schema = {"baz": 12}
+
+        if "requests" in sys.modules:
+            self.addCleanup(
+                sys.modules.__setitem__, "requests", sys.modules["requests"],
+            )
+        sys.modules["requests"] = ReallyFakeRequests({"http://bar": schema})
+
+        with self.resolver.resolving(ref) as resolved:
+            self.assertEqual(resolved, 12)
+
+    def test_it_retrieves_unstored_refs_via_urlopen(self):
+        ref = "http://bar#baz"
+        schema = {"baz": 12}
+
+        if "requests" in sys.modules:
+            self.addCleanup(
+                sys.modules.__setitem__, "requests", sys.modules["requests"],
+            )
+        sys.modules["requests"] = None
+
+        @contextmanager
+        def fake_urlopen(url):
+            self.assertEqual(url, "http://bar")
+            yield BytesIO(json.dumps(schema).encode("utf8"))
+
+        self.addCleanup(setattr, validators, "urlopen", validators.urlopen)
+        validators.urlopen = fake_urlopen
+
+        with self.resolver.resolving(ref) as resolved:
+            pass
+        self.assertEqual(resolved, 12)
+
+    def test_it_retrieves_local_refs_via_urlopen(self):
+        with tempfile.NamedTemporaryFile(delete=False, mode="wt") as tempf:
+            self.addCleanup(os.remove, tempf.name)
+            json.dump({"foo": "bar"}, tempf)
+
+        ref = "file://{}#foo".format(pathname2url(tempf.name))
+        with self.resolver.resolving(ref) as resolved:
+            self.assertEqual(resolved, "bar")
+
+    def test_it_can_construct_a_base_uri_from_a_schema(self):
+        schema = {"id": "foo"}
+        resolver = validators.RefResolver.from_schema(
+            schema,
+            id_of=lambda schema: schema.get(u"id", u""),
+        )
+        self.assertEqual(resolver.base_uri, "foo")
+        self.assertEqual(resolver.resolution_scope, "foo")
+        with resolver.resolving("") as resolved:
+            self.assertEqual(resolved, schema)
+        with resolver.resolving("#") as resolved:
+            self.assertEqual(resolved, schema)
+        with resolver.resolving("foo") as resolved:
+            self.assertEqual(resolved, schema)
+        with resolver.resolving("foo#") as resolved:
+            self.assertEqual(resolved, schema)
+
+    def test_it_can_construct_a_base_uri_from_a_schema_without_id(self):
+        schema = {}
+        resolver = validators.RefResolver.from_schema(schema)
+        self.assertEqual(resolver.base_uri, "")
+        self.assertEqual(resolver.resolution_scope, "")
+        with resolver.resolving("") as resolved:
+            self.assertEqual(resolved, schema)
+        with resolver.resolving("#") as resolved:
+            self.assertEqual(resolved, schema)
+
+    def test_custom_uri_scheme_handlers(self):
+        def handler(url):
+            self.assertEqual(url, ref)
+            return schema
+
+        schema = {"foo": "bar"}
+        ref = "foo://bar"
+        resolver = validators.RefResolver("", {}, handlers={"foo": handler})
+        with resolver.resolving(ref) as resolved:
+            self.assertEqual(resolved, schema)
+
+    def test_cache_remote_on(self):
+        response = [object()]
+
+        def handler(url):
+            try:
+                return response.pop()
+            except IndexError:  # pragma: no cover
+                self.fail("Response must not have been cached!")
+
+        ref = "foo://bar"
+        resolver = validators.RefResolver(
+            "", {}, cache_remote=True, handlers={"foo": handler},
+        )
+        with resolver.resolving(ref):
+            pass
+        with resolver.resolving(ref):
+            pass
+
+    def test_cache_remote_off(self):
+        response = [object()]
+
+        def handler(url):
+            try:
+                return response.pop()
+            except IndexError:  # pragma: no cover
+                self.fail("Handler called twice!")
+
+        ref = "foo://bar"
+        resolver = validators.RefResolver(
+            "", {}, cache_remote=False, handlers={"foo": handler},
+        )
+        with resolver.resolving(ref):
+            pass
+
+    def test_if_you_give_it_junk_you_get_a_resolution_error(self):
+        error = ValueError("Oh no! What's this?")
+
+        def handler(url):
+            raise error
+
+        ref = "foo://bar"
+        resolver = validators.RefResolver("", {}, handlers={"foo": handler})
+        with self.assertRaises(exceptions.RefResolutionError) as err:
+            with resolver.resolving(ref):
+                self.fail("Shouldn't get this far!")  # pragma: no cover
+        self.assertEqual(err.exception, exceptions.RefResolutionError(error))
+
+    def test_helpful_error_message_on_failed_pop_scope(self):
+        resolver = validators.RefResolver("", {})
+        resolver.pop_scope()
+        with self.assertRaises(exceptions.RefResolutionError) as exc:
+            resolver.pop_scope()
+        self.assertIn("Failed to pop the scope", str(exc.exception))
+
+
+def sorted_errors(errors):
+    def key(error):
+        return (
+            [str(e) for e in error.path],
+            [str(e) for e in error.schema_path],
+        )
+    return sorted(errors, key=key)
+
+
+@attr.s
+class ReallyFakeRequests(object):
+
+    _responses = attr.ib()
+
+    def get(self, url):
+        response = self._responses.get(url)
+        if url is None:  # pragma: no cover
+            raise ValueError("Unknown URL: " + repr(url))
+        return _ReallyFakeJSONResponse(json.dumps(response))
+
+
+@attr.s
+class _ReallyFakeJSONResponse(object):
+
+    _response = attr.ib()
+
+    def json(self):
+        return json.loads(self._response)
diff --git a/contrib/python/jsonschema/py3/jsonschema/validators.py b/contrib/python/jsonschema/py3/jsonschema/validators.py
new file mode 100644
index 00000000000..1dc420c70d2
--- /dev/null
+++ b/contrib/python/jsonschema/py3/jsonschema/validators.py
@@ -0,0 +1,970 @@
+"""
+Creation and extension of validators, with implementations for existing drafts.
+"""
+from __future__ import division
+
+from warnings import warn
+import contextlib
+import json
+import numbers
+
+from six import add_metaclass
+
+from jsonschema import (
+    _legacy_validators,
+    _types,
+    _utils,
+    _validators,
+    exceptions,
+)
+from jsonschema.compat import (
+    Sequence,
+    int_types,
+    iteritems,
+    lru_cache,
+    str_types,
+    unquote,
+    urldefrag,
+    urljoin,
+    urlopen,
+    urlsplit,
+)
+
+# Sigh. https://gitlab.com/pycqa/flake8/issues/280
+#       https://github.com/pyga/ebb-lint/issues/7
+# Imported for backwards compatibility.
+from jsonschema.exceptions import ErrorTree
+ErrorTree
+
+
+class _DontDoThat(Exception):
+    """
+    Raised when a Validators with non-default type checker is misused.
+
+    Asking one for DEFAULT_TYPES doesn't make sense, since type checkers
+    exist for the unrepresentable cases where DEFAULT_TYPES can't
+    represent the type relationship.
+    """
+
+    def __str__(self):
+        return "DEFAULT_TYPES cannot be used on Validators using TypeCheckers"
+
+
+validators = {}
+meta_schemas = _utils.URIDict()
+
+
+def _generate_legacy_type_checks(types=()):
+    """
+    Generate newer-style type checks out of JSON-type-name-to-type mappings.
+
+    Arguments:
+
+        types (dict):
+
+            A mapping of type names to their Python types
+
+    Returns:
+
+        A dictionary of definitions to pass to `TypeChecker`
+    """
+    types = dict(types)
+
+    def gen_type_check(pytypes):
+        pytypes = _utils.flatten(pytypes)
+
+        def type_check(checker, instance):
+            if isinstance(instance, bool):
+                if bool not in pytypes:
+                    return False
+            return isinstance(instance, pytypes)
+
+        return type_check
+
+    definitions = {}
+    for typename, pytypes in iteritems(types):
+        definitions[typename] = gen_type_check(pytypes)
+
+    return definitions
+
+
+_DEPRECATED_DEFAULT_TYPES = {
+    u"array": list,
+    u"boolean": bool,
+    u"integer": int_types,
+    u"null": type(None),
+    u"number": numbers.Number,
+    u"object": dict,
+    u"string": str_types,
+}
+_TYPE_CHECKER_FOR_DEPRECATED_DEFAULT_TYPES = _types.TypeChecker(
+    type_checkers=_generate_legacy_type_checks(_DEPRECATED_DEFAULT_TYPES),
+)
+
+
+def validates(version):
+    """
+    Register the decorated validator for a ``version`` of the specification.
+
+    Registered validators and their meta schemas will be considered when
+    parsing ``$schema`` properties' URIs.
+
+    Arguments:
+
+        version (str):
+
+            An identifier to use as the version's name
+
+    Returns:
+
+        collections.Callable:
+
+            a class decorator to decorate the validator with the version
+    """
+
+    def _validates(cls):
+        validators[version] = cls
+        meta_schema_id = cls.ID_OF(cls.META_SCHEMA)
+        if meta_schema_id:
+            meta_schemas[meta_schema_id] = cls
+        return cls
+    return _validates
+
+
+def _DEFAULT_TYPES(self):
+    if self._CREATED_WITH_DEFAULT_TYPES is None:
+        raise _DontDoThat()
+
+    warn(
+        (
+            "The DEFAULT_TYPES attribute is deprecated. "
+            "See the type checker attached to this validator instead."
+        ),
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return self._DEFAULT_TYPES
+
+
+class _DefaultTypesDeprecatingMetaClass(type):
+    DEFAULT_TYPES = property(_DEFAULT_TYPES)
+
+
+def _id_of(schema):
+    if schema is True or schema is False:
+        return u""
+    return schema.get(u"$id", u"")
+
+
+def create(
+    meta_schema,
+    validators=(),
+    version=None,
+    default_types=None,
+    type_checker=None,
+    id_of=_id_of,
+):
+    """
+    Create a new validator class.
+
+    Arguments:
+
+        meta_schema (collections.Mapping):
+
+            the meta schema for the new validator class
+
+        validators (collections.Mapping):
+
+            a mapping from names to callables, where each callable will
+            validate the schema property with the given name.
+
+            Each callable should take 4 arguments:
+
+                1. a validator instance,
+                2. the value of the property being validated within the
+                   instance
+                3. the instance
+                4. the schema
+
+        version (str):
+
+            an identifier for the version that this validator class will
+            validate. If provided, the returned validator class will
+            have its ``__name__`` set to include the version, and also
+            will have `jsonschema.validators.validates` automatically
+            called for the given version.
+
+        type_checker (jsonschema.TypeChecker):
+
+            a type checker, used when applying the :validator:`type` validator.
+
+            If unprovided, a `jsonschema.TypeChecker` will be created
+            with a set of default types typical of JSON Schema drafts.
+
+        default_types (collections.Mapping):
+
+            .. deprecated:: 3.0.0
+
+                Please use the type_checker argument instead.
+
+            If set, it provides mappings of JSON types to Python types
+            that will be converted to functions and redefined in this
+            object's `jsonschema.TypeChecker`.
+
+        id_of (collections.Callable):
+
+            A function that given a schema, returns its ID.
+
+    Returns:
+
+        a new `jsonschema.IValidator` class
+    """
+
+    if default_types is not None:
+        if type_checker is not None:
+            raise TypeError(
+                "Do not specify default_types when providing a type checker.",
+            )
+        _created_with_default_types = True
+        warn(
+            (
+                "The default_types argument is deprecated. "
+                "Use the type_checker argument instead."
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        type_checker = _types.TypeChecker(
+            type_checkers=_generate_legacy_type_checks(default_types),
+        )
+    else:
+        default_types = _DEPRECATED_DEFAULT_TYPES
+        if type_checker is None:
+            _created_with_default_types = False
+            type_checker = _TYPE_CHECKER_FOR_DEPRECATED_DEFAULT_TYPES
+        elif type_checker is _TYPE_CHECKER_FOR_DEPRECATED_DEFAULT_TYPES:
+            _created_with_default_types = False
+        else:
+            _created_with_default_types = None
+
+    @add_metaclass(_DefaultTypesDeprecatingMetaClass)
+    class Validator(object):
+
+        VALIDATORS = dict(validators)
+        META_SCHEMA = dict(meta_schema)
+        TYPE_CHECKER = type_checker
+        ID_OF = staticmethod(id_of)
+
+        DEFAULT_TYPES = property(_DEFAULT_TYPES)
+        _DEFAULT_TYPES = dict(default_types)
+        _CREATED_WITH_DEFAULT_TYPES = _created_with_default_types
+
+        def __init__(
+            self,
+            schema,
+            types=(),
+            resolver=None,
+            format_checker=None,
+        ):
+            if types:
+                warn(
+                    (
+                        "The types argument is deprecated. Provide "
+                        "a type_checker to jsonschema.validators.extend "
+                        "instead."
+                    ),
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+
+                self.TYPE_CHECKER = self.TYPE_CHECKER.redefine_many(
+                    _generate_legacy_type_checks(types),
+                )
+
+            if resolver is None:
+                resolver = RefResolver.from_schema(schema, id_of=id_of)
+
+            self.resolver = resolver
+            self.format_checker = format_checker
+            self.schema = schema
+
+        @classmethod
+        def check_schema(cls, schema):
+            for error in cls(cls.META_SCHEMA).iter_errors(schema):
+                raise exceptions.SchemaError.create_from(error)
+
+        def iter_errors(self, instance, _schema=None):
+            if _schema is None:
+                _schema = self.schema
+
+            if _schema is True:
+                return
+            elif _schema is False:
+                yield exceptions.ValidationError(
+                    "False schema does not allow %r" % (instance,),
+                    validator=None,
+                    validator_value=None,
+                    instance=instance,
+                    schema=_schema,
+                )
+                return
+
+            scope = id_of(_schema)
+            if scope:
+                self.resolver.push_scope(scope)
+            try:
+                ref = _schema.get(u"$ref")
+                if ref is not None:
+                    validators = [(u"$ref", ref)]
+                else:
+                    validators = iteritems(_schema)
+
+                for k, v in validators:
+                    validator = self.VALIDATORS.get(k)
+                    if validator is None:
+                        continue
+
+                    errors = validator(self, v, instance, _schema) or ()
+                    for error in errors:
+                        # set details if not already set by the called fn
+                        error._set(
+                            validator=k,
+                            validator_value=v,
+                            instance=instance,
+                            schema=_schema,
+                        )
+                        if k != u"$ref":
+                            error.schema_path.appendleft(k)
+                        yield error
+            finally:
+                if scope:
+                    self.resolver.pop_scope()
+
+        def descend(self, instance, schema, path=None, schema_path=None):
+            for error in self.iter_errors(instance, schema):
+                if path is not None:
+                    error.path.appendleft(path)
+                if schema_path is not None:
+                    error.schema_path.appendleft(schema_path)
+                yield error
+
+        def validate(self, *args, **kwargs):
+            for error in self.iter_errors(*args, **kwargs):
+                raise error
+
+        def is_type(self, instance, type):
+            try:
+                return self.TYPE_CHECKER.is_type(instance, type)
+            except exceptions.UndefinedTypeCheck:
+                raise exceptions.UnknownType(type, instance, self.schema)
+
+        def is_valid(self, instance, _schema=None):
+            error = next(self.iter_errors(instance, _schema), None)
+            return error is None
+
+    if version is not None:
+        Validator = validates(version)(Validator)
+        Validator.__name__ = version.title().replace(" ", "") + "Validator"
+
+    return Validator
+
+
+def extend(validator, validators=(), version=None, type_checker=None):
+    """
+    Create a new validator class by extending an existing one.
+
+    Arguments:
+
+        validator (jsonschema.IValidator):
+
+            an existing validator class
+
+        validators (collections.Mapping):
+
+            a mapping of new validator callables to extend with, whose
+            structure is as in `create`.
+
+            .. note::
+
+                Any validator callables with the same name as an
+                existing one will (silently) replace the old validator
+                callable entirely, effectively overriding any validation
+                done in the "parent" validator class.
+
+                If you wish to instead extend the behavior of a parent's
+                validator callable, delegate and call it directly in
+                the new validator function by retrieving it using
+                ``OldValidator.VALIDATORS["validator_name"]``.
+
+        version (str):
+
+            a version for the new validator class
+
+        type_checker (jsonschema.TypeChecker):
+
+            a type checker, used when applying the :validator:`type` validator.
+
+            If unprovided, the type checker of the extended
+            `jsonschema.IValidator` will be carried along.`
+
+    Returns:
+
+        a new `jsonschema.IValidator` class extending the one provided
+
+    .. note:: Meta Schemas
+
+        The new validator class will have its parent's meta schema.
+
+        If you wish to change or extend the meta schema in the new
+        validator class, modify ``META_SCHEMA`` directly on the returned
+        class. Note that no implicit copying is done, so a copy should
+        likely be made before modifying it, in order to not affect the
+        old validator.
+    """
+
+    all_validators = dict(validator.VALIDATORS)
+    all_validators.update(validators)
+
+    if type_checker is None:
+        type_checker = validator.TYPE_CHECKER
+    elif validator._CREATED_WITH_DEFAULT_TYPES:
+        raise TypeError(
+            "Cannot extend a validator created with default_types "
+            "with a type_checker. Update the validator to use a "
+            "type_checker when created."
+        )
+    return create(
+        meta_schema=validator.META_SCHEMA,
+        validators=all_validators,
+        version=version,
+        type_checker=type_checker,
+        id_of=validator.ID_OF,
+    )
+
+
+Draft3Validator = create(
+    meta_schema=_utils.load_schema("draft3"),
+    validators={
+        u"$ref": _validators.ref,
+        u"additionalItems": _validators.additionalItems,
+        u"additionalProperties": _validators.additionalProperties,
+        u"dependencies": _legacy_validators.dependencies_draft3,
+        u"disallow": _legacy_validators.disallow_draft3,
+        u"divisibleBy": _validators.multipleOf,
+        u"enum": _validators.enum,
+        u"extends": _legacy_validators.extends_draft3,
+        u"format": _validators.format,
+        u"items": _legacy_validators.items_draft3_draft4,
+        u"maxItems": _validators.maxItems,
+        u"maxLength": _validators.maxLength,
+        u"maximum": _legacy_validators.maximum_draft3_draft4,
+        u"minItems": _validators.minItems,
+        u"minLength": _validators.minLength,
+        u"minimum": _legacy_validators.minimum_draft3_draft4,
+        u"pattern": _validators.pattern,
+        u"patternProperties": _validators.patternProperties,
+        u"properties": _legacy_validators.properties_draft3,
+        u"type": _legacy_validators.type_draft3,
+        u"uniqueItems": _validators.uniqueItems,
+    },
+    type_checker=_types.draft3_type_checker,
+    version="draft3",
+    id_of=lambda schema: schema.get(u"id", ""),
+)
+
+Draft4Validator = create(
+    meta_schema=_utils.load_schema("draft4"),
+    validators={
+        u"$ref": _validators.ref,
+        u"additionalItems": _validators.additionalItems,
+        u"additionalProperties": _validators.additionalProperties,
+        u"allOf": _validators.allOf,
+        u"anyOf": _validators.anyOf,
+        u"dependencies": _validators.dependencies,
+        u"enum": _validators.enum,
+        u"format": _validators.format,
+        u"items": _legacy_validators.items_draft3_draft4,
+        u"maxItems": _validators.maxItems,
+        u"maxLength": _validators.maxLength,
+        u"maxProperties": _validators.maxProperties,
+        u"maximum": _legacy_validators.maximum_draft3_draft4,
+        u"minItems": _validators.minItems,
+        u"minLength": _validators.minLength,
+        u"minProperties": _validators.minProperties,
+        u"minimum": _legacy_validators.minimum_draft3_draft4,
+        u"multipleOf": _validators.multipleOf,
+        u"not": _validators.not_,
+        u"oneOf": _validators.oneOf,
+        u"pattern": _validators.pattern,
+        u"patternProperties": _validators.patternProperties,
+        u"properties": _validators.properties,
+        u"required": _validators.required,
+        u"type": _validators.type,
+        u"uniqueItems": _validators.uniqueItems,
+    },
+    type_checker=_types.draft4_type_checker,
+    version="draft4",
+    id_of=lambda schema: schema.get(u"id", ""),
+)
+
+Draft6Validator = create(
+    meta_schema=_utils.load_schema("draft6"),
+    validators={
+        u"$ref": _validators.ref,
+        u"additionalItems": _validators.additionalItems,
+        u"additionalProperties": _validators.additionalProperties,
+        u"allOf": _validators.allOf,
+        u"anyOf": _validators.anyOf,
+        u"const": _validators.const,
+        u"contains": _validators.contains,
+        u"dependencies": _validators.dependencies,
+        u"enum": _validators.enum,
+        u"exclusiveMaximum": _validators.exclusiveMaximum,
+        u"exclusiveMinimum": _validators.exclusiveMinimum,
+        u"format": _validators.format,
+        u"items": _validators.items,
+        u"maxItems": _validators.maxItems,
+        u"maxLength": _validators.maxLength,
+        u"maxProperties": _validators.maxProperties,
+        u"maximum": _validators.maximum,
+        u"minItems": _validators.minItems,
+        u"minLength": _validators.minLength,
+        u"minProperties": _validators.minProperties,
+        u"minimum": _validators.minimum,
+        u"multipleOf": _validators.multipleOf,
+        u"not": _validators.not_,
+        u"oneOf": _validators.oneOf,
+        u"pattern": _validators.pattern,
+        u"patternProperties": _validators.patternProperties,
+        u"properties": _validators.properties,
+        u"propertyNames": _validators.propertyNames,
+        u"required": _validators.required,
+        u"type": _validators.type,
+        u"uniqueItems": _validators.uniqueItems,
+    },
+    type_checker=_types.draft6_type_checker,
+    version="draft6",
+)
+
+Draft7Validator = create(
+    meta_schema=_utils.load_schema("draft7"),
+    validators={
+        u"$ref": _validators.ref,
+        u"additionalItems": _validators.additionalItems,
+        u"additionalProperties": _validators.additionalProperties,
+        u"allOf": _validators.allOf,
+        u"anyOf": _validators.anyOf,
+        u"const": _validators.const,
+        u"contains": _validators.contains,
+        u"dependencies": _validators.dependencies,
+        u"enum": _validators.enum,
+        u"exclusiveMaximum": _validators.exclusiveMaximum,
+        u"exclusiveMinimum": _validators.exclusiveMinimum,
+        u"format": _validators.format,
+        u"if": _validators.if_,
+        u"items": _validators.items,
+        u"maxItems": _validators.maxItems,
+        u"maxLength": _validators.maxLength,
+        u"maxProperties": _validators.maxProperties,
+        u"maximum": _validators.maximum,
+        u"minItems": _validators.minItems,
+        u"minLength": _validators.minLength,
+        u"minProperties": _validators.minProperties,
+        u"minimum": _validators.minimum,
+        u"multipleOf": _validators.multipleOf,
+        u"oneOf": _validators.oneOf,
+        u"not": _validators.not_,
+        u"pattern": _validators.pattern,
+        u"patternProperties": _validators.patternProperties,
+        u"properties": _validators.properties,
+        u"propertyNames": _validators.propertyNames,
+        u"required": _validators.required,
+        u"type": _validators.type,
+        u"uniqueItems": _validators.uniqueItems,
+    },
+    type_checker=_types.draft7_type_checker,
+    version="draft7",
+)
+
+_LATEST_VERSION = Draft7Validator
+
+
+class RefResolver(object):
+    """
+    Resolve JSON References.
+
+    Arguments:
+
+        base_uri (str):
+
+            The URI of the referring document
+
+        referrer:
+
+            The actual referring document
+
+        store (dict):
+
+            A mapping from URIs to documents to cache
+
+        cache_remote (bool):
+
+            Whether remote refs should be cached after first resolution
+
+        handlers (dict):
+
+            A mapping from URI schemes to functions that should be used
+            to retrieve them
+
+        urljoin_cache (:func:`functools.lru_cache`):
+
+            A cache that will be used for caching the results of joining
+            the resolution scope to subscopes.
+
+        remote_cache (:func:`functools.lru_cache`):
+
+            A cache that will be used for caching the results of
+            resolved remote URLs.
+
+    Attributes:
+
+        cache_remote (bool):
+
+            Whether remote refs should be cached after first resolution
+    """
+
+    def __init__(
+        self,
+        base_uri,
+        referrer,
+        store=(),
+        cache_remote=True,
+        handlers=(),
+        urljoin_cache=None,
+        remote_cache=None,
+    ):
+        if urljoin_cache is None:
+            urljoin_cache = lru_cache(1024)(urljoin)
+        if remote_cache is None:
+            remote_cache = lru_cache(1024)(self.resolve_from_url)
+
+        self.referrer = referrer
+        self.cache_remote = cache_remote
+        self.handlers = dict(handlers)
+
+        self._scopes_stack = [base_uri]
+        self.store = _utils.URIDict(
+            (id, validator.META_SCHEMA)
+            for id, validator in iteritems(meta_schemas)
+        )
+        self.store.update(store)
+        self.store[base_uri] = referrer
+
+        self._urljoin_cache = urljoin_cache
+        self._remote_cache = remote_cache
+
+    @classmethod
+    def from_schema(cls, schema, id_of=_id_of, *args, **kwargs):
+        """
+        Construct a resolver from a JSON schema object.
+
+        Arguments:
+
+            schema:
+
+                the referring schema
+
+        Returns:
+
+            `RefResolver`
+        """
+
+        return cls(base_uri=id_of(schema), referrer=schema, *args, **kwargs)
+
+    def push_scope(self, scope):
+        """
+        Enter a given sub-scope.
+
+        Treats further dereferences as being performed underneath the
+        given scope.
+        """
+        self._scopes_stack.append(
+            self._urljoin_cache(self.resolution_scope, scope),
+        )
+
+    def pop_scope(self):
+        """
+        Exit the most recent entered scope.
+
+        Treats further dereferences as being performed underneath the
+        original scope.
+
+        Don't call this method more times than `push_scope` has been
+        called.
+        """
+        try:
+            self._scopes_stack.pop()
+        except IndexError:
+            raise exceptions.RefResolutionError(
+                "Failed to pop the scope from an empty stack. "
+                "`pop_scope()` should only be called once for every "
+                "`push_scope()`"
+            )
+
+    @property
+    def resolution_scope(self):
+        """
+        Retrieve the current resolution scope.
+        """
+        return self._scopes_stack[-1]
+
+    @property
+    def base_uri(self):
+        """
+        Retrieve the current base URI, not including any fragment.
+        """
+        uri, _ = urldefrag(self.resolution_scope)
+        return uri
+
+    @contextlib.contextmanager
+    def in_scope(self, scope):
+        """
+        Temporarily enter the given scope for the duration of the context.
+        """
+        self.push_scope(scope)
+        try:
+            yield
+        finally:
+            self.pop_scope()
+
+    @contextlib.contextmanager
+    def resolving(self, ref):
+        """
+        Resolve the given ``ref`` and enter its resolution scope.
+
+        Exits the scope on exit of this context manager.
+
+        Arguments:
+
+            ref (str):
+
+                The reference to resolve
+        """
+
+        url, resolved = self.resolve(ref)
+        self.push_scope(url)
+        try:
+            yield resolved
+        finally:
+            self.pop_scope()
+
+    def resolve(self, ref):
+        """
+        Resolve the given reference.
+        """
+        url = self._urljoin_cache(self.resolution_scope, ref)
+        return url, self._remote_cache(url)
+
+    def resolve_from_url(self, url):
+        """
+        Resolve the given remote URL.
+        """
+        url, fragment = urldefrag(url)
+        try:
+            document = self.store[url]
+        except KeyError:
+            try:
+                document = self.resolve_remote(url)
+            except Exception as exc:
+                raise exceptions.RefResolutionError(exc)
+
+        return self.resolve_fragment(document, fragment)
+
+    def resolve_fragment(self, document, fragment):
+        """
+        Resolve a ``fragment`` within the referenced ``document``.
+
+        Arguments:
+
+            document:
+
+                The referent document
+
+            fragment (str):
+
+                a URI fragment to resolve within it
+        """
+
+        fragment = fragment.lstrip(u"/")
+        parts = unquote(fragment).split(u"/") if fragment else []
+
+        for part in parts:
+            part = part.replace(u"~1", u"/").replace(u"~0", u"~")
+
+            if isinstance(document, Sequence):
+                # Array indexes should be turned into integers
+                try:
+                    part = int(part)
+                except ValueError:
+                    pass
+            try:
+                document = document[part]
+            except (TypeError, LookupError):
+                raise exceptions.RefResolutionError(
+                    "Unresolvable JSON pointer: %r" % fragment
+                )
+
+        return document
+
+    def resolve_remote(self, uri):
+        """
+        Resolve a remote ``uri``.
+
+        If called directly, does not check the store first, but after
+        retrieving the document at the specified URI it will be saved in
+        the store if :attr:`cache_remote` is True.
+
+        .. note::
+
+            If the requests_ library is present, ``jsonschema`` will use it to
+            request the remote ``uri``, so that the correct encoding is
+            detected and used.
+
+            If it isn't, or if the scheme of the ``uri`` is not ``http`` or
+            ``https``, UTF-8 is assumed.
+
+        Arguments:
+
+            uri (str):
+
+                The URI to resolve
+
+        Returns:
+
+            The retrieved document
+
+        .. _requests: https://pypi.org/project/requests/
+        """
+        try:
+            import requests
+        except ImportError:
+            requests = None
+
+        scheme = urlsplit(uri).scheme
+
+        if scheme in self.handlers:
+            result = self.handlers[scheme](uri)
+        elif scheme in [u"http", u"https"] and requests:
+            # Requests has support for detecting the correct encoding of
+            # json over http
+            result = requests.get(uri).json()
+        else:
+            # Otherwise, pass off to urllib and assume utf-8
+            with urlopen(uri) as url:
+                result = json.loads(url.read().decode("utf-8"))
+
+        if self.cache_remote:
+            self.store[uri] = result
+        return result
+
+
+def validate(instance, schema, cls=None, *args, **kwargs):
+    """
+    Validate an instance under the given schema.
+
+        >>> validate([2, 3, 4], {"maxItems": 2})
+        Traceback (most recent call last):
+            ...
+        ValidationError: [2, 3, 4] is too long
+
+    :func:`validate` will first verify that the provided schema is
+    itself valid, since not doing so can lead to less obvious error
+    messages and fail in less obvious or consistent ways.
+
+    If you know you have a valid schema already, especially if you
+    intend to validate multiple instances with the same schema, you
+    likely would prefer using the `IValidator.validate` method directly
+    on a specific validator (e.g. ``Draft7Validator.validate``).
+
+
+    Arguments:
+
+        instance:
+
+            The instance to validate
+
+        schema:
+
+            The schema to validate with
+
+        cls (IValidator):
+
+            The class that will be used to validate the instance.
+
+    If the ``cls`` argument is not provided, two things will happen
+    in accordance with the specification. First, if the schema has a
+    :validator:`$schema` property containing a known meta-schema [#]_
+    then the proper validator will be used. The specification recommends
+    that all schemas contain :validator:`$schema` properties for this
+    reason. If no :validator:`$schema` property is found, the default
+    validator class is the latest released draft.
+
+    Any other provided positional and keyword arguments will be passed
+    on when instantiating the ``cls``.
+
+    Raises:
+
+        `jsonschema.exceptions.ValidationError` if the instance
+            is invalid
+
+        `jsonschema.exceptions.SchemaError` if the schema itself
+            is invalid
+
+    .. rubric:: Footnotes
+    .. [#] known by a validator registered with
+        `jsonschema.validators.validates`
+    """
+    if cls is None:
+        cls = validator_for(schema)
+
+    cls.check_schema(schema)
+    validator = cls(schema, *args, **kwargs)
+    error = exceptions.best_match(validator.iter_errors(instance))
+    if error is not None:
+        raise error
+
+
+def validator_for(schema, default=_LATEST_VERSION):
+    """
+    Retrieve the validator class appropriate for validating the given schema.
+
+    Uses the :validator:`$schema` property that should be present in the
+    given schema to look up the appropriate validator class.
+
+    Arguments:
+
+        schema (collections.Mapping or bool):
+
+            the schema to look at
+
+        default:
+
+            the default to return if the appropriate validator class
+            cannot be determined.
+
+            If unprovided, the default is to return the latest supported
+            draft.
+    """
+    if schema is True or schema is False or u"$schema" not in schema:
+        return default
+    if schema[u"$schema"] not in meta_schemas:
+        warn(
+            (
+                "The metaschema specified by $schema was not found. "
+                "Using the latest draft to validate, but this will raise "
+                "an error in the future."
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+    return meta_schemas.get(schema[u"$schema"], _LATEST_VERSION)
diff --git a/contrib/python/jsonschema/py3/tests/ya.make b/contrib/python/jsonschema/py3/tests/ya.make
new file mode 100644
index 00000000000..73af0ff813b
--- /dev/null
+++ b/contrib/python/jsonschema/py3/tests/ya.make
@@ -0,0 +1,32 @@
+PY3TEST()
+
+PEERDIR(
+    contrib/python/jsonschema
+    contrib/python/Twisted
+)
+
+IF (PYTHON2)
+    PEERDIR(
+        contrib/python/mock
+    )
+ENDIF()
+
+SRCDIR(contrib/python/jsonschema/py3/jsonschema/tests)
+
+PY_SRCS(
+    NAMESPACE jsonschema.tests
+    _helpers.py
+)
+
+TEST_SRCS(
+    __init__.py
+    test_cli.py
+    test_exceptions.py
+    test_format.py
+    test_types.py
+    test_validators.py
+)
+
+NO_LINT()
+
+END()
diff --git a/contrib/python/jsonschema/py3/ya.make b/contrib/python/jsonschema/py3/ya.make
new file mode 100644
index 00000000000..b40b94052b3
--- /dev/null
+++ b/contrib/python/jsonschema/py3/ya.make
@@ -0,0 +1,49 @@
+# Generated by devtools/yamaker (pypi).
+
+PY3_LIBRARY()
+
+VERSION(3.2.0)
+
+LICENSE(MIT)
+
+PEERDIR(
+    contrib/python/attrs
+    contrib/python/pyrsistent
+    contrib/python/setuptools
+    contrib/python/six
+)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    jsonschema/__init__.py
+    jsonschema/__main__.py
+    jsonschema/_format.py
+    jsonschema/_legacy_validators.py
+    jsonschema/_reflect.py
+    jsonschema/_types.py
+    jsonschema/_utils.py
+    jsonschema/_validators.py
+    jsonschema/cli.py
+    jsonschema/compat.py
+    jsonschema/exceptions.py
+    jsonschema/validators.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/jsonschema/py3/
+    .dist-info/METADATA
+    .dist-info/entry_points.txt
+    .dist-info/top_level.txt
+    jsonschema/schemas/draft3.json
+    jsonschema/schemas/draft4.json
+    jsonschema/schemas/draft6.json
+    jsonschema/schemas/draft7.json
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/jsonschema/ya.make b/contrib/python/jsonschema/ya.make
new file mode 100644
index 00000000000..7b62c0d5ca1
--- /dev/null
+++ b/contrib/python/jsonschema/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+    PEERDIR(contrib/python/jsonschema/py2)
+ELSE()
+    PEERDIR(contrib/python/jsonschema/py3)
+ENDIF()
+
+NO_LINT()
+
+END()
+
+RECURSE(
+    py2
+    py3
+)
diff --git a/contrib/python/pyOpenSSL/py2/.dist-info/METADATA b/contrib/python/pyOpenSSL/py2/.dist-info/METADATA
new file mode 100644
index 00000000000..43ea7f5813d
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/.dist-info/METADATA
@@ -0,0 +1,198 @@
+Metadata-Version: 2.1
+Name: pyOpenSSL
+Version: 21.0.0
+Summary: Python wrapper module around the OpenSSL library
+Home-page: https://pyopenssl.org/
+Author: The pyOpenSSL developers
+Author-email: cryptography-dev@python.org
+License: Apache License, Version 2.0
+Platform: UNKNOWN
+Classifier: Development Status :: 6 - Mature
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Security :: Cryptography
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Networking
+Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*
+Requires-Dist: cryptography (>=3.3)
+Requires-Dist: six (>=1.5.2)
+Provides-Extra: docs
+Requires-Dist: sphinx ; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
+Provides-Extra: test
+Requires-Dist: flaky ; extra == 'test'
+Requires-Dist: pretend ; extra == 'test'
+Requires-Dist: pytest (>=3.0.1) ; extra == 'test'
+
+========================================================
+pyOpenSSL -- A Python wrapper around the OpenSSL library
+========================================================
+
+.. image:: https://readthedocs.org/projects/pyopenssl/badge/?version=stable
+   :target: https://pyopenssl.org/en/stable/
+   :alt: Stable Docs
+
+.. image:: https://github.com/pyca/pyopenssl/workflows/CI/badge.svg?branch=main
+   :target: https://github.com/pyca/pyopenssl/actions?query=workflow%3ACI+branch%3Amain
+
+.. image:: https://codecov.io/github/pyca/pyopenssl/branch/main/graph/badge.svg
+   :target: https://codecov.io/github/pyca/pyopenssl
+   :alt: Test coverage
+
+**Note:** The Python Cryptographic Authority **strongly suggests** the use of `pyca/cryptography`_
+where possible. If you are using pyOpenSSL for anything other than making a TLS connection
+**you should move to cryptography and drop your pyOpenSSL dependency**.
+
+High-level wrapper around a subset of the OpenSSL library. Includes
+
+* ``SSL.Connection`` objects, wrapping the methods of Python's portable sockets
+* Callbacks written in Python
+* Extensive error-handling mechanism, mirroring OpenSSL's error codes
+
+... and much more.
+
+You can find more information in the documentation_.
+Development takes place on GitHub_.
+
+
+Discussion
+==========
+
+If you run into bugs, you can file them in our `issue tracker`_.
+
+We maintain a cryptography-dev_ mailing list for both user and development discussions.
+
+You can also join ``#cryptography-dev`` on Freenode to ask questions or get involved.
+
+
+.. _documentation: https://pyopenssl.org/
+.. _`issue tracker`: https://github.com/pyca/pyopenssl/issues
+.. _cryptography-dev: https://mail.python.org/mailman/listinfo/cryptography-dev
+.. _GitHub: https://github.com/pyca/pyopenssl
+.. _`pyca/cryptography`: https://github.com/pyca/cryptography
+
+
+Release Information
+===================
+
+21.0.0 (2020-09-28)
+-------------------
+
+Backward-incompatible changes:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+- The minimum ``cryptography`` version is now 3.3.
+- Drop support for Python 3.5
+
+Deprecations:
+^^^^^^^^^^^^^
+
+Changes:
+^^^^^^^^
+
+- Raise an error when an invalid ALPN value is set.
+  `#993 <https://github.com/pyca/pyopenssl/pull/993>`_
+- Added ``OpenSSL.SSL.Context.set_min_proto_version`` and ``OpenSSL.SSL.Context.set_max_proto_version``
+  to set the minimum and maximum supported TLS version `#985 <https://github.com/pyca/pyopenssl/pull/985>`_.
+- Updated ``to_cryptography`` and ``from_cryptography`` methods to support an upcoming release of ``cryptography`` without raising deprecation warnings.
+  `#1030 <https://github.com/pyca/pyopenssl/pull/1030>`_
+
+20.0.1 (2020-12-15)
+-------------------
+
+Backward-incompatible changes:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Deprecations:
+^^^^^^^^^^^^^
+
+Changes:
+^^^^^^^^
+
+- Fixed compatibility with OpenSSL 1.1.0.
+
+20.0.0 (2020-11-27)
+-------------------
+
+
+Backward-incompatible changes:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+- The minimum ``cryptography`` version is now 3.2.
+- Remove deprecated ``OpenSSL.tsafe`` module.
+- Removed deprecated ``OpenSSL.SSL.Context.set_npn_advertise_callback``, ``OpenSSL.SSL.Context.set_npn_select_callback``, and ``OpenSSL.SSL.Connection.get_next_proto_negotiated``.
+- Drop support for Python 3.4
+- Drop support for OpenSSL 1.0.1 and 1.0.2
+
+Deprecations:
+^^^^^^^^^^^^^
+
+- Deprecated ``OpenSSL.crypto.loads_pkcs7`` and ``OpenSSL.crypto.loads_pkcs12``.
+
+Changes:
+^^^^^^^^
+
+- Added a new optional ``chain`` parameter to ``OpenSSL.crypto.X509StoreContext()``
+  where additional untrusted certificates can be specified to help chain building.
+  `#948 <https://github.com/pyca/pyopenssl/pull/948>`_
+- Added ``OpenSSL.crypto.X509Store.load_locations`` to set trusted
+  certificate file bundles and/or directories for verification.
+  `#943 <https://github.com/pyca/pyopenssl/pull/943>`_
+- Added ``Context.set_keylog_callback`` to log key material.
+  `#910 <https://github.com/pyca/pyopenssl/pull/910>`_
+- Added ``OpenSSL.SSL.Connection.get_verified_chain`` to retrieve the
+  verified certificate chain of the peer.
+  `#894 <https://github.com/pyca/pyopenssl/pull/894>`_.
+- Make verification callback optional in ``Context.set_verify``.
+  If omitted, OpenSSL's default verification is used.
+  `#933 <https://github.com/pyca/pyopenssl/pull/933>`_
+- Fixed a bug that could truncate or cause a zero-length key error due to a
+  null byte in private key passphrase in ``OpenSSL.crypto.load_privatekey``
+  and ``OpenSSL.crypto.dump_privatekey``.
+  `#947 <https://github.com/pyca/pyopenssl/pull/947>`_
+
+19.1.0 (2019-11-18)
+-------------------
+
+
+Backward-incompatible changes:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+- Removed deprecated ``ContextType``, ``ConnectionType``, ``PKeyType``, ``X509NameType``, ``X509ReqType``, ``X509Type``, ``X509StoreType``, ``CRLType``, ``PKCS7Type``, ``PKCS12Type``, and ``NetscapeSPKIType`` aliases.
+  Use the classes without the ``Type`` suffix instead.
+  `#814 <https://github.com/pyca/pyopenssl/pull/814>`_
+- The minimum ``cryptography`` version is now 2.8 due to issues on macOS with a transitive dependency.
+  `#875 <https://github.com/pyca/pyopenssl/pull/875>`_
+
+Deprecations:
+^^^^^^^^^^^^^
+
+- Deprecated ``OpenSSL.SSL.Context.set_npn_advertise_callback``, ``OpenSSL.SSL.Context.set_npn_select_callback``, and ``OpenSSL.SSL.Connection.get_next_proto_negotiated``.
+  ALPN should be used instead.
+  `#820 <https://github.com/pyca/pyopenssl/pull/820>`_
+
+
+Changes:
+^^^^^^^^
+
+- Support ``bytearray`` in ``SSL.Connection.send()`` by using cffi's from_buffer.
+  `#852 <https://github.com/pyca/pyopenssl/pull/852>`_
+- The ``OpenSSL.SSL.Context.set_alpn_select_callback`` can return a new ``NO_OVERLAPPING_PROTOCOLS`` sentinel value
+  to allow a TLS handshake to complete without an application protocol.
+
+`Full changelog <https://pyopenssl.org/en/stable/changelog.html>`_.
+
+
+
diff --git a/contrib/python/pyOpenSSL/py2/.dist-info/top_level.txt b/contrib/python/pyOpenSSL/py2/.dist-info/top_level.txt
new file mode 100644
index 00000000000..effce34b618
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/.dist-info/top_level.txt
@@ -0,0 +1 @@
+OpenSSL
diff --git a/contrib/python/pyOpenSSL/py2/LICENSE b/contrib/python/pyOpenSSL/py2/LICENSE
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/contrib/python/pyOpenSSL/py2/OpenSSL/SSL.py b/contrib/python/pyOpenSSL/py2/OpenSSL/SSL.py
new file mode 100644
index 00000000000..e71b044cc02
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/OpenSSL/SSL.py
@@ -0,0 +1,2505 @@
+import os
+import socket
+from sys import platform
+from functools import wraps, partial
+from itertools import count, chain
+from weakref import WeakValueDictionary
+from errno import errorcode
+
+from six import integer_types, int2byte, indexbytes
+
+from OpenSSL._util import (
+    UNSPECIFIED as _UNSPECIFIED,
+    exception_from_error_queue as _exception_from_error_queue,
+    ffi as _ffi,
+    lib as _lib,
+    make_assert as _make_assert,
+    native as _native,
+    path_string as _path_string,
+    text_to_bytes_and_warn as _text_to_bytes_and_warn,
+    no_zero_allocator as _no_zero_allocator,
+)
+
+from OpenSSL.crypto import (
+    FILETYPE_PEM,
+    _PassphraseHelper,
+    PKey,
+    X509Name,
+    X509,
+    X509Store,
+)
+
+__all__ = [
+    "OPENSSL_VERSION_NUMBER",
+    "SSLEAY_VERSION",
+    "SSLEAY_CFLAGS",
+    "SSLEAY_PLATFORM",
+    "SSLEAY_DIR",
+    "SSLEAY_BUILT_ON",
+    "SENT_SHUTDOWN",
+    "RECEIVED_SHUTDOWN",
+    "SSLv2_METHOD",
+    "SSLv3_METHOD",
+    "SSLv23_METHOD",
+    "TLSv1_METHOD",
+    "TLSv1_1_METHOD",
+    "TLSv1_2_METHOD",
+    "TLS_METHOD",
+    "TLS_SERVER_METHOD",
+    "TLS_CLIENT_METHOD",
+    "SSL3_VERSION",
+    "TLS1_VERSION",
+    "TLS1_1_VERSION",
+    "TLS1_2_VERSION",
+    "TLS1_3_VERSION",
+    "OP_NO_SSLv2",
+    "OP_NO_SSLv3",
+    "OP_NO_TLSv1",
+    "OP_NO_TLSv1_1",
+    "OP_NO_TLSv1_2",
+    "OP_NO_TLSv1_3",
+    "MODE_RELEASE_BUFFERS",
+    "OP_SINGLE_DH_USE",
+    "OP_SINGLE_ECDH_USE",
+    "OP_EPHEMERAL_RSA",
+    "OP_MICROSOFT_SESS_ID_BUG",
+    "OP_NETSCAPE_CHALLENGE_BUG",
+    "OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG",
+    "OP_SSLREF2_REUSE_CERT_TYPE_BUG",
+    "OP_MICROSOFT_BIG_SSLV3_BUFFER",
+    "OP_MSIE_SSLV2_RSA_PADDING",
+    "OP_SSLEAY_080_CLIENT_DH_BUG",
+    "OP_TLS_D5_BUG",
+    "OP_TLS_BLOCK_PADDING_BUG",
+    "OP_DONT_INSERT_EMPTY_FRAGMENTS",
+    "OP_CIPHER_SERVER_PREFERENCE",
+    "OP_TLS_ROLLBACK_BUG",
+    "OP_PKCS1_CHECK_1",
+    "OP_PKCS1_CHECK_2",
+    "OP_NETSCAPE_CA_DN_BUG",
+    "OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG",
+    "OP_NO_COMPRESSION",
+    "OP_NO_QUERY_MTU",
+    "OP_COOKIE_EXCHANGE",
+    "OP_NO_TICKET",
+    "OP_ALL",
+    "VERIFY_PEER",
+    "VERIFY_FAIL_IF_NO_PEER_CERT",
+    "VERIFY_CLIENT_ONCE",
+    "VERIFY_NONE",
+    "SESS_CACHE_OFF",
+    "SESS_CACHE_CLIENT",
+    "SESS_CACHE_SERVER",
+    "SESS_CACHE_BOTH",
+    "SESS_CACHE_NO_AUTO_CLEAR",
+    "SESS_CACHE_NO_INTERNAL_LOOKUP",
+    "SESS_CACHE_NO_INTERNAL_STORE",
+    "SESS_CACHE_NO_INTERNAL",
+    "SSL_ST_CONNECT",
+    "SSL_ST_ACCEPT",
+    "SSL_ST_MASK",
+    "SSL_CB_LOOP",
+    "SSL_CB_EXIT",
+    "SSL_CB_READ",
+    "SSL_CB_WRITE",
+    "SSL_CB_ALERT",
+    "SSL_CB_READ_ALERT",
+    "SSL_CB_WRITE_ALERT",
+    "SSL_CB_ACCEPT_LOOP",
+    "SSL_CB_ACCEPT_EXIT",
+    "SSL_CB_CONNECT_LOOP",
+    "SSL_CB_CONNECT_EXIT",
+    "SSL_CB_HANDSHAKE_START",
+    "SSL_CB_HANDSHAKE_DONE",
+    "Error",
+    "WantReadError",
+    "WantWriteError",
+    "WantX509LookupError",
+    "ZeroReturnError",
+    "SysCallError",
+    "NO_OVERLAPPING_PROTOCOLS",
+    "SSLeay_version",
+    "Session",
+    "Context",
+    "Connection",
+]
+
+try:
+    _buffer = buffer
+except NameError:
+
+    class _buffer(object):
+        pass
+
+
+OPENSSL_VERSION_NUMBER = _lib.OPENSSL_VERSION_NUMBER
+SSLEAY_VERSION = _lib.SSLEAY_VERSION
+SSLEAY_CFLAGS = _lib.SSLEAY_CFLAGS
+SSLEAY_PLATFORM = _lib.SSLEAY_PLATFORM
+SSLEAY_DIR = _lib.SSLEAY_DIR
+SSLEAY_BUILT_ON = _lib.SSLEAY_BUILT_ON
+
+SENT_SHUTDOWN = _lib.SSL_SENT_SHUTDOWN
+RECEIVED_SHUTDOWN = _lib.SSL_RECEIVED_SHUTDOWN
+
+SSLv2_METHOD = 1
+SSLv3_METHOD = 2
+SSLv23_METHOD = 3
+TLSv1_METHOD = 4
+TLSv1_1_METHOD = 5
+TLSv1_2_METHOD = 6
+TLS_METHOD = 7
+TLS_SERVER_METHOD = 8
+TLS_CLIENT_METHOD = 9
+
+try:
+    SSL3_VERSION = _lib.SSL3_VERSION
+    TLS1_VERSION = _lib.TLS1_VERSION
+    TLS1_1_VERSION = _lib.TLS1_1_VERSION
+    TLS1_2_VERSION = _lib.TLS1_2_VERSION
+    TLS1_3_VERSION = _lib.TLS1_3_VERSION
+except AttributeError:
+    # Hardcode constants for cryptography < 3.4, see
+    # https://github.com/pyca/pyopenssl/pull/985#issuecomment-775186682
+    SSL3_VERSION = 768
+    TLS1_VERSION = 769
+    TLS1_1_VERSION = 770
+    TLS1_2_VERSION = 771
+    TLS1_3_VERSION = 772
+
+OP_NO_SSLv2 = _lib.SSL_OP_NO_SSLv2
+OP_NO_SSLv3 = _lib.SSL_OP_NO_SSLv3
+OP_NO_TLSv1 = _lib.SSL_OP_NO_TLSv1
+OP_NO_TLSv1_1 = _lib.SSL_OP_NO_TLSv1_1
+OP_NO_TLSv1_2 = _lib.SSL_OP_NO_TLSv1_2
+try:
+    OP_NO_TLSv1_3 = _lib.SSL_OP_NO_TLSv1_3
+except AttributeError:
+    pass
+
+MODE_RELEASE_BUFFERS = _lib.SSL_MODE_RELEASE_BUFFERS
+
+OP_SINGLE_DH_USE = _lib.SSL_OP_SINGLE_DH_USE
+OP_SINGLE_ECDH_USE = _lib.SSL_OP_SINGLE_ECDH_USE
+OP_EPHEMERAL_RSA = _lib.SSL_OP_EPHEMERAL_RSA
+OP_MICROSOFT_SESS_ID_BUG = _lib.SSL_OP_MICROSOFT_SESS_ID_BUG
+OP_NETSCAPE_CHALLENGE_BUG = _lib.SSL_OP_NETSCAPE_CHALLENGE_BUG
+OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG = (
+    _lib.SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG
+)
+OP_SSLREF2_REUSE_CERT_TYPE_BUG = _lib.SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG
+OP_MICROSOFT_BIG_SSLV3_BUFFER = _lib.SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER
+OP_MSIE_SSLV2_RSA_PADDING = _lib.SSL_OP_MSIE_SSLV2_RSA_PADDING
+OP_SSLEAY_080_CLIENT_DH_BUG = _lib.SSL_OP_SSLEAY_080_CLIENT_DH_BUG
+OP_TLS_D5_BUG = _lib.SSL_OP_TLS_D5_BUG
+OP_TLS_BLOCK_PADDING_BUG = _lib.SSL_OP_TLS_BLOCK_PADDING_BUG
+OP_DONT_INSERT_EMPTY_FRAGMENTS = _lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+OP_CIPHER_SERVER_PREFERENCE = _lib.SSL_OP_CIPHER_SERVER_PREFERENCE
+OP_TLS_ROLLBACK_BUG = _lib.SSL_OP_TLS_ROLLBACK_BUG
+OP_PKCS1_CHECK_1 = _lib.SSL_OP_PKCS1_CHECK_1
+OP_PKCS1_CHECK_2 = _lib.SSL_OP_PKCS1_CHECK_2
+OP_NETSCAPE_CA_DN_BUG = _lib.SSL_OP_NETSCAPE_CA_DN_BUG
+OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG = (
+    _lib.SSL_OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG
+)
+OP_NO_COMPRESSION = _lib.SSL_OP_NO_COMPRESSION
+
+OP_NO_QUERY_MTU = _lib.SSL_OP_NO_QUERY_MTU
+OP_COOKIE_EXCHANGE = _lib.SSL_OP_COOKIE_EXCHANGE
+OP_NO_TICKET = _lib.SSL_OP_NO_TICKET
+
+OP_ALL = _lib.SSL_OP_ALL
+
+VERIFY_PEER = _lib.SSL_VERIFY_PEER
+VERIFY_FAIL_IF_NO_PEER_CERT = _lib.SSL_VERIFY_FAIL_IF_NO_PEER_CERT
+VERIFY_CLIENT_ONCE = _lib.SSL_VERIFY_CLIENT_ONCE
+VERIFY_NONE = _lib.SSL_VERIFY_NONE
+
+SESS_CACHE_OFF = _lib.SSL_SESS_CACHE_OFF
+SESS_CACHE_CLIENT = _lib.SSL_SESS_CACHE_CLIENT
+SESS_CACHE_SERVER = _lib.SSL_SESS_CACHE_SERVER
+SESS_CACHE_BOTH = _lib.SSL_SESS_CACHE_BOTH
+SESS_CACHE_NO_AUTO_CLEAR = _lib.SSL_SESS_CACHE_NO_AUTO_CLEAR
+SESS_CACHE_NO_INTERNAL_LOOKUP = _lib.SSL_SESS_CACHE_NO_INTERNAL_LOOKUP
+SESS_CACHE_NO_INTERNAL_STORE = _lib.SSL_SESS_CACHE_NO_INTERNAL_STORE
+SESS_CACHE_NO_INTERNAL = _lib.SSL_SESS_CACHE_NO_INTERNAL
+
+SSL_ST_CONNECT = _lib.SSL_ST_CONNECT
+SSL_ST_ACCEPT = _lib.SSL_ST_ACCEPT
+SSL_ST_MASK = _lib.SSL_ST_MASK
+
+SSL_CB_LOOP = _lib.SSL_CB_LOOP
+SSL_CB_EXIT = _lib.SSL_CB_EXIT
+SSL_CB_READ = _lib.SSL_CB_READ
+SSL_CB_WRITE = _lib.SSL_CB_WRITE
+SSL_CB_ALERT = _lib.SSL_CB_ALERT
+SSL_CB_READ_ALERT = _lib.SSL_CB_READ_ALERT
+SSL_CB_WRITE_ALERT = _lib.SSL_CB_WRITE_ALERT
+SSL_CB_ACCEPT_LOOP = _lib.SSL_CB_ACCEPT_LOOP
+SSL_CB_ACCEPT_EXIT = _lib.SSL_CB_ACCEPT_EXIT
+SSL_CB_CONNECT_LOOP = _lib.SSL_CB_CONNECT_LOOP
+SSL_CB_CONNECT_EXIT = _lib.SSL_CB_CONNECT_EXIT
+SSL_CB_HANDSHAKE_START = _lib.SSL_CB_HANDSHAKE_START
+SSL_CB_HANDSHAKE_DONE = _lib.SSL_CB_HANDSHAKE_DONE
+
+# Taken from https://golang.org/src/crypto/x509/root_linux.go
+_CERTIFICATE_FILE_LOCATIONS = [
+    "/etc/ssl/certs/ca-certificates.crt",  # Debian/Ubuntu/Gentoo etc.
+    "/etc/pki/tls/certs/ca-bundle.crt",  # Fedora/RHEL 6
+    "/etc/ssl/ca-bundle.pem",  # OpenSUSE
+    "/etc/pki/tls/cacert.pem",  # OpenELEC
+    "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",  # CentOS/RHEL 7
+]
+
+_CERTIFICATE_PATH_LOCATIONS = [
+    "/etc/ssl/certs",  # SLES10/SLES11
+]
+
+# These values are compared to output from cffi's ffi.string so they must be
+# byte strings.
+_CRYPTOGRAPHY_MANYLINUX1_CA_DIR = b"/opt/pyca/cryptography/openssl/certs"
+_CRYPTOGRAPHY_MANYLINUX1_CA_FILE = b"/opt/pyca/cryptography/openssl/cert.pem"
+
+
+class Error(Exception):
+    """
+    An error occurred in an `OpenSSL.SSL` API.
+    """
+
+
+_raise_current_error = partial(_exception_from_error_queue, Error)
+_openssl_assert = _make_assert(Error)
+
+
+class WantReadError(Error):
+    pass
+
+
+class WantWriteError(Error):
+    pass
+
+
+class WantX509LookupError(Error):
+    pass
+
+
+class ZeroReturnError(Error):
+    pass
+
+
+class SysCallError(Error):
+    pass
+
+
+class _CallbackExceptionHelper(object):
+    """
+    A base class for wrapper classes that allow for intelligent exception
+    handling in OpenSSL callbacks.
+
+    :ivar list _problems: Any exceptions that occurred while executing in a
+        context where they could not be raised in the normal way.  Typically
+        this is because OpenSSL has called into some Python code and requires a
+        return value.  The exceptions are saved to be raised later when it is
+        possible to do so.
+    """
+
+    def __init__(self):
+        self._problems = []
+
+    def raise_if_problem(self):
+        """
+        Raise an exception from the OpenSSL error queue or that was previously
+        captured whe running a callback.
+        """
+        if self._problems:
+            try:
+                _raise_current_error()
+            except Error:
+                pass
+            raise self._problems.pop(0)
+
+
+class _VerifyHelper(_CallbackExceptionHelper):
+    """
+    Wrap a callback such that it can be used as a certificate verification
+    callback.
+    """
+
+    def __init__(self, callback):
+        _CallbackExceptionHelper.__init__(self)
+
+        @wraps(callback)
+        def wrapper(ok, store_ctx):
+            x509 = _lib.X509_STORE_CTX_get_current_cert(store_ctx)
+            _lib.X509_up_ref(x509)
+            cert = X509._from_raw_x509_ptr(x509)
+            error_number = _lib.X509_STORE_CTX_get_error(store_ctx)
+            error_depth = _lib.X509_STORE_CTX_get_error_depth(store_ctx)
+
+            index = _lib.SSL_get_ex_data_X509_STORE_CTX_idx()
+            ssl = _lib.X509_STORE_CTX_get_ex_data(store_ctx, index)
+            connection = Connection._reverse_mapping[ssl]
+
+            try:
+                result = callback(
+                    connection, cert, error_number, error_depth, ok
+                )
+            except Exception as e:
+                self._problems.append(e)
+                return 0
+            else:
+                if result:
+                    _lib.X509_STORE_CTX_set_error(store_ctx, _lib.X509_V_OK)
+                    return 1
+                else:
+                    return 0
+
+        self.callback = _ffi.callback(
+            "int (*)(int, X509_STORE_CTX *)", wrapper
+        )
+
+
+NO_OVERLAPPING_PROTOCOLS = object()
+
+
+class _ALPNSelectHelper(_CallbackExceptionHelper):
+    """
+    Wrap a callback such that it can be used as an ALPN selection callback.
+    """
+
+    def __init__(self, callback):
+        _CallbackExceptionHelper.__init__(self)
+
+        @wraps(callback)
+        def wrapper(ssl, out, outlen, in_, inlen, arg):
+            try:
+                conn = Connection._reverse_mapping[ssl]
+
+                # The string passed to us is made up of multiple
+                # length-prefixed bytestrings. We need to split that into a
+                # list.
+                instr = _ffi.buffer(in_, inlen)[:]
+                protolist = []
+                while instr:
+                    encoded_len = indexbytes(instr, 0)
+                    proto = instr[1 : encoded_len + 1]
+                    protolist.append(proto)
+                    instr = instr[encoded_len + 1 :]
+
+                # Call the callback
+                outbytes = callback(conn, protolist)
+                any_accepted = True
+                if outbytes is NO_OVERLAPPING_PROTOCOLS:
+                    outbytes = b""
+                    any_accepted = False
+                elif not isinstance(outbytes, bytes):
+                    raise TypeError(
+                        "ALPN callback must return a bytestring or the "
+                        "special NO_OVERLAPPING_PROTOCOLS sentinel value."
+                    )
+
+                # Save our callback arguments on the connection object to make
+                # sure that they don't get freed before OpenSSL can use them.
+                # Then, return them in the appropriate output parameters.
+                conn._alpn_select_callback_args = [
+                    _ffi.new("unsigned char *", len(outbytes)),
+                    _ffi.new("unsigned char[]", outbytes),
+                ]
+                outlen[0] = conn._alpn_select_callback_args[0][0]
+                out[0] = conn._alpn_select_callback_args[1]
+                if not any_accepted:
+                    return _lib.SSL_TLSEXT_ERR_NOACK
+                return _lib.SSL_TLSEXT_ERR_OK
+            except Exception as e:
+                self._problems.append(e)
+                return _lib.SSL_TLSEXT_ERR_ALERT_FATAL
+
+        self.callback = _ffi.callback(
+            (
+                "int (*)(SSL *, unsigned char **, unsigned char *, "
+                "const unsigned char *, unsigned int, void *)"
+            ),
+            wrapper,
+        )
+
+
+class _OCSPServerCallbackHelper(_CallbackExceptionHelper):
+    """
+    Wrap a callback such that it can be used as an OCSP callback for the server
+    side.
+
+    Annoyingly, OpenSSL defines one OCSP callback but uses it in two different
+    ways. For servers, that callback is expected to retrieve some OCSP data and
+    hand it to OpenSSL, and may return only SSL_TLSEXT_ERR_OK,
+    SSL_TLSEXT_ERR_FATAL, and SSL_TLSEXT_ERR_NOACK. For clients, that callback
+    is expected to check the OCSP data, and returns a negative value on error,
+    0 if the response is not acceptable, or positive if it is. These are
+    mutually exclusive return code behaviours, and they mean that we need two
+    helpers so that we always return an appropriate error code if the user's
+    code throws an exception.
+
+    Given that we have to have two helpers anyway, these helpers are a bit more
+    helpery than most: specifically, they hide a few more of the OpenSSL
+    functions so that the user has an easier time writing these callbacks.
+
+    This helper implements the server side.
+    """
+
+    def __init__(self, callback):
+        _CallbackExceptionHelper.__init__(self)
+
+        @wraps(callback)
+        def wrapper(ssl, cdata):
+            try:
+                conn = Connection._reverse_mapping[ssl]
+
+                # Extract the data if any was provided.
+                if cdata != _ffi.NULL:
+                    data = _ffi.from_handle(cdata)
+                else:
+                    data = None
+
+                # Call the callback.
+                ocsp_data = callback(conn, data)
+
+                if not isinstance(ocsp_data, bytes):
+                    raise TypeError("OCSP callback must return a bytestring.")
+
+                # If the OCSP data was provided, we will pass it to OpenSSL.
+                # However, we have an early exit here: if no OCSP data was
+                # provided we will just exit out and tell OpenSSL that there
+                # is nothing to do.
+                if not ocsp_data:
+                    return 3  # SSL_TLSEXT_ERR_NOACK
+
+                # OpenSSL takes ownership of this data and expects it to have
+                # been allocated by OPENSSL_malloc.
+                ocsp_data_length = len(ocsp_data)
+                data_ptr = _lib.OPENSSL_malloc(ocsp_data_length)
+                _ffi.buffer(data_ptr, ocsp_data_length)[:] = ocsp_data
+
+                _lib.SSL_set_tlsext_status_ocsp_resp(
+                    ssl, data_ptr, ocsp_data_length
+                )
+
+                return 0
+            except Exception as e:
+                self._problems.append(e)
+                return 2  # SSL_TLSEXT_ERR_ALERT_FATAL
+
+        self.callback = _ffi.callback("int (*)(SSL *, void *)", wrapper)
+
+
+class _OCSPClientCallbackHelper(_CallbackExceptionHelper):
+    """
+    Wrap a callback such that it can be used as an OCSP callback for the client
+    side.
+
+    Annoyingly, OpenSSL defines one OCSP callback but uses it in two different
+    ways. For servers, that callback is expected to retrieve some OCSP data and
+    hand it to OpenSSL, and may return only SSL_TLSEXT_ERR_OK,
+    SSL_TLSEXT_ERR_FATAL, and SSL_TLSEXT_ERR_NOACK. For clients, that callback
+    is expected to check the OCSP data, and returns a negative value on error,
+    0 if the response is not acceptable, or positive if it is. These are
+    mutually exclusive return code behaviours, and they mean that we need two
+    helpers so that we always return an appropriate error code if the user's
+    code throws an exception.
+
+    Given that we have to have two helpers anyway, these helpers are a bit more
+    helpery than most: specifically, they hide a few more of the OpenSSL
+    functions so that the user has an easier time writing these callbacks.
+
+    This helper implements the client side.
+    """
+
+    def __init__(self, callback):
+        _CallbackExceptionHelper.__init__(self)
+
+        @wraps(callback)
+        def wrapper(ssl, cdata):
+            try:
+                conn = Connection._reverse_mapping[ssl]
+
+                # Extract the data if any was provided.
+                if cdata != _ffi.NULL:
+                    data = _ffi.from_handle(cdata)
+                else:
+                    data = None
+
+                # Get the OCSP data.
+                ocsp_ptr = _ffi.new("unsigned char **")
+                ocsp_len = _lib.SSL_get_tlsext_status_ocsp_resp(ssl, ocsp_ptr)
+                if ocsp_len < 0:
+                    # No OCSP data.
+                    ocsp_data = b""
+                else:
+                    # Copy the OCSP data, then pass it to the callback.
+                    ocsp_data = _ffi.buffer(ocsp_ptr[0], ocsp_len)[:]
+
+                valid = callback(conn, ocsp_data, data)
+
+                # Return 1 on success or 0 on error.
+                return int(bool(valid))
+
+            except Exception as e:
+                self._problems.append(e)
+                # Return negative value if an exception is hit.
+                return -1
+
+        self.callback = _ffi.callback("int (*)(SSL *, void *)", wrapper)
+
+
+def _asFileDescriptor(obj):
+    fd = None
+    if not isinstance(obj, integer_types):
+        meth = getattr(obj, "fileno", None)
+        if meth is not None:
+            obj = meth()
+
+    if isinstance(obj, integer_types):
+        fd = obj
+
+    if not isinstance(fd, integer_types):
+        raise TypeError("argument must be an int, or have a fileno() method.")
+    elif fd < 0:
+        raise ValueError(
+            "file descriptor cannot be a negative integer (%i)" % (fd,)
+        )
+
+    return fd
+
+
+def SSLeay_version(type):
+    """
+    Return a string describing the version of OpenSSL in use.
+
+    :param type: One of the :const:`SSLEAY_` constants defined in this module.
+    """
+    return _ffi.string(_lib.SSLeay_version(type))
+
+
+def _make_requires(flag, error):
+    """
+    Builds a decorator that ensures that functions that rely on OpenSSL
+    functions that are not present in this build raise NotImplementedError,
+    rather than AttributeError coming out of cryptography.
+
+    :param flag: A cryptography flag that guards the functions, e.g.
+        ``Cryptography_HAS_NEXTPROTONEG``.
+    :param error: The string to be used in the exception if the flag is false.
+    """
+
+    def _requires_decorator(func):
+        if not flag:
+
+            @wraps(func)
+            def explode(*args, **kwargs):
+                raise NotImplementedError(error)
+
+            return explode
+        else:
+            return func
+
+    return _requires_decorator
+
+
+_requires_alpn = _make_requires(
+    _lib.Cryptography_HAS_ALPN, "ALPN not available"
+)
+
+
+_requires_keylog = _make_requires(
+    getattr(_lib, "Cryptography_HAS_KEYLOG", None), "Key logging not available"
+)
+
+
+class Session(object):
+    """
+    A class representing an SSL session.  A session defines certain connection
+    parameters which may be re-used to speed up the setup of subsequent
+    connections.
+
+    .. versionadded:: 0.14
+    """
+
+    pass
+
+
+class Context(object):
+    """
+    :class:`OpenSSL.SSL.Context` instances define the parameters for setting
+    up new SSL connections.
+
+    :param method: One of TLS_METHOD, TLS_CLIENT_METHOD, or TLS_SERVER_METHOD.
+                   SSLv23_METHOD, TLSv1_METHOD, etc. are deprecated and should
+                   not be used.
+    """
+
+    _methods = {
+        SSLv2_METHOD: "SSLv2_method",
+        SSLv3_METHOD: "SSLv3_method",
+        SSLv23_METHOD: "SSLv23_method",
+        TLSv1_METHOD: "TLSv1_method",
+        TLSv1_1_METHOD: "TLSv1_1_method",
+        TLSv1_2_METHOD: "TLSv1_2_method",
+        TLS_METHOD: "TLS_method",
+        TLS_SERVER_METHOD: "TLS_server_method",
+        TLS_CLIENT_METHOD: "TLS_client_method",
+    }
+    _methods = dict(
+        (identifier, getattr(_lib, name))
+        for (identifier, name) in _methods.items()
+        if getattr(_lib, name, None) is not None
+    )
+
+    def __init__(self, method):
+        if not isinstance(method, integer_types):
+            raise TypeError("method must be an integer")
+
+        try:
+            method_func = self._methods[method]
+        except KeyError:
+            raise ValueError("No such protocol")
+
+        method_obj = method_func()
+        _openssl_assert(method_obj != _ffi.NULL)
+
+        context = _lib.SSL_CTX_new(method_obj)
+        _openssl_assert(context != _ffi.NULL)
+        context = _ffi.gc(context, _lib.SSL_CTX_free)
+
+        # Set SSL_CTX_set_ecdh_auto so that the ECDH curve will be
+        # auto-selected. This function was added in 1.0.2 and made a noop in
+        # 1.1.0+ (where it is set automatically).
+        res = _lib.SSL_CTX_set_ecdh_auto(context, 1)
+        _openssl_assert(res == 1)
+
+        self._context = context
+        self._passphrase_helper = None
+        self._passphrase_callback = None
+        self._passphrase_userdata = None
+        self._verify_helper = None
+        self._verify_callback = None
+        self._info_callback = None
+        self._keylog_callback = None
+        self._tlsext_servername_callback = None
+        self._app_data = None
+        self._alpn_select_helper = None
+        self._alpn_select_callback = None
+        self._ocsp_helper = None
+        self._ocsp_callback = None
+        self._ocsp_data = None
+
+        self.set_mode(_lib.SSL_MODE_ENABLE_PARTIAL_WRITE)
+
+    def set_min_proto_version(self, version):
+        """
+        Set the minimum supported protocol version. Setting the minimum
+        version to 0 will enable protocol versions down to the lowest version
+        supported by the library.
+
+        If the underlying OpenSSL build is missing support for the selected
+        version, this method will raise an exception.
+        """
+        _openssl_assert(
+            _lib.SSL_CTX_set_min_proto_version(self._context, version) == 1
+        )
+
+    def set_max_proto_version(self, version):
+        """
+        Set the maximum supported protocol version. Setting the maximum
+        version to 0 will enable protocol versions up to the highest version
+        supported by the library.
+
+        If the underlying OpenSSL build is missing support for the selected
+        version, this method will raise an exception.
+        """
+        _openssl_assert(
+            _lib.SSL_CTX_set_max_proto_version(self._context, version) == 1
+        )
+
+    def load_verify_locations(self, cafile, capath=None):
+        """
+        Let SSL know where we can find trusted certificates for the certificate
+        chain.  Note that the certificates have to be in PEM format.
+
+        If capath is passed, it must be a directory prepared using the
+        ``c_rehash`` tool included with OpenSSL.  Either, but not both, of
+        *pemfile* or *capath* may be :data:`None`.
+
+        :param cafile: In which file we can find the certificates (``bytes`` or
+            ``unicode``).
+        :param capath: In which directory we can find the certificates
+            (``bytes`` or ``unicode``).
+
+        :return: None
+        """
+        if cafile is None:
+            cafile = _ffi.NULL
+        else:
+            cafile = _path_string(cafile)
+
+        if capath is None:
+            capath = _ffi.NULL
+        else:
+            capath = _path_string(capath)
+
+        load_result = _lib.SSL_CTX_load_verify_locations(
+            self._context, cafile, capath
+        )
+        if not load_result:
+            _raise_current_error()
+
+    def _wrap_callback(self, callback):
+        @wraps(callback)
+        def wrapper(size, verify, userdata):
+            return callback(size, verify, self._passphrase_userdata)
+
+        return _PassphraseHelper(
+            FILETYPE_PEM, wrapper, more_args=True, truncate=True
+        )
+
+    def set_passwd_cb(self, callback, userdata=None):
+        """
+        Set the passphrase callback.  This function will be called
+        when a private key with a passphrase is loaded.
+
+        :param callback: The Python callback to use.  This must accept three
+            positional arguments.  First, an integer giving the maximum length
+            of the passphrase it may return.  If the returned passphrase is
+            longer than this, it will be truncated.  Second, a boolean value
+            which will be true if the user should be prompted for the
+            passphrase twice and the callback should verify that the two values
+            supplied are equal. Third, the value given as the *userdata*
+            parameter to :meth:`set_passwd_cb`.  The *callback* must return
+            a byte string. If an error occurs, *callback* should return a false
+            value (e.g. an empty string).
+        :param userdata: (optional) A Python object which will be given as
+                         argument to the callback
+        :return: None
+        """
+        if not callable(callback):
+            raise TypeError("callback must be callable")
+
+        self._passphrase_helper = self._wrap_callback(callback)
+        self._passphrase_callback = self._passphrase_helper.callback
+        _lib.SSL_CTX_set_default_passwd_cb(
+            self._context, self._passphrase_callback
+        )
+        self._passphrase_userdata = userdata
+
+    def set_default_verify_paths(self):
+        """
+        Specify that the platform provided CA certificates are to be used for
+        verification purposes. This method has some caveats related to the
+        binary wheels that cryptography (pyOpenSSL's primary dependency) ships:
+
+        *   macOS will only load certificates using this method if the user has
+            the ``openssl@1.1`` `Homebrew <https://brew.sh>`_ formula installed
+            in the default location.
+        *   Windows will not work.
+        *   manylinux1 cryptography wheels will work on most common Linux
+            distributions in pyOpenSSL 17.1.0 and above.  pyOpenSSL detects the
+            manylinux1 wheel and attempts to load roots via a fallback path.
+
+        :return: None
+        """
+        # SSL_CTX_set_default_verify_paths will attempt to load certs from
+        # both a cafile and capath that are set at compile time. However,
+        # it will first check environment variables and, if present, load
+        # those paths instead
+        set_result = _lib.SSL_CTX_set_default_verify_paths(self._context)
+        _openssl_assert(set_result == 1)
+        # After attempting to set default_verify_paths we need to know whether
+        # to go down the fallback path.
+        # First we'll check to see if any env vars have been set. If so,
+        # we won't try to do anything else because the user has set the path
+        # themselves.
+        dir_env_var = _ffi.string(_lib.X509_get_default_cert_dir_env()).decode(
+            "ascii"
+        )
+        file_env_var = _ffi.string(
+            _lib.X509_get_default_cert_file_env()
+        ).decode("ascii")
+        if not self._check_env_vars_set(dir_env_var, file_env_var):
+            default_dir = _ffi.string(_lib.X509_get_default_cert_dir())
+            default_file = _ffi.string(_lib.X509_get_default_cert_file())
+            # Now we check to see if the default_dir and default_file are set
+            # to the exact values we use in our manylinux1 builds. If they are
+            # then we know to load the fallbacks
+            if (
+                default_dir == _CRYPTOGRAPHY_MANYLINUX1_CA_DIR
+                and default_file == _CRYPTOGRAPHY_MANYLINUX1_CA_FILE
+            ):
+                # This is manylinux1, let's load our fallback paths
+                self._fallback_default_verify_paths(
+                    _CERTIFICATE_FILE_LOCATIONS, _CERTIFICATE_PATH_LOCATIONS
+                )
+
+    def _check_env_vars_set(self, dir_env_var, file_env_var):
+        """
+        Check to see if the default cert dir/file environment vars are present.
+
+        :return: bool
+        """
+        return (
+            os.environ.get(file_env_var) is not None
+            or os.environ.get(dir_env_var) is not None
+        )
+
+    def _fallback_default_verify_paths(self, file_path, dir_path):
+        """
+        Default verify paths are based on the compiled version of OpenSSL.
+        However, when pyca/cryptography is compiled as a manylinux1 wheel
+        that compiled location can potentially be wrong. So, like Go, we
+        will try a predefined set of paths and attempt to load roots
+        from there.
+
+        :return: None
+        """
+        for cafile in file_path:
+            if os.path.isfile(cafile):
+                self.load_verify_locations(cafile)
+                break
+
+        for capath in dir_path:
+            if os.path.isdir(capath):
+                self.load_verify_locations(None, capath)
+                break
+
+    def use_certificate_chain_file(self, certfile):
+        """
+        Load a certificate chain from a file.
+
+        :param certfile: The name of the certificate chain file (``bytes`` or
+            ``unicode``).  Must be PEM encoded.
+
+        :return: None
+        """
+        certfile = _path_string(certfile)
+
+        result = _lib.SSL_CTX_use_certificate_chain_file(
+            self._context, certfile
+        )
+        if not result:
+            _raise_current_error()
+
+    def use_certificate_file(self, certfile, filetype=FILETYPE_PEM):
+        """
+        Load a certificate from a file
+
+        :param certfile: The name of the certificate file (``bytes`` or
+            ``unicode``).
+        :param filetype: (optional) The encoding of the file, which is either
+            :const:`FILETYPE_PEM` or :const:`FILETYPE_ASN1`.  The default is
+            :const:`FILETYPE_PEM`.
+
+        :return: None
+        """
+        certfile = _path_string(certfile)
+        if not isinstance(filetype, integer_types):
+            raise TypeError("filetype must be an integer")
+
+        use_result = _lib.SSL_CTX_use_certificate_file(
+            self._context, certfile, filetype
+        )
+        if not use_result:
+            _raise_current_error()
+
+    def use_certificate(self, cert):
+        """
+        Load a certificate from a X509 object
+
+        :param cert: The X509 object
+        :return: None
+        """
+        if not isinstance(cert, X509):
+            raise TypeError("cert must be an X509 instance")
+
+        use_result = _lib.SSL_CTX_use_certificate(self._context, cert._x509)
+        if not use_result:
+            _raise_current_error()
+
+    def add_extra_chain_cert(self, certobj):
+        """
+        Add certificate to chain
+
+        :param certobj: The X509 certificate object to add to the chain
+        :return: None
+        """
+        if not isinstance(certobj, X509):
+            raise TypeError("certobj must be an X509 instance")
+
+        copy = _lib.X509_dup(certobj._x509)
+        add_result = _lib.SSL_CTX_add_extra_chain_cert(self._context, copy)
+        if not add_result:
+            # TODO: This is untested.
+            _lib.X509_free(copy)
+            _raise_current_error()
+
+    def _raise_passphrase_exception(self):
+        if self._passphrase_helper is not None:
+            self._passphrase_helper.raise_if_problem(Error)
+
+        _raise_current_error()
+
+    def use_privatekey_file(self, keyfile, filetype=_UNSPECIFIED):
+        """
+        Load a private key from a file
+
+        :param keyfile: The name of the key file (``bytes`` or ``unicode``)
+        :param filetype: (optional) The encoding of the file, which is either
+            :const:`FILETYPE_PEM` or :const:`FILETYPE_ASN1`.  The default is
+            :const:`FILETYPE_PEM`.
+
+        :return: None
+        """
+        keyfile = _path_string(keyfile)
+
+        if filetype is _UNSPECIFIED:
+            filetype = FILETYPE_PEM
+        elif not isinstance(filetype, integer_types):
+            raise TypeError("filetype must be an integer")
+
+        use_result = _lib.SSL_CTX_use_PrivateKey_file(
+            self._context, keyfile, filetype
+        )
+        if not use_result:
+            self._raise_passphrase_exception()
+
+    def use_privatekey(self, pkey):
+        """
+        Load a private key from a PKey object
+
+        :param pkey: The PKey object
+        :return: None
+        """
+        if not isinstance(pkey, PKey):
+            raise TypeError("pkey must be a PKey instance")
+
+        use_result = _lib.SSL_CTX_use_PrivateKey(self._context, pkey._pkey)
+        if not use_result:
+            self._raise_passphrase_exception()
+
+    def check_privatekey(self):
+        """
+        Check if the private key (loaded with :meth:`use_privatekey`) matches
+        the certificate (loaded with :meth:`use_certificate`)
+
+        :return: :data:`None` (raises :exc:`Error` if something's wrong)
+        """
+        if not _lib.SSL_CTX_check_private_key(self._context):
+            _raise_current_error()
+
+    def load_client_ca(self, cafile):
+        """
+        Load the trusted certificates that will be sent to the client.  Does
+        not actually imply any of the certificates are trusted; that must be
+        configured separately.
+
+        :param bytes cafile: The path to a certificates file in PEM format.
+        :return: None
+        """
+        ca_list = _lib.SSL_load_client_CA_file(
+            _text_to_bytes_and_warn("cafile", cafile)
+        )
+        _openssl_assert(ca_list != _ffi.NULL)
+        _lib.SSL_CTX_set_client_CA_list(self._context, ca_list)
+
+    def set_session_id(self, buf):
+        """
+        Set the session id to *buf* within which a session can be reused for
+        this Context object.  This is needed when doing session resumption,
+        because there is no way for a stored session to know which Context
+        object it is associated with.
+
+        :param bytes buf: The session id.
+
+        :returns: None
+        """
+        buf = _text_to_bytes_and_warn("buf", buf)
+        _openssl_assert(
+            _lib.SSL_CTX_set_session_id_context(self._context, buf, len(buf))
+            == 1
+        )
+
+    def set_session_cache_mode(self, mode):
+        """
+        Set the behavior of the session cache used by all connections using
+        this Context.  The previously set mode is returned.  See
+        :const:`SESS_CACHE_*` for details about particular modes.
+
+        :param mode: One or more of the SESS_CACHE_* flags (combine using
+            bitwise or)
+        :returns: The previously set caching mode.
+
+        .. versionadded:: 0.14
+        """
+        if not isinstance(mode, integer_types):
+            raise TypeError("mode must be an integer")
+
+        return _lib.SSL_CTX_set_session_cache_mode(self._context, mode)
+
+    def get_session_cache_mode(self):
+        """
+        Get the current session cache mode.
+
+        :returns: The currently used cache mode.
+
+        .. versionadded:: 0.14
+        """
+        return _lib.SSL_CTX_get_session_cache_mode(self._context)
+
+    def set_verify(self, mode, callback=None):
+        """
+        Set the verification flags for this Context object to *mode* and
+        specify that *callback* should be used for verification callbacks.
+
+        :param mode: The verify mode, this should be one of
+            :const:`VERIFY_NONE` and :const:`VERIFY_PEER`. If
+            :const:`VERIFY_PEER` is used, *mode* can be OR:ed with
+            :const:`VERIFY_FAIL_IF_NO_PEER_CERT` and
+            :const:`VERIFY_CLIENT_ONCE` to further control the behaviour.
+        :param callback: The optional Python verification callback to use.
+            This should take five arguments: A Connection object, an X509
+            object, and three integer variables, which are in turn potential
+            error number, error depth and return code. *callback* should
+            return True if verification passes and False otherwise.
+            If omitted, OpenSSL's default verification is used.
+        :return: None
+
+        See SSL_CTX_set_verify(3SSL) for further details.
+        """
+        if not isinstance(mode, integer_types):
+            raise TypeError("mode must be an integer")
+
+        if callback is None:
+            self._verify_helper = None
+            self._verify_callback = None
+            _lib.SSL_CTX_set_verify(self._context, mode, _ffi.NULL)
+        else:
+            if not callable(callback):
+                raise TypeError("callback must be callable")
+
+            self._verify_helper = _VerifyHelper(callback)
+            self._verify_callback = self._verify_helper.callback
+            _lib.SSL_CTX_set_verify(self._context, mode, self._verify_callback)
+
+    def set_verify_depth(self, depth):
+        """
+        Set the maximum depth for the certificate chain verification that shall
+        be allowed for this Context object.
+
+        :param depth: An integer specifying the verify depth
+        :return: None
+        """
+        if not isinstance(depth, integer_types):
+            raise TypeError("depth must be an integer")
+
+        _lib.SSL_CTX_set_verify_depth(self._context, depth)
+
+    def get_verify_mode(self):
+        """
+        Retrieve the Context object's verify mode, as set by
+        :meth:`set_verify`.
+
+        :return: The verify mode
+        """
+        return _lib.SSL_CTX_get_verify_mode(self._context)
+
+    def get_verify_depth(self):
+        """
+        Retrieve the Context object's verify depth, as set by
+        :meth:`set_verify_depth`.
+
+        :return: The verify depth
+        """
+        return _lib.SSL_CTX_get_verify_depth(self._context)
+
+    def load_tmp_dh(self, dhfile):
+        """
+        Load parameters for Ephemeral Diffie-Hellman
+
+        :param dhfile: The file to load EDH parameters from (``bytes`` or
+            ``unicode``).
+
+        :return: None
+        """
+        dhfile = _path_string(dhfile)
+
+        bio = _lib.BIO_new_file(dhfile, b"r")
+        if bio == _ffi.NULL:
+            _raise_current_error()
+        bio = _ffi.gc(bio, _lib.BIO_free)
+
+        dh = _lib.PEM_read_bio_DHparams(bio, _ffi.NULL, _ffi.NULL, _ffi.NULL)
+        dh = _ffi.gc(dh, _lib.DH_free)
+        res = _lib.SSL_CTX_set_tmp_dh(self._context, dh)
+        _openssl_assert(res == 1)
+
+    def set_tmp_ecdh(self, curve):
+        """
+        Select a curve to use for ECDHE key exchange.
+
+        :param curve: A curve object to use as returned by either
+            :meth:`OpenSSL.crypto.get_elliptic_curve` or
+            :meth:`OpenSSL.crypto.get_elliptic_curves`.
+
+        :return: None
+        """
+        _lib.SSL_CTX_set_tmp_ecdh(self._context, curve._to_EC_KEY())
+
+    def set_cipher_list(self, cipher_list):
+        """
+        Set the list of ciphers to be used in this context.
+
+        See the OpenSSL manual for more information (e.g.
+        :manpage:`ciphers(1)`).
+
+        :param bytes cipher_list: An OpenSSL cipher string.
+        :return: None
+        """
+        cipher_list = _text_to_bytes_and_warn("cipher_list", cipher_list)
+
+        if not isinstance(cipher_list, bytes):
+            raise TypeError("cipher_list must be a byte string.")
+
+        _openssl_assert(
+            _lib.SSL_CTX_set_cipher_list(self._context, cipher_list) == 1
+        )
+        # In OpenSSL 1.1.1 setting the cipher list will always return TLS 1.3
+        # ciphers even if you pass an invalid cipher. Applications (like
+        # Twisted) have tests that depend on an error being raised if an
+        # invalid cipher string is passed, but without the following check
+        # for the TLS 1.3 specific cipher suites it would never error.
+        tmpconn = Connection(self, None)
+        if tmpconn.get_cipher_list() == [
+            "TLS_AES_256_GCM_SHA384",
+            "TLS_CHACHA20_POLY1305_SHA256",
+            "TLS_AES_128_GCM_SHA256",
+        ]:
+            raise Error(
+                [
+                    (
+                        "SSL routines",
+                        "SSL_CTX_set_cipher_list",
+                        "no cipher match",
+                    ),
+                ],
+            )
+
+    def set_client_ca_list(self, certificate_authorities):
+        """
+        Set the list of preferred client certificate signers for this server
+        context.
+
+        This list of certificate authorities will be sent to the client when
+        the server requests a client certificate.
+
+        :param certificate_authorities: a sequence of X509Names.
+        :return: None
+
+        .. versionadded:: 0.10
+        """
+        name_stack = _lib.sk_X509_NAME_new_null()
+        _openssl_assert(name_stack != _ffi.NULL)
+
+        try:
+            for ca_name in certificate_authorities:
+                if not isinstance(ca_name, X509Name):
+                    raise TypeError(
+                        "client CAs must be X509Name objects, not %s "
+                        "objects" % (type(ca_name).__name__,)
+                    )
+                copy = _lib.X509_NAME_dup(ca_name._name)
+                _openssl_assert(copy != _ffi.NULL)
+                push_result = _lib.sk_X509_NAME_push(name_stack, copy)
+                if not push_result:
+                    _lib.X509_NAME_free(copy)
+                    _raise_current_error()
+        except Exception:
+            _lib.sk_X509_NAME_free(name_stack)
+            raise
+
+        _lib.SSL_CTX_set_client_CA_list(self._context, name_stack)
+
+    def add_client_ca(self, certificate_authority):
+        """
+        Add the CA certificate to the list of preferred signers for this
+        context.
+
+        The list of certificate authorities will be sent to the client when the
+        server requests a client certificate.
+
+        :param certificate_authority: certificate authority's X509 certificate.
+        :return: None
+
+        .. versionadded:: 0.10
+        """
+        if not isinstance(certificate_authority, X509):
+            raise TypeError("certificate_authority must be an X509 instance")
+
+        add_result = _lib.SSL_CTX_add_client_CA(
+            self._context, certificate_authority._x509
+        )
+        _openssl_assert(add_result == 1)
+
+    def set_timeout(self, timeout):
+        """
+        Set the timeout for newly created sessions for this Context object to
+        *timeout*.  The default value is 300 seconds. See the OpenSSL manual
+        for more information (e.g. :manpage:`SSL_CTX_set_timeout(3)`).
+
+        :param timeout: The timeout in (whole) seconds
+        :return: The previous session timeout
+        """
+        if not isinstance(timeout, integer_types):
+            raise TypeError("timeout must be an integer")
+
+        return _lib.SSL_CTX_set_timeout(self._context, timeout)
+
+    def get_timeout(self):
+        """
+        Retrieve session timeout, as set by :meth:`set_timeout`. The default
+        is 300 seconds.
+
+        :return: The session timeout
+        """
+        return _lib.SSL_CTX_get_timeout(self._context)
+
+    def set_info_callback(self, callback):
+        """
+        Set the information callback to *callback*. This function will be
+        called from time to time during SSL handshakes.
+
+        :param callback: The Python callback to use.  This should take three
+            arguments: a Connection object and two integers.  The first integer
+            specifies where in the SSL handshake the function was called, and
+            the other the return code from a (possibly failed) internal
+            function call.
+        :return: None
+        """
+
+        @wraps(callback)
+        def wrapper(ssl, where, return_code):
+            callback(Connection._reverse_mapping[ssl], where, return_code)
+
+        self._info_callback = _ffi.callback(
+            "void (*)(const SSL *, int, int)", wrapper
+        )
+        _lib.SSL_CTX_set_info_callback(self._context, self._info_callback)
+
+    @_requires_keylog
+    def set_keylog_callback(self, callback):
+        """
+        Set the TLS key logging callback to *callback*. This function will be
+        called whenever TLS key material is generated or received, in order
+        to allow applications to store this keying material for debugging
+        purposes.
+
+        :param callback: The Python callback to use.  This should take two
+            arguments: a Connection object and a bytestring that contains
+            the key material in the format used by NSS for its SSLKEYLOGFILE
+            debugging output.
+        :return: None
+        """
+
+        @wraps(callback)
+        def wrapper(ssl, line):
+            line = _ffi.string(line)
+            callback(Connection._reverse_mapping[ssl], line)
+
+        self._keylog_callback = _ffi.callback(
+            "void (*)(const SSL *, const char *)", wrapper
+        )
+        _lib.SSL_CTX_set_keylog_callback(self._context, self._keylog_callback)
+
+    def get_app_data(self):
+        """
+        Get the application data (supplied via :meth:`set_app_data()`)
+
+        :return: The application data
+        """
+        return self._app_data
+
+    def set_app_data(self, data):
+        """
+        Set the application data (will be returned from get_app_data())
+
+        :param data: Any Python object
+        :return: None
+        """
+        self._app_data = data
+
+    def get_cert_store(self):
+        """
+        Get the certificate store for the context.  This can be used to add
+        "trusted" certificates without using the
+        :meth:`load_verify_locations` method.
+
+        :return: A X509Store object or None if it does not have one.
+        """
+        store = _lib.SSL_CTX_get_cert_store(self._context)
+        if store == _ffi.NULL:
+            # TODO: This is untested.
+            return None
+
+        pystore = X509Store.__new__(X509Store)
+        pystore._store = store
+        return pystore
+
+    def set_options(self, options):
+        """
+        Add options. Options set before are not cleared!
+        This method should be used with the :const:`OP_*` constants.
+
+        :param options: The options to add.
+        :return: The new option bitmask.
+        """
+        if not isinstance(options, integer_types):
+            raise TypeError("options must be an integer")
+
+        return _lib.SSL_CTX_set_options(self._context, options)
+
+    def set_mode(self, mode):
+        """
+        Add modes via bitmask. Modes set before are not cleared!  This method
+        should be used with the :const:`MODE_*` constants.
+
+        :param mode: The mode to add.
+        :return: The new mode bitmask.
+        """
+        if not isinstance(mode, integer_types):
+            raise TypeError("mode must be an integer")
+
+        return _lib.SSL_CTX_set_mode(self._context, mode)
+
+    def set_tlsext_servername_callback(self, callback):
+        """
+        Specify a callback function to be called when clients specify a server
+        name.
+
+        :param callback: The callback function.  It will be invoked with one
+            argument, the Connection instance.
+
+        .. versionadded:: 0.13
+        """
+
+        @wraps(callback)
+        def wrapper(ssl, alert, arg):
+            callback(Connection._reverse_mapping[ssl])
+            return 0
+
+        self._tlsext_servername_callback = _ffi.callback(
+            "int (*)(SSL *, int *, void *)", wrapper
+        )
+        _lib.SSL_CTX_set_tlsext_servername_callback(
+            self._context, self._tlsext_servername_callback
+        )
+
+    def set_tlsext_use_srtp(self, profiles):
+        """
+        Enable support for negotiating SRTP keying material.
+
+        :param bytes profiles: A colon delimited list of protection profile
+            names, like ``b'SRTP_AES128_CM_SHA1_80:SRTP_AES128_CM_SHA1_32'``.
+        :return: None
+        """
+        if not isinstance(profiles, bytes):
+            raise TypeError("profiles must be a byte string.")
+
+        _openssl_assert(
+            _lib.SSL_CTX_set_tlsext_use_srtp(self._context, profiles) == 0
+        )
+
+    @_requires_alpn
+    def set_alpn_protos(self, protos):
+        """
+        Specify the protocols that the client is prepared to speak after the
+        TLS connection has been negotiated using Application Layer Protocol
+        Negotiation.
+
+        :param protos: A list of the protocols to be offered to the server.
+            This list should be a Python list of bytestrings representing the
+            protocols to offer, e.g. ``[b'http/1.1', b'spdy/2']``.
+        """
+        # Take the list of protocols and join them together, prefixing them
+        # with their lengths.
+        protostr = b"".join(
+            chain.from_iterable((int2byte(len(p)), p) for p in protos)
+        )
+
+        # Build a C string from the list. We don't need to save this off
+        # because OpenSSL immediately copies the data out.
+        input_str = _ffi.new("unsigned char[]", protostr)
+
+        # https://www.openssl.org/docs/man1.1.0/man3/SSL_CTX_set_alpn_protos.html:
+        # SSL_CTX_set_alpn_protos() and SSL_set_alpn_protos()
+        # return 0 on success, and non-0 on failure.
+        # WARNING: these functions reverse the return value convention.
+        _openssl_assert(
+            _lib.SSL_CTX_set_alpn_protos(
+                self._context, input_str, len(protostr)
+            )
+            == 0
+        )
+
+    @_requires_alpn
+    def set_alpn_select_callback(self, callback):
+        """
+        Specify a callback function that will be called on the server when a
+        client offers protocols using ALPN.
+
+        :param callback: The callback function.  It will be invoked with two
+            arguments: the Connection, and a list of offered protocols as
+            bytestrings, e.g ``[b'http/1.1', b'spdy/2']``.  It can return
+            one of those bytestrings to indicate the chosen protocol, the
+            empty bytestring to terminate the TLS connection, or the
+            :py:obj:`NO_OVERLAPPING_PROTOCOLS` to indicate that no offered
+            protocol was selected, but that the connection should not be
+            aborted.
+        """
+        self._alpn_select_helper = _ALPNSelectHelper(callback)
+        self._alpn_select_callback = self._alpn_select_helper.callback
+        _lib.SSL_CTX_set_alpn_select_cb(
+            self._context, self._alpn_select_callback, _ffi.NULL
+        )
+
+    def _set_ocsp_callback(self, helper, data):
+        """
+        This internal helper does the common work for
+        ``set_ocsp_server_callback`` and ``set_ocsp_client_callback``, which is
+        almost all of it.
+        """
+        self._ocsp_helper = helper
+        self._ocsp_callback = helper.callback
+        if data is None:
+            self._ocsp_data = _ffi.NULL
+        else:
+            self._ocsp_data = _ffi.new_handle(data)
+
+        rc = _lib.SSL_CTX_set_tlsext_status_cb(
+            self._context, self._ocsp_callback
+        )
+        _openssl_assert(rc == 1)
+        rc = _lib.SSL_CTX_set_tlsext_status_arg(self._context, self._ocsp_data)
+        _openssl_assert(rc == 1)
+
+    def set_ocsp_server_callback(self, callback, data=None):
+        """
+        Set a callback to provide OCSP data to be stapled to the TLS handshake
+        on the server side.
+
+        :param callback: The callback function. It will be invoked with two
+            arguments: the Connection, and the optional arbitrary data you have
+            provided. The callback must return a bytestring that contains the
+            OCSP data to staple to the handshake. If no OCSP data is available
+            for this connection, return the empty bytestring.
+        :param data: Some opaque data that will be passed into the callback
+            function when called. This can be used to avoid needing to do
+            complex data lookups or to keep track of what context is being
+            used. This parameter is optional.
+        """
+        helper = _OCSPServerCallbackHelper(callback)
+        self._set_ocsp_callback(helper, data)
+
+    def set_ocsp_client_callback(self, callback, data=None):
+        """
+        Set a callback to validate OCSP data stapled to the TLS handshake on
+        the client side.
+
+        :param callback: The callback function. It will be invoked with three
+            arguments: the Connection, a bytestring containing the stapled OCSP
+            assertion, and the optional arbitrary data you have provided. The
+            callback must return a boolean that indicates the result of
+            validating the OCSP data: ``True`` if the OCSP data is valid and
+            the certificate can be trusted, or ``False`` if either the OCSP
+            data is invalid or the certificate has been revoked.
+        :param data: Some opaque data that will be passed into the callback
+            function when called. This can be used to avoid needing to do
+            complex data lookups or to keep track of what context is being
+            used. This parameter is optional.
+        """
+        helper = _OCSPClientCallbackHelper(callback)
+        self._set_ocsp_callback(helper, data)
+
+
+class Connection(object):
+    _reverse_mapping = WeakValueDictionary()
+
+    def __init__(self, context, socket=None):
+        """
+        Create a new Connection object, using the given OpenSSL.SSL.Context
+        instance and socket.
+
+        :param context: An SSL Context to use for this connection
+        :param socket: The socket to use for transport layer
+        """
+        if not isinstance(context, Context):
+            raise TypeError("context must be a Context instance")
+
+        ssl = _lib.SSL_new(context._context)
+        self._ssl = _ffi.gc(ssl, _lib.SSL_free)
+        # We set SSL_MODE_AUTO_RETRY to handle situations where OpenSSL returns
+        # an SSL_ERROR_WANT_READ when processing a non-application data packet
+        # even though there is still data on the underlying transport.
+        # See https://github.com/openssl/openssl/issues/6234 for more details.
+        _lib.SSL_set_mode(self._ssl, _lib.SSL_MODE_AUTO_RETRY)
+        self._context = context
+        self._app_data = None
+
+        # References to strings used for Application Layer Protocol
+        # Negotiation. These strings get copied at some point but it's well
+        # after the callback returns, so we have to hang them somewhere to
+        # avoid them getting freed.
+        self._alpn_select_callback_args = None
+
+        # Reference the verify_callback of the Context. This ensures that if
+        # set_verify is called again after the SSL object has been created we
+        # do not point to a dangling reference
+        self._verify_helper = context._verify_helper
+        self._verify_callback = context._verify_callback
+
+        self._reverse_mapping[self._ssl] = self
+
+        if socket is None:
+            self._socket = None
+            # Don't set up any gc for these, SSL_free will take care of them.
+            self._into_ssl = _lib.BIO_new(_lib.BIO_s_mem())
+            _openssl_assert(self._into_ssl != _ffi.NULL)
+
+            self._from_ssl = _lib.BIO_new(_lib.BIO_s_mem())
+            _openssl_assert(self._from_ssl != _ffi.NULL)
+
+            _lib.SSL_set_bio(self._ssl, self._into_ssl, self._from_ssl)
+        else:
+            self._into_ssl = None
+            self._from_ssl = None
+            self._socket = socket
+            set_result = _lib.SSL_set_fd(
+                self._ssl, _asFileDescriptor(self._socket)
+            )
+            _openssl_assert(set_result == 1)
+
+    def __getattr__(self, name):
+        """
+        Look up attributes on the wrapped socket object if they are not found
+        on the Connection object.
+        """
+        if self._socket is None:
+            raise AttributeError(
+                "'%s' object has no attribute '%s'"
+                % (self.__class__.__name__, name)
+            )
+        else:
+            return getattr(self._socket, name)
+
+    def _raise_ssl_error(self, ssl, result):
+        if self._context._verify_helper is not None:
+            self._context._verify_helper.raise_if_problem()
+        if self._context._alpn_select_helper is not None:
+            self._context._alpn_select_helper.raise_if_problem()
+        if self._context._ocsp_helper is not None:
+            self._context._ocsp_helper.raise_if_problem()
+
+        error = _lib.SSL_get_error(ssl, result)
+        if error == _lib.SSL_ERROR_WANT_READ:
+            raise WantReadError()
+        elif error == _lib.SSL_ERROR_WANT_WRITE:
+            raise WantWriteError()
+        elif error == _lib.SSL_ERROR_ZERO_RETURN:
+            raise ZeroReturnError()
+        elif error == _lib.SSL_ERROR_WANT_X509_LOOKUP:
+            # TODO: This is untested.
+            raise WantX509LookupError()
+        elif error == _lib.SSL_ERROR_SYSCALL:
+            if _lib.ERR_peek_error() == 0:
+                if result < 0:
+                    if platform == "win32":
+                        errno = _ffi.getwinerror()[0]
+                    else:
+                        errno = _ffi.errno
+
+                    if errno != 0:
+                        raise SysCallError(errno, errorcode.get(errno))
+                raise SysCallError(-1, "Unexpected EOF")
+            else:
+                # TODO: This is untested.
+                _raise_current_error()
+        elif error == _lib.SSL_ERROR_NONE:
+            pass
+        else:
+            _raise_current_error()
+
+    def get_context(self):
+        """
+        Retrieve the :class:`Context` object associated with this
+        :class:`Connection`.
+        """
+        return self._context
+
+    def set_context(self, context):
+        """
+        Switch this connection to a new session context.
+
+        :param context: A :class:`Context` instance giving the new session
+            context to use.
+        """
+        if not isinstance(context, Context):
+            raise TypeError("context must be a Context instance")
+
+        _lib.SSL_set_SSL_CTX(self._ssl, context._context)
+        self._context = context
+
+    def get_servername(self):
+        """
+        Retrieve the servername extension value if provided in the client hello
+        message, or None if there wasn't one.
+
+        :return: A byte string giving the server name or :data:`None`.
+
+        .. versionadded:: 0.13
+        """
+        name = _lib.SSL_get_servername(
+            self._ssl, _lib.TLSEXT_NAMETYPE_host_name
+        )
+        if name == _ffi.NULL:
+            return None
+
+        return _ffi.string(name)
+
+    def set_tlsext_host_name(self, name):
+        """
+        Set the value of the servername extension to send in the client hello.
+
+        :param name: A byte string giving the name.
+
+        .. versionadded:: 0.13
+        """
+        if not isinstance(name, bytes):
+            raise TypeError("name must be a byte string")
+        elif b"\0" in name:
+            raise TypeError("name must not contain NUL byte")
+
+        # XXX I guess this can fail sometimes?
+        _lib.SSL_set_tlsext_host_name(self._ssl, name)
+
+    def pending(self):
+        """
+        Get the number of bytes that can be safely read from the SSL buffer
+        (**not** the underlying transport buffer).
+
+        :return: The number of bytes available in the receive buffer.
+        """
+        return _lib.SSL_pending(self._ssl)
+
+    def send(self, buf, flags=0):
+        """
+        Send data on the connection. NOTE: If you get one of the WantRead,
+        WantWrite or WantX509Lookup exceptions on this, you have to call the
+        method again with the SAME buffer.
+
+        :param buf: The string, buffer or memoryview to send
+        :param flags: (optional) Included for compatibility with the socket
+                      API, the value is ignored
+        :return: The number of bytes written
+        """
+        # Backward compatibility
+        buf = _text_to_bytes_and_warn("buf", buf)
+
+        with _ffi.from_buffer(buf) as data:
+            # check len(buf) instead of len(data) for testability
+            if len(buf) > 2147483647:
+                raise ValueError(
+                    "Cannot send more than 2**31-1 bytes at once."
+                )
+
+            result = _lib.SSL_write(self._ssl, data, len(data))
+            self._raise_ssl_error(self._ssl, result)
+
+            return result
+
+    write = send
+
+    def sendall(self, buf, flags=0):
+        """
+        Send "all" data on the connection. This calls send() repeatedly until
+        all data is sent. If an error occurs, it's impossible to tell how much
+        data has been sent.
+
+        :param buf: The string, buffer or memoryview to send
+        :param flags: (optional) Included for compatibility with the socket
+                      API, the value is ignored
+        :return: The number of bytes written
+        """
+        buf = _text_to_bytes_and_warn("buf", buf)
+
+        with _ffi.from_buffer(buf) as data:
+
+            left_to_send = len(buf)
+            total_sent = 0
+
+            while left_to_send:
+                # SSL_write's num arg is an int,
+                # so we cannot send more than 2**31-1 bytes at once.
+                result = _lib.SSL_write(
+                    self._ssl, data + total_sent, min(left_to_send, 2147483647)
+                )
+                self._raise_ssl_error(self._ssl, result)
+                total_sent += result
+                left_to_send -= result
+
+            return total_sent
+
+    def recv(self, bufsiz, flags=None):
+        """
+        Receive data on the connection.
+
+        :param bufsiz: The maximum number of bytes to read
+        :param flags: (optional) The only supported flag is ``MSG_PEEK``,
+            all other flags are ignored.
+        :return: The string read from the Connection
+        """
+        buf = _no_zero_allocator("char[]", bufsiz)
+        if flags is not None and flags & socket.MSG_PEEK:
+            result = _lib.SSL_peek(self._ssl, buf, bufsiz)
+        else:
+            result = _lib.SSL_read(self._ssl, buf, bufsiz)
+        self._raise_ssl_error(self._ssl, result)
+        return _ffi.buffer(buf, result)[:]
+
+    read = recv
+
+    def recv_into(self, buffer, nbytes=None, flags=None):
+        """
+        Receive data on the connection and copy it directly into the provided
+        buffer, rather than creating a new string.
+
+        :param buffer: The buffer to copy into.
+        :param nbytes: (optional) The maximum number of bytes to read into the
+            buffer. If not present, defaults to the size of the buffer. If
+            larger than the size of the buffer, is reduced to the size of the
+            buffer.
+        :param flags: (optional) The only supported flag is ``MSG_PEEK``,
+            all other flags are ignored.
+        :return: The number of bytes read into the buffer.
+        """
+        if nbytes is None:
+            nbytes = len(buffer)
+        else:
+            nbytes = min(nbytes, len(buffer))
+
+        # We need to create a temporary buffer. This is annoying, it would be
+        # better if we could pass memoryviews straight into the SSL_read call,
+        # but right now we can't. Revisit this if CFFI gets that ability.
+        buf = _no_zero_allocator("char[]", nbytes)
+        if flags is not None and flags & socket.MSG_PEEK:
+            result = _lib.SSL_peek(self._ssl, buf, nbytes)
+        else:
+            result = _lib.SSL_read(self._ssl, buf, nbytes)
+        self._raise_ssl_error(self._ssl, result)
+
+        # This strange line is all to avoid a memory copy. The buffer protocol
+        # should allow us to assign a CFFI buffer to the LHS of this line, but
+        # on CPython 3.3+ that segfaults. As a workaround, we can temporarily
+        # wrap it in a memoryview.
+        buffer[:result] = memoryview(_ffi.buffer(buf, result))
+
+        return result
+
+    def _handle_bio_errors(self, bio, result):
+        if _lib.BIO_should_retry(bio):
+            if _lib.BIO_should_read(bio):
+                raise WantReadError()
+            elif _lib.BIO_should_write(bio):
+                # TODO: This is untested.
+                raise WantWriteError()
+            elif _lib.BIO_should_io_special(bio):
+                # TODO: This is untested.  I think io_special means the socket
+                # BIO has a not-yet connected socket.
+                raise ValueError("BIO_should_io_special")
+            else:
+                # TODO: This is untested.
+                raise ValueError("unknown bio failure")
+        else:
+            # TODO: This is untested.
+            _raise_current_error()
+
+    def bio_read(self, bufsiz):
+        """
+        If the Connection was created with a memory BIO, this method can be
+        used to read bytes from the write end of that memory BIO.  Many
+        Connection methods will add bytes which must be read in this manner or
+        the buffer will eventually fill up and the Connection will be able to
+        take no further actions.
+
+        :param bufsiz: The maximum number of bytes to read
+        :return: The string read.
+        """
+        if self._from_ssl is None:
+            raise TypeError("Connection sock was not None")
+
+        if not isinstance(bufsiz, integer_types):
+            raise TypeError("bufsiz must be an integer")
+
+        buf = _no_zero_allocator("char[]", bufsiz)
+        result = _lib.BIO_read(self._from_ssl, buf, bufsiz)
+        if result <= 0:
+            self._handle_bio_errors(self._from_ssl, result)
+
+        return _ffi.buffer(buf, result)[:]
+
+    def bio_write(self, buf):
+        """
+        If the Connection was created with a memory BIO, this method can be
+        used to add bytes to the read end of that memory BIO.  The Connection
+        can then read the bytes (for example, in response to a call to
+        :meth:`recv`).
+
+        :param buf: The string to put into the memory BIO.
+        :return: The number of bytes written
+        """
+        buf = _text_to_bytes_and_warn("buf", buf)
+
+        if self._into_ssl is None:
+            raise TypeError("Connection sock was not None")
+
+        with _ffi.from_buffer(buf) as data:
+            result = _lib.BIO_write(self._into_ssl, data, len(data))
+            if result <= 0:
+                self._handle_bio_errors(self._into_ssl, result)
+            return result
+
+    def renegotiate(self):
+        """
+        Renegotiate the session.
+
+        :return: True if the renegotiation can be started, False otherwise
+        :rtype: bool
+        """
+        if not self.renegotiate_pending():
+            _openssl_assert(_lib.SSL_renegotiate(self._ssl) == 1)
+            return True
+        return False
+
+    def do_handshake(self):
+        """
+        Perform an SSL handshake (usually called after :meth:`renegotiate` or
+        one of :meth:`set_accept_state` or :meth:`set_connect_state`). This can
+        raise the same exceptions as :meth:`send` and :meth:`recv`.
+
+        :return: None.
+        """
+        result = _lib.SSL_do_handshake(self._ssl)
+        self._raise_ssl_error(self._ssl, result)
+
+    def renegotiate_pending(self):
+        """
+        Check if there's a renegotiation in progress, it will return False once
+        a renegotiation is finished.
+
+        :return: Whether there's a renegotiation in progress
+        :rtype: bool
+        """
+        return _lib.SSL_renegotiate_pending(self._ssl) == 1
+
+    def total_renegotiations(self):
+        """
+        Find out the total number of renegotiations.
+
+        :return: The number of renegotiations.
+        :rtype: int
+        """
+        return _lib.SSL_total_renegotiations(self._ssl)
+
+    def connect(self, addr):
+        """
+        Call the :meth:`connect` method of the underlying socket and set up SSL
+        on the socket, using the :class:`Context` object supplied to this
+        :class:`Connection` object at creation.
+
+        :param addr: A remote address
+        :return: What the socket's connect method returns
+        """
+        _lib.SSL_set_connect_state(self._ssl)
+        return self._socket.connect(addr)
+
+    def connect_ex(self, addr):
+        """
+        Call the :meth:`connect_ex` method of the underlying socket and set up
+        SSL on the socket, using the Context object supplied to this Connection
+        object at creation. Note that if the :meth:`connect_ex` method of the
+        socket doesn't return 0, SSL won't be initialized.
+
+        :param addr: A remove address
+        :return: What the socket's connect_ex method returns
+        """
+        connect_ex = self._socket.connect_ex
+        self.set_connect_state()
+        return connect_ex(addr)
+
+    def accept(self):
+        """
+        Call the :meth:`accept` method of the underlying socket and set up SSL
+        on the returned socket, using the Context object supplied to this
+        :class:`Connection` object at creation.
+
+        :return: A *(conn, addr)* pair where *conn* is the new
+            :class:`Connection` object created, and *address* is as returned by
+            the socket's :meth:`accept`.
+        """
+        client, addr = self._socket.accept()
+        conn = Connection(self._context, client)
+        conn.set_accept_state()
+        return (conn, addr)
+
+    def bio_shutdown(self):
+        """
+        If the Connection was created with a memory BIO, this method can be
+        used to indicate that *end of file* has been reached on the read end of
+        that memory BIO.
+
+        :return: None
+        """
+        if self._from_ssl is None:
+            raise TypeError("Connection sock was not None")
+
+        _lib.BIO_set_mem_eof_return(self._into_ssl, 0)
+
+    def shutdown(self):
+        """
+        Send the shutdown message to the Connection.
+
+        :return: True if the shutdown completed successfully (i.e. both sides
+                 have sent closure alerts), False otherwise (in which case you
+                 call :meth:`recv` or :meth:`send` when the connection becomes
+                 readable/writeable).
+        """
+        result = _lib.SSL_shutdown(self._ssl)
+        if result < 0:
+            self._raise_ssl_error(self._ssl, result)
+        elif result > 0:
+            return True
+        else:
+            return False
+
+    def get_cipher_list(self):
+        """
+        Retrieve the list of ciphers used by the Connection object.
+
+        :return: A list of native cipher strings.
+        """
+        ciphers = []
+        for i in count():
+            result = _lib.SSL_get_cipher_list(self._ssl, i)
+            if result == _ffi.NULL:
+                break
+            ciphers.append(_native(_ffi.string(result)))
+        return ciphers
+
+    def get_client_ca_list(self):
+        """
+        Get CAs whose certificates are suggested for client authentication.
+
+        :return: If this is a server connection, the list of certificate
+            authorities that will be sent or has been sent to the client, as
+            controlled by this :class:`Connection`'s :class:`Context`.
+
+            If this is a client connection, the list will be empty until the
+            connection with the server is established.
+
+        .. versionadded:: 0.10
+        """
+        ca_names = _lib.SSL_get_client_CA_list(self._ssl)
+        if ca_names == _ffi.NULL:
+            # TODO: This is untested.
+            return []
+
+        result = []
+        for i in range(_lib.sk_X509_NAME_num(ca_names)):
+            name = _lib.sk_X509_NAME_value(ca_names, i)
+            copy = _lib.X509_NAME_dup(name)
+            _openssl_assert(copy != _ffi.NULL)
+
+            pyname = X509Name.__new__(X509Name)
+            pyname._name = _ffi.gc(copy, _lib.X509_NAME_free)
+            result.append(pyname)
+        return result
+
+    def makefile(self, *args, **kwargs):
+        """
+        The makefile() method is not implemented, since there is no dup
+        semantics for SSL connections
+
+        :raise: NotImplementedError
+        """
+        raise NotImplementedError(
+            "Cannot make file object of OpenSSL.SSL.Connection"
+        )
+
+    def get_app_data(self):
+        """
+        Retrieve application data as set by :meth:`set_app_data`.
+
+        :return: The application data
+        """
+        return self._app_data
+
+    def set_app_data(self, data):
+        """
+        Set application data
+
+        :param data: The application data
+        :return: None
+        """
+        self._app_data = data
+
+    def get_shutdown(self):
+        """
+        Get the shutdown state of the Connection.
+
+        :return: The shutdown state, a bitvector of SENT_SHUTDOWN,
+            RECEIVED_SHUTDOWN.
+        """
+        return _lib.SSL_get_shutdown(self._ssl)
+
+    def set_shutdown(self, state):
+        """
+        Set the shutdown state of the Connection.
+
+        :param state: bitvector of SENT_SHUTDOWN, RECEIVED_SHUTDOWN.
+        :return: None
+        """
+        if not isinstance(state, integer_types):
+            raise TypeError("state must be an integer")
+
+        _lib.SSL_set_shutdown(self._ssl, state)
+
+    def get_state_string(self):
+        """
+        Retrieve a verbose string detailing the state of the Connection.
+
+        :return: A string representing the state
+        :rtype: bytes
+        """
+        return _ffi.string(_lib.SSL_state_string_long(self._ssl))
+
+    def server_random(self):
+        """
+        Retrieve the random value used with the server hello message.
+
+        :return: A string representing the state
+        """
+        session = _lib.SSL_get_session(self._ssl)
+        if session == _ffi.NULL:
+            return None
+        length = _lib.SSL_get_server_random(self._ssl, _ffi.NULL, 0)
+        _openssl_assert(length > 0)
+        outp = _no_zero_allocator("unsigned char[]", length)
+        _lib.SSL_get_server_random(self._ssl, outp, length)
+        return _ffi.buffer(outp, length)[:]
+
+    def client_random(self):
+        """
+        Retrieve the random value used with the client hello message.
+
+        :return: A string representing the state
+        """
+        session = _lib.SSL_get_session(self._ssl)
+        if session == _ffi.NULL:
+            return None
+
+        length = _lib.SSL_get_client_random(self._ssl, _ffi.NULL, 0)
+        _openssl_assert(length > 0)
+        outp = _no_zero_allocator("unsigned char[]", length)
+        _lib.SSL_get_client_random(self._ssl, outp, length)
+        return _ffi.buffer(outp, length)[:]
+
+    def master_key(self):
+        """
+        Retrieve the value of the master key for this session.
+
+        :return: A string representing the state
+        """
+        session = _lib.SSL_get_session(self._ssl)
+        if session == _ffi.NULL:
+            return None
+
+        length = _lib.SSL_SESSION_get_master_key(session, _ffi.NULL, 0)
+        _openssl_assert(length > 0)
+        outp = _no_zero_allocator("unsigned char[]", length)
+        _lib.SSL_SESSION_get_master_key(session, outp, length)
+        return _ffi.buffer(outp, length)[:]
+
+    def export_keying_material(self, label, olen, context=None):
+        """
+        Obtain keying material for application use.
+
+        :param: label - a disambiguating label string as described in RFC 5705
+        :param: olen - the length of the exported key material in bytes
+        :param: context - a per-association context value
+        :return: the exported key material bytes or None
+        """
+        outp = _no_zero_allocator("unsigned char[]", olen)
+        context_buf = _ffi.NULL
+        context_len = 0
+        use_context = 0
+        if context is not None:
+            context_buf = context
+            context_len = len(context)
+            use_context = 1
+        success = _lib.SSL_export_keying_material(
+            self._ssl,
+            outp,
+            olen,
+            label,
+            len(label),
+            context_buf,
+            context_len,
+            use_context,
+        )
+        _openssl_assert(success == 1)
+        return _ffi.buffer(outp, olen)[:]
+
+    def sock_shutdown(self, *args, **kwargs):
+        """
+        Call the :meth:`shutdown` method of the underlying socket.
+        See :manpage:`shutdown(2)`.
+
+        :return: What the socket's shutdown() method returns
+        """
+        return self._socket.shutdown(*args, **kwargs)
+
+    def get_certificate(self):
+        """
+        Retrieve the local certificate (if any)
+
+        :return: The local certificate
+        """
+        cert = _lib.SSL_get_certificate(self._ssl)
+        if cert != _ffi.NULL:
+            _lib.X509_up_ref(cert)
+            return X509._from_raw_x509_ptr(cert)
+        return None
+
+    def get_peer_certificate(self):
+        """
+        Retrieve the other side's certificate (if any)
+
+        :return: The peer's certificate
+        """
+        cert = _lib.SSL_get_peer_certificate(self._ssl)
+        if cert != _ffi.NULL:
+            return X509._from_raw_x509_ptr(cert)
+        return None
+
+    @staticmethod
+    def _cert_stack_to_list(cert_stack):
+        """
+        Internal helper to convert a STACK_OF(X509) to a list of X509
+        instances.
+        """
+        result = []
+        for i in range(_lib.sk_X509_num(cert_stack)):
+            cert = _lib.sk_X509_value(cert_stack, i)
+            _openssl_assert(cert != _ffi.NULL)
+            res = _lib.X509_up_ref(cert)
+            _openssl_assert(res >= 1)
+            pycert = X509._from_raw_x509_ptr(cert)
+            result.append(pycert)
+        return result
+
+    def get_peer_cert_chain(self):
+        """
+        Retrieve the other side's certificate (if any)
+
+        :return: A list of X509 instances giving the peer's certificate chain,
+                 or None if it does not have one.
+        """
+        cert_stack = _lib.SSL_get_peer_cert_chain(self._ssl)
+        if cert_stack == _ffi.NULL:
+            return None
+
+        return self._cert_stack_to_list(cert_stack)
+
+    def get_verified_chain(self):
+        """
+        Retrieve the verified certificate chain of the peer including the
+        peer's end entity certificate. It must be called after a session has
+        been successfully established. If peer verification was not successful
+        the chain may be incomplete, invalid, or None.
+
+        :return: A list of X509 instances giving the peer's verified
+                 certificate chain, or None if it does not have one.
+
+        .. versionadded:: 20.0
+        """
+        # OpenSSL 1.1+
+        cert_stack = _lib.SSL_get0_verified_chain(self._ssl)
+        if cert_stack == _ffi.NULL:
+            return None
+
+        return self._cert_stack_to_list(cert_stack)
+
+    def want_read(self):
+        """
+        Checks if more data has to be read from the transport layer to complete
+        an operation.
+
+        :return: True iff more data has to be read
+        """
+        return _lib.SSL_want_read(self._ssl)
+
+    def want_write(self):
+        """
+        Checks if there is data to write to the transport layer to complete an
+        operation.
+
+        :return: True iff there is data to write
+        """
+        return _lib.SSL_want_write(self._ssl)
+
+    def set_accept_state(self):
+        """
+        Set the connection to work in server mode. The handshake will be
+        handled automatically by read/write.
+
+        :return: None
+        """
+        _lib.SSL_set_accept_state(self._ssl)
+
+    def set_connect_state(self):
+        """
+        Set the connection to work in client mode. The handshake will be
+        handled automatically by read/write.
+
+        :return: None
+        """
+        _lib.SSL_set_connect_state(self._ssl)
+
+    def get_session(self):
+        """
+        Returns the Session currently used.
+
+        :return: An instance of :class:`OpenSSL.SSL.Session` or
+            :obj:`None` if no session exists.
+
+        .. versionadded:: 0.14
+        """
+        session = _lib.SSL_get1_session(self._ssl)
+        if session == _ffi.NULL:
+            return None
+
+        pysession = Session.__new__(Session)
+        pysession._session = _ffi.gc(session, _lib.SSL_SESSION_free)
+        return pysession
+
+    def set_session(self, session):
+        """
+        Set the session to be used when the TLS/SSL connection is established.
+
+        :param session: A Session instance representing the session to use.
+        :returns: None
+
+        .. versionadded:: 0.14
+        """
+        if not isinstance(session, Session):
+            raise TypeError("session must be a Session instance")
+
+        result = _lib.SSL_set_session(self._ssl, session._session)
+        _openssl_assert(result == 1)
+
+    def _get_finished_message(self, function):
+        """
+        Helper to implement :meth:`get_finished` and
+        :meth:`get_peer_finished`.
+
+        :param function: Either :data:`SSL_get_finished`: or
+            :data:`SSL_get_peer_finished`.
+
+        :return: :data:`None` if the desired message has not yet been
+            received, otherwise the contents of the message.
+        :rtype: :class:`bytes` or :class:`NoneType`
+        """
+        # The OpenSSL documentation says nothing about what might happen if the
+        # count argument given is zero.  Specifically, it doesn't say whether
+        # the output buffer may be NULL in that case or not.  Inspection of the
+        # implementation reveals that it calls memcpy() unconditionally.
+        # Section 7.1.4, paragraph 1 of the C standard suggests that
+        # memcpy(NULL, source, 0) is not guaranteed to produce defined (let
+        # alone desirable) behavior (though it probably does on just about
+        # every implementation...)
+        #
+        # Allocate a tiny buffer to pass in (instead of just passing NULL as
+        # one might expect) for the initial call so as to be safe against this
+        # potentially undefined behavior.
+        empty = _ffi.new("char[]", 0)
+        size = function(self._ssl, empty, 0)
+        if size == 0:
+            # No Finished message so far.
+            return None
+
+        buf = _no_zero_allocator("char[]", size)
+        function(self._ssl, buf, size)
+        return _ffi.buffer(buf, size)[:]
+
+    def get_finished(self):
+        """
+        Obtain the latest TLS Finished message that we sent.
+
+        :return: The contents of the message or :obj:`None` if the TLS
+            handshake has not yet completed.
+        :rtype: :class:`bytes` or :class:`NoneType`
+
+        .. versionadded:: 0.15
+        """
+        return self._get_finished_message(_lib.SSL_get_finished)
+
+    def get_peer_finished(self):
+        """
+        Obtain the latest TLS Finished message that we received from the peer.
+
+        :return: The contents of the message or :obj:`None` if the TLS
+            handshake has not yet completed.
+        :rtype: :class:`bytes` or :class:`NoneType`
+
+        .. versionadded:: 0.15
+        """
+        return self._get_finished_message(_lib.SSL_get_peer_finished)
+
+    def get_cipher_name(self):
+        """
+        Obtain the name of the currently used cipher.
+
+        :returns: The name of the currently used cipher or :obj:`None`
+            if no connection has been established.
+        :rtype: :class:`unicode` or :class:`NoneType`
+
+        .. versionadded:: 0.15
+        """
+        cipher = _lib.SSL_get_current_cipher(self._ssl)
+        if cipher == _ffi.NULL:
+            return None
+        else:
+            name = _ffi.string(_lib.SSL_CIPHER_get_name(cipher))
+            return name.decode("utf-8")
+
+    def get_cipher_bits(self):
+        """
+        Obtain the number of secret bits of the currently used cipher.
+
+        :returns: The number of secret bits of the currently used cipher
+            or :obj:`None` if no connection has been established.
+        :rtype: :class:`int` or :class:`NoneType`
+
+        .. versionadded:: 0.15
+        """
+        cipher = _lib.SSL_get_current_cipher(self._ssl)
+        if cipher == _ffi.NULL:
+            return None
+        else:
+            return _lib.SSL_CIPHER_get_bits(cipher, _ffi.NULL)
+
+    def get_cipher_version(self):
+        """
+        Obtain the protocol version of the currently used cipher.
+
+        :returns: The protocol name of the currently used cipher
+            or :obj:`None` if no connection has been established.
+        :rtype: :class:`unicode` or :class:`NoneType`
+
+        .. versionadded:: 0.15
+        """
+        cipher = _lib.SSL_get_current_cipher(self._ssl)
+        if cipher == _ffi.NULL:
+            return None
+        else:
+            version = _ffi.string(_lib.SSL_CIPHER_get_version(cipher))
+            return version.decode("utf-8")
+
+    def get_protocol_version_name(self):
+        """
+        Retrieve the protocol version of the current connection.
+
+        :returns: The TLS version of the current connection, for example
+            the value for TLS 1.2 would be ``TLSv1.2``or ``Unknown``
+            for connections that were not successfully established.
+        :rtype: :class:`unicode`
+        """
+        version = _ffi.string(_lib.SSL_get_version(self._ssl))
+        return version.decode("utf-8")
+
+    def get_protocol_version(self):
+        """
+        Retrieve the SSL or TLS protocol version of the current connection.
+
+        :returns: The TLS version of the current connection.  For example,
+            it will return ``0x769`` for connections made over TLS version 1.
+        :rtype: :class:`int`
+        """
+        version = _lib.SSL_version(self._ssl)
+        return version
+
+    @_requires_alpn
+    def set_alpn_protos(self, protos):
+        """
+        Specify the client's ALPN protocol list.
+
+        These protocols are offered to the server during protocol negotiation.
+
+        :param protos: A list of the protocols to be offered to the server.
+            This list should be a Python list of bytestrings representing the
+            protocols to offer, e.g. ``[b'http/1.1', b'spdy/2']``.
+        """
+        # Take the list of protocols and join them together, prefixing them
+        # with their lengths.
+        protostr = b"".join(
+            chain.from_iterable((int2byte(len(p)), p) for p in protos)
+        )
+
+        # Build a C string from the list. We don't need to save this off
+        # because OpenSSL immediately copies the data out.
+        input_str = _ffi.new("unsigned char[]", protostr)
+
+        # https://www.openssl.org/docs/man1.1.0/man3/SSL_CTX_set_alpn_protos.html:
+        # SSL_CTX_set_alpn_protos() and SSL_set_alpn_protos()
+        # return 0 on success, and non-0 on failure.
+        # WARNING: these functions reverse the return value convention.
+        _openssl_assert(
+            _lib.SSL_set_alpn_protos(self._ssl, input_str, len(protostr)) == 0
+        )
+
+    @_requires_alpn
+    def get_alpn_proto_negotiated(self):
+        """
+        Get the protocol that was negotiated by ALPN.
+
+        :returns: A bytestring of the protocol name.  If no protocol has been
+            negotiated yet, returns an empty string.
+        """
+        data = _ffi.new("unsigned char **")
+        data_len = _ffi.new("unsigned int *")
+
+        _lib.SSL_get0_alpn_selected(self._ssl, data, data_len)
+
+        if not data_len:
+            return b""
+
+        return _ffi.buffer(data[0], data_len[0])[:]
+
+    def request_ocsp(self):
+        """
+        Called to request that the server sends stapled OCSP data, if
+        available. If this is not called on the client side then the server
+        will not send OCSP data. Should be used in conjunction with
+        :meth:`Context.set_ocsp_client_callback`.
+        """
+        rc = _lib.SSL_set_tlsext_status_type(
+            self._ssl, _lib.TLSEXT_STATUSTYPE_ocsp
+        )
+        _openssl_assert(rc == 1)
+
+
+# This is similar to the initialization calls at the end of OpenSSL/crypto.py
+# but is exercised mostly by the Context initializer.
+_lib.SSL_library_init()
diff --git a/contrib/python/pyOpenSSL/py2/OpenSSL/__init__.py b/contrib/python/pyOpenSSL/py2/OpenSSL/__init__.py
new file mode 100644
index 00000000000..11e896a4ea2
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/OpenSSL/__init__.py
@@ -0,0 +1,32 @@
+# Copyright (C) AB Strakt
+# See LICENSE for details.
+
+"""
+pyOpenSSL - A simple wrapper around the OpenSSL library
+"""
+
+from OpenSSL import crypto, SSL
+from OpenSSL.version import (
+    __author__,
+    __copyright__,
+    __email__,
+    __license__,
+    __summary__,
+    __title__,
+    __uri__,
+    __version__,
+)
+
+
+__all__ = [
+    "SSL",
+    "crypto",
+    "__author__",
+    "__copyright__",
+    "__email__",
+    "__license__",
+    "__summary__",
+    "__title__",
+    "__uri__",
+    "__version__",
+]
diff --git a/contrib/python/pyOpenSSL/py2/OpenSSL/_util.py b/contrib/python/pyOpenSSL/py2/OpenSSL/_util.py
new file mode 100644
index 00000000000..53c0b9e573c
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/OpenSSL/_util.py
@@ -0,0 +1,155 @@
+import sys
+import warnings
+
+from six import PY2, text_type
+
+from cryptography.hazmat.bindings.openssl.binding import Binding
+
+
+binding = Binding()
+binding.init_static_locks()
+ffi = binding.ffi
+lib = binding.lib
+
+
+# This is a special CFFI allocator that does not bother to zero its memory
+# after allocation. This has vastly better performance on large allocations and
+# so should be used whenever we don't need the memory zeroed out.
+no_zero_allocator = ffi.new_allocator(should_clear_after_alloc=False)
+
+
+def text(charp):
+    """
+    Get a native string type representing of the given CFFI ``char*`` object.
+
+    :param charp: A C-style string represented using CFFI.
+
+    :return: :class:`str`
+    """
+    if not charp:
+        return ""
+    return native(ffi.string(charp))
+
+
+def exception_from_error_queue(exception_type):
+    """
+    Convert an OpenSSL library failure into a Python exception.
+
+    When a call to the native OpenSSL library fails, this is usually signalled
+    by the return value, and an error code is stored in an error queue
+    associated with the current thread. The err library provides functions to
+    obtain these error codes and textual error messages.
+    """
+    errors = []
+
+    while True:
+        error = lib.ERR_get_error()
+        if error == 0:
+            break
+        errors.append(
+            (
+                text(lib.ERR_lib_error_string(error)),
+                text(lib.ERR_func_error_string(error)),
+                text(lib.ERR_reason_error_string(error)),
+            )
+        )
+
+    raise exception_type(errors)
+
+
+def make_assert(error):
+    """
+    Create an assert function that uses :func:`exception_from_error_queue` to
+    raise an exception wrapped by *error*.
+    """
+
+    def openssl_assert(ok):
+        """
+        If *ok* is not True, retrieve the error from OpenSSL and raise it.
+        """
+        if ok is not True:
+            exception_from_error_queue(error)
+
+    return openssl_assert
+
+
+def native(s):
+    """
+    Convert :py:class:`bytes` or :py:class:`unicode` to the native
+    :py:class:`str` type, using UTF-8 encoding if conversion is necessary.
+
+    :raise UnicodeError: The input string is not UTF-8 decodeable.
+
+    :raise TypeError: The input is neither :py:class:`bytes` nor
+        :py:class:`unicode`.
+    """
+    if not isinstance(s, (bytes, text_type)):
+        raise TypeError("%r is neither bytes nor unicode" % s)
+    if PY2:
+        if isinstance(s, text_type):
+            return s.encode("utf-8")
+    else:
+        if isinstance(s, bytes):
+            return s.decode("utf-8")
+    return s
+
+
+def path_string(s):
+    """
+    Convert a Python string to a :py:class:`bytes` string identifying the same
+    path and which can be passed into an OpenSSL API accepting a filename.
+
+    :param s: An instance of :py:class:`bytes` or :py:class:`unicode`.
+
+    :return: An instance of :py:class:`bytes`.
+    """
+    if isinstance(s, bytes):
+        return s
+    elif isinstance(s, text_type):
+        return s.encode(sys.getfilesystemencoding())
+    else:
+        raise TypeError("Path must be represented as bytes or unicode string")
+
+
+if PY2:
+
+    def byte_string(s):
+        return s
+
+
+else:
+
+    def byte_string(s):
+        return s.encode("charmap")
+
+
+# A marker object to observe whether some optional arguments are passed any
+# value or not.
+UNSPECIFIED = object()
+
+_TEXT_WARNING = (
+    text_type.__name__ + " for {0} is no longer accepted, use bytes"
+)
+
+
+def text_to_bytes_and_warn(label, obj):
+    """
+    If ``obj`` is text, emit a warning that it should be bytes instead and try
+    to convert it to bytes automatically.
+
+    :param str label: The name of the parameter from which ``obj`` was taken
+        (so a developer can easily find the source of the problem and correct
+        it).
+
+    :return: If ``obj`` is the text string type, a ``bytes`` object giving the
+        UTF-8 encoding of that text is returned.  Otherwise, ``obj`` itself is
+        returned.
+    """
+    if isinstance(obj, text_type):
+        warnings.warn(
+            _TEXT_WARNING.format(label),
+            category=DeprecationWarning,
+            stacklevel=3,
+        )
+        return obj.encode("utf-8")
+    return obj
diff --git a/contrib/python/pyOpenSSL/py2/OpenSSL/crypto.py b/contrib/python/pyOpenSSL/py2/OpenSSL/crypto.py
new file mode 100644
index 00000000000..eda4af6f9d9
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/OpenSSL/crypto.py
@@ -0,0 +1,3288 @@
+import calendar
+import datetime
+
+from base64 import b16encode
+from functools import partial
+from operator import __eq__, __ne__, __lt__, __le__, __gt__, __ge__
+
+from six import (
+    integer_types as _integer_types,
+    text_type as _text_type,
+    PY2 as _PY2,
+)
+
+from cryptography import utils, x509
+from cryptography.hazmat.primitives.asymmetric import dsa, rsa
+
+from OpenSSL._util import (
+    ffi as _ffi,
+    lib as _lib,
+    exception_from_error_queue as _exception_from_error_queue,
+    byte_string as _byte_string,
+    native as _native,
+    path_string as _path_string,
+    UNSPECIFIED as _UNSPECIFIED,
+    text_to_bytes_and_warn as _text_to_bytes_and_warn,
+    make_assert as _make_assert,
+)
+
+__all__ = [
+    "FILETYPE_PEM",
+    "FILETYPE_ASN1",
+    "FILETYPE_TEXT",
+    "TYPE_RSA",
+    "TYPE_DSA",
+    "Error",
+    "PKey",
+    "get_elliptic_curves",
+    "get_elliptic_curve",
+    "X509Name",
+    "X509Extension",
+    "X509Req",
+    "X509",
+    "X509StoreFlags",
+    "X509Store",
+    "X509StoreContextError",
+    "X509StoreContext",
+    "load_certificate",
+    "dump_certificate",
+    "dump_publickey",
+    "dump_privatekey",
+    "Revoked",
+    "CRL",
+    "PKCS7",
+    "PKCS12",
+    "NetscapeSPKI",
+    "load_publickey",
+    "load_privatekey",
+    "dump_certificate_request",
+    "load_certificate_request",
+    "sign",
+    "verify",
+    "dump_crl",
+    "load_crl",
+    "load_pkcs7_data",
+    "load_pkcs12",
+]
+
+FILETYPE_PEM = _lib.SSL_FILETYPE_PEM
+FILETYPE_ASN1 = _lib.SSL_FILETYPE_ASN1
+
+# TODO This was an API mistake.  OpenSSL has no such constant.
+FILETYPE_TEXT = 2 ** 16 - 1
+
+TYPE_RSA = _lib.EVP_PKEY_RSA
+TYPE_DSA = _lib.EVP_PKEY_DSA
+TYPE_DH = _lib.EVP_PKEY_DH
+TYPE_EC = _lib.EVP_PKEY_EC
+
+
+class Error(Exception):
+    """
+    An error occurred in an `OpenSSL.crypto` API.
+    """
+
+
+_raise_current_error = partial(_exception_from_error_queue, Error)
+_openssl_assert = _make_assert(Error)
+
+
+def _get_backend():
+    """
+    Importing the backend from cryptography has the side effect of activating
+    the osrandom engine. This mutates the global state of OpenSSL in the
+    process and causes issues for various programs that use subinterpreters or
+    embed Python. By putting the import in this function we can avoid
+    triggering this side effect unless _get_backend is called.
+    """
+    from cryptography.hazmat.backends.openssl.backend import backend
+
+    return backend
+
+
+def _untested_error(where):
+    """
+    An OpenSSL API failed somehow.  Additionally, the failure which was
+    encountered isn't one that's exercised by the test suite so future behavior
+    of pyOpenSSL is now somewhat less predictable.
+    """
+    raise RuntimeError("Unknown %s failure" % (where,))
+
+
+def _new_mem_buf(buffer=None):
+    """
+    Allocate a new OpenSSL memory BIO.
+
+    Arrange for the garbage collector to clean it up automatically.
+
+    :param buffer: None or some bytes to use to put into the BIO so that they
+        can be read out.
+    """
+    if buffer is None:
+        bio = _lib.BIO_new(_lib.BIO_s_mem())
+        free = _lib.BIO_free
+    else:
+        data = _ffi.new("char[]", buffer)
+        bio = _lib.BIO_new_mem_buf(data, len(buffer))
+
+        # Keep the memory alive as long as the bio is alive!
+        def free(bio, ref=data):
+            return _lib.BIO_free(bio)
+
+    _openssl_assert(bio != _ffi.NULL)
+
+    bio = _ffi.gc(bio, free)
+    return bio
+
+
+def _bio_to_string(bio):
+    """
+    Copy the contents of an OpenSSL BIO object into a Python byte string.
+    """
+    result_buffer = _ffi.new("char**")
+    buffer_length = _lib.BIO_get_mem_data(bio, result_buffer)
+    return _ffi.buffer(result_buffer[0], buffer_length)[:]
+
+
+def _set_asn1_time(boundary, when):
+    """
+    The the time value of an ASN1 time object.
+
+    @param boundary: An ASN1_TIME pointer (or an object safely
+        castable to that type) which will have its value set.
+    @param when: A string representation of the desired time value.
+
+    @raise TypeError: If C{when} is not a L{bytes} string.
+    @raise ValueError: If C{when} does not represent a time in the required
+        format.
+    @raise RuntimeError: If the time value cannot be set for some other
+        (unspecified) reason.
+    """
+    if not isinstance(when, bytes):
+        raise TypeError("when must be a byte string")
+
+    set_result = _lib.ASN1_TIME_set_string(boundary, when)
+    if set_result == 0:
+        raise ValueError("Invalid string")
+
+
+def _get_asn1_time(timestamp):
+    """
+    Retrieve the time value of an ASN1 time object.
+
+    @param timestamp: An ASN1_GENERALIZEDTIME* (or an object safely castable to
+        that type) from which the time value will be retrieved.
+
+    @return: The time value from C{timestamp} as a L{bytes} string in a certain
+        format.  Or C{None} if the object contains no time value.
+    """
+    string_timestamp = _ffi.cast("ASN1_STRING*", timestamp)
+    if _lib.ASN1_STRING_length(string_timestamp) == 0:
+        return None
+    elif (
+        _lib.ASN1_STRING_type(string_timestamp) == _lib.V_ASN1_GENERALIZEDTIME
+    ):
+        return _ffi.string(_lib.ASN1_STRING_data(string_timestamp))
+    else:
+        generalized_timestamp = _ffi.new("ASN1_GENERALIZEDTIME**")
+        _lib.ASN1_TIME_to_generalizedtime(timestamp, generalized_timestamp)
+        if generalized_timestamp[0] == _ffi.NULL:
+            # This may happen:
+            #   - if timestamp was not an ASN1_TIME
+            #   - if allocating memory for the ASN1_GENERALIZEDTIME failed
+            #   - if a copy of the time data from timestamp cannot be made for
+            #     the newly allocated ASN1_GENERALIZEDTIME
+            #
+            # These are difficult to test.  cffi enforces the ASN1_TIME type.
+            # Memory allocation failures are a pain to trigger
+            # deterministically.
+            _untested_error("ASN1_TIME_to_generalizedtime")
+        else:
+            string_timestamp = _ffi.cast(
+                "ASN1_STRING*", generalized_timestamp[0]
+            )
+            string_data = _lib.ASN1_STRING_data(string_timestamp)
+            string_result = _ffi.string(string_data)
+            _lib.ASN1_GENERALIZEDTIME_free(generalized_timestamp[0])
+            return string_result
+
+
+class _X509NameInvalidator(object):
+    def __init__(self):
+        self._names = []
+
+    def add(self, name):
+        self._names.append(name)
+
+    def clear(self):
+        for name in self._names:
+            # Breaks the object, but also prevents UAF!
+            del name._name
+
+
+class PKey(object):
+    """
+    A class representing an DSA or RSA public key or key pair.
+    """
+
+    _only_public = False
+    _initialized = True
+
+    def __init__(self):
+        pkey = _lib.EVP_PKEY_new()
+        self._pkey = _ffi.gc(pkey, _lib.EVP_PKEY_free)
+        self._initialized = False
+
+    def to_cryptography_key(self):
+        """
+        Export as a ``cryptography`` key.
+
+        :rtype: One of ``cryptography``'s `key interfaces`_.
+
+        .. _key interfaces: https://cryptography.io/en/latest/hazmat/\
+            primitives/asymmetric/rsa/#key-interfaces
+
+        .. versionadded:: 16.1.0
+        """
+        from cryptography.hazmat.primitives.serialization import (
+            load_der_private_key,
+            load_der_public_key,
+        )
+
+        backend = _get_backend()
+        if self._only_public:
+            der = dump_publickey(FILETYPE_ASN1, self)
+            return load_der_public_key(der, backend)
+        else:
+            der = dump_privatekey(FILETYPE_ASN1, self)
+            return load_der_private_key(der, None, backend)
+
+    @classmethod
+    def from_cryptography_key(cls, crypto_key):
+        """
+        Construct based on a ``cryptography`` *crypto_key*.
+
+        :param crypto_key: A ``cryptography`` key.
+        :type crypto_key: One of ``cryptography``'s `key interfaces`_.
+
+        :rtype: PKey
+
+        .. versionadded:: 16.1.0
+        """
+        if not isinstance(
+            crypto_key,
+            (
+                rsa.RSAPublicKey,
+                rsa.RSAPrivateKey,
+                dsa.DSAPublicKey,
+                dsa.DSAPrivateKey,
+            ),
+        ):
+            raise TypeError("Unsupported key type")
+
+        from cryptography.hazmat.primitives.serialization import (
+            Encoding,
+            NoEncryption,
+            PrivateFormat,
+            PublicFormat,
+        )
+
+        if isinstance(crypto_key, (rsa.RSAPublicKey, dsa.DSAPublicKey)):
+            return load_publickey(
+                FILETYPE_ASN1,
+                crypto_key.public_bytes(
+                    Encoding.DER, PublicFormat.SubjectPublicKeyInfo
+                ),
+            )
+        else:
+            der = crypto_key.private_bytes(
+                Encoding.DER, PrivateFormat.PKCS8, NoEncryption()
+            )
+            return load_privatekey(FILETYPE_ASN1, der)
+
+    def generate_key(self, type, bits):
+        """
+        Generate a key pair of the given type, with the given number of bits.
+
+        This generates a key "into" the this object.
+
+        :param type: The key type.
+        :type type: :py:data:`TYPE_RSA` or :py:data:`TYPE_DSA`
+        :param bits: The number of bits.
+        :type bits: :py:data:`int` ``>= 0``
+        :raises TypeError: If :py:data:`type` or :py:data:`bits` isn't
+            of the appropriate type.
+        :raises ValueError: If the number of bits isn't an integer of
+            the appropriate size.
+        :return: ``None``
+        """
+        if not isinstance(type, int):
+            raise TypeError("type must be an integer")
+
+        if not isinstance(bits, int):
+            raise TypeError("bits must be an integer")
+
+        if type == TYPE_RSA:
+            if bits <= 0:
+                raise ValueError("Invalid number of bits")
+
+            # TODO Check error return
+            exponent = _lib.BN_new()
+            exponent = _ffi.gc(exponent, _lib.BN_free)
+            _lib.BN_set_word(exponent, _lib.RSA_F4)
+
+            rsa = _lib.RSA_new()
+
+            result = _lib.RSA_generate_key_ex(rsa, bits, exponent, _ffi.NULL)
+            _openssl_assert(result == 1)
+
+            result = _lib.EVP_PKEY_assign_RSA(self._pkey, rsa)
+            _openssl_assert(result == 1)
+
+        elif type == TYPE_DSA:
+            dsa = _lib.DSA_new()
+            _openssl_assert(dsa != _ffi.NULL)
+
+            dsa = _ffi.gc(dsa, _lib.DSA_free)
+            res = _lib.DSA_generate_parameters_ex(
+                dsa, bits, _ffi.NULL, 0, _ffi.NULL, _ffi.NULL, _ffi.NULL
+            )
+            _openssl_assert(res == 1)
+
+            _openssl_assert(_lib.DSA_generate_key(dsa) == 1)
+            _openssl_assert(_lib.EVP_PKEY_set1_DSA(self._pkey, dsa) == 1)
+        else:
+            raise Error("No such key type")
+
+        self._initialized = True
+
+    def check(self):
+        """
+        Check the consistency of an RSA private key.
+
+        This is the Python equivalent of OpenSSL's ``RSA_check_key``.
+
+        :return: ``True`` if key is consistent.
+
+        :raise OpenSSL.crypto.Error: if the key is inconsistent.
+
+        :raise TypeError: if the key is of a type which cannot be checked.
+            Only RSA keys can currently be checked.
+        """
+        if self._only_public:
+            raise TypeError("public key only")
+
+        if _lib.EVP_PKEY_type(self.type()) != _lib.EVP_PKEY_RSA:
+            raise TypeError("key type unsupported")
+
+        rsa = _lib.EVP_PKEY_get1_RSA(self._pkey)
+        rsa = _ffi.gc(rsa, _lib.RSA_free)
+        result = _lib.RSA_check_key(rsa)
+        if result == 1:
+            return True
+        _raise_current_error()
+
+    def type(self):
+        """
+        Returns the type of the key
+
+        :return: The type of the key.
+        """
+        return _lib.EVP_PKEY_id(self._pkey)
+
+    def bits(self):
+        """
+        Returns the number of bits of the key
+
+        :return: The number of bits of the key.
+        """
+        return _lib.EVP_PKEY_bits(self._pkey)
+
+
+class _EllipticCurve(object):
+    """
+    A representation of a supported elliptic curve.
+
+    @cvar _curves: :py:obj:`None` until an attempt is made to load the curves.
+        Thereafter, a :py:type:`set` containing :py:type:`_EllipticCurve`
+        instances each of which represents one curve supported by the system.
+    @type _curves: :py:type:`NoneType` or :py:type:`set`
+    """
+
+    _curves = None
+
+    if not _PY2:
+        # This only necessary on Python 3.  Moreover, it is broken on Python 2.
+        def __ne__(self, other):
+            """
+            Implement cooperation with the right-hand side argument of ``!=``.
+
+            Python 3 seems to have dropped this cooperation in this very narrow
+            circumstance.
+            """
+            if isinstance(other, _EllipticCurve):
+                return super(_EllipticCurve, self).__ne__(other)
+            return NotImplemented
+
+    @classmethod
+    def _load_elliptic_curves(cls, lib):
+        """
+        Get the curves supported by OpenSSL.
+
+        :param lib: The OpenSSL library binding object.
+
+        :return: A :py:type:`set` of ``cls`` instances giving the names of the
+            elliptic curves the underlying library supports.
+        """
+        num_curves = lib.EC_get_builtin_curves(_ffi.NULL, 0)
+        builtin_curves = _ffi.new("EC_builtin_curve[]", num_curves)
+        # The return value on this call should be num_curves again.  We
+        # could check it to make sure but if it *isn't* then.. what could
+        # we do? Abort the whole process, I suppose...?  -exarkun
+        lib.EC_get_builtin_curves(builtin_curves, num_curves)
+        return set(cls.from_nid(lib, c.nid) for c in builtin_curves)
+
+    @classmethod
+    def _get_elliptic_curves(cls, lib):
+        """
+        Get, cache, and return the curves supported by OpenSSL.
+
+        :param lib: The OpenSSL library binding object.
+
+        :return: A :py:type:`set` of ``cls`` instances giving the names of the
+            elliptic curves the underlying library supports.
+        """
+        if cls._curves is None:
+            cls._curves = cls._load_elliptic_curves(lib)
+        return cls._curves
+
+    @classmethod
+    def from_nid(cls, lib, nid):
+        """
+        Instantiate a new :py:class:`_EllipticCurve` associated with the given
+        OpenSSL NID.
+
+        :param lib: The OpenSSL library binding object.
+
+        :param nid: The OpenSSL NID the resulting curve object will represent.
+            This must be a curve NID (and not, for example, a hash NID) or
+            subsequent operations will fail in unpredictable ways.
+        :type nid: :py:class:`int`
+
+        :return: The curve object.
+        """
+        return cls(lib, nid, _ffi.string(lib.OBJ_nid2sn(nid)).decode("ascii"))
+
+    def __init__(self, lib, nid, name):
+        """
+        :param _lib: The :py:mod:`cryptography` binding instance used to
+            interface with OpenSSL.
+
+        :param _nid: The OpenSSL NID identifying the curve this object
+            represents.
+        :type _nid: :py:class:`int`
+
+        :param name: The OpenSSL short name identifying the curve this object
+            represents.
+        :type name: :py:class:`unicode`
+        """
+        self._lib = lib
+        self._nid = nid
+        self.name = name
+
+    def __repr__(self):
+        return "<Curve %r>" % (self.name,)
+
+    def _to_EC_KEY(self):
+        """
+        Create a new OpenSSL EC_KEY structure initialized to use this curve.
+
+        The structure is automatically garbage collected when the Python object
+        is garbage collected.
+        """
+        key = self._lib.EC_KEY_new_by_curve_name(self._nid)
+        return _ffi.gc(key, _lib.EC_KEY_free)
+
+
+def get_elliptic_curves():
+    """
+    Return a set of objects representing the elliptic curves supported in the
+    OpenSSL build in use.
+
+    The curve objects have a :py:class:`unicode` ``name`` attribute by which
+    they identify themselves.
+
+    The curve objects are useful as values for the argument accepted by
+    :py:meth:`Context.set_tmp_ecdh` to specify which elliptical curve should be
+    used for ECDHE key exchange.
+    """
+    return _EllipticCurve._get_elliptic_curves(_lib)
+
+
+def get_elliptic_curve(name):
+    """
+    Return a single curve object selected by name.
+
+    See :py:func:`get_elliptic_curves` for information about curve objects.
+
+    :param name: The OpenSSL short name identifying the curve object to
+        retrieve.
+    :type name: :py:class:`unicode`
+
+    If the named curve is not supported then :py:class:`ValueError` is raised.
+    """
+    for curve in get_elliptic_curves():
+        if curve.name == name:
+            return curve
+    raise ValueError("unknown curve name", name)
+
+
+class X509Name(object):
+    """
+    An X.509 Distinguished Name.
+
+    :ivar countryName: The country of the entity.
+    :ivar C: Alias for  :py:attr:`countryName`.
+
+    :ivar stateOrProvinceName: The state or province of the entity.
+    :ivar ST: Alias for :py:attr:`stateOrProvinceName`.
+
+    :ivar localityName: The locality of the entity.
+    :ivar L: Alias for :py:attr:`localityName`.
+
+    :ivar organizationName: The organization name of the entity.
+    :ivar O: Alias for :py:attr:`organizationName`.
+
+    :ivar organizationalUnitName: The organizational unit of the entity.
+    :ivar OU: Alias for :py:attr:`organizationalUnitName`
+
+    :ivar commonName: The common name of the entity.
+    :ivar CN: Alias for :py:attr:`commonName`.
+
+    :ivar emailAddress: The e-mail address of the entity.
+    """
+
+    def __init__(self, name):
+        """
+        Create a new X509Name, copying the given X509Name instance.
+
+        :param name: The name to copy.
+        :type name: :py:class:`X509Name`
+        """
+        name = _lib.X509_NAME_dup(name._name)
+        self._name = _ffi.gc(name, _lib.X509_NAME_free)
+
+    def __setattr__(self, name, value):
+        if name.startswith("_"):
+            return super(X509Name, self).__setattr__(name, value)
+
+        # Note: we really do not want str subclasses here, so we do not use
+        # isinstance.
+        if type(name) is not str:
+            raise TypeError(
+                "attribute name must be string, not '%.200s'"
+                % (type(value).__name__,)
+            )
+
+        nid = _lib.OBJ_txt2nid(_byte_string(name))
+        if nid == _lib.NID_undef:
+            try:
+                _raise_current_error()
+            except Error:
+                pass
+            raise AttributeError("No such attribute")
+
+        # If there's an old entry for this NID, remove it
+        for i in range(_lib.X509_NAME_entry_count(self._name)):
+            ent = _lib.X509_NAME_get_entry(self._name, i)
+            ent_obj = _lib.X509_NAME_ENTRY_get_object(ent)
+            ent_nid = _lib.OBJ_obj2nid(ent_obj)
+            if nid == ent_nid:
+                ent = _lib.X509_NAME_delete_entry(self._name, i)
+                _lib.X509_NAME_ENTRY_free(ent)
+                break
+
+        if isinstance(value, _text_type):
+            value = value.encode("utf-8")
+
+        add_result = _lib.X509_NAME_add_entry_by_NID(
+            self._name, nid, _lib.MBSTRING_UTF8, value, -1, -1, 0
+        )
+        if not add_result:
+            _raise_current_error()
+
+    def __getattr__(self, name):
+        """
+        Find attribute. An X509Name object has the following attributes:
+        countryName (alias C), stateOrProvince (alias ST), locality (alias L),
+        organization (alias O), organizationalUnit (alias OU), commonName
+        (alias CN) and more...
+        """
+        nid = _lib.OBJ_txt2nid(_byte_string(name))
+        if nid == _lib.NID_undef:
+            # This is a bit weird.  OBJ_txt2nid indicated failure, but it seems
+            # a lower level function, a2d_ASN1_OBJECT, also feels the need to
+            # push something onto the error queue.  If we don't clean that up
+            # now, someone else will bump into it later and be quite confused.
+            # See lp#314814.
+            try:
+                _raise_current_error()
+            except Error:
+                pass
+            return super(X509Name, self).__getattr__(name)
+
+        entry_index = _lib.X509_NAME_get_index_by_NID(self._name, nid, -1)
+        if entry_index == -1:
+            return None
+
+        entry = _lib.X509_NAME_get_entry(self._name, entry_index)
+        data = _lib.X509_NAME_ENTRY_get_data(entry)
+
+        result_buffer = _ffi.new("unsigned char**")
+        data_length = _lib.ASN1_STRING_to_UTF8(result_buffer, data)
+        _openssl_assert(data_length >= 0)
+
+        try:
+            result = _ffi.buffer(result_buffer[0], data_length)[:].decode(
+                "utf-8"
+            )
+        finally:
+            # XXX untested
+            _lib.OPENSSL_free(result_buffer[0])
+        return result
+
+    def _cmp(op):
+        def f(self, other):
+            if not isinstance(other, X509Name):
+                return NotImplemented
+            result = _lib.X509_NAME_cmp(self._name, other._name)
+            return op(result, 0)
+
+        return f
+
+    __eq__ = _cmp(__eq__)
+    __ne__ = _cmp(__ne__)
+
+    __lt__ = _cmp(__lt__)
+    __le__ = _cmp(__le__)
+
+    __gt__ = _cmp(__gt__)
+    __ge__ = _cmp(__ge__)
+
+    def __repr__(self):
+        """
+        String representation of an X509Name
+        """
+        result_buffer = _ffi.new("char[]", 512)
+        format_result = _lib.X509_NAME_oneline(
+            self._name, result_buffer, len(result_buffer)
+        )
+        _openssl_assert(format_result != _ffi.NULL)
+
+        return "<X509Name object '%s'>" % (
+            _native(_ffi.string(result_buffer)),
+        )
+
+    def hash(self):
+        """
+        Return an integer representation of the first four bytes of the
+        MD5 digest of the DER representation of the name.
+
+        This is the Python equivalent of OpenSSL's ``X509_NAME_hash``.
+
+        :return: The (integer) hash of this name.
+        :rtype: :py:class:`int`
+        """
+        return _lib.X509_NAME_hash(self._name)
+
+    def der(self):
+        """
+        Return the DER encoding of this name.
+
+        :return: The DER encoded form of this name.
+        :rtype: :py:class:`bytes`
+        """
+        result_buffer = _ffi.new("unsigned char**")
+        encode_result = _lib.i2d_X509_NAME(self._name, result_buffer)
+        _openssl_assert(encode_result >= 0)
+
+        string_result = _ffi.buffer(result_buffer[0], encode_result)[:]
+        _lib.OPENSSL_free(result_buffer[0])
+        return string_result
+
+    def get_components(self):
+        """
+        Returns the components of this name, as a sequence of 2-tuples.
+
+        :return: The components of this name.
+        :rtype: :py:class:`list` of ``name, value`` tuples.
+        """
+        result = []
+        for i in range(_lib.X509_NAME_entry_count(self._name)):
+            ent = _lib.X509_NAME_get_entry(self._name, i)
+
+            fname = _lib.X509_NAME_ENTRY_get_object(ent)
+            fval = _lib.X509_NAME_ENTRY_get_data(ent)
+
+            nid = _lib.OBJ_obj2nid(fname)
+            name = _lib.OBJ_nid2sn(nid)
+
+            # ffi.string does not handle strings containing NULL bytes
+            # (which may have been generated by old, broken software)
+            value = _ffi.buffer(
+                _lib.ASN1_STRING_data(fval), _lib.ASN1_STRING_length(fval)
+            )[:]
+            result.append((_ffi.string(name), value))
+
+        return result
+
+
+class X509Extension(object):
+    """
+    An X.509 v3 certificate extension.
+    """
+
+    def __init__(self, type_name, critical, value, subject=None, issuer=None):
+        """
+        Initializes an X509 extension.
+
+        :param type_name: The name of the type of extension_ to create.
+        :type type_name: :py:data:`bytes`
+
+        :param bool critical: A flag indicating whether this is a critical
+            extension.
+
+        :param value: The value of the extension.
+        :type value: :py:data:`bytes`
+
+        :param subject: Optional X509 certificate to use as subject.
+        :type subject: :py:class:`X509`
+
+        :param issuer: Optional X509 certificate to use as issuer.
+        :type issuer: :py:class:`X509`
+
+        .. _extension: https://www.openssl.org/docs/manmaster/man5/
+            x509v3_config.html#STANDARD-EXTENSIONS
+        """
+        ctx = _ffi.new("X509V3_CTX*")
+
+        # A context is necessary for any extension which uses the r2i
+        # conversion method.  That is, X509V3_EXT_nconf may segfault if passed
+        # a NULL ctx. Start off by initializing most of the fields to NULL.
+        _lib.X509V3_set_ctx(ctx, _ffi.NULL, _ffi.NULL, _ffi.NULL, _ffi.NULL, 0)
+
+        # We have no configuration database - but perhaps we should (some
+        # extensions may require it).
+        _lib.X509V3_set_ctx_nodb(ctx)
+
+        # Initialize the subject and issuer, if appropriate.  ctx is a local,
+        # and as far as I can tell none of the X509V3_* APIs invoked here steal
+        # any references, so no need to mess with reference counts or
+        # duplicates.
+        if issuer is not None:
+            if not isinstance(issuer, X509):
+                raise TypeError("issuer must be an X509 instance")
+            ctx.issuer_cert = issuer._x509
+        if subject is not None:
+            if not isinstance(subject, X509):
+                raise TypeError("subject must be an X509 instance")
+            ctx.subject_cert = subject._x509
+
+        if critical:
+            # There are other OpenSSL APIs which would let us pass in critical
+            # separately, but they're harder to use, and since value is already
+            # a pile of crappy junk smuggling a ton of utterly important
+            # structured data, what's the point of trying to avoid nasty stuff
+            # with strings? (However, X509V3_EXT_i2d in particular seems like
+            # it would be a better API to invoke.  I do not know where to get
+            # the ext_struc it desires for its last parameter, though.)
+            value = b"critical," + value
+
+        extension = _lib.X509V3_EXT_nconf(_ffi.NULL, ctx, type_name, value)
+        if extension == _ffi.NULL:
+            _raise_current_error()
+        self._extension = _ffi.gc(extension, _lib.X509_EXTENSION_free)
+
+    @property
+    def _nid(self):
+        return _lib.OBJ_obj2nid(
+            _lib.X509_EXTENSION_get_object(self._extension)
+        )
+
+    _prefixes = {
+        _lib.GEN_EMAIL: "email",
+        _lib.GEN_DNS: "DNS",
+        _lib.GEN_URI: "URI",
+    }
+
+    def _subjectAltNameString(self):
+        names = _ffi.cast(
+            "GENERAL_NAMES*", _lib.X509V3_EXT_d2i(self._extension)
+        )
+
+        names = _ffi.gc(names, _lib.GENERAL_NAMES_free)
+        parts = []
+        for i in range(_lib.sk_GENERAL_NAME_num(names)):
+            name = _lib.sk_GENERAL_NAME_value(names, i)
+            try:
+                label = self._prefixes[name.type]
+            except KeyError:
+                bio = _new_mem_buf()
+                _lib.GENERAL_NAME_print(bio, name)
+                parts.append(_native(_bio_to_string(bio)))
+            else:
+                value = _native(
+                    _ffi.buffer(name.d.ia5.data, name.d.ia5.length)[:]
+                )
+                parts.append(label + ":" + value)
+        return ", ".join(parts)
+
+    def __str__(self):
+        """
+        :return: a nice text representation of the extension
+        """
+        if _lib.NID_subject_alt_name == self._nid:
+            return self._subjectAltNameString()
+
+        bio = _new_mem_buf()
+        print_result = _lib.X509V3_EXT_print(bio, self._extension, 0, 0)
+        _openssl_assert(print_result != 0)
+
+        return _native(_bio_to_string(bio))
+
+    def get_critical(self):
+        """
+        Returns the critical field of this X.509 extension.
+
+        :return: The critical field.
+        """
+        return _lib.X509_EXTENSION_get_critical(self._extension)
+
+    def get_short_name(self):
+        """
+        Returns the short type name of this X.509 extension.
+
+        The result is a byte string such as :py:const:`b"basicConstraints"`.
+
+        :return: The short type name.
+        :rtype: :py:data:`bytes`
+
+        .. versionadded:: 0.12
+        """
+        obj = _lib.X509_EXTENSION_get_object(self._extension)
+        nid = _lib.OBJ_obj2nid(obj)
+        return _ffi.string(_lib.OBJ_nid2sn(nid))
+
+    def get_data(self):
+        """
+        Returns the data of the X509 extension, encoded as ASN.1.
+
+        :return: The ASN.1 encoded data of this X509 extension.
+        :rtype: :py:data:`bytes`
+
+        .. versionadded:: 0.12
+        """
+        octet_result = _lib.X509_EXTENSION_get_data(self._extension)
+        string_result = _ffi.cast("ASN1_STRING*", octet_result)
+        char_result = _lib.ASN1_STRING_data(string_result)
+        result_length = _lib.ASN1_STRING_length(string_result)
+        return _ffi.buffer(char_result, result_length)[:]
+
+
+class X509Req(object):
+    """
+    An X.509 certificate signing requests.
+    """
+
+    def __init__(self):
+        req = _lib.X509_REQ_new()
+        self._req = _ffi.gc(req, _lib.X509_REQ_free)
+        # Default to version 0.
+        self.set_version(0)
+
+    def to_cryptography(self):
+        """
+        Export as a ``cryptography`` certificate signing request.
+
+        :rtype: ``cryptography.x509.CertificateSigningRequest``
+
+        .. versionadded:: 17.1.0
+        """
+        from cryptography.x509 import load_der_x509_csr
+
+        der = dump_certificate_request(FILETYPE_ASN1, self)
+
+        backend = _get_backend()
+        return load_der_x509_csr(der, backend)
+
+    @classmethod
+    def from_cryptography(cls, crypto_req):
+        """
+        Construct based on a ``cryptography`` *crypto_req*.
+
+        :param crypto_req: A ``cryptography`` X.509 certificate signing request
+        :type crypto_req: ``cryptography.x509.CertificateSigningRequest``
+
+        :rtype: X509Req
+
+        .. versionadded:: 17.1.0
+        """
+        if not isinstance(crypto_req, x509.CertificateSigningRequest):
+            raise TypeError("Must be a certificate signing request")
+
+        from cryptography.hazmat.primitives.serialization import Encoding
+
+        der = crypto_req.public_bytes(Encoding.DER)
+        return load_certificate_request(FILETYPE_ASN1, der)
+
+    def set_pubkey(self, pkey):
+        """
+        Set the public key of the certificate signing request.
+
+        :param pkey: The public key to use.
+        :type pkey: :py:class:`PKey`
+
+        :return: ``None``
+        """
+        set_result = _lib.X509_REQ_set_pubkey(self._req, pkey._pkey)
+        _openssl_assert(set_result == 1)
+
+    def get_pubkey(self):
+        """
+        Get the public key of the certificate signing request.
+
+        :return: The public key.
+        :rtype: :py:class:`PKey`
+        """
+        pkey = PKey.__new__(PKey)
+        pkey._pkey = _lib.X509_REQ_get_pubkey(self._req)
+        _openssl_assert(pkey._pkey != _ffi.NULL)
+        pkey._pkey = _ffi.gc(pkey._pkey, _lib.EVP_PKEY_free)
+        pkey._only_public = True
+        return pkey
+
+    def set_version(self, version):
+        """
+        Set the version subfield (RFC 2459, section 4.1.2.1) of the certificate
+        request.
+
+        :param int version: The version number.
+        :return: ``None``
+        """
+        set_result = _lib.X509_REQ_set_version(self._req, version)
+        _openssl_assert(set_result == 1)
+
+    def get_version(self):
+        """
+        Get the version subfield (RFC 2459, section 4.1.2.1) of the certificate
+        request.
+
+        :return: The value of the version subfield.
+        :rtype: :py:class:`int`
+        """
+        return _lib.X509_REQ_get_version(self._req)
+
+    def get_subject(self):
+        """
+        Return the subject of this certificate signing request.
+
+        This creates a new :class:`X509Name` that wraps the underlying subject
+        name field on the certificate signing request. Modifying it will modify
+        the underlying signing request, and will have the effect of modifying
+        any other :class:`X509Name` that refers to this subject.
+
+        :return: The subject of this certificate signing request.
+        :rtype: :class:`X509Name`
+        """
+        name = X509Name.__new__(X509Name)
+        name._name = _lib.X509_REQ_get_subject_name(self._req)
+        _openssl_assert(name._name != _ffi.NULL)
+
+        # The name is owned by the X509Req structure.  As long as the X509Name
+        # Python object is alive, keep the X509Req Python object alive.
+        name._owner = self
+
+        return name
+
+    def add_extensions(self, extensions):
+        """
+        Add extensions to the certificate signing request.
+
+        :param extensions: The X.509 extensions to add.
+        :type extensions: iterable of :py:class:`X509Extension`
+        :return: ``None``
+        """
+        stack = _lib.sk_X509_EXTENSION_new_null()
+        _openssl_assert(stack != _ffi.NULL)
+
+        stack = _ffi.gc(stack, _lib.sk_X509_EXTENSION_free)
+
+        for ext in extensions:
+            if not isinstance(ext, X509Extension):
+                raise ValueError("One of the elements is not an X509Extension")
+
+            # TODO push can fail (here and elsewhere)
+            _lib.sk_X509_EXTENSION_push(stack, ext._extension)
+
+        add_result = _lib.X509_REQ_add_extensions(self._req, stack)
+        _openssl_assert(add_result == 1)
+
+    def get_extensions(self):
+        """
+        Get X.509 extensions in the certificate signing request.
+
+        :return: The X.509 extensions in this request.
+        :rtype: :py:class:`list` of :py:class:`X509Extension` objects.
+
+        .. versionadded:: 0.15
+        """
+        exts = []
+        native_exts_obj = _lib.X509_REQ_get_extensions(self._req)
+        native_exts_obj = _ffi.gc(
+            native_exts_obj,
+            lambda x: _lib.sk_X509_EXTENSION_pop_free(
+                x,
+                _ffi.addressof(_lib._original_lib, "X509_EXTENSION_free"),
+            ),
+        )
+
+        for i in range(_lib.sk_X509_EXTENSION_num(native_exts_obj)):
+            ext = X509Extension.__new__(X509Extension)
+            extension = _lib.X509_EXTENSION_dup(
+                _lib.sk_X509_EXTENSION_value(native_exts_obj, i)
+            )
+            ext._extension = _ffi.gc(extension, _lib.X509_EXTENSION_free)
+            exts.append(ext)
+        return exts
+
+    def sign(self, pkey, digest):
+        """
+        Sign the certificate signing request with this key and digest type.
+
+        :param pkey: The key pair to sign with.
+        :type pkey: :py:class:`PKey`
+        :param digest: The name of the message digest to use for the signature,
+            e.g. :py:data:`b"sha256"`.
+        :type digest: :py:class:`bytes`
+        :return: ``None``
+        """
+        if pkey._only_public:
+            raise ValueError("Key has only public part")
+
+        if not pkey._initialized:
+            raise ValueError("Key is uninitialized")
+
+        digest_obj = _lib.EVP_get_digestbyname(_byte_string(digest))
+        if digest_obj == _ffi.NULL:
+            raise ValueError("No such digest method")
+
+        sign_result = _lib.X509_REQ_sign(self._req, pkey._pkey, digest_obj)
+        _openssl_assert(sign_result > 0)
+
+    def verify(self, pkey):
+        """
+        Verifies the signature on this certificate signing request.
+
+        :param PKey key: A public key.
+
+        :return: ``True`` if the signature is correct.
+        :rtype: bool
+
+        :raises OpenSSL.crypto.Error: If the signature is invalid or there is a
+            problem verifying the signature.
+        """
+        if not isinstance(pkey, PKey):
+            raise TypeError("pkey must be a PKey instance")
+
+        result = _lib.X509_REQ_verify(self._req, pkey._pkey)
+        if result <= 0:
+            _raise_current_error()
+
+        return result
+
+
+class X509(object):
+    """
+    An X.509 certificate.
+    """
+
+    def __init__(self):
+        x509 = _lib.X509_new()
+        _openssl_assert(x509 != _ffi.NULL)
+        self._x509 = _ffi.gc(x509, _lib.X509_free)
+
+        self._issuer_invalidator = _X509NameInvalidator()
+        self._subject_invalidator = _X509NameInvalidator()
+
+    @classmethod
+    def _from_raw_x509_ptr(cls, x509):
+        cert = cls.__new__(cls)
+        cert._x509 = _ffi.gc(x509, _lib.X509_free)
+        cert._issuer_invalidator = _X509NameInvalidator()
+        cert._subject_invalidator = _X509NameInvalidator()
+        return cert
+
+    def to_cryptography(self):
+        """
+        Export as a ``cryptography`` certificate.
+
+        :rtype: ``cryptography.x509.Certificate``
+
+        .. versionadded:: 17.1.0
+        """
+        from cryptography.x509 import load_der_x509_certificate
+
+        der = dump_certificate(FILETYPE_ASN1, self)
+        backend = _get_backend()
+        return load_der_x509_certificate(der, backend)
+
+    @classmethod
+    def from_cryptography(cls, crypto_cert):
+        """
+        Construct based on a ``cryptography`` *crypto_cert*.
+
+        :param crypto_key: A ``cryptography`` X.509 certificate.
+        :type crypto_key: ``cryptography.x509.Certificate``
+
+        :rtype: X509
+
+        .. versionadded:: 17.1.0
+        """
+        if not isinstance(crypto_cert, x509.Certificate):
+            raise TypeError("Must be a certificate")
+
+        from cryptography.hazmat.primitives.serialization import Encoding
+
+        der = crypto_cert.public_bytes(Encoding.DER)
+        return load_certificate(FILETYPE_ASN1, der)
+
+    def set_version(self, version):
+        """
+        Set the version number of the certificate. Note that the
+        version value is zero-based, eg. a value of 0 is V1.
+
+        :param version: The version number of the certificate.
+        :type version: :py:class:`int`
+
+        :return: ``None``
+        """
+        if not isinstance(version, int):
+            raise TypeError("version must be an integer")
+
+        _lib.X509_set_version(self._x509, version)
+
+    def get_version(self):
+        """
+        Return the version number of the certificate.
+
+        :return: The version number of the certificate.
+        :rtype: :py:class:`int`
+        """
+        return _lib.X509_get_version(self._x509)
+
+    def get_pubkey(self):
+        """
+        Get the public key of the certificate.
+
+        :return: The public key.
+        :rtype: :py:class:`PKey`
+        """
+        pkey = PKey.__new__(PKey)
+        pkey._pkey = _lib.X509_get_pubkey(self._x509)
+        if pkey._pkey == _ffi.NULL:
+            _raise_current_error()
+        pkey._pkey = _ffi.gc(pkey._pkey, _lib.EVP_PKEY_free)
+        pkey._only_public = True
+        return pkey
+
+    def set_pubkey(self, pkey):
+        """
+        Set the public key of the certificate.
+
+        :param pkey: The public key.
+        :type pkey: :py:class:`PKey`
+
+        :return: :py:data:`None`
+        """
+        if not isinstance(pkey, PKey):
+            raise TypeError("pkey must be a PKey instance")
+
+        set_result = _lib.X509_set_pubkey(self._x509, pkey._pkey)
+        _openssl_assert(set_result == 1)
+
+    def sign(self, pkey, digest):
+        """
+        Sign the certificate with this key and digest type.
+
+        :param pkey: The key to sign with.
+        :type pkey: :py:class:`PKey`
+
+        :param digest: The name of the message digest to use.
+        :type digest: :py:class:`bytes`
+
+        :return: :py:data:`None`
+        """
+        if not isinstance(pkey, PKey):
+            raise TypeError("pkey must be a PKey instance")
+
+        if pkey._only_public:
+            raise ValueError("Key only has public part")
+
+        if not pkey._initialized:
+            raise ValueError("Key is uninitialized")
+
+        evp_md = _lib.EVP_get_digestbyname(_byte_string(digest))
+        if evp_md == _ffi.NULL:
+            raise ValueError("No such digest method")
+
+        sign_result = _lib.X509_sign(self._x509, pkey._pkey, evp_md)
+        _openssl_assert(sign_result > 0)
+
+    def get_signature_algorithm(self):
+        """
+        Return the signature algorithm used in the certificate.
+
+        :return: The name of the algorithm.
+        :rtype: :py:class:`bytes`
+
+        :raises ValueError: If the signature algorithm is undefined.
+
+        .. versionadded:: 0.13
+        """
+        algor = _lib.X509_get0_tbs_sigalg(self._x509)
+        nid = _lib.OBJ_obj2nid(algor.algorithm)
+        if nid == _lib.NID_undef:
+            raise ValueError("Undefined signature algorithm")
+        return _ffi.string(_lib.OBJ_nid2ln(nid))
+
+    def digest(self, digest_name):
+        """
+        Return the digest of the X509 object.
+
+        :param digest_name: The name of the digest algorithm to use.
+        :type digest_name: :py:class:`bytes`
+
+        :return: The digest of the object, formatted as
+            :py:const:`b":"`-delimited hex pairs.
+        :rtype: :py:class:`bytes`
+        """
+        digest = _lib.EVP_get_digestbyname(_byte_string(digest_name))
+        if digest == _ffi.NULL:
+            raise ValueError("No such digest method")
+
+        result_buffer = _ffi.new("unsigned char[]", _lib.EVP_MAX_MD_SIZE)
+        result_length = _ffi.new("unsigned int[]", 1)
+        result_length[0] = len(result_buffer)
+
+        digest_result = _lib.X509_digest(
+            self._x509, digest, result_buffer, result_length
+        )
+        _openssl_assert(digest_result == 1)
+
+        return b":".join(
+            [
+                b16encode(ch).upper()
+                for ch in _ffi.buffer(result_buffer, result_length[0])
+            ]
+        )
+
+    def subject_name_hash(self):
+        """
+        Return the hash of the X509 subject.
+
+        :return: The hash of the subject.
+        :rtype: :py:class:`bytes`
+        """
+        return _lib.X509_subject_name_hash(self._x509)
+
+    def set_serial_number(self, serial):
+        """
+        Set the serial number of the certificate.
+
+        :param serial: The new serial number.
+        :type serial: :py:class:`int`
+
+        :return: :py:data`None`
+        """
+        if not isinstance(serial, _integer_types):
+            raise TypeError("serial must be an integer")
+
+        hex_serial = hex(serial)[2:]
+        if not isinstance(hex_serial, bytes):
+            hex_serial = hex_serial.encode("ascii")
+
+        bignum_serial = _ffi.new("BIGNUM**")
+
+        # BN_hex2bn stores the result in &bignum.  Unless it doesn't feel like
+        # it.  If bignum is still NULL after this call, then the return value
+        # is actually the result.  I hope.  -exarkun
+        small_serial = _lib.BN_hex2bn(bignum_serial, hex_serial)
+
+        if bignum_serial[0] == _ffi.NULL:
+            set_result = _lib.ASN1_INTEGER_set(
+                _lib.X509_get_serialNumber(self._x509), small_serial
+            )
+            if set_result:
+                # TODO Not tested
+                _raise_current_error()
+        else:
+            asn1_serial = _lib.BN_to_ASN1_INTEGER(bignum_serial[0], _ffi.NULL)
+            _lib.BN_free(bignum_serial[0])
+            if asn1_serial == _ffi.NULL:
+                # TODO Not tested
+                _raise_current_error()
+            asn1_serial = _ffi.gc(asn1_serial, _lib.ASN1_INTEGER_free)
+            set_result = _lib.X509_set_serialNumber(self._x509, asn1_serial)
+            _openssl_assert(set_result == 1)
+
+    def get_serial_number(self):
+        """
+        Return the serial number of this certificate.
+
+        :return: The serial number.
+        :rtype: int
+        """
+        asn1_serial = _lib.X509_get_serialNumber(self._x509)
+        bignum_serial = _lib.ASN1_INTEGER_to_BN(asn1_serial, _ffi.NULL)
+        try:
+            hex_serial = _lib.BN_bn2hex(bignum_serial)
+            try:
+                hexstring_serial = _ffi.string(hex_serial)
+                serial = int(hexstring_serial, 16)
+                return serial
+            finally:
+                _lib.OPENSSL_free(hex_serial)
+        finally:
+            _lib.BN_free(bignum_serial)
+
+    def gmtime_adj_notAfter(self, amount):
+        """
+        Adjust the time stamp on which the certificate stops being valid.
+
+        :param int amount: The number of seconds by which to adjust the
+            timestamp.
+        :return: ``None``
+        """
+        if not isinstance(amount, int):
+            raise TypeError("amount must be an integer")
+
+        notAfter = _lib.X509_getm_notAfter(self._x509)
+        _lib.X509_gmtime_adj(notAfter, amount)
+
+    def gmtime_adj_notBefore(self, amount):
+        """
+        Adjust the timestamp on which the certificate starts being valid.
+
+        :param amount: The number of seconds by which to adjust the timestamp.
+        :return: ``None``
+        """
+        if not isinstance(amount, int):
+            raise TypeError("amount must be an integer")
+
+        notBefore = _lib.X509_getm_notBefore(self._x509)
+        _lib.X509_gmtime_adj(notBefore, amount)
+
+    def has_expired(self):
+        """
+        Check whether the certificate has expired.
+
+        :return: ``True`` if the certificate has expired, ``False`` otherwise.
+        :rtype: bool
+        """
+        time_string = _native(self.get_notAfter())
+        not_after = datetime.datetime.strptime(time_string, "%Y%m%d%H%M%SZ")
+
+        return not_after < datetime.datetime.utcnow()
+
+    def _get_boundary_time(self, which):
+        return _get_asn1_time(which(self._x509))
+
+    def get_notBefore(self):
+        """
+        Get the timestamp at which the certificate starts being valid.
+
+        The timestamp is formatted as an ASN.1 TIME::
+
+            YYYYMMDDhhmmssZ
+
+        :return: A timestamp string, or ``None`` if there is none.
+        :rtype: bytes or NoneType
+        """
+        return self._get_boundary_time(_lib.X509_getm_notBefore)
+
+    def _set_boundary_time(self, which, when):
+        return _set_asn1_time(which(self._x509), when)
+
+    def set_notBefore(self, when):
+        """
+        Set the timestamp at which the certificate starts being valid.
+
+        The timestamp is formatted as an ASN.1 TIME::
+
+            YYYYMMDDhhmmssZ
+
+        :param bytes when: A timestamp string.
+        :return: ``None``
+        """
+        return self._set_boundary_time(_lib.X509_getm_notBefore, when)
+
+    def get_notAfter(self):
+        """
+        Get the timestamp at which the certificate stops being valid.
+
+        The timestamp is formatted as an ASN.1 TIME::
+
+            YYYYMMDDhhmmssZ
+
+        :return: A timestamp string, or ``None`` if there is none.
+        :rtype: bytes or NoneType
+        """
+        return self._get_boundary_time(_lib.X509_getm_notAfter)
+
+    def set_notAfter(self, when):
+        """
+        Set the timestamp at which the certificate stops being valid.
+
+        The timestamp is formatted as an ASN.1 TIME::
+
+            YYYYMMDDhhmmssZ
+
+        :param bytes when: A timestamp string.
+        :return: ``None``
+        """
+        return self._set_boundary_time(_lib.X509_getm_notAfter, when)
+
+    def _get_name(self, which):
+        name = X509Name.__new__(X509Name)
+        name._name = which(self._x509)
+        _openssl_assert(name._name != _ffi.NULL)
+
+        # The name is owned by the X509 structure.  As long as the X509Name
+        # Python object is alive, keep the X509 Python object alive.
+        name._owner = self
+
+        return name
+
+    def _set_name(self, which, name):
+        if not isinstance(name, X509Name):
+            raise TypeError("name must be an X509Name")
+        set_result = which(self._x509, name._name)
+        _openssl_assert(set_result == 1)
+
+    def get_issuer(self):
+        """
+        Return the issuer of this certificate.
+
+        This creates a new :class:`X509Name` that wraps the underlying issuer
+        name field on the certificate. Modifying it will modify the underlying
+        certificate, and will have the effect of modifying any other
+        :class:`X509Name` that refers to this issuer.
+
+        :return: The issuer of this certificate.
+        :rtype: :class:`X509Name`
+        """
+        name = self._get_name(_lib.X509_get_issuer_name)
+        self._issuer_invalidator.add(name)
+        return name
+
+    def set_issuer(self, issuer):
+        """
+        Set the issuer of this certificate.
+
+        :param issuer: The issuer.
+        :type issuer: :py:class:`X509Name`
+
+        :return: ``None``
+        """
+        self._set_name(_lib.X509_set_issuer_name, issuer)
+        self._issuer_invalidator.clear()
+
+    def get_subject(self):
+        """
+        Return the subject of this certificate.
+
+        This creates a new :class:`X509Name` that wraps the underlying subject
+        name field on the certificate. Modifying it will modify the underlying
+        certificate, and will have the effect of modifying any other
+        :class:`X509Name` that refers to this subject.
+
+        :return: The subject of this certificate.
+        :rtype: :class:`X509Name`
+        """
+        name = self._get_name(_lib.X509_get_subject_name)
+        self._subject_invalidator.add(name)
+        return name
+
+    def set_subject(self, subject):
+        """
+        Set the subject of this certificate.
+
+        :param subject: The subject.
+        :type subject: :py:class:`X509Name`
+
+        :return: ``None``
+        """
+        self._set_name(_lib.X509_set_subject_name, subject)
+        self._subject_invalidator.clear()
+
+    def get_extension_count(self):
+        """
+        Get the number of extensions on this certificate.
+
+        :return: The number of extensions.
+        :rtype: :py:class:`int`
+
+        .. versionadded:: 0.12
+        """
+        return _lib.X509_get_ext_count(self._x509)
+
+    def add_extensions(self, extensions):
+        """
+        Add extensions to the certificate.
+
+        :param extensions: The extensions to add.
+        :type extensions: An iterable of :py:class:`X509Extension` objects.
+        :return: ``None``
+        """
+        for ext in extensions:
+            if not isinstance(ext, X509Extension):
+                raise ValueError("One of the elements is not an X509Extension")
+
+            add_result = _lib.X509_add_ext(self._x509, ext._extension, -1)
+            if not add_result:
+                _raise_current_error()
+
+    def get_extension(self, index):
+        """
+        Get a specific extension of the certificate by index.
+
+        Extensions on a certificate are kept in order. The index
+        parameter selects which extension will be returned.
+
+        :param int index: The index of the extension to retrieve.
+        :return: The extension at the specified index.
+        :rtype: :py:class:`X509Extension`
+        :raises IndexError: If the extension index was out of bounds.
+
+        .. versionadded:: 0.12
+        """
+        ext = X509Extension.__new__(X509Extension)
+        ext._extension = _lib.X509_get_ext(self._x509, index)
+        if ext._extension == _ffi.NULL:
+            raise IndexError("extension index out of bounds")
+
+        extension = _lib.X509_EXTENSION_dup(ext._extension)
+        ext._extension = _ffi.gc(extension, _lib.X509_EXTENSION_free)
+        return ext
+
+
+class X509StoreFlags(object):
+    """
+    Flags for X509 verification, used to change the behavior of
+    :class:`X509Store`.
+
+    See `OpenSSL Verification Flags`_ for details.
+
+    .. _OpenSSL Verification Flags:
+        https://www.openssl.org/docs/manmaster/man3/X509_VERIFY_PARAM_set_flags.html
+    """
+
+    CRL_CHECK = _lib.X509_V_FLAG_CRL_CHECK
+    CRL_CHECK_ALL = _lib.X509_V_FLAG_CRL_CHECK_ALL
+    IGNORE_CRITICAL = _lib.X509_V_FLAG_IGNORE_CRITICAL
+    X509_STRICT = _lib.X509_V_FLAG_X509_STRICT
+    ALLOW_PROXY_CERTS = _lib.X509_V_FLAG_ALLOW_PROXY_CERTS
+    POLICY_CHECK = _lib.X509_V_FLAG_POLICY_CHECK
+    EXPLICIT_POLICY = _lib.X509_V_FLAG_EXPLICIT_POLICY
+    INHIBIT_MAP = _lib.X509_V_FLAG_INHIBIT_MAP
+    NOTIFY_POLICY = _lib.X509_V_FLAG_NOTIFY_POLICY
+    CHECK_SS_SIGNATURE = _lib.X509_V_FLAG_CHECK_SS_SIGNATURE
+
+
+class X509Store(object):
+    """
+    An X.509 store.
+
+    An X.509 store is used to describe a context in which to verify a
+    certificate. A description of a context may include a set of certificates
+    to trust, a set of certificate revocation lists, verification flags and
+    more.
+
+    An X.509 store, being only a description, cannot be used by itself to
+    verify a certificate. To carry out the actual verification process, see
+    :class:`X509StoreContext`.
+    """
+
+    def __init__(self):
+        store = _lib.X509_STORE_new()
+        self._store = _ffi.gc(store, _lib.X509_STORE_free)
+
+    def add_cert(self, cert):
+        """
+        Adds a trusted certificate to this store.
+
+        Adding a certificate with this method adds this certificate as a
+        *trusted* certificate.
+
+        :param X509 cert: The certificate to add to this store.
+
+        :raises TypeError: If the certificate is not an :class:`X509`.
+
+        :raises OpenSSL.crypto.Error: If OpenSSL was unhappy with your
+            certificate.
+
+        :return: ``None`` if the certificate was added successfully.
+        """
+        if not isinstance(cert, X509):
+            raise TypeError()
+
+        res = _lib.X509_STORE_add_cert(self._store, cert._x509)
+        _openssl_assert(res == 1)
+
+    def add_crl(self, crl):
+        """
+        Add a certificate revocation list to this store.
+
+        The certificate revocation lists added to a store will only be used if
+        the associated flags are configured to check certificate revocation
+        lists.
+
+        .. versionadded:: 16.1.0
+
+        :param CRL crl: The certificate revocation list to add to this store.
+        :return: ``None`` if the certificate revocation list was added
+            successfully.
+        """
+        _openssl_assert(_lib.X509_STORE_add_crl(self._store, crl._crl) != 0)
+
+    def set_flags(self, flags):
+        """
+        Set verification flags to this store.
+
+        Verification flags can be combined by oring them together.
+
+        .. note::
+
+          Setting a verification flag sometimes requires clients to add
+          additional information to the store, otherwise a suitable error will
+          be raised.
+
+          For example, in setting flags to enable CRL checking a
+          suitable CRL must be added to the store otherwise an error will be
+          raised.
+
+        .. versionadded:: 16.1.0
+
+        :param int flags: The verification flags to set on this store.
+            See :class:`X509StoreFlags` for available constants.
+        :return: ``None`` if the verification flags were successfully set.
+        """
+        _openssl_assert(_lib.X509_STORE_set_flags(self._store, flags) != 0)
+
+    def set_time(self, vfy_time):
+        """
+        Set the time against which the certificates are verified.
+
+        Normally the current time is used.
+
+        .. note::
+
+          For example, you can determine if a certificate was valid at a given
+          time.
+
+        .. versionadded:: 17.0.0
+
+        :param datetime vfy_time: The verification time to set on this store.
+        :return: ``None`` if the verification time was successfully set.
+        """
+        param = _lib.X509_VERIFY_PARAM_new()
+        param = _ffi.gc(param, _lib.X509_VERIFY_PARAM_free)
+
+        _lib.X509_VERIFY_PARAM_set_time(
+            param, calendar.timegm(vfy_time.timetuple())
+        )
+        _openssl_assert(_lib.X509_STORE_set1_param(self._store, param) != 0)
+
+    def load_locations(self, cafile, capath=None):
+        """
+        Let X509Store know where we can find trusted certificates for the
+        certificate chain.  Note that the certificates have to be in PEM
+        format.
+
+        If *capath* is passed, it must be a directory prepared using the
+        ``c_rehash`` tool included with OpenSSL.  Either, but not both, of
+        *cafile* or *capath* may be ``None``.
+
+        .. note::
+
+          Both *cafile* and *capath* may be set simultaneously.
+
+          Call this method multiple times to add more than one location.
+          For example, CA certificates, and certificate revocation list bundles
+          may be passed in *cafile* in subsequent calls to this method.
+
+        .. versionadded:: 20.0
+
+        :param cafile: In which file we can find the certificates (``bytes`` or
+                       ``unicode``).
+        :param capath: In which directory we can find the certificates
+                       (``bytes`` or ``unicode``).
+
+        :return: ``None`` if the locations were set successfully.
+
+        :raises OpenSSL.crypto.Error: If both *cafile* and *capath* is ``None``
+            or the locations could not be set for any reason.
+
+        """
+        if cafile is None:
+            cafile = _ffi.NULL
+        else:
+            cafile = _path_string(cafile)
+
+        if capath is None:
+            capath = _ffi.NULL
+        else:
+            capath = _path_string(capath)
+
+        load_result = _lib.X509_STORE_load_locations(
+            self._store, cafile, capath
+        )
+        if not load_result:
+            _raise_current_error()
+
+
+class X509StoreContextError(Exception):
+    """
+    An exception raised when an error occurred while verifying a certificate
+    using `OpenSSL.X509StoreContext.verify_certificate`.
+
+    :ivar certificate: The certificate which caused verificate failure.
+    :type certificate: :class:`X509`
+    """
+
+    def __init__(self, message, certificate):
+        super(X509StoreContextError, self).__init__(message)
+        self.certificate = certificate
+
+
+class X509StoreContext(object):
+    """
+    An X.509 store context.
+
+    An X.509 store context is used to carry out the actual verification process
+    of a certificate in a described context. For describing such a context, see
+    :class:`X509Store`.
+
+    :ivar _store_ctx: The underlying X509_STORE_CTX structure used by this
+        instance.  It is dynamically allocated and automatically garbage
+        collected.
+    :ivar _store: See the ``store`` ``__init__`` parameter.
+    :ivar _cert: See the ``certificate`` ``__init__`` parameter.
+    :ivar _chain: See the ``chain`` ``__init__`` parameter.
+    :param X509Store store: The certificates which will be trusted for the
+        purposes of any verifications.
+    :param X509 certificate: The certificate to be verified.
+    :param chain: List of untrusted certificates that may be used for building
+        the certificate chain. May be ``None``.
+    :type chain: :class:`list` of :class:`X509`
+    """
+
+    def __init__(self, store, certificate, chain=None):
+        store_ctx = _lib.X509_STORE_CTX_new()
+        self._store_ctx = _ffi.gc(store_ctx, _lib.X509_STORE_CTX_free)
+        self._store = store
+        self._cert = certificate
+        self._chain = self._build_certificate_stack(chain)
+        # Make the store context available for use after instantiating this
+        # class by initializing it now. Per testing, subsequent calls to
+        # :meth:`_init` have no adverse affect.
+        self._init()
+
+    @staticmethod
+    def _build_certificate_stack(certificates):
+        def cleanup(s):
+            # Equivalent to sk_X509_pop_free, but we don't
+            # currently have a CFFI binding for that available
+            for i in range(_lib.sk_X509_num(s)):
+                x = _lib.sk_X509_value(s, i)
+                _lib.X509_free(x)
+            _lib.sk_X509_free(s)
+
+        if certificates is None or len(certificates) == 0:
+            return _ffi.NULL
+
+        stack = _lib.sk_X509_new_null()
+        _openssl_assert(stack != _ffi.NULL)
+        stack = _ffi.gc(stack, cleanup)
+
+        for cert in certificates:
+            if not isinstance(cert, X509):
+                raise TypeError("One of the elements is not an X509 instance")
+
+            _openssl_assert(_lib.X509_up_ref(cert._x509) > 0)
+            if _lib.sk_X509_push(stack, cert._x509) <= 0:
+                _lib.X509_free(cert._x509)
+                _raise_current_error()
+
+        return stack
+
+    def _init(self):
+        """
+        Set up the store context for a subsequent verification operation.
+
+        Calling this method more than once without first calling
+        :meth:`_cleanup` will leak memory.
+        """
+        ret = _lib.X509_STORE_CTX_init(
+            self._store_ctx, self._store._store, self._cert._x509, self._chain
+        )
+        if ret <= 0:
+            _raise_current_error()
+
+    def _cleanup(self):
+        """
+        Internally cleans up the store context.
+
+        The store context can then be reused with a new call to :meth:`_init`.
+        """
+        _lib.X509_STORE_CTX_cleanup(self._store_ctx)
+
+    def _exception_from_context(self):
+        """
+        Convert an OpenSSL native context error failure into a Python
+        exception.
+
+        When a call to native OpenSSL X509_verify_cert fails, additional
+        information about the failure can be obtained from the store context.
+        """
+        errors = [
+            _lib.X509_STORE_CTX_get_error(self._store_ctx),
+            _lib.X509_STORE_CTX_get_error_depth(self._store_ctx),
+            _native(
+                _ffi.string(
+                    _lib.X509_verify_cert_error_string(
+                        _lib.X509_STORE_CTX_get_error(self._store_ctx)
+                    )
+                )
+            ),
+        ]
+        # A context error should always be associated with a certificate, so we
+        # expect this call to never return :class:`None`.
+        _x509 = _lib.X509_STORE_CTX_get_current_cert(self._store_ctx)
+        _cert = _lib.X509_dup(_x509)
+        pycert = X509._from_raw_x509_ptr(_cert)
+        return X509StoreContextError(errors, pycert)
+
+    def set_store(self, store):
+        """
+        Set the context's X.509 store.
+
+        .. versionadded:: 0.15
+
+        :param X509Store store: The store description which will be used for
+            the purposes of any *future* verifications.
+        """
+        self._store = store
+
+    def verify_certificate(self):
+        """
+        Verify a certificate in a context.
+
+        .. versionadded:: 0.15
+
+        :raises X509StoreContextError: If an error occurred when validating a
+          certificate in the context. Sets ``certificate`` attribute to
+          indicate which certificate caused the error.
+        """
+        # Always re-initialize the store context in case
+        # :meth:`verify_certificate` is called multiple times.
+        #
+        # :meth:`_init` is called in :meth:`__init__` so _cleanup is called
+        # before _init to ensure memory is not leaked.
+        self._cleanup()
+        self._init()
+        ret = _lib.X509_verify_cert(self._store_ctx)
+        self._cleanup()
+        if ret <= 0:
+            raise self._exception_from_context()
+
+    def get_verified_chain(self):
+        """
+        Verify a certificate in a context and return the complete validated
+        chain.
+
+        :raises X509StoreContextError: If an error occurred when validating a
+          certificate in the context. Sets ``certificate`` attribute to
+          indicate which certificate caused the error.
+
+        .. versionadded:: 20.0
+        """
+        # Always re-initialize the store context in case
+        # :meth:`verify_certificate` is called multiple times.
+        #
+        # :meth:`_init` is called in :meth:`__init__` so _cleanup is called
+        # before _init to ensure memory is not leaked.
+        self._cleanup()
+        self._init()
+        ret = _lib.X509_verify_cert(self._store_ctx)
+        if ret <= 0:
+            self._cleanup()
+            raise self._exception_from_context()
+
+        # Note: X509_STORE_CTX_get1_chain returns a deep copy of the chain.
+        cert_stack = _lib.X509_STORE_CTX_get1_chain(self._store_ctx)
+        _openssl_assert(cert_stack != _ffi.NULL)
+
+        result = []
+        for i in range(_lib.sk_X509_num(cert_stack)):
+            cert = _lib.sk_X509_value(cert_stack, i)
+            _openssl_assert(cert != _ffi.NULL)
+            pycert = X509._from_raw_x509_ptr(cert)
+            result.append(pycert)
+
+        # Free the stack but not the members which are freed by the X509 class.
+        _lib.sk_X509_free(cert_stack)
+        self._cleanup()
+        return result
+
+
+def load_certificate(type, buffer):
+    """
+    Load a certificate (X509) from the string *buffer* encoded with the
+    type *type*.
+
+    :param type: The file type (one of FILETYPE_PEM, FILETYPE_ASN1)
+
+    :param bytes buffer: The buffer the certificate is stored in
+
+    :return: The X509 object
+    """
+    if isinstance(buffer, _text_type):
+        buffer = buffer.encode("ascii")
+
+    bio = _new_mem_buf(buffer)
+
+    if type == FILETYPE_PEM:
+        x509 = _lib.PEM_read_bio_X509(bio, _ffi.NULL, _ffi.NULL, _ffi.NULL)
+    elif type == FILETYPE_ASN1:
+        x509 = _lib.d2i_X509_bio(bio, _ffi.NULL)
+    else:
+        raise ValueError("type argument must be FILETYPE_PEM or FILETYPE_ASN1")
+
+    if x509 == _ffi.NULL:
+        _raise_current_error()
+
+    return X509._from_raw_x509_ptr(x509)
+
+
+def dump_certificate(type, cert):
+    """
+    Dump the certificate *cert* into a buffer string encoded with the type
+    *type*.
+
+    :param type: The file type (one of FILETYPE_PEM, FILETYPE_ASN1, or
+        FILETYPE_TEXT)
+    :param cert: The certificate to dump
+    :return: The buffer with the dumped certificate in
+    """
+    bio = _new_mem_buf()
+
+    if type == FILETYPE_PEM:
+        result_code = _lib.PEM_write_bio_X509(bio, cert._x509)
+    elif type == FILETYPE_ASN1:
+        result_code = _lib.i2d_X509_bio(bio, cert._x509)
+    elif type == FILETYPE_TEXT:
+        result_code = _lib.X509_print_ex(bio, cert._x509, 0, 0)
+    else:
+        raise ValueError(
+            "type argument must be FILETYPE_PEM, FILETYPE_ASN1, or "
+            "FILETYPE_TEXT"
+        )
+
+    _openssl_assert(result_code == 1)
+    return _bio_to_string(bio)
+
+
+def dump_publickey(type, pkey):
+    """
+    Dump a public key to a buffer.
+
+    :param type: The file type (one of :data:`FILETYPE_PEM` or
+        :data:`FILETYPE_ASN1`).
+    :param PKey pkey: The public key to dump
+    :return: The buffer with the dumped key in it.
+    :rtype: bytes
+    """
+    bio = _new_mem_buf()
+    if type == FILETYPE_PEM:
+        write_bio = _lib.PEM_write_bio_PUBKEY
+    elif type == FILETYPE_ASN1:
+        write_bio = _lib.i2d_PUBKEY_bio
+    else:
+        raise ValueError("type argument must be FILETYPE_PEM or FILETYPE_ASN1")
+
+    result_code = write_bio(bio, pkey._pkey)
+    if result_code != 1:  # pragma: no cover
+        _raise_current_error()
+
+    return _bio_to_string(bio)
+
+
+def dump_privatekey(type, pkey, cipher=None, passphrase=None):
+    """
+    Dump the private key *pkey* into a buffer string encoded with the type
+    *type*.  Optionally (if *type* is :const:`FILETYPE_PEM`) encrypting it
+    using *cipher* and *passphrase*.
+
+    :param type: The file type (one of :const:`FILETYPE_PEM`,
+        :const:`FILETYPE_ASN1`, or :const:`FILETYPE_TEXT`)
+    :param PKey pkey: The PKey to dump
+    :param cipher: (optional) if encrypted PEM format, the cipher to use
+    :param passphrase: (optional) if encrypted PEM format, this can be either
+        the passphrase to use, or a callback for providing the passphrase.
+
+    :return: The buffer with the dumped key in
+    :rtype: bytes
+    """
+    bio = _new_mem_buf()
+
+    if not isinstance(pkey, PKey):
+        raise TypeError("pkey must be a PKey")
+
+    if cipher is not None:
+        if passphrase is None:
+            raise TypeError(
+                "if a value is given for cipher "
+                "one must also be given for passphrase"
+            )
+        cipher_obj = _lib.EVP_get_cipherbyname(_byte_string(cipher))
+        if cipher_obj == _ffi.NULL:
+            raise ValueError("Invalid cipher name")
+    else:
+        cipher_obj = _ffi.NULL
+
+    helper = _PassphraseHelper(type, passphrase)
+    if type == FILETYPE_PEM:
+        result_code = _lib.PEM_write_bio_PrivateKey(
+            bio,
+            pkey._pkey,
+            cipher_obj,
+            _ffi.NULL,
+            0,
+            helper.callback,
+            helper.callback_args,
+        )
+        helper.raise_if_problem()
+    elif type == FILETYPE_ASN1:
+        result_code = _lib.i2d_PrivateKey_bio(bio, pkey._pkey)
+    elif type == FILETYPE_TEXT:
+        if _lib.EVP_PKEY_id(pkey._pkey) != _lib.EVP_PKEY_RSA:
+            raise TypeError("Only RSA keys are supported for FILETYPE_TEXT")
+
+        rsa = _ffi.gc(_lib.EVP_PKEY_get1_RSA(pkey._pkey), _lib.RSA_free)
+        result_code = _lib.RSA_print(bio, rsa, 0)
+    else:
+        raise ValueError(
+            "type argument must be FILETYPE_PEM, FILETYPE_ASN1, or "
+            "FILETYPE_TEXT"
+        )
+
+    _openssl_assert(result_code != 0)
+
+    return _bio_to_string(bio)
+
+
+class Revoked(object):
+    """
+    A certificate revocation.
+    """
+
+    # https://www.openssl.org/docs/manmaster/man5/x509v3_config.html#CRL-distribution-points
+    # which differs from crl_reasons of crypto/x509v3/v3_enum.c that matches
+    # OCSP_crl_reason_str.  We use the latter, just like the command line
+    # program.
+    _crl_reasons = [
+        b"unspecified",
+        b"keyCompromise",
+        b"CACompromise",
+        b"affiliationChanged",
+        b"superseded",
+        b"cessationOfOperation",
+        b"certificateHold",
+        # b"removeFromCRL",
+    ]
+
+    def __init__(self):
+        revoked = _lib.X509_REVOKED_new()
+        self._revoked = _ffi.gc(revoked, _lib.X509_REVOKED_free)
+
+    def set_serial(self, hex_str):
+        """
+        Set the serial number.
+
+        The serial number is formatted as a hexadecimal number encoded in
+        ASCII.
+
+        :param bytes hex_str: The new serial number.
+
+        :return: ``None``
+        """
+        bignum_serial = _ffi.gc(_lib.BN_new(), _lib.BN_free)
+        bignum_ptr = _ffi.new("BIGNUM**")
+        bignum_ptr[0] = bignum_serial
+        bn_result = _lib.BN_hex2bn(bignum_ptr, hex_str)
+        if not bn_result:
+            raise ValueError("bad hex string")
+
+        asn1_serial = _ffi.gc(
+            _lib.BN_to_ASN1_INTEGER(bignum_serial, _ffi.NULL),
+            _lib.ASN1_INTEGER_free,
+        )
+        _lib.X509_REVOKED_set_serialNumber(self._revoked, asn1_serial)
+
+    def get_serial(self):
+        """
+        Get the serial number.
+
+        The serial number is formatted as a hexadecimal number encoded in
+        ASCII.
+
+        :return: The serial number.
+        :rtype: bytes
+        """
+        bio = _new_mem_buf()
+
+        asn1_int = _lib.X509_REVOKED_get0_serialNumber(self._revoked)
+        _openssl_assert(asn1_int != _ffi.NULL)
+        result = _lib.i2a_ASN1_INTEGER(bio, asn1_int)
+        _openssl_assert(result >= 0)
+        return _bio_to_string(bio)
+
+    def _delete_reason(self):
+        for i in range(_lib.X509_REVOKED_get_ext_count(self._revoked)):
+            ext = _lib.X509_REVOKED_get_ext(self._revoked, i)
+            obj = _lib.X509_EXTENSION_get_object(ext)
+            if _lib.OBJ_obj2nid(obj) == _lib.NID_crl_reason:
+                _lib.X509_EXTENSION_free(ext)
+                _lib.X509_REVOKED_delete_ext(self._revoked, i)
+                break
+
+    def set_reason(self, reason):
+        """
+        Set the reason of this revocation.
+
+        If :data:`reason` is ``None``, delete the reason instead.
+
+        :param reason: The reason string.
+        :type reason: :class:`bytes` or :class:`NoneType`
+
+        :return: ``None``
+
+        .. seealso::
+
+            :meth:`all_reasons`, which gives you a list of all supported
+            reasons which you might pass to this method.
+        """
+        if reason is None:
+            self._delete_reason()
+        elif not isinstance(reason, bytes):
+            raise TypeError("reason must be None or a byte string")
+        else:
+            reason = reason.lower().replace(b" ", b"")
+            reason_code = [r.lower() for r in self._crl_reasons].index(reason)
+
+            new_reason_ext = _lib.ASN1_ENUMERATED_new()
+            _openssl_assert(new_reason_ext != _ffi.NULL)
+            new_reason_ext = _ffi.gc(new_reason_ext, _lib.ASN1_ENUMERATED_free)
+
+            set_result = _lib.ASN1_ENUMERATED_set(new_reason_ext, reason_code)
+            _openssl_assert(set_result != _ffi.NULL)
+
+            self._delete_reason()
+            add_result = _lib.X509_REVOKED_add1_ext_i2d(
+                self._revoked, _lib.NID_crl_reason, new_reason_ext, 0, 0
+            )
+            _openssl_assert(add_result == 1)
+
+    def get_reason(self):
+        """
+        Get the reason of this revocation.
+
+        :return: The reason, or ``None`` if there is none.
+        :rtype: bytes or NoneType
+
+        .. seealso::
+
+            :meth:`all_reasons`, which gives you a list of all supported
+            reasons this method might return.
+        """
+        for i in range(_lib.X509_REVOKED_get_ext_count(self._revoked)):
+            ext = _lib.X509_REVOKED_get_ext(self._revoked, i)
+            obj = _lib.X509_EXTENSION_get_object(ext)
+            if _lib.OBJ_obj2nid(obj) == _lib.NID_crl_reason:
+                bio = _new_mem_buf()
+
+                print_result = _lib.X509V3_EXT_print(bio, ext, 0, 0)
+                if not print_result:
+                    print_result = _lib.M_ASN1_OCTET_STRING_print(
+                        bio, _lib.X509_EXTENSION_get_data(ext)
+                    )
+                    _openssl_assert(print_result != 0)
+
+                return _bio_to_string(bio)
+
+    def all_reasons(self):
+        """
+        Return a list of all the supported reason strings.
+
+        This list is a copy; modifying it does not change the supported reason
+        strings.
+
+        :return: A list of reason strings.
+        :rtype: :class:`list` of :class:`bytes`
+        """
+        return self._crl_reasons[:]
+
+    def set_rev_date(self, when):
+        """
+        Set the revocation timestamp.
+
+        :param bytes when: The timestamp of the revocation,
+            as ASN.1 TIME.
+        :return: ``None``
+        """
+        dt = _lib.X509_REVOKED_get0_revocationDate(self._revoked)
+        return _set_asn1_time(dt, when)
+
+    def get_rev_date(self):
+        """
+        Get the revocation timestamp.
+
+        :return: The timestamp of the revocation, as ASN.1 TIME.
+        :rtype: bytes
+        """
+        dt = _lib.X509_REVOKED_get0_revocationDate(self._revoked)
+        return _get_asn1_time(dt)
+
+
+class CRL(object):
+    """
+    A certificate revocation list.
+    """
+
+    def __init__(self):
+        crl = _lib.X509_CRL_new()
+        self._crl = _ffi.gc(crl, _lib.X509_CRL_free)
+
+    def to_cryptography(self):
+        """
+        Export as a ``cryptography`` CRL.
+
+        :rtype: ``cryptography.x509.CertificateRevocationList``
+
+        .. versionadded:: 17.1.0
+        """
+        from cryptography.x509 import load_der_x509_crl
+
+        der = dump_crl(FILETYPE_ASN1, self)
+
+        backend = _get_backend()
+        return load_der_x509_crl(der, backend)
+
+    @classmethod
+    def from_cryptography(cls, crypto_crl):
+        """
+        Construct based on a ``cryptography`` *crypto_crl*.
+
+        :param crypto_crl: A ``cryptography`` certificate revocation list
+        :type crypto_crl: ``cryptography.x509.CertificateRevocationList``
+
+        :rtype: CRL
+
+        .. versionadded:: 17.1.0
+        """
+        if not isinstance(crypto_crl, x509.CertificateRevocationList):
+            raise TypeError("Must be a certificate revocation list")
+
+        from cryptography.hazmat.primitives.serialization import Encoding
+
+        der = crypto_crl.public_bytes(Encoding.DER)
+        return load_crl(FILETYPE_ASN1, der)
+
+    def get_revoked(self):
+        """
+        Return the revocations in this certificate revocation list.
+
+        These revocations will be provided by value, not by reference.
+        That means it's okay to mutate them: it won't affect this CRL.
+
+        :return: The revocations in this CRL.
+        :rtype: :class:`tuple` of :class:`Revocation`
+        """
+        results = []
+        revoked_stack = _lib.X509_CRL_get_REVOKED(self._crl)
+        for i in range(_lib.sk_X509_REVOKED_num(revoked_stack)):
+            revoked = _lib.sk_X509_REVOKED_value(revoked_stack, i)
+            revoked_copy = _lib.Cryptography_X509_REVOKED_dup(revoked)
+            pyrev = Revoked.__new__(Revoked)
+            pyrev._revoked = _ffi.gc(revoked_copy, _lib.X509_REVOKED_free)
+            results.append(pyrev)
+        if results:
+            return tuple(results)
+
+    def add_revoked(self, revoked):
+        """
+        Add a revoked (by value not reference) to the CRL structure
+
+        This revocation will be added by value, not by reference. That
+        means it's okay to mutate it after adding: it won't affect
+        this CRL.
+
+        :param Revoked revoked: The new revocation.
+        :return: ``None``
+        """
+        copy = _lib.Cryptography_X509_REVOKED_dup(revoked._revoked)
+        _openssl_assert(copy != _ffi.NULL)
+
+        add_result = _lib.X509_CRL_add0_revoked(self._crl, copy)
+        _openssl_assert(add_result != 0)
+
+    def get_issuer(self):
+        """
+        Get the CRL's issuer.
+
+        .. versionadded:: 16.1.0
+
+        :rtype: X509Name
+        """
+        _issuer = _lib.X509_NAME_dup(_lib.X509_CRL_get_issuer(self._crl))
+        _openssl_assert(_issuer != _ffi.NULL)
+        _issuer = _ffi.gc(_issuer, _lib.X509_NAME_free)
+        issuer = X509Name.__new__(X509Name)
+        issuer._name = _issuer
+        return issuer
+
+    def set_version(self, version):
+        """
+        Set the CRL version.
+
+        .. versionadded:: 16.1.0
+
+        :param int version: The version of the CRL.
+        :return: ``None``
+        """
+        _openssl_assert(_lib.X509_CRL_set_version(self._crl, version) != 0)
+
+    def _set_boundary_time(self, which, when):
+        return _set_asn1_time(which(self._crl), when)
+
+    def set_lastUpdate(self, when):
+        """
+        Set when the CRL was last updated.
+
+        The timestamp is formatted as an ASN.1 TIME::
+
+            YYYYMMDDhhmmssZ
+
+        .. versionadded:: 16.1.0
+
+        :param bytes when: A timestamp string.
+        :return: ``None``
+        """
+        return self._set_boundary_time(_lib.X509_CRL_get_lastUpdate, when)
+
+    def set_nextUpdate(self, when):
+        """
+        Set when the CRL will next be updated.
+
+        The timestamp is formatted as an ASN.1 TIME::
+
+            YYYYMMDDhhmmssZ
+
+        .. versionadded:: 16.1.0
+
+        :param bytes when: A timestamp string.
+        :return: ``None``
+        """
+        return self._set_boundary_time(_lib.X509_CRL_get_nextUpdate, when)
+
+    def sign(self, issuer_cert, issuer_key, digest):
+        """
+        Sign the CRL.
+
+        Signing a CRL enables clients to associate the CRL itself with an
+        issuer. Before a CRL is meaningful to other OpenSSL functions, it must
+        be signed by an issuer.
+
+        This method implicitly sets the issuer's name based on the issuer
+        certificate and private key used to sign the CRL.
+
+        .. versionadded:: 16.1.0
+
+        :param X509 issuer_cert: The issuer's certificate.
+        :param PKey issuer_key: The issuer's private key.
+        :param bytes digest: The digest method to sign the CRL with.
+        """
+        digest_obj = _lib.EVP_get_digestbyname(digest)
+        _openssl_assert(digest_obj != _ffi.NULL)
+        _lib.X509_CRL_set_issuer_name(
+            self._crl, _lib.X509_get_subject_name(issuer_cert._x509)
+        )
+        _lib.X509_CRL_sort(self._crl)
+        result = _lib.X509_CRL_sign(self._crl, issuer_key._pkey, digest_obj)
+        _openssl_assert(result != 0)
+
+    def export(
+        self, cert, key, type=FILETYPE_PEM, days=100, digest=_UNSPECIFIED
+    ):
+        """
+        Export the CRL as a string.
+
+        :param X509 cert: The certificate used to sign the CRL.
+        :param PKey key: The key used to sign the CRL.
+        :param int type: The export format, either :data:`FILETYPE_PEM`,
+            :data:`FILETYPE_ASN1`, or :data:`FILETYPE_TEXT`.
+        :param int days: The number of days until the next update of this CRL.
+        :param bytes digest: The name of the message digest to use (eg
+            ``b"sha256"``).
+        :rtype: bytes
+        """
+
+        if not isinstance(cert, X509):
+            raise TypeError("cert must be an X509 instance")
+        if not isinstance(key, PKey):
+            raise TypeError("key must be a PKey instance")
+        if not isinstance(type, int):
+            raise TypeError("type must be an integer")
+
+        if digest is _UNSPECIFIED:
+            raise TypeError("digest must be provided")
+
+        digest_obj = _lib.EVP_get_digestbyname(digest)
+        if digest_obj == _ffi.NULL:
+            raise ValueError("No such digest method")
+
+        bio = _lib.BIO_new(_lib.BIO_s_mem())
+        _openssl_assert(bio != _ffi.NULL)
+
+        # A scratch time object to give different values to different CRL
+        # fields
+        sometime = _lib.ASN1_TIME_new()
+        _openssl_assert(sometime != _ffi.NULL)
+
+        _lib.X509_gmtime_adj(sometime, 0)
+        _lib.X509_CRL_set_lastUpdate(self._crl, sometime)
+
+        _lib.X509_gmtime_adj(sometime, days * 24 * 60 * 60)
+        _lib.X509_CRL_set_nextUpdate(self._crl, sometime)
+
+        _lib.X509_CRL_set_issuer_name(
+            self._crl, _lib.X509_get_subject_name(cert._x509)
+        )
+
+        sign_result = _lib.X509_CRL_sign(self._crl, key._pkey, digest_obj)
+        if not sign_result:
+            _raise_current_error()
+
+        return dump_crl(type, self)
+
+
+class PKCS7(object):
+    def type_is_signed(self):
+        """
+        Check if this NID_pkcs7_signed object
+
+        :return: True if the PKCS7 is of type signed
+        """
+        return bool(_lib.PKCS7_type_is_signed(self._pkcs7))
+
+    def type_is_enveloped(self):
+        """
+        Check if this NID_pkcs7_enveloped object
+
+        :returns: True if the PKCS7 is of type enveloped
+        """
+        return bool(_lib.PKCS7_type_is_enveloped(self._pkcs7))
+
+    def type_is_signedAndEnveloped(self):
+        """
+        Check if this NID_pkcs7_signedAndEnveloped object
+
+        :returns: True if the PKCS7 is of type signedAndEnveloped
+        """
+        return bool(_lib.PKCS7_type_is_signedAndEnveloped(self._pkcs7))
+
+    def type_is_data(self):
+        """
+        Check if this NID_pkcs7_data object
+
+        :return: True if the PKCS7 is of type data
+        """
+        return bool(_lib.PKCS7_type_is_data(self._pkcs7))
+
+    def get_type_name(self):
+        """
+        Returns the type name of the PKCS7 structure
+
+        :return: A string with the typename
+        """
+        nid = _lib.OBJ_obj2nid(self._pkcs7.type)
+        string_type = _lib.OBJ_nid2sn(nid)
+        return _ffi.string(string_type)
+
+
+class PKCS12(object):
+    """
+    A PKCS #12 archive.
+    """
+
+    def __init__(self):
+        self._pkey = None
+        self._cert = None
+        self._cacerts = None
+        self._friendlyname = None
+
+    def get_certificate(self):
+        """
+        Get the certificate in the PKCS #12 structure.
+
+        :return: The certificate, or :py:const:`None` if there is none.
+        :rtype: :py:class:`X509` or :py:const:`None`
+        """
+        return self._cert
+
+    def set_certificate(self, cert):
+        """
+        Set the certificate in the PKCS #12 structure.
+
+        :param cert: The new certificate, or :py:const:`None` to unset it.
+        :type cert: :py:class:`X509` or :py:const:`None`
+
+        :return: ``None``
+        """
+        if not isinstance(cert, X509):
+            raise TypeError("cert must be an X509 instance")
+        self._cert = cert
+
+    def get_privatekey(self):
+        """
+        Get the private key in the PKCS #12 structure.
+
+        :return: The private key, or :py:const:`None` if there is none.
+        :rtype: :py:class:`PKey`
+        """
+        return self._pkey
+
+    def set_privatekey(self, pkey):
+        """
+        Set the certificate portion of the PKCS #12 structure.
+
+        :param pkey: The new private key, or :py:const:`None` to unset it.
+        :type pkey: :py:class:`PKey` or :py:const:`None`
+
+        :return: ``None``
+        """
+        if not isinstance(pkey, PKey):
+            raise TypeError("pkey must be a PKey instance")
+        self._pkey = pkey
+
+    def get_ca_certificates(self):
+        """
+        Get the CA certificates in the PKCS #12 structure.
+
+        :return: A tuple with the CA certificates in the chain, or
+            :py:const:`None` if there are none.
+        :rtype: :py:class:`tuple` of :py:class:`X509` or :py:const:`None`
+        """
+        if self._cacerts is not None:
+            return tuple(self._cacerts)
+
+    def set_ca_certificates(self, cacerts):
+        """
+        Replace or set the CA certificates within the PKCS12 object.
+
+        :param cacerts: The new CA certificates, or :py:const:`None` to unset
+            them.
+        :type cacerts: An iterable of :py:class:`X509` or :py:const:`None`
+
+        :return: ``None``
+        """
+        if cacerts is None:
+            self._cacerts = None
+        else:
+            cacerts = list(cacerts)
+            for cert in cacerts:
+                if not isinstance(cert, X509):
+                    raise TypeError(
+                        "iterable must only contain X509 instances"
+                    )
+            self._cacerts = cacerts
+
+    def set_friendlyname(self, name):
+        """
+        Set the friendly name in the PKCS #12 structure.
+
+        :param name: The new friendly name, or :py:const:`None` to unset.
+        :type name: :py:class:`bytes` or :py:const:`None`
+
+        :return: ``None``
+        """
+        if name is None:
+            self._friendlyname = None
+        elif not isinstance(name, bytes):
+            raise TypeError(
+                "name must be a byte string or None (not %r)" % (name,)
+            )
+        self._friendlyname = name
+
+    def get_friendlyname(self):
+        """
+        Get the friendly name in the PKCS# 12 structure.
+
+        :returns: The friendly name,  or :py:const:`None` if there is none.
+        :rtype: :py:class:`bytes` or :py:const:`None`
+        """
+        return self._friendlyname
+
+    def export(self, passphrase=None, iter=2048, maciter=1):
+        """
+        Dump a PKCS12 object as a string.
+
+        For more information, see the :c:func:`PKCS12_create` man page.
+
+        :param passphrase: The passphrase used to encrypt the structure. Unlike
+            some other passphrase arguments, this *must* be a string, not a
+            callback.
+        :type passphrase: :py:data:`bytes`
+
+        :param iter: Number of times to repeat the encryption step.
+        :type iter: :py:data:`int`
+
+        :param maciter: Number of times to repeat the MAC step.
+        :type maciter: :py:data:`int`
+
+        :return: The string representation of the PKCS #12 structure.
+        :rtype:
+        """
+        passphrase = _text_to_bytes_and_warn("passphrase", passphrase)
+
+        if self._cacerts is None:
+            cacerts = _ffi.NULL
+        else:
+            cacerts = _lib.sk_X509_new_null()
+            cacerts = _ffi.gc(cacerts, _lib.sk_X509_free)
+            for cert in self._cacerts:
+                _lib.sk_X509_push(cacerts, cert._x509)
+
+        if passphrase is None:
+            passphrase = _ffi.NULL
+
+        friendlyname = self._friendlyname
+        if friendlyname is None:
+            friendlyname = _ffi.NULL
+
+        if self._pkey is None:
+            pkey = _ffi.NULL
+        else:
+            pkey = self._pkey._pkey
+
+        if self._cert is None:
+            cert = _ffi.NULL
+        else:
+            cert = self._cert._x509
+
+        pkcs12 = _lib.PKCS12_create(
+            passphrase,
+            friendlyname,
+            pkey,
+            cert,
+            cacerts,
+            _lib.NID_pbe_WithSHA1And3_Key_TripleDES_CBC,
+            _lib.NID_pbe_WithSHA1And3_Key_TripleDES_CBC,
+            iter,
+            maciter,
+            0,
+        )
+        if pkcs12 == _ffi.NULL:
+            _raise_current_error()
+        pkcs12 = _ffi.gc(pkcs12, _lib.PKCS12_free)
+
+        bio = _new_mem_buf()
+        _lib.i2d_PKCS12_bio(bio, pkcs12)
+        return _bio_to_string(bio)
+
+
+class NetscapeSPKI(object):
+    """
+    A Netscape SPKI object.
+    """
+
+    def __init__(self):
+        spki = _lib.NETSCAPE_SPKI_new()
+        self._spki = _ffi.gc(spki, _lib.NETSCAPE_SPKI_free)
+
+    def sign(self, pkey, digest):
+        """
+        Sign the certificate request with this key and digest type.
+
+        :param pkey: The private key to sign with.
+        :type pkey: :py:class:`PKey`
+
+        :param digest: The message digest to use.
+        :type digest: :py:class:`bytes`
+
+        :return: ``None``
+        """
+        if pkey._only_public:
+            raise ValueError("Key has only public part")
+
+        if not pkey._initialized:
+            raise ValueError("Key is uninitialized")
+
+        digest_obj = _lib.EVP_get_digestbyname(_byte_string(digest))
+        if digest_obj == _ffi.NULL:
+            raise ValueError("No such digest method")
+
+        sign_result = _lib.NETSCAPE_SPKI_sign(
+            self._spki, pkey._pkey, digest_obj
+        )
+        _openssl_assert(sign_result > 0)
+
+    def verify(self, key):
+        """
+        Verifies a signature on a certificate request.
+
+        :param PKey key: The public key that signature is supposedly from.
+
+        :return: ``True`` if the signature is correct.
+        :rtype: bool
+
+        :raises OpenSSL.crypto.Error: If the signature is invalid, or there was
+            a problem verifying the signature.
+        """
+        answer = _lib.NETSCAPE_SPKI_verify(self._spki, key._pkey)
+        if answer <= 0:
+            _raise_current_error()
+        return True
+
+    def b64_encode(self):
+        """
+        Generate a base64 encoded representation of this SPKI object.
+
+        :return: The base64 encoded string.
+        :rtype: :py:class:`bytes`
+        """
+        encoded = _lib.NETSCAPE_SPKI_b64_encode(self._spki)
+        result = _ffi.string(encoded)
+        _lib.OPENSSL_free(encoded)
+        return result
+
+    def get_pubkey(self):
+        """
+        Get the public key of this certificate.
+
+        :return: The public key.
+        :rtype: :py:class:`PKey`
+        """
+        pkey = PKey.__new__(PKey)
+        pkey._pkey = _lib.NETSCAPE_SPKI_get_pubkey(self._spki)
+        _openssl_assert(pkey._pkey != _ffi.NULL)
+        pkey._pkey = _ffi.gc(pkey._pkey, _lib.EVP_PKEY_free)
+        pkey._only_public = True
+        return pkey
+
+    def set_pubkey(self, pkey):
+        """
+        Set the public key of the certificate
+
+        :param pkey: The public key
+        :return: ``None``
+        """
+        set_result = _lib.NETSCAPE_SPKI_set_pubkey(self._spki, pkey._pkey)
+        _openssl_assert(set_result == 1)
+
+
+class _PassphraseHelper(object):
+    def __init__(self, type, passphrase, more_args=False, truncate=False):
+        if type != FILETYPE_PEM and passphrase is not None:
+            raise ValueError(
+                "only FILETYPE_PEM key format supports encryption"
+            )
+        self._passphrase = passphrase
+        self._more_args = more_args
+        self._truncate = truncate
+        self._problems = []
+
+    @property
+    def callback(self):
+        if self._passphrase is None:
+            return _ffi.NULL
+        elif isinstance(self._passphrase, bytes) or callable(self._passphrase):
+            return _ffi.callback("pem_password_cb", self._read_passphrase)
+        else:
+            raise TypeError(
+                "Last argument must be a byte string or a callable."
+            )
+
+    @property
+    def callback_args(self):
+        if self._passphrase is None:
+            return _ffi.NULL
+        elif isinstance(self._passphrase, bytes) or callable(self._passphrase):
+            return _ffi.NULL
+        else:
+            raise TypeError(
+                "Last argument must be a byte string or a callable."
+            )
+
+    def raise_if_problem(self, exceptionType=Error):
+        if self._problems:
+
+            # Flush the OpenSSL error queue
+            try:
+                _exception_from_error_queue(exceptionType)
+            except exceptionType:
+                pass
+
+            raise self._problems.pop(0)
+
+    def _read_passphrase(self, buf, size, rwflag, userdata):
+        try:
+            if callable(self._passphrase):
+                if self._more_args:
+                    result = self._passphrase(size, rwflag, userdata)
+                else:
+                    result = self._passphrase(rwflag)
+            else:
+                result = self._passphrase
+            if not isinstance(result, bytes):
+                raise ValueError("Bytes expected")
+            if len(result) > size:
+                if self._truncate:
+                    result = result[:size]
+                else:
+                    raise ValueError(
+                        "passphrase returned by callback is too long"
+                    )
+            for i in range(len(result)):
+                buf[i] = result[i : i + 1]
+            return len(result)
+        except Exception as e:
+            self._problems.append(e)
+            return 0
+
+
+def load_publickey(type, buffer):
+    """
+    Load a public key from a buffer.
+
+    :param type: The file type (one of :data:`FILETYPE_PEM`,
+        :data:`FILETYPE_ASN1`).
+    :param buffer: The buffer the key is stored in.
+    :type buffer: A Python string object, either unicode or bytestring.
+    :return: The PKey object.
+    :rtype: :class:`PKey`
+    """
+    if isinstance(buffer, _text_type):
+        buffer = buffer.encode("ascii")
+
+    bio = _new_mem_buf(buffer)
+
+    if type == FILETYPE_PEM:
+        evp_pkey = _lib.PEM_read_bio_PUBKEY(
+            bio, _ffi.NULL, _ffi.NULL, _ffi.NULL
+        )
+    elif type == FILETYPE_ASN1:
+        evp_pkey = _lib.d2i_PUBKEY_bio(bio, _ffi.NULL)
+    else:
+        raise ValueError("type argument must be FILETYPE_PEM or FILETYPE_ASN1")
+
+    if evp_pkey == _ffi.NULL:
+        _raise_current_error()
+
+    pkey = PKey.__new__(PKey)
+    pkey._pkey = _ffi.gc(evp_pkey, _lib.EVP_PKEY_free)
+    pkey._only_public = True
+    return pkey
+
+
+def load_privatekey(type, buffer, passphrase=None):
+    """
+    Load a private key (PKey) from the string *buffer* encoded with the type
+    *type*.
+
+    :param type: The file type (one of FILETYPE_PEM, FILETYPE_ASN1)
+    :param buffer: The buffer the key is stored in
+    :param passphrase: (optional) if encrypted PEM format, this can be
+                       either the passphrase to use, or a callback for
+                       providing the passphrase.
+
+    :return: The PKey object
+    """
+    if isinstance(buffer, _text_type):
+        buffer = buffer.encode("ascii")
+
+    bio = _new_mem_buf(buffer)
+
+    helper = _PassphraseHelper(type, passphrase)
+    if type == FILETYPE_PEM:
+        evp_pkey = _lib.PEM_read_bio_PrivateKey(
+            bio, _ffi.NULL, helper.callback, helper.callback_args
+        )
+        helper.raise_if_problem()
+    elif type == FILETYPE_ASN1:
+        evp_pkey = _lib.d2i_PrivateKey_bio(bio, _ffi.NULL)
+    else:
+        raise ValueError("type argument must be FILETYPE_PEM or FILETYPE_ASN1")
+
+    if evp_pkey == _ffi.NULL:
+        _raise_current_error()
+
+    pkey = PKey.__new__(PKey)
+    pkey._pkey = _ffi.gc(evp_pkey, _lib.EVP_PKEY_free)
+    return pkey
+
+
+def dump_certificate_request(type, req):
+    """
+    Dump the certificate request *req* into a buffer string encoded with the
+    type *type*.
+
+    :param type: The file type (one of FILETYPE_PEM, FILETYPE_ASN1)
+    :param req: The certificate request to dump
+    :return: The buffer with the dumped certificate request in
+    """
+    bio = _new_mem_buf()
+
+    if type == FILETYPE_PEM:
+        result_code = _lib.PEM_write_bio_X509_REQ(bio, req._req)
+    elif type == FILETYPE_ASN1:
+        result_code = _lib.i2d_X509_REQ_bio(bio, req._req)
+    elif type == FILETYPE_TEXT:
+        result_code = _lib.X509_REQ_print_ex(bio, req._req, 0, 0)
+    else:
+        raise ValueError(
+            "type argument must be FILETYPE_PEM, FILETYPE_ASN1, or "
+            "FILETYPE_TEXT"
+        )
+
+    _openssl_assert(result_code != 0)
+
+    return _bio_to_string(bio)
+
+
+def load_certificate_request(type, buffer):
+    """
+    Load a certificate request (X509Req) from the string *buffer* encoded with
+    the type *type*.
+
+    :param type: The file type (one of FILETYPE_PEM, FILETYPE_ASN1)
+    :param buffer: The buffer the certificate request is stored in
+    :return: The X509Req object
+    """
+    if isinstance(buffer, _text_type):
+        buffer = buffer.encode("ascii")
+
+    bio = _new_mem_buf(buffer)
+
+    if type == FILETYPE_PEM:
+        req = _lib.PEM_read_bio_X509_REQ(bio, _ffi.NULL, _ffi.NULL, _ffi.NULL)
+    elif type == FILETYPE_ASN1:
+        req = _lib.d2i_X509_REQ_bio(bio, _ffi.NULL)
+    else:
+        raise ValueError("type argument must be FILETYPE_PEM or FILETYPE_ASN1")
+
+    _openssl_assert(req != _ffi.NULL)
+
+    x509req = X509Req.__new__(X509Req)
+    x509req._req = _ffi.gc(req, _lib.X509_REQ_free)
+    return x509req
+
+
+def sign(pkey, data, digest):
+    """
+    Sign a data string using the given key and message digest.
+
+    :param pkey: PKey to sign with
+    :param data: data to be signed
+    :param digest: message digest to use
+    :return: signature
+
+    .. versionadded:: 0.11
+    """
+    data = _text_to_bytes_and_warn("data", data)
+
+    digest_obj = _lib.EVP_get_digestbyname(_byte_string(digest))
+    if digest_obj == _ffi.NULL:
+        raise ValueError("No such digest method")
+
+    md_ctx = _lib.Cryptography_EVP_MD_CTX_new()
+    md_ctx = _ffi.gc(md_ctx, _lib.Cryptography_EVP_MD_CTX_free)
+
+    _lib.EVP_SignInit(md_ctx, digest_obj)
+    _lib.EVP_SignUpdate(md_ctx, data, len(data))
+
+    length = _lib.EVP_PKEY_size(pkey._pkey)
+    _openssl_assert(length > 0)
+    signature_buffer = _ffi.new("unsigned char[]", length)
+    signature_length = _ffi.new("unsigned int *")
+    final_result = _lib.EVP_SignFinal(
+        md_ctx, signature_buffer, signature_length, pkey._pkey
+    )
+    _openssl_assert(final_result == 1)
+
+    return _ffi.buffer(signature_buffer, signature_length[0])[:]
+
+
+def verify(cert, signature, data, digest):
+    """
+    Verify the signature for a data string.
+
+    :param cert: signing certificate (X509 object) corresponding to the
+        private key which generated the signature.
+    :param signature: signature returned by sign function
+    :param data: data to be verified
+    :param digest: message digest to use
+    :return: ``None`` if the signature is correct, raise exception otherwise.
+
+    .. versionadded:: 0.11
+    """
+    data = _text_to_bytes_and_warn("data", data)
+
+    digest_obj = _lib.EVP_get_digestbyname(_byte_string(digest))
+    if digest_obj == _ffi.NULL:
+        raise ValueError("No such digest method")
+
+    pkey = _lib.X509_get_pubkey(cert._x509)
+    _openssl_assert(pkey != _ffi.NULL)
+    pkey = _ffi.gc(pkey, _lib.EVP_PKEY_free)
+
+    md_ctx = _lib.Cryptography_EVP_MD_CTX_new()
+    md_ctx = _ffi.gc(md_ctx, _lib.Cryptography_EVP_MD_CTX_free)
+
+    _lib.EVP_VerifyInit(md_ctx, digest_obj)
+    _lib.EVP_VerifyUpdate(md_ctx, data, len(data))
+    verify_result = _lib.EVP_VerifyFinal(
+        md_ctx, signature, len(signature), pkey
+    )
+
+    if verify_result != 1:
+        _raise_current_error()
+
+
+def dump_crl(type, crl):
+    """
+    Dump a certificate revocation list to a buffer.
+
+    :param type: The file type (one of ``FILETYPE_PEM``, ``FILETYPE_ASN1``, or
+        ``FILETYPE_TEXT``).
+    :param CRL crl: The CRL to dump.
+
+    :return: The buffer with the CRL.
+    :rtype: bytes
+    """
+    bio = _new_mem_buf()
+
+    if type == FILETYPE_PEM:
+        ret = _lib.PEM_write_bio_X509_CRL(bio, crl._crl)
+    elif type == FILETYPE_ASN1:
+        ret = _lib.i2d_X509_CRL_bio(bio, crl._crl)
+    elif type == FILETYPE_TEXT:
+        ret = _lib.X509_CRL_print(bio, crl._crl)
+    else:
+        raise ValueError(
+            "type argument must be FILETYPE_PEM, FILETYPE_ASN1, or "
+            "FILETYPE_TEXT"
+        )
+
+    _openssl_assert(ret == 1)
+    return _bio_to_string(bio)
+
+
+def load_crl(type, buffer):
+    """
+    Load Certificate Revocation List (CRL) data from a string *buffer*.
+    *buffer* encoded with the type *type*.
+
+    :param type: The file type (one of FILETYPE_PEM, FILETYPE_ASN1)
+    :param buffer: The buffer the CRL is stored in
+
+    :return: The PKey object
+    """
+    if isinstance(buffer, _text_type):
+        buffer = buffer.encode("ascii")
+
+    bio = _new_mem_buf(buffer)
+
+    if type == FILETYPE_PEM:
+        crl = _lib.PEM_read_bio_X509_CRL(bio, _ffi.NULL, _ffi.NULL, _ffi.NULL)
+    elif type == FILETYPE_ASN1:
+        crl = _lib.d2i_X509_CRL_bio(bio, _ffi.NULL)
+    else:
+        raise ValueError("type argument must be FILETYPE_PEM or FILETYPE_ASN1")
+
+    if crl == _ffi.NULL:
+        _raise_current_error()
+
+    result = CRL.__new__(CRL)
+    result._crl = _ffi.gc(crl, _lib.X509_CRL_free)
+    return result
+
+
+def load_pkcs7_data(type, buffer):
+    """
+    Load pkcs7 data from the string *buffer* encoded with the type
+    *type*.
+
+    :param type: The file type (one of FILETYPE_PEM or FILETYPE_ASN1)
+    :param buffer: The buffer with the pkcs7 data.
+    :return: The PKCS7 object
+    """
+    if isinstance(buffer, _text_type):
+        buffer = buffer.encode("ascii")
+
+    bio = _new_mem_buf(buffer)
+
+    if type == FILETYPE_PEM:
+        pkcs7 = _lib.PEM_read_bio_PKCS7(bio, _ffi.NULL, _ffi.NULL, _ffi.NULL)
+    elif type == FILETYPE_ASN1:
+        pkcs7 = _lib.d2i_PKCS7_bio(bio, _ffi.NULL)
+    else:
+        raise ValueError("type argument must be FILETYPE_PEM or FILETYPE_ASN1")
+
+    if pkcs7 == _ffi.NULL:
+        _raise_current_error()
+
+    pypkcs7 = PKCS7.__new__(PKCS7)
+    pypkcs7._pkcs7 = _ffi.gc(pkcs7, _lib.PKCS7_free)
+    return pypkcs7
+
+
+load_pkcs7_data = utils.deprecated(
+    load_pkcs7_data,
+    __name__,
+    (
+        "PKCS#7 support in pyOpenSSL is deprecated. You should use the APIs "
+        "in cryptography."
+    ),
+    DeprecationWarning,
+)
+
+
+def load_pkcs12(buffer, passphrase=None):
+    """
+    Load pkcs12 data from the string *buffer*. If the pkcs12 structure is
+    encrypted, a *passphrase* must be included.  The MAC is always
+    checked and thus required.
+
+    See also the man page for the C function :py:func:`PKCS12_parse`.
+
+    :param buffer: The buffer the certificate is stored in
+    :param passphrase: (Optional) The password to decrypt the PKCS12 lump
+    :returns: The PKCS12 object
+    """
+    passphrase = _text_to_bytes_and_warn("passphrase", passphrase)
+
+    if isinstance(buffer, _text_type):
+        buffer = buffer.encode("ascii")
+
+    bio = _new_mem_buf(buffer)
+
+    # Use null passphrase if passphrase is None or empty string. With PKCS#12
+    # password based encryption no password and a zero length password are two
+    # different things, but OpenSSL implementation will try both to figure out
+    # which one works.
+    if not passphrase:
+        passphrase = _ffi.NULL
+
+    p12 = _lib.d2i_PKCS12_bio(bio, _ffi.NULL)
+    if p12 == _ffi.NULL:
+        _raise_current_error()
+    p12 = _ffi.gc(p12, _lib.PKCS12_free)
+
+    pkey = _ffi.new("EVP_PKEY**")
+    cert = _ffi.new("X509**")
+    cacerts = _ffi.new("Cryptography_STACK_OF_X509**")
+
+    parse_result = _lib.PKCS12_parse(p12, passphrase, pkey, cert, cacerts)
+    if not parse_result:
+        _raise_current_error()
+
+    cacerts = _ffi.gc(cacerts[0], _lib.sk_X509_free)
+
+    # openssl 1.0.0 sometimes leaves an X509_check_private_key error in the
+    # queue for no particular reason.  This error isn't interesting to anyone
+    # outside this function.  It's not even interesting to us.  Get rid of it.
+    try:
+        _raise_current_error()
+    except Error:
+        pass
+
+    if pkey[0] == _ffi.NULL:
+        pykey = None
+    else:
+        pykey = PKey.__new__(PKey)
+        pykey._pkey = _ffi.gc(pkey[0], _lib.EVP_PKEY_free)
+
+    if cert[0] == _ffi.NULL:
+        pycert = None
+        friendlyname = None
+    else:
+        pycert = X509._from_raw_x509_ptr(cert[0])
+
+        friendlyname_length = _ffi.new("int*")
+        friendlyname_buffer = _lib.X509_alias_get0(
+            cert[0], friendlyname_length
+        )
+        friendlyname = _ffi.buffer(
+            friendlyname_buffer, friendlyname_length[0]
+        )[:]
+        if friendlyname_buffer == _ffi.NULL:
+            friendlyname = None
+
+    pycacerts = []
+    for i in range(_lib.sk_X509_num(cacerts)):
+        x509 = _lib.sk_X509_value(cacerts, i)
+        pycacert = X509._from_raw_x509_ptr(x509)
+        pycacerts.append(pycacert)
+    if not pycacerts:
+        pycacerts = None
+
+    pkcs12 = PKCS12.__new__(PKCS12)
+    pkcs12._pkey = pykey
+    pkcs12._cert = pycert
+    pkcs12._cacerts = pycacerts
+    pkcs12._friendlyname = friendlyname
+    return pkcs12
+
+
+load_pkcs12 = utils.deprecated(
+    load_pkcs12,
+    __name__,
+    (
+        "PKCS#12 support in pyOpenSSL is deprecated. You should use the APIs "
+        "in cryptography."
+    ),
+    DeprecationWarning,
+)
+
+
+# There are no direct unit tests for this initialization.  It is tested
+# indirectly since it is necessary for functions like dump_privatekey when
+# using encryption.
+#
+# Thus OpenSSL.test.test_crypto.FunctionTests.test_dump_privatekey_passphrase
+# and some other similar tests may fail without this (though they may not if
+# the Python runtime has already done some initialization of the underlying
+# OpenSSL library (and is linked against the same one that cryptography is
+# using)).
+_lib.OpenSSL_add_all_algorithms()
+
+# This is similar but exercised mainly by exception_from_error_queue.  It calls
+# both ERR_load_crypto_strings() and ERR_load_SSL_strings().
+_lib.SSL_load_error_strings()
+
+
+# Set the default string mask to match OpenSSL upstream (since 2005) and
+# RFC5280 recommendations.
+_lib.ASN1_STRING_set_default_mask_asc(b"utf8only")
diff --git a/contrib/python/pyOpenSSL/py2/OpenSSL/debug.py b/contrib/python/pyOpenSSL/py2/OpenSSL/debug.py
new file mode 100644
index 00000000000..04521d59226
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/OpenSSL/debug.py
@@ -0,0 +1,42 @@
+from __future__ import print_function
+
+import ssl
+import sys
+
+import OpenSSL.SSL
+import cffi
+import cryptography
+
+from . import version
+
+
+_env_info = u"""\
+pyOpenSSL: {pyopenssl}
+cryptography: {cryptography}
+cffi: {cffi}
+cryptography's compiled against OpenSSL: {crypto_openssl_compile}
+cryptography's linked OpenSSL: {crypto_openssl_link}
+Python's OpenSSL: {python_openssl}
+Python executable: {python}
+Python version: {python_version}
+Platform: {platform}
+sys.path: {sys_path}""".format(
+    pyopenssl=version.__version__,
+    crypto_openssl_compile=OpenSSL._util.ffi.string(
+        OpenSSL._util.lib.OPENSSL_VERSION_TEXT,
+    ).decode("ascii"),
+    crypto_openssl_link=OpenSSL.SSL.SSLeay_version(
+        OpenSSL.SSL.SSLEAY_VERSION
+    ).decode("ascii"),
+    python_openssl=getattr(ssl, "OPENSSL_VERSION", "n/a"),
+    cryptography=cryptography.__version__,
+    cffi=cffi.__version__,
+    python=sys.executable,
+    python_version=sys.version,
+    platform=sys.platform,
+    sys_path=sys.path,
+)
+
+
+if __name__ == "__main__":
+    print(_env_info)
diff --git a/contrib/python/pyOpenSSL/py2/OpenSSL/rand.py b/contrib/python/pyOpenSSL/py2/OpenSSL/rand.py
new file mode 100644
index 00000000000..d2c17673e5e
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/OpenSSL/rand.py
@@ -0,0 +1,40 @@
+"""
+PRNG management routines, thin wrappers.
+"""
+
+from OpenSSL._util import lib as _lib
+
+
+def add(buffer, entropy):
+    """
+    Mix bytes from *string* into the PRNG state.
+
+    The *entropy* argument is (the lower bound of) an estimate of how much
+    randomness is contained in *string*, measured in bytes.
+
+    For more information, see e.g. :rfc:`1750`.
+
+    This function is only relevant if you are forking Python processes and
+    need to reseed the CSPRNG after fork.
+
+    :param buffer: Buffer with random data.
+    :param entropy: The entropy (in bytes) measurement of the buffer.
+
+    :return: :obj:`None`
+    """
+    if not isinstance(buffer, bytes):
+        raise TypeError("buffer must be a byte string")
+
+    if not isinstance(entropy, int):
+        raise TypeError("entropy must be an integer")
+
+    _lib.RAND_add(buffer, len(buffer), entropy)
+
+
+def status():
+    """
+    Check whether the PRNG has been seeded with enough data.
+
+    :return: 1 if the PRNG is seeded enough, 0 otherwise.
+    """
+    return _lib.RAND_status()
diff --git a/contrib/python/pyOpenSSL/py2/OpenSSL/version.py b/contrib/python/pyOpenSSL/py2/OpenSSL/version.py
new file mode 100644
index 00000000000..c6fcecb0774
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/OpenSSL/version.py
@@ -0,0 +1,28 @@
+# Copyright (C) AB Strakt
+# Copyright (C) Jean-Paul Calderone
+# See LICENSE for details.
+
+"""
+pyOpenSSL - A simple wrapper around the OpenSSL library
+"""
+
+__all__ = [
+    "__author__",
+    "__copyright__",
+    "__email__",
+    "__license__",
+    "__summary__",
+    "__title__",
+    "__uri__",
+    "__version__",
+]
+
+__version__ = "21.0.0"
+
+__title__ = "pyOpenSSL"
+__uri__ = "https://pyopenssl.org/"
+__summary__ = "Python wrapper module around the OpenSSL library"
+__author__ = "The pyOpenSSL developers"
+__email__ = "cryptography-dev@python.org"
+__license__ = "Apache License, Version 2.0"
+__copyright__ = "Copyright 2001-2020 {0}".format(__author__)
diff --git a/contrib/python/pyOpenSSL/py2/README.rst b/contrib/python/pyOpenSSL/py2/README.rst
new file mode 100644
index 00000000000..a628c8aea9f
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/README.rst
@@ -0,0 +1,46 @@
+========================================================
+pyOpenSSL -- A Python wrapper around the OpenSSL library
+========================================================
+
+.. image:: https://readthedocs.org/projects/pyopenssl/badge/?version=stable
+   :target: https://pyopenssl.org/en/stable/
+   :alt: Stable Docs
+
+.. image:: https://github.com/pyca/pyopenssl/workflows/CI/badge.svg?branch=main
+   :target: https://github.com/pyca/pyopenssl/actions?query=workflow%3ACI+branch%3Amain
+
+.. image:: https://codecov.io/github/pyca/pyopenssl/branch/main/graph/badge.svg
+   :target: https://codecov.io/github/pyca/pyopenssl
+   :alt: Test coverage
+
+**Note:** The Python Cryptographic Authority **strongly suggests** the use of `pyca/cryptography`_
+where possible. If you are using pyOpenSSL for anything other than making a TLS connection
+**you should move to cryptography and drop your pyOpenSSL dependency**.
+
+High-level wrapper around a subset of the OpenSSL library. Includes
+
+* ``SSL.Connection`` objects, wrapping the methods of Python's portable sockets
+* Callbacks written in Python
+* Extensive error-handling mechanism, mirroring OpenSSL's error codes
+
+... and much more.
+
+You can find more information in the documentation_.
+Development takes place on GitHub_.
+
+
+Discussion
+==========
+
+If you run into bugs, you can file them in our `issue tracker`_.
+
+We maintain a cryptography-dev_ mailing list for both user and development discussions.
+
+You can also join ``#cryptography-dev`` on Freenode to ask questions or get involved.
+
+
+.. _documentation: https://pyopenssl.org/
+.. _`issue tracker`: https://github.com/pyca/pyopenssl/issues
+.. _cryptography-dev: https://mail.python.org/mailman/listinfo/cryptography-dev
+.. _GitHub: https://github.com/pyca/pyopenssl
+.. _`pyca/cryptography`: https://github.com/pyca/cryptography
diff --git a/contrib/python/pyOpenSSL/py2/ya.make b/contrib/python/pyOpenSSL/py2/ya.make
new file mode 100644
index 00000000000..6c1e686c4f8
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py2/ya.make
@@ -0,0 +1,33 @@
+# Generated by devtools/yamaker (pypi).
+
+PY2_LIBRARY()
+
+VERSION(21.0.0)
+
+LICENSE(Apache-2.0)
+
+PEERDIR(
+    contrib/python/cryptography
+    contrib/python/six
+)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    OpenSSL/SSL.py
+    OpenSSL/__init__.py
+    OpenSSL/_util.py
+    OpenSSL/crypto.py
+    OpenSSL/debug.py
+    OpenSSL/rand.py
+    OpenSSL/version.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/pyOpenSSL/py2/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+)
+
+END()
diff --git a/contrib/python/pyOpenSSL/py3/LICENSE b/contrib/python/pyOpenSSL/py3/LICENSE
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py3/LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/contrib/python/pyOpenSSL/py3/README.rst b/contrib/python/pyOpenSSL/py3/README.rst
new file mode 100644
index 00000000000..a628c8aea9f
--- /dev/null
+++ b/contrib/python/pyOpenSSL/py3/README.rst
@@ -0,0 +1,46 @@
+========================================================
+pyOpenSSL -- A Python wrapper around the OpenSSL library
+========================================================
+
+.. image:: https://readthedocs.org/projects/pyopenssl/badge/?version=stable
+   :target: https://pyopenssl.org/en/stable/
+   :alt: Stable Docs
+
+.. image:: https://github.com/pyca/pyopenssl/workflows/CI/badge.svg?branch=main
+   :target: https://github.com/pyca/pyopenssl/actions?query=workflow%3ACI+branch%3Amain
+
+.. image:: https://codecov.io/github/pyca/pyopenssl/branch/main/graph/badge.svg
+   :target: https://codecov.io/github/pyca/pyopenssl
+   :alt: Test coverage
+
+**Note:** The Python Cryptographic Authority **strongly suggests** the use of `pyca/cryptography`_
+where possible. If you are using pyOpenSSL for anything other than making a TLS connection
+**you should move to cryptography and drop your pyOpenSSL dependency**.
+
+High-level wrapper around a subset of the OpenSSL library. Includes
+
+* ``SSL.Connection`` objects, wrapping the methods of Python's portable sockets
+* Callbacks written in Python
+* Extensive error-handling mechanism, mirroring OpenSSL's error codes
+
+... and much more.
+
+You can find more information in the documentation_.
+Development takes place on GitHub_.
+
+
+Discussion
+==========
+
+If you run into bugs, you can file them in our `issue tracker`_.
+
+We maintain a cryptography-dev_ mailing list for both user and development discussions.
+
+You can also join ``#cryptography-dev`` on Freenode to ask questions or get involved.
+
+
+.. _documentation: https://pyopenssl.org/
+.. _`issue tracker`: https://github.com/pyca/pyopenssl/issues
+.. _cryptography-dev: https://mail.python.org/mailman/listinfo/cryptography-dev
+.. _GitHub: https://github.com/pyca/pyopenssl
+.. _`pyca/cryptography`: https://github.com/pyca/cryptography
diff --git a/contrib/python/pyOpenSSL/ya.make b/contrib/python/pyOpenSSL/ya.make
new file mode 100644
index 00000000000..62a5fd852f0
--- /dev/null
+++ b/contrib/python/pyOpenSSL/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+    PEERDIR(contrib/python/pyOpenSSL/py2)
+ELSE()
+    PEERDIR(contrib/python/pyOpenSSL/py3)
+ENDIF()
+
+NO_LINT()
+
+END()
+
+RECURSE(
+    py2
+    py3
+)
diff --git a/contrib/python/pyrsistent/py2/.dist-info/METADATA b/contrib/python/pyrsistent/py2/.dist-info/METADATA
new file mode 100644
index 00000000000..6b02f259000
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/.dist-info/METADATA
@@ -0,0 +1,744 @@
+Metadata-Version: 2.1
+Name: pyrsistent
+Version: 0.15.7
+Summary: Persistent/Functional/Immutable data structures
+Home-page: http://github.com/tobgu/pyrsistent/
+Author: Tobias Gustafsson
+Author-email: tobias.l.gustafsson@gmail.com
+License: MIT
+Platform: UNKNOWN
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Dist: six
+
+Pyrsistent
+==========
+.. image:: https://travis-ci.org/tobgu/pyrsistent.png?branch=master
+    :target: https://travis-ci.org/tobgu/pyrsistent
+
+.. image:: https://badge.fury.io/py/pyrsistent.svg
+    :target: https://badge.fury.io/py/pyrsistent
+
+.. image:: https://coveralls.io/repos/tobgu/pyrsistent/badge.svg?branch=master&service=github
+    :target: https://coveralls.io/github/tobgu/pyrsistent?branch=master
+
+
+.. _Pyrthon: https://www.github.com/tobgu/pyrthon/
+
+Pyrsistent is a number of persistent collections (by some referred to as functional data structures). Persistent in 
+the sense that they are immutable.
+
+All methods on a data structure that would normally mutate it instead return a new copy of the structure containing the
+requested updates. The original structure is left untouched.
+
+This will simplify the reasoning about what a program does since no hidden side effects ever can take place to these
+data structures. You can rest assured that the object you hold a reference to will remain the same throughout its
+lifetime and need not worry that somewhere five stack levels below you in the darkest corner of your application
+someone has decided to remove that element that you expected to be there.
+
+Pyrsistent is influenced by persistent data structures such as those found in the standard library of Clojure. The
+data structures are designed to share common elements through path copying.
+It aims at taking these concepts and make them as pythonic as possible so that they can be easily integrated into any python
+program without hassle.
+
+If you want to go all in on persistent data structures and use literal syntax to define them in your code rather
+than function calls check out Pyrthon_.
+
+Examples
+--------
+.. _Sequence: collections_
+.. _Hashable: collections_
+.. _Mapping: collections_
+.. _Mappings: collections_
+.. _Set: collections_
+.. _collections: https://docs.python.org/3/library/collections.abc.html
+.. _documentation: http://pyrsistent.readthedocs.org/
+
+The collection types and key features currently implemented are:
+
+* PVector_, similar to a python list
+* PMap_, similar to dict
+* PSet_, similar to set
+* PRecord_, a PMap on steroids with fixed fields, optional type and invariant checking and much more
+* PClass_, a Python class fixed fields, optional type and invariant checking and much more
+* `Checked collections`_, PVector, PMap and PSet with optional type and invariance checks and more
+* PBag, similar to collections.Counter
+* PList, a classic singly linked list
+* PDeque, similar to collections.deque
+* Immutable object type (immutable) built on the named tuple
+* freeze_ and thaw_ functions to convert between pythons standard collections and pyrsistent collections.
+* Flexible transformations_ of arbitrarily complex structures built from PMaps and PVectors.
+
+Below are examples of common usage patterns for some of the structures and features. More information and
+full documentation for all data structures is available in the documentation_.
+
+.. _PVector:
+
+PVector
+~~~~~~~
+With full support for the Sequence_ protocol PVector is meant as a drop in replacement to the built in list from a readers
+point of view. Write operations of course differ since no in place mutation is done but naming should be in line
+with corresponding operations on the built in list.
+
+Support for the Hashable_ protocol also means that it can be used as key in Mappings_.
+
+Appends are amortized O(1). Random access and insert is log32(n) where n is the size of the vector.
+
+.. code:: python
+
+    >>> from pyrsistent import v, pvector
+
+    # No mutation of vectors once created, instead they
+    # are "evolved" leaving the original untouched
+    >>> v1 = v(1, 2, 3)
+    >>> v2 = v1.append(4)
+    >>> v3 = v2.set(1, 5)
+    >>> v1
+    pvector([1, 2, 3])
+    >>> v2
+    pvector([1, 2, 3, 4])
+    >>> v3
+    pvector([1, 5, 3, 4])
+
+    # Random access and slicing
+    >>> v3[1]
+    5
+    >>> v3[1:3]
+    pvector([5, 3])
+
+    # Iteration
+    >>> list(x + 1 for x in v3)
+    [2, 6, 4, 5]
+    >>> pvector(2 * x for x in range(3))
+    pvector([0, 2, 4])
+
+.. _PMap:
+
+PMap
+~~~~
+With full support for the Mapping_ protocol PMap is meant as a drop in replacement to the built in dict from a readers point
+of view. Support for the Hashable_ protocol also means that it can be used as key in other Mappings_.
+
+Random access and insert is log32(n) where n is the size of the map.
+
+.. code:: python
+
+    >>> from pyrsistent import m, pmap, v
+
+    # No mutation of maps once created, instead they are
+    # "evolved" leaving the original untouched
+    >>> m1 = m(a=1, b=2)
+    >>> m2 = m1.set('c', 3)
+    >>> m3 = m2.set('a', 5)
+    >>> m1
+    pmap({'a': 1, 'b': 2})
+    >>> m2
+    pmap({'a': 1, 'c': 3, 'b': 2})
+    >>> m3
+    pmap({'a': 5, 'c': 3, 'b': 2})
+    >>> m3['a']
+    5
+
+    # Evolution of nested persistent structures
+    >>> m4 = m(a=5, b=6, c=v(1, 2))
+    >>> m4.transform(('c', 1), 17)
+    pmap({'a': 5, 'c': pvector([1, 17]), 'b': 6})
+    >>> m5 = m(a=1, b=2)
+
+    # Evolve by merging with other mappings
+    >>> m5.update(m(a=2, c=3), {'a': 17, 'd': 35})
+    pmap({'a': 17, 'c': 3, 'b': 2, 'd': 35})
+    >>> pmap({'x': 1, 'y': 2}) + pmap({'y': 3, 'z': 4})
+    pmap({'y': 3, 'x': 1, 'z': 4})
+
+    # Dict-like methods to convert to list and iterate
+    >>> m3.items()
+    pvector([('a', 5), ('c', 3), ('b', 2)])
+    >>> list(m3)
+    ['a', 'c', 'b']
+
+.. _PSet:
+
+PSet
+~~~~
+With full support for the Set_ protocol PSet is meant as a drop in replacement to the built in set from a readers point
+of view. Support for the Hashable_ protocol also means that it can be used as key in Mappings_.
+
+Random access and insert is log32(n) where n is the size of the set.
+
+.. code:: python
+
+    >>> from pyrsistent import s
+
+    # No mutation of sets once created, you know the story...
+    >>> s1 = s(1, 2, 3, 2)
+    >>> s2 = s1.add(4)
+    >>> s3 = s1.remove(1)
+    >>> s1
+    pset([1, 2, 3])
+    >>> s2
+    pset([1, 2, 3, 4])
+    >>> s3
+    pset([2, 3])
+
+    # Full support for set operations
+    >>> s1 | s(3, 4, 5)
+    pset([1, 2, 3, 4, 5])
+    >>> s1 & s(3, 4, 5)
+    pset([3])
+    >>> s1 < s2
+    True
+    >>> s1 < s(3, 4, 5)
+    False
+
+.. _PRecord:
+
+PRecord
+~~~~~~~
+A PRecord is a PMap with a fixed set of specified fields. Records are declared as python classes inheriting
+from PRecord. Because it is a PMap it has full support for all Mapping methods such as iteration and element
+access using subscript notation.
+
+.. code:: python
+
+    >>> from pyrsistent import PRecord, field
+    >>> class ARecord(PRecord):
+    ...     x = field()
+    ...
+    >>> r = ARecord(x=3)
+    >>> r
+    ARecord(x=3)
+    >>> r.x
+    3
+    >>> r.set(x=2)
+    ARecord(x=2)
+    >>> r.set(y=2)
+    Traceback (most recent call last):
+    AttributeError: 'y' is not among the specified fields for ARecord
+
+Type information
+****************
+It is possible to add type information to the record to enforce type checks. Multiple allowed types can be specified
+by providing an iterable of types.
+
+.. code:: python
+
+    >>> class BRecord(PRecord):
+    ...     x = field(type=int)
+    ...     y = field(type=(int, type(None)))
+    ...
+    >>> BRecord(x=3, y=None)
+    BRecord(y=None, x=3)
+    >>> BRecord(x=3.0)
+    Traceback (most recent call last):
+    PTypeError: Invalid type for field BRecord.x, was float
+
+
+Custom types (classes) that are iterable should be wrapped in a tuple to prevent their
+members being added to the set of valid types.  Although Enums in particular are now 
+supported without wrapping, see #83 for more information.
+
+Mandatory fields
+****************
+Fields are not mandatory by default but can be specified as such. If fields are missing an
+*InvariantException* will be thrown which contains information about the missing fields.
+
+.. code:: python
+
+    >>> from pyrsistent import InvariantException
+    >>> class CRecord(PRecord):
+    ...     x = field(mandatory=True)
+    ...
+    >>> r = CRecord(x=3)
+    >>> try:
+    ...    r.discard('x')
+    ... except InvariantException as e:
+    ...    print(e.missing_fields)
+    ...
+    ('CRecord.x',)
+
+Invariants
+**********
+It is possible to add invariants that must hold when evolving the record. Invariants can be
+specified on both field and record level. If invariants fail an *InvariantException* will be
+thrown which contains information about the failing invariants. An invariant function should
+return a tuple consisting of a boolean that tells if the invariant holds or not and an object
+describing the invariant. This object can later be used to identify which invariant that failed.
+
+The global invariant function is only executed if all field invariants hold.
+
+Global invariants are inherited to subclasses.
+
+.. code:: python
+
+    >>> class RestrictedVector(PRecord):
+    ...     __invariant__ = lambda r: (r.y >= r.x, 'x larger than y')
+    ...     x = field(invariant=lambda x: (x > 0, 'x negative'))
+    ...     y = field(invariant=lambda y: (y > 0, 'y negative'))
+    ...
+    >>> r = RestrictedVector(y=3, x=2)
+    >>> try:
+    ...    r.set(x=-1, y=-2)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    ('y negative', 'x negative')
+    >>> try:
+    ...    r.set(x=2, y=1)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    ('x larger than y',)
+
+Invariants may also contain multiple assertions. For those cases the invariant function should
+return a tuple of invariant tuples as described above. This structure is reflected in the
+invariant_errors attribute of the exception which will contain tuples with data from all failed
+invariants. Eg:
+
+.. code:: python
+
+    >>> class EvenX(PRecord):
+    ...     x = field(invariant=lambda x: ((x > 0, 'x negative'), (x % 2 == 0, 'x odd')))
+    ...
+    >>> try:
+    ...    EvenX(x=-1)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    (('x negative', 'x odd'),)
+
+
+Factories
+*********
+It's possible to specify factory functions for fields. The factory function receives whatever
+is supplied as field value and the actual returned by the factory is assigned to the field
+given that any type and invariant checks hold.
+PRecords have a default factory specified as a static function on the class, create(). It takes
+a *Mapping* as argument and returns an instance of the specific record.
+If a record has fields of type PRecord the create() method of that record will
+be called to create the "sub record" if no factory has explicitly been specified to override
+this behaviour.
+
+.. code:: python
+
+    >>> class DRecord(PRecord):
+    ...     x = field(factory=int)
+    ...
+    >>> class ERecord(PRecord):
+    ...     d = field(type=DRecord)
+    ...
+    >>> ERecord.create({'d': {'x': '1'}})
+    ERecord(d=DRecord(x=1))
+
+Collection fields
+*****************
+It is also possible to have fields with ``pyrsistent`` collections.
+
+.. code:: python
+
+   >>> from pyrsistent import pset_field, pmap_field, pvector_field
+   >>> class MultiRecord(PRecord):
+   ...     set_of_ints = pset_field(int)
+   ...     map_int_to_str = pmap_field(int, str)
+   ...     vector_of_strs = pvector_field(str)
+   ...
+
+Serialization
+*************
+PRecords support serialization back to dicts. Default serialization will take keys and values
+"as is" and output them into a dict. It is possible to specify custom serialization functions
+to take care of fields that require special treatment.
+
+.. code:: python
+
+    >>> from datetime import date
+    >>> class Person(PRecord):
+    ...     name = field(type=unicode)
+    ...     birth_date = field(type=date,
+    ...                        serializer=lambda format, d: d.strftime(format['date']))
+    ...
+    >>> john = Person(name=u'John', birth_date=date(1985, 10, 21))
+    >>> john.serialize({'date': '%Y-%m-%d'})
+    {'birth_date': '1985-10-21', 'name': u'John'}
+
+
+.. _instar: https://github.com/boxed/instar/
+
+.. _PClass:
+
+PClass
+~~~~~~
+A PClass is a python class with a fixed set of specified fields. PClasses are declared as python classes inheriting
+from PClass. It is defined the same way that PRecords are and behaves like a PRecord in all aspects except that it
+is not a PMap and hence not a collection but rather a plain Python object.
+
+.. code:: python
+
+    >>> from pyrsistent import PClass, field
+    >>> class AClass(PClass):
+    ...     x = field()
+    ...
+    >>> a = AClass(x=3)
+    >>> a
+    AClass(x=3)
+    >>> a.x
+    3
+
+
+Checked collections
+~~~~~~~~~~~~~~~~~~~
+Checked collections currently come in three flavors: CheckedPVector, CheckedPMap and CheckedPSet.
+
+.. code:: python
+
+    >>> from pyrsistent import CheckedPVector, CheckedPMap, CheckedPSet, thaw
+    >>> class Positives(CheckedPSet):
+    ...     __type__ = (long, int)
+    ...     __invariant__ = lambda n: (n >= 0, 'Negative')
+    ...
+    >>> class Lottery(PRecord):
+    ...     name = field(type=str)
+    ...     numbers = field(type=Positives, invariant=lambda p: (len(p) > 0, 'No numbers'))
+    ...
+    >>> class Lotteries(CheckedPVector):
+    ...     __type__ = Lottery
+    ...
+    >>> class LotteriesByDate(CheckedPMap):
+    ...     __key_type__ = date
+    ...     __value_type__ = Lotteries
+    ...
+    >>> lotteries = LotteriesByDate.create({date(2015, 2, 15): [{'name': 'SuperLotto', 'numbers': {1, 2, 3}},
+    ...                                                         {'name': 'MegaLotto',  'numbers': {4, 5, 6}}],
+    ...                                     date(2015, 2, 16): [{'name': 'SuperLotto', 'numbers': {3, 2, 1}},
+    ...                                                         {'name': 'MegaLotto',  'numbers': {6, 5, 4}}]})
+    >>> lotteries
+    LotteriesByDate({datetime.date(2015, 2, 15): Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')]), datetime.date(2015, 2, 16): Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')])})
+
+    # The checked versions support all operations that the corresponding
+    # unchecked types do
+    >>> lottery_0215 = lotteries[date(2015, 2, 15)]
+    >>> lottery_0215.transform([0, 'name'], 'SuperDuperLotto')
+    Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperDuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')])
+
+    # But also makes asserts that types and invariants hold
+    >>> lottery_0215.transform([0, 'name'], 999)
+    Traceback (most recent call last):
+    PTypeError: Invalid type for field Lottery.name, was int
+
+    >>> lottery_0215.transform([0, 'numbers'], set())
+    Traceback (most recent call last):
+    InvariantException: Field invariant failed
+
+    # They can be converted back to python built ins with either thaw()
+    # or serialize() (which provides possibilities to customize serialization)
+    >>> thaw(lottery_0215)
+    [{'numbers': set([1, 2, 3]), 'name': 'SuperLotto'}, {'numbers': set([4, 5, 6]), 'name': 'MegaLotto'}]
+    >>> lottery_0215.serialize()
+    [{'numbers': set([1, 2, 3]), 'name': 'SuperLotto'}, {'numbers': set([4, 5, 6]), 'name': 'MegaLotto'}]
+
+.. _transformations:
+
+Transformations
+~~~~~~~~~~~~~~~
+Transformations are inspired by the cool library instar_ for Clojure. They let you evolve PMaps and PVectors
+with arbitrarily deep/complex nesting using simple syntax and flexible matching syntax.
+
+The first argument to transformation is the path that points out the value to transform. The
+second is the transformation to perform. If the transformation is callable it will be applied
+to the value(s) matching the path. The path may also contain callables. In that case they are
+treated as matchers. If the matcher returns True for a specific key it is considered for transformation.
+
+.. code:: python
+
+    # Basic examples
+    >>> from pyrsistent import inc, freeze, thaw, rex, ny, discard
+    >>> v1 = freeze([1, 2, 3, 4, 5])
+    >>> v1.transform([2], inc)
+    pvector([1, 2, 4, 4, 5])
+    >>> v1.transform([lambda ix: 0 < ix < 4], 8)
+    pvector([1, 8, 8, 8, 5])
+    >>> v1.transform([lambda ix, v: ix == 0 or v == 5], 0)
+    pvector([0, 2, 3, 4, 0])
+
+    # The (a)ny matcher can be used to match anything
+    >>> v1.transform([ny], 8)
+    pvector([8, 8, 8, 8, 8])
+
+    # Regular expressions can be used for matching
+    >>> scores = freeze({'John': 12, 'Joseph': 34, 'Sara': 23})
+    >>> scores.transform([rex('^Jo')], 0)
+    pmap({'Joseph': 0, 'Sara': 23, 'John': 0})
+
+    # Transformations can be done on arbitrarily deep structures
+    >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
+    ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
+    ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
+    >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
+    >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
+    >>> very_short_news.articles[0].content
+    'A short article'
+    >>> very_short_news.articles[1].content
+    'A slightly long...'
+
+    # When nothing has been transformed the original data structure is kept
+    >>> short_news is news_paper
+    True
+    >>> very_short_news is news_paper
+    False
+    >>> very_short_news.articles[0] is news_paper.articles[0]
+    True
+
+    # There is a special transformation that can be used to discard elements. Also
+    # multiple transformations can be applied in one call
+    >>> thaw(news_paper.transform(['weather'], discard, ['articles', ny, 'content'], discard))
+    {'articles': [{'author': 'Sara'}, {'author': 'Steve'}]}
+
+Evolvers
+~~~~~~~~
+PVector, PMap and PSet all have support for a concept dubbed *evolvers*. An evolver acts like a mutable
+view of the underlying persistent data structure with "transaction like" semantics. No updates of the original
+data structure is ever performed, it is still fully immutable.
+
+The evolvers have a very limited API by design to discourage excessive, and inappropriate, usage as that would
+take us down the mutable road. In principle only basic mutation and element access functions are supported.
+Check out the documentation_ of each data structure for specific examples.
+
+Examples of when you may want to use an evolver instead of working directly with the data structure include:
+
+* Multiple updates are done to the same data structure and the intermediate results are of no
+  interest. In this case using an evolver may be a more efficient and easier to work with.
+* You need to pass a vector into a legacy function or a function that you have no control
+  over which performs in place mutations. In this case pass an evolver instance
+  instead and then create a new pvector from the evolver once the function returns.
+
+.. code:: python
+
+    >>> from pyrsistent import v
+
+    # In place mutation as when working with the built in counterpart
+    >>> v1 = v(1, 2, 3)
+    >>> e = v1.evolver()
+    >>> e[1] = 22
+    >>> e = e.append(4)
+    >>> e = e.extend([5, 6])
+    >>> e[5] += 1
+    >>> len(e)
+    6
+
+    # The evolver is considered *dirty* when it contains changes compared to the underlying vector
+    >>> e.is_dirty()
+    True
+
+    # But the underlying pvector still remains untouched
+    >>> v1
+    pvector([1, 2, 3])
+
+    # Once satisfied with the updates you can produce a new pvector containing the updates.
+    # The new pvector will share data with the original pvector in the same way that would have
+    # been done if only using operations on the pvector.
+    >>> v2 = e.persistent()
+    >>> v2
+    pvector([1, 22, 3, 4, 5, 7])
+
+    # The evolver is now no longer considered *dirty* as it contains no differences compared to the
+    # pvector just produced.
+    >>> e.is_dirty()
+    False
+
+    # You may continue to work with the same evolver without affecting the content of v2
+    >>> e[0] = 11
+
+    # Or create a new evolver from v2. The two evolvers can be updated independently but will both
+    # share data with v2 where possible.
+    >>> e2 = v2.evolver()
+    >>> e2[0] = 1111
+    >>> e.persistent()
+    pvector([11, 22, 3, 4, 5, 7])
+    >>> e2.persistent()
+    pvector([1111, 22, 3, 4, 5, 7])
+
+.. _freeze:
+.. _thaw:
+
+freeze and thaw
+~~~~~~~~~~~~~~~
+These functions are great when your cozy immutable world has to interact with the evil mutable world outside.
+
+.. code:: python
+
+    >>> from pyrsistent import freeze, thaw, v, m
+    >>> freeze([1, {'a': 3}])
+    pvector([1, pmap({'a': 3})])
+    >>> thaw(v(1, m(a=3)))
+    [1, {'a': 3}]
+
+Compatibility
+-------------
+
+Pyrsistent is developed and tested on Python 2.7, 3.5, 3.6, 3.7 and PyPy (Python 2 and 3 compatible). It will most
+likely work on all other versions >= 3.4 but no guarantees are given. :)
+
+Compatibility issues
+~~~~~~~~~~~~~~~~~~~~
+
+.. _27: https://github.com/tobgu/pyrsistent/issues/27
+
+There is currently one known compatibility issue when comparing built in sets and frozensets to PSets as discussed in 27_.
+It affects python 2 versions < 2.7.8 and python 3 versions < 3.4.0 and is due to a bug described in
+http://bugs.python.org/issue8743.
+
+Comparisons will fail or be incorrect when using the set/frozenset as left hand side of the comparison. As a workaround
+you need to either upgrade Python to a more recent version, avoid comparing sets/frozensets with PSets or always make
+sure to convert both sides of the comparison to the same type before performing the comparison.
+
+Performance
+-----------
+
+Pyrsistent is developed with performance in mind. Still, while some operations are nearly on par with their built in, 
+mutable, counterparts in terms of speed, other operations are slower. In the cases where attempts at
+optimizations have been done, speed has generally been valued over space.
+
+Pyrsistent comes with two API compatible flavors of PVector (on which PMap and PSet are based), one pure Python 
+implementation and one implemented as a C extension. The latter generally being 2 - 20 times faster than the former.
+The C extension will be used automatically when possible.
+
+The pure python implementation is fully PyPy compatible. Running it under PyPy speeds operations up considerably if 
+the structures are used heavily (if JITed), for some cases the performance is almost on par with the built in counterparts.
+
+Type hints
+----------
+
+PEP 561 style type hints for use with mypy and various editors are available for most types and functions in pyrsistent.
+
+Type classes for annotating your own code with pyrsistent types are also available under pyrsistent.typing.
+
+Installation
+------------
+
+pip install pyrsistent
+
+Documentation
+-------------
+
+Available at http://pyrsistent.readthedocs.org/
+
+Brief presentation available at http://slides.com/tobiasgustafsson/immutability-and-python/
+
+Contributors
+------------
+
+Tobias Gustafsson https://github.com/tobgu
+
+Christopher Armstrong https://github.com/radix
+
+Anders Hovmöller https://github.com/boxed
+
+Itamar Turner-Trauring https://github.com/itamarst
+
+Jonathan Lange https://github.com/jml
+
+Richard Futrell https://github.com/Futrell
+
+Jakob Hollenstein https://github.com/jkbjh
+
+David Honour https://github.com/foolswood
+
+David R. MacIver https://github.com/DRMacIver
+
+Marcus Ewert https://github.com/sarum90
+
+Jean-Paul Calderone https://github.com/exarkun
+
+Douglas Treadwell https://github.com/douglas-treadwell
+
+Travis Parker https://github.com/teepark
+
+Julian Berman https://github.com/Julian
+
+Dennis Tomas https://github.com/dtomas
+
+Neil Vyas https://github.com/neilvyas
+
+doozr https://github.com/doozr
+
+Kamil Galuszka https://github.com/galuszkak
+
+Tsuyoshi Hombashi https://github.com/thombashi
+
+nattofriends https://github.com/nattofriends
+
+agberk https://github.com/agberk
+
+Waleed Khan https://github.com/arxanas
+
+Jean-Louis Fuchs https://github.com/ganwell
+
+Carlos Corbacho https://github.com/ccorbacho
+
+Felix Yan https://github.com/felixonmars
+
+benrg https://github.com/benrg
+
+Jere Lahelma https://github.com/je-l
+
+Max Taggart https://github.com/MaxTaggart
+
+Vincent Philippon https://github.com/vphilippon
+
+Semen Zhydenko https://github.com/ss18
+
+Till Varoquaux  https://github.com/till-varoquaux
+
+Michal Kowalik https://github.com/michalvi
+
+ossdev07 https://github.com/ossdev07
+
+Kerry Olesen https://github.com/qhesz
+
+Contributing
+------------
+
+Want to contribute? That's great! If you experience problems please log them on GitHub. If you want to contribute code,
+please fork the repository and submit a pull request.
+
+Run tests
+~~~~~~~~~
+.. _tox: https://tox.readthedocs.io/en/latest/
+
+Tests can be executed using tox_.
+
+Install tox: ``pip install tox``
+
+Run test for Python 2.7: ``tox -epy27``
+
+Release
+~~~~~~~
+* Update CHANGES.txt
+* Update README with any new contributors and potential info needed.
+* Update _pyrsistent_version.py
+* python setup.py sdist upload
+* Commit and tag with new version: git add -u . && git commit -m 'Prepare version vX.Y.Z' && git tag -a vX.Y.Z -m 'vX.Y.Z'
+* Push commit and tags: git push && git push --tags
+
+Project status
+--------------
+Pyrsistent can be considered stable and mature (who knows, there may even be a 1.0 some day :-)). The project is
+maintained, bugs fixed, PRs reviewed and merged and new releases made. I currently do not have time for development
+of new features or functionality which I don't have use for myself. I'm more than happy to take PRs for new
+functionality though!
+
+There are a bunch of issues marked with ``enhancement`` and ``help wanted`` that contain requests for new functionality
+that would be nice to include. The level of difficulty and extend of the issues varies, please reach out to me if you're
+interested in working on any of them.
+
+If you feel that you have a grand master plan for where you would like Pyrsistent to go and have the time to put into
+it please don't hesitate to discuss this with me and submit PRs for it. If all goes well I'd be more than happy to add
+additional maintainers to the project!
+
+
diff --git a/contrib/python/pyrsistent/py2/.dist-info/top_level.txt b/contrib/python/pyrsistent/py2/.dist-info/top_level.txt
new file mode 100644
index 00000000000..f2460728a9d
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/.dist-info/top_level.txt
@@ -0,0 +1,3 @@
+_pyrsistent_version
+pvectorc
+pyrsistent
diff --git a/contrib/python/pyrsistent/py2/README.rst b/contrib/python/pyrsistent/py2/README.rst
new file mode 100644
index 00000000000..d728284b6e4
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/README.rst
@@ -0,0 +1,723 @@
+Pyrsistent
+==========
+.. image:: https://travis-ci.org/tobgu/pyrsistent.png?branch=master
+    :target: https://travis-ci.org/tobgu/pyrsistent
+
+.. image:: https://badge.fury.io/py/pyrsistent.svg
+    :target: https://badge.fury.io/py/pyrsistent
+
+.. image:: https://coveralls.io/repos/tobgu/pyrsistent/badge.svg?branch=master&service=github
+    :target: https://coveralls.io/github/tobgu/pyrsistent?branch=master
+
+
+.. _Pyrthon: https://www.github.com/tobgu/pyrthon/
+
+Pyrsistent is a number of persistent collections (by some referred to as functional data structures). Persistent in 
+the sense that they are immutable.
+
+All methods on a data structure that would normally mutate it instead return a new copy of the structure containing the
+requested updates. The original structure is left untouched.
+
+This will simplify the reasoning about what a program does since no hidden side effects ever can take place to these
+data structures. You can rest assured that the object you hold a reference to will remain the same throughout its
+lifetime and need not worry that somewhere five stack levels below you in the darkest corner of your application
+someone has decided to remove that element that you expected to be there.
+
+Pyrsistent is influenced by persistent data structures such as those found in the standard library of Clojure. The
+data structures are designed to share common elements through path copying.
+It aims at taking these concepts and make them as pythonic as possible so that they can be easily integrated into any python
+program without hassle.
+
+If you want to go all in on persistent data structures and use literal syntax to define them in your code rather
+than function calls check out Pyrthon_.
+
+Examples
+--------
+.. _Sequence: collections_
+.. _Hashable: collections_
+.. _Mapping: collections_
+.. _Mappings: collections_
+.. _Set: collections_
+.. _collections: https://docs.python.org/3/library/collections.abc.html
+.. _documentation: http://pyrsistent.readthedocs.org/
+
+The collection types and key features currently implemented are:
+
+* PVector_, similar to a python list
+* PMap_, similar to dict
+* PSet_, similar to set
+* PRecord_, a PMap on steroids with fixed fields, optional type and invariant checking and much more
+* PClass_, a Python class fixed fields, optional type and invariant checking and much more
+* `Checked collections`_, PVector, PMap and PSet with optional type and invariance checks and more
+* PBag, similar to collections.Counter
+* PList, a classic singly linked list
+* PDeque, similar to collections.deque
+* Immutable object type (immutable) built on the named tuple
+* freeze_ and thaw_ functions to convert between pythons standard collections and pyrsistent collections.
+* Flexible transformations_ of arbitrarily complex structures built from PMaps and PVectors.
+
+Below are examples of common usage patterns for some of the structures and features. More information and
+full documentation for all data structures is available in the documentation_.
+
+.. _PVector:
+
+PVector
+~~~~~~~
+With full support for the Sequence_ protocol PVector is meant as a drop in replacement to the built in list from a readers
+point of view. Write operations of course differ since no in place mutation is done but naming should be in line
+with corresponding operations on the built in list.
+
+Support for the Hashable_ protocol also means that it can be used as key in Mappings_.
+
+Appends are amortized O(1). Random access and insert is log32(n) where n is the size of the vector.
+
+.. code:: python
+
+    >>> from pyrsistent import v, pvector
+
+    # No mutation of vectors once created, instead they
+    # are "evolved" leaving the original untouched
+    >>> v1 = v(1, 2, 3)
+    >>> v2 = v1.append(4)
+    >>> v3 = v2.set(1, 5)
+    >>> v1
+    pvector([1, 2, 3])
+    >>> v2
+    pvector([1, 2, 3, 4])
+    >>> v3
+    pvector([1, 5, 3, 4])
+
+    # Random access and slicing
+    >>> v3[1]
+    5
+    >>> v3[1:3]
+    pvector([5, 3])
+
+    # Iteration
+    >>> list(x + 1 for x in v3)
+    [2, 6, 4, 5]
+    >>> pvector(2 * x for x in range(3))
+    pvector([0, 2, 4])
+
+.. _PMap:
+
+PMap
+~~~~
+With full support for the Mapping_ protocol PMap is meant as a drop in replacement to the built in dict from a readers point
+of view. Support for the Hashable_ protocol also means that it can be used as key in other Mappings_.
+
+Random access and insert is log32(n) where n is the size of the map.
+
+.. code:: python
+
+    >>> from pyrsistent import m, pmap, v
+
+    # No mutation of maps once created, instead they are
+    # "evolved" leaving the original untouched
+    >>> m1 = m(a=1, b=2)
+    >>> m2 = m1.set('c', 3)
+    >>> m3 = m2.set('a', 5)
+    >>> m1
+    pmap({'a': 1, 'b': 2})
+    >>> m2
+    pmap({'a': 1, 'c': 3, 'b': 2})
+    >>> m3
+    pmap({'a': 5, 'c': 3, 'b': 2})
+    >>> m3['a']
+    5
+
+    # Evolution of nested persistent structures
+    >>> m4 = m(a=5, b=6, c=v(1, 2))
+    >>> m4.transform(('c', 1), 17)
+    pmap({'a': 5, 'c': pvector([1, 17]), 'b': 6})
+    >>> m5 = m(a=1, b=2)
+
+    # Evolve by merging with other mappings
+    >>> m5.update(m(a=2, c=3), {'a': 17, 'd': 35})
+    pmap({'a': 17, 'c': 3, 'b': 2, 'd': 35})
+    >>> pmap({'x': 1, 'y': 2}) + pmap({'y': 3, 'z': 4})
+    pmap({'y': 3, 'x': 1, 'z': 4})
+
+    # Dict-like methods to convert to list and iterate
+    >>> m3.items()
+    pvector([('a', 5), ('c', 3), ('b', 2)])
+    >>> list(m3)
+    ['a', 'c', 'b']
+
+.. _PSet:
+
+PSet
+~~~~
+With full support for the Set_ protocol PSet is meant as a drop in replacement to the built in set from a readers point
+of view. Support for the Hashable_ protocol also means that it can be used as key in Mappings_.
+
+Random access and insert is log32(n) where n is the size of the set.
+
+.. code:: python
+
+    >>> from pyrsistent import s
+
+    # No mutation of sets once created, you know the story...
+    >>> s1 = s(1, 2, 3, 2)
+    >>> s2 = s1.add(4)
+    >>> s3 = s1.remove(1)
+    >>> s1
+    pset([1, 2, 3])
+    >>> s2
+    pset([1, 2, 3, 4])
+    >>> s3
+    pset([2, 3])
+
+    # Full support for set operations
+    >>> s1 | s(3, 4, 5)
+    pset([1, 2, 3, 4, 5])
+    >>> s1 & s(3, 4, 5)
+    pset([3])
+    >>> s1 < s2
+    True
+    >>> s1 < s(3, 4, 5)
+    False
+
+.. _PRecord:
+
+PRecord
+~~~~~~~
+A PRecord is a PMap with a fixed set of specified fields. Records are declared as python classes inheriting
+from PRecord. Because it is a PMap it has full support for all Mapping methods such as iteration and element
+access using subscript notation.
+
+.. code:: python
+
+    >>> from pyrsistent import PRecord, field
+    >>> class ARecord(PRecord):
+    ...     x = field()
+    ...
+    >>> r = ARecord(x=3)
+    >>> r
+    ARecord(x=3)
+    >>> r.x
+    3
+    >>> r.set(x=2)
+    ARecord(x=2)
+    >>> r.set(y=2)
+    Traceback (most recent call last):
+    AttributeError: 'y' is not among the specified fields for ARecord
+
+Type information
+****************
+It is possible to add type information to the record to enforce type checks. Multiple allowed types can be specified
+by providing an iterable of types.
+
+.. code:: python
+
+    >>> class BRecord(PRecord):
+    ...     x = field(type=int)
+    ...     y = field(type=(int, type(None)))
+    ...
+    >>> BRecord(x=3, y=None)
+    BRecord(y=None, x=3)
+    >>> BRecord(x=3.0)
+    Traceback (most recent call last):
+    PTypeError: Invalid type for field BRecord.x, was float
+
+
+Custom types (classes) that are iterable should be wrapped in a tuple to prevent their
+members being added to the set of valid types.  Although Enums in particular are now 
+supported without wrapping, see #83 for more information.
+
+Mandatory fields
+****************
+Fields are not mandatory by default but can be specified as such. If fields are missing an
+*InvariantException* will be thrown which contains information about the missing fields.
+
+.. code:: python
+
+    >>> from pyrsistent import InvariantException
+    >>> class CRecord(PRecord):
+    ...     x = field(mandatory=True)
+    ...
+    >>> r = CRecord(x=3)
+    >>> try:
+    ...    r.discard('x')
+    ... except InvariantException as e:
+    ...    print(e.missing_fields)
+    ...
+    ('CRecord.x',)
+
+Invariants
+**********
+It is possible to add invariants that must hold when evolving the record. Invariants can be
+specified on both field and record level. If invariants fail an *InvariantException* will be
+thrown which contains information about the failing invariants. An invariant function should
+return a tuple consisting of a boolean that tells if the invariant holds or not and an object
+describing the invariant. This object can later be used to identify which invariant that failed.
+
+The global invariant function is only executed if all field invariants hold.
+
+Global invariants are inherited to subclasses.
+
+.. code:: python
+
+    >>> class RestrictedVector(PRecord):
+    ...     __invariant__ = lambda r: (r.y >= r.x, 'x larger than y')
+    ...     x = field(invariant=lambda x: (x > 0, 'x negative'))
+    ...     y = field(invariant=lambda y: (y > 0, 'y negative'))
+    ...
+    >>> r = RestrictedVector(y=3, x=2)
+    >>> try:
+    ...    r.set(x=-1, y=-2)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    ('y negative', 'x negative')
+    >>> try:
+    ...    r.set(x=2, y=1)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    ('x larger than y',)
+
+Invariants may also contain multiple assertions. For those cases the invariant function should
+return a tuple of invariant tuples as described above. This structure is reflected in the
+invariant_errors attribute of the exception which will contain tuples with data from all failed
+invariants. Eg:
+
+.. code:: python
+
+    >>> class EvenX(PRecord):
+    ...     x = field(invariant=lambda x: ((x > 0, 'x negative'), (x % 2 == 0, 'x odd')))
+    ...
+    >>> try:
+    ...    EvenX(x=-1)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    (('x negative', 'x odd'),)
+
+
+Factories
+*********
+It's possible to specify factory functions for fields. The factory function receives whatever
+is supplied as field value and the actual returned by the factory is assigned to the field
+given that any type and invariant checks hold.
+PRecords have a default factory specified as a static function on the class, create(). It takes
+a *Mapping* as argument and returns an instance of the specific record.
+If a record has fields of type PRecord the create() method of that record will
+be called to create the "sub record" if no factory has explicitly been specified to override
+this behaviour.
+
+.. code:: python
+
+    >>> class DRecord(PRecord):
+    ...     x = field(factory=int)
+    ...
+    >>> class ERecord(PRecord):
+    ...     d = field(type=DRecord)
+    ...
+    >>> ERecord.create({'d': {'x': '1'}})
+    ERecord(d=DRecord(x=1))
+
+Collection fields
+*****************
+It is also possible to have fields with ``pyrsistent`` collections.
+
+.. code:: python
+
+   >>> from pyrsistent import pset_field, pmap_field, pvector_field
+   >>> class MultiRecord(PRecord):
+   ...     set_of_ints = pset_field(int)
+   ...     map_int_to_str = pmap_field(int, str)
+   ...     vector_of_strs = pvector_field(str)
+   ...
+	
+Serialization
+*************
+PRecords support serialization back to dicts. Default serialization will take keys and values
+"as is" and output them into a dict. It is possible to specify custom serialization functions
+to take care of fields that require special treatment.
+
+.. code:: python
+
+    >>> from datetime import date
+    >>> class Person(PRecord):
+    ...     name = field(type=unicode)
+    ...     birth_date = field(type=date,
+    ...                        serializer=lambda format, d: d.strftime(format['date']))
+    ...
+    >>> john = Person(name=u'John', birth_date=date(1985, 10, 21))
+    >>> john.serialize({'date': '%Y-%m-%d'})
+    {'birth_date': '1985-10-21', 'name': u'John'}
+
+
+.. _instar: https://github.com/boxed/instar/
+
+.. _PClass:
+
+PClass
+~~~~~~
+A PClass is a python class with a fixed set of specified fields. PClasses are declared as python classes inheriting
+from PClass. It is defined the same way that PRecords are and behaves like a PRecord in all aspects except that it
+is not a PMap and hence not a collection but rather a plain Python object.
+
+.. code:: python
+
+    >>> from pyrsistent import PClass, field
+    >>> class AClass(PClass):
+    ...     x = field()
+    ...
+    >>> a = AClass(x=3)
+    >>> a
+    AClass(x=3)
+    >>> a.x
+    3
+
+
+Checked collections
+~~~~~~~~~~~~~~~~~~~
+Checked collections currently come in three flavors: CheckedPVector, CheckedPMap and CheckedPSet.
+
+.. code:: python
+
+    >>> from pyrsistent import CheckedPVector, CheckedPMap, CheckedPSet, thaw
+    >>> class Positives(CheckedPSet):
+    ...     __type__ = (long, int)
+    ...     __invariant__ = lambda n: (n >= 0, 'Negative')
+    ...
+    >>> class Lottery(PRecord):
+    ...     name = field(type=str)
+    ...     numbers = field(type=Positives, invariant=lambda p: (len(p) > 0, 'No numbers'))
+    ...
+    >>> class Lotteries(CheckedPVector):
+    ...     __type__ = Lottery
+    ...
+    >>> class LotteriesByDate(CheckedPMap):
+    ...     __key_type__ = date
+    ...     __value_type__ = Lotteries
+    ...
+    >>> lotteries = LotteriesByDate.create({date(2015, 2, 15): [{'name': 'SuperLotto', 'numbers': {1, 2, 3}},
+    ...                                                         {'name': 'MegaLotto',  'numbers': {4, 5, 6}}],
+    ...                                     date(2015, 2, 16): [{'name': 'SuperLotto', 'numbers': {3, 2, 1}},
+    ...                                                         {'name': 'MegaLotto',  'numbers': {6, 5, 4}}]})
+    >>> lotteries
+    LotteriesByDate({datetime.date(2015, 2, 15): Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')]), datetime.date(2015, 2, 16): Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')])})
+
+    # The checked versions support all operations that the corresponding
+    # unchecked types do
+    >>> lottery_0215 = lotteries[date(2015, 2, 15)]
+    >>> lottery_0215.transform([0, 'name'], 'SuperDuperLotto')
+    Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperDuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')])
+
+    # But also makes asserts that types and invariants hold
+    >>> lottery_0215.transform([0, 'name'], 999)
+    Traceback (most recent call last):
+    PTypeError: Invalid type for field Lottery.name, was int
+
+    >>> lottery_0215.transform([0, 'numbers'], set())
+    Traceback (most recent call last):
+    InvariantException: Field invariant failed
+
+    # They can be converted back to python built ins with either thaw()
+    # or serialize() (which provides possibilities to customize serialization)
+    >>> thaw(lottery_0215)
+    [{'numbers': set([1, 2, 3]), 'name': 'SuperLotto'}, {'numbers': set([4, 5, 6]), 'name': 'MegaLotto'}]
+    >>> lottery_0215.serialize()
+    [{'numbers': set([1, 2, 3]), 'name': 'SuperLotto'}, {'numbers': set([4, 5, 6]), 'name': 'MegaLotto'}]
+
+.. _transformations:
+
+Transformations
+~~~~~~~~~~~~~~~
+Transformations are inspired by the cool library instar_ for Clojure. They let you evolve PMaps and PVectors
+with arbitrarily deep/complex nesting using simple syntax and flexible matching syntax.
+
+The first argument to transformation is the path that points out the value to transform. The
+second is the transformation to perform. If the transformation is callable it will be applied
+to the value(s) matching the path. The path may also contain callables. In that case they are
+treated as matchers. If the matcher returns True for a specific key it is considered for transformation.
+
+.. code:: python
+
+    # Basic examples
+    >>> from pyrsistent import inc, freeze, thaw, rex, ny, discard
+    >>> v1 = freeze([1, 2, 3, 4, 5])
+    >>> v1.transform([2], inc)
+    pvector([1, 2, 4, 4, 5])
+    >>> v1.transform([lambda ix: 0 < ix < 4], 8)
+    pvector([1, 8, 8, 8, 5])
+    >>> v1.transform([lambda ix, v: ix == 0 or v == 5], 0)
+    pvector([0, 2, 3, 4, 0])
+
+    # The (a)ny matcher can be used to match anything
+    >>> v1.transform([ny], 8)
+    pvector([8, 8, 8, 8, 8])
+
+    # Regular expressions can be used for matching
+    >>> scores = freeze({'John': 12, 'Joseph': 34, 'Sara': 23})
+    >>> scores.transform([rex('^Jo')], 0)
+    pmap({'Joseph': 0, 'Sara': 23, 'John': 0})
+
+    # Transformations can be done on arbitrarily deep structures
+    >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
+    ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
+    ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
+    >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
+    >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
+    >>> very_short_news.articles[0].content
+    'A short article'
+    >>> very_short_news.articles[1].content
+    'A slightly long...'
+
+    # When nothing has been transformed the original data structure is kept
+    >>> short_news is news_paper
+    True
+    >>> very_short_news is news_paper
+    False
+    >>> very_short_news.articles[0] is news_paper.articles[0]
+    True
+
+    # There is a special transformation that can be used to discard elements. Also
+    # multiple transformations can be applied in one call
+    >>> thaw(news_paper.transform(['weather'], discard, ['articles', ny, 'content'], discard))
+    {'articles': [{'author': 'Sara'}, {'author': 'Steve'}]}
+
+Evolvers
+~~~~~~~~
+PVector, PMap and PSet all have support for a concept dubbed *evolvers*. An evolver acts like a mutable
+view of the underlying persistent data structure with "transaction like" semantics. No updates of the original
+data structure is ever performed, it is still fully immutable.
+
+The evolvers have a very limited API by design to discourage excessive, and inappropriate, usage as that would
+take us down the mutable road. In principle only basic mutation and element access functions are supported.
+Check out the documentation_ of each data structure for specific examples.
+
+Examples of when you may want to use an evolver instead of working directly with the data structure include:
+
+* Multiple updates are done to the same data structure and the intermediate results are of no
+  interest. In this case using an evolver may be a more efficient and easier to work with.
+* You need to pass a vector into a legacy function or a function that you have no control
+  over which performs in place mutations. In this case pass an evolver instance
+  instead and then create a new pvector from the evolver once the function returns.
+
+.. code:: python
+
+    >>> from pyrsistent import v
+
+    # In place mutation as when working with the built in counterpart
+    >>> v1 = v(1, 2, 3)
+    >>> e = v1.evolver()
+    >>> e[1] = 22
+    >>> e = e.append(4)
+    >>> e = e.extend([5, 6])
+    >>> e[5] += 1
+    >>> len(e)
+    6
+
+    # The evolver is considered *dirty* when it contains changes compared to the underlying vector
+    >>> e.is_dirty()
+    True
+
+    # But the underlying pvector still remains untouched
+    >>> v1
+    pvector([1, 2, 3])
+
+    # Once satisfied with the updates you can produce a new pvector containing the updates.
+    # The new pvector will share data with the original pvector in the same way that would have
+    # been done if only using operations on the pvector.
+    >>> v2 = e.persistent()
+    >>> v2
+    pvector([1, 22, 3, 4, 5, 7])
+
+    # The evolver is now no longer considered *dirty* as it contains no differences compared to the
+    # pvector just produced.
+    >>> e.is_dirty()
+    False
+
+    # You may continue to work with the same evolver without affecting the content of v2
+    >>> e[0] = 11
+
+    # Or create a new evolver from v2. The two evolvers can be updated independently but will both
+    # share data with v2 where possible.
+    >>> e2 = v2.evolver()
+    >>> e2[0] = 1111
+    >>> e.persistent()
+    pvector([11, 22, 3, 4, 5, 7])
+    >>> e2.persistent()
+    pvector([1111, 22, 3, 4, 5, 7])
+
+.. _freeze:
+.. _thaw:
+
+freeze and thaw
+~~~~~~~~~~~~~~~
+These functions are great when your cozy immutable world has to interact with the evil mutable world outside.
+
+.. code:: python
+
+    >>> from pyrsistent import freeze, thaw, v, m
+    >>> freeze([1, {'a': 3}])
+    pvector([1, pmap({'a': 3})])
+    >>> thaw(v(1, m(a=3)))
+    [1, {'a': 3}]
+
+Compatibility
+-------------
+
+Pyrsistent is developed and tested on Python 2.7, 3.5, 3.6, 3.7 and PyPy (Python 2 and 3 compatible). It will most
+likely work on all other versions >= 3.4 but no guarantees are given. :)
+
+Compatibility issues
+~~~~~~~~~~~~~~~~~~~~
+
+.. _27: https://github.com/tobgu/pyrsistent/issues/27
+
+There is currently one known compatibility issue when comparing built in sets and frozensets to PSets as discussed in 27_.
+It affects python 2 versions < 2.7.8 and python 3 versions < 3.4.0 and is due to a bug described in
+http://bugs.python.org/issue8743.
+
+Comparisons will fail or be incorrect when using the set/frozenset as left hand side of the comparison. As a workaround
+you need to either upgrade Python to a more recent version, avoid comparing sets/frozensets with PSets or always make
+sure to convert both sides of the comparison to the same type before performing the comparison.
+
+Performance
+-----------
+
+Pyrsistent is developed with performance in mind. Still, while some operations are nearly on par with their built in, 
+mutable, counterparts in terms of speed, other operations are slower. In the cases where attempts at
+optimizations have been done, speed has generally been valued over space.
+
+Pyrsistent comes with two API compatible flavors of PVector (on which PMap and PSet are based), one pure Python 
+implementation and one implemented as a C extension. The latter generally being 2 - 20 times faster than the former.
+The C extension will be used automatically when possible.
+
+The pure python implementation is fully PyPy compatible. Running it under PyPy speeds operations up considerably if 
+the structures are used heavily (if JITed), for some cases the performance is almost on par with the built in counterparts.
+
+Type hints
+----------
+
+PEP 561 style type hints for use with mypy and various editors are available for most types and functions in pyrsistent.
+
+Type classes for annotating your own code with pyrsistent types are also available under pyrsistent.typing.
+
+Installation
+------------
+
+pip install pyrsistent
+
+Documentation
+-------------
+
+Available at http://pyrsistent.readthedocs.org/
+
+Brief presentation available at http://slides.com/tobiasgustafsson/immutability-and-python/
+
+Contributors
+------------
+
+Tobias Gustafsson https://github.com/tobgu
+
+Christopher Armstrong https://github.com/radix
+
+Anders Hovmöller https://github.com/boxed
+
+Itamar Turner-Trauring https://github.com/itamarst
+
+Jonathan Lange https://github.com/jml
+
+Richard Futrell https://github.com/Futrell
+
+Jakob Hollenstein https://github.com/jkbjh
+
+David Honour https://github.com/foolswood
+
+David R. MacIver https://github.com/DRMacIver
+
+Marcus Ewert https://github.com/sarum90
+
+Jean-Paul Calderone https://github.com/exarkun
+
+Douglas Treadwell https://github.com/douglas-treadwell
+
+Travis Parker https://github.com/teepark
+
+Julian Berman https://github.com/Julian
+
+Dennis Tomas https://github.com/dtomas
+
+Neil Vyas https://github.com/neilvyas
+
+doozr https://github.com/doozr
+
+Kamil Galuszka https://github.com/galuszkak
+
+Tsuyoshi Hombashi https://github.com/thombashi
+
+nattofriends https://github.com/nattofriends
+
+agberk https://github.com/agberk
+
+Waleed Khan https://github.com/arxanas
+
+Jean-Louis Fuchs https://github.com/ganwell
+
+Carlos Corbacho https://github.com/ccorbacho
+
+Felix Yan https://github.com/felixonmars
+
+benrg https://github.com/benrg
+
+Jere Lahelma https://github.com/je-l
+
+Max Taggart https://github.com/MaxTaggart
+
+Vincent Philippon https://github.com/vphilippon
+
+Semen Zhydenko https://github.com/ss18
+
+Till Varoquaux  https://github.com/till-varoquaux
+
+Michal Kowalik https://github.com/michalvi
+
+ossdev07 https://github.com/ossdev07
+
+Kerry Olesen https://github.com/qhesz
+
+Contributing
+------------
+
+Want to contribute? That's great! If you experience problems please log them on GitHub. If you want to contribute code,
+please fork the repository and submit a pull request.
+
+Run tests
+~~~~~~~~~
+.. _tox: https://tox.readthedocs.io/en/latest/
+
+Tests can be executed using tox_.
+
+Install tox: ``pip install tox``
+
+Run test for Python 2.7: ``tox -epy27``
+
+Release
+~~~~~~~
+* Update CHANGES.txt
+* Update README with any new contributors and potential info needed.
+* Update _pyrsistent_version.py
+* python setup.py sdist upload
+* Commit and tag with new version: git add -u . && git commit -m 'Prepare version vX.Y.Z' && git tag -a vX.Y.Z -m 'vX.Y.Z'
+* Push commit and tags: git push && git push --tags
+
+Project status
+--------------
+Pyrsistent can be considered stable and mature (who knows, there may even be a 1.0 some day :-)). The project is
+maintained, bugs fixed, PRs reviewed and merged and new releases made. I currently do not have time for development
+of new features or functionality which I don't have use for myself. I'm more than happy to take PRs for new
+functionality though!
+
+There are a bunch of issues marked with ``enhancement`` and ``help wanted`` that contain requests for new functionality
+that would be nice to include. The level of difficulty and extend of the issues varies, please reach out to me if you're
+interested in working on any of them.
+
+If you feel that you have a grand master plan for where you would like Pyrsistent to go and have the time to put into
+it please don't hesitate to discuss this with me and submit PRs for it. If all goes well I'd be more than happy to add
+additional maintainers to the project!
diff --git a/contrib/python/pyrsistent/py2/_pyrsistent_version.py b/contrib/python/pyrsistent/py2/_pyrsistent_version.py
new file mode 100644
index 00000000000..d9cd96d18b4
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/_pyrsistent_version.py
@@ -0,0 +1 @@
+__version__ = '0.15.7'
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/__init__.py b/contrib/python/pyrsistent/py2/pyrsistent/__init__.py
new file mode 100644
index 00000000000..be299658f3f
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/__init__.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+from pyrsistent._pmap import pmap, m, PMap
+
+from pyrsistent._pvector import pvector, v, PVector
+
+from pyrsistent._pset import pset, s, PSet
+
+from pyrsistent._pbag import pbag, b, PBag
+
+from pyrsistent._plist import plist, l, PList
+
+from pyrsistent._pdeque import pdeque, dq, PDeque
+
+from pyrsistent._checked_types import (
+    CheckedPMap, CheckedPVector, CheckedPSet, InvariantException, CheckedKeyTypeError,
+    CheckedValueTypeError, CheckedType, optional)
+
+from pyrsistent._field_common import (
+    field, PTypeError, pset_field, pmap_field, pvector_field)
+
+from pyrsistent._precord import PRecord
+
+from pyrsistent._pclass import PClass, PClassMeta
+
+from pyrsistent._immutable import immutable
+
+from pyrsistent._helpers import freeze, thaw, mutant
+
+from pyrsistent._transformations import inc, discard, rex, ny
+
+from pyrsistent._toolz import get_in
+
+
+__all__ = ('pmap', 'm', 'PMap',
+           'pvector', 'v', 'PVector',
+           'pset', 's', 'PSet',
+           'pbag', 'b', 'PBag',
+           'plist', 'l', 'PList',
+           'pdeque', 'dq', 'PDeque',
+           'CheckedPMap', 'CheckedPVector', 'CheckedPSet', 'InvariantException', 'CheckedKeyTypeError', 'CheckedValueTypeError', 'CheckedType', 'optional',
+           'PRecord', 'field', 'pset_field', 'pmap_field', 'pvector_field',
+           'PClass', 'PClassMeta',
+           'immutable',
+           'freeze', 'thaw', 'mutant',
+           'get_in',
+           'inc', 'discard', 'rex', 'ny')
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_checked_types.py b/contrib/python/pyrsistent/py2/pyrsistent/_checked_types.py
new file mode 100644
index 00000000000..cb8e4692e7f
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_checked_types.py
@@ -0,0 +1,542 @@
+from ._compat import Iterable
+import six
+
+from pyrsistent._compat import Enum, string_types
+from pyrsistent._pmap import PMap, pmap
+from pyrsistent._pset import PSet, pset
+from pyrsistent._pvector import PythonPVector, python_pvector
+
+
+class CheckedType(object):
+    """
+    Marker class to enable creation and serialization of checked object graphs.
+    """
+    __slots__ = ()
+
+    @classmethod
+    def create(cls, source_data, _factory_fields=None):
+        raise NotImplementedError()
+
+    def serialize(self, format=None):
+        raise NotImplementedError()
+
+
+def _restore_pickle(cls, data):
+    return cls.create(data, _factory_fields=set())
+
+
+class InvariantException(Exception):
+    """
+    Exception raised from a :py:class:`CheckedType` when invariant tests fail or when a mandatory
+    field is missing.
+
+    Contains two fields of interest:
+    invariant_errors, a tuple of error data for the failing invariants
+    missing_fields, a tuple of strings specifying the missing names
+    """
+
+    def __init__(self, error_codes=(), missing_fields=(), *args, **kwargs):
+        self.invariant_errors = tuple(e() if callable(e) else e for e in error_codes)
+        self.missing_fields = missing_fields
+        super(InvariantException, self).__init__(*args, **kwargs)
+
+    def __str__(self):
+        return super(InvariantException, self).__str__() + \
+            ", invariant_errors=[{invariant_errors}], missing_fields=[{missing_fields}]".format(
+            invariant_errors=', '.join(str(e) for e in self.invariant_errors),
+            missing_fields=', '.join(self.missing_fields))
+
+
+_preserved_iterable_types = (
+        Enum,
+)
+"""Some types are themselves iterable, but we want to use the type itself and
+not its members for the type specification. This defines a set of such types
+that we explicitly preserve.
+
+Note that strings are not such types because the string inputs we pass in are
+values, not types.
+"""
+
+
+def maybe_parse_user_type(t):
+    """Try to coerce a user-supplied type directive into a list of types.
+
+    This function should be used in all places where a user specifies a type,
+    for consistency.
+
+    The policy for what defines valid user input should be clear from the implementation.
+    """
+    is_type = isinstance(t, type)
+    is_preserved = isinstance(t, type) and issubclass(t, _preserved_iterable_types)
+    is_string = isinstance(t, string_types)
+    is_iterable = isinstance(t, Iterable)
+
+    if is_preserved:
+        return [t]
+    elif is_string:
+        return [t]
+    elif is_type and not is_iterable:
+        return [t]
+    elif is_iterable:
+        # Recur to validate contained types as well.
+        ts = t
+        return tuple(e for t in ts for e in maybe_parse_user_type(t))
+    else:
+        # If this raises because `t` cannot be formatted, so be it.
+        raise TypeError(
+            'Type specifications must be types or strings. Input: {}'.format(t)
+        )
+
+
+def maybe_parse_many_user_types(ts):
+    # Just a different name to communicate that you're parsing multiple user
+    # inputs. `maybe_parse_user_type` handles the iterable case anyway.
+    return maybe_parse_user_type(ts)
+
+
+def _store_types(dct, bases, destination_name, source_name):
+    maybe_types = maybe_parse_many_user_types([
+        d[source_name]
+        for d in ([dct] + [b.__dict__ for b in bases]) if source_name in d
+    ])
+
+    dct[destination_name] = maybe_types
+
+
+def _merge_invariant_results(result):
+    verdict = True
+    data = []
+    for verd, dat in result:
+        if not verd:
+            verdict = False
+            data.append(dat)
+
+    return verdict, tuple(data)
+
+
+def wrap_invariant(invariant):
+    # Invariant functions may return the outcome of several tests
+    # In those cases the results have to be merged before being passed
+    # back to the client.
+    def f(*args, **kwargs):
+        result = invariant(*args, **kwargs)
+        if isinstance(result[0], bool):
+            return result
+
+        return _merge_invariant_results(result)
+
+    return f
+
+
+def _all_dicts(bases, seen=None):
+    """
+    Yield each class in ``bases`` and each of their base classes.
+    """
+    if seen is None:
+        seen = set()
+    for cls in bases:
+        if cls in seen:
+            continue
+        seen.add(cls)
+        yield cls.__dict__
+        for b in _all_dicts(cls.__bases__, seen):
+            yield b
+
+
+def store_invariants(dct, bases, destination_name, source_name):
+    # Invariants are inherited
+    invariants = []
+    for ns in [dct] + list(_all_dicts(bases)):
+        try:
+            invariant = ns[source_name]
+        except KeyError:
+            continue
+        invariants.append(invariant)
+
+    if not all(callable(invariant) for invariant in invariants):
+        raise TypeError('Invariants must be callable')
+    dct[destination_name] = tuple(wrap_invariant(inv) for inv in invariants)
+
+
+class _CheckedTypeMeta(type):
+    def __new__(mcs, name, bases, dct):
+        _store_types(dct, bases, '_checked_types', '__type__')
+        store_invariants(dct, bases, '_checked_invariants', '__invariant__')
+
+        def default_serializer(self, _, value):
+            if isinstance(value, CheckedType):
+                return value.serialize()
+            return value
+
+        dct.setdefault('__serializer__', default_serializer)
+
+        dct['__slots__'] = ()
+
+        return super(_CheckedTypeMeta, mcs).__new__(mcs, name, bases, dct)
+
+
+class CheckedTypeError(TypeError):
+    def __init__(self, source_class, expected_types, actual_type, actual_value, *args, **kwargs):
+        super(CheckedTypeError, self).__init__(*args, **kwargs)
+        self.source_class = source_class
+        self.expected_types = expected_types
+        self.actual_type = actual_type
+        self.actual_value = actual_value
+
+
+class CheckedKeyTypeError(CheckedTypeError):
+    """
+    Raised when trying to set a value using a key with a type that doesn't match the declared type.
+
+    Attributes:
+    source_class -- The class of the collection
+    expected_types  -- Allowed types
+    actual_type -- The non matching type
+    actual_value -- Value of the variable with the non matching type
+    """
+    pass
+
+
+class CheckedValueTypeError(CheckedTypeError):
+    """
+    Raised when trying to set a value using a key with a type that doesn't match the declared type.
+
+    Attributes:
+    source_class -- The class of the collection
+    expected_types  -- Allowed types
+    actual_type -- The non matching type
+    actual_value -- Value of the variable with the non matching type
+    """
+    pass
+
+
+def _get_class(type_name):
+    module_name, class_name = type_name.rsplit('.', 1)
+    module = __import__(module_name, fromlist=[class_name])
+    return getattr(module, class_name)
+
+
+def get_type(typ):
+    if isinstance(typ, type):
+        return typ
+
+    return _get_class(typ)
+
+
+def get_types(typs):
+    return [get_type(typ) for typ in typs]
+
+
+def _check_types(it, expected_types, source_class, exception_type=CheckedValueTypeError):
+    if expected_types:
+        for e in it:
+            if not any(isinstance(e, get_type(t)) for t in expected_types):
+                actual_type = type(e)
+                msg = "Type {source_class} can only be used with {expected_types}, not {actual_type}".format(
+                    source_class=source_class.__name__,
+                    expected_types=tuple(get_type(et).__name__ for et in expected_types),
+                    actual_type=actual_type.__name__)
+                raise exception_type(source_class, expected_types, actual_type, e, msg)
+
+
+def _invariant_errors(elem, invariants):
+    return [data for valid, data in (invariant(elem) for invariant in invariants) if not valid]
+
+
+def _invariant_errors_iterable(it, invariants):
+    return sum([_invariant_errors(elem, invariants) for elem in it], [])
+
+
+def optional(*typs):
+    """ Convenience function to specify that a value may be of any of the types in type 'typs' or None """
+    return tuple(typs) + (type(None),)
+
+
+def _checked_type_create(cls, source_data, _factory_fields=None, ignore_extra=False):
+    if isinstance(source_data, cls):
+        return source_data
+
+    # Recursively apply create methods of checked types if the types of the supplied data
+    # does not match any of the valid types.
+    types = get_types(cls._checked_types)
+    checked_type = next((t for t in types if issubclass(t, CheckedType)), None)
+    if checked_type:
+        return cls([checked_type.create(data, ignore_extra=ignore_extra)
+                    if not any(isinstance(data, t) for t in types) else data
+                    for data in source_data])
+
+    return cls(source_data)
+
+@six.add_metaclass(_CheckedTypeMeta)
+class CheckedPVector(PythonPVector, CheckedType):
+    """
+    A CheckedPVector is a PVector which allows specifying type and invariant checks.
+
+    >>> class Positives(CheckedPVector):
+    ...     __type__ = (long, int)
+    ...     __invariant__ = lambda n: (n >= 0, 'Negative')
+    ...
+    >>> Positives([1, 2, 3])
+    Positives([1, 2, 3])
+    """
+
+    __slots__ = ()
+
+    def __new__(cls, initial=()):
+        if type(initial) == PythonPVector:
+            return super(CheckedPVector, cls).__new__(cls, initial._count, initial._shift, initial._root, initial._tail)
+
+        return CheckedPVector.Evolver(cls, python_pvector()).extend(initial).persistent()
+
+    def set(self, key, value):
+        return self.evolver().set(key, value).persistent()
+
+    def append(self, val):
+        return self.evolver().append(val).persistent()
+
+    def extend(self, it):
+        return self.evolver().extend(it).persistent()
+
+    create = classmethod(_checked_type_create)
+
+    def serialize(self, format=None):
+        serializer = self.__serializer__
+        return list(serializer(format, v) for v in self)
+
+    def __reduce__(self):
+        # Pickling support
+        return _restore_pickle, (self.__class__, list(self),)
+
+    class Evolver(PythonPVector.Evolver):
+        __slots__ = ('_destination_class', '_invariant_errors')
+
+        def __init__(self, destination_class, vector):
+            super(CheckedPVector.Evolver, self).__init__(vector)
+            self._destination_class = destination_class
+            self._invariant_errors = []
+
+        def _check(self, it):
+            _check_types(it, self._destination_class._checked_types, self._destination_class)
+            error_data = _invariant_errors_iterable(it, self._destination_class._checked_invariants)
+            self._invariant_errors.extend(error_data)
+
+        def __setitem__(self, key, value):
+            self._check([value])
+            return super(CheckedPVector.Evolver, self).__setitem__(key, value)
+
+        def append(self, elem):
+            self._check([elem])
+            return super(CheckedPVector.Evolver, self).append(elem)
+
+        def extend(self, it):
+            it = list(it)
+            self._check(it)
+            return super(CheckedPVector.Evolver, self).extend(it)
+
+        def persistent(self):
+            if self._invariant_errors:
+                raise InvariantException(error_codes=self._invariant_errors)
+
+            result = self._orig_pvector
+            if self.is_dirty() or (self._destination_class != type(self._orig_pvector)):
+                pv = super(CheckedPVector.Evolver, self).persistent().extend(self._extra_tail)
+                result = self._destination_class(pv)
+                self._reset(result)
+
+            return result
+
+    def __repr__(self):
+        return self.__class__.__name__ + "({0})".format(self.tolist())
+
+    __str__ = __repr__
+
+    def evolver(self):
+        return CheckedPVector.Evolver(self.__class__, self)
+
+
+@six.add_metaclass(_CheckedTypeMeta)
+class CheckedPSet(PSet, CheckedType):
+    """
+    A CheckedPSet is a PSet which allows specifying type and invariant checks.
+
+    >>> class Positives(CheckedPSet):
+    ...     __type__ = (long, int)
+    ...     __invariant__ = lambda n: (n >= 0, 'Negative')
+    ...
+    >>> Positives([1, 2, 3])
+    Positives([1, 2, 3])
+    """
+
+    __slots__ = ()
+
+    def __new__(cls, initial=()):
+        if type(initial) is PMap:
+            return super(CheckedPSet, cls).__new__(cls, initial)
+
+        evolver = CheckedPSet.Evolver(cls, pset())
+        for e in initial:
+            evolver.add(e)
+
+        return evolver.persistent()
+
+    def __repr__(self):
+        return self.__class__.__name__ + super(CheckedPSet, self).__repr__()[4:]
+
+    def __str__(self):
+        return self.__repr__()
+
+    def serialize(self, format=None):
+        serializer = self.__serializer__
+        return set(serializer(format, v) for v in self)
+
+    create = classmethod(_checked_type_create)
+
+    def __reduce__(self):
+        # Pickling support
+        return _restore_pickle, (self.__class__, list(self),)
+
+    def evolver(self):
+        return CheckedPSet.Evolver(self.__class__, self)
+
+    class Evolver(PSet._Evolver):
+        __slots__ = ('_destination_class', '_invariant_errors')
+
+        def __init__(self, destination_class, original_set):
+            super(CheckedPSet.Evolver, self).__init__(original_set)
+            self._destination_class = destination_class
+            self._invariant_errors = []
+
+        def _check(self, it):
+            _check_types(it, self._destination_class._checked_types, self._destination_class)
+            error_data = _invariant_errors_iterable(it, self._destination_class._checked_invariants)
+            self._invariant_errors.extend(error_data)
+
+        def add(self, element):
+            self._check([element])
+            self._pmap_evolver[element] = True
+            return self
+
+        def persistent(self):
+            if self._invariant_errors:
+                raise InvariantException(error_codes=self._invariant_errors)
+
+            if self.is_dirty() or self._destination_class != type(self._original_pset):
+                return self._destination_class(self._pmap_evolver.persistent())
+
+            return self._original_pset
+
+
+class _CheckedMapTypeMeta(type):
+    def __new__(mcs, name, bases, dct):
+        _store_types(dct, bases, '_checked_key_types', '__key_type__')
+        _store_types(dct, bases, '_checked_value_types', '__value_type__')
+        store_invariants(dct, bases, '_checked_invariants', '__invariant__')
+
+        def default_serializer(self, _, key, value):
+            sk = key
+            if isinstance(key, CheckedType):
+                sk = key.serialize()
+
+            sv = value
+            if isinstance(value, CheckedType):
+                sv = value.serialize()
+
+            return sk, sv
+
+        dct.setdefault('__serializer__', default_serializer)
+
+        dct['__slots__'] = ()
+
+        return super(_CheckedMapTypeMeta, mcs).__new__(mcs, name, bases, dct)
+
+# Marker object
+_UNDEFINED_CHECKED_PMAP_SIZE = object()
+
+
+@six.add_metaclass(_CheckedMapTypeMeta)
+class CheckedPMap(PMap, CheckedType):
+    """
+    A CheckedPMap is a PMap which allows specifying type and invariant checks.
+
+    >>> class IntToFloatMap(CheckedPMap):
+    ...     __key_type__ = int
+    ...     __value_type__ = float
+    ...     __invariant__ = lambda k, v: (int(v) == k, 'Invalid mapping')
+    ...
+    >>> IntToFloatMap({1: 1.5, 2: 2.25})
+    IntToFloatMap({1: 1.5, 2: 2.25})
+    """
+
+    __slots__ = ()
+
+    def __new__(cls, initial={}, size=_UNDEFINED_CHECKED_PMAP_SIZE):
+        if size is not _UNDEFINED_CHECKED_PMAP_SIZE:
+            return super(CheckedPMap, cls).__new__(cls, size, initial)
+
+        evolver = CheckedPMap.Evolver(cls, pmap())
+        for k, v in initial.items():
+            evolver.set(k, v)
+
+        return evolver.persistent()
+
+    def evolver(self):
+        return CheckedPMap.Evolver(self.__class__, self)
+
+    def __repr__(self):
+        return self.__class__.__name__ + "({0})".format(str(dict(self)))
+
+    __str__ = __repr__
+
+    def serialize(self, format=None):
+        serializer = self.__serializer__
+        return dict(serializer(format, k, v) for k, v in self.items())
+
+    @classmethod
+    def create(cls, source_data, _factory_fields=None):
+        if isinstance(source_data, cls):
+            return source_data
+
+        # Recursively apply create methods of checked types if the types of the supplied data
+        # does not match any of the valid types.
+        key_types = get_types(cls._checked_key_types)
+        checked_key_type = next((t for t in key_types if issubclass(t, CheckedType)), None)
+        value_types = get_types(cls._checked_value_types)
+        checked_value_type = next((t for t in value_types if issubclass(t, CheckedType)), None)
+
+        if checked_key_type or checked_value_type:
+            return cls(dict((checked_key_type.create(key) if checked_key_type and not any(isinstance(key, t) for t in key_types) else key,
+                             checked_value_type.create(value) if checked_value_type and not any(isinstance(value, t) for t in value_types) else value)
+                            for key, value in source_data.items()))
+
+        return cls(source_data)
+
+    def __reduce__(self):
+        # Pickling support
+        return _restore_pickle, (self.__class__, dict(self),)
+
+    class Evolver(PMap._Evolver):
+        __slots__ = ('_destination_class', '_invariant_errors')
+
+        def __init__(self, destination_class, original_map):
+            super(CheckedPMap.Evolver, self).__init__(original_map)
+            self._destination_class = destination_class
+            self._invariant_errors = []
+
+        def set(self, key, value):
+            _check_types([key], self._destination_class._checked_key_types, self._destination_class, CheckedKeyTypeError)
+            _check_types([value], self._destination_class._checked_value_types, self._destination_class)
+            self._invariant_errors.extend(data for valid, data in (invariant(key, value)
+                                                                   for invariant in self._destination_class._checked_invariants)
+                                          if not valid)
+
+            return super(CheckedPMap.Evolver, self).set(key, value)
+
+        def persistent(self):
+            if self._invariant_errors:
+                raise InvariantException(error_codes=self._invariant_errors)
+
+            if self.is_dirty() or type(self._original_pmap) != self._destination_class:
+                return self._destination_class(self._buckets_evolver.persistent(), self._size)
+
+            return self._original_pmap
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_compat.py b/contrib/python/pyrsistent/py2/pyrsistent/_compat.py
new file mode 100644
index 00000000000..e728586afe2
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_compat.py
@@ -0,0 +1,31 @@
+from six import string_types
+
+
+# enum compat
+try:
+    from enum import Enum
+except:
+    class Enum(object): pass
+    # no objects will be instances of this class
+
+# collections compat
+try:
+    from collections.abc import (
+        Container,
+        Hashable,
+        Iterable,
+        Mapping,
+        Sequence,
+        Set,
+        Sized,
+    )
+except ImportError:
+    from collections import (
+        Container,
+        Hashable,
+        Iterable,
+        Mapping,
+        Sequence,
+        Set,
+        Sized,
+    )
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_field_common.py b/contrib/python/pyrsistent/py2/pyrsistent/_field_common.py
new file mode 100644
index 00000000000..c2e461d2bd5
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_field_common.py
@@ -0,0 +1,330 @@
+import six
+import sys
+
+from pyrsistent._checked_types import (
+    CheckedPMap,
+    CheckedPSet,
+    CheckedPVector,
+    CheckedType,
+    InvariantException,
+    _restore_pickle,
+    get_type,
+    maybe_parse_user_type,
+    maybe_parse_many_user_types,
+)
+from pyrsistent._checked_types import optional as optional_type
+from pyrsistent._checked_types import wrap_invariant
+import inspect
+
+PY2 = sys.version_info[0] < 3
+
+
+def set_fields(dct, bases, name):
+    dct[name] = dict(sum([list(b.__dict__.get(name, {}).items()) for b in bases], []))
+
+    for k, v in list(dct.items()):
+        if isinstance(v, _PField):
+            dct[name][k] = v
+            del dct[k]
+
+
+def check_global_invariants(subject, invariants):
+    error_codes = tuple(error_code for is_ok, error_code in
+                        (invariant(subject) for invariant in invariants) if not is_ok)
+    if error_codes:
+        raise InvariantException(error_codes, (), 'Global invariant failed')
+
+
+def serialize(serializer, format, value):
+    if isinstance(value, CheckedType) and serializer is PFIELD_NO_SERIALIZER:
+        return value.serialize(format)
+
+    return serializer(format, value)
+
+
+def check_type(destination_cls, field, name, value):
+    if field.type and not any(isinstance(value, get_type(t)) for t in field.type):
+        actual_type = type(value)
+        message = "Invalid type for field {0}.{1}, was {2}".format(destination_cls.__name__, name, actual_type.__name__)
+        raise PTypeError(destination_cls, name, field.type, actual_type, message)
+
+
+def is_type_cls(type_cls, field_type):
+    if type(field_type) is set:
+        return True
+    types = tuple(field_type)
+    if len(types) == 0:
+        return False
+    return issubclass(get_type(types[0]), type_cls)
+
+
+def is_field_ignore_extra_complaint(type_cls, field, ignore_extra):
+    # ignore_extra param has default False value, for speed purpose no need to propagate False
+    if not ignore_extra:
+        return False
+
+    if not is_type_cls(type_cls, field.type):
+        return False
+
+    if PY2:
+        return 'ignore_extra' in inspect.getargspec(field.factory).args
+    else:
+        return 'ignore_extra' in inspect.signature(field.factory).parameters
+
+
+
+class _PField(object):
+    __slots__ = ('type', 'invariant', 'initial', 'mandatory', '_factory', 'serializer')
+
+    def __init__(self, type, invariant, initial, mandatory, factory, serializer):
+        self.type = type
+        self.invariant = invariant
+        self.initial = initial
+        self.mandatory = mandatory
+        self._factory = factory
+        self.serializer = serializer
+
+    @property
+    def factory(self):
+        # If no factory is specified and the type is another CheckedType use the factory method of that CheckedType
+        if self._factory is PFIELD_NO_FACTORY and len(self.type) == 1:
+            typ = get_type(tuple(self.type)[0])
+            if issubclass(typ, CheckedType):
+                return typ.create
+
+        return self._factory
+
+PFIELD_NO_TYPE = ()
+PFIELD_NO_INVARIANT = lambda _: (True, None)
+PFIELD_NO_FACTORY = lambda x: x
+PFIELD_NO_INITIAL = object()
+PFIELD_NO_SERIALIZER = lambda _, value: value
+
+
+def field(type=PFIELD_NO_TYPE, invariant=PFIELD_NO_INVARIANT, initial=PFIELD_NO_INITIAL,
+          mandatory=False, factory=PFIELD_NO_FACTORY, serializer=PFIELD_NO_SERIALIZER):
+    """
+    Field specification factory for :py:class:`PRecord`.
+
+    :param type: a type or iterable with types that are allowed for this field
+    :param invariant: a function specifying an invariant that must hold for the field
+    :param initial: value of field if not specified when instantiating the record
+    :param mandatory: boolean specifying if the field is mandatory or not
+    :param factory: function called when field is set.
+    :param serializer: function that returns a serialized version of the field
+    """
+
+    # NB: We have to check this predicate separately from the predicates in
+    # `maybe_parse_user_type` et al. because this one is related to supporting
+    # the argspec for `field`, while those are related to supporting the valid
+    # ways to specify types.
+
+    # Multiple types must be passed in one of the following containers. Note
+    # that a type that is a subclass of one of these containers, like a
+    # `collections.namedtuple`, will work as expected, since we check
+    # `isinstance` and not `issubclass`.
+    if isinstance(type, (list, set, tuple)):
+        types = set(maybe_parse_many_user_types(type))
+    else:
+        types = set(maybe_parse_user_type(type))
+
+    invariant_function = wrap_invariant(invariant) if invariant != PFIELD_NO_INVARIANT and callable(invariant) else invariant
+    field = _PField(type=types, invariant=invariant_function, initial=initial,
+                    mandatory=mandatory, factory=factory, serializer=serializer)
+
+    _check_field_parameters(field)
+
+    return field
+
+
+def _check_field_parameters(field):
+    for t in field.type:
+        if not isinstance(t, type) and not isinstance(t, six.string_types):
+            raise TypeError('Type parameter expected, not {0}'.format(type(t)))
+
+    if field.initial is not PFIELD_NO_INITIAL and \
+            not callable(field.initial) and \
+            field.type and not any(isinstance(field.initial, t) for t in field.type):
+        raise TypeError('Initial has invalid type {0}'.format(type(field.initial)))
+
+    if not callable(field.invariant):
+        raise TypeError('Invariant must be callable')
+
+    if not callable(field.factory):
+        raise TypeError('Factory must be callable')
+
+    if not callable(field.serializer):
+        raise TypeError('Serializer must be callable')
+
+
+class PTypeError(TypeError):
+    """
+    Raised when trying to assign a value with a type that doesn't match the declared type.
+
+    Attributes:
+    source_class -- The class of the record
+    field -- Field name
+    expected_types  -- Types allowed for the field
+    actual_type -- The non matching type
+    """
+    def __init__(self, source_class, field, expected_types, actual_type, *args, **kwargs):
+        super(PTypeError, self).__init__(*args, **kwargs)
+        self.source_class = source_class
+        self.field = field
+        self.expected_types = expected_types
+        self.actual_type = actual_type
+
+
+SEQ_FIELD_TYPE_SUFFIXES = {
+    CheckedPVector: "PVector",
+    CheckedPSet: "PSet",
+}
+
+# Global dictionary to hold auto-generated field types: used for unpickling
+_seq_field_types = {}
+
+def _restore_seq_field_pickle(checked_class, item_type, data):
+    """Unpickling function for auto-generated PVec/PSet field types."""
+    type_ = _seq_field_types[checked_class, item_type]
+    return _restore_pickle(type_, data)
+
+def _types_to_names(types):
+    """Convert a tuple of types to a human-readable string."""
+    return "".join(get_type(typ).__name__.capitalize() for typ in types)
+
+def _make_seq_field_type(checked_class, item_type):
+    """Create a subclass of the given checked class with the given item type."""
+    type_ = _seq_field_types.get((checked_class, item_type))
+    if type_ is not None:
+        return type_
+
+    class TheType(checked_class):
+        __type__ = item_type
+
+        def __reduce__(self):
+            return (_restore_seq_field_pickle,
+                    (checked_class, item_type, list(self)))
+
+    suffix = SEQ_FIELD_TYPE_SUFFIXES[checked_class]
+    TheType.__name__ = _types_to_names(TheType._checked_types) + suffix
+    _seq_field_types[checked_class, item_type] = TheType
+    return TheType
+
+def _sequence_field(checked_class, item_type, optional, initial):
+    """
+    Create checked field for either ``PSet`` or ``PVector``.
+
+    :param checked_class: ``CheckedPSet`` or ``CheckedPVector``.
+    :param item_type: The required type for the items in the set.
+    :param optional: If true, ``None`` can be used as a value for
+        this field.
+    :param initial: Initial value to pass to factory.
+
+    :return: A ``field`` containing a checked class.
+    """
+    TheType = _make_seq_field_type(checked_class, item_type)
+
+    if optional:
+        def factory(argument):
+            if argument is None:
+                return None
+            else:
+                return TheType.create(argument)
+    else:
+        factory = TheType.create
+
+    return field(type=optional_type(TheType) if optional else TheType,
+                 factory=factory, mandatory=True,
+                 initial=factory(initial))
+
+
+def pset_field(item_type, optional=False, initial=()):
+    """
+    Create checked ``PSet`` field.
+
+    :param item_type: The required type for the items in the set.
+    :param optional: If true, ``None`` can be used as a value for
+        this field.
+    :param initial: Initial value to pass to factory if no value is given
+        for the field.
+
+    :return: A ``field`` containing a ``CheckedPSet`` of the given type.
+    """
+    return _sequence_field(CheckedPSet, item_type, optional,
+                           initial)
+
+
+def pvector_field(item_type, optional=False, initial=()):
+    """
+    Create checked ``PVector`` field.
+
+    :param item_type: The required type for the items in the vector.
+    :param optional: If true, ``None`` can be used as a value for
+        this field.
+    :param initial: Initial value to pass to factory if no value is given
+        for the field.
+
+    :return: A ``field`` containing a ``CheckedPVector`` of the given type.
+    """
+    return _sequence_field(CheckedPVector, item_type, optional,
+                           initial)
+
+
+_valid = lambda item: (True, "")
+
+
+# Global dictionary to hold auto-generated field types: used for unpickling
+_pmap_field_types = {}
+
+def _restore_pmap_field_pickle(key_type, value_type, data):
+    """Unpickling function for auto-generated PMap field types."""
+    type_ = _pmap_field_types[key_type, value_type]
+    return _restore_pickle(type_, data)
+
+def _make_pmap_field_type(key_type, value_type):
+    """Create a subclass of CheckedPMap with the given key and value types."""
+    type_ = _pmap_field_types.get((key_type, value_type))
+    if type_ is not None:
+        return type_
+
+    class TheMap(CheckedPMap):
+        __key_type__ = key_type
+        __value_type__ = value_type
+
+        def __reduce__(self):
+            return (_restore_pmap_field_pickle,
+                    (self.__key_type__, self.__value_type__, dict(self)))
+
+    TheMap.__name__ = "{0}To{1}PMap".format(
+        _types_to_names(TheMap._checked_key_types),
+        _types_to_names(TheMap._checked_value_types))
+    _pmap_field_types[key_type, value_type] = TheMap
+    return TheMap
+
+
+def pmap_field(key_type, value_type, optional=False, invariant=PFIELD_NO_INVARIANT):
+    """
+    Create a checked ``PMap`` field.
+
+    :param key: The required type for the keys of the map.
+    :param value: The required type for the values of the map.
+    :param optional: If true, ``None`` can be used as a value for
+        this field.
+    :param invariant: Pass-through to ``field``.
+
+    :return: A ``field`` containing a ``CheckedPMap``.
+    """
+    TheMap = _make_pmap_field_type(key_type, value_type)
+
+    if optional:
+        def factory(argument):
+            if argument is None:
+                return None
+            else:
+                return TheMap.create(argument)
+    else:
+        factory = TheMap.create
+
+    return field(mandatory=True, initial=TheMap(),
+                 type=optional_type(TheMap) if optional else TheMap,
+                 factory=factory, invariant=invariant)
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_helpers.py b/contrib/python/pyrsistent/py2/pyrsistent/_helpers.py
new file mode 100644
index 00000000000..a56cc870e80
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_helpers.py
@@ -0,0 +1,82 @@
+from functools import wraps
+import six
+from pyrsistent._pmap import PMap, pmap
+from pyrsistent._pset import PSet, pset
+from pyrsistent._pvector import PVector, pvector
+
+
+def freeze(o):
+    """
+    Recursively convert simple Python containers into pyrsistent versions
+    of those containers.
+
+    - list is converted to pvector, recursively
+    - dict is converted to pmap, recursively on values (but not keys)
+    - set is converted to pset, but not recursively
+    - tuple is converted to tuple, recursively.
+
+    Sets and dict keys are not recursively frozen because they do not contain
+    mutable data by convention. The main exception to this rule is that
+    dict keys and set elements are often instances of mutable objects that
+    support hash-by-id, which this function can't convert anyway.
+
+    >>> freeze(set([1, 2]))
+    pset([1, 2])
+    >>> freeze([1, {'a': 3}])
+    pvector([1, pmap({'a': 3})])
+    >>> freeze((1, []))
+    (1, pvector([]))
+    """
+    typ = type(o)
+    if typ is dict:
+        return pmap(dict((k, freeze(v)) for k, v in six.iteritems(o)))
+    if typ is list:
+        return pvector(map(freeze, o))
+    if typ is tuple:
+        return tuple(map(freeze, o))
+    if typ is set:
+        return pset(o)
+    return o
+
+
+def thaw(o):
+    """
+    Recursively convert pyrsistent containers into simple Python containers.
+
+    - pvector is converted to list, recursively
+    - pmap is converted to dict, recursively on values (but not keys)
+    - pset is converted to set, but not recursively
+    - tuple is converted to tuple, recursively.
+
+    >>> from pyrsistent import s, m, v
+    >>> thaw(s(1, 2))
+    set([1, 2])
+    >>> thaw(v(1, m(a=3)))
+    [1, {'a': 3}]
+    >>> thaw((1, v()))
+    (1, [])
+    """
+    if isinstance(o, PVector):
+        return list(map(thaw, o))
+    if isinstance(o, PMap):
+        return dict((k, thaw(v)) for k, v in o.iteritems())
+    if isinstance(o, PSet):
+        return set(o)
+    if type(o) is tuple:
+        return tuple(map(thaw, o))
+    return o
+
+
+def mutant(fn):
+    """
+    Convenience decorator to isolate mutation to within the decorated function (with respect
+    to the input arguments).
+
+    All arguments to the decorated function will be frozen so that they are guaranteed not to change.
+    The return value is also frozen.
+    """
+    @wraps(fn)
+    def inner_f(*args, **kwargs):
+        return freeze(fn(*[freeze(e) for e in args], **dict(freeze(item) for item in kwargs.items())))
+
+    return inner_f
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_immutable.py b/contrib/python/pyrsistent/py2/pyrsistent/_immutable.py
new file mode 100644
index 00000000000..a89bd7552f0
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_immutable.py
@@ -0,0 +1,105 @@
+import sys
+
+import six
+
+
+def immutable(members='', name='Immutable', verbose=False):
+    """
+    Produces a class that either can be used standalone or as a base class for persistent classes.
+
+    This is a thin wrapper around a named tuple.
+
+    Constructing a type and using it to instantiate objects:
+
+    >>> Point = immutable('x, y', name='Point')
+    >>> p = Point(1, 2)
+    >>> p2 = p.set(x=3)
+    >>> p
+    Point(x=1, y=2)
+    >>> p2
+    Point(x=3, y=2)
+
+    Inheriting from a constructed type. In this case no type name needs to be supplied:
+
+    >>> class PositivePoint(immutable('x, y')):
+    ...     __slots__ = tuple()
+    ...     def __new__(cls, x, y):
+    ...         if x > 0 and y > 0:
+    ...             return super(PositivePoint, cls).__new__(cls, x, y)
+    ...         raise Exception('Coordinates must be positive!')
+    ...
+    >>> p = PositivePoint(1, 2)
+    >>> p.set(x=3)
+    PositivePoint(x=3, y=2)
+    >>> p.set(y=-3)
+    Traceback (most recent call last):
+    Exception: Coordinates must be positive!
+
+    The persistent class also supports the notion of frozen members. The value of a frozen member
+    cannot be updated. For example it could be used to implement an ID that should remain the same
+    over time. A frozen member is denoted by a trailing underscore.
+
+    >>> Point = immutable('x, y, id_', name='Point')
+    >>> p = Point(1, 2, id_=17)
+    >>> p.set(x=3)
+    Point(x=3, y=2, id_=17)
+    >>> p.set(id_=18)
+    Traceback (most recent call last):
+    AttributeError: Cannot set frozen members id_
+    """
+
+    if isinstance(members, six.string_types):
+        members = members.replace(',', ' ').split()
+
+    def frozen_member_test():
+        frozen_members = ["'%s'" % f for f in members if f.endswith('_')]
+        if frozen_members:
+            return """
+        frozen_fields = fields_to_modify & set([{frozen_members}])
+        if frozen_fields:
+            raise AttributeError('Cannot set frozen members %s' % ', '.join(frozen_fields))
+            """.format(frozen_members=', '.join(frozen_members))
+
+        return ''
+
+    verbose_string = ""
+    if sys.version_info < (3, 7):
+        # Verbose is no longer supported in Python 3.7
+        verbose_string = ", verbose={verbose}".format(verbose=verbose)
+
+    quoted_members = ', '.join("'%s'" % m for m in members)
+    template = """
+class {class_name}(namedtuple('ImmutableBase', [{quoted_members}]{verbose_string})):
+    __slots__ = tuple()
+
+    def __repr__(self):
+        return super({class_name}, self).__repr__().replace('ImmutableBase', self.__class__.__name__)
+
+    def set(self, **kwargs):
+        if not kwargs:
+            return self
+
+        fields_to_modify = set(kwargs.keys())
+        if not fields_to_modify <= {member_set}:
+            raise AttributeError("'%s' is not a member" % ', '.join(fields_to_modify - {member_set}))
+
+        {frozen_member_test}
+
+        return self.__class__.__new__(self.__class__, *map(kwargs.pop, [{quoted_members}], self))
+""".format(quoted_members=quoted_members,
+               member_set="set([%s])" % quoted_members if quoted_members else 'set()',
+               frozen_member_test=frozen_member_test(),
+               verbose_string=verbose_string,
+               class_name=name)
+
+    if verbose:
+        print(template)
+
+    from collections import namedtuple
+    namespace = dict(namedtuple=namedtuple, __name__='pyrsistent_immutable')
+    try:
+        six.exec_(template, namespace)
+    except SyntaxError as e:
+        raise SyntaxError(e.message + ':\n' + template)
+
+    return namespace[name]
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_pbag.py b/contrib/python/pyrsistent/py2/pyrsistent/_pbag.py
new file mode 100644
index 00000000000..9905e9a6e38
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_pbag.py
@@ -0,0 +1,267 @@
+from ._compat import Container, Iterable, Sized, Hashable
+from functools import reduce
+from pyrsistent._pmap import pmap
+
+
+def _add_to_counters(counters, element):
+    return counters.set(element, counters.get(element, 0) + 1)
+
+
+class PBag(object):
+    """
+    A persistent bag/multiset type.
+
+    Requires elements to be hashable, and allows duplicates, but has no
+    ordering. Bags are hashable.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`b`
+    or :py:func:`pbag` to create an instance.
+
+    Some examples:
+
+    >>> s = pbag([1, 2, 3, 1])
+    >>> s2 = s.add(4)
+    >>> s3 = s2.remove(1)
+    >>> s
+    pbag([1, 1, 2, 3])
+    >>> s2
+    pbag([1, 1, 2, 3, 4])
+    >>> s3
+    pbag([1, 2, 3, 4])
+    """
+
+    __slots__ = ('_counts', '__weakref__')
+
+    def __init__(self, counts):
+        self._counts = counts
+
+    def add(self, element):
+        """
+        Add an element to the bag.
+
+        >>> s = pbag([1])
+        >>> s2 = s.add(1)
+        >>> s3 = s.add(2)
+        >>> s2
+        pbag([1, 1])
+        >>> s3
+        pbag([1, 2])
+        """
+        return PBag(_add_to_counters(self._counts, element))
+
+    def update(self, iterable):
+        """
+        Update bag with all elements in iterable.
+
+        >>> s = pbag([1])
+        >>> s.update([1, 2])
+        pbag([1, 1, 2])
+        """
+        if iterable:
+            return PBag(reduce(_add_to_counters, iterable, self._counts))
+
+        return self
+
+    def remove(self, element):
+        """
+        Remove an element from the bag.
+
+        >>> s = pbag([1, 1, 2])
+        >>> s2 = s.remove(1)
+        >>> s3 = s.remove(2)
+        >>> s2
+        pbag([1, 2])
+        >>> s3
+        pbag([1, 1])
+        """
+        if element not in self._counts:
+            raise KeyError(element)
+        elif self._counts[element] == 1:
+            newc = self._counts.remove(element)
+        else:
+            newc = self._counts.set(element, self._counts[element] - 1)
+        return PBag(newc)
+
+    def count(self, element):
+        """
+        Return the number of times an element appears.
+
+
+        >>> pbag([]).count('non-existent')
+        0
+        >>> pbag([1, 1, 2]).count(1)
+        2
+        """
+        return self._counts.get(element, 0)
+
+    def __len__(self):
+        """
+        Return the length including duplicates.
+
+        >>> len(pbag([1, 1, 2]))
+        3
+        """
+        return sum(self._counts.itervalues())
+
+    def __iter__(self):
+        """
+        Return an iterator of all elements, including duplicates.
+
+        >>> list(pbag([1, 1, 2]))
+        [1, 1, 2]
+        >>> list(pbag([1, 2]))
+        [1, 2]
+        """
+        for elt, count in self._counts.iteritems():
+            for i in range(count):
+                yield elt
+
+    def __contains__(self, elt):
+        """
+        Check if an element is in the bag.
+
+        >>> 1 in pbag([1, 1, 2])
+        True
+        >>> 0 in pbag([1, 2])
+        False
+        """
+        return elt in self._counts
+
+    def __repr__(self):
+        return "pbag({0})".format(list(self))
+
+    def __eq__(self, other):
+        """
+        Check if two bags are equivalent, honoring the number of duplicates,
+        and ignoring insertion order.
+
+        >>> pbag([1, 1, 2]) == pbag([1, 2])
+        False
+        >>> pbag([2, 1, 0]) == pbag([0, 1, 2])
+        True
+        """
+        if type(other) is not PBag:
+            raise TypeError("Can only compare PBag with PBags")
+        return self._counts == other._counts
+
+    def __lt__(self, other):
+        raise TypeError('PBags are not orderable')
+
+    __le__ = __lt__
+    __gt__ = __lt__
+    __ge__ = __lt__
+
+    # Multiset-style operations similar to collections.Counter
+
+    def __add__(self, other):
+        """ 
+        Combine elements from two PBags.
+
+        >>> pbag([1, 2, 2]) + pbag([2, 3, 3])
+        pbag([1, 2, 2, 2, 3, 3])
+        """
+        if not isinstance(other, PBag):
+            return NotImplemented
+        result = self._counts.evolver()
+        for elem, other_count in other._counts.iteritems():
+            result[elem] = self.count(elem) + other_count
+        return PBag(result.persistent())
+
+    def __sub__(self, other):
+        """ 
+        Remove elements from one PBag that are present in another.
+
+        >>> pbag([1, 2, 2, 2, 3]) - pbag([2, 3, 3, 4])
+        pbag([1, 2, 2])
+        """
+        if not isinstance(other, PBag):
+            return NotImplemented
+        result = self._counts.evolver()
+        for elem, other_count in other._counts.iteritems():
+            newcount = self.count(elem) - other_count
+            if newcount > 0:
+                result[elem] = newcount
+            elif elem in self:
+                result.remove(elem)
+        return PBag(result.persistent())
+        
+    def __or__(self, other):
+        """ 
+        Union: Keep elements that are present in either of two PBags.
+
+        >>> pbag([1, 2, 2, 2]) | pbag([2, 3, 3])
+        pbag([1, 2, 2, 2, 3, 3])
+        """
+        if not isinstance(other, PBag):
+            return NotImplemented
+        result = self._counts.evolver()
+        for elem, other_count in other._counts.iteritems():
+            count = self.count(elem)
+            newcount = max(count, other_count)
+            result[elem] = newcount
+        return PBag(result.persistent())
+        
+    def __and__(self, other):
+        """
+        Intersection: Only keep elements that are present in both PBags.
+        
+        >>> pbag([1, 2, 2, 2]) & pbag([2, 3, 3])
+        pbag([2])
+        """
+        if not isinstance(other, PBag):
+            return NotImplemented
+        result = pmap().evolver()
+        for elem, count in self._counts.iteritems():
+            newcount = min(count, other.count(elem))
+            if newcount > 0:
+                result[elem] = newcount
+        return PBag(result.persistent())
+    
+    def __hash__(self):
+        """
+        Hash based on value of elements.
+
+        >>> m = pmap({pbag([1, 2]): "it's here!"})
+        >>> m[pbag([2, 1])]
+        "it's here!"
+        >>> pbag([1, 1, 2]) in m
+        False
+        """
+        return hash(self._counts)
+
+
+Container.register(PBag)
+Iterable.register(PBag)
+Sized.register(PBag)
+Hashable.register(PBag)
+
+
+def b(*elements):
+    """
+    Construct a persistent bag.
+
+    Takes an arbitrary number of arguments to insert into the new persistent
+    bag.
+
+    >>> b(1, 2, 3, 2)
+    pbag([1, 2, 2, 3])
+    """
+    return pbag(elements)
+
+
+def pbag(elements):
+    """
+    Convert an iterable to a persistent bag.
+
+    Takes an iterable with elements to insert.
+
+    >>> pbag([1, 2, 3, 2])
+    pbag([1, 2, 2, 3])
+    """
+    if not elements:
+        return _EMPTY_PBAG
+    return PBag(reduce(_add_to_counters, elements, pmap()))
+
+
+_EMPTY_PBAG = PBag(pmap())
+
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_pclass.py b/contrib/python/pyrsistent/py2/pyrsistent/_pclass.py
new file mode 100644
index 00000000000..a437f716482
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_pclass.py
@@ -0,0 +1,264 @@
+import six
+from pyrsistent._checked_types import (InvariantException, CheckedType, _restore_pickle, store_invariants)
+from pyrsistent._field_common import (
+    set_fields, check_type, is_field_ignore_extra_complaint, PFIELD_NO_INITIAL, serialize, check_global_invariants
+)
+from pyrsistent._transformations import transform
+
+
+def _is_pclass(bases):
+    return len(bases) == 1 and bases[0] == CheckedType
+
+
+class PClassMeta(type):
+    def __new__(mcs, name, bases, dct):
+        set_fields(dct, bases, name='_pclass_fields')
+        store_invariants(dct, bases, '_pclass_invariants', '__invariant__')
+        dct['__slots__'] = ('_pclass_frozen',) + tuple(key for key in dct['_pclass_fields'])
+
+        # There must only be one __weakref__ entry in the inheritance hierarchy,
+        # lets put it on the top level class.
+        if _is_pclass(bases):
+            dct['__slots__'] += ('__weakref__',)
+
+        return super(PClassMeta, mcs).__new__(mcs, name, bases, dct)
+
+_MISSING_VALUE = object()
+
+
+def _check_and_set_attr(cls, field, name, value, result, invariant_errors):
+    check_type(cls, field, name, value)
+    is_ok, error_code = field.invariant(value)
+    if not is_ok:
+        invariant_errors.append(error_code)
+    else:
+        setattr(result, name, value)
+
+
+@six.add_metaclass(PClassMeta)
+class PClass(CheckedType):
+    """
+    A PClass is a python class with a fixed set of specified fields. PClasses are declared as python classes inheriting
+    from PClass. It is defined the same way that PRecords are and behaves like a PRecord in all aspects except that it
+    is not a PMap and hence not a collection but rather a plain Python object.
+
+
+    More documentation and examples of PClass usage is available at https://github.com/tobgu/pyrsistent
+    """
+    def __new__(cls, **kwargs):    # Support *args?
+        result = super(PClass, cls).__new__(cls)
+        factory_fields = kwargs.pop('_factory_fields', None)
+        ignore_extra = kwargs.pop('ignore_extra', None)
+        missing_fields = []
+        invariant_errors = []
+        for name, field in cls._pclass_fields.items():
+            if name in kwargs:
+                if factory_fields is None or name in factory_fields:
+                    if is_field_ignore_extra_complaint(PClass, field, ignore_extra):
+                        value = field.factory(kwargs[name], ignore_extra=ignore_extra)
+                    else:
+                        value = field.factory(kwargs[name])
+                else:
+                    value = kwargs[name]
+                _check_and_set_attr(cls, field, name, value, result, invariant_errors)
+                del kwargs[name]
+            elif field.initial is not PFIELD_NO_INITIAL:
+                initial = field.initial() if callable(field.initial) else field.initial
+                _check_and_set_attr(
+                    cls, field, name, initial, result, invariant_errors)
+            elif field.mandatory:
+                missing_fields.append('{0}.{1}'.format(cls.__name__, name))
+
+        if invariant_errors or missing_fields:
+            raise InvariantException(tuple(invariant_errors), tuple(missing_fields), 'Field invariant failed')
+
+        if kwargs:
+            raise AttributeError("'{0}' are not among the specified fields for {1}".format(
+                ', '.join(kwargs), cls.__name__))
+
+        check_global_invariants(result, cls._pclass_invariants)
+
+        result._pclass_frozen = True
+        return result
+
+    def set(self, *args, **kwargs):
+        """
+        Set a field in the instance. Returns a new instance with the updated value. The original instance remains
+        unmodified. Accepts key-value pairs or single string representing the field name and a value.
+
+        >>> from pyrsistent import PClass, field
+        >>> class AClass(PClass):
+        ...     x = field()
+        ...
+        >>> a = AClass(x=1)
+        >>> a2 = a.set(x=2)
+        >>> a3 = a.set('x', 3)
+        >>> a
+        AClass(x=1)
+        >>> a2
+        AClass(x=2)
+        >>> a3
+        AClass(x=3)
+        """
+        if args:
+            kwargs[args[0]] = args[1]
+
+        factory_fields = set(kwargs)
+
+        for key in self._pclass_fields:
+            if key not in kwargs:
+                value = getattr(self, key, _MISSING_VALUE)
+                if value is not _MISSING_VALUE:
+                    kwargs[key] = value
+
+        return self.__class__(_factory_fields=factory_fields, **kwargs)
+
+    @classmethod
+    def create(cls, kwargs, _factory_fields=None, ignore_extra=False):
+        """
+        Factory method. Will create a new PClass of the current type and assign the values
+        specified in kwargs.
+
+        :param ignore_extra: A boolean which when set to True will ignore any keys which appear in kwargs that are not
+                             in the set of fields on the PClass.
+        """
+        if isinstance(kwargs, cls):
+            return kwargs
+
+        if ignore_extra:
+            kwargs = {k: kwargs[k] for k in cls._pclass_fields if k in kwargs}
+
+        return cls(_factory_fields=_factory_fields, ignore_extra=ignore_extra, **kwargs)
+
+    def serialize(self, format=None):
+        """
+        Serialize the current PClass using custom serializer functions for fields where
+        such have been supplied.
+        """
+        result = {}
+        for name in self._pclass_fields:
+            value = getattr(self, name, _MISSING_VALUE)
+            if value is not _MISSING_VALUE:
+                result[name] = serialize(self._pclass_fields[name].serializer, format, value)
+
+        return result
+
+    def transform(self, *transformations):
+        """
+        Apply transformations to the currency PClass. For more details on transformations see
+        the documentation for PMap. Transformations on PClasses do not support key matching
+        since the PClass is not a collection. Apart from that the transformations available
+        for other persistent types work as expected.
+        """
+        return transform(self, transformations)
+
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            for name in self._pclass_fields:
+                if getattr(self, name, _MISSING_VALUE) != getattr(other, name, _MISSING_VALUE):
+                    return False
+
+            return True
+
+        return NotImplemented
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __hash__(self):
+        # May want to optimize this by caching the hash somehow
+        return hash(tuple((key, getattr(self, key, _MISSING_VALUE)) for key in self._pclass_fields))
+
+    def __setattr__(self, key, value):
+        if getattr(self, '_pclass_frozen', False):
+            raise AttributeError("Can't set attribute, key={0}, value={1}".format(key, value))
+
+        super(PClass, self).__setattr__(key, value)
+
+    def __delattr__(self, key):
+            raise AttributeError("Can't delete attribute, key={0}, use remove()".format(key))
+
+    def _to_dict(self):
+        result = {}
+        for key in self._pclass_fields:
+            value = getattr(self, key, _MISSING_VALUE)
+            if value is not _MISSING_VALUE:
+                result[key] = value
+
+        return result
+
+    def __repr__(self):
+        return "{0}({1})".format(self.__class__.__name__,
+                                 ', '.join('{0}={1}'.format(k, repr(v)) for k, v in self._to_dict().items()))
+
+    def __reduce__(self):
+        # Pickling support
+        data = dict((key, getattr(self, key)) for key in self._pclass_fields if hasattr(self, key))
+        return _restore_pickle, (self.__class__, data,)
+
+    def evolver(self):
+        """
+        Returns an evolver for this object.
+        """
+        return _PClassEvolver(self, self._to_dict())
+
+    def remove(self, name):
+        """
+        Remove attribute given by name from the current instance. Raises AttributeError if the
+        attribute doesn't exist.
+        """
+        evolver = self.evolver()
+        del evolver[name]
+        return evolver.persistent()
+
+
+class _PClassEvolver(object):
+    __slots__ = ('_pclass_evolver_original', '_pclass_evolver_data', '_pclass_evolver_data_is_dirty', '_factory_fields')
+
+    def __init__(self, original, initial_dict):
+        self._pclass_evolver_original = original
+        self._pclass_evolver_data = initial_dict
+        self._pclass_evolver_data_is_dirty = False
+        self._factory_fields = set()
+
+    def __getitem__(self, item):
+        return self._pclass_evolver_data[item]
+
+    def set(self, key, value):
+        if self._pclass_evolver_data.get(key, _MISSING_VALUE) is not value:
+            self._pclass_evolver_data[key] = value
+            self._factory_fields.add(key)
+            self._pclass_evolver_data_is_dirty = True
+
+        return self
+
+    def __setitem__(self, key, value):
+        self.set(key, value)
+
+    def remove(self, item):
+        if item in self._pclass_evolver_data:
+            del self._pclass_evolver_data[item]
+            self._factory_fields.discard(item)
+            self._pclass_evolver_data_is_dirty = True
+            return self
+
+        raise AttributeError(item)
+
+    def __delitem__(self, item):
+        self.remove(item)
+
+    def persistent(self):
+        if self._pclass_evolver_data_is_dirty:
+            return self._pclass_evolver_original.__class__(_factory_fields=self._factory_fields,
+                                                           **self._pclass_evolver_data)
+
+        return self._pclass_evolver_original
+
+    def __setattr__(self, key, value):
+        if key not in self.__slots__:
+            self.set(key, value)
+        else:
+            super(_PClassEvolver, self).__setattr__(key, value)
+
+    def __getattr__(self, item):
+        return self[item]
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_pdeque.py b/contrib/python/pyrsistent/py2/pyrsistent/_pdeque.py
new file mode 100644
index 00000000000..5147b3fa6ad
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_pdeque.py
@@ -0,0 +1,376 @@
+from ._compat import Sequence, Hashable
+from itertools import islice, chain
+from numbers import Integral
+from pyrsistent._plist import plist
+
+
+class PDeque(object):
+    """
+    Persistent double ended queue (deque). Allows quick appends and pops in both ends. Implemented
+    using two persistent lists.
+
+    A maximum length can be specified to create a bounded queue.
+
+    Fully supports the Sequence and Hashable protocols including indexing and slicing but
+    if you need fast random access go for the PVector instead.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`dq` or :py:func:`pdeque` to
+    create an instance.
+
+    Some examples:
+
+    >>> x = pdeque([1, 2, 3])
+    >>> x.left
+    1
+    >>> x.right
+    3
+    >>> x[0] == x.left
+    True
+    >>> x[-1] == x.right
+    True
+    >>> x.pop()
+    pdeque([1, 2])
+    >>> x.pop() == x[:-1]
+    True
+    >>> x.popleft()
+    pdeque([2, 3])
+    >>> x.append(4)
+    pdeque([1, 2, 3, 4])
+    >>> x.appendleft(4)
+    pdeque([4, 1, 2, 3])
+
+    >>> y = pdeque([1, 2, 3], maxlen=3)
+    >>> y.append(4)
+    pdeque([2, 3, 4], maxlen=3)
+    >>> y.appendleft(4)
+    pdeque([4, 1, 2], maxlen=3)
+    """
+    __slots__ = ('_left_list', '_right_list', '_length', '_maxlen', '__weakref__')
+
+    def __new__(cls, left_list, right_list, length, maxlen=None):
+        instance = super(PDeque, cls).__new__(cls)
+        instance._left_list = left_list
+        instance._right_list = right_list
+        instance._length = length
+
+        if maxlen is not None:
+            if not isinstance(maxlen, Integral):
+                raise TypeError('An integer is required as maxlen')
+
+            if maxlen < 0:
+                raise ValueError("maxlen must be non-negative")
+
+        instance._maxlen = maxlen
+        return instance
+
+    @property
+    def right(self):
+        """
+        Rightmost element in dqueue.
+        """
+        return PDeque._tip_from_lists(self._right_list, self._left_list)
+
+    @property
+    def left(self):
+        """
+        Leftmost element in dqueue.
+        """
+        return PDeque._tip_from_lists(self._left_list, self._right_list)
+
+    @staticmethod
+    def _tip_from_lists(primary_list, secondary_list):
+        if primary_list:
+            return primary_list.first
+
+        if secondary_list:
+            return secondary_list[-1]
+
+        raise IndexError('No elements in empty deque')
+
+    def __iter__(self):
+        return chain(self._left_list, self._right_list.reverse())
+
+    def __repr__(self):
+        return "pdeque({0}{1})".format(list(self),
+                                       ', maxlen={0}'.format(self._maxlen) if self._maxlen is not None else '')
+    __str__ = __repr__
+
+    @property
+    def maxlen(self):
+        """
+        Maximum length of the queue.
+        """
+        return self._maxlen
+
+    def pop(self, count=1):
+        """
+        Return new deque with rightmost element removed. Popping the empty queue
+        will return the empty queue. A optional count can be given to indicate the
+        number of elements to pop. Popping with a negative index is the same as
+        popleft. Executes in amortized O(k) where k is the number of elements to pop.
+
+        >>> pdeque([1, 2]).pop()
+        pdeque([1])
+        >>> pdeque([1, 2]).pop(2)
+        pdeque([])
+        >>> pdeque([1, 2]).pop(-1)
+        pdeque([2])
+        """
+        if count < 0:
+            return self.popleft(-count)
+
+        new_right_list, new_left_list = PDeque._pop_lists(self._right_list, self._left_list, count)
+        return PDeque(new_left_list, new_right_list, max(self._length - count, 0), self._maxlen)
+
+    def popleft(self, count=1):
+        """
+        Return new deque with leftmost element removed. Otherwise functionally
+        equivalent to pop().
+
+        >>> pdeque([1, 2]).popleft()
+        pdeque([2])
+        """
+        if count < 0:
+            return self.pop(-count)
+
+        new_left_list, new_right_list = PDeque._pop_lists(self._left_list, self._right_list, count)
+        return PDeque(new_left_list, new_right_list, max(self._length - count, 0), self._maxlen)
+
+    @staticmethod
+    def _pop_lists(primary_list, secondary_list, count):
+        new_primary_list = primary_list
+        new_secondary_list = secondary_list
+
+        while count > 0 and (new_primary_list or new_secondary_list):
+            count -= 1
+            if new_primary_list.rest:
+                new_primary_list = new_primary_list.rest
+            elif new_primary_list:
+                new_primary_list = new_secondary_list.reverse()
+                new_secondary_list = plist()
+            else:
+                new_primary_list = new_secondary_list.reverse().rest
+                new_secondary_list = plist()
+
+        return new_primary_list, new_secondary_list
+
+    def _is_empty(self):
+        return not self._left_list and not self._right_list
+
+    def __lt__(self, other):
+        if not isinstance(other, PDeque):
+            return NotImplemented
+
+        return tuple(self) < tuple(other)
+
+    def __eq__(self, other):
+        if not isinstance(other, PDeque):
+            return NotImplemented
+
+        if tuple(self) == tuple(other):
+            # Sanity check of the length value since it is redundant (there for performance)
+            assert len(self) == len(other)
+            return True
+
+        return False
+
+    def __hash__(self):
+        return  hash(tuple(self))
+
+    def __len__(self):
+        return self._length
+
+    def append(self, elem):
+        """
+        Return new deque with elem as the rightmost element.
+
+        >>> pdeque([1, 2]).append(3)
+        pdeque([1, 2, 3])
+        """
+        new_left_list, new_right_list, new_length = self._append(self._left_list, self._right_list, elem)
+        return PDeque(new_left_list, new_right_list, new_length, self._maxlen)
+
+    def appendleft(self, elem):
+        """
+        Return new deque with elem as the leftmost element.
+
+        >>> pdeque([1, 2]).appendleft(3)
+        pdeque([3, 1, 2])
+        """
+        new_right_list, new_left_list, new_length = self._append(self._right_list, self._left_list, elem)
+        return PDeque(new_left_list, new_right_list, new_length, self._maxlen)
+
+    def _append(self, primary_list, secondary_list, elem):
+        if self._maxlen is not None and self._length == self._maxlen:
+            if self._maxlen == 0:
+                return primary_list, secondary_list, 0
+            new_primary_list, new_secondary_list = PDeque._pop_lists(primary_list, secondary_list, 1)
+            return new_primary_list, new_secondary_list.cons(elem), self._length
+
+        return primary_list, secondary_list.cons(elem), self._length + 1
+
+    @staticmethod
+    def _extend_list(the_list, iterable):
+        count = 0
+        for elem in iterable:
+            the_list = the_list.cons(elem)
+            count += 1
+
+        return the_list, count
+
+    def _extend(self, primary_list, secondary_list, iterable):
+        new_primary_list, extend_count = PDeque._extend_list(primary_list, iterable)
+        new_secondary_list = secondary_list
+        current_len = self._length + extend_count
+        if self._maxlen is not None and current_len > self._maxlen:
+            pop_len = current_len - self._maxlen
+            new_secondary_list, new_primary_list = PDeque._pop_lists(new_secondary_list, new_primary_list, pop_len)
+            extend_count -= pop_len
+
+        return new_primary_list, new_secondary_list, extend_count
+
+    def extend(self, iterable):
+        """
+        Return new deque with all elements of iterable appended to the right.
+
+        >>> pdeque([1, 2]).extend([3, 4])
+        pdeque([1, 2, 3, 4])
+        """
+        new_right_list, new_left_list, extend_count = self._extend(self._right_list, self._left_list, iterable)
+        return PDeque(new_left_list, new_right_list, self._length + extend_count, self._maxlen)
+
+    def extendleft(self, iterable):
+        """
+        Return new deque with all elements of iterable appended to the left.
+
+        NB! The elements will be inserted in reverse order compared to the order in the iterable.
+
+        >>> pdeque([1, 2]).extendleft([3, 4])
+        pdeque([4, 3, 1, 2])
+        """
+        new_left_list, new_right_list, extend_count = self._extend(self._left_list, self._right_list, iterable)
+        return PDeque(new_left_list, new_right_list, self._length + extend_count, self._maxlen)
+
+    def count(self, elem):
+        """
+        Return the number of elements equal to elem present in the queue
+
+        >>> pdeque([1, 2, 1]).count(1)
+        2
+        """
+        return self._left_list.count(elem) + self._right_list.count(elem)
+
+    def remove(self, elem):
+        """
+        Return new deque with first element from left equal to elem removed. If no such element is found
+        a ValueError is raised.
+
+        >>> pdeque([2, 1, 2]).remove(2)
+        pdeque([1, 2])
+        """
+        try:
+            return PDeque(self._left_list.remove(elem), self._right_list, self._length - 1)
+        except ValueError:
+            # Value not found in left list, try the right list
+            try:
+                # This is severely inefficient with a double reverse, should perhaps implement a remove_last()?
+                return PDeque(self._left_list,
+                               self._right_list.reverse().remove(elem).reverse(), self._length - 1)
+            except ValueError:
+                raise ValueError('{0} not found in PDeque'.format(elem))
+
+    def reverse(self):
+        """
+        Return reversed deque.
+
+        >>> pdeque([1, 2, 3]).reverse()
+        pdeque([3, 2, 1])
+
+        Also supports the standard python reverse function.
+
+        >>> reversed(pdeque([1, 2, 3]))
+        pdeque([3, 2, 1])
+        """
+        return PDeque(self._right_list, self._left_list, self._length)
+    __reversed__ = reverse
+
+    def rotate(self, steps):
+        """
+        Return deque with elements rotated steps steps.
+
+        >>> x = pdeque([1, 2, 3])
+        >>> x.rotate(1)
+        pdeque([3, 1, 2])
+        >>> x.rotate(-2)
+        pdeque([3, 1, 2])
+        """
+        popped_deque = self.pop(steps)
+        if steps >= 0:
+            return popped_deque.extendleft(islice(self.reverse(), steps))
+
+        return popped_deque.extend(islice(self, -steps))
+
+    def __reduce__(self):
+        # Pickling support
+        return pdeque, (list(self), self._maxlen)
+
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            if index.step is not None and index.step != 1:
+                # Too difficult, no structural sharing possible
+                return pdeque(tuple(self)[index], maxlen=self._maxlen)
+
+            result = self
+            if index.start is not None:
+                result = result.popleft(index.start % self._length)
+            if index.stop is not None:
+                result = result.pop(self._length - (index.stop % self._length))
+
+            return result
+
+        if not isinstance(index, Integral):
+            raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)
+
+        if index >= 0:
+            return self.popleft(index).left
+
+        shifted = len(self) + index
+        if shifted < 0:
+            raise IndexError(
+                "pdeque index {0} out of range {1}".format(index, len(self)),
+            )
+        return self.popleft(shifted).left
+
+    index = Sequence.index
+
+Sequence.register(PDeque)
+Hashable.register(PDeque)
+
+
+def pdeque(iterable=(), maxlen=None):
+    """
+    Return deque containing the elements of iterable. If maxlen is specified then
+    len(iterable) - maxlen elements are discarded from the left to if len(iterable) > maxlen.
+
+    >>> pdeque([1, 2, 3])
+    pdeque([1, 2, 3])
+    >>> pdeque([1, 2, 3, 4], maxlen=2)
+    pdeque([3, 4], maxlen=2)
+    """
+    t = tuple(iterable)
+    if maxlen is not None:
+        t = t[-maxlen:]
+    length = len(t)
+    pivot = int(length / 2)
+    left = plist(t[:pivot])
+    right = plist(t[pivot:], reverse=True)
+    return PDeque(left, right, length, maxlen)
+
+def dq(*elements):
+    """
+    Return deque containing all arguments.
+
+    >>> dq(1, 2, 3)
+    pdeque([1, 2, 3])
+    """
+    return pdeque(elements)
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_plist.py b/contrib/python/pyrsistent/py2/pyrsistent/_plist.py
new file mode 100644
index 00000000000..8b4267f5e3e
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_plist.py
@@ -0,0 +1,313 @@
+from ._compat import Sequence, Hashable
+from numbers import Integral
+from functools import reduce
+
+
+class _PListBuilder(object):
+    """
+    Helper class to allow construction of a list without
+    having to reverse it in the end.
+    """
+    __slots__ = ('_head', '_tail')
+
+    def __init__(self):
+        self._head = _EMPTY_PLIST
+        self._tail = _EMPTY_PLIST
+
+    def _append(self, elem, constructor):
+        if not self._tail:
+            self._head = constructor(elem)
+            self._tail = self._head
+        else:
+            self._tail.rest = constructor(elem)
+            self._tail = self._tail.rest
+
+        return self._head
+
+    def append_elem(self, elem):
+        return self._append(elem, lambda e: PList(e, _EMPTY_PLIST))
+
+    def append_plist(self, pl):
+        return self._append(pl, lambda l: l)
+
+    def build(self):
+        return self._head
+
+
+class _PListBase(object):
+    __slots__ = ('__weakref__',)
+
+    # Selected implementations can be taken straight from the Sequence
+    # class, other are less suitable. Especially those that work with
+    # index lookups.
+    count = Sequence.count
+    index = Sequence.index
+
+    def __reduce__(self):
+        # Pickling support
+        return plist, (list(self),)
+
+    def __len__(self):
+        """
+        Return the length of the list, computed by traversing it.
+
+        This is obviously O(n) but with the current implementation
+        where a list is also a node the overhead of storing the length
+        in every node would be quite significant.
+        """
+        return sum(1 for _ in self)
+
+    def __repr__(self):
+        return "plist({0})".format(list(self))
+    __str__ = __repr__
+
+    def cons(self, elem):
+        """
+        Return a new list with elem inserted as new head.
+
+        >>> plist([1, 2]).cons(3)
+        plist([3, 1, 2])
+        """
+        return PList(elem, self)
+
+    def mcons(self, iterable):
+        """
+        Return a new list with all elements of iterable repeatedly cons:ed to the current list.
+        NB! The elements will be inserted in the reverse order of the iterable.
+        Runs in O(len(iterable)).
+
+        >>> plist([1, 2]).mcons([3, 4])
+        plist([4, 3, 1, 2])
+        """
+        head = self
+        for elem in iterable:
+            head = head.cons(elem)
+
+        return head
+
+    def reverse(self):
+        """
+        Return a reversed version of list. Runs in O(n) where n is the length of the list.
+
+        >>> plist([1, 2, 3]).reverse()
+        plist([3, 2, 1])
+
+        Also supports the standard reversed function.
+
+        >>> reversed(plist([1, 2, 3]))
+        plist([3, 2, 1])
+        """
+        result = plist()
+        head = self
+        while head:
+            result = result.cons(head.first)
+            head = head.rest
+
+        return result
+    __reversed__ = reverse
+
+    def split(self, index):
+        """
+        Spilt the list at position specified by index. Returns a tuple containing the
+        list up until index and the list after the index. Runs in O(index).
+
+        >>> plist([1, 2, 3, 4]).split(2)
+        (plist([1, 2]), plist([3, 4]))
+        """
+        lb = _PListBuilder()
+        right_list = self
+        i = 0
+        while right_list and i < index:
+            lb.append_elem(right_list.first)
+            right_list = right_list.rest
+            i += 1
+
+        if not right_list:
+            # Just a small optimization in the cases where no split occurred
+            return self, _EMPTY_PLIST
+
+        return lb.build(), right_list
+
+    def __iter__(self):
+        li = self
+        while li:
+            yield li.first
+            li = li.rest
+
+    def __lt__(self, other):
+        if not isinstance(other, _PListBase):
+            return NotImplemented
+
+        return tuple(self) < tuple(other)
+
+    def __eq__(self, other):
+        """
+        Traverses the lists, checking equality of elements.
+        
+        This is an O(n) operation, but preserves the standard semantics of list equality.
+        """
+        if not isinstance(other, _PListBase):
+            return NotImplemented
+
+        self_head = self
+        other_head = other
+        while self_head and other_head:
+            if not self_head.first == other_head.first:
+                return False
+            self_head = self_head.rest
+            other_head = other_head.rest
+
+        return not self_head and not other_head
+
+    def __getitem__(self, index):
+        # Don't use this this data structure if you plan to do a lot of indexing, it is
+        # very inefficient! Use a PVector instead!
+
+        if isinstance(index, slice):
+            if index.start is not None and index.stop is None and (index.step is None or index.step == 1):
+                return self._drop(index.start)
+
+            # Take the easy way out for all other slicing cases, not much structural reuse possible anyway
+            return plist(tuple(self)[index])
+
+        if not isinstance(index, Integral):
+            raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)
+
+        if index < 0:
+            # NB: O(n)!
+            index += len(self)
+
+        try:
+            return self._drop(index).first
+        except AttributeError:
+            raise IndexError("PList index out of range")
+
+    def _drop(self, count):
+        if count < 0:
+            raise IndexError("PList index out of range")
+
+        head = self
+        while count > 0:
+            head = head.rest
+            count -= 1
+
+        return head
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+    def remove(self, elem):
+        """
+        Return new list with first element equal to elem removed. O(k) where k is the position
+        of the element that is removed.
+
+        Raises ValueError if no matching element is found.
+
+        >>> plist([1, 2, 1]).remove(1)
+        plist([2, 1])
+        """
+
+        builder = _PListBuilder()
+        head = self
+        while head:
+            if head.first == elem:
+                return builder.append_plist(head.rest)
+
+            builder.append_elem(head.first)
+            head = head.rest
+
+        raise ValueError('{0} not found in PList'.format(elem))
+
+
+class PList(_PListBase):
+    """
+    Classical Lisp style singly linked list. Adding elements to the head using cons is O(1).
+    Element access is O(k) where k is the position of the element in the list. Taking the
+    length of the list is O(n).
+
+    Fully supports the Sequence and Hashable protocols including indexing and slicing but
+    if you need fast random access go for the PVector instead.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`l` or :py:func:`plist` to
+    create an instance.
+
+    Some examples:
+
+    >>> x = plist([1, 2])
+    >>> y = x.cons(3)
+    >>> x
+    plist([1, 2])
+    >>> y
+    plist([3, 1, 2])
+    >>> y.first
+    3
+    >>> y.rest == x
+    True
+    >>> y[:2]
+    plist([3, 1])
+    """
+    __slots__ = ('first', 'rest')
+
+    def __new__(cls, first, rest):
+        instance = super(PList, cls).__new__(cls)
+        instance.first = first
+        instance.rest = rest
+        return instance
+
+    def __bool__(self):
+        return True
+    __nonzero__ = __bool__
+
+
+Sequence.register(PList)
+Hashable.register(PList)
+
+
+class _EmptyPList(_PListBase):
+    __slots__ = ()
+
+    def __bool__(self):
+        return False
+    __nonzero__ = __bool__
+
+    @property
+    def first(self):
+        raise AttributeError("Empty PList has no first")
+
+    @property
+    def rest(self):
+        return self
+
+
+Sequence.register(_EmptyPList)
+Hashable.register(_EmptyPList)
+
+_EMPTY_PLIST = _EmptyPList()
+
+
+def plist(iterable=(), reverse=False):
+    """
+    Creates a new persistent list containing all elements of iterable.
+    Optional parameter reverse specifies if the elements should be inserted in
+    reverse order or not.
+
+    >>> plist([1, 2, 3])
+    plist([1, 2, 3])
+    >>> plist([1, 2, 3], reverse=True)
+    plist([3, 2, 1])
+    """
+    if not reverse:
+        iterable = list(iterable)
+        iterable.reverse()
+
+    return reduce(lambda pl, elem: pl.cons(elem), iterable, _EMPTY_PLIST)
+
+
+def l(*elements):
+    """
+    Creates a new persistent list containing all arguments.
+
+    >>> l(1, 2, 3)
+    plist([1, 2, 3])
+    """
+    return plist(elements)
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_pmap.py b/contrib/python/pyrsistent/py2/pyrsistent/_pmap.py
new file mode 100644
index 00000000000..fe0bc55ed98
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_pmap.py
@@ -0,0 +1,460 @@
+from ._compat import Mapping, Hashable
+from itertools import chain
+import six
+from pyrsistent._pvector import pvector
+from pyrsistent._transformations import transform
+
+
+class PMap(object):
+    """
+    Persistent map/dict. Tries to follow the same naming conventions as the built in dict where feasible.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`m` or :py:func:`pmap` to
+    create an instance.
+
+    Was originally written as a very close copy of the Clojure equivalent but was later rewritten to closer
+    re-assemble the python dict. This means that a sparse vector (a PVector) of buckets is used. The keys are
+    hashed and the elements inserted at position hash % len(bucket_vector). Whenever the map size exceeds 2/3 of
+    the containing vectors size the map is reallocated to a vector of double the size. This is done to avoid
+    excessive hash collisions.
+
+    This structure corresponds most closely to the built in dict type and is intended as a replacement. Where the
+    semantics are the same (more or less) the same function names have been used but for some cases it is not possible,
+    for example assignments and deletion of values.
+
+    PMap implements the Mapping protocol and is Hashable. It also supports dot-notation for
+    element access.
+
+    Random access and insert is log32(n) where n is the size of the map.
+
+    The following are examples of some common operations on persistent maps
+
+    >>> m1 = m(a=1, b=3)
+    >>> m2 = m1.set('c', 3)
+    >>> m3 = m2.remove('a')
+    >>> m1
+    pmap({'a': 1, 'b': 3})
+    >>> m2
+    pmap({'a': 1, 'c': 3, 'b': 3})
+    >>> m3
+    pmap({'c': 3, 'b': 3})
+    >>> m3['c']
+    3
+    >>> m3.c
+    3
+    """
+    __slots__ = ('_size', '_buckets', '__weakref__', '_cached_hash')
+
+    def __new__(cls, size, buckets):
+        self = super(PMap, cls).__new__(cls)
+        self._size = size
+        self._buckets = buckets
+        return self
+
+    @staticmethod
+    def _get_bucket(buckets, key):
+        index = hash(key) % len(buckets)
+        bucket = buckets[index]
+        return index, bucket
+
+    @staticmethod
+    def _getitem(buckets, key):
+        _, bucket = PMap._get_bucket(buckets, key)
+        if bucket:
+            for k, v in bucket:
+                if k == key:
+                    return v
+
+        raise KeyError(key)
+
+    def __getitem__(self, key):
+        return PMap._getitem(self._buckets, key)
+
+    @staticmethod
+    def _contains(buckets, key):
+        _, bucket = PMap._get_bucket(buckets, key)
+        if bucket:
+            for k, _ in bucket:
+                if k == key:
+                    return True
+
+            return False
+
+        return False
+
+    def __contains__(self, key):
+        return self._contains(self._buckets, key)
+
+    get = Mapping.get
+
+    def __iter__(self):
+        return self.iterkeys()
+
+    def __getattr__(self, key):
+        try:
+            return self[key]
+        except KeyError:
+            raise AttributeError(
+                "{0} has no attribute '{1}'".format(type(self).__name__, key)
+            )
+
+    def iterkeys(self):
+        for k, _ in self.iteritems():
+            yield k
+
+    # These are more efficient implementations compared to the original
+    # methods that are based on the keys iterator and then calls the
+    # accessor functions to access the value for the corresponding key
+    def itervalues(self):
+        for _, v in self.iteritems():
+            yield v
+
+    def iteritems(self):
+        for bucket in self._buckets:
+            if bucket:
+                for k, v in bucket:
+                    yield k, v
+
+    def values(self):
+        return pvector(self.itervalues())
+
+    def keys(self):
+        return pvector(self.iterkeys())
+
+    def items(self):
+        return pvector(self.iteritems())
+
+    def __len__(self):
+        return self._size
+
+    def __repr__(self):
+        return 'pmap({0})'.format(str(dict(self)))
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if not isinstance(other, Mapping):
+            return NotImplemented
+        if len(self) != len(other):
+            return False
+        if isinstance(other, PMap):
+            if (hasattr(self, '_cached_hash') and hasattr(other, '_cached_hash')
+                    and self._cached_hash != other._cached_hash):
+                return False
+            if self._buckets == other._buckets:
+                return True
+            return dict(self.iteritems()) == dict(other.iteritems())
+        elif isinstance(other, dict):
+            return dict(self.iteritems()) == other
+        return dict(self.iteritems()) == dict(six.iteritems(other))
+
+    __ne__ = Mapping.__ne__
+
+    def __lt__(self, other):
+        raise TypeError('PMaps are not orderable')
+
+    __le__ = __lt__
+    __gt__ = __lt__
+    __ge__ = __lt__
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __hash__(self):
+        if not hasattr(self, '_cached_hash'):
+            self._cached_hash = hash(frozenset(self.iteritems()))
+        return self._cached_hash
+
+    def set(self, key, val):
+        """
+        Return a new PMap with key and val inserted.
+
+        >>> m1 = m(a=1, b=2)
+        >>> m2 = m1.set('a', 3)
+        >>> m3 = m1.set('c' ,4)
+        >>> m1
+        pmap({'a': 1, 'b': 2})
+        >>> m2
+        pmap({'a': 3, 'b': 2})
+        >>> m3
+        pmap({'a': 1, 'c': 4, 'b': 2})
+        """
+        return self.evolver().set(key, val).persistent()
+
+    def remove(self, key):
+        """
+        Return a new PMap without the element specified by key. Raises KeyError if the element
+        is not present.
+
+        >>> m1 = m(a=1, b=2)
+        >>> m1.remove('a')
+        pmap({'b': 2})
+        """
+        return self.evolver().remove(key).persistent()
+
+    def discard(self, key):
+        """
+        Return a new PMap without the element specified by key. Returns reference to itself
+        if element is not present.
+
+        >>> m1 = m(a=1, b=2)
+        >>> m1.discard('a')
+        pmap({'b': 2})
+        >>> m1 is m1.discard('c')
+        True
+        """
+        try:
+            return self.remove(key)
+        except KeyError:
+            return self
+
+    def update(self, *maps):
+        """
+        Return a new PMap with the items in Mappings inserted. If the same key is present in multiple
+        maps the rightmost (last) value is inserted.
+
+        >>> m1 = m(a=1, b=2)
+        >>> m1.update(m(a=2, c=3), {'a': 17, 'd': 35})
+        pmap({'a': 17, 'c': 3, 'b': 2, 'd': 35})
+        """
+        return self.update_with(lambda l, r: r, *maps)
+
+    def update_with(self, update_fn, *maps):
+        """
+        Return a new PMap with the items in Mappings maps inserted. If the same key is present in multiple
+        maps the values will be merged using merge_fn going from left to right.
+
+        >>> from operator import add
+        >>> m1 = m(a=1, b=2)
+        >>> m1.update_with(add, m(a=2))
+        pmap({'a': 3, 'b': 2})
+
+        The reverse behaviour of the regular merge. Keep the leftmost element instead of the rightmost.
+
+        >>> m1 = m(a=1)
+        >>> m1.update_with(lambda l, r: l, m(a=2), {'a':3})
+        pmap({'a': 1})
+        """
+        evolver = self.evolver()
+        for map in maps:
+            for key, value in map.items():
+                evolver.set(key, update_fn(evolver[key], value) if key in evolver else value)
+
+        return evolver.persistent()
+
+    def __add__(self, other):
+        return self.update(other)
+
+    def __reduce__(self):
+        # Pickling support
+        return pmap, (dict(self),)
+
+    def transform(self, *transformations):
+        """
+        Transform arbitrarily complex combinations of PVectors and PMaps. A transformation
+        consists of two parts. One match expression that specifies which elements to transform
+        and one transformation function that performs the actual transformation.
+
+        >>> from pyrsistent import freeze, ny
+        >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
+        ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
+        ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
+        >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
+        >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
+        >>> very_short_news.articles[0].content
+        'A short article'
+        >>> very_short_news.articles[1].content
+        'A slightly long...'
+
+        When nothing has been transformed the original data structure is kept
+
+        >>> short_news is news_paper
+        True
+        >>> very_short_news is news_paper
+        False
+        >>> very_short_news.articles[0] is news_paper.articles[0]
+        True
+        """
+        return transform(self, transformations)
+
+    def copy(self):
+        return self
+
+    class _Evolver(object):
+        __slots__ = ('_buckets_evolver', '_size', '_original_pmap')
+
+        def __init__(self, original_pmap):
+            self._original_pmap = original_pmap
+            self._buckets_evolver = original_pmap._buckets.evolver()
+            self._size = original_pmap._size
+
+        def __getitem__(self, key):
+            return PMap._getitem(self._buckets_evolver, key)
+
+        def __setitem__(self, key, val):
+            self.set(key, val)
+
+        def set(self, key, val):
+            if len(self._buckets_evolver) < 0.67 * self._size:
+                self._reallocate(2 * len(self._buckets_evolver))
+
+            kv = (key, val)
+            index, bucket = PMap._get_bucket(self._buckets_evolver, key)
+            if bucket:
+                for k, v in bucket:
+                    if k == key:
+                        if v is not val:
+                            new_bucket = [(k2, v2) if k2 != k else (k2, val) for k2, v2 in bucket]
+                            self._buckets_evolver[index] = new_bucket
+
+                        return self
+
+                new_bucket = [kv]
+                new_bucket.extend(bucket)
+                self._buckets_evolver[index] = new_bucket
+                self._size += 1
+            else:
+                self._buckets_evolver[index] = [kv]
+                self._size += 1
+
+            return self
+
+        def _reallocate(self, new_size):
+            new_list = new_size * [None]
+            buckets = self._buckets_evolver.persistent()
+            for k, v in chain.from_iterable(x for x in buckets if x):
+                index = hash(k) % new_size
+                if new_list[index]:
+                    new_list[index].append((k, v))
+                else:
+                    new_list[index] = [(k, v)]
+
+            # A reallocation should always result in a dirty buckets evolver to avoid
+            # possible loss of elements when doing the reallocation.
+            self._buckets_evolver = pvector().evolver()
+            self._buckets_evolver.extend(new_list)
+
+        def is_dirty(self):
+            return self._buckets_evolver.is_dirty()
+
+        def persistent(self):
+            if self.is_dirty():
+                self._original_pmap = PMap(self._size, self._buckets_evolver.persistent())
+
+            return self._original_pmap
+
+        def __len__(self):
+            return self._size
+
+        def __contains__(self, key):
+            return PMap._contains(self._buckets_evolver, key)
+
+        def __delitem__(self, key):
+            self.remove(key)
+
+        def remove(self, key):
+            index, bucket = PMap._get_bucket(self._buckets_evolver, key)
+
+            if bucket:
+                new_bucket = [(k, v) for (k, v) in bucket if k != key]
+                if len(bucket) > len(new_bucket):
+                    self._buckets_evolver[index] = new_bucket if new_bucket else None
+                    self._size -= 1
+                    return self
+
+            raise KeyError('{0}'.format(key))
+
+    def evolver(self):
+        """
+        Create a new evolver for this pmap. For a discussion on evolvers in general see the
+        documentation for the pvector evolver.
+
+        Create the evolver and perform various mutating updates to it:
+
+        >>> m1 = m(a=1, b=2)
+        >>> e = m1.evolver()
+        >>> e['c'] = 3
+        >>> len(e)
+        3
+        >>> del e['a']
+
+        The underlying pmap remains the same:
+
+        >>> m1
+        pmap({'a': 1, 'b': 2})
+
+        The changes are kept in the evolver. An updated pmap can be created using the
+        persistent() function on the evolver.
+
+        >>> m2 = e.persistent()
+        >>> m2
+        pmap({'c': 3, 'b': 2})
+
+        The new pmap will share data with the original pmap in the same way that would have
+        been done if only using operations on the pmap.
+        """
+        return self._Evolver(self)
+
+Mapping.register(PMap)
+Hashable.register(PMap)
+
+
+def _turbo_mapping(initial, pre_size):
+    if pre_size:
+        size = pre_size
+    else:
+        try:
+            size = 2 * len(initial) or 8
+        except Exception:
+            # Guess we can't figure out the length. Give up on length hinting,
+            # we can always reallocate later.
+            size = 8
+
+    buckets = size * [None]
+
+    if not isinstance(initial, Mapping):
+        # Make a dictionary of the initial data if it isn't already,
+        # that will save us some job further down since we can assume no
+        # key collisions
+        initial = dict(initial)
+
+    for k, v in six.iteritems(initial):
+        h = hash(k)
+        index = h % size
+        bucket = buckets[index]
+
+        if bucket:
+            bucket.append((k, v))
+        else:
+            buckets[index] = [(k, v)]
+
+    return PMap(len(initial), pvector().extend(buckets))
+
+
+_EMPTY_PMAP = _turbo_mapping({}, 0)
+
+
+def pmap(initial={}, pre_size=0):
+    """
+    Create new persistent map, inserts all elements in initial into the newly created map.
+    The optional argument pre_size may be used to specify an initial size of the underlying bucket vector. This
+    may have a positive performance impact in the cases where you know beforehand that a large number of elements
+    will be inserted into the map eventually since it will reduce the number of reallocations required.
+
+    >>> pmap({'a': 13, 'b': 14})
+    pmap({'a': 13, 'b': 14})
+    """
+    if not initial:
+        return _EMPTY_PMAP
+
+    return _turbo_mapping(initial, pre_size)
+
+
+def m(**kwargs):
+    """
+    Creates a new persitent map. Inserts all key value arguments into the newly created map.
+
+    >>> m(a=13, b=14)
+    pmap({'a': 13, 'b': 14})
+    """
+    return pmap(kwargs)
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_precord.py b/contrib/python/pyrsistent/py2/pyrsistent/_precord.py
new file mode 100644
index 00000000000..ec8d32c3dac
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_precord.py
@@ -0,0 +1,169 @@
+import six
+from pyrsistent._checked_types import CheckedType, _restore_pickle, InvariantException, store_invariants
+from pyrsistent._field_common import (
+    set_fields, check_type, is_field_ignore_extra_complaint, PFIELD_NO_INITIAL, serialize, check_global_invariants
+)
+from pyrsistent._pmap import PMap, pmap
+
+
+class _PRecordMeta(type):
+    def __new__(mcs, name, bases, dct):
+        set_fields(dct, bases, name='_precord_fields')
+        store_invariants(dct, bases, '_precord_invariants', '__invariant__')
+
+        dct['_precord_mandatory_fields'] = \
+            set(name for name, field in dct['_precord_fields'].items() if field.mandatory)
+
+        dct['_precord_initial_values'] = \
+            dict((k, field.initial) for k, field in dct['_precord_fields'].items() if field.initial is not PFIELD_NO_INITIAL)
+
+
+        dct['__slots__'] = ()
+
+        return super(_PRecordMeta, mcs).__new__(mcs, name, bases, dct)
+
+
+@six.add_metaclass(_PRecordMeta)
+class PRecord(PMap, CheckedType):
+    """
+    A PRecord is a PMap with a fixed set of specified fields. Records are declared as python classes inheriting
+    from PRecord. Because it is a PMap it has full support for all Mapping methods such as iteration and element
+    access using subscript notation.
+
+    More documentation and examples of PRecord usage is available at https://github.com/tobgu/pyrsistent
+    """
+    def __new__(cls, **kwargs):
+        # Hack total! If these two special attributes exist that means we can create
+        # ourselves. Otherwise we need to go through the Evolver to create the structures
+        # for us.
+        if '_precord_size' in kwargs and '_precord_buckets' in kwargs:
+            return super(PRecord, cls).__new__(cls, kwargs['_precord_size'], kwargs['_precord_buckets'])
+
+        factory_fields = kwargs.pop('_factory_fields', None)
+        ignore_extra = kwargs.pop('_ignore_extra', False)
+
+        initial_values = kwargs
+        if cls._precord_initial_values:
+            initial_values = dict((k, v() if callable(v) else v)
+                                  for k, v in cls._precord_initial_values.items())
+            initial_values.update(kwargs)
+
+        e = _PRecordEvolver(cls, pmap(), _factory_fields=factory_fields, _ignore_extra=ignore_extra)
+        for k, v in initial_values.items():
+            e[k] = v
+
+        return e.persistent()
+
+    def set(self, *args, **kwargs):
+        """
+        Set a field in the record. This set function differs slightly from that in the PMap
+        class. First of all it accepts key-value pairs. Second it accepts multiple key-value
+        pairs to perform one, atomic, update of multiple fields.
+        """
+
+        # The PRecord set() can accept kwargs since all fields that have been declared are
+        # valid python identifiers. Also allow multiple fields to be set in one operation.
+        if args:
+            return super(PRecord, self).set(args[0], args[1])
+
+        return self.update(kwargs)
+
+    def evolver(self):
+        """
+        Returns an evolver of this object.
+        """
+        return _PRecordEvolver(self.__class__, self)
+
+    def __repr__(self):
+        return "{0}({1})".format(self.__class__.__name__,
+                                 ', '.join('{0}={1}'.format(k, repr(v)) for k, v in self.items()))
+
+    @classmethod
+    def create(cls, kwargs, _factory_fields=None, ignore_extra=False):
+        """
+        Factory method. Will create a new PRecord of the current type and assign the values
+        specified in kwargs.
+
+        :param ignore_extra: A boolean which when set to True will ignore any keys which appear in kwargs that are not
+                             in the set of fields on the PRecord.
+        """
+        if isinstance(kwargs, cls):
+            return kwargs
+
+        if ignore_extra:
+            kwargs = {k: kwargs[k] for k in cls._precord_fields if k in kwargs}
+
+        return cls(_factory_fields=_factory_fields, _ignore_extra=ignore_extra, **kwargs)
+
+    def __reduce__(self):
+        # Pickling support
+        return _restore_pickle, (self.__class__, dict(self),)
+
+    def serialize(self, format=None):
+        """
+        Serialize the current PRecord using custom serializer functions for fields where
+        such have been supplied.
+        """
+        return dict((k, serialize(self._precord_fields[k].serializer, format, v)) for k, v in self.items())
+
+
+class _PRecordEvolver(PMap._Evolver):
+    __slots__ = ('_destination_cls', '_invariant_error_codes', '_missing_fields', '_factory_fields', '_ignore_extra')
+
+    def __init__(self, cls, original_pmap, _factory_fields=None, _ignore_extra=False):
+        super(_PRecordEvolver, self).__init__(original_pmap)
+        self._destination_cls = cls
+        self._invariant_error_codes = []
+        self._missing_fields = []
+        self._factory_fields = _factory_fields
+        self._ignore_extra = _ignore_extra
+
+    def __setitem__(self, key, original_value):
+        self.set(key, original_value)
+
+    def set(self, key, original_value):
+        field = self._destination_cls._precord_fields.get(key)
+        if field:
+            if self._factory_fields is None or field in self._factory_fields:
+                try:
+                    if is_field_ignore_extra_complaint(PRecord, field, self._ignore_extra):
+                        value = field.factory(original_value, ignore_extra=self._ignore_extra)
+                    else:
+                        value = field.factory(original_value)
+                except InvariantException as e:
+                    self._invariant_error_codes += e.invariant_errors
+                    self._missing_fields += e.missing_fields
+                    return self
+            else:
+                value = original_value
+
+            check_type(self._destination_cls, field, key, value)
+
+            is_ok, error_code = field.invariant(value)
+            if not is_ok:
+                self._invariant_error_codes.append(error_code)
+
+            return super(_PRecordEvolver, self).set(key, value)
+        else:
+            raise AttributeError("'{0}' is not among the specified fields for {1}".format(key, self._destination_cls.__name__))
+
+    def persistent(self):
+        cls = self._destination_cls
+        is_dirty = self.is_dirty()
+        pm = super(_PRecordEvolver, self).persistent()
+        if is_dirty or not isinstance(pm, cls):
+            result = cls(_precord_buckets=pm._buckets, _precord_size=pm._size)
+        else:
+            result = pm
+
+        if cls._precord_mandatory_fields:
+            self._missing_fields += tuple('{0}.{1}'.format(cls.__name__, f) for f
+                                          in (cls._precord_mandatory_fields - set(result.keys())))
+
+        if self._invariant_error_codes or self._missing_fields:
+            raise InvariantException(tuple(self._invariant_error_codes), tuple(self._missing_fields),
+                                     'Field invariant failed')
+
+        check_global_invariants(result, cls._precord_invariants)
+
+        return result
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_pset.py b/contrib/python/pyrsistent/py2/pyrsistent/_pset.py
new file mode 100644
index 00000000000..a972ec533b2
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_pset.py
@@ -0,0 +1,229 @@
+from ._compat import Set, Hashable
+import sys
+from pyrsistent._pmap import pmap
+
+PY2 = sys.version_info[0] < 3
+
+
+class PSet(object):
+    """
+    Persistent set implementation. Built on top of the persistent map. The set supports all operations
+    in the Set protocol and is Hashable.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`s` or :py:func:`pset`
+    to create an instance.
+
+    Random access and insert is log32(n) where n is the size of the set.
+
+    Some examples:
+
+    >>> s = pset([1, 2, 3, 1])
+    >>> s2 = s.add(4)
+    >>> s3 = s2.remove(2)
+    >>> s
+    pset([1, 2, 3])
+    >>> s2
+    pset([1, 2, 3, 4])
+    >>> s3
+    pset([1, 3, 4])
+    """
+    __slots__ = ('_map', '__weakref__')
+
+    def __new__(cls, m):
+        self = super(PSet, cls).__new__(cls)
+        self._map = m
+        return self
+
+    def __contains__(self, element):
+        return element in self._map
+
+    def __iter__(self):
+        return iter(self._map)
+
+    def __len__(self):
+        return len(self._map)
+
+    def __repr__(self):
+        if PY2 or not self:
+            return 'p' + str(set(self))
+
+        return 'pset([{0}])'.format(str(set(self))[1:-1])
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __hash__(self):
+        return hash(self._map)
+
+    def __reduce__(self):
+        # Pickling support
+        return pset, (list(self),)
+
+    @classmethod
+    def _from_iterable(cls, it, pre_size=8):
+        return PSet(pmap(dict((k, True) for k in it), pre_size=pre_size))
+
+    def add(self, element):
+        """
+        Return a new PSet with element added
+
+        >>> s1 = s(1, 2)
+        >>> s1.add(3)
+        pset([1, 2, 3])
+        """
+        return self.evolver().add(element).persistent()
+
+    def update(self, iterable):
+        """
+        Return a new PSet with elements in iterable added
+
+        >>> s1 = s(1, 2)
+        >>> s1.update([3, 4, 4])
+        pset([1, 2, 3, 4])
+        """
+        e = self.evolver()
+        for element in iterable:
+            e.add(element)
+
+        return e.persistent()
+
+    def remove(self, element):
+        """
+        Return a new PSet with element removed. Raises KeyError if element is not present.
+
+        >>> s1 = s(1, 2)
+        >>> s1.remove(2)
+        pset([1])
+        """
+        if element in self._map:
+            return self.evolver().remove(element).persistent()
+
+        raise KeyError("Element '%s' not present in PSet" % element)
+
+    def discard(self, element):
+        """
+        Return a new PSet with element removed. Returns itself if element is not present.
+        """
+        if element in self._map:
+            return self.evolver().remove(element).persistent()
+
+        return self
+
+    class _Evolver(object):
+        __slots__ = ('_original_pset', '_pmap_evolver')
+
+        def __init__(self, original_pset):
+            self._original_pset = original_pset
+            self._pmap_evolver = original_pset._map.evolver()
+
+        def add(self, element):
+            self._pmap_evolver[element] = True
+            return self
+
+        def remove(self, element):
+            del self._pmap_evolver[element]
+            return self
+
+        def is_dirty(self):
+            return self._pmap_evolver.is_dirty()
+
+        def persistent(self):
+            if not self.is_dirty():
+                return  self._original_pset
+
+            return PSet(self._pmap_evolver.persistent())
+
+        def __len__(self):
+            return len(self._pmap_evolver)
+
+    def copy(self):
+        return self
+
+    def evolver(self):
+        """
+        Create a new evolver for this pset. For a discussion on evolvers in general see the
+        documentation for the pvector evolver.
+
+        Create the evolver and perform various mutating updates to it:
+
+        >>> s1 = s(1, 2, 3)
+        >>> e = s1.evolver()
+        >>> _ = e.add(4)
+        >>> len(e)
+        4
+        >>> _ = e.remove(1)
+
+        The underlying pset remains the same:
+
+        >>> s1
+        pset([1, 2, 3])
+
+        The changes are kept in the evolver. An updated pmap can be created using the
+        persistent() function on the evolver.
+
+        >>> s2 = e.persistent()
+        >>> s2
+        pset([2, 3, 4])
+
+        The new pset will share data with the original pset in the same way that would have
+        been done if only using operations on the pset.
+        """
+        return PSet._Evolver(self)
+
+    # All the operations and comparisons you would expect on a set.
+    #
+    # This is not very beautiful. If we avoid inheriting from PSet we can use the
+    # __slots__ concepts (which requires a new style class) and hopefully save some memory.
+    __le__ = Set.__le__
+    __lt__ = Set.__lt__
+    __gt__ = Set.__gt__
+    __ge__ = Set.__ge__
+    __eq__ = Set.__eq__
+    __ne__ = Set.__ne__
+
+    __and__ = Set.__and__
+    __or__ = Set.__or__
+    __sub__ = Set.__sub__
+    __xor__ = Set.__xor__
+
+    issubset = __le__
+    issuperset = __ge__
+    union = __or__
+    intersection = __and__
+    difference = __sub__
+    symmetric_difference = __xor__
+
+    isdisjoint = Set.isdisjoint
+
+Set.register(PSet)
+Hashable.register(PSet)
+
+_EMPTY_PSET = PSet(pmap())
+
+
+def pset(iterable=(), pre_size=8):
+    """
+    Creates a persistent set from iterable. Optionally takes a sizing parameter equivalent to that
+    used for :py:func:`pmap`.
+
+    >>> s1 = pset([1, 2, 3, 2])
+    >>> s1
+    pset([1, 2, 3])
+    """
+    if not iterable:
+        return _EMPTY_PSET
+
+    return PSet._from_iterable(iterable, pre_size=pre_size)
+
+
+def s(*elements):
+    """
+    Create a persistent set.
+
+    Takes an arbitrary number of arguments to insert into the new set.
+
+    >>> s1 = s(1, 2, 3, 2)
+    >>> s1
+    pset([1, 2, 3])
+    """
+    return pset(elements)
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_pvector.py b/contrib/python/pyrsistent/py2/pyrsistent/_pvector.py
new file mode 100644
index 00000000000..82232782b76
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_pvector.py
@@ -0,0 +1,713 @@
+from abc import abstractmethod, ABCMeta
+from ._compat import Sequence, Hashable
+from numbers import Integral
+import operator
+import six
+from pyrsistent._transformations import transform
+
+
+def _bitcount(val):
+    return bin(val).count("1")
+
+BRANCH_FACTOR = 32
+BIT_MASK = BRANCH_FACTOR - 1
+SHIFT = _bitcount(BIT_MASK)
+
+
+def compare_pvector(v, other, operator):
+    return operator(v.tolist(), other.tolist() if isinstance(other, PVector) else other)
+
+
+def _index_or_slice(index, stop):
+    if stop is None:
+        return index
+
+    return slice(index, stop)
+
+
+class PythonPVector(object):
+    """
+    Support structure for PVector that implements structural sharing for vectors using a trie.
+    """
+    __slots__ = ('_count', '_shift', '_root', '_tail', '_tail_offset', '__weakref__')
+
+    def __new__(cls, count, shift, root, tail):
+        self = super(PythonPVector, cls).__new__(cls)
+        self._count = count
+        self._shift = shift
+        self._root = root
+        self._tail = tail
+
+        # Derived attribute stored for performance
+        self._tail_offset = self._count - len(self._tail)
+        return self
+
+    def __len__(self):
+        return self._count
+
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            # There are more conditions than the below where it would be OK to
+            # return ourselves, implement those...
+            if index.start is None and index.stop is None and index.step is None:
+                return self
+
+            # This is a bit nasty realizing the whole structure as a list before
+            # slicing it but it is the fastest way I've found to date, and it's easy :-)
+            return _EMPTY_PVECTOR.extend(self.tolist()[index])
+
+        if index < 0:
+            index += self._count
+
+        return PythonPVector._node_for(self, index)[index & BIT_MASK]
+
+    def __add__(self, other):
+        return self.extend(other)
+
+    def __repr__(self):
+        return 'pvector({0})'.format(str(self.tolist()))
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __iter__(self):
+        # This is kind of lazy and will produce some memory overhead but it is the fasted method
+        # by far of those tried since it uses the speed of the built in python list directly.
+        return iter(self.tolist())
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __eq__(self, other):
+        return self is other or (hasattr(other, '__len__') and self._count == len(other)) and compare_pvector(self, other, operator.eq)
+
+    def __gt__(self, other):
+        return compare_pvector(self, other, operator.gt)
+
+    def __lt__(self, other):
+        return compare_pvector(self, other, operator.lt)
+
+    def __ge__(self, other):
+        return compare_pvector(self, other, operator.ge)
+
+    def __le__(self, other):
+        return compare_pvector(self, other, operator.le)
+
+    def __mul__(self, times):
+        if times <= 0 or self is _EMPTY_PVECTOR:
+            return _EMPTY_PVECTOR
+
+        if times == 1:
+            return self
+
+        return _EMPTY_PVECTOR.extend(times * self.tolist())
+
+    __rmul__ = __mul__
+
+    def _fill_list(self, node, shift, the_list):
+        if shift:
+            shift -= SHIFT
+            for n in node:
+                self._fill_list(n, shift, the_list)
+        else:
+            the_list.extend(node)
+
+    def tolist(self):
+        """
+        The fastest way to convert the vector into a python list.
+        """
+        the_list = []
+        self._fill_list(self._root, self._shift, the_list)
+        the_list.extend(self._tail)
+        return the_list
+
+    def _totuple(self):
+        """
+        Returns the content as a python tuple.
+        """
+        return tuple(self.tolist())
+
+    def __hash__(self):
+        # Taking the easy way out again...
+        return hash(self._totuple())
+
+    def transform(self, *transformations):
+        return transform(self, transformations)
+
+    def __reduce__(self):
+        # Pickling support
+        return pvector, (self.tolist(),)
+
+    def mset(self, *args):
+        if len(args) % 2:
+            raise TypeError("mset expected an even number of arguments")
+
+        evolver = self.evolver()
+        for i in range(0, len(args), 2):
+            evolver[args[i]] = args[i+1]
+
+        return evolver.persistent()
+
+    class Evolver(object):
+        __slots__ = ('_count', '_shift', '_root', '_tail', '_tail_offset', '_dirty_nodes',
+                     '_extra_tail', '_cached_leafs', '_orig_pvector')
+
+        def __init__(self, v):
+            self._reset(v)
+
+        def __getitem__(self, index):
+            if not isinstance(index, Integral):
+                raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)
+
+            if index < 0:
+                index += self._count + len(self._extra_tail)
+
+            if self._count <= index < self._count + len(self._extra_tail):
+                return self._extra_tail[index - self._count]
+
+            return PythonPVector._node_for(self, index)[index & BIT_MASK]
+
+        def _reset(self, v):
+            self._count = v._count
+            self._shift = v._shift
+            self._root = v._root
+            self._tail = v._tail
+            self._tail_offset = v._tail_offset
+            self._dirty_nodes = {}
+            self._cached_leafs = {}
+            self._extra_tail = []
+            self._orig_pvector = v
+
+        def append(self, element):
+            self._extra_tail.append(element)
+            return self
+
+        def extend(self, iterable):
+            self._extra_tail.extend(iterable)
+            return self
+
+        def set(self, index, val):
+            self[index] = val
+            return self
+
+        def __setitem__(self, index, val):
+            if not isinstance(index, Integral):
+                raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)
+
+            if index < 0:
+                index += self._count + len(self._extra_tail)
+
+            if 0 <= index < self._count:
+                node = self._cached_leafs.get(index >> SHIFT)
+                if node:
+                    node[index & BIT_MASK] = val
+                elif index >= self._tail_offset:
+                    if id(self._tail) not in self._dirty_nodes:
+                        self._tail = list(self._tail)
+                        self._dirty_nodes[id(self._tail)] = True
+                        self._cached_leafs[index >> SHIFT] = self._tail
+                    self._tail[index & BIT_MASK] = val
+                else:
+                    self._root = self._do_set(self._shift, self._root, index, val)
+            elif self._count <= index < self._count + len(self._extra_tail):
+                self._extra_tail[index - self._count] = val
+            elif index == self._count + len(self._extra_tail):
+                self._extra_tail.append(val)
+            else:
+                raise IndexError("Index out of range: %s" % (index,))
+
+        def _do_set(self, level, node, i, val):
+            if id(node) in self._dirty_nodes:
+                ret = node
+            else:
+                ret = list(node)
+                self._dirty_nodes[id(ret)] = True
+
+            if level == 0:
+                ret[i & BIT_MASK] = val
+                self._cached_leafs[i >> SHIFT] = ret
+            else:
+                sub_index = (i >> level) & BIT_MASK  # >>>
+                ret[sub_index] = self._do_set(level - SHIFT, node[sub_index], i, val)
+
+            return ret
+
+        def delete(self, index):
+            del self[index]
+            return self
+
+        def __delitem__(self, key):
+            if self._orig_pvector:
+                # All structural sharing bets are off, base evolver on _extra_tail only
+                l = PythonPVector(self._count, self._shift, self._root, self._tail).tolist()
+                l.extend(self._extra_tail)
+                self._reset(_EMPTY_PVECTOR)
+                self._extra_tail = l
+
+            del self._extra_tail[key]
+
+        def persistent(self):
+            result = self._orig_pvector
+            if self.is_dirty():
+                result = PythonPVector(self._count, self._shift, self._root, self._tail).extend(self._extra_tail)
+                self._reset(result)
+
+            return result
+
+        def __len__(self):
+            return self._count + len(self._extra_tail)
+
+        def is_dirty(self):
+            return bool(self._dirty_nodes or self._extra_tail)
+
+    def evolver(self):
+        return PythonPVector.Evolver(self)
+
+    def set(self, i, val):
+        # This method could be implemented by a call to mset() but doing so would cause
+        # a ~5 X performance penalty on PyPy (considered the primary platform for this implementation
+        #  of PVector) so we're keeping this implementation for now.
+
+        if not isinstance(i, Integral):
+            raise TypeError("'%s' object cannot be interpreted as an index" % type(i).__name__)
+
+        if i < 0:
+            i += self._count
+
+        if 0 <= i < self._count:
+            if i >= self._tail_offset:
+                new_tail = list(self._tail)
+                new_tail[i & BIT_MASK] = val
+                return PythonPVector(self._count, self._shift, self._root, new_tail)
+
+            return PythonPVector(self._count, self._shift, self._do_set(self._shift, self._root, i, val), self._tail)
+
+        if i == self._count:
+            return self.append(val)
+
+        raise IndexError("Index out of range: %s" % (i,))
+
+    def _do_set(self, level, node, i, val):
+        ret = list(node)
+        if level == 0:
+            ret[i & BIT_MASK] = val
+        else:
+            sub_index = (i >> level) & BIT_MASK  # >>>
+            ret[sub_index] = self._do_set(level - SHIFT, node[sub_index], i, val)
+
+        return ret
+
+    @staticmethod
+    def _node_for(pvector_like, i):
+        if 0 <= i < pvector_like._count:
+            if i >= pvector_like._tail_offset:
+                return pvector_like._tail
+
+            node = pvector_like._root
+            for level in range(pvector_like._shift, 0, -SHIFT):
+                node = node[(i >> level) & BIT_MASK]  # >>>
+
+            return node
+
+        raise IndexError("Index out of range: %s" % (i,))
+
+    def _create_new_root(self):
+        new_shift = self._shift
+
+        # Overflow root?
+        if (self._count >> SHIFT) > (1 << self._shift): # >>>
+            new_root = [self._root, self._new_path(self._shift, self._tail)]
+            new_shift += SHIFT
+        else:
+            new_root = self._push_tail(self._shift, self._root, self._tail)
+
+        return new_root, new_shift
+
+    def append(self, val):
+        if len(self._tail) < BRANCH_FACTOR:
+            new_tail = list(self._tail)
+            new_tail.append(val)
+            return PythonPVector(self._count + 1, self._shift, self._root, new_tail)
+
+        # Full tail, push into tree
+        new_root, new_shift = self._create_new_root()
+        return PythonPVector(self._count + 1, new_shift, new_root, [val])
+
+    def _new_path(self, level, node):
+        if level == 0:
+            return node
+
+        return [self._new_path(level - SHIFT, node)]
+
+    def _mutating_insert_tail(self):
+        self._root, self._shift = self._create_new_root()
+        self._tail = []
+
+    def _mutating_fill_tail(self, offset, sequence):
+        max_delta_len = BRANCH_FACTOR - len(self._tail)
+        delta = sequence[offset:offset + max_delta_len]
+        self._tail.extend(delta)
+        delta_len = len(delta)
+        self._count += delta_len
+        return offset + delta_len
+
+    def _mutating_extend(self, sequence):
+        offset = 0
+        sequence_len = len(sequence)
+        while offset < sequence_len:
+            offset = self._mutating_fill_tail(offset, sequence)
+            if len(self._tail) == BRANCH_FACTOR:
+                self._mutating_insert_tail()
+
+        self._tail_offset = self._count - len(self._tail)
+
+    def extend(self, obj):
+        # Mutates the new vector directly for efficiency but that's only an
+        # implementation detail, once it is returned it should be considered immutable
+        l = obj.tolist() if isinstance(obj, PythonPVector) else list(obj)
+        if l:
+            new_vector = self.append(l[0])
+            new_vector._mutating_extend(l[1:])
+            return new_vector
+
+        return self
+
+    def _push_tail(self, level, parent, tail_node):
+        """
+        if parent is leaf, insert node,
+        else does it map to an existing child? ->
+             node_to_insert = push node one more level
+        else alloc new path
+
+        return  node_to_insert placed in copy of parent
+        """
+        ret = list(parent)
+
+        if level == SHIFT:
+            ret.append(tail_node)
+            return ret
+
+        sub_index = ((self._count - 1) >> level) & BIT_MASK  # >>>
+        if len(parent) > sub_index:
+            ret[sub_index] = self._push_tail(level - SHIFT, parent[sub_index], tail_node)
+            return ret
+
+        ret.append(self._new_path(level - SHIFT, tail_node))
+        return ret
+
+    def index(self, value, *args, **kwargs):
+        return self.tolist().index(value, *args, **kwargs)
+
+    def count(self, value):
+        return self.tolist().count(value)
+
+    def delete(self, index, stop=None):
+        l = self.tolist()
+        del l[_index_or_slice(index, stop)]
+        return _EMPTY_PVECTOR.extend(l)
+
+    def remove(self, value):
+        l = self.tolist()
+        l.remove(value)
+        return _EMPTY_PVECTOR.extend(l)
+
+@six.add_metaclass(ABCMeta)
+class PVector(object):
+    """
+    Persistent vector implementation. Meant as a replacement for the cases where you would normally
+    use a Python list.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`v` and :py:func:`pvector` to
+    create an instance.
+
+    Heavily influenced by the persistent vector available in Clojure. Initially this was more or
+    less just a port of the Java code for the Clojure vector. It has since been modified and to
+    some extent optimized for usage in Python.
+
+    The vector is organized as a trie, any mutating method will return a new vector that contains the changes. No
+    updates are done to the original vector. Structural sharing between vectors are applied where possible to save
+    space and to avoid making complete copies.
+
+    This structure corresponds most closely to the built in list type and is intended as a replacement. Where the
+    semantics are the same (more or less) the same function names have been used but for some cases it is not possible,
+    for example assignments.
+
+    The PVector implements the Sequence protocol and is Hashable.
+
+    Inserts are amortized O(1). Random access is log32(n) where n is the size of the vector.
+
+    The following are examples of some common operations on persistent vectors:
+
+    >>> p = v(1, 2, 3)
+    >>> p2 = p.append(4)
+    >>> p3 = p2.extend([5, 6, 7])
+    >>> p
+    pvector([1, 2, 3])
+    >>> p2
+    pvector([1, 2, 3, 4])
+    >>> p3
+    pvector([1, 2, 3, 4, 5, 6, 7])
+    >>> p3[5]
+    6
+    >>> p.set(1, 99)
+    pvector([1, 99, 3])
+    >>>
+    """
+
+    @abstractmethod
+    def __len__(self):
+        """
+        >>> len(v(1, 2, 3))
+        3
+        """
+
+    @abstractmethod
+    def __getitem__(self, index):
+        """
+        Get value at index. Full slicing support.
+
+        >>> v1 = v(5, 6, 7, 8)
+        >>> v1[2]
+        7
+        >>> v1[1:3]
+        pvector([6, 7])
+        """
+
+    @abstractmethod
+    def __add__(self, other):
+        """
+        >>> v1 = v(1, 2)
+        >>> v2 = v(3, 4)
+        >>> v1 + v2
+        pvector([1, 2, 3, 4])
+        """
+
+    @abstractmethod
+    def __mul__(self, times):
+        """
+        >>> v1 = v(1, 2)
+        >>> 3 * v1
+        pvector([1, 2, 1, 2, 1, 2])
+        """
+
+    @abstractmethod
+    def __hash__(self):
+        """
+        >>> v1 = v(1, 2, 3)
+        >>> v2 = v(1, 2, 3)
+        >>> hash(v1) == hash(v2)
+        True
+        """
+
+    @abstractmethod
+    def evolver(self):
+        """
+        Create a new evolver for this pvector. The evolver acts as a mutable view of the vector
+        with "transaction like" semantics. No part of the underlying vector i updated, it is still
+        fully immutable. Furthermore multiple evolvers created from the same pvector do not
+        interfere with each other.
+
+        You may want to use an evolver instead of working directly with the pvector in the
+        following cases:
+
+        * Multiple updates are done to the same vector and the intermediate results are of no
+          interest. In this case using an evolver may be a more efficient and easier to work with.
+        * You need to pass a vector into a legacy function or a function that you have no control
+          over which performs in place mutations of lists. In this case pass an evolver instance
+          instead and then create a new pvector from the evolver once the function returns.
+
+        The following example illustrates a typical workflow when working with evolvers. It also
+        displays most of the API (which i kept small by design, you should not be tempted to
+        use evolvers in excess ;-)).
+
+        Create the evolver and perform various mutating updates to it:
+
+        >>> v1 = v(1, 2, 3, 4, 5)
+        >>> e = v1.evolver()
+        >>> e[1] = 22
+        >>> _ = e.append(6)
+        >>> _ = e.extend([7, 8, 9])
+        >>> e[8] += 1
+        >>> len(e)
+        9
+
+        The underlying pvector remains the same:
+
+        >>> v1
+        pvector([1, 2, 3, 4, 5])
+
+        The changes are kept in the evolver. An updated pvector can be created using the
+        persistent() function on the evolver.
+
+        >>> v2 = e.persistent()
+        >>> v2
+        pvector([1, 22, 3, 4, 5, 6, 7, 8, 10])
+
+        The new pvector will share data with the original pvector in the same way that would have
+        been done if only using operations on the pvector.
+        """
+
+    @abstractmethod
+    def mset(self, *args):
+        """
+        Return a new vector with elements in specified positions replaced by values (multi set).
+
+        Elements on even positions in the argument list are interpreted as indexes while
+        elements on odd positions are considered values.
+
+        >>> v1 = v(1, 2, 3)
+        >>> v1.mset(0, 11, 2, 33)
+        pvector([11, 2, 33])
+        """
+
+    @abstractmethod
+    def set(self, i, val):
+        """
+        Return a new vector with element at position i replaced with val. The original vector remains unchanged.
+
+        Setting a value one step beyond the end of the vector is equal to appending. Setting beyond that will
+        result in an IndexError.
+
+        >>> v1 = v(1, 2, 3)
+        >>> v1.set(1, 4)
+        pvector([1, 4, 3])
+        >>> v1.set(3, 4)
+        pvector([1, 2, 3, 4])
+        >>> v1.set(-1, 4)
+        pvector([1, 2, 4])
+        """
+
+    @abstractmethod
+    def append(self, val):
+        """
+        Return a new vector with val appended.
+
+        >>> v1 = v(1, 2)
+        >>> v1.append(3)
+        pvector([1, 2, 3])
+        """
+
+    @abstractmethod
+    def extend(self, obj):
+        """
+        Return a new vector with all values in obj appended to it. Obj may be another
+        PVector or any other Iterable.
+
+        >>> v1 = v(1, 2, 3)
+        >>> v1.extend([4, 5])
+        pvector([1, 2, 3, 4, 5])
+        """
+
+    @abstractmethod
+    def index(self, value, *args, **kwargs):
+        """
+        Return first index of value. Additional indexes may be supplied to limit the search to a
+        sub range of the vector.
+
+        >>> v1 = v(1, 2, 3, 4, 3)
+        >>> v1.index(3)
+        2
+        >>> v1.index(3, 3, 5)
+        4
+        """
+
+    @abstractmethod
+    def count(self, value):
+        """
+        Return the number of times that value appears in the vector.
+
+        >>> v1 = v(1, 4, 3, 4)
+        >>> v1.count(4)
+        2
+        """
+
+    @abstractmethod
+    def transform(self, *transformations):
+        """
+        Transform arbitrarily complex combinations of PVectors and PMaps. A transformation
+        consists of two parts. One match expression that specifies which elements to transform
+        and one transformation function that performs the actual transformation.
+
+        >>> from pyrsistent import freeze, ny
+        >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
+        ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
+        ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
+        >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
+        >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
+        >>> very_short_news.articles[0].content
+        'A short article'
+        >>> very_short_news.articles[1].content
+        'A slightly long...'
+
+        When nothing has been transformed the original data structure is kept
+
+        >>> short_news is news_paper
+        True
+        >>> very_short_news is news_paper
+        False
+        >>> very_short_news.articles[0] is news_paper.articles[0]
+        True
+        """
+
+    @abstractmethod
+    def delete(self, index, stop=None):
+        """
+        Delete a portion of the vector by index or range.
+
+        >>> v1 = v(1, 2, 3, 4, 5)
+        >>> v1.delete(1)
+        pvector([1, 3, 4, 5])
+        >>> v1.delete(1, 3)
+        pvector([1, 4, 5])
+        """
+
+    @abstractmethod
+    def remove(self, value):
+        """
+        Remove the first occurrence of a value from the vector.
+
+        >>> v1 = v(1, 2, 3, 2, 1)
+        >>> v2 = v1.remove(1)
+        >>> v2
+        pvector([2, 3, 2, 1])
+        >>> v2.remove(1)
+        pvector([2, 3, 2])
+        """
+
+
+_EMPTY_PVECTOR = PythonPVector(0, SHIFT, [], [])
+PVector.register(PythonPVector)
+Sequence.register(PVector)
+Hashable.register(PVector)
+
+def python_pvector(iterable=()):
+    """
+    Create a new persistent vector containing the elements in iterable.
+
+    >>> v1 = pvector([1, 2, 3])
+    >>> v1
+    pvector([1, 2, 3])
+    """
+    return _EMPTY_PVECTOR.extend(iterable)
+
+try:
+    # Use the C extension as underlying trie implementation if it is available
+    import os
+    if os.environ.get('PYRSISTENT_NO_C_EXTENSION'):
+        pvector = python_pvector
+    else:
+        from pvectorc import pvector
+        PVector.register(type(pvector()))
+except ImportError:
+    pvector = python_pvector
+
+
+def v(*elements):
+    """
+    Create a new persistent vector containing all parameters to this function.
+
+    >>> v1 = v(1, 2, 3)
+    >>> v1
+    pvector([1, 2, 3])
+    """
+    return pvector(elements)
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_toolz.py b/contrib/python/pyrsistent/py2/pyrsistent/_toolz.py
new file mode 100644
index 00000000000..6643ee860dd
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_toolz.py
@@ -0,0 +1,83 @@
+"""
+Functionality copied from the toolz package to avoid having
+to add toolz as a dependency.
+
+See https://github.com/pytoolz/toolz/.
+
+toolz is relased under BSD licence. Below is the licence text
+from toolz as it appeared when copying the code.
+
+--------------------------------------------------------------
+
+Copyright (c) 2013 Matthew Rocklin
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  a. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  b. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  c. Neither the name of toolz nor the names of its contributors
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+"""
+import operator
+from six.moves import reduce
+
+
+def get_in(keys, coll, default=None, no_default=False):
+    """
+    NB: This is a straight copy of the get_in implementation found in
+        the toolz library (https://github.com/pytoolz/toolz/). It works
+        with persistent data structures as well as the corresponding
+        datastructures from the stdlib.
+
+    Returns coll[i0][i1]...[iX] where [i0, i1, ..., iX]==keys.
+
+    If coll[i0][i1]...[iX] cannot be found, returns ``default``, unless
+    ``no_default`` is specified, then it raises KeyError or IndexError.
+
+    ``get_in`` is a generalization of ``operator.getitem`` for nested data
+    structures such as dictionaries and lists.
+    >>> from pyrsistent import freeze
+    >>> transaction = freeze({'name': 'Alice',
+    ...                       'purchase': {'items': ['Apple', 'Orange'],
+    ...                                    'costs': [0.50, 1.25]},
+    ...                       'credit card': '5555-1234-1234-1234'})
+    >>> get_in(['purchase', 'items', 0], transaction)
+    'Apple'
+    >>> get_in(['name'], transaction)
+    'Alice'
+    >>> get_in(['purchase', 'total'], transaction)
+    >>> get_in(['purchase', 'items', 'apple'], transaction)
+    >>> get_in(['purchase', 'items', 10], transaction)
+    >>> get_in(['purchase', 'total'], transaction, 0)
+    0
+    >>> get_in(['y'], {}, no_default=True)
+    Traceback (most recent call last):
+        ...
+    KeyError: 'y'
+    """
+    try:
+        return reduce(operator.getitem, keys, coll)
+    except (KeyError, IndexError, TypeError):
+        if no_default:
+            raise
+        return default
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/_transformations.py b/contrib/python/pyrsistent/py2/pyrsistent/_transformations.py
new file mode 100644
index 00000000000..612098969b5
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/_transformations.py
@@ -0,0 +1,143 @@
+import re
+import six
+try:
+    from inspect import Parameter, signature
+except ImportError:
+    signature = None
+    try:
+        from inspect import getfullargspec as getargspec
+    except ImportError:
+        from inspect import getargspec
+
+
+_EMPTY_SENTINEL = object()
+
+
+def inc(x):
+    """ Add one to the current value """
+    return x + 1
+
+
+def dec(x):
+    """ Subtract one from the current value """
+    return x - 1
+
+
+def discard(evolver, key):
+    """ Discard the element and returns a structure without the discarded elements """
+    try:
+        del evolver[key]
+    except KeyError:
+        pass
+
+
+# Matchers
+def rex(expr):
+    """ Regular expression matcher to use together with transform functions """
+    r = re.compile(expr)
+    return lambda key: isinstance(key, six.string_types) and r.match(key)
+
+
+def ny(_):
+    """ Matcher that matches any value """
+    return True
+
+
+# Support functions
+def _chunks(l, n):
+    for i in range(0, len(l), n):
+        yield l[i:i + n]
+
+
+def transform(structure, transformations):
+    r = structure
+    for path, command in _chunks(transformations, 2):
+        r = _do_to_path(r, path, command)
+    return r
+
+
+def _do_to_path(structure, path, command):
+    if not path:
+        return command(structure) if callable(command) else command
+
+    kvs = _get_keys_and_values(structure, path[0])
+    return _update_structure(structure, kvs, path[1:], command)
+
+
+def _items(structure):
+    try:
+        return structure.items()
+    except AttributeError:
+        # Support wider range of structures by adding a transform_items() or similar?
+        return list(enumerate(structure))
+
+
+def _get(structure, key, default):
+    try:
+        if hasattr(structure, '__getitem__'):
+            return structure[key]
+
+        return getattr(structure, key)
+
+    except (IndexError, KeyError):
+        return default
+
+
+def _get_keys_and_values(structure, key_spec):
+    if callable(key_spec):
+        # Support predicates as callable objects in the path
+        arity = _get_arity(key_spec)
+        if arity == 1:
+            # Unary predicates are called with the "key" of the path
+            # - eg a key in a mapping, an index in a sequence.
+            return [(k, v) for k, v in _items(structure) if key_spec(k)]
+        elif arity == 2:
+            # Binary predicates are called with the key and the corresponding
+            # value.
+            return [(k, v) for k, v in _items(structure) if key_spec(k, v)]
+        else:
+            # Other arities are an error.
+            raise ValueError(
+                "callable in transform path must take 1 or 2 arguments"
+            )
+
+    # Non-callables are used as-is as a key.
+    return [(key_spec, _get(structure, key_spec, _EMPTY_SENTINEL))]
+
+
+if signature is None:
+    def _get_arity(f):
+        argspec = getargspec(f)
+        return len(argspec.args) - len(argspec.defaults or ())
+else:
+    def _get_arity(f):
+        return sum(
+            1
+            for p
+            in signature(f).parameters.values()
+            if p.default is Parameter.empty
+            and p.kind in (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD)
+        )
+
+
+def _update_structure(structure, kvs, path, command):
+    from pyrsistent._pmap import pmap
+    e = structure.evolver()
+    if not path and command is discard:
+        # Do this in reverse to avoid index problems with vectors. See #92.
+        for k, v in reversed(kvs):
+            discard(e, k)
+    else:
+        for k, v in kvs:
+            is_empty = False
+            if v is _EMPTY_SENTINEL:
+                # Allow expansion of structure but make sure to cover the case
+                # when an empty pmap is added as leaf node. See #154.
+                is_empty = True
+                v = pmap()
+
+            result = _do_to_path(v, path, command)
+            if result is not v or is_empty:
+                e[k] = result
+
+    return e.persistent()
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/py.typed b/contrib/python/pyrsistent/py2/pyrsistent/py.typed
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/pyrsistent/py2/pyrsistent/typing.py b/contrib/python/pyrsistent/py2/pyrsistent/typing.py
new file mode 100644
index 00000000000..6a86c831ba4
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/pyrsistent/typing.py
@@ -0,0 +1,80 @@
+"""Helpers for use with type annotation.
+
+Use the empty classes in this module when annotating the types of Pyrsistent
+objects, instead of using the actual collection class.
+
+For example,
+
+    from pyrsistent import pvector
+    from pyrsistent.typing import PVector
+
+    myvector: PVector[str] = pvector(['a', 'b', 'c'])
+
+"""
+from __future__ import absolute_import
+
+try:
+    from typing import Container
+    from typing import Hashable
+    from typing import Generic
+    from typing import Iterable
+    from typing import Mapping
+    from typing import Sequence
+    from typing import Sized
+    from typing import TypeVar
+
+    __all__ = [
+        'CheckedPMap',
+        'CheckedPSet',
+        'CheckedPVector',
+        'PBag',
+        'PDeque',
+        'PList',
+        'PMap',
+        'PSet',
+        'PVector',
+    ]
+
+    T = TypeVar('T')
+    KT = TypeVar('KT')
+    VT = TypeVar('VT')
+
+    class CheckedPMap(Mapping[KT, VT], Hashable):
+        pass
+
+    # PSet.add and PSet.discard have different type signatures than that of Set.
+    class CheckedPSet(Generic[T], Hashable):
+        pass
+
+    class CheckedPVector(Sequence[T], Hashable):
+        pass
+
+    class PBag(Container[T], Iterable[T], Sized, Hashable):
+        pass
+
+    class PDeque(Sequence[T], Hashable):
+        pass
+
+    class PList(Sequence[T], Hashable):
+        pass
+
+    class PMap(Mapping[KT, VT], Hashable):
+        pass
+
+    # PSet.add and PSet.discard have different type signatures than that of Set.
+    class PSet(Generic[T], Hashable):
+        pass
+
+    class PVector(Sequence[T], Hashable):
+        pass
+
+    class PVectorEvolver(Generic[T]):
+        pass
+
+    class PMapEvolver(Generic[KT, VT]):
+        pass
+
+    class PSetEvolver(Generic[T]):
+        pass
+except ImportError:
+    pass
diff --git a/contrib/python/pyrsistent/py2/tests/bag_test.py b/contrib/python/pyrsistent/py2/tests/bag_test.py
new file mode 100644
index 00000000000..fb80603108c
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/bag_test.py
@@ -0,0 +1,150 @@
+import pytest
+
+from pyrsistent import b, pbag
+
+
+def test_literalish_works():
+    assert b(1, 2) == pbag([1, 2])
+
+def test_empty_bag():
+    """
+    creating an empty pbag returns a singleton.
+
+    Note that this should NOT be relied upon in application code.
+    """
+    assert b() is b()
+
+def test_supports_hash():
+    assert hash(b(1, 2)) == hash(b(2, 1))
+
+def test_hash_in_dict():
+    assert {b(1,2,3,3): "hello"}[b(3,3,2,1)] == "hello"
+
+def test_empty_truthiness():
+    assert b(1)
+    assert not b()
+
+
+def test_repr_empty():
+    assert repr(b()) == 'pbag([])'
+
+def test_repr_elements():
+    assert repr(b(1, 2)) in ('pbag([1, 2])', 'pbag([2, 1])')
+
+
+def test_add_empty():
+    assert b().add(1) == b(1)
+
+def test_remove_final():
+    assert b().add(1).remove(1) == b()
+
+def test_remove_nonfinal():
+    assert b().add(1).add(1).remove(1) == b(1)
+
+def test_remove_nonexistent():
+    with pytest.raises(KeyError) as excinfo:
+        b().remove(1)
+    assert str(excinfo.exconly()) == 'KeyError: 1'
+
+
+def test_eq_empty():
+    assert b() == b()
+
+def test_neq():
+    assert b(1) != b()
+
+def test_eq_same_order():
+    assert b(1, 2, 1) == b(1, 2, 1)
+
+def test_eq_different_order():
+    assert b(2, 1, 2) == b(1, 2, 2)
+
+
+def test_count_non_existent():
+    assert b().count(1) == 0
+
+def test_count_unique():
+    assert b(1).count(1) == 1
+
+def test_count_duplicate():
+    assert b(1, 1).count(1) == 2
+
+
+def test_length_empty():
+    assert len(b()) == 0
+
+def test_length_unique():
+    assert len(b(1)) == 1
+
+def test_length_duplicates():
+    assert len(b(1, 1)) == 2
+
+def test_length_multiple_elements():
+    assert len(b(1, 1, 2, 3)) == 4
+
+
+def test_iter_duplicates():
+    assert list(b(1, 1)) == [1, 1]
+
+def test_iter_multiple_elements():
+    assert list(b(1, 2, 2)) in ([1, 2, 2], [2, 2, 1])
+
+def test_contains():
+    assert 1 in b(1)
+
+def test_not_contains():
+    assert 1 not in b(2)
+
+def test_add():
+    assert b(3, 3, 3, 2, 2, 1) + b(4, 3, 2, 1) == b(4,
+                                                    3, 3, 3, 3,
+                                                    2, 2, 2,
+                                                    1, 1)
+
+def test_sub():
+    assert b(1, 2, 3, 3) - b(3, 4) == b(1, 2, 3)
+
+def test_or():
+    assert b(1, 2, 2, 3, 3, 3) | b(1, 2, 3, 4, 4) == b(1,
+                                                       2, 2,
+                                                       3, 3, 3,
+                                                       4, 4)
+
+def test_and():
+    assert b(1, 2, 2, 3, 3, 3) & b(2, 3, 3, 4) == b(2, 3, 3)
+
+
+def test_pbag_is_unorderable():
+    with pytest.raises(TypeError):
+        _ = b(1) < b(2)  # type: ignore
+
+    with pytest.raises(TypeError):
+        _ = b(1) <= b(2)  # type: ignore
+
+    with pytest.raises(TypeError):
+        _ = b(1) > b(2)  # type: ignore
+
+    with pytest.raises(TypeError):
+        _ = b(1) >= b(2)  # type: ignore
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(b(1))
+
+
+def test_update():
+    assert pbag([1, 2, 2]).update([3, 3, 4]) == pbag([1, 2, 2, 3, 3, 4])
+
+
+def test_update_no_elements():
+    b = pbag([1, 2, 2])
+    assert b.update([]) is b
+
+
+def test_iterable():
+    """
+    PBags can be created from iterables even though they can't be len() hinted.
+    """
+
+    assert pbag(iter("a")) == pbag(iter("a"))
diff --git a/contrib/python/pyrsistent/py2/tests/checked_map_test.py b/contrib/python/pyrsistent/py2/tests/checked_map_test.py
new file mode 100644
index 00000000000..acc8816c6f7
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/checked_map_test.py
@@ -0,0 +1,151 @@
+import pickle
+import pytest
+from pyrsistent import CheckedPMap, InvariantException, PMap, CheckedType, CheckedPSet, CheckedPVector, \
+    CheckedKeyTypeError, CheckedValueTypeError
+
+
+class FloatToIntMap(CheckedPMap):
+    __key_type__ = float
+    __value_type__ = int
+    __invariant__ = lambda key, value: (int(key) == value, 'Invalid mapping')
+
+def test_instantiate():
+    x = FloatToIntMap({1.25: 1, 2.5: 2})
+
+    assert dict(x.items()) == {1.25: 1, 2.5: 2}
+    assert isinstance(x, FloatToIntMap)
+    assert isinstance(x, PMap)
+    assert isinstance(x, CheckedType)
+
+def test_instantiate_empty():
+    x = FloatToIntMap()
+
+    assert dict(x.items()) == {}
+    assert isinstance(x, FloatToIntMap)
+
+def test_set():
+     x = FloatToIntMap()
+     x2 = x.set(1.0, 1)
+
+     assert x2[1.0] == 1
+     assert isinstance(x2, FloatToIntMap)
+
+def test_invalid_key_type():
+     with pytest.raises(CheckedKeyTypeError):
+         FloatToIntMap({1: 1})
+
+def test_invalid_value_type():
+     with pytest.raises(CheckedValueTypeError):
+         FloatToIntMap({1.0: 1.0})
+
+def test_breaking_invariant():
+     try:
+         FloatToIntMap({1.5: 2})
+         assert False
+     except InvariantException as e:
+        assert e.invariant_errors == ('Invalid mapping',)
+
+def test_repr():
+    x = FloatToIntMap({1.25: 1})
+
+    assert str(x) == 'FloatToIntMap({1.25: 1})'
+
+def test_default_serialization():
+    x = FloatToIntMap({1.25: 1, 2.5: 2})
+
+    assert x.serialize() == {1.25: 1, 2.5: 2}
+
+class StringFloatToIntMap(FloatToIntMap):
+    @staticmethod
+    def __serializer__(format, key, value):
+        return format.format(key), format.format(value)
+
+def test_custom_serialization():
+    x = StringFloatToIntMap({1.25: 1, 2.5: 2})
+
+    assert x.serialize("{0}") == {"1.25": "1", "2.5": "2"}
+
+class FloatSet(CheckedPSet):
+    __type__ = float
+
+class IntToFloatSetMap(CheckedPMap):
+    __key_type__ = int
+    __value_type__ = FloatSet
+
+
+def test_multi_level_serialization():
+    x = IntToFloatSetMap.create({1: [1.25, 1.50], 2: [2.5, 2.75]})
+
+    assert str(x) == "IntToFloatSetMap({1: FloatSet([1.5, 1.25]), 2: FloatSet([2.75, 2.5])})"
+
+    sx = x.serialize()
+    assert sx == {1: set([1.5, 1.25]), 2: set([2.75, 2.5])}
+    assert isinstance(sx[1], set)
+
+def test_create_non_checked_types():
+    assert FloatToIntMap.create({1.25: 1, 2.5: 2}) == FloatToIntMap({1.25: 1, 2.5: 2})
+
+def test_create_checked_types():
+    class IntSet(CheckedPSet):
+        __type__ = int
+
+    class FloatVector(CheckedPVector):
+        __type__ = float
+
+    class IntSetToFloatVectorMap(CheckedPMap):
+        __key_type__ = IntSet
+        __value_type__ = FloatVector
+
+    x = IntSetToFloatVectorMap.create({frozenset([1, 2]): [1.25, 2.5]})
+
+    assert str(x) == "IntSetToFloatVectorMap({IntSet([1, 2]): FloatVector([1.25, 2.5])})"
+
+def test_evolver_returns_same_instance_when_no_updates():
+    x = FloatToIntMap({1.25: 1, 2.25: 2})
+
+    assert x.evolver().persistent() is x
+
+def test_map_with_no_types_or_invariants():
+    class NoCheckPMap(CheckedPMap):
+        pass
+
+    x = NoCheckPMap({1: 2, 3: 4})
+    assert x[1] == 2
+    assert x[3] == 4
+
+
+def test_pickling():
+    x = FloatToIntMap({1.25: 1, 2.5: 2})
+    y = pickle.loads(pickle.dumps(x, -1))
+
+    assert x == y
+    assert isinstance(y, FloatToIntMap)
+
+
+class FloatVector(CheckedPVector):
+    __type__ = float
+
+class VectorToSetMap(CheckedPMap):
+    __key_type__ = '__tests__.checked_map_test.FloatVector'
+    __value_type__ = '__tests__.checked_map_test.FloatSet'
+
+
+def test_type_check_with_string_specification():
+    content = [1.5, 2.0]
+    vec = FloatVector(content)
+    sett = FloatSet(content)
+    map = VectorToSetMap({vec: sett})
+
+    assert map[vec] == sett
+
+
+def test_type_creation_with_string_specification():
+    content = (1.5, 2.0)
+    map = VectorToSetMap.create({content: content})
+
+    assert map[FloatVector(content)] == set(content)
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(VectorToSetMap({}))
diff --git a/contrib/python/pyrsistent/py2/tests/checked_set_test.py b/contrib/python/pyrsistent/py2/tests/checked_set_test.py
new file mode 100644
index 00000000000..f0be4963e21
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/checked_set_test.py
@@ -0,0 +1,85 @@
+import pickle
+import pytest
+from pyrsistent import CheckedPSet, PSet, InvariantException, CheckedType, CheckedPVector, CheckedValueTypeError
+
+
+class Naturals(CheckedPSet):
+    __type__ = int
+    __invariant__ = lambda value: (value >= 0, 'Negative value')
+
+def test_instantiate():
+    x = Naturals([1, 2, 3, 3])
+
+    assert list(x) == [1, 2, 3]
+    assert isinstance(x, Naturals)
+    assert isinstance(x, PSet)
+    assert isinstance(x, CheckedType)
+
+def test_add():
+    x = Naturals()
+    x2 = x.add(1)
+
+    assert list(x2) == [1]
+    assert isinstance(x2, Naturals)
+
+def test_invalid_type():
+    with pytest.raises(CheckedValueTypeError):
+        Naturals([1, 2.0])
+
+def test_breaking_invariant():
+    try:
+        Naturals([1, -1])
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+def test_repr():
+    x = Naturals([1, 2])
+
+    assert str(x) == 'Naturals([1, 2])'
+
+def test_default_serialization():
+    x = Naturals([1, 2])
+
+    assert x.serialize() == set([1, 2])
+
+class StringNaturals(Naturals):
+    @staticmethod
+    def __serializer__(format, value):
+        return format.format(value)
+
+def test_custom_serialization():
+    x = StringNaturals([1, 2])
+
+    assert x.serialize("{0}") == set(["1", "2"])
+
+class NaturalsVector(CheckedPVector):
+    __type__ = Naturals
+
+def test_multi_level_serialization():
+    x = NaturalsVector.create([[1, 2], [3, 4]])
+
+    assert str(x) == "NaturalsVector([Naturals([1, 2]), Naturals([3, 4])])"
+
+    sx = x.serialize()
+    assert sx == [set([1, 2]), set([3, 4])]
+    assert isinstance(sx[0], set)
+
+def test_create():
+    assert Naturals.create([1, 2]) == Naturals([1, 2])
+
+def test_evolver_returns_same_instance_when_no_updates():
+    x = Naturals([1, 2])
+    assert x.evolver().persistent() is x
+
+def test_pickling():
+    x = Naturals([1, 2])
+    y = pickle.loads(pickle.dumps(x, -1))
+
+    assert x == y
+    assert isinstance(y, Naturals)
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(Naturals([1, 2]))
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py2/tests/checked_vector_test.py b/contrib/python/pyrsistent/py2/tests/checked_vector_test.py
new file mode 100644
index 00000000000..b2e3d43cd65
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/checked_vector_test.py
@@ -0,0 +1,213 @@
+import datetime
+import pickle
+import pytest
+from pyrsistent import CheckedPVector, InvariantException, optional, CheckedValueTypeError, PVector
+
+
+class Naturals(CheckedPVector):
+    __type__ = int
+    __invariant__ = lambda value: (value >= 0, 'Negative value')
+
+def test_instantiate():
+    x = Naturals([1, 2, 3])
+
+    assert list(x) == [1, 2, 3]
+    assert isinstance(x, Naturals)
+    assert isinstance(x, PVector)
+
+def test_append():
+    x = Naturals()
+    x2 = x.append(1)
+
+    assert list(x2) == [1]
+    assert isinstance(x2, Naturals)
+
+def test_extend():
+    x = Naturals()
+    x2 = x.extend([1])
+
+    assert list(x2) == [1]
+    assert isinstance(x2, Naturals)
+
+def test_set():
+    x = Naturals([1, 2])
+    x2 = x.set(1, 3)
+
+    assert list(x2) == [1, 3]
+    assert isinstance(x2, Naturals)
+
+
+def test_invalid_type():
+    try:
+        Naturals([1, 2.0])
+        assert False
+    except CheckedValueTypeError as e:
+        assert e.expected_types == (int,)
+        assert e.actual_type is float
+        assert e.actual_value == 2.0
+        assert e.source_class is Naturals
+
+    x = Naturals([1, 2])
+    with pytest.raises(TypeError):
+        x.append(3.0)
+
+    with pytest.raises(TypeError):
+        x.extend([3, 4.0])
+
+    with pytest.raises(TypeError):
+        x.set(1, 2.0)
+
+    with pytest.raises(TypeError):
+        x.evolver()[1] = 2.0
+
+def test_breaking_invariant():
+    try:
+        Naturals([1, -1])
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+    x = Naturals([1, 2])
+    try:
+        x.append(-1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+    try:
+        x.extend([-1])
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+    try:
+        x.set(1, -1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+def test_create_base_case():
+    x =  Naturals.create([1, 2, 3])
+
+    assert isinstance(x, Naturals)
+    assert x == Naturals([1, 2, 3])
+
+def test_create_with_instance_of_checked_pvector_returns_the_argument():
+    x =  Naturals([1, 2, 3])
+
+    assert Naturals.create(x) is x
+
+class OptionalNaturals(CheckedPVector):
+    __type__ = optional(int)
+    __invariant__ = lambda value: (value is None or value >= 0, 'Negative value')
+
+def test_multiple_allowed_types():
+    assert list(OptionalNaturals([1, None, 3])) == [1, None, 3]
+
+class NaturalsVector(CheckedPVector):
+    __type__ = optional(Naturals)
+
+def test_create_of_nested_structure():
+    assert NaturalsVector([Naturals([1, 2]), Naturals([3, 4]), None]) ==\
+           NaturalsVector.create([[1, 2], [3, 4], None])
+
+def test_serialize_default_case():
+    v = CheckedPVector([1, 2, 3])
+    assert v.serialize() == [1, 2, 3]
+
+class Dates(CheckedPVector):
+    __type__ = datetime.date
+
+    @staticmethod
+    def __serializer__(format, d):
+        return d.strftime(format)
+
+def test_serialize_custom_serializer():
+    d = datetime.date
+    v = Dates([d(2015, 2, 2), d(2015, 2, 3)])
+    assert v.serialize(format='%Y-%m-%d') == ['2015-02-02', '2015-02-03']
+
+def test_type_information_is_inherited():
+    class MultiDates(Dates):
+        __type__ = int
+
+    MultiDates([datetime.date(2015, 2, 4), 5])
+
+    with pytest.raises(TypeError):
+        MultiDates([5.0])
+
+def test_invariants_are_inherited():
+    class LimitNaturals(Naturals):
+        __invariant__ = lambda value: (value < 10, 'Too big')
+
+    try:
+        LimitNaturals([10, -1])
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Too big', 'Negative value')
+
+def test_invariant_must_be_callable():
+    with pytest.raises(TypeError):
+        class InvalidInvariant(CheckedPVector):
+            __invariant__ = 1
+
+def test_type_spec_must_be_type():
+    with pytest.raises(TypeError):
+        class InvalidType(CheckedPVector):
+            __type__ = 1
+
+def test_repr():
+    x = Naturals([1, 2])
+
+    assert str(x) == 'Naturals([1, 2])'
+
+def test_evolver_returns_same_instance_when_no_updates():
+    x = Naturals([1, 2])
+    assert x.evolver().persistent() is x
+
+def test_pickling():
+    x = Naturals([1, 2])
+    y = pickle.loads(pickle.dumps(x, -1))
+
+    assert x == y
+    assert isinstance(y, Naturals)
+
+def test_multiple_optional_types():
+    class Numbers(CheckedPVector):
+        __type__ = optional(int, float)
+
+    numbers = Numbers([1, 2.5, None])
+    assert numbers.serialize() == [1, 2.5, None]
+
+    with pytest.raises(TypeError):
+        numbers.append('foo')
+
+
+class NaturalsVectorStr(CheckedPVector):
+    __type__ = '__tests__.checked_vector_test.Naturals'
+
+
+def test_check_with_string_specification():
+    naturals_list = [Naturals([1, 2]), Naturals([3, 4])]
+    nv = NaturalsVectorStr(naturals_list)
+    assert nv == naturals_list
+
+
+def test_create_with_string_specification():
+    naturals_list = [[1, 2], [3, 4]]
+    nv = NaturalsVectorStr.create(naturals_list)
+    assert nv == naturals_list
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(Naturals([]))
+
+
+def test_create_with_generator_iterator():
+    # See issue #97
+    class Numbers(CheckedPVector):
+        __type__ = int
+
+    n = Numbers(i for i in [1, 2, 3])
+    assert n == Numbers([1, 2, 3])
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py2/tests/class_test.py b/contrib/python/pyrsistent/py2/tests/class_test.py
new file mode 100644
index 00000000000..f87c3e91ca6
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/class_test.py
@@ -0,0 +1,477 @@
+from pyrsistent._compat import Hashable
+import math
+import pickle
+import pytest
+import sys
+import uuid
+from pyrsistent import (
+    field, InvariantException, PClass, optional, CheckedPVector,
+    pmap_field, pset_field, pvector_field)
+
+
+class Point(PClass):
+    x = field(type=int, mandatory=True, invariant=lambda x: (x >= 0, 'X negative'))
+    y = field(type=int, serializer=lambda formatter, y: formatter(y))
+    z = field(type=int, initial=0)
+
+
+class Hierarchy(PClass):
+    point = field(type=Point)
+
+
+class TypedContainerObj(PClass):
+    map = pmap_field(str, str)
+    set = pset_field(str)
+    vec = pvector_field(str)
+
+
+class UniqueThing(PClass):
+    id = field(type=uuid.UUID, factory=uuid.UUID)
+    x = field(type=int)
+
+
+def test_create_ignore_extra():
+    p = Point.create({'x': 5, 'y': 10, 'z': 15, 'a': 0}, ignore_extra=True)
+    assert p.x == 5
+    assert p.y == 10
+    assert p.z == 15
+    assert isinstance(p, Point)
+
+
+def test_create_ignore_extra_false():
+    with pytest.raises(AttributeError):
+        _ = Point.create({'x': 5, 'y': 10, 'z': 15, 'a': 0})
+
+
+def test_create_ignore_extra_true():
+    h = Hierarchy.create(
+        {'point': {'x': 5, 'y': 10, 'z': 15, 'extra_field_0': 'extra_data_0'}, 'extra_field_1': 'extra_data_1'},
+        ignore_extra=True)
+    assert isinstance(h, Hierarchy)
+
+
+def test_evolve_pclass_instance():
+    p = Point(x=1, y=2)
+    p2 = p.set(x=p.x+2)
+
+    # Original remains
+    assert p.x == 1
+    assert p.y == 2
+
+    # Evolved object updated
+    assert p2.x == 3
+    assert p2.y == 2
+
+    p3 = p2.set('x', 4)
+    assert p3.x == 4
+    assert p3.y == 2
+
+
+def test_direct_assignment_not_possible():
+    p = Point(x=1, y=2)
+
+    with pytest.raises(AttributeError):
+        p.x = 1
+
+    with pytest.raises(AttributeError):
+        setattr(p, 'x', 1)
+
+
+def test_direct_delete_not_possible():
+    p = Point(x=1, y=2)
+    with pytest.raises(AttributeError):
+        del p.x
+
+    with pytest.raises(AttributeError):
+        delattr(p, 'x')
+
+
+def test_cannot_construct_with_undeclared_fields():
+    with pytest.raises(AttributeError):
+        Point(x=1, p=5)
+
+
+def test_cannot_construct_with_wrong_type():
+    with pytest.raises(TypeError):
+        Point(x='a')
+
+
+def test_cannot_construct_without_mandatory_fields():
+    try:
+        Point(y=1)
+        assert False
+    except InvariantException as e:
+        assert "[Point.x]" in str(e)
+
+
+def test_field_invariant_must_hold():
+    try:
+        Point(x=-1)
+        assert False
+    except InvariantException as e:
+        assert "X negative" in str(e)
+
+
+def test_initial_value_set_when_not_present_in_arguments():
+    p = Point(x=1, y=2)
+
+    assert p.z == 0
+
+
+class Line(PClass):
+    p1 = field(type=Point)
+    p2 = field(type=Point)
+
+
+def test_can_create_nested_structures_from_dict_and_serialize_back_to_dict():
+    source = dict(p1=dict(x=1, y=2, z=3), p2=dict(x=10, y=20, z=30))
+    l = Line.create(source)
+
+    assert l.p1.x == 1
+    assert l.p1.y == 2
+    assert l.p1.z == 3
+    assert l.p2.x == 10
+    assert l.p2.y == 20
+    assert l.p2.z == 30
+
+    assert l.serialize(format=lambda val: val) == source
+
+
+def test_can_serialize_with_custom_serializer():
+    p = Point(x=1, y=1, z=1)
+
+    assert p.serialize(format=lambda v: v + 17) == {'x': 1, 'y': 18, 'z': 1}
+
+
+def test_implements_proper_equality_based_on_equality_of_fields():
+    p1 = Point(x=1, y=2)
+    p2 = Point(x=3)
+    p3 = Point(x=1, y=2)
+
+    assert p1 == p3
+    assert not p1 != p3
+    assert p1 != p2
+    assert not p1 == p2
+
+
+def test_is_hashable():
+    p1 = Point(x=1, y=2)
+    p2 = Point(x=3, y=2)
+
+    d = {p1: 'A point', p2: 'Another point'}
+
+    p1_like = Point(x=1, y=2)
+    p2_like = Point(x=3, y=2)
+
+    assert isinstance(p1, Hashable)
+    assert d[p1_like] == 'A point'
+    assert d[p2_like] == 'Another point'
+    assert Point(x=10) not in d
+
+
+def test_supports_nested_transformation():
+    l1 = Line(p1=Point(x=2, y=1), p2=Point(x=20, y=10))
+
+    l2 = l1.transform(['p1', 'x'], 3)
+
+    assert l1.p1.x == 2
+
+    assert l2.p1.x == 3
+    assert l2.p1.y == 1
+    assert l2.p2.x == 20
+    assert l2.p2.y == 10
+
+
+def test_repr():
+    class ARecord(PClass):
+        a = field()
+        b = field()
+
+    assert repr(ARecord(a=1, b=2)) in ('ARecord(a=1, b=2)', 'ARecord(b=2, a=1)')
+
+
+def test_global_invariant_check():
+    class UnitCirclePoint(PClass):
+        __invariant__ = lambda cp: (0.99 < math.sqrt(cp.x*cp.x + cp.y*cp.y) < 1.01,
+                                    "Point not on unit circle")
+        x = field(type=float)
+        y = field(type=float)
+
+    UnitCirclePoint(x=1.0, y=0.0)
+
+    with pytest.raises(InvariantException):
+        UnitCirclePoint(x=1.0, y=1.0)
+
+
+def test_supports_pickling():
+    p1 = Point(x=2, y=1)
+    p2 = pickle.loads(pickle.dumps(p1, -1))
+
+    assert p1 == p2
+    assert isinstance(p2, Point)
+
+
+def test_supports_pickling_with_typed_container_fields():
+    obj = TypedContainerObj(map={'foo': 'bar'}, set=['hello', 'there'], vec=['a', 'b'])
+    obj2 = pickle.loads(pickle.dumps(obj))
+    assert obj == obj2
+
+
+def test_can_remove_optional_member():
+    p1 = Point(x=1, y=2)
+    p2 = p1.remove('y')
+
+    assert p2 == Point(x=1)
+
+
+def test_cannot_remove_mandatory_member():
+    p1 = Point(x=1, y=2)
+
+    with pytest.raises(InvariantException):
+        p1.remove('x')
+
+
+def test_cannot_remove_non_existing_member():
+    p1 = Point(x=1)
+
+    with pytest.raises(AttributeError):
+        p1.remove('y')
+
+
+def test_evolver_without_evolution_returns_original_instance():
+    p1 = Point(x=1)
+    e = p1.evolver()
+
+    assert e.persistent() is p1
+
+
+def test_evolver_with_evolution_to_same_element_returns_original_instance():
+    p1 = Point(x=1)
+    e = p1.evolver()
+    e.set('x', p1.x)
+
+    assert e.persistent() is p1
+
+
+def test_evolver_supports_chained_set_and_remove():
+    p1 = Point(x=1, y=2)
+
+    assert p1.evolver().set('x', 3).remove('y').persistent() == Point(x=3)
+
+
+def test_evolver_supports_dot_notation_for_setting_and_getting_elements():
+    e = Point(x=1, y=2).evolver()
+
+    e.x = 3
+    assert e.x == 3
+    assert e.persistent() == Point(x=3, y=2)
+
+
+class Numbers(CheckedPVector):
+    __type__ = int
+
+
+class LinkedList(PClass):
+    value = field(type='__tests__.class_test.Numbers')
+    next = field(type=optional('__tests__.class_test.LinkedList'))
+
+
+def test_string_as_type_specifier():
+    l = LinkedList(value=[1, 2], next=LinkedList(value=[3, 4], next=None))
+
+    assert isinstance(l.value, Numbers)
+    assert list(l.value) == [1, 2]
+    assert l.next.next is None
+
+
+def test_multiple_invariants_on_field():
+    # If the invariant returns a list of tests the results of running those tests will be
+    # a tuple containing result data of all failing tests.
+
+    class MultiInvariantField(PClass):
+        one = field(type=int, invariant=lambda x: ((False, 'one_one'),
+                                                   (False, 'one_two'),
+                                                   (True, 'one_three')))
+        two = field(invariant=lambda x: (False, 'two_one'))
+
+    try:
+        MultiInvariantField(one=1, two=2)
+        assert False
+    except InvariantException as e:
+        assert set(e.invariant_errors) == set([('one_one', 'one_two'), 'two_one'])
+
+
+def test_multiple_global_invariants():
+    class MultiInvariantGlobal(PClass):
+        __invariant__ = lambda self: ((False, 'x'), (False, 'y'))
+        one = field()
+
+    try:
+        MultiInvariantGlobal(one=1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == (('x', 'y'),)
+
+
+def test_inherited_global_invariants():
+    class Distant(object):
+        def __invariant__(self):
+            return [(self.distant, "distant")]
+
+    class Nearby(Distant):
+        def __invariant__(self):
+            return [(self.nearby, "nearby")]
+
+    class MultipleInvariantGlobal(Nearby, PClass):
+        distant = field()
+        nearby = field()
+
+    try:
+        MultipleInvariantGlobal(distant=False, nearby=False)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == (("nearby",), ("distant",),)
+
+
+def test_diamond_inherited_global_invariants():
+    counter = []
+    class Base(object):
+        def __invariant__(self):
+            counter.append(None)
+            return [(False, "base")]
+
+    class Left(Base):
+        pass
+
+    class Right(Base):
+        pass
+
+    class SingleInvariantGlobal(Left, Right, PClass):
+        pass
+
+    try:
+        SingleInvariantGlobal()
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == (("base",),)
+        assert counter == [None]
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(Point(x=1, y=2))
+
+
+def test_supports_weakref_with_multi_level_inheritance():
+    import weakref
+
+    class PPoint(Point):
+        a = field()
+
+    weakref.ref(PPoint(x=1, y=2))
+
+
+def test_supports_lazy_initial_value_for_field():
+    class MyClass(PClass):
+        a = field(int, initial=lambda: 2)
+
+    assert MyClass() == MyClass(a=2)
+
+
+def test_type_checks_lazy_initial_value_for_field():
+    class MyClass(PClass):
+        a = field(int, initial=lambda: "a")
+
+    with pytest.raises(TypeError):
+        MyClass()
+
+
+def test_invariant_checks_lazy_initial_value_for_field():
+    class MyClass(PClass):
+        a = field(int, invariant=lambda x: (x < 5, "Too large"), initial=lambda: 10)
+
+    with pytest.raises(InvariantException):
+        MyClass()
+
+
+def test_invariant_checks_static_initial_value():
+    class MyClass(PClass):
+        a = field(int, invariant=lambda x: (x < 5, "Too large"), initial=10)
+
+    with pytest.raises(InvariantException):
+        MyClass()
+
+
+def test_lazy_invariant_message():
+    class MyClass(PClass):
+        a = field(int, invariant=lambda x: (x < 5, lambda: "{x} is too large".format(x=x)))
+
+    try:
+        MyClass(a=5)
+        assert False
+    except InvariantException as e:
+        assert '5 is too large' in e.invariant_errors
+
+# Skipping this test for now but it describes a corner case with using Enums in
+# python 3 as types and a workaround to make it work.
+@pytest.mark.skipif(sys.version_info < (3, 4) or True, reason="requires python3.4")
+def test_enum_key_type():
+    import enum
+    class Foo(enum.Enum):
+        Bar = 1
+        Baz = 2
+
+    # This currently fails because the enum is iterable
+    class MyClass1(PClass):
+        f = pmap_field(key_type=Foo, value_type=int)
+
+    MyClass1()
+
+    # This is OK since it's wrapped in a tuple
+    class MyClass2(PClass):
+        f = pmap_field(key_type=(Foo,), value_type=int)
+
+    MyClass2()
+
+
+def test_pickle_with_one_way_factory():
+    thing = UniqueThing(id='25544626-86da-4bce-b6b6-9186c0804d64')
+    assert pickle.loads(pickle.dumps(thing)) == thing
+
+
+def test_evolver_with_one_way_factory():
+    thing = UniqueThing(id='cc65249a-56fe-4995-8719-ea02e124b234')
+    ev = thing.evolver()
+    ev.x = 5  # necessary to prevent persistent() returning the original
+    assert ev.persistent() == UniqueThing(id=str(thing.id), x=5)
+
+
+def test_set_doesnt_trigger_other_factories():
+    thing = UniqueThing(id='b413b280-de76-4e28-a8e3-5470ca83ea2c')
+    thing.set(x=5)
+
+
+def test_set_does_trigger_factories():
+    class SquaredPoint(PClass):
+        x = field(factory=lambda x: x ** 2)
+        y = field()
+
+    sp = SquaredPoint(x=3, y=10)
+    assert (sp.x, sp.y) == (9, 10)
+
+    sp2 = sp.set(x=4)
+    assert (sp2.x, sp2.y) == (16, 10)
+
+
+def test_value_can_be_overridden_in_subclass_new():
+    class X(PClass):
+        y = pvector_field(int)
+
+        def __new__(cls, **kwargs):
+            items = kwargs.get('y', None)
+            if items is None:
+                kwargs['y'] = ()
+            return super(X, cls).__new__(cls, **kwargs)
+
+    a = X(y=[])
+    b = a.set(y=None)
+    assert a == b
diff --git a/contrib/python/pyrsistent/py2/tests/deque_test.py b/contrib/python/pyrsistent/py2/tests/deque_test.py
new file mode 100644
index 00000000000..7798a755834
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/deque_test.py
@@ -0,0 +1,293 @@
+import pickle
+import pytest
+from pyrsistent import pdeque, dq
+
+
+def test_basic_right_and_left():
+    x = pdeque([1, 2])
+
+    assert x.right == 2
+    assert x.left == 1
+    assert len(x) == 2
+
+
+def test_construction_with_maxlen():
+    assert pdeque([1, 2, 3, 4], maxlen=2) == pdeque([3, 4])
+    assert pdeque([1, 2, 3, 4], maxlen=4) == pdeque([1, 2, 3, 4])
+    assert pdeque([], maxlen=2) == pdeque()
+
+
+def test_construction_with_invalid_maxlen():
+    with pytest.raises(TypeError):
+        pdeque([], maxlen='foo')
+
+    with pytest.raises(ValueError):
+        pdeque([], maxlen=-3)
+
+
+def test_pop():
+    x = pdeque([1, 2, 3, 4]).pop()
+    assert x.right == 3
+    assert x.left == 1
+
+    x = x.pop()
+    assert x.right == 2
+    assert x.left == 1
+
+    x = x.pop()
+    assert x.right == 1
+    assert x.left == 1
+
+    x = x.pop()
+    assert x == pdeque()
+
+    x = pdeque([1, 2]).pop()
+    assert x == pdeque([1])
+
+    x = x.pop()
+    assert x == pdeque()
+
+    assert pdeque().append(1).pop() == pdeque()
+    assert pdeque().appendleft(1).pop() == pdeque()
+
+
+def test_pop_multiple():
+    assert pdeque([1, 2, 3, 4]).pop(3) == pdeque([1])
+    assert pdeque([1, 2]).pop(3) == pdeque()
+
+
+def test_pop_with_negative_index():
+    assert pdeque([1, 2, 3]).pop(-1) == pdeque([1, 2, 3]).popleft(1)
+    assert pdeque([1, 2, 3]).popleft(-1) == pdeque([1, 2, 3]).pop(1)
+
+
+def test_popleft():
+    x = pdeque([1, 2, 3, 4]).popleft()
+    assert x.left == 2
+    assert x.right == 4
+
+    x = x.popleft()
+    assert x.left == 3
+    assert x.right == 4
+
+    x = x.popleft()
+    assert x.right == 4
+    assert x.left == 4
+
+    x = x.popleft()
+    assert x == pdeque()
+
+    x = pdeque([1, 2]).popleft()
+    assert x == pdeque([2])
+
+    x = x.popleft()
+    assert x == pdeque()
+
+    assert pdeque().append(1).popleft() == pdeque()
+    assert pdeque().appendleft(1).popleft() == pdeque()
+
+
+def test_popleft_multiple():
+    assert pdeque([1, 2, 3, 4]).popleft(3) == pdeque([4])
+
+
+def test_left_on_empty_deque():
+    with pytest.raises(IndexError):
+        pdeque().left
+
+
+def test_right_on_empty_deque():
+    with pytest.raises(IndexError):
+        pdeque().right
+
+
+def test_pop_empty_deque_returns_empty_deque():
+    # The other option is to throw an index error, this is what feels best for now though
+    assert pdeque().pop() == pdeque()
+    assert pdeque().popleft() == pdeque()
+
+
+def test_str():
+    assert str(pdeque([1, 2, 3])) == 'pdeque([1, 2, 3])'
+    assert str(pdeque([])) == 'pdeque([])'
+    assert str(pdeque([1, 2], maxlen=4)) == 'pdeque([1, 2], maxlen=4)'
+
+
+def test_append():
+    assert pdeque([1, 2]).append(3).append(4) == pdeque([1, 2, 3, 4])
+
+
+def test_append_with_maxlen():
+    assert pdeque([1, 2], maxlen=2).append(3).append(4) == pdeque([3, 4])
+    assert pdeque([1, 2], maxlen=3).append(3).append(4) == pdeque([2, 3, 4])
+    assert pdeque([], maxlen=0).append(1) == pdeque()
+
+
+def test_appendleft():
+    assert pdeque([2, 1]).appendleft(3).appendleft(4) == pdeque([4, 3, 2, 1])
+
+
+def test_appendleft_with_maxlen():
+    assert pdeque([2, 1], maxlen=2).appendleft(3).appendleft(4) == pdeque([4, 3])
+    assert pdeque([2, 1], maxlen=3).appendleft(3).appendleft(4) == pdeque([4, 3, 2])
+    assert pdeque([], maxlen=0).appendleft(1) == pdeque()
+
+
+def test_extend():
+    assert pdeque([1, 2]).extend([3, 4]) == pdeque([1, 2, 3, 4])
+
+
+def test_extend_with_maxlen():
+    assert pdeque([1, 2], maxlen=3).extend([3, 4]) == pdeque([2, 3, 4])
+    assert pdeque([1, 2], maxlen=2).extend([3, 4]) == pdeque([3, 4])
+    assert pdeque([], maxlen=2).extend([1, 2]) == pdeque([1, 2])
+    assert pdeque([], maxlen=0).extend([1, 2]) == pdeque([])
+
+
+def test_extendleft():
+    assert pdeque([2, 1]).extendleft([3, 4]) == pdeque([4, 3, 2, 1])
+
+
+def test_extendleft_with_maxlen():
+    assert pdeque([1, 2], maxlen=3).extendleft([3, 4]) == pdeque([4, 3, 1])
+    assert pdeque([1, 2], maxlen=2).extendleft([3, 4]) == pdeque([4, 3])
+    assert pdeque([], maxlen=2).extendleft([1, 2]) == pdeque([2, 1])
+    assert pdeque([], maxlen=0).extendleft([1, 2]) == pdeque([])
+
+
+def test_count():
+    x = pdeque([1, 2, 3, 2, 1])
+    assert x.count(1) == 2
+    assert x.count(2) == 2
+
+
+def test_remove():
+    assert pdeque([1, 2, 3, 4]).remove(2) == pdeque([1, 3, 4])
+    assert pdeque([1, 2, 3, 4]).remove(4) == pdeque([1, 2, 3])
+
+    # Right list must be reversed before removing element
+    assert pdeque([1, 2, 3, 3, 4, 5, 4, 6]).remove(4) == pdeque([1, 2, 3, 3, 5, 4, 6])
+
+
+def test_remove_element_missing():
+    with pytest.raises(ValueError):
+        pdeque().remove(2)
+
+    with pytest.raises(ValueError):
+        pdeque([1, 2, 3]).remove(4)
+
+
+def test_reverse():
+    assert pdeque([1, 2, 3, 4]).reverse() == pdeque([4, 3, 2, 1])
+    assert pdeque().reverse() == pdeque()
+
+
+def test_rotate_right():
+    assert pdeque([1, 2, 3, 4, 5]).rotate(2) == pdeque([4, 5, 1, 2, 3])
+    assert pdeque([1, 2]).rotate(0) == pdeque([1, 2])
+    assert pdeque().rotate(2) == pdeque()
+
+
+def test_rotate_left():
+    assert pdeque([1, 2, 3, 4, 5]).rotate(-2) == pdeque([3, 4, 5, 1, 2])
+    assert pdeque().rotate(-2) == pdeque()
+
+
+def test_set_maxlen():
+    x = pdeque([], maxlen=4)
+    assert x.maxlen == 4
+
+    with pytest.raises(AttributeError):
+        x.maxlen = 5
+
+
+def test_comparison():
+    small = pdeque([1, 2])
+    large = pdeque([1, 2, 3])
+
+    assert small < large
+    assert large > small
+    assert not small > large
+    assert not large < small
+    assert large != small
+
+    # Not equal to other types
+    assert small != [1, 2]
+
+
+def test_pickling():
+    input = pdeque([1, 2, 3], maxlen=5)
+    output = pickle.loads(pickle.dumps(input, -1))
+
+    assert output == input
+    assert output.maxlen == input.maxlen
+
+
+def test_indexing():
+    assert pdeque([1, 2, 3])[0] == 1
+    assert pdeque([1, 2, 3])[1] == 2
+    assert pdeque([1, 2, 3])[2] == 3
+    assert pdeque([1, 2, 3])[-1] == 3
+    assert pdeque([1, 2, 3])[-2] == 2
+    assert pdeque([1, 2, 3])[-3] == 1
+
+
+def test_one_element_indexing():
+    assert pdeque([2])[0] == 2
+    assert pdeque([2])[-1] == 2
+
+
+def test_empty_indexing():
+    with pytest.raises(IndexError):
+        assert pdeque([])[0] == 1
+
+
+def test_indexing_out_of_range():
+    with pytest.raises(IndexError):
+        pdeque([1, 2, 3])[-4]
+
+    with pytest.raises(IndexError):
+        pdeque([1, 2, 3])[3]
+
+    with pytest.raises(IndexError):
+        pdeque([2])[-2]
+
+
+def test_indexing_invalid_type():
+    with pytest.raises(TypeError) as e:
+        pdeque([1, 2, 3])['foo']
+
+    assert 'cannot be interpreted' in str(e.value)
+
+
+def test_slicing():
+    assert pdeque([1, 2, 3])[1:2] == pdeque([2])
+    assert pdeque([1, 2, 3])[2:1] == pdeque([])
+    assert pdeque([1, 2, 3])[-2:-1] == pdeque([2])
+    assert pdeque([1, 2, 3])[::2] == pdeque([1, 3])
+
+
+def test_hashing():
+    assert hash(pdeque([1, 2, 3])) == hash(pdeque().append(1).append(2).append(3))
+
+
+def test_index():
+    assert pdeque([1, 2, 3]).index(3) == 2
+
+
+def test_literalish():
+    assert dq(1, 2, 3) == pdeque([1, 2, 3])
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(dq(1, 2))
+
+
+def test_iterable():
+    """
+    PDeques can be created from iterables even though they can't be len()
+    hinted.
+    """
+
+    assert pdeque(iter("a")) == pdeque(iter("a"))
diff --git a/contrib/python/pyrsistent/py2/tests/field_test.py b/contrib/python/pyrsistent/py2/tests/field_test.py
new file mode 100644
index 00000000000..cf963cf00a1
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/field_test.py
@@ -0,0 +1,27 @@
+from pyrsistent._compat import Enum
+
+from pyrsistent import field, pvector_field
+
+
+# NB: This derives from the internal `pyrsistent._compat.Enum` in order to
+# simplify coverage across python versions. Since we use
+# `pyrsistent._compat.Enum` in `pyrsistent`'s implementation, it's useful to
+# use it in the test coverage as well, for consistency.
+class TestEnum(Enum):
+    x = 1
+    y = 2
+
+
+def test_enum():
+    f = field(type=TestEnum)
+
+    assert TestEnum in f.type
+    assert len(f.type) == 1
+
+
+# This is meant to exercise `_seq_field`.
+def test_pvector_field_enum_type():
+    f = pvector_field(TestEnum)
+
+    assert len(f.type) == 1
+    assert TestEnum is list(f.type)[0].__type__
diff --git a/contrib/python/pyrsistent/py2/tests/freeze_test.py b/contrib/python/pyrsistent/py2/tests/freeze_test.py
new file mode 100644
index 00000000000..eeb48896cdf
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/freeze_test.py
@@ -0,0 +1,101 @@
+"""Tests for freeze and thaw."""
+
+from pyrsistent import v, m, s, freeze, thaw, PRecord, field, mutant
+
+
+## Freeze
+
+def test_freeze_basic():
+    assert freeze(1) == 1
+    assert freeze('foo') == 'foo'
+
+def test_freeze_list():
+    assert freeze([1, 2]) == v(1, 2)
+
+def test_freeze_dict():
+    result = freeze({'a': 'b'})
+    assert result == m(a='b')
+    assert type(freeze({'a': 'b'})) is type(m())
+
+def test_freeze_set():
+    result = freeze(set([1, 2, 3]))
+    assert result == s(1, 2, 3)
+    assert type(result) is type(s())
+
+def test_freeze_recurse_in_dictionary_values():
+    result = freeze({'a': [1]})
+    assert result == m(a=v(1))
+    assert type(result['a']) is type(v())
+
+def test_freeze_recurse_in_lists():
+    result = freeze(['a', {'b': 3}])
+    assert result == v('a', m(b=3))
+    assert type(result[1]) is type(m())
+
+def test_freeze_recurse_in_tuples():
+    """Values in tuples are recursively frozen."""
+    result = freeze(('a', {}))
+    assert result == ('a', m())
+    assert type(result[1]) is type(m())
+
+
+
+## Thaw
+
+def test_thaw_basic():
+    assert thaw(1) == 1
+    assert thaw('foo') == 'foo'
+
+def test_thaw_list():
+    result = thaw(v(1, 2))
+    assert result == [1, 2]
+    assert type(result) is list
+
+def test_thaw_dict():
+    result = thaw(m(a='b'))
+    assert result == {'a': 'b'}
+    assert type(result) is dict
+
+def test_thaw_set():
+    result = thaw(s(1, 2))
+    assert result == set([1, 2])
+    assert type(result) is set
+
+def test_thaw_recurse_in_mapping_values():
+    result = thaw(m(a=v(1)))
+    assert result == {'a': [1]}
+    assert type(result['a']) is list
+
+def test_thaw_recurse_in_vectors():
+    result = thaw(v('a', m(b=3)))
+    assert result == ['a', {'b': 3}]
+    assert type(result[1]) is dict
+
+def test_thaw_recurse_in_tuples():
+    result = thaw(('a', m()))
+    assert result == ('a', {})
+    assert type(result[1]) is dict
+
+def test_thaw_can_handle_subclasses_of_persistent_base_types():
+    class R(PRecord):
+        x = field()
+
+    result = thaw(R(x=1))
+    assert result == {'x': 1}
+    assert type(result) is dict
+
+
+def test_mutant_decorator():
+    @mutant
+    def fn(a_list, a_dict):
+        assert a_list == v(1, 2, 3)
+        assert isinstance(a_dict, type(m()))
+        assert a_dict == {'a': 5}
+
+        return [1, 2, 3], {'a': 3}
+
+    pv, pm = fn([1, 2, 3], a_dict={'a': 5})
+
+    assert pv == v(1, 2, 3)
+    assert pm == m(a=3)
+    assert isinstance(pm, type(m()))
diff --git a/contrib/python/pyrsistent/py2/tests/hypothesis_vector_test.py b/contrib/python/pyrsistent/py2/tests/hypothesis_vector_test.py
new file mode 100644
index 00000000000..e634204ec2d
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/hypothesis_vector_test.py
@@ -0,0 +1,304 @@
+"""
+Hypothesis-based tests for pvector.
+"""
+
+import gc
+
+from pyrsistent._compat import Iterable
+from functools import wraps
+from pyrsistent import PClass, field
+
+from pytest import fixture
+
+from pyrsistent import pvector, discard
+
+from hypothesis import strategies as st, assume
+from hypothesis.stateful import RuleBasedStateMachine, Bundle, rule
+
+
+class TestObject(object):
+    """
+    An object that might catch reference count errors sometimes.
+    """
+    def __init__(self):
+        self.id = id(self)
+
+    def __repr__(self):
+        return "<%s>" % (self.id,)
+
+    def __del__(self):
+        # If self is a dangling memory reference this check might fail. Or
+        # segfault :)
+        if self.id != id(self):
+            raise RuntimeError()
+
+
+@fixture(scope="module")
+def gc_when_done(request):
+    request.addfinalizer(gc.collect)
+
+
+def test_setup(gc_when_done):
+    """
+    Ensure we GC when tests finish.
+    """
+
+
+# Pairs of a list and corresponding pvector:
+PVectorAndLists = st.lists(st.builds(TestObject)).map(
+    lambda l: (l, pvector(l)))
+
+
+def verify_inputs_unmodified(original):
+    """
+    Decorator that asserts that the wrapped function does not modify its
+    inputs.
+    """
+    def to_tuples(pairs):
+        return [(tuple(l), tuple(pv)) for (l, pv) in pairs]
+
+    @wraps(original)
+    def wrapper(self, **kwargs):
+        inputs = [k for k in kwargs.values() if isinstance(k, Iterable)]
+        tuple_inputs = to_tuples(inputs)
+        try:
+            return original(self, **kwargs)
+        finally:
+            # Ensure inputs were unmodified:
+            assert to_tuples(inputs) == tuple_inputs
+    return wrapper
+
+
+def assert_equal(l, pv):
+    assert l == pv
+    assert len(l) == len(pv)
+    length = len(l)
+    for i in range(length):
+        assert l[i] == pv[i]
+    for i in range(length):
+        for j in range(i, length):
+            assert l[i:j] == pv[i:j]
+    assert l == list(iter(pv))
+
+
+class PVectorBuilder(RuleBasedStateMachine):
+    """
+    Build a list and matching pvector step-by-step.
+
+    In each step in the state machine we do same operation on a list and
+    on a pvector, and then when we're done we compare the two.
+    """
+    sequences = Bundle("sequences")
+
+    @rule(target=sequences, start=PVectorAndLists)
+    def initial_value(self, start):
+        """
+        Some initial values generated by a hypothesis strategy.
+        """
+        return start
+
+    @rule(target=sequences, former=sequences)
+    @verify_inputs_unmodified
+    def append(self, former):
+        """
+        Append an item to the pair of sequences.
+        """
+        l, pv = former
+        obj = TestObject()
+        l2 = l[:]
+        l2.append(obj)
+        return l2, pv.append(obj)
+
+    @rule(target=sequences, start=sequences, end=sequences)
+    @verify_inputs_unmodified
+    def extend(self, start, end):
+        """
+        Extend a pair of sequences with another pair of sequences.
+        """
+        l, pv = start
+        l2, pv2 = end
+        # compare() has O(N**2) behavior, so don't want too-large lists:
+        assume(len(l) + len(l2) < 50)
+        l3 = l[:]
+        l3.extend(l2)
+        return l3, pv.extend(pv2)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def remove(self, former, data):
+        """
+        Remove an item from the sequences.
+        """
+        l, pv = former
+        assume(l)
+        l2 = l[:]
+        i = data.draw(st.sampled_from(range(len(l))))
+        del l2[i]
+        return l2, pv.delete(i)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def set(self, former, data):
+        """
+        Overwrite an item in the sequence.
+        """
+        l, pv = former
+        assume(l)
+        l2 = l[:]
+        i = data.draw(st.sampled_from(range(len(l))))
+        obj = TestObject()
+        l2[i] = obj
+        return l2, pv.set(i, obj)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def transform_set(self, former, data):
+        """
+        Transform the sequence by setting value.
+        """
+        l, pv = former
+        assume(l)
+        l2 = l[:]
+        i = data.draw(st.sampled_from(range(len(l))))
+        obj = TestObject()
+        l2[i] = obj
+        return l2, pv.transform([i], obj)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def transform_discard(self, former, data):
+        """
+        Transform the sequence by discarding a value.
+        """
+        l, pv = former
+        assume(l)
+        l2 = l[:]
+        i = data.draw(st.sampled_from(range(len(l))))
+        del l2[i]
+        return l2, pv.transform([i], discard)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def subset(self, former, data):
+        """
+        A subset of the previous sequence.
+        """
+        l, pv = former
+        assume(l)
+        i = data.draw(st.sampled_from(range(len(l))))
+        j = data.draw(st.sampled_from(range(len(l))))
+        return l[i:j], pv[i:j]
+
+    @rule(pair=sequences)
+    @verify_inputs_unmodified
+    def compare(self, pair):
+        """
+        The list and pvector must match.
+        """
+        l, pv = pair
+        # compare() has O(N**2) behavior, so don't want too-large lists:
+        assume(len(l) < 50)
+        assert_equal(l, pv)
+
+
+PVectorBuilderTests = PVectorBuilder.TestCase
+
+
+class EvolverItem(PClass):
+    original_list = field()
+    original_pvector = field()
+    current_list = field()
+    current_evolver = field()
+
+
+class PVectorEvolverBuilder(RuleBasedStateMachine):
+    """
+    Build a list and matching pvector evolver step-by-step.
+
+    In each step in the state machine we do same operation on a list and
+    on a pvector evolver, and then when we're done we compare the two.
+    """
+    sequences = Bundle("evolver_sequences")
+
+    @rule(target=sequences, start=PVectorAndLists)
+    def initial_value(self, start):
+        """
+        Some initial values generated by a hypothesis strategy.
+        """
+        l, pv = start
+        return EvolverItem(original_list=l,
+                           original_pvector=pv,
+                           current_list=l[:],
+                           current_evolver=pv.evolver())
+
+    @rule(item=sequences)
+    def append(self, item):
+        """
+        Append an item to the pair of sequences.
+        """
+        obj = TestObject()
+        item.current_list.append(obj)
+        item.current_evolver.append(obj)
+
+    @rule(start=sequences, end=sequences)
+    def extend(self, start, end):
+        """
+        Extend a pair of sequences with another pair of sequences.
+        """
+        # compare() has O(N**2) behavior, so don't want too-large lists:
+        assume(len(start.current_list) + len(end.current_list) < 50)
+        start.current_evolver.extend(end.current_list)
+        start.current_list.extend(end.current_list)
+
+    @rule(item=sequences, data=st.data())
+    def delete(self, item, data):
+        """
+        Remove an item from the sequences.
+        """
+        assume(item.current_list)
+        i = data.draw(st.sampled_from(range(len(item.current_list))))
+        del item.current_list[i]
+        del item.current_evolver[i]
+
+    @rule(item=sequences, data=st.data())
+    def setitem(self, item, data):
+        """
+        Overwrite an item in the sequence using ``__setitem__``.
+        """
+        assume(item.current_list)
+        i = data.draw(st.sampled_from(range(len(item.current_list))))
+        obj = TestObject()
+        item.current_list[i] = obj
+        item.current_evolver[i] = obj
+
+    @rule(item=sequences, data=st.data())
+    def set(self, item, data):
+        """
+        Overwrite an item in the sequence using ``set``.
+        """
+        assume(item.current_list)
+        i = data.draw(st.sampled_from(range(len(item.current_list))))
+        obj = TestObject()
+        item.current_list[i] = obj
+        item.current_evolver.set(i, obj)
+
+    @rule(item=sequences)
+    def compare(self, item):
+        """
+        The list and pvector evolver must match.
+        """
+        item.current_evolver.is_dirty()
+        # compare() has O(N**2) behavior, so don't want too-large lists:
+        assume(len(item.current_list) < 50)
+        # original object unmodified
+        assert item.original_list == item.original_pvector
+        # evolver matches:
+        for i in range(len(item.current_evolver)):
+            assert item.current_list[i] == item.current_evolver[i]
+        # persistent version matches
+        assert_equal(item.current_list, item.current_evolver.persistent())
+        # original object still unmodified
+        assert item.original_list == item.original_pvector
+
+
+PVectorEvolverBuilderTests = PVectorEvolverBuilder.TestCase
diff --git a/contrib/python/pyrsistent/py2/tests/immutable_object_test.py b/contrib/python/pyrsistent/py2/tests/immutable_object_test.py
new file mode 100644
index 00000000000..11ff513cbcb
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/immutable_object_test.py
@@ -0,0 +1,67 @@
+import pytest
+from pyrsistent import immutable
+
+class Empty(immutable(verbose=True)):
+    pass
+
+
+class Single(immutable('x')):
+    pass
+
+
+class FrozenMember(immutable('x, y_')):
+    pass
+
+
+class DerivedWithNew(immutable(['x', 'y'])):
+    def __new__(cls, x, y):
+        return super(DerivedWithNew, cls).__new__(cls, x, y)
+
+
+def test_instantiate_object_with_no_members():
+    t = Empty()
+    t2 = t.set()
+
+    assert t is t2
+
+
+def test_assign_non_existing_attribute():
+    t = Empty()
+
+    with pytest.raises(AttributeError):
+        t.set(a=1)
+
+
+def test_basic_instantiation():
+    t = Single(17)
+
+    assert t.x == 17
+    assert str(t) == 'Single(x=17)'
+
+
+def test_cannot_modify_member():
+    t = Single(17)
+
+    with pytest.raises(AttributeError):
+        t.x = 18
+
+def test_basic_replace():
+    t = Single(17)
+    t2 = t.set(x=18)
+
+    assert t.x == 17
+    assert t2.x == 18
+
+
+def test_cannot_replace_frozen_member():
+    t = FrozenMember(17, 18)
+
+    with pytest.raises(AttributeError):
+        t.set(y_=18)
+
+
+def test_derived_class_with_new():
+    d = DerivedWithNew(1, 2)
+    d2 = d.set(x=3)
+
+    assert d2.x == 3
diff --git a/contrib/python/pyrsistent/py2/tests/list_test.py b/contrib/python/pyrsistent/py2/tests/list_test.py
new file mode 100644
index 00000000000..ccbd83ba978
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/list_test.py
@@ -0,0 +1,209 @@
+import pickle
+import pytest
+from pyrsistent import plist, l
+
+
+def test_literalish_works():
+    assert l(1, 2, 3) == plist([1, 2, 3])
+
+
+def test_first_and_rest():
+    pl = plist([1, 2])
+    assert pl.first == 1
+    assert pl.rest.first == 2
+    assert pl.rest.rest is plist()
+
+
+def test_instantiate_large_list():
+    assert plist(range(1000)).first == 0
+
+
+def test_iteration():
+    assert list(plist()) == []
+    assert list(plist([1, 2, 3])) == [1, 2, 3]
+
+
+def test_cons():
+    assert plist([1, 2, 3]).cons(0) == plist([0, 1, 2, 3])
+
+
+def test_cons_empty_list():
+    assert plist().cons(0) == plist([0])
+
+
+def test_truthiness():
+    assert plist([1])
+    assert not plist()
+
+
+def test_len():
+    assert len(plist([1, 2, 3])) == 3
+    assert len(plist()) == 0
+
+
+def test_first_illegal_on_empty_list():
+    with pytest.raises(AttributeError):
+        plist().first
+
+
+def test_rest_return_self_on_empty_list():
+    assert plist().rest is plist()
+
+
+def test_reverse():
+    assert plist([1, 2, 3]).reverse() == plist([3, 2, 1])
+    assert reversed(plist([1, 2, 3])) == plist([3, 2, 1])
+
+    assert plist().reverse() == plist()
+    assert reversed(plist()) == plist()
+
+
+def test_inequality():
+    assert plist([1, 2]) != plist([1, 3])
+    assert plist([1, 2]) != plist([1, 2, 3])
+    assert plist() != plist([1, 2, 3])
+
+
+def test_repr():
+    assert str(plist()) == "plist([])"
+    assert str(plist([1, 2, 3])) == "plist([1, 2, 3])"
+
+
+def test_indexing():
+    assert plist([1, 2, 3])[2] == 3
+    assert plist([1, 2, 3])[-1] == 3
+
+
+def test_indexing_on_empty_list():
+    with pytest.raises(IndexError):
+        plist()[0]
+
+
+def test_index_out_of_range():
+    with pytest.raises(IndexError):
+        plist([1, 2])[2]
+
+    with pytest.raises(IndexError):
+        plist([1, 2])[-3]
+
+def test_index_invalid_type():
+    with pytest.raises(TypeError) as e:
+        plist([1, 2, 3])['foo']  # type: ignore
+
+    assert 'cannot be interpreted' in str(e.value)
+
+
+def test_slicing_take():
+    assert plist([1, 2, 3])[:2] == plist([1, 2])
+
+
+def test_slicing_take_out_of_range():
+    assert plist([1, 2, 3])[:20] == plist([1, 2, 3])
+
+
+def test_slicing_drop():
+    li = plist([1, 2, 3])
+    assert li[1:] is li.rest
+
+
+def test_slicing_drop_out_of_range():
+    assert plist([1, 2, 3])[3:] is plist()
+
+
+def test_contains():
+    assert 2 in plist([1, 2, 3])
+    assert 4 not in plist([1, 2, 3])
+    assert 1 not in plist()
+
+
+def test_count():
+    assert plist([1, 2, 1]).count(1) == 2
+    assert plist().count(1) == 0
+
+
+def test_index():
+    assert plist([1, 2, 3]).index(3) == 2
+
+
+def test_index_item_not_found():
+    with pytest.raises(ValueError):
+        plist().index(3)
+
+    with pytest.raises(ValueError):
+        plist([1, 2]).index(3)
+
+
+def test_pickling_empty_list():
+    assert pickle.loads(pickle.dumps(plist(), -1)) == plist()
+
+
+def test_pickling_non_empty_list():
+    assert pickle.loads(pickle.dumps(plist([1, 2, 3]), -1)) == plist([1, 2, 3])
+
+
+def test_comparison():
+    assert plist([1, 2]) < plist([1, 2, 3])
+    assert plist([2, 1]) > plist([1, 2, 3])
+    assert plist() < plist([1])
+    assert plist([1]) > plist()
+
+
+def test_comparison_with_other_type():
+    assert plist() != []
+
+
+def test_hashing():
+    assert hash(plist([1, 2])) == hash(plist([1, 2]))
+    assert hash(plist([1, 2])) != hash(plist([2, 1]))
+
+
+def test_split():
+    left_list, right_list = plist([1, 2, 3, 4, 5]).split(3)
+    assert left_list == plist([1, 2, 3])
+    assert right_list == plist([4, 5])
+
+
+def test_split_no_split_occurred():
+    x = plist([1, 2])
+    left_list, right_list = x.split(2)
+    assert left_list is x
+    assert right_list is plist()
+
+
+def test_split_empty_list():
+    left_list, right_list = plist().split(2)
+    assert left_list == plist()
+    assert right_list == plist()
+
+
+def test_remove():
+    assert plist([1, 2, 3, 2]).remove(2) == plist([1, 3, 2])
+    assert plist([1, 2, 3]).remove(1) == plist([2, 3])
+    assert plist([1, 2, 3]).remove(3) == plist([1, 2])
+
+
+def test_remove_missing_element():
+    with pytest.raises(ValueError):
+        plist([1, 2]).remove(3)
+
+    with pytest.raises(ValueError):
+        plist().remove(2)
+
+
+def test_mcons():
+    assert plist([1, 2]).mcons([3, 4]) == plist([4, 3, 1, 2])
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(plist())
+    weakref.ref(plist([1, 2]))
+
+
+def test_iterable():
+    """
+    PLists can be created from iterables even though they can't be len()
+    hinted.
+    """
+
+    assert plist(iter("a")) == plist(iter("a"))
diff --git a/contrib/python/pyrsistent/py2/tests/map_test.py b/contrib/python/pyrsistent/py2/tests/map_test.py
new file mode 100644
index 00000000000..4b79ad2d974
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/map_test.py
@@ -0,0 +1,497 @@
+from pyrsistent._compat import Mapping, Hashable
+import six
+from operator import add
+import pytest
+from pyrsistent import pmap, m, PVector
+import pickle
+
+def test_instance_of_hashable():
+    assert isinstance(m(), Hashable)
+
+
+def test_instance_of_map():
+    assert isinstance(m(), Mapping)
+
+
+def test_literalish_works():
+    assert m() is pmap()
+    assert m(a=1, b=2) == pmap({'a': 1, 'b': 2})
+
+
+def test_empty_initialization():
+    map = pmap()
+    assert len(map) == 0
+
+
+def test_initialization_with_one_element():
+    the_map = pmap({'a': 2})
+    assert len(the_map) == 1
+    assert the_map['a'] == 2
+    assert the_map.a == 2
+    assert 'a' in the_map
+    
+    assert the_map is the_map.discard('b')
+    
+    empty_map = the_map.remove('a')
+    assert len(empty_map) == 0
+    assert 'a' not in empty_map
+
+
+def test_get_non_existing_raises_key_error():
+    m1 = m()
+    with pytest.raises(KeyError) as error:
+        m1['foo']
+
+    assert str(error.value) == "'foo'"
+
+
+def test_remove_non_existing_element_raises_key_error():
+    m1 = m(a=1)
+
+    with pytest.raises(KeyError) as error:
+        m1.remove('b')
+
+    assert str(error.value) == "'b'"
+
+
+def test_various_iterations():
+    assert set(['a', 'b']) == set(m(a=1, b=2))
+    assert ['a', 'b'] == sorted(m(a=1, b=2).keys())
+    assert isinstance(m().keys(), PVector)
+
+    assert set([1, 2]) == set(m(a=1, b=2).itervalues())
+    assert [1, 2] == sorted(m(a=1, b=2).values())
+    assert isinstance(m().values(), PVector)
+
+    assert set([('a', 1), ('b', 2)]) == set(m(a=1, b=2).iteritems())
+    assert set([('a', 1), ('b', 2)]) == set(m(a=1, b=2).items())
+    assert isinstance(m().items(), PVector)
+
+
+def test_initialization_with_two_elements():
+    map = pmap({'a': 2, 'b': 3})
+    assert len(map) == 2
+    assert map['a'] == 2
+    assert map['b'] == 3
+
+    map2 = map.remove('a')
+    assert 'a' not in map2
+    assert map2['b'] == 3
+
+
+def test_initialization_with_many_elements():
+    init_dict = dict([(str(x), x) for x in range(1700)])
+    the_map = pmap(init_dict)
+
+    assert len(the_map) == 1700
+    assert the_map['16'] == 16
+    assert the_map['1699'] == 1699
+    assert the_map.set('256', 256) is the_map
+ 
+    new_map = the_map.remove('1600')
+    assert len(new_map) == 1699
+    assert '1600' not in new_map
+    assert new_map['1601'] == 1601
+    
+    # Some NOP properties
+    assert new_map.discard('18888') is new_map
+    assert '19999' not in new_map
+    assert new_map['1500'] == 1500
+    assert new_map.set('1500', new_map['1500']) is new_map
+
+
+def test_access_non_existing_element():
+    map1 = pmap()
+    assert len(map1) == 0
+    
+    map2 = map1.set('1', 1)
+    assert '1' not in map1
+    assert map2['1'] == 1
+    assert '2' not in map2
+    
+
+def test_overwrite_existing_element():
+    map1 = pmap({'a': 2})
+    map2 = map1.set('a', 3)
+
+    assert len(map2) == 1
+    assert map2['a'] == 3
+
+
+def test_hash():
+    x = m(a=1, b=2, c=3)
+    y = m(a=1, b=2, c=3)
+
+    assert hash(x) == hash(y)
+
+
+def test_same_hash_when_content_the_same_but_underlying_vector_size_differs():
+    x = pmap(dict((x, x) for x in range(1000)))
+    y = pmap({10: 10, 200: 200, 700: 700})
+
+    for z in x:
+        if z not in y:
+            x = x.remove(z)
+
+    assert x == y
+    assert hash(x) == hash(y)
+
+
+class HashabilityControlled(object):
+
+    hashable = True
+
+    def __hash__(self):
+        if self.hashable:
+            return 4 # Proven random
+        raise ValueError("I am not currently hashable.")
+
+
+def test_map_does_not_hash_values_on_second_hash_invocation():
+    hashable = HashabilityControlled()
+    x = pmap(dict(el=hashable))
+    hash(x)
+    hashable.hashable = False
+    hash(x)
+
+
+def test_equal():
+    x = m(a=1, b=2, c=3)
+    y = m(a=1, b=2, c=3)
+
+    assert x == y
+    assert not (x != y)
+
+    assert y == x
+    assert not (y != x)
+
+
+def test_equal_to_dict():
+    x = m(a=1, b=2, c=3)
+    y = dict(a=1, b=2, c=3)
+
+    assert x == y
+    assert not (x != y)
+
+    assert y == x
+    assert not (y != x)
+
+
+def test_equal_with_different_bucket_sizes():
+    x = pmap({'a': 1, 'b': 2}, 50)
+    y = pmap({'a': 1, 'b': 2}, 10)
+
+    assert x == y
+    assert not (x != y)
+
+    assert y == x
+    assert not (y != x)
+
+
+def test_equal_with_different_insertion_order():
+    x = pmap([(i, i) for i in range(50)], 10)
+    y = pmap([(i, i) for i in range(49, -1, -1)], 10)
+
+    assert x == y
+    assert not (x != y)
+
+    assert y == x
+    assert not (y != x)
+
+
+def test_not_equal():
+    x = m(a=1, b=2, c=3)
+    y = m(a=1, b=2)
+
+    assert x != y
+    assert not (x == y)
+
+    assert y != x
+    assert not (y == x)
+
+
+def test_not_equal_to_dict():
+    x = m(a=1, b=2, c=3)
+    y = dict(a=1, b=2, d=4)
+
+    assert x != y
+    assert not (x == y)
+
+    assert y != x
+    assert not (y == x)
+
+
+def test_update_with_multiple_arguments():
+    # If same value is present in multiple sources, the rightmost is used.
+    x = m(a=1, b=2, c=3)    
+    y = x.update(m(b=4, c=5), {'c': 6})
+
+    assert y == m(a=1, b=4, c=6)
+
+
+def test_update_one_argument():
+    x = m(a=1)
+
+    assert x.update(m(b=2)) == m(a=1, b=2)
+
+
+def test_update_no_arguments():
+    x = m(a=1)
+
+    assert x.update() is x
+
+
+def test_addition():
+    assert m(x=1, y=2) + m(y=3, z=4) == m(x=1, y=3, z=4)
+
+
+def test_transform_base_case():
+    # Works as set when called with only one key
+    x = m(a=1, b=2)
+    
+    assert x.transform(['a'], 3) == m(a=3, b=2)
+
+
+def test_transform_nested_maps():
+    x = m(a=1, b=m(c=3, d=m(e=6, f=7)))
+    
+    assert x.transform(['b', 'd', 'e'], 999) == m(a=1, b=m(c=3, d=m(e=999, f=7)))
+
+
+def test_transform_levels_missing():
+    x = m(a=1, b=m(c=3))
+    
+    assert x.transform(['b', 'd', 'e'], 999) == m(a=1, b=m(c=3, d=m(e=999)))
+
+
+class HashDummy(object):
+    def __hash__(self):
+        return 6528039219058920  # Hash of '33'
+
+    def __eq__(self, other):
+        return self is other
+
+
+def test_hash_collision_is_correctly_resolved():
+
+    dummy1 = HashDummy()
+    dummy2 = HashDummy()
+    dummy3 = HashDummy()
+    dummy4 = HashDummy()
+
+    map = pmap({dummy1: 1, dummy2: 2, dummy3: 3})
+    assert map[dummy1] == 1
+    assert map[dummy2] == 2
+    assert map[dummy3] == 3
+    assert dummy4 not in map
+    
+    keys = set()
+    values = set()
+    for k, v in map.iteritems():
+        keys.add(k)
+        values.add(v)
+
+    assert keys == set([dummy1, dummy2, dummy3])
+    assert values == set([1, 2, 3])
+
+    map2 = map.set(dummy1, 11)
+    assert map2[dummy1] == 11
+    
+    # Re-use existing structure when inserted element is the same
+    assert map2.set(dummy1, 11) is map2
+    
+    map3 = map.set('a', 22)
+    assert map3['a'] == 22
+    assert map3[dummy3] == 3
+    
+    # Remove elements
+    map4 = map.discard(dummy2)
+    assert len(map4) == 2
+    assert map4[dummy1] == 1
+    assert dummy2 not in map4
+    assert map4[dummy3] == 3
+    
+    assert map.discard(dummy4) is map
+    
+    # Empty map handling
+    empty_map = map4.remove(dummy1).remove(dummy3)
+    assert len(empty_map) == 0
+    assert empty_map.discard(dummy1) is empty_map
+    
+
+def test_bitmap_indexed_iteration():
+    map = pmap({'a': 2, 'b': 1})
+    keys = set()
+    values = set()
+    
+    count = 0
+    for k, v in map.iteritems():
+        count += 1
+        keys.add(k)
+        values.add(v)
+    
+    assert count == 2
+    assert keys == set(['a', 'b'])
+    assert values == set([2, 1])
+
+
+def test_iteration_with_many_elements():
+    values = list(range(0, 2000))
+    keys = [str(x) for x in values]
+    init_dict = dict(zip(keys, values))
+    
+    hash_dummy1 = HashDummy()
+    hash_dummy2 = HashDummy()
+    
+    # Throw in a couple of hash collision nodes to tests
+    # those properly as well
+    init_dict[hash_dummy1] = 12345
+    init_dict[hash_dummy2] = 54321
+    map = pmap(init_dict)
+
+    actual_values = set()
+    actual_keys = set()
+    
+    for k, v in map.iteritems():
+        actual_values.add(v)
+        actual_keys.add(k)
+        
+    assert actual_keys == set(keys + [hash_dummy1, hash_dummy2])
+    assert actual_values == set(values + [12345, 54321])
+
+
+def test_str():
+    assert str(pmap({1: 2, 3: 4})) == "pmap({1: 2, 3: 4})"
+
+
+def test_empty_truthiness():
+    assert m(a=1)
+    assert not m()
+
+def test_update_with():
+    assert m(a=1).update_with(add, m(a=2, b=4)) == m(a=3, b=4)
+    assert m(a=1).update_with(lambda l, r: l, m(a=2, b=4)) == m(a=1, b=4)
+
+    def map_add(l, r):
+        return dict(list(l.items()) + list(r.items()))
+
+    assert m(a={'c': 3}).update_with(map_add, m(a={'d': 4})) == m(a={'c': 3, 'd': 4})
+
+
+def test_pickling_empty_map():
+    assert pickle.loads(pickle.dumps(m(), -1)) == m()
+
+
+def test_pickling_non_empty_map():
+    assert pickle.loads(pickle.dumps(m(a=1, b=2), -1)) == m(a=1, b=2)
+
+
+def test_set_with_relocation():
+    x = pmap({'a':1000}, pre_size=1)
+    x = x.set('b', 3000)
+    x = x.set('c', 4000)
+    x = x.set('d', 5000)
+    x = x.set('d', 6000)
+
+    assert len(x) == 4
+    assert x == pmap({'a': 1000, 'b': 3000, 'c': 4000, 'd': 6000})
+
+
+def test_evolver_simple_update():
+    x = m(a=1000, b=2000)
+    e = x.evolver()
+    e['b'] = 3000
+
+    assert e['b'] == 3000
+    assert e.persistent()['b'] == 3000
+    assert x['b'] == 2000
+
+
+def test_evolver_update_with_relocation():
+    x = pmap({'a':1000}, pre_size=1)
+    e = x.evolver()
+    e['b'] = 3000
+    e['c'] = 4000
+    e['d'] = 5000
+    e['d'] = 6000
+
+    assert len(e) == 4
+    assert e.persistent() == pmap({'a': 1000, 'b': 3000, 'c': 4000, 'd': 6000})
+
+
+def test_evolver_set_with_reallocation_edge_case():
+    # Demonstrates a bug in evolver that also affects updates. Under certain
+    # circumstances, the result of `x.update(y)` will **not** have all the
+    # keys from `y`.
+    foo = object()
+    x = pmap({'a': foo}, pre_size=1)
+    e = x.evolver()
+    e['b'] = 3000
+    # Bug is triggered when we do a reallocation and the new value is
+    # identical to the old one.
+    e['a'] = foo
+
+    y = e.persistent()
+    assert 'b' in y
+    assert y is e.persistent()
+
+
+def test_evolver_remove_element():
+    e = m(a=1000, b=2000).evolver()
+    assert 'a' in e
+
+    del e['a']
+    assert 'a' not in e
+
+
+def test_evolver_remove_element_not_present():
+    e = m(a=1000, b=2000).evolver()
+
+    with pytest.raises(KeyError) as error:
+        del e['c']
+
+    assert str(error.value) == "'c'"
+
+
+def test_copy_returns_reference_to_self():
+    m1 = m(a=10)
+    assert m1.copy() is m1
+
+
+def test_dot_access_of_non_existing_element_raises_attribute_error():
+    m1 = m(a=10)
+
+    with pytest.raises(AttributeError) as error:
+        m1.b
+
+    error_message = str(error.value)
+
+    assert "'b'" in error_message
+    assert type(m1).__name__ in error_message
+
+
+def test_pmap_unorderable():
+    with pytest.raises(TypeError):
+        _ = m(a=1) < m(b=2)
+
+    with pytest.raises(TypeError):
+        _ = m(a=1) <= m(b=2)
+
+    with pytest.raises(TypeError):
+        _ = m(a=1) > m(b=2)
+
+    with pytest.raises(TypeError):
+        _ = m(a=1) >= m(b=2)
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(m(a=1))
+
+
+def test_iterable():
+    """
+    PMaps can be created from iterables even though they can't be len() hinted.
+    """
+
+    assert pmap(iter([("a", "b")])) == pmap([("a", "b")])
diff --git a/contrib/python/pyrsistent/py2/tests/memory_profiling.py b/contrib/python/pyrsistent/py2/tests/memory_profiling.py
new file mode 100644
index 00000000000..4b47dd9f8c6
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/memory_profiling.py
@@ -0,0 +1,44 @@
+"""
+Script to try do detect any memory leaks that may be lurking in the C implementation of the PVector.
+"""
+import inspect
+import sys
+import time
+import memory_profiler
+import vector_test
+from pyrsistent import pvector
+
+try:
+    import pvectorc
+except ImportError:
+    print("No C implementation of PVector available, terminating")
+    sys.exit()
+
+
+PROFILING_DURATION = 2.0
+
+def run_function(fn):
+    stop = time.time() + PROFILING_DURATION
+    while time.time() < stop:
+        fn(pvector)
+
+def detect_memory_leak(samples):
+    # Do not allow a memory usage difference larger than 5% between the beginning and the end.
+    # Skip the first samples to get rid of the build up period and the last sample since it seems
+    # a little less precise
+    return abs(1 - (sum(samples[5:8]) / sum(samples[-4:-1]))) > 0.05
+
+def profile_tests():
+    test_functions = [fn for fn in inspect.getmembers(vector_test, inspect.isfunction)
+                      if fn[0].startswith('test_')]
+
+    for name, fn in test_functions:
+        # There are a couple of tests that are not run for the C implementation, skip those
+        fn_args = inspect.getargspec(fn)[0]
+        if 'pvector' in fn_args:
+            print('Executing %s' % name)
+            result = memory_profiler.memory_usage((run_function, (fn,), {}), interval=.1)
+            assert not detect_memory_leak(result), (name, result)
+
+if __name__ == "__main__":
+    profile_tests()
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py2/tests/record_test.py b/contrib/python/pyrsistent/py2/tests/record_test.py
new file mode 100644
index 00000000000..286b10a60e0
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/record_test.py
@@ -0,0 +1,864 @@
+import pickle
+import datetime
+import pytest
+import six
+import uuid
+from pyrsistent import (
+    PRecord, field, InvariantException, ny, pset, PSet, CheckedPVector,
+    PTypeError, pset_field, pvector_field, pmap_field, pmap, PMap,
+    pvector, PVector, v, m)
+
+
+class ARecord(PRecord):
+    x = field(type=(int, float))
+    y = field()
+
+
+class Hierarchy(PRecord):
+    point1 = field(ARecord)
+    point2 = field(ARecord)
+    points = pvector_field(ARecord)
+
+
+class RecordContainingContainers(PRecord):
+    map = pmap_field(str, str)
+    vec = pvector_field(str)
+    set = pset_field(str)
+
+
+class UniqueThing(PRecord):
+    id = field(type=uuid.UUID, factory=uuid.UUID)
+
+
+class Something(object):
+    pass
+
+class Another(object):
+    pass
+
+def test_create_ignore_extra_true():
+    h = Hierarchy.create(
+        {'point1': {'x': 1, 'y': 'foo', 'extra_field_0': 'extra_data_0'},
+         'point2': {'x': 1, 'y': 'foo', 'extra_field_1': 'extra_data_1'},
+         'extra_field_2': 'extra_data_2',
+         }, ignore_extra=True
+    )
+    assert h
+
+
+def test_create_ignore_extra_true_sequence_hierarchy():
+    h = Hierarchy.create(
+        {'point1': {'x': 1, 'y': 'foo', 'extra_field_0': 'extra_data_0'},
+         'point2': {'x': 1, 'y': 'foo', 'extra_field_1': 'extra_data_1'},
+         'points': [{'x': 1, 'y': 'foo', 'extra_field_2': 'extra_data_2'},
+                    {'x': 1, 'y': 'foo', 'extra_field_3': 'extra_data_3'}],
+         'extra_field____': 'extra_data_2',
+         }, ignore_extra=True
+    )
+    assert h
+
+
+def test_create():
+    r = ARecord(x=1, y='foo')
+    assert r.x == 1
+    assert r.y == 'foo'
+    assert isinstance(r, ARecord)
+
+
+def test_create_ignore_extra():
+    r = ARecord.create({'x': 1, 'y': 'foo', 'z': None}, ignore_extra=True)
+    assert r.x == 1
+    assert r.y == 'foo'
+    assert isinstance(r, ARecord)
+
+
+def test_create_ignore_extra_false():
+    with pytest.raises(AttributeError):
+        _ = ARecord.create({'x': 1, 'y': 'foo', 'z': None})
+
+
+def test_correct_assignment():
+    r = ARecord(x=1, y='foo')
+    r2 = r.set('x', 2.0)
+    r3 = r2.set('y', 'bar')
+
+    assert r2 == {'x': 2.0, 'y': 'foo'}
+    assert r3 == {'x': 2.0, 'y': 'bar'}
+    assert isinstance(r3, ARecord)
+
+
+def test_direct_assignment_not_possible():
+    with pytest.raises(AttributeError):
+        ARecord().x = 1
+
+
+def test_cannot_assign_undeclared_fields():
+    with pytest.raises(AttributeError):
+        ARecord().set('z', 5)
+
+
+def test_cannot_assign_wrong_type_to_fields():
+    try:
+        ARecord().set('x', 'foo')
+        assert False
+    except PTypeError as e:
+        assert e.source_class == ARecord
+        assert e.field == 'x'
+        assert e.expected_types == set([int, float])
+        assert e.actual_type is type('foo')
+
+
+def test_cannot_construct_with_undeclared_fields():
+    with pytest.raises(AttributeError):
+        ARecord(z=5)
+
+
+def test_cannot_construct_with_fields_of_wrong_type():
+    with pytest.raises(TypeError):
+        ARecord(x='foo')
+
+
+def test_support_record_inheritance():
+    class BRecord(ARecord):
+        z = field()
+
+    r = BRecord(x=1, y='foo', z='bar')
+
+    assert isinstance(r, BRecord)
+    assert isinstance(r, ARecord)
+    assert r == {'x': 1, 'y': 'foo', 'z': 'bar'}
+
+
+def test_single_type_spec():
+    class A(PRecord):
+        x = field(type=int)
+
+    r = A(x=1)
+    assert r.x == 1
+
+    with pytest.raises(TypeError):
+        r.set('x', 'foo')
+
+
+def test_remove():
+    r = ARecord(x=1, y='foo')
+    r2 = r.remove('y')
+
+    assert isinstance(r2, ARecord)
+    assert r2 == {'x': 1}
+
+
+def test_remove_non_existing_member():
+    r = ARecord(x=1, y='foo')
+
+    with pytest.raises(KeyError):
+        r.remove('z')
+
+
+def test_field_invariant_must_hold():
+    class BRecord(PRecord):
+        x = field(invariant=lambda x: (x > 1, 'x too small'))
+        y = field(mandatory=True)
+
+    try:
+        BRecord(x=1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('x too small',)
+        assert e.missing_fields == ('BRecord.y',)
+
+
+def test_global_invariant_must_hold():
+    class BRecord(PRecord):
+        __invariant__ = lambda r: (r.x <= r.y, 'y smaller than x')
+        x = field()
+        y = field()
+
+    BRecord(x=1, y=2)
+
+    try:
+        BRecord(x=2, y=1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('y smaller than x',)
+        assert e.missing_fields == ()
+
+
+def test_set_multiple_fields():
+    a = ARecord(x=1, y='foo')
+    b = a.set(x=2, y='bar')
+
+    assert b == {'x': 2, 'y': 'bar'}
+
+
+def test_initial_value():
+    class BRecord(PRecord):
+        x = field(initial=1)
+        y = field(initial=2)
+
+    a = BRecord()
+    assert a.x == 1
+    assert a.y == 2
+
+
+def test_enum_field():
+    try:
+        from enum import Enum
+    except ImportError:
+        return  # Enum not supported in this environment
+
+    class TestEnum(Enum):
+        x = 1
+        y = 2
+
+    class RecordContainingEnum(PRecord):
+        enum_field = field(type=TestEnum)
+
+    r = RecordContainingEnum(enum_field=TestEnum.x)
+    assert r.enum_field == TestEnum.x
+
+def test_type_specification_must_be_a_type():
+    with pytest.raises(TypeError):
+        class BRecord(PRecord):
+            x = field(type=1)
+
+
+def test_initial_must_be_of_correct_type():
+    with pytest.raises(TypeError):
+        class BRecord(PRecord):
+            x = field(type=int, initial='foo')
+
+
+def test_invariant_must_be_callable():
+    with pytest.raises(TypeError):
+        class BRecord(PRecord):
+            x = field(invariant='foo')  # type: ignore
+
+
+def test_global_invariants_are_inherited():
+    class BRecord(PRecord):
+        __invariant__ = lambda r: (r.x % r.y == 0, 'modulo')
+        x = field()
+        y = field()
+
+    class CRecord(BRecord):
+        __invariant__ = lambda r: (r.x > r.y, 'size')
+
+    try:
+        CRecord(x=5, y=3)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('modulo',)
+
+
+def test_global_invariants_must_be_callable():
+    with pytest.raises(TypeError):
+        class CRecord(PRecord):
+            __invariant__ = 1
+
+
+def test_repr():
+    r = ARecord(x=1, y=2)
+    assert repr(r) == 'ARecord(x=1, y=2)' or repr(r) == 'ARecord(y=2, x=1)'
+
+
+def test_factory():
+    class BRecord(PRecord):
+        x = field(type=int, factory=int)
+
+    assert BRecord(x=2.5) == {'x': 2}
+
+
+def test_factory_must_be_callable():
+    with pytest.raises(TypeError):
+        class BRecord(PRecord):
+            x = field(type=int, factory=1)  # type: ignore
+
+
+def test_nested_record_construction():
+    class BRecord(PRecord):
+        x = field(int, factory=int)
+
+    class CRecord(PRecord):
+        a = field()
+        b = field(type=BRecord)
+
+    r = CRecord.create({'a': 'foo', 'b': {'x': '5'}})
+    assert isinstance(r, CRecord)
+    assert isinstance(r.b, BRecord)
+    assert r == {'a': 'foo', 'b': {'x': 5}}
+
+
+def test_pickling():
+    x = ARecord(x=2.0, y='bar')
+    y = pickle.loads(pickle.dumps(x, -1))
+
+    assert x == y
+    assert isinstance(y, ARecord)
+
+def test_supports_pickling_with_typed_container_fields():
+    obj = RecordContainingContainers(
+        map={'foo': 'bar'}, set=['hello', 'there'], vec=['a', 'b'])
+    obj2 = pickle.loads(pickle.dumps(obj))
+    assert obj == obj2
+
+def test_all_invariant_errors_reported():
+    class BRecord(PRecord):
+        x = field(factory=int, invariant=lambda x: (x >= 0, 'x negative'))
+        y = field(mandatory=True)
+
+    class CRecord(PRecord):
+        a = field(invariant=lambda x: (x != 0, 'a zero'))
+        b = field(type=BRecord)
+
+    try:
+        CRecord.create({'a': 0, 'b': {'x': -5}})
+        assert False
+    except InvariantException as e:
+        assert set(e.invariant_errors) == set(['x negative', 'a zero'])
+        assert e.missing_fields == ('BRecord.y',)
+
+
+def test_precord_factory_method_is_idempotent():
+    class BRecord(PRecord):
+        x = field()
+        y = field()
+
+    r = BRecord(x=1, y=2)
+    assert BRecord.create(r) is r
+
+
+def test_serialize():
+    class BRecord(PRecord):
+        d = field(type=datetime.date,
+                  factory=lambda d: datetime.datetime.strptime(d, "%d%m%Y").date(),
+                  serializer=lambda format, d: d.strftime('%Y-%m-%d') if format == 'ISO' else d.strftime('%d%m%Y'))
+
+    assert BRecord(d='14012015').serialize('ISO') == {'d': '2015-01-14'}
+    assert BRecord(d='14012015').serialize('other') == {'d': '14012015'}
+
+
+def test_nested_serialize():
+    class BRecord(PRecord):
+        d = field(serializer=lambda format, d: format)
+
+    class CRecord(PRecord):
+        b = field()
+
+    serialized = CRecord(b=BRecord(d='foo')).serialize('bar')
+
+    assert serialized == {'b': {'d': 'bar'}}
+    assert isinstance(serialized, dict)
+
+
+def test_serializer_must_be_callable():
+    with pytest.raises(TypeError):
+        class CRecord(PRecord):
+            x = field(serializer=1)  # type: ignore
+
+
+def test_transform_without_update_returns_same_precord():
+    r = ARecord(x=2.0, y='bar')
+    assert r.transform([ny], lambda x: x) is r
+
+
+class Application(PRecord):
+    name = field(type=(six.text_type,) + six.string_types)
+    image = field(type=(six.text_type,) + six.string_types)
+
+
+class ApplicationVector(CheckedPVector):
+    __type__ = Application
+
+
+class Node(PRecord):
+    applications = field(type=ApplicationVector)
+
+
+def test_nested_create_serialize():
+    node = Node(applications=[Application(name='myapp', image='myimage'),
+                              Application(name='b', image='c')])
+
+    node2 = Node.create({'applications': [{'name': 'myapp', 'image': 'myimage'},
+                                          {'name': 'b', 'image': 'c'}]})
+
+    assert node == node2
+
+    serialized = node.serialize()
+    restored = Node.create(serialized)
+
+    assert restored == node
+
+
+def test_pset_field_initial_value():
+    """
+    ``pset_field`` results in initial value that is empty.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    assert Record() == Record(value=[])
+
+def test_pset_field_custom_initial():
+    """
+    A custom initial value can be passed in.
+    """
+    class Record(PRecord):
+        value = pset_field(int, initial=(1, 2))
+    assert Record() == Record(value=[1, 2])
+
+def test_pset_field_factory():
+    """
+    ``pset_field`` has a factory that creates a ``PSet``.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    record = Record(value=[1, 2])
+    assert isinstance(record.value, PSet)
+
+def test_pset_field_checked_set():
+    """
+    ``pset_field`` results in a set that enforces its type.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    record = Record(value=[1, 2])
+    with pytest.raises(TypeError):
+        record.value.add("hello")  # type: ignore
+
+def test_pset_field_checked_vector_multiple_types():
+    """
+    ``pset_field`` results in a vector that enforces its types.
+    """
+    class Record(PRecord):
+        value = pset_field((int, str))
+    record = Record(value=[1, 2, "hello"])
+    with pytest.raises(TypeError):
+        record.value.add(object())
+
+def test_pset_field_type():
+    """
+    ``pset_field`` enforces its type.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    record = Record()
+    with pytest.raises(TypeError):
+        record.set("value", None)
+
+def test_pset_field_mandatory():
+    """
+    ``pset_field`` is a mandatory field.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    record = Record(value=[1])
+    with pytest.raises(InvariantException):
+        record.remove("value")
+
+def test_pset_field_default_non_optional():
+    """
+    By default ``pset_field`` is non-optional, i.e. does not allow
+    ``None``.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    with pytest.raises(TypeError):
+        Record(value=None)
+
+def test_pset_field_explicit_non_optional():
+    """
+    If ``optional`` argument is ``False`` then ``pset_field`` is
+    non-optional, i.e. does not allow ``None``.
+    """
+    class Record(PRecord):
+        value = pset_field(int, optional=False)
+    with pytest.raises(TypeError):
+        Record(value=None)
+
+def test_pset_field_optional():
+    """
+    If ``optional`` argument is true, ``None`` is acceptable alternative
+    to a set.
+    """
+    class Record(PRecord):
+        value = pset_field(int, optional=True)
+    assert ((Record(value=[1, 2]).value, Record(value=None).value) ==
+            (pset([1, 2]), None))
+
+def test_pset_field_name():
+    """
+    The created set class name is based on the type of items in the set.
+    """
+    class Record(PRecord):
+        value = pset_field(Something)
+        value2 = pset_field(int)
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingPSet", "IntPSet"))
+
+def test_pset_multiple_types_field_name():
+    """
+    The created set class name is based on the multiple given types of
+    items in the set.
+    """
+    class Record(PRecord):
+        value = pset_field((Something, int))
+
+    assert (Record().value.__class__.__name__ ==
+            "SomethingIntPSet")
+
+def test_pset_field_name_string_type():
+    """
+    The created set class name is based on the type of items specified by name
+    """
+    class Record(PRecord):
+        value = pset_field("__tests__.record_test.Something")
+    assert Record().value.__class__.__name__ == "SomethingPSet"
+
+
+def test_pset_multiple_string_types_field_name():
+    """
+    The created set class name is based on the multiple given types of
+    items in the set specified by name
+    """
+    class Record(PRecord):
+        value = pset_field(("__tests__.record_test.Something", "__tests__.record_test.Another"))
+
+    assert Record().value.__class__.__name__ == "SomethingAnotherPSet"
+
+def test_pvector_field_initial_value():
+    """
+    ``pvector_field`` results in initial value that is empty.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    assert Record() == Record(value=[])
+
+def test_pvector_field_custom_initial():
+    """
+    A custom initial value can be passed in.
+    """
+    class Record(PRecord):
+        value = pvector_field(int, initial=(1, 2))
+    assert Record() == Record(value=[1, 2])
+
+def test_pvector_field_factory():
+    """
+    ``pvector_field`` has a factory that creates a ``PVector``.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    record = Record(value=[1, 2])
+    assert isinstance(record.value, PVector)
+
+def test_pvector_field_checked_vector():
+    """
+    ``pvector_field`` results in a vector that enforces its type.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    record = Record(value=[1, 2])
+    with pytest.raises(TypeError):
+        record.value.append("hello")  # type: ignore
+
+def test_pvector_field_checked_vector_multiple_types():
+    """
+    ``pvector_field`` results in a vector that enforces its types.
+    """
+    class Record(PRecord):
+        value = pvector_field((int, str))
+    record = Record(value=[1, 2, "hello"])
+    with pytest.raises(TypeError):
+        record.value.append(object())
+
+def test_pvector_field_type():
+    """
+    ``pvector_field`` enforces its type.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    record = Record()
+    with pytest.raises(TypeError):
+        record.set("value", None)
+
+def test_pvector_field_mandatory():
+    """
+    ``pvector_field`` is a mandatory field.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    record = Record(value=[1])
+    with pytest.raises(InvariantException):
+        record.remove("value")
+
+def test_pvector_field_default_non_optional():
+    """
+    By default ``pvector_field`` is non-optional, i.e. does not allow
+    ``None``.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    with pytest.raises(TypeError):
+        Record(value=None)
+
+def test_pvector_field_explicit_non_optional():
+    """
+    If ``optional`` argument is ``False`` then ``pvector_field`` is
+    non-optional, i.e. does not allow ``None``.
+    """
+    class Record(PRecord):
+        value = pvector_field(int, optional=False)
+    with pytest.raises(TypeError):
+        Record(value=None)
+
+def test_pvector_field_optional():
+    """
+    If ``optional`` argument is true, ``None`` is acceptable alternative
+    to a sequence.
+    """
+    class Record(PRecord):
+        value = pvector_field(int, optional=True)
+    assert ((Record(value=[1, 2]).value, Record(value=None).value) ==
+            (pvector([1, 2]), None))
+
+def test_pvector_field_name():
+    """
+    The created set class name is based on the type of items in the set.
+    """
+    class Record(PRecord):
+        value = pvector_field(Something)
+        value2 = pvector_field(int)
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingPVector", "IntPVector"))
+
+def test_pvector_multiple_types_field_name():
+    """
+    The created vector class name is based on the multiple given types of
+    items in the vector.
+    """
+    class Record(PRecord):
+        value = pvector_field((Something, int))
+
+    assert (Record().value.__class__.__name__ ==
+            "SomethingIntPVector")
+
+def test_pvector_field_name_string_type():
+    """
+    The created set class name is based on the type of items in the set
+    specified by name.
+    """
+    class Record(PRecord):
+        value = pvector_field("__tests__.record_test.Something")
+    assert Record().value.__class__.__name__ == "SomethingPVector"
+
+def test_pvector_multiple_string_types_field_name():
+    """
+    The created vector class name is based on the multiple given types of
+    items in the vector.
+    """
+    class Record(PRecord):
+        value = pvector_field(("__tests__.record_test.Something", "__tests__.record_test.Another"))
+
+    assert Record().value.__class__.__name__ == "SomethingAnotherPVector"
+
+def test_pvector_field_create_from_nested_serialized_data():
+    class Foo(PRecord):
+        foo = field(type=str)
+
+    class Bar(PRecord):
+        bar = pvector_field(Foo)
+
+    data = Bar(bar=v(Foo(foo="foo")))
+    Bar.create(data.serialize()) == data
+
+def test_pmap_field_initial_value():
+    """
+    ``pmap_field`` results in initial value that is empty.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int)
+    assert Record() == Record(value={})
+
+def test_pmap_field_factory():
+    """
+    ``pmap_field`` has a factory that creates a ``PMap``.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int)
+    record = Record(value={1:  1234})
+    assert isinstance(record.value, PMap)
+
+def test_pmap_field_checked_map_key():
+    """
+    ``pmap_field`` results in a map that enforces its key type.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, type(None))
+    record = Record(value={1: None})
+    with pytest.raises(TypeError):
+        record.value.set("hello", None)  # type: ignore
+
+def test_pmap_field_checked_map_value():
+    """
+    ``pmap_field`` results in a map that enforces its value type.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, type(None))
+    record = Record(value={1: None})
+    with pytest.raises(TypeError):
+        record.value.set(2, 4)  # type: ignore
+
+def test_pmap_field_checked_map_key_multiple_types():
+    """
+    ``pmap_field`` results in a map that enforces its key types.
+    """
+    class Record(PRecord):
+        value = pmap_field((int, str), type(None))
+    record = Record(value={1: None, "hello": None})
+    with pytest.raises(TypeError):
+        record.value.set(object(), None)
+
+def test_pmap_field_checked_map_value_multiple_types():
+    """
+    ``pmap_field`` results in a map that enforces its value types.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, (str, type(None)))
+    record = Record(value={1: None, 3: "hello"})
+    with pytest.raises(TypeError):
+        record.value.set(2, 4)
+
+def test_pmap_field_mandatory():
+    """
+    ``pmap_field`` is a mandatory field.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int)
+    record = Record()
+    with pytest.raises(InvariantException):
+        record.remove("value")
+
+def test_pmap_field_default_non_optional():
+    """
+    By default ``pmap_field`` is non-optional, i.e. does not allow
+    ``None``.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int)
+    # Ought to be TypeError, but pyrsistent doesn't quite allow that:
+    with pytest.raises(AttributeError):
+        Record(value=None)
+
+def test_pmap_field_explicit_non_optional():
+    """
+    If ``optional`` argument is ``False`` then ``pmap_field`` is
+    non-optional, i.e. does not allow ``None``.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int, optional=False)
+    # Ought to be TypeError, but pyrsistent doesn't quite allow that:
+    with pytest.raises(AttributeError):
+        Record(value=None)
+
+def test_pmap_field_optional():
+    """
+    If ``optional`` argument is true, ``None`` is acceptable alternative
+    to a set.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int, optional=True)
+    assert (Record(value={1: 2}).value, Record(value=None).value) == \
+           (pmap({1: 2}), None)
+
+def test_pmap_field_name():
+    """
+    The created map class name is based on the types of items in the map.
+    """
+    class Record(PRecord):
+        value = pmap_field(Something, Another)
+        value2 = pmap_field(int, float)
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingToAnotherPMap", "IntToFloatPMap"))
+
+def test_pmap_field_name_multiple_types():
+    """
+    The created map class name is based on the types of items in the map,
+    including when there are multiple supported types.
+    """
+    class Record(PRecord):
+        value = pmap_field((Something, Another), int)
+        value2 = pmap_field(str, (int, float))
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingAnotherToIntPMap", "StrToIntFloatPMap"))
+
+def test_pmap_field_name_string_type():
+    """
+    The created map class name is based on the types of items in the map
+    specified by name.
+    """
+    class Record(PRecord):
+        value = pmap_field("__tests__.record_test.Something", "__tests__.record_test.Another")
+    assert Record().value.__class__.__name__ == "SomethingToAnotherPMap"
+
+def test_pmap_field_name_multiple_string_types():
+    """
+    The created map class name is based on the types of items in the map,
+    including when there are multiple supported types.
+    """
+    class Record(PRecord):
+        value = pmap_field(("__tests__.record_test.Something", "__tests__.record_test.Another"), int)
+        value2 = pmap_field(str, ("__tests__.record_test.Something", "__tests__.record_test.Another"))
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingAnotherToIntPMap", "StrToSomethingAnotherPMap"))
+
+def test_pmap_field_invariant():
+    """
+    The ``invariant`` parameter is passed through to ``field``.
+    """
+    class Record(PRecord):
+        value = pmap_field(
+            int, int,
+            invariant=(
+                lambda pmap: (len(pmap) == 1, "Exactly one item required.")
+            )
+        )
+    with pytest.raises(InvariantException):
+        Record(value={})
+    with pytest.raises(InvariantException):
+        Record(value={1: 2, 3: 4})
+    assert Record(value={1: 2}).value == {1: 2}
+
+
+def test_pmap_field_create_from_nested_serialized_data():
+    class Foo(PRecord):
+        foo = field(type=str)
+
+    class Bar(PRecord):
+        bar = pmap_field(str, Foo)
+
+    data = Bar(bar=m(foo_key=Foo(foo="foo")))
+    Bar.create(data.serialize()) == data
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(ARecord(x=1, y=2))
+
+
+def test_supports_lazy_initial_value_for_field():
+    class MyRecord(PRecord):
+        a = field(int, initial=lambda: 2)
+
+    assert MyRecord() == MyRecord(a=2)
+
+
+def test_pickle_with_one_way_factory():
+    """
+    A field factory isn't called when restoring from pickle.
+    """
+    thing = UniqueThing(id='25544626-86da-4bce-b6b6-9186c0804d64')
+    assert thing == pickle.loads(pickle.dumps(thing))
diff --git a/contrib/python/pyrsistent/py2/tests/regression_test.py b/contrib/python/pyrsistent/py2/tests/regression_test.py
new file mode 100644
index 00000000000..f8c11338348
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/regression_test.py
@@ -0,0 +1,30 @@
+from pyrsistent import pmap
+import random
+
+import gc
+
+
+def test_segfault_issue_52():
+    threshold = None
+    if hasattr(gc, 'get_threshold'):
+        # PyPy is lacking these functions
+        threshold = gc.get_threshold()
+        gc.set_threshold(1, 1, 1)  # fail fast
+
+    v = [pmap()]
+
+    def step():
+        depth = random.randint(1, 10)
+        path = random.sample(range(100000), depth)
+        v[0] = v[0].transform(path, "foo")
+
+    for i in range(1000):  # usually crashes after 10-20 steps
+        while True:
+            try:
+                step()
+                break
+            except AttributeError:  # evolver on string
+                continue
+
+    if threshold:
+        gc.set_threshold(*threshold)
diff --git a/contrib/python/pyrsistent/py2/tests/set_test.py b/contrib/python/pyrsistent/py2/tests/set_test.py
new file mode 100644
index 00000000000..6d33bb421bc
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/set_test.py
@@ -0,0 +1,178 @@
+from pyrsistent import pset, s
+import pytest
+import pickle
+
+
+def test_literalish_works():
+    assert s() is pset()
+    assert s(1, 2) == pset([1, 2])
+
+
+def test_supports_hash():
+    assert hash(s(1, 2)) == hash(s(1, 2))
+
+
+def test_empty_truthiness():
+    assert s(1)
+    assert not s()
+
+
+def test_contains_elements_that_it_was_initialized_with():
+    initial = [1, 2, 3]
+    s = pset(initial)
+
+    assert set(s) == set(initial)
+    assert len(s) == len(set(initial))
+
+
+def test_is_immutable():
+    s1 = pset([1])
+    s2 = s1.add(2)
+
+    assert s1 == pset([1])
+    assert s2 == pset([1, 2])
+
+    s3 = s2.remove(1)
+    assert s2 == pset([1, 2])
+    assert s3 == pset([2])
+
+
+def test_remove_when_not_present():
+    s1 = s(1, 2, 3)
+    with pytest.raises(KeyError):
+        s1.remove(4)
+
+
+def test_discard():
+    s1 = s(1, 2, 3)
+    assert s1.discard(3) == s(1, 2)
+    assert s1.discard(4) is s1
+
+
+def test_is_iterable():
+    assert sum(pset([1, 2, 3])) == 6
+
+
+def test_contains():
+    s = pset([1, 2, 3])
+
+    assert 2 in s
+    assert 4 not in s
+
+
+def test_supports_set_operations():
+    s1 = pset([1, 2, 3])
+    s2 = pset([3, 4, 5])
+
+    assert s1 | s2 == s(1, 2, 3, 4, 5)
+    assert s1.union(s2) == s1 | s2
+
+    assert s1 & s2 == s(3)
+    assert s1.intersection(s2) == s1 & s2
+
+    assert s1 - s2 == s(1, 2)
+    assert s1.difference(s2) == s1 - s2
+
+    assert s1 ^ s2 == s(1, 2, 4, 5)
+    assert s1.symmetric_difference(s2) == s1 ^ s2
+
+
+def test_supports_set_comparisons():
+    s1 = s(1, 2, 3)
+    s3 = s(1, 2)
+    s4 = s(1, 2, 3)
+
+    assert s(1, 2, 3, 3, 5) == s(1, 2, 3, 5)
+    assert s1 != s3
+
+    assert s3 < s1
+    assert s3 <= s1
+    assert s3 <= s4
+
+    assert s1 > s3
+    assert s1 >= s3
+    assert s4 >= s3
+
+
+def test_str():
+    rep = str(pset([1, 2, 3]))
+    assert rep == "pset([1, 2, 3])"
+
+
+def test_is_disjoint():
+    s1 = pset([1, 2, 3])
+    s2 = pset([3, 4, 5])
+    s3 = pset([4, 5])
+
+    assert not s1.isdisjoint(s2)
+    assert s1.isdisjoint(s3)
+
+
+def test_evolver_simple_add():
+    x = s(1, 2, 3)
+    e = x.evolver()
+    assert not e.is_dirty()
+
+    e.add(4)
+    assert e.is_dirty()
+
+    x2 = e.persistent()
+    assert not e.is_dirty()
+    assert x2 == s(1, 2, 3, 4)
+    assert x == s(1, 2, 3)
+
+def test_evolver_simple_remove():
+    x = s(1, 2, 3)
+    e = x.evolver()
+    e.remove(2)
+
+    x2 = e.persistent()
+    assert x2 == s(1, 3)
+    assert x == s(1, 2, 3)
+
+
+def test_evolver_no_update_produces_same_pset():
+    x = s(1, 2, 3)
+    e = x.evolver()
+    assert e.persistent() is x
+
+
+def test_evolver_len():
+    x = s(1, 2, 3)
+    e = x.evolver()
+    assert len(e) == 3
+
+
+def test_copy_returns_reference_to_self():
+    s1 = s(10)
+    assert s1.copy() is s1
+
+
+def test_pickling_empty_set():
+    assert pickle.loads(pickle.dumps(s(), -1)) == s()
+
+
+def test_pickling_non_empty_map():
+    assert pickle.loads(pickle.dumps(s(1, 2), -1)) == s(1, 2)
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(s(1))
+
+
+def test_update():
+    assert s(1, 2, 3).update([3, 4, 4, 5]) == s(1, 2, 3, 4, 5)
+
+
+def test_update_no_elements():
+    s1 = s(1, 2)
+    assert s1.update([]) is s1
+
+
+def test_iterable():
+    """
+    PSets can be created from iterables even though they can't be len() hinted.
+    """
+
+    assert pset(iter("a")) == pset(iter("a"))
diff --git a/contrib/python/pyrsistent/py2/tests/toolz_test.py b/contrib/python/pyrsistent/py2/tests/toolz_test.py
new file mode 100644
index 00000000000..d145704b864
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/toolz_test.py
@@ -0,0 +1,6 @@
+from pyrsistent import get_in, m, v
+
+
+def test_get_in():
+    # This is not an extensive test. The doctest covers that fairly good though.
+    get_in(m(a=v(1, 2, 3)), ['m', 1]) == 2
diff --git a/contrib/python/pyrsistent/py2/tests/transform_test.py b/contrib/python/pyrsistent/py2/tests/transform_test.py
new file mode 100644
index 00000000000..770fb47029e
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/transform_test.py
@@ -0,0 +1,117 @@
+from pyrsistent import freeze, inc, discard, rex, ny, field, PClass, pmap
+
+
+def test_callable_command():
+    m = freeze({'foo': {'bar': {'baz': 1}}})
+    assert m.transform(['foo', 'bar', 'baz'], inc) == {'foo': {'bar': {'baz': 2}}}
+
+
+def test_predicate():
+    m = freeze({'foo': {'bar': {'baz': 1}, 'qux': {'baz': 1}}})
+    assert m.transform(['foo', lambda x: x.startswith('b'), 'baz'], inc) == {'foo': {'bar': {'baz': 2}, 'qux': {'baz': 1}}}
+
+
+def test_broken_predicate():
+    broken_predicates = [
+        lambda: None,
+        lambda a, b, c: None,
+        lambda a, b, c, d=None: None,
+        lambda *args: None,
+        lambda **kwargs: None,
+    ]
+    for pred in broken_predicates:
+        try:
+            freeze({}).transform([pred], None)
+            assert False
+        except ValueError as e:
+            assert str(e) == "callable in transform path must take 1 or 2 arguments"
+
+
+def test_key_value_predicate():
+    m = freeze({
+        'foo': 1,
+        'bar': 2,
+    })
+    assert m.transform([
+        lambda k, v: (k, v) == ('foo', 1),
+    ], lambda v: v * 3) == {"foo": 3, "bar": 2}
+
+
+def test_remove():
+    m = freeze({'foo': {'bar': {'baz': 1}}})
+    assert m.transform(['foo', 'bar', 'baz'], discard) == {'foo': {'bar': {}}}
+
+
+def test_remove_pvector():
+    m = freeze({'foo': [1, 2, 3]})
+    assert m.transform(['foo', 1], discard) == {'foo': [1, 3]}
+
+
+def test_remove_pclass():
+    class MyClass(PClass):
+        a = field()
+        b = field()
+
+    m = freeze({'foo': MyClass(a=1, b=2)})
+    assert m.transform(['foo', 'b'], discard) == {'foo': MyClass(a=1)}
+
+
+def test_predicate_no_match():
+    m = freeze({'foo': {'bar': {'baz': 1}}})
+    assert m.transform(['foo', lambda x: x.startswith('c'), 'baz'], inc) == m
+
+
+def test_rex_predicate():
+    m = freeze({'foo': {'bar': {'baz': 1},
+                        'bof': {'baz': 1}}})
+    assert m.transform(['foo', rex('^bo.*'), 'baz'], inc) == {'foo': {'bar': {'baz': 1},
+                                                                      'bof': {'baz': 2}}}
+
+
+def test_rex_with_non_string_key():
+    m = freeze({'foo': 1, 5: 2})
+    assert m.transform([rex(".*")], 5) == {'foo': 5, 5: 2}
+
+
+def test_ny_predicated_matches_any_key():
+    m = freeze({'foo': 1, 5: 2})
+    assert m.transform([ny], 5) == {'foo': 5, 5: 5}
+
+
+def test_new_elements_created_when_missing():
+    m = freeze({})
+    assert m.transform(['foo', 'bar', 'baz'], 7) == {'foo': {'bar': {'baz': 7}}}
+
+
+def test_mixed_vector_and_map():
+    m = freeze({'foo': [1, 2, 3]})
+    assert m.transform(['foo', 1], 5) == freeze({'foo': [1, 5, 3]})
+
+
+def test_vector_predicate_callable_command():
+    v = freeze([1, 2, 3, 4, 5])
+    assert v.transform([lambda i: 0 < i < 4], inc) == freeze(freeze([1, 3, 4, 5, 5]))
+
+
+def test_vector_insert_map_one_step_beyond_end():
+    v = freeze([1, 2])
+    assert v.transform([2, 'foo'], 3) == freeze([1, 2, {'foo': 3}])
+
+
+def test_multiple_transformations():
+    v = freeze([1, 2])
+    assert v.transform([2, 'foo'], 3, [2, 'foo'], inc) == freeze([1, 2, {'foo': 4}])
+
+
+def test_no_transformation_returns_the_same_structure():
+    v = freeze([{'foo': 1}, {'bar': 2}])
+    assert v.transform([ny, ny], lambda x: x) is v
+
+
+def test_discard_multiple_elements_in_pvector():
+    assert freeze([0, 1, 2, 3, 4]).transform([lambda i: i % 2], discard) == freeze([0, 2, 4])
+
+
+def test_transform_insert_empty_pmap():
+    m = pmap().transform(['123'], pmap())
+    assert m == pmap({'123': pmap()})
diff --git a/contrib/python/pyrsistent/py2/tests/vector_test.py b/contrib/python/pyrsistent/py2/tests/vector_test.py
new file mode 100644
index 00000000000..aa59ea0c1c5
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/vector_test.py
@@ -0,0 +1,934 @@
+import os
+import pickle
+import pytest
+
+from pyrsistent._pvector import python_pvector
+
+
+@pytest.fixture(scope='session', params=['pyrsistent._pvector', 'pvectorc'])
+def pvector(request):
+    if request.param == 'pvectorc' and os.environ.get('PYRSISTENT_NO_C_EXTENSION'):
+        pytest.skip('Configured to not run tests for C extension')
+
+    m = pytest.importorskip(request.param)
+    if request.param == 'pyrsistent._pvector':
+        return m.python_pvector
+    return m.pvector
+
+
+def test_literalish_works():
+    from pyrsistent import pvector, v
+    assert v() is pvector()
+    assert v(1, 2) == pvector([1, 2])
+
+
+def test_empty_initialization(pvector):
+    seq = pvector()
+    assert len(seq) == 0
+
+    with pytest.raises(IndexError) as error:
+        x = seq[0]
+    assert str(error.value) == 'Index out of range: 0'
+
+
+def test_initialization_with_one_element(pvector):
+    seq = pvector([3])
+    assert len(seq) == 1
+    assert seq[0] == 3
+
+
+def test_append_works_and_does_not_affect_original_within_tail(pvector):
+    seq1 = pvector([3])
+    seq2 = seq1.append(2)
+
+    assert len(seq1) == 1
+    assert seq1[0] == 3
+
+    assert len(seq2) == 2
+    assert seq2[0] == 3
+    assert seq2[1] == 2
+
+
+def test_append_works_and_does_not_affect_original_outside_tail(pvector):
+    original = pvector([])
+    seq = original
+
+    for x in range(33):
+        seq = seq.append(x)
+
+    assert len(seq) == 33
+    assert seq[0] == 0
+    assert seq[31] == 31
+    assert seq[32] == 32
+
+    assert len(original) == 0
+
+
+def test_append_when_root_overflows(pvector):
+    seq = pvector([])
+
+    for x in range(32 * 33):
+        seq = seq.append(x)
+
+    seq = seq.append(10001)
+
+    for i in range(32 * 33):
+        assert seq[i] == i
+
+    assert seq[32 * 33] == 10001
+
+
+def test_multi_level_sequence(pvector):
+    seq = pvector(range(8000))
+    seq2 = seq.append(11)
+
+    assert seq[5] == 5
+    assert seq2[7373] == 7373
+    assert seq2[8000] == 11
+
+
+def test_multi_level_sequence_from_iterator(pvector):
+    seq = pvector(iter(range(8000)))
+    seq2 = seq.append(11)
+
+    assert seq[5] == 5
+    assert seq2[7373] == 7373
+    assert seq2[8000] == 11
+
+
+def test_random_insert_within_tail(pvector):
+    seq = pvector([1, 2, 3])
+
+    seq2 = seq.set(1, 4)
+
+    assert seq2[1] == 4
+    assert seq[1] == 2
+
+
+def test_random_insert_outside_tail(pvector):
+    seq = pvector(range(20000))
+
+    seq2 = seq.set(19000, 4)
+
+    assert seq2[19000] == 4
+    assert seq[19000] == 19000
+
+
+def test_insert_beyond_end(pvector):
+    seq = pvector(range(2))
+    seq2 = seq.set(2, 50)
+    assert seq2[2] == 50
+
+    with pytest.raises(IndexError) as error:
+        seq2.set(19, 4)
+
+    assert str(error.value) == 'Index out of range: 19'
+
+
+def test_insert_with_index_from_the_end(pvector):
+    x = pvector([1, 2, 3, 4])
+
+    assert x.set(-2, 5) == pvector([1, 2, 5, 4])
+
+
+def test_insert_with_too_negative_index(pvector):
+    x = pvector([1, 2, 3, 4])
+
+    with pytest.raises(IndexError):
+        x.set(-5, 17)
+
+
+def test_iteration(pvector):
+    y = 0
+    seq = pvector(range(2000))
+    for x in seq:
+        assert x == y
+        y += 1
+
+    assert y == 2000
+
+
+def test_zero_extend(pvector):
+    the_list = []
+    seq = pvector()
+    seq2 = seq.extend(the_list)
+    assert seq == seq2
+
+
+def test_short_extend(pvector):
+    # Extend within tail length
+    the_list = [1, 2]
+    seq = pvector()
+    seq2 = seq.extend(the_list)
+
+    assert len(seq2) == len(the_list)
+    assert seq2[0] == the_list[0]
+    assert seq2[1] == the_list[1]
+
+
+def test_long_extend(pvector):
+    # Multi level extend
+    seq = pvector()
+    length = 2137
+
+    # Extend from scratch
+    seq2 = seq.extend(range(length))
+    assert len(seq2) == length
+    for i in range(length):
+        assert seq2[i] == i
+
+    # Extend already filled vector
+    seq3 = seq2.extend(range(length, length + 5))
+    assert len(seq3) == length + 5
+    for i in range(length + 5):
+        assert seq3[i] == i
+
+    # Check that the original vector is still intact
+    assert len(seq2) == length
+    for i in range(length):
+        assert seq2[i] == i
+
+
+def test_slicing_zero_length_range(pvector):
+    seq = pvector(range(10))
+    seq2 = seq[2:2]
+
+    assert len(seq2) == 0
+
+
+def test_slicing_range(pvector):
+    seq = pvector(range(10))
+    seq2 = seq[2:4]
+
+    assert list(seq2) == [2, 3]
+
+
+def test_slice_identity(pvector):
+    # Pvector is immutable, no need to make a copy!
+    seq = pvector(range(10))
+
+    assert seq is seq[::]
+
+
+def test_slicing_range_with_step(pvector):
+    seq = pvector(range(100))
+    seq2 = seq[2:12:3]
+
+    assert list(seq2) == [2, 5, 8, 11]
+
+
+def test_slicing_no_range_but_step(pvector):
+    seq = pvector(range(10))
+    seq2 = seq[::2]
+
+    assert list(seq2) == [0, 2, 4, 6, 8]
+
+
+def test_slicing_reverse(pvector):
+    seq = pvector(range(10))
+    seq2 = seq[::-1]
+
+    assert seq2[0] == 9
+    assert seq2[1] == 8
+    assert len(seq2) == 10
+
+    seq3 = seq[-3: -7: -1]
+    assert seq3[0] == 7
+    assert seq3[3] == 4
+    assert len(seq3) == 4
+
+
+def test_delete_index(pvector):
+    seq = pvector([1, 2, 3])
+    assert seq.delete(0) == pvector([2, 3])
+    assert seq.delete(1) == pvector([1, 3])
+    assert seq.delete(2) == pvector([1, 2])
+    assert seq.delete(-1) == pvector([1, 2])
+    assert seq.delete(-2) == pvector([1, 3])
+    assert seq.delete(-3) == pvector([2, 3])
+
+
+def test_delete_index_out_of_bounds(pvector):
+    with pytest.raises(IndexError):
+        pvector([]).delete(0)
+    with pytest.raises(IndexError):
+        pvector([]).delete(-1)
+
+
+def test_delete_index_malformed(pvector):
+    with pytest.raises(TypeError):
+        pvector([]).delete('a')
+
+
+def test_delete_slice(pvector):
+    seq = pvector(range(5))
+    assert seq.delete(1, 4) == pvector([0, 4])
+    assert seq.delete(4, 1) == seq
+    assert seq.delete(0, 1) == pvector([1, 2, 3, 4])
+    assert seq.delete(6, 8) == seq
+    assert seq.delete(-1, 1) == seq
+    assert seq.delete(1, -1) == pvector([0, 4])
+
+
+def test_remove(pvector):
+    seq = pvector(range(5))
+    assert seq.remove(3) == pvector([0, 1, 2, 4])
+
+
+def test_remove_first_only(pvector):
+    seq = pvector([1, 2, 3, 2, 1])
+    assert seq.remove(2) == pvector([1, 3, 2, 1])
+
+
+def test_remove_index_out_of_bounds(pvector):
+    seq = pvector(range(5))
+    with pytest.raises(ValueError) as err:
+        seq.remove(5)
+    assert 'not in' in str(err.value)
+
+
+def test_addition(pvector):
+    v = pvector([1, 2]) + pvector([3, 4])
+
+    assert list(v) == [1, 2, 3, 4]
+
+
+def test_sorted(pvector):
+    seq = pvector([5, 2, 3, 1])
+    assert [1, 2, 3, 5] == sorted(seq)
+
+
+def test_boolean_conversion(pvector):
+    assert not bool(pvector())
+    assert bool(pvector([1]))
+
+
+def test_access_with_negative_index(pvector):
+    seq = pvector([1, 2, 3, 4])
+
+    assert seq[-1] == 4
+    assert seq[-4] == 1
+
+
+def test_index_error_positive(pvector):
+    with pytest.raises(IndexError):
+        pvector([1, 2, 3])[3]
+
+
+def test_index_error_negative(pvector):
+    with pytest.raises(IndexError):
+        pvector([1, 2, 3])[-4]
+
+
+def test_is_sequence(pvector):
+    from pyrsistent._compat import Sequence
+    assert isinstance(pvector(), Sequence)
+
+
+def test_empty_repr(pvector):
+    assert str(pvector()) == "pvector([])"
+
+
+def test_non_empty_repr(pvector):
+    v = pvector([1, 2, 3])
+    assert str(v) == "pvector([1, 2, 3])"
+
+    # There's some state that needs to be reset between calls in the native version,
+    # test that multiple invocations work.
+    assert str(v) == "pvector([1, 2, 3])"
+
+
+def test_repr_when_contained_object_contains_reference_to_self(pvector):
+    x = [1, 2, 3]
+    v = pvector([1, 2, x])
+    x.append(v)
+    assert str(v) == 'pvector([1, 2, [1, 2, 3, pvector([1, 2, [...]])]])'
+
+    # Run a GC to provoke any potential misbehavior
+    import gc
+    gc.collect()
+
+
+def test_is_hashable(pvector):
+    from pyrsistent._compat import Hashable
+    v = pvector([1, 2, 3])
+    v2 = pvector([1, 2, 3])
+
+    assert hash(v) == hash(v2)
+    assert isinstance(pvector(), Hashable)
+
+
+def test_refuses_to_hash_when_members_are_unhashable(pvector):
+    v = pvector([1, 2, [1, 2]])
+
+    with pytest.raises(TypeError):
+        hash(v)
+
+
+def test_compare_same_vectors(pvector):
+    v = pvector([1, 2])
+    assert v == v
+    assert pvector() == pvector()
+
+
+def test_compare_with_other_type_of_object(pvector):
+    assert pvector([1, 2]) != 'foo'
+
+
+def test_compare_equal_vectors(pvector):
+    v1 = pvector([1, 2])
+    v2 = pvector([1, 2])
+    assert v1 == v2
+    assert v1 >= v2
+    assert v1 <= v2
+
+
+def test_compare_different_vectors_same_size(pvector):
+    v1 = pvector([1, 2])
+    v2 = pvector([1, 3])
+    assert v1 != v2
+
+
+def test_compare_different_vectors_different_sizes(pvector):
+    v1 = pvector([1, 2])
+    v2 = pvector([1, 2, 3])
+    assert v1 != v2
+
+
+def test_compare_lt_gt(pvector):
+    v1 = pvector([1, 2])
+    v2 = pvector([1, 2, 3])
+    assert v1 < v2
+    assert v2 > v1
+
+
+def test_repeat(pvector):
+    v = pvector([1, 2])
+    assert 5 * pvector() is pvector()
+    assert v is 1 * v
+    assert 0 * v is pvector()
+    assert 2 * pvector([1, 2]) == pvector([1, 2, 1, 2])
+    assert -3 * pvector([1, 2]) is pvector()
+
+
+def test_transform_zero_key_length(pvector):
+    x = pvector([1, 2])
+
+    assert x.transform([], 3) == 3
+
+
+def test_transform_base_case(pvector):
+    x = pvector([1, 2])
+
+    assert x.transform([1], 3) == pvector([1, 3])
+
+
+def test_transform_nested_vectors(pvector):
+    x = pvector([1, 2, pvector([3, 4]), 5])
+
+    assert x.transform([2, 0], 999) == pvector([1, 2, pvector([999, 4]), 5])
+
+
+def test_transform_when_appending(pvector):
+    from pyrsistent import m
+    x = pvector([1, 2])
+
+    assert x.transform([2, 'd'], 999) == pvector([1, 2, m(d=999)])
+
+
+def test_transform_index_error_out_range(pvector):
+    x = pvector([1, 2, pvector([3, 4]), 5])
+
+    with pytest.raises(IndexError):
+        x.transform([2, 10], 999)
+
+
+def test_transform_index_error_wrong_type(pvector):
+    x = pvector([1, 2, pvector([3, 4]), 5])
+
+    with pytest.raises(TypeError):
+        x.transform([2, 'foo'], 999)
+
+
+def test_transform_non_setable_type(pvector):
+    x = pvector([1, 2, 5])
+
+    with pytest.raises(TypeError):
+        x.transform([2, 3], 999)
+
+
+def test_reverse(pvector):
+    x = pvector([1, 2, 5])
+
+    assert list(reversed(x)) == [5, 2, 1]
+
+
+def test_contains(pvector):
+    x = pvector([1, 2, 5])
+
+    assert 2 in x
+    assert 3 not in x
+
+
+def test_index(pvector):
+    x = pvector([1, 2, 5])
+
+    assert x.index(5) == 2
+
+
+def test_index_not_found(pvector):
+    x = pvector([1, 2, 5])
+
+    with pytest.raises(ValueError):
+        x.index(7)
+
+
+def test_index_not_found_with_limits(pvector):
+    x = pvector([1, 2, 5, 1])
+
+    with pytest.raises(ValueError):
+        x.index(1, 1, 3)
+
+
+def test_count(pvector):
+    x = pvector([1, 2, 5, 1])
+
+    assert x.count(1) == 2
+    assert x.count(4) == 0
+
+
+def test_empty_truthiness(pvector):
+    assert pvector([1])
+    assert not pvector([])
+
+
+def test_pickling_empty_vector(pvector):
+    assert pickle.loads(pickle.dumps(pvector(), -1)) == pvector()
+
+
+def test_pickling_non_empty_vector(pvector):
+    assert pickle.loads(pickle.dumps(pvector([1, 'a']), -1)) == pvector([1, 'a'])
+
+
+def test_mset_basic_assignments(pvector):
+    v1 = pvector(range(2000))
+    v2 = v1.mset(1, -1, 505, -505, 1998, -1998)
+
+    # Original not changed
+    assert v1[1] == 1
+    assert v1[505] == 505
+    assert v1[1998] == 1998
+
+    # Other updated
+    assert v2[1] == -1
+    assert v2[505] == -505
+    assert v2[1998] == -1998
+
+
+def test_mset_odd_number_of_arguments(pvector):
+    v = pvector([0, 1])
+
+    with pytest.raises(TypeError):
+        v.mset(0, 10, 1)
+
+
+def test_mset_index_out_of_range(pvector):
+    v = pvector([0, 1])
+
+    with pytest.raises(IndexError):
+        v.mset(3, 10)
+
+
+def test_evolver_no_update(pvector):
+    # This is mostly a test against memory leaks in the C implementation
+    v = pvector(range(40))
+
+    assert v.evolver().persistent() == v
+
+
+def test_evolver_deallocate_dirty_evolver(pvector):
+    # Ref count handling in native implementation
+    v = pvector(range(3220))
+    e = v.evolver()
+    e[10] = -10
+    e[3220] = -3220
+
+
+def test_evolver_simple_update_in_tree(pvector):
+    v = pvector(range(35))
+    e = v.evolver()
+    e[10] = -10
+
+    assert e[10] == -10
+    assert e.persistent()[10] == -10
+
+
+def test_evolver_set_out_of_range(pvector):
+    v = pvector([0])
+    e = v.evolver()
+    with pytest.raises(IndexError) as error:
+        e[10] = 1
+    assert str(error.value) == "Index out of range: 10"
+
+def test_evolver_multi_level_multi_update_in_tree(pvector):
+    # This test is mostly to detect memory/ref count issues in the native implementation
+    v = pvector(range(3500))
+    e = v.evolver()
+
+    # Update differs between first and second time since the
+    # corresponding node will be marked as dirty the first time only.
+    e[10] = -10
+    e[11] = -11
+    e[10] = -1000
+
+    # Update in neighbour node
+    e[50] = -50
+    e[50] = -5000
+
+    # Update in node in other half of vector
+    e[3000] = -3000
+    e[3000] = -30000
+
+    # Before freezing
+    assert e[10] == -1000
+    assert e[11] == -11
+    assert e[50] == -5000
+    assert e[3000] == -30000
+
+    # Run a GC to provoke any potential misbehavior
+    import gc
+    gc.collect()
+
+    v2 = e.persistent()
+    assert v2[10] == -1000
+    assert v2[50] == -5000
+    assert v2[3000] == -30000
+
+    # Run a GC to provoke any potential misbehavior
+    gc.collect()
+
+    # After freezing
+    assert e[10] == -1000
+    assert e[11] == -11
+    assert e[50] == -5000
+    assert e[3000] == -30000
+
+    # Original stays the same
+    assert v[10] == 10
+    assert v[50] == 50
+    assert v[3000] == 3000
+
+
+def test_evolver_simple_update_in_tail(pvector):
+    v = pvector(range(35))
+    e = v.evolver()
+    e[33] = -33
+
+    assert e[33] == -33
+    assert e.persistent()[33] == -33
+    assert v[33] == 33
+
+
+def test_evolver_simple_update_just_outside_vector(pvector):
+    v = pvector()
+    e = v.evolver()
+    e[0] = 1
+
+    assert e[0] == 1
+    assert e.persistent()[0] == 1
+    assert len(v) == 0
+
+
+def test_evolver_append(pvector):
+    v = pvector()
+    e = v.evolver()
+    e.append(1000)
+    assert e[0] == 1000
+
+    e[0] = 2000
+    assert e[0] == 2000
+    assert list(e.persistent()) == [2000]
+    assert list(v) == []
+
+
+def test_evolver_extend(pvector):
+    v = pvector([1000])
+    e = v.evolver()
+    e.extend([2000, 3000])
+    e[2] = 20000
+
+    assert list(e.persistent()) == [1000, 2000, 20000]
+    assert list(v) == [1000]
+
+
+def test_evolver_assign_and_read_with_negative_indices(pvector):
+    v = pvector([1, 2, 3])
+    e = v.evolver()
+    e[-1] = 4
+    e.extend([11, 12, 13])
+    e[-1] = 33
+
+    assert e[-1] == 33
+    assert list(e.persistent()) == [1, 2, 4, 11, 12, 33]
+
+
+def test_evolver_non_integral_access(pvector):
+    e = pvector([1]).evolver()
+
+    with pytest.raises(TypeError):
+        x = e['foo']
+
+
+def test_evolver_non_integral_assignment(pvector):
+    e = pvector([1]).evolver()
+
+    with pytest.raises(TypeError):
+        e['foo'] = 1
+
+
+def test_evolver_out_of_bounds_access(pvector):
+    e = pvector([1]).evolver()
+
+    with pytest.raises(IndexError):
+        x = e[1]
+
+
+def test_evolver_out_of_bounds_assignment(pvector):
+    e = pvector([1]).evolver()
+
+    with pytest.raises(IndexError):
+        e[2] = 1
+
+
+def test_no_dependencies_between_evolvers_from_the_same_pvector(pvector):
+    original_list = list(range(40))
+    v = pvector(original_list)
+    e1 = v.evolver()
+    e2 = v.evolver()
+
+    e1.extend([1, 2, 3])
+    e1[2] = 20
+    e1[35] = 350
+
+    e2.extend([-1, -2, -3])
+    e2[2] = -20
+    e2[35] = -350
+
+    e1_expected = original_list + [1, 2, 3]
+    e1_expected[2] = 20
+    e1_expected[35] = 350
+    assert list(e1.persistent()) == e1_expected
+
+    e2_expected = original_list + [-1, -2, -3]
+    e2_expected[2] = -20
+    e2_expected[35] = -350
+    assert list(e2.persistent()) == e2_expected
+
+
+def test_pvectors_produced_from_the_same_evolver_do_not_interfere(pvector):
+    original_list = list(range(40))
+    v = pvector(original_list)
+    e = v.evolver()
+
+    e.extend([1, 2, 3])
+    e[2] = 20
+    e[35] = 350
+
+    v1 = e.persistent()
+    v1_expected = original_list + [1, 2, 3]
+    v1_expected[2] = 20
+    v1_expected[35] = 350
+
+    e.extend([-1, -2, -3])
+    e[3] = -30
+    e[36] = -360
+
+    v2 = e.persistent()
+    v2_expected = v1_expected + [-1, -2, -3]
+    v2_expected[3] = -30
+    v2_expected[36] = -360
+
+    assert list(v1) == v1_expected
+    assert list(v2) == v2_expected
+
+
+def test_evolver_len(pvector):
+    e = pvector([1, 2, 3]).evolver()
+    e.extend([4, 5])
+
+    assert len(e) == 5
+
+
+def test_evolver_is_dirty(pvector):
+    e = pvector([1, 2, 3]).evolver()
+    assert not e.is_dirty()
+
+    e.append(4)
+    assert e.is_dirty
+
+    e.persistent()
+    assert not e.is_dirty()
+
+    e[2] = 2000
+    assert e.is_dirty
+
+    e.persistent()
+    assert not e.is_dirty()
+
+
+def test_vector_insert_one_step_beyond_end(pvector):
+    # This test exists to get the transform functionality under memory
+    # leak supervision. Most of the transformation tests are in test_transform.py.
+    v = pvector([1, 2])
+    assert v.transform([2], 3) == pvector([1, 2, 3])
+
+
+def test_evolver_with_no_updates_returns_same_pvector(pvector):
+    v = pvector([1, 2])
+    assert v.evolver().persistent() is v
+
+
+def test_evolver_returns_itself_on_evolving_operations(pvector):
+    # Does this to be able to chain operations
+    v = pvector([1, 2])
+    assert v.evolver().append(3).extend([4, 5]).set(1, 6).persistent() == pvector([1, 6, 3, 4, 5])
+
+
+def test_evolver_delete_by_index(pvector):
+    e = pvector([1, 2, 3]).evolver()
+
+    del e[0]
+
+    assert e.persistent() == python_pvector([2, 3])
+    assert e.append(4).persistent() == python_pvector([2, 3, 4])
+
+
+def test_evolver_delete_function_by_index(pvector):
+    e = pvector([1, 2, 3]).evolver()
+
+    assert e.delete(1).persistent() == python_pvector([1, 3])
+
+
+def test_evolver_delete_function_by_index_multiple_times(pvector):
+    SIZE = 40
+    e = pvector(range(SIZE)).evolver()
+    for i in range(SIZE):
+        assert e[0] == i
+        assert list(e.persistent()) == list(range(i, SIZE))
+        del e[0]
+
+    assert e.persistent() == list()
+
+
+def test_evolver_delete_function_invalid_index(pvector):
+    e = pvector([1, 2]).evolver()
+
+    with pytest.raises(TypeError):
+        del e["e"]
+
+
+def test_delete_of_non_existing_element(pvector):
+    e = pvector([1, 2]).evolver()
+
+    with pytest.raises(IndexError):
+        del e[2]
+
+    del e[0]
+    del e[0]
+
+    with pytest.raises(IndexError):
+        del e[0]
+
+    assert e.persistent() == pvector()
+
+
+def test_append_followed_by_delete(pvector):
+    e = pvector([1, 2]).evolver()
+
+    e.append(3)
+
+    del e[2]
+
+
+def test_evolver_set_followed_by_delete(pvector):
+    evolver = pvector([1, 2]).evolver()
+    evolver[1] = 3
+
+    assert [evolver[i] for i in range(len(evolver))] == [1, 3]
+
+    del evolver[0]
+
+    assert evolver.persistent() == pvector([3])
+
+
+def test_compare_with_list(pvector):
+    v = pvector([1, 2, 3])
+
+    assert v == [1, 2, 3]
+    assert v != [1, 2]
+    assert v > [1, 2]
+    assert v < [2, 2]
+    assert [1, 2] < v
+    assert v <= [1, 2, 3]
+    assert v <= [1, 2, 4]
+    assert v >= [1, 2, 3]
+    assert v >= [1, 2]
+
+
+def test_compare_with_non_iterable(pvector):
+    assert pvector([1, 2, 3]) != 5
+    assert not (pvector([1, 2, 3]) == 5)
+
+
+def test_python_no_c_extension_with_environment_variable():
+    from six.moves import reload_module
+    import pyrsistent._pvector
+    import pyrsistent
+    import os
+
+    os.environ['PYRSISTENT_NO_C_EXTENSION'] = 'TRUE'
+
+    reload_module(pyrsistent._pvector)
+    reload_module(pyrsistent)
+
+    assert type(pyrsistent.pvector()) is pyrsistent._pvector.PythonPVector
+
+    del os.environ['PYRSISTENT_NO_C_EXTENSION']
+
+    reload_module(pyrsistent._pvector)
+    reload_module(pyrsistent)
+
+
+def test_supports_weakref(pvector):
+    import weakref
+    weakref.ref(pvector())
+
+def test_get_evolver_referents(pvector):
+    """The C implementation of the evolver should expose the original PVector
+    to the gc only once.
+    """
+    if pvector.__module__ == 'pyrsistent._pvector':
+        pytest.skip("This test only applies to pvectorc")
+    import gc
+    v = pvector([1, 2, 3])
+    e = v.evolver()
+    assert len([x for x in gc.get_referents(e) if x is v]) == 1
+
+
+def test_failing_repr(pvector):
+    # See https://github.com/tobgu/pyrsistent/issues/84
+    class A(object):
+        def __repr__(self):
+            raise ValueError('oh no!')
+
+    with pytest.raises(ValueError):
+        repr(pvector([A()]))
+
+
+def test_iterable(pvector):
+    """
+    PVectors can be created from iterables even though they can't be len()
+    hinted.
+    """
+
+    assert pvector(iter("a")) == pvector(iter("a"))
diff --git a/contrib/python/pyrsistent/py2/tests/ya.make b/contrib/python/pyrsistent/py2/tests/ya.make
new file mode 100644
index 00000000000..22cadc93e09
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/tests/ya.make
@@ -0,0 +1,27 @@
+PY2TEST()
+
+PEERDIR(
+    contrib/python/pyrsistent
+)
+
+TEST_SRCS(
+    bag_test.py
+    checked_map_test.py
+    checked_set_test.py
+    checked_vector_test.py
+    class_test.py
+    deque_test.py
+    field_test.py
+    freeze_test.py
+    immutable_object_test.py
+    list_test.py
+    map_test.py
+    record_test.py
+    regression_test.py
+    set_test.py
+    toolz_test.py
+)
+
+NO_LINT()
+
+END()
diff --git a/contrib/python/pyrsistent/py2/ya.make b/contrib/python/pyrsistent/py2/ya.make
new file mode 100644
index 00000000000..4563cedf9d9
--- /dev/null
+++ b/contrib/python/pyrsistent/py2/ya.make
@@ -0,0 +1,52 @@
+# Generated by devtools/yamaker (pypi).
+
+PY2_LIBRARY()
+
+PROVIDES(python_pyrsistent)
+
+VERSION(0.15.7)
+
+LICENSE(MIT)
+
+PEERDIR(
+    contrib/python/six
+)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    _pyrsistent_version.py
+    pyrsistent/__init__.py
+    pyrsistent/__init__.pyi
+    pyrsistent/_checked_types.py
+    pyrsistent/_compat.py
+    pyrsistent/_field_common.py
+    pyrsistent/_helpers.py
+    pyrsistent/_immutable.py
+    pyrsistent/_pbag.py
+    pyrsistent/_pclass.py
+    pyrsistent/_pdeque.py
+    pyrsistent/_plist.py
+    pyrsistent/_pmap.py
+    pyrsistent/_precord.py
+    pyrsistent/_pset.py
+    pyrsistent/_pvector.py
+    pyrsistent/_toolz.py
+    pyrsistent/_transformations.py
+    pyrsistent/typing.py
+    pyrsistent/typing.pyi
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/pyrsistent/py2/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+    pyrsistent/py.typed
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/pyrsistent/py3/.dist-info/METADATA b/contrib/python/pyrsistent/py3/.dist-info/METADATA
new file mode 100644
index 00000000000..1ce3d375fd3
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/.dist-info/METADATA
@@ -0,0 +1,789 @@
+Metadata-Version: 2.1
+Name: pyrsistent
+Version: 0.20.0
+Summary: Persistent/Functional/Immutable data structures
+Home-page: https://github.com/tobgu/pyrsistent/
+Author: Tobias Gustafsson
+Author-email: tobias.l.gustafsson@gmail.com
+License: MIT
+Project-URL: Changelog, https://pyrsistent.readthedocs.io/en/latest/changes.html
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.8
+Description-Content-Type: text/x-rst
+License-File: LICENSE.mit
+
+Pyrsistent
+==========
+.. image:: https://github.com/tobgu/pyrsistent/actions/workflows/tests.yaml/badge.svg
+    :target: https://github.com/tobgu/pyrsistent/actions/workflows/tests.yaml
+
+
+.. _Pyrthon: https://www.github.com/tobgu/pyrthon
+.. _Pyrsistent_extras: https://github.com/mingmingrr/pyrsistent-extras
+
+Pyrsistent is a number of persistent collections (by some referred to as functional data structures). Persistent in
+the sense that they are immutable.
+
+All methods on a data structure that would normally mutate it instead return a new copy of the structure containing the
+requested updates. The original structure is left untouched.
+
+This will simplify the reasoning about what a program does since no hidden side effects ever can take place to these
+data structures. You can rest assured that the object you hold a reference to will remain the same throughout its
+lifetime and need not worry that somewhere five stack levels below you in the darkest corner of your application
+someone has decided to remove that element that you expected to be there.
+
+Pyrsistent is influenced by persistent data structures such as those found in the standard library of Clojure. The
+data structures are designed to share common elements through path copying.
+It aims at taking these concepts and make them as pythonic as possible so that they can be easily integrated into any python
+program without hassle.
+
+If you want use literal syntax to define them in your code rather
+than function calls check out Pyrthon_. Be aware, that one is experimental, unmaintained and alpha software. 
+
+If you cannot find the persistent data structure you're looking for here you may want to take a look at
+Pyrsistent_extras_ which is maintained by @mingmingrr. If you still don't find what you're looking for please
+open an issue for discussion. If we agree that functionality is missing you may want to go ahead and create
+a Pull Request implement the missing functionality.
+
+Examples
+--------
+.. _Sequence: collections_
+.. _Hashable: collections_
+.. _Mapping: collections_
+.. _Mappings: collections_
+.. _Set: collections_
+.. _collections: https://docs.python.org/3/library/collections.abc.html
+.. _documentation: http://pyrsistent.readthedocs.org/
+
+The collection types and key features currently implemented are:
+
+* PVector_, similar to a python list
+* PMap_, similar to dict
+* PSet_, similar to set
+* PRecord_, a PMap on steroids with fixed fields, optional type and invariant checking and much more
+* PClass_, a Python class fixed fields, optional type and invariant checking and much more
+* `Checked collections`_, PVector, PMap and PSet with optional type and invariance checks and more
+* PBag, similar to collections.Counter
+* PList, a classic singly linked list
+* PDeque, similar to collections.deque
+* Immutable object type (immutable) built on the named tuple
+* freeze_ and thaw_ functions to convert between pythons standard collections and pyrsistent collections.
+* Flexible transformations_ of arbitrarily complex structures built from PMaps and PVectors.
+
+Below are examples of common usage patterns for some of the structures and features. More information and
+full documentation for all data structures is available in the documentation_.
+
+.. _PVector:
+
+PVector
+~~~~~~~
+With full support for the Sequence_ protocol PVector is meant as a drop in replacement to the built in list from a readers
+point of view. Write operations of course differ since no in place mutation is done but naming should be in line
+with corresponding operations on the built in list.
+
+Support for the Hashable_ protocol also means that it can be used as key in Mappings_.
+
+Appends are amortized O(1). Random access and insert is log32(n) where n is the size of the vector.
+
+.. code:: python
+
+    >>> from pyrsistent import v, pvector
+
+    # No mutation of vectors once created, instead they
+    # are "evolved" leaving the original untouched
+    >>> v1 = v(1, 2, 3)
+    >>> v2 = v1.append(4)
+    >>> v3 = v2.set(1, 5)
+    >>> v1
+    pvector([1, 2, 3])
+    >>> v2
+    pvector([1, 2, 3, 4])
+    >>> v3
+    pvector([1, 5, 3, 4])
+
+    # Random access and slicing
+    >>> v3[1]
+    5
+    >>> v3[1:3]
+    pvector([5, 3])
+
+    # Iteration
+    >>> list(x + 1 for x in v3)
+    [2, 6, 4, 5]
+    >>> pvector(2 * x for x in range(3))
+    pvector([0, 2, 4])
+
+.. _PMap:
+
+PMap
+~~~~
+With full support for the Mapping_ protocol PMap is meant as a drop in replacement to the built in dict from a readers point
+of view. Support for the Hashable_ protocol also means that it can be used as key in other Mappings_.
+
+Random access and insert is log32(n) where n is the size of the map.
+
+.. code:: python
+
+    >>> from pyrsistent import m, pmap, v
+
+    # No mutation of maps once created, instead they are
+    # "evolved" leaving the original untouched
+    >>> m1 = m(a=1, b=2)
+    >>> m2 = m1.set('c', 3)
+    >>> m3 = m2.set('a', 5)
+    >>> m1
+    pmap({'a': 1, 'b': 2})
+    >>> m2
+    pmap({'a': 1, 'c': 3, 'b': 2})
+    >>> m3
+    pmap({'a': 5, 'c': 3, 'b': 2})
+    >>> m3['a']
+    5
+
+    # Evolution of nested persistent structures
+    >>> m4 = m(a=5, b=6, c=v(1, 2))
+    >>> m4.transform(('c', 1), 17)
+    pmap({'a': 5, 'c': pvector([1, 17]), 'b': 6})
+    >>> m5 = m(a=1, b=2)
+
+    # Evolve by merging with other mappings
+    >>> m5.update(m(a=2, c=3), {'a': 17, 'd': 35})
+    pmap({'a': 17, 'c': 3, 'b': 2, 'd': 35})
+    >>> pmap({'x': 1, 'y': 2}) + pmap({'y': 3, 'z': 4})
+    pmap({'y': 3, 'x': 1, 'z': 4})
+
+    # Dict-like methods to convert to list and iterate
+    >>> m3.items()
+    pvector([('a', 5), ('c', 3), ('b', 2)])
+    >>> list(m3)
+    ['a', 'c', 'b']
+
+.. _PSet:
+
+PSet
+~~~~
+With full support for the Set_ protocol PSet is meant as a drop in replacement to the built in set from a readers point
+of view. Support for the Hashable_ protocol also means that it can be used as key in Mappings_.
+
+Random access and insert is log32(n) where n is the size of the set.
+
+.. code:: python
+
+    >>> from pyrsistent import s
+
+    # No mutation of sets once created, you know the story...
+    >>> s1 = s(1, 2, 3, 2)
+    >>> s2 = s1.add(4)
+    >>> s3 = s1.remove(1)
+    >>> s1
+    pset([1, 2, 3])
+    >>> s2
+    pset([1, 2, 3, 4])
+    >>> s3
+    pset([2, 3])
+
+    # Full support for set operations
+    >>> s1 | s(3, 4, 5)
+    pset([1, 2, 3, 4, 5])
+    >>> s1 & s(3, 4, 5)
+    pset([3])
+    >>> s1 < s2
+    True
+    >>> s1 < s(3, 4, 5)
+    False
+
+.. _PRecord:
+
+PRecord
+~~~~~~~
+A PRecord is a PMap with a fixed set of specified fields. Records are declared as python classes inheriting
+from PRecord. Because it is a PMap it has full support for all Mapping methods such as iteration and element
+access using subscript notation.
+
+.. code:: python
+
+    >>> from pyrsistent import PRecord, field
+    >>> class ARecord(PRecord):
+    ...     x = field()
+    ...
+    >>> r = ARecord(x=3)
+    >>> r
+    ARecord(x=3)
+    >>> r.x
+    3
+    >>> r.set(x=2)
+    ARecord(x=2)
+    >>> r.set(y=2)
+    Traceback (most recent call last):
+    AttributeError: 'y' is not among the specified fields for ARecord
+
+Type information
+****************
+It is possible to add type information to the record to enforce type checks. Multiple allowed types can be specified
+by providing an iterable of types.
+
+.. code:: python
+
+    >>> class BRecord(PRecord):
+    ...     x = field(type=int)
+    ...     y = field(type=(int, type(None)))
+    ...
+    >>> BRecord(x=3, y=None)
+    BRecord(y=None, x=3)
+    >>> BRecord(x=3.0)
+    Traceback (most recent call last):
+    PTypeError: Invalid type for field BRecord.x, was float
+
+
+Custom types (classes) that are iterable should be wrapped in a tuple to prevent their
+members being added to the set of valid types.  Although Enums in particular are now
+supported without wrapping, see #83 for more information.
+
+Mandatory fields
+****************
+Fields are not mandatory by default but can be specified as such. If fields are missing an
+*InvariantException* will be thrown which contains information about the missing fields.
+
+.. code:: python
+
+    >>> from pyrsistent import InvariantException
+    >>> class CRecord(PRecord):
+    ...     x = field(mandatory=True)
+    ...
+    >>> r = CRecord(x=3)
+    >>> try:
+    ...    r.discard('x')
+    ... except InvariantException as e:
+    ...    print(e.missing_fields)
+    ...
+    ('CRecord.x',)
+
+Invariants
+**********
+It is possible to add invariants that must hold when evolving the record. Invariants can be
+specified on both field and record level. If invariants fail an *InvariantException* will be
+thrown which contains information about the failing invariants. An invariant function should
+return a tuple consisting of a boolean that tells if the invariant holds or not and an object
+describing the invariant. This object can later be used to identify which invariant that failed.
+
+The global invariant function is only executed if all field invariants hold.
+
+Global invariants are inherited to subclasses.
+
+.. code:: python
+
+    >>> class RestrictedVector(PRecord):
+    ...     __invariant__ = lambda r: (r.y >= r.x, 'x larger than y')
+    ...     x = field(invariant=lambda x: (x > 0, 'x negative'))
+    ...     y = field(invariant=lambda y: (y > 0, 'y negative'))
+    ...
+    >>> r = RestrictedVector(y=3, x=2)
+    >>> try:
+    ...    r.set(x=-1, y=-2)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    ('y negative', 'x negative')
+    >>> try:
+    ...    r.set(x=2, y=1)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    ('x larger than y',)
+
+Invariants may also contain multiple assertions. For those cases the invariant function should
+return a tuple of invariant tuples as described above. This structure is reflected in the
+invariant_errors attribute of the exception which will contain tuples with data from all failed
+invariants. Eg:
+
+.. code:: python
+
+    >>> class EvenX(PRecord):
+    ...     x = field(invariant=lambda x: ((x > 0, 'x negative'), (x % 2 == 0, 'x odd')))
+    ...
+    >>> try:
+    ...    EvenX(x=-1)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    (('x negative', 'x odd'),)
+
+
+Factories
+*********
+It's possible to specify factory functions for fields. The factory function receives whatever
+is supplied as field value and the actual returned by the factory is assigned to the field
+given that any type and invariant checks hold.
+PRecords have a default factory specified as a static function on the class, create(). It takes
+a *Mapping* as argument and returns an instance of the specific record.
+If a record has fields of type PRecord the create() method of that record will
+be called to create the "sub record" if no factory has explicitly been specified to override
+this behaviour.
+
+.. code:: python
+
+    >>> class DRecord(PRecord):
+    ...     x = field(factory=int)
+    ...
+    >>> class ERecord(PRecord):
+    ...     d = field(type=DRecord)
+    ...
+    >>> ERecord.create({'d': {'x': '1'}})
+    ERecord(d=DRecord(x=1))
+
+Collection fields
+*****************
+It is also possible to have fields with ``pyrsistent`` collections.
+
+.. code:: python
+
+   >>> from pyrsistent import pset_field, pmap_field, pvector_field
+   >>> class MultiRecord(PRecord):
+   ...     set_of_ints = pset_field(int)
+   ...     map_int_to_str = pmap_field(int, str)
+   ...     vector_of_strs = pvector_field(str)
+   ...
+
+Serialization
+*************
+PRecords support serialization back to dicts. Default serialization will take keys and values
+"as is" and output them into a dict. It is possible to specify custom serialization functions
+to take care of fields that require special treatment.
+
+.. code:: python
+
+    >>> from datetime import date
+    >>> class Person(PRecord):
+    ...     name = field(type=unicode)
+    ...     birth_date = field(type=date,
+    ...                        serializer=lambda format, d: d.strftime(format['date']))
+    ...
+    >>> john = Person(name=u'John', birth_date=date(1985, 10, 21))
+    >>> john.serialize({'date': '%Y-%m-%d'})
+    {'birth_date': '1985-10-21', 'name': u'John'}
+
+
+.. _instar: https://github.com/boxed/instar/
+
+.. _PClass:
+
+PClass
+~~~~~~
+A PClass is a python class with a fixed set of specified fields. PClasses are declared as python classes inheriting
+from PClass. It is defined the same way that PRecords are and behaves like a PRecord in all aspects except that it
+is not a PMap and hence not a collection but rather a plain Python object.
+
+.. code:: python
+
+    >>> from pyrsistent import PClass, field
+    >>> class AClass(PClass):
+    ...     x = field()
+    ...
+    >>> a = AClass(x=3)
+    >>> a
+    AClass(x=3)
+    >>> a.x
+    3
+
+
+Checked collections
+~~~~~~~~~~~~~~~~~~~
+Checked collections currently come in three flavors: CheckedPVector, CheckedPMap and CheckedPSet.
+
+.. code:: python
+
+    >>> from pyrsistent import CheckedPVector, CheckedPMap, CheckedPSet, thaw
+    >>> class Positives(CheckedPSet):
+    ...     __type__ = (long, int)
+    ...     __invariant__ = lambda n: (n >= 0, 'Negative')
+    ...
+    >>> class Lottery(PRecord):
+    ...     name = field(type=str)
+    ...     numbers = field(type=Positives, invariant=lambda p: (len(p) > 0, 'No numbers'))
+    ...
+    >>> class Lotteries(CheckedPVector):
+    ...     __type__ = Lottery
+    ...
+    >>> class LotteriesByDate(CheckedPMap):
+    ...     __key_type__ = date
+    ...     __value_type__ = Lotteries
+    ...
+    >>> lotteries = LotteriesByDate.create({date(2015, 2, 15): [{'name': 'SuperLotto', 'numbers': {1, 2, 3}},
+    ...                                                         {'name': 'MegaLotto',  'numbers': {4, 5, 6}}],
+    ...                                     date(2015, 2, 16): [{'name': 'SuperLotto', 'numbers': {3, 2, 1}},
+    ...                                                         {'name': 'MegaLotto',  'numbers': {6, 5, 4}}]})
+    >>> lotteries
+    LotteriesByDate({datetime.date(2015, 2, 15): Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')]), datetime.date(2015, 2, 16): Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')])})
+
+    # The checked versions support all operations that the corresponding
+    # unchecked types do
+    >>> lottery_0215 = lotteries[date(2015, 2, 15)]
+    >>> lottery_0215.transform([0, 'name'], 'SuperDuperLotto')
+    Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperDuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')])
+
+    # But also makes asserts that types and invariants hold
+    >>> lottery_0215.transform([0, 'name'], 999)
+    Traceback (most recent call last):
+    PTypeError: Invalid type for field Lottery.name, was int
+
+    >>> lottery_0215.transform([0, 'numbers'], set())
+    Traceback (most recent call last):
+    InvariantException: Field invariant failed
+
+    # They can be converted back to python built ins with either thaw()
+    # or serialize() (which provides possibilities to customize serialization)
+    >>> thaw(lottery_0215)
+    [{'numbers': set([1, 2, 3]), 'name': 'SuperLotto'}, {'numbers': set([4, 5, 6]), 'name': 'MegaLotto'}]
+    >>> lottery_0215.serialize()
+    [{'numbers': set([1, 2, 3]), 'name': 'SuperLotto'}, {'numbers': set([4, 5, 6]), 'name': 'MegaLotto'}]
+
+.. _transformations:
+
+Transformations
+~~~~~~~~~~~~~~~
+Transformations are inspired by the cool library instar_ for Clojure. They let you evolve PMaps and PVectors
+with arbitrarily deep/complex nesting using simple syntax and flexible matching syntax.
+
+The first argument to transformation is the path that points out the value to transform. The
+second is the transformation to perform. If the transformation is callable it will be applied
+to the value(s) matching the path. The path may also contain callables. In that case they are
+treated as matchers. If the matcher returns True for a specific key it is considered for transformation.
+
+.. code:: python
+
+    # Basic examples
+    >>> from pyrsistent import inc, freeze, thaw, rex, ny, discard
+    >>> v1 = freeze([1, 2, 3, 4, 5])
+    >>> v1.transform([2], inc)
+    pvector([1, 2, 4, 4, 5])
+    >>> v1.transform([lambda ix: 0 < ix < 4], 8)
+    pvector([1, 8, 8, 8, 5])
+    >>> v1.transform([lambda ix, v: ix == 0 or v == 5], 0)
+    pvector([0, 2, 3, 4, 0])
+
+    # The (a)ny matcher can be used to match anything
+    >>> v1.transform([ny], 8)
+    pvector([8, 8, 8, 8, 8])
+
+    # Regular expressions can be used for matching
+    >>> scores = freeze({'John': 12, 'Joseph': 34, 'Sara': 23})
+    >>> scores.transform([rex('^Jo')], 0)
+    pmap({'Joseph': 0, 'Sara': 23, 'John': 0})
+
+    # Transformations can be done on arbitrarily deep structures
+    >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
+    ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
+    ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
+    >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
+    >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
+    >>> very_short_news.articles[0].content
+    'A short article'
+    >>> very_short_news.articles[1].content
+    'A slightly long...'
+
+    # When nothing has been transformed the original data structure is kept
+    >>> short_news is news_paper
+    True
+    >>> very_short_news is news_paper
+    False
+    >>> very_short_news.articles[0] is news_paper.articles[0]
+    True
+
+    # There is a special transformation that can be used to discard elements. Also
+    # multiple transformations can be applied in one call
+    >>> thaw(news_paper.transform(['weather'], discard, ['articles', ny, 'content'], discard))
+    {'articles': [{'author': 'Sara'}, {'author': 'Steve'}]}
+
+Evolvers
+~~~~~~~~
+PVector, PMap and PSet all have support for a concept dubbed *evolvers*. An evolver acts like a mutable
+view of the underlying persistent data structure with "transaction like" semantics. No updates of the original
+data structure is ever performed, it is still fully immutable.
+
+The evolvers have a very limited API by design to discourage excessive, and inappropriate, usage as that would
+take us down the mutable road. In principle only basic mutation and element access functions are supported.
+Check out the documentation_ of each data structure for specific examples.
+
+Examples of when you may want to use an evolver instead of working directly with the data structure include:
+
+* Multiple updates are done to the same data structure and the intermediate results are of no
+  interest. In this case using an evolver may be a more efficient and easier to work with.
+* You need to pass a vector into a legacy function or a function that you have no control
+  over which performs in place mutations. In this case pass an evolver instance
+  instead and then create a new pvector from the evolver once the function returns.
+
+.. code:: python
+
+    >>> from pyrsistent import v
+
+    # In place mutation as when working with the built in counterpart
+    >>> v1 = v(1, 2, 3)
+    >>> e = v1.evolver()
+    >>> e[1] = 22
+    >>> e = e.append(4)
+    >>> e = e.extend([5, 6])
+    >>> e[5] += 1
+    >>> len(e)
+    6
+
+    # The evolver is considered *dirty* when it contains changes compared to the underlying vector
+    >>> e.is_dirty()
+    True
+
+    # But the underlying pvector still remains untouched
+    >>> v1
+    pvector([1, 2, 3])
+
+    # Once satisfied with the updates you can produce a new pvector containing the updates.
+    # The new pvector will share data with the original pvector in the same way that would have
+    # been done if only using operations on the pvector.
+    >>> v2 = e.persistent()
+    >>> v2
+    pvector([1, 22, 3, 4, 5, 7])
+
+    # The evolver is now no longer considered *dirty* as it contains no differences compared to the
+    # pvector just produced.
+    >>> e.is_dirty()
+    False
+
+    # You may continue to work with the same evolver without affecting the content of v2
+    >>> e[0] = 11
+
+    # Or create a new evolver from v2. The two evolvers can be updated independently but will both
+    # share data with v2 where possible.
+    >>> e2 = v2.evolver()
+    >>> e2[0] = 1111
+    >>> e.persistent()
+    pvector([11, 22, 3, 4, 5, 7])
+    >>> e2.persistent()
+    pvector([1111, 22, 3, 4, 5, 7])
+
+.. _freeze:
+.. _thaw:
+
+freeze and thaw
+~~~~~~~~~~~~~~~
+These functions are great when your cozy immutable world has to interact with the evil mutable world outside.
+
+.. code:: python
+
+    >>> from pyrsistent import freeze, thaw, v, m
+    >>> freeze([1, {'a': 3}])
+    pvector([1, pmap({'a': 3})])
+    >>> thaw(v(1, m(a=3)))
+    [1, {'a': 3}]
+
+By default, freeze will also recursively convert values inside PVectors and PMaps. This behaviour can be changed by providing freeze with the flag strict=False.
+
+.. code:: python
+
+    >>> from pyrsistent import freeze, v, m
+    >>> freeze(v(1, v(2, [3])))
+    pvector([1, pvector([2, pvector([3])])])
+    >>> freeze(v(1, v(2, [3])), strict=False)
+    pvector([1, pvector([2, [3]])])
+    >>> freeze(m(a=m(b={'c': 1})))
+    pmap({'a': pmap({'b': pmap({'c': 1})})})
+    >>> freeze(m(a=m(b={'c': 1})), strict=False)
+    pmap({'a': pmap({'b': {'c': 1}})})
+
+In this regard, thaw operates as the inverse of freeze so will thaw values inside native data structures unless passed the strict=False flag.
+
+
+Compatibility
+-------------
+
+Pyrsistent is developed and tested on Python 3.8+ and PyPy3.
+
+Performance
+-----------
+
+Pyrsistent is developed with performance in mind. Still, while some operations are nearly on par with their built in,
+mutable, counterparts in terms of speed, other operations are slower. In the cases where attempts at
+optimizations have been done, speed has generally been valued over space.
+
+Pyrsistent comes with two API compatible flavors of PVector (on which PMap and PSet are based), one pure Python
+implementation and one implemented as a C extension. The latter generally being 2 - 20 times faster than the former.
+The C extension will be used automatically when possible.
+
+The pure python implementation is fully PyPy compatible. Running it under PyPy speeds operations up considerably if
+the structures are used heavily (if JITed), for some cases the performance is almost on par with the built in counterparts.
+
+Type hints
+----------
+
+PEP 561 style type hints for use with mypy and various editors are available for most types and functions in pyrsistent.
+
+Type classes for annotating your own code with pyrsistent types are also available under pyrsistent.typing.
+
+Installation
+------------
+
+pip install pyrsistent
+
+Documentation
+-------------
+
+Available at http://pyrsistent.readthedocs.org/
+
+Brief presentation available at http://slides.com/tobiasgustafsson/immutability-and-python/
+
+Contributors
+------------
+
+Tobias Gustafsson https://github.com/tobgu
+
+Christopher Armstrong https://github.com/radix
+
+Anders Hovmöller https://github.com/boxed
+
+Itamar Turner-Trauring https://github.com/itamarst
+
+Jonathan Lange https://github.com/jml
+
+Richard Futrell https://github.com/Futrell
+
+Jakob Hollenstein https://github.com/jkbjh
+
+David Honour https://github.com/foolswood
+
+David R. MacIver https://github.com/DRMacIver
+
+Marcus Ewert https://github.com/sarum90
+
+Jean-Paul Calderone https://github.com/exarkun
+
+Douglas Treadwell https://github.com/douglas-treadwell
+
+Travis Parker https://github.com/teepark
+
+Julian Berman https://github.com/Julian
+
+Dennis Tomas https://github.com/dtomas
+
+Neil Vyas https://github.com/neilvyas
+
+doozr https://github.com/doozr
+
+Kamil Galuszka https://github.com/galuszkak
+
+Tsuyoshi Hombashi https://github.com/thombashi
+
+nattofriends https://github.com/nattofriends
+
+agberk https://github.com/agberk
+
+Waleed Khan https://github.com/arxanas
+
+Jean-Louis Fuchs https://github.com/ganwell
+
+Carlos Corbacho https://github.com/ccorbacho
+
+Felix Yan https://github.com/felixonmars
+
+benrg https://github.com/benrg
+
+Jere Lahelma https://github.com/je-l
+
+Max Taggart https://github.com/MaxTaggart
+
+Vincent Philippon https://github.com/vphilippon
+
+Semen Zhydenko https://github.com/ss18
+
+Till Varoquaux  https://github.com/till-varoquaux
+
+Michal Kowalik https://github.com/michalvi
+
+ossdev07 https://github.com/ossdev07
+
+Kerry Olesen https://github.com/qhesz
+
+johnthagen https://github.com/johnthagen
+
+Bastien Vallet https://github.com/djailla
+
+Ram Rachum  https://github.com/cool-RR
+
+Vincent Philippon https://github.com/vphilippon
+
+Andrey Bienkowski https://github.com/hexagonrecursion
+
+Ethan McCue https://github.com/bowbahdoe
+
+Jason R. Coombs https://github.com/jaraco
+
+Nathan https://github.com/ndowens
+
+Geert Barentsen https://github.com/barentsen
+
+phil-arh https://github.com/phil-arh
+
+Tamás Nepusz https://github.com/ntamas
+
+Hugo van Kemenade https://github.com/hugovk
+
+Ben Beasley https://github.com/musicinmybrain
+
+Noah C. Benson https://github.com/noahbenson
+
+dscrofts https://github.com/dscrofts
+
+Andy Reagan https://github.com/andyreagan
+
+Aaron Durant https://github.com/Aaron-Durant
+
+Joshua Munn https://github.com/jams2
+
+Lukas https://github.com/lukasK9999
+
+Arshad https://github.com/arshad-ml
+
+Contributing
+------------
+
+Want to contribute? That's great! If you experience problems please log them on GitHub. If you want to contribute code,
+please fork the repository and submit a pull request.
+
+Run tests
+~~~~~~~~~
+.. _tox: https://tox.readthedocs.io/en/latest/
+
+Tests can be executed using tox_.
+
+Install tox: ``pip install tox``
+
+Run test for Python 3.8: ``tox -e py38``
+
+Release
+~~~~~~~
+* `pip install -r requirements.txt`
+* Update CHANGES.txt
+* Update README.rst with any new contributors and potential info needed.
+* Update _pyrsistent_version.py
+* Commit and tag with new version: `git add -u . && git commit -m 'Prepare version vX.Y.Z' && git tag -a vX.Y.Z -m 'vX.Y.Z'`
+* Push commit and tags: `git push --follow-tags`
+* Build new release using Github actions
+
+Project status
+--------------
+Pyrsistent can be considered stable and mature (who knows, there may even be a 1.0 some day :-)). The project is
+maintained, bugs fixed, PRs reviewed and merged and new releases made. I currently do not have time for development
+of new features or functionality which I don't have use for myself. I'm more than happy to take PRs for new
+functionality though!
+
+There are a bunch of issues marked with ``enhancement`` and ``help wanted`` that contain requests for new functionality
+that would be nice to include. The level of difficulty and extend of the issues varies, please reach out to me if you're
+interested in working on any of them.
+
+If you feel that you have a grand master plan for where you would like Pyrsistent to go and have the time to put into
+it please don't hesitate to discuss this with me and submit PRs for it. If all goes well I'd be more than happy to add
+additional maintainers to the project!
diff --git a/contrib/python/pyrsistent/py3/.dist-info/top_level.txt b/contrib/python/pyrsistent/py3/.dist-info/top_level.txt
new file mode 100644
index 00000000000..f2460728a9d
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/.dist-info/top_level.txt
@@ -0,0 +1,3 @@
+_pyrsistent_version
+pvectorc
+pyrsistent
diff --git a/contrib/python/pyrsistent/py3/LICENSE.mit b/contrib/python/pyrsistent/py3/LICENSE.mit
new file mode 100644
index 00000000000..8a32be24498
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/LICENSE.mit
@@ -0,0 +1,22 @@
+Copyright (c) 2023 Tobias Gustafsson
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py3/README.rst b/contrib/python/pyrsistent/py3/README.rst
new file mode 100644
index 00000000000..64bb5854ca1
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/README.rst
@@ -0,0 +1,767 @@
+Pyrsistent
+==========
+.. image:: https://github.com/tobgu/pyrsistent/actions/workflows/tests.yaml/badge.svg
+    :target: https://github.com/tobgu/pyrsistent/actions/workflows/tests.yaml
+
+
+.. _Pyrthon: https://www.github.com/tobgu/pyrthon
+.. _Pyrsistent_extras: https://github.com/mingmingrr/pyrsistent-extras
+
+Pyrsistent is a number of persistent collections (by some referred to as functional data structures). Persistent in
+the sense that they are immutable.
+
+All methods on a data structure that would normally mutate it instead return a new copy of the structure containing the
+requested updates. The original structure is left untouched.
+
+This will simplify the reasoning about what a program does since no hidden side effects ever can take place to these
+data structures. You can rest assured that the object you hold a reference to will remain the same throughout its
+lifetime and need not worry that somewhere five stack levels below you in the darkest corner of your application
+someone has decided to remove that element that you expected to be there.
+
+Pyrsistent is influenced by persistent data structures such as those found in the standard library of Clojure. The
+data structures are designed to share common elements through path copying.
+It aims at taking these concepts and make them as pythonic as possible so that they can be easily integrated into any python
+program without hassle.
+
+If you want use literal syntax to define them in your code rather
+than function calls check out Pyrthon_. Be aware, that one is experimental, unmaintained and alpha software. 
+
+If you cannot find the persistent data structure you're looking for here you may want to take a look at
+Pyrsistent_extras_ which is maintained by @mingmingrr. If you still don't find what you're looking for please
+open an issue for discussion. If we agree that functionality is missing you may want to go ahead and create
+a Pull Request implement the missing functionality.
+
+Examples
+--------
+.. _Sequence: collections_
+.. _Hashable: collections_
+.. _Mapping: collections_
+.. _Mappings: collections_
+.. _Set: collections_
+.. _collections: https://docs.python.org/3/library/collections.abc.html
+.. _documentation: http://pyrsistent.readthedocs.org/
+
+The collection types and key features currently implemented are:
+
+* PVector_, similar to a python list
+* PMap_, similar to dict
+* PSet_, similar to set
+* PRecord_, a PMap on steroids with fixed fields, optional type and invariant checking and much more
+* PClass_, a Python class fixed fields, optional type and invariant checking and much more
+* `Checked collections`_, PVector, PMap and PSet with optional type and invariance checks and more
+* PBag, similar to collections.Counter
+* PList, a classic singly linked list
+* PDeque, similar to collections.deque
+* Immutable object type (immutable) built on the named tuple
+* freeze_ and thaw_ functions to convert between pythons standard collections and pyrsistent collections.
+* Flexible transformations_ of arbitrarily complex structures built from PMaps and PVectors.
+
+Below are examples of common usage patterns for some of the structures and features. More information and
+full documentation for all data structures is available in the documentation_.
+
+.. _PVector:
+
+PVector
+~~~~~~~
+With full support for the Sequence_ protocol PVector is meant as a drop in replacement to the built in list from a readers
+point of view. Write operations of course differ since no in place mutation is done but naming should be in line
+with corresponding operations on the built in list.
+
+Support for the Hashable_ protocol also means that it can be used as key in Mappings_.
+
+Appends are amortized O(1). Random access and insert is log32(n) where n is the size of the vector.
+
+.. code:: python
+
+    >>> from pyrsistent import v, pvector
+
+    # No mutation of vectors once created, instead they
+    # are "evolved" leaving the original untouched
+    >>> v1 = v(1, 2, 3)
+    >>> v2 = v1.append(4)
+    >>> v3 = v2.set(1, 5)
+    >>> v1
+    pvector([1, 2, 3])
+    >>> v2
+    pvector([1, 2, 3, 4])
+    >>> v3
+    pvector([1, 5, 3, 4])
+
+    # Random access and slicing
+    >>> v3[1]
+    5
+    >>> v3[1:3]
+    pvector([5, 3])
+
+    # Iteration
+    >>> list(x + 1 for x in v3)
+    [2, 6, 4, 5]
+    >>> pvector(2 * x for x in range(3))
+    pvector([0, 2, 4])
+
+.. _PMap:
+
+PMap
+~~~~
+With full support for the Mapping_ protocol PMap is meant as a drop in replacement to the built in dict from a readers point
+of view. Support for the Hashable_ protocol also means that it can be used as key in other Mappings_.
+
+Random access and insert is log32(n) where n is the size of the map.
+
+.. code:: python
+
+    >>> from pyrsistent import m, pmap, v
+
+    # No mutation of maps once created, instead they are
+    # "evolved" leaving the original untouched
+    >>> m1 = m(a=1, b=2)
+    >>> m2 = m1.set('c', 3)
+    >>> m3 = m2.set('a', 5)
+    >>> m1
+    pmap({'a': 1, 'b': 2})
+    >>> m2
+    pmap({'a': 1, 'c': 3, 'b': 2})
+    >>> m3
+    pmap({'a': 5, 'c': 3, 'b': 2})
+    >>> m3['a']
+    5
+
+    # Evolution of nested persistent structures
+    >>> m4 = m(a=5, b=6, c=v(1, 2))
+    >>> m4.transform(('c', 1), 17)
+    pmap({'a': 5, 'c': pvector([1, 17]), 'b': 6})
+    >>> m5 = m(a=1, b=2)
+
+    # Evolve by merging with other mappings
+    >>> m5.update(m(a=2, c=3), {'a': 17, 'd': 35})
+    pmap({'a': 17, 'c': 3, 'b': 2, 'd': 35})
+    >>> pmap({'x': 1, 'y': 2}) + pmap({'y': 3, 'z': 4})
+    pmap({'y': 3, 'x': 1, 'z': 4})
+
+    # Dict-like methods to convert to list and iterate
+    >>> m3.items()
+    pvector([('a', 5), ('c', 3), ('b', 2)])
+    >>> list(m3)
+    ['a', 'c', 'b']
+
+.. _PSet:
+
+PSet
+~~~~
+With full support for the Set_ protocol PSet is meant as a drop in replacement to the built in set from a readers point
+of view. Support for the Hashable_ protocol also means that it can be used as key in Mappings_.
+
+Random access and insert is log32(n) where n is the size of the set.
+
+.. code:: python
+
+    >>> from pyrsistent import s
+
+    # No mutation of sets once created, you know the story...
+    >>> s1 = s(1, 2, 3, 2)
+    >>> s2 = s1.add(4)
+    >>> s3 = s1.remove(1)
+    >>> s1
+    pset([1, 2, 3])
+    >>> s2
+    pset([1, 2, 3, 4])
+    >>> s3
+    pset([2, 3])
+
+    # Full support for set operations
+    >>> s1 | s(3, 4, 5)
+    pset([1, 2, 3, 4, 5])
+    >>> s1 & s(3, 4, 5)
+    pset([3])
+    >>> s1 < s2
+    True
+    >>> s1 < s(3, 4, 5)
+    False
+
+.. _PRecord:
+
+PRecord
+~~~~~~~
+A PRecord is a PMap with a fixed set of specified fields. Records are declared as python classes inheriting
+from PRecord. Because it is a PMap it has full support for all Mapping methods such as iteration and element
+access using subscript notation.
+
+.. code:: python
+
+    >>> from pyrsistent import PRecord, field
+    >>> class ARecord(PRecord):
+    ...     x = field()
+    ...
+    >>> r = ARecord(x=3)
+    >>> r
+    ARecord(x=3)
+    >>> r.x
+    3
+    >>> r.set(x=2)
+    ARecord(x=2)
+    >>> r.set(y=2)
+    Traceback (most recent call last):
+    AttributeError: 'y' is not among the specified fields for ARecord
+
+Type information
+****************
+It is possible to add type information to the record to enforce type checks. Multiple allowed types can be specified
+by providing an iterable of types.
+
+.. code:: python
+
+    >>> class BRecord(PRecord):
+    ...     x = field(type=int)
+    ...     y = field(type=(int, type(None)))
+    ...
+    >>> BRecord(x=3, y=None)
+    BRecord(y=None, x=3)
+    >>> BRecord(x=3.0)
+    Traceback (most recent call last):
+    PTypeError: Invalid type for field BRecord.x, was float
+
+
+Custom types (classes) that are iterable should be wrapped in a tuple to prevent their
+members being added to the set of valid types.  Although Enums in particular are now
+supported without wrapping, see #83 for more information.
+
+Mandatory fields
+****************
+Fields are not mandatory by default but can be specified as such. If fields are missing an
+*InvariantException* will be thrown which contains information about the missing fields.
+
+.. code:: python
+
+    >>> from pyrsistent import InvariantException
+    >>> class CRecord(PRecord):
+    ...     x = field(mandatory=True)
+    ...
+    >>> r = CRecord(x=3)
+    >>> try:
+    ...    r.discard('x')
+    ... except InvariantException as e:
+    ...    print(e.missing_fields)
+    ...
+    ('CRecord.x',)
+
+Invariants
+**********
+It is possible to add invariants that must hold when evolving the record. Invariants can be
+specified on both field and record level. If invariants fail an *InvariantException* will be
+thrown which contains information about the failing invariants. An invariant function should
+return a tuple consisting of a boolean that tells if the invariant holds or not and an object
+describing the invariant. This object can later be used to identify which invariant that failed.
+
+The global invariant function is only executed if all field invariants hold.
+
+Global invariants are inherited to subclasses.
+
+.. code:: python
+
+    >>> class RestrictedVector(PRecord):
+    ...     __invariant__ = lambda r: (r.y >= r.x, 'x larger than y')
+    ...     x = field(invariant=lambda x: (x > 0, 'x negative'))
+    ...     y = field(invariant=lambda y: (y > 0, 'y negative'))
+    ...
+    >>> r = RestrictedVector(y=3, x=2)
+    >>> try:
+    ...    r.set(x=-1, y=-2)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    ('y negative', 'x negative')
+    >>> try:
+    ...    r.set(x=2, y=1)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    ('x larger than y',)
+
+Invariants may also contain multiple assertions. For those cases the invariant function should
+return a tuple of invariant tuples as described above. This structure is reflected in the
+invariant_errors attribute of the exception which will contain tuples with data from all failed
+invariants. Eg:
+
+.. code:: python
+
+    >>> class EvenX(PRecord):
+    ...     x = field(invariant=lambda x: ((x > 0, 'x negative'), (x % 2 == 0, 'x odd')))
+    ...
+    >>> try:
+    ...    EvenX(x=-1)
+    ... except InvariantException as e:
+    ...    print(e.invariant_errors)
+    ...
+    (('x negative', 'x odd'),)
+
+
+Factories
+*********
+It's possible to specify factory functions for fields. The factory function receives whatever
+is supplied as field value and the actual returned by the factory is assigned to the field
+given that any type and invariant checks hold.
+PRecords have a default factory specified as a static function on the class, create(). It takes
+a *Mapping* as argument and returns an instance of the specific record.
+If a record has fields of type PRecord the create() method of that record will
+be called to create the "sub record" if no factory has explicitly been specified to override
+this behaviour.
+
+.. code:: python
+
+    >>> class DRecord(PRecord):
+    ...     x = field(factory=int)
+    ...
+    >>> class ERecord(PRecord):
+    ...     d = field(type=DRecord)
+    ...
+    >>> ERecord.create({'d': {'x': '1'}})
+    ERecord(d=DRecord(x=1))
+
+Collection fields
+*****************
+It is also possible to have fields with ``pyrsistent`` collections.
+
+.. code:: python
+
+   >>> from pyrsistent import pset_field, pmap_field, pvector_field
+   >>> class MultiRecord(PRecord):
+   ...     set_of_ints = pset_field(int)
+   ...     map_int_to_str = pmap_field(int, str)
+   ...     vector_of_strs = pvector_field(str)
+   ...
+
+Serialization
+*************
+PRecords support serialization back to dicts. Default serialization will take keys and values
+"as is" and output them into a dict. It is possible to specify custom serialization functions
+to take care of fields that require special treatment.
+
+.. code:: python
+
+    >>> from datetime import date
+    >>> class Person(PRecord):
+    ...     name = field(type=unicode)
+    ...     birth_date = field(type=date,
+    ...                        serializer=lambda format, d: d.strftime(format['date']))
+    ...
+    >>> john = Person(name=u'John', birth_date=date(1985, 10, 21))
+    >>> john.serialize({'date': '%Y-%m-%d'})
+    {'birth_date': '1985-10-21', 'name': u'John'}
+
+
+.. _instar: https://github.com/boxed/instar/
+
+.. _PClass:
+
+PClass
+~~~~~~
+A PClass is a python class with a fixed set of specified fields. PClasses are declared as python classes inheriting
+from PClass. It is defined the same way that PRecords are and behaves like a PRecord in all aspects except that it
+is not a PMap and hence not a collection but rather a plain Python object.
+
+.. code:: python
+
+    >>> from pyrsistent import PClass, field
+    >>> class AClass(PClass):
+    ...     x = field()
+    ...
+    >>> a = AClass(x=3)
+    >>> a
+    AClass(x=3)
+    >>> a.x
+    3
+
+
+Checked collections
+~~~~~~~~~~~~~~~~~~~
+Checked collections currently come in three flavors: CheckedPVector, CheckedPMap and CheckedPSet.
+
+.. code:: python
+
+    >>> from pyrsistent import CheckedPVector, CheckedPMap, CheckedPSet, thaw
+    >>> class Positives(CheckedPSet):
+    ...     __type__ = (long, int)
+    ...     __invariant__ = lambda n: (n >= 0, 'Negative')
+    ...
+    >>> class Lottery(PRecord):
+    ...     name = field(type=str)
+    ...     numbers = field(type=Positives, invariant=lambda p: (len(p) > 0, 'No numbers'))
+    ...
+    >>> class Lotteries(CheckedPVector):
+    ...     __type__ = Lottery
+    ...
+    >>> class LotteriesByDate(CheckedPMap):
+    ...     __key_type__ = date
+    ...     __value_type__ = Lotteries
+    ...
+    >>> lotteries = LotteriesByDate.create({date(2015, 2, 15): [{'name': 'SuperLotto', 'numbers': {1, 2, 3}},
+    ...                                                         {'name': 'MegaLotto',  'numbers': {4, 5, 6}}],
+    ...                                     date(2015, 2, 16): [{'name': 'SuperLotto', 'numbers': {3, 2, 1}},
+    ...                                                         {'name': 'MegaLotto',  'numbers': {6, 5, 4}}]})
+    >>> lotteries
+    LotteriesByDate({datetime.date(2015, 2, 15): Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')]), datetime.date(2015, 2, 16): Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')])})
+
+    # The checked versions support all operations that the corresponding
+    # unchecked types do
+    >>> lottery_0215 = lotteries[date(2015, 2, 15)]
+    >>> lottery_0215.transform([0, 'name'], 'SuperDuperLotto')
+    Lotteries([Lottery(numbers=Positives([1, 2, 3]), name='SuperDuperLotto'), Lottery(numbers=Positives([4, 5, 6]), name='MegaLotto')])
+
+    # But also makes asserts that types and invariants hold
+    >>> lottery_0215.transform([0, 'name'], 999)
+    Traceback (most recent call last):
+    PTypeError: Invalid type for field Lottery.name, was int
+
+    >>> lottery_0215.transform([0, 'numbers'], set())
+    Traceback (most recent call last):
+    InvariantException: Field invariant failed
+
+    # They can be converted back to python built ins with either thaw()
+    # or serialize() (which provides possibilities to customize serialization)
+    >>> thaw(lottery_0215)
+    [{'numbers': set([1, 2, 3]), 'name': 'SuperLotto'}, {'numbers': set([4, 5, 6]), 'name': 'MegaLotto'}]
+    >>> lottery_0215.serialize()
+    [{'numbers': set([1, 2, 3]), 'name': 'SuperLotto'}, {'numbers': set([4, 5, 6]), 'name': 'MegaLotto'}]
+
+.. _transformations:
+
+Transformations
+~~~~~~~~~~~~~~~
+Transformations are inspired by the cool library instar_ for Clojure. They let you evolve PMaps and PVectors
+with arbitrarily deep/complex nesting using simple syntax and flexible matching syntax.
+
+The first argument to transformation is the path that points out the value to transform. The
+second is the transformation to perform. If the transformation is callable it will be applied
+to the value(s) matching the path. The path may also contain callables. In that case they are
+treated as matchers. If the matcher returns True for a specific key it is considered for transformation.
+
+.. code:: python
+
+    # Basic examples
+    >>> from pyrsistent import inc, freeze, thaw, rex, ny, discard
+    >>> v1 = freeze([1, 2, 3, 4, 5])
+    >>> v1.transform([2], inc)
+    pvector([1, 2, 4, 4, 5])
+    >>> v1.transform([lambda ix: 0 < ix < 4], 8)
+    pvector([1, 8, 8, 8, 5])
+    >>> v1.transform([lambda ix, v: ix == 0 or v == 5], 0)
+    pvector([0, 2, 3, 4, 0])
+
+    # The (a)ny matcher can be used to match anything
+    >>> v1.transform([ny], 8)
+    pvector([8, 8, 8, 8, 8])
+
+    # Regular expressions can be used for matching
+    >>> scores = freeze({'John': 12, 'Joseph': 34, 'Sara': 23})
+    >>> scores.transform([rex('^Jo')], 0)
+    pmap({'Joseph': 0, 'Sara': 23, 'John': 0})
+
+    # Transformations can be done on arbitrarily deep structures
+    >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
+    ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
+    ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
+    >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
+    >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
+    >>> very_short_news.articles[0].content
+    'A short article'
+    >>> very_short_news.articles[1].content
+    'A slightly long...'
+
+    # When nothing has been transformed the original data structure is kept
+    >>> short_news is news_paper
+    True
+    >>> very_short_news is news_paper
+    False
+    >>> very_short_news.articles[0] is news_paper.articles[0]
+    True
+
+    # There is a special transformation that can be used to discard elements. Also
+    # multiple transformations can be applied in one call
+    >>> thaw(news_paper.transform(['weather'], discard, ['articles', ny, 'content'], discard))
+    {'articles': [{'author': 'Sara'}, {'author': 'Steve'}]}
+
+Evolvers
+~~~~~~~~
+PVector, PMap and PSet all have support for a concept dubbed *evolvers*. An evolver acts like a mutable
+view of the underlying persistent data structure with "transaction like" semantics. No updates of the original
+data structure is ever performed, it is still fully immutable.
+
+The evolvers have a very limited API by design to discourage excessive, and inappropriate, usage as that would
+take us down the mutable road. In principle only basic mutation and element access functions are supported.
+Check out the documentation_ of each data structure for specific examples.
+
+Examples of when you may want to use an evolver instead of working directly with the data structure include:
+
+* Multiple updates are done to the same data structure and the intermediate results are of no
+  interest. In this case using an evolver may be a more efficient and easier to work with.
+* You need to pass a vector into a legacy function or a function that you have no control
+  over which performs in place mutations. In this case pass an evolver instance
+  instead and then create a new pvector from the evolver once the function returns.
+
+.. code:: python
+
+    >>> from pyrsistent import v
+
+    # In place mutation as when working with the built in counterpart
+    >>> v1 = v(1, 2, 3)
+    >>> e = v1.evolver()
+    >>> e[1] = 22
+    >>> e = e.append(4)
+    >>> e = e.extend([5, 6])
+    >>> e[5] += 1
+    >>> len(e)
+    6
+
+    # The evolver is considered *dirty* when it contains changes compared to the underlying vector
+    >>> e.is_dirty()
+    True
+
+    # But the underlying pvector still remains untouched
+    >>> v1
+    pvector([1, 2, 3])
+
+    # Once satisfied with the updates you can produce a new pvector containing the updates.
+    # The new pvector will share data with the original pvector in the same way that would have
+    # been done if only using operations on the pvector.
+    >>> v2 = e.persistent()
+    >>> v2
+    pvector([1, 22, 3, 4, 5, 7])
+
+    # The evolver is now no longer considered *dirty* as it contains no differences compared to the
+    # pvector just produced.
+    >>> e.is_dirty()
+    False
+
+    # You may continue to work with the same evolver without affecting the content of v2
+    >>> e[0] = 11
+
+    # Or create a new evolver from v2. The two evolvers can be updated independently but will both
+    # share data with v2 where possible.
+    >>> e2 = v2.evolver()
+    >>> e2[0] = 1111
+    >>> e.persistent()
+    pvector([11, 22, 3, 4, 5, 7])
+    >>> e2.persistent()
+    pvector([1111, 22, 3, 4, 5, 7])
+
+.. _freeze:
+.. _thaw:
+
+freeze and thaw
+~~~~~~~~~~~~~~~
+These functions are great when your cozy immutable world has to interact with the evil mutable world outside.
+
+.. code:: python
+
+    >>> from pyrsistent import freeze, thaw, v, m
+    >>> freeze([1, {'a': 3}])
+    pvector([1, pmap({'a': 3})])
+    >>> thaw(v(1, m(a=3)))
+    [1, {'a': 3}]
+
+By default, freeze will also recursively convert values inside PVectors and PMaps. This behaviour can be changed by providing freeze with the flag strict=False.
+
+.. code:: python
+
+    >>> from pyrsistent import freeze, v, m
+    >>> freeze(v(1, v(2, [3])))
+    pvector([1, pvector([2, pvector([3])])])
+    >>> freeze(v(1, v(2, [3])), strict=False)
+    pvector([1, pvector([2, [3]])])
+    >>> freeze(m(a=m(b={'c': 1})))
+    pmap({'a': pmap({'b': pmap({'c': 1})})})
+    >>> freeze(m(a=m(b={'c': 1})), strict=False)
+    pmap({'a': pmap({'b': {'c': 1}})})
+
+In this regard, thaw operates as the inverse of freeze so will thaw values inside native data structures unless passed the strict=False flag.
+
+
+Compatibility
+-------------
+
+Pyrsistent is developed and tested on Python 3.8+ and PyPy3.
+
+Performance
+-----------
+
+Pyrsistent is developed with performance in mind. Still, while some operations are nearly on par with their built in,
+mutable, counterparts in terms of speed, other operations are slower. In the cases where attempts at
+optimizations have been done, speed has generally been valued over space.
+
+Pyrsistent comes with two API compatible flavors of PVector (on which PMap and PSet are based), one pure Python
+implementation and one implemented as a C extension. The latter generally being 2 - 20 times faster than the former.
+The C extension will be used automatically when possible.
+
+The pure python implementation is fully PyPy compatible. Running it under PyPy speeds operations up considerably if
+the structures are used heavily (if JITed), for some cases the performance is almost on par with the built in counterparts.
+
+Type hints
+----------
+
+PEP 561 style type hints for use with mypy and various editors are available for most types and functions in pyrsistent.
+
+Type classes for annotating your own code with pyrsistent types are also available under pyrsistent.typing.
+
+Installation
+------------
+
+pip install pyrsistent
+
+Documentation
+-------------
+
+Available at http://pyrsistent.readthedocs.org/
+
+Brief presentation available at http://slides.com/tobiasgustafsson/immutability-and-python/
+
+Contributors
+------------
+
+Tobias Gustafsson https://github.com/tobgu
+
+Christopher Armstrong https://github.com/radix
+
+Anders Hovmöller https://github.com/boxed
+
+Itamar Turner-Trauring https://github.com/itamarst
+
+Jonathan Lange https://github.com/jml
+
+Richard Futrell https://github.com/Futrell
+
+Jakob Hollenstein https://github.com/jkbjh
+
+David Honour https://github.com/foolswood
+
+David R. MacIver https://github.com/DRMacIver
+
+Marcus Ewert https://github.com/sarum90
+
+Jean-Paul Calderone https://github.com/exarkun
+
+Douglas Treadwell https://github.com/douglas-treadwell
+
+Travis Parker https://github.com/teepark
+
+Julian Berman https://github.com/Julian
+
+Dennis Tomas https://github.com/dtomas
+
+Neil Vyas https://github.com/neilvyas
+
+doozr https://github.com/doozr
+
+Kamil Galuszka https://github.com/galuszkak
+
+Tsuyoshi Hombashi https://github.com/thombashi
+
+nattofriends https://github.com/nattofriends
+
+agberk https://github.com/agberk
+
+Waleed Khan https://github.com/arxanas
+
+Jean-Louis Fuchs https://github.com/ganwell
+
+Carlos Corbacho https://github.com/ccorbacho
+
+Felix Yan https://github.com/felixonmars
+
+benrg https://github.com/benrg
+
+Jere Lahelma https://github.com/je-l
+
+Max Taggart https://github.com/MaxTaggart
+
+Vincent Philippon https://github.com/vphilippon
+
+Semen Zhydenko https://github.com/ss18
+
+Till Varoquaux  https://github.com/till-varoquaux
+
+Michal Kowalik https://github.com/michalvi
+
+ossdev07 https://github.com/ossdev07
+
+Kerry Olesen https://github.com/qhesz
+
+johnthagen https://github.com/johnthagen
+
+Bastien Vallet https://github.com/djailla
+
+Ram Rachum  https://github.com/cool-RR
+
+Vincent Philippon https://github.com/vphilippon
+
+Andrey Bienkowski https://github.com/hexagonrecursion
+
+Ethan McCue https://github.com/bowbahdoe
+
+Jason R. Coombs https://github.com/jaraco
+
+Nathan https://github.com/ndowens
+
+Geert Barentsen https://github.com/barentsen
+
+phil-arh https://github.com/phil-arh
+
+Tamás Nepusz https://github.com/ntamas
+
+Hugo van Kemenade https://github.com/hugovk
+
+Ben Beasley https://github.com/musicinmybrain
+
+Noah C. Benson https://github.com/noahbenson
+
+dscrofts https://github.com/dscrofts
+
+Andy Reagan https://github.com/andyreagan
+
+Aaron Durant https://github.com/Aaron-Durant
+
+Joshua Munn https://github.com/jams2
+
+Lukas https://github.com/lukasK9999
+
+Arshad https://github.com/arshad-ml
+
+Contributing
+------------
+
+Want to contribute? That's great! If you experience problems please log them on GitHub. If you want to contribute code,
+please fork the repository and submit a pull request.
+
+Run tests
+~~~~~~~~~
+.. _tox: https://tox.readthedocs.io/en/latest/
+
+Tests can be executed using tox_.
+
+Install tox: ``pip install tox``
+
+Run test for Python 3.8: ``tox -e py38``
+
+Release
+~~~~~~~
+* `pip install -r requirements.txt`
+* Update CHANGES.txt
+* Update README.rst with any new contributors and potential info needed.
+* Update _pyrsistent_version.py
+* Commit and tag with new version: `git add -u . && git commit -m 'Prepare version vX.Y.Z' && git tag -a vX.Y.Z -m 'vX.Y.Z'`
+* Push commit and tags: `git push --follow-tags`
+* Build new release using Github actions
+
+Project status
+--------------
+Pyrsistent can be considered stable and mature (who knows, there may even be a 1.0 some day :-)). The project is
+maintained, bugs fixed, PRs reviewed and merged and new releases made. I currently do not have time for development
+of new features or functionality which I don't have use for myself. I'm more than happy to take PRs for new
+functionality though!
+
+There are a bunch of issues marked with ``enhancement`` and ``help wanted`` that contain requests for new functionality
+that would be nice to include. The level of difficulty and extend of the issues varies, please reach out to me if you're
+interested in working on any of them.
+
+If you feel that you have a grand master plan for where you would like Pyrsistent to go and have the time to put into
+it please don't hesitate to discuss this with me and submit PRs for it. If all goes well I'd be more than happy to add
+additional maintainers to the project!
diff --git a/contrib/python/pyrsistent/py3/_pyrsistent_version.py b/contrib/python/pyrsistent/py3/_pyrsistent_version.py
new file mode 100644
index 00000000000..2f15b8cd378
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/_pyrsistent_version.py
@@ -0,0 +1 @@
+__version__ = '0.20.0'
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/__init__.py b/contrib/python/pyrsistent/py3/pyrsistent/__init__.py
new file mode 100644
index 00000000000..be299658f3f
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/__init__.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+from pyrsistent._pmap import pmap, m, PMap
+
+from pyrsistent._pvector import pvector, v, PVector
+
+from pyrsistent._pset import pset, s, PSet
+
+from pyrsistent._pbag import pbag, b, PBag
+
+from pyrsistent._plist import plist, l, PList
+
+from pyrsistent._pdeque import pdeque, dq, PDeque
+
+from pyrsistent._checked_types import (
+    CheckedPMap, CheckedPVector, CheckedPSet, InvariantException, CheckedKeyTypeError,
+    CheckedValueTypeError, CheckedType, optional)
+
+from pyrsistent._field_common import (
+    field, PTypeError, pset_field, pmap_field, pvector_field)
+
+from pyrsistent._precord import PRecord
+
+from pyrsistent._pclass import PClass, PClassMeta
+
+from pyrsistent._immutable import immutable
+
+from pyrsistent._helpers import freeze, thaw, mutant
+
+from pyrsistent._transformations import inc, discard, rex, ny
+
+from pyrsistent._toolz import get_in
+
+
+__all__ = ('pmap', 'm', 'PMap',
+           'pvector', 'v', 'PVector',
+           'pset', 's', 'PSet',
+           'pbag', 'b', 'PBag',
+           'plist', 'l', 'PList',
+           'pdeque', 'dq', 'PDeque',
+           'CheckedPMap', 'CheckedPVector', 'CheckedPSet', 'InvariantException', 'CheckedKeyTypeError', 'CheckedValueTypeError', 'CheckedType', 'optional',
+           'PRecord', 'field', 'pset_field', 'pmap_field', 'pvector_field',
+           'PClass', 'PClassMeta',
+           'immutable',
+           'freeze', 'thaw', 'mutant',
+           'get_in',
+           'inc', 'discard', 'rex', 'ny')
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_checked_types.py b/contrib/python/pyrsistent/py3/pyrsistent/_checked_types.py
new file mode 100644
index 00000000000..48446e51680
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_checked_types.py
@@ -0,0 +1,547 @@
+from enum import Enum
+
+from abc import abstractmethod, ABCMeta
+from collections.abc import Iterable
+from typing import TypeVar, Generic
+
+from pyrsistent._pmap import PMap, pmap
+from pyrsistent._pset import PSet, pset
+from pyrsistent._pvector import PythonPVector, python_pvector
+
+T_co = TypeVar('T_co', covariant=True)
+KT = TypeVar('KT')
+VT_co = TypeVar('VT_co', covariant=True)
+
+
+class CheckedType(object):
+    """
+    Marker class to enable creation and serialization of checked object graphs.
+    """
+    __slots__ = ()
+
+    @classmethod
+    @abstractmethod
+    def create(cls, source_data, _factory_fields=None):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def serialize(self, format=None):
+        raise NotImplementedError()
+
+
+def _restore_pickle(cls, data):
+    return cls.create(data, _factory_fields=set())
+
+
+class InvariantException(Exception):
+    """
+    Exception raised from a :py:class:`CheckedType` when invariant tests fail or when a mandatory
+    field is missing.
+
+    Contains two fields of interest:
+    invariant_errors, a tuple of error data for the failing invariants
+    missing_fields, a tuple of strings specifying the missing names
+    """
+
+    def __init__(self, error_codes=(), missing_fields=(), *args, **kwargs):
+        self.invariant_errors = tuple(e() if callable(e) else e for e in error_codes)
+        self.missing_fields = missing_fields
+        super(InvariantException, self).__init__(*args, **kwargs)
+
+    def __str__(self):
+        return super(InvariantException, self).__str__() + \
+            ", invariant_errors=[{invariant_errors}], missing_fields=[{missing_fields}]".format(
+            invariant_errors=', '.join(str(e) for e in self.invariant_errors),
+            missing_fields=', '.join(self.missing_fields))
+
+
+_preserved_iterable_types = (
+    Enum,
+)
+"""Some types are themselves iterable, but we want to use the type itself and
+not its members for the type specification. This defines a set of such types
+that we explicitly preserve.
+
+Note that strings are not such types because the string inputs we pass in are
+values, not types.
+"""
+
+
+def maybe_parse_user_type(t):
+    """Try to coerce a user-supplied type directive into a list of types.
+
+    This function should be used in all places where a user specifies a type,
+    for consistency.
+
+    The policy for what defines valid user input should be clear from the implementation.
+    """
+    is_type = isinstance(t, type)
+    is_preserved = isinstance(t, type) and issubclass(t, _preserved_iterable_types)
+    is_string = isinstance(t, str)
+    is_iterable = isinstance(t, Iterable)
+
+    if is_preserved:
+        return [t]
+    elif is_string:
+        return [t]
+    elif is_type and not is_iterable:
+        return [t]
+    elif is_iterable:
+        # Recur to validate contained types as well.
+        ts = t
+        return tuple(e for t in ts for e in maybe_parse_user_type(t))
+    else:
+        # If this raises because `t` cannot be formatted, so be it.
+        raise TypeError(
+            'Type specifications must be types or strings. Input: {}'.format(t)
+        )
+
+
+def maybe_parse_many_user_types(ts):
+    # Just a different name to communicate that you're parsing multiple user
+    # inputs. `maybe_parse_user_type` handles the iterable case anyway.
+    return maybe_parse_user_type(ts)
+
+
+def _store_types(dct, bases, destination_name, source_name):
+    maybe_types = maybe_parse_many_user_types([
+        d[source_name]
+        for d in ([dct] + [b.__dict__ for b in bases]) if source_name in d
+    ])
+
+    dct[destination_name] = maybe_types
+
+
+def _merge_invariant_results(result):
+    verdict = True
+    data = []
+    for verd, dat in result:
+        if not verd:
+            verdict = False
+            data.append(dat)
+
+    return verdict, tuple(data)
+
+
+def wrap_invariant(invariant):
+    # Invariant functions may return the outcome of several tests
+    # In those cases the results have to be merged before being passed
+    # back to the client.
+    def f(*args, **kwargs):
+        result = invariant(*args, **kwargs)
+        if isinstance(result[0], bool):
+            return result
+
+        return _merge_invariant_results(result)
+
+    return f
+
+
+def _all_dicts(bases, seen=None):
+    """
+    Yield each class in ``bases`` and each of their base classes.
+    """
+    if seen is None:
+        seen = set()
+    for cls in bases:
+        if cls in seen:
+            continue
+        seen.add(cls)
+        yield cls.__dict__
+        for b in _all_dicts(cls.__bases__, seen):
+            yield b
+
+
+def store_invariants(dct, bases, destination_name, source_name):
+    # Invariants are inherited
+    invariants = []
+    for ns in [dct] + list(_all_dicts(bases)):
+        try:
+            invariant = ns[source_name]
+        except KeyError:
+            continue
+        invariants.append(invariant)
+
+    if not all(callable(invariant) for invariant in invariants):
+        raise TypeError('Invariants must be callable')
+    dct[destination_name] = tuple(wrap_invariant(inv) for inv in invariants)
+
+
+class _CheckedTypeMeta(ABCMeta):
+    def __new__(mcs, name, bases, dct):
+        _store_types(dct, bases, '_checked_types', '__type__')
+        store_invariants(dct, bases, '_checked_invariants', '__invariant__')
+
+        def default_serializer(self, _, value):
+            if isinstance(value, CheckedType):
+                return value.serialize()
+            return value
+
+        dct.setdefault('__serializer__', default_serializer)
+
+        dct['__slots__'] = ()
+
+        return super(_CheckedTypeMeta, mcs).__new__(mcs, name, bases, dct)
+
+
+class CheckedTypeError(TypeError):
+    def __init__(self, source_class, expected_types, actual_type, actual_value, *args, **kwargs):
+        super(CheckedTypeError, self).__init__(*args, **kwargs)
+        self.source_class = source_class
+        self.expected_types = expected_types
+        self.actual_type = actual_type
+        self.actual_value = actual_value
+
+
+class CheckedKeyTypeError(CheckedTypeError):
+    """
+    Raised when trying to set a value using a key with a type that doesn't match the declared type.
+
+    Attributes:
+    source_class -- The class of the collection
+    expected_types  -- Allowed types
+    actual_type -- The non matching type
+    actual_value -- Value of the variable with the non matching type
+    """
+    pass
+
+
+class CheckedValueTypeError(CheckedTypeError):
+    """
+    Raised when trying to set a value using a key with a type that doesn't match the declared type.
+
+    Attributes:
+    source_class -- The class of the collection
+    expected_types  -- Allowed types
+    actual_type -- The non matching type
+    actual_value -- Value of the variable with the non matching type
+    """
+    pass
+
+
+def _get_class(type_name):
+    module_name, class_name = type_name.rsplit('.', 1)
+    module = __import__(module_name, fromlist=[class_name])
+    return getattr(module, class_name)
+
+
+def get_type(typ):
+    if isinstance(typ, type):
+        return typ
+
+    return _get_class(typ)
+
+
+def get_types(typs):
+    return [get_type(typ) for typ in typs]
+
+
+def _check_types(it, expected_types, source_class, exception_type=CheckedValueTypeError):
+    if expected_types:
+        for e in it:
+            if not any(isinstance(e, get_type(t)) for t in expected_types):
+                actual_type = type(e)
+                msg = "Type {source_class} can only be used with {expected_types}, not {actual_type}".format(
+                    source_class=source_class.__name__,
+                    expected_types=tuple(get_type(et).__name__ for et in expected_types),
+                    actual_type=actual_type.__name__)
+                raise exception_type(source_class, expected_types, actual_type, e, msg)
+
+
+def _invariant_errors(elem, invariants):
+    return [data for valid, data in (invariant(elem) for invariant in invariants) if not valid]
+
+
+def _invariant_errors_iterable(it, invariants):
+    return sum([_invariant_errors(elem, invariants) for elem in it], [])
+
+
+def optional(*typs):
+    """ Convenience function to specify that a value may be of any of the types in type 'typs' or None """
+    return tuple(typs) + (type(None),)
+
+
+def _checked_type_create(cls, source_data, _factory_fields=None, ignore_extra=False):
+    if isinstance(source_data, cls):
+        return source_data
+
+    # Recursively apply create methods of checked types if the types of the supplied data
+    # does not match any of the valid types.
+    types = get_types(cls._checked_types)
+    checked_type = next((t for t in types if issubclass(t, CheckedType)), None)
+    if checked_type:
+        return cls([checked_type.create(data, ignore_extra=ignore_extra)
+                    if not any(isinstance(data, t) for t in types) else data
+                    for data in source_data])
+
+    return cls(source_data)
+
+class CheckedPVector(Generic[T_co], PythonPVector, CheckedType, metaclass=_CheckedTypeMeta):
+    """
+    A CheckedPVector is a PVector which allows specifying type and invariant checks.
+
+    >>> class Positives(CheckedPVector):
+    ...     __type__ = (int, float)
+    ...     __invariant__ = lambda n: (n >= 0, 'Negative')
+    ...
+    >>> Positives([1, 2, 3])
+    Positives([1, 2, 3])
+    """
+
+    __slots__ = ()
+
+    def __new__(cls, initial=()):
+        if type(initial) == PythonPVector:
+            return super(CheckedPVector, cls).__new__(cls, initial._count, initial._shift, initial._root, initial._tail)
+
+        return CheckedPVector.Evolver(cls, python_pvector()).extend(initial).persistent()
+
+    def set(self, key, value):
+        return self.evolver().set(key, value).persistent()
+
+    def append(self, val):
+        return self.evolver().append(val).persistent()
+
+    def extend(self, it):
+        return self.evolver().extend(it).persistent()
+
+    create = classmethod(_checked_type_create)
+
+    def serialize(self, format=None):
+        serializer = self.__serializer__
+        return list(serializer(format, v) for v in self)
+
+    def __reduce__(self):
+        # Pickling support
+        return _restore_pickle, (self.__class__, list(self),)
+
+    class Evolver(PythonPVector.Evolver):
+        __slots__ = ('_destination_class', '_invariant_errors')
+
+        def __init__(self, destination_class, vector):
+            super(CheckedPVector.Evolver, self).__init__(vector)
+            self._destination_class = destination_class
+            self._invariant_errors = []
+
+        def _check(self, it):
+            _check_types(it, self._destination_class._checked_types, self._destination_class)
+            error_data = _invariant_errors_iterable(it, self._destination_class._checked_invariants)
+            self._invariant_errors.extend(error_data)
+
+        def __setitem__(self, key, value):
+            self._check([value])
+            return super(CheckedPVector.Evolver, self).__setitem__(key, value)
+
+        def append(self, elem):
+            self._check([elem])
+            return super(CheckedPVector.Evolver, self).append(elem)
+
+        def extend(self, it):
+            it = list(it)
+            self._check(it)
+            return super(CheckedPVector.Evolver, self).extend(it)
+
+        def persistent(self):
+            if self._invariant_errors:
+                raise InvariantException(error_codes=self._invariant_errors)
+
+            result = self._orig_pvector
+            if self.is_dirty() or (self._destination_class != type(self._orig_pvector)):
+                pv = super(CheckedPVector.Evolver, self).persistent().extend(self._extra_tail)
+                result = self._destination_class(pv)
+                self._reset(result)
+
+            return result
+
+    def __repr__(self):
+        return self.__class__.__name__ + "({0})".format(self.tolist())
+
+    __str__ = __repr__
+
+    def evolver(self):
+        return CheckedPVector.Evolver(self.__class__, self)
+
+
+class CheckedPSet(PSet[T_co], CheckedType, metaclass=_CheckedTypeMeta):
+    """
+    A CheckedPSet is a PSet which allows specifying type and invariant checks.
+
+    >>> class Positives(CheckedPSet):
+    ...     __type__ = (int, float)
+    ...     __invariant__ = lambda n: (n >= 0, 'Negative')
+    ...
+    >>> Positives([1, 2, 3])
+    Positives([1, 2, 3])
+    """
+
+    __slots__ = ()
+
+    def __new__(cls, initial=()):
+        if type(initial) is PMap:
+            return super(CheckedPSet, cls).__new__(cls, initial)
+
+        evolver = CheckedPSet.Evolver(cls, pset())
+        for e in initial:
+            evolver.add(e)
+
+        return evolver.persistent()
+
+    def __repr__(self):
+        return self.__class__.__name__ + super(CheckedPSet, self).__repr__()[4:]
+
+    def __str__(self):
+        return self.__repr__()
+
+    def serialize(self, format=None):
+        serializer = self.__serializer__
+        return set(serializer(format, v) for v in self)
+
+    create = classmethod(_checked_type_create)
+
+    def __reduce__(self):
+        # Pickling support
+        return _restore_pickle, (self.__class__, list(self),)
+
+    def evolver(self):
+        return CheckedPSet.Evolver(self.__class__, self)
+
+    class Evolver(PSet._Evolver):
+        __slots__ = ('_destination_class', '_invariant_errors')
+
+        def __init__(self, destination_class, original_set):
+            super(CheckedPSet.Evolver, self).__init__(original_set)
+            self._destination_class = destination_class
+            self._invariant_errors = []
+
+        def _check(self, it):
+            _check_types(it, self._destination_class._checked_types, self._destination_class)
+            error_data = _invariant_errors_iterable(it, self._destination_class._checked_invariants)
+            self._invariant_errors.extend(error_data)
+
+        def add(self, element):
+            self._check([element])
+            self._pmap_evolver[element] = True
+            return self
+
+        def persistent(self):
+            if self._invariant_errors:
+                raise InvariantException(error_codes=self._invariant_errors)
+
+            if self.is_dirty() or self._destination_class != type(self._original_pset):
+                return self._destination_class(self._pmap_evolver.persistent())
+
+            return self._original_pset
+
+
+class _CheckedMapTypeMeta(type):
+    def __new__(mcs, name, bases, dct):
+        _store_types(dct, bases, '_checked_key_types', '__key_type__')
+        _store_types(dct, bases, '_checked_value_types', '__value_type__')
+        store_invariants(dct, bases, '_checked_invariants', '__invariant__')
+
+        def default_serializer(self, _, key, value):
+            sk = key
+            if isinstance(key, CheckedType):
+                sk = key.serialize()
+
+            sv = value
+            if isinstance(value, CheckedType):
+                sv = value.serialize()
+
+            return sk, sv
+
+        dct.setdefault('__serializer__', default_serializer)
+
+        dct['__slots__'] = ()
+
+        return super(_CheckedMapTypeMeta, mcs).__new__(mcs, name, bases, dct)
+
+# Marker object
+_UNDEFINED_CHECKED_PMAP_SIZE = object()
+
+
+class CheckedPMap(PMap[KT, VT_co], CheckedType, metaclass=_CheckedMapTypeMeta):
+    """
+    A CheckedPMap is a PMap which allows specifying type and invariant checks.
+
+    >>> class IntToFloatMap(CheckedPMap):
+    ...     __key_type__ = int
+    ...     __value_type__ = float
+    ...     __invariant__ = lambda k, v: (int(v) == k, 'Invalid mapping')
+    ...
+    >>> IntToFloatMap({1: 1.5, 2: 2.25})
+    IntToFloatMap({1: 1.5, 2: 2.25})
+    """
+
+    __slots__ = ()
+
+    def __new__(cls, initial={}, size=_UNDEFINED_CHECKED_PMAP_SIZE):
+        if size is not _UNDEFINED_CHECKED_PMAP_SIZE:
+            return super(CheckedPMap, cls).__new__(cls, size, initial)
+
+        evolver = CheckedPMap.Evolver(cls, pmap())
+        for k, v in initial.items():
+            evolver.set(k, v)
+
+        return evolver.persistent()
+
+    def evolver(self):
+        return CheckedPMap.Evolver(self.__class__, self)
+
+    def __repr__(self):
+        return self.__class__.__name__ + "({0})".format(str(dict(self)))
+
+    __str__ = __repr__
+
+    def serialize(self, format=None):
+        serializer = self.__serializer__
+        return dict(serializer(format, k, v) for k, v in self.items())
+
+    @classmethod
+    def create(cls, source_data, _factory_fields=None):
+        if isinstance(source_data, cls):
+            return source_data
+
+        # Recursively apply create methods of checked types if the types of the supplied data
+        # does not match any of the valid types.
+        key_types = get_types(cls._checked_key_types)
+        checked_key_type = next((t for t in key_types if issubclass(t, CheckedType)), None)
+        value_types = get_types(cls._checked_value_types)
+        checked_value_type = next((t for t in value_types if issubclass(t, CheckedType)), None)
+
+        if checked_key_type or checked_value_type:
+            return cls(dict((checked_key_type.create(key) if checked_key_type and not any(isinstance(key, t) for t in key_types) else key,
+                             checked_value_type.create(value) if checked_value_type and not any(isinstance(value, t) for t in value_types) else value)
+                            for key, value in source_data.items()))
+
+        return cls(source_data)
+
+    def __reduce__(self):
+        # Pickling support
+        return _restore_pickle, (self.__class__, dict(self),)
+
+    class Evolver(PMap._Evolver):
+        __slots__ = ('_destination_class', '_invariant_errors')
+
+        def __init__(self, destination_class, original_map):
+            super(CheckedPMap.Evolver, self).__init__(original_map)
+            self._destination_class = destination_class
+            self._invariant_errors = []
+
+        def set(self, key, value):
+            _check_types([key], self._destination_class._checked_key_types, self._destination_class, CheckedKeyTypeError)
+            _check_types([value], self._destination_class._checked_value_types, self._destination_class)
+            self._invariant_errors.extend(data for valid, data in (invariant(key, value)
+                                                                   for invariant in self._destination_class._checked_invariants)
+                                          if not valid)
+
+            return super(CheckedPMap.Evolver, self).set(key, value)
+
+        def persistent(self):
+            if self._invariant_errors:
+                raise InvariantException(error_codes=self._invariant_errors)
+
+            if self.is_dirty() or type(self._original_pmap) != self._destination_class:
+                return self._destination_class(self._buckets_evolver.persistent(), self._size)
+
+            return self._original_pmap
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_field_common.py b/contrib/python/pyrsistent/py3/pyrsistent/_field_common.py
new file mode 100644
index 00000000000..508dd2f799e
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_field_common.py
@@ -0,0 +1,332 @@
+from pyrsistent._checked_types import (
+    CheckedPMap,
+    CheckedPSet,
+    CheckedPVector,
+    CheckedType,
+    InvariantException,
+    _restore_pickle,
+    get_type,
+    maybe_parse_user_type,
+    maybe_parse_many_user_types,
+)
+from pyrsistent._checked_types import optional as optional_type
+from pyrsistent._checked_types import wrap_invariant
+import inspect
+
+
+def set_fields(dct, bases, name):
+    dct[name] = dict(sum([list(b.__dict__.get(name, {}).items()) for b in bases], []))
+
+    for k, v in list(dct.items()):
+        if isinstance(v, _PField):
+            dct[name][k] = v
+            del dct[k]
+
+
+def check_global_invariants(subject, invariants):
+    error_codes = tuple(error_code for is_ok, error_code in
+                        (invariant(subject) for invariant in invariants) if not is_ok)
+    if error_codes:
+        raise InvariantException(error_codes, (), 'Global invariant failed')
+
+
+def serialize(serializer, format, value):
+    if isinstance(value, CheckedType) and serializer is PFIELD_NO_SERIALIZER:
+        return value.serialize(format)
+
+    return serializer(format, value)
+
+
+def check_type(destination_cls, field, name, value):
+    if field.type and not any(isinstance(value, get_type(t)) for t in field.type):
+        actual_type = type(value)
+        message = "Invalid type for field {0}.{1}, was {2}".format(destination_cls.__name__, name, actual_type.__name__)
+        raise PTypeError(destination_cls, name, field.type, actual_type, message)
+
+
+def is_type_cls(type_cls, field_type):
+    if type(field_type) is set:
+        return True
+    types = tuple(field_type)
+    if len(types) == 0:
+        return False
+    return issubclass(get_type(types[0]), type_cls)
+
+
+def is_field_ignore_extra_complaint(type_cls, field, ignore_extra):
+    # ignore_extra param has default False value, for speed purpose no need to propagate False
+    if not ignore_extra:
+        return False
+
+    if not is_type_cls(type_cls, field.type):
+        return False
+
+    return 'ignore_extra' in inspect.signature(field.factory).parameters
+
+
+
+class _PField(object):
+    __slots__ = ('type', 'invariant', 'initial', 'mandatory', '_factory', 'serializer')
+
+    def __init__(self, type, invariant, initial, mandatory, factory, serializer):
+        self.type = type
+        self.invariant = invariant
+        self.initial = initial
+        self.mandatory = mandatory
+        self._factory = factory
+        self.serializer = serializer
+
+    @property
+    def factory(self):
+        # If no factory is specified and the type is another CheckedType use the factory method of that CheckedType
+        if self._factory is PFIELD_NO_FACTORY and len(self.type) == 1:
+            typ = get_type(tuple(self.type)[0])
+            if issubclass(typ, CheckedType):
+                return typ.create
+
+        return self._factory
+
+PFIELD_NO_TYPE = ()
+PFIELD_NO_INVARIANT = lambda _: (True, None)
+PFIELD_NO_FACTORY = lambda x: x
+PFIELD_NO_INITIAL = object()
+PFIELD_NO_SERIALIZER = lambda _, value: value
+
+
+def field(type=PFIELD_NO_TYPE, invariant=PFIELD_NO_INVARIANT, initial=PFIELD_NO_INITIAL,
+          mandatory=False, factory=PFIELD_NO_FACTORY, serializer=PFIELD_NO_SERIALIZER):
+    """
+    Field specification factory for :py:class:`PRecord`.
+
+    :param type: a type or iterable with types that are allowed for this field
+    :param invariant: a function specifying an invariant that must hold for the field
+    :param initial: value of field if not specified when instantiating the record
+    :param mandatory: boolean specifying if the field is mandatory or not
+    :param factory: function called when field is set.
+    :param serializer: function that returns a serialized version of the field
+    """
+
+    # NB: We have to check this predicate separately from the predicates in
+    # `maybe_parse_user_type` et al. because this one is related to supporting
+    # the argspec for `field`, while those are related to supporting the valid
+    # ways to specify types.
+
+    # Multiple types must be passed in one of the following containers. Note
+    # that a type that is a subclass of one of these containers, like a
+    # `collections.namedtuple`, will work as expected, since we check
+    # `isinstance` and not `issubclass`.
+    if isinstance(type, (list, set, tuple)):
+        types = set(maybe_parse_many_user_types(type))
+    else:
+        types = set(maybe_parse_user_type(type))
+
+    invariant_function = wrap_invariant(invariant) if invariant != PFIELD_NO_INVARIANT and callable(invariant) else invariant
+    field = _PField(type=types, invariant=invariant_function, initial=initial,
+                    mandatory=mandatory, factory=factory, serializer=serializer)
+
+    _check_field_parameters(field)
+
+    return field
+
+
+def _check_field_parameters(field):
+    for t in field.type:
+        if not isinstance(t, type) and not isinstance(t, str):
+            raise TypeError('Type parameter expected, not {0}'.format(type(t)))
+
+    if field.initial is not PFIELD_NO_INITIAL and \
+            not callable(field.initial) and \
+            field.type and not any(isinstance(field.initial, t) for t in field.type):
+        raise TypeError('Initial has invalid type {0}'.format(type(field.initial)))
+
+    if not callable(field.invariant):
+        raise TypeError('Invariant must be callable')
+
+    if not callable(field.factory):
+        raise TypeError('Factory must be callable')
+
+    if not callable(field.serializer):
+        raise TypeError('Serializer must be callable')
+
+
+class PTypeError(TypeError):
+    """
+    Raised when trying to assign a value with a type that doesn't match the declared type.
+
+    Attributes:
+    source_class -- The class of the record
+    field -- Field name
+    expected_types  -- Types allowed for the field
+    actual_type -- The non matching type
+    """
+    def __init__(self, source_class, field, expected_types, actual_type, *args, **kwargs):
+        super(PTypeError, self).__init__(*args, **kwargs)
+        self.source_class = source_class
+        self.field = field
+        self.expected_types = expected_types
+        self.actual_type = actual_type
+
+
+SEQ_FIELD_TYPE_SUFFIXES = {
+    CheckedPVector: "PVector",
+    CheckedPSet: "PSet",
+}
+
+# Global dictionary to hold auto-generated field types: used for unpickling
+_seq_field_types = {}
+
+def _restore_seq_field_pickle(checked_class, item_type, data):
+    """Unpickling function for auto-generated PVec/PSet field types."""
+    type_ = _seq_field_types[checked_class, item_type]
+    return _restore_pickle(type_, data)
+
+def _types_to_names(types):
+    """Convert a tuple of types to a human-readable string."""
+    return "".join(get_type(typ).__name__.capitalize() for typ in types)
+
+def _make_seq_field_type(checked_class, item_type, item_invariant):
+    """Create a subclass of the given checked class with the given item type."""
+    type_ = _seq_field_types.get((checked_class, item_type))
+    if type_ is not None:
+        return type_
+
+    class TheType(checked_class):
+        __type__ = item_type
+        __invariant__ = item_invariant
+
+        def __reduce__(self):
+            return (_restore_seq_field_pickle,
+                    (checked_class, item_type, list(self)))
+
+    suffix = SEQ_FIELD_TYPE_SUFFIXES[checked_class]
+    TheType.__name__ = _types_to_names(TheType._checked_types) + suffix
+    _seq_field_types[checked_class, item_type] = TheType
+    return TheType
+
+def _sequence_field(checked_class, item_type, optional, initial,
+                    invariant=PFIELD_NO_INVARIANT,
+                    item_invariant=PFIELD_NO_INVARIANT):
+    """
+    Create checked field for either ``PSet`` or ``PVector``.
+
+    :param checked_class: ``CheckedPSet`` or ``CheckedPVector``.
+    :param item_type: The required type for the items in the set.
+    :param optional: If true, ``None`` can be used as a value for
+        this field.
+    :param initial: Initial value to pass to factory.
+
+    :return: A ``field`` containing a checked class.
+    """
+    TheType = _make_seq_field_type(checked_class, item_type, item_invariant)
+
+    if optional:
+        def factory(argument, _factory_fields=None, ignore_extra=False):
+            if argument is None:
+                return None
+            else:
+                return TheType.create(argument, _factory_fields=_factory_fields, ignore_extra=ignore_extra)
+    else:
+        factory = TheType.create
+
+    return field(type=optional_type(TheType) if optional else TheType,
+                 factory=factory, mandatory=True,
+                 invariant=invariant,
+                 initial=factory(initial))
+
+
+def pset_field(item_type, optional=False, initial=(),
+               invariant=PFIELD_NO_INVARIANT,
+               item_invariant=PFIELD_NO_INVARIANT):
+    """
+    Create checked ``PSet`` field.
+
+    :param item_type: The required type for the items in the set.
+    :param optional: If true, ``None`` can be used as a value for
+        this field.
+    :param initial: Initial value to pass to factory if no value is given
+        for the field.
+
+    :return: A ``field`` containing a ``CheckedPSet`` of the given type.
+    """
+    return _sequence_field(CheckedPSet, item_type, optional, initial,
+                           invariant=invariant,
+                           item_invariant=item_invariant)
+
+
+def pvector_field(item_type, optional=False, initial=(),
+                  invariant=PFIELD_NO_INVARIANT,
+                  item_invariant=PFIELD_NO_INVARIANT):
+    """
+    Create checked ``PVector`` field.
+
+    :param item_type: The required type for the items in the vector.
+    :param optional: If true, ``None`` can be used as a value for
+        this field.
+    :param initial: Initial value to pass to factory if no value is given
+        for the field.
+
+    :return: A ``field`` containing a ``CheckedPVector`` of the given type.
+    """
+    return _sequence_field(CheckedPVector, item_type, optional, initial,
+                           invariant=invariant,
+                           item_invariant=item_invariant)
+
+
+_valid = lambda item: (True, "")
+
+
+# Global dictionary to hold auto-generated field types: used for unpickling
+_pmap_field_types = {}
+
+def _restore_pmap_field_pickle(key_type, value_type, data):
+    """Unpickling function for auto-generated PMap field types."""
+    type_ = _pmap_field_types[key_type, value_type]
+    return _restore_pickle(type_, data)
+
+def _make_pmap_field_type(key_type, value_type):
+    """Create a subclass of CheckedPMap with the given key and value types."""
+    type_ = _pmap_field_types.get((key_type, value_type))
+    if type_ is not None:
+        return type_
+
+    class TheMap(CheckedPMap):
+        __key_type__ = key_type
+        __value_type__ = value_type
+
+        def __reduce__(self):
+            return (_restore_pmap_field_pickle,
+                    (self.__key_type__, self.__value_type__, dict(self)))
+
+    TheMap.__name__ = "{0}To{1}PMap".format(
+        _types_to_names(TheMap._checked_key_types),
+        _types_to_names(TheMap._checked_value_types))
+    _pmap_field_types[key_type, value_type] = TheMap
+    return TheMap
+
+
+def pmap_field(key_type, value_type, optional=False, invariant=PFIELD_NO_INVARIANT):
+    """
+    Create a checked ``PMap`` field.
+
+    :param key: The required type for the keys of the map.
+    :param value: The required type for the values of the map.
+    :param optional: If true, ``None`` can be used as a value for
+        this field.
+    :param invariant: Pass-through to ``field``.
+
+    :return: A ``field`` containing a ``CheckedPMap``.
+    """
+    TheMap = _make_pmap_field_type(key_type, value_type)
+
+    if optional:
+        def factory(argument):
+            if argument is None:
+                return None
+            else:
+                return TheMap.create(argument)
+    else:
+        factory = TheMap.create
+
+    return field(mandatory=True, initial=TheMap(),
+                 type=optional_type(TheMap) if optional else TheMap,
+                 factory=factory, invariant=invariant)
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_helpers.py b/contrib/python/pyrsistent/py3/pyrsistent/_helpers.py
new file mode 100644
index 00000000000..b44bfc5735b
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_helpers.py
@@ -0,0 +1,101 @@
+import collections
+from functools import wraps
+from pyrsistent._pmap import PMap, pmap
+from pyrsistent._pset import PSet, pset
+from pyrsistent._pvector import PVector, pvector
+
+def freeze(o, strict=True):
+    """
+    Recursively convert simple Python containers into pyrsistent versions
+    of those containers.
+
+    - list is converted to pvector, recursively
+    - dict is converted to pmap, recursively on values (but not keys)
+    - defaultdict is converted to pmap, recursively on values (but not keys)
+    - set is converted to pset, but not recursively
+    - tuple is converted to tuple, recursively.
+
+    If strict == True (default):
+
+    - freeze is called on elements of pvectors
+    - freeze is called on values of pmaps
+
+    Sets and dict keys are not recursively frozen because they do not contain
+    mutable data by convention. The main exception to this rule is that
+    dict keys and set elements are often instances of mutable objects that
+    support hash-by-id, which this function can't convert anyway.
+
+    >>> freeze(set([1, 2]))
+    pset([1, 2])
+    >>> freeze([1, {'a': 3}])
+    pvector([1, pmap({'a': 3})])
+    >>> freeze((1, []))
+    (1, pvector([]))
+    """
+    typ = type(o)
+    if typ is dict or (strict and isinstance(o, PMap)):
+        return pmap({k: freeze(v, strict) for k, v in o.items()})
+    if typ is collections.defaultdict or (strict and isinstance(o, PMap)):
+        return pmap({k: freeze(v, strict) for k, v in o.items()})
+    if typ is list or (strict and isinstance(o, PVector)):
+        curried_freeze = lambda x: freeze(x, strict)
+        return pvector(map(curried_freeze, o))
+    if typ is tuple:
+        curried_freeze = lambda x: freeze(x, strict)
+        return tuple(map(curried_freeze, o))
+    if typ is set:
+        # impossible to have anything that needs freezing inside a set or pset
+        return pset(o)
+    return o
+
+
+def thaw(o, strict=True):
+    """
+    Recursively convert pyrsistent containers into simple Python containers.
+
+    - pvector is converted to list, recursively
+    - pmap is converted to dict, recursively on values (but not keys)
+    - pset is converted to set, but not recursively
+    - tuple is converted to tuple, recursively.
+
+    If strict == True (the default):
+
+    - thaw is called on elements of lists
+    - thaw is called on values in dicts
+
+    >>> from pyrsistent import s, m, v
+    >>> thaw(s(1, 2))
+    {1, 2}
+    >>> thaw(v(1, m(a=3)))
+    [1, {'a': 3}]
+    >>> thaw((1, v()))
+    (1, [])
+    """
+    typ = type(o)
+    if isinstance(o, PVector) or (strict and typ is list):
+        curried_thaw = lambda x: thaw(x, strict)
+        return list(map(curried_thaw, o))
+    if isinstance(o, PMap) or (strict and typ is dict):
+        return {k: thaw(v, strict) for k, v in o.items()}
+    if typ is tuple:
+        curried_thaw = lambda x: thaw(x, strict)
+        return tuple(map(curried_thaw, o))
+    if isinstance(o, PSet):
+        # impossible to thaw inside psets or sets
+        return set(o)
+    return o
+
+
+def mutant(fn):
+    """
+    Convenience decorator to isolate mutation to within the decorated function (with respect
+    to the input arguments).
+
+    All arguments to the decorated function will be frozen so that they are guaranteed not to change.
+    The return value is also frozen.
+    """
+    @wraps(fn)
+    def inner_f(*args, **kwargs):
+        return freeze(fn(*[freeze(e) for e in args], **dict(freeze(item) for item in kwargs.items())))
+
+    return inner_f
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_immutable.py b/contrib/python/pyrsistent/py3/pyrsistent/_immutable.py
new file mode 100644
index 00000000000..d23deca7742
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_immutable.py
@@ -0,0 +1,97 @@
+import sys
+
+
+def immutable(members='', name='Immutable', verbose=False):
+    """
+    Produces a class that either can be used standalone or as a base class for persistent classes.
+
+    This is a thin wrapper around a named tuple.
+
+    Constructing a type and using it to instantiate objects:
+
+    >>> Point = immutable('x, y', name='Point')
+    >>> p = Point(1, 2)
+    >>> p2 = p.set(x=3)
+    >>> p
+    Point(x=1, y=2)
+    >>> p2
+    Point(x=3, y=2)
+
+    Inheriting from a constructed type. In this case no type name needs to be supplied:
+
+    >>> class PositivePoint(immutable('x, y')):
+    ...     __slots__ = tuple()
+    ...     def __new__(cls, x, y):
+    ...         if x > 0 and y > 0:
+    ...             return super(PositivePoint, cls).__new__(cls, x, y)
+    ...         raise Exception('Coordinates must be positive!')
+    ...
+    >>> p = PositivePoint(1, 2)
+    >>> p.set(x=3)
+    PositivePoint(x=3, y=2)
+    >>> p.set(y=-3)
+    Traceback (most recent call last):
+    Exception: Coordinates must be positive!
+
+    The persistent class also supports the notion of frozen members. The value of a frozen member
+    cannot be updated. For example it could be used to implement an ID that should remain the same
+    over time. A frozen member is denoted by a trailing underscore.
+
+    >>> Point = immutable('x, y, id_', name='Point')
+    >>> p = Point(1, 2, id_=17)
+    >>> p.set(x=3)
+    Point(x=3, y=2, id_=17)
+    >>> p.set(id_=18)
+    Traceback (most recent call last):
+    AttributeError: Cannot set frozen members id_
+    """
+
+    if isinstance(members, str):
+        members = members.replace(',', ' ').split()
+
+    def frozen_member_test():
+        frozen_members = ["'%s'" % f for f in members if f.endswith('_')]
+        if frozen_members:
+            return """
+        frozen_fields = fields_to_modify & set([{frozen_members}])
+        if frozen_fields:
+            raise AttributeError('Cannot set frozen members %s' % ', '.join(frozen_fields))
+            """.format(frozen_members=', '.join(frozen_members))
+
+        return ''
+
+    quoted_members = ', '.join("'%s'" % m for m in members)
+    template = """
+class {class_name}(namedtuple('ImmutableBase', [{quoted_members}])):
+    __slots__ = tuple()
+
+    def __repr__(self):
+        return super({class_name}, self).__repr__().replace('ImmutableBase', self.__class__.__name__)
+
+    def set(self, **kwargs):
+        if not kwargs:
+            return self
+
+        fields_to_modify = set(kwargs.keys())
+        if not fields_to_modify <= {member_set}:
+            raise AttributeError("'%s' is not a member" % ', '.join(fields_to_modify - {member_set}))
+
+        {frozen_member_test}
+
+        return self.__class__.__new__(self.__class__, *map(kwargs.pop, [{quoted_members}], self))
+""".format(quoted_members=quoted_members,
+               member_set="set([%s])" % quoted_members if quoted_members else 'set()',
+               frozen_member_test=frozen_member_test(),
+               class_name=name)
+
+    if verbose:
+        print(template)
+
+    from collections import namedtuple
+    namespace = dict(namedtuple=namedtuple, __name__='pyrsistent_immutable')
+    try:
+        exec(template, namespace)
+    except SyntaxError as e:
+        raise SyntaxError(str(e) + ':\n' + template) from e
+
+    return namespace[name]
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_pbag.py b/contrib/python/pyrsistent/py3/pyrsistent/_pbag.py
new file mode 100644
index 00000000000..50001f1919e
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_pbag.py
@@ -0,0 +1,270 @@
+from collections.abc import Container, Iterable, Sized, Hashable
+from functools import reduce
+from typing import Generic, TypeVar
+from pyrsistent._pmap import pmap
+
+T_co = TypeVar('T_co', covariant=True)
+
+
+def _add_to_counters(counters, element):
+    return counters.set(element, counters.get(element, 0) + 1)
+
+
+class PBag(Generic[T_co]):
+    """
+    A persistent bag/multiset type.
+
+    Requires elements to be hashable, and allows duplicates, but has no
+    ordering. Bags are hashable.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`b`
+    or :py:func:`pbag` to create an instance.
+
+    Some examples:
+
+    >>> s = pbag([1, 2, 3, 1])
+    >>> s2 = s.add(4)
+    >>> s3 = s2.remove(1)
+    >>> s
+    pbag([1, 1, 2, 3])
+    >>> s2
+    pbag([1, 1, 2, 3, 4])
+    >>> s3
+    pbag([1, 2, 3, 4])
+    """
+
+    __slots__ = ('_counts', '__weakref__')
+
+    def __init__(self, counts):
+        self._counts = counts
+
+    def add(self, element):
+        """
+        Add an element to the bag.
+
+        >>> s = pbag([1])
+        >>> s2 = s.add(1)
+        >>> s3 = s.add(2)
+        >>> s2
+        pbag([1, 1])
+        >>> s3
+        pbag([1, 2])
+        """
+        return PBag(_add_to_counters(self._counts, element))
+
+    def update(self, iterable):
+        """
+        Update bag with all elements in iterable.
+
+        >>> s = pbag([1])
+        >>> s.update([1, 2])
+        pbag([1, 1, 2])
+        """
+        if iterable:
+            return PBag(reduce(_add_to_counters, iterable, self._counts))
+
+        return self
+
+    def remove(self, element):
+        """
+        Remove an element from the bag.
+
+        >>> s = pbag([1, 1, 2])
+        >>> s2 = s.remove(1)
+        >>> s3 = s.remove(2)
+        >>> s2
+        pbag([1, 2])
+        >>> s3
+        pbag([1, 1])
+        """
+        if element not in self._counts:
+            raise KeyError(element)
+        elif self._counts[element] == 1:
+            newc = self._counts.remove(element)
+        else:
+            newc = self._counts.set(element, self._counts[element] - 1)
+        return PBag(newc)
+
+    def count(self, element):
+        """
+        Return the number of times an element appears.
+
+
+        >>> pbag([]).count('non-existent')
+        0
+        >>> pbag([1, 1, 2]).count(1)
+        2
+        """
+        return self._counts.get(element, 0)
+
+    def __len__(self):
+        """
+        Return the length including duplicates.
+
+        >>> len(pbag([1, 1, 2]))
+        3
+        """
+        return sum(self._counts.itervalues())
+
+    def __iter__(self):
+        """
+        Return an iterator of all elements, including duplicates.
+
+        >>> list(pbag([1, 1, 2]))
+        [1, 1, 2]
+        >>> list(pbag([1, 2]))
+        [1, 2]
+        """
+        for elt, count in self._counts.iteritems():
+            for i in range(count):
+                yield elt
+
+    def __contains__(self, elt):
+        """
+        Check if an element is in the bag.
+
+        >>> 1 in pbag([1, 1, 2])
+        True
+        >>> 0 in pbag([1, 2])
+        False
+        """
+        return elt in self._counts
+
+    def __repr__(self):
+        return "pbag({0})".format(list(self))
+
+    def __eq__(self, other):
+        """
+        Check if two bags are equivalent, honoring the number of duplicates,
+        and ignoring insertion order.
+
+        >>> pbag([1, 1, 2]) == pbag([1, 2])
+        False
+        >>> pbag([2, 1, 0]) == pbag([0, 1, 2])
+        True
+        """
+        if type(other) is not PBag:
+            raise TypeError("Can only compare PBag with PBags")
+        return self._counts == other._counts
+
+    def __lt__(self, other):
+        raise TypeError('PBags are not orderable')
+
+    __le__ = __lt__
+    __gt__ = __lt__
+    __ge__ = __lt__
+
+    # Multiset-style operations similar to collections.Counter
+
+    def __add__(self, other):
+        """
+        Combine elements from two PBags.
+
+        >>> pbag([1, 2, 2]) + pbag([2, 3, 3])
+        pbag([1, 2, 2, 2, 3, 3])
+        """
+        if not isinstance(other, PBag):
+            return NotImplemented
+        result = self._counts.evolver()
+        for elem, other_count in other._counts.iteritems():
+            result[elem] = self.count(elem) + other_count
+        return PBag(result.persistent())
+
+    def __sub__(self, other):
+        """
+        Remove elements from one PBag that are present in another.
+
+        >>> pbag([1, 2, 2, 2, 3]) - pbag([2, 3, 3, 4])
+        pbag([1, 2, 2])
+        """
+        if not isinstance(other, PBag):
+            return NotImplemented
+        result = self._counts.evolver()
+        for elem, other_count in other._counts.iteritems():
+            newcount = self.count(elem) - other_count
+            if newcount > 0:
+                result[elem] = newcount
+            elif elem in self:
+                result.remove(elem)
+        return PBag(result.persistent())
+
+    def __or__(self, other):
+        """
+        Union: Keep elements that are present in either of two PBags.
+
+        >>> pbag([1, 2, 2, 2]) | pbag([2, 3, 3])
+        pbag([1, 2, 2, 2, 3, 3])
+        """
+        if not isinstance(other, PBag):
+            return NotImplemented
+        result = self._counts.evolver()
+        for elem, other_count in other._counts.iteritems():
+            count = self.count(elem)
+            newcount = max(count, other_count)
+            result[elem] = newcount
+        return PBag(result.persistent())
+
+    def __and__(self, other):
+        """
+        Intersection: Only keep elements that are present in both PBags.
+
+        >>> pbag([1, 2, 2, 2]) & pbag([2, 3, 3])
+        pbag([2])
+        """
+        if not isinstance(other, PBag):
+            return NotImplemented
+        result = pmap().evolver()
+        for elem, count in self._counts.iteritems():
+            newcount = min(count, other.count(elem))
+            if newcount > 0:
+                result[elem] = newcount
+        return PBag(result.persistent())
+
+    def __hash__(self):
+        """
+        Hash based on value of elements.
+
+        >>> m = pmap({pbag([1, 2]): "it's here!"})
+        >>> m[pbag([2, 1])]
+        "it's here!"
+        >>> pbag([1, 1, 2]) in m
+        False
+        """
+        return hash(self._counts)
+
+
+Container.register(PBag)
+Iterable.register(PBag)
+Sized.register(PBag)
+Hashable.register(PBag)
+
+
+def b(*elements):
+    """
+    Construct a persistent bag.
+
+    Takes an arbitrary number of arguments to insert into the new persistent
+    bag.
+
+    >>> b(1, 2, 3, 2)
+    pbag([1, 2, 2, 3])
+    """
+    return pbag(elements)
+
+
+def pbag(elements):
+    """
+    Convert an iterable to a persistent bag.
+
+    Takes an iterable with elements to insert.
+
+    >>> pbag([1, 2, 3, 2])
+    pbag([1, 2, 2, 3])
+    """
+    if not elements:
+        return _EMPTY_PBAG
+    return PBag(reduce(_add_to_counters, elements, pmap()))
+
+
+_EMPTY_PBAG = PBag(pmap())
+
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_pclass.py b/contrib/python/pyrsistent/py3/pyrsistent/_pclass.py
new file mode 100644
index 00000000000..fd31a95d63a
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_pclass.py
@@ -0,0 +1,262 @@
+from pyrsistent._checked_types import (InvariantException, CheckedType, _restore_pickle, store_invariants)
+from pyrsistent._field_common import (
+    set_fields, check_type, is_field_ignore_extra_complaint, PFIELD_NO_INITIAL, serialize, check_global_invariants
+)
+from pyrsistent._transformations import transform
+
+
+def _is_pclass(bases):
+    return len(bases) == 1 and bases[0] == CheckedType
+
+
+class PClassMeta(type):
+    def __new__(mcs, name, bases, dct):
+        set_fields(dct, bases, name='_pclass_fields')
+        store_invariants(dct, bases, '_pclass_invariants', '__invariant__')
+        dct['__slots__'] = ('_pclass_frozen',) + tuple(key for key in dct['_pclass_fields'])
+
+        # There must only be one __weakref__ entry in the inheritance hierarchy,
+        # lets put it on the top level class.
+        if _is_pclass(bases):
+            dct['__slots__'] += ('__weakref__',)
+
+        return super(PClassMeta, mcs).__new__(mcs, name, bases, dct)
+
+_MISSING_VALUE = object()
+
+
+def _check_and_set_attr(cls, field, name, value, result, invariant_errors):
+    check_type(cls, field, name, value)
+    is_ok, error_code = field.invariant(value)
+    if not is_ok:
+        invariant_errors.append(error_code)
+    else:
+        setattr(result, name, value)
+
+
+class PClass(CheckedType, metaclass=PClassMeta):
+    """
+    A PClass is a python class with a fixed set of specified fields. PClasses are declared as python classes inheriting
+    from PClass. It is defined the same way that PRecords are and behaves like a PRecord in all aspects except that it
+    is not a PMap and hence not a collection but rather a plain Python object.
+
+
+    More documentation and examples of PClass usage is available at https://github.com/tobgu/pyrsistent
+    """
+    def __new__(cls, **kwargs):    # Support *args?
+        result = super(PClass, cls).__new__(cls)
+        factory_fields = kwargs.pop('_factory_fields', None)
+        ignore_extra = kwargs.pop('ignore_extra', None)
+        missing_fields = []
+        invariant_errors = []
+        for name, field in cls._pclass_fields.items():
+            if name in kwargs:
+                if factory_fields is None or name in factory_fields:
+                    if is_field_ignore_extra_complaint(PClass, field, ignore_extra):
+                        value = field.factory(kwargs[name], ignore_extra=ignore_extra)
+                    else:
+                        value = field.factory(kwargs[name])
+                else:
+                    value = kwargs[name]
+                _check_and_set_attr(cls, field, name, value, result, invariant_errors)
+                del kwargs[name]
+            elif field.initial is not PFIELD_NO_INITIAL:
+                initial = field.initial() if callable(field.initial) else field.initial
+                _check_and_set_attr(
+                    cls, field, name, initial, result, invariant_errors)
+            elif field.mandatory:
+                missing_fields.append('{0}.{1}'.format(cls.__name__, name))
+
+        if invariant_errors or missing_fields:
+            raise InvariantException(tuple(invariant_errors), tuple(missing_fields), 'Field invariant failed')
+
+        if kwargs:
+            raise AttributeError("'{0}' are not among the specified fields for {1}".format(
+                ', '.join(kwargs), cls.__name__))
+
+        check_global_invariants(result, cls._pclass_invariants)
+
+        result._pclass_frozen = True
+        return result
+
+    def set(self, *args, **kwargs):
+        """
+        Set a field in the instance. Returns a new instance with the updated value. The original instance remains
+        unmodified. Accepts key-value pairs or single string representing the field name and a value.
+
+        >>> from pyrsistent import PClass, field
+        >>> class AClass(PClass):
+        ...     x = field()
+        ...
+        >>> a = AClass(x=1)
+        >>> a2 = a.set(x=2)
+        >>> a3 = a.set('x', 3)
+        >>> a
+        AClass(x=1)
+        >>> a2
+        AClass(x=2)
+        >>> a3
+        AClass(x=3)
+        """
+        if args:
+            kwargs[args[0]] = args[1]
+
+        factory_fields = set(kwargs)
+
+        for key in self._pclass_fields:
+            if key not in kwargs:
+                value = getattr(self, key, _MISSING_VALUE)
+                if value is not _MISSING_VALUE:
+                    kwargs[key] = value
+
+        return self.__class__(_factory_fields=factory_fields, **kwargs)
+
+    @classmethod
+    def create(cls, kwargs, _factory_fields=None, ignore_extra=False):
+        """
+        Factory method. Will create a new PClass of the current type and assign the values
+        specified in kwargs.
+
+        :param ignore_extra: A boolean which when set to True will ignore any keys which appear in kwargs that are not
+                             in the set of fields on the PClass.
+        """
+        if isinstance(kwargs, cls):
+            return kwargs
+
+        if ignore_extra:
+            kwargs = {k: kwargs[k] for k in cls._pclass_fields if k in kwargs}
+
+        return cls(_factory_fields=_factory_fields, ignore_extra=ignore_extra, **kwargs)
+
+    def serialize(self, format=None):
+        """
+        Serialize the current PClass using custom serializer functions for fields where
+        such have been supplied.
+        """
+        result = {}
+        for name in self._pclass_fields:
+            value = getattr(self, name, _MISSING_VALUE)
+            if value is not _MISSING_VALUE:
+                result[name] = serialize(self._pclass_fields[name].serializer, format, value)
+
+        return result
+
+    def transform(self, *transformations):
+        """
+        Apply transformations to the currency PClass. For more details on transformations see
+        the documentation for PMap. Transformations on PClasses do not support key matching
+        since the PClass is not a collection. Apart from that the transformations available
+        for other persistent types work as expected.
+        """
+        return transform(self, transformations)
+
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            for name in self._pclass_fields:
+                if getattr(self, name, _MISSING_VALUE) != getattr(other, name, _MISSING_VALUE):
+                    return False
+
+            return True
+
+        return NotImplemented
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __hash__(self):
+        # May want to optimize this by caching the hash somehow
+        return hash(tuple((key, getattr(self, key, _MISSING_VALUE)) for key in self._pclass_fields))
+
+    def __setattr__(self, key, value):
+        if getattr(self, '_pclass_frozen', False):
+            raise AttributeError("Can't set attribute, key={0}, value={1}".format(key, value))
+
+        super(PClass, self).__setattr__(key, value)
+
+    def __delattr__(self, key):
+            raise AttributeError("Can't delete attribute, key={0}, use remove()".format(key))
+
+    def _to_dict(self):
+        result = {}
+        for key in self._pclass_fields:
+            value = getattr(self, key, _MISSING_VALUE)
+            if value is not _MISSING_VALUE:
+                result[key] = value
+
+        return result
+
+    def __repr__(self):
+        return "{0}({1})".format(self.__class__.__name__,
+                                 ', '.join('{0}={1}'.format(k, repr(v)) for k, v in self._to_dict().items()))
+
+    def __reduce__(self):
+        # Pickling support
+        data = dict((key, getattr(self, key)) for key in self._pclass_fields if hasattr(self, key))
+        return _restore_pickle, (self.__class__, data,)
+
+    def evolver(self):
+        """
+        Returns an evolver for this object.
+        """
+        return _PClassEvolver(self, self._to_dict())
+
+    def remove(self, name):
+        """
+        Remove attribute given by name from the current instance. Raises AttributeError if the
+        attribute doesn't exist.
+        """
+        evolver = self.evolver()
+        del evolver[name]
+        return evolver.persistent()
+
+
+class _PClassEvolver(object):
+    __slots__ = ('_pclass_evolver_original', '_pclass_evolver_data', '_pclass_evolver_data_is_dirty', '_factory_fields')
+
+    def __init__(self, original, initial_dict):
+        self._pclass_evolver_original = original
+        self._pclass_evolver_data = initial_dict
+        self._pclass_evolver_data_is_dirty = False
+        self._factory_fields = set()
+
+    def __getitem__(self, item):
+        return self._pclass_evolver_data[item]
+
+    def set(self, key, value):
+        if self._pclass_evolver_data.get(key, _MISSING_VALUE) is not value:
+            self._pclass_evolver_data[key] = value
+            self._factory_fields.add(key)
+            self._pclass_evolver_data_is_dirty = True
+
+        return self
+
+    def __setitem__(self, key, value):
+        self.set(key, value)
+
+    def remove(self, item):
+        if item in self._pclass_evolver_data:
+            del self._pclass_evolver_data[item]
+            self._factory_fields.discard(item)
+            self._pclass_evolver_data_is_dirty = True
+            return self
+
+        raise AttributeError(item)
+
+    def __delitem__(self, item):
+        self.remove(item)
+
+    def persistent(self):
+        if self._pclass_evolver_data_is_dirty:
+            return self._pclass_evolver_original.__class__(_factory_fields=self._factory_fields,
+                                                           **self._pclass_evolver_data)
+
+        return self._pclass_evolver_original
+
+    def __setattr__(self, key, value):
+        if key not in self.__slots__:
+            self.set(key, value)
+        else:
+            super(_PClassEvolver, self).__setattr__(key, value)
+
+    def __getattr__(self, item):
+        return self[item]
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_pdeque.py b/contrib/python/pyrsistent/py3/pyrsistent/_pdeque.py
new file mode 100644
index 00000000000..0f25936af7a
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_pdeque.py
@@ -0,0 +1,379 @@
+from collections.abc import Sequence, Hashable
+from itertools import islice, chain
+from numbers import Integral
+from typing import TypeVar, Generic
+from pyrsistent._plist import plist
+
+T_co = TypeVar('T_co', covariant=True)
+
+
+class PDeque(Generic[T_co]):
+    """
+    Persistent double ended queue (deque). Allows quick appends and pops in both ends. Implemented
+    using two persistent lists.
+
+    A maximum length can be specified to create a bounded queue.
+
+    Fully supports the Sequence and Hashable protocols including indexing and slicing but
+    if you need fast random access go for the PVector instead.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`dq` or :py:func:`pdeque` to
+    create an instance.
+
+    Some examples:
+
+    >>> x = pdeque([1, 2, 3])
+    >>> x.left
+    1
+    >>> x.right
+    3
+    >>> x[0] == x.left
+    True
+    >>> x[-1] == x.right
+    True
+    >>> x.pop()
+    pdeque([1, 2])
+    >>> x.pop() == x[:-1]
+    True
+    >>> x.popleft()
+    pdeque([2, 3])
+    >>> x.append(4)
+    pdeque([1, 2, 3, 4])
+    >>> x.appendleft(4)
+    pdeque([4, 1, 2, 3])
+
+    >>> y = pdeque([1, 2, 3], maxlen=3)
+    >>> y.append(4)
+    pdeque([2, 3, 4], maxlen=3)
+    >>> y.appendleft(4)
+    pdeque([4, 1, 2], maxlen=3)
+    """
+    __slots__ = ('_left_list', '_right_list', '_length', '_maxlen', '__weakref__')
+
+    def __new__(cls, left_list, right_list, length, maxlen=None):
+        instance = super(PDeque, cls).__new__(cls)
+        instance._left_list = left_list
+        instance._right_list = right_list
+        instance._length = length
+
+        if maxlen is not None:
+            if not isinstance(maxlen, Integral):
+                raise TypeError('An integer is required as maxlen')
+
+            if maxlen < 0:
+                raise ValueError("maxlen must be non-negative")
+
+        instance._maxlen = maxlen
+        return instance
+
+    @property
+    def right(self):
+        """
+        Rightmost element in dqueue.
+        """
+        return PDeque._tip_from_lists(self._right_list, self._left_list)
+
+    @property
+    def left(self):
+        """
+        Leftmost element in dqueue.
+        """
+        return PDeque._tip_from_lists(self._left_list, self._right_list)
+
+    @staticmethod
+    def _tip_from_lists(primary_list, secondary_list):
+        if primary_list:
+            return primary_list.first
+
+        if secondary_list:
+            return secondary_list[-1]
+
+        raise IndexError('No elements in empty deque')
+
+    def __iter__(self):
+        return chain(self._left_list, self._right_list.reverse())
+
+    def __repr__(self):
+        return "pdeque({0}{1})".format(list(self),
+                                       ', maxlen={0}'.format(self._maxlen) if self._maxlen is not None else '')
+    __str__ = __repr__
+
+    @property
+    def maxlen(self):
+        """
+        Maximum length of the queue.
+        """
+        return self._maxlen
+
+    def pop(self, count=1):
+        """
+        Return new deque with rightmost element removed. Popping the empty queue
+        will return the empty queue. A optional count can be given to indicate the
+        number of elements to pop. Popping with a negative index is the same as
+        popleft. Executes in amortized O(k) where k is the number of elements to pop.
+
+        >>> pdeque([1, 2]).pop()
+        pdeque([1])
+        >>> pdeque([1, 2]).pop(2)
+        pdeque([])
+        >>> pdeque([1, 2]).pop(-1)
+        pdeque([2])
+        """
+        if count < 0:
+            return self.popleft(-count)
+
+        new_right_list, new_left_list = PDeque._pop_lists(self._right_list, self._left_list, count)
+        return PDeque(new_left_list, new_right_list, max(self._length - count, 0), self._maxlen)
+
+    def popleft(self, count=1):
+        """
+        Return new deque with leftmost element removed. Otherwise functionally
+        equivalent to pop().
+
+        >>> pdeque([1, 2]).popleft()
+        pdeque([2])
+        """
+        if count < 0:
+            return self.pop(-count)
+
+        new_left_list, new_right_list = PDeque._pop_lists(self._left_list, self._right_list, count)
+        return PDeque(new_left_list, new_right_list, max(self._length - count, 0), self._maxlen)
+
+    @staticmethod
+    def _pop_lists(primary_list, secondary_list, count):
+        new_primary_list = primary_list
+        new_secondary_list = secondary_list
+
+        while count > 0 and (new_primary_list or new_secondary_list):
+            count -= 1
+            if new_primary_list.rest:
+                new_primary_list = new_primary_list.rest
+            elif new_primary_list:
+                new_primary_list = new_secondary_list.reverse()
+                new_secondary_list = plist()
+            else:
+                new_primary_list = new_secondary_list.reverse().rest
+                new_secondary_list = plist()
+
+        return new_primary_list, new_secondary_list
+
+    def _is_empty(self):
+        return not self._left_list and not self._right_list
+
+    def __lt__(self, other):
+        if not isinstance(other, PDeque):
+            return NotImplemented
+
+        return tuple(self) < tuple(other)
+
+    def __eq__(self, other):
+        if not isinstance(other, PDeque):
+            return NotImplemented
+
+        if tuple(self) == tuple(other):
+            # Sanity check of the length value since it is redundant (there for performance)
+            assert len(self) == len(other)
+            return True
+
+        return False
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+    def __len__(self):
+        return self._length
+
+    def append(self, elem):
+        """
+        Return new deque with elem as the rightmost element.
+
+        >>> pdeque([1, 2]).append(3)
+        pdeque([1, 2, 3])
+        """
+        new_left_list, new_right_list, new_length = self._append(self._left_list, self._right_list, elem)
+        return PDeque(new_left_list, new_right_list, new_length, self._maxlen)
+
+    def appendleft(self, elem):
+        """
+        Return new deque with elem as the leftmost element.
+
+        >>> pdeque([1, 2]).appendleft(3)
+        pdeque([3, 1, 2])
+        """
+        new_right_list, new_left_list, new_length = self._append(self._right_list, self._left_list, elem)
+        return PDeque(new_left_list, new_right_list, new_length, self._maxlen)
+
+    def _append(self, primary_list, secondary_list, elem):
+        if self._maxlen is not None and self._length == self._maxlen:
+            if self._maxlen == 0:
+                return primary_list, secondary_list, 0
+            new_primary_list, new_secondary_list = PDeque._pop_lists(primary_list, secondary_list, 1)
+            return new_primary_list, new_secondary_list.cons(elem), self._length
+
+        return primary_list, secondary_list.cons(elem), self._length + 1
+
+    @staticmethod
+    def _extend_list(the_list, iterable):
+        count = 0
+        for elem in iterable:
+            the_list = the_list.cons(elem)
+            count += 1
+
+        return the_list, count
+
+    def _extend(self, primary_list, secondary_list, iterable):
+        new_primary_list, extend_count = PDeque._extend_list(primary_list, iterable)
+        new_secondary_list = secondary_list
+        current_len = self._length + extend_count
+        if self._maxlen is not None and current_len > self._maxlen:
+            pop_len = current_len - self._maxlen
+            new_secondary_list, new_primary_list = PDeque._pop_lists(new_secondary_list, new_primary_list, pop_len)
+            extend_count -= pop_len
+
+        return new_primary_list, new_secondary_list, extend_count
+
+    def extend(self, iterable):
+        """
+        Return new deque with all elements of iterable appended to the right.
+
+        >>> pdeque([1, 2]).extend([3, 4])
+        pdeque([1, 2, 3, 4])
+        """
+        new_right_list, new_left_list, extend_count = self._extend(self._right_list, self._left_list, iterable)
+        return PDeque(new_left_list, new_right_list, self._length + extend_count, self._maxlen)
+
+    def extendleft(self, iterable):
+        """
+        Return new deque with all elements of iterable appended to the left.
+
+        NB! The elements will be inserted in reverse order compared to the order in the iterable.
+
+        >>> pdeque([1, 2]).extendleft([3, 4])
+        pdeque([4, 3, 1, 2])
+        """
+        new_left_list, new_right_list, extend_count = self._extend(self._left_list, self._right_list, iterable)
+        return PDeque(new_left_list, new_right_list, self._length + extend_count, self._maxlen)
+
+    def count(self, elem):
+        """
+        Return the number of elements equal to elem present in the queue
+
+        >>> pdeque([1, 2, 1]).count(1)
+        2
+        """
+        return self._left_list.count(elem) + self._right_list.count(elem)
+
+    def remove(self, elem):
+        """
+        Return new deque with first element from left equal to elem removed. If no such element is found
+        a ValueError is raised.
+
+        >>> pdeque([2, 1, 2]).remove(2)
+        pdeque([1, 2])
+        """
+        try:
+            return PDeque(self._left_list.remove(elem), self._right_list, self._length - 1)
+        except ValueError:
+            # Value not found in left list, try the right list
+            try:
+                # This is severely inefficient with a double reverse, should perhaps implement a remove_last()?
+                return PDeque(self._left_list,
+                              self._right_list.reverse().remove(elem).reverse(), self._length - 1)
+            except ValueError as e:
+                raise ValueError('{0} not found in PDeque'.format(elem)) from e
+
+    def reverse(self):
+        """
+        Return reversed deque.
+
+        >>> pdeque([1, 2, 3]).reverse()
+        pdeque([3, 2, 1])
+
+        Also supports the standard python reverse function.
+
+        >>> reversed(pdeque([1, 2, 3]))
+        pdeque([3, 2, 1])
+        """
+        return PDeque(self._right_list, self._left_list, self._length)
+    __reversed__ = reverse
+
+    def rotate(self, steps):
+        """
+        Return deque with elements rotated steps steps.
+
+        >>> x = pdeque([1, 2, 3])
+        >>> x.rotate(1)
+        pdeque([3, 1, 2])
+        >>> x.rotate(-2)
+        pdeque([3, 1, 2])
+        """
+        popped_deque = self.pop(steps)
+        if steps >= 0:
+            return popped_deque.extendleft(islice(self.reverse(), steps))
+
+        return popped_deque.extend(islice(self, -steps))
+
+    def __reduce__(self):
+        # Pickling support
+        return pdeque, (list(self), self._maxlen)
+
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            if index.step is not None and index.step != 1:
+                # Too difficult, no structural sharing possible
+                return pdeque(tuple(self)[index], maxlen=self._maxlen)
+
+            result = self
+            if index.start is not None:
+                result = result.popleft(index.start % self._length)
+            if index.stop is not None:
+                result = result.pop(self._length - (index.stop % self._length))
+
+            return result
+
+        if not isinstance(index, Integral):
+            raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)
+
+        if index >= 0:
+            return self.popleft(index).left
+
+        shifted = len(self) + index
+        if shifted < 0:
+            raise IndexError(
+                "pdeque index {0} out of range {1}".format(index, len(self)),
+            )
+        return self.popleft(shifted).left
+
+    index = Sequence.index
+
+Sequence.register(PDeque)
+Hashable.register(PDeque)
+
+
+def pdeque(iterable=(), maxlen=None):
+    """
+    Return deque containing the elements of iterable. If maxlen is specified then
+    len(iterable) - maxlen elements are discarded from the left to if len(iterable) > maxlen.
+
+    >>> pdeque([1, 2, 3])
+    pdeque([1, 2, 3])
+    >>> pdeque([1, 2, 3, 4], maxlen=2)
+    pdeque([3, 4], maxlen=2)
+    """
+    t = tuple(iterable)
+    if maxlen is not None:
+        t = t[-maxlen:]
+    length = len(t)
+    pivot = int(length / 2)
+    left = plist(t[:pivot])
+    right = plist(t[pivot:], reverse=True)
+    return PDeque(left, right, length, maxlen)
+
+def dq(*elements):
+    """
+    Return deque containing all arguments.
+
+    >>> dq(1, 2, 3)
+    pdeque([1, 2, 3])
+    """
+    return pdeque(elements)
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_plist.py b/contrib/python/pyrsistent/py3/pyrsistent/_plist.py
new file mode 100644
index 00000000000..322e15d649f
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_plist.py
@@ -0,0 +1,316 @@
+from collections.abc import Sequence, Hashable
+from numbers import Integral
+from functools import reduce
+from typing import Generic, TypeVar
+
+T_co = TypeVar('T_co', covariant=True)
+
+
+class _PListBuilder(object):
+    """
+    Helper class to allow construction of a list without
+    having to reverse it in the end.
+    """
+    __slots__ = ('_head', '_tail')
+
+    def __init__(self):
+        self._head = _EMPTY_PLIST
+        self._tail = _EMPTY_PLIST
+
+    def _append(self, elem, constructor):
+        if not self._tail:
+            self._head = constructor(elem)
+            self._tail = self._head
+        else:
+            self._tail.rest = constructor(elem)
+            self._tail = self._tail.rest
+
+        return self._head
+
+    def append_elem(self, elem):
+        return self._append(elem, lambda e: PList(e, _EMPTY_PLIST))
+
+    def append_plist(self, pl):
+        return self._append(pl, lambda l: l)
+
+    def build(self):
+        return self._head
+
+
+class _PListBase(object):
+    __slots__ = ('__weakref__',)
+
+    # Selected implementations can be taken straight from the Sequence
+    # class, other are less suitable. Especially those that work with
+    # index lookups.
+    count = Sequence.count
+    index = Sequence.index
+
+    def __reduce__(self):
+        # Pickling support
+        return plist, (list(self),)
+
+    def __len__(self):
+        """
+        Return the length of the list, computed by traversing it.
+
+        This is obviously O(n) but with the current implementation
+        where a list is also a node the overhead of storing the length
+        in every node would be quite significant.
+        """
+        return sum(1 for _ in self)
+
+    def __repr__(self):
+        return "plist({0})".format(list(self))
+    __str__ = __repr__
+
+    def cons(self, elem):
+        """
+        Return a new list with elem inserted as new head.
+
+        >>> plist([1, 2]).cons(3)
+        plist([3, 1, 2])
+        """
+        return PList(elem, self)
+
+    def mcons(self, iterable):
+        """
+        Return a new list with all elements of iterable repeatedly cons:ed to the current list.
+        NB! The elements will be inserted in the reverse order of the iterable.
+        Runs in O(len(iterable)).
+
+        >>> plist([1, 2]).mcons([3, 4])
+        plist([4, 3, 1, 2])
+        """
+        head = self
+        for elem in iterable:
+            head = head.cons(elem)
+
+        return head
+
+    def reverse(self):
+        """
+        Return a reversed version of list. Runs in O(n) where n is the length of the list.
+
+        >>> plist([1, 2, 3]).reverse()
+        plist([3, 2, 1])
+
+        Also supports the standard reversed function.
+
+        >>> reversed(plist([1, 2, 3]))
+        plist([3, 2, 1])
+        """
+        result = plist()
+        head = self
+        while head:
+            result = result.cons(head.first)
+            head = head.rest
+
+        return result
+    __reversed__ = reverse
+
+    def split(self, index):
+        """
+        Spilt the list at position specified by index. Returns a tuple containing the
+        list up until index and the list after the index. Runs in O(index).
+
+        >>> plist([1, 2, 3, 4]).split(2)
+        (plist([1, 2]), plist([3, 4]))
+        """
+        lb = _PListBuilder()
+        right_list = self
+        i = 0
+        while right_list and i < index:
+            lb.append_elem(right_list.first)
+            right_list = right_list.rest
+            i += 1
+
+        if not right_list:
+            # Just a small optimization in the cases where no split occurred
+            return self, _EMPTY_PLIST
+
+        return lb.build(), right_list
+
+    def __iter__(self):
+        li = self
+        while li:
+            yield li.first
+            li = li.rest
+
+    def __lt__(self, other):
+        if not isinstance(other, _PListBase):
+            return NotImplemented
+
+        return tuple(self) < tuple(other)
+
+    def __eq__(self, other):
+        """
+        Traverses the lists, checking equality of elements.
+
+        This is an O(n) operation, but preserves the standard semantics of list equality.
+        """
+        if not isinstance(other, _PListBase):
+            return NotImplemented
+
+        self_head = self
+        other_head = other
+        while self_head and other_head:
+            if not self_head.first == other_head.first:
+                return False
+            self_head = self_head.rest
+            other_head = other_head.rest
+
+        return not self_head and not other_head
+
+    def __getitem__(self, index):
+        # Don't use this this data structure if you plan to do a lot of indexing, it is
+        # very inefficient! Use a PVector instead!
+
+        if isinstance(index, slice):
+            if index.start is not None and index.stop is None and (index.step is None or index.step == 1):
+                return self._drop(index.start)
+
+            # Take the easy way out for all other slicing cases, not much structural reuse possible anyway
+            return plist(tuple(self)[index])
+
+        if not isinstance(index, Integral):
+            raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)
+
+        if index < 0:
+            # NB: O(n)!
+            index += len(self)
+
+        try:
+            return self._drop(index).first
+        except AttributeError as e:
+            raise IndexError("PList index out of range") from e
+
+    def _drop(self, count):
+        if count < 0:
+            raise IndexError("PList index out of range")
+
+        head = self
+        while count > 0:
+            head = head.rest
+            count -= 1
+
+        return head
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+    def remove(self, elem):
+        """
+        Return new list with first element equal to elem removed. O(k) where k is the position
+        of the element that is removed.
+
+        Raises ValueError if no matching element is found.
+
+        >>> plist([1, 2, 1]).remove(1)
+        plist([2, 1])
+        """
+
+        builder = _PListBuilder()
+        head = self
+        while head:
+            if head.first == elem:
+                return builder.append_plist(head.rest)
+
+            builder.append_elem(head.first)
+            head = head.rest
+
+        raise ValueError('{0} not found in PList'.format(elem))
+
+
+class PList(Generic[T_co], _PListBase):
+    """
+    Classical Lisp style singly linked list. Adding elements to the head using cons is O(1).
+    Element access is O(k) where k is the position of the element in the list. Taking the
+    length of the list is O(n).
+
+    Fully supports the Sequence and Hashable protocols including indexing and slicing but
+    if you need fast random access go for the PVector instead.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`l` or :py:func:`plist` to
+    create an instance.
+
+    Some examples:
+
+    >>> x = plist([1, 2])
+    >>> y = x.cons(3)
+    >>> x
+    plist([1, 2])
+    >>> y
+    plist([3, 1, 2])
+    >>> y.first
+    3
+    >>> y.rest == x
+    True
+    >>> y[:2]
+    plist([3, 1])
+    """
+    __slots__ = ('first', 'rest')
+
+    def __new__(cls, first, rest):
+        instance = super(PList, cls).__new__(cls)
+        instance.first = first
+        instance.rest = rest
+        return instance
+
+    def __bool__(self):
+        return True
+    __nonzero__ = __bool__
+
+
+Sequence.register(PList)
+Hashable.register(PList)
+
+
+class _EmptyPList(_PListBase):
+    __slots__ = ()
+
+    def __bool__(self):
+        return False
+    __nonzero__ = __bool__
+
+    @property
+    def first(self):
+        raise AttributeError("Empty PList has no first")
+
+    @property
+    def rest(self):
+        return self
+
+
+Sequence.register(_EmptyPList)
+Hashable.register(_EmptyPList)
+
+_EMPTY_PLIST = _EmptyPList()
+
+
+def plist(iterable=(), reverse=False):
+    """
+    Creates a new persistent list containing all elements of iterable.
+    Optional parameter reverse specifies if the elements should be inserted in
+    reverse order or not.
+
+    >>> plist([1, 2, 3])
+    plist([1, 2, 3])
+    >>> plist([1, 2, 3], reverse=True)
+    plist([3, 2, 1])
+    """
+    if not reverse:
+        iterable = list(iterable)
+        iterable.reverse()
+
+    return reduce(lambda pl, elem: pl.cons(elem), iterable, _EMPTY_PLIST)
+
+
+def l(*elements):
+    """
+    Creates a new persistent list containing all arguments.
+
+    >>> l(1, 2, 3)
+    plist([1, 2, 3])
+    """
+    return plist(elements)
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_pmap.py b/contrib/python/pyrsistent/py3/pyrsistent/_pmap.py
new file mode 100644
index 00000000000..0d82c4386a7
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_pmap.py
@@ -0,0 +1,583 @@
+from collections.abc import Mapping, Hashable
+from itertools import chain
+from typing import Generic, TypeVar
+
+from pyrsistent._pvector import pvector
+from pyrsistent._transformations import transform
+
+KT = TypeVar('KT')
+VT_co = TypeVar('VT_co', covariant=True)
+class PMapView:
+    """View type for the persistent map/dict type `PMap`.
+
+    Provides an equivalent of Python's built-in `dict_values` and `dict_items`
+    types that result from expreessions such as `{}.values()` and
+    `{}.items()`. The equivalent for `{}.keys()` is absent because the keys are
+    instead represented by a `PSet` object, which can be created in `O(1)` time.
+
+    The `PMapView` class is overloaded by the `PMapValues` and `PMapItems`
+    classes which handle the specific case of values and items, respectively
+
+    Parameters
+    ----------
+    m : mapping
+        The mapping/dict-like object of which a view is to be created. This
+        should generally be a `PMap` object.
+    """
+    # The public methods that use the above.
+    def __init__(self, m):
+        # Make sure this is a persistnt map
+        if not isinstance(m, PMap):
+            # We can convert mapping objects into pmap objects, I guess (but why?)
+            if isinstance(m, Mapping):
+                m = pmap(m)
+            else:
+                raise TypeError("PViewMap requires a Mapping object")
+        object.__setattr__(self, '_map', m)
+
+    def __len__(self):
+        return len(self._map)
+
+    def __setattr__(self, k, v):
+        raise TypeError("%s is immutable" % (type(self),))
+
+    def __reversed__(self):
+        raise TypeError("Persistent maps are not reversible")
+
+class PMapValues(PMapView):
+    """View type for the values of the persistent map/dict type `PMap`.
+
+    Provides an equivalent of Python's built-in `dict_values` type that result
+    from expreessions such as `{}.values()`. See also `PMapView`.
+
+    Parameters
+    ----------
+    m : mapping
+        The mapping/dict-like object of which a view is to be created. This
+        should generally be a `PMap` object.
+    """
+    def __iter__(self):
+        return self._map.itervalues()
+
+    def __contains__(self, arg):
+        return arg in self._map.itervalues()
+
+    # The str and repr methods imitate the dict_view style currently.
+    def __str__(self):
+        return f"pmap_values({list(iter(self))})"
+    
+    def __repr__(self):
+        return f"pmap_values({list(iter(self))})"
+    
+    def __eq__(self, x):
+        # For whatever reason, dict_values always seem to return False for ==
+        # (probably it's not implemented), so we mimic that.
+        if x is self: return True
+        else: return False
+    
+class PMapItems(PMapView):
+    """View type for the items of the persistent map/dict type `PMap`.
+
+    Provides an equivalent of Python's built-in `dict_items` type that result
+    from expreessions such as `{}.items()`. See also `PMapView`.
+
+    Parameters
+    ----------
+    m : mapping
+        The mapping/dict-like object of which a view is to be created. This
+        should generally be a `PMap` object.
+    """
+    def __iter__(self):
+        return self._map.iteritems()
+
+    def __contains__(self, arg):
+        try: (k,v) = arg
+        except Exception: return False
+        return k in self._map and self._map[k] == v
+
+    # The str and repr methods mitate the dict_view style currently.
+    def __str__(self):
+        return f"pmap_items({list(iter(self))})"
+    
+    def __repr__(self):
+        return f"pmap_items({list(iter(self))})"
+        
+    def __eq__(self, x):
+        if x is self: return True
+        elif not isinstance(x, type(self)): return False
+        else: return self._map == x._map
+
+class PMap(Generic[KT, VT_co]):
+    """
+    Persistent map/dict. Tries to follow the same naming conventions as the built in dict where feasible.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`m` or :py:func:`pmap` to
+    create an instance.
+
+    Was originally written as a very close copy of the Clojure equivalent but was later rewritten to closer
+    re-assemble the python dict. This means that a sparse vector (a PVector) of buckets is used. The keys are
+    hashed and the elements inserted at position hash % len(bucket_vector). Whenever the map size exceeds 2/3 of
+    the containing vectors size the map is reallocated to a vector of double the size. This is done to avoid
+    excessive hash collisions.
+
+    This structure corresponds most closely to the built in dict type and is intended as a replacement. Where the
+    semantics are the same (more or less) the same function names have been used but for some cases it is not possible,
+    for example assignments and deletion of values.
+
+    PMap implements the Mapping protocol and is Hashable. It also supports dot-notation for
+    element access.
+
+    Random access and insert is log32(n) where n is the size of the map.
+
+    The following are examples of some common operations on persistent maps
+
+    >>> m1 = m(a=1, b=3)
+    >>> m2 = m1.set('c', 3)
+    >>> m3 = m2.remove('a')
+    >>> m1 == {'a': 1, 'b': 3}
+    True
+    >>> m2 == {'a': 1, 'b': 3, 'c': 3}
+    True
+    >>> m3 == {'b': 3, 'c': 3}
+    True
+    >>> m3['c']
+    3
+    >>> m3.c
+    3
+    """
+    __slots__ = ('_size', '_buckets', '__weakref__', '_cached_hash')
+
+    def __new__(cls, size, buckets):
+        self = super(PMap, cls).__new__(cls)
+        self._size = size
+        self._buckets = buckets
+        return self
+
+    @staticmethod
+    def _get_bucket(buckets, key):
+        index = hash(key) % len(buckets)
+        bucket = buckets[index]
+        return index, bucket
+
+    @staticmethod
+    def _getitem(buckets, key):
+        _, bucket = PMap._get_bucket(buckets, key)
+        if bucket:
+            for k, v in bucket:
+                if k == key:
+                    return v
+
+        raise KeyError(key)
+
+    def __getitem__(self, key):
+        return PMap._getitem(self._buckets, key)
+
+    @staticmethod
+    def _contains(buckets, key):
+        _, bucket = PMap._get_bucket(buckets, key)
+        if bucket:
+            for k, _ in bucket:
+                if k == key:
+                    return True
+
+            return False
+
+        return False
+
+    def __contains__(self, key):
+        return self._contains(self._buckets, key)
+
+    get = Mapping.get
+
+    def __iter__(self):
+        return self.iterkeys()
+
+    # If this method is not defined, then reversed(pmap) will attempt to reverse
+    # the map using len() and getitem, usually resulting in a mysterious
+    # KeyError.
+    def __reversed__(self):
+        raise TypeError("Persistent maps are not reversible")
+
+    def __getattr__(self, key):
+        try:
+            return self[key]
+        except KeyError as e:
+            raise AttributeError(
+                "{0} has no attribute '{1}'".format(type(self).__name__, key)
+            ) from e
+
+    def iterkeys(self):
+        for k, _ in self.iteritems():
+            yield k
+
+    # These are more efficient implementations compared to the original
+    # methods that are based on the keys iterator and then calls the
+    # accessor functions to access the value for the corresponding key
+    def itervalues(self):
+        for _, v in self.iteritems():
+            yield v
+
+    def iteritems(self):
+        for bucket in self._buckets:
+            if bucket:
+                for k, v in bucket:
+                    yield k, v
+
+    def values(self):
+        return PMapValues(self)
+
+    def keys(self):
+        from ._pset import PSet
+        return PSet(self)
+
+    def items(self):
+        return PMapItems(self)
+
+    def __len__(self):
+        return self._size
+
+    def __repr__(self):
+        return 'pmap({0})'.format(str(dict(self)))
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if not isinstance(other, Mapping):
+            return NotImplemented
+        if len(self) != len(other):
+            return False
+        if isinstance(other, PMap):
+            if (hasattr(self, '_cached_hash') and hasattr(other, '_cached_hash')
+                    and self._cached_hash != other._cached_hash):
+                return False
+            if self._buckets == other._buckets:
+                return True
+            return dict(self.iteritems()) == dict(other.iteritems())
+        elif isinstance(other, dict):
+            return dict(self.iteritems()) == other
+        return dict(self.iteritems()) == dict(other.items())
+
+    __ne__ = Mapping.__ne__
+
+    def __lt__(self, other):
+        raise TypeError('PMaps are not orderable')
+
+    __le__ = __lt__
+    __gt__ = __lt__
+    __ge__ = __lt__
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __hash__(self):
+        if not hasattr(self, '_cached_hash'):
+            self._cached_hash = hash(frozenset(self.iteritems()))
+        return self._cached_hash
+
+    def set(self, key, val):
+        """
+        Return a new PMap with key and val inserted.
+
+        >>> m1 = m(a=1, b=2)
+        >>> m2 = m1.set('a', 3)
+        >>> m3 = m1.set('c' ,4)
+        >>> m1 == {'a': 1, 'b': 2}
+        True
+        >>> m2 == {'a': 3, 'b': 2}
+        True
+        >>> m3 == {'a': 1, 'b': 2, 'c': 4}
+        True
+        """
+        return self.evolver().set(key, val).persistent()
+
+    def remove(self, key):
+        """
+        Return a new PMap without the element specified by key. Raises KeyError if the element
+        is not present.
+
+        >>> m1 = m(a=1, b=2)
+        >>> m1.remove('a')
+        pmap({'b': 2})
+        """
+        return self.evolver().remove(key).persistent()
+
+    def discard(self, key):
+        """
+        Return a new PMap without the element specified by key. Returns reference to itself
+        if element is not present.
+
+        >>> m1 = m(a=1, b=2)
+        >>> m1.discard('a')
+        pmap({'b': 2})
+        >>> m1 is m1.discard('c')
+        True
+        """
+        try:
+            return self.remove(key)
+        except KeyError:
+            return self
+
+    def update(self, *maps):
+        """
+        Return a new PMap with the items in Mappings inserted. If the same key is present in multiple
+        maps the rightmost (last) value is inserted.
+
+        >>> m1 = m(a=1, b=2)
+        >>> m1.update(m(a=2, c=3), {'a': 17, 'd': 35}) == {'a': 17, 'b': 2, 'c': 3, 'd': 35}
+        True
+        """
+        return self.update_with(lambda l, r: r, *maps)
+
+    def update_with(self, update_fn, *maps):
+        """
+        Return a new PMap with the items in Mappings maps inserted. If the same key is present in multiple
+        maps the values will be merged using merge_fn going from left to right.
+
+        >>> from operator import add
+        >>> m1 = m(a=1, b=2)
+        >>> m1.update_with(add, m(a=2)) == {'a': 3, 'b': 2}
+        True
+
+        The reverse behaviour of the regular merge. Keep the leftmost element instead of the rightmost.
+
+        >>> m1 = m(a=1)
+        >>> m1.update_with(lambda l, r: l, m(a=2), {'a':3})
+        pmap({'a': 1})
+        """
+        evolver = self.evolver()
+        for map in maps:
+            for key, value in map.items():
+                evolver.set(key, update_fn(evolver[key], value) if key in evolver else value)
+
+        return evolver.persistent()
+
+    def __add__(self, other):
+        return self.update(other)
+
+    __or__ = __add__
+
+    def __reduce__(self):
+        # Pickling support
+        return pmap, (dict(self),)
+
+    def transform(self, *transformations):
+        """
+        Transform arbitrarily complex combinations of PVectors and PMaps. A transformation
+        consists of two parts. One match expression that specifies which elements to transform
+        and one transformation function that performs the actual transformation.
+
+        >>> from pyrsistent import freeze, ny
+        >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
+        ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
+        ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
+        >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
+        >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
+        >>> very_short_news.articles[0].content
+        'A short article'
+        >>> very_short_news.articles[1].content
+        'A slightly long...'
+
+        When nothing has been transformed the original data structure is kept
+
+        >>> short_news is news_paper
+        True
+        >>> very_short_news is news_paper
+        False
+        >>> very_short_news.articles[0] is news_paper.articles[0]
+        True
+        """
+        return transform(self, transformations)
+
+    def copy(self):
+        return self
+
+    class _Evolver(object):
+        __slots__ = ('_buckets_evolver', '_size', '_original_pmap')
+
+        def __init__(self, original_pmap):
+            self._original_pmap = original_pmap
+            self._buckets_evolver = original_pmap._buckets.evolver()
+            self._size = original_pmap._size
+
+        def __getitem__(self, key):
+            return PMap._getitem(self._buckets_evolver, key)
+
+        def __setitem__(self, key, val):
+            self.set(key, val)
+
+        def set(self, key, val):
+            kv = (key, val)
+            index, bucket = PMap._get_bucket(self._buckets_evolver, key)
+            reallocation_required = len(self._buckets_evolver) < 0.67 * self._size
+            if bucket:
+                for k, v in bucket:
+                    if k == key:
+                        if v is not val:
+                            # Use `not (k2 == k)` rather than `!=` to avoid relying on a well implemented `__ne__`, see #268.
+                            new_bucket = [(k2, v2) if not (k2 == k) else (k2, val) for k2, v2 in bucket]
+                            self._buckets_evolver[index] = new_bucket
+
+                        return self
+
+                # Only check and perform reallocation if not replacing an existing value.
+                # This is a performance tweak, see #247.
+                if reallocation_required:
+                    self._reallocate()
+                    return self.set(key, val)
+
+                new_bucket = [kv]
+                new_bucket.extend(bucket)
+                self._buckets_evolver[index] = new_bucket
+                self._size += 1
+            else:
+                if reallocation_required:
+                    self._reallocate()
+                    return self.set(key, val)
+
+                self._buckets_evolver[index] = [kv]
+                self._size += 1
+
+            return self
+
+        def _reallocate(self):
+            new_size = 2 * len(self._buckets_evolver)
+            new_list = new_size * [None]
+            buckets = self._buckets_evolver.persistent()
+            for k, v in chain.from_iterable(x for x in buckets if x):
+                index = hash(k) % new_size
+                if new_list[index]:
+                    new_list[index].append((k, v))
+                else:
+                    new_list[index] = [(k, v)]
+
+            # A reallocation should always result in a dirty buckets evolver to avoid
+            # possible loss of elements when doing the reallocation.
+            self._buckets_evolver = pvector().evolver()
+            self._buckets_evolver.extend(new_list)
+
+        def is_dirty(self):
+            return self._buckets_evolver.is_dirty()
+
+        def persistent(self):
+            if self.is_dirty():
+                self._original_pmap = PMap(self._size, self._buckets_evolver.persistent())
+
+            return self._original_pmap
+
+        def __len__(self):
+            return self._size
+
+        def __contains__(self, key):
+            return PMap._contains(self._buckets_evolver, key)
+
+        def __delitem__(self, key):
+            self.remove(key)
+
+        def remove(self, key):
+            index, bucket = PMap._get_bucket(self._buckets_evolver, key)
+
+            if bucket:
+                # Use `not (k == key)` rather than `!=` to avoid relying on a well implemented `__ne__`, see #268.
+                new_bucket = [(k, v) for (k, v) in bucket if not (k == key)]
+                size_diff = len(bucket) - len(new_bucket)
+                if size_diff > 0:
+                    self._buckets_evolver[index] = new_bucket if new_bucket else None
+                    self._size -= size_diff
+                    return self
+
+            raise KeyError('{0}'.format(key))
+
+    def evolver(self):
+        """
+        Create a new evolver for this pmap. For a discussion on evolvers in general see the
+        documentation for the pvector evolver.
+
+        Create the evolver and perform various mutating updates to it:
+
+        >>> m1 = m(a=1, b=2)
+        >>> e = m1.evolver()
+        >>> e['c'] = 3
+        >>> len(e)
+        3
+        >>> del e['a']
+
+        The underlying pmap remains the same:
+
+        >>> m1 == {'a': 1, 'b': 2}
+        True
+
+        The changes are kept in the evolver. An updated pmap can be created using the
+        persistent() function on the evolver.
+
+        >>> m2 = e.persistent()
+        >>> m2 == {'b': 2, 'c': 3}
+        True
+
+        The new pmap will share data with the original pmap in the same way that would have
+        been done if only using operations on the pmap.
+        """
+        return self._Evolver(self)
+
+Mapping.register(PMap)
+Hashable.register(PMap)
+
+
+def _turbo_mapping(initial, pre_size):
+    if pre_size:
+        size = pre_size
+    else:
+        try:
+            size = 2 * len(initial) or 8
+        except Exception:
+            # Guess we can't figure out the length. Give up on length hinting,
+            # we can always reallocate later.
+            size = 8
+
+    buckets = size * [None]
+
+    if not isinstance(initial, Mapping):
+        # Make a dictionary of the initial data if it isn't already,
+        # that will save us some job further down since we can assume no
+        # key collisions
+        initial = dict(initial)
+
+    for k, v in initial.items():
+        h = hash(k)
+        index = h % size
+        bucket = buckets[index]
+
+        if bucket:
+            bucket.append((k, v))
+        else:
+            buckets[index] = [(k, v)]
+
+    return PMap(len(initial), pvector().extend(buckets))
+
+
+_EMPTY_PMAP = _turbo_mapping({}, 0)
+
+
+def pmap(initial={}, pre_size=0):
+    """
+    Create new persistent map, inserts all elements in initial into the newly created map.
+    The optional argument pre_size may be used to specify an initial size of the underlying bucket vector. This
+    may have a positive performance impact in the cases where you know beforehand that a large number of elements
+    will be inserted into the map eventually since it will reduce the number of reallocations required.
+
+    >>> pmap({'a': 13, 'b': 14}) == {'a': 13, 'b': 14}
+    True
+    """
+    if not initial and pre_size == 0:
+        return _EMPTY_PMAP
+
+    return _turbo_mapping(initial, pre_size)
+
+
+def m(**kwargs):
+    """
+    Creates a new persistent map. Inserts all key value arguments into the newly created map.
+
+    >>> m(a=13, b=14) == {'a': 13, 'b': 14}
+    True
+    """
+    return pmap(kwargs)
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_precord.py b/contrib/python/pyrsistent/py3/pyrsistent/_precord.py
new file mode 100644
index 00000000000..1ee8198a1a3
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_precord.py
@@ -0,0 +1,167 @@
+from pyrsistent._checked_types import CheckedType, _restore_pickle, InvariantException, store_invariants
+from pyrsistent._field_common import (
+    set_fields, check_type, is_field_ignore_extra_complaint, PFIELD_NO_INITIAL, serialize, check_global_invariants
+)
+from pyrsistent._pmap import PMap, pmap
+
+
+class _PRecordMeta(type):
+    def __new__(mcs, name, bases, dct):
+        set_fields(dct, bases, name='_precord_fields')
+        store_invariants(dct, bases, '_precord_invariants', '__invariant__')
+
+        dct['_precord_mandatory_fields'] = \
+            set(name for name, field in dct['_precord_fields'].items() if field.mandatory)
+
+        dct['_precord_initial_values'] = \
+            dict((k, field.initial) for k, field in dct['_precord_fields'].items() if field.initial is not PFIELD_NO_INITIAL)
+
+
+        dct['__slots__'] = ()
+
+        return super(_PRecordMeta, mcs).__new__(mcs, name, bases, dct)
+
+
+class PRecord(PMap, CheckedType, metaclass=_PRecordMeta):
+    """
+    A PRecord is a PMap with a fixed set of specified fields. Records are declared as python classes inheriting
+    from PRecord. Because it is a PMap it has full support for all Mapping methods such as iteration and element
+    access using subscript notation.
+
+    More documentation and examples of PRecord usage is available at https://github.com/tobgu/pyrsistent
+    """
+    def __new__(cls, **kwargs):
+        # Hack total! If these two special attributes exist that means we can create
+        # ourselves. Otherwise we need to go through the Evolver to create the structures
+        # for us.
+        if '_precord_size' in kwargs and '_precord_buckets' in kwargs:
+            return super(PRecord, cls).__new__(cls, kwargs['_precord_size'], kwargs['_precord_buckets'])
+
+        factory_fields = kwargs.pop('_factory_fields', None)
+        ignore_extra = kwargs.pop('_ignore_extra', False)
+
+        initial_values = kwargs
+        if cls._precord_initial_values:
+            initial_values = dict((k, v() if callable(v) else v)
+                                  for k, v in cls._precord_initial_values.items())
+            initial_values.update(kwargs)
+
+        e = _PRecordEvolver(cls, pmap(pre_size=len(cls._precord_fields)), _factory_fields=factory_fields, _ignore_extra=ignore_extra)
+        for k, v in initial_values.items():
+            e[k] = v
+
+        return e.persistent()
+
+    def set(self, *args, **kwargs):
+        """
+        Set a field in the record. This set function differs slightly from that in the PMap
+        class. First of all it accepts key-value pairs. Second it accepts multiple key-value
+        pairs to perform one, atomic, update of multiple fields.
+        """
+
+        # The PRecord set() can accept kwargs since all fields that have been declared are
+        # valid python identifiers. Also allow multiple fields to be set in one operation.
+        if args:
+            return super(PRecord, self).set(args[0], args[1])
+
+        return self.update(kwargs)
+
+    def evolver(self):
+        """
+        Returns an evolver of this object.
+        """
+        return _PRecordEvolver(self.__class__, self)
+
+    def __repr__(self):
+        return "{0}({1})".format(self.__class__.__name__,
+                                 ', '.join('{0}={1}'.format(k, repr(v)) for k, v in self.items()))
+
+    @classmethod
+    def create(cls, kwargs, _factory_fields=None, ignore_extra=False):
+        """
+        Factory method. Will create a new PRecord of the current type and assign the values
+        specified in kwargs.
+
+        :param ignore_extra: A boolean which when set to True will ignore any keys which appear in kwargs that are not
+                             in the set of fields on the PRecord.
+        """
+        if isinstance(kwargs, cls):
+            return kwargs
+
+        if ignore_extra:
+            kwargs = {k: kwargs[k] for k in cls._precord_fields if k in kwargs}
+
+        return cls(_factory_fields=_factory_fields, _ignore_extra=ignore_extra, **kwargs)
+
+    def __reduce__(self):
+        # Pickling support
+        return _restore_pickle, (self.__class__, dict(self),)
+
+    def serialize(self, format=None):
+        """
+        Serialize the current PRecord using custom serializer functions for fields where
+        such have been supplied.
+        """
+        return dict((k, serialize(self._precord_fields[k].serializer, format, v)) for k, v in self.items())
+
+
+class _PRecordEvolver(PMap._Evolver):
+    __slots__ = ('_destination_cls', '_invariant_error_codes', '_missing_fields', '_factory_fields', '_ignore_extra')
+
+    def __init__(self, cls, original_pmap, _factory_fields=None, _ignore_extra=False):
+        super(_PRecordEvolver, self).__init__(original_pmap)
+        self._destination_cls = cls
+        self._invariant_error_codes = []
+        self._missing_fields = []
+        self._factory_fields = _factory_fields
+        self._ignore_extra = _ignore_extra
+
+    def __setitem__(self, key, original_value):
+        self.set(key, original_value)
+
+    def set(self, key, original_value):
+        field = self._destination_cls._precord_fields.get(key)
+        if field:
+            if self._factory_fields is None or field in self._factory_fields:
+                try:
+                    if is_field_ignore_extra_complaint(PRecord, field, self._ignore_extra):
+                        value = field.factory(original_value, ignore_extra=self._ignore_extra)
+                    else:
+                        value = field.factory(original_value)
+                except InvariantException as e:
+                    self._invariant_error_codes += e.invariant_errors
+                    self._missing_fields += e.missing_fields
+                    return self
+            else:
+                value = original_value
+
+            check_type(self._destination_cls, field, key, value)
+
+            is_ok, error_code = field.invariant(value)
+            if not is_ok:
+                self._invariant_error_codes.append(error_code)
+
+            return super(_PRecordEvolver, self).set(key, value)
+        else:
+            raise AttributeError("'{0}' is not among the specified fields for {1}".format(key, self._destination_cls.__name__))
+
+    def persistent(self):
+        cls = self._destination_cls
+        is_dirty = self.is_dirty()
+        pm = super(_PRecordEvolver, self).persistent()
+        if is_dirty or not isinstance(pm, cls):
+            result = cls(_precord_buckets=pm._buckets, _precord_size=pm._size)
+        else:
+            result = pm
+
+        if cls._precord_mandatory_fields:
+            self._missing_fields += tuple('{0}.{1}'.format(cls.__name__, f) for f
+                                          in (cls._precord_mandatory_fields - set(result.keys())))
+
+        if self._invariant_error_codes or self._missing_fields:
+            raise InvariantException(tuple(self._invariant_error_codes), tuple(self._missing_fields),
+                                     'Field invariant failed')
+
+        check_global_invariants(result, cls._precord_invariants)
+
+        return result
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_pset.py b/contrib/python/pyrsistent/py3/pyrsistent/_pset.py
new file mode 100644
index 00000000000..6247607db57
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_pset.py
@@ -0,0 +1,230 @@
+from collections.abc import Set, Hashable
+import sys
+from typing import TypeVar, Generic
+from pyrsistent._pmap import pmap
+
+T_co = TypeVar('T_co', covariant=True)
+
+
+class PSet(Generic[T_co]):
+    """
+    Persistent set implementation. Built on top of the persistent map. The set supports all operations
+    in the Set protocol and is Hashable.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`s` or :py:func:`pset`
+    to create an instance.
+
+    Random access and insert is log32(n) where n is the size of the set.
+
+    Some examples:
+
+    >>> s = pset([1, 2, 3, 1])
+    >>> s2 = s.add(4)
+    >>> s3 = s2.remove(2)
+    >>> s
+    pset([1, 2, 3])
+    >>> s2
+    pset([1, 2, 3, 4])
+    >>> s3
+    pset([1, 3, 4])
+    """
+    __slots__ = ('_map', '__weakref__')
+
+    def __new__(cls, m):
+        self = super(PSet, cls).__new__(cls)
+        self._map = m
+        return self
+
+    def __contains__(self, element):
+        return element in self._map
+
+    def __iter__(self):
+        return iter(self._map)
+
+    def __len__(self):
+        return len(self._map)
+
+    def __repr__(self):
+        if not self:
+            return 'p' + str(set(self))
+
+        return 'pset([{0}])'.format(str(set(self))[1:-1])
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __hash__(self):
+        return hash(self._map)
+
+    def __reduce__(self):
+        # Pickling support
+        return pset, (list(self),)
+
+    @classmethod
+    def _from_iterable(cls, it, pre_size=8):
+        return PSet(pmap(dict((k, True) for k in it), pre_size=pre_size))
+
+    def add(self, element):
+        """
+        Return a new PSet with element added
+
+        >>> s1 = s(1, 2)
+        >>> s1.add(3)
+        pset([1, 2, 3])
+        """
+        return self.evolver().add(element).persistent()
+
+    def update(self, iterable):
+        """
+        Return a new PSet with elements in iterable added
+
+        >>> s1 = s(1, 2)
+        >>> s1.update([3, 4, 4])
+        pset([1, 2, 3, 4])
+        """
+        e = self.evolver()
+        for element in iterable:
+            e.add(element)
+
+        return e.persistent()
+
+    def remove(self, element):
+        """
+        Return a new PSet with element removed. Raises KeyError if element is not present.
+
+        >>> s1 = s(1, 2)
+        >>> s1.remove(2)
+        pset([1])
+        """
+        if element in self._map:
+            return self.evolver().remove(element).persistent()
+
+        raise KeyError("Element '%s' not present in PSet" % repr(element))
+
+    def discard(self, element):
+        """
+        Return a new PSet with element removed. Returns itself if element is not present.
+        """
+        if element in self._map:
+            return self.evolver().remove(element).persistent()
+
+        return self
+
+    class _Evolver(object):
+        __slots__ = ('_original_pset', '_pmap_evolver')
+
+        def __init__(self, original_pset):
+            self._original_pset = original_pset
+            self._pmap_evolver = original_pset._map.evolver()
+
+        def add(self, element):
+            self._pmap_evolver[element] = True
+            return self
+
+        def remove(self, element):
+            del self._pmap_evolver[element]
+            return self
+
+        def is_dirty(self):
+            return self._pmap_evolver.is_dirty()
+
+        def persistent(self):
+            if not self.is_dirty():
+                return  self._original_pset
+
+            return PSet(self._pmap_evolver.persistent())
+
+        def __len__(self):
+            return len(self._pmap_evolver)
+
+    def copy(self):
+        return self
+
+    def evolver(self):
+        """
+        Create a new evolver for this pset. For a discussion on evolvers in general see the
+        documentation for the pvector evolver.
+
+        Create the evolver and perform various mutating updates to it:
+
+        >>> s1 = s(1, 2, 3)
+        >>> e = s1.evolver()
+        >>> _ = e.add(4)
+        >>> len(e)
+        4
+        >>> _ = e.remove(1)
+
+        The underlying pset remains the same:
+
+        >>> s1
+        pset([1, 2, 3])
+
+        The changes are kept in the evolver. An updated pmap can be created using the
+        persistent() function on the evolver.
+
+        >>> s2 = e.persistent()
+        >>> s2
+        pset([2, 3, 4])
+
+        The new pset will share data with the original pset in the same way that would have
+        been done if only using operations on the pset.
+        """
+        return PSet._Evolver(self)
+
+    # All the operations and comparisons you would expect on a set.
+    #
+    # This is not very beautiful. If we avoid inheriting from PSet we can use the
+    # __slots__ concepts (which requires a new style class) and hopefully save some memory.
+    __le__ = Set.__le__
+    __lt__ = Set.__lt__
+    __gt__ = Set.__gt__
+    __ge__ = Set.__ge__
+    __eq__ = Set.__eq__
+    __ne__ = Set.__ne__
+
+    __and__ = Set.__and__
+    __or__ = Set.__or__
+    __sub__ = Set.__sub__
+    __xor__ = Set.__xor__
+
+    issubset = __le__
+    issuperset = __ge__
+    union = __or__
+    intersection = __and__
+    difference = __sub__
+    symmetric_difference = __xor__
+
+    isdisjoint = Set.isdisjoint
+
+Set.register(PSet)
+Hashable.register(PSet)
+
+_EMPTY_PSET = PSet(pmap())
+
+
+def pset(iterable=(), pre_size=8):
+    """
+    Creates a persistent set from iterable. Optionally takes a sizing parameter equivalent to that
+    used for :py:func:`pmap`.
+
+    >>> s1 = pset([1, 2, 3, 2])
+    >>> s1
+    pset([1, 2, 3])
+    """
+    if not iterable:
+        return _EMPTY_PSET
+
+    return PSet._from_iterable(iterable, pre_size=pre_size)
+
+
+def s(*elements):
+    """
+    Create a persistent set.
+
+    Takes an arbitrary number of arguments to insert into the new set.
+
+    >>> s1 = s(1, 2, 3, 2)
+    >>> s1
+    pset([1, 2, 3])
+    """
+    return pset(elements)
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_pvector.py b/contrib/python/pyrsistent/py3/pyrsistent/_pvector.py
new file mode 100644
index 00000000000..51d8a227ba6
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_pvector.py
@@ -0,0 +1,715 @@
+from abc import abstractmethod, ABCMeta
+from collections.abc import Sequence, Hashable
+from numbers import Integral
+import operator
+from typing import TypeVar, Generic
+
+from pyrsistent._transformations import transform
+
+T_co = TypeVar('T_co', covariant=True)
+
+
+def _bitcount(val):
+    return bin(val).count("1")
+
+BRANCH_FACTOR = 32
+BIT_MASK = BRANCH_FACTOR - 1
+SHIFT = _bitcount(BIT_MASK)
+
+
+def compare_pvector(v, other, operator):
+    return operator(v.tolist(), other.tolist() if isinstance(other, PVector) else other)
+
+
+def _index_or_slice(index, stop):
+    if stop is None:
+        return index
+
+    return slice(index, stop)
+
+
+class PythonPVector(object):
+    """
+    Support structure for PVector that implements structural sharing for vectors using a trie.
+    """
+    __slots__ = ('_count', '_shift', '_root', '_tail', '_tail_offset', '__weakref__')
+
+    def __new__(cls, count, shift, root, tail):
+        self = super(PythonPVector, cls).__new__(cls)
+        self._count = count
+        self._shift = shift
+        self._root = root
+        self._tail = tail
+
+        # Derived attribute stored for performance
+        self._tail_offset = self._count - len(self._tail)
+        return self
+
+    def __len__(self):
+        return self._count
+
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            # There are more conditions than the below where it would be OK to
+            # return ourselves, implement those...
+            if index.start is None and index.stop is None and index.step is None:
+                return self
+
+            # This is a bit nasty realizing the whole structure as a list before
+            # slicing it but it is the fastest way I've found to date, and it's easy :-)
+            return _EMPTY_PVECTOR.extend(self.tolist()[index])
+
+        if index < 0:
+            index += self._count
+
+        return PythonPVector._node_for(self, index)[index & BIT_MASK]
+
+    def __add__(self, other):
+        return self.extend(other)
+
+    def __repr__(self):
+        return 'pvector({0})'.format(str(self.tolist()))
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __iter__(self):
+        # This is kind of lazy and will produce some memory overhead but it is the fasted method
+        # by far of those tried since it uses the speed of the built in python list directly.
+        return iter(self.tolist())
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __eq__(self, other):
+        return self is other or (hasattr(other, '__len__') and self._count == len(other)) and compare_pvector(self, other, operator.eq)
+
+    def __gt__(self, other):
+        return compare_pvector(self, other, operator.gt)
+
+    def __lt__(self, other):
+        return compare_pvector(self, other, operator.lt)
+
+    def __ge__(self, other):
+        return compare_pvector(self, other, operator.ge)
+
+    def __le__(self, other):
+        return compare_pvector(self, other, operator.le)
+
+    def __mul__(self, times):
+        if times <= 0 or self is _EMPTY_PVECTOR:
+            return _EMPTY_PVECTOR
+
+        if times == 1:
+            return self
+
+        return _EMPTY_PVECTOR.extend(times * self.tolist())
+
+    __rmul__ = __mul__
+
+    def _fill_list(self, node, shift, the_list):
+        if shift:
+            shift -= SHIFT
+            for n in node:
+                self._fill_list(n, shift, the_list)
+        else:
+            the_list.extend(node)
+
+    def tolist(self):
+        """
+        The fastest way to convert the vector into a python list.
+        """
+        the_list = []
+        self._fill_list(self._root, self._shift, the_list)
+        the_list.extend(self._tail)
+        return the_list
+
+    def _totuple(self):
+        """
+        Returns the content as a python tuple.
+        """
+        return tuple(self.tolist())
+
+    def __hash__(self):
+        # Taking the easy way out again...
+        return hash(self._totuple())
+
+    def transform(self, *transformations):
+        return transform(self, transformations)
+
+    def __reduce__(self):
+        # Pickling support
+        return pvector, (self.tolist(),)
+
+    def mset(self, *args):
+        if len(args) % 2:
+            raise TypeError("mset expected an even number of arguments")
+
+        evolver = self.evolver()
+        for i in range(0, len(args), 2):
+            evolver[args[i]] = args[i+1]
+
+        return evolver.persistent()
+
+    class Evolver(object):
+        __slots__ = ('_count', '_shift', '_root', '_tail', '_tail_offset', '_dirty_nodes',
+                     '_extra_tail', '_cached_leafs', '_orig_pvector')
+
+        def __init__(self, v):
+            self._reset(v)
+
+        def __getitem__(self, index):
+            if not isinstance(index, Integral):
+                raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)
+
+            if index < 0:
+                index += self._count + len(self._extra_tail)
+
+            if self._count <= index < self._count + len(self._extra_tail):
+                return self._extra_tail[index - self._count]
+
+            return PythonPVector._node_for(self, index)[index & BIT_MASK]
+
+        def _reset(self, v):
+            self._count = v._count
+            self._shift = v._shift
+            self._root = v._root
+            self._tail = v._tail
+            self._tail_offset = v._tail_offset
+            self._dirty_nodes = {}
+            self._cached_leafs = {}
+            self._extra_tail = []
+            self._orig_pvector = v
+
+        def append(self, element):
+            self._extra_tail.append(element)
+            return self
+
+        def extend(self, iterable):
+            self._extra_tail.extend(iterable)
+            return self
+
+        def set(self, index, val):
+            self[index] = val
+            return self
+
+        def __setitem__(self, index, val):
+            if not isinstance(index, Integral):
+                raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)
+
+            if index < 0:
+                index += self._count + len(self._extra_tail)
+
+            if 0 <= index < self._count:
+                node = self._cached_leafs.get(index >> SHIFT)
+                if node:
+                    node[index & BIT_MASK] = val
+                elif index >= self._tail_offset:
+                    if id(self._tail) not in self._dirty_nodes:
+                        self._tail = list(self._tail)
+                        self._dirty_nodes[id(self._tail)] = True
+                        self._cached_leafs[index >> SHIFT] = self._tail
+                    self._tail[index & BIT_MASK] = val
+                else:
+                    self._root = self._do_set(self._shift, self._root, index, val)
+            elif self._count <= index < self._count + len(self._extra_tail):
+                self._extra_tail[index - self._count] = val
+            elif index == self._count + len(self._extra_tail):
+                self._extra_tail.append(val)
+            else:
+                raise IndexError("Index out of range: %s" % (index,))
+
+        def _do_set(self, level, node, i, val):
+            if id(node) in self._dirty_nodes:
+                ret = node
+            else:
+                ret = list(node)
+                self._dirty_nodes[id(ret)] = True
+
+            if level == 0:
+                ret[i & BIT_MASK] = val
+                self._cached_leafs[i >> SHIFT] = ret
+            else:
+                sub_index = (i >> level) & BIT_MASK  # >>>
+                ret[sub_index] = self._do_set(level - SHIFT, node[sub_index], i, val)
+
+            return ret
+
+        def delete(self, index):
+            del self[index]
+            return self
+
+        def __delitem__(self, key):
+            if self._orig_pvector:
+                # All structural sharing bets are off, base evolver on _extra_tail only
+                l = PythonPVector(self._count, self._shift, self._root, self._tail).tolist()
+                l.extend(self._extra_tail)
+                self._reset(_EMPTY_PVECTOR)
+                self._extra_tail = l
+
+            del self._extra_tail[key]
+
+        def persistent(self):
+            result = self._orig_pvector
+            if self.is_dirty():
+                result = PythonPVector(self._count, self._shift, self._root, self._tail).extend(self._extra_tail)
+                self._reset(result)
+
+            return result
+
+        def __len__(self):
+            return self._count + len(self._extra_tail)
+
+        def is_dirty(self):
+            return bool(self._dirty_nodes or self._extra_tail)
+
+    def evolver(self):
+        return PythonPVector.Evolver(self)
+
+    def set(self, i, val):
+        # This method could be implemented by a call to mset() but doing so would cause
+        # a ~5 X performance penalty on PyPy (considered the primary platform for this implementation
+        #  of PVector) so we're keeping this implementation for now.
+
+        if not isinstance(i, Integral):
+            raise TypeError("'%s' object cannot be interpreted as an index" % type(i).__name__)
+
+        if i < 0:
+            i += self._count
+
+        if 0 <= i < self._count:
+            if i >= self._tail_offset:
+                new_tail = list(self._tail)
+                new_tail[i & BIT_MASK] = val
+                return PythonPVector(self._count, self._shift, self._root, new_tail)
+
+            return PythonPVector(self._count, self._shift, self._do_set(self._shift, self._root, i, val), self._tail)
+
+        if i == self._count:
+            return self.append(val)
+
+        raise IndexError("Index out of range: %s" % (i,))
+
+    def _do_set(self, level, node, i, val):
+        ret = list(node)
+        if level == 0:
+            ret[i & BIT_MASK] = val
+        else:
+            sub_index = (i >> level) & BIT_MASK  # >>>
+            ret[sub_index] = self._do_set(level - SHIFT, node[sub_index], i, val)
+
+        return ret
+
+    @staticmethod
+    def _node_for(pvector_like, i):
+        if 0 <= i < pvector_like._count:
+            if i >= pvector_like._tail_offset:
+                return pvector_like._tail
+
+            node = pvector_like._root
+            for level in range(pvector_like._shift, 0, -SHIFT):
+                node = node[(i >> level) & BIT_MASK]  # >>>
+
+            return node
+
+        raise IndexError("Index out of range: %s" % (i,))
+
+    def _create_new_root(self):
+        new_shift = self._shift
+
+        # Overflow root?
+        if (self._count >> SHIFT) > (1 << self._shift): # >>>
+            new_root = [self._root, self._new_path(self._shift, self._tail)]
+            new_shift += SHIFT
+        else:
+            new_root = self._push_tail(self._shift, self._root, self._tail)
+
+        return new_root, new_shift
+
+    def append(self, val):
+        if len(self._tail) < BRANCH_FACTOR:
+            new_tail = list(self._tail)
+            new_tail.append(val)
+            return PythonPVector(self._count + 1, self._shift, self._root, new_tail)
+
+        # Full tail, push into tree
+        new_root, new_shift = self._create_new_root()
+        return PythonPVector(self._count + 1, new_shift, new_root, [val])
+
+    def _new_path(self, level, node):
+        if level == 0:
+            return node
+
+        return [self._new_path(level - SHIFT, node)]
+
+    def _mutating_insert_tail(self):
+        self._root, self._shift = self._create_new_root()
+        self._tail = []
+
+    def _mutating_fill_tail(self, offset, sequence):
+        max_delta_len = BRANCH_FACTOR - len(self._tail)
+        delta = sequence[offset:offset + max_delta_len]
+        self._tail.extend(delta)
+        delta_len = len(delta)
+        self._count += delta_len
+        return offset + delta_len
+
+    def _mutating_extend(self, sequence):
+        offset = 0
+        sequence_len = len(sequence)
+        while offset < sequence_len:
+            offset = self._mutating_fill_tail(offset, sequence)
+            if len(self._tail) == BRANCH_FACTOR:
+                self._mutating_insert_tail()
+
+        self._tail_offset = self._count - len(self._tail)
+
+    def extend(self, obj):
+        # Mutates the new vector directly for efficiency but that's only an
+        # implementation detail, once it is returned it should be considered immutable
+        l = obj.tolist() if isinstance(obj, PythonPVector) else list(obj)
+        if l:
+            new_vector = self.append(l[0])
+            new_vector._mutating_extend(l[1:])
+            return new_vector
+
+        return self
+
+    def _push_tail(self, level, parent, tail_node):
+        """
+        if parent is leaf, insert node,
+        else does it map to an existing child? ->
+             node_to_insert = push node one more level
+        else alloc new path
+
+        return  node_to_insert placed in copy of parent
+        """
+        ret = list(parent)
+
+        if level == SHIFT:
+            ret.append(tail_node)
+            return ret
+
+        sub_index = ((self._count - 1) >> level) & BIT_MASK  # >>>
+        if len(parent) > sub_index:
+            ret[sub_index] = self._push_tail(level - SHIFT, parent[sub_index], tail_node)
+            return ret
+
+        ret.append(self._new_path(level - SHIFT, tail_node))
+        return ret
+
+    def index(self, value, *args, **kwargs):
+        return self.tolist().index(value, *args, **kwargs)
+
+    def count(self, value):
+        return self.tolist().count(value)
+
+    def delete(self, index, stop=None):
+        l = self.tolist()
+        del l[_index_or_slice(index, stop)]
+        return _EMPTY_PVECTOR.extend(l)
+
+    def remove(self, value):
+        l = self.tolist()
+        l.remove(value)
+        return _EMPTY_PVECTOR.extend(l)
+
+class PVector(Generic[T_co],metaclass=ABCMeta):
+    """
+    Persistent vector implementation. Meant as a replacement for the cases where you would normally
+    use a Python list.
+
+    Do not instantiate directly, instead use the factory functions :py:func:`v` and :py:func:`pvector` to
+    create an instance.
+
+    Heavily influenced by the persistent vector available in Clojure. Initially this was more or
+    less just a port of the Java code for the Clojure vector. It has since been modified and to
+    some extent optimized for usage in Python.
+
+    The vector is organized as a trie, any mutating method will return a new vector that contains the changes. No
+    updates are done to the original vector. Structural sharing between vectors are applied where possible to save
+    space and to avoid making complete copies.
+
+    This structure corresponds most closely to the built in list type and is intended as a replacement. Where the
+    semantics are the same (more or less) the same function names have been used but for some cases it is not possible,
+    for example assignments.
+
+    The PVector implements the Sequence protocol and is Hashable.
+
+    Inserts are amortized O(1). Random access is log32(n) where n is the size of the vector.
+
+    The following are examples of some common operations on persistent vectors:
+
+    >>> p = v(1, 2, 3)
+    >>> p2 = p.append(4)
+    >>> p3 = p2.extend([5, 6, 7])
+    >>> p
+    pvector([1, 2, 3])
+    >>> p2
+    pvector([1, 2, 3, 4])
+    >>> p3
+    pvector([1, 2, 3, 4, 5, 6, 7])
+    >>> p3[5]
+    6
+    >>> p.set(1, 99)
+    pvector([1, 99, 3])
+    >>>
+    """
+
+    @abstractmethod
+    def __len__(self):
+        """
+        >>> len(v(1, 2, 3))
+        3
+        """
+
+    @abstractmethod
+    def __getitem__(self, index):
+        """
+        Get value at index. Full slicing support.
+
+        >>> v1 = v(5, 6, 7, 8)
+        >>> v1[2]
+        7
+        >>> v1[1:3]
+        pvector([6, 7])
+        """
+
+    @abstractmethod
+    def __add__(self, other):
+        """
+        >>> v1 = v(1, 2)
+        >>> v2 = v(3, 4)
+        >>> v1 + v2
+        pvector([1, 2, 3, 4])
+        """
+
+    @abstractmethod
+    def __mul__(self, times):
+        """
+        >>> v1 = v(1, 2)
+        >>> 3 * v1
+        pvector([1, 2, 1, 2, 1, 2])
+        """
+
+    @abstractmethod
+    def __hash__(self):
+        """
+        >>> v1 = v(1, 2, 3)
+        >>> v2 = v(1, 2, 3)
+        >>> hash(v1) == hash(v2)
+        True
+        """
+
+    @abstractmethod
+    def evolver(self):
+        """
+        Create a new evolver for this pvector. The evolver acts as a mutable view of the vector
+        with "transaction like" semantics. No part of the underlying vector i updated, it is still
+        fully immutable. Furthermore multiple evolvers created from the same pvector do not
+        interfere with each other.
+
+        You may want to use an evolver instead of working directly with the pvector in the
+        following cases:
+
+        * Multiple updates are done to the same vector and the intermediate results are of no
+          interest. In this case using an evolver may be a more efficient and easier to work with.
+        * You need to pass a vector into a legacy function or a function that you have no control
+          over which performs in place mutations of lists. In this case pass an evolver instance
+          instead and then create a new pvector from the evolver once the function returns.
+
+        The following example illustrates a typical workflow when working with evolvers. It also
+        displays most of the API (which i kept small by design, you should not be tempted to
+        use evolvers in excess ;-)).
+
+        Create the evolver and perform various mutating updates to it:
+
+        >>> v1 = v(1, 2, 3, 4, 5)
+        >>> e = v1.evolver()
+        >>> e[1] = 22
+        >>> _ = e.append(6)
+        >>> _ = e.extend([7, 8, 9])
+        >>> e[8] += 1
+        >>> len(e)
+        9
+
+        The underlying pvector remains the same:
+
+        >>> v1
+        pvector([1, 2, 3, 4, 5])
+
+        The changes are kept in the evolver. An updated pvector can be created using the
+        persistent() function on the evolver.
+
+        >>> v2 = e.persistent()
+        >>> v2
+        pvector([1, 22, 3, 4, 5, 6, 7, 8, 10])
+
+        The new pvector will share data with the original pvector in the same way that would have
+        been done if only using operations on the pvector.
+        """
+
+    @abstractmethod
+    def mset(self, *args):
+        """
+        Return a new vector with elements in specified positions replaced by values (multi set).
+
+        Elements on even positions in the argument list are interpreted as indexes while
+        elements on odd positions are considered values.
+
+        >>> v1 = v(1, 2, 3)
+        >>> v1.mset(0, 11, 2, 33)
+        pvector([11, 2, 33])
+        """
+
+    @abstractmethod
+    def set(self, i, val):
+        """
+        Return a new vector with element at position i replaced with val. The original vector remains unchanged.
+
+        Setting a value one step beyond the end of the vector is equal to appending. Setting beyond that will
+        result in an IndexError.
+
+        >>> v1 = v(1, 2, 3)
+        >>> v1.set(1, 4)
+        pvector([1, 4, 3])
+        >>> v1.set(3, 4)
+        pvector([1, 2, 3, 4])
+        >>> v1.set(-1, 4)
+        pvector([1, 2, 4])
+        """
+
+    @abstractmethod
+    def append(self, val):
+        """
+        Return a new vector with val appended.
+
+        >>> v1 = v(1, 2)
+        >>> v1.append(3)
+        pvector([1, 2, 3])
+        """
+
+    @abstractmethod
+    def extend(self, obj):
+        """
+        Return a new vector with all values in obj appended to it. Obj may be another
+        PVector or any other Iterable.
+
+        >>> v1 = v(1, 2, 3)
+        >>> v1.extend([4, 5])
+        pvector([1, 2, 3, 4, 5])
+        """
+
+    @abstractmethod
+    def index(self, value, *args, **kwargs):
+        """
+        Return first index of value. Additional indexes may be supplied to limit the search to a
+        sub range of the vector.
+
+        >>> v1 = v(1, 2, 3, 4, 3)
+        >>> v1.index(3)
+        2
+        >>> v1.index(3, 3, 5)
+        4
+        """
+
+    @abstractmethod
+    def count(self, value):
+        """
+        Return the number of times that value appears in the vector.
+
+        >>> v1 = v(1, 4, 3, 4)
+        >>> v1.count(4)
+        2
+        """
+
+    @abstractmethod
+    def transform(self, *transformations):
+        """
+        Transform arbitrarily complex combinations of PVectors and PMaps. A transformation
+        consists of two parts. One match expression that specifies which elements to transform
+        and one transformation function that performs the actual transformation.
+
+        >>> from pyrsistent import freeze, ny
+        >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
+        ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
+        ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
+        >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
+        >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
+        >>> very_short_news.articles[0].content
+        'A short article'
+        >>> very_short_news.articles[1].content
+        'A slightly long...'
+
+        When nothing has been transformed the original data structure is kept
+
+        >>> short_news is news_paper
+        True
+        >>> very_short_news is news_paper
+        False
+        >>> very_short_news.articles[0] is news_paper.articles[0]
+        True
+        """
+
+    @abstractmethod
+    def delete(self, index, stop=None):
+        """
+        Delete a portion of the vector by index or range.
+
+        >>> v1 = v(1, 2, 3, 4, 5)
+        >>> v1.delete(1)
+        pvector([1, 3, 4, 5])
+        >>> v1.delete(1, 3)
+        pvector([1, 4, 5])
+        """
+
+    @abstractmethod
+    def remove(self, value):
+        """
+        Remove the first occurrence of a value from the vector.
+
+        >>> v1 = v(1, 2, 3, 2, 1)
+        >>> v2 = v1.remove(1)
+        >>> v2
+        pvector([2, 3, 2, 1])
+        >>> v2.remove(1)
+        pvector([2, 3, 2])
+        """
+
+
+_EMPTY_PVECTOR = PythonPVector(0, SHIFT, [], [])
+PVector.register(PythonPVector)
+Sequence.register(PVector)
+Hashable.register(PVector)
+
+def python_pvector(iterable=()):
+    """
+    Create a new persistent vector containing the elements in iterable.
+
+    >>> v1 = pvector([1, 2, 3])
+    >>> v1
+    pvector([1, 2, 3])
+    """
+    return _EMPTY_PVECTOR.extend(iterable)
+
+try:
+    # Use the C extension as underlying trie implementation if it is available
+    import os
+    if os.environ.get('PYRSISTENT_NO_C_EXTENSION'):
+        pvector = python_pvector
+    else:
+        from pvectorc import pvector
+        PVector.register(type(pvector()))
+except ImportError:
+    pvector = python_pvector
+
+
+def v(*elements):
+    """
+    Create a new persistent vector containing all parameters to this function.
+
+    >>> v1 = v(1, 2, 3)
+    >>> v1
+    pvector([1, 2, 3])
+    """
+    return pvector(elements)
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_toolz.py b/contrib/python/pyrsistent/py3/pyrsistent/_toolz.py
new file mode 100644
index 00000000000..0bf2cb14490
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_toolz.py
@@ -0,0 +1,83 @@
+"""
+Functionality copied from the toolz package to avoid having
+to add toolz as a dependency.
+
+See https://github.com/pytoolz/toolz/.
+
+toolz is released under BSD licence. Below is the licence text
+from toolz as it appeared when copying the code.
+
+--------------------------------------------------------------
+
+Copyright (c) 2013 Matthew Rocklin
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  a. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  b. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  c. Neither the name of toolz nor the names of its contributors
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+"""
+import operator
+from functools import reduce
+
+
+def get_in(keys, coll, default=None, no_default=False):
+    """
+    NB: This is a straight copy of the get_in implementation found in
+        the toolz library (https://github.com/pytoolz/toolz/). It works
+        with persistent data structures as well as the corresponding
+        datastructures from the stdlib.
+
+    Returns coll[i0][i1]...[iX] where [i0, i1, ..., iX]==keys.
+
+    If coll[i0][i1]...[iX] cannot be found, returns ``default``, unless
+    ``no_default`` is specified, then it raises KeyError or IndexError.
+
+    ``get_in`` is a generalization of ``operator.getitem`` for nested data
+    structures such as dictionaries and lists.
+    >>> from pyrsistent import freeze
+    >>> transaction = freeze({'name': 'Alice',
+    ...                       'purchase': {'items': ['Apple', 'Orange'],
+    ...                                    'costs': [0.50, 1.25]},
+    ...                       'credit card': '5555-1234-1234-1234'})
+    >>> get_in(['purchase', 'items', 0], transaction)
+    'Apple'
+    >>> get_in(['name'], transaction)
+    'Alice'
+    >>> get_in(['purchase', 'total'], transaction)
+    >>> get_in(['purchase', 'items', 'apple'], transaction)
+    >>> get_in(['purchase', 'items', 10], transaction)
+    >>> get_in(['purchase', 'total'], transaction, 0)
+    0
+    >>> get_in(['y'], {}, no_default=True)
+    Traceback (most recent call last):
+    ...
+    KeyError: 'y'
+    """
+    try:
+        return reduce(operator.getitem, keys, coll)
+    except (KeyError, IndexError, TypeError):
+        if no_default:
+            raise
+        return default
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/_transformations.py b/contrib/python/pyrsistent/py3/pyrsistent/_transformations.py
new file mode 100644
index 00000000000..6ef747f07e6
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/_transformations.py
@@ -0,0 +1,143 @@
+import re
+try:
+    from inspect import Parameter, signature
+except ImportError:
+    signature = None
+    from inspect import getfullargspec
+
+
+_EMPTY_SENTINEL = object()
+
+
+def inc(x):
+    """ Add one to the current value """
+    return x + 1
+
+
+def dec(x):
+    """ Subtract one from the current value """
+    return x - 1
+
+
+def discard(evolver, key):
+    """ Discard the element and returns a structure without the discarded elements """
+    try:
+        del evolver[key]
+    except KeyError:
+        pass
+
+
+# Matchers
+def rex(expr):
+    """ Regular expression matcher to use together with transform functions """
+    r = re.compile(expr)
+    return lambda key: isinstance(key, str) and r.match(key)
+
+
+def ny(_):
+    """ Matcher that matches any value """
+    return True
+
+
+# Support functions
+def _chunks(l, n):
+    for i in range(0, len(l), n):
+        yield l[i:i + n]
+
+
+def transform(structure, transformations):
+    r = structure
+    for path, command in _chunks(transformations, 2):
+        r = _do_to_path(r, path, command)
+    return r
+
+
+def _do_to_path(structure, path, command):
+    if not path:
+        return command(structure) if callable(command) else command
+
+    kvs = _get_keys_and_values(structure, path[0])
+    return _update_structure(structure, kvs, path[1:], command)
+
+
+def _items(structure):
+    try:
+        return structure.items()
+    except AttributeError:
+        # Support wider range of structures by adding a transform_items() or similar?
+        return list(enumerate(structure))
+
+
+def _get(structure, key, default):
+    try:
+        if hasattr(structure, '__getitem__'):
+            return structure[key]
+
+        return getattr(structure, key)
+
+    except (IndexError, KeyError):
+        return default
+
+
+def _get_keys_and_values(structure, key_spec):
+    if callable(key_spec):
+        # Support predicates as callable objects in the path
+        arity = _get_arity(key_spec)
+        if arity == 1:
+            # Unary predicates are called with the "key" of the path
+            # - eg a key in a mapping, an index in a sequence.
+            return [(k, v) for k, v in _items(structure) if key_spec(k)]
+        elif arity == 2:
+            # Binary predicates are called with the key and the corresponding
+            # value.
+            return [(k, v) for k, v in _items(structure) if key_spec(k, v)]
+        else:
+            # Other arities are an error.
+            raise ValueError(
+                "callable in transform path must take 1 or 2 arguments"
+            )
+
+    # Non-callables are used as-is as a key.
+    return [(key_spec, _get(structure, key_spec, _EMPTY_SENTINEL))]
+
+
+if signature is None:
+    def _get_arity(f):
+        argspec = getfullargspec(f)
+        return len(argspec.args) - len(argspec.defaults or ())
+else:
+    def _get_arity(f):
+        return sum(
+            1
+            for p
+            in signature(f).parameters.values()
+            if p.default is Parameter.empty
+            and p.kind in (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD)
+        )
+
+
+def _update_structure(structure, kvs, path, command):
+    from pyrsistent._pmap import pmap
+    e = structure.evolver()
+    if not path and command is discard:
+        # Do this in reverse to avoid index problems with vectors. See #92.
+        for k, v in reversed(kvs):
+            discard(e, k)
+    else:
+        for k, v in kvs:
+            is_empty = False
+            if v is _EMPTY_SENTINEL:
+                if command is discard:
+                    # If nothing there when discarding just move on, do not introduce new nodes
+                    continue
+
+                # Allow expansion of structure but make sure to cover the case
+                # when an empty pmap is added as leaf node. See #154.
+                is_empty = True
+                v = pmap()
+
+            result = _do_to_path(v, path, command)
+            if result is not v or is_empty:
+                e[k] = result
+
+    return e.persistent()
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/py.typed b/contrib/python/pyrsistent/py3/pyrsistent/py.typed
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/python/pyrsistent/py3/pyrsistent/typing.py b/contrib/python/pyrsistent/py3/pyrsistent/typing.py
new file mode 100644
index 00000000000..c97f9db520e
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/pyrsistent/typing.py
@@ -0,0 +1,82 @@
+"""Helpers for use with type annotation.
+
+Use the empty classes in this module when annotating the types of Pyrsistent
+objects, instead of using the actual collection class.
+
+For example,
+
+    from pyrsistent import pvector
+    from pyrsistent.typing import PVector
+
+    myvector: PVector[str] = pvector(['a', 'b', 'c'])
+
+"""
+from __future__ import absolute_import
+
+try:
+    from typing import Container
+    from typing import Hashable
+    from typing import Generic
+    from typing import Iterable
+    from typing import Mapping
+    from typing import Sequence
+    from typing import Sized
+    from typing import TypeVar
+
+    __all__ = [
+        'CheckedPMap',
+        'CheckedPSet',
+        'CheckedPVector',
+        'PBag',
+        'PDeque',
+        'PList',
+        'PMap',
+        'PSet',
+        'PVector',
+    ]
+
+    T = TypeVar('T')
+    T_co = TypeVar('T_co', covariant=True)
+    KT = TypeVar('KT')
+    VT = TypeVar('VT')
+    VT_co = TypeVar('VT_co', covariant=True)
+
+    class CheckedPMap(Mapping[KT, VT_co], Hashable):
+        pass
+
+    # PSet.add and PSet.discard have different type signatures than that of Set.
+    class CheckedPSet(Generic[T_co], Hashable):
+        pass
+
+    class CheckedPVector(Sequence[T_co], Hashable):
+        pass
+
+    class PBag(Container[T_co], Iterable[T_co], Sized, Hashable):
+        pass
+
+    class PDeque(Sequence[T_co], Hashable):
+        pass
+
+    class PList(Sequence[T_co], Hashable):
+        pass
+
+    class PMap(Mapping[KT, VT_co], Hashable):
+        pass
+
+    # PSet.add and PSet.discard have different type signatures than that of Set.
+    class PSet(Generic[T_co], Hashable):
+        pass
+
+    class PVector(Sequence[T_co], Hashable):
+        pass
+
+    class PVectorEvolver(Generic[T]):
+        pass
+
+    class PMapEvolver(Generic[KT, VT]):
+        pass
+
+    class PSetEvolver(Generic[T]):
+        pass
+except ImportError:
+    pass
diff --git a/contrib/python/pyrsistent/py3/tests/bag_test.py b/contrib/python/pyrsistent/py3/tests/bag_test.py
new file mode 100644
index 00000000000..fb80603108c
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/bag_test.py
@@ -0,0 +1,150 @@
+import pytest
+
+from pyrsistent import b, pbag
+
+
+def test_literalish_works():
+    assert b(1, 2) == pbag([1, 2])
+
+def test_empty_bag():
+    """
+    creating an empty pbag returns a singleton.
+
+    Note that this should NOT be relied upon in application code.
+    """
+    assert b() is b()
+
+def test_supports_hash():
+    assert hash(b(1, 2)) == hash(b(2, 1))
+
+def test_hash_in_dict():
+    assert {b(1,2,3,3): "hello"}[b(3,3,2,1)] == "hello"
+
+def test_empty_truthiness():
+    assert b(1)
+    assert not b()
+
+
+def test_repr_empty():
+    assert repr(b()) == 'pbag([])'
+
+def test_repr_elements():
+    assert repr(b(1, 2)) in ('pbag([1, 2])', 'pbag([2, 1])')
+
+
+def test_add_empty():
+    assert b().add(1) == b(1)
+
+def test_remove_final():
+    assert b().add(1).remove(1) == b()
+
+def test_remove_nonfinal():
+    assert b().add(1).add(1).remove(1) == b(1)
+
+def test_remove_nonexistent():
+    with pytest.raises(KeyError) as excinfo:
+        b().remove(1)
+    assert str(excinfo.exconly()) == 'KeyError: 1'
+
+
+def test_eq_empty():
+    assert b() == b()
+
+def test_neq():
+    assert b(1) != b()
+
+def test_eq_same_order():
+    assert b(1, 2, 1) == b(1, 2, 1)
+
+def test_eq_different_order():
+    assert b(2, 1, 2) == b(1, 2, 2)
+
+
+def test_count_non_existent():
+    assert b().count(1) == 0
+
+def test_count_unique():
+    assert b(1).count(1) == 1
+
+def test_count_duplicate():
+    assert b(1, 1).count(1) == 2
+
+
+def test_length_empty():
+    assert len(b()) == 0
+
+def test_length_unique():
+    assert len(b(1)) == 1
+
+def test_length_duplicates():
+    assert len(b(1, 1)) == 2
+
+def test_length_multiple_elements():
+    assert len(b(1, 1, 2, 3)) == 4
+
+
+def test_iter_duplicates():
+    assert list(b(1, 1)) == [1, 1]
+
+def test_iter_multiple_elements():
+    assert list(b(1, 2, 2)) in ([1, 2, 2], [2, 2, 1])
+
+def test_contains():
+    assert 1 in b(1)
+
+def test_not_contains():
+    assert 1 not in b(2)
+
+def test_add():
+    assert b(3, 3, 3, 2, 2, 1) + b(4, 3, 2, 1) == b(4,
+                                                    3, 3, 3, 3,
+                                                    2, 2, 2,
+                                                    1, 1)
+
+def test_sub():
+    assert b(1, 2, 3, 3) - b(3, 4) == b(1, 2, 3)
+
+def test_or():
+    assert b(1, 2, 2, 3, 3, 3) | b(1, 2, 3, 4, 4) == b(1,
+                                                       2, 2,
+                                                       3, 3, 3,
+                                                       4, 4)
+
+def test_and():
+    assert b(1, 2, 2, 3, 3, 3) & b(2, 3, 3, 4) == b(2, 3, 3)
+
+
+def test_pbag_is_unorderable():
+    with pytest.raises(TypeError):
+        _ = b(1) < b(2)  # type: ignore
+
+    with pytest.raises(TypeError):
+        _ = b(1) <= b(2)  # type: ignore
+
+    with pytest.raises(TypeError):
+        _ = b(1) > b(2)  # type: ignore
+
+    with pytest.raises(TypeError):
+        _ = b(1) >= b(2)  # type: ignore
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(b(1))
+
+
+def test_update():
+    assert pbag([1, 2, 2]).update([3, 3, 4]) == pbag([1, 2, 2, 3, 3, 4])
+
+
+def test_update_no_elements():
+    b = pbag([1, 2, 2])
+    assert b.update([]) is b
+
+
+def test_iterable():
+    """
+    PBags can be created from iterables even though they can't be len() hinted.
+    """
+
+    assert pbag(iter("a")) == pbag(iter("a"))
diff --git a/contrib/python/pyrsistent/py3/tests/checked_map_test.py b/contrib/python/pyrsistent/py3/tests/checked_map_test.py
new file mode 100644
index 00000000000..b0ffbceecfe
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/checked_map_test.py
@@ -0,0 +1,152 @@
+import pickle
+import pytest
+from pyrsistent import CheckedPMap, InvariantException, PMap, CheckedType, CheckedPSet, CheckedPVector, \
+    CheckedKeyTypeError, CheckedValueTypeError
+
+
+class FloatToIntMap(CheckedPMap):
+    __key_type__ = float
+    __value_type__ = int
+    __invariant__ = lambda key, value: (int(key) == value, 'Invalid mapping')
+
+def test_instantiate():
+    x = FloatToIntMap({1.25: 1, 2.5: 2})
+
+    assert dict(x.items()) == {1.25: 1, 2.5: 2}
+    assert isinstance(x, FloatToIntMap)
+    assert isinstance(x, PMap)
+    assert isinstance(x, CheckedType)
+
+def test_instantiate_empty():
+    x = FloatToIntMap()
+
+    assert dict(x.items()) == {}
+    assert isinstance(x, FloatToIntMap)
+
+def test_set():
+     x = FloatToIntMap()
+     x2 = x.set(1.0, 1)
+
+     assert x2[1.0] == 1
+     assert isinstance(x2, FloatToIntMap)
+
+def test_invalid_key_type():
+     with pytest.raises(CheckedKeyTypeError):
+         FloatToIntMap({1: 1})
+
+def test_invalid_value_type():
+     with pytest.raises(CheckedValueTypeError):
+         FloatToIntMap({1.0: 1.0})
+
+def test_breaking_invariant():
+     try:
+         FloatToIntMap({1.5: 2})
+         assert False
+     except InvariantException as e:
+        assert e.invariant_errors == ('Invalid mapping',)
+
+def test_repr():
+    x = FloatToIntMap({1.25: 1})
+
+    assert str(x) == 'FloatToIntMap({1.25: 1})'
+
+def test_default_serialization():
+    x = FloatToIntMap({1.25: 1, 2.5: 2})
+
+    assert x.serialize() == {1.25: 1, 2.5: 2}
+
+class StringFloatToIntMap(FloatToIntMap):
+    @staticmethod
+    def __serializer__(format, key, value):
+        return format.format(key), format.format(value)
+
+def test_custom_serialization():
+    x = StringFloatToIntMap({1.25: 1, 2.5: 2})
+
+    assert x.serialize("{0}") == {"1.25": "1", "2.5": "2"}
+
+class FloatSet(CheckedPSet):
+    __type__ = float
+
+class IntToFloatSetMap(CheckedPMap):
+    __key_type__ = int
+    __value_type__ = FloatSet
+
+
+def test_multi_level_serialization():
+    x = IntToFloatSetMap.create({1: [1.25, 1.50], 2: [2.5, 2.75]})
+
+    assert str(x) == "IntToFloatSetMap({1: FloatSet([1.5, 1.25]), 2: FloatSet([2.75, 2.5])})"
+
+    sx = x.serialize()
+    assert sx == {1: set([1.5, 1.25]), 2: set([2.75, 2.5])}
+    assert isinstance(sx[1], set)
+
+def test_create_non_checked_types():
+    assert FloatToIntMap.create({1.25: 1, 2.5: 2}) == FloatToIntMap({1.25: 1, 2.5: 2})
+
+def test_create_checked_types():
+    class IntSet(CheckedPSet):
+        __type__ = int
+
+    class FloatVector(CheckedPVector):
+        __type__ = float
+
+    class IntSetToFloatVectorMap(CheckedPMap):
+        __key_type__ = IntSet
+        __value_type__ = FloatVector
+
+    x = IntSetToFloatVectorMap.create({frozenset([1, 2]): [1.25, 2.5]})
+
+    assert str(x) == "IntSetToFloatVectorMap({IntSet([1, 2]): FloatVector([1.25, 2.5])})"
+
+def test_evolver_returns_same_instance_when_no_updates():
+    x = FloatToIntMap({1.25: 1, 2.25: 2})
+
+    assert x.evolver().persistent() is x
+
+def test_map_with_no_types_or_invariants():
+    class NoCheckPMap(CheckedPMap):
+        pass
+
+    x = NoCheckPMap({1: 2, 3: 4})
+    assert x[1] == 2
+    assert x[3] == 4
+
+
+def test_pickling():
+    x = FloatToIntMap({1.25: 1, 2.5: 2})
+    y = pickle.loads(pickle.dumps(x, -1))
+
+    assert x == y
+    assert isinstance(y, FloatToIntMap)
+
+
+class FloatVector(CheckedPVector):
+    __type__ = float
+
+
+class VectorToSetMap(CheckedPMap):
+    __key_type__ = '__tests__.checked_map_test.FloatVector'
+    __value_type__ = '__tests__.checked_map_test.FloatSet'
+
+
+def test_type_check_with_string_specification():
+    content = [1.5, 2.0]
+    vec = FloatVector(content)
+    sett = FloatSet(content)
+    map = VectorToSetMap({vec: sett})
+
+    assert map[vec] == sett
+
+
+def test_type_creation_with_string_specification():
+    content = (1.5, 2.0)
+    map = VectorToSetMap.create({content: content})
+
+    assert map[FloatVector(content)] == set(content)
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(VectorToSetMap({}))
diff --git a/contrib/python/pyrsistent/py3/tests/checked_set_test.py b/contrib/python/pyrsistent/py3/tests/checked_set_test.py
new file mode 100644
index 00000000000..f0be4963e21
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/checked_set_test.py
@@ -0,0 +1,85 @@
+import pickle
+import pytest
+from pyrsistent import CheckedPSet, PSet, InvariantException, CheckedType, CheckedPVector, CheckedValueTypeError
+
+
+class Naturals(CheckedPSet):
+    __type__ = int
+    __invariant__ = lambda value: (value >= 0, 'Negative value')
+
+def test_instantiate():
+    x = Naturals([1, 2, 3, 3])
+
+    assert list(x) == [1, 2, 3]
+    assert isinstance(x, Naturals)
+    assert isinstance(x, PSet)
+    assert isinstance(x, CheckedType)
+
+def test_add():
+    x = Naturals()
+    x2 = x.add(1)
+
+    assert list(x2) == [1]
+    assert isinstance(x2, Naturals)
+
+def test_invalid_type():
+    with pytest.raises(CheckedValueTypeError):
+        Naturals([1, 2.0])
+
+def test_breaking_invariant():
+    try:
+        Naturals([1, -1])
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+def test_repr():
+    x = Naturals([1, 2])
+
+    assert str(x) == 'Naturals([1, 2])'
+
+def test_default_serialization():
+    x = Naturals([1, 2])
+
+    assert x.serialize() == set([1, 2])
+
+class StringNaturals(Naturals):
+    @staticmethod
+    def __serializer__(format, value):
+        return format.format(value)
+
+def test_custom_serialization():
+    x = StringNaturals([1, 2])
+
+    assert x.serialize("{0}") == set(["1", "2"])
+
+class NaturalsVector(CheckedPVector):
+    __type__ = Naturals
+
+def test_multi_level_serialization():
+    x = NaturalsVector.create([[1, 2], [3, 4]])
+
+    assert str(x) == "NaturalsVector([Naturals([1, 2]), Naturals([3, 4])])"
+
+    sx = x.serialize()
+    assert sx == [set([1, 2]), set([3, 4])]
+    assert isinstance(sx[0], set)
+
+def test_create():
+    assert Naturals.create([1, 2]) == Naturals([1, 2])
+
+def test_evolver_returns_same_instance_when_no_updates():
+    x = Naturals([1, 2])
+    assert x.evolver().persistent() is x
+
+def test_pickling():
+    x = Naturals([1, 2])
+    y = pickle.loads(pickle.dumps(x, -1))
+
+    assert x == y
+    assert isinstance(y, Naturals)
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(Naturals([1, 2]))
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py3/tests/checked_vector_test.py b/contrib/python/pyrsistent/py3/tests/checked_vector_test.py
new file mode 100644
index 00000000000..b2e3d43cd65
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/checked_vector_test.py
@@ -0,0 +1,213 @@
+import datetime
+import pickle
+import pytest
+from pyrsistent import CheckedPVector, InvariantException, optional, CheckedValueTypeError, PVector
+
+
+class Naturals(CheckedPVector):
+    __type__ = int
+    __invariant__ = lambda value: (value >= 0, 'Negative value')
+
+def test_instantiate():
+    x = Naturals([1, 2, 3])
+
+    assert list(x) == [1, 2, 3]
+    assert isinstance(x, Naturals)
+    assert isinstance(x, PVector)
+
+def test_append():
+    x = Naturals()
+    x2 = x.append(1)
+
+    assert list(x2) == [1]
+    assert isinstance(x2, Naturals)
+
+def test_extend():
+    x = Naturals()
+    x2 = x.extend([1])
+
+    assert list(x2) == [1]
+    assert isinstance(x2, Naturals)
+
+def test_set():
+    x = Naturals([1, 2])
+    x2 = x.set(1, 3)
+
+    assert list(x2) == [1, 3]
+    assert isinstance(x2, Naturals)
+
+
+def test_invalid_type():
+    try:
+        Naturals([1, 2.0])
+        assert False
+    except CheckedValueTypeError as e:
+        assert e.expected_types == (int,)
+        assert e.actual_type is float
+        assert e.actual_value == 2.0
+        assert e.source_class is Naturals
+
+    x = Naturals([1, 2])
+    with pytest.raises(TypeError):
+        x.append(3.0)
+
+    with pytest.raises(TypeError):
+        x.extend([3, 4.0])
+
+    with pytest.raises(TypeError):
+        x.set(1, 2.0)
+
+    with pytest.raises(TypeError):
+        x.evolver()[1] = 2.0
+
+def test_breaking_invariant():
+    try:
+        Naturals([1, -1])
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+    x = Naturals([1, 2])
+    try:
+        x.append(-1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+    try:
+        x.extend([-1])
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+    try:
+        x.set(1, -1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Negative value',)
+
+def test_create_base_case():
+    x =  Naturals.create([1, 2, 3])
+
+    assert isinstance(x, Naturals)
+    assert x == Naturals([1, 2, 3])
+
+def test_create_with_instance_of_checked_pvector_returns_the_argument():
+    x =  Naturals([1, 2, 3])
+
+    assert Naturals.create(x) is x
+
+class OptionalNaturals(CheckedPVector):
+    __type__ = optional(int)
+    __invariant__ = lambda value: (value is None or value >= 0, 'Negative value')
+
+def test_multiple_allowed_types():
+    assert list(OptionalNaturals([1, None, 3])) == [1, None, 3]
+
+class NaturalsVector(CheckedPVector):
+    __type__ = optional(Naturals)
+
+def test_create_of_nested_structure():
+    assert NaturalsVector([Naturals([1, 2]), Naturals([3, 4]), None]) ==\
+           NaturalsVector.create([[1, 2], [3, 4], None])
+
+def test_serialize_default_case():
+    v = CheckedPVector([1, 2, 3])
+    assert v.serialize() == [1, 2, 3]
+
+class Dates(CheckedPVector):
+    __type__ = datetime.date
+
+    @staticmethod
+    def __serializer__(format, d):
+        return d.strftime(format)
+
+def test_serialize_custom_serializer():
+    d = datetime.date
+    v = Dates([d(2015, 2, 2), d(2015, 2, 3)])
+    assert v.serialize(format='%Y-%m-%d') == ['2015-02-02', '2015-02-03']
+
+def test_type_information_is_inherited():
+    class MultiDates(Dates):
+        __type__ = int
+
+    MultiDates([datetime.date(2015, 2, 4), 5])
+
+    with pytest.raises(TypeError):
+        MultiDates([5.0])
+
+def test_invariants_are_inherited():
+    class LimitNaturals(Naturals):
+        __invariant__ = lambda value: (value < 10, 'Too big')
+
+    try:
+        LimitNaturals([10, -1])
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('Too big', 'Negative value')
+
+def test_invariant_must_be_callable():
+    with pytest.raises(TypeError):
+        class InvalidInvariant(CheckedPVector):
+            __invariant__ = 1
+
+def test_type_spec_must_be_type():
+    with pytest.raises(TypeError):
+        class InvalidType(CheckedPVector):
+            __type__ = 1
+
+def test_repr():
+    x = Naturals([1, 2])
+
+    assert str(x) == 'Naturals([1, 2])'
+
+def test_evolver_returns_same_instance_when_no_updates():
+    x = Naturals([1, 2])
+    assert x.evolver().persistent() is x
+
+def test_pickling():
+    x = Naturals([1, 2])
+    y = pickle.loads(pickle.dumps(x, -1))
+
+    assert x == y
+    assert isinstance(y, Naturals)
+
+def test_multiple_optional_types():
+    class Numbers(CheckedPVector):
+        __type__ = optional(int, float)
+
+    numbers = Numbers([1, 2.5, None])
+    assert numbers.serialize() == [1, 2.5, None]
+
+    with pytest.raises(TypeError):
+        numbers.append('foo')
+
+
+class NaturalsVectorStr(CheckedPVector):
+    __type__ = '__tests__.checked_vector_test.Naturals'
+
+
+def test_check_with_string_specification():
+    naturals_list = [Naturals([1, 2]), Naturals([3, 4])]
+    nv = NaturalsVectorStr(naturals_list)
+    assert nv == naturals_list
+
+
+def test_create_with_string_specification():
+    naturals_list = [[1, 2], [3, 4]]
+    nv = NaturalsVectorStr.create(naturals_list)
+    assert nv == naturals_list
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(Naturals([]))
+
+
+def test_create_with_generator_iterator():
+    # See issue #97
+    class Numbers(CheckedPVector):
+        __type__ = int
+
+    n = Numbers(i for i in [1, 2, 3])
+    assert n == Numbers([1, 2, 3])
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py3/tests/class_test.py b/contrib/python/pyrsistent/py3/tests/class_test.py
new file mode 100644
index 00000000000..5e953965d5a
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/class_test.py
@@ -0,0 +1,474 @@
+from collections.abc import Hashable
+import math
+import pickle
+import pytest
+import uuid
+from pyrsistent import (
+    field, InvariantException, PClass, optional, CheckedPVector,
+    pmap_field, pset_field, pvector_field)
+
+
+class Point(PClass):
+    x = field(type=int, mandatory=True, invariant=lambda x: (x >= 0, 'X negative'))
+    y = field(type=int, serializer=lambda formatter, y: formatter(y))
+    z = field(type=int, initial=0)
+
+
+class Hierarchy(PClass):
+    point = field(type=Point)
+
+
+class TypedContainerObj(PClass):
+    map = pmap_field(str, str)
+    set = pset_field(str)
+    vec = pvector_field(str)
+
+
+class UniqueThing(PClass):
+    id = field(type=uuid.UUID, factory=uuid.UUID)
+    x = field(type=int)
+
+
+def test_create_ignore_extra():
+    p = Point.create({'x': 5, 'y': 10, 'z': 15, 'a': 0}, ignore_extra=True)
+    assert p.x == 5
+    assert p.y == 10
+    assert p.z == 15
+    assert isinstance(p, Point)
+
+
+def test_create_ignore_extra_false():
+    with pytest.raises(AttributeError):
+        _ = Point.create({'x': 5, 'y': 10, 'z': 15, 'a': 0})
+
+
+def test_create_ignore_extra_true():
+    h = Hierarchy.create(
+        {'point': {'x': 5, 'y': 10, 'z': 15, 'extra_field_0': 'extra_data_0'}, 'extra_field_1': 'extra_data_1'},
+        ignore_extra=True)
+    assert isinstance(h, Hierarchy)
+
+
+def test_evolve_pclass_instance():
+    p = Point(x=1, y=2)
+    p2 = p.set(x=p.x+2)
+
+    # Original remains
+    assert p.x == 1
+    assert p.y == 2
+
+    # Evolved object updated
+    assert p2.x == 3
+    assert p2.y == 2
+
+    p3 = p2.set('x', 4)
+    assert p3.x == 4
+    assert p3.y == 2
+
+
+def test_direct_assignment_not_possible():
+    p = Point(x=1, y=2)
+
+    with pytest.raises(AttributeError):
+        p.x = 1
+
+    with pytest.raises(AttributeError):
+        setattr(p, 'x', 1)
+
+
+def test_direct_delete_not_possible():
+    p = Point(x=1, y=2)
+    with pytest.raises(AttributeError):
+        del p.x
+
+    with pytest.raises(AttributeError):
+        delattr(p, 'x')
+
+
+def test_cannot_construct_with_undeclared_fields():
+    with pytest.raises(AttributeError):
+        Point(x=1, p=5)
+
+
+def test_cannot_construct_with_wrong_type():
+    with pytest.raises(TypeError):
+        Point(x='a')
+
+
+def test_cannot_construct_without_mandatory_fields():
+    try:
+        Point(y=1)
+        assert False
+    except InvariantException as e:
+        assert "[Point.x]" in str(e)
+
+
+def test_field_invariant_must_hold():
+    try:
+        Point(x=-1)
+        assert False
+    except InvariantException as e:
+        assert "X negative" in str(e)
+
+
+def test_initial_value_set_when_not_present_in_arguments():
+    p = Point(x=1, y=2)
+
+    assert p.z == 0
+
+
+class Line(PClass):
+    p1 = field(type=Point)
+    p2 = field(type=Point)
+
+
+def test_can_create_nested_structures_from_dict_and_serialize_back_to_dict():
+    source = dict(p1=dict(x=1, y=2, z=3), p2=dict(x=10, y=20, z=30))
+    l = Line.create(source)
+
+    assert l.p1.x == 1
+    assert l.p1.y == 2
+    assert l.p1.z == 3
+    assert l.p2.x == 10
+    assert l.p2.y == 20
+    assert l.p2.z == 30
+
+    assert l.serialize(format=lambda val: val) == source
+
+
+def test_can_serialize_with_custom_serializer():
+    p = Point(x=1, y=1, z=1)
+
+    assert p.serialize(format=lambda v: v + 17) == {'x': 1, 'y': 18, 'z': 1}
+
+
+def test_implements_proper_equality_based_on_equality_of_fields():
+    p1 = Point(x=1, y=2)
+    p2 = Point(x=3)
+    p3 = Point(x=1, y=2)
+
+    assert p1 == p3
+    assert not p1 != p3
+    assert p1 != p2
+    assert not p1 == p2
+
+
+def test_is_hashable():
+    p1 = Point(x=1, y=2)
+    p2 = Point(x=3, y=2)
+
+    d = {p1: 'A point', p2: 'Another point'}
+
+    p1_like = Point(x=1, y=2)
+    p2_like = Point(x=3, y=2)
+
+    assert isinstance(p1, Hashable)
+    assert d[p1_like] == 'A point'
+    assert d[p2_like] == 'Another point'
+    assert Point(x=10) not in d
+
+
+def test_supports_nested_transformation():
+    l1 = Line(p1=Point(x=2, y=1), p2=Point(x=20, y=10))
+
+    l2 = l1.transform(['p1', 'x'], 3)
+
+    assert l1.p1.x == 2
+
+    assert l2.p1.x == 3
+    assert l2.p1.y == 1
+    assert l2.p2.x == 20
+    assert l2.p2.y == 10
+
+
+def test_repr():
+    class ARecord(PClass):
+        a = field()
+        b = field()
+
+    assert repr(ARecord(a=1, b=2)) in ('ARecord(a=1, b=2)', 'ARecord(b=2, a=1)')
+
+
+def test_global_invariant_check():
+    class UnitCirclePoint(PClass):
+        __invariant__ = lambda cp: (0.99 < math.sqrt(cp.x*cp.x + cp.y*cp.y) < 1.01,
+                                    "Point not on unit circle")
+        x = field(type=float)
+        y = field(type=float)
+
+    UnitCirclePoint(x=1.0, y=0.0)
+
+    with pytest.raises(InvariantException):
+        UnitCirclePoint(x=1.0, y=1.0)
+
+
+def test_supports_pickling():
+    p1 = Point(x=2, y=1)
+    p2 = pickle.loads(pickle.dumps(p1, -1))
+
+    assert p1 == p2
+    assert isinstance(p2, Point)
+
+
+def test_supports_pickling_with_typed_container_fields():
+    obj = TypedContainerObj(map={'foo': 'bar'}, set=['hello', 'there'], vec=['a', 'b'])
+    obj2 = pickle.loads(pickle.dumps(obj))
+    assert obj == obj2
+
+
+def test_can_remove_optional_member():
+    p1 = Point(x=1, y=2)
+    p2 = p1.remove('y')
+
+    assert p2 == Point(x=1)
+
+
+def test_cannot_remove_mandatory_member():
+    p1 = Point(x=1, y=2)
+
+    with pytest.raises(InvariantException):
+        p1.remove('x')
+
+
+def test_cannot_remove_non_existing_member():
+    p1 = Point(x=1)
+
+    with pytest.raises(AttributeError):
+        p1.remove('y')
+
+
+def test_evolver_without_evolution_returns_original_instance():
+    p1 = Point(x=1)
+    e = p1.evolver()
+
+    assert e.persistent() is p1
+
+
+def test_evolver_with_evolution_to_same_element_returns_original_instance():
+    p1 = Point(x=1)
+    e = p1.evolver()
+    e.set('x', p1.x)
+
+    assert e.persistent() is p1
+
+
+def test_evolver_supports_chained_set_and_remove():
+    p1 = Point(x=1, y=2)
+
+    assert p1.evolver().set('x', 3).remove('y').persistent() == Point(x=3)
+
+
+def test_evolver_supports_dot_notation_for_setting_and_getting_elements():
+    e = Point(x=1, y=2).evolver()
+
+    e.x = 3
+    assert e.x == 3
+    assert e.persistent() == Point(x=3, y=2)
+
+
+class Numbers(CheckedPVector):
+    __type__ = int
+
+
+class LinkedList(PClass):
+    value = field(type='__tests__.class_test.Numbers')
+    next = field(type=optional('__tests__.class_test.LinkedList'))
+
+
+def test_string_as_type_specifier():
+    l = LinkedList(value=[1, 2], next=LinkedList(value=[3, 4], next=None))
+
+    assert isinstance(l.value, Numbers)
+    assert list(l.value) == [1, 2]
+    assert l.next.next is None
+
+
+def test_multiple_invariants_on_field():
+    # If the invariant returns a list of tests the results of running those tests will be
+    # a tuple containing result data of all failing tests.
+
+    class MultiInvariantField(PClass):
+        one = field(type=int, invariant=lambda x: ((False, 'one_one'),
+                                                   (False, 'one_two'),
+                                                   (True, 'one_three')))
+        two = field(invariant=lambda x: (False, 'two_one'))
+
+    try:
+        MultiInvariantField(one=1, two=2)
+        assert False
+    except InvariantException as e:
+        assert set(e.invariant_errors) == set([('one_one', 'one_two'), 'two_one'])
+
+
+def test_multiple_global_invariants():
+    class MultiInvariantGlobal(PClass):
+        __invariant__ = lambda self: ((False, 'x'), (False, 'y'))
+        one = field()
+
+    try:
+        MultiInvariantGlobal(one=1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == (('x', 'y'),)
+
+
+def test_inherited_global_invariants():
+    class Distant(object):
+        def __invariant__(self):
+            return [(self.distant, "distant")]
+
+    class Nearby(Distant):
+        def __invariant__(self):
+            return [(self.nearby, "nearby")]
+
+    class MultipleInvariantGlobal(Nearby, PClass):
+        distant = field()
+        nearby = field()
+
+    try:
+        MultipleInvariantGlobal(distant=False, nearby=False)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == (("nearby",), ("distant",),)
+
+
+def test_diamond_inherited_global_invariants():
+    counter = []
+    class Base(object):
+        def __invariant__(self):
+            counter.append(None)
+            return [(False, "base")]
+
+    class Left(Base):
+        pass
+
+    class Right(Base):
+        pass
+
+    class SingleInvariantGlobal(Left, Right, PClass):
+        pass
+
+    try:
+        SingleInvariantGlobal()
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == (("base",),)
+        assert counter == [None]
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(Point(x=1, y=2))
+
+
+def test_supports_weakref_with_multi_level_inheritance():
+    import weakref
+
+    class PPoint(Point):
+        a = field()
+
+    weakref.ref(PPoint(x=1, y=2))
+
+
+def test_supports_lazy_initial_value_for_field():
+    class MyClass(PClass):
+        a = field(int, initial=lambda: 2)
+
+    assert MyClass() == MyClass(a=2)
+
+
+def test_type_checks_lazy_initial_value_for_field():
+    class MyClass(PClass):
+        a = field(int, initial=lambda: "a")
+
+    with pytest.raises(TypeError):
+        MyClass()
+
+
+def test_invariant_checks_lazy_initial_value_for_field():
+    class MyClass(PClass):
+        a = field(int, invariant=lambda x: (x < 5, "Too large"), initial=lambda: 10)
+
+    with pytest.raises(InvariantException):
+        MyClass()
+
+
+def test_invariant_checks_static_initial_value():
+    class MyClass(PClass):
+        a = field(int, invariant=lambda x: (x < 5, "Too large"), initial=10)
+
+    with pytest.raises(InvariantException):
+        MyClass()
+
+
+def test_lazy_invariant_message():
+    class MyClass(PClass):
+        a = field(int, invariant=lambda x: (x < 5, lambda: "{x} is too large".format(x=x)))
+
+    try:
+        MyClass(a=5)
+        assert False
+    except InvariantException as e:
+        assert '5 is too large' in e.invariant_errors
+
+
+def test_enum_key_type():
+    import enum
+    class Foo(enum.Enum):
+        Bar = 1
+        Baz = 2
+
+    # This currently fails because the enum is iterable
+    class MyClass1(PClass):
+        f = pmap_field(key_type=Foo, value_type=int)
+
+    MyClass1()
+
+    # This is OK since it's wrapped in a tuple
+    class MyClass2(PClass):
+        f = pmap_field(key_type=(Foo,), value_type=int)
+
+    MyClass2()
+
+
+def test_pickle_with_one_way_factory():
+    thing = UniqueThing(id='25544626-86da-4bce-b6b6-9186c0804d64')
+    assert pickle.loads(pickle.dumps(thing)) == thing
+
+
+def test_evolver_with_one_way_factory():
+    thing = UniqueThing(id='cc65249a-56fe-4995-8719-ea02e124b234')
+    ev = thing.evolver()
+    ev.x = 5  # necessary to prevent persistent() returning the original
+    assert ev.persistent() == UniqueThing(id=str(thing.id), x=5)
+
+
+def test_set_doesnt_trigger_other_factories():
+    thing = UniqueThing(id='b413b280-de76-4e28-a8e3-5470ca83ea2c')
+    thing.set(x=5)
+
+
+def test_set_does_trigger_factories():
+    class SquaredPoint(PClass):
+        x = field(factory=lambda x: x ** 2)
+        y = field()
+
+    sp = SquaredPoint(x=3, y=10)
+    assert (sp.x, sp.y) == (9, 10)
+
+    sp2 = sp.set(x=4)
+    assert (sp2.x, sp2.y) == (16, 10)
+
+
+def test_value_can_be_overridden_in_subclass_new():
+    class X(PClass):
+        y = pvector_field(int)
+
+        def __new__(cls, **kwargs):
+            items = kwargs.get('y', None)
+            if items is None:
+                kwargs['y'] = ()
+            return super(X, cls).__new__(cls, **kwargs)
+
+    a = X(y=[])
+    b = a.set(y=None)
+    assert a == b
diff --git a/contrib/python/pyrsistent/py3/tests/deque_test.py b/contrib/python/pyrsistent/py3/tests/deque_test.py
new file mode 100644
index 00000000000..7798a755834
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/deque_test.py
@@ -0,0 +1,293 @@
+import pickle
+import pytest
+from pyrsistent import pdeque, dq
+
+
+def test_basic_right_and_left():
+    x = pdeque([1, 2])
+
+    assert x.right == 2
+    assert x.left == 1
+    assert len(x) == 2
+
+
+def test_construction_with_maxlen():
+    assert pdeque([1, 2, 3, 4], maxlen=2) == pdeque([3, 4])
+    assert pdeque([1, 2, 3, 4], maxlen=4) == pdeque([1, 2, 3, 4])
+    assert pdeque([], maxlen=2) == pdeque()
+
+
+def test_construction_with_invalid_maxlen():
+    with pytest.raises(TypeError):
+        pdeque([], maxlen='foo')
+
+    with pytest.raises(ValueError):
+        pdeque([], maxlen=-3)
+
+
+def test_pop():
+    x = pdeque([1, 2, 3, 4]).pop()
+    assert x.right == 3
+    assert x.left == 1
+
+    x = x.pop()
+    assert x.right == 2
+    assert x.left == 1
+
+    x = x.pop()
+    assert x.right == 1
+    assert x.left == 1
+
+    x = x.pop()
+    assert x == pdeque()
+
+    x = pdeque([1, 2]).pop()
+    assert x == pdeque([1])
+
+    x = x.pop()
+    assert x == pdeque()
+
+    assert pdeque().append(1).pop() == pdeque()
+    assert pdeque().appendleft(1).pop() == pdeque()
+
+
+def test_pop_multiple():
+    assert pdeque([1, 2, 3, 4]).pop(3) == pdeque([1])
+    assert pdeque([1, 2]).pop(3) == pdeque()
+
+
+def test_pop_with_negative_index():
+    assert pdeque([1, 2, 3]).pop(-1) == pdeque([1, 2, 3]).popleft(1)
+    assert pdeque([1, 2, 3]).popleft(-1) == pdeque([1, 2, 3]).pop(1)
+
+
+def test_popleft():
+    x = pdeque([1, 2, 3, 4]).popleft()
+    assert x.left == 2
+    assert x.right == 4
+
+    x = x.popleft()
+    assert x.left == 3
+    assert x.right == 4
+
+    x = x.popleft()
+    assert x.right == 4
+    assert x.left == 4
+
+    x = x.popleft()
+    assert x == pdeque()
+
+    x = pdeque([1, 2]).popleft()
+    assert x == pdeque([2])
+
+    x = x.popleft()
+    assert x == pdeque()
+
+    assert pdeque().append(1).popleft() == pdeque()
+    assert pdeque().appendleft(1).popleft() == pdeque()
+
+
+def test_popleft_multiple():
+    assert pdeque([1, 2, 3, 4]).popleft(3) == pdeque([4])
+
+
+def test_left_on_empty_deque():
+    with pytest.raises(IndexError):
+        pdeque().left
+
+
+def test_right_on_empty_deque():
+    with pytest.raises(IndexError):
+        pdeque().right
+
+
+def test_pop_empty_deque_returns_empty_deque():
+    # The other option is to throw an index error, this is what feels best for now though
+    assert pdeque().pop() == pdeque()
+    assert pdeque().popleft() == pdeque()
+
+
+def test_str():
+    assert str(pdeque([1, 2, 3])) == 'pdeque([1, 2, 3])'
+    assert str(pdeque([])) == 'pdeque([])'
+    assert str(pdeque([1, 2], maxlen=4)) == 'pdeque([1, 2], maxlen=4)'
+
+
+def test_append():
+    assert pdeque([1, 2]).append(3).append(4) == pdeque([1, 2, 3, 4])
+
+
+def test_append_with_maxlen():
+    assert pdeque([1, 2], maxlen=2).append(3).append(4) == pdeque([3, 4])
+    assert pdeque([1, 2], maxlen=3).append(3).append(4) == pdeque([2, 3, 4])
+    assert pdeque([], maxlen=0).append(1) == pdeque()
+
+
+def test_appendleft():
+    assert pdeque([2, 1]).appendleft(3).appendleft(4) == pdeque([4, 3, 2, 1])
+
+
+def test_appendleft_with_maxlen():
+    assert pdeque([2, 1], maxlen=2).appendleft(3).appendleft(4) == pdeque([4, 3])
+    assert pdeque([2, 1], maxlen=3).appendleft(3).appendleft(4) == pdeque([4, 3, 2])
+    assert pdeque([], maxlen=0).appendleft(1) == pdeque()
+
+
+def test_extend():
+    assert pdeque([1, 2]).extend([3, 4]) == pdeque([1, 2, 3, 4])
+
+
+def test_extend_with_maxlen():
+    assert pdeque([1, 2], maxlen=3).extend([3, 4]) == pdeque([2, 3, 4])
+    assert pdeque([1, 2], maxlen=2).extend([3, 4]) == pdeque([3, 4])
+    assert pdeque([], maxlen=2).extend([1, 2]) == pdeque([1, 2])
+    assert pdeque([], maxlen=0).extend([1, 2]) == pdeque([])
+
+
+def test_extendleft():
+    assert pdeque([2, 1]).extendleft([3, 4]) == pdeque([4, 3, 2, 1])
+
+
+def test_extendleft_with_maxlen():
+    assert pdeque([1, 2], maxlen=3).extendleft([3, 4]) == pdeque([4, 3, 1])
+    assert pdeque([1, 2], maxlen=2).extendleft([3, 4]) == pdeque([4, 3])
+    assert pdeque([], maxlen=2).extendleft([1, 2]) == pdeque([2, 1])
+    assert pdeque([], maxlen=0).extendleft([1, 2]) == pdeque([])
+
+
+def test_count():
+    x = pdeque([1, 2, 3, 2, 1])
+    assert x.count(1) == 2
+    assert x.count(2) == 2
+
+
+def test_remove():
+    assert pdeque([1, 2, 3, 4]).remove(2) == pdeque([1, 3, 4])
+    assert pdeque([1, 2, 3, 4]).remove(4) == pdeque([1, 2, 3])
+
+    # Right list must be reversed before removing element
+    assert pdeque([1, 2, 3, 3, 4, 5, 4, 6]).remove(4) == pdeque([1, 2, 3, 3, 5, 4, 6])
+
+
+def test_remove_element_missing():
+    with pytest.raises(ValueError):
+        pdeque().remove(2)
+
+    with pytest.raises(ValueError):
+        pdeque([1, 2, 3]).remove(4)
+
+
+def test_reverse():
+    assert pdeque([1, 2, 3, 4]).reverse() == pdeque([4, 3, 2, 1])
+    assert pdeque().reverse() == pdeque()
+
+
+def test_rotate_right():
+    assert pdeque([1, 2, 3, 4, 5]).rotate(2) == pdeque([4, 5, 1, 2, 3])
+    assert pdeque([1, 2]).rotate(0) == pdeque([1, 2])
+    assert pdeque().rotate(2) == pdeque()
+
+
+def test_rotate_left():
+    assert pdeque([1, 2, 3, 4, 5]).rotate(-2) == pdeque([3, 4, 5, 1, 2])
+    assert pdeque().rotate(-2) == pdeque()
+
+
+def test_set_maxlen():
+    x = pdeque([], maxlen=4)
+    assert x.maxlen == 4
+
+    with pytest.raises(AttributeError):
+        x.maxlen = 5
+
+
+def test_comparison():
+    small = pdeque([1, 2])
+    large = pdeque([1, 2, 3])
+
+    assert small < large
+    assert large > small
+    assert not small > large
+    assert not large < small
+    assert large != small
+
+    # Not equal to other types
+    assert small != [1, 2]
+
+
+def test_pickling():
+    input = pdeque([1, 2, 3], maxlen=5)
+    output = pickle.loads(pickle.dumps(input, -1))
+
+    assert output == input
+    assert output.maxlen == input.maxlen
+
+
+def test_indexing():
+    assert pdeque([1, 2, 3])[0] == 1
+    assert pdeque([1, 2, 3])[1] == 2
+    assert pdeque([1, 2, 3])[2] == 3
+    assert pdeque([1, 2, 3])[-1] == 3
+    assert pdeque([1, 2, 3])[-2] == 2
+    assert pdeque([1, 2, 3])[-3] == 1
+
+
+def test_one_element_indexing():
+    assert pdeque([2])[0] == 2
+    assert pdeque([2])[-1] == 2
+
+
+def test_empty_indexing():
+    with pytest.raises(IndexError):
+        assert pdeque([])[0] == 1
+
+
+def test_indexing_out_of_range():
+    with pytest.raises(IndexError):
+        pdeque([1, 2, 3])[-4]
+
+    with pytest.raises(IndexError):
+        pdeque([1, 2, 3])[3]
+
+    with pytest.raises(IndexError):
+        pdeque([2])[-2]
+
+
+def test_indexing_invalid_type():
+    with pytest.raises(TypeError) as e:
+        pdeque([1, 2, 3])['foo']
+
+    assert 'cannot be interpreted' in str(e.value)
+
+
+def test_slicing():
+    assert pdeque([1, 2, 3])[1:2] == pdeque([2])
+    assert pdeque([1, 2, 3])[2:1] == pdeque([])
+    assert pdeque([1, 2, 3])[-2:-1] == pdeque([2])
+    assert pdeque([1, 2, 3])[::2] == pdeque([1, 3])
+
+
+def test_hashing():
+    assert hash(pdeque([1, 2, 3])) == hash(pdeque().append(1).append(2).append(3))
+
+
+def test_index():
+    assert pdeque([1, 2, 3]).index(3) == 2
+
+
+def test_literalish():
+    assert dq(1, 2, 3) == pdeque([1, 2, 3])
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(dq(1, 2))
+
+
+def test_iterable():
+    """
+    PDeques can be created from iterables even though they can't be len()
+    hinted.
+    """
+
+    assert pdeque(iter("a")) == pdeque(iter("a"))
diff --git a/contrib/python/pyrsistent/py3/tests/field_test.py b/contrib/python/pyrsistent/py3/tests/field_test.py
new file mode 100644
index 00000000000..176b64cc6be
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/field_test.py
@@ -0,0 +1,23 @@
+from enum import Enum
+
+from pyrsistent import field, pvector_field
+
+
+class ExampleEnum(Enum):
+    x = 1
+    y = 2
+
+
+def test_enum():
+    f = field(type=ExampleEnum)
+
+    assert ExampleEnum in f.type
+    assert len(f.type) == 1
+
+
+# This is meant to exercise `_seq_field`.
+def test_pvector_field_enum_type():
+    f = pvector_field(ExampleEnum)
+
+    assert len(f.type) == 1
+    assert ExampleEnum is list(f.type)[0].__type__
diff --git a/contrib/python/pyrsistent/py3/tests/freeze_test.py b/contrib/python/pyrsistent/py3/tests/freeze_test.py
new file mode 100644
index 00000000000..158cf5d872a
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/freeze_test.py
@@ -0,0 +1,174 @@
+"""Tests for freeze and thaw."""
+import collections
+from pyrsistent import v, m, s, freeze, thaw, PRecord, field, mutant
+
+
+## Freeze (standard)
+
+def test_freeze_basic():
+    assert freeze(1) == 1
+    assert freeze('foo') == 'foo'
+
+def test_freeze_list():
+    assert freeze([1, 2]) == v(1, 2)
+
+def test_freeze_dict():
+    result = freeze({'a': 'b'})
+    assert result == m(a='b')
+    assert type(freeze({'a': 'b'})) is type(m())
+
+def test_freeze_defaultdict():
+    test_dict = collections.defaultdict(dict)
+    test_dict['a'] = 'b'
+    result = freeze(test_dict)
+    assert result == m(a='b')
+    assert type(freeze({'a': 'b'})) is type(m())
+
+def test_freeze_set():
+    result = freeze(set([1, 2, 3]))
+    assert result == s(1, 2, 3)
+    assert type(result) is type(s())
+
+def test_freeze_recurse_in_dictionary_values():
+    result = freeze({'a': [1]})
+    assert result == m(a=v(1))
+    assert type(result['a']) is type(v())
+
+def test_freeze_recurse_in_defaultdict_values():
+    test_dict = collections.defaultdict(dict)
+    test_dict['a'] = [1]
+    result = freeze(test_dict)
+    assert result == m(a=v(1))
+    assert type(result['a']) is type(v())
+
+def test_freeze_recurse_in_pmap_values():
+    input = {'a': m(b={'c': 1})}
+    result = freeze(input)
+    # PMap and PVector are == to their mutable equivalents
+    assert result == input
+    assert type(result) is type(m())
+    assert type(result['a']['b']) is type(m())
+
+def test_freeze_recurse_in_lists():
+    result = freeze(['a', {'b': 3}])
+    assert result == v('a', m(b=3))
+    assert type(result[1]) is type(m())
+
+def test_freeze_recurse_in_pvectors():
+    input = [1, v(2, [3])]
+    result = freeze(input)
+    # PMap and PVector are == to their mutable equivalents
+    assert result == input
+    assert type(result) is type(v())
+    assert type(result[1][1]) is type(v())
+
+def test_freeze_recurse_in_tuples():
+    """Values in tuples are recursively frozen."""
+    result = freeze(('a', {}))
+    assert result == ('a', m())
+    assert type(result[1]) is type(m())
+
+
+## Freeze (weak)
+
+def test_freeze_nonstrict_no_recurse_in_pmap_values():
+    input = {'a': m(b={'c': 1})}
+    result = freeze(input, strict=False)
+    # PMap and PVector are == to their mutable equivalents
+    assert result == input
+    assert type(result) is type(m())
+    assert type(result['a']['b']) is dict
+
+def test_freeze_nonstrict_no_recurse_in_pvectors():
+    input = [1, v(2, [3])]
+    result = freeze(input, strict=False)
+    # PMap and PVector are == to their mutable equivalents
+    assert result == input
+    assert type(result) is type(v())
+    assert type(result[1][1]) is list
+
+
+## Thaw
+
+def test_thaw_basic():
+    assert thaw(1) == 1
+    assert thaw('foo') == 'foo'
+
+def test_thaw_list():
+    result = thaw(v(1, 2))
+    assert result == [1, 2]
+    assert type(result) is list
+
+def test_thaw_dict():
+    result = thaw(m(a='b'))
+    assert result == {'a': 'b'}
+    assert type(result) is dict
+
+def test_thaw_set():
+    result = thaw(s(1, 2))
+    assert result == set([1, 2])
+    assert type(result) is set
+
+def test_thaw_recurse_in_mapping_values():
+    result = thaw(m(a=v(1)))
+    assert result == {'a': [1]}
+    assert type(result['a']) is list
+
+def test_thaw_recurse_in_dict_values():
+    result = thaw({'a': v(1, m(b=2))})
+    assert result == {'a': [1, {'b': 2}]}
+    assert type(result['a']) is list
+    assert type(result['a'][1]) is dict
+
+def test_thaw_recurse_in_vectors():
+    result = thaw(v('a', m(b=3)))
+    assert result == ['a', {'b': 3}]
+    assert type(result[1]) is dict
+
+def test_thaw_recurse_in_lists():
+    result = thaw(v(['a', m(b=1), v(2)]))
+    assert result == [['a', {'b': 1}, [2]]]
+    assert type(result[0]) is list
+    assert type(result[0][1]) is dict
+
+def test_thaw_recurse_in_tuples():
+    result = thaw(('a', m()))
+    assert result == ('a', {})
+    assert type(result[1]) is dict
+
+def test_thaw_can_handle_subclasses_of_persistent_base_types():
+    class R(PRecord):
+        x = field()
+
+    result = thaw(R(x=1))
+    assert result == {'x': 1}
+    assert type(result) is dict
+
+
+## Thaw (weak)
+
+def test_thaw_non_strict_no_recurse_in_dict_values():
+    result = thaw({'a': v(1, m(b=2))}, strict=False)
+    assert result == {'a': [1, {'b': 2}]}
+    assert type(result['a']) is type(v())
+    assert type(result['a'][1]) is type(m())
+
+def test_thaw_non_strict_no_recurse_in_lists():
+    result = thaw(v(['a', m(b=1), v(2)]), strict=False)
+    assert result == [['a', {'b': 1}, [2]]]
+    assert type(result[0][1]) is type(m())
+
+def test_mutant_decorator():
+    @mutant
+    def fn(a_list, a_dict):
+        assert a_list == v(1, 2, 3)
+        assert isinstance(a_dict, type(m()))
+        assert a_dict == {'a': 5}
+
+        return [1, 2, 3], {'a': 3}
+
+    pv, pm = fn([1, 2, 3], a_dict={'a': 5})
+
+    assert pv == v(1, 2, 3)
+    assert pm == m(a=3)
+    assert isinstance(pm, type(m()))
diff --git a/contrib/python/pyrsistent/py3/tests/hypothesis_vector_test.py b/contrib/python/pyrsistent/py3/tests/hypothesis_vector_test.py
new file mode 100644
index 00000000000..73e82abf0bc
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/hypothesis_vector_test.py
@@ -0,0 +1,304 @@
+"""
+Hypothesis-based tests for pvector.
+"""
+
+import gc
+
+from collections.abc import Iterable
+from functools import wraps
+from pyrsistent import PClass, field
+
+from pytest import fixture
+
+from pyrsistent import pvector, discard
+
+from hypothesis import strategies as st, assume
+from hypothesis.stateful import RuleBasedStateMachine, Bundle, rule
+
+
+class RefCountTracker:
+    """
+    An object that might catch reference count errors sometimes.
+    """
+    def __init__(self):
+        self.id = id(self)
+
+    def __repr__(self):
+        return "<%s>" % (self.id,)
+
+    def __del__(self):
+        # If self is a dangling memory reference this check might fail. Or
+        # segfault :)
+        if self.id != id(self):
+            raise RuntimeError()
+
+
+@fixture(scope="module")
+def gc_when_done(request):
+    request.addfinalizer(gc.collect)
+
+
+def test_setup(gc_when_done):
+    """
+    Ensure we GC when tests finish.
+    """
+
+
+# Pairs of a list and corresponding pvector:
+PVectorAndLists = st.lists(st.builds(RefCountTracker)).map(
+    lambda l: (l, pvector(l)))
+
+
+def verify_inputs_unmodified(original):
+    """
+    Decorator that asserts that the wrapped function does not modify its
+    inputs.
+    """
+    def to_tuples(pairs):
+        return [(tuple(l), tuple(pv)) for (l, pv) in pairs]
+
+    @wraps(original)
+    def wrapper(self, **kwargs):
+        inputs = [k for k in kwargs.values() if isinstance(k, Iterable)]
+        tuple_inputs = to_tuples(inputs)
+        try:
+            return original(self, **kwargs)
+        finally:
+            # Ensure inputs were unmodified:
+            assert to_tuples(inputs) == tuple_inputs
+    return wrapper
+
+
+def assert_equal(l, pv):
+    assert l == pv
+    assert len(l) == len(pv)
+    length = len(l)
+    for i in range(length):
+        assert l[i] == pv[i]
+    for i in range(length):
+        for j in range(i, length):
+            assert l[i:j] == pv[i:j]
+    assert l == list(iter(pv))
+
+
+class PVectorBuilder(RuleBasedStateMachine):
+    """
+    Build a list and matching pvector step-by-step.
+
+    In each step in the state machine we do same operation on a list and
+    on a pvector, and then when we're done we compare the two.
+    """
+    sequences = Bundle("sequences")
+
+    @rule(target=sequences, start=PVectorAndLists)
+    def initial_value(self, start):
+        """
+        Some initial values generated by a hypothesis strategy.
+        """
+        return start
+
+    @rule(target=sequences, former=sequences)
+    @verify_inputs_unmodified
+    def append(self, former):
+        """
+        Append an item to the pair of sequences.
+        """
+        l, pv = former
+        obj = RefCountTracker()
+        l2 = l[:]
+        l2.append(obj)
+        return l2, pv.append(obj)
+
+    @rule(target=sequences, start=sequences, end=sequences)
+    @verify_inputs_unmodified
+    def extend(self, start, end):
+        """
+        Extend a pair of sequences with another pair of sequences.
+        """
+        l, pv = start
+        l2, pv2 = end
+        # compare() has O(N**2) behavior, so don't want too-large lists:
+        assume(len(l) + len(l2) < 50)
+        l3 = l[:]
+        l3.extend(l2)
+        return l3, pv.extend(pv2)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def remove(self, former, data):
+        """
+        Remove an item from the sequences.
+        """
+        l, pv = former
+        assume(l)
+        l2 = l[:]
+        i = data.draw(st.sampled_from(range(len(l))))
+        del l2[i]
+        return l2, pv.delete(i)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def set(self, former, data):
+        """
+        Overwrite an item in the sequence.
+        """
+        l, pv = former
+        assume(l)
+        l2 = l[:]
+        i = data.draw(st.sampled_from(range(len(l))))
+        obj = RefCountTracker()
+        l2[i] = obj
+        return l2, pv.set(i, obj)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def transform_set(self, former, data):
+        """
+        Transform the sequence by setting value.
+        """
+        l, pv = former
+        assume(l)
+        l2 = l[:]
+        i = data.draw(st.sampled_from(range(len(l))))
+        obj = RefCountTracker()
+        l2[i] = obj
+        return l2, pv.transform([i], obj)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def transform_discard(self, former, data):
+        """
+        Transform the sequence by discarding a value.
+        """
+        l, pv = former
+        assume(l)
+        l2 = l[:]
+        i = data.draw(st.sampled_from(range(len(l))))
+        del l2[i]
+        return l2, pv.transform([i], discard)
+
+    @rule(target=sequences, former=sequences, data=st.data())
+    @verify_inputs_unmodified
+    def subset(self, former, data):
+        """
+        A subset of the previous sequence.
+        """
+        l, pv = former
+        assume(l)
+        i = data.draw(st.sampled_from(range(len(l))))
+        j = data.draw(st.sampled_from(range(len(l))))
+        return l[i:j], pv[i:j]
+
+    @rule(pair=sequences)
+    @verify_inputs_unmodified
+    def compare(self, pair):
+        """
+        The list and pvector must match.
+        """
+        l, pv = pair
+        # compare() has O(N**2) behavior, so don't want too-large lists:
+        assume(len(l) < 50)
+        assert_equal(l, pv)
+
+
+PVectorBuilderTests = PVectorBuilder.TestCase
+
+
+class EvolverItem(PClass):
+    original_list = field()
+    original_pvector = field()
+    current_list = field()
+    current_evolver = field()
+
+
+class PVectorEvolverBuilder(RuleBasedStateMachine):
+    """
+    Build a list and matching pvector evolver step-by-step.
+
+    In each step in the state machine we do same operation on a list and
+    on a pvector evolver, and then when we're done we compare the two.
+    """
+    sequences = Bundle("evolver_sequences")
+
+    @rule(target=sequences, start=PVectorAndLists)
+    def initial_value(self, start):
+        """
+        Some initial values generated by a hypothesis strategy.
+        """
+        l, pv = start
+        return EvolverItem(original_list=l,
+                           original_pvector=pv,
+                           current_list=l[:],
+                           current_evolver=pv.evolver())
+
+    @rule(item=sequences)
+    def append(self, item):
+        """
+        Append an item to the pair of sequences.
+        """
+        obj = RefCountTracker()
+        item.current_list.append(obj)
+        item.current_evolver.append(obj)
+
+    @rule(start=sequences, end=sequences)
+    def extend(self, start, end):
+        """
+        Extend a pair of sequences with another pair of sequences.
+        """
+        # compare() has O(N**2) behavior, so don't want too-large lists:
+        assume(len(start.current_list) + len(end.current_list) < 50)
+        start.current_evolver.extend(end.current_list)
+        start.current_list.extend(end.current_list)
+
+    @rule(item=sequences, data=st.data())
+    def delete(self, item, data):
+        """
+        Remove an item from the sequences.
+        """
+        assume(item.current_list)
+        i = data.draw(st.sampled_from(range(len(item.current_list))))
+        del item.current_list[i]
+        del item.current_evolver[i]
+
+    @rule(item=sequences, data=st.data())
+    def setitem(self, item, data):
+        """
+        Overwrite an item in the sequence using ``__setitem__``.
+        """
+        assume(item.current_list)
+        i = data.draw(st.sampled_from(range(len(item.current_list))))
+        obj = RefCountTracker()
+        item.current_list[i] = obj
+        item.current_evolver[i] = obj
+
+    @rule(item=sequences, data=st.data())
+    def set(self, item, data):
+        """
+        Overwrite an item in the sequence using ``set``.
+        """
+        assume(item.current_list)
+        i = data.draw(st.sampled_from(range(len(item.current_list))))
+        obj = RefCountTracker()
+        item.current_list[i] = obj
+        item.current_evolver.set(i, obj)
+
+    @rule(item=sequences)
+    def compare(self, item):
+        """
+        The list and pvector evolver must match.
+        """
+        item.current_evolver.is_dirty()
+        # compare() has O(N**2) behavior, so don't want too-large lists:
+        assume(len(item.current_list) < 50)
+        # original object unmodified
+        assert item.original_list == item.original_pvector
+        # evolver matches:
+        for i in range(len(item.current_evolver)):
+            assert item.current_list[i] == item.current_evolver[i]
+        # persistent version matches
+        assert_equal(item.current_list, item.current_evolver.persistent())
+        # original object still unmodified
+        assert item.original_list == item.original_pvector
+
+
+PVectorEvolverBuilderTests = PVectorEvolverBuilder.TestCase
diff --git a/contrib/python/pyrsistent/py3/tests/immutable_object_test.py b/contrib/python/pyrsistent/py3/tests/immutable_object_test.py
new file mode 100644
index 00000000000..11ff513cbcb
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/immutable_object_test.py
@@ -0,0 +1,67 @@
+import pytest
+from pyrsistent import immutable
+
+class Empty(immutable(verbose=True)):
+    pass
+
+
+class Single(immutable('x')):
+    pass
+
+
+class FrozenMember(immutable('x, y_')):
+    pass
+
+
+class DerivedWithNew(immutable(['x', 'y'])):
+    def __new__(cls, x, y):
+        return super(DerivedWithNew, cls).__new__(cls, x, y)
+
+
+def test_instantiate_object_with_no_members():
+    t = Empty()
+    t2 = t.set()
+
+    assert t is t2
+
+
+def test_assign_non_existing_attribute():
+    t = Empty()
+
+    with pytest.raises(AttributeError):
+        t.set(a=1)
+
+
+def test_basic_instantiation():
+    t = Single(17)
+
+    assert t.x == 17
+    assert str(t) == 'Single(x=17)'
+
+
+def test_cannot_modify_member():
+    t = Single(17)
+
+    with pytest.raises(AttributeError):
+        t.x = 18
+
+def test_basic_replace():
+    t = Single(17)
+    t2 = t.set(x=18)
+
+    assert t.x == 17
+    assert t2.x == 18
+
+
+def test_cannot_replace_frozen_member():
+    t = FrozenMember(17, 18)
+
+    with pytest.raises(AttributeError):
+        t.set(y_=18)
+
+
+def test_derived_class_with_new():
+    d = DerivedWithNew(1, 2)
+    d2 = d.set(x=3)
+
+    assert d2.x == 3
diff --git a/contrib/python/pyrsistent/py3/tests/list_test.py b/contrib/python/pyrsistent/py3/tests/list_test.py
new file mode 100644
index 00000000000..ccbd83ba978
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/list_test.py
@@ -0,0 +1,209 @@
+import pickle
+import pytest
+from pyrsistent import plist, l
+
+
+def test_literalish_works():
+    assert l(1, 2, 3) == plist([1, 2, 3])
+
+
+def test_first_and_rest():
+    pl = plist([1, 2])
+    assert pl.first == 1
+    assert pl.rest.first == 2
+    assert pl.rest.rest is plist()
+
+
+def test_instantiate_large_list():
+    assert plist(range(1000)).first == 0
+
+
+def test_iteration():
+    assert list(plist()) == []
+    assert list(plist([1, 2, 3])) == [1, 2, 3]
+
+
+def test_cons():
+    assert plist([1, 2, 3]).cons(0) == plist([0, 1, 2, 3])
+
+
+def test_cons_empty_list():
+    assert plist().cons(0) == plist([0])
+
+
+def test_truthiness():
+    assert plist([1])
+    assert not plist()
+
+
+def test_len():
+    assert len(plist([1, 2, 3])) == 3
+    assert len(plist()) == 0
+
+
+def test_first_illegal_on_empty_list():
+    with pytest.raises(AttributeError):
+        plist().first
+
+
+def test_rest_return_self_on_empty_list():
+    assert plist().rest is plist()
+
+
+def test_reverse():
+    assert plist([1, 2, 3]).reverse() == plist([3, 2, 1])
+    assert reversed(plist([1, 2, 3])) == plist([3, 2, 1])
+
+    assert plist().reverse() == plist()
+    assert reversed(plist()) == plist()
+
+
+def test_inequality():
+    assert plist([1, 2]) != plist([1, 3])
+    assert plist([1, 2]) != plist([1, 2, 3])
+    assert plist() != plist([1, 2, 3])
+
+
+def test_repr():
+    assert str(plist()) == "plist([])"
+    assert str(plist([1, 2, 3])) == "plist([1, 2, 3])"
+
+
+def test_indexing():
+    assert plist([1, 2, 3])[2] == 3
+    assert plist([1, 2, 3])[-1] == 3
+
+
+def test_indexing_on_empty_list():
+    with pytest.raises(IndexError):
+        plist()[0]
+
+
+def test_index_out_of_range():
+    with pytest.raises(IndexError):
+        plist([1, 2])[2]
+
+    with pytest.raises(IndexError):
+        plist([1, 2])[-3]
+
+def test_index_invalid_type():
+    with pytest.raises(TypeError) as e:
+        plist([1, 2, 3])['foo']  # type: ignore
+
+    assert 'cannot be interpreted' in str(e.value)
+
+
+def test_slicing_take():
+    assert plist([1, 2, 3])[:2] == plist([1, 2])
+
+
+def test_slicing_take_out_of_range():
+    assert plist([1, 2, 3])[:20] == plist([1, 2, 3])
+
+
+def test_slicing_drop():
+    li = plist([1, 2, 3])
+    assert li[1:] is li.rest
+
+
+def test_slicing_drop_out_of_range():
+    assert plist([1, 2, 3])[3:] is plist()
+
+
+def test_contains():
+    assert 2 in plist([1, 2, 3])
+    assert 4 not in plist([1, 2, 3])
+    assert 1 not in plist()
+
+
+def test_count():
+    assert plist([1, 2, 1]).count(1) == 2
+    assert plist().count(1) == 0
+
+
+def test_index():
+    assert plist([1, 2, 3]).index(3) == 2
+
+
+def test_index_item_not_found():
+    with pytest.raises(ValueError):
+        plist().index(3)
+
+    with pytest.raises(ValueError):
+        plist([1, 2]).index(3)
+
+
+def test_pickling_empty_list():
+    assert pickle.loads(pickle.dumps(plist(), -1)) == plist()
+
+
+def test_pickling_non_empty_list():
+    assert pickle.loads(pickle.dumps(plist([1, 2, 3]), -1)) == plist([1, 2, 3])
+
+
+def test_comparison():
+    assert plist([1, 2]) < plist([1, 2, 3])
+    assert plist([2, 1]) > plist([1, 2, 3])
+    assert plist() < plist([1])
+    assert plist([1]) > plist()
+
+
+def test_comparison_with_other_type():
+    assert plist() != []
+
+
+def test_hashing():
+    assert hash(plist([1, 2])) == hash(plist([1, 2]))
+    assert hash(plist([1, 2])) != hash(plist([2, 1]))
+
+
+def test_split():
+    left_list, right_list = plist([1, 2, 3, 4, 5]).split(3)
+    assert left_list == plist([1, 2, 3])
+    assert right_list == plist([4, 5])
+
+
+def test_split_no_split_occurred():
+    x = plist([1, 2])
+    left_list, right_list = x.split(2)
+    assert left_list is x
+    assert right_list is plist()
+
+
+def test_split_empty_list():
+    left_list, right_list = plist().split(2)
+    assert left_list == plist()
+    assert right_list == plist()
+
+
+def test_remove():
+    assert plist([1, 2, 3, 2]).remove(2) == plist([1, 3, 2])
+    assert plist([1, 2, 3]).remove(1) == plist([2, 3])
+    assert plist([1, 2, 3]).remove(3) == plist([1, 2])
+
+
+def test_remove_missing_element():
+    with pytest.raises(ValueError):
+        plist([1, 2]).remove(3)
+
+    with pytest.raises(ValueError):
+        plist().remove(2)
+
+
+def test_mcons():
+    assert plist([1, 2]).mcons([3, 4]) == plist([4, 3, 1, 2])
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(plist())
+    weakref.ref(plist([1, 2]))
+
+
+def test_iterable():
+    """
+    PLists can be created from iterables even though they can't be len()
+    hinted.
+    """
+
+    assert plist(iter("a")) == plist(iter("a"))
diff --git a/contrib/python/pyrsistent/py3/tests/map_test.py b/contrib/python/pyrsistent/py3/tests/map_test.py
new file mode 100644
index 00000000000..ae2317b2336
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/map_test.py
@@ -0,0 +1,551 @@
+from collections import namedtuple
+from collections.abc import Mapping, Hashable
+from operator import add
+import pytest
+from pyrsistent import pmap, m
+import pickle
+
+
+def test_instance_of_hashable():
+    assert isinstance(m(), Hashable)
+
+
+def test_instance_of_map():
+    assert isinstance(m(), Mapping)
+
+
+def test_literalish_works():
+    assert m() is pmap()
+    assert m(a=1, b=2) == pmap({'a': 1, 'b': 2})
+
+
+def test_empty_initialization():
+    a_map = pmap()
+    assert len(a_map) == 0
+
+
+def test_initialization_with_one_element():
+    the_map = pmap({'a': 2})
+    assert len(the_map) == 1
+    assert the_map['a'] == 2
+    assert the_map.a == 2
+    assert 'a' in the_map
+
+    assert the_map is the_map.discard('b')
+
+    empty_map = the_map.remove('a')
+    assert len(empty_map) == 0
+    assert 'a' not in empty_map
+
+
+def test_get_non_existing_raises_key_error():
+    m1 = m()
+    with pytest.raises(KeyError) as error:
+        m1['foo']
+
+    assert str(error.value) == "'foo'"
+
+
+def test_remove_non_existing_element_raises_key_error():
+    m1 = m(a=1)
+
+    with pytest.raises(KeyError) as error:
+        m1.remove('b')
+
+    assert str(error.value) == "'b'"
+
+
+def test_various_iterations():
+    assert {'a', 'b'} == set(m(a=1, b=2))
+    assert ['a', 'b'] == sorted(m(a=1, b=2).keys())
+
+    assert {1, 2} == set(m(a=1, b=2).itervalues())
+    assert [1, 2] == sorted(m(a=1, b=2).values())
+
+    assert {('a', 1), ('b', 2)} == set(m(a=1, b=2).iteritems())
+    assert {('a', 1), ('b', 2)} == set(m(a=1, b=2).items())
+
+    pm = pmap({k: k for k in range(100)})
+    assert len(pm) == len(pm.keys())
+    assert len(pm) == len(pm.values())
+    assert len(pm) == len(pm.items())
+    ks = pm.keys()
+    assert all(k in pm for k in ks)
+    assert all(k in ks for k in ks)
+    us = pm.items()
+    assert all(pm[k] == v for (k, v) in us)
+    vs = pm.values()
+    assert all(v in vs for v in vs)
+
+
+def test_initialization_with_two_elements():
+    map1 = pmap({'a': 2, 'b': 3})
+    assert len(map1) == 2
+    assert map1['a'] == 2
+    assert map1['b'] == 3
+
+    map2 = map1.remove('a')
+    assert 'a' not in map2
+    assert map2['b'] == 3
+
+
+def test_initialization_with_many_elements():
+    init_dict = dict([(str(x), x) for x in range(1700)])
+    the_map = pmap(init_dict)
+
+    assert len(the_map) == 1700
+    assert the_map['16'] == 16
+    assert the_map['1699'] == 1699
+    assert the_map.set('256', 256) is the_map
+
+    new_map = the_map.remove('1600')
+    assert len(new_map) == 1699
+    assert '1600' not in new_map
+    assert new_map['1601'] == 1601
+
+    # Some NOP properties
+    assert new_map.discard('18888') is new_map
+    assert '19999' not in new_map
+    assert new_map['1500'] == 1500
+    assert new_map.set('1500', new_map['1500']) is new_map
+
+
+def test_access_non_existing_element():
+    map1 = pmap()
+    assert len(map1) == 0
+
+    map2 = map1.set('1', 1)
+    assert '1' not in map1
+    assert map2['1'] == 1
+    assert '2' not in map2
+
+
+def test_overwrite_existing_element():
+    map1 = pmap({'a': 2})
+    map2 = map1.set('a', 3)
+
+    assert len(map2) == 1
+    assert map2['a'] == 3
+
+
+def test_hash():
+    x = m(a=1, b=2, c=3)
+    y = m(a=1, b=2, c=3)
+
+    assert hash(x) == hash(y)
+
+
+def test_same_hash_when_content_the_same_but_underlying_vector_size_differs():
+    x = pmap(dict((x, x) for x in range(1000)))
+    y = pmap({10: 10, 200: 200, 700: 700})
+
+    for z in x:
+        if z not in y:
+            x = x.remove(z)
+
+    assert x == y
+    assert hash(x) == hash(y)
+
+
+class HashabilityControlled(object):
+    hashable = True
+
+    def __hash__(self):
+        if self.hashable:
+            return 4  # Proven random
+        raise ValueError("I am not currently hashable.")
+
+
+def test_map_does_not_hash_values_on_second_hash_invocation():
+    hashable = HashabilityControlled()
+    x = pmap(dict(el=hashable))
+    hash(x)
+    hashable.hashable = False
+    hash(x)
+
+
+def test_equal():
+    x = m(a=1, b=2, c=3)
+    y = m(a=1, b=2, c=3)
+
+    assert x == y
+    assert not (x != y)
+
+    assert y == x
+    assert not (y != x)
+
+
+def test_equal_to_dict():
+    x = m(a=1, b=2, c=3)
+    y = dict(a=1, b=2, c=3)
+
+    assert x == y
+    assert not (x != y)
+
+    assert y == x
+    assert not (y != x)
+
+
+def test_equal_with_different_bucket_sizes():
+    x = pmap({'a': 1, 'b': 2}, 50)
+    y = pmap({'a': 1, 'b': 2}, 10)
+
+    assert x == y
+    assert not (x != y)
+
+    assert y == x
+    assert not (y != x)
+
+
+def test_equal_with_different_insertion_order():
+    x = pmap([(i, i) for i in range(50)], 10)
+    y = pmap([(i, i) for i in range(49, -1, -1)], 10)
+
+    assert x == y
+    assert not (x != y)
+
+    assert y == x
+    assert not (y != x)
+
+
+def test_not_equal():
+    x = m(a=1, b=2, c=3)
+    y = m(a=1, b=2)
+
+    assert x != y
+    assert not (x == y)
+
+    assert y != x
+    assert not (y == x)
+
+
+def test_not_equal_to_dict():
+    x = m(a=1, b=2, c=3)
+    y = dict(a=1, b=2, d=4)
+
+    assert x != y
+    assert not (x == y)
+
+    assert y != x
+    assert not (y == x)
+
+
+def test_update_with_multiple_arguments():
+    # If same value is present in multiple sources, the rightmost is used.
+    x = m(a=1, b=2, c=3)
+    y = x.update(m(b=4, c=5), {'c': 6})
+
+    assert y == m(a=1, b=4, c=6)
+
+
+def test_update_one_argument():
+    x = m(a=1)
+
+    assert x.update(m(b=2)) == m(a=1, b=2)
+
+
+def test_update_no_arguments():
+    x = m(a=1)
+
+    assert x.update() is x
+
+
+def test_addition():
+    assert m(x=1, y=2) + m(y=3, z=4) == m(x=1, y=3, z=4)
+
+
+def test_union_operator():
+    assert m(x=1, y=2) | m(y=3, z=4) == m(x=1, y=3, z=4)
+
+
+def test_transform_base_case():
+    # Works as set when called with only one key
+    x = m(a=1, b=2)
+
+    assert x.transform(['a'], 3) == m(a=3, b=2)
+
+
+def test_transform_nested_maps():
+    x = m(a=1, b=m(c=3, d=m(e=6, f=7)))
+
+    assert x.transform(['b', 'd', 'e'], 999) == m(a=1, b=m(c=3, d=m(e=999, f=7)))
+
+
+def test_transform_levels_missing():
+    x = m(a=1, b=m(c=3))
+
+    assert x.transform(['b', 'd', 'e'], 999) == m(a=1, b=m(c=3, d=m(e=999)))
+
+
+class HashDummy(object):
+    def __hash__(self):
+        return 6528039219058920  # Hash of '33'
+
+    def __eq__(self, other):
+        return self is other
+
+
+def test_hash_collision_is_correctly_resolved():
+    dummy1 = HashDummy()
+    dummy2 = HashDummy()
+    dummy3 = HashDummy()
+    dummy4 = HashDummy()
+
+    map1 = pmap({dummy1: 1, dummy2: 2, dummy3: 3})
+    assert map1[dummy1] == 1
+    assert map1[dummy2] == 2
+    assert map1[dummy3] == 3
+    assert dummy4 not in map1
+
+    keys = set()
+    values = set()
+    for k, v in map1.iteritems():
+        keys.add(k)
+        values.add(v)
+
+    assert keys == {dummy1, dummy2, dummy3}
+    assert values == {1, 2, 3}
+
+    map2 = map1.set(dummy1, 11)
+    assert map2[dummy1] == 11
+
+    # Re-use existing structure when inserted element is the same
+    assert map2.set(dummy1, 11) is map2
+
+    map3 = map1.set('a', 22)
+    assert map3['a'] == 22
+    assert map3[dummy3] == 3
+
+    # Remove elements
+    map4 = map1.discard(dummy2)
+    assert len(map4) == 2
+    assert map4[dummy1] == 1
+    assert dummy2 not in map4
+    assert map4[dummy3] == 3
+
+    assert map1.discard(dummy4) is map1
+
+    # Empty map handling
+    empty_map = map4.remove(dummy1).remove(dummy3)
+    assert len(empty_map) == 0
+    assert empty_map.discard(dummy1) is empty_map
+
+
+def test_bitmap_indexed_iteration():
+    a_map = pmap({'a': 2, 'b': 1})
+    keys = set()
+    values = set()
+
+    count = 0
+    for k, v in a_map.iteritems():
+        count += 1
+        keys.add(k)
+        values.add(v)
+
+    assert count == 2
+    assert keys == {'a', 'b'}
+    assert values == {2, 1}
+
+
+def test_iteration_with_many_elements():
+    values = list(range(0, 2000))
+    keys = [str(x) for x in values]
+    init_dict = dict(zip(keys, values))
+
+    hash_dummy1 = HashDummy()
+    hash_dummy2 = HashDummy()
+
+    # Throw in a couple of hash collision nodes to tests
+    # those properly as well
+    init_dict[hash_dummy1] = 12345
+    init_dict[hash_dummy2] = 54321
+    a_map = pmap(init_dict)
+
+    actual_values = set()
+    actual_keys = set()
+
+    for k, v in a_map.iteritems():
+        actual_values.add(v)
+        actual_keys.add(k)
+
+    assert actual_keys == set(keys + [hash_dummy1, hash_dummy2])
+    assert actual_values == set(values + [12345, 54321])
+
+
+def test_str():
+    assert str(pmap({1: 2, 3: 4})) == "pmap({1: 2, 3: 4})"
+
+
+def test_empty_truthiness():
+    assert m(a=1)
+    assert not m()
+
+
+def test_update_with():
+    assert m(a=1).update_with(add, m(a=2, b=4)) == m(a=3, b=4)
+    assert m(a=1).update_with(lambda l, r: l, m(a=2, b=4)) == m(a=1, b=4)
+
+    def map_add(l, r):
+        return dict(list(l.items()) + list(r.items()))
+
+    assert m(a={'c': 3}).update_with(map_add, m(a={'d': 4})) == m(a={'c': 3, 'd': 4})
+
+
+def test_pickling_empty_map():
+    assert pickle.loads(pickle.dumps(m(), -1)) == m()
+
+
+def test_pickling_non_empty_map():
+    assert pickle.loads(pickle.dumps(m(a=1, b=2), -1)) == m(a=1, b=2)
+
+
+def test_set_with_relocation():
+    x = pmap({'a': 1000}, pre_size=1)
+    x = x.set('b', 3000)
+    x = x.set('c', 4000)
+    x = x.set('d', 5000)
+    x = x.set('d', 6000)
+
+    assert len(x) == 4
+    assert x == pmap({'a': 1000, 'b': 3000, 'c': 4000, 'd': 6000})
+
+
+def test_evolver_simple_update():
+    x = m(a=1000, b=2000)
+    e = x.evolver()
+    e['b'] = 3000
+
+    assert e['b'] == 3000
+    assert e.persistent()['b'] == 3000
+    assert x['b'] == 2000
+
+
+def test_evolver_update_with_relocation():
+    x = pmap({'a': 1000}, pre_size=1)
+    e = x.evolver()
+    e['b'] = 3000
+    e['c'] = 4000
+    e['d'] = 5000
+    e['d'] = 6000
+
+    assert len(e) == 4
+    assert e.persistent() == pmap({'a': 1000, 'b': 3000, 'c': 4000, 'd': 6000})
+
+
+def test_evolver_set_with_reallocation_edge_case():
+    # Demonstrates a bug in evolver that also affects updates. Under certain
+    # circumstances, the result of `x.update(y)` will **not** have all the
+    # keys from `y`.
+    foo = object()
+    x = pmap({'a': foo}, pre_size=1)
+    e = x.evolver()
+    e['b'] = 3000
+    # Bug is triggered when we do a reallocation and the new value is
+    # identical to the old one.
+    e['a'] = foo
+
+    y = e.persistent()
+    assert 'b' in y
+    assert y is e.persistent()
+
+
+def test_evolver_remove_element():
+    e = m(a=1000, b=2000).evolver()
+    assert 'a' in e
+
+    del e['a']
+    assert 'a' not in e
+
+
+def test_evolver_remove_element_not_present():
+    e = m(a=1000, b=2000).evolver()
+
+    with pytest.raises(KeyError) as error:
+        del e['c']
+
+    assert str(error.value) == "'c'"
+
+
+def test_copy_returns_reference_to_self():
+    m1 = m(a=10)
+    assert m1.copy() is m1
+
+
+def test_dot_access_of_non_existing_element_raises_attribute_error():
+    m1 = m(a=10)
+
+    with pytest.raises(AttributeError) as error:
+        m1.b
+
+    error_message = str(error.value)
+
+    assert "'b'" in error_message
+    assert type(m1).__name__ in error_message
+
+
+def test_pmap_unorderable():
+    with pytest.raises(TypeError):
+        _ = m(a=1) < m(b=2)
+
+    with pytest.raises(TypeError):
+        _ = m(a=1) <= m(b=2)
+
+    with pytest.raises(TypeError):
+        _ = m(a=1) > m(b=2)
+
+    with pytest.raises(TypeError):
+        _ = m(a=1) >= m(b=2)
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(m(a=1))
+
+
+def test_insert_and_get_many_elements():
+    # This test case triggers reallocation of the underlying bucket structure.
+    a_map = m()
+    for x in range(1000):
+        a_map = a_map.set(str(x), x)
+
+    assert len(a_map) == 1000
+    for x in range(1000):
+        assert a_map[str(x)] == x, x
+
+
+def test_iterable():
+    """
+    PMaps can be created from iterables even though they can't be len() hinted.
+    """
+
+    assert pmap(iter([("a", "b")])) == pmap([("a", "b")])
+
+
+class BrokenPerson(namedtuple('Person', 'name')):
+    def __eq__(self, other):
+        return self.__class__ == other.__class__ and self.name == other.name
+
+    def __hash__(self):
+        return hash(self.name)
+
+
+class BrokenItem(namedtuple('Item', 'name')):
+    def __eq__(self, other):
+        return self.__class__ == other.__class__ and self.name == other.name
+
+    def __hash__(self):
+        return hash(self.name)
+
+
+def test_pmap_removal_with_broken_classes_deriving_from_namedtuple():
+    """
+    The two classes above implement __eq__ but also would need to implement __ne__ to compare
+    consistently. See issue https://github.com/tobgu/pyrsistent/issues/268 for details.
+    """
+    s = pmap({BrokenPerson('X'): 2, BrokenItem('X'): 3})
+    s = s.remove(BrokenPerson('X'))
+
+    # Both items are removed due to how they are compared for inequality
+    assert BrokenPerson('X') not in s
+    assert BrokenItem('X') in s
+    assert len(s) == 1
diff --git a/contrib/python/pyrsistent/py3/tests/memory_profiling.py b/contrib/python/pyrsistent/py3/tests/memory_profiling.py
new file mode 100644
index 00000000000..69036520cdc
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/memory_profiling.py
@@ -0,0 +1,48 @@
+"""
+Script to try do detect any memory leaks that may be lurking in the C implementation of the PVector.
+"""
+import inspect
+import sys
+import time
+import memory_profiler
+import vector_test
+from pyrsistent import pvector
+
+try:
+    import pvectorc
+except ImportError:
+    print("No C implementation of PVector available, terminating")
+    sys.exit()
+
+
+PROFILING_DURATION = 2.0
+
+
+def run_function(fn):
+    stop = time.time() + PROFILING_DURATION
+    while time.time() < stop:
+        fn(pvector)
+
+
+def detect_memory_leak(samples):
+    # Do not allow a memory usage difference larger than 5% between the beginning and the end.
+    # Skip the first samples to get rid of the build up period and the last sample since it seems
+    # a little less precise
+    return abs(1 - (sum(samples[5:8]) / sum(samples[-4:-1]))) > 0.05
+
+
+def profile_tests():
+    test_functions = [fn for fn in inspect.getmembers(vector_test, inspect.isfunction)
+                      if fn[0].startswith('test_')]
+
+    for name, fn in test_functions:
+        # There are a couple of tests that are not run for the C implementation, skip those
+        fn_args = inspect.getfullargspec(fn)[0]
+        if 'pvector' in fn_args:
+            print('Executing %s' % name)
+            result = memory_profiler.memory_usage((run_function, (fn,), {}), interval=.1)
+            assert not detect_memory_leak(result), (name, result)
+
+
+if __name__ == "__main__":
+    profile_tests()
\ No newline at end of file
diff --git a/contrib/python/pyrsistent/py3/tests/record_test.py b/contrib/python/pyrsistent/py3/tests/record_test.py
new file mode 100644
index 00000000000..95fc55b8f16
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/record_test.py
@@ -0,0 +1,878 @@
+import pickle
+import datetime
+import pytest
+import uuid
+from pyrsistent import (
+    PRecord, field, InvariantException, ny, pset, PSet, CheckedPVector,
+    PTypeError, pset_field, pvector_field, pmap_field, pmap, PMap,
+    pvector, PVector, v, m)
+
+
+class ARecord(PRecord):
+    x = field(type=(int, float))
+    y = field()
+
+
+class Hierarchy(PRecord):
+    point1 = field(ARecord)
+    point2 = field(ARecord)
+    points = pvector_field(ARecord)
+
+
+class RecordContainingContainers(PRecord):
+    map = pmap_field(str, str)
+    vec = pvector_field(str)
+    set = pset_field(str)
+
+
+class UniqueThing(PRecord):
+    id = field(type=uuid.UUID, factory=uuid.UUID)
+
+
+class Something(object):
+    pass
+
+class Another(object):
+    pass
+
+def test_create_ignore_extra_true():
+    h = Hierarchy.create(
+        {'point1': {'x': 1, 'y': 'foo', 'extra_field_0': 'extra_data_0'},
+         'point2': {'x': 1, 'y': 'foo', 'extra_field_1': 'extra_data_1'},
+         'extra_field_2': 'extra_data_2',
+         }, ignore_extra=True
+    )
+    assert h
+
+
+def test_create_ignore_extra_true_sequence_hierarchy():
+    h = Hierarchy.create(
+        {'point1': {'x': 1, 'y': 'foo', 'extra_field_0': 'extra_data_0'},
+         'point2': {'x': 1, 'y': 'foo', 'extra_field_1': 'extra_data_1'},
+         'points': [{'x': 1, 'y': 'foo', 'extra_field_2': 'extra_data_2'},
+                    {'x': 1, 'y': 'foo', 'extra_field_3': 'extra_data_3'}],
+         'extra_field____': 'extra_data_2',
+         }, ignore_extra=True
+    )
+    assert h
+
+
+def test_ignore_extra_for_pvector_field():
+    class HierarchyA(PRecord):
+        points = pvector_field(ARecord, optional=False)
+
+    class HierarchyB(PRecord):
+        points = pvector_field(ARecord, optional=True)
+
+    point_object = {'x': 1, 'y': 'foo', 'extra_field': 69}
+
+    h = HierarchyA.create({'points': [point_object]}, ignore_extra=True)
+    assert h
+    h = HierarchyB.create({'points': [point_object]}, ignore_extra=True)
+    assert h
+
+
+def test_create():
+    r = ARecord(x=1, y='foo')
+    assert r.x == 1
+    assert r.y == 'foo'
+    assert isinstance(r, ARecord)
+
+
+def test_create_ignore_extra():
+    r = ARecord.create({'x': 1, 'y': 'foo', 'z': None}, ignore_extra=True)
+    assert r.x == 1
+    assert r.y == 'foo'
+    assert isinstance(r, ARecord)
+
+
+def test_create_ignore_extra_false():
+    with pytest.raises(AttributeError):
+        _ = ARecord.create({'x': 1, 'y': 'foo', 'z': None})
+
+
+def test_correct_assignment():
+    r = ARecord(x=1, y='foo')
+    r2 = r.set('x', 2.0)
+    r3 = r2.set('y', 'bar')
+
+    assert r2 == {'x': 2.0, 'y': 'foo'}
+    assert r3 == {'x': 2.0, 'y': 'bar'}
+    assert isinstance(r3, ARecord)
+
+
+def test_direct_assignment_not_possible():
+    with pytest.raises(AttributeError):
+        ARecord().x = 1
+
+
+def test_cannot_assign_undeclared_fields():
+    with pytest.raises(AttributeError):
+        ARecord().set('z', 5)
+
+
+def test_cannot_assign_wrong_type_to_fields():
+    try:
+        ARecord().set('x', 'foo')
+        assert False
+    except PTypeError as e:
+        assert e.source_class == ARecord
+        assert e.field == 'x'
+        assert e.expected_types == set([int, float])
+        assert e.actual_type is type('foo')
+
+
+def test_cannot_construct_with_undeclared_fields():
+    with pytest.raises(AttributeError):
+        ARecord(z=5)
+
+
+def test_cannot_construct_with_fields_of_wrong_type():
+    with pytest.raises(TypeError):
+        ARecord(x='foo')
+
+
+def test_support_record_inheritance():
+    class BRecord(ARecord):
+        z = field()
+
+    r = BRecord(x=1, y='foo', z='bar')
+
+    assert isinstance(r, BRecord)
+    assert isinstance(r, ARecord)
+    assert r == {'x': 1, 'y': 'foo', 'z': 'bar'}
+
+
+def test_single_type_spec():
+    class A(PRecord):
+        x = field(type=int)
+
+    r = A(x=1)
+    assert r.x == 1
+
+    with pytest.raises(TypeError):
+        r.set('x', 'foo')
+
+
+def test_remove():
+    r = ARecord(x=1, y='foo')
+    r2 = r.remove('y')
+
+    assert isinstance(r2, ARecord)
+    assert r2 == {'x': 1}
+
+
+def test_remove_non_existing_member():
+    r = ARecord(x=1, y='foo')
+
+    with pytest.raises(KeyError):
+        r.remove('z')
+
+
+def test_field_invariant_must_hold():
+    class BRecord(PRecord):
+        x = field(invariant=lambda x: (x > 1, 'x too small'))
+        y = field(mandatory=True)
+
+    try:
+        BRecord(x=1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('x too small',)
+        assert e.missing_fields == ('BRecord.y',)
+
+
+def test_global_invariant_must_hold():
+    class BRecord(PRecord):
+        __invariant__ = lambda r: (r.x <= r.y, 'y smaller than x')
+        x = field()
+        y = field()
+
+    BRecord(x=1, y=2)
+
+    try:
+        BRecord(x=2, y=1)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('y smaller than x',)
+        assert e.missing_fields == ()
+
+
+def test_set_multiple_fields():
+    a = ARecord(x=1, y='foo')
+    b = a.set(x=2, y='bar')
+
+    assert b == {'x': 2, 'y': 'bar'}
+
+
+def test_initial_value():
+    class BRecord(PRecord):
+        x = field(initial=1)
+        y = field(initial=2)
+
+    a = BRecord()
+    assert a.x == 1
+    assert a.y == 2
+
+
+def test_enum_field():
+    try:
+        from enum import Enum
+    except ImportError:
+        return  # Enum not supported in this environment
+
+    class ExampleEnum(Enum):
+        x = 1
+        y = 2
+
+    class RecordContainingEnum(PRecord):
+        enum_field = field(type=ExampleEnum)
+
+    r = RecordContainingEnum(enum_field=ExampleEnum.x)
+    assert r.enum_field == ExampleEnum.x
+
+def test_type_specification_must_be_a_type():
+    with pytest.raises(TypeError):
+        class BRecord(PRecord):
+            x = field(type=1)
+
+
+def test_initial_must_be_of_correct_type():
+    with pytest.raises(TypeError):
+        class BRecord(PRecord):
+            x = field(type=int, initial='foo')
+
+
+def test_invariant_must_be_callable():
+    with pytest.raises(TypeError):
+        class BRecord(PRecord):
+            x = field(invariant='foo')  # type: ignore
+
+
+def test_global_invariants_are_inherited():
+    class BRecord(PRecord):
+        __invariant__ = lambda r: (r.x % r.y == 0, 'modulo')
+        x = field()
+        y = field()
+
+    class CRecord(BRecord):
+        __invariant__ = lambda r: (r.x > r.y, 'size')
+
+    try:
+        CRecord(x=5, y=3)
+        assert False
+    except InvariantException as e:
+        assert e.invariant_errors == ('modulo',)
+
+
+def test_global_invariants_must_be_callable():
+    with pytest.raises(TypeError):
+        class CRecord(PRecord):
+            __invariant__ = 1
+
+
+def test_repr():
+    r = ARecord(x=1, y=2)
+    assert repr(r) == 'ARecord(x=1, y=2)' or repr(r) == 'ARecord(y=2, x=1)'
+
+
+def test_factory():
+    class BRecord(PRecord):
+        x = field(type=int, factory=int)
+
+    assert BRecord(x=2.5) == {'x': 2}
+
+
+def test_factory_must_be_callable():
+    with pytest.raises(TypeError):
+        class BRecord(PRecord):
+            x = field(type=int, factory=1)  # type: ignore
+
+
+def test_nested_record_construction():
+    class BRecord(PRecord):
+        x = field(int, factory=int)
+
+    class CRecord(PRecord):
+        a = field()
+        b = field(type=BRecord)
+
+    r = CRecord.create({'a': 'foo', 'b': {'x': '5'}})
+    assert isinstance(r, CRecord)
+    assert isinstance(r.b, BRecord)
+    assert r == {'a': 'foo', 'b': {'x': 5}}
+
+
+def test_pickling():
+    x = ARecord(x=2.0, y='bar')
+    y = pickle.loads(pickle.dumps(x, -1))
+
+    assert x == y
+    assert isinstance(y, ARecord)
+
+def test_supports_pickling_with_typed_container_fields():
+    obj = RecordContainingContainers(
+        map={'foo': 'bar'}, set=['hello', 'there'], vec=['a', 'b'])
+    obj2 = pickle.loads(pickle.dumps(obj))
+    assert obj == obj2
+
+def test_all_invariant_errors_reported():
+    class BRecord(PRecord):
+        x = field(factory=int, invariant=lambda x: (x >= 0, 'x negative'))
+        y = field(mandatory=True)
+
+    class CRecord(PRecord):
+        a = field(invariant=lambda x: (x != 0, 'a zero'))
+        b = field(type=BRecord)
+
+    try:
+        CRecord.create({'a': 0, 'b': {'x': -5}})
+        assert False
+    except InvariantException as e:
+        assert set(e.invariant_errors) == set(['x negative', 'a zero'])
+        assert e.missing_fields == ('BRecord.y',)
+
+
+def test_precord_factory_method_is_idempotent():
+    class BRecord(PRecord):
+        x = field()
+        y = field()
+
+    r = BRecord(x=1, y=2)
+    assert BRecord.create(r) is r
+
+
+def test_serialize():
+    class BRecord(PRecord):
+        d = field(type=datetime.date,
+                  factory=lambda d: datetime.datetime.strptime(d, "%d%m%Y").date(),
+                  serializer=lambda format, d: d.strftime('%Y-%m-%d') if format == 'ISO' else d.strftime('%d%m%Y'))
+
+    assert BRecord(d='14012015').serialize('ISO') == {'d': '2015-01-14'}
+    assert BRecord(d='14012015').serialize('other') == {'d': '14012015'}
+
+
+def test_nested_serialize():
+    class BRecord(PRecord):
+        d = field(serializer=lambda format, d: format)
+
+    class CRecord(PRecord):
+        b = field()
+
+    serialized = CRecord(b=BRecord(d='foo')).serialize('bar')
+
+    assert serialized == {'b': {'d': 'bar'}}
+    assert isinstance(serialized, dict)
+
+
+def test_serializer_must_be_callable():
+    with pytest.raises(TypeError):
+        class CRecord(PRecord):
+            x = field(serializer=1)  # type: ignore
+
+
+def test_transform_without_update_returns_same_precord():
+    r = ARecord(x=2.0, y='bar')
+    assert r.transform([ny], lambda x: x) is r
+
+
+class Application(PRecord):
+    name = field(type=str)
+    image = field(type=str)
+
+
+class ApplicationVector(CheckedPVector):
+    __type__ = Application
+
+
+class Node(PRecord):
+    applications = field(type=ApplicationVector)
+
+
+def test_nested_create_serialize():
+    node = Node(applications=[Application(name='myapp', image='myimage'),
+                              Application(name='b', image='c')])
+
+    node2 = Node.create({'applications': [{'name': 'myapp', 'image': 'myimage'},
+                                          {'name': 'b', 'image': 'c'}]})
+
+    assert node == node2
+
+    serialized = node.serialize()
+    restored = Node.create(serialized)
+
+    assert restored == node
+
+
+def test_pset_field_initial_value():
+    """
+    ``pset_field`` results in initial value that is empty.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    assert Record() == Record(value=[])
+
+def test_pset_field_custom_initial():
+    """
+    A custom initial value can be passed in.
+    """
+    class Record(PRecord):
+        value = pset_field(int, initial=(1, 2))
+    assert Record() == Record(value=[1, 2])
+
+def test_pset_field_factory():
+    """
+    ``pset_field`` has a factory that creates a ``PSet``.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    record = Record(value=[1, 2])
+    assert isinstance(record.value, PSet)
+
+def test_pset_field_checked_set():
+    """
+    ``pset_field`` results in a set that enforces its type.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    record = Record(value=[1, 2])
+    with pytest.raises(TypeError):
+        record.value.add("hello")  # type: ignore
+
+def test_pset_field_checked_vector_multiple_types():
+    """
+    ``pset_field`` results in a vector that enforces its types.
+    """
+    class Record(PRecord):
+        value = pset_field((int, str))
+    record = Record(value=[1, 2, "hello"])
+    with pytest.raises(TypeError):
+        record.value.add(object())
+
+def test_pset_field_type():
+    """
+    ``pset_field`` enforces its type.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    record = Record()
+    with pytest.raises(TypeError):
+        record.set("value", None)
+
+def test_pset_field_mandatory():
+    """
+    ``pset_field`` is a mandatory field.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    record = Record(value=[1])
+    with pytest.raises(InvariantException):
+        record.remove("value")
+
+def test_pset_field_default_non_optional():
+    """
+    By default ``pset_field`` is non-optional, i.e. does not allow
+    ``None``.
+    """
+    class Record(PRecord):
+        value = pset_field(int)
+    with pytest.raises(TypeError):
+        Record(value=None)
+
+def test_pset_field_explicit_non_optional():
+    """
+    If ``optional`` argument is ``False`` then ``pset_field`` is
+    non-optional, i.e. does not allow ``None``.
+    """
+    class Record(PRecord):
+        value = pset_field(int, optional=False)
+    with pytest.raises(TypeError):
+        Record(value=None)
+
+def test_pset_field_optional():
+    """
+    If ``optional`` argument is true, ``None`` is acceptable alternative
+    to a set.
+    """
+    class Record(PRecord):
+        value = pset_field(int, optional=True)
+    assert ((Record(value=[1, 2]).value, Record(value=None).value) ==
+            (pset([1, 2]), None))
+
+def test_pset_field_name():
+    """
+    The created set class name is based on the type of items in the set.
+    """
+    class Record(PRecord):
+        value = pset_field(Something)
+        value2 = pset_field(int)
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingPSet", "IntPSet"))
+
+def test_pset_multiple_types_field_name():
+    """
+    The created set class name is based on the multiple given types of
+    items in the set.
+    """
+    class Record(PRecord):
+        value = pset_field((Something, int))
+
+    assert (Record().value.__class__.__name__ ==
+            "SomethingIntPSet")
+
+def test_pset_field_name_string_type():
+    """
+    The created set class name is based on the type of items specified by name
+    """
+    class Record(PRecord):
+        value = pset_field("record_test.Something")
+    assert Record().value.__class__.__name__ == "SomethingPSet"
+
+
+def test_pset_multiple_string_types_field_name():
+    """
+    The created set class name is based on the multiple given types of
+    items in the set specified by name
+    """
+    class Record(PRecord):
+        value = pset_field(("record_test.Something", "record_test.Another"))
+
+    assert Record().value.__class__.__name__ == "SomethingAnotherPSet"
+
+def test_pvector_field_initial_value():
+    """
+    ``pvector_field`` results in initial value that is empty.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    assert Record() == Record(value=[])
+
+def test_pvector_field_custom_initial():
+    """
+    A custom initial value can be passed in.
+    """
+    class Record(PRecord):
+        value = pvector_field(int, initial=(1, 2))
+    assert Record() == Record(value=[1, 2])
+
+def test_pvector_field_factory():
+    """
+    ``pvector_field`` has a factory that creates a ``PVector``.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    record = Record(value=[1, 2])
+    assert isinstance(record.value, PVector)
+
+def test_pvector_field_checked_vector():
+    """
+    ``pvector_field`` results in a vector that enforces its type.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    record = Record(value=[1, 2])
+    with pytest.raises(TypeError):
+        record.value.append("hello")  # type: ignore
+
+def test_pvector_field_checked_vector_multiple_types():
+    """
+    ``pvector_field`` results in a vector that enforces its types.
+    """
+    class Record(PRecord):
+        value = pvector_field((int, str))
+    record = Record(value=[1, 2, "hello"])
+    with pytest.raises(TypeError):
+        record.value.append(object())
+
+def test_pvector_field_type():
+    """
+    ``pvector_field`` enforces its type.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    record = Record()
+    with pytest.raises(TypeError):
+        record.set("value", None)
+
+def test_pvector_field_mandatory():
+    """
+    ``pvector_field`` is a mandatory field.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    record = Record(value=[1])
+    with pytest.raises(InvariantException):
+        record.remove("value")
+
+def test_pvector_field_default_non_optional():
+    """
+    By default ``pvector_field`` is non-optional, i.e. does not allow
+    ``None``.
+    """
+    class Record(PRecord):
+        value = pvector_field(int)
+    with pytest.raises(TypeError):
+        Record(value=None)
+
+def test_pvector_field_explicit_non_optional():
+    """
+    If ``optional`` argument is ``False`` then ``pvector_field`` is
+    non-optional, i.e. does not allow ``None``.
+    """
+    class Record(PRecord):
+        value = pvector_field(int, optional=False)
+    with pytest.raises(TypeError):
+        Record(value=None)
+
+def test_pvector_field_optional():
+    """
+    If ``optional`` argument is true, ``None`` is acceptable alternative
+    to a sequence.
+    """
+    class Record(PRecord):
+        value = pvector_field(int, optional=True)
+    assert ((Record(value=[1, 2]).value, Record(value=None).value) ==
+            (pvector([1, 2]), None))
+
+def test_pvector_field_name():
+    """
+    The created set class name is based on the type of items in the set.
+    """
+    class Record(PRecord):
+        value = pvector_field(Something)
+        value2 = pvector_field(int)
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingPVector", "IntPVector"))
+
+def test_pvector_multiple_types_field_name():
+    """
+    The created vector class name is based on the multiple given types of
+    items in the vector.
+    """
+    class Record(PRecord):
+        value = pvector_field((Something, int))
+
+    assert (Record().value.__class__.__name__ ==
+            "SomethingIntPVector")
+
+def test_pvector_field_name_string_type():
+    """
+    The created set class name is based on the type of items in the set
+    specified by name.
+    """
+    class Record(PRecord):
+        value = pvector_field("record_test.Something")
+    assert Record().value.__class__.__name__ == "SomethingPVector"
+
+def test_pvector_multiple_string_types_field_name():
+    """
+    The created vector class name is based on the multiple given types of
+    items in the vector.
+    """
+    class Record(PRecord):
+        value = pvector_field(("record_test.Something", "record_test.Another"))
+
+    assert Record().value.__class__.__name__ == "SomethingAnotherPVector"
+
+def test_pvector_field_create_from_nested_serialized_data():
+    class Foo(PRecord):
+        foo = field(type=str)
+
+    class Bar(PRecord):
+        bar = pvector_field(Foo)
+
+    data = Bar(bar=v(Foo(foo="foo")))
+    Bar.create(data.serialize()) == data
+
+def test_pmap_field_initial_value():
+    """
+    ``pmap_field`` results in initial value that is empty.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int)
+    assert Record() == Record(value={})
+
+def test_pmap_field_factory():
+    """
+    ``pmap_field`` has a factory that creates a ``PMap``.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int)
+    record = Record(value={1:  1234})
+    assert isinstance(record.value, PMap)
+
+def test_pmap_field_checked_map_key():
+    """
+    ``pmap_field`` results in a map that enforces its key type.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, type(None))
+    record = Record(value={1: None})
+    with pytest.raises(TypeError):
+        record.value.set("hello", None)  # type: ignore
+
+def test_pmap_field_checked_map_value():
+    """
+    ``pmap_field`` results in a map that enforces its value type.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, type(None))
+    record = Record(value={1: None})
+    with pytest.raises(TypeError):
+        record.value.set(2, 4)  # type: ignore
+
+def test_pmap_field_checked_map_key_multiple_types():
+    """
+    ``pmap_field`` results in a map that enforces its key types.
+    """
+    class Record(PRecord):
+        value = pmap_field((int, str), type(None))
+    record = Record(value={1: None, "hello": None})
+    with pytest.raises(TypeError):
+        record.value.set(object(), None)
+
+def test_pmap_field_checked_map_value_multiple_types():
+    """
+    ``pmap_field`` results in a map that enforces its value types.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, (str, type(None)))
+    record = Record(value={1: None, 3: "hello"})
+    with pytest.raises(TypeError):
+        record.value.set(2, 4)
+
+def test_pmap_field_mandatory():
+    """
+    ``pmap_field`` is a mandatory field.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int)
+    record = Record()
+    with pytest.raises(InvariantException):
+        record.remove("value")
+
+def test_pmap_field_default_non_optional():
+    """
+    By default ``pmap_field`` is non-optional, i.e. does not allow
+    ``None``.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int)
+    # Ought to be TypeError, but pyrsistent doesn't quite allow that:
+    with pytest.raises(AttributeError):
+        Record(value=None)
+
+def test_pmap_field_explicit_non_optional():
+    """
+    If ``optional`` argument is ``False`` then ``pmap_field`` is
+    non-optional, i.e. does not allow ``None``.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int, optional=False)
+    # Ought to be TypeError, but pyrsistent doesn't quite allow that:
+    with pytest.raises(AttributeError):
+        Record(value=None)
+
+def test_pmap_field_optional():
+    """
+    If ``optional`` argument is true, ``None`` is acceptable alternative
+    to a set.
+    """
+    class Record(PRecord):
+        value = pmap_field(int, int, optional=True)
+    assert (Record(value={1: 2}).value, Record(value=None).value) == \
+           (pmap({1: 2}), None)
+
+def test_pmap_field_name():
+    """
+    The created map class name is based on the types of items in the map.
+    """
+    class Record(PRecord):
+        value = pmap_field(Something, Another)
+        value2 = pmap_field(int, float)
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingToAnotherPMap", "IntToFloatPMap"))
+
+def test_pmap_field_name_multiple_types():
+    """
+    The created map class name is based on the types of items in the map,
+    including when there are multiple supported types.
+    """
+    class Record(PRecord):
+        value = pmap_field((Something, Another), int)
+        value2 = pmap_field(str, (int, float))
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingAnotherToIntPMap", "StrToIntFloatPMap"))
+
+def test_pmap_field_name_string_type():
+    """
+    The created map class name is based on the types of items in the map
+    specified by name.
+    """
+    class Record(PRecord):
+        value = pmap_field("record_test.Something", "record_test.Another")
+    assert Record().value.__class__.__name__ == "SomethingToAnotherPMap"
+
+def test_pmap_field_name_multiple_string_types():
+    """
+    The created map class name is based on the types of items in the map,
+    including when there are multiple supported types.
+    """
+    class Record(PRecord):
+        value = pmap_field(("record_test.Something", "record_test.Another"), int)
+        value2 = pmap_field(str, ("record_test.Something", "record_test.Another"))
+    assert ((Record().value.__class__.__name__,
+             Record().value2.__class__.__name__) ==
+            ("SomethingAnotherToIntPMap", "StrToSomethingAnotherPMap"))
+
+def test_pmap_field_invariant():
+    """
+    The ``invariant`` parameter is passed through to ``field``.
+    """
+    class Record(PRecord):
+        value = pmap_field(
+            int, int,
+            invariant=(
+                lambda pmap: (len(pmap) == 1, "Exactly one item required.")
+            )
+        )
+    with pytest.raises(InvariantException):
+        Record(value={})
+    with pytest.raises(InvariantException):
+        Record(value={1: 2, 3: 4})
+    assert Record(value={1: 2}).value == {1: 2}
+
+
+def test_pmap_field_create_from_nested_serialized_data():
+    class Foo(PRecord):
+        foo = field(type=str)
+
+    class Bar(PRecord):
+        bar = pmap_field(str, Foo)
+
+    data = Bar(bar=m(foo_key=Foo(foo="foo")))
+    Bar.create(data.serialize()) == data
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(ARecord(x=1, y=2))
+
+
+def test_supports_lazy_initial_value_for_field():
+    class MyRecord(PRecord):
+        a = field(int, initial=lambda: 2)
+
+    assert MyRecord() == MyRecord(a=2)
+
+
+def test_pickle_with_one_way_factory():
+    """
+    A field factory isn't called when restoring from pickle.
+    """
+    thing = UniqueThing(id='25544626-86da-4bce-b6b6-9186c0804d64')
+    assert thing == pickle.loads(pickle.dumps(thing))
diff --git a/contrib/python/pyrsistent/py3/tests/regression_test.py b/contrib/python/pyrsistent/py3/tests/regression_test.py
new file mode 100644
index 00000000000..f8c11338348
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/regression_test.py
@@ -0,0 +1,30 @@
+from pyrsistent import pmap
+import random
+
+import gc
+
+
+def test_segfault_issue_52():
+    threshold = None
+    if hasattr(gc, 'get_threshold'):
+        # PyPy is lacking these functions
+        threshold = gc.get_threshold()
+        gc.set_threshold(1, 1, 1)  # fail fast
+
+    v = [pmap()]
+
+    def step():
+        depth = random.randint(1, 10)
+        path = random.sample(range(100000), depth)
+        v[0] = v[0].transform(path, "foo")
+
+    for i in range(1000):  # usually crashes after 10-20 steps
+        while True:
+            try:
+                step()
+                break
+            except AttributeError:  # evolver on string
+                continue
+
+    if threshold:
+        gc.set_threshold(*threshold)
diff --git a/contrib/python/pyrsistent/py3/tests/set_test.py b/contrib/python/pyrsistent/py3/tests/set_test.py
new file mode 100644
index 00000000000..f605ee0d5ec
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/set_test.py
@@ -0,0 +1,181 @@
+from pyrsistent import pset, s
+import pytest
+import pickle
+
+def test_key_is_tuple():
+    with pytest.raises(KeyError):
+        pset().remove((1,1))
+
+def test_literalish_works():
+    assert s() is pset()
+    assert s(1, 2) == pset([1, 2])
+
+
+def test_supports_hash():
+    assert hash(s(1, 2)) == hash(s(1, 2))
+
+
+def test_empty_truthiness():
+    assert s(1)
+    assert not s()
+
+
+def test_contains_elements_that_it_was_initialized_with():
+    initial = [1, 2, 3]
+    s = pset(initial)
+
+    assert set(s) == set(initial)
+    assert len(s) == len(set(initial))
+
+
+def test_is_immutable():
+    s1 = pset([1])
+    s2 = s1.add(2)
+
+    assert s1 == pset([1])
+    assert s2 == pset([1, 2])
+
+    s3 = s2.remove(1)
+    assert s2 == pset([1, 2])
+    assert s3 == pset([2])
+
+
+def test_remove_when_not_present():
+    s1 = s(1, 2, 3)
+    with pytest.raises(KeyError):
+        s1.remove(4)
+
+
+def test_discard():
+    s1 = s(1, 2, 3)
+    assert s1.discard(3) == s(1, 2)
+    assert s1.discard(4) is s1
+
+
+def test_is_iterable():
+    assert sum(pset([1, 2, 3])) == 6
+
+
+def test_contains():
+    s = pset([1, 2, 3])
+
+    assert 2 in s
+    assert 4 not in s
+
+
+def test_supports_set_operations():
+    s1 = pset([1, 2, 3])
+    s2 = pset([3, 4, 5])
+
+    assert s1 | s2 == s(1, 2, 3, 4, 5)
+    assert s1.union(s2) == s1 | s2
+
+    assert s1 & s2 == s(3)
+    assert s1.intersection(s2) == s1 & s2
+
+    assert s1 - s2 == s(1, 2)
+    assert s1.difference(s2) == s1 - s2
+
+    assert s1 ^ s2 == s(1, 2, 4, 5)
+    assert s1.symmetric_difference(s2) == s1 ^ s2
+
+
+def test_supports_set_comparisons():
+    s1 = s(1, 2, 3)
+    s3 = s(1, 2)
+    s4 = s(1, 2, 3)
+
+    assert s(1, 2, 3, 3, 5) == s(1, 2, 3, 5)
+    assert s1 != s3
+
+    assert s3 < s1
+    assert s3 <= s1
+    assert s3 <= s4
+
+    assert s1 > s3
+    assert s1 >= s3
+    assert s4 >= s3
+
+
+def test_str():
+    rep = str(pset([1, 2, 3]))
+    assert rep == "pset([1, 2, 3])"
+
+
+def test_is_disjoint():
+    s1 = pset([1, 2, 3])
+    s2 = pset([3, 4, 5])
+    s3 = pset([4, 5])
+
+    assert not s1.isdisjoint(s2)
+    assert s1.isdisjoint(s3)
+
+
+def test_evolver_simple_add():
+    x = s(1, 2, 3)
+    e = x.evolver()
+    assert not e.is_dirty()
+
+    e.add(4)
+    assert e.is_dirty()
+
+    x2 = e.persistent()
+    assert not e.is_dirty()
+    assert x2 == s(1, 2, 3, 4)
+    assert x == s(1, 2, 3)
+
+def test_evolver_simple_remove():
+    x = s(1, 2, 3)
+    e = x.evolver()
+    e.remove(2)
+
+    x2 = e.persistent()
+    assert x2 == s(1, 3)
+    assert x == s(1, 2, 3)
+
+
+def test_evolver_no_update_produces_same_pset():
+    x = s(1, 2, 3)
+    e = x.evolver()
+    assert e.persistent() is x
+
+
+def test_evolver_len():
+    x = s(1, 2, 3)
+    e = x.evolver()
+    assert len(e) == 3
+
+
+def test_copy_returns_reference_to_self():
+    s1 = s(10)
+    assert s1.copy() is s1
+
+
+def test_pickling_empty_set():
+    assert pickle.loads(pickle.dumps(s(), -1)) == s()
+
+
+def test_pickling_non_empty_map():
+    assert pickle.loads(pickle.dumps(s(1, 2), -1)) == s(1, 2)
+
+
+def test_supports_weakref():
+    import weakref
+    weakref.ref(s(1))
+
+
+def test_update():
+    assert s(1, 2, 3).update([3, 4, 4, 5]) == s(1, 2, 3, 4, 5)
+
+
+def test_update_no_elements():
+    s1 = s(1, 2)
+    assert s1.update([]) is s1
+
+
+def test_iterable():
+    """
+    PSets can be created from iterables even though they can't be len() hinted.
+    """
+
+    assert pset(iter("a")) == pset(iter("a"))
diff --git a/contrib/python/pyrsistent/py3/tests/toolz_test.py b/contrib/python/pyrsistent/py3/tests/toolz_test.py
new file mode 100644
index 00000000000..d145704b864
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/toolz_test.py
@@ -0,0 +1,6 @@
+from pyrsistent import get_in, m, v
+
+
+def test_get_in():
+    # This is not an extensive test. The doctest covers that fairly good though.
+    get_in(m(a=v(1, 2, 3)), ['m', 1]) == 2
diff --git a/contrib/python/pyrsistent/py3/tests/transform_test.py b/contrib/python/pyrsistent/py3/tests/transform_test.py
new file mode 100644
index 00000000000..d133d14f65d
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/transform_test.py
@@ -0,0 +1,122 @@
+from pyrsistent import freeze, inc, discard, rex, ny, field, PClass, pmap
+
+
+def test_callable_command():
+    m = freeze({'foo': {'bar': {'baz': 1}}})
+    assert m.transform(['foo', 'bar', 'baz'], inc) == {'foo': {'bar': {'baz': 2}}}
+
+
+def test_predicate():
+    m = freeze({'foo': {'bar': {'baz': 1}, 'qux': {'baz': 1}}})
+    assert m.transform(['foo', lambda x: x.startswith('b'), 'baz'], inc) == {'foo': {'bar': {'baz': 2}, 'qux': {'baz': 1}}}
+
+
+def test_broken_predicate():
+    broken_predicates = [
+        lambda: None,
+        lambda a, b, c: None,
+        lambda a, b, c, d=None: None,
+        lambda *args: None,
+        lambda **kwargs: None,
+    ]
+    for pred in broken_predicates:
+        try:
+            freeze({}).transform([pred], None)
+            assert False
+        except ValueError as e:
+            assert str(e) == "callable in transform path must take 1 or 2 arguments"
+
+
+def test_key_value_predicate():
+    m = freeze({
+        'foo': 1,
+        'bar': 2,
+    })
+    assert m.transform([
+        lambda k, v: (k, v) == ('foo', 1),
+    ], lambda v: v * 3) == {"foo": 3, "bar": 2}
+
+
+def test_remove():
+    m = freeze({'foo': {'bar': {'baz': 1}}})
+    assert m.transform(['foo', 'bar', 'baz'], discard) == {'foo': {'bar': {}}}
+
+
+def test_remove_pvector():
+    m = freeze({'foo': [1, 2, 3]})
+    assert m.transform(['foo', 1], discard) == {'foo': [1, 3]}
+
+
+def test_remove_pclass():
+    class MyClass(PClass):
+        a = field()
+        b = field()
+
+    m = freeze({'foo': MyClass(a=1, b=2)})
+    assert m.transform(['foo', 'b'], discard) == {'foo': MyClass(a=1)}
+
+
+def test_predicate_no_match():
+    m = freeze({'foo': {'bar': {'baz': 1}}})
+    assert m.transform(['foo', lambda x: x.startswith('c'), 'baz'], inc) == m
+
+
+def test_rex_predicate():
+    m = freeze({'foo': {'bar': {'baz': 1},
+                        'bof': {'baz': 1}}})
+    assert m.transform(['foo', rex('^bo.*'), 'baz'], inc) == {'foo': {'bar': {'baz': 1},
+                                                                      'bof': {'baz': 2}}}
+
+
+def test_rex_with_non_string_key():
+    m = freeze({'foo': 1, 5: 2})
+    assert m.transform([rex(".*")], 5) == {'foo': 5, 5: 2}
+
+
+def test_ny_predicated_matches_any_key():
+    m = freeze({'foo': 1, 5: 2})
+    assert m.transform([ny], 5) == {'foo': 5, 5: 5}
+
+
+def test_new_elements_created_when_missing():
+    m = freeze({})
+    assert m.transform(['foo', 'bar', 'baz'], 7) == {'foo': {'bar': {'baz': 7}}}
+
+
+def test_mixed_vector_and_map():
+    m = freeze({'foo': [1, 2, 3]})
+    assert m.transform(['foo', 1], 5) == freeze({'foo': [1, 5, 3]})
+
+
+def test_vector_predicate_callable_command():
+    v = freeze([1, 2, 3, 4, 5])
+    assert v.transform([lambda i: 0 < i < 4], inc) == freeze(freeze([1, 3, 4, 5, 5]))
+
+
+def test_vector_insert_map_one_step_beyond_end():
+    v = freeze([1, 2])
+    assert v.transform([2, 'foo'], 3) == freeze([1, 2, {'foo': 3}])
+
+
+def test_multiple_transformations():
+    v = freeze([1, 2])
+    assert v.transform([2, 'foo'], 3, [2, 'foo'], inc) == freeze([1, 2, {'foo': 4}])
+
+
+def test_no_transformation_returns_the_same_structure():
+    v = freeze([{'foo': 1}, {'bar': 2}])
+    assert v.transform([ny, ny], lambda x: x) is v
+
+
+def test_discard_multiple_elements_in_pvector():
+    assert freeze([0, 1, 2, 3, 4]).transform([lambda i: i % 2], discard) == freeze([0, 2, 4])
+
+
+def test_transform_insert_empty_pmap():
+    m = pmap().transform(['123'], pmap())
+    assert m == pmap({'123': pmap()})
+
+
+def test_discard_does_not_insert_nodes():
+    m = freeze({}).transform(['foo', 'bar'], discard)
+    assert m == pmap({})
diff --git a/contrib/python/pyrsistent/py3/tests/vector_test.py b/contrib/python/pyrsistent/py3/tests/vector_test.py
new file mode 100644
index 00000000000..e5c4bf69c3b
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/vector_test.py
@@ -0,0 +1,934 @@
+from collections.abc import Hashable, Sequence
+import os
+import pickle
+import pytest
+
+from pyrsistent._pvector import python_pvector
+
+
+@pytest.fixture(scope='session', params=['pyrsistent._pvector', 'pvectorc'])
+def pvector(request):
+    if request.param == 'pvectorc' and os.environ.get('PYRSISTENT_NO_C_EXTENSION'):
+        pytest.skip('Configured to not run tests for C extension')
+
+    m = pytest.importorskip(request.param)
+    if request.param == 'pyrsistent._pvector':
+        return m.python_pvector
+    return m.pvector
+
+
+def test_literalish_works():
+    from pyrsistent import pvector, v
+    assert v() is pvector()
+    assert v(1, 2) == pvector([1, 2])
+
+
+def test_empty_initialization(pvector):
+    seq = pvector()
+    assert len(seq) == 0
+
+    with pytest.raises(IndexError) as error:
+        x = seq[0]
+    assert str(error.value) == 'Index out of range: 0'
+
+
+def test_initialization_with_one_element(pvector):
+    seq = pvector([3])
+    assert len(seq) == 1
+    assert seq[0] == 3
+
+
+def test_append_works_and_does_not_affect_original_within_tail(pvector):
+    seq1 = pvector([3])
+    seq2 = seq1.append(2)
+
+    assert len(seq1) == 1
+    assert seq1[0] == 3
+
+    assert len(seq2) == 2
+    assert seq2[0] == 3
+    assert seq2[1] == 2
+
+
+def test_append_works_and_does_not_affect_original_outside_tail(pvector):
+    original = pvector([])
+    seq = original
+
+    for x in range(33):
+        seq = seq.append(x)
+
+    assert len(seq) == 33
+    assert seq[0] == 0
+    assert seq[31] == 31
+    assert seq[32] == 32
+
+    assert len(original) == 0
+
+
+def test_append_when_root_overflows(pvector):
+    seq = pvector([])
+
+    for x in range(32 * 33):
+        seq = seq.append(x)
+
+    seq = seq.append(10001)
+
+    for i in range(32 * 33):
+        assert seq[i] == i
+
+    assert seq[32 * 33] == 10001
+
+
+def test_multi_level_sequence(pvector):
+    seq = pvector(range(8000))
+    seq2 = seq.append(11)
+
+    assert seq[5] == 5
+    assert seq2[7373] == 7373
+    assert seq2[8000] == 11
+
+
+def test_multi_level_sequence_from_iterator(pvector):
+    seq = pvector(iter(range(8000)))
+    seq2 = seq.append(11)
+
+    assert seq[5] == 5
+    assert seq2[7373] == 7373
+    assert seq2[8000] == 11
+
+
+def test_random_insert_within_tail(pvector):
+    seq = pvector([1, 2, 3])
+
+    seq2 = seq.set(1, 4)
+
+    assert seq2[1] == 4
+    assert seq[1] == 2
+
+
+def test_random_insert_outside_tail(pvector):
+    seq = pvector(range(20000))
+
+    seq2 = seq.set(19000, 4)
+
+    assert seq2[19000] == 4
+    assert seq[19000] == 19000
+
+
+def test_insert_beyond_end(pvector):
+    seq = pvector(range(2))
+    seq2 = seq.set(2, 50)
+    assert seq2[2] == 50
+
+    with pytest.raises(IndexError) as error:
+        seq2.set(19, 4)
+
+    assert str(error.value) == 'Index out of range: 19'
+
+
+def test_insert_with_index_from_the_end(pvector):
+    x = pvector([1, 2, 3, 4])
+
+    assert x.set(-2, 5) == pvector([1, 2, 5, 4])
+
+
+def test_insert_with_too_negative_index(pvector):
+    x = pvector([1, 2, 3, 4])
+
+    with pytest.raises(IndexError):
+        x.set(-5, 17)
+
+
+def test_iteration(pvector):
+    y = 0
+    seq = pvector(range(2000))
+    for x in seq:
+        assert x == y
+        y += 1
+
+    assert y == 2000
+
+
+def test_zero_extend(pvector):
+    the_list = []
+    seq = pvector()
+    seq2 = seq.extend(the_list)
+    assert seq == seq2
+
+
+def test_short_extend(pvector):
+    # Extend within tail length
+    the_list = [1, 2]
+    seq = pvector()
+    seq2 = seq.extend(the_list)
+
+    assert len(seq2) == len(the_list)
+    assert seq2[0] == the_list[0]
+    assert seq2[1] == the_list[1]
+
+
+def test_long_extend(pvector):
+    # Multi level extend
+    seq = pvector()
+    length = 2137
+
+    # Extend from scratch
+    seq2 = seq.extend(range(length))
+    assert len(seq2) == length
+    for i in range(length):
+        assert seq2[i] == i
+
+    # Extend already filled vector
+    seq3 = seq2.extend(range(length, length + 5))
+    assert len(seq3) == length + 5
+    for i in range(length + 5):
+        assert seq3[i] == i
+
+    # Check that the original vector is still intact
+    assert len(seq2) == length
+    for i in range(length):
+        assert seq2[i] == i
+
+
+def test_slicing_zero_length_range(pvector):
+    seq = pvector(range(10))
+    seq2 = seq[2:2]
+
+    assert len(seq2) == 0
+
+
+def test_slicing_range(pvector):
+    seq = pvector(range(10))
+    seq2 = seq[2:4]
+
+    assert list(seq2) == [2, 3]
+
+
+def test_slice_identity(pvector):
+    # Pvector is immutable, no need to make a copy!
+    seq = pvector(range(10))
+
+    assert seq is seq[::]
+
+
+def test_slicing_range_with_step(pvector):
+    seq = pvector(range(100))
+    seq2 = seq[2:12:3]
+
+    assert list(seq2) == [2, 5, 8, 11]
+
+
+def test_slicing_no_range_but_step(pvector):
+    seq = pvector(range(10))
+    seq2 = seq[::2]
+
+    assert list(seq2) == [0, 2, 4, 6, 8]
+
+
+def test_slicing_reverse(pvector):
+    seq = pvector(range(10))
+    seq2 = seq[::-1]
+
+    assert seq2[0] == 9
+    assert seq2[1] == 8
+    assert len(seq2) == 10
+
+    seq3 = seq[-3: -7: -1]
+    assert seq3[0] == 7
+    assert seq3[3] == 4
+    assert len(seq3) == 4
+
+
+def test_delete_index(pvector):
+    seq = pvector([1, 2, 3])
+    assert seq.delete(0) == pvector([2, 3])
+    assert seq.delete(1) == pvector([1, 3])
+    assert seq.delete(2) == pvector([1, 2])
+    assert seq.delete(-1) == pvector([1, 2])
+    assert seq.delete(-2) == pvector([1, 3])
+    assert seq.delete(-3) == pvector([2, 3])
+
+
+def test_delete_index_out_of_bounds(pvector):
+    with pytest.raises(IndexError):
+        pvector([]).delete(0)
+    with pytest.raises(IndexError):
+        pvector([]).delete(-1)
+
+
+def test_delete_index_malformed(pvector):
+    with pytest.raises(TypeError):
+        pvector([]).delete('a')
+
+
+def test_delete_slice(pvector):
+    seq = pvector(range(5))
+    assert seq.delete(1, 4) == pvector([0, 4])
+    assert seq.delete(4, 1) == seq
+    assert seq.delete(0, 1) == pvector([1, 2, 3, 4])
+    assert seq.delete(6, 8) == seq
+    assert seq.delete(-1, 1) == seq
+    assert seq.delete(1, -1) == pvector([0, 4])
+
+
+def test_remove(pvector):
+    seq = pvector(range(5))
+    assert seq.remove(3) == pvector([0, 1, 2, 4])
+
+
+def test_remove_first_only(pvector):
+    seq = pvector([1, 2, 3, 2, 1])
+    assert seq.remove(2) == pvector([1, 3, 2, 1])
+
+
+def test_remove_index_out_of_bounds(pvector):
+    seq = pvector(range(5))
+    with pytest.raises(ValueError) as err:
+        seq.remove(5)
+    assert 'not in' in str(err.value)
+
+
+def test_addition(pvector):
+    v = pvector([1, 2]) + pvector([3, 4])
+
+    assert list(v) == [1, 2, 3, 4]
+
+
+def test_sorted(pvector):
+    seq = pvector([5, 2, 3, 1])
+    assert [1, 2, 3, 5] == sorted(seq)
+
+
+def test_boolean_conversion(pvector):
+    assert not bool(pvector())
+    assert bool(pvector([1]))
+
+
+def test_access_with_negative_index(pvector):
+    seq = pvector([1, 2, 3, 4])
+
+    assert seq[-1] == 4
+    assert seq[-4] == 1
+
+
+def test_index_error_positive(pvector):
+    with pytest.raises(IndexError):
+        pvector([1, 2, 3])[3]
+
+
+def test_index_error_negative(pvector):
+    with pytest.raises(IndexError):
+        pvector([1, 2, 3])[-4]
+
+
+def test_is_sequence(pvector):
+    assert isinstance(pvector(), Sequence)
+
+
+def test_empty_repr(pvector):
+    assert str(pvector()) == "pvector([])"
+
+
+def test_non_empty_repr(pvector):
+    v = pvector([1, 2, 3])
+    assert str(v) == "pvector([1, 2, 3])"
+
+    # There's some state that needs to be reset between calls in the native version,
+    # test that multiple invocations work.
+    assert str(v) == "pvector([1, 2, 3])"
+
+
+def test_repr_when_contained_object_contains_reference_to_self(pvector):
+    x = [1, 2, 3]
+    v = pvector([1, 2, x])
+    x.append(v)
+    assert str(v) == 'pvector([1, 2, [1, 2, 3, pvector([1, 2, [...]])]])'
+
+    # Run a GC to provoke any potential misbehavior
+    import gc
+    gc.collect()
+
+
+def test_is_hashable(pvector):
+
+    v = pvector([1, 2, 3])
+    v2 = pvector([1, 2, 3])
+
+    assert hash(v) == hash(v2)
+    assert isinstance(pvector(), Hashable)
+
+
+def test_refuses_to_hash_when_members_are_unhashable(pvector):
+    v = pvector([1, 2, [1, 2]])
+
+    with pytest.raises(TypeError):
+        hash(v)
+
+
+def test_compare_same_vectors(pvector):
+    v = pvector([1, 2])
+    assert v == v
+    assert pvector() == pvector()
+
+
+def test_compare_with_other_type_of_object(pvector):
+    assert pvector([1, 2]) != 'foo'
+
+
+def test_compare_equal_vectors(pvector):
+    v1 = pvector([1, 2])
+    v2 = pvector([1, 2])
+    assert v1 == v2
+    assert v1 >= v2
+    assert v1 <= v2
+
+
+def test_compare_different_vectors_same_size(pvector):
+    v1 = pvector([1, 2])
+    v2 = pvector([1, 3])
+    assert v1 != v2
+
+
+def test_compare_different_vectors_different_sizes(pvector):
+    v1 = pvector([1, 2])
+    v2 = pvector([1, 2, 3])
+    assert v1 != v2
+
+
+def test_compare_lt_gt(pvector):
+    v1 = pvector([1, 2])
+    v2 = pvector([1, 2, 3])
+    assert v1 < v2
+    assert v2 > v1
+
+
+def test_repeat(pvector):
+    v = pvector([1, 2])
+    assert 5 * pvector() is pvector()
+    assert v is 1 * v
+    assert 0 * v is pvector()
+    assert 2 * pvector([1, 2]) == pvector([1, 2, 1, 2])
+    assert -3 * pvector([1, 2]) is pvector()
+
+
+def test_transform_zero_key_length(pvector):
+    x = pvector([1, 2])
+
+    assert x.transform([], 3) == 3
+
+
+def test_transform_base_case(pvector):
+    x = pvector([1, 2])
+
+    assert x.transform([1], 3) == pvector([1, 3])
+
+
+def test_transform_nested_vectors(pvector):
+    x = pvector([1, 2, pvector([3, 4]), 5])
+
+    assert x.transform([2, 0], 999) == pvector([1, 2, pvector([999, 4]), 5])
+
+
+def test_transform_when_appending(pvector):
+    from pyrsistent import m
+    x = pvector([1, 2])
+
+    assert x.transform([2, 'd'], 999) == pvector([1, 2, m(d=999)])
+
+
+def test_transform_index_error_out_range(pvector):
+    x = pvector([1, 2, pvector([3, 4]), 5])
+
+    with pytest.raises(IndexError):
+        x.transform([2, 10], 999)
+
+
+def test_transform_index_error_wrong_type(pvector):
+    x = pvector([1, 2, pvector([3, 4]), 5])
+
+    with pytest.raises(TypeError):
+        x.transform([2, 'foo'], 999)
+
+
+def test_transform_non_setable_type(pvector):
+    x = pvector([1, 2, 5])
+
+    with pytest.raises(TypeError):
+        x.transform([2, 3], 999)
+
+
+def test_reverse(pvector):
+    x = pvector([1, 2, 5])
+
+    assert list(reversed(x)) == [5, 2, 1]
+
+
+def test_contains(pvector):
+    x = pvector([1, 2, 5])
+
+    assert 2 in x
+    assert 3 not in x
+
+
+def test_index(pvector):
+    x = pvector([1, 2, 5])
+
+    assert x.index(5) == 2
+
+
+def test_index_not_found(pvector):
+    x = pvector([1, 2, 5])
+
+    with pytest.raises(ValueError):
+        x.index(7)
+
+
+def test_index_not_found_with_limits(pvector):
+    x = pvector([1, 2, 5, 1])
+
+    with pytest.raises(ValueError):
+        x.index(1, 1, 3)
+
+
+def test_count(pvector):
+    x = pvector([1, 2, 5, 1])
+
+    assert x.count(1) == 2
+    assert x.count(4) == 0
+
+
+def test_empty_truthiness(pvector):
+    assert pvector([1])
+    assert not pvector([])
+
+
+def test_pickling_empty_vector(pvector):
+    assert pickle.loads(pickle.dumps(pvector(), -1)) == pvector()
+
+
+def test_pickling_non_empty_vector(pvector):
+    assert pickle.loads(pickle.dumps(pvector([1, 'a']), -1)) == pvector([1, 'a'])
+
+
+def test_mset_basic_assignments(pvector):
+    v1 = pvector(range(2000))
+    v2 = v1.mset(1, -1, 505, -505, 1998, -1998)
+
+    # Original not changed
+    assert v1[1] == 1
+    assert v1[505] == 505
+    assert v1[1998] == 1998
+
+    # Other updated
+    assert v2[1] == -1
+    assert v2[505] == -505
+    assert v2[1998] == -1998
+
+
+def test_mset_odd_number_of_arguments(pvector):
+    v = pvector([0, 1])
+
+    with pytest.raises(TypeError):
+        v.mset(0, 10, 1)
+
+
+def test_mset_index_out_of_range(pvector):
+    v = pvector([0, 1])
+
+    with pytest.raises(IndexError):
+        v.mset(3, 10)
+
+
+def test_evolver_no_update(pvector):
+    # This is mostly a test against memory leaks in the C implementation
+    v = pvector(range(40))
+
+    assert v.evolver().persistent() == v
+
+
+def test_evolver_deallocate_dirty_evolver(pvector):
+    # Ref count handling in native implementation
+    v = pvector(range(3220))
+    e = v.evolver()
+    e[10] = -10
+    e[3220] = -3220
+
+
+def test_evolver_simple_update_in_tree(pvector):
+    v = pvector(range(35))
+    e = v.evolver()
+    e[10] = -10
+
+    assert e[10] == -10
+    assert e.persistent()[10] == -10
+
+
+def test_evolver_set_out_of_range(pvector):
+    v = pvector([0])
+    e = v.evolver()
+    with pytest.raises(IndexError) as error:
+        e[10] = 1
+    assert str(error.value) == "Index out of range: 10"
+
+def test_evolver_multi_level_multi_update_in_tree(pvector):
+    # This test is mostly to detect memory/ref count issues in the native implementation
+    v = pvector(range(3500))
+    e = v.evolver()
+
+    # Update differs between first and second time since the
+    # corresponding node will be marked as dirty the first time only.
+    e[10] = -10
+    e[11] = -11
+    e[10] = -1000
+
+    # Update in neighbour node
+    e[50] = -50
+    e[50] = -5000
+
+    # Update in node in other half of vector
+    e[3000] = -3000
+    e[3000] = -30000
+
+    # Before freezing
+    assert e[10] == -1000
+    assert e[11] == -11
+    assert e[50] == -5000
+    assert e[3000] == -30000
+
+    # Run a GC to provoke any potential misbehavior
+    import gc
+    gc.collect()
+
+    v2 = e.persistent()
+    assert v2[10] == -1000
+    assert v2[50] == -5000
+    assert v2[3000] == -30000
+
+    # Run a GC to provoke any potential misbehavior
+    gc.collect()
+
+    # After freezing
+    assert e[10] == -1000
+    assert e[11] == -11
+    assert e[50] == -5000
+    assert e[3000] == -30000
+
+    # Original stays the same
+    assert v[10] == 10
+    assert v[50] == 50
+    assert v[3000] == 3000
+
+
+def test_evolver_simple_update_in_tail(pvector):
+    v = pvector(range(35))
+    e = v.evolver()
+    e[33] = -33
+
+    assert e[33] == -33
+    assert e.persistent()[33] == -33
+    assert v[33] == 33
+
+
+def test_evolver_simple_update_just_outside_vector(pvector):
+    v = pvector()
+    e = v.evolver()
+    e[0] = 1
+
+    assert e[0] == 1
+    assert e.persistent()[0] == 1
+    assert len(v) == 0
+
+
+def test_evolver_append(pvector):
+    v = pvector()
+    e = v.evolver()
+    e.append(1000)
+    assert e[0] == 1000
+
+    e[0] = 2000
+    assert e[0] == 2000
+    assert list(e.persistent()) == [2000]
+    assert list(v) == []
+
+
+def test_evolver_extend(pvector):
+    v = pvector([1000])
+    e = v.evolver()
+    e.extend([2000, 3000])
+    e[2] = 20000
+
+    assert list(e.persistent()) == [1000, 2000, 20000]
+    assert list(v) == [1000]
+
+
+def test_evolver_assign_and_read_with_negative_indices(pvector):
+    v = pvector([1, 2, 3])
+    e = v.evolver()
+    e[-1] = 4
+    e.extend([11, 12, 13])
+    e[-1] = 33
+
+    assert e[-1] == 33
+    assert list(e.persistent()) == [1, 2, 4, 11, 12, 33]
+
+
+def test_evolver_non_integral_access(pvector):
+    e = pvector([1]).evolver()
+
+    with pytest.raises(TypeError):
+        x = e['foo']
+
+
+def test_evolver_non_integral_assignment(pvector):
+    e = pvector([1]).evolver()
+
+    with pytest.raises(TypeError):
+        e['foo'] = 1
+
+
+def test_evolver_out_of_bounds_access(pvector):
+    e = pvector([1]).evolver()
+
+    with pytest.raises(IndexError):
+        x = e[1]
+
+
+def test_evolver_out_of_bounds_assignment(pvector):
+    e = pvector([1]).evolver()
+
+    with pytest.raises(IndexError):
+        e[2] = 1
+
+
+def test_no_dependencies_between_evolvers_from_the_same_pvector(pvector):
+    original_list = list(range(40))
+    v = pvector(original_list)
+    e1 = v.evolver()
+    e2 = v.evolver()
+
+    e1.extend([1, 2, 3])
+    e1[2] = 20
+    e1[35] = 350
+
+    e2.extend([-1, -2, -3])
+    e2[2] = -20
+    e2[35] = -350
+
+    e1_expected = original_list + [1, 2, 3]
+    e1_expected[2] = 20
+    e1_expected[35] = 350
+    assert list(e1.persistent()) == e1_expected
+
+    e2_expected = original_list + [-1, -2, -3]
+    e2_expected[2] = -20
+    e2_expected[35] = -350
+    assert list(e2.persistent()) == e2_expected
+
+
+def test_pvectors_produced_from_the_same_evolver_do_not_interfere(pvector):
+    original_list = list(range(40))
+    v = pvector(original_list)
+    e = v.evolver()
+
+    e.extend([1, 2, 3])
+    e[2] = 20
+    e[35] = 350
+
+    v1 = e.persistent()
+    v1_expected = original_list + [1, 2, 3]
+    v1_expected[2] = 20
+    v1_expected[35] = 350
+
+    e.extend([-1, -2, -3])
+    e[3] = -30
+    e[36] = -360
+
+    v2 = e.persistent()
+    v2_expected = v1_expected + [-1, -2, -3]
+    v2_expected[3] = -30
+    v2_expected[36] = -360
+
+    assert list(v1) == v1_expected
+    assert list(v2) == v2_expected
+
+
+def test_evolver_len(pvector):
+    e = pvector([1, 2, 3]).evolver()
+    e.extend([4, 5])
+
+    assert len(e) == 5
+
+
+def test_evolver_is_dirty(pvector):
+    e = pvector([1, 2, 3]).evolver()
+    assert not e.is_dirty()
+
+    e.append(4)
+    assert e.is_dirty
+
+    e.persistent()
+    assert not e.is_dirty()
+
+    e[2] = 2000
+    assert e.is_dirty
+
+    e.persistent()
+    assert not e.is_dirty()
+
+
+def test_vector_insert_one_step_beyond_end(pvector):
+    # This test exists to get the transform functionality under memory
+    # leak supervision. Most of the transformation tests are in test_transform.py.
+    v = pvector([1, 2])
+    assert v.transform([2], 3) == pvector([1, 2, 3])
+
+
+def test_evolver_with_no_updates_returns_same_pvector(pvector):
+    v = pvector([1, 2])
+    assert v.evolver().persistent() is v
+
+
+def test_evolver_returns_itself_on_evolving_operations(pvector):
+    # Does this to be able to chain operations
+    v = pvector([1, 2])
+    assert v.evolver().append(3).extend([4, 5]).set(1, 6).persistent() == pvector([1, 6, 3, 4, 5])
+
+
+def test_evolver_delete_by_index(pvector):
+    e = pvector([1, 2, 3]).evolver()
+
+    del e[0]
+
+    assert e.persistent() == python_pvector([2, 3])
+    assert e.append(4).persistent() == python_pvector([2, 3, 4])
+
+
+def test_evolver_delete_function_by_index(pvector):
+    e = pvector([1, 2, 3]).evolver()
+
+    assert e.delete(1).persistent() == python_pvector([1, 3])
+
+
+def test_evolver_delete_function_by_index_multiple_times(pvector):
+    SIZE = 40
+    e = pvector(range(SIZE)).evolver()
+    for i in range(SIZE):
+        assert e[0] == i
+        assert list(e.persistent()) == list(range(i, SIZE))
+        del e[0]
+
+    assert e.persistent() == list()
+
+
+def test_evolver_delete_function_invalid_index(pvector):
+    e = pvector([1, 2]).evolver()
+
+    with pytest.raises(TypeError):
+        del e["e"]
+
+
+def test_delete_of_non_existing_element(pvector):
+    e = pvector([1, 2]).evolver()
+
+    with pytest.raises(IndexError):
+        del e[2]
+
+    del e[0]
+    del e[0]
+
+    with pytest.raises(IndexError):
+        del e[0]
+
+    assert e.persistent() == pvector()
+
+
+def test_append_followed_by_delete(pvector):
+    e = pvector([1, 2]).evolver()
+
+    e.append(3)
+
+    del e[2]
+
+
+def test_evolver_set_followed_by_delete(pvector):
+    evolver = pvector([1, 2]).evolver()
+    evolver[1] = 3
+
+    assert [evolver[i] for i in range(len(evolver))] == [1, 3]
+
+    del evolver[0]
+
+    assert evolver.persistent() == pvector([3])
+
+
+def test_compare_with_list(pvector):
+    v = pvector([1, 2, 3])
+
+    assert v == [1, 2, 3]
+    assert v != [1, 2]
+    assert v > [1, 2]
+    assert v < [2, 2]
+    assert [1, 2] < v
+    assert v <= [1, 2, 3]
+    assert v <= [1, 2, 4]
+    assert v >= [1, 2, 3]
+    assert v >= [1, 2]
+
+
+def test_compare_with_non_iterable(pvector):
+    assert pvector([1, 2, 3]) != 5
+    assert not (pvector([1, 2, 3]) == 5)
+
+
+def test_python_no_c_extension_with_environment_variable():
+    from importlib import reload as reload_module
+    import pyrsistent._pvector
+    import pyrsistent
+    import os
+
+    os.environ['PYRSISTENT_NO_C_EXTENSION'] = 'TRUE'
+
+    reload_module(pyrsistent._pvector)
+    reload_module(pyrsistent)
+
+    assert type(pyrsistent.pvector()) is pyrsistent._pvector.PythonPVector
+
+    del os.environ['PYRSISTENT_NO_C_EXTENSION']
+
+    reload_module(pyrsistent._pvector)
+    reload_module(pyrsistent)
+
+
+def test_supports_weakref(pvector):
+    import weakref
+    weakref.ref(pvector())
+
+def test_get_evolver_referents(pvector):
+    """The C implementation of the evolver should expose the original PVector
+    to the gc only once.
+    """
+    if pvector.__module__ == 'pyrsistent._pvector':
+        pytest.skip("This test only applies to pvectorc")
+    import gc
+    v = pvector([1, 2, 3])
+    e = v.evolver()
+    assert len([x for x in gc.get_referents(e) if x is v]) == 1
+
+
+def test_failing_repr(pvector):
+    # See https://github.com/tobgu/pyrsistent/issues/84
+    class A(object):
+        def __repr__(self):
+            raise ValueError('oh no!')
+
+    with pytest.raises(ValueError):
+        repr(pvector([A()]))
+
+
+def test_iterable(pvector):
+    """
+    PVectors can be created from iterables even though they can't be len()
+    hinted.
+    """
+
+    assert pvector(iter("a")) == pvector(iter("a"))
diff --git a/contrib/python/pyrsistent/py3/tests/ya.make b/contrib/python/pyrsistent/py3/tests/ya.make
new file mode 100644
index 00000000000..8bb63ae5597
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/tests/ya.make
@@ -0,0 +1,27 @@
+PY3TEST()
+
+PEERDIR(
+    contrib/python/pyrsistent
+)
+
+TEST_SRCS(
+    bag_test.py
+    checked_map_test.py
+    checked_set_test.py
+    checked_vector_test.py
+    class_test.py
+    deque_test.py
+    field_test.py
+    freeze_test.py
+    immutable_object_test.py
+    list_test.py
+    map_test.py
+    record_test.py
+    regression_test.py
+    set_test.py
+    toolz_test.py
+)
+
+NO_LINT()
+
+END()
diff --git a/contrib/python/pyrsistent/py3/ya.make b/contrib/python/pyrsistent/py3/ya.make
new file mode 100644
index 00000000000..bec491d8beb
--- /dev/null
+++ b/contrib/python/pyrsistent/py3/ya.make
@@ -0,0 +1,47 @@
+# Generated by devtools/yamaker (pypi).
+
+PY3_LIBRARY()
+
+PROVIDES(python_pyrsistent)
+
+VERSION(0.20.0)
+
+LICENSE(MIT)
+
+NO_LINT()
+
+PY_SRCS(
+    TOP_LEVEL
+    _pyrsistent_version.py
+    pyrsistent/__init__.py
+    pyrsistent/__init__.pyi
+    pyrsistent/_checked_types.py
+    pyrsistent/_field_common.py
+    pyrsistent/_helpers.py
+    pyrsistent/_immutable.py
+    pyrsistent/_pbag.py
+    pyrsistent/_pclass.py
+    pyrsistent/_pdeque.py
+    pyrsistent/_plist.py
+    pyrsistent/_pmap.py
+    pyrsistent/_precord.py
+    pyrsistent/_pset.py
+    pyrsistent/_pvector.py
+    pyrsistent/_toolz.py
+    pyrsistent/_transformations.py
+    pyrsistent/typing.py
+    pyrsistent/typing.pyi
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/pyrsistent/py3/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+    pyrsistent/py.typed
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/pyrsistent/ya.make b/contrib/python/pyrsistent/ya.make
new file mode 100644
index 00000000000..41b7591417b
--- /dev/null
+++ b/contrib/python/pyrsistent/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+    PEERDIR(contrib/python/pyrsistent/py2)
+ELSE()
+    PEERDIR(contrib/python/pyrsistent/py3)
+ENDIF()
+
+NO_LINT()
+
+END()
+
+RECURSE(
+    py2
+    py3
+)
diff --git a/contrib/python/zope.interface/py2/.dist-info/METADATA b/contrib/python/zope.interface/py2/.dist-info/METADATA
new file mode 100644
index 00000000000..7f8e6998e31
--- /dev/null
+++ b/contrib/python/zope.interface/py2/.dist-info/METADATA
@@ -0,0 +1,1094 @@
+Metadata-Version: 2.1
+Name: zope.interface
+Version: 5.5.2
+Summary: Interfaces for Python
+Home-page: https://github.com/zopefoundation/zope.interface
+Author: Zope Foundation and Contributors
+Author-email: zope-dev@zope.org
+License: ZPL 2.1
+Keywords: interface,components,plugins
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Zope Public License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Framework :: Zope :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
+License-File: LICENSE.txt
+Requires-Dist: setuptools
+Provides-Extra: docs
+Requires-Dist: Sphinx ; extra == 'docs'
+Requires-Dist: repoze.sphinx.autointerface ; extra == 'docs'
+Provides-Extra: test
+Requires-Dist: coverage (>=5.0.3) ; extra == 'test'
+Requires-Dist: zope.event ; extra == 'test'
+Requires-Dist: zope.testing ; extra == 'test'
+Provides-Extra: testing
+Requires-Dist: coverage (>=5.0.3) ; extra == 'testing'
+Requires-Dist: zope.event ; extra == 'testing'
+Requires-Dist: zope.testing ; extra == 'testing'
+
+====================
+ ``zope.interface``
+====================
+
+.. image:: https://img.shields.io/pypi/v/zope.interface.svg
+    :target: https://pypi.python.org/pypi/zope.interface/
+    :alt: Latest Version
+
+.. image:: https://img.shields.io/pypi/pyversions/zope.interface.svg
+        :target: https://pypi.org/project/zope.interface/
+        :alt: Supported Python versions
+
+.. image:: https://github.com/zopefoundation/zope.interface/actions/workflows/tests.yml/badge.svg
+        :target: https://github.com/zopefoundation/zope.interface/actions/workflows/tests.yml
+
+.. image:: https://readthedocs.org/projects/zopeinterface/badge/?version=latest
+        :target: https://zopeinterface.readthedocs.io/en/latest/
+        :alt: Documentation Status
+
+This package is intended to be independently reusable in any Python
+project. It is maintained by the `Zope Toolkit project
+<https://zopetoolkit.readthedocs.io/>`_.
+
+This package provides an implementation of "object interfaces" for Python.
+Interfaces are a mechanism for labeling objects as conforming to a given
+API or contract. So, this package can be considered as implementation of
+the `Design By Contract`_ methodology support in Python.
+
+.. _Design By Contract: http://en.wikipedia.org/wiki/Design_by_contract
+
+For detailed documentation, please see https://zopeinterface.readthedocs.io/en/latest/
+
+=========
+ Changes
+=========
+
+5.5.2 (2022-11-17)
+==================
+
+- Add support for building arm64 wheels on macOS.
+
+
+5.5.1 (2022-11-03)
+==================
+
+- Add support for final Python 3.11 release.
+
+
+5.5.0 (2022-10-10)
+==================
+
+- Add support for Python 3.10 and 3.11 (as of 3.11.0rc2).
+
+- Add missing Trove classifier showing support for Python 3.9.
+
+- Add some more entries to ``zope.interface.interfaces.__all__``.
+
+- Disable unsafe math optimizations in C code.  See `pull request 262
+  <https://github.com/zopefoundation/zope.interface/pull/262>`_.
+
+
+5.4.0 (2021-04-15)
+==================
+
+- Make the C implementation of the ``__providedBy__`` descriptor stop
+  ignoring all errors raised when accessing the instance's
+  ``__provides__``. Now it behaves like the Python version and only
+  catches ``AttributeError``. The previous behaviour could lead to
+  crashing the interpreter in cases of recursion and errors. See
+  `issue 239 <https://github.com/zopefoundation/zope.interface/issues>`_.
+
+- Update the ``repr()`` and ``str()`` of various objects to be shorter
+  and more informative. In many cases, the ``repr()`` is now something
+  that can be evaluated to produce an equal object. For example, what
+  was previously printed as ``<implementedBy builtins.list>`` is now
+  shown as ``classImplements(list, IMutableSequence, IIterable)``. See
+  `issue 236 <https://github.com/zopefoundation/zope.interface/issues/236>`_.
+
+- Make ``Declaration.__add__`` (as in ``implementedBy(Cls) +
+  ISomething``) try harder to preserve a consistent resolution order
+  when the two arguments share overlapping pieces of the interface
+  inheritance hierarchy. Previously, the right hand side was always
+  put at the end of the resolution order, which could easily produce
+  invalid orders. See `issue 193
+  <https://github.com/zopefoundation/zope.interface/issues/193>`_.
+
+5.3.0 (2020-03-21)
+==================
+
+- No changes from 5.3.0a1
+
+
+5.3.0a1 (2021-03-18)
+====================
+
+- Improve the repr of ``zope.interface.Provides`` to remove ambiguity
+  about what is being provided. This is especially helpful diagnosing
+  IRO issues.
+
+- Allow subclasses of ``BaseAdapterRegistry`` (including
+  ``AdapterRegistry`` and ``VerifyingAdapterRegistry``) to have
+  control over the data structures. This allows persistent
+  implementations such as those based on ZODB to choose more scalable
+  options (e.g., BTrees instead of dicts). See `issue 224
+  <https://github.com/zopefoundation/zope.interface/issues/224>`_.
+
+- Fix a reference counting issue in ``BaseAdapterRegistry`` that could
+  lead to references to interfaces being kept around even when all
+  utilities/adapters/subscribers providing that interface have been
+  removed. This is mostly an issue for persistent implementations.
+  Note that this only corrects the issue moving forward, it does not
+  solve any already corrupted reference counts. See `issue 227
+  <https://github.com/zopefoundation/zope.interface/issues/227>`_.
+
+- Add the method ``BaseAdapterRegistry.rebuild()``. This can be used
+  to fix the reference counting issue mentioned above, as well as to
+  update the data structures when custom data types have changed.
+
+- Add the interface method ``IAdapterRegistry.subscribed()`` and
+  implementation ``BaseAdapterRegistry.subscribed()`` for querying
+  directly registered subscribers. See `issue 230
+  <https://github.com/zopefoundation/zope.interface/issues/230>`_.
+
+- Add the maintenance method
+  ``Components.rebuildUtilityRegistryFromLocalCache()``. Most users
+  will not need this, but it can be useful if the ``Components.utilities``
+  registry is suspected to be out of sync with the ``Components``
+  object itself (this might happen to persistent ``Components``
+  implementations in the face of bugs).
+
+- Fix the ``Provides`` and ``ClassProvides`` descriptors to stop
+  allowing redundant interfaces (those already implemented by the
+  underlying class or meta class) to produce an inconsistent
+  resolution order. This is similar to the change in ``@implementer``
+  in 5.1.0, and resolves inconsistent resolution orders with
+  ``zope.proxy`` and ``zope.location``. See `issue 207
+  <https://github.com/zopefoundation/zope.interface/issues/207>`_.
+
+5.2.0 (2020-11-05)
+==================
+
+- Add documentation section ``Persistency and Equality``
+  (`#218 <https://github.com/zopefoundation/zope.interface/issues/218>`_).
+
+- Create arm64 wheels.
+
+- Add support for Python 3.9.
+
+
+5.1.2 (2020-10-01)
+==================
+
+- Make sure to call each invariant only once when validating invariants.
+  Previously, invariants could be called multiple times because when an
+  invariant is defined in an interface, it's found by in all interfaces
+  inheriting from that interface.  See `pull request 215
+  <https://github.com/zopefoundation/zope.interface/pull/215/>`_.
+
+5.1.1 (2020-09-30)
+==================
+
+- Fix the method definitions of ``IAdapterRegistry.subscribe``,
+  ``subscriptions`` and ``subscribers``. Previously, they all were
+  defined to accept a ``name`` keyword argument, but subscribers have
+  no names and the implementation of that interface did not accept
+  that argument. See `issue 208
+  <https://github.com/zopefoundation/zope.interface/issues/208>`_.
+
+- Fix a potential reference leak in the C optimizations. Previously,
+  applications that dynamically created unique ``Specification``
+  objects (e.g., used ``@implementer`` on dynamic classes) could
+  notice a growth of small objects over time leading to increased
+  garbage collection times. See `issue 216
+  <https://github.com/zopefoundation/zope.interface/issues/216>`_.
+
+  .. caution::
+
+     This leak could prevent interfaces used as the bases of
+     other interfaces from being garbage collected. Those interfaces
+     will now be collected.
+
+     One way in which this would manifest was that ``weakref.ref``
+     objects (and things built upon them, like
+     ``Weak[Key|Value]Dictionary``) would continue to have access to
+     the original object even if there were no other visible
+     references to Python and the original object *should* have been
+     collected. This could be especially problematic for the
+     ``WeakKeyDictionary`` when combined with dynamic or local
+     (created in the scope of a function) interfaces, since interfaces
+     are hashed based just on their name and module name. See the
+     linked issue for an example of a resulting ``KeyError``.
+
+     Note that such potential errors are not new, they are just once
+     again a possibility.
+
+5.1.0 (2020-04-08)
+==================
+
+- Make ``@implementer(*iface)`` and ``classImplements(cls, *iface)``
+  ignore redundant interfaces. If the class already implements an
+  interface through inheritance, it is no longer redeclared
+  specifically for *cls*. This solves many instances of inconsistent
+  resolution orders, while still allowing the interface to be declared
+  for readability and maintenance purposes. See `issue 199
+  <https://github.com/zopefoundation/zope.interface/issues/199>`_.
+
+- Remove all bare ``except:`` statements. Previously, when accessing
+  special attributes such as ``__provides__``, ``__providedBy__``,
+  ``__class__`` and ``__conform__``, this package wrapped such access
+  in a bare ``except:`` statement, meaning that many errors could pass
+  silently; typically this would result in a fallback path being taken
+  and sometimes (like with ``providedBy()``) the result would be
+  non-sensical. This is especially true when those attributes are
+  implemented with descriptors. Now, only ``AttributeError`` is
+  caught. This makes errors more obvious.
+
+  Obviously, this means that some exceptions will be propagated
+  differently than before. In particular, ``RuntimeError`` raised by
+  Acquisition in the case of circular containment will now be
+  propagated. Previously, when adapting such a broken object, a
+  ``TypeError`` would be the common result, but now it will be a more
+  informative ``RuntimeError``.
+
+  In addition, ZODB errors like ``POSKeyError`` could now be
+  propagated where previously they would ignored by this package.
+
+  See `issue 200 <https://github.com/zopefoundation/zope.interface/issues/200>`_.
+
+- Require that the second argument (*bases*) to ``InterfaceClass`` is
+  a tuple. This only matters when directly using ``InterfaceClass`` to
+  create new interfaces dynamically. Previously, an individual
+  interface was allowed, but did not work correctly. Now it is
+  consistent with ``type`` and requires a tuple.
+
+- Let interfaces define custom ``__adapt__`` methods. This implements
+  the other side of the :pep:`246` adaptation protocol: objects being
+  adapted could already implement ``__conform__`` if they know about
+  the interface, and now interfaces can implement ``__adapt__`` if
+  they know about particular objects. There is no performance penalty
+  for interfaces that do not supply custom ``__adapt__`` methods.
+
+  This includes the ability to add new methods, or override existing
+  interface methods using the new ``@interfacemethod`` decorator.
+
+  See `issue 3 <https://github.com/zopefoundation/zope.interface/issues/3>`_.
+
+- Make the internal singleton object returned by APIs like
+  ``implementedBy`` and ``directlyProvidedBy`` for objects that
+  implement or provide no interfaces more immutable. Previously an
+  internal cache could be mutated. See `issue 204
+  <https://github.com/zopefoundation/zope.interface/issues/204>`_.
+
+5.0.2 (2020-03-30)
+==================
+
+- Ensure that objects that implement no interfaces (such as direct
+  subclasses of ``object``) still include ``Interface`` itself in
+  their ``__iro___`` and ``__sro___``. This fixes adapter registry
+  lookups for such objects when the adapter is registered for
+  ``Interface``. See `issue 197
+  <https://github.com/zopefoundation/zope.interface/issues/197>`_.
+
+
+5.0.1 (2020-03-21)
+==================
+
+- Ensure the resolution order for ``InterfaceClass`` is consistent.
+  See `issue 192 <https://github.com/zopefoundation/zope.interface/issues/192>`_.
+
+- Ensure the resolution order for ``collections.OrderedDict`` is
+  consistent on CPython 2. (It was already consistent on Python 3 and PyPy).
+
+- Fix the handling of the ``ZOPE_INTERFACE_STRICT_IRO`` environment
+  variable. Previously, ``ZOPE_INTERFACE_STRICT_RO`` was read, in
+  contrast with the documentation. See `issue 194
+  <https://github.com/zopefoundation/zope.interface/issues/194>`_.
+
+
+5.0.0 (2020-03-19)
+==================
+
+- Make an internal singleton object returned by APIs like
+  ``implementedBy`` and ``directlyProvidedBy`` immutable. Previously,
+  it was fully mutable and allowed changing its ``__bases___``. That
+  could potentially lead to wrong results in pathological corner
+  cases. See `issue 158
+  <https://github.com/zopefoundation/zope.interface/issues/158>`_.
+
+- Support the ``PURE_PYTHON`` environment variable at runtime instead
+  of just at wheel build time. A value of 0 forces the C extensions to
+  be used (even on PyPy) failing if they aren't present. Any other
+  value forces the Python implementation to be used, ignoring the C
+  extensions. See `PR 151 <https://github.com/zopefoundation/zope.interface/pull/151>`_.
+
+- Cache the result of ``__hash__`` method in ``InterfaceClass`` as a
+  speed optimization. The method is called very often (i.e several
+  hundred thousand times during Plone 5.2 startup). Because the hash value never
+  changes it can be cached. This improves test performance from 0.614s
+  down to 0.575s (1.07x faster). In a real world Plone case a reindex
+  index came down from 402s to 320s (1.26x faster). See `PR 156
+  <https://github.com/zopefoundation/zope.interface/pull/156>`_.
+
+- Change the C classes ``SpecificationBase`` and its subclass
+  ``ClassProvidesBase`` to store implementation attributes in their structures
+  instead of their instance dictionaries. This eliminates the use of
+  an undocumented private C API function, and helps make some
+  instances require less memory. See `PR 154 <https://github.com/zopefoundation/zope.interface/pull/154>`_.
+
+- Reduce memory usage in other ways based on observations of usage
+  patterns in Zope (3) and Plone code bases.
+
+  - Specifications with no dependents are common (more than 50%) so
+    avoid allocating a ``WeakKeyDictionary`` unless we need it.
+  - Likewise, tagged values are relatively rare, so don't allocate a
+    dictionary to hold them until they are used.
+  - Use ``__slots___`` or the C equivalent ``tp_members`` in more
+    common places. Note that this removes the ability to set arbitrary
+    instance variables on certain objects.
+    See `PR 155 <https://github.com/zopefoundation/zope.interface/pull/155>`_.
+
+  The changes in this release resulted in a 7% memory reduction after
+  loading about 6,000 modules that define about 2,200 interfaces.
+
+  .. caution::
+
+     Details of many private attributes have changed, and external use
+     of those private attributes may break. In particular, the
+     lifetime and default value of ``_v_attrs`` has changed.
+
+- Remove support for hashing uninitialized interfaces. This could only
+  be done by subclassing ``InterfaceClass``. This has generated a
+  warning since it was first added in 2011 (3.6.5). Please call the
+  ``InterfaceClass`` constructor or otherwise set the appropriate
+  fields in your subclass before attempting to hash or sort it. See
+  `issue 157 <https://github.com/zopefoundation/zope.interface/issues/157>`_.
+
+- Remove unneeded override of the ``__hash__`` method from
+  ``zope.interface.declarations.Implements``. Watching a reindex index
+  process in ZCatalog with on a Py-Spy after 10k samples the time for
+  ``.adapter._lookup`` was reduced from 27.5s to 18.8s (~1.5x faster).
+  Overall reindex index time shrunk from 369s to 293s (1.26x faster).
+  See `PR 161
+  <https://github.com/zopefoundation/zope.interface/pull/161>`_.
+
+- Make the Python implementation closer to the C implementation by
+  ignoring all exceptions, not just ``AttributeError``, during (parts
+  of) interface adaptation. See `issue 163
+  <https://github.com/zopefoundation/zope.interface/issues/163>`_.
+
+- Micro-optimization in ``.adapter._lookup`` , ``.adapter._lookupAll``
+  and ``.adapter._subscriptions``: By loading ``components.get`` into
+  a local variable before entering the loop a bytcode "LOAD_FAST 0
+  (components)" in the loop can be eliminated. In Plone, while running
+  all tests, average speedup of the "owntime" of ``_lookup`` is ~5x.
+  See `PR 167
+  <https://github.com/zopefoundation/zope.interface/pull/167>`_.
+
+- Add ``__all__`` declarations to all modules. This helps tools that
+  do auto-completion and documentation and results in less cluttered
+  results. Wildcard ("*") are not recommended and may be affected. See
+  `issue 153
+  <https://github.com/zopefoundation/zope.interface/issues/153>`_.
+
+- Fix ``verifyClass`` and ``verifyObject`` for builtin types like
+  ``dict`` that have methods taking an optional, unnamed argument with
+  no default value like ``dict.pop``. On PyPy3, the verification is
+  strict, but on PyPy2 (as on all versions of CPython) those methods
+  cannot be verified and are ignored. See `issue 118
+  <https://github.com/zopefoundation/zope.interface/issues/118>`_.
+
+- Update the common interfaces ``IEnumerableMapping``,
+  ``IExtendedReadMapping``, ``IExtendedWriteMapping``,
+  ``IReadSequence`` and ``IUniqueMemberWriteSequence`` to no longer
+  require methods that were removed from Python 3 on Python 3, such as
+  ``__setslice___``. Now, ``dict``, ``list`` and ``tuple`` properly
+  verify as ``IFullMapping``, ``ISequence`` and ``IReadSequence,``
+  respectively on all versions of Python.
+
+- Add human-readable ``__str___`` and ``__repr___`` to ``Attribute``
+  and ``Method``. These contain the name of the defining interface
+  and the attribute. For methods, it also includes the signature.
+
+- Change the error strings raised by ``verifyObject`` and
+  ``verifyClass``. They now include more human-readable information
+  and exclude extraneous lines and spaces. See `issue 170
+  <https://github.com/zopefoundation/zope.interface/issues/170>`_.
+
+  .. caution:: This will break consumers (such as doctests) that
+               depended on the exact error messages.
+
+- Make ``verifyObject`` and ``verifyClass`` report all errors, if the
+  candidate object has multiple detectable violations. Previously they
+  reported only the first error. See `issue
+  <https://github.com/zopefoundation/zope.interface/issues/171>`_.
+
+  Like the above, this will break consumers depending on the exact
+  output of error messages if more than one error is present.
+
+- Add ``zope.interface.common.collections``,
+  ``zope.interface.common.numbers``, and ``zope.interface.common.io``.
+  These modules define interfaces based on the ABCs defined in the
+  standard library ``collections.abc``, ``numbers`` and ``io``
+  modules, respectively. Importing these modules will make the
+  standard library concrete classes that are registered with those
+  ABCs declare the appropriate interface. See `issue 138
+  <https://github.com/zopefoundation/zope.interface/issues/138>`_.
+
+- Add ``zope.interface.common.builtins``. This module defines
+  interfaces of common builtin types, such as ``ITextString`` and
+  ``IByteString``, ``IDict``, etc. These interfaces extend the
+  appropriate interfaces from ``collections`` and ``numbers``, and the
+  standard library classes implement them after importing this module.
+  This is intended as a replacement for third-party packages like
+  `dolmen.builtins <https://pypi.org/project/dolmen.builtins/>`_.
+  See `issue 138 <https://github.com/zopefoundation/zope.interface/issues/138>`_.
+
+- Make ``providedBy()`` and ``implementedBy()`` respect ``super``
+  objects. For instance, if class ``Derived`` implements ``IDerived``
+  and extends ``Base`` which in turn implements ``IBase``, then
+  ``providedBy(super(Derived, derived))`` will return ``[IBase]``.
+  Previously it would have returned ``[IDerived]`` (in general, it
+  would previously have returned whatever would have been returned
+  without ``super``).
+
+  Along with this change, adapter registries will unpack ``super``
+  objects into their ``__self___`` before passing it to the factory.
+  Together, this means that ``component.getAdapter(super(Derived,
+  self), ITarget)`` is now meaningful.
+
+  See `issue 11 <https://github.com/zopefoundation/zope.interface/issues/11>`_.
+
+- Fix a potential interpreter crash in the low-level adapter
+  registry lookup functions. See issue 11.
+
+- Adopt Python's standard `C3 resolution order
+  <https://www.python.org/download/releases/2.3/mro/>`_ to compute the
+  ``__iro__`` and ``__sro__`` of interfaces, with tweaks to support
+  additional cases that are common in interfaces but disallowed for
+  Python classes. Previously, an ad-hoc ordering that made no
+  particular guarantees was used.
+
+  This has many beneficial properties, including the fact that base
+  interface and base classes tend to appear near the end of the
+  resolution order instead of the beginning. The resolution order in
+  general should be more predictable and consistent.
+
+  .. caution::
+     In some cases, especially with complex interface inheritance
+     trees or when manually providing or implementing interfaces, the
+     resulting IRO may be quite different. This may affect adapter
+     lookup.
+
+  The C3 order enforces some constraints in order to be able to
+  guarantee a sensible ordering. Older versions of zope.interface did
+  not impose similar constraints, so it was possible to create
+  interfaces and declarations that are inconsistent with the C3
+  constraints. In that event, zope.interface will still produce a
+  resolution order equal to the old order, but it won't be guaranteed
+  to be fully C3 compliant. In the future, strict enforcement of C3
+  order may be the default.
+
+  A set of environment variables and module constants allows
+  controlling several aspects of this new behaviour. It is possible to
+  request warnings about inconsistent resolution orders encountered,
+  and even to forbid them. Differences between the C3 resolution order
+  and the previous order can be logged, and, in extreme cases, the
+  previous order can still be used (this ability will be removed in
+  the future). For details, see the documentation for
+  ``zope.interface.ro``.
+
+- Make inherited tagged values in interfaces respect the resolution
+  order (``__iro__``), as method and attribute lookup does. Previously
+  tagged values could give inconsistent results. See `issue 190
+  <https://github.com/zopefoundation/zope.interface/issues/190>`_.
+
+- Add ``getDirectTaggedValue`` (and related methods) to interfaces to
+  allow accessing tagged values irrespective of inheritance. See
+  `issue 190
+  <https://github.com/zopefoundation/zope.interface/issues/190>`_.
+
+- Ensure that ``Interface`` is always the last item in the ``__iro__``
+  and ``__sro__``. This is usually the case, but if classes that do
+  not implement any interfaces are part of a class inheritance
+  hierarchy, ``Interface`` could be assigned too high a priority.
+  See `issue 8 <https://github.com/zopefoundation/zope.interface/issues/8>`_.
+
+- Implement sorting, equality, and hashing in C for ``Interface``
+  objects. In micro benchmarks, this makes those operations 40% to 80%
+  faster. This translates to a 20% speed up in querying adapters.
+
+  Note that this changes certain implementation details. In
+  particular, ``InterfaceClass`` now has a non-default metaclass, and
+  it is enforced that ``__module__`` in instances of
+  ``InterfaceClass`` is read-only.
+
+  See `PR 183 <https://github.com/zopefoundation/zope.interface/pull/183>`_.
+
+
+4.7.2 (2020-03-10)
+==================
+
+- Remove deprecated use of setuptools features.  See `issue 30
+  <https://github.com/zopefoundation/zope.interface/issues/30>`_.
+
+
+4.7.1 (2019-11-11)
+==================
+
+- Use Python 3 syntax in the documentation.  See `issue 119
+  <https://github.com/zopefoundation/zope.interface/issues/119>`_.
+
+
+4.7.0 (2019-11-11)
+==================
+
+- Drop support for Python 3.4.
+
+- Change ``queryTaggedValue``, ``getTaggedValue``,
+  ``getTaggedValueTags`` in interfaces. They now include inherited
+  values by following ``__bases__``. See `PR 144
+  <https://github.com/zopefoundation/zope.interface/pull/144>`_.
+
+  .. caution:: This may be a breaking change.
+
+- Add support for Python 3.8.
+
+
+4.6.0 (2018-10-23)
+==================
+
+- Add support for Python 3.7
+
+- Fix ``verifyObject`` for class objects with staticmethods on
+  Python 3. See `issue 126
+  <https://github.com/zopefoundation/zope.interface/issues/126>`_.
+
+
+4.5.0 (2018-04-19)
+==================
+
+- Drop support for 3.3, avoid accidental dependence breakage via setup.py.
+  See `PR 110 <https://github.com/zopefoundation/zope.interface/pull/110>`_.
+- Allow registering and unregistering instance methods as listeners.
+  See `issue 12 <https://github.com/zopefoundation/zope.interface/issues/12>`_
+  and `PR 102 <https://github.com/zopefoundation/zope.interface/pull/102>`_.
+- Synchronize and simplify zope/__init__.py. See `issue 114
+  <https://github.com/zopefoundation/zope.interface/issues/114>`_
+
+
+4.4.3 (2017-09-22)
+==================
+
+- Avoid exceptions when the ``__annotations__`` attribute is added to
+  interface definitions with Python 3.x type hints. See `issue 98
+  <https://github.com/zopefoundation/zope.interface/issues/98>`_.
+- Fix the possibility of a rare crash in the C extension when
+  deallocating items. See `issue 100
+  <https://github.com/zopefoundation/zope.interface/issues/100>`_.
+
+
+4.4.2 (2017-06-14)
+==================
+
+- Fix a regression storing
+  ``zope.component.persistentregistry.PersistentRegistry`` instances.
+  See `issue 85 <https://github.com/zopefoundation/zope.interface/issues/85>`_.
+
+- Fix a regression that could lead to the utility registration cache
+  of ``Components`` getting out of sync. See `issue 93
+  <https://github.com/zopefoundation/zope.interface/issues/93>`_.
+
+4.4.1 (2017-05-13)
+==================
+
+- Simplify the caching of utility-registration data. In addition to
+  simplification, avoids spurious test failures when checking for
+  leaks in tests with persistent registries. See `pull 84
+  <https://github.com/zopefoundation/zope.interface/pull/84>`_.
+
+- Raise ``ValueError`` when non-text names are passed to adapter registry
+  methods:  prevents corruption of lookup caches.
+
+4.4.0 (2017-04-21)
+==================
+
+- Avoid a warning from the C compiler.
+  (https://github.com/zopefoundation/zope.interface/issues/71)
+
+- Add support for Python 3.6.
+
+4.3.3 (2016-12-13)
+==================
+
+- Correct typos and ReST formatting errors in documentation.
+
+- Add API documentation for the adapter registry.
+
+- Ensure that the ``LICENSE.txt`` file is included in built wheels.
+
+- Fix C optimizations broken on Py3k.  See the Python bug at:
+  http://bugs.python.org/issue15657
+  (https://github.com/zopefoundation/zope.interface/issues/60)
+
+
+4.3.2 (2016-09-05)
+==================
+
+- Fix equality testing of ``implementedBy`` objects and proxies.
+  (https://github.com/zopefoundation/zope.interface/issues/55)
+
+
+4.3.1 (2016-08-31)
+==================
+
+- Support Components subclasses that are not hashable.
+  (https://github.com/zopefoundation/zope.interface/issues/53)
+
+
+4.3.0 (2016-08-31)
+==================
+
+- Add the ability to sort the objects returned by ``implementedBy``.
+  This is compatible with the way interface classes sort so they can
+  be used together in ordered containers like BTrees.
+  (https://github.com/zopefoundation/zope.interface/issues/42)
+
+- Make ``setuptools`` a hard dependency of ``setup.py``.
+  (https://github.com/zopefoundation/zope.interface/issues/13)
+
+- Change a linear algorithm (O(n)) in ``Components.registerUtility`` and
+  ``Components.unregisterUtility`` into a dictionary lookup (O(1)) for
+  hashable components. This substantially improves the time taken to
+  manipulate utilities in large registries at the cost of some
+  additional memory usage. (https://github.com/zopefoundation/zope.interface/issues/46)
+
+
+4.2.0 (2016-06-10)
+==================
+
+- Add support for Python 3.5
+
+- Drop support for Python 2.6 and 3.2.
+
+
+4.1.3 (2015-10-05)
+==================
+
+- Fix installation without a C compiler on Python 3.5
+  (https://github.com/zopefoundation/zope.interface/issues/24).
+
+
+4.1.2 (2014-12-27)
+==================
+
+- Add support for PyPy3.
+
+- Remove unittest assertions deprecated in Python3.x.
+
+- Add ``zope.interface.document.asReStructuredText``, which formats the
+  generated text for an interface using ReST double-backtick markers.
+
+
+4.1.1 (2014-03-19)
+==================
+
+- Add support for Python 3.4.
+
+
+4.1.0 (2014-02-05)
+==================
+
+- Update ``boostrap.py`` to version 2.2.
+
+- Add ``@named(name)`` declaration, that specifies the component name, so it
+  does not have to be passed in during registration.
+
+
+4.0.5 (2013-02-28)
+==================
+
+- Fix a bug where a decorated method caused false positive failures on
+  ``verifyClass()``.
+
+
+4.0.4 (2013-02-21)
+==================
+
+- Fix a bug that was revealed by porting zope.traversing. During a loop, the
+  loop body modified a weakref dict causing a ``RuntimeError`` error.
+
+4.0.3 (2012-12-31)
+==================
+
+- Fleshed out PyPI Trove classifiers.
+
+4.0.2 (2012-11-21)
+==================
+
+- Add support for Python 3.3.
+
+- Restored ability to install the package in the absence of ``setuptools``.
+
+- LP #1055223:  Fix test which depended on dictionary order and failed randomly
+  in Python 3.3.
+
+4.0.1 (2012-05-22)
+==================
+
+- Drop explicit ``DeprecationWarnings`` for "class advice" APIS (these
+  APIs are still deprecated under Python 2.x, and still raise an exception
+  under Python 3.x, but no longer cause a warning to be emitted under
+  Python 2.x).
+
+4.0.0 (2012-05-16)
+==================
+
+- Automated build of Sphinx HTML docs and running doctest snippets via tox.
+
+- Deprecate the "class advice" APIs from ``zope.interface.declarations``:
+  ``implements``, ``implementsOnly``, and ``classProvides``.  In their place,
+  prefer the equivalent class decorators: ``@implementer``,
+  ``@implementer_only``, and ``@provider``.  Code which uses the deprecated
+  APIs will not work as expected under Py3k.
+
+- Remove use of '2to3' and associated fixers when installing under Py3k.
+  The code is now in a "compatible subset" which supports Python 2.6, 2.7,
+  and 3.2, including PyPy 1.8 (the version compatible with the 2.7 language
+  spec).
+
+- Drop explicit support for Python 2.4 / 2.5 / 3.1.
+
+- Add support for PyPy.
+
+- Add support for continuous integration using ``tox`` and ``jenkins``.
+
+- Add 'setup.py dev' alias (runs ``setup.py develop`` plus installs
+  ``nose`` and ``coverage``).
+
+- Add 'setup.py docs' alias (installs ``Sphinx`` and dependencies).
+
+- Replace all unittest coverage previously accomplished via doctests with
+  unittests.  The doctests have been moved into a ``docs`` section, managed
+  as a Sphinx collection.
+
+- LP #910987:  Ensure that the semantics of the ``lookup`` method of
+  ``zope.interface.adapter.LookupBase`` are the same in both the C and
+  Python implementations.
+
+- LP #900906:  Avoid exceptions due to tne new ``__qualname__`` attribute
+  added in Python 3.3 (see PEP 3155 for rationale).  Thanks to Antoine
+  Pitrou for the patch.
+
+3.8.0 (2011-09-22)
+==================
+
+- New module ``zope.interface.registry``.  This is code moved from
+  ``zope.component.registry`` which implements a basic nonperistent component
+  registry as ``zope.interface.registry.Components``.  This class was moved
+  from ``zope.component`` to make porting systems (such as Pyramid) that rely
+  only on a basic component registry to Python 3 possible without needing to
+  port the entirety of the ``zope.component`` package.  Backwards
+  compatibility import shims have been left behind in ``zope.component``, so
+  this change will not break any existing code.
+
+- New ``tests_require`` dependency: ``zope.event`` to test events sent by
+  Components implementation.  The ``zope.interface`` package does not have a
+  hard dependency on ``zope.event``, but if ``zope.event`` is importable, it
+  will send component registration events when methods of an instance of
+  ``zope.interface.registry.Components`` are called.
+
+- New interfaces added to support ``zope.interface.registry.Components``
+  addition: ``ComponentLookupError``, ``Invalid``, ``IObjectEvent``,
+  ``ObjectEvent``, ``IComponentLookup``, ``IRegistration``,
+  ``IUtilityRegistration``, ``IAdapterRegistration``,
+  ``ISubscriptionAdapterRegistration``, ``IHandlerRegistration``,
+  ``IRegistrationEvent``, ``RegistrationEvent``, ``IRegistered``,
+  ``Registered``, ``IUnregistered``, ``Unregistered``,
+  ``IComponentRegistry``, and ``IComponents``.
+
+- No longer Python 2.4 compatible (tested under 2.5, 2.6, 2.7, and 3.2).
+
+3.7.0 (2011-08-13)
+==================
+
+- Move changes from 3.6.2 - 3.6.5 to a new 3.7.x release line.
+
+3.6.7 (2011-08-20)
+==================
+
+- Fix sporadic failures on x86-64 platforms in tests of rich comparisons
+  of interfaces.
+
+3.6.6 (2011-08-13)
+==================
+
+- LP #570942:  Now correctly compare interfaces  from different modules but
+  with the same names.
+
+  N.B.: This is a less intrusive / destabilizing fix than the one applied in
+  3.6.3:  we only fix the underlying cmp-alike function, rather than adding
+  the other "rich comparison" functions.
+
+- Revert to software as released with 3.6.1 for "stable" 3.6 release branch.
+
+3.6.5 (2011-08-11)
+==================
+
+- LP #811792:  work around buggy behavior in some subclasses of
+  ``zope.interface.interface.InterfaceClass``, which invoke ``__hash__``
+  before initializing ``__module__`` and ``__name__``.  The workaround
+  returns a fixed constant hash in such cases, and issues a ``UserWarning``.
+
+- LP #804832:  Under PyPy, ``zope.interface`` should not build its C
+  extension.  Also, prevent attempting to build it under Jython.
+
+- Add a tox.ini for easier xplatform testing.
+
+- Fix testing deprecation warnings issued when tested under Py3K.
+
+3.6.4 (2011-07-04)
+==================
+
+- LP 804951:  InterfaceClass instances were unhashable under Python 3.x.
+
+3.6.3 (2011-05-26)
+==================
+
+- LP #570942:  Now correctly compare interfaces  from different modules but
+  with the same names.
+
+3.6.2 (2011-05-17)
+==================
+
+- Moved detailed documentation out-of-line from PyPI page, linking instead to
+  http://docs.zope.org/zope.interface .
+
+- Fixes for small issues when running tests under Python 3.2 using
+  ``zope.testrunner``.
+
+- LP # 675064:  Specify return value type for C optimizations module init
+  under Python 3:  undeclared value caused warnings, and segfaults on some
+  64 bit architectures.
+
+- setup.py now raises RuntimeError if you don't have Distutils installed when
+  running under Python 3.
+
+3.6.1 (2010-05-03)
+==================
+
+- A non-ASCII character in the changelog made 3.6.0 uninstallable on
+  Python 3 systems with another default encoding than UTF-8.
+
+- Fix compiler warnings under GCC 4.3.3.
+
+3.6.0 (2010-04-29)
+==================
+
+- LP #185974:  Clear the cache used by ``Specificaton.get`` inside
+  ``Specification.changed``.  Thanks to Jacob Holm for the patch.
+
+- Add support for Python 3.1. Contributors:
+
+    Lennart Regebro
+    Martin v Loewis
+    Thomas Lotze
+    Wolfgang Schnerring
+
+  The 3.1 support is completely backwards compatible. However, the implements
+  syntax used under Python 2.X does not work under 3.X, since it depends on
+  how metaclasses are implemented and this has changed. Instead it now supports
+  a decorator syntax (also under Python 2.X)::
+
+    class Foo:
+        implements(IFoo)
+        ...
+
+  can now also be written::
+
+    @implementer(IFoo):
+    class Foo:
+        ...
+
+  There are 2to3 fixers available to do this change automatically in the
+  zope.fixers package.
+
+- Python 2.3 is no longer supported.
+
+
+3.5.4 (2009-12-23)
+==================
+
+- Use the standard Python doctest module instead of zope.testing.doctest, which
+  has been deprecated.
+
+
+3.5.3 (2009-12-08)
+==================
+
+- Fix an edge case: make providedBy() work when a class has '__provides__' in
+  its __slots__ (see http://thread.gmane.org/gmane.comp.web.zope.devel/22490)
+
+
+3.5.2 (2009-07-01)
+==================
+
+- BaseAdapterRegistry.unregister, unsubscribe: Remove empty portions of
+  the data structures when something is removed.  This avoids leaving
+  references to global objects (interfaces) that may be slated for
+  removal from the calling application.
+
+
+3.5.1 (2009-03-18)
+==================
+
+- verifyObject: use getattr instead of hasattr to test for object attributes
+  in order to let exceptions other than AttributeError raised by properties
+  propagate to the caller
+
+- Add Sphinx-based documentation building to the package buildout
+  configuration. Use the ``bin/docs`` command after buildout.
+
+- Improve package description a bit. Unify changelog entries formatting.
+
+- Change package's mailing list address to zope-dev at zope.org as
+  zope3-dev at zope.org is now retired.
+
+
+3.5.0 (2008-10-26)
+==================
+
+- Fix declaration of _zope_interface_coptimizations, it's not a top level
+  package.
+
+- Add a DocTestSuite for odd.py module, so their tests are run.
+
+- Allow to bootstrap on Jython.
+
+- Fix https://bugs.launchpad.net/zope3/3.3/+bug/98388: ISpecification
+  was missing a declaration for __iro__.
+
+- Add optional code optimizations support, which allows the building
+  of C code optimizations to fail (Jython).
+
+- Replace `_flatten` with a non-recursive implementation, effectively making
+  it 3x faster.
+
+
+3.4.1 (2007-10-02)
+==================
+
+- Fix a setup bug that prevented installation from source on systems
+  without setuptools.
+
+
+3.4.0 (2007-07-19)
+==================
+
+- Final release for 3.4.0.
+
+
+3.4.0b3 (2007-05-22)
+====================
+
+
+- When checking whether an object is already registered, use identity
+  comparison, to allow adding registering with picky custom comparison methods.
+
+
+3.3.0.1 (2007-01-03)
+====================
+
+- Made a reference to OverflowWarning, which disappeared in Python
+  2.5, conditional.
+
+
+3.3.0 (2007/01/03)
+==================
+
+New Features
+------------
+
+- Refactor the adapter-lookup algorithim to make it much simpler and faster.
+
+  Also, implement more of the adapter-lookup logic in C, making
+  debugging of application code easier, since there is less
+  infrastructre code to step through.
+
+- Treat objects without interface declarations as if they
+  declared that they provide ``zope.interface.Interface``.
+
+- Add a number of richer new adapter-registration interfaces
+  that provide greater control and introspection.
+
+- Add a new interface decorator to zope.interface that allows the
+  setting of tagged values on an interface at definition time (see
+  zope.interface.taggedValue).
+
+Bug Fixes
+---------
+
+- A bug in multi-adapter lookup sometimes caused incorrect adapters to
+  be returned.
+
+
+3.2.0.2 (2006-04-15)
+====================
+
+- Fix packaging bug:  'package_dir' must be a *relative* path.
+
+
+3.2.0.1 (2006-04-14)
+====================
+
+- Packaging change:  suppress inclusion of 'setup.cfg' in 'sdist' builds.
+
+
+3.2.0 (2006-01-05)
+==================
+
+- Corresponds to the version of the zope.interface package shipped as part of
+  the Zope 3.2.0 release.
+
+
+3.1.0 (2005-10-03)
+==================
+
+- Corresponds to the version of the zope.interface package shipped as part of
+  the Zope 3.1.0 release.
+
+- Made attribute resolution order consistent with component lookup order,
+  i.e. new-style class MRO semantics.
+
+- Deprecate 'isImplementedBy' and 'isImplementedByInstancesOf' APIs in
+  favor of 'implementedBy' and 'providedBy'.
+
+
+3.0.1 (2005-07-27)
+==================
+
+- Corresponds to the version of the zope.interface package shipped as part of
+  the Zope X3.0.1 release.
+
+- Fix a bug reported by James Knight, which caused adapter registries
+  to fail occasionally to reflect declaration changes.
+
+
+3.0.0 (2004-11-07)
+==================
+
+- Corresponds to the version of the zope.interface package shipped as part of
+  the Zope X3.0.0 release.
diff --git a/contrib/python/zope.interface/py2/.dist-info/top_level.txt b/contrib/python/zope.interface/py2/.dist-info/top_level.txt
new file mode 100644
index 00000000000..66179d49851
--- /dev/null
+++ b/contrib/python/zope.interface/py2/.dist-info/top_level.txt
@@ -0,0 +1 @@
+zope
diff --git a/contrib/python/zope.interface/py2/COPYRIGHT.txt b/contrib/python/zope.interface/py2/COPYRIGHT.txt
new file mode 100644
index 00000000000..79859e06010
--- /dev/null
+++ b/contrib/python/zope.interface/py2/COPYRIGHT.txt
@@ -0,0 +1 @@
+Zope Foundation and Contributors
\ No newline at end of file
diff --git a/contrib/python/zope.interface/py2/LICENSE.txt b/contrib/python/zope.interface/py2/LICENSE.txt
new file mode 100644
index 00000000000..e1f9ad7b3b4
--- /dev/null
+++ b/contrib/python/zope.interface/py2/LICENSE.txt
@@ -0,0 +1,44 @@
+Zope Public License (ZPL) Version 2.1
+
+A copyright notice accompanies this license document that identifies the
+copyright holders.
+
+This license has been certified as open source. It has also been designated as
+GPL compatible by the Free Software Foundation (FSF).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions in source code must retain the accompanying copyright
+notice, this list of conditions, and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the accompanying copyright
+notice, this list of conditions, and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Names of the copyright holders must not be used to endorse or promote
+products derived from this software without prior written permission from the
+copyright holders.
+
+4. The right to distribute this software or to use it for any purpose does not
+give you the right to use Servicemarks (sm) or Trademarks (tm) of the
+copyright
+holders. Use of them is covered by separate agreement with the copyright
+holders.
+
+5. If any files are modified, you must cause the modified files to carry
+prominent notices stating that you changed the files and the date of any
+change.
+
+Disclaimer
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/python/zope.interface/py2/README.rst b/contrib/python/zope.interface/py2/README.rst
new file mode 100644
index 00000000000..e8a18e905dc
--- /dev/null
+++ b/contrib/python/zope.interface/py2/README.rst
@@ -0,0 +1,31 @@
+====================
+ ``zope.interface``
+====================
+
+.. image:: https://img.shields.io/pypi/v/zope.interface.svg
+    :target: https://pypi.python.org/pypi/zope.interface/
+    :alt: Latest Version
+
+.. image:: https://img.shields.io/pypi/pyversions/zope.interface.svg
+        :target: https://pypi.org/project/zope.interface/
+        :alt: Supported Python versions
+
+.. image:: https://github.com/zopefoundation/zope.interface/actions/workflows/tests.yml/badge.svg
+        :target: https://github.com/zopefoundation/zope.interface/actions/workflows/tests.yml
+
+.. image:: https://readthedocs.org/projects/zopeinterface/badge/?version=latest
+        :target: https://zopeinterface.readthedocs.io/en/latest/
+        :alt: Documentation Status
+
+This package is intended to be independently reusable in any Python
+project. It is maintained by the `Zope Toolkit project
+<https://zopetoolkit.readthedocs.io/>`_.
+
+This package provides an implementation of "object interfaces" for Python.
+Interfaces are a mechanism for labeling objects as conforming to a given
+API or contract. So, this package can be considered as implementation of
+the `Design By Contract`_ methodology support in Python.
+
+.. _Design By Contract: http://en.wikipedia.org/wiki/Design_by_contract
+
+For detailed documentation, please see https://zopeinterface.readthedocs.io/en/latest/
diff --git a/contrib/python/zope.interface/py2/ya.make b/contrib/python/zope.interface/py2/ya.make
new file mode 100644
index 00000000000..70358bd2904
--- /dev/null
+++ b/contrib/python/zope.interface/py2/ya.make
@@ -0,0 +1,61 @@
+# Generated by devtools/yamaker (pypi).
+
+PY2_LIBRARY()
+
+VERSION(5.5.2)
+
+LICENSE(ZPL-2.1)
+
+PEERDIR(
+    contrib/python/setuptools
+)
+
+NO_COMPILER_WARNINGS()
+
+NO_LINT()
+
+SRCS(
+    zope/interface/_zope_interface_coptimizations.c
+)
+
+PY_REGISTER(
+    zope.interface._zope_interface_coptimizations
+)
+
+PY_SRCS(
+    TOP_LEVEL
+    zope/interface/__init__.py
+    zope/interface/_compat.py
+    zope/interface/_flatten.py
+    zope/interface/adapter.py
+    zope/interface/advice.py
+    zope/interface/common/__init__.py
+    zope/interface/common/builtins.py
+    zope/interface/common/collections.py
+    zope/interface/common/idatetime.py
+    zope/interface/common/interfaces.py
+    zope/interface/common/io.py
+    zope/interface/common/mapping.py
+    zope/interface/common/numbers.py
+    zope/interface/common/sequence.py
+    zope/interface/declarations.py
+    zope/interface/document.py
+    zope/interface/exceptions.py
+    zope/interface/interface.py
+    zope/interface/interfaces.py
+    zope/interface/registry.py
+    zope/interface/ro.py
+    zope/interface/verify.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/zope.interface/py2/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/zope.interface/py2/zope/interface/__init__.py b/contrib/python/zope.interface/py2/zope/interface/__init__.py
new file mode 100644
index 00000000000..3372103e61c
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/__init__.py
@@ -0,0 +1,96 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interfaces
+
+This package implements the Python "scarecrow" proposal.
+
+The package exports two objects, `Interface` and `Attribute` directly. It also
+exports several helper methods. Interface is used to create an interface with
+a class statement, as in:
+
+  class IMyInterface(Interface):
+    '''Interface documentation
+    '''
+
+    def meth(arg1, arg2):
+        '''Documentation for meth
+        '''
+
+    # Note that there is no self argument
+
+To find out what you can do with interfaces, see the interface
+interface, `IInterface` in the `interfaces` module.
+
+The package has several public modules:
+
+  o `declarations` provides utilities to declare interfaces on objects. It
+    also provides a wide range of helpful utilities that aid in managing
+    declared interfaces. Most of its public names are however imported here.
+
+  o `document` has a utility for documenting an interface as structured text.
+
+  o `exceptions` has the interface-defined exceptions
+
+  o `interfaces` contains a list of all public interfaces for this package.
+
+  o `verify` has utilities for verifying implementations of interfaces.
+
+See the module doc strings for more information.
+"""
+__docformat__ = 'restructuredtext'
+# pylint:disable=wrong-import-position,unused-import
+from zope.interface.interface import Interface
+from zope.interface.interface import _wire
+
+# Need to actually get the interface elements to implement the right interfaces
+_wire()
+del _wire
+
+from zope.interface.declarations import Declaration
+from zope.interface.declarations import alsoProvides
+from zope.interface.declarations import classImplements
+from zope.interface.declarations import classImplementsFirst
+from zope.interface.declarations import classImplementsOnly
+from zope.interface.declarations import classProvides
+from zope.interface.declarations import directlyProvidedBy
+from zope.interface.declarations import directlyProvides
+from zope.interface.declarations import implementedBy
+from zope.interface.declarations import implementer
+from zope.interface.declarations import implementer_only
+from zope.interface.declarations import implements
+from zope.interface.declarations import implementsOnly
+from zope.interface.declarations import moduleProvides
+from zope.interface.declarations import named
+from zope.interface.declarations import noLongerProvides
+from zope.interface.declarations import providedBy
+from zope.interface.declarations import provider
+
+from zope.interface.exceptions import Invalid
+
+from zope.interface.interface import Attribute
+from zope.interface.interface import interfacemethod
+from zope.interface.interface import invariant
+from zope.interface.interface import taggedValue
+
+# The following are to make spec pickles cleaner
+from zope.interface.declarations import Provides
+
+
+from zope.interface.interfaces import IInterfaceDeclaration
+
+moduleProvides(IInterfaceDeclaration)
+
+__all__ = ('Interface', 'Attribute') + tuple(IInterfaceDeclaration)
+
+assert all(k in globals() for k in __all__)
diff --git a/contrib/python/zope.interface/py2/zope/interface/_compat.py b/contrib/python/zope.interface/py2/zope/interface/_compat.py
new file mode 100644
index 00000000000..3587463c431
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/_compat.py
@@ -0,0 +1,170 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Support functions for dealing with differences in platforms, including Python
+versions and implementations.
+
+This file should have no imports from the rest of zope.interface because it is
+used during early bootstrapping.
+"""
+import os
+import sys
+import types
+
+if sys.version_info[0] < 3:
+
+    def _normalize_name(name):
+        if isinstance(name, basestring):
+            return unicode(name)
+        raise TypeError("name must be a regular or unicode string")
+
+    CLASS_TYPES = (type, types.ClassType)
+    STRING_TYPES = (basestring,)
+
+    _BUILTINS = '__builtin__'
+
+    PYTHON3 = False
+    PYTHON2 = True
+
+else:
+
+    def _normalize_name(name):
+        if isinstance(name, bytes):
+            name = str(name, 'ascii')
+        if isinstance(name, str):
+            return name
+        raise TypeError("name must be a string or ASCII-only bytes")
+
+    CLASS_TYPES = (type,)
+    STRING_TYPES = (str,)
+
+    _BUILTINS = 'builtins'
+
+    PYTHON3 = True
+    PYTHON2 = False
+
+PYPY = hasattr(sys, 'pypy_version_info')
+PYPY2 = PYTHON2 and PYPY
+
+def _skip_under_py3k(test_method):
+    import unittest
+    return unittest.skipIf(sys.version_info[0] >= 3, "Only on Python 2")(test_method)
+
+
+def _skip_under_py2(test_method):
+    import unittest
+    return unittest.skipIf(sys.version_info[0] < 3, "Only on Python 3")(test_method)
+
+
+def _c_optimizations_required():
+    """
+    Return a true value if the C optimizations are required.
+
+    This uses the ``PURE_PYTHON`` variable as documented in `_use_c_impl`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    require_c = pure_env == "0"
+    return require_c
+
+
+def _c_optimizations_available():
+    """
+    Return the C optimization module, if available, otherwise
+    a false value.
+
+    If the optimizations are required but not available, this
+    raises the ImportError.
+
+    This does not say whether they should be used or not.
+    """
+    catch = () if _c_optimizations_required() else (ImportError,)
+    try:
+        from zope.interface import _zope_interface_coptimizations as c_opt
+        return c_opt
+    except catch: # pragma: no cover (only Jython doesn't build extensions)
+        return False
+
+
+def _c_optimizations_ignored():
+    """
+    The opposite of `_c_optimizations_required`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    return pure_env is not None and pure_env != "0"
+
+
+def _should_attempt_c_optimizations():
+    """
+    Return a true value if we should attempt to use the C optimizations.
+
+    This takes into account whether we're on PyPy and the value of the
+    ``PURE_PYTHON`` environment variable, as defined in `_use_c_impl`.
+    """
+    is_pypy = hasattr(sys, 'pypy_version_info')
+
+    if _c_optimizations_required():
+        return True
+    if is_pypy:
+        return False
+    return not _c_optimizations_ignored()
+
+
+def _use_c_impl(py_impl, name=None, globs=None):
+    """
+    Decorator. Given an object implemented in Python, with a name like
+    ``Foo``, import the corresponding C implementation from
+    ``zope.interface._zope_interface_coptimizations`` with the name
+    ``Foo`` and use it instead.
+
+    If the ``PURE_PYTHON`` environment variable is set to any value
+    other than ``"0"``, or we're on PyPy, ignore the C implementation
+    and return the Python version. If the C implementation cannot be
+    imported, return the Python version. If ``PURE_PYTHON`` is set to
+    0, *require* the C implementation (let the ImportError propagate);
+    note that PyPy can import the C implementation in this case (and all
+    tests pass).
+
+    In all cases, the Python version is kept available. in the module
+    globals with the name ``FooPy`` and the name ``FooFallback`` (both
+    conventions have been used; the C implementation of some functions
+    looks for the ``Fallback`` version, as do some of the Sphinx
+    documents).
+
+    Example::
+
+        @_use_c_impl
+        class Foo(object):
+            ...
+    """
+    name = name or py_impl.__name__
+    globs = globs or sys._getframe(1).f_globals
+
+    def find_impl():
+        if not _should_attempt_c_optimizations():
+            return py_impl
+
+        c_opt = _c_optimizations_available()
+        if not c_opt: # pragma: no cover (only Jython doesn't build extensions)
+            return py_impl
+
+        __traceback_info__ = c_opt
+        return getattr(c_opt, name)
+
+    c_impl = find_impl()
+    # Always make available by the FooPy name and FooFallback
+    # name (for testing and documentation)
+    globs[name + 'Py'] = py_impl
+    globs[name + 'Fallback'] = py_impl
+
+    return c_impl
diff --git a/contrib/python/zope.interface/py2/zope/interface/_flatten.py b/contrib/python/zope.interface/py2/zope/interface/_flatten.py
new file mode 100644
index 00000000000..a80c2de49ab
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/_flatten.py
@@ -0,0 +1,35 @@
+##############################################################################
+#
+# Copyright (c) 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Adapter-style interface registry
+
+See Adapter class.
+"""
+from zope.interface import Declaration
+
+def _flatten(implements, include_None=0):
+
+    try:
+        r = implements.flattened()
+    except AttributeError:
+        if implements is None:
+            r=()
+        else:
+            r = Declaration(implements).flattened()
+
+    if not include_None:
+        return r
+
+    r = list(r)
+    r.append(None)
+    return r
diff --git a/contrib/python/zope.interface/py2/zope/interface/_zope_interface_coptimizations.c b/contrib/python/zope.interface/py2/zope/interface/_zope_interface_coptimizations.c
new file mode 100644
index 00000000000..af52a0aa730
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/_zope_interface_coptimizations.c
@@ -0,0 +1,2122 @@
+/*###########################################################################
+ #
+ # Copyright (c) 2003 Zope Foundation and Contributors.
+ # All Rights Reserved.
+ #
+ # This software is subject to the provisions of the Zope Public License,
+ # Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+ # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+ # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+ # FOR A PARTICULAR PURPOSE.
+ #
+ ############################################################################*/
+
+#include "Python.h"
+#include "structmember.h"
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-parameter"
+#pragma clang diagnostic ignored "-Wmissing-field-initializers"
+#endif
+
+#define TYPE(O) ((PyTypeObject*)(O))
+#define OBJECT(O) ((PyObject*)(O))
+#define CLASSIC(O) ((PyClassObject*)(O))
+#ifndef PyVarObject_HEAD_INIT
+#define PyVarObject_HEAD_INIT(a, b) PyObject_HEAD_INIT(a) b,
+#endif
+#ifndef Py_TYPE
+#define Py_TYPE(o) ((o)->ob_type)
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define PY3K
+#define PyNative_FromString PyUnicode_FromString
+#else
+#define PyNative_FromString PyString_FromString
+#endif
+
+static PyObject *str__dict__, *str__implemented__, *strextends;
+static PyObject *BuiltinImplementationSpecifications, *str__provides__;
+static PyObject *str__class__, *str__providedBy__;
+static PyObject *empty, *fallback;
+static PyObject *str__conform__, *str_call_conform, *adapter_hooks;
+static PyObject *str_uncached_lookup, *str_uncached_lookupAll;
+static PyObject *str_uncached_subscriptions;
+static PyObject *str_registry, *strro, *str_generation, *strchanged;
+static PyObject *str__self__;
+static PyObject *str__module__;
+static PyObject *str__name__;
+static PyObject *str__adapt__;
+static PyObject *str_CALL_CUSTOM_ADAPT;
+
+static PyTypeObject *Implements;
+
+static int imported_declarations = 0;
+
+static int
+import_declarations(void)
+{
+  PyObject *declarations, *i;
+
+  declarations = PyImport_ImportModule("zope.interface.declarations");
+  if (declarations == NULL)
+    return -1;
+
+  BuiltinImplementationSpecifications = PyObject_GetAttrString(
+                    declarations, "BuiltinImplementationSpecifications");
+  if (BuiltinImplementationSpecifications == NULL)
+    return -1;
+
+  empty = PyObject_GetAttrString(declarations, "_empty");
+  if (empty == NULL)
+    return -1;
+
+  fallback = PyObject_GetAttrString(declarations, "implementedByFallback");
+  if (fallback == NULL)
+    return -1;
+
+
+
+  i = PyObject_GetAttrString(declarations, "Implements");
+  if (i == NULL)
+    return -1;
+
+  if (! PyType_Check(i))
+    {
+      PyErr_SetString(PyExc_TypeError,
+                      "zope.interface.declarations.Implements is not a type");
+      return -1;
+    }
+
+  Implements = (PyTypeObject *)i;
+
+  Py_DECREF(declarations);
+
+  imported_declarations = 1;
+  return 0;
+}
+
+
+static PyTypeObject SpecificationBaseType;   /* Forward */
+
+static PyObject *
+implementedByFallback(PyObject *cls)
+{
+  if (imported_declarations == 0 && import_declarations() < 0)
+    return NULL;
+
+  return PyObject_CallFunctionObjArgs(fallback, cls, NULL);
+}
+
+static PyObject *
+implementedBy(PyObject *ignored, PyObject *cls)
+{
+  /* Fast retrieval of implements spec, if possible, to optimize
+     common case.  Use fallback code if we get stuck.
+  */
+
+  PyObject *dict = NULL, *spec;
+
+  if (PyObject_TypeCheck(cls, &PySuper_Type))
+  {
+      // Let merging be handled by Python.
+      return implementedByFallback(cls);
+  }
+
+  if (PyType_Check(cls))
+    {
+      dict = TYPE(cls)->tp_dict;
+      Py_XINCREF(dict);
+    }
+
+  if (dict == NULL)
+    dict = PyObject_GetAttr(cls, str__dict__);
+
+  if (dict == NULL)
+    {
+      /* Probably a security proxied class, use more expensive fallback code */
+      PyErr_Clear();
+      return implementedByFallback(cls);
+    }
+
+  spec = PyObject_GetItem(dict, str__implemented__);
+  Py_DECREF(dict);
+  if (spec)
+    {
+      if (imported_declarations == 0 && import_declarations() < 0)
+        return NULL;
+
+      if (PyObject_TypeCheck(spec, Implements))
+        return spec;
+
+      /* Old-style declaration, use more expensive fallback code */
+      Py_DECREF(spec);
+      return implementedByFallback(cls);
+    }
+
+  PyErr_Clear();
+
+  /* Maybe we have a builtin */
+  if (imported_declarations == 0 && import_declarations() < 0)
+    return NULL;
+
+  spec = PyDict_GetItem(BuiltinImplementationSpecifications, cls);
+  if (spec != NULL)
+    {
+      Py_INCREF(spec);
+      return spec;
+    }
+
+  /* We're stuck, use fallback */
+  return implementedByFallback(cls);
+}
+
+static PyObject *
+getObjectSpecification(PyObject *ignored, PyObject *ob)
+{
+  PyObject *cls, *result;
+
+  result = PyObject_GetAttr(ob, str__provides__);
+  if (!result)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          /* Propagate non AttributeError exceptions. */
+          return NULL;
+      }
+      PyErr_Clear();
+  }
+  else
+  {
+      int is_instance = -1;
+      is_instance = PyObject_IsInstance(result, (PyObject*)&SpecificationBaseType);
+      if (is_instance < 0)
+      {
+          /* Propagate all errors */
+          return NULL;
+      }
+      if (is_instance)
+      {
+          return result;
+      }
+  }
+
+  /* We do a getattr here so as not to be defeated by proxies */
+  cls = PyObject_GetAttr(ob, str__class__);
+  if (cls == NULL)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          /* Propagate non-AttributeErrors */
+          return NULL;
+      }
+      PyErr_Clear();
+      if (imported_declarations == 0 && import_declarations() < 0)
+          return NULL;
+
+      Py_INCREF(empty);
+      return empty;
+  }
+  result = implementedBy(NULL, cls);
+  Py_DECREF(cls);
+
+  return result;
+}
+
+static PyObject *
+providedBy(PyObject *ignored, PyObject *ob)
+{
+  PyObject *result, *cls, *cp;
+  int is_instance = -1;
+  result = NULL;
+
+  is_instance = PyObject_IsInstance(ob, (PyObject*)&PySuper_Type);
+  if (is_instance < 0)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          /* Propagate non-AttributeErrors */
+          return NULL;
+      }
+      PyErr_Clear();
+  }
+  if (is_instance)
+  {
+      return implementedBy(NULL, ob);
+  }
+
+  result = PyObject_GetAttr(ob, str__providedBy__);
+
+  if (result == NULL)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          return NULL;
+      }
+
+      PyErr_Clear();
+      return getObjectSpecification(NULL, ob);
+  }
+
+
+  /* We want to make sure we have a spec. We can't do a type check
+     because we may have a proxy, so we'll just try to get the
+     only attribute.
+  */
+  if (PyObject_TypeCheck(result, &SpecificationBaseType)
+      ||
+      PyObject_HasAttr(result, strextends)
+      )
+    return result;
+
+  /*
+    The object's class doesn't understand descriptors.
+    Sigh. We need to get an object descriptor, but we have to be
+    careful.  We want to use the instance's __provides__,l if
+    there is one, but only if it didn't come from the class.
+  */
+  Py_DECREF(result);
+
+  cls = PyObject_GetAttr(ob, str__class__);
+  if (cls == NULL)
+    return NULL;
+
+  result = PyObject_GetAttr(ob, str__provides__);
+  if (result == NULL)
+    {
+      /* No __provides__, so just fall back to implementedBy */
+      PyErr_Clear();
+      result = implementedBy(NULL, cls);
+      Py_DECREF(cls);
+      return result;
+    }
+
+  cp = PyObject_GetAttr(cls, str__provides__);
+  if (cp == NULL)
+    {
+      /* The the class has no provides, assume we're done: */
+      PyErr_Clear();
+      Py_DECREF(cls);
+      return result;
+    }
+
+  if (cp == result)
+    {
+      /*
+        Oops, we got the provides from the class. This means
+        the object doesn't have it's own. We should use implementedBy
+      */
+      Py_DECREF(result);
+      result = implementedBy(NULL, cls);
+    }
+
+  Py_DECREF(cls);
+  Py_DECREF(cp);
+
+  return result;
+}
+
+typedef struct {
+    PyObject_HEAD
+    PyObject* weakreflist;
+    /*
+      In the past, these fields were stored in the __dict__
+      and were technically allowed to contain any Python object, though
+      other type checks would fail or fall back to generic code paths if
+      they didn't have the expected type. We preserve that behaviour and don't
+      make any assumptions about contents.
+    */
+    PyObject* _implied;
+    /*
+      The remainder aren't used in C code but must be stored here
+      to prevent instance layout conflicts.
+    */
+    PyObject* _dependents;
+    PyObject* _bases;
+    PyObject* _v_attrs;
+    PyObject* __iro__;
+    PyObject* __sro__;
+} Spec;
+
+/*
+  We know what the fields are *supposed* to define, but
+  they could have anything, so we need to traverse them.
+*/
+static int
+Spec_traverse(Spec* self, visitproc visit, void* arg)
+{
+    Py_VISIT(self->_implied);
+    Py_VISIT(self->_dependents);
+    Py_VISIT(self->_bases);
+    Py_VISIT(self->_v_attrs);
+    Py_VISIT(self->__iro__);
+    Py_VISIT(self->__sro__);
+    return 0;
+}
+
+static int
+Spec_clear(Spec* self)
+{
+    Py_CLEAR(self->_implied);
+    Py_CLEAR(self->_dependents);
+    Py_CLEAR(self->_bases);
+    Py_CLEAR(self->_v_attrs);
+    Py_CLEAR(self->__iro__);
+    Py_CLEAR(self->__sro__);
+    return 0;
+}
+
+static void
+Spec_dealloc(Spec* self)
+{
+    /* PyType_GenericAlloc that you get when you don't
+       specify a tp_alloc always tracks the object. */
+    PyObject_GC_UnTrack((PyObject *)self);
+    if (self->weakreflist != NULL) {
+        PyObject_ClearWeakRefs(OBJECT(self));
+    }
+    Spec_clear(self);
+    Py_TYPE(self)->tp_free(OBJECT(self));
+}
+
+static PyObject *
+Spec_extends(Spec *self, PyObject *other)
+{
+  PyObject *implied;
+
+  implied = self->_implied;
+  if (implied == NULL) {
+    return NULL;
+  }
+
+  if (PyDict_GetItem(implied, other) != NULL)
+    Py_RETURN_TRUE;
+  Py_RETURN_FALSE;
+}
+
+static char Spec_extends__doc__[] =
+"Test whether a specification is or extends another"
+;
+
+static char Spec_providedBy__doc__[] =
+"Test whether an interface is implemented by the specification"
+;
+
+static PyObject *
+Spec_call(Spec *self, PyObject *args, PyObject *kw)
+{
+  PyObject *spec;
+
+  if (! PyArg_ParseTuple(args, "O", &spec))
+    return NULL;
+  return Spec_extends(self, spec);
+}
+
+static PyObject *
+Spec_providedBy(PyObject *self, PyObject *ob)
+{
+  PyObject *decl, *item;
+
+  decl = providedBy(NULL, ob);
+  if (decl == NULL)
+    return NULL;
+
+  if (PyObject_TypeCheck(decl, &SpecificationBaseType))
+    item = Spec_extends((Spec*)decl, self);
+  else
+    /* decl is probably a security proxy.  We have to go the long way
+       around.
+    */
+    item = PyObject_CallFunctionObjArgs(decl, self, NULL);
+
+  Py_DECREF(decl);
+  return item;
+}
+
+
+static char Spec_implementedBy__doc__[] =
+"Test whether the specification is implemented by a class or factory.\n"
+"Raise TypeError if argument is neither a class nor a callable."
+;
+
+static PyObject *
+Spec_implementedBy(PyObject *self, PyObject *cls)
+{
+  PyObject *decl, *item;
+
+  decl = implementedBy(NULL, cls);
+  if (decl == NULL)
+    return NULL;
+
+  if (PyObject_TypeCheck(decl, &SpecificationBaseType))
+    item = Spec_extends((Spec*)decl, self);
+  else
+    item = PyObject_CallFunctionObjArgs(decl, self, NULL);
+
+  Py_DECREF(decl);
+  return item;
+}
+
+static struct PyMethodDef Spec_methods[] = {
+        {"providedBy",
+         (PyCFunction)Spec_providedBy,          METH_O,
+         Spec_providedBy__doc__},
+        {"implementedBy",
+         (PyCFunction)Spec_implementedBy,       METH_O,
+         Spec_implementedBy__doc__},
+        {"isOrExtends", (PyCFunction)Spec_extends,      METH_O,
+         Spec_extends__doc__},
+
+        {NULL,          NULL}           /* sentinel */
+};
+
+static PyMemberDef Spec_members[] = {
+  {"_implied", T_OBJECT_EX, offsetof(Spec, _implied), 0, ""},
+  {"_dependents", T_OBJECT_EX, offsetof(Spec, _dependents), 0, ""},
+  {"_bases", T_OBJECT_EX, offsetof(Spec, _bases), 0, ""},
+  {"_v_attrs", T_OBJECT_EX, offsetof(Spec, _v_attrs), 0, ""},
+  {"__iro__", T_OBJECT_EX, offsetof(Spec, __iro__), 0, ""},
+  {"__sro__", T_OBJECT_EX, offsetof(Spec, __sro__), 0, ""},
+  {NULL},
+};
+
+
+static PyTypeObject SpecificationBaseType = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_interface_coptimizations."
+                                "SpecificationBase",
+        /* tp_basicsize      */ sizeof(Spec),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)Spec_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)Spec_call,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+        "Base type for Specification objects",
+        /* tp_traverse       */ (traverseproc)Spec_traverse,
+        /* tp_clear          */ (inquiry)Spec_clear,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ offsetof(Spec, weakreflist),
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ Spec_methods,
+        /* tp_members        */ Spec_members,
+};
+
+static PyObject *
+OSD_descr_get(PyObject *self, PyObject *inst, PyObject *cls)
+{
+  PyObject *provides;
+
+  if (inst == NULL)
+    return getObjectSpecification(NULL, cls);
+
+  provides = PyObject_GetAttr(inst, str__provides__);
+  /* Return __provides__ if we got it, or return NULL and propagate non-AttributeError. */
+  if (provides != NULL || !PyErr_ExceptionMatches(PyExc_AttributeError))
+    return provides;
+
+  PyErr_Clear();
+  return implementedBy(NULL, cls);
+}
+
+static PyTypeObject OSDType = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_interface_coptimizations."
+                                "ObjectSpecificationDescriptor",
+        /* tp_basicsize      */ 0,
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)0,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)0,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT
+                                | Py_TPFLAGS_BASETYPE ,
+        "Object Specification Descriptor",
+        /* tp_traverse       */ (traverseproc)0,
+        /* tp_clear          */ (inquiry)0,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ 0,
+        /* tp_members        */ 0,
+        /* tp_getset         */ 0,
+        /* tp_base           */ 0,
+        /* tp_dict           */ 0, /* internal use */
+        /* tp_descr_get      */ (descrgetfunc)OSD_descr_get,
+};
+
+typedef struct {
+    Spec spec;
+    /* These members are handled generically, as for Spec members. */
+    PyObject* _cls;
+    PyObject* _implements;
+} CPB;
+
+static PyObject *
+CPB_descr_get(CPB *self, PyObject *inst, PyObject *cls)
+{
+  PyObject *implements;
+
+  if (self->_cls == NULL)
+    return NULL;
+
+  if (cls == self->_cls)
+    {
+      if (inst == NULL)
+        {
+          Py_INCREF(self);
+          return OBJECT(self);
+        }
+
+      implements = self->_implements;
+      Py_XINCREF(implements);
+      return implements;
+    }
+
+  PyErr_SetObject(PyExc_AttributeError, str__provides__);
+  return NULL;
+}
+
+static int
+CPB_traverse(CPB* self, visitproc visit, void* arg)
+{
+    Py_VISIT(self->_cls);
+    Py_VISIT(self->_implements);
+    return Spec_traverse((Spec*)self, visit, arg);
+}
+
+static int
+CPB_clear(CPB* self)
+{
+    Py_CLEAR(self->_cls);
+    Py_CLEAR(self->_implements);
+    Spec_clear((Spec*)self);
+    return 0;
+}
+
+static void
+CPB_dealloc(CPB* self)
+{
+    PyObject_GC_UnTrack((PyObject *)self);
+    CPB_clear(self);
+    Spec_dealloc((Spec*)self);
+}
+
+static PyMemberDef CPB_members[] = {
+  {"_cls", T_OBJECT_EX, offsetof(CPB, _cls), 0, "Defining class."},
+  {"_implements", T_OBJECT_EX, offsetof(CPB, _implements), 0, "Result of implementedBy."},
+  {NULL}
+};
+
+static PyTypeObject CPBType = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_interface_coptimizations."
+                                "ClassProvidesBase",
+        /* tp_basicsize      */ sizeof(CPB),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)CPB_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)0,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+        "C Base class for ClassProvides",
+        /* tp_traverse       */ (traverseproc)CPB_traverse,
+        /* tp_clear          */ (inquiry)CPB_clear,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ 0,
+        /* tp_members        */ CPB_members,
+        /* tp_getset         */ 0,
+        /* tp_base           */ &SpecificationBaseType,
+        /* tp_dict           */ 0, /* internal use */
+        /* tp_descr_get      */ (descrgetfunc)CPB_descr_get,
+        /* tp_descr_set      */ 0,
+        /* tp_dictoffset     */ 0,
+        /* tp_init           */ 0,
+        /* tp_alloc          */ 0,
+        /* tp_new            */ 0,
+};
+
+/* ==================================================================== */
+/* ========== Begin: __call__ and __adapt__ =========================== */
+
+/*
+    def __adapt__(self, obj):
+        """Adapt an object to the receiver
+        """
+        if self.providedBy(obj):
+            return obj
+
+        for hook in adapter_hooks:
+            adapter = hook(self, obj)
+            if adapter is not None:
+                return adapter
+
+
+*/
+static PyObject *
+__adapt__(PyObject *self, PyObject *obj)
+{
+  PyObject *decl, *args, *adapter;
+  int implements, i, l;
+
+  decl = providedBy(NULL, obj);
+  if (decl == NULL)
+    return NULL;
+
+  if (PyObject_TypeCheck(decl, &SpecificationBaseType))
+    {
+      PyObject *implied;
+
+      implied = ((Spec*)decl)->_implied;
+      if (implied == NULL)
+        {
+          Py_DECREF(decl);
+          return NULL;
+        }
+
+      implements = PyDict_GetItem(implied, self) != NULL;
+      Py_DECREF(decl);
+    }
+  else
+    {
+      /* decl is probably a security proxy.  We have to go the long way
+         around.
+      */
+      PyObject *r;
+      r = PyObject_CallFunctionObjArgs(decl, self, NULL);
+      Py_DECREF(decl);
+      if (r == NULL)
+        return NULL;
+      implements = PyObject_IsTrue(r);
+      Py_DECREF(r);
+    }
+
+  if (implements)
+    {
+      Py_INCREF(obj);
+      return obj;
+    }
+
+  l = PyList_GET_SIZE(adapter_hooks);
+  args = PyTuple_New(2);
+  if (args == NULL)
+    return NULL;
+  Py_INCREF(self);
+  PyTuple_SET_ITEM(args, 0, self);
+  Py_INCREF(obj);
+  PyTuple_SET_ITEM(args, 1, obj);
+  for (i = 0; i < l; i++)
+    {
+      adapter = PyObject_CallObject(PyList_GET_ITEM(adapter_hooks, i), args);
+      if (adapter == NULL || adapter != Py_None)
+        {
+          Py_DECREF(args);
+          return adapter;
+        }
+      Py_DECREF(adapter);
+    }
+
+  Py_DECREF(args);
+
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+#ifndef PY3K
+typedef long Py_hash_t;
+#endif
+
+typedef struct {
+    Spec spec;
+    PyObject* __name__;
+    PyObject* __module__;
+    Py_hash_t _v_cached_hash;
+} IB;
+
+static struct PyMethodDef ib_methods[] = {
+  {"__adapt__", (PyCFunction)__adapt__, METH_O,
+   "Adapt an object to the receiver"},
+  {NULL,                NULL}           /* sentinel */
+};
+
+/*
+    def __call__(self, obj, alternate=_marker):
+        try:
+            conform = obj.__conform__
+        except AttributeError: # pylint:disable=bare-except
+            conform = None
+
+        if conform is not None:
+            adapter = self._call_conform(conform)
+            if adapter is not None:
+                return adapter
+
+        adapter = self.__adapt__(obj)
+
+        if adapter is not None:
+            return adapter
+        if alternate is not _marker:
+            return alternate
+        raise TypeError("Could not adapt", obj, self)
+
+*/
+static PyObject *
+IB_call(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+  PyObject *conform, *obj, *alternate, *adapter;
+  static char *kwlist[] = {"obj", "alternate", NULL};
+  conform = obj = alternate = adapter = NULL;
+
+
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", kwlist,
+                                   &obj, &alternate))
+    return NULL;
+
+  conform = PyObject_GetAttr(obj, str__conform__);
+  if (conform == NULL)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          /* Propagate non-AttributeErrors */
+          return NULL;
+      }
+      PyErr_Clear();
+
+      Py_INCREF(Py_None);
+      conform = Py_None;
+  }
+
+  if (conform != Py_None)
+  {
+      adapter = PyObject_CallMethodObjArgs(self, str_call_conform,
+                                           conform, NULL);
+      Py_DECREF(conform);
+      if (adapter == NULL || adapter != Py_None)
+          return adapter;
+      Py_DECREF(adapter);
+  }
+  else
+  {
+      Py_DECREF(conform);
+  }
+
+  /* We differ from the Python code here. For speed, instead of always calling
+     self.__adapt__(), we check to see if the type has defined it. Checking in
+     the dict for __adapt__ isn't sufficient because there's no cheap way to
+     tell if it's the __adapt__ that InterfaceBase itself defines (our type
+     will *never* be InterfaceBase, we're always subclassed by
+     InterfaceClass). Instead, we cooperate with InterfaceClass in Python to
+     set a flag in a new subclass when this is necessary. */
+  if (PyDict_GetItem(self->ob_type->tp_dict, str_CALL_CUSTOM_ADAPT))
+  {
+      /* Doesn't matter what the value is. Simply being present is enough. */
+      adapter = PyObject_CallMethodObjArgs(self, str__adapt__, obj, NULL);
+  }
+  else
+  {
+      adapter = __adapt__(self, obj);
+  }
+
+  if (adapter == NULL || adapter != Py_None)
+  {
+      return adapter;
+  }
+  Py_DECREF(adapter);
+
+  if (alternate != NULL)
+  {
+      Py_INCREF(alternate);
+      return alternate;
+  }
+
+  adapter = Py_BuildValue("sOO", "Could not adapt", obj, self);
+  if (adapter != NULL)
+  {
+      PyErr_SetObject(PyExc_TypeError, adapter);
+      Py_DECREF(adapter);
+  }
+  return NULL;
+}
+
+
+static int
+IB_traverse(IB* self, visitproc visit, void* arg)
+{
+    Py_VISIT(self->__name__);
+    Py_VISIT(self->__module__);
+    return Spec_traverse((Spec*)self, visit, arg);
+}
+
+static int
+IB_clear(IB* self)
+{
+    Py_CLEAR(self->__name__);
+    Py_CLEAR(self->__module__);
+    return Spec_clear((Spec*)self);
+}
+
+static void
+IB_dealloc(IB* self)
+{
+    PyObject_GC_UnTrack((PyObject *)self);
+    IB_clear(self);
+    Spec_dealloc((Spec*)self);
+}
+
+static PyMemberDef IB_members[] = {
+  {"__name__", T_OBJECT_EX, offsetof(IB, __name__), 0, ""},
+  // The redundancy between __module__ and __ibmodule__ is because
+  // __module__ is often shadowed by subclasses.
+  {"__module__", T_OBJECT_EX, offsetof(IB, __module__), READONLY, ""},
+  {"__ibmodule__", T_OBJECT_EX, offsetof(IB, __module__), 0, ""},
+  {NULL}
+};
+
+static Py_hash_t
+IB_hash(IB* self)
+{
+    PyObject* tuple;
+    if (!self->__module__) {
+        PyErr_SetString(PyExc_AttributeError, "__module__");
+        return -1;
+    }
+    if (!self->__name__) {
+        PyErr_SetString(PyExc_AttributeError, "__name__");
+        return -1;
+    }
+
+    if (self->_v_cached_hash) {
+        return self->_v_cached_hash;
+    }
+
+    tuple = PyTuple_Pack(2, self->__name__, self->__module__);
+    if (!tuple) {
+        return -1;
+    }
+    self->_v_cached_hash = PyObject_Hash(tuple);
+    Py_CLEAR(tuple);
+    return self->_v_cached_hash;
+}
+
+static PyTypeObject InterfaceBaseType;
+
+static PyObject*
+IB_richcompare(IB* self, PyObject* other, int op)
+{
+    PyObject* othername;
+    PyObject* othermod;
+    PyObject* oresult;
+    IB* otherib;
+    int result;
+
+    otherib = NULL;
+    oresult = othername = othermod = NULL;
+
+    if (OBJECT(self) == other) {
+        switch(op) {
+        case Py_EQ:
+        case Py_LE:
+        case Py_GE:
+            Py_RETURN_TRUE;
+            break;
+        case Py_NE:
+            Py_RETURN_FALSE;
+        }
+    }
+
+    if (other == Py_None) {
+        switch(op) {
+        case Py_LT:
+        case Py_LE:
+        case Py_NE:
+            Py_RETURN_TRUE;
+        default:
+            Py_RETURN_FALSE;
+        }
+    }
+
+    if (PyObject_TypeCheck(other, &InterfaceBaseType)) {
+        // This branch borrows references. No need to clean
+        // up if otherib is not null.
+        otherib = (IB*)other;
+        othername = otherib->__name__;
+        othermod = otherib->__module__;
+    }
+    else {
+        othername = PyObject_GetAttrString(other, "__name__");
+        if (othername) {
+            othermod = PyObject_GetAttrString(other, "__module__");
+        }
+        if (!othername || !othermod) {
+            if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_AttributeError)) {
+                PyErr_Clear();
+                oresult = Py_NotImplemented;
+            }
+            goto cleanup;
+        }
+    }
+#if 0
+// This is the simple, straightforward version of what Python does.
+    PyObject* pt1 = PyTuple_Pack(2, self->__name__, self->__module__);
+    PyObject* pt2 = PyTuple_Pack(2, othername, othermod);
+    oresult = PyObject_RichCompare(pt1, pt2, op);
+#endif
+
+    // tuple comparison is decided by the first non-equal element.
+    result = PyObject_RichCompareBool(self->__name__, othername, Py_EQ);
+    if (result == 0) {
+        result = PyObject_RichCompareBool(self->__name__, othername, op);
+    }
+    else if (result == 1) {
+        result = PyObject_RichCompareBool(self->__module__, othermod, op);
+    }
+    // If either comparison failed, we have an error set.
+    // Leave oresult NULL so we raise it.
+    if (result == -1) {
+        goto cleanup;
+    }
+
+    oresult = result ? Py_True : Py_False;
+
+
+cleanup:
+    Py_XINCREF(oresult);
+
+    if (!otherib) {
+        Py_XDECREF(othername);
+        Py_XDECREF(othermod);
+    }
+    return oresult;
+
+}
+
+static int
+IB_init(IB* self, PyObject* args, PyObject* kwargs)
+{
+    static char *kwlist[] = {"__name__", "__module__", NULL};
+    PyObject* module = NULL;
+    PyObject* name = NULL;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OO:InterfaceBase.__init__", kwlist,
+                                      &name, &module)) {
+        return -1;
+    }
+    IB_clear(self);
+    self->__module__ = module ? module : Py_None;
+    Py_INCREF(self->__module__);
+    self->__name__ = name ? name : Py_None;
+    Py_INCREF(self->__name__);
+    return 0;
+}
+
+
+static PyTypeObject InterfaceBaseType = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_zope_interface_coptimizations."
+                                "InterfaceBase",
+        /* tp_basicsize      */ sizeof(IB),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)IB_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)IB_hash,
+        /* tp_call           */ (ternaryfunc)IB_call,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT
+                                | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+        /* tp_doc */ "Interface base type providing __call__ and __adapt__",
+        /* tp_traverse       */ (traverseproc)IB_traverse,
+        /* tp_clear          */ (inquiry)IB_clear,
+        /* tp_richcompare    */ (richcmpfunc)IB_richcompare,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ ib_methods,
+        /* tp_members        */ IB_members,
+        /* tp_getset         */ 0,
+        /* tp_base           */ &SpecificationBaseType,
+        /* tp_dict           */ 0,
+        /* tp_descr_get      */ 0,
+        /* tp_descr_set      */ 0,
+        /* tp_dictoffset     */ 0,
+        /* tp_init           */ (initproc)IB_init,
+};
+
+/* =================== End: __call__ and __adapt__ ==================== */
+/* ==================================================================== */
+
+/* ==================================================================== */
+/* ========================== Begin: Lookup Bases ===================== */
+
+typedef struct {
+  PyObject_HEAD
+  PyObject *_cache;
+  PyObject *_mcache;
+  PyObject *_scache;
+} lookup;
+
+typedef struct {
+  PyObject_HEAD
+  PyObject *_cache;
+  PyObject *_mcache;
+  PyObject *_scache;
+  PyObject *_verify_ro;
+  PyObject *_verify_generations;
+} verify;
+
+static int
+lookup_traverse(lookup *self, visitproc visit, void *arg)
+{
+  int vret;
+
+  if (self->_cache) {
+    vret = visit(self->_cache, arg);
+    if (vret != 0)
+      return vret;
+  }
+
+  if (self->_mcache) {
+    vret = visit(self->_mcache, arg);
+    if (vret != 0)
+      return vret;
+  }
+
+  if (self->_scache) {
+    vret = visit(self->_scache, arg);
+    if (vret != 0)
+      return vret;
+  }
+
+  return 0;
+}
+
+static int
+lookup_clear(lookup *self)
+{
+  Py_CLEAR(self->_cache);
+  Py_CLEAR(self->_mcache);
+  Py_CLEAR(self->_scache);
+  return 0;
+}
+
+static void
+lookup_dealloc(lookup *self)
+{
+  PyObject_GC_UnTrack((PyObject *)self);
+  lookup_clear(self);
+  Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+/*
+    def changed(self, ignored=None):
+        self._cache.clear()
+        self._mcache.clear()
+        self._scache.clear()
+*/
+static PyObject *
+lookup_changed(lookup *self, PyObject *ignored)
+{
+  lookup_clear(self);
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+#define ASSURE_DICT(N) if (N == NULL) { N = PyDict_New(); \
+                                        if (N == NULL) return NULL; \
+                                       }
+
+/*
+    def _getcache(self, provided, name):
+        cache = self._cache.get(provided)
+        if cache is None:
+            cache = {}
+            self._cache[provided] = cache
+        if name:
+            c = cache.get(name)
+            if c is None:
+                c = {}
+                cache[name] = c
+            cache = c
+        return cache
+*/
+static PyObject *
+_subcache(PyObject *cache, PyObject *key)
+{
+  PyObject *subcache;
+
+  subcache = PyDict_GetItem(cache, key);
+  if (subcache == NULL)
+    {
+      int status;
+
+      subcache = PyDict_New();
+      if (subcache == NULL)
+        return NULL;
+      status = PyDict_SetItem(cache, key, subcache);
+      Py_DECREF(subcache);
+      if (status < 0)
+        return NULL;
+    }
+
+  return subcache;
+}
+static PyObject *
+_getcache(lookup *self, PyObject *provided, PyObject *name)
+{
+  PyObject *cache;
+
+  ASSURE_DICT(self->_cache);
+  cache = _subcache(self->_cache, provided);
+  if (cache == NULL)
+    return NULL;
+
+  if (name != NULL && PyObject_IsTrue(name))
+    cache = _subcache(cache, name);
+
+  return cache;
+}
+
+
+/*
+    def lookup(self, required, provided, name=u'', default=None):
+        cache = self._getcache(provided, name)
+        if len(required) == 1:
+            result = cache.get(required[0], _not_in_mapping)
+        else:
+            result = cache.get(tuple(required), _not_in_mapping)
+
+        if result is _not_in_mapping:
+            result = self._uncached_lookup(required, provided, name)
+            if len(required) == 1:
+                cache[required[0]] = result
+            else:
+                cache[tuple(required)] = result
+
+        if result is None:
+            return default
+
+        return result
+*/
+
+static PyObject *
+_lookup(lookup *self,
+        PyObject *required, PyObject *provided, PyObject *name,
+        PyObject *default_)
+{
+  PyObject *result, *key, *cache;
+  result = key = cache = NULL;
+#ifdef PY3K
+  if ( name && !PyUnicode_Check(name) )
+#else
+  if ( name && !PyString_Check(name) && !PyUnicode_Check(name) )
+#endif
+  {
+    PyErr_SetString(PyExc_ValueError,
+                    "name is not a string or unicode");
+    return NULL;
+  }
+
+  /* If `required` is a lazy sequence, it could have arbitrary side-effects,
+     such as clearing our caches. So we must not retrieve the cache until
+     after resolving it. */
+  required = PySequence_Tuple(required);
+  if (required == NULL)
+    return NULL;
+
+
+  cache = _getcache(self, provided, name);
+  if (cache == NULL)
+    return NULL;
+
+  if (PyTuple_GET_SIZE(required) == 1)
+    key = PyTuple_GET_ITEM(required, 0);
+  else
+    key = required;
+
+  result = PyDict_GetItem(cache, key);
+  if (result == NULL)
+    {
+      int status;
+
+      result = PyObject_CallMethodObjArgs(OBJECT(self), str_uncached_lookup,
+                                          required, provided, name, NULL);
+      if (result == NULL)
+        {
+          Py_DECREF(required);
+          return NULL;
+        }
+      status = PyDict_SetItem(cache, key, result);
+      Py_DECREF(required);
+      if (status < 0)
+        {
+          Py_DECREF(result);
+          return NULL;
+        }
+    }
+  else
+    {
+      Py_INCREF(result);
+      Py_DECREF(required);
+    }
+
+  if (result == Py_None && default_ != NULL)
+    {
+      Py_DECREF(Py_None);
+      Py_INCREF(default_);
+      return default_;
+    }
+
+  return result;
+}
+static PyObject *
+lookup_lookup(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", "name", "default", NULL};
+  PyObject *required, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:LookupBase.lookup", kwlist,
+                                    &required, &provided, &name, &default_))
+    return NULL;
+
+  return _lookup(self, required, provided, name, default_);
+}
+
+
+/*
+    def lookup1(self, required, provided, name=u'', default=None):
+        cache = self._getcache(provided, name)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            return self.lookup((required, ), provided, name, default)
+
+        if result is None:
+            return default
+
+        return result
+*/
+static PyObject *
+_lookup1(lookup *self,
+        PyObject *required, PyObject *provided, PyObject *name,
+        PyObject *default_)
+{
+  PyObject *result, *cache;
+
+#ifdef PY3K
+  if ( name && !PyUnicode_Check(name) )
+#else
+  if ( name && !PyString_Check(name) && !PyUnicode_Check(name) )
+#endif
+  {
+    PyErr_SetString(PyExc_ValueError,
+                    "name is not a string or unicode");
+    return NULL;
+  }
+
+  cache = _getcache(self, provided, name);
+  if (cache == NULL)
+    return NULL;
+
+  result = PyDict_GetItem(cache, required);
+  if (result == NULL)
+    {
+      PyObject *tup;
+
+      tup = PyTuple_New(1);
+      if (tup == NULL)
+        return NULL;
+      Py_INCREF(required);
+      PyTuple_SET_ITEM(tup, 0, required);
+      result = _lookup(self, tup, provided, name, default_);
+      Py_DECREF(tup);
+    }
+  else
+    {
+      if (result == Py_None && default_ != NULL)
+        {
+          result = default_;
+        }
+      Py_INCREF(result);
+    }
+
+  return result;
+}
+static PyObject *
+lookup_lookup1(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", "name", "default", NULL};
+  PyObject *required, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:LookupBase.lookup1", kwlist,
+                                    &required, &provided, &name, &default_))
+    return NULL;
+
+  return _lookup1(self, required, provided, name, default_);
+}
+
+/*
+    def adapter_hook(self, provided, object, name=u'', default=None):
+        required = providedBy(object)
+        cache = self._getcache(provided, name)
+        factory = cache.get(required, _not_in_mapping)
+        if factory is _not_in_mapping:
+            factory = self.lookup((required, ), provided, name)
+
+        if factory is not None:
+            if isinstance(object, super):
+                object = object.__self__
+            result = factory(object)
+            if result is not None:
+                return result
+
+        return default
+*/
+static PyObject *
+_adapter_hook(lookup *self,
+              PyObject *provided, PyObject *object,  PyObject *name,
+              PyObject *default_)
+{
+  PyObject *required, *factory, *result;
+
+#ifdef PY3K
+  if ( name && !PyUnicode_Check(name) )
+#else
+  if ( name && !PyString_Check(name) && !PyUnicode_Check(name) )
+#endif
+  {
+    PyErr_SetString(PyExc_ValueError,
+                    "name is not a string or unicode");
+    return NULL;
+  }
+
+  required = providedBy(NULL, object);
+  if (required == NULL)
+    return NULL;
+
+  factory = _lookup1(self, required, provided, name, Py_None);
+  Py_DECREF(required);
+  if (factory == NULL)
+    return NULL;
+
+  if (factory != Py_None)
+  {
+      if (PyObject_TypeCheck(object, &PySuper_Type)) {
+          PyObject* self = PyObject_GetAttr(object, str__self__);
+          if (self == NULL)
+          {
+              Py_DECREF(factory);
+              return NULL;
+          }
+          // Borrow the reference to self
+          Py_DECREF(self);
+          object = self;
+      }
+      result = PyObject_CallFunctionObjArgs(factory, object, NULL);
+      Py_DECREF(factory);
+      if (result == NULL || result != Py_None)
+        return result;
+  }
+  else
+    result = factory; /* None */
+
+  if (default_ == NULL || default_ == result) /* No default specified, */
+    return result;   /* Return None.  result is owned None */
+
+  Py_DECREF(result);
+  Py_INCREF(default_);
+
+  return default_;
+}
+static PyObject *
+lookup_adapter_hook(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"provided", "object", "name", "default", NULL};
+  PyObject *object, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:LookupBase.adapter_hook", kwlist,
+                                    &provided, &object, &name, &default_))
+    return NULL;
+
+  return _adapter_hook(self, provided, object, name, default_);
+}
+
+static PyObject *
+lookup_queryAdapter(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"object", "provided", "name", "default", NULL};
+  PyObject *object, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:LookupBase.queryAdapter", kwlist,
+                                    &object, &provided, &name, &default_))
+    return NULL;
+
+  return _adapter_hook(self, provided, object, name, default_);
+}
+
+/*
+    def lookupAll(self, required, provided):
+        cache = self._mcache.get(provided)
+        if cache is None:
+            cache = {}
+            self._mcache[provided] = cache
+
+        required = tuple(required)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            result = self._uncached_lookupAll(required, provided)
+            cache[required] = result
+
+        return result
+*/
+static PyObject *
+_lookupAll(lookup *self, PyObject *required, PyObject *provided)
+{
+  PyObject *cache, *result;
+
+  /* resolve before getting cache. See note in _lookup. */
+  required = PySequence_Tuple(required);
+  if (required == NULL)
+    return NULL;
+
+  ASSURE_DICT(self->_mcache);
+  cache = _subcache(self->_mcache, provided);
+  if (cache == NULL)
+    return NULL;
+
+  result = PyDict_GetItem(cache, required);
+  if (result == NULL)
+    {
+      int status;
+
+      result = PyObject_CallMethodObjArgs(OBJECT(self), str_uncached_lookupAll,
+                                          required, provided, NULL);
+      if (result == NULL)
+        {
+          Py_DECREF(required);
+          return NULL;
+        }
+      status = PyDict_SetItem(cache, required, result);
+      Py_DECREF(required);
+      if (status < 0)
+        {
+          Py_DECREF(result);
+          return NULL;
+        }
+    }
+  else
+    {
+      Py_INCREF(result);
+      Py_DECREF(required);
+    }
+
+  return result;
+}
+static PyObject *
+lookup_lookupAll(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", NULL};
+  PyObject *required, *provided;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO:LookupBase.lookupAll", kwlist,
+                                    &required, &provided))
+    return NULL;
+
+  return _lookupAll(self, required, provided);
+}
+
+/*
+    def subscriptions(self, required, provided):
+        cache = self._scache.get(provided)
+        if cache is None:
+            cache = {}
+            self._scache[provided] = cache
+
+        required = tuple(required)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            result = self._uncached_subscriptions(required, provided)
+            cache[required] = result
+
+        return result
+*/
+static PyObject *
+_subscriptions(lookup *self, PyObject *required, PyObject *provided)
+{
+  PyObject *cache, *result;
+
+  /* resolve before getting cache. See note in _lookup. */
+  required = PySequence_Tuple(required);
+  if (required == NULL)
+    return NULL;
+
+  ASSURE_DICT(self->_scache);
+  cache = _subcache(self->_scache, provided);
+  if (cache == NULL)
+    return NULL;
+
+  result = PyDict_GetItem(cache, required);
+  if (result == NULL)
+    {
+      int status;
+
+      result = PyObject_CallMethodObjArgs(
+                                 OBJECT(self), str_uncached_subscriptions,
+                                 required, provided, NULL);
+      if (result == NULL)
+        {
+          Py_DECREF(required);
+          return NULL;
+        }
+      status = PyDict_SetItem(cache, required, result);
+      Py_DECREF(required);
+      if (status < 0)
+        {
+          Py_DECREF(result);
+          return NULL;
+        }
+    }
+  else
+    {
+      Py_INCREF(result);
+      Py_DECREF(required);
+    }
+
+  return result;
+}
+static PyObject *
+lookup_subscriptions(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", NULL};
+  PyObject *required, *provided;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist,
+                                    &required, &provided))
+    return NULL;
+
+  return _subscriptions(self, required, provided);
+}
+
+static struct PyMethodDef lookup_methods[] = {
+  {"changed",       (PyCFunction)lookup_changed,       METH_O,        ""},
+  {"lookup",        (PyCFunction)lookup_lookup,        METH_KEYWORDS | METH_VARARGS, ""},
+  {"lookup1",       (PyCFunction)lookup_lookup1,       METH_KEYWORDS | METH_VARARGS, ""},
+  {"queryAdapter",  (PyCFunction)lookup_queryAdapter,  METH_KEYWORDS | METH_VARARGS, ""},
+  {"adapter_hook",  (PyCFunction)lookup_adapter_hook,  METH_KEYWORDS | METH_VARARGS, ""},
+  {"lookupAll",     (PyCFunction)lookup_lookupAll,     METH_KEYWORDS | METH_VARARGS, ""},
+  {"subscriptions", (PyCFunction)lookup_subscriptions, METH_KEYWORDS | METH_VARARGS, ""},
+  {NULL,            NULL}               /* sentinel */
+};
+
+static PyTypeObject LookupBase = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_zope_interface_coptimizations."
+                                "LookupBase",
+        /* tp_basicsize      */ sizeof(lookup),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)&lookup_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)0,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT
+                                | Py_TPFLAGS_BASETYPE
+                                | Py_TPFLAGS_HAVE_GC,
+        /* tp_doc            */ "",
+        /* tp_traverse       */ (traverseproc)lookup_traverse,
+        /* tp_clear          */ (inquiry)lookup_clear,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ lookup_methods,
+};
+
+static int
+verifying_traverse(verify *self, visitproc visit, void *arg)
+{
+  int vret;
+
+  vret = lookup_traverse((lookup *)self, visit, arg);
+  if (vret != 0)
+    return vret;
+
+  if (self->_verify_ro) {
+    vret = visit(self->_verify_ro, arg);
+    if (vret != 0)
+      return vret;
+  }
+  if (self->_verify_generations) {
+    vret = visit(self->_verify_generations, arg);
+    if (vret != 0)
+      return vret;
+  }
+
+  return 0;
+}
+
+static int
+verifying_clear(verify *self)
+{
+  lookup_clear((lookup *)self);
+  Py_CLEAR(self->_verify_generations);
+  Py_CLEAR(self->_verify_ro);
+  return 0;
+}
+
+
+static void
+verifying_dealloc(verify *self)
+{
+  PyObject_GC_UnTrack((PyObject *)self);
+  verifying_clear(self);
+  Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+/*
+    def changed(self, originally_changed):
+        super(VerifyingBasePy, self).changed(originally_changed)
+        self._verify_ro = self._registry.ro[1:]
+        self._verify_generations = [r._generation for r in self._verify_ro]
+*/
+static PyObject *
+_generations_tuple(PyObject *ro)
+{
+  int i, l;
+  PyObject *generations;
+
+  l = PyTuple_GET_SIZE(ro);
+  generations = PyTuple_New(l);
+  for (i=0; i < l; i++)
+    {
+      PyObject *generation;
+
+      generation = PyObject_GetAttr(PyTuple_GET_ITEM(ro, i), str_generation);
+      if (generation == NULL)
+        {
+          Py_DECREF(generations);
+          return NULL;
+        }
+      PyTuple_SET_ITEM(generations, i, generation);
+    }
+
+  return generations;
+}
+static PyObject *
+verifying_changed(verify *self, PyObject *ignored)
+{
+  PyObject *t, *ro;
+
+  verifying_clear(self);
+
+  t = PyObject_GetAttr(OBJECT(self), str_registry);
+  if (t == NULL)
+    return NULL;
+  ro = PyObject_GetAttr(t, strro);
+  Py_DECREF(t);
+  if (ro == NULL)
+    return NULL;
+
+  t = PyObject_CallFunctionObjArgs(OBJECT(&PyTuple_Type), ro, NULL);
+  Py_DECREF(ro);
+  if (t == NULL)
+    return NULL;
+
+  ro = PyTuple_GetSlice(t, 1, PyTuple_GET_SIZE(t));
+  Py_DECREF(t);
+  if (ro == NULL)
+    return NULL;
+
+  self->_verify_generations = _generations_tuple(ro);
+  if (self->_verify_generations == NULL)
+    {
+      Py_DECREF(ro);
+      return NULL;
+    }
+
+  self->_verify_ro = ro;
+
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+/*
+    def _verify(self):
+        if ([r._generation for r in self._verify_ro]
+            != self._verify_generations):
+            self.changed(None)
+*/
+static int
+_verify(verify *self)
+{
+  PyObject *changed_result;
+
+  if (self->_verify_ro != NULL && self->_verify_generations != NULL)
+    {
+      PyObject *generations;
+      int changed;
+
+      generations = _generations_tuple(self->_verify_ro);
+      if (generations == NULL)
+        return -1;
+
+      changed = PyObject_RichCompareBool(self->_verify_generations,
+                                         generations, Py_NE);
+      Py_DECREF(generations);
+      if (changed == -1)
+        return -1;
+
+      if (changed == 0)
+        return 0;
+    }
+
+  changed_result = PyObject_CallMethodObjArgs(OBJECT(self), strchanged,
+                                              Py_None, NULL);
+  if (changed_result == NULL)
+    return -1;
+
+  Py_DECREF(changed_result);
+  return 0;
+}
+
+static PyObject *
+verifying_lookup(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", "name", "default", NULL};
+  PyObject *required, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO", kwlist,
+                                    &required, &provided, &name, &default_))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _lookup((lookup *)self, required, provided, name, default_);
+}
+
+static PyObject *
+verifying_lookup1(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", "name", "default", NULL};
+  PyObject *required, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO", kwlist,
+                                    &required, &provided, &name, &default_))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _lookup1((lookup *)self, required, provided, name, default_);
+}
+
+static PyObject *
+verifying_adapter_hook(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"provided", "object", "name", "default", NULL};
+  PyObject *object, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO", kwlist,
+                                    &provided, &object, &name, &default_))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _adapter_hook((lookup *)self, provided, object, name, default_);
+}
+
+static PyObject *
+verifying_queryAdapter(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"object", "provided", "name", "default", NULL};
+  PyObject *object, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO", kwlist,
+                                    &object, &provided, &name, &default_))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _adapter_hook((lookup *)self, provided, object, name, default_);
+}
+
+static PyObject *
+verifying_lookupAll(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", NULL};
+  PyObject *required, *provided;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist,
+                                    &required, &provided))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _lookupAll((lookup *)self, required, provided);
+}
+
+static PyObject *
+verifying_subscriptions(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", NULL};
+  PyObject *required, *provided;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist,
+                                    &required, &provided))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _subscriptions((lookup *)self, required, provided);
+}
+
+static struct PyMethodDef verifying_methods[] = {
+  {"changed",      (PyCFunction)verifying_changed,        METH_O,        ""},
+  {"lookup",       (PyCFunction)verifying_lookup,         METH_KEYWORDS | METH_VARARGS, ""},
+  {"lookup1",      (PyCFunction)verifying_lookup1,        METH_KEYWORDS | METH_VARARGS, ""},
+  {"queryAdapter",  (PyCFunction)verifying_queryAdapter,  METH_KEYWORDS | METH_VARARGS, ""},
+  {"adapter_hook",  (PyCFunction)verifying_adapter_hook,  METH_KEYWORDS | METH_VARARGS, ""},
+  {"lookupAll",    (PyCFunction)verifying_lookupAll,      METH_KEYWORDS | METH_VARARGS, ""},
+  {"subscriptions", (PyCFunction)verifying_subscriptions, METH_KEYWORDS | METH_VARARGS, ""},
+  {NULL,            NULL}               /* sentinel */
+};
+
+static PyTypeObject VerifyingBase = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_zope_interface_coptimizations."
+                                "VerifyingBase",
+        /* tp_basicsize      */ sizeof(verify),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)&verifying_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)0,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT
+                                | Py_TPFLAGS_BASETYPE
+                                | Py_TPFLAGS_HAVE_GC,
+        /* tp_doc            */ "",
+        /* tp_traverse       */ (traverseproc)verifying_traverse,
+        /* tp_clear          */ (inquiry)verifying_clear,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ verifying_methods,
+        /* tp_members        */ 0,
+        /* tp_getset         */ 0,
+        /* tp_base           */ &LookupBase,
+};
+
+/* ========================== End: Lookup Bases ======================= */
+/* ==================================================================== */
+
+
+
+static struct PyMethodDef m_methods[] = {
+  {"implementedBy", (PyCFunction)implementedBy, METH_O,
+   "Interfaces implemented by a class or factory.\n"
+   "Raises TypeError if argument is neither a class nor a callable."},
+  {"getObjectSpecification", (PyCFunction)getObjectSpecification, METH_O,
+   "Get an object's interfaces (internal api)"},
+  {"providedBy", (PyCFunction)providedBy, METH_O,
+   "Get an object's interfaces"},
+
+  {NULL,         (PyCFunction)NULL, 0, NULL}            /* sentinel */
+};
+
+#if  PY_MAJOR_VERSION >= 3
+static char module_doc[] = "C optimizations for zope.interface\n\n";
+
+static struct PyModuleDef _zic_module = {
+        PyModuleDef_HEAD_INIT,
+        "_zope_interface_coptimizations",
+        module_doc,
+        -1,
+        m_methods,
+        NULL,
+        NULL,
+        NULL,
+        NULL
+};
+#endif
+
+static PyObject *
+init(void)
+{
+  PyObject *m;
+
+#if  PY_MAJOR_VERSION < 3
+#define DEFINE_STRING(S) \
+  if(! (str ## S = PyString_FromString(# S))) return NULL
+#else
+#define DEFINE_STRING(S) \
+  if(! (str ## S = PyUnicode_FromString(# S))) return NULL
+#endif
+
+  DEFINE_STRING(__dict__);
+  DEFINE_STRING(__implemented__);
+  DEFINE_STRING(__provides__);
+  DEFINE_STRING(__class__);
+  DEFINE_STRING(__providedBy__);
+  DEFINE_STRING(extends);
+  DEFINE_STRING(__conform__);
+  DEFINE_STRING(_call_conform);
+  DEFINE_STRING(_uncached_lookup);
+  DEFINE_STRING(_uncached_lookupAll);
+  DEFINE_STRING(_uncached_subscriptions);
+  DEFINE_STRING(_registry);
+  DEFINE_STRING(_generation);
+  DEFINE_STRING(ro);
+  DEFINE_STRING(changed);
+  DEFINE_STRING(__self__);
+  DEFINE_STRING(__name__);
+  DEFINE_STRING(__module__);
+  DEFINE_STRING(__adapt__);
+  DEFINE_STRING(_CALL_CUSTOM_ADAPT);
+#undef DEFINE_STRING
+  adapter_hooks = PyList_New(0);
+  if (adapter_hooks == NULL)
+    return NULL;
+
+  /* Initialize types: */
+  SpecificationBaseType.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&SpecificationBaseType) < 0)
+    return NULL;
+  OSDType.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&OSDType) < 0)
+    return NULL;
+  CPBType.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&CPBType) < 0)
+    return NULL;
+
+  InterfaceBaseType.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&InterfaceBaseType) < 0)
+    return NULL;
+
+  LookupBase.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&LookupBase) < 0)
+    return NULL;
+
+  VerifyingBase.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&VerifyingBase) < 0)
+    return NULL;
+
+  #if PY_MAJOR_VERSION < 3
+  /* Create the module and add the functions */
+  m = Py_InitModule3("_zope_interface_coptimizations", m_methods,
+                     "C optimizations for zope.interface\n\n");
+  #else
+  m = PyModule_Create(&_zic_module);
+  #endif
+  if (m == NULL)
+    return NULL;
+
+  /* Add types: */
+  if (PyModule_AddObject(m, "SpecificationBase", OBJECT(&SpecificationBaseType)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "ObjectSpecificationDescriptor",
+                         (PyObject *)&OSDType) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "ClassProvidesBase", OBJECT(&CPBType)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "InterfaceBase", OBJECT(&InterfaceBaseType)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "LookupBase", OBJECT(&LookupBase)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "VerifyingBase", OBJECT(&VerifyingBase)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "adapter_hooks", adapter_hooks) < 0)
+    return NULL;
+  return m;
+}
+
+PyMODINIT_FUNC
+#if PY_MAJOR_VERSION < 3
+init_zope_interface_coptimizations(void)
+{
+  init();
+}
+#else
+PyInit__zope_interface_coptimizations(void)
+{
+  return init();
+}
+#endif
+
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
diff --git a/contrib/python/zope.interface/py2/zope/interface/adapter.py b/contrib/python/zope.interface/py2/zope/interface/adapter.py
new file mode 100644
index 00000000000..9a542db3c70
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/adapter.py
@@ -0,0 +1,1018 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Adapter management
+"""
+import itertools
+import weakref
+
+from zope.interface import implementer
+from zope.interface import providedBy
+from zope.interface import Interface
+from zope.interface import ro
+from zope.interface.interfaces import IAdapterRegistry
+
+from zope.interface._compat import _normalize_name
+from zope.interface._compat import STRING_TYPES
+from zope.interface._compat import _use_c_impl
+
+__all__ = [
+    'AdapterRegistry',
+    'VerifyingAdapterRegistry',
+]
+
+# In the CPython implementation,
+# ``tuple`` and ``list`` cooperate so that ``tuple([some list])``
+# directly allocates and iterates at the C level without using a
+# Python iterator. That's not the case for
+# ``tuple(generator_expression)`` or ``tuple(map(func, it))``.
+##
+# 3.8
+# ``tuple([t for t in range(10)])``      -> 610ns
+# ``tuple(t for t in range(10))``        -> 696ns
+# ``tuple(map(lambda t: t, range(10)))`` -> 881ns
+##
+# 2.7
+# ``tuple([t fon t in range(10)])``      -> 625ns
+# ``tuple(t for t in range(10))``        -> 665ns
+# ``tuple(map(lambda t: t, range(10)))`` -> 958ns
+#
+# All three have substantial variance.
+##
+# On PyPy, this is also the best option.
+##
+# PyPy 2.7.18-7.3.3
+# ``tuple([t fon t in range(10)])``      -> 128ns
+# ``tuple(t for t in range(10))``        -> 175ns
+# ``tuple(map(lambda t: t, range(10)))`` -> 153ns
+##
+# PyPy 3.7.9 7.3.3-beta
+# ``tuple([t fon t in range(10)])``      ->  82ns
+# ``tuple(t for t in range(10))``        -> 177ns
+# ``tuple(map(lambda t: t, range(10)))`` -> 168ns
+#
+
+class BaseAdapterRegistry(object):
+    """
+    A basic implementation of the data storage and algorithms required
+    for a :class:`zope.interface.interfaces.IAdapterRegistry`.
+
+    Subclasses can set the following attributes to control how the data
+    is stored; in particular, these hooks can be helpful for ZODB
+    persistence. They can be class attributes that are the named (or similar) type, or
+    they can be methods that act as a constructor for an object that behaves
+    like the types defined here; this object will not assume that they are type
+    objects, but subclasses are free to do so:
+
+    _sequenceType = list
+      This is the type used for our two mutable top-level "byorder" sequences.
+      Must support mutation operations like ``append()`` and ``del seq[index]``.
+      These are usually small (< 10). Although at least one of them is
+      accessed when performing lookups or queries on this object, the other
+      is untouched. In many common scenarios, both are only required when
+      mutating registrations and subscriptions (like what
+      :meth:`zope.interface.interfaces.IComponents.registerUtility` does).
+      This use pattern makes it an ideal candidate to be a
+      :class:`~persistent.list.PersistentList`.
+    _leafSequenceType = tuple
+      This is the type used for the leaf sequences of subscribers.
+      It could be set to a ``PersistentList`` to avoid many unnecessary data
+      loads when subscribers aren't being used. Mutation operations are directed
+      through :meth:`_addValueToLeaf` and :meth:`_removeValueFromLeaf`; if you use
+      a mutable type, you'll need to override those.
+    _mappingType = dict
+      This is the mutable mapping type used for the keyed mappings.
+      A :class:`~persistent.mapping.PersistentMapping`
+      could be used to help reduce the number of data loads when the registry is large
+      and parts of it are rarely used. Further reductions in data loads can come from
+      using a :class:`~BTrees.OOBTree.OOBTree`, but care is required
+      to be sure that all required/provided
+      values are fully ordered (e.g., no required or provided values that are classes
+      can be used).
+    _providedType = dict
+      This is the mutable mapping type used for the ``_provided`` mapping.
+      This is separate from the generic mapping type because the values
+      are always integers, so one might choose to use a more optimized data
+      structure such as a :class:`~BTrees.OIBTree.OIBTree`.
+      The same caveats regarding key types
+      apply as for ``_mappingType``.
+
+    It is possible to also set these on an instance, but because of the need to
+    potentially also override :meth:`_addValueToLeaf` and :meth:`_removeValueFromLeaf`,
+    this may be less useful in a persistent scenario; using a subclass is recommended.
+
+    .. versionchanged:: 5.3.0
+        Add support for customizing the way internal data
+        structures are created.
+    .. versionchanged:: 5.3.0
+        Add methods :meth:`rebuild`, :meth:`allRegistrations`
+        and :meth:`allSubscriptions`.
+    """
+
+    # List of methods copied from lookup sub-objects:
+    _delegated = ('lookup', 'queryMultiAdapter', 'lookup1', 'queryAdapter',
+                  'adapter_hook', 'lookupAll', 'names',
+                  'subscriptions', 'subscribers')
+
+    # All registries maintain a generation that can be used by verifying
+    # registries
+    _generation = 0
+
+    def __init__(self, bases=()):
+
+        # The comments here could be improved. Possibly this bit needs
+        # explaining in a separate document, as the comments here can
+        # be quite confusing. /regebro
+
+        # {order -> {required -> {provided -> {name -> value}}}}
+        # Here "order" is actually an index in a list, "required" and
+        # "provided" are interfaces, and "required" is really a nested
+        # key.  So, for example:
+        # for order == 0 (that is, self._adapters[0]), we have:
+        #   {provided -> {name -> value}}
+        # but for order == 2 (that is, self._adapters[2]), we have:
+        #   {r1 -> {r2 -> {provided -> {name -> value}}}}
+        #
+        self._adapters = self._sequenceType()
+
+        # {order -> {required -> {provided -> {name -> [value]}}}}
+        # where the remarks about adapters above apply
+        self._subscribers = self._sequenceType()
+
+        # Set, with a reference count, keeping track of the interfaces
+        # for which we have provided components:
+        self._provided = self._providedType()
+
+        # Create ``_v_lookup`` object to perform lookup.  We make this a
+        # separate object to to make it easier to implement just the
+        # lookup functionality in C.  This object keeps track of cache
+        # invalidation data in two kinds of registries.
+
+        #   Invalidating registries have caches that are invalidated
+        #     when they or their base registies change.  An invalidating
+        #     registry can only have invalidating registries as bases.
+        #     See LookupBaseFallback below for the pertinent logic.
+
+        #   Verifying registies can't rely on getting invalidation messages,
+        #     so have to check the generations of base registries to determine
+        #     if their cache data are current.  See VerifyingBasePy below
+        #     for the pertinent object.
+        self._createLookup()
+
+        # Setting the bases causes the registries described above
+        # to be initialized (self._setBases -> self.changed ->
+        # self._v_lookup.changed).
+
+        self.__bases__ = bases
+
+    def _setBases(self, bases):
+        """
+        If subclasses need to track when ``__bases__`` changes, they
+        can override this method.
+
+        Subclasses must still call this method.
+        """
+        self.__dict__['__bases__'] = bases
+        self.ro = ro.ro(self)
+        self.changed(self)
+
+    __bases__ = property(lambda self: self.__dict__['__bases__'],
+                         lambda self, bases: self._setBases(bases),
+                         )
+
+    def _createLookup(self):
+        self._v_lookup = self.LookupClass(self)
+        for name in self._delegated:
+            self.__dict__[name] = getattr(self._v_lookup, name)
+
+    # Hooks for subclasses to define the types of objects used in
+    # our data structures.
+    # These have to be documented in the docstring, instead of local
+    # comments, because Sphinx autodoc ignores the comment and just writes
+    # "alias of list"
+    _sequenceType = list
+    _leafSequenceType = tuple
+    _mappingType = dict
+    _providedType = dict
+
+    def _addValueToLeaf(self, existing_leaf_sequence, new_item):
+        """
+        Add the value *new_item* to the *existing_leaf_sequence*, which may
+        be ``None``.
+
+        Subclasses that redefine `_leafSequenceType` should override this method.
+
+        :param existing_leaf_sequence:
+            If *existing_leaf_sequence* is not *None*, it will be an instance
+            of `_leafSequenceType`. (Unless the object has been unpickled
+            from an old pickle and the class definition has changed, in which case
+            it may be an instance of a previous definition, commonly a `tuple`.)
+
+        :return:
+           This method returns the new value to be stored. It may mutate the
+           sequence in place if it was not ``None`` and the type is mutable, but
+           it must also return it.
+
+        .. versionadded:: 5.3.0
+        """
+        if existing_leaf_sequence is None:
+            return (new_item,)
+        return existing_leaf_sequence + (new_item,)
+
+    def _removeValueFromLeaf(self, existing_leaf_sequence, to_remove):
+        """
+        Remove the item *to_remove* from the (non-``None``, non-empty)
+        *existing_leaf_sequence* and return the mutated sequence.
+
+        If there is more than one item that is equal to *to_remove*
+        they must all be removed.
+
+        Subclasses that redefine `_leafSequenceType` should override
+        this method. Note that they can call this method to help
+        in their implementation; this implementation will always
+        return a new tuple constructed by iterating across
+        the *existing_leaf_sequence* and omitting items equal to *to_remove*.
+
+        :param existing_leaf_sequence:
+           As for `_addValueToLeaf`, probably an instance of
+           `_leafSequenceType` but possibly an older type; never `None`.
+        :return:
+           A version of *existing_leaf_sequence* with all items equal to
+           *to_remove* removed. Must not return `None`. However,
+           returning an empty
+           object, even of another type such as the empty tuple, ``()`` is
+           explicitly allowed; such an object will never be stored.
+
+        .. versionadded:: 5.3.0
+        """
+        return tuple([v for v in existing_leaf_sequence if v != to_remove])
+
+    def changed(self, originally_changed):
+        self._generation += 1
+        self._v_lookup.changed(originally_changed)
+
+    def register(self, required, provided, name, value):
+        if not isinstance(name, STRING_TYPES):
+            raise ValueError('name is not a string')
+        if value is None:
+            self.unregister(required, provided, name, value)
+            return
+
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        name = _normalize_name(name)
+        order = len(required)
+        byorder = self._adapters
+        while len(byorder) <= order:
+            byorder.append(self._mappingType())
+        components = byorder[order]
+        key = required + (provided,)
+
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                d = self._mappingType()
+                components[k] = d
+            components = d
+
+        if components.get(name) is value:
+            return
+
+        components[name] = value
+
+        n = self._provided.get(provided, 0) + 1
+        self._provided[provided] = n
+        if n == 1:
+            self._v_lookup.add_extendor(provided)
+
+        self.changed(self)
+
+    def _find_leaf(self, byorder, required, provided, name):
+        # Find the leaf value, if any, in the *byorder* list
+        # for the interface sequence *required* and the interface
+        # *provided*, given the already normalized *name*.
+        #
+        # If no such leaf value exists, returns ``None``
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        order = len(required)
+        if len(byorder) <= order:
+            return None
+
+        components = byorder[order]
+        key = required + (provided,)
+
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                return None
+            components = d
+
+        return components.get(name)
+
+    def registered(self, required, provided, name=u''):
+        return self._find_leaf(
+            self._adapters,
+            required,
+            provided,
+            _normalize_name(name)
+        )
+
+    @classmethod
+    def _allKeys(cls, components, i, parent_k=()):
+        if i == 0:
+            for k, v in components.items():
+                yield parent_k + (k,), v
+        else:
+            for k, v in components.items():
+                new_parent_k = parent_k + (k,)
+                for x, y in cls._allKeys(v, i - 1, new_parent_k):
+                    yield x, y
+
+    def _all_entries(self, byorder):
+        # Recurse through the mapping levels of the `byorder` sequence,
+        # reconstructing a flattened sequence of ``(required, provided, name, value)``
+        # tuples that can be used to reconstruct the sequence with the appropriate
+        # registration methods.
+        #
+        # Locally reference the `byorder` data; it might be replaced while
+        # this method is running (see ``rebuild``).
+        for i, components in enumerate(byorder):
+            # We will have *i* levels of dictionaries to go before
+            # we get to the leaf.
+            for key, value in self._allKeys(components, i + 1):
+                assert len(key) == i + 2
+                required = key[:i]
+                provided = key[-2]
+                name = key[-1]
+                yield (required, provided, name, value)
+
+    def allRegistrations(self):
+        """
+        Yields tuples ``(required, provided, name, value)`` for all
+        the registrations that this object holds.
+
+        These tuples could be passed as the arguments to the
+        :meth:`register` method on another adapter registry to
+        duplicate the registrations this object holds.
+
+        .. versionadded:: 5.3.0
+        """
+        for t in self._all_entries(self._adapters):
+            yield t
+
+    def unregister(self, required, provided, name, value=None):
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        order = len(required)
+        byorder = self._adapters
+        if order >= len(byorder):
+            return False
+        components = byorder[order]
+        key = required + (provided,)
+
+        # Keep track of how we got to `components`:
+        lookups = []
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                return
+            lookups.append((components, k))
+            components = d
+
+        old = components.get(name)
+        if old is None:
+            return
+        if (value is not None) and (old is not value):
+            return
+
+        del components[name]
+        if not components:
+            # Clean out empty containers, since we don't want our keys
+            # to reference global objects (interfaces) unnecessarily.
+            # This is often a problem when an interface is slated for
+            # removal; a hold-over entry in the registry can make it
+            # difficult to remove such interfaces.
+            for comp, k in reversed(lookups):
+                d = comp[k]
+                if d:
+                    break
+                else:
+                    del comp[k]
+            while byorder and not byorder[-1]:
+                del byorder[-1]
+        n = self._provided[provided] - 1
+        if n == 0:
+            del self._provided[provided]
+            self._v_lookup.remove_extendor(provided)
+        else:
+            self._provided[provided] = n
+
+        self.changed(self)
+
+    def subscribe(self, required, provided, value):
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        name = u''
+        order = len(required)
+        byorder = self._subscribers
+        while len(byorder) <= order:
+            byorder.append(self._mappingType())
+        components = byorder[order]
+        key = required + (provided,)
+
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                d = self._mappingType()
+                components[k] = d
+            components = d
+
+        components[name] = self._addValueToLeaf(components.get(name), value)
+
+        if provided is not None:
+            n = self._provided.get(provided, 0) + 1
+            self._provided[provided] = n
+            if n == 1:
+                self._v_lookup.add_extendor(provided)
+
+        self.changed(self)
+
+    def subscribed(self, required, provided, subscriber):
+        subscribers = self._find_leaf(
+            self._subscribers,
+            required,
+            provided,
+            u''
+        ) or ()
+        return subscriber if subscriber in subscribers else None
+
+    def allSubscriptions(self):
+        """
+        Yields tuples ``(required, provided, value)`` for all the
+        subscribers that this object holds.
+
+        These tuples could be passed as the arguments to the
+        :meth:`subscribe` method on another adapter registry to
+        duplicate the registrations this object holds.
+
+        .. versionadded:: 5.3.0
+        """
+        for required, provided, _name, value in self._all_entries(self._subscribers):
+            for v in value:
+                yield (required, provided, v)
+
+    def unsubscribe(self, required, provided, value=None):
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        order = len(required)
+        byorder = self._subscribers
+        if order >= len(byorder):
+            return
+        components = byorder[order]
+        key = required + (provided,)
+
+        # Keep track of how we got to `components`:
+        lookups = []
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                return
+            lookups.append((components, k))
+            components = d
+
+        old = components.get(u'')
+        if not old:
+            # this is belt-and-suspenders against the failure of cleanup below
+            return  # pragma: no cover
+        len_old = len(old)
+        if value is None:
+            # Removing everything; note that the type of ``new`` won't
+            # necessarily match the ``_leafSequenceType``, but that's
+            # OK because we're about to delete the entire entry
+            # anyway.
+            new = ()
+        else:
+            new = self._removeValueFromLeaf(old, value)
+        # ``new`` may be the same object as ``old``, just mutated in place,
+        # so we cannot compare it to ``old`` to check for changes. Remove
+        # our reference to it now to avoid trying to do so below.
+        del old
+
+        if len(new) == len_old:
+            # No changes, so nothing could have been removed.
+            return
+
+        if new:
+            components[u''] = new
+        else:
+            # Instead of setting components[u''] = new, we clean out
+            # empty containers, since we don't want our keys to
+            # reference global objects (interfaces) unnecessarily.  This
+            # is often a problem when an interface is slated for
+            # removal; a hold-over entry in the registry can make it
+            # difficult to remove such interfaces.
+            del components[u'']
+            for comp, k in reversed(lookups):
+                d = comp[k]
+                if d:
+                    break
+                else:
+                    del comp[k]
+            while byorder and not byorder[-1]:
+                del byorder[-1]
+
+        if provided is not None:
+            n = self._provided[provided] + len(new) - len_old
+            if n == 0:
+                del self._provided[provided]
+                self._v_lookup.remove_extendor(provided)
+            else:
+                self._provided[provided] = n
+
+        self.changed(self)
+
+    def rebuild(self):
+        """
+        Rebuild (and replace) all the internal data structures of this
+        object.
+
+        This is useful, especially for persistent implementations, if
+        you suspect an issue with reference counts keeping interfaces
+        alive even though they are no longer used.
+
+        It is also useful if you or a subclass change the data types
+        (``_mappingType`` and friends) that are to be used.
+
+        This method replaces all internal data structures with new objects;
+        it specifically does not re-use any storage.
+
+        .. versionadded:: 5.3.0
+        """
+
+        # Grab the iterators, we're about to discard their data.
+        registrations = self.allRegistrations()
+        subscriptions = self.allSubscriptions()
+
+        def buffer(it):
+            # The generator doesn't actually start running until we
+            # ask for its next(), by which time the attributes will change
+            # unless we do so before calling __init__.
+            try:
+                first = next(it)
+            except StopIteration:
+                return iter(())
+
+            return itertools.chain((first,), it)
+
+        registrations = buffer(registrations)
+        subscriptions = buffer(subscriptions)
+
+
+        # Replace the base data structures as well as _v_lookup.
+        self.__init__(self.__bases__)
+        # Re-register everything previously registered and subscribed.
+        #
+        # XXX: This is going to call ``self.changed()`` a lot, all of
+        # which is unnecessary (because ``self.__init__`` just
+        # re-created those dependent objects and also called
+        # ``self.changed()``). Is this a bottleneck that needs fixed?
+        # (We could do ``self.changed = lambda _: None`` before
+        # beginning and remove it after to disable the presumably expensive
+        # part of passing that notification to the change of objects.)
+        for args in registrations:
+            self.register(*args)
+        for args in subscriptions:
+            self.subscribe(*args)
+
+    # XXX hack to fake out twisted's use of a private api.  We need to get them
+    # to use the new registered method.
+    def get(self, _): # pragma: no cover
+        class XXXTwistedFakeOut:
+            selfImplied = {}
+        return XXXTwistedFakeOut
+
+
+_not_in_mapping = object()
+
+@_use_c_impl
+class LookupBase(object):
+
+    def __init__(self):
+        self._cache = {}
+        self._mcache = {}
+        self._scache = {}
+
+    def changed(self, ignored=None):
+        self._cache.clear()
+        self._mcache.clear()
+        self._scache.clear()
+
+    def _getcache(self, provided, name):
+        cache = self._cache.get(provided)
+        if cache is None:
+            cache = {}
+            self._cache[provided] = cache
+        if name:
+            c = cache.get(name)
+            if c is None:
+                c = {}
+                cache[name] = c
+            cache = c
+        return cache
+
+    def lookup(self, required, provided, name=u'', default=None):
+        if not isinstance(name, STRING_TYPES):
+            raise ValueError('name is not a string')
+        cache = self._getcache(provided, name)
+        required = tuple(required)
+        if len(required) == 1:
+            result = cache.get(required[0], _not_in_mapping)
+        else:
+            result = cache.get(tuple(required), _not_in_mapping)
+
+        if result is _not_in_mapping:
+            result = self._uncached_lookup(required, provided, name)
+            if len(required) == 1:
+                cache[required[0]] = result
+            else:
+                cache[tuple(required)] = result
+
+        if result is None:
+            return default
+
+        return result
+
+    def lookup1(self, required, provided, name=u'', default=None):
+        if not isinstance(name, STRING_TYPES):
+            raise ValueError('name is not a string')
+        cache = self._getcache(provided, name)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            return self.lookup((required, ), provided, name, default)
+
+        if result is None:
+            return default
+
+        return result
+
+    def queryAdapter(self, object, provided, name=u'', default=None):
+        return self.adapter_hook(provided, object, name, default)
+
+    def adapter_hook(self, provided, object, name=u'', default=None):
+        if not isinstance(name, STRING_TYPES):
+            raise ValueError('name is not a string')
+        required = providedBy(object)
+        cache = self._getcache(provided, name)
+        factory = cache.get(required, _not_in_mapping)
+        if factory is _not_in_mapping:
+            factory = self.lookup((required, ), provided, name)
+
+        if factory is not None:
+            if isinstance(object, super):
+                object = object.__self__
+            result = factory(object)
+            if result is not None:
+                return result
+
+        return default
+
+    def lookupAll(self, required, provided):
+        cache = self._mcache.get(provided)
+        if cache is None:
+            cache = {}
+            self._mcache[provided] = cache
+
+        required = tuple(required)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            result = self._uncached_lookupAll(required, provided)
+            cache[required] = result
+
+        return result
+
+
+    def subscriptions(self, required, provided):
+        cache = self._scache.get(provided)
+        if cache is None:
+            cache = {}
+            self._scache[provided] = cache
+
+        required = tuple(required)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            result = self._uncached_subscriptions(required, provided)
+            cache[required] = result
+
+        return result
+
+
+@_use_c_impl
+class VerifyingBase(LookupBaseFallback):
+    # Mixin for lookups against registries which "chain" upwards, and
+    # whose lookups invalidate their own caches whenever a parent registry
+    # bumps its own '_generation' counter.  E.g., used by
+    # zope.component.persistentregistry
+
+    def changed(self, originally_changed):
+        LookupBaseFallback.changed(self, originally_changed)
+        self._verify_ro = self._registry.ro[1:]
+        self._verify_generations = [r._generation for r in self._verify_ro]
+
+    def _verify(self):
+        if ([r._generation for r in self._verify_ro]
+            != self._verify_generations):
+            self.changed(None)
+
+    def _getcache(self, provided, name):
+        self._verify()
+        return LookupBaseFallback._getcache(self, provided, name)
+
+    def lookupAll(self, required, provided):
+        self._verify()
+        return LookupBaseFallback.lookupAll(self, required, provided)
+
+    def subscriptions(self, required, provided):
+        self._verify()
+        return LookupBaseFallback.subscriptions(self, required, provided)
+
+
+class AdapterLookupBase(object):
+
+    def __init__(self, registry):
+        self._registry = registry
+        self._required = {}
+        self.init_extendors()
+        super(AdapterLookupBase, self).__init__()
+
+    def changed(self, ignored=None):
+        super(AdapterLookupBase, self).changed(None)
+        for r in self._required.keys():
+            r = r()
+            if r is not None:
+                r.unsubscribe(self)
+        self._required.clear()
+
+
+    # Extendors
+    # ---------
+
+    # When given an target interface for an adapter lookup, we need to consider
+    # adapters for interfaces that extend the target interface.  This is
+    # what the extendors dictionary is about.  It tells us all of the
+    # interfaces that extend an interface for which there are adapters
+    # registered.
+
+    # We could separate this by order and name, thus reducing the
+    # number of provided interfaces to search at run time.  The tradeoff,
+    # however, is that we have to store more information.  For example,
+    # if the same interface is provided for multiple names and if the
+    # interface extends many interfaces, we'll have to keep track of
+    # a fair bit of information for each name.  It's better to
+    # be space efficient here and be time efficient in the cache
+    # implementation.
+
+    # TODO: add invalidation when a provided interface changes, in case
+    # the interface's __iro__ has changed.  This is unlikely enough that
+    # we'll take our chances for now.
+
+    def init_extendors(self):
+        self._extendors = {}
+        for p in self._registry._provided:
+            self.add_extendor(p)
+
+    def add_extendor(self, provided):
+        _extendors = self._extendors
+        for i in provided.__iro__:
+            extendors = _extendors.get(i, ())
+            _extendors[i] = (
+                [e for e in extendors if provided.isOrExtends(e)]
+                +
+                [provided]
+                +
+                [e for e in extendors if not provided.isOrExtends(e)]
+                )
+
+    def remove_extendor(self, provided):
+        _extendors = self._extendors
+        for i in provided.__iro__:
+            _extendors[i] = [e for e in _extendors.get(i, ())
+                             if e != provided]
+
+
+    def _subscribe(self, *required):
+        _refs = self._required
+        for r in required:
+            ref = r.weakref()
+            if ref not in _refs:
+                r.subscribe(self)
+                _refs[ref] = 1
+
+    def _uncached_lookup(self, required, provided, name=u''):
+        required = tuple(required)
+        result = None
+        order = len(required)
+        for registry in self._registry.ro:
+            byorder = registry._adapters
+            if order >= len(byorder):
+                continue
+
+            extendors = registry._v_lookup._extendors.get(provided)
+            if not extendors:
+                continue
+
+            components = byorder[order]
+            result = _lookup(components, required, extendors, name, 0,
+                             order)
+            if result is not None:
+                break
+
+        self._subscribe(*required)
+
+        return result
+
+    def queryMultiAdapter(self, objects, provided, name=u'', default=None):
+        factory = self.lookup([providedBy(o) for o in objects], provided, name)
+        if factory is None:
+            return default
+
+        result = factory(*[o.__self__ if isinstance(o, super) else o for o in objects])
+        if result is None:
+            return default
+
+        return result
+
+    def _uncached_lookupAll(self, required, provided):
+        required = tuple(required)
+        order = len(required)
+        result = {}
+        for registry in reversed(self._registry.ro):
+            byorder = registry._adapters
+            if order >= len(byorder):
+                continue
+            extendors = registry._v_lookup._extendors.get(provided)
+            if not extendors:
+                continue
+            components = byorder[order]
+            _lookupAll(components, required, extendors, result, 0, order)
+
+        self._subscribe(*required)
+
+        return tuple(result.items())
+
+    def names(self, required, provided):
+        return [c[0] for c in self.lookupAll(required, provided)]
+
+    def _uncached_subscriptions(self, required, provided):
+        required = tuple(required)
+        order = len(required)
+        result = []
+        for registry in reversed(self._registry.ro):
+            byorder = registry._subscribers
+            if order >= len(byorder):
+                continue
+
+            if provided is None:
+                extendors = (provided, )
+            else:
+                extendors = registry._v_lookup._extendors.get(provided)
+                if extendors is None:
+                    continue
+
+            _subscriptions(byorder[order], required, extendors, u'',
+                           result, 0, order)
+
+        self._subscribe(*required)
+
+        return result
+
+    def subscribers(self, objects, provided):
+        subscriptions = self.subscriptions([providedBy(o) for o in objects], provided)
+        if provided is None:
+            result = ()
+            for subscription in subscriptions:
+                subscription(*objects)
+        else:
+            result = []
+            for subscription in subscriptions:
+                subscriber = subscription(*objects)
+                if subscriber is not None:
+                    result.append(subscriber)
+        return result
+
+class AdapterLookup(AdapterLookupBase, LookupBase):
+    pass
+
+@implementer(IAdapterRegistry)
+class AdapterRegistry(BaseAdapterRegistry):
+    """
+    A full implementation of ``IAdapterRegistry`` that adds support for
+    sub-registries.
+    """
+
+    LookupClass = AdapterLookup
+
+    def __init__(self, bases=()):
+        # AdapterRegisties are invalidating registries, so
+        # we need to keep track of our invalidating subregistries.
+        self._v_subregistries = weakref.WeakKeyDictionary()
+
+        super(AdapterRegistry, self).__init__(bases)
+
+    def _addSubregistry(self, r):
+        self._v_subregistries[r] = 1
+
+    def _removeSubregistry(self, r):
+        if r in self._v_subregistries:
+            del self._v_subregistries[r]
+
+    def _setBases(self, bases):
+        old = self.__dict__.get('__bases__', ())
+        for r in old:
+            if r not in bases:
+                r._removeSubregistry(self)
+        for r in bases:
+            if r not in old:
+                r._addSubregistry(self)
+
+        super(AdapterRegistry, self)._setBases(bases)
+
+    def changed(self, originally_changed):
+        super(AdapterRegistry, self).changed(originally_changed)
+
+        for sub in self._v_subregistries.keys():
+            sub.changed(originally_changed)
+
+
+class VerifyingAdapterLookup(AdapterLookupBase, VerifyingBase):
+    pass
+
+@implementer(IAdapterRegistry)
+class VerifyingAdapterRegistry(BaseAdapterRegistry):
+    """
+    The most commonly-used adapter registry.
+    """
+
+    LookupClass = VerifyingAdapterLookup
+
+def _convert_None_to_Interface(x):
+    if x is None:
+        return Interface
+    else:
+        return x
+
+def _lookup(components, specs, provided, name, i, l):
+    # this function is called very often.
+    # The components.get in loops is executed 100 of 1000s times.
+    # by loading get into a local variable the bytecode
+    # "LOAD_FAST 0 (components)" in the loop can be eliminated.
+    components_get = components.get
+    if i < l:
+        for spec in specs[i].__sro__:
+            comps = components_get(spec)
+            if comps:
+                r = _lookup(comps, specs, provided, name, i+1, l)
+                if r is not None:
+                    return r
+    else:
+        for iface in provided:
+            comps = components_get(iface)
+            if comps:
+                r = comps.get(name)
+                if r is not None:
+                    return r
+
+    return None
+
+def _lookupAll(components, specs, provided, result, i, l):
+    components_get = components.get  # see _lookup above
+    if i < l:
+        for spec in reversed(specs[i].__sro__):
+            comps = components_get(spec)
+            if comps:
+                _lookupAll(comps, specs, provided, result, i+1, l)
+    else:
+        for iface in reversed(provided):
+            comps = components_get(iface)
+            if comps:
+                result.update(comps)
+
+def _subscriptions(components, specs, provided, name, result, i, l):
+    components_get = components.get  # see _lookup above
+    if i < l:
+        for spec in reversed(specs[i].__sro__):
+            comps = components_get(spec)
+            if comps:
+                _subscriptions(comps, specs, provided, name, result, i+1, l)
+    else:
+        for iface in reversed(provided):
+            comps = components_get(iface)
+            if comps:
+                comps = comps.get(name)
+                if comps:
+                    result.extend(comps)
diff --git a/contrib/python/zope.interface/py2/zope/interface/advice.py b/contrib/python/zope.interface/py2/zope/interface/advice.py
new file mode 100644
index 00000000000..86d0f11aa4d
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/advice.py
@@ -0,0 +1,213 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Class advice.
+
+This module was adapted from 'protocols.advice', part of the Python
+Enterprise Application Kit (PEAK).  Please notify the PEAK authors
+(pje@telecommunity.com and tsarna@sarna.org) if bugs are found or
+Zope-specific changes are required, so that the PEAK version of this module
+can be kept in sync.
+
+PEAK is a Python application framework that interoperates with (but does
+not require) Zope 3 and Twisted.  It provides tools for manipulating UML
+models, object-relational persistence, aspect-oriented programming, and more.
+Visit the PEAK home page at http://peak.telecommunity.com for more information.
+"""
+
+from types import FunctionType
+try:
+    from types import ClassType
+except ImportError:
+    __python3 = True
+else:
+    __python3 = False
+
+__all__ = [
+    'addClassAdvisor',
+    'determineMetaclass',
+    'getFrameInfo',
+    'isClassAdvisor',
+    'minimalBases',
+]
+
+import sys
+
+def getFrameInfo(frame):
+    """Return (kind,module,locals,globals) for a frame
+
+    'kind' is one of "exec", "module", "class", "function call", or "unknown".
+    """
+
+    f_locals = frame.f_locals
+    f_globals = frame.f_globals
+
+    sameNamespace = f_locals is f_globals
+    hasModule = '__module__' in f_locals
+    hasName = '__name__' in f_globals
+
+    sameName = hasModule and hasName
+    sameName = sameName and f_globals['__name__']==f_locals['__module__']
+
+    module = hasName and sys.modules.get(f_globals['__name__']) or None
+
+    namespaceIsModule = module and module.__dict__ is f_globals
+
+    if not namespaceIsModule:
+        # some kind of funky exec
+        kind = "exec"
+    elif sameNamespace and not hasModule:
+        kind = "module"
+    elif sameName and not sameNamespace:
+        kind = "class"
+    elif not sameNamespace:
+        kind = "function call"
+    else: # pragma: no cover
+        # How can you have f_locals is f_globals, and have '__module__' set?
+        # This is probably module-level code, but with a '__module__' variable.
+        kind = "unknown"
+    return kind, module, f_locals, f_globals
+
+
+def addClassAdvisor(callback, depth=2):
+    """Set up 'callback' to be passed the containing class upon creation
+
+    This function is designed to be called by an "advising" function executed
+    in a class suite.  The "advising" function supplies a callback that it
+    wishes to have executed when the containing class is created.  The
+    callback will be given one argument: the newly created containing class.
+    The return value of the callback will be used in place of the class, so
+    the callback should return the input if it does not wish to replace the
+    class.
+
+    The optional 'depth' argument to this function determines the number of
+    frames between this function and the targeted class suite.  'depth'
+    defaults to 2, since this skips this function's frame and one calling
+    function frame.  If you use this function from a function called directly
+    in the class suite, the default will be correct, otherwise you will need
+    to determine the correct depth yourself.
+
+    This function works by installing a special class factory function in
+    place of the '__metaclass__' of the containing class.  Therefore, only
+    callbacks *after* the last '__metaclass__' assignment in the containing
+    class will be executed.  Be sure that classes using "advising" functions
+    declare any '__metaclass__' *first*, to ensure all callbacks are run."""
+    # This entire approach is invalid under Py3K.  Don't even try to fix
+    # the coverage for this block there. :(
+    if __python3: # pragma: no cover
+        raise TypeError('Class advice impossible in Python3')
+
+    frame = sys._getframe(depth)
+    kind, module, caller_locals, caller_globals = getFrameInfo(frame)
+
+    # This causes a problem when zope interfaces are used from doctest.
+    # In these cases, kind == "exec".
+    #
+    #if kind != "class":
+    #    raise SyntaxError(
+    #        "Advice must be in the body of a class statement"
+    #    )
+
+    previousMetaclass = caller_locals.get('__metaclass__')
+    if __python3:   # pragma: no cover
+        defaultMetaclass  = caller_globals.get('__metaclass__', type)
+    else:
+        defaultMetaclass  = caller_globals.get('__metaclass__', ClassType)
+
+
+    def advise(name, bases, cdict):
+
+        if '__metaclass__' in cdict:
+            del cdict['__metaclass__']
+
+        if previousMetaclass is None:
+            if bases:
+                # find best metaclass or use global __metaclass__ if no bases
+                meta = determineMetaclass(bases)
+            else:
+                meta = defaultMetaclass
+
+        elif isClassAdvisor(previousMetaclass):
+            # special case: we can't compute the "true" metaclass here,
+            # so we need to invoke the previous metaclass and let it
+            # figure it out for us (and apply its own advice in the process)
+            meta = previousMetaclass
+
+        else:
+            meta = determineMetaclass(bases, previousMetaclass)
+
+        newClass = meta(name,bases,cdict)
+
+        # this lets the callback replace the class completely, if it wants to
+        return callback(newClass)
+
+    # introspection data only, not used by inner function
+    advise.previousMetaclass = previousMetaclass
+    advise.callback = callback
+
+    # install the advisor
+    caller_locals['__metaclass__'] = advise
+
+
+def isClassAdvisor(ob):
+    """True if 'ob' is a class advisor function"""
+    return isinstance(ob,FunctionType) and hasattr(ob,'previousMetaclass')
+
+
+def determineMetaclass(bases, explicit_mc=None):
+    """Determine metaclass from 1+ bases and optional explicit __metaclass__"""
+
+    meta = [getattr(b,'__class__',type(b)) for b in bases]
+
+    if explicit_mc is not None:
+        # The explicit metaclass needs to be verified for compatibility
+        # as well, and allowed to resolve the incompatible bases, if any
+        meta.append(explicit_mc)
+
+    if len(meta)==1:
+        # easy case
+        return meta[0]
+
+    candidates = minimalBases(meta) # minimal set of metaclasses
+
+    if not candidates: # pragma: no cover
+        # they're all "classic" classes
+        assert(not __python3) # This should not happen under Python 3
+        return ClassType
+
+    elif len(candidates)>1:
+        # We could auto-combine, but for now we won't...
+        raise TypeError("Incompatible metatypes",bases)
+
+    # Just one, return it
+    return candidates[0]
+
+
+def minimalBases(classes):
+    """Reduce a list of base classes to its ordered minimum equivalent"""
+
+    if not __python3: # pragma: no cover
+        classes = [c for c in classes if c is not ClassType]
+    candidates = []
+
+    for m in classes:
+        for n in classes:
+            if issubclass(n,m) and m is not n:
+                break
+        else:
+            # m has no subclasses in 'classes'
+            if m in candidates:
+                candidates.remove(m)    # ensure that we're later in the list
+            candidates.append(m)
+
+    return candidates
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/__init__.py b/contrib/python/zope.interface/py2/zope/interface/common/__init__.py
new file mode 100644
index 00000000000..137e93867ee
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/__init__.py
@@ -0,0 +1,272 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+
+import itertools
+from types import FunctionType
+
+from zope.interface import classImplements
+from zope.interface import Interface
+from zope.interface.interface import fromFunction
+from zope.interface.interface import InterfaceClass
+from zope.interface.interface import _decorator_non_return
+
+__all__ = [
+    # Nothing public here.
+]
+
+
+# pylint:disable=inherit-non-class,
+# pylint:disable=no-self-argument,no-method-argument
+# pylint:disable=unexpected-special-method-signature
+
+class optional(object):
+    # Apply this decorator to a method definition to make it
+    # optional (remove it from the list of required names), overriding
+    # the definition inherited from the ABC.
+    def __init__(self, method):
+        self.__doc__ = method.__doc__
+
+
+class ABCInterfaceClass(InterfaceClass):
+    """
+    An interface that is automatically derived from a
+    :class:`abc.ABCMeta` type.
+
+    Internal use only.
+
+    The body of the interface definition *must* define
+    a property ``abc`` that is the ABC to base the interface on.
+
+    If ``abc`` is *not* in the interface definition, a regular
+    interface will be defined instead (but ``extra_classes`` is still
+    respected).
+
+    Use the ``@optional`` decorator on method definitions if
+    the ABC defines methods that are not actually required in all cases
+    because the Python language has multiple ways to implement a protocol.
+    For example, the ``iter()`` protocol can be implemented with
+    ``__iter__`` or the pair ``__len__`` and ``__getitem__``.
+
+    When created, any existing classes that are registered to conform
+    to the ABC are declared to implement this interface. This is *not*
+    automatically updated as the ABC registry changes. If the body of the
+    interface definition defines ``extra_classes``, it should be a
+    tuple giving additional classes to declare implement the interface.
+
+    Note that this is not fully symmetric. For example, it is usually
+    the case that a subclass relationship carries the interface
+    declarations over::
+
+        >>> from zope.interface import Interface
+        >>> class I1(Interface):
+        ...     pass
+        ...
+        >>> from zope.interface import implementer
+        >>> @implementer(I1)
+        ... class Root(object):
+        ...     pass
+        ...
+        >>> class Child(Root):
+        ...     pass
+        ...
+        >>> child = Child()
+        >>> isinstance(child, Root)
+        True
+        >>> from zope.interface import providedBy
+        >>> list(providedBy(child))
+        [<InterfaceClass __main__.I1>]
+
+    However, that's not the case with ABCs and ABC interfaces. Just
+    because ``isinstance(A(), AnABC)`` and ``isinstance(B(), AnABC)``
+    are both true, that doesn't mean there's any class hierarchy
+    relationship between ``A`` and ``B``, or between either of them
+    and ``AnABC``. Thus, if ``AnABC`` implemented ``IAnABC``, it would
+    not follow that either ``A`` or ``B`` implements ``IAnABC`` (nor
+    their instances provide it)::
+
+        >>> class SizedClass(object):
+        ...     def __len__(self): return 1
+        ...
+        >>> from collections.abc import Sized
+        >>> isinstance(SizedClass(), Sized)
+        True
+        >>> from zope.interface import classImplements
+        >>> classImplements(Sized, I1)
+        None
+        >>> list(providedBy(SizedClass()))
+        []
+
+    Thus, to avoid conflicting assumptions, ABCs should not be
+    declared to implement their parallel ABC interface. Only concrete
+    classes specifically registered with the ABC should be declared to
+    do so.
+
+    .. versionadded:: 5.0.0
+    """
+
+    # If we could figure out invalidation, and used some special
+    # Specification/Declaration instances, and override the method ``providedBy`` here,
+    # perhaps we could more closely integrate with ABC virtual inheritance?
+
+    def __init__(self, name, bases, attrs):
+        # go ahead and give us a name to ease debugging.
+        self.__name__ = name
+        extra_classes = attrs.pop('extra_classes', ())
+        ignored_classes = attrs.pop('ignored_classes', ())
+
+        if 'abc' not in attrs:
+            # Something like ``IList(ISequence)``: We're extending
+            # abc interfaces but not an ABC interface ourself.
+            InterfaceClass.__init__(self, name, bases, attrs)
+            ABCInterfaceClass.__register_classes(self, extra_classes, ignored_classes)
+            self.__class__ = InterfaceClass
+            return
+
+        based_on = attrs.pop('abc')
+        self.__abc = based_on
+        self.__extra_classes = tuple(extra_classes)
+        self.__ignored_classes = tuple(ignored_classes)
+
+        assert name[1:] == based_on.__name__, (name, based_on)
+        methods = {
+            # Passing the name is important in case of aliases,
+            # e.g., ``__ror__ = __or__``.
+            k: self.__method_from_function(v, k)
+            for k, v in vars(based_on).items()
+            if isinstance(v, FunctionType) and not self.__is_private_name(k)
+            and not self.__is_reverse_protocol_name(k)
+        }
+
+        methods['__doc__'] = self.__create_class_doc(attrs)
+        # Anything specified in the body takes precedence.
+        methods.update(attrs)
+        InterfaceClass.__init__(self, name, bases, methods)
+        self.__register_classes()
+
+    @staticmethod
+    def __optional_methods_to_docs(attrs):
+        optionals = {k: v for k, v in attrs.items() if isinstance(v, optional)}
+        for k in optionals:
+            attrs[k] = _decorator_non_return
+
+        if not optionals:
+            return ''
+
+        docs = "\n\nThe following methods are optional:\n - " + "\n-".join(
+            "%s\n%s" % (k, v.__doc__) for k, v in optionals.items()
+        )
+        return docs
+
+    def __create_class_doc(self, attrs):
+        based_on = self.__abc
+        def ref(c):
+            mod = c.__module__
+            name = c.__name__
+            if mod == str.__module__:
+                return "`%s`" % name
+            if mod == '_io':
+                mod = 'io'
+            return "`%s.%s`" % (mod, name)
+        implementations_doc = "\n - ".join(
+            ref(c)
+            for c in sorted(self.getRegisteredConformers(), key=ref)
+        )
+        if implementations_doc:
+            implementations_doc = "\n\nKnown implementations are:\n\n - " + implementations_doc
+
+        based_on_doc = (based_on.__doc__ or '')
+        based_on_doc = based_on_doc.splitlines()
+        based_on_doc = based_on_doc[0] if based_on_doc else ''
+
+        doc = """Interface for the ABC `%s.%s`.\n\n%s%s%s""" % (
+            based_on.__module__, based_on.__name__,
+            attrs.get('__doc__', based_on_doc),
+            self.__optional_methods_to_docs(attrs),
+            implementations_doc
+        )
+        return doc
+
+
+    @staticmethod
+    def __is_private_name(name):
+        if name.startswith('__') and name.endswith('__'):
+            return False
+        return name.startswith('_')
+
+    @staticmethod
+    def __is_reverse_protocol_name(name):
+        # The reverse names, like __rand__,
+        # aren't really part of the protocol. The interpreter has
+        # very complex behaviour around invoking those. PyPy
+        # doesn't always even expose them as attributes.
+        return name.startswith('__r') and name.endswith('__')
+
+    def __method_from_function(self, function, name):
+        method = fromFunction(function, self, name=name)
+        # Eliminate the leading *self*, which is implied in
+        # an interface, but explicit in an ABC.
+        method.positional = method.positional[1:]
+        return method
+
+    def __register_classes(self, conformers=None, ignored_classes=None):
+        # Make the concrete classes already present in our ABC's registry
+        # declare that they implement this interface.
+        conformers = conformers if conformers is not None else self.getRegisteredConformers()
+        ignored = ignored_classes if ignored_classes is not None else self.__ignored_classes
+        for cls in conformers:
+            if cls in ignored:
+                continue
+            classImplements(cls, self)
+
+    def getABC(self):
+        """
+        Return the ABC this interface represents.
+        """
+        return self.__abc
+
+    def getRegisteredConformers(self):
+        """
+        Return an iterable of the classes that are known to conform to
+        the ABC this interface parallels.
+        """
+        based_on = self.__abc
+
+        # The registry only contains things that aren't already
+        # known to be subclasses of the ABC. But the ABC is in charge
+        # of checking that, so its quite possible that registrations
+        # are in fact ignored, winding up just in the _abc_cache.
+        try:
+            registered = list(based_on._abc_registry) + list(based_on._abc_cache)
+        except AttributeError:
+            # Rewritten in C in CPython 3.7.
+            # These expose the underlying weakref.
+            from abc import _get_dump
+            data = _get_dump(based_on)
+            registry = data[0]
+            cache = data[1]
+            registered = [x() for x in itertools.chain(registry, cache)]
+            registered = [x for x in registered if x is not None]
+
+        return set(itertools.chain(registered, self.__extra_classes))
+
+
+def _create_ABCInterface():
+    # It's a two-step process to create the root ABCInterface, because
+    # without specifying a corresponding ABC, using the normal constructor
+    # gets us a plain InterfaceClass object, and there is no ABC to associate with the
+    # root.
+    abc_name_bases_attrs = ('ABCInterface', (Interface,), {})
+    instance = ABCInterfaceClass.__new__(ABCInterfaceClass, *abc_name_bases_attrs)
+    InterfaceClass.__init__(instance, *abc_name_bases_attrs)
+    return instance
+
+ABCInterface = _create_ABCInterface()
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/builtins.py b/contrib/python/zope.interface/py2/zope/interface/common/builtins.py
new file mode 100644
index 00000000000..a07c0a36e72
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/builtins.py
@@ -0,0 +1,125 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""
+Interface definitions for builtin types.
+
+After this module is imported, the standard library types will declare
+that they implement the appropriate interface.
+
+.. versionadded:: 5.0.0
+"""
+from __future__ import absolute_import
+
+from zope.interface import classImplements
+
+from zope.interface.common import collections
+from zope.interface.common import numbers
+from zope.interface.common import io
+
+__all__ = [
+    'IList',
+    'ITuple',
+    'ITextString',
+    'IByteString',
+    'INativeString',
+    'IBool',
+    'IDict',
+    'IFile',
+]
+
+# pylint:disable=no-self-argument
+class IList(collections.IMutableSequence):
+    """
+    Interface for :class:`list`
+    """
+    extra_classes = (list,)
+
+    def sort(key=None, reverse=False):
+        """
+        Sort the list in place and return None.
+
+        *key* and *reverse* must be passed by name only.
+        """
+
+
+class ITuple(collections.ISequence):
+    """
+    Interface for :class:`tuple`
+    """
+    extra_classes = (tuple,)
+
+
+class ITextString(collections.ISequence):
+    """
+    Interface for text (unicode) strings.
+
+    On Python 2, this is :class:`unicode`. On Python 3,
+    this is :class:`str`
+    """
+    extra_classes = (type(u'unicode'),)
+
+
+class IByteString(collections.IByteString):
+    """
+    Interface for immutable byte strings.
+
+    On all Python versions this is :class:`bytes`.
+
+    Unlike :class:`zope.interface.common.collections.IByteString`
+    (the parent of this interface) this does *not* include
+    :class:`bytearray`.
+    """
+    extra_classes = (bytes,)
+
+
+class INativeString(IByteString if str is bytes else ITextString):
+    """
+    Interface for native strings.
+
+    On all Python versions, this is :class:`str`. On Python 2,
+    this extends :class:`IByteString`, while on Python 3 it extends
+    :class:`ITextString`.
+    """
+# We're not extending ABCInterface so extra_classes won't work
+classImplements(str, INativeString)
+
+
+class IBool(numbers.IIntegral):
+    """
+    Interface for :class:`bool`
+    """
+    extra_classes = (bool,)
+
+
+class IDict(collections.IMutableMapping):
+    """
+    Interface for :class:`dict`
+    """
+    extra_classes = (dict,)
+
+
+class IFile(io.IIOBase):
+    """
+    Interface for :class:`file`.
+
+    It is recommended to use the interfaces from :mod:`zope.interface.common.io`
+    instead of this interface.
+
+    On Python 3, there is no single implementation of this interface;
+    depending on the arguments, the :func:`open` builtin can return
+    many different classes that implement different interfaces from
+    :mod:`zope.interface.common.io`.
+    """
+    try:
+        extra_classes = (file,)
+    except NameError:
+        extra_classes = ()
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/collections.py b/contrib/python/zope.interface/py2/zope/interface/common/collections.py
new file mode 100644
index 00000000000..00e2b8c2871
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/collections.py
@@ -0,0 +1,284 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""
+Interface definitions paralleling the abstract base classes defined in
+:mod:`collections.abc`.
+
+After this module is imported, the standard library types will declare
+that they implement the appropriate interface. While most standard
+library types will properly implement that interface (that
+is, ``verifyObject(ISequence, list()))`` will pass, for example), a few might not:
+
+    - `memoryview` doesn't feature all the defined methods of
+      ``ISequence`` such as ``count``; it is still declared to provide
+      ``ISequence`` though.
+
+    - `collections.deque.pop` doesn't accept the ``index`` argument of
+      `collections.abc.MutableSequence.pop`
+
+    - `range.index` does not accept the ``start`` and ``stop`` arguments.
+
+.. versionadded:: 5.0.0
+"""
+from __future__ import absolute_import
+
+import sys
+
+from abc import ABCMeta
+# The collections imports are here, and not in
+# zope.interface._compat to avoid importing collections
+# unless requested. It's a big import.
+try:
+    from collections import abc
+except ImportError:
+    import collections as abc
+from collections import OrderedDict
+try:
+    # On Python 3, all of these extend the appropriate collection ABC,
+    # but on Python 2, UserDict does not (though it is registered as a
+    # MutableMapping). (Importantly, UserDict on Python 2 is *not*
+    # registered, because it's not iterable.) Extending the ABC is not
+    # taken into account for interface declarations, though, so we
+    # need to be explicit about it.
+    from collections import UserList
+    from collections import UserDict
+    from collections import UserString
+except ImportError:
+    # Python 2
+    from UserList import UserList
+    from UserDict import IterableUserDict as UserDict
+    from UserString import UserString
+
+from zope.interface._compat import PYTHON2 as PY2
+from zope.interface._compat import PYTHON3 as PY3
+from zope.interface.common import ABCInterface
+from zope.interface.common import optional
+
+# pylint:disable=inherit-non-class,
+# pylint:disable=no-self-argument,no-method-argument
+# pylint:disable=unexpected-special-method-signature
+# pylint:disable=no-value-for-parameter
+
+PY35 = sys.version_info[:2] >= (3, 5)
+PY36 = sys.version_info[:2] >= (3, 6)
+
+def _new_in_ver(name, ver,
+                bases_if_missing=(ABCMeta,),
+                register_if_missing=()):
+    if ver:
+        return getattr(abc, name)
+
+    # TODO: It's a shame to have to repeat the bases when
+    # the ABC is missing. Can we DRY that?
+    missing = ABCMeta(name, bases_if_missing, {
+        '__doc__': "The ABC %s is not defined in this version of Python." % (
+            name
+        ),
+    })
+
+    for c in register_if_missing:
+        missing.register(c)
+
+    return missing
+
+__all__ = [
+    'IAsyncGenerator',
+    'IAsyncIterable',
+    'IAsyncIterator',
+    'IAwaitable',
+    'ICollection',
+    'IContainer',
+    'ICoroutine',
+    'IGenerator',
+    'IHashable',
+    'IItemsView',
+    'IIterable',
+    'IIterator',
+    'IKeysView',
+    'IMapping',
+    'IMappingView',
+    'IMutableMapping',
+    'IMutableSequence',
+    'IMutableSet',
+    'IReversible',
+    'ISequence',
+    'ISet',
+    'ISized',
+    'IValuesView',
+]
+
+class IContainer(ABCInterface):
+    abc = abc.Container
+
+    @optional
+    def __contains__(other):
+        """
+        Optional method. If not provided, the interpreter will use
+        ``__iter__`` or the old ``__getitem__`` protocol
+        to implement ``in``.
+        """
+
+class IHashable(ABCInterface):
+    abc = abc.Hashable
+
+class IIterable(ABCInterface):
+    abc = abc.Iterable
+
+    @optional
+    def __iter__():
+        """
+        Optional method. If not provided, the interpreter will
+        implement `iter` using the old ``__getitem__`` protocol.
+        """
+
+class IIterator(IIterable):
+    abc = abc.Iterator
+
+class IReversible(IIterable):
+    abc = _new_in_ver('Reversible', PY36, (IIterable.getABC(),))
+
+    @optional
+    def __reversed__():
+        """
+        Optional method. If this isn't present, the interpreter
+        will use ``__len__`` and ``__getitem__`` to implement the
+        `reversed` builtin.
+        """
+
+class IGenerator(IIterator):
+    # New in 3.5
+    abc = _new_in_ver('Generator', PY35, (IIterator.getABC(),))
+
+
+class ISized(ABCInterface):
+    abc = abc.Sized
+
+
+# ICallable is not defined because there's no standard signature.
+
+class ICollection(ISized,
+                  IIterable,
+                  IContainer):
+    abc = _new_in_ver('Collection', PY36,
+                      (ISized.getABC(), IIterable.getABC(), IContainer.getABC()))
+
+
+class ISequence(IReversible,
+                ICollection):
+    abc = abc.Sequence
+    extra_classes = (UserString,)
+    # On Python 2, basestring is registered as an ISequence, and
+    # its subclass str is an IByteString. If we also register str as
+    # an ISequence, that tends to lead to inconsistent resolution order.
+    ignored_classes = (basestring,) if str is bytes else () # pylint:disable=undefined-variable
+
+    @optional
+    def __reversed__():
+        """
+        Optional method. If this isn't present, the interpreter
+        will use ``__len__`` and ``__getitem__`` to implement the
+        `reversed` builtin.
+        """
+
+    @optional
+    def __iter__():
+        """
+        Optional method. If not provided, the interpreter will
+        implement `iter` using the old ``__getitem__`` protocol.
+        """
+
+class IMutableSequence(ISequence):
+    abc = abc.MutableSequence
+    extra_classes = (UserList,)
+
+
+class IByteString(ISequence):
+    """
+    This unifies `bytes` and `bytearray`.
+    """
+    abc = _new_in_ver('ByteString', PY3,
+                      (ISequence.getABC(),),
+                      (bytes, bytearray))
+
+
+class ISet(ICollection):
+    abc = abc.Set
+
+
+class IMutableSet(ISet):
+    abc = abc.MutableSet
+
+
+class IMapping(ICollection):
+    abc = abc.Mapping
+    extra_classes = (dict,)
+    # OrderedDict is a subclass of dict. On CPython 2,
+    # it winds up registered as a IMutableMapping, which
+    # produces an inconsistent IRO if we also try to register it
+    # here.
+    ignored_classes = (OrderedDict,)
+    if PY2:
+        @optional
+        def __eq__(other):
+            """
+            The interpreter will supply one.
+            """
+
+        __ne__ = __eq__
+
+
+class IMutableMapping(IMapping):
+    abc = abc.MutableMapping
+    extra_classes = (dict, UserDict,)
+    ignored_classes = (OrderedDict,)
+
+class IMappingView(ISized):
+    abc = abc.MappingView
+
+
+class IItemsView(IMappingView, ISet):
+    abc = abc.ItemsView
+
+
+class IKeysView(IMappingView, ISet):
+    abc = abc.KeysView
+
+
+class IValuesView(IMappingView, ICollection):
+    abc = abc.ValuesView
+
+    @optional
+    def __contains__(other):
+        """
+        Optional method. If not provided, the interpreter will use
+        ``__iter__`` or the old ``__len__`` and ``__getitem__`` protocol
+        to implement ``in``.
+        """
+
+class IAwaitable(ABCInterface):
+    abc = _new_in_ver('Awaitable', PY35)
+
+
+class ICoroutine(IAwaitable):
+    abc = _new_in_ver('Coroutine', PY35)
+
+
+class IAsyncIterable(ABCInterface):
+    abc = _new_in_ver('AsyncIterable', PY35)
+
+
+class IAsyncIterator(IAsyncIterable):
+    abc = _new_in_ver('AsyncIterator', PY35)
+
+
+class IAsyncGenerator(IAsyncIterator):
+    abc = _new_in_ver('AsyncGenerator', PY36)
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/idatetime.py b/contrib/python/zope.interface/py2/zope/interface/common/idatetime.py
new file mode 100644
index 00000000000..82f0059c851
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/idatetime.py
@@ -0,0 +1,606 @@
+##############################################################################
+# Copyright (c) 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""Datetime interfaces.
+
+This module is called idatetime because if it were called datetime the import
+of the real datetime would fail.
+"""
+from datetime import timedelta, date, datetime, time, tzinfo
+
+from zope.interface import Interface, Attribute
+from zope.interface import classImplements
+
+
+class ITimeDeltaClass(Interface):
+    """This is the timedelta class interface.
+
+    This is symbolic; this module does **not** make
+    `datetime.timedelta` provide this interface.
+    """
+
+    min = Attribute("The most negative timedelta object")
+
+    max = Attribute("The most positive timedelta object")
+
+    resolution = Attribute(
+        "The smallest difference between non-equal timedelta objects")
+
+
+class ITimeDelta(ITimeDeltaClass):
+    """Represent the difference between two datetime objects.
+
+    Implemented by `datetime.timedelta`.
+
+    Supported operators:
+
+    - add, subtract timedelta
+    - unary plus, minus, abs
+    - compare to timedelta
+    - multiply, divide by int/long
+
+    In addition, `.datetime` supports subtraction of two `.datetime` objects
+    returning a `.timedelta`, and addition or subtraction of a `.datetime`
+    and a `.timedelta` giving a `.datetime`.
+
+    Representation: (days, seconds, microseconds).
+    """
+
+    days = Attribute("Days between -999999999 and 999999999 inclusive")
+
+    seconds = Attribute("Seconds between 0 and 86399 inclusive")
+
+    microseconds = Attribute("Microseconds between 0 and 999999 inclusive")
+
+
+class IDateClass(Interface):
+    """This is the date class interface.
+
+    This is symbolic; this module does **not** make
+    `datetime.date` provide this interface.
+    """
+
+    min = Attribute("The earliest representable date")
+
+    max = Attribute("The latest representable date")
+
+    resolution = Attribute(
+        "The smallest difference between non-equal date objects")
+
+    def today():
+        """Return the current local time.
+
+        This is equivalent to ``date.fromtimestamp(time.time())``"""
+
+    def fromtimestamp(timestamp):
+        """Return the local date from a POSIX timestamp (like time.time())
+
+        This may raise `ValueError`, if the timestamp is out of the range of
+        values supported by the platform C ``localtime()`` function. It's common
+        for this to be restricted to years from 1970 through 2038. Note that
+        on non-POSIX systems that include leap seconds in their notion of a
+        timestamp, leap seconds are ignored by `fromtimestamp`.
+        """
+
+    def fromordinal(ordinal):
+        """Return the date corresponding to the proleptic Gregorian ordinal.
+
+         January 1 of year 1 has ordinal 1. `ValueError` is raised unless
+         1 <= ordinal <= date.max.toordinal().
+
+         For any date *d*, ``date.fromordinal(d.toordinal()) == d``.
+         """
+
+
+class IDate(IDateClass):
+    """Represents a date (year, month and day) in an idealized calendar.
+
+    Implemented by `datetime.date`.
+
+    Operators:
+
+    __repr__, __str__
+    __cmp__, __hash__
+    __add__, __radd__, __sub__ (add/radd only with timedelta arg)
+    """
+
+    year = Attribute("Between MINYEAR and MAXYEAR inclusive.")
+
+    month = Attribute("Between 1 and 12 inclusive")
+
+    day = Attribute(
+        "Between 1 and the number of days in the given month of the given year.")
+
+    def replace(year, month, day):
+        """Return a date with the same value.
+
+        Except for those members given new values by whichever keyword
+        arguments are specified. For example, if ``d == date(2002, 12, 31)``, then
+        ``d.replace(day=26) == date(2000, 12, 26)``.
+        """
+
+    def timetuple():
+        """Return a 9-element tuple of the form returned by `time.localtime`.
+
+        The hours, minutes and seconds are 0, and the DST flag is -1.
+        ``d.timetuple()`` is equivalent to
+        ``(d.year, d.month, d.day, 0, 0, 0, d.weekday(), d.toordinal() -
+        date(d.year, 1, 1).toordinal() + 1, -1)``
+        """
+
+    def toordinal():
+        """Return the proleptic Gregorian ordinal of the date
+
+        January 1 of year 1 has ordinal 1. For any date object *d*,
+        ``date.fromordinal(d.toordinal()) == d``.
+        """
+
+    def weekday():
+        """Return the day of the week as an integer.
+
+        Monday is 0 and Sunday is 6. For example,
+        ``date(2002, 12, 4).weekday() == 2``, a Wednesday.
+
+        .. seealso:: `isoweekday`.
+        """
+
+    def isoweekday():
+        """Return the day of the week as an integer.
+
+        Monday is 1 and Sunday is 7. For example,
+        date(2002, 12, 4).isoweekday() == 3, a Wednesday.
+
+        .. seealso:: `weekday`, `isocalendar`.
+        """
+
+    def isocalendar():
+        """Return a 3-tuple, (ISO year, ISO week number, ISO weekday).
+
+        The ISO calendar is a widely used variant of the Gregorian calendar.
+        See http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm for a good
+        explanation.
+
+        The ISO year consists of 52 or 53 full weeks, and where a week starts
+        on a Monday and ends on a Sunday. The first week of an ISO year is the
+        first (Gregorian) calendar week of a year containing a Thursday. This
+        is called week number 1, and the ISO year of that Thursday is the same
+        as its Gregorian year.
+
+        For example, 2004 begins on a Thursday, so the first week of ISO year
+        2004 begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004, so
+        that ``date(2003, 12, 29).isocalendar() == (2004, 1, 1)`` and
+        ``date(2004, 1, 4).isocalendar() == (2004, 1, 7)``.
+        """
+
+    def isoformat():
+        """Return a string representing the date in ISO 8601 format.
+
+        This is 'YYYY-MM-DD'.
+        For example, ``date(2002, 12, 4).isoformat() == '2002-12-04'``.
+        """
+
+    def __str__():
+        """For a date *d*, ``str(d)`` is equivalent to ``d.isoformat()``."""
+
+    def ctime():
+        """Return a string representing the date.
+
+        For example date(2002, 12, 4).ctime() == 'Wed Dec 4 00:00:00 2002'.
+        d.ctime() is equivalent to time.ctime(time.mktime(d.timetuple()))
+        on platforms where the native C ctime() function
+        (which `time.ctime` invokes, but which date.ctime() does not invoke)
+        conforms to the C standard.
+        """
+
+    def strftime(format):
+        """Return a string representing the date.
+
+        Controlled by an explicit format string. Format codes referring to
+        hours, minutes or seconds will see 0 values.
+        """
+
+
+class IDateTimeClass(Interface):
+    """This is the datetime class interface.
+
+    This is symbolic; this module does **not** make
+    `datetime.datetime` provide this interface.
+    """
+
+    min = Attribute("The earliest representable datetime")
+
+    max = Attribute("The latest representable datetime")
+
+    resolution = Attribute(
+        "The smallest possible difference between non-equal datetime objects")
+
+    def today():
+        """Return the current local datetime, with tzinfo None.
+
+        This is equivalent to ``datetime.fromtimestamp(time.time())``.
+
+        .. seealso:: `now`, `fromtimestamp`.
+        """
+
+    def now(tz=None):
+        """Return the current local date and time.
+
+        If optional argument *tz* is None or not specified, this is like `today`,
+        but, if possible, supplies more precision than can be gotten from going
+        through a `time.time` timestamp (for example, this may be possible on
+        platforms supplying the C ``gettimeofday()`` function).
+
+        Else tz must be an instance of a class tzinfo subclass, and the current
+        date and time are converted to tz's time zone. In this case the result
+        is equivalent to tz.fromutc(datetime.utcnow().replace(tzinfo=tz)).
+
+        .. seealso:: `today`, `utcnow`.
+        """
+
+    def utcnow():
+        """Return the current UTC date and time, with tzinfo None.
+
+        This is like `now`, but returns the current UTC date and time, as a
+        naive datetime object.
+
+        .. seealso:: `now`.
+        """
+
+    def fromtimestamp(timestamp, tz=None):
+        """Return the local date and time corresponding to the POSIX timestamp.
+
+        Same as is returned by time.time(). If optional argument tz is None or
+        not specified, the timestamp is converted to the platform's local date
+        and time, and the returned datetime object is naive.
+
+        Else tz must be an instance of a class tzinfo subclass, and the
+        timestamp is converted to tz's time zone. In this case the result is
+        equivalent to
+        ``tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))``.
+
+        fromtimestamp() may raise `ValueError`, if the timestamp is out of the
+        range of values supported by the platform C localtime() or gmtime()
+        functions. It's common for this to be restricted to years in 1970
+        through 2038. Note that on non-POSIX systems that include leap seconds
+        in their notion of a timestamp, leap seconds are ignored by
+        fromtimestamp(), and then it's possible to have two timestamps
+        differing by a second that yield identical datetime objects.
+
+        .. seealso:: `utcfromtimestamp`.
+        """
+
+    def utcfromtimestamp(timestamp):
+        """Return the UTC datetime from the POSIX timestamp with tzinfo None.
+
+        This may raise `ValueError`, if the timestamp is out of the range of
+        values supported by the platform C ``gmtime()`` function. It's common for
+        this to be restricted to years in 1970 through 2038.
+
+        .. seealso:: `fromtimestamp`.
+        """
+
+    def fromordinal(ordinal):
+        """Return the datetime from the proleptic Gregorian ordinal.
+
+        January 1 of year 1 has ordinal 1. `ValueError` is raised unless
+        1 <= ordinal <= datetime.max.toordinal().
+        The hour, minute, second and microsecond of the result are all 0, and
+        tzinfo is None.
+        """
+
+    def combine(date, time):
+        """Return a new datetime object.
+
+        Its date members are equal to the given date object's, and whose time
+        and tzinfo members are equal to the given time object's. For any
+        datetime object *d*, ``d == datetime.combine(d.date(), d.timetz())``.
+        If date is a datetime object, its time and tzinfo members are ignored.
+        """
+
+
+class IDateTime(IDate, IDateTimeClass):
+    """Object contains all the information from a date object and a time object.
+
+    Implemented by `datetime.datetime`.
+    """
+
+    year = Attribute("Year between MINYEAR and MAXYEAR inclusive")
+
+    month = Attribute("Month between 1 and 12 inclusive")
+
+    day = Attribute(
+        "Day between 1 and the number of days in the given month of the year")
+
+    hour = Attribute("Hour in range(24)")
+
+    minute = Attribute("Minute in range(60)")
+
+    second = Attribute("Second in range(60)")
+
+    microsecond = Attribute("Microsecond in range(1000000)")
+
+    tzinfo = Attribute(
+        """The object passed as the tzinfo argument to the datetime constructor
+        or None if none was passed""")
+
+    def date():
+         """Return date object with same year, month and day."""
+
+    def time():
+        """Return time object with same hour, minute, second, microsecond.
+
+        tzinfo is None.
+
+        .. seealso:: Method :meth:`timetz`.
+        """
+
+    def timetz():
+        """Return time object with same hour, minute, second, microsecond,
+        and tzinfo.
+
+        .. seealso:: Method :meth:`time`.
+        """
+
+    def replace(year, month, day, hour, minute, second, microsecond, tzinfo):
+        """Return a datetime with the same members, except for those members
+        given new values by whichever keyword arguments are specified.
+
+        Note that ``tzinfo=None`` can be specified to create a naive datetime from
+        an aware datetime with no conversion of date and time members.
+        """
+
+    def astimezone(tz):
+        """Return a datetime object with new tzinfo member tz, adjusting the
+        date and time members so the result is the same UTC time as self, but
+        in tz's local time.
+
+        tz must be an instance of a tzinfo subclass, and its utcoffset() and
+        dst() methods must not return None. self must be aware (self.tzinfo
+        must not be None, and self.utcoffset() must not return None).
+
+        If self.tzinfo is tz, self.astimezone(tz) is equal to self: no
+        adjustment of date or time members is performed. Else the result is
+        local time in time zone tz, representing the same UTC time as self:
+
+            after astz = dt.astimezone(tz), astz - astz.utcoffset()
+
+        will usually have the same date and time members as dt - dt.utcoffset().
+        The discussion of class `datetime.tzinfo` explains the cases at Daylight Saving
+        Time transition boundaries where this cannot be achieved (an issue only
+        if tz models both standard and daylight time).
+
+        If you merely want to attach a time zone object *tz* to a datetime *dt*
+        without adjustment of date and time members, use ``dt.replace(tzinfo=tz)``.
+        If you merely want to remove the time zone object from an aware
+        datetime dt without conversion of date and time members, use
+        ``dt.replace(tzinfo=None)``.
+
+        Note that the default `tzinfo.fromutc` method can be overridden in a
+        tzinfo subclass to effect the result returned by `astimezone`.
+        """
+
+    def utcoffset():
+        """Return the timezone offset in minutes east of UTC (negative west of
+        UTC)."""
+
+    def dst():
+        """Return 0 if DST is not in effect, or the DST offset (in minutes
+        eastward) if DST is in effect.
+        """
+
+    def tzname():
+        """Return the timezone name."""
+
+    def timetuple():
+        """Return a 9-element tuple of the form returned by `time.localtime`."""
+
+    def utctimetuple():
+        """Return UTC time tuple compatilble with `time.gmtime`."""
+
+    def toordinal():
+        """Return the proleptic Gregorian ordinal of the date.
+
+        The same as self.date().toordinal().
+        """
+
+    def weekday():
+        """Return the day of the week as an integer.
+
+        Monday is 0 and Sunday is 6. The same as self.date().weekday().
+        See also isoweekday().
+        """
+
+    def isoweekday():
+        """Return the day of the week as an integer.
+
+        Monday is 1 and Sunday is 7. The same as self.date().isoweekday.
+
+        .. seealso:: `weekday`, `isocalendar`.
+        """
+
+    def isocalendar():
+        """Return a 3-tuple, (ISO year, ISO week number, ISO weekday).
+
+        The same as self.date().isocalendar().
+        """
+
+    def isoformat(sep='T'):
+        """Return a string representing the date and time in ISO 8601 format.
+
+        YYYY-MM-DDTHH:MM:SS.mmmmmm or YYYY-MM-DDTHH:MM:SS if microsecond is 0
+
+        If `utcoffset` does not return None, a 6-character string is appended,
+        giving the UTC offset in (signed) hours and minutes:
+
+        YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM or YYYY-MM-DDTHH:MM:SS+HH:MM
+        if microsecond is 0.
+
+        The optional argument sep (default 'T') is a one-character separator,
+        placed between the date and time portions of the result.
+        """
+
+    def __str__():
+        """For a datetime instance *d*, ``str(d)`` is equivalent to ``d.isoformat(' ')``.
+        """
+
+    def ctime():
+        """Return a string representing the date and time.
+
+        ``datetime(2002, 12, 4, 20, 30, 40).ctime() == 'Wed Dec 4 20:30:40 2002'``.
+        ``d.ctime()`` is equivalent to ``time.ctime(time.mktime(d.timetuple()))`` on
+        platforms where the native C ``ctime()`` function (which `time.ctime`
+        invokes, but which `datetime.ctime` does not invoke) conforms to the
+        C standard.
+        """
+
+    def strftime(format):
+        """Return a string representing the date and time.
+
+        This is controlled by an explicit format string.
+        """
+
+
+class ITimeClass(Interface):
+    """This is the time class interface.
+
+    This is symbolic; this module does **not** make
+    `datetime.time` provide this interface.
+
+    """
+
+    min = Attribute("The earliest representable time")
+
+    max = Attribute("The latest representable time")
+
+    resolution = Attribute(
+        "The smallest possible difference between non-equal time objects")
+
+
+class ITime(ITimeClass):
+    """Represent time with time zone.
+
+    Implemented by `datetime.time`.
+
+    Operators:
+
+    __repr__, __str__
+    __cmp__, __hash__
+    """
+
+    hour = Attribute("Hour in range(24)")
+
+    minute = Attribute("Minute in range(60)")
+
+    second = Attribute("Second in range(60)")
+
+    microsecond = Attribute("Microsecond in range(1000000)")
+
+    tzinfo = Attribute(
+        """The object passed as the tzinfo argument to the time constructor
+        or None if none was passed.""")
+
+    def replace(hour, minute, second, microsecond, tzinfo):
+        """Return a time with the same value.
+
+        Except for those members given new values by whichever keyword
+        arguments are specified. Note that tzinfo=None can be specified
+        to create a naive time from an aware time, without conversion of the
+        time members.
+        """
+
+    def isoformat():
+        """Return a string representing the time in ISO 8601 format.
+
+        That is HH:MM:SS.mmmmmm or, if self.microsecond is 0, HH:MM:SS
+        If utcoffset() does not return None, a 6-character string is appended,
+        giving the UTC offset in (signed) hours and minutes:
+        HH:MM:SS.mmmmmm+HH:MM or, if self.microsecond is 0, HH:MM:SS+HH:MM
+        """
+
+    def __str__():
+        """For a time t, str(t) is equivalent to t.isoformat()."""
+
+    def strftime(format):
+        """Return a string representing the time.
+
+        This is controlled by an explicit format string.
+        """
+
+    def utcoffset():
+        """Return the timezone offset in minutes east of UTC (negative west of
+        UTC).
+
+        If tzinfo is None, returns None, else returns
+        self.tzinfo.utcoffset(None), and raises an exception if the latter
+        doesn't return None or a timedelta object representing a whole number
+        of minutes with magnitude less than one day.
+        """
+
+    def dst():
+        """Return 0 if DST is not in effect, or the DST offset (in minutes
+        eastward) if DST is in effect.
+
+        If tzinfo is None, returns None, else returns self.tzinfo.dst(None),
+        and raises an exception if the latter doesn't return None, or a
+        timedelta object representing a whole number of minutes with
+        magnitude less than one day.
+        """
+
+    def tzname():
+        """Return the timezone name.
+
+        If tzinfo is None, returns None, else returns self.tzinfo.tzname(None),
+        or raises an exception if the latter doesn't return None or a string
+        object.
+        """
+
+
+class ITZInfo(Interface):
+    """Time zone info class.
+    """
+
+    def utcoffset(dt):
+        """Return offset of local time from UTC, in minutes east of UTC.
+
+        If local time is west of UTC, this should be negative.
+        Note that this is intended to be the total offset from UTC;
+        for example, if a tzinfo object represents both time zone and DST
+        adjustments, utcoffset() should return their sum. If the UTC offset
+        isn't known, return None. Else the value returned must be a timedelta
+        object specifying a whole number of minutes in the range -1439 to 1439
+        inclusive (1440 = 24*60; the magnitude of the offset must be less
+        than one day).
+        """
+
+    def dst(dt):
+        """Return the daylight saving time (DST) adjustment, in minutes east
+        of UTC, or None if DST information isn't known.
+        """
+
+    def tzname(dt):
+        """Return the time zone name corresponding to the datetime object as
+        a string.
+        """
+
+    def fromutc(dt):
+        """Return an equivalent datetime in self's local time."""
+
+
+classImplements(timedelta, ITimeDelta)
+classImplements(date, IDate)
+classImplements(datetime, IDateTime)
+classImplements(time, ITime)
+classImplements(tzinfo, ITZInfo)
+
+## directlyProvides(timedelta, ITimeDeltaClass)
+## directlyProvides(date, IDateClass)
+## directlyProvides(datetime, IDateTimeClass)
+## directlyProvides(time, ITimeClass)
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/interfaces.py b/contrib/python/zope.interface/py2/zope/interface/common/interfaces.py
new file mode 100644
index 00000000000..4308e0ac325
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/interfaces.py
@@ -0,0 +1,212 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interfaces for standard python exceptions
+"""
+from zope.interface import Interface
+from zope.interface import classImplements
+
+class IException(Interface):
+    "Interface for `Exception`"
+classImplements(Exception, IException)
+
+
+class IStandardError(IException):
+    "Interface for `StandardError` (Python 2 only.)"
+try:
+    classImplements(StandardError, IStandardError)
+except NameError:  #pragma NO COVER
+    pass # StandardError does not exist in Python 3
+
+
+class IWarning(IException):
+    "Interface for `Warning`"
+classImplements(Warning, IWarning)
+
+
+class ISyntaxError(IStandardError):
+    "Interface for `SyntaxError`"
+classImplements(SyntaxError, ISyntaxError)
+
+
+class ILookupError(IStandardError):
+    "Interface for `LookupError`"
+classImplements(LookupError, ILookupError)
+
+
+class IValueError(IStandardError):
+    "Interface for `ValueError`"
+classImplements(ValueError, IValueError)
+
+
+class IRuntimeError(IStandardError):
+    "Interface for `RuntimeError`"
+classImplements(RuntimeError, IRuntimeError)
+
+
+class IArithmeticError(IStandardError):
+    "Interface for `ArithmeticError`"
+classImplements(ArithmeticError, IArithmeticError)
+
+
+class IAssertionError(IStandardError):
+    "Interface for `AssertionError`"
+classImplements(AssertionError, IAssertionError)
+
+
+class IAttributeError(IStandardError):
+    "Interface for `AttributeError`"
+classImplements(AttributeError, IAttributeError)
+
+
+class IDeprecationWarning(IWarning):
+    "Interface for `DeprecationWarning`"
+classImplements(DeprecationWarning, IDeprecationWarning)
+
+
+class IEOFError(IStandardError):
+    "Interface for `EOFError`"
+classImplements(EOFError, IEOFError)
+
+
+class IEnvironmentError(IStandardError):
+    "Interface for `EnvironmentError`"
+classImplements(EnvironmentError, IEnvironmentError)
+
+
+class IFloatingPointError(IArithmeticError):
+    "Interface for `FloatingPointError`"
+classImplements(FloatingPointError, IFloatingPointError)
+
+
+class IIOError(IEnvironmentError):
+    "Interface for `IOError`"
+classImplements(IOError, IIOError)
+
+
+class IImportError(IStandardError):
+    "Interface for `ImportError`"
+classImplements(ImportError, IImportError)
+
+
+class IIndentationError(ISyntaxError):
+    "Interface for `IndentationError`"
+classImplements(IndentationError, IIndentationError)
+
+
+class IIndexError(ILookupError):
+    "Interface for `IndexError`"
+classImplements(IndexError, IIndexError)
+
+
+class IKeyError(ILookupError):
+    "Interface for `KeyError`"
+classImplements(KeyError, IKeyError)
+
+
+class IKeyboardInterrupt(IStandardError):
+    "Interface for `KeyboardInterrupt`"
+classImplements(KeyboardInterrupt, IKeyboardInterrupt)
+
+
+class IMemoryError(IStandardError):
+    "Interface for `MemoryError`"
+classImplements(MemoryError, IMemoryError)
+
+
+class INameError(IStandardError):
+    "Interface for `NameError`"
+classImplements(NameError, INameError)
+
+
+class INotImplementedError(IRuntimeError):
+    "Interface for `NotImplementedError`"
+classImplements(NotImplementedError, INotImplementedError)
+
+
+class IOSError(IEnvironmentError):
+    "Interface for `OSError`"
+classImplements(OSError, IOSError)
+
+
+class IOverflowError(IArithmeticError):
+    "Interface for `ArithmeticError`"
+classImplements(OverflowError, IOverflowError)
+
+
+class IOverflowWarning(IWarning):
+    """Deprecated, no standard class implements this.
+
+    This was the interface for ``OverflowWarning`` prior to Python 2.5,
+    but that class was removed for all versions after that.
+    """
+
+
+class IReferenceError(IStandardError):
+    "Interface for `ReferenceError`"
+classImplements(ReferenceError, IReferenceError)
+
+
+class IRuntimeWarning(IWarning):
+    "Interface for `RuntimeWarning`"
+classImplements(RuntimeWarning, IRuntimeWarning)
+
+
+class IStopIteration(IException):
+    "Interface for `StopIteration`"
+classImplements(StopIteration, IStopIteration)
+
+
+class ISyntaxWarning(IWarning):
+    "Interface for `SyntaxWarning`"
+classImplements(SyntaxWarning, ISyntaxWarning)
+
+
+class ISystemError(IStandardError):
+    "Interface for `SystemError`"
+classImplements(SystemError, ISystemError)
+
+
+class ISystemExit(IException):
+    "Interface for `SystemExit`"
+classImplements(SystemExit, ISystemExit)
+
+
+class ITabError(IIndentationError):
+    "Interface for `TabError`"
+classImplements(TabError, ITabError)
+
+
+class ITypeError(IStandardError):
+    "Interface for `TypeError`"
+classImplements(TypeError, ITypeError)
+
+
+class IUnboundLocalError(INameError):
+    "Interface for `UnboundLocalError`"
+classImplements(UnboundLocalError, IUnboundLocalError)
+
+
+class IUnicodeError(IValueError):
+    "Interface for `UnicodeError`"
+classImplements(UnicodeError, IUnicodeError)
+
+
+class IUserWarning(IWarning):
+    "Interface for `UserWarning`"
+classImplements(UserWarning, IUserWarning)
+
+
+class IZeroDivisionError(IArithmeticError):
+    "Interface for `ZeroDivisionError`"
+classImplements(ZeroDivisionError, IZeroDivisionError)
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/io.py b/contrib/python/zope.interface/py2/zope/interface/common/io.py
new file mode 100644
index 00000000000..540d53ac945
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/io.py
@@ -0,0 +1,53 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""
+Interface definitions paralleling the abstract base classes defined in
+:mod:`io`.
+
+After this module is imported, the standard library types will declare
+that they implement the appropriate interface.
+
+.. versionadded:: 5.0.0
+"""
+from __future__ import absolute_import
+
+import io as abc
+
+from zope.interface.common import ABCInterface
+
+# pylint:disable=inherit-non-class,
+# pylint:disable=no-member
+
+class IIOBase(ABCInterface):
+    abc = abc.IOBase
+
+
+class IRawIOBase(IIOBase):
+    abc = abc.RawIOBase
+
+
+class IBufferedIOBase(IIOBase):
+    abc = abc.BufferedIOBase
+    try:
+        import cStringIO
+    except ImportError:
+        # Python 3
+        extra_classes = ()
+    else:
+        import StringIO
+        extra_classes = (StringIO.StringIO, cStringIO.InputType, cStringIO.OutputType)
+        del cStringIO
+        del StringIO
+
+
+class ITextIOBase(IIOBase):
+    abc = abc.TextIOBase
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/mapping.py b/contrib/python/zope.interface/py2/zope/interface/common/mapping.py
new file mode 100644
index 00000000000..de56cf84935
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/mapping.py
@@ -0,0 +1,184 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Mapping Interfaces.
+
+Importing this module does *not* mark any standard classes as
+implementing any of these interfaces.
+
+While this module is not deprecated, new code should generally use
+:mod:`zope.interface.common.collections`, specifically
+:class:`~zope.interface.common.collections.IMapping` and
+:class:`~zope.interface.common.collections.IMutableMapping`. This
+module is occasionally useful for its extremely fine grained breakdown
+of interfaces.
+
+The standard library :class:`dict` and :class:`collections.UserDict`
+implement ``IMutableMapping``, but *do not* implement any of the
+interfaces in this module.
+"""
+from zope.interface import Interface
+from zope.interface._compat import PYTHON2 as PY2
+from zope.interface.common import collections
+
+class IItemMapping(Interface):
+    """Simplest readable mapping object
+    """
+
+    def __getitem__(key):
+        """Get a value for a key
+
+        A `KeyError` is raised if there is no value for the key.
+        """
+
+
+class IReadMapping(collections.IContainer, IItemMapping):
+    """
+    Basic mapping interface.
+
+    .. versionchanged:: 5.0.0
+       Extend ``IContainer``
+    """
+
+    def get(key, default=None):
+        """Get a value for a key
+
+        The default is returned if there is no value for the key.
+        """
+
+    def __contains__(key):
+        """Tell if a key exists in the mapping."""
+        # Optional in IContainer, required by this interface.
+
+
+class IWriteMapping(Interface):
+    """Mapping methods for changing data"""
+
+    def __delitem__(key):
+        """Delete a value from the mapping using the key."""
+
+    def __setitem__(key, value):
+        """Set a new item in the mapping."""
+
+
+class IEnumerableMapping(collections.ISized, IReadMapping):
+    """
+    Mapping objects whose items can be enumerated.
+
+    .. versionchanged:: 5.0.0
+       Extend ``ISized``
+    """
+
+    def keys():
+        """Return the keys of the mapping object.
+        """
+
+    def __iter__():
+        """Return an iterator for the keys of the mapping object.
+        """
+
+    def values():
+        """Return the values of the mapping object.
+        """
+
+    def items():
+        """Return the items of the mapping object.
+        """
+
+class IMapping(IWriteMapping, IEnumerableMapping):
+    ''' Simple mapping interface '''
+
+class IIterableMapping(IEnumerableMapping):
+    """A mapping that has distinct methods for iterating
+    without copying.
+
+    On Python 2, a `dict` has these methods, but on Python 3
+    the methods defined in `IEnumerableMapping` already iterate
+    without copying.
+    """
+
+    if PY2:
+        def iterkeys():
+            "iterate over keys; equivalent to ``__iter__``"
+
+        def itervalues():
+            "iterate over values"
+
+        def iteritems():
+            "iterate over items"
+
+class IClonableMapping(Interface):
+    """Something that can produce a copy of itself.
+
+    This is available in `dict`.
+    """
+
+    def copy():
+        "return copy of dict"
+
+class IExtendedReadMapping(IIterableMapping):
+    """
+    Something with a particular method equivalent to ``__contains__``.
+
+    On Python 2, `dict` provides this method, but it was removed
+    in Python 3.
+    """
+
+    if PY2:
+        def has_key(key):
+            """Tell if a key exists in the mapping; equivalent to ``__contains__``"""
+
+class IExtendedWriteMapping(IWriteMapping):
+    """Additional mutation methods.
+
+    These are all provided by `dict`.
+    """
+
+    def clear():
+        "delete all items"
+
+    def update(d):
+        " Update D from E: for k in E.keys(): D[k] = E[k]"
+
+    def setdefault(key, default=None):
+        "D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D"
+
+    def pop(k, default=None):
+        """
+        pop(k[,default]) -> value
+
+        Remove specified key and return the corresponding value.
+
+        If key is not found, *default* is returned if given, otherwise
+        `KeyError` is raised. Note that *default* must not be passed by
+        name.
+        """
+
+    def popitem():
+        """remove and return some (key, value) pair as a
+        2-tuple; but raise KeyError if mapping is empty"""
+
+class IFullMapping(
+        collections.IMutableMapping,
+        IExtendedReadMapping, IExtendedWriteMapping, IClonableMapping, IMapping,):
+    """
+    Full mapping interface.
+
+    Most uses of this interface should instead use
+    :class:`~zope.interface.commons.collections.IMutableMapping` (one of the
+    bases of this interface). The required methods are the same.
+
+    .. versionchanged:: 5.0.0
+       Extend ``IMutableMapping``
+    """
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/numbers.py b/contrib/python/zope.interface/py2/zope/interface/common/numbers.py
new file mode 100644
index 00000000000..3bf9206b5e3
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/numbers.py
@@ -0,0 +1,84 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""
+Interface definitions paralleling the abstract base classes defined in
+:mod:`numbers`.
+
+After this module is imported, the standard library types will declare
+that they implement the appropriate interface.
+
+.. versionadded:: 5.0.0
+"""
+from __future__ import absolute_import
+
+import numbers as abc
+
+from zope.interface.common import ABCInterface
+from zope.interface.common import optional
+
+from zope.interface._compat import PYTHON2 as PY2
+
+# pylint:disable=inherit-non-class,
+# pylint:disable=no-self-argument,no-method-argument
+# pylint:disable=unexpected-special-method-signature
+# pylint:disable=no-value-for-parameter
+
+
+class INumber(ABCInterface):
+    abc = abc.Number
+
+
+class IComplex(INumber):
+    abc = abc.Complex
+
+    @optional
+    def __complex__():
+        """
+        Rarely implemented, even in builtin types.
+        """
+
+    if PY2:
+        @optional
+        def __eq__(other):
+            """
+            The interpreter may supply one through complicated rules.
+            """
+
+        __ne__ = __eq__
+
+class IReal(IComplex):
+    abc = abc.Real
+
+    @optional
+    def __complex__():
+        """
+        Rarely implemented, even in builtin types.
+        """
+
+    __floor__ = __ceil__ = __complex__
+
+    if PY2:
+        @optional
+        def __le__(other):
+            """
+            The interpreter may supply one through complicated rules.
+            """
+
+        __lt__ = __le__
+
+
+class IRational(IReal):
+    abc = abc.Rational
+
+
+class IIntegral(IRational):
+    abc = abc.Integral
diff --git a/contrib/python/zope.interface/py2/zope/interface/common/sequence.py b/contrib/python/zope.interface/py2/zope/interface/common/sequence.py
new file mode 100644
index 00000000000..da4bc84a09c
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/common/sequence.py
@@ -0,0 +1,215 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Sequence Interfaces
+
+Importing this module does *not* mark any standard classes as
+implementing any of these interfaces.
+
+While this module is not deprecated, new code should generally use
+:mod:`zope.interface.common.collections`, specifically
+:class:`~zope.interface.common.collections.ISequence` and
+:class:`~zope.interface.common.collections.IMutableSequence`. This
+module is occasionally useful for its fine-grained breakdown of interfaces.
+
+The standard library :class:`list`, :class:`tuple` and
+:class:`collections.UserList`, among others, implement ``ISequence``
+or ``IMutableSequence`` but *do not* implement any of the interfaces
+in this module.
+"""
+
+__docformat__ = 'restructuredtext'
+from zope.interface import Interface
+from zope.interface.common import collections
+from zope.interface._compat import PYTHON2 as PY2
+
+class IMinimalSequence(collections.IIterable):
+    """Most basic sequence interface.
+
+    All sequences are iterable.  This requires at least one of the
+    following:
+
+    - a `__getitem__()` method that takes a single argument; integer
+      values starting at 0 must be supported, and `IndexError` should
+      be raised for the first index for which there is no value, or
+
+    - an `__iter__()` method that returns an iterator as defined in
+      the Python documentation (http://docs.python.org/lib/typeiter.html).
+
+    """
+
+    def __getitem__(index):
+        """``x.__getitem__(index) <==> x[index]``
+
+        Declaring this interface does not specify whether `__getitem__`
+        supports slice objects."""
+
+class IFiniteSequence(collections.ISized, IMinimalSequence):
+    """
+    A sequence of bound size.
+
+    .. versionchanged:: 5.0.0
+       Extend ``ISized``
+    """
+
+class IReadSequence(collections.IContainer, IFiniteSequence):
+    """
+    read interface shared by tuple and list
+
+    This interface is similar to
+    :class:`~zope.interface.common.collections.ISequence`, but
+    requires that all instances be totally ordered. Most users
+    should prefer ``ISequence``.
+
+    .. versionchanged:: 5.0.0
+       Extend ``IContainer``
+    """
+
+    def __contains__(item):
+        """``x.__contains__(item) <==> item in x``"""
+        # Optional in IContainer, required here.
+
+    def __lt__(other):
+        """``x.__lt__(other) <==> x < other``"""
+
+    def __le__(other):
+        """``x.__le__(other) <==> x <= other``"""
+
+    def __eq__(other):
+        """``x.__eq__(other) <==> x == other``"""
+
+    def __ne__(other):
+        """``x.__ne__(other) <==> x != other``"""
+
+    def __gt__(other):
+        """``x.__gt__(other) <==> x > other``"""
+
+    def __ge__(other):
+        """``x.__ge__(other) <==> x >= other``"""
+
+    def __add__(other):
+        """``x.__add__(other) <==> x + other``"""
+
+    def __mul__(n):
+        """``x.__mul__(n) <==> x * n``"""
+
+    def __rmul__(n):
+        """``x.__rmul__(n) <==> n * x``"""
+
+    if PY2:
+        def __getslice__(i, j):
+            """``x.__getslice__(i, j) <==> x[i:j]``
+
+            Use of negative indices is not supported.
+
+            Deprecated since Python 2.0 but still a part of `UserList`.
+            """
+
+class IExtendedReadSequence(IReadSequence):
+    """Full read interface for lists"""
+
+    def count(item):
+        """Return number of occurrences of value"""
+
+    def index(item, *args):
+        """index(value, [start, [stop]]) -> int
+
+        Return first index of *value*
+        """
+
+class IUniqueMemberWriteSequence(Interface):
+    """The write contract for a sequence that may enforce unique members"""
+
+    def __setitem__(index, item):
+        """``x.__setitem__(index, item) <==> x[index] = item``
+
+        Declaring this interface does not specify whether `__setitem__`
+        supports slice objects.
+        """
+
+    def __delitem__(index):
+        """``x.__delitem__(index) <==> del x[index]``
+
+        Declaring this interface does not specify whether `__delitem__`
+        supports slice objects.
+        """
+
+    if PY2:
+        def __setslice__(i, j, other):
+            """``x.__setslice__(i, j, other) <==> x[i:j] = other``
+
+            Use of negative indices is not supported.
+
+            Deprecated since Python 2.0 but still a part of `UserList`.
+            """
+
+        def __delslice__(i, j):
+            """``x.__delslice__(i, j) <==> del x[i:j]``
+
+            Use of negative indices is not supported.
+
+            Deprecated since Python 2.0 but still a part of `UserList`.
+            """
+
+    def __iadd__(y):
+        """``x.__iadd__(y) <==> x += y``"""
+
+    def append(item):
+        """Append item to end"""
+
+    def insert(index, item):
+        """Insert item before index"""
+
+    def pop(index=-1):
+        """Remove and return item at index (default last)"""
+
+    def remove(item):
+        """Remove first occurrence of value"""
+
+    def reverse():
+        """Reverse *IN PLACE*"""
+
+    def sort(cmpfunc=None):
+        """Stable sort *IN PLACE*; `cmpfunc(x, y)` -> -1, 0, 1"""
+
+    def extend(iterable):
+        """Extend list by appending elements from the iterable"""
+
+class IWriteSequence(IUniqueMemberWriteSequence):
+    """Full write contract for sequences"""
+
+    def __imul__(n):
+        """``x.__imul__(n) <==> x *= n``"""
+
+class ISequence(IReadSequence, IWriteSequence):
+    """
+    Full sequence contract.
+
+    New code should prefer
+    :class:`~zope.interface.common.collections.IMutableSequence`.
+
+    Compared to that interface, which is implemented by :class:`list`
+    (:class:`~zope.interface.common.builtins.IList`), among others,
+    this interface is missing the following methods:
+
+        - clear
+
+        - count
+
+        - index
+
+    This interface adds the following methods:
+
+        - sort
+    """
diff --git a/contrib/python/zope.interface/py2/zope/interface/declarations.py b/contrib/python/zope.interface/py2/zope/interface/declarations.py
new file mode 100644
index 00000000000..59bd650fdcc
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/declarations.py
@@ -0,0 +1,1313 @@
+##############################################################################
+# Copyright (c) 2003 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""Implementation of interface declarations
+
+There are three flavors of declarations:
+
+  - Declarations are used to simply name declared interfaces.
+
+  - ImplementsDeclarations are used to express the interfaces that a
+    class implements (that instances of the class provides).
+
+    Implements specifications support inheriting interfaces.
+
+  - ProvidesDeclarations are used to express interfaces directly
+    provided by objects.
+
+"""
+__docformat__ = 'restructuredtext'
+
+import sys
+from types import FunctionType
+from types import MethodType
+from types import ModuleType
+import weakref
+
+from zope.interface.advice import addClassAdvisor
+from zope.interface.interface import Interface
+from zope.interface.interface import InterfaceClass
+from zope.interface.interface import SpecificationBase
+from zope.interface.interface import Specification
+from zope.interface.interface import NameAndModuleComparisonMixin
+from zope.interface._compat import CLASS_TYPES as DescriptorAwareMetaClasses
+from zope.interface._compat import PYTHON3
+from zope.interface._compat import _use_c_impl
+
+__all__ = [
+    # None. The public APIs of this module are
+    # re-exported from zope.interface directly.
+]
+
+# pylint:disable=too-many-lines
+
+# Registry of class-implementation specifications
+BuiltinImplementationSpecifications = {}
+
+_ADVICE_ERROR = ('Class advice impossible in Python3.  '
+                 'Use the @%s class decorator instead.')
+
+_ADVICE_WARNING = ('The %s API is deprecated, and will not work in Python3  '
+                   'Use the @%s class decorator instead.')
+
+def _next_super_class(ob):
+    # When ``ob`` is an instance of ``super``, return
+    # the next class in the MRO that we should actually be
+    # looking at. Watch out for diamond inheritance!
+    self_class = ob.__self_class__
+    class_that_invoked_super = ob.__thisclass__
+    complete_mro = self_class.__mro__
+    next_class = complete_mro[complete_mro.index(class_that_invoked_super) + 1]
+    return next_class
+
+class named(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __call__(self, ob):
+        ob.__component_name__ = self.name
+        return ob
+
+
+class Declaration(Specification):
+    """Interface declarations"""
+
+    __slots__ = ()
+
+    def __init__(self, *bases):
+        Specification.__init__(self, _normalizeargs(bases))
+
+    def __contains__(self, interface):
+        """Test whether an interface is in the specification
+        """
+
+        return self.extends(interface) and interface in self.interfaces()
+
+    def __iter__(self):
+        """Return an iterator for the interfaces in the specification
+        """
+        return self.interfaces()
+
+    def flattened(self):
+        """Return an iterator of all included and extended interfaces
+        """
+        return iter(self.__iro__)
+
+    def __sub__(self, other):
+        """Remove interfaces from a specification
+        """
+        return Declaration(*[
+            i for i in self.interfaces()
+            if not [
+                j
+                for j in other.interfaces()
+                if i.extends(j, 0) # non-strict extends
+            ]
+        ])
+
+    def __add__(self, other):
+        """
+        Add two specifications or a specification and an interface
+        and produce a new declaration.
+
+        .. versionchanged:: 5.4.0
+           Now tries to preserve a consistent resolution order. Interfaces
+           being added to this object are added to the front of the resulting resolution
+           order if they already extend an interface in this object. Previously,
+           they were always added to the end of the order, which easily resulted in
+           invalid orders.
+        """
+        before = []
+        result = list(self.interfaces())
+        seen = set(result)
+        for i in other.interfaces():
+            if i in seen:
+                continue
+            seen.add(i)
+            if any(i.extends(x) for x in result):
+                # It already extends us, e.g., is a subclass,
+                # so it needs to go at the front of the RO.
+                before.append(i)
+            else:
+                result.append(i)
+        return Declaration(*(before + result))
+
+    # XXX: Is __radd__ needed? No tests break if it's removed.
+    # If it is needed, does it need to handle the C3 ordering differently?
+    # I (JAM) don't *think* it does.
+    __radd__ = __add__
+
+    @staticmethod
+    def _add_interfaces_to_cls(interfaces, cls):
+        # Strip redundant interfaces already provided
+        # by the cls so we don't produce invalid
+        # resolution orders.
+        implemented_by_cls = implementedBy(cls)
+        interfaces = tuple([
+            iface
+            for iface in interfaces
+            if not implemented_by_cls.isOrExtends(iface)
+        ])
+        return interfaces + (implemented_by_cls,)
+
+    @staticmethod
+    def _argument_names_for_repr(interfaces):
+        # These don't actually have to be interfaces, they could be other
+        # Specification objects like Implements. Also, the first
+        # one is typically/nominally the cls.
+        ordered_names = []
+        names = set()
+        for iface in interfaces:
+            duplicate_transform = repr
+            if isinstance(iface, InterfaceClass):
+                # Special case to get 'foo.bar.IFace'
+                # instead of '<InterfaceClass foo.bar.IFace>'
+                this_name = iface.__name__
+                duplicate_transform = str
+            elif isinstance(iface, type):
+                # Likewise for types. (Ignoring legacy old-style
+                # classes.)
+                this_name = iface.__name__
+                duplicate_transform = _implements_name
+            elif (isinstance(iface, Implements)
+                  and not iface.declared
+                  and iface.inherit in interfaces):
+                # If nothing is declared, there's no need to even print this;
+                # it would just show as ``classImplements(Class)``, and the
+                # ``Class`` has typically already.
+                continue
+            else:
+                this_name = repr(iface)
+
+            already_seen = this_name in names
+            names.add(this_name)
+            if already_seen:
+                this_name = duplicate_transform(iface)
+
+            ordered_names.append(this_name)
+        return ', '.join(ordered_names)
+
+
+class _ImmutableDeclaration(Declaration):
+    # A Declaration that is immutable. Used as a singleton to
+    # return empty answers for things like ``implementedBy``.
+    # We have to define the actual singleton after normalizeargs
+    # is defined, and that in turn is defined after InterfaceClass and
+    # Implements.
+
+    __slots__ = ()
+
+    __instance = None
+
+    def __new__(cls):
+        if _ImmutableDeclaration.__instance is None:
+            _ImmutableDeclaration.__instance = object.__new__(cls)
+        return _ImmutableDeclaration.__instance
+
+    def __reduce__(self):
+        return "_empty"
+
+    @property
+    def __bases__(self):
+        return ()
+
+    @__bases__.setter
+    def __bases__(self, new_bases):
+        # We expect the superclass constructor to set ``self.__bases__ = ()``.
+        # Rather than attempt to special case that in the constructor and allow
+        # setting __bases__ only at that time, it's easier to just allow setting
+        # the empty tuple at any time. That makes ``x.__bases__ = x.__bases__`` a nice
+        # no-op too. (Skipping the superclass constructor altogether is a recipe
+        # for maintenance headaches.)
+        if new_bases != ():
+            raise TypeError("Cannot set non-empty bases on shared empty Declaration.")
+
+    # As the immutable empty declaration, we cannot be changed.
+    # This means there's no logical reason for us to have dependents
+    # or subscriptions: we'll never notify them. So there's no need for
+    # us to keep track of any of that.
+    @property
+    def dependents(self):
+        return {}
+
+    changed = subscribe = unsubscribe = lambda self, _ignored: None
+
+    def interfaces(self):
+        # An empty iterator
+        return iter(())
+
+    def extends(self, interface, strict=True):
+        return interface is self._ROOT
+
+    def get(self, name, default=None):
+        return default
+
+    def weakref(self, callback=None):
+        # We're a singleton, we never go away. So there's no need to return
+        # distinct weakref objects here; their callbacks will never
+        # be called. Instead, we only need to return a callable that
+        # returns ourself. The easiest one is to return _ImmutableDeclaration
+        # itself; testing on Python 3.8 shows that's faster than a function that
+        # returns _empty. (Remember, one goal is to avoid allocating any
+        # object, and that includes a method.)
+        return _ImmutableDeclaration
+
+    @property
+    def _v_attrs(self):
+        # _v_attrs is not a public, documented property, but some client
+        # code uses it anyway as a convenient place to cache things. To keep
+        # the empty declaration truly immutable, we must ignore that. That includes
+        # ignoring assignments as well.
+        return {}
+
+    @_v_attrs.setter
+    def _v_attrs(self, new_attrs):
+        pass
+
+
+##############################################################################
+#
+# Implementation specifications
+#
+# These specify interfaces implemented by instances of classes
+
+class Implements(NameAndModuleComparisonMixin,
+                 Declaration):
+    # Inherit from NameAndModuleComparisonMixin to be
+    # mutually comparable with InterfaceClass objects.
+    # (The two must be mutually comparable to be able to work in e.g., BTrees.)
+    # Instances of this class generally don't have a __module__ other than
+    # `zope.interface.declarations`, whereas they *do* have a __name__ that is the
+    # fully qualified name of the object they are representing.
+
+    # Note, though, that equality and hashing are still identity based. This
+    # accounts for things like nested objects that have the same name (typically
+    # only in tests) and is consistent with pickling. As far as comparisons to InterfaceClass
+    # goes, we'll never have equal name and module to those, so we're still consistent there.
+    # Instances of this class are essentially intended to be unique and are
+    # heavily cached (note how our __reduce__ handles this) so having identity
+    # based hash and eq should also work.
+
+    # We want equality and hashing to be based on identity. However, we can't actually
+    # implement __eq__/__ne__ to do this because sometimes we get wrapped in a proxy.
+    # We need to let the proxy types implement these methods so they can handle unwrapping
+    # and then rely on: (1) the interpreter automatically changing `implements == proxy` into
+    # `proxy == implements` (which will call proxy.__eq__ to do the unwrapping) and then
+    # (2) the default equality and hashing semantics being identity based.
+
+    # class whose specification should be used as additional base
+    inherit = None
+
+    # interfaces actually declared for a class
+    declared = ()
+
+    # Weak cache of {class: <implements>} for super objects.
+    # Created on demand. These are rare, as of 5.0 anyway. Using a class
+    # level default doesn't take space in instances. Using _v_attrs would be
+    # another place to store this without taking space unless needed.
+    _super_cache = None
+
+    __name__ = '?'
+
+    @classmethod
+    def named(cls, name, *bases):
+        # Implementation method: Produce an Implements interface with
+        # a fully fleshed out __name__ before calling the constructor, which
+        # sets bases to the given interfaces and which may pass this object to
+        # other objects (e.g., to adjust dependents). If they're sorting or comparing
+        # by name, this needs to be set.
+        inst = cls.__new__(cls)
+        inst.__name__ = name
+        inst.__init__(*bases)
+        return inst
+
+    def changed(self, originally_changed):
+        try:
+            del self._super_cache
+        except AttributeError:
+            pass
+        return super(Implements, self).changed(originally_changed)
+
+    def __repr__(self):
+        if self.inherit:
+            name = getattr(self.inherit, '__name__', None) or _implements_name(self.inherit)
+        else:
+            name = self.__name__
+        declared_names = self._argument_names_for_repr(self.declared)
+        if declared_names:
+            declared_names = ', ' + declared_names
+        return 'classImplements(%s%s)' % (name, declared_names)
+
+    def __reduce__(self):
+        return implementedBy, (self.inherit, )
+
+
+def _implements_name(ob):
+    # Return the __name__ attribute to be used by its __implemented__
+    # property.
+    # This must be stable for the "same" object across processes
+    # because it is used for sorting. It needn't be unique, though, in cases
+    # like nested classes named Foo created by different functions, because
+    # equality and hashing is still based on identity.
+    # It might be nice to use __qualname__ on Python 3, but that would produce
+    # different values between Py2 and Py3.
+    return (getattr(ob, '__module__', '?') or '?') + \
+        '.' + (getattr(ob, '__name__', '?') or '?')
+
+
+def _implementedBy_super(sup):
+    # TODO: This is now simple enough we could probably implement
+    # in C if needed.
+
+    # If the class MRO is strictly linear, we could just
+    # follow the normal algorithm for the next class in the
+    # search order (e.g., just return
+    # ``implemented_by_next``). But when diamond inheritance
+    # or mixins + interface declarations are present, we have
+    # to consider the whole MRO and compute a new Implements
+    # that excludes the classes being skipped over but
+    # includes everything else.
+    implemented_by_self = implementedBy(sup.__self_class__)
+    cache = implemented_by_self._super_cache # pylint:disable=protected-access
+    if cache is None:
+        cache = implemented_by_self._super_cache = weakref.WeakKeyDictionary()
+
+    key = sup.__thisclass__
+    try:
+        return cache[key]
+    except KeyError:
+        pass
+
+    next_cls = _next_super_class(sup)
+    # For ``implementedBy(cls)``:
+    # .__bases__ is .declared + [implementedBy(b) for b in cls.__bases__]
+    # .inherit is cls
+
+    implemented_by_next = implementedBy(next_cls)
+    mro = sup.__self_class__.__mro__
+    ix_next_cls = mro.index(next_cls)
+    classes_to_keep = mro[ix_next_cls:]
+    new_bases = [implementedBy(c) for c in classes_to_keep]
+
+    new = Implements.named(
+        implemented_by_self.__name__ + ':' + implemented_by_next.__name__,
+        *new_bases
+    )
+    new.inherit = implemented_by_next.inherit
+    new.declared = implemented_by_next.declared
+    # I don't *think* that new needs to subscribe to ``implemented_by_self``;
+    # it auto-subscribed to its bases, and that should be good enough.
+    cache[key] = new
+
+    return new
+
+
+@_use_c_impl
+def implementedBy(cls): # pylint:disable=too-many-return-statements,too-many-branches
+    """Return the interfaces implemented for a class' instances
+
+      The value returned is an `~zope.interface.interfaces.IDeclaration`.
+    """
+    try:
+        if isinstance(cls, super):
+            # Yes, this needs to be inside the try: block. Some objects
+            # like security proxies even break isinstance.
+            return _implementedBy_super(cls)
+
+        spec = cls.__dict__.get('__implemented__')
+    except AttributeError:
+
+        # we can't get the class dict. This is probably due to a
+        # security proxy.  If this is the case, then probably no
+        # descriptor was installed for the class.
+
+        # We don't want to depend directly on zope.security in
+        # zope.interface, but we'll try to make reasonable
+        # accommodations in an indirect way.
+
+        # We'll check to see if there's an implements:
+
+        spec = getattr(cls, '__implemented__', None)
+        if spec is None:
+            # There's no spec stred in the class. Maybe its a builtin:
+            spec = BuiltinImplementationSpecifications.get(cls)
+            if spec is not None:
+                return spec
+            return _empty
+
+        if spec.__class__ == Implements:
+            # we defaulted to _empty or there was a spec. Good enough.
+            # Return it.
+            return spec
+
+        # TODO: need old style __implements__ compatibility?
+        # Hm, there's an __implemented__, but it's not a spec. Must be
+        # an old-style declaration. Just compute a spec for it
+        return Declaration(*_normalizeargs((spec, )))
+
+    if isinstance(spec, Implements):
+        return spec
+
+    if spec is None:
+        spec = BuiltinImplementationSpecifications.get(cls)
+        if spec is not None:
+            return spec
+
+    # TODO: need old style __implements__ compatibility?
+    spec_name = _implements_name(cls)
+    if spec is not None:
+        # old-style __implemented__ = foo declaration
+        spec = (spec, ) # tuplefy, as it might be just an int
+        spec = Implements.named(spec_name, *_normalizeargs(spec))
+        spec.inherit = None    # old-style implies no inherit
+        del cls.__implemented__ # get rid of the old-style declaration
+    else:
+        try:
+            bases = cls.__bases__
+        except AttributeError:
+            if not callable(cls):
+                raise TypeError("ImplementedBy called for non-factory", cls)
+            bases = ()
+
+        spec = Implements.named(spec_name, *[implementedBy(c) for c in bases])
+        spec.inherit = cls
+
+    try:
+        cls.__implemented__ = spec
+        if not hasattr(cls, '__providedBy__'):
+            cls.__providedBy__ = objectSpecificationDescriptor
+
+        if (isinstance(cls, DescriptorAwareMetaClasses)
+                and '__provides__' not in cls.__dict__):
+            # Make sure we get a __provides__ descriptor
+            cls.__provides__ = ClassProvides(
+                cls,
+                getattr(cls, '__class__', type(cls)),
+                )
+
+    except TypeError:
+        if not isinstance(cls, type):
+            raise TypeError("ImplementedBy called for non-type", cls)
+        BuiltinImplementationSpecifications[cls] = spec
+
+    return spec
+
+
+def classImplementsOnly(cls, *interfaces):
+    """
+    Declare the only interfaces implemented by instances of a class
+
+    The arguments after the class are one or more interfaces or interface
+    specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+    The interfaces given (including the interfaces in the specifications)
+    replace any previous declarations, *including* inherited definitions. If you
+    wish to preserve inherited declarations, you can pass ``implementedBy(cls)``
+    in *interfaces*. This can be used to alter the interface resolution order.
+    """
+    spec = implementedBy(cls)
+    # Clear out everything inherited. It's important to
+    # also clear the bases right now so that we don't improperly discard
+    # interfaces that are already implemented by *old* bases that we're
+    # about to get rid of.
+    spec.declared = ()
+    spec.inherit = None
+    spec.__bases__ = ()
+    _classImplements_ordered(spec, interfaces, ())
+
+
+def classImplements(cls, *interfaces):
+    """
+    Declare additional interfaces implemented for instances of a class
+
+    The arguments after the class are one or more interfaces or
+    interface specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+    The interfaces given (including the interfaces in the specifications)
+    are added to any interfaces previously declared. An effort is made to
+    keep a consistent C3 resolution order, but this cannot be guaranteed.
+
+    .. versionchanged:: 5.0.0
+       Each individual interface in *interfaces* may be added to either the
+       beginning or end of the list of interfaces declared for *cls*,
+       based on inheritance, in order to try to maintain a consistent
+       resolution order. Previously, all interfaces were added to the end.
+    .. versionchanged:: 5.1.0
+       If *cls* is already declared to implement an interface (or derived interface)
+       in *interfaces* through inheritance, the interface is ignored. Previously, it
+       would redundantly be made direct base of *cls*, which often produced inconsistent
+       interface resolution orders. Now, the order will be consistent, but may change.
+       Also, if the ``__bases__`` of the *cls* are later changed, the *cls* will no
+       longer be considered to implement such an interface (changing the ``__bases__`` of *cls*
+       has never been supported).
+    """
+    spec = implementedBy(cls)
+    interfaces = tuple(_normalizeargs(interfaces))
+
+    before = []
+    after = []
+
+    # Take steps to try to avoid producing an invalid resolution
+    # order, while still allowing for BWC (in the past, we always
+    # appended)
+    for iface in interfaces:
+        for b in spec.declared:
+            if iface.extends(b):
+                before.append(iface)
+                break
+        else:
+            after.append(iface)
+    _classImplements_ordered(spec, tuple(before), tuple(after))
+
+
+def classImplementsFirst(cls, iface):
+    """
+    Declare that instances of *cls* additionally provide *iface*.
+
+    The second argument is an interface or interface specification.
+    It is added as the highest priority (first in the IRO) interface;
+    no attempt is made to keep a consistent resolution order.
+
+    .. versionadded:: 5.0.0
+    """
+    spec = implementedBy(cls)
+    _classImplements_ordered(spec, (iface,), ())
+
+
+def _classImplements_ordered(spec, before=(), after=()):
+    # Elide everything already inherited.
+    # Except, if it is the root, and we don't already declare anything else
+    # that would imply it, allow the root through. (TODO: When we disallow non-strict
+    # IRO, this part of the check can be removed because it's not possible to re-declare
+    # like that.)
+    before = [
+        x
+        for x in before
+        if not spec.isOrExtends(x) or (x is Interface and not spec.declared)
+    ]
+    after = [
+        x
+        for x in after
+        if not spec.isOrExtends(x) or (x is Interface and not spec.declared)
+    ]
+
+    # eliminate duplicates
+    new_declared = []
+    seen = set()
+    for l in before, spec.declared, after:
+        for b in l:
+            if b not in seen:
+                new_declared.append(b)
+                seen.add(b)
+
+    spec.declared = tuple(new_declared)
+
+    # compute the bases
+    bases = new_declared # guaranteed no dupes
+
+    if spec.inherit is not None:
+        for c in spec.inherit.__bases__:
+            b = implementedBy(c)
+            if b not in seen:
+                seen.add(b)
+                bases.append(b)
+
+    spec.__bases__ = tuple(bases)
+
+
+def _implements_advice(cls):
+    interfaces, do_classImplements = cls.__dict__['__implements_advice_data__']
+    del cls.__implements_advice_data__
+    do_classImplements(cls, *interfaces)
+    return cls
+
+
+class implementer(object):
+    """
+    Declare the interfaces implemented by instances of a class.
+
+    This function is called as a class decorator.
+
+    The arguments are one or more interfaces or interface
+    specifications (`~zope.interface.interfaces.IDeclaration`
+    objects).
+
+    The interfaces given (including the interfaces in the
+    specifications) are added to any interfaces previously declared,
+    unless the interface is already implemented.
+
+    Previous declarations include declarations for base classes unless
+    implementsOnly was used.
+
+    This function is provided for convenience. It provides a more
+    convenient way to call `classImplements`. For example::
+
+        @implementer(I1)
+        class C(object):
+            pass
+
+    is equivalent to calling::
+
+        classImplements(C, I1)
+
+    after the class has been created.
+
+    .. seealso:: `classImplements`
+       The change history provided there applies to this function too.
+    """
+    __slots__ = ('interfaces',)
+
+    def __init__(self, *interfaces):
+        self.interfaces = interfaces
+
+    def __call__(self, ob):
+        if isinstance(ob, DescriptorAwareMetaClasses):
+            # This is the common branch for new-style (object) and
+            # on Python 2 old-style classes.
+            classImplements(ob, *self.interfaces)
+            return ob
+
+        spec_name = _implements_name(ob)
+        spec = Implements.named(spec_name, *self.interfaces)
+        try:
+            ob.__implemented__ = spec
+        except AttributeError:
+            raise TypeError("Can't declare implements", ob)
+        return ob
+
+class implementer_only(object):
+    """Declare the only interfaces implemented by instances of a class
+
+      This function is called as a class decorator.
+
+      The arguments are one or more interfaces or interface
+      specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+      Previous declarations including declarations for base classes
+      are overridden.
+
+      This function is provided for convenience. It provides a more
+      convenient way to call `classImplementsOnly`. For example::
+
+        @implementer_only(I1)
+        class C(object): pass
+
+      is equivalent to calling::
+
+        classImplementsOnly(I1)
+
+      after the class has been created.
+      """
+
+    def __init__(self, *interfaces):
+        self.interfaces = interfaces
+
+    def __call__(self, ob):
+        if isinstance(ob, (FunctionType, MethodType)):
+            # XXX Does this decorator make sense for anything but classes?
+            # I don't think so. There can be no inheritance of interfaces
+            # on a method or function....
+            raise ValueError('The implementer_only decorator is not '
+                             'supported for methods or functions.')
+
+        # Assume it's a class:
+        classImplementsOnly(ob, *self.interfaces)
+        return ob
+
+def _implements(name, interfaces, do_classImplements):
+    # This entire approach is invalid under Py3K.  Don't even try to fix
+    # the coverage for this block there. :(
+    frame = sys._getframe(2) # pylint:disable=protected-access
+    locals = frame.f_locals # pylint:disable=redefined-builtin
+
+    # Try to make sure we were called from a class def. In 2.2.0 we can't
+    # check for __module__ since it doesn't seem to be added to the locals
+    # until later on.
+    if locals is frame.f_globals or '__module__' not in locals:
+        raise TypeError(name+" can be used only from a class definition.")
+
+    if '__implements_advice_data__' in locals:
+        raise TypeError(name+" can be used only once in a class definition.")
+
+    locals['__implements_advice_data__'] = interfaces, do_classImplements
+    addClassAdvisor(_implements_advice, depth=3)
+
+def implements(*interfaces):
+    """
+    Declare interfaces implemented by instances of a class.
+
+    .. deprecated:: 5.0
+        This only works for Python 2. The `implementer` decorator
+        is preferred for all versions.
+
+    This function is called in a class definition.
+
+    The arguments are one or more interfaces or interface
+    specifications (`~zope.interface.interfaces.IDeclaration`
+    objects).
+
+    The interfaces given (including the interfaces in the
+    specifications) are added to any interfaces previously declared.
+
+    Previous declarations include declarations for base classes unless
+    `implementsOnly` was used.
+
+    This function is provided for convenience. It provides a more
+    convenient way to call `classImplements`. For example::
+
+        implements(I1)
+
+    is equivalent to calling::
+
+        classImplements(C, I1)
+
+    after the class has been created.
+    """
+    # This entire approach is invalid under Py3K.  Don't even try to fix
+    # the coverage for this block there. :(
+    if PYTHON3:
+        raise TypeError(_ADVICE_ERROR % 'implementer')
+    _implements("implements", interfaces, classImplements)
+
+def implementsOnly(*interfaces):
+    """Declare the only interfaces implemented by instances of a class
+
+      This function is called in a class definition.
+
+      The arguments are one or more interfaces or interface
+      specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+      Previous declarations including declarations for base classes
+      are overridden.
+
+      This function is provided for convenience. It provides a more
+      convenient way to call `classImplementsOnly`. For example::
+
+        implementsOnly(I1)
+
+      is equivalent to calling::
+
+        classImplementsOnly(I1)
+
+      after the class has been created.
+    """
+    # This entire approach is invalid under Py3K.  Don't even try to fix
+    # the coverage for this block there. :(
+    if PYTHON3:
+        raise TypeError(_ADVICE_ERROR % 'implementer_only')
+    _implements("implementsOnly", interfaces, classImplementsOnly)
+
+##############################################################################
+#
+# Instance declarations
+
+class Provides(Declaration):  # Really named ProvidesClass
+    """Implement ``__provides__``, the instance-specific specification
+
+    When an object is pickled, we pickle the interfaces that it implements.
+    """
+
+    def __init__(self, cls, *interfaces):
+        self.__args = (cls, ) + interfaces
+        self._cls = cls
+        Declaration.__init__(self, *self._add_interfaces_to_cls(interfaces, cls))
+
+    # Added to by ``moduleProvides``, et al
+    _v_module_names = ()
+
+    def __repr__(self):
+        # The typical way to create instances of this
+        # object is via calling ``directlyProvides(...)`` or ``alsoProvides()``,
+        # but that's not the only way. Proxies, for example,
+        # directly use the ``Provides(...)`` function (which is the
+        # more generic method, and what we pickle as). We're after the most
+        # readable, useful repr in the common case, so we use the most
+        # common name.
+        #
+        # We also cooperate with ``moduleProvides`` to attempt to do the
+        # right thing for that API. See it for details.
+        function_name = 'directlyProvides'
+        if self._cls is ModuleType and self._v_module_names:
+            # See notes in ``moduleProvides``/``directlyProvides``
+            providing_on_module = True
+            interfaces = self.__args[1:]
+        else:
+            providing_on_module = False
+            interfaces = (self._cls,) + self.__bases__
+        ordered_names = self._argument_names_for_repr(interfaces)
+        if providing_on_module:
+            mod_names = self._v_module_names
+            if len(mod_names) == 1:
+                mod_names = "sys.modules[%r]" % mod_names[0]
+            ordered_names = (
+                '%s, ' % (mod_names,)
+            ) + ordered_names
+        return "%s(%s)" % (
+            function_name,
+            ordered_names,
+        )
+
+    def __reduce__(self):
+        # This reduces to the Provides *function*, not
+        # this class.
+        return Provides, self.__args
+
+    __module__ = 'zope.interface'
+
+    def __get__(self, inst, cls):
+        """Make sure that a class __provides__ doesn't leak to an instance
+        """
+        if inst is None and cls is self._cls:
+            # We were accessed through a class, so we are the class'
+            # provides spec. Just return this object, but only if we are
+            # being called on the same class that we were defined for:
+            return self
+
+        raise AttributeError('__provides__')
+
+ProvidesClass = Provides
+
+# Registry of instance declarations
+# This is a memory optimization to allow objects to share specifications.
+InstanceDeclarations = weakref.WeakValueDictionary()
+
+def Provides(*interfaces): # pylint:disable=function-redefined
+    """Cache instance declarations
+
+      Instance declarations are shared among instances that have the same
+      declaration. The declarations are cached in a weak value dictionary.
+    """
+    spec = InstanceDeclarations.get(interfaces)
+    if spec is None:
+        spec = ProvidesClass(*interfaces)
+        InstanceDeclarations[interfaces] = spec
+
+    return spec
+
+Provides.__safe_for_unpickling__ = True
+
+
+def directlyProvides(object, *interfaces): # pylint:disable=redefined-builtin
+    """Declare interfaces declared directly for an object
+
+      The arguments after the object are one or more interfaces or interface
+      specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+      The interfaces given (including the interfaces in the specifications)
+      replace interfaces previously declared for the object.
+    """
+    cls = getattr(object, '__class__', None)
+    if cls is not None and getattr(cls, '__class__', None) is cls:
+        # It's a meta class (well, at least it it could be an extension class)
+        # Note that we can't get here from Py3k tests:  there is no normal
+        # class which isn't descriptor aware.
+        if not isinstance(object,
+                          DescriptorAwareMetaClasses):
+            raise TypeError("Attempt to make an interface declaration on a "
+                            "non-descriptor-aware class")
+
+    interfaces = _normalizeargs(interfaces)
+    if cls is None:
+        cls = type(object)
+
+    issub = False
+    for damc in DescriptorAwareMetaClasses:
+        if issubclass(cls, damc):
+            issub = True
+            break
+    if issub:
+        # we have a class or type.  We'll use a special descriptor
+        # that provides some extra caching
+        object.__provides__ = ClassProvides(object, cls, *interfaces)
+    else:
+        provides = object.__provides__ = Provides(cls, *interfaces)
+        # See notes in ``moduleProvides``.
+        if issubclass(cls, ModuleType) and hasattr(object, '__name__'):
+            provides._v_module_names += (object.__name__,)
+
+
+
+def alsoProvides(object, *interfaces): # pylint:disable=redefined-builtin
+    """Declare interfaces declared directly for an object
+
+    The arguments after the object are one or more interfaces or interface
+    specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+    The interfaces given (including the interfaces in the specifications) are
+    added to the interfaces previously declared for the object.
+    """
+    directlyProvides(object, directlyProvidedBy(object), *interfaces)
+
+
+def noLongerProvides(object, interface): # pylint:disable=redefined-builtin
+    """ Removes a directly provided interface from an object.
+    """
+    directlyProvides(object, directlyProvidedBy(object) - interface)
+    if interface.providedBy(object):
+        raise ValueError("Can only remove directly provided interfaces.")
+
+
+@_use_c_impl
+class ClassProvidesBase(SpecificationBase):
+
+    __slots__ = (
+        '_cls',
+        '_implements',
+    )
+
+    def __get__(self, inst, cls):
+        # member slots are set by subclass
+        # pylint:disable=no-member
+        if cls is self._cls:
+            # We only work if called on the class we were defined for
+
+            if inst is None:
+                # We were accessed through a class, so we are the class'
+                # provides spec. Just return this object as is:
+                return self
+
+            return self._implements
+
+        raise AttributeError('__provides__')
+
+
+class ClassProvides(Declaration, ClassProvidesBase):
+    """Special descriptor for class ``__provides__``
+
+    The descriptor caches the implementedBy info, so that
+    we can get declarations for objects without instance-specific
+    interfaces a bit quicker.
+    """
+
+    __slots__ = (
+        '__args',
+    )
+
+    def __init__(self, cls, metacls, *interfaces):
+        self._cls = cls
+        self._implements = implementedBy(cls)
+        self.__args = (cls, metacls, ) + interfaces
+        Declaration.__init__(self, *self._add_interfaces_to_cls(interfaces, metacls))
+
+    def __repr__(self):
+        # There are two common ways to get instances of this object:
+        # The most interesting way is calling ``@provider(..)`` as a decorator
+        # of a class; this is the same as calling ``directlyProvides(cls, ...)``.
+        #
+        # The other way is by default: anything that invokes ``implementedBy(x)``
+        # will wind up putting an instance in ``type(x).__provides__``; this includes
+        # the ``@implementer(...)`` decorator. Those instances won't have any
+        # interfaces.
+        #
+        # Thus, as our repr, we go with the ``directlyProvides()`` syntax.
+        interfaces = (self._cls, ) + self.__args[2:]
+        ordered_names = self._argument_names_for_repr(interfaces)
+        return "directlyProvides(%s)" % (ordered_names,)
+
+    def __reduce__(self):
+        return self.__class__, self.__args
+
+    # Copy base-class method for speed
+    __get__ = ClassProvidesBase.__get__
+
+
+def directlyProvidedBy(object): # pylint:disable=redefined-builtin
+    """Return the interfaces directly provided by the given object
+
+    The value returned is an `~zope.interface.interfaces.IDeclaration`.
+    """
+    provides = getattr(object, "__provides__", None)
+    if (
+            provides is None # no spec
+            # We might have gotten the implements spec, as an
+            # optimization. If so, it's like having only one base, that we
+            # lop off to exclude class-supplied declarations:
+            or isinstance(provides, Implements)
+    ):
+        return _empty
+
+    # Strip off the class part of the spec:
+    return Declaration(provides.__bases__[:-1])
+
+
+def classProvides(*interfaces):
+    """Declare interfaces provided directly by a class
+
+      This function is called in a class definition.
+
+      The arguments are one or more interfaces or interface specifications
+      (`~zope.interface.interfaces.IDeclaration` objects).
+
+      The given interfaces (including the interfaces in the specifications)
+      are used to create the class's direct-object interface specification.
+      An error will be raised if the module class has an direct interface
+      specification. In other words, it is an error to call this function more
+      than once in a class definition.
+
+      Note that the given interfaces have nothing to do with the interfaces
+      implemented by instances of the class.
+
+      This function is provided for convenience. It provides a more convenient
+      way to call `directlyProvides` for a class. For example::
+
+        classProvides(I1)
+
+      is equivalent to calling::
+
+        directlyProvides(theclass, I1)
+
+      after the class has been created.
+    """
+    # This entire approach is invalid under Py3K.  Don't even try to fix
+    # the coverage for this block there. :(
+
+    if PYTHON3:
+        raise TypeError(_ADVICE_ERROR % 'provider')
+
+    frame = sys._getframe(1) # pylint:disable=protected-access
+    locals = frame.f_locals # pylint:disable=redefined-builtin
+
+    # Try to make sure we were called from a class def
+    if (locals is frame.f_globals) or ('__module__' not in locals):
+        raise TypeError("classProvides can be used only from a "
+                        "class definition.")
+
+    if '__provides__' in locals:
+        raise TypeError(
+            "classProvides can only be used once in a class definition.")
+
+    locals["__provides__"] = _normalizeargs(interfaces)
+
+    addClassAdvisor(_classProvides_advice, depth=2)
+
+def _classProvides_advice(cls):
+    # This entire approach is invalid under Py3K.  Don't even try to fix
+    # the coverage for this block there. :(
+    interfaces = cls.__dict__['__provides__']
+    del cls.__provides__
+    directlyProvides(cls, *interfaces)
+    return cls
+
+
+class provider(object):
+    """Class decorator version of classProvides"""
+
+    def __init__(self, *interfaces):
+        self.interfaces = interfaces
+
+    def __call__(self, ob):
+        directlyProvides(ob, *self.interfaces)
+        return ob
+
+
+def moduleProvides(*interfaces):
+    """Declare interfaces provided by a module
+
+    This function is used in a module definition.
+
+    The arguments are one or more interfaces or interface specifications
+    (`~zope.interface.interfaces.IDeclaration` objects).
+
+    The given interfaces (including the interfaces in the specifications) are
+    used to create the module's direct-object interface specification.  An
+    error will be raised if the module already has an interface specification.
+    In other words, it is an error to call this function more than once in a
+    module definition.
+
+    This function is provided for convenience. It provides a more convenient
+    way to call directlyProvides. For example::
+
+      moduleProvides(I1)
+
+    is equivalent to::
+
+      directlyProvides(sys.modules[__name__], I1)
+    """
+    frame = sys._getframe(1) # pylint:disable=protected-access
+    locals = frame.f_locals # pylint:disable=redefined-builtin
+
+    # Try to make sure we were called from a module body
+    if (locals is not frame.f_globals) or ('__name__' not in locals):
+        raise TypeError(
+            "moduleProvides can only be used from a module definition.")
+
+    if '__provides__' in locals:
+        raise TypeError(
+            "moduleProvides can only be used once in a module definition.")
+
+    # Note: This is cached based on the key ``(ModuleType, *interfaces)``;
+    # One consequence is that any module that provides the same interfaces
+    # gets the same ``__repr__``, meaning that you can't tell what module
+    # such a declaration came from. Adding the module name to ``_v_module_names``
+    # attempts to correct for this; it works in some common situations, but fails
+    # (1) after pickling (the data is lost) and (2) if declarations are
+    # actually shared and (3) if the alternate spelling of ``directlyProvides()``
+    # is used. Problem (3)  is fixed by cooperating with ``directlyProvides``
+    # to maintain this information, and problem (2) is worked around by
+    # printing all the names, but (1) is unsolvable without introducing
+    # new classes or changing the stored data...but it doesn't actually matter,
+    # because ``ModuleType`` can't be pickled!
+    p = locals["__provides__"] = Provides(ModuleType,
+                                          *_normalizeargs(interfaces))
+    p._v_module_names += (locals['__name__'],)
+
+
+##############################################################################
+#
+# Declaration querying support
+
+# XXX:  is this a fossil?  Nobody calls it, no unit tests exercise it, no
+#       doctests import it, and the package __init__ doesn't import it.
+#       (Answer: Versions of zope.container prior to 4.4.0 called this,
+#        and zope.proxy.decorator up through at least 4.3.5 called this.)
+def ObjectSpecification(direct, cls):
+    """Provide object specifications
+
+    These combine information for the object and for it's classes.
+    """
+    return Provides(cls, direct) # pragma: no cover fossil
+
+@_use_c_impl
+def getObjectSpecification(ob):
+    try:
+        provides = ob.__provides__
+    except AttributeError:
+        provides = None
+
+    if provides is not None:
+        if isinstance(provides, SpecificationBase):
+            return provides
+
+    try:
+        cls = ob.__class__
+    except AttributeError:
+        # We can't get the class, so just consider provides
+        return _empty
+    return implementedBy(cls)
+
+
+@_use_c_impl
+def providedBy(ob):
+    """
+    Return the interfaces provided by *ob*.
+
+    If *ob* is a :class:`super` object, then only interfaces implemented
+    by the remainder of the classes in the method resolution order are
+    considered. Interfaces directly provided by the object underlying *ob*
+    are not.
+    """
+    # Here we have either a special object, an old-style declaration
+    # or a descriptor
+
+    # Try to get __providedBy__
+    try:
+        if isinstance(ob, super): # Some objects raise errors on isinstance()
+            return implementedBy(ob)
+
+        r = ob.__providedBy__
+    except AttributeError:
+        # Not set yet. Fall back to lower-level thing that computes it
+        return getObjectSpecification(ob)
+
+    try:
+        # We might have gotten a descriptor from an instance of a
+        # class (like an ExtensionClass) that doesn't support
+        # descriptors.  We'll make sure we got one by trying to get
+        # the only attribute, which all specs have.
+        r.extends
+    except AttributeError:
+
+        # The object's class doesn't understand descriptors.
+        # Sigh. We need to get an object descriptor, but we have to be
+        # careful.  We want to use the instance's __provides__, if
+        # there is one, but only if it didn't come from the class.
+
+        try:
+            r = ob.__provides__
+        except AttributeError:
+            # No __provides__, so just fall back to implementedBy
+            return implementedBy(ob.__class__)
+
+        # We need to make sure we got the __provides__ from the
+        # instance. We'll do this by making sure we don't get the same
+        # thing from the class:
+
+        try:
+            cp = ob.__class__.__provides__
+        except AttributeError:
+            # The ob doesn't have a class or the class has no
+            # provides, assume we're done:
+            return r
+
+        if r is cp:
+            # Oops, we got the provides from the class. This means
+            # the object doesn't have it's own. We should use implementedBy
+            return implementedBy(ob.__class__)
+
+    return r
+
+
+@_use_c_impl
+class ObjectSpecificationDescriptor(object):
+    """Implement the ``__providedBy__`` attribute
+
+    The ``__providedBy__`` attribute computes the interfaces provided by
+    an object. If an object has an ``__provides__`` attribute, that is returned.
+    Otherwise, `implementedBy` the *cls* is returned.
+
+    .. versionchanged:: 5.4.0
+       Both the default (C) implementation and the Python implementation
+       now let exceptions raised by accessing ``__provides__`` propagate.
+       Previously, the C version ignored all exceptions.
+    .. versionchanged:: 5.4.0
+       The Python implementation now matches the C implementation and lets
+       a ``__provides__`` of ``None`` override what the class is declared to
+       implement.
+    """
+
+    def __get__(self, inst, cls):
+        """Get an object specification for an object
+        """
+        if inst is None:
+            return getObjectSpecification(cls)
+
+        try:
+            return inst.__provides__
+        except AttributeError:
+            return implementedBy(cls)
+
+
+##############################################################################
+
+def _normalizeargs(sequence, output=None):
+    """Normalize declaration arguments
+
+    Normalization arguments might contain Declarions, tuples, or single
+    interfaces.
+
+    Anything but individual interfaces or implements specs will be expanded.
+    """
+    if output is None:
+        output = []
+
+    cls = sequence.__class__
+    if InterfaceClass in cls.__mro__ or Implements in cls.__mro__:
+        output.append(sequence)
+    else:
+        for v in sequence:
+            _normalizeargs(v, output)
+
+    return output
+
+_empty = _ImmutableDeclaration()
+
+objectSpecificationDescriptor = ObjectSpecificationDescriptor()
diff --git a/contrib/python/zope.interface/py2/zope/interface/document.py b/contrib/python/zope.interface/py2/zope/interface/document.py
new file mode 100644
index 00000000000..309bb575ee6
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/document.py
@@ -0,0 +1,124 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+""" Pretty-Print an Interface object as structured text (Yum)
+
+This module provides a function, asStructuredText, for rendering an
+interface as structured text.
+"""
+import zope.interface
+
+__all__ = [
+    'asReStructuredText',
+    'asStructuredText',
+]
+
+def asStructuredText(I, munge=0, rst=False):
+    """ Output structured text format.  Note, this will whack any existing
+    'structured' format of the text.
+
+    If `rst=True`, then the output will quote all code as inline literals in
+    accordance with 'reStructuredText' markup principles.
+    """
+
+    if rst:
+        inline_literal = lambda s: "``%s``" % (s,)
+    else:
+        inline_literal = lambda s: s
+
+    r = [inline_literal(I.getName())]
+    outp = r.append
+    level = 1
+
+    if I.getDoc():
+        outp(_justify_and_indent(_trim_doc_string(I.getDoc()), level))
+
+    bases = [base
+             for base in I.__bases__
+             if base is not zope.interface.Interface
+             ]
+    if bases:
+        outp(_justify_and_indent("This interface extends:", level, munge))
+        level += 1
+        for b in bases:
+            item = "o %s" % inline_literal(b.getName())
+            outp(_justify_and_indent(_trim_doc_string(item), level, munge))
+        level -= 1
+
+    namesAndDescriptions = sorted(I.namesAndDescriptions())
+
+    outp(_justify_and_indent("Attributes:", level, munge))
+    level += 1
+    for name, desc in namesAndDescriptions:
+        if not hasattr(desc, 'getSignatureString'):   # ugh...
+            item = "%s -- %s" % (inline_literal(desc.getName()),
+                                 desc.getDoc() or 'no documentation')
+            outp(_justify_and_indent(_trim_doc_string(item), level, munge))
+    level -= 1
+
+    outp(_justify_and_indent("Methods:", level, munge))
+    level += 1
+    for name, desc in namesAndDescriptions:
+        if hasattr(desc, 'getSignatureString'):   # ugh...
+            _call = "%s%s" % (desc.getName(), desc.getSignatureString())
+            item = "%s -- %s" % (inline_literal(_call),
+                                 desc.getDoc() or 'no documentation')
+            outp(_justify_and_indent(_trim_doc_string(item), level, munge))
+
+    return "\n\n".join(r) + "\n\n"
+
+
+def asReStructuredText(I, munge=0):
+    """ Output reStructuredText format.  Note, this will whack any existing
+    'structured' format of the text."""
+    return asStructuredText(I, munge=munge, rst=True)
+
+
+def _trim_doc_string(text):
+    """ Trims a doc string to make it format
+    correctly with structured text. """
+
+    lines = text.replace('\r\n', '\n').split('\n')
+    nlines = [lines.pop(0)]
+    if lines:
+        min_indent = min([len(line) - len(line.lstrip())
+                          for line in lines])
+        for line in lines:
+            nlines.append(line[min_indent:])
+
+    return '\n'.join(nlines)
+
+
+def _justify_and_indent(text, level, munge=0, width=72):
+    """ indent and justify text, rejustify (munge) if specified """
+
+    indent = " " * level
+
+    if munge:
+        lines = []
+        line = indent
+        text = text.split()
+
+        for word in text:
+            line = ' '.join([line, word])
+            if len(line) > width:
+                lines.append(line)
+                line = indent
+        else:
+            lines.append(line)
+
+        return '\n'.join(lines)
+
+    else:
+        return indent + \
+            text.strip().replace("\r\n", "\n") .replace("\n", "\n" + indent)
diff --git a/contrib/python/zope.interface/py2/zope/interface/exceptions.py b/contrib/python/zope.interface/py2/zope/interface/exceptions.py
new file mode 100644
index 00000000000..47c351b25f9
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/exceptions.py
@@ -0,0 +1,275 @@
+##############################################################################
+#
+# Copyright (c) 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interface-specific exceptions
+"""
+
+__all__ = [
+    # Invalid tree
+    'Invalid',
+    'DoesNotImplement',
+    'BrokenImplementation',
+    'BrokenMethodImplementation',
+    'MultipleInvalid',
+    # Other
+    'BadImplements',
+    'InvalidInterface',
+]
+
+class Invalid(Exception):
+    """A specification is violated
+    """
+
+
+class _TargetInvalid(Invalid):
+    # Internal use. Subclass this when you're describing
+    # a particular target object that's invalid according
+    # to a specific interface.
+    #
+    # For backwards compatibility, the *target* and *interface* are
+    # optional, and the signatures are inconsistent in their ordering.
+    #
+    # We deal with the inconsistency in ordering by defining the index
+    # of the two values in ``self.args``. *target* uses a marker object to
+    # distinguish "not given" from "given, but None", because the latter
+    # can be a value that gets passed to validation. For this reason, it must
+    # always be the last argument (we detect absence by the ``IndexError``).
+
+    _IX_INTERFACE = 0
+    _IX_TARGET = 1
+    # The exception to catch when indexing self.args indicating that
+    # an argument was not given. If all arguments are expected,
+    # a subclass should set this to ().
+    _NOT_GIVEN_CATCH = IndexError
+    _NOT_GIVEN = '<Not Given>'
+
+    def _get_arg_or_default(self, ix, default=None):
+        try:
+            return self.args[ix] # pylint:disable=unsubscriptable-object
+        except self._NOT_GIVEN_CATCH:
+            return default
+
+    @property
+    def interface(self):
+        return self._get_arg_or_default(self._IX_INTERFACE)
+
+    @property
+    def target(self):
+        return self._get_arg_or_default(self._IX_TARGET, self._NOT_GIVEN)
+
+    ###
+    # str
+    #
+    # The ``__str__`` of self is implemented by concatenating (%s), in order,
+    # these properties (none of which should have leading or trailing
+    # whitespace):
+    #
+    # - self._str_subject
+    #   Begin the message, including a description of the target.
+    # - self._str_description
+    #   Provide a general description of the type of error, including
+    #   the interface name if possible and relevant.
+    # - self._str_conjunction
+    #   Join the description to the details. Defaults to ": ".
+    # - self._str_details
+    #   Provide details about how this particular instance of the error.
+    # - self._str_trailer
+    #   End the message. Usually just a period.
+    ###
+
+    @property
+    def _str_subject(self):
+        target = self.target
+        if target is self._NOT_GIVEN:
+            return "An object"
+        return "The object %r" % (target,)
+
+    @property
+    def _str_description(self):
+        return "has failed to implement interface %s" % (
+            self.interface or '<Unknown>'
+        )
+
+    _str_conjunction = ": "
+    _str_details = "<unknown>"
+    _str_trailer = '.'
+
+    def __str__(self):
+        return "%s %s%s%s%s" % (
+            self._str_subject,
+            self._str_description,
+            self._str_conjunction,
+            self._str_details,
+            self._str_trailer
+        )
+
+
+class DoesNotImplement(_TargetInvalid):
+    """
+    DoesNotImplement(interface[, target])
+
+    The *target* (optional) does not implement the *interface*.
+
+    .. versionchanged:: 5.0.0
+       Add the *target* argument and attribute, and change the resulting
+       string value of this object accordingly.
+    """
+
+    _str_details = "Does not declaratively implement the interface"
+
+
+class BrokenImplementation(_TargetInvalid):
+    """
+    BrokenImplementation(interface, name[, target])
+
+    The *target* (optional) is missing the attribute *name*.
+
+    .. versionchanged:: 5.0.0
+       Add the *target* argument and attribute, and change the resulting
+       string value of this object accordingly.
+
+       The *name* can either be a simple string or a ``Attribute`` object.
+    """
+
+    _IX_NAME = _TargetInvalid._IX_INTERFACE + 1
+    _IX_TARGET = _IX_NAME + 1
+
+    @property
+    def name(self):
+        return self.args[1] # pylint:disable=unsubscriptable-object
+
+    @property
+    def _str_details(self):
+        return "The %s attribute was not provided" % (
+            repr(self.name) if isinstance(self.name, str) else self.name
+        )
+
+
+class BrokenMethodImplementation(_TargetInvalid):
+    """
+    BrokenMethodImplementation(method, message[, implementation, interface, target])
+
+    The *target* (optional) has a *method* in *implementation* that violates
+    its contract in a way described by *mess*.
+
+    .. versionchanged:: 5.0.0
+       Add the *interface* and *target* argument and attribute,
+       and change the resulting string value of this object accordingly.
+
+       The *method* can either be a simple string or a ``Method`` object.
+
+    .. versionchanged:: 5.0.0
+       If *implementation* is given, then the *message* will have the
+       string "implementation" replaced with an short but informative
+       representation of *implementation*.
+
+    """
+
+    _IX_IMPL = 2
+    _IX_INTERFACE = _IX_IMPL + 1
+    _IX_TARGET = _IX_INTERFACE + 1
+
+    @property
+    def method(self):
+        return self.args[0] # pylint:disable=unsubscriptable-object
+
+    @property
+    def mess(self):
+        return self.args[1] # pylint:disable=unsubscriptable-object
+
+    @staticmethod
+    def __implementation_str(impl):
+        # It could be a callable or some arbitrary object, we don't
+        # know yet.
+        import inspect # Inspect is a heavy-weight dependency, lots of imports
+        try:
+            sig = inspect.signature
+            formatsig = str
+        except AttributeError:
+            sig = inspect.getargspec
+            f = inspect.formatargspec
+            formatsig = lambda sig: f(*sig) # pylint:disable=deprecated-method
+
+        try:
+            sig = sig(impl)
+        except (ValueError, TypeError):
+            # Unable to introspect. Darn.
+            # This could be a non-callable, or a particular builtin,
+            # or a bound method that doesn't even accept 'self', e.g.,
+            # ``Class.method = lambda: None; Class().method``
+            return repr(impl)
+
+        try:
+            name = impl.__qualname__
+        except AttributeError:
+            name = impl.__name__
+
+        return name + formatsig(sig)
+
+    @property
+    def _str_details(self):
+        impl = self._get_arg_or_default(self._IX_IMPL, self._NOT_GIVEN)
+        message = self.mess
+        if impl is not self._NOT_GIVEN and 'implementation' in message:
+            message = message.replace("implementation", '%r')
+            message = message % (self.__implementation_str(impl),)
+
+        return 'The contract of %s is violated because %s' % (
+            repr(self.method) if isinstance(self.method, str) else self.method,
+            message,
+        )
+
+
+class MultipleInvalid(_TargetInvalid):
+    """
+    The *target* has failed to implement the *interface* in
+    multiple ways.
+
+    The failures are described by *exceptions*, a collection of
+    other `Invalid` instances.
+
+    .. versionadded:: 5.0
+    """
+
+    _NOT_GIVEN_CATCH = ()
+
+    def __init__(self, interface, target, exceptions):
+        super(MultipleInvalid, self).__init__(interface, target, tuple(exceptions))
+
+    @property
+    def exceptions(self):
+        return self.args[2] # pylint:disable=unsubscriptable-object
+
+    @property
+    def _str_details(self):
+        # It would be nice to use tabs here, but that
+        # is hard to represent in doctests.
+        return '\n    ' + '\n    '.join(
+            x._str_details.strip() if isinstance(x, _TargetInvalid) else str(x)
+            for x in self.exceptions
+        )
+
+    _str_conjunction = ':' # We don't want a trailing space, messes up doctests
+    _str_trailer = ''
+
+
+class InvalidInterface(Exception):
+    """The interface has invalid contents
+    """
+
+class BadImplements(TypeError):
+    """An implementation assertion is invalid
+
+    because it doesn't contain an interface or a sequence of valid
+    implementation assertions.
+    """
diff --git a/contrib/python/zope.interface/py2/zope/interface/interface.py b/contrib/python/zope.interface/py2/zope/interface/interface.py
new file mode 100644
index 00000000000..74476418b72
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/interface.py
@@ -0,0 +1,1153 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interface object implementation
+"""
+# pylint:disable=protected-access
+import sys
+from types import MethodType
+from types import FunctionType
+import weakref
+
+from zope.interface._compat import _use_c_impl
+from zope.interface._compat import PYTHON2 as PY2
+from zope.interface.exceptions import Invalid
+from zope.interface.ro import ro as calculate_ro
+from zope.interface import ro
+
+__all__ = [
+    # Most of the public API from this module is directly exported
+    # from zope.interface. The only remaining public API intended to
+    # be imported from here should be those few things documented as
+    # such.
+    'InterfaceClass',
+    'Specification',
+    'adapter_hooks',
+]
+
+CO_VARARGS = 4
+CO_VARKEYWORDS = 8
+# Put in the attrs dict of an interface by ``taggedValue`` and ``invariants``
+TAGGED_DATA = '__interface_tagged_values__'
+# Put in the attrs dict of an interface by ``interfacemethod``
+INTERFACE_METHODS = '__interface_methods__'
+
+_decorator_non_return = object()
+_marker = object()
+
+
+
+def invariant(call):
+    f_locals = sys._getframe(1).f_locals
+    tags = f_locals.setdefault(TAGGED_DATA, {})
+    invariants = tags.setdefault('invariants', [])
+    invariants.append(call)
+    return _decorator_non_return
+
+
+def taggedValue(key, value):
+    """Attaches a tagged value to an interface at definition time."""
+    f_locals = sys._getframe(1).f_locals
+    tagged_values = f_locals.setdefault(TAGGED_DATA, {})
+    tagged_values[key] = value
+    return _decorator_non_return
+
+
+class Element(object):
+    """
+    Default implementation of `zope.interface.interfaces.IElement`.
+    """
+
+    # We can't say this yet because we don't have enough
+    # infrastructure in place.
+    #
+    #implements(IElement)
+
+    def __init__(self, __name__, __doc__=''): # pylint:disable=redefined-builtin
+        if not __doc__ and __name__.find(' ') >= 0:
+            __doc__ = __name__
+            __name__ = None
+
+        self.__name__ = __name__
+        self.__doc__ = __doc__
+        # Tagged values are rare, especially on methods or attributes.
+        # Deferring the allocation can save substantial memory.
+        self.__tagged_values = None
+
+    def getName(self):
+        """ Returns the name of the object. """
+        return self.__name__
+
+    def getDoc(self):
+        """ Returns the documentation for the object. """
+        return self.__doc__
+
+    ###
+    # Tagged values.
+    #
+    # Direct tagged values are set only in this instance. Others
+    # may be inherited (for those subclasses that have that concept).
+    ###
+
+    def getTaggedValue(self, tag):
+        """ Returns the value associated with 'tag'. """
+        if not self.__tagged_values:
+            raise KeyError(tag)
+        return self.__tagged_values[tag]
+
+    def queryTaggedValue(self, tag, default=None):
+        """ Returns the value associated with 'tag'. """
+        return self.__tagged_values.get(tag, default) if self.__tagged_values else default
+
+    def getTaggedValueTags(self):
+        """ Returns a collection of all tags. """
+        return self.__tagged_values.keys() if self.__tagged_values else ()
+
+    def setTaggedValue(self, tag, value):
+        """ Associates 'value' with 'key'. """
+        if self.__tagged_values is None:
+            self.__tagged_values = {}
+        self.__tagged_values[tag] = value
+
+    queryDirectTaggedValue = queryTaggedValue
+    getDirectTaggedValue = getTaggedValue
+    getDirectTaggedValueTags = getTaggedValueTags
+
+
+SpecificationBasePy = object # filled by _use_c_impl.
+
+
+@_use_c_impl
+class SpecificationBase(object):
+    # This object is the base of the inheritance hierarchy for ClassProvides:
+    #
+    # ClassProvides < ClassProvidesBase, Declaration
+    # Declaration < Specification < SpecificationBase
+    # ClassProvidesBase < SpecificationBase
+    #
+    # In order to have compatible instance layouts, we need to declare
+    # the storage used by Specification and Declaration here (and
+    # those classes must have ``__slots__ = ()``); fortunately this is
+    # not a waste of space because those are the only two inheritance
+    # trees. These all translate into tp_members in C.
+    __slots__ = (
+        # Things used here.
+        '_implied',
+        # Things used in Specification.
+        '_dependents',
+        '_bases',
+        '_v_attrs',
+        '__iro__',
+        '__sro__',
+        '__weakref__',
+    )
+
+    def providedBy(self, ob):
+        """Is the interface implemented by an object
+        """
+        spec = providedBy(ob)
+        return self in spec._implied
+
+    def implementedBy(self, cls):
+        """Test whether the specification is implemented by a class or factory.
+
+        Raise TypeError if argument is neither a class nor a callable.
+        """
+        spec = implementedBy(cls)
+        return self in spec._implied
+
+    def isOrExtends(self, interface):
+        """Is the interface the same as or extend the given interface
+        """
+        return interface in self._implied # pylint:disable=no-member
+
+    __call__ = isOrExtends
+
+
+class NameAndModuleComparisonMixin(object):
+    # Internal use. Implement the basic sorting operators (but not (in)equality
+    # or hashing). Subclasses must provide ``__name__`` and ``__module__``
+    # attributes. Subclasses will be mutually comparable; but because equality
+    # and hashing semantics are missing from this class, take care in how
+    # you define those two attributes: If you stick with the default equality
+    # and hashing (identity based) you should make sure that all possible ``__name__``
+    # and ``__module__`` pairs are unique ACROSS ALL SUBCLASSES. (Actually, pretty
+    # much the same thing goes if you define equality and hashing to be based on
+    # those two attributes: they must still be consistent ACROSS ALL SUBCLASSES.)
+
+    # pylint:disable=assigning-non-slot
+    __slots__ = ()
+
+    def _compare(self, other):
+        """
+        Compare *self* to *other* based on ``__name__`` and ``__module__``.
+
+        Return 0 if they are equal, return 1 if *self* is
+        greater than *other*, and return -1 if *self* is less than
+        *other*.
+
+        If *other* does not have ``__name__`` or ``__module__``, then
+        return ``NotImplemented``.
+
+        .. caution::
+           This allows comparison to things well outside the type hierarchy,
+           perhaps not symmetrically.
+
+           For example, ``class Foo(object)`` and ``class Foo(Interface)``
+           in the same file would compare equal, depending on the order of
+           operands. Writing code like this by hand would be unusual, but it could
+           happen with dynamic creation of types and interfaces.
+
+        None is treated as a pseudo interface that implies the loosest
+        contact possible, no contract. For that reason, all interfaces
+        sort before None.
+        """
+        if other is self:
+            return 0
+
+        if other is None:
+            return -1
+
+        n1 = (self.__name__, self.__module__)
+        try:
+            n2 = (other.__name__, other.__module__)
+        except AttributeError:
+            return NotImplemented
+
+        # This spelling works under Python3, which doesn't have cmp().
+        return (n1 > n2) - (n1 < n2)
+
+    def __lt__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c < 0
+
+    def __le__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c <= 0
+
+    def __gt__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c > 0
+
+    def __ge__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c >= 0
+
+
+@_use_c_impl
+class InterfaceBase(NameAndModuleComparisonMixin, SpecificationBasePy):
+    """Base class that wants to be replaced with a C base :)
+    """
+
+    __slots__ = (
+        '__name__',
+        '__ibmodule__',
+        '_v_cached_hash',
+    )
+
+    def __init__(self, name=None, module=None):
+        self.__name__ = name
+        self.__ibmodule__ = module
+
+    def _call_conform(self, conform):
+        raise NotImplementedError
+
+    @property
+    def __module_property__(self):
+        # This is for _InterfaceMetaClass
+        return self.__ibmodule__
+
+    def __call__(self, obj, alternate=_marker):
+        """Adapt an object to the interface
+        """
+        try:
+            conform = obj.__conform__
+        except AttributeError:
+            conform = None
+
+        if conform is not None:
+            adapter = self._call_conform(conform)
+            if adapter is not None:
+                return adapter
+
+        adapter = self.__adapt__(obj)
+
+        if adapter is not None:
+            return adapter
+        if alternate is not _marker:
+            return alternate
+        raise TypeError("Could not adapt", obj, self)
+
+    def __adapt__(self, obj):
+        """Adapt an object to the receiver
+        """
+        if self.providedBy(obj):
+            return obj
+
+        for hook in adapter_hooks:
+            adapter = hook(self, obj)
+            if adapter is not None:
+                return adapter
+
+        return None
+
+    def __hash__(self):
+        # pylint:disable=assigning-non-slot,attribute-defined-outside-init
+        try:
+            return self._v_cached_hash
+        except AttributeError:
+            self._v_cached_hash = hash((self.__name__, self.__module__))
+        return self._v_cached_hash
+
+    def __eq__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c == 0
+
+    def __ne__(self, other):
+        if other is self:
+            return False
+
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c != 0
+
+adapter_hooks = _use_c_impl([], 'adapter_hooks')
+
+
+class Specification(SpecificationBase):
+    """Specifications
+
+    An interface specification is used to track interface declarations
+    and component registrations.
+
+    This class is a base class for both interfaces themselves and for
+    interface specifications (declarations).
+
+    Specifications are mutable.  If you reassign their bases, their
+    relations with other specifications are adjusted accordingly.
+    """
+    __slots__ = ()
+
+    # The root of all Specifications. This will be assigned `Interface`,
+    # once it is defined.
+    _ROOT = None
+
+    # Copy some base class methods for speed
+    isOrExtends = SpecificationBase.isOrExtends
+    providedBy = SpecificationBase.providedBy
+
+    def __init__(self, bases=()):
+        # There are many leaf interfaces with no dependents,
+        # and a few with very many. It's a heavily left-skewed
+        # distribution. In a survey of Plone and Zope related packages
+        # that loaded 2245 InterfaceClass objects and 2235 ClassProvides
+        # instances, there were a total of 7000 Specification objects created.
+        # 4700 had 0 dependents, 1400 had 1, 382 had 2 and so on. Only one
+        # for <type> had 1664. So there's savings to be had deferring
+        # the creation of dependents.
+        self._dependents = None # type: weakref.WeakKeyDictionary
+        self._bases = ()
+        self._implied = {}
+        self._v_attrs = None
+        self.__iro__ = ()
+        self.__sro__ = ()
+
+        self.__bases__ = tuple(bases)
+
+    @property
+    def dependents(self):
+        if self._dependents is None:
+            self._dependents = weakref.WeakKeyDictionary()
+        return self._dependents
+
+    def subscribe(self, dependent):
+        self._dependents[dependent] = self.dependents.get(dependent, 0) + 1
+
+    def unsubscribe(self, dependent):
+        try:
+            n = self._dependents[dependent]
+        except TypeError:
+            raise KeyError(dependent)
+        n -= 1
+        if not n:
+            del self.dependents[dependent]
+        else:
+            assert n > 0
+            self.dependents[dependent] = n
+
+    def __setBases(self, bases):
+        # Remove ourselves as a dependent of our old bases
+        for b in self.__bases__:
+            b.unsubscribe(self)
+
+        # Register ourselves as a dependent of our new bases
+        self._bases = bases
+        for b in bases:
+            b.subscribe(self)
+
+        self.changed(self)
+
+    __bases__ = property(
+        lambda self: self._bases,
+        __setBases,
+        )
+
+    # This method exists for tests to override the way we call
+    # ro.calculate_ro(), usually by adding extra kwargs. We don't
+    # want to have a mutable dictionary as a class member that we pass
+    # ourself because mutability is bad, and passing **kw is slower than
+    # calling the bound function.
+    _do_calculate_ro = calculate_ro
+
+    def _calculate_sro(self):
+        """
+        Calculate and return the resolution order for this object, using its ``__bases__``.
+
+        Ensures that ``Interface`` is always the last (lowest priority) element.
+        """
+        # We'd like to make Interface the lowest priority as a
+        # property of the resolution order algorithm. That almost
+        # works out naturally, but it fails when class inheritance has
+        # some bases that DO implement an interface, and some that DO
+        # NOT. In such a mixed scenario, you wind up with a set of
+        # bases to consider that look like this: [[..., Interface],
+        # [..., object], ...]. Depending on the order of inheritance,
+        # Interface can wind up before or after object, and that can
+        # happen at any point in the tree, meaning Interface can wind
+        # up somewhere in the middle of the order. Since Interface is
+        # treated as something that everything winds up implementing
+        # anyway (a catch-all for things like adapters), having it high up
+        # the order is bad. It's also bad to have it at the end, just before
+        # some concrete class: concrete classes should be HIGHER priority than
+        # interfaces (because there's only one class, but many implementations).
+        #
+        # One technically nice way to fix this would be to have
+        # ``implementedBy(object).__bases__ = (Interface,)``
+        #
+        # But: (1) That fails for old-style classes and (2) that causes
+        # everything to appear to *explicitly* implement Interface, when up
+        # to this point it's been an implicit virtual sort of relationship.
+        #
+        # So we force the issue by mutating the resolution order.
+
+        # Note that we let C3 use pre-computed __sro__ for our bases.
+        # This requires that by the time this method is invoked, our bases
+        # have settled their SROs. Thus, ``changed()`` must first
+        # update itself before telling its descendents of changes.
+        sro = self._do_calculate_ro(base_mros={
+            b: b.__sro__
+            for b in self.__bases__
+        })
+        root = self._ROOT
+        if root is not None and sro and sro[-1] is not root:
+            # In one dataset of 1823 Interface objects, 1117 ClassProvides objects,
+            # sro[-1] was root 4496 times, and only not root 118 times. So it's
+            # probably worth checking.
+
+            # Once we don't have to deal with old-style classes,
+            # we can add a check and only do this if base_count > 1,
+            # if we tweak the bootstrapping for ``<implementedBy object>``
+            sro = [
+                x
+                for x in sro
+                if x is not root
+            ]
+            sro.append(root)
+
+        return sro
+
+    def changed(self, originally_changed):
+        """
+        We, or something we depend on, have changed.
+
+        By the time this is called, the things we depend on,
+        such as our bases, should themselves be stable.
+        """
+        self._v_attrs = None
+
+        implied = self._implied
+        implied.clear()
+
+        ancestors = self._calculate_sro()
+        self.__sro__ = tuple(ancestors)
+        self.__iro__ = tuple([ancestor for ancestor in ancestors
+                              if isinstance(ancestor, InterfaceClass)
+                              ])
+
+        for ancestor in ancestors:
+            # We directly imply our ancestors:
+            implied[ancestor] = ()
+
+        # Now, advise our dependents of change
+        # (being careful not to create the WeakKeyDictionary if not needed):
+        for dependent in tuple(self._dependents.keys() if self._dependents else ()):
+            dependent.changed(originally_changed)
+
+        # Just in case something called get() at some point
+        # during that process and we have a cycle of some sort
+        # make sure we didn't cache incomplete results.
+        self._v_attrs = None
+
+    def interfaces(self):
+        """Return an iterator for the interfaces in the specification.
+        """
+        seen = {}
+        for base in self.__bases__:
+            for interface in base.interfaces():
+                if interface not in seen:
+                    seen[interface] = 1
+                    yield interface
+
+    def extends(self, interface, strict=True):
+        """Does the specification extend the given interface?
+
+        Test whether an interface in the specification extends the
+        given interface
+        """
+        return ((interface in self._implied)
+                and
+                ((not strict) or (self != interface))
+                )
+
+    def weakref(self, callback=None):
+        return weakref.ref(self, callback)
+
+    def get(self, name, default=None):
+        """Query for an attribute description
+        """
+        attrs = self._v_attrs
+        if attrs is None:
+            attrs = self._v_attrs = {}
+        attr = attrs.get(name)
+        if attr is None:
+            for iface in self.__iro__:
+                attr = iface.direct(name)
+                if attr is not None:
+                    attrs[name] = attr
+                    break
+
+        return default if attr is None else attr
+
+
+class _InterfaceMetaClass(type):
+    # Handling ``__module__`` on ``InterfaceClass`` is tricky. We need
+    # to be able to read it on a type and get the expected string. We
+    # also need to be able to set it on an instance and get the value
+    # we set. So far so good. But what gets tricky is that we'd like
+    # to store the value in the C structure (``InterfaceBase.__ibmodule__``) for
+    # direct access during equality, sorting, and hashing. "No
+    # problem, you think, I'll just use a property" (well, the C
+    # equivalents, ``PyMemberDef`` or ``PyGetSetDef``).
+    #
+    # Except there is a problem. When a subclass is created, the
+    # metaclass (``type``) always automatically puts the expected
+    # string in the class's dictionary under ``__module__``, thus
+    # overriding the property inherited from the superclass. Writing
+    # ``Subclass.__module__`` still works, but
+    # ``Subclass().__module__`` fails.
+    #
+    # There are multiple ways to work around this:
+    #
+    # (1) Define ``InterfaceBase.__getattribute__`` to watch for
+    # ``__module__`` and return the C storage.
+    #
+    # This works, but slows down *all* attribute access (except,
+    # ironically, to ``__module__``) by about 25% (40ns becomes 50ns)
+    # (when implemented in C). Since that includes methods like
+    # ``providedBy``, that's probably not acceptable.
+    #
+    # All the other methods involve modifying subclasses. This can be
+    # done either on the fly in some cases, as instances are
+    # constructed, or by using a metaclass. These next few can be done on the fly.
+    #
+    # (2) Make ``__module__`` a descriptor in each subclass dictionary.
+    # It can't be a straight up ``@property`` descriptor, though, because accessing
+    # it on the class returns a ``property`` object, not the desired string.
+    #
+    # (3) Implement a data descriptor (``__get__`` and ``__set__``)
+    # that is both a subclass of string, and also does the redirect of
+    # ``__module__`` to ``__ibmodule__`` and does the correct thing
+    # with the ``instance`` argument to ``__get__`` is None (returns
+    # the class's value.) (Why must it be a subclass of string? Because
+    # when it' s in the class's dict, it's defined on an *instance* of the
+    # metaclass; descriptors in an instance's dict aren't honored --- their
+    # ``__get__`` is never invoked --- so it must also *be* the value we want
+    # returned.)
+    #
+    # This works, preserves the ability to read and write
+    # ``__module__``, and eliminates any penalty accessing other
+    # attributes. But it slows down accessing ``__module__`` of
+    # instances by 200% (40ns to 124ns), requires editing class dicts on the fly
+    # (in InterfaceClass.__init__), thus slightly slowing down all interface creation,
+    # and is ugly.
+    #
+    # (4) As in the last step, but make it a non-data descriptor (no ``__set__``).
+    #
+    # If you then *also* store a copy of ``__ibmodule__`` in
+    # ``__module__`` in the instance's dict, reading works for both
+    # class and instance and is full speed for instances. But the cost
+    # is storage space, and you can't write to it anymore, not without
+    # things getting out of sync.
+    #
+    # (Actually, ``__module__`` was never meant to be writable. Doing
+    # so would break BTrees and normal dictionaries, as well as the
+    # repr, maybe more.)
+    #
+    # That leaves us with a metaclass. (Recall that a class is an
+    # instance of its metaclass, so properties/descriptors defined in
+    # the metaclass are used when accessing attributes on the
+    # instance/class. We'll use that to define ``__module__``.) Here
+    # we can have our cake and eat it too: no extra storage, and
+    # C-speed access to the underlying storage. The only substantial
+    # cost is that metaclasses tend to make people's heads hurt. (But
+    # still less than the descriptor-is-string, hopefully.)
+
+    __slots__ = ()
+
+    def __new__(cls, name, bases, attrs):
+        # Figure out what module defined the interface.
+        # This is copied from ``InterfaceClass.__init__``;
+        # reviewers aren't sure how AttributeError or KeyError
+        # could be raised.
+        __module__ = sys._getframe(1).f_globals['__name__']
+        # Get the C optimized __module__ accessor and give it
+        # to the new class.
+        moduledescr = InterfaceBase.__dict__['__module__']
+        if isinstance(moduledescr, str):
+            # We're working with the Python implementation,
+            # not the C version
+            moduledescr = InterfaceBase.__dict__['__module_property__']
+        attrs['__module__'] = moduledescr
+        kind = type.__new__(cls, name, bases, attrs)
+        kind.__module = __module__
+        return kind
+
+    @property
+    def __module__(cls):
+        return cls.__module
+
+    def __repr__(cls):
+        return "<class '%s.%s'>" % (
+            cls.__module,
+            cls.__name__,
+        )
+
+
+_InterfaceClassBase = _InterfaceMetaClass(
+    'InterfaceClass',
+    # From least specific to most specific.
+    (InterfaceBase, Specification, Element),
+    {'__slots__': ()}
+)
+
+
+def interfacemethod(func):
+    """
+    Convert a method specification to an actual method of the interface.
+
+    This is a decorator that functions like `staticmethod` et al.
+
+    The primary use of this decorator is to allow interface definitions to
+    define the ``__adapt__`` method, but other interface methods can be
+    overridden this way too.
+
+    .. seealso:: `zope.interface.interfaces.IInterfaceDeclaration.interfacemethod`
+    """
+    f_locals = sys._getframe(1).f_locals
+    methods = f_locals.setdefault(INTERFACE_METHODS, {})
+    methods[func.__name__] = func
+    return _decorator_non_return
+
+
+class InterfaceClass(_InterfaceClassBase):
+    """
+    Prototype (scarecrow) Interfaces Implementation.
+
+    Note that it is not possible to change the ``__name__`` or ``__module__``
+    after an instance of this object has been constructed.
+    """
+
+    # We can't say this yet because we don't have enough
+    # infrastructure in place.
+    #
+    #implements(IInterface)
+
+    def __new__(cls, name=None, bases=(), attrs=None, __doc__=None, # pylint:disable=redefined-builtin
+                __module__=None):
+        assert isinstance(bases, tuple)
+        attrs = attrs or {}
+        needs_custom_class = attrs.pop(INTERFACE_METHODS, None)
+        if needs_custom_class:
+            needs_custom_class.update(
+                {'__classcell__': attrs.pop('__classcell__')}
+                if '__classcell__' in attrs
+                else {}
+            )
+            if '__adapt__' in needs_custom_class:
+                # We need to tell the C code to call this.
+                needs_custom_class['_CALL_CUSTOM_ADAPT'] = 1
+
+            if issubclass(cls, _InterfaceClassWithCustomMethods):
+                cls_bases = (cls,)
+            elif cls is InterfaceClass:
+                cls_bases = (_InterfaceClassWithCustomMethods,)
+            else:
+                cls_bases = (cls, _InterfaceClassWithCustomMethods)
+
+            cls = type(cls)( # pylint:disable=self-cls-assignment
+                name + "<WithCustomMethods>",
+                cls_bases,
+                needs_custom_class
+            )
+        elif PY2 and bases and len(bases) > 1:
+            bases_with_custom_methods = tuple(
+                type(b)
+                for b in bases
+                if issubclass(type(b), _InterfaceClassWithCustomMethods)
+            )
+
+            # If we have a subclass of InterfaceClass in *bases*,
+            # Python 3 is smart enough to pass that as *cls*, but Python
+            # 2 just passes whatever the first base in *bases* is. This means that if
+            # we have multiple inheritance, and one of our bases has already defined
+            # a custom method like ``__adapt__``, we do the right thing automatically
+            # and extend it on Python 3, but not necessarily on Python 2. To fix this, we need
+            # to run the MRO algorithm and get the most derived base manually.
+            # Note that this only works for consistent resolution orders
+            if bases_with_custom_methods:
+                cls = type( # pylint:disable=self-cls-assignment
+                    name + "<WithCustomMethods>",
+                    bases_with_custom_methods,
+                    {}
+                ).__mro__[1] # Not the class we created, the most derived.
+
+        return _InterfaceClassBase.__new__(cls)
+
+    def __init__(self, name, bases=(), attrs=None, __doc__=None,  # pylint:disable=redefined-builtin
+                 __module__=None):
+        # We don't call our metaclass parent directly
+        # pylint:disable=non-parent-init-called
+        # pylint:disable=super-init-not-called
+        if not all(isinstance(base, InterfaceClass) for base in bases):
+            raise TypeError('Expected base interfaces')
+
+        if attrs is None:
+            attrs = {}
+
+        if __module__ is None:
+            __module__ = attrs.get('__module__')
+            if isinstance(__module__, str):
+                del attrs['__module__']
+            else:
+                try:
+                    # Figure out what module defined the interface.
+                    # This is how cPython figures out the module of
+                    # a class, but of course it does it in C. :-/
+                    __module__ = sys._getframe(1).f_globals['__name__']
+                except (AttributeError, KeyError): # pragma: no cover
+                    pass
+
+        InterfaceBase.__init__(self, name, __module__)
+        # These asserts assisted debugging the metaclass
+        # assert '__module__' not in self.__dict__
+        # assert self.__ibmodule__ is self.__module__ is __module__
+
+        d = attrs.get('__doc__')
+        if d is not None:
+            if not isinstance(d, Attribute):
+                if __doc__ is None:
+                    __doc__ = d
+                del attrs['__doc__']
+
+        if __doc__ is None:
+            __doc__ = ''
+
+        Element.__init__(self, name, __doc__)
+
+        tagged_data = attrs.pop(TAGGED_DATA, None)
+        if tagged_data is not None:
+            for key, val in tagged_data.items():
+                self.setTaggedValue(key, val)
+
+        Specification.__init__(self, bases)
+        self.__attrs = self.__compute_attrs(attrs)
+
+        self.__identifier__ = "%s.%s" % (__module__, name)
+
+    def __compute_attrs(self, attrs):
+        # Make sure that all recorded attributes (and methods) are of type
+        # `Attribute` and `Method`
+        def update_value(aname, aval):
+            if isinstance(aval, Attribute):
+                aval.interface = self
+                if not aval.__name__:
+                    aval.__name__ = aname
+            elif isinstance(aval, FunctionType):
+                aval = fromFunction(aval, self, name=aname)
+            else:
+                raise InvalidInterface("Concrete attribute, " + aname)
+            return aval
+
+        return {
+            aname: update_value(aname, aval)
+            for aname, aval in attrs.items()
+            if aname not in (
+                # __locals__: Python 3 sometimes adds this.
+                '__locals__',
+                # __qualname__: PEP 3155 (Python 3.3+)
+                '__qualname__',
+                # __annotations__: PEP 3107 (Python 3.0+)
+                '__annotations__',
+            )
+            and aval is not _decorator_non_return
+        }
+
+    def interfaces(self):
+        """Return an iterator for the interfaces in the specification.
+        """
+        yield self
+
+    def getBases(self):
+        return self.__bases__
+
+    def isEqualOrExtendedBy(self, other):
+        """Same interface or extends?"""
+        return self == other or other.extends(self)
+
+    def names(self, all=False): # pylint:disable=redefined-builtin
+        """Return the attribute names defined by the interface."""
+        if not all:
+            return self.__attrs.keys()
+
+        r = self.__attrs.copy()
+
+        for base in self.__bases__:
+            r.update(dict.fromkeys(base.names(all)))
+
+        return r.keys()
+
+    def __iter__(self):
+        return iter(self.names(all=True))
+
+    def namesAndDescriptions(self, all=False): # pylint:disable=redefined-builtin
+        """Return attribute names and descriptions defined by interface."""
+        if not all:
+            return self.__attrs.items()
+
+        r = {}
+        for base in self.__bases__[::-1]:
+            r.update(dict(base.namesAndDescriptions(all)))
+
+        r.update(self.__attrs)
+
+        return r.items()
+
+    def getDescriptionFor(self, name):
+        """Return the attribute description for the given name."""
+        r = self.get(name)
+        if r is not None:
+            return r
+
+        raise KeyError(name)
+
+    __getitem__ = getDescriptionFor
+
+    def __contains__(self, name):
+        return self.get(name) is not None
+
+    def direct(self, name):
+        return self.__attrs.get(name)
+
+    def queryDescriptionFor(self, name, default=None):
+        return self.get(name, default)
+
+    def validateInvariants(self, obj, errors=None):
+        """validate object to defined invariants."""
+
+        for iface in self.__iro__:
+            for invariant in iface.queryDirectTaggedValue('invariants', ()):
+                try:
+                    invariant(obj)
+                except Invalid as error:
+                    if errors is not None:
+                        errors.append(error)
+                    else:
+                        raise
+
+        if errors:
+            raise Invalid(errors)
+
+    def queryTaggedValue(self, tag, default=None):
+        """
+        Queries for the value associated with *tag*, returning it from the nearest
+        interface in the ``__iro__``.
+
+        If not found, returns *default*.
+        """
+        for iface in self.__iro__:
+            value = iface.queryDirectTaggedValue(tag, _marker)
+            if value is not _marker:
+                return value
+        return default
+
+    def getTaggedValue(self, tag):
+        """ Returns the value associated with 'tag'. """
+        value = self.queryTaggedValue(tag, default=_marker)
+        if value is _marker:
+            raise KeyError(tag)
+        return value
+
+    def getTaggedValueTags(self):
+        """ Returns a list of all tags. """
+        keys = set()
+        for base in self.__iro__:
+            keys.update(base.getDirectTaggedValueTags())
+        return keys
+
+    def __repr__(self):
+        try:
+            return self._v_repr
+        except AttributeError:
+            name = str(self)
+            r = "<%s %s>" % (self.__class__.__name__, name)
+            self._v_repr = r # pylint:disable=attribute-defined-outside-init
+            return r
+
+    def __str__(self):
+        name = self.__name__
+        m = self.__ibmodule__
+        if m:
+            name = '%s.%s' % (m, name)
+        return name
+
+    def _call_conform(self, conform):
+        try:
+            return conform(self)
+        except TypeError: # pragma: no cover
+            # We got a TypeError. It might be an error raised by
+            # the __conform__ implementation, or *we* may have
+            # made the TypeError by calling an unbound method
+            # (object is a class).  In the later case, we behave
+            # as though there is no __conform__ method. We can
+            # detect this case by checking whether there is more
+            # than one traceback object in the traceback chain:
+            if sys.exc_info()[2].tb_next is not None:
+                # There is more than one entry in the chain, so
+                # reraise the error:
+                raise
+            # This clever trick is from Phillip Eby
+
+        return None # pragma: no cover
+
+    def __reduce__(self):
+        return self.__name__
+
+Interface = InterfaceClass("Interface", __module__='zope.interface')
+# Interface is the only member of its own SRO.
+Interface._calculate_sro = lambda: (Interface,)
+Interface.changed(Interface)
+assert Interface.__sro__ == (Interface,)
+Specification._ROOT = Interface
+ro._ROOT = Interface
+
+class _InterfaceClassWithCustomMethods(InterfaceClass):
+    """
+    Marker class for interfaces with custom methods that override InterfaceClass methods.
+    """
+
+
+class Attribute(Element):
+    """Attribute descriptions
+    """
+
+    # We can't say this yet because we don't have enough
+    # infrastructure in place.
+    #
+    # implements(IAttribute)
+
+    interface = None
+
+    def _get_str_info(self):
+        """Return extra data to put at the end of __str__."""
+        return ""
+
+    def __str__(self):
+        of = ''
+        if self.interface is not None:
+            of = self.interface.__module__ + '.' + self.interface.__name__ + '.'
+        # self.__name__ may be None during construction (e.g., debugging)
+        return of + (self.__name__ or '<unknown>') + self._get_str_info()
+
+    def __repr__(self):
+        return "<%s.%s object at 0x%x %s>" % (
+            type(self).__module__,
+            type(self).__name__,
+            id(self),
+            self
+        )
+
+
+class Method(Attribute):
+    """Method interfaces
+
+    The idea here is that you have objects that describe methods.
+    This provides an opportunity for rich meta-data.
+    """
+
+    # We can't say this yet because we don't have enough
+    # infrastructure in place.
+    #
+    # implements(IMethod)
+
+    positional = required = ()
+    _optional = varargs = kwargs = None
+    def _get_optional(self):
+        if self._optional is None:
+            return {}
+        return self._optional
+    def _set_optional(self, opt):
+        self._optional = opt
+    def _del_optional(self):
+        self._optional = None
+    optional = property(_get_optional, _set_optional, _del_optional)
+
+    def __call__(self, *args, **kw):
+        raise BrokenImplementation(self.interface, self.__name__)
+
+    def getSignatureInfo(self):
+        return {'positional': self.positional,
+                'required': self.required,
+                'optional': self.optional,
+                'varargs': self.varargs,
+                'kwargs': self.kwargs,
+                }
+
+    def getSignatureString(self):
+        sig = []
+        for v in self.positional:
+            sig.append(v)
+            if v in self.optional.keys():
+                sig[-1] += "=" + repr(self.optional[v])
+        if self.varargs:
+            sig.append("*" + self.varargs)
+        if self.kwargs:
+            sig.append("**" + self.kwargs)
+
+        return "(%s)" % ", ".join(sig)
+
+    _get_str_info = getSignatureString
+
+
+def fromFunction(func, interface=None, imlevel=0, name=None):
+    name = name or func.__name__
+    method = Method(name, func.__doc__)
+    defaults = getattr(func, '__defaults__', None) or ()
+    code = func.__code__
+    # Number of positional arguments
+    na = code.co_argcount - imlevel
+    names = code.co_varnames[imlevel:]
+    opt = {}
+    # Number of required arguments
+    defaults_count = len(defaults)
+    if not defaults_count:
+        # PyPy3 uses ``__defaults_count__`` for builtin methods
+        # like ``dict.pop``. Surprisingly, these don't have recorded
+        # ``__defaults__``
+        defaults_count = getattr(func, '__defaults_count__', 0)
+
+    nr = na - defaults_count
+    if nr < 0:
+        defaults = defaults[-nr:]
+        nr = 0
+
+    # Determine the optional arguments.
+    opt.update(dict(zip(names[nr:], defaults)))
+
+    method.positional = names[:na]
+    method.required = names[:nr]
+    method.optional = opt
+
+    argno = na
+
+    # Determine the function's variable argument's name (i.e. *args)
+    if code.co_flags & CO_VARARGS:
+        method.varargs = names[argno]
+        argno = argno + 1
+    else:
+        method.varargs = None
+
+    # Determine the function's keyword argument's name (i.e. **kw)
+    if code.co_flags & CO_VARKEYWORDS:
+        method.kwargs = names[argno]
+    else:
+        method.kwargs = None
+
+    method.interface = interface
+
+    for key, value in func.__dict__.items():
+        method.setTaggedValue(key, value)
+
+    return method
+
+
+def fromMethod(meth, interface=None, name=None):
+    if isinstance(meth, MethodType):
+        func = meth.__func__
+    else:
+        func = meth
+    return fromFunction(func, interface, imlevel=1, name=name)
+
+
+# Now we can create the interesting interfaces and wire them up:
+def _wire():
+    from zope.interface.declarations import classImplements
+    # From lest specific to most specific.
+    from zope.interface.interfaces import IElement
+    classImplements(Element, IElement)
+
+    from zope.interface.interfaces import IAttribute
+    classImplements(Attribute, IAttribute)
+
+    from zope.interface.interfaces import IMethod
+    classImplements(Method, IMethod)
+
+    from zope.interface.interfaces import ISpecification
+    classImplements(Specification, ISpecification)
+
+    from zope.interface.interfaces import IInterface
+    classImplements(InterfaceClass, IInterface)
+
+
+# We import this here to deal with module dependencies.
+# pylint:disable=wrong-import-position
+from zope.interface.declarations import implementedBy
+from zope.interface.declarations import providedBy
+from zope.interface.exceptions import InvalidInterface
+from zope.interface.exceptions import BrokenImplementation
+
+# This ensures that ``Interface`` winds up in the flattened()
+# list of the immutable declaration. It correctly overrides changed()
+# as a no-op, so we bypass that.
+from zope.interface.declarations import _empty
+Specification.changed(_empty, _empty)
diff --git a/contrib/python/zope.interface/py2/zope/interface/interfaces.py b/contrib/python/zope.interface/py2/zope/interface/interfaces.py
new file mode 100644
index 00000000000..66aecb90967
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/interfaces.py
@@ -0,0 +1,1593 @@
+##############################################################################
+#
+# Copyright (c) 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interface Package Interfaces
+"""
+__docformat__ = 'restructuredtext'
+
+from zope.interface.interface import Attribute
+from zope.interface.interface import Interface
+from zope.interface.declarations import implementer
+
+__all__ = [
+    'ComponentLookupError',
+    'IAdapterRegistration',
+    'IAdapterRegistry',
+    'IAttribute',
+    'IComponentLookup',
+    'IComponentRegistry',
+    'IComponents',
+    'IDeclaration',
+    'IElement',
+    'IHandlerRegistration',
+    'IInterface',
+    'IInterfaceDeclaration',
+    'IMethod',
+    'Invalid',
+    'IObjectEvent',
+    'IRegistered',
+    'IRegistration',
+    'IRegistrationEvent',
+    'ISpecification',
+    'ISubscriptionAdapterRegistration',
+    'IUnregistered',
+    'IUtilityRegistration',
+    'ObjectEvent',
+    'Registered',
+    'Unregistered',
+]
+
+# pylint:disable=inherit-non-class,no-method-argument,no-self-argument
+# pylint:disable=unexpected-special-method-signature
+# pylint:disable=too-many-lines
+
+class IElement(Interface):
+    """
+    Objects that have basic documentation and tagged values.
+
+    Known derivatives include :class:`IAttribute` and its derivative
+    :class:`IMethod`; these have no notion of inheritance.
+    :class:`IInterface` is also a derivative, and it does have a
+    notion of inheritance, expressed through its ``__bases__`` and
+    ordered in its ``__iro__`` (both defined by
+    :class:`ISpecification`).
+    """
+
+    # pylint:disable=arguments-differ
+
+    # Note that defining __doc__ as an Attribute hides the docstring
+    # from introspection. When changing it, also change it in the Sphinx
+    # ReST files.
+
+    __name__ = Attribute('__name__', 'The object name')
+    __doc__ = Attribute('__doc__', 'The object doc string')
+
+    ###
+    # Tagged values.
+    #
+    # Direct values are established in this instance. Others may be
+    # inherited. Although ``IElement`` itself doesn't have a notion of
+    # inheritance, ``IInterface`` *does*. It might have been better to
+    # make ``IInterface`` define new methods
+    # ``getIndirectTaggedValue``, etc, to include inheritance instead
+    # of overriding ``getTaggedValue`` to do that, but that ship has sailed.
+    # So to keep things nice and symmetric, we define the ``Direct`` methods here.
+    ###
+
+    def getTaggedValue(tag):
+        """Returns the value associated with *tag*.
+
+        Raise a `KeyError` if the tag isn't set.
+
+        If the object has a notion of inheritance, this searches
+        through the inheritance hierarchy and returns the nearest result.
+        If there is no such notion, this looks only at this object.
+
+        .. versionchanged:: 4.7.0
+           This method should respect inheritance if present.
+        """
+
+    def queryTaggedValue(tag, default=None):
+        """
+        As for `getTaggedValue`, but instead of raising a `KeyError`, returns *default*.
+
+
+        .. versionchanged:: 4.7.0
+           This method should respect inheritance if present.
+        """
+
+    def getTaggedValueTags():
+        """
+        Returns a collection of all tags in no particular order.
+
+        If the object has a notion of inheritance, this
+        includes all the inherited tagged values. If there is
+        no such notion, this looks only at this object.
+
+        .. versionchanged:: 4.7.0
+           This method should respect inheritance if present.
+        """
+
+    def setTaggedValue(tag, value):
+        """
+        Associates *value* with *key* directly in this object.
+        """
+
+    def getDirectTaggedValue(tag):
+        """
+        As for `getTaggedValue`, but never includes inheritance.
+
+        .. versionadded:: 5.0.0
+        """
+
+    def queryDirectTaggedValue(tag, default=None):
+        """
+        As for `queryTaggedValue`, but never includes inheritance.
+
+        .. versionadded:: 5.0.0
+        """
+
+    def getDirectTaggedValueTags():
+        """
+        As for `getTaggedValueTags`, but includes only tags directly
+        set on this object.
+
+        .. versionadded:: 5.0.0
+        """
+
+
+class IAttribute(IElement):
+    """Attribute descriptors"""
+
+    interface = Attribute('interface',
+                          'Stores the interface instance in which the '
+                          'attribute is located.')
+
+
+class IMethod(IAttribute):
+    """Method attributes"""
+
+    def getSignatureInfo():
+        """Returns the signature information.
+
+        This method returns a dictionary with the following string keys:
+
+        - positional
+            A sequence of the names of positional arguments.
+        - required
+            A sequence of the names of required arguments.
+        - optional
+            A dictionary mapping argument names to their default values.
+        - varargs
+            The name of the varargs argument (or None).
+        - kwargs
+            The name of the kwargs argument (or None).
+        """
+
+    def getSignatureString():
+        """Return a signature string suitable for inclusion in documentation.
+
+        This method returns the function signature string. For example, if you
+        have ``def func(a, b, c=1, d='f')``, then the signature string is ``"(a, b,
+        c=1, d='f')"``.
+        """
+
+class ISpecification(Interface):
+    """Object Behavioral specifications"""
+    # pylint:disable=arguments-differ
+    def providedBy(object): # pylint:disable=redefined-builtin
+        """Test whether the interface is implemented by the object
+
+        Return true of the object asserts that it implements the
+        interface, including asserting that it implements an extended
+        interface.
+        """
+
+    def implementedBy(class_):
+        """Test whether the interface is implemented by instances of the class
+
+        Return true of the class asserts that its instances implement the
+        interface, including asserting that they implement an extended
+        interface.
+        """
+
+    def isOrExtends(other):
+        """Test whether the specification is or extends another
+        """
+
+    def extends(other, strict=True):
+        """Test whether a specification extends another
+
+        The specification extends other if it has other as a base
+        interface or if one of it's bases extends other.
+
+        If strict is false, then the specification extends itself.
+        """
+
+    def weakref(callback=None):
+        """Return a weakref to the specification
+
+        This method is, regrettably, needed to allow weakrefs to be
+        computed to security-proxied specifications.  While the
+        zope.interface package does not require zope.security or
+        zope.proxy, it has to be able to coexist with it.
+
+        """
+
+    __bases__ = Attribute("""Base specifications
+
+    A tuple of specifications from which this specification is
+    directly derived.
+
+    """)
+
+    __sro__ = Attribute("""Specification-resolution order
+
+    A tuple of the specification and all of it's ancestor
+    specifications from most specific to least specific. The specification
+    itself is the first element.
+
+    (This is similar to the method-resolution order for new-style classes.)
+    """)
+
+    __iro__ = Attribute("""Interface-resolution order
+
+    A tuple of the specification's ancestor interfaces from
+    most specific to least specific.  The specification itself is
+    included if it is an interface.
+
+    (This is similar to the method-resolution order for new-style classes.)
+    """)
+
+    def get(name, default=None):
+        """Look up the description for a name
+
+        If the named attribute is not defined, the default is
+        returned.
+        """
+
+
+class IInterface(ISpecification, IElement):
+    """Interface objects
+
+    Interface objects describe the behavior of an object by containing
+    useful information about the object.  This information includes:
+
+    - Prose documentation about the object.  In Python terms, this
+      is called the "doc string" of the interface.  In this element,
+      you describe how the object works in prose language and any
+      other useful information about the object.
+
+    - Descriptions of attributes.  Attribute descriptions include
+      the name of the attribute and prose documentation describing
+      the attributes usage.
+
+    - Descriptions of methods.  Method descriptions can include:
+
+        - Prose "doc string" documentation about the method and its
+          usage.
+
+        - A description of the methods arguments; how many arguments
+          are expected, optional arguments and their default values,
+          the position or arguments in the signature, whether the
+          method accepts arbitrary arguments and whether the method
+          accepts arbitrary keyword arguments.
+
+    - Optional tagged data.  Interface objects (and their attributes and
+      methods) can have optional, application specific tagged data
+      associated with them.  Examples uses for this are examples,
+      security assertions, pre/post conditions, and other possible
+      information you may want to associate with an Interface or its
+      attributes.
+
+    Not all of this information is mandatory.  For example, you may
+    only want the methods of your interface to have prose
+    documentation and not describe the arguments of the method in
+    exact detail.  Interface objects are flexible and let you give or
+    take any of these components.
+
+    Interfaces are created with the Python class statement using
+    either `zope.interface.Interface` or another interface, as in::
+
+      from zope.interface import Interface
+
+      class IMyInterface(Interface):
+        '''Interface documentation'''
+
+        def meth(arg1, arg2):
+            '''Documentation for meth'''
+
+        # Note that there is no self argument
+
+     class IMySubInterface(IMyInterface):
+        '''Interface documentation'''
+
+        def meth2():
+            '''Documentation for meth2'''
+
+    You use interfaces in two ways:
+
+    - You assert that your object implement the interfaces.
+
+      There are several ways that you can declare that an object
+      provides an interface:
+
+      1. Call `zope.interface.implementer` on your class definition.
+
+      2. Call `zope.interface.directlyProvides` on your object.
+
+      3. Call `zope.interface.classImplements` to declare that instances
+         of a class implement an interface.
+
+         For example::
+
+           from zope.interface import classImplements
+
+           classImplements(some_class, some_interface)
+
+         This approach is useful when it is not an option to modify
+         the class source.  Note that this doesn't affect what the
+         class itself implements, but only what its instances
+         implement.
+
+    - You query interface meta-data. See the IInterface methods and
+      attributes for details.
+
+    """
+    # pylint:disable=arguments-differ
+    def names(all=False): # pylint:disable=redefined-builtin
+        """Get the interface attribute names
+
+        Return a collection of the names of the attributes, including
+        methods, included in the interface definition.
+
+        Normally, only directly defined attributes are included. If
+        a true positional or keyword argument is given, then
+        attributes defined by base classes will be included.
+        """
+
+    def namesAndDescriptions(all=False): # pylint:disable=redefined-builtin
+        """Get the interface attribute names and descriptions
+
+        Return a collection of the names and descriptions of the
+        attributes, including methods, as name-value pairs, included
+        in the interface definition.
+
+        Normally, only directly defined attributes are included. If
+        a true positional or keyword argument is given, then
+        attributes defined by base classes will be included.
+        """
+
+    def __getitem__(name):
+        """Get the description for a name
+
+        If the named attribute is not defined, a `KeyError` is raised.
+        """
+
+    def direct(name):
+        """Get the description for the name if it was defined by the interface
+
+        If the interface doesn't define the name, returns None.
+        """
+
+    def validateInvariants(obj, errors=None):
+        """Validate invariants
+
+        Validate object to defined invariants.  If errors is None,
+        raises first Invalid error; if errors is a list, appends all errors
+        to list, then raises Invalid with the errors as the first element
+        of the "args" tuple."""
+
+    def __contains__(name):
+        """Test whether the name is defined by the interface"""
+
+    def __iter__():
+        """Return an iterator over the names defined by the interface
+
+        The names iterated include all of the names defined by the
+        interface directly and indirectly by base interfaces.
+        """
+
+    __module__ = Attribute("""The name of the module defining the interface""")
+
+
+class IDeclaration(ISpecification):
+    """Interface declaration
+
+    Declarations are used to express the interfaces implemented by
+    classes or provided by objects.
+    """
+
+    def __contains__(interface):
+        """Test whether an interface is in the specification
+
+        Return true if the given interface is one of the interfaces in
+        the specification and false otherwise.
+        """
+
+    def __iter__():
+        """Return an iterator for the interfaces in the specification
+        """
+
+    def flattened():
+        """Return an iterator of all included and extended interfaces
+
+        An iterator is returned for all interfaces either included in
+        or extended by interfaces included in the specifications
+        without duplicates. The interfaces are in "interface
+        resolution order". The interface resolution order is such that
+        base interfaces are listed after interfaces that extend them
+        and, otherwise, interfaces are included in the order that they
+        were defined in the specification.
+        """
+
+    def __sub__(interfaces):
+        """Create an interface specification with some interfaces excluded
+
+        The argument can be an interface or an interface
+        specifications.  The interface or interfaces given in a
+        specification are subtracted from the interface specification.
+
+        Removing an interface that is not in the specification does
+        not raise an error. Doing so has no effect.
+
+        Removing an interface also removes sub-interfaces of the interface.
+
+        """
+
+    def __add__(interfaces):
+        """Create an interface specification with some interfaces added
+
+        The argument can be an interface or an interface
+        specifications.  The interface or interfaces given in a
+        specification are added to the interface specification.
+
+        Adding an interface that is already in the specification does
+        not raise an error. Doing so has no effect.
+        """
+
+    def __nonzero__():
+        """Return a true value of the interface specification is non-empty
+        """
+
+class IInterfaceDeclaration(Interface):
+    """
+    Declare and check the interfaces of objects.
+
+    The functions defined in this interface are used to declare the
+    interfaces that objects provide and to query the interfaces that
+    have been declared.
+
+    Interfaces can be declared for objects in two ways:
+
+        - Interfaces are declared for instances of the object's class
+
+        - Interfaces are declared for the object directly.
+
+    The interfaces declared for an object are, therefore, the union of
+    interfaces declared for the object directly and the interfaces
+    declared for instances of the object's class.
+
+    Note that we say that a class implements the interfaces provided
+    by it's instances. An instance can also provide interfaces
+    directly. The interfaces provided by an object are the union of
+    the interfaces provided directly and the interfaces implemented by
+    the class.
+
+    This interface is implemented by :mod:`zope.interface`.
+    """
+    # pylint:disable=arguments-differ
+    ###
+    # Defining interfaces
+    ###
+
+    Interface = Attribute("The base class used to create new interfaces")
+
+    def taggedValue(key, value):
+        """
+        Attach a tagged value to an interface while defining the interface.
+
+        This is a way of executing :meth:`IElement.setTaggedValue` from
+        the definition of the interface. For example::
+
+             class IFoo(Interface):
+                 taggedValue('key', 'value')
+
+        .. seealso:: `zope.interface.taggedValue`
+        """
+
+    def invariant(checker_function):
+        """
+        Attach an invariant checker function to an interface while defining it.
+
+        Invariants can later be validated against particular implementations by
+        calling :meth:`IInterface.validateInvariants`.
+
+        For example::
+
+             def check_range(ob):
+                 if ob.max < ob.min:
+                     raise ValueError("max value is less than min value")
+
+             class IRange(Interface):
+                 min = Attribute("The min value")
+                 max = Attribute("The max value")
+
+                 invariant(check_range)
+
+        .. seealso:: `zope.interface.invariant`
+        """
+
+    def interfacemethod(method):
+        """
+        A decorator that transforms a method specification into an
+        implementation method.
+
+        This is used to override methods of ``Interface`` or provide new methods.
+        Definitions using this decorator will not appear in :meth:`IInterface.names()`.
+        It is possible to have an implementation method and a method specification
+        of the same name.
+
+        For example::
+
+             class IRange(Interface):
+                 @interfacemethod
+                 def __adapt__(self, obj):
+                     if isinstance(obj, range):
+                         # Return the builtin ``range`` as-is
+                         return obj
+                     return super(type(IRange), self).__adapt__(obj)
+
+        You can use ``super`` to call the parent class functionality. Note that
+        the zero-argument version (``super().__adapt__``) works on Python 3.6 and above, but
+        prior to that the two-argument version must be used, and the class must be explicitly
+        passed as the first argument.
+
+        .. versionadded:: 5.1.0
+        .. seealso:: `zope.interface.interfacemethod`
+        """
+
+    ###
+    # Querying interfaces
+    ###
+
+    def providedBy(ob):
+        """
+        Return the interfaces provided by an object.
+
+        This is the union of the interfaces directly provided by an
+        object and interfaces implemented by it's class.
+
+        The value returned is an `IDeclaration`.
+
+        .. seealso:: `zope.interface.providedBy`
+        """
+
+    def implementedBy(class_):
+        """
+        Return the interfaces implemented for a class's instances.
+
+        The value returned is an `IDeclaration`.
+
+        .. seealso:: `zope.interface.implementedBy`
+        """
+
+    ###
+    # Declaring interfaces
+    ###
+
+    def classImplements(class_, *interfaces):
+        """
+        Declare additional interfaces implemented for instances of a class.
+
+        The arguments after the class are one or more interfaces or
+        interface specifications (`IDeclaration` objects).
+
+        The interfaces given (including the interfaces in the
+        specifications) are added to any interfaces previously
+        declared.
+
+        Consider the following example::
+
+          class C(A, B):
+             ...
+
+          classImplements(C, I1, I2)
+
+
+        Instances of ``C`` provide ``I1``, ``I2``, and whatever interfaces
+        instances of ``A`` and ``B`` provide. This is equivalent to::
+
+            @implementer(I1, I2)
+            class C(A, B):
+                pass
+
+        .. seealso:: `zope.interface.classImplements`
+        .. seealso:: `zope.interface.implementer`
+        """
+
+    def classImplementsFirst(cls, interface):
+        """
+        See :func:`zope.interface.classImplementsFirst`.
+        """
+
+    def implementer(*interfaces):
+        """
+        Create a decorator for declaring interfaces implemented by a
+        factory.
+
+        A callable is returned that makes an implements declaration on
+        objects passed to it.
+
+        .. seealso:: :meth:`classImplements`
+        """
+
+    def classImplementsOnly(class_, *interfaces):
+        """
+        Declare the only interfaces implemented by instances of a class.
+
+        The arguments after the class are one or more interfaces or
+        interface specifications (`IDeclaration` objects).
+
+        The interfaces given (including the interfaces in the
+        specifications) replace any previous declarations.
+
+        Consider the following example::
+
+          class C(A, B):
+             ...
+
+          classImplements(C, IA, IB. IC)
+          classImplementsOnly(C. I1, I2)
+
+        Instances of ``C`` provide only ``I1``, ``I2``, and regardless of
+        whatever interfaces instances of ``A`` and ``B`` implement.
+
+        .. seealso:: `zope.interface.classImplementsOnly`
+        """
+
+    def implementer_only(*interfaces):
+        """
+        Create a decorator for declaring the only interfaces implemented.
+
+        A callable is returned that makes an implements declaration on
+        objects passed to it.
+
+        .. seealso:: `zope.interface.implementer_only`
+        """
+
+    def directlyProvidedBy(object): # pylint:disable=redefined-builtin
+        """
+        Return the interfaces directly provided by the given object.
+
+        The value returned is an `IDeclaration`.
+
+        .. seealso:: `zope.interface.directlyProvidedBy`
+        """
+
+    def directlyProvides(object, *interfaces): # pylint:disable=redefined-builtin
+        """
+        Declare interfaces declared directly for an object.
+
+        The arguments after the object are one or more interfaces or
+        interface specifications (`IDeclaration` objects).
+
+        .. caution::
+           The interfaces given (including the interfaces in the
+           specifications) *replace* interfaces previously
+           declared for the object. See :meth:`alsoProvides` to add
+           additional interfaces.
+
+        Consider the following example::
+
+          class C(A, B):
+             ...
+
+          ob = C()
+          directlyProvides(ob, I1, I2)
+
+        The object, ``ob`` provides ``I1``, ``I2``, and whatever interfaces
+        instances have been declared for instances of ``C``.
+
+        To remove directly provided interfaces, use `directlyProvidedBy` and
+        subtract the unwanted interfaces. For example::
+
+          directlyProvides(ob, directlyProvidedBy(ob)-I2)
+
+        removes I2 from the interfaces directly provided by
+        ``ob``. The object, ``ob`` no longer directly provides ``I2``,
+        although it might still provide ``I2`` if it's class
+        implements ``I2``.
+
+        To add directly provided interfaces, use `directlyProvidedBy` and
+        include additional interfaces.  For example::
+
+          directlyProvides(ob, directlyProvidedBy(ob), I2)
+
+        adds I2 to the interfaces directly provided by ob.
+
+        .. seealso:: `zope.interface.directlyProvides`
+        """
+
+    def alsoProvides(object, *interfaces): # pylint:disable=redefined-builtin
+        """
+        Declare additional interfaces directly for an object.
+
+        For example::
+
+          alsoProvides(ob, I1)
+
+        is equivalent to::
+
+          directlyProvides(ob, directlyProvidedBy(ob), I1)
+
+        .. seealso:: `zope.interface.alsoProvides`
+        """
+
+    def noLongerProvides(object, interface): # pylint:disable=redefined-builtin
+        """
+        Remove an interface from the list of an object's directly provided
+        interfaces.
+
+        For example::
+
+          noLongerProvides(ob, I1)
+
+        is equivalent to::
+
+          directlyProvides(ob, directlyProvidedBy(ob) - I1)
+
+        with the exception that if ``I1`` is an interface that is
+        provided by ``ob`` through the class's implementation,
+        `ValueError` is raised.
+
+        .. seealso:: `zope.interface.noLongerProvides`
+        """
+
+    def implements(*interfaces):
+        """
+        Declare interfaces implemented by instances of a class.
+
+        .. deprecated:: 5.0
+           This only works for Python 2. The `implementer` decorator
+           is preferred for all versions.
+
+        This function is called in a class definition (Python 2.x only).
+
+        The arguments are one or more interfaces or interface
+        specifications (`IDeclaration` objects).
+
+        The interfaces given (including the interfaces in the
+        specifications) are added to any interfaces previously
+        declared.
+
+        Previous declarations include declarations for base classes
+        unless implementsOnly was used.
+
+        This function is provided for convenience. It provides a more
+        convenient way to call `classImplements`. For example::
+
+          implements(I1)
+
+        is equivalent to calling::
+
+          classImplements(C, I1)
+
+        after the class has been created.
+
+        Consider the following example (Python 2.x only)::
+
+          class C(A, B):
+            implements(I1, I2)
+
+
+        Instances of ``C`` implement ``I1``, ``I2``, and whatever interfaces
+        instances of ``A`` and ``B`` implement.
+        """
+
+    def implementsOnly(*interfaces):
+        """
+        Declare the only interfaces implemented by instances of a class.
+
+        .. deprecated:: 5.0
+           This only works for Python 2. The `implementer_only` decorator
+           is preferred for all versions.
+
+        This function is called in a class definition (Python 2.x only).
+
+        The arguments are one or more interfaces or interface
+        specifications (`IDeclaration` objects).
+
+        Previous declarations including declarations for base classes
+        are overridden.
+
+        This function is provided for convenience. It provides a more
+        convenient way to call `classImplementsOnly`. For example::
+
+          implementsOnly(I1)
+
+        is equivalent to calling::
+
+          classImplementsOnly(I1)
+
+        after the class has been created.
+
+        Consider the following example (Python 2.x only)::
+
+          class C(A, B):
+            implementsOnly(I1, I2)
+
+
+        Instances of ``C`` implement ``I1``, ``I2``, regardless of what
+        instances of ``A`` and ``B`` implement.
+        """
+
+    def classProvides(*interfaces):
+        """
+        Declare interfaces provided directly by a class.
+
+        .. deprecated:: 5.0
+           This only works for Python 2. The `provider` decorator
+           is preferred for all versions.
+
+        This function is called in a class definition.
+
+        The arguments are one or more interfaces or interface
+        specifications (`IDeclaration` objects).
+
+        The given interfaces (including the interfaces in the
+        specifications) are used to create the class's direct-object
+        interface specification.  An error will be raised if the module
+        class has an direct interface specification.  In other words, it is
+        an error to call this function more than once in a class
+        definition.
+
+        Note that the given interfaces have nothing to do with the
+        interfaces implemented by instances of the class.
+
+        This function is provided for convenience. It provides a more
+        convenient way to call `directlyProvides` for a class. For example::
+
+          classProvides(I1)
+
+        is equivalent to calling::
+
+          directlyProvides(theclass, I1)
+
+        after the class has been created.
+        """
+
+    def provider(*interfaces):
+        """
+        A class decorator version of `classProvides`.
+
+        .. seealso:: `zope.interface.provider`
+        """
+
+    def moduleProvides(*interfaces):
+        """
+        Declare interfaces provided by a module.
+
+        This function is used in a module definition.
+
+        The arguments are one or more interfaces or interface
+        specifications (`IDeclaration` objects).
+
+        The given interfaces (including the interfaces in the
+        specifications) are used to create the module's direct-object
+        interface specification.  An error will be raised if the module
+        already has an interface specification.  In other words, it is
+        an error to call this function more than once in a module
+        definition.
+
+        This function is provided for convenience. It provides a more
+        convenient way to call `directlyProvides` for a module. For example::
+
+          moduleImplements(I1)
+
+        is equivalent to::
+
+          directlyProvides(sys.modules[__name__], I1)
+
+        .. seealso:: `zope.interface.moduleProvides`
+        """
+
+    def Declaration(*interfaces):
+        """
+        Create an interface specification.
+
+        The arguments are one or more interfaces or interface
+        specifications (`IDeclaration` objects).
+
+        A new interface specification (`IDeclaration`) with the given
+        interfaces is returned.
+
+        .. seealso:: `zope.interface.Declaration`
+        """
+
+class IAdapterRegistry(Interface):
+    """Provide an interface-based registry for adapters
+
+    This registry registers objects that are in some sense "from" a
+    sequence of specification to an interface and a name.
+
+    No specific semantics are assumed for the registered objects,
+    however, the most common application will be to register factories
+    that adapt objects providing required specifications to a provided
+    interface.
+    """
+
+    def register(required, provided, name, value):
+        """Register a value
+
+        A value is registered for a *sequence* of required specifications, a
+        provided interface, and a name, which must be text.
+        """
+
+    def registered(required, provided, name=u''):
+        """Return the component registered for the given interfaces and name
+
+        name must be text.
+
+        Unlike the lookup method, this methods won't retrieve
+        components registered for more specific required interfaces or
+        less specific provided interfaces.
+
+        If no component was registered exactly for the given
+        interfaces and name, then None is returned.
+
+        """
+
+    def lookup(required, provided, name='', default=None):
+        """Lookup a value
+
+        A value is looked up based on a *sequence* of required
+        specifications, a provided interface, and a name, which must be
+        text.
+        """
+
+    def queryMultiAdapter(objects, provided, name=u'', default=None):
+        """Adapt a sequence of objects to a named, provided, interface
+        """
+
+    def lookup1(required, provided, name=u'', default=None):
+        """Lookup a value using a single required interface
+
+        A value is looked up based on a single required
+        specifications, a provided interface, and a name, which must be
+        text.
+        """
+
+    def queryAdapter(object, provided, name=u'', default=None): # pylint:disable=redefined-builtin
+        """Adapt an object using a registered adapter factory.
+        """
+
+    def adapter_hook(provided, object, name=u'', default=None): # pylint:disable=redefined-builtin
+        """Adapt an object using a registered adapter factory.
+
+        name must be text.
+        """
+
+    def lookupAll(required, provided):
+        """Find all adapters from the required to the provided interfaces
+
+        An iterable object is returned that provides name-value two-tuples.
+        """
+
+    def names(required, provided): # pylint:disable=arguments-differ
+        """Return the names for which there are registered objects
+        """
+
+    def subscribe(required, provided, subscriber): # pylint:disable=arguments-differ
+        """Register a subscriber
+
+        A subscriber is registered for a *sequence* of required
+        specifications, a provided interface, and a name.
+
+        Multiple subscribers may be registered for the same (or
+        equivalent) interfaces.
+
+        .. versionchanged:: 5.1.1
+           Correct the method signature to remove the ``name`` parameter.
+           Subscribers have no names.
+        """
+
+    def subscribed(required, provided, subscriber):
+        """
+        Check whether the object *subscriber* is registered directly
+        with this object via a previous call to
+        ``subscribe(required, provided, subscriber)``.
+
+        If the *subscriber*, or one equal to it, has been subscribed,
+        for the given *required* sequence and *provided* interface,
+        return that object. (This does not guarantee whether the *subscriber*
+        itself is returned, or an object equal to it.)
+
+        If it has not, return ``None``.
+
+        Unlike :meth:`subscriptions`, this method won't retrieve
+        components registered for more specific required interfaces or
+        less specific provided interfaces.
+
+        .. versionadded:: 5.3.0
+        """
+
+    def subscriptions(required, provided):
+        """
+        Get a sequence of subscribers.
+
+        Subscribers for a sequence of *required* interfaces, and a *provided*
+        interface are returned. This takes into account subscribers
+        registered with this object, as well as those registered with
+        base adapter registries in the resolution order, and interfaces that
+        extend *provided*.
+
+        .. versionchanged:: 5.1.1
+           Correct the method signature to remove the ``name`` parameter.
+           Subscribers have no names.
+        """
+
+    def subscribers(objects, provided):
+        """
+        Get a sequence of subscription **adapters**.
+
+        This is like :meth:`subscriptions`, but calls the returned
+        subscribers with *objects* (and optionally returns the results
+        of those calls), instead of returning the subscribers directly.
+
+        :param objects: A sequence of objects; they will be used to
+            determine the *required* argument to :meth:`subscriptions`.
+        :param provided: A single interface, or ``None``, to pass
+            as the *provided* parameter to :meth:`subscriptions`.
+            If an interface is given, the results of calling each returned
+            subscriber with the the *objects* are collected and returned
+            from this method; each result should be an object implementing
+            the *provided* interface. If ``None``, the resulting subscribers
+            are still called, but the results are ignored.
+        :return: A sequence of the results of calling the subscribers
+            if *provided* is not ``None``. If there are no registered
+            subscribers, or *provided* is ``None``, this will be an empty
+            sequence.
+
+        .. versionchanged:: 5.1.1
+           Correct the method signature to remove the ``name`` parameter.
+           Subscribers have no names.
+        """
+
+# begin formerly in zope.component
+
+class ComponentLookupError(LookupError):
+    """A component could not be found."""
+
+class Invalid(Exception):
+    """A component doesn't satisfy a promise."""
+
+class IObjectEvent(Interface):
+    """An event related to an object.
+
+    The object that generated this event is not necessarily the object
+    referred to by location.
+    """
+
+    object = Attribute("The subject of the event.")
+
+
+@implementer(IObjectEvent)
+class ObjectEvent(object):
+
+    def __init__(self, object): # pylint:disable=redefined-builtin
+        self.object = object
+
+
+class IComponentLookup(Interface):
+    """Component Manager for a Site
+
+    This object manages the components registered at a particular site. The
+    definition of a site is intentionally vague.
+    """
+
+    adapters = Attribute(
+        "Adapter Registry to manage all registered adapters.")
+
+    utilities = Attribute(
+        "Adapter Registry to manage all registered utilities.")
+
+    def queryAdapter(object, interface, name=u'', default=None): # pylint:disable=redefined-builtin
+        """Look for a named adapter to an interface for an object
+
+        If a matching adapter cannot be found, returns the default.
+        """
+
+    def getAdapter(object, interface, name=u''): # pylint:disable=redefined-builtin
+        """Look for a named adapter to an interface for an object
+
+        If a matching adapter cannot be found, a `ComponentLookupError`
+        is raised.
+        """
+
+    def queryMultiAdapter(objects, interface, name=u'', default=None):
+        """Look for a multi-adapter to an interface for multiple objects
+
+        If a matching adapter cannot be found, returns the default.
+        """
+
+    def getMultiAdapter(objects, interface, name=u''):
+        """Look for a multi-adapter to an interface for multiple objects
+
+        If a matching adapter cannot be found, a `ComponentLookupError`
+        is raised.
+        """
+
+    def getAdapters(objects, provided):
+        """Look for all matching adapters to a provided interface for objects
+
+        Return an iterable of name-adapter pairs for adapters that
+        provide the given interface.
+        """
+
+    def subscribers(objects, provided):
+        """Get subscribers
+
+        Subscribers are returned that provide the provided interface
+        and that depend on and are computed from the sequence of
+        required objects.
+        """
+
+    def handle(*objects):
+        """Call handlers for the given objects
+
+        Handlers registered for the given objects are called.
+        """
+
+    def queryUtility(interface, name='', default=None):
+        """Look up a utility that provides an interface.
+
+        If one is not found, returns default.
+        """
+
+    def getUtilitiesFor(interface):
+        """Look up the registered utilities that provide an interface.
+
+        Returns an iterable of name-utility pairs.
+        """
+
+    def getAllUtilitiesRegisteredFor(interface):
+        """Return all registered utilities for an interface
+
+        This includes overridden utilities.
+
+        An iterable of utility instances is returned.  No names are
+        returned.
+        """
+
+class IRegistration(Interface):
+    """A registration-information object
+    """
+
+    registry = Attribute("The registry having the registration")
+
+    name = Attribute("The registration name")
+
+    info = Attribute("""Information about the registration
+
+    This is information deemed useful to people browsing the
+    configuration of a system. It could, for example, include
+    commentary or information about the source of the configuration.
+    """)
+
+class IUtilityRegistration(IRegistration):
+    """Information about the registration of a utility
+    """
+
+    factory = Attribute("The factory used to create the utility. Optional.")
+    component = Attribute("The object registered")
+    provided = Attribute("The interface provided by the component")
+
+class _IBaseAdapterRegistration(IRegistration):
+    """Information about the registration of an adapter
+    """
+
+    factory = Attribute("The factory used to create adapters")
+
+    required = Attribute("""The adapted interfaces
+
+    This is a sequence of interfaces adapters by the registered
+    factory.  The factory will be caled with a sequence of objects, as
+    positional arguments, that provide these interfaces.
+    """)
+
+    provided = Attribute("""The interface provided by the adapters.
+
+    This interface is implemented by the factory
+    """)
+
+class IAdapterRegistration(_IBaseAdapterRegistration):
+    """Information about the registration of an adapter
+    """
+
+class ISubscriptionAdapterRegistration(_IBaseAdapterRegistration):
+    """Information about the registration of a subscription adapter
+    """
+
+class IHandlerRegistration(IRegistration):
+
+    handler = Attribute("An object called used to handle an event")
+
+    required = Attribute("""The handled interfaces
+
+    This is a sequence of interfaces handled by the registered
+    handler.  The handler will be caled with a sequence of objects, as
+    positional arguments, that provide these interfaces.
+    """)
+
+class IRegistrationEvent(IObjectEvent):
+    """An event that involves a registration"""
+
+
+@implementer(IRegistrationEvent)
+class RegistrationEvent(ObjectEvent):
+    """There has been a change in a registration
+    """
+    def __repr__(self):
+        return "%s event:\n%r" % (self.__class__.__name__, self.object)
+
+class IRegistered(IRegistrationEvent):
+    """A component or factory was registered
+    """
+
+@implementer(IRegistered)
+class Registered(RegistrationEvent):
+    pass
+
+class IUnregistered(IRegistrationEvent):
+    """A component or factory was unregistered
+    """
+
+@implementer(IUnregistered)
+class Unregistered(RegistrationEvent):
+    """A component or factory was unregistered
+    """
+
+
+class IComponentRegistry(Interface):
+    """Register components
+    """
+
+    def registerUtility(component=None, provided=None, name=u'',
+                        info=u'', factory=None):
+        """Register a utility
+
+        :param factory:
+           Factory for the component to be registered.
+
+        :param component:
+           The registered component
+
+        :param provided:
+           This is the interface provided by the utility.  If the
+           component provides a single interface, then this
+           argument is optional and the component-implemented
+           interface will be used.
+
+        :param name:
+           The utility name.
+
+        :param info:
+           An object that can be converted to a string to provide
+           information about the registration.
+
+        Only one of *component* and *factory* can be used.
+
+        A `IRegistered` event is generated with an `IUtilityRegistration`.
+        """
+
+    def unregisterUtility(component=None, provided=None, name=u'',
+                          factory=None):
+        """Unregister a utility
+
+        :returns:
+            A boolean is returned indicating whether the registry was
+            changed.  If the given *component* is None and there is no
+            component registered, or if the given *component* is not
+            None and is not registered, then the function returns
+            False, otherwise it returns True.
+
+        :param factory:
+           Factory for the component to be unregistered.
+
+        :param component:
+           The registered component The given component can be
+           None, in which case any component registered to provide
+           the given provided interface with the given name is
+           unregistered.
+
+        :param provided:
+           This is the interface provided by the utility.  If the
+           component is not None and provides a single interface,
+           then this argument is optional and the
+           component-implemented interface will be used.
+
+        :param name:
+           The utility name.
+
+        Only one of *component* and *factory* can be used.
+        An `IUnregistered` event is generated with an `IUtilityRegistration`.
+        """
+
+    def registeredUtilities():
+        """Return an iterable of `IUtilityRegistration` instances.
+
+        These registrations describe the current utility registrations
+        in the object.
+        """
+
+    def registerAdapter(factory, required=None, provided=None, name=u'',
+                        info=u''):
+        """Register an adapter factory
+
+        :param factory:
+            The object used to compute the adapter
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set in class definitions using
+            the `.adapter`
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param provided:
+            This is the interface provided by the adapter and
+            implemented by the factory.  If the factory
+            implements a single interface, then this argument is
+            optional and the factory-implemented interface will be
+            used.
+
+        :param name:
+            The adapter name.
+
+        :param info:
+           An object that can be converted to a string to provide
+           information about the registration.
+
+        A `IRegistered` event is generated with an `IAdapterRegistration`.
+        """
+
+    def unregisterAdapter(factory=None, required=None,
+                          provided=None, name=u''):
+        """Unregister an adapter factory
+
+        :returns:
+            A boolean is returned indicating whether the registry was
+            changed.  If the given component is None and there is no
+            component registered, or if the given component is not
+            None and is not registered, then the function returns
+            False, otherwise it returns True.
+
+        :param factory:
+            This is the object used to compute the adapter. The
+            factory can be None, in which case any factory
+            registered to implement the given provided interface
+            for the given required specifications with the given
+            name is unregistered.
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If the factory is not None and the required
+            arguments is omitted, then the value of the factory's
+            __component_adapts__ attribute will be used.  The
+            __component_adapts__ attribute attribute is normally
+            set in class definitions using adapts function, or for
+            callables using the adapter decorator.  If the factory
+            is None or doesn't have a __component_adapts__ adapts
+            attribute, then this argument is required.
+
+        :param provided:
+            This is the interface provided by the adapter and
+            implemented by the factory.  If the factory is not
+            None and implements a single interface, then this
+            argument is optional and the factory-implemented
+            interface will be used.
+
+        :param name:
+            The adapter name.
+
+        An `IUnregistered` event is generated with an `IAdapterRegistration`.
+        """
+
+    def registeredAdapters():
+        """Return an iterable of `IAdapterRegistration` instances.
+
+        These registrations describe the current adapter registrations
+        in the object.
+        """
+
+    def registerSubscriptionAdapter(factory, required=None, provides=None,
+                                    name=u'', info=''):
+        """Register a subscriber factory
+
+        :param factory:
+            The object used to compute the adapter
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set using the adapter
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param provided:
+            This is the interface provided by the adapter and
+            implemented by the factory.  If the factory implements
+            a single interface, then this argument is optional and
+            the factory-implemented interface will be used.
+
+        :param name:
+            The adapter name.
+
+            Currently, only the empty string is accepted.  Other
+            strings will be accepted in the future when support for
+            named subscribers is added.
+
+        :param info:
+           An object that can be converted to a string to provide
+           information about the registration.
+
+        A `IRegistered` event is generated with an
+        `ISubscriptionAdapterRegistration`.
+        """
+
+    def unregisterSubscriptionAdapter(factory=None, required=None,
+                                      provides=None, name=u''):
+        """Unregister a subscriber factory.
+
+        :returns:
+            A boolean is returned indicating whether the registry was
+            changed.  If the given component is None and there is no
+            component registered, or if the given component is not
+            None and is not registered, then the function returns
+            False, otherwise it returns True.
+
+        :param factory:
+            This is the object used to compute the adapter. The
+            factory can be None, in which case any factories
+            registered to implement the given provided interface
+            for the given required specifications with the given
+            name are unregistered.
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set using the adapter
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param provided:
+            This is the interface provided by the adapter and
+            implemented by the factory.  If the factory is not
+            None implements a single interface, then this argument
+            is optional and the factory-implemented interface will
+            be used.
+
+        :param name:
+            The adapter name.
+
+            Currently, only the empty string is accepted.  Other
+            strings will be accepted in the future when support for
+            named subscribers is added.
+
+        An `IUnregistered` event is generated with an
+        `ISubscriptionAdapterRegistration`.
+        """
+
+    def registeredSubscriptionAdapters():
+        """Return an iterable of `ISubscriptionAdapterRegistration` instances.
+
+        These registrations describe the current subscription adapter
+        registrations in the object.
+        """
+
+    def registerHandler(handler, required=None, name=u'', info=''):
+        """Register a handler.
+
+        A handler is a subscriber that doesn't compute an adapter
+        but performs some function when called.
+
+        :param handler:
+            The object used to handle some event represented by
+            the objects passed to it.
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set using the adapter
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param name:
+            The handler name.
+
+            Currently, only the empty string is accepted.  Other
+            strings will be accepted in the future when support for
+            named handlers is added.
+
+        :param info:
+           An object that can be converted to a string to provide
+           information about the registration.
+
+
+        A `IRegistered` event is generated with an `IHandlerRegistration`.
+        """
+
+    def unregisterHandler(handler=None, required=None, name=u''):
+        """Unregister a handler.
+
+        A handler is a subscriber that doesn't compute an adapter
+        but performs some function when called.
+
+        :returns: A boolean is returned indicating whether the registry was
+            changed.
+
+        :param handler:
+            This is the object used to handle some event
+            represented by the objects passed to it. The handler
+            can be None, in which case any handlers registered for
+            the given required specifications with the given are
+            unregistered.
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set using the adapter
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param name:
+            The handler name.
+
+            Currently, only the empty string is accepted.  Other
+            strings will be accepted in the future when support for
+            named handlers is added.
+
+        An `IUnregistered` event is generated with an `IHandlerRegistration`.
+        """
+
+    def registeredHandlers():
+        """Return an iterable of `IHandlerRegistration` instances.
+
+        These registrations describe the current handler registrations
+        in the object.
+        """
+
+
+class IComponents(IComponentLookup, IComponentRegistry):
+    """Component registration and access
+    """
+
+
+# end formerly in zope.component
diff --git a/contrib/python/zope.interface/py2/zope/interface/registry.py b/contrib/python/zope.interface/py2/zope/interface/registry.py
new file mode 100644
index 00000000000..4fdb120b7f5
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/registry.py
@@ -0,0 +1,726 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Basic components support
+"""
+from collections import defaultdict
+
+try:
+    from zope.event import notify
+except ImportError: # pragma: no cover
+    def notify(*arg, **kw): pass
+
+from zope.interface.interfaces import ISpecification
+from zope.interface.interfaces import ComponentLookupError
+from zope.interface.interfaces import IAdapterRegistration
+from zope.interface.interfaces import IComponents
+from zope.interface.interfaces import IHandlerRegistration
+from zope.interface.interfaces import ISubscriptionAdapterRegistration
+from zope.interface.interfaces import IUtilityRegistration
+from zope.interface.interfaces import Registered
+from zope.interface.interfaces import Unregistered
+
+from zope.interface.interface import Interface
+from zope.interface.declarations import implementedBy
+from zope.interface.declarations import implementer
+from zope.interface.declarations import implementer_only
+from zope.interface.declarations import providedBy
+from zope.interface.adapter import AdapterRegistry
+from zope.interface._compat import CLASS_TYPES
+from zope.interface._compat import STRING_TYPES
+
+__all__ = [
+    # Components is public API, but
+    # the *Registration classes are just implementations
+    # of public interfaces.
+    'Components',
+]
+
+class _UnhashableComponentCounter(object):
+    # defaultdict(int)-like object for unhashable components
+
+    def __init__(self, otherdict):
+        # [(component, count)]
+        self._data = [item for item in otherdict.items()]
+
+    def __getitem__(self, key):
+        for component, count in self._data:
+            if component == key:
+                return count
+        return 0
+
+    def __setitem__(self, component, count):
+        for i, data in enumerate(self._data):
+            if data[0] == component:
+                self._data[i] = component, count
+                return
+        self._data.append((component, count))
+
+    def __delitem__(self, component):
+        for i, data in enumerate(self._data):
+            if data[0] == component:
+                del self._data[i]
+                return
+        raise KeyError(component) # pragma: no cover
+
+def _defaultdict_int():
+    return defaultdict(int)
+
+class _UtilityRegistrations(object):
+
+    def __init__(self, utilities, utility_registrations):
+        # {provided -> {component: count}}
+        self._cache = defaultdict(_defaultdict_int)
+        self._utilities = utilities
+        self._utility_registrations = utility_registrations
+
+        self.__populate_cache()
+
+    def __populate_cache(self):
+        for ((p, _), data) in iter(self._utility_registrations.items()):
+            component = data[0]
+            self.__cache_utility(p, component)
+
+    def __cache_utility(self, provided, component):
+        try:
+            self._cache[provided][component] += 1
+        except TypeError:
+            # The component is not hashable, and we have a dict. Switch to a strategy
+            # that doesn't use hashing.
+            prov = self._cache[provided] = _UnhashableComponentCounter(self._cache[provided])
+            prov[component] += 1
+
+    def __uncache_utility(self, provided, component):
+        provided = self._cache[provided]
+        # It seems like this line could raise a TypeError if component isn't
+        # hashable and we haven't yet switched to _UnhashableComponentCounter. However,
+        # we can't actually get in that situation. In order to get here, we would
+        # have had to cache the utility already which would have switched
+        # the datastructure if needed.
+        count = provided[component]
+        count -= 1
+        if count == 0:
+            del provided[component]
+        else:
+            provided[component] = count
+        return count > 0
+
+    def _is_utility_subscribed(self, provided, component):
+        try:
+            return self._cache[provided][component] > 0
+        except TypeError:
+            # Not hashable and we're still using a dict
+            return False
+
+    def registerUtility(self, provided, name, component, info, factory):
+        subscribed = self._is_utility_subscribed(provided, component)
+
+        self._utility_registrations[(provided, name)] = component, info, factory
+        self._utilities.register((), provided, name, component)
+
+        if not subscribed:
+            self._utilities.subscribe((), provided, component)
+
+        self.__cache_utility(provided, component)
+
+    def unregisterUtility(self, provided, name, component):
+        del self._utility_registrations[(provided, name)]
+        self._utilities.unregister((), provided, name)
+
+        subscribed = self.__uncache_utility(provided, component)
+
+        if not subscribed:
+            self._utilities.unsubscribe((), provided, component)
+
+
+@implementer(IComponents)
+class Components(object):
+
+    _v_utility_registrations_cache = None
+
+    def __init__(self, name='', bases=()):
+        # __init__ is used for test cleanup as well as initialization.
+        # XXX add a separate API for test cleanup.
+        assert isinstance(name, STRING_TYPES)
+        self.__name__ = name
+        self._init_registries()
+        self._init_registrations()
+        self.__bases__ = tuple(bases)
+        self._v_utility_registrations_cache = None
+
+    def __repr__(self):
+        return "<%s %s>" % (self.__class__.__name__, self.__name__)
+
+    def __reduce__(self):
+        # Mimic what a persistent.Persistent object does and elide
+        # _v_ attributes so that they don't get saved in ZODB.
+        # This allows us to store things that cannot be pickled in such
+        # attributes.
+        reduction = super(Components, self).__reduce__()
+        # (callable, args, state, listiter, dictiter)
+        # We assume the state is always a dict; the last three items
+        # are technically optional and can be missing or None.
+        filtered_state = {k: v for k, v in reduction[2].items()
+                          if not k.startswith('_v_')}
+        reduction = list(reduction)
+        reduction[2] = filtered_state
+        return tuple(reduction)
+
+    def _init_registries(self):
+        # Subclasses have never been required to call this method
+        # if they override it, merely to fill in these two attributes.
+        self.adapters = AdapterRegistry()
+        self.utilities = AdapterRegistry()
+
+    def _init_registrations(self):
+        self._utility_registrations = {}
+        self._adapter_registrations = {}
+        self._subscription_registrations = []
+        self._handler_registrations = []
+
+    @property
+    def _utility_registrations_cache(self):
+        # We use a _v_ attribute internally so that data aren't saved in ZODB,
+        # because this object cannot be pickled.
+        cache = self._v_utility_registrations_cache
+        if (cache is None
+            or cache._utilities is not self.utilities
+            or cache._utility_registrations is not self._utility_registrations):
+            cache = self._v_utility_registrations_cache = _UtilityRegistrations(
+                self.utilities,
+                self._utility_registrations)
+        return cache
+
+    def _getBases(self):
+        # Subclasses might override
+        return self.__dict__.get('__bases__', ())
+
+    def _setBases(self, bases):
+        # Subclasses might override
+        self.adapters.__bases__ = tuple([
+            base.adapters for base in bases])
+        self.utilities.__bases__ = tuple([
+            base.utilities for base in bases])
+        self.__dict__['__bases__'] = tuple(bases)
+
+    __bases__ = property(
+        lambda self: self._getBases(),
+        lambda self, bases: self._setBases(bases),
+        )
+
+    def registerUtility(self, component=None, provided=None, name=u'',
+                        info=u'', event=True, factory=None):
+        if factory:
+            if component:
+                raise TypeError("Can't specify factory and component.")
+            component = factory()
+
+        if provided is None:
+            provided = _getUtilityProvided(component)
+
+        if name == u'':
+            name = _getName(component)
+
+        reg = self._utility_registrations.get((provided, name))
+        if reg is not None:
+            if reg[:2] == (component, info):
+                # already registered
+                return
+            self.unregisterUtility(reg[0], provided, name)
+
+        self._utility_registrations_cache.registerUtility(
+            provided, name, component, info, factory)
+
+        if event:
+            notify(Registered(
+                UtilityRegistration(self, provided, name, component, info,
+                                    factory)
+                ))
+
+    def unregisterUtility(self, component=None, provided=None, name=u'',
+                          factory=None):
+        if factory:
+            if component:
+                raise TypeError("Can't specify factory and component.")
+            component = factory()
+
+        if provided is None:
+            if component is None:
+                raise TypeError("Must specify one of component, factory and "
+                                "provided")
+            provided = _getUtilityProvided(component)
+
+        old = self._utility_registrations.get((provided, name))
+        if (old is None) or ((component is not None) and
+                             (component != old[0])):
+            return False
+
+        if component is None:
+            component = old[0]
+
+        # Note that component is now the old thing registered
+        self._utility_registrations_cache.unregisterUtility(
+            provided, name, component)
+
+        notify(Unregistered(
+            UtilityRegistration(self, provided, name, component, *old[1:])
+            ))
+
+        return True
+
+    def registeredUtilities(self):
+        for ((provided, name), data
+             ) in iter(self._utility_registrations.items()):
+            yield UtilityRegistration(self, provided, name, *data)
+
+    def queryUtility(self, provided, name=u'', default=None):
+        return self.utilities.lookup((), provided, name, default)
+
+    def getUtility(self, provided, name=u''):
+        utility = self.utilities.lookup((), provided, name)
+        if utility is None:
+            raise ComponentLookupError(provided, name)
+        return utility
+
+    def getUtilitiesFor(self, interface):
+        for name, utility in self.utilities.lookupAll((), interface):
+            yield name, utility
+
+    def getAllUtilitiesRegisteredFor(self, interface):
+        return self.utilities.subscriptions((), interface)
+
+    def registerAdapter(self, factory, required=None, provided=None,
+                        name=u'', info=u'', event=True):
+        if provided is None:
+            provided = _getAdapterProvided(factory)
+        required = _getAdapterRequired(factory, required)
+        if name == u'':
+            name = _getName(factory)
+        self._adapter_registrations[(required, provided, name)
+                                    ] = factory, info
+        self.adapters.register(required, provided, name, factory)
+
+        if event:
+            notify(Registered(
+                AdapterRegistration(self, required, provided, name,
+                                    factory, info)
+                ))
+
+
+    def unregisterAdapter(self, factory=None,
+                          required=None, provided=None, name=u'',
+                          ):
+        if provided is None:
+            if factory is None:
+                raise TypeError("Must specify one of factory and provided")
+            provided = _getAdapterProvided(factory)
+
+        if (required is None) and (factory is None):
+            raise TypeError("Must specify one of factory and required")
+
+        required = _getAdapterRequired(factory, required)
+        old = self._adapter_registrations.get((required, provided, name))
+        if (old is None) or ((factory is not None) and
+                             (factory != old[0])):
+            return False
+
+        del self._adapter_registrations[(required, provided, name)]
+        self.adapters.unregister(required, provided, name)
+
+        notify(Unregistered(
+            AdapterRegistration(self, required, provided, name,
+                                *old)
+            ))
+
+        return True
+
+    def registeredAdapters(self):
+        for ((required, provided, name), (component, info)
+             ) in iter(self._adapter_registrations.items()):
+            yield AdapterRegistration(self, required, provided, name,
+                                      component, info)
+
+    def queryAdapter(self, object, interface, name=u'', default=None):
+        return self.adapters.queryAdapter(object, interface, name, default)
+
+    def getAdapter(self, object, interface, name=u''):
+        adapter = self.adapters.queryAdapter(object, interface, name)
+        if adapter is None:
+            raise ComponentLookupError(object, interface, name)
+        return adapter
+
+    def queryMultiAdapter(self, objects, interface, name=u'',
+                          default=None):
+        return self.adapters.queryMultiAdapter(
+            objects, interface, name, default)
+
+    def getMultiAdapter(self, objects, interface, name=u''):
+        adapter = self.adapters.queryMultiAdapter(objects, interface, name)
+        if adapter is None:
+            raise ComponentLookupError(objects, interface, name)
+        return adapter
+
+    def getAdapters(self, objects, provided):
+        for name, factory in self.adapters.lookupAll(
+            list(map(providedBy, objects)),
+            provided):
+            adapter = factory(*objects)
+            if adapter is not None:
+                yield name, adapter
+
+    def registerSubscriptionAdapter(self,
+                                    factory, required=None, provided=None,
+                                    name=u'', info=u'',
+                                    event=True):
+        if name:
+            raise TypeError("Named subscribers are not yet supported")
+        if provided is None:
+            provided = _getAdapterProvided(factory)
+        required = _getAdapterRequired(factory, required)
+        self._subscription_registrations.append(
+            (required, provided, name, factory, info)
+            )
+        self.adapters.subscribe(required, provided, factory)
+
+        if event:
+            notify(Registered(
+                SubscriptionRegistration(self, required, provided, name,
+                                         factory, info)
+                ))
+
+    def registeredSubscriptionAdapters(self):
+        for data in self._subscription_registrations:
+            yield SubscriptionRegistration(self, *data)
+
+    def unregisterSubscriptionAdapter(self, factory=None,
+                          required=None, provided=None, name=u'',
+                          ):
+        if name:
+            raise TypeError("Named subscribers are not yet supported")
+        if provided is None:
+            if factory is None:
+                raise TypeError("Must specify one of factory and provided")
+            provided = _getAdapterProvided(factory)
+
+        if (required is None) and (factory is None):
+            raise TypeError("Must specify one of factory and required")
+
+        required = _getAdapterRequired(factory, required)
+
+        if factory is None:
+            new = [(r, p, n, f, i)
+                   for (r, p, n, f, i)
+                   in self._subscription_registrations
+                   if not (r == required and p == provided)
+                   ]
+        else:
+            new = [(r, p, n, f, i)
+                   for (r, p, n, f, i)
+                   in self._subscription_registrations
+                   if not (r == required and p == provided and f == factory)
+                   ]
+
+        if len(new) == len(self._subscription_registrations):
+            return False
+
+
+        self._subscription_registrations[:] = new
+        self.adapters.unsubscribe(required, provided, factory)
+
+        notify(Unregistered(
+            SubscriptionRegistration(self, required, provided, name,
+                                     factory, '')
+            ))
+
+        return True
+
+    def subscribers(self, objects, provided):
+        return self.adapters.subscribers(objects, provided)
+
+    def registerHandler(self,
+                        factory, required=None,
+                        name=u'', info=u'',
+                        event=True):
+        if name:
+            raise TypeError("Named handlers are not yet supported")
+        required = _getAdapterRequired(factory, required)
+        self._handler_registrations.append(
+            (required, name, factory, info)
+            )
+        self.adapters.subscribe(required, None, factory)
+
+        if event:
+            notify(Registered(
+                HandlerRegistration(self, required, name, factory, info)
+                ))
+
+    def registeredHandlers(self):
+        for data in self._handler_registrations:
+            yield HandlerRegistration(self, *data)
+
+    def unregisterHandler(self, factory=None, required=None, name=u''):
+        if name:
+            raise TypeError("Named subscribers are not yet supported")
+
+        if (required is None) and (factory is None):
+            raise TypeError("Must specify one of factory and required")
+
+        required = _getAdapterRequired(factory, required)
+
+        if factory is None:
+            new = [(r, n, f, i)
+                   for (r, n, f, i)
+                   in self._handler_registrations
+                   if r != required
+                   ]
+        else:
+            new = [(r, n, f, i)
+                   for (r, n, f, i)
+                   in self._handler_registrations
+                   if not (r == required and f == factory)
+                   ]
+
+        if len(new) == len(self._handler_registrations):
+            return False
+
+        self._handler_registrations[:] = new
+        self.adapters.unsubscribe(required, None, factory)
+
+        notify(Unregistered(
+            HandlerRegistration(self, required, name, factory, '')
+            ))
+
+        return True
+
+    def handle(self, *objects):
+        self.adapters.subscribers(objects, None)
+
+    def rebuildUtilityRegistryFromLocalCache(self, rebuild=False):
+        """
+        Emergency maintenance method to rebuild the ``.utilities``
+        registry from the local copy maintained in this object, or
+        detect the need to do so.
+
+        Most users will never need to call this, but it can be helpful
+        in the event of suspected corruption.
+
+        By default, this method only checks for corruption. To make it
+        actually rebuild the registry, pass `True` for *rebuild*.
+
+        :param bool rebuild: If set to `True` (not the default),
+           this method will actually register and subscribe utilities
+           in the registry as needed to synchronize with the local cache.
+
+        :return: A dictionary that's meant as diagnostic data. The keys
+           and values may change over time. When called with a false *rebuild*,
+           the keys ``"needed_registered"`` and ``"needed_subscribed"`` will be
+           non-zero if any corruption was detected, but that will not be corrected.
+
+        .. versionadded:: 5.3.0
+        """
+        regs = dict(self._utility_registrations)
+        utils = self.utilities
+        needed_registered = 0
+        did_not_register = 0
+        needed_subscribed = 0
+        did_not_subscribe = 0
+
+
+        # Avoid the expensive change process during this; we'll call
+        # it once at the end if needed.
+        assert 'changed' not in utils.__dict__
+        utils.changed = lambda _: None
+
+        if rebuild:
+            register = utils.register
+            subscribe = utils.subscribe
+        else:
+            register = subscribe = lambda *args: None
+
+        try:
+            for (provided, name), (value, _info, _factory) in regs.items():
+                if utils.registered((), provided, name) != value:
+                    register((), provided, name, value)
+                    needed_registered += 1
+                else:
+                    did_not_register += 1
+
+                if utils.subscribed((), provided, value) is None:
+                    needed_subscribed += 1
+                    subscribe((), provided, value)
+                else:
+                    did_not_subscribe += 1
+        finally:
+            del utils.changed
+            if rebuild and (needed_subscribed or needed_registered):
+                utils.changed(utils)
+
+        return {
+            'needed_registered': needed_registered,
+            'did_not_register': did_not_register,
+            'needed_subscribed': needed_subscribed,
+            'did_not_subscribe': did_not_subscribe
+        }
+
+def _getName(component):
+    try:
+        return component.__component_name__
+    except AttributeError:
+        return u''
+
+def _getUtilityProvided(component):
+    provided = list(providedBy(component))
+    if len(provided) == 1:
+        return provided[0]
+    raise TypeError(
+        "The utility doesn't provide a single interface "
+        "and no provided interface was specified.")
+
+def _getAdapterProvided(factory):
+    provided = list(implementedBy(factory))
+    if len(provided) == 1:
+        return provided[0]
+    raise TypeError(
+        "The adapter factory doesn't implement a single interface "
+        "and no provided interface was specified.")
+
+def _getAdapterRequired(factory, required):
+    if required is None:
+        try:
+            required = factory.__component_adapts__
+        except AttributeError:
+            raise TypeError(
+                "The adapter factory doesn't have a __component_adapts__ "
+                "attribute and no required specifications were specified"
+                )
+    elif ISpecification.providedBy(required):
+        raise TypeError("the required argument should be a list of "
+                        "interfaces, not a single interface")
+
+    result = []
+    for r in required:
+        if r is None:
+            r = Interface
+        elif not ISpecification.providedBy(r):
+            if isinstance(r, CLASS_TYPES):
+                r = implementedBy(r)
+            else:
+                raise TypeError("Required specification must be a "
+                                "specification or class, not %r" % type(r)
+                                )
+        result.append(r)
+    return tuple(result)
+
+
+@implementer(IUtilityRegistration)
+class UtilityRegistration(object):
+
+    def __init__(self, registry, provided, name, component, doc, factory=None):
+        (self.registry, self.provided, self.name, self.component, self.info,
+         self.factory
+         ) = registry, provided, name, component, doc, factory
+
+    def __repr__(self):
+        return '%s(%r, %s, %r, %s, %r, %r)' % (
+                self.__class__.__name__,
+                self.registry,
+                getattr(self.provided, '__name__', None), self.name,
+                getattr(self.component, '__name__', repr(self.component)),
+                self.factory, self.info,
+                )
+
+    def __hash__(self):
+        return id(self)
+
+    def __eq__(self, other):
+        return repr(self) == repr(other)
+
+    def __ne__(self, other):
+        return repr(self) != repr(other)
+
+    def __lt__(self, other):
+        return repr(self) < repr(other)
+
+    def __le__(self, other):
+        return repr(self) <= repr(other)
+
+    def __gt__(self, other):
+        return repr(self) > repr(other)
+
+    def __ge__(self, other):
+        return repr(self) >= repr(other)
+
+@implementer(IAdapterRegistration)
+class AdapterRegistration(object):
+
+    def __init__(self, registry, required, provided, name, component, doc):
+        (self.registry, self.required, self.provided, self.name,
+         self.factory, self.info
+         ) = registry, required, provided, name, component, doc
+
+    def __repr__(self):
+        return '%s(%r, %s, %s, %r, %s, %r)' % (
+            self.__class__.__name__,
+            self.registry,
+            '[' + ", ".join([r.__name__ for r in self.required]) + ']',
+            getattr(self.provided, '__name__', None), self.name,
+            getattr(self.factory, '__name__', repr(self.factory)), self.info,
+            )
+
+    def __hash__(self):
+        return id(self)
+
+    def __eq__(self, other):
+        return repr(self) == repr(other)
+
+    def __ne__(self, other):
+        return repr(self) != repr(other)
+
+    def __lt__(self, other):
+        return repr(self) < repr(other)
+
+    def __le__(self, other):
+        return repr(self) <= repr(other)
+
+    def __gt__(self, other):
+        return repr(self) > repr(other)
+
+    def __ge__(self, other):
+        return repr(self) >= repr(other)
+
+@implementer_only(ISubscriptionAdapterRegistration)
+class SubscriptionRegistration(AdapterRegistration):
+    pass
+
+
+@implementer_only(IHandlerRegistration)
+class HandlerRegistration(AdapterRegistration):
+
+    def __init__(self, registry, required, name, handler, doc):
+        (self.registry, self.required, self.name, self.handler, self.info
+         ) = registry, required, name, handler, doc
+
+    @property
+    def factory(self):
+        return self.handler
+
+    provided = None
+
+    def __repr__(self):
+        return '%s(%r, %s, %r, %s, %r)' % (
+            self.__class__.__name__,
+            self.registry,
+            '[' + ", ".join([r.__name__ for r in self.required]) + ']',
+            self.name,
+            getattr(self.factory, '__name__', repr(self.factory)), self.info,
+            )
diff --git a/contrib/python/zope.interface/py2/zope/interface/ro.py b/contrib/python/zope.interface/py2/zope/interface/ro.py
new file mode 100644
index 00000000000..89dde6799bd
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/ro.py
@@ -0,0 +1,666 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Compute a resolution order for an object and its bases.
+
+.. versionchanged:: 5.0
+   The resolution order is now based on the same C3 order that Python
+   uses for classes. In complex instances of multiple inheritance, this
+   may result in a different ordering.
+
+   In older versions, the ordering wasn't required to be C3 compliant,
+   and for backwards compatibility, it still isn't. If the ordering
+   isn't C3 compliant (if it is *inconsistent*), zope.interface will
+   make a best guess to try to produce a reasonable resolution order.
+   Still (just as before), the results in such cases may be
+   surprising.
+
+.. rubric:: Environment Variables
+
+Due to the change in 5.0, certain environment variables can be used to control errors
+and warnings about inconsistent resolution orders. They are listed in priority order, with
+variables at the bottom generally overriding variables above them.
+
+ZOPE_INTERFACE_WARN_BAD_IRO
+    If this is set to "1", then if there is at least one inconsistent resolution
+    order discovered, a warning (:class:`InconsistentResolutionOrderWarning`) will
+    be issued. Use the usual warning mechanisms to control this behaviour. The warning
+    text will contain additional information on debugging.
+ZOPE_INTERFACE_TRACK_BAD_IRO
+    If this is set to "1", then zope.interface will log information about each
+    inconsistent resolution order discovered, and keep those details in memory in this module
+    for later inspection.
+ZOPE_INTERFACE_STRICT_IRO
+    If this is set to "1", any attempt to use :func:`ro` that would produce a non-C3
+    ordering will fail by raising :class:`InconsistentResolutionOrderError`.
+
+.. important::
+
+   ``ZOPE_INTERFACE_STRICT_IRO`` is intended to become the default in the future.
+
+There are two environment variables that are independent.
+
+ZOPE_INTERFACE_LOG_CHANGED_IRO
+    If this is set to "1", then if the C3 resolution order is different from
+    the legacy resolution order for any given object, a message explaining the differences
+    will be logged. This is intended to be used for debugging complicated IROs.
+ZOPE_INTERFACE_USE_LEGACY_IRO
+    If this is set to "1", then the C3 resolution order will *not* be used. The
+    legacy IRO will be used instead. This is a temporary measure and will be removed in the
+    future. It is intended to help during the transition.
+    It implies ``ZOPE_INTERFACE_LOG_CHANGED_IRO``.
+
+.. rubric:: Debugging Behaviour Changes in zope.interface 5
+
+Most behaviour changes from zope.interface 4 to 5 are related to
+inconsistent resolution orders. ``ZOPE_INTERFACE_STRICT_IRO`` is the
+most effective tool to find such inconsistent resolution orders, and
+we recommend running your code with this variable set if at all
+possible. Doing so will ensure that all interface resolution orders
+are consistent, and if they're not, will immediately point the way to
+where this is violated.
+
+Occasionally, however, this may not be enough. This is because in some
+cases, a C3 ordering can be found (the resolution order is fully
+consistent) that is substantially different from the ad-hoc legacy
+ordering. In such cases, you may find that you get an unexpected value
+returned when adapting one or more objects to an interface. To debug
+this, *also* enable ``ZOPE_INTERFACE_LOG_CHANGED_IRO`` and examine the
+output. The main thing to look for is changes in the relative
+positions of interfaces for which there are registered adapters.
+"""
+from __future__ import print_function
+__docformat__ = 'restructuredtext'
+
+__all__ = [
+    'ro',
+    'InconsistentResolutionOrderError',
+    'InconsistentResolutionOrderWarning',
+]
+
+__logger = None
+
+def _logger():
+    global __logger # pylint:disable=global-statement
+    if __logger is None:
+        import logging
+        __logger = logging.getLogger(__name__)
+    return __logger
+
+def _legacy_mergeOrderings(orderings):
+    """Merge multiple orderings so that within-ordering order is preserved
+
+    Orderings are constrained in such a way that if an object appears
+    in two or more orderings, then the suffix that begins with the
+    object must be in both orderings.
+
+    For example:
+
+    >>> _mergeOrderings([
+    ... ['x', 'y', 'z'],
+    ... ['q', 'z'],
+    ... [1, 3, 5],
+    ... ['z']
+    ... ])
+    ['x', 'y', 'q', 1, 3, 5, 'z']
+
+    """
+
+    seen = set()
+    result = []
+    for ordering in reversed(orderings):
+        for o in reversed(ordering):
+            if o not in seen:
+                seen.add(o)
+                result.insert(0, o)
+
+    return result
+
+def _legacy_flatten(begin):
+    result = [begin]
+    i = 0
+    for ob in iter(result):
+        i += 1
+        # The recursive calls can be avoided by inserting the base classes
+        # into the dynamically growing list directly after the currently
+        # considered object;  the iterator makes sure this will keep working
+        # in the future, since it cannot rely on the length of the list
+        # by definition.
+        result[i:i] = ob.__bases__
+    return result
+
+def _legacy_ro(ob):
+    return _legacy_mergeOrderings([_legacy_flatten(ob)])
+
+###
+# Compare base objects using identity, not equality. This matches what
+# the CPython MRO algorithm does, and is *much* faster to boot: that,
+# plus some other small tweaks makes the difference between 25s and 6s
+# in loading 446 plone/zope interface.py modules (1925 InterfaceClass,
+# 1200 Implements, 1100 ClassProvides objects)
+###
+
+
+class InconsistentResolutionOrderWarning(PendingDeprecationWarning):
+    """
+    The warning issued when an invalid IRO is requested.
+    """
+
+class InconsistentResolutionOrderError(TypeError):
+    """
+    The error raised when an invalid IRO is requested in strict mode.
+    """
+
+    def __init__(self, c3, base_tree_remaining):
+        self.C = c3.leaf
+        base_tree = c3.base_tree
+        self.base_ros = {
+            base: base_tree[i + 1]
+            for i, base in enumerate(self.C.__bases__)
+        }
+        # Unfortunately, this doesn't necessarily directly match
+        # up to any transformation on C.__bases__, because
+        # if any were fully used up, they were removed already.
+        self.base_tree_remaining = base_tree_remaining
+
+        TypeError.__init__(self)
+
+    def __str__(self):
+        import pprint
+        return "%s: For object %r.\nBase ROs:\n%s\nConflict Location:\n%s" % (
+            self.__class__.__name__,
+            self.C,
+            pprint.pformat(self.base_ros),
+            pprint.pformat(self.base_tree_remaining),
+        )
+
+
+class _NamedBool(int): # cannot actually inherit bool
+
+    def __new__(cls, val, name):
+        inst = super(cls, _NamedBool).__new__(cls, val)
+        inst.__name__ = name
+        return inst
+
+
+class _ClassBoolFromEnv(object):
+    """
+    Non-data descriptor that reads a transformed environment variable
+    as a boolean, and caches the result in the class.
+    """
+
+    def __get__(self, inst, klass):
+        import os
+        for cls in klass.__mro__:
+            my_name = None
+            for k in dir(klass):
+                if k in cls.__dict__ and cls.__dict__[k] is self:
+                    my_name = k
+                    break
+            if my_name is not None:
+                break
+        else: # pragma: no cover
+            raise RuntimeError("Unable to find self")
+
+        env_name = 'ZOPE_INTERFACE_' + my_name
+        val = os.environ.get(env_name, '') == '1'
+        val = _NamedBool(val, my_name)
+        setattr(klass, my_name, val)
+        setattr(klass, 'ORIG_' + my_name, self)
+        return val
+
+
+class _StaticMRO(object):
+    # A previously resolved MRO, supplied by the caller.
+    # Used in place of calculating it.
+
+    had_inconsistency = None # We don't know...
+
+    def __init__(self, C, mro):
+        self.leaf = C
+        self.__mro = tuple(mro)
+
+    def mro(self):
+        return list(self.__mro)
+
+
+class C3(object):
+    # Holds the shared state during computation of an MRO.
+
+    @staticmethod
+    def resolver(C, strict, base_mros):
+        strict = strict if strict is not None else C3.STRICT_IRO
+        factory = C3
+        if strict:
+            factory = _StrictC3
+        elif C3.TRACK_BAD_IRO:
+            factory = _TrackingC3
+
+        memo = {}
+        base_mros = base_mros or {}
+        for base, mro in base_mros.items():
+            assert base in C.__bases__
+            memo[base] = _StaticMRO(base, mro)
+
+        return factory(C, memo)
+
+    __mro = None
+    __legacy_ro = None
+    direct_inconsistency = False
+
+    def __init__(self, C, memo):
+        self.leaf = C
+        self.memo = memo
+        kind = self.__class__
+
+        base_resolvers = []
+        for base in C.__bases__:
+            if base not in memo:
+                resolver = kind(base, memo)
+                memo[base] = resolver
+            base_resolvers.append(memo[base])
+
+        self.base_tree = [
+            [C]
+        ] + [
+            memo[base].mro() for base in C.__bases__
+        ] + [
+            list(C.__bases__)
+        ]
+
+        self.bases_had_inconsistency = any(base.had_inconsistency for base in base_resolvers)
+
+        if len(C.__bases__) == 1:
+            self.__mro = [C] + memo[C.__bases__[0]].mro()
+
+    @property
+    def had_inconsistency(self):
+        return self.direct_inconsistency or self.bases_had_inconsistency
+
+    @property
+    def legacy_ro(self):
+        if self.__legacy_ro is None:
+            self.__legacy_ro = tuple(_legacy_ro(self.leaf))
+        return list(self.__legacy_ro)
+
+    TRACK_BAD_IRO = _ClassBoolFromEnv()
+    STRICT_IRO = _ClassBoolFromEnv()
+    WARN_BAD_IRO = _ClassBoolFromEnv()
+    LOG_CHANGED_IRO = _ClassBoolFromEnv()
+    USE_LEGACY_IRO = _ClassBoolFromEnv()
+    BAD_IROS = ()
+
+    def _warn_iro(self):
+        if not self.WARN_BAD_IRO:
+            # For the initial release, one must opt-in to see the warning.
+            # In the future (2021?) seeing at least the first warning will
+            # be the default
+            return
+        import warnings
+        warnings.warn(
+            "An inconsistent resolution order is being requested. "
+            "(Interfaces should follow the Python class rules known as C3.) "
+            "For backwards compatibility, zope.interface will allow this, "
+            "making the best guess it can to produce as meaningful an order as possible. "
+            "In the future this might be an error. Set the warning filter to error, or set "
+            "the environment variable 'ZOPE_INTERFACE_TRACK_BAD_IRO' to '1' and examine "
+            "ro.C3.BAD_IROS to debug, or set 'ZOPE_INTERFACE_STRICT_IRO' to raise exceptions.",
+            InconsistentResolutionOrderWarning,
+        )
+
+    @staticmethod
+    def _can_choose_base(base, base_tree_remaining):
+        # From C3:
+        # nothead = [s for s in nonemptyseqs if cand in s[1:]]
+        for bases in base_tree_remaining:
+            if not bases or bases[0] is base:
+                continue
+
+            for b in bases:
+                if b is base:
+                    return False
+        return True
+
+    @staticmethod
+    def _nonempty_bases_ignoring(base_tree, ignoring):
+        return list(filter(None, [
+            [b for b in bases if b is not ignoring]
+            for bases
+            in base_tree
+        ]))
+
+    def _choose_next_base(self, base_tree_remaining):
+        """
+        Return the next base.
+
+        The return value will either fit the C3 constraints or be our best
+        guess about what to do. If we cannot guess, this may raise an exception.
+        """
+        base = self._find_next_C3_base(base_tree_remaining)
+        if base is not None:
+            return base
+        return self._guess_next_base(base_tree_remaining)
+
+    def _find_next_C3_base(self, base_tree_remaining):
+        """
+        Return the next base that fits the constraints, or ``None`` if there isn't one.
+        """
+        for bases in base_tree_remaining:
+            base = bases[0]
+            if self._can_choose_base(base, base_tree_remaining):
+                return base
+        return None
+
+    class _UseLegacyRO(Exception):
+        pass
+
+    def _guess_next_base(self, base_tree_remaining):
+        # Narf. We may have an inconsistent order (we won't know for
+        # sure until we check all the bases). Python cannot create
+        # classes like this:
+        #
+        # class B1:
+        #   pass
+        # class B2(B1):
+        #   pass
+        # class C(B1, B2): # -> TypeError; this is like saying C(B1, B2, B1).
+        #  pass
+        #
+        # However, older versions of zope.interface were fine with this order.
+        # A good example is ``providedBy(IOError())``. Because of the way
+        # ``classImplements`` works, it winds up with ``__bases__`` ==
+        # ``[IEnvironmentError, IIOError, IOSError, <implementedBy Exception>]``
+        # (on Python 3). But ``IEnvironmentError`` is a base of both ``IIOError``
+        # and ``IOSError``. Previously, we would get a resolution order of
+        # ``[IIOError, IOSError, IEnvironmentError, IStandardError, IException, Interface]``
+        # but the standard Python algorithm would forbid creating that order entirely.
+
+        # Unlike Python's MRO, we attempt to resolve the issue. A few
+        # heuristics have been tried. One was:
+        #
+        # Strip off the first (highest priority) base of each direct
+        # base one at a time and seeing if we can come to an agreement
+        # with the other bases. (We're trying for a partial ordering
+        # here.) This often resolves cases (such as the IOSError case
+        # above), and frequently produces the same ordering as the
+        # legacy MRO did. If we looked at all the highest priority
+        # bases and couldn't find any partial ordering, then we strip
+        # them *all* out and begin the C3 step again. We take care not
+        # to promote a common root over all others.
+        #
+        # If we only did the first part, stripped off the first
+        # element of the first item, we could resolve simple cases.
+        # But it tended to fail badly. If we did the whole thing, it
+        # could be extremely painful from a performance perspective
+        # for deep/wide things like Zope's OFS.SimpleItem.Item. Plus,
+        # anytime you get ExtensionClass.Base into the mix, you're
+        # likely to wind up in trouble, because it messes with the MRO
+        # of classes. Sigh.
+        #
+        # So now, we fall back to the old linearization (fast to compute).
+        self._warn_iro()
+        self.direct_inconsistency = InconsistentResolutionOrderError(self, base_tree_remaining)
+        raise self._UseLegacyRO
+
+    def _merge(self):
+        # Returns a merged *list*.
+        result = self.__mro = []
+        base_tree_remaining = self.base_tree
+        base = None
+        while 1:
+            # Take last picked base out of the base tree wherever it is.
+            # This differs slightly from the standard Python MRO and is needed
+            # because we have no other step that prevents duplicates
+            # from coming in (e.g., in the inconsistent fallback path)
+            base_tree_remaining = self._nonempty_bases_ignoring(base_tree_remaining, base)
+
+            if not base_tree_remaining:
+                return result
+            try:
+                base = self._choose_next_base(base_tree_remaining)
+            except self._UseLegacyRO:
+                self.__mro = self.legacy_ro
+                return self.legacy_ro
+
+            result.append(base)
+
+    def mro(self):
+        if self.__mro is None:
+            self.__mro = tuple(self._merge())
+        return list(self.__mro)
+
+
+class _StrictC3(C3):
+    __slots__ = ()
+    def _guess_next_base(self, base_tree_remaining):
+        raise InconsistentResolutionOrderError(self, base_tree_remaining)
+
+
+class _TrackingC3(C3):
+    __slots__ = ()
+    def _guess_next_base(self, base_tree_remaining):
+        import traceback
+        bad_iros = C3.BAD_IROS
+        if self.leaf not in bad_iros:
+            if bad_iros == ():
+                import weakref
+                # This is a race condition, but it doesn't matter much.
+                bad_iros = C3.BAD_IROS = weakref.WeakKeyDictionary()
+            bad_iros[self.leaf] = t = (
+                InconsistentResolutionOrderError(self, base_tree_remaining),
+                traceback.format_stack()
+            )
+            _logger().warning("Tracking inconsistent IRO: %s", t[0])
+        return C3._guess_next_base(self, base_tree_remaining)
+
+
+class _ROComparison(object):
+    # Exists to compute and print a pretty string comparison
+    # for differing ROs.
+    # Since we're used in a logging context, and may actually never be printed,
+    # this is a class so we can defer computing the diff until asked.
+
+    # Components we use to build up the comparison report
+    class Item(object):
+        prefix = '  '
+        def __init__(self, item):
+            self.item = item
+        def __str__(self):
+            return "%s%s" % (
+                self.prefix,
+                self.item,
+            )
+
+    class Deleted(Item):
+        prefix = '- '
+
+    class Inserted(Item):
+        prefix = '+ '
+
+    Empty = str
+
+    class ReplacedBy(object): # pragma: no cover
+        prefix = '- '
+        suffix = ''
+        def __init__(self, chunk, total_count):
+            self.chunk = chunk
+            self.total_count = total_count
+
+        def __iter__(self):
+            lines = [
+                self.prefix + str(item) + self.suffix
+                for item in self.chunk
+            ]
+            while len(lines) < self.total_count:
+                lines.append('')
+
+            return iter(lines)
+
+    class Replacing(ReplacedBy):
+        prefix = "+ "
+        suffix = ''
+
+
+    _c3_report = None
+    _legacy_report = None
+
+    def __init__(self, c3, c3_ro, legacy_ro):
+        self.c3 = c3
+        self.c3_ro = c3_ro
+        self.legacy_ro = legacy_ro
+
+    def __move(self, from_, to_, chunk, operation):
+        for x in chunk:
+            to_.append(operation(x))
+            from_.append(self.Empty())
+
+    def _generate_report(self):
+        if self._c3_report is None:
+            import difflib
+            # The opcodes we get describe how to turn 'a' into 'b'. So
+            # the old one (legacy) needs to be first ('a')
+            matcher = difflib.SequenceMatcher(None, self.legacy_ro, self.c3_ro)
+            # The reports are equal length sequences. We're going for a
+            # side-by-side diff.
+            self._c3_report = c3_report = []
+            self._legacy_report = legacy_report = []
+            for opcode, leg1, leg2, c31, c32 in matcher.get_opcodes():
+                c3_chunk = self.c3_ro[c31:c32]
+                legacy_chunk = self.legacy_ro[leg1:leg2]
+
+                if opcode == 'equal':
+                    # Guaranteed same length
+                    c3_report.extend((self.Item(x) for x in c3_chunk))
+                    legacy_report.extend(self.Item(x) for x in legacy_chunk)
+                if opcode == 'delete':
+                    # Guaranteed same length
+                    assert not c3_chunk
+                    self.__move(c3_report, legacy_report, legacy_chunk, self.Deleted)
+                if opcode == 'insert':
+                    # Guaranteed same length
+                    assert not legacy_chunk
+                    self.__move(legacy_report, c3_report, c3_chunk, self.Inserted)
+                if opcode == 'replace': # pragma: no cover (How do you make it output this?)
+                    # Either side could be longer.
+                    chunk_size = max(len(c3_chunk), len(legacy_chunk))
+                    c3_report.extend(self.Replacing(c3_chunk, chunk_size))
+                    legacy_report.extend(self.ReplacedBy(legacy_chunk, chunk_size))
+
+        return self._c3_report, self._legacy_report
+
+    @property
+    def _inconsistent_label(self):
+        inconsistent = []
+        if self.c3.direct_inconsistency:
+            inconsistent.append('direct')
+        if self.c3.bases_had_inconsistency:
+            inconsistent.append('bases')
+        return '+'.join(inconsistent) if inconsistent else 'no'
+
+    def __str__(self):
+        c3_report, legacy_report = self._generate_report()
+        assert len(c3_report) == len(legacy_report)
+
+        left_lines = [str(x) for x in legacy_report]
+        right_lines = [str(x) for x in c3_report]
+
+        # We have the same number of lines in the report; this is not
+        # necessarily the same as the number of items in either RO.
+        assert len(left_lines) == len(right_lines)
+
+        padding = ' ' * 2
+        max_left = max(len(x) for x in left_lines)
+        max_right = max(len(x) for x in right_lines)
+
+        left_title = 'Legacy RO (len=%s)' % (len(self.legacy_ro),)
+
+        right_title = 'C3 RO (len=%s; inconsistent=%s)' % (
+            len(self.c3_ro),
+            self._inconsistent_label,
+        )
+        lines = [
+            (padding + left_title.ljust(max_left) + padding + right_title.ljust(max_right)),
+            padding + '=' * (max_left + len(padding) + max_right)
+        ]
+        lines += [
+            padding + left.ljust(max_left) + padding + right
+            for left, right in zip(left_lines, right_lines)
+        ]
+
+        return '\n'.join(lines)
+
+
+# Set to `Interface` once it is defined. This is used to
+# avoid logging false positives about changed ROs.
+_ROOT = None
+
+def ro(C, strict=None, base_mros=None, log_changed_ro=None, use_legacy_ro=None):
+    """
+    ro(C) -> list
+
+    Compute the precedence list (mro) according to C3.
+
+    :return: A fresh `list` object.
+
+    .. versionchanged:: 5.0.0
+       Add the *strict*, *log_changed_ro* and *use_legacy_ro*
+       keyword arguments. These are provisional and likely to be
+       removed in the future. They are most useful for testing.
+    """
+    # The ``base_mros`` argument is for internal optimization and
+    # not documented.
+    resolver = C3.resolver(C, strict, base_mros)
+    mro = resolver.mro()
+
+    log_changed = log_changed_ro if log_changed_ro is not None else resolver.LOG_CHANGED_IRO
+    use_legacy = use_legacy_ro if use_legacy_ro is not None else resolver.USE_LEGACY_IRO
+
+    if log_changed or use_legacy:
+        legacy_ro = resolver.legacy_ro
+        assert isinstance(legacy_ro, list)
+        assert isinstance(mro, list)
+        changed = legacy_ro != mro
+        if changed:
+            # Did only Interface move? The fix for issue #8 made that
+            # somewhat common. It's almost certainly not a problem, though,
+            # so allow ignoring it.
+            legacy_without_root = [x for x in legacy_ro if x is not _ROOT]
+            mro_without_root = [x for x in mro if x is not _ROOT]
+            changed = legacy_without_root != mro_without_root
+
+        if changed:
+            comparison = _ROComparison(resolver, mro, legacy_ro)
+            _logger().warning(
+                "Object %r has different legacy and C3 MROs:\n%s",
+                C, comparison
+            )
+        if resolver.had_inconsistency and legacy_ro == mro:
+            comparison = _ROComparison(resolver, mro, legacy_ro)
+            _logger().warning(
+                "Object %r had inconsistent IRO and used the legacy RO:\n%s"
+                "\nInconsistency entered at:\n%s",
+                C, comparison, resolver.direct_inconsistency
+            )
+        if use_legacy:
+            return legacy_ro
+
+    return mro
+
+
+def is_consistent(C):
+    """
+    Check if the resolution order for *C*, as computed by :func:`ro`, is consistent
+    according to C3.
+    """
+    return not C3.resolver(C, False, None).had_inconsistency
diff --git a/contrib/python/zope.interface/py2/zope/interface/verify.py b/contrib/python/zope.interface/py2/zope/interface/verify.py
new file mode 100644
index 00000000000..0a64aeb65c6
--- /dev/null
+++ b/contrib/python/zope.interface/py2/zope/interface/verify.py
@@ -0,0 +1,218 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Verify interface implementations
+"""
+from __future__ import print_function
+import inspect
+import sys
+from types import FunctionType
+from types import MethodType
+
+from zope.interface._compat import PYPY2
+
+from zope.interface.exceptions import BrokenImplementation
+from zope.interface.exceptions import BrokenMethodImplementation
+from zope.interface.exceptions import DoesNotImplement
+from zope.interface.exceptions import Invalid
+from zope.interface.exceptions import MultipleInvalid
+
+from zope.interface.interface import fromMethod, fromFunction, Method
+
+__all__ = [
+    'verifyObject',
+    'verifyClass',
+]
+
+# This will be monkey-patched when running under Zope 2, so leave this
+# here:
+MethodTypes = (MethodType, )
+
+
+def _verify(iface, candidate, tentative=False, vtype=None):
+    """
+    Verify that *candidate* might correctly provide *iface*.
+
+    This involves:
+
+    - Making sure the candidate claims that it provides the
+      interface using ``iface.providedBy`` (unless *tentative* is `True`,
+      in which case this step is skipped). This means that the candidate's class
+      declares that it `implements <zope.interface.implementer>` the interface,
+      or the candidate itself declares that it `provides <zope.interface.provider>`
+      the interface
+
+    - Making sure the candidate defines all the necessary methods
+
+    - Making sure the methods have the correct signature (to the
+      extent possible)
+
+    - Making sure the candidate defines all the necessary attributes
+
+    :return bool: Returns a true value if everything that could be
+       checked passed.
+    :raises zope.interface.Invalid: If any of the previous
+       conditions does not hold.
+
+    .. versionchanged:: 5.0
+        If multiple methods or attributes are invalid, all such errors
+        are collected and reported. Previously, only the first error was reported.
+        As a special case, if only one such error is present, it is raised
+        alone, like before.
+    """
+
+    if vtype == 'c':
+        tester = iface.implementedBy
+    else:
+        tester = iface.providedBy
+
+    excs = []
+    if not tentative and not tester(candidate):
+        excs.append(DoesNotImplement(iface, candidate))
+
+    for name, desc in iface.namesAndDescriptions(all=True):
+        try:
+            _verify_element(iface, name, desc, candidate, vtype)
+        except Invalid as e:
+            excs.append(e)
+
+    if excs:
+        if len(excs) == 1:
+            raise excs[0]
+        raise MultipleInvalid(iface, candidate, excs)
+
+    return True
+
+def _verify_element(iface, name, desc, candidate, vtype):
+    # Here the `desc` is either an `Attribute` or `Method` instance
+    try:
+        attr = getattr(candidate, name)
+    except AttributeError:
+        if (not isinstance(desc, Method)) and vtype == 'c':
+            # We can't verify non-methods on classes, since the
+            # class may provide attrs in it's __init__.
+            return
+        # TODO: On Python 3, this should use ``raise...from``
+        raise BrokenImplementation(iface, desc, candidate)
+
+    if not isinstance(desc, Method):
+        # If it's not a method, there's nothing else we can test
+        return
+
+    if inspect.ismethoddescriptor(attr) or inspect.isbuiltin(attr):
+        # The first case is what you get for things like ``dict.pop``
+        # on CPython (e.g., ``verifyClass(IFullMapping, dict))``). The
+        # second case is what you get for things like ``dict().pop`` on
+        # CPython (e.g., ``verifyObject(IFullMapping, dict()))``.
+        # In neither case can we get a signature, so there's nothing
+        # to verify. Even the inspect module gives up and raises
+        # ValueError: no signature found. The ``__text_signature__`` attribute
+        # isn't typically populated either.
+        #
+        # Note that on PyPy 2 or 3 (up through 7.3 at least), these are
+        # not true for things like ``dict.pop`` (but might be true for C extensions?)
+        return
+
+    if isinstance(attr, FunctionType):
+        if sys.version_info[0] >= 3 and isinstance(candidate, type) and vtype == 'c':
+            # This is an "unbound method" in Python 3.
+            # Only unwrap this if we're verifying implementedBy;
+            # otherwise we can unwrap @staticmethod on classes that directly
+            # provide an interface.
+            meth = fromFunction(attr, iface, name=name,
+                                imlevel=1)
+        else:
+            # Nope, just a normal function
+            meth = fromFunction(attr, iface, name=name)
+    elif (isinstance(attr, MethodTypes)
+          and type(attr.__func__) is FunctionType):
+        meth = fromMethod(attr, iface, name)
+    elif isinstance(attr, property) and vtype == 'c':
+        # Without an instance we cannot be sure it's not a
+        # callable.
+        # TODO: This should probably check inspect.isdatadescriptor(),
+        # a more general form than ``property``
+        return
+
+    else:
+        if not callable(attr):
+            raise BrokenMethodImplementation(desc, "implementation is not a method",
+                                             attr, iface, candidate)
+        # sigh, it's callable, but we don't know how to introspect it, so
+        # we have to give it a pass.
+        return
+
+    # Make sure that the required and implemented method signatures are
+    # the same.
+    mess = _incompat(desc.getSignatureInfo(), meth.getSignatureInfo())
+    if mess:
+        if PYPY2 and _pypy2_false_positive(mess, candidate, vtype):
+            return
+        raise BrokenMethodImplementation(desc, mess, attr, iface, candidate)
+
+
+
+def verifyClass(iface, candidate, tentative=False):
+    """
+    Verify that the *candidate* might correctly provide *iface*.
+    """
+    return _verify(iface, candidate, tentative, vtype='c')
+
+def verifyObject(iface, candidate, tentative=False):
+    return _verify(iface, candidate, tentative, vtype='o')
+
+verifyObject.__doc__ = _verify.__doc__
+
+_MSG_TOO_MANY = 'implementation requires too many arguments'
+_KNOWN_PYPY2_FALSE_POSITIVES = frozenset((
+    _MSG_TOO_MANY,
+))
+
+
+def _pypy2_false_positive(msg, candidate, vtype):
+    # On PyPy2, builtin methods and functions like
+    # ``dict.pop`` that take pseudo-optional arguments
+    # (those with no default, something you can't express in Python 2
+    # syntax; CPython uses special internal APIs to implement these methods)
+    # return false failures because PyPy2 doesn't expose any way
+    # to detect this pseudo-optional status. PyPy3 doesn't have this problem
+    # because of __defaults_count__, and CPython never gets here because it
+    # returns true for ``ismethoddescriptor`` or ``isbuiltin``.
+    #
+    # We can't catch all such cases, but we can handle the common ones.
+    #
+    if msg not in _KNOWN_PYPY2_FALSE_POSITIVES:
+        return False
+
+    known_builtin_types = vars(__builtins__).values()
+    candidate_type = candidate if vtype == 'c' else type(candidate)
+    if candidate_type in known_builtin_types:
+        return True
+
+    return False
+
+
+def _incompat(required, implemented):
+    #if (required['positional'] !=
+    #    implemented['positional'][:len(required['positional'])]
+    #    and implemented['kwargs'] is None):
+    #    return 'imlementation has different argument names'
+    if len(implemented['required']) > len(required['required']):
+        return _MSG_TOO_MANY
+    if ((len(implemented['positional']) < len(required['positional']))
+        and not implemented['varargs']):
+        return "implementation doesn't allow enough arguments"
+    if required['kwargs'] and not implemented['kwargs']:
+        return "implementation doesn't support keyword arguments"
+    if required['varargs'] and not implemented['varargs']:
+        return "implementation doesn't support variable arguments"
diff --git a/contrib/python/zope.interface/py3/.dist-info/METADATA b/contrib/python/zope.interface/py3/.dist-info/METADATA
new file mode 100644
index 00000000000..f0bfe5f5850
--- /dev/null
+++ b/contrib/python/zope.interface/py3/.dist-info/METADATA
@@ -0,0 +1,1120 @@
+Metadata-Version: 2.1
+Name: zope.interface
+Version: 6.1
+Summary: Interfaces for Python
+Home-page: https://github.com/zopefoundation/zope.interface
+Author: Zope Foundation and Contributors
+Author-email: zope-dev@zope.org
+License: ZPL 2.1
+Keywords: interface,components,plugins
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Zope Public License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Framework :: Zope :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.7
+License-File: LICENSE.txt
+Requires-Dist: setuptools
+Provides-Extra: docs
+Requires-Dist: Sphinx ; extra == 'docs'
+Requires-Dist: repoze.sphinx.autointerface ; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
+Provides-Extra: test
+Requires-Dist: coverage >=5.0.3 ; extra == 'test'
+Requires-Dist: zope.event ; extra == 'test'
+Requires-Dist: zope.testing ; extra == 'test'
+Provides-Extra: testing
+Requires-Dist: coverage >=5.0.3 ; extra == 'testing'
+Requires-Dist: zope.event ; extra == 'testing'
+Requires-Dist: zope.testing ; extra == 'testing'
+
+====================
+ ``zope.interface``
+====================
+
+.. image:: https://img.shields.io/pypi/v/zope.interface.svg
+    :target: https://pypi.python.org/pypi/zope.interface/
+    :alt: Latest Version
+
+.. image:: https://img.shields.io/pypi/pyversions/zope.interface.svg
+        :target: https://pypi.org/project/zope.interface/
+        :alt: Supported Python versions
+
+.. image:: https://github.com/zopefoundation/zope.interface/actions/workflows/tests.yml/badge.svg
+        :target: https://github.com/zopefoundation/zope.interface/actions/workflows/tests.yml
+
+.. image:: https://readthedocs.org/projects/zopeinterface/badge/?version=latest
+        :target: https://zopeinterface.readthedocs.io/en/latest/
+        :alt: Documentation Status
+
+This package is intended to be independently reusable in any Python
+project. It is maintained by the `Zope Toolkit project
+<https://zopetoolkit.readthedocs.io/>`_.
+
+This package provides an implementation of "object interfaces" for Python.
+Interfaces are a mechanism for labeling objects as conforming to a given
+API or contract. So, this package can be considered as implementation of
+the `Design By Contract`_ methodology support in Python.
+
+.. _Design By Contract: http://en.wikipedia.org/wiki/Design_by_contract
+
+For detailed documentation, please see https://zopeinterface.readthedocs.io/en/latest/
+
+=========
+ Changes
+=========
+
+6.1 (2023-10-05)
+================
+
+- Build Linux binary wheels for Python 3.12.
+
+- Add support for Python 3.12.
+
+- Fix building of the docs for non-final versions.
+
+
+6.0 (2023-03-17)
+================
+
+- Build Linux binary wheels for Python 3.11.
+
+- Drop support for Python 2.7, 3.5, 3.6.
+
+- Fix test deprecation warning on Python 3.11.
+
+- Add preliminary support for Python 3.12 as of 3.12a5.
+
+- Drop:
+
+    + `zope.interface.implements`
+    + `zope.interface.implementsOnly`
+    + `zope.interface.classProvides`
+
+
+5.5.2 (2022-11-17)
+==================
+
+- Add support for building arm64 wheels on macOS.
+
+
+5.5.1 (2022-11-03)
+==================
+
+- Add support for final Python 3.11 release.
+
+
+5.5.0 (2022-10-10)
+==================
+
+- Add support for Python 3.10 and 3.11 (as of 3.11.0rc2).
+
+- Add missing Trove classifier showing support for Python 3.9.
+
+- Add some more entries to ``zope.interface.interfaces.__all__``.
+
+- Disable unsafe math optimizations in C code.  See `pull request 262
+  <https://github.com/zopefoundation/zope.interface/pull/262>`_.
+
+
+5.4.0 (2021-04-15)
+==================
+
+- Make the C implementation of the ``__providedBy__`` descriptor stop
+  ignoring all errors raised when accessing the instance's
+  ``__provides__``. Now it behaves like the Python version and only
+  catches ``AttributeError``. The previous behaviour could lead to
+  crashing the interpreter in cases of recursion and errors. See
+  `issue 239 <https://github.com/zopefoundation/zope.interface/issues>`_.
+
+- Update the ``repr()`` and ``str()`` of various objects to be shorter
+  and more informative. In many cases, the ``repr()`` is now something
+  that can be evaluated to produce an equal object. For example, what
+  was previously printed as ``<implementedBy builtins.list>`` is now
+  shown as ``classImplements(list, IMutableSequence, IIterable)``. See
+  `issue 236 <https://github.com/zopefoundation/zope.interface/issues/236>`_.
+
+- Make ``Declaration.__add__`` (as in ``implementedBy(Cls) +
+  ISomething``) try harder to preserve a consistent resolution order
+  when the two arguments share overlapping pieces of the interface
+  inheritance hierarchy. Previously, the right hand side was always
+  put at the end of the resolution order, which could easily produce
+  invalid orders. See `issue 193
+  <https://github.com/zopefoundation/zope.interface/issues/193>`_.
+
+5.3.0 (2020-03-21)
+==================
+
+- No changes from 5.3.0a1
+
+
+5.3.0a1 (2021-03-18)
+====================
+
+- Improve the repr of ``zope.interface.Provides`` to remove ambiguity
+  about what is being provided. This is especially helpful diagnosing
+  IRO issues.
+
+- Allow subclasses of ``BaseAdapterRegistry`` (including
+  ``AdapterRegistry`` and ``VerifyingAdapterRegistry``) to have
+  control over the data structures. This allows persistent
+  implementations such as those based on ZODB to choose more scalable
+  options (e.g., BTrees instead of dicts). See `issue 224
+  <https://github.com/zopefoundation/zope.interface/issues/224>`_.
+
+- Fix a reference counting issue in ``BaseAdapterRegistry`` that could
+  lead to references to interfaces being kept around even when all
+  utilities/adapters/subscribers providing that interface have been
+  removed. This is mostly an issue for persistent implementations.
+  Note that this only corrects the issue moving forward, it does not
+  solve any already corrupted reference counts. See `issue 227
+  <https://github.com/zopefoundation/zope.interface/issues/227>`_.
+
+- Add the method ``BaseAdapterRegistry.rebuild()``. This can be used
+  to fix the reference counting issue mentioned above, as well as to
+  update the data structures when custom data types have changed.
+
+- Add the interface method ``IAdapterRegistry.subscribed()`` and
+  implementation ``BaseAdapterRegistry.subscribed()`` for querying
+  directly registered subscribers. See `issue 230
+  <https://github.com/zopefoundation/zope.interface/issues/230>`_.
+
+- Add the maintenance method
+  ``Components.rebuildUtilityRegistryFromLocalCache()``. Most users
+  will not need this, but it can be useful if the ``Components.utilities``
+  registry is suspected to be out of sync with the ``Components``
+  object itself (this might happen to persistent ``Components``
+  implementations in the face of bugs).
+
+- Fix the ``Provides`` and ``ClassProvides`` descriptors to stop
+  allowing redundant interfaces (those already implemented by the
+  underlying class or meta class) to produce an inconsistent
+  resolution order. This is similar to the change in ``@implementer``
+  in 5.1.0, and resolves inconsistent resolution orders with
+  ``zope.proxy`` and ``zope.location``. See `issue 207
+  <https://github.com/zopefoundation/zope.interface/issues/207>`_.
+
+5.2.0 (2020-11-05)
+==================
+
+- Add documentation section ``Persistency and Equality``
+  (`#218 <https://github.com/zopefoundation/zope.interface/issues/218>`_).
+
+- Create arm64 wheels.
+
+- Add support for Python 3.9.
+
+
+5.1.2 (2020-10-01)
+==================
+
+- Make sure to call each invariant only once when validating invariants.
+  Previously, invariants could be called multiple times because when an
+  invariant is defined in an interface, it's found by in all interfaces
+  inheriting from that interface.  See `pull request 215
+  <https://github.com/zopefoundation/zope.interface/pull/215/>`_.
+
+5.1.1 (2020-09-30)
+==================
+
+- Fix the method definitions of ``IAdapterRegistry.subscribe``,
+  ``subscriptions`` and ``subscribers``. Previously, they all were
+  defined to accept a ``name`` keyword argument, but subscribers have
+  no names and the implementation of that interface did not accept
+  that argument. See `issue 208
+  <https://github.com/zopefoundation/zope.interface/issues/208>`_.
+
+- Fix a potential reference leak in the C optimizations. Previously,
+  applications that dynamically created unique ``Specification``
+  objects (e.g., used ``@implementer`` on dynamic classes) could
+  notice a growth of small objects over time leading to increased
+  garbage collection times. See `issue 216
+  <https://github.com/zopefoundation/zope.interface/issues/216>`_.
+
+  .. caution::
+
+     This leak could prevent interfaces used as the bases of
+     other interfaces from being garbage collected. Those interfaces
+     will now be collected.
+
+     One way in which this would manifest was that ``weakref.ref``
+     objects (and things built upon them, like
+     ``Weak[Key|Value]Dictionary``) would continue to have access to
+     the original object even if there were no other visible
+     references to Python and the original object *should* have been
+     collected. This could be especially problematic for the
+     ``WeakKeyDictionary`` when combined with dynamic or local
+     (created in the scope of a function) interfaces, since interfaces
+     are hashed based just on their name and module name. See the
+     linked issue for an example of a resulting ``KeyError``.
+
+     Note that such potential errors are not new, they are just once
+     again a possibility.
+
+5.1.0 (2020-04-08)
+==================
+
+- Make ``@implementer(*iface)`` and ``classImplements(cls, *iface)``
+  ignore redundant interfaces. If the class already implements an
+  interface through inheritance, it is no longer redeclared
+  specifically for *cls*. This solves many instances of inconsistent
+  resolution orders, while still allowing the interface to be declared
+  for readability and maintenance purposes. See `issue 199
+  <https://github.com/zopefoundation/zope.interface/issues/199>`_.
+
+- Remove all bare ``except:`` statements. Previously, when accessing
+  special attributes such as ``__provides__``, ``__providedBy__``,
+  ``__class__`` and ``__conform__``, this package wrapped such access
+  in a bare ``except:`` statement, meaning that many errors could pass
+  silently; typically this would result in a fallback path being taken
+  and sometimes (like with ``providedBy()``) the result would be
+  non-sensical. This is especially true when those attributes are
+  implemented with descriptors. Now, only ``AttributeError`` is
+  caught. This makes errors more obvious.
+
+  Obviously, this means that some exceptions will be propagated
+  differently than before. In particular, ``RuntimeError`` raised by
+  Acquisition in the case of circular containment will now be
+  propagated. Previously, when adapting such a broken object, a
+  ``TypeError`` would be the common result, but now it will be a more
+  informative ``RuntimeError``.
+
+  In addition, ZODB errors like ``POSKeyError`` could now be
+  propagated where previously they would ignored by this package.
+
+  See `issue 200 <https://github.com/zopefoundation/zope.interface/issues/200>`_.
+
+- Require that the second argument (*bases*) to ``InterfaceClass`` is
+  a tuple. This only matters when directly using ``InterfaceClass`` to
+  create new interfaces dynamically. Previously, an individual
+  interface was allowed, but did not work correctly. Now it is
+  consistent with ``type`` and requires a tuple.
+
+- Let interfaces define custom ``__adapt__`` methods. This implements
+  the other side of the :pep:`246` adaptation protocol: objects being
+  adapted could already implement ``__conform__`` if they know about
+  the interface, and now interfaces can implement ``__adapt__`` if
+  they know about particular objects. There is no performance penalty
+  for interfaces that do not supply custom ``__adapt__`` methods.
+
+  This includes the ability to add new methods, or override existing
+  interface methods using the new ``@interfacemethod`` decorator.
+
+  See `issue 3 <https://github.com/zopefoundation/zope.interface/issues/3>`_.
+
+- Make the internal singleton object returned by APIs like
+  ``implementedBy`` and ``directlyProvidedBy`` for objects that
+  implement or provide no interfaces more immutable. Previously an
+  internal cache could be mutated. See `issue 204
+  <https://github.com/zopefoundation/zope.interface/issues/204>`_.
+
+5.0.2 (2020-03-30)
+==================
+
+- Ensure that objects that implement no interfaces (such as direct
+  subclasses of ``object``) still include ``Interface`` itself in
+  their ``__iro___`` and ``__sro___``. This fixes adapter registry
+  lookups for such objects when the adapter is registered for
+  ``Interface``. See `issue 197
+  <https://github.com/zopefoundation/zope.interface/issues/197>`_.
+
+
+5.0.1 (2020-03-21)
+==================
+
+- Ensure the resolution order for ``InterfaceClass`` is consistent.
+  See `issue 192 <https://github.com/zopefoundation/zope.interface/issues/192>`_.
+
+- Ensure the resolution order for ``collections.OrderedDict`` is
+  consistent on CPython 2. (It was already consistent on Python 3 and PyPy).
+
+- Fix the handling of the ``ZOPE_INTERFACE_STRICT_IRO`` environment
+  variable. Previously, ``ZOPE_INTERFACE_STRICT_RO`` was read, in
+  contrast with the documentation. See `issue 194
+  <https://github.com/zopefoundation/zope.interface/issues/194>`_.
+
+
+5.0.0 (2020-03-19)
+==================
+
+- Make an internal singleton object returned by APIs like
+  ``implementedBy`` and ``directlyProvidedBy`` immutable. Previously,
+  it was fully mutable and allowed changing its ``__bases___``. That
+  could potentially lead to wrong results in pathological corner
+  cases. See `issue 158
+  <https://github.com/zopefoundation/zope.interface/issues/158>`_.
+
+- Support the ``PURE_PYTHON`` environment variable at runtime instead
+  of just at wheel build time. A value of 0 forces the C extensions to
+  be used (even on PyPy) failing if they aren't present. Any other
+  value forces the Python implementation to be used, ignoring the C
+  extensions. See `PR 151 <https://github.com/zopefoundation/zope.interface/pull/151>`_.
+
+- Cache the result of ``__hash__`` method in ``InterfaceClass`` as a
+  speed optimization. The method is called very often (i.e several
+  hundred thousand times during Plone 5.2 startup). Because the hash value never
+  changes it can be cached. This improves test performance from 0.614s
+  down to 0.575s (1.07x faster). In a real world Plone case a reindex
+  index came down from 402s to 320s (1.26x faster). See `PR 156
+  <https://github.com/zopefoundation/zope.interface/pull/156>`_.
+
+- Change the C classes ``SpecificationBase`` and its subclass
+  ``ClassProvidesBase`` to store implementation attributes in their structures
+  instead of their instance dictionaries. This eliminates the use of
+  an undocumented private C API function, and helps make some
+  instances require less memory. See `PR 154 <https://github.com/zopefoundation/zope.interface/pull/154>`_.
+
+- Reduce memory usage in other ways based on observations of usage
+  patterns in Zope (3) and Plone code bases.
+
+  - Specifications with no dependents are common (more than 50%) so
+    avoid allocating a ``WeakKeyDictionary`` unless we need it.
+  - Likewise, tagged values are relatively rare, so don't allocate a
+    dictionary to hold them until they are used.
+  - Use ``__slots___`` or the C equivalent ``tp_members`` in more
+    common places. Note that this removes the ability to set arbitrary
+    instance variables on certain objects.
+    See `PR 155 <https://github.com/zopefoundation/zope.interface/pull/155>`_.
+
+  The changes in this release resulted in a 7% memory reduction after
+  loading about 6,000 modules that define about 2,200 interfaces.
+
+  .. caution::
+
+     Details of many private attributes have changed, and external use
+     of those private attributes may break. In particular, the
+     lifetime and default value of ``_v_attrs`` has changed.
+
+- Remove support for hashing uninitialized interfaces. This could only
+  be done by subclassing ``InterfaceClass``. This has generated a
+  warning since it was first added in 2011 (3.6.5). Please call the
+  ``InterfaceClass`` constructor or otherwise set the appropriate
+  fields in your subclass before attempting to hash or sort it. See
+  `issue 157 <https://github.com/zopefoundation/zope.interface/issues/157>`_.
+
+- Remove unneeded override of the ``__hash__`` method from
+  ``zope.interface.declarations.Implements``. Watching a reindex index
+  process in ZCatalog with on a Py-Spy after 10k samples the time for
+  ``.adapter._lookup`` was reduced from 27.5s to 18.8s (~1.5x faster).
+  Overall reindex index time shrunk from 369s to 293s (1.26x faster).
+  See `PR 161
+  <https://github.com/zopefoundation/zope.interface/pull/161>`_.
+
+- Make the Python implementation closer to the C implementation by
+  ignoring all exceptions, not just ``AttributeError``, during (parts
+  of) interface adaptation. See `issue 163
+  <https://github.com/zopefoundation/zope.interface/issues/163>`_.
+
+- Micro-optimization in ``.adapter._lookup`` , ``.adapter._lookupAll``
+  and ``.adapter._subscriptions``: By loading ``components.get`` into
+  a local variable before entering the loop a bytcode "LOAD_FAST 0
+  (components)" in the loop can be eliminated. In Plone, while running
+  all tests, average speedup of the "owntime" of ``_lookup`` is ~5x.
+  See `PR 167
+  <https://github.com/zopefoundation/zope.interface/pull/167>`_.
+
+- Add ``__all__`` declarations to all modules. This helps tools that
+  do auto-completion and documentation and results in less cluttered
+  results. Wildcard ("*") are not recommended and may be affected. See
+  `issue 153
+  <https://github.com/zopefoundation/zope.interface/issues/153>`_.
+
+- Fix ``verifyClass`` and ``verifyObject`` for builtin types like
+  ``dict`` that have methods taking an optional, unnamed argument with
+  no default value like ``dict.pop``. On PyPy3, the verification is
+  strict, but on PyPy2 (as on all versions of CPython) those methods
+  cannot be verified and are ignored. See `issue 118
+  <https://github.com/zopefoundation/zope.interface/issues/118>`_.
+
+- Update the common interfaces ``IEnumerableMapping``,
+  ``IExtendedReadMapping``, ``IExtendedWriteMapping``,
+  ``IReadSequence`` and ``IUniqueMemberWriteSequence`` to no longer
+  require methods that were removed from Python 3 on Python 3, such as
+  ``__setslice___``. Now, ``dict``, ``list`` and ``tuple`` properly
+  verify as ``IFullMapping``, ``ISequence`` and ``IReadSequence,``
+  respectively on all versions of Python.
+
+- Add human-readable ``__str___`` and ``__repr___`` to ``Attribute``
+  and ``Method``. These contain the name of the defining interface
+  and the attribute. For methods, it also includes the signature.
+
+- Change the error strings raised by ``verifyObject`` and
+  ``verifyClass``. They now include more human-readable information
+  and exclude extraneous lines and spaces. See `issue 170
+  <https://github.com/zopefoundation/zope.interface/issues/170>`_.
+
+  .. caution:: This will break consumers (such as doctests) that
+               depended on the exact error messages.
+
+- Make ``verifyObject`` and ``verifyClass`` report all errors, if the
+  candidate object has multiple detectable violations. Previously they
+  reported only the first error. See `issue
+  <https://github.com/zopefoundation/zope.interface/issues/171>`_.
+
+  Like the above, this will break consumers depending on the exact
+  output of error messages if more than one error is present.
+
+- Add ``zope.interface.common.collections``,
+  ``zope.interface.common.numbers``, and ``zope.interface.common.io``.
+  These modules define interfaces based on the ABCs defined in the
+  standard library ``collections.abc``, ``numbers`` and ``io``
+  modules, respectively. Importing these modules will make the
+  standard library concrete classes that are registered with those
+  ABCs declare the appropriate interface. See `issue 138
+  <https://github.com/zopefoundation/zope.interface/issues/138>`_.
+
+- Add ``zope.interface.common.builtins``. This module defines
+  interfaces of common builtin types, such as ``ITextString`` and
+  ``IByteString``, ``IDict``, etc. These interfaces extend the
+  appropriate interfaces from ``collections`` and ``numbers``, and the
+  standard library classes implement them after importing this module.
+  This is intended as a replacement for third-party packages like
+  `dolmen.builtins <https://pypi.org/project/dolmen.builtins/>`_.
+  See `issue 138 <https://github.com/zopefoundation/zope.interface/issues/138>`_.
+
+- Make ``providedBy()`` and ``implementedBy()`` respect ``super``
+  objects. For instance, if class ``Derived`` implements ``IDerived``
+  and extends ``Base`` which in turn implements ``IBase``, then
+  ``providedBy(super(Derived, derived))`` will return ``[IBase]``.
+  Previously it would have returned ``[IDerived]`` (in general, it
+  would previously have returned whatever would have been returned
+  without ``super``).
+
+  Along with this change, adapter registries will unpack ``super``
+  objects into their ``__self___`` before passing it to the factory.
+  Together, this means that ``component.getAdapter(super(Derived,
+  self), ITarget)`` is now meaningful.
+
+  See `issue 11 <https://github.com/zopefoundation/zope.interface/issues/11>`_.
+
+- Fix a potential interpreter crash in the low-level adapter
+  registry lookup functions. See issue 11.
+
+- Adopt Python's standard `C3 resolution order
+  <https://www.python.org/download/releases/2.3/mro/>`_ to compute the
+  ``__iro__`` and ``__sro__`` of interfaces, with tweaks to support
+  additional cases that are common in interfaces but disallowed for
+  Python classes. Previously, an ad-hoc ordering that made no
+  particular guarantees was used.
+
+  This has many beneficial properties, including the fact that base
+  interface and base classes tend to appear near the end of the
+  resolution order instead of the beginning. The resolution order in
+  general should be more predictable and consistent.
+
+  .. caution::
+     In some cases, especially with complex interface inheritance
+     trees or when manually providing or implementing interfaces, the
+     resulting IRO may be quite different. This may affect adapter
+     lookup.
+
+  The C3 order enforces some constraints in order to be able to
+  guarantee a sensible ordering. Older versions of zope.interface did
+  not impose similar constraints, so it was possible to create
+  interfaces and declarations that are inconsistent with the C3
+  constraints. In that event, zope.interface will still produce a
+  resolution order equal to the old order, but it won't be guaranteed
+  to be fully C3 compliant. In the future, strict enforcement of C3
+  order may be the default.
+
+  A set of environment variables and module constants allows
+  controlling several aspects of this new behaviour. It is possible to
+  request warnings about inconsistent resolution orders encountered,
+  and even to forbid them. Differences between the C3 resolution order
+  and the previous order can be logged, and, in extreme cases, the
+  previous order can still be used (this ability will be removed in
+  the future). For details, see the documentation for
+  ``zope.interface.ro``.
+
+- Make inherited tagged values in interfaces respect the resolution
+  order (``__iro__``), as method and attribute lookup does. Previously
+  tagged values could give inconsistent results. See `issue 190
+  <https://github.com/zopefoundation/zope.interface/issues/190>`_.
+
+- Add ``getDirectTaggedValue`` (and related methods) to interfaces to
+  allow accessing tagged values irrespective of inheritance. See
+  `issue 190
+  <https://github.com/zopefoundation/zope.interface/issues/190>`_.
+
+- Ensure that ``Interface`` is always the last item in the ``__iro__``
+  and ``__sro__``. This is usually the case, but if classes that do
+  not implement any interfaces are part of a class inheritance
+  hierarchy, ``Interface`` could be assigned too high a priority.
+  See `issue 8 <https://github.com/zopefoundation/zope.interface/issues/8>`_.
+
+- Implement sorting, equality, and hashing in C for ``Interface``
+  objects. In micro benchmarks, this makes those operations 40% to 80%
+  faster. This translates to a 20% speed up in querying adapters.
+
+  Note that this changes certain implementation details. In
+  particular, ``InterfaceClass`` now has a non-default metaclass, and
+  it is enforced that ``__module__`` in instances of
+  ``InterfaceClass`` is read-only.
+
+  See `PR 183 <https://github.com/zopefoundation/zope.interface/pull/183>`_.
+
+
+4.7.2 (2020-03-10)
+==================
+
+- Remove deprecated use of setuptools features.  See `issue 30
+  <https://github.com/zopefoundation/zope.interface/issues/30>`_.
+
+
+4.7.1 (2019-11-11)
+==================
+
+- Use Python 3 syntax in the documentation.  See `issue 119
+  <https://github.com/zopefoundation/zope.interface/issues/119>`_.
+
+
+4.7.0 (2019-11-11)
+==================
+
+- Drop support for Python 3.4.
+
+- Change ``queryTaggedValue``, ``getTaggedValue``,
+  ``getTaggedValueTags`` in interfaces. They now include inherited
+  values by following ``__bases__``. See `PR 144
+  <https://github.com/zopefoundation/zope.interface/pull/144>`_.
+
+  .. caution:: This may be a breaking change.
+
+- Add support for Python 3.8.
+
+
+4.6.0 (2018-10-23)
+==================
+
+- Add support for Python 3.7
+
+- Fix ``verifyObject`` for class objects with staticmethods on
+  Python 3. See `issue 126
+  <https://github.com/zopefoundation/zope.interface/issues/126>`_.
+
+
+4.5.0 (2018-04-19)
+==================
+
+- Drop support for 3.3, avoid accidental dependence breakage via setup.py.
+  See `PR 110 <https://github.com/zopefoundation/zope.interface/pull/110>`_.
+- Allow registering and unregistering instance methods as listeners.
+  See `issue 12 <https://github.com/zopefoundation/zope.interface/issues/12>`_
+  and `PR 102 <https://github.com/zopefoundation/zope.interface/pull/102>`_.
+- Synchronize and simplify zope/__init__.py. See `issue 114
+  <https://github.com/zopefoundation/zope.interface/issues/114>`_
+
+
+4.4.3 (2017-09-22)
+==================
+
+- Avoid exceptions when the ``__annotations__`` attribute is added to
+  interface definitions with Python 3.x type hints. See `issue 98
+  <https://github.com/zopefoundation/zope.interface/issues/98>`_.
+- Fix the possibility of a rare crash in the C extension when
+  deallocating items. See `issue 100
+  <https://github.com/zopefoundation/zope.interface/issues/100>`_.
+
+
+4.4.2 (2017-06-14)
+==================
+
+- Fix a regression storing
+  ``zope.component.persistentregistry.PersistentRegistry`` instances.
+  See `issue 85 <https://github.com/zopefoundation/zope.interface/issues/85>`_.
+
+- Fix a regression that could lead to the utility registration cache
+  of ``Components`` getting out of sync. See `issue 93
+  <https://github.com/zopefoundation/zope.interface/issues/93>`_.
+
+4.4.1 (2017-05-13)
+==================
+
+- Simplify the caching of utility-registration data. In addition to
+  simplification, avoids spurious test failures when checking for
+  leaks in tests with persistent registries. See `pull 84
+  <https://github.com/zopefoundation/zope.interface/pull/84>`_.
+
+- Raise ``ValueError`` when non-text names are passed to adapter registry
+  methods:  prevents corruption of lookup caches.
+
+4.4.0 (2017-04-21)
+==================
+
+- Avoid a warning from the C compiler.
+  (https://github.com/zopefoundation/zope.interface/issues/71)
+
+- Add support for Python 3.6.
+
+4.3.3 (2016-12-13)
+==================
+
+- Correct typos and ReST formatting errors in documentation.
+
+- Add API documentation for the adapter registry.
+
+- Ensure that the ``LICENSE.txt`` file is included in built wheels.
+
+- Fix C optimizations broken on Py3k.  See the Python bug at:
+  http://bugs.python.org/issue15657
+  (https://github.com/zopefoundation/zope.interface/issues/60)
+
+
+4.3.2 (2016-09-05)
+==================
+
+- Fix equality testing of ``implementedBy`` objects and proxies.
+  (https://github.com/zopefoundation/zope.interface/issues/55)
+
+
+4.3.1 (2016-08-31)
+==================
+
+- Support Components subclasses that are not hashable.
+  (https://github.com/zopefoundation/zope.interface/issues/53)
+
+
+4.3.0 (2016-08-31)
+==================
+
+- Add the ability to sort the objects returned by ``implementedBy``.
+  This is compatible with the way interface classes sort so they can
+  be used together in ordered containers like BTrees.
+  (https://github.com/zopefoundation/zope.interface/issues/42)
+
+- Make ``setuptools`` a hard dependency of ``setup.py``.
+  (https://github.com/zopefoundation/zope.interface/issues/13)
+
+- Change a linear algorithm (O(n)) in ``Components.registerUtility`` and
+  ``Components.unregisterUtility`` into a dictionary lookup (O(1)) for
+  hashable components. This substantially improves the time taken to
+  manipulate utilities in large registries at the cost of some
+  additional memory usage. (https://github.com/zopefoundation/zope.interface/issues/46)
+
+
+4.2.0 (2016-06-10)
+==================
+
+- Add support for Python 3.5
+
+- Drop support for Python 2.6 and 3.2.
+
+
+4.1.3 (2015-10-05)
+==================
+
+- Fix installation without a C compiler on Python 3.5
+  (https://github.com/zopefoundation/zope.interface/issues/24).
+
+
+4.1.2 (2014-12-27)
+==================
+
+- Add support for PyPy3.
+
+- Remove unittest assertions deprecated in Python3.x.
+
+- Add ``zope.interface.document.asReStructuredText``, which formats the
+  generated text for an interface using ReST double-backtick markers.
+
+
+4.1.1 (2014-03-19)
+==================
+
+- Add support for Python 3.4.
+
+
+4.1.0 (2014-02-05)
+==================
+
+- Update ``boostrap.py`` to version 2.2.
+
+- Add ``@named(name)`` declaration, that specifies the component name, so it
+  does not have to be passed in during registration.
+
+
+4.0.5 (2013-02-28)
+==================
+
+- Fix a bug where a decorated method caused false positive failures on
+  ``verifyClass()``.
+
+
+4.0.4 (2013-02-21)
+==================
+
+- Fix a bug that was revealed by porting zope.traversing. During a loop, the
+  loop body modified a weakref dict causing a ``RuntimeError`` error.
+
+4.0.3 (2012-12-31)
+==================
+
+- Fleshed out PyPI Trove classifiers.
+
+4.0.2 (2012-11-21)
+==================
+
+- Add support for Python 3.3.
+
+- Restored ability to install the package in the absence of ``setuptools``.
+
+- LP #1055223:  Fix test which depended on dictionary order and failed randomly
+  in Python 3.3.
+
+4.0.1 (2012-05-22)
+==================
+
+- Drop explicit ``DeprecationWarnings`` for "class advice" APIS (these
+  APIs are still deprecated under Python 2.x, and still raise an exception
+  under Python 3.x, but no longer cause a warning to be emitted under
+  Python 2.x).
+
+4.0.0 (2012-05-16)
+==================
+
+- Automated build of Sphinx HTML docs and running doctest snippets via tox.
+
+- Deprecate the "class advice" APIs from ``zope.interface.declarations``:
+  ``implements``, ``implementsOnly``, and ``classProvides``.  In their place,
+  prefer the equivalent class decorators: ``@implementer``,
+  ``@implementer_only``, and ``@provider``.  Code which uses the deprecated
+  APIs will not work as expected under Py3k.
+
+- Remove use of '2to3' and associated fixers when installing under Py3k.
+  The code is now in a "compatible subset" which supports Python 2.6, 2.7,
+  and 3.2, including PyPy 1.8 (the version compatible with the 2.7 language
+  spec).
+
+- Drop explicit support for Python 2.4 / 2.5 / 3.1.
+
+- Add support for PyPy.
+
+- Add support for continuous integration using ``tox`` and ``jenkins``.
+
+- Add 'setup.py dev' alias (runs ``setup.py develop`` plus installs
+  ``nose`` and ``coverage``).
+
+- Add 'setup.py docs' alias (installs ``Sphinx`` and dependencies).
+
+- Replace all unittest coverage previously accomplished via doctests with
+  unittests.  The doctests have been moved into a ``docs`` section, managed
+  as a Sphinx collection.
+
+- LP #910987:  Ensure that the semantics of the ``lookup`` method of
+  ``zope.interface.adapter.LookupBase`` are the same in both the C and
+  Python implementations.
+
+- LP #900906:  Avoid exceptions due to tne new ``__qualname__`` attribute
+  added in Python 3.3 (see PEP 3155 for rationale).  Thanks to Antoine
+  Pitrou for the patch.
+
+3.8.0 (2011-09-22)
+==================
+
+- New module ``zope.interface.registry``.  This is code moved from
+  ``zope.component.registry`` which implements a basic nonperistent component
+  registry as ``zope.interface.registry.Components``.  This class was moved
+  from ``zope.component`` to make porting systems (such as Pyramid) that rely
+  only on a basic component registry to Python 3 possible without needing to
+  port the entirety of the ``zope.component`` package.  Backwards
+  compatibility import shims have been left behind in ``zope.component``, so
+  this change will not break any existing code.
+
+- New ``tests_require`` dependency: ``zope.event`` to test events sent by
+  Components implementation.  The ``zope.interface`` package does not have a
+  hard dependency on ``zope.event``, but if ``zope.event`` is importable, it
+  will send component registration events when methods of an instance of
+  ``zope.interface.registry.Components`` are called.
+
+- New interfaces added to support ``zope.interface.registry.Components``
+  addition: ``ComponentLookupError``, ``Invalid``, ``IObjectEvent``,
+  ``ObjectEvent``, ``IComponentLookup``, ``IRegistration``,
+  ``IUtilityRegistration``, ``IAdapterRegistration``,
+  ``ISubscriptionAdapterRegistration``, ``IHandlerRegistration``,
+  ``IRegistrationEvent``, ``RegistrationEvent``, ``IRegistered``,
+  ``Registered``, ``IUnregistered``, ``Unregistered``,
+  ``IComponentRegistry``, and ``IComponents``.
+
+- No longer Python 2.4 compatible (tested under 2.5, 2.6, 2.7, and 3.2).
+
+3.7.0 (2011-08-13)
+==================
+
+- Move changes from 3.6.2 - 3.6.5 to a new 3.7.x release line.
+
+3.6.7 (2011-08-20)
+==================
+
+- Fix sporadic failures on x86-64 platforms in tests of rich comparisons
+  of interfaces.
+
+3.6.6 (2011-08-13)
+==================
+
+- LP #570942:  Now correctly compare interfaces  from different modules but
+  with the same names.
+
+  N.B.: This is a less intrusive / destabilizing fix than the one applied in
+  3.6.3:  we only fix the underlying cmp-alike function, rather than adding
+  the other "rich comparison" functions.
+
+- Revert to software as released with 3.6.1 for "stable" 3.6 release branch.
+
+3.6.5 (2011-08-11)
+==================
+
+- LP #811792:  work around buggy behavior in some subclasses of
+  ``zope.interface.interface.InterfaceClass``, which invoke ``__hash__``
+  before initializing ``__module__`` and ``__name__``.  The workaround
+  returns a fixed constant hash in such cases, and issues a ``UserWarning``.
+
+- LP #804832:  Under PyPy, ``zope.interface`` should not build its C
+  extension.  Also, prevent attempting to build it under Jython.
+
+- Add a tox.ini for easier xplatform testing.
+
+- Fix testing deprecation warnings issued when tested under Py3K.
+
+3.6.4 (2011-07-04)
+==================
+
+- LP 804951:  InterfaceClass instances were unhashable under Python 3.x.
+
+3.6.3 (2011-05-26)
+==================
+
+- LP #570942:  Now correctly compare interfaces  from different modules but
+  with the same names.
+
+3.6.2 (2011-05-17)
+==================
+
+- Moved detailed documentation out-of-line from PyPI page, linking instead to
+  http://docs.zope.org/zope.interface .
+
+- Fixes for small issues when running tests under Python 3.2 using
+  ``zope.testrunner``.
+
+- LP # 675064:  Specify return value type for C optimizations module init
+  under Python 3:  undeclared value caused warnings, and segfaults on some
+  64 bit architectures.
+
+- setup.py now raises RuntimeError if you don't have Distutils installed when
+  running under Python 3.
+
+3.6.1 (2010-05-03)
+==================
+
+- A non-ASCII character in the changelog made 3.6.0 uninstallable on
+  Python 3 systems with another default encoding than UTF-8.
+
+- Fix compiler warnings under GCC 4.3.3.
+
+3.6.0 (2010-04-29)
+==================
+
+- LP #185974:  Clear the cache used by ``Specificaton.get`` inside
+  ``Specification.changed``.  Thanks to Jacob Holm for the patch.
+
+- Add support for Python 3.1. Contributors:
+
+    Lennart Regebro
+    Martin v Loewis
+    Thomas Lotze
+    Wolfgang Schnerring
+
+  The 3.1 support is completely backwards compatible. However, the implements
+  syntax used under Python 2.X does not work under 3.X, since it depends on
+  how metaclasses are implemented and this has changed. Instead it now supports
+  a decorator syntax (also under Python 2.X)::
+
+    class Foo:
+        implements(IFoo)
+        ...
+
+  can now also be written::
+
+    @implementer(IFoo):
+    class Foo:
+        ...
+
+  There are 2to3 fixers available to do this change automatically in the
+  zope.fixers package.
+
+- Python 2.3 is no longer supported.
+
+
+3.5.4 (2009-12-23)
+==================
+
+- Use the standard Python doctest module instead of zope.testing.doctest, which
+  has been deprecated.
+
+
+3.5.3 (2009-12-08)
+==================
+
+- Fix an edge case: make providedBy() work when a class has '__provides__' in
+  its __slots__ (see http://thread.gmane.org/gmane.comp.web.zope.devel/22490)
+
+
+3.5.2 (2009-07-01)
+==================
+
+- BaseAdapterRegistry.unregister, unsubscribe: Remove empty portions of
+  the data structures when something is removed.  This avoids leaving
+  references to global objects (interfaces) that may be slated for
+  removal from the calling application.
+
+
+3.5.1 (2009-03-18)
+==================
+
+- verifyObject: use getattr instead of hasattr to test for object attributes
+  in order to let exceptions other than AttributeError raised by properties
+  propagate to the caller
+
+- Add Sphinx-based documentation building to the package buildout
+  configuration. Use the ``bin/docs`` command after buildout.
+
+- Improve package description a bit. Unify changelog entries formatting.
+
+- Change package's mailing list address to zope-dev at zope.org as
+  zope3-dev at zope.org is now retired.
+
+
+3.5.0 (2008-10-26)
+==================
+
+- Fix declaration of _zope_interface_coptimizations, it's not a top level
+  package.
+
+- Add a DocTestSuite for odd.py module, so their tests are run.
+
+- Allow to bootstrap on Jython.
+
+- Fix https://bugs.launchpad.net/zope3/3.3/+bug/98388: ISpecification
+  was missing a declaration for __iro__.
+
+- Add optional code optimizations support, which allows the building
+  of C code optimizations to fail (Jython).
+
+- Replace `_flatten` with a non-recursive implementation, effectively making
+  it 3x faster.
+
+
+3.4.1 (2007-10-02)
+==================
+
+- Fix a setup bug that prevented installation from source on systems
+  without setuptools.
+
+
+3.4.0 (2007-07-19)
+==================
+
+- Final release for 3.4.0.
+
+
+3.4.0b3 (2007-05-22)
+====================
+
+
+- When checking whether an object is already registered, use identity
+  comparison, to allow adding registering with picky custom comparison methods.
+
+
+3.3.0.1 (2007-01-03)
+====================
+
+- Made a reference to OverflowWarning, which disappeared in Python
+  2.5, conditional.
+
+
+3.3.0 (2007/01/03)
+==================
+
+New Features
+------------
+
+- Refactor the adapter-lookup algorithim to make it much simpler and faster.
+
+  Also, implement more of the adapter-lookup logic in C, making
+  debugging of application code easier, since there is less
+  infrastructre code to step through.
+
+- Treat objects without interface declarations as if they
+  declared that they provide ``zope.interface.Interface``.
+
+- Add a number of richer new adapter-registration interfaces
+  that provide greater control and introspection.
+
+- Add a new interface decorator to zope.interface that allows the
+  setting of tagged values on an interface at definition time (see
+  zope.interface.taggedValue).
+
+Bug Fixes
+---------
+
+- A bug in multi-adapter lookup sometimes caused incorrect adapters to
+  be returned.
+
+
+3.2.0.2 (2006-04-15)
+====================
+
+- Fix packaging bug:  'package_dir' must be a *relative* path.
+
+
+3.2.0.1 (2006-04-14)
+====================
+
+- Packaging change:  suppress inclusion of 'setup.cfg' in 'sdist' builds.
+
+
+3.2.0 (2006-01-05)
+==================
+
+- Corresponds to the version of the zope.interface package shipped as part of
+  the Zope 3.2.0 release.
+
+
+3.1.0 (2005-10-03)
+==================
+
+- Corresponds to the version of the zope.interface package shipped as part of
+  the Zope 3.1.0 release.
+
+- Made attribute resolution order consistent with component lookup order,
+  i.e. new-style class MRO semantics.
+
+- Deprecate 'isImplementedBy' and 'isImplementedByInstancesOf' APIs in
+  favor of 'implementedBy' and 'providedBy'.
+
+
+3.0.1 (2005-07-27)
+==================
+
+- Corresponds to the version of the zope.interface package shipped as part of
+  the Zope X3.0.1 release.
+
+- Fix a bug reported by James Knight, which caused adapter registries
+  to fail occasionally to reflect declaration changes.
+
+
+3.0.0 (2004-11-07)
+==================
+
+- Corresponds to the version of the zope.interface package shipped as part of
+  the Zope X3.0.0 release.
diff --git a/contrib/python/zope.interface/py3/.dist-info/top_level.txt b/contrib/python/zope.interface/py3/.dist-info/top_level.txt
new file mode 100644
index 00000000000..66179d49851
--- /dev/null
+++ b/contrib/python/zope.interface/py3/.dist-info/top_level.txt
@@ -0,0 +1 @@
+zope
diff --git a/contrib/python/zope.interface/py3/COPYRIGHT.txt b/contrib/python/zope.interface/py3/COPYRIGHT.txt
new file mode 100644
index 00000000000..79859e06010
--- /dev/null
+++ b/contrib/python/zope.interface/py3/COPYRIGHT.txt
@@ -0,0 +1 @@
+Zope Foundation and Contributors
\ No newline at end of file
diff --git a/contrib/python/zope.interface/py3/LICENSE.txt b/contrib/python/zope.interface/py3/LICENSE.txt
new file mode 100644
index 00000000000..e1f9ad7b3b4
--- /dev/null
+++ b/contrib/python/zope.interface/py3/LICENSE.txt
@@ -0,0 +1,44 @@
+Zope Public License (ZPL) Version 2.1
+
+A copyright notice accompanies this license document that identifies the
+copyright holders.
+
+This license has been certified as open source. It has also been designated as
+GPL compatible by the Free Software Foundation (FSF).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions in source code must retain the accompanying copyright
+notice, this list of conditions, and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the accompanying copyright
+notice, this list of conditions, and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Names of the copyright holders must not be used to endorse or promote
+products derived from this software without prior written permission from the
+copyright holders.
+
+4. The right to distribute this software or to use it for any purpose does not
+give you the right to use Servicemarks (sm) or Trademarks (tm) of the
+copyright
+holders. Use of them is covered by separate agreement with the copyright
+holders.
+
+5. If any files are modified, you must cause the modified files to carry
+prominent notices stating that you changed the files and the date of any
+change.
+
+Disclaimer
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/python/zope.interface/py3/README.rst b/contrib/python/zope.interface/py3/README.rst
new file mode 100644
index 00000000000..e8a18e905dc
--- /dev/null
+++ b/contrib/python/zope.interface/py3/README.rst
@@ -0,0 +1,31 @@
+====================
+ ``zope.interface``
+====================
+
+.. image:: https://img.shields.io/pypi/v/zope.interface.svg
+    :target: https://pypi.python.org/pypi/zope.interface/
+    :alt: Latest Version
+
+.. image:: https://img.shields.io/pypi/pyversions/zope.interface.svg
+        :target: https://pypi.org/project/zope.interface/
+        :alt: Supported Python versions
+
+.. image:: https://github.com/zopefoundation/zope.interface/actions/workflows/tests.yml/badge.svg
+        :target: https://github.com/zopefoundation/zope.interface/actions/workflows/tests.yml
+
+.. image:: https://readthedocs.org/projects/zopeinterface/badge/?version=latest
+        :target: https://zopeinterface.readthedocs.io/en/latest/
+        :alt: Documentation Status
+
+This package is intended to be independently reusable in any Python
+project. It is maintained by the `Zope Toolkit project
+<https://zopetoolkit.readthedocs.io/>`_.
+
+This package provides an implementation of "object interfaces" for Python.
+Interfaces are a mechanism for labeling objects as conforming to a given
+API or contract. So, this package can be considered as implementation of
+the `Design By Contract`_ methodology support in Python.
+
+.. _Design By Contract: http://en.wikipedia.org/wiki/Design_by_contract
+
+For detailed documentation, please see https://zopeinterface.readthedocs.io/en/latest/
diff --git a/contrib/python/zope.interface/py3/ya.make b/contrib/python/zope.interface/py3/ya.make
new file mode 100644
index 00000000000..f11d0d940c7
--- /dev/null
+++ b/contrib/python/zope.interface/py3/ya.make
@@ -0,0 +1,61 @@
+# Generated by devtools/yamaker (pypi).
+
+PY3_LIBRARY()
+
+VERSION(6.1)
+
+LICENSE(ZPL-2.1)
+
+PEERDIR(
+    contrib/python/setuptools
+)
+
+NO_COMPILER_WARNINGS()
+
+NO_LINT()
+
+SRCS(
+    zope/interface/_zope_interface_coptimizations.c
+)
+
+PY_REGISTER(
+    zope.interface._zope_interface_coptimizations
+)
+
+PY_SRCS(
+    TOP_LEVEL
+    zope/interface/__init__.py
+    zope/interface/_compat.py
+    zope/interface/_flatten.py
+    zope/interface/adapter.py
+    zope/interface/advice.py
+    zope/interface/common/__init__.py
+    zope/interface/common/builtins.py
+    zope/interface/common/collections.py
+    zope/interface/common/idatetime.py
+    zope/interface/common/interfaces.py
+    zope/interface/common/io.py
+    zope/interface/common/mapping.py
+    zope/interface/common/numbers.py
+    zope/interface/common/sequence.py
+    zope/interface/declarations.py
+    zope/interface/document.py
+    zope/interface/exceptions.py
+    zope/interface/interface.py
+    zope/interface/interfaces.py
+    zope/interface/registry.py
+    zope/interface/ro.py
+    zope/interface/verify.py
+)
+
+RESOURCE_FILES(
+    PREFIX contrib/python/zope.interface/py3/
+    .dist-info/METADATA
+    .dist-info/top_level.txt
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    tests
+)
diff --git a/contrib/python/zope.interface/py3/zope/interface/__init__.py b/contrib/python/zope.interface/py3/zope/interface/__init__.py
new file mode 100644
index 00000000000..17a272f1dae
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/__init__.py
@@ -0,0 +1,93 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interfaces
+
+This package implements the Python "scarecrow" proposal.
+
+The package exports two objects, `Interface` and `Attribute` directly. It also
+exports several helper methods. Interface is used to create an interface with
+a class statement, as in:
+
+  class IMyInterface(Interface):
+    '''Interface documentation
+    '''
+
+    def meth(arg1, arg2):
+        '''Documentation for meth
+        '''
+
+    # Note that there is no self argument
+
+To find out what you can do with interfaces, see the interface
+interface, `IInterface` in the `interfaces` module.
+
+The package has several public modules:
+
+  o `declarations` provides utilities to declare interfaces on objects. It
+    also provides a wide range of helpful utilities that aid in managing
+    declared interfaces. Most of its public names are however imported here.
+
+  o `document` has a utility for documenting an interface as structured text.
+
+  o `exceptions` has the interface-defined exceptions
+
+  o `interfaces` contains a list of all public interfaces for this package.
+
+  o `verify` has utilities for verifying implementations of interfaces.
+
+See the module doc strings for more information.
+"""
+__docformat__ = 'restructuredtext'
+# pylint:disable=wrong-import-position,unused-import
+from zope.interface.interface import Interface
+from zope.interface.interface import _wire
+
+# Need to actually get the interface elements to implement the right interfaces
+_wire()
+del _wire
+
+from zope.interface.declarations import Declaration
+from zope.interface.declarations import alsoProvides
+from zope.interface.declarations import classImplements
+from zope.interface.declarations import classImplementsFirst
+from zope.interface.declarations import classImplementsOnly
+from zope.interface.declarations import directlyProvidedBy
+from zope.interface.declarations import directlyProvides
+from zope.interface.declarations import implementedBy
+from zope.interface.declarations import implementer
+from zope.interface.declarations import implementer_only
+from zope.interface.declarations import moduleProvides
+from zope.interface.declarations import named
+from zope.interface.declarations import noLongerProvides
+from zope.interface.declarations import providedBy
+from zope.interface.declarations import provider
+
+from zope.interface.exceptions import Invalid
+
+from zope.interface.interface import Attribute
+from zope.interface.interface import interfacemethod
+from zope.interface.interface import invariant
+from zope.interface.interface import taggedValue
+
+# The following are to make spec pickles cleaner
+from zope.interface.declarations import Provides
+
+
+from zope.interface.interfaces import IInterfaceDeclaration
+
+moduleProvides(IInterfaceDeclaration)
+
+__all__ = ('Interface', 'Attribute') + tuple(IInterfaceDeclaration)
+
+assert all(k in globals() for k in __all__)
diff --git a/contrib/python/zope.interface/py3/zope/interface/_compat.py b/contrib/python/zope.interface/py3/zope/interface/_compat.py
new file mode 100644
index 00000000000..2ff8d83eafe
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/_compat.py
@@ -0,0 +1,135 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Support functions for dealing with differences in platforms, including Python
+versions and implementations.
+
+This file should have no imports from the rest of zope.interface because it is
+used during early bootstrapping.
+"""
+import os
+import sys
+
+
+def _normalize_name(name):
+    if isinstance(name, bytes):
+        name = str(name, 'ascii')
+    if isinstance(name, str):
+        return name
+    raise TypeError("name must be a string or ASCII-only bytes")
+
+PYPY = hasattr(sys, 'pypy_version_info')
+
+
+def _c_optimizations_required():
+    """
+    Return a true value if the C optimizations are required.
+
+    This uses the ``PURE_PYTHON`` variable as documented in `_use_c_impl`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    require_c = pure_env == "0"
+    return require_c
+
+
+def _c_optimizations_available():
+    """
+    Return the C optimization module, if available, otherwise
+    a false value.
+
+    If the optimizations are required but not available, this
+    raises the ImportError.
+
+    This does not say whether they should be used or not.
+    """
+    catch = () if _c_optimizations_required() else (ImportError,)
+    try:
+        from zope.interface import _zope_interface_coptimizations as c_opt
+        return c_opt
+    except catch: # pragma: no cover (only Jython doesn't build extensions)
+        return False
+
+
+def _c_optimizations_ignored():
+    """
+    The opposite of `_c_optimizations_required`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    return pure_env is not None and pure_env != "0"
+
+
+def _should_attempt_c_optimizations():
+    """
+    Return a true value if we should attempt to use the C optimizations.
+
+    This takes into account whether we're on PyPy and the value of the
+    ``PURE_PYTHON`` environment variable, as defined in `_use_c_impl`.
+    """
+    is_pypy = hasattr(sys, 'pypy_version_info')
+
+    if _c_optimizations_required():
+        return True
+    if is_pypy:
+        return False
+    return not _c_optimizations_ignored()
+
+
+def _use_c_impl(py_impl, name=None, globs=None):
+    """
+    Decorator. Given an object implemented in Python, with a name like
+    ``Foo``, import the corresponding C implementation from
+    ``zope.interface._zope_interface_coptimizations`` with the name
+    ``Foo`` and use it instead.
+
+    If the ``PURE_PYTHON`` environment variable is set to any value
+    other than ``"0"``, or we're on PyPy, ignore the C implementation
+    and return the Python version. If the C implementation cannot be
+    imported, return the Python version. If ``PURE_PYTHON`` is set to
+    0, *require* the C implementation (let the ImportError propagate);
+    note that PyPy can import the C implementation in this case (and all
+    tests pass).
+
+    In all cases, the Python version is kept available. in the module
+    globals with the name ``FooPy`` and the name ``FooFallback`` (both
+    conventions have been used; the C implementation of some functions
+    looks for the ``Fallback`` version, as do some of the Sphinx
+    documents).
+
+    Example::
+
+        @_use_c_impl
+        class Foo(object):
+            ...
+    """
+    name = name or py_impl.__name__
+    globs = globs or sys._getframe(1).f_globals
+
+    def find_impl():
+        if not _should_attempt_c_optimizations():
+            return py_impl
+
+        c_opt = _c_optimizations_available()
+        if not c_opt: # pragma: no cover (only Jython doesn't build extensions)
+            return py_impl
+
+        __traceback_info__ = c_opt
+        return getattr(c_opt, name)
+
+    c_impl = find_impl()
+    # Always make available by the FooPy name and FooFallback
+    # name (for testing and documentation)
+    globs[name + 'Py'] = py_impl
+    globs[name + 'Fallback'] = py_impl
+
+    return c_impl
diff --git a/contrib/python/zope.interface/py3/zope/interface/_flatten.py b/contrib/python/zope.interface/py3/zope/interface/_flatten.py
new file mode 100644
index 00000000000..a80c2de49ab
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/_flatten.py
@@ -0,0 +1,35 @@
+##############################################################################
+#
+# Copyright (c) 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Adapter-style interface registry
+
+See Adapter class.
+"""
+from zope.interface import Declaration
+
+def _flatten(implements, include_None=0):
+
+    try:
+        r = implements.flattened()
+    except AttributeError:
+        if implements is None:
+            r=()
+        else:
+            r = Declaration(implements).flattened()
+
+    if not include_None:
+        return r
+
+    r = list(r)
+    r.append(None)
+    return r
diff --git a/contrib/python/zope.interface/py3/zope/interface/_zope_interface_coptimizations.c b/contrib/python/zope.interface/py3/zope/interface/_zope_interface_coptimizations.c
new file mode 100644
index 00000000000..91899283c0b
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/_zope_interface_coptimizations.c
@@ -0,0 +1,2101 @@
+/*###########################################################################
+ #
+ # Copyright (c) 2003 Zope Foundation and Contributors.
+ # All Rights Reserved.
+ #
+ # This software is subject to the provisions of the Zope Public License,
+ # Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+ # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+ # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+ # FOR A PARTICULAR PURPOSE.
+ #
+ ############################################################################*/
+
+#include "Python.h"
+#include "structmember.h"
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-parameter"
+#pragma clang diagnostic ignored "-Wmissing-field-initializers"
+#endif
+
+#define TYPE(O) ((PyTypeObject*)(O))
+#define OBJECT(O) ((PyObject*)(O))
+#define CLASSIC(O) ((PyClassObject*)(O))
+#ifndef PyVarObject_HEAD_INIT
+#define PyVarObject_HEAD_INIT(a, b) PyObject_HEAD_INIT(a) b,
+#endif
+#ifndef Py_TYPE
+#define Py_TYPE(o) ((o)->ob_type)
+#endif
+
+#define PyNative_FromString PyUnicode_FromString
+
+static PyObject *str__dict__, *str__implemented__, *strextends;
+static PyObject *BuiltinImplementationSpecifications, *str__provides__;
+static PyObject *str__class__, *str__providedBy__;
+static PyObject *empty, *fallback;
+static PyObject *str__conform__, *str_call_conform, *adapter_hooks;
+static PyObject *str_uncached_lookup, *str_uncached_lookupAll;
+static PyObject *str_uncached_subscriptions;
+static PyObject *str_registry, *strro, *str_generation, *strchanged;
+static PyObject *str__self__;
+static PyObject *str__module__;
+static PyObject *str__name__;
+static PyObject *str__adapt__;
+static PyObject *str_CALL_CUSTOM_ADAPT;
+
+static PyTypeObject *Implements;
+
+static int imported_declarations = 0;
+
+static int
+import_declarations(void)
+{
+  PyObject *declarations, *i;
+
+  declarations = PyImport_ImportModule("zope.interface.declarations");
+  if (declarations == NULL)
+    return -1;
+
+  BuiltinImplementationSpecifications = PyObject_GetAttrString(
+                    declarations, "BuiltinImplementationSpecifications");
+  if (BuiltinImplementationSpecifications == NULL)
+    return -1;
+
+  empty = PyObject_GetAttrString(declarations, "_empty");
+  if (empty == NULL)
+    return -1;
+
+  fallback = PyObject_GetAttrString(declarations, "implementedByFallback");
+  if (fallback == NULL)
+    return -1;
+
+
+
+  i = PyObject_GetAttrString(declarations, "Implements");
+  if (i == NULL)
+    return -1;
+
+  if (! PyType_Check(i))
+    {
+      PyErr_SetString(PyExc_TypeError,
+                      "zope.interface.declarations.Implements is not a type");
+      return -1;
+    }
+
+  Implements = (PyTypeObject *)i;
+
+  Py_DECREF(declarations);
+
+  imported_declarations = 1;
+  return 0;
+}
+
+
+static PyTypeObject SpecificationBaseType;   /* Forward */
+
+static PyObject *
+implementedByFallback(PyObject *cls)
+{
+  if (imported_declarations == 0 && import_declarations() < 0)
+    return NULL;
+
+  return PyObject_CallFunctionObjArgs(fallback, cls, NULL);
+}
+
+static PyObject *
+implementedBy(PyObject *ignored, PyObject *cls)
+{
+  /* Fast retrieval of implements spec, if possible, to optimize
+     common case.  Use fallback code if we get stuck.
+  */
+
+  PyObject *dict = NULL, *spec;
+
+  if (PyObject_TypeCheck(cls, &PySuper_Type))
+  {
+      // Let merging be handled by Python.
+      return implementedByFallback(cls);
+  }
+
+  if (PyType_Check(cls))
+    {
+      dict = TYPE(cls)->tp_dict;
+      Py_XINCREF(dict);
+    }
+
+  if (dict == NULL)
+    dict = PyObject_GetAttr(cls, str__dict__);
+
+  if (dict == NULL)
+    {
+      /* Probably a security proxied class, use more expensive fallback code */
+      PyErr_Clear();
+      return implementedByFallback(cls);
+    }
+
+  spec = PyObject_GetItem(dict, str__implemented__);
+  Py_DECREF(dict);
+  if (spec)
+    {
+      if (imported_declarations == 0 && import_declarations() < 0)
+        return NULL;
+
+      if (PyObject_TypeCheck(spec, Implements))
+        return spec;
+
+      /* Old-style declaration, use more expensive fallback code */
+      Py_DECREF(spec);
+      return implementedByFallback(cls);
+    }
+
+  PyErr_Clear();
+
+  /* Maybe we have a builtin */
+  if (imported_declarations == 0 && import_declarations() < 0)
+    return NULL;
+
+  spec = PyDict_GetItem(BuiltinImplementationSpecifications, cls);
+  if (spec != NULL)
+    {
+      Py_INCREF(spec);
+      return spec;
+    }
+
+  /* We're stuck, use fallback */
+  return implementedByFallback(cls);
+}
+
+static PyObject *
+getObjectSpecification(PyObject *ignored, PyObject *ob)
+{
+  PyObject *cls, *result;
+
+  result = PyObject_GetAttr(ob, str__provides__);
+  if (!result)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          /* Propagate non AttributeError exceptions. */
+          return NULL;
+      }
+      PyErr_Clear();
+  }
+  else
+  {
+      int is_instance = -1;
+      is_instance = PyObject_IsInstance(result, (PyObject*)&SpecificationBaseType);
+      if (is_instance < 0)
+      {
+          /* Propagate all errors */
+          return NULL;
+      }
+      if (is_instance)
+      {
+          return result;
+      }
+  }
+
+  /* We do a getattr here so as not to be defeated by proxies */
+  cls = PyObject_GetAttr(ob, str__class__);
+  if (cls == NULL)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          /* Propagate non-AttributeErrors */
+          return NULL;
+      }
+      PyErr_Clear();
+      if (imported_declarations == 0 && import_declarations() < 0)
+          return NULL;
+
+      Py_INCREF(empty);
+      return empty;
+  }
+  result = implementedBy(NULL, cls);
+  Py_DECREF(cls);
+
+  return result;
+}
+
+static PyObject *
+providedBy(PyObject *ignored, PyObject *ob)
+{
+  PyObject *result, *cls, *cp;
+  int is_instance = -1;
+  result = NULL;
+
+  is_instance = PyObject_IsInstance(ob, (PyObject*)&PySuper_Type);
+  if (is_instance < 0)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          /* Propagate non-AttributeErrors */
+          return NULL;
+      }
+      PyErr_Clear();
+  }
+  if (is_instance)
+  {
+      return implementedBy(NULL, ob);
+  }
+
+  result = PyObject_GetAttr(ob, str__providedBy__);
+
+  if (result == NULL)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          return NULL;
+      }
+
+      PyErr_Clear();
+      return getObjectSpecification(NULL, ob);
+  }
+
+
+  /* We want to make sure we have a spec. We can't do a type check
+     because we may have a proxy, so we'll just try to get the
+     only attribute.
+  */
+  if (PyObject_TypeCheck(result, &SpecificationBaseType)
+      ||
+      PyObject_HasAttr(result, strextends)
+      )
+    return result;
+
+  /*
+    The object's class doesn't understand descriptors.
+    Sigh. We need to get an object descriptor, but we have to be
+    careful.  We want to use the instance's __provides__,l if
+    there is one, but only if it didn't come from the class.
+  */
+  Py_DECREF(result);
+
+  cls = PyObject_GetAttr(ob, str__class__);
+  if (cls == NULL)
+    return NULL;
+
+  result = PyObject_GetAttr(ob, str__provides__);
+  if (result == NULL)
+    {
+      /* No __provides__, so just fall back to implementedBy */
+      PyErr_Clear();
+      result = implementedBy(NULL, cls);
+      Py_DECREF(cls);
+      return result;
+    }
+
+  cp = PyObject_GetAttr(cls, str__provides__);
+  if (cp == NULL)
+    {
+      /* The the class has no provides, assume we're done: */
+      PyErr_Clear();
+      Py_DECREF(cls);
+      return result;
+    }
+
+  if (cp == result)
+    {
+      /*
+        Oops, we got the provides from the class. This means
+        the object doesn't have it's own. We should use implementedBy
+      */
+      Py_DECREF(result);
+      result = implementedBy(NULL, cls);
+    }
+
+  Py_DECREF(cls);
+  Py_DECREF(cp);
+
+  return result;
+}
+
+typedef struct {
+    PyObject_HEAD
+    PyObject* weakreflist;
+    /*
+      In the past, these fields were stored in the __dict__
+      and were technically allowed to contain any Python object, though
+      other type checks would fail or fall back to generic code paths if
+      they didn't have the expected type. We preserve that behaviour and don't
+      make any assumptions about contents.
+    */
+    PyObject* _implied;
+    /*
+      The remainder aren't used in C code but must be stored here
+      to prevent instance layout conflicts.
+    */
+    PyObject* _dependents;
+    PyObject* _bases;
+    PyObject* _v_attrs;
+    PyObject* __iro__;
+    PyObject* __sro__;
+} Spec;
+
+/*
+  We know what the fields are *supposed* to define, but
+  they could have anything, so we need to traverse them.
+*/
+static int
+Spec_traverse(Spec* self, visitproc visit, void* arg)
+{
+    Py_VISIT(self->_implied);
+    Py_VISIT(self->_dependents);
+    Py_VISIT(self->_bases);
+    Py_VISIT(self->_v_attrs);
+    Py_VISIT(self->__iro__);
+    Py_VISIT(self->__sro__);
+    return 0;
+}
+
+static int
+Spec_clear(Spec* self)
+{
+    Py_CLEAR(self->_implied);
+    Py_CLEAR(self->_dependents);
+    Py_CLEAR(self->_bases);
+    Py_CLEAR(self->_v_attrs);
+    Py_CLEAR(self->__iro__);
+    Py_CLEAR(self->__sro__);
+    return 0;
+}
+
+static void
+Spec_dealloc(Spec* self)
+{
+    /* PyType_GenericAlloc that you get when you don't
+       specify a tp_alloc always tracks the object. */
+    PyObject_GC_UnTrack((PyObject *)self);
+    if (self->weakreflist != NULL) {
+        PyObject_ClearWeakRefs(OBJECT(self));
+    }
+    Spec_clear(self);
+    Py_TYPE(self)->tp_free(OBJECT(self));
+}
+
+static PyObject *
+Spec_extends(Spec *self, PyObject *other)
+{
+  PyObject *implied;
+
+  implied = self->_implied;
+  if (implied == NULL) {
+    return NULL;
+  }
+
+  if (PyDict_GetItem(implied, other) != NULL)
+    Py_RETURN_TRUE;
+  Py_RETURN_FALSE;
+}
+
+static char Spec_extends__doc__[] =
+"Test whether a specification is or extends another"
+;
+
+static char Spec_providedBy__doc__[] =
+"Test whether an interface is implemented by the specification"
+;
+
+static PyObject *
+Spec_call(Spec *self, PyObject *args, PyObject *kw)
+{
+  PyObject *spec;
+
+  if (! PyArg_ParseTuple(args, "O", &spec))
+    return NULL;
+  return Spec_extends(self, spec);
+}
+
+static PyObject *
+Spec_providedBy(PyObject *self, PyObject *ob)
+{
+  PyObject *decl, *item;
+
+  decl = providedBy(NULL, ob);
+  if (decl == NULL)
+    return NULL;
+
+  if (PyObject_TypeCheck(decl, &SpecificationBaseType))
+    item = Spec_extends((Spec*)decl, self);
+  else
+    /* decl is probably a security proxy.  We have to go the long way
+       around.
+    */
+    item = PyObject_CallFunctionObjArgs(decl, self, NULL);
+
+  Py_DECREF(decl);
+  return item;
+}
+
+
+static char Spec_implementedBy__doc__[] =
+"Test whether the specification is implemented by a class or factory.\n"
+"Raise TypeError if argument is neither a class nor a callable."
+;
+
+static PyObject *
+Spec_implementedBy(PyObject *self, PyObject *cls)
+{
+  PyObject *decl, *item;
+
+  decl = implementedBy(NULL, cls);
+  if (decl == NULL)
+    return NULL;
+
+  if (PyObject_TypeCheck(decl, &SpecificationBaseType))
+    item = Spec_extends((Spec*)decl, self);
+  else
+    item = PyObject_CallFunctionObjArgs(decl, self, NULL);
+
+  Py_DECREF(decl);
+  return item;
+}
+
+static struct PyMethodDef Spec_methods[] = {
+        {"providedBy",
+         (PyCFunction)Spec_providedBy,          METH_O,
+         Spec_providedBy__doc__},
+        {"implementedBy",
+         (PyCFunction)Spec_implementedBy,       METH_O,
+         Spec_implementedBy__doc__},
+        {"isOrExtends", (PyCFunction)Spec_extends,      METH_O,
+         Spec_extends__doc__},
+
+        {NULL,          NULL}           /* sentinel */
+};
+
+static PyMemberDef Spec_members[] = {
+  {"_implied", T_OBJECT_EX, offsetof(Spec, _implied), 0, ""},
+  {"_dependents", T_OBJECT_EX, offsetof(Spec, _dependents), 0, ""},
+  {"_bases", T_OBJECT_EX, offsetof(Spec, _bases), 0, ""},
+  {"_v_attrs", T_OBJECT_EX, offsetof(Spec, _v_attrs), 0, ""},
+  {"__iro__", T_OBJECT_EX, offsetof(Spec, __iro__), 0, ""},
+  {"__sro__", T_OBJECT_EX, offsetof(Spec, __sro__), 0, ""},
+  {NULL},
+};
+
+
+static PyTypeObject SpecificationBaseType = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_interface_coptimizations."
+                                "SpecificationBase",
+        /* tp_basicsize      */ sizeof(Spec),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)Spec_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)Spec_call,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+        "Base type for Specification objects",
+        /* tp_traverse       */ (traverseproc)Spec_traverse,
+        /* tp_clear          */ (inquiry)Spec_clear,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ offsetof(Spec, weakreflist),
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ Spec_methods,
+        /* tp_members        */ Spec_members,
+};
+
+static PyObject *
+OSD_descr_get(PyObject *self, PyObject *inst, PyObject *cls)
+{
+  PyObject *provides;
+
+  if (inst == NULL)
+    return getObjectSpecification(NULL, cls);
+
+  provides = PyObject_GetAttr(inst, str__provides__);
+  /* Return __provides__ if we got it, or return NULL and propagate non-AttributeError. */
+  if (provides != NULL || !PyErr_ExceptionMatches(PyExc_AttributeError))
+    return provides;
+
+  PyErr_Clear();
+  return implementedBy(NULL, cls);
+}
+
+static PyTypeObject OSDType = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_interface_coptimizations."
+                                "ObjectSpecificationDescriptor",
+        /* tp_basicsize      */ 0,
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)0,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)0,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT
+                                | Py_TPFLAGS_BASETYPE ,
+        "Object Specification Descriptor",
+        /* tp_traverse       */ (traverseproc)0,
+        /* tp_clear          */ (inquiry)0,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ 0,
+        /* tp_members        */ 0,
+        /* tp_getset         */ 0,
+        /* tp_base           */ 0,
+        /* tp_dict           */ 0, /* internal use */
+        /* tp_descr_get      */ (descrgetfunc)OSD_descr_get,
+};
+
+typedef struct {
+    Spec spec;
+    /* These members are handled generically, as for Spec members. */
+    PyObject* _cls;
+    PyObject* _implements;
+} CPB;
+
+static PyObject *
+CPB_descr_get(CPB *self, PyObject *inst, PyObject *cls)
+{
+  PyObject *implements;
+
+  if (self->_cls == NULL)
+    return NULL;
+
+  if (cls == self->_cls)
+    {
+      if (inst == NULL)
+        {
+          Py_INCREF(self);
+          return OBJECT(self);
+        }
+
+      implements = self->_implements;
+      Py_XINCREF(implements);
+      return implements;
+    }
+
+  PyErr_SetObject(PyExc_AttributeError, str__provides__);
+  return NULL;
+}
+
+static int
+CPB_traverse(CPB* self, visitproc visit, void* arg)
+{
+    Py_VISIT(self->_cls);
+    Py_VISIT(self->_implements);
+    return Spec_traverse((Spec*)self, visit, arg);
+}
+
+static int
+CPB_clear(CPB* self)
+{
+    Py_CLEAR(self->_cls);
+    Py_CLEAR(self->_implements);
+    Spec_clear((Spec*)self);
+    return 0;
+}
+
+static void
+CPB_dealloc(CPB* self)
+{
+    PyObject_GC_UnTrack((PyObject *)self);
+    CPB_clear(self);
+    Spec_dealloc((Spec*)self);
+}
+
+static PyMemberDef CPB_members[] = {
+  {"_cls", T_OBJECT_EX, offsetof(CPB, _cls), 0, "Defining class."},
+  {"_implements", T_OBJECT_EX, offsetof(CPB, _implements), 0, "Result of implementedBy."},
+  {NULL}
+};
+
+static PyTypeObject CPBType = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_interface_coptimizations."
+                                "ClassProvidesBase",
+        /* tp_basicsize      */ sizeof(CPB),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)CPB_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)0,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+        "C Base class for ClassProvides",
+        /* tp_traverse       */ (traverseproc)CPB_traverse,
+        /* tp_clear          */ (inquiry)CPB_clear,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ 0,
+        /* tp_members        */ CPB_members,
+        /* tp_getset         */ 0,
+        /* tp_base           */ &SpecificationBaseType,
+        /* tp_dict           */ 0, /* internal use */
+        /* tp_descr_get      */ (descrgetfunc)CPB_descr_get,
+        /* tp_descr_set      */ 0,
+        /* tp_dictoffset     */ 0,
+        /* tp_init           */ 0,
+        /* tp_alloc          */ 0,
+        /* tp_new            */ 0,
+};
+
+/* ==================================================================== */
+/* ========== Begin: __call__ and __adapt__ =========================== */
+
+/*
+    def __adapt__(self, obj):
+        """Adapt an object to the receiver
+        """
+        if self.providedBy(obj):
+            return obj
+
+        for hook in adapter_hooks:
+            adapter = hook(self, obj)
+            if adapter is not None:
+                return adapter
+
+
+*/
+static PyObject *
+__adapt__(PyObject *self, PyObject *obj)
+{
+  PyObject *decl, *args, *adapter;
+  int implements, i, l;
+
+  decl = providedBy(NULL, obj);
+  if (decl == NULL)
+    return NULL;
+
+  if (PyObject_TypeCheck(decl, &SpecificationBaseType))
+    {
+      PyObject *implied;
+
+      implied = ((Spec*)decl)->_implied;
+      if (implied == NULL)
+        {
+          Py_DECREF(decl);
+          return NULL;
+        }
+
+      implements = PyDict_GetItem(implied, self) != NULL;
+      Py_DECREF(decl);
+    }
+  else
+    {
+      /* decl is probably a security proxy.  We have to go the long way
+         around.
+      */
+      PyObject *r;
+      r = PyObject_CallFunctionObjArgs(decl, self, NULL);
+      Py_DECREF(decl);
+      if (r == NULL)
+        return NULL;
+      implements = PyObject_IsTrue(r);
+      Py_DECREF(r);
+    }
+
+  if (implements)
+    {
+      Py_INCREF(obj);
+      return obj;
+    }
+
+  l = PyList_GET_SIZE(adapter_hooks);
+  args = PyTuple_New(2);
+  if (args == NULL)
+    return NULL;
+  Py_INCREF(self);
+  PyTuple_SET_ITEM(args, 0, self);
+  Py_INCREF(obj);
+  PyTuple_SET_ITEM(args, 1, obj);
+  for (i = 0; i < l; i++)
+    {
+      adapter = PyObject_CallObject(PyList_GET_ITEM(adapter_hooks, i), args);
+      if (adapter == NULL || adapter != Py_None)
+        {
+          Py_DECREF(args);
+          return adapter;
+        }
+      Py_DECREF(adapter);
+    }
+
+  Py_DECREF(args);
+
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+typedef struct {
+    Spec spec;
+    PyObject* __name__;
+    PyObject* __module__;
+    Py_hash_t _v_cached_hash;
+} IB;
+
+static struct PyMethodDef ib_methods[] = {
+  {"__adapt__", (PyCFunction)__adapt__, METH_O,
+   "Adapt an object to the receiver"},
+  {NULL,                NULL}           /* sentinel */
+};
+
+/*
+    def __call__(self, obj, alternate=_marker):
+        try:
+            conform = obj.__conform__
+        except AttributeError: # pylint:disable=bare-except
+            conform = None
+
+        if conform is not None:
+            adapter = self._call_conform(conform)
+            if adapter is not None:
+                return adapter
+
+        adapter = self.__adapt__(obj)
+
+        if adapter is not None:
+            return adapter
+        if alternate is not _marker:
+            return alternate
+        raise TypeError("Could not adapt", obj, self)
+
+*/
+static PyObject *
+IB_call(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+  PyObject *conform, *obj, *alternate, *adapter;
+  static char *kwlist[] = {"obj", "alternate", NULL};
+  conform = obj = alternate = adapter = NULL;
+
+
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", kwlist,
+                                   &obj, &alternate))
+    return NULL;
+
+  conform = PyObject_GetAttr(obj, str__conform__);
+  if (conform == NULL)
+  {
+      if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+      {
+          /* Propagate non-AttributeErrors */
+          return NULL;
+      }
+      PyErr_Clear();
+
+      Py_INCREF(Py_None);
+      conform = Py_None;
+  }
+
+  if (conform != Py_None)
+  {
+      adapter = PyObject_CallMethodObjArgs(self, str_call_conform,
+                                           conform, NULL);
+      Py_DECREF(conform);
+      if (adapter == NULL || adapter != Py_None)
+          return adapter;
+      Py_DECREF(adapter);
+  }
+  else
+  {
+      Py_DECREF(conform);
+  }
+
+  /* We differ from the Python code here. For speed, instead of always calling
+     self.__adapt__(), we check to see if the type has defined it. Checking in
+     the dict for __adapt__ isn't sufficient because there's no cheap way to
+     tell if it's the __adapt__ that InterfaceBase itself defines (our type
+     will *never* be InterfaceBase, we're always subclassed by
+     InterfaceClass). Instead, we cooperate with InterfaceClass in Python to
+     set a flag in a new subclass when this is necessary. */
+  if (PyDict_GetItem(self->ob_type->tp_dict, str_CALL_CUSTOM_ADAPT))
+  {
+      /* Doesn't matter what the value is. Simply being present is enough. */
+      adapter = PyObject_CallMethodObjArgs(self, str__adapt__, obj, NULL);
+  }
+  else
+  {
+      adapter = __adapt__(self, obj);
+  }
+
+  if (adapter == NULL || adapter != Py_None)
+  {
+      return adapter;
+  }
+  Py_DECREF(adapter);
+
+  if (alternate != NULL)
+  {
+      Py_INCREF(alternate);
+      return alternate;
+  }
+
+  adapter = Py_BuildValue("sOO", "Could not adapt", obj, self);
+  if (adapter != NULL)
+  {
+      PyErr_SetObject(PyExc_TypeError, adapter);
+      Py_DECREF(adapter);
+  }
+  return NULL;
+}
+
+
+static int
+IB_traverse(IB* self, visitproc visit, void* arg)
+{
+    Py_VISIT(self->__name__);
+    Py_VISIT(self->__module__);
+    return Spec_traverse((Spec*)self, visit, arg);
+}
+
+static int
+IB_clear(IB* self)
+{
+    Py_CLEAR(self->__name__);
+    Py_CLEAR(self->__module__);
+    return Spec_clear((Spec*)self);
+}
+
+static void
+IB_dealloc(IB* self)
+{
+    PyObject_GC_UnTrack((PyObject *)self);
+    IB_clear(self);
+    Spec_dealloc((Spec*)self);
+}
+
+static PyMemberDef IB_members[] = {
+  {"__name__", T_OBJECT_EX, offsetof(IB, __name__), 0, ""},
+  // The redundancy between __module__ and __ibmodule__ is because
+  // __module__ is often shadowed by subclasses.
+  {"__module__", T_OBJECT_EX, offsetof(IB, __module__), READONLY, ""},
+  {"__ibmodule__", T_OBJECT_EX, offsetof(IB, __module__), 0, ""},
+  {NULL}
+};
+
+static Py_hash_t
+IB_hash(IB* self)
+{
+    PyObject* tuple;
+    if (!self->__module__) {
+        PyErr_SetString(PyExc_AttributeError, "__module__");
+        return -1;
+    }
+    if (!self->__name__) {
+        PyErr_SetString(PyExc_AttributeError, "__name__");
+        return -1;
+    }
+
+    if (self->_v_cached_hash) {
+        return self->_v_cached_hash;
+    }
+
+    tuple = PyTuple_Pack(2, self->__name__, self->__module__);
+    if (!tuple) {
+        return -1;
+    }
+    self->_v_cached_hash = PyObject_Hash(tuple);
+    Py_CLEAR(tuple);
+    return self->_v_cached_hash;
+}
+
+static PyTypeObject InterfaceBaseType;
+
+static PyObject*
+IB_richcompare(IB* self, PyObject* other, int op)
+{
+    PyObject* othername;
+    PyObject* othermod;
+    PyObject* oresult;
+    IB* otherib;
+    int result;
+
+    otherib = NULL;
+    oresult = othername = othermod = NULL;
+
+    if (OBJECT(self) == other) {
+        switch(op) {
+        case Py_EQ:
+        case Py_LE:
+        case Py_GE:
+            Py_RETURN_TRUE;
+            break;
+        case Py_NE:
+            Py_RETURN_FALSE;
+        }
+    }
+
+    if (other == Py_None) {
+        switch(op) {
+        case Py_LT:
+        case Py_LE:
+        case Py_NE:
+            Py_RETURN_TRUE;
+        default:
+            Py_RETURN_FALSE;
+        }
+    }
+
+    if (PyObject_TypeCheck(other, &InterfaceBaseType)) {
+        // This branch borrows references. No need to clean
+        // up if otherib is not null.
+        otherib = (IB*)other;
+        othername = otherib->__name__;
+        othermod = otherib->__module__;
+    }
+    else {
+        othername = PyObject_GetAttrString(other, "__name__");
+        if (othername) {
+            othermod = PyObject_GetAttrString(other, "__module__");
+        }
+        if (!othername || !othermod) {
+            if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_AttributeError)) {
+                PyErr_Clear();
+                oresult = Py_NotImplemented;
+            }
+            goto cleanup;
+        }
+    }
+#if 0
+// This is the simple, straightforward version of what Python does.
+    PyObject* pt1 = PyTuple_Pack(2, self->__name__, self->__module__);
+    PyObject* pt2 = PyTuple_Pack(2, othername, othermod);
+    oresult = PyObject_RichCompare(pt1, pt2, op);
+#endif
+
+    // tuple comparison is decided by the first non-equal element.
+    result = PyObject_RichCompareBool(self->__name__, othername, Py_EQ);
+    if (result == 0) {
+        result = PyObject_RichCompareBool(self->__name__, othername, op);
+    }
+    else if (result == 1) {
+        result = PyObject_RichCompareBool(self->__module__, othermod, op);
+    }
+    // If either comparison failed, we have an error set.
+    // Leave oresult NULL so we raise it.
+    if (result == -1) {
+        goto cleanup;
+    }
+
+    oresult = result ? Py_True : Py_False;
+
+
+cleanup:
+    Py_XINCREF(oresult);
+
+    if (!otherib) {
+        Py_XDECREF(othername);
+        Py_XDECREF(othermod);
+    }
+    return oresult;
+
+}
+
+static int
+IB_init(IB* self, PyObject* args, PyObject* kwargs)
+{
+    static char *kwlist[] = {"__name__", "__module__", NULL};
+    PyObject* module = NULL;
+    PyObject* name = NULL;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OO:InterfaceBase.__init__", kwlist,
+                                      &name, &module)) {
+        return -1;
+    }
+    IB_clear(self);
+    self->__module__ = module ? module : Py_None;
+    Py_INCREF(self->__module__);
+    self->__name__ = name ? name : Py_None;
+    Py_INCREF(self->__name__);
+    return 0;
+}
+
+
+static PyTypeObject InterfaceBaseType = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_zope_interface_coptimizations."
+                                "InterfaceBase",
+        /* tp_basicsize      */ sizeof(IB),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)IB_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)IB_hash,
+        /* tp_call           */ (ternaryfunc)IB_call,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT
+                                | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+        /* tp_doc */ "Interface base type providing __call__ and __adapt__",
+        /* tp_traverse       */ (traverseproc)IB_traverse,
+        /* tp_clear          */ (inquiry)IB_clear,
+        /* tp_richcompare    */ (richcmpfunc)IB_richcompare,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ ib_methods,
+        /* tp_members        */ IB_members,
+        /* tp_getset         */ 0,
+        /* tp_base           */ &SpecificationBaseType,
+        /* tp_dict           */ 0,
+        /* tp_descr_get      */ 0,
+        /* tp_descr_set      */ 0,
+        /* tp_dictoffset     */ 0,
+        /* tp_init           */ (initproc)IB_init,
+};
+
+/* =================== End: __call__ and __adapt__ ==================== */
+/* ==================================================================== */
+
+/* ==================================================================== */
+/* ========================== Begin: Lookup Bases ===================== */
+
+typedef struct {
+  PyObject_HEAD
+  PyObject *_cache;
+  PyObject *_mcache;
+  PyObject *_scache;
+} lookup;
+
+typedef struct {
+  PyObject_HEAD
+  PyObject *_cache;
+  PyObject *_mcache;
+  PyObject *_scache;
+  PyObject *_verify_ro;
+  PyObject *_verify_generations;
+} verify;
+
+static int
+lookup_traverse(lookup *self, visitproc visit, void *arg)
+{
+  int vret;
+
+  if (self->_cache) {
+    vret = visit(self->_cache, arg);
+    if (vret != 0)
+      return vret;
+  }
+
+  if (self->_mcache) {
+    vret = visit(self->_mcache, arg);
+    if (vret != 0)
+      return vret;
+  }
+
+  if (self->_scache) {
+    vret = visit(self->_scache, arg);
+    if (vret != 0)
+      return vret;
+  }
+
+  return 0;
+}
+
+static int
+lookup_clear(lookup *self)
+{
+  Py_CLEAR(self->_cache);
+  Py_CLEAR(self->_mcache);
+  Py_CLEAR(self->_scache);
+  return 0;
+}
+
+static void
+lookup_dealloc(lookup *self)
+{
+  PyObject_GC_UnTrack((PyObject *)self);
+  lookup_clear(self);
+  Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+/*
+    def changed(self, ignored=None):
+        self._cache.clear()
+        self._mcache.clear()
+        self._scache.clear()
+*/
+static PyObject *
+lookup_changed(lookup *self, PyObject *ignored)
+{
+  lookup_clear(self);
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+#define ASSURE_DICT(N) if (N == NULL) { N = PyDict_New(); \
+                                        if (N == NULL) return NULL; \
+                                       }
+
+/*
+    def _getcache(self, provided, name):
+        cache = self._cache.get(provided)
+        if cache is None:
+            cache = {}
+            self._cache[provided] = cache
+        if name:
+            c = cache.get(name)
+            if c is None:
+                c = {}
+                cache[name] = c
+            cache = c
+        return cache
+*/
+static PyObject *
+_subcache(PyObject *cache, PyObject *key)
+{
+  PyObject *subcache;
+
+  subcache = PyDict_GetItem(cache, key);
+  if (subcache == NULL)
+    {
+      int status;
+
+      subcache = PyDict_New();
+      if (subcache == NULL)
+        return NULL;
+      status = PyDict_SetItem(cache, key, subcache);
+      Py_DECREF(subcache);
+      if (status < 0)
+        return NULL;
+    }
+
+  return subcache;
+}
+static PyObject *
+_getcache(lookup *self, PyObject *provided, PyObject *name)
+{
+  PyObject *cache;
+
+  ASSURE_DICT(self->_cache);
+  cache = _subcache(self->_cache, provided);
+  if (cache == NULL)
+    return NULL;
+
+  if (name != NULL && PyObject_IsTrue(name))
+    cache = _subcache(cache, name);
+
+  return cache;
+}
+
+
+/*
+    def lookup(self, required, provided, name=u'', default=None):
+        cache = self._getcache(provided, name)
+        if len(required) == 1:
+            result = cache.get(required[0], _not_in_mapping)
+        else:
+            result = cache.get(tuple(required), _not_in_mapping)
+
+        if result is _not_in_mapping:
+            result = self._uncached_lookup(required, provided, name)
+            if len(required) == 1:
+                cache[required[0]] = result
+            else:
+                cache[tuple(required)] = result
+
+        if result is None:
+            return default
+
+        return result
+*/
+
+static PyObject *
+_lookup(lookup *self,
+        PyObject *required, PyObject *provided, PyObject *name,
+        PyObject *default_)
+{
+  PyObject *result, *key, *cache;
+  result = key = cache = NULL;
+  if ( name && !PyUnicode_Check(name) )
+  {
+    PyErr_SetString(PyExc_ValueError,
+                    "name is not a string or unicode");
+    return NULL;
+  }
+
+  /* If `required` is a lazy sequence, it could have arbitrary side-effects,
+     such as clearing our caches. So we must not retrieve the cache until
+     after resolving it. */
+  required = PySequence_Tuple(required);
+  if (required == NULL)
+    return NULL;
+
+
+  cache = _getcache(self, provided, name);
+  if (cache == NULL)
+    return NULL;
+
+  if (PyTuple_GET_SIZE(required) == 1)
+    key = PyTuple_GET_ITEM(required, 0);
+  else
+    key = required;
+
+  result = PyDict_GetItem(cache, key);
+  if (result == NULL)
+    {
+      int status;
+
+      result = PyObject_CallMethodObjArgs(OBJECT(self), str_uncached_lookup,
+                                          required, provided, name, NULL);
+      if (result == NULL)
+        {
+          Py_DECREF(required);
+          return NULL;
+        }
+      status = PyDict_SetItem(cache, key, result);
+      Py_DECREF(required);
+      if (status < 0)
+        {
+          Py_DECREF(result);
+          return NULL;
+        }
+    }
+  else
+    {
+      Py_INCREF(result);
+      Py_DECREF(required);
+    }
+
+  if (result == Py_None && default_ != NULL)
+    {
+      Py_DECREF(Py_None);
+      Py_INCREF(default_);
+      return default_;
+    }
+
+  return result;
+}
+static PyObject *
+lookup_lookup(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", "name", "default", NULL};
+  PyObject *required, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:LookupBase.lookup", kwlist,
+                                    &required, &provided, &name, &default_))
+    return NULL;
+
+  return _lookup(self, required, provided, name, default_);
+}
+
+
+/*
+    def lookup1(self, required, provided, name=u'', default=None):
+        cache = self._getcache(provided, name)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            return self.lookup((required, ), provided, name, default)
+
+        if result is None:
+            return default
+
+        return result
+*/
+static PyObject *
+_lookup1(lookup *self,
+        PyObject *required, PyObject *provided, PyObject *name,
+        PyObject *default_)
+{
+  PyObject *result, *cache;
+
+  if ( name && !PyUnicode_Check(name) )
+  {
+    PyErr_SetString(PyExc_ValueError,
+                    "name is not a string or unicode");
+    return NULL;
+  }
+
+  cache = _getcache(self, provided, name);
+  if (cache == NULL)
+    return NULL;
+
+  result = PyDict_GetItem(cache, required);
+  if (result == NULL)
+    {
+      PyObject *tup;
+
+      tup = PyTuple_New(1);
+      if (tup == NULL)
+        return NULL;
+      Py_INCREF(required);
+      PyTuple_SET_ITEM(tup, 0, required);
+      result = _lookup(self, tup, provided, name, default_);
+      Py_DECREF(tup);
+    }
+  else
+    {
+      if (result == Py_None && default_ != NULL)
+        {
+          result = default_;
+        }
+      Py_INCREF(result);
+    }
+
+  return result;
+}
+static PyObject *
+lookup_lookup1(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", "name", "default", NULL};
+  PyObject *required, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:LookupBase.lookup1", kwlist,
+                                    &required, &provided, &name, &default_))
+    return NULL;
+
+  return _lookup1(self, required, provided, name, default_);
+}
+
+/*
+    def adapter_hook(self, provided, object, name=u'', default=None):
+        required = providedBy(object)
+        cache = self._getcache(provided, name)
+        factory = cache.get(required, _not_in_mapping)
+        if factory is _not_in_mapping:
+            factory = self.lookup((required, ), provided, name)
+
+        if factory is not None:
+            if isinstance(object, super):
+                object = object.__self__
+            result = factory(object)
+            if result is not None:
+                return result
+
+        return default
+*/
+static PyObject *
+_adapter_hook(lookup *self,
+              PyObject *provided, PyObject *object,  PyObject *name,
+              PyObject *default_)
+{
+  PyObject *required, *factory, *result;
+
+  if ( name && !PyUnicode_Check(name) )
+  {
+    PyErr_SetString(PyExc_ValueError,
+                    "name is not a string or unicode");
+    return NULL;
+  }
+
+  required = providedBy(NULL, object);
+  if (required == NULL)
+    return NULL;
+
+  factory = _lookup1(self, required, provided, name, Py_None);
+  Py_DECREF(required);
+  if (factory == NULL)
+    return NULL;
+
+  if (factory != Py_None)
+  {
+      if (PyObject_TypeCheck(object, &PySuper_Type)) {
+          PyObject* self = PyObject_GetAttr(object, str__self__);
+          if (self == NULL)
+          {
+              Py_DECREF(factory);
+              return NULL;
+          }
+          // Borrow the reference to self
+          Py_DECREF(self);
+          object = self;
+      }
+      result = PyObject_CallFunctionObjArgs(factory, object, NULL);
+      Py_DECREF(factory);
+      if (result == NULL || result != Py_None)
+        return result;
+  }
+  else
+    result = factory; /* None */
+
+  if (default_ == NULL || default_ == result) /* No default specified, */
+    return result;   /* Return None.  result is owned None */
+
+  Py_DECREF(result);
+  Py_INCREF(default_);
+
+  return default_;
+}
+static PyObject *
+lookup_adapter_hook(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"provided", "object", "name", "default", NULL};
+  PyObject *object, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:LookupBase.adapter_hook", kwlist,
+                                    &provided, &object, &name, &default_))
+    return NULL;
+
+  return _adapter_hook(self, provided, object, name, default_);
+}
+
+static PyObject *
+lookup_queryAdapter(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"object", "provided", "name", "default", NULL};
+  PyObject *object, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:LookupBase.queryAdapter", kwlist,
+                                    &object, &provided, &name, &default_))
+    return NULL;
+
+  return _adapter_hook(self, provided, object, name, default_);
+}
+
+/*
+    def lookupAll(self, required, provided):
+        cache = self._mcache.get(provided)
+        if cache is None:
+            cache = {}
+            self._mcache[provided] = cache
+
+        required = tuple(required)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            result = self._uncached_lookupAll(required, provided)
+            cache[required] = result
+
+        return result
+*/
+static PyObject *
+_lookupAll(lookup *self, PyObject *required, PyObject *provided)
+{
+  PyObject *cache, *result;
+
+  /* resolve before getting cache. See note in _lookup. */
+  required = PySequence_Tuple(required);
+  if (required == NULL)
+    return NULL;
+
+  ASSURE_DICT(self->_mcache);
+  cache = _subcache(self->_mcache, provided);
+  if (cache == NULL)
+    return NULL;
+
+  result = PyDict_GetItem(cache, required);
+  if (result == NULL)
+    {
+      int status;
+
+      result = PyObject_CallMethodObjArgs(OBJECT(self), str_uncached_lookupAll,
+                                          required, provided, NULL);
+      if (result == NULL)
+        {
+          Py_DECREF(required);
+          return NULL;
+        }
+      status = PyDict_SetItem(cache, required, result);
+      Py_DECREF(required);
+      if (status < 0)
+        {
+          Py_DECREF(result);
+          return NULL;
+        }
+    }
+  else
+    {
+      Py_INCREF(result);
+      Py_DECREF(required);
+    }
+
+  return result;
+}
+static PyObject *
+lookup_lookupAll(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", NULL};
+  PyObject *required, *provided;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO:LookupBase.lookupAll", kwlist,
+                                    &required, &provided))
+    return NULL;
+
+  return _lookupAll(self, required, provided);
+}
+
+/*
+    def subscriptions(self, required, provided):
+        cache = self._scache.get(provided)
+        if cache is None:
+            cache = {}
+            self._scache[provided] = cache
+
+        required = tuple(required)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            result = self._uncached_subscriptions(required, provided)
+            cache[required] = result
+
+        return result
+*/
+static PyObject *
+_subscriptions(lookup *self, PyObject *required, PyObject *provided)
+{
+  PyObject *cache, *result;
+
+  /* resolve before getting cache. See note in _lookup. */
+  required = PySequence_Tuple(required);
+  if (required == NULL)
+    return NULL;
+
+  ASSURE_DICT(self->_scache);
+  cache = _subcache(self->_scache, provided);
+  if (cache == NULL)
+    return NULL;
+
+  result = PyDict_GetItem(cache, required);
+  if (result == NULL)
+    {
+      int status;
+
+      result = PyObject_CallMethodObjArgs(
+                                 OBJECT(self), str_uncached_subscriptions,
+                                 required, provided, NULL);
+      if (result == NULL)
+        {
+          Py_DECREF(required);
+          return NULL;
+        }
+      status = PyDict_SetItem(cache, required, result);
+      Py_DECREF(required);
+      if (status < 0)
+        {
+          Py_DECREF(result);
+          return NULL;
+        }
+    }
+  else
+    {
+      Py_INCREF(result);
+      Py_DECREF(required);
+    }
+
+  return result;
+}
+static PyObject *
+lookup_subscriptions(lookup *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", NULL};
+  PyObject *required, *provided;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist,
+                                    &required, &provided))
+    return NULL;
+
+  return _subscriptions(self, required, provided);
+}
+
+static struct PyMethodDef lookup_methods[] = {
+  {"changed",       (PyCFunction)lookup_changed,       METH_O,        ""},
+  {"lookup",        (PyCFunction)lookup_lookup,        METH_KEYWORDS | METH_VARARGS, ""},
+  {"lookup1",       (PyCFunction)lookup_lookup1,       METH_KEYWORDS | METH_VARARGS, ""},
+  {"queryAdapter",  (PyCFunction)lookup_queryAdapter,  METH_KEYWORDS | METH_VARARGS, ""},
+  {"adapter_hook",  (PyCFunction)lookup_adapter_hook,  METH_KEYWORDS | METH_VARARGS, ""},
+  {"lookupAll",     (PyCFunction)lookup_lookupAll,     METH_KEYWORDS | METH_VARARGS, ""},
+  {"subscriptions", (PyCFunction)lookup_subscriptions, METH_KEYWORDS | METH_VARARGS, ""},
+  {NULL,            NULL}               /* sentinel */
+};
+
+static PyTypeObject LookupBase = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_zope_interface_coptimizations."
+                                "LookupBase",
+        /* tp_basicsize      */ sizeof(lookup),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)&lookup_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)0,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT
+                                | Py_TPFLAGS_BASETYPE
+                                | Py_TPFLAGS_HAVE_GC,
+        /* tp_doc            */ "",
+        /* tp_traverse       */ (traverseproc)lookup_traverse,
+        /* tp_clear          */ (inquiry)lookup_clear,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ lookup_methods,
+};
+
+static int
+verifying_traverse(verify *self, visitproc visit, void *arg)
+{
+  int vret;
+
+  vret = lookup_traverse((lookup *)self, visit, arg);
+  if (vret != 0)
+    return vret;
+
+  if (self->_verify_ro) {
+    vret = visit(self->_verify_ro, arg);
+    if (vret != 0)
+      return vret;
+  }
+  if (self->_verify_generations) {
+    vret = visit(self->_verify_generations, arg);
+    if (vret != 0)
+      return vret;
+  }
+
+  return 0;
+}
+
+static int
+verifying_clear(verify *self)
+{
+  lookup_clear((lookup *)self);
+  Py_CLEAR(self->_verify_generations);
+  Py_CLEAR(self->_verify_ro);
+  return 0;
+}
+
+
+static void
+verifying_dealloc(verify *self)
+{
+  PyObject_GC_UnTrack((PyObject *)self);
+  verifying_clear(self);
+  Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+/*
+    def changed(self, originally_changed):
+        super(VerifyingBasePy, self).changed(originally_changed)
+        self._verify_ro = self._registry.ro[1:]
+        self._verify_generations = [r._generation for r in self._verify_ro]
+*/
+static PyObject *
+_generations_tuple(PyObject *ro)
+{
+  int i, l;
+  PyObject *generations;
+
+  l = PyTuple_GET_SIZE(ro);
+  generations = PyTuple_New(l);
+  for (i=0; i < l; i++)
+    {
+      PyObject *generation;
+
+      generation = PyObject_GetAttr(PyTuple_GET_ITEM(ro, i), str_generation);
+      if (generation == NULL)
+        {
+          Py_DECREF(generations);
+          return NULL;
+        }
+      PyTuple_SET_ITEM(generations, i, generation);
+    }
+
+  return generations;
+}
+static PyObject *
+verifying_changed(verify *self, PyObject *ignored)
+{
+  PyObject *t, *ro;
+
+  verifying_clear(self);
+
+  t = PyObject_GetAttr(OBJECT(self), str_registry);
+  if (t == NULL)
+    return NULL;
+  ro = PyObject_GetAttr(t, strro);
+  Py_DECREF(t);
+  if (ro == NULL)
+    return NULL;
+
+  t = PyObject_CallFunctionObjArgs(OBJECT(&PyTuple_Type), ro, NULL);
+  Py_DECREF(ro);
+  if (t == NULL)
+    return NULL;
+
+  ro = PyTuple_GetSlice(t, 1, PyTuple_GET_SIZE(t));
+  Py_DECREF(t);
+  if (ro == NULL)
+    return NULL;
+
+  self->_verify_generations = _generations_tuple(ro);
+  if (self->_verify_generations == NULL)
+    {
+      Py_DECREF(ro);
+      return NULL;
+    }
+
+  self->_verify_ro = ro;
+
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+/*
+    def _verify(self):
+        if ([r._generation for r in self._verify_ro]
+            != self._verify_generations):
+            self.changed(None)
+*/
+static int
+_verify(verify *self)
+{
+  PyObject *changed_result;
+
+  if (self->_verify_ro != NULL && self->_verify_generations != NULL)
+    {
+      PyObject *generations;
+      int changed;
+
+      generations = _generations_tuple(self->_verify_ro);
+      if (generations == NULL)
+        return -1;
+
+      changed = PyObject_RichCompareBool(self->_verify_generations,
+                                         generations, Py_NE);
+      Py_DECREF(generations);
+      if (changed == -1)
+        return -1;
+
+      if (changed == 0)
+        return 0;
+    }
+
+  changed_result = PyObject_CallMethodObjArgs(OBJECT(self), strchanged,
+                                              Py_None, NULL);
+  if (changed_result == NULL)
+    return -1;
+
+  Py_DECREF(changed_result);
+  return 0;
+}
+
+static PyObject *
+verifying_lookup(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", "name", "default", NULL};
+  PyObject *required, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO", kwlist,
+                                    &required, &provided, &name, &default_))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _lookup((lookup *)self, required, provided, name, default_);
+}
+
+static PyObject *
+verifying_lookup1(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", "name", "default", NULL};
+  PyObject *required, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO", kwlist,
+                                    &required, &provided, &name, &default_))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _lookup1((lookup *)self, required, provided, name, default_);
+}
+
+static PyObject *
+verifying_adapter_hook(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"provided", "object", "name", "default", NULL};
+  PyObject *object, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO", kwlist,
+                                    &provided, &object, &name, &default_))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _adapter_hook((lookup *)self, provided, object, name, default_);
+}
+
+static PyObject *
+verifying_queryAdapter(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"object", "provided", "name", "default", NULL};
+  PyObject *object, *provided, *name=NULL, *default_=NULL;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO", kwlist,
+                                    &object, &provided, &name, &default_))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _adapter_hook((lookup *)self, provided, object, name, default_);
+}
+
+static PyObject *
+verifying_lookupAll(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", NULL};
+  PyObject *required, *provided;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist,
+                                    &required, &provided))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _lookupAll((lookup *)self, required, provided);
+}
+
+static PyObject *
+verifying_subscriptions(verify *self, PyObject *args, PyObject *kwds)
+{
+  static char *kwlist[] = {"required", "provided", NULL};
+  PyObject *required, *provided;
+
+  if (! PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist,
+                                    &required, &provided))
+    return NULL;
+
+  if (_verify(self) < 0)
+    return NULL;
+
+  return _subscriptions((lookup *)self, required, provided);
+}
+
+static struct PyMethodDef verifying_methods[] = {
+  {"changed",      (PyCFunction)verifying_changed,        METH_O,        ""},
+  {"lookup",       (PyCFunction)verifying_lookup,         METH_KEYWORDS | METH_VARARGS, ""},
+  {"lookup1",      (PyCFunction)verifying_lookup1,        METH_KEYWORDS | METH_VARARGS, ""},
+  {"queryAdapter",  (PyCFunction)verifying_queryAdapter,  METH_KEYWORDS | METH_VARARGS, ""},
+  {"adapter_hook",  (PyCFunction)verifying_adapter_hook,  METH_KEYWORDS | METH_VARARGS, ""},
+  {"lookupAll",    (PyCFunction)verifying_lookupAll,      METH_KEYWORDS | METH_VARARGS, ""},
+  {"subscriptions", (PyCFunction)verifying_subscriptions, METH_KEYWORDS | METH_VARARGS, ""},
+  {NULL,            NULL}               /* sentinel */
+};
+
+static PyTypeObject VerifyingBase = {
+        PyVarObject_HEAD_INIT(NULL, 0)
+        /* tp_name           */ "_zope_interface_coptimizations."
+                                "VerifyingBase",
+        /* tp_basicsize      */ sizeof(verify),
+        /* tp_itemsize       */ 0,
+        /* tp_dealloc        */ (destructor)&verifying_dealloc,
+        /* tp_print          */ (printfunc)0,
+        /* tp_getattr        */ (getattrfunc)0,
+        /* tp_setattr        */ (setattrfunc)0,
+        /* tp_compare        */ 0,
+        /* tp_repr           */ (reprfunc)0,
+        /* tp_as_number      */ 0,
+        /* tp_as_sequence    */ 0,
+        /* tp_as_mapping     */ 0,
+        /* tp_hash           */ (hashfunc)0,
+        /* tp_call           */ (ternaryfunc)0,
+        /* tp_str            */ (reprfunc)0,
+        /* tp_getattro       */ (getattrofunc)0,
+        /* tp_setattro       */ (setattrofunc)0,
+        /* tp_as_buffer      */ 0,
+        /* tp_flags          */ Py_TPFLAGS_DEFAULT
+                                | Py_TPFLAGS_BASETYPE
+                                | Py_TPFLAGS_HAVE_GC,
+        /* tp_doc            */ "",
+        /* tp_traverse       */ (traverseproc)verifying_traverse,
+        /* tp_clear          */ (inquiry)verifying_clear,
+        /* tp_richcompare    */ (richcmpfunc)0,
+        /* tp_weaklistoffset */ (long)0,
+        /* tp_iter           */ (getiterfunc)0,
+        /* tp_iternext       */ (iternextfunc)0,
+        /* tp_methods        */ verifying_methods,
+        /* tp_members        */ 0,
+        /* tp_getset         */ 0,
+        /* tp_base           */ &LookupBase,
+};
+
+/* ========================== End: Lookup Bases ======================= */
+/* ==================================================================== */
+
+
+
+static struct PyMethodDef m_methods[] = {
+  {"implementedBy", (PyCFunction)implementedBy, METH_O,
+   "Interfaces implemented by a class or factory.\n"
+   "Raises TypeError if argument is neither a class nor a callable."},
+  {"getObjectSpecification", (PyCFunction)getObjectSpecification, METH_O,
+   "Get an object's interfaces (internal api)"},
+  {"providedBy", (PyCFunction)providedBy, METH_O,
+   "Get an object's interfaces"},
+
+  {NULL,         (PyCFunction)NULL, 0, NULL}            /* sentinel */
+};
+
+#if  PY_MAJOR_VERSION >= 3
+static char module_doc[] = "C optimizations for zope.interface\n\n";
+
+static struct PyModuleDef _zic_module = {
+        PyModuleDef_HEAD_INIT,
+        "_zope_interface_coptimizations",
+        module_doc,
+        -1,
+        m_methods,
+        NULL,
+        NULL,
+        NULL,
+        NULL
+};
+#endif
+
+static PyObject *
+init(void)
+{
+  PyObject *m;
+
+#if  PY_MAJOR_VERSION < 3
+#define DEFINE_STRING(S) \
+  if(! (str ## S = PyString_FromString(# S))) return NULL
+#else
+#define DEFINE_STRING(S) \
+  if(! (str ## S = PyUnicode_FromString(# S))) return NULL
+#endif
+
+  DEFINE_STRING(__dict__);
+  DEFINE_STRING(__implemented__);
+  DEFINE_STRING(__provides__);
+  DEFINE_STRING(__class__);
+  DEFINE_STRING(__providedBy__);
+  DEFINE_STRING(extends);
+  DEFINE_STRING(__conform__);
+  DEFINE_STRING(_call_conform);
+  DEFINE_STRING(_uncached_lookup);
+  DEFINE_STRING(_uncached_lookupAll);
+  DEFINE_STRING(_uncached_subscriptions);
+  DEFINE_STRING(_registry);
+  DEFINE_STRING(_generation);
+  DEFINE_STRING(ro);
+  DEFINE_STRING(changed);
+  DEFINE_STRING(__self__);
+  DEFINE_STRING(__name__);
+  DEFINE_STRING(__module__);
+  DEFINE_STRING(__adapt__);
+  DEFINE_STRING(_CALL_CUSTOM_ADAPT);
+#undef DEFINE_STRING
+  adapter_hooks = PyList_New(0);
+  if (adapter_hooks == NULL)
+    return NULL;
+
+  /* Initialize types: */
+  SpecificationBaseType.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&SpecificationBaseType) < 0)
+    return NULL;
+  OSDType.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&OSDType) < 0)
+    return NULL;
+  CPBType.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&CPBType) < 0)
+    return NULL;
+
+  InterfaceBaseType.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&InterfaceBaseType) < 0)
+    return NULL;
+
+  LookupBase.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&LookupBase) < 0)
+    return NULL;
+
+  VerifyingBase.tp_new = PyBaseObject_Type.tp_new;
+  if (PyType_Ready(&VerifyingBase) < 0)
+    return NULL;
+
+  #if PY_MAJOR_VERSION < 3
+  /* Create the module and add the functions */
+  m = Py_InitModule3("_zope_interface_coptimizations", m_methods,
+                     "C optimizations for zope.interface\n\n");
+  #else
+  m = PyModule_Create(&_zic_module);
+  #endif
+  if (m == NULL)
+    return NULL;
+
+  /* Add types: */
+  if (PyModule_AddObject(m, "SpecificationBase", OBJECT(&SpecificationBaseType)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "ObjectSpecificationDescriptor",
+                         (PyObject *)&OSDType) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "ClassProvidesBase", OBJECT(&CPBType)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "InterfaceBase", OBJECT(&InterfaceBaseType)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "LookupBase", OBJECT(&LookupBase)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "VerifyingBase", OBJECT(&VerifyingBase)) < 0)
+    return NULL;
+  if (PyModule_AddObject(m, "adapter_hooks", adapter_hooks) < 0)
+    return NULL;
+  return m;
+}
+
+PyMODINIT_FUNC
+#if PY_MAJOR_VERSION < 3
+init_zope_interface_coptimizations(void)
+{
+  init();
+}
+#else
+PyInit__zope_interface_coptimizations(void)
+{
+  return init();
+}
+#endif
+
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
diff --git a/contrib/python/zope.interface/py3/zope/interface/adapter.py b/contrib/python/zope.interface/py3/zope/interface/adapter.py
new file mode 100644
index 00000000000..dbff0d19da2
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/adapter.py
@@ -0,0 +1,1015 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Adapter management
+"""
+import itertools
+import weakref
+
+from zope.interface import implementer
+from zope.interface import providedBy
+from zope.interface import Interface
+from zope.interface import ro
+from zope.interface.interfaces import IAdapterRegistry
+
+from zope.interface._compat import _normalize_name
+from zope.interface._compat import _use_c_impl
+
+__all__ = [
+    'AdapterRegistry',
+    'VerifyingAdapterRegistry',
+]
+
+# In the CPython implementation,
+# ``tuple`` and ``list`` cooperate so that ``tuple([some list])``
+# directly allocates and iterates at the C level without using a
+# Python iterator. That's not the case for
+# ``tuple(generator_expression)`` or ``tuple(map(func, it))``.
+##
+# 3.8
+# ``tuple([t for t in range(10)])``      -> 610ns
+# ``tuple(t for t in range(10))``        -> 696ns
+# ``tuple(map(lambda t: t, range(10)))`` -> 881ns
+##
+# 2.7
+# ``tuple([t fon t in range(10)])``      -> 625ns
+# ``tuple(t for t in range(10))``        -> 665ns
+# ``tuple(map(lambda t: t, range(10)))`` -> 958ns
+#
+# All three have substantial variance.
+##
+# On PyPy, this is also the best option.
+##
+# PyPy 2.7.18-7.3.3
+# ``tuple([t fon t in range(10)])``      -> 128ns
+# ``tuple(t for t in range(10))``        -> 175ns
+# ``tuple(map(lambda t: t, range(10)))`` -> 153ns
+##
+# PyPy 3.7.9 7.3.3-beta
+# ``tuple([t fon t in range(10)])``      ->  82ns
+# ``tuple(t for t in range(10))``        -> 177ns
+# ``tuple(map(lambda t: t, range(10)))`` -> 168ns
+#
+
+class BaseAdapterRegistry:
+    """
+    A basic implementation of the data storage and algorithms required
+    for a :class:`zope.interface.interfaces.IAdapterRegistry`.
+
+    Subclasses can set the following attributes to control how the data
+    is stored; in particular, these hooks can be helpful for ZODB
+    persistence. They can be class attributes that are the named (or similar) type, or
+    they can be methods that act as a constructor for an object that behaves
+    like the types defined here; this object will not assume that they are type
+    objects, but subclasses are free to do so:
+
+    _sequenceType = list
+      This is the type used for our two mutable top-level "byorder" sequences.
+      Must support mutation operations like ``append()`` and ``del seq[index]``.
+      These are usually small (< 10). Although at least one of them is
+      accessed when performing lookups or queries on this object, the other
+      is untouched. In many common scenarios, both are only required when
+      mutating registrations and subscriptions (like what
+      :meth:`zope.interface.interfaces.IComponents.registerUtility` does).
+      This use pattern makes it an ideal candidate to be a
+      :class:`~persistent.list.PersistentList`.
+    _leafSequenceType = tuple
+      This is the type used for the leaf sequences of subscribers.
+      It could be set to a ``PersistentList`` to avoid many unnecessary data
+      loads when subscribers aren't being used. Mutation operations are directed
+      through :meth:`_addValueToLeaf` and :meth:`_removeValueFromLeaf`; if you use
+      a mutable type, you'll need to override those.
+    _mappingType = dict
+      This is the mutable mapping type used for the keyed mappings.
+      A :class:`~persistent.mapping.PersistentMapping`
+      could be used to help reduce the number of data loads when the registry is large
+      and parts of it are rarely used. Further reductions in data loads can come from
+      using a :class:`~BTrees.OOBTree.OOBTree`, but care is required
+      to be sure that all required/provided
+      values are fully ordered (e.g., no required or provided values that are classes
+      can be used).
+    _providedType = dict
+      This is the mutable mapping type used for the ``_provided`` mapping.
+      This is separate from the generic mapping type because the values
+      are always integers, so one might choose to use a more optimized data
+      structure such as a :class:`~BTrees.OIBTree.OIBTree`.
+      The same caveats regarding key types
+      apply as for ``_mappingType``.
+
+    It is possible to also set these on an instance, but because of the need to
+    potentially also override :meth:`_addValueToLeaf` and :meth:`_removeValueFromLeaf`,
+    this may be less useful in a persistent scenario; using a subclass is recommended.
+
+    .. versionchanged:: 5.3.0
+        Add support for customizing the way internal data
+        structures are created.
+    .. versionchanged:: 5.3.0
+        Add methods :meth:`rebuild`, :meth:`allRegistrations`
+        and :meth:`allSubscriptions`.
+    """
+
+    # List of methods copied from lookup sub-objects:
+    _delegated = ('lookup', 'queryMultiAdapter', 'lookup1', 'queryAdapter',
+                  'adapter_hook', 'lookupAll', 'names',
+                  'subscriptions', 'subscribers')
+
+    # All registries maintain a generation that can be used by verifying
+    # registries
+    _generation = 0
+
+    def __init__(self, bases=()):
+
+        # The comments here could be improved. Possibly this bit needs
+        # explaining in a separate document, as the comments here can
+        # be quite confusing. /regebro
+
+        # {order -> {required -> {provided -> {name -> value}}}}
+        # Here "order" is actually an index in a list, "required" and
+        # "provided" are interfaces, and "required" is really a nested
+        # key.  So, for example:
+        # for order == 0 (that is, self._adapters[0]), we have:
+        #   {provided -> {name -> value}}
+        # but for order == 2 (that is, self._adapters[2]), we have:
+        #   {r1 -> {r2 -> {provided -> {name -> value}}}}
+        #
+        self._adapters = self._sequenceType()
+
+        # {order -> {required -> {provided -> {name -> [value]}}}}
+        # where the remarks about adapters above apply
+        self._subscribers = self._sequenceType()
+
+        # Set, with a reference count, keeping track of the interfaces
+        # for which we have provided components:
+        self._provided = self._providedType()
+
+        # Create ``_v_lookup`` object to perform lookup.  We make this a
+        # separate object to to make it easier to implement just the
+        # lookup functionality in C.  This object keeps track of cache
+        # invalidation data in two kinds of registries.
+
+        #   Invalidating registries have caches that are invalidated
+        #     when they or their base registies change.  An invalidating
+        #     registry can only have invalidating registries as bases.
+        #     See LookupBaseFallback below for the pertinent logic.
+
+        #   Verifying registies can't rely on getting invalidation messages,
+        #     so have to check the generations of base registries to determine
+        #     if their cache data are current.  See VerifyingBasePy below
+        #     for the pertinent object.
+        self._createLookup()
+
+        # Setting the bases causes the registries described above
+        # to be initialized (self._setBases -> self.changed ->
+        # self._v_lookup.changed).
+
+        self.__bases__ = bases
+
+    def _setBases(self, bases):
+        """
+        If subclasses need to track when ``__bases__`` changes, they
+        can override this method.
+
+        Subclasses must still call this method.
+        """
+        self.__dict__['__bases__'] = bases
+        self.ro = ro.ro(self)
+        self.changed(self)
+
+    __bases__ = property(lambda self: self.__dict__['__bases__'],
+                         lambda self, bases: self._setBases(bases),
+                         )
+
+    def _createLookup(self):
+        self._v_lookup = self.LookupClass(self)
+        for name in self._delegated:
+            self.__dict__[name] = getattr(self._v_lookup, name)
+
+    # Hooks for subclasses to define the types of objects used in
+    # our data structures.
+    # These have to be documented in the docstring, instead of local
+    # comments, because Sphinx autodoc ignores the comment and just writes
+    # "alias of list"
+    _sequenceType = list
+    _leafSequenceType = tuple
+    _mappingType = dict
+    _providedType = dict
+
+    def _addValueToLeaf(self, existing_leaf_sequence, new_item):
+        """
+        Add the value *new_item* to the *existing_leaf_sequence*, which may
+        be ``None``.
+
+        Subclasses that redefine `_leafSequenceType` should override this method.
+
+        :param existing_leaf_sequence:
+            If *existing_leaf_sequence* is not *None*, it will be an instance
+            of `_leafSequenceType`. (Unless the object has been unpickled
+            from an old pickle and the class definition has changed, in which case
+            it may be an instance of a previous definition, commonly a `tuple`.)
+
+        :return:
+           This method returns the new value to be stored. It may mutate the
+           sequence in place if it was not ``None`` and the type is mutable, but
+           it must also return it.
+
+        .. versionadded:: 5.3.0
+        """
+        if existing_leaf_sequence is None:
+            return (new_item,)
+        return existing_leaf_sequence + (new_item,)
+
+    def _removeValueFromLeaf(self, existing_leaf_sequence, to_remove):
+        """
+        Remove the item *to_remove* from the (non-``None``, non-empty)
+        *existing_leaf_sequence* and return the mutated sequence.
+
+        If there is more than one item that is equal to *to_remove*
+        they must all be removed.
+
+        Subclasses that redefine `_leafSequenceType` should override
+        this method. Note that they can call this method to help
+        in their implementation; this implementation will always
+        return a new tuple constructed by iterating across
+        the *existing_leaf_sequence* and omitting items equal to *to_remove*.
+
+        :param existing_leaf_sequence:
+           As for `_addValueToLeaf`, probably an instance of
+           `_leafSequenceType` but possibly an older type; never `None`.
+        :return:
+           A version of *existing_leaf_sequence* with all items equal to
+           *to_remove* removed. Must not return `None`. However,
+           returning an empty
+           object, even of another type such as the empty tuple, ``()`` is
+           explicitly allowed; such an object will never be stored.
+
+        .. versionadded:: 5.3.0
+        """
+        return tuple([v for v in existing_leaf_sequence if v != to_remove])
+
+    def changed(self, originally_changed):
+        self._generation += 1
+        self._v_lookup.changed(originally_changed)
+
+    def register(self, required, provided, name, value):
+        if not isinstance(name, str):
+            raise ValueError('name is not a string')
+        if value is None:
+            self.unregister(required, provided, name, value)
+            return
+
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        name = _normalize_name(name)
+        order = len(required)
+        byorder = self._adapters
+        while len(byorder) <= order:
+            byorder.append(self._mappingType())
+        components = byorder[order]
+        key = required + (provided,)
+
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                d = self._mappingType()
+                components[k] = d
+            components = d
+
+        if components.get(name) is value:
+            return
+
+        components[name] = value
+
+        n = self._provided.get(provided, 0) + 1
+        self._provided[provided] = n
+        if n == 1:
+            self._v_lookup.add_extendor(provided)
+
+        self.changed(self)
+
+    def _find_leaf(self, byorder, required, provided, name):
+        # Find the leaf value, if any, in the *byorder* list
+        # for the interface sequence *required* and the interface
+        # *provided*, given the already normalized *name*.
+        #
+        # If no such leaf value exists, returns ``None``
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        order = len(required)
+        if len(byorder) <= order:
+            return None
+
+        components = byorder[order]
+        key = required + (provided,)
+
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                return None
+            components = d
+
+        return components.get(name)
+
+    def registered(self, required, provided, name=''):
+        return self._find_leaf(
+            self._adapters,
+            required,
+            provided,
+            _normalize_name(name)
+        )
+
+    @classmethod
+    def _allKeys(cls, components, i, parent_k=()):
+        if i == 0:
+            for k, v in components.items():
+                yield parent_k + (k,), v
+        else:
+            for k, v in components.items():
+                new_parent_k = parent_k + (k,)
+                yield from cls._allKeys(v, i - 1, new_parent_k)
+
+    def _all_entries(self, byorder):
+        # Recurse through the mapping levels of the `byorder` sequence,
+        # reconstructing a flattened sequence of ``(required, provided, name, value)``
+        # tuples that can be used to reconstruct the sequence with the appropriate
+        # registration methods.
+        #
+        # Locally reference the `byorder` data; it might be replaced while
+        # this method is running (see ``rebuild``).
+        for i, components in enumerate(byorder):
+            # We will have *i* levels of dictionaries to go before
+            # we get to the leaf.
+            for key, value in self._allKeys(components, i + 1):
+                assert len(key) == i + 2
+                required = key[:i]
+                provided = key[-2]
+                name = key[-1]
+                yield (required, provided, name, value)
+
+    def allRegistrations(self):
+        """
+        Yields tuples ``(required, provided, name, value)`` for all
+        the registrations that this object holds.
+
+        These tuples could be passed as the arguments to the
+        :meth:`register` method on another adapter registry to
+        duplicate the registrations this object holds.
+
+        .. versionadded:: 5.3.0
+        """
+        yield from self._all_entries(self._adapters)
+
+    def unregister(self, required, provided, name, value=None):
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        order = len(required)
+        byorder = self._adapters
+        if order >= len(byorder):
+            return False
+        components = byorder[order]
+        key = required + (provided,)
+
+        # Keep track of how we got to `components`:
+        lookups = []
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                return
+            lookups.append((components, k))
+            components = d
+
+        old = components.get(name)
+        if old is None:
+            return
+        if (value is not None) and (old is not value):
+            return
+
+        del components[name]
+        if not components:
+            # Clean out empty containers, since we don't want our keys
+            # to reference global objects (interfaces) unnecessarily.
+            # This is often a problem when an interface is slated for
+            # removal; a hold-over entry in the registry can make it
+            # difficult to remove such interfaces.
+            for comp, k in reversed(lookups):
+                d = comp[k]
+                if d:
+                    break
+                else:
+                    del comp[k]
+            while byorder and not byorder[-1]:
+                del byorder[-1]
+        n = self._provided[provided] - 1
+        if n == 0:
+            del self._provided[provided]
+            self._v_lookup.remove_extendor(provided)
+        else:
+            self._provided[provided] = n
+
+        self.changed(self)
+
+    def subscribe(self, required, provided, value):
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        name = ''
+        order = len(required)
+        byorder = self._subscribers
+        while len(byorder) <= order:
+            byorder.append(self._mappingType())
+        components = byorder[order]
+        key = required + (provided,)
+
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                d = self._mappingType()
+                components[k] = d
+            components = d
+
+        components[name] = self._addValueToLeaf(components.get(name), value)
+
+        if provided is not None:
+            n = self._provided.get(provided, 0) + 1
+            self._provided[provided] = n
+            if n == 1:
+                self._v_lookup.add_extendor(provided)
+
+        self.changed(self)
+
+    def subscribed(self, required, provided, subscriber):
+        subscribers = self._find_leaf(
+            self._subscribers,
+            required,
+            provided,
+            ''
+        ) or ()
+        return subscriber if subscriber in subscribers else None
+
+    def allSubscriptions(self):
+        """
+        Yields tuples ``(required, provided, value)`` for all the
+        subscribers that this object holds.
+
+        These tuples could be passed as the arguments to the
+        :meth:`subscribe` method on another adapter registry to
+        duplicate the registrations this object holds.
+
+        .. versionadded:: 5.3.0
+        """
+        for required, provided, _name, value in self._all_entries(self._subscribers):
+            for v in value:
+                yield (required, provided, v)
+
+    def unsubscribe(self, required, provided, value=None):
+        required = tuple([_convert_None_to_Interface(r) for r in required])
+        order = len(required)
+        byorder = self._subscribers
+        if order >= len(byorder):
+            return
+        components = byorder[order]
+        key = required + (provided,)
+
+        # Keep track of how we got to `components`:
+        lookups = []
+        for k in key:
+            d = components.get(k)
+            if d is None:
+                return
+            lookups.append((components, k))
+            components = d
+
+        old = components.get('')
+        if not old:
+            # this is belt-and-suspenders against the failure of cleanup below
+            return  # pragma: no cover
+        len_old = len(old)
+        if value is None:
+            # Removing everything; note that the type of ``new`` won't
+            # necessarily match the ``_leafSequenceType``, but that's
+            # OK because we're about to delete the entire entry
+            # anyway.
+            new = ()
+        else:
+            new = self._removeValueFromLeaf(old, value)
+        # ``new`` may be the same object as ``old``, just mutated in place,
+        # so we cannot compare it to ``old`` to check for changes. Remove
+        # our reference to it now to avoid trying to do so below.
+        del old
+
+        if len(new) == len_old:
+            # No changes, so nothing could have been removed.
+            return
+
+        if new:
+            components[''] = new
+        else:
+            # Instead of setting components[u''] = new, we clean out
+            # empty containers, since we don't want our keys to
+            # reference global objects (interfaces) unnecessarily.  This
+            # is often a problem when an interface is slated for
+            # removal; a hold-over entry in the registry can make it
+            # difficult to remove such interfaces.
+            del components['']
+            for comp, k in reversed(lookups):
+                d = comp[k]
+                if d:
+                    break
+                else:
+                    del comp[k]
+            while byorder and not byorder[-1]:
+                del byorder[-1]
+
+        if provided is not None:
+            n = self._provided[provided] + len(new) - len_old
+            if n == 0:
+                del self._provided[provided]
+                self._v_lookup.remove_extendor(provided)
+            else:
+                self._provided[provided] = n
+
+        self.changed(self)
+
+    def rebuild(self):
+        """
+        Rebuild (and replace) all the internal data structures of this
+        object.
+
+        This is useful, especially for persistent implementations, if
+        you suspect an issue with reference counts keeping interfaces
+        alive even though they are no longer used.
+
+        It is also useful if you or a subclass change the data types
+        (``_mappingType`` and friends) that are to be used.
+
+        This method replaces all internal data structures with new objects;
+        it specifically does not re-use any storage.
+
+        .. versionadded:: 5.3.0
+        """
+
+        # Grab the iterators, we're about to discard their data.
+        registrations = self.allRegistrations()
+        subscriptions = self.allSubscriptions()
+
+        def buffer(it):
+            # The generator doesn't actually start running until we
+            # ask for its next(), by which time the attributes will change
+            # unless we do so before calling __init__.
+            try:
+                first = next(it)
+            except StopIteration:
+                return iter(())
+
+            return itertools.chain((first,), it)
+
+        registrations = buffer(registrations)
+        subscriptions = buffer(subscriptions)
+
+
+        # Replace the base data structures as well as _v_lookup.
+        self.__init__(self.__bases__)
+        # Re-register everything previously registered and subscribed.
+        #
+        # XXX: This is going to call ``self.changed()`` a lot, all of
+        # which is unnecessary (because ``self.__init__`` just
+        # re-created those dependent objects and also called
+        # ``self.changed()``). Is this a bottleneck that needs fixed?
+        # (We could do ``self.changed = lambda _: None`` before
+        # beginning and remove it after to disable the presumably expensive
+        # part of passing that notification to the change of objects.)
+        for args in registrations:
+            self.register(*args)
+        for args in subscriptions:
+            self.subscribe(*args)
+
+    # XXX hack to fake out twisted's use of a private api.  We need to get them
+    # to use the new registered method.
+    def get(self, _): # pragma: no cover
+        class XXXTwistedFakeOut:
+            selfImplied = {}
+        return XXXTwistedFakeOut
+
+
+_not_in_mapping = object()
+
+@_use_c_impl
+class LookupBase:
+
+    def __init__(self):
+        self._cache = {}
+        self._mcache = {}
+        self._scache = {}
+
+    def changed(self, ignored=None):
+        self._cache.clear()
+        self._mcache.clear()
+        self._scache.clear()
+
+    def _getcache(self, provided, name):
+        cache = self._cache.get(provided)
+        if cache is None:
+            cache = {}
+            self._cache[provided] = cache
+        if name:
+            c = cache.get(name)
+            if c is None:
+                c = {}
+                cache[name] = c
+            cache = c
+        return cache
+
+    def lookup(self, required, provided, name='', default=None):
+        if not isinstance(name, str):
+            raise ValueError('name is not a string')
+        cache = self._getcache(provided, name)
+        required = tuple(required)
+        if len(required) == 1:
+            result = cache.get(required[0], _not_in_mapping)
+        else:
+            result = cache.get(tuple(required), _not_in_mapping)
+
+        if result is _not_in_mapping:
+            result = self._uncached_lookup(required, provided, name)
+            if len(required) == 1:
+                cache[required[0]] = result
+            else:
+                cache[tuple(required)] = result
+
+        if result is None:
+            return default
+
+        return result
+
+    def lookup1(self, required, provided, name='', default=None):
+        if not isinstance(name, str):
+            raise ValueError('name is not a string')
+        cache = self._getcache(provided, name)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            return self.lookup((required, ), provided, name, default)
+
+        if result is None:
+            return default
+
+        return result
+
+    def queryAdapter(self, object, provided, name='', default=None):
+        return self.adapter_hook(provided, object, name, default)
+
+    def adapter_hook(self, provided, object, name='', default=None):
+        if not isinstance(name, str):
+            raise ValueError('name is not a string')
+        required = providedBy(object)
+        cache = self._getcache(provided, name)
+        factory = cache.get(required, _not_in_mapping)
+        if factory is _not_in_mapping:
+            factory = self.lookup((required, ), provided, name)
+
+        if factory is not None:
+            if isinstance(object, super):
+                object = object.__self__
+            result = factory(object)
+            if result is not None:
+                return result
+
+        return default
+
+    def lookupAll(self, required, provided):
+        cache = self._mcache.get(provided)
+        if cache is None:
+            cache = {}
+            self._mcache[provided] = cache
+
+        required = tuple(required)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            result = self._uncached_lookupAll(required, provided)
+            cache[required] = result
+
+        return result
+
+
+    def subscriptions(self, required, provided):
+        cache = self._scache.get(provided)
+        if cache is None:
+            cache = {}
+            self._scache[provided] = cache
+
+        required = tuple(required)
+        result = cache.get(required, _not_in_mapping)
+        if result is _not_in_mapping:
+            result = self._uncached_subscriptions(required, provided)
+            cache[required] = result
+
+        return result
+
+
+@_use_c_impl
+class VerifyingBase(LookupBaseFallback):
+    # Mixin for lookups against registries which "chain" upwards, and
+    # whose lookups invalidate their own caches whenever a parent registry
+    # bumps its own '_generation' counter.  E.g., used by
+    # zope.component.persistentregistry
+
+    def changed(self, originally_changed):
+        LookupBaseFallback.changed(self, originally_changed)
+        self._verify_ro = self._registry.ro[1:]
+        self._verify_generations = [r._generation for r in self._verify_ro]
+
+    def _verify(self):
+        if ([r._generation for r in self._verify_ro]
+            != self._verify_generations):
+            self.changed(None)
+
+    def _getcache(self, provided, name):
+        self._verify()
+        return LookupBaseFallback._getcache(self, provided, name)
+
+    def lookupAll(self, required, provided):
+        self._verify()
+        return LookupBaseFallback.lookupAll(self, required, provided)
+
+    def subscriptions(self, required, provided):
+        self._verify()
+        return LookupBaseFallback.subscriptions(self, required, provided)
+
+
+class AdapterLookupBase:
+
+    def __init__(self, registry):
+        self._registry = registry
+        self._required = {}
+        self.init_extendors()
+        super().__init__()
+
+    def changed(self, ignored=None):
+        super().changed(None)
+        for r in self._required.keys():
+            r = r()
+            if r is not None:
+                r.unsubscribe(self)
+        self._required.clear()
+
+
+    # Extendors
+    # ---------
+
+    # When given an target interface for an adapter lookup, we need to consider
+    # adapters for interfaces that extend the target interface.  This is
+    # what the extendors dictionary is about.  It tells us all of the
+    # interfaces that extend an interface for which there are adapters
+    # registered.
+
+    # We could separate this by order and name, thus reducing the
+    # number of provided interfaces to search at run time.  The tradeoff,
+    # however, is that we have to store more information.  For example,
+    # if the same interface is provided for multiple names and if the
+    # interface extends many interfaces, we'll have to keep track of
+    # a fair bit of information for each name.  It's better to
+    # be space efficient here and be time efficient in the cache
+    # implementation.
+
+    # TODO: add invalidation when a provided interface changes, in case
+    # the interface's __iro__ has changed.  This is unlikely enough that
+    # we'll take our chances for now.
+
+    def init_extendors(self):
+        self._extendors = {}
+        for p in self._registry._provided:
+            self.add_extendor(p)
+
+    def add_extendor(self, provided):
+        _extendors = self._extendors
+        for i in provided.__iro__:
+            extendors = _extendors.get(i, ())
+            _extendors[i] = (
+                [e for e in extendors if provided.isOrExtends(e)]
+                +
+                [provided]
+                +
+                [e for e in extendors if not provided.isOrExtends(e)]
+                )
+
+    def remove_extendor(self, provided):
+        _extendors = self._extendors
+        for i in provided.__iro__:
+            _extendors[i] = [e for e in _extendors.get(i, ())
+                             if e != provided]
+
+
+    def _subscribe(self, *required):
+        _refs = self._required
+        for r in required:
+            ref = r.weakref()
+            if ref not in _refs:
+                r.subscribe(self)
+                _refs[ref] = 1
+
+    def _uncached_lookup(self, required, provided, name=''):
+        required = tuple(required)
+        result = None
+        order = len(required)
+        for registry in self._registry.ro:
+            byorder = registry._adapters
+            if order >= len(byorder):
+                continue
+
+            extendors = registry._v_lookup._extendors.get(provided)
+            if not extendors:
+                continue
+
+            components = byorder[order]
+            result = _lookup(components, required, extendors, name, 0,
+                             order)
+            if result is not None:
+                break
+
+        self._subscribe(*required)
+
+        return result
+
+    def queryMultiAdapter(self, objects, provided, name='', default=None):
+        factory = self.lookup([providedBy(o) for o in objects], provided, name)
+        if factory is None:
+            return default
+
+        result = factory(*[o.__self__ if isinstance(o, super) else o for o in objects])
+        if result is None:
+            return default
+
+        return result
+
+    def _uncached_lookupAll(self, required, provided):
+        required = tuple(required)
+        order = len(required)
+        result = {}
+        for registry in reversed(self._registry.ro):
+            byorder = registry._adapters
+            if order >= len(byorder):
+                continue
+            extendors = registry._v_lookup._extendors.get(provided)
+            if not extendors:
+                continue
+            components = byorder[order]
+            _lookupAll(components, required, extendors, result, 0, order)
+
+        self._subscribe(*required)
+
+        return tuple(result.items())
+
+    def names(self, required, provided):
+        return [c[0] for c in self.lookupAll(required, provided)]
+
+    def _uncached_subscriptions(self, required, provided):
+        required = tuple(required)
+        order = len(required)
+        result = []
+        for registry in reversed(self._registry.ro):
+            byorder = registry._subscribers
+            if order >= len(byorder):
+                continue
+
+            if provided is None:
+                extendors = (provided, )
+            else:
+                extendors = registry._v_lookup._extendors.get(provided)
+                if extendors is None:
+                    continue
+
+            _subscriptions(byorder[order], required, extendors, '',
+                           result, 0, order)
+
+        self._subscribe(*required)
+
+        return result
+
+    def subscribers(self, objects, provided):
+        subscriptions = self.subscriptions([providedBy(o) for o in objects], provided)
+        if provided is None:
+            result = ()
+            for subscription in subscriptions:
+                subscription(*objects)
+        else:
+            result = []
+            for subscription in subscriptions:
+                subscriber = subscription(*objects)
+                if subscriber is not None:
+                    result.append(subscriber)
+        return result
+
+class AdapterLookup(AdapterLookupBase, LookupBase):
+    pass
+
+@implementer(IAdapterRegistry)
+class AdapterRegistry(BaseAdapterRegistry):
+    """
+    A full implementation of ``IAdapterRegistry`` that adds support for
+    sub-registries.
+    """
+
+    LookupClass = AdapterLookup
+
+    def __init__(self, bases=()):
+        # AdapterRegisties are invalidating registries, so
+        # we need to keep track of our invalidating subregistries.
+        self._v_subregistries = weakref.WeakKeyDictionary()
+
+        super().__init__(bases)
+
+    def _addSubregistry(self, r):
+        self._v_subregistries[r] = 1
+
+    def _removeSubregistry(self, r):
+        if r in self._v_subregistries:
+            del self._v_subregistries[r]
+
+    def _setBases(self, bases):
+        old = self.__dict__.get('__bases__', ())
+        for r in old:
+            if r not in bases:
+                r._removeSubregistry(self)
+        for r in bases:
+            if r not in old:
+                r._addSubregistry(self)
+
+        super()._setBases(bases)
+
+    def changed(self, originally_changed):
+        super().changed(originally_changed)
+
+        for sub in self._v_subregistries.keys():
+            sub.changed(originally_changed)
+
+
+class VerifyingAdapterLookup(AdapterLookupBase, VerifyingBase):
+    pass
+
+@implementer(IAdapterRegistry)
+class VerifyingAdapterRegistry(BaseAdapterRegistry):
+    """
+    The most commonly-used adapter registry.
+    """
+
+    LookupClass = VerifyingAdapterLookup
+
+def _convert_None_to_Interface(x):
+    if x is None:
+        return Interface
+    else:
+        return x
+
+def _lookup(components, specs, provided, name, i, l):
+    # this function is called very often.
+    # The components.get in loops is executed 100 of 1000s times.
+    # by loading get into a local variable the bytecode
+    # "LOAD_FAST 0 (components)" in the loop can be eliminated.
+    components_get = components.get
+    if i < l:
+        for spec in specs[i].__sro__:
+            comps = components_get(spec)
+            if comps:
+                r = _lookup(comps, specs, provided, name, i+1, l)
+                if r is not None:
+                    return r
+    else:
+        for iface in provided:
+            comps = components_get(iface)
+            if comps:
+                r = comps.get(name)
+                if r is not None:
+                    return r
+
+    return None
+
+def _lookupAll(components, specs, provided, result, i, l):
+    components_get = components.get  # see _lookup above
+    if i < l:
+        for spec in reversed(specs[i].__sro__):
+            comps = components_get(spec)
+            if comps:
+                _lookupAll(comps, specs, provided, result, i+1, l)
+    else:
+        for iface in reversed(provided):
+            comps = components_get(iface)
+            if comps:
+                result.update(comps)
+
+def _subscriptions(components, specs, provided, name, result, i, l):
+    components_get = components.get  # see _lookup above
+    if i < l:
+        for spec in reversed(specs[i].__sro__):
+            comps = components_get(spec)
+            if comps:
+                _subscriptions(comps, specs, provided, name, result, i+1, l)
+    else:
+        for iface in reversed(provided):
+            comps = components_get(iface)
+            if comps:
+                comps = comps.get(name)
+                if comps:
+                    result.extend(comps)
diff --git a/contrib/python/zope.interface/py3/zope/interface/advice.py b/contrib/python/zope.interface/py3/zope/interface/advice.py
new file mode 100644
index 00000000000..54e356e672b
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/advice.py
@@ -0,0 +1,118 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Class advice.
+
+This module was adapted from 'protocols.advice', part of the Python
+Enterprise Application Kit (PEAK).  Please notify the PEAK authors
+(pje@telecommunity.com and tsarna@sarna.org) if bugs are found or
+Zope-specific changes are required, so that the PEAK version of this module
+can be kept in sync.
+
+PEAK is a Python application framework that interoperates with (but does
+not require) Zope 3 and Twisted.  It provides tools for manipulating UML
+models, object-relational persistence, aspect-oriented programming, and more.
+Visit the PEAK home page at http://peak.telecommunity.com for more information.
+"""
+
+from types import FunctionType
+
+__all__ = [
+    'determineMetaclass',
+    'getFrameInfo',
+    'isClassAdvisor',
+    'minimalBases',
+]
+
+import sys
+
+def getFrameInfo(frame):
+    """Return (kind,module,locals,globals) for a frame
+
+    'kind' is one of "exec", "module", "class", "function call", or "unknown".
+    """
+
+    f_locals = frame.f_locals
+    f_globals = frame.f_globals
+
+    sameNamespace = f_locals is f_globals
+    hasModule = '__module__' in f_locals
+    hasName = '__name__' in f_globals
+
+    sameName = hasModule and hasName
+    sameName = sameName and f_globals['__name__']==f_locals['__module__']
+
+    module = hasName and sys.modules.get(f_globals['__name__']) or None
+
+    namespaceIsModule = module and module.__dict__ is f_globals
+
+    if not namespaceIsModule:
+        # some kind of funky exec
+        kind = "exec"
+    elif sameNamespace and not hasModule:
+        kind = "module"
+    elif sameName and not sameNamespace:
+        kind = "class"
+    elif not sameNamespace:
+        kind = "function call"
+    else: # pragma: no cover
+        # How can you have f_locals is f_globals, and have '__module__' set?
+        # This is probably module-level code, but with a '__module__' variable.
+        kind = "unknown"
+    return kind, module, f_locals, f_globals
+
+
+def isClassAdvisor(ob):
+    """True if 'ob' is a class advisor function"""
+    return isinstance(ob,FunctionType) and hasattr(ob,'previousMetaclass')
+
+
+def determineMetaclass(bases, explicit_mc=None):
+    """Determine metaclass from 1+ bases and optional explicit __metaclass__"""
+
+    meta = [getattr(b,'__class__',type(b)) for b in bases]
+
+    if explicit_mc is not None:
+        # The explicit metaclass needs to be verified for compatibility
+        # as well, and allowed to resolve the incompatible bases, if any
+        meta.append(explicit_mc)
+
+    if len(meta)==1:
+        # easy case
+        return meta[0]
+
+    candidates = minimalBases(meta) # minimal set of metaclasses
+
+    if len(candidates)>1:
+        # We could auto-combine, but for now we won't...
+        raise TypeError("Incompatible metatypes", bases)
+
+    # Just one, return it
+    return candidates[0]
+
+
+def minimalBases(classes):
+    """Reduce a list of base classes to its ordered minimum equivalent"""
+    candidates = []
+
+    for m in classes:
+        for n in classes:
+            if issubclass(n,m) and m is not n:
+                break
+        else:
+            # m has no subclasses in 'classes'
+            if m in candidates:
+                candidates.remove(m)    # ensure that we're later in the list
+            candidates.append(m)
+
+    return candidates
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/__init__.py b/contrib/python/zope.interface/py3/zope/interface/common/__init__.py
new file mode 100644
index 00000000000..56f4566a24e
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/__init__.py
@@ -0,0 +1,272 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+
+import itertools
+from types import FunctionType
+
+from zope.interface import classImplements
+from zope.interface import Interface
+from zope.interface.interface import fromFunction
+from zope.interface.interface import InterfaceClass
+from zope.interface.interface import _decorator_non_return
+
+__all__ = [
+    # Nothing public here.
+]
+
+
+# pylint:disable=inherit-non-class,
+# pylint:disable=no-self-argument,no-method-argument
+# pylint:disable=unexpected-special-method-signature
+
+class optional:
+    # Apply this decorator to a method definition to make it
+    # optional (remove it from the list of required names), overriding
+    # the definition inherited from the ABC.
+    def __init__(self, method):
+        self.__doc__ = method.__doc__
+
+
+class ABCInterfaceClass(InterfaceClass):
+    """
+    An interface that is automatically derived from a
+    :class:`abc.ABCMeta` type.
+
+    Internal use only.
+
+    The body of the interface definition *must* define
+    a property ``abc`` that is the ABC to base the interface on.
+
+    If ``abc`` is *not* in the interface definition, a regular
+    interface will be defined instead (but ``extra_classes`` is still
+    respected).
+
+    Use the ``@optional`` decorator on method definitions if
+    the ABC defines methods that are not actually required in all cases
+    because the Python language has multiple ways to implement a protocol.
+    For example, the ``iter()`` protocol can be implemented with
+    ``__iter__`` or the pair ``__len__`` and ``__getitem__``.
+
+    When created, any existing classes that are registered to conform
+    to the ABC are declared to implement this interface. This is *not*
+    automatically updated as the ABC registry changes. If the body of the
+    interface definition defines ``extra_classes``, it should be a
+    tuple giving additional classes to declare implement the interface.
+
+    Note that this is not fully symmetric. For example, it is usually
+    the case that a subclass relationship carries the interface
+    declarations over::
+
+        >>> from zope.interface import Interface
+        >>> class I1(Interface):
+        ...     pass
+        ...
+        >>> from zope.interface import implementer
+        >>> @implementer(I1)
+        ... class Root(object):
+        ...     pass
+        ...
+        >>> class Child(Root):
+        ...     pass
+        ...
+        >>> child = Child()
+        >>> isinstance(child, Root)
+        True
+        >>> from zope.interface import providedBy
+        >>> list(providedBy(child))
+        [<InterfaceClass __main__.I1>]
+
+    However, that's not the case with ABCs and ABC interfaces. Just
+    because ``isinstance(A(), AnABC)`` and ``isinstance(B(), AnABC)``
+    are both true, that doesn't mean there's any class hierarchy
+    relationship between ``A`` and ``B``, or between either of them
+    and ``AnABC``. Thus, if ``AnABC`` implemented ``IAnABC``, it would
+    not follow that either ``A`` or ``B`` implements ``IAnABC`` (nor
+    their instances provide it)::
+
+        >>> class SizedClass(object):
+        ...     def __len__(self): return 1
+        ...
+        >>> from collections.abc import Sized
+        >>> isinstance(SizedClass(), Sized)
+        True
+        >>> from zope.interface import classImplements
+        >>> classImplements(Sized, I1)
+        None
+        >>> list(providedBy(SizedClass()))
+        []
+
+    Thus, to avoid conflicting assumptions, ABCs should not be
+    declared to implement their parallel ABC interface. Only concrete
+    classes specifically registered with the ABC should be declared to
+    do so.
+
+    .. versionadded:: 5.0.0
+    """
+
+    # If we could figure out invalidation, and used some special
+    # Specification/Declaration instances, and override the method ``providedBy`` here,
+    # perhaps we could more closely integrate with ABC virtual inheritance?
+
+    def __init__(self, name, bases, attrs):
+        # go ahead and give us a name to ease debugging.
+        self.__name__ = name
+        extra_classes = attrs.pop('extra_classes', ())
+        ignored_classes = attrs.pop('ignored_classes', ())
+
+        if 'abc' not in attrs:
+            # Something like ``IList(ISequence)``: We're extending
+            # abc interfaces but not an ABC interface ourself.
+            InterfaceClass.__init__(self, name, bases, attrs)
+            ABCInterfaceClass.__register_classes(self, extra_classes, ignored_classes)
+            self.__class__ = InterfaceClass
+            return
+
+        based_on = attrs.pop('abc')
+        self.__abc = based_on
+        self.__extra_classes = tuple(extra_classes)
+        self.__ignored_classes = tuple(ignored_classes)
+
+        assert name[1:] == based_on.__name__, (name, based_on)
+        methods = {
+            # Passing the name is important in case of aliases,
+            # e.g., ``__ror__ = __or__``.
+            k: self.__method_from_function(v, k)
+            for k, v in vars(based_on).items()
+            if isinstance(v, FunctionType) and not self.__is_private_name(k)
+            and not self.__is_reverse_protocol_name(k)
+        }
+
+        methods['__doc__'] = self.__create_class_doc(attrs)
+        # Anything specified in the body takes precedence.
+        methods.update(attrs)
+        InterfaceClass.__init__(self, name, bases, methods)
+        self.__register_classes()
+
+    @staticmethod
+    def __optional_methods_to_docs(attrs):
+        optionals = {k: v for k, v in attrs.items() if isinstance(v, optional)}
+        for k in optionals:
+            attrs[k] = _decorator_non_return
+
+        if not optionals:
+            return ''
+
+        docs = "\n\nThe following methods are optional:\n - " + "\n-".join(
+            "{}\n{}".format(k, v.__doc__) for k, v in optionals.items()
+        )
+        return docs
+
+    def __create_class_doc(self, attrs):
+        based_on = self.__abc
+        def ref(c):
+            mod = c.__module__
+            name = c.__name__
+            if mod == str.__module__:
+                return "`%s`" % name
+            if mod == '_io':
+                mod = 'io'
+            return "`{}.{}`".format(mod, name)
+        implementations_doc = "\n - ".join(
+            ref(c)
+            for c in sorted(self.getRegisteredConformers(), key=ref)
+        )
+        if implementations_doc:
+            implementations_doc = "\n\nKnown implementations are:\n\n - " + implementations_doc
+
+        based_on_doc = (based_on.__doc__ or '')
+        based_on_doc = based_on_doc.splitlines()
+        based_on_doc = based_on_doc[0] if based_on_doc else ''
+
+        doc = """Interface for the ABC `{}.{}`.\n\n{}{}{}""".format(
+            based_on.__module__, based_on.__name__,
+            attrs.get('__doc__', based_on_doc),
+            self.__optional_methods_to_docs(attrs),
+            implementations_doc
+        )
+        return doc
+
+
+    @staticmethod
+    def __is_private_name(name):
+        if name.startswith('__') and name.endswith('__'):
+            return False
+        return name.startswith('_')
+
+    @staticmethod
+    def __is_reverse_protocol_name(name):
+        # The reverse names, like __rand__,
+        # aren't really part of the protocol. The interpreter has
+        # very complex behaviour around invoking those. PyPy
+        # doesn't always even expose them as attributes.
+        return name.startswith('__r') and name.endswith('__')
+
+    def __method_from_function(self, function, name):
+        method = fromFunction(function, self, name=name)
+        # Eliminate the leading *self*, which is implied in
+        # an interface, but explicit in an ABC.
+        method.positional = method.positional[1:]
+        return method
+
+    def __register_classes(self, conformers=None, ignored_classes=None):
+        # Make the concrete classes already present in our ABC's registry
+        # declare that they implement this interface.
+        conformers = conformers if conformers is not None else self.getRegisteredConformers()
+        ignored = ignored_classes if ignored_classes is not None else self.__ignored_classes
+        for cls in conformers:
+            if cls in ignored:
+                continue
+            classImplements(cls, self)
+
+    def getABC(self):
+        """
+        Return the ABC this interface represents.
+        """
+        return self.__abc
+
+    def getRegisteredConformers(self):
+        """
+        Return an iterable of the classes that are known to conform to
+        the ABC this interface parallels.
+        """
+        based_on = self.__abc
+
+        # The registry only contains things that aren't already
+        # known to be subclasses of the ABC. But the ABC is in charge
+        # of checking that, so its quite possible that registrations
+        # are in fact ignored, winding up just in the _abc_cache.
+        try:
+            registered = list(based_on._abc_registry) + list(based_on._abc_cache)
+        except AttributeError:
+            # Rewritten in C in CPython 3.7.
+            # These expose the underlying weakref.
+            from abc import _get_dump
+            data = _get_dump(based_on)
+            registry = data[0]
+            cache = data[1]
+            registered = [x() for x in itertools.chain(registry, cache)]
+            registered = [x for x in registered if x is not None]
+
+        return set(itertools.chain(registered, self.__extra_classes))
+
+
+def _create_ABCInterface():
+    # It's a two-step process to create the root ABCInterface, because
+    # without specifying a corresponding ABC, using the normal constructor
+    # gets us a plain InterfaceClass object, and there is no ABC to associate with the
+    # root.
+    abc_name_bases_attrs = ('ABCInterface', (Interface,), {})
+    instance = ABCInterfaceClass.__new__(ABCInterfaceClass, *abc_name_bases_attrs)
+    InterfaceClass.__init__(instance, *abc_name_bases_attrs)
+    return instance
+
+ABCInterface = _create_ABCInterface()
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/builtins.py b/contrib/python/zope.interface/py3/zope/interface/common/builtins.py
new file mode 100644
index 00000000000..17090e4a791
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/builtins.py
@@ -0,0 +1,119 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""
+Interface definitions for builtin types.
+
+After this module is imported, the standard library types will declare
+that they implement the appropriate interface.
+
+.. versionadded:: 5.0.0
+"""
+
+from zope.interface import classImplements
+
+from zope.interface.common import collections
+from zope.interface.common import numbers
+from zope.interface.common import io
+
+__all__ = [
+    'IList',
+    'ITuple',
+    'ITextString',
+    'IByteString',
+    'INativeString',
+    'IBool',
+    'IDict',
+    'IFile',
+]
+
+# pylint:disable=no-self-argument
+class IList(collections.IMutableSequence):
+    """
+    Interface for :class:`list`
+    """
+    extra_classes = (list,)
+
+    def sort(key=None, reverse=False):
+        """
+        Sort the list in place and return None.
+
+        *key* and *reverse* must be passed by name only.
+        """
+
+
+class ITuple(collections.ISequence):
+    """
+    Interface for :class:`tuple`
+    """
+    extra_classes = (tuple,)
+
+
+class ITextString(collections.ISequence):
+    """
+    Interface for text ("unicode") strings.
+
+    This is :class:`str`
+    """
+    extra_classes = (str,)
+
+
+class IByteString(collections.IByteString):
+    """
+    Interface for immutable byte strings.
+
+    On all Python versions this is :class:`bytes`.
+
+    Unlike :class:`zope.interface.common.collections.IByteString`
+    (the parent of this interface) this does *not* include
+    :class:`bytearray`.
+    """
+    extra_classes = (bytes,)
+
+
+class INativeString(ITextString):
+    """
+    Interface for native strings.
+
+    On all Python versions, this is :class:`str`. Tt extends
+    :class:`ITextString`.
+    """
+# We're not extending ABCInterface so extra_classes won't work
+classImplements(str, INativeString)
+
+
+class IBool(numbers.IIntegral):
+    """
+    Interface for :class:`bool`
+    """
+    extra_classes = (bool,)
+
+
+class IDict(collections.IMutableMapping):
+    """
+    Interface for :class:`dict`
+    """
+    extra_classes = (dict,)
+
+
+class IFile(io.IIOBase):
+    """
+    Interface for :class:`file`.
+
+    It is recommended to use the interfaces from :mod:`zope.interface.common.io`
+    instead of this interface.
+
+    On Python 3, there is no single implementation of this interface;
+    depending on the arguments, the :func:`open` builtin can return
+    many different classes that implement different interfaces from
+    :mod:`zope.interface.common.io`.
+    """
+    extra_classes = ()
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/collections.py b/contrib/python/zope.interface/py3/zope/interface/common/collections.py
new file mode 100644
index 00000000000..c5490282688
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/collections.py
@@ -0,0 +1,253 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""
+Interface definitions paralleling the abstract base classes defined in
+:mod:`collections.abc`.
+
+After this module is imported, the standard library types will declare
+that they implement the appropriate interface. While most standard
+library types will properly implement that interface (that
+is, ``verifyObject(ISequence, list()))`` will pass, for example), a few might not:
+
+    - `memoryview` doesn't feature all the defined methods of
+      ``ISequence`` such as ``count``; it is still declared to provide
+      ``ISequence`` though.
+
+    - `collections.deque.pop` doesn't accept the ``index`` argument of
+      `collections.abc.MutableSequence.pop`
+
+    - `range.index` does not accept the ``start`` and ``stop`` arguments.
+
+.. versionadded:: 5.0.0
+"""
+
+import sys
+
+from abc import ABCMeta
+from collections import abc
+from collections import OrderedDict
+from collections import UserList
+from collections import UserDict
+from collections import UserString
+
+from zope.interface.common import ABCInterface
+from zope.interface.common import optional
+
+# pylint:disable=inherit-non-class,
+# pylint:disable=no-self-argument,no-method-argument
+# pylint:disable=unexpected-special-method-signature
+# pylint:disable=no-value-for-parameter
+
+
+def _new_in_ver(name, ver,
+                bases_if_missing=(ABCMeta,),
+                register_if_missing=()):
+    if ver:
+        return getattr(abc, name)
+
+    # TODO: It's a shame to have to repeat the bases when
+    # the ABC is missing. Can we DRY that?
+    missing = ABCMeta(name, bases_if_missing, {
+        '__doc__': "The ABC %s is not defined in this version of Python." % (
+            name
+        ),
+    })
+
+    for c in register_if_missing:
+        missing.register(c)
+
+    return missing
+
+__all__ = [
+    'IAsyncGenerator',
+    'IAsyncIterable',
+    'IAsyncIterator',
+    'IAwaitable',
+    'ICollection',
+    'IContainer',
+    'ICoroutine',
+    'IGenerator',
+    'IHashable',
+    'IItemsView',
+    'IIterable',
+    'IIterator',
+    'IKeysView',
+    'IMapping',
+    'IMappingView',
+    'IMutableMapping',
+    'IMutableSequence',
+    'IMutableSet',
+    'IReversible',
+    'ISequence',
+    'ISet',
+    'ISized',
+    'IValuesView',
+]
+
+class IContainer(ABCInterface):
+    abc = abc.Container
+
+    @optional
+    def __contains__(other):
+        """
+        Optional method. If not provided, the interpreter will use
+        ``__iter__`` or the old ``__getitem__`` protocol
+        to implement ``in``.
+        """
+
+class IHashable(ABCInterface):
+    abc = abc.Hashable
+
+class IIterable(ABCInterface):
+    abc = abc.Iterable
+
+    @optional
+    def __iter__():
+        """
+        Optional method. If not provided, the interpreter will
+        implement `iter` using the old ``__getitem__`` protocol.
+        """
+
+class IIterator(IIterable):
+    abc = abc.Iterator
+
+class IReversible(IIterable):
+    abc = _new_in_ver('Reversible', True, (IIterable.getABC(),))
+
+    @optional
+    def __reversed__():
+        """
+        Optional method. If this isn't present, the interpreter
+        will use ``__len__`` and ``__getitem__`` to implement the
+        `reversed` builtin.
+        """
+
+class IGenerator(IIterator):
+    # New in Python 3.5
+    abc = _new_in_ver('Generator', True, (IIterator.getABC(),))
+
+
+class ISized(ABCInterface):
+    abc = abc.Sized
+
+
+# ICallable is not defined because there's no standard signature.
+
+class ICollection(ISized,
+                  IIterable,
+                  IContainer):
+    abc = _new_in_ver('Collection', True,
+                      (ISized.getABC(), IIterable.getABC(), IContainer.getABC()))
+
+
+class ISequence(IReversible,
+                ICollection):
+    abc = abc.Sequence
+    extra_classes = (UserString,)
+    # On Python 2, basestring is registered as an ISequence, and
+    # its subclass str is an IByteString. If we also register str as
+    # an ISequence, that tends to lead to inconsistent resolution order.
+    ignored_classes = (basestring,) if str is bytes else () # pylint:disable=undefined-variable
+
+    @optional
+    def __reversed__():
+        """
+        Optional method. If this isn't present, the interpreter
+        will use ``__len__`` and ``__getitem__`` to implement the
+        `reversed` builtin.
+        """
+
+    @optional
+    def __iter__():
+        """
+        Optional method. If not provided, the interpreter will
+        implement `iter` using the old ``__getitem__`` protocol.
+        """
+
+class IMutableSequence(ISequence):
+    abc = abc.MutableSequence
+    extra_classes = (UserList,)
+
+
+class IByteString(ISequence):
+    """
+    This unifies `bytes` and `bytearray`.
+    """
+    abc = _new_in_ver('ByteString', True,
+                      (ISequence.getABC(),),
+                      (bytes, bytearray))
+
+
+class ISet(ICollection):
+    abc = abc.Set
+
+
+class IMutableSet(ISet):
+    abc = abc.MutableSet
+
+
+class IMapping(ICollection):
+    abc = abc.Mapping
+    extra_classes = (dict,)
+    # OrderedDict is a subclass of dict. On CPython 2,
+    # it winds up registered as a IMutableMapping, which
+    # produces an inconsistent IRO if we also try to register it
+    # here.
+    ignored_classes = (OrderedDict,)
+
+
+class IMutableMapping(IMapping):
+    abc = abc.MutableMapping
+    extra_classes = (dict, UserDict,)
+    ignored_classes = (OrderedDict,)
+
+class IMappingView(ISized):
+    abc = abc.MappingView
+
+
+class IItemsView(IMappingView, ISet):
+    abc = abc.ItemsView
+
+
+class IKeysView(IMappingView, ISet):
+    abc = abc.KeysView
+
+
+class IValuesView(IMappingView, ICollection):
+    abc = abc.ValuesView
+
+    @optional
+    def __contains__(other):
+        """
+        Optional method. If not provided, the interpreter will use
+        ``__iter__`` or the old ``__len__`` and ``__getitem__`` protocol
+        to implement ``in``.
+        """
+
+class IAwaitable(ABCInterface):
+    abc = _new_in_ver('Awaitable', True)
+
+
+class ICoroutine(IAwaitable):
+    abc = _new_in_ver('Coroutine', True)
+
+
+class IAsyncIterable(ABCInterface):
+    abc = _new_in_ver('AsyncIterable', True)
+
+
+class IAsyncIterator(IAsyncIterable):
+    abc = _new_in_ver('AsyncIterator', True)
+
+
+class IAsyncGenerator(IAsyncIterator):
+    abc = _new_in_ver('AsyncGenerator', True)
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/idatetime.py b/contrib/python/zope.interface/py3/zope/interface/common/idatetime.py
new file mode 100644
index 00000000000..82f0059c851
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/idatetime.py
@@ -0,0 +1,606 @@
+##############################################################################
+# Copyright (c) 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""Datetime interfaces.
+
+This module is called idatetime because if it were called datetime the import
+of the real datetime would fail.
+"""
+from datetime import timedelta, date, datetime, time, tzinfo
+
+from zope.interface import Interface, Attribute
+from zope.interface import classImplements
+
+
+class ITimeDeltaClass(Interface):
+    """This is the timedelta class interface.
+
+    This is symbolic; this module does **not** make
+    `datetime.timedelta` provide this interface.
+    """
+
+    min = Attribute("The most negative timedelta object")
+
+    max = Attribute("The most positive timedelta object")
+
+    resolution = Attribute(
+        "The smallest difference between non-equal timedelta objects")
+
+
+class ITimeDelta(ITimeDeltaClass):
+    """Represent the difference between two datetime objects.
+
+    Implemented by `datetime.timedelta`.
+
+    Supported operators:
+
+    - add, subtract timedelta
+    - unary plus, minus, abs
+    - compare to timedelta
+    - multiply, divide by int/long
+
+    In addition, `.datetime` supports subtraction of two `.datetime` objects
+    returning a `.timedelta`, and addition or subtraction of a `.datetime`
+    and a `.timedelta` giving a `.datetime`.
+
+    Representation: (days, seconds, microseconds).
+    """
+
+    days = Attribute("Days between -999999999 and 999999999 inclusive")
+
+    seconds = Attribute("Seconds between 0 and 86399 inclusive")
+
+    microseconds = Attribute("Microseconds between 0 and 999999 inclusive")
+
+
+class IDateClass(Interface):
+    """This is the date class interface.
+
+    This is symbolic; this module does **not** make
+    `datetime.date` provide this interface.
+    """
+
+    min = Attribute("The earliest representable date")
+
+    max = Attribute("The latest representable date")
+
+    resolution = Attribute(
+        "The smallest difference between non-equal date objects")
+
+    def today():
+        """Return the current local time.
+
+        This is equivalent to ``date.fromtimestamp(time.time())``"""
+
+    def fromtimestamp(timestamp):
+        """Return the local date from a POSIX timestamp (like time.time())
+
+        This may raise `ValueError`, if the timestamp is out of the range of
+        values supported by the platform C ``localtime()`` function. It's common
+        for this to be restricted to years from 1970 through 2038. Note that
+        on non-POSIX systems that include leap seconds in their notion of a
+        timestamp, leap seconds are ignored by `fromtimestamp`.
+        """
+
+    def fromordinal(ordinal):
+        """Return the date corresponding to the proleptic Gregorian ordinal.
+
+         January 1 of year 1 has ordinal 1. `ValueError` is raised unless
+         1 <= ordinal <= date.max.toordinal().
+
+         For any date *d*, ``date.fromordinal(d.toordinal()) == d``.
+         """
+
+
+class IDate(IDateClass):
+    """Represents a date (year, month and day) in an idealized calendar.
+
+    Implemented by `datetime.date`.
+
+    Operators:
+
+    __repr__, __str__
+    __cmp__, __hash__
+    __add__, __radd__, __sub__ (add/radd only with timedelta arg)
+    """
+
+    year = Attribute("Between MINYEAR and MAXYEAR inclusive.")
+
+    month = Attribute("Between 1 and 12 inclusive")
+
+    day = Attribute(
+        "Between 1 and the number of days in the given month of the given year.")
+
+    def replace(year, month, day):
+        """Return a date with the same value.
+
+        Except for those members given new values by whichever keyword
+        arguments are specified. For example, if ``d == date(2002, 12, 31)``, then
+        ``d.replace(day=26) == date(2000, 12, 26)``.
+        """
+
+    def timetuple():
+        """Return a 9-element tuple of the form returned by `time.localtime`.
+
+        The hours, minutes and seconds are 0, and the DST flag is -1.
+        ``d.timetuple()`` is equivalent to
+        ``(d.year, d.month, d.day, 0, 0, 0, d.weekday(), d.toordinal() -
+        date(d.year, 1, 1).toordinal() + 1, -1)``
+        """
+
+    def toordinal():
+        """Return the proleptic Gregorian ordinal of the date
+
+        January 1 of year 1 has ordinal 1. For any date object *d*,
+        ``date.fromordinal(d.toordinal()) == d``.
+        """
+
+    def weekday():
+        """Return the day of the week as an integer.
+
+        Monday is 0 and Sunday is 6. For example,
+        ``date(2002, 12, 4).weekday() == 2``, a Wednesday.
+
+        .. seealso:: `isoweekday`.
+        """
+
+    def isoweekday():
+        """Return the day of the week as an integer.
+
+        Monday is 1 and Sunday is 7. For example,
+        date(2002, 12, 4).isoweekday() == 3, a Wednesday.
+
+        .. seealso:: `weekday`, `isocalendar`.
+        """
+
+    def isocalendar():
+        """Return a 3-tuple, (ISO year, ISO week number, ISO weekday).
+
+        The ISO calendar is a widely used variant of the Gregorian calendar.
+        See http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm for a good
+        explanation.
+
+        The ISO year consists of 52 or 53 full weeks, and where a week starts
+        on a Monday and ends on a Sunday. The first week of an ISO year is the
+        first (Gregorian) calendar week of a year containing a Thursday. This
+        is called week number 1, and the ISO year of that Thursday is the same
+        as its Gregorian year.
+
+        For example, 2004 begins on a Thursday, so the first week of ISO year
+        2004 begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004, so
+        that ``date(2003, 12, 29).isocalendar() == (2004, 1, 1)`` and
+        ``date(2004, 1, 4).isocalendar() == (2004, 1, 7)``.
+        """
+
+    def isoformat():
+        """Return a string representing the date in ISO 8601 format.
+
+        This is 'YYYY-MM-DD'.
+        For example, ``date(2002, 12, 4).isoformat() == '2002-12-04'``.
+        """
+
+    def __str__():
+        """For a date *d*, ``str(d)`` is equivalent to ``d.isoformat()``."""
+
+    def ctime():
+        """Return a string representing the date.
+
+        For example date(2002, 12, 4).ctime() == 'Wed Dec 4 00:00:00 2002'.
+        d.ctime() is equivalent to time.ctime(time.mktime(d.timetuple()))
+        on platforms where the native C ctime() function
+        (which `time.ctime` invokes, but which date.ctime() does not invoke)
+        conforms to the C standard.
+        """
+
+    def strftime(format):
+        """Return a string representing the date.
+
+        Controlled by an explicit format string. Format codes referring to
+        hours, minutes or seconds will see 0 values.
+        """
+
+
+class IDateTimeClass(Interface):
+    """This is the datetime class interface.
+
+    This is symbolic; this module does **not** make
+    `datetime.datetime` provide this interface.
+    """
+
+    min = Attribute("The earliest representable datetime")
+
+    max = Attribute("The latest representable datetime")
+
+    resolution = Attribute(
+        "The smallest possible difference between non-equal datetime objects")
+
+    def today():
+        """Return the current local datetime, with tzinfo None.
+
+        This is equivalent to ``datetime.fromtimestamp(time.time())``.
+
+        .. seealso:: `now`, `fromtimestamp`.
+        """
+
+    def now(tz=None):
+        """Return the current local date and time.
+
+        If optional argument *tz* is None or not specified, this is like `today`,
+        but, if possible, supplies more precision than can be gotten from going
+        through a `time.time` timestamp (for example, this may be possible on
+        platforms supplying the C ``gettimeofday()`` function).
+
+        Else tz must be an instance of a class tzinfo subclass, and the current
+        date and time are converted to tz's time zone. In this case the result
+        is equivalent to tz.fromutc(datetime.utcnow().replace(tzinfo=tz)).
+
+        .. seealso:: `today`, `utcnow`.
+        """
+
+    def utcnow():
+        """Return the current UTC date and time, with tzinfo None.
+
+        This is like `now`, but returns the current UTC date and time, as a
+        naive datetime object.
+
+        .. seealso:: `now`.
+        """
+
+    def fromtimestamp(timestamp, tz=None):
+        """Return the local date and time corresponding to the POSIX timestamp.
+
+        Same as is returned by time.time(). If optional argument tz is None or
+        not specified, the timestamp is converted to the platform's local date
+        and time, and the returned datetime object is naive.
+
+        Else tz must be an instance of a class tzinfo subclass, and the
+        timestamp is converted to tz's time zone. In this case the result is
+        equivalent to
+        ``tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))``.
+
+        fromtimestamp() may raise `ValueError`, if the timestamp is out of the
+        range of values supported by the platform C localtime() or gmtime()
+        functions. It's common for this to be restricted to years in 1970
+        through 2038. Note that on non-POSIX systems that include leap seconds
+        in their notion of a timestamp, leap seconds are ignored by
+        fromtimestamp(), and then it's possible to have two timestamps
+        differing by a second that yield identical datetime objects.
+
+        .. seealso:: `utcfromtimestamp`.
+        """
+
+    def utcfromtimestamp(timestamp):
+        """Return the UTC datetime from the POSIX timestamp with tzinfo None.
+
+        This may raise `ValueError`, if the timestamp is out of the range of
+        values supported by the platform C ``gmtime()`` function. It's common for
+        this to be restricted to years in 1970 through 2038.
+
+        .. seealso:: `fromtimestamp`.
+        """
+
+    def fromordinal(ordinal):
+        """Return the datetime from the proleptic Gregorian ordinal.
+
+        January 1 of year 1 has ordinal 1. `ValueError` is raised unless
+        1 <= ordinal <= datetime.max.toordinal().
+        The hour, minute, second and microsecond of the result are all 0, and
+        tzinfo is None.
+        """
+
+    def combine(date, time):
+        """Return a new datetime object.
+
+        Its date members are equal to the given date object's, and whose time
+        and tzinfo members are equal to the given time object's. For any
+        datetime object *d*, ``d == datetime.combine(d.date(), d.timetz())``.
+        If date is a datetime object, its time and tzinfo members are ignored.
+        """
+
+
+class IDateTime(IDate, IDateTimeClass):
+    """Object contains all the information from a date object and a time object.
+
+    Implemented by `datetime.datetime`.
+    """
+
+    year = Attribute("Year between MINYEAR and MAXYEAR inclusive")
+
+    month = Attribute("Month between 1 and 12 inclusive")
+
+    day = Attribute(
+        "Day between 1 and the number of days in the given month of the year")
+
+    hour = Attribute("Hour in range(24)")
+
+    minute = Attribute("Minute in range(60)")
+
+    second = Attribute("Second in range(60)")
+
+    microsecond = Attribute("Microsecond in range(1000000)")
+
+    tzinfo = Attribute(
+        """The object passed as the tzinfo argument to the datetime constructor
+        or None if none was passed""")
+
+    def date():
+         """Return date object with same year, month and day."""
+
+    def time():
+        """Return time object with same hour, minute, second, microsecond.
+
+        tzinfo is None.
+
+        .. seealso:: Method :meth:`timetz`.
+        """
+
+    def timetz():
+        """Return time object with same hour, minute, second, microsecond,
+        and tzinfo.
+
+        .. seealso:: Method :meth:`time`.
+        """
+
+    def replace(year, month, day, hour, minute, second, microsecond, tzinfo):
+        """Return a datetime with the same members, except for those members
+        given new values by whichever keyword arguments are specified.
+
+        Note that ``tzinfo=None`` can be specified to create a naive datetime from
+        an aware datetime with no conversion of date and time members.
+        """
+
+    def astimezone(tz):
+        """Return a datetime object with new tzinfo member tz, adjusting the
+        date and time members so the result is the same UTC time as self, but
+        in tz's local time.
+
+        tz must be an instance of a tzinfo subclass, and its utcoffset() and
+        dst() methods must not return None. self must be aware (self.tzinfo
+        must not be None, and self.utcoffset() must not return None).
+
+        If self.tzinfo is tz, self.astimezone(tz) is equal to self: no
+        adjustment of date or time members is performed. Else the result is
+        local time in time zone tz, representing the same UTC time as self:
+
+            after astz = dt.astimezone(tz), astz - astz.utcoffset()
+
+        will usually have the same date and time members as dt - dt.utcoffset().
+        The discussion of class `datetime.tzinfo` explains the cases at Daylight Saving
+        Time transition boundaries where this cannot be achieved (an issue only
+        if tz models both standard and daylight time).
+
+        If you merely want to attach a time zone object *tz* to a datetime *dt*
+        without adjustment of date and time members, use ``dt.replace(tzinfo=tz)``.
+        If you merely want to remove the time zone object from an aware
+        datetime dt without conversion of date and time members, use
+        ``dt.replace(tzinfo=None)``.
+
+        Note that the default `tzinfo.fromutc` method can be overridden in a
+        tzinfo subclass to effect the result returned by `astimezone`.
+        """
+
+    def utcoffset():
+        """Return the timezone offset in minutes east of UTC (negative west of
+        UTC)."""
+
+    def dst():
+        """Return 0 if DST is not in effect, or the DST offset (in minutes
+        eastward) if DST is in effect.
+        """
+
+    def tzname():
+        """Return the timezone name."""
+
+    def timetuple():
+        """Return a 9-element tuple of the form returned by `time.localtime`."""
+
+    def utctimetuple():
+        """Return UTC time tuple compatilble with `time.gmtime`."""
+
+    def toordinal():
+        """Return the proleptic Gregorian ordinal of the date.
+
+        The same as self.date().toordinal().
+        """
+
+    def weekday():
+        """Return the day of the week as an integer.
+
+        Monday is 0 and Sunday is 6. The same as self.date().weekday().
+        See also isoweekday().
+        """
+
+    def isoweekday():
+        """Return the day of the week as an integer.
+
+        Monday is 1 and Sunday is 7. The same as self.date().isoweekday.
+
+        .. seealso:: `weekday`, `isocalendar`.
+        """
+
+    def isocalendar():
+        """Return a 3-tuple, (ISO year, ISO week number, ISO weekday).
+
+        The same as self.date().isocalendar().
+        """
+
+    def isoformat(sep='T'):
+        """Return a string representing the date and time in ISO 8601 format.
+
+        YYYY-MM-DDTHH:MM:SS.mmmmmm or YYYY-MM-DDTHH:MM:SS if microsecond is 0
+
+        If `utcoffset` does not return None, a 6-character string is appended,
+        giving the UTC offset in (signed) hours and minutes:
+
+        YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM or YYYY-MM-DDTHH:MM:SS+HH:MM
+        if microsecond is 0.
+
+        The optional argument sep (default 'T') is a one-character separator,
+        placed between the date and time portions of the result.
+        """
+
+    def __str__():
+        """For a datetime instance *d*, ``str(d)`` is equivalent to ``d.isoformat(' ')``.
+        """
+
+    def ctime():
+        """Return a string representing the date and time.
+
+        ``datetime(2002, 12, 4, 20, 30, 40).ctime() == 'Wed Dec 4 20:30:40 2002'``.
+        ``d.ctime()`` is equivalent to ``time.ctime(time.mktime(d.timetuple()))`` on
+        platforms where the native C ``ctime()`` function (which `time.ctime`
+        invokes, but which `datetime.ctime` does not invoke) conforms to the
+        C standard.
+        """
+
+    def strftime(format):
+        """Return a string representing the date and time.
+
+        This is controlled by an explicit format string.
+        """
+
+
+class ITimeClass(Interface):
+    """This is the time class interface.
+
+    This is symbolic; this module does **not** make
+    `datetime.time` provide this interface.
+
+    """
+
+    min = Attribute("The earliest representable time")
+
+    max = Attribute("The latest representable time")
+
+    resolution = Attribute(
+        "The smallest possible difference between non-equal time objects")
+
+
+class ITime(ITimeClass):
+    """Represent time with time zone.
+
+    Implemented by `datetime.time`.
+
+    Operators:
+
+    __repr__, __str__
+    __cmp__, __hash__
+    """
+
+    hour = Attribute("Hour in range(24)")
+
+    minute = Attribute("Minute in range(60)")
+
+    second = Attribute("Second in range(60)")
+
+    microsecond = Attribute("Microsecond in range(1000000)")
+
+    tzinfo = Attribute(
+        """The object passed as the tzinfo argument to the time constructor
+        or None if none was passed.""")
+
+    def replace(hour, minute, second, microsecond, tzinfo):
+        """Return a time with the same value.
+
+        Except for those members given new values by whichever keyword
+        arguments are specified. Note that tzinfo=None can be specified
+        to create a naive time from an aware time, without conversion of the
+        time members.
+        """
+
+    def isoformat():
+        """Return a string representing the time in ISO 8601 format.
+
+        That is HH:MM:SS.mmmmmm or, if self.microsecond is 0, HH:MM:SS
+        If utcoffset() does not return None, a 6-character string is appended,
+        giving the UTC offset in (signed) hours and minutes:
+        HH:MM:SS.mmmmmm+HH:MM or, if self.microsecond is 0, HH:MM:SS+HH:MM
+        """
+
+    def __str__():
+        """For a time t, str(t) is equivalent to t.isoformat()."""
+
+    def strftime(format):
+        """Return a string representing the time.
+
+        This is controlled by an explicit format string.
+        """
+
+    def utcoffset():
+        """Return the timezone offset in minutes east of UTC (negative west of
+        UTC).
+
+        If tzinfo is None, returns None, else returns
+        self.tzinfo.utcoffset(None), and raises an exception if the latter
+        doesn't return None or a timedelta object representing a whole number
+        of minutes with magnitude less than one day.
+        """
+
+    def dst():
+        """Return 0 if DST is not in effect, or the DST offset (in minutes
+        eastward) if DST is in effect.
+
+        If tzinfo is None, returns None, else returns self.tzinfo.dst(None),
+        and raises an exception if the latter doesn't return None, or a
+        timedelta object representing a whole number of minutes with
+        magnitude less than one day.
+        """
+
+    def tzname():
+        """Return the timezone name.
+
+        If tzinfo is None, returns None, else returns self.tzinfo.tzname(None),
+        or raises an exception if the latter doesn't return None or a string
+        object.
+        """
+
+
+class ITZInfo(Interface):
+    """Time zone info class.
+    """
+
+    def utcoffset(dt):
+        """Return offset of local time from UTC, in minutes east of UTC.
+
+        If local time is west of UTC, this should be negative.
+        Note that this is intended to be the total offset from UTC;
+        for example, if a tzinfo object represents both time zone and DST
+        adjustments, utcoffset() should return their sum. If the UTC offset
+        isn't known, return None. Else the value returned must be a timedelta
+        object specifying a whole number of minutes in the range -1439 to 1439
+        inclusive (1440 = 24*60; the magnitude of the offset must be less
+        than one day).
+        """
+
+    def dst(dt):
+        """Return the daylight saving time (DST) adjustment, in minutes east
+        of UTC, or None if DST information isn't known.
+        """
+
+    def tzname(dt):
+        """Return the time zone name corresponding to the datetime object as
+        a string.
+        """
+
+    def fromutc(dt):
+        """Return an equivalent datetime in self's local time."""
+
+
+classImplements(timedelta, ITimeDelta)
+classImplements(date, IDate)
+classImplements(datetime, IDateTime)
+classImplements(time, ITime)
+classImplements(tzinfo, ITZInfo)
+
+## directlyProvides(timedelta, ITimeDeltaClass)
+## directlyProvides(date, IDateClass)
+## directlyProvides(datetime, IDateTimeClass)
+## directlyProvides(time, ITimeClass)
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/interfaces.py b/contrib/python/zope.interface/py3/zope/interface/common/interfaces.py
new file mode 100644
index 00000000000..70bd294f358
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/interfaces.py
@@ -0,0 +1,208 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interfaces for standard python exceptions
+"""
+from zope.interface import Interface
+from zope.interface import classImplements
+
+class IException(Interface):
+    "Interface for `Exception`"
+classImplements(Exception, IException)
+
+
+class IStandardError(IException):
+    "Interface for `StandardError` (no longer existing.)"
+
+
+class IWarning(IException):
+    "Interface for `Warning`"
+classImplements(Warning, IWarning)
+
+
+class ISyntaxError(IStandardError):
+    "Interface for `SyntaxError`"
+classImplements(SyntaxError, ISyntaxError)
+
+
+class ILookupError(IStandardError):
+    "Interface for `LookupError`"
+classImplements(LookupError, ILookupError)
+
+
+class IValueError(IStandardError):
+    "Interface for `ValueError`"
+classImplements(ValueError, IValueError)
+
+
+class IRuntimeError(IStandardError):
+    "Interface for `RuntimeError`"
+classImplements(RuntimeError, IRuntimeError)
+
+
+class IArithmeticError(IStandardError):
+    "Interface for `ArithmeticError`"
+classImplements(ArithmeticError, IArithmeticError)
+
+
+class IAssertionError(IStandardError):
+    "Interface for `AssertionError`"
+classImplements(AssertionError, IAssertionError)
+
+
+class IAttributeError(IStandardError):
+    "Interface for `AttributeError`"
+classImplements(AttributeError, IAttributeError)
+
+
+class IDeprecationWarning(IWarning):
+    "Interface for `DeprecationWarning`"
+classImplements(DeprecationWarning, IDeprecationWarning)
+
+
+class IEOFError(IStandardError):
+    "Interface for `EOFError`"
+classImplements(EOFError, IEOFError)
+
+
+class IEnvironmentError(IStandardError):
+    "Interface for `EnvironmentError`"
+classImplements(EnvironmentError, IEnvironmentError)
+
+
+class IFloatingPointError(IArithmeticError):
+    "Interface for `FloatingPointError`"
+classImplements(FloatingPointError, IFloatingPointError)
+
+
+class IIOError(IEnvironmentError):
+    "Interface for `IOError`"
+classImplements(IOError, IIOError)
+
+
+class IImportError(IStandardError):
+    "Interface for `ImportError`"
+classImplements(ImportError, IImportError)
+
+
+class IIndentationError(ISyntaxError):
+    "Interface for `IndentationError`"
+classImplements(IndentationError, IIndentationError)
+
+
+class IIndexError(ILookupError):
+    "Interface for `IndexError`"
+classImplements(IndexError, IIndexError)
+
+
+class IKeyError(ILookupError):
+    "Interface for `KeyError`"
+classImplements(KeyError, IKeyError)
+
+
+class IKeyboardInterrupt(IStandardError):
+    "Interface for `KeyboardInterrupt`"
+classImplements(KeyboardInterrupt, IKeyboardInterrupt)
+
+
+class IMemoryError(IStandardError):
+    "Interface for `MemoryError`"
+classImplements(MemoryError, IMemoryError)
+
+
+class INameError(IStandardError):
+    "Interface for `NameError`"
+classImplements(NameError, INameError)
+
+
+class INotImplementedError(IRuntimeError):
+    "Interface for `NotImplementedError`"
+classImplements(NotImplementedError, INotImplementedError)
+
+
+class IOSError(IEnvironmentError):
+    "Interface for `OSError`"
+classImplements(OSError, IOSError)
+
+
+class IOverflowError(IArithmeticError):
+    "Interface for `ArithmeticError`"
+classImplements(OverflowError, IOverflowError)
+
+
+class IOverflowWarning(IWarning):
+    """Deprecated, no standard class implements this.
+
+    This was the interface for ``OverflowWarning`` prior to Python 2.5,
+    but that class was removed for all versions after that.
+    """
+
+
+class IReferenceError(IStandardError):
+    "Interface for `ReferenceError`"
+classImplements(ReferenceError, IReferenceError)
+
+
+class IRuntimeWarning(IWarning):
+    "Interface for `RuntimeWarning`"
+classImplements(RuntimeWarning, IRuntimeWarning)
+
+
+class IStopIteration(IException):
+    "Interface for `StopIteration`"
+classImplements(StopIteration, IStopIteration)
+
+
+class ISyntaxWarning(IWarning):
+    "Interface for `SyntaxWarning`"
+classImplements(SyntaxWarning, ISyntaxWarning)
+
+
+class ISystemError(IStandardError):
+    "Interface for `SystemError`"
+classImplements(SystemError, ISystemError)
+
+
+class ISystemExit(IException):
+    "Interface for `SystemExit`"
+classImplements(SystemExit, ISystemExit)
+
+
+class ITabError(IIndentationError):
+    "Interface for `TabError`"
+classImplements(TabError, ITabError)
+
+
+class ITypeError(IStandardError):
+    "Interface for `TypeError`"
+classImplements(TypeError, ITypeError)
+
+
+class IUnboundLocalError(INameError):
+    "Interface for `UnboundLocalError`"
+classImplements(UnboundLocalError, IUnboundLocalError)
+
+
+class IUnicodeError(IValueError):
+    "Interface for `UnicodeError`"
+classImplements(UnicodeError, IUnicodeError)
+
+
+class IUserWarning(IWarning):
+    "Interface for `UserWarning`"
+classImplements(UserWarning, IUserWarning)
+
+
+class IZeroDivisionError(IArithmeticError):
+    "Interface for `ZeroDivisionError`"
+classImplements(ZeroDivisionError, IZeroDivisionError)
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/io.py b/contrib/python/zope.interface/py3/zope/interface/common/io.py
new file mode 100644
index 00000000000..0d6f3badfcf
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/io.py
@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""
+Interface definitions paralleling the abstract base classes defined in
+:mod:`io`.
+
+After this module is imported, the standard library types will declare
+that they implement the appropriate interface.
+
+.. versionadded:: 5.0.0
+"""
+
+import io as abc
+
+from zope.interface.common import ABCInterface
+
+# pylint:disable=inherit-non-class,
+# pylint:disable=no-member
+
+class IIOBase(ABCInterface):
+    abc = abc.IOBase
+
+
+class IRawIOBase(IIOBase):
+    abc = abc.RawIOBase
+
+
+class IBufferedIOBase(IIOBase):
+    abc = abc.BufferedIOBase
+    extra_classes = ()
+
+
+class ITextIOBase(IIOBase):
+    abc = abc.TextIOBase
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/mapping.py b/contrib/python/zope.interface/py3/zope/interface/common/mapping.py
new file mode 100644
index 00000000000..d04333357f8
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/mapping.py
@@ -0,0 +1,168 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Mapping Interfaces.
+
+Importing this module does *not* mark any standard classes as
+implementing any of these interfaces.
+
+While this module is not deprecated, new code should generally use
+:mod:`zope.interface.common.collections`, specifically
+:class:`~zope.interface.common.collections.IMapping` and
+:class:`~zope.interface.common.collections.IMutableMapping`. This
+module is occasionally useful for its extremely fine grained breakdown
+of interfaces.
+
+The standard library :class:`dict` and :class:`collections.UserDict`
+implement ``IMutableMapping``, but *do not* implement any of the
+interfaces in this module.
+"""
+from zope.interface import Interface
+from zope.interface.common import collections
+
+class IItemMapping(Interface):
+    """Simplest readable mapping object
+    """
+
+    def __getitem__(key):
+        """Get a value for a key
+
+        A `KeyError` is raised if there is no value for the key.
+        """
+
+
+class IReadMapping(collections.IContainer, IItemMapping):
+    """
+    Basic mapping interface.
+
+    .. versionchanged:: 5.0.0
+       Extend ``IContainer``
+    """
+
+    def get(key, default=None):
+        """Get a value for a key
+
+        The default is returned if there is no value for the key.
+        """
+
+    def __contains__(key):
+        """Tell if a key exists in the mapping."""
+        # Optional in IContainer, required by this interface.
+
+
+class IWriteMapping(Interface):
+    """Mapping methods for changing data"""
+
+    def __delitem__(key):
+        """Delete a value from the mapping using the key."""
+
+    def __setitem__(key, value):
+        """Set a new item in the mapping."""
+
+
+class IEnumerableMapping(collections.ISized, IReadMapping):
+    """
+    Mapping objects whose items can be enumerated.
+
+    .. versionchanged:: 5.0.0
+       Extend ``ISized``
+    """
+
+    def keys():
+        """Return the keys of the mapping object.
+        """
+
+    def __iter__():
+        """Return an iterator for the keys of the mapping object.
+        """
+
+    def values():
+        """Return the values of the mapping object.
+        """
+
+    def items():
+        """Return the items of the mapping object.
+        """
+
+class IMapping(IWriteMapping, IEnumerableMapping):
+    ''' Simple mapping interface '''
+
+class IIterableMapping(IEnumerableMapping):
+    """A mapping that has distinct methods for iterating
+    without copying.
+
+    """
+
+
+class IClonableMapping(Interface):
+    """Something that can produce a copy of itself.
+
+    This is available in `dict`.
+    """
+
+    def copy():
+        "return copy of dict"
+
+class IExtendedReadMapping(IIterableMapping):
+    """
+    Something with a particular method equivalent to ``__contains__``.
+
+    On Python 2, `dict` provided the ``has_key`` method, but it was removed
+    in Python 3.
+    """
+
+
+class IExtendedWriteMapping(IWriteMapping):
+    """Additional mutation methods.
+
+    These are all provided by `dict`.
+    """
+
+    def clear():
+        "delete all items"
+
+    def update(d):
+        " Update D from E: for k in E.keys(): D[k] = E[k]"
+
+    def setdefault(key, default=None):
+        "D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D"
+
+    def pop(k, default=None):
+        """
+        pop(k[,default]) -> value
+
+        Remove specified key and return the corresponding value.
+
+        If key is not found, *default* is returned if given, otherwise
+        `KeyError` is raised. Note that *default* must not be passed by
+        name.
+        """
+
+    def popitem():
+        """remove and return some (key, value) pair as a
+        2-tuple; but raise KeyError if mapping is empty"""
+
+class IFullMapping(
+        collections.IMutableMapping,
+        IExtendedReadMapping, IExtendedWriteMapping, IClonableMapping, IMapping,):
+    """
+    Full mapping interface.
+
+    Most uses of this interface should instead use
+    :class:`~zope.interface.commons.collections.IMutableMapping` (one of the
+    bases of this interface). The required methods are the same.
+
+    .. versionchanged:: 5.0.0
+       Extend ``IMutableMapping``
+    """
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/numbers.py b/contrib/python/zope.interface/py3/zope/interface/common/numbers.py
new file mode 100644
index 00000000000..6b20e09d324
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/numbers.py
@@ -0,0 +1,65 @@
+##############################################################################
+# Copyright (c) 2020 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""
+Interface definitions paralleling the abstract base classes defined in
+:mod:`numbers`.
+
+After this module is imported, the standard library types will declare
+that they implement the appropriate interface.
+
+.. versionadded:: 5.0.0
+"""
+
+import numbers as abc
+
+from zope.interface.common import ABCInterface
+from zope.interface.common import optional
+
+
+# pylint:disable=inherit-non-class,
+# pylint:disable=no-self-argument,no-method-argument
+# pylint:disable=unexpected-special-method-signature
+# pylint:disable=no-value-for-parameter
+
+
+class INumber(ABCInterface):
+    abc = abc.Number
+
+
+class IComplex(INumber):
+    abc = abc.Complex
+
+    @optional
+    def __complex__():
+        """
+        Rarely implemented, even in builtin types.
+        """
+
+
+class IReal(IComplex):
+    abc = abc.Real
+
+    @optional
+    def __complex__():
+        """
+        Rarely implemented, even in builtin types.
+        """
+
+    __floor__ = __ceil__ = __complex__
+
+
+class IRational(IReal):
+    abc = abc.Rational
+
+
+class IIntegral(IRational):
+    abc = abc.Integral
diff --git a/contrib/python/zope.interface/py3/zope/interface/common/sequence.py b/contrib/python/zope.interface/py3/zope/interface/common/sequence.py
new file mode 100644
index 00000000000..5edc73dc6ae
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/common/sequence.py
@@ -0,0 +1,189 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Sequence Interfaces
+
+Importing this module does *not* mark any standard classes as
+implementing any of these interfaces.
+
+While this module is not deprecated, new code should generally use
+:mod:`zope.interface.common.collections`, specifically
+:class:`~zope.interface.common.collections.ISequence` and
+:class:`~zope.interface.common.collections.IMutableSequence`. This
+module is occasionally useful for its fine-grained breakdown of interfaces.
+
+The standard library :class:`list`, :class:`tuple` and
+:class:`collections.UserList`, among others, implement ``ISequence``
+or ``IMutableSequence`` but *do not* implement any of the interfaces
+in this module.
+"""
+
+__docformat__ = 'restructuredtext'
+from zope.interface import Interface
+from zope.interface.common import collections
+
+class IMinimalSequence(collections.IIterable):
+    """Most basic sequence interface.
+
+    All sequences are iterable.  This requires at least one of the
+    following:
+
+    - a `__getitem__()` method that takes a single argument; integer
+      values starting at 0 must be supported, and `IndexError` should
+      be raised for the first index for which there is no value, or
+
+    - an `__iter__()` method that returns an iterator as defined in
+      the Python documentation (http://docs.python.org/lib/typeiter.html).
+
+    """
+
+    def __getitem__(index):
+        """``x.__getitem__(index) <==> x[index]``
+
+        Declaring this interface does not specify whether `__getitem__`
+        supports slice objects."""
+
+class IFiniteSequence(collections.ISized, IMinimalSequence):
+    """
+    A sequence of bound size.
+
+    .. versionchanged:: 5.0.0
+       Extend ``ISized``
+    """
+
+class IReadSequence(collections.IContainer, IFiniteSequence):
+    """
+    read interface shared by tuple and list
+
+    This interface is similar to
+    :class:`~zope.interface.common.collections.ISequence`, but
+    requires that all instances be totally ordered. Most users
+    should prefer ``ISequence``.
+
+    .. versionchanged:: 5.0.0
+       Extend ``IContainer``
+    """
+
+    def __contains__(item):
+        """``x.__contains__(item) <==> item in x``"""
+        # Optional in IContainer, required here.
+
+    def __lt__(other):
+        """``x.__lt__(other) <==> x < other``"""
+
+    def __le__(other):
+        """``x.__le__(other) <==> x <= other``"""
+
+    def __eq__(other):
+        """``x.__eq__(other) <==> x == other``"""
+
+    def __ne__(other):
+        """``x.__ne__(other) <==> x != other``"""
+
+    def __gt__(other):
+        """``x.__gt__(other) <==> x > other``"""
+
+    def __ge__(other):
+        """``x.__ge__(other) <==> x >= other``"""
+
+    def __add__(other):
+        """``x.__add__(other) <==> x + other``"""
+
+    def __mul__(n):
+        """``x.__mul__(n) <==> x * n``"""
+
+    def __rmul__(n):
+        """``x.__rmul__(n) <==> n * x``"""
+
+
+class IExtendedReadSequence(IReadSequence):
+    """Full read interface for lists"""
+
+    def count(item):
+        """Return number of occurrences of value"""
+
+    def index(item, *args):
+        """index(value, [start, [stop]]) -> int
+
+        Return first index of *value*
+        """
+
+class IUniqueMemberWriteSequence(Interface):
+    """The write contract for a sequence that may enforce unique members"""
+
+    def __setitem__(index, item):
+        """``x.__setitem__(index, item) <==> x[index] = item``
+
+        Declaring this interface does not specify whether `__setitem__`
+        supports slice objects.
+        """
+
+    def __delitem__(index):
+        """``x.__delitem__(index) <==> del x[index]``
+
+        Declaring this interface does not specify whether `__delitem__`
+        supports slice objects.
+        """
+
+    def __iadd__(y):
+        """``x.__iadd__(y) <==> x += y``"""
+
+    def append(item):
+        """Append item to end"""
+
+    def insert(index, item):
+        """Insert item before index"""
+
+    def pop(index=-1):
+        """Remove and return item at index (default last)"""
+
+    def remove(item):
+        """Remove first occurrence of value"""
+
+    def reverse():
+        """Reverse *IN PLACE*"""
+
+    def sort(cmpfunc=None):
+        """Stable sort *IN PLACE*; `cmpfunc(x, y)` -> -1, 0, 1"""
+
+    def extend(iterable):
+        """Extend list by appending elements from the iterable"""
+
+class IWriteSequence(IUniqueMemberWriteSequence):
+    """Full write contract for sequences"""
+
+    def __imul__(n):
+        """``x.__imul__(n) <==> x *= n``"""
+
+class ISequence(IReadSequence, IWriteSequence):
+    """
+    Full sequence contract.
+
+    New code should prefer
+    :class:`~zope.interface.common.collections.IMutableSequence`.
+
+    Compared to that interface, which is implemented by :class:`list`
+    (:class:`~zope.interface.common.builtins.IList`), among others,
+    this interface is missing the following methods:
+
+        - clear
+
+        - count
+
+        - index
+
+    This interface adds the following methods:
+
+        - sort
+    """
diff --git a/contrib/python/zope.interface/py3/zope/interface/declarations.py b/contrib/python/zope.interface/py3/zope/interface/declarations.py
new file mode 100644
index 00000000000..61e2543929d
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/declarations.py
@@ -0,0 +1,1188 @@
+##############################################################################
+# Copyright (c) 2003 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+##############################################################################
+"""Implementation of interface declarations
+
+There are three flavors of declarations:
+
+  - Declarations are used to simply name declared interfaces.
+
+  - ImplementsDeclarations are used to express the interfaces that a
+    class implements (that instances of the class provides).
+
+    Implements specifications support inheriting interfaces.
+
+  - ProvidesDeclarations are used to express interfaces directly
+    provided by objects.
+
+"""
+__docformat__ = 'restructuredtext'
+
+import sys
+from types import FunctionType
+from types import MethodType
+from types import ModuleType
+import weakref
+
+from zope.interface.interface import Interface
+from zope.interface.interface import InterfaceClass
+from zope.interface.interface import SpecificationBase
+from zope.interface.interface import Specification
+from zope.interface.interface import NameAndModuleComparisonMixin
+from zope.interface._compat import _use_c_impl
+
+__all__ = [
+    # None. The public APIs of this module are
+    # re-exported from zope.interface directly.
+]
+
+# pylint:disable=too-many-lines
+
+# Registry of class-implementation specifications
+BuiltinImplementationSpecifications = {}
+
+
+def _next_super_class(ob):
+    # When ``ob`` is an instance of ``super``, return
+    # the next class in the MRO that we should actually be
+    # looking at. Watch out for diamond inheritance!
+    self_class = ob.__self_class__
+    class_that_invoked_super = ob.__thisclass__
+    complete_mro = self_class.__mro__
+    next_class = complete_mro[complete_mro.index(class_that_invoked_super) + 1]
+    return next_class
+
+class named:
+
+    def __init__(self, name):
+        self.name = name
+
+    def __call__(self, ob):
+        ob.__component_name__ = self.name
+        return ob
+
+
+class Declaration(Specification):
+    """Interface declarations"""
+
+    __slots__ = ()
+
+    def __init__(self, *bases):
+        Specification.__init__(self, _normalizeargs(bases))
+
+    def __contains__(self, interface):
+        """Test whether an interface is in the specification
+        """
+
+        return self.extends(interface) and interface in self.interfaces()
+
+    def __iter__(self):
+        """Return an iterator for the interfaces in the specification
+        """
+        return self.interfaces()
+
+    def flattened(self):
+        """Return an iterator of all included and extended interfaces
+        """
+        return iter(self.__iro__)
+
+    def __sub__(self, other):
+        """Remove interfaces from a specification
+        """
+        return Declaration(*[
+            i for i in self.interfaces()
+            if not [
+                j
+                for j in other.interfaces()
+                if i.extends(j, 0) # non-strict extends
+            ]
+        ])
+
+    def __add__(self, other):
+        """
+        Add two specifications or a specification and an interface
+        and produce a new declaration.
+
+        .. versionchanged:: 5.4.0
+           Now tries to preserve a consistent resolution order. Interfaces
+           being added to this object are added to the front of the resulting resolution
+           order if they already extend an interface in this object. Previously,
+           they were always added to the end of the order, which easily resulted in
+           invalid orders.
+        """
+        before = []
+        result = list(self.interfaces())
+        seen = set(result)
+        for i in other.interfaces():
+            if i in seen:
+                continue
+            seen.add(i)
+            if any(i.extends(x) for x in result):
+                # It already extends us, e.g., is a subclass,
+                # so it needs to go at the front of the RO.
+                before.append(i)
+            else:
+                result.append(i)
+        return Declaration(*(before + result))
+
+    # XXX: Is __radd__ needed? No tests break if it's removed.
+    # If it is needed, does it need to handle the C3 ordering differently?
+    # I (JAM) don't *think* it does.
+    __radd__ = __add__
+
+    @staticmethod
+    def _add_interfaces_to_cls(interfaces, cls):
+        # Strip redundant interfaces already provided
+        # by the cls so we don't produce invalid
+        # resolution orders.
+        implemented_by_cls = implementedBy(cls)
+        interfaces = tuple([
+            iface
+            for iface in interfaces
+            if not implemented_by_cls.isOrExtends(iface)
+        ])
+        return interfaces + (implemented_by_cls,)
+
+    @staticmethod
+    def _argument_names_for_repr(interfaces):
+        # These don't actually have to be interfaces, they could be other
+        # Specification objects like Implements. Also, the first
+        # one is typically/nominally the cls.
+        ordered_names = []
+        names = set()
+        for iface in interfaces:
+            duplicate_transform = repr
+            if isinstance(iface, InterfaceClass):
+                # Special case to get 'foo.bar.IFace'
+                # instead of '<InterfaceClass foo.bar.IFace>'
+                this_name = iface.__name__
+                duplicate_transform = str
+            elif isinstance(iface, type):
+                # Likewise for types. (Ignoring legacy old-style
+                # classes.)
+                this_name = iface.__name__
+                duplicate_transform = _implements_name
+            elif (isinstance(iface, Implements)
+                  and not iface.declared
+                  and iface.inherit in interfaces):
+                # If nothing is declared, there's no need to even print this;
+                # it would just show as ``classImplements(Class)``, and the
+                # ``Class`` has typically already.
+                continue
+            else:
+                this_name = repr(iface)
+
+            already_seen = this_name in names
+            names.add(this_name)
+            if already_seen:
+                this_name = duplicate_transform(iface)
+
+            ordered_names.append(this_name)
+        return ', '.join(ordered_names)
+
+
+class _ImmutableDeclaration(Declaration):
+    # A Declaration that is immutable. Used as a singleton to
+    # return empty answers for things like ``implementedBy``.
+    # We have to define the actual singleton after normalizeargs
+    # is defined, and that in turn is defined after InterfaceClass and
+    # Implements.
+
+    __slots__ = ()
+
+    __instance = None
+
+    def __new__(cls):
+        if _ImmutableDeclaration.__instance is None:
+            _ImmutableDeclaration.__instance = object.__new__(cls)
+        return _ImmutableDeclaration.__instance
+
+    def __reduce__(self):
+        return "_empty"
+
+    @property
+    def __bases__(self):
+        return ()
+
+    @__bases__.setter
+    def __bases__(self, new_bases):
+        # We expect the superclass constructor to set ``self.__bases__ = ()``.
+        # Rather than attempt to special case that in the constructor and allow
+        # setting __bases__ only at that time, it's easier to just allow setting
+        # the empty tuple at any time. That makes ``x.__bases__ = x.__bases__`` a nice
+        # no-op too. (Skipping the superclass constructor altogether is a recipe
+        # for maintenance headaches.)
+        if new_bases != ():
+            raise TypeError("Cannot set non-empty bases on shared empty Declaration.")
+
+    # As the immutable empty declaration, we cannot be changed.
+    # This means there's no logical reason for us to have dependents
+    # or subscriptions: we'll never notify them. So there's no need for
+    # us to keep track of any of that.
+    @property
+    def dependents(self):
+        return {}
+
+    changed = subscribe = unsubscribe = lambda self, _ignored: None
+
+    def interfaces(self):
+        # An empty iterator
+        return iter(())
+
+    def extends(self, interface, strict=True):
+        return interface is self._ROOT
+
+    def get(self, name, default=None):
+        return default
+
+    def weakref(self, callback=None):
+        # We're a singleton, we never go away. So there's no need to return
+        # distinct weakref objects here; their callbacks will never
+        # be called. Instead, we only need to return a callable that
+        # returns ourself. The easiest one is to return _ImmutableDeclaration
+        # itself; testing on Python 3.8 shows that's faster than a function that
+        # returns _empty. (Remember, one goal is to avoid allocating any
+        # object, and that includes a method.)
+        return _ImmutableDeclaration
+
+    @property
+    def _v_attrs(self):
+        # _v_attrs is not a public, documented property, but some client code
+        # uses it anyway as a convenient place to cache things. To keep the
+        # empty declaration truly immutable, we must ignore that. That includes
+        # ignoring assignments as well.
+        return {}
+
+    @_v_attrs.setter
+    def _v_attrs(self, new_attrs):
+        pass
+
+
+##############################################################################
+#
+# Implementation specifications
+#
+# These specify interfaces implemented by instances of classes
+
+class Implements(NameAndModuleComparisonMixin,
+                 Declaration):
+    # Inherit from NameAndModuleComparisonMixin to be
+    # mutually comparable with InterfaceClass objects.
+    # (The two must be mutually comparable to be able to work in e.g., BTrees.)
+    # Instances of this class generally don't have a __module__ other than
+    # `zope.interface.declarations`, whereas they *do* have a __name__ that is the
+    # fully qualified name of the object they are representing.
+
+    # Note, though, that equality and hashing are still identity based. This
+    # accounts for things like nested objects that have the same name (typically
+    # only in tests) and is consistent with pickling. As far as comparisons to InterfaceClass
+    # goes, we'll never have equal name and module to those, so we're still consistent there.
+    # Instances of this class are essentially intended to be unique and are
+    # heavily cached (note how our __reduce__ handles this) so having identity
+    # based hash and eq should also work.
+
+    # We want equality and hashing to be based on identity. However, we can't actually
+    # implement __eq__/__ne__ to do this because sometimes we get wrapped in a proxy.
+    # We need to let the proxy types implement these methods so they can handle unwrapping
+    # and then rely on: (1) the interpreter automatically changing `implements == proxy` into
+    # `proxy == implements` (which will call proxy.__eq__ to do the unwrapping) and then
+    # (2) the default equality and hashing semantics being identity based.
+
+    # class whose specification should be used as additional base
+    inherit = None
+
+    # interfaces actually declared for a class
+    declared = ()
+
+    # Weak cache of {class: <implements>} for super objects.
+    # Created on demand. These are rare, as of 5.0 anyway. Using a class
+    # level default doesn't take space in instances. Using _v_attrs would be
+    # another place to store this without taking space unless needed.
+    _super_cache = None
+
+    __name__ = '?'
+
+    @classmethod
+    def named(cls, name, *bases):
+        # Implementation method: Produce an Implements interface with
+        # a fully fleshed out __name__ before calling the constructor, which
+        # sets bases to the given interfaces and which may pass this object to
+        # other objects (e.g., to adjust dependents). If they're sorting or comparing
+        # by name, this needs to be set.
+        inst = cls.__new__(cls)
+        inst.__name__ = name
+        inst.__init__(*bases)
+        return inst
+
+    def changed(self, originally_changed):
+        try:
+            del self._super_cache
+        except AttributeError:
+            pass
+        return super().changed(originally_changed)
+
+    def __repr__(self):
+        if self.inherit:
+            name = getattr(self.inherit, '__name__', None) or _implements_name(self.inherit)
+        else:
+            name = self.__name__
+        declared_names = self._argument_names_for_repr(self.declared)
+        if declared_names:
+            declared_names = ', ' + declared_names
+        return 'classImplements({}{})'.format(name, declared_names)
+
+    def __reduce__(self):
+        return implementedBy, (self.inherit, )
+
+
+def _implements_name(ob):
+    # Return the __name__ attribute to be used by its __implemented__
+    # property.
+    # This must be stable for the "same" object across processes
+    # because it is used for sorting. It needn't be unique, though, in cases
+    # like nested classes named Foo created by different functions, because
+    # equality and hashing is still based on identity.
+    # It might be nice to use __qualname__ on Python 3, but that would produce
+    # different values between Py2 and Py3.
+    return (getattr(ob, '__module__', '?') or '?') + \
+        '.' + (getattr(ob, '__name__', '?') or '?')
+
+
+def _implementedBy_super(sup):
+    # TODO: This is now simple enough we could probably implement
+    # in C if needed.
+
+    # If the class MRO is strictly linear, we could just
+    # follow the normal algorithm for the next class in the
+    # search order (e.g., just return
+    # ``implemented_by_next``). But when diamond inheritance
+    # or mixins + interface declarations are present, we have
+    # to consider the whole MRO and compute a new Implements
+    # that excludes the classes being skipped over but
+    # includes everything else.
+    implemented_by_self = implementedBy(sup.__self_class__)
+    cache = implemented_by_self._super_cache # pylint:disable=protected-access
+    if cache is None:
+        cache = implemented_by_self._super_cache = weakref.WeakKeyDictionary()
+
+    key = sup.__thisclass__
+    try:
+        return cache[key]
+    except KeyError:
+        pass
+
+    next_cls = _next_super_class(sup)
+    # For ``implementedBy(cls)``:
+    # .__bases__ is .declared + [implementedBy(b) for b in cls.__bases__]
+    # .inherit is cls
+
+    implemented_by_next = implementedBy(next_cls)
+    mro = sup.__self_class__.__mro__
+    ix_next_cls = mro.index(next_cls)
+    classes_to_keep = mro[ix_next_cls:]
+    new_bases = [implementedBy(c) for c in classes_to_keep]
+
+    new = Implements.named(
+        implemented_by_self.__name__ + ':' + implemented_by_next.__name__,
+        *new_bases
+    )
+    new.inherit = implemented_by_next.inherit
+    new.declared = implemented_by_next.declared
+    # I don't *think* that new needs to subscribe to ``implemented_by_self``;
+    # it auto-subscribed to its bases, and that should be good enough.
+    cache[key] = new
+
+    return new
+
+
+@_use_c_impl
+def implementedBy(cls): # pylint:disable=too-many-return-statements,too-many-branches
+    """Return the interfaces implemented for a class' instances
+
+      The value returned is an `~zope.interface.interfaces.IDeclaration`.
+    """
+    try:
+        if isinstance(cls, super):
+            # Yes, this needs to be inside the try: block. Some objects
+            # like security proxies even break isinstance.
+            return _implementedBy_super(cls)
+
+        spec = cls.__dict__.get('__implemented__')
+    except AttributeError:
+
+        # we can't get the class dict. This is probably due to a
+        # security proxy.  If this is the case, then probably no
+        # descriptor was installed for the class.
+
+        # We don't want to depend directly on zope.security in
+        # zope.interface, but we'll try to make reasonable
+        # accommodations in an indirect way.
+
+        # We'll check to see if there's an implements:
+
+        spec = getattr(cls, '__implemented__', None)
+        if spec is None:
+            # There's no spec stred in the class. Maybe its a builtin:
+            spec = BuiltinImplementationSpecifications.get(cls)
+            if spec is not None:
+                return spec
+            return _empty
+
+        if spec.__class__ == Implements:
+            # we defaulted to _empty or there was a spec. Good enough.
+            # Return it.
+            return spec
+
+        # TODO: need old style __implements__ compatibility?
+        # Hm, there's an __implemented__, but it's not a spec. Must be
+        # an old-style declaration. Just compute a spec for it
+        return Declaration(*_normalizeargs((spec, )))
+
+    if isinstance(spec, Implements):
+        return spec
+
+    if spec is None:
+        spec = BuiltinImplementationSpecifications.get(cls)
+        if spec is not None:
+            return spec
+
+    # TODO: need old style __implements__ compatibility?
+    spec_name = _implements_name(cls)
+    if spec is not None:
+        # old-style __implemented__ = foo declaration
+        spec = (spec, ) # tuplefy, as it might be just an int
+        spec = Implements.named(spec_name, *_normalizeargs(spec))
+        spec.inherit = None    # old-style implies no inherit
+        del cls.__implemented__ # get rid of the old-style declaration
+    else:
+        try:
+            bases = cls.__bases__
+        except AttributeError:
+            if not callable(cls):
+                raise TypeError("ImplementedBy called for non-factory", cls)
+            bases = ()
+
+        spec = Implements.named(spec_name, *[implementedBy(c) for c in bases])
+        spec.inherit = cls
+
+    try:
+        cls.__implemented__ = spec
+        if not hasattr(cls, '__providedBy__'):
+            cls.__providedBy__ = objectSpecificationDescriptor
+
+        if isinstance(cls, type) and '__provides__' not in cls.__dict__:
+            # Make sure we get a __provides__ descriptor
+            cls.__provides__ = ClassProvides(
+                cls,
+                getattr(cls, '__class__', type(cls)),
+                )
+
+    except TypeError:
+        if not isinstance(cls, type):
+            raise TypeError("ImplementedBy called for non-type", cls)
+        BuiltinImplementationSpecifications[cls] = spec
+
+    return spec
+
+
+def classImplementsOnly(cls, *interfaces):
+    """
+    Declare the only interfaces implemented by instances of a class
+
+    The arguments after the class are one or more interfaces or interface
+    specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+    The interfaces given (including the interfaces in the specifications)
+    replace any previous declarations, *including* inherited definitions. If you
+    wish to preserve inherited declarations, you can pass ``implementedBy(cls)``
+    in *interfaces*. This can be used to alter the interface resolution order.
+    """
+    spec = implementedBy(cls)
+    # Clear out everything inherited. It's important to
+    # also clear the bases right now so that we don't improperly discard
+    # interfaces that are already implemented by *old* bases that we're
+    # about to get rid of.
+    spec.declared = ()
+    spec.inherit = None
+    spec.__bases__ = ()
+    _classImplements_ordered(spec, interfaces, ())
+
+
+def classImplements(cls, *interfaces):
+    """
+    Declare additional interfaces implemented for instances of a class
+
+    The arguments after the class are one or more interfaces or
+    interface specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+    The interfaces given (including the interfaces in the specifications)
+    are added to any interfaces previously declared. An effort is made to
+    keep a consistent C3 resolution order, but this cannot be guaranteed.
+
+    .. versionchanged:: 5.0.0
+       Each individual interface in *interfaces* may be added to either the
+       beginning or end of the list of interfaces declared for *cls*,
+       based on inheritance, in order to try to maintain a consistent
+       resolution order. Previously, all interfaces were added to the end.
+    .. versionchanged:: 5.1.0
+       If *cls* is already declared to implement an interface (or derived interface)
+       in *interfaces* through inheritance, the interface is ignored. Previously, it
+       would redundantly be made direct base of *cls*, which often produced inconsistent
+       interface resolution orders. Now, the order will be consistent, but may change.
+       Also, if the ``__bases__`` of the *cls* are later changed, the *cls* will no
+       longer be considered to implement such an interface (changing the ``__bases__`` of *cls*
+       has never been supported).
+    """
+    spec = implementedBy(cls)
+    interfaces = tuple(_normalizeargs(interfaces))
+
+    before = []
+    after = []
+
+    # Take steps to try to avoid producing an invalid resolution
+    # order, while still allowing for BWC (in the past, we always
+    # appended)
+    for iface in interfaces:
+        for b in spec.declared:
+            if iface.extends(b):
+                before.append(iface)
+                break
+        else:
+            after.append(iface)
+    _classImplements_ordered(spec, tuple(before), tuple(after))
+
+
+def classImplementsFirst(cls, iface):
+    """
+    Declare that instances of *cls* additionally provide *iface*.
+
+    The second argument is an interface or interface specification.
+    It is added as the highest priority (first in the IRO) interface;
+    no attempt is made to keep a consistent resolution order.
+
+    .. versionadded:: 5.0.0
+    """
+    spec = implementedBy(cls)
+    _classImplements_ordered(spec, (iface,), ())
+
+
+def _classImplements_ordered(spec, before=(), after=()):
+    # Elide everything already inherited.
+    # Except, if it is the root, and we don't already declare anything else
+    # that would imply it, allow the root through. (TODO: When we disallow non-strict
+    # IRO, this part of the check can be removed because it's not possible to re-declare
+    # like that.)
+    before = [
+        x
+        for x in before
+        if not spec.isOrExtends(x) or (x is Interface and not spec.declared)
+    ]
+    after = [
+        x
+        for x in after
+        if not spec.isOrExtends(x) or (x is Interface and not spec.declared)
+    ]
+
+    # eliminate duplicates
+    new_declared = []
+    seen = set()
+    for l in before, spec.declared, after:
+        for b in l:
+            if b not in seen:
+                new_declared.append(b)
+                seen.add(b)
+
+    spec.declared = tuple(new_declared)
+
+    # compute the bases
+    bases = new_declared # guaranteed no dupes
+
+    if spec.inherit is not None:
+        for c in spec.inherit.__bases__:
+            b = implementedBy(c)
+            if b not in seen:
+                seen.add(b)
+                bases.append(b)
+
+    spec.__bases__ = tuple(bases)
+
+
+def _implements_advice(cls):
+    interfaces, do_classImplements = cls.__dict__['__implements_advice_data__']
+    del cls.__implements_advice_data__
+    do_classImplements(cls, *interfaces)
+    return cls
+
+
+class implementer:
+    """
+    Declare the interfaces implemented by instances of a class.
+
+    This function is called as a class decorator.
+
+    The arguments are one or more interfaces or interface
+    specifications (`~zope.interface.interfaces.IDeclaration`
+    objects).
+
+    The interfaces given (including the interfaces in the
+    specifications) are added to any interfaces previously declared,
+    unless the interface is already implemented.
+
+    Previous declarations include declarations for base classes unless
+    implementsOnly was used.
+
+    This function is provided for convenience. It provides a more
+    convenient way to call `classImplements`. For example::
+
+        @implementer(I1)
+        class C(object):
+            pass
+
+    is equivalent to calling::
+
+        classImplements(C, I1)
+
+    after the class has been created.
+
+    .. seealso:: `classImplements`
+       The change history provided there applies to this function too.
+    """
+    __slots__ = ('interfaces',)
+
+    def __init__(self, *interfaces):
+        self.interfaces = interfaces
+
+    def __call__(self, ob):
+        if isinstance(ob, type):
+            # This is the common branch for classes.
+            classImplements(ob, *self.interfaces)
+            return ob
+
+        spec_name = _implements_name(ob)
+        spec = Implements.named(spec_name, *self.interfaces)
+        try:
+            ob.__implemented__ = spec
+        except AttributeError:
+            raise TypeError("Can't declare implements", ob)
+        return ob
+
+class implementer_only:
+    """Declare the only interfaces implemented by instances of a class
+
+      This function is called as a class decorator.
+
+      The arguments are one or more interfaces or interface
+      specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+      Previous declarations including declarations for base classes
+      are overridden.
+
+      This function is provided for convenience. It provides a more
+      convenient way to call `classImplementsOnly`. For example::
+
+        @implementer_only(I1)
+        class C(object): pass
+
+      is equivalent to calling::
+
+        classImplementsOnly(I1)
+
+      after the class has been created.
+      """
+
+    def __init__(self, *interfaces):
+        self.interfaces = interfaces
+
+    def __call__(self, ob):
+        if isinstance(ob, (FunctionType, MethodType)):
+            # XXX Does this decorator make sense for anything but classes?
+            # I don't think so. There can be no inheritance of interfaces
+            # on a method or function....
+            raise ValueError('The implementer_only decorator is not '
+                             'supported for methods or functions.')
+
+        # Assume it's a class:
+        classImplementsOnly(ob, *self.interfaces)
+        return ob
+
+
+##############################################################################
+#
+# Instance declarations
+
+class Provides(Declaration):  # Really named ProvidesClass
+    """Implement ``__provides__``, the instance-specific specification
+
+    When an object is pickled, we pickle the interfaces that it implements.
+    """
+
+    def __init__(self, cls, *interfaces):
+        self.__args = (cls, ) + interfaces
+        self._cls = cls
+        Declaration.__init__(self, *self._add_interfaces_to_cls(interfaces, cls))
+
+    # Added to by ``moduleProvides``, et al
+    _v_module_names = ()
+
+    def __repr__(self):
+        # The typical way to create instances of this
+        # object is via calling ``directlyProvides(...)`` or ``alsoProvides()``,
+        # but that's not the only way. Proxies, for example,
+        # directly use the ``Provides(...)`` function (which is the
+        # more generic method, and what we pickle as). We're after the most
+        # readable, useful repr in the common case, so we use the most
+        # common name.
+        #
+        # We also cooperate with ``moduleProvides`` to attempt to do the
+        # right thing for that API. See it for details.
+        function_name = 'directlyProvides'
+        if self._cls is ModuleType and self._v_module_names:
+            # See notes in ``moduleProvides``/``directlyProvides``
+            providing_on_module = True
+            interfaces = self.__args[1:]
+        else:
+            providing_on_module = False
+            interfaces = (self._cls,) + self.__bases__
+        ordered_names = self._argument_names_for_repr(interfaces)
+        if providing_on_module:
+            mod_names = self._v_module_names
+            if len(mod_names) == 1:
+                mod_names = "sys.modules[%r]" % mod_names[0]
+            ordered_names = (
+                '{}, '.format(mod_names)
+            ) + ordered_names
+        return "{}({})".format(
+            function_name,
+            ordered_names,
+        )
+
+    def __reduce__(self):
+        # This reduces to the Provides *function*, not
+        # this class.
+        return Provides, self.__args
+
+    __module__ = 'zope.interface'
+
+    def __get__(self, inst, cls):
+        """Make sure that a class __provides__ doesn't leak to an instance
+        """
+        if inst is None and cls is self._cls:
+            # We were accessed through a class, so we are the class'
+            # provides spec. Just return this object, but only if we are
+            # being called on the same class that we were defined for:
+            return self
+
+        raise AttributeError('__provides__')
+
+ProvidesClass = Provides
+
+# Registry of instance declarations
+# This is a memory optimization to allow objects to share specifications.
+InstanceDeclarations = weakref.WeakValueDictionary()
+
+def Provides(*interfaces): # pylint:disable=function-redefined
+    """Declaration for an instance of *cls*.
+
+       The correct signature is ``cls, *interfaces``.
+       The *cls* is necessary to avoid the
+       construction of inconsistent resolution orders.
+
+      Instance declarations are shared among instances that have the same
+      declaration. The declarations are cached in a weak value dictionary.
+    """
+    spec = InstanceDeclarations.get(interfaces)
+    if spec is None:
+        spec = ProvidesClass(*interfaces)
+        InstanceDeclarations[interfaces] = spec
+
+    return spec
+
+Provides.__safe_for_unpickling__ = True
+
+
+def directlyProvides(object, *interfaces): # pylint:disable=redefined-builtin
+    """Declare interfaces declared directly for an object
+
+      The arguments after the object are one or more interfaces or interface
+      specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+      The interfaces given (including the interfaces in the specifications)
+      replace interfaces previously declared for the object.
+    """
+    cls = getattr(object, '__class__', None)
+    if cls is not None and getattr(cls, '__class__', None) is cls:
+        # It's a meta class (well, at least it it could be an extension class)
+        # Note that we can't get here from the tests:  there is no normal
+        # class which isn't descriptor aware.
+        if not isinstance(object, type):
+            raise TypeError("Attempt to make an interface declaration on a "
+                            "non-descriptor-aware class")
+
+    interfaces = _normalizeargs(interfaces)
+    if cls is None:
+        cls = type(object)
+
+    if issubclass(cls, type):
+        # we have a class or type.  We'll use a special descriptor
+        # that provides some extra caching
+        object.__provides__ = ClassProvides(object, cls, *interfaces)
+    else:
+        provides = object.__provides__ = Provides(cls, *interfaces)
+        # See notes in ``moduleProvides``.
+        if issubclass(cls, ModuleType) and hasattr(object, '__name__'):
+            provides._v_module_names += (object.__name__,)
+
+
+
+def alsoProvides(object, *interfaces): # pylint:disable=redefined-builtin
+    """Declare interfaces declared directly for an object
+
+    The arguments after the object are one or more interfaces or interface
+    specifications (`~zope.interface.interfaces.IDeclaration` objects).
+
+    The interfaces given (including the interfaces in the specifications) are
+    added to the interfaces previously declared for the object.
+    """
+    directlyProvides(object, directlyProvidedBy(object), *interfaces)
+
+
+def noLongerProvides(object, interface): # pylint:disable=redefined-builtin
+    """ Removes a directly provided interface from an object.
+    """
+    directlyProvides(object, directlyProvidedBy(object) - interface)
+    if interface.providedBy(object):
+        raise ValueError("Can only remove directly provided interfaces.")
+
+
+@_use_c_impl
+class ClassProvidesBase(SpecificationBase):
+
+    __slots__ = (
+        '_cls',
+        '_implements',
+    )
+
+    def __get__(self, inst, cls):
+        # member slots are set by subclass
+        # pylint:disable=no-member
+        if cls is self._cls:
+            # We only work if called on the class we were defined for
+
+            if inst is None:
+                # We were accessed through a class, so we are the class'
+                # provides spec. Just return this object as is:
+                return self
+
+            return self._implements
+
+        raise AttributeError('__provides__')
+
+
+class ClassProvides(Declaration, ClassProvidesBase):
+    """Special descriptor for class ``__provides__``
+
+    The descriptor caches the implementedBy info, so that
+    we can get declarations for objects without instance-specific
+    interfaces a bit quicker.
+    """
+
+    __slots__ = (
+        '__args',
+    )
+
+    def __init__(self, cls, metacls, *interfaces):
+        self._cls = cls
+        self._implements = implementedBy(cls)
+        self.__args = (cls, metacls, ) + interfaces
+        Declaration.__init__(self, *self._add_interfaces_to_cls(interfaces, metacls))
+
+    def __repr__(self):
+        # There are two common ways to get instances of this object:
+        # The most interesting way is calling ``@provider(..)`` as a decorator
+        # of a class; this is the same as calling ``directlyProvides(cls, ...)``.
+        #
+        # The other way is by default: anything that invokes ``implementedBy(x)``
+        # will wind up putting an instance in ``type(x).__provides__``; this includes
+        # the ``@implementer(...)`` decorator. Those instances won't have any
+        # interfaces.
+        #
+        # Thus, as our repr, we go with the ``directlyProvides()`` syntax.
+        interfaces = (self._cls, ) + self.__args[2:]
+        ordered_names = self._argument_names_for_repr(interfaces)
+        return "directlyProvides({})".format(ordered_names)
+
+    def __reduce__(self):
+        return self.__class__, self.__args
+
+    # Copy base-class method for speed
+    __get__ = ClassProvidesBase.__get__
+
+
+def directlyProvidedBy(object): # pylint:disable=redefined-builtin
+    """Return the interfaces directly provided by the given object
+
+    The value returned is an `~zope.interface.interfaces.IDeclaration`.
+    """
+    provides = getattr(object, "__provides__", None)
+    if (
+            provides is None # no spec
+            # We might have gotten the implements spec, as an
+            # optimization. If so, it's like having only one base, that we
+            # lop off to exclude class-supplied declarations:
+            or isinstance(provides, Implements)
+    ):
+        return _empty
+
+    # Strip off the class part of the spec:
+    return Declaration(provides.__bases__[:-1])
+
+
+class provider:
+    """Declare interfaces provided directly by a class
+
+      This function is called in a class definition.
+
+      The arguments are one or more interfaces or interface specifications
+      (`~zope.interface.interfaces.IDeclaration` objects).
+
+      The given interfaces (including the interfaces in the specifications)
+      are used to create the class's direct-object interface specification.
+      An error will be raised if the module class has an direct interface
+      specification. In other words, it is an error to call this function more
+      than once in a class definition.
+
+      Note that the given interfaces have nothing to do with the interfaces
+      implemented by instances of the class.
+
+      This function is provided for convenience. It provides a more convenient
+      way to call `directlyProvides` for a class. For example::
+
+        @provider(I1)
+        class C:
+            pass
+
+      is equivalent to calling::
+
+        directlyProvides(C, I1)
+
+      after the class has been created.
+    """
+
+    def __init__(self, *interfaces):
+        self.interfaces = interfaces
+
+    def __call__(self, ob):
+        directlyProvides(ob, *self.interfaces)
+        return ob
+
+
+def moduleProvides(*interfaces):
+    """Declare interfaces provided by a module
+
+    This function is used in a module definition.
+
+    The arguments are one or more interfaces or interface specifications
+    (`~zope.interface.interfaces.IDeclaration` objects).
+
+    The given interfaces (including the interfaces in the specifications) are
+    used to create the module's direct-object interface specification.  An
+    error will be raised if the module already has an interface specification.
+    In other words, it is an error to call this function more than once in a
+    module definition.
+
+    This function is provided for convenience. It provides a more convenient
+    way to call directlyProvides. For example::
+
+      moduleProvides(I1)
+
+    is equivalent to::
+
+      directlyProvides(sys.modules[__name__], I1)
+    """
+    frame = sys._getframe(1) # pylint:disable=protected-access
+    locals = frame.f_locals # pylint:disable=redefined-builtin
+
+    # Try to make sure we were called from a module body
+    if (locals is not frame.f_globals) or ('__name__' not in locals):
+        raise TypeError(
+            "moduleProvides can only be used from a module definition.")
+
+    if '__provides__' in locals:
+        raise TypeError(
+            "moduleProvides can only be used once in a module definition.")
+
+    # Note: This is cached based on the key ``(ModuleType, *interfaces)``;
+    # One consequence is that any module that provides the same interfaces
+    # gets the same ``__repr__``, meaning that you can't tell what module
+    # such a declaration came from. Adding the module name to ``_v_module_names``
+    # attempts to correct for this; it works in some common situations, but fails
+    # (1) after pickling (the data is lost) and (2) if declarations are
+    # actually shared and (3) if the alternate spelling of ``directlyProvides()``
+    # is used. Problem (3)  is fixed by cooperating with ``directlyProvides``
+    # to maintain this information, and problem (2) is worked around by
+    # printing all the names, but (1) is unsolvable without introducing
+    # new classes or changing the stored data...but it doesn't actually matter,
+    # because ``ModuleType`` can't be pickled!
+    p = locals["__provides__"] = Provides(ModuleType,
+                                          *_normalizeargs(interfaces))
+    p._v_module_names += (locals['__name__'],)
+
+
+##############################################################################
+#
+# Declaration querying support
+
+# XXX:  is this a fossil?  Nobody calls it, no unit tests exercise it, no
+#       doctests import it, and the package __init__ doesn't import it.
+#       (Answer: Versions of zope.container prior to 4.4.0 called this,
+#        and zope.proxy.decorator up through at least 4.3.5 called this.)
+def ObjectSpecification(direct, cls):
+    """Provide object specifications
+
+    These combine information for the object and for it's classes.
+    """
+    return Provides(cls, direct) # pragma: no cover fossil
+
+@_use_c_impl
+def getObjectSpecification(ob):
+    try:
+        provides = ob.__provides__
+    except AttributeError:
+        provides = None
+
+    if provides is not None:
+        if isinstance(provides, SpecificationBase):
+            return provides
+
+    try:
+        cls = ob.__class__
+    except AttributeError:
+        # We can't get the class, so just consider provides
+        return _empty
+    return implementedBy(cls)
+
+
+@_use_c_impl
+def providedBy(ob):
+    """
+    Return the interfaces provided by *ob*.
+
+    If *ob* is a :class:`super` object, then only interfaces implemented
+    by the remainder of the classes in the method resolution order are
+    considered. Interfaces directly provided by the object underlying *ob*
+    are not.
+    """
+    # Here we have either a special object, an old-style declaration
+    # or a descriptor
+
+    # Try to get __providedBy__
+    try:
+        if isinstance(ob, super): # Some objects raise errors on isinstance()
+            return implementedBy(ob)
+
+        r = ob.__providedBy__
+    except AttributeError:
+        # Not set yet. Fall back to lower-level thing that computes it
+        return getObjectSpecification(ob)
+
+    try:
+        # We might have gotten a descriptor from an instance of a
+        # class (like an ExtensionClass) that doesn't support
+        # descriptors.  We'll make sure we got one by trying to get
+        # the only attribute, which all specs have.
+        r.extends
+    except AttributeError:
+
+        # The object's class doesn't understand descriptors.
+        # Sigh. We need to get an object descriptor, but we have to be
+        # careful.  We want to use the instance's __provides__, if
+        # there is one, but only if it didn't come from the class.
+
+        try:
+            r = ob.__provides__
+        except AttributeError:
+            # No __provides__, so just fall back to implementedBy
+            return implementedBy(ob.__class__)
+
+        # We need to make sure we got the __provides__ from the
+        # instance. We'll do this by making sure we don't get the same
+        # thing from the class:
+
+        try:
+            cp = ob.__class__.__provides__
+        except AttributeError:
+            # The ob doesn't have a class or the class has no
+            # provides, assume we're done:
+            return r
+
+        if r is cp:
+            # Oops, we got the provides from the class. This means
+            # the object doesn't have it's own. We should use implementedBy
+            return implementedBy(ob.__class__)
+
+    return r
+
+
+@_use_c_impl
+class ObjectSpecificationDescriptor:
+    """Implement the ``__providedBy__`` attribute
+
+    The ``__providedBy__`` attribute computes the interfaces provided by
+    an object. If an object has an ``__provides__`` attribute, that is returned.
+    Otherwise, `implementedBy` the *cls* is returned.
+
+    .. versionchanged:: 5.4.0
+       Both the default (C) implementation and the Python implementation
+       now let exceptions raised by accessing ``__provides__`` propagate.
+       Previously, the C version ignored all exceptions.
+    .. versionchanged:: 5.4.0
+       The Python implementation now matches the C implementation and lets
+       a ``__provides__`` of ``None`` override what the class is declared to
+       implement.
+    """
+
+    def __get__(self, inst, cls):
+        """Get an object specification for an object
+        """
+        if inst is None:
+            return getObjectSpecification(cls)
+
+        try:
+            return inst.__provides__
+        except AttributeError:
+            return implementedBy(cls)
+
+
+##############################################################################
+
+def _normalizeargs(sequence, output=None):
+    """Normalize declaration arguments
+
+    Normalization arguments might contain Declarions, tuples, or single
+    interfaces.
+
+    Anything but individual interfaces or implements specs will be expanded.
+    """
+    if output is None:
+        output = []
+
+    cls = sequence.__class__
+    if InterfaceClass in cls.__mro__ or Implements in cls.__mro__:
+        output.append(sequence)
+    else:
+        for v in sequence:
+            _normalizeargs(v, output)
+
+    return output
+
+_empty = _ImmutableDeclaration()
+
+objectSpecificationDescriptor = ObjectSpecificationDescriptor()
diff --git a/contrib/python/zope.interface/py3/zope/interface/document.py b/contrib/python/zope.interface/py3/zope/interface/document.py
new file mode 100644
index 00000000000..84cfaa0b718
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/document.py
@@ -0,0 +1,124 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+""" Pretty-Print an Interface object as structured text (Yum)
+
+This module provides a function, asStructuredText, for rendering an
+interface as structured text.
+"""
+import zope.interface
+
+__all__ = [
+    'asReStructuredText',
+    'asStructuredText',
+]
+
+def asStructuredText(I, munge=0, rst=False):
+    """ Output structured text format.  Note, this will whack any existing
+    'structured' format of the text.
+
+    If `rst=True`, then the output will quote all code as inline literals in
+    accordance with 'reStructuredText' markup principles.
+    """
+
+    if rst:
+        inline_literal = lambda s: "``{}``".format(s)
+    else:
+        inline_literal = lambda s: s
+
+    r = [inline_literal(I.getName())]
+    outp = r.append
+    level = 1
+
+    if I.getDoc():
+        outp(_justify_and_indent(_trim_doc_string(I.getDoc()), level))
+
+    bases = [base
+             for base in I.__bases__
+             if base is not zope.interface.Interface
+             ]
+    if bases:
+        outp(_justify_and_indent("This interface extends:", level, munge))
+        level += 1
+        for b in bases:
+            item = "o %s" % inline_literal(b.getName())
+            outp(_justify_and_indent(_trim_doc_string(item), level, munge))
+        level -= 1
+
+    namesAndDescriptions = sorted(I.namesAndDescriptions())
+
+    outp(_justify_and_indent("Attributes:", level, munge))
+    level += 1
+    for name, desc in namesAndDescriptions:
+        if not hasattr(desc, 'getSignatureString'):   # ugh...
+            item = "{} -- {}".format(inline_literal(desc.getName()),
+                                 desc.getDoc() or 'no documentation')
+            outp(_justify_and_indent(_trim_doc_string(item), level, munge))
+    level -= 1
+
+    outp(_justify_and_indent("Methods:", level, munge))
+    level += 1
+    for name, desc in namesAndDescriptions:
+        if hasattr(desc, 'getSignatureString'):   # ugh...
+            _call = "{}{}".format(desc.getName(), desc.getSignatureString())
+            item = "{} -- {}".format(inline_literal(_call),
+                                 desc.getDoc() or 'no documentation')
+            outp(_justify_and_indent(_trim_doc_string(item), level, munge))
+
+    return "\n\n".join(r) + "\n\n"
+
+
+def asReStructuredText(I, munge=0):
+    """ Output reStructuredText format.  Note, this will whack any existing
+    'structured' format of the text."""
+    return asStructuredText(I, munge=munge, rst=True)
+
+
+def _trim_doc_string(text):
+    """ Trims a doc string to make it format
+    correctly with structured text. """
+
+    lines = text.replace('\r\n', '\n').split('\n')
+    nlines = [lines.pop(0)]
+    if lines:
+        min_indent = min([len(line) - len(line.lstrip())
+                          for line in lines])
+        for line in lines:
+            nlines.append(line[min_indent:])
+
+    return '\n'.join(nlines)
+
+
+def _justify_and_indent(text, level, munge=0, width=72):
+    """ indent and justify text, rejustify (munge) if specified """
+
+    indent = " " * level
+
+    if munge:
+        lines = []
+        line = indent
+        text = text.split()
+
+        for word in text:
+            line = ' '.join([line, word])
+            if len(line) > width:
+                lines.append(line)
+                line = indent
+        else:
+            lines.append(line)
+
+        return '\n'.join(lines)
+
+    else:
+        return indent + \
+            text.strip().replace("\r\n", "\n") .replace("\n", "\n" + indent)
diff --git a/contrib/python/zope.interface/py3/zope/interface/exceptions.py b/contrib/python/zope.interface/py3/zope/interface/exceptions.py
new file mode 100644
index 00000000000..d5c234a6dab
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/exceptions.py
@@ -0,0 +1,275 @@
+##############################################################################
+#
+# Copyright (c) 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interface-specific exceptions
+"""
+
+__all__ = [
+    # Invalid tree
+    'Invalid',
+    'DoesNotImplement',
+    'BrokenImplementation',
+    'BrokenMethodImplementation',
+    'MultipleInvalid',
+    # Other
+    'BadImplements',
+    'InvalidInterface',
+]
+
+class Invalid(Exception):
+    """A specification is violated
+    """
+
+
+class _TargetInvalid(Invalid):
+    # Internal use. Subclass this when you're describing
+    # a particular target object that's invalid according
+    # to a specific interface.
+    #
+    # For backwards compatibility, the *target* and *interface* are
+    # optional, and the signatures are inconsistent in their ordering.
+    #
+    # We deal with the inconsistency in ordering by defining the index
+    # of the two values in ``self.args``. *target* uses a marker object to
+    # distinguish "not given" from "given, but None", because the latter
+    # can be a value that gets passed to validation. For this reason, it must
+    # always be the last argument (we detect absence by the ``IndexError``).
+
+    _IX_INTERFACE = 0
+    _IX_TARGET = 1
+    # The exception to catch when indexing self.args indicating that
+    # an argument was not given. If all arguments are expected,
+    # a subclass should set this to ().
+    _NOT_GIVEN_CATCH = IndexError
+    _NOT_GIVEN = '<Not Given>'
+
+    def _get_arg_or_default(self, ix, default=None):
+        try:
+            return self.args[ix] # pylint:disable=unsubscriptable-object
+        except self._NOT_GIVEN_CATCH:
+            return default
+
+    @property
+    def interface(self):
+        return self._get_arg_or_default(self._IX_INTERFACE)
+
+    @property
+    def target(self):
+        return self._get_arg_or_default(self._IX_TARGET, self._NOT_GIVEN)
+
+    ###
+    # str
+    #
+    # The ``__str__`` of self is implemented by concatenating (%s), in order,
+    # these properties (none of which should have leading or trailing
+    # whitespace):
+    #
+    # - self._str_subject
+    #   Begin the message, including a description of the target.
+    # - self._str_description
+    #   Provide a general description of the type of error, including
+    #   the interface name if possible and relevant.
+    # - self._str_conjunction
+    #   Join the description to the details. Defaults to ": ".
+    # - self._str_details
+    #   Provide details about how this particular instance of the error.
+    # - self._str_trailer
+    #   End the message. Usually just a period.
+    ###
+
+    @property
+    def _str_subject(self):
+        target = self.target
+        if target is self._NOT_GIVEN:
+            return "An object"
+        return "The object {!r}".format(target)
+
+    @property
+    def _str_description(self):
+        return "has failed to implement interface %s" % (
+            self.interface or '<Unknown>'
+        )
+
+    _str_conjunction = ": "
+    _str_details = "<unknown>"
+    _str_trailer = '.'
+
+    def __str__(self):
+        return "{} {}{}{}{}".format(
+            self._str_subject,
+            self._str_description,
+            self._str_conjunction,
+            self._str_details,
+            self._str_trailer
+        )
+
+
+class DoesNotImplement(_TargetInvalid):
+    """
+    DoesNotImplement(interface[, target])
+
+    The *target* (optional) does not implement the *interface*.
+
+    .. versionchanged:: 5.0.0
+       Add the *target* argument and attribute, and change the resulting
+       string value of this object accordingly.
+    """
+
+    _str_details = "Does not declaratively implement the interface"
+
+
+class BrokenImplementation(_TargetInvalid):
+    """
+    BrokenImplementation(interface, name[, target])
+
+    The *target* (optional) is missing the attribute *name*.
+
+    .. versionchanged:: 5.0.0
+       Add the *target* argument and attribute, and change the resulting
+       string value of this object accordingly.
+
+       The *name* can either be a simple string or a ``Attribute`` object.
+    """
+
+    _IX_NAME = _TargetInvalid._IX_INTERFACE + 1
+    _IX_TARGET = _IX_NAME + 1
+
+    @property
+    def name(self):
+        return self.args[1] # pylint:disable=unsubscriptable-object
+
+    @property
+    def _str_details(self):
+        return "The %s attribute was not provided" % (
+            repr(self.name) if isinstance(self.name, str) else self.name
+        )
+
+
+class BrokenMethodImplementation(_TargetInvalid):
+    """
+    BrokenMethodImplementation(method, message[, implementation, interface, target])
+
+    The *target* (optional) has a *method* in *implementation* that violates
+    its contract in a way described by *mess*.
+
+    .. versionchanged:: 5.0.0
+       Add the *interface* and *target* argument and attribute,
+       and change the resulting string value of this object accordingly.
+
+       The *method* can either be a simple string or a ``Method`` object.
+
+    .. versionchanged:: 5.0.0
+       If *implementation* is given, then the *message* will have the
+       string "implementation" replaced with an short but informative
+       representation of *implementation*.
+
+    """
+
+    _IX_IMPL = 2
+    _IX_INTERFACE = _IX_IMPL + 1
+    _IX_TARGET = _IX_INTERFACE + 1
+
+    @property
+    def method(self):
+        return self.args[0] # pylint:disable=unsubscriptable-object
+
+    @property
+    def mess(self):
+        return self.args[1] # pylint:disable=unsubscriptable-object
+
+    @staticmethod
+    def __implementation_str(impl):
+        # It could be a callable or some arbitrary object, we don't
+        # know yet.
+        import inspect # Inspect is a heavy-weight dependency, lots of imports
+        try:
+            sig = inspect.signature
+            formatsig = str
+        except AttributeError:
+            sig = inspect.getargspec
+            f = inspect.formatargspec
+            formatsig = lambda sig: f(*sig) # pylint:disable=deprecated-method
+
+        try:
+            sig = sig(impl)
+        except (ValueError, TypeError):
+            # Unable to introspect. Darn.
+            # This could be a non-callable, or a particular builtin,
+            # or a bound method that doesn't even accept 'self', e.g.,
+            # ``Class.method = lambda: None; Class().method``
+            return repr(impl)
+
+        try:
+            name = impl.__qualname__
+        except AttributeError:
+            name = impl.__name__
+
+        return name + formatsig(sig)
+
+    @property
+    def _str_details(self):
+        impl = self._get_arg_or_default(self._IX_IMPL, self._NOT_GIVEN)
+        message = self.mess
+        if impl is not self._NOT_GIVEN and 'implementation' in message:
+            message = message.replace("implementation", '%r')
+            message = message % (self.__implementation_str(impl),)
+
+        return 'The contract of {} is violated because {}'.format(
+            repr(self.method) if isinstance(self.method, str) else self.method,
+            message,
+        )
+
+
+class MultipleInvalid(_TargetInvalid):
+    """
+    The *target* has failed to implement the *interface* in
+    multiple ways.
+
+    The failures are described by *exceptions*, a collection of
+    other `Invalid` instances.
+
+    .. versionadded:: 5.0
+    """
+
+    _NOT_GIVEN_CATCH = ()
+
+    def __init__(self, interface, target, exceptions):
+        super().__init__(interface, target, tuple(exceptions))
+
+    @property
+    def exceptions(self):
+        return self.args[2] # pylint:disable=unsubscriptable-object
+
+    @property
+    def _str_details(self):
+        # It would be nice to use tabs here, but that
+        # is hard to represent in doctests.
+        return '\n    ' + '\n    '.join(
+            x._str_details.strip() if isinstance(x, _TargetInvalid) else str(x)
+            for x in self.exceptions
+        )
+
+    _str_conjunction = ':' # We don't want a trailing space, messes up doctests
+    _str_trailer = ''
+
+
+class InvalidInterface(Exception):
+    """The interface has invalid contents
+    """
+
+class BadImplements(TypeError):
+    """An implementation assertion is invalid
+
+    because it doesn't contain an interface or a sequence of valid
+    implementation assertions.
+    """
diff --git a/contrib/python/zope.interface/py3/zope/interface/interface.py b/contrib/python/zope.interface/py3/zope/interface/interface.py
new file mode 100644
index 00000000000..1bd6f9e818f
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/interface.py
@@ -0,0 +1,1131 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interface object implementation
+"""
+# pylint:disable=protected-access
+import sys
+from types import MethodType
+from types import FunctionType
+import weakref
+
+from zope.interface._compat import _use_c_impl
+from zope.interface.exceptions import Invalid
+from zope.interface.ro import ro as calculate_ro
+from zope.interface import ro
+
+__all__ = [
+    # Most of the public API from this module is directly exported
+    # from zope.interface. The only remaining public API intended to
+    # be imported from here should be those few things documented as
+    # such.
+    'InterfaceClass',
+    'Specification',
+    'adapter_hooks',
+]
+
+CO_VARARGS = 4
+CO_VARKEYWORDS = 8
+# Put in the attrs dict of an interface by ``taggedValue`` and ``invariants``
+TAGGED_DATA = '__interface_tagged_values__'
+# Put in the attrs dict of an interface by ``interfacemethod``
+INTERFACE_METHODS = '__interface_methods__'
+
+_decorator_non_return = object()
+_marker = object()
+
+
+
+def invariant(call):
+    f_locals = sys._getframe(1).f_locals
+    tags = f_locals.setdefault(TAGGED_DATA, {})
+    invariants = tags.setdefault('invariants', [])
+    invariants.append(call)
+    return _decorator_non_return
+
+
+def taggedValue(key, value):
+    """Attaches a tagged value to an interface at definition time."""
+    f_locals = sys._getframe(1).f_locals
+    tagged_values = f_locals.setdefault(TAGGED_DATA, {})
+    tagged_values[key] = value
+    return _decorator_non_return
+
+
+class Element:
+    """
+    Default implementation of `zope.interface.interfaces.IElement`.
+    """
+
+    # We can't say this yet because we don't have enough
+    # infrastructure in place.
+    #
+    #implements(IElement)
+
+    def __init__(self, __name__, __doc__=''): # pylint:disable=redefined-builtin
+        if not __doc__ and __name__.find(' ') >= 0:
+            __doc__ = __name__
+            __name__ = None
+
+        self.__name__ = __name__
+        self.__doc__ = __doc__
+        # Tagged values are rare, especially on methods or attributes.
+        # Deferring the allocation can save substantial memory.
+        self.__tagged_values = None
+
+    def getName(self):
+        """ Returns the name of the object. """
+        return self.__name__
+
+    def getDoc(self):
+        """ Returns the documentation for the object. """
+        return self.__doc__
+
+    ###
+    # Tagged values.
+    #
+    # Direct tagged values are set only in this instance. Others
+    # may be inherited (for those subclasses that have that concept).
+    ###
+
+    def getTaggedValue(self, tag):
+        """ Returns the value associated with 'tag'. """
+        if not self.__tagged_values:
+            raise KeyError(tag)
+        return self.__tagged_values[tag]
+
+    def queryTaggedValue(self, tag, default=None):
+        """ Returns the value associated with 'tag'. """
+        return self.__tagged_values.get(tag, default) if self.__tagged_values else default
+
+    def getTaggedValueTags(self):
+        """ Returns a collection of all tags. """
+        return self.__tagged_values.keys() if self.__tagged_values else ()
+
+    def setTaggedValue(self, tag, value):
+        """ Associates 'value' with 'key'. """
+        if self.__tagged_values is None:
+            self.__tagged_values = {}
+        self.__tagged_values[tag] = value
+
+    queryDirectTaggedValue = queryTaggedValue
+    getDirectTaggedValue = getTaggedValue
+    getDirectTaggedValueTags = getTaggedValueTags
+
+
+SpecificationBasePy = object # filled by _use_c_impl.
+
+
+@_use_c_impl
+class SpecificationBase:
+    # This object is the base of the inheritance hierarchy for ClassProvides:
+    #
+    # ClassProvides < ClassProvidesBase, Declaration
+    # Declaration < Specification < SpecificationBase
+    # ClassProvidesBase < SpecificationBase
+    #
+    # In order to have compatible instance layouts, we need to declare
+    # the storage used by Specification and Declaration here (and
+    # those classes must have ``__slots__ = ()``); fortunately this is
+    # not a waste of space because those are the only two inheritance
+    # trees. These all translate into tp_members in C.
+    __slots__ = (
+        # Things used here.
+        '_implied',
+        # Things used in Specification.
+        '_dependents',
+        '_bases',
+        '_v_attrs',
+        '__iro__',
+        '__sro__',
+        '__weakref__',
+    )
+
+    def providedBy(self, ob):
+        """Is the interface implemented by an object
+        """
+        spec = providedBy(ob)
+        return self in spec._implied
+
+    def implementedBy(self, cls):
+        """Test whether the specification is implemented by a class or factory.
+
+        Raise TypeError if argument is neither a class nor a callable.
+        """
+        spec = implementedBy(cls)
+        return self in spec._implied
+
+    def isOrExtends(self, interface):
+        """Is the interface the same as or extend the given interface
+        """
+        return interface in self._implied # pylint:disable=no-member
+
+    __call__ = isOrExtends
+
+
+class NameAndModuleComparisonMixin:
+    # Internal use. Implement the basic sorting operators (but not (in)equality
+    # or hashing). Subclasses must provide ``__name__`` and ``__module__``
+    # attributes. Subclasses will be mutually comparable; but because equality
+    # and hashing semantics are missing from this class, take care in how
+    # you define those two attributes: If you stick with the default equality
+    # and hashing (identity based) you should make sure that all possible ``__name__``
+    # and ``__module__`` pairs are unique ACROSS ALL SUBCLASSES. (Actually, pretty
+    # much the same thing goes if you define equality and hashing to be based on
+    # those two attributes: they must still be consistent ACROSS ALL SUBCLASSES.)
+
+    # pylint:disable=assigning-non-slot
+    __slots__ = ()
+
+    def _compare(self, other):
+        """
+        Compare *self* to *other* based on ``__name__`` and ``__module__``.
+
+        Return 0 if they are equal, return 1 if *self* is
+        greater than *other*, and return -1 if *self* is less than
+        *other*.
+
+        If *other* does not have ``__name__`` or ``__module__``, then
+        return ``NotImplemented``.
+
+        .. caution::
+           This allows comparison to things well outside the type hierarchy,
+           perhaps not symmetrically.
+
+           For example, ``class Foo(object)`` and ``class Foo(Interface)``
+           in the same file would compare equal, depending on the order of
+           operands. Writing code like this by hand would be unusual, but it could
+           happen with dynamic creation of types and interfaces.
+
+        None is treated as a pseudo interface that implies the loosest
+        contact possible, no contract. For that reason, all interfaces
+        sort before None.
+        """
+        if other is self:
+            return 0
+
+        if other is None:
+            return -1
+
+        n1 = (self.__name__, self.__module__)
+        try:
+            n2 = (other.__name__, other.__module__)
+        except AttributeError:
+            return NotImplemented
+
+        # This spelling works under Python3, which doesn't have cmp().
+        return (n1 > n2) - (n1 < n2)
+
+    def __lt__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c < 0
+
+    def __le__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c <= 0
+
+    def __gt__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c > 0
+
+    def __ge__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c >= 0
+
+
+@_use_c_impl
+class InterfaceBase(NameAndModuleComparisonMixin, SpecificationBasePy):
+    """Base class that wants to be replaced with a C base :)
+    """
+
+    __slots__ = (
+        '__name__',
+        '__ibmodule__',
+        '_v_cached_hash',
+    )
+
+    def __init__(self, name=None, module=None):
+        self.__name__ = name
+        self.__ibmodule__ = module
+
+    def _call_conform(self, conform):
+        raise NotImplementedError
+
+    @property
+    def __module_property__(self):
+        # This is for _InterfaceMetaClass
+        return self.__ibmodule__
+
+    def __call__(self, obj, alternate=_marker):
+        """Adapt an object to the interface
+        """
+        try:
+            conform = obj.__conform__
+        except AttributeError:
+            conform = None
+
+        if conform is not None:
+            adapter = self._call_conform(conform)
+            if adapter is not None:
+                return adapter
+
+        adapter = self.__adapt__(obj)
+
+        if adapter is not None:
+            return adapter
+        if alternate is not _marker:
+            return alternate
+        raise TypeError("Could not adapt", obj, self)
+
+    def __adapt__(self, obj):
+        """Adapt an object to the receiver
+        """
+        if self.providedBy(obj):
+            return obj
+
+        for hook in adapter_hooks:
+            adapter = hook(self, obj)
+            if adapter is not None:
+                return adapter
+
+        return None
+
+    def __hash__(self):
+        # pylint:disable=assigning-non-slot,attribute-defined-outside-init
+        try:
+            return self._v_cached_hash
+        except AttributeError:
+            self._v_cached_hash = hash((self.__name__, self.__module__))
+        return self._v_cached_hash
+
+    def __eq__(self, other):
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c == 0
+
+    def __ne__(self, other):
+        if other is self:
+            return False
+
+        c = self._compare(other)
+        if c is NotImplemented:
+            return c
+        return c != 0
+
+adapter_hooks = _use_c_impl([], 'adapter_hooks')
+
+
+class Specification(SpecificationBase):
+    """Specifications
+
+    An interface specification is used to track interface declarations
+    and component registrations.
+
+    This class is a base class for both interfaces themselves and for
+    interface specifications (declarations).
+
+    Specifications are mutable.  If you reassign their bases, their
+    relations with other specifications are adjusted accordingly.
+    """
+    __slots__ = ()
+
+    # The root of all Specifications. This will be assigned `Interface`,
+    # once it is defined.
+    _ROOT = None
+
+    # Copy some base class methods for speed
+    isOrExtends = SpecificationBase.isOrExtends
+    providedBy = SpecificationBase.providedBy
+
+    def __init__(self, bases=()):
+        # There are many leaf interfaces with no dependents,
+        # and a few with very many. It's a heavily left-skewed
+        # distribution. In a survey of Plone and Zope related packages
+        # that loaded 2245 InterfaceClass objects and 2235 ClassProvides
+        # instances, there were a total of 7000 Specification objects created.
+        # 4700 had 0 dependents, 1400 had 1, 382 had 2 and so on. Only one
+        # for <type> had 1664. So there's savings to be had deferring
+        # the creation of dependents.
+        self._dependents = None # type: weakref.WeakKeyDictionary
+        self._bases = ()
+        self._implied = {}
+        self._v_attrs = None
+        self.__iro__ = ()
+        self.__sro__ = ()
+
+        self.__bases__ = tuple(bases)
+
+    @property
+    def dependents(self):
+        if self._dependents is None:
+            self._dependents = weakref.WeakKeyDictionary()
+        return self._dependents
+
+    def subscribe(self, dependent):
+        self._dependents[dependent] = self.dependents.get(dependent, 0) + 1
+
+    def unsubscribe(self, dependent):
+        try:
+            n = self._dependents[dependent]
+        except TypeError:
+            raise KeyError(dependent)
+        n -= 1
+        if not n:
+            del self.dependents[dependent]
+        else:
+            assert n > 0
+            self.dependents[dependent] = n
+
+    def __setBases(self, bases):
+        # Remove ourselves as a dependent of our old bases
+        for b in self.__bases__:
+            b.unsubscribe(self)
+
+        # Register ourselves as a dependent of our new bases
+        self._bases = bases
+        for b in bases:
+            b.subscribe(self)
+
+        self.changed(self)
+
+    __bases__ = property(
+        lambda self: self._bases,
+        __setBases,
+        )
+
+    # This method exists for tests to override the way we call
+    # ro.calculate_ro(), usually by adding extra kwargs. We don't
+    # want to have a mutable dictionary as a class member that we pass
+    # ourself because mutability is bad, and passing **kw is slower than
+    # calling the bound function.
+    _do_calculate_ro = calculate_ro
+
+    def _calculate_sro(self):
+        """
+        Calculate and return the resolution order for this object, using its ``__bases__``.
+
+        Ensures that ``Interface`` is always the last (lowest priority) element.
+        """
+        # We'd like to make Interface the lowest priority as a
+        # property of the resolution order algorithm. That almost
+        # works out naturally, but it fails when class inheritance has
+        # some bases that DO implement an interface, and some that DO
+        # NOT. In such a mixed scenario, you wind up with a set of
+        # bases to consider that look like this: [[..., Interface],
+        # [..., object], ...]. Depending on the order of inheritance,
+        # Interface can wind up before or after object, and that can
+        # happen at any point in the tree, meaning Interface can wind
+        # up somewhere in the middle of the order. Since Interface is
+        # treated as something that everything winds up implementing
+        # anyway (a catch-all for things like adapters), having it high up
+        # the order is bad. It's also bad to have it at the end, just before
+        # some concrete class: concrete classes should be HIGHER priority than
+        # interfaces (because there's only one class, but many implementations).
+        #
+        # One technically nice way to fix this would be to have
+        # ``implementedBy(object).__bases__ = (Interface,)``
+        #
+        # But: (1) That fails for old-style classes and (2) that causes
+        # everything to appear to *explicitly* implement Interface, when up
+        # to this point it's been an implicit virtual sort of relationship.
+        #
+        # So we force the issue by mutating the resolution order.
+
+        # Note that we let C3 use pre-computed __sro__ for our bases.
+        # This requires that by the time this method is invoked, our bases
+        # have settled their SROs. Thus, ``changed()`` must first
+        # update itself before telling its descendents of changes.
+        sro = self._do_calculate_ro(base_mros={
+            b: b.__sro__
+            for b in self.__bases__
+        })
+        root = self._ROOT
+        if root is not None and sro and sro[-1] is not root:
+            # In one dataset of 1823 Interface objects, 1117 ClassProvides objects,
+            # sro[-1] was root 4496 times, and only not root 118 times. So it's
+            # probably worth checking.
+
+            # Once we don't have to deal with old-style classes,
+            # we can add a check and only do this if base_count > 1,
+            # if we tweak the bootstrapping for ``<implementedBy object>``
+            sro = [
+                x
+                for x in sro
+                if x is not root
+            ]
+            sro.append(root)
+
+        return sro
+
+    def changed(self, originally_changed):
+        """
+        We, or something we depend on, have changed.
+
+        By the time this is called, the things we depend on,
+        such as our bases, should themselves be stable.
+        """
+        self._v_attrs = None
+
+        implied = self._implied
+        implied.clear()
+
+        ancestors = self._calculate_sro()
+        self.__sro__ = tuple(ancestors)
+        self.__iro__ = tuple([ancestor for ancestor in ancestors
+                              if isinstance(ancestor, InterfaceClass)
+                              ])
+
+        for ancestor in ancestors:
+            # We directly imply our ancestors:
+            implied[ancestor] = ()
+
+        # Now, advise our dependents of change
+        # (being careful not to create the WeakKeyDictionary if not needed):
+        for dependent in tuple(self._dependents.keys() if self._dependents else ()):
+            dependent.changed(originally_changed)
+
+        # Just in case something called get() at some point
+        # during that process and we have a cycle of some sort
+        # make sure we didn't cache incomplete results.
+        self._v_attrs = None
+
+    def interfaces(self):
+        """Return an iterator for the interfaces in the specification.
+        """
+        seen = {}
+        for base in self.__bases__:
+            for interface in base.interfaces():
+                if interface not in seen:
+                    seen[interface] = 1
+                    yield interface
+
+    def extends(self, interface, strict=True):
+        """Does the specification extend the given interface?
+
+        Test whether an interface in the specification extends the
+        given interface
+        """
+        return ((interface in self._implied)
+                and
+                ((not strict) or (self != interface))
+                )
+
+    def weakref(self, callback=None):
+        return weakref.ref(self, callback)
+
+    def get(self, name, default=None):
+        """Query for an attribute description
+        """
+        attrs = self._v_attrs
+        if attrs is None:
+            attrs = self._v_attrs = {}
+        attr = attrs.get(name)
+        if attr is None:
+            for iface in self.__iro__:
+                attr = iface.direct(name)
+                if attr is not None:
+                    attrs[name] = attr
+                    break
+
+        return default if attr is None else attr
+
+
+class _InterfaceMetaClass(type):
+    # Handling ``__module__`` on ``InterfaceClass`` is tricky. We need
+    # to be able to read it on a type and get the expected string. We
+    # also need to be able to set it on an instance and get the value
+    # we set. So far so good. But what gets tricky is that we'd like
+    # to store the value in the C structure (``InterfaceBase.__ibmodule__``) for
+    # direct access during equality, sorting, and hashing. "No
+    # problem, you think, I'll just use a property" (well, the C
+    # equivalents, ``PyMemberDef`` or ``PyGetSetDef``).
+    #
+    # Except there is a problem. When a subclass is created, the
+    # metaclass (``type``) always automatically puts the expected
+    # string in the class's dictionary under ``__module__``, thus
+    # overriding the property inherited from the superclass. Writing
+    # ``Subclass.__module__`` still works, but
+    # ``Subclass().__module__`` fails.
+    #
+    # There are multiple ways to work around this:
+    #
+    # (1) Define ``InterfaceBase.__getattribute__`` to watch for
+    # ``__module__`` and return the C storage.
+    #
+    # This works, but slows down *all* attribute access (except,
+    # ironically, to ``__module__``) by about 25% (40ns becomes 50ns)
+    # (when implemented in C). Since that includes methods like
+    # ``providedBy``, that's probably not acceptable.
+    #
+    # All the other methods involve modifying subclasses. This can be
+    # done either on the fly in some cases, as instances are
+    # constructed, or by using a metaclass. These next few can be done on the fly.
+    #
+    # (2) Make ``__module__`` a descriptor in each subclass dictionary.
+    # It can't be a straight up ``@property`` descriptor, though, because accessing
+    # it on the class returns a ``property`` object, not the desired string.
+    #
+    # (3) Implement a data descriptor (``__get__`` and ``__set__``)
+    # that is both a subclass of string, and also does the redirect of
+    # ``__module__`` to ``__ibmodule__`` and does the correct thing
+    # with the ``instance`` argument to ``__get__`` is None (returns
+    # the class's value.) (Why must it be a subclass of string? Because
+    # when it' s in the class's dict, it's defined on an *instance* of the
+    # metaclass; descriptors in an instance's dict aren't honored --- their
+    # ``__get__`` is never invoked --- so it must also *be* the value we want
+    # returned.)
+    #
+    # This works, preserves the ability to read and write
+    # ``__module__``, and eliminates any penalty accessing other
+    # attributes. But it slows down accessing ``__module__`` of
+    # instances by 200% (40ns to 124ns), requires editing class dicts on the fly
+    # (in InterfaceClass.__init__), thus slightly slowing down all interface creation,
+    # and is ugly.
+    #
+    # (4) As in the last step, but make it a non-data descriptor (no ``__set__``).
+    #
+    # If you then *also* store a copy of ``__ibmodule__`` in
+    # ``__module__`` in the instance's dict, reading works for both
+    # class and instance and is full speed for instances. But the cost
+    # is storage space, and you can't write to it anymore, not without
+    # things getting out of sync.
+    #
+    # (Actually, ``__module__`` was never meant to be writable. Doing
+    # so would break BTrees and normal dictionaries, as well as the
+    # repr, maybe more.)
+    #
+    # That leaves us with a metaclass. (Recall that a class is an
+    # instance of its metaclass, so properties/descriptors defined in
+    # the metaclass are used when accessing attributes on the
+    # instance/class. We'll use that to define ``__module__``.) Here
+    # we can have our cake and eat it too: no extra storage, and
+    # C-speed access to the underlying storage. The only substantial
+    # cost is that metaclasses tend to make people's heads hurt. (But
+    # still less than the descriptor-is-string, hopefully.)
+
+    __slots__ = ()
+
+    def __new__(cls, name, bases, attrs):
+        # Figure out what module defined the interface.
+        # This is copied from ``InterfaceClass.__init__``;
+        # reviewers aren't sure how AttributeError or KeyError
+        # could be raised.
+        __module__ = sys._getframe(1).f_globals['__name__']
+        # Get the C optimized __module__ accessor and give it
+        # to the new class.
+        moduledescr = InterfaceBase.__dict__['__module__']
+        if isinstance(moduledescr, str):
+            # We're working with the Python implementation,
+            # not the C version
+            moduledescr = InterfaceBase.__dict__['__module_property__']
+        attrs['__module__'] = moduledescr
+        kind = type.__new__(cls, name, bases, attrs)
+        kind.__module = __module__
+        return kind
+
+    @property
+    def __module__(cls):
+        return cls.__module
+
+    def __repr__(cls):
+        return "<class '{}.{}'>".format(
+            cls.__module,
+            cls.__name__,
+        )
+
+
+_InterfaceClassBase = _InterfaceMetaClass(
+    'InterfaceClass',
+    # From least specific to most specific.
+    (InterfaceBase, Specification, Element),
+    {'__slots__': ()}
+)
+
+
+def interfacemethod(func):
+    """
+    Convert a method specification to an actual method of the interface.
+
+    This is a decorator that functions like `staticmethod` et al.
+
+    The primary use of this decorator is to allow interface definitions to
+    define the ``__adapt__`` method, but other interface methods can be
+    overridden this way too.
+
+    .. seealso:: `zope.interface.interfaces.IInterfaceDeclaration.interfacemethod`
+    """
+    f_locals = sys._getframe(1).f_locals
+    methods = f_locals.setdefault(INTERFACE_METHODS, {})
+    methods[func.__name__] = func
+    return _decorator_non_return
+
+
+class InterfaceClass(_InterfaceClassBase):
+    """
+    Prototype (scarecrow) Interfaces Implementation.
+
+    Note that it is not possible to change the ``__name__`` or ``__module__``
+    after an instance of this object has been constructed.
+    """
+
+    # We can't say this yet because we don't have enough
+    # infrastructure in place.
+    #
+    #implements(IInterface)
+
+    def __new__(cls, name=None, bases=(), attrs=None, __doc__=None, # pylint:disable=redefined-builtin
+                __module__=None):
+        assert isinstance(bases, tuple)
+        attrs = attrs or {}
+        needs_custom_class = attrs.pop(INTERFACE_METHODS, None)
+        if needs_custom_class:
+            needs_custom_class.update(
+                {'__classcell__': attrs.pop('__classcell__')}
+                if '__classcell__' in attrs
+                else {}
+            )
+            if '__adapt__' in needs_custom_class:
+                # We need to tell the C code to call this.
+                needs_custom_class['_CALL_CUSTOM_ADAPT'] = 1
+
+            if issubclass(cls, _InterfaceClassWithCustomMethods):
+                cls_bases = (cls,)
+            elif cls is InterfaceClass:
+                cls_bases = (_InterfaceClassWithCustomMethods,)
+            else:
+                cls_bases = (cls, _InterfaceClassWithCustomMethods)
+
+            cls = type(cls)( # pylint:disable=self-cls-assignment
+                name + "<WithCustomMethods>",
+                cls_bases,
+                needs_custom_class
+            )
+
+        return _InterfaceClassBase.__new__(cls)
+
+    def __init__(self, name, bases=(), attrs=None, __doc__=None,  # pylint:disable=redefined-builtin
+                 __module__=None):
+        # We don't call our metaclass parent directly
+        # pylint:disable=non-parent-init-called
+        # pylint:disable=super-init-not-called
+        if not all(isinstance(base, InterfaceClass) for base in bases):
+            raise TypeError('Expected base interfaces')
+
+        if attrs is None:
+            attrs = {}
+
+        if __module__ is None:
+            __module__ = attrs.get('__module__')
+            if isinstance(__module__, str):
+                del attrs['__module__']
+            else:
+                try:
+                    # Figure out what module defined the interface.
+                    # This is how cPython figures out the module of
+                    # a class, but of course it does it in C. :-/
+                    __module__ = sys._getframe(1).f_globals['__name__']
+                except (AttributeError, KeyError): # pragma: no cover
+                    pass
+
+        InterfaceBase.__init__(self, name, __module__)
+        # These asserts assisted debugging the metaclass
+        # assert '__module__' not in self.__dict__
+        # assert self.__ibmodule__ is self.__module__ is __module__
+
+        d = attrs.get('__doc__')
+        if d is not None:
+            if not isinstance(d, Attribute):
+                if __doc__ is None:
+                    __doc__ = d
+                del attrs['__doc__']
+
+        if __doc__ is None:
+            __doc__ = ''
+
+        Element.__init__(self, name, __doc__)
+
+        tagged_data = attrs.pop(TAGGED_DATA, None)
+        if tagged_data is not None:
+            for key, val in tagged_data.items():
+                self.setTaggedValue(key, val)
+
+        Specification.__init__(self, bases)
+        self.__attrs = self.__compute_attrs(attrs)
+
+        self.__identifier__ = "{}.{}".format(__module__, name)
+
+    def __compute_attrs(self, attrs):
+        # Make sure that all recorded attributes (and methods) are of type
+        # `Attribute` and `Method`
+        def update_value(aname, aval):
+            if isinstance(aval, Attribute):
+                aval.interface = self
+                if not aval.__name__:
+                    aval.__name__ = aname
+            elif isinstance(aval, FunctionType):
+                aval = fromFunction(aval, self, name=aname)
+            else:
+                raise InvalidInterface("Concrete attribute, " + aname)
+            return aval
+
+        return {
+            aname: update_value(aname, aval)
+            for aname, aval in attrs.items()
+            if aname not in (
+                # __locals__: Python 3 sometimes adds this.
+                '__locals__',
+                # __qualname__: PEP 3155 (Python 3.3+)
+                '__qualname__',
+                # __annotations__: PEP 3107 (Python 3.0+)
+                '__annotations__',
+            )
+            and aval is not _decorator_non_return
+        }
+
+    def interfaces(self):
+        """Return an iterator for the interfaces in the specification.
+        """
+        yield self
+
+    def getBases(self):
+        return self.__bases__
+
+    def isEqualOrExtendedBy(self, other):
+        """Same interface or extends?"""
+        return self == other or other.extends(self)
+
+    def names(self, all=False): # pylint:disable=redefined-builtin
+        """Return the attribute names defined by the interface."""
+        if not all:
+            return self.__attrs.keys()
+
+        r = self.__attrs.copy()
+
+        for base in self.__bases__:
+            r.update(dict.fromkeys(base.names(all)))
+
+        return r.keys()
+
+    def __iter__(self):
+        return iter(self.names(all=True))
+
+    def namesAndDescriptions(self, all=False): # pylint:disable=redefined-builtin
+        """Return attribute names and descriptions defined by interface."""
+        if not all:
+            return self.__attrs.items()
+
+        r = {}
+        for base in self.__bases__[::-1]:
+            r.update(dict(base.namesAndDescriptions(all)))
+
+        r.update(self.__attrs)
+
+        return r.items()
+
+    def getDescriptionFor(self, name):
+        """Return the attribute description for the given name."""
+        r = self.get(name)
+        if r is not None:
+            return r
+
+        raise KeyError(name)
+
+    __getitem__ = getDescriptionFor
+
+    def __contains__(self, name):
+        return self.get(name) is not None
+
+    def direct(self, name):
+        return self.__attrs.get(name)
+
+    def queryDescriptionFor(self, name, default=None):
+        return self.get(name, default)
+
+    def validateInvariants(self, obj, errors=None):
+        """validate object to defined invariants."""
+
+        for iface in self.__iro__:
+            for invariant in iface.queryDirectTaggedValue('invariants', ()):
+                try:
+                    invariant(obj)
+                except Invalid as error:
+                    if errors is not None:
+                        errors.append(error)
+                    else:
+                        raise
+
+        if errors:
+            raise Invalid(errors)
+
+    def queryTaggedValue(self, tag, default=None):
+        """
+        Queries for the value associated with *tag*, returning it from the nearest
+        interface in the ``__iro__``.
+
+        If not found, returns *default*.
+        """
+        for iface in self.__iro__:
+            value = iface.queryDirectTaggedValue(tag, _marker)
+            if value is not _marker:
+                return value
+        return default
+
+    def getTaggedValue(self, tag):
+        """ Returns the value associated with 'tag'. """
+        value = self.queryTaggedValue(tag, default=_marker)
+        if value is _marker:
+            raise KeyError(tag)
+        return value
+
+    def getTaggedValueTags(self):
+        """ Returns a list of all tags. """
+        keys = set()
+        for base in self.__iro__:
+            keys.update(base.getDirectTaggedValueTags())
+        return keys
+
+    def __repr__(self):
+        try:
+            return self._v_repr
+        except AttributeError:
+            name = str(self)
+            r = "<{} {}>".format(self.__class__.__name__, name)
+            self._v_repr = r # pylint:disable=attribute-defined-outside-init
+            return r
+
+    def __str__(self):
+        name = self.__name__
+        m = self.__ibmodule__
+        if m:
+            name = '{}.{}'.format(m, name)
+        return name
+
+    def _call_conform(self, conform):
+        try:
+            return conform(self)
+        except TypeError: # pragma: no cover
+            # We got a TypeError. It might be an error raised by
+            # the __conform__ implementation, or *we* may have
+            # made the TypeError by calling an unbound method
+            # (object is a class).  In the later case, we behave
+            # as though there is no __conform__ method. We can
+            # detect this case by checking whether there is more
+            # than one traceback object in the traceback chain:
+            if sys.exc_info()[2].tb_next is not None:
+                # There is more than one entry in the chain, so
+                # reraise the error:
+                raise
+            # This clever trick is from Phillip Eby
+
+        return None # pragma: no cover
+
+    def __reduce__(self):
+        return self.__name__
+
+Interface = InterfaceClass("Interface", __module__='zope.interface')
+# Interface is the only member of its own SRO.
+Interface._calculate_sro = lambda: (Interface,)
+Interface.changed(Interface)
+assert Interface.__sro__ == (Interface,)
+Specification._ROOT = Interface
+ro._ROOT = Interface
+
+class _InterfaceClassWithCustomMethods(InterfaceClass):
+    """
+    Marker class for interfaces with custom methods that override InterfaceClass methods.
+    """
+
+
+class Attribute(Element):
+    """Attribute descriptions
+    """
+
+    # We can't say this yet because we don't have enough
+    # infrastructure in place.
+    #
+    # implements(IAttribute)
+
+    interface = None
+
+    def _get_str_info(self):
+        """Return extra data to put at the end of __str__."""
+        return ""
+
+    def __str__(self):
+        of = ''
+        if self.interface is not None:
+            of = self.interface.__module__ + '.' + self.interface.__name__ + '.'
+        # self.__name__ may be None during construction (e.g., debugging)
+        return of + (self.__name__ or '<unknown>') + self._get_str_info()
+
+    def __repr__(self):
+        return "<{}.{} object at 0x{:x} {}>".format(
+            type(self).__module__,
+            type(self).__name__,
+            id(self),
+            self
+        )
+
+
+class Method(Attribute):
+    """Method interfaces
+
+    The idea here is that you have objects that describe methods.
+    This provides an opportunity for rich meta-data.
+    """
+
+    # We can't say this yet because we don't have enough
+    # infrastructure in place.
+    #
+    # implements(IMethod)
+
+    positional = required = ()
+    _optional = varargs = kwargs = None
+    def _get_optional(self):
+        if self._optional is None:
+            return {}
+        return self._optional
+    def _set_optional(self, opt):
+        self._optional = opt
+    def _del_optional(self):
+        self._optional = None
+    optional = property(_get_optional, _set_optional, _del_optional)
+
+    def __call__(self, *args, **kw):
+        raise BrokenImplementation(self.interface, self.__name__)
+
+    def getSignatureInfo(self):
+        return {'positional': self.positional,
+                'required': self.required,
+                'optional': self.optional,
+                'varargs': self.varargs,
+                'kwargs': self.kwargs,
+                }
+
+    def getSignatureString(self):
+        sig = []
+        for v in self.positional:
+            sig.append(v)
+            if v in self.optional.keys():
+                sig[-1] += "=" + repr(self.optional[v])
+        if self.varargs:
+            sig.append("*" + self.varargs)
+        if self.kwargs:
+            sig.append("**" + self.kwargs)
+
+        return "(%s)" % ", ".join(sig)
+
+    _get_str_info = getSignatureString
+
+
+def fromFunction(func, interface=None, imlevel=0, name=None):
+    name = name or func.__name__
+    method = Method(name, func.__doc__)
+    defaults = getattr(func, '__defaults__', None) or ()
+    code = func.__code__
+    # Number of positional arguments
+    na = code.co_argcount - imlevel
+    names = code.co_varnames[imlevel:]
+    opt = {}
+    # Number of required arguments
+    defaults_count = len(defaults)
+    if not defaults_count:
+        # PyPy3 uses ``__defaults_count__`` for builtin methods
+        # like ``dict.pop``. Surprisingly, these don't have recorded
+        # ``__defaults__``
+        defaults_count = getattr(func, '__defaults_count__', 0)
+
+    nr = na - defaults_count
+    if nr < 0:
+        defaults = defaults[-nr:]
+        nr = 0
+
+    # Determine the optional arguments.
+    opt.update(dict(zip(names[nr:], defaults)))
+
+    method.positional = names[:na]
+    method.required = names[:nr]
+    method.optional = opt
+
+    argno = na
+
+    # Determine the function's variable argument's name (i.e. *args)
+    if code.co_flags & CO_VARARGS:
+        method.varargs = names[argno]
+        argno = argno + 1
+    else:
+        method.varargs = None
+
+    # Determine the function's keyword argument's name (i.e. **kw)
+    if code.co_flags & CO_VARKEYWORDS:
+        method.kwargs = names[argno]
+    else:
+        method.kwargs = None
+
+    method.interface = interface
+
+    for key, value in func.__dict__.items():
+        method.setTaggedValue(key, value)
+
+    return method
+
+
+def fromMethod(meth, interface=None, name=None):
+    if isinstance(meth, MethodType):
+        func = meth.__func__
+    else:
+        func = meth
+    return fromFunction(func, interface, imlevel=1, name=name)
+
+
+# Now we can create the interesting interfaces and wire them up:
+def _wire():
+    from zope.interface.declarations import classImplements
+    # From lest specific to most specific.
+    from zope.interface.interfaces import IElement
+    classImplements(Element, IElement)
+
+    from zope.interface.interfaces import IAttribute
+    classImplements(Attribute, IAttribute)
+
+    from zope.interface.interfaces import IMethod
+    classImplements(Method, IMethod)
+
+    from zope.interface.interfaces import ISpecification
+    classImplements(Specification, ISpecification)
+
+    from zope.interface.interfaces import IInterface
+    classImplements(InterfaceClass, IInterface)
+
+
+# We import this here to deal with module dependencies.
+# pylint:disable=wrong-import-position
+from zope.interface.declarations import implementedBy
+from zope.interface.declarations import providedBy
+from zope.interface.exceptions import InvalidInterface
+from zope.interface.exceptions import BrokenImplementation
+
+# This ensures that ``Interface`` winds up in the flattened()
+# list of the immutable declaration. It correctly overrides changed()
+# as a no-op, so we bypass that.
+from zope.interface.declarations import _empty
+Specification.changed(_empty, _empty)
diff --git a/contrib/python/zope.interface/py3/zope/interface/interfaces.py b/contrib/python/zope.interface/py3/zope/interface/interfaces.py
new file mode 100644
index 00000000000..2b67ce1a9e0
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/interfaces.py
@@ -0,0 +1,1480 @@
+##############################################################################
+#
+# Copyright (c) 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interface Package Interfaces
+"""
+__docformat__ = 'restructuredtext'
+
+from zope.interface.interface import Attribute
+from zope.interface.interface import Interface
+from zope.interface.declarations import implementer
+
+__all__ = [
+    'ComponentLookupError',
+    'IAdapterRegistration',
+    'IAdapterRegistry',
+    'IAttribute',
+    'IComponentLookup',
+    'IComponentRegistry',
+    'IComponents',
+    'IDeclaration',
+    'IElement',
+    'IHandlerRegistration',
+    'IInterface',
+    'IInterfaceDeclaration',
+    'IMethod',
+    'Invalid',
+    'IObjectEvent',
+    'IRegistered',
+    'IRegistration',
+    'IRegistrationEvent',
+    'ISpecification',
+    'ISubscriptionAdapterRegistration',
+    'IUnregistered',
+    'IUtilityRegistration',
+    'ObjectEvent',
+    'Registered',
+    'Unregistered',
+]
+
+# pylint:disable=inherit-non-class,no-method-argument,no-self-argument
+# pylint:disable=unexpected-special-method-signature
+# pylint:disable=too-many-lines
+
+class IElement(Interface):
+    """
+    Objects that have basic documentation and tagged values.
+
+    Known derivatives include :class:`IAttribute` and its derivative
+    :class:`IMethod`; these have no notion of inheritance.
+    :class:`IInterface` is also a derivative, and it does have a
+    notion of inheritance, expressed through its ``__bases__`` and
+    ordered in its ``__iro__`` (both defined by
+    :class:`ISpecification`).
+    """
+
+    # pylint:disable=arguments-differ
+
+    # Note that defining __doc__ as an Attribute hides the docstring
+    # from introspection. When changing it, also change it in the Sphinx
+    # ReST files.
+
+    __name__ = Attribute('__name__', 'The object name')
+    __doc__ = Attribute('__doc__', 'The object doc string')
+
+    ###
+    # Tagged values.
+    #
+    # Direct values are established in this instance. Others may be
+    # inherited. Although ``IElement`` itself doesn't have a notion of
+    # inheritance, ``IInterface`` *does*. It might have been better to
+    # make ``IInterface`` define new methods
+    # ``getIndirectTaggedValue``, etc, to include inheritance instead
+    # of overriding ``getTaggedValue`` to do that, but that ship has sailed.
+    # So to keep things nice and symmetric, we define the ``Direct`` methods here.
+    ###
+
+    def getTaggedValue(tag):
+        """Returns the value associated with *tag*.
+
+        Raise a `KeyError` if the tag isn't set.
+
+        If the object has a notion of inheritance, this searches
+        through the inheritance hierarchy and returns the nearest result.
+        If there is no such notion, this looks only at this object.
+
+        .. versionchanged:: 4.7.0
+           This method should respect inheritance if present.
+        """
+
+    def queryTaggedValue(tag, default=None):
+        """
+        As for `getTaggedValue`, but instead of raising a `KeyError`, returns *default*.
+
+
+        .. versionchanged:: 4.7.0
+           This method should respect inheritance if present.
+        """
+
+    def getTaggedValueTags():
+        """
+        Returns a collection of all tags in no particular order.
+
+        If the object has a notion of inheritance, this
+        includes all the inherited tagged values. If there is
+        no such notion, this looks only at this object.
+
+        .. versionchanged:: 4.7.0
+           This method should respect inheritance if present.
+        """
+
+    def setTaggedValue(tag, value):
+        """
+        Associates *value* with *key* directly in this object.
+        """
+
+    def getDirectTaggedValue(tag):
+        """
+        As for `getTaggedValue`, but never includes inheritance.
+
+        .. versionadded:: 5.0.0
+        """
+
+    def queryDirectTaggedValue(tag, default=None):
+        """
+        As for `queryTaggedValue`, but never includes inheritance.
+
+        .. versionadded:: 5.0.0
+        """
+
+    def getDirectTaggedValueTags():
+        """
+        As for `getTaggedValueTags`, but includes only tags directly
+        set on this object.
+
+        .. versionadded:: 5.0.0
+        """
+
+
+class IAttribute(IElement):
+    """Attribute descriptors"""
+
+    interface = Attribute('interface',
+                          'Stores the interface instance in which the '
+                          'attribute is located.')
+
+
+class IMethod(IAttribute):
+    """Method attributes"""
+
+    def getSignatureInfo():
+        """Returns the signature information.
+
+        This method returns a dictionary with the following string keys:
+
+        - positional
+            A sequence of the names of positional arguments.
+        - required
+            A sequence of the names of required arguments.
+        - optional
+            A dictionary mapping argument names to their default values.
+        - varargs
+            The name of the varargs argument (or None).
+        - kwargs
+            The name of the kwargs argument (or None).
+        """
+
+    def getSignatureString():
+        """Return a signature string suitable for inclusion in documentation.
+
+        This method returns the function signature string. For example, if you
+        have ``def func(a, b, c=1, d='f')``, then the signature string is ``"(a, b,
+        c=1, d='f')"``.
+        """
+
+class ISpecification(Interface):
+    """Object Behavioral specifications"""
+    # pylint:disable=arguments-differ
+    def providedBy(object): # pylint:disable=redefined-builtin
+        """Test whether the interface is implemented by the object
+
+        Return true of the object asserts that it implements the
+        interface, including asserting that it implements an extended
+        interface.
+        """
+
+    def implementedBy(class_):
+        """Test whether the interface is implemented by instances of the class
+
+        Return true of the class asserts that its instances implement the
+        interface, including asserting that they implement an extended
+        interface.
+        """
+
+    def isOrExtends(other):
+        """Test whether the specification is or extends another
+        """
+
+    def extends(other, strict=True):
+        """Test whether a specification extends another
+
+        The specification extends other if it has other as a base
+        interface or if one of it's bases extends other.
+
+        If strict is false, then the specification extends itself.
+        """
+
+    def weakref(callback=None):
+        """Return a weakref to the specification
+
+        This method is, regrettably, needed to allow weakrefs to be
+        computed to security-proxied specifications.  While the
+        zope.interface package does not require zope.security or
+        zope.proxy, it has to be able to coexist with it.
+
+        """
+
+    __bases__ = Attribute("""Base specifications
+
+    A tuple of specifications from which this specification is
+    directly derived.
+
+    """)
+
+    __sro__ = Attribute("""Specification-resolution order
+
+    A tuple of the specification and all of it's ancestor
+    specifications from most specific to least specific. The specification
+    itself is the first element.
+
+    (This is similar to the method-resolution order for new-style classes.)
+    """)
+
+    __iro__ = Attribute("""Interface-resolution order
+
+    A tuple of the specification's ancestor interfaces from
+    most specific to least specific.  The specification itself is
+    included if it is an interface.
+
+    (This is similar to the method-resolution order for new-style classes.)
+    """)
+
+    def get(name, default=None):
+        """Look up the description for a name
+
+        If the named attribute is not defined, the default is
+        returned.
+        """
+
+
+class IInterface(ISpecification, IElement):
+    """Interface objects
+
+    Interface objects describe the behavior of an object by containing
+    useful information about the object.  This information includes:
+
+    - Prose documentation about the object.  In Python terms, this
+      is called the "doc string" of the interface.  In this element,
+      you describe how the object works in prose language and any
+      other useful information about the object.
+
+    - Descriptions of attributes.  Attribute descriptions include
+      the name of the attribute and prose documentation describing
+      the attributes usage.
+
+    - Descriptions of methods.  Method descriptions can include:
+
+        - Prose "doc string" documentation about the method and its
+          usage.
+
+        - A description of the methods arguments; how many arguments
+          are expected, optional arguments and their default values,
+          the position or arguments in the signature, whether the
+          method accepts arbitrary arguments and whether the method
+          accepts arbitrary keyword arguments.
+
+    - Optional tagged data.  Interface objects (and their attributes and
+      methods) can have optional, application specific tagged data
+      associated with them.  Examples uses for this are examples,
+      security assertions, pre/post conditions, and other possible
+      information you may want to associate with an Interface or its
+      attributes.
+
+    Not all of this information is mandatory.  For example, you may
+    only want the methods of your interface to have prose
+    documentation and not describe the arguments of the method in
+    exact detail.  Interface objects are flexible and let you give or
+    take any of these components.
+
+    Interfaces are created with the Python class statement using
+    either `zope.interface.Interface` or another interface, as in::
+
+      from zope.interface import Interface
+
+      class IMyInterface(Interface):
+        '''Interface documentation'''
+
+        def meth(arg1, arg2):
+            '''Documentation for meth'''
+
+        # Note that there is no self argument
+
+     class IMySubInterface(IMyInterface):
+        '''Interface documentation'''
+
+        def meth2():
+            '''Documentation for meth2'''
+
+    You use interfaces in two ways:
+
+    - You assert that your object implement the interfaces.
+
+      There are several ways that you can declare that an object
+      provides an interface:
+
+      1. Call `zope.interface.implementer` on your class definition.
+
+      2. Call `zope.interface.directlyProvides` on your object.
+
+      3. Call `zope.interface.classImplements` to declare that instances
+         of a class implement an interface.
+
+         For example::
+
+           from zope.interface import classImplements
+
+           classImplements(some_class, some_interface)
+
+         This approach is useful when it is not an option to modify
+         the class source.  Note that this doesn't affect what the
+         class itself implements, but only what its instances
+         implement.
+
+    - You query interface meta-data. See the IInterface methods and
+      attributes for details.
+
+    """
+    # pylint:disable=arguments-differ
+    def names(all=False): # pylint:disable=redefined-builtin
+        """Get the interface attribute names
+
+        Return a collection of the names of the attributes, including
+        methods, included in the interface definition.
+
+        Normally, only directly defined attributes are included. If
+        a true positional or keyword argument is given, then
+        attributes defined by base classes will be included.
+        """
+
+    def namesAndDescriptions(all=False): # pylint:disable=redefined-builtin
+        """Get the interface attribute names and descriptions
+
+        Return a collection of the names and descriptions of the
+        attributes, including methods, as name-value pairs, included
+        in the interface definition.
+
+        Normally, only directly defined attributes are included. If
+        a true positional or keyword argument is given, then
+        attributes defined by base classes will be included.
+        """
+
+    def __getitem__(name):
+        """Get the description for a name
+
+        If the named attribute is not defined, a `KeyError` is raised.
+        """
+
+    def direct(name):
+        """Get the description for the name if it was defined by the interface
+
+        If the interface doesn't define the name, returns None.
+        """
+
+    def validateInvariants(obj, errors=None):
+        """Validate invariants
+
+        Validate object to defined invariants.  If errors is None,
+        raises first Invalid error; if errors is a list, appends all errors
+        to list, then raises Invalid with the errors as the first element
+        of the "args" tuple."""
+
+    def __contains__(name):
+        """Test whether the name is defined by the interface"""
+
+    def __iter__():
+        """Return an iterator over the names defined by the interface
+
+        The names iterated include all of the names defined by the
+        interface directly and indirectly by base interfaces.
+        """
+
+    __module__ = Attribute("""The name of the module defining the interface""")
+
+
+class IDeclaration(ISpecification):
+    """Interface declaration
+
+    Declarations are used to express the interfaces implemented by
+    classes or provided by objects.
+    """
+
+    def __contains__(interface):
+        """Test whether an interface is in the specification
+
+        Return true if the given interface is one of the interfaces in
+        the specification and false otherwise.
+        """
+
+    def __iter__():
+        """Return an iterator for the interfaces in the specification
+        """
+
+    def flattened():
+        """Return an iterator of all included and extended interfaces
+
+        An iterator is returned for all interfaces either included in
+        or extended by interfaces included in the specifications
+        without duplicates. The interfaces are in "interface
+        resolution order". The interface resolution order is such that
+        base interfaces are listed after interfaces that extend them
+        and, otherwise, interfaces are included in the order that they
+        were defined in the specification.
+        """
+
+    def __sub__(interfaces):
+        """Create an interface specification with some interfaces excluded
+
+        The argument can be an interface or an interface
+        specifications.  The interface or interfaces given in a
+        specification are subtracted from the interface specification.
+
+        Removing an interface that is not in the specification does
+        not raise an error. Doing so has no effect.
+
+        Removing an interface also removes sub-interfaces of the interface.
+
+        """
+
+    def __add__(interfaces):
+        """Create an interface specification with some interfaces added
+
+        The argument can be an interface or an interface
+        specifications.  The interface or interfaces given in a
+        specification are added to the interface specification.
+
+        Adding an interface that is already in the specification does
+        not raise an error. Doing so has no effect.
+        """
+
+    def __nonzero__():
+        """Return a true value of the interface specification is non-empty
+        """
+
+class IInterfaceDeclaration(Interface):
+    """
+    Declare and check the interfaces of objects.
+
+    The functions defined in this interface are used to declare the
+    interfaces that objects provide and to query the interfaces that
+    have been declared.
+
+    Interfaces can be declared for objects in two ways:
+
+        - Interfaces are declared for instances of the object's class
+
+        - Interfaces are declared for the object directly.
+
+    The interfaces declared for an object are, therefore, the union of
+    interfaces declared for the object directly and the interfaces
+    declared for instances of the object's class.
+
+    Note that we say that a class implements the interfaces provided
+    by it's instances. An instance can also provide interfaces
+    directly. The interfaces provided by an object are the union of
+    the interfaces provided directly and the interfaces implemented by
+    the class.
+
+    This interface is implemented by :mod:`zope.interface`.
+    """
+    # pylint:disable=arguments-differ
+    ###
+    # Defining interfaces
+    ###
+
+    Interface = Attribute("The base class used to create new interfaces")
+
+    def taggedValue(key, value):
+        """
+        Attach a tagged value to an interface while defining the interface.
+
+        This is a way of executing :meth:`IElement.setTaggedValue` from
+        the definition of the interface. For example::
+
+             class IFoo(Interface):
+                 taggedValue('key', 'value')
+
+        .. seealso:: `zope.interface.taggedValue`
+        """
+
+    def invariant(checker_function):
+        """
+        Attach an invariant checker function to an interface while defining it.
+
+        Invariants can later be validated against particular implementations by
+        calling :meth:`IInterface.validateInvariants`.
+
+        For example::
+
+             def check_range(ob):
+                 if ob.max < ob.min:
+                     raise ValueError("max value is less than min value")
+
+             class IRange(Interface):
+                 min = Attribute("The min value")
+                 max = Attribute("The max value")
+
+                 invariant(check_range)
+
+        .. seealso:: `zope.interface.invariant`
+        """
+
+    def interfacemethod(method):
+        """
+        A decorator that transforms a method specification into an
+        implementation method.
+
+        This is used to override methods of ``Interface`` or provide new methods.
+        Definitions using this decorator will not appear in :meth:`IInterface.names()`.
+        It is possible to have an implementation method and a method specification
+        of the same name.
+
+        For example::
+
+             class IRange(Interface):
+                 @interfacemethod
+                 def __adapt__(self, obj):
+                     if isinstance(obj, range):
+                         # Return the builtin ``range`` as-is
+                         return obj
+                     return super(type(IRange), self).__adapt__(obj)
+
+        You can use ``super`` to call the parent class functionality. Note that
+        the zero-argument version (``super().__adapt__``) works on Python 3.6 and above, but
+        prior to that the two-argument version must be used, and the class must be explicitly
+        passed as the first argument.
+
+        .. versionadded:: 5.1.0
+        .. seealso:: `zope.interface.interfacemethod`
+        """
+
+    ###
+    # Querying interfaces
+    ###
+
+    def providedBy(ob):
+        """
+        Return the interfaces provided by an object.
+
+        This is the union of the interfaces directly provided by an
+        object and interfaces implemented by it's class.
+
+        The value returned is an `IDeclaration`.
+
+        .. seealso:: `zope.interface.providedBy`
+        """
+
+    def implementedBy(class_):
+        """
+        Return the interfaces implemented for a class's instances.
+
+        The value returned is an `IDeclaration`.
+
+        .. seealso:: `zope.interface.implementedBy`
+        """
+
+    ###
+    # Declaring interfaces
+    ###
+
+    def classImplements(class_, *interfaces):
+        """
+        Declare additional interfaces implemented for instances of a class.
+
+        The arguments after the class are one or more interfaces or
+        interface specifications (`IDeclaration` objects).
+
+        The interfaces given (including the interfaces in the
+        specifications) are added to any interfaces previously
+        declared.
+
+        Consider the following example::
+
+          class C(A, B):
+             ...
+
+          classImplements(C, I1, I2)
+
+
+        Instances of ``C`` provide ``I1``, ``I2``, and whatever interfaces
+        instances of ``A`` and ``B`` provide. This is equivalent to::
+
+            @implementer(I1, I2)
+            class C(A, B):
+                pass
+
+        .. seealso:: `zope.interface.classImplements`
+        .. seealso:: `zope.interface.implementer`
+        """
+
+    def classImplementsFirst(cls, interface):
+        """
+        See :func:`zope.interface.classImplementsFirst`.
+        """
+
+    def implementer(*interfaces):
+        """
+        Create a decorator for declaring interfaces implemented by a
+        factory.
+
+        A callable is returned that makes an implements declaration on
+        objects passed to it.
+
+        .. seealso:: :meth:`classImplements`
+        """
+
+    def classImplementsOnly(class_, *interfaces):
+        """
+        Declare the only interfaces implemented by instances of a class.
+
+        The arguments after the class are one or more interfaces or
+        interface specifications (`IDeclaration` objects).
+
+        The interfaces given (including the interfaces in the
+        specifications) replace any previous declarations.
+
+        Consider the following example::
+
+          class C(A, B):
+             ...
+
+          classImplements(C, IA, IB. IC)
+          classImplementsOnly(C. I1, I2)
+
+        Instances of ``C`` provide only ``I1``, ``I2``, and regardless of
+        whatever interfaces instances of ``A`` and ``B`` implement.
+
+        .. seealso:: `zope.interface.classImplementsOnly`
+        """
+
+    def implementer_only(*interfaces):
+        """
+        Create a decorator for declaring the only interfaces implemented.
+
+        A callable is returned that makes an implements declaration on
+        objects passed to it.
+
+        .. seealso:: `zope.interface.implementer_only`
+        """
+
+    def directlyProvidedBy(object): # pylint:disable=redefined-builtin
+        """
+        Return the interfaces directly provided by the given object.
+
+        The value returned is an `IDeclaration`.
+
+        .. seealso:: `zope.interface.directlyProvidedBy`
+        """
+
+    def directlyProvides(object, *interfaces): # pylint:disable=redefined-builtin
+        """
+        Declare interfaces declared directly for an object.
+
+        The arguments after the object are one or more interfaces or
+        interface specifications (`IDeclaration` objects).
+
+        .. caution::
+           The interfaces given (including the interfaces in the
+           specifications) *replace* interfaces previously
+           declared for the object. See :meth:`alsoProvides` to add
+           additional interfaces.
+
+        Consider the following example::
+
+          class C(A, B):
+             ...
+
+          ob = C()
+          directlyProvides(ob, I1, I2)
+
+        The object, ``ob`` provides ``I1``, ``I2``, and whatever interfaces
+        instances have been declared for instances of ``C``.
+
+        To remove directly provided interfaces, use `directlyProvidedBy` and
+        subtract the unwanted interfaces. For example::
+
+          directlyProvides(ob, directlyProvidedBy(ob)-I2)
+
+        removes I2 from the interfaces directly provided by
+        ``ob``. The object, ``ob`` no longer directly provides ``I2``,
+        although it might still provide ``I2`` if it's class
+        implements ``I2``.
+
+        To add directly provided interfaces, use `directlyProvidedBy` and
+        include additional interfaces.  For example::
+
+          directlyProvides(ob, directlyProvidedBy(ob), I2)
+
+        adds I2 to the interfaces directly provided by ob.
+
+        .. seealso:: `zope.interface.directlyProvides`
+        """
+
+    def alsoProvides(object, *interfaces): # pylint:disable=redefined-builtin
+        """
+        Declare additional interfaces directly for an object.
+
+        For example::
+
+          alsoProvides(ob, I1)
+
+        is equivalent to::
+
+          directlyProvides(ob, directlyProvidedBy(ob), I1)
+
+        .. seealso:: `zope.interface.alsoProvides`
+        """
+
+    def noLongerProvides(object, interface): # pylint:disable=redefined-builtin
+        """
+        Remove an interface from the list of an object's directly provided
+        interfaces.
+
+        For example::
+
+          noLongerProvides(ob, I1)
+
+        is equivalent to::
+
+          directlyProvides(ob, directlyProvidedBy(ob) - I1)
+
+        with the exception that if ``I1`` is an interface that is
+        provided by ``ob`` through the class's implementation,
+        `ValueError` is raised.
+
+        .. seealso:: `zope.interface.noLongerProvides`
+        """
+
+    def provider(*interfaces):
+        """
+        Declare interfaces provided directly by a class.
+
+        .. seealso:: `zope.interface.provider`
+        """
+
+    def moduleProvides(*interfaces):
+        """
+        Declare interfaces provided by a module.
+
+        This function is used in a module definition.
+
+        The arguments are one or more interfaces or interface
+        specifications (`IDeclaration` objects).
+
+        The given interfaces (including the interfaces in the
+        specifications) are used to create the module's direct-object
+        interface specification.  An error will be raised if the module
+        already has an interface specification.  In other words, it is
+        an error to call this function more than once in a module
+        definition.
+
+        This function is provided for convenience. It provides a more
+        convenient way to call `directlyProvides` for a module. For example::
+
+          moduleImplements(I1)
+
+        is equivalent to::
+
+          directlyProvides(sys.modules[__name__], I1)
+
+        .. seealso:: `zope.interface.moduleProvides`
+        """
+
+    def Declaration(*interfaces):
+        """
+        Create an interface specification.
+
+        The arguments are one or more interfaces or interface
+        specifications (`IDeclaration` objects).
+
+        A new interface specification (`IDeclaration`) with the given
+        interfaces is returned.
+
+        .. seealso:: `zope.interface.Declaration`
+        """
+
+class IAdapterRegistry(Interface):
+    """Provide an interface-based registry for adapters
+
+    This registry registers objects that are in some sense "from" a
+    sequence of specification to an interface and a name.
+
+    No specific semantics are assumed for the registered objects,
+    however, the most common application will be to register factories
+    that adapt objects providing required specifications to a provided
+    interface.
+    """
+
+    def register(required, provided, name, value):
+        """Register a value
+
+        A value is registered for a *sequence* of required specifications, a
+        provided interface, and a name, which must be text.
+        """
+
+    def registered(required, provided, name=''):
+        """Return the component registered for the given interfaces and name
+
+        name must be text.
+
+        Unlike the lookup method, this methods won't retrieve
+        components registered for more specific required interfaces or
+        less specific provided interfaces.
+
+        If no component was registered exactly for the given
+        interfaces and name, then None is returned.
+
+        """
+
+    def lookup(required, provided, name='', default=None):
+        """Lookup a value
+
+        A value is looked up based on a *sequence* of required
+        specifications, a provided interface, and a name, which must be
+        text.
+        """
+
+    def queryMultiAdapter(objects, provided, name='', default=None):
+        """Adapt a sequence of objects to a named, provided, interface
+        """
+
+    def lookup1(required, provided, name='', default=None):
+        """Lookup a value using a single required interface
+
+        A value is looked up based on a single required
+        specifications, a provided interface, and a name, which must be
+        text.
+        """
+
+    def queryAdapter(object, provided, name='', default=None): # pylint:disable=redefined-builtin
+        """Adapt an object using a registered adapter factory.
+        """
+
+    def adapter_hook(provided, object, name='', default=None): # pylint:disable=redefined-builtin
+        """Adapt an object using a registered adapter factory.
+
+        name must be text.
+        """
+
+    def lookupAll(required, provided):
+        """Find all adapters from the required to the provided interfaces
+
+        An iterable object is returned that provides name-value two-tuples.
+        """
+
+    def names(required, provided): # pylint:disable=arguments-differ
+        """Return the names for which there are registered objects
+        """
+
+    def subscribe(required, provided, subscriber): # pylint:disable=arguments-differ
+        """Register a subscriber
+
+        A subscriber is registered for a *sequence* of required
+        specifications, a provided interface, and a name.
+
+        Multiple subscribers may be registered for the same (or
+        equivalent) interfaces.
+
+        .. versionchanged:: 5.1.1
+           Correct the method signature to remove the ``name`` parameter.
+           Subscribers have no names.
+        """
+
+    def subscribed(required, provided, subscriber):
+        """
+        Check whether the object *subscriber* is registered directly
+        with this object via a previous call to
+        ``subscribe(required, provided, subscriber)``.
+
+        If the *subscriber*, or one equal to it, has been subscribed,
+        for the given *required* sequence and *provided* interface,
+        return that object. (This does not guarantee whether the *subscriber*
+        itself is returned, or an object equal to it.)
+
+        If it has not, return ``None``.
+
+        Unlike :meth:`subscriptions`, this method won't retrieve
+        components registered for more specific required interfaces or
+        less specific provided interfaces.
+
+        .. versionadded:: 5.3.0
+        """
+
+    def subscriptions(required, provided):
+        """
+        Get a sequence of subscribers.
+
+        Subscribers for a sequence of *required* interfaces, and a *provided*
+        interface are returned. This takes into account subscribers
+        registered with this object, as well as those registered with
+        base adapter registries in the resolution order, and interfaces that
+        extend *provided*.
+
+        .. versionchanged:: 5.1.1
+           Correct the method signature to remove the ``name`` parameter.
+           Subscribers have no names.
+        """
+
+    def subscribers(objects, provided):
+        """
+        Get a sequence of subscription **adapters**.
+
+        This is like :meth:`subscriptions`, but calls the returned
+        subscribers with *objects* (and optionally returns the results
+        of those calls), instead of returning the subscribers directly.
+
+        :param objects: A sequence of objects; they will be used to
+            determine the *required* argument to :meth:`subscriptions`.
+        :param provided: A single interface, or ``None``, to pass
+            as the *provided* parameter to :meth:`subscriptions`.
+            If an interface is given, the results of calling each returned
+            subscriber with the the *objects* are collected and returned
+            from this method; each result should be an object implementing
+            the *provided* interface. If ``None``, the resulting subscribers
+            are still called, but the results are ignored.
+        :return: A sequence of the results of calling the subscribers
+            if *provided* is not ``None``. If there are no registered
+            subscribers, or *provided* is ``None``, this will be an empty
+            sequence.
+
+        .. versionchanged:: 5.1.1
+           Correct the method signature to remove the ``name`` parameter.
+           Subscribers have no names.
+        """
+
+# begin formerly in zope.component
+
+class ComponentLookupError(LookupError):
+    """A component could not be found."""
+
+class Invalid(Exception):
+    """A component doesn't satisfy a promise."""
+
+class IObjectEvent(Interface):
+    """An event related to an object.
+
+    The object that generated this event is not necessarily the object
+    referred to by location.
+    """
+
+    object = Attribute("The subject of the event.")
+
+
+@implementer(IObjectEvent)
+class ObjectEvent:
+
+    def __init__(self, object): # pylint:disable=redefined-builtin
+        self.object = object
+
+
+class IComponentLookup(Interface):
+    """Component Manager for a Site
+
+    This object manages the components registered at a particular site. The
+    definition of a site is intentionally vague.
+    """
+
+    adapters = Attribute(
+        "Adapter Registry to manage all registered adapters.")
+
+    utilities = Attribute(
+        "Adapter Registry to manage all registered utilities.")
+
+    def queryAdapter(object, interface, name='', default=None): # pylint:disable=redefined-builtin
+        """Look for a named adapter to an interface for an object
+
+        If a matching adapter cannot be found, returns the default.
+        """
+
+    def getAdapter(object, interface, name=''): # pylint:disable=redefined-builtin
+        """Look for a named adapter to an interface for an object
+
+        If a matching adapter cannot be found, a `ComponentLookupError`
+        is raised.
+        """
+
+    def queryMultiAdapter(objects, interface, name='', default=None):
+        """Look for a multi-adapter to an interface for multiple objects
+
+        If a matching adapter cannot be found, returns the default.
+        """
+
+    def getMultiAdapter(objects, interface, name=''):
+        """Look for a multi-adapter to an interface for multiple objects
+
+        If a matching adapter cannot be found, a `ComponentLookupError`
+        is raised.
+        """
+
+    def getAdapters(objects, provided):
+        """Look for all matching adapters to a provided interface for objects
+
+        Return an iterable of name-adapter pairs for adapters that
+        provide the given interface.
+        """
+
+    def subscribers(objects, provided):
+        """Get subscribers
+
+        Subscribers are returned that provide the provided interface
+        and that depend on and are computed from the sequence of
+        required objects.
+        """
+
+    def handle(*objects):
+        """Call handlers for the given objects
+
+        Handlers registered for the given objects are called.
+        """
+
+    def queryUtility(interface, name='', default=None):
+        """Look up a utility that provides an interface.
+
+        If one is not found, returns default.
+        """
+
+    def getUtilitiesFor(interface):
+        """Look up the registered utilities that provide an interface.
+
+        Returns an iterable of name-utility pairs.
+        """
+
+    def getAllUtilitiesRegisteredFor(interface):
+        """Return all registered utilities for an interface
+
+        This includes overridden utilities.
+
+        An iterable of utility instances is returned.  No names are
+        returned.
+        """
+
+class IRegistration(Interface):
+    """A registration-information object
+    """
+
+    registry = Attribute("The registry having the registration")
+
+    name = Attribute("The registration name")
+
+    info = Attribute("""Information about the registration
+
+    This is information deemed useful to people browsing the
+    configuration of a system. It could, for example, include
+    commentary or information about the source of the configuration.
+    """)
+
+class IUtilityRegistration(IRegistration):
+    """Information about the registration of a utility
+    """
+
+    factory = Attribute("The factory used to create the utility. Optional.")
+    component = Attribute("The object registered")
+    provided = Attribute("The interface provided by the component")
+
+class _IBaseAdapterRegistration(IRegistration):
+    """Information about the registration of an adapter
+    """
+
+    factory = Attribute("The factory used to create adapters")
+
+    required = Attribute("""The adapted interfaces
+
+    This is a sequence of interfaces adapters by the registered
+    factory.  The factory will be caled with a sequence of objects, as
+    positional arguments, that provide these interfaces.
+    """)
+
+    provided = Attribute("""The interface provided by the adapters.
+
+    This interface is implemented by the factory
+    """)
+
+class IAdapterRegistration(_IBaseAdapterRegistration):
+    """Information about the registration of an adapter
+    """
+
+class ISubscriptionAdapterRegistration(_IBaseAdapterRegistration):
+    """Information about the registration of a subscription adapter
+    """
+
+class IHandlerRegistration(IRegistration):
+
+    handler = Attribute("An object called used to handle an event")
+
+    required = Attribute("""The handled interfaces
+
+    This is a sequence of interfaces handled by the registered
+    handler.  The handler will be caled with a sequence of objects, as
+    positional arguments, that provide these interfaces.
+    """)
+
+class IRegistrationEvent(IObjectEvent):
+    """An event that involves a registration"""
+
+
+@implementer(IRegistrationEvent)
+class RegistrationEvent(ObjectEvent):
+    """There has been a change in a registration
+    """
+    def __repr__(self):
+        return "{} event:\n{!r}".format(self.__class__.__name__, self.object)
+
+class IRegistered(IRegistrationEvent):
+    """A component or factory was registered
+    """
+
+@implementer(IRegistered)
+class Registered(RegistrationEvent):
+    pass
+
+class IUnregistered(IRegistrationEvent):
+    """A component or factory was unregistered
+    """
+
+@implementer(IUnregistered)
+class Unregistered(RegistrationEvent):
+    """A component or factory was unregistered
+    """
+
+
+class IComponentRegistry(Interface):
+    """Register components
+    """
+
+    def registerUtility(component=None, provided=None, name='',
+                        info='', factory=None):
+        """Register a utility
+
+        :param factory:
+           Factory for the component to be registered.
+
+        :param component:
+           The registered component
+
+        :param provided:
+           This is the interface provided by the utility.  If the
+           component provides a single interface, then this
+           argument is optional and the component-implemented
+           interface will be used.
+
+        :param name:
+           The utility name.
+
+        :param info:
+           An object that can be converted to a string to provide
+           information about the registration.
+
+        Only one of *component* and *factory* can be used.
+
+        A `IRegistered` event is generated with an `IUtilityRegistration`.
+        """
+
+    def unregisterUtility(component=None, provided=None, name='',
+                          factory=None):
+        """Unregister a utility
+
+        :returns:
+            A boolean is returned indicating whether the registry was
+            changed.  If the given *component* is None and there is no
+            component registered, or if the given *component* is not
+            None and is not registered, then the function returns
+            False, otherwise it returns True.
+
+        :param factory:
+           Factory for the component to be unregistered.
+
+        :param component:
+           The registered component The given component can be
+           None, in which case any component registered to provide
+           the given provided interface with the given name is
+           unregistered.
+
+        :param provided:
+           This is the interface provided by the utility.  If the
+           component is not None and provides a single interface,
+           then this argument is optional and the
+           component-implemented interface will be used.
+
+        :param name:
+           The utility name.
+
+        Only one of *component* and *factory* can be used.
+        An `IUnregistered` event is generated with an `IUtilityRegistration`.
+        """
+
+    def registeredUtilities():
+        """Return an iterable of `IUtilityRegistration` instances.
+
+        These registrations describe the current utility registrations
+        in the object.
+        """
+
+    def registerAdapter(factory, required=None, provided=None, name='',
+                        info=''):
+        """Register an adapter factory
+
+        :param factory:
+            The object used to compute the adapter
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set in class definitions using
+            the `.adapter`
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param provided:
+            This is the interface provided by the adapter and
+            implemented by the factory.  If the factory
+            implements a single interface, then this argument is
+            optional and the factory-implemented interface will be
+            used.
+
+        :param name:
+            The adapter name.
+
+        :param info:
+           An object that can be converted to a string to provide
+           information about the registration.
+
+        A `IRegistered` event is generated with an `IAdapterRegistration`.
+        """
+
+    def unregisterAdapter(factory=None, required=None,
+                          provided=None, name=''):
+        """Unregister an adapter factory
+
+        :returns:
+            A boolean is returned indicating whether the registry was
+            changed.  If the given component is None and there is no
+            component registered, or if the given component is not
+            None and is not registered, then the function returns
+            False, otherwise it returns True.
+
+        :param factory:
+            This is the object used to compute the adapter. The
+            factory can be None, in which case any factory
+            registered to implement the given provided interface
+            for the given required specifications with the given
+            name is unregistered.
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If the factory is not None and the required
+            arguments is omitted, then the value of the factory's
+            __component_adapts__ attribute will be used.  The
+            __component_adapts__ attribute attribute is normally
+            set in class definitions using adapts function, or for
+            callables using the adapter decorator.  If the factory
+            is None or doesn't have a __component_adapts__ adapts
+            attribute, then this argument is required.
+
+        :param provided:
+            This is the interface provided by the adapter and
+            implemented by the factory.  If the factory is not
+            None and implements a single interface, then this
+            argument is optional and the factory-implemented
+            interface will be used.
+
+        :param name:
+            The adapter name.
+
+        An `IUnregistered` event is generated with an `IAdapterRegistration`.
+        """
+
+    def registeredAdapters():
+        """Return an iterable of `IAdapterRegistration` instances.
+
+        These registrations describe the current adapter registrations
+        in the object.
+        """
+
+    def registerSubscriptionAdapter(factory, required=None, provides=None,
+                                    name='', info=''):
+        """Register a subscriber factory
+
+        :param factory:
+            The object used to compute the adapter
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set using the adapter
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param provided:
+            This is the interface provided by the adapter and
+            implemented by the factory.  If the factory implements
+            a single interface, then this argument is optional and
+            the factory-implemented interface will be used.
+
+        :param name:
+            The adapter name.
+
+            Currently, only the empty string is accepted.  Other
+            strings will be accepted in the future when support for
+            named subscribers is added.
+
+        :param info:
+           An object that can be converted to a string to provide
+           information about the registration.
+
+        A `IRegistered` event is generated with an
+        `ISubscriptionAdapterRegistration`.
+        """
+
+    def unregisterSubscriptionAdapter(factory=None, required=None,
+                                      provides=None, name=''):
+        """Unregister a subscriber factory.
+
+        :returns:
+            A boolean is returned indicating whether the registry was
+            changed.  If the given component is None and there is no
+            component registered, or if the given component is not
+            None and is not registered, then the function returns
+            False, otherwise it returns True.
+
+        :param factory:
+            This is the object used to compute the adapter. The
+            factory can be None, in which case any factories
+            registered to implement the given provided interface
+            for the given required specifications with the given
+            name are unregistered.
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set using the adapter
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param provided:
+            This is the interface provided by the adapter and
+            implemented by the factory.  If the factory is not
+            None implements a single interface, then this argument
+            is optional and the factory-implemented interface will
+            be used.
+
+        :param name:
+            The adapter name.
+
+            Currently, only the empty string is accepted.  Other
+            strings will be accepted in the future when support for
+            named subscribers is added.
+
+        An `IUnregistered` event is generated with an
+        `ISubscriptionAdapterRegistration`.
+        """
+
+    def registeredSubscriptionAdapters():
+        """Return an iterable of `ISubscriptionAdapterRegistration` instances.
+
+        These registrations describe the current subscription adapter
+        registrations in the object.
+        """
+
+    def registerHandler(handler, required=None, name='', info=''):
+        """Register a handler.
+
+        A handler is a subscriber that doesn't compute an adapter
+        but performs some function when called.
+
+        :param handler:
+            The object used to handle some event represented by
+            the objects passed to it.
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set using the adapter
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param name:
+            The handler name.
+
+            Currently, only the empty string is accepted.  Other
+            strings will be accepted in the future when support for
+            named handlers is added.
+
+        :param info:
+           An object that can be converted to a string to provide
+           information about the registration.
+
+
+        A `IRegistered` event is generated with an `IHandlerRegistration`.
+        """
+
+    def unregisterHandler(handler=None, required=None, name=''):
+        """Unregister a handler.
+
+        A handler is a subscriber that doesn't compute an adapter
+        but performs some function when called.
+
+        :returns: A boolean is returned indicating whether the registry was
+            changed.
+
+        :param handler:
+            This is the object used to handle some event
+            represented by the objects passed to it. The handler
+            can be None, in which case any handlers registered for
+            the given required specifications with the given are
+            unregistered.
+
+        :param required:
+            This is a sequence of specifications for objects to be
+            adapted.  If omitted, then the value of the factory's
+            ``__component_adapts__`` attribute will be used.  The
+            ``__component_adapts__`` attribute is
+            normally set using the adapter
+            decorator.  If the factory doesn't have a
+            ``__component_adapts__`` adapts attribute, then this
+            argument is required.
+
+        :param name:
+            The handler name.
+
+            Currently, only the empty string is accepted.  Other
+            strings will be accepted in the future when support for
+            named handlers is added.
+
+        An `IUnregistered` event is generated with an `IHandlerRegistration`.
+        """
+
+    def registeredHandlers():
+        """Return an iterable of `IHandlerRegistration` instances.
+
+        These registrations describe the current handler registrations
+        in the object.
+        """
+
+
+class IComponents(IComponentLookup, IComponentRegistry):
+    """Component registration and access
+    """
+
+
+# end formerly in zope.component
diff --git a/contrib/python/zope.interface/py3/zope/interface/registry.py b/contrib/python/zope.interface/py3/zope/interface/registry.py
new file mode 100644
index 00000000000..292499dbeca
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/registry.py
@@ -0,0 +1,723 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Basic components support
+"""
+from collections import defaultdict
+
+try:
+    from zope.event import notify
+except ImportError: # pragma: no cover
+    def notify(*arg, **kw): pass
+
+from zope.interface.interfaces import ISpecification
+from zope.interface.interfaces import ComponentLookupError
+from zope.interface.interfaces import IAdapterRegistration
+from zope.interface.interfaces import IComponents
+from zope.interface.interfaces import IHandlerRegistration
+from zope.interface.interfaces import ISubscriptionAdapterRegistration
+from zope.interface.interfaces import IUtilityRegistration
+from zope.interface.interfaces import Registered
+from zope.interface.interfaces import Unregistered
+
+from zope.interface.interface import Interface
+from zope.interface.declarations import implementedBy
+from zope.interface.declarations import implementer
+from zope.interface.declarations import implementer_only
+from zope.interface.declarations import providedBy
+from zope.interface.adapter import AdapterRegistry
+
+__all__ = [
+    # Components is public API, but
+    # the *Registration classes are just implementations
+    # of public interfaces.
+    'Components',
+]
+
+class _UnhashableComponentCounter:
+    # defaultdict(int)-like object for unhashable components
+
+    def __init__(self, otherdict):
+        # [(component, count)]
+        self._data = [item for item in otherdict.items()]
+
+    def __getitem__(self, key):
+        for component, count in self._data:
+            if component == key:
+                return count
+        return 0
+
+    def __setitem__(self, component, count):
+        for i, data in enumerate(self._data):
+            if data[0] == component:
+                self._data[i] = component, count
+                return
+        self._data.append((component, count))
+
+    def __delitem__(self, component):
+        for i, data in enumerate(self._data):
+            if data[0] == component:
+                del self._data[i]
+                return
+        raise KeyError(component) # pragma: no cover
+
+def _defaultdict_int():
+    return defaultdict(int)
+
+class _UtilityRegistrations:
+
+    def __init__(self, utilities, utility_registrations):
+        # {provided -> {component: count}}
+        self._cache = defaultdict(_defaultdict_int)
+        self._utilities = utilities
+        self._utility_registrations = utility_registrations
+
+        self.__populate_cache()
+
+    def __populate_cache(self):
+        for ((p, _), data) in iter(self._utility_registrations.items()):
+            component = data[0]
+            self.__cache_utility(p, component)
+
+    def __cache_utility(self, provided, component):
+        try:
+            self._cache[provided][component] += 1
+        except TypeError:
+            # The component is not hashable, and we have a dict. Switch to a strategy
+            # that doesn't use hashing.
+            prov = self._cache[provided] = _UnhashableComponentCounter(self._cache[provided])
+            prov[component] += 1
+
+    def __uncache_utility(self, provided, component):
+        provided = self._cache[provided]
+        # It seems like this line could raise a TypeError if component isn't
+        # hashable and we haven't yet switched to _UnhashableComponentCounter. However,
+        # we can't actually get in that situation. In order to get here, we would
+        # have had to cache the utility already which would have switched
+        # the datastructure if needed.
+        count = provided[component]
+        count -= 1
+        if count == 0:
+            del provided[component]
+        else:
+            provided[component] = count
+        return count > 0
+
+    def _is_utility_subscribed(self, provided, component):
+        try:
+            return self._cache[provided][component] > 0
+        except TypeError:
+            # Not hashable and we're still using a dict
+            return False
+
+    def registerUtility(self, provided, name, component, info, factory):
+        subscribed = self._is_utility_subscribed(provided, component)
+
+        self._utility_registrations[(provided, name)] = component, info, factory
+        self._utilities.register((), provided, name, component)
+
+        if not subscribed:
+            self._utilities.subscribe((), provided, component)
+
+        self.__cache_utility(provided, component)
+
+    def unregisterUtility(self, provided, name, component):
+        del self._utility_registrations[(provided, name)]
+        self._utilities.unregister((), provided, name)
+
+        subscribed = self.__uncache_utility(provided, component)
+
+        if not subscribed:
+            self._utilities.unsubscribe((), provided, component)
+
+
+@implementer(IComponents)
+class Components:
+
+    _v_utility_registrations_cache = None
+
+    def __init__(self, name='', bases=()):
+        # __init__ is used for test cleanup as well as initialization.
+        # XXX add a separate API for test cleanup.
+        assert isinstance(name, str)
+        self.__name__ = name
+        self._init_registries()
+        self._init_registrations()
+        self.__bases__ = tuple(bases)
+        self._v_utility_registrations_cache = None
+
+    def __repr__(self):
+        return "<{} {}>".format(self.__class__.__name__, self.__name__)
+
+    def __reduce__(self):
+        # Mimic what a persistent.Persistent object does and elide
+        # _v_ attributes so that they don't get saved in ZODB.
+        # This allows us to store things that cannot be pickled in such
+        # attributes.
+        reduction = super().__reduce__()
+        # (callable, args, state, listiter, dictiter)
+        # We assume the state is always a dict; the last three items
+        # are technically optional and can be missing or None.
+        filtered_state = {k: v for k, v in reduction[2].items()
+                          if not k.startswith('_v_')}
+        reduction = list(reduction)
+        reduction[2] = filtered_state
+        return tuple(reduction)
+
+    def _init_registries(self):
+        # Subclasses have never been required to call this method
+        # if they override it, merely to fill in these two attributes.
+        self.adapters = AdapterRegistry()
+        self.utilities = AdapterRegistry()
+
+    def _init_registrations(self):
+        self._utility_registrations = {}
+        self._adapter_registrations = {}
+        self._subscription_registrations = []
+        self._handler_registrations = []
+
+    @property
+    def _utility_registrations_cache(self):
+        # We use a _v_ attribute internally so that data aren't saved in ZODB,
+        # because this object cannot be pickled.
+        cache = self._v_utility_registrations_cache
+        if (cache is None
+            or cache._utilities is not self.utilities
+            or cache._utility_registrations is not self._utility_registrations):
+            cache = self._v_utility_registrations_cache = _UtilityRegistrations(
+                self.utilities,
+                self._utility_registrations)
+        return cache
+
+    def _getBases(self):
+        # Subclasses might override
+        return self.__dict__.get('__bases__', ())
+
+    def _setBases(self, bases):
+        # Subclasses might override
+        self.adapters.__bases__ = tuple([
+            base.adapters for base in bases])
+        self.utilities.__bases__ = tuple([
+            base.utilities for base in bases])
+        self.__dict__['__bases__'] = tuple(bases)
+
+    __bases__ = property(
+        lambda self: self._getBases(),
+        lambda self, bases: self._setBases(bases),
+        )
+
+    def registerUtility(self, component=None, provided=None, name='',
+                        info='', event=True, factory=None):
+        if factory:
+            if component:
+                raise TypeError("Can't specify factory and component.")
+            component = factory()
+
+        if provided is None:
+            provided = _getUtilityProvided(component)
+
+        if name == '':
+            name = _getName(component)
+
+        reg = self._utility_registrations.get((provided, name))
+        if reg is not None:
+            if reg[:2] == (component, info):
+                # already registered
+                return
+            self.unregisterUtility(reg[0], provided, name)
+
+        self._utility_registrations_cache.registerUtility(
+            provided, name, component, info, factory)
+
+        if event:
+            notify(Registered(
+                UtilityRegistration(self, provided, name, component, info,
+                                    factory)
+                ))
+
+    def unregisterUtility(self, component=None, provided=None, name='',
+                          factory=None):
+        if factory:
+            if component:
+                raise TypeError("Can't specify factory and component.")
+            component = factory()
+
+        if provided is None:
+            if component is None:
+                raise TypeError("Must specify one of component, factory and "
+                                "provided")
+            provided = _getUtilityProvided(component)
+
+        old = self._utility_registrations.get((provided, name))
+        if (old is None) or ((component is not None) and
+                             (component != old[0])):
+            return False
+
+        if component is None:
+            component = old[0]
+
+        # Note that component is now the old thing registered
+        self._utility_registrations_cache.unregisterUtility(
+            provided, name, component)
+
+        notify(Unregistered(
+            UtilityRegistration(self, provided, name, component, *old[1:])
+            ))
+
+        return True
+
+    def registeredUtilities(self):
+        for ((provided, name), data
+             ) in iter(self._utility_registrations.items()):
+            yield UtilityRegistration(self, provided, name, *data)
+
+    def queryUtility(self, provided, name='', default=None):
+        return self.utilities.lookup((), provided, name, default)
+
+    def getUtility(self, provided, name=''):
+        utility = self.utilities.lookup((), provided, name)
+        if utility is None:
+            raise ComponentLookupError(provided, name)
+        return utility
+
+    def getUtilitiesFor(self, interface):
+        yield from self.utilities.lookupAll((), interface)
+
+    def getAllUtilitiesRegisteredFor(self, interface):
+        return self.utilities.subscriptions((), interface)
+
+    def registerAdapter(self, factory, required=None, provided=None,
+                        name='', info='', event=True):
+        if provided is None:
+            provided = _getAdapterProvided(factory)
+        required = _getAdapterRequired(factory, required)
+        if name == '':
+            name = _getName(factory)
+        self._adapter_registrations[(required, provided, name)
+                                    ] = factory, info
+        self.adapters.register(required, provided, name, factory)
+
+        if event:
+            notify(Registered(
+                AdapterRegistration(self, required, provided, name,
+                                    factory, info)
+                ))
+
+
+    def unregisterAdapter(self, factory=None,
+                          required=None, provided=None, name='',
+                          ):
+        if provided is None:
+            if factory is None:
+                raise TypeError("Must specify one of factory and provided")
+            provided = _getAdapterProvided(factory)
+
+        if (required is None) and (factory is None):
+            raise TypeError("Must specify one of factory and required")
+
+        required = _getAdapterRequired(factory, required)
+        old = self._adapter_registrations.get((required, provided, name))
+        if (old is None) or ((factory is not None) and
+                             (factory != old[0])):
+            return False
+
+        del self._adapter_registrations[(required, provided, name)]
+        self.adapters.unregister(required, provided, name)
+
+        notify(Unregistered(
+            AdapterRegistration(self, required, provided, name,
+                                *old)
+            ))
+
+        return True
+
+    def registeredAdapters(self):
+        for ((required, provided, name), (component, info)
+             ) in iter(self._adapter_registrations.items()):
+            yield AdapterRegistration(self, required, provided, name,
+                                      component, info)
+
+    def queryAdapter(self, object, interface, name='', default=None):
+        return self.adapters.queryAdapter(object, interface, name, default)
+
+    def getAdapter(self, object, interface, name=''):
+        adapter = self.adapters.queryAdapter(object, interface, name)
+        if adapter is None:
+            raise ComponentLookupError(object, interface, name)
+        return adapter
+
+    def queryMultiAdapter(self, objects, interface, name='',
+                          default=None):
+        return self.adapters.queryMultiAdapter(
+            objects, interface, name, default)
+
+    def getMultiAdapter(self, objects, interface, name=''):
+        adapter = self.adapters.queryMultiAdapter(objects, interface, name)
+        if adapter is None:
+            raise ComponentLookupError(objects, interface, name)
+        return adapter
+
+    def getAdapters(self, objects, provided):
+        for name, factory in self.adapters.lookupAll(
+            list(map(providedBy, objects)),
+            provided):
+            adapter = factory(*objects)
+            if adapter is not None:
+                yield name, adapter
+
+    def registerSubscriptionAdapter(self,
+                                    factory, required=None, provided=None,
+                                    name='', info='',
+                                    event=True):
+        if name:
+            raise TypeError("Named subscribers are not yet supported")
+        if provided is None:
+            provided = _getAdapterProvided(factory)
+        required = _getAdapterRequired(factory, required)
+        self._subscription_registrations.append(
+            (required, provided, name, factory, info)
+            )
+        self.adapters.subscribe(required, provided, factory)
+
+        if event:
+            notify(Registered(
+                SubscriptionRegistration(self, required, provided, name,
+                                         factory, info)
+                ))
+
+    def registeredSubscriptionAdapters(self):
+        for data in self._subscription_registrations:
+            yield SubscriptionRegistration(self, *data)
+
+    def unregisterSubscriptionAdapter(self, factory=None,
+                          required=None, provided=None, name='',
+                          ):
+        if name:
+            raise TypeError("Named subscribers are not yet supported")
+        if provided is None:
+            if factory is None:
+                raise TypeError("Must specify one of factory and provided")
+            provided = _getAdapterProvided(factory)
+
+        if (required is None) and (factory is None):
+            raise TypeError("Must specify one of factory and required")
+
+        required = _getAdapterRequired(factory, required)
+
+        if factory is None:
+            new = [(r, p, n, f, i)
+                   for (r, p, n, f, i)
+                   in self._subscription_registrations
+                   if not (r == required and p == provided)
+                   ]
+        else:
+            new = [(r, p, n, f, i)
+                   for (r, p, n, f, i)
+                   in self._subscription_registrations
+                   if not (r == required and p == provided and f == factory)
+                   ]
+
+        if len(new) == len(self._subscription_registrations):
+            return False
+
+
+        self._subscription_registrations[:] = new
+        self.adapters.unsubscribe(required, provided, factory)
+
+        notify(Unregistered(
+            SubscriptionRegistration(self, required, provided, name,
+                                     factory, '')
+            ))
+
+        return True
+
+    def subscribers(self, objects, provided):
+        return self.adapters.subscribers(objects, provided)
+
+    def registerHandler(self,
+                        factory, required=None,
+                        name='', info='',
+                        event=True):
+        if name:
+            raise TypeError("Named handlers are not yet supported")
+        required = _getAdapterRequired(factory, required)
+        self._handler_registrations.append(
+            (required, name, factory, info)
+            )
+        self.adapters.subscribe(required, None, factory)
+
+        if event:
+            notify(Registered(
+                HandlerRegistration(self, required, name, factory, info)
+                ))
+
+    def registeredHandlers(self):
+        for data in self._handler_registrations:
+            yield HandlerRegistration(self, *data)
+
+    def unregisterHandler(self, factory=None, required=None, name=''):
+        if name:
+            raise TypeError("Named subscribers are not yet supported")
+
+        if (required is None) and (factory is None):
+            raise TypeError("Must specify one of factory and required")
+
+        required = _getAdapterRequired(factory, required)
+
+        if factory is None:
+            new = [(r, n, f, i)
+                   for (r, n, f, i)
+                   in self._handler_registrations
+                   if r != required
+                   ]
+        else:
+            new = [(r, n, f, i)
+                   for (r, n, f, i)
+                   in self._handler_registrations
+                   if not (r == required and f == factory)
+                   ]
+
+        if len(new) == len(self._handler_registrations):
+            return False
+
+        self._handler_registrations[:] = new
+        self.adapters.unsubscribe(required, None, factory)
+
+        notify(Unregistered(
+            HandlerRegistration(self, required, name, factory, '')
+            ))
+
+        return True
+
+    def handle(self, *objects):
+        self.adapters.subscribers(objects, None)
+
+    def rebuildUtilityRegistryFromLocalCache(self, rebuild=False):
+        """
+        Emergency maintenance method to rebuild the ``.utilities``
+        registry from the local copy maintained in this object, or
+        detect the need to do so.
+
+        Most users will never need to call this, but it can be helpful
+        in the event of suspected corruption.
+
+        By default, this method only checks for corruption. To make it
+        actually rebuild the registry, pass `True` for *rebuild*.
+
+        :param bool rebuild: If set to `True` (not the default),
+           this method will actually register and subscribe utilities
+           in the registry as needed to synchronize with the local cache.
+
+        :return: A dictionary that's meant as diagnostic data. The keys
+           and values may change over time. When called with a false *rebuild*,
+           the keys ``"needed_registered"`` and ``"needed_subscribed"`` will be
+           non-zero if any corruption was detected, but that will not be corrected.
+
+        .. versionadded:: 5.3.0
+        """
+        regs = dict(self._utility_registrations)
+        utils = self.utilities
+        needed_registered = 0
+        did_not_register = 0
+        needed_subscribed = 0
+        did_not_subscribe = 0
+
+
+        # Avoid the expensive change process during this; we'll call
+        # it once at the end if needed.
+        assert 'changed' not in utils.__dict__
+        utils.changed = lambda _: None
+
+        if rebuild:
+            register = utils.register
+            subscribe = utils.subscribe
+        else:
+            register = subscribe = lambda *args: None
+
+        try:
+            for (provided, name), (value, _info, _factory) in regs.items():
+                if utils.registered((), provided, name) != value:
+                    register((), provided, name, value)
+                    needed_registered += 1
+                else:
+                    did_not_register += 1
+
+                if utils.subscribed((), provided, value) is None:
+                    needed_subscribed += 1
+                    subscribe((), provided, value)
+                else:
+                    did_not_subscribe += 1
+        finally:
+            del utils.changed
+            if rebuild and (needed_subscribed or needed_registered):
+                utils.changed(utils)
+
+        return {
+            'needed_registered': needed_registered,
+            'did_not_register': did_not_register,
+            'needed_subscribed': needed_subscribed,
+            'did_not_subscribe': did_not_subscribe
+        }
+
+def _getName(component):
+    try:
+        return component.__component_name__
+    except AttributeError:
+        return ''
+
+def _getUtilityProvided(component):
+    provided = list(providedBy(component))
+    if len(provided) == 1:
+        return provided[0]
+    raise TypeError(
+        "The utility doesn't provide a single interface "
+        "and no provided interface was specified.")
+
+def _getAdapterProvided(factory):
+    provided = list(implementedBy(factory))
+    if len(provided) == 1:
+        return provided[0]
+    raise TypeError(
+        "The adapter factory doesn't implement a single interface "
+        "and no provided interface was specified.")
+
+def _getAdapterRequired(factory, required):
+    if required is None:
+        try:
+            required = factory.__component_adapts__
+        except AttributeError:
+            raise TypeError(
+                "The adapter factory doesn't have a __component_adapts__ "
+                "attribute and no required specifications were specified"
+                )
+    elif ISpecification.providedBy(required):
+        raise TypeError("the required argument should be a list of "
+                        "interfaces, not a single interface")
+
+    result = []
+    for r in required:
+        if r is None:
+            r = Interface
+        elif not ISpecification.providedBy(r):
+            if isinstance(r, type):
+                r = implementedBy(r)
+            else:
+                raise TypeError("Required specification must be a "
+                                "specification or class, not %r" % type(r)
+                                )
+        result.append(r)
+    return tuple(result)
+
+
+@implementer(IUtilityRegistration)
+class UtilityRegistration:
+
+    def __init__(self, registry, provided, name, component, doc, factory=None):
+        (self.registry, self.provided, self.name, self.component, self.info,
+         self.factory
+         ) = registry, provided, name, component, doc, factory
+
+    def __repr__(self):
+        return '{}({!r}, {}, {!r}, {}, {!r}, {!r})'.format(
+                self.__class__.__name__,
+                self.registry,
+                getattr(self.provided, '__name__', None), self.name,
+                getattr(self.component, '__name__', repr(self.component)),
+                self.factory, self.info,
+                )
+
+    def __hash__(self):
+        return id(self)
+
+    def __eq__(self, other):
+        return repr(self) == repr(other)
+
+    def __ne__(self, other):
+        return repr(self) != repr(other)
+
+    def __lt__(self, other):
+        return repr(self) < repr(other)
+
+    def __le__(self, other):
+        return repr(self) <= repr(other)
+
+    def __gt__(self, other):
+        return repr(self) > repr(other)
+
+    def __ge__(self, other):
+        return repr(self) >= repr(other)
+
+@implementer(IAdapterRegistration)
+class AdapterRegistration:
+
+    def __init__(self, registry, required, provided, name, component, doc):
+        (self.registry, self.required, self.provided, self.name,
+         self.factory, self.info
+         ) = registry, required, provided, name, component, doc
+
+    def __repr__(self):
+        return '{}({!r}, {}, {}, {!r}, {}, {!r})'.format(
+            self.__class__.__name__,
+            self.registry,
+            '[' + ", ".join([r.__name__ for r in self.required]) + ']',
+            getattr(self.provided, '__name__', None), self.name,
+            getattr(self.factory, '__name__', repr(self.factory)), self.info,
+            )
+
+    def __hash__(self):
+        return id(self)
+
+    def __eq__(self, other):
+        return repr(self) == repr(other)
+
+    def __ne__(self, other):
+        return repr(self) != repr(other)
+
+    def __lt__(self, other):
+        return repr(self) < repr(other)
+
+    def __le__(self, other):
+        return repr(self) <= repr(other)
+
+    def __gt__(self, other):
+        return repr(self) > repr(other)
+
+    def __ge__(self, other):
+        return repr(self) >= repr(other)
+
+@implementer_only(ISubscriptionAdapterRegistration)
+class SubscriptionRegistration(AdapterRegistration):
+    pass
+
+
+@implementer_only(IHandlerRegistration)
+class HandlerRegistration(AdapterRegistration):
+
+    def __init__(self, registry, required, name, handler, doc):
+        (self.registry, self.required, self.name, self.handler, self.info
+         ) = registry, required, name, handler, doc
+
+    @property
+    def factory(self):
+        return self.handler
+
+    provided = None
+
+    def __repr__(self):
+        return '{}({!r}, {}, {!r}, {}, {!r})'.format(
+            self.__class__.__name__,
+            self.registry,
+            '[' + ", ".join([r.__name__ for r in self.required]) + ']',
+            self.name,
+            getattr(self.factory, '__name__', repr(self.factory)), self.info,
+            )
diff --git a/contrib/python/zope.interface/py3/zope/interface/ro.py b/contrib/python/zope.interface/py3/zope/interface/ro.py
new file mode 100644
index 00000000000..17468e92318
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/ro.py
@@ -0,0 +1,665 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Compute a resolution order for an object and its bases.
+
+.. versionchanged:: 5.0
+   The resolution order is now based on the same C3 order that Python
+   uses for classes. In complex instances of multiple inheritance, this
+   may result in a different ordering.
+
+   In older versions, the ordering wasn't required to be C3 compliant,
+   and for backwards compatibility, it still isn't. If the ordering
+   isn't C3 compliant (if it is *inconsistent*), zope.interface will
+   make a best guess to try to produce a reasonable resolution order.
+   Still (just as before), the results in such cases may be
+   surprising.
+
+.. rubric:: Environment Variables
+
+Due to the change in 5.0, certain environment variables can be used to control errors
+and warnings about inconsistent resolution orders. They are listed in priority order, with
+variables at the bottom generally overriding variables above them.
+
+ZOPE_INTERFACE_WARN_BAD_IRO
+    If this is set to "1", then if there is at least one inconsistent resolution
+    order discovered, a warning (:class:`InconsistentResolutionOrderWarning`) will
+    be issued. Use the usual warning mechanisms to control this behaviour. The warning
+    text will contain additional information on debugging.
+ZOPE_INTERFACE_TRACK_BAD_IRO
+    If this is set to "1", then zope.interface will log information about each
+    inconsistent resolution order discovered, and keep those details in memory in this module
+    for later inspection.
+ZOPE_INTERFACE_STRICT_IRO
+    If this is set to "1", any attempt to use :func:`ro` that would produce a non-C3
+    ordering will fail by raising :class:`InconsistentResolutionOrderError`.
+
+.. important::
+
+   ``ZOPE_INTERFACE_STRICT_IRO`` is intended to become the default in the future.
+
+There are two environment variables that are independent.
+
+ZOPE_INTERFACE_LOG_CHANGED_IRO
+    If this is set to "1", then if the C3 resolution order is different from
+    the legacy resolution order for any given object, a message explaining the differences
+    will be logged. This is intended to be used for debugging complicated IROs.
+ZOPE_INTERFACE_USE_LEGACY_IRO
+    If this is set to "1", then the C3 resolution order will *not* be used. The
+    legacy IRO will be used instead. This is a temporary measure and will be removed in the
+    future. It is intended to help during the transition.
+    It implies ``ZOPE_INTERFACE_LOG_CHANGED_IRO``.
+
+.. rubric:: Debugging Behaviour Changes in zope.interface 5
+
+Most behaviour changes from zope.interface 4 to 5 are related to
+inconsistent resolution orders. ``ZOPE_INTERFACE_STRICT_IRO`` is the
+most effective tool to find such inconsistent resolution orders, and
+we recommend running your code with this variable set if at all
+possible. Doing so will ensure that all interface resolution orders
+are consistent, and if they're not, will immediately point the way to
+where this is violated.
+
+Occasionally, however, this may not be enough. This is because in some
+cases, a C3 ordering can be found (the resolution order is fully
+consistent) that is substantially different from the ad-hoc legacy
+ordering. In such cases, you may find that you get an unexpected value
+returned when adapting one or more objects to an interface. To debug
+this, *also* enable ``ZOPE_INTERFACE_LOG_CHANGED_IRO`` and examine the
+output. The main thing to look for is changes in the relative
+positions of interfaces for which there are registered adapters.
+"""
+__docformat__ = 'restructuredtext'
+
+__all__ = [
+    'ro',
+    'InconsistentResolutionOrderError',
+    'InconsistentResolutionOrderWarning',
+]
+
+__logger = None
+
+def _logger():
+    global __logger # pylint:disable=global-statement
+    if __logger is None:
+        import logging
+        __logger = logging.getLogger(__name__)
+    return __logger
+
+def _legacy_mergeOrderings(orderings):
+    """Merge multiple orderings so that within-ordering order is preserved
+
+    Orderings are constrained in such a way that if an object appears
+    in two or more orderings, then the suffix that begins with the
+    object must be in both orderings.
+
+    For example:
+
+    >>> _mergeOrderings([
+    ... ['x', 'y', 'z'],
+    ... ['q', 'z'],
+    ... [1, 3, 5],
+    ... ['z']
+    ... ])
+    ['x', 'y', 'q', 1, 3, 5, 'z']
+
+    """
+
+    seen = set()
+    result = []
+    for ordering in reversed(orderings):
+        for o in reversed(ordering):
+            if o not in seen:
+                seen.add(o)
+                result.insert(0, o)
+
+    return result
+
+def _legacy_flatten(begin):
+    result = [begin]
+    i = 0
+    for ob in iter(result):
+        i += 1
+        # The recursive calls can be avoided by inserting the base classes
+        # into the dynamically growing list directly after the currently
+        # considered object;  the iterator makes sure this will keep working
+        # in the future, since it cannot rely on the length of the list
+        # by definition.
+        result[i:i] = ob.__bases__
+    return result
+
+def _legacy_ro(ob):
+    return _legacy_mergeOrderings([_legacy_flatten(ob)])
+
+###
+# Compare base objects using identity, not equality. This matches what
+# the CPython MRO algorithm does, and is *much* faster to boot: that,
+# plus some other small tweaks makes the difference between 25s and 6s
+# in loading 446 plone/zope interface.py modules (1925 InterfaceClass,
+# 1200 Implements, 1100 ClassProvides objects)
+###
+
+
+class InconsistentResolutionOrderWarning(PendingDeprecationWarning):
+    """
+    The warning issued when an invalid IRO is requested.
+    """
+
+class InconsistentResolutionOrderError(TypeError):
+    """
+    The error raised when an invalid IRO is requested in strict mode.
+    """
+
+    def __init__(self, c3, base_tree_remaining):
+        self.C = c3.leaf
+        base_tree = c3.base_tree
+        self.base_ros = {
+            base: base_tree[i + 1]
+            for i, base in enumerate(self.C.__bases__)
+        }
+        # Unfortunately, this doesn't necessarily directly match
+        # up to any transformation on C.__bases__, because
+        # if any were fully used up, they were removed already.
+        self.base_tree_remaining = base_tree_remaining
+
+        TypeError.__init__(self)
+
+    def __str__(self):
+        import pprint
+        return "{}: For object {!r}.\nBase ROs:\n{}\nConflict Location:\n{}".format(
+            self.__class__.__name__,
+            self.C,
+            pprint.pformat(self.base_ros),
+            pprint.pformat(self.base_tree_remaining),
+        )
+
+
+class _NamedBool(int): # cannot actually inherit bool
+
+    def __new__(cls, val, name):
+        inst = super(cls, _NamedBool).__new__(cls, val)
+        inst.__name__ = name
+        return inst
+
+
+class _ClassBoolFromEnv:
+    """
+    Non-data descriptor that reads a transformed environment variable
+    as a boolean, and caches the result in the class.
+    """
+
+    def __get__(self, inst, klass):
+        import os
+        for cls in klass.__mro__:
+            my_name = None
+            for k in dir(klass):
+                if k in cls.__dict__ and cls.__dict__[k] is self:
+                    my_name = k
+                    break
+            if my_name is not None:
+                break
+        else: # pragma: no cover
+            raise RuntimeError("Unable to find self")
+
+        env_name = 'ZOPE_INTERFACE_' + my_name
+        val = os.environ.get(env_name, '') == '1'
+        val = _NamedBool(val, my_name)
+        setattr(klass, my_name, val)
+        setattr(klass, 'ORIG_' + my_name, self)
+        return val
+
+
+class _StaticMRO:
+    # A previously resolved MRO, supplied by the caller.
+    # Used in place of calculating it.
+
+    had_inconsistency = None # We don't know...
+
+    def __init__(self, C, mro):
+        self.leaf = C
+        self.__mro = tuple(mro)
+
+    def mro(self):
+        return list(self.__mro)
+
+
+class C3:
+    # Holds the shared state during computation of an MRO.
+
+    @staticmethod
+    def resolver(C, strict, base_mros):
+        strict = strict if strict is not None else C3.STRICT_IRO
+        factory = C3
+        if strict:
+            factory = _StrictC3
+        elif C3.TRACK_BAD_IRO:
+            factory = _TrackingC3
+
+        memo = {}
+        base_mros = base_mros or {}
+        for base, mro in base_mros.items():
+            assert base in C.__bases__
+            memo[base] = _StaticMRO(base, mro)
+
+        return factory(C, memo)
+
+    __mro = None
+    __legacy_ro = None
+    direct_inconsistency = False
+
+    def __init__(self, C, memo):
+        self.leaf = C
+        self.memo = memo
+        kind = self.__class__
+
+        base_resolvers = []
+        for base in C.__bases__:
+            if base not in memo:
+                resolver = kind(base, memo)
+                memo[base] = resolver
+            base_resolvers.append(memo[base])
+
+        self.base_tree = [
+            [C]
+        ] + [
+            memo[base].mro() for base in C.__bases__
+        ] + [
+            list(C.__bases__)
+        ]
+
+        self.bases_had_inconsistency = any(base.had_inconsistency for base in base_resolvers)
+
+        if len(C.__bases__) == 1:
+            self.__mro = [C] + memo[C.__bases__[0]].mro()
+
+    @property
+    def had_inconsistency(self):
+        return self.direct_inconsistency or self.bases_had_inconsistency
+
+    @property
+    def legacy_ro(self):
+        if self.__legacy_ro is None:
+            self.__legacy_ro = tuple(_legacy_ro(self.leaf))
+        return list(self.__legacy_ro)
+
+    TRACK_BAD_IRO = _ClassBoolFromEnv()
+    STRICT_IRO = _ClassBoolFromEnv()
+    WARN_BAD_IRO = _ClassBoolFromEnv()
+    LOG_CHANGED_IRO = _ClassBoolFromEnv()
+    USE_LEGACY_IRO = _ClassBoolFromEnv()
+    BAD_IROS = ()
+
+    def _warn_iro(self):
+        if not self.WARN_BAD_IRO:
+            # For the initial release, one must opt-in to see the warning.
+            # In the future (2021?) seeing at least the first warning will
+            # be the default
+            return
+        import warnings
+        warnings.warn(
+            "An inconsistent resolution order is being requested. "
+            "(Interfaces should follow the Python class rules known as C3.) "
+            "For backwards compatibility, zope.interface will allow this, "
+            "making the best guess it can to produce as meaningful an order as possible. "
+            "In the future this might be an error. Set the warning filter to error, or set "
+            "the environment variable 'ZOPE_INTERFACE_TRACK_BAD_IRO' to '1' and examine "
+            "ro.C3.BAD_IROS to debug, or set 'ZOPE_INTERFACE_STRICT_IRO' to raise exceptions.",
+            InconsistentResolutionOrderWarning,
+        )
+
+    @staticmethod
+    def _can_choose_base(base, base_tree_remaining):
+        # From C3:
+        # nothead = [s for s in nonemptyseqs if cand in s[1:]]
+        for bases in base_tree_remaining:
+            if not bases or bases[0] is base:
+                continue
+
+            for b in bases:
+                if b is base:
+                    return False
+        return True
+
+    @staticmethod
+    def _nonempty_bases_ignoring(base_tree, ignoring):
+        return list(filter(None, [
+            [b for b in bases if b is not ignoring]
+            for bases
+            in base_tree
+        ]))
+
+    def _choose_next_base(self, base_tree_remaining):
+        """
+        Return the next base.
+
+        The return value will either fit the C3 constraints or be our best
+        guess about what to do. If we cannot guess, this may raise an exception.
+        """
+        base = self._find_next_C3_base(base_tree_remaining)
+        if base is not None:
+            return base
+        return self._guess_next_base(base_tree_remaining)
+
+    def _find_next_C3_base(self, base_tree_remaining):
+        """
+        Return the next base that fits the constraints, or ``None`` if there isn't one.
+        """
+        for bases in base_tree_remaining:
+            base = bases[0]
+            if self._can_choose_base(base, base_tree_remaining):
+                return base
+        return None
+
+    class _UseLegacyRO(Exception):
+        pass
+
+    def _guess_next_base(self, base_tree_remaining):
+        # Narf. We may have an inconsistent order (we won't know for
+        # sure until we check all the bases). Python cannot create
+        # classes like this:
+        #
+        # class B1:
+        #   pass
+        # class B2(B1):
+        #   pass
+        # class C(B1, B2): # -> TypeError; this is like saying C(B1, B2, B1).
+        #  pass
+        #
+        # However, older versions of zope.interface were fine with this order.
+        # A good example is ``providedBy(IOError())``. Because of the way
+        # ``classImplements`` works, it winds up with ``__bases__`` ==
+        # ``[IEnvironmentError, IIOError, IOSError, <implementedBy Exception>]``
+        # (on Python 3). But ``IEnvironmentError`` is a base of both ``IIOError``
+        # and ``IOSError``. Previously, we would get a resolution order of
+        # ``[IIOError, IOSError, IEnvironmentError, IStandardError, IException, Interface]``
+        # but the standard Python algorithm would forbid creating that order entirely.
+
+        # Unlike Python's MRO, we attempt to resolve the issue. A few
+        # heuristics have been tried. One was:
+        #
+        # Strip off the first (highest priority) base of each direct
+        # base one at a time and seeing if we can come to an agreement
+        # with the other bases. (We're trying for a partial ordering
+        # here.) This often resolves cases (such as the IOSError case
+        # above), and frequently produces the same ordering as the
+        # legacy MRO did. If we looked at all the highest priority
+        # bases and couldn't find any partial ordering, then we strip
+        # them *all* out and begin the C3 step again. We take care not
+        # to promote a common root over all others.
+        #
+        # If we only did the first part, stripped off the first
+        # element of the first item, we could resolve simple cases.
+        # But it tended to fail badly. If we did the whole thing, it
+        # could be extremely painful from a performance perspective
+        # for deep/wide things like Zope's OFS.SimpleItem.Item. Plus,
+        # anytime you get ExtensionClass.Base into the mix, you're
+        # likely to wind up in trouble, because it messes with the MRO
+        # of classes. Sigh.
+        #
+        # So now, we fall back to the old linearization (fast to compute).
+        self._warn_iro()
+        self.direct_inconsistency = InconsistentResolutionOrderError(self, base_tree_remaining)
+        raise self._UseLegacyRO
+
+    def _merge(self):
+        # Returns a merged *list*.
+        result = self.__mro = []
+        base_tree_remaining = self.base_tree
+        base = None
+        while 1:
+            # Take last picked base out of the base tree wherever it is.
+            # This differs slightly from the standard Python MRO and is needed
+            # because we have no other step that prevents duplicates
+            # from coming in (e.g., in the inconsistent fallback path)
+            base_tree_remaining = self._nonempty_bases_ignoring(base_tree_remaining, base)
+
+            if not base_tree_remaining:
+                return result
+            try:
+                base = self._choose_next_base(base_tree_remaining)
+            except self._UseLegacyRO:
+                self.__mro = self.legacy_ro
+                return self.legacy_ro
+
+            result.append(base)
+
+    def mro(self):
+        if self.__mro is None:
+            self.__mro = tuple(self._merge())
+        return list(self.__mro)
+
+
+class _StrictC3(C3):
+    __slots__ = ()
+    def _guess_next_base(self, base_tree_remaining):
+        raise InconsistentResolutionOrderError(self, base_tree_remaining)
+
+
+class _TrackingC3(C3):
+    __slots__ = ()
+    def _guess_next_base(self, base_tree_remaining):
+        import traceback
+        bad_iros = C3.BAD_IROS
+        if self.leaf not in bad_iros:
+            if bad_iros == ():
+                import weakref
+                # This is a race condition, but it doesn't matter much.
+                bad_iros = C3.BAD_IROS = weakref.WeakKeyDictionary()
+            bad_iros[self.leaf] = t = (
+                InconsistentResolutionOrderError(self, base_tree_remaining),
+                traceback.format_stack()
+            )
+            _logger().warning("Tracking inconsistent IRO: %s", t[0])
+        return C3._guess_next_base(self, base_tree_remaining)
+
+
+class _ROComparison:
+    # Exists to compute and print a pretty string comparison
+    # for differing ROs.
+    # Since we're used in a logging context, and may actually never be printed,
+    # this is a class so we can defer computing the diff until asked.
+
+    # Components we use to build up the comparison report
+    class Item:
+        prefix = '  '
+        def __init__(self, item):
+            self.item = item
+        def __str__(self):
+            return "{}{}".format(
+                self.prefix,
+                self.item,
+            )
+
+    class Deleted(Item):
+        prefix = '- '
+
+    class Inserted(Item):
+        prefix = '+ '
+
+    Empty = str
+
+    class ReplacedBy: # pragma: no cover
+        prefix = '- '
+        suffix = ''
+        def __init__(self, chunk, total_count):
+            self.chunk = chunk
+            self.total_count = total_count
+
+        def __iter__(self):
+            lines = [
+                self.prefix + str(item) + self.suffix
+                for item in self.chunk
+            ]
+            while len(lines) < self.total_count:
+                lines.append('')
+
+            return iter(lines)
+
+    class Replacing(ReplacedBy):
+        prefix = "+ "
+        suffix = ''
+
+
+    _c3_report = None
+    _legacy_report = None
+
+    def __init__(self, c3, c3_ro, legacy_ro):
+        self.c3 = c3
+        self.c3_ro = c3_ro
+        self.legacy_ro = legacy_ro
+
+    def __move(self, from_, to_, chunk, operation):
+        for x in chunk:
+            to_.append(operation(x))
+            from_.append(self.Empty())
+
+    def _generate_report(self):
+        if self._c3_report is None:
+            import difflib
+            # The opcodes we get describe how to turn 'a' into 'b'. So
+            # the old one (legacy) needs to be first ('a')
+            matcher = difflib.SequenceMatcher(None, self.legacy_ro, self.c3_ro)
+            # The reports are equal length sequences. We're going for a
+            # side-by-side diff.
+            self._c3_report = c3_report = []
+            self._legacy_report = legacy_report = []
+            for opcode, leg1, leg2, c31, c32 in matcher.get_opcodes():
+                c3_chunk = self.c3_ro[c31:c32]
+                legacy_chunk = self.legacy_ro[leg1:leg2]
+
+                if opcode == 'equal':
+                    # Guaranteed same length
+                    c3_report.extend(self.Item(x) for x in c3_chunk)
+                    legacy_report.extend(self.Item(x) for x in legacy_chunk)
+                if opcode == 'delete':
+                    # Guaranteed same length
+                    assert not c3_chunk
+                    self.__move(c3_report, legacy_report, legacy_chunk, self.Deleted)
+                if opcode == 'insert':
+                    # Guaranteed same length
+                    assert not legacy_chunk
+                    self.__move(legacy_report, c3_report, c3_chunk, self.Inserted)
+                if opcode == 'replace': # pragma: no cover (How do you make it output this?)
+                    # Either side could be longer.
+                    chunk_size = max(len(c3_chunk), len(legacy_chunk))
+                    c3_report.extend(self.Replacing(c3_chunk, chunk_size))
+                    legacy_report.extend(self.ReplacedBy(legacy_chunk, chunk_size))
+
+        return self._c3_report, self._legacy_report
+
+    @property
+    def _inconsistent_label(self):
+        inconsistent = []
+        if self.c3.direct_inconsistency:
+            inconsistent.append('direct')
+        if self.c3.bases_had_inconsistency:
+            inconsistent.append('bases')
+        return '+'.join(inconsistent) if inconsistent else 'no'
+
+    def __str__(self):
+        c3_report, legacy_report = self._generate_report()
+        assert len(c3_report) == len(legacy_report)
+
+        left_lines = [str(x) for x in legacy_report]
+        right_lines = [str(x) for x in c3_report]
+
+        # We have the same number of lines in the report; this is not
+        # necessarily the same as the number of items in either RO.
+        assert len(left_lines) == len(right_lines)
+
+        padding = ' ' * 2
+        max_left = max(len(x) for x in left_lines)
+        max_right = max(len(x) for x in right_lines)
+
+        left_title = 'Legacy RO (len={})'.format(len(self.legacy_ro))
+
+        right_title = 'C3 RO (len={}; inconsistent={})'.format(
+            len(self.c3_ro),
+            self._inconsistent_label,
+        )
+        lines = [
+            (padding + left_title.ljust(max_left) + padding + right_title.ljust(max_right)),
+            padding + '=' * (max_left + len(padding) + max_right)
+        ]
+        lines += [
+            padding + left.ljust(max_left) + padding + right
+            for left, right in zip(left_lines, right_lines)
+        ]
+
+        return '\n'.join(lines)
+
+
+# Set to `Interface` once it is defined. This is used to
+# avoid logging false positives about changed ROs.
+_ROOT = None
+
+def ro(C, strict=None, base_mros=None, log_changed_ro=None, use_legacy_ro=None):
+    """
+    ro(C) -> list
+
+    Compute the precedence list (mro) according to C3.
+
+    :return: A fresh `list` object.
+
+    .. versionchanged:: 5.0.0
+       Add the *strict*, *log_changed_ro* and *use_legacy_ro*
+       keyword arguments. These are provisional and likely to be
+       removed in the future. They are most useful for testing.
+    """
+    # The ``base_mros`` argument is for internal optimization and
+    # not documented.
+    resolver = C3.resolver(C, strict, base_mros)
+    mro = resolver.mro()
+
+    log_changed = log_changed_ro if log_changed_ro is not None else resolver.LOG_CHANGED_IRO
+    use_legacy = use_legacy_ro if use_legacy_ro is not None else resolver.USE_LEGACY_IRO
+
+    if log_changed or use_legacy:
+        legacy_ro = resolver.legacy_ro
+        assert isinstance(legacy_ro, list)
+        assert isinstance(mro, list)
+        changed = legacy_ro != mro
+        if changed:
+            # Did only Interface move? The fix for issue #8 made that
+            # somewhat common. It's almost certainly not a problem, though,
+            # so allow ignoring it.
+            legacy_without_root = [x for x in legacy_ro if x is not _ROOT]
+            mro_without_root = [x for x in mro if x is not _ROOT]
+            changed = legacy_without_root != mro_without_root
+
+        if changed:
+            comparison = _ROComparison(resolver, mro, legacy_ro)
+            _logger().warning(
+                "Object %r has different legacy and C3 MROs:\n%s",
+                C, comparison
+            )
+        if resolver.had_inconsistency and legacy_ro == mro:
+            comparison = _ROComparison(resolver, mro, legacy_ro)
+            _logger().warning(
+                "Object %r had inconsistent IRO and used the legacy RO:\n%s"
+                "\nInconsistency entered at:\n%s",
+                C, comparison, resolver.direct_inconsistency
+            )
+        if use_legacy:
+            return legacy_ro
+
+    return mro
+
+
+def is_consistent(C):
+    """
+    Check if the resolution order for *C*, as computed by :func:`ro`, is consistent
+    according to C3.
+    """
+    return not C3.resolver(C, False, None).had_inconsistency
diff --git a/contrib/python/zope.interface/py3/zope/interface/verify.py b/contrib/python/zope.interface/py3/zope/interface/verify.py
new file mode 100644
index 00000000000..0ab0b3f96ba
--- /dev/null
+++ b/contrib/python/zope.interface/py3/zope/interface/verify.py
@@ -0,0 +1,185 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Verify interface implementations
+"""
+import inspect
+import sys
+from types import FunctionType
+from types import MethodType
+
+from zope.interface.exceptions import BrokenImplementation
+from zope.interface.exceptions import BrokenMethodImplementation
+from zope.interface.exceptions import DoesNotImplement
+from zope.interface.exceptions import Invalid
+from zope.interface.exceptions import MultipleInvalid
+
+from zope.interface.interface import fromMethod, fromFunction, Method
+
+__all__ = [
+    'verifyObject',
+    'verifyClass',
+]
+
+# This will be monkey-patched when running under Zope 2, so leave this
+# here:
+MethodTypes = (MethodType, )
+
+
+def _verify(iface, candidate, tentative=False, vtype=None):
+    """
+    Verify that *candidate* might correctly provide *iface*.
+
+    This involves:
+
+    - Making sure the candidate claims that it provides the
+      interface using ``iface.providedBy`` (unless *tentative* is `True`,
+      in which case this step is skipped). This means that the candidate's class
+      declares that it `implements <zope.interface.implementer>` the interface,
+      or the candidate itself declares that it `provides <zope.interface.provider>`
+      the interface
+
+    - Making sure the candidate defines all the necessary methods
+
+    - Making sure the methods have the correct signature (to the
+      extent possible)
+
+    - Making sure the candidate defines all the necessary attributes
+
+    :return bool: Returns a true value if everything that could be
+       checked passed.
+    :raises zope.interface.Invalid: If any of the previous
+       conditions does not hold.
+
+    .. versionchanged:: 5.0
+        If multiple methods or attributes are invalid, all such errors
+        are collected and reported. Previously, only the first error was reported.
+        As a special case, if only one such error is present, it is raised
+        alone, like before.
+    """
+
+    if vtype == 'c':
+        tester = iface.implementedBy
+    else:
+        tester = iface.providedBy
+
+    excs = []
+    if not tentative and not tester(candidate):
+        excs.append(DoesNotImplement(iface, candidate))
+
+    for name, desc in iface.namesAndDescriptions(all=True):
+        try:
+            _verify_element(iface, name, desc, candidate, vtype)
+        except Invalid as e:
+            excs.append(e)
+
+    if excs:
+        if len(excs) == 1:
+            raise excs[0]
+        raise MultipleInvalid(iface, candidate, excs)
+
+    return True
+
+def _verify_element(iface, name, desc, candidate, vtype):
+    # Here the `desc` is either an `Attribute` or `Method` instance
+    try:
+        attr = getattr(candidate, name)
+    except AttributeError:
+        if (not isinstance(desc, Method)) and vtype == 'c':
+            # We can't verify non-methods on classes, since the
+            # class may provide attrs in it's __init__.
+            return
+        # TODO: This should use ``raise...from``
+        raise BrokenImplementation(iface, desc, candidate)
+
+    if not isinstance(desc, Method):
+        # If it's not a method, there's nothing else we can test
+        return
+
+    if inspect.ismethoddescriptor(attr) or inspect.isbuiltin(attr):
+        # The first case is what you get for things like ``dict.pop``
+        # on CPython (e.g., ``verifyClass(IFullMapping, dict))``). The
+        # second case is what you get for things like ``dict().pop`` on
+        # CPython (e.g., ``verifyObject(IFullMapping, dict()))``.
+        # In neither case can we get a signature, so there's nothing
+        # to verify. Even the inspect module gives up and raises
+        # ValueError: no signature found. The ``__text_signature__`` attribute
+        # isn't typically populated either.
+        #
+        # Note that on PyPy 2 or 3 (up through 7.3 at least), these are
+        # not true for things like ``dict.pop`` (but might be true for C extensions?)
+        return
+
+    if isinstance(attr, FunctionType):
+        if isinstance(candidate, type) and vtype == 'c':
+            # This is an "unbound method".
+            # Only unwrap this if we're verifying implementedBy;
+            # otherwise we can unwrap @staticmethod on classes that directly
+            # provide an interface.
+            meth = fromFunction(attr, iface, name=name, imlevel=1)
+        else:
+            # Nope, just a normal function
+            meth = fromFunction(attr, iface, name=name)
+    elif (isinstance(attr, MethodTypes)
+          and type(attr.__func__) is FunctionType):
+        meth = fromMethod(attr, iface, name)
+    elif isinstance(attr, property) and vtype == 'c':
+        # Without an instance we cannot be sure it's not a
+        # callable.
+        # TODO: This should probably check inspect.isdatadescriptor(),
+        # a more general form than ``property``
+        return
+
+    else:
+        if not callable(attr):
+            raise BrokenMethodImplementation(desc, "implementation is not a method",
+                                             attr, iface, candidate)
+        # sigh, it's callable, but we don't know how to introspect it, so
+        # we have to give it a pass.
+        return
+
+    # Make sure that the required and implemented method signatures are
+    # the same.
+    mess = _incompat(desc.getSignatureInfo(), meth.getSignatureInfo())
+    if mess:
+        raise BrokenMethodImplementation(desc, mess, attr, iface, candidate)
+
+
+
+def verifyClass(iface, candidate, tentative=False):
+    """
+    Verify that the *candidate* might correctly provide *iface*.
+    """
+    return _verify(iface, candidate, tentative, vtype='c')
+
+def verifyObject(iface, candidate, tentative=False):
+    return _verify(iface, candidate, tentative, vtype='o')
+
+verifyObject.__doc__ = _verify.__doc__
+
+_MSG_TOO_MANY = 'implementation requires too many arguments'
+
+def _incompat(required, implemented):
+    #if (required['positional'] !=
+    #    implemented['positional'][:len(required['positional'])]
+    #    and implemented['kwargs'] is None):
+    #    return 'imlementation has different argument names'
+    if len(implemented['required']) > len(required['required']):
+        return _MSG_TOO_MANY
+    if ((len(implemented['positional']) < len(required['positional']))
+        and not implemented['varargs']):
+        return "implementation doesn't allow enough arguments"
+    if required['kwargs'] and not implemented['kwargs']:
+        return "implementation doesn't support keyword arguments"
+    if required['varargs'] and not implemented['varargs']:
+        return "implementation doesn't support variable arguments"
diff --git a/contrib/python/zope.interface/ya.make b/contrib/python/zope.interface/ya.make
new file mode 100644
index 00000000000..1987d80896a
--- /dev/null
+++ b/contrib/python/zope.interface/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+    PEERDIR(contrib/python/zope.interface/py2)
+ELSE()
+    PEERDIR(contrib/python/zope.interface/py3)
+ENDIF()
+
+NO_LINT()
+
+END()
+
+RECURSE(
+    py2
+    py3
+)
-- 
cgit v1.3